From 91d319e849ca912e1ff77046cb277985db5844d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 14:06:25 -0400 Subject: [PATCH 0001/8313] moved from my doc repo --- git-annex.mdwn | 165 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 git-annex.mdwn diff --git a/git-annex.mdwn b/git-annex.mdwn new file mode 100644 index 0000000000..bc3550398c --- /dev/null +++ b/git-annex.mdwn @@ -0,0 +1,165 @@ +git-annex allows managing files with git, without checking the file +contents into git. This is useful when dealing with files larger than git +can currently easily handle, whether due to limitations in memory, +checksumming time, or disk space (only one copy need be stored of an +annexed file). + +Even without file content tracking, being able to manage file metadata with +git, move files around and delete files with versioned directory trees, and use +branches and distributed clone, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly versioned +files, which is convenient for maintaining code, Makefiles, etc that are +associated with annexed files but that benefit from full revision control. + +Enough broad picture, here's how it actually looks: + +* `git annex --add $file` moves the file into `.git/annex/`, and replaces + it with a symlink pointing at the annexed file, and then calls `git add` + to version the *symlink*. (If the file has already been annexed, it does + nothing.) +* You can move the symlink around, copy it, delete it, etc, and commit changes + as desired using git. Reading the symlink will always get you the annexed + file content, or the link may be broken if the content is not currently + available. +* If you use normal git push/pull commands, the annexed file contents + won't be sent, but the symlinks will be. So different clones of a repository + can have different sets of annexed files available. +* `git annex --push $repository` pushes *all* annexed files to the specified + repository. +* `git annex --pull $repository` pulls *all* annexed files from the specified + repository. +* `git annex --want $file` indicates that you want access to a file's + content, without immediatly transferring it. +* `git annex --get $file` is used to transfer a specified file, and/or + files previously indicated with --want. If a configured repository has it, + or it is available from other key/value storage, it will be immediatly + downloaded. +* `git annex --drop $file` indicates that you no longer want the file's + content to be available in this repository. +* `git annex $file` is a shorthand for either --add or --get. If the file + is already known, it does --get, otherwise it does --add. + +## copies + +git-annex can be configured to try to keep N copies of a file's content +available across all repositories. By default, N is 1 (configured by +annex.numcopies). + +`git annex --drop` attempts to communicate with all other configured +repositories, to check that N copies of the file exist. If enough +repositories cannot be contacted, it will retain the file content. +You can later use `git annex --drop --retry` to retry pending drops. +Or you can use `git annex --drop --force $file` to force dropping of +file content. + +For example, consider three repositories: Server, Laptop, and USB. Both Server +and USB have a copy of a file, and N=1. If on Laptop, you `git annex --get +$file`, this will transfer it from either Server or USB (depending on which +is available), and there are now 3 copies of the file. + +Suppose you want to free up space on laptop again, and you --drop the file +there. If USB is connected, or Server can be contacted, git-annex can check +that it still has a copy of the file, and the content is removed from +Laptop. But if USB is currently disconnected, and Server also cannot be +contacted, it can't check that and will retain the file content. + +With N=2, in order to drop the file content from Laptop, it would need access +to both USB and Server. + +Note that different repositories can be configured with different values of +N. So just because Laptop has N=2, this does not prevent the number of +copies falling to 1, when USB and Server have N=1, and of they have the +only copies of a file. + +## the .git-annex directory + +The `.git-annex` directory at the top of the repository, is used to store +git-annex information that should be propigated between repositories. + +Data is stored here in files that are arranged to avoid conflicts in most +cases. A conflict could occur if a file with the same name but different +content was added to multiple clones. + +## key/value storage + +git-annex uses a key/value abstraction layer to allow files contents to be +stored in different ways. In theory, any key/value storage system could be +used to store the file contents, and git-annex would then retrieve them +as needed and put them in `.git/annex/`. + +When a file is annexed, a key is generated from its content and/or metadata. +This key can later be used to retrieve the file's content (its value). This +key generation must be stable for a given file content, name, and size. + +The mapping from filename to its key is stored in the .git-annex directory, +in a file named `$filename.$backend` + +Multiple pluggable backends are supported, and more than one can be used +to store different files' contents in a given repository. + +* `file` -- This backend stores the file's content in + `.git/annex/`, and assumes that any file with the same basename + has the same content. So with this backend, files can be moved around, + but should never be added to or changed. This is the default, and + the least expensive backend. +* `sha1sum` -- This backend stores the file's content in + `.git/annex/`, with a name based on its sha1 checksum. This backend allows + modifications of files to be tracked. Its need to generate checksums + can make it slow for large files. +* `url` -- This backend downloads the file's content from an external URL. + +## location tracking + +git-annex keeps track of on which repository it last saw a file's content. +This can be useful when using it for archiving with offline storage. When +you indicate you --want a file, git-annex will tell you which repositories +have the file's content. + +Location tracking information is stored in `.git-annex/$filename.log`. +Repositories record their name and the date when they --get or --drop +a file's content. (Git is configured to use a union merge for this file, +so the lines may be in arbitrary order, but it will never conflict.) + +## configuration + +* `annex.numcopies` -- number of copies of files to keep +* `annex.backend` -- name of the default key/value backend to use to + store new files +* `annex.name` -- allows specifying a unique name for this repository. + If not specified, the name is derived from its directory's location and + the hostname. When a repository is on removable media it is useful to give + it a more stable name. Typically the name of a repository is the same + name configured as a git remote to allow pulling from that repository. +* `remote..annex-cost` -- When determining which repository to + transfer annexed files from or to, ones with lower costs are preferred. + The default cost is 50. Note that other factors may be configured + when pushing files to repositories, in particular, whether the repository + is on a filesystem with sufficient free space. + +## issues + +### symlinks + +If the symlink to annexed content is relative, moving it to a subdir will +break it. But it it's absolute, moving the git repo (or mounting its drive +elsewhere) will break it. Either: + +* Use relative links and need `git annex mv` to move (or post-commit + hook that caches moves and updates links). +* Use absolute links and need `git annex fixlinks` when location changes; + note that would also mean that git would see the symlink targets changed + and want to commit the change. + +### free space determination + +Need a way to tell how much free space is available on the disk containing +a given repository. The repository may be remote, so ssh may need to be +used. + +Similarly, need a way to tell the size of a file before downloading it from +remote, to check local disk space. + +### auto-drop files on rm + +When git-rm removed a file, it should get dropped too. Of course, it may +not be dropped right away, depending on number of copies available. From a667d99cd1aa90691ded4fc110a1e11e965fc3a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 19:22:40 -0400 Subject: [PATCH 0002/8313] first module --- LocationLog.hs | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 LocationLog.hs diff --git a/LocationLog.hs b/LocationLog.hs new file mode 100644 index 0000000000..c756a17b05 --- /dev/null +++ b/LocationLog.hs @@ -0,0 +1,81 @@ +{- git-annex location log + - + - git-annex keeps track of on which repository it last saw a file's content. + - This can be useful when using it for archiving with offline storage. + - When you indicate you --want a file, git-annex will tell you which + - repositories have the file's content. + - + - Location tracking information is stored in `.git-annex/$filename.log`. + - Repositories record their name and the date when they --get or --drop + - a file's content. (Git is configured to use a union merge for this file, + - so the lines may be in arbitrary order, but it will never conflict.) + - + - A line of the log will look like: "date reponame filename" + - + -} + +module LocationLog where + +import Data.DateTime +import System.IO +import System.Posix.IO + +data LogLine = LogLine { + date :: DateTime, + repo :: String, + file :: String +} deriving (Eq) + +-- a special value representing a log file line that could not be parsed +unparsable = (LogLine (fromSeconds 0) "" "") + +instance Show LogLine where + show (LogLine date repo file) = unwords + [(show (toSeconds date)), repo, file] + +instance Read LogLine where + -- this parser is robust in that even unparsable log lines are + -- read without an exception being thrown + readsPrec _ string = if (length w >= 3) + then [((LogLine time repo file), "")] + else [(unparsable, "")] + where + time = fromSeconds $ read $ w !! 0 + repo = w !! 1 + file = unwords $ rest w + w = words string + rest (_:_:l) = l + +{- Reads a log file -} +readLog :: String -> IO [LogLine] +readLog file = do + h <- openLocked file ReadMode + s <- hGetContents h + -- hClose handle' -- TODO disabled due to lazy IO issue + -- filter out any unparsable lines + return $ filter ( /= unparsable ) $ map read $ lines s + +{- Adds a LogLine to a log file -} +writeLog :: String -> LogLine -> IO () +writeLog file line = do + h <- openLocked file AppendMode + hPutStrLn h $ show line + hClose h + +{- Let's just say that Haskell makes reading/writing a file with + - file locking excessively difficult. -} +openLocked file mode = do + handle <- openFile file mode + lockfd <- handleToFd handle -- closes handle + waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) + handle' <- fdToHandle lockfd + return handle' + where + lockType ReadMode = ReadLock + lockType _ = WriteLock + +{- Generates a new log line with the current date. -} +logNow :: String -> String -> IO LogLine +logNow repo file = do + now <- getCurrentTime + return $ LogLine now repo file From 6b54817f2688cffc8751b3b1552dca0a34744e61 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 21:06:46 -0400 Subject: [PATCH 0003/8313] second module! --- GitRepo.hs | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 GitRepo.hs diff --git a/GitRepo.hs b/GitRepo.hs new file mode 100644 index 0000000000..fece797858 --- /dev/null +++ b/GitRepo.hs @@ -0,0 +1,57 @@ +{- git repository handling -} + +module GitRepo where + +import Directory +import System.Directory +import Data.String.Utils + +{- Returns the path to the current repository's .git directory. + - (For a bare repository, that is the root of the repository.) -} +gitDir :: IO String +gitDir = do + repo <- repoTop + bare <- isBareRepo repo + if (bare) + then return repo + else return $ repo ++ "/.git" + +{- Finds the top of the current git repository, which may be in a parent + - directory. -} +repoTop :: IO String +repoTop = do + dir <- getCurrentDirectory + top <- seekUp dir isRepoTop + case top of + (Just dir) -> return dir + Nothing -> error "Not in a git repository." + +seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) +seekUp dir want = do + ok <- want dir + if ok + then return (Just dir) + else case (parentDir dir) of + (Just d) -> seekUp d want + Nothing -> return Nothing + +parentDir :: String -> Maybe String +parentDir dir = + if length dirs > 0 + then Just ("/" ++ (join "/" $ take ((length dirs) - 1) dirs)) + else Nothing + where + dirs = filter (\x -> length x > 0) $ split "/" dir + +isRepoTop dir = do + r <- isGitRepo dir + b <- isBareRepo dir + return (r || b) + +isGitRepo dir = gitSignature dir ".git" ".git/config" +isBareRepo dir = gitSignature dir "objects" "config" + +gitSignature dir subdir file = do + s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) + f <- (doesFileExist (dir ++ "/" ++ file)) + return (s && f) From c920505a52ab3c42b7892b7f7a1c5244c39e916f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:09:10 -0400 Subject: [PATCH 0004/8313] add gitRelative function --- GitRepo.hs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index fece797858..f1372bf3a1 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -4,8 +4,25 @@ module GitRepo where import Directory import System.Directory +import System.Path import Data.String.Utils +{- Given a relative or absolute filename, calculates the name to use + - relative to a git repository directory (which must be absolute). + - This is the same form displayed and used by git. -} +gitRelative :: String -> String -> String +gitRelative file repo = drop (length absrepo) absfile + where + -- normalize both repo and file, so that repo + -- will be substring of file + absrepo = case (absNormPath "/" repo) of + Just f -> f ++ "/" + Nothing -> error $ "bad repo" ++ repo + absfile = case (secureAbsNormPath absrepo file) of + Just f -> f + Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo + + {- Returns the path to the current repository's .git directory. - (For a bare repository, that is the root of the repository.) -} gitDir :: IO String @@ -20,8 +37,8 @@ gitDir = do - directory. -} repoTop :: IO String repoTop = do - dir <- getCurrentDirectory - top <- seekUp dir isRepoTop + cwd <- getCurrentDirectory + top <- seekUp cwd isRepoTop case top of (Just dir) -> return dir Nothing -> error "Not in a git repository." From dcfb24e5b5764c8a7bde0a1410022a903ba3c99b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:14:13 -0400 Subject: [PATCH 0005/8313] add logFile --- GitRepo.hs | 2 +- LocationLog.hs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index f1372bf3a1..01e6746aef 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -11,7 +11,7 @@ import Data.String.Utils - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} gitRelative :: String -> String -> String -gitRelative file repo = drop (length absrepo) absfile +gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file diff --git a/LocationLog.hs b/LocationLog.hs index c756a17b05..ab109460ad 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -19,6 +19,7 @@ module LocationLog where import Data.DateTime import System.IO import System.Posix.IO +import GitRepo data LogLine = LogLine { date :: DateTime, @@ -79,3 +80,10 @@ logNow :: String -> String -> IO LogLine logNow repo file = do now <- getCurrentTime return $ LogLine now repo file + +{- Returns the filename of the log file for a given annexed file. -} +logFile :: String -> IO String +logFile annexedFile = do + repo <- repoTop + return $ repo ++ "/.git-annex/" ++ + (gitRelative repo annexedFile) ++ ".log" From 9ae522bb7689842e1d0251d486c22d26bb6461da Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:29:16 -0400 Subject: [PATCH 0006/8313] add status field to log --- LocationLog.hs | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index ab109460ad..ff357aaecd 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -10,7 +10,8 @@ - a file's content. (Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict.) - - - A line of the log will look like: "date reponame filename" + - A line of the log will look like: "date N reponame filename" + - Where N=1 when the repo has the file, and 0 otherwise. - -} @@ -21,31 +22,44 @@ import System.IO import System.Posix.IO import GitRepo +data LogStatus = FilePresent | FileMissing | Undefined + deriving (Eq) + +instance Show LogStatus where + show FilePresent = "1" + show FileMissing = "0" + show Undefined = "undefined" + +instance Read LogStatus where + readsPrec _ "1" = [(FilePresent, "")] + readsPrec _ "0" = [(FileMissing, "")] + readsPrec _ _ = [(Undefined, "")] + data LogLine = LogLine { date :: DateTime, + status :: LogStatus, repo :: String, file :: String } deriving (Eq) --- a special value representing a log file line that could not be parsed -unparsable = (LogLine (fromSeconds 0) "" "") - instance Show LogLine where - show (LogLine date repo file) = unwords - [(show (toSeconds date)), repo, file] + show (LogLine date status repo file) = unwords + [(show (toSeconds date)), (show status), repo, file] instance Read LogLine where - -- this parser is robust in that even unparsable log lines are - -- read without an exception being thrown + -- This parser is robust in that even unparsable log lines are + -- read without an exception being thrown. + -- Such lines have a status of Undefined. readsPrec _ string = if (length w >= 3) - then [((LogLine time repo file), "")] - else [(unparsable, "")] + then [((LogLine date status repo file), "")] + else [((LogLine (fromSeconds 0) Undefined "" ""), "")] where - time = fromSeconds $ read $ w !! 0 - repo = w !! 1 + date = fromSeconds $ read $ w !! 0 + status = read $ w !! 1 + repo = w !! 2 file = unwords $ rest w w = words string - rest (_:_:l) = l + rest (_:_:_:l) = l {- Reads a log file -} readLog :: String -> IO [LogLine] @@ -54,7 +68,7 @@ readLog file = do s <- hGetContents h -- hClose handle' -- TODO disabled due to lazy IO issue -- filter out any unparsable lines - return $ filter ( /= unparsable ) $ map read $ lines s + return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s {- Adds a LogLine to a log file -} writeLog :: String -> LogLine -> IO () @@ -76,10 +90,10 @@ openLocked file mode = do lockType _ = WriteLock {- Generates a new log line with the current date. -} -logNow :: String -> String -> IO LogLine -logNow repo file = do +logNow :: LogStatus -> String -> String -> IO LogLine +logNow status repo file = do now <- getCurrentTime - return $ LogLine now repo file + return $ LogLine now status repo file {- Returns the filename of the log file for a given annexed file. -} logFile :: String -> IO String From 011118dbdff84458a5f9eea05547d79fbf7e88ac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 22:46:35 -0400 Subject: [PATCH 0007/8313] adding file presence calculation code --- LocationLog.hs | 55 +++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index ff357aaecd..911e4765b4 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -5,12 +5,12 @@ - When you indicate you --want a file, git-annex will tell you which - repositories have the file's content. - - - Location tracking information is stored in `.git-annex/$filename.log`. + - Location tracking information is stored in `.git-annex/filename.log`. - Repositories record their name and the date when they --get or --drop - a file's content. (Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict.) - - - A line of the log will look like: "date N reponame filename" + - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. - -} @@ -19,8 +19,8 @@ module LocationLog where import Data.DateTime import System.IO -import System.Posix.IO import GitRepo +import Utility data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) @@ -38,28 +38,26 @@ instance Read LogStatus where data LogLine = LogLine { date :: DateTime, status :: LogStatus, - repo :: String, - file :: String + repo :: String } deriving (Eq) instance Show LogLine where - show (LogLine date status repo file) = unwords - [(show (toSeconds date)), (show status), repo, file] + show (LogLine date status repo) = unwords + [(show (toSeconds date)), (show status), repo] instance Read LogLine where -- This parser is robust in that even unparsable log lines are -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = if (length w >= 3) - then [((LogLine date status repo file), "")] - else [((LogLine (fromSeconds 0) Undefined "" ""), "")] + then [((LogLine date status repo), "")] + else [((LogLine (fromSeconds 0) Undefined ""), "")] where date = fromSeconds $ read $ w !! 0 status = read $ w !! 1 - repo = w !! 2 - file = unwords $ rest w + repo = unwords $ rest w w = words string - rest (_:_:_:l) = l + rest (_:_:l) = l {- Reads a log file -} readLog :: String -> IO [LogLine] @@ -77,23 +75,11 @@ writeLog file line = do hPutStrLn h $ show line hClose h -{- Let's just say that Haskell makes reading/writing a file with - - file locking excessively difficult. -} -openLocked file mode = do - handle <- openFile file mode - lockfd <- handleToFd handle -- closes handle - waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) - handle' <- fdToHandle lockfd - return handle' - where - lockType ReadMode = ReadLock - lockType _ = WriteLock - -{- Generates a new log line with the current date. -} -logNow :: LogStatus -> String -> String -> IO LogLine -logNow status repo file = do +{- Generates a new LogLine with the current date. -} +logNow :: LogStatus -> String -> IO LogLine +logNow status repo = do now <- getCurrentTime - return $ LogLine now status repo file + return $ LogLine now status repo {- Returns the filename of the log file for a given annexed file. -} logFile :: String -> IO String @@ -101,3 +87,16 @@ logFile annexedFile = do repo <- repoTop return $ repo ++ "/.git-annex/" ++ (gitRelative repo annexedFile) ++ ".log" + +{- Returns a list of repositories that, according to the log, have + - the content of a file -} +fileLocations :: String -> IO [String] +fileLocations file = do + log <- logFile file + lines <- readLog log + return $ map repo (filterPresent lines) + +{- Filters the list of LogLines to find repositories where the file + - is (or should still be) present. -} +filterPresent :: [LogLine] -> [LogLine] +filterPresent lines = From 381e6f84e5f4ddc64ed86f08064ebaf2313b18db Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Oct 2010 23:35:05 -0400 Subject: [PATCH 0008/8313] robustness --- GitRepo.hs | 13 +++---------- LocationLog.hs | 26 +++++++++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 01e6746aef..2e7fff22ea 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -6,6 +6,7 @@ import Directory import System.Directory import System.Path import Data.String.Utils +import Utility {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). @@ -49,16 +50,8 @@ seekUp dir want = do if ok then return (Just dir) else case (parentDir dir) of - (Just d) -> seekUp d want - Nothing -> return Nothing - -parentDir :: String -> Maybe String -parentDir dir = - if length dirs > 0 - then Just ("/" ++ (join "/" $ take ((length dirs) - 1) dirs)) - else Nothing - where - dirs = filter (\x -> length x > 0) $ split "/" dir + "" -> return Nothing + d -> seekUp d want isRepoTop dir = do r <- isGitRepo dir diff --git a/LocationLog.hs b/LocationLog.hs index 911e4765b4..b6c85113d8 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -19,6 +19,7 @@ module LocationLog where import Data.DateTime import System.IO +import System.Directory import GitRepo import Utility @@ -55,22 +56,29 @@ instance Read LogLine where where date = fromSeconds $ read $ w !! 0 status = read $ w !! 1 - repo = unwords $ rest w + repo = unwords $ drop 2 w w = words string - rest (_:_:l) = l -{- Reads a log file -} +{- Reads a log file. + - Note that the LogLines returned may be in any order. -} readLog :: String -> IO [LogLine] readLog file = do - h <- openLocked file ReadMode - s <- hGetContents h - -- hClose handle' -- TODO disabled due to lazy IO issue - -- filter out any unparsable lines - return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s + exists <- doesFileExist file + if exists + then do + h <- openLocked file ReadMode + s <- hGetContents h + -- hClose handle' -- TODO disabled due to lazy IO issue + -- filter out any unparsable lines + return $ filter (\l -> (status l) /= Undefined ) + $ map read $ lines s + else do + return [] {- Adds a LogLine to a log file -} writeLog :: String -> LogLine -> IO () writeLog file line = do + createDirectoryIfMissing True (parentDir file) h <- openLocked file AppendMode hPutStrLn h $ show line hClose h @@ -99,4 +107,4 @@ fileLocations file = do {- Filters the list of LogLines to find repositories where the file - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = +filterPresent lines = error "unimplimented" -- TODO From e64d1becf429489f8c6ded230e6e17b63a89c483 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 00:02:07 -0400 Subject: [PATCH 0009/8313] robustness fix avoid crash if the seconds field is not numeric --- LocationLog.hs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index b6c85113d8..db1fac1444 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -20,6 +20,7 @@ module LocationLog where import Data.DateTime import System.IO import System.Directory +import Data.Char import GitRepo import Utility @@ -50,14 +51,15 @@ instance Read LogLine where -- This parser is robust in that even unparsable log lines are -- read without an exception being thrown. -- Such lines have a status of Undefined. - readsPrec _ string = if (length w >= 3) - then [((LogLine date status repo), "")] - else [((LogLine (fromSeconds 0) Undefined ""), "")] + readsPrec _ string = + if (length w >= 3 && all isDigit date) + then [((LogLine (fromSeconds $ read date) status repo), "")] + else [((LogLine (fromSeconds 0) Undefined ""), "")] where - date = fromSeconds $ read $ w !! 0 + w = words string + date = w !! 0 status = read $ w !! 1 repo = unwords $ drop 2 w - w = words string {- Reads a log file. - Note that the LogLines returned may be in any order. -} From d0e82d0b9218a9ff3a693e066c4320c08d4d1c47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 00:18:10 -0400 Subject: [PATCH 0010/8313] add --- Utility.hs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 Utility.hs diff --git a/Utility.hs b/Utility.hs new file mode 100644 index 0000000000..05b06dea79 --- /dev/null +++ b/Utility.hs @@ -0,0 +1,29 @@ +{- git-annex utility functions + -} + +module Utility where + +import System.IO +import System.Posix.IO +import Data.String.Utils + +{- Let's just say that Haskell makes reading/writing a file with + - file locking excessively difficult. -} +openLocked file mode = do + handle <- openFile file mode + lockfd <- handleToFd handle -- closes handle + waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) + handle' <- fdToHandle lockfd + return handle' + where + lockType ReadMode = ReadLock + lockType _ = WriteLock + +{- Returns the parent directory of a path. Parent of / is "" -} +parentDir :: String -> String +parentDir dir = + if length dirs > 0 + then "/" ++ (join "/" $ take ((length dirs) - 1) dirs) + else "" + where + dirs = filter (\x -> length x > 0) $ split "/" dir From c67521741abd9a49ebf43d6c649fe0356fa68fb3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 00:18:16 -0400 Subject: [PATCH 0011/8313] add --- .gitignore | 5 +++++ Makefile | 7 +++++++ demo.log | 8 ++++++++ git-annex.hs | 8 ++++++++ 4 files changed, 28 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 demo.log create mode 100644 git-annex.hs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..7dd8869b16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.o +*.hi +*.ho +*.a +git-annex diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..8b7c9d3a0e --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +git-annex: + ghc --make git-annex + +clean: + rm -f git-annex *.o *.hi *.ho *.a + +.PHONY: git-annex diff --git a/demo.log b/demo.log new file mode 100644 index 0000000000..251a84c520 --- /dev/null +++ b/demo.log @@ -0,0 +1,8 @@ +1286654242 1 repo +1286652724 0 foo +a a a +a 1 a +-1 a a +1286652724 1 foo +1286656282 1 foo +1286656282 0 repo diff --git a/git-annex.hs b/git-annex.hs new file mode 100644 index 0000000000..a57e9e2dbc --- /dev/null +++ b/git-annex.hs @@ -0,0 +1,8 @@ +{- git-annex main program + - -} + +import LocationLog + +main = do + l <- readLog "demo.log" + putStrLn "hi" From 60c672e444decf59c20beb70b89f030ad9d62b3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:22:35 -0400 Subject: [PATCH 0012/8313] strictness and handle closing --- LocationLog.hs | 7 +++---- Utility.hs | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index db1fac1444..c921a20052 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -12,7 +12,6 @@ - - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. - - -} module LocationLog where @@ -69,8 +68,8 @@ readLog file = do if exists then do h <- openLocked file ReadMode - s <- hGetContents h - -- hClose handle' -- TODO disabled due to lazy IO issue + s <- hGetContentsStrict h + hClose h -- filter out any unparsable lines return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s @@ -95,7 +94,7 @@ logNow status repo = do logFile :: String -> IO String logFile annexedFile = do repo <- repoTop - return $ repo ++ "/.git-annex/" ++ + return $ (gitStateDir repo) ++ (gitRelative repo annexedFile) ++ ".log" {- Returns a list of repositories that, according to the log, have diff --git a/Utility.hs b/Utility.hs index 05b06dea79..ab9ce04f36 100644 --- a/Utility.hs +++ b/Utility.hs @@ -19,6 +19,10 @@ openLocked file mode = do lockType ReadMode = ReadLock lockType _ = WriteLock +{- A version of hgetContents that is not lazy. Ensures file is + - all read before it gets closed. -} +hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s + {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = From 852ead470756744cd6663ee2d537f3d281f1e7c8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:22:47 -0400 Subject: [PATCH 0013/8313] add gitPrep to handle .gitattributes --- GitRepo.hs | 27 ++++++++++++++++++++++++++- git-annex.hs | 4 ++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2e7fff22ea..21c683bd27 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,6 +8,12 @@ import System.Path import Data.String.Utils import Utility +{- Long-term state is stored in files inside the .git-annex directory + - in the git repository. -} +stateLoc = ".git-annex" +gitStateDir :: String -> String +gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" + {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} @@ -23,9 +29,28 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo +{- Sets up the current git repo for git-annex. May be called repeatedly. -} +gitPrep :: IO () +gitPrep = do + repo <- repoTop + bare <- isBareRepo repo + -- configure git to use union merge driver on state files + let attributes = repo ++ "/.gitattributes" + let attrLine = stateLoc ++ "/* merge=union" + exists <- doesFileExist attributes + if (not bare) + then if (not exists) + then writeFile attributes $ attrLine ++ "\n" + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then appendFile attributes $ attrLine ++ "\n" + else return () + else return () {- Returns the path to the current repository's .git directory. - - (For a bare repository, that is the root of the repository.) -} + - (For a bare repository, that is the root of the repository.) + - TODO: support GIT_DIR -} gitDir :: IO String gitDir = do repo <- repoTop diff --git a/git-annex.hs b/git-annex.hs index a57e9e2dbc..e8032a1325 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -2,7 +2,7 @@ - -} import LocationLog +import GitRepo main = do - l <- readLog "demo.log" - putStrLn "hi" + gitPrep From 058cef945023843219e09d4cec80bb7e137b9876 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:27:14 -0400 Subject: [PATCH 0014/8313] handle bare repo right for gitattributes also simplere code! --- GitRepo.hs | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 21c683bd27..2e2c1d52b8 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -32,21 +32,26 @@ gitRelative repo file = drop (length absrepo) absfile {- Sets up the current git repo for git-annex. May be called repeatedly. -} gitPrep :: IO () gitPrep = do + -- configure git to use union merge driver on state files + let attrLine = stateLoc ++ "/* merge=union" + attributes <- gitAttributes + exists <- doesFileExist attributes + if (not exists) + then writeFile attributes $ attrLine ++ "\n" + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then appendFile attributes $ attrLine ++ "\n" + else return () + +{- Returns the path to the current repository's gitattributes file. -} +gitAttributes :: IO String +gitAttributes = do repo <- repoTop bare <- isBareRepo repo - -- configure git to use union merge driver on state files - let attributes = repo ++ "/.gitattributes" - let attrLine = stateLoc ++ "/* merge=union" - exists <- doesFileExist attributes - if (not bare) - then if (not exists) - then writeFile attributes $ attrLine ++ "\n" - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then appendFile attributes $ attrLine ++ "\n" - else return () - else return () + if (bare) + then return $ repo ++ "/info/.gitattributes" + else return $ repo ++ "/.gitattributes" {- Returns the path to the current repository's .git directory. - (For a bare repository, that is the root of the repository.) @@ -56,7 +61,7 @@ gitDir = do repo <- repoTop bare <- isBareRepo repo if (bare) - then return repo + then return $ repo else return $ repo ++ "/.git" {- Finds the top of the current git repository, which may be in a parent From 11ad93f023fa5e867b5b7bd47f45393caceb401a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 02:29:58 -0400 Subject: [PATCH 0015/8313] reorg --- GitRepo.hs | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2e2c1d52b8..8737d82512 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -14,6 +14,26 @@ stateLoc = ".git-annex" gitStateDir :: String -> String gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" +{- Path to the current repository's gitattributes file. -} +gitAttributes :: IO String +gitAttributes = do + repo <- repoTop + bare <- isBareRepo repo + if (bare) + then return $ repo ++ "/info/.gitattributes" + else return $ repo ++ "/.gitattributes" + +{- Path to the current repository's .git directory. + - (For a bare repository, that is the root of the repository.) + - TODO: support GIT_DIR -} +gitDir :: IO String +gitDir = do + repo <- repoTop + bare <- isBareRepo repo + if (bare) + then return $ repo + else return $ repo ++ "/.git" + {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} @@ -41,29 +61,11 @@ gitPrep = do else do content <- readFile attributes if (all (/= attrLine) (lines content)) - then appendFile attributes $ attrLine ++ "\n" + then do + appendFile attributes $ attrLine ++ "\n" + -- TODO check attributes file into git? else return () -{- Returns the path to the current repository's gitattributes file. -} -gitAttributes :: IO String -gitAttributes = do - repo <- repoTop - bare <- isBareRepo repo - if (bare) - then return $ repo ++ "/info/.gitattributes" - else return $ repo ++ "/.gitattributes" - -{- Returns the path to the current repository's .git directory. - - (For a bare repository, that is the root of the repository.) - - TODO: support GIT_DIR -} -gitDir :: IO String -gitDir = do - repo <- repoTop - bare <- isBareRepo repo - if (bare) - then return $ repo - else return $ repo ++ "/.git" - {- Finds the top of the current git repository, which may be in a parent - directory. -} repoTop :: IO String From f98fa53d7f6d851b8a1ae804c02780769c98e07c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 11:08:40 -0400 Subject: [PATCH 0016/8313] fixed close after locking --- LocationLog.hs | 15 +++++++++------ Utility.hs | 7 +++++-- git-annex.hs | 2 ++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index c921a20052..1523901df7 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -12,6 +12,11 @@ - - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. + - + - TOOD: compact logs, by storing only current presence infomation when + - writing them. + - + - TODO: use ByteString -} module LocationLog where @@ -67,9 +72,8 @@ readLog file = do exists <- doesFileExist file if exists then do - h <- openLocked file ReadMode - s <- hGetContentsStrict h - hClose h + s <- withFileLocked file ReadMode $ \h -> + hGetContentsStrict h -- filter out any unparsable lines return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s @@ -80,9 +84,8 @@ readLog file = do writeLog :: String -> LogLine -> IO () writeLog file line = do createDirectoryIfMissing True (parentDir file) - h <- openLocked file AppendMode - hPutStrLn h $ show line - hClose h + withFileLocked file AppendMode $ \h -> + hPutStrLn h $ show line {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine diff --git a/Utility.hs b/Utility.hs index ab9ce04f36..d1eb247d3c 100644 --- a/Utility.hs +++ b/Utility.hs @@ -9,12 +9,15 @@ import Data.String.Utils {- Let's just say that Haskell makes reading/writing a file with - file locking excessively difficult. -} -openLocked file mode = do +withFileLocked file mode action = do + -- TODO: find a way to use bracket here handle <- openFile file mode lockfd <- handleToFd handle -- closes handle waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) handle' <- fdToHandle lockfd - return handle' + ret <- action handle' + hClose handle' + return ret where lockType ReadMode = ReadLock lockType _ = WriteLock diff --git a/git-annex.hs b/git-annex.hs index e8032a1325..66b9491bde 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,3 +6,5 @@ import GitRepo main = do gitPrep + l <- readLog "demo.log" + writeLog "demo2.log" $ l !! 0 From 7ad4a0bb7d4beb469f0aba017fae1ac48060e862 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 12:31:14 -0400 Subject: [PATCH 0017/8313] log compaction --- LocationLog.hs | 44 +++++++++++++++++++++++++++++++++++--------- demo.log | 7 +++++-- git-annex.hs | 2 +- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 1523901df7..028ceed5fb 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -7,21 +7,19 @@ - - Location tracking information is stored in `.git-annex/filename.log`. - Repositories record their name and the date when they --get or --drop - - a file's content. (Git is configured to use a union merge for this file, - - so the lines may be in arbitrary order, but it will never conflict.) + - a file's content. - - A line of the log will look like: "date N reponame" - Where N=1 when the repo has the file, and 0 otherwise. - - - - TOOD: compact logs, by storing only current presence infomation when - - writing them. - - - - TODO: use ByteString + - + - Git is configured to use a union merge for this file, + - so the lines may be in arbitrary order, but it will never conflict. -} module LocationLog where import Data.DateTime +import qualified Data.Map as Map import System.IO import System.Directory import Data.Char @@ -81,12 +79,19 @@ readLog file = do return [] {- Adds a LogLine to a log file -} -writeLog :: String -> LogLine -> IO () -writeLog file line = do +appendLog :: String -> LogLine -> IO () +appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line +{- Writes a set of lines to a log file -} +writeLog :: String -> [LogLine] -> IO () +writeLog file lines = do + createDirectoryIfMissing True (parentDir file) + withFileLocked file WriteMode $ \h -> + hPutStr h $ unlines $ map show lines + {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine logNow status repo = do @@ -112,3 +117,24 @@ fileLocations file = do - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] filterPresent lines = error "unimplimented" -- TODO + +{- Compacts a set of logs, returning a subset that contains the current + - status. -} +compactLog :: [LogLine] -> [LogLine] +compactLog lines = compactLog' Map.empty lines +compactLog' map [] = Map.elems map +compactLog' map (l:ls) = compactLog' (mapLog map l) ls + +{- Inserts a log into a map of logs, if the log has better (ie, newer) + - information about a repo than the other logs in the map -} +mapLog map log = + if (better) + then Map.insert (repo log) log map + else map + where + better = case (Map.lookup (repo log) map) of + -- <= used because two log entries could + -- have the same timestamp; if so the one that + -- is seen last should win. + Just l -> (date l <= date log) + Nothing -> True diff --git a/demo.log b/demo.log index 251a84c520..7a42630566 100644 --- a/demo.log +++ b/demo.log @@ -1,8 +1,11 @@ 1286654242 1 repo 1286652724 0 foo +1286656282 1 foo +1286656282 0 repo +1286656281 0 foo +# some garbage, should be ignored a a a + a 1 a -1 a a 1286652724 1 foo -1286656282 1 foo -1286656282 0 repo diff --git a/git-annex.hs b/git-annex.hs index 66b9491bde..cae72f00db 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,4 +7,4 @@ import GitRepo main = do gitPrep l <- readLog "demo.log" - writeLog "demo2.log" $ l !! 0 + writeLog "demo2.log" $ compactLog l From a745043e7db87ef43dbbb3f93cdf5807ff9958ec Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 12:35:28 -0400 Subject: [PATCH 0018/8313] don't repeatedly call repoTop, it's a bit expensive --- GitRepo.hs | 24 +++++++++++------------- git-annex.hs | 3 ++- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 8737d82512..140fb628a7 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -14,21 +14,19 @@ stateLoc = ".git-annex" gitStateDir :: String -> String gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" -{- Path to the current repository's gitattributes file. -} -gitAttributes :: IO String -gitAttributes = do - repo <- repoTop +{- Path to a repository's gitattributes file. -} +gitAttributes :: FilePath -> IO String +gitAttributes repo = do bare <- isBareRepo repo if (bare) then return $ repo ++ "/info/.gitattributes" else return $ repo ++ "/.gitattributes" -{- Path to the current repository's .git directory. +{- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} -gitDir :: IO String -gitDir = do - repo <- repoTop +gitDir :: FilePath -> IO String +gitDir repo = do bare <- isBareRepo repo if (bare) then return $ repo @@ -37,7 +35,7 @@ gitDir = do {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} -gitRelative :: String -> String -> String +gitRelative :: FilePath -> String -> String gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo @@ -49,12 +47,12 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Sets up the current git repo for git-annex. May be called repeatedly. -} -gitPrep :: IO () -gitPrep = do +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitPrep :: FilePath -> IO () +gitPrep repo = do -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/* merge=union" - attributes <- gitAttributes + attributes <- gitAttributes repo exists <- doesFileExist attributes if (not exists) then writeFile attributes $ attrLine ++ "\n" diff --git a/git-annex.hs b/git-annex.hs index cae72f00db..0f274e674b 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,6 +5,7 @@ import LocationLog import GitRepo main = do - gitPrep + repo <- repoTop + gitPrep repo l <- readLog "demo.log" writeLog "demo2.log" $ compactLog l From a55f49efb6c05c5ddb031f077690e90ed7358642 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 12:41:20 -0400 Subject: [PATCH 0019/8313] update --- LocationLog.hs | 14 +++++++------- git-annex.hs | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 028ceed5fb..f9421cd9a0 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -65,7 +65,7 @@ instance Read LogLine where {- Reads a log file. - Note that the LogLines returned may be in any order. -} -readLog :: String -> IO [LogLine] +readLog :: FilePath -> IO [LogLine] readLog file = do exists <- doesFileExist file if exists @@ -79,14 +79,14 @@ readLog file = do return [] {- Adds a LogLine to a log file -} -appendLog :: String -> LogLine -> IO () +appendLog :: FilePath -> LogLine -> IO () appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line {- Writes a set of lines to a log file -} -writeLog :: String -> [LogLine] -> IO () +writeLog :: FilePath -> [LogLine] -> IO () writeLog file lines = do createDirectoryIfMissing True (parentDir file) withFileLocked file WriteMode $ \h -> @@ -99,7 +99,7 @@ logNow status repo = do return $ LogLine now status repo {- Returns the filename of the log file for a given annexed file. -} -logFile :: String -> IO String +logFile :: FilePath -> IO String logFile annexedFile = do repo <- repoTop return $ (gitStateDir repo) ++ @@ -107,16 +107,16 @@ logFile annexedFile = do {- Returns a list of repositories that, according to the log, have - the content of a file -} -fileLocations :: String -> IO [String] +fileLocations :: FilePath -> IO [String] fileLocations file = do log <- logFile file lines <- readLog log return $ map repo (filterPresent lines) -{- Filters the list of LogLines to find repositories where the file +{- Filters the list of LogLines to find ones where the file - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = error "unimplimented" -- TODO +filterPresent lines = filter (\l -> FilePresent == status l) $ compactLog lines {- Compacts a set of logs, returning a subset that contains the current - status. -} diff --git a/git-annex.hs b/git-annex.hs index 0f274e674b..8944b50f55 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,5 +7,6 @@ import GitRepo main = do repo <- repoTop gitPrep repo + l <- readLog "demo.log" writeLog "demo2.log" $ compactLog l From 80ce5f90db1de10a5fa42583efcb7390cf185662 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 13:47:04 -0400 Subject: [PATCH 0020/8313] update --- Backend.hs | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ BackendFile.hs | 17 +++++++++++++++ BackendUrl.hs | 17 +++++++++++++++ GitRepo.hs | 14 ++++++------ git-annex.hs | 6 ++++++ 5 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 Backend.hs create mode 100644 BackendFile.hs create mode 100644 BackendUrl.hs diff --git a/Backend.hs b/Backend.hs new file mode 100644 index 0000000000..cb91325c61 --- /dev/null +++ b/Backend.hs @@ -0,0 +1,58 @@ +{- git-annex key/value storage backends + - + - git-annex uses a key/value abstraction layer to allow files contents to be + - stored in different ways. In theory, any key/value storage system could be + - used to store the file contents, and git-annex would then retrieve them + - as needed and put them in `.git/annex/`. + - + - When a file is annexed, a key is generated from its content and/or metadata. + - This key can later be used to retrieve the file's content (its value). This + - key generation must be stable for a given file content, name, and size. + - + - The mapping from filename to its key is stored in the .git-annex directory, + - in a file named `$filename.$backend` + - + - Multiple pluggable backends are supported, and more than one can be used + - to store different files' contents in a given repository. + - -} + +module Backend where + +import GitRepo +import System.Directory + +data Backend = Backend { + name :: String, -- name of this backend + keyvalue :: FilePath -> Maybe String, -- maps from key to value + retrievekey :: IO String -> IO (Bool) -- retrieves value given key +} + +{- Name of state file that holds the key for an annexed file, + - using a given backend. -} +backendFile :: Backend -> GitRepo -> FilePath -> String +backendFile backend repo file = gitStateDir repo ++ + (gitRelative repo file) ++ "." ++ (name backend) + +{- Looks up the backend used for an already annexed file. -} +lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) +lookupBackend [] repo file = return Nothing +lookupBackend (b:bs) repo file = do + present <- checkBackend b repo file + if present + then + return $ Just b + else + lookupBackend bs repo file + +{- Checks if a file is available via a given backend. -} +checkBackend :: Backend -> GitRepo -> FilePath -> IO (Bool) +checkBackend backend repo file = doesFileExist $ backendFile backend repo file + +{- Attempts to retrieve an annexed file from one of the backends. -} +retrieveFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) +retrieveFile backends repo file = do + result <- lookupBackend backends repo file + case (result) of + Nothing -> return False + Just b -> (retrievekey b) key + where key = readFile (backendFile b repo file) diff --git a/BackendFile.hs b/BackendFile.hs new file mode 100644 index 0000000000..b1a3be58a6 --- /dev/null +++ b/BackendFile.hs @@ -0,0 +1,17 @@ +{- git-annex "file" backend + - -} + +module BackendFile (backend) where + +import Backend + +backend = Backend { + name = "file", + keyvalue = keyValue, + retrievekey = copyFile +} + +-- direct mapping from filename to key +keyValue k = Just $ id k + +copyFile f = error "unimplemented" diff --git a/BackendUrl.hs b/BackendUrl.hs new file mode 100644 index 0000000000..f95c53bbfc --- /dev/null +++ b/BackendUrl.hs @@ -0,0 +1,17 @@ +{- git-annex "url" backend + - -} + +module BackendUrl (backend) where + +import Backend + +backend = Backend { + name = "url", + keyvalue = keyValue, + retrievekey = downloadUrl +} + +-- cannot generate url from filename +keyValue k = Nothing + +downloadUrl k = error "unimplemented" diff --git a/GitRepo.hs b/GitRepo.hs index 140fb628a7..8974d9db6c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,14 +8,16 @@ import System.Path import Data.String.Utils import Utility +type GitRepo = FilePath + {- Long-term state is stored in files inside the .git-annex directory - in the git repository. -} stateLoc = ".git-annex" -gitStateDir :: String -> String +gitStateDir :: GitRepo -> FilePath gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" {- Path to a repository's gitattributes file. -} -gitAttributes :: FilePath -> IO String +gitAttributes :: GitRepo -> IO String gitAttributes repo = do bare <- isBareRepo repo if (bare) @@ -25,7 +27,7 @@ gitAttributes repo = do {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} -gitDir :: FilePath -> IO String +gitDir :: GitRepo -> IO String gitDir repo = do bare <- isBareRepo repo if (bare) @@ -35,7 +37,7 @@ gitDir repo = do {- Given a relative or absolute filename, calculates the name to use - relative to a git repository directory (which must be absolute). - This is the same form displayed and used by git. -} -gitRelative :: FilePath -> String -> String +gitRelative :: GitRepo -> String -> String gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo @@ -48,7 +50,7 @@ gitRelative repo file = drop (length absrepo) absfile Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitPrep :: FilePath -> IO () +gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/* merge=union" @@ -66,7 +68,7 @@ gitPrep repo = do {- Finds the top of the current git repository, which may be in a parent - directory. -} -repoTop :: IO String +repoTop :: IO GitRepo repoTop = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop diff --git a/git-annex.hs b/git-annex.hs index 8944b50f55..77faea2b78 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -3,6 +3,12 @@ import LocationLog import GitRepo +import Backend + +-- When adding a new backend, import it here and add it to the backends list. +import qualified BackendFile +import qualified BackendUrl +backends = [BackendFile.backend, BackendUrl.backend] main = do repo <- repoTop From f4d2a05e86df464790fb183148717e7ac7f49cda Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:04:07 -0400 Subject: [PATCH 0021/8313] got annexing working --- Annex.hs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Annex.hs diff --git a/Annex.hs b/Annex.hs new file mode 100644 index 0000000000..7d89c882aa --- /dev/null +++ b/Annex.hs @@ -0,0 +1,36 @@ +{- git-annex + -} + +module Annex where + +import Backend +import System.Posix.Files +import System.Directory +import GitRepo +import Utility + +{- An annexed file's content is stored in .git/annex/. -} +annexedFileLocation repo file = do + dir <- gitDir repo + return $ dir ++ "/annex/" ++ (gitRelative repo file) + +{- Annexes a file, storing it in a backend, and then moving it into + - the annex directory and setting up the symlink pointing to its + - content. -} +annexFile :: [Backend] -> GitRepo -> FilePath -> IO () +annexFile backends repo file = do + alreadyannexed <- lookupBackend backends repo file + case (alreadyannexed) of + Just _ -> error $ "already annexed " ++ file + Nothing -> do + stored <- storeFile backends repo file + if (not stored) + then error $ "no backend could store " ++ file + else symlink + where + symlink = do + dest <- annexedFileLocation repo file + createDirectoryIfMissing True (parentDir dest) + renameFile file dest + createSymbolicLink dest file + gitAdd repo file From cc235192353561a374c431485c6c3834659e0fa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:04:18 -0400 Subject: [PATCH 0022/8313] update --- Backend.hs | 55 +++++++++++++++++++++++++++++++++++++------------- BackendFile.hs | 15 ++++++++++---- BackendUrl.hs | 15 ++++++++++---- GitRepo.hs | 11 ++++++++-- LocationLog.hs | 12 +++++------ git-annex.hs | 1 + 6 files changed, 79 insertions(+), 30 deletions(-) diff --git a/Backend.hs b/Backend.hs index cb91325c61..c55634a681 100644 --- a/Backend.hs +++ b/Backend.hs @@ -18,24 +18,60 @@ module Backend where -import GitRepo import System.Directory +import GitRepo +import Utility + +type Key = String data Backend = Backend { - name :: String, -- name of this backend - keyvalue :: FilePath -> Maybe String, -- maps from key to value - retrievekey :: IO String -> IO (Bool) -- retrieves value given key + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: FilePath -> Key -> IO (Bool), + -- retrieves a key's contents to a file + retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) } +instance Show Backend where + show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" + {- Name of state file that holds the key for an annexed file, - using a given backend. -} backendFile :: Backend -> GitRepo -> FilePath -> String backendFile backend repo file = gitStateDir repo ++ (gitRelative repo file) ++ "." ++ (name backend) +{- Attempts to Stores a file in one of the backends. -} +storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) +storeFile [] _ _ = return False +storeFile (b:bs) repo file = do + try <- (getKey b) (gitRelative repo file) + case (try) of + Nothing -> storeFile bs repo file + Just key -> do + (storeFileKey b) file key + createDirectoryIfMissing True (parentDir backendfile) + writeFile backendfile key + return True + where backendfile = backendFile b repo file + +{- Attempts to retrieve an file from one of the backends, saving it to + - a specified location. -} +retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO (Bool) +retrieveFile backends repo file dest = do + result <- lookupBackend backends repo file + case (result) of + Nothing -> return False + Just b -> (retrieveKeyFile b) key dest + where + key = readFile (backendFile b repo file) + {- Looks up the backend used for an already annexed file. -} lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) -lookupBackend [] repo file = return Nothing +lookupBackend [] _ _ = return Nothing lookupBackend (b:bs) repo file = do present <- checkBackend b repo file if present @@ -47,12 +83,3 @@ lookupBackend (b:bs) repo file = do {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> GitRepo -> FilePath -> IO (Bool) checkBackend backend repo file = doesFileExist $ backendFile backend repo file - -{- Attempts to retrieve an annexed file from one of the backends. -} -retrieveFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) -retrieveFile backends repo file = do - result <- lookupBackend backends repo file - case (result) of - Nothing -> return False - Just b -> (retrievekey b) key - where key = readFile (backendFile b repo file) diff --git a/BackendFile.hs b/BackendFile.hs index b1a3be58a6..324a4d8cd9 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -7,11 +7,18 @@ import Backend backend = Backend { name = "file", - keyvalue = keyValue, - retrievekey = copyFile + getKey = keyValue, + storeFileKey = moveToAnnex, + retrieveKeyFile = copyFromOtherRepo } -- direct mapping from filename to key -keyValue k = Just $ id k +keyValue :: FilePath -> IO (Maybe Key) +keyValue k = return $ Just $ id k + +moveToAnnex :: FilePath -> Key -> IO (Bool) +moveToAnnex file key = return False + +copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) +copyFromOtherRepo key file = return False -copyFile f = error "unimplemented" diff --git a/BackendUrl.hs b/BackendUrl.hs index f95c53bbfc..9b4c83d618 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -7,11 +7,18 @@ import Backend backend = Backend { name = "url", - keyvalue = keyValue, - retrievekey = downloadUrl + getKey = keyValue, + storeFileKey = dummyStore, + retrieveKeyFile = downloadUrl } -- cannot generate url from filename -keyValue k = Nothing +keyValue :: FilePath -> IO (Maybe Key) +keyValue k = return Nothing -downloadUrl k = error "unimplemented" +-- cannot store to urls +dummyStore :: FilePath -> Key -> IO (Bool) +dummyStore file url = return False + +downloadUrl :: IO Key -> FilePath -> IO (Bool) +downloadUrl url file = error "unimplemented" diff --git a/GitRepo.hs b/GitRepo.hs index 8974d9db6c..690782f0dc 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -57,15 +57,22 @@ gitPrep repo = do attributes <- gitAttributes repo exists <- doesFileExist attributes if (not exists) - then writeFile attributes $ attrLine ++ "\n" + then do + writeFile attributes $ attrLine ++ "\n" + gitAdd repo attributes else do content <- readFile attributes if (all (/= attrLine) (lines content)) then do appendFile attributes $ attrLine ++ "\n" - -- TODO check attributes file into git? + gitAdd repo attributes else return () +{- Stages a changed file in git's index. -} +gitAdd repo file = do + -- TODO + return () + {- Finds the top of the current git repository, which may be in a parent - directory. -} repoTop :: IO GitRepo diff --git a/LocationLog.hs b/LocationLog.hs index f9421cd9a0..32af824612 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -84,6 +84,7 @@ appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line + -- TODO git add log {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () @@ -99,17 +100,16 @@ logNow status repo = do return $ LogLine now status repo {- Returns the filename of the log file for a given annexed file. -} -logFile :: FilePath -> IO String -logFile annexedFile = do - repo <- repoTop +logFile :: GitRepo -> FilePath -> IO String +logFile repo annexedFile = do return $ (gitStateDir repo) ++ (gitRelative repo annexedFile) ++ ".log" {- Returns a list of repositories that, according to the log, have - the content of a file -} -fileLocations :: FilePath -> IO [String] -fileLocations file = do - log <- logFile file +fileLocations :: GitRepo -> FilePath -> IO [String] +fileLocations thisrepo file = do + log <- logFile thisrepo file lines <- readLog log return $ map repo (filterPresent lines) diff --git a/git-annex.hs b/git-annex.hs index 77faea2b78..556e0607e8 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -4,6 +4,7 @@ import LocationLog import GitRepo import Backend +import Annex -- When adding a new backend, import it here and add it to the backends list. import qualified BackendFile From 4631927a5c7b14605725f1c6f272fee19d8b4318 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:21:17 -0400 Subject: [PATCH 0023/8313] fix storing files in .git/annex by key --- Annex.hs | 19 +++++++++---------- Backend.hs | 24 ++++++++++++++++-------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index 7d89c882aa..bd9ce92a4e 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,14 +9,13 @@ import System.Directory import GitRepo import Utility -{- An annexed file's content is stored in .git/annex/. -} -annexedFileLocation repo file = do +{- An annexed file's content is stored somewhere under .git/annex/ -} +annexLoc repo key = do dir <- gitDir repo - return $ dir ++ "/annex/" ++ (gitRelative repo file) + return $ dir ++ "/annex/" ++ key {- Annexes a file, storing it in a backend, and then moving it into - - the annex directory and setting up the symlink pointing to its - - content. -} + - the annex directory and setting up the symlink pointing to its content. -} annexFile :: [Backend] -> GitRepo -> FilePath -> IO () annexFile backends repo file = do alreadyannexed <- lookupBackend backends repo file @@ -24,12 +23,12 @@ annexFile backends repo file = do Just _ -> error $ "already annexed " ++ file Nothing -> do stored <- storeFile backends repo file - if (not stored) - then error $ "no backend could store " ++ file - else symlink + case (stored) of + Nothing -> error $ "no backend could store " ++ file + Just key -> symlink key where - symlink = do - dest <- annexedFileLocation repo file + symlink key = do + dest <- annexLoc repo key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file diff --git a/Backend.hs b/Backend.hs index c55634a681..d6b4339890 100644 --- a/Backend.hs +++ b/Backend.hs @@ -44,21 +44,29 @@ backendFile :: Backend -> GitRepo -> FilePath -> String backendFile backend repo file = gitStateDir repo ++ (gitRelative repo file) ++ "." ++ (name backend) -{- Attempts to Stores a file in one of the backends. -} -storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Bool) -storeFile [] _ _ = return False +{- Attempts to store a file in one of the backends, and returns + - its key. -} +storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) +storeFile [] _ _ = return Nothing storeFile (b:bs) repo file = do try <- (getKey b) (gitRelative repo file) case (try) of - Nothing -> storeFile bs repo file + Nothing -> nextbackend Just key -> do - (storeFileKey b) file key + stored <- (storeFileKey b) file key + if (not stored) + then nextbackend + else do + bookkeeping key + return $ Just key + where + nextbackend = storeFile bs repo file + backendfile = backendFile b repo file + bookkeeping key = do createDirectoryIfMissing True (parentDir backendfile) writeFile backendfile key - return True - where backendfile = backendFile b repo file -{- Attempts to retrieve an file from one of the backends, saving it to +{- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO (Bool) retrieveFile backends repo file dest = do From eb577ee37ff1d631aa3580a235b9954043d0fb27 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:27:49 -0400 Subject: [PATCH 0024/8313] stub checksum backend --- BackendChecksum.hs | 18 ++++++++++++++++++ BackendFile.hs | 1 - git-annex.hs | 3 ++- 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 BackendChecksum.hs diff --git a/BackendChecksum.hs b/BackendChecksum.hs new file mode 100644 index 0000000000..267f8099c7 --- /dev/null +++ b/BackendChecksum.hs @@ -0,0 +1,18 @@ +{- git-annex "checksum" backend + - -} + +module BackendChecksum (backend) where + +import Backend +import qualified BackendFile +import Data.Digest.Pure.SHA + +-- based on BackendFile just with a different key type +backend = BackendFile.backend { + name = "checksum", + getKey = keyValue +} + +-- +keyValue :: FilePath -> IO (Maybe Key) +keyValue k = error "unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index 324a4d8cd9..dd6ff595aa 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -21,4 +21,3 @@ moveToAnnex file key = return False copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) copyFromOtherRepo key file = return False - diff --git a/git-annex.hs b/git-annex.hs index 556e0607e8..cce49050b3 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -8,8 +8,9 @@ import Annex -- When adding a new backend, import it here and add it to the backends list. import qualified BackendFile +import qualified BackendChecksum import qualified BackendUrl -backends = [BackendFile.backend, BackendUrl.backend] +backends = [BackendFile.backend, BackendChecksum.backend, BackendUrl.backend] main = do repo <- repoTop From 7880dc16fef81bb6a8812c6b4e9578a6ae2b2879 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:41:35 -0400 Subject: [PATCH 0025/8313] update --- Backend.hs | 8 ++++---- BackendChecksum.hs | 7 ++++--- BackendFile.hs | 16 ++++++++++------ BackendUrl.hs | 11 ++++++----- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Backend.hs b/Backend.hs index d6b4339890..40279866fd 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,9 +28,9 @@ data Backend = Backend { -- name of this backend name :: String, -- converts a filename to a key - getKey :: FilePath -> IO (Maybe Key), + getKey :: GitRepo -> FilePath -> IO (Maybe Key), -- stores a file's contents to a key - storeFileKey :: FilePath -> Key -> IO (Bool), + storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), -- retrieves a key's contents to a file retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) } @@ -49,11 +49,11 @@ backendFile backend repo file = gitStateDir repo ++ storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) storeFile [] _ _ = return Nothing storeFile (b:bs) repo file = do - try <- (getKey b) (gitRelative repo file) + try <- (getKey b) repo (gitRelative repo file) case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) file key + stored <- (storeFileKey b) repo file key if (not stored) then nextbackend else do diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 267f8099c7..7b8d2c281e 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -4,6 +4,7 @@ module BackendChecksum (backend) where import Backend +import GitRepo import qualified BackendFile import Data.Digest.Pure.SHA @@ -13,6 +14,6 @@ backend = BackendFile.backend { getKey = keyValue } --- -keyValue :: FilePath -> IO (Maybe Key) -keyValue k = error "unimplemented" -- TODO +-- checksum the file to get its key +keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue k = error "checksum keyValue unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index dd6ff595aa..6caf30f65d 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -4,20 +4,24 @@ module BackendFile (backend) where import Backend +import GitRepo backend = Backend { name = "file", getKey = keyValue, - storeFileKey = moveToAnnex, + storeFileKey = dummyStore, retrieveKeyFile = copyFromOtherRepo } -- direct mapping from filename to key -keyValue :: FilePath -> IO (Maybe Key) -keyValue k = return $ Just $ id k +keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue repo file = return $ Just file -moveToAnnex :: FilePath -> Key -> IO (Bool) -moveToAnnex file key = return False +-- This backend does not really do any independant data storage, +-- it relies on the file contents in .git/annex/ in this repo, +-- and other accessible repos. So storing a file is a no-op. +dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) +dummyStore repo file key = return True copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) -copyFromOtherRepo key file = return False +copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/BackendUrl.hs b/BackendUrl.hs index 9b4c83d618..1aa5224b58 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -4,6 +4,7 @@ module BackendUrl (backend) where import Backend +import GitRepo backend = Backend { name = "url", @@ -13,12 +14,12 @@ backend = Backend { } -- cannot generate url from filename -keyValue :: FilePath -> IO (Maybe Key) -keyValue k = return Nothing +keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue repo file = return Nothing -- cannot store to urls -dummyStore :: FilePath -> Key -> IO (Bool) -dummyStore file url = return False +dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) +dummyStore repo file url = return False downloadUrl :: IO Key -> FilePath -> IO (Bool) -downloadUrl url file = error "unimplemented" +downloadUrl url file = error "downloadUrl unimplemented" From dce9c2e0804d2c94f46dcac8c9884766bb22dcc7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 15:54:02 -0400 Subject: [PATCH 0026/8313] convert GitRepo to struct with constructor --- Annex.hs | 30 +++++++++++++++++++------ Backend.hs | 1 + BackendFile.hs | 8 ++++--- GitRepo.hs | 61 ++++++++++++++++++-------------------------------- LocationLog.hs | 1 + Locations.hs | 18 +++++++++++++++ git-annex.hs | 2 +- 7 files changed, 71 insertions(+), 50 deletions(-) create mode 100644 Locations.hs diff --git a/Annex.hs b/Annex.hs index bd9ce92a4e..f23358bf63 100644 --- a/Annex.hs +++ b/Annex.hs @@ -3,16 +3,12 @@ module Annex where -import Backend import System.Posix.Files import System.Directory import GitRepo import Utility - -{- An annexed file's content is stored somewhere under .git/annex/ -} -annexLoc repo key = do - dir <- gitDir repo - return $ dir ++ "/annex/" ++ key +import Locations +import Backend {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} @@ -28,8 +24,28 @@ annexFile backends repo file = do Just key -> symlink key where symlink key = do - dest <- annexLoc repo key + dest <- annexDir repo key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file gitAdd repo file + +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitPrep :: GitRepo -> IO () +gitPrep repo = do + -- configure git to use union merge driver on state files + let attrLine = stateLoc ++ "/* merge=union" + attributes <- gitAttributes repo + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + gitAdd repo attributes + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + gitAdd repo attributes + else return () + diff --git a/Backend.hs b/Backend.hs index 40279866fd..e01f122395 100644 --- a/Backend.hs +++ b/Backend.hs @@ -19,6 +19,7 @@ module Backend where import System.Directory +import Locations import GitRepo import Utility diff --git a/BackendFile.hs b/BackendFile.hs index 6caf30f65d..92f708696b 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -17,11 +17,13 @@ backend = Backend { keyValue :: GitRepo -> FilePath -> IO (Maybe Key) keyValue repo file = return $ Just file --- This backend does not really do any independant data storage, --- it relies on the file contents in .git/annex/ in this repo, --- and other accessible repos. So storing a file is a no-op. +{- This backend does not really do any independant data storage, + - it relies on the file contents in .git/annex/ in this repo, + - and other accessible repos. So storing a file is a no-op. -} dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) dummyStore repo file key = return True +{- Try to find a copy of the file in one of the other repos, + - and copy it over to this one. -} copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/GitRepo.hs b/GitRepo.hs index 690782f0dc..fda83f7d8e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,79 +8,62 @@ import System.Path import Data.String.Utils import Utility -type GitRepo = FilePath +data GitRepo = GitRepo { + top :: FilePath, + remotes :: [GitRepo] +} deriving (Eq, Show, Read) -{- Long-term state is stored in files inside the .git-annex directory - - in the git repository. -} -stateLoc = ".git-annex" -gitStateDir :: GitRepo -> FilePath -gitStateDir repo = repo ++ "/" ++ stateLoc ++ "/" +{- GitRepo constructor -} +gitRepo :: FilePath -> IO GitRepo +gitRepo dir = do + -- TOOD query repo for configuration settings; other repositories; etc + return GitRepo { top = dir, remotes = [] } {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String gitAttributes repo = do - bare <- isBareRepo repo + bare <- isBareRepo (top repo) if (bare) - then return $ repo ++ "/info/.gitattributes" - else return $ repo ++ "/.gitattributes" + then return $ (top repo) ++ "/info/.gitattributes" + else return $ (top repo) ++ "/.gitattributes" {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} gitDir :: GitRepo -> IO String gitDir repo = do - bare <- isBareRepo repo + bare <- isBareRepo (top repo) if (bare) - then return $ repo - else return $ repo ++ "/.git" + then return $ (top repo) + else return $ (top repo) ++ "/.git" {- Given a relative or absolute filename, calculates the name to use - - relative to a git repository directory (which must be absolute). + - to refer to the file relative to a git repository directory. - This is the same form displayed and used by git. -} gitRelative :: GitRepo -> String -> String gitRelative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file - absrepo = case (absNormPath "/" repo) of + absrepo = case (absNormPath "/" (top repo)) of Just f -> f ++ "/" - Nothing -> error $ "bad repo" ++ repo + Nothing -> error $ "bad repo" ++ (top repo) absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitPrep :: GitRepo -> IO () -gitPrep repo = do - -- configure git to use union merge driver on state files - let attrLine = stateLoc ++ "/* merge=union" - attributes <- gitAttributes repo - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - gitAdd repo attributes - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - gitAdd repo attributes - else return () - {- Stages a changed file in git's index. -} gitAdd repo file = do -- TODO return () -{- Finds the top of the current git repository, which may be in a parent - - directory. -} -repoTop :: IO GitRepo -repoTop = do +{- Finds the current git repository, which may be in a parent directory. -} +currentRepo :: IO GitRepo +currentRepo = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> return dir + (Just dir) -> gitRepo dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) diff --git a/LocationLog.hs b/LocationLog.hs index 32af824612..73e9f1c6d0 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -25,6 +25,7 @@ import System.Directory import Data.Char import GitRepo import Utility +import Locations data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) diff --git a/Locations.hs b/Locations.hs new file mode 100644 index 0000000000..7273797ef9 --- /dev/null +++ b/Locations.hs @@ -0,0 +1,18 @@ +{- git-annex file locations + -} + +module Locations where + +import GitRepo + +{- An annexed file's content is stored somewhere under .git/annex/ -} +annexDir :: GitRepo -> String -> IO FilePath +annexDir repo key = do + dir <- gitDir repo + return $ dir ++ "/annex/" ++ key + +{- Long-term state is stored in files inside the .git-annex directory + - in the git repository. -} +stateLoc = ".git-annex" +gitStateDir :: GitRepo -> FilePath +gitStateDir repo = (top repo) ++ "/" ++ stateLoc ++ "/" diff --git a/git-annex.hs b/git-annex.hs index cce49050b3..f8c67b1fdd 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -13,7 +13,7 @@ import qualified BackendUrl backends = [BackendFile.backend, BackendChecksum.backend, BackendUrl.backend] main = do - repo <- repoTop + repo <- currentRepo gitPrep repo l <- readLog "demo.log" From e5514e0cb0809848645814e8c1f67cd89cb16c4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 18:05:37 -0400 Subject: [PATCH 0027/8313] update --- Annex.hs | 1 + Backend.hs | 14 +------------- BackendChecksum.hs | 3 +-- BackendFile.hs | 3 +-- BackendList.hs | 14 ++++++++++++++ BackendUrl.hs | 3 +-- CmdLine.hs | 41 +++++++++++++++++++++++++++++++++++++++++ GitRepo.hs | 13 +++++++------ LocationLog.hs | 1 + Locations.hs | 3 ++- Types.hs | 24 ++++++++++++++++++++++++ git-annex.hs | 17 +++++++---------- 12 files changed, 101 insertions(+), 36 deletions(-) create mode 100644 BackendList.hs create mode 100644 CmdLine.hs create mode 100644 Types.hs diff --git a/Annex.hs b/Annex.hs index f23358bf63..bddff1e13d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,6 +8,7 @@ import System.Directory import GitRepo import Utility import Locations +import Types import Backend {- Annexes a file, storing it in a backend, and then moving it into diff --git a/Backend.hs b/Backend.hs index e01f122395..93ceee234a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -22,19 +22,7 @@ import System.Directory import Locations import GitRepo import Utility - -type Key = String - -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), - -- retrieves a key's contents to a file - retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) -} +import Types instance Show Backend where show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 7b8d2c281e..18ff0cb57c 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -3,8 +3,7 @@ module BackendChecksum (backend) where -import Backend -import GitRepo +import Types import qualified BackendFile import Data.Digest.Pure.SHA diff --git a/BackendFile.hs b/BackendFile.hs index 92f708696b..deb4bce7e8 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,8 +3,7 @@ module BackendFile (backend) where -import Backend -import GitRepo +import Types backend = Backend { name = "file", diff --git a/BackendList.hs b/BackendList.hs new file mode 100644 index 0000000000..c744949b6c --- /dev/null +++ b/BackendList.hs @@ -0,0 +1,14 @@ +{- git-annex backend list + - -} + +module BackendList where + +-- When adding a new backend, import it here and add it to the list. +import qualified BackendFile +import qualified BackendChecksum +import qualified BackendUrl +supportedBackends = + [ BackendFile.backend + , BackendChecksum.backend + , BackendUrl.backend + ] diff --git a/BackendUrl.hs b/BackendUrl.hs index 1aa5224b58..2bc34434be 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,8 +3,7 @@ module BackendUrl (backend) where -import Backend -import GitRepo +import Types backend = Backend { name = "url", diff --git a/CmdLine.hs b/CmdLine.hs new file mode 100644 index 0000000000..79bd55cd9e --- /dev/null +++ b/CmdLine.hs @@ -0,0 +1,41 @@ +{- git-annex command line + - + - TODO: This is very rough and stupid; I would like to use + - System.Console.CmdArgs.Implicit but it is not yet packaged in Debian. + -} + +module CmdLine where + +import System.Console.GetOpt +import Types +import Annex + +data Flag = Add FilePath | Push String | Pull String | + Want FilePath | Get (Maybe FilePath) | Drop FilePath + deriving Show + +options :: [OptDescr Flag] +options = + [ Option ['a'] ["add"] (ReqArg Add "FILE") "add file to annex" + , Option ['p'] ["push"] (ReqArg Push "REPO") "push annex to repo" + , Option ['P'] ["pull"] (ReqArg Pull "REPO") "pull annex from repo" + , Option ['w'] ["want"] (ReqArg Want "FILE") "request file contents" + , Option ['g'] ["get"] (OptArg Get "FILE") "transfer file contents" + , Option ['d'] ["drop"] (ReqArg Drop "FILE") "indicate file content not needed" + ] + +argvToFlags argv = do + case getOpt Permute options argv of + -- no options? add listed files + ([],p,[] ) -> return $ map (\f -> Add f) p + -- all options parsed, return flags + (o,[],[] ) -> return o + -- error case + (_,n,errs) -> ioError (userError (concat errs ++ usageInfo header options)) + where header = "Usage: git-annex [option] file" + +dispatch :: Flag -> [Backend] -> GitRepo -> IO () +dispatch flag backends repo = do + case (flag) of + Add f -> annexFile backends repo f + _ -> error "not implemented" diff --git a/GitRepo.hs b/GitRepo.hs index fda83f7d8e..a0909d5ecd 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,17 +7,18 @@ import System.Directory import System.Path import Data.String.Utils import Utility - -data GitRepo = GitRepo { - top :: FilePath, - remotes :: [GitRepo] -} deriving (Eq, Show, Read) +import Types +import BackendList {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo gitRepo dir = do -- TOOD query repo for configuration settings; other repositories; etc - return GitRepo { top = dir, remotes = [] } + return GitRepo { + top = dir, + remotes = [], + backends = supportedBackends + } {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String diff --git a/LocationLog.hs b/LocationLog.hs index 73e9f1c6d0..a5e9a2679a 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -26,6 +26,7 @@ import Data.Char import GitRepo import Utility import Locations +import Types data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) diff --git a/Locations.hs b/Locations.hs index 7273797ef9..50f94a727d 100644 --- a/Locations.hs +++ b/Locations.hs @@ -3,10 +3,11 @@ module Locations where +import Types import GitRepo {- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> String -> IO FilePath +annexDir :: GitRepo -> Key -> IO FilePath annexDir repo key = do dir <- gitDir repo return $ dir ++ "/annex/" ++ key diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..2308b6fde9 --- /dev/null +++ b/Types.hs @@ -0,0 +1,24 @@ +{- git-annex data types + - -} + +module Types where + +type Key = String + +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: GitRepo -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), + -- retrieves a key's contents to a file + retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) +} + +data GitRepo = GitRepo { + top :: FilePath, + remotes :: [GitRepo], + backends :: [Backend] +} + diff --git a/git-annex.hs b/git-annex.hs index f8c67b1fdd..590a7c0519 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,20 +1,17 @@ {- git-annex main program - -} -import LocationLog +import System.Environment import GitRepo -import Backend +import CmdLine import Annex - --- When adding a new backend, import it here and add it to the backends list. -import qualified BackendFile -import qualified BackendChecksum -import qualified BackendUrl -backends = [BackendFile.backend, BackendChecksum.backend, BackendUrl.backend] +import BackendList main = do + args <- getArgs + flags <- argvToFlags args + repo <- currentRepo gitPrep repo - l <- readLog "demo.log" - writeLog "demo2.log" $ compactLog l + mapM (\f -> dispatch f supportedBackends repo) flags From 026adce5a01381e9a802747f2ddf4ca5635468c9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 18:25:31 -0400 Subject: [PATCH 0028/8313] update --- Annex.hs | 18 ++++++++++++------ CmdLine.hs | 6 +++--- GitRepo.hs | 4 +--- Types.hs | 11 +++++++++-- git-annex.hs | 5 ++--- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/Annex.hs b/Annex.hs index bddff1e13d..bd3cade583 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,26 +10,32 @@ import Utility import Locations import Types import Backend +import BackendList + +startAnnex :: IO State +startAnnex = do + r <- currentRepo + return State { repo = r, backends = supportedBackends } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexFile :: [Backend] -> GitRepo -> FilePath -> IO () -annexFile backends repo file = do - alreadyannexed <- lookupBackend backends repo file +annexFile :: State -> FilePath -> IO () +annexFile state file = do + alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed " ++ file Nothing -> do - stored <- storeFile backends repo file + stored <- storeFile (backends state) (repo state) file case (stored) of Nothing -> error $ "no backend could store " ++ file Just key -> symlink key where symlink key = do - dest <- annexDir repo key + dest <- annexDir (repo state) key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file - gitAdd repo file + gitAdd (repo state) file {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () diff --git a/CmdLine.hs b/CmdLine.hs index 79bd55cd9e..d848ee8f9c 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -34,8 +34,8 @@ argvToFlags argv = do (_,n,errs) -> ioError (userError (concat errs ++ usageInfo header options)) where header = "Usage: git-annex [option] file" -dispatch :: Flag -> [Backend] -> GitRepo -> IO () -dispatch flag backends repo = do +dispatch :: Flag -> State -> IO () +dispatch flag state = do case (flag) of - Add f -> annexFile backends repo f + Add f -> annexFile state f _ -> error "not implemented" diff --git a/GitRepo.hs b/GitRepo.hs index a0909d5ecd..06da2ff883 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -8,7 +8,6 @@ import System.Path import Data.String.Utils import Utility import Types -import BackendList {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo @@ -16,8 +15,7 @@ gitRepo dir = do -- TOOD query repo for configuration settings; other repositories; etc return GitRepo { top = dir, - remotes = [], - backends = supportedBackends + remotes = [] } {- Path to a repository's gitattributes file. -} diff --git a/Types.hs b/Types.hs index 2308b6fde9..cab4b2016e 100644 --- a/Types.hs +++ b/Types.hs @@ -3,8 +3,10 @@ module Types where +-- annexed filenames are mapped into keys type Key = String +-- this structure represents a key/value backend data Backend = Backend { -- name of this backend name :: String, @@ -16,9 +18,14 @@ data Backend = Backend { retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) } +-- a git repository data GitRepo = GitRepo { top :: FilePath, - remotes :: [GitRepo], - backends :: [Backend] + remotes :: [GitRepo] } +-- git-annex's runtime state +data State = State { + repo :: GitRepo, + backends :: [Backend] +} diff --git a/git-annex.hs b/git-annex.hs index 590a7c0519..2c9b1315fe 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -11,7 +11,6 @@ main = do args <- getArgs flags <- argvToFlags args - repo <- currentRepo - gitPrep repo + state <- startAnnex - mapM (\f -> dispatch f supportedBackends repo) flags + mapM (\f -> dispatch f state) flags From 586266e444f72808101a055323887fe08ae7fce3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 19:00:08 -0400 Subject: [PATCH 0029/8313] robustness --- Annex.hs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index bd3cade583..402c767daa 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,4 +1,4 @@ -{- git-annex +{- git-annex toplevel code -} module Annex where @@ -12,15 +12,21 @@ import Types import Backend import BackendList +{- On startup, examine the git repo, prepare it, and record state for + - later. -} startAnnex :: IO State startAnnex = do r <- currentRepo + gitPrep r + -- TODO query git repo for configuration return State { repo = r, backends = supportedBackends } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do + checkExists file + checkLegal file alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed " ++ file @@ -36,6 +42,16 @@ annexFile state file = do renameFile file dest createSymbolicLink dest file gitAdd (repo state) file + checkExists file = do + exists <- doesFileExist file + case (exists) of + False -> error $ "does not exist: " ++ file + True -> return () + checkLegal file = do + s <- getFileStatus file + case (not (isSymbolicLink s) && not (isRegularFile s)) of + False -> error $ "not a regular file: " ++ file + True -> return () {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () From 93d2dc0d6878ccb1067376d2a03193c222429d3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 19:14:32 -0400 Subject: [PATCH 0030/8313] cache whether a repo is bare --- GitRepo.hs | 9 +++++---- Types.hs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 06da2ff883..ef76fb9766 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -12,17 +12,19 @@ import Types {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo gitRepo dir = do + b <- isBareRepo dir + -- TOOD query repo for configuration settings; other repositories; etc return GitRepo { top = dir, + bare = b, remotes = [] } {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String gitAttributes repo = do - bare <- isBareRepo (top repo) - if (bare) + if (bare repo) then return $ (top repo) ++ "/info/.gitattributes" else return $ (top repo) ++ "/.gitattributes" @@ -31,8 +33,7 @@ gitAttributes repo = do - TODO: support GIT_DIR -} gitDir :: GitRepo -> IO String gitDir repo = do - bare <- isBareRepo (top repo) - if (bare) + if (bare repo) then return $ (top repo) else return $ (top repo) ++ "/.git" diff --git a/Types.hs b/Types.hs index cab4b2016e..e1f598f0f5 100644 --- a/Types.hs +++ b/Types.hs @@ -21,6 +21,7 @@ data Backend = Backend { -- a git repository data GitRepo = GitRepo { top :: FilePath, + bare :: Bool, remotes :: [GitRepo] } From 344f13394fe5b12cbdd5eeb99bb63892c7096bfd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 19:53:31 -0400 Subject: [PATCH 0031/8313] update --- Annex.hs | 30 +++++++++++++++++++++++------- Backend.hs | 23 +++++++++++++++++++---- BackendFile.hs | 10 +++++++--- BackendUrl.hs | 11 +++++++---- CmdLine.hs | 6 ++++-- Types.hs | 6 ++++-- git-annex.mdwn | 1 + 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/Annex.hs b/Annex.hs index 402c767daa..964532f3f2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -44,14 +44,30 @@ annexFile state file = do gitAdd (repo state) file checkExists file = do exists <- doesFileExist file - case (exists) of - False -> error $ "does not exist: " ++ file - True -> return () + if (not exists) + then error $ "does not exist: " ++ file + else return () checkLegal file = do - s <- getFileStatus file - case (not (isSymbolicLink s) && not (isRegularFile s)) of - False -> error $ "not a regular file: " ++ file - True -> return () + s <- getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then error $ "not a regular file: " ++ file + else return () + +{- Inverse of annexFile. -} +unannexFile :: State -> FilePath -> IO () +unannexFile state file = do + alreadyannexed <- lookupBackend (backends state) (repo state) file + case (alreadyannexed) of + Nothing -> error $ "not annexed " ++ file + Just _ -> do + mkey <- dropFile (backends state) (repo state) file + case (mkey) of + Nothing -> return () + Just key -> do + src <- annexDir (repo state) key + removeFile file + renameFile src file + return () {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () diff --git a/Backend.hs b/Backend.hs index 93ceee234a..5ddd3aac65 100644 --- a/Backend.hs +++ b/Backend.hs @@ -57,14 +57,29 @@ storeFile (b:bs) repo file = do {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} -retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO (Bool) +retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO Bool retrieveFile backends repo file dest = do result <- lookupBackend backends repo file case (result) of Nothing -> return False - Just b -> (retrieveKeyFile b) key dest - where - key = readFile (backendFile b repo file) + Just b -> do + key <- lookupKey b repo file + (retrieveKeyFile b) key dest + +{- Drops the key for a file from the backend that has it. -} +dropFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) +dropFile backends repo file = do + result <- lookupBackend backends repo file + case (result) of + Nothing -> return Nothing + Just b -> do + key <- lookupKey b repo file + (removeKey b) key + return $ Just key + +{- Looks up the key a backend uses for an already annexed file. -} +lookupKey :: Backend -> GitRepo -> FilePath -> IO Key +lookupKey backend repo file = readFile (backendFile backend repo file) {- Looks up the backend used for an already annexed file. -} lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) diff --git a/BackendFile.hs b/BackendFile.hs index deb4bce7e8..de60803c36 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -9,7 +9,8 @@ backend = Backend { name = "file", getKey = keyValue, storeFileKey = dummyStore, - retrieveKeyFile = copyFromOtherRepo + retrieveKeyFile = copyFromOtherRepo, + removeKey = dummyRemove } -- direct mapping from filename to key @@ -18,11 +19,14 @@ keyValue repo file = return $ Just file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - - and other accessible repos. So storing a file is a no-op. -} + - and other accessible repos. So storing or removing a key is + - a no-op. -} dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) dummyStore repo file key = return True +dummyRemove :: Key -> IO Bool +dummyRemove url = return False {- Try to find a copy of the file in one of the other repos, - and copy it over to this one. -} -copyFromOtherRepo :: IO Key -> FilePath -> IO (Bool) +copyFromOtherRepo :: Key -> FilePath -> IO (Bool) copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/BackendUrl.hs b/BackendUrl.hs index 2bc34434be..ddeab9e042 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -9,16 +9,19 @@ backend = Backend { name = "url", getKey = keyValue, storeFileKey = dummyStore, - retrieveKeyFile = downloadUrl + retrieveKeyFile = downloadUrl, + removeKey = dummyRemove } -- cannot generate url from filename keyValue :: GitRepo -> FilePath -> IO (Maybe Key) keyValue repo file = return Nothing --- cannot store to urls -dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) +-- cannot change urls +dummyStore :: GitRepo -> FilePath -> Key -> IO Bool dummyStore repo file url = return False +dummyRemove :: Key -> IO Bool +dummyRemove url = return False -downloadUrl :: IO Key -> FilePath -> IO (Bool) +downloadUrl :: Key -> FilePath -> IO Bool downloadUrl url file = error "downloadUrl unimplemented" diff --git a/CmdLine.hs b/CmdLine.hs index d848ee8f9c..3709f836bd 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,8 +10,8 @@ import System.Console.GetOpt import Types import Annex -data Flag = Add FilePath | Push String | Pull String | - Want FilePath | Get (Maybe FilePath) | Drop FilePath +data Flag = Add FilePath | Push String | Pull String | Want FilePath | + Get (Maybe FilePath) | Drop FilePath | Unannex FilePath deriving Show options :: [OptDescr Flag] @@ -22,6 +22,7 @@ options = , Option ['w'] ["want"] (ReqArg Want "FILE") "request file contents" , Option ['g'] ["get"] (OptArg Get "FILE") "transfer file contents" , Option ['d'] ["drop"] (ReqArg Drop "FILE") "indicate file content not needed" + , Option ['u'] ["unannex"] (ReqArg Unannex "FILE") "undo --add" ] argvToFlags argv = do @@ -38,4 +39,5 @@ dispatch :: Flag -> State -> IO () dispatch flag state = do case (flag) of Add f -> annexFile state f + Unannex f -> unannexFile state f _ -> error "not implemented" diff --git a/Types.hs b/Types.hs index e1f598f0f5..6e3727e25a 100644 --- a/Types.hs +++ b/Types.hs @@ -13,9 +13,11 @@ data Backend = Backend { -- converts a filename to a key getKey :: GitRepo -> FilePath -> IO (Maybe Key), -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO (Bool), + storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: IO Key -> FilePath -> IO (Bool) + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool } -- a git repository diff --git a/git-annex.mdwn b/git-annex.mdwn index bc3550398c..2996a90b51 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -36,6 +36,7 @@ Enough broad picture, here's how it actually looks: downloaded. * `git annex --drop $file` indicates that you no longer want the file's content to be available in this repository. +* `git annex --unannex $file` undoes a `git annex --add`. * `git annex $file` is a shorthand for either --add or --get. If the file is already known, it does --get, otherwise it does --add. From 200bc6fdb84658593bfb02f34f984531b6710d26 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 21:00:42 -0400 Subject: [PATCH 0032/8313] better option handling multiple-file support for all modes --- Annex.hs | 12 +++--------- CmdLine.hs | 45 +++++++++++++++++++++++---------------------- git-annex.hs | 29 +++++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/Annex.hs b/Annex.hs index 964532f3f2..ee94a98097 100644 --- a/Annex.hs +++ b/Annex.hs @@ -25,15 +25,14 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - checkExists file - checkLegal file alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of - Just _ -> error $ "already annexed " ++ file + Just _ -> error $ "already annexed: " ++ file Nothing -> do + checkLegal file stored <- storeFile (backends state) (repo state) file case (stored) of - Nothing -> error $ "no backend could store " ++ file + Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key where symlink key = do @@ -42,11 +41,6 @@ annexFile state file = do renameFile file dest createSymbolicLink dest file gitAdd (repo state) file - checkExists file = do - exists <- doesFileExist file - if (not exists) - then error $ "does not exist: " ++ file - else return () checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) diff --git a/CmdLine.hs b/CmdLine.hs index 3709f836bd..c956f29a55 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,34 +10,35 @@ import System.Console.GetOpt import Types import Annex -data Flag = Add FilePath | Push String | Pull String | Want FilePath | - Get (Maybe FilePath) | Drop FilePath | Unannex FilePath +data Mode = Add | Push | Pull | Want | Get | Drop | Unannex deriving Show -options :: [OptDescr Flag] +options :: [OptDescr Mode] options = - [ Option ['a'] ["add"] (ReqArg Add "FILE") "add file to annex" - , Option ['p'] ["push"] (ReqArg Push "REPO") "push annex to repo" - , Option ['P'] ["pull"] (ReqArg Pull "REPO") "pull annex from repo" - , Option ['w'] ["want"] (ReqArg Want "FILE") "request file contents" - , Option ['g'] ["get"] (OptArg Get "FILE") "transfer file contents" - , Option ['d'] ["drop"] (ReqArg Drop "FILE") "indicate file content not needed" - , Option ['u'] ["unannex"] (ReqArg Unannex "FILE") "undo --add" + [ Option ['a'] ["add"] (NoArg Add) "add files to annex" + , Option ['p'] ["push"] (NoArg Push) "push annex to repos" + , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" + , Option ['w'] ["want"] (NoArg Want) "request file contents" + , Option ['g'] ["get"] (NoArg Get) "transfer file contents" + , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" + , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" ] -argvToFlags argv = do +argvToMode argv = do case getOpt Permute options argv of - -- no options? add listed files - ([],p,[] ) -> return $ map (\f -> Add f) p - -- all options parsed, return flags - (o,[],[] ) -> return o + -- default mode is Add + ([],files,[]) -> return (Add, files) + -- one mode is normal case + (m:[],files,[]) -> return (m, files) + -- multiple modes is an error + (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) -- error case - (_,n,errs) -> ioError (userError (concat errs ++ usageInfo header options)) - where header = "Usage: git-annex [option] file" + (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) + where header = "Usage: git-annex [mode] file" -dispatch :: Flag -> State -> IO () -dispatch flag state = do - case (flag) of - Add f -> annexFile state f - Unannex f -> unannexFile state f +dispatch :: State -> Mode -> FilePath -> IO () +dispatch state mode file = do + case (mode) of + Add -> annexFile state file + Unannex -> unannexFile state file _ -> error "not implemented" diff --git a/git-annex.hs b/git-annex.hs index 2c9b1315fe..22fbe60ca3 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,16 +1,37 @@ {- git-annex main program - -} +import System.IO import System.Environment -import GitRepo +import Control.Exception import CmdLine import Annex -import BackendList main = do args <- getArgs - flags <- argvToFlags args + (mode, files) <- argvToMode args state <- startAnnex - mapM (\f -> dispatch f state) flags + tryRun 0 $ map (\f -> dispatch state mode f) files + +{- Tries to run a series of actions, not stopping if some error out, + - and propigating an overall error status at the end. -} +tryRun errflag [] = do + if (errflag > 0) + then error "unsuccessful" + else return () +tryRun errflag (a:as) = do + result <- try (a)::IO (Either SomeException ()) + case (result) of + Left err -> do + showErr err + tryRun 1 as + Right _ -> tryRun errflag as + +{- Exception pretty-printing. -} +showErr :: SomeException -> IO () +showErr e = do + let err = show e + hPutStrLn stderr $ "git-annex: " ++ err + return () From dc5e8853f3b0857f2023df6cb23e57bf42b5b858 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 21:04:25 -0400 Subject: [PATCH 0033/8313] missed a file --- Backend.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Backend.hs b/Backend.hs index 5ddd3aac65..eb4a948c4e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -75,6 +75,7 @@ dropFile backends repo file = do Just b -> do key <- lookupKey b repo file (removeKey b) key + removeFile $ backendFile b repo file return $ Just key {- Looks up the key a backend uses for an already annexed file. -} From c5d7ca0a5a2c6837d394e23d1a18a1005ee6f1b6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 10 Oct 2010 22:20:52 -0400 Subject: [PATCH 0034/8313] use Data.Time instead of Data.DateTime The latter has shady rounding. The new module is a bit harder to use, but worth it, it adds subsecond timestamps too. --- Annex.hs | 1 + LocationLog.hs | 50 ++++++++++++++++++++++++++++---------------------- demo.log | 12 ++++++------ 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/Annex.hs b/Annex.hs index ee94a98097..ad94758c50 100644 --- a/Annex.hs +++ b/Annex.hs @@ -11,6 +11,7 @@ import Locations import Types import Backend import BackendList +import LocationLog {- On startup, examine the git repo, prepare it, and record state for - later. -} diff --git a/LocationLog.hs b/LocationLog.hs index a5e9a2679a..195596bda3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -18,7 +18,9 @@ module LocationLog where -import Data.DateTime +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale import qualified Data.Map as Map import System.IO import System.Directory @@ -28,6 +30,12 @@ import Utility import Locations import Types +data LogLine = LogLine { + date :: POSIXTime, + status :: LogStatus, + reponame :: String +} deriving (Eq) + data LogStatus = FilePresent | FileMissing | Undefined deriving (Eq) @@ -41,29 +49,30 @@ instance Read LogStatus where readsPrec _ "0" = [(FileMissing, "")] readsPrec _ _ = [(Undefined, "")] -data LogLine = LogLine { - date :: DateTime, - status :: LogStatus, - repo :: String -} deriving (Eq) - instance Show LogLine where - show (LogLine date status repo) = unwords - [(show (toSeconds date)), (show status), repo] + show (LogLine date status reponame) = unwords + [(show date), (show status), reponame] instance Read LogLine where -- This parser is robust in that even unparsable log lines are -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = - if (length w >= 3 && all isDigit date) - then [((LogLine (fromSeconds $ read date) status repo), "")] - else [((LogLine (fromSeconds 0) Undefined ""), "")] + if (length w >= 3) + then case (pdate) of + Just v -> good v + Nothing -> undefined + else undefined where w = words string date = w !! 0 status = read $ w !! 1 - repo = unwords $ drop 2 w + reponame = unwords $ drop 2 w + pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime + + good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status reponame + undefined = ret $ LogLine (0) Undefined "" + ret v = [(v, "")] {- Reads a log file. - Note that the LogLines returned may be in any order. -} @@ -97,9 +106,9 @@ writeLog file lines = do {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine -logNow status repo = do - now <- getCurrentTime - return $ LogLine now status repo +logNow status reponame = do + now <- getPOSIXTime + return $ LogLine now status reponame {- Returns the filename of the log file for a given annexed file. -} logFile :: GitRepo -> FilePath -> IO String @@ -113,7 +122,7 @@ fileLocations :: GitRepo -> FilePath -> IO [String] fileLocations thisrepo file = do log <- logFile thisrepo file lines <- readLog log - return $ map repo (filterPresent lines) + return $ map reponame (filterPresent lines) {- Filters the list of LogLines to find ones where the file - is (or should still be) present. -} @@ -131,12 +140,9 @@ compactLog' map (l:ls) = compactLog' (mapLog map l) ls - information about a repo than the other logs in the map -} mapLog map log = if (better) - then Map.insert (repo log) log map + then Map.insert (reponame log) log map else map where - better = case (Map.lookup (repo log) map) of - -- <= used because two log entries could - -- have the same timestamp; if so the one that - -- is seen last should win. + better = case (Map.lookup (reponame log) map) of Just l -> (date l <= date log) Nothing -> True diff --git a/demo.log b/demo.log index 7a42630566..bdecb7d401 100644 --- a/demo.log +++ b/demo.log @@ -1,11 +1,11 @@ -1286654242 1 repo -1286652724 0 foo -1286656282 1 foo -1286656282 0 repo -1286656281 0 foo +1286654242s 1 repo +1286652724s 0 foo +1286656282s 1 foo +1286656282s 0 repo +1286656281s 0 foo # some garbage, should be ignored a a a a 1 a -1 a a -1286652724 1 foo +1286652724.0001s 1 foo From 2bd3eea0318fe52452fa7077fe94ae3f224ae9c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 00:19:38 -0400 Subject: [PATCH 0035/8313] add git config lookups for annex.name, annex.backends, etc --- Annex.hs | 33 +++++++++++++++++++++++++++------ BackendList.hs | 18 ++++++++++++++++++ GitRepo.hs | 16 +++++++++++++--- Types.hs | 11 ++++++++--- git-annex.mdwn | 13 +++++++++---- 5 files changed, 75 insertions(+), 16 deletions(-) diff --git a/Annex.hs b/Annex.hs index ad94758c50..882ed2761b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -18,20 +18,38 @@ import LocationLog startAnnex :: IO State startAnnex = do r <- currentRepo + config <- getConfig r gitPrep r - -- TODO query git repo for configuration - return State { repo = r, backends = supportedBackends } + return State { + repo = r, + gitconfig = config + } + +{- Query the git repo for relevant configuration settings. -} +getConfig :: GitRepo -> IO GitConfig +getConfig repo = do + -- a name can be configured, if none is, use the repository path + name <- gitConfigGet "annex.name" (top repo) + -- default number of copies to keep of file contents is 1 + numcopies <- gitConfigGet "annex.numcopies" "1" + backends <- gitConfigGet "annex.backends" "" + + return GitConfig { + annex_name = name, + annex_numcopies = read numcopies, + annex_backends = parseBackendList backends + } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend backends (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (backends state) (repo state) file + stored <- storeFile (annex_backends $ gitconfig state) (repo state) file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -47,15 +65,16 @@ annexFile state file = do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () + backends = annex_backends $ gitconfig state {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend backends (repo state) file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile (backends state) (repo state) file + mkey <- dropFile backends (repo state) file case (mkey) of Nothing -> return () Just key -> do @@ -63,6 +82,8 @@ unannexFile state file = do removeFile file renameFile src file return () + where + backends = annex_backends $ gitconfig state {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () diff --git a/BackendList.hs b/BackendList.hs index c744949b6c..77e4bd817f 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -4,6 +4,7 @@ module BackendList where -- When adding a new backend, import it here and add it to the list. +import Types import qualified BackendFile import qualified BackendChecksum import qualified BackendUrl @@ -12,3 +13,20 @@ supportedBackends = , BackendChecksum.backend , BackendUrl.backend ] + +{- Parses a string with a list of backend names into + - a list of Backend objects. If the list is empty, + - defaults to supportedBackends. -} +parseBackendList :: String -> [Backend] +parseBackendList s = + if (length s == 0) + then supportedBackends + else map (lookupBackendName) $ words s + +{- Looks up a supported backed by name. -} +lookupBackendName :: String -> Backend +lookupBackendName s = + if ((length matches) /= 1) + then error $ "unknown backend " ++ s + else matches !! 0 + where matches = filter (\b -> s == name b) supportedBackends diff --git a/GitRepo.hs b/GitRepo.hs index ef76fb9766..3a8a8110dd 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -5,7 +5,10 @@ module GitRepo where import Directory import System.Directory import System.Path +import System.Cmd.Utils +import System.IO import Data.String.Utils +import Control.Exception import Utility import Types @@ -14,11 +17,9 @@ gitRepo :: FilePath -> IO GitRepo gitRepo dir = do b <- isBareRepo dir - -- TOOD query repo for configuration settings; other repositories; etc return GitRepo { top = dir, - bare = b, - remotes = [] + bare = b } {- Path to a repository's gitattributes file. -} @@ -53,10 +54,19 @@ gitRelative repo file = drop (length absrepo) absfile Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo {- Stages a changed file in git's index. -} +gitAdd :: GitRepo -> FilePath -> IO () gitAdd repo file = do -- TODO return () +{- Queries git-config. -} +gitConfigGet :: String -> String -> IO String +gitConfigGet name defaultValue = + handle ((\_ -> return defaultValue)::SomeException -> IO String) $ + pOpen ReadFromPipe "git" ["config", "--get", name] $ \h -> do + ret <- hGetLine h + return ret + {- Finds the current git repository, which may be in a parent directory. -} currentRepo :: IO GitRepo currentRepo = do diff --git a/Types.hs b/Types.hs index 6e3727e25a..5c5a428d59 100644 --- a/Types.hs +++ b/Types.hs @@ -23,12 +23,17 @@ data Backend = Backend { -- a git repository data GitRepo = GitRepo { top :: FilePath, - bare :: Bool, - remotes :: [GitRepo] + bare :: Bool } -- git-annex's runtime state data State = State { repo :: GitRepo, - backends :: [Backend] + gitconfig :: GitConfig +} + +data GitConfig = GitConfig { + annex_name :: String, + annex_numcopies :: Int, + annex_backends :: [Backend] } diff --git a/git-annex.mdwn b/git-annex.mdwn index 2996a90b51..6bfdd57c7f 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -124,8 +124,9 @@ so the lines may be in arbitrary order, but it will never conflict.) ## configuration * `annex.numcopies` -- number of copies of files to keep -* `annex.backend` -- name of the default key/value backend to use to - store new files +* `annex.backends` -- space-separated list of names of + the key/value backends to use. The first listed is used to store + new files. * `annex.name` -- allows specifying a unique name for this repository. If not specified, the name is derived from its directory's location and the hostname. When a repository is on removable media it is useful to give @@ -145,11 +146,15 @@ If the symlink to annexed content is relative, moving it to a subdir will break it. But it it's absolute, moving the git repo (or mounting its drive elsewhere) will break it. Either: -* Use relative links and need `git annex mv` to move (or post-commit +* Use relative links and need `git annex --mv` to move (or post-commit hook that caches moves and updates links). * Use absolute links and need `git annex fixlinks` when location changes; note that would also mean that git would see the symlink targets changed - and want to commit the change. + and want to commit the change. And, other clones of the repo would + diverge and there would be conflicts on the symlink text. Ugh. + +Hard links are not an option, because git would then happily commit the +file content. Amoung other reasons.. ### free space determination From 779ebba96153e712803c8284a0502d7080c609bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 00:22:50 -0400 Subject: [PATCH 0036/8313] adjust merge config *.log will merge, but foo.$backend files will not --- Annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index 882ed2761b..e3956cbf20 100644 --- a/Annex.hs +++ b/Annex.hs @@ -89,7 +89,7 @@ unannexFile state file = do gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files - let attrLine = stateLoc ++ "/* merge=union" + let attrLine = stateLoc ++ "/*.log merge=union" attributes <- gitAttributes repo exists <- doesFileExist attributes if (not exists) From ecf19abf76d449d2e69d89518566a7cce899708d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 00:23:49 -0400 Subject: [PATCH 0037/8313] foo --- Locations.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index 50f94a727d..b25e99197a 100644 --- a/Locations.hs +++ b/Locations.hs @@ -12,8 +12,8 @@ annexDir repo key = do dir <- gitDir repo return $ dir ++ "/annex/" ++ key -{- Long-term state is stored in files inside the .git-annex directory - - in the git repository. -} +{- Long-term, cross-repo state is stored in files inside the .git-annex + - directory, in the git repository. -} stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (top repo) ++ "/" ++ stateLoc ++ "/" From de3dafae80f45af8db56dc95e11863f049cf3cb2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 16:35:28 -0400 Subject: [PATCH 0038/8313] update --- TODO | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 0000000000..ea930f8034 --- /dev/null +++ b/TODO @@ -0,0 +1,9 @@ +* bug when annexing files in a subdir of a git repo +* how to handle git mv file? + +* query remotes for their annex.name settings + +* hook up LocationLog +* --push/--pull/--get/--want/--drop + +* finish BackendUrl and BackendChecksum From af82586adff96f18fe768e432f501c647401262f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 17:19:55 -0400 Subject: [PATCH 0039/8313] split up Types --- Annex.hs | 19 ++++++++++++++++++- Backend.hs | 18 +++++++++++++++++- BackendChecksum.hs | 3 ++- BackendFile.hs | 3 ++- BackendList.hs | 3 ++- BackendUrl.hs | 3 ++- CmdLine.hs | 1 - GitRepo.hs | 7 ++++++- LocationLog.hs | 1 - Locations.hs | 7 ------- Types.hs | 39 --------------------------------------- 11 files changed, 49 insertions(+), 55 deletions(-) delete mode 100644 Types.hs diff --git a/Annex.hs b/Annex.hs index e3956cbf20..cedc478a43 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,11 +8,28 @@ import System.Directory import GitRepo import Utility import Locations -import Types import Backend import BackendList import LocationLog +-- git-annex's runtime state +data State = State { + repo :: GitRepo, + gitconfig :: GitConfig +} + +data GitConfig = GitConfig { + annex_name :: String, + annex_numcopies :: Int, + annex_backends :: [Backend] +} + +{- An annexed file's content is stored somewhere under .git/annex/ -} +annexDir :: GitRepo -> Key -> IO FilePath +annexDir repo key = do + dir <- gitDir repo + return $ dir ++ "/annex/" ++ key + {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State diff --git a/Backend.hs b/Backend.hs index eb4a948c4e..fb7d5666f3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -22,7 +22,23 @@ import System.Directory import Locations import GitRepo import Utility -import Types + +-- annexed filenames are mapped into keys +type Key = FilePath + +-- this structure represents a key/value backend +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: GitRepo -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool +} instance Show Backend where show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 18ff0cb57c..e262962cab 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -3,9 +3,10 @@ module BackendChecksum (backend) where -import Types import qualified BackendFile import Data.Digest.Pure.SHA +import Backend +import GitRepo -- based on BackendFile just with a different key type backend = BackendFile.backend { diff --git a/BackendFile.hs b/BackendFile.hs index de60803c36..831f804173 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,7 +3,8 @@ module BackendFile (backend) where -import Types +import Backend +import GitRepo backend = Backend { name = "file", diff --git a/BackendList.hs b/BackendList.hs index 77e4bd817f..c3a1b13a1b 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -3,8 +3,9 @@ module BackendList where +import Backend + -- When adding a new backend, import it here and add it to the list. -import Types import qualified BackendFile import qualified BackendChecksum import qualified BackendUrl diff --git a/BackendUrl.hs b/BackendUrl.hs index ddeab9e042..f08f0bdb46 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,7 +3,8 @@ module BackendUrl (backend) where -import Types +import Backend +import GitRepo backend = Backend { name = "url", diff --git a/CmdLine.hs b/CmdLine.hs index c956f29a55..53707cd716 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -7,7 +7,6 @@ module CmdLine where import System.Console.GetOpt -import Types import Annex data Mode = Add | Push | Pull | Want | Get | Drop | Unannex diff --git a/GitRepo.hs b/GitRepo.hs index 3a8a8110dd..c26f752ef8 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -10,7 +10,12 @@ import System.IO import Data.String.Utils import Control.Exception import Utility -import Types + +-- a git repository +data GitRepo = GitRepo { + top :: FilePath, + bare :: Bool +} {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo diff --git a/LocationLog.hs b/LocationLog.hs index 195596bda3..8e6b56fe8e 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -28,7 +28,6 @@ import Data.Char import GitRepo import Utility import Locations -import Types data LogLine = LogLine { date :: POSIXTime, diff --git a/Locations.hs b/Locations.hs index b25e99197a..22a0db8e2c 100644 --- a/Locations.hs +++ b/Locations.hs @@ -3,15 +3,8 @@ module Locations where -import Types import GitRepo -{- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> Key -> IO FilePath -annexDir repo key = do - dir <- gitDir repo - return $ dir ++ "/annex/" ++ key - {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository. -} stateLoc = ".git-annex" diff --git a/Types.hs b/Types.hs deleted file mode 100644 index 5c5a428d59..0000000000 --- a/Types.hs +++ /dev/null @@ -1,39 +0,0 @@ -{- git-annex data types - - -} - -module Types where - --- annexed filenames are mapped into keys -type Key = String - --- this structure represents a key/value backend -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, - -- removes a key - removeKey :: Key -> IO Bool -} - --- a git repository -data GitRepo = GitRepo { - top :: FilePath, - bare :: Bool -} - --- git-annex's runtime state -data State = State { - repo :: GitRepo, - gitconfig :: GitConfig -} - -data GitConfig = GitConfig { - annex_name :: String, - annex_numcopies :: Int, - annex_backends :: [Backend] -} From ebc3fbe9ae2c5cc52332c77a92697c2517ce8263 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 17:52:46 -0400 Subject: [PATCH 0040/8313] explicit exports --- Annex.hs | 11 ++++++++--- Backend.hs | 29 ++++++++--------------------- BackendChecksum.hs | 2 +- BackendFile.hs | 2 +- BackendList.hs | 8 ++++++-- BackendUrl.hs | 2 +- CmdLine.hs | 5 ++++- GitRepo.hs | 22 +++++++++++++++++----- LocationLog.hs | 3 ++- Locations.hs | 7 +++++-- Utility.hs | 6 +++++- 11 files changed, 58 insertions(+), 39 deletions(-) diff --git a/Annex.hs b/Annex.hs index cedc478a43..8660febd57 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,7 +1,12 @@ {- git-annex toplevel code -} -module Annex where +module Annex ( + State, + startAnnex, + annexFile, + unannexFile +) where import System.Posix.Files import System.Directory @@ -34,7 +39,7 @@ annexDir repo key = do - later. -} startAnnex :: IO State startAnnex = do - r <- currentRepo + r <- gitRepoCurrent config <- getConfig r gitPrep r return State { @@ -46,7 +51,7 @@ startAnnex = do getConfig :: GitRepo -> IO GitConfig getConfig repo = do -- a name can be configured, if none is, use the repository path - name <- gitConfigGet "annex.name" (top repo) + name <- gitConfigGet "annex.name" (gitRepoTop repo) -- default number of copies to keep of file contents is 1 numcopies <- gitConfigGet "annex.numcopies" "1" backends <- gitConfigGet "annex.backends" "" diff --git a/Backend.hs b/Backend.hs index fb7d5666f3..2d3ea42d64 100644 --- a/Backend.hs +++ b/Backend.hs @@ -16,32 +16,19 @@ - to store different files' contents in a given repository. - -} -module Backend where +module Backend ( + Key, + Backend, -- note only data type is exported, not destructors + lookupBackend, + storeFile, + dropFile +) where import System.Directory import Locations import GitRepo import Utility - --- annexed filenames are mapped into keys -type Key = FilePath - --- this structure represents a key/value backend -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, - -- removes a key - removeKey :: Key -> IO Bool -} - -instance Show Backend where - show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" +import BackendType {- Name of state file that holds the key for an annexed file, - using a given backend. -} diff --git a/BackendChecksum.hs b/BackendChecksum.hs index e262962cab..e80dbe793c 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -5,7 +5,7 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA -import Backend +import BackendType import GitRepo -- based on BackendFile just with a different key type diff --git a/BackendFile.hs b/BackendFile.hs index 831f804173..ae53f460f1 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,7 +3,7 @@ module BackendFile (backend) where -import Backend +import BackendType import GitRepo backend = Backend { diff --git a/BackendList.hs b/BackendList.hs index c3a1b13a1b..f733a44be9 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -1,9 +1,13 @@ {- git-annex backend list - -} -module BackendList where +module BackendList ( + supportedBackends, + parseBackendList, + lookupBackendName +) where -import Backend +import BackendType -- When adding a new backend, import it here and add it to the list. import qualified BackendFile diff --git a/BackendUrl.hs b/BackendUrl.hs index f08f0bdb46..4ba1dbadb5 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,7 +3,7 @@ module BackendUrl (backend) where -import Backend +import BackendType import GitRepo backend = Backend { diff --git a/CmdLine.hs b/CmdLine.hs index 53707cd716..cc87088898 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -4,7 +4,10 @@ - System.Console.CmdArgs.Implicit but it is not yet packaged in Debian. -} -module CmdLine where +module CmdLine ( + argvToMode, + dispatch +) where import System.Console.GetOpt import Annex diff --git a/GitRepo.hs b/GitRepo.hs index c26f752ef8..d01ba642b0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -1,6 +1,15 @@ {- git repository handling -} -module GitRepo where +module GitRepo ( + GitRepo, + gitRepoCurrent, + gitRepoTop, + gitDir, + gitRelative, + gitConfigGet, + gitAdd, + gitAttributes +) where import Directory import System.Directory @@ -13,7 +22,7 @@ import Utility -- a git repository data GitRepo = GitRepo { - top :: FilePath, + gitRepoTop :: FilePath, bare :: Bool } @@ -23,10 +32,13 @@ gitRepo dir = do b <- isBareRepo dir return GitRepo { - top = dir, + gitRepoTop = dir, bare = b } +{- Short name used in here for top of repo. -} +top = gitRepoTop + {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String gitAttributes repo = do @@ -73,8 +85,8 @@ gitConfigGet name defaultValue = return ret {- Finds the current git repository, which may be in a parent directory. -} -currentRepo :: IO GitRepo -currentRepo = do +gitRepoCurrent :: IO GitRepo +gitRepoCurrent = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of diff --git a/LocationLog.hs b/LocationLog.hs index 8e6b56fe8e..31d454f103 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -16,7 +16,8 @@ - so the lines may be in arbitrary order, but it will never conflict. -} -module LocationLog where +module LocationLog ( +) where import Data.Time.Clock.POSIX import Data.Time diff --git a/Locations.hs b/Locations.hs index 22a0db8e2c..31bb3d9de3 100644 --- a/Locations.hs +++ b/Locations.hs @@ -1,7 +1,10 @@ {- git-annex file locations -} -module Locations where +module Locations ( + gitStateDir, + stateLoc +) where import GitRepo @@ -9,4 +12,4 @@ import GitRepo - directory, in the git repository. -} stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath -gitStateDir repo = (top repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir repo = (gitRepoTop repo) ++ "/" ++ stateLoc ++ "/" diff --git a/Utility.hs b/Utility.hs index d1eb247d3c..dea53967fc 100644 --- a/Utility.hs +++ b/Utility.hs @@ -1,7 +1,11 @@ {- git-annex utility functions -} -module Utility where +module Utility ( + withFileLocked, + hGetContentsStrict, + parentDir +) where import System.IO import System.Posix.IO From 104fe9132af2553a29631b1cd38cc79169e9d9f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:15:05 -0400 Subject: [PATCH 0041/8313] cleanup --- Annex.hs | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8660febd57..972cb3e0f0 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,10 +20,10 @@ import LocationLog -- git-annex's runtime state data State = State { repo :: GitRepo, - gitconfig :: GitConfig + config :: Config } -data GitConfig = GitConfig { +data Config = Config { annex_name :: String, annex_numcopies :: Int, annex_backends :: [Backend] @@ -40,26 +40,11 @@ annexDir repo key = do startAnnex :: IO State startAnnex = do r <- gitRepoCurrent - config <- getConfig r + config <- queryConfig r gitPrep r return State { repo = r, - gitconfig = config - } - -{- Query the git repo for relevant configuration settings. -} -getConfig :: GitRepo -> IO GitConfig -getConfig repo = do - -- a name can be configured, if none is, use the repository path - name <- gitConfigGet "annex.name" (gitRepoTop repo) - -- default number of copies to keep of file contents is 1 - numcopies <- gitConfigGet "annex.numcopies" "1" - backends <- gitConfigGet "annex.backends" "" - - return GitConfig { - annex_name = name, - annex_numcopies = read numcopies, - annex_backends = parseBackendList backends + config = config } {- Annexes a file, storing it in a backend, and then moving it into @@ -71,7 +56,7 @@ annexFile state file = do Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (annex_backends $ gitconfig state) (repo state) file + stored <- storeFile backends (repo state) file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -87,7 +72,7 @@ annexFile state file = do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - backends = annex_backends $ gitconfig state + backends = getConfig state annex_backends {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () @@ -105,7 +90,22 @@ unannexFile state file = do renameFile src file return () where - backends = annex_backends $ gitconfig state + backends = getConfig state annex_backends + +{- Query the git repo for relevant configuration settings. -} +queryConfig :: GitRepo -> IO Config +queryConfig repo = do + -- a name can be configured, if none is, use the repository path + name <- gitConfigGet "annex.name" (gitRepoTop repo) + -- default number of copies to keep of file contents is 1 + numcopies <- gitConfigGet "annex.numcopies" "1" + backends <- gitConfigGet "annex.backends" "" + + return Config { + annex_name = name, + annex_numcopies = read numcopies, + annex_backends = parseBackendList backends + } {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () @@ -126,3 +126,6 @@ gitPrep repo = do gitAdd repo attributes else return () +{- Looks up a key in a State's Config -} +getConfig :: State -> (Config -> b) -> b +getConfig state key = key $ config state From f516b820caa702ee76c85b005fef285b8372c4da Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:15:14 -0400 Subject: [PATCH 0042/8313] add --- BackendType.hs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 BackendType.hs diff --git a/BackendType.hs b/BackendType.hs new file mode 100644 index 0000000000..3bc822f329 --- /dev/null +++ b/BackendType.hs @@ -0,0 +1,31 @@ +{- git-annex backend data types + - -} + +module BackendType ( + -- the entire types are exported, for use in backend implementations + Key(..), + Backend(..) +) where + +import GitRepo + +-- annexed filenames are mapped into keys +type Key = FilePath + +-- this structure represents a key/value backend +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: GitRepo -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool +} + +instance Show Backend where + show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" + From 8f99409518d343ded6a1355b4366bd21ee4cf66d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:31:41 -0400 Subject: [PATCH 0043/8313] simpler exception handling --- GitRepo.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index d01ba642b0..de54f6dca6 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -16,8 +16,8 @@ import System.Directory import System.Path import System.Cmd.Utils import System.IO +import System.IO.Error import Data.String.Utils -import Control.Exception import Utility -- a git repository @@ -79,7 +79,7 @@ gitAdd repo file = do {- Queries git-config. -} gitConfigGet :: String -> String -> IO String gitConfigGet name defaultValue = - handle ((\_ -> return defaultValue)::SomeException -> IO String) $ + flip catch (\_ -> return defaultValue) $ pOpen ReadFromPipe "git" ["config", "--get", name] $ \h -> do ret <- hGetLine h return ret From 530f16b980bbfe70b49d5112ad9c48a9754e69c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:39:09 -0400 Subject: [PATCH 0044/8313] better result summary --- git-annex.hs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/git-annex.hs b/git-annex.hs index 22fbe60ca3..ec80359b6c 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -13,21 +13,21 @@ main = do state <- startAnnex - tryRun 0 $ map (\f -> dispatch state mode f) files + tryRun 0 0 $ map (\f -> dispatch state mode f) files {- Tries to run a series of actions, not stopping if some error out, - and propigating an overall error status at the end. -} -tryRun errflag [] = do - if (errflag > 0) - then error "unsuccessful" +tryRun errnum oknum [] = do + if (errnum > 0) + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " succeeded" else return () -tryRun errflag (a:as) = do +tryRun errnum oknum (a:as) = do result <- try (a)::IO (Either SomeException ()) case (result) of Left err -> do showErr err - tryRun 1 as - Right _ -> tryRun errflag as + tryRun (errnum + 1) oknum as + Right _ -> tryRun errnum (oknum + 1) as {- Exception pretty-printing. -} showErr :: SomeException -> IO () From f6306bc301af7db3da7afa6e095014de37e2bce3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 18:39:36 -0400 Subject: [PATCH 0045/8313] wording --- git-annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-annex.hs b/git-annex.hs index ec80359b6c..7bcd4de226 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -19,7 +19,7 @@ main = do - and propigating an overall error status at the end. -} tryRun errnum oknum [] = do if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " succeeded" + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () tryRun errnum oknum (a:as) = do result <- try (a)::IO (Either SomeException ()) From cd1e39b127e96298685906e455ff186312d08029 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 23:22:38 -0400 Subject: [PATCH 0046/8313] moved config reading into GitRepo --- Annex.hs | 52 +++++++------------------------ GitRepo.hs | 89 +++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 76 insertions(+), 65 deletions(-) diff --git a/Annex.hs b/Annex.hs index 972cb3e0f0..abb7bff6e8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,49 +20,41 @@ import LocationLog -- git-annex's runtime state data State = State { repo :: GitRepo, - config :: Config -} - -data Config = Config { - annex_name :: String, - annex_numcopies :: Int, - annex_backends :: [Backend] + backends :: [Backend] } {- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> Key -> IO FilePath -annexDir repo key = do - dir <- gitDir repo - return $ dir ++ "/annex/" ++ key +annexDir :: GitRepo -> Key -> FilePath +annexDir repo key = gitDir repo ++ "/annex/" ++ key {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State startAnnex = do r <- gitRepoCurrent - config <- queryConfig r gitPrep r + return State { repo = r, - config = config + backends = parseBackendList $ gitConfig r "annex.backends" "" } {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend backends (repo state) file + alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile backends (repo state) file + stored <- storeFile (backends state) (repo state) file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key where symlink key = do - dest <- annexDir (repo state) key + let dest = annexDir (repo state) key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file @@ -72,40 +64,22 @@ annexFile state file = do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - backends = getConfig state annex_backends {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend backends (repo state) file + alreadyannexed <- lookupBackend (backends state) (repo state) file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile backends (repo state) file + mkey <- dropFile (backends state) (repo state) file case (mkey) of Nothing -> return () Just key -> do - src <- annexDir (repo state) key + let src = annexDir (repo state) key removeFile file renameFile src file return () - where - backends = getConfig state annex_backends - -{- Query the git repo for relevant configuration settings. -} -queryConfig :: GitRepo -> IO Config -queryConfig repo = do - -- a name can be configured, if none is, use the repository path - name <- gitConfigGet "annex.name" (gitRepoTop repo) - -- default number of copies to keep of file contents is 1 - numcopies <- gitConfigGet "annex.numcopies" "1" - backends <- gitConfigGet "annex.backends" "" - - return Config { - annex_name = name, - annex_numcopies = read numcopies, - annex_backends = parseBackendList backends - } {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () @@ -125,7 +99,3 @@ gitPrep repo = do appendFile attributes $ attrLine ++ "\n" gitAdd repo attributes else return () - -{- Looks up a key in a State's Config -} -getConfig :: State -> (Config -> b) -> b -getConfig state key = key $ config state diff --git a/GitRepo.hs b/GitRepo.hs index de54f6dca6..7ae6584dd4 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -1,4 +1,9 @@ -{- git repository handling -} +{- git repository handling + - + - This is written to be completely independant of git-annex and should be + - suitable for other uses. + - + - -} module GitRepo ( GitRepo, @@ -6,38 +11,46 @@ module GitRepo ( gitRepoTop, gitDir, gitRelative, - gitConfigGet, + gitConfig, gitAdd, gitAttributes ) where import Directory +import System import System.Directory import System.Path import System.Cmd.Utils import System.IO -import System.IO.Error +import System.Posix.Process import Data.String.Utils +import Data.Map as Map (fromList, empty, lookup, Map) import Utility -- a git repository data GitRepo = GitRepo { - gitRepoTop :: FilePath, - bare :: Bool -} + top :: FilePath, + bare :: Bool, + config :: Map String String +} deriving (Show, Read, Eq) {- GitRepo constructor -} gitRepo :: FilePath -> IO GitRepo gitRepo dir = do b <- isBareRepo dir - return GitRepo { - gitRepoTop = dir, - bare = b + let r = GitRepo { + top = dir, + bare = b, + config = Map.empty } + r' <- gitConfigRead r -{- Short name used in here for top of repo. -} -top = gitRepoTop + return r' + +{- Field accessor. -} +gitRepoTop :: GitRepo -> FilePath +gitRepoTop repo = top repo {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> IO String @@ -49,11 +62,11 @@ gitAttributes repo = do {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} -gitDir :: GitRepo -> IO String -gitDir repo = do +gitDir :: GitRepo -> String +gitDir repo = if (bare repo) - then return $ (top repo) - else return $ (top repo) ++ "/.git" + then top repo + else top repo ++ "/.git" {- Given a relative or absolute filename, calculates the name to use - to refer to the file relative to a git repository directory. @@ -72,17 +85,45 @@ gitRelative repo file = drop (length absrepo) absfile {- Stages a changed file in git's index. -} gitAdd :: GitRepo -> FilePath -> IO () -gitAdd repo file = do - -- TODO +gitAdd repo file = runGit repo ["add", file] + +{- Constructs a git command line operating on the specified repo. -} +gitCommandLine :: GitRepo -> [String] -> [String] +gitCommandLine repo params = + -- force use of specified repo via --git-dir and --work-tree + ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + +{- Runs git in the specified repo. -} +runGit :: GitRepo -> [String] -> IO () +runGit repo params = do + r <- executeFile "git" True (gitCommandLine repo params) Nothing return () -{- Queries git-config. -} -gitConfigGet :: String -> String -> IO String -gitConfigGet name defaultValue = - flip catch (\_ -> return defaultValue) $ - pOpen ReadFromPipe "git" ["config", "--get", name] $ \h -> do - ret <- hGetLine h - return ret +{- Runs a git subcommand and returns its output. -} +gitPipeRead :: GitRepo -> [String] -> IO String +gitPipeRead repo params = + pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do + ret <- hGetContentsStrict h + return ret + +{- Runs git config and populates a repo with its settings. -} +gitConfigRead :: GitRepo -> IO GitRepo +gitConfigRead repo = do + c <- gitPipeRead repo ["config", "--list"] + return repo { config = Map.fromList $ parse c } + where + parse s = map ( \l -> (key l, val l) ) $ lines s + keyval l = split sep l :: [String] + key l = (keyval l) !! 0 + val l = join sep $ drop 1 $ keyval l + sep = "=" + +{- Returns a single git config setting, or a default value if not set. -} +gitConfig :: GitRepo -> String -> String -> String +gitConfig repo key defaultValue = + case (Map.lookup key $ config repo) of + Just value -> value + Nothing -> defaultValue {- Finds the current git repository, which may be in a parent directory. -} gitRepoCurrent :: IO GitRepo From 16b551726d9d846a51656b7b1d4736a3b1b438f4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 23:27:48 -0400 Subject: [PATCH 0047/8313] minor --- GitRepo.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 7ae6584dd4..2f9084ff7c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -112,10 +112,11 @@ gitConfigRead repo = do c <- gitPipeRead repo ["config", "--list"] return repo { config = Map.fromList $ parse c } where - parse s = map ( \l -> (key l, val l) ) $ lines s - keyval l = split sep l :: [String] + parse s = map pair $ lines s + pair l = (key l, val l) key l = (keyval l) !! 0 val l = join sep $ drop 1 $ keyval l + keyval l = split sep l :: [String] sep = "=" {- Returns a single git config setting, or a default value if not set. -} From 107074d6623a687d046615a5034af10be7ff1756 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 11 Oct 2010 23:41:12 -0400 Subject: [PATCH 0048/8313] fiddle --- GitRepo.hs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2f9084ff7c..2a97e60704 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,6 +7,7 @@ module GitRepo ( GitRepo, + gitRepoFromPath, gitRepoCurrent, gitRepoTop, gitDir, @@ -35,8 +36,8 @@ data GitRepo = GitRepo { } deriving (Show, Read, Eq) {- GitRepo constructor -} -gitRepo :: FilePath -> IO GitRepo -gitRepo dir = do +gitRepoFromPath :: FilePath -> IO GitRepo +gitRepoFromPath dir = do b <- isBareRepo dir let r = GitRepo { @@ -110,14 +111,17 @@ gitPipeRead repo params = gitConfigRead :: GitRepo -> IO GitRepo gitConfigRead repo = do c <- gitPipeRead repo ["config", "--list"] - return repo { config = Map.fromList $ parse c } - where - parse s = map pair $ lines s - pair l = (key l, val l) - key l = (keyval l) !! 0 - val l = join sep $ drop 1 $ keyval l - keyval l = split sep l :: [String] - sep = "=" + return repo { config = gitConfigParse c } + +{- Parses git config --list output into a config map. -} +gitConfigParse :: String -> Map.Map String String +gitConfigParse s = Map.fromList $ map pair $ lines s + where + pair l = (key l, val l) + key l = (keyval l) !! 0 + val l = join sep $ drop 1 $ keyval l + keyval l = split sep l :: [String] + sep = "=" {- Returns a single git config setting, or a default value if not set. -} gitConfig :: GitRepo -> String -> String -> String @@ -132,7 +136,7 @@ gitRepoCurrent = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> gitRepo dir + (Just dir) -> gitRepoFromPath dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) From 92bf408c664e4fa66132a7095e1f856312ce667c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 00:53:42 -0400 Subject: [PATCH 0049/8313] faddle --- Annex.hs | 2 +- GitRepo.hs | 99 ++++++++++++++++++++++++++++++++++++---------------- Locations.hs | 4 +-- 3 files changed, 72 insertions(+), 33 deletions(-) diff --git a/Annex.hs b/Annex.hs index abb7bff6e8..5a7274ca15 100644 --- a/Annex.hs +++ b/Annex.hs @@ -86,7 +86,7 @@ gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" - attributes <- gitAttributes repo + let attributes = gitAttributes repo exists <- doesFileExist attributes if (not exists) then do diff --git a/GitRepo.hs b/GitRepo.hs index 2a97e60704..dc1c52b478 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,9 +7,10 @@ module GitRepo ( GitRepo, - gitRepoFromPath, gitRepoCurrent, - gitRepoTop, + gitRepoFromPath, + gitRepoFromUrl, + gitWorkTree, gitDir, gitRelative, gitConfig, @@ -26,21 +27,28 @@ import System.IO import System.Posix.Process import Data.String.Utils import Data.Map as Map (fromList, empty, lookup, Map) +import Network.URI +import Maybe import Utility --- a git repository -data GitRepo = GitRepo { - top :: FilePath, - bare :: Bool, - config :: Map String String -} deriving (Show, Read, Eq) +{- A git repository can be local or remote. -} +data GitRepo = + LocalGitRepo { + top :: FilePath, + bare :: Bool, + config :: Map String String + } | RemoteGitRepo { + url :: String, + top :: FilePath, + config :: Map String String + } deriving (Show, Read, Eq) -{- GitRepo constructor -} +{- Local GitRepo constructor. -} gitRepoFromPath :: FilePath -> IO GitRepo gitRepoFromPath dir = do b <- isBareRepo dir - let r = GitRepo { + let r = LocalGitRepo { top = dir, bare = b, config = Map.empty @@ -49,28 +57,49 @@ gitRepoFromPath dir = do return r' -{- Field accessor. -} -gitRepoTop :: GitRepo -> FilePath -gitRepoTop repo = top repo +{- Remote GitRepo constructor. Note that remote repo config is not read. + - Throws exception on invalid url. -} +gitRepoFromUrl :: String -> IO GitRepo +gitRepoFromUrl url = do + return RemoteGitRepo { + url = url, + top = path url, + config = Map.empty + } + where path url = uriPath $ fromJust $ parseURI url + +{- Some code needs to vary between remote and local repos. -} +local repo = case (repo) of + LocalGitRepo {} -> True + RemoteGitRepo {} -> False +remote repo = not $ local repo +assertlocal repo action = + if (local repo) + then action + else error "acting on remote git repo not supported" {- Path to a repository's gitattributes file. -} -gitAttributes :: GitRepo -> IO String -gitAttributes repo = do +gitAttributes :: GitRepo -> String +gitAttributes repo = assertlocal repo $ do if (bare repo) - then return $ (top repo) ++ "/info/.gitattributes" - else return $ (top repo) ++ "/.gitattributes" + then (top repo) ++ "/info/.gitattributes" + else (top repo) ++ "/.gitattributes" {- Path to a repository's .git directory. - (For a bare repository, that is the root of the repository.) - TODO: support GIT_DIR -} gitDir :: GitRepo -> String -gitDir repo = +gitDir repo = assertlocal repo $ if (bare repo) then top repo else top repo ++ "/.git" -{- Given a relative or absolute filename, calculates the name to use - - to refer to the file relative to a git repository directory. +{- Path to a repository's --work-tree. -} +gitWorkTree :: GitRepo -> FilePath +gitWorkTree repo = top repo + +{- Given a relative or absolute filename in a repository, calculates the + - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} gitRelative :: GitRepo -> String -> String gitRelative repo file = drop (length absrepo) absfile @@ -92,26 +121,36 @@ gitAdd repo file = runGit repo ["add", file] gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + if (local repo) + then ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + else error "gitCommandLine not implemented for remote repo" {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () -runGit repo params = do - r <- executeFile "git" True (gitCommandLine repo params) Nothing - return () +runGit repo params = + if (local repo) + then do + r <- executeFile "git" True (gitCommandLine repo params) Nothing + return () + else error "runGit not implemented for remote repo" {- Runs a git subcommand and returns its output. -} gitPipeRead :: GitRepo -> [String] -> IO String gitPipeRead repo params = - pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret + if (local repo) + then pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do + ret <- hGetContentsStrict h + return ret + else error "gitPipeRead not implemented for remote repo" {- Runs git config and populates a repo with its settings. -} gitConfigRead :: GitRepo -> IO GitRepo -gitConfigRead repo = do - c <- gitPipeRead repo ["config", "--list"] - return repo { config = gitConfigParse c } +gitConfigRead repo = + if (local repo) + then do + c <- gitPipeRead repo ["config", "--list"] + return repo { config = gitConfigParse c } + else error "gitConfigRead not implemented for remote repo" {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String diff --git a/Locations.hs b/Locations.hs index 31bb3d9de3..300f443f7f 100644 --- a/Locations.hs +++ b/Locations.hs @@ -9,7 +9,7 @@ module Locations ( import GitRepo {- Long-term, cross-repo state is stored in files inside the .git-annex - - directory, in the git repository. -} + - directory, in the git repository's working tree. -} stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath -gitStateDir repo = (gitRepoTop repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" From eea55856e9db85884a7fb28ce1b408fdbc05f90f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 01:35:32 -0400 Subject: [PATCH 0050/8313] remotes lookup --- GitRepo.hs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index dc1c52b478..fb3ddbaf85 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -26,7 +26,7 @@ import System.Cmd.Utils import System.IO import System.Posix.Process import Data.String.Utils -import Data.Map as Map (fromList, empty, lookup, Map) +import Data.Map as Map hiding (map, split) import Network.URI import Maybe import Utility @@ -61,14 +61,15 @@ gitRepoFromPath dir = do - Throws exception on invalid url. -} gitRepoFromUrl :: String -> IO GitRepo gitRepoFromUrl url = do - return RemoteGitRepo { + return $ RemoteGitRepo { url = url, top = path url, config = Map.empty } where path url = uriPath $ fromJust $ parseURI url -{- Some code needs to vary between remote and local repos. -} +{- Some code needs to vary between remote and local repos, these functions + - help with that. -} local repo = case (repo) of LocalGitRepo {} -> True RemoteGitRepo {} -> False @@ -165,9 +166,19 @@ gitConfigParse s = Map.fromList $ map pair $ lines s {- Returns a single git config setting, or a default value if not set. -} gitConfig :: GitRepo -> String -> String -> String gitConfig repo key defaultValue = - case (Map.lookup key $ config repo) of - Just value -> value - Nothing -> defaultValue + Map.findWithDefault key defaultValue (config repo) + +{- Returns a list of a repo's configured remotes. -} +gitConfigRemotes :: GitRepo -> IO [GitRepo] +gitConfigRemotes repo = mapM construct remotes + where + remotes = elems $ filter $ config repo + filter = filterWithKey (\k _ -> isremote k) + isremote k = (startswith "remote." k) && (endswith ".url" k) + construct r = + if (isURI r) + then gitRepoFromUrl r + else gitRepoFromPath r {- Finds the current git repository, which may be in a parent directory. -} gitRepoCurrent :: IO GitRepo From c8002bd91b03b66c195014ecaa9111c50fa5e716 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:00:29 -0400 Subject: [PATCH 0051/8313] update --- GitRepo.hs | 4 +++- TODO | 34 +++++++++++++++++++++++++++++++++- git-annex.mdwn | 24 +++++++++++++++--------- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index fb3ddbaf85..27fc0632c6 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -31,7 +31,9 @@ import Network.URI import Maybe import Utility -{- A git repository can be local or remote. -} +{- A git repository can be on local disk or remote. Not to be confused + - with a git repo's configured remotes, some of which may be on local + - disk. -} data GitRepo = LocalGitRepo { top :: FilePath, diff --git a/TODO b/TODO index ea930f8034..b08784ec24 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,39 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? -* query remotes for their annex.name settings +* query remotes for their annex.name settings, or figure out a different + solution to nameing problem? + + - querying network remotes all the time will be slow. local caching in + .git/config? + - having a git annex name and a git remote name that are distinct + will be confusing + - but git remote names are repo-local, I want a global name + - really, I don't want a name at all, I want a per-repo UUID + + So, each repo has a UUID, stored in annex.uuid. + + And also, the last seen UUID for each remote is listed: + + remote.origin.annex-uuid=d3d2474c-d5c3-11df-80a9-002170d25c55 + + Then when it need to find a repo by UUID, it can see if a known remote + has it -- and then query the remote to confirm the repo there still has + that UUID (a different repo may have been mounted there). + + Failing that, it can force a refresh of all uuids, updating .git/config, + and check again. + + - Only downside for this is that if I put a repo on a usb disk, + and it is disconnected and I have no remote for it, + git-annex will have to say: + + "You asked for a file that is only present on a git repo with + UUID d3d2474c-d5c3-11df-80a9-002170d25c55 -- and I don't know + where it is." + + To solve that, let .git-annex/uuid.map map between uuids and descriptions, + like "d3d2474c-d5c3-11df-80a9-002170d25c55 SATA drive labeled '* arch-2'" * hook up LocationLog * --push/--pull/--get/--want/--drop diff --git a/git-annex.mdwn b/git-annex.mdwn index 6bfdd57c7f..1348886f2b 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -79,7 +79,7 @@ git-annex information that should be propigated between repositories. Data is stored here in files that are arranged to avoid conflicts in most cases. A conflict could occur if a file with the same name but different -content was added to multiple clones. +content was added to multiple repositories. ## key/value storage @@ -117,26 +117,32 @@ you indicate you --want a file, git-annex will tell you which repositories have the file's content. Location tracking information is stored in `.git-annex/$filename.log`. -Repositories record their name and the date when they --get or --drop +Repositories record their UUID and the date when they --get or --drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) +The optional file `.git-annex/uuid.map` can be created to add a description +to a UUID. If git-annex needs a file from a repository and it cannot find +the repository amoung the remotes, it will use the description from this +file when asking for the repository to be made available. The file format +is a UUID, a space, and the rest of the line is its description. For +example: + + UUID d3d2474c-d5c3-11df-80a9-002170d25c55 USB drive in red enclosure + ## configuration -* `annex.numcopies` -- number of copies of files to keep +* `annex.uuid` -- a unique UUID for this repository +* `annex.numcopies` -- number of copies of files to keep (default: 1) * `annex.backends` -- space-separated list of names of the key/value backends to use. The first listed is used to store - new files. -* `annex.name` -- allows specifying a unique name for this repository. - If not specified, the name is derived from its directory's location and - the hostname. When a repository is on removable media it is useful to give - it a more stable name. Typically the name of a repository is the same - name configured as a git remote to allow pulling from that repository. + new files. (default: file, checksum, url) * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 50. Note that other factors may be configured when pushing files to repositories, in particular, whether the repository is on a filesystem with sufficient free space. +* `remote..annex-uuid` -- git-annex caches UUIDs of remotes here ## issues From 97b31a525e31abf1db95154d09c7efa368d3f59c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:05:33 -0400 Subject: [PATCH 0052/8313] foo --- git-annex.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/git-annex.mdwn b/git-annex.mdwn index 1348886f2b..9dd2d44ef7 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -129,6 +129,7 @@ is a UUID, a space, and the rest of the line is its description. For example: UUID d3d2474c-d5c3-11df-80a9-002170d25c55 USB drive in red enclosure + UUID 60cf39c8-d5c6-11df-aa8b-93fda39008d6 my colocated server ## configuration From 8f069bd2875022cfceb0c50cb9a5667a9bae88d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:40:09 -0400 Subject: [PATCH 0053/8313] tweak --- Annex.hs | 2 +- GitRepo.hs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Annex.hs b/Annex.hs index 5a7274ca15..3461a92cde 100644 --- a/Annex.hs +++ b/Annex.hs @@ -31,7 +31,7 @@ annexDir repo key = gitDir repo ++ "/annex/" ++ key - later. -} startAnnex :: IO State startAnnex = do - r <- gitRepoCurrent + r <- gitRepoFromCwd gitPrep r return State { diff --git a/GitRepo.hs b/GitRepo.hs index 27fc0632c6..643b725e67 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -7,7 +7,7 @@ module GitRepo ( GitRepo, - gitRepoCurrent, + gitRepoFromCwd, gitRepoFromPath, gitRepoFromUrl, gitWorkTree, @@ -183,8 +183,8 @@ gitConfigRemotes repo = mapM construct remotes else gitRepoFromPath r {- Finds the current git repository, which may be in a parent directory. -} -gitRepoCurrent :: IO GitRepo -gitRepoCurrent = do +gitRepoFromCwd :: IO GitRepo +gitRepoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of From b430f55b80e0c4efba352817d8eecded586d0726 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 02:51:44 -0400 Subject: [PATCH 0054/8313] tweak --- GitRepo.hs | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 643b725e67..241dd4009b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -15,6 +15,7 @@ module GitRepo ( gitRelative, gitConfig, gitAdd, + gitRm, gitAttributes ) where @@ -79,7 +80,8 @@ remote repo = not $ local repo assertlocal repo action = if (local repo) then action - else error "acting on remote git repo not supported" + else error $ "acting on remote git repo " ++ (url repo) ++ + " not supported" {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> String @@ -116,44 +118,38 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Stages a changed file in git's index. -} +{- Stages a changed/new file in git's index. -} gitAdd :: GitRepo -> FilePath -> IO () gitAdd repo file = runGit repo ["add", file] +{- Removes a file. -} +gitRm :: GitRepo -> FilePath -> IO () +gitRm repo file = runGit repo ["rm", file] + {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: GitRepo -> [String] -> [String] -gitCommandLine repo params = +gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - if (local repo) - then ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params - else error "gitCommandLine not implemented for remote repo" + ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () -runGit repo params = - if (local repo) - then do - r <- executeFile "git" True (gitCommandLine repo params) Nothing - return () - else error "runGit not implemented for remote repo" +runGit repo params = assertlocal repo $ do + r <- executeFile "git" True (gitCommandLine repo params) Nothing + return () {- Runs a git subcommand and returns its output. -} gitPipeRead :: GitRepo -> [String] -> IO String -gitPipeRead repo params = - if (local repo) - then pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret - else error "gitPipeRead not implemented for remote repo" +gitPipeRead repo params = assertlocal repo $ do + pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do + ret <- hGetContentsStrict h + return ret {- Runs git config and populates a repo with its settings. -} gitConfigRead :: GitRepo -> IO GitRepo -gitConfigRead repo = - if (local repo) - then do - c <- gitPipeRead repo ["config", "--list"] - return repo { config = gitConfigParse c } - else error "gitConfigRead not implemented for remote repo" +gitConfigRead repo = assertlocal repo $ do + c <- gitPipeRead repo ["config", "--list"] + return repo { config = gitConfigParse c } {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String From 10b7c405fa427b5657d2336974a7e0a19ed098ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:23:34 -0400 Subject: [PATCH 0055/8313] better git repo querying and bare repo detection --- GitRepo.hs | 67 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 241dd4009b..21b37519bd 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -38,7 +38,6 @@ import Utility data GitRepo = LocalGitRepo { top :: FilePath, - bare :: Bool, config :: Map String String } | RemoteGitRepo { url :: String, @@ -46,24 +45,20 @@ data GitRepo = config :: Map String String } deriving (Show, Read, Eq) -{- Local GitRepo constructor. -} -gitRepoFromPath :: FilePath -> IO GitRepo -gitRepoFromPath dir = do - b <- isBareRepo dir - +{- Local GitRepo constructor. Can optionally query the repo for its config. -} +gitRepoFromPath :: FilePath -> Bool -> IO GitRepo +gitRepoFromPath dir query = do let r = LocalGitRepo { top = dir, - bare = b, config = Map.empty } - r' <- gitConfigRead r + if (query) + then gitConfigRead r + else return r - return r' - -{- Remote GitRepo constructor. Note that remote repo config is not read. - - Throws exception on invalid url. -} -gitRepoFromUrl :: String -> IO GitRepo -gitRepoFromUrl url = do +{- Remote GitRepo constructor. Throws exception on invalid url. -} +gitRepoFromUrl :: String -> Bool -> IO GitRepo +gitRepoFromUrl url query = do return $ RemoteGitRepo { url = url, top = path url, @@ -71,8 +66,11 @@ gitRepoFromUrl url = do } where path url = uriPath $ fromJust $ parseURI url -{- Some code needs to vary between remote and local repos, these functions - - help with that. -} +{- User-visible description of a git repo by path or url -} +describe repo = if (local repo) then top repo else url repo + +{- Some code needs to vary between remote and local repos, or bare and + - non-bare, these functions help with that. -} local repo = case (repo) of LocalGitRepo {} -> True RemoteGitRepo {} -> False @@ -80,8 +78,16 @@ remote repo = not $ local repo assertlocal repo action = if (local repo) then action - else error $ "acting on remote git repo " ++ (url repo) ++ + else error $ "acting on remote git repo " ++ (describe repo) ++ " not supported" +bare :: GitRepo -> Bool +bare repo = + if (member b (config repo)) + then ("true" == fromJust (Map.lookup b (config repo))) + else error $ "it is not known if git repo " ++ (describe repo) ++ + " is a bare repository; config not read" + where + b = "core.bare" {- Path to a repository's gitattributes file. -} gitAttributes :: GitRepo -> String @@ -130,7 +136,11 @@ gitRm repo file = runGit repo ["rm", file] gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + -- gitDir cannot be used for --git-dir because the config may + -- not have been read (and gitConfigRead relies on this function). + -- So this relies on git doing the right thing when told that + -- --git-dir is the top of a work tree. + ["--git-dir="++(top repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () @@ -175,8 +185,8 @@ gitConfigRemotes repo = mapM construct remotes isremote k = (startswith "remote." k) && (endswith ".url" k) construct r = if (isURI r) - then gitRepoFromUrl r - else gitRepoFromPath r + then gitRepoFromUrl r False + else gitRepoFromPath r False {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo @@ -184,7 +194,7 @@ gitRepoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> gitRepoFromPath dir + (Just dir) -> gitRepoFromPath dir True Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) @@ -200,11 +210,10 @@ isRepoTop dir = do r <- isGitRepo dir b <- isBareRepo dir return (r || b) - -isGitRepo dir = gitSignature dir ".git" ".git/config" -isBareRepo dir = gitSignature dir "objects" "config" - -gitSignature dir subdir file = do - s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) - f <- (doesFileExist (dir ++ "/" ++ file)) - return (s && f) + where + isGitRepo dir = gitSignature dir ".git" ".git/config" + isBareRepo dir = gitSignature dir "objects" "config" + gitSignature dir subdir file = do + s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) + f <- (doesFileExist (dir ++ "/" ++ file)) + return (s && f) From 8a3ea4edcbf1cc7059d5382ca84d0033cd9152c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:31:19 -0400 Subject: [PATCH 0056/8313] typo --- GitRepo.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 21b37519bd..f3c959bec4 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -174,7 +174,7 @@ gitConfigParse s = Map.fromList $ map pair $ lines s {- Returns a single git config setting, or a default value if not set. -} gitConfig :: GitRepo -> String -> String -> String gitConfig repo key defaultValue = - Map.findWithDefault key defaultValue (config repo) + Map.findWithDefault defaultValue key (config repo) {- Returns a list of a repo's configured remotes. -} gitConfigRemotes :: GitRepo -> IO [GitRepo] From e4bc7e599a799d758c4d948dce65a7fa05dd50cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:35:05 -0400 Subject: [PATCH 0057/8313] revert bad change I was wrong about git-config's level of smarts --- GitRepo.hs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f3c959bec4..5657803119 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -136,11 +136,7 @@ gitRm repo file = runGit repo ["rm", file] gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - -- gitDir cannot be used for --git-dir because the config may - -- not have been read (and gitConfigRead relies on this function). - -- So this relies on git doing the right thing when told that - -- --git-dir is the top of a work tree. - ["--git-dir="++(top repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} runGit :: GitRepo -> [String] -> IO () From 31b24348d25f5aec7ff521b7452fab6833a1d051 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 12:47:11 -0400 Subject: [PATCH 0058/8313] new git config read method --- GitRepo.hs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 5657803119..c87bd355e0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -22,9 +22,11 @@ module GitRepo ( import Directory import System import System.Directory +import System.Posix.Directory import System.Path import System.Cmd.Utils import System.IO +import IO (bracket_) import System.Posix.Process import Data.String.Utils import Data.Map as Map hiding (map, split) @@ -151,11 +153,17 @@ gitPipeRead repo params = assertlocal repo $ do ret <- hGetContentsStrict h return ret -{- Runs git config and populates a repo with its settings. -} +{- Runs git config and populates a repo with its config. -} gitConfigRead :: GitRepo -> IO GitRepo gitConfigRead repo = assertlocal repo $ do - c <- gitPipeRead repo ["config", "--list"] - return repo { config = gitConfigParse c } + {- Cannot use gitPipeRead because it relies on the config having + been already read. Instead, chdir to the repo. -} + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory (top repo)) + (\_ -> changeWorkingDirectory cwd) $ do + pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do + val <- hGetContentsStrict h + return repo { config = gitConfigParse val } {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String From ea5d7fe07a5c40349e66848fc9cd06a9f748b724 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 13:02:41 -0400 Subject: [PATCH 0059/8313] add uuid --- .gitattributes | 1 + Annex.hs | 15 +++++++++++++++ GitRepo.hs | 9 +++++---- 3 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..b98b07d7d2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git-annex/*.log merge=union diff --git a/Annex.hs b/Annex.hs index 3461a92cde..5adc739964 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,6 +10,8 @@ module Annex ( import System.Posix.Files import System.Directory +import System.Cmd.Utils +import System.IO import GitRepo import Utility import Locations @@ -84,6 +86,13 @@ unannexFile state file = do {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do + -- Make sure that the repo has an annex.uuid setting. + if ("" == gitConfig repo "annex.uuid" "") + then do + uuid <- genUUID + gitRun repo ["config", "annex.uuid", uuid] + else return () + -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" let attributes = gitAttributes repo @@ -99,3 +108,9 @@ gitPrep repo = do appendFile attributes $ attrLine ++ "\n" gitAdd repo attributes else return () + +{- Generates a UUID. There is a library for this, but it's not packaged, + - so use the command line tool. -} +genUUID :: IO String +genUUID = do + pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h diff --git a/GitRepo.hs b/GitRepo.hs index c87bd355e0..b166e32814 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -16,6 +16,7 @@ module GitRepo ( gitConfig, gitAdd, gitRm, + gitRun, gitAttributes ) where @@ -128,11 +129,11 @@ gitRelative repo file = drop (length absrepo) absfile {- Stages a changed/new file in git's index. -} gitAdd :: GitRepo -> FilePath -> IO () -gitAdd repo file = runGit repo ["add", file] +gitAdd repo file = gitRun repo ["add", file] {- Removes a file. -} gitRm :: GitRepo -> FilePath -> IO () -gitRm repo file = runGit repo ["rm", file] +gitRm repo file = gitRun repo ["rm", file] {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: GitRepo -> [String] -> [String] @@ -141,8 +142,8 @@ gitCommandLine repo params = assertlocal repo $ ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} -runGit :: GitRepo -> [String] -> IO () -runGit repo params = assertlocal repo $ do +gitRun :: GitRepo -> [String] -> IO () +gitRun repo params = assertlocal repo $ do r <- executeFile "git" True (gitCommandLine repo params) Nothing return () From dc1d5e68317b85043c8c30a82f53f78b0a9a9f51 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 13:10:07 -0400 Subject: [PATCH 0060/8313] update --- Annex.hs | 16 ++-------------- TODO | 34 +--------------------------------- UUID.hs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 47 deletions(-) create mode 100644 UUID.hs diff --git a/Annex.hs b/Annex.hs index 5adc739964..31897479d4 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,13 +10,12 @@ module Annex ( import System.Posix.Files import System.Directory -import System.Cmd.Utils -import System.IO import GitRepo import Utility import Locations import Backend import BackendList +import UUID import LocationLog -- git-annex's runtime state @@ -86,12 +85,7 @@ unannexFile state file = do {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do - -- Make sure that the repo has an annex.uuid setting. - if ("" == gitConfig repo "annex.uuid" "") - then do - uuid <- genUUID - gitRun repo ["config", "annex.uuid", uuid] - else return () + prepUUID repo -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" @@ -108,9 +102,3 @@ gitPrep repo = do appendFile attributes $ attrLine ++ "\n" gitAdd repo attributes else return () - -{- Generates a UUID. There is a library for this, but it's not packaged, - - so use the command line tool. -} -genUUID :: IO String -genUUID = do - pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h diff --git a/TODO b/TODO index b08784ec24..c951eb3f1e 100644 --- a/TODO +++ b/TODO @@ -1,39 +1,7 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? -* query remotes for their annex.name settings, or figure out a different - solution to nameing problem? - - - querying network remotes all the time will be slow. local caching in - .git/config? - - having a git annex name and a git remote name that are distinct - will be confusing - - but git remote names are repo-local, I want a global name - - really, I don't want a name at all, I want a per-repo UUID - - So, each repo has a UUID, stored in annex.uuid. - - And also, the last seen UUID for each remote is listed: - - remote.origin.annex-uuid=d3d2474c-d5c3-11df-80a9-002170d25c55 - - Then when it need to find a repo by UUID, it can see if a known remote - has it -- and then query the remote to confirm the repo there still has - that UUID (a different repo may have been mounted there). - - Failing that, it can force a refresh of all uuids, updating .git/config, - and check again. - - - Only downside for this is that if I put a repo on a usb disk, - and it is disconnected and I have no remote for it, - git-annex will have to say: - - "You asked for a file that is only present on a git repo with - UUID d3d2474c-d5c3-11df-80a9-002170d25c55 -- and I don't know - where it is." - - To solve that, let .git-annex/uuid.map map between uuids and descriptions, - like "d3d2474c-d5c3-11df-80a9-002170d25c55 SATA drive labeled '* arch-2'" +* query remotes for their annex.uuid settings * hook up LocationLog * --push/--pull/--get/--want/--drop diff --git a/UUID.hs b/UUID.hs new file mode 100644 index 0000000000..a0e0784821 --- /dev/null +++ b/UUID.hs @@ -0,0 +1,36 @@ +{- git-annex uuids + - + - Each git repository used by git-annex has an annex.uuid setting that + - uniquely identifies that repository. + - + -} + +module UUID ( + getUUID, + prepUUID, + genUUID +) where + +import System.Cmd.Utils +import System.IO +import GitRepo + +configkey="annex.uuid" + +{- Generates a UUID. There is a library for this, but it's not packaged, + - so use the command line tool. -} +genUUID :: IO String +genUUID = do + pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h + +getUUID :: GitRepo -> String +getUUID repo = gitConfig repo "annex.uuid" "" + +{- Make sure that the repo has an annex.uuid setting. -} +prepUUID :: GitRepo -> IO () +prepUUID repo = + if ("" == getUUID repo) + then do + uuid <- genUUID + gitRun repo ["config", configkey, uuid] + else return () From 4fbdb197d524720d1ea77795b33cb5d24152bce9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 13:12:47 -0400 Subject: [PATCH 0061/8313] correctness --- Annex.hs | 9 ++++----- GitRepo.hs | 1 + UUID.hs | 6 ++++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Annex.hs b/Annex.hs index 31897479d4..ed3e4e33ad 100644 --- a/Annex.hs +++ b/Annex.hs @@ -33,11 +33,12 @@ annexDir repo key = gitDir repo ++ "/annex/" ++ key startAnnex :: IO State startAnnex = do r <- gitRepoFromCwd - gitPrep r + r' <- prepUUID r + gitPrep r' return State { - repo = r, - backends = parseBackendList $ gitConfig r "annex.backends" "" + repo = r', + backends = parseBackendList $ gitConfig r' "annex.backends" "" } {- Annexes a file, storing it in a backend, and then moving it into @@ -85,8 +86,6 @@ unannexFile state file = do {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do - prepUUID repo - -- configure git to use union merge driver on state files let attrLine = stateLoc ++ "/*.log merge=union" let attributes = gitAttributes repo diff --git a/GitRepo.hs b/GitRepo.hs index b166e32814..9a919128e6 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -14,6 +14,7 @@ module GitRepo ( gitDir, gitRelative, gitConfig, + gitConfigRead, gitAdd, gitRm, gitRun, diff --git a/UUID.hs b/UUID.hs index a0e0784821..0b795b8c10 100644 --- a/UUID.hs +++ b/UUID.hs @@ -23,14 +23,16 @@ genUUID :: IO String genUUID = do pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h +{- Looks up a repo's UUID -} getUUID :: GitRepo -> String getUUID repo = gitConfig repo "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} -prepUUID :: GitRepo -> IO () +prepUUID :: GitRepo -> IO GitRepo prepUUID repo = if ("" == getUUID repo) then do uuid <- genUUID gitRun repo ["config", configkey, uuid] - else return () + gitConfigRead repo -- return new repo with updated config + else return repo From e53900a54597437818d785aa6b1683b3b49d2afb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 15:44:54 -0400 Subject: [PATCH 0062/8313] stub --- Annex.hs | 33 ++++++++++++++++++++++++++++++++- CmdLine.hs | 12 ++++++++---- UUID.hs | 3 ++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/Annex.hs b/Annex.hs index ed3e4e33ad..af25697198 100644 --- a/Annex.hs +++ b/Annex.hs @@ -5,7 +5,12 @@ module Annex ( State, startAnnex, annexFile, - unannexFile + unannexFile, + annexGetFile, + annexWantFile, + annexDropFile, + annexPushRepo, + annexPullRepo ) where import System.Posix.Files @@ -83,6 +88,32 @@ unannexFile state file = do renameFile src file return () +{- Transfers the file from a remote. -} +annexGetFile :: State -> FilePath -> IO () +annexGetFile state file = do + alreadyannexed <- lookupBackend (backends state) (repo state) file + case (alreadyannexed) of + Nothing -> error $ "not annexed " ++ file + Just _ -> do error "not implemented" -- TODO + -- 1. find remote with file + -- 2. copy file from remote + +{- Indicates a file is wanted. -} +annexWantFile :: State -> FilePath -> IO () +annexWantFile state file = do error "not implemented" -- TODO + +{- Indicates a file is now wanted. -} +annexDropFile :: State -> FilePath -> IO () +annexDropFile state file = do error "not implemented" -- TODO + +{- Pushes all files to a remote repository. -} +annexPushRepo :: State -> String -> IO () +annexPushRepo state reponame = do error "not implemented" -- TODO + +{- Pulls all files from a remote repository. -} +annexPullRepo :: State -> String -> IO () +annexPullRepo state reponame = do error "not implemented" -- TODO + {- Sets up a git repo for git-annex. May be called repeatedly. -} gitPrep :: GitRepo -> IO () gitPrep repo = do diff --git a/CmdLine.hs b/CmdLine.hs index cc87088898..60ba81d30b 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -39,8 +39,12 @@ argvToMode argv = do where header = "Usage: git-annex [mode] file" dispatch :: State -> Mode -> FilePath -> IO () -dispatch state mode file = do +dispatch state mode item = do case (mode) of - Add -> annexFile state file - Unannex -> unannexFile state file - _ -> error "not implemented" + Add -> annexFile state item + Push -> annexPushRepo state item + Pull -> annexPullRepo state item + Want -> annexWantFile state item + Get -> annexGetFile state item + Drop -> annexDropFile state item + Unannex -> unannexFile state item diff --git a/UUID.hs b/UUID.hs index 0b795b8c10..40c2330ee9 100644 --- a/UUID.hs +++ b/UUID.hs @@ -34,5 +34,6 @@ prepUUID repo = then do uuid <- genUUID gitRun repo ["config", configkey, uuid] - gitConfigRead repo -- return new repo with updated config + -- return new repo with updated config + gitConfigRead repo else return repo From d257bad93c4ae6f8e6ef6a9c848e63d0f46eb225 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 15:48:00 -0400 Subject: [PATCH 0063/8313] uuid type --- Annex.hs | 2 +- UUID.hs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Annex.hs b/Annex.hs index af25697198..f695985e1f 100644 --- a/Annex.hs +++ b/Annex.hs @@ -102,7 +102,7 @@ annexGetFile state file = do annexWantFile :: State -> FilePath -> IO () annexWantFile state file = do error "not implemented" -- TODO -{- Indicates a file is now wanted. -} +{- Indicates a file is not wanted. -} annexDropFile :: State -> FilePath -> IO () annexDropFile state file = do error "not implemented" -- TODO diff --git a/UUID.hs b/UUID.hs index 40c2330ee9..4364e20700 100644 --- a/UUID.hs +++ b/UUID.hs @@ -15,16 +15,18 @@ import System.Cmd.Utils import System.IO import GitRepo +type UUID = String + configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} -genUUID :: IO String +genUUID :: IO UUID genUUID = do pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h {- Looks up a repo's UUID -} -getUUID :: GitRepo -> String +getUUID :: GitRepo -> UUID getUUID repo = gitConfig repo "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} From 759f146d0fd5857cbbb796367c3dd8c695550b46 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 15:52:18 -0400 Subject: [PATCH 0064/8313] update --- Annex.hs | 7 +------ Types.hs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 Types.hs diff --git a/Annex.hs b/Annex.hs index f695985e1f..830f619197 100644 --- a/Annex.hs +++ b/Annex.hs @@ -22,12 +22,7 @@ import Backend import BackendList import UUID import LocationLog - --- git-annex's runtime state -data State = State { - repo :: GitRepo, - backends :: [Backend] -} +import Types {- An annexed file's content is stored somewhere under .git/annex/ -} annexDir :: GitRepo -> Key -> FilePath diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..df95880274 --- /dev/null +++ b/Types.hs @@ -0,0 +1,14 @@ +{- git-annex core data types -} + +module Types ( + State(..) +) where + +import BackendType +import GitRepo + +-- git-annex's runtime state +data State = State { + repo :: GitRepo, + backends :: [Backend] +} From 2ac47a3a59b2b9b8980b4a9d3277bcb653bcb026 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:06:10 -0400 Subject: [PATCH 0065/8313] thread State thru to backends --- Annex.hs | 10 ++++----- Backend.hs | 55 +++++++++++++++++++++++----------------------- BackendChecksum.hs | 4 ++-- BackendFile.hs | 10 ++++----- BackendList.hs | 2 +- BackendType.hs | 31 -------------------------- BackendUrl.hs | 7 +++--- Types.hs | 25 +++++++++++++++++++-- 8 files changed, 67 insertions(+), 77 deletions(-) delete mode 100644 BackendType.hs diff --git a/Annex.hs b/Annex.hs index 830f619197..78d990eacb 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,12 +45,12 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend (backends state) state file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (backends state) (repo state) file + stored <- storeFile (backends state) state file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -70,11 +70,11 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend (backends state) state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile (backends state) (repo state) file + mkey <- dropFile (backends state) state file case (mkey) of Nothing -> return () Just key -> do @@ -86,7 +86,7 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - alreadyannexed <- lookupBackend (backends state) (repo state) file + alreadyannexed <- lookupBackend (backends state) state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do error "not implemented" -- TODO diff --git a/Backend.hs b/Backend.hs index 2d3ea42d64..ddfd8b19d9 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,74 +28,75 @@ import System.Directory import Locations import GitRepo import Utility -import BackendType +import Types {- Name of state file that holds the key for an annexed file, - using a given backend. -} -backendFile :: Backend -> GitRepo -> FilePath -> String -backendFile backend repo file = gitStateDir repo ++ - (gitRelative repo file) ++ "." ++ (name backend) +backendFile :: Backend -> State -> FilePath -> String +backendFile backend state file = + gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + "." ++ (name backend) {- Attempts to store a file in one of the backends, and returns - its key. -} -storeFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) +storeFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) storeFile [] _ _ = return Nothing -storeFile (b:bs) repo file = do - try <- (getKey b) repo (gitRelative repo file) +storeFile (b:bs) state file = do + try <- (getKey b) state (gitRelative (repo state) file) case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) repo file key + stored <- (storeFileKey b) state file key if (not stored) then nextbackend else do bookkeeping key return $ Just key where - nextbackend = storeFile bs repo file - backendfile = backendFile b repo file + nextbackend = storeFile bs state file + backendfile = backendFile b state file bookkeeping key = do createDirectoryIfMissing True (parentDir backendfile) writeFile backendfile key {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} -retrieveFile :: [Backend] -> GitRepo -> FilePath -> FilePath -> IO Bool -retrieveFile backends repo file dest = do - result <- lookupBackend backends repo file +retrieveFile :: [Backend] -> State -> FilePath -> FilePath -> IO Bool +retrieveFile backends state file dest = do + result <- lookupBackend backends state file case (result) of Nothing -> return False Just b -> do - key <- lookupKey b repo file + key <- lookupKey b state file (retrieveKeyFile b) key dest {- Drops the key for a file from the backend that has it. -} -dropFile :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Key) -dropFile backends repo file = do - result <- lookupBackend backends repo file +dropFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) +dropFile backends state file = do + result <- lookupBackend backends state file case (result) of Nothing -> return Nothing Just b -> do - key <- lookupKey b repo file + key <- lookupKey b state file (removeKey b) key - removeFile $ backendFile b repo file + removeFile $ backendFile b state file return $ Just key {- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: Backend -> GitRepo -> FilePath -> IO Key -lookupKey backend repo file = readFile (backendFile backend repo file) +lookupKey :: Backend -> State -> FilePath -> IO Key +lookupKey backend state file = readFile (backendFile backend state file) {- Looks up the backend used for an already annexed file. -} -lookupBackend :: [Backend] -> GitRepo -> FilePath -> IO (Maybe Backend) +lookupBackend :: [Backend] -> State -> FilePath -> IO (Maybe Backend) lookupBackend [] _ _ = return Nothing -lookupBackend (b:bs) repo file = do - present <- checkBackend b repo file +lookupBackend (b:bs) state file = do + present <- checkBackend b state file if present then return $ Just b else - lookupBackend bs repo file + lookupBackend bs state file {- Checks if a file is available via a given backend. -} -checkBackend :: Backend -> GitRepo -> FilePath -> IO (Bool) -checkBackend backend repo file = doesFileExist $ backendFile backend repo file +checkBackend :: Backend -> State -> FilePath -> IO (Bool) +checkBackend backend state file = doesFileExist $ backendFile backend state file diff --git a/BackendChecksum.hs b/BackendChecksum.hs index e80dbe793c..72b4744e31 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -5,7 +5,7 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA -import BackendType +import Types import GitRepo -- based on BackendFile just with a different key type @@ -15,5 +15,5 @@ backend = BackendFile.backend { } -- checksum the file to get its key -keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue :: State -> FilePath -> IO (Maybe Key) keyValue k = error "checksum keyValue unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index ae53f460f1..33c2985bca 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,7 +3,7 @@ module BackendFile (backend) where -import BackendType +import Types import GitRepo backend = Backend { @@ -15,15 +15,15 @@ backend = Backend { } -- direct mapping from filename to key -keyValue :: GitRepo -> FilePath -> IO (Maybe Key) -keyValue repo file = return $ Just file +keyValue :: State -> FilePath -> IO (Maybe Key) +keyValue state file = return $ Just file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing or removing a key is - a no-op. -} -dummyStore :: GitRepo -> FilePath -> Key -> IO (Bool) -dummyStore repo file key = return True +dummyStore :: State -> FilePath -> Key -> IO (Bool) +dummyStore state file key = return True dummyRemove :: Key -> IO Bool dummyRemove url = return False diff --git a/BackendList.hs b/BackendList.hs index f733a44be9..104444dc20 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -7,7 +7,7 @@ module BackendList ( lookupBackendName ) where -import BackendType +import Types -- When adding a new backend, import it here and add it to the list. import qualified BackendFile diff --git a/BackendType.hs b/BackendType.hs deleted file mode 100644 index 3bc822f329..0000000000 --- a/BackendType.hs +++ /dev/null @@ -1,31 +0,0 @@ -{- git-annex backend data types - - -} - -module BackendType ( - -- the entire types are exported, for use in backend implementations - Key(..), - Backend(..) -) where - -import GitRepo - --- annexed filenames are mapped into keys -type Key = FilePath - --- this structure represents a key/value backend -data Backend = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: GitRepo -> FilePath -> IO (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: GitRepo -> FilePath -> Key -> IO Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, - -- removes a key - removeKey :: Key -> IO Bool -} - -instance Show Backend where - show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" - diff --git a/BackendUrl.hs b/BackendUrl.hs index 4ba1dbadb5..aad6477443 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,8 +3,7 @@ module BackendUrl (backend) where -import BackendType -import GitRepo +import Types backend = Backend { name = "url", @@ -15,11 +14,11 @@ backend = Backend { } -- cannot generate url from filename -keyValue :: GitRepo -> FilePath -> IO (Maybe Key) +keyValue :: State -> FilePath -> IO (Maybe Key) keyValue repo file = return Nothing -- cannot change urls -dummyStore :: GitRepo -> FilePath -> Key -> IO Bool +dummyStore :: State -> FilePath -> Key -> IO Bool dummyStore repo file url = return False dummyRemove :: Key -> IO Bool dummyRemove url = return False diff --git a/Types.hs b/Types.hs index df95880274..de6bff9ff6 100644 --- a/Types.hs +++ b/Types.hs @@ -1,14 +1,35 @@ {- git-annex core data types -} module Types ( - State(..) + State(..), + Key(..), + Backend(..) ) where -import BackendType import GitRepo -- git-annex's runtime state data State = State { repo :: GitRepo, backends :: [Backend] +} deriving (Show) + +-- annexed filenames are mapped into keys +type Key = FilePath + +-- this structure represents a key/value backend +data Backend = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: State -> FilePath -> IO (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: State -> FilePath -> Key -> IO Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> IO Bool, + -- removes a key + removeKey :: Key -> IO Bool } + +instance Show Backend where + show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" From 20acda0423b1a00eae64296835679887ca79ea2f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:10:15 -0400 Subject: [PATCH 0066/8313] more state --- Backend.hs | 4 ++-- BackendChecksum.hs | 1 - BackendFile.hs | 9 ++++----- BackendUrl.hs | 8 ++++---- Types.hs | 4 ++-- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Backend.hs b/Backend.hs index ddfd8b19d9..622d558e31 100644 --- a/Backend.hs +++ b/Backend.hs @@ -68,7 +68,7 @@ retrieveFile backends state file dest = do Nothing -> return False Just b -> do key <- lookupKey b state file - (retrieveKeyFile b) key dest + (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} dropFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) @@ -78,7 +78,7 @@ dropFile backends state file = do Nothing -> return Nothing Just b -> do key <- lookupKey b state file - (removeKey b) key + (removeKey b) state key removeFile $ backendFile b state file return $ Just key diff --git a/BackendChecksum.hs b/BackendChecksum.hs index 72b4744e31..efa2244125 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -6,7 +6,6 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA import Types -import GitRepo -- based on BackendFile just with a different key type backend = BackendFile.backend { diff --git a/BackendFile.hs b/BackendFile.hs index 33c2985bca..c59cbcbaae 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -4,7 +4,6 @@ module BackendFile (backend) where import Types -import GitRepo backend = Backend { name = "file", @@ -24,10 +23,10 @@ keyValue state file = return $ Just file - a no-op. -} dummyStore :: State -> FilePath -> Key -> IO (Bool) dummyStore state file key = return True -dummyRemove :: Key -> IO Bool -dummyRemove url = return False +dummyRemove :: State -> Key -> IO Bool +dummyRemove state url = return False {- Try to find a copy of the file in one of the other repos, - and copy it over to this one. -} -copyFromOtherRepo :: Key -> FilePath -> IO (Bool) -copyFromOtherRepo key file = error "copyFromOtherRepo unimplemented" -- TODO +copyFromOtherRepo :: State -> Key -> FilePath -> IO (Bool) +copyFromOtherRepo state key file = error "copyFromOtherRepo unimplemented" -- TODO diff --git a/BackendUrl.hs b/BackendUrl.hs index aad6477443..71503c5c13 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -20,8 +20,8 @@ keyValue repo file = return Nothing -- cannot change urls dummyStore :: State -> FilePath -> Key -> IO Bool dummyStore repo file url = return False -dummyRemove :: Key -> IO Bool -dummyRemove url = return False +dummyRemove :: State -> Key -> IO Bool +dummyRemove state url = return False -downloadUrl :: Key -> FilePath -> IO Bool -downloadUrl url file = error "downloadUrl unimplemented" +downloadUrl :: State -> Key -> FilePath -> IO Bool +downloadUrl state url file = error "downloadUrl unimplemented" diff --git a/Types.hs b/Types.hs index de6bff9ff6..26ba2a9043 100644 --- a/Types.hs +++ b/Types.hs @@ -26,9 +26,9 @@ data Backend = Backend { -- stores a file's contents to a key storeFileKey :: State -> FilePath -> Key -> IO Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> IO Bool, + retrieveKeyFile :: State -> Key -> FilePath -> IO Bool, -- removes a key - removeKey :: Key -> IO Bool + removeKey :: State -> Key -> IO Bool } instance Show Backend where From 603e01e96ce8d76f4b689b4503c3f4528c39957f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:20:41 -0400 Subject: [PATCH 0067/8313] simplify some function signatures using state --- Annex.hs | 11 +++++------ Backend.hs | 44 ++++++++++++++++++++++---------------------- CmdLine.hs | 1 + 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Annex.hs b/Annex.hs index 78d990eacb..1752cabffc 100644 --- a/Annex.hs +++ b/Annex.hs @@ -2,7 +2,6 @@ -} module Annex ( - State, startAnnex, annexFile, unannexFile, @@ -45,12 +44,12 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend (backends state) state file + alreadyannexed <- lookupBackend state file case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do checkLegal file - stored <- storeFile (backends state) state file + stored <- storeFile state file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just key -> symlink key @@ -70,11 +69,11 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend (backends state) state file + alreadyannexed <- lookupBackend state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile (backends state) state file + mkey <- dropFile state file case (mkey) of Nothing -> return () Just key -> do @@ -86,7 +85,7 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - alreadyannexed <- lookupBackend (backends state) state file + alreadyannexed <- lookupBackend state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do error "not implemented" -- TODO diff --git a/Backend.hs b/Backend.hs index 622d558e31..2e5275ba68 100644 --- a/Backend.hs +++ b/Backend.hs @@ -17,8 +17,6 @@ - -} module Backend ( - Key, - Backend, -- note only data type is exported, not destructors lookupBackend, storeFile, dropFile @@ -32,16 +30,17 @@ import Types {- Name of state file that holds the key for an annexed file, - using a given backend. -} -backendFile :: Backend -> State -> FilePath -> String -backendFile backend state file = +backendFile :: State -> Backend -> FilePath -> String +backendFile state backend file = gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ "." ++ (name backend) {- Attempts to store a file in one of the backends, and returns - its key. -} -storeFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) -storeFile [] _ _ = return Nothing -storeFile (b:bs) state file = do +storeFile :: State -> FilePath -> IO (Maybe Key) +storeFile state file = storeFile' (backends state) state file +storeFile' [] _ _ = return Nothing +storeFile' (b:bs) state file = do try <- (getKey b) state (gitRelative (repo state) file) case (try) of Nothing -> nextbackend @@ -53,17 +52,17 @@ storeFile (b:bs) state file = do bookkeeping key return $ Just key where - nextbackend = storeFile bs state file - backendfile = backendFile b state file + nextbackend = storeFile' bs state file + backendfile = backendFile state b file bookkeeping key = do createDirectoryIfMissing True (parentDir backendfile) writeFile backendfile key {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} -retrieveFile :: [Backend] -> State -> FilePath -> FilePath -> IO Bool -retrieveFile backends state file dest = do - result <- lookupBackend backends state file +retrieveFile :: State -> FilePath -> FilePath -> IO Bool +retrieveFile state file dest = do + result <- lookupBackend state file case (result) of Nothing -> return False Just b -> do @@ -71,32 +70,33 @@ retrieveFile backends state file dest = do (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} -dropFile :: [Backend] -> State -> FilePath -> IO (Maybe Key) -dropFile backends state file = do - result <- lookupBackend backends state file +dropFile :: State -> FilePath -> IO (Maybe Key) +dropFile state file = do + result <- lookupBackend state file case (result) of Nothing -> return Nothing Just b -> do key <- lookupKey b state file (removeKey b) state key - removeFile $ backendFile b state file + removeFile $ backendFile state b file return $ Just key {- Looks up the key a backend uses for an already annexed file. -} lookupKey :: Backend -> State -> FilePath -> IO Key -lookupKey backend state file = readFile (backendFile backend state file) +lookupKey backend state file = readFile (backendFile state backend file) {- Looks up the backend used for an already annexed file. -} -lookupBackend :: [Backend] -> State -> FilePath -> IO (Maybe Backend) -lookupBackend [] _ _ = return Nothing -lookupBackend (b:bs) state file = do +lookupBackend :: State -> FilePath -> IO (Maybe Backend) +lookupBackend state file = lookupBackend' (backends state) state file +lookupBackend' [] _ _ = return Nothing +lookupBackend' (b:bs) state file = do present <- checkBackend b state file if present then return $ Just b else - lookupBackend bs state file + lookupBackend' bs state file {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) -checkBackend backend state file = doesFileExist $ backendFile backend state file +checkBackend backend state file = doesFileExist $ backendFile state backend file diff --git a/CmdLine.hs b/CmdLine.hs index 60ba81d30b..9da2b6493c 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,6 +10,7 @@ module CmdLine ( ) where import System.Console.GetOpt +import Types import Annex data Mode = Add | Push | Pull | Want | Get | Drop | Unannex From 570899ed0c16121705ad5db1cb7aa96181a306a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:39:10 -0400 Subject: [PATCH 0068/8313] handle newlines on keys --- Backend.hs | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Backend.hs b/Backend.hs index 2e5275ba68..fab82a0935 100644 --- a/Backend.hs +++ b/Backend.hs @@ -23,18 +23,12 @@ module Backend ( ) where import System.Directory +import Data.String.Utils import Locations import GitRepo import Utility import Types -{- Name of state file that holds the key for an annexed file, - - using a given backend. -} -backendFile :: State -> Backend -> FilePath -> String -backendFile state backend file = - gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ - "." ++ (name backend) - {- Attempts to store a file in one of the backends, and returns - its key. -} storeFile :: State -> FilePath -> IO (Maybe Key) @@ -49,14 +43,10 @@ storeFile' (b:bs) state file = do if (not stored) then nextbackend else do - bookkeeping key + recordKey state b file key return $ Just key where nextbackend = storeFile' bs state file - backendfile = backendFile state b file - bookkeeping key = do - createDirectoryIfMissing True (parentDir backendfile) - writeFile backendfile key {- Attempts to retrieve an file from one of the backends, saving it to - a specified location. -} @@ -81,10 +71,6 @@ dropFile state file = do removeFile $ backendFile state b file return $ Just key -{- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: Backend -> State -> FilePath -> IO Key -lookupKey backend state file = readFile (backendFile state backend file) - {- Looks up the backend used for an already annexed file. -} lookupBackend :: State -> FilePath -> IO (Maybe Backend) lookupBackend state file = lookupBackend' (backends state) state file @@ -97,6 +83,31 @@ lookupBackend' (b:bs) state file = do else lookupBackend' bs state file +{- Name of state file that holds the key for an annexed file, + - using a given backend. -} +backendFile :: State -> Backend -> FilePath -> String +backendFile state backend file = + gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + "." ++ (name backend) + {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) checkBackend backend state file = doesFileExist $ backendFile state backend file + +{- Looks up the key a backend uses for an already annexed file. -} +lookupKey :: Backend -> State -> FilePath -> IO Key +lookupKey backend state file = do + k <- readFile (backendFile state backend file) + return $ chomp k + where + chomp s = if (endswith s "\n") + then (reverse . (drop 1) . reverse) s + else s + +{- Records the key a backend uses for an annexed file. -} +recordKey :: State -> Backend -> FilePath -> Key -> IO () +recordKey state backend file key = do + createDirectoryIfMissing True (parentDir record) + writeFile record $ key ++ "\n" + where + record = backendFile state backend file From 921313bcc706f054c33c3eb923c47955710cd0a3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:40:17 -0400 Subject: [PATCH 0069/8313] consistency --- Backend.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Backend.hs b/Backend.hs index fab82a0935..f03f098cf1 100644 --- a/Backend.hs +++ b/Backend.hs @@ -56,7 +56,7 @@ retrieveFile state file dest = do case (result) of Nothing -> return False Just b -> do - key <- lookupKey b state file + key <- lookupKey state b file (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} @@ -66,7 +66,7 @@ dropFile state file = do case (result) of Nothing -> return Nothing Just b -> do - key <- lookupKey b state file + key <- lookupKey state b file (removeKey b) state key removeFile $ backendFile state b file return $ Just key @@ -95,8 +95,8 @@ checkBackend :: Backend -> State -> FilePath -> IO (Bool) checkBackend backend state file = doesFileExist $ backendFile state backend file {- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: Backend -> State -> FilePath -> IO Key -lookupKey backend state file = do +lookupKey :: State -> Backend -> FilePath -> IO Key +lookupKey state backend file = do k <- readFile (backendFile state backend file) return $ chomp k where From cad916d92695c7c04d3cdacbcd333a2dcd109d53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 16:52:01 -0400 Subject: [PATCH 0070/8313] hookup annexgetfile --- Annex.hs | 19 ++++++++++++------- Backend.hs | 2 ++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Annex.hs b/Annex.hs index 1752cabffc..7ac9932f1d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,8 +24,8 @@ import LocationLog import Types {- An annexed file's content is stored somewhere under .git/annex/ -} -annexDir :: GitRepo -> Key -> FilePath -annexDir repo key = gitDir repo ++ "/annex/" ++ key +annexLocation :: GitRepo -> Key -> FilePath +annexLocation repo key = gitDir repo ++ "/annex/" ++ key {- On startup, examine the git repo, prepare it, and record state for - later. -} @@ -55,7 +55,7 @@ annexFile state file = do Just key -> symlink key where symlink key = do - let dest = annexDir (repo state) key + let dest = annexLocation (repo state) key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file @@ -77,7 +77,7 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just key -> do - let src = annexDir (repo state) key + let src = annexLocation (repo state) key removeFile file renameFile src file return () @@ -88,9 +88,14 @@ annexGetFile state file = do alreadyannexed <- lookupBackend state file case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file - Just _ -> do error "not implemented" -- TODO - -- 1. find remote with file - -- 2. copy file from remote + Just backend -> do + key <- lookupKey state backend file + let dest = annexLocation (repo state) key + createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile state file dest + if (success) + then return () + else error $ "failed to get " ++ file {- Indicates a file is wanted. -} annexWantFile :: State -> FilePath -> IO () diff --git a/Backend.hs b/Backend.hs index f03f098cf1..9d1b0cdbee 100644 --- a/Backend.hs +++ b/Backend.hs @@ -19,6 +19,8 @@ module Backend ( lookupBackend, storeFile, + retrieveFile, + lookupKey, dropFile ) where From a36c39ad0af168259948a360087d2ff05df2857e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:26:34 -0400 Subject: [PATCH 0071/8313] getting files via http working! --- Annex.hs | 10 ++++++++-- Backend.hs | 2 +- BackendUrl.hs | 11 +++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Annex.hs b/Annex.hs index 7ac9932f1d..68379cf203 100644 --- a/Annex.hs +++ b/Annex.hs @@ -14,6 +14,7 @@ module Annex ( import System.Posix.Files import System.Directory +import Data.String.Utils import GitRepo import Utility import Locations @@ -23,9 +24,14 @@ import UUID import LocationLog import Types -{- An annexed file's content is stored somewhere under .git/annex/ -} +{- An annexed file's content is stored somewhere under .git/annex/, + - based on the key. Since the symlink is user-visible, the filename + - used should be as close to the key as possible, in case the key is a + - filename or url. Just escape "/" in the key name, to keep a flat + - tree of files and avoid issues with files ending with "/" etc. -} annexLocation :: GitRepo -> Key -> FilePath -annexLocation repo key = gitDir repo ++ "/annex/" ++ key +annexLocation repo key = gitDir repo ++ "/annex/" ++ (transform key) + where transform s = replace "/" "%" $ replace "%" "%%" s {- On startup, examine the git repo, prepare it, and record state for - later. -} diff --git a/Backend.hs b/Backend.hs index 9d1b0cdbee..a16dfab6a3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -102,7 +102,7 @@ lookupKey state backend file = do k <- readFile (backendFile state backend file) return $ chomp k where - chomp s = if (endswith s "\n") + chomp s = if (endswith "\n" s) then (reverse . (drop 1) . reverse) s else s diff --git a/BackendUrl.hs b/BackendUrl.hs index 71503c5c13..ca44a5c37b 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,6 +3,8 @@ module BackendUrl (backend) where +import System.Posix.Process +import IO import Types backend = Backend { @@ -17,11 +19,16 @@ backend = Backend { keyValue :: State -> FilePath -> IO (Maybe Key) keyValue repo file = return Nothing --- cannot change urls +-- cannot change url contents dummyStore :: State -> FilePath -> Key -> IO Bool dummyStore repo file url = return False dummyRemove :: State -> Key -> IO Bool dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool -downloadUrl state url file = error "downloadUrl unimplemented" +downloadUrl state url file = do + putStrLn $ "download: " ++ url + result <- try $ executeFile "curl" True ["-o", file, url] Nothing + case (result) of + Left _ -> return False + Right _ -> return True From 0561ea1b2873962199aca5ba6529254aa5b2632b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:43:54 -0400 Subject: [PATCH 0072/8313] oops, wrong system --- BackendUrl.hs | 4 ++-- GitRepo.hs | 4 ++-- TODO | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/BackendUrl.hs b/BackendUrl.hs index ca44a5c37b..f18c800e96 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,7 +3,7 @@ module BackendUrl (backend) where -import System.Posix.Process +import System.Cmd import IO import Types @@ -28,7 +28,7 @@ dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool downloadUrl state url file = do putStrLn $ "download: " ++ url - result <- try $ executeFile "curl" True ["-o", file, url] Nothing + result <- try $ rawSystem "curl" ["-o", file, url] case (result) of Left _ -> return False Right _ -> return True diff --git a/GitRepo.hs b/GitRepo.hs index 9a919128e6..068b2569c0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -26,10 +26,10 @@ import System import System.Directory import System.Posix.Directory import System.Path +import System.Cmd import System.Cmd.Utils import System.IO import IO (bracket_) -import System.Posix.Process import Data.String.Utils import Data.Map as Map hiding (map, split) import Network.URI @@ -145,7 +145,7 @@ gitCommandLine repo params = assertlocal repo $ {- Runs git in the specified repo. -} gitRun :: GitRepo -> [String] -> IO () gitRun repo params = assertlocal repo $ do - r <- executeFile "git" True (gitCommandLine repo params) Nothing + r <- rawSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} diff --git a/TODO b/TODO index c951eb3f1e..ad0389f850 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,7 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? +* if curl fails to download, git-annex crashes and does not complete + further actions.. exception seems to somehow not get caught * query remotes for their annex.uuid settings From 3d2b44ffe58ddc2f235f71cb548ba4a43b6fe641 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:51:41 -0400 Subject: [PATCH 0073/8313] better progress --- BackendUrl.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BackendUrl.hs b/BackendUrl.hs index f18c800e96..3f0846885d 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -28,7 +28,7 @@ dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool downloadUrl state url file = do putStrLn $ "download: " ++ url - result <- try $ rawSystem "curl" ["-o", file, url] + result <- try $ rawSystem "curl" ["-#", "-o", file, url] case (result) of Left _ -> return False Right _ -> return True From 10992b90c97e8c6abfd26da3d6cb50011b4230b6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:56:29 -0400 Subject: [PATCH 0074/8313] avoid redownload --- Annex.hs | 26 +++++++++++++++++--------- TODO | 2 -- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Annex.hs b/Annex.hs index 68379cf203..725009fd2c 100644 --- a/Annex.hs +++ b/Annex.hs @@ -29,10 +29,14 @@ import Types - used should be as close to the key as possible, in case the key is a - filename or url. Just escape "/" in the key name, to keep a flat - tree of files and avoid issues with files ending with "/" etc. -} -annexLocation :: GitRepo -> Key -> FilePath -annexLocation repo key = gitDir repo ++ "/annex/" ++ (transform key) +annexLocation :: State -> Key -> FilePath +annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (transform key) where transform s = replace "/" "%" $ replace "%" "%%" s +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: State -> Key -> IO Bool +inAnnex state key = doesFileExist $ annexLocation state key + {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State @@ -61,7 +65,7 @@ annexFile state file = do Just key -> symlink key where symlink key = do - let dest = annexLocation (repo state) key + let dest = annexLocation state key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file @@ -83,7 +87,7 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just key -> do - let src = annexLocation (repo state) key + let src = annexLocation state key removeFile file renameFile src file return () @@ -96,12 +100,16 @@ annexGetFile state file = do Nothing -> error $ "not annexed " ++ file Just backend -> do key <- lookupKey state backend file - let dest = annexLocation (repo state) key - createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state file dest - if (success) + inannex <- inAnnex state key + if (inannex) then return () - else error $ "failed to get " ++ file + else do + let dest = annexLocation state key + createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile state file dest + if (success) + then return () + else error $ "failed to get " ++ file {- Indicates a file is wanted. -} annexWantFile :: State -> FilePath -> IO () diff --git a/TODO b/TODO index ad0389f850..c951eb3f1e 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,5 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? -* if curl fails to download, git-annex crashes and does not complete - further actions.. exception seems to somehow not get caught * query remotes for their annex.uuid settings From f1eb4fef99aa553899256f3542e5bf30523e3512 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 17:57:23 -0400 Subject: [PATCH 0075/8313] todo --- TODO | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index c951eb3f1e..36ec65c2f5 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,11 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? +* implement retrieval for backendfile + * query remotes for their annex.uuid settings * hook up LocationLog -* --push/--pull/--get/--want/--drop +* --push/--pull/--want/--drop * finish BackendUrl and BackendChecksum From b882fe8410f33b2c8b170e6a60b55d156e336d47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 18:06:34 -0400 Subject: [PATCH 0076/8313] locationlog will use uuids --- BackendFile.hs | 7 ++++++- LocationLog.hs | 25 +++++++++++++------------ UUID.hs | 1 + 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index c59cbcbaae..43ca2191ce 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -29,4 +29,9 @@ dummyRemove state url = return False {- Try to find a copy of the file in one of the other repos, - and copy it over to this one. -} copyFromOtherRepo :: State -> Key -> FilePath -> IO (Bool) -copyFromOtherRepo state key file = error "copyFromOtherRepo unimplemented" -- TODO +copyFromOtherRepo state key file = + -- 1. get ordered list of remotes (local repos, then remote repos) + -- 2. read locationlog for file + -- 3. filter remotes list to ones that have file + -- 4. attempt to transfer from each remote until success + error "copyFromOtherRepo unimplemented" -- TODO diff --git a/LocationLog.hs b/LocationLog.hs index 31d454f103..2cd84db1f7 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -6,10 +6,10 @@ - repositories have the file's content. - - Location tracking information is stored in `.git-annex/filename.log`. - - Repositories record their name and the date when they --get or --drop + - Repositories record their UUID and the date when they --get or --drop - a file's content. - - - A line of the log will look like: "date N reponame" + - A line of the log will look like: "date N UUID" - Where N=1 when the repo has the file, and 0 otherwise. - - Git is configured to use a union merge for this file, @@ -28,12 +28,13 @@ import System.Directory import Data.Char import GitRepo import Utility +import UUID import Locations data LogLine = LogLine { date :: POSIXTime, status :: LogStatus, - reponame :: String + uuid :: UUID } deriving (Eq) data LogStatus = FilePresent | FileMissing | Undefined @@ -50,8 +51,8 @@ instance Read LogStatus where readsPrec _ _ = [(Undefined, "")] instance Show LogLine where - show (LogLine date status reponame) = unwords - [(show date), (show status), reponame] + show (LogLine date status uuid) = unwords + [(show date), (show status), uuid] instance Read LogLine where -- This parser is robust in that even unparsable log lines are @@ -67,10 +68,10 @@ instance Read LogLine where w = words string date = w !! 0 status = read $ w !! 1 - reponame = unwords $ drop 2 w + uuid = unwords $ drop 2 w pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime - good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status reponame + good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] @@ -106,9 +107,9 @@ writeLog file lines = do {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> String -> IO LogLine -logNow status reponame = do +logNow status uuid = do now <- getPOSIXTime - return $ LogLine now status reponame + return $ LogLine now status uuid {- Returns the filename of the log file for a given annexed file. -} logFile :: GitRepo -> FilePath -> IO String @@ -122,7 +123,7 @@ fileLocations :: GitRepo -> FilePath -> IO [String] fileLocations thisrepo file = do log <- logFile thisrepo file lines <- readLog log - return $ map reponame (filterPresent lines) + return $ map uuid (filterPresent lines) {- Filters the list of LogLines to find ones where the file - is (or should still be) present. -} @@ -140,9 +141,9 @@ compactLog' map (l:ls) = compactLog' (mapLog map l) ls - information about a repo than the other logs in the map -} mapLog map log = if (better) - then Map.insert (reponame log) log map + then Map.insert (uuid log) log map else map where - better = case (Map.lookup (reponame log) map) of + better = case (Map.lookup (uuid log) map) of Just l -> (date l <= date log) Nothing -> True diff --git a/UUID.hs b/UUID.hs index 4364e20700..e2b624d69c 100644 --- a/UUID.hs +++ b/UUID.hs @@ -6,6 +6,7 @@ -} module UUID ( + UUID, getUUID, prepUUID, genUUID From 3b89924f53cd88c0b5c21767dfd03b65d9d32f09 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 18:25:41 -0400 Subject: [PATCH 0077/8313] record annexed files in log --- Annex.hs | 1 + LocationLog.hs | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Annex.hs b/Annex.hs index 725009fd2c..29cd7b0fda 100644 --- a/Annex.hs +++ b/Annex.hs @@ -68,6 +68,7 @@ annexFile state file = do let dest = annexLocation state key createDirectoryIfMissing True (parentDir dest) renameFile file dest + logChange (repo state) file (getUUID (repo state)) FilePresent createSymbolicLink dest file gitAdd (repo state) file checkLegal file = do diff --git a/LocationLog.hs b/LocationLog.hs index 2cd84db1f7..d3dd07a4e2 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -17,6 +17,8 @@ -} module LocationLog ( + LogStatus(..), + logChange ) where import Data.Time.Clock.POSIX @@ -75,6 +77,22 @@ instance Read LogLine where undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] +{- Log a change in the presence of a file in a repository, + - and add the log to git so it will propigate to other repos. -} +logChange :: GitRepo -> FilePath -> UUID -> LogStatus -> IO () +logChange repo file uuid status = do + log <- logNow status uuid + if (status == FilePresent) + -- file added; just append to log + then appendLog logfile log + -- file removed; compact log + else do + ls <- readLog logfile + writeLog logfile (log:ls) + gitAdd repo logfile + where + logfile = logFile repo file + {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> IO [LogLine] @@ -106,23 +124,21 @@ writeLog file lines = do hPutStr h $ unlines $ map show lines {- Generates a new LogLine with the current date. -} -logNow :: LogStatus -> String -> IO LogLine +logNow :: LogStatus -> UUID -> IO LogLine logNow status uuid = do now <- getPOSIXTime return $ LogLine now status uuid {- Returns the filename of the log file for a given annexed file. -} -logFile :: GitRepo -> FilePath -> IO String -logFile repo annexedFile = do - return $ (gitStateDir repo) ++ +logFile :: GitRepo -> FilePath -> String +logFile repo annexedFile = (gitStateDir repo) ++ (gitRelative repo annexedFile) ++ ".log" -{- Returns a list of repositories that, according to the log, have +{- Returns a list of repository UUIDs that, according to the log, have - the content of a file -} -fileLocations :: GitRepo -> FilePath -> IO [String] +fileLocations :: GitRepo -> FilePath -> IO [UUID] fileLocations thisrepo file = do - log <- logFile thisrepo file - lines <- readLog log + lines <- readLog $ logFile thisrepo file return $ map uuid (filterPresent lines) {- Filters the list of LogLines to find ones where the file From 476f66abb99ad2baa18b699c26ac9ee7250eca76 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 18:31:02 -0400 Subject: [PATCH 0078/8313] now that a uuid is used, don't need to rejoin --- LocationLog.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LocationLog.hs b/LocationLog.hs index d3dd07a4e2..da702d6503 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -70,7 +70,7 @@ instance Read LogLine where w = words string date = w !! 0 status = read $ w !! 1 - uuid = unwords $ drop 2 w + uuid = w !! 3 pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid From b7858ada038084c8455cdf9d3598382308dc52b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:04:36 -0400 Subject: [PATCH 0079/8313] bugfixes --- Annex.hs | 51 +++++++++++++++++++++++++--------------- Backend.hs | 20 +++++----------- GitRepo.hs | 10 -------- LocationLog.hs | 64 +++++++++++++++++++++++--------------------------- Locations.hs | 25 +++++++++++++++++++- Types.hs | 1 + 6 files changed, 93 insertions(+), 78 deletions(-) diff --git a/Annex.hs b/Annex.hs index 29cd7b0fda..b8e70e6c8d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,15 +24,6 @@ import UUID import LocationLog import Types -{- An annexed file's content is stored somewhere under .git/annex/, - - based on the key. Since the symlink is user-visible, the filename - - used should be as close to the key as possible, in case the key is a - - filename or url. Just escape "/" in the key name, to keep a flat - - tree of files and avoid issues with files ending with "/" etc. -} -annexLocation :: State -> Key -> FilePath -annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (transform key) - where transform s = replace "/" "%" $ replace "%" "%%" s - {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Key -> IO Bool inAnnex state key = doesFileExist $ annexLocation state key @@ -62,15 +53,18 @@ annexFile state file = do stored <- storeFile state file case (stored) of Nothing -> error $ "no backend could store: " ++ file - Just key -> symlink key + Just (key, backend) -> setup key backend where - symlink key = do + setup key backend = do let dest = annexLocation state key createDirectoryIfMissing True (parentDir dest) renameFile file dest - logChange (repo state) file (getUUID (repo state)) FilePresent createSymbolicLink dest file - gitAdd (repo state) file + gitRun (repo state) ["add", file, bfile] + gitRun (repo state) ["commit", "-m", + ("git-annex annexed " ++ file), file, bfile] + logStatus state key ValuePresent + where bfile = backendFile state backend file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -87,11 +81,17 @@ unannexFile state file = do mkey <- dropFile state file case (mkey) of Nothing -> return () - Just key -> do + Just (key, backend) -> do let src = annexLocation state key removeFile file + gitRun (repo state) ["rm", file, bfile] + gitRun (repo state) ["commit", "-m", + ("git-annex unannexed " ++ file), + file, bfile] renameFile src file + logStatus state key ValueMissing return () + where bfile = backendFile state backend file {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () @@ -109,7 +109,9 @@ annexGetFile state file = do createDirectoryIfMissing True (parentDir dest) success <- retrieveFile state file dest if (success) - then return () + then do + logStatus state key ValuePresent + return () else error $ "failed to get " ++ file {- Indicates a file is wanted. -} @@ -132,17 +134,28 @@ annexPullRepo state reponame = do error "not implemented" -- TODO gitPrep :: GitRepo -> IO () gitPrep repo = do -- configure git to use union merge driver on state files - let attrLine = stateLoc ++ "/*.log merge=union" - let attributes = gitAttributes repo exists <- doesFileExist attributes if (not exists) then do writeFile attributes $ attrLine ++ "\n" - gitAdd repo attributes + commit else do content <- readFile attributes if (all (/= attrLine) (lines content)) then do appendFile attributes $ attrLine ++ "\n" - gitAdd repo attributes + commit else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = gitAttributes repo + commit = do + gitRun repo ["add", attributes] + gitRun repo ["commit", "-m", "git-annex setup", + attributes] + +{- Updates the LocationLog when a key's presence changes. -} +logStatus state key status = do + f <- logChange (repo state) key (getUUID (repo state)) status + gitRun (repo state) ["add", f] + gitRun (repo state) ["commit", "-m", "git-annex log update", f] diff --git a/Backend.hs b/Backend.hs index a16dfab6a3..d7bde241a3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -31,9 +31,8 @@ import GitRepo import Utility import Types -{- Attempts to store a file in one of the backends, and returns - - its key. -} -storeFile :: State -> FilePath -> IO (Maybe Key) +{- Attempts to store a file in one of the backends. -} +storeFile :: State -> FilePath -> IO (Maybe (Key, Backend)) storeFile state file = storeFile' (backends state) state file storeFile' [] _ _ = return Nothing storeFile' (b:bs) state file = do @@ -46,7 +45,7 @@ storeFile' (b:bs) state file = do then nextbackend else do recordKey state b file key - return $ Just key + return $ Just (key, b) where nextbackend = storeFile' bs state file @@ -62,7 +61,7 @@ retrieveFile state file dest = do (retrieveKeyFile b) state key dest {- Drops the key for a file from the backend that has it. -} -dropFile :: State -> FilePath -> IO (Maybe Key) +dropFile :: State -> FilePath -> IO (Maybe (Key, Backend)) dropFile state file = do result <- lookupBackend state file case (result) of @@ -71,7 +70,7 @@ dropFile state file = do key <- lookupKey state b file (removeKey b) state key removeFile $ backendFile state b file - return $ Just key + return $ Just (key, b) {- Looks up the backend used for an already annexed file. -} lookupBackend :: State -> FilePath -> IO (Maybe Backend) @@ -85,13 +84,6 @@ lookupBackend' (b:bs) state file = do else lookupBackend' bs state file -{- Name of state file that holds the key for an annexed file, - - using a given backend. -} -backendFile :: State -> Backend -> FilePath -> String -backendFile state backend file = - gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ - "." ++ (name backend) - {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) checkBackend backend state file = doesFileExist $ backendFile state backend file @@ -106,7 +98,7 @@ lookupKey state backend file = do then (reverse . (drop 1) . reverse) s else s -{- Records the key a backend uses for an annexed file. -} +{- Records the key used for an annexed file. -} recordKey :: State -> Backend -> FilePath -> Key -> IO () recordKey state backend file key = do createDirectoryIfMissing True (parentDir record) diff --git a/GitRepo.hs b/GitRepo.hs index 068b2569c0..fcaae1253f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -15,8 +15,6 @@ module GitRepo ( gitRelative, gitConfig, gitConfigRead, - gitAdd, - gitRm, gitRun, gitAttributes ) where @@ -128,14 +126,6 @@ gitRelative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -{- Stages a changed/new file in git's index. -} -gitAdd :: GitRepo -> FilePath -> IO () -gitAdd repo file = gitRun repo ["add", file] - -{- Removes a file. -} -gitRm :: GitRepo -> FilePath -> IO () -gitRm repo file = gitRun repo ["rm", file] - {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ diff --git a/LocationLog.hs b/LocationLog.hs index da702d6503..2eab4815ed 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -1,13 +1,13 @@ {- git-annex location log - - - git-annex keeps track of on which repository it last saw a file's content. + - git-annex keeps track of on which repository it last saw a value. - This can be useful when using it for archiving with offline storage. - When you indicate you --want a file, git-annex will tell you which - - repositories have the file's content. + - repositories have the value. - - - Location tracking information is stored in `.git-annex/filename.log`. + - Location tracking information is stored in `.git-annex/key.log`. - Repositories record their UUID and the date when they --get or --drop - - a file's content. + - a value. - - A line of the log will look like: "date N UUID" - Where N=1 when the repo has the file, and 0 otherwise. @@ -31,6 +31,7 @@ import Data.Char import GitRepo import Utility import UUID +import Types import Locations data LogLine = LogLine { @@ -39,17 +40,17 @@ data LogLine = LogLine { uuid :: UUID } deriving (Eq) -data LogStatus = FilePresent | FileMissing | Undefined +data LogStatus = ValuePresent | ValueMissing | Undefined deriving (Eq) instance Show LogStatus where - show FilePresent = "1" - show FileMissing = "0" + show ValuePresent = "1" + show ValueMissing = "0" show Undefined = "undefined" instance Read LogStatus where - readsPrec _ "1" = [(FilePresent, "")] - readsPrec _ "0" = [(FileMissing, "")] + readsPrec _ "1" = [(ValuePresent, "")] + readsPrec _ "0" = [(ValueMissing, "")] readsPrec _ _ = [(Undefined, "")] instance Show LogLine where @@ -61,7 +62,7 @@ instance Read LogLine where -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = - if (length w >= 3) + if (length w == 3) then case (pdate) of Just v -> good v Nothing -> undefined @@ -70,28 +71,23 @@ instance Read LogLine where w = words string date = w !! 0 status = read $ w !! 1 - uuid = w !! 3 + uuid = w !! 2 pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] -{- Log a change in the presence of a file in a repository, - - and add the log to git so it will propigate to other repos. -} -logChange :: GitRepo -> FilePath -> UUID -> LogStatus -> IO () -logChange repo file uuid status = do +{- Log a change in the presence of a key's value in a repository, + - and return the log filename. -} +logChange :: GitRepo -> Key -> UUID -> LogStatus -> IO FilePath +logChange repo key uuid status = do log <- logNow status uuid - if (status == FilePresent) - -- file added; just append to log - then appendLog logfile log - -- file removed; compact log - else do - ls <- readLog logfile - writeLog logfile (log:ls) - gitAdd repo logfile + ls <- readLog logfile + writeLog logfile (compactLog $ log:ls) + return logfile where - logfile = logFile repo file + logfile = logFile repo key {- Reads a log file. - Note that the LogLines returned may be in any order. -} @@ -129,22 +125,22 @@ logNow status uuid = do now <- getPOSIXTime return $ LogLine now status uuid -{- Returns the filename of the log file for a given annexed file. -} -logFile :: GitRepo -> FilePath -> String -logFile repo annexedFile = (gitStateDir repo) ++ - (gitRelative repo annexedFile) ++ ".log" +{- Returns the filename of the log file for a given key. -} +logFile :: GitRepo -> Key -> String +logFile repo key = + (gitStateDir repo) ++ (gitRelative repo (keyFile key)) ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - - the content of a file -} -fileLocations :: GitRepo -> FilePath -> IO [UUID] -fileLocations thisrepo file = do - lines <- readLog $ logFile thisrepo file + - the value of a key. -} +keyLocations :: GitRepo -> Key -> IO [UUID] +keyLocations thisrepo key = do + lines <- readLog $ logFile thisrepo key return $ map uuid (filterPresent lines) -{- Filters the list of LogLines to find ones where the file +{- Filters the list of LogLines to find ones where the value - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = filter (\l -> FilePresent == status l) $ compactLog lines +filterPresent lines = filter (\l -> ValuePresent == status l) $ compactLog lines {- Compacts a set of logs, returning a subset that contains the current - status. -} diff --git a/Locations.hs b/Locations.hs index 300f443f7f..59f9df727a 100644 --- a/Locations.hs +++ b/Locations.hs @@ -3,9 +3,14 @@ module Locations ( gitStateDir, - stateLoc + stateLoc, + keyFile, + annexLocation, + backendFile ) where +import Data.String.Utils +import Types import GitRepo {- Long-term, cross-repo state is stored in files inside the .git-annex @@ -13,3 +18,21 @@ import GitRepo stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" + +{- Generates a filename that can be used to record a key somewhere to disk. + - Just escape "/" in the key name, to keep a flat + - tree of files and avoid issues with files ending with "/" etc. -} +keyFile :: Key -> FilePath +keyFile key = replace "/" "%" $ replace "%" "%%" key + +{- An annexed file's content is stored somewhere under .git/annex/, + - based on the key. -} +annexLocation :: State -> Key -> FilePath +annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (keyFile key) + +{- Name of state file that holds the key for an annexed file, + - using a given backend. -} +backendFile :: State -> Backend -> FilePath -> String +backendFile state backend file = + gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + "." ++ (name backend) diff --git a/Types.hs b/Types.hs index 26ba2a9043..73492dfc38 100644 --- a/Types.hs +++ b/Types.hs @@ -6,6 +6,7 @@ module Types ( Backend(..) ) where +import Data.String.Utils import GitRepo -- git-annex's runtime state From 490eb66be40d4e9e6a5e4d89f67610e073e7574f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:20:59 -0400 Subject: [PATCH 0080/8313] update --- Backend.hs | 3 --- Locations.hs | 7 ++++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Backend.hs b/Backend.hs index d7bde241a3..525a52beea 100644 --- a/Backend.hs +++ b/Backend.hs @@ -9,9 +9,6 @@ - This key can later be used to retrieve the file's content (its value). This - key generation must be stable for a given file content, name, and size. - - - The mapping from filename to its key is stored in the .git-annex directory, - - in a file named `$filename.$backend` - - - Multiple pluggable backends are supported, and more than one can be used - to store different files' contents in a given repository. - -} diff --git a/Locations.hs b/Locations.hs index 59f9df727a..925aa39e56 100644 --- a/Locations.hs +++ b/Locations.hs @@ -30,9 +30,10 @@ keyFile key = replace "/" "%" $ replace "%" "%%" key annexLocation :: State -> Key -> FilePath annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (keyFile key) -{- Name of state file that holds the key for an annexed file, - - using a given backend. -} +{- The mapping from filename to its key is stored in the .git-annex directory, + - in a file named `key/$filename.$backend` -} backendFile :: State -> Backend -> FilePath -> String backendFile state backend file = - gitStateDir (repo state) ++ (gitRelative (repo state) file) ++ + gitStateDir (repo state) ++ "key/" ++ + (gitRelative (repo state) file) ++ "." ++ (name backend) From d029c48d0ae5485329b4e8757b45f320bb8bfea9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:26:02 -0400 Subject: [PATCH 0081/8313] docs --- git-annex.mdwn | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 9dd2d44ef7..84030bfcaa 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -36,7 +36,9 @@ Enough broad picture, here's how it actually looks: downloaded. * `git annex --drop $file` indicates that you no longer want the file's content to be available in this repository. -* `git annex --unannex $file` undoes a `git annex --add`. +* `git annex --unannex $file` undoes a `git annex --add`. But use `--drop` + if you're just done with a file; only use `--unannex` if you + accidentially added a file. * `git annex $file` is a shorthand for either --add or --get. If the file is already known, it does --get, otherwise it does --add. @@ -74,12 +76,12 @@ only copies of a file. ## the .git-annex directory -The `.git-annex` directory at the top of the repository, is used to store +The `.git-annex` directory at the top of the repository is used to store git-annex information that should be propigated between repositories. Data is stored here in files that are arranged to avoid conflicts in most cases. A conflict could occur if a file with the same name but different -content was added to multiple repositories. +content was added to different repositories. ## key/value storage @@ -93,7 +95,7 @@ This key can later be used to retrieve the file's content (its value). This key generation must be stable for a given file content, name, and size. The mapping from filename to its key is stored in the .git-annex directory, -in a file named `$filename.$backend` +in a file named `key/$filename.$backend` Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. @@ -116,7 +118,7 @@ This can be useful when using it for archiving with offline storage. When you indicate you --want a file, git-annex will tell you which repositories have the file's content. -Location tracking information is stored in `.git-annex/$filename.log`. +Location tracking information is stored in `.git-annex/$key.log`. Repositories record their UUID and the date when they --get or --drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) @@ -140,10 +142,12 @@ example: new files. (default: file, checksum, url) * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. - The default cost is 50. Note that other factors may be configured - when pushing files to repositories, in particular, whether the repository - is on a filesystem with sufficient free space. -* `remote..annex-uuid` -- git-annex caches UUIDs of remotes here + The default cost is 100 for local repositories, and 200 for remote + repositories. Note that other factors may be configured when pushing + files to repositories, in particular, whether the repository is on + a filesystem with sufficient free space. +* `remote..annex-uuid` -- git-annex caches UUIDs of repositories + here. ## issues From fe612bac0361b42af2cdff590797e929cebbb53d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 12 Oct 2010 20:35:20 -0400 Subject: [PATCH 0082/8313] thought --- git-annex.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/git-annex.mdwn b/git-annex.mdwn index 84030bfcaa..3a8db6dcfb 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -180,3 +180,11 @@ remote, to check local disk space. When git-rm removed a file, it should get dropped too. Of course, it may not be dropped right away, depending on number of copies available. + +### branching + +The use of `.git-annex` to store state means that if a repo has branches +and the user switched between them, git-annex will see different state in +the different branches. Whether that is a bug or a feature may depend on +point of view -- call it Too Be Determined. An alternative would be to +store data directly in the git repo as `pristine-tar` does. From 9926fe5c8a1479f734c0a5b68c7c4e6ddfc2f8cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:21:34 -0400 Subject: [PATCH 0083/8313] bugfix in escaping --- Locations.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index 925aa39e56..c12b9fadce 100644 --- a/Locations.hs +++ b/Locations.hs @@ -19,11 +19,11 @@ stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" -{- Generates a filename that can be used to record a key somewhere to disk. +{- Converts a key into a filename fragment. - Just escape "/" in the key name, to keep a flat - tree of files and avoid issues with files ending with "/" etc. -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "%%" key +keyFile key = replace "/" "&s" $ replace "&" "&a" key {- An annexed file's content is stored somewhere under .git/annex/, - based on the key. -} From 208bba8d3062133733d27a5db521013e3a2ead57 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:42:46 -0400 Subject: [PATCH 0084/8313] got rid of the .git-annex/key.backend files --- Annex.hs | 24 +++++++++++------------- Backend.hs | 46 ++++++++++++++++++---------------------------- Locations.hs | 22 ++++++++-------------- TODO | 7 +++++++ git-annex.mdwn | 18 ++++++++---------- 5 files changed, 52 insertions(+), 65 deletions(-) diff --git a/Annex.hs b/Annex.hs index b8e70e6c8d..82efd543d4 100644 --- a/Annex.hs +++ b/Annex.hs @@ -25,8 +25,8 @@ import LocationLog import Types {- Checks if a given key is currently present in the annexLocation -} -inAnnex :: State -> Key -> IO Bool -inAnnex state key = doesFileExist $ annexLocation state key +inAnnex :: State -> Backend -> Key -> IO Bool +inAnnex state backend key = doesFileExist $ annexLocation state backend key {- On startup, examine the git repo, prepare it, and record state for - later. -} @@ -56,15 +56,14 @@ annexFile state file = do Just (key, backend) -> setup key backend where setup key backend = do - let dest = annexLocation state key + let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink dest file - gitRun (repo state) ["add", file, bfile] + gitRun (repo state) ["add", file] gitRun (repo state) ["commit", "-m", - ("git-annex annexed " ++ file), file, bfile] + ("git-annex annexed " ++ file), file] logStatus state key ValuePresent - where bfile = backendFile state backend file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -82,16 +81,15 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just (key, backend) -> do - let src = annexLocation state key + let src = annexLocation state backend key removeFile file - gitRun (repo state) ["rm", file, bfile] + gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", ("git-annex unannexed " ++ file), - file, bfile] + file] renameFile src file logStatus state key ValueMissing return () - where bfile = backendFile state backend file {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () @@ -100,12 +98,12 @@ annexGetFile state file = do case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just backend -> do - key <- lookupKey state backend file - inannex <- inAnnex state key + key <- fileKey file + inannex <- inAnnex state backend key if (inannex) then return () else do - let dest = annexLocation state key + let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) success <- retrieveFile state file dest if (success) diff --git a/Backend.hs b/Backend.hs index 525a52beea..68d70feec7 100644 --- a/Backend.hs +++ b/Backend.hs @@ -17,12 +17,14 @@ module Backend ( lookupBackend, storeFile, retrieveFile, - lookupKey, + fileKey, dropFile ) where import System.Directory +import System.FilePath import Data.String.Utils +import System.Posix.Files import Locations import GitRepo import Utility @@ -41,7 +43,6 @@ storeFile' (b:bs) state file = do if (not stored) then nextbackend else do - recordKey state b file key return $ Just (key, b) where nextbackend = storeFile' bs state file @@ -53,9 +54,9 @@ retrieveFile state file dest = do result <- lookupBackend state file case (result) of Nothing -> return False - Just b -> do - key <- lookupKey state b file - (retrieveKeyFile b) state key dest + Just backend -> do + key <- fileKey file + (retrieveKeyFile backend) state key dest {- Drops the key for a file from the backend that has it. -} dropFile :: State -> FilePath -> IO (Maybe (Key, Backend)) @@ -63,11 +64,10 @@ dropFile state file = do result <- lookupBackend state file case (result) of Nothing -> return Nothing - Just b -> do - key <- lookupKey state b file - (removeKey b) state key - removeFile $ backendFile state b file - return $ Just (key, b) + Just backend -> do + key <- fileKey file + (removeKey backend) state key + return $ Just (key, backend) {- Looks up the backend used for an already annexed file. -} lookupBackend :: State -> FilePath -> IO (Maybe Backend) @@ -83,22 +83,12 @@ lookupBackend' (b:bs) state file = do {- Checks if a file is available via a given backend. -} checkBackend :: Backend -> State -> FilePath -> IO (Bool) -checkBackend backend state file = doesFileExist $ backendFile state backend file +checkBackend backend state file = + doesFileExist $ annexLocation state backend file -{- Looks up the key a backend uses for an already annexed file. -} -lookupKey :: State -> Backend -> FilePath -> IO Key -lookupKey state backend file = do - k <- readFile (backendFile state backend file) - return $ chomp k - where - chomp s = if (endswith "\n" s) - then (reverse . (drop 1) . reverse) s - else s - -{- Records the key used for an annexed file. -} -recordKey :: State -> Backend -> FilePath -> Key -> IO () -recordKey state backend file key = do - createDirectoryIfMissing True (parentDir record) - writeFile record $ key ++ "\n" - where - record = backendFile state backend file +{- Looks up the key corresponding to an annexed file, + - by examining what the file symlinks to. -} +fileKey :: FilePath -> IO Key +fileKey file = do + l <- readSymbolicLink (file) + return $ takeFileName $ l diff --git a/Locations.hs b/Locations.hs index c12b9fadce..b859fd2f27 100644 --- a/Locations.hs +++ b/Locations.hs @@ -5,8 +5,7 @@ module Locations ( gitStateDir, stateLoc, keyFile, - annexLocation, - backendFile + annexLocation ) where import Data.String.Utils @@ -25,15 +24,10 @@ gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" keyFile :: Key -> FilePath keyFile key = replace "/" "&s" $ replace "&" "&a" key -{- An annexed file's content is stored somewhere under .git/annex/, - - based on the key. -} -annexLocation :: State -> Key -> FilePath -annexLocation state key = gitDir (repo state) ++ "/annex/" ++ (keyFile key) - -{- The mapping from filename to its key is stored in the .git-annex directory, - - in a file named `key/$filename.$backend` -} -backendFile :: State -> Backend -> FilePath -> String -backendFile state backend file = - gitStateDir (repo state) ++ "key/" ++ - (gitRelative (repo state) file) ++ - "." ++ (name backend) +{- An annexed file's content is stored in + - .git/annex// ; this allows deriving the key and backend + - by looking at the symlink to it. -} +annexLocation :: State -> Backend -> Key -> FilePath +annexLocation state backend key = + gitDir (repo state) ++ "/annex/" ++ (name backend) ++ + "/" ++ (keyFile key) diff --git a/TODO b/TODO index 36ec65c2f5..392ec4990e 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,13 @@ * bug when annexing files in a subdir of a git repo * how to handle git mv file? +* if the annexed files were in .git/annex//key, and + files in the repo symlink to that, the .git-annex/key/. + would be redundant, and not needed + + -- no separate merge problem with it + -- want to add an url? `ln -s .git/annex//http:%%kitenet.net%foo myfile` + * implement retrieval for backendfile * query remotes for their annex.uuid settings diff --git a/git-annex.mdwn b/git-annex.mdwn index 3a8db6dcfb..dd0b3bc07a 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -91,11 +91,9 @@ used to store the file contents, and git-annex would then retrieve them as needed and put them in `.git/annex/`. When a file is annexed, a key is generated from its content and/or metadata. -This key can later be used to retrieve the file's content (its value). This -key generation must be stable for a given file content, name, and size. - -The mapping from filename to its key is stored in the .git-annex directory, -in a file named `key/$filename.$backend` +The file checked into git symlinks to the key. This key can later be used +to retrieve the file's content (its value). This key generation must be +stable for a given file content, name, and size. Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. @@ -183,8 +181,8 @@ not be dropped right away, depending on number of copies available. ### branching -The use of `.git-annex` to store state means that if a repo has branches -and the user switched between them, git-annex will see different state in -the different branches. Whether that is a bug or a feature may depend on -point of view -- call it Too Be Determined. An alternative would be to -store data directly in the git repo as `pristine-tar` does. +The use of `.git-annex` to store logs means that if a repo has branches +and the user switched between them, git-annex will see different logs in +the different branches, and so may miss info about what remotes have which +files (though it can re-learn). An alternative would be to +store the log data directly in the git repo as `pristine-tar` does. From 14d7b2ac13318ba513bbab4f08b98434741f0e12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:45:09 -0400 Subject: [PATCH 0085/8313] update --- Annex.hs | 6 +++--- demo.log | 11 ----------- git-annex.mdwn | 4 ---- 3 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 demo.log diff --git a/Annex.hs b/Annex.hs index 82efd543d4..e5eb1894f8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -34,7 +34,7 @@ startAnnex :: IO State startAnnex = do r <- gitRepoFromCwd r' <- prepUUID r - gitPrep r' + gitSetup r' return State { repo = r', @@ -129,8 +129,8 @@ annexPullRepo :: State -> String -> IO () annexPullRepo state reponame = do error "not implemented" -- TODO {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitPrep :: GitRepo -> IO () -gitPrep repo = do +gitSetup :: GitRepo -> IO () +gitSetup repo = do -- configure git to use union merge driver on state files exists <- doesFileExist attributes if (not exists) diff --git a/demo.log b/demo.log deleted file mode 100644 index bdecb7d401..0000000000 --- a/demo.log +++ /dev/null @@ -1,11 +0,0 @@ -1286654242s 1 repo -1286652724s 0 foo -1286656282s 1 foo -1286656282s 0 repo -1286656281s 0 foo -# some garbage, should be ignored -a a a - -a 1 a --1 a a -1286652724.0001s 1 foo diff --git a/git-annex.mdwn b/git-annex.mdwn index dd0b3bc07a..6852ed0080 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -79,10 +79,6 @@ only copies of a file. The `.git-annex` directory at the top of the repository is used to store git-annex information that should be propigated between repositories. -Data is stored here in files that are arranged to avoid conflicts in most -cases. A conflict could occur if a file with the same name but different -content was added to different repositories. - ## key/value storage git-annex uses a key/value abstraction layer to allow files contents to be From 67ae9d7fa109503e4b798e2b7703282b92ce3deb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 00:58:59 -0400 Subject: [PATCH 0086/8313] relative symlink to annexed file --- Annex.hs | 2 +- GitRepo.hs | 10 ++++------ Locations.hs | 9 +++++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Annex.hs b/Annex.hs index e5eb1894f8..1c369be926 100644 --- a/Annex.hs +++ b/Annex.hs @@ -59,7 +59,7 @@ annexFile state file = do let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest - createSymbolicLink dest file + createSymbolicLink (annexLocationRelative state backend key) file gitRun (repo state) ["add", file] gitRun (repo state) ["commit", "-m", ("git-annex annexed " ++ file), file] diff --git a/GitRepo.hs b/GitRepo.hs index fcaae1253f..f0686ff203 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -98,14 +98,12 @@ gitAttributes repo = assertlocal repo $ do then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" -{- Path to a repository's .git directory. - - (For a bare repository, that is the root of the repository.) - - TODO: support GIT_DIR -} +{- Path to a repository's .git directory, relative to its topdir. -} gitDir :: GitRepo -> String gitDir repo = assertlocal repo $ if (bare repo) - then top repo - else top repo ++ "/.git" + then "" + else ".git" {- Path to a repository's --work-tree. -} gitWorkTree :: GitRepo -> FilePath @@ -130,7 +128,7 @@ gitRelative repo file = drop (length absrepo) absfile gitCommandLine :: GitRepo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(gitDir repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(top repo)++"/"++(gitDir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} gitRun :: GitRepo -> [String] -> IO () diff --git a/Locations.hs b/Locations.hs index b859fd2f27..faf29235f6 100644 --- a/Locations.hs +++ b/Locations.hs @@ -5,7 +5,8 @@ module Locations ( gitStateDir, stateLoc, keyFile, - annexLocation + annexLocation, + annexLocationRelative ) where import Data.String.Utils @@ -28,6 +29,10 @@ keyFile key = replace "/" "&s" $ replace "&" "&a" key - .git/annex// ; this allows deriving the key and backend - by looking at the symlink to it. -} annexLocation :: State -> Backend -> Key -> FilePath -annexLocation state backend key = +annexLocation state backend key = + (gitWorkTree $ repo state) ++ "/" ++ + (annexLocationRelative state backend key) +annexLocationRelative :: State -> Backend -> Key -> FilePath +annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) From 16cd682290b065fee59575b077525d20713e4b4b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:04:06 -0400 Subject: [PATCH 0087/8313] better key to file mapping --- Locations.hs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Locations.hs b/Locations.hs index faf29235f6..6aba0ed1a2 100644 --- a/Locations.hs +++ b/Locations.hs @@ -20,10 +20,17 @@ gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" {- Converts a key into a filename fragment. - - Just escape "/" in the key name, to keep a flat - - tree of files and avoid issues with files ending with "/" etc. -} + - + - Escape "/" in the key name, to keep a flat tree of files and avoid + - issues with keys containing "/../" or ending with "/" etc. + - + - "/" is escaped to "%" because it's short and rarely used, and resembles + - a slash + - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping + - is one to one. + - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "&s" $ replace "&" "&a" key +keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" key {- An annexed file's content is stored in - .git/annex// ; this allows deriving the key and backend @@ -32,6 +39,7 @@ annexLocation :: State -> Backend -> Key -> FilePath annexLocation state backend key = (gitWorkTree $ repo state) ++ "/" ++ (annexLocationRelative state backend key) + annexLocationRelative :: State -> Backend -> Key -> FilePath annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ From 3e65384f06400e06f78173d64b13da07c5d024d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:36:20 -0400 Subject: [PATCH 0088/8313] fix relative symlink 2 --- Annex.hs | 28 +++++++++++++++++++--------- Locations.hs | 3 ++- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Annex.hs b/Annex.hs index 1c369be926..1ad9569f97 100644 --- a/Annex.hs +++ b/Annex.hs @@ -55,20 +55,30 @@ annexFile state file = do Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> setup key backend where - setup key backend = do - let dest = annexLocation state backend key - createDirectoryIfMissing True (parentDir dest) - renameFile file dest - createSymbolicLink (annexLocationRelative state backend key) file - gitRun (repo state) ["add", file] - gitRun (repo state) ["commit", "-m", - ("git-annex annexed " ++ file), file] - logStatus state key ValuePresent checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () + setup key backend = do + let dest = annexLocation state backend key + let reldest = annexLocationRelative state backend key + createDirectoryIfMissing True (parentDir dest) + renameFile file dest + createSymbolicLink ((linkTarget file) ++ reldest) file + gitRun (repo state) ["add", file] + gitRun (repo state) ["commit", "-m", + ("git-annex annexed " ++ file), file] + logStatus state key ValuePresent + linkTarget file = + -- relies on file being relative to the top of the + -- git repo; just replace each subdirectory with ".." + if (subdirs > 0) + then (join "/" $ take subdirs $ repeat "..") ++ "/" + else "" + where + subdirs = (length $ split "/" file) - 1 + {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () diff --git a/Locations.hs b/Locations.hs index 6aba0ed1a2..72f4c451fd 100644 --- a/Locations.hs +++ b/Locations.hs @@ -40,7 +40,8 @@ annexLocation state backend key = (gitWorkTree $ repo state) ++ "/" ++ (annexLocationRelative state backend key) +{- Annexed file's location relative to the gitWorkTree -} annexLocationRelative :: State -> Backend -> Key -> FilePath -annexLocationRelative state backend key = +annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) From 3a18b6d2ae95f8b536640f2438437d1d0a99082e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:42:39 -0400 Subject: [PATCH 0089/8313] bugfix --- Annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index 1ad9569f97..dd6912d852 100644 --- a/Annex.hs +++ b/Annex.hs @@ -91,7 +91,7 @@ unannexFile state file = do case (mkey) of Nothing -> return () Just (key, backend) -> do - let src = annexLocation state backend key + let src = annexLocation state backend file removeFile file gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", From 4ecebfb218e58fb85a8e4484af93b5178a8046e7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:43:24 -0400 Subject: [PATCH 0090/8313] bugfx --- Annex.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Annex.hs b/Annex.hs index dd6912d852..63cf0d2fb1 100644 --- a/Annex.hs +++ b/Annex.hs @@ -97,6 +97,9 @@ unannexFile state file = do gitRun (repo state) ["commit", "-m", ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) renameFile src file logStatus state key ValueMissing return () From 77d052af3c527b3ebe349329305d80c9c5a2bf36 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:49:21 -0400 Subject: [PATCH 0091/8313] fix parentDir to work for relative too --- Utility.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index dea53967fc..349dd9355f 100644 --- a/Utility.hs +++ b/Utility.hs @@ -34,7 +34,8 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then "/" ++ (join "/" $ take ((length dirs) - 1) dirs) + then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) else "" where dirs = filter (\x -> length x > 0) $ split "/" dir + absolute = if ((dir !! 0) == '/') then "/" else "" From 3f2ce326fac36cef36a2ab9667d20d921ab91a6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:53:16 -0400 Subject: [PATCH 0092/8313] update --- TODO | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/TODO b/TODO index 392ec4990e..0bba79f102 100644 --- a/TODO +++ b/TODO @@ -1,12 +1,5 @@ -* bug when annexing files in a subdir of a git repo -* how to handle git mv file? - -* if the annexed files were in .git/annex//key, and - files in the repo symlink to that, the .git-annex/key/. - would be redundant, and not needed - - -- no separate merge problem with it - -- want to add an url? `ln -s .git/annex//http:%%kitenet.net%foo myfile` +* bug when annexing files while in a subdir of a git repo +* bug when specifying full path to files when annexing * implement retrieval for backendfile @@ -15,4 +8,6 @@ * hook up LocationLog * --push/--pull/--want/--drop -* finish BackendUrl and BackendChecksum +* how to handle git mv file? + +* finish BackendChecksum From 490a3a828cbb5a4046178b36fc0f9fe0696d0e9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 01:55:44 -0400 Subject: [PATCH 0093/8313] update --- TODO | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO b/TODO index 0bba79f102..a0f7c8b5fa 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,10 @@ * bug when annexing files while in a subdir of a git repo -* bug when specifying full path to files when annexing +* bug when specifying absolute path to files when annexing * implement retrieval for backendfile * query remotes for their annex.uuid settings -* hook up LocationLog * --push/--pull/--want/--drop * how to handle git mv file? From d1071bd1fe879abb3ebb229f9347f7855a697b8c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 02:31:24 -0400 Subject: [PATCH 0094/8313] autobugfixing! Converted Key to a real data type and caught all the places where I used an unconverted filename as a key. Had to loose some sanity checks around whether something is already annexed, but I guess I can add those back other ways. --- Annex.hs | 21 ++++++++++++-------- Backend.hs | 54 ++++++++++++++++++++++++++++---------------------- BackendFile.hs | 5 +++-- BackendUrl.hs | 4 ++-- Locations.hs | 2 +- Types.hs | 6 +++++- 6 files changed, 54 insertions(+), 38 deletions(-) diff --git a/Annex.hs b/Annex.hs index 63cf0d2fb1..8a7b8d860c 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,7 +45,8 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - alreadyannexed <- lookupBackend state file + -- TODO check if already annexed + let alreadyannexed = Nothing case (alreadyannexed) of Just _ -> error $ "already annexed: " ++ file Nothing -> do @@ -83,15 +84,17 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - alreadyannexed <- lookupBackend state file + -- TODO check if already annexed + let alreadyannexed = Just 1 case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file Just _ -> do - mkey <- dropFile state file - case (mkey) of + key <- fileKey file + dropped <- dropFile state key + case (dropped) of Nothing -> return () Just (key, backend) -> do - let src = annexLocation state backend file + let src = annexLocation state backend key removeFile file gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", @@ -107,18 +110,20 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - alreadyannexed <- lookupBackend state file + -- TODO check if already annexed + let alreadyannexed = Just 1 case (alreadyannexed) of Nothing -> error $ "not annexed " ++ file - Just backend -> do + Just _ -> do key <- fileKey file + backend <- fileBackend file inannex <- inAnnex state backend key if (inannex) then return () else do let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state file dest + success <- retrieveFile state key dest if (success) then do logStatus state key ValuePresent diff --git a/Backend.hs b/Backend.hs index 68d70feec7..dbb0064a51 100644 --- a/Backend.hs +++ b/Backend.hs @@ -16,15 +16,17 @@ module Backend ( lookupBackend, storeFile, + dropFile, retrieveFile, fileKey, - dropFile + fileBackend ) where import System.Directory import System.FilePath import Data.String.Utils import System.Posix.Files +import BackendList import Locations import GitRepo import Utility @@ -47,48 +49,52 @@ storeFile' (b:bs) state file = do where nextbackend = storeFile' bs state file -{- Attempts to retrieve an file from one of the backends, saving it to +{- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: State -> FilePath -> FilePath -> IO Bool -retrieveFile state file dest = do - result <- lookupBackend state file +retrieveFile :: State -> Key -> FilePath -> IO Bool +retrieveFile state key dest = do + result <- lookupBackend state key case (result) of Nothing -> return False - Just backend -> do - key <- fileKey file - (retrieveKeyFile backend) state key dest + Just backend -> (retrieveKeyFile backend) state key dest -{- Drops the key for a file from the backend that has it. -} -dropFile :: State -> FilePath -> IO (Maybe (Key, Backend)) -dropFile state file = do - result <- lookupBackend state file +{- Drops a key from the backend that has it. -} +dropFile :: State -> Key -> IO (Maybe (Key, Backend)) +dropFile state key = do + result <- lookupBackend state key case (result) of Nothing -> return Nothing Just backend -> do - key <- fileKey file (removeKey backend) state key return $ Just (key, backend) -{- Looks up the backend used for an already annexed file. -} -lookupBackend :: State -> FilePath -> IO (Maybe Backend) -lookupBackend state file = lookupBackend' (backends state) state file +{- Looks up the backend that has a key. -} +lookupBackend :: State -> Key -> IO (Maybe Backend) +lookupBackend state key = lookupBackend' (backends state) state key lookupBackend' [] _ _ = return Nothing -lookupBackend' (b:bs) state file = do - present <- checkBackend b state file +lookupBackend' (b:bs) state key = do + present <- checkBackend b state key if present then return $ Just b else - lookupBackend' bs state file + lookupBackend' bs state key -{- Checks if a file is available via a given backend. -} -checkBackend :: Backend -> State -> FilePath -> IO (Bool) -checkBackend backend state file = - doesFileExist $ annexLocation state backend file +{- Checks if a key is available via a given backend. -} +checkBackend :: Backend -> State -> Key -> IO (Bool) +checkBackend backend state key = + doesFileExist $ annexLocation state backend key {- Looks up the key corresponding to an annexed file, - by examining what the file symlinks to. -} fileKey :: FilePath -> IO Key fileKey file = do l <- readSymbolicLink (file) - return $ takeFileName $ l + return $ Key $ takeFileName $ l + +{- Looks up the backend corresponding to an annexed file, + - by examining what the file symlinks to. -} +fileBackend :: FilePath -> IO Backend +fileBackend file = do + l <- readSymbolicLink (file) + return $ lookupBackendName $ takeFileName $ parentDir $ l diff --git a/BackendFile.hs b/BackendFile.hs index 43ca2191ce..15b23536bf 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -15,12 +15,13 @@ backend = Backend { -- direct mapping from filename to key keyValue :: State -> FilePath -> IO (Maybe Key) -keyValue state file = return $ Just file +keyValue state file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing or removing a key is - - a no-op. -} + - a no-op. TODO until support is added for git annex --push otherrepo, + - then these could implement that.. -} dummyStore :: State -> FilePath -> Key -> IO (Bool) dummyStore state file key = return True dummyRemove :: State -> Key -> IO Bool diff --git a/BackendUrl.hs b/BackendUrl.hs index 3f0846885d..5b586497c4 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -27,8 +27,8 @@ dummyRemove state url = return False downloadUrl :: State -> Key -> FilePath -> IO Bool downloadUrl state url file = do - putStrLn $ "download: " ++ url - result <- try $ rawSystem "curl" ["-#", "-o", file, url] + putStrLn $ "download: " ++ (show url) + result <- try $ rawSystem "curl" ["-#", "-o", file, (show url)] case (result) of Left _ -> return False Right _ -> return True diff --git a/Locations.hs b/Locations.hs index 72f4c451fd..a99ad6ec40 100644 --- a/Locations.hs +++ b/Locations.hs @@ -30,7 +30,7 @@ gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" - is one to one. - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" key +keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" $ show key {- An annexed file's content is stored in - .git/annex// ; this allows deriving the key and backend diff --git a/Types.hs b/Types.hs index 73492dfc38..9b0bb00fd5 100644 --- a/Types.hs +++ b/Types.hs @@ -16,7 +16,11 @@ data State = State { } deriving (Show) -- annexed filenames are mapped into keys -type Key = FilePath +data Key = Key String deriving (Eq) + +-- show a key to convert it to a string +instance Show Key where + show (Key v) = v -- this structure represents a key/value backend data Backend = Backend { From 4b801b265afa94b1219a1abb6e52e08e0790582a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:20:05 -0400 Subject: [PATCH 0095/8313] error handling --- Annex.hs | 34 ++++++++++++--------------- Backend.hs | 67 ++++++++++++++++-------------------------------------- 2 files changed, 34 insertions(+), 67 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8a7b8d860c..30ec0843a5 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,10 +45,9 @@ startAnnex = do - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () annexFile state file = do - -- TODO check if already annexed - let alreadyannexed = Nothing - case (alreadyannexed) of - Just _ -> error $ "already annexed: " ++ file + r <- lookupFile file + case (r) of + Just _ -> error $ "already annexed " ++ file Nothing -> do checkLegal file stored <- storeFile state file @@ -84,16 +83,14 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = do - -- TODO check if already annexed - let alreadyannexed = Just 1 - case (alreadyannexed) of + r <- lookupFile file + case (r) of Nothing -> error $ "not annexed " ++ file - Just _ -> do - key <- fileKey file - dropped <- dropFile state key - case (dropped) of - Nothing -> return () - Just (key, backend) -> do + Just (key, backend) -> do + dropped <- dropFile state backend key + if (not dropped) + then error $ "backend refused to drop " ++ file + else do let src = annexLocation state backend key removeFile file gitRun (repo state) ["rm", file] @@ -110,20 +107,17 @@ unannexFile state file = do {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = do - -- TODO check if already annexed - let alreadyannexed = Just 1 - case (alreadyannexed) of + r <- lookupFile file + case (r) of Nothing -> error $ "not annexed " ++ file - Just _ -> do - key <- fileKey file - backend <- fileBackend file + Just (key, backend) -> do inannex <- inAnnex state backend key if (inannex) then return () else do let dest = annexLocation state backend key createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state key dest + success <- retrieveFile state backend key dest if (success) then do logStatus state key ValuePresent diff --git a/Backend.hs b/Backend.hs index dbb0064a51..2697f43d4a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -14,14 +14,13 @@ - -} module Backend ( - lookupBackend, storeFile, dropFile, retrieveFile, - fileKey, - fileBackend + lookupFile ) where +import Control.Exception import System.Directory import System.FilePath import Data.String.Utils @@ -51,50 +50,24 @@ storeFile' (b:bs) state file = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: State -> Key -> FilePath -> IO Bool -retrieveFile state key dest = do - result <- lookupBackend state key - case (result) of - Nothing -> return False - Just backend -> (retrieveKeyFile backend) state key dest +retrieveFile :: State -> Backend -> Key -> FilePath -> IO Bool +retrieveFile state backend key dest = (retrieveKeyFile backend) state key dest -{- Drops a key from the backend that has it. -} -dropFile :: State -> Key -> IO (Maybe (Key, Backend)) -dropFile state key = do - result <- lookupBackend state key - case (result) of - Nothing -> return Nothing - Just backend -> do - (removeKey backend) state key - return $ Just (key, backend) +{- Drops a key from a backend. -} +dropFile :: State -> Backend -> Key -> IO Bool +dropFile state backend key = (removeKey backend) state key -{- Looks up the backend that has a key. -} -lookupBackend :: State -> Key -> IO (Maybe Backend) -lookupBackend state key = lookupBackend' (backends state) state key -lookupBackend' [] _ _ = return Nothing -lookupBackend' (b:bs) state key = do - present <- checkBackend b state key - if present - then - return $ Just b - else - lookupBackend' bs state key - -{- Checks if a key is available via a given backend. -} -checkBackend :: Backend -> State -> Key -> IO (Bool) -checkBackend backend state key = - doesFileExist $ annexLocation state backend key - -{- Looks up the key corresponding to an annexed file, +{- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} -fileKey :: FilePath -> IO Key -fileKey file = do - l <- readSymbolicLink (file) - return $ Key $ takeFileName $ l - -{- Looks up the backend corresponding to an annexed file, - - by examining what the file symlinks to. -} -fileBackend :: FilePath -> IO Backend -fileBackend file = do - l <- readSymbolicLink (file) - return $ lookupBackendName $ takeFileName $ parentDir $ l +lookupFile :: FilePath -> IO (Maybe (Key, Backend)) +lookupFile file = do + result <- try (lookup)::IO (Either SomeException (Maybe (Key, Backend))) + case (result) of + Left err -> return Nothing + Right succ -> return succ + where + lookup = do + l <- readSymbolicLink file + return $ Just (k l, b l) + k l = Key $ takeFileName $ l + b l = lookupBackendName $ takeFileName $ parentDir $ l From cc5cf0093ea1aacc4c5460dfdd4d35f2963687bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:30:51 -0400 Subject: [PATCH 0096/8313] cleanup --- Annex.hs | 64 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/Annex.hs b/Annex.hs index 30ec0843a5..7cee3c4cba 100644 --- a/Annex.hs +++ b/Annex.hs @@ -41,20 +41,24 @@ startAnnex = do backends = parseBackendList $ gitConfig r' "annex.backends" "" } +inBackend file yes no = do + r <- lookupFile file + case (r) of + Just v -> yes v + Nothing -> no +notinBackend file yes no = inBackend file no yes + {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexFile :: State -> FilePath -> IO () -annexFile state file = do - r <- lookupFile file - case (r) of - Just _ -> error $ "already annexed " ++ file - Nothing -> do - checkLegal file - stored <- storeFile state file - case (stored) of - Nothing -> error $ "no backend could store: " ++ file - Just (key, backend) -> setup key backend +annexFile state file = inBackend file err $ do + checkLegal file + stored <- storeFile state file + case (stored) of + Nothing -> error $ "no backend could store: " ++ file + Just (key, backend) -> setup key backend where + err = error $ "already annexed " ++ file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -82,27 +86,25 @@ annexFile state file = do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () -unannexFile state file = do - r <- lookupFile file - case (r) of - Nothing -> error $ "not annexed " ++ file - Just (key, backend) -> do - dropped <- dropFile state backend key - if (not dropped) - then error $ "backend refused to drop " ++ file - else do - let src = annexLocation state backend key - removeFile file - gitRun (repo state) ["rm", file] - gitRun (repo state) ["commit", "-m", - ("git-annex unannexed " ++ file), - file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file - logStatus state key ValueMissing - return () +unannexFile state file = notinBackend file err $ \(key, backend) -> do + dropped <- dropFile state backend key + if (not dropped) + then error $ "backend refused to drop " ++ file + else cleanup key backend + where + err = error $ "not annexed " ++ file + cleanup key backend = do + let src = annexLocation state backend key + removeFile file + gitRun (repo state) ["rm", file] + gitRun (repo state) ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + logStatus state key ValueMissing + return () {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () From 99b2029236248f6b4ce68e126b70fa0855fac37f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:41:12 -0400 Subject: [PATCH 0097/8313] key conversion back from file bugfixes --- Annex.hs | 6 ++---- Backend.hs | 2 +- Locations.hs | 31 ++++++++++++++++++------------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index 7cee3c4cba..936e62502d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -87,10 +87,8 @@ annexFile state file = inBackend file err $ do {- Inverse of annexFile. -} unannexFile :: State -> FilePath -> IO () unannexFile state file = notinBackend file err $ \(key, backend) -> do - dropped <- dropFile state backend key - if (not dropped) - then error $ "backend refused to drop " ++ file - else cleanup key backend + dropFile state backend key + cleanup key backend where err = error $ "not annexed " ++ file cleanup key backend = do diff --git a/Backend.hs b/Backend.hs index 2697f43d4a..bc7eb206ff 100644 --- a/Backend.hs +++ b/Backend.hs @@ -69,5 +69,5 @@ lookupFile file = do lookup = do l <- readSymbolicLink file return $ Just (k l, b l) - k l = Key $ takeFileName $ l + k l = fileKey $ takeFileName $ l b l = lookupBackendName $ takeFileName $ parentDir $ l diff --git a/Locations.hs b/Locations.hs index a99ad6ec40..304ca060e9 100644 --- a/Locations.hs +++ b/Locations.hs @@ -5,6 +5,7 @@ module Locations ( gitStateDir, stateLoc, keyFile, + fileKey, annexLocation, annexLocationRelative ) where @@ -19,19 +20,6 @@ stateLoc = ".git-annex" gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" -{- Converts a key into a filename fragment. - - - - Escape "/" in the key name, to keep a flat tree of files and avoid - - issues with keys containing "/../" or ending with "/" etc. - - - - "/" is escaped to "%" because it's short and rarely used, and resembles - - a slash - - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping - - is one to one. - - -} -keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "%s" $ replace "&" "&a" $ show key - {- An annexed file's content is stored in - .git/annex// ; this allows deriving the key and backend - by looking at the symlink to it. -} @@ -45,3 +33,20 @@ annexLocationRelative :: State -> Backend -> Key -> FilePath annexLocationRelative state backend key = gitDir (repo state) ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) + +{- Converts a key into a filename fragment. + - + - Escape "/" in the key name, to keep a flat tree of files and avoid + - issues with keys containing "/../" or ending with "/" etc. + - + - "/" is escaped to "%" because it's short and rarely used, and resembles + - a slash + - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping + - is one to one. + - -} +keyFile :: Key -> FilePath +keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key + +{- Reverses keyFile -} +fileKey :: FilePath -> Key +fileKey file = Key $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From ff998a9a677ba4a4af9e4bd45a651653421760cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:46:40 -0400 Subject: [PATCH 0098/8313] cleanup2 --- Annex.hs | 58 ++++++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/Annex.hs b/Annex.hs index 936e62502d..eb57e2d57a 100644 --- a/Annex.hs +++ b/Annex.hs @@ -65,6 +65,7 @@ annexFile state file = inBackend file err $ do then error $ "not a regular file: " ++ file else return () setup key backend = do + logStatus state key ValuePresent let dest = annexLocation state backend key let reldest = annexLocationRelative state backend key createDirectoryIfMissing True (parentDir dest) @@ -73,7 +74,6 @@ annexFile state file = inBackend file err $ do gitRun (repo state) ["add", file] gitRun (repo state) ["commit", "-m", ("git-annex annexed " ++ file), file] - logStatus state key ValuePresent linkTarget file = -- relies on file being relative to the top of the -- git repo; just replace each subdirectory with ".." @@ -88,41 +88,37 @@ annexFile state file = inBackend file err $ do unannexFile :: State -> FilePath -> IO () unannexFile state file = notinBackend file err $ \(key, backend) -> do dropFile state backend key - cleanup key backend + logStatus state key ValueMissing + let src = annexLocation state backend key + removeFile file + gitRun (repo state) ["rm", file] + gitRun (repo state) ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + return () where err = error $ "not annexed " ++ file - cleanup key backend = do - let src = annexLocation state backend key - removeFile file - gitRun (repo state) ["rm", file] - gitRun (repo state) ["commit", "-m", - ("git-annex unannexed " ++ file), file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file - logStatus state key ValueMissing - return () {- Transfers the file from a remote. -} annexGetFile :: State -> FilePath -> IO () -annexGetFile state file = do - r <- lookupFile file - case (r) of - Nothing -> error $ "not annexed " ++ file - Just (key, backend) -> do - inannex <- inAnnex state backend key - if (inannex) - then return () - else do - let dest = annexLocation state backend key - createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state backend key dest - if (success) - then do - logStatus state key ValuePresent - return () - else error $ "failed to get " ++ file +annexGetFile state file = notinBackend file err $ \(key, backend) -> do + inannex <- inAnnex state backend key + if (inannex) + then return () + else do + let dest = annexLocation state backend key + createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile state backend key dest + if (success) + then do + logStatus state key ValuePresent + return () + else error $ "failed to get " ++ file + where + err = error $ "not annexed " ++ file {- Indicates a file is wanted. -} annexWantFile :: State -> FilePath -> IO () From 1dbf36bf9a951b7a92770ea0b57bc79c8b465795 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 03:51:55 -0400 Subject: [PATCH 0099/8313] cleanup3 --- Annex.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Annex.hs b/Annex.hs index eb57e2d57a..012e901997 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,10 +24,6 @@ import UUID import LocationLog import Types -{- Checks if a given key is currently present in the annexLocation -} -inAnnex :: State -> Backend -> Key -> IO Bool -inAnnex state backend key = doesFileExist $ annexLocation state backend key - {- On startup, examine the git repo, prepare it, and record state for - later. -} startAnnex :: IO State @@ -89,7 +85,6 @@ unannexFile :: State -> FilePath -> IO () unannexFile state file = notinBackend file err $ \(key, backend) -> do dropFile state backend key logStatus state key ValueMissing - let src = annexLocation state backend key removeFile file gitRun (repo state) ["rm", file] gitRun (repo state) ["commit", "-m", @@ -97,12 +92,13 @@ unannexFile state file = notinBackend file err $ \(key, backend) -> do -- git rm deletes empty directories; -- put them back createDirectoryIfMissing True (parentDir file) + let src = annexLocation state backend key renameFile src file return () where err = error $ "not annexed " ++ file -{- Transfers the file from a remote. -} +{- Gets an annexed file from one of the backends. -} annexGetFile :: State -> FilePath -> IO () annexGetFile state file = notinBackend file err $ \(key, backend) -> do inannex <- inAnnex state backend key @@ -165,3 +161,7 @@ logStatus state key status = do f <- logChange (repo state) key (getUUID (repo state)) status gitRun (repo state) ["add", f] gitRun (repo state) ["commit", "-m", "git-annex log update", f] + +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: State -> Backend -> Key -> IO Bool +inAnnex state backend key = doesFileExist $ annexLocation state backend key From 794d44cf1daf073f05d1a27b2a02c47db37c443a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 14:01:17 -0400 Subject: [PATCH 0100/8313] add remoteName --- GitRepo.hs | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f0686ff203..489c9cf758 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -40,11 +40,14 @@ import Utility data GitRepo = LocalGitRepo { top :: FilePath, - config :: Map String String + config :: Map String String, + -- remoteName holds the name used for this repo in remotes + remoteName :: Maybe String } | RemoteGitRepo { url :: String, top :: FilePath, - config :: Map String String + config :: Map String String, + remoteName :: Maybe String } deriving (Show, Read, Eq) {- Local GitRepo constructor. Can optionally query the repo for its config. -} @@ -52,7 +55,8 @@ gitRepoFromPath :: FilePath -> Bool -> IO GitRepo gitRepoFromPath dir query = do let r = LocalGitRepo { top = dir, - config = Map.empty + config = Map.empty, + remoteName = Nothing } if (query) then gitConfigRead r @@ -64,7 +68,8 @@ gitRepoFromUrl url query = do return $ RemoteGitRepo { url = url, top = path url, - config = Map.empty + config = Map.empty, + remoteName = Nothing } where path url = uriPath $ fromJust $ parseURI url @@ -174,13 +179,15 @@ gitConfig repo key defaultValue = gitConfigRemotes :: GitRepo -> IO [GitRepo] gitConfigRemotes repo = mapM construct remotes where - remotes = elems $ filter $ config repo + remotes = toList $ filter $ config repo filter = filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) - construct r = - if (isURI r) - then gitRepoFromUrl r False - else gitRepoFromPath r False + remotename k = (split "." k) !! 1 + construct (k,v) = do + r <- if (isURI v) + then gitRepoFromUrl v False + else gitRepoFromPath v False + return r { remoteName = Just $ remotename k } {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo From 771a6b36e1527571b9a38baacbee6e864f44172a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 14:40:56 -0400 Subject: [PATCH 0101/8313] cost ordering --- Annex.hs | 25 +++++++++++++++++++++++++ GitRepo.hs | 21 ++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/Annex.hs b/Annex.hs index 012e901997..bd57514ead 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,12 +9,14 @@ module Annex ( annexWantFile, annexDropFile, annexPushRepo, + repoCost, annexPullRepo ) where import System.Posix.Files import System.Directory import Data.String.Utils +import List import GitRepo import Utility import Locations @@ -165,3 +167,26 @@ logStatus state key status = do {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Backend -> Key -> IO Bool inAnnex state backend key = doesFileExist $ annexLocation state backend key + +{- Orders a list of git repos by cost. -} +reposByCost :: State -> [GitRepo] -> [GitRepo] +reposByCost state l = + fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l + where + costpairs l = map (\r -> (r, repoCost state r)) l + +{- Calculates cost for a repo. + - + - The default cost is 100 for local repositories, and 200 for remote + - repositories; it can also be configured by remote..annex-cost + -} +repoCost :: State -> GitRepo -> Int +repoCost state r = + if ((length $ config state r) > 0) + then read $ config state r + else if (gitRepoIsLocal r) + then 100 + else 200 + where + config state r = gitConfig (repo state) (configkey r) "" + configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/GitRepo.hs b/GitRepo.hs index 489c9cf758..c4a55863d2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -10,13 +10,17 @@ module GitRepo ( gitRepoFromCwd, gitRepoFromPath, gitRepoFromUrl, + gitRepoIsLocal, + gitRepoIsRemote, + gitConfigRemotes, gitWorkTree, gitDir, gitRelative, gitConfig, gitConfigRead, gitRun, - gitAttributes + gitAttributes, + gitRepoRemoteName ) where import Directory @@ -74,16 +78,23 @@ gitRepoFromUrl url query = do where path url = uriPath $ fromJust $ parseURI url {- User-visible description of a git repo by path or url -} -describe repo = if (local repo) then top repo else url repo +describe repo = if (gitRepoIsLocal repo) then top repo else url repo + +{- Returns the name of the remote that corresponds to the repo, if + - it is a remote. Otherwise, "" -} +gitRepoRemoteName r = + if (isJust $ remoteName r) + then fromJust $ remoteName r + else "" {- Some code needs to vary between remote and local repos, or bare and - non-bare, these functions help with that. -} -local repo = case (repo) of +gitRepoIsLocal repo = case (repo) of LocalGitRepo {} -> True RemoteGitRepo {} -> False -remote repo = not $ local repo +gitRepoIsRemote repo = not $ gitRepoIsLocal repo assertlocal repo action = - if (local repo) + if (gitRepoIsLocal repo) then action else error $ "acting on remote git repo " ++ (describe repo) ++ " not supported" From 77055f5ff82d2712f599ba77e03d5d2cc022ff65 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 14:51:09 -0400 Subject: [PATCH 0102/8313] move some stuff out of IO --- Annex.hs | 11 ++++++++--- GitRepo.hs | 32 ++++++++++++++------------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/Annex.hs b/Annex.hs index bd57514ead..e06bd84bc2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,7 +9,7 @@ module Annex ( annexWantFile, annexDropFile, annexPushRepo, - repoCost, + annexRemotes, annexPullRepo ) where @@ -31,8 +31,9 @@ import Types startAnnex :: IO State startAnnex = do r <- gitRepoFromCwd - r' <- prepUUID r - gitSetup r' + r' <- gitConfigRead r + r'' <- prepUUID r' + gitSetup r'' return State { repo = r', @@ -168,6 +169,10 @@ logStatus state key status = do inAnnex :: State -> Backend -> Key -> IO Bool inAnnex state backend key = doesFileExist $ annexLocation state backend key +{- Ordered list of remotes for the annex. -} +annexRemotes :: State -> [GitRepo] +annexRemotes state = reposByCost state $ gitConfigRemotes (repo state) + {- Orders a list of git repos by cost. -} reposByCost :: State -> [GitRepo] -> [GitRepo] reposByCost state l = diff --git a/GitRepo.hs b/GitRepo.hs index c4a55863d2..06e244d6bc 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -54,22 +54,19 @@ data GitRepo = remoteName :: Maybe String } deriving (Show, Read, Eq) -{- Local GitRepo constructor. Can optionally query the repo for its config. -} -gitRepoFromPath :: FilePath -> Bool -> IO GitRepo -gitRepoFromPath dir query = do - let r = LocalGitRepo { +{- Local GitRepo constructor. -} +gitRepoFromPath :: FilePath -> GitRepo +gitRepoFromPath dir = + LocalGitRepo { top = dir, config = Map.empty, remoteName = Nothing } - if (query) - then gitConfigRead r - else return r {- Remote GitRepo constructor. Throws exception on invalid url. -} -gitRepoFromUrl :: String -> Bool -> IO GitRepo -gitRepoFromUrl url query = do - return $ RemoteGitRepo { +gitRepoFromUrl :: String -> GitRepo +gitRepoFromUrl url = + RemoteGitRepo { url = url, top = path url, config = Map.empty, @@ -187,18 +184,17 @@ gitConfig repo key defaultValue = Map.findWithDefault defaultValue key (config repo) {- Returns a list of a repo's configured remotes. -} -gitConfigRemotes :: GitRepo -> IO [GitRepo] -gitConfigRemotes repo = mapM construct remotes +gitConfigRemotes :: GitRepo -> [GitRepo] +gitConfigRemotes repo = map construct remotes where remotes = toList $ filter $ config repo filter = filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 - construct (k,v) = do - r <- if (isURI v) - then gitRepoFromUrl v False - else gitRepoFromPath v False - return r { remoteName = Just $ remotename k } + construct (k,v) = (gen v) { remoteName = Just $ remotename k } + gen v = if (isURI v) + then gitRepoFromUrl v + else gitRepoFromPath v {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo @@ -206,7 +202,7 @@ gitRepoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> gitRepoFromPath dir True + (Just dir) -> return $ gitRepoFromPath dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) From e28ff5bdaf7ce56c0c928904ff883c1e2cd093de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 15:55:18 -0400 Subject: [PATCH 0103/8313] almost able to get files from remotes now! --- Annex.hs | 30 +----------------------------- BackendFile.hs | 36 +++++++++++++++++++++++++++--------- GitRepo.hs | 14 ++++++++++---- LocationLog.hs | 3 ++- Remotes.hs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ UUID.hs | 31 ++++++++++++++++++++++++++----- 6 files changed, 116 insertions(+), 48 deletions(-) create mode 100644 Remotes.hs diff --git a/Annex.hs b/Annex.hs index e06bd84bc2..834c26115b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -9,7 +9,6 @@ module Annex ( annexWantFile, annexDropFile, annexPushRepo, - annexRemotes, annexPullRepo ) where @@ -161,37 +160,10 @@ gitSetup repo = do {- Updates the LocationLog when a key's presence changes. -} logStatus state key status = do - f <- logChange (repo state) key (getUUID (repo state)) status + f <- logChange (repo state) key (getUUID state (repo state)) status gitRun (repo state) ["add", f] gitRun (repo state) ["commit", "-m", "git-annex log update", f] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Backend -> Key -> IO Bool inAnnex state backend key = doesFileExist $ annexLocation state backend key - -{- Ordered list of remotes for the annex. -} -annexRemotes :: State -> [GitRepo] -annexRemotes state = reposByCost state $ gitConfigRemotes (repo state) - -{- Orders a list of git repos by cost. -} -reposByCost :: State -> [GitRepo] -> [GitRepo] -reposByCost state l = - fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l - where - costpairs l = map (\r -> (r, repoCost state r)) l - -{- Calculates cost for a repo. - - - - The default cost is 100 for local repositories, and 200 for remote - - repositories; it can also be configured by remote..annex-cost - -} -repoCost :: State -> GitRepo -> Int -repoCost state r = - if ((length $ config state r) > 0) - then read $ config state r - else if (gitRepoIsLocal r) - then 100 - else 200 - where - config state r = gitConfig (repo state) (configkey r) "" - configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/BackendFile.hs b/BackendFile.hs index 15b23536bf..d4d137e53b 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -4,12 +4,16 @@ module BackendFile (backend) where import Types +import LocationLog +import Locations +import Remotes +import GitRepo backend = Backend { name = "file", getKey = keyValue, storeFileKey = dummyStore, - retrieveKeyFile = copyFromOtherRepo, + retrieveKeyFile = copyKeyFile, removeKey = dummyRemove } @@ -27,12 +31,26 @@ dummyStore state file key = return True dummyRemove :: State -> Key -> IO Bool dummyRemove state url = return False -{- Try to find a copy of the file in one of the other repos, +{- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} -copyFromOtherRepo :: State -> Key -> FilePath -> IO (Bool) -copyFromOtherRepo state key file = - -- 1. get ordered list of remotes (local repos, then remote repos) - -- 2. read locationlog for file - -- 3. filter remotes list to ones that have file - -- 4. attempt to transfer from each remote until success - error "copyFromOtherRepo unimplemented" -- TODO +copyKeyFile :: State -> Key -> FilePath -> IO (Bool) +copyKeyFile state key file = do + remotes <- remotesWithKey state key + if (0 == length remotes) + then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ + "(Perhaps you need to git remote add a repository?)" + else trycopy remotes remotes + where + trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ + "To get that file, need access to one of these remotes: " ++ + (remotesList full) + trycopy full (r:rs) = do + ok <- copyFromRemote r key file + if (ok) + then return True + else trycopy full rs + +{- Tries to copy a file from a remote. -} +copyFromRemote :: GitRepo -> Key -> FilePath -> IO (Bool) +copyFromRemote r key file = do + return False -- TODO diff --git a/GitRepo.hs b/GitRepo.hs index 06e244d6bc..e1f086b693 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -13,6 +13,7 @@ module GitRepo ( gitRepoIsLocal, gitRepoIsRemote, gitConfigRemotes, + gitRepoDescribe, gitWorkTree, gitDir, gitRelative, @@ -74,8 +75,13 @@ gitRepoFromUrl url = } where path url = uriPath $ fromJust $ parseURI url -{- User-visible description of a git repo by path or url -} -describe repo = if (gitRepoIsLocal repo) then top repo else url repo +{- User-visible description of a git repo. -} +gitRepoDescribe repo = + if (isJust $ remoteName repo) + then fromJust $ remoteName repo + else if (gitRepoIsLocal repo) + then top repo + else url repo {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} @@ -93,13 +99,13 @@ gitRepoIsRemote repo = not $ gitRepoIsLocal repo assertlocal repo action = if (gitRepoIsLocal repo) then action - else error $ "acting on remote git repo " ++ (describe repo) ++ + else error $ "acting on remote git repo " ++ (gitRepoDescribe repo) ++ " not supported" bare :: GitRepo -> Bool bare repo = if (member b (config repo)) then ("true" == fromJust (Map.lookup b (config repo))) - else error $ "it is not known if git repo " ++ (describe repo) ++ + else error $ "it is not known if git repo " ++ (gitRepoDescribe repo) ++ " is a bare repository; config not read" where b = "core.bare" diff --git a/LocationLog.hs b/LocationLog.hs index 2eab4815ed..28ac46b90c 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -18,7 +18,8 @@ module LocationLog ( LogStatus(..), - logChange + logChange, + keyLocations ) where import Data.Time.Clock.POSIX diff --git a/Remotes.hs b/Remotes.hs new file mode 100644 index 0000000000..ae709a3c23 --- /dev/null +++ b/Remotes.hs @@ -0,0 +1,50 @@ +{- git-annex remote repositories -} + +module Remotes ( + remotesList, + remotesWithKey +) where + +import Types +import GitRepo +import LocationLog +import Data.String.Utils +import UUID +import List + +{- Human visible list of remotes. -} +remotesList :: [GitRepo] -> String +remotesList remotes = join " " $ map gitRepoDescribe remotes + +{- Cost ordered list of remotes that the LocationLog indicate may have a key. -} +remotesWithKey :: State -> Key -> IO [GitRepo] +remotesWithKey state key = do + uuids <- keyLocations (repo state) key + return $ reposByUUID state (remotesByCost state) uuids + +{- Cost Ordered list of remotes. -} +remotesByCost :: State -> [GitRepo] +remotesByCost state = reposByCost state $ gitConfigRemotes (repo state) + +{- Orders a list of git repos by cost. -} +reposByCost :: State -> [GitRepo] -> [GitRepo] +reposByCost state l = + fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l + where + costpairs l = map (\r -> (r, repoCost state r)) l + +{- Calculates cost for a repo. + - + - The default cost is 100 for local repositories, and 200 for remote + - repositories; it can also be configured by remote..annex-cost + -} +repoCost :: State -> GitRepo -> Int +repoCost state r = + if ((length $ config state r) > 0) + then read $ config state r + else if (gitRepoIsLocal r) + then 100 + else 200 + where + config state r = gitConfig (repo state) (configkey r) "" + configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/UUID.hs b/UUID.hs index e2b624d69c..b4c4c0cc0b 100644 --- a/UUID.hs +++ b/UUID.hs @@ -9,12 +9,16 @@ module UUID ( UUID, getUUID, prepUUID, - genUUID + genUUID, + reposByUUID ) where +import Maybe +import List import System.Cmd.Utils import System.IO import GitRepo +import Types type UUID = String @@ -26,17 +30,34 @@ genUUID :: IO UUID genUUID = do pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h -{- Looks up a repo's UUID -} -getUUID :: GitRepo -> UUID -getUUID repo = gitConfig repo "annex.uuid" "" +{- Looks up a repo's UUID. May return "" if none is known. + - + - UUIDs of remotes are cached in git config, using keys named + - remote..annex-uuid + - + - -} +getUUID :: State -> GitRepo -> UUID +getUUID s r = + if ("" /= getUUID' r) + then getUUID' r + else cached s r + where + cached s r = gitConfig (repo s) (configkey r) "" + configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-uuid" +getUUID' r = gitConfig r "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: GitRepo -> IO GitRepo prepUUID repo = - if ("" == getUUID repo) + if ("" == getUUID' repo) then do uuid <- genUUID gitRun repo ["config", configkey, uuid] -- return new repo with updated config gitConfigRead repo else return repo + +{- Filters a list of repos to ones that have listed UUIDs. -} +reposByUUID :: State -> [GitRepo] -> [UUID] -> [GitRepo] +reposByUUID state repos uuids = + filter (\r -> isJust $ elemIndex (getUUID state r) uuids) repos From f87c5ed9496f50646d9f5e8be540f8bc059db242 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 16:21:50 -0400 Subject: [PATCH 0104/8313] copying almost working --- Annex.hs | 10 +++++----- BackendFile.hs | 35 ++++++++++++++++++++++++++++------- Locations.hs | 20 ++++++++++---------- git-annex.hs | 3 +-- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/Annex.hs b/Annex.hs index 834c26115b..8489c2ca6b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -64,8 +64,8 @@ annexFile state file = inBackend file err $ do else return () setup key backend = do logStatus state key ValuePresent - let dest = annexLocation state backend key - let reldest = annexLocationRelative state backend key + let dest = annexLocation (repo state) backend key + let reldest = annexLocationRelative (repo state) backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file @@ -94,7 +94,7 @@ unannexFile state file = notinBackend file err $ \(key, backend) -> do -- git rm deletes empty directories; -- put them back createDirectoryIfMissing True (parentDir file) - let src = annexLocation state backend key + let src = annexLocation (repo state) backend key renameFile src file return () where @@ -107,7 +107,7 @@ annexGetFile state file = notinBackend file err $ \(key, backend) -> do if (inannex) then return () else do - let dest = annexLocation state backend key + let dest = annexLocation (repo state) backend key createDirectoryIfMissing True (parentDir dest) success <- retrieveFile state backend key dest if (success) @@ -166,4 +166,4 @@ logStatus state key status = do {- Checks if a given key is currently present in the annexLocation -} inAnnex :: State -> Backend -> Key -> IO Bool -inAnnex state backend key = doesFileExist $ annexLocation state backend key +inAnnex state backend key = doesFileExist $ annexLocation (repo state) backend key diff --git a/BackendFile.hs b/BackendFile.hs index d4d137e53b..adb8da8bd7 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,6 +3,9 @@ module BackendFile (backend) where +import System.IO +import System.Cmd +import Control.Exception import Types import LocationLog import Locations @@ -45,12 +48,30 @@ copyKeyFile state key file = do "To get that file, need access to one of these remotes: " ++ (remotesList full) trycopy full (r:rs) = do - ok <- copyFromRemote r key file - if (ok) - then return True - else trycopy full rs + putStrLn "trying a remote" + result <- try (copyFromRemote r key file)::IO (Either SomeException ()) + case (result) of + Left err -> do + showerr err r + trycopy full rs + Right succ -> return True + showerr err r = do + hPutStrLn stderr $ "git-annex: copy from " ++ + (gitRepoDescribe r ) ++ " failed: " ++ + (show err) -{- Tries to copy a file from a remote. -} -copyFromRemote :: GitRepo -> Key -> FilePath -> IO (Bool) +{- Tries to copy a file from a remote, exception on error. -} +copyFromRemote :: GitRepo -> Key -> FilePath -> IO () copyFromRemote r key file = do - return False -- TODO + r <- if (gitRepoIsLocal r) + then getlocal + else getremote + return () + where + getlocal = do + putStrLn $ "get: " ++ location + rawSystem "cp" ["-a", location, file] + getremote = do + putStrLn $ "get: " ++ location + error "get via network not yet implemented!" + location = annexLocation r backend key diff --git a/Locations.hs b/Locations.hs index 304ca060e9..d6d7d42480 100644 --- a/Locations.hs +++ b/Locations.hs @@ -21,18 +21,18 @@ gitStateDir :: GitRepo -> FilePath gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" {- An annexed file's content is stored in - - .git/annex// ; this allows deriving the key and backend - - by looking at the symlink to it. -} -annexLocation :: State -> Backend -> Key -> FilePath -annexLocation state backend key = - (gitWorkTree $ repo state) ++ "/" ++ - (annexLocationRelative state backend key) + - /path/to/repo/.git/annex// + - + - (That allows deriving the key and backend by looking at the symlink to it.) + -} +annexLocation :: GitRepo -> Backend -> Key -> FilePath +annexLocation r backend key = + (gitWorkTree r) ++ "/" ++ (annexLocationRelative r backend key) {- Annexed file's location relative to the gitWorkTree -} -annexLocationRelative :: State -> Backend -> Key -> FilePath -annexLocationRelative state backend key = - gitDir (repo state) ++ "/annex/" ++ (name backend) ++ - "/" ++ (keyFile key) +annexLocationRelative :: GitRepo -> Backend -> Key -> FilePath +annexLocationRelative r backend key = + gitDir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) {- Converts a key into a filename fragment. - diff --git a/git-annex.hs b/git-annex.hs index 7bcd4de226..7785e4f2d7 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -32,6 +32,5 @@ tryRun errnum oknum (a:as) = do {- Exception pretty-printing. -} showErr :: SomeException -> IO () showErr e = do - let err = show e - hPutStrLn stderr $ "git-annex: " ++ err + hPutStrLn stderr $ "git-annex: " ++ (show e) return () From e5c1db355f5fa31af14ed8474aee89872b934f1a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 16:32:16 -0400 Subject: [PATCH 0105/8313] it works!! --- BackendFile.hs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index adb8da8bd7..a31cbfeb1e 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -48,30 +48,31 @@ copyKeyFile state key file = do "To get that file, need access to one of these remotes: " ++ (remotesList full) trycopy full (r:rs) = do - putStrLn "trying a remote" result <- try (copyFromRemote r key file)::IO (Either SomeException ()) case (result) of Left err -> do - showerr err r + hPutStrLn stderr (show err) trycopy full rs Right succ -> return True - showerr err r = do - hPutStrLn stderr $ "git-annex: copy from " ++ - (gitRepoDescribe r ) ++ " failed: " ++ - (show err) {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: GitRepo -> Key -> FilePath -> IO () copyFromRemote r key file = do - r <- if (gitRepoIsLocal r) - then getlocal - else getremote + putStrLn $ "copy from " ++ (gitRepoDescribe r ) ++ " " ++ file + + -- annexLocation needs the git config read for the remote first. + -- FIXME: Having this here means git-config is run repeatedly when + -- copying a series of files; need to use state monad to avoid + -- this. + r' <- gitConfigRead r + + _ <- if (gitRepoIsLocal r') + then getlocal r' + else getremote r' return () where - getlocal = do - putStrLn $ "get: " ++ location - rawSystem "cp" ["-a", location, file] - getremote = do - putStrLn $ "get: " ++ location + getlocal r = do + rawSystem "cp" ["-a", location r, file] + getremote r = do error "get via network not yet implemented!" - location = annexLocation r backend key + location r = annexLocation r backend key From b1607485168e851f69fe3a5b74d73f3c36edf886 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 21:28:47 -0400 Subject: [PATCH 0106/8313] use a state monad enormous reworking --- Annex.hs | 138 ++++++++++++++++++++++++++------------------- Backend.hs | 25 ++++---- BackendChecksum.hs | 2 +- BackendFile.hs | 21 +++---- BackendUrl.hs | 21 +++---- CmdLine.hs | 21 +++---- Remotes.hs | 44 +++++++++------ TODO | 4 +- Types.hs | 51 +++++++++++++++-- UUID.hs | 50 +++++++++------- git-annex.hs | 31 ++++++---- 11 files changed, 251 insertions(+), 157 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8489c2ca6b..f3c8f533a4 100644 --- a/Annex.hs +++ b/Annex.hs @@ -12,6 +12,7 @@ module Annex ( annexPullRepo ) where +import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory import Data.String.Utils @@ -25,22 +26,27 @@ import UUID import LocationLog import Types -{- On startup, examine the git repo, prepare it, and record state for - - later. -} -startAnnex :: IO State +{- Create and returns an Annex state object. + - Examines and prepares the git repo. + -} +startAnnex :: IO AnnexState startAnnex = do - r <- gitRepoFromCwd - r' <- gitConfigRead r - r'' <- prepUUID r' - gitSetup r'' - - return State { - repo = r', - backends = parseBackendList $ gitConfig r' "annex.backends" "" - } + g <- gitRepoFromCwd + let s = makeAnnexState g + (_,s') <- runAnnexState s (prep g) + return s' + where + prep g = do + -- setup git and read its config; update state + liftIO $ gitSetup g + g' <- liftIO $ gitConfigRead g + gitAnnexChange g' + backendsAnnexChange $ parseBackendList $ + gitConfig g' "annex.backends" "" + prepUUID inBackend file yes no = do - r <- lookupFile file + r <- liftIO $ lookupFile file case (r) of Just v -> yes v Nothing -> no @@ -48,13 +54,16 @@ notinBackend file yes no = inBackend file no yes {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexFile :: State -> FilePath -> IO () -annexFile state file = inBackend file err $ do - checkLegal file - stored <- storeFile state file +annexFile :: FilePath -> Annex () +annexFile file = inBackend file err $ do + liftIO $ checkLegal file + stored <- storeFile file + g <- gitAnnex case (stored) of Nothing -> error $ "no backend could store: " ++ file - Just (key, backend) -> setup key backend + Just (key, backend) -> do + logStatus key ValuePresent + liftIO $ setup g key backend where err = error $ "already annexed " ++ file checkLegal file = do @@ -62,15 +71,14 @@ annexFile state file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - setup key backend = do - logStatus state key ValuePresent - let dest = annexLocation (repo state) backend key - let reldest = annexLocationRelative (repo state) backend key + setup g key backend = do + let dest = annexLocation g backend key + let reldest = annexLocationRelative g backend key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file - gitRun (repo state) ["add", file] - gitRun (repo state) ["commit", "-m", + gitRun g ["add", file] + gitRun g ["commit", "-m", ("git-annex annexed " ++ file), file] linkTarget file = -- relies on file being relative to the top of the @@ -83,56 +91,60 @@ annexFile state file = inBackend file err $ do {- Inverse of annexFile. -} -unannexFile :: State -> FilePath -> IO () -unannexFile state file = notinBackend file err $ \(key, backend) -> do - dropFile state backend key - logStatus state key ValueMissing - removeFile file - gitRun (repo state) ["rm", file] - gitRun (repo state) ["commit", "-m", - ("git-annex unannexed " ++ file), file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - let src = annexLocation (repo state) backend key - renameFile src file - return () +unannexFile :: FilePath -> Annex () +unannexFile file = notinBackend file err $ \(key, backend) -> do + dropFile backend key + logStatus key ValueMissing + g <- gitAnnex + let src = annexLocation g backend key + liftIO $ moveout g src where err = error $ "not annexed " ++ file + moveout g src = do + removeFile file + gitRun g ["rm", file] + gitRun g ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + return () {- Gets an annexed file from one of the backends. -} -annexGetFile :: State -> FilePath -> IO () -annexGetFile state file = notinBackend file err $ \(key, backend) -> do - inannex <- inAnnex state backend key +annexGetFile :: FilePath -> Annex () +annexGetFile file = notinBackend file err $ \(key, backend) -> do + inannex <- inAnnex backend key if (inannex) then return () else do - let dest = annexLocation (repo state) backend key - createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile state backend key dest + g <- gitAnnex + let dest = annexLocation g backend key + liftIO $ createDirectoryIfMissing True (parentDir dest) + success <- retrieveFile backend key dest if (success) then do - logStatus state key ValuePresent + logStatus key ValuePresent return () else error $ "failed to get " ++ file where err = error $ "not annexed " ++ file {- Indicates a file is wanted. -} -annexWantFile :: State -> FilePath -> IO () -annexWantFile state file = do error "not implemented" -- TODO +annexWantFile :: FilePath -> Annex () +annexWantFile file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} -annexDropFile :: State -> FilePath -> IO () -annexDropFile state file = do error "not implemented" -- TODO +annexDropFile :: FilePath -> Annex () +annexDropFile file = do error "not implemented" -- TODO {- Pushes all files to a remote repository. -} -annexPushRepo :: State -> String -> IO () -annexPushRepo state reponame = do error "not implemented" -- TODO +annexPushRepo :: String -> Annex () +annexPushRepo reponame = do error "not implemented" -- TODO {- Pulls all files from a remote repository. -} -annexPullRepo :: State -> String -> IO () -annexPullRepo state reponame = do error "not implemented" -- TODO +annexPullRepo :: String -> Annex () +annexPullRepo reponame = do error "not implemented" -- TODO {- Sets up a git repo for git-annex. May be called repeatedly. -} gitSetup :: GitRepo -> IO () @@ -159,11 +171,19 @@ gitSetup repo = do attributes] {- Updates the LocationLog when a key's presence changes. -} -logStatus state key status = do - f <- logChange (repo state) key (getUUID state (repo state)) status - gitRun (repo state) ["add", f] - gitRun (repo state) ["commit", "-m", "git-annex log update", f] +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = do + g <- gitAnnex + u <- getUUID g + f <- liftIO $ logChange g key u status + liftIO $ commit g f + where + commit g f = do + gitRun g ["add", f] + gitRun g ["commit", "-m", "git-annex log update", f] {- Checks if a given key is currently present in the annexLocation -} -inAnnex :: State -> Backend -> Key -> IO Bool -inAnnex state backend key = doesFileExist $ annexLocation (repo state) backend key +inAnnex :: Backend -> Key -> Annex Bool +inAnnex backend key = do + g <- gitAnnex + liftIO $ doesFileExist $ annexLocation g backend key diff --git a/Backend.hs b/Backend.hs index bc7eb206ff..775c4a02f2 100644 --- a/Backend.hs +++ b/Backend.hs @@ -20,6 +20,7 @@ module Backend ( lookupFile ) where +import Control.Monad.State import Control.Exception import System.Directory import System.FilePath @@ -32,30 +33,34 @@ import Utility import Types {- Attempts to store a file in one of the backends. -} -storeFile :: State -> FilePath -> IO (Maybe (Key, Backend)) -storeFile state file = storeFile' (backends state) state file +storeFile :: FilePath -> Annex (Maybe (Key, Backend)) +storeFile file = do + g <- gitAnnex + let relfile = gitRelative g file + b <- backendsAnnex + storeFile' b file relfile storeFile' [] _ _ = return Nothing -storeFile' (b:bs) state file = do - try <- (getKey b) state (gitRelative (repo state) file) +storeFile' (b:bs) file relfile = do + try <- (getKey b) relfile case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) state file key + stored <- (storeFileKey b) file key if (not stored) then nextbackend else do return $ Just (key, b) where - nextbackend = storeFile' bs state file + nextbackend = storeFile' bs file relfile {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: State -> Backend -> Key -> FilePath -> IO Bool -retrieveFile state backend key dest = (retrieveKeyFile backend) state key dest +retrieveFile :: Backend -> Key -> FilePath -> Annex Bool +retrieveFile backend key dest = (retrieveKeyFile backend) key dest {- Drops a key from a backend. -} -dropFile :: State -> Backend -> Key -> IO Bool -dropFile state backend key = (removeKey backend) state key +dropFile :: Backend -> Key -> Annex Bool +dropFile backend key = (removeKey backend) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/BackendChecksum.hs b/BackendChecksum.hs index efa2244125..c6e68ffed4 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -14,5 +14,5 @@ backend = BackendFile.backend { } -- checksum the file to get its key -keyValue :: State -> FilePath -> IO (Maybe Key) +keyValue :: FilePath -> Annex (Maybe Key) keyValue k = error "checksum keyValue unimplemented" -- TODO diff --git a/BackendFile.hs b/BackendFile.hs index a31cbfeb1e..9b82a0b202 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -3,6 +3,7 @@ module BackendFile (backend) where +import Control.Monad.State import System.IO import System.Cmd import Control.Exception @@ -21,28 +22,28 @@ backend = Backend { } -- direct mapping from filename to key -keyValue :: State -> FilePath -> IO (Maybe Key) -keyValue state file = return $ Just $ Key file +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing or removing a key is - a no-op. TODO until support is added for git annex --push otherrepo, - then these could implement that.. -} -dummyStore :: State -> FilePath -> Key -> IO (Bool) -dummyStore state file key = return True -dummyRemove :: State -> Key -> IO Bool -dummyRemove state url = return False +dummyStore :: FilePath -> Key -> Annex (Bool) +dummyStore file key = return True +dummyRemove :: Key -> Annex Bool +dummyRemove url = return False {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} -copyKeyFile :: State -> Key -> FilePath -> IO (Bool) -copyKeyFile state key file = do - remotes <- remotesWithKey state key +copyKeyFile :: Key -> FilePath -> Annex (Bool) +copyKeyFile key file = do + remotes <- remotesWithKey key if (0 == length remotes) then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ "(Perhaps you need to git remote add a repository?)" - else trycopy remotes remotes + else liftIO $ trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ diff --git a/BackendUrl.hs b/BackendUrl.hs index 5b586497c4..43b0bc75a8 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -3,6 +3,7 @@ module BackendUrl (backend) where +import Control.Monad.State import System.Cmd import IO import Types @@ -16,19 +17,19 @@ backend = Backend { } -- cannot generate url from filename -keyValue :: State -> FilePath -> IO (Maybe Key) -keyValue repo file = return Nothing +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = return Nothing -- cannot change url contents -dummyStore :: State -> FilePath -> Key -> IO Bool -dummyStore repo file url = return False -dummyRemove :: State -> Key -> IO Bool -dummyRemove state url = return False +dummyStore :: FilePath -> Key -> Annex Bool +dummyStore file url = return False +dummyRemove :: Key -> Annex Bool +dummyRemove url = return False -downloadUrl :: State -> Key -> FilePath -> IO Bool -downloadUrl state url file = do - putStrLn $ "download: " ++ (show url) - result <- try $ rawSystem "curl" ["-#", "-o", file, (show url)] +downloadUrl :: Key -> FilePath -> Annex Bool +downloadUrl url file = do + liftIO $ putStrLn $ "download: " ++ (show url) + result <- liftIO $ try $ rawSystem "curl" ["-#", "-o", file, (show url)] case (result) of Left _ -> return False Right _ -> return True diff --git a/CmdLine.hs b/CmdLine.hs index 9da2b6493c..d23508aa27 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -6,7 +6,8 @@ module CmdLine ( argvToMode, - dispatch + dispatch, + Mode ) where import System.Console.GetOpt @@ -39,13 +40,13 @@ argvToMode argv = do (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) where header = "Usage: git-annex [mode] file" -dispatch :: State -> Mode -> FilePath -> IO () -dispatch state mode item = do +dispatch :: Mode -> FilePath -> Annex () +dispatch mode item = do case (mode) of - Add -> annexFile state item - Push -> annexPushRepo state item - Pull -> annexPullRepo state item - Want -> annexWantFile state item - Get -> annexGetFile state item - Drop -> annexDropFile state item - Unannex -> unannexFile state item + Add -> annexFile item + Push -> annexPushRepo item + Pull -> annexPullRepo item + Want -> annexWantFile item + Get -> annexGetFile item + Drop -> annexDropFile item + Unannex -> unannexFile item diff --git a/Remotes.hs b/Remotes.hs index ae709a3c23..3992914678 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -5,6 +5,7 @@ module Remotes ( remotesWithKey ) where +import Control.Monad.State (liftIO) import Types import GitRepo import LocationLog @@ -17,34 +18,43 @@ remotesList :: [GitRepo] -> String remotesList remotes = join " " $ map gitRepoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -remotesWithKey :: State -> Key -> IO [GitRepo] -remotesWithKey state key = do - uuids <- keyLocations (repo state) key - return $ reposByUUID state (remotesByCost state) uuids +remotesWithKey :: Key -> Annex [GitRepo] +remotesWithKey key = do + g <- gitAnnex + uuids <- liftIO $ keyLocations g key + remotes <- remotesByCost + reposByUUID remotes uuids {- Cost Ordered list of remotes. -} -remotesByCost :: State -> [GitRepo] -remotesByCost state = reposByCost state $ gitConfigRemotes (repo state) +remotesByCost :: Annex [GitRepo] +remotesByCost = do + g <- gitAnnex + reposByCost $ gitConfigRemotes g {- Orders a list of git repos by cost. -} -reposByCost :: State -> [GitRepo] -> [GitRepo] -reposByCost state l = - fst $ unzip $ sortBy (\(r1, c1) (r2, c2) -> compare c1 c2) $ costpairs l +reposByCost :: [GitRepo] -> Annex [GitRepo] +reposByCost l = do + costpairs <- mapM costpair l + return $ fst $ unzip $ sortBy bycost $ costpairs where - costpairs l = map (\r -> (r, repoCost state r)) l + costpair r = do + cost <- repoCost r + return (r, cost) + bycost (_, c1) (_, c2) = compare c1 c2 {- Calculates cost for a repo. - - The default cost is 100 for local repositories, and 200 for remote - repositories; it can also be configured by remote..annex-cost -} -repoCost :: State -> GitRepo -> Int -repoCost state r = - if ((length $ config state r) > 0) - then read $ config state r +repoCost :: GitRepo -> Annex Int +repoCost r = do + g <- gitAnnex + if ((length $ config g r) > 0) + then return $ read $ config g r else if (gitRepoIsLocal r) - then 100 - else 200 + then return 100 + else return 200 where - config state r = gitConfig (repo state) (configkey r) "" + config g r = gitConfig g (configkey r) "" configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" diff --git a/TODO b/TODO index a0f7c8b5fa..ea3f87c11b 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,9 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* implement retrieval for backendfile +* state monad -* query remotes for their annex.uuid settings +* query remotes for their annex.uuid settings and cache * --push/--pull/--want/--drop diff --git a/Types.hs b/Types.hs index 9b0bb00fd5..15c2ec89fd 100644 --- a/Types.hs +++ b/Types.hs @@ -1,20 +1,59 @@ {- git-annex core data types -} module Types ( - State(..), + Annex(..), + makeAnnexState, + runAnnexState, + gitAnnex, + gitAnnexChange, + backendsAnnex, + backendsAnnexChange, + + AnnexState(..), Key(..), Backend(..) ) where +import Control.Monad.State import Data.String.Utils import GitRepo -- git-annex's runtime state -data State = State { +data AnnexState = AnnexState { repo :: GitRepo, backends :: [Backend] } deriving (Show) +-- git-annex's monad +type Annex = StateT AnnexState IO + +-- constructor +makeAnnexState :: GitRepo -> AnnexState +makeAnnexState g = AnnexState { repo = g, backends = [] } + +-- performs an action in the Annex monad +runAnnexState state action = runStateT (action) state + +-- state accessors +gitAnnex :: Annex GitRepo +gitAnnex = do + state <- get + return (repo state) +gitAnnexChange :: GitRepo -> Annex () +gitAnnexChange r = do + state <- get + put state { repo = r } + return () +backendsAnnex :: Annex [Backend] +backendsAnnex = do + state <- get + return (backends state) +backendsAnnexChange :: [Backend] -> Annex () +backendsAnnexChange b = do + state <- get + put state { backends = b } + return () + -- annexed filenames are mapped into keys data Key = Key String deriving (Eq) @@ -27,13 +66,13 @@ data Backend = Backend { -- name of this backend name :: String, -- converts a filename to a key - getKey :: State -> FilePath -> IO (Maybe Key), + getKey :: FilePath -> Annex (Maybe Key), -- stores a file's contents to a key - storeFileKey :: State -> FilePath -> Key -> IO Bool, + storeFileKey :: FilePath -> Key -> Annex Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: State -> Key -> FilePath -> IO Bool, + retrieveKeyFile :: Key -> FilePath -> Annex Bool, -- removes a key - removeKey :: State -> Key -> IO Bool + removeKey :: Key -> Annex Bool } instance Show Backend where diff --git a/UUID.hs b/UUID.hs index b4c4c0cc0b..5c9f9179ea 100644 --- a/UUID.hs +++ b/UUID.hs @@ -13,6 +13,7 @@ module UUID ( reposByUUID ) where +import Control.Monad.State import Maybe import List import System.Cmd.Utils @@ -26,9 +27,8 @@ configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} -genUUID :: IO UUID -genUUID = do - pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h +genUUID :: Annex UUID +genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h {- Looks up a repo's UUID. May return "" if none is known. - @@ -36,28 +36,38 @@ genUUID = do - remote..annex-uuid - - -} -getUUID :: State -> GitRepo -> UUID -getUUID s r = - if ("" /= getUUID' r) - then getUUID' r - else cached s r +getUUID :: GitRepo -> Annex UUID +getUUID r = do + if ("" /= configured r) + then return $ configured r + else cached r where - cached s r = gitConfig (repo s) (configkey r) "" + configured r = gitConfig r "annex.uuid" "" + cached r = do + g <- gitAnnex + return $ gitConfig g (configkey r) "" configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-uuid" -getUUID' r = gitConfig r "annex.uuid" "" {- Make sure that the repo has an annex.uuid setting. -} -prepUUID :: GitRepo -> IO GitRepo -prepUUID repo = - if ("" == getUUID' repo) +prepUUID :: Annex () +prepUUID = do + g <- gitAnnex + u <- getUUID g + if ("" == u) then do uuid <- genUUID - gitRun repo ["config", configkey, uuid] - -- return new repo with updated config - gitConfigRead repo - else return repo + liftIO $ gitRun g ["config", configkey, uuid] + -- re-read git config and update the repo's state + u' <- liftIO $ gitConfigRead g + gitAnnexChange u' + return () + else return () {- Filters a list of repos to ones that have listed UUIDs. -} -reposByUUID :: State -> [GitRepo] -> [UUID] -> [GitRepo] -reposByUUID state repos uuids = - filter (\r -> isJust $ elemIndex (getUUID state r) uuids) repos +reposByUUID :: [GitRepo] -> [UUID] -> Annex [GitRepo] +reposByUUID repos uuids = do + filterM match repos + where + match r = do + u <- getUUID r + return $ isJust $ elemIndex u uuids diff --git a/git-annex.hs b/git-annex.hs index 7785e4f2d7..935be2f1ef 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,36 +1,43 @@ {- git-annex main program - -} +import Control.Monad.State import System.IO import System.Environment import Control.Exception import CmdLine +import Types import Annex main = do args <- getArgs - (mode, files) <- argvToMode args - + (mode, params) <- argvToMode args state <- startAnnex + tryRun state mode 0 0 params - tryRun 0 0 $ map (\f -> dispatch state mode f) files - -{- Tries to run a series of actions, not stopping if some error out, - - and propigating an overall error status at the end. -} -tryRun errnum oknum [] = do +{- Processes each param in the list by dispatching the handler function + - for the user-selection operation mode. Catches exceptions, not stopping + - if some error out, and propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> Mode -> Int -> Int -> [String] -> IO () +tryRun state mode errnum oknum [] = do if (errnum > 0) then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () -tryRun errnum oknum (a:as) = do - result <- try (a)::IO (Either SomeException ()) +tryRun state mode errnum oknum (f:fs) = do + result <- try (runAnnexState state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err - tryRun (errnum + 1) oknum as - Right _ -> tryRun errnum (oknum + 1) as + tryRun state mode (errnum + 1) oknum fs + Right (_,state') -> tryRun state' mode errnum (oknum + 1) fs {- Exception pretty-printing. -} -showErr :: SomeException -> IO () showErr e = do hPutStrLn stderr $ "git-annex: " ++ (show e) return () From 89654751daacd9336c114bfc1c88c952dcc4ffe9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 21:35:10 -0400 Subject: [PATCH 0107/8313] bugfix --- Annex.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index f3c8f533a4..f55a62c4d6 100644 --- a/Annex.hs +++ b/Annex.hs @@ -38,9 +38,9 @@ startAnnex = do where prep g = do -- setup git and read its config; update state - liftIO $ gitSetup g g' <- liftIO $ gitConfigRead g gitAnnexChange g' + liftIO $ gitSetup g' backendsAnnexChange $ parseBackendList $ gitConfig g' "annex.backends" "" prepUUID From 912d10e78b725b4d3d4105a0ffe5696c21fc0e10 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 22:59:43 -0400 Subject: [PATCH 0108/8313] implemented remotes config caching --- BackendFile.hs | 30 +++++++++++++----------------- GitRepo.hs | 50 +++++++++++++++++++++++++++++++++++--------------- Remotes.hs | 28 ++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 34 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index 9b82a0b202..d16f3611b1 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -43,16 +43,20 @@ copyKeyFile key file = do if (0 == length remotes) then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ "(Perhaps you need to git remote add a repository?)" - else liftIO $ trycopy remotes remotes + else trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ (remotesList full) trycopy full (r:rs) = do - result <- try (copyFromRemote r key file)::IO (Either SomeException ()) + -- annexLocation needs the git config to have been + -- read for a remote, so do that now, + -- if it hasn't been already + r' <- remoteEnsureGitConfigRead r + result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) case (result) of Left err -> do - hPutStrLn stderr (show err) + liftIO $ hPutStrLn stderr (show err) trycopy full rs Right succ -> return True @@ -61,19 +65,11 @@ copyFromRemote :: GitRepo -> Key -> FilePath -> IO () copyFromRemote r key file = do putStrLn $ "copy from " ++ (gitRepoDescribe r ) ++ " " ++ file - -- annexLocation needs the git config read for the remote first. - -- FIXME: Having this here means git-config is run repeatedly when - -- copying a series of files; need to use state monad to avoid - -- this. - r' <- gitConfigRead r - - _ <- if (gitRepoIsLocal r') - then getlocal r' - else getremote r' + if (gitRepoIsLocal r) + then getlocal + else getremote return () where - getlocal r = do - rawSystem "cp" ["-a", location r, file] - getremote r = do - error "get via network not yet implemented!" - location r = annexLocation r backend key + getlocal = rawSystem "cp" ["-a", location, file] + getremote = error "get via network not yet implemented!" + location = annexLocation r backend key diff --git a/GitRepo.hs b/GitRepo.hs index e1f086b693..d22218219c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -12,15 +12,17 @@ module GitRepo ( gitRepoFromUrl, gitRepoIsLocal, gitRepoIsRemote, - gitConfigRemotes, gitRepoDescribe, gitWorkTree, gitDir, gitRelative, gitConfig, + gitConfigMap, gitConfigRead, gitRun, gitAttributes, + gitRepoRemotes, + gitRepoRemotesAdd, gitRepoRemoteName ) where @@ -46,12 +48,14 @@ data GitRepo = LocalGitRepo { top :: FilePath, config :: Map String String, + remotes :: [GitRepo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String } | RemoteGitRepo { url :: String, top :: FilePath, config :: Map String String, + remotes :: [GitRepo], remoteName :: Maybe String } deriving (Show, Read, Eq) @@ -61,6 +65,7 @@ gitRepoFromPath dir = LocalGitRepo { top = dir, config = Map.empty, + remotes = [], remoteName = Nothing } @@ -71,6 +76,7 @@ gitRepoFromUrl url = url = url, top = path url, config = Map.empty, + remotes = [], remoteName = Nothing } where path url = uriPath $ fromJust $ parseURI url @@ -83,6 +89,15 @@ gitRepoDescribe repo = then top repo else url repo +{- Returns the list of a repo's remotes. -} +gitRepoRemotes :: GitRepo -> [GitRepo] +gitRepoRemotes r = remotes r + +{- Constructs and returns an updated version of a repo with + - different remotes list. -} +gitRepoRemotesAdd :: GitRepo -> [GitRepo] -> GitRepo +gitRepoRemotesAdd repo rs = repo { remotes = rs } + {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} gitRepoRemoteName r = @@ -169,10 +184,24 @@ gitConfigRead repo = assertlocal repo $ do been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory bracket_ (changeWorkingDirectory (top repo)) - (\_ -> changeWorkingDirectory cwd) $ do + (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do val <- hGetContentsStrict h - return repo { config = gitConfigParse val } + let r = repo { config = gitConfigParse val } + return r { remotes = gitConfigRemotes r } + +{- Calculates a list of a repo's configured remotes, by parsing its config. -} +gitConfigRemotes :: GitRepo -> [GitRepo] +gitConfigRemotes repo = map construct remotes + where + remotes = toList $ filter $ config repo + filter = filterWithKey (\k _ -> isremote k) + isremote k = (startswith "remote." k) && (endswith ".url" k) + remotename k = (split "." k) !! 1 + construct (k,v) = (gen v) { remoteName = Just $ remotename k } + gen v = if (isURI v) + then gitRepoFromUrl v + else gitRepoFromPath v {- Parses git config --list output into a config map. -} gitConfigParse :: String -> Map.Map String String @@ -189,18 +218,9 @@ gitConfig :: GitRepo -> String -> String -> String gitConfig repo key defaultValue = Map.findWithDefault defaultValue key (config repo) -{- Returns a list of a repo's configured remotes. -} -gitConfigRemotes :: GitRepo -> [GitRepo] -gitConfigRemotes repo = map construct remotes - where - remotes = toList $ filter $ config repo - filter = filterWithKey (\k _ -> isremote k) - isremote k = (startswith "remote." k) && (endswith ".url" k) - remotename k = (split "." k) !! 1 - construct (k,v) = (gen v) { remoteName = Just $ remotename k } - gen v = if (isURI v) - then gitRepoFromUrl v - else gitRepoFromPath v +{- Access to raw config Map -} +gitConfigMap :: GitRepo -> Map String String +gitConfigMap repo = config repo {- Finds the current git repository, which may be in a parent directory. -} gitRepoFromCwd :: IO GitRepo diff --git a/Remotes.hs b/Remotes.hs index 3992914678..13b87982c9 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -2,10 +2,12 @@ module Remotes ( remotesList, - remotesWithKey + remotesWithKey, + remoteEnsureGitConfigRead ) where import Control.Monad.State (liftIO) +import qualified Data.Map as Map import Types import GitRepo import LocationLog @@ -29,7 +31,7 @@ remotesWithKey key = do remotesByCost :: Annex [GitRepo] remotesByCost = do g <- gitAnnex - reposByCost $ gitConfigRemotes g + reposByCost $ gitRepoRemotes g {- Orders a list of git repos by cost. -} reposByCost :: [GitRepo] -> Annex [GitRepo] @@ -58,3 +60,25 @@ repoCost r = do where config g r = gitConfig g (configkey r) "" configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" + +{- The git configs for the git repo's remotes is not read on startup + - because reading it may be expensive. This function ensures that it is + - read for a specified remote, and updates state. It returns the + - updated git repo also. -} +remoteEnsureGitConfigRead :: GitRepo -> Annex GitRepo +remoteEnsureGitConfigRead r = do + if (Map.null $ gitConfigMap r) + then do + r' <- liftIO $ gitConfigRead r + g <- gitAnnex + let l = gitRepoRemotes g + let g' = gitRepoRemotesAdd g $ exchange l r' + gitAnnexChange g' + return r' + else return r + where + exchange [] new = [] + exchange (old:ls) new = + if ((gitRepoRemoteName old) == (gitRepoRemoteName new)) + then new:(exchange ls new) + else old:(exchange ls new) From 64b5167b0f9620bd96cd57b58f0e40be741e5420 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 23:03:01 -0400 Subject: [PATCH 0109/8313] update --- Types.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Types.hs b/Types.hs index 15c2ec89fd..ce377dda11 100644 --- a/Types.hs +++ b/Types.hs @@ -1,7 +1,8 @@ {- git-annex core data types -} module Types ( - Annex(..), + Annex, + AnnexState, makeAnnexState, runAnnexState, gitAnnex, @@ -9,7 +10,6 @@ module Types ( backendsAnnex, backendsAnnexChange, - AnnexState(..), Key(..), Backend(..) ) where @@ -34,7 +34,7 @@ makeAnnexState g = AnnexState { repo = g, backends = [] } -- performs an action in the Annex monad runAnnexState state action = runStateT (action) state --- state accessors +-- Annex monad state accessors gitAnnex :: Annex GitRepo gitAnnex = do state <- get From 8ab54401b609f49a603f3ed69bb8493a53f28db8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 23:18:58 -0400 Subject: [PATCH 0110/8313] update --- BackendFile.hs | 5 +---- Remotes.hs | 12 +++++++++--- TODO | 2 -- UUID.hs | 10 +++++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/BackendFile.hs b/BackendFile.hs index d16f3611b1..e821ac22b2 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -40,10 +40,7 @@ dummyRemove url = return False copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- remotesWithKey key - if (0 == length remotes) - then error $ "no known remotes have: " ++ (keyFile key) ++ "\n" ++ - "(Perhaps you need to git remote add a repository?)" - else trycopy remotes remotes + trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ diff --git a/Remotes.hs b/Remotes.hs index 13b87982c9..f3af81f23d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -8,10 +8,11 @@ module Remotes ( import Control.Monad.State (liftIO) import qualified Data.Map as Map +import Data.String.Utils import Types import GitRepo import LocationLog -import Data.String.Utils +import Locations import UUID import List @@ -24,8 +25,13 @@ remotesWithKey :: Key -> Annex [GitRepo] remotesWithKey key = do g <- gitAnnex uuids <- liftIO $ keyLocations g key - remotes <- remotesByCost - reposByUUID remotes uuids + allremotes <- remotesByCost + remotes <- reposByUUID allremotes uuids + if (0 == length remotes) + then error $ "no configured git remotes have: " ++ (keyFile key) ++ "\n" ++ + "It has been seen before in these repositories:\n" ++ + prettyPrintUUIDs uuids + else return remotes {- Cost Ordered list of remotes. -} remotesByCost :: Annex [GitRepo] diff --git a/TODO b/TODO index ea3f87c11b..40017c816a 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,6 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* state monad - * query remotes for their annex.uuid settings and cache * --push/--pull/--want/--drop diff --git a/UUID.hs b/UUID.hs index 5c9f9179ea..af6003bfb7 100644 --- a/UUID.hs +++ b/UUID.hs @@ -10,7 +10,8 @@ module UUID ( getUUID, prepUUID, genUUID, - reposByUUID + reposByUUID, + prettyPrintUUIDs ) where import Control.Monad.State @@ -71,3 +72,10 @@ reposByUUID repos uuids = do match r = do u <- getUUID r return $ isJust $ elemIndex u uuids + +{- Pretty-prints a list of UUIDs + - TODO: use lookup file to really show pretty names. -} +prettyPrintUUIDs :: [UUID] -> String +prettyPrintUUIDs uuids = + unwords $ map (\u -> "\tUUID "++u++"\n") uuids + From e47dca162a0ca0144172c9a61a47d1e0b5ad04b7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Oct 2010 23:31:08 -0400 Subject: [PATCH 0111/8313] update --- Locations.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Locations.hs b/Locations.hs index d6d7d42480..68a9581924 100644 --- a/Locations.hs +++ b/Locations.hs @@ -47,6 +47,7 @@ annexLocationRelative r backend key = keyFile :: Key -> FilePath keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key -{- Reverses keyFile -} +{- Reverses keyFile, converting a filename fragment (ie, the basename of + - the symlink target) into a key. -} fileKey :: FilePath -> Key fileKey file = Key $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From eda80e44c5fb399fa4e5625388d6e0f993b0f779 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:12:41 -0400 Subject: [PATCH 0112/8313] add module that only exports abstract types --- AbstractTypes.hs | 17 +++++++++++++++++ Annex.hs | 2 +- CmdLine.hs | 2 +- LocationLog.hs | 2 +- Remotes.hs | 2 +- UUID.hs | 2 +- git-annex.hs | 2 +- 7 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 AbstractTypes.hs diff --git a/AbstractTypes.hs b/AbstractTypes.hs new file mode 100644 index 0000000000..510a37f0cc --- /dev/null +++ b/AbstractTypes.hs @@ -0,0 +1,17 @@ +{- git-annex data types, abstract only -} + +module AbstractTypes ( + Annex, + AnnexState, + makeAnnexState, + runAnnexState, + gitAnnex, + gitAnnexChange, + backendsAnnex, + backendsAnnexChange, + + Key, + Backend +) where + +import Types diff --git a/Annex.hs b/Annex.hs index f55a62c4d6..68bf0136a9 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,7 +24,7 @@ import Backend import BackendList import UUID import LocationLog -import Types +import AbstractTypes {- Create and returns an Annex state object. - Examines and prepares the git repo. diff --git a/CmdLine.hs b/CmdLine.hs index d23508aa27..bb908a2e40 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,7 +11,7 @@ module CmdLine ( ) where import System.Console.GetOpt -import Types +import AbstractTypes import Annex data Mode = Add | Push | Pull | Want | Get | Drop | Unannex diff --git a/LocationLog.hs b/LocationLog.hs index 28ac46b90c..a6d998e0af 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,7 +32,7 @@ import Data.Char import GitRepo import Utility import UUID -import Types +import AbstractTypes import Locations data LogLine = LogLine { diff --git a/Remotes.hs b/Remotes.hs index f3af81f23d..711cd6c83a 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -9,7 +9,7 @@ module Remotes ( import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils -import Types +import AbstractTypes import GitRepo import LocationLog import Locations diff --git a/UUID.hs b/UUID.hs index af6003bfb7..f334afdc9d 100644 --- a/UUID.hs +++ b/UUID.hs @@ -20,7 +20,7 @@ import List import System.Cmd.Utils import System.IO import GitRepo -import Types +import AbstractTypes type UUID = String diff --git a/git-annex.hs b/git-annex.hs index 935be2f1ef..be5168755f 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,7 +6,7 @@ import System.IO import System.Environment import Control.Exception import CmdLine -import Types +import AbstractTypes import Annex main = do From 48643b68b3ff05399b72f44b8b02ff34d6de046c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:36:41 -0400 Subject: [PATCH 0113/8313] convert GitRepo to qualified import --- Annex.hs | 78 ++++++++++++------------ Backend.hs | 4 +- BackendFile.hs | 8 +-- CmdLine.hs | 14 ++--- GitRepo.hs | 158 ++++++++++++++++++++++++------------------------- LocationLog.hs | 10 ++-- Locations.hs | 14 ++--- Remotes.hs | 34 +++++------ Types.hs | 24 ++------ UUID.hs | 16 ++--- git-annex.hs | 2 +- 11 files changed, 173 insertions(+), 189 deletions(-) diff --git a/Annex.hs b/Annex.hs index 68bf0136a9..54f9b9dff8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -2,14 +2,14 @@ -} module Annex ( - startAnnex, - annexFile, - unannexFile, - annexGetFile, - annexWantFile, - annexDropFile, - annexPushRepo, - annexPullRepo + start, + annexCmd, + unannexCmd, + getCmd, + wantCmd, + dropCmd, + pushCmd, + pullCmd ) where import Control.Monad.State (liftIO) @@ -17,7 +17,7 @@ import System.Posix.Files import System.Directory import Data.String.Utils import List -import GitRepo +import qualified GitRepo as Git import Utility import Locations import Backend @@ -29,20 +29,20 @@ import AbstractTypes {- Create and returns an Annex state object. - Examines and prepares the git repo. -} -startAnnex :: IO AnnexState -startAnnex = do - g <- gitRepoFromCwd +start :: IO AnnexState +start = do + g <- Git.repoFromCwd let s = makeAnnexState g (_,s') <- runAnnexState s (prep g) return s' where prep g = do -- setup git and read its config; update state - g' <- liftIO $ gitConfigRead g + g' <- liftIO $ Git.configRead g gitAnnexChange g' liftIO $ gitSetup g' backendsAnnexChange $ parseBackendList $ - gitConfig g' "annex.backends" "" + Git.configGet g' "annex.backends" "" prepUUID inBackend file yes no = do @@ -54,8 +54,8 @@ notinBackend file yes no = inBackend file no yes {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexFile :: FilePath -> Annex () -annexFile file = inBackend file err $ do +annexCmd :: FilePath -> Annex () +annexCmd file = inBackend file err $ do liftIO $ checkLegal file stored <- storeFile file g <- gitAnnex @@ -77,8 +77,8 @@ annexFile file = inBackend file err $ do createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file - gitRun g ["add", file] - gitRun g ["commit", "-m", + Git.run g ["add", file] + Git.run g ["commit", "-m", ("git-annex annexed " ++ file), file] linkTarget file = -- relies on file being relative to the top of the @@ -90,9 +90,9 @@ annexFile file = inBackend file err $ do subdirs = (length $ split "/" file) - 1 -{- Inverse of annexFile. -} -unannexFile :: FilePath -> Annex () -unannexFile file = notinBackend file err $ \(key, backend) -> do +{- Inverse of annexCmd. -} +unannexCmd :: FilePath -> Annex () +unannexCmd file = notinBackend file err $ \(key, backend) -> do dropFile backend key logStatus key ValueMissing g <- gitAnnex @@ -102,8 +102,8 @@ unannexFile file = notinBackend file err $ \(key, backend) -> do err = error $ "not annexed " ++ file moveout g src = do removeFile file - gitRun g ["rm", file] - gitRun g ["commit", "-m", + Git.run g ["rm", file] + Git.run g ["commit", "-m", ("git-annex unannexed " ++ file), file] -- git rm deletes empty directories; -- put them back @@ -112,8 +112,8 @@ unannexFile file = notinBackend file err $ \(key, backend) -> do return () {- Gets an annexed file from one of the backends. -} -annexGetFile :: FilePath -> Annex () -annexGetFile file = notinBackend file err $ \(key, backend) -> do +getCmd :: FilePath -> Annex () +getCmd file = notinBackend file err $ \(key, backend) -> do inannex <- inAnnex backend key if (inannex) then return () @@ -131,23 +131,23 @@ annexGetFile file = notinBackend file err $ \(key, backend) -> do err = error $ "not annexed " ++ file {- Indicates a file is wanted. -} -annexWantFile :: FilePath -> Annex () -annexWantFile file = do error "not implemented" -- TODO +wantCmd :: FilePath -> Annex () +wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} -annexDropFile :: FilePath -> Annex () -annexDropFile file = do error "not implemented" -- TODO +dropCmd :: FilePath -> Annex () +dropCmd file = do error "not implemented" -- TODO {- Pushes all files to a remote repository. -} -annexPushRepo :: String -> Annex () -annexPushRepo reponame = do error "not implemented" -- TODO +pushCmd :: String -> Annex () +pushCmd reponame = do error "not implemented" -- TODO {- Pulls all files from a remote repository. -} -annexPullRepo :: String -> Annex () -annexPullRepo reponame = do error "not implemented" -- TODO +pullCmd :: String -> Annex () +pullCmd reponame = do error "not implemented" -- TODO {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: GitRepo -> IO () +gitSetup :: Git.Repo -> IO () gitSetup repo = do -- configure git to use union merge driver on state files exists <- doesFileExist attributes @@ -164,10 +164,10 @@ gitSetup repo = do else return () where attrLine = stateLoc ++ "/*.log merge=union" - attributes = gitAttributes repo + attributes = Git.attributes repo commit = do - gitRun repo ["add", attributes] - gitRun repo ["commit", "-m", "git-annex setup", + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", attributes] {- Updates the LocationLog when a key's presence changes. -} @@ -179,8 +179,8 @@ logStatus key status = do liftIO $ commit g f where commit g f = do - gitRun g ["add", f] - gitRun g ["commit", "-m", "git-annex log update", f] + Git.run g ["add", f] + Git.run g ["commit", "-m", "git-annex log update", f] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool diff --git a/Backend.hs b/Backend.hs index 775c4a02f2..1bd4efc1e8 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,7 +28,7 @@ import Data.String.Utils import System.Posix.Files import BackendList import Locations -import GitRepo +import qualified GitRepo as Git import Utility import Types @@ -36,7 +36,7 @@ import Types storeFile :: FilePath -> Annex (Maybe (Key, Backend)) storeFile file = do g <- gitAnnex - let relfile = gitRelative g file + let relfile = Git.relative g file b <- backendsAnnex storeFile' b file relfile storeFile' [] _ _ = return Nothing diff --git a/BackendFile.hs b/BackendFile.hs index e821ac22b2..6c1dc06233 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -11,7 +11,7 @@ import Types import LocationLog import Locations import Remotes -import GitRepo +import qualified GitRepo as Git backend = Backend { name = "file", @@ -58,11 +58,11 @@ copyKeyFile key file = do Right succ -> return True {- Tries to copy a file from a remote, exception on error. -} -copyFromRemote :: GitRepo -> Key -> FilePath -> IO () +copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () copyFromRemote r key file = do - putStrLn $ "copy from " ++ (gitRepoDescribe r ) ++ " " ++ file + putStrLn $ "copy from " ++ (Git.repoDescribe r ) ++ " " ++ file - if (gitRepoIsLocal r) + if (Git.repoIsLocal r) then getlocal else getremote return () diff --git a/CmdLine.hs b/CmdLine.hs index bb908a2e40..479be7e8b5 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -43,10 +43,10 @@ argvToMode argv = do dispatch :: Mode -> FilePath -> Annex () dispatch mode item = do case (mode) of - Add -> annexFile item - Push -> annexPushRepo item - Pull -> annexPullRepo item - Want -> annexWantFile item - Get -> annexGetFile item - Drop -> annexDropFile item - Unannex -> unannexFile item + Add -> annexCmd item + Push -> pushCmd item + Pull -> pullCmd item + Want -> wantCmd item + Get -> getCmd item + Drop -> dropCmd item + Unannex -> unannexCmd item diff --git a/GitRepo.hs b/GitRepo.hs index d22218219c..f3bb5427ad 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -3,27 +3,27 @@ - This is written to be completely independant of git-annex and should be - suitable for other uses. - - - -} + -} module GitRepo ( - GitRepo, - gitRepoFromCwd, - gitRepoFromPath, - gitRepoFromUrl, - gitRepoIsLocal, - gitRepoIsRemote, - gitRepoDescribe, - gitWorkTree, - gitDir, - gitRelative, - gitConfig, - gitConfigMap, - gitConfigRead, - gitRun, - gitAttributes, - gitRepoRemotes, - gitRepoRemotesAdd, - gitRepoRemoteName + Repo, + repoFromCwd, + repoFromPath, + repoFromUrl, + repoIsLocal, + repoIsRemote, + repoDescribe, + workTree, + dir, + relative, + configGet, + configMap, + configRead, + run, + attributes, + remotes, + remotesAdd, + repoRemoteName ) where import Directory @@ -44,35 +44,35 @@ import Utility {- A git repository can be on local disk or remote. Not to be confused - with a git repo's configured remotes, some of which may be on local - disk. -} -data GitRepo = - LocalGitRepo { +data Repo = + LocalRepo { top :: FilePath, config :: Map String String, - remotes :: [GitRepo], + remotes :: [Repo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String - } | RemoteGitRepo { + } | RemoteRepo { url :: String, top :: FilePath, config :: Map String String, - remotes :: [GitRepo], + remotes :: [Repo], remoteName :: Maybe String } deriving (Show, Read, Eq) -{- Local GitRepo constructor. -} -gitRepoFromPath :: FilePath -> GitRepo -gitRepoFromPath dir = - LocalGitRepo { +{- Local Repo constructor. -} +repoFromPath :: FilePath -> Repo +repoFromPath dir = + LocalRepo { top = dir, config = Map.empty, remotes = [], remoteName = Nothing } -{- Remote GitRepo constructor. Throws exception on invalid url. -} -gitRepoFromUrl :: String -> GitRepo -gitRepoFromUrl url = - RemoteGitRepo { +{- Remote Repo constructor. Throws exception on invalid url. -} +repoFromUrl :: String -> Repo +repoFromUrl url = + RemoteRepo { url = url, top = path url, config = Map.empty, @@ -82,72 +82,68 @@ gitRepoFromUrl url = where path url = uriPath $ fromJust $ parseURI url {- User-visible description of a git repo. -} -gitRepoDescribe repo = +repoDescribe repo = if (isJust $ remoteName repo) then fromJust $ remoteName repo - else if (gitRepoIsLocal repo) + else if (repoIsLocal repo) then top repo else url repo -{- Returns the list of a repo's remotes. -} -gitRepoRemotes :: GitRepo -> [GitRepo] -gitRepoRemotes r = remotes r - {- Constructs and returns an updated version of a repo with - different remotes list. -} -gitRepoRemotesAdd :: GitRepo -> [GitRepo] -> GitRepo -gitRepoRemotesAdd repo rs = repo { remotes = rs } +remotesAdd :: Repo -> [Repo] -> Repo +remotesAdd repo rs = repo { remotes = rs } {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} -gitRepoRemoteName r = +repoRemoteName r = if (isJust $ remoteName r) then fromJust $ remoteName r else "" {- Some code needs to vary between remote and local repos, or bare and - non-bare, these functions help with that. -} -gitRepoIsLocal repo = case (repo) of - LocalGitRepo {} -> True - RemoteGitRepo {} -> False -gitRepoIsRemote repo = not $ gitRepoIsLocal repo +repoIsLocal repo = case (repo) of + LocalRepo {} -> True + RemoteRepo {} -> False +repoIsRemote repo = not $ repoIsLocal repo assertlocal repo action = - if (gitRepoIsLocal repo) + if (repoIsLocal repo) then action - else error $ "acting on remote git repo " ++ (gitRepoDescribe repo) ++ + else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" -bare :: GitRepo -> Bool +bare :: Repo -> Bool bare repo = if (member b (config repo)) then ("true" == fromJust (Map.lookup b (config repo))) - else error $ "it is not known if git repo " ++ (gitRepoDescribe repo) ++ + else error $ "it is not known if git repo " ++ (repoDescribe repo) ++ " is a bare repository; config not read" where b = "core.bare" {- Path to a repository's gitattributes file. -} -gitAttributes :: GitRepo -> String -gitAttributes repo = assertlocal repo $ do +attributes :: Repo -> String +attributes repo = assertlocal repo $ do if (bare repo) then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its topdir. -} -gitDir :: GitRepo -> String -gitDir repo = assertlocal repo $ +dir :: Repo -> String +dir repo = assertlocal repo $ if (bare repo) then "" else ".git" {- Path to a repository's --work-tree. -} -gitWorkTree :: GitRepo -> FilePath -gitWorkTree repo = top repo +workTree :: Repo -> FilePath +workTree repo = top repo {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} -gitRelative :: GitRepo -> String -> String -gitRelative repo file = drop (length absrepo) absfile +relative :: Repo -> String -> String +relative repo file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file @@ -159,27 +155,27 @@ gitRelative repo file = drop (length absrepo) absfile Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo {- Constructs a git command line operating on the specified repo. -} -gitCommandLine :: GitRepo -> [String] -> [String] +gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo params = assertlocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(top repo)++"/"++(gitDir repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(top repo)++"/"++(dir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} -gitRun :: GitRepo -> [String] -> IO () -gitRun repo params = assertlocal repo $ do +run :: Repo -> [String] -> IO () +run repo params = assertlocal repo $ do r <- rawSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} -gitPipeRead :: GitRepo -> [String] -> IO String +gitPipeRead :: Repo -> [String] -> IO String gitPipeRead repo params = assertlocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do ret <- hGetContentsStrict h return ret {- Runs git config and populates a repo with its config. -} -gitConfigRead :: GitRepo -> IO GitRepo -gitConfigRead repo = assertlocal repo $ do +configRead :: Repo -> IO Repo +configRead repo = assertlocal repo $ do {- Cannot use gitPipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory @@ -187,12 +183,12 @@ gitConfigRead repo = assertlocal repo $ do (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do val <- hGetContentsStrict h - let r = repo { config = gitConfigParse val } - return r { remotes = gitConfigRemotes r } + let r = repo { config = configParse val } + return r { remotes = configRemotes r } {- Calculates a list of a repo's configured remotes, by parsing its config. -} -gitConfigRemotes :: GitRepo -> [GitRepo] -gitConfigRemotes repo = map construct remotes +configRemotes :: Repo -> [Repo] +configRemotes repo = map construct remotes where remotes = toList $ filter $ config repo filter = filterWithKey (\k _ -> isremote k) @@ -200,12 +196,12 @@ gitConfigRemotes repo = map construct remotes remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } gen v = if (isURI v) - then gitRepoFromUrl v - else gitRepoFromPath v + then repoFromUrl v + else repoFromPath v {- Parses git config --list output into a config map. -} -gitConfigParse :: String -> Map.Map String String -gitConfigParse s = Map.fromList $ map pair $ lines s +configParse :: String -> Map.Map String String +configParse s = Map.fromList $ map pair $ lines s where pair l = (key l, val l) key l = (keyval l) !! 0 @@ -214,21 +210,21 @@ gitConfigParse s = Map.fromList $ map pair $ lines s sep = "=" {- Returns a single git config setting, or a default value if not set. -} -gitConfig :: GitRepo -> String -> String -> String -gitConfig repo key defaultValue = +configGet :: Repo -> String -> String -> String +configGet repo key defaultValue = Map.findWithDefault defaultValue key (config repo) {- Access to raw config Map -} -gitConfigMap :: GitRepo -> Map String String -gitConfigMap repo = config repo +configMap :: Repo -> Map String String +configMap repo = config repo {- Finds the current git repository, which may be in a parent directory. -} -gitRepoFromCwd :: IO GitRepo -gitRepoFromCwd = do +repoFromCwd :: IO Repo +repoFromCwd = do cwd <- getCurrentDirectory top <- seekUp cwd isRepoTop case top of - (Just dir) -> return $ gitRepoFromPath dir + (Just dir) -> return $ repoFromPath dir Nothing -> error "Not in a git repository." seekUp :: String -> (String -> IO Bool) -> IO (Maybe String) @@ -241,11 +237,11 @@ seekUp dir want = do d -> seekUp d want isRepoTop dir = do - r <- isGitRepo dir + r <- isRepo dir b <- isBareRepo dir return (r || b) where - isGitRepo dir = gitSignature dir ".git" ".git/config" + isRepo dir = gitSignature dir ".git" ".git/config" isBareRepo dir = gitSignature dir "objects" "config" gitSignature dir subdir file = do s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) diff --git a/LocationLog.hs b/LocationLog.hs index a6d998e0af..7953b345f8 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -29,7 +29,7 @@ import qualified Data.Map as Map import System.IO import System.Directory import Data.Char -import GitRepo +import qualified GitRepo as Git import Utility import UUID import AbstractTypes @@ -81,7 +81,7 @@ instance Read LogLine where {- Log a change in the presence of a key's value in a repository, - and return the log filename. -} -logChange :: GitRepo -> Key -> UUID -> LogStatus -> IO FilePath +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key uuid status = do log <- logNow status uuid ls <- readLog logfile @@ -127,13 +127,13 @@ logNow status uuid = do return $ LogLine now status uuid {- Returns the filename of the log file for a given key. -} -logFile :: GitRepo -> Key -> String +logFile :: Git.Repo -> Key -> String logFile repo key = - (gitStateDir repo) ++ (gitRelative repo (keyFile key)) ++ ".log" + (gitStateDir repo) ++ (Git.relative repo (keyFile key)) ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} -keyLocations :: GitRepo -> Key -> IO [UUID] +keyLocations :: Git.Repo -> Key -> IO [UUID] keyLocations thisrepo key = do lines <- readLog $ logFile thisrepo key return $ map uuid (filterPresent lines) diff --git a/Locations.hs b/Locations.hs index 68a9581924..5d701681c1 100644 --- a/Locations.hs +++ b/Locations.hs @@ -12,27 +12,27 @@ module Locations ( import Data.String.Utils import Types -import GitRepo +import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository's working tree. -} stateLoc = ".git-annex" -gitStateDir :: GitRepo -> FilePath -gitStateDir repo = (gitWorkTree repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir :: Git.Repo -> FilePath +gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc ++ "/" {- An annexed file's content is stored in - /path/to/repo/.git/annex// - - (That allows deriving the key and backend by looking at the symlink to it.) -} -annexLocation :: GitRepo -> Backend -> Key -> FilePath +annexLocation :: Git.Repo -> Backend -> Key -> FilePath annexLocation r backend key = - (gitWorkTree r) ++ "/" ++ (annexLocationRelative r backend key) + (Git.workTree r) ++ "/" ++ (annexLocationRelative r backend key) {- Annexed file's location relative to the gitWorkTree -} -annexLocationRelative :: GitRepo -> Backend -> Key -> FilePath +annexLocationRelative :: Git.Repo -> Backend -> Key -> FilePath annexLocationRelative r backend key = - gitDir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) + Git.dir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) {- Converts a key into a filename fragment. - diff --git a/Remotes.hs b/Remotes.hs index 711cd6c83a..39404bf191 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -10,18 +10,18 @@ import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils import AbstractTypes -import GitRepo +import qualified GitRepo as Git import LocationLog import Locations import UUID import List {- Human visible list of remotes. -} -remotesList :: [GitRepo] -> String -remotesList remotes = join " " $ map gitRepoDescribe remotes +remotesList :: [Git.Repo] -> String +remotesList remotes = join " " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -remotesWithKey :: Key -> Annex [GitRepo] +remotesWithKey :: Key -> Annex [Git.Repo] remotesWithKey key = do g <- gitAnnex uuids <- liftIO $ keyLocations g key @@ -34,13 +34,13 @@ remotesWithKey key = do else return remotes {- Cost Ordered list of remotes. -} -remotesByCost :: Annex [GitRepo] +remotesByCost :: Annex [Git.Repo] remotesByCost = do g <- gitAnnex - reposByCost $ gitRepoRemotes g + reposByCost $ Git.remotes g {- Orders a list of git repos by cost. -} -reposByCost :: [GitRepo] -> Annex [GitRepo] +reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do costpairs <- mapM costpair l return $ fst $ unzip $ sortBy bycost $ costpairs @@ -55,36 +55,36 @@ reposByCost l = do - The default cost is 100 for local repositories, and 200 for remote - repositories; it can also be configured by remote..annex-cost -} -repoCost :: GitRepo -> Annex Int +repoCost :: Git.Repo -> Annex Int repoCost r = do g <- gitAnnex if ((length $ config g r) > 0) then return $ read $ config g r - else if (gitRepoIsLocal r) + else if (Git.repoIsLocal r) then return 100 else return 200 where - config g r = gitConfig g (configkey r) "" - configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-cost" + config g r = Git.configGet g (configkey r) "" + configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function ensures that it is - read for a specified remote, and updates state. It returns the - updated git repo also. -} -remoteEnsureGitConfigRead :: GitRepo -> Annex GitRepo +remoteEnsureGitConfigRead :: Git.Repo -> Annex Git.Repo remoteEnsureGitConfigRead r = do - if (Map.null $ gitConfigMap r) + if (Map.null $ Git.configMap r) then do - r' <- liftIO $ gitConfigRead r + r' <- liftIO $ Git.configRead r g <- gitAnnex - let l = gitRepoRemotes g - let g' = gitRepoRemotesAdd g $ exchange l r' + let l = Git.remotes g + let g' = Git.remotesAdd g $ exchange l r' gitAnnexChange g' return r' else return r where exchange [] new = [] exchange (old:ls) new = - if ((gitRepoRemoteName old) == (gitRepoRemoteName new)) + if ((Git.repoRemoteName old) == (Git.repoRemoteName new)) then new:(exchange ls new) else old:(exchange ls new) diff --git a/Types.hs b/Types.hs index ce377dda11..c9d33affdd 100644 --- a/Types.hs +++ b/Types.hs @@ -1,26 +1,14 @@ {- git-annex core data types -} -module Types ( - Annex, - AnnexState, - makeAnnexState, - runAnnexState, - gitAnnex, - gitAnnexChange, - backendsAnnex, - backendsAnnexChange, - - Key(..), - Backend(..) -) where +module Types where import Control.Monad.State import Data.String.Utils -import GitRepo +import qualified GitRepo as Git -- git-annex's runtime state data AnnexState = AnnexState { - repo :: GitRepo, + repo :: Git.Repo, backends :: [Backend] } deriving (Show) @@ -28,18 +16,18 @@ data AnnexState = AnnexState { type Annex = StateT AnnexState IO -- constructor -makeAnnexState :: GitRepo -> AnnexState +makeAnnexState :: Git.Repo -> AnnexState makeAnnexState g = AnnexState { repo = g, backends = [] } -- performs an action in the Annex monad runAnnexState state action = runStateT (action) state -- Annex monad state accessors -gitAnnex :: Annex GitRepo +gitAnnex :: Annex Git.Repo gitAnnex = do state <- get return (repo state) -gitAnnexChange :: GitRepo -> Annex () +gitAnnexChange :: Git.Repo -> Annex () gitAnnexChange r = do state <- get put state { repo = r } diff --git a/UUID.hs b/UUID.hs index f334afdc9d..9c8b23a96a 100644 --- a/UUID.hs +++ b/UUID.hs @@ -19,7 +19,7 @@ import Maybe import List import System.Cmd.Utils import System.IO -import GitRepo +import qualified GitRepo as Git import AbstractTypes type UUID = String @@ -37,17 +37,17 @@ genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h - remote..annex-uuid - - -} -getUUID :: GitRepo -> Annex UUID +getUUID :: Git.Repo -> Annex UUID getUUID r = do if ("" /= configured r) then return $ configured r else cached r where - configured r = gitConfig r "annex.uuid" "" + configured r = Git.configGet r "annex.uuid" "" cached r = do g <- gitAnnex - return $ gitConfig g (configkey r) "" - configkey r = "remote." ++ (gitRepoRemoteName r) ++ ".annex-uuid" + return $ Git.configGet g (configkey r) "" + configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -57,15 +57,15 @@ prepUUID = do if ("" == u) then do uuid <- genUUID - liftIO $ gitRun g ["config", configkey, uuid] + liftIO $ Git.run g ["config", configkey, uuid] -- re-read git config and update the repo's state - u' <- liftIO $ gitConfigRead g + u' <- liftIO $ Git.configRead g gitAnnexChange u' return () else return () {- Filters a list of repos to ones that have listed UUIDs. -} -reposByUUID :: [GitRepo] -> [UUID] -> Annex [GitRepo] +reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] reposByUUID repos uuids = do filterM match repos where diff --git a/git-annex.hs b/git-annex.hs index be5168755f..2cf1c5305d 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -12,7 +12,7 @@ import Annex main = do args <- getArgs (mode, params) <- argvToMode args - state <- startAnnex + state <- start tryRun state mode 0 0 params {- Processes each param in the list by dispatching the handler function From 4c1d8b9689043c18214b1da7d5c145fb0859443d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:41:54 -0400 Subject: [PATCH 0114/8313] more namespace cleanup --- Annex.hs | 10 +++++----- BackendFile.hs | 8 ++++---- Remotes.hs | 18 +++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index 54f9b9dff8..c26baabef5 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,7 +20,7 @@ import List import qualified GitRepo as Git import Utility import Locations -import Backend +import qualified Backend import BackendList import UUID import LocationLog @@ -46,7 +46,7 @@ start = do prepUUID inBackend file yes no = do - r <- liftIO $ lookupFile file + r <- liftIO $ Backend.lookupFile file case (r) of Just v -> yes v Nothing -> no @@ -57,7 +57,7 @@ notinBackend file yes no = inBackend file no yes annexCmd :: FilePath -> Annex () annexCmd file = inBackend file err $ do liftIO $ checkLegal file - stored <- storeFile file + stored <- Backend.storeFile file g <- gitAnnex case (stored) of Nothing -> error $ "no backend could store: " ++ file @@ -93,7 +93,7 @@ annexCmd file = inBackend file err $ do {- Inverse of annexCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - dropFile backend key + Backend.dropFile backend key logStatus key ValueMissing g <- gitAnnex let src = annexLocation g backend key @@ -121,7 +121,7 @@ getCmd file = notinBackend file err $ \(key, backend) -> do g <- gitAnnex let dest = annexLocation g backend key liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- retrieveFile backend key dest + success <- Backend.retrieveFile backend key dest if (success) then do logStatus key ValuePresent diff --git a/BackendFile.hs b/BackendFile.hs index 6c1dc06233..a0396f51da 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -10,7 +10,7 @@ import Control.Exception import Types import LocationLog import Locations -import Remotes +import qualified Remotes import qualified GitRepo as Git backend = Backend { @@ -39,17 +39,17 @@ dummyRemove url = return False - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do - remotes <- remotesWithKey key + remotes <- Remotes.withKey key trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ - (remotesList full) + (Remotes.list full) trycopy full (r:rs) = do -- annexLocation needs the git config to have been -- read for a remote, so do that now, -- if it hasn't been already - r' <- remoteEnsureGitConfigRead r + r' <- Remotes.ensureGitConfigRead r result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) case (result) of Left err -> do diff --git a/Remotes.hs b/Remotes.hs index 39404bf191..918ae2290c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -1,9 +1,9 @@ {- git-annex remote repositories -} module Remotes ( - remotesList, - remotesWithKey, - remoteEnsureGitConfigRead + list, + withKey, + ensureGitConfigRead ) where import Control.Monad.State (liftIO) @@ -17,12 +17,12 @@ import UUID import List {- Human visible list of remotes. -} -remotesList :: [Git.Repo] -> String -remotesList remotes = join " " $ map Git.repoDescribe remotes +list :: [Git.Repo] -> String +list remotes = join " " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -remotesWithKey :: Key -> Annex [Git.Repo] -remotesWithKey key = do +withKey :: Key -> Annex [Git.Repo] +withKey key = do g <- gitAnnex uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost @@ -71,8 +71,8 @@ repoCost r = do - because reading it may be expensive. This function ensures that it is - read for a specified remote, and updates state. It returns the - updated git repo also. -} -remoteEnsureGitConfigRead :: Git.Repo -> Annex Git.Repo -remoteEnsureGitConfigRead r = do +ensureGitConfigRead :: Git.Repo -> Annex Git.Repo +ensureGitConfigRead r = do if (Map.null $ Git.configMap r) then do r' <- liftIO $ Git.configRead r From 0b55bd05de7b83a474ea58e9d45676934667f4bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 02:52:17 -0400 Subject: [PATCH 0115/8313] more namespace cleanup --- AbstractTypes.hs | 32 +++++++++++++++++++++++++++++++- Backend.hs | 3 ++- BackendChecksum.hs | 2 +- BackendFile.hs | 2 +- BackendList.hs | 2 +- Types.hs => BackendTypes.hs | 37 +++++++------------------------------ BackendUrl.hs | 2 +- Locations.hs | 8 +++++--- 8 files changed, 49 insertions(+), 39 deletions(-) rename Types.hs => BackendTypes.hs (55%) diff --git a/AbstractTypes.hs b/AbstractTypes.hs index 510a37f0cc..935d1de2f5 100644 --- a/AbstractTypes.hs +++ b/AbstractTypes.hs @@ -14,4 +14,34 @@ module AbstractTypes ( Backend ) where -import Types +import Control.Monad.State +import qualified GitRepo as Git +import BackendTypes + +-- constructor +makeAnnexState :: Git.Repo -> AnnexState +makeAnnexState g = AnnexState { repo = g, backends = [] } + +-- performs an action in the Annex monad +runAnnexState state action = runStateT (action) state + +-- Annex monad state accessors +gitAnnex :: Annex Git.Repo +gitAnnex = do + state <- get + return (repo state) +gitAnnexChange :: Git.Repo -> Annex () +gitAnnexChange r = do + state <- get + put state { repo = r } + return () +backendsAnnex :: Annex [Backend] +backendsAnnex = do + state <- get + return (backends state) +backendsAnnexChange :: [Backend] -> Annex () +backendsAnnexChange b = do + state <- get + put state { backends = b } + return () + diff --git a/Backend.hs b/Backend.hs index 1bd4efc1e8..251e436c75 100644 --- a/Backend.hs +++ b/Backend.hs @@ -30,7 +30,8 @@ import BackendList import Locations import qualified GitRepo as Git import Utility -import Types +import AbstractTypes +import BackendTypes {- Attempts to store a file in one of the backends. -} storeFile :: FilePath -> Annex (Maybe (Key, Backend)) diff --git a/BackendChecksum.hs b/BackendChecksum.hs index c6e68ffed4..50ef2ae6f6 100644 --- a/BackendChecksum.hs +++ b/BackendChecksum.hs @@ -5,7 +5,7 @@ module BackendChecksum (backend) where import qualified BackendFile import Data.Digest.Pure.SHA -import Types +import BackendTypes -- based on BackendFile just with a different key type backend = BackendFile.backend { diff --git a/BackendFile.hs b/BackendFile.hs index a0396f51da..284daca88e 100644 --- a/BackendFile.hs +++ b/BackendFile.hs @@ -7,7 +7,7 @@ import Control.Monad.State import System.IO import System.Cmd import Control.Exception -import Types +import BackendTypes import LocationLog import Locations import qualified Remotes diff --git a/BackendList.hs b/BackendList.hs index 104444dc20..91a2fa7fc3 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -7,7 +7,7 @@ module BackendList ( lookupBackendName ) where -import Types +import BackendTypes -- When adding a new backend, import it here and add it to the list. import qualified BackendFile diff --git a/Types.hs b/BackendTypes.hs similarity index 55% rename from Types.hs rename to BackendTypes.hs index c9d33affdd..2ef65f4691 100644 --- a/Types.hs +++ b/BackendTypes.hs @@ -1,12 +1,16 @@ -{- git-annex core data types -} +{- git-annex backend data types + - + - Mostly only backend implementations should need to import this. + -} -module Types where +module BackendTypes where import Control.Monad.State import Data.String.Utils import qualified GitRepo as Git --- git-annex's runtime state +-- git-annex's runtime state type doesn't really belong here, +-- but it uses Backend, so has to be here to avoid a depends loop. data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend] @@ -15,33 +19,6 @@ data AnnexState = AnnexState { -- git-annex's monad type Annex = StateT AnnexState IO --- constructor -makeAnnexState :: Git.Repo -> AnnexState -makeAnnexState g = AnnexState { repo = g, backends = [] } - --- performs an action in the Annex monad -runAnnexState state action = runStateT (action) state - --- Annex monad state accessors -gitAnnex :: Annex Git.Repo -gitAnnex = do - state <- get - return (repo state) -gitAnnexChange :: Git.Repo -> Annex () -gitAnnexChange r = do - state <- get - put state { repo = r } - return () -backendsAnnex :: Annex [Backend] -backendsAnnex = do - state <- get - return (backends state) -backendsAnnexChange :: [Backend] -> Annex () -backendsAnnexChange b = do - state <- get - put state { backends = b } - return () - -- annexed filenames are mapped into keys data Key = Key String deriving (Eq) diff --git a/BackendUrl.hs b/BackendUrl.hs index 43b0bc75a8..fc0a8ae586 100644 --- a/BackendUrl.hs +++ b/BackendUrl.hs @@ -6,7 +6,7 @@ module BackendUrl (backend) where import Control.Monad.State import System.Cmd import IO -import Types +import BackendTypes backend = Backend { name = "url", diff --git a/Locations.hs b/Locations.hs index 5d701681c1..8c1915b021 100644 --- a/Locations.hs +++ b/Locations.hs @@ -11,7 +11,8 @@ module Locations ( ) where import Data.String.Utils -import Types +import AbstractTypes +import qualified BackendTypes as Backend import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex @@ -32,7 +33,7 @@ annexLocation r backend key = {- Annexed file's location relative to the gitWorkTree -} annexLocationRelative :: Git.Repo -> Backend -> Key -> FilePath annexLocationRelative r backend key = - Git.dir r ++ "/annex/" ++ (name backend) ++ "/" ++ (keyFile key) + Git.dir r ++ "/annex/" ++ (Backend.name backend) ++ "/" ++ (keyFile key) {- Converts a key into a filename fragment. - @@ -50,4 +51,5 @@ keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Key -fileKey file = Key $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file +fileKey file = Backend.Key $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From 6f3572e47f57bbe5cc76b58c8bcdc9c6c455dce0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:18:11 -0400 Subject: [PATCH 0116/8313] more reorg, spiffed up state monad --- AbstractTypes.hs | 47 ----------- Annex.hs | 215 ++++++++--------------------------------------- Backend.hs | 7 +- CmdLine.hs | 4 +- Commands.hs | 189 +++++++++++++++++++++++++++++++++++++++++ LocationLog.hs | 2 +- Locations.hs | 2 +- Remotes.hs | 13 +-- Types.hs | 10 +++ UUID.hs | 11 +-- git-annex.hs | 8 +- 11 files changed, 259 insertions(+), 249 deletions(-) delete mode 100644 AbstractTypes.hs create mode 100644 Commands.hs create mode 100644 Types.hs diff --git a/AbstractTypes.hs b/AbstractTypes.hs deleted file mode 100644 index 935d1de2f5..0000000000 --- a/AbstractTypes.hs +++ /dev/null @@ -1,47 +0,0 @@ -{- git-annex data types, abstract only -} - -module AbstractTypes ( - Annex, - AnnexState, - makeAnnexState, - runAnnexState, - gitAnnex, - gitAnnexChange, - backendsAnnex, - backendsAnnexChange, - - Key, - Backend -) where - -import Control.Monad.State -import qualified GitRepo as Git -import BackendTypes - --- constructor -makeAnnexState :: Git.Repo -> AnnexState -makeAnnexState g = AnnexState { repo = g, backends = [] } - --- performs an action in the Annex monad -runAnnexState state action = runStateT (action) state - --- Annex monad state accessors -gitAnnex :: Annex Git.Repo -gitAnnex = do - state <- get - return (repo state) -gitAnnexChange :: Git.Repo -> Annex () -gitAnnexChange r = do - state <- get - put state { repo = r } - return () -backendsAnnex :: Annex [Backend] -backendsAnnex = do - state <- get - return (backends state) -backendsAnnexChange :: [Backend] -> Annex () -backendsAnnexChange b = do - state <- get - put state { backends = b } - return () - diff --git a/Annex.hs b/Annex.hs index c26baabef5..fcd19ba033 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,189 +1,42 @@ -{- git-annex toplevel code - -} +{- git-annex monad -} module Annex ( - start, - annexCmd, - unannexCmd, - getCmd, - wantCmd, - dropCmd, - pushCmd, - pullCmd + new, + run, + gitRepo, + gitRepoChange, + backends, + backendsChange, ) where -import Control.Monad.State (liftIO) -import System.Posix.Files -import System.Directory -import Data.String.Utils -import List +import Control.Monad.State import qualified GitRepo as Git -import Utility -import Locations -import qualified Backend -import BackendList -import UUID -import LocationLog -import AbstractTypes +import Types +import qualified BackendTypes as Backend -{- Create and returns an Annex state object. - - Examines and prepares the git repo. - -} -start :: IO AnnexState -start = do - g <- Git.repoFromCwd - let s = makeAnnexState g - (_,s') <- runAnnexState s (prep g) - return s' - where - prep g = do - -- setup git and read its config; update state - g' <- liftIO $ Git.configRead g - gitAnnexChange g' - liftIO $ gitSetup g' - backendsAnnexChange $ parseBackendList $ - Git.configGet g' "annex.backends" "" - prepUUID +-- constructor +new :: Git.Repo -> AnnexState +new g = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } -inBackend file yes no = do - r <- liftIO $ Backend.lookupFile file - case (r) of - Just v -> yes v - Nothing -> no -notinBackend file yes no = inBackend file no yes +-- performs an action in the Annex monad +run state action = runStateT (action) state -{- Annexes a file, storing it in a backend, and then moving it into - - the annex directory and setting up the symlink pointing to its content. -} -annexCmd :: FilePath -> Annex () -annexCmd file = inBackend file err $ do - liftIO $ checkLegal file - stored <- Backend.storeFile file - g <- gitAnnex - case (stored) of - Nothing -> error $ "no backend could store: " ++ file - Just (key, backend) -> do - logStatus key ValuePresent - liftIO $ setup g key backend - where - err = error $ "already annexed " ++ file - checkLegal file = do - s <- getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) - then error $ "not a regular file: " ++ file - else return () - setup g key backend = do - let dest = annexLocation g backend key - let reldest = annexLocationRelative g backend key - createDirectoryIfMissing True (parentDir dest) - renameFile file dest - createSymbolicLink ((linkTarget file) ++ reldest) file - Git.run g ["add", file] - Git.run g ["commit", "-m", - ("git-annex annexed " ++ file), file] - linkTarget file = - -- relies on file being relative to the top of the - -- git repo; just replace each subdirectory with ".." - if (subdirs > 0) - then (join "/" $ take subdirs $ repeat "..") ++ "/" - else "" - where - subdirs = (length $ split "/" file) - 1 - - -{- Inverse of annexCmd. -} -unannexCmd :: FilePath -> Annex () -unannexCmd file = notinBackend file err $ \(key, backend) -> do - Backend.dropFile backend key - logStatus key ValueMissing - g <- gitAnnex - let src = annexLocation g backend key - liftIO $ moveout g src - where - err = error $ "not annexed " ++ file - moveout g src = do - removeFile file - Git.run g ["rm", file] - Git.run g ["commit", "-m", - ("git-annex unannexed " ++ file), file] - -- git rm deletes empty directories; - -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file - return () - -{- Gets an annexed file from one of the backends. -} -getCmd :: FilePath -> Annex () -getCmd file = notinBackend file err $ \(key, backend) -> do - inannex <- inAnnex backend key - if (inannex) - then return () - else do - g <- gitAnnex - let dest = annexLocation g backend key - liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- Backend.retrieveFile backend key dest - if (success) - then do - logStatus key ValuePresent - return () - else error $ "failed to get " ++ file - where - err = error $ "not annexed " ++ file - -{- Indicates a file is wanted. -} -wantCmd :: FilePath -> Annex () -wantCmd file = do error "not implemented" -- TODO - -{- Indicates a file is not wanted. -} -dropCmd :: FilePath -> Annex () -dropCmd file = do error "not implemented" -- TODO - -{- Pushes all files to a remote repository. -} -pushCmd :: String -> Annex () -pushCmd reponame = do error "not implemented" -- TODO - -{- Pulls all files from a remote repository. -} -pullCmd :: String -> Annex () -pullCmd reponame = do error "not implemented" -- TODO - -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Git.Repo -> IO () -gitSetup repo = do - -- configure git to use union merge driver on state files - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () - where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] - -{- Updates the LocationLog when a key's presence changes. -} -logStatus :: Key -> LogStatus -> Annex () -logStatus key status = do - g <- gitAnnex - u <- getUUID g - f <- liftIO $ logChange g key u status - liftIO $ commit g f - where - commit g f = do - Git.run g ["add", f] - Git.run g ["commit", "-m", "git-annex log update", f] - -{- Checks if a given key is currently present in the annexLocation -} -inAnnex :: Backend -> Key -> Annex Bool -inAnnex backend key = do - g <- gitAnnex - liftIO $ doesFileExist $ annexLocation g backend key +-- Annex monad state accessors +gitRepo :: Annex Git.Repo +gitRepo = do + state <- get + return (Backend.repo state) +gitRepoChange :: Git.Repo -> Annex () +gitRepoChange r = do + state <- get + put state { Backend.repo = r } + return () +backends :: Annex [Backend] +backends = do + state <- get + return (Backend.backends state) +backendsChange :: [Backend] -> Annex () +backendsChange b = do + state <- get + put state { Backend.backends = b } + return () diff --git a/Backend.hs b/Backend.hs index 251e436c75..2829fef9d2 100644 --- a/Backend.hs +++ b/Backend.hs @@ -29,16 +29,17 @@ import System.Posix.Files import BackendList import Locations import qualified GitRepo as Git +import qualified Annex import Utility -import AbstractTypes +import Types import BackendTypes {- Attempts to store a file in one of the backends. -} storeFile :: FilePath -> Annex (Maybe (Key, Backend)) storeFile file = do - g <- gitAnnex + g <- Annex.gitRepo let relfile = Git.relative g file - b <- backendsAnnex + b <- Annex.backends storeFile' b file relfile storeFile' [] _ _ = return Nothing storeFile' (b:bs) file relfile = do diff --git a/CmdLine.hs b/CmdLine.hs index 479be7e8b5..9737e0eb01 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,8 +11,8 @@ module CmdLine ( ) where import System.Console.GetOpt -import AbstractTypes -import Annex +import Types +import Commands data Mode = Add | Push | Pull | Want | Get | Drop | Unannex deriving Show diff --git a/Commands.hs b/Commands.hs new file mode 100644 index 0000000000..98e65b126a --- /dev/null +++ b/Commands.hs @@ -0,0 +1,189 @@ +{- git-annex subcommands -} + +module Commands ( + start, + annexCmd, + unannexCmd, + getCmd, + wantCmd, + dropCmd, + pushCmd, + pullCmd +) where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory +import Data.String.Utils +import List +import qualified GitRepo as Git +import qualified Annex +import Utility +import Locations +import qualified Backend +import BackendList +import UUID +import LocationLog +import Types + +{- Create and returns an Annex state object. + - Examines and prepares the git repo. + -} +start :: IO AnnexState +start = do + g <- Git.repoFromCwd + let s = Annex.new g + (_,s') <- Annex.run s (prep g) + return s' + where + prep g = do + -- setup git and read its config; update state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' + liftIO $ gitSetup g' + Annex.backendsChange $ parseBackendList $ + Git.configGet g' "annex.backends" "" + prepUUID + +inBackend file yes no = do + r <- liftIO $ Backend.lookupFile file + case (r) of + Just v -> yes v + Nothing -> no +notinBackend file yes no = inBackend file no yes + +{- Annexes a file, storing it in a backend, and then moving it into + - the annex directory and setting up the symlink pointing to its content. -} +annexCmd :: FilePath -> Annex () +annexCmd file = inBackend file err $ do + liftIO $ checkLegal file + stored <- Backend.storeFile file + g <- Annex.gitRepo + case (stored) of + Nothing -> error $ "no backend could store: " ++ file + Just (key, backend) -> do + logStatus key ValuePresent + liftIO $ setup g key backend + where + err = error $ "already annexed " ++ file + checkLegal file = do + s <- getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then error $ "not a regular file: " ++ file + else return () + setup g key backend = do + let dest = annexLocation g backend key + let reldest = annexLocationRelative g backend key + createDirectoryIfMissing True (parentDir dest) + renameFile file dest + createSymbolicLink ((linkTarget file) ++ reldest) file + Git.run g ["add", file] + Git.run g ["commit", "-m", + ("git-annex annexed " ++ file), file] + linkTarget file = + -- relies on file being relative to the top of the + -- git repo; just replace each subdirectory with ".." + if (subdirs > 0) + then (join "/" $ take subdirs $ repeat "..") ++ "/" + else "" + where + subdirs = (length $ split "/" file) - 1 + + +{- Inverse of annexCmd. -} +unannexCmd :: FilePath -> Annex () +unannexCmd file = notinBackend file err $ \(key, backend) -> do + Backend.dropFile backend key + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g backend key + liftIO $ moveout g src + where + err = error $ "not annexed " ++ file + moveout g src = do + removeFile file + Git.run g ["rm", file] + Git.run g ["commit", "-m", + ("git-annex unannexed " ++ file), file] + -- git rm deletes empty directories; + -- put them back + createDirectoryIfMissing True (parentDir file) + renameFile src file + return () + +{- Gets an annexed file from one of the backends. -} +getCmd :: FilePath -> Annex () +getCmd file = notinBackend file err $ \(key, backend) -> do + inannex <- inAnnex backend key + if (inannex) + then return () + else do + g <- Annex.gitRepo + let dest = annexLocation g backend key + liftIO $ createDirectoryIfMissing True (parentDir dest) + success <- Backend.retrieveFile backend key dest + if (success) + then do + logStatus key ValuePresent + return () + else error $ "failed to get " ++ file + where + err = error $ "not annexed " ++ file + +{- Indicates a file is wanted. -} +wantCmd :: FilePath -> Annex () +wantCmd file = do error "not implemented" -- TODO + +{- Indicates a file is not wanted. -} +dropCmd :: FilePath -> Annex () +dropCmd file = do error "not implemented" -- TODO + +{- Pushes all files to a remote repository. -} +pushCmd :: String -> Annex () +pushCmd reponame = do error "not implemented" -- TODO + +{- Pulls all files from a remote repository. -} +pullCmd :: String -> Annex () +pullCmd reponame = do error "not implemented" -- TODO + +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitSetup :: Git.Repo -> IO () +gitSetup repo = do + -- configure git to use union merge driver on state files + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + commit + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] + +{- Updates the LocationLog when a key's presence changes. -} +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = do + g <- Annex.gitRepo + u <- getUUID g + f <- liftIO $ logChange g key u status + liftIO $ commit g f + where + commit g f = do + Git.run g ["add", f] + Git.run g ["commit", "-m", "git-annex log update", f] + +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: Backend -> Key -> Annex Bool +inAnnex backend key = do + g <- Annex.gitRepo + liftIO $ doesFileExist $ annexLocation g backend key diff --git a/LocationLog.hs b/LocationLog.hs index 7953b345f8..ba91787049 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,7 +32,7 @@ import Data.Char import qualified GitRepo as Git import Utility import UUID -import AbstractTypes +import Types import Locations data LogLine = LogLine { diff --git a/Locations.hs b/Locations.hs index 8c1915b021..7b8beb14f0 100644 --- a/Locations.hs +++ b/Locations.hs @@ -11,7 +11,7 @@ module Locations ( ) where import Data.String.Utils -import AbstractTypes +import Types import qualified BackendTypes as Backend import qualified GitRepo as Git diff --git a/Remotes.hs b/Remotes.hs index 918ae2290c..1802ff28eb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -9,8 +9,9 @@ module Remotes ( import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils -import AbstractTypes +import Types import qualified GitRepo as Git +import qualified Annex import LocationLog import Locations import UUID @@ -23,7 +24,7 @@ list remotes = join " " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} withKey :: Key -> Annex [Git.Repo] withKey key = do - g <- gitAnnex + g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost remotes <- reposByUUID allremotes uuids @@ -36,7 +37,7 @@ withKey key = do {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] remotesByCost = do - g <- gitAnnex + g <- Annex.gitRepo reposByCost $ Git.remotes g {- Orders a list of git repos by cost. -} @@ -57,7 +58,7 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - g <- gitAnnex + g <- Annex.gitRepo if ((length $ config g r) > 0) then return $ read $ config g r else if (Git.repoIsLocal r) @@ -76,10 +77,10 @@ ensureGitConfigRead r = do if (Map.null $ Git.configMap r) then do r' <- liftIO $ Git.configRead r - g <- gitAnnex + g <- Annex.gitRepo let l = Git.remotes g let g' = Git.remotesAdd g $ exchange l r' - gitAnnexChange g' + Annex.gitRepoChange g' return r' else return r where diff --git a/Types.hs b/Types.hs new file mode 100644 index 0000000000..4262ed567d --- /dev/null +++ b/Types.hs @@ -0,0 +1,10 @@ +{- git-annex abstract data types -} + +module Types ( + Annex, + AnnexState, + Key, + Backend +) where + +import BackendTypes diff --git a/UUID.hs b/UUID.hs index 9c8b23a96a..1c31a343fa 100644 --- a/UUID.hs +++ b/UUID.hs @@ -20,7 +20,8 @@ import List import System.Cmd.Utils import System.IO import qualified GitRepo as Git -import AbstractTypes +import Types +import qualified Annex type UUID = String @@ -45,22 +46,22 @@ getUUID r = do where configured r = Git.configGet r "annex.uuid" "" cached r = do - g <- gitAnnex + g <- Annex.gitRepo return $ Git.configGet g (configkey r) "" configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () prepUUID = do - g <- gitAnnex + g <- Annex.gitRepo u <- getUUID g if ("" == u) then do uuid <- genUUID liftIO $ Git.run g ["config", configkey, uuid] -- re-read git config and update the repo's state - u' <- liftIO $ Git.configRead g - gitAnnexChange u' + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' return () else return () diff --git a/git-annex.hs b/git-annex.hs index 2cf1c5305d..ce3b2ac428 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,8 +6,9 @@ import System.IO import System.Environment import Control.Exception import CmdLine -import AbstractTypes -import Annex +import Types +import Commands +import qualified Annex main = do args <- getArgs @@ -30,7 +31,8 @@ tryRun state mode errnum oknum [] = do then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () tryRun state mode errnum oknum (f:fs) = do - result <- try (runAnnexState state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + result <- try + (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err From f407f23a54d9152a382ee8e48629f40e1a72a26f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:40:26 -0400 Subject: [PATCH 0117/8313] more refactor --- Commands.hs | 58 +++++------------------------------ Core.hs | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++ git-annex.hs | 41 +++---------------------- 3 files changed, 98 insertions(+), 88 deletions(-) create mode 100644 Core.hs diff --git a/Commands.hs b/Commands.hs index 98e65b126a..be61c7c64e 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,7 +1,6 @@ {- git-annex subcommands -} module Commands ( - start, annexCmd, unannexCmd, getCmd, @@ -26,32 +25,6 @@ import UUID import LocationLog import Types -{- Create and returns an Annex state object. - - Examines and prepares the git repo. - -} -start :: IO AnnexState -start = do - g <- Git.repoFromCwd - let s = Annex.new g - (_,s') <- Annex.run s (prep g) - return s' - where - prep g = do - -- setup git and read its config; update state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' - liftIO $ gitSetup g' - Annex.backendsChange $ parseBackendList $ - Git.configGet g' "annex.backends" "" - prepUUID - -inBackend file yes no = do - r <- liftIO $ Backend.lookupFile file - case (r) of - Just v -> yes v - Nothing -> no -notinBackend file yes no = inBackend file no yes - {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} annexCmd :: FilePath -> Annex () @@ -146,30 +119,6 @@ pushCmd reponame = do error "not implemented" -- TODO pullCmd :: String -> Annex () pullCmd reponame = do error "not implemented" -- TODO -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Git.Repo -> IO () -gitSetup repo = do - -- configure git to use union merge driver on state files - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () - where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] - {- Updates the LocationLog when a key's presence changes. -} logStatus :: Key -> LogStatus -> Annex () logStatus key status = do @@ -182,6 +131,13 @@ logStatus key status = do Git.run g ["add", f] Git.run g ["commit", "-m", "git-annex log update", f] +inBackend file yes no = do + r <- liftIO $ Backend.lookupFile file + case (r) of + Just v -> yes v + Nothing -> no +notinBackend file yes no = inBackend file no yes + {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool inAnnex backend key = do diff --git a/Core.hs b/Core.hs new file mode 100644 index 0000000000..e3d2c64039 --- /dev/null +++ b/Core.hs @@ -0,0 +1,87 @@ +{- git-annex core functions -} + +module Core where + +import System.IO +import System.Directory +import Control.Monad.State (liftIO) +import Control.Exception +import CmdLine +import Types +import BackendList +import Locations +import UUID +import qualified GitRepo as Git +import qualified Annex + +{- Create and returns an Annex state object. + - Examines and prepares the git repo. + -} +start :: IO AnnexState +start = do + g <- Git.repoFromCwd + let s = Annex.new g + (_,s') <- Annex.run s (prep g) + return s' + where + prep g = do + -- setup git and read its config; update state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' + liftIO $ gitSetup g' + Annex.backendsChange $ parseBackendList $ + Git.configGet g' "annex.backends" "" + prepUUID + +{- Processes each param in the list by dispatching the handler function + - for the user-selection operation mode. Catches exceptions, not stopping + - if some error out, and propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> Mode -> [String] -> IO () +tryRun state mode params = tryRun' state mode 0 0 params +tryRun' state mode errnum oknum [] = do + if (errnum > 0) + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" + else return () +tryRun' state mode errnum oknum (f:fs) = do + result <- try + (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + case (result) of + Left err -> do + showErr err + tryRun' state mode (errnum + 1) oknum fs + Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs + +{- Exception pretty-printing. -} +showErr e = do + hPutStrLn stderr $ "git-annex: " ++ (show e) + return () + +{- Sets up a git repo for git-annex. May be called repeatedly. -} +gitSetup :: Git.Repo -> IO () +gitSetup repo = do + -- configure git to use union merge driver on state files + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + commit + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] diff --git a/git-annex.hs b/git-annex.hs index ce3b2ac428..b326b2b199 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,45 +1,12 @@ -{- git-annex main program - - -} +{- git-annex main program -} -import Control.Monad.State -import System.IO import System.Environment -import Control.Exception -import CmdLine -import Types -import Commands import qualified Annex +import Core +import CmdLine main = do args <- getArgs (mode, params) <- argvToMode args state <- start - tryRun state mode 0 0 params - -{- Processes each param in the list by dispatching the handler function - - for the user-selection operation mode. Catches exceptions, not stopping - - if some error out, and propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. - -} -tryRun :: AnnexState -> Mode -> Int -> Int -> [String] -> IO () -tryRun state mode errnum oknum [] = do - if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" - else return () -tryRun state mode errnum oknum (f:fs) = do - result <- try - (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) - case (result) of - Left err -> do - showErr err - tryRun state mode (errnum + 1) oknum fs - Right (_,state') -> tryRun state' mode errnum (oknum + 1) fs - -{- Exception pretty-printing. -} -showErr e = do - hPutStrLn stderr $ "git-annex: " ++ (show e) - return () + tryRun state mode params From 50630840ee6802fef9db136505975db40a81920a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:46:34 -0400 Subject: [PATCH 0118/8313] build in subdir --- .gitignore | 5 +---- Makefile | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 7dd8869b16..2b3e3aef1f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,2 @@ -*.o -*.hi -*.ho -*.a +build/* git-annex diff --git a/Makefile b/Makefile index 8b7c9d3a0e..876407de09 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ git-annex: - ghc --make git-annex + mkdir -p build + ghc -odir build -hidir build --make git-annex clean: - rm -f git-annex *.o *.hi *.ho *.a + rm -rf build git-annex .PHONY: git-annex From 7117702fddf521ed4f3675a91cd87119207eba02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:47:06 -0400 Subject: [PATCH 0119/8313] foo --- .gitattributes | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index b98b07d7d2..0000000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -.git-annex/*.log merge=union From 0f12bd16d829432f7b1c2efbba386262ed36fc27 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 03:50:28 -0400 Subject: [PATCH 0120/8313] subdir --- BackendChecksum.hs => Backend/Checksum.hs | 6 +++--- BackendFile.hs => Backend/File.hs | 2 +- BackendUrl.hs => Backend/Url.hs | 2 +- BackendList.hs | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) rename BackendChecksum.hs => Backend/Checksum.hs (75%) rename BackendFile.hs => Backend/File.hs (98%) rename BackendUrl.hs => Backend/Url.hs (95%) diff --git a/BackendChecksum.hs b/Backend/Checksum.hs similarity index 75% rename from BackendChecksum.hs rename to Backend/Checksum.hs index 50ef2ae6f6..bfc789e40e 100644 --- a/BackendChecksum.hs +++ b/Backend/Checksum.hs @@ -1,14 +1,14 @@ {- git-annex "checksum" backend - -} -module BackendChecksum (backend) where +module Backend.Checksum (backend) where -import qualified BackendFile +import qualified Backend.File import Data.Digest.Pure.SHA import BackendTypes -- based on BackendFile just with a different key type -backend = BackendFile.backend { +backend = Backend.File.backend { name = "checksum", getKey = keyValue } diff --git a/BackendFile.hs b/Backend/File.hs similarity index 98% rename from BackendFile.hs rename to Backend/File.hs index 284daca88e..107ef38517 100644 --- a/BackendFile.hs +++ b/Backend/File.hs @@ -1,7 +1,7 @@ {- git-annex "file" backend - -} -module BackendFile (backend) where +module Backend.File (backend) where import Control.Monad.State import System.IO diff --git a/BackendUrl.hs b/Backend/Url.hs similarity index 95% rename from BackendUrl.hs rename to Backend/Url.hs index fc0a8ae586..e4ba58e6d7 100644 --- a/BackendUrl.hs +++ b/Backend/Url.hs @@ -1,7 +1,7 @@ {- git-annex "url" backend - -} -module BackendUrl (backend) where +module Backend.Url (backend) where import Control.Monad.State import System.Cmd diff --git a/BackendList.hs b/BackendList.hs index 91a2fa7fc3..e9f926ce2a 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,13 +10,13 @@ module BackendList ( import BackendTypes -- When adding a new backend, import it here and add it to the list. -import qualified BackendFile -import qualified BackendChecksum -import qualified BackendUrl +import qualified Backend.File +import qualified Backend.Checksum +import qualified Backend.Url supportedBackends = - [ BackendFile.backend - , BackendChecksum.backend - , BackendUrl.backend + [ Backend.File.backend + , Backend.Checksum.backend + , Backend.Url.backend ] {- Parses a string with a list of backend names into From 282d9853682f457cc6dc8b095b230bd892f0a5f3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 12:36:40 -0400 Subject: [PATCH 0121/8313] default command --- CmdLine.hs | 8 ++++---- Commands.hs | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 9737e0eb01..98971a733b 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -14,7 +14,7 @@ import System.Console.GetOpt import Types import Commands -data Mode = Add | Push | Pull | Want | Get | Drop | Unannex +data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex deriving Show options :: [OptDescr Mode] @@ -30,8 +30,7 @@ options = argvToMode argv = do case getOpt Permute options argv of - -- default mode is Add - ([],files,[]) -> return (Add, files) + ([],files,[]) -> return (Default, files) -- one mode is normal case (m:[],files,[]) -> return (m, files) -- multiple modes is an error @@ -43,7 +42,8 @@ argvToMode argv = do dispatch :: Mode -> FilePath -> Annex () dispatch mode item = do case (mode) of - Add -> annexCmd item + Default -> defaultCmd item + Add -> addCmd item Push -> pushCmd item Pull -> pullCmd item Want -> wantCmd item diff --git a/Commands.hs b/Commands.hs index be61c7c64e..b4f57d6fe4 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,7 +1,8 @@ {- git-annex subcommands -} module Commands ( - annexCmd, + defaultCmd, + addCmd, unannexCmd, getCmd, wantCmd, @@ -25,10 +26,19 @@ import UUID import LocationLog import Types +{- Default mode is to annex a file if it is not already, and otherwise + - get its content. -} +defaultCmd :: FilePath -> Annex () +defaultCmd file = do + r <- liftIO $ Backend.lookupFile file + case (r) of + Just v -> getCmd file + Nothing -> addCmd file + {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} -annexCmd :: FilePath -> Annex () -annexCmd file = inBackend file err $ do +addCmd :: FilePath -> Annex () +addCmd file = inBackend file err $ do liftIO $ checkLegal file stored <- Backend.storeFile file g <- Annex.gitRepo @@ -63,7 +73,7 @@ annexCmd file = inBackend file err $ do subdirs = (length $ split "/" file) - 1 -{- Inverse of annexCmd. -} +{- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do Backend.dropFile backend key From 8df3e2aa0227e426ade1d92f430e02e31bb97ad9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 13:11:42 -0400 Subject: [PATCH 0122/8313] query remotes for uuids (not cached yet) --- Backend/File.hs | 21 ++++++++++++-------- Remotes.hs | 52 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 107ef38517..78e1f55635 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -49,13 +49,16 @@ copyKeyFile key file = do -- annexLocation needs the git config to have been -- read for a remote, so do that now, -- if it hasn't been already - r' <- Remotes.ensureGitConfigRead r - result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) - case (result) of - Left err -> do - liftIO $ hPutStrLn stderr (show err) - trycopy full rs - Right succ -> return True + result <- Remotes.tryGitConfigRead r + case (result) of + Nothing -> trycopy full rs + Just r' -> do + result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) + case (result) of + Left err -> do + liftIO $ hPutStrLn stderr (show err) + trycopy full rs + Right succ -> return True {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () @@ -67,6 +70,8 @@ copyFromRemote r key file = do else getremote return () where - getlocal = rawSystem "cp" ["-a", location, file] + getlocal = do + rawSystem "cp" ["-a", location, file] + putStrLn "cp done" getremote = error "get via network not yet implemented!" location = annexLocation r backend key diff --git a/Remotes.hs b/Remotes.hs index 1802ff28eb..ecb0d96e38 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -3,19 +3,21 @@ module Remotes ( list, withKey, - ensureGitConfigRead + tryGitConfigRead ) where import Control.Monad.State (liftIO) +import IO import qualified Data.Map as Map import Data.String.Utils +import List +import Maybe import Types import qualified GitRepo as Git import qualified Annex import LocationLog import Locations import UUID -import List {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -27,12 +29,25 @@ withKey key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost + -- this only uses cached data, so may not find new remotes remotes <- reposByUUID allremotes uuids if (0 == length remotes) - then error $ "no configured git remotes have: " ++ (keyFile key) ++ "\n" ++ + then tryharder allremotes uuids + else return remotes + where + tryharder allremotes uuids = do + -- more expensive; check each remote's config + mayberemotes <- mapM tryGitConfigRead allremotes + let allremotes' = catMaybes mayberemotes + remotes' <- reposByUUID allremotes' uuids + if (0 == length remotes') + then err uuids + else return remotes' + err uuids = + error $ "no available git remotes have: " ++ + (keyFile key) ++ "\n" ++ "It has been seen before in these repositories:\n" ++ prettyPrintUUIDs uuids - else return remotes {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] @@ -69,20 +84,25 @@ repoCost r = do configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- The git configs for the git repo's remotes is not read on startup - - because reading it may be expensive. This function ensures that it is - - read for a specified remote, and updates state. It returns the - - updated git repo also. -} -ensureGitConfigRead :: Git.Repo -> Annex Git.Repo -ensureGitConfigRead r = do + - because reading it may be expensive. This function tries to read the + - config for a specified remote, and updates state. If successful, it + - returns the updated git repo. -} +tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) +tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do - r' <- liftIO $ Git.configRead r - g <- Annex.gitRepo - let l = Git.remotes g - let g' = Git.remotesAdd g $ exchange l r' - Annex.gitRepoChange g' - return r' - else return r + liftIO $ putStrLn $ "read config for " ++ (show r) + result <- liftIO $ try (Git.configRead r) + case (result) of + Left err -> return Nothing + Right r' -> do + g <- Annex.gitRepo + let l = Git.remotes g + let g' = Git.remotesAdd g $ + exchange l r' + Annex.gitRepoChange g' + return $ Just r' + else return $ Just r where exchange [] new = [] exchange (old:ls) new = From 7c975eab07d842e3d91626871027f803f34c6372 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 13:17:43 -0400 Subject: [PATCH 0123/8313] check rawSystem exit codes --- Backend/File.hs | 8 +++++--- Backend/Url.hs | 10 +++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 78e1f55635..2ac12487e1 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -6,6 +6,7 @@ module Backend.File (backend) where import Control.Monad.State import System.IO import System.Cmd +import System.Exit import Control.Exception import BackendTypes import LocationLog @@ -68,10 +69,11 @@ copyFromRemote r key file = do if (Git.repoIsLocal r) then getlocal else getremote - return () where getlocal = do - rawSystem "cp" ["-a", location, file] - putStrLn "cp done" + res <-rawSystem "cp" ["-a", location, file] + if (res == ExitSuccess) + then return () + else error "cp failed" getremote = error "get via network not yet implemented!" location = annexLocation r backend key diff --git a/Backend/Url.hs b/Backend/Url.hs index e4ba58e6d7..9831c337b8 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -5,7 +5,7 @@ module Backend.Url (backend) where import Control.Monad.State import System.Cmd -import IO +import System.Exit import BackendTypes backend = Backend { @@ -29,7 +29,7 @@ dummyRemove url = return False downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl url file = do liftIO $ putStrLn $ "download: " ++ (show url) - result <- liftIO $ try $ rawSystem "curl" ["-#", "-o", file, (show url)] - case (result) of - Left _ -> return False - Right _ -> return True + result <- liftIO $ rawSystem "curl" ["-#", "-o", file, (show url)] + if (result == ExitSuccess) + then return True + else return False From f9557d7c5e2aa7ef19a5d589594154a21c7f2caa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 13:49:45 -0400 Subject: [PATCH 0124/8313] uuid cache done --- Remotes.hs | 1 - TODO | 2 -- UUID.hs | 42 +++++++++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index ecb0d96e38..4f4e5a26cb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -91,7 +91,6 @@ tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do - liftIO $ putStrLn $ "read config for " ++ (show r) result <- liftIO $ try (Git.configRead r) case (result) of Left err -> return Nothing diff --git a/TODO b/TODO index 40017c816a..54411185a8 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,6 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* query remotes for their annex.uuid settings and cache - * --push/--pull/--want/--drop * how to handle git mv file? diff --git a/UUID.hs b/UUID.hs index 1c31a343fa..c770045270 100644 --- a/UUID.hs +++ b/UUID.hs @@ -40,15 +40,25 @@ genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h - -} getUUID :: Git.Repo -> Annex UUID getUUID r = do - if ("" /= configured r) - then return $ configured r - else cached r + g <- Annex.gitRepo + let uuid = cached r g + if (uuid /= "") + then return $ uuid + else do + let uuid = uncached r + if (uuid /= "") + then do + updatecache r g uuid + return uuid + else return "" where - configured r = Git.configGet r "annex.uuid" "" - cached r = do - g <- Annex.gitRepo - return $ Git.configGet g (configkey r) "" - configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" + uncached r = Git.configGet r "annex.uuid" "" + cached r g = Git.configGet g (cachekey r) "" + updatecache r g uuid = do + if (g /= r) + then setConfig (cachekey r) uuid + else return () + cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -58,13 +68,19 @@ prepUUID = do if ("" == u) then do uuid <- genUUID - liftIO $ Git.run g ["config", configkey, uuid] - -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' - return () + setConfig configkey uuid else return () +{- Changes a git config setting in both internal state and .git/config -} +setConfig :: String -> String -> Annex () +setConfig key value = do + g <- Annex.gitRepo + liftIO $ Git.run g ["config", key, value] + -- re-read git config and update the repo's state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' + return () + {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] reposByUUID repos uuids = do From a200761e66f01a271c90ce67482105befca6ef09 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:14:19 -0400 Subject: [PATCH 0125/8313] implemented basic --drop --- Backend.hs | 34 +++++++++++++++++----------------- Backend/File.hs | 6 ++++-- Backend/Url.hs | 4 +++- Commands.hs | 24 ++++++++++++++++++++---- Remotes.hs | 7 ++++--- TODO | 2 +- 6 files changed, 49 insertions(+), 28 deletions(-) diff --git a/Backend.hs b/Backend.hs index 2829fef9d2..7a8a41a4ba 100644 --- a/Backend.hs +++ b/Backend.hs @@ -14,9 +14,9 @@ - -} module Backend ( - storeFile, - dropFile, - retrieveFile, + storeFileKey, + removeKey, + retrieveKeyFile, lookupFile ) where @@ -32,37 +32,37 @@ import qualified GitRepo as Git import qualified Annex import Utility import Types -import BackendTypes +import qualified BackendTypes as B {- Attempts to store a file in one of the backends. -} -storeFile :: FilePath -> Annex (Maybe (Key, Backend)) -storeFile file = do +storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) +storeFileKey file = do g <- Annex.gitRepo let relfile = Git.relative g file b <- Annex.backends - storeFile' b file relfile -storeFile' [] _ _ = return Nothing -storeFile' (b:bs) file relfile = do - try <- (getKey b) relfile + storeFileKey' b file relfile +storeFileKey' [] _ _ = return Nothing +storeFileKey' (b:bs) file relfile = do + try <- (B.getKey b) relfile case (try) of Nothing -> nextbackend Just key -> do - stored <- (storeFileKey b) file key + stored <- (B.storeFileKey b) file key if (not stored) then nextbackend else do return $ Just (key, b) where - nextbackend = storeFile' bs file relfile + nextbackend = storeFileKey' bs file relfile {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveFile :: Backend -> Key -> FilePath -> Annex Bool -retrieveFile backend key dest = (retrieveKeyFile backend) key dest +retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool +retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest -{- Drops a key from a backend. -} -dropFile :: Backend -> Key -> Annex Bool -dropFile backend key = (removeKey backend) key +{- Removes a key from a backend. -} +removeKey :: Backend -> Key -> Annex Bool +removeKey backend key = (B.removeKey backend) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index 2ac12487e1..311fe820b2 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -28,13 +28,15 @@ keyValue file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - - and other accessible repos. So storing or removing a key is + - and other accessible repos. So storing a key is - a no-op. TODO until support is added for git annex --push otherrepo, - then these could implement that.. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True + +{- Allow keys to be removed. -} dummyRemove :: Key -> Annex Bool -dummyRemove url = return False +dummyRemove url = return True {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} diff --git a/Backend/Url.hs b/Backend/Url.hs index 9831c337b8..3d971864ab 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -23,8 +23,10 @@ keyValue file = return Nothing -- cannot change url contents dummyStore :: FilePath -> Key -> Annex Bool dummyStore file url = return False + +-- allow keys to be removed dummyRemove :: Key -> Annex Bool -dummyRemove url = return False +dummyRemove url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl url file = do diff --git a/Commands.hs b/Commands.hs index b4f57d6fe4..65f6f6efd6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -40,7 +40,7 @@ defaultCmd file = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file - stored <- Backend.storeFile file + stored <- Backend.storeFileKey file g <- Annex.gitRepo case (stored) of Nothing -> error $ "no backend could store: " ++ file @@ -76,7 +76,7 @@ addCmd file = inBackend file err $ do {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - Backend.dropFile backend key + Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo let src = annexLocation g backend key @@ -104,7 +104,7 @@ getCmd file = notinBackend file err $ \(key, backend) -> do g <- Annex.gitRepo let dest = annexLocation g backend key liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- Backend.retrieveFile backend key dest + success <- Backend.retrieveKeyFile backend key dest if (success) then do logStatus key ValuePresent @@ -119,7 +119,23 @@ wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} dropCmd :: FilePath -> Annex () -dropCmd file = do error "not implemented" -- TODO +dropCmd file = notinBackend file err $ \(key, backend) -> do + -- TODO only remove if enough copies are present elsewhere + success <- Backend.removeKey backend key + if (success) + then do + logStatus key ValueMissing + inannex <- inAnnex backend key + if (inannex) + then do + g <- Annex.gitRepo + let loc = annexLocation g backend key + liftIO $ removeFile loc + return () + else return () + else error $ "backend refused to drop " ++ file + where + err = error $ "not annexed " ++ file {- Pushes all files to a remote repository. -} pushCmd :: String -> Annex () diff --git a/Remotes.hs b/Remotes.hs index 4f4e5a26cb..f20d51ab35 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -45,9 +45,10 @@ withKey key = do else return remotes' err uuids = error $ "no available git remotes have: " ++ - (keyFile key) ++ "\n" ++ - "It has been seen before in these repositories:\n" ++ - prettyPrintUUIDs uuids + (keyFile key) ++ (uuidlist uuids) + uuidlist [] = "" + uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ + prettyPrintUUIDs uuids {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] diff --git a/TODO b/TODO index 54411185a8..c4ce74e198 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,7 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* --push/--pull/--want/--drop +* --push/--pull/--want * how to handle git mv file? From 65e4f9cc73f4800fd4dcb5503f7a428539e1e959 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:22:05 -0400 Subject: [PATCH 0126/8313] update cached uuids if it's noticed they changed --- UUID.hs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/UUID.hs b/UUID.hs index c770045270..9348c7b437 100644 --- a/UUID.hs +++ b/UUID.hs @@ -41,22 +41,21 @@ genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h getUUID :: Git.Repo -> Annex UUID getUUID r = do g <- Annex.gitRepo - let uuid = cached r g - if (uuid /= "") - then return $ uuid - else do - let uuid = uncached r - if (uuid /= "") - then do - updatecache r g uuid - return uuid - else return "" + + let c = cached r g + let u = uncached r + + if (c /= u && u /= "") + then do + updatecache g r u + return u + else return c where uncached r = Git.configGet r "annex.uuid" "" cached r g = Git.configGet g (cachekey r) "" - updatecache r g uuid = do - if (g /= r) - then setConfig (cachekey r) uuid + updatecache g r u = do + if (g /= r) + then setConfig (cachekey r) u else return () cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" From 90cdc61c7c8d08590e054018c54c542c463be7e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:38:29 -0400 Subject: [PATCH 0127/8313] refactor --- CmdLine.hs | 52 -------------------------------------------- Commands.hs | 58 +++++++++++++++++++++++++++++++++++++------------- Core.hs | 37 ++++++-------------------------- git-annex.hs | 34 ++++++++++++++++++++++++++++- git-annex.mdwn | 6 ++++-- 5 files changed, 86 insertions(+), 101 deletions(-) delete mode 100644 CmdLine.hs diff --git a/CmdLine.hs b/CmdLine.hs deleted file mode 100644 index 98971a733b..0000000000 --- a/CmdLine.hs +++ /dev/null @@ -1,52 +0,0 @@ -{- git-annex command line - - - - TODO: This is very rough and stupid; I would like to use - - System.Console.CmdArgs.Implicit but it is not yet packaged in Debian. - -} - -module CmdLine ( - argvToMode, - dispatch, - Mode -) where - -import System.Console.GetOpt -import Types -import Commands - -data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex - deriving Show - -options :: [OptDescr Mode] -options = - [ Option ['a'] ["add"] (NoArg Add) "add files to annex" - , Option ['p'] ["push"] (NoArg Push) "push annex to repos" - , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" - , Option ['w'] ["want"] (NoArg Want) "request file contents" - , Option ['g'] ["get"] (NoArg Get) "transfer file contents" - , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" - , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" - ] - -argvToMode argv = do - case getOpt Permute options argv of - ([],files,[]) -> return (Default, files) - -- one mode is normal case - (m:[],files,[]) -> return (m, files) - -- multiple modes is an error - (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) - -- error case - (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) - where header = "Usage: git-annex [mode] file" - -dispatch :: Mode -> FilePath -> Annex () -dispatch mode item = do - case (mode) of - Default -> defaultCmd item - Add -> addCmd item - Push -> pushCmd item - Pull -> pullCmd item - Want -> wantCmd item - Get -> getCmd item - Drop -> dropCmd item - Unannex -> unannexCmd item diff --git a/Commands.hs b/Commands.hs index 65f6f6efd6..b631664d61 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,16 +1,12 @@ -{- git-annex subcommands -} +{- git-annex command line -} module Commands ( - defaultCmd, - addCmd, - unannexCmd, - getCmd, - wantCmd, - dropCmd, - pushCmd, - pullCmd + argvToMode, + dispatch, + Mode ) where +import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory @@ -25,6 +21,44 @@ import BackendList import UUID import LocationLog import Types +import Core + +data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex + deriving Show + +options :: [OptDescr Mode] +options = + [ Option ['a'] ["add"] (NoArg Add) "add files to annex" + , Option ['p'] ["push"] (NoArg Push) "push annex to repos" + , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" + , Option ['w'] ["want"] (NoArg Want) "request file contents" + , Option ['g'] ["get"] (NoArg Get) "transfer file contents" + , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" + , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" + ] + +argvToMode argv = do + case getOpt Permute options argv of + ([],files,[]) -> return (Default, files) + -- one mode is normal case + (m:[],files,[]) -> return (m, files) + -- multiple modes is an error + (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) + -- error case + (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) + where header = "Usage: git-annex [mode] file" + +dispatch :: Mode -> FilePath -> Annex () +dispatch mode item = do + case (mode) of + Default -> defaultCmd item + Add -> addCmd item + Push -> pushCmd item + Pull -> pullCmd item + Want -> wantCmd item + Get -> getCmd item + Drop -> dropCmd item + Unannex -> unannexCmd item {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} @@ -163,9 +197,3 @@ inBackend file yes no = do Just v -> yes v Nothing -> no notinBackend file yes no = inBackend file no yes - -{- Checks if a given key is currently present in the annexLocation -} -inAnnex :: Backend -> Key -> Annex Bool -inAnnex backend key = do - g <- Annex.gitRepo - liftIO $ doesFileExist $ annexLocation g backend key diff --git a/Core.hs b/Core.hs index e3d2c64039..1eb9da687d 100644 --- a/Core.hs +++ b/Core.hs @@ -5,8 +5,6 @@ module Core where import System.IO import System.Directory import Control.Monad.State (liftIO) -import Control.Exception -import CmdLine import Types import BackendList import Locations @@ -33,35 +31,6 @@ start = do Git.configGet g' "annex.backends" "" prepUUID -{- Processes each param in the list by dispatching the handler function - - for the user-selection operation mode. Catches exceptions, not stopping - - if some error out, and propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. - -} -tryRun :: AnnexState -> Mode -> [String] -> IO () -tryRun state mode params = tryRun' state mode 0 0 params -tryRun' state mode errnum oknum [] = do - if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" - else return () -tryRun' state mode errnum oknum (f:fs) = do - result <- try - (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) - case (result) of - Left err -> do - showErr err - tryRun' state mode (errnum + 1) oknum fs - Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs - -{- Exception pretty-printing. -} -showErr e = do - hPutStrLn stderr $ "git-annex: " ++ (show e) - return () - {- Sets up a git repo for git-annex. May be called repeatedly. -} gitSetup :: Git.Repo -> IO () gitSetup repo = do @@ -85,3 +54,9 @@ gitSetup repo = do Git.run repo ["add", attributes] Git.run repo ["commit", "-m", "git-annex setup", attributes] + +{- Checks if a given key is currently present in the annexLocation -} +inAnnex :: Backend -> Key -> Annex Bool +inAnnex backend key = do + g <- Annex.gitRepo + liftIO $ doesFileExist $ annexLocation g backend key diff --git a/git-annex.hs b/git-annex.hs index b326b2b199..a038107e9e 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,12 +1,44 @@ {- git-annex main program -} +import Control.Exception +import System.IO import System.Environment import qualified Annex +import Types import Core -import CmdLine +import Commands main = do args <- getArgs (mode, params) <- argvToMode args state <- start tryRun state mode params + +{- Processes each param in the list by dispatching the handler function + - for the user-selection operation mode. Catches exceptions, not stopping + - if some error out, and propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> Mode -> [String] -> IO () +tryRun state mode params = tryRun' state mode 0 0 params +tryRun' state mode errnum oknum (f:fs) = do + result <- try + (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + case (result) of + Left err -> do + showErr err + tryRun' state mode (errnum + 1) oknum fs + Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs +tryRun' state mode errnum oknum [] = do + if (errnum > 0) + then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" + else return () + +{- Exception pretty-printing. -} +showErr e = do + hPutStrLn stderr $ "git-annex: " ++ (show e) + return () diff --git a/git-annex.mdwn b/git-annex.mdwn index 6852ed0080..a7a0539077 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -48,9 +48,11 @@ git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1 (configured by annex.numcopies). -`git annex --drop` attempts to communicate with all other configured +`git annex --drop` attempts to check all other configured repositories, to check that N copies of the file exist. If enough -repositories cannot be contacted, it will retain the file content. +repositories cannot be verified to have it, it will retain the file content +to avoid data loss. + You can later use `git annex --drop --retry` to retry pending drops. Or you can use `git annex --drop --force $file` to force dropping of file content. From a0c013605699a4b1509ba1b04b4522ac43f033c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:49:19 -0400 Subject: [PATCH 0128/8313] cooler command-line handling --- Commands.hs | 44 +++++++++++++++----------------------------- git-annex.hs | 18 +++++++++--------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/Commands.hs b/Commands.hs index b631664d61..05ea658801 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,9 +1,7 @@ {- git-annex command line -} module Commands ( - argvToMode, - dispatch, - Mode + argvToActions ) where import System.Console.GetOpt @@ -23,43 +21,31 @@ import LocationLog import Types import Core -data Mode = Default | Add | Push | Pull | Want | Get | Drop | Unannex - deriving Show - -options :: [OptDescr Mode] +options :: [OptDescr (String -> Annex ())] options = - [ Option ['a'] ["add"] (NoArg Add) "add files to annex" - , Option ['p'] ["push"] (NoArg Push) "push annex to repos" - , Option ['P'] ["pull"] (NoArg Pull) "pull annex from repos" - , Option ['w'] ["want"] (NoArg Want) "request file contents" - , Option ['g'] ["get"] (NoArg Get) "transfer file contents" - , Option ['d'] ["drop"] (NoArg Drop) "indicate file contents not needed" - , Option ['u'] ["unannex"] (NoArg Unannex) "undo --add" + [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" + , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" + , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" + , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" + , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" + , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" + , Option ['u'] ["unannex"] (NoArg unannexCmd) "undo --add" ] -argvToMode argv = do +{- Parses command line and returns a list of actons to be run in the Annex + - monad. -} +argvToActions :: [String] -> IO [Annex ()] +argvToActions argv = do case getOpt Permute options argv of - ([],files,[]) -> return (Default, files) + ([],files,[]) -> return $ map defaultCmd files -- one mode is normal case - (m:[],files,[]) -> return (m, files) + (m:[],files,[]) -> return $ map m files -- multiple modes is an error (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) -- error case (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) where header = "Usage: git-annex [mode] file" -dispatch :: Mode -> FilePath -> Annex () -dispatch mode item = do - case (mode) of - Default -> defaultCmd item - Add -> addCmd item - Push -> pushCmd item - Pull -> pullCmd item - Want -> wantCmd item - Get -> getCmd item - Drop -> dropCmd item - Unannex -> unannexCmd item - {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} defaultCmd :: FilePath -> Annex () diff --git a/git-annex.hs b/git-annex.hs index a038107e9e..77dd29fac7 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -10,9 +10,9 @@ import Commands main = do args <- getArgs - (mode, params) <- argvToMode args + actions <- argvToActions args state <- start - tryRun state mode params + tryRun state actions {- Processes each param in the list by dispatching the handler function - for the user-selection operation mode. Catches exceptions, not stopping @@ -23,17 +23,17 @@ main = do - or more likely I missed an easy way to do it. So, I have to laboriously - thread AnnexState through this function. -} -tryRun :: AnnexState -> Mode -> [String] -> IO () -tryRun state mode params = tryRun' state mode 0 0 params -tryRun' state mode errnum oknum (f:fs) = do +tryRun :: AnnexState -> [Annex ()] -> IO () +tryRun state actions = tryRun' state 0 0 actions +tryRun' state errnum oknum (a:as) = do result <- try - (Annex.run state (dispatch mode f))::IO (Either SomeException ((), AnnexState)) + (Annex.run state a)::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err - tryRun' state mode (errnum + 1) oknum fs - Right (_,state') -> tryRun' state' mode errnum (oknum + 1) fs -tryRun' state mode errnum oknum [] = do + tryRun' state (errnum + 1) oknum as + Right (_,state') -> tryRun' state' errnum (oknum + 1) as +tryRun' state errnum oknum [] = do if (errnum > 0) then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" else return () From 1d628ff2380d1bec0c260bc19349c67360fa7a1f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:50:46 -0400 Subject: [PATCH 0129/8313] comment --- Commands.hs | 4 +--- git-annex.hs | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 05ea658801..feb35d1fb7 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,8 +1,6 @@ {- git-annex command line -} -module Commands ( - argvToActions -) where +module Commands (argvToActions) where import System.Console.GetOpt import Control.Monad.State (liftIO) diff --git a/git-annex.hs b/git-annex.hs index 77dd29fac7..78e8750147 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -14,8 +14,7 @@ main = do state <- start tryRun state actions -{- Processes each param in the list by dispatching the handler function - - for the user-selection operation mode. Catches exceptions, not stopping +{- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. - - This runs in the IO monad, not in the Annex monad. It seems that From 40df205881c6bfa180dd37d1a6e67afb3ce3593f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 14:54:56 -0400 Subject: [PATCH 0130/8313] indent --- Commands.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index feb35d1fb7..730663b0d6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,12 +21,12 @@ import Core options :: [OptDescr (String -> Annex ())] options = - [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" - , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" - , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" - , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" - , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" - , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" + [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" + , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" + , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" + , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" + , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" + , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" , Option ['u'] ["unannex"] (NoArg unannexCmd) "undo --add" ] From 4b7e54eddb953ee3ec3ce1734e2748fc3e2f2f9f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 15:05:10 -0400 Subject: [PATCH 0131/8313] tweak docs --- git-annex.mdwn | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index a7a0539077..62e4301eb5 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -53,10 +53,6 @@ repositories, to check that N copies of the file exist. If enough repositories cannot be verified to have it, it will retain the file content to avoid data loss. -You can later use `git annex --drop --retry` to retry pending drops. -Or you can use `git annex --drop --force $file` to force dropping of -file content. - For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex --get $file`, this will transfer it from either Server or USB (depending on which @@ -66,14 +62,15 @@ Suppose you want to free up space on laptop again, and you --drop the file there. If USB is connected, or Server can be contacted, git-annex can check that it still has a copy of the file, and the content is removed from Laptop. But if USB is currently disconnected, and Server also cannot be -contacted, it can't check that and will retain the file content. +contacted, it can't verify that it is safe to drop the file, and will +refuse to do so. With N=2, in order to drop the file content from Laptop, it would need access to both USB and Server. Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1, and of they have the +copies falling to 1, when USB and Server have N=1, and if they have the only copies of a file. ## the .git-annex directory From 859731ee5b09072d112296a073cb152007d7785a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 15:31:44 -0400 Subject: [PATCH 0132/8313] add hasKey --- Backend/File.hs | 12 +++++++++--- Backend/Url.hs | 9 +++++---- BackendTypes.hs | 4 +++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 311fe820b2..893850a695 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -13,13 +13,16 @@ import LocationLog import Locations import qualified Remotes import qualified GitRepo as Git +import Utility +import Core backend = Backend { name = "file", getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, - removeKey = dummyRemove + removeKey = dummyRemove, + hasKey = checkKeyFile } -- direct mapping from filename to key @@ -29,8 +32,7 @@ keyValue file = return $ Just $ Key file {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. So storing a key is - - a no-op. TODO until support is added for git annex --push otherrepo, - - then these could implement that.. -} + - a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True @@ -38,6 +40,10 @@ dummyStore file key = return True dummyRemove :: Key -> Annex Bool dummyRemove url = return True +{- Just check if the .git/annex/ file for the key exists. -} +checkKeyFile :: Key -> Annex Bool +checkKeyFile k = inAnnex backend k + {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex (Bool) diff --git a/Backend/Url.hs b/Backend/Url.hs index 3d971864ab..325a7e217a 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -13,7 +13,8 @@ backend = Backend { getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = downloadUrl, - removeKey = dummyRemove + removeKey = dummyOk, + hasKey = dummyOk } -- cannot generate url from filename @@ -24,9 +25,9 @@ keyValue file = return Nothing dummyStore :: FilePath -> Key -> Annex Bool dummyStore file url = return False --- allow keys to be removed -dummyRemove :: Key -> Annex Bool -dummyRemove url = return True +-- allow keys to be removed; presumably they can always be downloaded again +dummyOk :: Key -> Annex Bool +dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl url file = do diff --git a/BackendTypes.hs b/BackendTypes.hs index 2ef65f4691..e480f725b8 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -37,7 +37,9 @@ data Backend = Backend { -- retrieves a key's contents to a file retrieveKeyFile :: Key -> FilePath -> Annex Bool, -- removes a key - removeKey :: Key -> Annex Bool + removeKey :: Key -> Annex Bool, + -- checks if a backend is storing the content of a key + hasKey :: Key -> Annex Bool } instance Show Backend where From d4ce0724527fa0155f737b5d3e94e190c27d29dc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 15:58:53 -0400 Subject: [PATCH 0133/8313] break depends cycle --- Backend.hs | 13 +++++++++++++ Backend/File.hs | 1 - Core.hs | 2 -- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Backend.hs b/Backend.hs index 7a8a41a4ba..01c7c6823f 100644 --- a/Backend.hs +++ b/Backend.hs @@ -33,6 +33,19 @@ import qualified Annex import Utility import Types import qualified BackendTypes as B +import BackendList + +{- List of backends in the order to try them when storing a new key. -} +backendList :: Annex [Backend] +backendList = do + l <- Annex.backends + if (0 < length l) + then return l + else do + g <- Annex.gitRepo + let l = parseBackendList $ Git.configGet g "annex.backends" "" + Annex.backendsChange l + return l {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) diff --git a/Backend/File.hs b/Backend/File.hs index 893850a695..92f5932ce5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -14,7 +14,6 @@ import Locations import qualified Remotes import qualified GitRepo as Git import Utility -import Core backend = Backend { name = "file", diff --git a/Core.hs b/Core.hs index 1eb9da687d..431c9c9e63 100644 --- a/Core.hs +++ b/Core.hs @@ -27,8 +27,6 @@ start = do g' <- liftIO $ Git.configRead g Annex.gitRepoChange g' liftIO $ gitSetup g' - Annex.backendsChange $ parseBackendList $ - Git.configGet g' "annex.backends" "" prepUUID {- Sets up a git repo for git-annex. May be called repeatedly. -} From aa2f4bd81049e3bcaad6f5f1334864ce14887527 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 16:13:43 -0400 Subject: [PATCH 0134/8313] bug --- Annex.hs | 15 ++++++++--- Backend.hs | 2 +- Backend/File.hs | 1 + Core.hs | 67 ++++++++++++++++++++----------------------------- git-annex.hs | 7 ++++-- 5 files changed, 46 insertions(+), 46 deletions(-) diff --git a/Annex.hs b/Annex.hs index fcd19ba033..9be86c9481 100644 --- a/Annex.hs +++ b/Annex.hs @@ -14,9 +14,18 @@ import qualified GitRepo as Git import Types import qualified BackendTypes as Backend --- constructor -new :: Git.Repo -> AnnexState -new g = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } +{- Create and returns an Annex state object for the specified git repo. + -} +new :: Git.Repo -> IO AnnexState +new g = do + let s = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } + (_,s') <- Annex.run s (prep g) + return s' + where + prep g = do + -- read git config and update state + g' <- liftIO $ Git.configRead g + Annex.gitRepoChange g' -- performs an action in the Annex monad run state action = runStateT (action) state diff --git a/Backend.hs b/Backend.hs index 01c7c6823f..7a8178a8eb 100644 --- a/Backend.hs +++ b/Backend.hs @@ -52,7 +52,7 @@ storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) storeFileKey file = do g <- Annex.gitRepo let relfile = Git.relative g file - b <- Annex.backends + b <- backendList storeFileKey' b file relfile storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do diff --git a/Backend/File.hs b/Backend/File.hs index 92f5932ce5..893850a695 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -14,6 +14,7 @@ import Locations import qualified Remotes import qualified GitRepo as Git import Utility +import Core backend = Backend { name = "file", diff --git a/Core.hs b/Core.hs index 431c9c9e63..644bedd00f 100644 --- a/Core.hs +++ b/Core.hs @@ -6,52 +6,39 @@ import System.IO import System.Directory import Control.Monad.State (liftIO) import Types -import BackendList import Locations import UUID import qualified GitRepo as Git import qualified Annex - -{- Create and returns an Annex state object. - - Examines and prepares the git repo. - -} -start :: IO AnnexState -start = do - g <- Git.repoFromCwd - let s = Annex.new g - (_,s') <- Annex.run s (prep g) - return s' - where - prep g = do - -- setup git and read its config; update state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' - liftIO $ gitSetup g' - prepUUID - + {- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Git.Repo -> IO () -gitSetup repo = do - -- configure git to use union merge driver on state files - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () +gitSetup :: Annex () +gitSetup = do + g <- Annex.gitRepo + liftIO $ setupattributes g + prepUUID where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] + -- configure git to use union merge driver on state files + setupattributes repo = do + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) + then do + appendFile attributes $ attrLine ++ "\n" + commit + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool diff --git a/git-annex.hs b/git-annex.hs index 78e8750147..f9d9311eb2 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,12 +7,15 @@ import qualified Annex import Types import Core import Commands +import Annex +import qualified GitRepo as Git main = do args <- getArgs actions <- argvToActions args - state <- start - tryRun state actions + gitrepo <- Git.repoFromCwd + state <- new gitrepo + tryRun state (gitSetup:actions) {- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. From 508a3b65ed675c9322940578614f088ea2c74e7f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 17:37:20 -0400 Subject: [PATCH 0135/8313] annex.numcopies works --- Backend.hs | 15 ++++++++++++++- Backend/File.hs | 13 +++++++++++++ Commands.hs | 41 ++++++++++++++++++++++++++++++++++++++++- Remotes.hs | 10 +--------- TODO | 3 +++ 5 files changed, 71 insertions(+), 11 deletions(-) diff --git a/Backend.hs b/Backend.hs index 7a8178a8eb..47e42b8229 100644 --- a/Backend.hs +++ b/Backend.hs @@ -15,8 +15,9 @@ module Backend ( storeFileKey, - removeKey, retrieveKeyFile, + removeKey, + hasKey, lookupFile ) where @@ -77,6 +78,18 @@ retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest removeKey :: Backend -> Key -> Annex Bool removeKey backend key = (B.removeKey backend) key +{- Checks if any backend has a key. -} +hasKey :: Key -> Annex Bool +hasKey key = do + b <- backendList + hasKey' b key +hasKey' [] key = return False +hasKey' (b:bs) key = do + has <- (B.hasKey b) key + if (has) + then return True + else hasKey' bs key + {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} lookupFile :: FilePath -> IO (Maybe (Key, Backend)) diff --git a/Backend/File.hs b/Backend/File.hs index 893850a695..def2f30910 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,8 @@ import qualified Remotes import qualified GitRepo as Git import Utility import Core +import qualified Annex +import UUID backend = Backend { name = "file", @@ -49,6 +51,9 @@ checkKeyFile k = inAnnex backend k copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- Remotes.withKey key + if (0 == length remotes) + then cantfind + else return () trycopy remotes remotes where trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ @@ -68,6 +73,14 @@ copyKeyFile key file = do liftIO $ hPutStrLn stderr (show err) trycopy full rs Right succ -> return True + cantfind = do + g <- Annex.gitRepo + uuids <- liftIO $ keyLocations g key + error $ "no available git remotes have: " ++ + (keyFile key) ++ (uuidlist uuids) + uuidlist [] = "" + uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ + prettyPrintUUIDs uuids {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () diff --git a/Commands.hs b/Commands.hs index 730663b0d6..6128b76aad 100644 --- a/Commands.hs +++ b/Commands.hs @@ -8,6 +8,7 @@ import System.Posix.Files import System.Directory import Data.String.Utils import List +import IO import qualified GitRepo as Git import qualified Annex import Utility @@ -18,6 +19,7 @@ import UUID import LocationLog import Types import Core +import qualified Remotes options :: [OptDescr (String -> Annex ())] options = @@ -138,7 +140,7 @@ wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do - -- TODO only remove if enough copies are present elsewhere + requireEnoughCopies key success <- Backend.removeKey backend key if (success) then do @@ -181,3 +183,40 @@ inBackend file yes no = do Just v -> yes v Nothing -> no notinBackend file yes no = inBackend file no yes + +{- Checks remotes to verify that enough copies of a key exist to allow + - for a key to be safely removed (with no data loss), and fails with an + - error if not. -} +requireEnoughCopies :: Key -> Annex () +requireEnoughCopies key = do + g <- Annex.gitRepo + let numcopies = read $ Git.configGet g config "1" + remotes <- Remotes.withKey key + if (numcopies > length remotes) + then error $ "I only know about " ++ (show $ length remotes) ++ + " out of " ++ (show numcopies) ++ + " necessary copies of: " ++ (keyFile key) ++ + unsafe + else findcopies numcopies remotes [] + where + findcopies 0 _ _ = return () -- success, enough copies found + findcopies _ [] bad = die bad + findcopies n (r:rs) bad = do + result <- liftIO $ try $ haskey r + case (result) of + Right True -> findcopies (n-1) rs bad + Left _ -> findcopies n rs (r:bad) + haskey r = do + -- To check if a remote has a key, construct a new + -- Annex monad and query its backend. + a <- Annex.new r + (result, _) <- Annex.run a (Backend.hasKey key) + return result + die bad = + error $ "I failed to find enough other copies of: " ++ + (keyFile key) ++ "\n" ++ + "I was unable to access these remotes: " ++ + (Remotes.list bad) ++ unsafe + unsafe = "\n -- According to the " ++ config ++ + " setting, it is not safe to remove it!" + config = "annex.numcopies" diff --git a/Remotes.hs b/Remotes.hs index f20d51ab35..2fffcffa7e 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -40,15 +40,7 @@ withKey key = do mayberemotes <- mapM tryGitConfigRead allremotes let allremotes' = catMaybes mayberemotes remotes' <- reposByUUID allremotes' uuids - if (0 == length remotes') - then err uuids - else return remotes' - err uuids = - error $ "no available git remotes have: " ++ - (keyFile key) ++ (uuidlist uuids) - uuidlist [] = "" - uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ - prettyPrintUUIDs uuids + return remotes' {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] diff --git a/TODO b/TODO index c4ce74e198..70ace863ea 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,9 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing +* need to include backend name as part of the key, because currently + if two backends have overlapping key spaces, it can confuse things + * --push/--pull/--want * how to handle git mv file? From 467c4b2751921818f86561d85b0927254e48d956 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 17:57:04 -0400 Subject: [PATCH 0136/8313] better shutdown --- Commands.hs | 9 +++------ Core.hs | 57 ++++++++++++++++++++++++++++++---------------------- git-annex.hs | 14 ++++++------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/Commands.hs b/Commands.hs index 6128b76aad..58d88aa3b4 100644 --- a/Commands.hs +++ b/Commands.hs @@ -171,11 +171,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - liftIO $ commit g f - where - commit g f = do - Git.run g ["add", f] - Git.run g ["commit", "-m", "git-annex log update", f] + liftIO $ Git.run g ["add", f] -- committed at shutdown inBackend file yes no = do r <- liftIO $ Backend.lookupFile file @@ -204,7 +200,8 @@ requireEnoughCopies key = do findcopies n (r:rs) bad = do result <- liftIO $ try $ haskey r case (result) of - Right True -> findcopies (n-1) rs bad + Right True -> do + findcopies (n-1) rs bad Left _ -> findcopies n rs (r:bad) haskey r = do -- To check if a remote has a key, construct a new diff --git a/Core.hs b/Core.hs index 644bedd00f..5182a6855a 100644 --- a/Core.hs +++ b/Core.hs @@ -11,34 +11,43 @@ import UUID import qualified GitRepo as Git import qualified Annex -{- Sets up a git repo for git-annex. May be called repeatedly. -} -gitSetup :: Annex () -gitSetup = do +{- Sets up a git repo for git-annex. -} +setup :: Annex () +setup = do g <- Annex.gitRepo - liftIO $ setupattributes g + liftIO $ gitAttributes g prepUUID - where - -- configure git to use union merge driver on state files - setupattributes repo = do - exists <- doesFileExist attributes - if (not exists) + +{- When git-annex is done, it runs this. -} +shutdown :: Annex () +shutdown = do + g <- Annex.gitRepo + liftIO $ Git.run g ["commit", "-m", + "git-annex log update", ".git-annex"] + +{- configure git to use union merge driver on state files, if it is not + - already -} +gitAttributes :: Git.Repo -> IO () +gitAttributes repo = do + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + if (all (/= attrLine) (lines content)) then do - writeFile attributes $ attrLine ++ "\n" + appendFile attributes $ attrLine ++ "\n" commit - else do - content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () - where - attrLine = stateLoc ++ "/*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] + else return () + where + attrLine = stateLoc ++ "/*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] {- Checks if a given key is currently present in the annexLocation -} inAnnex :: Backend -> Key -> Annex Bool diff --git a/git-annex.hs b/git-annex.hs index f9d9311eb2..e147391957 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -15,7 +15,7 @@ main = do actions <- argvToActions args gitrepo <- Git.repoFromCwd state <- new gitrepo - tryRun state (gitSetup:actions) + tryRun state $ [setup] ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. @@ -26,18 +26,18 @@ main = do - thread AnnexState through this function. -} tryRun :: AnnexState -> [Annex ()] -> IO () -tryRun state actions = tryRun' state 0 0 actions -tryRun' state errnum oknum (a:as) = do +tryRun state actions = tryRun' state 0 actions +tryRun' state errnum (a:as) = do result <- try (Annex.run state a)::IO (Either SomeException ((), AnnexState)) case (result) of Left err -> do showErr err - tryRun' state (errnum + 1) oknum as - Right (_,state') -> tryRun' state' errnum (oknum + 1) as -tryRun' state errnum oknum [] = do + tryRun' state (errnum + 1) as + Right (_,state') -> tryRun' state' errnum as +tryRun' state errnum [] = do if (errnum > 0) - then error $ (show errnum) ++ " failed ; " ++ show (oknum) ++ " ok" + then error $ (show errnum) ++ " failed" else return () {- Exception pretty-printing. -} From c4959fee47f168857998dea6d11395158251158d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 18:48:21 -0400 Subject: [PATCH 0137/8313] bugfix --- Commands.hs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index 58d88aa3b4..ce8f00fd6f 100644 --- a/Commands.hs +++ b/Commands.hs @@ -200,9 +200,9 @@ requireEnoughCopies key = do findcopies n (r:rs) bad = do result <- liftIO $ try $ haskey r case (result) of - Right True -> do - findcopies (n-1) rs bad - Left _ -> findcopies n rs (r:bad) + Right True -> findcopies (n-1) rs bad + Right False -> findcopies n rs bad + Left _ -> findcopies n rs (r:bad) haskey r = do -- To check if a remote has a key, construct a new -- Annex monad and query its backend. @@ -211,9 +211,11 @@ requireEnoughCopies key = do return result die bad = error $ "I failed to find enough other copies of: " ++ - (keyFile key) ++ "\n" ++ - "I was unable to access these remotes: " ++ - (Remotes.list bad) ++ unsafe + (keyFile key) ++ + (if (0 /= length bad) then listbad bad else "") + ++ unsafe + listbad bad = "\nI was unable to access these remotes: " ++ + (Remotes.list bad) unsafe = "\n -- According to the " ++ config ++ " setting, it is not safe to remove it!" config = "annex.numcopies" From b8ba60428a0b4c077482560757e830e9ba02a823 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 19:36:11 -0400 Subject: [PATCH 0138/8313] changed key to include backend name --- Backend/File.hs | 6 +++--- BackendTypes.hs | 20 +++++++++++++++----- Commands.hs | 18 +++++++++--------- Core.hs | 6 +++--- Locations.hs | 19 ++++++++++--------- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index def2f30910..6267b478a0 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -29,7 +29,7 @@ backend = Backend { -- direct mapping from filename to key keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key file +keyValue file = return $ Just $ Key ((name backend), file) {- This backend does not really do any independant data storage, - it relies on the file contents in .git/annex/ in this repo, @@ -44,7 +44,7 @@ dummyRemove url = return True {- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool -checkKeyFile k = inAnnex backend k +checkKeyFile k = inAnnex k {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} @@ -97,4 +97,4 @@ copyFromRemote r key file = do then return () else error "cp failed" getremote = error "get via network not yet implemented!" - location = annexLocation r backend key + location = annexLocation r key diff --git a/BackendTypes.hs b/BackendTypes.hs index e480f725b8..e0f5f73738 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -5,7 +5,7 @@ module BackendTypes where -import Control.Monad.State +import Control.Monad.State (StateT) import Data.String.Utils import qualified GitRepo as Git @@ -19,12 +19,22 @@ data AnnexState = AnnexState { -- git-annex's monad type Annex = StateT AnnexState IO --- annexed filenames are mapped into keys -data Key = Key String deriving (Eq) +-- annexed filenames are mapped through a backend into keys +type KeyFrag = String +type BackendName = String +data Key = Key (BackendName, KeyFrag) deriving (Eq) --- show a key to convert it to a string +-- show a key to convert it to a string; the string includes the +-- name of the backend to avoid collisions between key strings instance Show Key where - show (Key v) = v + show (Key (b, k)) = b ++ ":" ++ k + +instance Read Key where + readsPrec _ s = [((Key (b,k)) ,"")] + where + l = split ":" s + b = l !! 0 + k = join ":" $ drop 1 l -- this structure represents a key/value backend data Backend = Backend { diff --git a/Commands.hs b/Commands.hs index ce8f00fd6f..7ff33ab020 100644 --- a/Commands.hs +++ b/Commands.hs @@ -66,7 +66,7 @@ addCmd file = inBackend file err $ do Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - liftIO $ setup g key backend + liftIO $ setup g key where err = error $ "already annexed " ++ file checkLegal file = do @@ -74,9 +74,9 @@ addCmd file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - setup g key backend = do - let dest = annexLocation g backend key - let reldest = annexLocationRelative g backend key + setup g key = do + let dest = annexLocation g key + let reldest = annexLocationRelative g key createDirectoryIfMissing True (parentDir dest) renameFile file dest createSymbolicLink ((linkTarget file) ++ reldest) file @@ -99,7 +99,7 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo - let src = annexLocation g backend key + let src = annexLocation g key liftIO $ moveout g src where err = error $ "not annexed " ++ file @@ -117,12 +117,12 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () getCmd file = notinBackend file err $ \(key, backend) -> do - inannex <- inAnnex backend key + inannex <- inAnnex key if (inannex) then return () else do g <- Annex.gitRepo - let dest = annexLocation g backend key + let dest = annexLocation g key liftIO $ createDirectoryIfMissing True (parentDir dest) success <- Backend.retrieveKeyFile backend key dest if (success) @@ -145,11 +145,11 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (success) then do logStatus key ValueMissing - inannex <- inAnnex backend key + inannex <- inAnnex key if (inannex) then do g <- Annex.gitRepo - let loc = annexLocation g backend key + let loc = annexLocation g key liftIO $ removeFile loc return () else return () diff --git a/Core.hs b/Core.hs index 5182a6855a..6f05394bb7 100644 --- a/Core.hs +++ b/Core.hs @@ -50,7 +50,7 @@ gitAttributes repo = do attributes] {- Checks if a given key is currently present in the annexLocation -} -inAnnex :: Backend -> Key -> Annex Bool -inAnnex backend key = do +inAnnex :: Key -> Annex Bool +inAnnex key = do g <- Annex.gitRepo - liftIO $ doesFileExist $ annexLocation g backend key + liftIO $ doesFileExist $ annexLocation g key diff --git a/Locations.hs b/Locations.hs index 7b8beb14f0..960a8938d8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -22,18 +22,19 @@ gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc ++ "/" {- An annexed file's content is stored in - - /path/to/repo/.git/annex// + - /path/to/repo/.git/annex/, where is of the form + - - - - (That allows deriving the key and backend by looking at the symlink to it.) + - That allows deriving the key and backend by looking at the symlink to it. -} -annexLocation :: Git.Repo -> Backend -> Key -> FilePath -annexLocation r backend key = - (Git.workTree r) ++ "/" ++ (annexLocationRelative r backend key) +annexLocation :: Git.Repo -> Key -> FilePath +annexLocation r key = + (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) {- Annexed file's location relative to the gitWorkTree -} -annexLocationRelative :: Git.Repo -> Backend -> Key -> FilePath -annexLocationRelative r backend key = - Git.dir r ++ "/annex/" ++ (Backend.name backend) ++ "/" ++ (keyFile key) +annexLocationRelative :: Git.Repo -> Key -> FilePath +annexLocationRelative r key = + Git.dir r ++ "/annex/" ++ (keyFile key) {- Converts a key into a filename fragment. - @@ -51,5 +52,5 @@ keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Key -fileKey file = Backend.Key $ +fileKey file = read $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file From 4c3ad80f320d3c4cccc3e41e4f2364155bae22a1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 20:05:04 -0400 Subject: [PATCH 0139/8313] bugfix --- Backend.hs | 20 +++++++------------- Backend/Url.hs | 11 +++++++---- BackendList.hs | 2 +- BackendTypes.hs | 8 ++++++++ TODO | 3 --- Types.hs | 4 +++- 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Backend.hs b/Backend.hs index 47e42b8229..f419831d28 100644 --- a/Backend.hs +++ b/Backend.hs @@ -78,17 +78,9 @@ retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest removeKey :: Backend -> Key -> Annex Bool removeKey backend key = (B.removeKey backend) key -{- Checks if any backend has a key. -} +{- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool -hasKey key = do - b <- backendList - hasKey' b key -hasKey' [] key = return False -hasKey' (b:bs) key = do - has <- (B.hasKey b) key - if (has) - then return True - else hasKey' bs key +hasKey key = (B.hasKey (lookupBackendName $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} @@ -101,6 +93,8 @@ lookupFile file = do where lookup = do l <- readSymbolicLink file - return $ Just (k l, b l) - k l = fileKey $ takeFileName $ l - b l = lookupBackendName $ takeFileName $ parentDir $ l + return $ Just $ pair $ takeFileName l + pair file = (k, b) + where + k = fileKey file + b = lookupBackendName $ backendName k diff --git a/Backend/Url.hs b/Backend/Url.hs index 325a7e217a..e237672084 100644 --- a/Backend/Url.hs +++ b/Backend/Url.hs @@ -3,7 +3,8 @@ module Backend.Url (backend) where -import Control.Monad.State +import Control.Monad.State (liftIO) +import Data.String.Utils import System.Cmd import System.Exit import BackendTypes @@ -30,9 +31,11 @@ dummyOk :: Key -> Annex Bool dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool -downloadUrl url file = do - liftIO $ putStrLn $ "download: " ++ (show url) - result <- liftIO $ rawSystem "curl" ["-#", "-o", file, (show url)] +downloadUrl key file = do + liftIO $ putStrLn $ "download: " ++ url + result <- liftIO $ rawSystem "curl" ["-#", "-o", file, url] if (result == ExitSuccess) then return True else return False + where + url = join ":" $ drop 1 $ split ":" $ show key diff --git a/BackendList.hs b/BackendList.hs index e9f926ce2a..b66110905a 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -28,7 +28,7 @@ parseBackendList s = then supportedBackends else map (lookupBackendName) $ words s -{- Looks up a supported backed by name. -} +{- Looks up a supported backend by name. -} lookupBackendName :: String -> Backend lookupBackendName s = if ((length matches) /= 1) diff --git a/BackendTypes.hs b/BackendTypes.hs index e0f5f73738..41ff7e506f 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -36,6 +36,14 @@ instance Read Key where b = l !! 0 k = join ":" $ drop 1 l +-- pulls the backend name out +backendName :: Key -> BackendName +backendName (Key (b,k)) = b + +-- pulls the key fragment out +keyFrag :: Key -> KeyFrag +keyFrag (Key (b,k)) = k + -- this structure represents a key/value backend data Backend = Backend { -- name of this backend diff --git a/TODO b/TODO index 70ace863ea..c4ce74e198 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ * bug when annexing files while in a subdir of a git repo * bug when specifying absolute path to files when annexing -* need to include backend name as part of the key, because currently - if two backends have overlapping key spaces, it can confuse things - * --push/--pull/--want * how to handle git mv file? diff --git a/Types.hs b/Types.hs index 4262ed567d..a0f120db0b 100644 --- a/Types.hs +++ b/Types.hs @@ -3,8 +3,10 @@ module Types ( Annex, AnnexState, + Backend, Key, - Backend + backendName, + keyFrag ) where import BackendTypes From 29039fdf97f541a1077c9af65ccbe09dd2ae2b28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 21:10:59 -0400 Subject: [PATCH 0140/8313] add flags, and change to subcommand style --- Annex.hs | 18 +++++++++++++++++- BackendTypes.hs | 7 ++++++- Commands.hs | 50 ++++++++++++++++++++++++++----------------------- Core.hs | 5 +++-- TODO | 2 ++ Types.hs | 3 ++- git-annex.hs | 4 ++-- 7 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Annex.hs b/Annex.hs index 9be86c9481..9e76b9b042 100644 --- a/Annex.hs +++ b/Annex.hs @@ -7,6 +7,9 @@ module Annex ( gitRepoChange, backends, backendsChange, + flagIsSet, + flagsChange, + Flag(..) ) where import Control.Monad.State @@ -18,7 +21,11 @@ import qualified BackendTypes as Backend -} new :: Git.Repo -> IO AnnexState new g = do - let s = Backend.AnnexState { Backend.repo = g, Backend.backends = [] } + let s = Backend.AnnexState { + Backend.repo = g, + Backend.backends = [], + Backend.flags = [] + } (_,s') <- Annex.run s (prep g) return s' where @@ -49,3 +56,12 @@ backendsChange b = do state <- get put state { Backend.backends = b } return () +flagIsSet :: Flag -> Annex Bool +flagIsSet flag = do + state <- get + return $ elem flag $ Backend.flags state +flagsChange :: [Flag] -> Annex () +flagsChange b = do + state <- get + put state { Backend.flags = b } + return () diff --git a/BackendTypes.hs b/BackendTypes.hs index 41ff7e506f..1b67ef584d 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -9,11 +9,16 @@ import Control.Monad.State (StateT) import Data.String.Utils import qualified GitRepo as Git +-- command-line flags +data Flag = Force + deriving (Eq, Read, Show) + -- git-annex's runtime state type doesn't really belong here, -- but it uses Backend, so has to be here to avoid a depends loop. data AnnexState = AnnexState { repo :: Git.Repo, - backends :: [Backend] + backends :: [Backend], + flags :: [Flag] } deriving (Show) -- git-annex's monad diff --git a/Commands.hs b/Commands.hs index 7ff33ab020..a16470fe38 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,6 +1,6 @@ {- git-annex command line -} -module Commands (argvToActions) where +module Commands (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) @@ -21,30 +21,34 @@ import Types import Core import qualified Remotes -options :: [OptDescr (String -> Annex ())] -options = - [ Option ['a'] ["add"] (NoArg addCmd) "add files to annex" - , Option ['p'] ["push"] (NoArg pushCmd) "push annex to repos" - , Option ['P'] ["pull"] (NoArg pullCmd) "pull annex from repos" - , Option ['w'] ["want"] (NoArg wantCmd) "request file contents" - , Option ['g'] ["get"] (NoArg getCmd) "transfer file contents" - , Option ['d'] ["drop"] (NoArg dropCmd) "indicate file contents not needed" - , Option ['u'] ["unannex"] (NoArg unannexCmd) "undo --add" - ] - {- Parses command line and returns a list of actons to be run in the Annex - monad. -} -argvToActions :: [String] -> IO [Annex ()] -argvToActions argv = do - case getOpt Permute options argv of - ([],files,[]) -> return $ map defaultCmd files - -- one mode is normal case - (m:[],files,[]) -> return $ map m files - -- multiple modes is an error - (ms,files,[]) -> ioError (userError ("only one mode should be specified\n" ++ usageInfo header options)) - -- error case - (_,files,errs) -> ioError (userError (concat errs ++ usageInfo header options)) - where header = "Usage: git-annex [mode] file" +parseCmd :: [String] -> IO ([Flag], [Annex ()]) +parseCmd argv = do + (flags, nonopts) <- getopt + case (length nonopts) of + 0 -> error header + _ -> do + let c = lookupCmd (nonopts !! 0) + if (0 == length c) + then return $ (flags, map defaultCmd nonopts) + else do + return $ (flags, map (snd $ c !! 0) $ drop 1 nonopts) + where + getopt = case getOpt Permute options argv of + (flags, nonopts, []) -> return (flags, nonopts) + (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) + lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds + cmds = [ ("add", addCmd) + , ("push", pushCmd) + , ("pull", pullCmd) + , ("want", wantCmd) + , ("drop", dropCmd) + , ("unannex", unannexCmd) + ] + header = "Usage: git-annex [" ++ + (join "|" $ map fst cmds) ++ "] file ..." + options = [ Option ['f'] ["force"] (NoArg Force) "" ] {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} diff --git a/Core.hs b/Core.hs index 6f05394bb7..765b1e6a7e 100644 --- a/Core.hs +++ b/Core.hs @@ -12,8 +12,9 @@ import qualified GitRepo as Git import qualified Annex {- Sets up a git repo for git-annex. -} -setup :: Annex () -setup = do +startup :: [Flag] -> Annex () +startup flags = do + Annex.flagsChange flags g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID diff --git a/TODO b/TODO index c4ce74e198..b800097a0e 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,8 @@ * --push/--pull/--want +* recurse on directories + * how to handle git mv file? * finish BackendChecksum diff --git a/Types.hs b/Types.hs index a0f120db0b..6bf26d36e6 100644 --- a/Types.hs +++ b/Types.hs @@ -6,7 +6,8 @@ module Types ( Backend, Key, backendName, - keyFrag + keyFrag, + Flag(..), ) where import BackendTypes diff --git a/git-annex.hs b/git-annex.hs index e147391957..cd67242afa 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -12,10 +12,10 @@ import qualified GitRepo as Git main = do args <- getArgs - actions <- argvToActions args + (flags, actions) <- parseCmd args gitrepo <- Git.repoFromCwd state <- new gitrepo - tryRun state $ [setup] ++ actions ++ [shutdown] + tryRun state $ [startup flags] ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches exceptions, not stopping - if some error out, and propigates an overall error status at the end. From 1ab3e54ca8e56f8d7b8fd6ad4ceda888e19205f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 21:12:54 -0400 Subject: [PATCH 0141/8313] docs --- git-annex.mdwn | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 62e4301eb5..70bd66e954 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -13,7 +13,7 @@ associated with annexed files but that benefit from full revision control. Enough broad picture, here's how it actually looks: -* `git annex --add $file` moves the file into `.git/annex/`, and replaces +* `git annex add $file` moves the file into `.git/annex/`, and replaces it with a symlink pointing at the annexed file, and then calls `git add` to version the *symlink*. (If the file has already been annexed, it does nothing.) @@ -24,23 +24,23 @@ Enough broad picture, here's how it actually looks: * If you use normal git push/pull commands, the annexed file contents won't be sent, but the symlinks will be. So different clones of a repository can have different sets of annexed files available. -* `git annex --push $repository` pushes *all* annexed files to the specified +* `git annex push $repository` pushes *all* annexed files to the specified repository. -* `git annex --pull $repository` pulls *all* annexed files from the specified +* `git annex pull $repository` pulls *all* annexed files from the specified repository. -* `git annex --want $file` indicates that you want access to a file's +* `git annex want $file` indicates that you want access to a file's content, without immediatly transferring it. -* `git annex --get $file` is used to transfer a specified file, and/or - files previously indicated with --want. If a configured repository has it, +* `git annex get $file` is used to transfer a specified file, and/or + files previously indicated with `git annex want`. If a configured repository has it, or it is available from other key/value storage, it will be immediatly downloaded. -* `git annex --drop $file` indicates that you no longer want the file's +* `git annex drop $file` indicates that you no longer want the file's content to be available in this repository. -* `git annex --unannex $file` undoes a `git annex --add`. But use `--drop` - if you're just done with a file; only use `--unannex` if you +* `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` + if you're just done with a file; only use `unannex` if you accidentially added a file. -* `git annex $file` is a shorthand for either --add or --get. If the file - is already known, it does --get, otherwise it does --add. +* `git annex $file` is a shorthand. If the file + is already known, it does `git annex get`, otherwise it does `git annex add`. ## copies @@ -48,17 +48,17 @@ git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1 (configured by annex.numcopies). -`git annex --drop` attempts to check all other configured +`git annex drop` attempts to check all other configured repositories, to check that N copies of the file exist. If enough repositories cannot be verified to have it, it will retain the file content to avoid data loss. For example, consider three repositories: Server, Laptop, and USB. Both Server -and USB have a copy of a file, and N=1. If on Laptop, you `git annex --get +and USB have a copy of a file, and N=1. If on Laptop, you `git annex get $file`, this will transfer it from either Server or USB (depending on which is available), and there are now 3 copies of the file. -Suppose you want to free up space on laptop again, and you --drop the file +Suppose you want to free up space on laptop again, and you `git annex drop` the file there. If USB is connected, or Server can be contacted, git-annex can check that it still has a copy of the file, and the content is removed from Laptop. But if USB is currently disconnected, and Server also cannot be @@ -108,11 +108,11 @@ to store different files' contents in a given repository. git-annex keeps track of on which repository it last saw a file's content. This can be useful when using it for archiving with offline storage. When -you indicate you --want a file, git-annex will tell you which repositories +you indicate you want a file, git-annex will tell you which repositories have the file's content. Location tracking information is stored in `.git-annex/$key.log`. -Repositories record their UUID and the date when they --get or --drop +Repositories record their UUID and the date when they get or drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) @@ -150,7 +150,7 @@ If the symlink to annexed content is relative, moving it to a subdir will break it. But it it's absolute, moving the git repo (or mounting its drive elsewhere) will break it. Either: -* Use relative links and need `git annex --mv` to move (or post-commit +* Use relative links and need `git annex mv` to move (or post-commit hook that caches moves and updates links). * Use absolute links and need `git annex fixlinks` when location changes; note that would also mean that git would see the symlink targets changed From c977b6b1f3833ed1ead9212d956d8f83a4ca9028 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 22:09:03 -0400 Subject: [PATCH 0142/8313] forcing --- Commands.hs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index a16470fe38..6c519c294a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -144,7 +144,10 @@ wantCmd file = do error "not implemented" -- TODO {- Indicates a file is not wanted. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do - requireEnoughCopies key + force <- Annex.flagIsSet Force + if (not force) + then requireEnoughCopies key + else return () success <- Backend.removeKey backend key if (success) then do @@ -220,6 +223,9 @@ requireEnoughCopies key = do ++ unsafe listbad bad = "\nI was unable to access these remotes: " ++ (Remotes.list bad) - unsafe = "\n -- According to the " ++ config ++ - " setting, it is not safe to remove it!" + unsafe = "\n" ++ + " -- According to the " ++ config ++ + " setting, it is not safe to remove it!\n" ++ + " (Use --force to override.)" + config = "annex.numcopies" From bbbe9858fe2e83767661282f7ab8ed3470ec6568 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 23:52:45 -0400 Subject: [PATCH 0143/8313] avoid empty commits --- Annex.hs | 11 +++++++---- BackendTypes.hs | 2 +- Commands.hs | 3 ++- Core.hs | 9 ++++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Annex.hs b/Annex.hs index 9e76b9b042..08607cafa8 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,7 +8,7 @@ module Annex ( backends, backendsChange, flagIsSet, - flagsChange, + flagChange, Flag(..) ) where @@ -60,8 +60,11 @@ flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get return $ elem flag $ Backend.flags state -flagsChange :: [Flag] -> Annex () -flagsChange b = do +flagChange :: Flag -> Bool -> Annex () +flagChange flag set = do state <- get - put state { Backend.flags = b } + let f = filter (/= flag) $ Backend.flags state + if (set) + then put state { Backend.flags = (flag:f) } + else put state { Backend.flags = f } return () diff --git a/BackendTypes.hs b/BackendTypes.hs index 1b67ef584d..13ffde7f89 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -10,7 +10,7 @@ import Data.String.Utils import qualified GitRepo as Git -- command-line flags -data Flag = Force +data Flag = Force | NeedCommit deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, diff --git a/Commands.hs b/Commands.hs index 6c519c294a..5b5bc269bc 100644 --- a/Commands.hs +++ b/Commands.hs @@ -178,7 +178,8 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - liftIO $ Git.run g ["add", f] -- committed at shutdown + liftIO $ Git.run g ["add", f] + Annex.flagChange NeedCommit True inBackend file yes no = do r <- liftIO $ Backend.lookupFile file diff --git a/Core.hs b/Core.hs index 765b1e6a7e..8f1c9cc802 100644 --- a/Core.hs +++ b/Core.hs @@ -14,7 +14,7 @@ import qualified Annex {- Sets up a git repo for git-annex. -} startup :: [Flag] -> Annex () startup flags = do - Annex.flagsChange flags + mapM (\f -> Annex.flagChange f True) flags g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID @@ -23,8 +23,11 @@ startup flags = do shutdown :: Annex () shutdown = do g <- Annex.gitRepo - liftIO $ Git.run g ["commit", "-m", - "git-annex log update", ".git-annex"] + needcommit <- Annex.flagIsSet NeedCommit + if (needcommit) + then liftIO $ Git.run g ["commit", "-m", + "git-annex log update", ".git-annex"] + else return () {- configure git to use union merge driver on state files, if it is not - already -} From e7ffa5b594deb9d89d555b24f8ed7842951905af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 14 Oct 2010 23:57:22 -0400 Subject: [PATCH 0144/8313] update --- Commands.hs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 5b5bc269bc..5021ba260a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,8 +21,8 @@ import Types import Core import qualified Remotes -{- Parses command line and returns a list of actons to be run in the Annex - - monad. -} +{- Parses command line and returns a list of flags and a list of + - actions to be run in the Annex monad. -} parseCmd :: [String] -> IO ([Flag], [Annex ()]) parseCmd argv = do (flags, nonopts) <- getopt @@ -40,15 +40,16 @@ parseCmd argv = do (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds cmds = [ ("add", addCmd) + , ("get", getCmd) + , ("drop", dropCmd) + , ("want", wantCmd) , ("push", pushCmd) , ("pull", pullCmd) - , ("want", wantCmd) - , ("drop", dropCmd) , ("unannex", unannexCmd) ] header = "Usage: git-annex [" ++ (join "|" $ map fst cmds) ++ "] file ..." - options = [ Option ['f'] ["force"] (NoArg Force) "" ] + options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Default mode is to annex a file if it is not already, and otherwise - get its content. -} From d7b170c9a2f9a7f52b6ef88243e249a04685764a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 14:26:16 -0400 Subject: [PATCH 0145/8313] cleanup --- Commands.hs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index 5021ba260a..aed3a19d32 100644 --- a/Commands.hs +++ b/Commands.hs @@ -25,16 +25,17 @@ import qualified Remotes - actions to be run in the Annex monad. -} parseCmd :: [String] -> IO ([Flag], [Annex ()]) parseCmd argv = do - (flags, nonopts) <- getopt - case (length nonopts) of + (flags, files) <- getopt + case (length files) of 0 -> error header _ -> do - let c = lookupCmd (nonopts !! 0) + let c = lookupCmd (files !! 0) if (0 == length c) - then return $ (flags, map defaultCmd nonopts) - else do - return $ (flags, map (snd $ c !! 0) $ drop 1 nonopts) + then ret flags defaultCmd files + else ret flags (snd $ c !! 0) $ drop 1 files where + ret flags cmd files = return (flags, makeactions cmd files) + makeactions cmd files = map cmd files getopt = case getOpt Permute options argv of (flags, nonopts, []) -> return (flags, nonopts) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) From 80104eab9a28b9a94fb36653b7cd95b734e16e4d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 14:31:06 -0400 Subject: [PATCH 0146/8313] bugfix --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 8f1c9cc802..7e719888a0 100644 --- a/Core.hs +++ b/Core.hs @@ -26,7 +26,7 @@ shutdown = do needcommit <- Annex.flagIsSet NeedCommit if (needcommit) then liftIO $ Git.run g ["commit", "-m", - "git-annex log update", ".git-annex"] + "git-annex log update", gitStateDir g] else return () {- configure git to use union merge driver on state files, if it is not From e577656fea6f66ef64547374e962adb7fd4ce80a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:09:30 -0400 Subject: [PATCH 0147/8313] relative link fix --- Commands.hs | 25 ++++++++++++------------- Locations.hs | 5 ++--- Utility.hs | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/Commands.hs b/Commands.hs index aed3a19d32..a403a5a48a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -6,6 +6,7 @@ import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory +import System.Path import Data.String.Utils import List import IO @@ -66,13 +67,14 @@ defaultCmd file = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file - stored <- Backend.storeFileKey file g <- Annex.gitRepo + link <- liftIO $ calcGitLink file g + stored <- Backend.storeFileKey file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - liftIO $ setup g key + liftIO $ setup g key link where err = error $ "already annexed " ++ file checkLegal file = do @@ -80,24 +82,21 @@ addCmd file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - setup g key = do + calcGitLink file g = do + cwd <- getCurrentDirectory + let absfile = case (absNormPath cwd file) of + Just f -> f + Nothing -> error $ "unable to normalize " ++ file + return $ relPathDirToDir (parentDir absfile) (Git.workTree g) + setup g key link = do let dest = annexLocation g key let reldest = annexLocationRelative g key createDirectoryIfMissing True (parentDir dest) renameFile file dest - createSymbolicLink ((linkTarget file) ++ reldest) file + createSymbolicLink (link ++ reldest) file Git.run g ["add", file] Git.run g ["commit", "-m", ("git-annex annexed " ++ file), file] - linkTarget file = - -- relies on file being relative to the top of the - -- git repo; just replace each subdirectory with ".." - if (subdirs > 0) - then (join "/" $ take subdirs $ repeat "..") ++ "/" - else "" - where - subdirs = (length $ split "/" file) - 1 - {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () diff --git a/Locations.hs b/Locations.hs index 960a8938d8..733e745537 100644 --- a/Locations.hs +++ b/Locations.hs @@ -31,10 +31,9 @@ annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) -{- Annexed file's location relative to the gitWorkTree -} +{- Annexed file's location relative to git's working tree. -} annexLocationRelative :: Git.Repo -> Key -> FilePath -annexLocationRelative r key = - Git.dir r ++ "/annex/" ++ (keyFile key) +annexLocationRelative r key = Git.dir r ++ "/annex/" ++ (keyFile key) {- Converts a key into a filename fragment. - diff --git a/Utility.hs b/Utility.hs index 349dd9355f..a8324815e5 100644 --- a/Utility.hs +++ b/Utility.hs @@ -4,12 +4,16 @@ module Utility ( withFileLocked, hGetContentsStrict, - parentDir + parentDir, + relPathCwdToDir, + relPathDirToDir, ) where import System.IO import System.Posix.IO import Data.String.Utils +import System.Path +import System.Directory {- Let's just say that Haskell makes reading/writing a file with - file locking excessively difficult. -} @@ -39,3 +43,45 @@ parentDir dir = where dirs = filter (\x -> length x > 0) $ split "/" dir absolute = if ((dir !! 0) == '/') then "/" else "" + +{- Constructs a relative path from the CWD to a directory. + - + - For example, assuming CWD is /tmp/foo/bar: + - relPathCwdToDir "/tmp/foo" == "../" + - relPathCwdToDir "/tmp/foo/bar" == "" + - relPathCwdToDir "/tmp/foo/bar" == "" + -} +relPathCwdToDir :: FilePath -> IO FilePath +relPathCwdToDir dir = do + cwd <- getCurrentDirectory + let absdir = abs cwd dir + return $ relPathDirToDir cwd absdir + where + -- absolute, normalized form of the directory + abs cwd dir = + case (absNormPath cwd dir) of + Just d -> d + Nothing -> error $ "unable to normalize " ++ dir + +{- Constructs a relative path from one directory to another. + - + - Both directories must be absolute, and normalized (eg with absNormpath). + - + - The path will end with "/", unless it is empty. + - -} +relPathDirToDir :: FilePath -> FilePath -> FilePath +relPathDirToDir from to = + if (0 < length path) + then if (endswith "/" path) + then path + else path ++ "/" + else "" + where + pfrom = split "/" from + pto = split "/" to + common = map fst $ filter same $ zip pfrom pto + same (c,d) = c == d + uncommon = drop numcommon pto + dotdots = take ((length pfrom) - numcommon) $ repeat ".." + numcommon = length $ common + path = join "/" $ dotdots ++ uncommon From 0e8cb63aabaa4a80769792fb07b0db2594efd6b0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:25:52 -0400 Subject: [PATCH 0148/8313] update --- TODO | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO b/TODO index b800097a0e..ddd0761658 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,4 @@ -* bug when annexing files while in a subdir of a git repo -* bug when specifying absolute path to files when annexing +* bug: cannot "git annex ../foo" (GitRepo.relative is buggy) * --push/--pull/--want From 395625d0a7c00457f63925beb31078f3eb3d9f79 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:42:36 -0400 Subject: [PATCH 0149/8313] rename file -> WORM --- Backend/Checksum.hs | 1 - Backend/File.hs | 22 ++++++++++------------ Backend/Worm.hs | 16 ++++++++++++++++ BackendList.hs | 4 ++-- git-annex.mdwn | 4 ++-- 5 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 Backend/Worm.hs diff --git a/Backend/Checksum.hs b/Backend/Checksum.hs index bfc789e40e..de98fbf446 100644 --- a/Backend/Checksum.hs +++ b/Backend/Checksum.hs @@ -7,7 +7,6 @@ import qualified Backend.File import Data.Digest.Pure.SHA import BackendTypes --- based on BackendFile just with a different key type backend = Backend.File.backend { name = "checksum", getKey = keyValue diff --git a/Backend/File.hs b/Backend/File.hs index 6267b478a0..eba4b88f83 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -1,5 +1,12 @@ -{- git-annex "file" backend - - -} +{- git-annex pseudo-backend + - + - This backend does not really do any independant data storage, + - it relies on the file contents in .git/annex/ in this repo, + - and other accessible repos. + - + - This is an abstract backend; getKey has to be implemented to complete + - it. + -} module Backend.File (backend) where @@ -19,22 +26,13 @@ import qualified Annex import UUID backend = Backend { - name = "file", - getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = dummyRemove, hasKey = checkKeyFile } --- direct mapping from filename to key -keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key ((name backend), file) - -{- This backend does not really do any independant data storage, - - it relies on the file contents in .git/annex/ in this repo, - - and other accessible repos. So storing a key is - - a no-op. -} +{- Storing a key is a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True diff --git a/Backend/Worm.hs b/Backend/Worm.hs new file mode 100644 index 0000000000..26fffab521 --- /dev/null +++ b/Backend/Worm.hs @@ -0,0 +1,16 @@ +{- git-annex "WORM" backend -- Write Once, Read Many + - -} + +module Backend.Worm (backend) where + +import qualified Backend.File +import BackendTypes + +backend = Backend.File.backend { + name = "WORM", + getKey = keyValue +} + +-- direct mapping from filename to key +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = return $ Just $ Key ((name backend), file) diff --git a/BackendList.hs b/BackendList.hs index b66110905a..93c0464f1b 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,11 +10,11 @@ module BackendList ( import BackendTypes -- When adding a new backend, import it here and add it to the list. -import qualified Backend.File +import qualified Backend.Worm import qualified Backend.Checksum import qualified Backend.Url supportedBackends = - [ Backend.File.backend + [ Backend.Worm.backend , Backend.Checksum.backend , Backend.Url.backend ] diff --git a/git-annex.mdwn b/git-annex.mdwn index 70bd66e954..fba9648dba 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -93,8 +93,8 @@ stable for a given file content, name, and size. Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. -* `file` -- This backend stores the file's content in - `.git/annex/`, and assumes that any file with the same basename +* `WORM` ("Write Once, Read Many") This backend stores the file's content + in `.git/annex/`, and assumes that any file with the same basename has the same content. So with this backend, files can be moved around, but should never be added to or changed. This is the default, and the least expensive backend. From 1f585912e2097234ecad599a072610000e7744f0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 16:52:47 -0400 Subject: [PATCH 0150/8313] use basename as key --- Backend/Worm.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Backend/Worm.hs b/Backend/Worm.hs index 26fffab521..ba79428efa 100644 --- a/Backend/Worm.hs +++ b/Backend/Worm.hs @@ -5,12 +5,14 @@ module Backend.Worm (backend) where import qualified Backend.File import BackendTypes +import Utility +import System.FilePath backend = Backend.File.backend { name = "WORM", getKey = keyValue } --- direct mapping from filename to key +-- direct mapping from basename of filename to key keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key ((name backend), file) +keyValue file = return $ Just $ Key ((name backend), (takeFileName file)) From 8e742bd89e6bd3d83c44847c0455043809c64c89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 18:24:31 -0400 Subject: [PATCH 0151/8313] use some library functions --- Utility.hs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Utility.hs b/Utility.hs index a8324815e5..105c533ece 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,6 +13,7 @@ import System.IO import System.Posix.IO import Data.String.Utils import System.Path +import System.FilePath import System.Directory {- Let's just say that Haskell makes reading/writing a file with @@ -38,11 +39,13 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) + then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ split "/" dir - absolute = if ((dir !! 0) == '/') then "/" else "" + dirs = filter (\x -> length x > 0) $ + split s dir + slash = if (isAbsolute dir) then "" else s + s = [pathSeparator] {- Constructs a relative path from the CWD to a directory. - @@ -68,20 +71,19 @@ relPathCwdToDir dir = do - Both directories must be absolute, and normalized (eg with absNormpath). - - The path will end with "/", unless it is empty. - - -} + -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = if (0 < length path) - then if (endswith "/" path) - then path - else path ++ "/" + then addTrailingPathSeparator path else "" where - pfrom = split "/" from - pto = split "/" to + s = [pathSeparator] + pfrom = split s from + pto = split s to common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common - path = join "/" $ dotdots ++ uncommon + path = join s $ dotdots ++ uncommon From 44b8f7c95de84018044ce3669e62d40eac1b91a7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 18:57:05 -0400 Subject: [PATCH 0152/8313] better worm keys --- Backend/Worm.hs | 24 ++++++++++++++++++++++-- git-annex.mdwn | 8 ++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Backend/Worm.hs b/Backend/Worm.hs index ba79428efa..89fe4bf572 100644 --- a/Backend/Worm.hs +++ b/Backend/Worm.hs @@ -3,16 +3,36 @@ module Backend.Worm (backend) where +import Control.Monad.State import qualified Backend.File import BackendTypes import Utility import System.FilePath +import System.Posix.Files +import Data.Digest.Pure.SHA -- slow, but we only checksum filenames +import qualified Data.ByteString.Lazy.Char8 as B backend = Backend.File.backend { name = "WORM", getKey = keyValue } --- direct mapping from basename of filename to key +-- A SHA1 of the basename of the filename, plus the file size and +-- modification time, is used as the unique part of the key. That +-- allows multiple files with the same names to have different keys, +-- while also allowing a file to be moved around while retaining the +-- same key. +-- +-- The basename of the filename is also included in the key, so it's clear +-- what the original filename was when a user sees the value. keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return $ Just $ Key ((name backend), (takeFileName file)) +keyValue file = do + stat <- liftIO $ getFileStatus file + return $ Just $ Key ((name backend), key stat) + where + key stat = (checksum $ uniqueid stat) ++ sep ++ base + checksum s = show $ sha1 $ B.pack s + uniqueid stat = (show $ fileSize stat) ++ sep ++ + (show $ modificationTime stat) + base = takeFileName file + sep = ":" diff --git a/git-annex.mdwn b/git-annex.mdwn index fba9648dba..2079b5b466 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -94,10 +94,10 @@ Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. * `WORM` ("Write Once, Read Many") This backend stores the file's content - in `.git/annex/`, and assumes that any file with the same basename - has the same content. So with this backend, files can be moved around, - but should never be added to or changed. This is the default, and - the least expensive backend. + in `.git/annex/`, and assumes that any file with the same basename, + size, and modification time has the same content. So with this backend, + files can be moved around, but should never be added to or changed. + This is the default, and the least expensive backend. * `sha1sum` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums From 0989dd2694e4be1bc851d0a50903ceaaa988907a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 18:58:26 -0400 Subject: [PATCH 0153/8313] Revert "use some library functions" This reverts commit 8e742bd89e6bd3d83c44847c0455043809c64c89. meh? --- Utility.hs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Utility.hs b/Utility.hs index 105c533ece..a8324815e5 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,7 +13,6 @@ import System.IO import System.Posix.IO import Data.String.Utils import System.Path -import System.FilePath import System.Directory {- Let's just say that Haskell makes reading/writing a file with @@ -39,13 +38,11 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then slash ++ (join s $ take ((length dirs) - 1) dirs) + then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ - split s dir - slash = if (isAbsolute dir) then "" else s - s = [pathSeparator] + dirs = filter (\x -> length x > 0) $ split "/" dir + absolute = if ((dir !! 0) == '/') then "/" else "" {- Constructs a relative path from the CWD to a directory. - @@ -71,19 +68,20 @@ relPathCwdToDir dir = do - Both directories must be absolute, and normalized (eg with absNormpath). - - The path will end with "/", unless it is empty. - -} + - -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = if (0 < length path) - then addTrailingPathSeparator path + then if (endswith "/" path) + then path + else path ++ "/" else "" where - s = [pathSeparator] - pfrom = split s from - pto = split s to + pfrom = split "/" from + pto = split "/" to common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common - path = join s $ dotdots ++ uncommon + path = join "/" $ dotdots ++ uncommon From 23f95ac6df5f25613ac2904c23821f3ca3054246 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:01:20 -0400 Subject: [PATCH 0154/8313] use some library functions retry with a bugfix --- Utility.hs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Utility.hs b/Utility.hs index a8324815e5..e04b44e6ff 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,6 +13,7 @@ import System.IO import System.Posix.IO import Data.String.Utils import System.Path +import System.FilePath import System.Directory {- Let's just say that Haskell makes reading/writing a file with @@ -38,11 +39,13 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s parentDir :: String -> String parentDir dir = if length dirs > 0 - then absolute ++ (join "/" $ take ((length dirs) - 1) dirs) + then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ split "/" dir - absolute = if ((dir !! 0) == '/') then "/" else "" + dirs = filter (\x -> length x > 0) $ + split s dir + slash = if (not $ isAbsolute dir) then "" else s + s = [pathSeparator] {- Constructs a relative path from the CWD to a directory. - @@ -68,20 +71,19 @@ relPathCwdToDir dir = do - Both directories must be absolute, and normalized (eg with absNormpath). - - The path will end with "/", unless it is empty. - - -} + -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = if (0 < length path) - then if (endswith "/" path) - then path - else path ++ "/" + then addTrailingPathSeparator path else "" where - pfrom = split "/" from - pto = split "/" to + s = [pathSeparator] + pfrom = split s from + pto = split s to common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common - path = join "/" $ dotdots ++ uncommon + path = join s $ dotdots ++ uncommon From 946a7f3f2128704c7b4eeea265a1375c1b60c622 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:32:33 -0400 Subject: [PATCH 0155/8313] update --- git-annex.mdwn | 59 ++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 2079b5b466..21649bfd16 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -1,15 +1,15 @@ git-annex allows managing files with git, without checking the file -contents into git. This is useful when dealing with files larger than git -can currently easily handle, whether due to limitations in memory, -checksumming time, or disk space (only one copy need be stored of an -annexed file). +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. -Even without file content tracking, being able to manage file metadata with -git, move files around and delete files with versioned directory trees, and use -branches and distributed clone, are all very handy reasons to use git. And -annexed files can co-exist in the same git repository with regularly versioned -files, which is convenient for maintaining code, Makefiles, etc that are -associated with annexed files but that benefit from full revision control. +Even without file content tracking, being able to manage files with git, +move files around and delete files with versioned directory trees, and use +branches and distributed clones, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly +versioned files, which is convenient for maintaining documents, Makefiles, +etc that are associated with annexed files but that benefit from full +revision control. Enough broad picture, here's how it actually looks: @@ -17,13 +17,13 @@ Enough broad picture, here's how it actually looks: it with a symlink pointing at the annexed file, and then calls `git add` to version the *symlink*. (If the file has already been annexed, it does nothing.) +* If you use normal git push/pull commands, the annexed file content + won't be transferred, but the symlinks will be. So different clones of a + repository can have different sets of annexed files available. * You can move the symlink around, copy it, delete it, etc, and commit changes as desired using git. Reading the symlink will always get you the annexed file content, or the link may be broken if the content is not currently available. -* If you use normal git push/pull commands, the annexed file contents - won't be sent, but the symlinks will be. So different clones of a repository - can have different sets of annexed files available. * `git annex push $repository` pushes *all* annexed files to the specified repository. * `git annex pull $repository` pulls *all* annexed files from the specified @@ -31,9 +31,9 @@ Enough broad picture, here's how it actually looks: * `git annex want $file` indicates that you want access to a file's content, without immediatly transferring it. * `git annex get $file` is used to transfer a specified file, and/or - files previously indicated with `git annex want`. If a configured repository has it, - or it is available from other key/value storage, it will be immediatly - downloaded. + files previously indicated with `git annex want`. If a configured + repository has it, or it is available from other key/value storage, + it will be immediatly downloaded. * `git annex drop $file` indicates that you no longer want the file's content to be available in this repository. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` @@ -48,17 +48,16 @@ git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1 (configured by annex.numcopies). -`git annex drop` attempts to check all other configured -repositories, to check that N copies of the file exist. If enough -repositories cannot be verified to have it, it will retain the file content -to avoid data loss. +`git annex drop` attempts to check with other git remotes, to check that N +copies of the file exist. If enough repositories cannot be verified to have +it, it will retain the file content to avoid data loss. For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex get $file`, this will transfer it from either Server or USB (depending on which is available), and there are now 3 copies of the file. -Suppose you want to free up space on laptop again, and you `git annex drop` the file +Suppose you want to free up space on Laptop again, and you `git annex drop` the file there. If USB is connected, or Server can be contacted, git-annex can check that it still has a copy of the file, and the content is removed from Laptop. But if USB is currently disconnected, and Server also cannot be @@ -70,17 +69,11 @@ to both USB and Server. Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1, and if they have the -only copies of a file. - -## the .git-annex directory - -The `.git-annex` directory at the top of the repository is used to store -git-annex information that should be propigated between repositories. +copies falling to 1, when USB and Server have N=1. ## key/value storage -git-annex uses a key/value abstraction layer to allow files contents to be +git-annex uses a key/value abstraction layer to allow file contents to be stored in different ways. In theory, any key/value storage system could be used to store the file contents, and git-annex would then retrieve them as needed and put them in `.git/annex/`. @@ -94,15 +87,15 @@ Multiple pluggable backends are supported, and more than one can be used to store different files' contents in a given repository. * `WORM` ("Write Once, Read Many") This backend stores the file's content - in `.git/annex/`, and assumes that any file with the same basename, + only in `.git/annex/`, and assumes that any file with the same basename, size, and modification time has the same content. So with this backend, files can be moved around, but should never be added to or changed. This is the default, and the least expensive backend. -* `sha1sum` -- This backend stores the file's content in +* `SHA1` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums can make it slow for large files. -* `url` -- This backend downloads the file's content from an external URL. +* `URL` -- This backend downloads the file's content from an external URL. ## location tracking @@ -132,7 +125,7 @@ example: * `annex.numcopies` -- number of copies of files to keep (default: 1) * `annex.backends` -- space-separated list of names of the key/value backends to use. The first listed is used to store - new files. (default: file, checksum, url) + new files. (default: "WORM SHA1 URL") * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote From e67887d98b61aeabffc9d1a231421bb00848dd13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:32:56 -0400 Subject: [PATCH 0156/8313] lift to IO --- Backend/File.hs | 13 +++++++++---- UUID.hs | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index eba4b88f83..b2c5c90ebb 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -26,12 +26,16 @@ import qualified Annex import UUID backend = Backend { + name = mustProvide, + getKey = mustProvide, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = dummyRemove, hasKey = checkKeyFile } +mustProvide = error "must provide this field" + {- Storing a key is a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True @@ -74,11 +78,12 @@ copyKeyFile key file = do cantfind = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key + ppuuids <- prettyPrintUUIDs uuids error $ "no available git remotes have: " ++ - (keyFile key) ++ (uuidlist uuids) - uuidlist [] = "" - uuidlist uuids = "\nIt has been seen before in these repositories:\n" ++ - prettyPrintUUIDs uuids + (keyFile key) ++ + if (0 < length uuids) + then "\nIt has been seen before in these repositories:\n" ++ ppuuids + else "" {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () diff --git a/UUID.hs b/UUID.hs index 9348c7b437..3653eeec42 100644 --- a/UUID.hs +++ b/UUID.hs @@ -91,7 +91,7 @@ reposByUUID repos uuids = do {- Pretty-prints a list of UUIDs - TODO: use lookup file to really show pretty names. -} -prettyPrintUUIDs :: [UUID] -> String +prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = - unwords $ map (\u -> "\tUUID "++u++"\n") uuids + return $ unwords $ map (\u -> "\tUUID "++u++"\n") uuids From 5de102d5b90fb621bdb1bd81cf5f562a9a2549e4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 19:33:10 -0400 Subject: [PATCH 0157/8313] rename backends more --- Backend/{Checksum.hs => SHA1.hs} | 8 ++++---- Backend/{Url.hs => URL.hs} | 6 +++--- Backend/{Worm.hs => WORM.hs} | 2 +- BackendList.hs | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) rename Backend/{Checksum.hs => SHA1.hs} (58%) rename Backend/{Url.hs => URL.hs} (92%) rename Backend/{Worm.hs => WORM.hs} (97%) diff --git a/Backend/Checksum.hs b/Backend/SHA1.hs similarity index 58% rename from Backend/Checksum.hs rename to Backend/SHA1.hs index de98fbf446..8c7c99bbd7 100644 --- a/Backend/Checksum.hs +++ b/Backend/SHA1.hs @@ -1,17 +1,17 @@ -{- git-annex "checksum" backend +{- git-annex "SHA1" backend - -} -module Backend.Checksum (backend) where +module Backend.SHA1 (backend) where import qualified Backend.File import Data.Digest.Pure.SHA import BackendTypes backend = Backend.File.backend { - name = "checksum", + name = "SHA1", getKey = keyValue } -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) -keyValue k = error "checksum keyValue unimplemented" -- TODO +keyValue k = error "SHA1 keyValue unimplemented" -- TODO diff --git a/Backend/Url.hs b/Backend/URL.hs similarity index 92% rename from Backend/Url.hs rename to Backend/URL.hs index e237672084..4e87ca4c26 100644 --- a/Backend/Url.hs +++ b/Backend/URL.hs @@ -1,7 +1,7 @@ -{- git-annex "url" backend +{- git-annex "URL" backend - -} -module Backend.Url (backend) where +module Backend.URL (backend) where import Control.Monad.State (liftIO) import Data.String.Utils @@ -10,7 +10,7 @@ import System.Exit import BackendTypes backend = Backend { - name = "url", + name = "URL", getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = downloadUrl, diff --git a/Backend/Worm.hs b/Backend/WORM.hs similarity index 97% rename from Backend/Worm.hs rename to Backend/WORM.hs index 89fe4bf572..9a1e17ec5f 100644 --- a/Backend/Worm.hs +++ b/Backend/WORM.hs @@ -1,7 +1,7 @@ {- git-annex "WORM" backend -- Write Once, Read Many - -} -module Backend.Worm (backend) where +module Backend.WORM (backend) where import Control.Monad.State import qualified Backend.File diff --git a/BackendList.hs b/BackendList.hs index 93c0464f1b..42e237204f 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,13 +10,13 @@ module BackendList ( import BackendTypes -- When adding a new backend, import it here and add it to the list. -import qualified Backend.Worm -import qualified Backend.Checksum -import qualified Backend.Url +import qualified Backend.WORM +import qualified Backend.SHA1 +import qualified Backend.URL supportedBackends = - [ Backend.Worm.backend - , Backend.Checksum.backend - , Backend.Url.backend + [ Backend.WORM.backend + , Backend.SHA1.backend + , Backend.URL.backend ] {- Parses a string with a list of backend names into From 46ac19a51d8994aa0ac978fef3359729ed91c6ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Oct 2010 20:20:16 -0400 Subject: [PATCH 0158/8313] implemented uuid.log --- UUID.hs | 50 +++++++++++++++++++++++++++++++++++++++++++------- git-annex.mdwn | 2 +- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/UUID.hs b/UUID.hs index 3653eeec42..8cdee43de6 100644 --- a/UUID.hs +++ b/UUID.hs @@ -11,7 +11,8 @@ module UUID ( prepUUID, genUUID, reposByUUID, - prettyPrintUUIDs + prettyPrintUUIDs, + describeUUID ) where import Control.Monad.State @@ -19,8 +20,10 @@ import Maybe import List import System.Cmd.Utils import System.IO +import qualified Data.Map as M import qualified GitRepo as Git import Types +import Locations import qualified Annex type UUID = String @@ -29,7 +32,7 @@ configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} -genUUID :: Annex UUID +genUUID :: IO UUID genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h {- Looks up a repo's UUID. May return "" if none is known. @@ -66,7 +69,7 @@ prepUUID = do u <- getUUID g if ("" == u) then do - uuid <- genUUID + uuid <- liftIO $ genUUID setConfig configkey uuid else return () @@ -89,9 +92,42 @@ reposByUUID repos uuids = do u <- getUUID r return $ isJust $ elemIndex u uuids -{- Pretty-prints a list of UUIDs - - TODO: use lookup file to really show pretty names. -} +{- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String -prettyPrintUUIDs uuids = - return $ unwords $ map (\u -> "\tUUID "++u++"\n") uuids +prettyPrintUUIDs uuids = do + m <- uuidMap + return $ unwords $ map (\u -> " "++(prettify m u)++"\n") uuids + where + prettify m u = + if (0 < (length $ findlog m u)) + then u ++ " -- " ++ (findlog m u) + else u + findlog m u = M.findWithDefault "" u m +{- Records a description for a uuid in the uuidLog. -} +describeUUID :: UUID -> String -> Annex () +describeUUID uuid desc = do + m <- uuidMap + let m' = M.insert uuid desc m + log <- uuidLog + liftIO $ writeFile log $ serialize m' + where + serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m + +{- Read and parse the uuidLog into a Map -} +uuidMap :: Annex (M.Map UUID String) +uuidMap = do + log <- uuidLog + s <- liftIO $ catch (readFile log) (\error -> return "") + return $ M.fromList $ map (\l -> pair l) $ lines s + where + pair l = + if (1 < (length $ words l)) + then ((words l) !! 0, unwords $ drop 1 $ words l) + else ("", "") + +{- Filename of uuid.log. -} +uuidLog :: Annex String +uuidLog = do + g <- Annex.gitRepo + return $ (gitStateDir g) ++ "uuid.log" diff --git a/git-annex.mdwn b/git-annex.mdwn index 21649bfd16..1261a196fe 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -109,7 +109,7 @@ Repositories record their UUID and the date when they get or drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) -The optional file `.git-annex/uuid.map` can be created to add a description +The optional file `.git-annex/uuid.log` can be created to add a description to a UUID. If git-annex needs a file from a repository and it cannot find the repository amoung the remotes, it will use the description from this file when asking for the repository to be made available. The file format From 645bc94d3d9e5f08bda74a99e0584768b32da81c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 13:22:48 -0400 Subject: [PATCH 0159/8313] quiet commit of logs --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 7e719888a0..006973fc9d 100644 --- a/Core.hs +++ b/Core.hs @@ -25,7 +25,7 @@ shutdown = do g <- Annex.gitRepo needcommit <- Annex.flagIsSet NeedCommit if (needcommit) - then liftIO $ Git.run g ["commit", "-m", + then liftIO $ Git.run g ["commit", "-q", "-m", "git-annex log update", gitStateDir g] else return () From 1260adbd7700ab9e35f61f4ad94b9cc0536f243e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 13:38:59 -0400 Subject: [PATCH 0160/8313] basic recursion done; skipping git stuff still todo --- Commands.hs | 17 +++++++++-------- TODO | 2 ++ Utility.hs | 14 ++++++++++++++ git-annex.mdwn | 5 +++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index a403a5a48a..f28b3e72b9 100644 --- a/Commands.hs +++ b/Commands.hs @@ -26,20 +26,21 @@ import qualified Remotes - actions to be run in the Annex monad. -} parseCmd :: [String] -> IO ([Flag], [Annex ()]) parseCmd argv = do - (flags, files) <- getopt - case (length files) of + (flags, params) <- getopt + case (length params) of 0 -> error header _ -> do - let c = lookupCmd (files !! 0) - if (0 == length c) - then ret flags defaultCmd files - else ret flags (snd $ c !! 0) $ drop 1 files + let (cmd, locs) = takeCmd params $ lookupCmd (params !! 0) + files <- mapM recurseFiles locs + return (flags, map cmd $ foldl (++) [] files) where - ret flags cmd files = return (flags, makeactions cmd files) - makeactions cmd files = map cmd files getopt = case getOpt Permute options argv of (flags, nonopts, []) -> return (flags, nonopts) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) + takeCmd files cmds = + if (0 == length cmds) + then (defaultCmd, files) + else ((snd $ cmds !! 0), drop 1 files) lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds cmds = [ ("add", addCmd) , ("get", getCmd) diff --git a/TODO b/TODO index ddd0761658..8cb77fe9fb 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,8 @@ * --push/--pull/--want +* isn't pull the same as get? + * recurse on directories * how to handle git mv file? diff --git a/Utility.hs b/Utility.hs index e04b44e6ff..8005fd17cc 100644 --- a/Utility.hs +++ b/Utility.hs @@ -7,12 +7,14 @@ module Utility ( parentDir, relPathCwdToDir, relPathDirToDir, + recurseFiles, ) where import System.IO import System.Posix.IO import Data.String.Utils import System.Path +import System.IO.HVFS import System.FilePath import System.Directory @@ -87,3 +89,15 @@ relPathDirToDir from to = dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common path = join s $ dotdots ++ uncommon + +{- Recursively returns all files and symlinks (to anything) in the specified + - path. If the path is a file, returns only it. Does not follow symlinks to + - directories. -} +recurseFiles :: FilePath -> IO [FilePath] +recurseFiles path = do + find <- recurseDirStat SystemFS path + return $ filesOnly find + where + filesOnly l = map (\(f,s) -> f) $ filter isFile l + isFile (f, HVFSStatEncap s) = + vIsRegularFile s || vIsSymbolicLink s diff --git a/git-annex.mdwn b/git-annex.mdwn index 1261a196fe..1922a1b63c 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -42,6 +42,11 @@ Enough broad picture, here's how it actually looks: * `git annex $file` is a shorthand. If the file is already known, it does `git annex get`, otherwise it does `git annex add`. +Oh yeah, "$file" in the above can be any number of files, or directories. +git-annex automatically recurses into directories, but skips files that are +checked into git (as well as skipping `.git` itself), so "git annex ." works +fine. + ## copies git-annex can be configured to try to keep N copies of a file's content From 5f73fd5b661ecdeae164cc3d5a6c4d0b6113eba7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 13:59:48 -0400 Subject: [PATCH 0161/8313] dropped defaultCmd With recusrion, it doesn't really make sense. --- Commands.hs | 23 ++++++----------------- Core.hs | 2 ++ git-annex.mdwn | 2 -- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Commands.hs b/Commands.hs index f28b3e72b9..3d85b12b93 100644 --- a/Commands.hs +++ b/Commands.hs @@ -29,18 +29,16 @@ parseCmd argv = do (flags, params) <- getopt case (length params) of 0 -> error header - _ -> do - let (cmd, locs) = takeCmd params $ lookupCmd (params !! 0) - files <- mapM recurseFiles locs - return (flags, map cmd $ foldl (++) [] files) + _ -> case (lookupCmd (params !! 0)) of + [] -> error header + [(_,cmd)] -> do + let locs = drop 1 params + files <- mapM recurseFiles locs + return (flags, map cmd $ foldl (++) [] files) where getopt = case getOpt Permute options argv of (flags, nonopts, []) -> return (flags, nonopts) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) - takeCmd files cmds = - if (0 == length cmds) - then (defaultCmd, files) - else ((snd $ cmds !! 0), drop 1 files) lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds cmds = [ ("add", addCmd) , ("get", getCmd) @@ -54,15 +52,6 @@ parseCmd argv = do (join "|" $ map fst cmds) ++ "] file ..." options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] -{- Default mode is to annex a file if it is not already, and otherwise - - get its content. -} -defaultCmd :: FilePath -> Annex () -defaultCmd file = do - r <- liftIO $ Backend.lookupFile file - case (r) of - Just v -> getCmd file - Nothing -> addCmd file - {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () diff --git a/Core.hs b/Core.hs index 006973fc9d..19d1737c3a 100644 --- a/Core.hs +++ b/Core.hs @@ -58,3 +58,5 @@ inAnnex :: Key -> Annex Bool inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key + +{- -} diff --git a/git-annex.mdwn b/git-annex.mdwn index 1922a1b63c..2796f48fb1 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -39,8 +39,6 @@ Enough broad picture, here's how it actually looks: * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you accidentially added a file. -* `git annex $file` is a shorthand. If the file - is already known, it does `git annex get`, otherwise it does `git annex add`. Oh yeah, "$file" in the above can be any number of files, or directories. git-annex automatically recurses into directories, but skips files that are From 5a32804115a73d3c6fb2de17a1f9a6c628beba5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 14:20:43 -0400 Subject: [PATCH 0162/8313] add inGit/notInGit --- GitRepo.hs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f3bb5427ad..5981a6ca11 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -20,10 +20,13 @@ module GitRepo ( configMap, configRead, run, + pipeRead, attributes, remotes, remotesAdd, - repoRemoteName + repoRemoteName, + inGit, + notInGit ) where import Directory @@ -167,16 +170,30 @@ run repo params = assertlocal repo $ do return () {- Runs a git subcommand and returns its output. -} -gitPipeRead :: Repo -> [String] -> IO String -gitPipeRead repo params = assertlocal repo $ do +pipeRead :: Repo -> [String] -> IO String +pipeRead repo params = assertlocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do ret <- hGetContentsStrict h return ret +{- Passed a location, recursively scans for all files that + - are checked into git at that location. -} +inGit :: Repo -> FilePath -> IO [FilePath] +inGit repo location = do + s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard"] + return $ lines s + +{- Passed a location, recursively scans for all files that are not checked + - into git, and not gitignored. -} +notInGit :: Repo -> FilePath -> IO [FilePath] +notInGit repo location = do + s <- pipeRead repo ["ls-files", "--others", "--exclude-standard"] + return $ lines s + {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo configRead repo = assertlocal repo $ do - {- Cannot use gitPipeRead because it relies on the config having + {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory bracket_ (changeWorkingDirectory (top repo)) From bfa581a218719c46dbc19a212a005b0cf2e145c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 14:58:14 -0400 Subject: [PATCH 0163/8313] bugfix --- GitRepo.hs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 5981a6ca11..76150b3095 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -25,8 +25,8 @@ module GitRepo ( remotes, remotesAdd, repoRemoteName, - inGit, - notInGit + inRepo, + notInRepo ) where import Directory @@ -178,16 +178,16 @@ pipeRead repo params = assertlocal repo $ do {- Passed a location, recursively scans for all files that - are checked into git at that location. -} -inGit :: Repo -> FilePath -> IO [FilePath] -inGit repo location = do - s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard"] +inRepo :: Repo -> FilePath -> IO [FilePath] +inRepo repo location = do + s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard", location] return $ lines s {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} -notInGit :: Repo -> FilePath -> IO [FilePath] -notInGit repo location = do - s <- pipeRead repo ["ls-files", "--others", "--exclude-standard"] +notInRepo :: Repo -> FilePath -> IO [FilePath] +notInRepo repo location = do + s <- pipeRead repo ["ls-files", "--others", "--exclude-standard", location] return $ lines s {- Runs git config and populates a repo with its config. -} From e80160380a16fbeb38f21f4683917b49a9221a91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 14:58:35 -0400 Subject: [PATCH 0164/8313] now finds files in git or not depending on what command wants --- Commands.hs | 56 ++++++++++++++++++++++++++++++++++++---------------- Utility.hs | 15 +------------- git-annex.hs | 2 +- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Commands.hs b/Commands.hs index 3d85b12b93..a2535001e7 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,35 +21,57 @@ import LocationLog import Types import Core import qualified Remotes +import qualified BackendTypes + +data CmdWants = FilesInGit | FilesNotInGit | RepoName +data Command = Command { + cmdname :: String, + cmdaction :: (String -> Annex ()), + cmdwants :: CmdWants +} + +cmds :: [Command] +cmds = [ (Command "add" addCmd FilesNotInGit) + , (Command "get" getCmd FilesInGit) + , (Command "drop" dropCmd FilesInGit) + , (Command "want" wantCmd FilesInGit) + , (Command "push" pushCmd RepoName) + , (Command "pull" pullCmd RepoName) + , (Command "unannex" unannexCmd FilesInGit) + ] + +{- Finds the type of parameters a command wants, from among the passed + - parameter list. -} +findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] +findWanted FilesNotInGit params repo = do + files <- mapM (Git.notInRepo repo) params + return $ foldl (++) [] files +findWanted FilesInGit params repo = do + files <- mapM (Git.inRepo repo) params + return $ foldl (++) [] files +findWanted RepoName params _ = do + return $ params {- Parses command line and returns a list of flags and a list of - actions to be run in the Annex monad. -} -parseCmd :: [String] -> IO ([Flag], [Annex ()]) -parseCmd argv = do +parseCmd :: [String] -> AnnexState -> IO ([Flag], [Annex ()]) +parseCmd argv state = do (flags, params) <- getopt case (length params) of 0 -> error header _ -> case (lookupCmd (params !! 0)) of [] -> error header - [(_,cmd)] -> do - let locs = drop 1 params - files <- mapM recurseFiles locs - return (flags, map cmd $ foldl (++) [] files) + [Command _ action want] -> do + f <- findWanted want (drop 1 params) + (BackendTypes.repo state) + return (flags, map action f) where getopt = case getOpt Permute options argv of - (flags, nonopts, []) -> return (flags, nonopts) + (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) - lookupCmd cmd = filter (\(c, a) -> c == cmd) cmds - cmds = [ ("add", addCmd) - , ("get", getCmd) - , ("drop", dropCmd) - , ("want", wantCmd) - , ("push", pushCmd) - , ("pull", pullCmd) - , ("unannex", unannexCmd) - ] + lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ - (join "|" $ map fst cmds) ++ "] file ..." + (join "|" $ map cmdname cmds) ++ "] file ..." options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Annexes a file, storing it in a backend, and then moving it into diff --git a/Utility.hs b/Utility.hs index 8005fd17cc..e4278ff3f6 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,8 +6,7 @@ module Utility ( hGetContentsStrict, parentDir, relPathCwdToDir, - relPathDirToDir, - recurseFiles, + relPathDirToDir ) where import System.IO @@ -89,15 +88,3 @@ relPathDirToDir from to = dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common path = join s $ dotdots ++ uncommon - -{- Recursively returns all files and symlinks (to anything) in the specified - - path. If the path is a file, returns only it. Does not follow symlinks to - - directories. -} -recurseFiles :: FilePath -> IO [FilePath] -recurseFiles path = do - find <- recurseDirStat SystemFS path - return $ filesOnly find - where - filesOnly l = map (\(f,s) -> f) $ filter isFile l - isFile (f, HVFSStatEncap s) = - vIsRegularFile s || vIsSymbolicLink s diff --git a/git-annex.hs b/git-annex.hs index cd67242afa..01416f6dd9 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -12,9 +12,9 @@ import qualified GitRepo as Git main = do args <- getArgs - (flags, actions) <- parseCmd args gitrepo <- Git.repoFromCwd state <- new gitrepo + (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches exceptions, not stopping From eed4a7fcdfbae821485d120055c8aec4824ecb3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:10:07 -0400 Subject: [PATCH 0165/8313] tweak --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index a2535001e7..11f808c211 100644 --- a/Commands.hs +++ b/Commands.hs @@ -71,7 +71,7 @@ parseCmd argv state = do (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ - (join "|" $ map cmdname cmds) ++ "] file ..." + (join "|" $ map cmdname cmds) ++ "] ..." options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Annexes a file, storing it in a backend, and then moving it into From 684011175cc75bb6a667e65ba0ec6cabd1f0897a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:22:47 -0400 Subject: [PATCH 0166/8313] update --- git-annex.mdwn | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/git-annex.mdwn b/git-annex.mdwn index 2796f48fb1..bb216f0383 100644 --- a/git-annex.mdwn +++ b/git-annex.mdwn @@ -40,16 +40,15 @@ Enough broad picture, here's how it actually looks: if you're just done with a file; only use `unannex` if you accidentially added a file. -Oh yeah, "$file" in the above can be any number of files, or directories. -git-annex automatically recurses into directories, but skips files that are -checked into git (as well as skipping `.git` itself), so "git annex ." works -fine. +Oh yeah, "$file" in the above can be any number of files, or directories, +same as you'd pass to "git add" or "git rm". +So "git annex add ." or "git annex get dir/" work fine. ## copies git-annex can be configured to try to keep N copies of a file's content -available across all repositories. By default, N is 1 (configured by -annex.numcopies). +available across all repositories. By default, N is 1; it is configured by +annex.numcopies. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have @@ -105,7 +104,11 @@ to store different files' contents in a given repository. git-annex keeps track of on which repository it last saw a file's content. This can be useful when using it for archiving with offline storage. When you indicate you want a file, git-annex will tell you which repositories -have the file's content. +have the file's content. For example: + + # git annex get myfile + git-annex: unable to get: myfile + To get that file, need access to one of these remotes: usbdrive Location tracking information is stored in `.git-annex/$key.log`. Repositories record their UUID and the date when they get or drop @@ -113,7 +116,7 @@ a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) The optional file `.git-annex/uuid.log` can be created to add a description -to a UUID. If git-annex needs a file from a repository and it cannot find +to a UUID. If git-annex needs a file from some repository, and it cannot find the repository amoung the remotes, it will use the description from this file when asking for the repository to be made available. The file format is a UUID, a space, and the rest of the line is its description. For From a31dc74806f165e01f56dbc3322e738a921cc6e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:23:03 -0400 Subject: [PATCH 0167/8313] update --- TODO | 7 ++----- git-annex.mdwn => doc/git-annex.mdwn | 0 2 files changed, 2 insertions(+), 5 deletions(-) rename git-annex.mdwn => doc/git-annex.mdwn (100%) diff --git a/TODO b/TODO index 8cb77fe9fb..8fc17fca99 100644 --- a/TODO +++ b/TODO @@ -1,11 +1,8 @@ -* bug: cannot "git annex ../foo" (GitRepo.relative is buggy) +* bug: cannot "git annex ../foo" (GitRepo.relative is buggy and + git-ls-files also refuses w/o --full-name, which would need other changes) * --push/--pull/--want -* isn't pull the same as get? - -* recurse on directories - * how to handle git mv file? * finish BackendChecksum diff --git a/git-annex.mdwn b/doc/git-annex.mdwn similarity index 100% rename from git-annex.mdwn rename to doc/git-annex.mdwn From 81d628a8cd6f20c2ef336271ae03376dc75b6920 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 15:58:42 -0400 Subject: [PATCH 0168/8313] updatte --- doc/git-annex.mdwn | 102 +++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index bb216f0383..ad45c08426 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -11,34 +11,48 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. +My motivation for git-annex was the growing number of external drives I +use. Some are used to archive data, others hold backups, and yet others +come with me when I'm away from home to carry data that doesn't fit on my +netbook. Maintaining all that was a nightmare, lots of ad-hoc moving files +around, rsyncing files (unison is too slow), and deleting multiple copies +of files from multiple places. I realized what what I needed was revision +control where each drive was a repository, and where copying the files +around, and deciding which copies were safe to delete was automated. +I posted about this to the VCS-home mailing list and got a great suggestion +to make it support arbitrary key-value stores. A week of coding later, +and git-annex is born. + Enough broad picture, here's how it actually looks: * `git annex add $file` moves the file into `.git/annex/`, and replaces it with a symlink pointing at the annexed file, and then calls `git add` to version the *symlink*. (If the file has already been annexed, it does - nothing.) -* If you use normal git push/pull commands, the annexed file content - won't be transferred, but the symlinks will be. So different clones of a - repository can have different sets of annexed files available. -* You can move the symlink around, copy it, delete it, etc, and commit changes + nothing.) + + If you then use normal git push/pull commands, the annexed file content + won't be transferred between repositories, but the symlinks will be. + So different clones of a repository can have different sets of annexed + files available. + + You can move the symlink around, copy it, delete it, etc, and commit changes as desired using git. Reading the symlink will always get you the annexed file content, or the link may be broken if the content is not currently available. +* `git annex get $file` is used to transfer a specified file from the + backend storage to the current repository. +* `git annex drop $file` indicates that you no longer want the file's + content to be available in this repository. * `git annex push $repository` pushes *all* annexed files to the specified repository. * `git annex pull $repository` pulls *all* annexed files from the specified repository. -* `git annex want $file` indicates that you want access to a file's - content, without immediatly transferring it. -* `git annex get $file` is used to transfer a specified file, and/or - files previously indicated with `git annex want`. If a configured - repository has it, or it is available from other key/value storage, - it will be immediatly downloaded. -* `git annex drop $file` indicates that you no longer want the file's - content to be available in this repository. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you accidentially added a file. +* `git annex describe "some description"` allows associating some description + (such as "USB archive drive 1") with a repository. This can help with + finding it later, see "Location Tracking" below. Oh yeah, "$file" in the above can be any number of files, or directories, same as you'd pass to "git add" or "git rm". @@ -73,10 +87,10 @@ Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of copies falling to 1, when USB and Server have N=1. -## key/value storage +## key-value storage -git-annex uses a key/value abstraction layer to allow file contents to be -stored in different ways. In theory, any key/value storage system could be +git-annex uses a key-value abstraction layer to allow file contents to be +stored in different ways. In theory, any key-value storage system could be used to store the file contents, and git-annex would then retrieve them as needed and put them in `.git/annex/`. @@ -101,36 +115,40 @@ to store different files' contents in a given repository. ## location tracking -git-annex keeps track of on which repository it last saw a file's content. -This can be useful when using it for archiving with offline storage. When -you indicate you want a file, git-annex will tell you which repositories -have the file's content. For example: - - # git annex get myfile - git-annex: unable to get: myfile - To get that file, need access to one of these remotes: usbdrive - -Location tracking information is stored in `.git-annex/$key.log`. +git-annex keeps track of in which repositories it last saw a file's content. +This location tracking information is stored in `.git-annex/$key.log`. Repositories record their UUID and the date when they get or drop a file's content. (Git is configured to use a union merge for this file, so the lines may be in arbitrary order, but it will never conflict.) -The optional file `.git-annex/uuid.log` can be created to add a description -to a UUID. If git-annex needs a file from some repository, and it cannot find -the repository amoung the remotes, it will use the description from this -file when asking for the repository to be made available. The file format -is a UUID, a space, and the rest of the line is its description. For -example: +This location tracking information is useful if you have multiple +repositories, and not all are always accessible. For example, perhaps one +is on a home file server, and you are away from home. Then git-annex can +tell you what git remote it needs access to in order to get a file: - UUID d3d2474c-d5c3-11df-80a9-002170d25c55 USB drive in red enclosure - UUID 60cf39c8-d5c6-11df-aa8b-93fda39008d6 my colocated server + # git annex get myfile + git-annex: unable to get file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile + To get that file, need access to one of these remotes: home + +Another way the location tracking comes in handy is if you put repositories +on removable USB drives, that might be archived away offline in a safe +place. In this sort of case, you probably don't have a git remotes +configured for every USB drive. So git-annex may have to resort to talking +about repository UUIDs. If you have previously used "git annex describe" +in those repositories, it will include their description to help you with +finding them: + + git-annex: no available git remotes have file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile + It has been seen before in these repositories: + c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 + e1938fee-d95b-11df-96cc-002170d25c55 ## configuration * `annex.uuid` -- a unique UUID for this repository * `annex.numcopies` -- number of copies of files to keep (default: 1) * `annex.backends` -- space-separated list of names of - the key/value backends to use. The first listed is used to store + the key-value backends to use. The first listed is used to store new files. (default: "WORM SHA1 URL") * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. @@ -165,13 +183,13 @@ Need a way to tell how much free space is available on the disk containing a given repository. The repository may be remote, so ssh may need to be used. -Similarly, need a way to tell the size of a file before downloading it from -remote, to check local disk space. +Similarly, need a way to tell the size of a file before copying it from +a remote, to check local disk space. -### auto-drop files on rm +### auto-drop on rm -When git-rm removed a file, it should get dropped too. Of course, it may -not be dropped right away, depending on number of copies available. +When git-rm removed a file, its key should get dropped too. Of course, it +may not be dropped right away, depending on number of copies available. ### branching @@ -180,3 +198,7 @@ and the user switched between them, git-annex will see different logs in the different branches, and so may miss info about what remotes have which files (though it can re-learn). An alternative would be to store the log data directly in the git repo as `pristine-tar` does. + +## contact + +Joey Hess From 6d13ae10cf1d295b64855984f5a526f8209f3341 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:15:31 -0400 Subject: [PATCH 0169/8313] git annex describe --- Backend.hs | 6 +++--- Backend/File.hs | 4 ++-- BackendTypes.hs | 2 +- Commands.hs | 24 +++++++++++++++++------- UUID.hs | 10 +++++++--- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/Backend.hs b/Backend.hs index f419831d28..636557d7d4 100644 --- a/Backend.hs +++ b/Backend.hs @@ -1,7 +1,7 @@ -{- git-annex key/value storage backends +{- git-annex key-value storage backends - - - git-annex uses a key/value abstraction layer to allow files contents to be - - stored in different ways. In theory, any key/value storage system could be + - git-annex uses a key-value abstraction layer to allow files contents to be + - stored in different ways. In theory, any key-value storage system could be - used to store the file contents, and git-annex would then retrieve them - as needed and put them in `.git/annex/`. - diff --git a/Backend/File.hs b/Backend/File.hs index b2c5c90ebb..c443b4f7ad 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -58,7 +58,7 @@ copyKeyFile key file = do else return () trycopy remotes remotes where - trycopy full [] = error $ "unable to get: " ++ (keyFile key) ++ "\n" ++ + trycopy full [] = error $ "unable to get file with key: " ++ (keyFile key) ++ "\n" ++ "To get that file, need access to one of these remotes: " ++ (Remotes.list full) trycopy full (r:rs) = do @@ -79,7 +79,7 @@ copyKeyFile key file = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key ppuuids <- prettyPrintUUIDs uuids - error $ "no available git remotes have: " ++ + error $ "no available git remotes have file with key: " ++ (keyFile key) ++ if (0 < length uuids) then "\nIt has been seen before in these repositories:\n" ++ ppuuids diff --git a/BackendTypes.hs b/BackendTypes.hs index 13ffde7f89..41bc77858e 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -49,7 +49,7 @@ backendName (Key (b,k)) = b keyFrag :: Key -> KeyFrag keyFrag (Key (b,k)) = k --- this structure represents a key/value backend +-- this structure represents a key-value backend data Backend = Backend { -- name of this backend name :: String, diff --git a/Commands.hs b/Commands.hs index 11f808c211..1f91280112 100644 --- a/Commands.hs +++ b/Commands.hs @@ -23,7 +23,7 @@ import Core import qualified Remotes import qualified BackendTypes -data CmdWants = FilesInGit | FilesNotInGit | RepoName +data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -34,10 +34,10 @@ cmds :: [Command] cmds = [ (Command "add" addCmd FilesNotInGit) , (Command "get" getCmd FilesInGit) , (Command "drop" dropCmd FilesInGit) - , (Command "want" wantCmd FilesInGit) , (Command "push" pushCmd RepoName) , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) + , (Command "describe" describeCmd SingleString) ] {- Finds the type of parameters a command wants, from among the passed @@ -49,6 +49,8 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files +findWanted SingleString params _ = do + return $ [unwords params] findWanted RepoName params _ = do return $ params @@ -150,11 +152,8 @@ getCmd file = notinBackend file err $ \(key, backend) -> do where err = error $ "not annexed " ++ file -{- Indicates a file is wanted. -} -wantCmd :: FilePath -> Annex () -wantCmd file = do error "not implemented" -- TODO - -{- Indicates a file is not wanted. -} +{- Indicates a file's content is not wanted anymore, and should be removed + - if it's safe to do so. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do force <- Annex.flagIsSet Force @@ -185,6 +184,17 @@ pushCmd reponame = do error "not implemented" -- TODO pullCmd :: String -> Annex () pullCmd reponame = do error "not implemented" -- TODO +{- Stores description for the repository. -} +describeCmd :: String -> Annex () +describeCmd description = do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + log <- uuidLog + liftIO $ Git.run g ["add", log] + Annex.flagChange NeedCommit True + liftIO $ putStrLn "description set" + {- Updates the LocationLog when a key's presence changes. -} logStatus :: Key -> LogStatus -> Annex () logStatus key status = do diff --git a/UUID.hs b/UUID.hs index 8cdee43de6..3e6991d485 100644 --- a/UUID.hs +++ b/UUID.hs @@ -12,7 +12,8 @@ module UUID ( genUUID, reposByUUID, prettyPrintUUIDs, - describeUUID + describeUUID, + uuidLog ) where import Control.Monad.State @@ -25,6 +26,7 @@ import qualified GitRepo as Git import Types import Locations import qualified Annex +import Utility type UUID = String @@ -110,7 +112,7 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m log <- uuidLog - liftIO $ writeFile log $ serialize m' + liftIO $ withFileLocked log WriteMode (\h -> hPutStr h $ serialize m') where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m @@ -118,7 +120,9 @@ describeUUID uuid desc = do uuidMap :: Annex (M.Map UUID String) uuidMap = do log <- uuidLog - s <- liftIO $ catch (readFile log) (\error -> return "") + s <- liftIO $ catch + (withFileLocked log ReadMode $ \h -> hGetContentsStrict h) + (\error -> return "") return $ M.fromList $ map (\l -> pair l) $ lines s where pair l = From 909f619c07699fe6c76d40bb4649e07737a0b9ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:20:49 -0400 Subject: [PATCH 0170/8313] tweaks --- Annex.hs | 1 + Backend.hs | 1 + Backend/File.hs | 1 + Backend/SHA1.hs | 3 ++- Backend/URL.hs | 1 + Backend/WORM.hs | 7 ++++--- BackendTypes.hs | 1 + Commands.hs | 1 + Core.hs | 1 + GitRepo.hs | 1 + LocationLog.hs | 1 + Locations.hs | 1 + Remotes.hs | 1 + UUID.hs | 3 +++ doc/git-annex.mdwn | 10 +++++----- git-annex.hs | 4 ++-- 16 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Annex.hs b/Annex.hs index 08607cafa8..68c0cb88e2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -13,6 +13,7 @@ module Annex ( ) where import Control.Monad.State + import qualified GitRepo as Git import Types import qualified BackendTypes as Backend diff --git a/Backend.hs b/Backend.hs index 636557d7d4..874191924d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,6 +27,7 @@ import System.Directory import System.FilePath import Data.String.Utils import System.Posix.Files + import BackendList import Locations import qualified GitRepo as Git diff --git a/Backend/File.hs b/Backend/File.hs index c443b4f7ad..f5237f7210 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,7 @@ import System.IO import System.Cmd import System.Exit import Control.Exception + import BackendTypes import LocationLog import Locations diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 8c7c99bbd7..c01e01a723 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -3,8 +3,9 @@ module Backend.SHA1 (backend) where -import qualified Backend.File import Data.Digest.Pure.SHA + +import qualified Backend.File import BackendTypes backend = Backend.File.backend { diff --git a/Backend/URL.hs b/Backend/URL.hs index 4e87ca4c26..9e64e04996 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -7,6 +7,7 @@ import Control.Monad.State (liftIO) import Data.String.Utils import System.Cmd import System.Exit + import BackendTypes backend = Backend { diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 9a1e17ec5f..420f336e90 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -4,14 +4,15 @@ module Backend.WORM (backend) where import Control.Monad.State -import qualified Backend.File -import BackendTypes -import Utility import System.FilePath import System.Posix.Files import Data.Digest.Pure.SHA -- slow, but we only checksum filenames import qualified Data.ByteString.Lazy.Char8 as B +import qualified Backend.File +import BackendTypes +import Utility + backend = Backend.File.backend { name = "WORM", getKey = keyValue diff --git a/BackendTypes.hs b/BackendTypes.hs index 41bc77858e..49bd1bceb8 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -7,6 +7,7 @@ module BackendTypes where import Control.Monad.State (StateT) import Data.String.Utils + import qualified GitRepo as Git -- command-line flags diff --git a/Commands.hs b/Commands.hs index 1f91280112..c477a81fde 100644 --- a/Commands.hs +++ b/Commands.hs @@ -10,6 +10,7 @@ import System.Path import Data.String.Utils import List import IO + import qualified GitRepo as Git import qualified Annex import Utility diff --git a/Core.hs b/Core.hs index 19d1737c3a..fcbce4163a 100644 --- a/Core.hs +++ b/Core.hs @@ -5,6 +5,7 @@ module Core where import System.IO import System.Directory import Control.Monad.State (liftIO) + import Types import Locations import UUID diff --git a/GitRepo.hs b/GitRepo.hs index 76150b3095..32383197b5 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -42,6 +42,7 @@ import Data.String.Utils import Data.Map as Map hiding (map, split) import Network.URI import Maybe + import Utility {- A git repository can be on local disk or remote. Not to be confused diff --git a/LocationLog.hs b/LocationLog.hs index ba91787049..c0d6170b2e 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -29,6 +29,7 @@ import qualified Data.Map as Map import System.IO import System.Directory import Data.Char + import qualified GitRepo as Git import Utility import UUID diff --git a/Locations.hs b/Locations.hs index 733e745537..4978500624 100644 --- a/Locations.hs +++ b/Locations.hs @@ -11,6 +11,7 @@ module Locations ( ) where import Data.String.Utils + import Types import qualified BackendTypes as Backend import qualified GitRepo as Git diff --git a/Remotes.hs b/Remotes.hs index 2fffcffa7e..3774f993ce 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -12,6 +12,7 @@ import qualified Data.Map as Map import Data.String.Utils import List import Maybe + import Types import qualified GitRepo as Git import qualified Annex diff --git a/UUID.hs b/UUID.hs index 3e6991d485..6bd483a18c 100644 --- a/UUID.hs +++ b/UUID.hs @@ -21,7 +21,9 @@ import Maybe import List import System.Cmd.Utils import System.IO +import System.Directory import qualified Data.Map as M + import qualified GitRepo as Git import Types import Locations @@ -112,6 +114,7 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m log <- uuidLog + liftIO $ createDirectoryIfMissing True (parentDir log) liftIO $ withFileLocked log WriteMode (\h -> hPutStr h $ serialize m') where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index ad45c08426..e552dc770a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -16,12 +16,12 @@ use. Some are used to archive data, others hold backups, and yet others come with me when I'm away from home to carry data that doesn't fit on my netbook. Maintaining all that was a nightmare, lots of ad-hoc moving files around, rsyncing files (unison is too slow), and deleting multiple copies -of files from multiple places. I realized what what I needed was revision -control where each drive was a repository, and where copying the files -around, and deciding which copies were safe to delete was automated. +of files from multiple places. I realized what what I needed was a form of +revision control where each drive was a repository, and where copying the +files around, and deciding which copies were safe to delete was automated. I posted about this to the VCS-home mailing list and got a great suggestion -to make it support arbitrary key-value stores. A week of coding later, -and git-annex is born. +to make it support arbitrary key-value stores, for more generality and +flexability. A week of coding later, and git-annex is born. Enough broad picture, here's how it actually looks: diff --git a/git-annex.hs b/git-annex.hs index 01416f6dd9..f4f0cfcdfd 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -3,17 +3,17 @@ import Control.Exception import System.IO import System.Environment + import qualified Annex import Types import Core import Commands -import Annex import qualified GitRepo as Git main = do args <- getArgs gitrepo <- Git.repoFromCwd - state <- new gitrepo + state <- Annex.new gitrepo (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] From 117e97ea30f6e414a99f413d1e2050da84edd9df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:41:24 -0400 Subject: [PATCH 0171/8313] debianization --- INSTALL | 5 +++++ Makefile | 4 ++++ debian/changelog | 5 +++++ debian/compat | 1 + debian/control | 26 ++++++++++++++++++++++++++ debian/docs | 1 + debian/rules | 7 +++++++ 7 files changed, 49 insertions(+) create mode 100644 INSTALL create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/docs create mode 100755 debian/rules diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000000..5c149dc453 --- /dev/null +++ b/INSTALL @@ -0,0 +1,5 @@ +To build and use git-annex, you will need: + +* ghc +* These haskell libraries: MissingH +* a "uuid" command diff --git a/Makefile b/Makefile index 876407de09..d1fcbbeeeb 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ git-annex: mkdir -p build ghc -odir build -hidir build --make git-annex +install: + install -d $(DESTDIR)/usr/bin + install git-annex $(DESTDIR)/usr/bin + clean: rm -rf build git-annex diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000..998754777e --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +git-annex (0.01) UNRELEASED; urgency=low + + * First release + + -- Joey Hess Thu, 09 Sep 2010 08:24:58 -0400 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000000..7f8f011eb7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..fa4fd61159 --- /dev/null +++ b/debian/control @@ -0,0 +1,26 @@ +Source: git-annex +Section: utils +Priority: optional +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev +Maintainer: Joey Hess +Standards-Version: 3.9.1 +Vcs-Git: git://git.kitenet.net/git-annex +Homepage: http://kitenet.net/~joey/code/git-annex/ + +Package: git-annex +Architecture: any +Section: utils +Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid +Description: manage files with git, without checking their contents into git + git-annex allows managing files with git, without checking the file + contents into git. While that may seem paradoxical, it is useful when + dealing with files larger than git can currently easily handle, whether due + to limitations in memory, checksumming time, or disk space. + . + Even without file content tracking, being able to manage files with git, + move files around and delete files with versioned directory trees, and use + branches and distributed clones, are all very handy reasons to use git. And + annexed files can co-exist in the same git repository with regularly + versioned files, which is convenient for maintaining documents, Makefiles, + etc that are associated with annexed files but that benefit from full + revision control. diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000000..9de86edc7e --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +doc/*.mdwn diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..e0a209a72c --- /dev/null +++ b/debian/rules @@ -0,0 +1,7 @@ +#!/usr/bin/make -f +%: + dh $@ + +# Not intended for use by anyone except the author. +announcedir: + @echo ${HOME}/src/joeywiki/code/git-annex/news From d1a455bdb4bd96c4ecf590b56f0517538c9d8eb0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 16:52:20 -0400 Subject: [PATCH 0172/8313] need SHA too --- INSTALL | 2 +- debian/control | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index 5c149dc453..11f573488c 100644 --- a/INSTALL +++ b/INSTALL @@ -1,5 +1,5 @@ To build and use git-annex, you will need: * ghc -* These haskell libraries: MissingH +* These haskell libraries: MissingH SHA * a "uuid" command diff --git a/debian/control b/debian/control index fa4fd61159..e58f55af9e 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From 4da793b51441e65c48bbf680d8650c57a4c9874d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 17:08:08 -0400 Subject: [PATCH 0173/8313] up --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 8fc17fca99..ed150cff02 100644 --- a/TODO +++ b/TODO @@ -5,4 +5,7 @@ * how to handle git mv file? +* Support for remote git repositories (ssh:// specifically can be made to + work, although the other end probably needs to have git-annex installed..) + * finish BackendChecksum From 91c9cd2b8eb9934eebf9a20adde7794a103d144a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 17:15:58 -0400 Subject: [PATCH 0174/8313] todo --- INSTALL | 2 +- TODO | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 11f573488c..9594448e70 100644 --- a/INSTALL +++ b/INSTALL @@ -2,4 +2,4 @@ To build and use git-annex, you will need: * ghc * These haskell libraries: MissingH SHA -* a "uuid" command +* uuid diff --git a/TODO b/TODO index ed150cff02..5bef280e21 100644 --- a/TODO +++ b/TODO @@ -8,4 +8,8 @@ * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) +* Find a way to copy a file with a progress bar, while still preserving + stat. Easiest way might be to use pv and fix up the permissions etc + after? + * finish BackendChecksum From b3e5590fb2995d73d5e69a3954fcb11d9fe98d28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 17:44:59 -0400 Subject: [PATCH 0175/8313] update --- Remotes.hs | 6 ++++-- TODO | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 3774f993ce..f21f5a6ba4 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -6,8 +6,8 @@ module Remotes ( tryGitConfigRead ) where +import Control.Exception import Control.Monad.State (liftIO) -import IO import qualified Data.Map as Map import Data.String.Utils import List @@ -85,7 +85,9 @@ tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do - result <- liftIO $ try (Git.configRead r) + -- configRead can fail due to IO error or + -- for other reasons; catch all possible exceptions + result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of Left err -> return Nothing Right r' -> do diff --git a/TODO b/TODO index 5bef280e21..cd94f03bcb 100644 --- a/TODO +++ b/TODO @@ -8,6 +8,9 @@ * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) +* Copy files atomically, don't leaf a partial key on interrupt. + (Fix for URL download too..) + * Find a way to copy a file with a progress bar, while still preserving stat. Easiest way might be to use pv and fix up the permissions etc after? From be5b1defeb2f3b5499fd3c002fcdba5b5e9d15f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 19:43:32 -0400 Subject: [PATCH 0176/8313] add --no-commit option --- BackendTypes.hs | 2 +- Commands.hs | 37 ++++++++++++++++++++++++++----------- Core.hs | 3 ++- TODO | 3 +++ 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/BackendTypes.hs b/BackendTypes.hs index 49bd1bceb8..06ecfb8fe5 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -11,7 +11,7 @@ import Data.String.Utils import qualified GitRepo as Git -- command-line flags -data Flag = Force | NeedCommit +data Flag = Force | NoCommit | NeedCommit deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, diff --git a/Commands.hs b/Commands.hs index c477a81fde..63ca6b5e4e 100644 --- a/Commands.hs +++ b/Commands.hs @@ -32,7 +32,8 @@ data Command = Command { } cmds :: [Command] -cmds = [ (Command "add" addCmd FilesNotInGit) +cmds = [ + (Command "add" addCmd FilesNotInGit) , (Command "get" getCmd FilesInGit) , (Command "drop" dropCmd FilesInGit) , (Command "push" pushCmd RepoName) @@ -41,6 +42,11 @@ cmds = [ (Command "add" addCmd FilesNotInGit) , (Command "describe" describeCmd SingleString) ] +options = [ + Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" + , Option ['N'] ["no-commit"] (NoArg NoCommit) "do not stage or commit changes" + ] + {- Finds the type of parameters a command wants, from among the passed - parameter list. -} findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] @@ -75,7 +81,6 @@ parseCmd argv state = do lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." - options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" ] {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} @@ -89,7 +94,7 @@ addCmd file = inBackend file err $ do Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - liftIO $ setup g key link + setup g key link where err = error $ "already annexed " ++ file checkLegal file = do @@ -106,12 +111,16 @@ addCmd file = inBackend file err $ do setup g key link = do let dest = annexLocation g key let reldest = annexLocationRelative g key - createDirectoryIfMissing True (parentDir dest) - renameFile file dest - createSymbolicLink (link ++ reldest) file - Git.run g ["add", file] - Git.run g ["commit", "-m", - ("git-annex annexed " ++ file), file] + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile file dest + liftIO $ createSymbolicLink (link ++ reldest) file + nocommit <- Annex.flagIsSet NoCommit + if (not nocommit) + then do + liftIO $ Git.run g ["add", file] + liftIO $ Git.run g ["commit", "-m", + ("git-annex annexed " ++ file), file] + else return () {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () @@ -192,7 +201,10 @@ describeCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - liftIO $ Git.run g ["add", log] + nocommit <- Annex.flagIsSet NoCommit + if (not nocommit) + then liftIO $ Git.run g ["add", log] + else return () Annex.flagChange NeedCommit True liftIO $ putStrLn "description set" @@ -202,7 +214,10 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - liftIO $ Git.run g ["add", f] + nocommit <- Annex.flagIsSet NoCommit + if (not nocommit) + then liftIO $ Git.run g ["add", f] + else return () Annex.flagChange NeedCommit True inBackend file yes no = do diff --git a/Core.hs b/Core.hs index fcbce4163a..6e48068f96 100644 --- a/Core.hs +++ b/Core.hs @@ -24,8 +24,9 @@ startup flags = do shutdown :: Annex () shutdown = do g <- Annex.gitRepo + nocommit <- Annex.flagIsSet NoCommit needcommit <- Annex.flagIsSet NeedCommit - if (needcommit) + if (needcommit && not nocommit) then liftIO $ Git.run g ["commit", "-q", "-m", "git-annex log update", gitStateDir g] else return () diff --git a/TODO b/TODO index cd94f03bcb..fedcce6dd9 100644 --- a/TODO +++ b/TODO @@ -5,6 +5,9 @@ * how to handle git mv file? +* how to handle git rm file? (should try to drop keys that have no + referring file, if it seems safe..) + * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) From c69e747d383d308d0cf65d88dc1c3be139d056a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 19:57:56 -0400 Subject: [PATCH 0177/8313] refactor --- Commands.hs | 20 +++----------------- Core.hs | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Commands.hs b/Commands.hs index 63ca6b5e4e..f33be5393c 100644 --- a/Commands.hs +++ b/Commands.hs @@ -114,13 +114,7 @@ addCmd file = inBackend file err $ do liftIO $ createDirectoryIfMissing True (parentDir dest) liftIO $ renameFile file dest liftIO $ createSymbolicLink (link ++ reldest) file - nocommit <- Annex.flagIsSet NoCommit - if (not nocommit) - then do - liftIO $ Git.run g ["add", file] - liftIO $ Git.run g ["commit", "-m", - ("git-annex annexed " ++ file), file] - else return () + gitAdd file $ Just $ "git-annex annexed " ++ file {- Inverse of addCmd. -} unannexCmd :: FilePath -> Annex () @@ -201,11 +195,7 @@ describeCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - nocommit <- Annex.flagIsSet NoCommit - if (not nocommit) - then liftIO $ Git.run g ["add", log] - else return () - Annex.flagChange NeedCommit True + gitAdd log Nothing -- all logs are committed at end liftIO $ putStrLn "description set" {- Updates the LocationLog when a key's presence changes. -} @@ -214,11 +204,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - nocommit <- Annex.flagIsSet NoCommit - if (not nocommit) - then liftIO $ Git.run g ["add", f] - else return () - Annex.flagChange NeedCommit True + gitAdd f Nothing -- all logs are committed at end inBackend file yes no = do r <- liftIO $ Backend.lookupFile file diff --git a/Core.hs b/Core.hs index 6e48068f96..5f5cba2957 100644 --- a/Core.hs +++ b/Core.hs @@ -2,6 +2,7 @@ module Core where +import Maybe import System.IO import System.Directory import Control.Monad.State (liftIO) @@ -61,4 +62,22 @@ inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key -{- -} +{- Adds, optionally also commits a file to git. + - + - All changes to the git repository should go through this function. + - + - This is careful to not rely on the index. It may have staged changes, + - so only use operations that avoid committing such changes. + -} +gitAdd :: FilePath -> Maybe String -> Annex () +gitAdd file commitmessage = do + nocommit <- Annex.flagIsSet NoCommit + if (nocommit) + then Annex.flagChange NeedCommit True + else do + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] + if (isJust commitmessage) + then liftIO $ Git.run g ["commit", "-m", + (fromJust commitmessage), file] + else return () From 96347a25a26d01ae4814e9eeb44e7c82a68fb560 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 20:03:41 -0400 Subject: [PATCH 0178/8313] show full usage --- Commands.hs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index f33be5393c..b9f31a56cd 100644 --- a/Commands.hs +++ b/Commands.hs @@ -67,9 +67,9 @@ parseCmd :: [String] -> AnnexState -> IO ([Flag], [Annex ()]) parseCmd argv state = do (flags, params) <- getopt case (length params) of - 0 -> error header + 0 -> error usage _ -> case (lookupCmd (params !! 0)) of - [] -> error header + [] -> error usage [Command _ action want] -> do f <- findWanted want (drop 1 params) (BackendTypes.repo state) @@ -77,10 +77,11 @@ parseCmd argv state = do where getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usageInfo header options)) + (_, _, errs) -> ioError (userError (concat errs ++ usage)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." + usage = usageInfo header options {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} From b02a3b3f5b264ca12fcbf225db3c3ddd341ac51a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:03:25 -0400 Subject: [PATCH 0179/8313] add fix subcommand --- Commands.hs | 37 +++++++++++++++++++++++++------------ Core.hs | 13 +++++++++++++ doc/git-annex.mdwn | 2 ++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/Commands.hs b/Commands.hs index b9f31a56cd..8afe66b91b 100644 --- a/Commands.hs +++ b/Commands.hs @@ -40,6 +40,7 @@ cmds = [ , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) , (Command "describe" describeCmd SingleString) + , (Command "fix" fixCmd FilesInGit) ] options = [ @@ -89,13 +90,12 @@ addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file g <- Annex.gitRepo - link <- liftIO $ calcGitLink file g stored <- Backend.storeFileKey file case (stored) of Nothing -> error $ "no backend could store: " ++ file Just (key, backend) -> do logStatus key ValuePresent - setup g key link + setup g key where err = error $ "already annexed " ++ file checkLegal file = do @@ -103,21 +103,15 @@ addCmd file = inBackend file err $ do if ((isSymbolicLink s) || (not $ isRegularFile s)) then error $ "not a regular file: " ++ file else return () - calcGitLink file g = do - cwd <- getCurrentDirectory - let absfile = case (absNormPath cwd file) of - Just f -> f - Nothing -> error $ "unable to normalize " ++ file - return $ relPathDirToDir (parentDir absfile) (Git.workTree g) - setup g key link = do + setup g key = do let dest = annexLocation g key - let reldest = annexLocationRelative g key liftIO $ createDirectoryIfMissing True (parentDir dest) liftIO $ renameFile file dest - liftIO $ createSymbolicLink (link ++ reldest) file + link <- calcGitLink file key + liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex annexed " ++ file -{- Inverse of addCmd. -} +{- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do Backend.removeKey backend key @@ -181,6 +175,25 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do where err = error $ "not annexed " ++ file +{- Fixes the symlink to an annexed file. -} +fixCmd :: String -> Annex () +fixCmd file = notinBackend file err $ \(key, backend) -> do + link <- calcGitLink file key + checkLegal file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + gitAdd file $ Just $ "git-annex fix " ++ file + where + checkLegal file = do + s <- liftIO $ getSymbolicLinkStatus file + force <- Annex.flagIsSet Force + if (not (isSymbolicLink s) && not force) + then error $ "not a symbolic link : " ++ file ++ + " (use --force to override this sanity check)" + else return () + err = error $ "not annexed " ++ file + {- Pushes all files to a remote repository. -} pushCmd :: String -> Annex () pushCmd reponame = do error "not implemented" -- TODO diff --git a/Core.hs b/Core.hs index 5f5cba2957..021595f8b6 100644 --- a/Core.hs +++ b/Core.hs @@ -6,12 +6,14 @@ import Maybe import System.IO import System.Directory import Control.Monad.State (liftIO) +import System.Path import Types import Locations import UUID import qualified GitRepo as Git import qualified Annex +import Utility {- Sets up a git repo for git-annex. -} startup :: [Flag] -> Annex () @@ -81,3 +83,14 @@ gitAdd file commitmessage = do then liftIO $ Git.run g ["commit", "-m", (fromJust commitmessage), file] else return () + +{- Calculates the relative path to use to link a file to a key. -} +calcGitLink :: FilePath -> Key -> Annex FilePath +calcGitLink file key = do + g <- Annex.gitRepo + cwd <- liftIO $ getCurrentDirectory + let absfile = case (absNormPath cwd file) of + Just f -> f + Nothing -> error $ "unable to normalize " ++ file + return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ + annexLocationRelative g key diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e552dc770a..e65ad5b020 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -47,6 +47,8 @@ Enough broad picture, here's how it actually looks: repository. * `git annex pull $repository` pulls *all* annexed files from the specified repository. +* `git annex file $file` adjusts the symlink for the file to point to its + content again. Use this if you've moved the file around. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you accidentially added a file. From 0c0ae028386aaf17aed1771eee6731c62b72e839 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:15:23 -0400 Subject: [PATCH 0180/8313] add fix subcommand --- Commands.hs | 20 +++++++++++++++----- TODO | 6 ++++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8afe66b91b..9a3f925244 100644 --- a/Commands.hs +++ b/Commands.hs @@ -24,7 +24,8 @@ import Core import qualified Remotes import qualified BackendTypes -data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString +data CmdWants = FilesInGit | FilesNotInGit | FilesInOrNotInGit | + RepoName | SingleString data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -40,7 +41,7 @@ cmds = [ , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) , (Command "describe" describeCmd SingleString) - , (Command "fix" fixCmd FilesInGit) + , (Command "fix" fixCmd FilesInOrNotInGit) ] options = [ @@ -57,6 +58,10 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files +findWanted FilesInOrNotInGit params repo = do + a <- findWanted FilesInGit params repo + b <- findWanted FilesNotInGit params repo + return $ union a b findWanted SingleString params _ = do return $ [unwords params] findWanted RepoName params _ = do @@ -178,20 +183,25 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () fixCmd file = notinBackend file err $ \(key, backend) -> do + liftIO $ putStrLn $ "fix " ++ file link <- calcGitLink file key - checkLegal file + checkLegal file link liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex fix " ++ file where - checkLegal file = do + checkLegal file link = do s <- liftIO $ getSymbolicLinkStatus file force <- Annex.flagIsSet Force if (not (isSymbolicLink s) && not force) then error $ "not a symbolic link : " ++ file ++ " (use --force to override this sanity check)" - else return () + else do + l <- liftIO $ readSymbolicLink file + if (link == l) + then error $ "symbolic link already ok for: " ++ file + else return () err = error $ "not annexed " ++ file {- Pushes all files to a remote repository. -} diff --git a/TODO b/TODO index fedcce6dd9..038ed0d4a4 100644 --- a/TODO +++ b/TODO @@ -3,15 +3,17 @@ * --push/--pull/--want -* how to handle git mv file? +* how to handle git mv file? -> git annex fix -> run automatically? * how to handle git rm file? (should try to drop keys that have no referring file, if it seems safe..) +* add a git annex fsck that finds keys that have no referring file + * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) -* Copy files atomically, don't leaf a partial key on interrupt. +* Copy files atomically, don't leave a partial key on interrupt. (Fix for URL download too..) * Find a way to copy a file with a progress bar, while still preserving From 38825f48645c0220b6b2e0368c2ebdbb625f6703 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:18:21 -0400 Subject: [PATCH 0181/8313] remove useless checks the file will always be a symlink at this point --- Commands.hs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Commands.hs b/Commands.hs index 9a3f925244..4346a35e22 100644 --- a/Commands.hs +++ b/Commands.hs @@ -192,16 +192,10 @@ fixCmd file = notinBackend file err $ \(key, backend) -> do gitAdd file $ Just $ "git-annex fix " ++ file where checkLegal file link = do - s <- liftIO $ getSymbolicLinkStatus file - force <- Annex.flagIsSet Force - if (not (isSymbolicLink s) && not force) - then error $ "not a symbolic link : " ++ file ++ - " (use --force to override this sanity check)" - else do - l <- liftIO $ readSymbolicLink file - if (link == l) - then error $ "symbolic link already ok for: " ++ file - else return () + l <- liftIO $ readSymbolicLink file + if (link == l) + then error $ "symbolic link already ok for: " ++ file + else return () err = error $ "not annexed " ++ file {- Pushes all files to a remote repository. -} From 19daf3fca40d99dd305a75e10dcaa8fbc734598b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 21:50:33 -0400 Subject: [PATCH 0182/8313] oops, should commit descriptions! --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 4346a35e22..af6b5aad78 100644 --- a/Commands.hs +++ b/Commands.hs @@ -213,7 +213,7 @@ describeCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - gitAdd log Nothing -- all logs are committed at end + gitAdd log $ Just $ "description for UUID " ++ (show u) liftIO $ putStrLn "description set" {- Updates the LocationLog when a key's presence changes. -} From da453ba70149444672b8cd64e36fe34604edce73 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:36:35 -0400 Subject: [PATCH 0183/8313] bugfix: don't add files under .git-annex That could happen if git annex add -N were used repeatedly.. --- Commands.hs | 5 ++++- Core.hs | 2 +- Locations.hs | 4 ++-- doc/git-annex.mdwn | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index af6b5aad78..1364a1b35e 100644 --- a/Commands.hs +++ b/Commands.hs @@ -54,7 +54,10 @@ options = [ findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] findWanted FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params - return $ foldl (++) [] files + return $ filter notstate $ foldl (++) [] files + where + -- never include files in the state directory + notstate f = f /= take (length stateLoc) f findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files diff --git a/Core.hs b/Core.hs index 021595f8b6..70e6e66804 100644 --- a/Core.hs +++ b/Core.hs @@ -51,7 +51,7 @@ gitAttributes repo = do commit else return () where - attrLine = stateLoc ++ "/*.log merge=union" + attrLine = stateLoc ++ "*.log merge=union" attributes = Git.attributes repo commit = do Git.run repo ["add", attributes] diff --git a/Locations.hs b/Locations.hs index 4978500624..76516224cf 100644 --- a/Locations.hs +++ b/Locations.hs @@ -18,9 +18,9 @@ import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository's working tree. -} -stateLoc = ".git-annex" +stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath -gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc ++ "/" +gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc {- An annexed file's content is stored in - /path/to/repo/.git/annex/, where is of the form diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e65ad5b020..50fd28e82a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -51,7 +51,8 @@ Enough broad picture, here's how it actually looks: content again. Use this if you've moved the file around. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` if you're just done with a file; only use `unannex` if you - accidentially added a file. + accidentially added a file. (You can also run this on all your annexed + files come the Singularity. ;-) * `git annex describe "some description"` allows associating some description (such as "USB archive drive 1") with a repository. This can help with finding it later, see "Location Tracking" below. From 96451ac392d42973f508da08d7c1197c83c659a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:43:38 -0400 Subject: [PATCH 0184/8313] nocommit does not make sense in unannex mode --- Commands.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 1364a1b35e..be0a8e37ff 100644 --- a/Commands.hs +++ b/Commands.hs @@ -122,11 +122,15 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - Backend.removeKey backend key - logStatus key ValueMissing - g <- Annex.gitRepo - let src = annexLocation g key - liftIO $ moveout g src + nocommit <- Annex.flagIsSet NoCommit + if (nocommit) + then error "--nocommit cannot be used in unannex mode" + else do + Backend.removeKey backend key + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g key + liftIO $ moveout g src where err = error $ "not annexed " ++ file moveout g src = do From c0b16e0a7306bacce96564d80911bbdb5a246847 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:49:09 -0400 Subject: [PATCH 0185/8313] actually, unannex w/o commit can work just have to git rm --- Commands.hs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index be0a8e37ff..67b8fe8ad3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -122,26 +122,25 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - nocommit <- Annex.flagIsSet NoCommit - if (nocommit) - then error "--nocommit cannot be used in unannex mode" - else do - Backend.removeKey backend key - logStatus key ValueMissing - g <- Annex.gitRepo - let src = annexLocation g key - liftIO $ moveout g src + Backend.removeKey backend key + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g key + moveout g src where err = error $ "not annexed " ++ file moveout g src = do - removeFile file - Git.run g ["rm", file] - Git.run g ["commit", "-m", - ("git-annex unannexed " ++ file), file] + nocommit <- Annex.flagIsSet NoCommit + liftIO removeFile file + liftIO Git.run g ["rm", file] + if (not nocommit) + then liftIO Git.run g ["commit", "-m", + ("git-annex unannexed " ++ file), file] + else return () -- git rm deletes empty directories; -- put them back - createDirectoryIfMissing True (parentDir file) - renameFile src file + liftIO createDirectoryIfMissing True (parentDir file) + liftIO renameFile src file return () {- Gets an annexed file from one of the backends. -} From 20b447782a974408594692c7aa6ce8bc26f87858 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 22:59:19 -0400 Subject: [PATCH 0186/8313] typo --- Commands.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Commands.hs b/Commands.hs index 67b8fe8ad3..dfb3eef436 100644 --- a/Commands.hs +++ b/Commands.hs @@ -131,16 +131,16 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do err = error $ "not annexed " ++ file moveout g src = do nocommit <- Annex.flagIsSet NoCommit - liftIO removeFile file - liftIO Git.run g ["rm", file] + liftIO $ removeFile file + liftIO $ Git.run g ["rm", file] if (not nocommit) - then liftIO Git.run g ["commit", "-m", + then liftIO $ Git.run g ["commit", "-m", ("git-annex unannexed " ++ file), file] else return () -- git rm deletes empty directories; -- put them back - liftIO createDirectoryIfMissing True (parentDir file) - liftIO renameFile src file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ renameFile src file return () {- Gets an annexed file from one of the backends. -} From aaee8e231f111b9b4a2ead95eaaeb3d635cc1699 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Oct 2010 23:36:45 -0400 Subject: [PATCH 0187/8313] bugfix --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index dfb3eef436..258490996d 100644 --- a/Commands.hs +++ b/Commands.hs @@ -57,7 +57,7 @@ findWanted FilesNotInGit params repo = do return $ filter notstate $ foldl (++) [] files where -- never include files in the state directory - notstate f = f /= take (length stateLoc) f + notstate f = stateLoc /= take (length stateLoc) f findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files From c57b1a697c5de4e20ef10c2c4a39a77c20fde85b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 00:10:04 -0400 Subject: [PATCH 0188/8313] add visible size and time to worm keys --- Backend/WORM.hs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 420f336e90..7e86d4d243 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -23,7 +23,10 @@ backend = Backend.File.backend { -- allows multiple files with the same names to have different keys, -- while also allowing a file to be moved around while retaining the -- same key. --- +-- +-- The file size and modification time are also included in the key, +-- unhashed. This could be used as a sanity check. +-- -- The basename of the filename is also included in the key, so it's clear -- what the original filename was when a user sees the value. keyValue :: FilePath -> Annex (Maybe Key) @@ -31,9 +34,10 @@ keyValue file = do stat <- liftIO $ getFileStatus file return $ Just $ Key ((name backend), key stat) where - key stat = (checksum $ uniqueid stat) ++ sep ++ base + key stat = (checksum $ uniqueid stat) ++ sep ++ + uniqueid stat ++ sep ++ base checksum s = show $ sha1 $ B.pack s - uniqueid stat = (show $ fileSize stat) ++ sep ++ - (show $ modificationTime stat) + uniqueid stat = (show $ modificationTime stat) ++ sep ++ + (show $ fileSize stat) base = takeFileName file sep = ":" From a0b040524a595c16ddb2dbead205ca8ccb6890aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 00:33:05 -0400 Subject: [PATCH 0189/8313] remove checksum from WORM with size and mtime in the key, it's redundant --- Backend/WORM.hs | 18 +++++------------- Commands.hs | 9 ++++----- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 7e86d4d243..463b0ac8e2 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -6,7 +6,6 @@ module Backend.WORM (backend) where import Control.Monad.State import System.FilePath import System.Posix.Files -import Data.Digest.Pure.SHA -- slow, but we only checksum filenames import qualified Data.ByteString.Lazy.Char8 as B import qualified Backend.File @@ -18,25 +17,18 @@ backend = Backend.File.backend { getKey = keyValue } --- A SHA1 of the basename of the filename, plus the file size and --- modification time, is used as the unique part of the key. That --- allows multiple files with the same names to have different keys, +-- The key is formed from the file size, modification time, and the +-- basename of the filename. +-- +-- That allows multiple files with the same names to have different keys, -- while also allowing a file to be moved around while retaining the -- same key. --- --- The file size and modification time are also included in the key, --- unhashed. This could be used as a sanity check. --- --- The basename of the filename is also included in the key, so it's clear --- what the original filename was when a user sees the value. keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do stat <- liftIO $ getFileStatus file return $ Just $ Key ((name backend), key stat) where - key stat = (checksum $ uniqueid stat) ++ sep ++ - uniqueid stat ++ sep ++ base - checksum s = show $ sha1 $ B.pack s + key stat = uniqueid stat ++ sep ++ base uniqueid stat = (show $ modificationTime stat) ++ sep ++ (show $ fileSize stat) base = takeFileName file diff --git a/Commands.hs b/Commands.hs index 258490996d..5931bf0a9a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -54,10 +54,7 @@ options = [ findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] findWanted FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params - return $ filter notstate $ foldl (++) [] files - where - -- never include files in the state directory - notstate f = stateLoc /= take (length stateLoc) f + return $ foldl (++) [] files findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files @@ -82,8 +79,10 @@ parseCmd argv state = do [Command _ action want] -> do f <- findWanted want (drop 1 params) (BackendTypes.repo state) - return (flags, map action f) + return (flags, map action $ filter notstate f) where + -- never include files from the state directory + notstate f = stateLoc /= take (length stateLoc) f getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) From 6bfa534aa4d7552c4ccfdb9523b55da19fac8883 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 10:47:46 -0400 Subject: [PATCH 0190/8313] git annex drop -- do not try to drop if key is not in backend --- Commands.hs | 23 ++++++++++++++--------- TODO | 6 +++++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Commands.hs b/Commands.hs index 5931bf0a9a..b446dbfac6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -165,13 +165,20 @@ getCmd file = notinBackend file err $ \(key, backend) -> do - if it's safe to do so. -} dropCmd :: FilePath -> Annex () dropCmd file = notinBackend file err $ \(key, backend) -> do - force <- Annex.flagIsSet Force - if (not force) - then requireEnoughCopies key - else return () - success <- Backend.removeKey backend key - if (success) - then do + inbackend <- Backend.hasKey key + if (not inbackend) + then return () -- no-op + else do + force <- Annex.flagIsSet Force + if (not force) + then requireEnoughCopies key + else return () + success <- Backend.removeKey backend key + if (success) + then cleanup key + else error $ "backend refused to drop " ++ file + where + cleanup key = do logStatus key ValueMissing inannex <- inAnnex key if (inannex) @@ -181,8 +188,6 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do liftIO $ removeFile loc return () else return () - else error $ "backend refused to drop " ++ file - where err = error $ "not annexed " ++ file {- Fixes the symlink to an annexed file. -} diff --git a/TODO b/TODO index 038ed0d4a4..807df32d84 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,11 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) -* --push/--pull/--want +* bug: git annex add file is silent if file was a symlink and got replaced + with a file. The you then git command -a, you'll check in the fil contents.. + +* --push/--pull should take a reponame and files, and push those files + to that repo; dropping them from the current repo * how to handle git mv file? -> git annex fix -> run automatically? From b471822cfe4476995f539c6e7e7da7f7bf2b5e02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 11:47:36 -0400 Subject: [PATCH 0191/8313] move supportedBackends list into annex monad This was necessary so the File backend could import Backend w/o a cycle. Moved code that checks whether enough backends have a file into File backend. --- Annex.hs | 26 +++++++++++++++------ Backend.hs | 35 ++++++++++++++++++++--------- Backend/File.hs | 60 ++++++++++++++++++++++++++++++++++++++++++++----- BackendList.hs | 25 ++------------------- BackendTypes.hs | 1 + Commands.hs | 50 +---------------------------------------- git-annex.hs | 3 ++- 7 files changed, 105 insertions(+), 95 deletions(-) diff --git a/Annex.hs b/Annex.hs index 68c0cb88e2..e76ccd1dcb 100644 --- a/Annex.hs +++ b/Annex.hs @@ -7,6 +7,8 @@ module Annex ( gitRepoChange, backends, backendsChange, + supportedBackends, + supportedBackendsChange, flagIsSet, flagChange, Flag(..) @@ -20,20 +22,21 @@ import qualified BackendTypes as Backend {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> IO AnnexState -new g = do +new :: Git.Repo -> [Backend] -> IO AnnexState +new gitrepo allbackends = do let s = Backend.AnnexState { - Backend.repo = g, + Backend.repo = gitrepo, Backend.backends = [], + Backend.supportedBackends = allbackends, Backend.flags = [] } - (_,s') <- Annex.run s (prep g) + (_,s') <- Annex.run s (prep gitrepo) return s' where - prep g = do + prep gitrepo = do -- read git config and update state - g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' + gitrepo' <- liftIO $ Git.configRead gitrepo + Annex.gitRepoChange gitrepo' -- performs an action in the Annex monad run state action = runStateT (action) state @@ -57,6 +60,15 @@ backendsChange b = do state <- get put state { Backend.backends = b } return () +supportedBackends :: Annex [Backend] +supportedBackends = do + state <- get + return (Backend.supportedBackends state) +supportedBackendsChange :: [Backend] -> Annex () +supportedBackendsChange b = do + state <- get + put state { Backend.supportedBackends = b } + return () flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get diff --git a/Backend.hs b/Backend.hs index 874191924d..dfaa559702 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,14 +28,12 @@ import System.FilePath import Data.String.Utils import System.Posix.Files -import BackendList import Locations import qualified GitRepo as Git import qualified Annex import Utility import Types import qualified BackendTypes as B -import BackendList {- List of backends in the order to try them when storing a new key. -} backendList :: Annex [Backend] @@ -44,10 +42,24 @@ backendList = do if (0 < length l) then return l else do + all <- Annex.supportedBackends g <- Annex.gitRepo - let l = parseBackendList $ Git.configGet g "annex.backends" "" + let l = parseBackendList all $ Git.configGet g "annex.backends" "" Annex.backendsChange l return l + where + parseBackendList all s = + if (length s == 0) + then all + else map (lookupBackendName all) $ words s + +{- Looks up a backend in the list of supportedBackends -} +lookupBackendName :: [Backend] -> String -> Backend +lookupBackendName all s = + if ((length matches) /= 1) + then error $ "unknown backend " ++ s + else matches !! 0 + where matches = filter (\b -> s == B.name b) all {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) @@ -81,21 +93,24 @@ removeKey backend key = (B.removeKey backend) key {- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool -hasKey key = (B.hasKey (lookupBackendName $ backendName key)) key +hasKey key = do + all <- Annex.supportedBackends + (B.hasKey (lookupBackendName all $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} -lookupFile :: FilePath -> IO (Maybe (Key, Backend)) +lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) lookupFile file = do - result <- try (lookup)::IO (Either SomeException (Maybe (Key, Backend))) + all <- Annex.supportedBackends + result <- liftIO $ (try (lookup all)::IO (Either SomeException (Maybe (Key, Backend)))) case (result) of Left err -> return Nothing Right succ -> return succ where - lookup = do + lookup all = do l <- readSymbolicLink file - return $ Just $ pair $ takeFileName l - pair file = (k, b) + return $ Just $ pair all $ takeFileName l + pair all file = (k, b) where k = fileKey file - b = lookupBackendName $ backendName k + b = lookupBackendName all $ backendName k diff --git a/Backend/File.hs b/Backend/File.hs index f5237f7210..591ff3db41 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -25,13 +25,14 @@ import Utility import Core import qualified Annex import UUID +import qualified Backend backend = Backend { name = mustProvide, getKey = mustProvide, storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, - removeKey = dummyRemove, + removeKey = checkRemoveKey, hasKey = checkKeyFile } @@ -41,10 +42,6 @@ mustProvide = error "must provide this field" dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True -{- Allow keys to be removed. -} -dummyRemove :: Key -> Annex Bool -dummyRemove url = return True - {- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool checkKeyFile k = inAnnex k @@ -102,3 +99,56 @@ copyFromRemote r key file = do else error "cp failed" getremote = error "get via network not yet implemented!" location = annexLocation r key + +{- Checks remotes to verify that enough copies of a key exist to allow + - for a key to be safely removed (with no data loss), and fails with an + - error if not. -} +checkRemoveKey :: Key -> Annex (Bool) +checkRemoveKey key = do + force <- Annex.flagIsSet Force + if (force) + then return True + else do + g <- Annex.gitRepo + let numcopies = read $ Git.configGet g config "1" + remotes <- Remotes.withKey key + if (numcopies > length remotes) + then retNotEnoughCopiesKnown remotes numcopies + else findcopies numcopies remotes [] + where + failMsg w = do + liftIO $ hPutStrLn stderr $ "git-annex: " ++ w + return False -- failure, not enough copies found + findcopies 0 _ _ = return True -- success, enough copies found + findcopies _ [] bad = notEnoughCopiesSeen bad + findcopies n (r:rs) bad = do + all <- Annex.supportedBackends + result <- liftIO $ ((try $ remoteHasKey r all)::IO (Either SomeException Bool)) + case (result) of + Right True -> findcopies (n-1) rs bad + Right False -> findcopies n rs bad + Left _ -> findcopies n rs (r:bad) + remoteHasKey r all = do + -- To check if a remote has a key, construct a new + -- Annex monad and query its backend. + a <- Annex.new r all + (result, _) <- Annex.run a (Backend.hasKey key) + return result + notEnoughCopiesSeen bad = failMsg $ + "I failed to find enough other copies of: " ++ + (keyFile key) ++ + (if (0 /= length bad) then listbad bad else "") + ++ unsafe + listbad bad = "\nI was unable to access these remotes: " ++ + (Remotes.list bad) + retNotEnoughCopiesKnown remotes numcopies = failMsg $ + "I only know about " ++ (show $ length remotes) ++ + " out of " ++ (show numcopies) ++ + " necessary copies of: " ++ (keyFile key) ++ + unsafe + unsafe = "\n" ++ + " -- According to the " ++ config ++ + " setting, it is not safe to remove it!\n" ++ + " (Use --force to override.)" + + config = "annex.numcopies" diff --git a/BackendList.hs b/BackendList.hs index 42e237204f..920f8fc0a6 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -1,11 +1,7 @@ {- git-annex backend list - -} -module BackendList ( - supportedBackends, - parseBackendList, - lookupBackendName -) where +module BackendList (allBackends) where import BackendTypes @@ -13,25 +9,8 @@ import BackendTypes import qualified Backend.WORM import qualified Backend.SHA1 import qualified Backend.URL -supportedBackends = +allBackends = [ Backend.WORM.backend , Backend.SHA1.backend , Backend.URL.backend ] - -{- Parses a string with a list of backend names into - - a list of Backend objects. If the list is empty, - - defaults to supportedBackends. -} -parseBackendList :: String -> [Backend] -parseBackendList s = - if (length s == 0) - then supportedBackends - else map (lookupBackendName) $ words s - -{- Looks up a supported backend by name. -} -lookupBackendName :: String -> Backend -lookupBackendName s = - if ((length matches) /= 1) - then error $ "unknown backend " ++ s - else matches !! 0 - where matches = filter (\b -> s == name b) supportedBackends diff --git a/BackendTypes.hs b/BackendTypes.hs index 06ecfb8fe5..e372099b2d 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -19,6 +19,7 @@ data Flag = Force | NoCommit | NeedCommit data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend], + supportedBackends :: [Backend], flags :: [Flag] } deriving (Show) diff --git a/Commands.hs b/Commands.hs index b446dbfac6..62376e4dde 100644 --- a/Commands.hs +++ b/Commands.hs @@ -16,7 +16,6 @@ import qualified Annex import Utility import Locations import qualified Backend -import BackendList import UUID import LocationLog import Types @@ -169,10 +168,6 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (not inbackend) then return () -- no-op else do - force <- Annex.flagIsSet Force - if (not force) - then requireEnoughCopies key - else return () success <- Backend.removeKey backend key if (success) then cleanup key @@ -235,51 +230,8 @@ logStatus key status = do gitAdd f Nothing -- all logs are committed at end inBackend file yes no = do - r <- liftIO $ Backend.lookupFile file + r <- Backend.lookupFile file case (r) of Just v -> yes v Nothing -> no notinBackend file yes no = inBackend file no yes - -{- Checks remotes to verify that enough copies of a key exist to allow - - for a key to be safely removed (with no data loss), and fails with an - - error if not. -} -requireEnoughCopies :: Key -> Annex () -requireEnoughCopies key = do - g <- Annex.gitRepo - let numcopies = read $ Git.configGet g config "1" - remotes <- Remotes.withKey key - if (numcopies > length remotes) - then error $ "I only know about " ++ (show $ length remotes) ++ - " out of " ++ (show numcopies) ++ - " necessary copies of: " ++ (keyFile key) ++ - unsafe - else findcopies numcopies remotes [] - where - findcopies 0 _ _ = return () -- success, enough copies found - findcopies _ [] bad = die bad - findcopies n (r:rs) bad = do - result <- liftIO $ try $ haskey r - case (result) of - Right True -> findcopies (n-1) rs bad - Right False -> findcopies n rs bad - Left _ -> findcopies n rs (r:bad) - haskey r = do - -- To check if a remote has a key, construct a new - -- Annex monad and query its backend. - a <- Annex.new r - (result, _) <- Annex.run a (Backend.hasKey key) - return result - die bad = - error $ "I failed to find enough other copies of: " ++ - (keyFile key) ++ - (if (0 /= length bad) then listbad bad else "") - ++ unsafe - listbad bad = "\nI was unable to access these remotes: " ++ - (Remotes.list bad) - unsafe = "\n" ++ - " -- According to the " ++ config ++ - " setting, it is not safe to remove it!\n" ++ - " (Use --force to override.)" - - config = "annex.numcopies" diff --git a/git-annex.hs b/git-annex.hs index f4f0cfcdfd..947868f238 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -9,11 +9,12 @@ import Types import Core import Commands import qualified GitRepo as Git +import BackendList main = do args <- getArgs gitrepo <- Git.repoFromCwd - state <- Annex.new gitrepo + state <- Annex.new gitrepo allBackends (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] From cb1a0a387f93e882ced50709f938bd0a28cd14be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 11:51:53 -0400 Subject: [PATCH 0192/8313] update --- doc/git-annex.mdwn | 63 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 50fd28e82a..4d2872aa3d 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -61,35 +61,6 @@ Oh yeah, "$file" in the above can be any number of files, or directories, same as you'd pass to "git add" or "git rm". So "git annex add ." or "git annex get dir/" work fine. -## copies - -git-annex can be configured to try to keep N copies of a file's content -available across all repositories. By default, N is 1; it is configured by -annex.numcopies. - -`git annex drop` attempts to check with other git remotes, to check that N -copies of the file exist. If enough repositories cannot be verified to have -it, it will retain the file content to avoid data loss. - -For example, consider three repositories: Server, Laptop, and USB. Both Server -and USB have a copy of a file, and N=1. If on Laptop, you `git annex get -$file`, this will transfer it from either Server or USB (depending on which -is available), and there are now 3 copies of the file. - -Suppose you want to free up space on Laptop again, and you `git annex drop` the file -there. If USB is connected, or Server can be contacted, git-annex can check -that it still has a copy of the file, and the content is removed from -Laptop. But if USB is currently disconnected, and Server also cannot be -contacted, it can't verify that it is safe to drop the file, and will -refuse to do so. - -With N=2, in order to drop the file content from Laptop, it would need access -to both USB and Server. - -Note that different repositories can be configured with different values of -N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1. - ## key-value storage git-annex uses a key-value abstraction layer to allow file contents to be @@ -116,6 +87,37 @@ to store different files' contents in a given repository. can make it slow for large files. * `URL` -- This backend downloads the file's content from an external URL. +## copies + +The WORM and SHA1 key-value backends store data inside your git repository. +It's important that data not get lost by an ill-though `git annex drop` +command. So, then using those backends, git-annex can be configured to try +to keep N copies of a file's content available across all repositories. By +default, N is 1; it is configured by annex.numcopies. + +`git annex drop` attempts to check with other git remotes, to check that N +copies of the file exist. If enough repositories cannot be verified to have +it, it will retain the file content to avoid data loss. + +For example, consider three repositories: Server, Laptop, and USB. Both Server +and USB have a copy of a file, and N=1. If on Laptop, you `git annex get +$file`, this will transfer it from either Server or USB (depending on which +is available), and there are now 3 copies of the file. + +Suppose you want to free up space on Laptop again, and you `git annex drop` the file +there. If USB is connected, or Server can be contacted, git-annex can check +that it still has a copy of the file, and the content is removed from +Laptop. But if USB is currently disconnected, and Server also cannot be +contacted, it can't verify that it is safe to drop the file, and will +refuse to do so. + +With N=2, in order to drop the file content from Laptop, it would need access +to both USB and Server. + +Note that different repositories can be configured with different values of +N. So just because Laptop has N=2, this does not prevent the number of +copies falling to 1, when USB and Server have N=1. + ## location tracking git-annex keeps track of in which repositories it last saw a file's content. @@ -149,7 +151,8 @@ finding them: ## configuration * `annex.uuid` -- a unique UUID for this repository -* `annex.numcopies` -- number of copies of files to keep (default: 1) +* `annex.numcopies` -- number of copies of files to keep across all + repositories (default: 1) * `annex.backends` -- space-separated list of names of the key-value backends to use. The first listed is used to store new files. (default: "WORM SHA1 URL") From ae4d20d157eb288046dddf4555bfc9f2660ed675 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 11:57:39 -0400 Subject: [PATCH 0193/8313] bugfix --- Core.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core.hs b/Core.hs index 70e6e66804..1832b35138 100644 --- a/Core.hs +++ b/Core.hs @@ -75,14 +75,14 @@ gitAdd :: FilePath -> Maybe String -> Annex () gitAdd file commitmessage = do nocommit <- Annex.flagIsSet NoCommit if (nocommit) - then Annex.flagChange NeedCommit True + then return () else do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] if (isJust commitmessage) then liftIO $ Git.run g ["commit", "-m", (fromJust commitmessage), file] - else return () + else Annex.flagChange NeedCommit True {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath From 76ba2d003072da67bd9b0fb5b84bf7a268a956ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 12:08:59 -0400 Subject: [PATCH 0194/8313] reorg --- Commands.hs | 9 +-------- Core.hs | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index 62376e4dde..2ff8d0d7b3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -221,14 +221,7 @@ describeCmd description = do gitAdd log $ Just $ "description for UUID " ++ (show u) liftIO $ putStrLn "description set" -{- Updates the LocationLog when a key's presence changes. -} -logStatus :: Key -> LogStatus -> Annex () -logStatus key status = do - g <- Annex.gitRepo - u <- getUUID g - f <- liftIO $ logChange g key u status - gitAdd f Nothing -- all logs are committed at end - +-- helpers inBackend file yes no = do r <- Backend.lookupFile file case (r) of diff --git a/Core.hs b/Core.hs index 1832b35138..70ec2eca01 100644 --- a/Core.hs +++ b/Core.hs @@ -10,6 +10,7 @@ import System.Path import Types import Locations +import LocationLog import UUID import qualified GitRepo as Git import qualified Annex @@ -94,3 +95,12 @@ calcGitLink file key = do Nothing -> error $ "unable to normalize " ++ file return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ annexLocationRelative g key + +{- Updates the LocationLog when a key's presence changes. -} +logStatus :: Key -> LogStatus -> Annex () +logStatus key status = do + g <- Annex.gitRepo + u <- getUUID g + f <- liftIO $ logChange g key u status + gitAdd f Nothing -- all logs are committed at end + From 98676928c8dec5183951006450cc26cf5fb6a985 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 12:09:52 -0400 Subject: [PATCH 0195/8313] prune --- Annex.hs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Annex.hs b/Annex.hs index e76ccd1dcb..5fd500d949 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,7 +8,6 @@ module Annex ( backends, backendsChange, supportedBackends, - supportedBackendsChange, flagIsSet, flagChange, Flag(..) @@ -64,11 +63,6 @@ supportedBackends :: Annex [Backend] supportedBackends = do state <- get return (Backend.supportedBackends state) -supportedBackendsChange :: [Backend] -> Annex () -supportedBackendsChange b = do - state <- get - put state { Backend.supportedBackends = b } - return () flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get From 8f6e5da18f80ea5dfc124afed0ff2671a3909d56 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 12:12:23 -0400 Subject: [PATCH 0196/8313] verbosity --- Commands.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Commands.hs b/Commands.hs index 2ff8d0d7b3..e9b5ddcbdf 100644 --- a/Commands.hs +++ b/Commands.hs @@ -95,6 +95,7 @@ parseCmd argv state = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file + liftIO $ putStrLn $ "add " ++ file g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of @@ -120,6 +121,7 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do + liftIO $ putStrLn $ "unannex " ++ file Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo @@ -168,6 +170,7 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (not inbackend) then return () -- no-op else do + liftIO $ putStrLn $ "drop " ++ file success <- Backend.removeKey backend key if (success) then cleanup key From 6d4fc0ca7eb220298e42d368ead57622e80929a3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 13:13:49 -0400 Subject: [PATCH 0197/8313] command output cleanup --- Backend/File.hs | 59 +++++++++++++++++++++++++------------------------ Commands.hs | 33 ++++++++++++++++----------- Core.hs | 26 ++++++++++++++++++++-- UUID.hs | 2 +- 4 files changed, 75 insertions(+), 45 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 591ff3db41..f7796532be 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -53,12 +53,13 @@ copyKeyFile key file = do remotes <- Remotes.withKey key if (0 == length remotes) then cantfind - else return () - trycopy remotes remotes + else trycopy remotes remotes where - trycopy full [] = error $ "unable to get file with key: " ++ (keyFile key) ++ "\n" ++ - "To get that file, need access to one of these remotes: " ++ - (Remotes.list full) + trycopy full [] = do + showNote $ + "need access to one of these remotes: " ++ + (Remotes.list full) + return False trycopy full (r:rs) = do -- annexLocation needs the git config to have been -- read for a remote, so do that now, @@ -67,6 +68,7 @@ copyKeyFile key file = do case (result) of Nothing -> trycopy full rs Just r' -> do + showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) case (result) of Left err -> do @@ -77,17 +79,15 @@ copyKeyFile key file = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key ppuuids <- prettyPrintUUIDs uuids - error $ "no available git remotes have file with key: " ++ - (keyFile key) ++ - if (0 < length uuids) - then "\nIt has been seen before in these repositories:\n" ++ ppuuids - else "" + showNote $ "No available git remotes have the file." + if (0 < length uuids) + then showLongNote $ "It has been seen before in these repositories:\n" ++ ppuuids + else return () + return False {- Tries to copy a file from a remote, exception on error. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () copyFromRemote r key file = do - putStrLn $ "copy from " ++ (Git.repoDescribe r ) ++ " " ++ file - if (Git.repoIsLocal r) then getlocal else getremote @@ -116,9 +116,6 @@ checkRemoveKey key = do then retNotEnoughCopiesKnown remotes numcopies else findcopies numcopies remotes [] where - failMsg w = do - liftIO $ hPutStrLn stderr $ "git-annex: " ++ w - return False -- failure, not enough copies found findcopies 0 _ _ = return True -- success, enough copies found findcopies _ [] bad = notEnoughCopiesSeen bad findcopies n (r:rs) bad = do @@ -134,21 +131,25 @@ checkRemoveKey key = do a <- Annex.new r all (result, _) <- Annex.run a (Backend.hasKey key) return result - notEnoughCopiesSeen bad = failMsg $ - "I failed to find enough other copies of: " ++ - (keyFile key) ++ - (if (0 /= length bad) then listbad bad else "") - ++ unsafe - listbad bad = "\nI was unable to access these remotes: " ++ - (Remotes.list bad) - retNotEnoughCopiesKnown remotes numcopies = failMsg $ + notEnoughCopiesSeen bad = do + showNote "failed to find enough other copies of the file" + if (0 /= length bad) then listbad bad else return () + unsafe + return False + listbad bad = + showLongNote $ + "I was unable to access these remotes: " ++ + (Remotes.list bad) + retNotEnoughCopiesKnown remotes numcopies = do + showNote $ "I only know about " ++ (show $ length remotes) ++ " out of " ++ (show numcopies) ++ - " necessary copies of: " ++ (keyFile key) ++ - unsafe - unsafe = "\n" ++ - " -- According to the " ++ config ++ - " setting, it is not safe to remove it!\n" ++ - " (Use --force to override.)" + " necessary copies of the file" + unsafe + return False + unsafe = do + showLongNote $ "According to the " ++ config ++ + " setting, it is not safe to remove it!" + showLongNote "(Use --force to override.)" config = "annex.numcopies" diff --git a/Commands.hs b/Commands.hs index e9b5ddcbdf..8591dbf6ac 100644 --- a/Commands.hs +++ b/Commands.hs @@ -95,11 +95,11 @@ parseCmd argv state = do addCmd :: FilePath -> Annex () addCmd file = inBackend file err $ do liftIO $ checkLegal file - liftIO $ putStrLn $ "add " ++ file + showStart "add" file g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of - Nothing -> error $ "no backend could store: " ++ file + Nothing -> showEndFail "no backend could store" file Just (key, backend) -> do logStatus key ValuePresent setup g key @@ -117,11 +117,13 @@ addCmd file = inBackend file err $ do link <- calcGitLink file key liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex annexed " ++ file + showEndOk {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () unannexCmd file = notinBackend file err $ \(key, backend) -> do - liftIO $ putStrLn $ "unannex " ++ file + showStart "unannex" file + Annex.flagChange Force True -- force backend to always remove Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo @@ -132,16 +134,17 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do moveout g src = do nocommit <- Annex.flagIsSet NoCommit liftIO $ removeFile file - liftIO $ Git.run g ["rm", file] + liftIO $ Git.run g ["rm", "--quiet", file] if (not nocommit) - then liftIO $ Git.run g ["commit", "-m", - ("git-annex unannexed " ++ file), file] + then liftIO $ Git.run g ["commit", "--quiet", + "-m", ("git-annex unannexed " ++ file), + file] else return () -- git rm deletes empty directories; -- put them back liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ renameFile src file - return () + showEndOk {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () @@ -150,6 +153,7 @@ getCmd file = notinBackend file err $ \(key, backend) -> do if (inannex) then return () else do + showStart "get" file g <- Annex.gitRepo let dest = annexLocation g key liftIO $ createDirectoryIfMissing True (parentDir dest) @@ -157,8 +161,8 @@ getCmd file = notinBackend file err $ \(key, backend) -> do if (success) then do logStatus key ValuePresent - return () - else error $ "failed to get " ++ file + showEndOk + else showEndFail "get" file where err = error $ "not annexed " ++ file @@ -170,11 +174,13 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do if (not inbackend) then return () -- no-op else do - liftIO $ putStrLn $ "drop " ++ file + showStart "drop" file success <- Backend.removeKey backend key if (success) - then cleanup key - else error $ "backend refused to drop " ++ file + then do + cleanup key + showEndOk + else showEndFail "backend refused to drop" file where cleanup key = do logStatus key ValueMissing @@ -191,13 +197,14 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () fixCmd file = notinBackend file err $ \(key, backend) -> do - liftIO $ putStrLn $ "fix " ++ file link <- calcGitLink file key checkLegal file link + showStart "fix" file liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex fix " ++ file + showEndOk where checkLegal file link = do l <- liftIO $ readSymbolicLink file diff --git a/Core.hs b/Core.hs index 70ec2eca01..27411b2e64 100644 --- a/Core.hs +++ b/Core.hs @@ -7,6 +7,7 @@ import System.IO import System.Directory import Control.Monad.State (liftIO) import System.Path +import Data.String.Utils import Types import Locations @@ -81,8 +82,8 @@ gitAdd file commitmessage = do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] if (isJust commitmessage) - then liftIO $ Git.run g ["commit", "-m", - (fromJust commitmessage), file] + then liftIO $ Git.run g ["commit", "--quiet", + "-m", (fromJust commitmessage), file] else Annex.flagChange NeedCommit True {- Calculates the relative path to use to link a file to a key. -} @@ -104,3 +105,24 @@ logStatus key status = do f <- liftIO $ logChange g key u status gitAdd f Nothing -- all logs are committed at end +{- Output logging -} +showStart :: String -> String -> Annex () +showStart command file = do + liftIO $ putStr $ command ++ " " ++ file + liftIO $ hFlush stdout +showNote :: String -> Annex () +showNote s = do + liftIO $ putStr $ " (" ++ s ++ ")" + liftIO $ hFlush stdout +showLongNote :: String -> Annex () +showLongNote s = do + liftIO $ putStr $ "\n" ++ (indent s) + where + indent s = join "\n" $ map (\l -> " " ++ l) $ lines s +showEndOk :: Annex () +showEndOk = do + liftIO $ putStrLn " ok" +showEndFail :: String -> String -> Annex () +showEndFail command file = do + liftIO $ putStrLn "" + error $ command ++ " " ++ file ++ " failed" diff --git a/UUID.hs b/UUID.hs index 6bd483a18c..b665c27e93 100644 --- a/UUID.hs +++ b/UUID.hs @@ -100,7 +100,7 @@ reposByUUID repos uuids = do prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do m <- uuidMap - return $ unwords $ map (\u -> " "++(prettify m u)++"\n") uuids + return $ unwords $ map (\u -> "\t"++(prettify m u)++"\n") uuids where prettify m u = if (0 < (length $ findlog m u)) From 8398b9ab4a654f3f6ec570b70229a8a0030e8ab6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 13:17:34 -0400 Subject: [PATCH 0198/8313] cleanup output --- Backend/URL.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index 9e64e04996..7535207661 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -9,6 +9,7 @@ import System.Cmd import System.Exit import BackendTypes +import Core backend = Backend { name = "URL", @@ -33,7 +34,8 @@ dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do - liftIO $ putStrLn $ "download: " ++ url + showNote "downloading" + liftIO $ putStrLn "" -- make way for curl progress bar result <- liftIO $ rawSystem "curl" ["-#", "-o", file, url] if (result == ExitSuccess) then return True From a020b0c25c4e7c2e14d685eac8c4d3aa0e1fef8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 16:39:30 -0400 Subject: [PATCH 0199/8313] atomic file retrieval from backends --- Commands.hs | 9 ++++++--- Core.hs | 9 +++++++++ Locations.hs | 8 +++++++- TODO | 6 ------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8591dbf6ac..05af0ab2db 100644 --- a/Commands.hs +++ b/Commands.hs @@ -156,13 +156,16 @@ getCmd file = notinBackend file err $ \(key, backend) -> do showStart "get" file g <- Annex.gitRepo let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - success <- Backend.retrieveKeyFile backend key dest + let tmp = (annexTmpLocation g) ++ (keyFile key) + liftIO $ createDirectoryIfMissing True (parentDir tmp) + success <- Backend.retrieveKeyFile backend key tmp if (success) then do + liftIO $ renameFile tmp dest logStatus key ValuePresent showEndOk - else showEndFail "get" file + else do + showEndFail "get" file where err = error $ "not annexed " ++ file diff --git a/Core.hs b/Core.hs index 27411b2e64..302e304e49 100644 --- a/Core.hs +++ b/Core.hs @@ -29,6 +29,8 @@ startup flags = do shutdown :: Annex () shutdown = do g <- Annex.gitRepo + + -- handle pending commits nocommit <- Annex.flagIsSet NoCommit needcommit <- Annex.flagIsSet NeedCommit if (needcommit && not nocommit) @@ -36,6 +38,13 @@ shutdown = do "git-annex log update", gitStateDir g] else return () + -- clean up any files left in the temp directory + let tmp = annexTmpLocation g + exists <- liftIO $ doesDirectoryExist tmp + if (exists) + then liftIO $ removeDirectoryRecursive $ tmp + else return () + {- configure git to use union merge driver on state files, if it is not - already -} gitAttributes :: Git.Repo -> IO () diff --git a/Locations.hs b/Locations.hs index 76516224cf..2b0adb7ba8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -7,7 +7,8 @@ module Locations ( keyFile, fileKey, annexLocation, - annexLocationRelative + annexLocationRelative, + annexTmpLocation ) where import Data.String.Utils @@ -36,6 +37,11 @@ annexLocation r key = annexLocationRelative :: Git.Repo -> Key -> FilePath annexLocationRelative r key = Git.dir r ++ "/annex/" ++ (keyFile key) +{- .git-annex/tmp is used for temp files + -} +annexTmpLocation :: Git.Repo -> FilePath +annexTmpLocation r = (Git.workTree r) ++ "/" ++ Git.dir r ++ "/annex/tmp/" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/TODO b/TODO index 807df32d84..410c694c28 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) -* bug: git annex add file is silent if file was a symlink and got replaced - with a file. The you then git command -a, you'll check in the fil contents.. - * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo @@ -17,9 +14,6 @@ * Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) -* Copy files atomically, don't leave a partial key on interrupt. - (Fix for URL download too..) - * Find a way to copy a file with a progress bar, while still preserving stat. Easiest way might be to use pv and fix up the permissions etc after? From a4dc920f6b2c31cbdd2c727f1ba7550216303991 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 16:44:06 -0400 Subject: [PATCH 0200/8313] remove some old todos --- Commands.hs | 10 ---------- LocationLog.hs | 1 - doc/git-annex.mdwn | 4 ---- 3 files changed, 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 05af0ab2db..48186928ae 100644 --- a/Commands.hs +++ b/Commands.hs @@ -36,8 +36,6 @@ cmds = [ (Command "add" addCmd FilesNotInGit) , (Command "get" getCmd FilesInGit) , (Command "drop" dropCmd FilesInGit) - , (Command "push" pushCmd RepoName) - , (Command "pull" pullCmd RepoName) , (Command "unannex" unannexCmd FilesInGit) , (Command "describe" describeCmd SingleString) , (Command "fix" fixCmd FilesInOrNotInGit) @@ -216,14 +214,6 @@ fixCmd file = notinBackend file err $ \(key, backend) -> do else return () err = error $ "not annexed " ++ file -{- Pushes all files to a remote repository. -} -pushCmd :: String -> Annex () -pushCmd reponame = do error "not implemented" -- TODO - -{- Pulls all files from a remote repository. -} -pullCmd :: String -> Annex () -pullCmd reponame = do error "not implemented" -- TODO - {- Stores description for the repository. -} describeCmd :: String -> Annex () describeCmd description = do diff --git a/LocationLog.hs b/LocationLog.hs index c0d6170b2e..4a5fe449cd 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -112,7 +112,6 @@ appendLog file line = do createDirectoryIfMissing True (parentDir file) withFileLocked file AppendMode $ \h -> hPutStrLn h $ show line - -- TODO git add log {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4d2872aa3d..66d9897d02 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -43,10 +43,6 @@ Enough broad picture, here's how it actually looks: backend storage to the current repository. * `git annex drop $file` indicates that you no longer want the file's content to be available in this repository. -* `git annex push $repository` pushes *all* annexed files to the specified - repository. -* `git annex pull $repository` pulls *all* annexed files from the specified - repository. * `git annex file $file` adjusts the symlink for the file to point to its content again. Use this if you've moved the file around. * `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` From 632a4e2c6de54aec47a5553d68edd4921231d3c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 17:10:20 -0400 Subject: [PATCH 0201/8313] rename describe to init and show usage --- Commands.hs | 93 +++++++++++++++++++++++++++------------------- doc/git-annex.mdwn | 22 ++++++----- 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/Commands.hs b/Commands.hs index 48186928ae..6d68fc4e75 100644 --- a/Commands.hs +++ b/Commands.hs @@ -23,22 +23,28 @@ import Core import qualified Remotes import qualified BackendTypes -data CmdWants = FilesInGit | FilesNotInGit | FilesInOrNotInGit | - RepoName | SingleString +data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), - cmdwants :: CmdWants + cmdwants :: CmdWants, + cmddesc :: String } cmds :: [Command] cmds = [ - (Command "add" addCmd FilesNotInGit) - , (Command "get" getCmd FilesInGit) - , (Command "drop" dropCmd FilesInGit) - , (Command "unannex" unannexCmd FilesInGit) - , (Command "describe" describeCmd SingleString) - , (Command "fix" fixCmd FilesInOrNotInGit) + (Command "add" addCmd FilesNotInGit + "add files to annex") + , (Command "get" getCmd FilesInGit + "make content of annexed files available") + , (Command "drop" dropCmd FilesInGit + "indicate content of files not currently wanted") + , (Command "unannex" unannexCmd FilesInGit + "undo accidential add command") + , (Command "init" initCmd SingleString + "initialize git-annex with repository description") + , (Command "fix" fixCmd FilesInGit + "fix up files' symlinks to point to annexed content") ] options = [ @@ -46,6 +52,17 @@ options = [ , Option ['N'] ["no-commit"] (NoArg NoCommit) "do not stage or commit changes" ] +header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." + +usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs + where + cmddescs = unlines $ map (\c -> indent $ showcmd c) cmds + showcmd c = + (cmdname c) ++ + (take (10 - (length (cmdname c))) $ repeat ' ') ++ + (cmddesc c) + indent l = " " ++ l + {- Finds the type of parameters a command wants, from among the passed - parameter list. -} findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] @@ -55,10 +72,6 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files -findWanted FilesInOrNotInGit params repo = do - a <- findWanted FilesInGit params repo - b <- findWanted FilesNotInGit params repo - return $ union a b findWanted SingleString params _ = do return $ [unwords params] findWanted RepoName params _ = do @@ -73,7 +86,7 @@ parseCmd argv state = do 0 -> error usage _ -> case (lookupCmd (params !! 0)) of [] -> error usage - [Command _ action want] -> do + [Command _ action want _] -> do f <- findWanted want (drop 1 params) (BackendTypes.repo state) return (flags, map action $ filter notstate f) @@ -84,9 +97,6 @@ parseCmd argv state = do (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds - header = "Usage: git-annex [" ++ - (join "|" $ map cmdname cmds) ++ "] ..." - usage = usageInfo header options {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} @@ -197,32 +207,37 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () -fixCmd file = notinBackend file err $ \(key, backend) -> do +fixCmd file = notinBackend file skip $ \(key, backend) -> do link <- calcGitLink file key - checkLegal file link - showStart "fix" file - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file - liftIO $ createSymbolicLink link file - gitAdd file $ Just $ "git-annex fix " ++ file - showEndOk + l <- liftIO $ readSymbolicLink file + if (link == l) + then skip + else do + showStart "fix" file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + gitAdd file $ Just $ "git-annex fix " ++ file + showEndOk where - checkLegal file link = do - l <- liftIO $ readSymbolicLink file - if (link == l) - then error $ "symbolic link already ok for: " ++ file - else return () - err = error $ "not annexed " ++ file + -- quietly skip non-annexed files, so this can be used + -- as a commit hook + skip = return () {- Stores description for the repository. -} -describeCmd :: String -> Annex () -describeCmd description = do - g <- Annex.gitRepo - u <- getUUID g - describeUUID u description - log <- uuidLog - gitAdd log $ Just $ "description for UUID " ++ (show u) - liftIO $ putStrLn "description set" +initCmd :: String -> Annex () +initCmd description = do + if (0 == length description) + then error $ + "please specify a description of this repository\n" ++ + usage + else do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + log <- uuidLog + gitAdd log $ Just $ "description for UUID " ++ (show u) + liftIO $ putStrLn "description set" -- helpers inBackend file yes no = do diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 66d9897d02..4647eb0587 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -49,7 +49,7 @@ Enough broad picture, here's how it actually looks: if you're just done with a file; only use `unannex` if you accidentially added a file. (You can also run this on all your annexed files come the Singularity. ;-) -* `git annex describe "some description"` allows associating some description +* `git annex init "some description"` allows associating some description (such as "USB archive drive 1") with a repository. This can help with finding it later, see "Location Tracking" below. @@ -128,21 +128,23 @@ is on a home file server, and you are away from home. Then git-annex can tell you what git remote it needs access to in order to get a file: # git annex get myfile - git-annex: unable to get file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile - To get that file, need access to one of these remotes: home + get myfile (need access to one of these remotes: home) + git-annex: get myfile failed Another way the location tracking comes in handy is if you put repositories on removable USB drives, that might be archived away offline in a safe place. In this sort of case, you probably don't have a git remotes configured for every USB drive. So git-annex may have to resort to talking -about repository UUIDs. If you have previously used "git annex describe" -in those repositories, it will include their description to help you with -finding them: +about repository UUIDs. If you have previously used "git annex init" +to attach descriptions to those repositories, it will include their +descriptions to help you with finding them: - git-annex: no available git remotes have file with key: WORM:8b01f6d371178722367393eb26043482e1820306:myfile - It has been seen before in these repositories: - c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 - e1938fee-d95b-11df-96cc-002170d25c55 + # git annex get myfile + get myfile (No available git remotes have the file.) + It has been seen before in these repositories: + c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 + e1938fee-d95b-11df-96cc-002170d25c55 + git-annex: get myfile failed ## configuration From 97b20e7ffeae1e6b678a77c72871af8b03d62cc6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 17:13:37 -0400 Subject: [PATCH 0202/8313] update --- INSTALL | 2 ++ 1 file changed, 2 insertions(+) diff --git a/INSTALL b/INSTALL index 9594448e70..a7fc7f6f31 100644 --- a/INSTALL +++ b/INSTALL @@ -3,3 +3,5 @@ To build and use git-annex, you will need: * ghc * These haskell libraries: MissingH SHA * uuid + +Then just run make; make install From e602238cd8cd309cea812ae85044f7d9f79be530 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 18:27:37 -0400 Subject: [PATCH 0203/8313] don't complain if a file is not annexed --- Commands.hs | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/Commands.hs b/Commands.hs index 6d68fc4e75..028c458cae 100644 --- a/Commands.hs +++ b/Commands.hs @@ -101,7 +101,7 @@ parseCmd argv state = do {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () -addCmd file = inBackend file err $ do +addCmd file = inBackend file $ do liftIO $ checkLegal file showStart "add" file g <- Annex.gitRepo @@ -112,7 +112,6 @@ addCmd file = inBackend file err $ do logStatus key ValuePresent setup g key where - err = error $ "already annexed " ++ file checkLegal file = do s <- getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -129,7 +128,7 @@ addCmd file = inBackend file err $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () -unannexCmd file = notinBackend file err $ \(key, backend) -> do +unannexCmd file = notinBackend file $ \(key, backend) -> do showStart "unannex" file Annex.flagChange Force True -- force backend to always remove Backend.removeKey backend key @@ -138,7 +137,6 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do let src = annexLocation g key moveout g src where - err = error $ "not annexed " ++ file moveout g src = do nocommit <- Annex.flagIsSet NoCommit liftIO $ removeFile file @@ -156,7 +154,7 @@ unannexCmd file = notinBackend file err $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () -getCmd file = notinBackend file err $ \(key, backend) -> do +getCmd file = notinBackend file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return () @@ -174,13 +172,11 @@ getCmd file = notinBackend file err $ \(key, backend) -> do showEndOk else do showEndFail "get" file - where - err = error $ "not annexed " ++ file {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () -dropCmd file = notinBackend file err $ \(key, backend) -> do +dropCmd file = notinBackend file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return () -- no-op @@ -203,15 +199,14 @@ dropCmd file = notinBackend file err $ \(key, backend) -> do liftIO $ removeFile loc return () else return () - err = error $ "not annexed " ++ file {- Fixes the symlink to an annexed file. -} fixCmd :: String -> Annex () -fixCmd file = notinBackend file skip $ \(key, backend) -> do +fixCmd file = notinBackend file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) - then skip + then return () else do showStart "fix" file liftIO $ createDirectoryIfMissing True (parentDir file) @@ -219,10 +214,6 @@ fixCmd file = notinBackend file skip $ \(key, backend) -> do liftIO $ createSymbolicLink link file gitAdd file $ Just $ "git-annex fix " ++ file showEndOk - where - -- quietly skip non-annexed files, so this can be used - -- as a commit hook - skip = return () {- Stores description for the repository. -} initCmd :: String -> Annex () @@ -240,9 +231,13 @@ initCmd description = do liftIO $ putStrLn "description set" -- helpers -inBackend file yes no = do +inBackend file a = do r <- Backend.lookupFile file case (r) of - Just v -> yes v - Nothing -> no -notinBackend file yes no = inBackend file no yes + Just v -> return () + Nothing -> a +notinBackend file a = do + r <- Backend.lookupFile file + case (r) of + Just v -> a v + Nothing -> return () From bb6707020d08f7509c21c1229088bb6017438caf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 18:52:01 -0400 Subject: [PATCH 0204/8313] update --- doc/git-annex.mdwn | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4647eb0587..d15ca4a9f2 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -146,6 +146,19 @@ descriptions to help you with finding them: e1938fee-d95b-11df-96cc-002170d25c55 git-annex: get myfile failed +## symlink farming commit hook + +git-annex does use a lot of symlinks. Specicially, relative symlinks, +that are checked into git. To allow you to move those around without +annoyance, git-annex can run as a post-commit hook. This way, you can `git mv` +a symlink to an annexed file, and as soon as you commit, it will be fixed +up. + +`git annex init` tries to set up a post-commit hook that is itself a symlink +back to git-annex. If you want to have your own shell script in the post-commit +hook, just make it call `git annex` with no parameters. git-annex will detect +when it's run from a git hook and do the necessary fixups. + ## configuration * `annex.uuid` -- a unique UUID for this repository @@ -165,22 +178,6 @@ descriptions to help you with finding them: ## issues -### symlinks - -If the symlink to annexed content is relative, moving it to a subdir will -break it. But it it's absolute, moving the git repo (or mounting its drive -elsewhere) will break it. Either: - -* Use relative links and need `git annex mv` to move (or post-commit - hook that caches moves and updates links). -* Use absolute links and need `git annex fixlinks` when location changes; - note that would also mean that git would see the symlink targets changed - and want to commit the change. And, other clones of the repo would - diverge and there would be conflicts on the symlink text. Ugh. - -Hard links are not an option, because git would then happily commit the -file content. Amoung other reasons.. - ### free space determination Need a way to tell how much free space is available on the disk containing From 335c06171ac9a45a76b3b92d647615142bcc6ba0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 18:52:09 -0400 Subject: [PATCH 0205/8313] commit logs at end; faster --- Backend/File.hs | 4 ++-- Commands.hs | 6 +++--- Core.hs | 20 +++++++++----------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index f7796532be..9b81bef9a1 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -116,6 +116,8 @@ checkRemoveKey key = do then retNotEnoughCopiesKnown remotes numcopies else findcopies numcopies remotes [] where + config = "annex.numcopies" + findcopies 0 _ _ = return True -- success, enough copies found findcopies _ [] bad = notEnoughCopiesSeen bad findcopies n (r:rs) bad = do @@ -151,5 +153,3 @@ checkRemoveKey key = do showLongNote $ "According to the " ++ config ++ " setting, it is not safe to remove it!" showLongNote "(Use --force to override.)" - - config = "annex.numcopies" diff --git a/Commands.hs b/Commands.hs index 028c458cae..9a79e9d0c5 100644 --- a/Commands.hs +++ b/Commands.hs @@ -123,7 +123,7 @@ addCmd file = inBackend file $ do liftIO $ renameFile file dest link <- calcGitLink file key liftIO $ createSymbolicLink link file - gitAdd file $ Just $ "git-annex annexed " ++ file + gitAdd file $ "git-annex annexed " ++ file showEndOk {- Undo addCmd. -} @@ -212,7 +212,7 @@ fixCmd file = notinBackend file $ \(key, backend) -> do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file - gitAdd file $ Just $ "git-annex fix " ++ file + gitAdd file $ "git-annex fix " ++ file showEndOk {- Stores description for the repository. -} @@ -227,7 +227,7 @@ initCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - gitAdd log $ Just $ "description for UUID " ++ (show u) + gitAdd log $ "description for UUID " ++ (show u) liftIO $ putStrLn "description set" -- helpers diff --git a/Core.hs b/Core.hs index 302e304e49..5f63002c1f 100644 --- a/Core.hs +++ b/Core.hs @@ -34,8 +34,10 @@ shutdown = do nocommit <- Annex.flagIsSet NoCommit needcommit <- Annex.flagIsSet NeedCommit if (needcommit && not nocommit) - then liftIO $ Git.run g ["commit", "-q", "-m", - "git-annex log update", gitStateDir g] + then do + liftIO $ Git.run g ["add", gitStateDir g] + liftIO $ Git.run g ["commit", "-q", "-m", + "git-annex log update", gitStateDir g] else return () -- clean up any files left in the temp directory @@ -75,14 +77,12 @@ inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key -{- Adds, optionally also commits a file to git. - - - - All changes to the git repository should go through this function. +{- Adds and commits a file to git. - - This is careful to not rely on the index. It may have staged changes, - so only use operations that avoid committing such changes. -} -gitAdd :: FilePath -> Maybe String -> Annex () +gitAdd :: FilePath -> String -> Annex () gitAdd file commitmessage = do nocommit <- Annex.flagIsSet NoCommit if (nocommit) @@ -90,10 +90,8 @@ gitAdd file commitmessage = do else do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] - if (isJust commitmessage) - then liftIO $ Git.run g ["commit", "--quiet", - "-m", (fromJust commitmessage), file] - else Annex.flagChange NeedCommit True + liftIO $ Git.run g ["commit", "--quiet", + "-m", commitmessage, file] {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath @@ -112,7 +110,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g f <- liftIO $ logChange g key u status - gitAdd f Nothing -- all logs are committed at end + Annex.flagChange NeedCommit True -- commit all logs at end {- Output logging -} showStart :: String -> String -> Annex () From 8f634a5e16a7fae58c5bb24628e19c319905606e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 20:35:33 -0400 Subject: [PATCH 0206/8313] cleanup --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 9a79e9d0c5..de3089a566 100644 --- a/Commands.hs +++ b/Commands.hs @@ -187,7 +187,7 @@ dropCmd file = notinBackend file $ \(key, backend) -> do then do cleanup key showEndOk - else showEndFail "backend refused to drop" file + else showEndFail "drop" file where cleanup key = do logStatus key ValueMissing From 939a6f860e1a2eea58e46a05861076e1b174cbd2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Oct 2010 23:53:01 -0400 Subject: [PATCH 0207/8313] thoughts --- doc/git-annex.mdwn | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d15ca4a9f2..4c85a03b69 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -197,8 +197,39 @@ may not be dropped right away, depending on number of copies available. The use of `.git-annex` to store logs means that if a repo has branches and the user switched between them, git-annex will see different logs in the different branches, and so may miss info about what remotes have which -files (though it can re-learn). An alternative would be to -store the log data directly in the git repo as `pristine-tar` does. +files (though it can re-learn). + +An alternative would be to store the log data directly in the git repo +as `pristine-tar` does. Problem with that approach is that git won't merge +conflicting changes to log files if they are not in the currently checked +out branch. + +It would be possible to use a branch with a tree like this, to avoid +conflicts: + +key/uuid/time/status + +As long as new files are only added, and old timestamped files deleted, +there would be no conflicts. + +A related problem though is the size of the tree objects git needs to +commit. Having the logs in a separate branch doesn't help with that. +As more keys are added, the tree object size will increase, and git will +take longer and longer to commit, and use more space. One way to deal with +this is simply by splitting the logs amoung subdirectories. Git then can +reuse trees for most directories. (Check: Does it still have to build +dup trees in memory?) + +Another approach would be to have git-annex *delete* old logs. Keep logs +for the currently available files, or something like that. If other log +info is needed, look back through history to find the first occurance of a +log. Maybe even look at other branches -- so if the logs were on master, +a new empty branch could be made and git-annex would still know where to +get keys in that branch. + +Would have to be careful about conflicts when deleting and bringing back +files with the same name. And would need to avoid expensive searching thru +all history to try to find an old log file. ## contact From 4b1086cc7d1dd9cb4eba78210976a731a683948d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 01:52:06 -0400 Subject: [PATCH 0208/8313] experimentally, removing all actual git committing Idea is the user will commit when ready, just stage everything. --- BackendTypes.hs | 3 +-- Commands.hs | 14 ++++---------- Core.hs | 29 ++--------------------------- LocationLog.hs | 6 ++---- 4 files changed, 9 insertions(+), 43 deletions(-) diff --git a/BackendTypes.hs b/BackendTypes.hs index e372099b2d..548ef17a2e 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -11,8 +11,7 @@ import Data.String.Utils import qualified GitRepo as Git -- command-line flags -data Flag = Force | NoCommit | NeedCommit - deriving (Eq, Read, Show) +data Flag = Force deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, -- but it uses Backend, so has to be here to avoid a depends loop. diff --git a/Commands.hs b/Commands.hs index de3089a566..718e991c9a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -49,7 +49,6 @@ cmds = [ options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" - , Option ['N'] ["no-commit"] (NoArg NoCommit) "do not stage or commit changes" ] header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." @@ -123,7 +122,7 @@ addCmd file = inBackend file $ do liftIO $ renameFile file dest link <- calcGitLink file key liftIO $ createSymbolicLink link file - gitAdd file $ "git-annex annexed " ++ file + liftIO $ Git.run g ["add", file] showEndOk {- Undo addCmd. -} @@ -138,14 +137,8 @@ unannexCmd file = notinBackend file $ \(key, backend) -> do moveout g src where moveout g src = do - nocommit <- Annex.flagIsSet NoCommit liftIO $ removeFile file liftIO $ Git.run g ["rm", "--quiet", file] - if (not nocommit) - then liftIO $ Git.run g ["commit", "--quiet", - "-m", ("git-annex unannexed " ++ file), - file] - else return () -- git rm deletes empty directories; -- put them back liftIO $ createDirectoryIfMissing True (parentDir file) @@ -212,7 +205,8 @@ fixCmd file = notinBackend file $ \(key, backend) -> do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file - gitAdd file $ "git-annex fix " ++ file + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] showEndOk {- Stores description for the repository. -} @@ -227,7 +221,7 @@ initCmd description = do u <- getUUID g describeUUID u description log <- uuidLog - gitAdd log $ "description for UUID " ++ (show u) + liftIO $ Git.run g ["add", log] liftIO $ putStrLn "description set" -- helpers diff --git a/Core.hs b/Core.hs index 5f63002c1f..0af22ee73e 100644 --- a/Core.hs +++ b/Core.hs @@ -30,15 +30,7 @@ shutdown :: Annex () shutdown = do g <- Annex.gitRepo - -- handle pending commits - nocommit <- Annex.flagIsSet NoCommit - needcommit <- Annex.flagIsSet NeedCommit - if (needcommit && not nocommit) - then do - liftIO $ Git.run g ["add", gitStateDir g] - liftIO $ Git.run g ["commit", "-q", "-m", - "git-annex log update", gitStateDir g] - else return () + liftIO $ Git.run g ["add", gitStateDir g] -- clean up any files left in the temp directory let tmp = annexTmpLocation g @@ -77,22 +69,6 @@ inAnnex key = do g <- Annex.gitRepo liftIO $ doesFileExist $ annexLocation g key -{- Adds and commits a file to git. - - - - This is careful to not rely on the index. It may have staged changes, - - so only use operations that avoid committing such changes. - -} -gitAdd :: FilePath -> String -> Annex () -gitAdd file commitmessage = do - nocommit <- Annex.flagIsSet NoCommit - if (nocommit) - then return () - else do - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] - liftIO $ Git.run g ["commit", "--quiet", - "-m", commitmessage, file] - {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath calcGitLink file key = do @@ -109,8 +85,7 @@ logStatus :: Key -> LogStatus -> Annex () logStatus key status = do g <- Annex.gitRepo u <- getUUID g - f <- liftIO $ logChange g key u status - Annex.flagChange NeedCommit True -- commit all logs at end + liftIO $ logChange g key u status {- Output logging -} showStart :: String -> String -> Annex () diff --git a/LocationLog.hs b/LocationLog.hs index 4a5fe449cd..785b3330db 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -80,14 +80,12 @@ instance Read LogLine where undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] -{- Log a change in the presence of a key's value in a repository, - - and return the log filename. -} -logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath +{- Log a change in the presence of a key's value in a repository. -} +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO () logChange repo key uuid status = do log <- logNow status uuid ls <- readLog logfile writeLog logfile (compactLog $ log:ls) - return logfile where logfile = logFile repo key From 0382d26cdbdc52c1e985cba9667a4d50d0653216 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 01:57:32 -0400 Subject: [PATCH 0209/8313] speling --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 718e991c9a..bdeab5fc91 100644 --- a/Commands.hs +++ b/Commands.hs @@ -48,7 +48,7 @@ cmds = [ ] options = [ - Option ['f'] ["force"] (NoArg Force) "allow actions that may loose annexed data" + Option ['f'] ["force"] (NoArg Force) "allow actions that may lose annexed data" ] header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." From f3dcc8489d7b7f9417f9752987a298976838ce47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 02:06:27 -0400 Subject: [PATCH 0210/8313] gratuitous rename --- Annex.hs | 30 ++++++++++++++--------------- Backend.hs | 14 +++++++------- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- BackendList.hs | 2 -- Commands.hs | 4 ++-- Locations.hs | 2 +- BackendTypes.hs => TypeInternals.hs | 6 +++--- Types.hs | 2 +- 11 files changed, 33 insertions(+), 35 deletions(-) rename BackendTypes.hs => TypeInternals.hs (93%) diff --git a/Annex.hs b/Annex.hs index 5fd500d949..b68e513553 100644 --- a/Annex.hs +++ b/Annex.hs @@ -17,17 +17,17 @@ import Control.Monad.State import qualified GitRepo as Git import Types -import qualified BackendTypes as Backend +import qualified TypeInternals as Internals {- Create and returns an Annex state object for the specified git repo. -} new :: Git.Repo -> [Backend] -> IO AnnexState new gitrepo allbackends = do - let s = Backend.AnnexState { - Backend.repo = gitrepo, - Backend.backends = [], - Backend.supportedBackends = allbackends, - Backend.flags = [] + let s = Internals.AnnexState { + Internals.repo = gitrepo, + Internals.backends = [], + Internals.supportedBackends = allbackends, + Internals.flags = [] } (_,s') <- Annex.run s (prep gitrepo) return s' @@ -44,34 +44,34 @@ run state action = runStateT (action) state gitRepo :: Annex Git.Repo gitRepo = do state <- get - return (Backend.repo state) + return (Internals.repo state) gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get - put state { Backend.repo = r } + put state { Internals.repo = r } return () backends :: Annex [Backend] backends = do state <- get - return (Backend.backends state) + return (Internals.backends state) backendsChange :: [Backend] -> Annex () backendsChange b = do state <- get - put state { Backend.backends = b } + put state { Internals.backends = b } return () supportedBackends :: Annex [Backend] supportedBackends = do state <- get - return (Backend.supportedBackends state) + return (Internals.supportedBackends state) flagIsSet :: Flag -> Annex Bool flagIsSet flag = do state <- get - return $ elem flag $ Backend.flags state + return $ elem flag $ Internals.flags state flagChange :: Flag -> Bool -> Annex () flagChange flag set = do state <- get - let f = filter (/= flag) $ Backend.flags state + let f = filter (/= flag) $ Internals.flags state if (set) - then put state { Backend.flags = (flag:f) } - else put state { Backend.flags = f } + then put state { Internals.flags = (flag:f) } + else put state { Internals.flags = f } return () diff --git a/Backend.hs b/Backend.hs index dfaa559702..a427234d7c 100644 --- a/Backend.hs +++ b/Backend.hs @@ -33,7 +33,7 @@ import qualified GitRepo as Git import qualified Annex import Utility import Types -import qualified BackendTypes as B +import qualified TypeInternals as Internals {- List of backends in the order to try them when storing a new key. -} backendList :: Annex [Backend] @@ -59,7 +59,7 @@ lookupBackendName all s = if ((length matches) /= 1) then error $ "unknown backend " ++ s else matches !! 0 - where matches = filter (\b -> s == B.name b) all + where matches = filter (\b -> s == Internals.name b) all {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) @@ -70,11 +70,11 @@ storeFileKey file = do storeFileKey' b file relfile storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do - try <- (B.getKey b) relfile + try <- (Internals.getKey b) relfile case (try) of Nothing -> nextbackend Just key -> do - stored <- (B.storeFileKey b) file key + stored <- (Internals.storeFileKey b) file key if (not stored) then nextbackend else do @@ -85,17 +85,17 @@ storeFileKey' (b:bs) file relfile = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool -retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest +retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} removeKey :: Backend -> Key -> Annex Bool -removeKey backend key = (B.removeKey backend) key +removeKey backend key = (Internals.removeKey backend) key {- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool hasKey key = do all <- Annex.supportedBackends - (B.hasKey (lookupBackendName all $ backendName key)) key + (Internals.hasKey (lookupBackendName all $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index 9b81bef9a1..c97a354d0c 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -16,7 +16,7 @@ import System.Cmd import System.Exit import Control.Exception -import BackendTypes +import TypeInternals import LocationLog import Locations import qualified Remotes diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index c01e01a723..2143a6af59 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -6,7 +6,7 @@ module Backend.SHA1 (backend) where import Data.Digest.Pure.SHA import qualified Backend.File -import BackendTypes +import TypeInternals backend = Backend.File.backend { name = "SHA1", diff --git a/Backend/URL.hs b/Backend/URL.hs index 7535207661..5c1fd74c93 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -8,7 +8,7 @@ import Data.String.Utils import System.Cmd import System.Exit -import BackendTypes +import TypeInternals import Core backend = Backend { diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 463b0ac8e2..0588ddaf83 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -9,7 +9,7 @@ import System.Posix.Files import qualified Data.ByteString.Lazy.Char8 as B import qualified Backend.File -import BackendTypes +import TypeInternals import Utility backend = Backend.File.backend { diff --git a/BackendList.hs b/BackendList.hs index 920f8fc0a6..25f3ae5eac 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -3,8 +3,6 @@ module BackendList (allBackends) where -import BackendTypes - -- When adding a new backend, import it here and add it to the list. import qualified Backend.WORM import qualified Backend.SHA1 diff --git a/Commands.hs b/Commands.hs index bdeab5fc91..fab72160a2 100644 --- a/Commands.hs +++ b/Commands.hs @@ -21,7 +21,7 @@ import LocationLog import Types import Core import qualified Remotes -import qualified BackendTypes +import qualified TypeInternals data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString data Command = Command { @@ -87,7 +87,7 @@ parseCmd argv state = do [] -> error usage [Command _ action want _] -> do f <- findWanted want (drop 1 params) - (BackendTypes.repo state) + (TypeInternals.repo state) return (flags, map action $ filter notstate f) where -- never include files from the state directory diff --git a/Locations.hs b/Locations.hs index 2b0adb7ba8..18d416eb4a 100644 --- a/Locations.hs +++ b/Locations.hs @@ -14,7 +14,7 @@ module Locations ( import Data.String.Utils import Types -import qualified BackendTypes as Backend +import qualified TypeInternals as Internals import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex diff --git a/BackendTypes.hs b/TypeInternals.hs similarity index 93% rename from BackendTypes.hs rename to TypeInternals.hs index 548ef17a2e..e8f7cb9e74 100644 --- a/BackendTypes.hs +++ b/TypeInternals.hs @@ -1,9 +1,9 @@ -{- git-annex backend data types +{- git-annex internal data types - - - Mostly only backend implementations should need to import this. + - Most things should not need this, using Types and/or Annex instead. -} -module BackendTypes where +module TypeInternals where import Control.Monad.State (StateT) import Data.String.Utils diff --git a/Types.hs b/Types.hs index 6bf26d36e6..2284d92674 100644 --- a/Types.hs +++ b/Types.hs @@ -10,4 +10,4 @@ module Types ( Flag(..), ) where -import BackendTypes +import TypeInternals From 33432a32775d23dff5e5b6f499656b247e4ed604 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 21:08:27 -0400 Subject: [PATCH 0211/8313] bug --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 410c694c28..6fa34046db 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,9 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) +* bug: ctrl+c does not stop it from running another action; need to + not catch UserInterrupt exceptions. + * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo From 15986f01d1fd565da151dcb08697e21a94fc9037 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 Oct 2010 21:36:26 -0400 Subject: [PATCH 0212/8313] bug --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 6fa34046db..eca1922b7a 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,9 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) +* bug: doesn't learn new remote's uuids if a known (but maybe not accessible) + uuids has a wanted file + * bug: ctrl+c does not stop it from running another action; need to not catch UserInterrupt exceptions. From c7664588f81fe27b3e88d49523ef3c483ac6481a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:19:56 -0400 Subject: [PATCH 0213/8313] use safesystem --- Backend/File.hs | 7 ++----- Backend/URL.hs | 11 +++++++---- GitRepo.hs | 2 +- TODO | 3 --- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index c97a354d0c..8969d7556e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -13,6 +13,7 @@ module Backend.File (backend) where import Control.Monad.State import System.IO import System.Cmd +import System.Cmd.Utils import System.Exit import Control.Exception @@ -92,11 +93,7 @@ copyFromRemote r key file = do then getlocal else getremote where - getlocal = do - res <-rawSystem "cp" ["-a", location, file] - if (res == ExitSuccess) - then return () - else error "cp failed" + getlocal = safeSystem "cp" ["-a", location, file] getremote = error "get via network not yet implemented!" location = annexLocation r key diff --git a/Backend/URL.hs b/Backend/URL.hs index 5c1fd74c93..c9b6ab6df8 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -3,9 +3,11 @@ module Backend.URL (backend) where +import Control.Exception import Control.Monad.State (liftIO) import Data.String.Utils import System.Cmd +import System.Cmd.Utils import System.Exit import TypeInternals @@ -36,9 +38,10 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" liftIO $ putStrLn "" -- make way for curl progress bar - result <- liftIO $ rawSystem "curl" ["-#", "-o", file, url] - if (result == ExitSuccess) - then return True - else return False + result <- liftIO $ (try curl::IO (Either SomeException ())) + case result of + Left err -> return False + Right succ -> return True where + curl = safeSystem "curl" ["-#", "-o", file, url] url = join ":" $ drop 1 $ split ":" $ show key diff --git a/GitRepo.hs b/GitRepo.hs index 32383197b5..5b0e68cd62 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -167,7 +167,7 @@ gitCommandLine repo params = assertlocal repo $ {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () run repo params = assertlocal repo $ do - r <- rawSystem "git" (gitCommandLine repo params) + r <- safeSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} diff --git a/TODO b/TODO index eca1922b7a..e6fdcd0b29 100644 --- a/TODO +++ b/TODO @@ -4,9 +4,6 @@ * bug: doesn't learn new remote's uuids if a known (but maybe not accessible) uuids has a wanted file -* bug: ctrl+c does not stop it from running another action; need to - not catch UserInterrupt exceptions. - * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo From 7afac113443b8e93e19ad87d769a24c52706f551 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:45:45 -0400 Subject: [PATCH 0214/8313] add boolSystem --- Utility.hs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index e4278ff3f6..09b973002f 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,10 +6,15 @@ module Utility ( hGetContentsStrict, parentDir, relPathCwdToDir, - relPathDirToDir + relPathDirToDir, + boolSystem ) where import System.IO +import System.Cmd +import System.Exit +import System.Posix.Signals +import Data.Typeable import System.Posix.IO import Data.String.Utils import System.Path @@ -88,3 +93,18 @@ relPathDirToDir from to = dotdots = take ((length pfrom) - numcommon) $ repeat ".." numcommon = length $ common path = join s $ dotdots ++ uncommon + +{- Run a system command, and returns True or False + - if it succeeded or failed. + - + - An error is thrown if the command exits due to SIGINT, + - to propigate ctrl-c. + -} +boolSystem :: FilePath -> [String] -> IO Bool +boolSystem command params = do + r <- rawSystem command params + case r of + ExitSuccess -> return True + ExitFailure e -> if Just e == cast sigINT + then error $ command ++ "interrupted" + else return False From 470e0a2fbd1f554df677127212643d534c2f7857 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:46:07 -0400 Subject: [PATCH 0215/8313] use boolSystem --- Backend/File.hs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 8969d7556e..6b2e82726d 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -14,7 +14,6 @@ import Control.Monad.State import System.IO import System.Cmd import System.Cmd.Utils -import System.Exit import Control.Exception import TypeInternals @@ -70,12 +69,7 @@ copyKeyFile key file = do Nothing -> trycopy full rs Just r' -> do showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." - result <- liftIO $ (try (copyFromRemote r' key file)::IO (Either SomeException ())) - case (result) of - Left err -> do - liftIO $ hPutStrLn stderr (show err) - trycopy full rs - Right succ -> return True + liftIO $ copyFromRemote r' key file cantfind = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key @@ -86,15 +80,15 @@ copyKeyFile key file = do else return () return False -{- Tries to copy a file from a remote, exception on error. -} -copyFromRemote :: Git.Repo -> Key -> FilePath -> IO () +{- Tries to copy a file from a remote. -} +copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool copyFromRemote r key file = do if (Git.repoIsLocal r) then getlocal else getremote where - getlocal = safeSystem "cp" ["-a", location, file] - getremote = error "get via network not yet implemented!" + getlocal = boolSystem "cp" ["-a", location, file] + getremote = return False -- TODO implement get from remote location = annexLocation r key {- Checks remotes to verify that enough copies of a key exist to allow From 2caf711827470976f935bb06bb3b6b87e1776299 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 01:46:20 -0400 Subject: [PATCH 0216/8313] stop trapping all exceptions Need to allow exceptions to be thrown for SIGPIPE propigation. Converted places that used error unncessarily to not. --- Commands.hs | 26 ++++++++++++-------------- git-annex.hs | 10 +++++----- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/Commands.hs b/Commands.hs index fab72160a2..115c3b3edb 100644 --- a/Commands.hs +++ b/Commands.hs @@ -101,21 +101,19 @@ parseCmd argv state = do - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () addCmd file = inBackend file $ do - liftIO $ checkLegal file - showStart "add" file - g <- Annex.gitRepo - stored <- Backend.storeFileKey file - case (stored) of - Nothing -> showEndFail "no backend could store" file - Just (key, backend) -> do - logStatus key ValuePresent - setup g key + s <- liftIO $ getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then return () + else do + showStart "add" file + g <- Annex.gitRepo + stored <- Backend.storeFileKey file + case (stored) of + Nothing -> showEndFail "no backend could store" file + Just (key, backend) -> do + logStatus key ValuePresent + setup g key where - checkLegal file = do - s <- getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) - then error $ "not a regular file: " ++ file - else return () setup g key = do let dest = annexLocation g key liftIO $ createDirectoryIfMissing True (parentDir dest) diff --git a/git-annex.hs b/git-annex.hs index 947868f238..71a21379dc 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,6 +1,6 @@ {- git-annex main program -} -import Control.Exception +import IO (try) import System.IO import System.Environment @@ -18,8 +18,9 @@ main = do (flags, actions) <- parseCmd args state tryRun state $ [startup flags] ++ actions ++ [shutdown] -{- Runs a list of Annex actions. Catches exceptions, not stopping - - if some error out, and propigates an overall error status at the end. +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + - Propigates an overall error status at the end. - - This runs in the IO monad, not in the Annex monad. It seems that - exceptions can only be caught in the IO monad, not in a stacked monad; @@ -29,8 +30,7 @@ main = do tryRun :: AnnexState -> [Annex ()] -> IO () tryRun state actions = tryRun' state 0 actions tryRun' state errnum (a:as) = do - result <- try - (Annex.run state a)::IO (Either SomeException ((), AnnexState)) + result <- try $ Annex.run state a case (result) of Left err -> do showErr err From 2ea589e117bcc36ee613454ffb52b8e52cc96bc9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 12:53:51 -0400 Subject: [PATCH 0217/8313] don't throw a fatal error --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 0af22ee73e..5bf1083938 100644 --- a/Core.hs +++ b/Core.hs @@ -107,4 +107,4 @@ showEndOk = do showEndFail :: String -> String -> Annex () showEndFail command file = do liftIO $ putStrLn "" - error $ command ++ " " ++ file ++ " failed" + liftIO $ hPutStrLn stderr $ command ++ " " ++ file ++ " failed" From d23fc22f0e17c95765f940f81f733f9580e19107 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 12:55:40 -0400 Subject: [PATCH 0218/8313] less verbose failures seem better here --- Commands.hs | 6 +++--- Core.hs | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Commands.hs b/Commands.hs index 115c3b3edb..e38f9c3725 100644 --- a/Commands.hs +++ b/Commands.hs @@ -109,7 +109,7 @@ addCmd file = inBackend file $ do g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of - Nothing -> showEndFail "no backend could store" file + Nothing -> showEndFail Just (key, backend) -> do logStatus key ValuePresent setup g key @@ -162,7 +162,7 @@ getCmd file = notinBackend file $ \(key, backend) -> do logStatus key ValuePresent showEndOk else do - showEndFail "get" file + showEndFail {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} @@ -178,7 +178,7 @@ dropCmd file = notinBackend file $ \(key, backend) -> do then do cleanup key showEndOk - else showEndFail "drop" file + else showEndFail where cleanup key = do logStatus key ValueMissing diff --git a/Core.hs b/Core.hs index 5bf1083938..3532c71d52 100644 --- a/Core.hs +++ b/Core.hs @@ -104,7 +104,6 @@ showLongNote s = do showEndOk :: Annex () showEndOk = do liftIO $ putStrLn " ok" -showEndFail :: String -> String -> Annex () -showEndFail command file = do - liftIO $ putStrLn "" - liftIO $ hPutStrLn stderr $ command ++ " " ++ file ++ " failed" +showEndFail :: Annex () +showEndFail = do + liftIO $ putStrLn " failed" From 3531ce5c54e380d15d54d838c90f4ebe311782af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 13:08:05 -0400 Subject: [PATCH 0219/8313] fix remote uuid learning bug --- Remotes.hs | 11 ++++++++--- TODO | 3 --- TypeInternals.hs | 6 ++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index f21f5a6ba4..828dc753fe 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -30,17 +30,22 @@ withKey key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost - -- this only uses cached data, so may not find new remotes + -- This only uses cached data, so may not include new remotes + -- or remotes whose uuid has changed (eg by a different drive being + -- mounted at their location). So unless it happens to find all + -- remotes, try harder, loading the remotes' configs. remotes <- reposByUUID allremotes uuids - if (0 == length remotes) + remotesread <- Annex.flagIsSet RemotesRead + if ((length allremotes /= length remotes) && not remotesread) then tryharder allremotes uuids else return remotes where tryharder allremotes uuids = do - -- more expensive; check each remote's config + -- more expensive; read each remote's config mayberemotes <- mapM tryGitConfigRead allremotes let allremotes' = catMaybes mayberemotes remotes' <- reposByUUID allremotes' uuids + Annex.flagChange RemotesRead True return remotes' {- Cost Ordered list of remotes. -} diff --git a/TODO b/TODO index e6fdcd0b29..410c694c28 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,6 @@ * bug: cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) -* bug: doesn't learn new remote's uuids if a known (but maybe not accessible) - uuids has a wanted file - * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo diff --git a/TypeInternals.hs b/TypeInternals.hs index e8f7cb9e74..4a9d2653e1 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -10,8 +10,10 @@ import Data.String.Utils import qualified GitRepo as Git --- command-line flags -data Flag = Force deriving (Eq, Read, Show) +data Flag = + Force | -- command-line flags + RemotesRead -- indicates that remote repo configs have been read + deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, -- but it uses Backend, so has to be here to avoid a depends loop. From ed3f6653b664d72e4b89c4dd0c56f4b7db7cbab9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 13:39:53 -0400 Subject: [PATCH 0220/8313] better drop error messages --- Backend/File.hs | 49 +++++++++++++++++++++++++++++-------------------- UUID.hs | 2 +- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 6b2e82726d..6944a8b62f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,8 @@ import System.IO import System.Cmd import System.Cmd.Utils import Control.Exception +import List +import Maybe import TypeInternals import LocationLog @@ -52,7 +54,10 @@ copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- Remotes.withKey key if (0 == length remotes) - then cantfind + then do + showNote $ "No available git remotes have the file." + showLocations key + return False else trycopy remotes remotes where trycopy full [] = do @@ -70,15 +75,6 @@ copyKeyFile key file = do Just r' -> do showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." liftIO $ copyFromRemote r' key file - cantfind = do - g <- Annex.gitRepo - uuids <- liftIO $ keyLocations g key - ppuuids <- prettyPrintUUIDs uuids - showNote $ "No available git remotes have the file." - if (0 < length uuids) - then showLongNote $ "It has been seen before in these repositories:\n" ++ ppuuids - else return () - return False {- Tries to copy a file from a remote. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool @@ -90,6 +86,17 @@ copyFromRemote r key file = do getlocal = boolSystem "cp" ["-a", location, file] getremote = return False -- TODO implement get from remote location = annexLocation r key + +showLocations :: Key -> Annex () +showLocations key = do + g <- Annex.gitRepo + u <- getUUID g + uuids <- liftIO $ keyLocations g key + let uuidsf = filter (\v -> v /= u) uuids + ppuuids <- prettyPrintUUIDs uuidsf + if (0 < length uuidsf) + then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids + else showLongNote $ "No other repository is known to contain the file." {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an @@ -125,22 +132,24 @@ checkRemoveKey key = do (result, _) <- Annex.run a (Backend.hasKey key) return result notEnoughCopiesSeen bad = do - showNote "failed to find enough other copies of the file" - if (0 /= length bad) then listbad bad else return () unsafe + if (0 /= length bad) then listbad bad else return () + showLocations key + hint return False listbad bad = showLongNote $ "I was unable to access these remotes: " ++ (Remotes.list bad) retNotEnoughCopiesKnown remotes numcopies = do - showNote $ - "I only know about " ++ (show $ length remotes) ++ - " out of " ++ (show numcopies) ++ - " necessary copies of the file" unsafe + showLongNote $ + "Could only verify the existence of " ++ + (show $ length remotes) ++ + " out of " ++ (show numcopies) ++ + " necessary copies" + showLocations key + hint return False - unsafe = do - showLongNote $ "According to the " ++ config ++ - " setting, it is not safe to remove it!" - showLongNote "(Use --force to override.)" + unsafe = showNote "unsafe" + hint = showLongNote $ "(Use --force to override this check, or adjust annex.numcopies.)" diff --git a/UUID.hs b/UUID.hs index b665c27e93..47d305c4fb 100644 --- a/UUID.hs +++ b/UUID.hs @@ -51,7 +51,7 @@ getUUID r = do let c = cached r g let u = uncached r - + if (c /= u && u /= "") then do updatecache g r u From 4f8d28819da8e13bfa741dae8d84b0000e38e083 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:13:48 -0400 Subject: [PATCH 0221/8313] improved messages when a file is not available in remotes --- Backend/File.hs | 55 ++++++++++++++++++++++--------------------------- Remotes.hs | 13 ++++++------ 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 6944a8b62f..4ea25daa73 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -55,15 +55,15 @@ copyKeyFile key file = do remotes <- Remotes.withKey key if (0 == length remotes) then do - showNote $ "No available git remotes have the file." + showNote "not available" showLocations key return False else trycopy remotes remotes where trycopy full [] = do - showNote $ - "need access to one of these remotes: " ++ - (Remotes.list full) + showNote "not available" + showTriedRemotes full + showLocations key return False trycopy full (r:rs) = do -- annexLocation needs the git config to have been @@ -71,8 +71,8 @@ copyKeyFile key file = do -- if it hasn't been already result <- Remotes.tryGitConfigRead r case (result) of - Nothing -> trycopy full rs - Just r' -> do + Left err -> trycopy full rs + Right r' -> do showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." liftIO $ copyFromRemote r' key file @@ -86,7 +86,7 @@ copyFromRemote r key file = do getlocal = boolSystem "cp" ["-a", location, file] getremote = return False -- TODO implement get from remote location = annexLocation r key - + showLocations :: Key -> Annex () showLocations key = do g <- Annex.gitRepo @@ -97,6 +97,10 @@ showLocations key = do if (0 < length uuidsf) then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids else showLongNote $ "No other repository is known to contain the file." + +showTriedRemotes remotes = + showLongNote $ "I was unable to access these remotes: " ++ + (Remotes.list remotes) {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an @@ -108,46 +112,37 @@ checkRemoveKey key = do then return True else do g <- Annex.gitRepo - let numcopies = read $ Git.configGet g config "1" remotes <- Remotes.withKey key + let numcopies = read $ Git.configGet g config "1" if (numcopies > length remotes) - then retNotEnoughCopiesKnown remotes numcopies - else findcopies numcopies remotes [] + then notEnoughCopies numcopies (length remotes) [] + else findcopies numcopies 0 remotes [] where config = "annex.numcopies" - - findcopies 0 _ _ = return True -- success, enough copies found - findcopies _ [] bad = notEnoughCopiesSeen bad - findcopies n (r:rs) bad = do + findcopies need have [] bad = + if (have >= need) + then return True + else notEnoughCopies need have bad + findcopies need have (r:rs) bad = do all <- Annex.supportedBackends result <- liftIO $ ((try $ remoteHasKey r all)::IO (Either SomeException Bool)) case (result) of - Right True -> findcopies (n-1) rs bad - Right False -> findcopies n rs bad - Left _ -> findcopies n rs (r:bad) + Right True -> findcopies need (have+1) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) remoteHasKey r all = do -- To check if a remote has a key, construct a new -- Annex monad and query its backend. a <- Annex.new r all (result, _) <- Annex.run a (Backend.hasKey key) return result - notEnoughCopiesSeen bad = do - unsafe - if (0 /= length bad) then listbad bad else return () - showLocations key - hint - return False - listbad bad = - showLongNote $ - "I was unable to access these remotes: " ++ - (Remotes.list bad) - retNotEnoughCopiesKnown remotes numcopies = do + notEnoughCopies need have bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - (show $ length remotes) ++ - " out of " ++ (show numcopies) ++ + (show have) ++ " out of " ++ (show need) ++ " necessary copies" + if (0 /= length bad) then showTriedRemotes bad else return () showLocations key hint return False diff --git a/Remotes.hs b/Remotes.hs index 828dc753fe..a0894f418b 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -10,6 +10,7 @@ import Control.Exception import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils +import Data.Either.Utils import List import Maybe @@ -42,8 +43,8 @@ withKey key = do where tryharder allremotes uuids = do -- more expensive; read each remote's config - mayberemotes <- mapM tryGitConfigRead allremotes - let allremotes' = catMaybes mayberemotes + eitherremotes <- mapM tryGitConfigRead allremotes + let allremotes' = map fromEither eitherremotes remotes' <- reposByUUID allremotes' uuids Annex.flagChange RemotesRead True return remotes' @@ -86,7 +87,7 @@ repoCost r = do - because reading it may be expensive. This function tries to read the - config for a specified remote, and updates state. If successful, it - returns the updated git repo. -} -tryGitConfigRead :: Git.Repo -> Annex (Maybe Git.Repo) +tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do if (Map.null $ Git.configMap r) then do @@ -94,15 +95,15 @@ tryGitConfigRead r = do -- for other reasons; catch all possible exceptions result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of - Left err -> return Nothing + Left err -> return $ Left r Right r' -> do g <- Annex.gitRepo let l = Git.remotes g let g' = Git.remotesAdd g $ exchange l r' Annex.gitRepoChange g' - return $ Just r' - else return $ Just r + return $ Right r' + else return $ Right r -- config already read where exchange [] new = [] exchange (old:ls) new = From 731cabbe3d083bcca6535fc9751b63cc16a83b83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:15:53 -0400 Subject: [PATCH 0222/8313] newlines before failed message needed if a long message was shown --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 3532c71d52..8dc4bff6fc 100644 --- a/Core.hs +++ b/Core.hs @@ -106,4 +106,4 @@ showEndOk = do liftIO $ putStrLn " ok" showEndFail :: Annex () showEndFail = do - liftIO $ putStrLn " failed" + liftIO $ putStrLn "\nfailed" From f3c5a8543b2793f507b4a4801315d1f333e758cc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:17:41 -0400 Subject: [PATCH 0223/8313] update --- TODO | 2 +- debian/docs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/TODO b/TODO index 410c694c28..a804597b88 100644 --- a/TODO +++ b/TODO @@ -4,7 +4,7 @@ * --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo -* how to handle git mv file? -> git annex fix -> run automatically? +* how to handle git mv file? -> git annex fix -> run automatically on commit * how to handle git rm file? (should try to drop keys that have no referring file, if it seems safe..) diff --git a/debian/docs b/debian/docs index 9de86edc7e..d6fa57dd7b 100644 --- a/debian/docs +++ b/debian/docs @@ -1 +1,2 @@ doc/*.mdwn +TODO From e8267f1b9e99cce79209eb2f47fce02d52d60b56 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:37:19 -0400 Subject: [PATCH 0224/8313] add doc wiki --- Makefile | 12 +++++++ debian/control | 2 +- doc/Makefile | 15 +++++++++ doc/bugs.mdwn | 4 +++ doc/bugs/branching.mdwn | 36 ++++++++++++++++++++ doc/bugs/done.mdwn | 3 ++ doc/bugs/free_space_checking.mdwn | 8 +++++ doc/contact.mdwn | 4 +++ doc/download.mdwn | 5 +++ doc/git-annex.mdwn | 55 ------------------------------- doc/index.mdwn | 23 +++++++++++++ doc/news.mdwn | 5 +++ 12 files changed, 116 insertions(+), 56 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/bugs.mdwn create mode 100644 doc/bugs/branching.mdwn create mode 100644 doc/bugs/done.mdwn create mode 100644 doc/bugs/free_space_checking.mdwn create mode 100644 doc/contact.mdwn create mode 100644 doc/download.mdwn create mode 100644 doc/index.mdwn create mode 100644 doc/news.mdwn diff --git a/Makefile b/Makefile index d1fcbbeeeb..87c725efea 100644 --- a/Makefile +++ b/Makefile @@ -8,5 +8,17 @@ install: clean: rm -rf build git-annex + rm -rf doc/.ikiwiki html + +# Build static html docs suitable for being shipped in the software +# package. This depends on ikiwiki being installed to build the docs. +ifeq ($(shell which ikiwiki),) +IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki +else +IKIWIKI=ikiwiki +endif + +docs: + $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff .PHONY: git-annex diff --git a/debian/control b/debian/control index e58f55af9e..846844c32a 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev, ikiwiki Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000000..f2c4d8e54d --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,15 @@ +# Build static html docs suitable for being shipped in the software +# package. This depends on ikiwiki being installed to build the docs. + +ifeq ($(shell which ikiwiki),) +IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki +else +IKIWIKI=ikiwiki +endif + +all: + $(IKIWIKI) `pwd` html -v --wikiname FooBar --plugin=goodstuff \ + --exclude=html --exclude=Makefile + +clean: + rm -rf .ikiwiki html diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn new file mode 100644 index 0000000000..dd2a9b4035 --- /dev/null +++ b/doc/bugs.mdwn @@ -0,0 +1,4 @@ +This is git-annex's bug list. Link bugs to [[bugs/done]] when done. + +[[!inline pages="./bugs/* and !./bugs/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0]] diff --git a/doc/bugs/branching.mdwn b/doc/bugs/branching.mdwn new file mode 100644 index 0000000000..21996ecc00 --- /dev/null +++ b/doc/bugs/branching.mdwn @@ -0,0 +1,36 @@ +The use of `.git-annex` to store logs means that if a repo has branches +and the user switched between them, git-annex will see different logs in +the different branches, and so may miss info about what remotes have which +files (though it can re-learn). + +An alternative would be to store the log data directly in the git repo +as `pristine-tar` does. Problem with that approach is that git won't merge +conflicting changes to log files if they are not in the currently checked +out branch. + +It would be possible to use a branch with a tree like this, to avoid +conflicts: + +key/uuid/time/status + +As long as new files are only added, and old timestamped files deleted, +there would be no conflicts. + +A related problem though is the size of the tree objects git needs to +commit. Having the logs in a separate branch doesn't help with that. +As more keys are added, the tree object size will increase, and git will +take longer and longer to commit, and use more space. One way to deal with +this is simply by splitting the logs amoung subdirectories. Git then can +reuse trees for most directories. (Check: Does it still have to build +dup trees in memory?) + +Another approach would be to have git-annex *delete* old logs. Keep logs +for the currently available files, or something like that. If other log +info is needed, look back through history to find the first occurance of a +log. Maybe even look at other branches -- so if the logs were on master, +a new empty branch could be made and git-annex would still know where to +get keys in that branch. + +Would have to be careful about conflicts when deleting and bringing back +files with the same name. And would need to avoid expensive searching thru +all history to try to find an old log file. diff --git a/doc/bugs/done.mdwn b/doc/bugs/done.mdwn new file mode 100644 index 0000000000..ad332e2a26 --- /dev/null +++ b/doc/bugs/done.mdwn @@ -0,0 +1,3 @@ +recently fixed [[bugs]] + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10]] diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn new file mode 100644 index 0000000000..34528a7b35 --- /dev/null +++ b/doc/bugs/free_space_checking.mdwn @@ -0,0 +1,8 @@ +Should check that there is enough free space before trying to copy a +file around. + +* Need a way to tell how much free space is available on the disk containing + a given repository. + +* And, need a way to tell the size of a file before copying it from + a remote, to check local disk space. diff --git a/doc/contact.mdwn b/doc/contact.mdwn new file mode 100644 index 0000000000..1238ca0403 --- /dev/null +++ b/doc/contact.mdwn @@ -0,0 +1,4 @@ +Joey Hess is the author of git-annex. + +The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home) +is a good place to discuss it. diff --git a/doc/download.mdwn b/doc/download.mdwn new file mode 100644 index 0000000000..2ceb73193a --- /dev/null +++ b/doc/download.mdwn @@ -0,0 +1,5 @@ +The main git repository for git-annex is `git://git.kitenet.net/git-annex` +[[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] + +There are no binary packages yet, but you can build Debian packages from +the source tree with `dpkg-buildpackage`. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4c85a03b69..eb5fa9ced9 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -176,61 +176,6 @@ when it's run from a git hook and do the necessary fixups. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. -## issues - -### free space determination - -Need a way to tell how much free space is available on the disk containing -a given repository. The repository may be remote, so ssh may need to be -used. - -Similarly, need a way to tell the size of a file before copying it from -a remote, to check local disk space. - -### auto-drop on rm - -When git-rm removed a file, its key should get dropped too. Of course, it -may not be dropped right away, depending on number of copies available. - -### branching - -The use of `.git-annex` to store logs means that if a repo has branches -and the user switched between them, git-annex will see different logs in -the different branches, and so may miss info about what remotes have which -files (though it can re-learn). - -An alternative would be to store the log data directly in the git repo -as `pristine-tar` does. Problem with that approach is that git won't merge -conflicting changes to log files if they are not in the currently checked -out branch. - -It would be possible to use a branch with a tree like this, to avoid -conflicts: - -key/uuid/time/status - -As long as new files are only added, and old timestamped files deleted, -there would be no conflicts. - -A related problem though is the size of the tree objects git needs to -commit. Having the logs in a separate branch doesn't help with that. -As more keys are added, the tree object size will increase, and git will -take longer and longer to commit, and use more space. One way to deal with -this is simply by splitting the logs amoung subdirectories. Git then can -reuse trees for most directories. (Check: Does it still have to build -dup trees in memory?) - -Another approach would be to have git-annex *delete* old logs. Keep logs -for the currently available files, or something like that. If other log -info is needed, look back through history to find the first occurance of a -log. Maybe even look at other branches -- so if the logs were on master, -a new empty branch could be made and git-annex would still know where to -get keys in that branch. - -Would have to be careful about conflicts when deleting and bringing back -files with the same name. And would need to avoid expensive searching thru -all history to try to find an old log file. - ## contact Joey Hess diff --git a/doc/index.mdwn b/doc/index.mdwn new file mode 100644 index 0000000000..8d2cb1ef57 --- /dev/null +++ b/doc/index.mdwn @@ -0,0 +1,23 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +Even without file content tracking, being able to manage files with git, +move files around and delete files with versioned directory trees, and use +branches and distributed clones, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly +versioned files, which is convenient for maintaining documents, Makefiles, +etc that are associated with annexed files but that benefit from full +revision control. + +* [[man page|git-annex]] +* **[[download]]** +* [[news]] +* [[bugs]] +* [[contact]] + +---- + +git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and +hosted by [Branchable](http://branchable.com/). diff --git a/doc/news.mdwn b/doc/news.mdwn new file mode 100644 index 0000000000..d0ff1ca2c1 --- /dev/null +++ b/doc/news.mdwn @@ -0,0 +1,5 @@ +This is where announcements of new releases, features, and other news is +posted. git-annex users are recommended to subscribe to this page's RSS +feed. + +[[!inline pages="./news/* and !*/Discussion" rootpage="news" show="30"]] From 21128c88e71cb2050b78f996888d9e251a448807 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 14:39:40 -0400 Subject: [PATCH 0225/8313] tweak --- Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 87c725efea..a0911df04c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +all: git-annex docs + git-annex: mkdir -p build ghc -odir build -hidir build --make git-annex @@ -10,15 +12,16 @@ clean: rm -rf build git-annex rm -rf doc/.ikiwiki html -# Build static html docs suitable for being shipped in the software -# package. This depends on ikiwiki being installed to build the docs. +# If ikiwiki is available, build static html docs suitable for being +# shipped in the software package. ifeq ($(shell which ikiwiki),) -IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki +IKIWIKI=echo "** ikiwiki not found, skipping building docs" >&2 else IKIWIKI=ikiwiki endif docs: - $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff + $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ + --no-usedirs .PHONY: git-annex From 6ef1c2d2daf37dc92b4c364ea34802b62688018b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 15:17:26 -0400 Subject: [PATCH 0226/8313] allow lines with leading tab, to be preformatted text --- mdwn2man | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 mdwn2man diff --git a/mdwn2man b/mdwn2man new file mode 100755 index 0000000000..c212539454 --- /dev/null +++ b/mdwn2man @@ -0,0 +1,43 @@ +#!/usr/bin/perl +# Warning: hack + +my $prog=shift; +my $section=shift; + +print ".TH $prog $section\n"; + +while (<>) { + s{(\\?)\[\[([^\s\|\]]+)(\|[^\s\]]+)?\]\]}{$1 ? "[[$2]]" : $2}eg; + s/\`//g; + s/^\s*\./\\&./g; + if (/^#\s/) { + s/^#\s/.SH /; + <>; # blank; + } + s/^ +//; + s/^\t/ /; + s/-/\\-/g; + s/^Warning:.*//g; + s/^$/.PP\n/; + s/^\*\s+(.*)/.IP "$1"/; + next if $_ eq ".PP\n" && $skippara; + if (/^.IP /) { + $inlist=1; + $spippara=0; + } + elsif (/.SH/) { + $skippara=0; + $inlist=0; + } + elsif (/^\./) { + $skippara=1; + } + else { + $skippara=0; + } + if ($inlist && $_ eq ".PP\n") { + $_=".IP\n"; + } + + print $_; +} From 7bc4435ffdc1760a7ac8638cdc1cfac78aebaabb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 15:59:40 -0400 Subject: [PATCH 0227/8313] update --- .gitignore | 3 + Commands.hs | 2 +- Makefile | 9 +- doc/Makefile | 15 -- doc/backends.mdwn | 21 ++ doc/bugs/symlink_farming_commit_hook.mdwn | 12 ++ doc/copies.mdwn | 30 +++ doc/git-annex.mdwn | 223 +++++++++------------- doc/index.mdwn | 9 +- doc/location_tracking.mdwn | 28 +++ 10 files changed, 202 insertions(+), 150 deletions(-) delete mode 100644 doc/Makefile create mode 100644 doc/backends.mdwn create mode 100644 doc/bugs/symlink_farming_commit_hook.mdwn create mode 100644 doc/copies.mdwn create mode 100644 doc/location_tracking.mdwn diff --git a/.gitignore b/.gitignore index 2b3e3aef1f..13deb526a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ build/* git-annex +git-annex.1 +doc/.ikiwiki +html diff --git a/Commands.hs b/Commands.hs index e38f9c3725..2addf714e8 100644 --- a/Commands.hs +++ b/Commands.hs @@ -51,7 +51,7 @@ options = [ Option ['f'] ["force"] (NoArg Force) "allow actions that may lose annexed data" ] -header = "Usage: git-annex [" ++ (join "|" $ map cmdname cmds) ++ "] ..." +header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) ++ " [path ...]" usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where diff --git a/Makefile b/Makefile index a0911df04c..f18f076da9 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,6 @@ install: install -d $(DESTDIR)/usr/bin install git-annex $(DESTDIR)/usr/bin -clean: - rm -rf build git-annex - rm -rf doc/.ikiwiki html - # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) @@ -21,7 +17,12 @@ IKIWIKI=ikiwiki endif docs: + ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs +clean: + rm -rf build git-annex git-annex.1 + rm -rf doc/.ikiwiki html + .PHONY: git-annex diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index f2c4d8e54d..0000000000 --- a/doc/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# Build static html docs suitable for being shipped in the software -# package. This depends on ikiwiki being installed to build the docs. - -ifeq ($(shell which ikiwiki),) -IKIWIKI=echo "** ikiwiki not found" >&2 ; echo ikiwiki -else -IKIWIKI=ikiwiki -endif - -all: - $(IKIWIKI) `pwd` html -v --wikiname FooBar --plugin=goodstuff \ - --exclude=html --exclude=Makefile - -clean: - rm -rf .ikiwiki html diff --git a/doc/backends.mdwn b/doc/backends.mdwn new file mode 100644 index 0000000000..d3ccaec491 --- /dev/null +++ b/doc/backends.mdwn @@ -0,0 +1,21 @@ +git-annex uses a key-value abstraction layer to allow file contents to be +stored in different ways. In theory, any key-value storage system could be +used to store file contents. + +When a file is annexed, a key is generated from its content and/or metadata. +The file checked into git symlinks to the key. This key can later be used +to retrieve the file's content (its value). + +Multiple pluggable backends are supported, and more than one can be used +to store different files' contents in a given repository. + +* `WORM` ("Write Once, Read Many") This backend stores the file's content + only in `.git/annex/`, and assumes that any file with the same basename, + size, and modification time has the same content. So with this backend, + files can be moved around, but should never be added to or changed. + This is the default, and the least expensive backend. +* `SHA1` -- This backend stores the file's content in + `.git/annex/`, with a name based on its sha1 checksum. This backend allows + modifications of files to be tracked. Its need to generate checksums + can make it slower for large files. +* `URL` -- This backend downloads the file's content from an external URL. diff --git a/doc/bugs/symlink_farming_commit_hook.mdwn b/doc/bugs/symlink_farming_commit_hook.mdwn new file mode 100644 index 0000000000..af03beb70e --- /dev/null +++ b/doc/bugs/symlink_farming_commit_hook.mdwn @@ -0,0 +1,12 @@ +TODO: implement below + +git-annex does use a lot of symlinks. Specicially, relative symlinks, +that are checked into git. To allow you to move those around without +annoyance, git-annex can run as a post-commit hook. This way, you can `git mv` +a symlink to an annexed file, and as soon as you commit, it will be fixed +up. + +`git annex init` tries to set up a post-commit hook that is itself a symlink +back to git-annex. If you want to have your own shell script in the post-commit +hook, just make it call `git annex` with no parameters. git-annex will detect +when it's run from a git hook and do the necessary fixups. diff --git a/doc/copies.mdwn b/doc/copies.mdwn new file mode 100644 index 0000000000..ff66f4e8ae --- /dev/null +++ b/doc/copies.mdwn @@ -0,0 +1,30 @@ +The WORM and SHA1 key-value [[backends|backend]] store data inside +your git repository's `.git` directory, not in some external data store. + +It's important that data not get lost by an ill-considered `git annex drop` +command. So, then using those backends, git-annex can be configured to try +to keep N copies of a file's content available across all repositories. By +default, N is 1; it is configured by annex.numcopies. + +`git annex drop` attempts to check with other git remotes, to check that N +copies of the file exist. If enough repositories cannot be verified to have +it, it will retain the file content to avoid data loss. + +For example, consider three repositories: Server, Laptop, and USB. Both Server +and USB have a copy of a file, and N=1. If on Laptop, you `git annex get +$file`, this will transfer it from either Server or USB (depending on which +is available), and there are now 3 copies of the file. + +Suppose you want to free up space on Laptop again, and you `git annex drop` the file +there. If USB is connected, or Server can be contacted, git-annex can check +that it still has a copy of the file, and the content is removed from +Laptop. But if USB is currently disconnected, and Server also cannot be +contacted, it can't verify that it is safe to drop the file, and will +refuse to do so. + +With N=2, in order to drop the file content from Laptop, it would need access +to both USB and Server. + +Note that different repositories can be configured with different values of +N. So just because Laptop has N=2, this does not prevent the number of +copies falling to 1, when USB and Server have N=1. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index eb5fa9ced9..25cf6f776c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -1,3 +1,13 @@ +# NAME + +git-annex - manage files with git, without checking their contents in + +# SYNOPSIS + +git annex subcommand [path ...] + +# DESCRIPTION + git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when dealing with files larger than git can currently easily handle, whether due @@ -11,157 +21,94 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. -My motivation for git-annex was the growing number of external drives I -use. Some are used to archive data, others hold backups, and yet others -come with me when I'm away from home to carry data that doesn't fit on my -netbook. Maintaining all that was a nightmare, lots of ad-hoc moving files -around, rsyncing files (unison is too slow), and deleting multiple copies -of files from multiple places. I realized what what I needed was a form of -revision control where each drive was a repository, and where copying the -files around, and deciding which copies were safe to delete was automated. -I posted about this to the VCS-home mailing list and got a great suggestion -to make it support arbitrary key-value stores, for more generality and -flexability. A week of coding later, and git-annex is born. +When a file is annexed, its content is moved into a key-value store, and +a symlink is made that points to the content. These symlinks are checked into +git and versioned like regular files. You can move them around, delete +them, and so on. Pushing to another git repository will make git-annex +there aware of the annexed file, and it can be used to retrieve its +content from the key-value store. -Enough broad picture, here's how it actually looks: +# EXAMPLES -* `git annex add $file` moves the file into `.git/annex/`, and replaces - it with a symlink pointing at the annexed file, and then calls `git add` - to version the *symlink*. (If the file has already been annexed, it does - nothing.) - - If you then use normal git push/pull commands, the annexed file content - won't be transferred between repositories, but the symlinks will be. - So different clones of a repository can have different sets of annexed - files available. - - You can move the symlink around, copy it, delete it, etc, and commit changes - as desired using git. Reading the symlink will always get you the annexed - file content, or the link may be broken if the content is not currently - available. -* `git annex get $file` is used to transfer a specified file from the - backend storage to the current repository. -* `git annex drop $file` indicates that you no longer want the file's - content to be available in this repository. -* `git annex file $file` adjusts the symlink for the file to point to its - content again. Use this if you've moved the file around. -* `git annex unannex $file` undoes a `git annex add`. But use `git annex drop` - if you're just done with a file; only use `unannex` if you - accidentially added a file. (You can also run this on all your annexed - files come the Singularity. ;-) -* `git annex init "some description"` allows associating some description - (such as "USB archive drive 1") with a repository. This can help with - finding it later, see "Location Tracking" below. + # git annex get video/hackity_hack_and_kaxxt.mov + get video/_why_hackity_hack_and_kaxxt.mov (not available) + I was unable to access these remotes: server + Try making some of these repositories available: + 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + failed + # sudo mount /media/usb + # git remote add usbdrive /media/usb + # git annex get video/hackity_hack_and_kaxxt.mov + get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok + # git commit -a -m "got a video I want to rewatch on the plane" + + # git annex add iso + add iso/Debian_5.0.iso ok + # git commit -a -m "saving Debian CD for later" + + # git annex push usbdrive iso + error: push not yet implemented! + # git annex drop iso + drop iso/Debian_5.0.iso ok + # git commit -a -m "freed up space" -Oh yeah, "$file" in the above can be any number of files, or directories, -same as you'd pass to "git add" or "git rm". -So "git annex add ." or "git annex get dir/" work fine. +# SUBCOMMANDS -## key-value storage +Like many git commands, git-annex can be passed a path that +is either a file or a directory. In the latter case it acts on all relevant +files in the directory. -git-annex uses a key-value abstraction layer to allow file contents to be -stored in different ways. In theory, any key-value storage system could be -used to store the file contents, and git-annex would then retrieve them -as needed and put them in `.git/annex/`. +Many git-annex subcommands will stage changes for later `git commit` by you. -When a file is annexed, a key is generated from its content and/or metadata. -The file checked into git symlinks to the key. This key can later be used -to retrieve the file's content (its value). This key generation must be -stable for a given file content, name, and size. +* add [path ...] -Multiple pluggable backends are supported, and more than one can be used -to store different files' contents in a given repository. + Adds files in the path to the annex. Files that are already checked into + git, or that git has been configured to ignore will be silently skipped. -* `WORM` ("Write Once, Read Many") This backend stores the file's content - only in `.git/annex/`, and assumes that any file with the same basename, - size, and modification time has the same content. So with this backend, - files can be moved around, but should never be added to or changed. - This is the default, and the least expensive backend. -* `SHA1` -- This backend stores the file's content in - `.git/annex/`, with a name based on its sha1 checksum. This backend allows - modifications of files to be tracked. Its need to generate checksums - can make it slow for large files. -* `URL` -- This backend downloads the file's content from an external URL. +* get [path ...] -## copies + Makes the content of annexed files available in this repository. Depending + on the backend used, this will involve copying them from another repository, + or downloading them, or transferring them from some kind of key-value store. -The WORM and SHA1 key-value backends store data inside your git repository. -It's important that data not get lost by an ill-though `git annex drop` -command. So, then using those backends, git-annex can be configured to try -to keep N copies of a file's content available across all repositories. By -default, N is 1; it is configured by annex.numcopies. +* drop [path ...] -`git annex drop` attempts to check with other git remotes, to check that N -copies of the file exist. If enough repositories cannot be verified to have -it, it will retain the file content to avoid data loss. + Drops the content of annexed files from this repository. -For example, consider three repositories: Server, Laptop, and USB. Both Server -and USB have a copy of a file, and N=1. If on Laptop, you `git annex get -$file`, this will transfer it from either Server or USB (depending on which -is available), and there are now 3 copies of the file. + git-annex may refuse to drop a content if the backend does not think + it is safe to do so. -Suppose you want to free up space on Laptop again, and you `git annex drop` the file -there. If USB is connected, or Server can be contacted, git-annex can check -that it still has a copy of the file, and the content is removed from -Laptop. But if USB is currently disconnected, and Server also cannot be -contacted, it can't verify that it is safe to drop the file, and will -refuse to do so. +* unannex [path ...] -With N=2, in order to drop the file content from Laptop, it would need access -to both USB and Server. + Use this to undo an accidental add command. This is not the command you + should use if you intentionally annexed a file and don't want its contents + any more. In that case you should use `git annex drop` instead, and you + can also `git rm` the file. -Note that different repositories can be configured with different values of -N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1. +* init "description" -## location tracking + Initializes git-annex with a descripotion of the git repository. + This is an optional, but recommended step. -git-annex keeps track of in which repositories it last saw a file's content. -This location tracking information is stored in `.git-annex/$key.log`. -Repositories record their UUID and the date when they get or drop -a file's content. (Git is configured to use a union merge for this file, -so the lines may be in arbitrary order, but it will never conflict.) +* fix [path ...] -This location tracking information is useful if you have multiple -repositories, and not all are always accessible. For example, perhaps one -is on a home file server, and you are away from home. Then git-annex can -tell you what git remote it needs access to in order to get a file: + Fixes up symlinks that have become broken to again point to annexed content. + This is useful to run if you have been moving the symlinks around. - # git annex get myfile - get myfile (need access to one of these remotes: home) - git-annex: get myfile failed +# OPTIONS -Another way the location tracking comes in handy is if you put repositories -on removable USB drives, that might be archived away offline in a safe -place. In this sort of case, you probably don't have a git remotes -configured for every USB drive. So git-annex may have to resort to talking -about repository UUIDs. If you have previously used "git annex init" -to attach descriptions to those repositories, it will include their -descriptions to help you with finding them: +* --force - # git annex get myfile - get myfile (No available git remotes have the file.) - It has been seen before in these repositories: - c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 - e1938fee-d95b-11df-96cc-002170d25c55 - git-annex: get myfile failed + Force unsafe actions, such as dropping a file's content when no other + source of it can be verified to still exist. Use with care. -## symlink farming commit hook +## CONFIGURATION -git-annex does use a lot of symlinks. Specicially, relative symlinks, -that are checked into git. To allow you to move those around without -annoyance, git-annex can run as a post-commit hook. This way, you can `git mv` -a symlink to an annexed file, and as soon as you commit, it will be fixed -up. +Like other git commands, git-annex is configured via `.git/config`. -`git annex init` tries to set up a post-commit hook that is itself a symlink -back to git-annex. If you want to have your own shell script in the post-commit -hook, just make it call `git annex` with no parameters. git-annex will detect -when it's run from a git hook and do the necessary fixups. - -## configuration - -* `annex.uuid` -- a unique UUID for this repository +* `annex.uuid` -- a unique UUID for this repository (automatically set) * `annex.numcopies` -- number of copies of files to keep across all repositories (default: 1) * `annex.backends` -- space-separated list of names of @@ -176,6 +123,24 @@ when it's run from a git hook and do the necessary fixups. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. -## contact +# FILES -Joey Hess +These files are used, in your git repository: + +`.git/annex/` contains the annexed file contents that are currently +available. Annexed files in your git repository symlink to that content. + +`.git-annex/uuid.log` is used to map between repository UUID and +decscriptions. You may edit it. + +`.git-annex/*.log` is where git-annex records its content tracking +information. These files should be committed to git. + +`.git-annex/.gitattributes` is configured to use git's union merge driver +to avoid conflicts when merging files in the `.git-annex` directory. + +# AUTHOR + +Joey Hess + +Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD). Edit with care diff --git a/doc/index.mdwn b/doc/index.mdwn index 8d2cb1ef57..df42eabc11 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -11,12 +11,19 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. -* [[man page|git-annex]] * **[[download]]** * [[news]] * [[bugs]] * [[contact]] +## documentation + +* [[man page|git-annex]] +* [[key-value backends|backends]] for data storage +* [[location_tracking]] reminds you where git-annex has seen files +* git-annex prevents accidential data loss by [[tracking copies|copies]] + of your files + ---- git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn new file mode 100644 index 0000000000..a7d5c150b1 --- /dev/null +++ b/doc/location_tracking.mdwn @@ -0,0 +1,28 @@ +git-annex keeps track of in which repositories it last saw a file's content. +This location tracking information is stored in `.git-annex/$key.log`. +Repositories record their UUID and the date when they get or drop +a file's content. (Git is configured to use a union merge for this file, +so the lines may be in arbitrary order, but it will never conflict.) + +This location tracking information is useful if you have multiple +repositories, and not all are always accessible. For example, perhaps one +is on a home file server, and you are away from home. Then git-annex can +tell you what git remote it needs access to in order to get a file: + + # git annex get myfile + get myfile(not available) + I was unable to access these remotes: home + +Another way the location tracking comes in handy is if you put repositories +on removable USB drives, that might be archived away offline in a safe +place. In this sort of case, you probably don't have a git remotes +configured for every USB drive. So git-annex may have to resort to talking +about repository UUIDs. If you have previously used "git annex init" +to attach descriptions to those repositories, it will include their +descriptions to help you with finding them: + + # git annex get myfile + get myfile (not available) + Try making some of these repositories available: + c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 + e1938fee-d95b-11df-96cc-002170d25c55 From 05539c773ea1246de253e1373ca1711412b91503 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:01:48 -0400 Subject: [PATCH 0228/8313] split TODO --- TODO | 21 --------------------- debian/docs | 3 +-- doc/bugs/backendchecksum.mdwn | 1 + doc/bugs/dotdot_problem.mdwn | 2 ++ doc/bugs/file_copy_progress_bar.mdwn | 3 +++ doc/bugs/fsck.mdwn | 1 + doc/bugs/gitrm.mdwn | 2 ++ doc/bugs/network_remotes.mdwn | 3 +++ doc/bugs/pushpull.mdwn | 2 ++ 9 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 TODO create mode 100644 doc/bugs/backendchecksum.mdwn create mode 100644 doc/bugs/dotdot_problem.mdwn create mode 100644 doc/bugs/file_copy_progress_bar.mdwn create mode 100644 doc/bugs/fsck.mdwn create mode 100644 doc/bugs/gitrm.mdwn create mode 100644 doc/bugs/network_remotes.mdwn create mode 100644 doc/bugs/pushpull.mdwn diff --git a/TODO b/TODO deleted file mode 100644 index a804597b88..0000000000 --- a/TODO +++ /dev/null @@ -1,21 +0,0 @@ -* bug: cannot "git annex ../foo" (GitRepo.relative is buggy and - git-ls-files also refuses w/o --full-name, which would need other changes) - -* --push/--pull should take a reponame and files, and push those files - to that repo; dropping them from the current repo - -* how to handle git mv file? -> git annex fix -> run automatically on commit - -* how to handle git rm file? (should try to drop keys that have no - referring file, if it seems safe..) - -* add a git annex fsck that finds keys that have no referring file - -* Support for remote git repositories (ssh:// specifically can be made to - work, although the other end probably needs to have git-annex installed..) - -* Find a way to copy a file with a progress bar, while still preserving - stat. Easiest way might be to use pv and fix up the permissions etc - after? - -* finish BackendChecksum diff --git a/debian/docs b/debian/docs index d6fa57dd7b..1936cc1d44 100644 --- a/debian/docs +++ b/debian/docs @@ -1,2 +1 @@ -doc/*.mdwn -TODO +html diff --git a/doc/bugs/backendchecksum.mdwn b/doc/bugs/backendchecksum.mdwn new file mode 100644 index 0000000000..40ff868c22 --- /dev/null +++ b/doc/bugs/backendchecksum.mdwn @@ -0,0 +1 @@ +This backend is not finished. diff --git a/doc/bugs/dotdot_problem.mdwn b/doc/bugs/dotdot_problem.mdwn new file mode 100644 index 0000000000..9d247a9c09 --- /dev/null +++ b/doc/bugs/dotdot_problem.mdwn @@ -0,0 +1,2 @@ +cannot "git annex ../foo" (GitRepo.relative is buggy and +git-ls-files also refuses w/o --full-name, which would need other changes) diff --git a/doc/bugs/file_copy_progress_bar.mdwn b/doc/bugs/file_copy_progress_bar.mdwn new file mode 100644 index 0000000000..cd4ea33b78 --- /dev/null +++ b/doc/bugs/file_copy_progress_bar.mdwn @@ -0,0 +1,3 @@ +Find a way to copy a file with a progress bar, while still preserving +stat. Easiest way might be to use pv and fix up the permissions etc +after? diff --git a/doc/bugs/fsck.mdwn b/doc/bugs/fsck.mdwn new file mode 100644 index 0000000000..308a1cb63a --- /dev/null +++ b/doc/bugs/fsck.mdwn @@ -0,0 +1 @@ +add a git annex fsck that finds keys that have no referring file diff --git a/doc/bugs/gitrm.mdwn b/doc/bugs/gitrm.mdwn new file mode 100644 index 0000000000..d771aa32a7 --- /dev/null +++ b/doc/bugs/gitrm.mdwn @@ -0,0 +1,2 @@ +how to handle git rm file? (should try to drop keys that have no +referring file, if it seems safe..) diff --git a/doc/bugs/network_remotes.mdwn b/doc/bugs/network_remotes.mdwn new file mode 100644 index 0000000000..be43ee20be --- /dev/null +++ b/doc/bugs/network_remotes.mdwn @@ -0,0 +1,3 @@ +Support for remote git repositories (ssh:// specifically can be made to +work, although the other end probably needs to have git-annex +installed..) diff --git a/doc/bugs/pushpull.mdwn b/doc/bugs/pushpull.mdwn new file mode 100644 index 0000000000..47da2107f2 --- /dev/null +++ b/doc/bugs/pushpull.mdwn @@ -0,0 +1,2 @@ +--push/--pull should take a reponame and files, and push those files + to that repo; dropping them from the current repo From b1d22b3229a1cc86e0df8d4899b44d3e44e960d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:07:33 -0400 Subject: [PATCH 0229/8313] update --- doc/bugs/using_url_backend.mdwn | 9 +++++++++ doc/index.mdwn | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 doc/bugs/using_url_backend.mdwn diff --git a/doc/bugs/using_url_backend.mdwn b/doc/bugs/using_url_backend.mdwn new file mode 100644 index 0000000000..a0d447c6e2 --- /dev/null +++ b/doc/bugs/using_url_backend.mdwn @@ -0,0 +1,9 @@ +There is no way to `git annex add` a file using the URL [[backend|backends]]. + +For now, we have to manually make the symlink. Something like this: + + ln -s .git/annex/URL:http:%%www.example.com%foo.tar.gz + +Note the escaping of slashes. + +A `git annex register ` command could do this.. diff --git a/doc/index.mdwn b/doc/index.mdwn index df42eabc11..285f84f132 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -18,7 +18,7 @@ revision control. ## documentation -* [[man page|git-annex]] +* [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidential data loss by [[tracking copies|copies]] From d6911f57b76a6469484076dc991adb39d8d9b8e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:17:29 -0400 Subject: [PATCH 0230/8313] update --- Makefile | 2 +- debian/control | 2 +- debian/copyright | 5 + debian/manpages | 1 + doc/GPL | 339 +++++++++++++++++++++++++++++++++++++++++++++ doc/git-annex.mdwn | 4 +- doc/index.mdwn | 1 + 7 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 debian/copyright create mode 100644 debian/manpages create mode 100644 doc/GPL diff --git a/Makefile b/Makefile index f18f076da9..39f5ba8ad7 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ endif docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ - --no-usedirs + --no-usedirs --disable-plugin=openid clean: rm -rf build git-annex git-annex.1 diff --git a/debian/control b/debian/control index 846844c32a..4e3ad01bdb 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,7 @@ Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-de Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex -Homepage: http://kitenet.net/~joey/code/git-annex/ +Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000000..5d0ae13c87 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,5 @@ +Files: * +Copyright: © 2010 Joey Hess +License: GPL-2+ + The full text of the GPL is distributed as doc/GPL in this package's + source, or in /usr/share/common-licenses/GPL on Debian systems. diff --git a/debian/manpages b/debian/manpages new file mode 100644 index 0000000000..ca34203aa0 --- /dev/null +++ b/debian/manpages @@ -0,0 +1 @@ +git-annex.1 diff --git a/doc/GPL b/doc/GPL new file mode 100644 index 0000000000..d159169d10 --- /dev/null +++ b/doc/GPL @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 25cf6f776c..09b245497c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -87,7 +87,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. -* init "description" +* init description Initializes git-annex with a descripotion of the git repository. This is an optional, but recommended step. @@ -143,4 +143,6 @@ to avoid conflicts when merging files in the `.git-annex` directory. Joey Hess + + Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD). Edit with care diff --git a/doc/index.mdwn b/doc/index.mdwn index 285f84f132..3541847fa4 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -23,6 +23,7 @@ revision control. * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidential data loss by [[tracking copies|copies]] of your files +* git-annex is Free Software, licensed under the [[GPL]]. ---- From d53519fade3baf9dfb3a92adcfa3c03d852171d2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:30:17 -0400 Subject: [PATCH 0231/8313] probably won't use SHA module --- Backend/SHA1.hs | 2 -- debian/control | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 2143a6af59..caece6b78c 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -3,8 +3,6 @@ module Backend.SHA1 (backend) where -import Data.Digest.Pure.SHA - import qualified Backend.File import TypeInternals diff --git a/debian/control b/debian/control index 4e3ad01bdb..83bc8c82bf 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-sha-dev, ikiwiki +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, ikiwiki Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From 0f153765b7552054cb459730b95477f8f4f1ae21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:32:40 -0400 Subject: [PATCH 0232/8313] update --- INSTALL | 8 +------- doc/download.mdwn | 2 ++ doc/index.mdwn | 1 + 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/INSTALL b/INSTALL index a7fc7f6f31..f024541c1a 100644 --- a/INSTALL +++ b/INSTALL @@ -1,7 +1 @@ -To build and use git-annex, you will need: - -* ghc -* These haskell libraries: MissingH SHA -* uuid - -Then just run make; make install +See doc/install.mdwn for installation instructions. diff --git a/doc/download.mdwn b/doc/download.mdwn index 2ceb73193a..664f46ed95 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -3,3 +3,5 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` There are no binary packages yet, but you can build Debian packages from the source tree with `dpkg-buildpackage`. + +Next: [[install]] diff --git a/doc/index.mdwn b/doc/index.mdwn index 3541847fa4..e30326853e 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -12,6 +12,7 @@ etc that are associated with annexed files but that benefit from full revision control. * **[[download]]** +* [[install]] * [[news]] * [[bugs]] * [[contact]] From e7572f9249f0e4c3f757bb8da889a41f53fd9e34 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 16:32:50 -0400 Subject: [PATCH 0233/8313] add --- doc/install.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/install.mdwn diff --git a/doc/install.mdwn b/doc/install.mdwn new file mode 100644 index 0000000000..cc6fb6fb35 --- /dev/null +++ b/doc/install.mdwn @@ -0,0 +1,7 @@ +To build and use git-annex, you will need: + +* The Haskell Platform: +* MissingH: +* uuid: + +Then just [[download]] git-annex and run: `make; make install` From 9d5b8ebab0e247c5c9c05a5216dcd4c638299190 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:28:29 -0400 Subject: [PATCH 0234/8313] update --- Makefile | 3 ++- doc/bugs.mdwn | 2 +- doc/bugs/done.mdwn | 3 ++- doc/index.mdwn | 18 ++++++++++++++++++ doc/not.mdwn | 10 ++++++++++ doc/templates/bare.tmpl | 1 + doc/use_case/Alice.mdwn | 18 ++++++++++++++++++ doc/use_case/Bob.mdwn | 18 ++++++++++++++++++ 8 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 doc/not.mdwn create mode 100644 doc/templates/bare.tmpl create mode 100644 doc/use_case/Alice.mdwn create mode 100644 doc/use_case/Bob.mdwn diff --git a/Makefile b/Makefile index 39f5ba8ad7..d35e82ad55 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ endif docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ - --no-usedirs --disable-plugin=openid + --no-usedirs --disable-plugin=openid --plugin=sidebar \ + --underlaydir=/dev/null clean: rm -rf build git-annex git-annex.1 diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn index dd2a9b4035..2786e5bf74 100644 --- a/doc/bugs.mdwn +++ b/doc/bugs.mdwn @@ -1,4 +1,4 @@ This is git-annex's bug list. Link bugs to [[bugs/done]] when done. [[!inline pages="./bugs/* and !./bugs/done and !link(done) -and !*/Discussion" actions=yes postform=yes show=0]] +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] diff --git a/doc/bugs/done.mdwn b/doc/bugs/done.mdwn index ad332e2a26..a35d427198 100644 --- a/doc/bugs/done.mdwn +++ b/doc/bugs/done.mdwn @@ -1,3 +1,4 @@ recently fixed [[bugs]] -[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10]] +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10 +archive=yes]] diff --git a/doc/index.mdwn b/doc/index.mdwn index e30326853e..b3a871627f 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -11,11 +11,28 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. +[[!sidebar content=""" * **[[download]]** * [[install]] * [[news]] * [[bugs]] * [[contact]] +"""]] + + +## sample use cases + + + + + + +
[[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]]
+ +If that describes you, or if you're some from column A and some from column +B, then git-annex may be the tool you've been looking for to expand from +keeping all your small important files in git, to managing your large +files with git. ## documentation @@ -24,6 +41,7 @@ revision control. * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidential data loss by [[tracking copies|copies]] of your files +* [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. ---- diff --git a/doc/not.mdwn b/doc/not.mdwn new file mode 100644 index 0000000000..2697a9b1fd --- /dev/null +++ b/doc/not.mdwn @@ -0,0 +1,10 @@ +[[!meta title="what git-annex is not"]] + +* git-annex is not a backup system. It may be a useful component of an + [[archival|use_case/bob]] system, or a way to deliver files to a backup + system. + + For a backup system that uses git, take a look at + [bup](http://github.com/apenwarr/bup). + +* probably several other things.. diff --git a/doc/templates/bare.tmpl b/doc/templates/bare.tmpl new file mode 100644 index 0000000000..2d476b716f --- /dev/null +++ b/doc/templates/bare.tmpl @@ -0,0 +1 @@ + diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn new file mode 100644 index 0000000000..c42eb3a74c --- /dev/null +++ b/doc/use_case/Alice.mdwn @@ -0,0 +1,18 @@ +### The Nomad + +Alice is always on the move, often with her trusty netbook and a small +handheld terabyte USB drive, or a smaller USB keydrive. She has a server +out there on the net. All these things can have different files on them, +but Alice no longer has to deal with the tedious process of keeping them +manually in sync. + +When she has 1 bar on her cell, Alice queues up interesting files on her +server for later. At a coffee shop, she has git-annex download them to her +USB drive. High in the sky or in a remote cabin, she catches up on +podcasts, videos, and games, first letting git-annex copy them from +her USB drive to the netbook (this saves battery power). + +When she's done, she tells git-annex which to keep and which to remove. +They're all removed from her netbook to save space, and Alice knowns +that next time she syncs up to the net, her changes will be synced back +to her server. diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn new file mode 100644 index 0000000000..a5dc01b373 --- /dev/null +++ b/doc/use_case/Bob.mdwn @@ -0,0 +1,18 @@ +### The Archivist + +Bob has many drives to archive his data, most of them kept offline, in a +safe place. + +With git-annex, Bob has a single directory tree that includes all +his files, even if their content is being stored offline. He can +reorganize his files using that tree, committing new versions to git, +without worry about accidentially deleting anything. + +When Bob needs access to some files, git-annex can tell him which drive(s) +they're on, and easily make them available. Indeed, every drive knows what +is on every other drive. + +Run in a cron job, git-annex adds new files to achival drives at night. It +also helps Bob keep track of intentional, and unintentional copies of +files, and logs information he can use to decide when it's time to duplicate +the content of old drives. From b08b45815dffeadd5f5fb2492bb4e5c36b921aee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:30:08 -0400 Subject: [PATCH 0235/8313] update --- doc/not.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index 2697a9b1fd..a91a6a7a04 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -7,4 +7,8 @@ For a backup system that uses git, take a look at [bup](http://github.com/apenwarr/bup). +* git-annex is not unison, but if you're finding unison's checksumming + too slow, or its strict mirroring of everything to both places too + limiting, then git-annex could be a useful alternative. + * probably several other things.. From 0b84d02b8427a7bb2fb2da5e9abf606f0e67dca9 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 19 Oct 2010 23:35:13 +0000 Subject: [PATCH 0236/8313] initial commit --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..b84c806686 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.ikiwiki +/recentchanges From 972639d85c663855dd0c7476b732dcb319efdb2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:35:56 -0400 Subject: [PATCH 0237/8313] update --- doc/not.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index a91a6a7a04..affcb57f11 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -11,4 +11,6 @@ too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. -* probably several other things.. +* git-annex is not some flaky script that was quickly thrown together. + I wrote it in Haskell because I wanted it to be solid and to compile + down to a binary. From 54db046c5ae2e7fce73fcd5ce4da278b5f8b445c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 19:38:14 -0400 Subject: [PATCH 0238/8313] clean --- .gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index b84c806686..0000000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.ikiwiki -/recentchanges From cdfea4debc86d49fb4003fc4b27e76d056d318fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:07:50 -0400 Subject: [PATCH 0239/8313] add walkthrough --- doc/index.mdwn | 3 +- doc/walkthrough.mdwn | 112 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 doc/walkthrough.mdwn diff --git a/doc/index.mdwn b/doc/index.mdwn index b3a871627f..de5fe55a3c 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -11,6 +11,8 @@ versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. +To get a feel for it, see the [[walkthrough]]. + [[!sidebar content=""" * **[[download]]** * [[install]] @@ -19,7 +21,6 @@ revision control. * [[contact]] """]] - ## sample use cases diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn new file mode 100644 index 0000000000..056361e13b --- /dev/null +++ b/doc/walkthrough.mdwn @@ -0,0 +1,112 @@ +## creating a repository + +This is very straightforward. Just tell it a description of the repository. + + # mkdir ~/annex + # cd ~/annex + # git init + # git annex init "my laptop" + +## adding a remote + +This could be a USB drive, or a sshfs or NFS mount to a file server, for +example. + + # sudo mount /media/usb + # cd /media/usb + # git clone ~/annex + # cd annex + # git annex init "portable USB drive" + # git remote add home ~/annex + # cd ~/annex + # git remote add usbdrive /media/usb + +There was nothing git-annex specific about that, except telling it the name +of the new repository created on the USB drive. + +Notice that both repos are set up as remotes of the other one. This lets +either get annexed files from the other. + +## adding files + + # cd ~/annex + # cp /tmp/big_file . + # cp /tmp/debian.iso . + # git annex add . + add big_file ok + add debian.iso ok + # git commit -a -m added + +Notice you commit at the end, this checks in git-annex's record of the +files but not thier actual, large, content. + +## transferring files around + +Let's copy everything in the laptop's home annex to the USB drive. + + # cd /media/usb/annex + # git pull home master + # git annex get . + get big_file (copying from home...) ok + get debian.iso (copying from home...) ok + +Notice that you had to git pull from home first, this lets git-annex know +what has changed in home, and so it knows about the files you added and +can get them. + +## transferring files: When things go wrong + +After a while, you'll have serveral annexes, with different file contents. +You don't have to try to keep all that straight; git-annex does +[[location_tracking] for you. If you ask it to get a file and the drive +or file server is not accessible, it will let you know what it needs to get +it: + + # git annex get video/hackity_hack_and_kaxxt.mov + get video/_why_hackity_hack_and_kaxxt.mov (not available) + I was unable to access these remotes: server + Try making some of these repositories available: + 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + failed + # sudo mount /media/usb + # git annex get video/hackity_hack_and_kaxxt.mov + get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok + # git commit -a -m "got a video I want to rewatch on the plane" + +## removing files + +You can always drop files safely. Git-annex checks that some other annex +has the file before removing it. + + # git annex drop debian.iso + drop iso/Debian_5.0.iso ok + # git commit -a -m "freed up space" + +## removing files: When things go wrong + +Before dropping a file, git-annex wants to be able to look at other +remotes, and verify that they still have a file. After all, it could +have been dropped from them too. If the remotes are not mounted/available, +you'll see something like this. + + # git annex drop important_file other.iso + drop important_file (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + I was unable to access these remotes: usbdrive + Try making some of these repositories available: + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + (Use --force to override this check, or adjust annex.numcopies.) + failed + drop other.iso (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + No other repository is known to contain the file. + (Use --force to override this check, or adjust annex.numcopies.) + failed + +Here you might --force it to drop `important_file` if you trust your backup. +But `other.iso` looks to have never been copied to anywhere else, so if +it's something you want to hold onto, you'd need to transfer it to +some other repository before dropping it. From bbac7178e8798751d0c2a2b8ee2f4f119aaece6c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:10:45 -0400 Subject: [PATCH 0240/8313] on bare --- doc/walkthrough.mdwn | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 056361e13b..24c3d71368 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -21,11 +21,16 @@ example. # cd ~/annex # git remote add usbdrive /media/usb -There was nothing git-annex specific about that, except telling it the name +This is all standard ad-hoc distributed git repository setup. Or you +could have added a centralized bare repository on a server if you prefer +doing things that way. + +Anyway, the only git-annex specific part is telling it the name of the new repository created on the USB drive. Notice that both repos are set up as remotes of the other one. This lets -either get annexed files from the other. +either get annexed files from the other. You'll want to do that even +if you are using a centralized bare repository. ## adding files From a7cc89f1eb96add4ad18812f47721e9e6c652580 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:19:52 -0400 Subject: [PATCH 0241/8313] fix --- doc/walkthrough.mdwn | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 24c3d71368..e5cbc7df19 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -45,6 +45,21 @@ if you are using a centralized bare repository. Notice you commit at the end, this checks in git-annex's record of the files but not thier actual, large, content. +## renaming files + + # cd ~/annex + # git mv big_file my_cool_big_file + # mkdir iso + # git mv debian.iso iso + # git annex fix . + fix iso/debian.iso ok + # git commit -m moved + +You can use any normal git operations to move files around, or even +make copies or delete them. `git-annex fix` needs to be run if a file +is moved into a different directory, in order to fix up the symlink +pointing to the file's content. + ## transferring files around Let's copy everything in the laptop's home annex to the USB drive. From 14dd4dc316d8d30b972f51b72459ca6e63f44f39 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:22:37 -0400 Subject: [PATCH 0242/8313] tweak --- doc/walkthrough.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index e5cbc7df19..fb824c995a 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -67,8 +67,8 @@ Let's copy everything in the laptop's home annex to the USB drive. # cd /media/usb/annex # git pull home master # git annex get . - get big_file (copying from home...) ok - get debian.iso (copying from home...) ok + get my_cool_big_file (copying from home...) ok + get iso/debian.iso (copying from home...) ok Notice that you had to git pull from home first, this lets git-annex know what has changed in home, and so it knows about the files you added and @@ -100,7 +100,7 @@ it: You can always drop files safely. Git-annex checks that some other annex has the file before removing it. - # git annex drop debian.iso + # git annex drop iso/debian.iso drop iso/Debian_5.0.iso ok # git commit -a -m "freed up space" From 277b0f0de10d0ca10007c61255eb5a8486e702f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:27:07 -0400 Subject: [PATCH 0243/8313] add --- doc/bugs/{backendchecksum.mdwn => backendSHA1.mdwn} | 0 doc/news/prerelease.mdwn | 9 +++++++++ 2 files changed, 9 insertions(+) rename doc/bugs/{backendchecksum.mdwn => backendSHA1.mdwn} (100%) create mode 100644 doc/news/prerelease.mdwn diff --git a/doc/bugs/backendchecksum.mdwn b/doc/bugs/backendSHA1.mdwn similarity index 100% rename from doc/bugs/backendchecksum.mdwn rename to doc/bugs/backendSHA1.mdwn diff --git a/doc/news/prerelease.mdwn b/doc/news/prerelease.mdwn new file mode 100644 index 0000000000..3ecdf01241 --- /dev/null +++ b/doc/news/prerelease.mdwn @@ -0,0 +1,9 @@ +git-annex is not yet formally released. If you [[download]] it from git, +everything described on this web site should already work, and I hope work +well, with these exceptions: + +* The SHA1 backend is incomplete. + +I reserve the right to change its data formats before the first release, +though I already have enough drives using it that I'd probably have to +write a transition tool anyway! From a069ed164fa9da63f62d0b4f63bde61b9117630f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:29:38 -0400 Subject: [PATCH 0244/8313] sign --- doc/news/prerelease.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/news/prerelease.mdwn b/doc/news/prerelease.mdwn index 3ecdf01241..3a1227ff0a 100644 --- a/doc/news/prerelease.mdwn +++ b/doc/news/prerelease.mdwn @@ -6,4 +6,4 @@ well, with these exceptions: I reserve the right to change its data formats before the first release, though I already have enough drives using it that I'd probably have to -write a transition tool anyway! +write a transition tool anyway! --[[Joey]] From bed33d540ca43a616fca491ed0f197aefcb0edb2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 20:36:08 -0400 Subject: [PATCH 0245/8313] fix --- doc/copies.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index ff66f4e8ae..aec10ab7af 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -1,4 +1,4 @@ -The WORM and SHA1 key-value [[backends|backend]] store data inside +The WORM and SHA1 key-value [[backends]] store data inside your git repository's `.git` directory, not in some external data store. It's important that data not get lost by an ill-considered `git annex drop` From b79a53eb242f6b46772457e893fe2b95684ed111 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Oct 2010 21:25:59 -0400 Subject: [PATCH 0246/8313] forgot git :) --- doc/install.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/install.mdwn b/doc/install.mdwn index cc6fb6fb35..e217799a32 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -1,5 +1,6 @@ To build and use git-annex, you will need: +* git: * The Haskell Platform: * MissingH: * uuid: From ee10027a246aa9c5fcd9503314242c65734e8d8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 11:56:36 -0400 Subject: [PATCH 0247/8313] 3 --- debian/copyright | 2 +- doc/GPL | 833 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 585 insertions(+), 250 deletions(-) diff --git a/debian/copyright b/debian/copyright index 5d0ae13c87..55638653c1 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,5 @@ Files: * Copyright: © 2010 Joey Hess -License: GPL-2+ +License: GPL-3+ The full text of the GPL is distributed as doc/GPL in this package's source, or in /usr/share/common-licenses/GPL on Debian systems. diff --git a/doc/GPL b/doc/GPL index d159169d10..94a9ed024d 100644 --- a/doc/GPL +++ b/doc/GPL @@ -1,281 +1,622 @@ GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + Version 3, 29 June 2007 - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + TERMS AND CONDITIONS - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". + 0. Definitions. -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. + "This License" refers to version 3 of the GNU General Public License. - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. + A "covered work" means either the unmodified Program or a work based +on the Program. - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. + 1. Source Code. -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. + The Corresponding Source for a work in source code form is that +same work. -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. + 2. Basic Permissions. - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of this License. - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. + 13. Use with the GNU Affero General Public License. -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. + 14. Revised Versions of this License. - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. - NO WARRANTY + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. + 15. Disclaimer of Warranty. - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. END OF TERMS AND CONDITIONS @@ -287,15 +628,15 @@ free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least +state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) - This program is free software; you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -303,37 +644,31 @@ the "copyright" line and a pointer to where the full notice is found. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + You should have received a copy of the GNU General Public License + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From d42f8a9f392e92a8145fb9d68aad4d2028a6210b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 12:07:24 -0400 Subject: [PATCH 0248/8313] autocommit on init --- Commands.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Commands.hs b/Commands.hs index 2addf714e8..1cc046c03a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -220,6 +220,7 @@ initCmd description = do describeUUID u description log <- uuidLog liftIO $ Git.run g ["add", log] + liftIO $ Git.run g ["commit", "-m", "git annex init", log] liftIO $ putStrLn "description set" -- helpers From 4e3f4e9b0528e1b7f087ffc56ec9da0e88a1cc36 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 12:16:49 -0400 Subject: [PATCH 0249/8313] typo --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index fb824c995a..0da11a8b31 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -78,7 +78,7 @@ can get them. After a while, you'll have serveral annexes, with different file contents. You don't have to try to keep all that straight; git-annex does -[[location_tracking] for you. If you ask it to get a file and the drive +[[location_tracking]] for you. If you ask it to get a file and the drive or file server is not accessible, it will let you know what it needs to get it: From a68e36f518589bd15fea32da273ad6fd2f288bb5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Oct 2010 12:54:40 -0400 Subject: [PATCH 0250/8313] entry for the prerelease --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 998754777e..85d105e1b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -git-annex (0.01) UNRELEASED; urgency=low +git-annex (0.01) unstable; urgency=low - * First release + * First prerelease. - -- Joey Hess Thu, 09 Sep 2010 08:24:58 -0400 + -- Joey Hess Wed, 20 Oct 2010 12:54:24 -0400 From 19fde4960dc1d6c8c05efd0f5b4293c2fb52ebf9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:30:16 -0400 Subject: [PATCH 0251/8313] new fromkey subcommand, for registering urls, etc had to redo Annex monad's flag storage --- Annex.hs | 27 ++++++---- Backend.hs | 19 ++++--- Backend/File.hs | 2 +- Commands.hs | 93 ++++++++++++++++++++++++++------- Core.hs | 5 +- Makefile | 2 +- Remotes.hs | 4 +- TypeInternals.hs | 19 ++++--- Types.hs | 5 +- doc/bugs/using_url_backend.mdwn | 2 + doc/git-annex.mdwn | 21 +++++++- doc/walkthrough.mdwn | 31 +++++++++++ git-annex.hs | 4 +- 13 files changed, 179 insertions(+), 55 deletions(-) diff --git a/Annex.hs b/Annex.hs index b68e513553..d021f936ef 100644 --- a/Annex.hs +++ b/Annex.hs @@ -10,10 +10,12 @@ module Annex ( supportedBackends, flagIsSet, flagChange, + flagGet, Flag(..) ) where import Control.Monad.State +import qualified Data.Map as M import qualified GitRepo as Git import Types @@ -27,7 +29,7 @@ new gitrepo allbackends = do Internals.repo = gitrepo, Internals.backends = [], Internals.supportedBackends = allbackends, - Internals.flags = [] + Internals.flags = M.empty } (_,s') <- Annex.run s (prep gitrepo) return s' @@ -63,15 +65,20 @@ supportedBackends :: Annex [Backend] supportedBackends = do state <- get return (Internals.supportedBackends state) -flagIsSet :: Flag -> Annex Bool -flagIsSet flag = do +flagIsSet :: FlagName -> Annex Bool +flagIsSet name = do state <- get - return $ elem flag $ Internals.flags state -flagChange :: Flag -> Bool -> Annex () -flagChange flag set = do + case (M.lookup name $ Internals.flags state) of + Just (FlagBool True) -> return True + _ -> return False +flagChange :: FlagName -> Flag -> Annex () +flagChange name val = do state <- get - let f = filter (/= flag) $ Internals.flags state - if (set) - then put state { Internals.flags = (flag:f) } - else put state { Internals.flags = f } + put state { Internals.flags = M.insert name val $ Internals.flags state } return () +flagGet :: FlagName -> Annex String +flagGet name = do + state <- get + case (M.lookup name $ Internals.flags state) of + Just (FlagString s) -> return s + _ -> return "" diff --git a/Backend.hs b/Backend.hs index a427234d7c..b8def21cd8 100644 --- a/Backend.hs +++ b/Backend.hs @@ -14,6 +14,7 @@ - -} module Backend ( + list, storeFileKey, retrieveKeyFile, removeKey, @@ -36,24 +37,28 @@ import Types import qualified TypeInternals as Internals {- List of backends in the order to try them when storing a new key. -} -backendList :: Annex [Backend] -backendList = do - l <- Annex.backends +list :: Annex [Backend] +list = do + l <- Annex.backends -- list is cached here if (0 < length l) then return l else do all <- Annex.supportedBackends g <- Annex.gitRepo let l = parseBackendList all $ Git.configGet g "annex.backends" "" - Annex.backendsChange l - return l + backendflag <- Annex.flagGet "backend" + let l' = if (0 < length backendflag) + then (lookupBackendName all backendflag):l + else l + Annex.backendsChange $ l' + return l' where parseBackendList all s = if (length s == 0) then all else map (lookupBackendName all) $ words s -{- Looks up a backend in the list of supportedBackends -} +{- Looks up a backend in a list -} lookupBackendName :: [Backend] -> String -> Backend lookupBackendName all s = if ((length matches) /= 1) @@ -66,7 +71,7 @@ storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) storeFileKey file = do g <- Annex.gitRepo let relfile = Git.relative g file - b <- backendList + b <- list storeFileKey' b file relfile storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do diff --git a/Backend/File.hs b/Backend/File.hs index 4ea25daa73..d1ed1ec64e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -107,7 +107,7 @@ showTriedRemotes remotes = - error if not. -} checkRemoveKey :: Key -> Annex (Bool) checkRemoveKey key = do - force <- Annex.flagIsSet Force + force <- Annex.flagIsSet "force" if (force) then return True else do diff --git a/Commands.hs b/Commands.hs index 1cc046c03a..59915f59c2 100644 --- a/Commands.hs +++ b/Commands.hs @@ -8,6 +8,7 @@ import System.Posix.Files import System.Directory import System.Path import Data.String.Utils +import Control.Monad (filterM) import List import IO @@ -23,7 +24,8 @@ import Core import qualified Remotes import qualified TypeInternals -data CmdWants = FilesInGit | FilesNotInGit | RepoName | SingleString +data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | + RepoName | Description data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -41,26 +43,49 @@ cmds = [ "indicate content of files not currently wanted") , (Command "unannex" unannexCmd FilesInGit "undo accidential add command") - , (Command "init" initCmd SingleString + , (Command "init" initCmd Description "initialize git-annex with repository description") , (Command "fix" fixCmd FilesInGit "fix up files' symlinks to point to annexed content") + , (Command "fromkey" fromKeyCmd FilesMissing + "adds a file using a specific key") ] +-- Each dashed command-line option results in generation of an action +-- in the Annex monad that performs the necessary setting. +options :: [OptDescr (Annex ())] options = [ - Option ['f'] ["force"] (NoArg Force) "allow actions that may lose annexed data" + Option ['f'] ["force"] + (NoArg (Annex.flagChange "force" $ FlagBool True)) + "allow actions that may lose annexed data" + , Option ['b'] ["backend"] + (ReqArg (\s -> Annex.flagChange "backend" $ FlagString s) "NAME") + "specify default key-value backend to use" + , Option ['k'] ["key"] + (ReqArg (\s -> Annex.flagChange "key" $ FlagString s) "KEY") + "specify a key to use" ] -header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) ++ " [path ...]" +header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) +usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where cmddescs = unlines $ map (\c -> indent $ showcmd c) cmds showcmd c = (cmdname c) ++ - (take (10 - (length (cmdname c))) $ repeat ' ') ++ + (pad 10 (cmdname c)) ++ + (descWanted (cmdwants c)) ++ + (pad 13 (descWanted (cmdwants c))) ++ (cmddesc c) indent l = " " ++ l + pad n s = take (n - (length s)) $ repeat ' ' + +{- Generate descrioptions of wanted parameters for subcommands. -} +descWanted :: CmdWants -> String +descWanted Description = "DESCRIPTION" +descWanted RepoName = "REPO" +descWanted _ = "PATH ..." {- Finds the type of parameters a command wants, from among the passed - parameter list. -} @@ -71,14 +96,23 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files -findWanted SingleString params _ = do +findWanted FilesMissing params repo = do + files <- liftIO $ filterM missing params + return $ files + where + missing f = do + e <- doesFileExist f + if (e) then return False else return True +findWanted Description params _ = do return $ [unwords params] findWanted RepoName params _ = do return $ params -{- Parses command line and returns a list of flags and a list of - - actions to be run in the Annex monad. -} -parseCmd :: [String] -> AnnexState -> IO ([Flag], [Annex ()]) +{- Parses command line and returns two lists of actions to be + - run in the Annex monad. The first actions configure it + - according to command line options, while the second actions + - handle subcommands. -} +parseCmd :: [String] -> AnnexState -> IO ([Annex ()], [Annex ()]) parseCmd argv state = do (flags, params) <- getopt case (length params) of @@ -100,7 +134,7 @@ parseCmd argv state = do {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () -addCmd file = inBackend file $ do +addCmd file = notInBackend file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return () @@ -125,9 +159,9 @@ addCmd file = inBackend file $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () -unannexCmd file = notinBackend file $ \(key, backend) -> do +unannexCmd file = inBackend file $ \(key, backend) -> do showStart "unannex" file - Annex.flagChange Force True -- force backend to always remove + Annex.flagChange "force" $ FlagBool True -- force backend to always remove Backend.removeKey backend key logStatus key ValueMissing g <- Annex.gitRepo @@ -145,7 +179,7 @@ unannexCmd file = notinBackend file $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () -getCmd file = notinBackend file $ \(key, backend) -> do +getCmd file = inBackend file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return () @@ -167,7 +201,7 @@ getCmd file = notinBackend file $ \(key, backend) -> do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () -dropCmd file = notinBackend file $ \(key, backend) -> do +dropCmd file = inBackend file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return () -- no-op @@ -192,8 +226,8 @@ dropCmd file = notinBackend file $ \(key, backend) -> do else return () {- Fixes the symlink to an annexed file. -} -fixCmd :: String -> Annex () -fixCmd file = notinBackend file $ \(key, backend) -> do +fixCmd :: FilePath -> Annex () +fixCmd file = inBackend file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -223,13 +257,36 @@ initCmd description = do liftIO $ Git.run g ["commit", "-m", "git annex init", log] liftIO $ putStrLn "description set" +{- Adds a file pointing at a manually-specified key -} +fromKeyCmd :: FilePath -> Annex () +fromKeyCmd file = do + keyname <- Annex.flagGet "key" + if (0 == length keyname) + then error "please specify the key with --key" + else return () + backends <- Backend.list + let key = genKey (backends !! 0) keyname + + inbackend <- Backend.hasKey key + if (not inbackend) + then error $ "key ("++keyname++") is not present in backend" + else return () + + link <- calcGitLink file key + showStart "fromkey" file + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ createSymbolicLink link file + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] + showEndOk + -- helpers -inBackend file a = do +notInBackend file a = do r <- Backend.lookupFile file case (r) of Just v -> return () Nothing -> a -notinBackend file a = do +inBackend file a = do r <- Backend.lookupFile file case (r) of Just v -> a v diff --git a/Core.hs b/Core.hs index 8dc4bff6fc..4941dc26bf 100644 --- a/Core.hs +++ b/Core.hs @@ -18,9 +18,8 @@ import qualified Annex import Utility {- Sets up a git repo for git-annex. -} -startup :: [Flag] -> Annex () -startup flags = do - mapM (\f -> Annex.flagChange f True) flags +startup :: Annex () +startup = do g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID diff --git a/Makefile b/Makefile index d35e82ad55..5f8bb50122 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ - --underlaydir=/dev/null + --underlaydir=/dev/null --disable-plugin=shortcut clean: rm -rf build git-annex git-annex.1 diff --git a/Remotes.hs b/Remotes.hs index a0894f418b..07aafe51b7 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -36,7 +36,7 @@ withKey key = do -- mounted at their location). So unless it happens to find all -- remotes, try harder, loading the remotes' configs. remotes <- reposByUUID allremotes uuids - remotesread <- Annex.flagIsSet RemotesRead + remotesread <- Annex.flagIsSet "remotesread" if ((length allremotes /= length remotes) && not remotesread) then tryharder allremotes uuids else return remotes @@ -46,7 +46,7 @@ withKey key = do eitherremotes <- mapM tryGitConfigRead allremotes let allremotes' = map fromEither eitherremotes remotes' <- reposByUUID allremotes' uuids - Annex.flagChange RemotesRead True + Annex.flagChange "remotesread" $ FlagBool True return remotes' {- Cost Ordered list of remotes. -} diff --git a/TypeInternals.hs b/TypeInternals.hs index 4a9d2653e1..6d1c72d2ea 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -7,12 +7,15 @@ module TypeInternals where import Control.Monad.State (StateT) import Data.String.Utils +import qualified Data.Map as M import qualified GitRepo as Git -data Flag = - Force | -- command-line flags - RemotesRead -- indicates that remote repo configs have been read +-- command-line flags +type FlagName = String +data Flag = + FlagBool Bool | + FlagString String deriving (Eq, Read, Show) -- git-annex's runtime state type doesn't really belong here, @@ -21,7 +24,7 @@ data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend], supportedBackends :: [Backend], - flags :: [Flag] + flags :: M.Map FlagName Flag } deriving (Show) -- git-annex's monad @@ -32,6 +35,10 @@ type KeyFrag = String type BackendName = String data Key = Key (BackendName, KeyFrag) deriving (Eq) +-- constructs a key in a backend +genKey :: Backend -> KeyFrag -> Key +genKey b f = Key (name b,f) + -- show a key to convert it to a string; the string includes the -- name of the backend to avoid collisions between key strings instance Show Key where @@ -48,10 +55,6 @@ instance Read Key where backendName :: Key -> BackendName backendName (Key (b,k)) = b --- pulls the key fragment out -keyFrag :: Key -> KeyFrag -keyFrag (Key (b,k)) = k - -- this structure represents a key-value backend data Backend = Backend { -- name of this backend diff --git a/Types.hs b/Types.hs index 2284d92674..50597962ce 100644 --- a/Types.hs +++ b/Types.hs @@ -5,9 +5,10 @@ module Types ( AnnexState, Backend, Key, + genKey, backendName, - keyFrag, - Flag(..), + FlagName, + Flag(..) ) where import TypeInternals diff --git a/doc/bugs/using_url_backend.mdwn b/doc/bugs/using_url_backend.mdwn index a0d447c6e2..1f3cd56281 100644 --- a/doc/bugs/using_url_backend.mdwn +++ b/doc/bugs/using_url_backend.mdwn @@ -7,3 +7,5 @@ For now, we have to manually make the symlink. Something like this: Note the escaping of slashes. A `git annex register ` command could do this.. + +[[done]] diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 09b245497c..81c229c513 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -4,7 +4,7 @@ git-annex - manage files with git, without checking their contents in # SYNOPSIS -git annex subcommand [path ...] +git annex subcommand [params ...] # DESCRIPTION @@ -97,6 +97,16 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Fixes up symlinks that have become broken to again point to annexed content. This is useful to run if you have been moving the symlinks around. +* fromkey file + + This can be used to maually set up a file to link to a specified key + in the key-value backend. How you determine an existing key in the backend + varies. For the URL backend, the key is just a URL to the content. + + Example: + + git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + # OPTIONS * --force @@ -104,6 +114,15 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Force unsafe actions, such as dropping a file's content when no other source of it can be verified to still exist. Use with care. +* --backend=name + + Specify the default key-value backend to use, adding it to the front + of the list normally configured by `annex.backends`. + +* --key=name + + Specifies a key to operate on, for use with the addkey subcommand. + ## CONFIGURATION Like other git commands, git-annex is configured via `.git/config`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 0da11a8b31..7018a839e9 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -1,3 +1,7 @@ +A walkthrough of the basic features of git-annex. + +[[!toc]] + ## creating a repository This is very straightforward. Just tell it a description of the repository. @@ -130,3 +134,30 @@ Here you might --force it to drop `important_file` if you trust your backup. But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. + +## using other backends: manually adding a remote URL + +git-annex has multiple key-value [[backends]]. So far this walkthrough has +demonstrated the default, WORM (Write Once, Read Many) backend. + +Another handy backend is the URL backend, which can fetch file's content +from remote URLs. Here's how to set up some files in your repository +that use this backend: + + # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + add somefile ok + # git commit -m "added a file from the Internet Archive" + +Now you if you ask git-annex to get that file, it will download it, +and cache it locally, until you have it drop it. + + # git annex get somefile + get somefile (downloading) + #########################################################################100.0% + ok + +You can always drop files downloaded by the URL backend. It is assumed +that the URL is stable; no local backup is kept. + + # git annex drop somefile + drop somefile (ok) diff --git a/git-annex.hs b/git-annex.hs index 71a21379dc..602f672c5b 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -15,8 +15,8 @@ main = do args <- getArgs gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends - (flags, actions) <- parseCmd args state - tryRun state $ [startup flags] ++ actions ++ [shutdown] + (configure, actions) <- parseCmd args state + tryRun state $ [startup] ++ configure ++ actions ++ [shutdown] {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). From fd5d6403f05ed346e5660ccefb7254ff062505cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:33:04 -0400 Subject: [PATCH 0252/8313] typo --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7018a839e9..e25c36fb40 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -47,7 +47,7 @@ if you are using a centralized bare repository. # git commit -a -m added Notice you commit at the end, this checks in git-annex's record of the -files but not thier actual, large, content. +files but not their actual, large, content. ## renaming files From 42eab98f56c0641467441d1d868f850a1e7ef81c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:35:04 -0400 Subject: [PATCH 0253/8313] typo --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index e25c36fb40..7114cceae2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -145,7 +145,7 @@ from remote URLs. Here's how to set up some files in your repository that use this backend: # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile - add somefile ok + fromkey somefile ok # git commit -m "added a file from the Internet Archive" Now you if you ask git-annex to get that file, it will download it, From 409285d8e18fb27d4c1723693d6ac22d67ebec08 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:36:01 -0400 Subject: [PATCH 0254/8313] clarify --- doc/walkthrough.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7114cceae2..4069fb87bf 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -135,7 +135,7 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. -## using other backends: manually adding a remote URL +## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has demonstrated the default, WORM (Write Once, Read Many) backend. @@ -149,7 +149,7 @@ that use this backend: # git commit -m "added a file from the Internet Archive" Now you if you ask git-annex to get that file, it will download it, -and cache it locally, until you have it drop it. +and cache it locally. # git annex get somefile get somefile (downloading) From e8e397036f8f04c4ef088d5a4a1b12b37b3a6118 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 16:38:14 -0400 Subject: [PATCH 0255/8313] changelog --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 85d105e1b5..1c36652302 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.02) UNRELEASED; urgency=low + + * New fromkey subcommand, for registering urls, etc. + + -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 + git-annex (0.01) unstable; urgency=low * First prerelease. From 514b98ff5574fe09ec365d5881684c16f752df6c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 17:59:32 -0400 Subject: [PATCH 0256/8313] document move subcommand --- Commands.hs | 38 +++++++++++++++++++++++++------------- doc/git-annex.mdwn | 33 ++++++++++++++++++++++++--------- doc/walkthrough.mdwn | 24 ++++++++++++++++++++---- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/Commands.hs b/Commands.hs index 59915f59c2..62eb08e83d 100644 --- a/Commands.hs +++ b/Commands.hs @@ -24,8 +24,7 @@ import Core import qualified Remotes import qualified TypeInternals -data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | - RepoName | Description +data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description data Command = Command { cmdname :: String, cmdaction :: (String -> Annex ()), @@ -41,10 +40,12 @@ cmds = [ "make content of annexed files available") , (Command "drop" dropCmd FilesInGit "indicate content of files not currently wanted") - , (Command "unannex" unannexCmd FilesInGit - "undo accidential add command") + , (Command "move" moveCmd FilesInGit + "transfer content of files to another repository") , (Command "init" initCmd Description "initialize git-annex with repository description") + , (Command "unannex" unannexCmd FilesInGit + "undo accidential add command") , (Command "fix" fixCmd FilesInGit "fix up files' symlinks to point to annexed content") , (Command "fromkey" fromKeyCmd FilesMissing @@ -55,16 +56,18 @@ cmds = [ -- in the Annex monad that performs the necessary setting. options :: [OptDescr (Annex ())] options = [ - Option ['f'] ["force"] - (NoArg (Annex.flagChange "force" $ FlagBool True)) + Option ['f'] ["force"] (NoArg (storebool "force" True)) "allow actions that may lose annexed data" - , Option ['b'] ["backend"] - (ReqArg (\s -> Annex.flagChange "backend" $ FlagString s) "NAME") + , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") "specify default key-value backend to use" - , Option ['k'] ["key"] - (ReqArg (\s -> Annex.flagChange "key" $ FlagString s) "KEY") + , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" + , Option ['r'] ["repository"] (ReqArg (storestring "repository") "REPOSITORY") + "specify a repository" ] + where + storebool n b = Annex.flagChange n $ FlagBool b + storestring n s = Annex.flagChange n $ FlagString s header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) @@ -84,7 +87,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Generate descrioptions of wanted parameters for subcommands. -} descWanted :: CmdWants -> String descWanted Description = "DESCRIPTION" -descWanted RepoName = "REPO" descWanted _ = "PATH ..." {- Finds the type of parameters a command wants, from among the passed @@ -105,8 +107,6 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] -findWanted RepoName params _ = do - return $ params {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -198,6 +198,18 @@ getCmd file = inBackend file $ \(key, backend) -> do else do showEndFail +{- Moves the content of an annexed file to another repository, + - removing it from the current repository, and updates locationlog + - information on both. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +moveCmd :: FilePath -> Annex () +moveCmd file = inBackend file $ \(key, backend) -> do + error "TODO" + {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 81c229c513..86ac9c6359 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -48,11 +48,12 @@ content from the key-value store. add iso/Debian_5.0.iso ok # git commit -a -m "saving Debian CD for later" - # git annex push usbdrive iso - error: push not yet implemented! # git annex drop iso drop iso/Debian_5.0.iso ok # git commit -a -m "freed up space" + + # git annex move video --to=usbdrive + move iso/Debian_5.0.iso (to usbdrive...) ok # SUBCOMMANDS @@ -77,8 +78,22 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Drops the content of annexed files from this repository. - git-annex may refuse to drop a content if the backend does not think - it is safe to do so. + git-annex may refuse to drop content if the backend does not think + it is safe to do so, typically because of the setting of annex.numcopies. + +* move [path ...] + + Moves the content of annexed files from the current repository to + another one. Use with the --to option. + + Note that unlike drop, this does not honor annex.numcopies. + A file's content can be moved even if there are insufficient + copies to allow it to be dropped. + +* init description + + Initializes git-annex with a descripotion of the git repository. + This is an optional, but recommended step. * unannex [path ...] @@ -87,11 +102,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. -* init description - - Initializes git-annex with a descripotion of the git repository. - This is an optional, but recommended step. - * fix [path ...] Fixes up symlinks that have become broken to again point to annexed content. @@ -123,6 +133,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a key to operate on, for use with the addkey subcommand. +* --to=repository + + Specifies a git repository that content will be sent to. + It can be specified by a path, url, or remote name. + ## CONFIGURATION Like other git commands, git-annex is configured via `.git/config`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 4069fb87bf..7dbe62bdc2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -64,9 +64,14 @@ make copies or delete them. `git-annex fix` needs to be run if a file is moved into a different directory, in order to fix up the symlink pointing to the file's content. -## transferring files around +## getting file content -Let's copy everything in the laptop's home annex to the USB drive. +A repository does not always have all annexed file contents available. +When you need the content of a file, you can use "git annex get" to +make it available. + +We can use this to copy everything in the laptop's home annex to the +USB drive. # cd /media/usb/annex # git pull home master @@ -75,8 +80,8 @@ Let's copy everything in the laptop's home annex to the USB drive. get iso/debian.iso (copying from home...) ok Notice that you had to git pull from home first, this lets git-annex know -what has changed in home, and so it knows about the files you added and -can get them. +what has changed in home, and so it knows about the files present there and +can get them. See below for an easier way. ## transferring files: When things go wrong @@ -135,6 +140,17 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. +## moving file content to another repository + +Often you will want to transfer some file contents from a repository to +some other one, and then drop it from the first repository. For example, +your laptop's disk is getting full; time to move some files to an external +disk. Doing that by hand is possible, but a bit of a pain. `git annex move` +makes it very easy. + + # git annex move my_cool_big_file --to usbdrive + move my_cool_big_file (to usbdrive...) ok + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 014f7f650de9e272628fd5031c8c9a00b1eb69ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Oct 2010 19:27:29 -0400 Subject: [PATCH 0257/8313] crazy idea --- doc/bugs/add_a_git_backend.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/bugs/add_a_git_backend.mdwn diff --git a/doc/bugs/add_a_git_backend.mdwn b/doc/bugs/add_a_git_backend.mdwn new file mode 100644 index 0000000000..91a5001cc9 --- /dev/null +++ b/doc/bugs/add_a_git_backend.mdwn @@ -0,0 +1,6 @@ +There should be a backend where the file content is stored.. in a git +repository! + +This way, you know your annexed content is safe & versioned, but you only +have to deal with the pain of git with large files in one place, and can +use all of git-annex's features everywhere else. From 8da596feff4f402fec08b6fd3815fd32e9770af6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 12:38:20 -0400 Subject: [PATCH 0258/8313] support reading config over ssh --- GitRepo.hs | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 5b0e68cd62..e8504a8410 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -56,12 +56,11 @@ data Repo = -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String } | RemoteRepo { - url :: String, - top :: FilePath, + url :: URI, config :: Map String String, remotes :: [Repo], remoteName :: Maybe String - } deriving (Show, Read, Eq) + } deriving (Show, Eq) {- Local Repo constructor. -} repoFromPath :: FilePath -> Repo @@ -77,13 +76,11 @@ repoFromPath dir = repoFromUrl :: String -> Repo repoFromUrl url = RemoteRepo { - url = url, - top = path url, + url = fromJust $ parseURI url, config = Map.empty, remotes = [], remoteName = Nothing } - where path url = uriPath $ fromJust $ parseURI url {- User-visible description of a git repo. -} repoDescribe repo = @@ -91,7 +88,7 @@ repoDescribe repo = then fromJust $ remoteName repo else if (repoIsLocal repo) then top repo - else url repo + else show (url repo) {- Constructs and returns an updated version of a repo with - different remotes list. -} @@ -105,8 +102,8 @@ repoRemoteName r = then fromJust $ remoteName r else "" -{- Some code needs to vary between remote and local repos, or bare and - - non-bare, these functions help with that. -} +{- Some code needs to vary between remote and local repos, + - or bare and non-bare, these functions help with that. -} repoIsLocal repo = case (repo) of LocalRepo {} -> True RemoteRepo {} -> False @@ -116,6 +113,10 @@ assertlocal repo action = then action else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" +assertssh repo action = + case (uriScheme $ url repo) of + "ssh:" -> action + _ -> error $ "unsupported remote repo type " ++ (show $ url repo) bare :: Repo -> Bool bare repo = if (member b (config repo)) @@ -193,16 +194,24 @@ notInRepo repo location = do {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo -configRead repo = assertlocal repo $ do +configRead repo = if (repoIsLocal repo) {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} - cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory (top repo)) - (\_ -> changeWorkingDirectory cwd) $ - pOpen ReadFromPipe "git" ["config", "--list"] $ \h -> do - val <- hGetContentsStrict h - let r = repo { config = configParse val } - return r { remotes = configRemotes r } + then do + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory (top repo)) + (\_ -> changeWorkingDirectory cwd) $ + pOpen ReadFromPipe "git" ["config", "--list"] proc + else assertssh repo $ do + pOpen ReadFromPipe "ssh" [sshhost, sshcommand] proc + where + sshhost = (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + where a = fromJust $ uriAuthority $ url repo + sshcommand = "cd '" ++ (uriPath $ url repo) ++ "' && git config --list" + proc h = do + val <- hGetContentsStrict h + let r = repo { config = configParse val } + return r { remotes = configRemotes r } {- Calculates a list of a repo's configured remotes, by parsing its config. -} configRemotes :: Repo -> [Repo] From 9f13f3ac5e7d20df91cb57af5e630fd48776d775 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 13:21:43 -0400 Subject: [PATCH 0259/8313] support using scp for ssh remotes --- Backend/File.hs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index d1ed1ec64e..d28b927287 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -73,7 +73,7 @@ copyKeyFile key file = do case (result) of Left err -> trycopy full rs Right r' -> do - showNote $ "copying from " ++ (Git.repoDescribe r ) ++ "..." + showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." liftIO $ copyFromRemote r' key file {- Tries to copy a file from a remote. -} @@ -81,11 +81,15 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool copyFromRemote r key file = do if (Git.repoIsLocal r) then getlocal - else getremote + else if (Git.repoIsSsh r) + then getssh + else error "copying from non-ssh repo not supported" where - getlocal = boolSystem "cp" ["-a", location, file] - getremote = return False -- TODO implement get from remote location = annexLocation r key + getlocal = boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + boolSystem "scp" [location, file] showLocations :: Key -> Annex () showLocations key = do @@ -130,10 +134,10 @@ checkRemoveKey key = do Right True -> findcopies need (have+1) rs bad Right False -> findcopies need have rs bad Left _ -> findcopies need have rs (r:bad) - remoteHasKey r all = do + remoteHasKey remote all = do -- To check if a remote has a key, construct a new -- Annex monad and query its backend. - a <- Annex.new r all + a <- Annex.new remote all (result, _) <- Annex.run a (Backend.hasKey key) return result notEnoughCopies need have bad = do From 897bf49b4eddba41b5ca36210ccf36f34df84e01 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 13:40:19 -0400 Subject: [PATCH 0260/8313] support ssh repo in workTree --- GitRepo.hs | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index e8504a8410..80a3722cec 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -12,6 +12,7 @@ module GitRepo ( repoFromUrl, repoIsLocal, repoIsRemote, + repoIsSsh, repoDescribe, workTree, dir, @@ -108,15 +109,21 @@ repoIsLocal repo = case (repo) of LocalRepo {} -> True RemoteRepo {} -> False repoIsRemote repo = not $ repoIsLocal repo +repoIsSsh repo = repoIsRemote repo && (uriScheme $ url repo) == "ssh:" assertlocal repo action = if (repoIsLocal repo) then action else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" -assertssh repo action = - case (uriScheme $ url repo) of - "ssh:" -> action - _ -> error $ "unsupported remote repo type " ++ (show $ url repo) +assertremote repo action = + if (repoIsRemote repo) + then action + else error $ "acting on local git repo " ++ (repoDescribe repo) ++ + " not supported" +assertssh repo action = + if (repoIsSsh repo) + then action + else error $ "unsupported url " ++ (show $ url repo) bare :: Repo -> Bool bare repo = if (member b (config repo)) @@ -135,14 +142,21 @@ attributes repo = assertlocal repo $ do {- Path to a repository's .git directory, relative to its topdir. -} dir :: Repo -> String -dir repo = assertlocal repo $ - if (bare repo) - then "" - else ".git" +dir repo = if (bare repo) then "" else ".git" {- Path to a repository's --work-tree. -} workTree :: Repo -> FilePath -workTree repo = top repo +workTree repo = + if (repoIsLocal) repo + then top repo + else assertssh repo $ (remoteHost repo) ++ ":" ++ (uriPath $ url repo) + +{- Hostname for a remote repo. (May include a username and/or port too.) -} +remoteHost :: Repo -> String +remoteHost repo = assertremote repo $ + (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + where + a = fromJust $ uriAuthority $ url repo {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. @@ -203,10 +217,8 @@ configRead repo = if (repoIsLocal repo) (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] proc else assertssh repo $ do - pOpen ReadFromPipe "ssh" [sshhost, sshcommand] proc + pOpen ReadFromPipe "ssh" [remoteHost repo, sshcommand] proc where - sshhost = (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) - where a = fromJust $ uriAuthority $ url repo sshcommand = "cd '" ++ (uriPath $ url repo) ++ "' && git config --list" proc h = do val <- hGetContentsStrict h From 46ac66a4380e238865f213ff28411283a4efbbbd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 14:05:30 -0400 Subject: [PATCH 0261/8313] fix remote vs remote wording confusion --- GitRepo.hs | 56 ++++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 80a3722cec..ea9e8a8b76 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -10,8 +10,7 @@ module GitRepo ( repoFromCwd, repoFromPath, repoFromUrl, - repoIsLocal, - repoIsRemote, + repoIsUrl, repoIsSsh, repoDescribe, workTree, @@ -46,17 +45,16 @@ import Maybe import Utility -{- A git repository can be on local disk or remote. Not to be confused - - with a git repo's configured remotes, some of which may be on local - - disk. -} +{- There are two types of repositories; those on local disk and those + - accessed via an URL. -} data Repo = - LocalRepo { + Repo { top :: FilePath, config :: Map String String, remotes :: [Repo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String - } | RemoteRepo { + } | UrlRepo { url :: URI, config :: Map String String, remotes :: [Repo], @@ -66,7 +64,7 @@ data Repo = {- Local Repo constructor. -} repoFromPath :: FilePath -> Repo repoFromPath dir = - LocalRepo { + Repo { top = dir, config = Map.empty, remotes = [], @@ -76,7 +74,7 @@ repoFromPath dir = {- Remote Repo constructor. Throws exception on invalid url. -} repoFromUrl :: String -> Repo repoFromUrl url = - RemoteRepo { + UrlRepo { url = fromJust $ parseURI url, config = Map.empty, remotes = [], @@ -87,7 +85,7 @@ repoFromUrl url = repoDescribe repo = if (isJust $ remoteName repo) then fromJust $ remoteName repo - else if (repoIsLocal repo) + else if (not $ repoIsUrl repo) then top repo else show (url repo) @@ -103,20 +101,19 @@ repoRemoteName r = then fromJust $ remoteName r else "" -{- Some code needs to vary between remote and local repos, +{- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} -repoIsLocal repo = case (repo) of - LocalRepo {} -> True - RemoteRepo {} -> False -repoIsRemote repo = not $ repoIsLocal repo -repoIsSsh repo = repoIsRemote repo && (uriScheme $ url repo) == "ssh:" -assertlocal repo action = - if (repoIsLocal repo) +repoIsUrl repo = case (repo) of + UrlRepo {} -> True + Repo {} -> False +repoIsSsh repo = repoIsUrl repo && (uriScheme $ url repo) == "ssh:" +assertLocal repo action = + if (not $ repoIsUrl repo) then action else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ " not supported" -assertremote repo action = - if (repoIsRemote repo) +assertUrl repo action = + if (repoIsUrl repo) then action else error $ "acting on local git repo " ++ (repoDescribe repo) ++ " not supported" @@ -135,7 +132,7 @@ bare repo = {- Path to a repository's gitattributes file. -} attributes :: Repo -> String -attributes repo = assertlocal repo $ do +attributes repo = assertLocal repo $ do if (bare repo) then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" @@ -147,13 +144,13 @@ dir repo = if (bare repo) then "" else ".git" {- Path to a repository's --work-tree. -} workTree :: Repo -> FilePath workTree repo = - if (repoIsLocal) repo + if (not $ repoIsUrl repo) then top repo else assertssh repo $ (remoteHost repo) ++ ":" ++ (uriPath $ url repo) {- Hostname for a remote repo. (May include a username and/or port too.) -} remoteHost :: Repo -> String -remoteHost repo = assertremote repo $ +remoteHost repo = assertUrl repo $ (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) where a = fromJust $ uriAuthority $ url repo @@ -175,19 +172,19 @@ relative repo file = drop (length absrepo) absfile {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [String] -> [String] -gitCommandLine repo params = assertlocal repo $ +gitCommandLine repo params = assertLocal repo $ -- force use of specified repo via --git-dir and --work-tree ["--git-dir="++(top repo)++"/"++(dir repo), "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () -run repo params = assertlocal repo $ do +run repo params = assertLocal repo $ do r <- safeSystem "git" (gitCommandLine repo params) return () {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String -pipeRead repo params = assertlocal repo $ do +pipeRead repo params = assertLocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do ret <- hGetContentsStrict h return ret @@ -208,10 +205,11 @@ notInRepo repo location = do {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo -configRead repo = if (repoIsLocal repo) - {- Cannot use pipeRead because it relies on the config having - been already read. Instead, chdir to the repo. -} +configRead repo = + if (not $ repoIsUrl repo) then do + {- Cannot use pipeRead because it relies on the config having + been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory bracket_ (changeWorkingDirectory (top repo)) (\_ -> changeWorkingDirectory cwd) $ From 9ec5d90b6a28e28f6349635d33df0000ce9203ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 14:28:47 -0400 Subject: [PATCH 0262/8313] avoid reading configs for URL remotes every time --- Backend/File.hs | 2 +- Remotes.hs | 67 +++++++++++++++++++++++++++++++--------------- debian/changelog | 1 + doc/git-annex.mdwn | 2 ++ 4 files changed, 50 insertions(+), 22 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index d28b927287..b99a064ae3 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -79,7 +79,7 @@ copyKeyFile key file = do {- Tries to copy a file from a remote. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool copyFromRemote r key file = do - if (Git.repoIsLocal r) + if (not $ Git.repoIsUrl r) then getlocal else if (Git.repoIsSsh r) then getssh diff --git a/Remotes.hs b/Remotes.hs index 07aafe51b7..cb8f4d1312 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -8,6 +8,7 @@ module Remotes ( import Control.Exception import Control.Monad.State (liftIO) +import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils import Data.Either.Utils @@ -20,6 +21,7 @@ import qualified Annex import LocationLog import Locations import UUID +import Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -31,23 +33,36 @@ withKey key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost - -- This only uses cached data, so may not include new remotes - -- or remotes whose uuid has changed (eg by a different drive being - -- mounted at their location). So unless it happens to find all - -- remotes, try harder, loading the remotes' configs. - remotes <- reposByUUID allremotes uuids + -- To determine if a remote has a key, its UUID needs to be known. + -- The locally cached UIIDs of remotes can fall out of date if + -- eg, a different drive is mounted at the same location. + -- But, reading the config of remotes can be expensive, so make + -- sure we only do it once per git-annex run. remotesread <- Annex.flagIsSet "remotesread" - if ((length allremotes /= length remotes) && not remotesread) - then tryharder allremotes uuids - else return remotes + if (remotesread) + then reposByUUID allremotes uuids + else do + -- We assume that it's cheap to read the config + -- of non-URL remotes, so that is done each time. + -- But reading the config of an URL remote is + -- only done when there is no cached UUID value. + let cheap = filter (not . Git.repoIsUrl) allremotes + let expensive = filter Git.repoIsUrl allremotes + doexpensive <- filterM cachedUUID expensive + if (0 < length doexpensive) + then showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." + else return () + let todo = cheap ++ doexpensive + if (0 < length todo) + then do + e <- mapM tryGitConfigRead todo + Annex.flagChange "remotesread" $ FlagBool True + withKey key + else reposByUUID allremotes uuids where - tryharder allremotes uuids = do - -- more expensive; read each remote's config - eitherremotes <- mapM tryGitConfigRead allremotes - let allremotes' = map fromEither eitherremotes - remotes' <- reposByUUID allremotes' uuids - Annex.flagChange "remotesread" $ FlagBool True - return remotes' + cachedUUID r = do + u <- getUUID r + return $ 0 == length u {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] @@ -55,10 +70,11 @@ remotesByCost = do g <- Annex.gitRepo reposByCost $ Git.remotes g -{- Orders a list of git repos by cost. -} +{- Orders a list of git repos by cost, and throws out ignored ones. -} reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do - costpairs <- mapM costpair l + notignored <- filterM repoNotIgnored l + costpairs <- mapM costpair notignored return $ fst $ unzip $ sortBy bycost $ costpairs where costpair r = do @@ -76,13 +92,22 @@ repoCost r = do g <- Annex.gitRepo if ((length $ config g r) > 0) then return $ read $ config g r - else if (Git.repoIsLocal r) - then return 100 - else return 200 + else if (Git.repoIsUrl r) + then return 200 + else return 100 where config g r = Git.configGet g (configkey r) "" configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" +{- Checks if a repo should be ignored. -} +repoNotIgnored :: Git.Repo -> Annex Bool +repoNotIgnored r = do + g <- Annex.gitRepo + return ("true" /= config g r) + where + config g r = Git.configGet g (configkey r) "" + configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" + {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the - config for a specified remote, and updates state. If successful, it @@ -95,7 +120,7 @@ tryGitConfigRead r = do -- for other reasons; catch all possible exceptions result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of - Left err -> return $ Left r + Left e -> return $ Left r Right r' -> do g <- Annex.gitRepo let l = Git.remotes g diff --git a/debian/changelog b/debian/changelog index 1c36652302..2800e54a33 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.02) UNRELEASED; urgency=low * New fromkey subcommand, for registering urls, etc. + * Can scp annexed files from remotes. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 86ac9c6359..a7c6b9e48c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -154,6 +154,8 @@ Like other git commands, git-annex is configured via `.git/config`. repositories. Note that other factors may be configured when pushing files to repositories, in particular, whether the repository is on a filesystem with sufficient free space. +* `remote..annex-ignore` -- If set to "true", prevents git-annex + from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. From 91e6625eb56671472abd9532a5635f541d025a60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 14:57:02 -0400 Subject: [PATCH 0263/8313] add shellEscape ugly, but sometimes necessary There is a haskell shell-escape module, but it is not packaged in Debian --- Utility.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index 09b973002f..8bffde0577 100644 --- a/Utility.hs +++ b/Utility.hs @@ -7,7 +7,8 @@ module Utility ( parentDir, relPathCwdToDir, relPathDirToDir, - boolSystem + boolSystem, + shellEscape ) where import System.IO @@ -108,3 +109,9 @@ boolSystem command params = do ExitFailure e -> if Just e == cast sigINT then error $ command ++ "interrupted" else return False + +{- Escapes a filename to be safely able to be exposed to the shell. -} +shellEscape f = "'" ++ quote ++ "'" + where + -- replace ' with '"'"' + quote = join "'\"'\"'" $ split "'" f From aafb63edb13ac87eb5d741c75b90de6115f06452 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:06:14 -0400 Subject: [PATCH 0264/8313] support checking network remotes when dropping --- Backend/File.hs | 19 +++++++++++++++---- GitRepo.hs | 35 ++++++++++++++++++++++------------- debian/changelog | 3 ++- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index b99a064ae3..15d8f4a263 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -44,9 +44,18 @@ mustProvide = error "must provide this field" dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True -{- Just check if the .git/annex/ file for the key exists. -} +{- Just check if the .git/annex/ file for the key exists. + - + - But, if running against a remote annex, need to use ssh to do it. -} checkKeyFile :: Key -> Annex Bool -checkKeyFile k = inAnnex k +checkKeyFile k = do + g <- Annex.gitRepo + if (not $ Git.repoIsUrl g) + then inAnnex k + else do + showNote ("checking " ++ Git.repoDescribe g ++ "...") + liftIO $ boolSystem "ssh" [Git.urlHost g, + "test -e " ++ (shellEscape $ annexLocation g k)] {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} @@ -85,11 +94,13 @@ copyFromRemote r key file = do then getssh else error "copying from non-ssh repo not supported" where - location = annexLocation r key getlocal = boolSystem "cp" ["-a", location, file] getssh = do liftIO $ putStrLn "" -- make way for scp progress bar - boolSystem "scp" [location, file] + -- TODO double-shell-quote path for scp + boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location showLocations :: Key -> Annex () showLocations key = do diff --git a/GitRepo.hs b/GitRepo.hs index ea9e8a8b76..553e91fec0 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -16,6 +16,8 @@ module GitRepo ( workTree, dir, relative, + urlPath, + urlHost, configGet, configMap, configRead, @@ -110,7 +112,7 @@ repoIsSsh repo = repoIsUrl repo && (uriScheme $ url repo) == "ssh:" assertLocal repo action = if (not $ repoIsUrl repo) then action - else error $ "acting on remote git repo " ++ (repoDescribe repo) ++ + else error $ "acting on URL git repo " ++ (repoDescribe repo) ++ " not supported" assertUrl repo action = if (repoIsUrl repo) @@ -137,23 +139,18 @@ attributes repo = assertLocal repo $ do then (top repo) ++ "/info/.gitattributes" else (top repo) ++ "/.gitattributes" -{- Path to a repository's .git directory, relative to its topdir. -} +{- Path to a repository's .git directory, relative to its workTree. -} dir :: Repo -> String dir repo = if (bare repo) then "" else ".git" -{- Path to a repository's --work-tree. -} +{- Path to a repository's --work-tree, that is, its top. + - + - Note that for URL repositories, this is relative to the urlHost -} workTree :: Repo -> FilePath workTree repo = if (not $ repoIsUrl repo) then top repo - else assertssh repo $ (remoteHost repo) ++ ":" ++ (uriPath $ url repo) - -{- Hostname for a remote repo. (May include a username and/or port too.) -} -remoteHost :: Repo -> String -remoteHost repo = assertUrl repo $ - (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) - where - a = fromJust $ uriAuthority $ url repo + else urlPath repo {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. @@ -170,6 +167,18 @@ relative repo file = drop (length absrepo) absfile Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo +{- Hostname of an URL repo. (May include a username and/or port too.) -} +urlHost :: Repo -> String +urlHost repo = assertUrl repo $ + (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + where + a = fromJust $ uriAuthority $ url repo + +{- Path of an URL repo. -} +urlPath :: Repo -> String +urlPath repo = assertUrl repo $ + uriPath $ url repo + {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo params = assertLocal repo $ @@ -215,9 +224,9 @@ configRead repo = (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] proc else assertssh repo $ do - pOpen ReadFromPipe "ssh" [remoteHost repo, sshcommand] proc + pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] proc where - sshcommand = "cd '" ++ (uriPath $ url repo) ++ "' && git config --list" + sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" proc h = do val <- hGetContentsStrict h let r = repo { config = configParse val } diff --git a/debian/changelog b/debian/changelog index 2800e54a33..3d791ffe57 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ git-annex (0.02) UNRELEASED; urgency=low * New fromkey subcommand, for registering urls, etc. - * Can scp annexed files from remotes. + * Can scp annexed files from remote hosts, and check remote hosts for + file content when dropping files. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From 599cb15f305468bd5d5d23b7950018c8a767cc4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:08:53 -0400 Subject: [PATCH 0265/8313] update --- debian/changelog | 2 ++ doc/bugs/network_remotes.mdwn | 2 ++ doc/git-annex.mdwn | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 3d791ffe57..68f9f02cde 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (0.02) UNRELEASED; urgency=low * New fromkey subcommand, for registering urls, etc. * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. + * Add remote.annex-ignore git config setting to allow completly disabling + a given remote. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/bugs/network_remotes.mdwn b/doc/bugs/network_remotes.mdwn index be43ee20be..42efa832f5 100644 --- a/doc/bugs/network_remotes.mdwn +++ b/doc/bugs/network_remotes.mdwn @@ -1,3 +1,5 @@ Support for remote git repositories (ssh:// specifically can be made to work, although the other end probably needs to have git-annex installed..) + +[[done]], at least get and put work.. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a7c6b9e48c..37100fceef 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -92,7 +92,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * init description - Initializes git-annex with a descripotion of the git repository. + Initializes git-annex with a description of the git repository. This is an optional, but recommended step. * unannex [path ...] From 1e40562b0284d81e0509e61369b28c7f6b14152a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:10:21 -0400 Subject: [PATCH 0266/8313] todo --- doc/bugs/rsync.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/bugs/rsync.mdwn diff --git a/doc/bugs/rsync.mdwn b/doc/bugs/rsync.mdwn new file mode 100644 index 0000000000..75e0175c86 --- /dev/null +++ b/doc/bugs/rsync.mdwn @@ -0,0 +1,2 @@ +Transferring a file from a ssh:// remote should use rsync to allow resuming +of a prior transfer. From 5b4272aba3ee04c89c8f2be509377e7dd68cc5f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:12:48 -0400 Subject: [PATCH 0267/8313] update --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f8bb50122..921eb70670 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,8 @@ docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ - --underlaydir=/dev/null --disable-plugin=shortcut + --underlaydir=/dev/null --disable-plugin=shortcut \ + --disable-plugin=smiley clean: rm -rf build git-annex git-annex.1 From 26005d23bae2ab213fda694d2744ed239e412cbe Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:21:23 -0400 Subject: [PATCH 0268/8313] separate remote names with commas --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index cb8f4d1312..0e7dd31eaf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -25,7 +25,7 @@ import Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String -list remotes = join " " $ map Git.repoDescribe remotes +list remotes = join ", " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} withKey :: Key -> Annex [Git.Repo] From e6048b0389f8da9bd1e56e2f1f7d18abc773f03f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:38:31 -0400 Subject: [PATCH 0269/8313] document using ssh remotes --- debian/control | 2 +- doc/walkthrough.mdwn | 46 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/debian/control b/debian/control index 83bc8c82bf..d8abc487cb 100644 --- a/debian/control +++ b/debian/control @@ -10,7 +10,7 @@ Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid +Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7dbe62bdc2..5c91f2a9c6 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -13,8 +13,8 @@ This is very straightforward. Just tell it a description of the repository. ## adding a remote -This could be a USB drive, or a sshfs or NFS mount to a file server, for -example. +Like any other git repository, git-annex repositories have remotes. +Let's start by adding a USB drive as a remote. # sudo mount /media/usb # cd /media/usb @@ -25,11 +25,8 @@ example. # cd ~/annex # git remote add usbdrive /media/usb -This is all standard ad-hoc distributed git repository setup. Or you -could have added a centralized bare repository on a server if you prefer -doing things that way. - -Anyway, the only git-annex specific part is telling it the name +This is all standard ad-hoc distributed git repository setup. +The only git-annex specific part is telling it the name of the new repository created on the USB drive. Notice that both repos are set up as remotes of the other one. This lets @@ -151,6 +148,41 @@ makes it very easy. # git annex move my_cool_big_file --to usbdrive move my_cool_big_file (to usbdrive...) ok +## using ssh remotes + +So far git-annex has been used with a remote repository on a USB drive. +But it can also be used with a remote that is truely remote, a host +accessed by ssh. + +Say you have a desktop on the same network as your laptop and want +to clone the laptop's annex to it: + + # git clone ssh://mylaptop/home/me/annex ~/annex + # cd ~/annex + # git annex init "my desktop" + +Now you can get files and they will be transferred by `scp`: + + # git annex get my_cool_big_file + get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) + my_cool_big_file 100% 2159 2.1KB/s 00:00 + ok + +When you drop files, git-annex will ssh over to the remote and make +sure the file's content is still there before removing it locally: + + # git annex drop my_cool_big_file + drop my_cool_big_file (checking origin..) ok + +Note that normally git-annex prefers to use non-ssh remotes, like +a USB drive, before ssh remotes. They are assumed to be faster/cheaper to +access, if available. There is a annex-cost setting you can configure in +`.git/config` to adjust which repositories it prefers. See +[[the_man_page|git-annex]] for details. + +Also, note that you need full shell access for this to work -- +git-annex needs to be able to ssh in and run commands. + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 6f2a6a42d1fce3d5aa8e0b585829f4004c498165 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:42:59 -0400 Subject: [PATCH 0270/8313] real output --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 5c91f2a9c6..41e0332402 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -165,7 +165,7 @@ Now you can get files and they will be transferred by `scp`: # git annex get my_cool_big_file get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) - my_cool_big_file 100% 2159 2.1KB/s 00:00 + WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 ok When you drop files, git-annex will ssh over to the remote and make From 3dbba26275d1c7ae895962b7c66e7774a716cc91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:56:57 -0400 Subject: [PATCH 0271/8313] reorg --- Backend/File.hs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 15d8f4a263..797d487479 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -102,21 +102,6 @@ copyFromRemote r key file = do location = annexLocation r key sshlocation = (Git.urlHost r) ++ ":" ++ location -showLocations :: Key -> Annex () -showLocations key = do - g <- Annex.gitRepo - u <- getUUID g - uuids <- liftIO $ keyLocations g key - let uuidsf = filter (\v -> v /= u) uuids - ppuuids <- prettyPrintUUIDs uuidsf - if (0 < length uuidsf) - then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - else showLongNote $ "No other repository is known to contain the file." - -showTriedRemotes remotes = - showLongNote $ "I was unable to access these remotes: " ++ - (Remotes.list remotes) - {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an - error if not. -} @@ -163,3 +148,18 @@ checkRemoveKey key = do return False unsafe = showNote "unsafe" hint = showLongNote $ "(Use --force to override this check, or adjust annex.numcopies.)" + +showLocations :: Key -> Annex () +showLocations key = do + g <- Annex.gitRepo + u <- getUUID g + uuids <- liftIO $ keyLocations g key + let uuidsf = filter (\v -> v /= u) uuids + ppuuids <- prettyPrintUUIDs uuidsf + if (0 < length uuidsf) + then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids + else showLongNote $ "No other repository is known to contain the file." + +showTriedRemotes remotes = + showLongNote $ "I was unable to access these remotes: " ++ + (Remotes.list remotes) From 8ff3c73556704f94dcf80a1ddbed9430a1579a54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 15:59:13 -0400 Subject: [PATCH 0272/8313] bug --- doc/bugs/scp_interrupt_to_background.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/bugs/scp_interrupt_to_background.mdwn diff --git a/doc/bugs/scp_interrupt_to_background.mdwn b/doc/bugs/scp_interrupt_to_background.mdwn new file mode 100644 index 0000000000..700c81c650 --- /dev/null +++ b/doc/bugs/scp_interrupt_to_background.mdwn @@ -0,0 +1,2 @@ +When getting a file with scp, SIGINT is blocked, exposing the git +subcommand fork to background bug again. From ff38e49eb453ccfd58ce0e424aeca97389ab0100 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 20:35:39 -0400 Subject: [PATCH 0273/8313] --from/--to can be used to limit the remote repository that git-annex uses. --- Commands.hs | 6 ++++-- Remotes.hs | 17 ++++++++++++----- debian/changelog | 2 ++ doc/git-annex.mdwn | 9 ++++++++- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index 62eb08e83d..3e7447c5b1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -62,8 +62,10 @@ options = [ "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" - , Option ['r'] ["repository"] (ReqArg (storestring "repository") "REPOSITORY") - "specify a repository" + , Option ['t'] ["to"] (ReqArg (storestring "repository") "REPOSITORY") + "specify a repository to transfer content to" + , Option ['f'] ["from"] (ReqArg (storestring "repository") "REPOSITORY") + "specify a repository to transfer content from" ] where storebool n b = Annex.flagChange n $ FlagBool b diff --git a/Remotes.hs b/Remotes.hs index 0e7dd31eaf..8418a42253 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -70,7 +70,7 @@ remotesByCost = do g <- Annex.gitRepo reposByCost $ Git.remotes g -{- Orders a list of git repos by cost, and throws out ignored ones. -} +{- Orders a list of git repos by cost. Throws out ignored ones. -} reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do notignored <- filterM repoNotIgnored l @@ -99,14 +99,21 @@ repoCost r = do config g r = Git.configGet g (configkey r) "" configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" -{- Checks if a repo should be ignored. -} +{- Checks if a repo should be ignored, based either on annex-ignore + - setting, or on command-line options. Allows command-line to override + - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do g <- Annex.gitRepo - return ("true" /= config g r) + name <- Annex.flagGet "repository" + if (not $ null name) + then return $ match name + else return $ notignored g where - config g r = Git.configGet g (configkey r) "" - configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" + match name = name == Git.repoRemoteName r + notignored g = "true" /= config g + config g = Git.configGet g configkey "" + configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the diff --git a/debian/changelog b/debian/changelog index 68f9f02cde..0a27cd12ed 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.02) UNRELEASED; urgency=low file content when dropping files. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. + * --from/--to can be used to limit the remote repository that git-annex + uses. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 37100fceef..e67d9092c5 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -133,10 +133,17 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a key to operate on, for use with the addkey subcommand. +* --from=repository + + Specifies a repository that content will be retrieved from. + It should be specified using the name of a configured git remote. + + This can be used to limit the repository used by 'git annex get'. + * --to=repository Specifies a git repository that content will be sent to. - It can be specified by a path, url, or remote name. + It should be specified using the name of a configured git remote. ## CONFIGURATION From f4e2dde8a8ceaf689ec5391174b53cb1b213ea8b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Oct 2010 20:47:14 -0400 Subject: [PATCH 0274/8313] fix perl refugee code --- Backend.hs | 6 +++--- Backend/File.hs | 10 +++++----- Commands.hs | 10 +++++----- Remotes.hs | 8 ++++---- UUID.hs | 2 +- Utility.hs | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Backend.hs b/Backend.hs index b8def21cd8..c70b7b707b 100644 --- a/Backend.hs +++ b/Backend.hs @@ -40,21 +40,21 @@ import qualified TypeInternals as Internals list :: Annex [Backend] list = do l <- Annex.backends -- list is cached here - if (0 < length l) + if (not $ null l) then return l else do all <- Annex.supportedBackends g <- Annex.gitRepo let l = parseBackendList all $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" - let l' = if (0 < length backendflag) + let l' = if (not $ null backendflag) then (lookupBackendName all backendflag):l else l Annex.backendsChange $ l' return l' where parseBackendList all s = - if (length s == 0) + if (null s) then all else map (lookupBackendName all) $ words s diff --git a/Backend/File.hs b/Backend/File.hs index 797d487479..3396db3e58 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -62,7 +62,7 @@ checkKeyFile k = do copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do remotes <- Remotes.withKey key - if (0 == length remotes) + if (null remotes) then do showNote "not available" showLocations key @@ -142,7 +142,7 @@ checkRemoveKey key = do "Could only verify the existence of " ++ (show have) ++ " out of " ++ (show need) ++ " necessary copies" - if (0 /= length bad) then showTriedRemotes bad else return () + if (not $ null bad) then showTriedRemotes bad else return () showLocations key hint return False @@ -156,9 +156,9 @@ showLocations key = do uuids <- liftIO $ keyLocations g key let uuidsf = filter (\v -> v /= u) uuids ppuuids <- prettyPrintUUIDs uuidsf - if (0 < length uuidsf) - then showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - else showLongNote $ "No other repository is known to contain the file." + if (null uuidsf) + then showLongNote $ "No other repository is known to contain the file." + else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ diff --git a/Commands.hs b/Commands.hs index 3e7447c5b1..cb923d5e59 100644 --- a/Commands.hs +++ b/Commands.hs @@ -117,9 +117,9 @@ findWanted Description params _ = do parseCmd :: [String] -> AnnexState -> IO ([Annex ()], [Annex ()]) parseCmd argv state = do (flags, params) <- getopt - case (length params) of - 0 -> error usage - _ -> case (lookupCmd (params !! 0)) of + if (null params) + then error usage + else case (lookupCmd (params !! 0)) of [] -> error usage [Command _ action want _] -> do f <- findWanted want (drop 1 params) @@ -258,7 +258,7 @@ fixCmd file = inBackend file $ \(key, backend) -> do {- Stores description for the repository. -} initCmd :: String -> Annex () initCmd description = do - if (0 == length description) + if (null description) then error $ "please specify a description of this repository\n" ++ usage @@ -275,7 +275,7 @@ initCmd description = do fromKeyCmd :: FilePath -> Annex () fromKeyCmd file = do keyname <- Annex.flagGet "key" - if (0 == length keyname) + if (null keyname) then error "please specify the key with --key" else return () backends <- Backend.list diff --git a/Remotes.hs b/Remotes.hs index 8418a42253..48ab9ef8c0 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -49,11 +49,11 @@ withKey key = do let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive - if (0 < length doexpensive) + if (not $ null doexpensive) then showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." else return () let todo = cheap ++ doexpensive - if (0 < length todo) + if (not $ null todo) then do e <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True @@ -62,7 +62,7 @@ withKey key = do where cachedUUID r = do u <- getUUID r - return $ 0 == length u + return $ null u {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] @@ -90,7 +90,7 @@ reposByCost l = do repoCost :: Git.Repo -> Annex Int repoCost r = do g <- Annex.gitRepo - if ((length $ config g r) > 0) + if (not $ null $ config g r) then return $ read $ config g r else if (Git.repoIsUrl r) then return 200 diff --git a/UUID.hs b/UUID.hs index 47d305c4fb..a7783d6140 100644 --- a/UUID.hs +++ b/UUID.hs @@ -103,7 +103,7 @@ prettyPrintUUIDs uuids = do return $ unwords $ map (\u -> "\t"++(prettify m u)++"\n") uuids where prettify m u = - if (0 < (length $ findlog m u)) + if (not $ null $ findlog m u) then u ++ " -- " ++ (findlog m u) else u findlog m u = M.findWithDefault "" u m diff --git a/Utility.hs b/Utility.hs index 8bffde0577..8e620c64cd 100644 --- a/Utility.hs +++ b/Utility.hs @@ -45,7 +45,7 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = - if length dirs > 0 + if (not $ null dirs) then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where @@ -81,7 +81,7 @@ relPathCwdToDir dir = do -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = - if (0 < length path) + if (not $ null path) then addTrailingPathSeparator path else "" where From 2fbbbd34bcf8e925f84d96510eb063bdfbfe3f9b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 12:35:10 -0400 Subject: [PATCH 0275/8313] store from and to repositories separately --- Commands.hs | 4 ++-- Remotes.hs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index cb923d5e59..8be930bcc1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -62,9 +62,9 @@ options = [ "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storestring "repository") "REPOSITORY") + , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") "specify a repository to transfer content to" - , Option ['f'] ["from"] (ReqArg (storestring "repository") "REPOSITORY") + , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") "specify a repository to transfer content from" ] where diff --git a/Remotes.hs b/Remotes.hs index 48ab9ef8c0..f24da2c222 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -105,7 +105,9 @@ repoCost r = do repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do g <- Annex.gitRepo - name <- Annex.flagGet "repository" + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + let name = if (not $ null fromName) then fromName else toName if (not $ null name) then return $ match name else return $ notignored g From 5a91543be33719d6da7b53c4c449be8f75481375 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 12:41:13 -0400 Subject: [PATCH 0276/8313] update --- doc/git-annex.mdwn | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e67d9092c5..522be75700 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -48,11 +48,11 @@ content from the key-value store. add iso/Debian_5.0.iso ok # git commit -a -m "saving Debian CD for later" - # git annex drop iso - drop iso/Debian_5.0.iso ok + # git annex drop iso/Debian_4.0.iso + drop iso/Debian_4.0.iso ok # git commit -a -m "freed up space" - # git annex move video --to=usbdrive + # git annex move iso --to=usbdrive move iso/Debian_5.0.iso (to usbdrive...) ok # SUBCOMMANDS @@ -83,12 +83,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * move [path ...] - Moves the content of annexed files from the current repository to - another one. Use with the --to option. + When used with the --to option, moves the content of annexed files from + the current repository to the specified one. - Note that unlike drop, this does not honor annex.numcopies. - A file's content can be moved even if there are insufficient - copies to allow it to be dropped. + When used with the --from option, moves the content of annexed files + from the specified repository to the current one. * init description @@ -138,8 +137,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a repository that content will be retrieved from. It should be specified using the name of a configured git remote. - This can be used to limit the repository used by 'git annex get'. - * --to=repository Specifies a git repository that content will be sent to. From 9dfbf40d1a8493ec191f8e79410ed9d2a9508141 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 13:18:47 -0400 Subject: [PATCH 0277/8313] reorg remote key presense checking code Also, it now checks if a key is inAnnex, ie, cached in .git/annex, not if it is present in a remote. For the File Backend, these are equivilant, not so for other backends. --- Backend/File.hs | 29 ++++++----------------------- Core.hs | 12 ++++++++++-- Remotes.hs | 31 ++++++++++++++++++++++++------- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 3396db3e58..dbd0674286 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -44,24 +44,15 @@ mustProvide = error "must provide this field" dummyStore :: FilePath -> Key -> Annex (Bool) dummyStore file key = return True -{- Just check if the .git/annex/ file for the key exists. - - - - But, if running against a remote annex, need to use ssh to do it. -} +{- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool -checkKeyFile k = do - g <- Annex.gitRepo - if (not $ Git.repoIsUrl g) - then inAnnex k - else do - showNote ("checking " ++ Git.repoDescribe g ++ "...") - liftIO $ boolSystem "ssh" [Git.urlHost g, - "test -e " ++ (shellEscape $ annexLocation g k)] +checkKeyFile k = inAnnex k {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex (Bool) copyKeyFile key file = do - remotes <- Remotes.withKey key + remotes <- Remotes.keyPossibilities key if (null remotes) then do showNote "not available" @@ -97,7 +88,6 @@ copyFromRemote r key file = do getlocal = boolSystem "cp" ["-a", location, file] getssh = do liftIO $ putStrLn "" -- make way for scp progress bar - -- TODO double-shell-quote path for scp boolSystem "scp" [sshlocation, file] location = annexLocation r key sshlocation = (Git.urlHost r) ++ ":" ++ location @@ -112,7 +102,7 @@ checkRemoveKey key = do then return True else do g <- Annex.gitRepo - remotes <- Remotes.withKey key + remotes <- Remotes.keyPossibilities key let numcopies = read $ Git.configGet g config "1" if (numcopies > length remotes) then notEnoughCopies numcopies (length remotes) [] @@ -124,18 +114,11 @@ checkRemoveKey key = do then return True else notEnoughCopies need have bad findcopies need have (r:rs) bad = do - all <- Annex.supportedBackends - result <- liftIO $ ((try $ remoteHasKey r all)::IO (Either SomeException Bool)) - case (result) of + haskey <- Remotes.inAnnex r key + case (haskey) of Right True -> findcopies need (have+1) rs bad Right False -> findcopies need have rs bad Left _ -> findcopies need have rs (r:bad) - remoteHasKey remote all = do - -- To check if a remote has a key, construct a new - -- Annex monad and query its backend. - a <- Annex.new remote all - (result, _) <- Annex.run a (Backend.hasKey key) - return result notEnoughCopies need have bad = do unsafe showLongNote $ diff --git a/Core.hs b/Core.hs index 4941dc26bf..da05823bb3 100644 --- a/Core.hs +++ b/Core.hs @@ -62,11 +62,19 @@ gitAttributes repo = do Git.run repo ["commit", "-m", "git-annex setup", attributes] -{- Checks if a given key is currently present in the annexLocation -} +{- Checks if a given key is currently present in the annexLocation. + - + - This can be run against a remote repository to check the key there. -} inAnnex :: Key -> Annex Bool inAnnex key = do g <- Annex.gitRepo - liftIO $ doesFileExist $ annexLocation g key + if (not $ Git.repoIsUrl g) + then liftIO $ doesFileExist $ annexLocation g key + else do + showNote ("checking " ++ Git.repoDescribe g ++ "...") + liftIO $ boolSystem "ssh" [Git.urlHost g, + "test -e " ++ + (shellEscape $ annexLocation g key)] {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath diff --git a/Remotes.hs b/Remotes.hs index f24da2c222..13f66aae23 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -2,8 +2,9 @@ module Remotes ( list, - withKey, - tryGitConfigRead + keyPossibilities, + tryGitConfigRead, + inAnnex ) where import Control.Exception @@ -18,18 +19,19 @@ import Maybe import Types import qualified GitRepo as Git import qualified Annex +import qualified Backend import LocationLog import Locations import UUID -import Core +import qualified Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes {- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -withKey :: Key -> Annex [Git.Repo] -withKey key = do +keyPossibilities :: Key -> Annex [Git.Repo] +keyPossibilities key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost @@ -50,20 +52,35 @@ withKey key = do let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive if (not $ null doexpensive) - then showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." + then Core.showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." else return () let todo = cheap ++ doexpensive if (not $ null todo) then do e <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True - withKey key + keyPossibilities key else reposByUUID allremotes uuids where cachedUUID r = do u <- getUUID r return $ null u +{- Checks if a given remote has the content for a key inAnnex. + - + - This is done by constructing a new Annex monad using the remote. + - + - If the remote cannot be accessed, returns a Left error. + -} +inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) +inAnnex remote key = do + a <- liftIO $ Annex.new remote [] + liftIO $ ((try $ check a)::IO (Either IOException Bool)) + where + check a = do + (result, _) <- Annex.run a (Core.inAnnex key) + return result + {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] remotesByCost = do From 03bcb8d8b3033726eb4a5b1122fbadc1adb6b3a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 13:59:47 -0400 Subject: [PATCH 0278/8313] better message display --- Core.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Core.hs b/Core.hs index da05823bb3..176aa2bf38 100644 --- a/Core.hs +++ b/Core.hs @@ -97,11 +97,11 @@ logStatus key status = do {- Output logging -} showStart :: String -> String -> Annex () showStart command file = do - liftIO $ putStr $ command ++ " " ++ file + liftIO $ putStr $ command ++ " " ++ file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () showNote s = do - liftIO $ putStr $ " (" ++ s ++ ")" + liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout showLongNote :: String -> Annex () showLongNote s = do @@ -110,7 +110,7 @@ showLongNote s = do indent s = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () showEndOk = do - liftIO $ putStrLn " ok" + liftIO $ putStrLn "ok" showEndFail :: Annex () showEndFail = do liftIO $ putStrLn "\nfailed" From 08236e780fd0047b1dbab2ef7c50d96be8709cab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:14:36 -0400 Subject: [PATCH 0279/8313] reorg --- Backend/File.hs | 29 ++++------------------------- Remotes.hs | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 27 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index dbd0674286..14b4b9dae5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -66,31 +66,10 @@ copyKeyFile key file = do showLocations key return False trycopy full (r:rs) = do - -- annexLocation needs the git config to have been - -- read for a remote, so do that now, - -- if it hasn't been already - result <- Remotes.tryGitConfigRead r - case (result) of - Left err -> trycopy full rs - Right r' -> do - showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." - liftIO $ copyFromRemote r' key file - -{- Tries to copy a file from a remote. -} -copyFromRemote :: Git.Repo -> Key -> FilePath -> IO Bool -copyFromRemote r key file = do - if (not $ Git.repoIsUrl r) - then getlocal - else if (Git.repoIsSsh r) - then getssh - else error "copying from non-ssh repo not supported" - where - getlocal = boolSystem "cp" ["-a", location, file] - getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar - boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + copied <- Remotes.copyFromRemote r key file + if (copied) + then return True + else trycopy full rs {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an diff --git a/Remotes.hs b/Remotes.hs index 13f66aae23..a4b358c779 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -4,7 +4,9 @@ module Remotes ( list, keyPossibilities, tryGitConfigRead, - inAnnex + inAnnex, + commandLineRemote, + copyFromRemote ) where import Control.Exception @@ -20,10 +22,11 @@ import Types import qualified GitRepo as Git import qualified Annex import qualified Backend +import qualified Core import LocationLog import Locations import UUID -import qualified Core +import Utility {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -134,6 +137,22 @@ repoNotIgnored r = do config g = Git.configGet g configkey "" configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" +{- Returns the remote specified by --from or --to, may fail with error. -} +commandLineRemote :: Annex Git.Repo +commandLineRemote = do + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + let name = if (not $ null fromName) then fromName else toName + if (null name) + then error "no remote specified" + else do + g <- Annex.gitRepo + let match = filter (\r -> name == Git.repoRemoteName r) $ + Git.remotes g + if (null match) + then error $ "there is no git remote named \"" ++ name ++ "\"" + else return $ match !! 0 + {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the - config for a specified remote, and updates state. If successful, it @@ -161,3 +180,27 @@ tryGitConfigRead r = do if ((Git.repoRemoteName old) == (Git.repoRemoteName new)) then new:(exchange ls new) else old:(exchange ls new) + +{- Tries to copy a file from a remote. -} +copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool +copyFromRemote r key file = do + -- annexLocation needs the git config to have been read for a remote, + -- so do that now if it hasn't been already + result <- tryGitConfigRead r + case (result) of + Left err -> return False + Right r' -> copy r' + where + copy r = do + Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + if (not $ Git.repoIsUrl r) + then getlocal + else if (Git.repoIsSsh r) + then getssh + else error "copying from non-ssh repo not supported" + getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location From 5601a8db02e516d77d62fe18972a882ebad53b29 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:14:50 -0400 Subject: [PATCH 0280/8313] update --- doc/walkthrough.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 41e0332402..c9eba0a575 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -166,7 +166,7 @@ Now you can get files and they will be transferred by `scp`: # git annex get my_cool_big_file get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 - ok + ok When you drop files, git-annex will ssh over to the remote and make sure the file's content is still there before removing it locally: @@ -202,7 +202,7 @@ and cache it locally. # git annex get somefile get somefile (downloading) #########################################################################100.0% - ok + ok You can always drop files downloaded by the URL backend. It is assumed that the URL is stable; no local backup is kept. From fcd30ce99251cf8bd98ab25fb73f1fcb5b081a4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:26:23 -0400 Subject: [PATCH 0281/8313] bugfix --- Remotes.hs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index a4b358c779..c1cab73c62 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -198,9 +198,10 @@ copyFromRemote r key file = do else if (Git.repoIsSsh r) then getssh else error "copying from non-ssh repo not supported" - getlocal = liftIO $ boolSystem "cp" ["-a", location, file] - getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + where + getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location From 4c7248c77998d011f8978a32328b8f3817d7acbc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:26:38 -0400 Subject: [PATCH 0282/8313] factored out getViaTmp --- Core.hs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Core.hs b/Core.hs index 176aa2bf38..acb4489c8e 100644 --- a/Core.hs +++ b/Core.hs @@ -94,6 +94,24 @@ logStatus key status = do u <- getUUID g liftIO $ logChange g key u status +{- Runs an action, passing it a temporary filename to download, + - and if the action succeeds, moves the temp file into + - the annex as a key's content. -} +getViaTmp :: Key -> (FilePath -> Annex (Bool)) -> Annex () +getViaTmp key action = do + g <- Annex.gitRepo + let dest = annexLocation g key + let tmp = (annexTmpLocation g) ++ (keyFile key) + liftIO $ createDirectoryIfMissing True (parentDir tmp) + success <- action tmp + if (success) + then do + liftIO $ renameFile tmp dest + logStatus key ValuePresent + showEndOk + else do + showEndFail + {- Output logging -} showStart :: String -> String -> Annex () showStart command file = do From f05ed818f9e8e49b646805402be928f9c89c9a7f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:27:04 -0400 Subject: [PATCH 0283/8313] implemented 1/4th of move subcommand --- Commands.hs | 115 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 34 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8be930bcc1..011481bd8f 100644 --- a/Commands.hs +++ b/Commands.hs @@ -41,7 +41,7 @@ cmds = [ , (Command "drop" dropCmd FilesInGit "indicate content of files not currently wanted") , (Command "move" moveCmd FilesInGit - "transfer content of files to another repository") + "transfer content of files to/from another repository") , (Command "init" initCmd Description "initialize git-annex with repository description") , (Command "unannex" unannexCmd FilesInGit @@ -63,9 +63,9 @@ options = [ , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") "specify a key to use" , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") - "specify a repository to transfer content to" + "specify to where to transfer content" , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") - "specify a repository to transfer content from" + "specify from where to transfer content" ] where storebool n b = Annex.flagChange n $ FlagBool b @@ -136,7 +136,7 @@ parseCmd argv state = do {- Annexes a file, storing it in a backend, and then moving it into - the annex directory and setting up the symlink pointing to its content. -} addCmd :: FilePath -> Annex () -addCmd file = notInBackend file $ do +addCmd file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return () @@ -161,7 +161,7 @@ addCmd file = notInBackend file $ do {- Undo addCmd. -} unannexCmd :: FilePath -> Annex () -unannexCmd file = inBackend file $ \(key, backend) -> do +unannexCmd file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file Annex.flagChange "force" $ FlagBool True -- force backend to always remove Backend.removeKey backend key @@ -181,41 +181,18 @@ unannexCmd file = inBackend file $ \(key, backend) -> do {- Gets an annexed file from one of the backends. -} getCmd :: FilePath -> Annex () -getCmd file = inBackend file $ \(key, backend) -> do +getCmd file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return () else do showStart "get" file - g <- Annex.gitRepo - let dest = annexLocation g key - let tmp = (annexTmpLocation g) ++ (keyFile key) - liftIO $ createDirectoryIfMissing True (parentDir tmp) - success <- Backend.retrieveKeyFile backend key tmp - if (success) - then do - liftIO $ renameFile tmp dest - logStatus key ValuePresent - showEndOk - else do - showEndFail - -{- Moves the content of an annexed file to another repository, - - removing it from the current repository, and updates locationlog - - information on both. - - - - Note that unlike drop, this does not honor annex.numcopies. - - A file's content can be moved even if there are insufficient copies to - - allow it to be dropped. - -} -moveCmd :: FilePath -> Annex () -moveCmd file = inBackend file $ \(key, backend) -> do - error "TODO" + getViaTmp key (Backend.retrieveKeyFile backend key) {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} dropCmd :: FilePath -> Annex () -dropCmd file = inBackend file $ \(key, backend) -> do +dropCmd file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return () -- no-op @@ -241,7 +218,7 @@ dropCmd file = inBackend file $ \(key, backend) -> do {- Fixes the symlink to an annexed file. -} fixCmd :: FilePath -> Annex () -fixCmd file = inBackend file $ \(key, backend) -> do +fixCmd file = isAnnexed file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -294,13 +271,83 @@ fromKeyCmd file = do liftIO $ Git.run g ["add", file] showEndOk +{- Move a file either --to or --from a repository. + - + - This only operates on the cached file content; it does not involve + - moving data in the key-value backend. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +moveCmd :: FilePath -> Annex () +moveCmd file = do + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + case (fromName, toName) of + ("", "") -> error "specify either --from or --to" + ("", to) -> moveTo file + (from, "") -> moveFrom file + (_, _) -> error "only one of --from or --to can be specified" + +{- Moves the content of an annexed file to another repository, + - removing it from the current repository, and updates locationlog + - information on both. + - + - If the destination already has the content, it is still removed + - from the current repository. + -} +moveTo :: FilePath -> Annex () +moveTo file = isAnnexed file $ \(key, backend) -> do + ishere <- inAnnex key + if (not ishere) + then return () -- not here, so nothing to do + else do + showStart "move" file + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case isthere of + Left err -> error (show err) + Right True -> removeit + Right False -> moveit + where + moveit = do + error $ "TODO move" ++ file + removeit = do + error $ "TODO remove" ++ file + +{- Moves the content of an annexed file from another repository to the current + - repository and updates locationlog information on both. + - + - If the current repository already has the content, it is still removed + - from the other repository. + -} +moveFrom :: FilePath -> Annex () +moveFrom file = isAnnexed file $ \(key, backend) -> do + showStart "move" file -- have to show this before checking remote + ishere <- inAnnex key + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case (ishere, isthere) of + (_, Left err) -> error (show err) + (_, Right False) -> showEndFail + (False, Right True) -> moveit remote key + (True, Right True) -> removeit remote key + where + moveit remote key = do + getViaTmp key (Remotes.copyFromRemote remote key) + removeit remote key + removeit remote key = do + error $ "TODO remove" ++ file + showEndOk + -- helpers -notInBackend file a = do +notAnnexed file a = do r <- Backend.lookupFile file case (r) of Just v -> return () Nothing -> a -inBackend file a = do +isAnnexed file a = do r <- Backend.lookupFile file case (r) of Just v -> a v From 3cf16c9883d63247cceb1f8cf2ce41d43b7a214e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 14:58:14 -0400 Subject: [PATCH 0284/8313] incomplete --- Commands.hs | 11 ++++++----- Remotes.hs | 42 ++++++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/Commands.hs b/Commands.hs index 011481bd8f..ffa3576cf9 100644 --- a/Commands.hs +++ b/Commands.hs @@ -308,12 +308,13 @@ moveTo file = isAnnexed file $ \(key, backend) -> do isthere <- Remotes.inAnnex remote key case isthere of Left err -> error (show err) - Right True -> removeit - Right False -> moveit + Right False -> moveit remote key + Right True -> removeit remote key where - moveit = do - error $ "TODO move" ++ file - removeit = do + moveit remote key = do + Remotes.copyToRemote remote key + removeit remote key + removeit remote key = do error $ "TODO remove" ++ file {- Moves the content of an annexed file from another repository to the current diff --git a/Remotes.hs b/Remotes.hs index c1cab73c62..4cfcfdffdc 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -6,7 +6,8 @@ module Remotes ( tryGitConfigRead, inAnnex, commandLineRemote, - copyFromRemote + copyFromRemote, + copyToRemote ) where import Control.Exception @@ -70,13 +71,11 @@ keyPossibilities key = do return $ null u {- Checks if a given remote has the content for a key inAnnex. - - - - This is done by constructing a new Annex monad using the remote. - - - If the remote cannot be accessed, returns a Left error. -} inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) inAnnex remote key = do + -- the check needs to run in an Annex monad using the remote a <- liftIO $ Annex.new remote [] liftIO $ ((try $ check a)::IO (Either IOException Bool)) where @@ -181,7 +180,7 @@ tryGitConfigRead r = do then new:(exchange ls new) else old:(exchange ls new) -{- Tries to copy a file from a remote. -} +{- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file = do -- annexLocation needs the git config to have been read for a remote, @@ -189,13 +188,13 @@ copyFromRemote r key file = do result <- tryGitConfigRead r case (result) of Left err -> return False - Right r' -> copy r' + Right from -> copy from where - copy r = do - Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." - if (not $ Git.repoIsUrl r) + copy from = do + Core.showNote $ "copying from " ++ (Git.repoDescribe from) ++ "..." + if (not $ Git.repoIsUrl from) then getlocal - else if (Git.repoIsSsh r) + else if (Git.repoIsSsh from) then getssh else error "copying from non-ssh repo not supported" where @@ -203,5 +202,24 @@ copyFromRemote r key file = do getssh = do liftIO $ putStrLn "" -- make way for scp progress bar liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + location = annexLocation from key + sshlocation = (Git.urlHost from) ++ ":" ++ location + +{- Tries to copy a key's content to a remote. -} +copyToRemote :: Git.Repo -> Key -> Annex Bool +copyToRemote r key = do + g <- Annex.gitRepo + Core.showNote $ "copying to " ++ (Git.repoDescribe r) ++ "..." + if (not $ Git.repoIsUrl r) + then sendlocal g + else if (Git.repoIsSsh r) + then sendssh g + else error "copying to non-ssh repo not supported" + where + sendlocal g = liftIO $ boolSystem "cp" ["-a", location g, file] + sendssh g = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [location g, sshlocation] + location g = annexLocation g key + sshlocation = (Git.urlHost r) ++ ":" ++ file + file = error "TODO" From fed1d3c1f534e01414292cd99724e15f5b19cda7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Oct 2010 22:25:07 -0400 Subject: [PATCH 0285/8313] bugfix Annex.new can throw an IO exception if a remote git repo is not available. --- Remotes.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 4cfcfdffdc..aec38a363e 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -76,10 +76,10 @@ keyPossibilities key = do inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) inAnnex remote key = do -- the check needs to run in an Annex monad using the remote - a <- liftIO $ Annex.new remote [] - liftIO $ ((try $ check a)::IO (Either IOException Bool)) + liftIO $ ((try $ check)::IO (Either IOException Bool)) where - check a = do + check = do + a <- Annex.new remote [] (result, _) <- Annex.run a (Core.inAnnex key) return result From 81f71e57b9ac68b8d79c30fb27a22c5e3941fcee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 11:47:45 -0400 Subject: [PATCH 0286/8313] reorg --- doc/index.mdwn | 1 + doc/todo.mdwn | 4 ++++ doc/{bugs => todo}/add_a_git_backend.mdwn | 0 doc/{bugs => todo}/backendSHA1.mdwn | 0 doc/{bugs => todo}/branching.mdwn | 0 doc/todo/done.mdwn | 4 ++++ doc/{bugs => todo}/file_copy_progress_bar.mdwn | 0 doc/{bugs => todo}/fsck.mdwn | 0 doc/{bugs => todo}/gitrm.mdwn | 0 doc/{bugs => todo}/network_remotes.mdwn | 0 doc/todo/parallel_possibilities.mdwn | 13 +++++++++++++ doc/{bugs => todo}/pushpull.mdwn | 0 doc/{bugs => todo}/rsync.mdwn | 0 doc/{bugs => todo}/symlink_farming_commit_hook.mdwn | 0 doc/{bugs => todo}/using_url_backend.mdwn | 0 15 files changed, 22 insertions(+) create mode 100644 doc/todo.mdwn rename doc/{bugs => todo}/add_a_git_backend.mdwn (100%) rename doc/{bugs => todo}/backendSHA1.mdwn (100%) rename doc/{bugs => todo}/branching.mdwn (100%) create mode 100644 doc/todo/done.mdwn rename doc/{bugs => todo}/file_copy_progress_bar.mdwn (100%) rename doc/{bugs => todo}/fsck.mdwn (100%) rename doc/{bugs => todo}/gitrm.mdwn (100%) rename doc/{bugs => todo}/network_remotes.mdwn (100%) create mode 100644 doc/todo/parallel_possibilities.mdwn rename doc/{bugs => todo}/pushpull.mdwn (100%) rename doc/{bugs => todo}/rsync.mdwn (100%) rename doc/{bugs => todo}/symlink_farming_commit_hook.mdwn (100%) rename doc/{bugs => todo}/using_url_backend.mdwn (100%) diff --git a/doc/index.mdwn b/doc/index.mdwn index de5fe55a3c..ae5dbf85e5 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -18,6 +18,7 @@ To get a feel for it, see the [[walkthrough]]. * [[install]] * [[news]] * [[bugs]] +* [[todo]] * [[contact]] """]] diff --git a/doc/todo.mdwn b/doc/todo.mdwn new file mode 100644 index 0000000000..79552298b6 --- /dev/null +++ b/doc/todo.mdwn @@ -0,0 +1,4 @@ +This is git-annex's todo list. Link items to [[todo/done]] when done. + +[[!inline pages="./todo/* and !./todo/done and !link(done) +and !*/Discussion" actions=yes postform=yes show=0 archive=yes]] diff --git a/doc/bugs/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn similarity index 100% rename from doc/bugs/add_a_git_backend.mdwn rename to doc/todo/add_a_git_backend.mdwn diff --git a/doc/bugs/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn similarity index 100% rename from doc/bugs/backendSHA1.mdwn rename to doc/todo/backendSHA1.mdwn diff --git a/doc/bugs/branching.mdwn b/doc/todo/branching.mdwn similarity index 100% rename from doc/bugs/branching.mdwn rename to doc/todo/branching.mdwn diff --git a/doc/todo/done.mdwn b/doc/todo/done.mdwn new file mode 100644 index 0000000000..e7c98081b7 --- /dev/null +++ b/doc/todo/done.mdwn @@ -0,0 +1,4 @@ +recently fixed [[todo]] items. + +[[!inline pages="./* and link(./done) and !*/Discussion" sort=mtime show=10 +archive=yes]] diff --git a/doc/bugs/file_copy_progress_bar.mdwn b/doc/todo/file_copy_progress_bar.mdwn similarity index 100% rename from doc/bugs/file_copy_progress_bar.mdwn rename to doc/todo/file_copy_progress_bar.mdwn diff --git a/doc/bugs/fsck.mdwn b/doc/todo/fsck.mdwn similarity index 100% rename from doc/bugs/fsck.mdwn rename to doc/todo/fsck.mdwn diff --git a/doc/bugs/gitrm.mdwn b/doc/todo/gitrm.mdwn similarity index 100% rename from doc/bugs/gitrm.mdwn rename to doc/todo/gitrm.mdwn diff --git a/doc/bugs/network_remotes.mdwn b/doc/todo/network_remotes.mdwn similarity index 100% rename from doc/bugs/network_remotes.mdwn rename to doc/todo/network_remotes.mdwn diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn new file mode 100644 index 0000000000..178f95021b --- /dev/null +++ b/doc/todo/parallel_possibilities.mdwn @@ -0,0 +1,13 @@ +One of my reasons for using haskell was that it provides the possibility of +some parallell processing. Although since git-annex hits the filesystem +heavily and mostly runs other git commands, maybe not a whole lot. + +Anyway, each git-annex command is broken down into a series of independant +actions, which has some potential for parallelism. + +Probably they would need to be split further. Each action currently has 3 +distinct phases, basically "check", "do", and "record". If the check action +returned a do action that returned a record action, then it could easily +make sense to parallelize the check actions and start on the do actions +(which probably won't parallelize well) while they are still being +generated, and possibly parallelize the record actions at the end. diff --git a/doc/bugs/pushpull.mdwn b/doc/todo/pushpull.mdwn similarity index 100% rename from doc/bugs/pushpull.mdwn rename to doc/todo/pushpull.mdwn diff --git a/doc/bugs/rsync.mdwn b/doc/todo/rsync.mdwn similarity index 100% rename from doc/bugs/rsync.mdwn rename to doc/todo/rsync.mdwn diff --git a/doc/bugs/symlink_farming_commit_hook.mdwn b/doc/todo/symlink_farming_commit_hook.mdwn similarity index 100% rename from doc/bugs/symlink_farming_commit_hook.mdwn rename to doc/todo/symlink_farming_commit_hook.mdwn diff --git a/doc/bugs/using_url_backend.mdwn b/doc/todo/using_url_backend.mdwn similarity index 100% rename from doc/bugs/using_url_backend.mdwn rename to doc/todo/using_url_backend.mdwn From e29210d1dddb79abc0f93fc5175add8e10455688 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 14:10:38 -0400 Subject: [PATCH 0287/8313] update --- Commands.hs | 17 ++++++++++++++--- Core.hs | 6 +++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Commands.hs b/Commands.hs index ffa3576cf9..8c9dca9adb 100644 --- a/Commands.hs +++ b/Commands.hs @@ -86,7 +86,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Generate descrioptions of wanted parameters for subcommands. -} +{- Generate descriptions of wanted parameters for subcommands. -} descWanted :: CmdWants -> String descWanted Description = "DESCRIPTION" descWanted _ = "PATH ..." @@ -187,7 +187,10 @@ getCmd file = isAnnexed file $ \(key, backend) -> do then return () else do showStart "get" file - getViaTmp key (Backend.retrieveKeyFile backend key) + ok <- getViaTmp key (Backend.retrieveKeyFile backend key) + if (ok) + then showEndOk + else showEndFail {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} @@ -315,7 +318,15 @@ moveTo file = isAnnexed file $ \(key, backend) -> do Remotes.copyToRemote remote key removeit remote key removeit remote key = do - error $ "TODO remove" ++ file + error "TODO: drop key from local" + -- Update local location log; key is present + -- there and missing here. + logStatus key ValueMissing + u <- getUUID remote + liftIO $ logChange remote key u ValuePresent + -- Propigate location log to remote. + error "TODO: update remote locationlog" + showEndOk {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. diff --git a/Core.hs b/Core.hs index acb4489c8e..1d887792ad 100644 --- a/Core.hs +++ b/Core.hs @@ -97,7 +97,7 @@ logStatus key status = do {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into - the annex as a key's content. -} -getViaTmp :: Key -> (FilePath -> Annex (Bool)) -> Annex () +getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool getViaTmp key action = do g <- Annex.gitRepo let dest = annexLocation g key @@ -108,9 +108,9 @@ getViaTmp key action = do then do liftIO $ renameFile tmp dest logStatus key ValuePresent - showEndOk + return True else do - showEndFail + return False {- Output logging -} showStart :: String -> String -> Annex () From 7fe4bfa20fc9e6ced0b0e933891becb0546b79bb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 15:44:27 -0400 Subject: [PATCH 0288/8313] split commands into 3 phases I feel like I just leveled up in Haskell. --- Commands.hs | 393 ++++++++++++++++----------- Core.hs | 2 + doc/todo/parallel_possibilities.mdwn | 11 +- 3 files changed, 236 insertions(+), 170 deletions(-) diff --git a/Commands.hs b/Commands.hs index 8c9dca9adb..f4286abf5b 100644 --- a/Commands.hs +++ b/Commands.hs @@ -24,31 +24,66 @@ import Core import qualified Remotes import qualified TypeInternals -data CmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description -data Command = Command { - cmdname :: String, - cmdaction :: (String -> Annex ()), - cmdwants :: CmdWants, - cmddesc :: String +{- A subcommand runs in three stages. Each stage can return the next stage + - to run. + - + - 1. The start stage is run before anything is printed about the + - subcommand, and can early abort it if the input does not make sense. + - It should run quickly and should not modify Annex state. + - + - 2. The perform stage is run after a message is printed about the subcommand + - being run. + - + - 3. The cleanup stage is run only if the do stage succeeds, and it returns + - the overall success/fail of the subcommand. + -} +type SubCmdStart = String -> Annex (Maybe SubCmdPerform) +type SubCmdPerform = Annex (Maybe SubCmdCleanup) +type SubCmdCleanup = Annex Bool + +{- Runs a subcommand through its three stages. -} +doSubCmd :: String -> SubCmdStart -> String -> Annex () +doSubCmd cmdname start param = do + res <- start param :: Annex (Maybe SubCmdPerform) + case (res) of + Nothing -> return () + Just perform -> do + showStart cmdname param + res <- perform :: Annex (Maybe SubCmdCleanup) + case (res) of + Nothing -> showEndFail + Just cleanup -> do + res <- cleanup + if (res) + then showEndOk + else showEndFail + + +data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description +data SubCommand = Command { + subcmdname :: String, + subcmdaction :: SubCmdStart, + subcmdwants :: SubCmdWants, + subcmddesc :: String } -cmds :: [Command] -cmds = [ - (Command "add" addCmd FilesNotInGit +subCmds :: [SubCommand] +subCmds = [ + (Command "add" addStart FilesNotInGit "add files to annex") - , (Command "get" getCmd FilesInGit + , (Command "get" getStart FilesInGit "make content of annexed files available") - , (Command "drop" dropCmd FilesInGit + , (Command "drop" dropStart FilesInGit "indicate content of files not currently wanted") - , (Command "move" moveCmd FilesInGit + , (Command "move" moveStart FilesInGit "transfer content of files to/from another repository") - , (Command "init" initCmd Description + , (Command "init" initStart Description "initialize git-annex with repository description") - , (Command "unannex" unannexCmd FilesInGit + , (Command "unannex" unannexStart FilesInGit "undo accidential add command") - , (Command "fix" fixCmd FilesInGit + , (Command "fix" fixStart FilesInGit "fix up files' symlinks to point to annexed content") - , (Command "fromkey" fromKeyCmd FilesMissing + , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") ] @@ -71,29 +106,29 @@ options = [ storebool n b = Annex.flagChange n $ FlagBool b storestring n s = Annex.flagChange n $ FlagString s -header = "Usage: git-annex " ++ (join "|" $ map cmdname cmds) +header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where - cmddescs = unlines $ map (\c -> indent $ showcmd c) cmds + cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds showcmd c = - (cmdname c) ++ - (pad 10 (cmdname c)) ++ - (descWanted (cmdwants c)) ++ - (pad 13 (descWanted (cmdwants c))) ++ - (cmddesc c) + (subcmdname c) ++ + (pad 10 (subcmdname c)) ++ + (descWanted (subcmdwants c)) ++ + (pad 13 (descWanted (subcmdwants c))) ++ + (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' {- Generate descriptions of wanted parameters for subcommands. -} -descWanted :: CmdWants -> String +descWanted :: SubCmdWants -> String descWanted Description = "DESCRIPTION" descWanted _ = "PATH ..." -{- Finds the type of parameters a command wants, from among the passed +{- Finds the type of parameters a subcommand wants, from among the passed - parameter list. -} -findWanted :: CmdWants -> [String] -> Git.Repo -> IO [String] +findWanted :: SubCmdWants -> [String] -> Git.Repo -> IO [String] findWanted FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params return $ foldl (++) [] files @@ -121,139 +156,154 @@ parseCmd argv state = do then error usage else case (lookupCmd (params !! 0)) of [] -> error usage - [Command _ action want _] -> do + [Command name action want _] -> do f <- findWanted want (drop 1 params) (TypeInternals.repo state) - return (flags, map action $ filter notstate f) + return (flags, map (doSubCmd name action) $ + filter notstate f) where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds + lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds -{- Annexes a file, storing it in a backend, and then moving it into - - the annex directory and setting up the symlink pointing to its content. -} -addCmd :: FilePath -> Annex () -addCmd file = notAnnexed file $ do +{- The add subcommand annexes a file, storing it in a backend, and then + - moving it into the annex directory and setting up the symlink pointing + - to its content. -} +addStart :: FilePath -> Annex (Maybe SubCmdPerform) +addStart file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) - then return () - else do - showStart "add" file - g <- Annex.gitRepo - stored <- Backend.storeFileKey file - case (stored) of - Nothing -> showEndFail - Just (key, backend) -> do - logStatus key ValuePresent - setup g key - where - setup g key = do - let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile file dest - link <- calcGitLink file key - liftIO $ createSymbolicLink link file - liftIO $ Git.run g ["add", file] - showEndOk + then return Nothing + else return $ Just $ addPerform file +addPerform :: FilePath -> Annex (Maybe SubCmdCleanup) +addPerform file = do + g <- Annex.gitRepo + stored <- Backend.storeFileKey file + case (stored) of + Nothing -> return Nothing + Just (key, backend) -> return $ Just $ addCleanup file key +addCleanup :: FilePath -> Key -> Annex Bool +addCleanup file key = do + logStatus key ValuePresent + g <- Annex.gitRepo + let dest = annexLocation g key + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile file dest + link <- calcGitLink file key + liftIO $ createSymbolicLink link file + liftIO $ Git.run g ["add", file] + return True -{- Undo addCmd. -} -unannexCmd :: FilePath -> Annex () -unannexCmd file = isAnnexed file $ \(key, backend) -> do - showStart "unannex" file - Annex.flagChange "force" $ FlagBool True -- force backend to always remove +{- The unannex subcommand undoes an add. -} +unannexStart :: FilePath -> Annex (Maybe SubCmdPerform) +unannexStart file = isAnnexed file $ \(key, backend) -> do + return $ Just $ unannexPerform file key backend +unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +unannexPerform file key backend = do + -- force backend to always remove + Annex.flagChange "force" $ FlagBool True Backend.removeKey backend key + return $ Just $ unannexCleanup file key +unannexCleanup :: FilePath -> Key -> Annex Bool +unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo let src = annexLocation g key - moveout g src - where - moveout g src = do - liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", file] - -- git rm deletes empty directories; - -- put them back - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ renameFile src file - showEndOk + liftIO $ removeFile file + liftIO $ Git.run g ["rm", "--quiet", file] + -- git rm deletes empty directories; put them back + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ renameFile src file + return True {- Gets an annexed file from one of the backends. -} -getCmd :: FilePath -> Annex () -getCmd file = isAnnexed file $ \(key, backend) -> do +getStart :: FilePath -> Annex (Maybe SubCmdPerform) +getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) - then return () - else do - showStart "get" file - ok <- getViaTmp key (Backend.retrieveKeyFile backend key) - if (ok) - then showEndOk - else showEndFail + then return Nothing + else return $ Just $ getPerform file key backend +getPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +getPerform file key backend = do + ok <- getViaTmp key (Backend.retrieveKeyFile backend key) + if (ok) + then return $ Just $ return True + else return Nothing {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropCmd :: FilePath -> Annex () -dropCmd file = isAnnexed file $ \(key, backend) -> do +dropStart :: FilePath -> Annex (Maybe SubCmdPerform) +dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) - then return () -- no-op - else do - showStart "drop" file - success <- Backend.removeKey backend key - if (success) - then do - cleanup key - showEndOk - else showEndFail - where - cleanup key = do - logStatus key ValueMissing - inannex <- inAnnex key - if (inannex) - then do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return () - else return () + then return Nothing + else return $ Just $ dropPerform file key backend +dropPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +dropPerform file key backend = do + success <- Backend.removeKey backend key + if (success) + then return $ Just $ dropCleanup key + else return Nothing +dropCleanup :: Key -> Annex Bool +dropCleanup key = do + logStatus key ValueMissing + inannex <- inAnnex key + if (inannex) + then do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return True + else return True {- Fixes the symlink to an annexed file. -} -fixCmd :: FilePath -> Annex () -fixCmd file = isAnnexed file $ \(key, backend) -> do +fixStart :: FilePath -> Annex (Maybe SubCmdPerform) +fixStart file = isAnnexed file $ \(key, backend) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) - then return () - else do - showStart "fix" file - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file - liftIO $ createSymbolicLink link file - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] - showEndOk + then return Nothing + else return $ Just $ fixPerform file link +fixPerform :: FilePath -> FilePath -> Annex (Maybe SubCmdCleanup) +fixPerform file link = do + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + g <- Annex.gitRepo + liftIO $ Git.run g ["add", file] + return $ Just $ fixCleanup +fixCleanup :: Annex Bool +fixCleanup = do + return True {- Stores description for the repository. -} -initCmd :: String -> Annex () -initCmd description = do +initStart :: String -> Annex (Maybe SubCmdPerform) +initStart description = do if (null description) then error $ "please specify a description of this repository\n" ++ usage - else do - g <- Annex.gitRepo - u <- getUUID g - describeUUID u description - log <- uuidLog - liftIO $ Git.run g ["add", log] - liftIO $ Git.run g ["commit", "-m", "git annex init", log] - liftIO $ putStrLn "description set" + else return $ Just $ initPerform description +initPerform :: String -> Annex (Maybe SubCmdCleanup) +initPerform description = do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + return $ Just $ initCleanup +initCleanup :: Annex Bool +initCleanup = do + g <- Annex.gitRepo + log <- uuidLog + liftIO $ Git.run g ["add", log] + liftIO $ Git.run g ["commit", "-m", "git annex init", log] + return True {- Adds a file pointing at a manually-specified key -} -fromKeyCmd :: FilePath -> Annex () -fromKeyCmd file = do +fromKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) +fromKeyStart file = do keyname <- Annex.flagGet "key" if (null keyname) then error "please specify the key with --key" @@ -264,33 +314,31 @@ fromKeyCmd file = do inbackend <- Backend.hasKey key if (not inbackend) then error $ "key ("++keyname++") is not present in backend" - else return () - + else return $ Just $ fromKeyPerform file key +fromKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +fromKeyPerform file key = do link <- calcGitLink file key - showStart "fromkey" file liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file + return $ Just $ fromKeyCleanup file +fromKeyCleanup :: FilePath -> Annex Bool +fromKeyCleanup file = do g <- Annex.gitRepo liftIO $ Git.run g ["add", file] - showEndOk + return True {- Move a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve - - moving data in the key-value backend. - - - - Note that unlike drop, this does not honor annex.numcopies. - - A file's content can be moved even if there are insufficient copies to - - allow it to be dropped. - -} -moveCmd :: FilePath -> Annex () -moveCmd file = do + - moving data in the key-value backend. -} +moveStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of ("", "") -> error "specify either --from or --to" - ("", to) -> moveTo file - (from, "") -> moveFrom file + ("", to) -> moveToStart file + (from, "") -> moveFromStart file (_, _) -> error "only one of --from or --to can be specified" {- Moves the content of an annexed file to another repository, @@ -299,34 +347,42 @@ moveCmd file = do - - If the destination already has the content, it is still removed - from the current repository. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. -} -moveTo :: FilePath -> Annex () -moveTo file = isAnnexed file $ \(key, backend) -> do +moveToStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveToStart file = isAnnexed file $ \(key, backend) -> do ishere <- inAnnex key if (not ishere) - then return () -- not here, so nothing to do - else do - showStart "move" file - remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key - case isthere of - Left err -> error (show err) - Right False -> moveit remote key - Right True -> removeit remote key + then return Nothing -- not here, so nothing to do + else return $ Just $ moveToPerform file key +moveToPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +moveToPerform file key = do + -- checking the remote is expensive, so not done in the start step + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case isthere of + Left err -> error (show err) + Right False -> moveit remote key + Right True -> removeit remote key where moveit remote key = do Remotes.copyToRemote remote key removeit remote key removeit remote key = do error "TODO: drop key from local" - -- Update local location log; key is present - -- there and missing here. - logStatus key ValueMissing - u <- getUUID remote - liftIO $ logChange remote key u ValuePresent - -- Propigate location log to remote. - error "TODO: update remote locationlog" - showEndOk + return $ Just $ moveToCleanup remote key +moveToCleanup :: Git.Repo -> Key -> Annex Bool +moveToCleanup remote key = do + -- Update local location log; key is present there and missing here. + logStatus key ValueMissing + u <- getUUID remote + liftIO $ logChange remote key u ValuePresent + -- Propigate location log to remote. + error "TODO: update remote locationlog" + return True {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. @@ -334,33 +390,42 @@ moveTo file = isAnnexed file $ \(key, backend) -> do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFrom :: FilePath -> Annex () -moveFrom file = isAnnexed file $ \(key, backend) -> do - showStart "move" file -- have to show this before checking remote - ishere <- inAnnex key +moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveFromStart file = isAnnexed file $ \(key, backend) -> do + return $ Just $ moveFromPerform file key +moveFromPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +moveFromPerform file key = do + -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key + ishere <- inAnnex key case (ishere, isthere) of (_, Left err) -> error (show err) - (_, Right False) -> showEndFail + (_, Right False) -> return Nothing -- not in remote; fail (False, Right True) -> moveit remote key (True, Right True) -> removeit remote key where moveit remote key = do - getViaTmp key (Remotes.copyFromRemote remote key) - removeit remote key + ok <- getViaTmp key (Remotes.copyFromRemote remote key) + if (ok) + then removeit remote key + else return Nothing -- fail removeit remote key = do error $ "TODO remove" ++ file - showEndOk + return $ Just moveFromCleanup +moveFromCleanup :: Annex Bool +moveFromCleanup = do + error "update location logs" + return True -- helpers notAnnexed file a = do r <- Backend.lookupFile file case (r) of - Just v -> return () + Just v -> return Nothing Nothing -> a isAnnexed file a = do r <- Backend.lookupFile file case (r) of Just v -> a v - Nothing -> return () + Nothing -> return Nothing diff --git a/Core.hs b/Core.hs index 1d887792ad..a97bf5090d 100644 --- a/Core.hs +++ b/Core.hs @@ -110,6 +110,8 @@ getViaTmp key action = do logStatus key ValuePresent return True else do + -- the tmp file is left behind, in case caller wants + -- to resume its transfer return False {- Output logging -} diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn index 178f95021b..15e5171ca3 100644 --- a/doc/todo/parallel_possibilities.mdwn +++ b/doc/todo/parallel_possibilities.mdwn @@ -5,9 +5,8 @@ heavily and mostly runs other git commands, maybe not a whole lot. Anyway, each git-annex command is broken down into a series of independant actions, which has some potential for parallelism. -Probably they would need to be split further. Each action currently has 3 -distinct phases, basically "check", "do", and "record". If the check action -returned a do action that returned a record action, then it could easily -make sense to parallelize the check actions and start on the do actions -(which probably won't parallelize well) while they are still being -generated, and possibly parallelize the record actions at the end. +Each action has 3 distinct phases, basically "check", "perform", and +"cleanup". The perform actions are not parellizable; the cleanup may be, +and the check should be easily parallelizable, although they may access the +disk or run minor git query commands, so would probably not want to run +too many of them at once. From 1f3088fb945bbe241e5309a961a5b2df0fa9f93a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 15:46:53 -0400 Subject: [PATCH 0289/8313] wording --- Commands.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Commands.hs b/Commands.hs index f4286abf5b..cda34d1bdf 100644 --- a/Commands.hs +++ b/Commands.hs @@ -25,17 +25,17 @@ import qualified Remotes import qualified TypeInternals {- A subcommand runs in three stages. Each stage can return the next stage - - to run. + - to run. - - 1. The start stage is run before anything is printed about the - subcommand, and can early abort it if the input does not make sense. - It should run quickly and should not modify Annex state. - - 2. The perform stage is run after a message is printed about the subcommand - - being run. + - being run, and it should be where the bulk of the work happens. - - - 3. The cleanup stage is run only if the do stage succeeds, and it returns - - the overall success/fail of the subcommand. + - 3. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the subcommand. -} type SubCmdStart = String -> Annex (Maybe SubCmdPerform) type SubCmdPerform = Annex (Maybe SubCmdCleanup) From 3f0de706dd37f6b50db224bef19139eb780afef0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 15:49:52 -0400 Subject: [PATCH 0290/8313] comments --- Commands.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index cda34d1bdf..9c35c22e18 100644 --- a/Commands.hs +++ b/Commands.hs @@ -59,14 +59,16 @@ doSubCmd cmdname start param = do else showEndFail +{- A subcommand can broadly want one of several kinds of input parameters. + - This allows a first stage of filtering before starting a subcommand. -} data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description + data SubCommand = Command { subcmdname :: String, subcmdaction :: SubCmdStart, subcmdwants :: SubCmdWants, subcmddesc :: String } - subCmds :: [SubCommand] subCmds = [ (Command "add" addStart FilesNotInGit @@ -108,6 +110,7 @@ options = [ header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) +{- Usage message with lists of options and subcommands. -} usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where From 1aa19422ac8748eeff219ac4f46df166dae783c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 17:17:03 -0400 Subject: [PATCH 0291/8313] git annex move --from remote almost working --- Commands.hs | 69 ++++++++++++++++++++++++++++---------------------- LocationLog.hs | 4 ++- Locations.hs | 3 +++ Remotes.hs | 44 +++++++++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 32 deletions(-) diff --git a/Commands.hs b/Commands.hs index 9c35c22e18..cf05164636 100644 --- a/Commands.hs +++ b/Commands.hs @@ -367,25 +367,28 @@ moveToPerform file key = do remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key case isthere of - Left err -> error (show err) - Right False -> moveit remote key - Right True -> removeit remote key - where - moveit remote key = do - Remotes.copyToRemote remote key - removeit remote key - removeit remote key = do - error "TODO: drop key from local" - return $ Just $ moveToCleanup remote key + Left err -> do + showNote $ show err + return Nothing + Right False -> do + ok <- Remotes.copyToRemote remote key + if (ok) + then return $ Just $ moveToCleanup remote key + else return Nothing -- failed + Right True -> return $ Just $ moveToCleanup remote key moveToCleanup :: Git.Repo -> Key -> Annex Bool moveToCleanup remote key = do - -- Update local location log; key is present there and missing here. - logStatus key ValueMissing - u <- getUUID remote - liftIO $ logChange remote key u ValuePresent - -- Propigate location log to remote. - error "TODO: update remote locationlog" - return True + -- cleanup on the local side is the same as done for the drop subcommand + ok <- dropCleanup key + if (not ok) + then return False + else do + -- Record that the key is present on the remote. + u <- getUUID remote + liftIO $ logChange remote key u ValuePresent + -- Propigate location log to remote. + error "TODO: update remote locationlog" + return True {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. @@ -403,22 +406,28 @@ moveFromPerform file key = do isthere <- Remotes.inAnnex remote key ishere <- inAnnex key case (ishere, isthere) of - (_, Left err) -> error (show err) - (_, Right False) -> return Nothing -- not in remote; fail - (False, Right True) -> moveit remote key - (True, Right True) -> removeit remote key - where - moveit remote key = do + (_, Left err) -> do + showNote $ show err + return Nothing + (_, Right False) -> do + showNote $ "not present in " ++ (Git.repoDescribe remote) + return Nothing + (False, Right True) -> do + -- copy content from remote ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) - then removeit remote key + then return $ Just $ moveFromCleanup remote key else return Nothing -- fail - removeit remote key = do - error $ "TODO remove" ++ file - return $ Just moveFromCleanup -moveFromCleanup :: Annex Bool -moveFromCleanup = do - error "update location logs" + (True, Right True) -> do + -- the content is already here, just remove from remote + return $ Just $ moveFromCleanup remote key +moveFromCleanup :: Git.Repo -> Key -> Annex Bool +moveFromCleanup remote key = do + Remotes.removeRemoteFile remote $ annexLocation remote key + -- Record that the key is not on the remote. + u <- getUUID remote + liftIO $ logChange remote key u ValueMissing + Remotes.updateRemoteLogStatus remote key return True -- helpers diff --git a/LocationLog.hs b/LocationLog.hs index 785b3330db..9ec71ce232 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -19,7 +19,9 @@ module LocationLog ( LogStatus(..), logChange, - keyLocations + keyLocations, + logFile, + readLog ) where import Data.Time.Clock.POSIX diff --git a/Locations.hs b/Locations.hs index 18d416eb4a..92918a7e0c 100644 --- a/Locations.hs +++ b/Locations.hs @@ -28,6 +28,9 @@ gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc - - - That allows deriving the key and backend by looking at the symlink to it. + - + - Note that even if the repo is a bare repo, the annex is put in a .git + - sub -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = diff --git a/Remotes.hs b/Remotes.hs index aec38a363e..67ebd75f97 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -7,7 +7,9 @@ module Remotes ( inAnnex, commandLineRemote, copyFromRemote, - copyToRemote + copyToRemote, + removeRemoteFile, + updateRemoteLogStatus ) where import Control.Exception @@ -16,8 +18,11 @@ import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils import Data.Either.Utils +import System.Cmd.Utils +import System.Directory import List import Maybe +import IO (hPutStrLn) import Types import qualified GitRepo as Git @@ -223,3 +228,40 @@ copyToRemote r key = do location g = annexLocation g key sshlocation = (Git.urlHost r) ++ ":" ++ file file = error "TODO" + +{- Removes a file from a remote. -} +removeRemoteFile :: Git.Repo -> FilePath -> Annex () +removeRemoteFile r file = do + if (not $ Git.repoIsUrl r) + then liftIO $ removeFile file + else if (Git.repoIsSsh r) + then do + ok <- liftIO $ boolSystem "ssh" + [Git.urlHost r, "rm -f " ++ + (shellEscape file)] + if (ok) + then return () + else error "failed to remove file from remote" + else error "removing file from non-ssh repo not supported" + +{- Update's a remote's location log for a key, by merging the local + - location log into it. -} +updateRemoteLogStatus :: Git.Repo -> Key -> Annex () +updateRemoteLogStatus r key = do + -- To merge, just append data to the remote's + -- log. Since the log is timestamped, the presumably newer + -- information from the local will superscede the older + -- information in the remote's log. + -- TODO: remote log locking + let mergecmd = "cat >> " ++ (shellEscape $ logFile r key) ++ " && " ++ + "cd " ++ (shellEscape $ Git.workTree r) ++ " && " ++ + "git add " ++ (shellEscape $ gitStateDir r) + let shellcmd = if (not $ Git.repoIsUrl r) + then pOpen WriteToPipe "sh" ["-c", mergecmd] + else if (Git.repoIsSsh r) + then pOpen WriteToPipe "ssh" [Git.urlHost r, mergecmd] + else error "updating non-ssh repo not supported" + g <- Annex.gitRepo + liftIO $ shellcmd $ \h -> do + lines <- readLog $ logFile g key + hPutStrLn h $ unlines $ map show lines From 8beed17168aab12bb4045b6d8635b37503d5099b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 17:31:07 -0400 Subject: [PATCH 0292/8313] drop incomplete bare repo support Added a bug about it. Now git annex move --from works fully --- Locations.hs | 14 +++++++------- Remotes.hs | 34 +++++++++++++--------------------- doc/bugs/bare_git_repos.mdwn | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 doc/bugs/bare_git_repos.mdwn diff --git a/Locations.hs b/Locations.hs index 92918a7e0c..c2c747e582 100644 --- a/Locations.hs +++ b/Locations.hs @@ -28,22 +28,22 @@ gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc - - - That allows deriving the key and backend by looking at the symlink to it. - - - - Note that even if the repo is a bare repo, the annex is put in a .git - - sub -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) -{- Annexed file's location relative to git's working tree. -} +{- Annexed file's location relative to git's working tree. + - + - Note: Assumes repo is NOT bare.-} annexLocationRelative :: Git.Repo -> Key -> FilePath -annexLocationRelative r key = Git.dir r ++ "/annex/" ++ (keyFile key) +annexLocationRelative r key = ".git/annex/" ++ (keyFile key) {- .git-annex/tmp is used for temp files - -} + - + - Note: Assumes repo is NOT bare. -} annexTmpLocation :: Git.Repo -> FilePath -annexTmpLocation r = (Git.workTree r) ++ "/" ++ Git.dir r ++ "/annex/tmp/" +annexTmpLocation r = (Git.workTree r) ++ ".git/annex/tmp/" {- Converts a key into a filename fragment. - diff --git a/Remotes.hs b/Remotes.hs index 67ebd75f97..c9c65babeb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -188,27 +188,19 @@ tryGitConfigRead r = do {- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file = do - -- annexLocation needs the git config to have been read for a remote, - -- so do that now if it hasn't been already - result <- tryGitConfigRead r - case (result) of - Left err -> return False - Right from -> copy from + Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + if (not $ Git.repoIsUrl r) + then getlocal + else if (Git.repoIsSsh r) + then getssh + else error "copying from non-ssh repo not supported" where - copy from = do - Core.showNote $ "copying from " ++ (Git.repoDescribe from) ++ "..." - if (not $ Git.repoIsUrl from) - then getlocal - else if (Git.repoIsSsh from) - then getssh - else error "copying from non-ssh repo not supported" - where - getlocal = liftIO $ boolSystem "cp" ["-a", location, file] - getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation from key - sshlocation = (Git.urlHost from) ++ ":" ++ location + getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getssh = do + liftIO $ putStrLn "" -- make way for scp progress bar + liftIO $ boolSystem "scp" [sshlocation, file] + location = annexLocation r key + sshlocation = (Git.urlHost r) ++ ":" ++ location {- Tries to copy a key's content to a remote. -} copyToRemote :: Git.Repo -> Key -> Annex Bool @@ -255,7 +247,7 @@ updateRemoteLogStatus r key = do -- TODO: remote log locking let mergecmd = "cat >> " ++ (shellEscape $ logFile r key) ++ " && " ++ "cd " ++ (shellEscape $ Git.workTree r) ++ " && " ++ - "git add " ++ (shellEscape $ gitStateDir r) + "git add " ++ (shellEscape $ stateLoc) let shellcmd = if (not $ Git.repoIsUrl r) then pOpen WriteToPipe "sh" ["-c", mergecmd] else if (Git.repoIsSsh r) diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn new file mode 100644 index 0000000000..e24d7a7ee3 --- /dev/null +++ b/doc/bugs/bare_git_repos.mdwn @@ -0,0 +1,14 @@ +It would be nice if git-annex could be used in bare git repos. +However, that is not currently supported. Problems include: + +* git-annex often does not read a git repo's config before touching it, + so it doesn't know if the repo is bare or not + (reading the config when operating on ssh repos would be a pain and SLOW; + I had some of that code in as of 1aa19422ac8748eeff219ac4f46df166dae783c5, + but ripped it all out) +* .. which results in creating `.git/annex` in a bare repo, which mightily + confuses git (so it will complain that the bare repo is not + a git repo at all!) +* `.git-annex/` needs to have state recorded to it and committed, and that + is not possible with a bare repo. (If [[todo/branching]] were done, + that might be fixed.) From d0a9cdadafca1ee0da100a993b23e8a063f86bf8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 18:32:29 -0400 Subject: [PATCH 0293/8313] add dropkey subcommand and --quiet Needed for better git annex move --from --- Commands.hs | 51 +++++++++++++++++++++++++++++++++++-------- Core.hs | 14 +++++++----- GitRepo.hs | 2 +- Remotes.hs | 54 +++++++++++++++------------------------------- TypeInternals.hs | 9 ++++---- Types.hs | 1 + doc/git-annex.mdwn | 12 +++++++++++ 7 files changed, 87 insertions(+), 56 deletions(-) diff --git a/Commands.hs b/Commands.hs index cf05164636..2b8da585e3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -61,7 +61,8 @@ doSubCmd cmdname start param = do {- A subcommand can broadly want one of several kinds of input parameters. - This allows a first stage of filtering before starting a subcommand. -} -data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description +data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing + | Description | Keys data SubCommand = Command { subcmdname :: String, @@ -87,6 +88,8 @@ subCmds = [ "fix up files' symlinks to point to annexed content") , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") + , (Command "dropkey" fromKeyStart Keys + "drops cached content for specified keys") ] -- Each dashed command-line option results in generation of an action @@ -95,6 +98,8 @@ options :: [OptDescr (Annex ())] options = [ Option ['f'] ["force"] (NoArg (storebool "force" True)) "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) + "avoid verbose output" , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") @@ -127,6 +132,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Generate descriptions of wanted parameters for subcommands. -} descWanted :: SubCmdWants -> String descWanted Description = "DESCRIPTION" +descWanted Keys = "KEY ..." descWanted _ = "PATH ..." {- Finds the type of parameters a subcommand wants, from among the passed @@ -147,6 +153,7 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] +findWanted Keys params _ = return params {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -243,9 +250,9 @@ dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing - else return $ Just $ dropPerform file key backend -dropPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) -dropPerform file key backend = do + else return $ Just $ dropPerform key backend +dropPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key @@ -262,6 +269,29 @@ dropCleanup key = do return True else return True +{- Drops cached content for a key. -} +dropKeyStart :: String -> Annex (Maybe SubCmdPerform) +dropKeyStart keyname = do + backends <- Backend.list + let key = genKey (backends !! 0) keyname + present <- inAnnex key + force <- Annex.flagIsSet "force" + if (not present) + then return Nothing + else if (not force) + then error "dropkey is can cause data loss; use --force if you're sure you want to do this" + else return $ Just $ dropKeyPerform key +dropKeyPerform :: Key -> Annex (Maybe SubCmdCleanup) +dropKeyPerform key = do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return $ Just $ dropKeyCleanup key +dropKeyCleanup :: Key -> Annex Bool +dropKeyCleanup key = do + logStatus key ValueMissing + return True + {- Fixes the symlink to an annexed file. -} fixStart :: FilePath -> Annex (Maybe SubCmdPerform) fixStart file = isAnnexed file $ \(key, backend) -> do @@ -423,11 +453,14 @@ moveFromPerform file key = do return $ Just $ moveFromCleanup remote key moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - Remotes.removeRemoteFile remote $ annexLocation remote key - -- Record that the key is not on the remote. - u <- getUUID remote - liftIO $ logChange remote key u ValueMissing - Remotes.updateRemoteLogStatus remote key + -- Force drop content from the remote. + Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", + "--backend=" ++ (backendName key), + keyName key] + -- Record locally that the key is not on the remote. + remoteuuid <- getUUID remote + g <- Annex.gitRepo + liftIO $ logChange g key remoteuuid ValueMissing return True -- helpers diff --git a/Core.hs b/Core.hs index a97bf5090d..8717aee818 100644 --- a/Core.hs +++ b/Core.hs @@ -115,22 +115,26 @@ getViaTmp key action = do return False {- Output logging -} +verbose :: Annex () -> Annex () +verbose a = do + q <- Annex.flagIsSet "quiet" + if (q) then return () else a showStart :: String -> String -> Annex () -showStart command file = do +showStart command file = verbose $ do liftIO $ putStr $ command ++ " " ++ file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () -showNote s = do +showNote s = verbose $ do liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout showLongNote :: String -> Annex () -showLongNote s = do +showLongNote s = verbose $ do liftIO $ putStr $ "\n" ++ (indent s) where indent s = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () -showEndOk = do +showEndOk = verbose $ do liftIO $ putStrLn "ok" showEndFail :: Annex () -showEndFail = do +showEndFail = verbose $ do liftIO $ putStrLn "\nfailed" diff --git a/GitRepo.hs b/GitRepo.hs index 553e91fec0..ee1bdba344 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -156,7 +156,7 @@ workTree repo = - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} relative :: Repo -> String -> String -relative repo file = drop (length absrepo) absfile +relative repo file = assertLocal repo $ drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file diff --git a/Remotes.hs b/Remotes.hs index c9c65babeb..985199e1cd 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -8,11 +8,11 @@ module Remotes ( commandLineRemote, copyFromRemote, copyToRemote, - removeRemoteFile, - updateRemoteLogStatus + runCmd ) where -import Control.Exception +import IO (bracket_) +import Control.Exception hiding (bracket_) import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map @@ -20,9 +20,9 @@ import Data.String.Utils import Data.Either.Utils import System.Cmd.Utils import System.Directory +import System.Posix.Directory import List import Maybe -import IO (hPutStrLn) import Types import qualified GitRepo as Git @@ -221,39 +221,19 @@ copyToRemote r key = do sshlocation = (Git.urlHost r) ++ ":" ++ file file = error "TODO" -{- Removes a file from a remote. -} -removeRemoteFile :: Git.Repo -> FilePath -> Annex () -removeRemoteFile r file = do +{- Runs a command in a remote. -} +runCmd :: Git.Repo -> String -> [String] -> Annex Bool +runCmd r command params = do if (not $ Git.repoIsUrl r) - then liftIO $ removeFile file + then do + cwd <- liftIO $ getCurrentDirectory + liftIO $ bracket_ (changeWorkingDirectory (Git.workTree r)) + (\_ -> changeWorkingDirectory cwd) $ + boolSystem command params else if (Git.repoIsSsh r) then do - ok <- liftIO $ boolSystem "ssh" - [Git.urlHost r, "rm -f " ++ - (shellEscape file)] - if (ok) - then return () - else error "failed to remove file from remote" - else error "removing file from non-ssh repo not supported" - -{- Update's a remote's location log for a key, by merging the local - - location log into it. -} -updateRemoteLogStatus :: Git.Repo -> Key -> Annex () -updateRemoteLogStatus r key = do - -- To merge, just append data to the remote's - -- log. Since the log is timestamped, the presumably newer - -- information from the local will superscede the older - -- information in the remote's log. - -- TODO: remote log locking - let mergecmd = "cat >> " ++ (shellEscape $ logFile r key) ++ " && " ++ - "cd " ++ (shellEscape $ Git.workTree r) ++ " && " ++ - "git add " ++ (shellEscape $ stateLoc) - let shellcmd = if (not $ Git.repoIsUrl r) - then pOpen WriteToPipe "sh" ["-c", mergecmd] - else if (Git.repoIsSsh r) - then pOpen WriteToPipe "ssh" [Git.urlHost r, mergecmd] - else error "updating non-ssh repo not supported" - g <- Annex.gitRepo - liftIO $ shellcmd $ \h -> do - lines <- readLog $ logFile g key - hPutStrLn h $ unlines $ map show lines + liftIO $ boolSystem "ssh" [Git.urlHost r, + "cd " ++ (shellEscape $ Git.workTree r) ++ + " && " ++ command ++ " " ++ + unwords params] + else error "running command in non-ssh repo not supported" diff --git a/TypeInternals.hs b/TypeInternals.hs index 6d1c72d2ea..188f5e534b 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -31,12 +31,12 @@ data AnnexState = AnnexState { type Annex = StateT AnnexState IO -- annexed filenames are mapped through a backend into keys -type KeyFrag = String +type KeyName = String type BackendName = String -data Key = Key (BackendName, KeyFrag) deriving (Eq) +data Key = Key (BackendName, KeyName) deriving (Eq) -- constructs a key in a backend -genKey :: Backend -> KeyFrag -> Key +genKey :: Backend -> KeyName -> Key genKey b f = Key (name b,f) -- show a key to convert it to a string; the string includes the @@ -51,9 +51,10 @@ instance Read Key where b = l !! 0 k = join ":" $ drop 1 l --- pulls the backend name out backendName :: Key -> BackendName backendName (Key (b,k)) = b +keyName :: Key -> KeyName +keyName (Key (b,k)) = k -- this structure represents a key-value backend data Backend = Backend { diff --git a/Types.hs b/Types.hs index 50597962ce..c3d6467a3f 100644 --- a/Types.hs +++ b/Types.hs @@ -7,6 +7,7 @@ module Types ( Key, genKey, backendName, + keyName, FlagName, Flag(..) ) where diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 522be75700..e7057afeea 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -116,6 +116,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile +* dropkey [key ...] + + Drops the cached data for the specified keys from this repository. + + This can be used to drop content for arbitrary keys, which do not need + to have a file in the git repository pointing at them. + # OPTIONS * --force @@ -123,6 +130,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Force unsafe actions, such as dropping a file's content when no other source of it can be verified to still exist. Use with care. +* --quiet + + Avoid the default verbose logging of what is done; only show errors + and progress displays. + * --backend=name Specify the default key-value backend to use, adding it to the front From 3b6b9ab4e1cbe3e174418e032795a598ab11fb8e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 18:33:59 -0400 Subject: [PATCH 0294/8313] typo --- Commands.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Commands.hs b/Commands.hs index 2b8da585e3..78e1ab32cf 100644 --- a/Commands.hs +++ b/Commands.hs @@ -88,7 +88,7 @@ subCmds = [ "fix up files' symlinks to point to annexed content") , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") - , (Command "dropkey" fromKeyStart Keys + , (Command "dropkey" dropKeyStart Keys "drops cached content for specified keys") ] From 47892ced883b96c3a9c2903aa8a59b3b8a2f1731 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 18:36:41 -0400 Subject: [PATCH 0295/8313] new bug --- doc/bugs/error_propigation.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/error_propigation.mdwn diff --git a/doc/bugs/error_propigation.mdwn b/doc/bugs/error_propigation.mdwn new file mode 100644 index 0000000000..0a0b38f5ee --- /dev/null +++ b/doc/bugs/error_propigation.mdwn @@ -0,0 +1,3 @@ +If a subcommand fails w/o throwing an error, no error is propigated to the +git-annex exit code. With --quiet, this makes it look like the command +succeeded. From e87287c11b81ea6f339628bcbebfb239d0ccadd0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 19:17:11 -0400 Subject: [PATCH 0296/8313] fix failure propigation --- Commands.hs | 26 ++++++++++++++++++-------- Core.hs | 7 +++++-- doc/bugs/error_propigation.mdwn | 2 +- git-annex.hs | 5 +++-- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Commands.hs b/Commands.hs index 78e1ab32cf..729eae1240 100644 --- a/Commands.hs +++ b/Commands.hs @@ -42,21 +42,27 @@ type SubCmdPerform = Annex (Maybe SubCmdCleanup) type SubCmdCleanup = Annex Bool {- Runs a subcommand through its three stages. -} -doSubCmd :: String -> SubCmdStart -> String -> Annex () +doSubCmd :: String -> SubCmdStart -> String -> Annex Bool doSubCmd cmdname start param = do res <- start param :: Annex (Maybe SubCmdPerform) case (res) of - Nothing -> return () + Nothing -> return True Just perform -> do showStart cmdname param res <- perform :: Annex (Maybe SubCmdCleanup) case (res) of - Nothing -> showEndFail + Nothing -> do + showEndFail + return False Just cleanup -> do res <- cleanup if (res) - then showEndOk - else showEndFail + then do + showEndOk + return True + else do + showEndFail + return False {- A subcommand can broadly want one of several kinds of input parameters. @@ -159,7 +165,7 @@ findWanted Keys params _ = return params - run in the Annex monad. The first actions configure it - according to command line options, while the second actions - handle subcommands. -} -parseCmd :: [String] -> AnnexState -> IO ([Annex ()], [Annex ()]) +parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) parseCmd argv state = do (flags, params) <- getopt if (null params) @@ -169,8 +175,12 @@ parseCmd argv state = do [Command name action want _] -> do f <- findWanted want (drop 1 params) (TypeInternals.repo state) - return (flags, map (doSubCmd name action) $ - filter notstate f) + let actions = map (doSubCmd name action) $ + filter notstate f + let configactions = map (\f -> do + f + return True) flags + return (configactions, actions) where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f diff --git a/Core.hs b/Core.hs index 8717aee818..0d95e382b0 100644 --- a/Core.hs +++ b/Core.hs @@ -18,14 +18,15 @@ import qualified Annex import Utility {- Sets up a git repo for git-annex. -} -startup :: Annex () +startup :: Annex Bool startup = do g <- Annex.gitRepo liftIO $ gitAttributes g prepUUID + return True {- When git-annex is done, it runs this. -} -shutdown :: Annex () +shutdown :: Annex Bool shutdown = do g <- Annex.gitRepo @@ -38,6 +39,8 @@ shutdown = do then liftIO $ removeDirectoryRecursive $ tmp else return () + return True + {- configure git to use union merge driver on state files, if it is not - already -} gitAttributes :: Git.Repo -> IO () diff --git a/doc/bugs/error_propigation.mdwn b/doc/bugs/error_propigation.mdwn index 0a0b38f5ee..25998907e8 100644 --- a/doc/bugs/error_propigation.mdwn +++ b/doc/bugs/error_propigation.mdwn @@ -1,3 +1,3 @@ If a subcommand fails w/o throwing an error, no error is propigated to the git-annex exit code. With --quiet, this makes it look like the command -succeeded. +succeeded. [[done]] diff --git a/git-annex.hs b/git-annex.hs index 602f672c5b..d7b26cd968 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -27,7 +27,7 @@ main = do - or more likely I missed an easy way to do it. So, I have to laboriously - thread AnnexState through this function. -} -tryRun :: AnnexState -> [Annex ()] -> IO () +tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions tryRun' state errnum (a:as) = do result <- try $ Annex.run state a @@ -35,7 +35,8 @@ tryRun' state errnum (a:as) = do Left err -> do showErr err tryRun' state (errnum + 1) as - Right (_,state') -> tryRun' state' errnum as + Right (True,state') -> tryRun' state' errnum as + Right (False,state') -> tryRun' state' (errnum + 1) as tryRun' state errnum [] = do if (errnum > 0) then error $ (show errnum) ++ " failed" From a8fbd5d91fc56ebedb08614bb89db2d383c9a0ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 19:34:44 -0400 Subject: [PATCH 0297/8313] speed up git annex move --from Avoid extra ssh to check if the remote has the key, just trust the location log (and propigate error if it's wrong). Quick exit when asked to move files that are not on the remote, so this is now suitable to be used on a big directory. --- Commands.hs | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 729eae1240..d201b6f799 100644 --- a/Commands.hs +++ b/Commands.hs @@ -438,32 +438,27 @@ moveToCleanup remote key = do -} moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) moveFromStart file = isAnnexed file $ \(key, backend) -> do - return $ Just $ moveFromPerform file key + g <- Annex.gitRepo + remote <- Remotes.commandLineRemote + l <- Remotes.keyPossibilities key + if (elem remote l) + then return $ Just $ moveFromPerform file key + else return Nothing moveFromPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) moveFromPerform file key = do - -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key ishere <- inAnnex key - case (ishere, isthere) of - (_, Left err) -> do - showNote $ show err - return Nothing - (_, Right False) -> do - showNote $ "not present in " ++ (Git.repoDescribe remote) - return Nothing - (False, Right True) -> do + if (ishere) + then return $ Just $ moveFromCleanup remote key + else do -- copy content from remote ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail - (True, Right True) -> do - -- the content is already here, just remove from remote - return $ Just $ moveFromCleanup remote key moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - -- Force drop content from the remote. + showNote $ "dropping from " ++ (Git.repoDescribe remote) ++ "..." Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), keyName key] From a0e8ba37c69a7ce69a6bca42b4e4a20d046b3566 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 19:38:59 -0400 Subject: [PATCH 0298/8313] changelog --- debian/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 0a27cd12ed..292931d0f3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,15 @@ git-annex (0.02) UNRELEASED; urgency=low + * New move subcommand, that can move files from or to a remote. * New fromkey subcommand, for registering urls, etc. * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. - * --from/--to can be used to limit the remote repository that git-annex + * --from/--to can be used to control the remote repository that git-annex uses. + * --quiet can be used to avoid verbose output + * New plumbing-level dropkey subcommand. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From fec9f611df1a5e973f4847ac71fe85bd85abdff4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:19:08 -0400 Subject: [PATCH 0299/8313] add setkey subcommand And finished implementing move --to --- Commands.hs | 65 ++++++++++++++++++++++++++++++++-------------- Core.hs | 4 ++- Remotes.hs | 30 ++++++++++----------- debian/changelog | 2 +- doc/git-annex.mdwn | 11 +++++++- 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/Commands.hs b/Commands.hs index d201b6f799..3c64083918 100644 --- a/Commands.hs +++ b/Commands.hs @@ -68,7 +68,7 @@ doSubCmd cmdname start param = do {- A subcommand can broadly want one of several kinds of input parameters. - This allows a first stage of filtering before starting a subcommand. -} data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing - | Description | Keys + | Description | Keys | Tempfile data SubCommand = Command { subcmdname :: String, @@ -95,7 +95,9 @@ subCmds = [ , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") , (Command "dropkey" dropKeyStart Keys - "drops cached content for specified keys") + "drops annexed content for specified keys") + , (Command "setkey" setKeyStart Tempfile + "sets annexed content for a key using a temp file") ] -- Each dashed command-line option results in generation of an action @@ -159,7 +161,7 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] -findWanted Keys params _ = return params +findWanted _ params _ = return params {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -302,6 +304,29 @@ dropKeyCleanup key = do logStatus key ValueMissing return True +{- Sets cached content for a key. -} +setKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) +setKeyStart tmpfile = do + keyname <- Annex.flagGet "key" + if (null keyname) + then error "please specify the key with --key" + else return () + backends <- Backend.list + let key = genKey (backends !! 0) keyname + return $ Just $ setKeyPerform tmpfile key +setKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +setKeyPerform tmpfile key = do + g <- Annex.gitRepo + let loc = annexLocation g key + ok <- liftIO $ boolSystem "mv" [tmpfile, loc] + if (not ok) + then error "mv failed!" + else return $ Just $ setKeyCleanup tmpfile key +setKeyCleanup :: FilePath -> Key -> Annex Bool +setKeyCleanup tmpfile key = do + logStatus key ValuePresent + return True + {- Fixes the symlink to an annexed file. -} fixStart :: FilePath -> Annex (Maybe SubCmdPerform) fixStart file = isAnnexed file $ \(key, backend) -> do @@ -411,24 +436,26 @@ moveToPerform file key = do showNote $ show err return Nothing Right False -> do - ok <- Remotes.copyToRemote remote key + let tmpfile = (annexTmpLocation remote) ++ (keyFile key) + ok <- Remotes.copyToRemote remote key tmpfile if (ok) - then return $ Just $ moveToCleanup remote key + then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed - Right True -> return $ Just $ moveToCleanup remote key -moveToCleanup :: Git.Repo -> Key -> Annex Bool -moveToCleanup remote key = do - -- cleanup on the local side is the same as done for the drop subcommand - ok <- dropCleanup key - if (not ok) - then return False - else do - -- Record that the key is present on the remote. - u <- getUUID remote - liftIO $ logChange remote key u ValuePresent - -- Propigate location log to remote. - error "TODO: update remote locationlog" - return True + Right True -> return $ Just $ dropCleanup key +moveToCleanup :: Git.Repo -> Key -> FilePath -> Annex Bool +moveToCleanup remote key tmpfile = do + -- Tell remote to use the transferred content. + Remotes.runCmd remote "git-annex" ["setkey", "--quiet", + "--backend=" ++ (backendName key), + "--key=" ++ keyName key, + tmpfile] + -- Record that the key is present on the remote. + g <- Annex.gitRepo + remoteuuid <- getUUID remote + liftIO $ logChange g key remoteuuid ValuePresent + -- Cleanup on the local side is the same as done for the + -- drop subcommand. + dropCleanup key {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. diff --git a/Core.hs b/Core.hs index 0d95e382b0..881b668e0d 100644 --- a/Core.hs +++ b/Core.hs @@ -32,12 +32,14 @@ shutdown = do liftIO $ Git.run g ["add", gitStateDir g] - -- clean up any files left in the temp directory + -- clean up any files left in the temp directory, but leave + -- the tmp directory itself let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp if (exists) then liftIO $ removeDirectoryRecursive $ tmp else return () + liftIO $ createDirectoryIfMissing True tmp return True diff --git a/Remotes.hs b/Remotes.hs index 985199e1cd..1d5992704a 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -195,31 +195,31 @@ copyFromRemote r key file = do then getssh else error "copying from non-ssh repo not supported" where - getlocal = liftIO $ boolSystem "cp" ["-a", location, file] + getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] getssh = do liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshlocation, file] - location = annexLocation r key - sshlocation = (Git.urlHost r) ++ ":" ++ location + liftIO $ boolSystem "scp" [sshLocation r keyloc, file] + keyloc = annexLocation r key -{- Tries to copy a key's content to a remote. -} -copyToRemote :: Git.Repo -> Key -> Annex Bool -copyToRemote r key = do +{- Tries to copy a key's content to a file on a remote. -} +copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool +copyToRemote r key file = do g <- Annex.gitRepo + let keyloc = annexLocation g key Core.showNote $ "copying to " ++ (Git.repoDescribe r) ++ "..." if (not $ Git.repoIsUrl r) - then sendlocal g + then putlocal keyloc else if (Git.repoIsSsh r) - then sendssh g + then putssh keyloc else error "copying to non-ssh repo not supported" where - sendlocal g = liftIO $ boolSystem "cp" ["-a", location g, file] - sendssh g = do + putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] + putssh src = do liftIO $ putStrLn "" -- make way for scp progress bar - liftIO $ boolSystem "scp" [location g, sshlocation] - location g = annexLocation g key - sshlocation = (Git.urlHost r) ++ ":" ++ file - file = error "TODO" + liftIO $ boolSystem "scp" [src, sshLocation r file] + +sshLocation :: Git.Repo -> FilePath -> FilePath +sshLocation r file = (Git.urlHost r) ++ ":" ++ file {- Runs a command in a remote. -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool diff --git a/debian/changelog b/debian/changelog index 292931d0f3..46ad2ac5db 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,7 +9,7 @@ git-annex (0.02) UNRELEASED; urgency=low * --from/--to can be used to control the remote repository that git-annex uses. * --quiet can be used to avoid verbose output - * New plumbing-level dropkey subcommand. + * New plumbing-level dropkey and setkey subcommands. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e7057afeea..cba634f205 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -118,11 +118,20 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * dropkey [key ...] - Drops the cached data for the specified keys from this repository. + Drops the annexed data for the specified keys from this repository. This can be used to drop content for arbitrary keys, which do not need to have a file in the git repository pointing at them. +* setkey file + + Sets the annxed data for a key to the content of the specified file, + and then removes the file. + + Example: + + git annex setkey --key=1287765018:3 /tmp/file + # OPTIONS * --force From 4a69cb8ade2dc4572c0e8cada6bc30f38ed6bc30 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:20:17 -0400 Subject: [PATCH 0300/8313] done --- doc/todo/pushpull.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/pushpull.mdwn b/doc/todo/pushpull.mdwn index 47da2107f2..6828b35b2f 100644 --- a/doc/todo/pushpull.mdwn +++ b/doc/todo/pushpull.mdwn @@ -1,2 +1,4 @@ --push/--pull should take a reponame and files, and push those files to that repo; dropping them from the current repo + +[[done]] (move --from/--to) From 0a4235e26e0bce14ee8d65de9e0480d23b87ed87 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:22:37 -0400 Subject: [PATCH 0301/8313] update --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 46ad2ac5db..fd1fddafda 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,15 +1,15 @@ git-annex (0.02) UNRELEASED; urgency=low - * New move subcommand, that can move files from or to a remote. - * New fromkey subcommand, for registering urls, etc. * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. + * New move subcommand, that can move files from or to a remote. + * New fromkey subcommand, for registering urls, etc. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. * --from/--to can be used to control the remote repository that git-annex uses. * --quiet can be used to avoid verbose output - * New plumbing-level dropkey and setkey subcommands. + * New plumbing-level dropkey and addkey subcommands. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From 0788c12ffe9c07104ca70f4dc0636ed0f6879ad2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:45:50 -0400 Subject: [PATCH 0302/8313] improve git annex move in walkthrough --- doc/walkthrough.mdwn | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index c9eba0a575..15d19db2b2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -78,7 +78,7 @@ USB drive. Notice that you had to git pull from home first, this lets git-annex know what has changed in home, and so it knows about the files present there and -can get them. See below for an easier way. +can get them. ## transferring files: When things go wrong @@ -137,17 +137,6 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. -## moving file content to another repository - -Often you will want to transfer some file contents from a repository to -some other one, and then drop it from the first repository. For example, -your laptop's disk is getting full; time to move some files to an external -disk. Doing that by hand is possible, but a bit of a pain. `git annex move` -makes it very easy. - - # git annex move my_cool_big_file --to usbdrive - move my_cool_big_file (to usbdrive...) ok - ## using ssh remotes So far git-annex has been used with a remote repository on a USB drive. @@ -183,6 +172,21 @@ access, if available. There is a annex-cost setting you can configure in Also, note that you need full shell access for this to work -- git-annex needs to be able to ssh in and run commands. +## moving file content to another repository + +Often you will want to move some file contents from a repository to some +other one. For example, your laptop's disk is getting full; time to move +some files to an external disk before moving another file from home to your +laptop. Doing that by hand (by using `git annex get` and `git annex drop`) +is possible, but a bit of a pain. `git annex move` makes it very easy. + + # git annex move my_cool_big_file --to usbdrive + move my_cool_big_file (copying to usbdrive...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --from home + move video/hackity_hack_and_kaxxt.mov (copying from home...)a + WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 + (dropping from home...) ok + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 99eaf41da57819ed49e554b09a61b74ac91c7ddb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:48:32 -0400 Subject: [PATCH 0303/8313] better messages --- Commands.hs | 4 ++-- Remotes.hs | 2 -- doc/git-annex.mdwn | 2 +- doc/walkthrough.mdwn | 6 +++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Commands.hs b/Commands.hs index 3c64083918..2941d619e1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -436,6 +436,7 @@ moveToPerform file key = do showNote $ show err return Nothing Right False -> do + Core.showNote $ "moving to " ++ (Git.repoDescribe r) ++ "..." let tmpfile = (annexTmpLocation remote) ++ (keyFile key) ok <- Remotes.copyToRemote remote key tmpfile if (ok) @@ -478,14 +479,13 @@ moveFromPerform file key = do if (ishere) then return $ Just $ moveFromCleanup remote key else do - -- copy content from remote + Core.showNote $ "moving from " ++ (Git.repoDescribe r) ++ "..." ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - showNote $ "dropping from " ++ (Git.repoDescribe remote) ++ "..." Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), keyName key] diff --git a/Remotes.hs b/Remotes.hs index 1d5992704a..81a4af47ac 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -188,7 +188,6 @@ tryGitConfigRead r = do {- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file = do - Core.showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." if (not $ Git.repoIsUrl r) then getlocal else if (Git.repoIsSsh r) @@ -206,7 +205,6 @@ copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyToRemote r key file = do g <- Annex.gitRepo let keyloc = annexLocation g key - Core.showNote $ "copying to " ++ (Git.repoDescribe r) ++ "..." if (not $ Git.repoIsUrl r) then putlocal keyloc else if (Git.repoIsSsh r) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index cba634f205..3645d37f2a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -53,7 +53,7 @@ content from the key-value store. # git commit -a -m "freed up space" # git annex move iso --to=usbdrive - move iso/Debian_5.0.iso (to usbdrive...) ok + move iso/Debian_5.0.iso (moving to usbdrive...) ok # SUBCOMMANDS diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 15d19db2b2..ce68a77f7a 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -181,11 +181,11 @@ laptop. Doing that by hand (by using `git annex get` and `git annex drop`) is possible, but a bit of a pain. `git annex move` makes it very easy. # git annex move my_cool_big_file --to usbdrive - move my_cool_big_file (copying to usbdrive...) ok + move my_cool_big_file (moving to usbdrive...) ok # git annex move video/hackity_hack_and_kaxxt.mov --from home - move video/hackity_hack_and_kaxxt.mov (copying from home...)a + move video/hackity_hack_and_kaxxt.mov (moving from home...) WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 - (dropping from home...) ok + ok ## using the URL backend From 5d4ff035ba0f5a9bd9c61def3effd68bfeb02448 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 20:52:03 -0400 Subject: [PATCH 0304/8313] bugfix --- Commands.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Commands.hs b/Commands.hs index 2941d619e1..da4d840ae1 100644 --- a/Commands.hs +++ b/Commands.hs @@ -436,7 +436,7 @@ moveToPerform file key = do showNote $ show err return Nothing Right False -> do - Core.showNote $ "moving to " ++ (Git.repoDescribe r) ++ "..." + Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." let tmpfile = (annexTmpLocation remote) ++ (keyFile key) ok <- Remotes.copyToRemote remote key tmpfile if (ok) @@ -479,7 +479,7 @@ moveFromPerform file key = do if (ishere) then return $ Just $ moveFromCleanup remote key else do - Core.showNote $ "moving from " ++ (Git.repoDescribe r) ++ "..." + Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key From eea140c90dadfc8cc5ac4aec65c9d710f859c310 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 21:06:31 -0400 Subject: [PATCH 0305/8313] handle better the case of a disconnected drive remote --- Backend/File.hs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 14b4b9dae5..4b7d7917c0 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,6 +15,7 @@ import System.IO import System.Cmd import System.Cmd.Utils import Control.Exception +import System.Directory import List import Maybe @@ -66,10 +67,27 @@ copyKeyFile key file = do showLocations key return False trycopy full (r:rs) = do - copied <- Remotes.copyFromRemote r key file - if (copied) - then return True + probablythere <- probablyPresent r + if (probablythere) + then do + showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + copied <- Remotes.copyFromRemote r key file + if (copied) + then return True + else trycopy full rs else trycopy full rs + probablyPresent r = do + -- This check is to avoid an ugly message if a + -- remote is a drive that is not mounted. + -- Avoid checking inAnnex for ssh remotes because + -- that is unnecessarily slow, and the locationlog + -- should be trusted. (If the ssh remote is down + -- or really lacks the file, it's ok to show + -- an ugly message before going on to the next + -- remote.) + if (not $ Git.repoIsUrl r) + then liftIO $ doesFileExist $ annexLocation r key + else return True {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an From 68e2687661cdc83197e78fd56a10803890401159 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 21:15:29 -0400 Subject: [PATCH 0306/8313] short-circuit as soon as enough remotes are verified to have a file --- Backend/File.hs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 4b7d7917c0..4273ba36be 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -111,11 +111,14 @@ checkRemoveKey key = do then return True else notEnoughCopies need have bad findcopies need have (r:rs) bad = do - haskey <- Remotes.inAnnex r key - case (haskey) of - Right True -> findcopies need (have+1) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) + if (have >= need) + then return True + else do + haskey <- Remotes.inAnnex r key + case (haskey) of + Right True -> findcopies need (have+1) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) notEnoughCopies need have bad = do unsafe showLongNote $ From 4cda7b6e7c2f08c99b0c4c14bb86e691903a985b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Oct 2010 21:35:45 -0400 Subject: [PATCH 0307/8313] bugfix --- Locations.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Locations.hs b/Locations.hs index c2c747e582..970f89aa8d 100644 --- a/Locations.hs +++ b/Locations.hs @@ -43,7 +43,7 @@ annexLocationRelative r key = ".git/annex/" ++ (keyFile key) - - Note: Assumes repo is NOT bare. -} annexTmpLocation :: Git.Repo -> FilePath -annexTmpLocation r = (Git.workTree r) ++ ".git/annex/tmp/" +annexTmpLocation r = (Git.workTree r) ++ "/.git/annex/tmp/" {- Converts a key into a filename fragment. - From ef26076a5a3df9b8740883e3f7b3b68585b47ad5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Oct 2010 15:59:50 -0400 Subject: [PATCH 0308/8313] add git queue to Annex monad not used anywhere just yet.. --- Annex.hs | 42 ++++++++++++++++++++++++++----- Core.hs | 9 +++++++ GitQueue.hs | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ TypeInternals.hs | 4 ++- 4 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 GitQueue.hs diff --git a/Annex.hs b/Annex.hs index d021f936ef..1963d19c69 100644 --- a/Annex.hs +++ b/Annex.hs @@ -11,25 +11,28 @@ module Annex ( flagIsSet, flagChange, flagGet, - Flag(..) + Flag(..), + queue, + queueGet ) where import Control.Monad.State import qualified Data.Map as M import qualified GitRepo as Git +import qualified GitQueue import Types import qualified TypeInternals as Internals -{- Create and returns an Annex state object for the specified git repo. - -} +{- Create and returns an Annex state object for the specified git repo. -} new :: Git.Repo -> [Backend] -> IO AnnexState new gitrepo allbackends = do let s = Internals.AnnexState { Internals.repo = gitrepo, Internals.backends = [], Internals.supportedBackends = allbackends, - Internals.flags = M.empty + Internals.flags = M.empty, + Internals.repoqueue = GitQueue.empty } (_,s') <- Annex.run s (prep gitrepo) return s' @@ -39,46 +42,73 @@ new gitrepo allbackends = do gitrepo' <- liftIO $ Git.configRead gitrepo Annex.gitRepoChange gitrepo' --- performs an action in the Annex monad +{- performs an action in the Annex monad -} run state action = runStateT (action) state --- Annex monad state accessors +{- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo gitRepo = do state <- get return (Internals.repo state) + +{- Changes the git repository being acted on. -} gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get put state { Internals.repo = r } return () + +{- Returns the backends being used. -} backends :: Annex [Backend] backends = do state <- get return (Internals.backends state) + +{- Sets the backends to use. -} backendsChange :: [Backend] -> Annex () backendsChange b = do state <- get put state { Internals.backends = b } return () + +{- Returns the full list of supported backends. -} supportedBackends :: Annex [Backend] supportedBackends = do state <- get return (Internals.supportedBackends state) + +{- Return True if a Bool flag is set. -} flagIsSet :: FlagName -> Annex Bool flagIsSet name = do state <- get case (M.lookup name $ Internals.flags state) of Just (FlagBool True) -> return True _ -> return False + +{- Sets the value of a flag. -} flagChange :: FlagName -> Flag -> Annex () flagChange name val = do state <- get put state { Internals.flags = M.insert name val $ Internals.flags state } return () + +{- Gets the value of a String flag (or "" if there is no such String flag) -} flagGet :: FlagName -> Annex String flagGet name = do state <- get case (M.lookup name $ Internals.flags state) of Just (FlagString s) -> return s _ -> return "" + +{- Adds a git command to the queue. -} +queue :: String -> [String] -> FilePath -> Annex () +queue subcommand params file = do + state <- get + let q = Internals.repoqueue state + put state { Internals.repoqueue = GitQueue.add q subcommand params file } + +{- Returns the queue. -} +queueGet :: Annex GitQueue.Queue +queueGet = do + state <- get + return (Internals.repoqueue state) diff --git a/Core.hs b/Core.hs index 881b668e0d..4c7c9205e6 100644 --- a/Core.hs +++ b/Core.hs @@ -14,6 +14,7 @@ import Locations import LocationLog import UUID import qualified GitRepo as Git +import qualified GitQueue import qualified Annex import Utility @@ -30,6 +31,14 @@ shutdown :: Annex Bool shutdown = do g <- Annex.gitRepo + -- Runs all queued git commands. + q <- Annex.queueGet + if (q == GitQueue.empty) + then return () + else do + liftIO $ putStrLn "Recording state in git..." + liftIO $ GitQueue.run g q + liftIO $ Git.run g ["add", gitStateDir g] -- clean up any files left in the temp directory, but leave diff --git a/GitQueue.hs b/GitQueue.hs new file mode 100644 index 0000000000..b7210ccb59 --- /dev/null +++ b/GitQueue.hs @@ -0,0 +1,64 @@ +{- git repository command queues + -} + +module GitQueue ( + Queue, + empty, + add, + run +) where + +import qualified Data.Map as M + +import qualified GitRepo as Git + +{- An action to perform in a git repository. The file to act on + - is not included, and must be able to be appended after the params. -} +data Action = Action { + subcommand :: String, + params :: [String] + } deriving (Show, Eq, Ord) + +{- A queue of actions to perform (in any order) on a git repository, + - with lists of files to perform them on. This allows coalescing + - similar git commands. -} +type Queue = M.Map Action [FilePath] + +{- Constructor for empty queue. -} +empty :: Queue +empty = M.empty + +{- Adds an action to a queue. -} +add :: Queue -> String -> [String] -> FilePath -> Queue +add queue subcommand params file = M.insertWith (++) action [file] queue + where + action = Action subcommand params + +{- Runs a queue on a git repository. -} +run :: Git.Repo -> Queue -> IO () +run repo queue = do + mapM (\(k, v) -> runAction repo k v) $ M.toList queue + return () + +{- Runs an Action on a list of files in a git repository. + - + - Complicated by commandline length limits. -} +runAction :: Git.Repo -> Action -> [FilePath] -> IO () +runAction repo action files = do + xargs [] 0 files + where + arg_max = 2048 -- TODO get better ARG_MAX + maxlen = arg_max - cmdlen + c = (subcommand action):(params action) + cmdlen = (length "git") + + (foldl (\a b -> a + b + 1) 1 $ map length c) + xargs collect _ [] = exec collect + xargs collect len (f:fs) = do + let len' = len + 1 + length f + if (len' >= maxlen) + then do + exec collect + xargs [f] (length f) fs + else xargs (f:collect) len' fs + exec [] = return () + exec fs = Git.run repo $ c ++ fs diff --git a/TypeInternals.hs b/TypeInternals.hs index 188f5e534b..d4404fd390 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -10,6 +10,7 @@ import Data.String.Utils import qualified Data.Map as M import qualified GitRepo as Git +import qualified GitQueue -- command-line flags type FlagName = String @@ -24,7 +25,8 @@ data AnnexState = AnnexState { repo :: Git.Repo, backends :: [Backend], supportedBackends :: [Backend], - flags :: M.Map FlagName Flag + flags :: M.Map FlagName Flag, + repoqueue :: GitQueue.Queue } deriving (Show) -- git-annex's monad From 24ee4439d4783cde7102f4ffd857c521367ce16f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Oct 2010 16:15:29 -0400 Subject: [PATCH 0309/8313] use git command queue --- Commands.hs | 20 ++++++++++---------- Core.hs | 5 ++--- LocationLog.hs | 6 ++++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index da4d840ae1..6018ed8223 100644 --- a/Commands.hs +++ b/Commands.hs @@ -216,7 +216,7 @@ addCleanup file key = do liftIO $ renameFile file dest link <- calcGitLink file key liftIO $ createSymbolicLink link file - liftIO $ Git.run g ["add", file] + Annex.queue "add" [] file return True {- The unannex subcommand undoes an add. -} @@ -340,11 +340,10 @@ fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] - return $ Just $ fixCleanup -fixCleanup :: Annex Bool -fixCleanup = do + return $ Just $ fixCleanup file +fixCleanup :: FilePath -> Annex Bool +fixCleanup file = do + Annex.queue "add" [] file return True {- Stores description for the repository. -} @@ -391,8 +390,7 @@ fromKeyPerform file key = do return $ Just $ fromKeyCleanup file fromKeyCleanup :: FilePath -> Annex Bool fromKeyCleanup file = do - g <- Annex.gitRepo - liftIO $ Git.run g ["add", file] + Annex.queue "add" [] file return True {- Move a file either --to or --from a repository. @@ -453,7 +451,8 @@ moveToCleanup remote key tmpfile = do -- Record that the key is present on the remote. g <- Annex.gitRepo remoteuuid <- getUUID remote - liftIO $ logChange g key remoteuuid ValuePresent + log <- liftIO $ logChange g key remoteuuid ValuePresent + Annex.queue "add" [] log -- Cleanup on the local side is the same as done for the -- drop subcommand. dropCleanup key @@ -492,7 +491,8 @@ moveFromCleanup remote key = do -- Record locally that the key is not on the remote. remoteuuid <- getUUID remote g <- Annex.gitRepo - liftIO $ logChange g key remoteuuid ValueMissing + log <- liftIO $ logChange g key remoteuuid ValueMissing + Annex.queue "add" [] log return True -- helpers diff --git a/Core.hs b/Core.hs index 4c7c9205e6..27baba28e6 100644 --- a/Core.hs +++ b/Core.hs @@ -39,8 +39,6 @@ shutdown = do liftIO $ putStrLn "Recording state in git..." liftIO $ GitQueue.run g q - liftIO $ Git.run g ["add", gitStateDir g] - -- clean up any files left in the temp directory, but leave -- the tmp directory itself let tmp = annexTmpLocation g @@ -106,7 +104,8 @@ logStatus :: Key -> LogStatus -> Annex () logStatus key status = do g <- Annex.gitRepo u <- getUUID g - liftIO $ logChange g key u status + log <- liftIO $ logChange g key u status + Annex.queue "add" [] log {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/LocationLog.hs b/LocationLog.hs index 9ec71ce232..d027c4b809 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -82,12 +82,14 @@ instance Read LogLine where undefined = ret $ LogLine (0) Undefined "" ret v = [(v, "")] -{- Log a change in the presence of a key's value in a repository. -} -logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO () +{- Log a change in the presence of a key's value in a repository, + - and returns the filename of the logfile. -} +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO (FilePath) logChange repo key uuid status = do log <- logNow status uuid ls <- readLog logfile writeLog logfile (compactLog $ log:ls) + return logfile where logfile = logFile repo key From 5b4fa4aeca9c67dc1d40dee9782a4806d7702b4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 13:12:02 -0400 Subject: [PATCH 0310/8313] use xargs --- Core.hs | 2 +- GitQueue.hs | 28 +++++++++++----------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/Core.hs b/Core.hs index 27baba28e6..13177588d0 100644 --- a/Core.hs +++ b/Core.hs @@ -36,7 +36,7 @@ shutdown = do if (q == GitQueue.empty) then return () else do - liftIO $ putStrLn "Recording state in git..." + verbose $ liftIO $ putStrLn "Recording state in git..." liftIO $ GitQueue.run g q -- clean up any files left in the temp directory, but leave diff --git a/GitQueue.hs b/GitQueue.hs index b7210ccb59..34a89f17b7 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -1,4 +1,4 @@ -{- git repository command queues +{- git repository command queue -} module GitQueue ( @@ -9,6 +9,9 @@ module GitQueue ( ) where import qualified Data.Map as M +import System.IO +import System.Cmd.Utils +import Data.String.Utils import qualified GitRepo as Git @@ -45,20 +48,11 @@ run repo queue = do - Complicated by commandline length limits. -} runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do - xargs [] 0 files + if (null files) + then return () + else runxargs where - arg_max = 2048 -- TODO get better ARG_MAX - maxlen = arg_max - cmdlen - c = (subcommand action):(params action) - cmdlen = (length "git") + - (foldl (\a b -> a + b + 1) 1 $ map length c) - xargs collect _ [] = exec collect - xargs collect len (f:fs) = do - let len' = len + 1 + length f - if (len' >= maxlen) - then do - exec collect - xargs [f] (length f) fs - else xargs (f:collect) len' fs - exec [] = return () - exec fs = Git.run repo $ c ++ fs + runxargs = pOpen WriteToPipe "xargs" + (["-0", "git", subcommand action] ++ (params action)) + feedxargs + feedxargs h = hPutStr h $ join "\0" files From 4e7c27f58bc8af8be50bca82523ad18c1dfe2a4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 13:39:45 -0400 Subject: [PATCH 0311/8313] tweak --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index 81a4af47ac..8f4fcf0f0c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -61,7 +61,7 @@ keyPossibilities key = do let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive if (not $ null doexpensive) - then Core.showNote $ "getting UUIDs for " ++ (list doexpensive) ++ "..." + then Core.showNote $ "getting UUID for " ++ (list doexpensive) ++ "..." else return () let todo = cheap ++ doexpensive if (not $ null todo) From 3281a1cb19e25f964ca9b91877a194fcb216ca5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 13:55:28 -0400 Subject: [PATCH 0312/8313] don't try to set up .gitattributes every time; only do it on git annex init --- Commands.hs | 3 ++- Core.hs | 1 - doc/git-annex.mdwn | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Commands.hs b/Commands.hs index 6018ed8223..015a2d822a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -346,7 +346,7 @@ fixCleanup file = do Annex.queue "add" [] file return True -{- Stores description for the repository. -} +{- Stores description for the repository etc. -} initStart :: String -> Annex (Maybe SubCmdPerform) initStart description = do if (null description) @@ -359,6 +359,7 @@ initPerform description = do g <- Annex.gitRepo u <- getUUID g describeUUID u description + liftIO $ gitAttributes g return $ Just $ initCleanup initCleanup :: Annex Bool initCleanup = do diff --git a/Core.hs b/Core.hs index 13177588d0..80bf56cc40 100644 --- a/Core.hs +++ b/Core.hs @@ -22,7 +22,6 @@ import Utility startup :: Annex Bool startup = do g <- Annex.gitRepo - liftIO $ gitAttributes g prepUUID return True diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3645d37f2a..417ff7e58d 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -91,7 +91,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * init description - Initializes git-annex with a description of the git repository. + Initializes git-annex with a description of the git repository, + and sets up `.gitattributes` and the pre-commit hook. This is an optional, but recommended step. * unannex [path ...] @@ -106,6 +107,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Fixes up symlinks that have become broken to again point to annexed content. This is useful to run if you have been moving the symlinks around. + You do not normally need to run this by hand since `git-annex init` + installs a pre-commit hook that automatically fixes up symlinks when + they are committed. + * fromkey file This can be used to maually set up a file to link to a specified key From 563484e1354878df4a6877e4506af0d70e41b13e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:33:44 -0400 Subject: [PATCH 0313/8313] pre-commit hook --- Commands.hs | 20 +++++++++++++++++--- Core.hs | 15 +++++++++++++++ debian/changelog | 3 +++ doc/git-annex.mdwn | 12 ++++++++---- doc/todo/symlink_farming_commit_hook.mdwn | 2 ++ doc/walkthrough.mdwn | 18 ++++++++++-------- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 015a2d822a..b6bc6dfc22 100644 --- a/Commands.hs +++ b/Commands.hs @@ -68,7 +68,7 @@ doSubCmd cmdname start param = do {- A subcommand can broadly want one of several kinds of input parameters. - This allows a first stage of filtering before starting a subcommand. -} data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing - | Description | Keys | Tempfile + | Description | Keys | Tempfile | FilesToBeCommitted data SubCommand = Command { subcmdname :: String, @@ -91,7 +91,9 @@ subCmds = [ , (Command "unannex" unannexStart FilesInGit "undo accidential add command") , (Command "fix" fixStart FilesInGit - "fix up files' symlinks to point to annexed content") + "fix up symlinks to point to annexed content") + , (Command "pre-commit" fixStart FilesToBeCommitted + "fix up symlinks before they are committed") , (Command "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") , (Command "dropkey" dropKeyStart Keys @@ -130,7 +132,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds showcmd c = (subcmdname c) ++ - (pad 10 (subcmdname c)) ++ + (pad 11 (subcmdname c)) ++ (descWanted (subcmdwants c)) ++ (pad 13 (descWanted (subcmdwants c))) ++ (subcmddesc c) @@ -161,6 +163,17 @@ findWanted FilesMissing params repo = do if (e) then return False else return True findWanted Description params _ = do return $ [unwords params] +findWanted FilesToBeCommitted params repo = do + files <- mapM gitcached params + return $ foldl (++) [] files + where + gitcached p = do + -- ask git for files staged for commit that + -- are being added, moved, or changed (but not deleted) + fs0 <- Git.pipeRead repo ["diff", "--cached", + "--name-only", "--diff-filter=ACMRT", + "-z", "HEAD", p] + return $ filter (not . null) $ split "\0" fs0 findWanted _ params _ = return params {- Parses command line and returns two lists of actions to be @@ -360,6 +373,7 @@ initPerform description = do u <- getUUID g describeUUID u description liftIO $ gitAttributes g + liftIO $ gitPreCommitHook g return $ Just $ initCleanup initCleanup :: Annex Bool initCleanup = do diff --git a/Core.hs b/Core.hs index 80bf56cc40..254bcec517 100644 --- a/Core.hs +++ b/Core.hs @@ -73,6 +73,21 @@ gitAttributes repo = do Git.run repo ["commit", "-m", "git-annex setup", attributes] +{- set up a git pre-commit hook, if one is not already present -} +gitPreCommitHook :: Git.Repo -> IO () +gitPreCommitHook repo = do + let hook = (Git.workTree repo) ++ "/" ++ (Git.dir repo) ++ + "/hooks/pre-commit" + exists <- doesFileExist hook + if (exists) + then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + else do + writeFile hook $ "#!/bin/sh\n" ++ + "# automatically configured by git-annex\n" ++ + "git annex pre-commit .\n" + p <- getPermissions hook + setPermissions hook $ p {executable = True} + {- Checks if a given key is currently present in the annexLocation. - - This can be run against a remote repository to check the key there. -} diff --git a/debian/changelog b/debian/changelog index fd1fddafda..d0922c1c5b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (0.02) UNRELEASED; urgency=low file content when dropping files. * New move subcommand, that can move files from or to a remote. * New fromkey subcommand, for registering urls, etc. + * git-annex init will now set up a pre-commit hook that fixes up symlinks + before they are committed, to ensure that moving symlinks around does not + break them. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. * --from/--to can be used to control the remote repository that git-annex diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 417ff7e58d..f7bb64988d 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -107,9 +107,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Fixes up symlinks that have become broken to again point to annexed content. This is useful to run if you have been moving the symlinks around. - You do not normally need to run this by hand since `git-annex init` - installs a pre-commit hook that automatically fixes up symlinks when - they are committed. +* pre-commit [path ...] + + Fixes up symlinks that are staged as part of a commit, to ensure they + point to annexed content. + + This is meant to be called from git's pre-commit hook. `git annex init` + automatically creates a pre-commit hook using this. * fromkey file @@ -202,7 +206,7 @@ decscriptions. You may edit it. `.git-annex/*.log` is where git-annex records its content tracking information. These files should be committed to git. -`.git-annex/.gitattributes` is configured to use git's union merge driver +`.gitattributes` is configured to use git's union merge driver to avoid conflicts when merging files in the `.git-annex` directory. # AUTHOR diff --git a/doc/todo/symlink_farming_commit_hook.mdwn b/doc/todo/symlink_farming_commit_hook.mdwn index af03beb70e..3e93cb34b8 100644 --- a/doc/todo/symlink_farming_commit_hook.mdwn +++ b/doc/todo/symlink_farming_commit_hook.mdwn @@ -10,3 +10,5 @@ up. back to git-annex. If you want to have your own shell script in the post-commit hook, just make it call `git annex` with no parameters. git-annex will detect when it's run from a git hook and do the necessary fixups. + +[[done]] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ce68a77f7a..b49a00f195 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -43,23 +43,25 @@ if you are using a centralized bare repository. add debian.iso ok # git commit -a -m added -Notice you commit at the end, this checks in git-annex's record of the -files but not their actual, large, content. +When you add a file to the annex and commit it, only a symlink to +the annexed content is committed. The content itself is stored in +git-annex's backend. ## renaming files # cd ~/annex # git mv big_file my_cool_big_file # mkdir iso - # git mv debian.iso iso - # git annex fix . - fix iso/debian.iso ok + # git mv debian.iso iso/ # git commit -m moved You can use any normal git operations to move files around, or even -make copies or delete them. `git-annex fix` needs to be run if a file -is moved into a different directory, in order to fix up the symlink -pointing to the file's content. +make copies or delete them. + +Notice that, since annexed files are represented by symlinks, +the symlink will break when the file is moved into a subdirectory. +But, git-annex will fix this up for you when you commit -- +it has a pre-commit hook that watches for and corrects broken symlinks. ## getting file content From 3874b978ab36993fe3df8b5749a6d9c53eec2bfa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:39:21 -0400 Subject: [PATCH 0314/8313] update --- debian/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index d0922c1c5b..c3255ba271 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,17 +2,20 @@ git-annex (0.02) UNRELEASED; urgency=low * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. - * New move subcommand, that can move files from or to a remote. + * New move subcommand, that makes it easy to move file contents from + or to a remote. * New fromkey subcommand, for registering urls, etc. * git-annex init will now set up a pre-commit hook that fixes up symlinks before they are committed, to ensure that moving symlinks around does not break them. + * More intelligent and fast staging of modified files; git add coalescing. * Add remote.annex-ignore git config setting to allow completly disabling a given remote. * --from/--to can be used to control the remote repository that git-annex uses. * --quiet can be used to avoid verbose output * New plumbing-level dropkey and addkey subcommands. + * Lots of bug fixes. -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 From 7c65a18f1f748f575a5ad0ecc271f76b85a76c92 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:40:50 -0400 Subject: [PATCH 0315/8313] doc pointer --- doc/git-annex.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index f7bb64988d..03e6ce444f 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -209,6 +209,14 @@ information. These files should be committed to git. `.gitattributes` is configured to use git's union merge driver to avoid conflicts when merging files in the `.git-annex` directory. +# SEE ALSO + +Most of git-annex's documentation is available on its web site, + + +If git-annex is installed from a package, a copy of its documentation +should be included, in, for example, `/usr/share/doc/git-annex/` + # AUTHOR Joey Hess From d92730bef6cc40cdd96ff24957a9c2d2bc3cb730 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 14:48:59 -0400 Subject: [PATCH 0316/8313] tweaks --- doc/walkthrough.mdwn | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b49a00f195..ab0067470d 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -21,7 +21,7 @@ Let's start by adding a USB drive as a remote. # git clone ~/annex # cd annex # git annex init "portable USB drive" - # git remote add home ~/annex + # git remote add laptop ~/annex # cd ~/annex # git remote add usbdrive /media/usb @@ -29,9 +29,9 @@ This is all standard ad-hoc distributed git repository setup. The only git-annex specific part is telling it the name of the new repository created on the USB drive. -Notice that both repos are set up as remotes of the other one. This lets +Notice that both repos are set up as remotes of one another. This lets either get annexed files from the other. You'll want to do that even -if you are using a centralized bare repository. +if you are using git in a more centralized fashion. ## adding files @@ -69,17 +69,17 @@ A repository does not always have all annexed file contents available. When you need the content of a file, you can use "git annex get" to make it available. -We can use this to copy everything in the laptop's home annex to the +We can use this to copy everything in the laptop's annex to the USB drive. # cd /media/usb/annex - # git pull home master + # git pull laptop master # git annex get . - get my_cool_big_file (copying from home...) ok - get iso/debian.iso (copying from home...) ok + get my_cool_big_file (copying from laptop...) ok + get iso/debian.iso (copying from laptop...) ok -Notice that you had to git pull from home first, this lets git-annex know -what has changed in home, and so it knows about the files present there and +Notice that you had to git pull from laptop first, this lets git-annex know +what has changed in laptop, and so it knows about the files present there and can get them. ## transferring files: When things go wrong @@ -92,7 +92,7 @@ it: # git annex get video/hackity_hack_and_kaxxt.mov get video/_why_hackity_hack_and_kaxxt.mov (not available) - I was unable to access these remotes: server + I was unable to access these remotes: usbdrive, server Try making some of these repositories available: 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive @@ -141,9 +141,9 @@ some other repository before dropping it. ## using ssh remotes -So far git-annex has been used with a remote repository on a USB drive. -But it can also be used with a remote that is truely remote, a host -accessed by ssh. +So far in this walkthrough, git-annex has been used with a remote +repository on a USB drive. But it can also be used with a git remote +that is truely remote, a host accessed by ssh. Say you have a desktop on the same network as your laptop and want to clone the laptop's annex to it: @@ -155,7 +155,7 @@ to clone the laptop's annex to it: Now you can get files and they will be transferred by `scp`: # git annex get my_cool_big_file - get my_cool_big_file (getting UUIDs for origin...) (copying from origin...) + get my_cool_big_file (getting UUID for origin...) (copying from origin...) WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 ok @@ -174,18 +174,19 @@ access, if available. There is a annex-cost setting you can configure in Also, note that you need full shell access for this to work -- git-annex needs to be able to ssh in and run commands. -## moving file content to another repository +## moving file content between repositories Often you will want to move some file contents from a repository to some other one. For example, your laptop's disk is getting full; time to move -some files to an external disk before moving another file from home to your -laptop. Doing that by hand (by using `git annex get` and `git annex drop`) -is possible, but a bit of a pain. `git annex move` makes it very easy. +some files to an external disk before moving another file from a file +server to your laptop. Doing that by hand (by using `git annex get` and +`git annex drop`) is possible, but a bit of a pain. `git annex move` +makes it very easy. # git annex move my_cool_big_file --to usbdrive move my_cool_big_file (moving to usbdrive...) ok - # git annex move video/hackity_hack_and_kaxxt.mov --from home - move video/hackity_hack_and_kaxxt.mov (moving from home...) + # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver + move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok From 7bd7cca39932858a08dd50a15e5a6f2dc9daa6c0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:00:41 -0400 Subject: [PATCH 0317/8313] some work on the sha1 backend; still incomplete --- Backend/SHA1.hs | 12 +++++++++++- GitRepo.hs | 4 ++-- doc/todo/backendSHA1.mdwn | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index caece6b78c..9e8d6df6dd 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -3,6 +3,11 @@ module Backend.SHA1 (backend) where +import Control.Monad.State +import Data.String.Utils +import System.Cmd.Utils +import System.IO + import qualified Backend.File import TypeInternals @@ -13,4 +18,9 @@ backend = Backend.File.backend { -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) -keyValue k = error "SHA1 keyValue unimplemented" -- TODO +keyValue file = liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do + line <- hGetLine h + let bits = split " " line + if (null bits) + then error "sha1sum parse error" + else return $ Just $ Key ((name backend), bits !! 0) diff --git a/GitRepo.hs b/GitRepo.hs index ee1bdba344..9418102355 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -195,8 +195,8 @@ run repo params = assertLocal repo $ do pipeRead :: Repo -> [String] -> IO String pipeRead repo params = assertLocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret + ret <- hGetContentsStrict h + return ret {- Passed a location, recursively scans for all files that - are checked into git at that location. -} diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn index 40ff868c22..fa9728af64 100644 --- a/doc/todo/backendSHA1.mdwn +++ b/doc/todo/backendSHA1.mdwn @@ -1 +1,5 @@ This backend is not finished. + +In particular, while files can be added using it, git-annex will not notice +when their content changes, and will not create a new key for the new sha1 +of the net content. From 46f9525351e51f3cc3069181fae5f2ba22f869c1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:02:05 -0400 Subject: [PATCH 0318/8313] warning about sha1 --- doc/backends.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/backends.mdwn b/doc/backends.mdwn index d3ccaec491..cd587726c0 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -17,5 +17,6 @@ to store different files' contents in a given repository. * `SHA1` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums - can make it slower for large files. + can make it slower for large files. **Warning** this backend is not ready + for use. * `URL` -- This backend downloads the file's content from an external URL. From e44c7d113605941285015f0953d1e8be562a848b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:08:46 -0400 Subject: [PATCH 0319/8313] update --- debian/rules | 2 +- doc/git-annex.mdwn | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/rules b/debian/rules index e0a209a72c..9079120a80 100755 --- a/debian/rules +++ b/debian/rules @@ -4,4 +4,4 @@ # Not intended for use by anyone except the author. announcedir: - @echo ${HOME}/src/joeywiki/code/git-annex/news + @echo ${HOME}/src/git-annex/doc/news diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 03e6ce444f..3c48e9dc30 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -172,7 +172,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a git repository that content will be sent to. It should be specified using the name of a configured git remote. -## CONFIGURATION +# CONFIGURATION Like other git commands, git-annex is configured via `.git/config`. @@ -188,7 +188,7 @@ Like other git commands, git-annex is configured via `.git/config`. repositories. Note that other factors may be configured when pushing files to repositories, in particular, whether the repository is on a filesystem with sufficient free space. -* `remote..annex-ignore` -- If set to "true", prevents git-annex +* `remote..annex-ignore` -- If set to `true`, prevents git-annex from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. From d47fb4c11ef75e4029c8da1f3ed2e68f86e62486 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 15:14:59 -0400 Subject: [PATCH 0320/8313] symlinks --- CHANGELOG | 1 + GPL | 1 + INSTALL | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) create mode 120000 CHANGELOG create mode 120000 GPL mode change 100644 => 120000 INSTALL diff --git a/CHANGELOG b/CHANGELOG new file mode 120000 index 0000000000..d526672ce2 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1 @@ +debian/changelog \ No newline at end of file diff --git a/GPL b/GPL new file mode 120000 index 0000000000..9539e93f72 --- /dev/null +++ b/GPL @@ -0,0 +1 @@ +doc/GPL \ No newline at end of file diff --git a/INSTALL b/INSTALL deleted file mode 100644 index f024541c1a..0000000000 --- a/INSTALL +++ /dev/null @@ -1 +0,0 @@ -See doc/install.mdwn for installation instructions. diff --git a/INSTALL b/INSTALL new file mode 120000 index 0000000000..67566818f0 --- /dev/null +++ b/INSTALL @@ -0,0 +1 @@ +doc/install.mdwn \ No newline at end of file From 833d4b342e7909e6770edb19fbc58d781e922205 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 16:53:54 -0400 Subject: [PATCH 0321/8313] copyright statements --- Annex.hs | 7 ++++++- Backend.hs | 6 +++++- Backend/File.hs | 4 ++++ Backend/SHA1.hs | 6 +++++- Backend/URL.hs | 6 +++++- Backend/WORM.hs | 6 +++++- BackendList.hs | 6 +++++- Commands.hs | 7 ++++++- Core.hs | 7 ++++++- GitQueue.hs | 4 ++++ GitRepo.hs | 3 +++ LocationLog.hs | 4 ++++ Locations.hs | 4 ++++ Remotes.hs | 7 ++++++- TypeInternals.hs | 4 ++++ Types.hs | 7 ++++++- UUID.hs | 3 +++ Utility.hs | 4 ++++ debian/changelog | 4 ++-- git-annex.hs | 7 ++++++- 20 files changed, 93 insertions(+), 13 deletions(-) diff --git a/Annex.hs b/Annex.hs index 1963d19c69..0e8ec2b7bc 100644 --- a/Annex.hs +++ b/Annex.hs @@ -1,4 +1,9 @@ -{- git-annex monad -} +{- git-annex monad + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Annex ( new, diff --git a/Backend.hs b/Backend.hs index c70b7b707b..b7c52ddec4 100644 --- a/Backend.hs +++ b/Backend.hs @@ -11,7 +11,11 @@ - - Multiple pluggable backends are supported, and more than one can be used - to store different files' contents in a given repository. - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend ( list, diff --git a/Backend/File.hs b/Backend/File.hs index 4273ba36be..4b9a3b45b5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -6,6 +6,10 @@ - - This is an abstract backend; getKey has to be implemented to complete - it. + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Backend.File (backend) where diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 9e8d6df6dd..f6daeffec4 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -1,5 +1,9 @@ {- git-annex "SHA1" backend - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend.SHA1 (backend) where diff --git a/Backend/URL.hs b/Backend/URL.hs index c9b6ab6df8..fd55ddf01e 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -1,5 +1,9 @@ {- git-annex "URL" backend - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend.URL (backend) where diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 0588ddaf83..b5ae11807e 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -1,5 +1,9 @@ {- git-annex "WORM" backend -- Write Once, Read Many - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Backend.WORM (backend) where diff --git a/BackendList.hs b/BackendList.hs index 25f3ae5eac..e628e3a612 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -1,5 +1,9 @@ {- git-annex backend list - - -} + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module BackendList (allBackends) where diff --git a/Commands.hs b/Commands.hs index b6bc6dfc22..6974b697c3 100644 --- a/Commands.hs +++ b/Commands.hs @@ -1,4 +1,9 @@ -{- git-annex command line -} +{- git-annex command line + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Commands (parseCmd) where diff --git a/Core.hs b/Core.hs index 254bcec517..e0993a53e7 100644 --- a/Core.hs +++ b/Core.hs @@ -1,4 +1,9 @@ -{- git-annex core functions -} +{- git-annex core functions + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Core where diff --git a/GitQueue.hs b/GitQueue.hs index 34a89f17b7..6a68edb25b 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -1,4 +1,8 @@ {- git repository command queue + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module GitQueue ( diff --git a/GitRepo.hs b/GitRepo.hs index 9418102355..229b76847f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -3,6 +3,9 @@ - This is written to be completely independant of git-annex and should be - suitable for other uses. - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module GitRepo ( diff --git a/LocationLog.hs b/LocationLog.hs index d027c4b809..10a6377080 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -14,6 +14,10 @@ - - Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict. + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module LocationLog ( diff --git a/Locations.hs b/Locations.hs index 970f89aa8d..49ee878c85 100644 --- a/Locations.hs +++ b/Locations.hs @@ -1,4 +1,8 @@ {- git-annex file locations + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Locations ( diff --git a/Remotes.hs b/Remotes.hs index 8f4fcf0f0c..665de38ae5 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -1,4 +1,9 @@ -{- git-annex remote repositories -} +{- git-annex remote repositories + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Remotes ( list, diff --git a/TypeInternals.hs b/TypeInternals.hs index d4404fd390..f45be4760c 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -1,6 +1,10 @@ {- git-annex internal data types - - Most things should not need this, using Types and/or Annex instead. + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module TypeInternals where diff --git a/Types.hs b/Types.hs index c3d6467a3f..b94a4170af 100644 --- a/Types.hs +++ b/Types.hs @@ -1,4 +1,9 @@ -{- git-annex abstract data types -} +{- git-annex abstract data types + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} module Types ( Annex, diff --git a/UUID.hs b/UUID.hs index a7783d6140..79b2b55fa2 100644 --- a/UUID.hs +++ b/UUID.hs @@ -3,6 +3,9 @@ - Each git repository used by git-annex has an annex.uuid setting that - uniquely identifies that repository. - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module UUID ( diff --git a/Utility.hs b/Utility.hs index 8e620c64cd..ab90c51608 100644 --- a/Utility.hs +++ b/Utility.hs @@ -1,4 +1,8 @@ {- git-annex utility functions + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Utility ( diff --git a/debian/changelog b/debian/changelog index c3255ba271..72ae91c02e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.02) UNRELEASED; urgency=low +git-annex (0.02) unstable; urgency=low * Can scp annexed files from remote hosts, and check remote hosts for file content when dropping files. @@ -17,7 +17,7 @@ git-annex (0.02) UNRELEASED; urgency=low * New plumbing-level dropkey and addkey subcommands. * Lots of bug fixes. - -- Joey Hess Thu, 21 Oct 2010 16:38:00 -0400 + -- Joey Hess Wed, 27 Oct 2010 16:39:29 -0400 git-annex (0.01) unstable; urgency=low diff --git a/git-annex.hs b/git-annex.hs index d7b26cd968..e9e7ff0273 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,4 +1,9 @@ -{- git-annex main program -} +{- git-annex main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} import IO (try) import System.IO From 27d28b0bf2f3b723f404407d1814c38a3c37076d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 16:57:36 -0400 Subject: [PATCH 0322/8313] add news item for git-annex 0.02 --- doc/news/version_0.02.mdwn | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/news/version_0.02.mdwn diff --git a/doc/news/version_0.02.mdwn b/doc/news/version_0.02.mdwn new file mode 100644 index 0000000000..689124ab4e --- /dev/null +++ b/doc/news/version_0.02.mdwn @@ -0,0 +1,18 @@ +git-annex 0.02 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Can scp annexed files from remote hosts, and check remote hosts for + file content when dropping files. + * New move subcommand, that makes it easy to move file contents from + or to a remote. + * New fromkey subcommand, for registering urls, etc. + * git-annex init will now set up a pre-commit hook that fixes up symlinks + before they are committed, to ensure that moving symlinks around does not + break them. + * More intelligent and fast staging of modified files; git add coalescing. + * Add remote.annex-ignore git config setting to allow completly disabling + a given remote. + * --from/--to can be used to control the remote repository that git-annex + uses. + * --quiet can be used to avoid verbose output + * New plumbing-level dropkey and addkey subcommands. + * Lots of bug fixes."""]] \ No newline at end of file From 2de5f51a57f011654007829850e7b24992497e8d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 19:10:28 -0400 Subject: [PATCH 0323/8313] xargs needed --- doc/install.mdwn | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index e217799a32..9e3b385c5c 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -1,8 +1,9 @@ To build and use git-annex, you will need: -* git: +* `git`: * The Haskell Platform: * MissingH: -* uuid: +* `uuid`: +* `xargs`: Then just [[download]] git-annex and run: `make; make install` From 1118b4a6466f3453f5c517ff8eadbfbd1a4895f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Oct 2010 19:27:01 -0400 Subject: [PATCH 0324/8313] idea --- doc/todo/union_mounting.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/todo/union_mounting.mdwn diff --git a/doc/todo/union_mounting.mdwn b/doc/todo/union_mounting.mdwn new file mode 100644 index 0000000000..c42a055021 --- /dev/null +++ b/doc/todo/union_mounting.mdwn @@ -0,0 +1,10 @@ +It should be possible to union mount annexes. So if multiple drives have +content, an annex mounting them both would have available all the +content from all the drives. + +This could be done by just making .git/annex/KEY link to the actual content +on the mounted annex. + +(Need to make sure the [[copy_tracking|copies]] code does not +confused and think the symlink is a copy of the content.. Also need to make +sure that code that writes to .git/annex does not follow symlinks.)) From ca667f5612d0d13f1aaf6ca973681a9448ed3864 Mon Sep 17 00:00:00 2001 From: Eugen_Paiuc Date: Thu, 28 Oct 2010 03:07:02 +0000 Subject: [PATCH 0325/8313] --- doc/install.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index 9e3b385c5c..e49100f786 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,5 +5,5 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: - -Then just [[download]] git-annex and run: `make; make install` +* `ikiwiki`: <> +* Then just [[download]] git-annex and run: `make; make install` From 9c7b3dce9e8f964ed60dd45bca580a46ff8a5ed5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 12:15:21 -0400 Subject: [PATCH 0326/8313] tweaks --- GitRepo.hs | 18 +++++++++++------- Remotes.hs | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 229b76847f..0e87c9526d 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -24,6 +24,7 @@ module GitRepo ( configGet, configMap, configRead, + configTrue, run, pipeRead, attributes, @@ -47,6 +48,7 @@ import Data.String.Utils import Data.Map as Map hiding (map, split) import Network.URI import Maybe +import Char import Utility @@ -127,13 +129,11 @@ assertssh repo action = then action else error $ "unsupported url " ++ (show $ url repo) bare :: Repo -> Bool -bare repo = - if (member b (config repo)) - then ("true" == fromJust (Map.lookup b (config repo))) - else error $ "it is not known if git repo " ++ (repoDescribe repo) ++ +bare repo = case Map.lookup "core.bare" $ config repo of + Just v -> configTrue v + Nothing -> error $ "it is not known if git repo " ++ + (repoDescribe repo) ++ " is a bare repository; config not read" - where - b = "core.bare" {- Path to a repository's gitattributes file. -} attributes :: Repo -> String @@ -173,7 +173,7 @@ relative repo file = assertLocal repo $ drop (length absrepo) absfile {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String urlHost repo = assertUrl repo $ - (uriUserInfo a) ++ (uriRegName a) ++ (uriPort a) + uriUserInfo a ++ uriRegName a ++ uriPort a where a = fromJust $ uriAuthority $ url repo @@ -235,6 +235,10 @@ configRead repo = let r = repo { config = configParse val } return r { remotes = configRemotes r } +{- Checks if a string fron git config is a true value. -} +configTrue :: String -> Bool +configTrue s = map toLower s == "true" + {- Calculates a list of a repo's configured remotes, by parsing its config. -} configRemotes :: Repo -> [Repo] configRemotes repo = map construct remotes diff --git a/Remotes.hs b/Remotes.hs index 665de38ae5..bee98a6f3d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -139,10 +139,10 @@ repoNotIgnored r = do let name = if (not $ null fromName) then fromName else toName if (not $ null name) then return $ match name - else return $ notignored g + else return $ not $ ignored g where match name = name == Git.repoRemoteName r - notignored g = "true" /= config g + ignored g = Git.configTrue $ config g config g = Git.configGet g configkey "" configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" From 045b051ec10023afc2e62895032527d5b5130495 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 12:40:05 -0400 Subject: [PATCH 0327/8313] got rid of almost all 'return ()' --- Annex.hs | 3 --- Backend/File.hs | 5 +++-- Commands.hs | 9 +++------ Core.hs | 23 +++++++++-------------- GitQueue.hs | 5 ++--- GitRepo.hs | 3 +-- Remotes.hs | 7 ++++--- UUID.hs | 14 ++++---------- git-annex.hs | 11 ++++------- 9 files changed, 30 insertions(+), 50 deletions(-) diff --git a/Annex.hs b/Annex.hs index 0e8ec2b7bc..60ae91708b 100644 --- a/Annex.hs +++ b/Annex.hs @@ -61,7 +61,6 @@ gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get put state { Internals.repo = r } - return () {- Returns the backends being used. -} backends :: Annex [Backend] @@ -74,7 +73,6 @@ backendsChange :: [Backend] -> Annex () backendsChange b = do state <- get put state { Internals.backends = b } - return () {- Returns the full list of supported backends. -} supportedBackends :: Annex [Backend] @@ -95,7 +93,6 @@ flagChange :: FlagName -> Flag -> Annex () flagChange name val = do state <- get put state { Internals.flags = M.insert name val $ Internals.flags state } - return () {- Gets the value of a String flag (or "" if there is no such String flag) -} flagGet :: FlagName -> Annex String diff --git a/Backend/File.hs b/Backend/File.hs index 4b9a3b45b5..b45354752f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -129,7 +129,7 @@ checkRemoveKey key = do "Could only verify the existence of " ++ (show have) ++ " out of " ++ (show need) ++ " necessary copies" - if (not $ null bad) then showTriedRemotes bad else return () + showTriedRemotes bad showLocations key hint return False @@ -146,7 +146,8 @@ showLocations key = do if (null uuidsf) then showLongNote $ "No other repository is known to contain the file." else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - + +showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ (Remotes.list remotes) diff --git a/Commands.hs b/Commands.hs index 6974b697c3..a46b9fb163 100644 --- a/Commands.hs +++ b/Commands.hs @@ -14,6 +14,7 @@ import System.Directory import System.Path import Data.String.Utils import Control.Monad (filterM) +import Monad (when) import List import IO @@ -326,9 +327,7 @@ dropKeyCleanup key = do setKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) setKeyStart tmpfile = do keyname <- Annex.flagGet "key" - if (null keyname) - then error "please specify the key with --key" - else return () + when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname return $ Just $ setKeyPerform tmpfile key @@ -392,9 +391,7 @@ initCleanup = do fromKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) fromKeyStart file = do keyname <- Annex.flagGet "key" - if (null keyname) - then error "please specify the key with --key" - else return () + when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname diff --git a/Core.hs b/Core.hs index e0993a53e7..cf97768c73 100644 --- a/Core.hs +++ b/Core.hs @@ -13,6 +13,7 @@ import System.Directory import Control.Monad.State (liftIO) import System.Path import Data.String.Utils +import Monad (when, unless) import Types import Locations @@ -37,19 +38,15 @@ shutdown = do -- Runs all queued git commands. q <- Annex.queueGet - if (q == GitQueue.empty) - then return () - else do - verbose $ liftIO $ putStrLn "Recording state in git..." - liftIO $ GitQueue.run g q + unless (q == GitQueue.empty) $ do + verbose $ liftIO $ putStrLn "Recording state in git..." + liftIO $ GitQueue.run g q -- clean up any files left in the temp directory, but leave -- the tmp directory itself let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp - if (exists) - then liftIO $ removeDirectoryRecursive $ tmp - else return () + when (exists) $ liftIO $ removeDirectoryRecursive $ tmp liftIO $ createDirectoryIfMissing True tmp return True @@ -65,11 +62,9 @@ gitAttributes repo = do commit else do content <- readFile attributes - if (all (/= attrLine) (lines content)) - then do - appendFile attributes $ attrLine ++ "\n" - commit - else return () + when (all (/= attrLine) (lines content)) $ do + appendFile attributes $ attrLine ++ "\n" + commit where attrLine = stateLoc ++ "*.log merge=union" attributes = Git.attributes repo @@ -150,7 +145,7 @@ getViaTmp key action = do verbose :: Annex () -> Annex () verbose a = do q <- Annex.flagIsSet "quiet" - if (q) then return () else a + unless q a showStart :: String -> String -> Annex () showStart command file = verbose $ do liftIO $ putStr $ command ++ " " ++ file ++ " " diff --git a/GitQueue.hs b/GitQueue.hs index 6a68edb25b..09b8037e64 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -16,6 +16,7 @@ import qualified Data.Map as M import System.IO import System.Cmd.Utils import Data.String.Utils +import Monad (unless) import qualified GitRepo as Git @@ -52,9 +53,7 @@ run repo queue = do - Complicated by commandline length limits. -} runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do - if (null files) - then return () - else runxargs + unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" (["-0", "git", subcommand action] ++ (params action)) diff --git a/GitRepo.hs b/GitRepo.hs index 0e87c9526d..d0fac96c1a 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -191,8 +191,7 @@ gitCommandLine repo params = assertLocal repo $ {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do - r <- safeSystem "git" (gitCommandLine repo params) - return () + safeSystem "git" (gitCommandLine repo params) {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String diff --git a/Remotes.hs b/Remotes.hs index bee98a6f3d..02d4dc9c27 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -28,6 +28,7 @@ import System.Directory import System.Posix.Directory import List import Maybe +import Monad (when, unless) import Types import qualified GitRepo as Git @@ -65,9 +66,9 @@ keyPossibilities key = do let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive - if (not $ null doexpensive) - then Core.showNote $ "getting UUID for " ++ (list doexpensive) ++ "..." - else return () + unless (null doexpensive) $ do + Core.showNote $ "getting UUID for " ++ + (list doexpensive) ++ "..." let todo = cheap ++ doexpensive if (not $ null todo) then do diff --git a/UUID.hs b/UUID.hs index 79b2b55fa2..f2235e4b62 100644 --- a/UUID.hs +++ b/UUID.hs @@ -63,10 +63,7 @@ getUUID r = do where uncached r = Git.configGet r "annex.uuid" "" cached r g = Git.configGet g (cachekey r) "" - updatecache g r u = do - if (g /= r) - then setConfig (cachekey r) u - else return () + updatecache g r u = when (g /= r) $ setConfig (cachekey r) u cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} @@ -74,11 +71,9 @@ prepUUID :: Annex () prepUUID = do g <- Annex.gitRepo u <- getUUID g - if ("" == u) - then do - uuid <- liftIO $ genUUID - setConfig configkey uuid - else return () + when ("" == u) $ do + uuid <- liftIO $ genUUID + setConfig configkey uuid {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () @@ -88,7 +83,6 @@ setConfig key value = do -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g Annex.gitRepoChange g' - return () {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] diff --git a/git-annex.hs b/git-annex.hs index e9e7ff0273..5011fade2b 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -8,6 +8,7 @@ import IO (try) import System.IO import System.Environment +import Monad import qualified Annex import Types @@ -42,12 +43,8 @@ tryRun' state errnum (a:as) = do tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' state errnum [] = do - if (errnum > 0) - then error $ (show errnum) ++ " failed" - else return () +tryRun' state errnum [] = + when (errnum > 0) $ error $ (show errnum) ++ " failed" {- Exception pretty-printing. -} -showErr e = do - hPutStrLn stderr $ "git-annex: " ++ (show e) - return () +showErr e = hPutStrLn stderr $ "git-annex: " ++ (show e) From 7109e20e5d2026c25a42a2784c1e3a430a67d3cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 12:52:40 -0400 Subject: [PATCH 0328/8313] tweak --- GitRepo.hs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index d0fac96c1a..4bad8a50d3 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -150,10 +150,8 @@ dir repo = if (bare repo) then "" else ".git" - - Note that for URL repositories, this is relative to the urlHost -} workTree :: Repo -> FilePath -workTree repo = - if (not $ repoIsUrl repo) - then top repo - else urlPath repo +workTree r@(UrlRepo { }) = urlPath r +workTree (Repo { top = p }) = p {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. @@ -186,7 +184,8 @@ urlPath repo = assertUrl repo $ gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo params = assertLocal repo $ -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(top repo)++"/"++(dir repo), "--work-tree="++(top repo)] ++ params + ["--git-dir="++(top repo)++"/"++(dir repo), + "--work-tree="++(top repo)] ++ params {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () From 3e02977814e77b47e4db020e9b0dedadad1b6e7e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 13:40:10 -0400 Subject: [PATCH 0329/8313] took Josh's asvice and unified the Repo data types & used pattern matching more --- GitRepo.hs | 154 +++++++++++++++++++++++++---------------------------- 1 file changed, 73 insertions(+), 81 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 4bad8a50d3..cd2c80691b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -54,47 +54,37 @@ import Utility {- There are two types of repositories; those on local disk and those - accessed via an URL. -} -data Repo = +data RepoLocation = Dir FilePath | Url URI + deriving (Show, Eq) + +data Repo = Repo { + location :: RepoLocation, + config :: Map String String, + remotes :: [Repo], + -- remoteName holds the name used for this repo in remotes + remoteName :: Maybe String +} deriving (Show, Eq) + +newFrom l = Repo { - top :: FilePath, - config :: Map String String, - remotes :: [Repo], - -- remoteName holds the name used for this repo in remotes - remoteName :: Maybe String - } | UrlRepo { - url :: URI, - config :: Map String String, - remotes :: [Repo], - remoteName :: Maybe String - } deriving (Show, Eq) + location = l, + config = Map.empty, + remotes = [], + remoteName = Nothing + } {- Local Repo constructor. -} repoFromPath :: FilePath -> Repo -repoFromPath dir = - Repo { - top = dir, - config = Map.empty, - remotes = [], - remoteName = Nothing - } +repoFromPath dir = newFrom $ Dir dir {- Remote Repo constructor. Throws exception on invalid url. -} repoFromUrl :: String -> Repo -repoFromUrl url = - UrlRepo { - url = fromJust $ parseURI url, - config = Map.empty, - remotes = [], - remoteName = Nothing - } +repoFromUrl url = newFrom $ Url $ fromJust $ parseURI url {- User-visible description of a git repo. -} -repoDescribe repo = - if (isJust $ remoteName repo) - then fromJust $ remoteName repo - else if (not $ repoIsUrl repo) - then top repo - else show (url repo) +repoDescribe Repo { remoteName = Just name } = name +repoDescribe Repo { location = Url url } = show url +repoDescribe Repo { location = Dir dir } = dir {- Constructs and returns an updated version of a repo with - different remotes list. -} @@ -103,17 +93,19 @@ remotesAdd repo rs = repo { remotes = rs } {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} -repoRemoteName r = - if (isJust $ remoteName r) - then fromJust $ remoteName r - else "" +repoRemoteName Repo { remoteName = Just name } = name +repoRemoteName _ = "" {- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} -repoIsUrl repo = case (repo) of - UrlRepo {} -> True - Repo {} -> False -repoIsSsh repo = repoIsUrl repo && (uriScheme $ url repo) == "ssh:" +repoIsUrl Repo { location = Url _ } = True +repoIsUrl _ = False + +repoIsSsh Repo { location = Url url } + | uriScheme url == "ssh:" = True + | otherwise = False +repoIsSsh _ = False + assertLocal repo action = if (not $ repoIsUrl repo) then action @@ -124,10 +116,10 @@ assertUrl repo action = then action else error $ "acting on local git repo " ++ (repoDescribe repo) ++ " not supported" -assertssh repo action = +assertSsh repo action = if (repoIsSsh repo) then action - else error $ "unsupported url " ++ (show $ url repo) + else error $ "unsupported url in repo " ++ (repoDescribe repo) bare :: Repo -> Bool bare repo = case Map.lookup "core.bare" $ config repo of Just v -> configTrue v @@ -137,55 +129,56 @@ bare repo = case Map.lookup "core.bare" $ config repo of {- Path to a repository's gitattributes file. -} attributes :: Repo -> String -attributes repo = assertLocal repo $ do - if (bare repo) - then (top repo) ++ "/info/.gitattributes" - else (top repo) ++ "/.gitattributes" +attributes repo + | bare repo = (workTree repo) ++ "/info/.gitattributes" + | otherwise = (workTree repo) ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its workTree. -} dir :: Repo -> String -dir repo = if (bare repo) then "" else ".git" +dir repo + | bare repo = "" + | otherwise = ".git" {- Path to a repository's --work-tree, that is, its top. - - - Note that for URL repositories, this is relative to the urlHost -} + - Note that for URL repositories, this is the path on the remote host. -} workTree :: Repo -> FilePath -workTree r@(UrlRepo { }) = urlPath r -workTree (Repo { top = p }) = p +workTree r@(Repo { location = Url _ }) = urlPath r +workTree (Repo { location = Dir d }) = d {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} relative :: Repo -> String -> String -relative repo file = assertLocal repo $ drop (length absrepo) absfile +relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile where -- normalize both repo and file, so that repo -- will be substring of file - absrepo = case (absNormPath "/" (top repo)) of + absrepo = case (absNormPath "/" d) of Just f -> f ++ "/" - Nothing -> error $ "bad repo" ++ (top repo) + Nothing -> error $ "bad repo" ++ (repoDescribe repo) absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo +relative repo file = assertLocal repo $ error "internal" {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String -urlHost repo = assertUrl repo $ - uriUserInfo a ++ uriRegName a ++ uriPort a - where - a = fromJust $ uriAuthority $ url repo +urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a + where a = fromJust $ uriAuthority $ u +urlHost repo = assertUrl repo $ error "internal" {- Path of an URL repo. -} urlPath :: Repo -> String -urlPath repo = assertUrl repo $ - uriPath $ url repo +urlPath Repo { location = Url u } = uriPath u +urlPath repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [String] -> [String] -gitCommandLine repo params = assertLocal repo $ +gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++(top repo)++"/"++(dir repo), - "--work-tree="++(top repo)] ++ params + ["--git-dir="++d++"/"++(dir repo), "--work-tree="++d] ++ params +gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo. -} run :: Repo -> [String] -> IO () @@ -215,23 +208,23 @@ notInRepo repo location = do {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo -configRead repo = - if (not $ repoIsUrl repo) - then do - {- Cannot use pipeRead because it relies on the config having - been already read. Instead, chdir to the repo. -} - cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory (top repo)) - (\_ -> changeWorkingDirectory cwd) $ - pOpen ReadFromPipe "git" ["config", "--list"] proc - else assertssh repo $ do - pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] proc +configRead repo@(Repo { location = Dir d }) = do + {- Cannot use pipeRead because it relies on the config having + been already read. Instead, chdir to the repo. -} + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory d) + (\_ -> changeWorkingDirectory cwd) $ + pOpen ReadFromPipe "git" ["config", "--list"] $ + hConfigRead repo +configRead repo = assertSsh repo $ do + pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] $ hConfigRead repo where - sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" - proc h = do - val <- hGetContentsStrict h - let r = repo { config = configParse val } - return r { remotes = configRemotes r } + sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ + " && git config --list" +hConfigRead repo h = do + val <- hGetContentsStrict h + let r = repo { config = configParse val } + return r { remotes = configRemotes r } {- Checks if a string fron git config is a true value. -} configTrue :: String -> Bool @@ -246,9 +239,8 @@ configRemotes repo = map construct remotes isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } - gen v = if (isURI v) - then repoFromUrl v - else repoFromPath v + gen v | isURI v = repoFromUrl v + | otherwise = repoFromPath v {- Parses git config --list output into a config map. -} configParse :: String -> Map.Map String String From 5c2c652d7dd6346990143f10f9a7d520496cc871 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 13:47:10 -0400 Subject: [PATCH 0330/8313] Fix support for file:// remotes. --- GitRepo.hs | 5 ++++- debian/changelog | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index cd2c80691b..fd69ec21ae 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -79,7 +79,10 @@ repoFromPath dir = newFrom $ Dir dir {- Remote Repo constructor. Throws exception on invalid url. -} repoFromUrl :: String -> Repo -repoFromUrl url = newFrom $ Url $ fromJust $ parseURI url +repoFromUrl url + | startswith "file://" url = repoFromPath $ uriPath u + | otherwise = newFrom $ Url u + where u = fromJust $ parseURI url {- User-visible description of a git repo. -} repoDescribe Repo { remoteName = Just name } = name diff --git a/debian/changelog b/debian/changelog index 72ae91c02e..ac1ed0ab40 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.03) UNRELEASED; urgency=low + + * Fix support for file:// remotes. + + -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 + git-annex (0.02) unstable; urgency=low * Can scp annexed files from remote hosts, and check remote hosts for From ecfbc01ff8b55d4cbe02c02c604264a627e759bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 14:04:22 -0400 Subject: [PATCH 0331/8313] Add --verbose --- Commands.hs | 2 ++ debian/changelog | 1 + doc/git-annex.mdwn | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/Commands.hs b/Commands.hs index a46b9fb163..ac17c34d1a 100644 --- a/Commands.hs +++ b/Commands.hs @@ -116,6 +116,8 @@ options = [ "allow actions that may lose annexed data" , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) + "allow verbose output" , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") "specify default key-value backend to use" , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") diff --git a/debian/changelog b/debian/changelog index ac1ed0ab40..c7297d1c3f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix support for file:// remotes. + * Add --verbose -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3c48e9dc30..47fbb37601 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -153,6 +153,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Avoid the default verbose logging of what is done; only show errors and progress displays. +* --verbose + + Enable verbose logging. + * --backend=name Specify the default key-value backend to use, adding it to the front From 694a33e91b290373649594688ad880203d140dcb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Oct 2010 14:20:02 -0400 Subject: [PATCH 0332/8313] syntax tweaks --- Commands.hs | 45 +++++++++++++++++++++------------------------ LocationLog.hs | 2 +- Remotes.hs | 18 ++++++++---------- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/Commands.hs b/Commands.hs index ac17c34d1a..41e9ad54dc 100644 --- a/Commands.hs +++ b/Commands.hs @@ -14,7 +14,7 @@ import System.Directory import System.Path import Data.String.Utils import Control.Monad (filterM) -import Monad (when) +import Monad (when, unless) import List import IO @@ -168,7 +168,7 @@ findWanted FilesMissing params repo = do where missing f = do e <- doesFileExist f - if (e) then return False else return True + return $ not e findWanted Description params _ = do return $ [unwords params] findWanted FilesToBeCommitted params repo = do @@ -191,19 +191,18 @@ findWanted _ params _ = return params parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) parseCmd argv state = do (flags, params) <- getopt - if (null params) - then error usage - else case (lookupCmd (params !! 0)) of - [] -> error usage - [Command name action want _] -> do - f <- findWanted want (drop 1 params) - (TypeInternals.repo state) - let actions = map (doSubCmd name action) $ - filter notstate f - let configactions = map (\f -> do - f - return True) flags - return (configactions, actions) + when (null params) $ error usage + case lookupCmd (params !! 0) of + [] -> error usage + [Command name action want _] -> do + f <- findWanted want (drop 1 params) + (TypeInternals.repo state) + let actions = map (doSubCmd name action) $ + filter notstate f + let configactions = map (\f -> do + f + return True) flags + return (configactions, actions) where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f @@ -273,7 +272,7 @@ getPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) getPerform file key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) - then return $ Just $ return True + then return $ Just $ return True -- no cleanup needed else return Nothing {- Indicates a file's content is not wanted anymore, and should be removed @@ -368,11 +367,9 @@ fixCleanup file = do {- Stores description for the repository etc. -} initStart :: String -> Annex (Maybe SubCmdPerform) initStart description = do - if (null description) - then error $ - "please specify a description of this repository\n" ++ - usage - else return $ Just $ initPerform description + when (null description) $ error $ + "please specify a description of this repository\n" ++ usage + return $ Just $ initPerform description initPerform :: String -> Annex (Maybe SubCmdCleanup) initPerform description = do g <- Annex.gitRepo @@ -398,9 +395,9 @@ fromKeyStart file = do let key = genKey (backends !! 0) keyname inbackend <- Backend.hasKey key - if (not inbackend) - then error $ "key ("++keyname++") is not present in backend" - else return $ Just $ fromKeyPerform file key + unless (inbackend) $ error $ + "key ("++keyname++") is not present in backend" + return $ Just $ fromKeyPerform file key fromKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) fromKeyPerform file key = do link <- calcGitLink file key diff --git a/LocationLog.hs b/LocationLog.hs index 10a6377080..f92dee652d 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -163,6 +163,6 @@ mapLog map log = then Map.insert (uuid log) log map else map where - better = case (Map.lookup (uuid log) map) of + better = case Map.lookup (uuid log) map of Just l -> (date l <= date log) Nothing -> True diff --git a/Remotes.hs b/Remotes.hs index 02d4dc9c27..a432e1b5d6 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -153,15 +153,13 @@ commandLineRemote = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if (not $ null fromName) then fromName else toName - if (null name) - then error "no remote specified" - else do - g <- Annex.gitRepo - let match = filter (\r -> name == Git.repoRemoteName r) $ - Git.remotes g - if (null match) - then error $ "there is no git remote named \"" ++ name ++ "\"" - else return $ match !! 0 + when (null name) $ error "no remote specified" + g <- Annex.gitRepo + let match = filter (\r -> name == Git.repoRemoteName r) $ + Git.remotes g + when (null match) $ error $ + "there is no git remote named \"" ++ name ++ "\"" + return $ match !! 0 {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the @@ -187,7 +185,7 @@ tryGitConfigRead r = do where exchange [] new = [] exchange (old:ls) new = - if ((Git.repoRemoteName old) == (Git.repoRemoteName new)) + if (Git.repoRemoteName old == Git.repoRemoteName new) then new:(exchange ls new) else old:(exchange ls new) From fde01e52f382d7903749609301f99d78791c2f2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 12:38:41 -0400 Subject: [PATCH 0333/8313] Fix SIGINT handling. --- Utility.hs | 3 +-- debian/changelog | 1 + doc/bugs/scp_interrupt_to_background.mdwn | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Utility.hs b/Utility.hs index ab90c51608..338aca7a39 100644 --- a/Utility.hs +++ b/Utility.hs @@ -19,7 +19,6 @@ import System.IO import System.Cmd import System.Exit import System.Posix.Signals -import Data.Typeable import System.Posix.IO import Data.String.Utils import System.Path @@ -110,7 +109,7 @@ boolSystem command params = do r <- rawSystem command params case r of ExitSuccess -> return True - ExitFailure e -> if Just e == cast sigINT + ExitFailure e -> if toInteger e == toInteger sigINT then error $ command ++ "interrupted" else return False diff --git a/debian/changelog b/debian/changelog index c7297d1c3f..77e01482fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix support for file:// remotes. * Add --verbose + * Fix SIGINT handling. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/bugs/scp_interrupt_to_background.mdwn b/doc/bugs/scp_interrupt_to_background.mdwn index 700c81c650..381f5cd736 100644 --- a/doc/bugs/scp_interrupt_to_background.mdwn +++ b/doc/bugs/scp_interrupt_to_background.mdwn @@ -1,2 +1,2 @@ When getting a file with scp, SIGINT is blocked, exposing the git -subcommand fork to background bug again. +subcommand fork to background bug again. [[done]] From e3030196b6616690d365597470a7123476444e57 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 13:57:22 -0400 Subject: [PATCH 0334/8313] really fix SIGINT handling Have to completly avoid SIGINT being trapped, which means going very low-level. --- Utility.hs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Utility.hs b/Utility.hs index 338aca7a39..233825b651 100644 --- a/Utility.hs +++ b/Utility.hs @@ -16,8 +16,10 @@ module Utility ( ) where import System.IO -import System.Cmd +import System.Cmd.Utils import System.Exit +import System.Posix.Process +import System.Posix.Process.Internals import System.Posix.Signals import System.Posix.IO import Data.String.Utils @@ -101,17 +103,30 @@ relPathDirToDir from to = {- Run a system command, and returns True or False - if it succeeded or failed. - - - An error is thrown if the command exits due to SIGINT, - - to propigate ctrl-c. + - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} boolSystem :: FilePath -> [String] -> IO Bool boolSystem command params = do - r <- rawSystem command params - case r of - ExitSuccess -> return True - ExitFailure e -> if toInteger e == toInteger sigINT - then error $ command ++ "interrupted" - else return False + -- Going low-level because all the high-level system functions + -- block SIGINT etc. We need to block SIGCHLD, but allow + -- SIGINT to do its default program termination. + let sigset = addSignal sigCHLD emptySignalSet + oldint <- installHandler sigINT Default Nothing + oldset <- getSignalMask + blockSignals sigset + childpid <- forkProcess $ childaction oldint oldset + mps <- getProcessStatus True False childpid + restoresignals oldint oldset + case mps of + Just (Exited ExitSuccess) -> return True + _ -> return False + where + restoresignals oldint oldset = do + installHandler sigINT oldint Nothing + setSignalMask oldset + childaction oldint oldset = do + restoresignals oldint oldset + executeFile command True params Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} shellEscape f = "'" ++ quote ++ "'" From fa04c36fbe6bd9e936a74230bc8e887a90250bfd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 13:59:48 -0400 Subject: [PATCH 0335/8313] ikiwiki is not really needed --- doc/install.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index e49100f786..ec9ea92c7f 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,5 +5,7 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: -* `ikiwiki`: <> * Then just [[download]] git-annex and run: `make; make install` + +([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, +but that will be skipped if it is not installed.) From d92f186fc4bc34e0999a6e47f15e54717e0ab206 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 14:07:26 -0400 Subject: [PATCH 0336/8313] convert safeSystem to boolSystem to fix ctrl-c handling --- Backend/URL.hs | 8 ++------ GitRepo.hs | 6 ++++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index fd55ddf01e..7f0bd66733 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -7,7 +7,6 @@ module Backend.URL (backend) where -import Control.Exception import Control.Monad.State (liftIO) import Data.String.Utils import System.Cmd @@ -16,6 +15,7 @@ import System.Exit import TypeInternals import Core +import Utility backend = Backend { name = "URL", @@ -42,10 +42,6 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" liftIO $ putStrLn "" -- make way for curl progress bar - result <- liftIO $ (try curl::IO (Either SomeException ())) - case result of - Left err -> return False - Right succ -> return True + liftIO $ boolSystem "curl" ["-#", "-o", file, url] where - curl = safeSystem "curl" ["-#", "-o", file, url] url = join ":" $ drop 1 $ split ":" $ show key diff --git a/GitRepo.hs b/GitRepo.hs index fd69ec21ae..b9980826ca 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -35,6 +35,7 @@ module GitRepo ( notInRepo ) where +import Monad (when, unless) import Directory import System import System.Directory @@ -183,10 +184,11 @@ gitCommandLine repo@(Repo { location = Dir d} ) params = ["--git-dir="++d++"/"++(dir repo), "--work-tree="++d] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" -{- Runs git in the specified repo. -} +{- Runs git in the specified repo, throwing an error if it fails. -} run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do - safeSystem "git" (gitCommandLine repo params) + ok <- boolSystem "git" (gitCommandLine repo params) + unless (ok) $ error $ "git " ++ (show params) ++ " failed" {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String From 7c0777c60d5381cc16342483747e1fc9c92e41e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 14:10:55 -0400 Subject: [PATCH 0337/8313] avoid unnessary newlines before progress in quiet mode --- Backend/URL.hs | 2 +- Core.hs | 2 ++ Remotes.hs | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index 7f0bd66733..384f933ebf 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -41,7 +41,7 @@ dummyOk url = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" - liftIO $ putStrLn "" -- make way for curl progress bar + showProgress -- make way for curl progress bar liftIO $ boolSystem "curl" ["-#", "-o", file, url] where url = join ":" $ drop 1 $ split ":" $ show key diff --git a/Core.hs b/Core.hs index cf97768c73..8cdc6979ab 100644 --- a/Core.hs +++ b/Core.hs @@ -154,6 +154,8 @@ showNote :: String -> Annex () showNote s = verbose $ do liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout +showProgress :: Annex () +showProgress = verbose $ liftIO $ putStr $ "\n" showLongNote :: String -> Annex () showLongNote s = verbose $ do liftIO $ putStr $ "\n" ++ (indent s) diff --git a/Remotes.hs b/Remotes.hs index a432e1b5d6..66395adcda 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -39,6 +39,7 @@ import LocationLog import Locations import UUID import Utility +import qualified Core {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -200,7 +201,7 @@ copyFromRemote r key file = do where getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] getssh = do - liftIO $ putStrLn "" -- make way for scp progress bar + Core.showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" [sshLocation r keyloc, file] keyloc = annexLocation r key @@ -217,7 +218,7 @@ copyToRemote r key file = do where putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] putssh src = do - liftIO $ putStrLn "" -- make way for scp progress bar + Core.showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath From c9347693d78a1287cc8e3d066642931f28523263 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 17:26:26 -0400 Subject: [PATCH 0338/8313] factor out stagedFiles --- Commands.hs | 10 +--------- GitRepo.hs | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Commands.hs b/Commands.hs index 41e9ad54dc..2af8874e5c 100644 --- a/Commands.hs +++ b/Commands.hs @@ -172,16 +172,8 @@ findWanted FilesMissing params repo = do findWanted Description params _ = do return $ [unwords params] findWanted FilesToBeCommitted params repo = do - files <- mapM gitcached params + files <- mapM (Git.stagedFiles repo) params return $ foldl (++) [] files - where - gitcached p = do - -- ask git for files staged for commit that - -- are being added, moved, or changed (but not deleted) - fs0 <- Git.pipeRead repo ["diff", "--cached", - "--name-only", "--diff-filter=ACMRT", - "-z", "HEAD", p] - return $ filter (not . null) $ split "\0" fs0 findWanted _ params _ = return params {- Parses command line and returns two lists of actions to be diff --git a/GitRepo.hs b/GitRepo.hs index b9980826ca..355600248e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -32,7 +32,8 @@ module GitRepo ( remotesAdd, repoRemoteName, inRepo, - notInRepo + notInRepo, + stagedFiles ) where import Monad (when, unless) @@ -46,7 +47,7 @@ import System.Cmd.Utils import System.IO import IO (bracket_) import Data.String.Utils -import Data.Map as Map hiding (map, split) +import qualified Data.Map as Map hiding (map, split) import Network.URI import Maybe import Char @@ -60,7 +61,7 @@ data RepoLocation = Dir FilePath | Url URI data Repo = Repo { location :: RepoLocation, - config :: Map String String, + config :: Map.Map String String, remotes :: [Repo], -- remoteName holds the name used for this repo in remotes remoteName :: Maybe String @@ -211,6 +212,14 @@ notInRepo repo location = do s <- pipeRead repo ["ls-files", "--others", "--exclude-standard", location] return $ lines s +{- Passed a location, returns a list of the files that are staged for + - commit that are being added, moved, or changed (but not deleted). -} +stagedFiles :: Repo -> FilePath -> IO [FilePath] +stagedFiles repo location = do + fs0 <- pipeRead repo ["diff", "--cached", "--name-only", + "--diff-filter=ACMRT", "-z", "HEAD", location] + return $ filter (not . null) $ split "\0" fs0 + {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo configRead repo@(Repo { location = Dir d }) = do @@ -239,8 +248,8 @@ configTrue s = map toLower s == "true" configRemotes :: Repo -> [Repo] configRemotes repo = map construct remotes where - remotes = toList $ filter $ config repo - filter = filterWithKey (\k _ -> isremote k) + remotes = Map.toList $ filter $ config repo + filter = Map.filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } @@ -263,7 +272,7 @@ configGet repo key defaultValue = Map.findWithDefault defaultValue key (config repo) {- Access to raw config Map -} -configMap :: Repo -> Map String String +configMap :: Repo -> Map.Map String String configMap repo = config repo {- Finds the current git repository, which may be in a parent directory. -} From 8e158b7cec7baa77af511e77a6251d4954fb26d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 17:37:05 -0400 Subject: [PATCH 0339/8313] use -z with git-ls-files, to support files with odd chars --- GitRepo.hs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 355600248e..c8ba77133c 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -201,24 +201,30 @@ pipeRead repo params = assertLocal repo $ do {- Passed a location, recursively scans for all files that - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] -inRepo repo location = do - s <- pipeRead repo ["ls-files", "--cached", "--exclude-standard", location] - return $ lines s +inRepo repo location = pipeNullSplit repo + ["ls-files", "--cached", "--exclude-standard", "-z", location] {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} notInRepo :: Repo -> FilePath -> IO [FilePath] -notInRepo repo location = do - s <- pipeRead repo ["ls-files", "--others", "--exclude-standard", location] - return $ lines s +notInRepo repo location = pipeNullSplit repo + ["ls-files", "--others", "--exclude-standard", "-z", location] -{- Passed a location, returns a list of the files that are staged for - - commit that are being added, moved, or changed (but not deleted). -} +{- Passed a location, returns a list of the files, staged for + - commit, that are being added, moved, or changed (but not deleted). -} stagedFiles :: Repo -> FilePath -> IO [FilePath] -stagedFiles repo location = do - fs0 <- pipeRead repo ["diff", "--cached", "--name-only", - "--diff-filter=ACMRT", "-z", "HEAD", location] - return $ filter (not . null) $ split "\0" fs0 +stagedFiles repo location = pipeNullSplit repo + ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", + "HEAD", location] + +{- Reads null terminated output of a git command (as enabled by the -z + - parameter), and splits it into a list of files. -} +pipeNullSplit :: Repo -> [String] -> IO [FilePath] +pipeNullSplit repo params = do + fs0 <- pipeRead repo params + return $ split0 fs0 + where + split0 s = filter (not . null) $ split "\0" s {- Runs git config and populates a repo with its config. -} configRead :: Repo -> IO Repo From c88d4939453845efee04da811d64aa41046f9c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Oct 2010 17:38:12 -0400 Subject: [PATCH 0340/8313] changelog --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index 77e01482fd..4759f7e515 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix support for file:// remotes. * Add --verbose * Fix SIGINT handling. + * Fix handling of files with unusual characters in their name. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From d1fd2c14285adf59f1c4219f0b24fb96b54a9c6c Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sat, 30 Oct 2010 17:25:38 +0000 Subject: [PATCH 0341/8313] --- doc/bugs/building_on_lenny.mdwn | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 doc/bugs/building_on_lenny.mdwn diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn new file mode 100644 index 0000000000..3b24e0ecf0 --- /dev/null +++ b/doc/bugs/building_on_lenny.mdwn @@ -0,0 +1,37 @@ +hi, + +I am trying to build git annex on lenny. + +I checked out the latest from git c88d4939453845efee04da811d64aa41046f9c11, +installed all the packages (some from backports) as required by dpkg-buildpackage + +Then I get this: + + ... + mkdir -p build + ghc -odir build -hidir build --make git-annex + [ 1 of 19] Compiling Utility ( Utility.hs, build/Utility.o ) + [ 2 of 19] Compiling GitRepo ( GitRepo.hs, build/GitRepo.o ) + [ 3 of 19] Compiling GitQueue ( GitQueue.hs, build/GitQueue.o ) + [ 4 of 19] Compiling TypeInternals ( TypeInternals.hs, build/TypeInternals.o ) + [ 5 of 19] Compiling Types ( Types.hs, build/Types.o ) + [ 6 of 19] Compiling Annex ( Annex.hs, build/Annex.o ) + [ 7 of 19] Compiling Locations ( Locations.hs, build/Locations.o ) + [ 8 of 19] Compiling UUID ( UUID.hs, build/UUID.o ) + [ 9 of 19] Compiling LocationLog ( LocationLog.hs, build/LocationLog.o ) + [10 of 19] Compiling Core ( Core.hs, build/Core.o ) + [11 of 19] Compiling Backend.URL ( Backend/URL.hs, build/Backend/URL.o ) + [12 of 19] Compiling Backend ( Backend.hs, build/Backend.o ) + + Backend.hs:114:50: + Not in scope: type constructor or class `SomeException' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/cstamas/tmp/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [build] Error 2 + dpkg-buildpackage: failure: debian/rules build gave error exit status 2 + +I will try to check the mentioned file for error, but I do not know how to program in haskell. + +Thanks for your help! + From 83715949c95be7f50e6ec07e3a7356b2b1f7cad1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:03:34 -0400 Subject: [PATCH 0342/8313] response --- doc/bugs/building_on_lenny.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 3b24e0ecf0..c0e45912ab 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -35,3 +35,6 @@ I will try to check the mentioned file for error, but I do not know how to progr Thanks for your help! +> Newer versions of ghc changed their exception handling types, and +> I coded git-annex to use the new style and not the old. gch6 6.12 will +> work. I do not think there is a backport available though. --[[Joey]] From 23da029b75417fc6acac795204e2579b13011ab0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:10:10 -0400 Subject: [PATCH 0343/8313] Support building with Debian stable's ghc. --- Backend.hs | 1 + Makefile | 2 +- Portability.hs | 9 +++++++++ Remotes.hs | 1 + debian/changelog | 1 + doc/bugs/building_on_lenny.mdwn | 3 +++ 6 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Portability.hs diff --git a/Backend.hs b/Backend.hs index b7c52ddec4..59df37fc4d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -39,6 +39,7 @@ import qualified Annex import Utility import Types import qualified TypeInternals as Internals +import Portability {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend] diff --git a/Makefile b/Makefile index 921eb70670..3ffa29e3b3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs git-annex: mkdir -p build - ghc -odir build -hidir build --make git-annex + ghc -cpp -odir build -hidir build --make git-annex install: install -d $(DESTDIR)/usr/bin diff --git a/Portability.hs b/Portability.hs new file mode 100644 index 0000000000..9cb386a9be --- /dev/null +++ b/Portability.hs @@ -0,0 +1,9 @@ +{- git-annex - Nasty portability workarounds. -} +module Portability where + +-- old ghc does not know about SomeException. +-- http://haskell.1045720.n5.nabble.com/Help-using-catch-in-6-10-td3127921.html#a3127921 +-- This needs ghc -cpp +#if __GLASGOW_HASKELL__ < 610 +type SomeException = Exception +#endif diff --git a/Remotes.hs b/Remotes.hs index 66395adcda..32016b7758 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -40,6 +40,7 @@ import Locations import UUID import Utility import qualified Core +import Portability {- Human visible list of remotes. -} list :: [Git.Repo] -> String diff --git a/debian/changelog b/debian/changelog index 4759f7e515..d6cc73e49e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Add --verbose * Fix SIGINT handling. * Fix handling of files with unusual characters in their name. + * Support building with Debian stable's ghc. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index c0e45912ab..cf5a20a7cf 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -38,3 +38,6 @@ Thanks for your help! > Newer versions of ghc changed their exception handling types, and > I coded git-annex to use the new style and not the old. gch6 6.12 will > work. I do not think there is a backport available though. --[[Joey]] +> +> Ok, found and deployed a workaround. It is not tested. Let me know how it +> works for you. --[[Joey]] From 128eaa7073f7399ffcb4ae7ccd503b00cd4225bc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:11:11 -0400 Subject: [PATCH 0344/8313] note --- Portability.hs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Portability.hs b/Portability.hs index 9cb386a9be..3c2be0e5fc 100644 --- a/Portability.hs +++ b/Portability.hs @@ -2,8 +2,13 @@ module Portability where -- old ghc does not know about SomeException. +-- -- http://haskell.1045720.n5.nabble.com/Help-using-catch-in-6-10-td3127921.html#a3127921 +-- -- This needs ghc -cpp +-- +-- This would be better, but is not packaged in Debian yet. +-- http://hackage.haskell.org/package/extensible-exceptions #if __GLASGOW_HASKELL__ < 610 type SomeException = Exception #endif From b4a218e0784cebfba45fc2ffba81557915565fbd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 15:15:04 -0400 Subject: [PATCH 0345/8313] meh --- Portability.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Portability.hs b/Portability.hs index 3c2be0e5fc..d864d3b938 100644 --- a/Portability.hs +++ b/Portability.hs @@ -7,7 +7,7 @@ module Portability where -- -- This needs ghc -cpp -- --- This would be better, but is not packaged in Debian yet. +-- This would be better, but then users of old ghc would need to install it. -- http://hackage.haskell.org/package/extensible-exceptions #if __GLASGOW_HASKELL__ < 610 type SomeException = Exception From 2be74a60df229c0e887b9f6c30ea1195a64e83a0 Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sat, 30 Oct 2010 19:40:17 +0000 Subject: [PATCH 0346/8313] reply to joey --- doc/bugs/building_on_lenny.mdwn | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index cf5a20a7cf..d394182500 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -33,7 +33,7 @@ Then I get this: I will try to check the mentioned file for error, but I do not know how to program in haskell. -Thanks for your help! +Thanks for your help! --[[cstamas]] > Newer versions of ghc changed their exception handling types, and > I coded git-annex to use the new style and not the old. gch6 6.12 will @@ -41,3 +41,19 @@ Thanks for your help! > > Ok, found and deployed a workaround. It is not tested. Let me know how it > works for you. --[[Joey]] +>> +>> I did a git pull and now I get: + + mkdir -p build + ghc -cpp -odir build -hidir build --make git-annex + [ 1 of 20] Compiling Portability ( Portability.hs, build/Portability.o ) + + Portability.hs:13:21: + Not in scope: type constructor or class `Exception' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/cstamas/tmp/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [build] Error 2 + dpkg-buildpackage: failure: debian/rules build gave error exit status 2 + +>> --[[cstamas]] From 8edc7a0e434f9574e1f46682acb3f5c5321e825e Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sat, 30 Oct 2010 19:46:39 +0000 Subject: [PATCH 0347/8313] formatting fix --- doc/bugs/building_on_lenny.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index d394182500..07b359e855 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -41,7 +41,7 @@ Thanks for your help! --[[cstamas]] > > Ok, found and deployed a workaround. It is not tested. Let me know how it > works for you. --[[Joey]] ->> + >> I did a git pull and now I get: mkdir -p build From fad1616e68b2219a7e1a6c4b7e1cfe8485ad8c2b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 17:26:45 -0400 Subject: [PATCH 0348/8313] build fix --- Portability.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Portability.hs b/Portability.hs index d864d3b938..91c8334cc9 100644 --- a/Portability.hs +++ b/Portability.hs @@ -1,5 +1,7 @@ {- git-annex - Nasty portability workarounds. -} -module Portability where +module Portability (SomeException) where + +import Control.Exception -- old ghc does not know about SomeException. -- From 583e8118d4e77fc4f76474717a821c56e3cf772d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 17:29:11 -0400 Subject: [PATCH 0349/8313] ok, let's just use extensible-exceptions --- Backend.hs | 3 +-- Portability.hs | 16 ---------------- Remotes.hs | 3 +-- 3 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 Portability.hs diff --git a/Backend.hs b/Backend.hs index 59df37fc4d..00b2833e03 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,7 +27,7 @@ module Backend ( ) where import Control.Monad.State -import Control.Exception +import Control.Exception.Extensible import System.Directory import System.FilePath import Data.String.Utils @@ -39,7 +39,6 @@ import qualified Annex import Utility import Types import qualified TypeInternals as Internals -import Portability {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend] diff --git a/Portability.hs b/Portability.hs deleted file mode 100644 index 91c8334cc9..0000000000 --- a/Portability.hs +++ /dev/null @@ -1,16 +0,0 @@ -{- git-annex - Nasty portability workarounds. -} -module Portability (SomeException) where - -import Control.Exception - --- old ghc does not know about SomeException. --- --- http://haskell.1045720.n5.nabble.com/Help-using-catch-in-6-10-td3127921.html#a3127921 --- --- This needs ghc -cpp --- --- This would be better, but then users of old ghc would need to install it. --- http://hackage.haskell.org/package/extensible-exceptions -#if __GLASGOW_HASKELL__ < 610 -type SomeException = Exception -#endif diff --git a/Remotes.hs b/Remotes.hs index 32016b7758..27bd39ead7 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -17,7 +17,7 @@ module Remotes ( ) where import IO (bracket_) -import Control.Exception hiding (bracket_) +import Control.Exception.Extensible hiding (bracket_) import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map @@ -40,7 +40,6 @@ import Locations import UUID import Utility import qualified Core -import Portability {- Human visible list of remotes. -} list :: [Git.Repo] -> String From 765c9fa82d958259d7beaf8edc51f06a0db31a00 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 17:38:20 -0400 Subject: [PATCH 0350/8313] meh --- doc/bugs/building_on_lenny.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 07b359e855..388db2010d 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -57,3 +57,9 @@ Thanks for your help! --[[cstamas]] dpkg-buildpackage: failure: debian/rules build gave error exit status 2 >> --[[cstamas]] + +>>> Ok well, I'm not going to try to reimplement all of +>>> Control.Exception.Extensible so I've made it use it. You will have to +>>> figure out how to install that library yourself though, I don't know +>>> how to use cabal with such an old ghc. Library is here: +>>> --[[Joey]] From 465fb0ebc4e9173606303a719cc753b45e133b20 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 20:43:05 -0400 Subject: [PATCH 0351/8313] link to ask.debian.net --- doc/bugs/building_on_lenny.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 388db2010d..654b1283e0 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -62,4 +62,6 @@ Thanks for your help! --[[cstamas]] >>> Control.Exception.Extensible so I've made it use it. You will have to >>> figure out how to install that library yourself though, I don't know >>> how to use cabal with such an old ghc. Library is here: ->>> --[[Joey]] +>>> +>>> and I asked how to get it on stable here: +>>> --[[Joey]] From 963bfa967369511bbe3261860c3609b5ae9554ec Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 22:47:34 -0400 Subject: [PATCH 0352/8313] cpp not needed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3ffa29e3b3..921eb70670 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs git-annex: mkdir -p build - ghc -cpp -odir build -hidir build --make git-annex + ghc -odir build -hidir build --make git-annex install: install -d $(DESTDIR)/usr/bin From c2651d64bc8a384f8669879c051577ee61da27ed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 23:19:33 -0400 Subject: [PATCH 0353/8313] Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. --- GitRepo.hs | 16 +++++++++++++++- debian/changelog | 2 ++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index c8ba77133c..b6670e2ebe 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -27,6 +27,7 @@ module GitRepo ( configTrue, run, pipeRead, + hPipeRead, attributes, remotes, remotesAdd, @@ -198,6 +199,17 @@ pipeRead repo params = assertLocal repo $ do ret <- hGetContentsStrict h return ret +{- Like pipeRead, but does not read output strictly; recommended + - for git commands that produce a lot of output that will be processed + - lazily. + - + - ONLY AFTER the string has been read completely, You must call either + - getProcessStatus or forceSuccess on the PipeHandle. Zombies will result + - otherwise.-} +hPipeRead :: Repo -> [String] -> IO (PipeHandle, String) +hPipeRead repo params = assertLocal repo $ do + pipeFrom "git" (gitCommandLine repo params) + {- Passed a location, recursively scans for all files that - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] @@ -221,7 +233,9 @@ stagedFiles repo location = pipeNullSplit repo - parameter), and splits it into a list of files. -} pipeNullSplit :: Repo -> [String] -> IO [FilePath] pipeNullSplit repo params = do - fs0 <- pipeRead repo params + -- XXX handle is left open, this is ok for git-annex, but may need + -- to be cleaned up for other uses. + (handle, fs0) <- hPipeRead repo params return $ split0 fs0 where split0 s = filter (not . null) $ split "\0" s diff --git a/debian/changelog b/debian/changelog index d6cc73e49e..70e9052231 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.03) UNRELEASED; urgency=low * Fix SIGINT handling. * Fix handling of files with unusual characters in their name. * Support building with Debian stable's ghc. + * Fixed memory leak; git-annex no longer reads the whole file list + from git before starting, and will be much faster with large repos. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From d5a0c16298a625d80288276a92dcfebf1cb9b3de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 30 Oct 2010 23:34:40 -0400 Subject: [PATCH 0354/8313] more space saving by not locking location log for read Actions that need to read all the location logs, like "git annex get .", were still using a lot of memory, and profiling pointed at the location log reading as the problem. Not locking them for read, and thus avoiding the strict reading fixes the problem, although I don't quite understand why. (Oddly, -sstderr profiling did not show the memory as used, though top showed dozens of MB being used.) Anyway, it's fine to not lock location logs for read, since the log format and parser should be safe if a partial read of a file being written happens. Note that that could easily happen anyway, if doing a git pull, etc, especially if git needs to union merge in changes from elsewhere. The worst that will happen is git-annex could get a bad or out of date idea about locations and refuse to eg, --drop something. --- LocationLog.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index f92dee652d..20e777c567 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -104,8 +104,7 @@ readLog file = do exists <- doesFileExist file if exists then do - s <- withFileLocked file ReadMode $ \h -> - hGetContentsStrict h + s <- readFile file -- filter out any unparsable lines return $ filter (\l -> (status l) /= Undefined ) $ map read $ lines s From 59672d32edd9cc30942a9200670b3e1f817d26aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 01:06:58 -0400 Subject: [PATCH 0355/8313] write to tmp file Writing to a tmp file means no locking is needed, and it fixes a bug introduced by the last commit, which made log files be read lazily, so they could still be open when written, which breaks due to haskell's internal locking. --- LocationLog.hs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 20e777c567..14ae88abc3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -35,6 +35,7 @@ import qualified Data.Map as Map import System.IO import System.Directory import Data.Char +import System.Posix.Process import qualified GitRepo as Git import Utility @@ -111,19 +112,14 @@ readLog file = do else do return [] -{- Adds a LogLine to a log file -} -appendLog :: FilePath -> LogLine -> IO () -appendLog file line = do - createDirectoryIfMissing True (parentDir file) - withFileLocked file AppendMode $ \h -> - hPutStrLn h $ show line - {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () writeLog file lines = do + pid <- getProcessID + let tmpfile = file ++ ".tmp" ++ show pid createDirectoryIfMissing True (parentDir file) - withFileLocked file WriteMode $ \h -> - hPutStr h $ unlines $ map show lines + writeFile tmpfile $ unlines $ map show lines + renameFile tmpfile file {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> UUID -> IO LogLine From 6a9a9bd5a316f143964574e5ed12eb69b1d17c41 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 01:51:50 -0400 Subject: [PATCH 0356/8313] another memory optimisation This time memory leaked if lots of UUIDs needed to be pretty-printed, as in a get or drop of many files. Essentially the same strict read buffering problem that affected the LocationLog underneath though. uuidMap really could stand to be cached, as the uuid log is read many times in this case. But it is a fairly edge case. --- UUID.hs | 10 ++++++---- Utility.hs | 16 ---------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/UUID.hs b/UUID.hs index f2235e4b62..20f85bae12 100644 --- a/UUID.hs +++ b/UUID.hs @@ -26,6 +26,7 @@ import System.Cmd.Utils import System.IO import System.Directory import qualified Data.Map as M +import System.Posix.Process import qualified GitRepo as Git import Types @@ -111,8 +112,11 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m log <- uuidLog + pid <- liftIO $ getProcessID + let tmplog = log ++ ".tmp" ++ show pid liftIO $ createDirectoryIfMissing True (parentDir log) - liftIO $ withFileLocked log WriteMode (\h -> hPutStr h $ serialize m') + liftIO $ writeFile tmplog $ serialize m' + liftIO $ renameFile tmplog log where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m @@ -120,9 +124,7 @@ describeUUID uuid desc = do uuidMap :: Annex (M.Map UUID String) uuidMap = do log <- uuidLog - s <- liftIO $ catch - (withFileLocked log ReadMode $ \h -> hGetContentsStrict h) - (\error -> return "") + s <- liftIO $ catch (readFile log) (\error -> return "") return $ M.fromList $ map (\l -> pair l) $ lines s where pair l = diff --git a/Utility.hs b/Utility.hs index 233825b651..e7b4b510b8 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,7 +6,6 @@ -} module Utility ( - withFileLocked, hGetContentsStrict, parentDir, relPathCwdToDir, @@ -28,21 +27,6 @@ import System.IO.HVFS import System.FilePath import System.Directory -{- Let's just say that Haskell makes reading/writing a file with - - file locking excessively difficult. -} -withFileLocked file mode action = do - -- TODO: find a way to use bracket here - handle <- openFile file mode - lockfd <- handleToFd handle -- closes handle - waitToSetLock lockfd (lockType mode, AbsoluteSeek, 0, 0) - handle' <- fdToHandle lockfd - ret <- action handle' - hClose handle' - return ret - where - lockType ReadMode = ReadLock - lockType _ = WriteLock - {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s From bcfd0d908a3e52ca61aa295ddebf7b1898224ab1 Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sun, 31 Oct 2010 16:06:00 +0000 Subject: [PATCH 0357/8313] building on squeeze --- doc/bugs/building_on_lenny.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index 654b1283e0..fee13717ff 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -65,3 +65,15 @@ Thanks for your help! --[[cstamas]] >>> >>> and I asked how to get it on stable here: >>> --[[Joey]] + +>>>> I made some effort with cabal on lenny. I can install (and I did it) cabal +>>>> from squeeze as dependencies are ok. Then I installed extensible +>>>> exceptions, but it places it in some local dir that git-annex's installer +>>>> (or ghc itself) does not know about. +>>>> +>>>> Later I realized that *only* for the compilation ghc6 and its friends are +>>>> needed. So I built the package on my other machine running squeeze. Then +>>>> resulting deb packages cleanly installs on lenny +>>>> +>>>> For me this is OK. Thanks! --[[cstamas]] + From 081517b6e9111536cd67c1f501adf07497862ce5 Mon Sep 17 00:00:00 2001 From: "http://users.itk.ppke.hu/~cstamas/openid/" Date: Sun, 31 Oct 2010 16:10:22 +0000 Subject: [PATCH 0358/8313] add 'done' link --- doc/bugs/building_on_lenny.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/bugs/building_on_lenny.mdwn b/doc/bugs/building_on_lenny.mdwn index fee13717ff..48386bde47 100644 --- a/doc/bugs/building_on_lenny.mdwn +++ b/doc/bugs/building_on_lenny.mdwn @@ -77,3 +77,4 @@ Thanks for your help! --[[cstamas]] >>>> >>>> For me this is OK. Thanks! --[[cstamas]] +[[done]] From dc12ce762e521a5db052346eb67590ca62e4f2f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:23:51 -0400 Subject: [PATCH 0359/8313] -Wall clean --- Remotes.hs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 27bd39ead7..b9c1b48f3d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -22,19 +22,14 @@ import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils -import Data.Either.Utils -import System.Cmd.Utils import System.Directory import System.Posix.Directory import List -import Maybe import Monad (when, unless) import Types import qualified GitRepo as Git import qualified Annex -import qualified Backend -import qualified Core import LocationLog import Locations import UUID @@ -73,7 +68,7 @@ keyPossibilities key = do let todo = cheap ++ doexpensive if (not $ null todo) then do - e <- mapM tryGitConfigRead todo + _ <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True keyPossibilities key else reposByUUID allremotes uuids @@ -121,14 +116,14 @@ reposByCost l = do repoCost :: Git.Repo -> Annex Int repoCost r = do g <- Annex.gitRepo - if (not $ null $ config g r) - then return $ read $ config g r + if (not $ null $ config g) + then return $ read $ config g else if (Git.repoIsUrl r) then return 200 else return 100 where - config g r = Git.configGet g (configkey r) "" - configkey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" + config g = Git.configGet g configkey "" + configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- Checks if a repo should be ignored, based either on annex-ignore - setting, or on command-line options. Allows command-line to override @@ -174,7 +169,7 @@ tryGitConfigRead r = do -- for other reasons; catch all possible exceptions result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) case (result) of - Left e -> return $ Left r + Left _ -> return $ Left r Right r' -> do g <- Annex.gitRepo let l = Git.remotes g @@ -184,7 +179,7 @@ tryGitConfigRead r = do return $ Right r' else return $ Right r -- config already read where - exchange [] new = [] + exchange [] _ = [] exchange (old:ls) new = if (Git.repoRemoteName old == Git.repoRemoteName new) then new:(exchange ls new) From 1576c48c80e4806b6021ec66f0dc645cf0a83486 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:32:18 -0400 Subject: [PATCH 0360/8313] more Wall cleaning --- Annex.hs | 5 +++-- Makefile | 2 +- git-annex.hs | 5 ++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Annex.hs b/Annex.hs index 60ae91708b..63eef51589 100644 --- a/Annex.hs +++ b/Annex.hs @@ -39,15 +39,16 @@ new gitrepo allbackends = do Internals.flags = M.empty, Internals.repoqueue = GitQueue.empty } - (_,s') <- Annex.run s (prep gitrepo) + (_,s') <- Annex.run s prep return s' where - prep gitrepo = do + prep = do -- read git config and update state gitrepo' <- liftIO $ Git.configRead gitrepo Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} +run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) run state action = runStateT (action) state {- Returns the git repository being acted on -} diff --git a/Makefile b/Makefile index 921eb70670..ed262333a3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs git-annex: mkdir -p build - ghc -odir build -hidir build --make git-annex + ghc -Wall -odir build -hidir build --make git-annex install: install -d $(DESTDIR)/usr/bin diff --git a/git-annex.hs b/git-annex.hs index 5011fade2b..e9e0d6f027 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -17,6 +17,7 @@ import Commands import qualified GitRepo as Git import BackendList +main :: IO () main = do args <- getArgs gitrepo <- Git.repoFromCwd @@ -35,6 +36,7 @@ main = do -} tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions +tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case (result) of @@ -43,8 +45,9 @@ tryRun' state errnum (a:as) = do tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' state errnum [] = +tryRun' _ errnum [] = when (errnum > 0) $ error $ (show errnum) ++ " failed" {- Exception pretty-printing. -} +showErr :: (Show a) => a -> IO () showErr e = hPutStrLn stderr $ "git-annex: " ++ (show e) From 2d893b3331f5515b179d7541c9b4cb1f30162fce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:39:53 -0400 Subject: [PATCH 0361/8313] more Wall cleaning --- Backend.hs | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/Backend.hs b/Backend.hs index 00b2833e03..d75c2a761e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -28,15 +28,12 @@ module Backend ( import Control.Monad.State import Control.Exception.Extensible -import System.Directory import System.FilePath -import Data.String.Utils import System.Posix.Files import Locations import qualified GitRepo as Git import qualified Annex -import Utility import Types import qualified TypeInternals as Internals @@ -47,28 +44,28 @@ list = do if (not $ null l) then return l else do - all <- Annex.supportedBackends + bs <- Annex.supportedBackends g <- Annex.gitRepo - let l = parseBackendList all $ Git.configGet g "annex.backends" "" + let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" let l' = if (not $ null backendflag) - then (lookupBackendName all backendflag):l - else l + then (lookupBackendName bs backendflag):defaults + else defaults Annex.backendsChange $ l' return l' where - parseBackendList all s = + parseBackendList bs s = if (null s) - then all - else map (lookupBackendName all) $ words s + then bs + else map (lookupBackendName bs) $ words s {- Looks up a backend in a list -} lookupBackendName :: [Backend] -> String -> Backend -lookupBackendName all s = +lookupBackendName bs s = if ((length matches) /= 1) then error $ "unknown backend " ++ s else matches !! 0 - where matches = filter (\b -> s == Internals.name b) all + where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) @@ -77,10 +74,11 @@ storeFileKey file = do let relfile = Git.relative g file b <- list storeFileKey' b file relfile +storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend)) storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do - try <- (Internals.getKey b) relfile - case (try) of + result <- (Internals.getKey b) relfile + case (result) of Nothing -> nextbackend Just key -> do stored <- (Internals.storeFileKey b) file key @@ -103,23 +101,23 @@ removeKey backend key = (Internals.removeKey backend) key {- Checks if a backend has its key. -} hasKey :: Key -> Annex Bool hasKey key = do - all <- Annex.supportedBackends - (Internals.hasKey (lookupBackendName all $ backendName key)) key + bs <- Annex.supportedBackends + (Internals.hasKey (lookupBackendName bs $ backendName key)) key {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) lookupFile file = do - all <- Annex.supportedBackends - result <- liftIO $ (try (lookup all)::IO (Either SomeException (Maybe (Key, Backend)))) + bs <- Annex.supportedBackends + result <- liftIO $ (try (find bs)::IO (Either SomeException (Maybe (Key, Backend)))) case (result) of - Left err -> return Nothing - Right succ -> return succ + Left _ -> return Nothing + Right val -> return val where - lookup all = do + find bs = do l <- readSymbolicLink file - return $ Just $ pair all $ takeFileName l - pair all file = (k, b) + return $ Just $ pair bs $ takeFileName l + pair bs f = (k, b) where - k = fileKey file - b = lookupBackendName all $ backendName k + k = fileKey f + b = lookupBackendName bs $ backendName k From 40729e4bfdf4fe9753705a5a7a93cf1e0012a92c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 14:40:34 -0400 Subject: [PATCH 0362/8313] more Wall cleaning --- BackendList.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BackendList.hs b/BackendList.hs index e628e3a612..59ec026d91 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -11,6 +11,9 @@ module BackendList (allBackends) where import qualified Backend.WORM import qualified Backend.SHA1 import qualified Backend.URL +import TypeInternals + +allBackends :: [Backend] allBackends = [ Backend.WORM.backend , Backend.SHA1.backend From 28b5a9fa2048f4e6425cc1d8dbe13b0e4c36a15b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:09:50 -0400 Subject: [PATCH 0363/8313] changelog --- Commands.hs | 146 +++++++++++++++++++++++++++------------------------- 1 file changed, 76 insertions(+), 70 deletions(-) diff --git a/Commands.hs b/Commands.hs index 2af8874e5c..fb76e5502f 100644 --- a/Commands.hs +++ b/Commands.hs @@ -11,12 +11,9 @@ import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory -import System.Path import Data.String.Utils import Control.Monad (filterM) import Monad (when, unless) -import List -import IO import qualified GitRepo as Git import qualified Annex @@ -50,19 +47,19 @@ type SubCmdCleanup = Annex Bool {- Runs a subcommand through its three stages. -} doSubCmd :: String -> SubCmdStart -> String -> Annex Bool doSubCmd cmdname start param = do - res <- start param :: Annex (Maybe SubCmdPerform) - case (res) of + startres <- start param :: Annex (Maybe SubCmdPerform) + case (startres) of Nothing -> return True Just perform -> do showStart cmdname param - res <- perform :: Annex (Maybe SubCmdCleanup) - case (res) of + performres <- perform :: Annex (Maybe SubCmdCleanup) + case (performres) of Nothing -> do showEndFail return False Just cleanup -> do - res <- cleanup - if (res) + cleanupres <- cleanup + if (cleanupres) then do showEndOk return True @@ -76,7 +73,7 @@ doSubCmd cmdname start param = do data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing | Description | Keys | Tempfile | FilesToBeCommitted -data SubCommand = Command { +data SubCommand = SubCommand { subcmdname :: String, subcmdaction :: SubCmdStart, subcmdwants :: SubCmdWants, @@ -84,27 +81,27 @@ data SubCommand = Command { } subCmds :: [SubCommand] subCmds = [ - (Command "add" addStart FilesNotInGit + (SubCommand "add" addStart FilesNotInGit "add files to annex") - , (Command "get" getStart FilesInGit + , (SubCommand "get" getStart FilesInGit "make content of annexed files available") - , (Command "drop" dropStart FilesInGit + , (SubCommand "drop" dropStart FilesInGit "indicate content of files not currently wanted") - , (Command "move" moveStart FilesInGit + , (SubCommand "move" moveStart FilesInGit "transfer content of files to/from another repository") - , (Command "init" initStart Description + , (SubCommand "init" initStart Description "initialize git-annex with repository description") - , (Command "unannex" unannexStart FilesInGit + , (SubCommand "unannex" unannexStart FilesInGit "undo accidential add command") - , (Command "fix" fixStart FilesInGit + , (SubCommand "fix" fixStart FilesInGit "fix up symlinks to point to annexed content") - , (Command "pre-commit" fixStart FilesToBeCommitted + , (SubCommand "pre-commit" fixStart FilesToBeCommitted "fix up symlinks before they are committed") - , (Command "fromkey" fromKeyStart FilesMissing + , (SubCommand "fromkey" fromKeyStart FilesMissing "adds a file using a specific key") - , (Command "dropkey" dropKeyStart Keys + , (SubCommand "dropkey" dropKeyStart Keys "drops annexed content for specified keys") - , (Command "setkey" setKeyStart Tempfile + , (SubCommand "setkey" setKeyStart Tempfile "sets annexed content for a key using a temp file") ] @@ -131,6 +128,7 @@ options = [ storebool n b = Annex.flagChange n $ FlagBool b storestring n s = Annex.flagChange n $ FlagString s +header :: String header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) {- Usage message with lists of options and subcommands. -} @@ -162,7 +160,7 @@ findWanted FilesNotInGit params repo = do findWanted FilesInGit params repo = do files <- mapM (Git.inRepo repo) params return $ foldl (++) [] files -findWanted FilesMissing params repo = do +findWanted FilesMissing params _ = do files <- liftIO $ filterM missing params return $ files where @@ -186,15 +184,17 @@ parseCmd argv state = do when (null params) $ error usage case lookupCmd (params !! 0) of [] -> error usage - [Command name action want _] -> do - f <- findWanted want (drop 1 params) + [SubCommand { subcmdname = name, subcmdaction = action, + subcmdwants = want, subcmddesc = _ }] -> do + files <- findWanted want (drop 1 params) (TypeInternals.repo state) let actions = map (doSubCmd name action) $ - filter notstate f - let configactions = map (\f -> do - f + filter notstate files + let configactions = map (\flag -> do + flag return True) flags return (configactions, actions) + _ -> error "internal error: multiple matching subcommands" where -- never include files from the state directory notstate f = stateLoc /= take (length stateLoc) f @@ -214,11 +214,10 @@ addStart file = notAnnexed file $ do else return $ Just $ addPerform file addPerform :: FilePath -> Annex (Maybe SubCmdCleanup) addPerform file = do - g <- Annex.gitRepo stored <- Backend.storeFileKey file case (stored) of Nothing -> return Nothing - Just (key, backend) -> return $ Just $ addCleanup file key + Just (key, _) -> return $ Just $ addCleanup file key addCleanup :: FilePath -> Key -> Annex Bool addCleanup file key = do logStatus key ValuePresent @@ -239,8 +238,10 @@ unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True - Backend.removeKey backend key - return $ Just $ unannexCleanup file key + ok <- Backend.removeKey backend key + if (ok) + then return $ Just $ unannexCleanup file key + else return Nothing unannexCleanup :: FilePath -> Key -> Annex Bool unannexCleanup file key = do logStatus key ValueMissing @@ -259,9 +260,9 @@ getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing - else return $ Just $ getPerform file key backend -getPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) -getPerform file key backend = do + else return $ Just $ getPerform key backend +getPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) then return $ Just $ return True -- no cleanup needed @@ -331,15 +332,15 @@ setKeyPerform tmpfile key = do ok <- liftIO $ boolSystem "mv" [tmpfile, loc] if (not ok) then error "mv failed!" - else return $ Just $ setKeyCleanup tmpfile key -setKeyCleanup :: FilePath -> Key -> Annex Bool -setKeyCleanup tmpfile key = do + else return $ Just $ setKeyCleanup key +setKeyCleanup :: Key -> Annex Bool +setKeyCleanup key = do logStatus key ValuePresent return True {- Fixes the symlink to an annexed file. -} fixStart :: FilePath -> Annex (Maybe SubCmdPerform) -fixStart file = isAnnexed file $ \(key, backend) -> do +fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -373,9 +374,9 @@ initPerform description = do initCleanup :: Annex Bool initCleanup = do g <- Annex.gitRepo - log <- uuidLog - liftIO $ Git.run g ["add", log] - liftIO $ Git.run g ["commit", "-m", "git annex init", log] + logfile <- uuidLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] return True {- Adds a file pointing at a manually-specified key -} @@ -411,9 +412,9 @@ moveStart file = do toName <- Annex.flagGet "torepository" case (fromName, toName) of ("", "") -> error "specify either --from or --to" - ("", to) -> moveToStart file - (from, "") -> moveFromStart file - (_, _) -> error "only one of --from or --to can be specified" + ("", _) -> moveToStart file + (_ , "") -> moveFromStart file + (_ , _) -> error "only one of --from or --to can be specified" {- Moves the content of an annexed file to another repository, - removing it from the current repository, and updates locationlog @@ -427,13 +428,13 @@ moveStart file = do - allow it to be dropped. -} moveToStart :: FilePath -> Annex (Maybe SubCmdPerform) -moveToStart file = isAnnexed file $ \(key, backend) -> do +moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) then return Nothing -- not here, so nothing to do - else return $ Just $ moveToPerform file key -moveToPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) -moveToPerform file key = do + else return $ Just $ moveToPerform key +moveToPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key @@ -452,18 +453,21 @@ moveToPerform file key = do moveToCleanup :: Git.Repo -> Key -> FilePath -> Annex Bool moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. - Remotes.runCmd remote "git-annex" ["setkey", "--quiet", + ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", "--backend=" ++ (backendName key), "--key=" ++ keyName key, tmpfile] - -- Record that the key is present on the remote. - g <- Annex.gitRepo - remoteuuid <- getUUID remote - log <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" [] log - -- Cleanup on the local side is the same as done for the - -- drop subcommand. - dropCleanup key + if ok + then do + -- Record that the key is present on the remote. + g <- Annex.gitRepo + remoteuuid <- getUUID remote + logfile <- liftIO $ logChange g key remoteuuid ValuePresent + Annex.queue "add" [] logfile + -- Cleanup on the local side is the same as done for the + -- drop subcommand. + dropCleanup key + else return False {- Moves the content of an annexed file from another repository to the current - repository and updates locationlog information on both. @@ -472,15 +476,14 @@ moveToCleanup remote key tmpfile = do - from the other repository. -} moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) -moveFromStart file = isAnnexed file $ \(key, backend) -> do - g <- Annex.gitRepo +moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key if (elem remote l) - then return $ Just $ moveFromPerform file key + then return $ Just $ moveFromPerform key else return Nothing -moveFromPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) -moveFromPerform file key = do +moveFromPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key if (ishere) @@ -493,22 +496,25 @@ moveFromPerform file key = do else return Nothing -- fail moveFromCleanup :: Git.Repo -> Key -> Annex Bool moveFromCleanup remote key = do - Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", + ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), keyName key] - -- Record locally that the key is not on the remote. - remoteuuid <- getUUID remote - g <- Annex.gitRepo - log <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" [] log - return True + when ok $ do + -- Record locally that the key is not on the remote. + remoteuuid <- getUUID remote + g <- Annex.gitRepo + logfile <- liftIO $ logChange g key remoteuuid ValueMissing + Annex.queue "add" [] logfile + return ok -- helpers +notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) notAnnexed file a = do r <- Backend.lookupFile file case (r) of - Just v -> return Nothing + Just _ -> return Nothing Nothing -> a +isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) isAnnexed file a = do r <- Backend.lookupFile file case (r) of From 96fa6b89acdd77498140790553a42c81250816ed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:12:56 -0400 Subject: [PATCH 0364/8313] more Wall cleaning --- Core.hs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Core.hs b/Core.hs index 8cdc6979ab..5f799161cf 100644 --- a/Core.hs +++ b/Core.hs @@ -7,7 +7,6 @@ module Core where -import Maybe import System.IO import System.Directory import Control.Monad.State (liftIO) @@ -27,7 +26,6 @@ import Utility {- Sets up a git repo for git-annex. -} startup :: Annex Bool startup = do - g <- Annex.gitRepo prepUUID return True @@ -118,8 +116,8 @@ logStatus :: Key -> LogStatus -> Annex () logStatus key status = do g <- Annex.gitRepo u <- getUUID g - log <- liftIO $ logChange g key u status - Annex.queue "add" [] log + logfile <- liftIO $ logChange g key u status + Annex.queue "add" [] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into @@ -158,9 +156,9 @@ showProgress :: Annex () showProgress = verbose $ liftIO $ putStr $ "\n" showLongNote :: String -> Annex () showLongNote s = verbose $ do - liftIO $ putStr $ "\n" ++ (indent s) + liftIO $ putStr $ "\n" ++ indented where - indent s = join "\n" $ map (\l -> " " ++ l) $ lines s + indented = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () showEndOk = verbose $ do liftIO $ putStrLn "ok" From 435ec21d58f9c7ddb622f290648b31da2c9a8f1e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:25:55 -0400 Subject: [PATCH 0365/8313] bugfix: really run GitQueue against specified repo, not necessarily pwd --- GitQueue.hs | 12 ++++++------ GitRepo.hs | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/GitQueue.hs b/GitQueue.hs index 09b8037e64..632d1d3910 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -23,8 +23,8 @@ import qualified GitRepo as Git {- An action to perform in a git repository. The file to act on - is not included, and must be able to be appended after the params. -} data Action = Action { - subcommand :: String, - params :: [String] + getSubcommand :: String, + getParams :: [String] } deriving (Show, Eq, Ord) {- A queue of actions to perform (in any order) on a git repository, @@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue {- Runs a queue on a git repository. -} run :: Git.Repo -> Queue -> IO () run repo queue = do - mapM (\(k, v) -> runAction repo k v) $ M.toList queue + _ <- mapM (\(k, v) -> runAction repo k v) $ M.toList queue return () {- Runs an Action on a list of files in a git repository. @@ -55,7 +55,7 @@ runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do unless (null files) runxargs where - runxargs = pOpen WriteToPipe "xargs" - (["-0", "git", subcommand action] ++ (params action)) - feedxargs + runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs + gitcmd = ["git"] ++ Git.gitCommandLine repo + ((getSubcommand action):(getParams action)) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index b6670e2ebe..9ecd3923aa 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -25,6 +25,7 @@ module GitRepo ( configMap, configRead, configTrue, + gitCommandLine, run, pipeRead, hPipeRead, From aa05859410b43da72c3165b7915d8c917bc015ca Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:38:47 -0400 Subject: [PATCH 0366/8313] more Wall cleaning --- Core.hs | 2 +- GitRepo.hs | 57 ++++++++++++++++++++++++++++++------------------------ 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/Core.hs b/Core.hs index 5f799161cf..c5026c6a5f 100644 --- a/Core.hs +++ b/Core.hs @@ -74,7 +74,7 @@ gitAttributes repo = do {- set up a git pre-commit hook, if one is not already present -} gitPreCommitHook :: Git.Repo -> IO () gitPreCommitHook repo = do - let hook = (Git.workTree repo) ++ "/" ++ (Git.dir repo) ++ + let hook = (Git.workTree repo) ++ "/" ++ (Git.gitDir repo) ++ "/hooks/pre-commit" exists <- doesFileExist hook if (exists) diff --git a/GitRepo.hs b/GitRepo.hs index 9ecd3923aa..874b5c3c9b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -17,7 +17,7 @@ module GitRepo ( repoIsSsh, repoDescribe, workTree, - dir, + gitDir, relative, urlPath, urlHost, @@ -38,17 +38,14 @@ module GitRepo ( stagedFiles ) where -import Monad (when, unless) +import Monad (unless) import Directory -import System -import System.Directory import System.Posix.Directory import System.Path -import System.Cmd import System.Cmd.Utils -import System.IO import IO (bracket_) import Data.String.Utils +import System.IO import qualified Data.Map as Map hiding (map, split) import Network.URI import Maybe @@ -69,6 +66,7 @@ data Repo = Repo { remoteName :: Maybe String } deriving (Show, Eq) +newFrom :: RepoLocation -> Repo newFrom l = Repo { location = l, @@ -89,6 +87,7 @@ repoFromUrl url where u = fromJust $ parseURI url {- User-visible description of a git repo. -} +repoDescribe :: Repo -> String repoDescribe Repo { remoteName = Just name } = name repoDescribe Repo { location = Url url } = show url repoDescribe Repo { location = Dir dir } = dir @@ -100,29 +99,35 @@ remotesAdd repo rs = repo { remotes = rs } {- Returns the name of the remote that corresponds to the repo, if - it is a remote. Otherwise, "" -} +repoRemoteName :: Repo -> String repoRemoteName Repo { remoteName = Just name } = name repoRemoteName _ = "" {- Some code needs to vary between URL and normal repos, - or bare and non-bare, these functions help with that. -} +repoIsUrl :: Repo -> Bool repoIsUrl Repo { location = Url _ } = True repoIsUrl _ = False +repoIsSsh :: Repo -> Bool repoIsSsh Repo { location = Url url } | uriScheme url == "ssh:" = True | otherwise = False repoIsSsh _ = False +assertLocal :: Repo -> a -> a assertLocal repo action = if (not $ repoIsUrl repo) then action else error $ "acting on URL git repo " ++ (repoDescribe repo) ++ " not supported" +assertUrl :: Repo -> a -> a assertUrl repo action = if (repoIsUrl repo) then action else error $ "acting on local git repo " ++ (repoDescribe repo) ++ " not supported" +assertSsh :: Repo -> a -> a assertSsh repo action = if (repoIsSsh repo) then action @@ -141,8 +146,8 @@ attributes repo | otherwise = (workTree repo) ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its workTree. -} -dir :: Repo -> String -dir repo +gitDir :: Repo -> String +gitDir repo | bare repo = "" | otherwise = ".git" @@ -167,7 +172,7 @@ relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo -relative repo file = assertLocal repo $ error "internal" +relative repo _ = assertLocal repo $ error "internal" {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String @@ -184,7 +189,7 @@ urlPath repo = assertUrl repo $ error "internal" gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++d++"/"++(dir repo), "--work-tree="++d] ++ params + ["--git-dir="++d++"/"++(gitDir repo), "--work-tree="++d] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} @@ -214,21 +219,21 @@ hPipeRead repo params = assertLocal repo $ do {- Passed a location, recursively scans for all files that - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] -inRepo repo location = pipeNullSplit repo - ["ls-files", "--cached", "--exclude-standard", "-z", location] +inRepo repo l = pipeNullSplit repo + ["ls-files", "--cached", "--exclude-standard", "-z", l] {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} notInRepo :: Repo -> FilePath -> IO [FilePath] -notInRepo repo location = pipeNullSplit repo - ["ls-files", "--others", "--exclude-standard", "-z", location] +notInRepo repo l = pipeNullSplit repo + ["ls-files", "--others", "--exclude-standard", "-z", l] {- Passed a location, returns a list of the files, staged for - commit, that are being added, moved, or changed (but not deleted). -} stagedFiles :: Repo -> FilePath -> IO [FilePath] -stagedFiles repo location = pipeNullSplit repo +stagedFiles repo l = pipeNullSplit repo ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", - "HEAD", location] + "HEAD", l] {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} @@ -236,7 +241,7 @@ pipeNullSplit :: Repo -> [String] -> IO [FilePath] pipeNullSplit repo params = do -- XXX handle is left open, this is ok for git-annex, but may need -- to be cleaned up for other uses. - (handle, fs0) <- hPipeRead repo params + (_, fs0) <- hPipeRead repo params return $ split0 fs0 where split0 s = filter (not . null) $ split "\0" s @@ -256,6 +261,7 @@ configRead repo = assertSsh repo $ do where sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" +hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do val <- hGetContentsStrict h let r = repo { config = configParse val } @@ -267,10 +273,10 @@ configTrue s = map toLower s == "true" {- Calculates a list of a repo's configured remotes, by parsing its config. -} configRemotes :: Repo -> [Repo] -configRemotes repo = map construct remotes +configRemotes repo = map construct remotepairs where - remotes = Map.toList $ filter $ config repo - filter = Map.filterWithKey (\k _ -> isremote k) + remotepairs = Map.toList $ filterremotes $ config repo + filterremotes = Map.filterWithKey (\k _ -> isremote k) isremote k = (startswith "remote." k) && (endswith ".url" k) remotename k = (split "." k) !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } @@ -314,14 +320,15 @@ seekUp dir want = do "" -> return Nothing d -> seekUp d want +isRepoTop :: FilePath -> IO Bool isRepoTop dir = do - r <- isRepo dir - b <- isBareRepo dir + r <- isRepo + b <- isBareRepo return (r || b) where - isRepo dir = gitSignature dir ".git" ".git/config" - isBareRepo dir = gitSignature dir "objects" "config" - gitSignature dir subdir file = do + isRepo = gitSignature ".git" ".git/config" + isBareRepo = gitSignature "objects" "config" + gitSignature subdir file = do s <- (doesDirectoryExist (dir ++ "/" ++ subdir)) f <- (doesFileExist (dir ++ "/" ++ file)) return (s && f) From b2c28c1ac0700eadc8689cdfb6f065d5147108bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 15:50:07 -0400 Subject: [PATCH 0367/8313] more Wall cleaning --- LocationLog.hs | 61 +++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 14ae88abc3..9a9dad1333 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,9 +32,7 @@ import Data.Time.Clock.POSIX import Data.Time import System.Locale import qualified Data.Map as Map -import System.IO import System.Directory -import Data.Char import System.Posix.Process import qualified GitRepo as Git @@ -63,8 +61,7 @@ instance Read LogStatus where readsPrec _ _ = [(Undefined, "")] instance Show LogLine where - show (LogLine date status uuid) = unwords - [(show date), (show status), uuid] + show (LogLine d s u) = unwords [show d, show s, u] instance Read LogLine where -- This parser is robust in that even unparsable log lines are @@ -74,26 +71,25 @@ instance Read LogLine where if (length w == 3) then case (pdate) of Just v -> good v - Nothing -> undefined - else undefined + Nothing -> bad + else bad where w = words string - date = w !! 0 - status = read $ w !! 1 - uuid = w !! 2 - pdate = (parseTime defaultTimeLocale "%s%Qs" date) :: Maybe UTCTime + s = read $ w !! 1 + u = w !! 2 + pdate = (parseTime defaultTimeLocale "%s%Qs" $ w !! 0) :: Maybe UTCTime - good v = ret $ LogLine (utcTimeToPOSIXSeconds v) status uuid - undefined = ret $ LogLine (0) Undefined "" + good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s u + bad = ret $ LogLine (0) Undefined "" ret v = [(v, "")] {- Log a change in the presence of a key's value in a repository, - and returns the filename of the logfile. -} logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO (FilePath) -logChange repo key uuid status = do - log <- logNow status uuid +logChange repo key u s = do + line <- logNow s u ls <- readLog logfile - writeLog logfile (compactLog $ log:ls) + writeLog logfile (compactLog $ line:ls) return logfile where logfile = logFile repo key @@ -114,18 +110,18 @@ readLog file = do {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () -writeLog file lines = do +writeLog file ls = do pid <- getProcessID let tmpfile = file ++ ".tmp" ++ show pid createDirectoryIfMissing True (parentDir file) - writeFile tmpfile $ unlines $ map show lines + writeFile tmpfile $ unlines $ map show ls renameFile tmpfile file {- Generates a new LogLine with the current date. -} logNow :: LogStatus -> UUID -> IO LogLine -logNow status uuid = do +logNow s u = do now <- getPOSIXTime - return $ LogLine now status uuid + return $ LogLine now s u {- Returns the filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String @@ -136,28 +132,33 @@ logFile repo key = - the value of a key. -} keyLocations :: Git.Repo -> Key -> IO [UUID] keyLocations thisrepo key = do - lines <- readLog $ logFile thisrepo key - return $ map uuid (filterPresent lines) + ls <- readLog $ logFile thisrepo key + return $ map uuid $ filterPresent ls {- Filters the list of LogLines to find ones where the value - is (or should still be) present. -} filterPresent :: [LogLine] -> [LogLine] -filterPresent lines = filter (\l -> ValuePresent == status l) $ compactLog lines +filterPresent ls = filter (\l -> ValuePresent == status l) $ compactLog ls + +type LogMap = Map.Map UUID LogLine {- Compacts a set of logs, returning a subset that contains the current - status. -} compactLog :: [LogLine] -> [LogLine] -compactLog lines = compactLog' Map.empty lines -compactLog' map [] = Map.elems map -compactLog' map (l:ls) = compactLog' (mapLog map l) ls +compactLog ls = compactLog' Map.empty ls +compactLog' :: LogMap -> [LogLine] -> [LogLine] +compactLog' m [] = Map.elems m +compactLog' m (l:ls) = compactLog' (mapLog m l) ls {- Inserts a log into a map of logs, if the log has better (ie, newer) - information about a repo than the other logs in the map -} -mapLog map log = +mapLog :: LogMap -> LogLine -> LogMap +mapLog m l = if (better) - then Map.insert (uuid log) log map - else map + then Map.insert u l m + else m where - better = case Map.lookup (uuid log) map of - Just l -> (date l <= date log) + better = case Map.lookup u m of + Just l' -> (date l' <= date l) Nothing -> True + u = uuid l From cf4c926f2e78bd315988c1aae063f6b4148a2fae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 16:00:32 -0400 Subject: [PATCH 0368/8313] more Wall cleaning --- Backend/File.hs | 15 +++++---------- Backend/SHA1.hs | 1 + Backend/URL.hs | 10 ++++------ Backend/WORM.hs | 3 +-- Core.hs | 2 +- Locations.hs | 8 ++++---- TypeInternals.hs | 4 ++-- Utility.hs | 12 +++++------- 8 files changed, 23 insertions(+), 32 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index b45354752f..5b93d8227e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -15,25 +15,18 @@ module Backend.File (backend) where import Control.Monad.State -import System.IO -import System.Cmd -import System.Cmd.Utils -import Control.Exception import System.Directory -import List -import Maybe import TypeInternals import LocationLog import Locations import qualified Remotes import qualified GitRepo as Git -import Utility import Core import qualified Annex import UUID -import qualified Backend +backend :: Backend backend = Backend { name = mustProvide, getKey = mustProvide, @@ -43,11 +36,12 @@ backend = Backend { hasKey = checkKeyFile } +mustProvide :: a mustProvide = error "must provide this field" {- Storing a key is a no-op. -} dummyStore :: FilePath -> Key -> Annex (Bool) -dummyStore file key = return True +dummyStore _ _ = return True {- Just check if the .git/annex/ file for the key exists. -} checkKeyFile :: Key -> Annex Bool @@ -146,7 +140,8 @@ showLocations key = do if (null uuidsf) then showLongNote $ "No other repository is known to contain the file." else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids - + +showTriedRemotes :: [Git.Repo] -> Annex () showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index f6daeffec4..76c368f84e 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -15,6 +15,7 @@ import System.IO import qualified Backend.File import TypeInternals +backend :: Backend backend = Backend.File.backend { name = "SHA1", getKey = keyValue diff --git a/Backend/URL.hs b/Backend/URL.hs index 384f933ebf..e6d3eb1ae5 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -9,14 +9,12 @@ module Backend.URL (backend) where import Control.Monad.State (liftIO) import Data.String.Utils -import System.Cmd -import System.Cmd.Utils -import System.Exit import TypeInternals import Core import Utility +backend :: Backend backend = Backend { name = "URL", getKey = keyValue, @@ -28,15 +26,15 @@ backend = Backend { -- cannot generate url from filename keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = return Nothing +keyValue _ = return Nothing -- cannot change url contents dummyStore :: FilePath -> Key -> Annex Bool -dummyStore file url = return False +dummyStore _ _ = return False -- allow keys to be removed; presumably they can always be downloaded again dummyOk :: Key -> Annex Bool -dummyOk url = return True +dummyOk _ = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do diff --git a/Backend/WORM.hs b/Backend/WORM.hs index b5ae11807e..848386ecd1 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,12 +10,11 @@ module Backend.WORM (backend) where import Control.Monad.State import System.FilePath import System.Posix.Files -import qualified Data.ByteString.Lazy.Char8 as B import qualified Backend.File import TypeInternals -import Utility +backend :: Backend backend = Backend.File.backend { name = "WORM", getKey = keyValue diff --git a/Core.hs b/Core.hs index c5026c6a5f..ebe5d2966c 100644 --- a/Core.hs +++ b/Core.hs @@ -109,7 +109,7 @@ calcGitLink file key = do Just f -> f Nothing -> error $ "unable to normalize " ++ file return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ - annexLocationRelative g key + annexLocationRelative key {- Updates the LocationLog when a key's presence changes. -} logStatus :: Key -> LogStatus -> Annex () diff --git a/Locations.hs b/Locations.hs index 49ee878c85..a296f8d876 100644 --- a/Locations.hs +++ b/Locations.hs @@ -18,11 +18,11 @@ module Locations ( import Data.String.Utils import Types -import qualified TypeInternals as Internals import qualified GitRepo as Git {- Long-term, cross-repo state is stored in files inside the .git-annex - directory, in the git repository's working tree. -} +stateLoc :: String stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc @@ -35,13 +35,13 @@ gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = - (Git.workTree r) ++ "/" ++ (annexLocationRelative r key) + (Git.workTree r) ++ "/" ++ (annexLocationRelative key) {- Annexed file's location relative to git's working tree. - - Note: Assumes repo is NOT bare.-} -annexLocationRelative :: Git.Repo -> Key -> FilePath -annexLocationRelative r key = ".git/annex/" ++ (keyFile key) +annexLocationRelative :: Key -> FilePath +annexLocationRelative key = ".git/annex/" ++ (keyFile key) {- .git-annex/tmp is used for temp files - diff --git a/TypeInternals.hs b/TypeInternals.hs index f45be4760c..46c92cb59b 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -58,9 +58,9 @@ instance Read Key where k = join ":" $ drop 1 l backendName :: Key -> BackendName -backendName (Key (b,k)) = b +backendName (Key (b,_)) = b keyName :: Key -> KeyName -keyName (Key (b,k)) = k +keyName (Key (_,k)) = k -- this structure represents a key-value backend data Backend = Backend { diff --git a/Utility.hs b/Utility.hs index e7b4b510b8..6867f473a7 100644 --- a/Utility.hs +++ b/Utility.hs @@ -15,20 +15,17 @@ module Utility ( ) where import System.IO -import System.Cmd.Utils import System.Exit import System.Posix.Process -import System.Posix.Process.Internals import System.Posix.Signals -import System.Posix.IO import Data.String.Utils import System.Path -import System.IO.HVFS import System.FilePath import System.Directory {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} +hGetContentsStrict :: Handle -> IO String hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} @@ -53,11 +50,11 @@ parentDir dir = relPathCwdToDir :: FilePath -> IO FilePath relPathCwdToDir dir = do cwd <- getCurrentDirectory - let absdir = abs cwd dir + let absdir = absnorm cwd return $ relPathDirToDir cwd absdir where -- absolute, normalized form of the directory - abs cwd dir = + absnorm cwd = case (absNormPath cwd dir) of Just d -> d Nothing -> error $ "unable to normalize " ++ dir @@ -106,13 +103,14 @@ boolSystem command params = do _ -> return False where restoresignals oldint oldset = do - installHandler sigINT oldint Nothing + _ <- installHandler sigINT oldint Nothing setSignalMask oldset childaction oldint oldset = do restoresignals oldint oldset executeFile command True params Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} +shellEscape :: FilePath -> FilePath shellEscape f = "'" ++ quote ++ "'" where -- replace ' with '"'"' From dda0679290aabdb7fe61e52e7a0c84f8baf3d547 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 16:04:19 -0400 Subject: [PATCH 0369/8313] all Walls are clean! --- UUID.hs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/UUID.hs b/UUID.hs index 20f85bae12..d683e0b98e 100644 --- a/UUID.hs +++ b/UUID.hs @@ -36,6 +36,7 @@ import Utility type UUID = String +configkey :: String configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, @@ -53,19 +54,19 @@ getUUID :: Git.Repo -> Annex UUID getUUID r = do g <- Annex.gitRepo - let c = cached r g - let u = uncached r + let c = cached g + let u = uncached if (c /= u && u /= "") then do - updatecache g r u + updatecache g u return u else return c where - uncached r = Git.configGet r "annex.uuid" "" - cached r g = Git.configGet g (cachekey r) "" - updatecache g r u = when (g /= r) $ setConfig (cachekey r) u - cachekey r = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" + uncached = Git.configGet r "annex.uuid" "" + cached g = Git.configGet g cachekey "" + updatecache g u = when (g /= r) $ setConfig cachekey u + cachekey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -111,26 +112,27 @@ describeUUID :: UUID -> String -> Annex () describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m - log <- uuidLog + logfile <- uuidLog pid <- liftIO $ getProcessID - let tmplog = log ++ ".tmp" ++ show pid - liftIO $ createDirectoryIfMissing True (parentDir log) - liftIO $ writeFile tmplog $ serialize m' - liftIO $ renameFile tmplog log + let tmplogfile = logfile ++ ".tmp" ++ show pid + liftIO $ createDirectoryIfMissing True (parentDir logfile) + liftIO $ writeFile tmplogfile $ serialize m' + liftIO $ renameFile tmplogfile logfile where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m {- Read and parse the uuidLog into a Map -} uuidMap :: Annex (M.Map UUID String) uuidMap = do - log <- uuidLog - s <- liftIO $ catch (readFile log) (\error -> return "") + logfile <- uuidLog + s <- liftIO $ catch (readFile logfile) ignoreerror return $ M.fromList $ map (\l -> pair l) $ lines s where pair l = if (1 < (length $ words l)) then ((words l) !! 0, unwords $ drop 1 $ words l) else ("", "") + ignoreerror _ = return "" {- Filename of uuid.log. -} uuidLog :: Annex String From 9429cdb9dab1321cb69d40996c31572f4657538c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 17:02:49 -0400 Subject: [PATCH 0370/8313] idea --- doc/todo/immutable_annexed_files.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/todo/immutable_annexed_files.mdwn diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn new file mode 100644 index 0000000000..8f19f4f744 --- /dev/null +++ b/doc/todo/immutable_annexed_files.mdwn @@ -0,0 +1,3 @@ +> josh: Do you do anything in git-annex to try to make the files immutable? +> For instance, removing write permission, or even chattr? +> joey: I don't, but that's a very good idea From 763882925daee55584e68f9b0915ecd72877b14f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 17:04:28 -0400 Subject: [PATCH 0371/8313] idea2 --- doc/todo/immutable_annexed_files.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn index 8f19f4f744..e5207dc163 100644 --- a/doc/todo/immutable_annexed_files.mdwn +++ b/doc/todo/immutable_annexed_files.mdwn @@ -1,3 +1,6 @@ > josh: Do you do anything in git-annex to try to make the files immutable? > For instance, removing write permission, or even chattr? > joey: I don't, but that's a very good idea +> josh: Oh, I just thought of another slightly crazy but handy idea. +> josh: I'd hate to run into a program which somehow followed the symlink and then did an unlink to replace the file. +> josh: To break that, you could create a new directory under annex's internal directory for each file, and make the directory have no write permission. From b220e117f2e06133919ad31e9cc225470f501566 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 17:20:05 -0400 Subject: [PATCH 0372/8313] idea --- doc/todo/checkout.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/todo/checkout.mdwn diff --git a/doc/todo/checkout.mdwn b/doc/todo/checkout.mdwn new file mode 100644 index 0000000000..70d31a5ff5 --- /dev/null +++ b/doc/todo/checkout.mdwn @@ -0,0 +1,5 @@ +The checkout subcommand replaces the symlink that normally points at a +file's content, with a copy of the file. Once you've checked a file out, +you can edit it, and `git commit` it. On commit, git-annex will detect +if the file has been changed, and if it has, `add` its content to the +annex. From fd6611f9554e43f2bc365f7ef70f90877c9403d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 18:04:34 -0400 Subject: [PATCH 0373/8313] Fix crash on unknown symlinks. --- Backend.hs | 41 ++++++++++++++++++++++++++++------------- Core.hs | 7 +++++++ debian/changelog | 1 + git-annex.hs | 7 +------ 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/Backend.hs b/Backend.hs index d75c2a761e..f1b4c28979 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,9 +27,10 @@ module Backend ( ) where import Control.Monad.State -import Control.Exception.Extensible +import IO (try) import System.FilePath import System.Posix.Files +import Core import Locations import qualified GitRepo as Git @@ -59,12 +60,17 @@ list = do then bs else map (lookupBackendName bs) $ words s -{- Looks up a backend in a list -} +{- Looks up a backend in a list. May fail if unknown. -} lookupBackendName :: [Backend] -> String -> Backend lookupBackendName bs s = + case maybeLookupBackendName bs s of + Just b -> b + Nothing -> error $ "unknown backend " ++ s +maybeLookupBackendName :: [Backend] -> String -> Maybe Backend +maybeLookupBackendName bs s = if ((length matches) /= 1) - then error $ "unknown backend " ++ s - else matches !! 0 + then Nothing + else Just $ matches !! 0 where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} @@ -109,15 +115,24 @@ hasKey key = do lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) lookupFile file = do bs <- Annex.supportedBackends - result <- liftIO $ (try (find bs)::IO (Either SomeException (Maybe (Key, Backend)))) - case (result) of + tl <- liftIO $ try getsymlink + case tl of Left _ -> return Nothing - Right val -> return val - where - find bs = do + Right l -> makekey bs l + where + getsymlink = do l <- readSymbolicLink file - return $ Just $ pair bs $ takeFileName l - pair bs f = (k, b) + return $ takeFileName l + makekey bs l = do + case maybeLookupBackendName bs $ bname of + Nothing -> do + unless (null kname || null bname) $ + warning skip + return Nothing + Just backend -> return $ Just (k, backend) where - k = fileKey f - b = lookupBackendName bs $ backendName k + k = fileKey l + bname = backendName k + kname = keyName k + skip = "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" diff --git a/Core.hs b/Core.hs index ebe5d2966c..e2e6eaa0cf 100644 --- a/Core.hs +++ b/Core.hs @@ -165,3 +165,10 @@ showEndOk = verbose $ do showEndFail :: Annex () showEndFail = verbose $ do liftIO $ putStrLn "\nfailed" + +{- Exception pretty-printing. -} +showErr :: (Show a) => a -> Annex () +showErr e = warning $ show e + +warning :: String -> Annex () +warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s diff --git a/debian/changelog b/debian/changelog index 70e9052231..83e76b6ec7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Support building with Debian stable's ghc. * Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. + * Fix crash on unknown symlinks. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/git-annex.hs b/git-annex.hs index e9e0d6f027..d798d417b5 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,7 +6,6 @@ -} import IO (try) -import System.IO import System.Environment import Monad @@ -41,13 +40,9 @@ tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case (result) of Left err -> do - showErr err + _ <- Annex.run state $ showErr err tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as tryRun' _ errnum [] = when (errnum > 0) $ error $ (show errnum) ++ " failed" - -{- Exception pretty-printing. -} -showErr :: (Show a) => a -> IO () -showErr e = hPutStrLn stderr $ "git-annex: " ++ (show e) From f4383532e69630266c1188527714138dfff8bea0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:52:11 -0400 Subject: [PATCH 0374/8313] add toggles --- doc/use_case/Alice.mdwn | 8 ++++++-- doc/use_case/Bob.mdwn | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index c42eb3a74c..5d0360a2b8 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -3,9 +3,12 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. All these things can have different files on them, -but Alice no longer has to deal with the tedious process of keeping them -manually in sync. +but with git-annex Alice no longer has to deal with the tedious process +of keeping them manually in sync. +[[!toggle id=alice text="more..."]] + +[[!toggleable id=alice text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on @@ -16,3 +19,4 @@ When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knowns that next time she syncs up to the net, her changes will be synced back to her server. +"""]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index a5dc01b373..3aad78eef9 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -4,10 +4,13 @@ Bob has many drives to archive his data, most of them kept offline, in a safe place. With git-annex, Bob has a single directory tree that includes all -his files, even if their content is being stored offline. He can +his files, even those whose content is stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. +[[!toggle id=bob text="more..."]] + +[[!toggleable id=bob text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. @@ -16,3 +19,4 @@ Run in a cron job, git-annex adds new files to achival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. +"""]] From 371906efb010c3b441d958adecb2a6fddd498237 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:53:17 -0400 Subject: [PATCH 0375/8313] foo --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 5d0360a2b8..590f7b6527 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -6,9 +6,9 @@ out there on the net. All these things can have different files on them, but with git-annex Alice no longer has to deal with the tedious process of keeping them manually in sync. -[[!toggle id=alice text="more..."]] +[[!toggle id=use text="more..."]] -[[!toggleable id=alice text=""" +[[!toggleable id=use text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 3aad78eef9..0f1247ab22 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -8,9 +8,9 @@ his files, even those whose content is stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. -[[!toggle id=bob text="more..."]] +[[!toggle id=use text="more..."]] -[[!toggleable id=bob text=""" +[[!toggleable id=use text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. From b5411926abc5724f7f156ceaea8fa7e37a745cd1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:54:12 -0400 Subject: [PATCH 0376/8313] Revert "foo" This reverts commit 371906efb010c3b441d958adecb2a6fddd498237. --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 590f7b6527..5d0360a2b8 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -6,9 +6,9 @@ out there on the net. All these things can have different files on them, but with git-annex Alice no longer has to deal with the tedious process of keeping them manually in sync. -[[!toggle id=use text="more..."]] +[[!toggle id=alice text="more..."]] -[[!toggleable id=use text=""" +[[!toggleable id=alice text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 0f1247ab22..3aad78eef9 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -8,9 +8,9 @@ his files, even those whose content is stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. -[[!toggle id=use text="more..."]] +[[!toggle id=bob text="more..."]] -[[!toggleable id=use text=""" +[[!toggleable id=bob text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. From 4e742d62f45fad486d8772a7b9649b42315e1be7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 19:57:05 -0400 Subject: [PATCH 0377/8313] Revert "add toggles" This reverts commit f4383532e69630266c1188527714138dfff8bea0. --- doc/use_case/Alice.mdwn | 8 ++------ doc/use_case/Bob.mdwn | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 5d0360a2b8..c42eb3a74c 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -3,12 +3,9 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. All these things can have different files on them, -but with git-annex Alice no longer has to deal with the tedious process -of keeping them manually in sync. +but Alice no longer has to deal with the tedious process of keeping them +manually in sync. -[[!toggle id=alice text="more..."]] - -[[!toggleable id=alice text=""" When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on @@ -19,4 +16,3 @@ When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knowns that next time she syncs up to the net, her changes will be synced back to her server. -"""]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 3aad78eef9..a5dc01b373 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -4,13 +4,10 @@ Bob has many drives to archive his data, most of them kept offline, in a safe place. With git-annex, Bob has a single directory tree that includes all -his files, even those whose content is stored offline. He can +his files, even if their content is being stored offline. He can reorganize his files using that tree, committing new versions to git, without worry about accidentially deleting anything. -[[!toggle id=bob text="more..."]] - -[[!toggleable id=bob text=""" When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. @@ -19,4 +16,3 @@ Run in a cron job, git-annex adds new files to achival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -"""]] From 228b32d473abb72bb26ff8eca008387190428c44 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:13:43 -0400 Subject: [PATCH 0378/8313] bugfix: shell escape for scp --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index b9c1b48f3d..b36856b543 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -217,7 +217,7 @@ copyToRemote r key file = do liftIO $ boolSystem "scp" [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath -sshLocation r file = (Git.urlHost r) ++ ":" ++ file +sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file {- Runs a command in a remote. -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool From 11efa7ef609a99de2841772e415cd2808ee438b8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:19:25 -0400 Subject: [PATCH 0379/8313] more escaping for ssh --- Remotes.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index b36856b543..135a701ffb 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -232,6 +232,6 @@ runCmd r command params = do then do liftIO $ boolSystem "ssh" [Git.urlHost r, "cd " ++ (shellEscape $ Git.workTree r) ++ - " && " ++ command ++ " " ++ - unwords params] + " && " ++ (shellEscape command) ++ " " ++ + (unwords $ map shellEscape params)] else error "running command in non-ssh repo not supported" From d04bfceaad3e96c91c7effa9cebdc0811d336278 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:20:07 -0400 Subject: [PATCH 0380/8313] more accurate type --- Utility.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index 6867f473a7..47c059ded7 100644 --- a/Utility.hs +++ b/Utility.hs @@ -110,7 +110,7 @@ boolSystem command params = do executeFile command True params Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} -shellEscape :: FilePath -> FilePath +shellEscape :: FilePath -> String shellEscape f = "'" ++ quote ++ "'" where -- replace ' with '"'"' From c6206c4560adf35427569a9d9a637f9ee23cb2b8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:22:01 -0400 Subject: [PATCH 0381/8313] tweak --- Utility.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utility.hs b/Utility.hs index 47c059ded7..4e56289e22 100644 --- a/Utility.hs +++ b/Utility.hs @@ -111,7 +111,7 @@ boolSystem command params = do {- Escapes a filename to be safely able to be exposed to the shell. -} shellEscape :: FilePath -> String -shellEscape f = "'" ++ quote ++ "'" +shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' - quote = join "'\"'\"'" $ split "'" f + escaped = join "'\"'\"'" $ split "'" f From e70812eca9748ee53468e68563366ebd856b7e82 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:29:11 -0400 Subject: [PATCH 0382/8313] use ssh -p to preserve perms and refactor --- Remotes.hs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 135a701ffb..a5c4597eba 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -196,8 +196,7 @@ copyFromRemote r key file = do where getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] getssh = do - Core.showProgress -- make way for scp progress bar - liftIO $ boolSystem "scp" [sshLocation r keyloc, file] + scp [sshLocation r keyloc, file] keyloc = annexLocation r key {- Tries to copy a key's content to a file on a remote. -} @@ -213,12 +212,16 @@ copyToRemote r key file = do where putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] putssh src = do - Core.showProgress -- make way for scp progress bar - liftIO $ boolSystem "scp" [src, sshLocation r file] + scp [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file +scp :: [String] -> Annex Bool +scp params = do + Core.showProgress -- make way for scp progress bar + liftIO $ boolSystem "scp" ("-p":params) + {- Runs a command in a remote. -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do From 0194394be61dba0324a5a99f462aa2206066771c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 22:56:56 -0400 Subject: [PATCH 0383/8313] Added remote.annex-scp-options and remote.annex-ssh-options. --- Remotes.hs | 49 ++++++++++++++++++++++++++-------------------- debian/changelog | 1 + doc/git-annex.mdwn | 5 +++++ 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index a5c4597eba..778ad89e25 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -115,33 +115,28 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - g <- Annex.gitRepo - if (not $ null $ config g) - then return $ read $ config g + cost <- repoConfig r "annex-cost" "" + if (not $ null cost) + then return $ read cost else if (Git.repoIsUrl r) then return 200 else return 100 - where - config g = Git.configGet g configkey "" - configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-cost" {- Checks if a repo should be ignored, based either on annex-ignore - setting, or on command-line options. Allows command-line to override - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do - g <- Annex.gitRepo + ignored <- repoConfig r "annex-ignore" "false" fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if (not $ null fromName) then fromName else toName if (not $ null name) then return $ match name - else return $ not $ ignored g + else return $ not $ isIgnored ignored where match name = name == Git.repoRemoteName r - ignored g = Git.configTrue $ config g - config g = Git.configGet g configkey "" - configkey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-ignore" + isIgnored ignored = Git.configTrue ignored {- Returns the remote specified by --from or --to, may fail with error. -} commandLineRemote :: Annex Git.Repo @@ -195,8 +190,7 @@ copyFromRemote r key file = do else error "copying from non-ssh repo not supported" where getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] - getssh = do - scp [sshLocation r keyloc, file] + getssh = scp r [sshLocation r keyloc, file] keyloc = annexLocation r key {- Tries to copy a key's content to a file on a remote. -} @@ -211,20 +205,23 @@ copyToRemote r key file = do else error "copying to non-ssh repo not supported" where putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] - putssh src = do - scp [src, sshLocation r file] + putssh src = scp r [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file -scp :: [String] -> Annex Bool -scp params = do +{- Runs scp against a specified remote. (Honors annex-scp-options.) -} +scp :: Git.Repo -> [String] -> Annex Bool +scp r params = do + scpoptions <- repoConfig r "annex-scp-options" "" Core.showProgress -- make way for scp progress bar - liftIO $ boolSystem "scp" ("-p":params) + liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params -{- Runs a command in a remote. -} +{- Runs a command in a remote, using ssh if necessary. + - (Honors annex-ssh-options.) -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do + sshoptions <- repoConfig r "annex-ssh-options" "" if (not $ Git.repoIsUrl r) then do cwd <- liftIO $ getCurrentDirectory @@ -233,8 +230,18 @@ runCmd r command params = do boolSystem command params else if (Git.repoIsSsh r) then do - liftIO $ boolSystem "ssh" [Git.urlHost r, - "cd " ++ (shellEscape $ Git.workTree r) ++ + liftIO $ boolSystem "ssh" $ + (words sshoptions) ++ + [Git.urlHost r, "cd " ++ + (shellEscape $ Git.workTree r) ++ " && " ++ (shellEscape command) ++ " " ++ (unwords $ map shellEscape params)] else error "running command in non-ssh repo not supported" + +{- Looks up a per-remote config option in git config. -} +repoConfig :: Git.Repo -> String -> String -> Annex String +repoConfig r key def = do + g <- Annex.gitRepo + return $ Git.configGet g fullkey def + where + fullkey = "remote." ++ (Git.repoRemoteName r) ++ "." ++ key diff --git a/debian/changelog b/debian/changelog index 83e76b6ec7..cb6a0ef868 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. * Fix crash on unknown symlinks. + * Added remote.annex-scp-options and remote.annex-ssh-options. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 47fbb37601..d176a04c04 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -196,6 +196,11 @@ Like other git commands, git-annex is configured via `.git/config`. from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. +* `remote..annex-scp-options` -- Options to use when using scp + to or from this repository. For example, to force ipv6, and limit + the bandwidth to 100Kbit/s, set it to "-6 -l 100" +* `remote..annex-ssh-options` -- Options to use when using ssh + to talk to this repository. # FILES From f3e4633e359d0a4afc7a0f90a89e4d276b597d84 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:21:16 -0400 Subject: [PATCH 0384/8313] refactor inAnnex remote checking to Remotes --- Core.hs | 13 +++---------- Remotes.hs | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Core.hs b/Core.hs index e2e6eaa0cf..ab7291affa 100644 --- a/Core.hs +++ b/Core.hs @@ -86,19 +86,12 @@ gitPreCommitHook repo = do p <- getPermissions hook setPermissions hook $ p {executable = True} -{- Checks if a given key is currently present in the annexLocation. - - - - This can be run against a remote repository to check the key there. -} +{- Checks if a given key is currently present in the annexLocation. -} inAnnex :: Key -> Annex Bool inAnnex key = do g <- Annex.gitRepo - if (not $ Git.repoIsUrl g) - then liftIO $ doesFileExist $ annexLocation g key - else do - showNote ("checking " ++ Git.repoDescribe g ++ "...") - liftIO $ boolSystem "ssh" [Git.urlHost g, - "test -e " ++ - (shellEscape $ annexLocation g key)] + when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo" + liftIO $ doesFileExist $ annexLocation g key {- Calculates the relative path to use to link a file to a key. -} calcGitLink :: FilePath -> Key -> Annex FilePath diff --git a/Remotes.hs b/Remotes.hs index 778ad89e25..a01da7d488 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -81,14 +81,24 @@ keyPossibilities key = do - If the remote cannot be accessed, returns a Left error. -} inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) -inAnnex remote key = do - -- the check needs to run in an Annex monad using the remote - liftIO $ ((try $ check)::IO (Either IOException Bool)) +inAnnex r key = do + if (not $ Git.repoIsUrl r) + then check local + else do + Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") + check remote where - check = do - a <- Annex.new remote [] + check a = liftIO $ ((try a)::IO (Either IOException Bool)) + local = do + -- run a local check by making an Annex monad + -- using the remote + a <- Annex.new r [] (result, _) <- Annex.run a (Core.inAnnex key) return result + remote = do + -- remote check via ssh in and test + boolSystem "ssh" [Git.urlHost r, "test -e " ++ + (shellEscape $ annexLocation r key)] {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] From 00d4c7cd01b6f6e863a22483b9ea20ca5260da43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:24:16 -0400 Subject: [PATCH 0385/8313] simplify evals --- Annex.hs | 3 +++ Remotes.hs | 3 +-- git-annex.hs | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Annex.hs b/Annex.hs index 63eef51589..8f60a0bf49 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,6 +8,7 @@ module Annex ( new, run, + eval, gitRepo, gitRepoChange, backends, @@ -50,6 +51,8 @@ new gitrepo allbackends = do {- performs an action in the Annex monad -} run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) run state action = runStateT (action) state +eval :: AnnexState -> StateT AnnexState IO a -> IO a +eval state action = evalStateT (action) state {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo diff --git a/Remotes.hs b/Remotes.hs index a01da7d488..2879516fe1 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -93,8 +93,7 @@ inAnnex r key = do -- run a local check by making an Annex monad -- using the remote a <- Annex.new r [] - (result, _) <- Annex.run a (Core.inAnnex key) - return result + Annex.eval a (Core.inAnnex key) remote = do -- remote check via ssh in and test boolSystem "ssh" [Git.urlHost r, "test -e " ++ diff --git a/git-annex.hs b/git-annex.hs index d798d417b5..e958ac2f96 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -40,7 +40,7 @@ tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case (result) of Left err -> do - _ <- Annex.run state $ showErr err + Annex.eval state $ showErr err tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as From cec25153ecd4f824cf150afbd294ad0c5ed1413e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:38:07 -0400 Subject: [PATCH 0386/8313] bugfix: git annex move --from The data structure comparison didn't work because for a file remote, the config gets read for one structure but not the other. --- Commands.hs | 2 +- Remotes.hs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Commands.hs b/Commands.hs index fb76e5502f..507c82cccc 100644 --- a/Commands.hs +++ b/Commands.hs @@ -479,7 +479,7 @@ moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key - if (elem remote l) + if (not $ null $ filter (\r -> Remotes.same r remote) l) then return $ Just $ moveFromPerform key else return Nothing moveFromPerform :: Key -> Annex (Maybe SubCmdCleanup) diff --git a/Remotes.hs b/Remotes.hs index 2879516fe1..dffbe8f502 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -10,6 +10,7 @@ module Remotes ( keyPossibilities, tryGitConfigRead, inAnnex, + same, commandLineRemote, copyFromRemote, copyToRemote, @@ -47,7 +48,7 @@ keyPossibilities key = do uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost -- To determine if a remote has a key, its UUID needs to be known. - -- The locally cached UIIDs of remotes can fall out of date if + -- The locally cached UUIDs of remotes can fall out of date if -- eg, a different drive is mounted at the same location. -- But, reading the config of remotes can be expensive, so make -- sure we only do it once per git-annex run. @@ -147,6 +148,10 @@ repoNotIgnored r = do match name = name == Git.repoRemoteName r isIgnored ignored = Git.configTrue ignored +{- Checks if two repos are the same, by comparing their remote names. -} +same :: Git.Repo -> Git.Repo -> Bool +same a b = Git.repoRemoteName a == Git.repoRemoteName b + {- Returns the remote specified by --from or --to, may fail with error. -} commandLineRemote :: Annex Git.Repo commandLineRemote = do From 0bd7ebbf356c2753fbd6e29b925efef0b4eb3f74 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 31 Oct 2010 23:50:58 -0400 Subject: [PATCH 0387/8313] make a ssh call honor annex-ssh-options --- Remotes.hs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index dffbe8f502..c28ba5afed 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -84,21 +84,21 @@ keyPossibilities key = do inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) inAnnex r key = do if (not $ Git.repoIsUrl r) - then check local - else do - Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") - check remote + then liftIO $ ((try checklocal)::IO (Either IOException Bool)) + else checkremote where - check a = liftIO $ ((try a)::IO (Either IOException Bool)) - local = do + checklocal = do -- run a local check by making an Annex monad -- using the remote a <- Annex.new r [] Annex.eval a (Core.inAnnex key) - remote = do - -- remote check via ssh in and test - boolSystem "ssh" [Git.urlHost r, "test -e " ++ - (shellEscape $ annexLocation r key)] + checkremote = do + Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") + inannex <- runCmd r ("test -e " ++ + (shellEscape $ annexLocation r key)) [] + -- XXX Note that ssh failing and the file not existing + -- are not currently differentiated. + return $ Right inannex {- Cost Ordered list of remotes. -} remotesByCost :: Annex [Git.Repo] From 99c522edeff6add3ece41922f862c59b5afdddbe Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:04:53 -0400 Subject: [PATCH 0388/8313] finished adding support for annex.ssh-options --- Annex.hs | 2 +- GitRepo.hs | 17 +++++++++++------ Remotes.hs | 7 ++++--- UUID.hs | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Annex.hs b/Annex.hs index 8f60a0bf49..303881fa09 100644 --- a/Annex.hs +++ b/Annex.hs @@ -45,7 +45,7 @@ new gitrepo allbackends = do where prep = do -- read git config and update state - gitrepo' <- liftIO $ Git.configRead gitrepo + gitrepo' <- liftIO $ Git.configRead gitrepo Nothing Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} diff --git a/GitRepo.hs b/GitRepo.hs index 874b5c3c9b..505dd06eb2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -246,9 +246,11 @@ pipeNullSplit repo params = do where split0 s = filter (not . null) $ split "\0" s -{- Runs git config and populates a repo with its config. -} -configRead :: Repo -> IO Repo -configRead repo@(Repo { location = Dir d }) = do +{- Runs git config and populates a repo with its config. + - + - For a ssh repository, a list of ssh options may optionally be specified. -} +configRead :: Repo -> Maybe [String] -> IO Repo +configRead repo@(Repo { location = Dir d }) _ = do {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory @@ -256,10 +258,13 @@ configRead repo@(Repo { location = Dir d }) = do (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo -configRead repo = assertSsh repo $ do - pOpen ReadFromPipe "ssh" [urlHost repo, sshcommand] $ hConfigRead repo +configRead repo sshopts = assertSsh repo $ do + pOpen ReadFromPipe "ssh" params $ hConfigRead repo where - sshcommand = "cd " ++ (shellEscape $ urlPath repo) ++ + params = case sshopts of + Nothing -> [urlHost repo, command] + Just l -> l ++ [urlHost repo, command] + command = "cd " ++ (shellEscape $ urlPath repo) ++ " && git config --list" hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do diff --git a/Remotes.hs b/Remotes.hs index c28ba5afed..bb77b00b31 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -94,8 +94,8 @@ inAnnex r key = do Annex.eval a (Core.inAnnex key) checkremote = do Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- runCmd r ("test -e " ++ - (shellEscape $ annexLocation r key)) [] + inannex <- runCmd r "test" + [ "-e", (shellEscape $ annexLocation r key)] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. return $ Right inannex @@ -172,11 +172,12 @@ commandLineRemote = do - returns the updated git repo. -} tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do + sshoptions <- repoConfig r "annex-ssh-options" "" if (Map.null $ Git.configMap r) then do -- configRead can fail due to IO error or -- for other reasons; catch all possible exceptions - result <- liftIO $ (try (Git.configRead r)::IO (Either SomeException (Git.Repo))) + result <- liftIO $ (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException (Git.Repo))) case (result) of Left _ -> return $ Left r Right r' -> do diff --git a/UUID.hs b/UUID.hs index d683e0b98e..ffd2cd46dc 100644 --- a/UUID.hs +++ b/UUID.hs @@ -83,7 +83,7 @@ setConfig key value = do g <- Annex.gitRepo liftIO $ Git.run g ["config", key, value] -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g + g' <- liftIO $ Git.configRead g Nothing Annex.gitRepoChange g' {- Filters a list of repos to ones that have listed UUIDs. -} From e638f9647d10ca0cc63799698456a387ca9b8382 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:17:26 -0400 Subject: [PATCH 0389/8313] add global fallback for per-repo options --- Remotes.hs | 19 +++++++++++-------- doc/git-annex.mdwn | 5 ++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index bb77b00b31..7bb1bcd222 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -125,7 +125,7 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - cost <- repoConfig r "annex-cost" "" + cost <- repoConfig r "cost" "" if (not $ null cost) then return $ read cost else if (Git.repoIsUrl r) @@ -137,7 +137,7 @@ repoCost r = do - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do - ignored <- repoConfig r "annex-ignore" "false" + ignored <- repoConfig r "ignore" "false" fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if (not $ null fromName) then fromName else toName @@ -172,7 +172,7 @@ commandLineRemote = do - returns the updated git repo. -} tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do - sshoptions <- repoConfig r "annex-ssh-options" "" + sshoptions <- repoConfig r "ssh-options" "" if (Map.null $ Git.configMap r) then do -- configRead can fail due to IO error or @@ -228,7 +228,7 @@ sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file {- Runs scp against a specified remote. (Honors annex-scp-options.) -} scp :: Git.Repo -> [String] -> Annex Bool scp r params = do - scpoptions <- repoConfig r "annex-scp-options" "" + scpoptions <- repoConfig r "scp-options" "" Core.showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params @@ -236,7 +236,7 @@ scp r params = do - (Honors annex-ssh-options.) -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do - sshoptions <- repoConfig r "annex-ssh-options" "" + sshoptions <- repoConfig r "ssh-options" "" if (not $ Git.repoIsUrl r) then do cwd <- liftIO $ getCurrentDirectory @@ -253,10 +253,13 @@ runCmd r command params = do (unwords $ map shellEscape params)] else error "running command in non-ssh repo not supported" -{- Looks up a per-remote config option in git config. -} +{- Looks up a per-remote config option in git config. + - Failing that, tries looking for a global config option. -} repoConfig :: Git.Repo -> String -> String -> Annex String repoConfig r key def = do g <- Annex.gitRepo - return $ Git.configGet g fullkey def + let def' = Git.configGet g global def + return $ Git.configGet g local def' where - fullkey = "remote." ++ (Git.repoRemoteName r) ++ "." ++ key + local = "remote." ++ (Git.repoRemoteName r) ++ ".annex-" ++ key + global = "annex." ++ key diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d176a04c04..fa584d360c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -178,7 +178,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you. # CONFIGURATION -Like other git commands, git-annex is configured via `.git/config`. +Like other git commands, git-annex is configured via `git-config`. +Here are all the supported configuration settings. * `annex.uuid` -- a unique UUID for this repository (automatically set) * `annex.numcopies` -- number of copies of files to keep across all @@ -201,6 +202,8 @@ Like other git commands, git-annex is configured via `.git/config`. the bandwidth to 100Kbit/s, set it to "-6 -l 100" * `remote..annex-ssh-options` -- Options to use when using ssh to talk to this repository. +* `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh + options to use if a remote does not have specific options. # FILES From 11215b5b118d296dc0d0ac3328802b87964915be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:26:47 -0400 Subject: [PATCH 0390/8313] cleanup --- doc/git-annex.mdwn | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index fa584d360c..c60810424b 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -190,9 +190,7 @@ Here are all the supported configuration settings. * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote - repositories. Note that other factors may be configured when pushing - files to repositories, in particular, whether the repository is on - a filesystem with sufficient free space. + repositories. * `remote..annex-ignore` -- If set to `true`, prevents git-annex from ever using this remote. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories From 4b9990194c239a42fe2916ddf5db83a4be06bf80 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:27:27 -0400 Subject: [PATCH 0391/8313] expand --- doc/git-annex.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index c60810424b..1f2f57fba7 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -192,7 +192,8 @@ Here are all the supported configuration settings. The default cost is 100 for local repositories, and 200 for remote repositories. * `remote..annex-ignore` -- If set to `true`, prevents git-annex - from ever using this remote. + from ever using this remote. This is, for example, useful if the + remote is a bare repository, which git-annex does not currently support. * `remote..annex-uuid` -- git-annex caches UUIDs of repositories here. * `remote..annex-scp-options` -- Options to use when using scp From 524125e52e8d2a392a2662ac09f8658307513a25 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 00:28:52 -0400 Subject: [PATCH 0392/8313] fix --- doc/git-annex.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 1f2f57fba7..52a8c712f6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -230,7 +230,7 @@ should be included, in, for example, `/usr/share/doc/git-annex/` # AUTHOR -Joey Hess +Joey Hess From 0655ae4b8a4c3b55351a37ddad03bb33a7cc9353 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 03:01:58 -0400 Subject: [PATCH 0393/8313] move --- Core.hs | 24 ++++++++++++++++++++++++ git-annex.hs | 26 -------------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/Core.hs b/Core.hs index ab7291affa..d3d0d7899c 100644 --- a/Core.hs +++ b/Core.hs @@ -7,6 +7,7 @@ module Core where +import IO (try) import System.IO import System.Directory import Control.Monad.State (liftIO) @@ -22,6 +23,29 @@ import qualified GitRepo as Git import qualified GitQueue import qualified Annex import Utility + +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + - Propigates an overall error status at the end. + - + - This runs in the IO monad, not in the Annex monad. It seems that + - exceptions can only be caught in the IO monad, not in a stacked monad; + - or more likely I missed an easy way to do it. So, I have to laboriously + - thread AnnexState through this function. + -} +tryRun :: AnnexState -> [Annex Bool] -> IO () +tryRun state actions = tryRun' state 0 actions +tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () +tryRun' state errnum (a:as) = do + result <- try $ Annex.run state a + case (result) of + Left err -> do + Annex.eval state $ showErr err + tryRun' state (errnum + 1) as + Right (True,state') -> tryRun' state' errnum as + Right (False,state') -> tryRun' state' (errnum + 1) as +tryRun' _ errnum [] = + when (errnum > 0) $ error $ (show errnum) ++ " failed" {- Sets up a git repo for git-annex. -} startup :: Annex Bool diff --git a/git-annex.hs b/git-annex.hs index e958ac2f96..370c22a1ef 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,12 +5,9 @@ - Licensed under the GNU GPL version 3 or higher. -} -import IO (try) import System.Environment -import Monad import qualified Annex -import Types import Core import Commands import qualified GitRepo as Git @@ -23,26 +20,3 @@ main = do state <- Annex.new gitrepo allBackends (configure, actions) <- parseCmd args state tryRun state $ [startup] ++ configure ++ actions ++ [shutdown] - -{- Runs a list of Annex actions. Catches IO errors and continues - - (but explicitly thrown errors terminate the whole command). - - Propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. - -} -tryRun :: AnnexState -> [Annex Bool] -> IO () -tryRun state actions = tryRun' state 0 actions -tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () -tryRun' state errnum (a:as) = do - result <- try $ Annex.run state a - case (result) of - Left err -> do - Annex.eval state $ showErr err - tryRun' state (errnum + 1) as - Right (True,state') -> tryRun' state' errnum as - Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' _ errnum [] = - when (errnum > 0) $ error $ (show errnum) ++ " failed" From 4da551827fd0e5a437037b316f60b5a000e447e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 03:12:28 -0400 Subject: [PATCH 0394/8313] trim --- Core.hs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Core.hs b/Core.hs index d3d0d7899c..327a9af483 100644 --- a/Core.hs +++ b/Core.hs @@ -27,11 +27,6 @@ import Utility {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). - Propigates an overall error status at the end. - - - - This runs in the IO monad, not in the Annex monad. It seems that - - exceptions can only be caught in the IO monad, not in a stacked monad; - - or more likely I missed an easy way to do it. So, I have to laboriously - - thread AnnexState through this function. -} tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions From 59e49ae083eb9e6211eec10c901264abcf3e5676 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 14:49:05 -0400 Subject: [PATCH 0395/8313] rework subcommand invocation logic --- Commands.hs | 132 ++++++++++++++++++++++++--------------------- debian/changelog | 2 + doc/git-annex.mdwn | 14 +++-- 3 files changed, 85 insertions(+), 63 deletions(-) diff --git a/Commands.hs b/Commands.hs index 507c82cccc..0f3c6ac347 100644 --- a/Commands.hs +++ b/Commands.hs @@ -27,12 +27,18 @@ import Core import qualified Remotes import qualified TypeInternals +{- A subcommand can take one of several kinds of input parameters. -} +data SubCmdInput = FilesInGit FilePath | FilesNotInGit FilePath | + FilesMissing FilePath | Description String | Keys String | + Tempfile FilePath | FilesToBeCommitted FilePath + {- A subcommand runs in three stages. Each stage can return the next stage - to run. - - 1. The start stage is run before anything is printed about the - - subcommand, and can early abort it if the input does not make sense. - - It should run quickly and should not modify Annex state. + - subcommand, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. - - 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. @@ -40,18 +46,18 @@ import qualified TypeInternals - 3. The cleanup stage is run only if the perform stage succeeds, and it - returns the overall success/fail of the subcommand. -} -type SubCmdStart = String -> Annex (Maybe SubCmdPerform) +type SubCmdStart = Annex (Maybe SubCmdPerform) type SubCmdPerform = Annex (Maybe SubCmdCleanup) type SubCmdCleanup = Annex Bool {- Runs a subcommand through its three stages. -} -doSubCmd :: String -> SubCmdStart -> String -> Annex Bool -doSubCmd cmdname start param = do - startres <- start param :: Annex (Maybe SubCmdPerform) +doSubCmd :: String -> SubCmdStart -> Annex Bool +doSubCmd cmdname start = do + startres <- start :: Annex (Maybe SubCmdPerform) case (startres) of Nothing -> return True Just perform -> do - showStart cmdname param + --showStart cmdname param performres <- perform :: Annex (Maybe SubCmdCleanup) case (performres) of Nothing -> do @@ -68,15 +74,10 @@ doSubCmd cmdname start param = do return False -{- A subcommand can broadly want one of several kinds of input parameters. - - This allows a first stage of filtering before starting a subcommand. -} -data SubCmdWants = FilesInGit | FilesNotInGit | FilesMissing - | Description | Keys | Tempfile | FilesToBeCommitted - data SubCommand = SubCommand { subcmdname :: String, - subcmdaction :: SubCmdStart, - subcmdwants :: SubCmdWants, + subcmdaction :: (SubCmdInput -> SubCmdStart), + subcmdinput :: (String -> SubCmdInput), subcmddesc :: String } subCmds :: [SubCommand] @@ -139,40 +140,53 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs showcmd c = (subcmdname c) ++ (pad 11 (subcmdname c)) ++ - (descWanted (subcmdwants c)) ++ - (pad 13 (descWanted (subcmdwants c))) ++ + (descSubCmdInput (subcmdinput c)) ++ + (pad 13 (descSubCmdInput (subcmdinput c))) ++ (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' {- Generate descriptions of wanted parameters for subcommands. -} -descWanted :: SubCmdWants -> String -descWanted Description = "DESCRIPTION" -descWanted Keys = "KEY ..." -descWanted _ = "PATH ..." +descSubCmdInput :: (String -> SubCmdInput) -> String +descSubCmdInput Description = "DESCRIPTION" +descSubCmdInput Keys = "KEY ..." +descSubCmdInput _ = "PATH ..." + +{- Prepares a set of actions to run to handle a subcommand, based on + - the parameters passed to it. -} +prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdname = name, subcmdaction = action, + subcmdinput = input, subcmddesc = _ } repo params = do + input <- findInput input params repo + return $ map (doSubCmd name action) input {- Finds the type of parameters a subcommand wants, from among the passed - parameter list. -} -findWanted :: SubCmdWants -> [String] -> Git.Repo -> IO [String] -findWanted FilesNotInGit params repo = do +findInput :: (String -> SubCmdInput) -> [String] -> Git.Repo -> IO [SubCmdInput] +findInput FilesNotInGit params repo = do files <- mapM (Git.notInRepo repo) params - return $ foldl (++) [] files -findWanted FilesInGit params repo = do + return $ map FilesNotInGit $ notState $ foldl (++) [] files +findInput FilesInGit params repo = do files <- mapM (Git.inRepo repo) params - return $ foldl (++) [] files -findWanted FilesMissing params _ = do + return $ map FilesInGit $ notState $ foldl (++) [] files +findInput FilesMissing params _ = do files <- liftIO $ filterM missing params - return $ files + return $ map FilesMissing $ notState $ files where missing f = do e <- doesFileExist f return $ not e -findWanted Description params _ = do - return $ [unwords params] -findWanted FilesToBeCommitted params repo = do +findInput Description params _ = do + return $ map Description $ [unwords params] +findInput FilesToBeCommitted params repo = do files <- mapM (Git.stagedFiles repo) params - return $ foldl (++) [] files -findWanted _ params _ = return params + return $ map FilesToBeCommitted $ notState $ foldl (++) [] files +findInput Keys params _ = return $ map Keys params +findInput Tempfile params _ = return $ map Tempfile params + +{- filter out files from the state directory -} +notState :: [FilePath] -> [FilePath] +notState fs = filter (\f -> stateLoc /= take (length stateLoc) f) fs {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -184,20 +198,15 @@ parseCmd argv state = do when (null params) $ error usage case lookupCmd (params !! 0) of [] -> error usage - [SubCommand { subcmdname = name, subcmdaction = action, - subcmdwants = want, subcmddesc = _ }] -> do - files <- findWanted want (drop 1 params) - (TypeInternals.repo state) - let actions = map (doSubCmd name action) $ - filter notstate files + [subcommand] -> do + let repo = TypeInternals.repo state + actions <- prepSubCmd subcommand repo (drop 1 params) let configactions = map (\flag -> do flag return True) flags return (configactions, actions) _ -> error "internal error: multiple matching subcommands" where - -- never include files from the state directory - notstate f = stateLoc /= take (length stateLoc) f getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) @@ -206,8 +215,8 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: FilePath -> Annex (Maybe SubCmdPerform) -addStart file = notAnnexed file $ do +addStart :: SubCmdInput -> SubCmdStart +addStart (FilesNotInGit file) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing @@ -231,8 +240,8 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: FilePath -> Annex (Maybe SubCmdPerform) -unannexStart file = isAnnexed file $ \(key, backend) -> do +unannexStart :: SubCmdInput -> SubCmdStart +unannexStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do return $ Just $ unannexPerform file key backend unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) unannexPerform file key backend = do @@ -255,8 +264,8 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: FilePath -> Annex (Maybe SubCmdPerform) -getStart file = isAnnexed file $ \(key, backend) -> do +getStart :: SubCmdInput -> Annex (Maybe SubCmdPerform) +getStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing @@ -270,8 +279,8 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: FilePath -> Annex (Maybe SubCmdPerform) -dropStart file = isAnnexed file $ \(key, backend) -> do +dropStart :: SubCmdInput -> SubCmdStart +dropStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing @@ -295,8 +304,8 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: String -> Annex (Maybe SubCmdPerform) -dropKeyStart keyname = do +dropKeyStart :: SubCmdInput -> SubCmdStart +dropKeyStart (Keys keyname) = do backends <- Backend.list let key = genKey (backends !! 0) keyname present <- inAnnex key @@ -318,8 +327,8 @@ dropKeyCleanup key = do return True {- Sets cached content for a key. -} -setKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) -setKeyStart tmpfile = do +setKeyStart :: SubCmdInput -> SubCmdStart +setKeyStart (Tempfile tmpfile) = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list @@ -339,8 +348,11 @@ setKeyCleanup key = do return True {- Fixes the symlink to an annexed file. -} -fixStart :: FilePath -> Annex (Maybe SubCmdPerform) -fixStart file = isAnnexed file $ \(key, _) -> do +fixStart :: SubCmdInput -> SubCmdStart +fixStart (FilesInGit file) = fixStart' file +fixStart (FilesToBeCommitted file) = fixStart' file +fixStart' :: FilePath -> SubCmdStart +fixStart' file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) @@ -358,8 +370,8 @@ fixCleanup file = do return True {- Stores description for the repository etc. -} -initStart :: String -> Annex (Maybe SubCmdPerform) -initStart description = do +initStart :: SubCmdInput -> SubCmdStart +initStart (Description description) = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage return $ Just $ initPerform description @@ -380,8 +392,8 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: FilePath -> Annex (Maybe SubCmdPerform) -fromKeyStart file = do +fromKeyStart :: SubCmdInput -> SubCmdStart +fromKeyStart (FilesMissing file) = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list @@ -406,8 +418,8 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: FilePath -> Annex (Maybe SubCmdPerform) -moveStart file = do +moveStart :: SubCmdInput -> SubCmdStart +moveStart (FilesInGit file) = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of diff --git a/debian/changelog b/debian/changelog index cb6a0ef868..1709f63465 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ git-annex (0.03) UNRELEASED; urgency=low from git before starting, and will be much faster with large repos. * Fix crash on unknown symlinks. * Added remote.annex-scp-options and remote.annex-ssh-options. + * The backends to use when adding different sets of files can be configured + via gitattributes. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 52a8c712f6..bbd7e8cab1 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -159,8 +159,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * --backend=name - Specify the default key-value backend to use, adding it to the front - of the list normally configured by `annex.backends`. + Specifies the key-value backend to use when adding a file. * --key=name @@ -186,7 +185,7 @@ Here are all the supported configuration settings. repositories (default: 1) * `annex.backends` -- space-separated list of names of the key-value backends to use. The first listed is used to store - new files. (default: "WORM SHA1 URL") + new files by default. (default: "WORM SHA1 URL") * `remote..annex-cost` -- When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote @@ -204,6 +203,15 @@ Here are all the supported configuration settings. * `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh options to use if a remote does not have specific options. +The backend used when adding a new file to the annex can be configured +on a per-file-type basis via the `.gitattributes` file. In the file, +the `git-annex-backend` attribute can be set to the name of the backend to +use. For example, this here's how to use the WORM backend by default, +but the SHA1 backend for ogg files: + + * git-annex-backend=WORM + *.ogg git-annex-backend=SHA1 + # FILES These files are used, in your git repository: From fefaa5cc486cc435aa720a1fea29974c1ae17c4a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:01:27 -0400 Subject: [PATCH 0396/8313] big subcommand dispatch rework not quite done.. head hurts --- Commands.hs | 242 +++++++++++++++++++++++++--------------------------- 1 file changed, 114 insertions(+), 128 deletions(-) diff --git a/Commands.hs b/Commands.hs index 0f3c6ac347..9a41e21b73 100644 --- a/Commands.hs +++ b/Commands.hs @@ -27,82 +27,54 @@ import Core import qualified Remotes import qualified TypeInternals -{- A subcommand can take one of several kinds of input parameters. -} -data SubCmdInput = FilesInGit FilePath | FilesNotInGit FilePath | - FilesMissing FilePath | Description String | Keys String | - Tempfile FilePath | FilesToBeCommitted FilePath - -{- A subcommand runs in three stages. Each stage can return the next stage +{- A subcommand runs in four stages. Each stage can return the next stage - to run. - - - 1. The start stage is run before anything is printed about the - - subcommand, is passed some input, and can early abort it - - if the input does not make sense. It should run quickly and - - should not modify Annex state. - - - - 2. The perform stage is run after a message is printed about the subcommand - - being run, and it should be where the bulk of the work happens. - - - - 3. The cleanup stage is run only if the perform stage succeeds, and it - - returns the overall success/fail of the subcommand. - -} + - 0. The parse stage takes the parameters passed to the subcommand, + - looks through the repo to find the ones that are relevant + - to that subcommand (ie, new files to add), and returns a list of + - start stage actions to run. -} +type SubCmdParse = [String] -> Git.Repo -> IO [SubCmdStart] +{- 1. The start stage is run before anything is printed about the + - subcommand, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. -} type SubCmdStart = Annex (Maybe SubCmdPerform) +{- 2. The perform stage is run after a message is printed about the subcommand + - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) +{- 3. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the subcommand. -} type SubCmdCleanup = Annex Bool -{- Runs a subcommand through its three stages. -} -doSubCmd :: String -> SubCmdStart -> Annex Bool -doSubCmd cmdname start = do - startres <- start :: Annex (Maybe SubCmdPerform) - case (startres) of - Nothing -> return True - Just perform -> do - --showStart cmdname param - performres <- perform :: Annex (Maybe SubCmdCleanup) - case (performres) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - cleanupres <- cleanup - if (cleanupres) - then do - showEndOk - return True - else do - showEndFail - return False - - data SubCommand = SubCommand { subcmdname :: String, - subcmdaction :: (SubCmdInput -> SubCmdStart), - subcmdinput :: (String -> SubCmdInput), + subcmdparse :: SubCmdParse, subcmddesc :: String } subCmds :: [SubCommand] subCmds = [ - (SubCommand "add" addStart FilesNotInGit + (SubCommand "add" (withFilesNotInGit addStart) "add files to annex") - , (SubCommand "get" getStart FilesInGit + , (SubCommand "get" (withFilesInGit getStart) "make content of annexed files available") - , (SubCommand "drop" dropStart FilesInGit + , (SubCommand "drop" (withFilesInGit dropStart) "indicate content of files not currently wanted") - , (SubCommand "move" moveStart FilesInGit + , (SubCommand "move" (withFilesInGit moveStart) "transfer content of files to/from another repository") - , (SubCommand "init" initStart Description + , (SubCommand "init" (withDescription initStart) "initialize git-annex with repository description") - , (SubCommand "unannex" unannexStart FilesInGit + , (SubCommand "unannex" (withFilesInGit unannexStart) "undo accidential add command") - , (SubCommand "fix" fixStart FilesInGit + , (SubCommand "fix" (withFilesInGit fixStart) "fix up symlinks to point to annexed content") - , (SubCommand "pre-commit" fixStart FilesToBeCommitted + , (SubCommand "pre-commit" (withFilesToBeCommitted fixStart) "fix up symlinks before they are committed") - , (SubCommand "fromkey" fromKeyStart FilesMissing + , (SubCommand "fromkey" (withFilesMissing fromKeyStart) "adds a file using a specific key") - , (SubCommand "dropkey" dropKeyStart Keys + , (SubCommand "dropkey" (withKeys dropKeyStart) "drops annexed content for specified keys") - , (SubCommand "setkey" setKeyStart Tempfile + , (SubCommand "setkey" (withTempFile setKeyStart) "sets annexed content for a key using a temp file") ] @@ -140,49 +112,66 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs showcmd c = (subcmdname c) ++ (pad 11 (subcmdname c)) ++ - (descSubCmdInput (subcmdinput c)) ++ - (pad 13 (descSubCmdInput (subcmdinput c))) ++ (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Generate descriptions of wanted parameters for subcommands. -} -descSubCmdInput :: (String -> SubCmdInput) -> String -descSubCmdInput Description = "DESCRIPTION" -descSubCmdInput Keys = "KEY ..." -descSubCmdInput _ = "PATH ..." - -{- Prepares a set of actions to run to handle a subcommand, based on +{- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdname = name, subcmdaction = action, - subcmdinput = input, subcmddesc = _ } repo params = do - input <- findInput input params repo - return $ map (doSubCmd name action) input +prepSubCmd SubCommand { subcmdname = name, subcmdparse = parse, + subcmddesc = _ } repo params = do + list <- parse params repo :: IO [SubCmdStart] + return map (\a -> doSubCmd name a) list -{- Finds the type of parameters a subcommand wants, from among the passed - - parameter list. -} -findInput :: (String -> SubCmdInput) -> [String] -> Git.Repo -> IO [SubCmdInput] -findInput FilesNotInGit params repo = do +{- Runs a subcommand through the perform and cleanup stages -} +doSubCmd :: String -> SubCmdPerform -> SubCmdCleanup +doSubCmd cmdname perform = do + p <- perform + case (p) of + Nothing -> do + showEndFail + return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False + +{- These functions parse a user's parameters into a list of SubCmdStart + actions to perform. -} +type ParseStrings = (String -> SubCmdStart) -> SubCmdParse +withFilesNotInGit :: ParseStrings +withFilesNotInGit a params repo = do files <- mapM (Git.notInRepo repo) params - return $ map FilesNotInGit $ notState $ foldl (++) [] files -findInput FilesInGit params repo = do + return $ map a $ notState $ foldl (++) [] files +withFilesInGit :: ParseStrings +withFilesInGit a params repo = do files <- mapM (Git.inRepo repo) params - return $ map FilesInGit $ notState $ foldl (++) [] files -findInput FilesMissing params _ = do + return $ map a $ notState $ foldl (++) [] files +withFilesMissing :: ParseStrings +withFilesMissing a params _ = do files <- liftIO $ filterM missing params - return $ map FilesMissing $ notState $ files + return $ map a $ notState files where missing f = do e <- doesFileExist f return $ not e -findInput Description params _ = do - return $ map Description $ [unwords params] -findInput FilesToBeCommitted params repo = do +withDescription :: ParseStrings +withDescription a params _ = do + return $ [a $ unwords params] +withFilesToBeCommitted :: ParseStrings +withFilesToBeCommitted a params repo = do files <- mapM (Git.stagedFiles repo) params - return $ map FilesToBeCommitted $ notState $ foldl (++) [] files -findInput Keys params _ = return $ map Keys params -findInput Tempfile params _ = return $ map Tempfile params + return $ map a $ notState $ foldl (++) [] files +withKeys :: ParseStrings +withKeys a params _ = return $ map a params +withTempFile :: ParseStrings +withTempFile a params _ = return $ map a params {- filter out files from the state directory -} notState :: [FilePath] -> [FilePath] @@ -215,19 +204,19 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: SubCmdInput -> SubCmdStart -addStart (FilesNotInGit file) = notAnnexed file $ do +addStart :: FilePath -> SubCmdStart +addStart file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing else return $ Just $ addPerform file -addPerform :: FilePath -> Annex (Maybe SubCmdCleanup) +addPerform :: FilePath -> SubCmdPerform addPerform file = do stored <- Backend.storeFileKey file case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> Annex Bool +addCleanup :: FilePath -> Key -> SubCmdCleanup addCleanup file key = do logStatus key ValuePresent g <- Annex.gitRepo @@ -240,10 +229,10 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: SubCmdInput -> SubCmdStart -unannexStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do +unannexStart :: FilePath -> SubCmdStart +unannexStart file = isAnnexed file $ \(key, backend) -> do return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> Annex (Maybe SubCmdCleanup) +unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True @@ -251,7 +240,7 @@ unannexPerform file key backend = do if (ok) then return $ Just $ unannexCleanup file key else return Nothing -unannexCleanup :: FilePath -> Key -> Annex Bool +unannexCleanup :: FilePath -> Key -> SubCmdCleanup unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo @@ -264,13 +253,13 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: SubCmdInput -> Annex (Maybe SubCmdPerform) -getStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do +getStart :: FilePath -> SubCmdStart +getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing else return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +getPerform :: Key -> Backend -> SubCmdPerform getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) @@ -279,19 +268,19 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: SubCmdInput -> SubCmdStart -dropStart (FilesInGit file) = isAnnexed file $ \(key, backend) -> do +dropStart :: FilePath -> SubCmdStart +dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing else return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> Annex (Maybe SubCmdCleanup) +dropPerform :: Key -> Backend -> SubCmdPerform dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key else return Nothing -dropCleanup :: Key -> Annex Bool +dropCleanup :: Key -> SubCmdCleanup dropCleanup key = do logStatus key ValueMissing inannex <- inAnnex key @@ -304,8 +293,8 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: SubCmdInput -> SubCmdStart -dropKeyStart (Keys keyname) = do +dropKeyStart :: String -> SubCmdStart +dropKeyStart keyname = do backends <- Backend.list let key = genKey (backends !! 0) keyname present <- inAnnex key @@ -315,26 +304,26 @@ dropKeyStart (Keys keyname) = do else if (not force) then error "dropkey is can cause data loss; use --force if you're sure you want to do this" else return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> Annex (Maybe SubCmdCleanup) +dropKeyPerform :: Key -> SubCmdPerform dropKeyPerform key = do g <- Annex.gitRepo let loc = annexLocation g key liftIO $ removeFile loc return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> Annex Bool +dropKeyCleanup :: Key -> SubCmdCleanup dropKeyCleanup key = do logStatus key ValueMissing return True {- Sets cached content for a key. -} -setKeyStart :: SubCmdInput -> SubCmdStart -setKeyStart (Tempfile tmpfile) = do +setKeyStart :: FilePath -> SubCmdStart +setKeyStart tmpfile = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +setKeyPerform :: FilePath -> Key -> SubCmdPerform setKeyPerform tmpfile key = do g <- Annex.gitRepo let loc = annexLocation g key @@ -342,40 +331,37 @@ setKeyPerform tmpfile key = do if (not ok) then error "mv failed!" else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> Annex Bool +setKeyCleanup :: Key -> SubCmdCleanup setKeyCleanup key = do logStatus key ValuePresent return True {- Fixes the symlink to an annexed file. -} -fixStart :: SubCmdInput -> SubCmdStart -fixStart (FilesInGit file) = fixStart' file -fixStart (FilesToBeCommitted file) = fixStart' file -fixStart' :: FilePath -> SubCmdStart -fixStart' file = isAnnexed file $ \(key, _) -> do +fixStart :: FilePath -> SubCmdStart +fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file if (link == l) then return Nothing else return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> Annex (Maybe SubCmdCleanup) +fixPerform :: FilePath -> FilePath -> SubCmdPerform fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ fixCleanup file -fixCleanup :: FilePath -> Annex Bool +fixCleanup :: FilePath -> SubCmdCleanup fixCleanup file = do Annex.queue "add" [] file return True {- Stores description for the repository etc. -} -initStart :: SubCmdInput -> SubCmdStart -initStart (Description description) = do +initStart :: String -> SubCmdStart +initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage return $ Just $ initPerform description -initPerform :: String -> Annex (Maybe SubCmdCleanup) +initPerform :: String -> SubCmdPerform initPerform description = do g <- Annex.gitRepo u <- getUUID g @@ -383,7 +369,7 @@ initPerform description = do liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ initCleanup -initCleanup :: Annex Bool +initCleanup :: SubCmdCleanup initCleanup = do g <- Annex.gitRepo logfile <- uuidLog @@ -392,8 +378,8 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: SubCmdInput -> SubCmdStart -fromKeyStart (FilesMissing file) = do +fromKeyStart :: FilePath -> SubCmdStart +fromKeyStart file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list @@ -403,13 +389,13 @@ fromKeyStart (FilesMissing file) = do unless (inbackend) $ error $ "key ("++keyname++") is not present in backend" return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> Annex (Maybe SubCmdCleanup) +fromKeyPerform :: FilePath -> Key -> SubCmdPerform fromKeyPerform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> Annex Bool +fromKeyCleanup :: FilePath -> SubCmdCleanup fromKeyCleanup file = do Annex.queue "add" [] file return True @@ -418,8 +404,8 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: SubCmdInput -> SubCmdStart -moveStart (FilesInGit file) = do +moveStart :: FilePath -> SubCmdStart +moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of @@ -439,13 +425,13 @@ moveStart (FilesInGit file) = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveToStart :: FilePath -> SubCmdStart moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) then return Nothing -- not here, so nothing to do else return $ Just $ moveToPerform key -moveToPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveToPerform :: Key -> SubCmdPerform moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -462,7 +448,7 @@ moveToPerform key = do then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> Annex Bool +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -487,14 +473,14 @@ moveToCleanup remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFromStart :: FilePath -> Annex (Maybe SubCmdPerform) +moveFromStart :: FilePath -> SubCmdStart moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key if (not $ null $ filter (\r -> Remotes.same r remote) l) then return $ Just $ moveFromPerform key else return Nothing -moveFromPerform :: Key -> Annex (Maybe SubCmdCleanup) +moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -506,7 +492,7 @@ moveFromPerform key = do if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> Annex Bool +moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), From 15e7d5913757ca8a76cbd83b3626a463fdea1767 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:12:58 -0400 Subject: [PATCH 0397/8313] rework complete --- Commands.hs | 75 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/Commands.hs b/Commands.hs index 9a41e21b73..99de9bbc86 100644 --- a/Commands.hs +++ b/Commands.hs @@ -119,28 +119,31 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdname = name, subcmdparse = parse, - subcmddesc = _ } repo params = do +prepSubCmd SubCommand { subcmdparse = parse } repo params = do list <- parse params repo :: IO [SubCmdStart] - return map (\a -> doSubCmd name a) list + return $ map (\a -> doSubCmd a) list -{- Runs a subcommand through the perform and cleanup stages -} -doSubCmd :: String -> SubCmdPerform -> SubCmdCleanup -doSubCmd cmdname perform = do - p <- perform - case (p) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - c <- cleanup - if (c) - then do - showEndOk - return True - else do +{- Runs a subcommand through the start, perform and cleanup stages -} +doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd start = do + s <- start + case (s) of + Nothing -> return True + Just perform -> do + p <- perform + case (p) of + Nothing -> do showEndFail return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False {- These functions parse a user's parameters into a list of SubCmdStart actions to perform. -} @@ -209,7 +212,9 @@ addStart file = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing - else return $ Just $ addPerform file + else do + showStart "add" file + return $ Just $ addPerform file addPerform :: FilePath -> SubCmdPerform addPerform file = do stored <- Backend.storeFileKey file @@ -231,6 +236,7 @@ addCleanup file key = do {- The unannex subcommand undoes an add. -} unannexStart :: FilePath -> SubCmdStart unannexStart file = isAnnexed file $ \(key, backend) -> do + showStart "unannex" file return $ Just $ unannexPerform file key backend unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform unannexPerform file key backend = do @@ -258,7 +264,9 @@ getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) then return Nothing - else return $ Just $ getPerform key backend + else do + showStart "get" file + return $ Just $ getPerform key backend getPerform :: Key -> Backend -> SubCmdPerform getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) @@ -273,7 +281,9 @@ dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) then return Nothing - else return $ Just $ dropPerform key backend + else do + showStart "drop" file + return $ Just $ dropPerform key backend dropPerform :: Key -> Backend -> SubCmdPerform dropPerform key backend = do success <- Backend.removeKey backend key @@ -303,7 +313,9 @@ dropKeyStart keyname = do then return Nothing else if (not force) then error "dropkey is can cause data loss; use --force if you're sure you want to do this" - else return $ Just $ dropKeyPerform key + else do + showStart "dropkey" keyname + return $ Just $ dropKeyPerform key dropKeyPerform :: Key -> SubCmdPerform dropKeyPerform key = do g <- Annex.gitRepo @@ -322,6 +334,7 @@ setKeyStart tmpfile = do when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname + showStart "setkey" tmpfile return $ Just $ setKeyPerform tmpfile key setKeyPerform :: FilePath -> Key -> SubCmdPerform setKeyPerform tmpfile key = do @@ -343,7 +356,9 @@ fixStart file = isAnnexed file $ \(key, _) -> do l <- liftIO $ readSymbolicLink file if (link == l) then return Nothing - else return $ Just $ fixPerform file link + else do + showStart "fix" file + return $ Just $ fixPerform file link fixPerform :: FilePath -> FilePath -> SubCmdPerform fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) @@ -360,6 +375,7 @@ initStart :: String -> SubCmdStart initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage + showStart "init" description return $ Just $ initPerform description initPerform :: String -> SubCmdPerform initPerform description = do @@ -388,6 +404,7 @@ fromKeyStart file = do inbackend <- Backend.hasKey key unless (inbackend) $ error $ "key ("++keyname++") is not present in backend" + showStart "fromkey" file return $ Just $ fromKeyPerform file key fromKeyPerform :: FilePath -> Key -> SubCmdPerform fromKeyPerform file key = do @@ -430,7 +447,9 @@ moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) then return Nothing -- not here, so nothing to do - else return $ Just $ moveToPerform key + else do + showStart "move" file + return $ Just $ moveToPerform key moveToPerform :: Key -> SubCmdPerform moveToPerform key = do -- checking the remote is expensive, so not done in the start step @@ -477,9 +496,11 @@ moveFromStart :: FilePath -> SubCmdStart moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key - if (not $ null $ filter (\r -> Remotes.same r remote) l) - then return $ Just $ moveFromPerform key - else return Nothing + if (null $ filter (\r -> Remotes.same r remote) l) + then return Nothing + else do + showStart "move" file + return $ Just $ moveFromPerform key moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote From 287e6e5c1328071ec9b934f75d5250b37a066afe Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:18:32 -0400 Subject: [PATCH 0398/8313] bring back param descriptions in usage --- Commands.hs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/Commands.hs b/Commands.hs index 99de9bbc86..80b355f792 100644 --- a/Commands.hs +++ b/Commands.hs @@ -49,34 +49,39 @@ type SubCmdCleanup = Annex Bool data SubCommand = SubCommand { subcmdname :: String, + subcmdparams :: String, subcmdparse :: SubCmdParse, subcmddesc :: String } subCmds :: [SubCommand] subCmds = [ - (SubCommand "add" (withFilesNotInGit addStart) + (SubCommand "add" path (withFilesNotInGit addStart) "add files to annex") - , (SubCommand "get" (withFilesInGit getStart) + , (SubCommand "get" path (withFilesInGit getStart) "make content of annexed files available") - , (SubCommand "drop" (withFilesInGit dropStart) + , (SubCommand "drop" path (withFilesInGit dropStart) "indicate content of files not currently wanted") - , (SubCommand "move" (withFilesInGit moveStart) + , (SubCommand "move" path (withFilesInGit moveStart) "transfer content of files to/from another repository") - , (SubCommand "init" (withDescription initStart) + , (SubCommand "init" desc (withDescription initStart) "initialize git-annex with repository description") - , (SubCommand "unannex" (withFilesInGit unannexStart) + , (SubCommand "unannex" path (withFilesInGit unannexStart) "undo accidential add command") - , (SubCommand "fix" (withFilesInGit fixStart) + , (SubCommand "fix" path (withFilesInGit fixStart) "fix up symlinks to point to annexed content") - , (SubCommand "pre-commit" (withFilesToBeCommitted fixStart) + , (SubCommand "pre-commit" path (withFilesToBeCommitted fixStart) "fix up symlinks before they are committed") - , (SubCommand "fromkey" (withFilesMissing fromKeyStart) + , (SubCommand "fromkey" key (withFilesMissing fromKeyStart) "adds a file using a specific key") - , (SubCommand "dropkey" (withKeys dropKeyStart) + , (SubCommand "dropkey" key (withKeys dropKeyStart) "drops annexed content for specified keys") - , (SubCommand "setkey" (withTempFile setKeyStart) + , (SubCommand "setkey" key (withTempFile setKeyStart) "sets annexed content for a key using a temp file") ] + where + path = "PATH ..." + key = "KEY ..." + desc = "DESCRIPTION" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. @@ -112,6 +117,8 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs showcmd c = (subcmdname c) ++ (pad 11 (subcmdname c)) ++ + (subcmdparams c) ++ + (pad 13 (subcmdparams c)) ++ (subcmddesc c) indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' From 899a86f8f9601e359a894fe2839dea9cf7f47def Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 17:50:37 -0400 Subject: [PATCH 0399/8313] now only need to add gitattributes lookup --- Backend.hs | 22 +++++++++++++----- Commands.hs | 64 ++++++++++++++++++++++++++++------------------------- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/Backend.hs b/Backend.hs index f1b4c28979..693e1371b5 100644 --- a/Backend.hs +++ b/Backend.hs @@ -23,7 +23,8 @@ module Backend ( retrieveKeyFile, removeKey, hasKey, - lookupFile + lookupFile, + chooseBackends ) where import Control.Monad.State @@ -74,12 +75,15 @@ maybeLookupBackendName bs s = where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} -storeFileKey :: FilePath -> Annex (Maybe (Key, Backend)) -storeFileKey file = do +storeFileKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend)) +storeFileKey file trybackend = do g <- Annex.gitRepo let relfile = Git.relative g file - b <- list - storeFileKey' b file relfile + bs <- list + let bs' = case trybackend of + Nothing -> bs + Just backend -> backend:bs + storeFileKey' bs' file relfile storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend)) storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do @@ -136,3 +140,11 @@ lookupFile file = do kname = keyName k skip = "skipping " ++ file ++ " (unknown backend " ++ bname ++ ")" + +{- Looks up the backends that should be used for each file in a list. + - That can be configured on a per-file basis in the gitattributes file. + -} +chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] +chooseBackends fs = do + -- TODO + return $ map (\f -> (f, Nothing)) fs diff --git a/Commands.hs b/Commands.hs index 80b355f792..c012cdca03 100644 --- a/Commands.hs +++ b/Commands.hs @@ -25,7 +25,6 @@ import LocationLog import Types import Core import qualified Remotes -import qualified TypeInternals {- A subcommand runs in four stages. Each stage can return the next stage - to run. @@ -34,7 +33,7 @@ import qualified TypeInternals - looks through the repo to find the ones that are relevant - to that subcommand (ie, new files to add), and returns a list of - start stage actions to run. -} -type SubCmdParse = [String] -> Git.Repo -> IO [SubCmdStart] +type SubCmdParse = [String] -> Annex [SubCmdStart] {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and @@ -125,9 +124,9 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} -prepSubCmd :: SubCommand -> Git.Repo -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdparse = parse } repo params = do - list <- parse params repo :: IO [SubCmdStart] +prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdparse = parse } state params = do + list <- Annex.eval state $ parse params return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} @@ -155,37 +154,43 @@ doSubCmd start = do {- These functions parse a user's parameters into a list of SubCmdStart actions to perform. -} type ParseStrings = (String -> SubCmdStart) -> SubCmdParse -withFilesNotInGit :: ParseStrings -withFilesNotInGit a params repo = do - files <- mapM (Git.notInRepo repo) params - return $ map a $ notState $ foldl (++) [] files +type ParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdStart) -> SubCmdParse +withFilesNotInGit :: ParseBackendFiles +withFilesNotInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.notInRepo repo) params + let files' = foldl (++) [] files + pairs <- Backend.chooseBackends files' + return $ map a $ filter (\(f,_) -> notState f) pairs withFilesInGit :: ParseStrings -withFilesInGit a params repo = do - files <- mapM (Git.inRepo repo) params - return $ map a $ notState $ foldl (++) [] files +withFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + return $ map a $ filter notState $ foldl (++) [] files withFilesMissing :: ParseStrings -withFilesMissing a params _ = do +withFilesMissing a params = do files <- liftIO $ filterM missing params - return $ map a $ notState files + return $ map a $ filter notState files where missing f = do e <- doesFileExist f return $ not e withDescription :: ParseStrings -withDescription a params _ = do +withDescription a params = do return $ [a $ unwords params] withFilesToBeCommitted :: ParseStrings -withFilesToBeCommitted a params repo = do - files <- mapM (Git.stagedFiles repo) params - return $ map a $ notState $ foldl (++) [] files +withFilesToBeCommitted a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] files withKeys :: ParseStrings -withKeys a params _ = return $ map a params +withKeys a params = return $ map a params withTempFile :: ParseStrings -withTempFile a params _ = return $ map a params +withTempFile a params = return $ map a params {- filter out files from the state directory -} -notState :: [FilePath] -> [FilePath] -notState fs = filter (\f -> stateLoc /= take (length stateLoc) f) fs +notState :: FilePath -> Bool +notState f = stateLoc /= take (length stateLoc) f {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it @@ -198,8 +203,7 @@ parseCmd argv state = do case lookupCmd (params !! 0) of [] -> error usage [subcommand] -> do - let repo = TypeInternals.repo state - actions <- prepSubCmd subcommand repo (drop 1 params) + actions <- prepSubCmd subcommand state (drop 1 params) let configactions = map (\flag -> do flag return True) flags @@ -214,17 +218,17 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: FilePath -> SubCmdStart -addStart file = notAnnexed file $ do +addStart :: (FilePath, Maybe Backend) -> SubCmdStart +addStart pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) then return Nothing else do showStart "add" file - return $ Just $ addPerform file -addPerform :: FilePath -> SubCmdPerform -addPerform file = do - stored <- Backend.storeFileKey file + return $ Just $ addPerform pair +addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform +addPerform (file, backend) = do + stored <- Backend.storeFileKey file backend case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key From ab3294f1dd734e7573c859e123158eda8e3551ac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 18:24:19 -0400 Subject: [PATCH 0400/8313] wrote checkAttr --- GitRepo.hs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 505dd06eb2..e8dd0a5dc3 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -35,7 +35,8 @@ module GitRepo ( repoRemoteName, inRepo, notInRepo, - stagedFiles + stagedFiles, + checkAttr ) where import Monad (unless) @@ -145,6 +146,26 @@ attributes repo | bare repo = (workTree repo) ++ "/info/.gitattributes" | otherwise = (workTree repo) ++ "/.gitattributes" +{- Looks up a gitattributes value for each file in a list. -} +checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] +checkAttr repo attr files = do + (pid, fromh, toh) <- hPipeBoth "git" $ + gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] + -- git-check-attr reads all its stdin before outputting anything, + -- so we don't need to worry about deadlock + hPutStr toh files0 + hClose toh + c <- hGetContentsStrict fromh + hClose fromh + forceSuccess pid + return $ map topair $ lines c + where + files0 = join "\0" files + topair l = (bits !! 0, join sep $ drop 1 $ bits) + where + bits = split sep l + sep = ": " ++ attr ++ ": " + {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String gitDir repo From 2926cc64fb838d1ddaa4b9e123a2eaec6c19cc71 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 19:05:38 -0400 Subject: [PATCH 0401/8313] In .gitattributes, the git-annex-backend attribute can be set to the names of backends to use when adding different types of files. --- Backend.hs | 6 ++++-- debian/changelog | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Backend.hs b/Backend.hs index 693e1371b5..e1d0e0a686 100644 --- a/Backend.hs +++ b/Backend.hs @@ -146,5 +146,7 @@ lookupFile file = do -} chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] chooseBackends fs = do - -- TODO - return $ map (\f -> (f, Nothing)) fs + g <- Annex.gitRepo + bs <- Annex.supportedBackends + pairs <- liftIO $ Git.checkAttr g "git-annex-backend" fs + return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs diff --git a/debian/changelog b/debian/changelog index 1709f63465..b433ec62fd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,8 @@ git-annex (0.03) UNRELEASED; urgency=low * Added remote.annex-scp-options and remote.annex-ssh-options. * The backends to use when adding different sets of files can be configured via gitattributes. + * In .gitattributes, the git-annex-backend attribute can be set to the + names of backends to use when adding different types of files. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From f1f4bdcd6054303a5711944e146093d2f4f8069b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 19:11:06 -0400 Subject: [PATCH 0402/8313] expand docs --- doc/backends.mdwn | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/backends.mdwn b/doc/backends.mdwn index cd587726c0..dc359174ae 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -6,8 +6,8 @@ When a file is annexed, a key is generated from its content and/or metadata. The file checked into git symlinks to the key. This key can later be used to retrieve the file's content (its value). -Multiple pluggable backends are supported, and more than one can be used -to store different files' contents in a given repository. +Multiple pluggable backends are supported, and a single repository +can use different backends for different files. * `WORM` ("Write Once, Read Many") This backend stores the file's content only in `.git/annex/`, and assumes that any file with the same basename, @@ -20,3 +20,18 @@ to store different files' contents in a given repository. can make it slower for large files. **Warning** this backend is not ready for use. * `URL` -- This backend downloads the file's content from an external URL. + +The `annex.backends` git-config setting can be used to list the backends +git-annex should use. The first one listed will be used by default when +new files are added. + +For finer control of what backend is used when adding different types of +files, the `.gitattributes` file can be used. The `git-annex-backend` +attribute can be set to the name of the backend to use for matching files. + +For example, to use the SHA1 backend for sound files, which tend to be +smallish and might be modified over time, you could set in +`.gitattributes`: + + *.mp3 git-annex-backend=SHA1 + *.ogg git-annex-backend=SHA1 From 1f9996f7424a6581ed92b56e3ad3f298348c1daf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 19:18:47 -0400 Subject: [PATCH 0403/8313] less confusing names for the subcommand stage types --- Commands.hs | 96 ++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/Commands.hs b/Commands.hs index c012cdca03..f4f675d025 100644 --- a/Commands.hs +++ b/Commands.hs @@ -33,23 +33,25 @@ import qualified Remotes - looks through the repo to find the ones that are relevant - to that subcommand (ie, new files to add), and returns a list of - start stage actions to run. -} -type SubCmdParse = [String] -> Annex [SubCmdStart] +type SubCmdParseStrings = (String -> SubCmdPerform) -> SubCmdStart +type SubCmdParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdPerform) -> SubCmdStart {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStart = Annex (Maybe SubCmdPerform) +type SubCmdStart = [String] -> Annex [SubCmdPerform] {- 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) {- 3. The cleanup stage is run only if the perform stage succeeds, and it - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex Bool +type SubCmdCleanup = Annex (Maybe SubCmdStatus) +type SubCmdStatus = Annex Bool data SubCommand = SubCommand { subcmdname :: String, subcmdparams :: String, - subcmdparse :: SubCmdParse, + subcmdparse :: SubCmdStart, subcmddesc :: String } subCmds :: [SubCommand] @@ -130,7 +132,7 @@ prepSubCmd SubCommand { subcmdparse = parse } state params = do return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd :: SubCmdPerform -> SubCmdStatus doSubCmd start = do s <- start case (s) of @@ -151,23 +153,21 @@ doSubCmd start = do showEndFail return False -{- These functions parse a user's parameters into a list of SubCmdStart +{- These functions parse a user's parameters into a list of SubCmdPerform actions to perform. -} -type ParseStrings = (String -> SubCmdStart) -> SubCmdParse -type ParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdStart) -> SubCmdParse -withFilesNotInGit :: ParseBackendFiles +withFilesNotInGit :: SubCmdParseBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.notInRepo repo) params let files' = foldl (++) [] files pairs <- Backend.chooseBackends files' return $ map a $ filter (\(f,_) -> notState f) pairs -withFilesInGit :: ParseStrings +withFilesInGit :: SubCmdParseStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: ParseStrings +withFilesMissing :: SubCmdParseStrings withFilesMissing a params = do files <- liftIO $ filterM missing params return $ map a $ filter notState files @@ -175,17 +175,17 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withDescription :: ParseStrings +withDescription :: SubCmdParseStrings withDescription a params = do return $ [a $ unwords params] -withFilesToBeCommitted :: ParseStrings +withFilesToBeCommitted :: SubCmdParseStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.stagedFiles repo) params return $ map a $ filter notState $ foldl (++) [] files -withKeys :: ParseStrings +withKeys :: SubCmdParseStrings withKeys a params = return $ map a params -withTempFile :: ParseStrings +withTempFile :: SubCmdParseStrings withTempFile a params = return $ map a params {- filter out files from the state directory -} @@ -218,7 +218,7 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: (FilePath, Maybe Backend) -> SubCmdStart +addStart :: (FilePath, Maybe Backend) -> SubCmdPerform addStart pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -226,13 +226,13 @@ addStart pair@(file, _) = notAnnexed file $ do else do showStart "add" file return $ Just $ addPerform pair -addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform +addPerform :: (FilePath, Maybe Backend) -> SubCmdCleanup addPerform (file, backend) = do stored <- Backend.storeFileKey file backend case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> SubCmdCleanup +addCleanup :: FilePath -> Key -> SubCmdStatus addCleanup file key = do logStatus key ValuePresent g <- Annex.gitRepo @@ -245,11 +245,11 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: FilePath -> SubCmdStart +unannexStart :: FilePath -> SubCmdPerform unannexStart file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform +unannexPerform :: FilePath -> Key -> Backend -> SubCmdCleanup unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True @@ -257,7 +257,7 @@ unannexPerform file key backend = do if (ok) then return $ Just $ unannexCleanup file key else return Nothing -unannexCleanup :: FilePath -> Key -> SubCmdCleanup +unannexCleanup :: FilePath -> Key -> SubCmdStatus unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo @@ -270,7 +270,7 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: FilePath -> SubCmdStart +getStart :: FilePath -> SubCmdPerform getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) @@ -278,7 +278,7 @@ getStart file = isAnnexed file $ \(key, backend) -> do else do showStart "get" file return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> SubCmdPerform +getPerform :: Key -> Backend -> SubCmdCleanup getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) @@ -287,7 +287,7 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: FilePath -> SubCmdStart +dropStart :: FilePath -> SubCmdPerform dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) @@ -295,13 +295,13 @@ dropStart file = isAnnexed file $ \(key, backend) -> do else do showStart "drop" file return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> SubCmdPerform +dropPerform :: Key -> Backend -> SubCmdCleanup dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key else return Nothing -dropCleanup :: Key -> SubCmdCleanup +dropCleanup :: Key -> SubCmdStatus dropCleanup key = do logStatus key ValueMissing inannex <- inAnnex key @@ -314,7 +314,7 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: String -> SubCmdStart +dropKeyStart :: String -> SubCmdPerform dropKeyStart keyname = do backends <- Backend.list let key = genKey (backends !! 0) keyname @@ -327,19 +327,19 @@ dropKeyStart keyname = do else do showStart "dropkey" keyname return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> SubCmdPerform +dropKeyPerform :: Key -> SubCmdCleanup dropKeyPerform key = do g <- Annex.gitRepo let loc = annexLocation g key liftIO $ removeFile loc return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> SubCmdCleanup +dropKeyCleanup :: Key -> SubCmdStatus dropKeyCleanup key = do logStatus key ValueMissing return True {- Sets cached content for a key. -} -setKeyStart :: FilePath -> SubCmdStart +setKeyStart :: FilePath -> SubCmdPerform setKeyStart tmpfile = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -347,7 +347,7 @@ setKeyStart tmpfile = do let key = genKey (backends !! 0) keyname showStart "setkey" tmpfile return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> SubCmdPerform +setKeyPerform :: FilePath -> Key -> SubCmdCleanup setKeyPerform tmpfile key = do g <- Annex.gitRepo let loc = annexLocation g key @@ -355,13 +355,13 @@ setKeyPerform tmpfile key = do if (not ok) then error "mv failed!" else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> SubCmdCleanup +setKeyCleanup :: Key -> SubCmdStatus setKeyCleanup key = do logStatus key ValuePresent return True {- Fixes the symlink to an annexed file. -} -fixStart :: FilePath -> SubCmdStart +fixStart :: FilePath -> SubCmdPerform fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file @@ -370,25 +370,25 @@ fixStart file = isAnnexed file $ \(key, _) -> do else do showStart "fix" file return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> SubCmdPerform +fixPerform :: FilePath -> FilePath -> SubCmdCleanup fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ fixCleanup file -fixCleanup :: FilePath -> SubCmdCleanup +fixCleanup :: FilePath -> SubCmdStatus fixCleanup file = do Annex.queue "add" [] file return True {- Stores description for the repository etc. -} -initStart :: String -> SubCmdStart +initStart :: String -> SubCmdPerform initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage showStart "init" description return $ Just $ initPerform description -initPerform :: String -> SubCmdPerform +initPerform :: String -> SubCmdCleanup initPerform description = do g <- Annex.gitRepo u <- getUUID g @@ -396,7 +396,7 @@ initPerform description = do liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ initCleanup -initCleanup :: SubCmdCleanup +initCleanup :: SubCmdStatus initCleanup = do g <- Annex.gitRepo logfile <- uuidLog @@ -405,7 +405,7 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: FilePath -> SubCmdStart +fromKeyStart :: FilePath -> SubCmdPerform fromKeyStart file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -417,13 +417,13 @@ fromKeyStart file = do "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> SubCmdPerform +fromKeyPerform :: FilePath -> Key -> SubCmdCleanup fromKeyPerform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> SubCmdCleanup +fromKeyCleanup :: FilePath -> SubCmdStatus fromKeyCleanup file = do Annex.queue "add" [] file return True @@ -432,7 +432,7 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: FilePath -> SubCmdStart +moveStart :: FilePath -> SubCmdPerform moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" @@ -453,7 +453,7 @@ moveStart file = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: FilePath -> SubCmdStart +moveToStart :: FilePath -> SubCmdPerform moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) @@ -461,7 +461,7 @@ moveToStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdPerform +moveToPerform :: Key -> SubCmdCleanup moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -478,7 +478,7 @@ moveToPerform key = do then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdStatus moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -503,7 +503,7 @@ moveToCleanup remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFromStart :: FilePath -> SubCmdStart +moveFromStart :: FilePath -> SubCmdPerform moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key @@ -512,7 +512,7 @@ moveFromStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdPerform +moveFromPerform :: Key -> SubCmdCleanup moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -524,7 +524,7 @@ moveFromPerform key = do if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup +moveFromCleanup :: Git.Repo -> Key -> SubCmdStatus moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), From f0bf94f76021952f0a4803ab13c8dfdc3c78b148 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 20:03:21 -0400 Subject: [PATCH 0404/8313] better types --- Commands.hs | 65 +++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/Commands.hs b/Commands.hs index f4f675d025..01761d0309 100644 --- a/Commands.hs +++ b/Commands.hs @@ -26,20 +26,21 @@ import Types import Core import qualified Remotes -{- A subcommand runs in four stages. Each stage can return the next stage - - to run. +{- A subcommand runs in four stages. - - - 0. The parse stage takes the parameters passed to the subcommand, + - 0. The seek stage takes the parameters passed to the subcommand, - looks through the repo to find the ones that are relevant - - to that subcommand (ie, new files to add), and returns a list of - - start stage actions to run. -} -type SubCmdParseStrings = (String -> SubCmdPerform) -> SubCmdStart -type SubCmdParseBackendFiles = ((FilePath, Maybe Backend) -> SubCmdPerform) -> SubCmdStart + - to that subcommand (ie, new files to add), and generates + - a start stage action. -} +type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek +type SubCmdSeek = [String] -> Annex [SubCmdPerform] {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStart = [String] -> Annex [SubCmdPerform] +type SubCmdStartString = String -> SubCmdPerform +type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdPerform {- 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) @@ -51,7 +52,7 @@ type SubCmdStatus = Annex Bool data SubCommand = SubCommand { subcmdname :: String, subcmdparams :: String, - subcmdparse :: SubCmdStart, + subcmdseek :: SubCmdSeek, subcmddesc :: String } subCmds :: [SubCommand] @@ -127,8 +128,8 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- Prepares a set of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdparse = parse } state params = do - list <- Annex.eval state $ parse params +prepSubCmd SubCommand { subcmdseek = seek } state params = do + list <- Annex.eval state $ seek params return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} @@ -153,21 +154,21 @@ doSubCmd start = do showEndFail return False -{- These functions parse a user's parameters into a list of SubCmdPerform - actions to perform. -} -withFilesNotInGit :: SubCmdParseBackendFiles +{- These functions find appropriate files or other things based on a + user's parameters. -} +withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.notInRepo repo) params let files' = foldl (++) [] files pairs <- Backend.chooseBackends files' return $ map a $ filter (\(f,_) -> notState f) pairs -withFilesInGit :: SubCmdParseStrings +withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: SubCmdParseStrings +withFilesMissing :: SubCmdSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params return $ map a $ filter notState files @@ -175,17 +176,17 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withDescription :: SubCmdParseStrings +withDescription :: SubCmdSeekStrings withDescription a params = do return $ [a $ unwords params] -withFilesToBeCommitted :: SubCmdParseStrings +withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.stagedFiles repo) params return $ map a $ filter notState $ foldl (++) [] files -withKeys :: SubCmdParseStrings +withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params -withTempFile :: SubCmdParseStrings +withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params {- filter out files from the state directory -} @@ -218,7 +219,7 @@ parseCmd argv state = do {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -addStart :: (FilePath, Maybe Backend) -> SubCmdPerform +addStart :: SubCmdStartBackendFile addStart pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if ((isSymbolicLink s) || (not $ isRegularFile s)) @@ -245,7 +246,7 @@ addCleanup file key = do return True {- The unannex subcommand undoes an add. -} -unannexStart :: FilePath -> SubCmdPerform +unannexStart :: SubCmdStartString unannexStart file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ unannexPerform file key backend @@ -270,7 +271,7 @@ unannexCleanup file key = do return True {- Gets an annexed file from one of the backends. -} -getStart :: FilePath -> SubCmdPerform +getStart :: SubCmdStartString getStart file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if (inannex) @@ -287,7 +288,7 @@ getPerform key backend = do {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -dropStart :: FilePath -> SubCmdPerform +dropStart :: SubCmdStartString dropStart file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if (not inbackend) @@ -314,7 +315,7 @@ dropCleanup key = do else return True {- Drops cached content for a key. -} -dropKeyStart :: String -> SubCmdPerform +dropKeyStart :: SubCmdStartString dropKeyStart keyname = do backends <- Backend.list let key = genKey (backends !! 0) keyname @@ -339,7 +340,7 @@ dropKeyCleanup key = do return True {- Sets cached content for a key. -} -setKeyStart :: FilePath -> SubCmdPerform +setKeyStart :: SubCmdStartString setKeyStart tmpfile = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -361,7 +362,7 @@ setKeyCleanup key = do return True {- Fixes the symlink to an annexed file. -} -fixStart :: FilePath -> SubCmdPerform +fixStart :: SubCmdStartString fixStart file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file @@ -382,7 +383,7 @@ fixCleanup file = do return True {- Stores description for the repository etc. -} -initStart :: String -> SubCmdPerform +initStart :: SubCmdStartString initStart description = do when (null description) $ error $ "please specify a description of this repository\n" ++ usage @@ -405,7 +406,7 @@ initCleanup = do return True {- Adds a file pointing at a manually-specified key -} -fromKeyStart :: FilePath -> SubCmdPerform +fromKeyStart :: SubCmdStartString fromKeyStart file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -432,7 +433,7 @@ fromKeyCleanup file = do - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -moveStart :: FilePath -> SubCmdPerform +moveStart :: SubCmdStartString moveStart file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" @@ -453,7 +454,7 @@ moveStart file = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: FilePath -> SubCmdPerform +moveToStart :: SubCmdStartString moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if (not ishere) @@ -503,7 +504,7 @@ moveToCleanup remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository. -} -moveFromStart :: FilePath -> SubCmdPerform +moveFromStart :: SubCmdStartString moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key From 82d5a46c5618c6d35ef7b85c5cc257875de4a34b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 20:13:10 -0400 Subject: [PATCH 0405/8313] finally got the types clear enough --- Commands.hs | 64 +++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/Commands.hs b/Commands.hs index 01761d0309..199b83abf6 100644 --- a/Commands.hs +++ b/Commands.hs @@ -31,23 +31,25 @@ import qualified Remotes - 0. The seek stage takes the parameters passed to the subcommand, - looks through the repo to find the ones that are relevant - to that subcommand (ie, new files to add), and generates - - a start stage action. -} -type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdSeek = [String] -> Annex [SubCmdPerform] + - a list of start stage actions. -} +type SubCmdSeek = [String] -> Annex [SubCmdStart] {- 1. The start stage is run before anything is printed about the - subcommand, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStartString = String -> SubCmdPerform -type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdPerform +type SubCmdStart = Annex (Maybe SubCmdPerform) {- 2. The perform stage is run after a message is printed about the subcommand - being run, and it should be where the bulk of the work happens. -} type SubCmdPerform = Annex (Maybe SubCmdCleanup) {- 3. The cleanup stage is run only if the perform stage succeeds, and it - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex (Maybe SubCmdStatus) -type SubCmdStatus = Annex Bool +type SubCmdCleanup = Annex Bool +{- Some helper functions are used to build up SubCmdSeek and SubCmdStart + - functions. -} +type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek +type SubCmdStartString = String -> SubCmdStart +type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart data SubCommand = SubCommand { subcmdname :: String, @@ -125,7 +127,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Prepares a set of actions to run to perform a subcommand, based on +{- Prepares a list of actions to run to perform a subcommand, based on - the parameters passed to it. -} prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] prepSubCmd SubCommand { subcmdseek = seek } state params = do @@ -133,7 +135,7 @@ prepSubCmd SubCommand { subcmdseek = seek } state params = do return $ map (\a -> doSubCmd a) list {- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdPerform -> SubCmdStatus +doSubCmd :: SubCmdStart -> SubCmdCleanup doSubCmd start = do s <- start case (s) of @@ -227,13 +229,13 @@ addStart pair@(file, _) = notAnnexed file $ do else do showStart "add" file return $ Just $ addPerform pair -addPerform :: (FilePath, Maybe Backend) -> SubCmdCleanup +addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform addPerform (file, backend) = do stored <- Backend.storeFileKey file backend case (stored) of Nothing -> return Nothing Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> SubCmdStatus +addCleanup :: FilePath -> Key -> SubCmdCleanup addCleanup file key = do logStatus key ValuePresent g <- Annex.gitRepo @@ -250,7 +252,7 @@ unannexStart :: SubCmdStartString unannexStart file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> SubCmdCleanup +unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform unannexPerform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True @@ -258,7 +260,7 @@ unannexPerform file key backend = do if (ok) then return $ Just $ unannexCleanup file key else return Nothing -unannexCleanup :: FilePath -> Key -> SubCmdStatus +unannexCleanup :: FilePath -> Key -> SubCmdCleanup unannexCleanup file key = do logStatus key ValueMissing g <- Annex.gitRepo @@ -279,7 +281,7 @@ getStart file = isAnnexed file $ \(key, backend) -> do else do showStart "get" file return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> SubCmdCleanup +getPerform :: Key -> Backend -> SubCmdPerform getPerform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if (ok) @@ -296,13 +298,13 @@ dropStart file = isAnnexed file $ \(key, backend) -> do else do showStart "drop" file return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> SubCmdCleanup +dropPerform :: Key -> Backend -> SubCmdPerform dropPerform key backend = do success <- Backend.removeKey backend key if (success) then return $ Just $ dropCleanup key else return Nothing -dropCleanup :: Key -> SubCmdStatus +dropCleanup :: Key -> SubCmdCleanup dropCleanup key = do logStatus key ValueMissing inannex <- inAnnex key @@ -328,13 +330,13 @@ dropKeyStart keyname = do else do showStart "dropkey" keyname return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> SubCmdCleanup +dropKeyPerform :: Key -> SubCmdPerform dropKeyPerform key = do g <- Annex.gitRepo let loc = annexLocation g key liftIO $ removeFile loc return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> SubCmdStatus +dropKeyCleanup :: Key -> SubCmdCleanup dropKeyCleanup key = do logStatus key ValueMissing return True @@ -348,7 +350,7 @@ setKeyStart tmpfile = do let key = genKey (backends !! 0) keyname showStart "setkey" tmpfile return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> SubCmdCleanup +setKeyPerform :: FilePath -> Key -> SubCmdPerform setKeyPerform tmpfile key = do g <- Annex.gitRepo let loc = annexLocation g key @@ -356,7 +358,7 @@ setKeyPerform tmpfile key = do if (not ok) then error "mv failed!" else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> SubCmdStatus +setKeyCleanup :: Key -> SubCmdCleanup setKeyCleanup key = do logStatus key ValuePresent return True @@ -371,13 +373,13 @@ fixStart file = isAnnexed file $ \(key, _) -> do else do showStart "fix" file return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> SubCmdCleanup +fixPerform :: FilePath -> FilePath -> SubCmdPerform fixPerform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ fixCleanup file -fixCleanup :: FilePath -> SubCmdStatus +fixCleanup :: FilePath -> SubCmdCleanup fixCleanup file = do Annex.queue "add" [] file return True @@ -389,7 +391,7 @@ initStart description = do "please specify a description of this repository\n" ++ usage showStart "init" description return $ Just $ initPerform description -initPerform :: String -> SubCmdCleanup +initPerform :: String -> SubCmdPerform initPerform description = do g <- Annex.gitRepo u <- getUUID g @@ -397,7 +399,7 @@ initPerform description = do liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ initCleanup -initCleanup :: SubCmdStatus +initCleanup :: SubCmdCleanup initCleanup = do g <- Annex.gitRepo logfile <- uuidLog @@ -418,13 +420,13 @@ fromKeyStart file = do "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> SubCmdCleanup +fromKeyPerform :: FilePath -> Key -> SubCmdPerform fromKeyPerform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> SubCmdStatus +fromKeyCleanup :: FilePath -> SubCmdCleanup fromKeyCleanup file = do Annex.queue "add" [] file return True @@ -462,7 +464,7 @@ moveToStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdCleanup +moveToPerform :: Key -> SubCmdPerform moveToPerform key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -479,7 +481,7 @@ moveToPerform key = do then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdStatus +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -513,7 +515,7 @@ moveFromStart file = isAnnexed file $ \(key, _) -> do else do showStart "move" file return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdCleanup +moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -525,7 +527,7 @@ moveFromPerform key = do if (ok) then return $ Just $ moveFromCleanup remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdStatus +moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", "--backend=" ++ (backendName key), From e379307900ba571a5104f3bb2e7ad6c6900d3062 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 20:27:30 -0400 Subject: [PATCH 0406/8313] better message when ikiwiki is not available --- Commands.hs | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Commands.hs b/Commands.hs index 199b83abf6..330b71ed27 100644 --- a/Commands.hs +++ b/Commands.hs @@ -47,8 +47,8 @@ type SubCmdCleanup = Annex Bool {- Some helper functions are used to build up SubCmdSeek and SubCmdStart - functions. -} type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartString = String -> SubCmdStart +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart data SubCommand = SubCommand { diff --git a/Makefile b/Makefile index ed262333a3..5e27c1e904 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ install: # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) -IKIWIKI=echo "** ikiwiki not found, skipping building docs" >&2 +IKIWIKI=@echo "** ikiwiki not found, skipping building docs" >&2; true else IKIWIKI=ikiwiki endif From 13514b6afc6cb043dfa742dd0e69efca7a4374df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 21:21:13 -0400 Subject: [PATCH 0407/8313] fix checkAttr to not deadlock and not need strictness Yeah, "will never deadlock" is never a good sign in code comments. ;) --- GitRepo.hs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index e8dd0a5dc3..bc6d450664 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -149,17 +149,12 @@ attributes repo {- Looks up a gitattributes value for each file in a list. -} checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] checkAttr repo attr files = do - (pid, fromh, toh) <- hPipeBoth "git" $ - gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] - -- git-check-attr reads all its stdin before outputting anything, - -- so we don't need to worry about deadlock - hPutStr toh files0 - hClose toh - c <- hGetContentsStrict fromh - hClose fromh - forceSuccess pid - return $ map topair $ lines c + (handle, s) <- pipeBoth "git" params files0 + return $ map topair $ lines s + -- XXX handle is left open, this is ok for git-annex, but may need + -- to be cleaned up for other uses. where + params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] files0 = join "\0" files topair l = (bits !! 0, join sep $ drop 1 $ bits) where From 58c89565e9f9b3a05b0b4ff33944b543d226c398 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 1 Nov 2010 22:50:53 -0400 Subject: [PATCH 0408/8313] updat --- Backend/SHA1.hs | 5 ++++- GitRepo.hs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 76c368f84e..76869b071d 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -14,6 +14,7 @@ import System.IO import qualified Backend.File import TypeInternals +import Core backend :: Backend backend = Backend.File.backend { @@ -23,7 +24,9 @@ backend = Backend.File.backend { -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do +keyValue file = do + showNote "checksum" + liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line if (null bits) diff --git a/GitRepo.hs b/GitRepo.hs index bc6d450664..244acda862 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -149,7 +149,7 @@ attributes repo {- Looks up a gitattributes value for each file in a list. -} checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] checkAttr repo attr files = do - (handle, s) <- pipeBoth "git" params files0 + (_, s) <- pipeBoth "git" params files0 return $ map topair $ lines s -- XXX handle is left open, this is ok for git-annex, but may need -- to be cleaned up for other uses. From 790613333c2d946d28fe62ef7da8ed3a23428f99 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 01:07:34 -0400 Subject: [PATCH 0409/8313] deal with git's insane octal filename encoding --- GitRepo.hs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 244acda862..389aa35fe1 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -156,11 +156,40 @@ checkAttr repo attr files = do where params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] files0 = join "\0" files - topair l = (bits !! 0, join sep $ drop 1 $ bits) + topair l = (file, value) where + file = decodeGitFile $ join sep $ take end bits + value = bits !! end + end = length bits - 1 bits = split sep l sep = ": " ++ attr ++ ": " +{- Some git commands output encoded filenames. Such a filename + - will always be double-quoted, and then \nnn (in octal) is used + - to escape high characters. -} +decodeGitFile :: String -> FilePath +decodeGitFile [] = [] +decodeGitFile f@(c:s) + | c == '"' = unescape middle + | otherwise = f + where + e = "\\" + middle = take (length s - 1) s + unescape v = foldl (++) beginning $ map decode $ split e rest + where + pair = span (/= '\\') v + beginning = fst pair + rest = snd pair + decode [] = "" + decode n + | length num == 3 = (chr $ readoctal num):rest + | otherwise = e++n + where + pair = span isOctDigit n + num = fst pair + rest = snd pair + readoctal o = read $ "0o" ++ o :: Int + {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String gitDir repo From bdb7ed76cf233a5c7415b2235123fe448f130ab9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 02:24:19 -0400 Subject: [PATCH 0410/8313] tweak --- Backend/SHA1.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 76869b071d..4858922585 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -25,7 +25,7 @@ backend = Backend.File.backend { -- checksum the file to get its key keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do - showNote "checksum" + showNote "checksum..." liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line From 82056a79213268c4d7d93c64c013ca44056f70ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 12:43:34 -0400 Subject: [PATCH 0411/8313] reorg --- GitRepo.hs | 88 +++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 389aa35fe1..9fda0a23fe 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -146,50 +146,6 @@ attributes repo | bare repo = (workTree repo) ++ "/info/.gitattributes" | otherwise = (workTree repo) ++ "/.gitattributes" -{- Looks up a gitattributes value for each file in a list. -} -checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] -checkAttr repo attr files = do - (_, s) <- pipeBoth "git" params files0 - return $ map topair $ lines s - -- XXX handle is left open, this is ok for git-annex, but may need - -- to be cleaned up for other uses. - where - params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] - files0 = join "\0" files - topair l = (file, value) - where - file = decodeGitFile $ join sep $ take end bits - value = bits !! end - end = length bits - 1 - bits = split sep l - sep = ": " ++ attr ++ ": " - -{- Some git commands output encoded filenames. Such a filename - - will always be double-quoted, and then \nnn (in octal) is used - - to escape high characters. -} -decodeGitFile :: String -> FilePath -decodeGitFile [] = [] -decodeGitFile f@(c:s) - | c == '"' = unescape middle - | otherwise = f - where - e = "\\" - middle = take (length s - 1) s - unescape v = foldl (++) beginning $ map decode $ split e rest - where - pair = span (/= '\\') v - beginning = fst pair - rest = snd pair - decode [] = "" - decode n - | length num == 3 = (chr $ readoctal num):rest - | otherwise = e++n - where - pair = span isOctDigit n - num = fst pair - rest = snd pair - readoctal o = read $ "0o" ++ o :: Int - {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String gitDir repo @@ -352,6 +308,50 @@ configGet repo key defaultValue = configMap :: Repo -> Map.Map String String configMap repo = config repo +{- Looks up a gitattributes value for each file in a list. -} +checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] +checkAttr repo attr files = do + (_, s) <- pipeBoth "git" params files0 + return $ map topair $ lines s + -- XXX handle is left open, this is ok for git-annex, but may need + -- to be cleaned up for other uses. + where + params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] + files0 = join "\0" files + topair l = (file, value) + where + file = decodeGitFile $ join sep $ take end bits + value = bits !! end + end = length bits - 1 + bits = split sep l + sep = ": " ++ attr ++ ": " + +{- Some git commands output encoded filenames. Such a filename + - will always be double-quoted, and then \nnn (in octal) is used + - to escape high characters. -} +decodeGitFile :: String -> FilePath +decodeGitFile [] = [] +decodeGitFile f@(c:s) + | c == '"' = unescape middle + | otherwise = f + where + e = "\\" + middle = take (length s - 1) s + unescape v = foldl (++) beginning $ map decode $ split e rest + where + pair = span (/= '\\') v + beginning = fst pair + rest = snd pair + decode [] = "" + decode n + | length num == 3 = (chr $ readoctal num):rest + | otherwise = e++n + where + pair = span isOctDigit n + num = fst pair + rest = snd pair + readoctal o = read $ "0o" ++ o :: Int + {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo repoFromCwd = do From 9e5985ff983c4b489d545e99946cf62e34fadcd9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 15:54:43 -0400 Subject: [PATCH 0412/8313] blew several hours on getting the decodeGitFile 100% right with quickcheck --- GitRepo.hs | 76 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 9fda0a23fe..c62a820d8e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -36,7 +36,11 @@ module GitRepo ( inRepo, notInRepo, stagedFiles, - checkAttr + checkAttr, + decodeGitFile, + encodeGitFile, + + prop_idempotent_encode ) where import Monad (unless) @@ -51,6 +55,9 @@ import qualified Data.Map as Map hiding (map, split) import Network.URI import Maybe import Char +import Text.Printf +import Data.Word (Word8) +import Codec.Binary.UTF8.String (encode) import Utility @@ -332,25 +339,70 @@ checkAttr repo attr files = do decodeGitFile :: String -> FilePath decodeGitFile [] = [] decodeGitFile f@(c:s) - | c == '"' = unescape middle + | c == '"' = unescape ("", middle) | otherwise = f where - e = "\\" middle = take (length s - 1) s - unescape v = foldl (++) beginning $ map decode $ split e rest + unescape (b, []) = b + unescape (b, v) = b ++ beginning ++ unescape (decode rest) where pair = span (/= '\\') v beginning = fst pair rest = snd pair - decode [] = "" - decode n - | length num == 3 = (chr $ readoctal num):rest - | otherwise = e++n + isescape c = c == '\\' + decode (e:n1:n2:n3:rest) + | isescape e && alloctal = (fromoctal, rest) + where + alloctal = isOctDigit n1 && + isOctDigit n2 && + isOctDigit n3 + fromoctal = [chr $ readoctal (n1:n2:n3:[])] + readoctal o = read $ "0o" ++ o :: Int + decode (e:nc:rest) + | isescape e = ([echar nc], rest) where - pair = span isOctDigit n - num = fst pair - rest = snd pair - readoctal o = read $ "0o" ++ o :: Int + -- special character escapes + echar 'a' = '\a' + echar 'b' = '\b' + echar 'f' = '\f' + echar 'n' = '\n' + echar 'r' = '\r' + echar 't' = '\t' + echar 'v' = '\v' + echar x = x + decode n = ("", n) + +{- Should not need to use this, except for testing decodeGitFile. -} +encodeGitFile :: FilePath -> String +encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" + where + e c = "\\" ++ [c] + echar '\a' = e 'a' + echar '\b' = e 'b' + echar '\f' = e 'f' + echar '\n' = e 'n' + echar '\r' = e 'r' + echar '\t' = e 't' + echar '\v' = e 'v' + echar '\\' = e '\\' + echar '"' = e '"' + echar x + | ord x < 0x20 = e_num x -- low ascii + | ord x >= 256 = e_utf x + | ord x > 0x7E = e_num x -- high ascii + | otherwise = [x] -- printable ascii + where + showoctal i = "\\" ++ (printf "%03o" i) + e_num c = showoctal $ ord c + -- unicode character is decomposed to Word8 + -- and each is shown with e_num + e_utf c = foldl (++) "" $ map showoctal $ + (encode [c] :: [Word8]) + + +{- for quickcheck -} +prop_idempotent_encode :: String -> Bool +prop_idempotent_encode s = s == (decodeGitFile $ encodeGitFile s) {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo From cecb1cbeb2017e9e3be252c253ee7f2b49235c63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 16:00:55 -0400 Subject: [PATCH 0413/8313] clean up --- GitRepo.hs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index c62a820d8e..752253b416 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -333,23 +333,26 @@ checkAttr repo attr files = do bits = split sep l sep = ": " ++ attr ++ ": " -{- Some git commands output encoded filenames. Such a filename - - will always be double-quoted, and then \nnn (in octal) is used - - to escape high characters. -} +{- Some git commands output encoded filenames. Decode that (annoyingly + - complex) encoding. -} decodeGitFile :: String -> FilePath decodeGitFile [] = [] decodeGitFile f@(c:s) + -- encoded strings will be inside double quotes | c == '"' = unescape ("", middle) | otherwise = f where + e = '\\' middle = take (length s - 1) s unescape (b, []) = b + -- look for escapes starting with '\' unescape (b, v) = b ++ beginning ++ unescape (decode rest) where - pair = span (/= '\\') v + pair = span (/= e) v beginning = fst pair rest = snd pair - isescape c = c == '\\' + isescape c = c == e + -- \NNN is an octal encoded character decode (e:n1:n2:n3:rest) | isescape e && alloctal = (fromoctal, rest) where @@ -358,10 +361,10 @@ decodeGitFile f@(c:s) isOctDigit n3 fromoctal = [chr $ readoctal (n1:n2:n3:[])] readoctal o = read $ "0o" ++ o :: Int + -- \C is used for a few special characters decode (e:nc:rest) | isescape e = ([echar nc], rest) where - -- special character escapes echar 'a' = '\a' echar 'b' = '\b' echar 'f' = '\f' From c7b0f60fba9f53ed97c43f3ad9a48e7698b97760 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 16:02:43 -0400 Subject: [PATCH 0414/8313] clean up --- GitRepo.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 752253b416..d9dd086f2a 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -397,8 +397,8 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" where showoctal i = "\\" ++ (printf "%03o" i) e_num c = showoctal $ ord c - -- unicode character is decomposed to Word8 - -- and each is shown with e_num + -- unicode character is decomposed to + -- Word8s and each is shown in octal e_utf c = foldl (++) "" $ map showoctal $ (encode [c] :: [Word8]) From d93a37289406fbeb0cedf05f1c8009f68cdb2570 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 16:49:35 -0400 Subject: [PATCH 0415/8313] add a stupid test harness --- GitRepo.hs | 18 +++++++++--------- Makefile | 3 +++ test.hs | 8 ++++++++ 3 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 test.hs diff --git a/GitRepo.hs b/GitRepo.hs index d9dd086f2a..7d5291ff1e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -40,7 +40,7 @@ module GitRepo ( decodeGitFile, encodeGitFile, - prop_idempotent_encode + prop_idempotent_deencode ) where import Monad (unless) @@ -351,10 +351,10 @@ decodeGitFile f@(c:s) pair = span (/= e) v beginning = fst pair rest = snd pair - isescape c = c == e + isescape x = x == e -- \NNN is an octal encoded character - decode (e:n1:n2:n3:rest) - | isescape e && alloctal = (fromoctal, rest) + decode (x:n1:n2:n3:rest) + | isescape x && alloctal = (fromoctal, rest) where alloctal = isOctDigit n1 && isOctDigit n2 && @@ -362,8 +362,8 @@ decodeGitFile f@(c:s) fromoctal = [chr $ readoctal (n1:n2:n3:[])] readoctal o = read $ "0o" ++ o :: Int -- \C is used for a few special characters - decode (e:nc:rest) - | isescape e = ([echar nc], rest) + decode (x:nc:rest) + | isescape x = ([echar nc], rest) where echar 'a' = '\a' echar 'b' = '\b' @@ -372,7 +372,7 @@ decodeGitFile f@(c:s) echar 'r' = '\r' echar 't' = '\t' echar 'v' = '\v' - echar x = x + echar a = a decode n = ("", n) {- Should not need to use this, except for testing decodeGitFile. -} @@ -404,8 +404,8 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" {- for quickcheck -} -prop_idempotent_encode :: String -> Bool -prop_idempotent_encode s = s == (decodeGitFile $ encodeGitFile s) +prop_idempotent_deencode :: String -> Bool +prop_idempotent_deencode s = s == (decodeGitFile $ encodeGitFile s) {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo diff --git a/Makefile b/Makefile index 5e27c1e904..eb74bb7273 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ else IKIWIKI=ikiwiki endif +test: + runghc test.hs + docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ diff --git a/test.hs b/test.hs new file mode 100644 index 0000000000..e9ea684593 --- /dev/null +++ b/test.hs @@ -0,0 +1,8 @@ +-- TODO find a test harness that is actually in Debian and use it. + +import Test.QuickCheck +import GitRepo + +main = do + putStr "prop_idempotent_deencode " + quickCheck prop_idempotent_deencode From 606ed6bb3566fa86c1783e3f1c7d799a6f1be8d1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 17:08:16 -0400 Subject: [PATCH 0416/8313] better test framework --- test.hs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test.hs b/test.hs index e9ea684593..9897236176 100644 --- a/test.hs +++ b/test.hs @@ -1,8 +1,13 @@ -- TODO find a test harness that is actually in Debian and use it. import Test.QuickCheck +import Test.HUnit +import Test.HUnit.Tools + import GitRepo -main = do - putStr "prop_idempotent_deencode " - quickCheck prop_idempotent_deencode +alltests = [ + qctest "prop_idempotent_deencode" prop_idempotent_deencode + ] + +main = runVerboseTests (TestList alltests) From 0eae5b806c76b0fa3e21fbae6e5f2d9a39a04cce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 2 Nov 2010 19:04:24 -0400 Subject: [PATCH 0417/8313] broke subcommands out into separate modules --- CmdLine.hs | 201 ++++++++++++++++ Command.hs | 50 ++++ Command/Add.hs | 52 +++++ Command/Drop.hs | 50 ++++ Command/DropKey.hs | 47 ++++ Command/Fix.hs | 40 ++++ Command/FromKey.hs | 44 ++++ Command/Get.hs | 31 +++ Command/Init.hs | 42 ++++ Command/Move.hs | 131 +++++++++++ Command/SetKey.hs | 43 ++++ Command/Unannex.hs | 48 ++++ Commands.hs | 555 --------------------------------------------- git-annex.hs | 2 +- 14 files changed, 780 insertions(+), 556 deletions(-) create mode 100644 CmdLine.hs create mode 100644 Command.hs create mode 100644 Command/Add.hs create mode 100644 Command/Drop.hs create mode 100644 Command/DropKey.hs create mode 100644 Command/Fix.hs create mode 100644 Command/FromKey.hs create mode 100644 Command/Get.hs create mode 100644 Command/Init.hs create mode 100644 Command/Move.hs create mode 100644 Command/SetKey.hs create mode 100644 Command/Unannex.hs delete mode 100644 Commands.hs diff --git a/CmdLine.hs b/CmdLine.hs new file mode 100644 index 0000000000..494da2873c --- /dev/null +++ b/CmdLine.hs @@ -0,0 +1,201 @@ +{- git-annex command line + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CmdLine (parseCmd) where + +import System.Console.GetOpt +import Control.Monad.State (liftIO) +import System.Directory +import Data.String.Utils +import Control.Monad (filterM) +import Monad (when) + +import qualified GitRepo as Git +import qualified Annex +import Locations +import qualified Backend +import Types +import Core + +import Command +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey +import qualified Command.Fix +import qualified Command.Init + +data SubCommand = SubCommand { + subcmdname :: String, + subcmdparams :: String, + subcmdseek :: SubCmdSeek, + subcmddesc :: String +} +subCmds :: [SubCommand] +subCmds = [ + (SubCommand "add" path (withFilesNotInGit Command.Add.start) + "add files to annex") + , (SubCommand "get" path (withFilesInGit Command.Get.start) + "make content of annexed files available") + , (SubCommand "drop" path (withFilesInGit Command.Drop.start) + "indicate content of files not currently wanted") + , (SubCommand "move" path (withFilesInGit Command.Move.start) + "transfer content of files to/from another repository") + , (SubCommand "init" desc (withDescription Command.Init.start) + "initialize git-annex with repository description") + , (SubCommand "unannex" path (withFilesInGit Command.Unannex.start) + "undo accidential add command") + , (SubCommand "fix" path (withFilesInGit Command.Fix.start) + "fix up symlinks to point to annexed content") + , (SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) + "fix up symlinks before they are committed") + , (SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) + "adds a file using a specific key") + , (SubCommand "dropkey" key (withKeys Command.DropKey.start) + "drops annexed content for specified keys") + , (SubCommand "setkey" key (withTempFile Command.SetKey.start) + "sets annexed content for a key using a temp file") + ] + where + path = "PATH ..." + key = "KEY ..." + desc = "DESCRIPTION" + +-- Each dashed command-line option results in generation of an action +-- in the Annex monad that performs the necessary setting. +options :: [OptDescr (Annex ())] +options = [ + Option ['f'] ["force"] (NoArg (storebool "force" True)) + "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) + "allow verbose output" + , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") + "specify default key-value backend to use" + , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") + "specify from where to transfer content" + ] + where + storebool n b = Annex.flagChange n $ FlagBool b + storestring n s = Annex.flagChange n $ FlagString s + +header :: String +header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) + +{- Usage message with lists of options and subcommands. -} +usage :: String +usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs + where + cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds + showcmd c = + (subcmdname c) ++ + (pad 11 (subcmdname c)) ++ + (subcmdparams c) ++ + (pad 13 (subcmdparams c)) ++ + (subcmddesc c) + indent l = " " ++ l + pad n s = take (n - (length s)) $ repeat ' ' + +{- Prepares a list of actions to run to perform a subcommand, based on + - the parameters passed to it. -} +prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdseek = seek } state params = do + list <- Annex.eval state $ seek params + return $ map (\a -> doSubCmd a) list + +{- Runs a subcommand through the start, perform and cleanup stages -} +doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd start = do + s <- start + case (s) of + Nothing -> return True + Just perform -> do + p <- perform + case (p) of + Nothing -> do + showEndFail + return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False + +{- These functions find appropriate files or other things based on a + user's parameters. -} +withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.notInRepo repo) params + let files' = foldl (++) [] files + pairs <- Backend.chooseBackends files' + return $ map a $ filter (\(f,_) -> notState f) pairs +withFilesInGit :: SubCmdSeekStrings +withFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + return $ map a $ filter notState $ foldl (++) [] files +withFilesMissing :: SubCmdSeekStrings +withFilesMissing a params = do + files <- liftIO $ filterM missing params + return $ map a $ filter notState files + where + missing f = do + e <- doesFileExist f + return $ not e +withDescription :: SubCmdSeekStrings +withDescription a params = do + return $ [a $ unwords params] +withFilesToBeCommitted :: SubCmdSeekStrings +withFilesToBeCommitted a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] files +withKeys :: SubCmdSeekStrings +withKeys a params = return $ map a params +withTempFile :: SubCmdSeekStrings +withTempFile a params = return $ map a params + +{- filter out files from the state directory -} +notState :: FilePath -> Bool +notState f = stateLoc /= take (length stateLoc) f + +{- Parses command line and returns two lists of actions to be + - run in the Annex monad. The first actions configure it + - according to command line options, while the second actions + - handle subcommands. -} +parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) +parseCmd argv state = do + (flags, params) <- getopt + when (null params) $ error usage + case lookupCmd (params !! 0) of + [] -> error usage + [subcommand] -> do + actions <- prepSubCmd subcommand state (drop 1 params) + let configactions = map (\flag -> do + flag + return True) flags + return (configactions, actions) + _ -> error "internal error: multiple matching subcommands" + where + getopt = case getOpt Permute options argv of + (flags, params, []) -> return (flags, params) + (_, _, errs) -> ioError (userError (concat errs ++ usage)) + lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds diff --git a/Command.hs b/Command.hs new file mode 100644 index 0000000000..3d1e75e5f5 --- /dev/null +++ b/Command.hs @@ -0,0 +1,50 @@ +{- git-annex command types + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command where + +import Types +import Backend + +{- A subcommand runs in four stages. + - + - 0. The seek stage takes the parameters passed to the subcommand, + - looks through the repo to find the ones that are relevant + - to that subcommand (ie, new files to add), and generates + - a list of start stage actions. -} +type SubCmdSeek = [String] -> Annex [SubCmdStart] +{- 1. The start stage is run before anything is printed about the + - subcommand, is passed some input, and can early abort it + - if the input does not make sense. It should run quickly and + - should not modify Annex state. -} +type SubCmdStart = Annex (Maybe SubCmdPerform) +{- 2. The perform stage is run after a message is printed about the subcommand + - being run, and it should be where the bulk of the work happens. -} +type SubCmdPerform = Annex (Maybe SubCmdCleanup) +{- 3. The cleanup stage is run only if the perform stage succeeds, and it + - returns the overall success/fail of the subcommand. -} +type SubCmdCleanup = Annex Bool +{- Some helper functions are used to build up SubCmdSeek and SubCmdStart + - functions. -} +type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek +type SubCmdStartString = String -> SubCmdStart +type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek +type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart + +notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) +notAnnexed file a = do + r <- Backend.lookupFile file + case (r) of + Just _ -> return Nothing + Nothing -> a + +isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) +isAnnexed file a = do + r <- Backend.lookupFile file + case (r) of + Just v -> a v + Nothing -> return Nothing diff --git a/Command/Add.hs b/Command/Add.hs new file mode 100644 index 0000000000..825c1d8c1e --- /dev/null +++ b/Command/Add.hs @@ -0,0 +1,52 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Add where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import Utility +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- The add subcommand annexes a file, storing it in a backend, and then + - moving it into the annex directory and setting up the symlink pointing + - to its content. -} +start :: SubCmdStartBackendFile +start pair@(file, _) = notAnnexed file $ do + s <- liftIO $ getSymbolicLinkStatus file + if ((isSymbolicLink s) || (not $ isRegularFile s)) + then return Nothing + else do + showStart "add" file + return $ Just $ perform pair + +perform :: (FilePath, Maybe Backend) -> SubCmdPerform +perform (file, backend) = do + stored <- Backend.storeFileKey file backend + case (stored) of + Nothing -> return Nothing + Just (key, _) -> return $ Just $ cleanup file key + +cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup file key = do + logStatus key ValuePresent + g <- Annex.gitRepo + let dest = annexLocation g key + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile file dest + link <- calcGitLink file key + liftIO $ createSymbolicLink link file + Annex.queue "add" [] file + return True diff --git a/Command/Drop.hs b/Command/Drop.hs new file mode 100644 index 0000000000..6cdf216f41 --- /dev/null +++ b/Command/Drop.hs @@ -0,0 +1,50 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Drop where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- Indicates a file's content is not wanted anymore, and should be removed + - if it's safe to do so. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + inbackend <- Backend.hasKey key + if (not inbackend) + then return Nothing + else do + showStart "drop" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + success <- Backend.removeKey backend key + if (success) + then return $ Just $ cleanup key + else return Nothing + +cleanup :: Key -> SubCmdCleanup +cleanup key = do + logStatus key ValueMissing + inannex <- inAnnex key + if (inannex) + then do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return True + else return True + diff --git a/Command/DropKey.hs b/Command/DropKey.hs new file mode 100644 index 0000000000..bdd9b55b12 --- /dev/null +++ b/Command/DropKey.hs @@ -0,0 +1,47 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropKey where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- Drops cached content for a key. -} +start :: SubCmdStartString +start keyname = do + backends <- Backend.list + let key = genKey (backends !! 0) keyname + present <- inAnnex key + force <- Annex.flagIsSet "force" + if (not present) + then return Nothing + else if (not force) + then error "dropkey is can cause data loss; use --force if you're sure you want to do this" + else do + showStart "dropkey" keyname + return $ Just $ perform key + +perform :: Key -> SubCmdPerform +perform key = do + g <- Annex.gitRepo + let loc = annexLocation g key + liftIO $ removeFile loc + return $ Just $ cleanup key + +cleanup :: Key -> SubCmdCleanup +cleanup key = do + logStatus key ValueMissing + return True + diff --git a/Command/Fix.hs b/Command/Fix.hs new file mode 100644 index 0000000000..90257a8a53 --- /dev/null +++ b/Command/Fix.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Fix where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import Utility +import Core + +{- Fixes the symlink to an annexed file. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, _) -> do + link <- calcGitLink file key + l <- liftIO $ readSymbolicLink file + if (link == l) + then return Nothing + else do + showStart "fix" file + return $ Just $ perform file link + +perform :: FilePath -> FilePath -> SubCmdPerform +perform file link = do + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ removeFile file + liftIO $ createSymbolicLink link file + return $ Just $ cleanup file + +cleanup :: FilePath -> SubCmdCleanup +cleanup file = do + Annex.queue "add" [] file + return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs new file mode 100644 index 0000000000..3071f218f4 --- /dev/null +++ b/Command/FromKey.hs @@ -0,0 +1,44 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FromKey where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory +import Control.Monad (when, unless) + +import Command +import qualified Annex +import Utility +import qualified Backend +import Types +import Core + +{- Adds a file pointing at a manually-specified key -} +start :: SubCmdStartString +start file = do + keyname <- Annex.flagGet "key" + when (null keyname) $ error "please specify the key with --key" + backends <- Backend.list + let key = genKey (backends !! 0) keyname + + inbackend <- Backend.hasKey key + unless (inbackend) $ error $ + "key ("++keyname++") is not present in backend" + showStart "fromkey" file + return $ Just $ perform file key +perform :: FilePath -> Key -> SubCmdPerform +perform file key = do + link <- calcGitLink file key + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ createSymbolicLink link file + return $ Just $ cleanup file +cleanup :: FilePath -> SubCmdCleanup +cleanup file = do + Annex.queue "add" [] file + return True diff --git a/Command/Get.hs b/Command/Get.hs new file mode 100644 index 0000000000..1433bc8d00 --- /dev/null +++ b/Command/Get.hs @@ -0,0 +1,31 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Get where + +import Command +import qualified Backend +import Types +import Core + +{- Gets an annexed file from one of the backends. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + inannex <- inAnnex key + if (inannex) + then return Nothing + else do + showStart "get" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + ok <- getViaTmp key (Backend.retrieveKeyFile backend key) + if (ok) + then return $ Just $ return True -- no cleanup needed + else return Nothing + diff --git a/Command/Init.hs b/Command/Init.hs new file mode 100644 index 0000000000..b1e4e0e066 --- /dev/null +++ b/Command/Init.hs @@ -0,0 +1,42 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Init where + +import Control.Monad.State (liftIO) +import Control.Monad (when) + +import Command +import qualified Annex +import Core +import qualified GitRepo as Git +import UUID + +{- Stores description for the repository etc. -} +start :: SubCmdStartString +start description = do + when (null description) $ error $ + "please specify a description of this repository\n" + showStart "init" description + return $ Just $ perform description + +perform :: String -> SubCmdPerform +perform description = do + g <- Annex.gitRepo + u <- getUUID g + describeUUID u description + liftIO $ gitAttributes g + liftIO $ gitPreCommitHook g + return $ Just $ cleanup + +cleanup :: SubCmdCleanup +cleanup = do + g <- Annex.gitRepo + logfile <- uuidLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] + return True diff --git a/Command/Move.hs b/Command/Move.hs new file mode 100644 index 0000000000..cee9416222 --- /dev/null +++ b/Command/Move.hs @@ -0,0 +1,131 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Move where + +import Control.Monad.State (liftIO) +import Monad (when) + +import Command +import Command.Drop +import qualified Annex +import Locations +import LocationLog +import Types +import Core +import qualified GitRepo as Git +import qualified Remotes +import UUID + +{- Move a file either --to or --from a repository. + - + - This only operates on the cached file content; it does not involve + - moving data in the key-value backend. -} +start :: SubCmdStartString +start file = do + fromName <- Annex.flagGet "fromrepository" + toName <- Annex.flagGet "torepository" + case (fromName, toName) of + ("", "") -> error "specify either --from or --to" + ("", _) -> moveToStart file + (_ , "") -> moveFromStart file + (_ , _) -> error "only one of --from or --to can be specified" + +{- Moves the content of an annexed file to another repository, + - removing it from the current repository, and updates locationlog + - information on both. + - + - If the destination already has the content, it is still removed + - from the current repository. + - + - Note that unlike drop, this does not honor annex.numcopies. + - A file's content can be moved even if there are insufficient copies to + - allow it to be dropped. + -} +moveToStart :: SubCmdStartString +moveToStart file = isAnnexed file $ \(key, _) -> do + ishere <- inAnnex key + if (not ishere) + then return Nothing -- not here, so nothing to do + else do + showStart "move" file + return $ Just $ moveToPerform key +moveToPerform :: Key -> SubCmdPerform +moveToPerform key = do + -- checking the remote is expensive, so not done in the start step + remote <- Remotes.commandLineRemote + isthere <- Remotes.inAnnex remote key + case isthere of + Left err -> do + showNote $ show err + return Nothing + Right False -> do + Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." + let tmpfile = (annexTmpLocation remote) ++ (keyFile key) + ok <- Remotes.copyToRemote remote key tmpfile + if (ok) + then return $ Just $ moveToCleanup remote key tmpfile + else return Nothing -- failed + Right True -> return $ Just $ Command.Drop.cleanup key +moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup +moveToCleanup remote key tmpfile = do + -- Tell remote to use the transferred content. + ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", + "--backend=" ++ (backendName key), + "--key=" ++ keyName key, + tmpfile] + if ok + then do + -- Record that the key is present on the remote. + g <- Annex.gitRepo + remoteuuid <- getUUID remote + logfile <- liftIO $ logChange g key remoteuuid ValuePresent + Annex.queue "add" [] logfile + -- Cleanup on the local side is the same as done for the + -- drop subcommand. + Command.Drop.cleanup key + else return False + +{- Moves the content of an annexed file from another repository to the current + - repository and updates locationlog information on both. + - + - If the current repository already has the content, it is still removed + - from the other repository. + -} +moveFromStart :: SubCmdStartString +moveFromStart file = isAnnexed file $ \(key, _) -> do + remote <- Remotes.commandLineRemote + l <- Remotes.keyPossibilities key + if (null $ filter (\r -> Remotes.same r remote) l) + then return Nothing + else do + showStart "move" file + return $ Just $ moveFromPerform key +moveFromPerform :: Key -> SubCmdPerform +moveFromPerform key = do + remote <- Remotes.commandLineRemote + ishere <- inAnnex key + if (ishere) + then return $ Just $ moveFromCleanup remote key + else do + Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." + ok <- getViaTmp key (Remotes.copyFromRemote remote key) + if (ok) + then return $ Just $ moveFromCleanup remote key + else return Nothing -- fail +moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup +moveFromCleanup remote key = do + ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", + "--backend=" ++ (backendName key), + keyName key] + when ok $ do + -- Record locally that the key is not on the remote. + remoteuuid <- getUUID remote + g <- Annex.gitRepo + logfile <- liftIO $ logChange g key remoteuuid ValueMissing + Annex.queue "add" [] logfile + return ok diff --git a/Command/SetKey.hs b/Command/SetKey.hs new file mode 100644 index 0000000000..a5710643ec --- /dev/null +++ b/Command/SetKey.hs @@ -0,0 +1,43 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SetKey where + +import Control.Monad.State (liftIO) +import Control.Monad (when) + +import Command +import qualified Annex +import Utility +import Locations +import qualified Backend +import LocationLog +import Types +import Core + +{- Sets cached content for a key. -} +start :: SubCmdStartString +start tmpfile = do + keyname <- Annex.flagGet "key" + when (null keyname) $ error "please specify the key with --key" + backends <- Backend.list + let key = genKey (backends !! 0) keyname + showStart "setkey" tmpfile + return $ Just $ perform tmpfile key +perform :: FilePath -> Key -> SubCmdPerform +perform tmpfile key = do + g <- Annex.gitRepo + let loc = annexLocation g key + ok <- liftIO $ boolSystem "mv" [tmpfile, loc] + if (not ok) + then error "mv failed!" + else return $ Just $ cleanup key +cleanup :: Key -> SubCmdCleanup +cleanup key = do + logStatus key ValuePresent + return True + diff --git a/Command/Unannex.hs b/Command/Unannex.hs new file mode 100644 index 0000000000..5cffb2d894 --- /dev/null +++ b/Command/Unannex.hs @@ -0,0 +1,48 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unannex where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Utility +import Locations +import qualified Backend +import LocationLog +import Types +import Core +import qualified GitRepo as Git + +{- The unannex subcommand undoes an add. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + showStart "unannex" file + return $ Just $ perform file key backend + +perform :: FilePath -> Key -> Backend -> SubCmdPerform +perform file key backend = do + -- force backend to always remove + Annex.flagChange "force" $ FlagBool True + ok <- Backend.removeKey backend key + if (ok) + then return $ Just $ cleanup file key + else return Nothing + +cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup file key = do + logStatus key ValueMissing + g <- Annex.gitRepo + let src = annexLocation g key + liftIO $ removeFile file + liftIO $ Git.run g ["rm", "--quiet", file] + -- git rm deletes empty directories; put them back + liftIO $ createDirectoryIfMissing True (parentDir file) + liftIO $ renameFile src file + return True diff --git a/Commands.hs b/Commands.hs deleted file mode 100644 index 330b71ed27..0000000000 --- a/Commands.hs +++ /dev/null @@ -1,555 +0,0 @@ -{- git-annex command line - - - - Copyright 2010 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Commands (parseCmd) where - -import System.Console.GetOpt -import Control.Monad.State (liftIO) -import System.Posix.Files -import System.Directory -import Data.String.Utils -import Control.Monad (filterM) -import Monad (when, unless) - -import qualified GitRepo as Git -import qualified Annex -import Utility -import Locations -import qualified Backend -import UUID -import LocationLog -import Types -import Core -import qualified Remotes - -{- A subcommand runs in four stages. - - - - 0. The seek stage takes the parameters passed to the subcommand, - - looks through the repo to find the ones that are relevant - - to that subcommand (ie, new files to add), and generates - - a list of start stage actions. -} -type SubCmdSeek = [String] -> Annex [SubCmdStart] -{- 1. The start stage is run before anything is printed about the - - subcommand, is passed some input, and can early abort it - - if the input does not make sense. It should run quickly and - - should not modify Annex state. -} -type SubCmdStart = Annex (Maybe SubCmdPerform) -{- 2. The perform stage is run after a message is printed about the subcommand - - being run, and it should be where the bulk of the work happens. -} -type SubCmdPerform = Annex (Maybe SubCmdCleanup) -{- 3. The cleanup stage is run only if the perform stage succeeds, and it - - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex Bool -{- Some helper functions are used to build up SubCmdSeek and SubCmdStart - - functions. -} -type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdStartString = String -> SubCmdStart -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart - -data SubCommand = SubCommand { - subcmdname :: String, - subcmdparams :: String, - subcmdseek :: SubCmdSeek, - subcmddesc :: String -} -subCmds :: [SubCommand] -subCmds = [ - (SubCommand "add" path (withFilesNotInGit addStart) - "add files to annex") - , (SubCommand "get" path (withFilesInGit getStart) - "make content of annexed files available") - , (SubCommand "drop" path (withFilesInGit dropStart) - "indicate content of files not currently wanted") - , (SubCommand "move" path (withFilesInGit moveStart) - "transfer content of files to/from another repository") - , (SubCommand "init" desc (withDescription initStart) - "initialize git-annex with repository description") - , (SubCommand "unannex" path (withFilesInGit unannexStart) - "undo accidential add command") - , (SubCommand "fix" path (withFilesInGit fixStart) - "fix up symlinks to point to annexed content") - , (SubCommand "pre-commit" path (withFilesToBeCommitted fixStart) - "fix up symlinks before they are committed") - , (SubCommand "fromkey" key (withFilesMissing fromKeyStart) - "adds a file using a specific key") - , (SubCommand "dropkey" key (withKeys dropKeyStart) - "drops annexed content for specified keys") - , (SubCommand "setkey" key (withTempFile setKeyStart) - "sets annexed content for a key using a temp file") - ] - where - path = "PATH ..." - key = "KEY ..." - desc = "DESCRIPTION" - --- Each dashed command-line option results in generation of an action --- in the Annex monad that performs the necessary setting. -options :: [OptDescr (Annex ())] -options = [ - Option ['f'] ["force"] (NoArg (storebool "force" True)) - "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) - "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) - "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") - "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") - "specify from where to transfer content" - ] - where - storebool n b = Annex.flagChange n $ FlagBool b - storestring n s = Annex.flagChange n $ FlagString s - -header :: String -header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) - -{- Usage message with lists of options and subcommands. -} -usage :: String -usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs - where - cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds - showcmd c = - (subcmdname c) ++ - (pad 11 (subcmdname c)) ++ - (subcmdparams c) ++ - (pad 13 (subcmdparams c)) ++ - (subcmddesc c) - indent l = " " ++ l - pad n s = take (n - (length s)) $ repeat ' ' - -{- Prepares a list of actions to run to perform a subcommand, based on - - the parameters passed to it. -} -prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } state params = do - list <- Annex.eval state $ seek params - return $ map (\a -> doSubCmd a) list - -{- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup -doSubCmd start = do - s <- start - case (s) of - Nothing -> return True - Just perform -> do - p <- perform - case (p) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - c <- cleanup - if (c) - then do - showEndOk - return True - else do - showEndFail - return False - -{- These functions find appropriate files or other things based on a - user's parameters. -} -withFilesNotInGit :: SubCmdSeekBackendFiles -withFilesNotInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.notInRepo repo) params - let files' = foldl (++) [] files - pairs <- Backend.chooseBackends files' - return $ map a $ filter (\(f,_) -> notState f) pairs -withFilesInGit :: SubCmdSeekStrings -withFilesInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: SubCmdSeekStrings -withFilesMissing a params = do - files <- liftIO $ filterM missing params - return $ map a $ filter notState files - where - missing f = do - e <- doesFileExist f - return $ not e -withDescription :: SubCmdSeekStrings -withDescription a params = do - return $ [a $ unwords params] -withFilesToBeCommitted :: SubCmdSeekStrings -withFilesToBeCommitted a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] files -withKeys :: SubCmdSeekStrings -withKeys a params = return $ map a params -withTempFile :: SubCmdSeekStrings -withTempFile a params = return $ map a params - -{- filter out files from the state directory -} -notState :: FilePath -> Bool -notState f = stateLoc /= take (length stateLoc) f - -{- Parses command line and returns two lists of actions to be - - run in the Annex monad. The first actions configure it - - according to command line options, while the second actions - - handle subcommands. -} -parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) -parseCmd argv state = do - (flags, params) <- getopt - when (null params) $ error usage - case lookupCmd (params !! 0) of - [] -> error usage - [subcommand] -> do - actions <- prepSubCmd subcommand state (drop 1 params) - let configactions = map (\flag -> do - flag - return True) flags - return (configactions, actions) - _ -> error "internal error: multiple matching subcommands" - where - getopt = case getOpt Permute options argv of - (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds - -{- The add subcommand annexes a file, storing it in a backend, and then - - moving it into the annex directory and setting up the symlink pointing - - to its content. -} -addStart :: SubCmdStartBackendFile -addStart pair@(file, _) = notAnnexed file $ do - s <- liftIO $ getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) - then return Nothing - else do - showStart "add" file - return $ Just $ addPerform pair -addPerform :: (FilePath, Maybe Backend) -> SubCmdPerform -addPerform (file, backend) = do - stored <- Backend.storeFileKey file backend - case (stored) of - Nothing -> return Nothing - Just (key, _) -> return $ Just $ addCleanup file key -addCleanup :: FilePath -> Key -> SubCmdCleanup -addCleanup file key = do - logStatus key ValuePresent - g <- Annex.gitRepo - let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile file dest - link <- calcGitLink file key - liftIO $ createSymbolicLink link file - Annex.queue "add" [] file - return True - -{- The unannex subcommand undoes an add. -} -unannexStart :: SubCmdStartString -unannexStart file = isAnnexed file $ \(key, backend) -> do - showStart "unannex" file - return $ Just $ unannexPerform file key backend -unannexPerform :: FilePath -> Key -> Backend -> SubCmdPerform -unannexPerform file key backend = do - -- force backend to always remove - Annex.flagChange "force" $ FlagBool True - ok <- Backend.removeKey backend key - if (ok) - then return $ Just $ unannexCleanup file key - else return Nothing -unannexCleanup :: FilePath -> Key -> SubCmdCleanup -unannexCleanup file key = do - logStatus key ValueMissing - g <- Annex.gitRepo - let src = annexLocation g key - liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", file] - -- git rm deletes empty directories; put them back - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ renameFile src file - return True - -{- Gets an annexed file from one of the backends. -} -getStart :: SubCmdStartString -getStart file = isAnnexed file $ \(key, backend) -> do - inannex <- inAnnex key - if (inannex) - then return Nothing - else do - showStart "get" file - return $ Just $ getPerform key backend -getPerform :: Key -> Backend -> SubCmdPerform -getPerform key backend = do - ok <- getViaTmp key (Backend.retrieveKeyFile backend key) - if (ok) - then return $ Just $ return True -- no cleanup needed - else return Nothing - -{- Indicates a file's content is not wanted anymore, and should be removed - - if it's safe to do so. -} -dropStart :: SubCmdStartString -dropStart file = isAnnexed file $ \(key, backend) -> do - inbackend <- Backend.hasKey key - if (not inbackend) - then return Nothing - else do - showStart "drop" file - return $ Just $ dropPerform key backend -dropPerform :: Key -> Backend -> SubCmdPerform -dropPerform key backend = do - success <- Backend.removeKey backend key - if (success) - then return $ Just $ dropCleanup key - else return Nothing -dropCleanup :: Key -> SubCmdCleanup -dropCleanup key = do - logStatus key ValueMissing - inannex <- inAnnex key - if (inannex) - then do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return True - else return True - -{- Drops cached content for a key. -} -dropKeyStart :: SubCmdStartString -dropKeyStart keyname = do - backends <- Backend.list - let key = genKey (backends !! 0) keyname - present <- inAnnex key - force <- Annex.flagIsSet "force" - if (not present) - then return Nothing - else if (not force) - then error "dropkey is can cause data loss; use --force if you're sure you want to do this" - else do - showStart "dropkey" keyname - return $ Just $ dropKeyPerform key -dropKeyPerform :: Key -> SubCmdPerform -dropKeyPerform key = do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return $ Just $ dropKeyCleanup key -dropKeyCleanup :: Key -> SubCmdCleanup -dropKeyCleanup key = do - logStatus key ValueMissing - return True - -{- Sets cached content for a key. -} -setKeyStart :: SubCmdStartString -setKeyStart tmpfile = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (backends !! 0) keyname - showStart "setkey" tmpfile - return $ Just $ setKeyPerform tmpfile key -setKeyPerform :: FilePath -> Key -> SubCmdPerform -setKeyPerform tmpfile key = do - g <- Annex.gitRepo - let loc = annexLocation g key - ok <- liftIO $ boolSystem "mv" [tmpfile, loc] - if (not ok) - then error "mv failed!" - else return $ Just $ setKeyCleanup key -setKeyCleanup :: Key -> SubCmdCleanup -setKeyCleanup key = do - logStatus key ValuePresent - return True - -{- Fixes the symlink to an annexed file. -} -fixStart :: SubCmdStartString -fixStart file = isAnnexed file $ \(key, _) -> do - link <- calcGitLink file key - l <- liftIO $ readSymbolicLink file - if (link == l) - then return Nothing - else do - showStart "fix" file - return $ Just $ fixPerform file link -fixPerform :: FilePath -> FilePath -> SubCmdPerform -fixPerform file link = do - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ removeFile file - liftIO $ createSymbolicLink link file - return $ Just $ fixCleanup file -fixCleanup :: FilePath -> SubCmdCleanup -fixCleanup file = do - Annex.queue "add" [] file - return True - -{- Stores description for the repository etc. -} -initStart :: SubCmdStartString -initStart description = do - when (null description) $ error $ - "please specify a description of this repository\n" ++ usage - showStart "init" description - return $ Just $ initPerform description -initPerform :: String -> SubCmdPerform -initPerform description = do - g <- Annex.gitRepo - u <- getUUID g - describeUUID u description - liftIO $ gitAttributes g - liftIO $ gitPreCommitHook g - return $ Just $ initCleanup -initCleanup :: SubCmdCleanup -initCleanup = do - g <- Annex.gitRepo - logfile <- uuidLog - liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] - return True - -{- Adds a file pointing at a manually-specified key -} -fromKeyStart :: SubCmdStartString -fromKeyStart file = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (backends !! 0) keyname - - inbackend <- Backend.hasKey key - unless (inbackend) $ error $ - "key ("++keyname++") is not present in backend" - showStart "fromkey" file - return $ Just $ fromKeyPerform file key -fromKeyPerform :: FilePath -> Key -> SubCmdPerform -fromKeyPerform file key = do - link <- calcGitLink file key - liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ createSymbolicLink link file - return $ Just $ fromKeyCleanup file -fromKeyCleanup :: FilePath -> SubCmdCleanup -fromKeyCleanup file = do - Annex.queue "add" [] file - return True - -{- Move a file either --to or --from a repository. - - - - This only operates on the cached file content; it does not involve - - moving data in the key-value backend. -} -moveStart :: SubCmdStartString -moveStart file = do - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - case (fromName, toName) of - ("", "") -> error "specify either --from or --to" - ("", _) -> moveToStart file - (_ , "") -> moveFromStart file - (_ , _) -> error "only one of --from or --to can be specified" - -{- Moves the content of an annexed file to another repository, - - removing it from the current repository, and updates locationlog - - information on both. - - - - If the destination already has the content, it is still removed - - from the current repository. - - - - Note that unlike drop, this does not honor annex.numcopies. - - A file's content can be moved even if there are insufficient copies to - - allow it to be dropped. - -} -moveToStart :: SubCmdStartString -moveToStart file = isAnnexed file $ \(key, _) -> do - ishere <- inAnnex key - if (not ishere) - then return Nothing -- not here, so nothing to do - else do - showStart "move" file - return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdPerform -moveToPerform key = do - -- checking the remote is expensive, so not done in the start step - remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key - case isthere of - Left err -> do - showNote $ show err - return Nothing - Right False -> do - Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." - let tmpfile = (annexTmpLocation remote) ++ (keyFile key) - ok <- Remotes.copyToRemote remote key tmpfile - if (ok) - then return $ Just $ moveToCleanup remote key tmpfile - else return Nothing -- failed - Right True -> return $ Just $ dropCleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup -moveToCleanup remote key tmpfile = do - -- Tell remote to use the transferred content. - ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", - "--backend=" ++ (backendName key), - "--key=" ++ keyName key, - tmpfile] - if ok - then do - -- Record that the key is present on the remote. - g <- Annex.gitRepo - remoteuuid <- getUUID remote - logfile <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" [] logfile - -- Cleanup on the local side is the same as done for the - -- drop subcommand. - dropCleanup key - else return False - -{- Moves the content of an annexed file from another repository to the current - - repository and updates locationlog information on both. - - - - If the current repository already has the content, it is still removed - - from the other repository. - -} -moveFromStart :: SubCmdStartString -moveFromStart file = isAnnexed file $ \(key, _) -> do - remote <- Remotes.commandLineRemote - l <- Remotes.keyPossibilities key - if (null $ filter (\r -> Remotes.same r remote) l) - then return Nothing - else do - showStart "move" file - return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdPerform -moveFromPerform key = do - remote <- Remotes.commandLineRemote - ishere <- inAnnex key - if (ishere) - then return $ Just $ moveFromCleanup remote key - else do - Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." - ok <- getViaTmp key (Remotes.copyFromRemote remote key) - if (ok) - then return $ Just $ moveFromCleanup remote key - else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup -moveFromCleanup remote key = do - ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", - "--backend=" ++ (backendName key), - keyName key] - when ok $ do - -- Record locally that the key is not on the remote. - remoteuuid <- getUUID remote - g <- Annex.gitRepo - logfile <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" [] logfile - return ok - --- helpers -notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) -notAnnexed file a = do - r <- Backend.lookupFile file - case (r) of - Just _ -> return Nothing - Nothing -> a -isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) -isAnnexed file a = do - r <- Backend.lookupFile file - case (r) of - Just v -> a v - Nothing -> return Nothing diff --git a/git-annex.hs b/git-annex.hs index 370c22a1ef..098ccac2d4 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -9,7 +9,7 @@ import System.Environment import qualified Annex import Core -import Commands +import CmdLine import qualified GitRepo as Git import BackendList From df4b461df1bf70016ffa02d08e822067f8544a63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Nov 2010 13:28:49 -0400 Subject: [PATCH 0418/8313] refactor --- CmdLine.hs | 35 ----------------------------------- Command.hs | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 494da2873c..98bdab12fb 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -33,12 +33,6 @@ import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init -data SubCommand = SubCommand { - subcmdname :: String, - subcmdparams :: String, - subcmdseek :: SubCmdSeek, - subcmddesc :: String -} subCmds :: [SubCommand] subCmds = [ (SubCommand "add" path (withFilesNotInGit Command.Add.start) @@ -109,35 +103,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = take (n - (length s)) $ repeat ' ' -{- Prepares a list of actions to run to perform a subcommand, based on - - the parameters passed to it. -} -prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } state params = do - list <- Annex.eval state $ seek params - return $ map (\a -> doSubCmd a) list - -{- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup -doSubCmd start = do - s <- start - case (s) of - Nothing -> return True - Just perform -> do - p <- perform - case (p) of - Nothing -> do - showEndFail - return False - Just cleanup -> do - c <- cleanup - if (c) - then do - showEndOk - return True - else do - showEndFail - return False - {- These functions find appropriate files or other things based on a user's parameters. -} withFilesNotInGit :: SubCmdSeekBackendFiles diff --git a/Command.hs b/Command.hs index 3d1e75e5f5..47c73370f3 100644 --- a/Command.hs +++ b/Command.hs @@ -8,7 +8,9 @@ module Command where import Types -import Backend +import qualified Backend +import Core +import qualified Annex {- A subcommand runs in four stages. - @@ -35,6 +37,42 @@ type SubCmdStartString = String -> SubCmdStart type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart +data SubCommand = SubCommand { + subcmdname :: String, + subcmdparams :: String, + subcmdseek :: SubCmdSeek, + subcmddesc :: String +} + +{- Prepares a list of actions to run to perform a subcommand, based on + - the parameters passed to it. -} +prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] +prepSubCmd SubCommand { subcmdseek = seek } state params = do + list <- Annex.eval state $ seek params + return $ map (\a -> doSubCmd a) list + +{- Runs a subcommand through the start, perform and cleanup stages -} +doSubCmd :: SubCmdStart -> SubCmdCleanup +doSubCmd start = do + s <- start + case (s) of + Nothing -> return True + Just perform -> do + p <- perform + case (p) of + Nothing -> do + showEndFail + return False + Just cleanup -> do + c <- cleanup + if (c) + then do + showEndOk + return True + else do + showEndFail + return False + notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) notAnnexed file a = do r <- Backend.lookupFile file From cc4794ce85f8e8e511a4aadb62db53bfff35ca8d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Nov 2010 13:37:06 -0400 Subject: [PATCH 0419/8313] support subcommands that take no params --- CmdLine.hs | 10 ++++++++-- Command.hs | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 98bdab12fb..e03c4d4c6d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -32,6 +32,7 @@ import qualified Command.DropKey import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init +import qualified Command.Fsck subCmds :: [SubCommand] subCmds = [ @@ -47,8 +48,6 @@ subCmds = [ "initialize git-annex with repository description") , (SubCommand "unannex" path (withFilesInGit Command.Unannex.start) "undo accidential add command") - , (SubCommand "fix" path (withFilesInGit Command.Fix.start) - "fix up symlinks to point to annexed content") , (SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) "fix up symlinks before they are committed") , (SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) @@ -57,11 +56,16 @@ subCmds = [ "drops annexed content for specified keys") , (SubCommand "setkey" key (withTempFile Command.SetKey.start) "sets annexed content for a key using a temp file") + , (SubCommand "fix" path (withFilesInGit Command.Fix.start) + "fix up symlinks to point to annexed content") + , (SubCommand "fsck" nothing (withNothing Command.Fsck.start) + "check annex for problems") ] where path = "PATH ..." key = "KEY ..." desc = "DESCRIPTION" + nothing = "" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. @@ -137,6 +141,8 @@ withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params +withNothing :: SubCmdSeekNothing +withNothing a params = return [a] {- filter out files from the state directory -} notState :: FilePath -> Bool diff --git a/Command.hs b/Command.hs index 47c73370f3..d557651aa3 100644 --- a/Command.hs +++ b/Command.hs @@ -36,6 +36,7 @@ type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek type SubCmdStartString = String -> SubCmdStart type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart +type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek data SubCommand = SubCommand { subcmdname :: String, From 6b80356f6de05efef1f14fd2af9835cf5abe69a0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 4 Nov 2010 13:40:00 -0400 Subject: [PATCH 0420/8313] fixes --- CmdLine.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index e03c4d4c6d..7aaa1c842e 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,7 +10,6 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Directory -import Data.String.Utils import Control.Monad (filterM) import Monad (when) @@ -19,7 +18,6 @@ import qualified Annex import Locations import qualified Backend import Types -import Core import Command import qualified Command.Add @@ -91,7 +89,7 @@ options = [ storestring n s = Annex.flagChange n $ FlagString s header :: String -header = "Usage: git-annex " ++ (join "|" $ map subcmdname subCmds) +header = "Usage: git-annex subcommand [option ..]" {- Usage message with lists of options and subcommands. -} usage :: String @@ -142,7 +140,7 @@ withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params withNothing :: SubCmdSeekNothing -withNothing a params = return [a] +withNothing a _ = return [a] {- filter out files from the state directory -} notState :: FilePath -> Bool From 016b6a59e7187ead0ed630699c85d0fec729a30d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 6 Nov 2010 17:06:19 -0400 Subject: [PATCH 0421/8313] add fsck subcommand (stub) --- CmdLine.hs | 72 ++++++++++++++++++++++------------------------ Command.hs | 2 +- Command/Fsck.hs | 39 +++++++++++++++++++++++++ debian/changelog | 1 + doc/git-annex.mdwn | 5 ++++ 5 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 Command/Fsck.hs diff --git a/CmdLine.hs b/CmdLine.hs index 7aaa1c842e..3823c72476 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,8 +10,7 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Directory -import Control.Monad (filterM) -import Monad (when) +import Control.Monad (filterM, when) import qualified GitRepo as Git import qualified Annex @@ -33,31 +32,31 @@ import qualified Command.Init import qualified Command.Fsck subCmds :: [SubCommand] -subCmds = [ - (SubCommand "add" path (withFilesNotInGit Command.Add.start) - "add files to annex") - , (SubCommand "get" path (withFilesInGit Command.Get.start) - "make content of annexed files available") - , (SubCommand "drop" path (withFilesInGit Command.Drop.start) - "indicate content of files not currently wanted") - , (SubCommand "move" path (withFilesInGit Command.Move.start) - "transfer content of files to/from another repository") - , (SubCommand "init" desc (withDescription Command.Init.start) - "initialize git-annex with repository description") - , (SubCommand "unannex" path (withFilesInGit Command.Unannex.start) - "undo accidential add command") - , (SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) - "fix up symlinks before they are committed") - , (SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) - "adds a file using a specific key") - , (SubCommand "dropkey" key (withKeys Command.DropKey.start) - "drops annexed content for specified keys") - , (SubCommand "setkey" key (withTempFile Command.SetKey.start) - "sets annexed content for a key using a temp file") - , (SubCommand "fix" path (withFilesInGit Command.Fix.start) - "fix up symlinks to point to annexed content") - , (SubCommand "fsck" nothing (withNothing Command.Fsck.start) - "check annex for problems") +subCmds = + [ SubCommand "add" path (withFilesNotInGit Command.Add.start) + "add files to annex" + , SubCommand "get" path (withFilesInGit Command.Get.start) + "make content of annexed files available" + , SubCommand "drop" path (withFilesInGit Command.Drop.start) + "indicate content of files not currently wanted" + , SubCommand "move" path (withFilesInGit Command.Move.start) + "transfer content of files to/from another repository" + , SubCommand "init" desc (withDescription Command.Init.start) + "initialize git-annex with repository description" + , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) + "undo accidential add command" + , SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) + "fix up symlinks before they are committed" + , SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) + "adds a file using a specific key" + , SubCommand "dropkey" key (withKeys Command.DropKey.start) + "drops annexed content for specified keys" + , SubCommand "setkey" key (withTempFile Command.SetKey.start) + "sets annexed content for a key using a temp file" + , SubCommand "fix" path (withFilesInGit Command.Fix.start) + "fix up symlinks to point to annexed content" + , SubCommand "fsck" nothing (withNothing Command.Fsck.start) + "check annex for problems" ] where path = "PATH ..." @@ -95,15 +94,15 @@ header = "Usage: git-annex subcommand [option ..]" usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where - cmddescs = unlines $ map (\c -> indent $ showcmd c) subCmds + cmddescs = unlines $ map (indent . showcmd) subCmds showcmd c = - (subcmdname c) ++ - (pad 11 (subcmdname c)) ++ - (subcmdparams c) ++ - (pad 13 (subcmdparams c)) ++ - (subcmddesc c) + subcmdname c ++ + pad 11 (subcmdname c) ++ + subcmdparams c ++ + pad 13 (subcmdparams c) ++ + subcmddesc c indent l = " " ++ l - pad n s = take (n - (length s)) $ repeat ' ' + pad n s = replicate (n - length s) ' ' {- These functions find appropriate files or other things based on a user's parameters. -} @@ -128,8 +127,7 @@ withFilesMissing a params = do e <- doesFileExist f return $ not e withDescription :: SubCmdSeekStrings -withDescription a params = do - return $ [a $ unwords params] +withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo @@ -154,7 +152,7 @@ parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) parseCmd argv state = do (flags, params) <- getopt when (null params) $ error usage - case lookupCmd (params !! 0) of + case lookupCmd (head params) of [] -> error usage [subcommand] -> do actions <- prepSubCmd subcommand state (drop 1 params) diff --git a/Command.hs b/Command.hs index d557651aa3..a0e3280d6b 100644 --- a/Command.hs +++ b/Command.hs @@ -50,7 +50,7 @@ data SubCommand = SubCommand { prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] prepSubCmd SubCommand { subcmdseek = seek } state params = do list <- Annex.eval state $ seek params - return $ map (\a -> doSubCmd a) list + return $ map doSubCmd list {- Runs a subcommand through the start, perform and cleanup stages -} doSubCmd :: SubCmdStart -> SubCmdCleanup diff --git a/Command/Fsck.hs b/Command/Fsck.hs new file mode 100644 index 0000000000..bd5a9ad7f0 --- /dev/null +++ b/Command/Fsck.hs @@ -0,0 +1,39 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Fsck where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import Types +import Utility +import Core + +{- Checks the whole annex for problems. -} +start :: SubCmdStart +start = do + showStart "fsck" "" + return $ Just perform + +perform :: SubCmdPerform +perform = do + ok <- checkUnused + if (ok) + then return $ Just $ return True + else do + showLongNote "Possible problems detected." + return Nothing + +checkUnused :: Annex Bool +checkUnused = do + showNote "checking for unused data..." + -- TODO + return False diff --git a/debian/changelog b/debian/changelog index b433ec62fd..ae68f657b9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -13,6 +13,7 @@ git-annex (0.03) UNRELEASED; urgency=low via gitattributes. * In .gitattributes, the git-annex-backend attribute can be set to the names of backends to use when adding different types of files. + * Add fsck subcommand. -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index bbd7e8cab1..856b474e05 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -141,6 +141,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex setkey --key=1287765018:3 /tmp/file +* fsck + + This subcommand checks the whole annex for consistency, and warns + about any problems found. + # OPTIONS * --force From a3519c365feec45b5ab1236c7610863678b868a0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 6 Nov 2010 17:07:11 -0400 Subject: [PATCH 0422/8313] hlinted a few files --- Annex.hs | 4 ++-- Backend.hs | 13 ++++++------- Core.hs | 14 +++++++------- GitQueue.hs | 4 ++-- GitRepo.hs | 31 +++++++++++++++---------------- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/Annex.hs b/Annex.hs index 303881fa09..e86e1967e3 100644 --- a/Annex.hs +++ b/Annex.hs @@ -50,9 +50,9 @@ new gitrepo allbackends = do {- performs an action in the Annex monad -} run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) -run state action = runStateT (action) state +run state action = runStateT action state eval :: AnnexState -> StateT AnnexState IO a -> IO a -eval state action = evalStateT (action) state +eval state action = evalStateT action state {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo diff --git a/Backend.hs b/Backend.hs index e1d0e0a686..e2c8a43b6a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -53,7 +53,7 @@ list = do let l' = if (not $ null backendflag) then (lookupBackendName bs backendflag):defaults else defaults - Annex.backendsChange $ l' + Annex.backendsChange l' return l' where parseBackendList bs s = @@ -71,7 +71,7 @@ maybeLookupBackendName :: [Backend] -> String -> Maybe Backend maybeLookupBackendName bs s = if ((length matches) /= 1) then Nothing - else Just $ matches !! 0 + else Just $ head matches where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} @@ -88,14 +88,13 @@ storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend storeFileKey' [] _ _ = return Nothing storeFileKey' (b:bs) file relfile = do result <- (Internals.getKey b) relfile - case (result) of + case result of Nothing -> nextbackend Just key -> do stored <- (Internals.storeFileKey b) file key if (not stored) then nextbackend - else do - return $ Just (key, b) + else return $ Just (key, b) where nextbackend = storeFileKey' bs file relfile @@ -127,8 +126,8 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey bs l = do - case maybeLookupBackendName bs $ bname of + makekey bs l = + case maybeLookupBackendName bs bname of Nothing -> do unless (null kname || null bname) $ warning skip diff --git a/Core.hs b/Core.hs index 327a9af483..f34b2ebbeb 100644 --- a/Core.hs +++ b/Core.hs @@ -13,7 +13,7 @@ import System.Directory import Control.Monad.State (liftIO) import System.Path import Data.String.Utils -import Monad (when, unless) +import Control.Monad (when, unless) import Types import Locations @@ -40,7 +40,7 @@ tryRun' state errnum (a:as) = do Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as tryRun' _ errnum [] = - when (errnum > 0) $ error $ (show errnum) ++ " failed" + when (errnum > 0) $ error $ show errnum ++ " failed" {- Sets up a git repo for git-annex. -} startup :: Annex Bool @@ -63,7 +63,7 @@ shutdown = do -- the tmp directory itself let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp - when (exists) $ liftIO $ removeDirectoryRecursive $ tmp + when (exists) $ liftIO $ removeDirectoryRecursive tmp liftIO $ createDirectoryIfMissing True tmp return True @@ -93,7 +93,7 @@ gitAttributes repo = do {- set up a git pre-commit hook, if one is not already present -} gitPreCommitHook :: Git.Repo -> IO () gitPreCommitHook repo = do - let hook = (Git.workTree repo) ++ "/" ++ (Git.gitDir repo) ++ + let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit" exists <- doesFileExist hook if (exists) @@ -120,7 +120,7 @@ calcGitLink file key = do let absfile = case (absNormPath cwd file) of Just f -> f Nothing -> error $ "unable to normalize " ++ file - return $ (relPathDirToDir (parentDir absfile) (Git.workTree g)) ++ + return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ annexLocationRelative key {- Updates the LocationLog when a key's presence changes. -} @@ -138,7 +138,7 @@ getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool getViaTmp key action = do g <- Annex.gitRepo let dest = annexLocation g key - let tmp = (annexTmpLocation g) ++ (keyFile key) + let tmp = annexTmpLocation g ++ keyFile key liftIO $ createDirectoryIfMissing True (parentDir tmp) success <- action tmp if (success) @@ -165,7 +165,7 @@ showNote s = verbose $ do liftIO $ putStr $ "(" ++ s ++ ") " liftIO $ hFlush stdout showProgress :: Annex () -showProgress = verbose $ liftIO $ putStr $ "\n" +showProgress = verbose $ liftIO $ putStr "\n" showLongNote :: String -> Annex () showLongNote s = verbose $ do liftIO $ putStr $ "\n" ++ indented diff --git a/GitQueue.hs b/GitQueue.hs index 632d1d3910..2d44a8f108 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -16,7 +16,7 @@ import qualified Data.Map as M import System.IO import System.Cmd.Utils import Data.String.Utils -import Monad (unless) +import Control.Monad (unless) import qualified GitRepo as Git @@ -57,5 +57,5 @@ runAction repo action files = do where runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs gitcmd = ["git"] ++ Git.gitCommandLine repo - ((getSubcommand action):(getParams action)) + (getSubcommand action:getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index 7d5291ff1e..da86c225e2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -43,8 +43,8 @@ module GitRepo ( prop_idempotent_deencode ) where -import Monad (unless) -import Directory +import Control.Monad (unless) +import System.Directory import System.Posix.Directory import System.Path import System.Cmd.Utils @@ -53,11 +53,11 @@ import Data.String.Utils import System.IO import qualified Data.Map as Map hiding (map, split) import Network.URI -import Maybe -import Char -import Text.Printf +import Data.Maybe +import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) +import Text.Printf import Utility @@ -127,31 +127,31 @@ assertLocal :: Repo -> a -> a assertLocal repo action = if (not $ repoIsUrl repo) then action - else error $ "acting on URL git repo " ++ (repoDescribe repo) ++ + else error $ "acting on URL git repo " ++ repoDescribe repo ++ " not supported" assertUrl :: Repo -> a -> a assertUrl repo action = if (repoIsUrl repo) then action - else error $ "acting on local git repo " ++ (repoDescribe repo) ++ + else error $ "acting on local git repo " ++ repoDescribe repo ++ " not supported" assertSsh :: Repo -> a -> a assertSsh repo action = if (repoIsSsh repo) then action - else error $ "unsupported url in repo " ++ (repoDescribe repo) + else error $ "unsupported url in repo " ++ repoDescribe repo bare :: Repo -> Bool bare repo = case Map.lookup "core.bare" $ config repo of Just v -> configTrue v Nothing -> error $ "it is not known if git repo " ++ - (repoDescribe repo) ++ + repoDescribe repo ++ " is a bare repository; config not read" {- Path to a repository's gitattributes file. -} attributes :: Repo -> String attributes repo - | bare repo = (workTree repo) ++ "/info/.gitattributes" - | otherwise = (workTree repo) ++ "/.gitattributes" + | bare repo = workTree repo ++ "/info/.gitattributes" + | otherwise = workTree repo ++ "/.gitattributes" {- Path to a repository's .git directory, relative to its workTree. -} gitDir :: Repo -> String @@ -176,7 +176,7 @@ relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile -- will be substring of file absrepo = case (absNormPath "/" d) of Just f -> f ++ "/" - Nothing -> error $ "bad repo" ++ (repoDescribe repo) + Nothing -> error $ "bad repo" ++ repoDescribe repo absfile = case (secureAbsNormPath absrepo file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo @@ -185,7 +185,7 @@ relative repo _ = assertLocal repo $ error "internal" {- Hostname of an URL repo. (May include a username and/or port too.) -} urlHost :: Repo -> String urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a - where a = fromJust $ uriAuthority $ u + where a = fromJust $ uriAuthority u urlHost repo = assertUrl repo $ error "internal" {- Path of an URL repo. -} @@ -204,14 +204,13 @@ gitCommandLine repo _ = assertLocal repo $ error "internal" run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo params) - unless (ok) $ error $ "git " ++ (show params) ++ " failed" + unless (ok) $ error $ "git " ++ show params ++ " failed" {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String pipeRead repo params = assertLocal repo $ do pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - ret <- hGetContentsStrict h - return ret + hGetContentsStrict h {- Like pipeRead, but does not read output strictly; recommended - for git commands that produce a lot of output that will be processed From b1e26b19c6055b8c635a158068a2ad0188f17d17 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 00:26:44 +0000 Subject: [PATCH 0423/8313] idea --- debian/changelog | 1 - doc/todo/auto_remotes.mdwn | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 doc/todo/auto_remotes.mdwn diff --git a/debian/changelog b/debian/changelog index ae68f657b9..1d3c6427f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,6 @@ git-annex (0.03) UNRELEASED; urgency=low * Add --verbose * Fix SIGINT handling. * Fix handling of files with unusual characters in their name. - * Support building with Debian stable's ghc. * Fixed memory leak; git-annex no longer reads the whole file list from git before starting, and will be much faster with large repos. * Fix crash on unknown symlinks. diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn new file mode 100644 index 0000000000..4976a2a778 --- /dev/null +++ b/doc/todo/auto_remotes.mdwn @@ -0,0 +1,20 @@ +It should be possible for clones to learn about how to contact +each other without remotes needing to always be explicitly set +up. Say that `.git-annex/remote.log` is maintained by git-annex +to contain: + + UUID hostname URI + +The URI comes from configured remotes and maybe from +`file://$(pwd)` for the current repo. This format will merge +without conflicts or data loss. + +Then when content is belived to be in a UUID, and no +configured remote has it, the remote.log can be consulted and +URIs that look likely tried. (file:// ones if the hostname +is the same (or maybe always -- a removable drive might tend +to be mounted at the same location on different hosts), +otherwise ssh:// ones.) + +Question: When should git-annex update the remote.log? +(Not just on init.) Whenever it reads in a repo's remotes? From 8156af90edc27c041bd1c22d8401aff21f4b8947 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 00:28:17 +0000 Subject: [PATCH 0424/8313] update --- doc/todo/auto_remotes.mdwn | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn index 4976a2a778..2ac4790af9 100644 --- a/doc/todo/auto_remotes.mdwn +++ b/doc/todo/auto_remotes.mdwn @@ -6,8 +6,9 @@ to contain: UUID hostname URI The URI comes from configured remotes and maybe from -`file://$(pwd)` for the current repo. This format will merge -without conflicts or data loss. +`file://$(pwd)`, or even `ssh://$(hostname -f)` +for the current repo. This format will merge without +conflicts or data loss. Then when content is belived to be in a UUID, and no configured remote has it, the remote.log can be consulted and @@ -17,4 +18,4 @@ to be mounted at the same location on different hosts), otherwise ssh:// ones.) Question: When should git-annex update the remote.log? -(Not just on init.) Whenever it reads in a repo's remotes? +(If not just on init.) Whenever it reads in a repo's remotes? From 55b92860ceb099614ac9ebe4c37e92b57ad6a430 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 6 Nov 2010 21:12:45 -0400 Subject: [PATCH 0425/8313] bigfix: doubled shell escape --- Remotes.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 7bb1bcd222..280543968c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -94,8 +94,7 @@ inAnnex r key = do Annex.eval a (Core.inAnnex key) checkremote = do Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- runCmd r "test" - [ "-e", (shellEscape $ annexLocation r key)] + inannex <- runCmd r "test" ["-e", annexLocation r key] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. return $ Right inannex From ea8ccaa3d5416044ca69e4a3dcb7b879aec0ff4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 17:26:21 -0400 Subject: [PATCH 0426/8313] rough in fsck --- Command/Fsck.hs | 46 ++++++++++++++++++++++++++++++++++++++-------- TypeInternals.hs | 2 +- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index bd5a9ad7f0..c86f30ff80 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,15 +7,10 @@ module Command.Fsck where -import Control.Monad.State (liftIO) -import System.Posix.Files -import System.Directory - import Command -import qualified Annex import Types -import Utility import Core +import qualified Data.Map as M {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -35,5 +30,40 @@ perform = do checkUnused :: Annex Bool checkUnused = do showNote "checking for unused data..." - -- TODO - return False + unused <- unusedKeys + if (null unused) + then return True + else do + showLongNote $ w unused + return False + where + w u = unlines $ [ + "Some annexed data is no longer pointed to by any file.", + "If this data is no longer needed, it can be removed using git-annex dropkey:" + ] ++ map show u + +unusedKeys :: Annex [Key] +unusedKeys = do + present <- getKeysPresent + referenced <- getKeysReferenced + + -- Constructing a single map, of the set that tends to be smaller, + -- appears more efficient in both memory and CPU than constructing + -- and taking the M.difference of two maps. + let present_m = existsMap present + let unused_m = remove referenced present_m + return $ M.keys unused_m + where + remove [] m = m + remove (x:xs) m = remove xs $ M.delete x m + +existsMap :: Ord k => [k] -> M.Map k Int +existsMap l = M.fromList $ map (\k -> (k, 1)) l + +getKeysPresent :: Annex [Key] +getKeysPresent = do + return [] + +getKeysReferenced :: Annex [Key] +getKeysReferenced = do + return [] diff --git a/TypeInternals.hs b/TypeInternals.hs index 46c92cb59b..4b5cff9d9f 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -39,7 +39,7 @@ type Annex = StateT AnnexState IO -- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String -data Key = Key (BackendName, KeyName) deriving (Eq) +data Key = Key (BackendName, KeyName) deriving (Eq, Ord) -- constructs a key in a backend genKey :: Backend -> KeyName -> Key From 316264f3e8f6dbcbd2c3752566d7d754dbfe9994 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 17:36:24 -0400 Subject: [PATCH 0427/8313] add annexDir --- Locations.hs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Locations.hs b/Locations.hs index a296f8d876..951924c407 100644 --- a/Locations.hs +++ b/Locations.hs @@ -12,7 +12,8 @@ module Locations ( fileKey, annexLocation, annexLocationRelative, - annexTmpLocation + annexTmpLocation, + annexDir ) where import Data.String.Utils @@ -43,11 +44,15 @@ annexLocation r key = annexLocationRelative :: Key -> FilePath annexLocationRelative key = ".git/annex/" ++ (keyFile key) -{- .git-annex/tmp is used for temp files +{- The annex directory of a repository. - - Note: Assumes repo is NOT bare. -} +annexDir :: Git.Repo -> FilePath +annexDir r = Git.workTree r ++ "/.git/annex" + +{- .git-annex/tmp is used for temp files -} annexTmpLocation :: Git.Repo -> FilePath -annexTmpLocation r = (Git.workTree r) ++ "/.git/annex/tmp/" +annexTmpLocation r = annexDir r ++ "/tmp/" {- Converts a key into a filename fragment. - From 009873e0eb296b6f373f9f2d847659038a1d2bde Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:22:25 -0400 Subject: [PATCH 0428/8313] fsck works --- Backend.hs | 2 +- Command/Fsck.hs | 33 ++++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/Backend.hs b/Backend.hs index e2c8a43b6a..456a98bd41 100644 --- a/Backend.hs +++ b/Backend.hs @@ -107,7 +107,7 @@ retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest removeKey :: Backend -> Key -> Annex Bool removeKey backend key = (Internals.removeKey backend) key -{- Checks if a backend has its key. -} +{- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool hasKey key = do bs <- Annex.supportedBackends diff --git a/Command/Fsck.hs b/Command/Fsck.hs index c86f30ff80..785aecd8af 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,10 +7,20 @@ module Command.Fsck where +import qualified Data.Map as M +import System.Directory +import System.Posix.Files +import Monad (filterM) +import Control.Monad.State (liftIO) +import Data.Maybe + import Command import Types import Core -import qualified Data.Map as M +import Locations +import qualified Annex +import qualified GitRepo as Git +import qualified Backend {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -38,10 +48,12 @@ checkUnused = do return False where w u = unlines $ [ - "Some annexed data is no longer pointed to by any file.", + "Some annexed data is no longer pointed to by any files in the repository.", "If this data is no longer needed, it can be removed using git-annex dropkey:" - ] ++ map show u + ] ++ map (\k -> " " ++ show k) u +{- Finds keys whose content is present, but that do not seem to be used + - by any files in the git repo. -} unusedKeys :: Annex [Key] unusedKeys = do present <- getKeysPresent @@ -62,8 +74,19 @@ existsMap l = M.fromList $ map (\k -> (k, 1)) l getKeysPresent :: Annex [Key] getKeysPresent = do - return [] + g <- Annex.gitRepo + let top = annexDir g + contents <- liftIO $ getDirectoryContents top + files <- liftIO $ filterM (isreg top) contents + return $ map fileKey files + where + isreg top f = do + s <- getFileStatus $ top ++ "/" ++ f + return $ isRegularFile s getKeysReferenced :: Annex [Key] getKeysReferenced = do - return [] + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g $ Git.workTree g + keypairs <- mapM Backend.lookupFile files + return $ map fst $ catMaybes keypairs From bbee90cd557f0e1acd3744a120bf5251928b7056 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:23:25 -0400 Subject: [PATCH 0429/8313] update --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1d3c6427f6..ccfaa0d074 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,7 +12,8 @@ git-annex (0.03) UNRELEASED; urgency=low via gitattributes. * In .gitattributes, the git-annex-backend attribute can be set to the names of backends to use when adding different types of files. - * Add fsck subcommand. + * Add fsck subcommand. (For now it only finds unused key contents in the + annex.) -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 From 05d321ace65291322e0393b1d931f29c5058f5c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:24:44 -0400 Subject: [PATCH 0430/8313] done --- doc/todo/fsck.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn index 308a1cb63a..6126154e9f 100644 --- a/doc/todo/fsck.mdwn +++ b/doc/todo/fsck.mdwn @@ -1 +1,3 @@ add a git annex fsck that finds keys that have no referring file + +[[done]] From 8f949b164404e70be56d283280045ba4c72426d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:30:21 -0400 Subject: [PATCH 0431/8313] releasing version 0.03 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index ccfaa0d074..f9cec02c44 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.03) UNRELEASED; urgency=low +git-annex (0.03) unstable; urgency=low * Fix support for file:// remotes. * Add --verbose @@ -15,7 +15,7 @@ git-annex (0.03) UNRELEASED; urgency=low * Add fsck subcommand. (For now it only finds unused key contents in the annex.) - -- Joey Hess Thu, 28 Oct 2010 13:46:59 -0400 + -- Joey Hess Sun, 07 Nov 2010 18:26:04 -0400 git-annex (0.02) unstable; urgency=low From 714619d9e834a3a04340b164255ff2c92f654228 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 18:30:38 -0400 Subject: [PATCH 0432/8313] add news item for git-annex 0.03 --- doc/news/version_0.03.mdwn | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/news/version_0.03.mdwn diff --git a/doc/news/version_0.03.mdwn b/doc/news/version_0.03.mdwn new file mode 100644 index 0000000000..b22aad91af --- /dev/null +++ b/doc/news/version_0.03.mdwn @@ -0,0 +1,16 @@ +git-annex 0.03 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix support for file:// remotes. + * Add --verbose + * Fix SIGINT handling. + * Fix handling of files with unusual characters in their name. + * Fixed memory leak; git-annex no longer reads the whole file list + from git before starting, and will be much faster with large repos. + * Fix crash on unknown symlinks. + * Added remote.annex-scp-options and remote.annex-ssh-options. + * The backends to use when adding different sets of files can be configured + via gitattributes. + * In .gitattributes, the git-annex-backend attribute can be set to the + names of backends to use when adding different types of files. + * Add fsck subcommand. (For now it only finds unused key contents in the + annex.)"""]] \ No newline at end of file From 377bf24d9a951186b374cd7a3f920b6bc9deb8f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 7 Nov 2010 21:02:25 -0400 Subject: [PATCH 0433/8313] documentation for checkout --- debian/changelog | 8 ++++++++ doc/backends.mdwn | 3 +-- doc/git-annex.mdwn | 14 +++++++++++++- doc/todo/backendSHA1.mdwn | 2 ++ doc/todo/checkout.mdwn | 18 ++++++++++++++++++ doc/walkthrough.mdwn | 34 ++++++++++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index f9cec02c44..9205dbe06f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (0.04) UNRELEASED; urgency=low + + * Add checkout subcommand, which allows checking out file content + in preparation of changing it. + * Add uncheckout subcommand. + + -- Joey Hess Sun, 07 Nov 2010 21:01:29 -0400 + git-annex (0.03) unstable; urgency=low * Fix support for file:// remotes. diff --git a/doc/backends.mdwn b/doc/backends.mdwn index dc359174ae..69c17649a4 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -17,8 +17,7 @@ can use different backends for different files. * `SHA1` -- This backend stores the file's content in `.git/annex/`, with a name based on its sha1 checksum. This backend allows modifications of files to be tracked. Its need to generate checksums - can make it slower for large files. **Warning** this backend is not ready - for use. + can make it slower for large files. * `URL` -- This backend downloads the file's content from an external URL. The `annex.backends` git-config setting can be used to list the backends diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 856b474e05..80680820fa 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -81,6 +81,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git-annex may refuse to drop content if the backend does not think it is safe to do so, typically because of the setting of annex.numcopies. +* checkout [path ...] + + Checks out annexed files so they can be modified. This replaces the + symlink for each specified file with a copy of the file content. + When you `git commit`, the file, the new content is injected back into + the annex. + * move [path ...] When used with the --to option, moves the content of annexed files from @@ -95,6 +102,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. and sets up `.gitattributes` and the pre-commit hook. This is an optional, but recommended step. +* uncheckout [path ...] + + Use this to undo a checkout command if you don't want to modify + the checked out files, or have made modifications you want to discard. + * unannex [path ...] Use this to undo an accidental add command. This is not the command you @@ -110,7 +122,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * pre-commit [path ...] Fixes up symlinks that are staged as part of a commit, to ensure they - point to annexed content. + point to annexed content. Also handles committing checked-out files. This is meant to be called from git's pre-commit hook. `git annex init` automatically creates a pre-commit hook using this. diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn index fa9728af64..44df406dea 100644 --- a/doc/todo/backendSHA1.mdwn +++ b/doc/todo/backendSHA1.mdwn @@ -3,3 +3,5 @@ This backend is not finished. In particular, while files can be added using it, git-annex will not notice when their content changes, and will not create a new key for the new sha1 of the net content. + +[[done]]; use checkout subcommand diff --git a/doc/todo/checkout.mdwn b/doc/todo/checkout.mdwn index 70d31a5ff5..50da2d62e1 100644 --- a/doc/todo/checkout.mdwn +++ b/doc/todo/checkout.mdwn @@ -3,3 +3,21 @@ file's content, with a copy of the file. Once you've checked a file out, you can edit it, and `git commit` it. On commit, git-annex will detect if the file has been changed, and if it has, `add` its content to the annex. + +> Internally, this will need to store the original symlink to the file, in +> `.git/annex/checkedout/$filename`. +> +> * git-annex uncheckout moves that back +> * git-annex pre-commit hook checks each file being committed to see if +> it has a symlink there, and if so, removes the symlink and adds the new +> content to the annex. +> +> And it seems the file content should be copied, not moved or hard linked: +> +> * Makes sure other annexes can find it if transferring it from +> this annex. +> * Ensures it's always available for uncheckout. +> * Avoids the last copy of a file's content being lost when +> the checked out file is modified. + +[[done]] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ab0067470d..cb564fa979 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -190,6 +190,26 @@ makes it very easy. WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok +## modifying annexed files + +Normally, the content of files in the annex cannot be modified. +In order to modify a file, it should first be checked out: + + # git annex checkout my_cool_big_file + checkout my_cool_big_file (copying...) ok + +Checking a file out replaces the symlink that normally points at its content +with a copy of the content. You can then modify the file like any regular +file. Because it is a regular file. + +When you `git commit`, git-annex's pre-commit hook will automatically +notice that you are committing a checked-out file, and add its new content +to the annex. The file will be replaced with a symlink to the new content, +and this symlink is what gets committed to git. + +If you decide you don't need to modify the file after all, or want to discard +modifications, just use the uncheckout subcommand to undo the checkout. + ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has @@ -216,3 +236,17 @@ that the URL is stable; no local backup is kept. # git annex drop somefile drop somefile (ok) + +## using the SHA1 backend + +Another handy alternative to the default [[backend|backends]] is the +SHA1 backend. This backend provides more git-style assurance that your data +has not been damanged. And the checksum means that when you add the same +content to the annex twice, only one copy need be stored in the backend. + +The only reason it's not the default is that it needs to checksum +files when they're added to the annex, and this can slow things down +significantly for really big files. To make SHA1 the detault, just +add something like this to `.gitattributes`: + + * git-annex-backend=SHA1 From f03adec793d378cc4807392400d09e70e293a991 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 12:36:55 -0400 Subject: [PATCH 0434/8313] Add build dep on libghc6-testpack-dev. --- debian/changelog | 6 ++++++ debian/control | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index f9cec02c44..a363d442c9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.04) UNRELEASED; urgency=low + + * Add build dep on libghc6-testpack-dev. + + -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 + git-annex (0.03) unstable; urgency=low * Fix support for file:// remotes. diff --git a/debian/control b/debian/control index d8abc487cb..3fba367427 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, ikiwiki +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From ab4de454914954676aa1e05ef26dc8a85bd8f6f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 14:39:12 -0400 Subject: [PATCH 0435/8313] Add annex.version, which will be used to automate upgrades. --- Annex.hs | 12 +++++++++++- Core.hs | 16 ++++++++++++++++ UUID.hs | 13 ++----------- debian/changelog | 1 + doc/git-annex.mdwn | 2 ++ 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/Annex.hs b/Annex.hs index e86e1967e3..7c046b141d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -19,7 +19,8 @@ module Annex ( flagGet, Flag(..), queue, - queueGet + queueGet, + setConfig ) where import Control.Monad.State @@ -118,3 +119,12 @@ queueGet :: Annex GitQueue.Queue queueGet = do state <- get return (Internals.repoqueue state) + +{- Changes a git config setting in both internal state and .git/config -} +setConfig :: String -> String -> Annex () +setConfig key value = do + g <- Annex.gitRepo + liftIO $ Git.run g ["config", key, value] + -- re-read git config and update the repo's state + g' <- liftIO $ Git.configRead g Nothing + Annex.gitRepoChange g' diff --git a/Core.hs b/Core.hs index f34b2ebbeb..347e635939 100644 --- a/Core.hs +++ b/Core.hs @@ -46,6 +46,7 @@ tryRun' _ errnum [] = startup :: Annex Bool startup = do prepUUID + autoUpgrade return True {- When git-annex is done, it runs this. -} @@ -151,6 +152,21 @@ getViaTmp key action = do -- to resume its transfer return False +{- Uses the annex.version git config setting to automate upgrades. -} +autoUpgrade :: Annex () +autoUpgrade = do + g <- Annex.gitRepo + + case Git.configGet g field "0" of + "0" -> do -- before there was repo versioning + setVersion + v | v == currentVersion -> return () + _ -> error "this version of git-annex is too old for this git repository!" + where + currentVersion = "1" + setVersion = Annex.setConfig field currentVersion + field = "annex.version" + {- Output logging -} verbose :: Annex () -> Annex () verbose a = do diff --git a/UUID.hs b/UUID.hs index ffd2cd46dc..0f8a2173ef 100644 --- a/UUID.hs +++ b/UUID.hs @@ -65,7 +65,7 @@ getUUID r = do where uncached = Git.configGet r "annex.uuid" "" cached g = Git.configGet g cachekey "" - updatecache g u = when (g /= r) $ setConfig cachekey u + updatecache g u = when (g /= r) $ Annex.setConfig cachekey u cachekey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} @@ -75,16 +75,7 @@ prepUUID = do u <- getUUID g when ("" == u) $ do uuid <- liftIO $ genUUID - setConfig configkey uuid - -{- Changes a git config setting in both internal state and .git/config -} -setConfig :: String -> String -> Annex () -setConfig key value = do - g <- Annex.gitRepo - liftIO $ Git.run g ["config", key, value] - -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g Nothing - Annex.gitRepoChange g' + Annex.setConfig configkey uuid {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] diff --git a/debian/changelog b/debian/changelog index a363d442c9..98b814eb94 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.04) UNRELEASED; urgency=low * Add build dep on libghc6-testpack-dev. + * Add annex.version, which will be used to automate upgrades. -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 856b474e05..6f2c85d573 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -207,6 +207,8 @@ Here are all the supported configuration settings. to talk to this repository. * `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh options to use if a remote does not have specific options. +* `annex.version` -- Automatically maintained, and used to automate upgrades + between versions. The backend used when adding a new file to the annex can be configured on a per-file-type basis via the `.gitattributes` file. In the file, From 02a21d7f274568a2e2f94498607955aab8713a24 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 15:14:54 -0400 Subject: [PATCH 0436/8313] reorg .git/annex --- Locations.hs | 13 ++++++++++--- debian/changelog | 2 ++ doc/backends.mdwn | 12 ++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Locations.hs b/Locations.hs index 951924c407..78c0bff4b3 100644 --- a/Locations.hs +++ b/Locations.hs @@ -13,7 +13,8 @@ module Locations ( annexLocation, annexLocationRelative, annexTmpLocation, - annexDir + annexDir, + annexObjectDir ) where import Data.String.Utils @@ -29,7 +30,7 @@ gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc {- An annexed file's content is stored in - - /path/to/repo/.git/annex/, where is of the form + - /path/to/repo/.git/annex/objects//, where is of the form - - - That allows deriving the key and backend by looking at the symlink to it. @@ -42,7 +43,8 @@ annexLocation r key = - - Note: Assumes repo is NOT bare.-} annexLocationRelative :: Key -> FilePath -annexLocationRelative key = ".git/annex/" ++ (keyFile key) +annexLocationRelative key = ".git/annex/objects/" ++ f ++ f + where f = keyFile key {- The annex directory of a repository. - @@ -50,6 +52,11 @@ annexLocationRelative key = ".git/annex/" ++ (keyFile key) annexDir :: Git.Repo -> FilePath annexDir r = Git.workTree r ++ "/.git/annex" +{- The part of the annex directory where file contents are stored. + -} +annexObjectDir :: Git.Repo -> FilePath +annexObjectDir r = annexDir r ++ "/objects" + {- .git-annex/tmp is used for temp files -} annexTmpLocation :: Git.Repo -> FilePath annexTmpLocation r = annexDir r ++ "/tmp/" diff --git a/debian/changelog b/debian/changelog index 98b814eb94..49aa9829a0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ git-annex (0.04) UNRELEASED; urgency=low * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades. + * Reorganised the layout of .git/annex/ , moving cached file contents + to .git/annex/objects// -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/backends.mdwn b/doc/backends.mdwn index dc359174ae..fde23df5ed 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -10,13 +10,13 @@ Multiple pluggable backends are supported, and a single repository can use different backends for different files. * `WORM` ("Write Once, Read Many") This backend stores the file's content - only in `.git/annex/`, and assumes that any file with the same basename, - size, and modification time has the same content. So with this backend, - files can be moved around, but should never be added to or changed. - This is the default, and the least expensive backend. + only in `.git/annex/objects/`, and assumes that any file with the same + basename, size, and modification time has the same content. So with + this backend, files can be moved around, but should never be added to + or changed. This is the default, and the least expensive backend. * `SHA1` -- This backend stores the file's content in - `.git/annex/`, with a name based on its sha1 checksum. This backend allows - modifications of files to be tracked. Its need to generate checksums + `.git/annex/objects/`, with a name based on its sha1 checksum. This backend + allows modifications of files to be tracked. Its need to generate checksums can make it slower for large files. **Warning** this backend is not ready for use. * `URL` -- This backend downloads the file's content from an external URL. From 070e8530c1151dc96dec099eac8b967277751b10 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 15:15:21 -0400 Subject: [PATCH 0437/8313] refactoring, no code changes really --- Backend.hs | 2 +- Backend/File.hs | 1 + Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Command.hs | 2 +- Command/Add.hs | 1 + Command/Drop.hs | 1 + Command/DropKey.hs | 1 + Command/Fix.hs | 1 + Command/FromKey.hs | 1 + Command/Fsck.hs | 29 +------------------- Command/Get.hs | 1 + Command/Init.hs | 1 + Command/Move.hs | 5 ++-- Command/SetKey.hs | 1 + Command/Unannex.hs | 1 + Core.hs | 66 ++++++++++++++++++++-------------------------- Messages.hs | 54 +++++++++++++++++++++++++++++++++++++ Remotes.hs | 7 ++--- 19 files changed, 105 insertions(+), 74 deletions(-) create mode 100644 Messages.hs diff --git a/Backend.hs b/Backend.hs index 456a98bd41..43b450736d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -31,13 +31,13 @@ import Control.Monad.State import IO (try) import System.FilePath import System.Posix.Files -import Core import Locations import qualified GitRepo as Git import qualified Annex import Types import qualified TypeInternals as Internals +import Messages {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend] diff --git a/Backend/File.hs b/Backend/File.hs index 5b93d8227e..9178b830a5 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -25,6 +25,7 @@ import qualified GitRepo as Git import Core import qualified Annex import UUID +import Messages backend :: Backend backend = Backend { diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 4858922585..5a232ec1db 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -14,7 +14,7 @@ import System.IO import qualified Backend.File import TypeInternals -import Core +import Messages backend :: Backend backend = Backend.File.backend { diff --git a/Backend/URL.hs b/Backend/URL.hs index e6d3eb1ae5..830d343c53 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -11,8 +11,8 @@ import Control.Monad.State (liftIO) import Data.String.Utils import TypeInternals -import Core import Utility +import Messages backend :: Backend backend = Backend { diff --git a/Command.hs b/Command.hs index a0e3280d6b..f896a53f6f 100644 --- a/Command.hs +++ b/Command.hs @@ -9,7 +9,7 @@ module Command where import Types import qualified Backend -import Core +import Messages import qualified Annex {- A subcommand runs in four stages. diff --git a/Command/Add.hs b/Command/Add.hs index 825c1d8c1e..3cc681f69a 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -19,6 +19,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing diff --git a/Command/Drop.hs b/Command/Drop.hs index 6cdf216f41..d1ebd7f64d 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -17,6 +17,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} diff --git a/Command/DropKey.hs b/Command/DropKey.hs index bdd9b55b12..8076e6fd3f 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -17,6 +17,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- Drops cached content for a key. -} start :: SubCmdStartString diff --git a/Command/Fix.hs b/Command/Fix.hs index 90257a8a53..7963a1d2ea 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -15,6 +15,7 @@ import Command import qualified Annex import Utility import Core +import Messages {- Fixes the symlink to an annexed file. -} start :: SubCmdStartString diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 3071f218f4..de555475c1 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -18,6 +18,7 @@ import Utility import qualified Backend import Types import Core +import Messages {- Adds a file pointing at a manually-specified key -} start :: SubCmdStartString diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 785aecd8af..5405ce1201 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -8,19 +8,11 @@ module Command.Fsck where import qualified Data.Map as M -import System.Directory -import System.Posix.Files -import Monad (filterM) -import Control.Monad.State (liftIO) -import Data.Maybe import Command import Types import Core -import Locations -import qualified Annex -import qualified GitRepo as Git -import qualified Backend +import Messages {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -71,22 +63,3 @@ unusedKeys = do existsMap :: Ord k => [k] -> M.Map k Int existsMap l = M.fromList $ map (\k -> (k, 1)) l - -getKeysPresent :: Annex [Key] -getKeysPresent = do - g <- Annex.gitRepo - let top = annexDir g - contents <- liftIO $ getDirectoryContents top - files <- liftIO $ filterM (isreg top) contents - return $ map fileKey files - where - isreg top f = do - s <- getFileStatus $ top ++ "/" ++ f - return $ isRegularFile s - -getKeysReferenced :: Annex [Key] -getKeysReferenced = do - g <- Annex.gitRepo - files <- liftIO $ Git.inRepo g $ Git.workTree g - keypairs <- mapM Backend.lookupFile files - return $ map fst $ catMaybes keypairs diff --git a/Command/Get.hs b/Command/Get.hs index 1433bc8d00..c50b5a3775 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -11,6 +11,7 @@ import Command import qualified Backend import Types import Core +import Messages {- Gets an annexed file from one of the backends. -} start :: SubCmdStartString diff --git a/Command/Init.hs b/Command/Init.hs index b1e4e0e066..fd55242a46 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -15,6 +15,7 @@ import qualified Annex import Core import qualified GitRepo as Git import UUID +import Messages {- Stores description for the repository etc. -} start :: SubCmdStartString diff --git a/Command/Move.hs b/Command/Move.hs index cee9416222..6ca923a310 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,6 +20,7 @@ import Core import qualified GitRepo as Git import qualified Remotes import UUID +import Messages {- Move a file either --to or --from a repository. - @@ -64,7 +65,7 @@ moveToPerform key = do showNote $ show err return Nothing Right False -> do - Core.showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." + showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." let tmpfile = (annexTmpLocation remote) ++ (keyFile key) ok <- Remotes.copyToRemote remote key tmpfile if (ok) @@ -112,7 +113,7 @@ moveFromPerform key = do if (ishere) then return $ Just $ moveFromCleanup remote key else do - Core.showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." + showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." ok <- getViaTmp key (Remotes.copyFromRemote remote key) if (ok) then return $ Just $ moveFromCleanup remote key diff --git a/Command/SetKey.hs b/Command/SetKey.hs index a5710643ec..9286e740b6 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -18,6 +18,7 @@ import qualified Backend import LocationLog import Types import Core +import Messages {- Sets cached content for a key. -} start :: SubCmdStartString diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 5cffb2d894..e0848cd4a0 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -19,6 +19,7 @@ import LocationLog import Types import Core import qualified GitRepo as Git +import Messages {- The unannex subcommand undoes an add. -} start :: SubCmdStartString diff --git a/Core.hs b/Core.hs index 347e635939..7aadfb5fbf 100644 --- a/Core.hs +++ b/Core.hs @@ -8,12 +8,12 @@ module Core where import IO (try) -import System.IO import System.Directory import Control.Monad.State (liftIO) import System.Path -import Data.String.Utils -import Control.Monad (when, unless) +import Control.Monad (when, unless, filterM) +import System.Posix.Files +import Data.Maybe import Types import Locations @@ -22,7 +22,9 @@ import UUID import qualified GitRepo as Git import qualified GitQueue import qualified Annex +import qualified Backend import Utility +import Messages {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). @@ -152,6 +154,27 @@ getViaTmp key action = do -- to resume its transfer return False +{- List of keys whose content exists in .git/annex/objects/ -} +getKeysPresent :: Annex [Key] +getKeysPresent = do + g <- Annex.gitRepo + let top = annexObjectDir g + contents <- liftIO $ getDirectoryContents top + files <- liftIO $ filterM (isreg top) contents + return $ map fileKey files + where + isreg top f = do + s <- getFileStatus $ top ++ "/" ++ f + return $ isRegularFile s + +{- List of keys referenced by symlinks in the git repo. -} +getKeysReferenced :: Annex [Key] +getKeysReferenced = do + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g $ Git.workTree g + keypairs <- mapM Backend.lookupFile files + return $ map fst $ catMaybes keypairs + {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do @@ -159,6 +182,8 @@ autoUpgrade = do case Git.configGet g field "0" of "0" -> do -- before there was repo versioning + upgradeNote "Upgrading object directory layout..." + setVersion v | v == currentVersion -> return () _ -> error "this version of git-annex is too old for this git repository!" @@ -166,37 +191,4 @@ autoUpgrade = do currentVersion = "1" setVersion = Annex.setConfig field currentVersion field = "annex.version" - -{- Output logging -} -verbose :: Annex () -> Annex () -verbose a = do - q <- Annex.flagIsSet "quiet" - unless q a -showStart :: String -> String -> Annex () -showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ file ++ " " - liftIO $ hFlush stdout -showNote :: String -> Annex () -showNote s = verbose $ do - liftIO $ putStr $ "(" ++ s ++ ") " - liftIO $ hFlush stdout -showProgress :: Annex () -showProgress = verbose $ liftIO $ putStr "\n" -showLongNote :: String -> Annex () -showLongNote s = verbose $ do - liftIO $ putStr $ "\n" ++ indented - where - indented = join "\n" $ map (\l -> " " ++ l) $ lines s -showEndOk :: Annex () -showEndOk = verbose $ do - liftIO $ putStrLn "ok" -showEndFail :: Annex () -showEndFail = verbose $ do - liftIO $ putStrLn "\nfailed" - -{- Exception pretty-printing. -} -showErr :: (Show a) => a -> Annex () -showErr e = warning $ show e - -warning :: String -> Annex () -warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s + upgradeNote s = verbose $ liftIO $ putStrLn $ "("++s++")" diff --git a/Messages.hs b/Messages.hs new file mode 100644 index 0000000000..89f78e2441 --- /dev/null +++ b/Messages.hs @@ -0,0 +1,54 @@ +{- git-annex output messages + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Messages where + +import Control.Monad.State (liftIO) +import System.IO +import Control.Monad (unless) +import Data.String.Utils + +import Types +import qualified Annex + +verbose :: Annex () -> Annex () +verbose a = do + q <- Annex.flagIsSet "quiet" + unless q a + +showStart :: String -> String -> Annex () +showStart command file = verbose $ do + liftIO $ putStr $ command ++ " " ++ file ++ " " + liftIO $ hFlush stdout + +showNote :: String -> Annex () +showNote s = verbose $ do + liftIO $ putStr $ "(" ++ s ++ ") " + liftIO $ hFlush stdout + +showProgress :: Annex () +showProgress = verbose $ liftIO $ putStr "\n" + +showLongNote :: String -> Annex () +showLongNote s = verbose $ do + liftIO $ putStr $ "\n" ++ indented + where + indented = join "\n" $ map (\l -> " " ++ l) $ lines s +showEndOk :: Annex () +showEndOk = verbose $ do + liftIO $ putStrLn "ok" + +showEndFail :: Annex () +showEndFail = verbose $ do + liftIO $ putStrLn "\nfailed" + +{- Exception pretty-printing. -} +showErr :: (Show a) => a -> Annex () +showErr e = warning $ show e + +warning :: String -> Annex () +warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s diff --git a/Remotes.hs b/Remotes.hs index 280543968c..7aad6c2a06 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -36,6 +36,7 @@ import Locations import UUID import Utility import qualified Core +import Messages {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -64,7 +65,7 @@ keyPossibilities key = do let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive unless (null doexpensive) $ do - Core.showNote $ "getting UUID for " ++ + showNote $ "getting UUID for " ++ (list doexpensive) ++ "..." let todo = cheap ++ doexpensive if (not $ null todo) @@ -93,7 +94,7 @@ inAnnex r key = do a <- Annex.new r [] Annex.eval a (Core.inAnnex key) checkremote = do - Core.showNote ("checking " ++ Git.repoDescribe r ++ "...") + showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- runCmd r "test" ["-e", annexLocation r key] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. @@ -228,7 +229,7 @@ sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file scp :: Git.Repo -> [String] -> Annex Bool scp r params = do scpoptions <- repoConfig r "scp-options" "" - Core.showProgress -- make way for scp progress bar + showProgress -- make way for scp progress bar liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params {- Runs a command in a remote, using ssh if necessary. From c281747b0eb39c10eb7bae0ea3202dca6077b74f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:40:02 -0400 Subject: [PATCH 0438/8313] add queueRun --- Annex.hs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Annex.hs b/Annex.hs index 7c046b141d..af761051dc 100644 --- a/Annex.hs +++ b/Annex.hs @@ -20,6 +20,7 @@ module Annex ( Flag(..), queue, queueGet, + queueRun, setConfig ) where @@ -120,6 +121,15 @@ queueGet = do state <- get return (Internals.repoqueue state) +{- Runs (and empties) the queue. -} +queueRun :: Annex () +queueRun = do + state <- get + let q = Internals.repoqueue state + g <- gitRepo + liftIO $ GitQueue.run g q + put state { Internals.repoqueue = GitQueue.empty } + {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () setConfig key value = do From 50ec22e322ecc0538a0629e32313c0d8ec4ffd45 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:40:28 -0400 Subject: [PATCH 0439/8313] set version on init --- Command/Init.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Command/Init.hs b/Command/Init.hs index fd55242a46..fa5725c48f 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -15,6 +15,7 @@ import qualified Annex import Core import qualified GitRepo as Git import UUID +import Version import Messages {- Stores description for the repository etc. -} @@ -30,6 +31,7 @@ perform description = do g <- Annex.gitRepo u <- getUUID g describeUUID u description + setVersion liftIO $ gitAttributes g liftIO $ gitPreCommitHook g return $ Just $ cleanup From 98a77ab7256e484d29d67fcffc1f173fcb830f60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:40:42 -0400 Subject: [PATCH 0440/8313] add --- Version.hs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Version.hs diff --git a/Version.hs b/Version.hs new file mode 100644 index 0000000000..ce39c0c1b1 --- /dev/null +++ b/Version.hs @@ -0,0 +1,39 @@ +{- git-annex repository versioning + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Version where + +import Control.Monad.State (liftIO) +import System.Directory + +import Types +import qualified Annex +import qualified GitRepo as Git +import Locations + +currentVersion :: String +currentVersion = "1" + +versionField :: String +versionField = "annex.version" + +getVersion :: Annex (Maybe String) +getVersion = do + g <- Annex.gitRepo + let v = Git.configGet g versionField "" + if (not $ null v) + then return $ Just v + else do + -- version 0 was not recorded in .git/config; + -- such a repo should have an annexDir + d <- liftIO $ doesDirectoryExist $ annexDir g + if (d) + then return $ Just "0" + else return Nothing -- no version yet + +setVersion :: Annex () +setVersion = Annex.setConfig versionField currentVersion From ba59ac13b25d5be671e38cb7b4c40257f3fdac4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:45:41 -0400 Subject: [PATCH 0441/8313] add showSideAction --- Messages.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Messages.hs b/Messages.hs index 89f78e2441..ed4f3b90a1 100644 --- a/Messages.hs +++ b/Messages.hs @@ -20,6 +20,9 @@ verbose a = do q <- Annex.flagIsSet "quiet" unless q a +showSideAction :: String -> Annex () +showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" + showStart :: String -> String -> Annex () showStart command file = verbose $ do liftIO $ putStr $ command ++ " " ++ file ++ " " From 6395b790ce3d2f97803f0c642af71d1a9eb169c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 16:47:36 -0400 Subject: [PATCH 0442/8313] Reorganised the layout of .git/annex/ --- Core.hs | 77 ++++++++++++++++++++++++++++++++-------------- Locations.hs | 23 ++++++++------ debian/changelog | 10 ++++-- doc/git-annex.mdwn | 2 +- test.hs | 4 ++- 5 files changed, 78 insertions(+), 38 deletions(-) diff --git a/Core.hs b/Core.hs index 7aadfb5fbf..90af62eb67 100644 --- a/Core.hs +++ b/Core.hs @@ -25,6 +25,7 @@ import qualified Annex import qualified Backend import Utility import Messages +import Version {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). @@ -54,16 +55,14 @@ startup = do {- When git-annex is done, it runs this. -} shutdown :: Annex Bool shutdown = do - g <- Annex.gitRepo - - -- Runs all queued git commands. q <- Annex.queueGet unless (q == GitQueue.empty) $ do - verbose $ liftIO $ putStrLn "Recording state in git..." - liftIO $ GitQueue.run g q + showSideAction "Recording state in git..." + Annex.queueRun -- clean up any files left in the temp directory, but leave -- the tmp directory itself + g <- Annex.gitRepo let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp when (exists) $ liftIO $ removeDirectoryRecursive tmp @@ -140,13 +139,12 @@ logStatus key status = do getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool getViaTmp key action = do g <- Annex.gitRepo - let dest = annexLocation g key let tmp = annexTmpLocation g ++ keyFile key liftIO $ createDirectoryIfMissing True (parentDir tmp) success <- action tmp if (success) then do - liftIO $ renameFile tmp dest + moveToObjectDir key tmp logStatus key ValuePresent return True else do @@ -154,17 +152,28 @@ getViaTmp key action = do -- to resume its transfer return False +{- Moves a file into .git/annex/objects/ -} +moveToObjectDir :: Key -> FilePath -> Annex () +moveToObjectDir key src = do + g <- Annex.gitRepo + let dest = annexLocation g key + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ renameFile src dest + -- TODO directory and file mode tweaks + {- List of keys whose content exists in .git/annex/objects/ -} getKeysPresent :: Annex [Key] getKeysPresent = do g <- Annex.gitRepo - let top = annexObjectDir g - contents <- liftIO $ getDirectoryContents top - files <- liftIO $ filterM (isreg top) contents + getKeysPresent' $ annexObjectDir g +getKeysPresent' :: FilePath -> Annex [Key] +getKeysPresent' dir = do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM isreg contents return $ map fileKey files where - isreg top f = do - s <- getFileStatus $ top ++ "/" ++ f + isreg f = do + s <- getFileStatus $ dir ++ "/" ++ f return $ isRegularFile s {- List of keys referenced by symlinks in the git repo. -} @@ -178,17 +187,39 @@ getKeysReferenced = do {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do + version <- getVersion + case version of + Just "0" -> upgradeFrom0 + Nothing -> return () -- repo not initted yet, no version + Just v | v == currentVersion -> return () + Just _ -> error "this version of git-annex is too old for this git repository!" + +upgradeFrom0 :: Annex () +upgradeFrom0 = do + showSideAction "Upgrading object directory layout for git-annex 0.04..." g <- Annex.gitRepo - case Git.configGet g field "0" of - "0" -> do -- before there was repo versioning - upgradeNote "Upgrading object directory layout..." - - setVersion - v | v == currentVersion -> return () - _ -> error "this version of git-annex is too old for this git repository!" + -- do the reorganisation of the files + let olddir = annexDir g + keys <- getKeysPresent' olddir + _ <- mapM (\k -> moveToObjectDir k $ olddir ++ "/" ++ keyFile k) keys + + -- update the symlinks to the files + files <- liftIO $ Git.inRepo g $ Git.workTree g + fixlinks files + Annex.queueRun + + setVersion + where - currentVersion = "1" - setVersion = Annex.setConfig field currentVersion - field = "annex.version" - upgradeNote s = verbose $ liftIO $ putStrLn $ "("++s++")" + fixlinks [] = return () + fixlinks (f:fs) = do + r <- Backend.lookupFile f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [] f + fixlinks fs diff --git a/Locations.hs b/Locations.hs index 78c0bff4b3..e5f78a31ce 100644 --- a/Locations.hs +++ b/Locations.hs @@ -14,7 +14,9 @@ module Locations ( annexLocationRelative, annexTmpLocation, annexDir, - annexObjectDir + annexObjectDir, + + prop_idempotent_fileKey ) where import Data.String.Utils @@ -29,12 +31,7 @@ stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc -{- An annexed file's content is stored in - - /path/to/repo/.git/annex/objects//, where is of the form - - - - - - That allows deriving the key and backend by looking at the symlink to it. - -} +{- Annexed file's absolute location. -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = (Git.workTree r) ++ "/" ++ (annexLocationRelative key) @@ -43,8 +40,9 @@ annexLocation r key = - - Note: Assumes repo is NOT bare.-} annexLocationRelative :: Key -> FilePath -annexLocationRelative key = ".git/annex/objects/" ++ f ++ f - where f = keyFile key +annexLocationRelative key = ".git/annex/objects/" ++ f ++ "/" ++ f + where + f = keyFile key {- The annex directory of a repository. - @@ -72,10 +70,15 @@ annexTmpLocation r = annexDir r ++ "/tmp/" - is one to one. - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key +keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Key fileKey file = read $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +{- for quickcheck -} +prop_idempotent_fileKey :: String -> Bool +prop_idempotent_fileKey s = k == (fileKey $ keyFile k) + where k = read "test:s" diff --git a/debian/changelog b/debian/changelog index 49aa9829a0..dc9dcedc2b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,13 @@ git-annex (0.04) UNRELEASED; urgency=low * Add build dep on libghc6-testpack-dev. - * Add annex.version, which will be used to automate upgrades. - * Reorganised the layout of .git/annex/ , moving cached file contents - to .git/annex/objects// + * Add annex.version, which will be used to automate upgrades + between incompatable versions. + * Reorganised the layout of .git/annex/ + * The new layout will be automatically upgraded to the first time + git-annex is used in a repository with the old layout. + * Note that git-annex 0.04 cannot transfer content from old repositories + that have not yet been upgraded. -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 6f2c85d573..6a580f0050 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -223,7 +223,7 @@ but the SHA1 backend for ogg files: These files are used, in your git repository: -`.git/annex/` contains the annexed file contents that are currently +`.git/annex/objects/` contains the annexed file contents that are currently available. Annexed files in your git repository symlink to that content. `.git-annex/uuid.log` is used to map between repository UUID and diff --git a/test.hs b/test.hs index 9897236176..288532d7be 100644 --- a/test.hs +++ b/test.hs @@ -5,9 +5,11 @@ import Test.HUnit import Test.HUnit.Tools import GitRepo +import Locations alltests = [ - qctest "prop_idempotent_deencode" prop_idempotent_deencode + qctest "prop_idempotent_deencode" prop_idempotent_deencode, + qctest "prop_idempotent_fileKey" prop_idempotent_fileKey ] main = runVerboseTests (TestList alltests) From 40a815d873a828fbccee453f45fc519feffe15fd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 17:44:08 -0400 Subject: [PATCH 0443/8313] add unsetFileMode --- Utility.hs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Utility.hs b/Utility.hs index 4e56289e22..0053c687bb 100644 --- a/Utility.hs +++ b/Utility.hs @@ -11,17 +11,21 @@ module Utility ( relPathCwdToDir, relPathDirToDir, boolSystem, - shellEscape + shellEscape, + unsetFileMode ) where import System.IO import System.Exit import System.Posix.Process import System.Posix.Signals +import System.Posix.Files +import System.Posix.Types import Data.String.Utils import System.Path import System.FilePath import System.Directory +import Foreign (complement) {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} @@ -115,3 +119,10 @@ shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' escaped = join "'\"'\"'" $ split "'" f + +{- Removes a FileMode from a file. + - For example, call with otherWriteMode to chmod o-w -} +unsetFileMode :: FilePath -> FileMode -> IO () +unsetFileMode f m = do + s <- getFileStatus f + setFileMode f $ (fileMode s) `intersectFileModes` (complement m) From 8dd9f8e49eae081e7503facff6d5a53285194c09 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 17:44:30 -0400 Subject: [PATCH 0444/8313] typo --- Locations.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Locations.hs b/Locations.hs index e5f78a31ce..58244cef0e 100644 --- a/Locations.hs +++ b/Locations.hs @@ -81,4 +81,4 @@ fileKey file = read $ {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool prop_idempotent_fileKey s = k == (fileKey $ keyFile k) - where k = read "test:s" + where k = read $ "test:" ++ s From 1d32d902c95a49c53c46951641852c209476cb3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 8 Nov 2010 19:26:37 -0400 Subject: [PATCH 0445/8313] Annexed file contents are now made unwritable and put in unwriteable directories, to avoid them accidentially being removed or modified. (Thanks Josh Triplett for the idea.) --- Command/Add.hs | 9 +---- Command/Drop.hs | 17 ++------- Command/DropKey.hs | 8 +--- Command/SetKey.hs | 22 +++++------ Command/Unannex.hs | 9 +++-- Core.hs | 53 +++++++++++++++++++++++---- debian/changelog | 3 ++ doc/todo/immutable_annexed_files.mdwn | 2 + 8 files changed, 74 insertions(+), 49 deletions(-) diff --git a/Command/Add.hs b/Command/Add.hs index 3cc681f69a..6c5d24f842 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -9,12 +9,9 @@ module Command.Add where import Control.Monad.State (liftIO) import System.Posix.Files -import System.Directory import Command import qualified Annex -import Utility -import Locations import qualified Backend import LocationLog import Types @@ -42,11 +39,9 @@ perform (file, backend) = do cleanup :: FilePath -> Key -> SubCmdCleanup cleanup file key = do + moveAnnex key file logStatus key ValuePresent - g <- Annex.gitRepo - let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile file dest + link <- calcGitLink file key liftIO $ createSymbolicLink link file Annex.queue "add" [] file diff --git a/Command/Drop.hs b/Command/Drop.hs index d1ebd7f64d..48433b14cf 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -7,12 +7,9 @@ module Command.Drop where -import Control.Monad.State (liftIO) -import System.Directory +import Control.Monad (when) import Command -import qualified Annex -import Locations import qualified Backend import LocationLog import Types @@ -39,13 +36,7 @@ perform key backend = do cleanup :: Key -> SubCmdCleanup cleanup key = do - logStatus key ValueMissing inannex <- inAnnex key - if (inannex) - then do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc - return True - else return True - + when (inannex) $ removeAnnex key + logStatus key ValueMissing + return True diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 8076e6fd3f..e0b20918cb 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -7,12 +7,8 @@ module Command.DropKey where -import Control.Monad.State (liftIO) -import System.Directory - import Command import qualified Annex -import Locations import qualified Backend import LocationLog import Types @@ -36,9 +32,7 @@ start keyname = do perform :: Key -> SubCmdPerform perform key = do - g <- Annex.gitRepo - let loc = annexLocation g key - liftIO $ removeFile loc + removeAnnex key return $ Just $ cleanup key cleanup :: Key -> SubCmdCleanup diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 9286e740b6..50e9a590b1 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -13,7 +13,6 @@ import Control.Monad (when) import Command import qualified Annex import Utility -import Locations import qualified Backend import LocationLog import Types @@ -22,21 +21,22 @@ import Messages {- Sets cached content for a key. -} start :: SubCmdStartString -start tmpfile = do +start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list let key = genKey (backends !! 0) keyname - showStart "setkey" tmpfile - return $ Just $ perform tmpfile key + showStart "setkey" file + return $ Just $ perform file key perform :: FilePath -> Key -> SubCmdPerform -perform tmpfile key = do - g <- Annex.gitRepo - let loc = annexLocation g key - ok <- liftIO $ boolSystem "mv" [tmpfile, loc] - if (not ok) - then error "mv failed!" - else return $ Just $ cleanup key +perform file key = do + -- the file might be on a different filesystem, so mv is used + -- rather than simply calling moveToObjectDir key file + ok <- getViaTmp key $ \dest -> liftIO $ boolSystem "mv" [file, dest] + if ok + then return $ Just $ cleanup key + else error "mv failed!" + cleanup :: Key -> SubCmdCleanup cleanup key = do logStatus key ValuePresent diff --git a/Command/Unannex.hs b/Command/Unannex.hs index e0848cd4a0..a9c18f765e 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -13,7 +13,6 @@ import System.Directory import Command import qualified Annex import Utility -import Locations import qualified Backend import LocationLog import Types @@ -38,12 +37,14 @@ perform file key backend = do cleanup :: FilePath -> Key -> SubCmdCleanup cleanup file key = do - logStatus key ValueMissing g <- Annex.gitRepo - let src = annexLocation g key + liftIO $ removeFile file liftIO $ Git.run g ["rm", "--quiet", file] -- git rm deletes empty directories; put them back liftIO $ createDirectoryIfMissing True (parentDir file) - liftIO $ renameFile src file + + fromAnnex key file + logStatus key ValueMissing + return True diff --git a/Core.hs b/Core.hs index 90af62eb67..f04a3dfac8 100644 --- a/Core.hs +++ b/Core.hs @@ -144,7 +144,7 @@ getViaTmp key action = do success <- action tmp if (success) then do - moveToObjectDir key tmp + moveAnnex key tmp logStatus key ValuePresent return True else do @@ -152,14 +152,53 @@ getViaTmp key action = do -- to resume its transfer return False +{- Removes the write bits from a file. -} +preventWrite :: FilePath -> IO () +preventWrite f = unsetFileMode f writebits + where + writebits = foldl unionFileModes ownerWriteMode + [groupWriteMode, otherWriteMode] + +{- Turns a file's write bit back on. -} +allowWrite :: FilePath -> IO () +allowWrite f = do + s <- getFileStatus f + setFileMode f $ (fileMode s) `unionFileModes` ownerWriteMode + {- Moves a file into .git/annex/objects/ -} -moveToObjectDir :: Key -> FilePath -> Annex () -moveToObjectDir key src = do +moveAnnex :: Key -> FilePath -> Annex () +moveAnnex key src = do g <- Annex.gitRepo let dest = annexLocation g key - liftIO $ createDirectoryIfMissing True (parentDir dest) - liftIO $ renameFile src dest - -- TODO directory and file mode tweaks + let dir = parentDir dest + liftIO $ do + createDirectoryIfMissing True dir + renameFile src dest + preventWrite dest + preventWrite dir + +{- Removes a key's file from .git/annex/objects/ -} +removeAnnex :: Key -> Annex () +removeAnnex key = do + g <- Annex.gitRepo + let file = annexLocation g key + let dir = parentDir file + liftIO $ do + allowWrite dir + removeFile file + removeDirectory dir + +{- Moves a key's file out of .git/annex/objects/ -} +fromAnnex :: Key -> FilePath -> Annex () +fromAnnex key dest = do + g <- Annex.gitRepo + let file = annexLocation g key + let dir = parentDir file + liftIO $ do + allowWrite dir + allowWrite file + renameFile file dest + removeDirectory dir {- List of keys whose content exists in .git/annex/objects/ -} getKeysPresent :: Annex [Key] @@ -202,7 +241,7 @@ upgradeFrom0 = do -- do the reorganisation of the files let olddir = annexDir g keys <- getKeysPresent' olddir - _ <- mapM (\k -> moveToObjectDir k $ olddir ++ "/" ++ keyFile k) keys + _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys -- update the symlinks to the files files <- liftIO $ Git.inRepo g $ Git.workTree g diff --git a/debian/changelog b/debian/changelog index dc9dcedc2b..1ce6a2debe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,9 @@ git-annex (0.04) UNRELEASED; urgency=low git-annex is used in a repository with the old layout. * Note that git-annex 0.04 cannot transfer content from old repositories that have not yet been upgraded. + * Annexed file contents are now made unwritable and put in unwriteable + directories, to avoid them accidentially being removed or modified. + (Thanks Josh Triplett for the idea.) -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/todo/immutable_annexed_files.mdwn b/doc/todo/immutable_annexed_files.mdwn index e5207dc163..b26838e95e 100644 --- a/doc/todo/immutable_annexed_files.mdwn +++ b/doc/todo/immutable_annexed_files.mdwn @@ -4,3 +4,5 @@ > josh: Oh, I just thought of another slightly crazy but handy idea. > josh: I'd hate to run into a program which somehow followed the symlink and then did an unlink to replace the file. > josh: To break that, you could create a new directory under annex's internal directory for each file, and make the directory have no write permission. + +[[done]] and done --[[Joey]] From 8d5374f4a33f398baa166035e5fafb716a78fd1d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 Nov 2010 15:04:13 -0400 Subject: [PATCH 0446/8313] tweak --- Core.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index f04a3dfac8..304c8a9232 100644 --- a/Core.hs +++ b/Core.hs @@ -235,7 +235,7 @@ autoUpgrade = do upgradeFrom0 :: Annex () upgradeFrom0 = do - showSideAction "Upgrading object directory layout for git-annex 0.04..." + showSideAction "Upgrading object directory layout..." g <- Annex.gitRepo -- do the reorganisation of the files From d56feda25dd82ffa34fe5e3f28eff3ecf9eac5b5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 Nov 2010 15:11:45 -0400 Subject: [PATCH 0447/8313] maybe call it unlock and not checkout It'd be confusing to have a git-annex subcommand with the same name as a git subcommand. --- debian/changelog | 6 +++--- doc/git-annex.mdwn | 11 ++++++----- doc/todo/backendSHA1.mdwn | 2 +- doc/walkthrough.mdwn | 18 +++++++++--------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/debian/changelog b/debian/changelog index fa8bb0f938..9fd96de95b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ git-annex (0.04) UNRELEASED; urgency=low - * Add checkout subcommand, which allows checking out file content - in preparation of changing it. - * Add uncheckout subcommand. + * Add unlock subcommand, which replaces the symlink with a copy of + the file's content in preparation of changing it. + * Add lock subcommand. * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades between incompatable versions. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3bb3b08357..61f0f8fca6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -81,9 +81,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git-annex may refuse to drop content if the backend does not think it is safe to do so, typically because of the setting of annex.numcopies. -* checkout [path ...] +* unlock [path ...] - Checks out annexed files so they can be modified. This replaces the + Normally, the content of annexed files is protected from being changed. + Unlocking a annexed file allows it to be modified. This replaces the symlink for each specified file with a copy of the file content. When you `git commit`, the file, the new content is injected back into the annex. @@ -102,10 +103,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. and sets up `.gitattributes` and the pre-commit hook. This is an optional, but recommended step. -* uncheckout [path ...] +* lock [path ...] - Use this to undo a checkout command if you don't want to modify - the checked out files, or have made modifications you want to discard. + Use this to undo an unlock command if you don't want to modify + the files, or have made modifications you want to discard. * unannex [path ...] diff --git a/doc/todo/backendSHA1.mdwn b/doc/todo/backendSHA1.mdwn index 44df406dea..8c16b75ad0 100644 --- a/doc/todo/backendSHA1.mdwn +++ b/doc/todo/backendSHA1.mdwn @@ -4,4 +4,4 @@ In particular, while files can be added using it, git-annex will not notice when their content changes, and will not create a new key for the new sha1 of the net content. -[[done]]; use checkout subcommand +[[done]]; use unlock subcommand and commit changes with git diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index cb564fa979..29922bd75e 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -192,23 +192,23 @@ makes it very easy. ## modifying annexed files -Normally, the content of files in the annex cannot be modified. -In order to modify a file, it should first be checked out: +Normally, the content of files in the annex is prevented from being modified. +In order to modify a file, it should first be unlocked: - # git annex checkout my_cool_big_file - checkout my_cool_big_file (copying...) ok + # git annex unlock my_cool_big_file + unlock my_cool_big_file (copying...) ok -Checking a file out replaces the symlink that normally points at its content -with a copy of the content. You can then modify the file like any regular -file. Because it is a regular file. +They replaces the symlink that normally points at its content with a copy +of the content. You can then modify the file like any regular file. Because +it is a regular file. When you `git commit`, git-annex's pre-commit hook will automatically -notice that you are committing a checked-out file, and add its new content +notice that you are committing an unlocked file, and add its new content to the annex. The file will be replaced with a symlink to the new content, and this symlink is what gets committed to git. If you decide you don't need to modify the file after all, or want to discard -modifications, just use the uncheckout subcommand to undo the checkout. +modifications, just use the lock subcommand. ## using the URL backend From 536bc97d25479ac969273b49442c2fd8c31358c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 9 Nov 2010 15:59:49 -0400 Subject: [PATCH 0448/8313] lock and unlock subcommands --- CmdLine.hs | 6 ++++++ Command/Lock.hs | 41 +++++++++++++++++++++++++++++++++++++++++ Command/Unlock.hs | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 Command/Lock.hs create mode 100644 Command/Unlock.hs diff --git a/CmdLine.hs b/CmdLine.hs index 3823c72476..adcf25e9a9 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -30,6 +30,8 @@ import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init import qualified Command.Fsck +import qualified Command.Unlock +import qualified Command.Lock subCmds :: [SubCommand] subCmds = @@ -41,6 +43,10 @@ subCmds = "indicate content of files not currently wanted" , SubCommand "move" path (withFilesInGit Command.Move.start) "transfer content of files to/from another repository" + , SubCommand "unlock" path (withFilesInGit Command.Unlock.start) + "unlock files for modification" + , SubCommand "lock" path (withFilesInGit Command.Lock.start) + "undo unlock command" , SubCommand "init" desc (withDescription Command.Init.start) "initialize git-annex with repository description" , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) diff --git a/Command/Lock.hs b/Command/Lock.hs new file mode 100644 index 0000000000..c28e643276 --- /dev/null +++ b/Command/Lock.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Lock where + +import Control.Monad.State (liftIO) +import System.Directory +import System.Posix.Files + +import Command +import Messages +import qualified Annex +import qualified GitRepo as Git + +{- Undo unlock -} +start :: SubCmdStartString +start file = do + -- Want to avoid calling git checkout on files that are not + -- annexed -- but without the symlink to the annex, cannot tell + -- for sure if the file was annexed. So, check if git thinks the + -- file's type has changed (from a symlink to a regular file). + g <- Annex.gitRepo + test <- liftIO $ + Git.pipeRead g ["diff", "--name-only", "--diff-filter=T", file] + s <- liftIO $ getSymbolicLinkStatus file + if (null test || isSymbolicLink s) + then return Nothing + else do + showStart "lock" file + return $ Just $ perform file + +perform :: FilePath -> SubCmdPerform +perform file = do + liftIO $ removeFile file + g <- Annex.gitRepo + liftIO $ Git.run g ["checkout", file] + return $ Just $ return True -- no cleanup needed diff --git a/Command/Unlock.hs b/Command/Unlock.hs new file mode 100644 index 0000000000..57d4ad87af --- /dev/null +++ b/Command/Unlock.hs @@ -0,0 +1,36 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unlock where + +import Control.Monad.State (liftIO) +import System.Directory + +import Command +import qualified Annex +import Types +import Messages +import Locations +import Utility + +{- The unlock subcommand replaces the symlink with a copy of the file's + - content. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, _) -> do + showStart "unlock" file + return $ Just $ perform file key + +perform :: FilePath -> Key -> SubCmdPerform +perform dest key = do + g <- Annex.gitRepo + let src = annexLocation g key + liftIO $ removeFile dest + showNote "copying..." + ok <- liftIO $ boolSystem "cp" ["-p", src, dest] + if ok + then return $ Just $ return True -- no cleanup needed + else error "cp failed!" From 515d6b6c7d90e1ff44e791421066450cf4322b47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:49:35 -0400 Subject: [PATCH 0449/8313] Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 --- CmdLine.hs | 5 +++-- Makefile | 11 +++++++---- debian/changelog | 2 ++ doc/git-annex.mdwn | 4 ++-- test.hs | 3 ++- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index adcf25e9a9..1c73533df2 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -32,6 +32,7 @@ import qualified Command.Init import qualified Command.Fsck import qualified Command.Unlock import qualified Command.Lock +import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = @@ -51,8 +52,8 @@ subCmds = "initialize git-annex with repository description" , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) "undo accidential add command" - , SubCommand "pre-commit" path (withFilesToBeCommitted Command.Fix.start) - "fix up symlinks before they are committed" + , SubCommand "pre-commit" path (withFilesToBeCommitted Command.PreCommit.start) + "run by git pre-commit hook" , SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) "adds a file using a specific key" , SubCommand "dropkey" key (withKeys Command.DropKey.start) diff --git a/Makefile b/Makefile index eb74bb7273..ee4d50f9fc 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ all: git-annex docs +ghcmake=ghc -Wall -odir build -hidir build --make + git-annex: mkdir -p build - ghc -Wall -odir build -hidir build --make git-annex + $(ghcmake) git-annex install: install -d $(DESTDIR)/usr/bin @@ -17,7 +19,8 @@ IKIWIKI=ikiwiki endif test: - runghc test.hs + $(ghcmake) test + ./test docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 @@ -27,7 +30,7 @@ docs: --disable-plugin=smiley clean: - rm -rf build git-annex git-annex.1 + rm -rf build git-annex git-annex.1 test rm -rf doc/.ikiwiki html -.PHONY: git-annex +.PHONY: git-annex test diff --git a/debian/changelog b/debian/changelog index 9fd96de95b..97939af661 100644 --- a/debian/changelog +++ b/debian/changelog @@ -14,6 +14,8 @@ git-annex (0.04) UNRELEASED; urgency=low * Annexed file contents are now made unwritable and put in unwriteable directories, to avoid them accidentially being removed or modified. (Thanks Josh Triplett for the idea.) + * Avoid using runghc to run test suite as it is not available on all + architectures. Closes: #603006 -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 61f0f8fca6..d3cf594aa8 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -101,7 +101,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Initializes git-annex with a description of the git repository, and sets up `.gitattributes` and the pre-commit hook. - This is an optional, but recommended step. * lock [path ...] @@ -123,7 +122,8 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * pre-commit [path ...] Fixes up symlinks that are staged as part of a commit, to ensure they - point to annexed content. Also handles committing checked-out files. + point to annexed content. Also handles injecting changes to unlocked + files into the annex. This is meant to be called from git's pre-commit hook. `git annex init` automatically creates a pre-commit hook using this. diff --git a/test.hs b/test.hs index 288532d7be..2faf579a20 100644 --- a/test.hs +++ b/test.hs @@ -1,15 +1,16 @@ -- TODO find a test harness that is actually in Debian and use it. -import Test.QuickCheck import Test.HUnit import Test.HUnit.Tools import GitRepo import Locations +alltests :: [Test] alltests = [ qctest "prop_idempotent_deencode" prop_idempotent_deencode, qctest "prop_idempotent_fileKey" prop_idempotent_fileKey ] +main :: IO (Counts, Int) main = runVerboseTests (TestList alltests) From b52aa2d12f064310d193955b15ef7dccc281eec8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:51:17 -0400 Subject: [PATCH 0450/8313] in debian unstable! --- doc/download.mdwn | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index 664f46ed95..ccf21c091b 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,7 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -There are no binary packages yet, but you can build Debian packages from -the source tree with `dpkg-buildpackage`. +Users of Debian unstable can `apt-get install git-annex` Next: [[install]] From 05ca2bebff521b1fa9b79014b1856b828d897b6d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:51:17 -0400 Subject: [PATCH 0451/8313] in debian unstable! --- doc/download.mdwn | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index 664f46ed95..ccf21c091b 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,7 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -There are no binary packages yet, but you can build Debian packages from -the source tree with `dpkg-buildpackage`. +Users of Debian unstable can `apt-get install git-annex` Next: [[install]] From 91c5fe71af0f43461d933abc9a9b9a8b94594897 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:52:43 -0400 Subject: [PATCH 0452/8313] add --- Command/PreCommit.hs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Command/PreCommit.hs diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs new file mode 100644 index 0000000000..2128f207da --- /dev/null +++ b/Command/PreCommit.hs @@ -0,0 +1,22 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.PreCommit where + +import Command +import qualified Command.Fix + +{- Run by git pre-commit hook. -} +start :: SubCmdStartString +start file = do + -- If a file is unlocked for edit, inject its content into the + -- annex, and replace it with a symlink to the content. Git will + -- then commit that. + error "TODO" + + -- fix symlinks + Command.Fix.start file From 25b014ec26650c549bcb6dc900bf1c0646df57e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:52:59 -0400 Subject: [PATCH 0453/8313] update --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 13deb526a9..6956c49dd2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build/* +test git-annex git-annex.1 doc/.ikiwiki From f1c4a5a8dc222f19245f26e3deb7b25237bfc712 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 10:55:00 -0400 Subject: [PATCH 0454/8313] close --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index 97939af661..0e121fda21 100644 --- a/debian/changelog +++ b/debian/changelog @@ -16,6 +16,7 @@ git-annex (0.04) UNRELEASED; urgency=low (Thanks Josh Triplett for the idea.) * Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 + * Missing build dep. Closes: #603016 -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 From 55720885aeb90e9b8d3e4153145b6a13cac1c0c7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 12:50:00 -0400 Subject: [PATCH 0455/8313] set write bit on unlocked file --- Command/Unlock.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 57d4ad87af..de21988de5 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -16,6 +16,7 @@ import Types import Messages import Locations import Utility +import Core {- The unlock subcommand replaces the symlink with a copy of the file's - content. -} @@ -32,5 +33,7 @@ perform dest key = do showNote "copying..." ok <- liftIO $ boolSystem "cp" ["-p", src, dest] if ok - then return $ Just $ return True -- no cleanup needed + then do + liftIO $ allowWrite dest + return $ Just $ return True else error "cp failed!" From 2ab448276c90cfb588b8608c2e84ebbe88c87e57 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:01:01 -0400 Subject: [PATCH 0456/8313] fix handling of staged unlocked files --- Command/Lock.hs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/Command/Lock.hs b/Command/Lock.hs index c28e643276..1be35d6126 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -11,6 +11,7 @@ import Control.Monad.State (liftIO) import System.Directory import System.Posix.Files +import Types import Command import Messages import qualified Annex @@ -19,15 +20,8 @@ import qualified GitRepo as Git {- Undo unlock -} start :: SubCmdStartString start file = do - -- Want to avoid calling git checkout on files that are not - -- annexed -- but without the symlink to the annex, cannot tell - -- for sure if the file was annexed. So, check if git thinks the - -- file's type has changed (from a symlink to a regular file). - g <- Annex.gitRepo - test <- liftIO $ - Git.pipeRead g ["diff", "--name-only", "--diff-filter=T", file] - s <- liftIO $ getSymbolicLinkStatus file - if (null test || isSymbolicLink s) + locked <- isLocked file + if locked then return Nothing else do showStart "lock" file @@ -37,5 +31,26 @@ perform :: FilePath -> SubCmdPerform perform file = do liftIO $ removeFile file g <- Annex.gitRepo - liftIO $ Git.run g ["checkout", file] + -- first reset the file to drop any changes checked into the index + liftIO $ Git.run g ["reset", "-q", "--", file] + -- checkout the symlink + liftIO $ Git.run g ["checkout", "--", file] return $ Just $ return True -- no cleanup needed + +{- Checks if a file is unlocked for edit. + - + - But, without the symlink to the annex, cannot tell for sure if the + - file was annexed before. So, check if git thinks the file's type has + - changed (from a symlink to a regular file). -} +isLocked :: FilePath -> Annex Bool +isLocked file = do + g <- Annex.gitRepo + changed <- typechanged g Nothing + changedCached <- typechanged g $ Just "--cached" + s <- liftIO $ getSymbolicLinkStatus file + return $ null (changed++changedCached) || isSymbolicLink s + where + typechanged g Nothing = typechanged' g params + typechanged g (Just param) = typechanged' g $ params++[param] + typechanged' g p = liftIO $ Git.pipeRead g $ p++[file] + params = ["diff", "--name-only", "--diff-filter=T"] From 361d28e1389e218c1d0baf378a740570a139f730 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:01:17 -0400 Subject: [PATCH 0457/8313] Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the pre-commit hook. --- Command/PreCommit.hs | 28 +++++++++++++++++++++++----- debian/changelog | 3 +++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 2128f207da..117546359d 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -7,16 +7,34 @@ module Command.PreCommit where +import Control.Monad.State (liftIO) +import Control.Monad (when, unless) + import Command +import qualified Annex +import qualified Backend +import qualified GitRepo as Git import qualified Command.Fix +import qualified Command.Lock +import qualified Command.Add {- Run by git pre-commit hook. -} start :: SubCmdStartString start file = do - -- If a file is unlocked for edit, inject its content into the - -- annex, and replace it with a symlink to the content. Git will - -- then commit that. - error "TODO" + -- If a file is unlocked for edit, add its new content to the + -- annex, -} + locked <- Command.Lock.isLocked file + when (not locked) $ do + pairs <- Backend.chooseBackends [file] + ok <- doSubCmd $ Command.Add.start $ pairs !! 0 + unless (ok) $ do + error $ "failed to add " ++ file ++ "; canceling commit" + -- git commit will have staged the file's content; + -- drop that and stage the symlink + g <- Annex.gitRepo + liftIO $ Git.run g ["reset", "-q", "--", file] + liftIO $ Git.run g ["add", "--", file] - -- fix symlinks + -- Fix symlinks as they are committed, this ensures the + -- relative links are not broken when moved around. Command.Fix.start file diff --git a/debian/changelog b/debian/changelog index 0e121fda21..02c4ec295a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,9 @@ git-annex (0.04) UNRELEASED; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of the file's content in preparation of changing it. * Add lock subcommand. + * Unlocked files will now automatically be added back into the annex when + committed (and the updated symlink committed), by some magic in the + pre-commit hook. * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades between incompatable versions. From 43412419ea8c2f26620a0bc837acf6f14f0afb12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:08:29 -0400 Subject: [PATCH 0458/8313] bugfix The object's directory might exist if the backend picked the same key as was already present. That could happen, for example, if the sha1 is the same. Note that I chose to go ahead and replace the old content with the new. We don't know if they are the same (even with sha1, their timestamp or perms could differ), so have to assume the newer one is preffered. Although it won't propigate to other annexes, so it had better not be significantly different! --- Core.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Core.hs b/Core.hs index 304c8a9232..f9c9417bd8 100644 --- a/Core.hs +++ b/Core.hs @@ -173,6 +173,7 @@ moveAnnex key src = do let dir = parentDir dest liftIO $ do createDirectoryIfMissing True dir + allowWrite dir -- in case the directory already exists renameFile src dest preventWrite dest preventWrite dir From ee6052cbf8337f60d4c1ee590f21b24f77f3b210 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:27:52 -0400 Subject: [PATCH 0459/8313] update for modifying files --- doc/walkthrough.mdwn | 65 ++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 29922bd75e..cfdf0ddabb 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -139,6 +139,51 @@ But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. +## modifying annexed files + +Normally, the content of files in the annex is prevented from being modified. + + # echo oops > my_cool_big_file + bash: my_cool_big_file: Permission deined + +In order to modify a file, it should first be unlocked. + + # git annex unlock my_cool_big_file + unlock my_cool_big_file (copying...) ok + +They replaces the symlink that normally points at its content with a copy +of the content. You can then modify the file like any regular file. Because +it is a regular file. + +If you decide you don't need to modify the file after all, or want to discard +modifications, just use `git annex lock`. + +When you `git commit`, git-annex's pre-commit hook will automatically +notice that you are committing an unlocked file, and add its new content +to the annex. The file will be replaced with a symlink to the new content, +and this symlink is what gets committed to git in the end. + + # echo "now smaller, but even cooler" > my_cool_big_file + # git commit my_cool_big_file -m "changed an annexed file" + add my_cool_big_file ok + (Recording state in git...) + [master 64cda67] changed an annexed file + 2 files changed, 2 insertions(+), 1 deletions(-) + create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log + +There is one problem with using `git commit` like this: Git wants to first +stage the entire contents of the file in its index. That can be slow for +big files (sorta why git-annex exists in the first place). So, the +automatic handling on commit is a nice safety feature, since it prevents +the file content being accidentially commited into git. But when working with +big files, it's faster to explicitly add them to the annex yourself +before committing. + + # echo "now smaller, but even cooler yet" > my_cool_big_file + # git annex add my_cool_big_file + add my_cool_big_file ok + # git commit my_cool_big_file -m "changed an annexed file" + ## using ssh remotes So far in this walkthrough, git-annex has been used with a remote @@ -190,26 +235,6 @@ makes it very easy. WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok -## modifying annexed files - -Normally, the content of files in the annex is prevented from being modified. -In order to modify a file, it should first be unlocked: - - # git annex unlock my_cool_big_file - unlock my_cool_big_file (copying...) ok - -They replaces the symlink that normally points at its content with a copy -of the content. You can then modify the file like any regular file. Because -it is a regular file. - -When you `git commit`, git-annex's pre-commit hook will automatically -notice that you are committing an unlocked file, and add its new content -to the annex. The file will be replaced with a symlink to the new content, -and this symlink is what gets committed to git. - -If you decide you don't need to modify the file after all, or want to discard -modifications, just use the lock subcommand. - ## using the URL backend git-annex has multiple key-value [[backends]]. So far this walkthrough has From 99d9c1cf893f1ad6bf400219a47593cf70993768 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:28:04 -0400 Subject: [PATCH 0460/8313] edit an alias for unlock --- CmdLine.hs | 2 ++ debian/changelog | 3 ++- doc/git-annex.mdwn | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CmdLine.hs b/CmdLine.hs index 1c73533df2..3f2b1e94eb 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -46,6 +46,8 @@ subCmds = "transfer content of files to/from another repository" , SubCommand "unlock" path (withFilesInGit Command.Unlock.start) "unlock files for modification" + , SubCommand "edit" path (withFilesInGit Command.Unlock.start) + "same as unlock" , SubCommand "lock" path (withFilesInGit Command.Lock.start) "undo unlock command" , SubCommand "init" desc (withDescription Command.Init.start) diff --git a/debian/changelog b/debian/changelog index 02c4ec295a..d3733aeb8e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ git-annex (0.04) UNRELEASED; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of - the file's content in preparation of changing it. + the file's content in preparation of changing it. The "edit" subcommand + is an alias for unlock. * Add lock subcommand. * Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d3cf594aa8..cb3cde0a73 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -89,6 +89,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. When you `git commit`, the file, the new content is injected back into the annex. +* edit [path ...] + + This is an alias for the unlock subcommand. May be easier to remember, + if you think of this as allowing you to edit an annexed file. + * move [path ...] When used with the --to option, moves the content of annexed files from From d0886a9ac7b662b2cdfe386af1d81af74925d0b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 13:32:46 -0400 Subject: [PATCH 0461/8313] explicity run queue to git add files --- Command/PreCommit.hs | 2 +- doc/walkthrough.mdwn | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 117546359d..cd6ce6f080 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -33,7 +33,7 @@ start file = do -- drop that and stage the symlink g <- Annex.gitRepo liftIO $ Git.run g ["reset", "-q", "--", file] - liftIO $ Git.run g ["add", "--", file] + Annex.queueRun -- Fix symlinks as they are committed, this ensures the -- relative links are not broken when moved around. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index cfdf0ddabb..61cf29b89f 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -166,7 +166,6 @@ and this symlink is what gets committed to git in the end. # echo "now smaller, but even cooler" > my_cool_big_file # git commit my_cool_big_file -m "changed an annexed file" add my_cool_big_file ok - (Recording state in git...) [master 64cda67] changed an annexed file 2 files changed, 2 insertions(+), 1 deletions(-) create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log From 81524d19a7e5bc5f7b573260babe2def279187f7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:01:41 -0400 Subject: [PATCH 0462/8313] add typeChangedFiles --- GitRepo.hs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index da86c225e2..5fc077c449 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -39,6 +39,7 @@ module GitRepo ( checkAttr, decodeGitFile, encodeGitFile, + typeChangedFiles, prop_idempotent_deencode ) where @@ -58,6 +59,7 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf +import Data.List import Utility @@ -227,20 +229,31 @@ hPipeRead repo params = assertLocal repo $ do - are checked into git at that location. -} inRepo :: Repo -> FilePath -> IO [FilePath] inRepo repo l = pipeNullSplit repo - ["ls-files", "--cached", "--exclude-standard", "-z", l] + ["ls-files", "--cached", "--exclude-standard", "-z", "--", l] {- Passed a location, recursively scans for all files that are not checked - into git, and not gitignored. -} notInRepo :: Repo -> FilePath -> IO [FilePath] notInRepo repo l = pipeNullSplit repo - ["ls-files", "--others", "--exclude-standard", "-z", l] + ["ls-files", "--others", "--exclude-standard", "-z", "--", l] {- Passed a location, returns a list of the files, staged for - commit, that are being added, moved, or changed (but not deleted). -} stagedFiles :: Repo -> FilePath -> IO [FilePath] stagedFiles repo l = pipeNullSplit repo - ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", - "HEAD", l] + ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", + "--", l] + +{- Passed a location, returns a list of the files whose type has changed. -} +typeChangedFiles :: Repo -> FilePath -> IO [FilePath] +typeChangedFiles repo l = do + changed <- pipeNullSplit repo $ start ++ end + changedCached <- pipeNullSplit repo $ start ++ ["--cached"] ++ end + -- a file can be found twice by the above, so nub + return $ nub $ changed ++ changedCached + where + start = ["diff", "--name-only", "--diff-filter=T", "-z"] + end = ["--", l] {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} From e826368cecb2e515cc3c4f5f8d0385c025b069a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:02:08 -0400 Subject: [PATCH 0463/8313] allow adding unlocked files --- CmdLine.hs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 3f2b1e94eb..7e6626573a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -36,7 +36,7 @@ import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = - [ SubCommand "add" path (withFilesNotInGit Command.Add.start) + [ SubCommand "add" path (withFilesToAdd Command.Add.start) "add files to annex" , SubCommand "get" path (withFilesInGit Command.Get.start) "make content of annexed files available" @@ -115,13 +115,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs {- These functions find appropriate files or other things based on a user's parameters. -} -withFilesNotInGit :: SubCmdSeekBackendFiles -withFilesNotInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.notInRepo repo) params - let files' = foldl (++) [] files - pairs <- Backend.chooseBackends files' - return $ map a $ filter (\(f,_) -> notState f) pairs withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo @@ -135,6 +128,14 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e +withFilesToAdd :: SubCmdSeekBackendFiles +withFilesToAdd a params = do + repo <- Annex.gitRepo + newfiles <- liftIO $ mapM (Git.notInRepo repo) params + unlockedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params + let files = foldl (++) [] $ newfiles ++ unlockedfiles + pairs <- Backend.chooseBackends files + return $ map a $ filter (\(f,_) -> notState f) pairs withDescription :: SubCmdSeekStrings withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings From 31101a8b278a1b875defdea627307ef90ac3df21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:08:28 -0400 Subject: [PATCH 0464/8313] use new git function --- Command/Lock.hs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Command/Lock.hs b/Command/Lock.hs index 1be35d6126..955749e93d 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -45,12 +45,6 @@ perform file = do isLocked :: FilePath -> Annex Bool isLocked file = do g <- Annex.gitRepo - changed <- typechanged g Nothing - changedCached <- typechanged g $ Just "--cached" + typechanged <- liftIO $ Git.typeChangedFiles g file s <- liftIO $ getSymbolicLinkStatus file - return $ null (changed++changedCached) || isSymbolicLink s - where - typechanged g Nothing = typechanged' g params - typechanged g (Just param) = typechanged' g $ params++[param] - typechanged' g p = liftIO $ Git.pipeRead g $ p++[file] - params = ["diff", "--name-only", "--diff-filter=T"] + return $ (not $ elem file typechanged) || isSymbolicLink s From 16ba23d48de00daccbfb481dd1baca91047f8b3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:10:48 -0400 Subject: [PATCH 0465/8313] tweak --- doc/git-annex.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index cb3cde0a73..d0bd3a754e 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -85,9 +85,9 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Normally, the content of annexed files is protected from being changed. Unlocking a annexed file allows it to be modified. This replaces the - symlink for each specified file with a copy of the file content. - When you `git commit`, the file, the new content is injected back into - the annex. + symlink for each specified file with a copy of the file's content. + You can then modify it and `git annex add` (or `git commit`) to inject + it back into the annex. * edit [path ...] From fb824f7eb03c10301ad897d9e1eeb0aa40492a3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:15:21 -0400 Subject: [PATCH 0466/8313] use -- before filenames when running git add, git rm, etc --- Command/Add.hs | 2 +- Command/Fix.hs | 2 +- Command/FromKey.hs | 2 +- Command/Move.hs | 4 ++-- Command/Unannex.hs | 2 +- Core.hs | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Command/Add.hs b/Command/Add.hs index 6c5d24f842..649b466bb3 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -44,5 +44,5 @@ cleanup file key = do link <- calcGitLink file key liftIO $ createSymbolicLink link file - Annex.queue "add" [] file + Annex.queue "add" ["--"] file return True diff --git a/Command/Fix.hs b/Command/Fix.hs index 7963a1d2ea..9db832cc76 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -37,5 +37,5 @@ perform file link = do cleanup :: FilePath -> SubCmdCleanup cleanup file = do - Annex.queue "add" [] file + Annex.queue "add" ["--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index de555475c1..229a93684a 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -41,5 +41,5 @@ perform file key = do return $ Just $ cleanup file cleanup :: FilePath -> SubCmdCleanup cleanup file = do - Annex.queue "add" [] file + Annex.queue "add" ["--"] file return True diff --git a/Command/Move.hs b/Command/Move.hs index 6ca923a310..e0b079193a 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -85,7 +85,7 @@ moveToCleanup remote key tmpfile = do g <- Annex.gitRepo remoteuuid <- getUUID remote logfile <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" [] logfile + Annex.queue "add" ["--"] logfile -- Cleanup on the local side is the same as done for the -- drop subcommand. Command.Drop.cleanup key @@ -128,5 +128,5 @@ moveFromCleanup remote key = do remoteuuid <- getUUID remote g <- Annex.gitRepo logfile <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" [] logfile + Annex.queue "add" ["--"] logfile return ok diff --git a/Command/Unannex.hs b/Command/Unannex.hs index a9c18f765e..f5e78e55af 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -40,7 +40,7 @@ cleanup file key = do g <- Annex.gitRepo liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", file] + liftIO $ Git.run g ["rm", "--quiet", "--", file] -- git rm deletes empty directories; put them back liftIO $ createDirectoryIfMissing True (parentDir file) diff --git a/Core.hs b/Core.hs index f9c9417bd8..8497a7f368 100644 --- a/Core.hs +++ b/Core.hs @@ -131,7 +131,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g logfile <- liftIO $ logChange g key u status - Annex.queue "add" [] logfile + Annex.queue "add" ["--"] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into @@ -261,5 +261,5 @@ upgradeFrom0 = do link <- calcGitLink f k liftIO $ removeFile f liftIO $ createSymbolicLink link f - Annex.queue "add" [] f + Annex.queue "add" ["--"] f fixlinks fs From d9d6b256fc4ad430dad48374bd64a82913e7b080 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:16:53 -0400 Subject: [PATCH 0467/8313] cleanup --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index d3733aeb8e..9020f60855 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,7 +7,6 @@ git-annex (0.04) UNRELEASED; urgency=low * Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the pre-commit hook. - * Add build dep on libghc6-testpack-dev. * Add annex.version, which will be used to automate upgrades between incompatable versions. * Reorganised the layout of .git/annex/ @@ -18,9 +17,9 @@ git-annex (0.04) UNRELEASED; urgency=low * Annexed file contents are now made unwritable and put in unwriteable directories, to avoid them accidentially being removed or modified. (Thanks Josh Triplett for the idea.) + * Add build dep on libghc6-testpack-dev. Closes: #603016 * Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 - * Missing build dep. Closes: #603016 -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 From 10f35dceb62882d90dd019357ebd257b7e7f64fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:20:50 -0400 Subject: [PATCH 0468/8313] tweak --- doc/walkthrough.mdwn | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 61cf29b89f..d6c0214ffd 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -142,6 +142,8 @@ some other repository before dropping it. ## modifying annexed files Normally, the content of files in the annex is prevented from being modified. +That's a good thing, because it might be the only copy, you wouldn't +want to lose it in a fumblefingered mistake. # echo oops > my_cool_big_file bash: my_cool_big_file: Permission deined @@ -151,12 +153,12 @@ In order to modify a file, it should first be unlocked. # git annex unlock my_cool_big_file unlock my_cool_big_file (copying...) ok -They replaces the symlink that normally points at its content with a copy +That replaces the symlink that normally points at its content with a copy of the content. You can then modify the file like any regular file. Because it is a regular file. -If you decide you don't need to modify the file after all, or want to discard -modifications, just use `git annex lock`. +(If you decide you don't need to modify the file after all, or want to discard +modifications, just use `git annex lock`.) When you `git commit`, git-annex's pre-commit hook will automatically notice that you are committing an unlocked file, and add its new content From b9d7e67f614b0da96130c99206ad82a690bc826e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:29:51 -0400 Subject: [PATCH 0469/8313] releasing version 0.04 --- debian/changelog | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 9020f60855..0c09eb7ea4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.04) UNRELEASED; urgency=low +git-annex (0.04) unstable; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of the file's content in preparation of changing it. The "edit" subcommand @@ -7,8 +7,9 @@ git-annex (0.04) UNRELEASED; urgency=low * Unlocked files will now automatically be added back into the annex when committed (and the updated symlink committed), by some magic in the pre-commit hook. + * The SHA1 backend is now fully usable. * Add annex.version, which will be used to automate upgrades - between incompatable versions. + between incompatible versions. * Reorganised the layout of .git/annex/ * The new layout will be automatically upgraded to the first time git-annex is used in a repository with the old layout. @@ -21,7 +22,7 @@ git-annex (0.04) UNRELEASED; urgency=low * Avoid using runghc to run test suite as it is not available on all architectures. Closes: #603006 - -- Joey Hess Mon, 08 Nov 2010 12:36:39 -0400 + -- Joey Hess Wed, 10 Nov 2010 14:23:23 -0400 git-annex (0.03) unstable; urgency=low From 8a8987579f8efe290e5084202a416f72eec491a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 10 Nov 2010 14:30:33 -0400 Subject: [PATCH 0470/8313] add news item for git-annex 0.04 --- doc/news/version_0.04.mdwn | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 doc/news/version_0.04.mdwn diff --git a/doc/news/version_0.04.mdwn b/doc/news/version_0.04.mdwn new file mode 100644 index 0000000000..c5a9fd3ead --- /dev/null +++ b/doc/news/version_0.04.mdwn @@ -0,0 +1,23 @@ +git-annex 0.04 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add unlock subcommand, which replaces the symlink with a copy of + the file's content in preparation of changing it. The "edit" subcommand + is an alias for unlock. + * Add lock subcommand. + * Unlocked files will now automatically be added back into the annex when + committed (and the updated symlink committed), by some magic in the + pre-commit hook. + * The SHA1 backend is now fully usable. + * Add annex.version, which will be used to automate upgrades + between incompatible versions. + * Reorganised the layout of .git/annex/ + * The new layout will be automatically upgraded to the first time + git-annex is used in a repository with the old layout. + * Note that git-annex 0.04 cannot transfer content from old repositories + that have not yet been upgraded. + * Annexed file contents are now made unwritable and put in unwriteable + directories, to avoid them accidentially being removed or modified. + (Thanks Josh Triplett for the idea.) + * Add build dep on libghc6-testpack-dev. Closes: #[603016](http://bugs.debian.org/603016) + * Avoid using runghc to run test suite as it is not available on all + architectures. Closes: #[603006](http://bugs.debian.org/603006)"""]] \ No newline at end of file From dffe9499630a8dded76b23ef762f03aa264dc45c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 14:54:29 -0400 Subject: [PATCH 0471/8313] Optimize both pre-commit and lock subcommands. isLocked was doing the expensive check before the cheap one. Let's not fork git diff twice per file when committing, especially. git diff is still run more than strictly necessary (ie, more than once) if multiple unlocked files are being committed. But much better now. --- Command/Lock.hs | 18 ++++++++++-------- Command/PreCommit.hs | 5 +++-- debian/changelog | 6 ++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Command/Lock.hs b/Command/Lock.hs index 955749e93d..6ae59221c8 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -37,14 +37,16 @@ perform file = do liftIO $ Git.run g ["checkout", "--", file] return $ Just $ return True -- no cleanup needed -{- Checks if a file is unlocked for edit. - - - - But, without the symlink to the annex, cannot tell for sure if the - - file was annexed before. So, check if git thinks the file's type has - - changed (from a symlink to a regular file). -} +{- Checks if a file is unlocked for edit. -} isLocked :: FilePath -> Annex Bool isLocked file = do - g <- Annex.gitRepo - typechanged <- liftIO $ Git.typeChangedFiles g file + -- check if it's a symlink first, as that's cheapest s <- liftIO $ getSymbolicLinkStatus file - return $ (not $ elem file typechanged) || isSymbolicLink s + if (isSymbolicLink s) + then return True -- Symlinked files are always locked. + else do + -- Not a symlink, so see if the type has changed, + -- if so it is presumed to have been unlocked. + g <- Annex.gitRepo + typechanged <- liftIO $ Git.typeChangedFiles g file + return $ not $ elem file typechanged diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index cd6ce6f080..72cece8d50 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -22,7 +22,7 @@ import qualified Command.Add start :: SubCmdStartString start file = do -- If a file is unlocked for edit, add its new content to the - -- annex, -} + -- annex. -} locked <- Command.Lock.isLocked file when (not locked) $ do pairs <- Backend.chooseBackends [file] @@ -30,7 +30,8 @@ start file = do unless (ok) $ do error $ "failed to add " ++ file ++ "; canceling commit" -- git commit will have staged the file's content; - -- drop that and stage the symlink + -- drop that and run command queued by Add.state to + -- stage the symlink g <- Annex.gitRepo liftIO $ Git.run g ["reset", "-q", "--", file] Annex.queueRun diff --git a/debian/changelog b/debian/changelog index 0c09eb7ea4..a4c8bceac0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.05) UNRELEASED; urgency=low + + * Optimize both pre-commit and lock subcommands. + + -- Joey Hess Thu, 11 Nov 2010 14:52:05 -0400 + git-annex (0.04) unstable; urgency=low * Add unlock subcommand, which replaces the symlink with a copy of From 58fffdb73be115ad095342eba2f4dd90190e998f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 15:34:28 -0400 Subject: [PATCH 0472/8313] remove unnecessary mkdir --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index ee4d50f9fc..ffefbb2a39 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ all: git-annex docs ghcmake=ghc -Wall -odir build -hidir build --make git-annex: - mkdir -p build $(ghcmake) git-annex install: From b5ce88dd2aa2d6cc5eac6fd014f94d387c38bce0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 17:01:19 -0400 Subject: [PATCH 0473/8313] build with -O2 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ffefbb2a39..c94fbbc8f2 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: git-annex docs -ghcmake=ghc -Wall -odir build -hidir build --make +ghcmake=ghc -Wall -odir build -hidir build -O2 --make git-annex: $(ghcmake) git-annex From ce62f5abf16e578f9f4b86cd140ea2ddfb1e4217 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 17:58:55 -0400 Subject: [PATCH 0474/8313] rework command dispatching for add and pre-commit Both subcommands do two different operations on different sets of files, so allowing a subcommand to perform a list of operations cleans things up. --- CmdLine.hs | 57 ++++++++++++++++++++++++++++---------------- Command.hs | 6 ++--- Command/Lock.hs | 28 ++++------------------ Command/PreCommit.hs | 42 ++++++++++++++++---------------- Core.hs | 14 +++++++++++ debian/changelog | 8 ++++++- 6 files changed, 84 insertions(+), 71 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 7e6626573a..7c9d75c18d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -17,6 +17,7 @@ import qualified Annex import Locations import qualified Backend import Types +import Core import Command import qualified Command.Add @@ -36,35 +37,37 @@ import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = - [ SubCommand "add" path (withFilesToAdd Command.Add.start) + [ SubCommand "add" path [withFilesNotInGit Command.Add.start, + withFilesUnlocked Command.Add.start] "add files to annex" - , SubCommand "get" path (withFilesInGit Command.Get.start) + , SubCommand "get" path [withFilesInGit Command.Get.start] "make content of annexed files available" - , SubCommand "drop" path (withFilesInGit Command.Drop.start) + , SubCommand "drop" path [withFilesInGit Command.Drop.start] "indicate content of files not currently wanted" - , SubCommand "move" path (withFilesInGit Command.Move.start) + , SubCommand "move" path [withFilesInGit Command.Move.start] "transfer content of files to/from another repository" - , SubCommand "unlock" path (withFilesInGit Command.Unlock.start) + , SubCommand "unlock" path [withFilesInGit Command.Unlock.start] "unlock files for modification" - , SubCommand "edit" path (withFilesInGit Command.Unlock.start) + , SubCommand "edit" path [withFilesInGit Command.Unlock.start] "same as unlock" - , SubCommand "lock" path (withFilesInGit Command.Lock.start) + , SubCommand "lock" path [withFilesUnlocked Command.Lock.start] "undo unlock command" - , SubCommand "init" desc (withDescription Command.Init.start) + , SubCommand "init" desc [withDescription Command.Init.start] "initialize git-annex with repository description" - , SubCommand "unannex" path (withFilesInGit Command.Unannex.start) + , SubCommand "unannex" path [withFilesInGit Command.Unannex.start] "undo accidential add command" - , SubCommand "pre-commit" path (withFilesToBeCommitted Command.PreCommit.start) + , SubCommand "pre-commit" path [withFilesToBeCommitted Command.Fix.start, + withUnlockedFilesToBeCommitted Command.PreCommit.start] "run by git pre-commit hook" - , SubCommand "fromkey" key (withFilesMissing Command.FromKey.start) + , SubCommand "fromkey" key [withFilesMissing Command.FromKey.start] "adds a file using a specific key" - , SubCommand "dropkey" key (withKeys Command.DropKey.start) + , SubCommand "dropkey" key [withKeys Command.DropKey.start] "drops annexed content for specified keys" - , SubCommand "setkey" key (withTempFile Command.SetKey.start) + , SubCommand "setkey" key [withTempFile Command.SetKey.start] "sets annexed content for a key using a temp file" - , SubCommand "fix" path (withFilesInGit Command.Fix.start) + , SubCommand "fix" path [withFilesInGit Command.Fix.start] "fix up symlinks to point to annexed content" - , SubCommand "fsck" nothing (withNothing Command.Fsck.start) + , SubCommand "fsck" nothing [withNothing Command.Fsck.start] "check annex for problems" ] where @@ -128,12 +131,17 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withFilesToAdd :: SubCmdSeekBackendFiles -withFilesToAdd a params = do +withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params - unlockedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params - let files = foldl (++) [] $ newfiles ++ unlockedfiles + backendPairs a $ foldl (++) [] newfiles +withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked a params = do + unlocked <- mapM unlockedFiles params + backendPairs a $ foldl (++) [] unlocked +backendPairs :: SubCmdSeekBackendFiles +backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a $ filter (\(f,_) -> notState f) pairs withDescription :: SubCmdSeekStrings @@ -141,8 +149,15 @@ withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] files + tocommit <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] tocommit +withUnlockedFilesToBeCommitted :: SubCmdSeekStrings +withUnlockedFilesToBeCommitted a params = do + repo <- Annex.gitRepo + unlocked <- mapM unlockedFiles params + tocommit <- liftIO $ mapM (Git.stagedFiles repo) $ + filter notState $ foldl (++) [] unlocked + return $ map a $ foldl (++) [] tocommit withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings diff --git a/Command.hs b/Command.hs index f896a53f6f..90c4d53854 100644 --- a/Command.hs +++ b/Command.hs @@ -41,7 +41,7 @@ type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek data SubCommand = SubCommand { subcmdname :: String, subcmdparams :: String, - subcmdseek :: SubCmdSeek, + subcmdseek :: [SubCmdSeek], subcmddesc :: String } @@ -49,8 +49,8 @@ data SubCommand = SubCommand { - the parameters passed to it. -} prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] prepSubCmd SubCommand { subcmdseek = seek } state params = do - list <- Annex.eval state $ seek params - return $ map doSubCmd list + lists <- Annex.eval state $ mapM (\s -> s params) seek + return $ map doSubCmd $ foldl (++) [] lists {- Runs a subcommand through the start, perform and cleanup stages -} doSubCmd :: SubCmdStart -> SubCmdCleanup diff --git a/Command/Lock.hs b/Command/Lock.hs index 6ae59221c8..f03d6b6c88 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -9,23 +9,17 @@ module Command.Lock where import Control.Monad.State (liftIO) import System.Directory -import System.Posix.Files -import Types import Command import Messages import qualified Annex import qualified GitRepo as Git {- Undo unlock -} -start :: SubCmdStartString -start file = do - locked <- isLocked file - if locked - then return Nothing - else do - showStart "lock" file - return $ Just $ perform file +start :: SubCmdStartBackendFile +start (file, _) = do + showStart "lock" file + return $ Just $ perform file perform :: FilePath -> SubCmdPerform perform file = do @@ -36,17 +30,3 @@ perform file = do -- checkout the symlink liftIO $ Git.run g ["checkout", "--", file] return $ Just $ return True -- no cleanup needed - -{- Checks if a file is unlocked for edit. -} -isLocked :: FilePath -> Annex Bool -isLocked file = do - -- check if it's a symlink first, as that's cheapest - s <- liftIO $ getSymbolicLinkStatus file - if (isSymbolicLink s) - then return True -- Symlinked files are always locked. - else do - -- Not a symlink, so see if the type has changed, - -- if so it is presumed to have been unlocked. - g <- Annex.gitRepo - typechanged <- liftIO $ Git.typeChangedFiles g file - return $ not $ elem file typechanged diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 72cece8d50..b3b940cdd1 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -8,34 +8,32 @@ module Command.PreCommit where import Control.Monad.State (liftIO) -import Control.Monad (when, unless) import Command import qualified Annex import qualified Backend import qualified GitRepo as Git -import qualified Command.Fix -import qualified Command.Lock import qualified Command.Add -{- Run by git pre-commit hook. -} +{- Run by git pre-commit hook; passed unlocked files that are being + - committed. -} start :: SubCmdStartString -start file = do - -- If a file is unlocked for edit, add its new content to the - -- annex. -} - locked <- Command.Lock.isLocked file - when (not locked) $ do - pairs <- Backend.chooseBackends [file] - ok <- doSubCmd $ Command.Add.start $ pairs !! 0 - unless (ok) $ do - error $ "failed to add " ++ file ++ "; canceling commit" - -- git commit will have staged the file's content; - -- drop that and run command queued by Add.state to - -- stage the symlink - g <- Annex.gitRepo - liftIO $ Git.run g ["reset", "-q", "--", file] - Annex.queueRun +start file = return $ Just $ perform file - -- Fix symlinks as they are committed, this ensures the - -- relative links are not broken when moved around. - Command.Fix.start file +perform :: FilePath -> SubCmdPerform +perform file = do + pairs <- Backend.chooseBackends [file] + ok <- doSubCmd $ Command.Add.start $ pairs !! 0 + if ok + then return $ Just $ cleanup file + else error $ "failed to add " ++ file ++ "; canceling commit" + +cleanup :: FilePath -> SubCmdCleanup +cleanup file = do + -- git commit will have staged the file's content; + -- drop that and run command queued by Add.state to + -- stage the symlink + g <- Annex.gitRepo + liftIO $ Git.run g ["reset", "-q", "--", file] + Annex.queueRun + return True diff --git a/Core.hs b/Core.hs index 8497a7f368..0c06d23101 100644 --- a/Core.hs +++ b/Core.hs @@ -224,6 +224,20 @@ getKeysReferenced = do keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs +{- Passed a location (a directory or a single file, returns + - files there that are unlocked for editing. -} +unlockedFiles :: FilePath -> Annex [FilePath] +unlockedFiles l = do + -- unlocked files have changed type from a symlink to a regular file + g <- Annex.gitRepo + typechangedfiles <- liftIO $ Git.typeChangedFiles g l + unlockedfiles <- filterM notsymlink typechangedfiles + return unlockedfiles + where + notsymlink f = do + s <- liftIO $ getSymbolicLinkStatus f + return $ not $ isSymbolicLink s + {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do diff --git a/debian/changelog b/debian/changelog index a4c8bceac0..f705bfaf54 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,12 @@ git-annex (0.05) UNRELEASED; urgency=low - * Optimize both pre-commit and lock subcommands. + * Optimize both pre-commit and lock subcommands to not call git diff + on every file being committed or locked. + (This actually also works around a bug in ghc 6.12.1, that caused + git-annex 0.04 pre-commit to sometimes corrupt filenames and fail. + The excessive number of calls made by pre-commit exposed the ghc bug. + Thanks Josh Triplett for the debugging.) + * Build with -O3. -- Joey Hess Thu, 11 Nov 2010 14:52:05 -0400 From f2c7a6e73d342f4f82be4e2839a2022f97539c65 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:21:54 -0400 Subject: [PATCH 0475/8313] got rid of several more calls to git when finding unlocked files --- CmdLine.hs | 22 +++++++++++++++------- Core.hs | 14 -------------- GitRepo.hs | 20 +++++++++++++------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 7c9d75c18d..5c25b41c3b 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -10,6 +10,7 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad.State (liftIO) import System.Directory +import System.Posix.Files import Control.Monad (filterM, when) import qualified GitRepo as Git @@ -17,7 +18,6 @@ import qualified Annex import Locations import qualified Backend import Types -import Core import Command import qualified Command.Add @@ -138,8 +138,11 @@ withFilesNotInGit a params = do backendPairs a $ foldl (++) [] newfiles withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked a params = do - unlocked <- mapM unlockedFiles params - backendPairs a $ foldl (++) [] unlocked + -- unlocked files have changed type from a symlink to a regular file + repo <- Annex.gitRepo + typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + backendPairs a $ filter notState unlockedfiles backendPairs :: SubCmdSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files @@ -154,10 +157,9 @@ withFilesToBeCommitted a params = do withUnlockedFilesToBeCommitted :: SubCmdSeekStrings withUnlockedFilesToBeCommitted a params = do repo <- Annex.gitRepo - unlocked <- mapM unlockedFiles params - tocommit <- liftIO $ mapM (Git.stagedFiles repo) $ - filter notState $ foldl (++) [] unlocked - return $ map a $ foldl (++) [] tocommit + typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + return $ map a $ filter notState unlockedfiles withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings @@ -168,6 +170,12 @@ withNothing a _ = return [a] {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f + +{- filter out symlinks -} +notSymlink :: FilePath -> IO Bool +notSymlink f = do + s <- liftIO $ getSymbolicLinkStatus f + return $ not $ isSymbolicLink s {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it diff --git a/Core.hs b/Core.hs index 0c06d23101..8497a7f368 100644 --- a/Core.hs +++ b/Core.hs @@ -224,20 +224,6 @@ getKeysReferenced = do keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs -{- Passed a location (a directory or a single file, returns - - files there that are unlocked for editing. -} -unlockedFiles :: FilePath -> Annex [FilePath] -unlockedFiles l = do - -- unlocked files have changed type from a symlink to a regular file - g <- Annex.gitRepo - typechangedfiles <- liftIO $ Git.typeChangedFiles g l - unlockedfiles <- filterM notsymlink typechangedfiles - return unlockedfiles - where - notsymlink f = do - s <- liftIO $ getSymbolicLinkStatus f - return $ not $ isSymbolicLink s - {- Uses the annex.version git config setting to automate upgrades. -} autoUpgrade :: Annex () autoUpgrade = do diff --git a/GitRepo.hs b/GitRepo.hs index 5fc077c449..fa78e51222 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -40,6 +40,7 @@ module GitRepo ( decodeGitFile, encodeGitFile, typeChangedFiles, + typeChangedStagedFiles, prop_idempotent_deencode ) where @@ -59,7 +60,6 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf -import Data.List import Utility @@ -244,16 +244,22 @@ stagedFiles repo l = pipeNullSplit repo ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", "--", l] -{- Passed a location, returns a list of the files whose type has changed. -} +{- Passed a location, returns a list of the files, staged for + - commit, whose type has changed. -} +typeChangedStagedFiles :: Repo -> FilePath -> IO [FilePath] +typeChangedStagedFiles repo l = typeChangedFiles' repo l ["--cached"] + +{- Passed a location, returns a list of the files whose type has changed. + - Files only staged for commit will not be included. -} typeChangedFiles :: Repo -> FilePath -> IO [FilePath] -typeChangedFiles repo l = do - changed <- pipeNullSplit repo $ start ++ end - changedCached <- pipeNullSplit repo $ start ++ ["--cached"] ++ end - -- a file can be found twice by the above, so nub - return $ nub $ changed ++ changedCached +typeChangedFiles repo l = typeChangedFiles' repo l [] + +typeChangedFiles' :: Repo -> FilePath -> [String] -> IO [FilePath] +typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = ["diff", "--name-only", "--diff-filter=T", "-z"] end = ["--", l] + {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} From 5357d3a37af9e3d3a0aec207a8ba7fb94bfea953 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:26:27 -0400 Subject: [PATCH 0476/8313] remove dup filter --- CmdLine.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 5c25b41c3b..93404e546d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -117,7 +117,7 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs pad n s = replicate (n - length s) ' ' {- These functions find appropriate files or other things based on a - user's parameters. -} + user's parameters, and run a specified action on them. -} withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo @@ -135,7 +135,7 @@ withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params - backendPairs a $ foldl (++) [] newfiles + backendPairs a $ filter notState $ foldl (++) [] newfiles withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked a params = do -- unlocked files have changed type from a symlink to a regular file @@ -146,7 +146,7 @@ withFilesUnlocked a params = do backendPairs :: SubCmdSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files - return $ map a $ filter (\(f,_) -> notState f) pairs + return $ map a pairs withDescription :: SubCmdSeekStrings withDescription a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings From da0de293d16ace6aac574d0cdc37ec41715b7d66 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:54:52 -0400 Subject: [PATCH 0477/8313] refactor param seeking --- CmdLine.hs | 101 +++++++------------------------------------ Command.hs | 70 +++++++++++++++++++++++++++++- Command/Add.hs | 4 ++ Command/Drop.hs | 3 ++ Command/DropKey.hs | 3 ++ Command/Fix.hs | 3 ++ Command/FromKey.hs | 3 ++ Command/Fsck.hs | 3 ++ Command/Get.hs | 3 ++ Command/Init.hs | 3 ++ Command/Lock.hs | 3 ++ Command/Move.hs | 5 ++- Command/PreCommit.hs | 9 +++- Command/SetKey.hs | 3 ++ Command/Unannex.hs | 3 ++ Command/Unlock.hs | 3 ++ debian/changelog | 9 ++-- 17 files changed, 138 insertions(+), 93 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 93404e546d..efa541ebcc 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -8,15 +8,9 @@ module CmdLine (parseCmd) where import System.Console.GetOpt -import Control.Monad.State (liftIO) -import System.Directory -import System.Posix.Files -import Control.Monad (filterM, when) +import Control.Monad (when) -import qualified GitRepo as Git import qualified Annex -import Locations -import qualified Backend import Types import Command @@ -37,37 +31,35 @@ import qualified Command.PreCommit subCmds :: [SubCommand] subCmds = - [ SubCommand "add" path [withFilesNotInGit Command.Add.start, - withFilesUnlocked Command.Add.start] + [ SubCommand "add" path Command.Add.seek "add files to annex" - , SubCommand "get" path [withFilesInGit Command.Get.start] + , SubCommand "get" path Command.Get.seek "make content of annexed files available" - , SubCommand "drop" path [withFilesInGit Command.Drop.start] + , SubCommand "drop" path Command.Drop.seek "indicate content of files not currently wanted" - , SubCommand "move" path [withFilesInGit Command.Move.start] + , SubCommand "move" path Command.Move.seek "transfer content of files to/from another repository" - , SubCommand "unlock" path [withFilesInGit Command.Unlock.start] + , SubCommand "unlock" path Command.Unlock.seek "unlock files for modification" - , SubCommand "edit" path [withFilesInGit Command.Unlock.start] + , SubCommand "edit" path Command.Unlock.seek "same as unlock" - , SubCommand "lock" path [withFilesUnlocked Command.Lock.start] + , SubCommand "lock" path Command.Lock.seek "undo unlock command" - , SubCommand "init" desc [withDescription Command.Init.start] + , SubCommand "init" desc Command.Init.seek "initialize git-annex with repository description" - , SubCommand "unannex" path [withFilesInGit Command.Unannex.start] + , SubCommand "unannex" path Command.Unannex.seek "undo accidential add command" - , SubCommand "pre-commit" path [withFilesToBeCommitted Command.Fix.start, - withUnlockedFilesToBeCommitted Command.PreCommit.start] + , SubCommand "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" - , SubCommand "fromkey" key [withFilesMissing Command.FromKey.start] + , SubCommand "fromkey" key Command.FromKey.seek "adds a file using a specific key" - , SubCommand "dropkey" key [withKeys Command.DropKey.start] + , SubCommand "dropkey" key Command.DropKey.seek "drops annexed content for specified keys" - , SubCommand "setkey" key [withTempFile Command.SetKey.start] + , SubCommand "setkey" key Command.SetKey.seek "sets annexed content for a key using a temp file" - , SubCommand "fix" path [withFilesInGit Command.Fix.start] + , SubCommand "fix" path Command.Fix.seek "fix up symlinks to point to annexed content" - , SubCommand "fsck" nothing [withNothing Command.Fsck.start] + , SubCommand "fsck" nothing Command.Fsck.seek "check annex for problems" ] where @@ -116,67 +108,6 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = replicate (n - length s) ' ' -{- These functions find appropriate files or other things based on a - user's parameters, and run a specified action on them. -} -withFilesInGit :: SubCmdSeekStrings -withFilesInGit a params = do - repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - return $ map a $ filter notState $ foldl (++) [] files -withFilesMissing :: SubCmdSeekStrings -withFilesMissing a params = do - files <- liftIO $ filterM missing params - return $ map a $ filter notState files - where - missing f = do - e <- doesFileExist f - return $ not e -withFilesNotInGit :: SubCmdSeekBackendFiles -withFilesNotInGit a params = do - repo <- Annex.gitRepo - newfiles <- liftIO $ mapM (Git.notInRepo repo) params - backendPairs a $ filter notState $ foldl (++) [] newfiles -withFilesUnlocked :: SubCmdSeekBackendFiles -withFilesUnlocked a params = do - -- unlocked files have changed type from a symlink to a regular file - repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - backendPairs a $ filter notState unlockedfiles -backendPairs :: SubCmdSeekBackendFiles -backendPairs a files = do - pairs <- Backend.chooseBackends files - return $ map a pairs -withDescription :: SubCmdSeekStrings -withDescription a params = return [a $ unwords params] -withFilesToBeCommitted :: SubCmdSeekStrings -withFilesToBeCommitted a params = do - repo <- Annex.gitRepo - tocommit <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] tocommit -withUnlockedFilesToBeCommitted :: SubCmdSeekStrings -withUnlockedFilesToBeCommitted a params = do - repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - return $ map a $ filter notState unlockedfiles -withKeys :: SubCmdSeekStrings -withKeys a params = return $ map a params -withTempFile :: SubCmdSeekStrings -withTempFile a params = return $ map a params -withNothing :: SubCmdSeekNothing -withNothing a _ = return [a] - -{- filter out files from the state directory -} -notState :: FilePath -> Bool -notState f = stateLoc /= take (length stateLoc) f - -{- filter out symlinks -} -notSymlink :: FilePath -> IO Bool -notSymlink f = do - s <- liftIO $ getSymbolicLinkStatus f - return $ not $ isSymbolicLink s - {- Parses command line and returns two lists of actions to be - run in the Annex monad. The first actions configure it - according to command line options, while the second actions diff --git a/Command.hs b/Command.hs index 90c4d53854..21d636463e 100644 --- a/Command.hs +++ b/Command.hs @@ -1,4 +1,4 @@ -{- git-annex command types +{- git-annex commands - - Copyright 2010 Joey Hess - @@ -7,10 +7,17 @@ module Command where +import Control.Monad.State (liftIO) +import System.Directory +import System.Posix.Files +import Control.Monad (filterM) + import Types import qualified Backend import Messages import qualified Annex +import qualified GitRepo as Git +import Locations {- A subcommand runs in four stages. - @@ -87,3 +94,64 @@ isAnnexed file a = do case (r) of Just v -> a v Nothing -> return Nothing + +{- These functions find appropriate files or other things based on a + user's parameters, and run a specified action on them. -} +withFilesInGit :: SubCmdSeekStrings +withFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + return $ map a $ filter notState $ foldl (++) [] files +withFilesMissing :: SubCmdSeekStrings +withFilesMissing a params = do + files <- liftIO $ filterM missing params + return $ map a $ filter notState files + where + missing f = do + e <- doesFileExist f + return $ not e +withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit a params = do + repo <- Annex.gitRepo + newfiles <- liftIO $ mapM (Git.notInRepo repo) params + backendPairs a $ filter notState $ foldl (++) [] newfiles +withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked a params = do + -- unlocked files have changed type from a symlink to a regular file + repo <- Annex.gitRepo + typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + backendPairs a $ filter notState unlockedfiles +backendPairs :: SubCmdSeekBackendFiles +backendPairs a files = do + pairs <- Backend.chooseBackends files + return $ map a pairs +withDescription :: SubCmdSeekStrings +withDescription a params = return [a $ unwords params] +withFilesToBeCommitted :: SubCmdSeekStrings +withFilesToBeCommitted a params = do + repo <- Annex.gitRepo + tocommit <- liftIO $ mapM (Git.stagedFiles repo) params + return $ map a $ filter notState $ foldl (++) [] tocommit +withUnlockedFilesToBeCommitted :: SubCmdSeekStrings +withUnlockedFilesToBeCommitted a params = do + repo <- Annex.gitRepo + typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params + unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + return $ map a $ filter notState unlockedfiles +withKeys :: SubCmdSeekStrings +withKeys a params = return $ map a params +withTempFile :: SubCmdSeekStrings +withTempFile a params = return $ map a params +withNothing :: SubCmdSeekNothing +withNothing a _ = return [a] + +{- filter out files from the state directory -} +notState :: FilePath -> Bool +notState f = stateLoc /= take (length stateLoc) f + +{- filter out symlinks -} +notSymlink :: FilePath -> IO Bool +notSymlink f = do + s <- liftIO $ getSymbolicLinkStatus f + return $ not $ isSymbolicLink s diff --git a/Command/Add.hs b/Command/Add.hs index 649b466bb3..586807b530 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,6 +18,10 @@ import Types import Core import Messages +{- Add acts on both files not checked into git yet, and unlocked files. -} +seek :: [SubCmdSeek] +seek = [withFilesNotInGit start, withFilesUnlocked start] + {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} diff --git a/Command/Drop.hs b/Command/Drop.hs index 48433b14cf..1e73d8b821 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -16,6 +16,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} start :: SubCmdStartString diff --git a/Command/DropKey.hs b/Command/DropKey.hs index e0b20918cb..34010481dd 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -15,6 +15,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withKeys start] + {- Drops cached content for a key. -} start :: SubCmdStartString start keyname = do diff --git a/Command/Fix.hs b/Command/Fix.hs index 9db832cc76..323aca95e3 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,6 +17,9 @@ import Utility import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Fixes the symlink to an annexed file. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, _) -> do diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 229a93684a..f25de23a2c 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -20,6 +20,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesMissing start] + {- Adds a file pointing at a manually-specified key -} start :: SubCmdStartString start file = do diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 5405ce1201..e5f0debe0f 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -14,6 +14,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withNothing start] + {- Checks the whole annex for problems. -} start :: SubCmdStart start = do diff --git a/Command/Get.hs b/Command/Get.hs index c50b5a3775..13d1375377 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -13,6 +13,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Gets an annexed file from one of the backends. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do diff --git a/Command/Init.hs b/Command/Init.hs index fa5725c48f..e3b05a83fa 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -18,6 +18,9 @@ import UUID import Version import Messages +seek :: [SubCmdSeek] +seek = [withDescription start] + {- Stores description for the repository etc. -} start :: SubCmdStartString start description = do diff --git a/Command/Lock.hs b/Command/Lock.hs index f03d6b6c88..27a030bc22 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -15,6 +15,9 @@ import Messages import qualified Annex import qualified GitRepo as Git +seek :: [SubCmdSeek] +seek = [withFilesUnlocked start] + {- Undo unlock -} start :: SubCmdStartBackendFile start (file, _) = do diff --git a/Command/Move.hs b/Command/Move.hs index e0b079193a..7f8f40737d 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Monad (when) import Command -import Command.Drop +import qualified Command.Drop import qualified Annex import Locations import LocationLog @@ -22,6 +22,9 @@ import qualified Remotes import UUID import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- Move a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index b3b940cdd1..a15510bd93 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -14,9 +14,14 @@ import qualified Annex import qualified Backend import qualified GitRepo as Git import qualified Command.Add +import qualified Command.Fix + +{- The pre-commit hook needs to fix symlinks to all files being committed. + - And, it needs to inject unlocked files into the annex. -} +seek :: [SubCmdSeek] +seek = [withFilesToBeCommitted Command.Fix.start, + withUnlockedFilesToBeCommitted start] -{- Run by git pre-commit hook; passed unlocked files that are being - - committed. -} start :: SubCmdStartString start file = return $ Just $ perform file diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 50e9a590b1..e8d407b832 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -19,6 +19,9 @@ import Types import Core import Messages +seek :: [SubCmdSeek] +seek = [withTempFile start] + {- Sets cached content for a key. -} start :: SubCmdStartString start file = do diff --git a/Command/Unannex.hs b/Command/Unannex.hs index f5e78e55af..e85e8486f5 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -20,6 +20,9 @@ import Core import qualified GitRepo as Git import Messages +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- The unannex subcommand undoes an add. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do diff --git a/Command/Unlock.hs b/Command/Unlock.hs index de21988de5..3ff3023b24 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -18,6 +18,9 @@ import Locations import Utility import Core +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + {- The unlock subcommand replaces the symlink with a copy of the file's - content. -} start :: SubCmdStartString diff --git a/debian/changelog b/debian/changelog index f705bfaf54..b9f9569abc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,14 +1,15 @@ -git-annex (0.05) UNRELEASED; urgency=low +git-annex (0.05) unstable; urgency=low * Optimize both pre-commit and lock subcommands to not call git diff - on every file being committed or locked. + on every file being committed/locked. (This actually also works around a bug in ghc 6.12.1, that caused - git-annex 0.04 pre-commit to sometimes corrupt filenames and fail. + git-annex 0.04 pre-commit to sometimes corrupt filename being read + from git ls-files and fail. The excessive number of calls made by pre-commit exposed the ghc bug. Thanks Josh Triplett for the debugging.) * Build with -O3. - -- Joey Hess Thu, 11 Nov 2010 14:52:05 -0400 + -- Joey Hess Thu, 11 Nov 2010 18:31:09 -0400 git-annex (0.04) unstable; urgency=low From 3e60c3a3f921d7e6e3d4e227aeea3b0125ded93e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:59:19 -0400 Subject: [PATCH 0478/8313] releasing version 0.05 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b9f9569abc..7b59883bb3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,12 +2,12 @@ git-annex (0.05) unstable; urgency=low * Optimize both pre-commit and lock subcommands to not call git diff on every file being committed/locked. - (This actually also works around a bug in ghc 6.12.1, that caused + (This actually also works around a bug in ghc, that caused git-annex 0.04 pre-commit to sometimes corrupt filename being read from git ls-files and fail. The excessive number of calls made by pre-commit exposed the ghc bug. Thanks Josh Triplett for the debugging.) - * Build with -O3. + * Build with -O2. -- Joey Hess Thu, 11 Nov 2010 18:31:09 -0400 From 728fc4134e8c1f1040356b752a9be2c894080bc9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 18:59:37 -0400 Subject: [PATCH 0479/8313] add news item for git-annex 0.05 --- doc/news/version_0.05.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/news/version_0.05.mdwn diff --git a/doc/news/version_0.05.mdwn b/doc/news/version_0.05.mdwn new file mode 100644 index 0000000000..88c7391c13 --- /dev/null +++ b/doc/news/version_0.05.mdwn @@ -0,0 +1,10 @@ +git-annex 0.05 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Optimize both pre-commit and lock subcommands to not call git diff + on every file being committed/locked. + (This actually also works around a bug in ghc, that caused + git-annex 0.04 pre-commit to sometimes corrupt filename being read + from git ls-files and fail. + The excessive number of calls made by pre-commit exposed the ghc bug. + Thanks Josh Triplett for the debugging.) + * Build with -O2."""]] \ No newline at end of file From fc6b972da6f35be21429b96f8c50c1771edf1230 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 19:00:38 -0400 Subject: [PATCH 0480/8313] link to ghc bug --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 7b59883bb3..d1c3344ffa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,8 @@ git-annex (0.05) unstable; urgency=low on every file being committed/locked. (This actually also works around a bug in ghc, that caused git-annex 0.04 pre-commit to sometimes corrupt filename being read - from git ls-files and fail. + from git ls-files and fail. + See The excessive number of calls made by pre-commit exposed the ghc bug. Thanks Josh Triplett for the debugging.) * Build with -O2. From c7ec38b829192ce6c29acf37d49a30f28ebd6cb6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 11 Nov 2010 20:00:49 -0400 Subject: [PATCH 0481/8313] bug --- doc/bugs/fat_support.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/fat_support.mdwn diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn new file mode 100644 index 0000000000..05164d4571 --- /dev/null +++ b/doc/bugs/fat_support.mdwn @@ -0,0 +1,9 @@ +Klaus pointed out that there are two problems that keep +git-annex from being used on USB keys, that would typically +be VFAT formatted: + +- Use of symlinks, which VFAT does not support. Very hard to fix. +- Use of ":" in filenames of object files, also not supported. + Could easily be fixed by reorganizing the object directory. + +[[!tag wishlist]] From 09da0da02f683a218c56cbba0fe0d2c2d5909fe9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 12 Nov 2010 00:56:08 -0400 Subject: [PATCH 0482/8313] add --- doc/todo/use_cp_reflink.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/todo/use_cp_reflink.mdwn diff --git a/doc/todo/use_cp_reflink.mdwn b/doc/todo/use_cp_reflink.mdwn new file mode 100644 index 0000000000..d7974928ce --- /dev/null +++ b/doc/todo/use_cp_reflink.mdwn @@ -0,0 +1,5 @@ +The unlock command needs to copy a file, and it would be great to use this: + cp --reflink=auto src dst + +O(1) overhead on BTRFS. Needs coreutils 7.6; and remember that git-annex +may be used on systems without coreutils.. From 3a5efc54d0c4e3cf4bdb54e830a2bb2a9125f68c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 12 Nov 2010 11:00:20 -0400 Subject: [PATCH 0483/8313] fullfledged design for moving location tracking info into branches --- doc/todo/branching.mdwn | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index 21996ecc00..81f230a92a 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -34,3 +34,41 @@ get keys in that branch. Would have to be careful about conflicts when deleting and bringing back files with the same name. And would need to avoid expensive searching thru all history to try to find an old log file. + +## fleshed out proposal + +Let's use one branch per uuid, named git-annex/$UUID. + +- I came to realize this would be a good idea when thinking about how + to upgrade. Each individual annex will be upgraded independantly, + so each will want to make a branch, and if the branches aren't distinct, + they will merge conflict for sure. +- TODO: What will need to be done to git to make it push/pull these new + branches? +- A given repo only ever writes to its UUID branch. So no conflicts. +- (BTW, UUIDs probably don't compress well, and this reduces the bloat of having + them repeated lots of times in the tree.) +- Per UUID branches mean that if it wants to find a file's location + amoung configured remotes, it can examine only their branches, if + desired. + +In the branch, only one file is needed. Call it locationlog. git-annex +can cache location log changes and write them all to locationlog in +a single git operation on shutdown. + +- TODO: what if it's ctrl-c'd with changes pending? Perhaps it should + collect them to ,git/annex/locationlog, and inject that file on shutdown? +- This will be less overhead than the current staging of all the log files. + +The log is not appended to, so in git we have a series of commits each of +which replaces the log's entire contens. + +To find locations of a key, all (or all relevant) branches need to be +examined, looking backward through the history of each until a log +with a indication of the presense/absense of the key is found. + +- This will be less expensive for files that have recently been added + or transfered. +- It could get pretty slow when digging deeper. +- Only 3 places in git-annex will be affected by any slowdown: move --from, + get and drop. From d4d65a3c923de1eece50463145e875326bfe57e9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 13:11:41 -0400 Subject: [PATCH 0484/8313] new fsck items --- doc/todo/fsck.mdwn | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn index 6126154e9f..eebe6446f4 100644 --- a/doc/todo/fsck.mdwn +++ b/doc/todo/fsck.mdwn @@ -1,3 +1,9 @@ add a git annex fsck that finds keys that have no referring file -[[done]] +(done) + +* Need per-backend fsck support. sha1 can checksum all files in the annex. + WORM can check filesize. + +* Both can check that annex.numcopies is satisfied. Probably only + querying the locationlog, not doing an online verification. From 5fa25a812a8a03af9f6a5fdb3d06eb4d89ee06f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 14:59:27 -0400 Subject: [PATCH 0485/8313] fsck improvements * fsck: Check if annex.numcopies is satisfied. * fsck: Verify the sha1 of files when the SHA1 backend is used. * fsck: Verify the size of files when the WORM backend is used. * fsck: Allow specifying individual files to fsk if fscking everything is not desired. * fsck: Fix bug, introduced in 0.04, in detection of unused data. --- Backend.hs | 7 ++++++- Backend/File.hs | 43 ++++++++++++++++++++++++++++++++++++------- Backend/SHA1.hs | 37 ++++++++++++++++++++++++++++++++----- Backend/URL.hs | 8 ++++++-- Backend/WORM.hs | 34 +++++++++++++++++++++++++++++++++- CmdLine.hs | 6 +++--- Command.hs | 10 ++++++++++ Command/Fsck.hs | 9 ++++----- Command/FsckFile.hs | 33 +++++++++++++++++++++++++++++++++ Core.hs | 18 +++++++++++++++--- Locations.hs | 5 +++++ TypeInternals.hs | 4 +++- debian/changelog | 11 +++++++++++ doc/git-annex.mdwn | 8 +++++--- doc/walkthrough.mdwn | 34 ++++++++++++++++++++++++++++++++++ 15 files changed, 236 insertions(+), 31 deletions(-) create mode 100644 Command/FsckFile.hs diff --git a/Backend.hs b/Backend.hs index 43b450736d..14af56bbfa 100644 --- a/Backend.hs +++ b/Backend.hs @@ -23,6 +23,7 @@ module Backend ( retrieveKeyFile, removeKey, hasKey, + fsckKey, lookupFile, chooseBackends ) where @@ -105,7 +106,7 @@ retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} removeKey :: Backend -> Key -> Annex Bool -removeKey backend key = (Internals.removeKey backend) key +removeKey backend key = (Internals.removeKey backend) key {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool @@ -113,6 +114,10 @@ hasKey key = do bs <- Annex.supportedBackends (Internals.hasKey (lookupBackendName bs $ backendName key)) key +{- Checks a key's backend for problems. -} +fsckKey :: Backend -> Key -> Annex Bool +fsckKey backend key = (Internals.fsckKey backend) key + {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) diff --git a/Backend/File.hs b/Backend/File.hs index 9178b830a5..9bda0d5718 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -4,15 +4,15 @@ - it relies on the file contents in .git/annex/ in this repo, - and other accessible repos. - - - This is an abstract backend; getKey has to be implemented to complete - - it. + - This is an abstract backend; name, getKey and fsckKey have to be implemented + - to complete it. - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Backend.File (backend) where +module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory @@ -34,7 +34,8 @@ backend = Backend { storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = checkRemoveKey, - hasKey = checkKeyFile + hasKey = checkKeyFile, + fsckKey = mustProvide } mustProvide :: a @@ -97,14 +98,12 @@ checkRemoveKey key = do if (force) then return True else do - g <- Annex.gitRepo remotes <- Remotes.keyPossibilities key - let numcopies = read $ Git.configGet g config "1" + numcopies <- getNumCopies if (numcopies > length remotes) then notEnoughCopies numcopies (length remotes) [] else findcopies numcopies 0 remotes [] where - config = "annex.numcopies" findcopies need have [] bad = if (have >= need) then return True @@ -147,3 +146,33 @@ showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ (Remotes.list remotes) + +getNumCopies :: Annex Int +getNumCopies = do + g <- Annex.gitRepo + return $ read $ Git.configGet g config "1" + where + config = "annex.numcopies" + +{- This is used to check that numcopies is satisfied for the key on fsck. + - This trusts the location log, and so checks all keys, even those with + - data not present in the current annex. + - + - The passed action is first run to allow backends deriving this one + - to do their own checks. + -} +checkKey :: (Key -> Annex Bool) -> Key -> Annex Bool +checkKey a key = do + a_ok <- a key + copies_ok <- checkKeyNumCopies key + return $ a_ok && copies_ok + +checkKeyNumCopies :: Key -> Annex Bool +checkKeyNumCopies key = do + remotes <- Remotes.keyPossibilities key + numcopies <- getNumCopies + if (length remotes < numcopies) + then do + showLongNote $ "only " ++ show (length remotes) ++ " of " ++ show numcopies ++ " copies" + return False + else return True diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 5a232ec1db..8852e72e94 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -11,24 +11,51 @@ import Control.Monad.State import Data.String.Utils import System.Cmd.Utils import System.IO +import System.Directory import qualified Backend.File import TypeInternals import Messages +import qualified Annex +import Locations +import Core backend :: Backend backend = Backend.File.backend { name = "SHA1", - getKey = keyValue + getKey = keyValue, + fsckKey = Backend.File.checkKey checkKeySHA1 } --- checksum the file to get its key -keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = do +sha1 :: FilePath -> Annex String +sha1 file = do showNote "checksum..." liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line if (null bits) then error "sha1sum parse error" - else return $ Just $ Key ((name backend), bits !! 0) + else return $ bits !! 0 + +-- A key is a sha1 of its contents. +keyValue :: FilePath -> Annex (Maybe Key) +keyValue file = do + s <- sha1 file + return $ Just $ Key ((name backend), s) + +-- A key's sha1 is checked during fsck. +checkKeySHA1 :: Key -> Annex Bool +checkKeySHA1 key = do + g <- Annex.gitRepo + let file = annexLocation g key + present <- liftIO $ doesFileExist file + if (not present) + then return True + else do + s <- sha1 file + if (s == keyName key) + then return True + else do + dest <- moveBad key + showNote $ "bad file content (moved to "++dest++")" + return False diff --git a/Backend/URL.hs b/Backend/URL.hs index 830d343c53..b38ea71c96 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -20,8 +20,13 @@ backend = Backend { getKey = keyValue, storeFileKey = dummyStore, retrieveKeyFile = downloadUrl, + -- allow keys to be removed; presumably they can always be + -- downloaded again removeKey = dummyOk, - hasKey = dummyOk + -- similarly, keys are always assumed to be out there on the web + hasKey = dummyOk, + -- and nothing needed to fsck + fsckKey = dummyOk } -- cannot generate url from filename @@ -32,7 +37,6 @@ keyValue _ = return Nothing dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return False --- allow keys to be removed; presumably they can always be downloaded again dummyOk :: Key -> Annex Bool dummyOk _ = return True diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 848386ecd1..21b3876b90 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,14 +10,22 @@ module Backend.WORM (backend) where import Control.Monad.State import System.FilePath import System.Posix.Files +import System.Posix.Types +import System.Directory +import Data.String.Utils import qualified Backend.File import TypeInternals +import Locations +import qualified Annex +import Core +import Messages backend :: Backend backend = Backend.File.backend { name = "WORM", - getKey = keyValue + getKey = keyValue, + fsckKey = Backend.File.checkKey checkKeySize } -- The key is formed from the file size, modification time, and the @@ -36,3 +44,27 @@ keyValue file = do (show $ fileSize stat) base = takeFileName file sep = ":" + +{- Extracts the file size from a key. -} +keySize :: Key -> FileOffset +keySize key = read $ section !! 2 + where + section = split ":" (keyName key) + +{- The size of the data for a key is checked against the size encoded in + - the key. Note that the modification time is not checked. -} +checkKeySize :: Key -> Annex Bool +checkKeySize key = do + g <- Annex.gitRepo + let file = annexLocation g key + present <- liftIO $ doesFileExist file + if (not present) + then return True + else do + s <- liftIO $ getFileStatus file + if (fileSize s == keySize key) + then return True + else do + dest <- moveBad key + showNote $ "bad file size (moved to "++dest++")" + return False diff --git a/CmdLine.hs b/CmdLine.hs index efa541ebcc..a683be5c53 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -59,14 +59,14 @@ subCmds = "sets annexed content for a key using a temp file" , SubCommand "fix" path Command.Fix.seek "fix up symlinks to point to annexed content" - , SubCommand "fsck" nothing Command.Fsck.seek - "check annex for problems" + , SubCommand "fsck" maybepath Command.Fsck.seek + "check for problems" ] where path = "PATH ..." + maybepath = "[PATH ...]" key = "KEY ..." desc = "DESCRIPTION" - nothing = "" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. diff --git a/Command.hs b/Command.hs index 21d636463e..4180155faf 100644 --- a/Command.hs +++ b/Command.hs @@ -146,6 +146,16 @@ withTempFile a params = return $ map a params withNothing :: SubCmdSeekNothing withNothing a _ = return [a] +{- Default to acting on all files matching the seek action if + - none are specified. -} +withAll :: SubCmdSeekStrings -> SubCmdSeekStrings +withAll w a params = do + if null params + then do + g <- Annex.gitRepo + w a [Git.workTree g] + else w a params + {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f diff --git a/Command/Fsck.hs b/Command/Fsck.hs index e5f0debe0f..b0b9f7bb6b 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,9 +13,10 @@ import Command import Types import Core import Messages +import qualified Command.FsckFile seek :: [SubCmdSeek] -seek = [withNothing start] +seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] {- Checks the whole annex for problems. -} start :: SubCmdStart @@ -26,11 +27,9 @@ start = do perform :: SubCmdPerform perform = do ok <- checkUnused - if (ok) + if ok then return $ Just $ return True - else do - showLongNote "Possible problems detected." - return Nothing + else return Nothing checkUnused :: Annex Bool checkUnused = do diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs new file mode 100644 index 0000000000..2f9efa56ee --- /dev/null +++ b/Command/FsckFile.hs @@ -0,0 +1,33 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.FsckFile where + +import Command +import qualified Backend +import Types +import Messages + +seek :: [SubCmdSeek] +seek = [withFilesInGit start] + +{- Checks a file's backend data for problems. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + inbackend <- Backend.hasKey key + if (not inbackend) + then return Nothing + else do + showStart "fsck" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + success <- Backend.fsckKey backend key + if (success) + then return $ Just $ return True + else return Nothing diff --git a/Core.hs b/Core.hs index 8497a7f368..789b369cc8 100644 --- a/Core.hs +++ b/Core.hs @@ -14,6 +14,7 @@ import System.Path import Control.Monad (when, unless, filterM) import System.Posix.Files import Data.Maybe +import System.FilePath import Types import Locations @@ -201,6 +202,16 @@ fromAnnex key dest = do renameFile file dest removeDirectory dir +{- Moves a key out of .git/annex/objects/ into .git/annex/bad, and + - returns the directory it was moved to. -} +moveBad :: Key -> Annex FilePath +moveBad key = do + g <- Annex.gitRepo + let src = parentDir $ annexLocation g key + let dest = annexBadLocation g + liftIO $ renameDirectory src dest + return dest + {- List of keys whose content exists in .git/annex/objects/ -} getKeysPresent :: Annex [Key] getKeysPresent = do @@ -209,11 +220,12 @@ getKeysPresent = do getKeysPresent' :: FilePath -> Annex [Key] getKeysPresent' dir = do contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM isreg contents + files <- liftIO $ filterM present contents return $ map fileKey files where - isreg f = do - s <- getFileStatus $ dir ++ "/" ++ f + present d = do + s <- getFileStatus $ dir ++ "/" ++ d ++ "/" + ++ (takeFileName d) return $ isRegularFile s {- List of keys referenced by symlinks in the git repo. -} diff --git a/Locations.hs b/Locations.hs index 58244cef0e..c3bab285d4 100644 --- a/Locations.hs +++ b/Locations.hs @@ -13,6 +13,7 @@ module Locations ( annexLocation, annexLocationRelative, annexTmpLocation, + annexBadLocation, annexDir, annexObjectDir, @@ -59,6 +60,10 @@ annexObjectDir r = annexDir r ++ "/objects" annexTmpLocation :: Git.Repo -> FilePath annexTmpLocation r = annexDir r ++ "/tmp/" +{- .git-annex/bad is used for bad files found during fsck -} +annexBadLocation :: Git.Repo -> FilePath +annexBadLocation r = annexDir r ++ "/bad/" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/TypeInternals.hs b/TypeInternals.hs index 4b5cff9d9f..3078224b15 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -75,7 +75,9 @@ data Backend = Backend { -- removes a key removeKey :: Key -> Annex Bool, -- checks if a backend is storing the content of a key - hasKey :: Key -> Annex Bool + hasKey :: Key -> Annex Bool, + -- called during fsck to check a key + fsckKey :: Key -> Annex Bool } instance Show Backend where diff --git a/debian/changelog b/debian/changelog index d1c3344ffa..71d163d116 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +git-annex (0.06) UNRELEASED; urgency=low + + * fsck: Check if annex.numcopies is satisfied. + * fsck: Verify the sha1 of files when the SHA1 backend is used. + * fsck: Verify the size of files when the WORM backend is used. + * fsck: Allow specifying individual files to fsk if fscking everything + is not desired. + * fsck: Fix bug, introduced in 0.04, in detection of unused data. + + -- Joey Hess Sat, 13 Nov 2010 14:08:58 -0400 + git-annex (0.05) unstable; urgency=low * Optimize both pre-commit and lock subcommands to not call git diff diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d0bd3a754e..61a5962f1f 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -159,10 +159,12 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex setkey --key=1287765018:3 /tmp/file -* fsck +* fsck [path ...] - This subcommand checks the whole annex for consistency, and warns - about any problems found. + With no parameters, this subcommand checks the whole annex for consistency, + and warns about any problems found. + + With parameters, only the specified files are checked. # OPTIONS diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index d6c0214ffd..7effb53178 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -276,3 +276,37 @@ significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: * git-annex-backend=SHA1 + +## fsck: verifying your data + +You can use the fsck subcommand to check for problems in your data. +What can be checked depends on the [[backend|backends]] you've used to store +the data. For example, when you use the SHA1 backend, fsck will verify that +the checksums of your files are good. Fsck also checks that the annex.numcopies +setting is satisfied for all files, and it warns about any dangling values +in `.git/annex/objects/`. + + # git annex fsck + fsck (checking for unused data...) (checking files...) ok + +Fsck checks the entire repository for problems by default. But you can +also specify the files to check. +This is particularly useful if you're using sha1 and don't want to spend +a long time checksumming everything. + + # git annex fsck my_cool_big_file + fsck my_cool_big_file (checksum..) ok + +## fsck: When things go wrong + +Fsck never deletes possibly bad data; instead it will be moved to +`.git/annex/bad/` for you to review. Here is a sample of what fsck +might say about a badly messed up annex: + + # git annex fsck + fsck (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository. + If this data is no longer needed, it can be removed using git-annex dropkey: + WORM:1289672605:3:file + (checking files...) + From abebbcfd544953f3a2c9dab042368069fd2d916a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:24:36 -0400 Subject: [PATCH 0486/8313] fsck improvements --- Backend/File.hs | 14 +++++++++++--- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- doc/walkthrough.mdwn | 26 +++++++++++++++++--------- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 9bda0d5718..8351778568 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -169,10 +169,18 @@ checkKey a key = do checkKeyNumCopies :: Key -> Annex Bool checkKeyNumCopies key = do + needed <- getNumCopies remotes <- Remotes.keyPossibilities key - numcopies <- getNumCopies - if (length remotes < numcopies) + inannex <- inAnnex key + let present = length remotes + if inannex then 1 else 0 + if (present < needed) then do - showLongNote $ "only " ++ show (length remotes) ++ " of " ++ show numcopies ++ " copies" + showLongNote $ note present needed return False else return True + where + note 0 _ = "** No known copies of the file exist!" + note present needed = + "Only " ++ show present ++ " of " ++ show needed ++ + " copies exist. " ++ + "Run git annex get somewhere else to back it up." diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 8852e72e94..9e9000ba93 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -57,5 +57,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - showNote $ "bad file content (moved to "++dest++")" + showLongNote $ "Bad file content; moved to "++dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 21b3876b90..e53ec6de76 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -66,5 +66,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - showNote $ "bad file size (moved to "++dest++")" + showLongNote $ "Bad file size; moved to "++dest return False diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 7effb53178..887fde48cd 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -170,7 +170,7 @@ and this symlink is what gets committed to git in the end. add my_cool_big_file ok [master 64cda67] changed an annexed file 2 files changed, 2 insertions(+), 1 deletions(-) - create mode 100644 .git-annex/SHA1:0b1d8616d0238cb9418a0e0a649bdad2e9e7faae.log + create mode 100644 .git-annex/WORM:1289672605:30:file.log There is one problem with using `git commit` like this: Git wants to first stage the entire contents of the file in its index. That can be slow for @@ -287,20 +287,21 @@ setting is satisfied for all files, and it warns about any dangling values in `.git/annex/objects/`. # git annex fsck - fsck (checking for unused data...) (checking files...) ok + fsck (checking for unused data...) ok + fsck my_cool_big_file (checksum..) ok + ...... -Fsck checks the entire repository for problems by default. But you can -also specify the files to check. -This is particularly useful if you're using sha1 and don't want to spend -a long time checksumming everything. +You can also specifiy the files to check. This is particularly useful if +you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file + fsck (checking for unused data...) ok fsck my_cool_big_file (checksum..) ok ## fsck: When things go wrong Fsck never deletes possibly bad data; instead it will be moved to -`.git/annex/bad/` for you to review. Here is a sample of what fsck +`.git/annex/bad/` for you to recover. Here is a sample of what fsck might say about a badly messed up annex: # git annex fsck @@ -308,5 +309,12 @@ might say about a badly messed up annex: Some annexed data is no longer pointed to by any files in the repository. If this data is no longer needed, it can be removed using git-annex dropkey: WORM:1289672605:3:file - (checking files...) - + failed + fsck my_cool_big_file (checksum..) + Bad file content; moved to .git/annex/bad/ + ** No known copies of the file exist! + failed + fsck important_file + Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. + failed + git-annex: 3 failed From dd573e70105706bd1a769f74af9c3006dab903d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:40:12 -0400 Subject: [PATCH 0487/8313] fsck bugfixes --- Backend/WORM.hs | 2 +- Core.hs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Backend/WORM.hs b/Backend/WORM.hs index e53ec6de76..3749899968 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -47,7 +47,7 @@ keyValue file = do {- Extracts the file size from a key. -} keySize :: Key -> FileOffset -keySize key = read $ section !! 2 +keySize key = read $ section !! 1 where section = split ":" (keyName key) diff --git a/Core.hs b/Core.hs index 789b369cc8..2a81678aa6 100644 --- a/Core.hs +++ b/Core.hs @@ -207,9 +207,11 @@ fromAnnex key dest = do moveBad :: Key -> Annex FilePath moveBad key = do g <- Annex.gitRepo - let src = parentDir $ annexLocation g key + let src = annexLocation g key let dest = annexBadLocation g - liftIO $ renameDirectory src dest + liftIO $ createDirectoryIfMissing True dest + liftIO $ renameFile src (dest ++ takeFileName src) + liftIO $ removeDirectory (parentDir src) return dest {- List of keys whose content exists in .git/annex/objects/ -} From aec34ee1bda9b8bcdfe54903331eae582ee32792 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:42:56 -0400 Subject: [PATCH 0488/8313] tweak --- Core.hs | 6 +++--- doc/walkthrough.mdwn | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Core.hs b/Core.hs index 2a81678aa6..9faaada56f 100644 --- a/Core.hs +++ b/Core.hs @@ -203,14 +203,14 @@ fromAnnex key dest = do removeDirectory dir {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and - - returns the directory it was moved to. -} + - returns the file it was moved to. -} moveBad :: Key -> Annex FilePath moveBad key = do g <- Annex.gitRepo let src = annexLocation g key - let dest = annexBadLocation g + let dest = annexBadLocation g ++ takeFileName src liftIO $ createDirectoryIfMissing True dest - liftIO $ renameFile src (dest ++ takeFileName src) + liftIO $ renameFile src dest liftIO $ removeDirectory (parentDir src) return dest diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 887fde48cd..049dc40f67 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -311,7 +311,7 @@ might say about a badly messed up annex: WORM:1289672605:3:file failed fsck my_cool_big_file (checksum..) - Bad file content; moved to .git/annex/bad/ + Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 ** No known copies of the file exist! failed fsck important_file From 14d897a33827450799a713dac9d6a5a384c80e40 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:43:08 -0400 Subject: [PATCH 0489/8313] close --- doc/todo/fsck.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/fsck.mdwn b/doc/todo/fsck.mdwn index eebe6446f4..1dcaad9a51 100644 --- a/doc/todo/fsck.mdwn +++ b/doc/todo/fsck.mdwn @@ -7,3 +7,5 @@ add a git annex fsck that finds keys that have no referring file * Both can check that annex.numcopies is satisfied. Probably only querying the locationlog, not doing an online verification. + +[[done]] From 147affa252989b99e876a052433bd5d2ce32c672 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:44:02 -0400 Subject: [PATCH 0490/8313] tweak --- doc/walkthrough.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 049dc40f67..a15f0a9a44 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -288,7 +288,7 @@ in `.git/annex/objects/`. # git annex fsck fsck (checking for unused data...) ok - fsck my_cool_big_file (checksum..) ok + fsck my_cool_big_file (checksum...) ok ...... You can also specifiy the files to check. This is particularly useful if @@ -296,7 +296,7 @@ you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file fsck (checking for unused data...) ok - fsck my_cool_big_file (checksum..) ok + fsck my_cool_big_file (checksum...) ok ## fsck: When things go wrong @@ -310,7 +310,7 @@ might say about a badly messed up annex: If this data is no longer needed, it can be removed using git-annex dropkey: WORM:1289672605:3:file failed - fsck my_cool_big_file (checksum..) + fsck my_cool_big_file (checksum...) Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 ** No known copies of the file exist! failed From d9d79a7980b5c9a84823c0cf38750a0189d03b0a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 15:46:56 -0400 Subject: [PATCH 0491/8313] idea --- doc/bugs/fat_support.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 05164d4571..2a998cf1ad 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -3,6 +3,8 @@ git-annex from being used on USB keys, that would typically be VFAT formatted: - Use of symlinks, which VFAT does not support. Very hard to fix. + One possibility is to add [[bare_git_repos]] support, then + a git repo on a thumb drive could be used to transfer data. - Use of ":" in filenames of object files, also not supported. Could easily be fixed by reorganizing the object directory. From 498c8e8544f04841d3221ce7a47b2409858e9f38 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:03:25 -0400 Subject: [PATCH 0492/8313] fsck: avoid global checks if files specified --- Command.hs | 6 ++---- Command/Fsck.hs | 19 +++++++++++++------ Command/Init.hs | 2 +- doc/walkthrough.mdwn | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Command.hs b/Command.hs index 4180155faf..3f68cf661f 100644 --- a/Command.hs +++ b/Command.hs @@ -126,8 +126,8 @@ backendPairs :: SubCmdSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a pairs -withDescription :: SubCmdSeekStrings -withDescription a params = return [a $ unwords params] +withString :: SubCmdSeekStrings +withString a params = return [a $ unwords params] withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo @@ -143,8 +143,6 @@ withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params -withNothing :: SubCmdSeekNothing -withNothing a _ = return [a] {- Default to acting on all files matching the seek action if - none are specified. -} diff --git a/Command/Fsck.hs b/Command/Fsck.hs index b0b9f7bb6b..85a26a89b9 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -16,13 +16,20 @@ import Messages import qualified Command.FsckFile seek :: [SubCmdSeek] -seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] +seek = [withString start, withAll withFilesInGit Command.FsckFile.start] -{- Checks the whole annex for problems. -} -start :: SubCmdStart -start = do - showStart "fsck" "" - return $ Just perform +{- Checks the whole annex for problems, only if specific files were not + - specified. -} +start :: SubCmdStartString +start whatspecified = do + if (null whatspecified) + then do + showStart "fsck" "" + return $ Just perform + else do + showStart "fsck" "" + showNote "only checking specified files" + return $ Just $ return $ Just $ return True perform :: SubCmdPerform perform = do diff --git a/Command/Init.hs b/Command/Init.hs index e3b05a83fa..8110948a41 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -19,7 +19,7 @@ import Version import Messages seek :: [SubCmdSeek] -seek = [withDescription start] +seek = [withString start] {- Stores description for the repository etc. -} start :: SubCmdStartString diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index a15f0a9a44..cdb2392e7a 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -295,7 +295,7 @@ You can also specifiy the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file - fsck (checking for unused data...) ok + fsck (only checking specified files) ok fsck my_cool_big_file (checksum...) ok ## fsck: When things go wrong From 7293ba2940dc7dcefa7376667fa4462b21e05ee0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:12:02 -0400 Subject: [PATCH 0493/8313] fsck even files not in backend --- Command/FsckFile.hs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs index 2f9efa56ee..c74e94e62f 100644 --- a/Command/FsckFile.hs +++ b/Command/FsckFile.hs @@ -18,12 +18,8 @@ seek = [withFilesInGit start] {- Checks a file's backend data for problems. -} start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do - inbackend <- Backend.hasKey key - if (not inbackend) - then return Nothing - else do - showStart "fsck" file - return $ Just $ perform key backend + showStart "fsck" file + return $ Just $ perform key backend perform :: Key -> Backend -> SubCmdPerform perform key backend = do From 19ee56559a191e0e794380363a1bd0b0c57ea22a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:15:45 -0400 Subject: [PATCH 0494/8313] better fsck file handling --- Command.hs | 4 ++++ Command/Fsck.hs | 16 +++++----------- doc/walkthrough.mdwn | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Command.hs b/Command.hs index 3f68cf661f..40a21cacc8 100644 --- a/Command.hs +++ b/Command.hs @@ -44,6 +44,7 @@ type SubCmdStartString = String -> SubCmdStart type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek +type SubCmdStartNothing = SubCmdStart data SubCommand = SubCommand { subcmdname :: String, @@ -143,6 +144,9 @@ withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings withTempFile a params = return $ map a params +withNothing :: SubCmdSeekNothing +withNothing a [] = return [a] +withNothing _ _ = return [] {- Default to acting on all files matching the seek action if - none are specified. -} diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 85a26a89b9..5b731a6968 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -16,20 +16,14 @@ import Messages import qualified Command.FsckFile seek :: [SubCmdSeek] -seek = [withString start, withAll withFilesInGit Command.FsckFile.start] +seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] {- Checks the whole annex for problems, only if specific files were not - specified. -} -start :: SubCmdStartString -start whatspecified = do - if (null whatspecified) - then do - showStart "fsck" "" - return $ Just perform - else do - showStart "fsck" "" - showNote "only checking specified files" - return $ Just $ return $ Just $ return True +start :: SubCmdStartNothing +start = do + showStart "fsck" "" + return $ Just perform perform :: SubCmdPerform perform = do diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index cdb2392e7a..9a6ee220c2 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -295,7 +295,6 @@ You can also specifiy the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file - fsck (only checking specified files) ok fsck my_cool_big_file (checksum...) ok ## fsck: When things go wrong From 2403fece78b2f25d7369babd0852bc214127e5f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:29:33 -0400 Subject: [PATCH 0495/8313] releasing version 0.06 --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 71d163d116..64973557ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,13 +1,13 @@ -git-annex (0.06) UNRELEASED; urgency=low +git-annex (0.06) unstable; urgency=low * fsck: Check if annex.numcopies is satisfied. * fsck: Verify the sha1 of files when the SHA1 backend is used. * fsck: Verify the size of files when the WORM backend is used. - * fsck: Allow specifying individual files to fsk if fscking everything + * fsck: Allow specifying individual files if fscking everything is not desired. * fsck: Fix bug, introduced in 0.04, in detection of unused data. - -- Joey Hess Sat, 13 Nov 2010 14:08:58 -0400 + -- Joey Hess Sat, 13 Nov 2010 16:24:29 -0400 git-annex (0.05) unstable; urgency=low From dd5aa59cb2242c8c2070fc3faa56bd06d58b736c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 16:29:40 -0400 Subject: [PATCH 0496/8313] add news item for git-annex 0.06 --- doc/news/version_0.06.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/news/version_0.06.mdwn diff --git a/doc/news/version_0.06.mdwn b/doc/news/version_0.06.mdwn new file mode 100644 index 0000000000..39689232a7 --- /dev/null +++ b/doc/news/version_0.06.mdwn @@ -0,0 +1,8 @@ +git-annex 0.06 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * fsck: Check if annex.numcopies is satisfied. + * fsck: Verify the sha1 of files when the SHA1 backend is used. + * fsck: Verify the size of files when the WORM backend is used. + * fsck: Allow specifying individual files if fscking everything + is not desired. + * fsck: Fix bug, introduced in 0.04, in detection of unused data."""]] \ No newline at end of file From bfacd10b9628c070f190ce642cb82c8bf9514632 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 21:29:30 -0400 Subject: [PATCH 0497/8313] add logo --- doc/index.mdwn | 2 ++ doc/logo.png | Bin 0 -> 15170 bytes 2 files changed, 2 insertions(+) create mode 100644 doc/logo.png diff --git a/doc/index.mdwn b/doc/index.mdwn index ae5dbf85e5..add0505d97 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -14,6 +14,8 @@ revision control. To get a feel for it, see the [[walkthrough]]. [[!sidebar content=""" +[[!img logo.png size=200x]] + * **[[download]]** * [[install]] * [[news]] diff --git a/doc/logo.png b/doc/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..a92ff42f4768ee1f55480dcb5c0f323f2a083577 GIT binary patch literal 15170 zcmZ8|by!s2_w|4vARS7B4Bg!yKxs+o?(URMC8VWGKtPaA>Fx&U?(Xh-58wB{-#o-K z_s*Pq?mm0(v-aBShJ2QjKz&2-1_FVgev%Ybgg{^n!LJh{EI1POHx3*8gK<)n5QdZt z6YYXGNcNH%P7nw(IDw1}14;Rb2M!`Qf07kLm_tT`A*ApMNvMH9-a|f#epGgwJ4|!+ z!kL`w85z7vSn@0U^^6{&N2ywQcGk@2)-b4Ux#34^KWD_> z7-N#h-mwR!_k;I6Kl*;a%}}IP-`_vAZ4Vk(4|XIk3y$!PxTAGdFu*|k&P6)lAdp`0 z%Ld-bfNO)f{hzmxWC-K}mQ@DQs-Ia?twLzECxdEJ&DB&TYc^RKpc1B{sF{Unxmgq!|GZ{sCFMdo;Q%G@rF%%SkQ*)c7{gQXXJh6eWdGPL*#?t8NYGvD&1&Ip4PO# z!h}w2J|zlbcqTS<+LSeqZBy#mFg(OrF-O%#P@vZ{o+T&Oj?h}8MT-NY~U&=9772SxNqD)O>7b#?H zZGB~fi}~o=Rq12_qJ`vwDJRw6u?(zX{rt%rD36DY&`$_3iz6T0*6t_`!h+>~yq znTT)Bt{(lhDSTPoQKy->m77yrFo9JvR;E7Q3_OyAv`kyr(h0xqo?g7Kvvf`2s+_h9 z^uUJvpsIB7eO$E=jBWg#F6UxpCD+uevWBek+L!G#Z7E^^*x%lCPd1{4SSP1)4~dLE zD}fPN@9VEu5PLL1w$gd1?bcidj;)Eu=DQEzqmJ(awcnP8PzJ!!!+Itvhl0;-_vo{e zSn#ADu#Rc%y_ipzEmfY>MzV>^*#8_=i;w8w4qiK&PwBD4LS6&Us<~ukQ{nmRM<$eR zDnI&)!aH=^Qq-F!J$&g>{oZa~$Gd{rKaf@& ziEMegIjKD55279$g4y&Ed@5It2}z|q3nY+m{&(I+W|KfE$c6LKC#2f)+JUloc;^jR zke`WzTjSbBaS~aDi=Mw0GOR$j00TSAN0gUzjY%P5Dj8De(T^$#$G2%`tLuUM^UE`I zPMVjpKFYpr9$hVs(MQdw;F}EOpFrSYOBU0P6o&9Zr&-vw1Jhe$SU=l5t|@y|q2ws! zrRI9?;|eY8>k_E-21gm)%(P^}v`?;585x`GbKhgPiW%rQelagTJ@_w=AB=xa zW~J9@V8FoScAsOjsZE~eT}cZ3Gi~(m2x7ISGdN6XW1>x-B{1>^_=?%h(8v8D)Do9t zl!2d?!&`IOm^$Foiz|z|w-vJsYnIo#dCb-2T*BXLCI%$7P+v>{i({53xpiqwS>@Df z^DKsle;7THjVny$XN6fDx4`xwtK8O=IcNhi`S!7;B7|DM#$^ zY%r~2NoxYpx#q%>s!npT8v7U516VH7r?}epDJ%51RU2UN;jmOu_HOcIPoF)R;SA$T ziJ_&6YDeMk!W4RVUM9>-Fl;c|(Vj+6ymREc^*;hx_h`18m5_~Ql`rt8o4GdVu60#r zTO&6hJnULwHNJ`$_}d~jsSoA#!A?XcFYyu1wO&0&tbJeoJ=OCMCb-~Fd~72wR#Rcg zzkF+Zx)D+*lO*GgP)Yf?9(yN`xj*|O8pHTXyvc=&MmQh{FL^%5zkXPpQp zWj*8BmR9?_ah%P#>wC@NyNm*t2I&Q^oW^~|%dcSFFcnkBKteB^QXV0GO$$2}?#0l? z_~bQ>yDQxuM+UX5Jz6d=cJ7f^X+RP58#t9zKAh|Q=U%u9T{@pn-~Ti{{@JLg~7*yJypIX?$Is@o1^TdSo_7QFHzgGXge=4CL^}SP6E~r z_JX8Jy*MH7FdnI!3b)U*M*8rBq2VAY?$h=D*>Um}X$#1vBoQskN_(S+mlkt_+)m5u zH6`x@I8sstY3Drj2Gcq=pQ8ewW?pK$=4Nn#lv)7f6rjhnx#CJ7Cm3tWiRac@)e6BpRp6xAI z8P2uhAE^A;|CzE|Gu&L<_h-0KL%XaV*2zhBxCCbP*=^}>_k2y39m9wb$sD7-G;U(L zGN=EGc|u84KQ+7}B&2TS?5wOqzU;DM(bZ<7)e2vUCS(BKRiO9*VKb1B*Q1{8dN1{mA-TF)F(B~=D zC(GeGGi*}Q3fD)g%Z$>k#-YO@5ll<>&WIYZC7DYtWyEAiN4%-@fTx+@7JVC*aPHl2 zut;__a-v{cC~my2OmVtlCtnm_z`Y-dwk5w|Cky?Bu&!D>q$`1VTj{bKi-dliMWXFH zZfN06f7~Qw^}K}^5ADK6Lf6^juhJ930XIO(U;t9a`8W2|zX_&C;lZ7c%`^OF_ag@$ z4zU^yoX{fi@R&y@661t2f z3_Sb!lMkOZ{@ii~GMIjNOm#)g*^5lKMLxM>RR=zyAClTc5|F>}khI z!-{cmcCj9I;|aa1IAvnd<0LyG|9H!e!G?Z&MCLr%S$_Db)LHkmo`D+|hyB zbIXodQ08f<`r9pnRF~Vzph!sd=iT|F6b&S!Q|E@ttew=4@4QTW?GuEwg8)TAJ53>EBjRO)YJRW z4A;2l&(qEC12M1)BXx;gxm!%|UpB>0j-+*zn@z~tz-8Jj;A9ofUzMoids{6hDw|bp zSTiokA>NidPB;3^0yM&FucAT?fD}Ky&u#S@wUehN0p&U5%4MCBe8r5xCkmkoSn8yS@$ysBSar-*r0&zGW& z(e~rYf}GW@9ZFNl(!>hoWj&qwk=sbQ$R8IpV^|bHl!1eM{u8mjNG}~2UqwdMMFIz< znv%}tFTsK|Go6fP=4@8e5$+E9&1Rv~HzB&~ksL(J3Qr=IMg-Ib@47weDYu=aC4Bw_ zcP`2wWn<756~#kwIx6@IOGXzu|Jd1|1`ZeKy=xY|c_LphTqeGr!+1ddDpL!+!u`FP zNh>AFTZ_g&P!Zp~w_NZ%r%@L+!axzwD^_9k6?Xy4=bQL~d%kCR^F`W2lp?%XdG-Y6 zD+|8R&%m$cuF}`msj z;0L2aDA3%sf=rt8XK1VVcTCi&J{$@4c{aYn#Z9vvetU;T#HLyQ-h63jb=oye>Yv6X zfVhvDXSlXa*kqkLYc8G}jL6@%#wYZT$Vbg>rEH`IZ2RcP661CT2!>?06E)}bwg`(* z-lzpdUK6oTQUmBg)VlVq<}`;$Hax^-qL~gS`pRBE+yI~6iV|&)!BBJIlfwd~tr~)! zyUoeh65eU9sp?9{>8-30fb16V!PZM>z7(2gB65N-YtHWYV#V!76nsXd?;A&0-01lY zNdyT`{9hw8!$wUD-WpF!TI{}mvL3@>%pv01V1h#FJ-JprQbv9!`;0SRrp+Dx@JZov z5_#$oZUJy<2f^?-^m8EU&@*S9B#E(vAdfLDPf+6&yO$QN;*O1toRgIoixBt988tS^ z(Nzk8(c(RP#zFq#NCuKB+!C?3ggiSf#JD{f6D_?#dhzl#bzH~?_Yc!8DR}4Gl+X7S z7lh5Nal#ZWybnQOqd^O#fbE6D6aHOuY%GCz`lI$dCo2ufPGdW>U&!*xN4YkqTfK)^ zF`$0oc?@Ks3q2nWKN%D?yo68Frf0$1g{T4xO-QR>!@P9bf*zvA;arLTgt)UD*&6uM z0yd@HKjbT@x_lERMDk(3iZ~0?_yVLME0%Fs^dP3LZ)uOq57XPpvJ$|RW*2oT=`!~u z5z@@`Y~VDy-43f1_O(>=okMmdEjh5j&sbU5g|9iy_#p9X;F@@rS%a6ov1ZQ27F6+apFuO1@1gC9t? z(Pw2dZx-Ze3XCX6Q#yMx0Ydm^l)}i^Ue7@EP$9YfL0G!mjRKw)mRx1^hBR2lPq!9} zdSpFgfC#T?zt|>jkrs(zAk;kir;9qTI4fl{+ezraL+*MDAsdVbCARG1K<9agyd}|J z{%;XgkLNd!S9Y2m7*wBP?OiX3BMZ1cDkztYGiC?K`1!A^yDo{ALG3G|I$Yc<*i^0L zv+xoq3CcZw&m;D1iB>R3ll3|BeCU7^359xpYRb>tzcs!Vl@QDjU}o;P+Ec;3q&*P= zOoG6c`Fe=$2(>3gD-U4C5Xm-f+>N25sBFNbei+zB@1~Mg9AeGeRWi%a0c+-1K`T=0 zL?IvJ`Sq2w<|{}tUN;fHd}nmP_gA1&>bdP^#$E(v8W<>_Z7QxR-c^7(vx|ITO`jBa z!-lztWzDwHgaQpUFt;CuaQzobM6$jkaswi8HUI<5A-yW*6J8SJ8|suZ%G<5ig9Yr9 ziGSmDR()^5etWy8@(F-;V_17h^V7rV8Vq4MPP20Y%-@J7F&^zNvmYj_yyaLvY9X@u zHuOG+(nZ(W#M-o529c{2EJdcj{r*4EYoFN@DbbLr2po!me7=dDoQcn~pU32rA{#-_(FN0?#&gMmWc zdO0N31kXq{s4nX#k8~Y%XckUiHl_az8~}6&EA}0%%>sQ^?G|lRe#GmETw3c_BPqT& zL0M^?b-yTLT76sAp3r;YjA3W|%ObBNuW$4UeE0WcF@(+Y<5cNr<}|o2F~exf5ufkz zQXwv6xP7g_@FA}@4`5NW`oEwJ`U09bPIIKGOGfs-`Y0l=B=41)THD>=0woD-zo;A$%b2#5p! z@Pwte-uwrESY2TdQaM~wJ&TVTyblQohTVK(LBy-j&o#2=8I5}arOF!-mHMlR;eH(! zM1X&WadxF|T;elRFvg6ha=CIilL!rn{C>T{aI}rQLaeC;WJLcuI#+TPU}d_I?Cu91 z^`Q{E_8f~!{IS2u534xGA@ZjGjTDpGmH*APl5Z1*u&wU><}`Uzpz}J2R^-3mM-O`V z?X9AakHLjbzfWub#p|ncHBd$gjoYMf4X|}~^T>L?4$VahmHj}~3Xg#%BXgDhc}_LA zX0J|!JA_Jf*2JAt*2;7^`gJg$Y!i}jm@(_jDOjsvk<8{=kRGc{{yN7=D3<=WhKV4%flr*9Iv!2lDlLQZ) zw;bqKFh7t!L~^F<&s3hLDHI_3)*ZKcl}uVzX%;gK(BXuZki#w{Z*6VCRDM&;kCUXp zO9j?GGRf+QKS`b;B_;E^y72JU)z#I|(6F(wF)|__s0e$<7#b0A$S3|II=b9py*oEI z7sk!jhHHMI&dIKRmnvU5lnJl~+pT2>(cKerudqQrACZ2d&dI3>U{*!kGrUqD4-E}% zetx{OEhzMGA`;e>rHK0ZatNDRZZL_J914YPf_;hJ z2Fk?c(E-`4)v-L1Po7)hB|k6}?JV7(tC@m60bPdq1qE}}Hd?l}C8E$wixdpun#4mh zjRb>w`BRS6p^H}4YoTRsV3NeG4;`4zX=|M6xjSRkjxV$^Z}F|SGqitTpymFkVPZl# zB{xTrw%rHhFCRp9D!@;@4>%$C_)=?L$-S@jGn_8b;dedytImKG0tX9HVMh#skxpiF zlGQlQ%eWYh^>s#$&cC(1j(;Ln&OVGvkmls(wts{C2r)R4LaZpjS`@GS#1VjDfY#MZ zVmsJoAu26B&>cZoSy{gsM|^C>_=_W&XLl2EQ@s;w}4PD3_L& zK7RZtl~WPBml+I5mBoonvVe`h&Kug|h2(XySM~=p>)BES==SGHxJP|7$ zsOBKiIkU5{<}~;t@&6IlNZ?GCxxYTH(JXFyV)G`#{AHoZl<2gMEZ|YN*x>4WIjM#h z_2C~(y+jgGTa;evFEFMd`o4q}Ke41sbDz~c~ z>%5Manldvp!Gy$0K{z_O(b&GUL*k%ad$lSH$sjd`<-}VRb{70Ad3)X9L@7fxKAJ5N zDB^e3i3`4cd$tKiC6B~oGuBA()Cv*BA|`&k7?*#(S&tHWx*_ji!Syx^%_>sezh3su zNJ`o{I*N=05i5$J)LBY;A9^D5$3Ns{8 z))f;IQ(-wt7a`uk%;kC$VSmhP&>j#>>N8`_HP0?g8!vTzIIs5kGb94sG;;K0hXq<_ z)?dwHxj&WLqlOw&rj@|2inZ%1p5d~_sk?i6mOYP~u4H^a<4|ISRNE~_+WrqCWL0q- z=P&9}%uIw~fF2lpJ=8$ba6~IUK0aB7ieE^rSatc!w2tvFENq%kLk*XOS_f<@nX>b{ z`uh5_Gsg>z-hjgFn(AU6kLQ*OU0rvtDtneJNj+;@z1LgR=jD~=DzdT>k8G?S7dw7_ ze*Q9wD_#M!m1bdGE)g#HC_#;l|2boUAXuI>I~hI%@J8Lv^u@ zMjAAVr@Fhv`Adn|O;L886kSR`$;qh}PF{I=n3&`YnTn|S%hWRE%UY;j9rTB>m`MJB zo>S2YkSe_i|-EO0CRbGfar^SvDR9JZ@HtAd;yTKya2_Ai(U#N_U%TU5JC&C$xwOTtOkogM+47`4z>NI#+@}Z@L%SVa~kIno*2379Fq#&L{tkxi!SZ2D3YYm ztTc^C!{xVJd38I7$oGN!r0W2*TeAPdx<4*IZt1zq!p)xHZbbS=eH2=+!>G7hbVu0n zE`+xHQ9k!Ey-~IB^5g`s({pfm_(=K_GFToTI#iX?ts*-X?xlAQ=M2yIX-{;xeX632 z`@5b}6j*wCdc~?%Bh2mw2CKCW>#ynEGHD753f_#ueMPe;vla9*1O1<(V#AK)DB_!& zS!*uObb)BexReyB*R)NcTCY?;f7btV{B0}=pa4<3$@4>Tr`R@A=iYg#oUzu{51u4+ zeZIIuC&7N51f=U?COz%Xr= zg=UW8?Crc&m1YzXFB?q>9GUU;(V`0zmI+?>_&B!rYa?u?9{;<@?bvyFbskfP?NfwH z)h4MT(8}cZcYG!h@b~fYF)W`U$tjz)MnEIN6Y%(}%|xo70p;YZDk&L$`c6(mgXsNw zDZr|&Y%n@#{ei575~0WGkM5Q*R#O$DPU`G9_x1UCWy|T96c+?S(EFkVz}U)H`cqL^ zSr`2kCX5H&Ja1e8cN?V;_~^=HukbTH-8rMt$q7b+`MXzd&w>P z&pnEmgv5LuI>yVc4cb4OK~g1*nUt}OBeWz|M*Jklpin-VWPZ5V9k)oSezw@A6K#4p z8_=c-G+KPqZ0hRnep^`DY{^03dGVx}&%wrqrJ?=s}-Z|zd zCdb7E*s^Q42IQ?HP%)h*kKJZMClIig7PM-FwDA)*(*N!EIJSL-f7$31u5iQ@-i2*$ z^u-pcl{TOC)0+(@ktGfUE;S1rQ#A&eXnbq?_wQes`}x+|nqi3On9`qC)5-`!ZaKPm z!}Zct*ua3<=5gaGT~~YF4z{0A>x_z!dBvI4S1=&9zM%_B%F7>j&Cw;8Fun^@%ny%< z*p$)_4hp(5XC47)c5Q8q-E7#{%xrsmTbd$9_rP;dHn2pmW$9_?GfX3r*@~nQ-5-F3 z)Mb@@&EYIEc7L*)$pK$wwp4Utu2E+IXcxTLlEdl6HV+REkB>>;z3UzlM5Ly5a&-Ky zXaNARmX?-=M)Hs;d$Nr7q*t=c6u(5PH5apb*@t}P_IN*;+IRQ+Xs3tyOR;yCmEz%e zm?R|Wv9aS}6=(}aeUDb<>J0C~{&Tg?W<1(m2 zBJgze5i3iR@SQA`G=pVsWcEDT64q??&xMLur!EDK-Uk$E*C|rP&MMiBk*oroLxJ@P zETkwjUdjZvHWnJ)s}OHW6Pf~WYO7iFtKQD{PFb{RdPsZp=OL5MOH$iIjn~yUHut$+ z+e*pLNB2hE6-DCN8ciNmY5acZ#I-hUx$+s2|H}|)Ttshcv1oC4w_Rf>513V2oDQMh zA)}^_-xWnAB=o#5Cjkb;3&ITRx?b|Mi&7K<{1U!ClIM}~&W0SWGJe5rwc_u%5TYOH zgJN$0c94{m+)*ItdT$w^&E$JAD#7ZAlx#2Ns5Ly1I&3OZ*xAoY(wD}J!La##ywmxN z^KPwGI%jOMP%R3N)o{mdhW=exBC{UJ+qdulE7U>}2>7%^0Jll;_xCuwJ8bgGfF*xt z=KFwu9FH?bQIu3vXd2w<(j+<+#(kY3*e{$6?&fj*H{w^=OSC%7y;h7X^W+}QEEidZ z9~CAFR0`MD*MnD%mQXqXq@A~G@!lFt{)gBb5T1&S0oT*T=p zaEB{b-hIUTpY7Y8v&6!@yuDoyW=5x`y4~@=(O?8%*x4ORkhpjD^dJzxB-=YA^9I@f9_-cyb1#Crj-D|;d@m2?1~n=o0u>#7WN`4|;d~@DH5KfNegZk#yn{0{GjHitZ|?7% zZZThOSK29WH7P-P!@ zO@~tA6aM##kIxz31%*WR1F|hSTjAv1(NX%(pZ@`*#cI@jQpHGy0g72fGRi|IqJSS3 zi(*7ya_<1aVl^OkwlvvtkZqQ!O!r4FjnAMR?pqtEPG3NKEVcaEK$FKQi_=~!3ySPG z5)x9S=}>4$hyo4H!aKq5Y=Da(0uq6jwmQEYw$rMA2;LV80k;wKnhU49i`|8a!ok78 z#>U2nhlik`*Cnc`*BjWvMXK~dIZ_HZ*C$RXP^=IzXQ>=1Il0KtQ1Ew4Y8^2L1}qz7 z+urUV4Jy@%b#=5e-;mbMLv18XEbv`VMLi=jD{tfFWi0%C$kYm$uef+}|8jw&K(%mF zsRaPq#6;W<7P@$;VpUIfcMA@J)dQq!+aIrc;dq$A-s(#&;O>unrp5o7>dy0_;*Uq0 zzy>gU_NX~WN_wP6tE)ZdkHA^IpS;x1Y?gW6O-lkBFW( z-SIzTLENxK!hqVz_AfpX8Op#`_165lY^OQUWw5PEogmExVC3915{{vy&3*>O>?^5= z@`O@xF(v1kFXo73uT#sjyyp#4vWzC8S%^UdzX>LcJU=wP28>6PEi?VUw`_-X;Xp!n zTQ`peC`5Zx<*zi^lUe-BCTU}T4Y*(C9kYmC_xN$0W0bJF#ix1TY@}HHRfGR38kMN0 zU(15)c&95pib{0BNr~8B_>U5p%Op^XImid9*K9|hb!I-6RZKzK2+r{+WC5}D3mY|8 z9C^GP8%V)Wxd1(2kLKg3*WBq6I_-uB#{+|NA|+TT0$7;0xje< zleoQJVk-qgGdi(1>gXS+ev*D0ay#_T(T;RUV%unm74(yZ00f67K@jN#$O8lQ*RR3w z|B{=&Al$grOgaFnt(HSPbr&#tPxiUXl(@~!bLt>TFZOF1@U3vXw_Ae}+5Bf~9>w#p z*$hZhCVL0KianeBXZAN4&;rY(&##LRR%C18d?yap=M(Sb8cqa%70L@5{7ptAn`Nwf z8$|aqt=FW*IqpC-#HZvvEi{=*fb85=tg5pNMOz09P2~5EYba*0{ZJlD-y2|!SHZ&t zKz)!lVY*Shm0`hXU5Q_NJ^;1V#;l8Lv33P*;LYX^fjok8J}F|>eu2}36GD&r-22ZR ztl96^@ODXj{lf?cBObW)*fa?WP{glce}wpFcdK7Z94-O&bgMdj*G1~gH)ojvy+uJd zqbbcMhyRB3T(yu%$CD2H=&9YDysgK=jpPX+bP~-c7mY()ME*c^M~rMF z@Y&bRY_lYG&Ux#rg}#3%L>KTkZK17(ZFlsMK=9Wtl2FE1s4F)R$b7I-NAy;dDDhs; zgH>LFdV=$U5t6*47tS?){Z`0ZV0|Fm6AJg;5connc=A@*d$h@9l#&snc}%0F}J zfYCd0;TF>8^7W6e5{$hCD#vxYa!Vj@AQUcHee(%%R?F4T_0w0-#9dVUZ&cu;a2R8F zeC`ft^@%UT%60sZp5kKPBBgWn);$vv96lC}qca4;glOB-PeIuK(+77I43F?A)=DGe*&r-O#Z8ecMh~~b-eIT8 z>-Kf9kwFnScUz`kIW^|xKR^=Nu)ViJKdJ^UX3sJwvwy~QQ1Vn6hQNAXvNJ!dwz^b4 zqdpJXD?*HSS#1kL_W9unJ_0s{nnm?H5fAE@g>eZFttPmWh{C<&$IBb?CianxVIcgP|mQ{%@4J!J8=vl8;?-ifw&n4jC{YjhKANHs6IzKe5se#BdGbIXxDJ? zSEL9(kb>c!&tIAG2j2pt_Q0{R9vW)7L0E|JzgqxRom_Pq57kZnQOZ2L@%}fqI|w z>9e}p7}DnumDon=y&{jNL8asovzm?n%^v-xal_Zk14?EbdM!%LzMRvnL_&B9@iqUO zoIdDA!dV4l+^&1}1wJ*b<}?^|)%S(p@nyh2W4d)cmeU7*oLb)<(RJ&i1q5s5L7_lz zV$EfoMCoPcmbMI`b`YZfFlr$bns(c7za2lx_{pOq|bg1&A* zh=NPW|8E{*{1s@U;^pIBNDdjD1G0ffR#(b;*TG|Jy1T}4s$59zPZ^xOu;&u~PEteY zXW_66X)j@>7>8ihsXMPJD-Si|3b1tvBi_v- z+^xXz(r4XD38t7*{M#9hj880x_C9uN*Ly8DjE&;8U7)RHFpm7scTwc%7GnJtrjNj?doGy$q%53&7(dp2UgTc<==n8-+}&VZUr8+%hoP@ zggF{0NA%epZRGP2eBOHg8p3&$eDLNDUS6S^%IlN@;H6l`7;%iU+avrdt*6wl1yXT{ z9EtPB-;L83w8cUR%s0GIoxVE%i3Ewo%(KT^C4{#ERWW|QRAtzLag6^IfO&} zLv+s`cGa#Yj{YE4@f{smeSL=c2H8HPdUM&;j;Hn2lD?_EUlW|B4k!!o=Jw5AntPoJ z*^UlT^yXcbvkwa<_S_v%Fw8W5(D6m&f*B^qj_aqX z`cWI*S@Sye_3Au$9s|tb8+SRAtB8L|?FY37%V7_^@?$tesR@qcD&Uc?lYyi&9rgEo ze{s=Umjm+lPbTC>0U!O?djm9!*5(-#RA_wZf0qXkSIMLu}~J?eEMD?7-U27-@Ik@1@@(XjqnfQyrJk`d;7sG5qq#WI9+| zCy$;to4yvv=Ota0OZVu95=@&JiLPPv@dm&ZJp^|@C5rE!|zwocb%Pt4Pq_$gT?kRb7So7dWu;5>x>V zqYo+#q*DjPpicOo=SnWTmZQW5efaJ9hQj9nPS2j*1nrlG9?5H%mlO9o{ros$cQF7B zvY0a?#<+%S1R+T!yjnqFTcy2B+Tn~8TF|{?M43b0LKyx8qVq?s5D=GA93z3ivb?CO zxB{{yd9$Zrp$Er)Dr?x+kj4p&y=3n%Fa1i>~wy81xdn8|rqPR_mD|Kb5FN<94RLr`~h`U1$^sQLqVf zzleo=i|y#0Sr?PH!*XNKPOm7IpZjDGL{I)OLH^W?Nd!$$F?7F9AHKpSVvssIK z(dqA`cS5XdZ2&$0JA^cX{V95Mtg6onXwNrC@NByIql%vY0oIHm`bdeZc4w8R4Btucd9Jjpf{k3=da#_LFB7%FB_b{ zCH2xftvoRXqVF_^Qd+S$)YS|f6?U5~*CJia!sz=EtjrX3QKB`kPN;?-A)!7rN9n=q zNAZX*{}}(hCB^uS$4mM;^|Y?2Dve{UYlHJ2A|#oI8Ts3G3-{9(gUH;(i4F}LO@qAp z=gxH?Q(PON5erQe7oaA}hk<;QxCVCuBvIf2GNs0DMzn?mXxuse1@MT(%4N%{J-Dws zbzHZzAT93Lya*fcY;v!W1$Q~#U;~vVza^?7V_TW{f3>7~W(BxPh9j$_em^Y*9~G}s z)80G)neAbIK=1~X_YNPF5i!6GXB7X+N=ZMZT}*5kXu0W{P1-TtJ?x*622hK|nTt(S zq_&qh^|u1}iuWEbLFeS!u(@|~cy~T{@r)(N#8u_F8Ax)eDplPCMKLcfdL=-eI@$!n zNAROkTp7la1iTh_w#5z)|HxRWTOu&S6#43sJm7PHCf&D| zD8A~bkpD)kZ~C_WYl&~u!tq!`mfdyYeHJ?a)-5dazNY>q=AOIUY|$5>3JCh;?^8H? z{*){a37{%6?DjBHt<0!UUATBEt!%18(W!9|7rma^2TV%8wu&aI2>V@@)AVgc5seZzATB}qGJ)`RFDzsDMn1lLi|9EN%h$J$ zD_*@XV^rgkO_M6KYljK)vFG#Z*2(_z@cpmxK2R`3ai>!RNLpAlQc+0}W71rsD+Sd{ zFD&HP(nSO9sRJeBg{pvvdhF1IwFMZZD%Yb}!xPLhF!Ssbvrxj5;`I3T%j~cm;6i|8 z3}<_4Kp{+JhX%6L`D5MOXb(VGQcIqGo{RlaoJWg(&&)&rj)W{5tk#F=}E!dh2Zg zB6y+!nk9OphtQgA0t;a|`NsnDe;+5+-#L5E Date: Sat, 13 Nov 2010 21:30:58 -0400 Subject: [PATCH 0498/8313] update --- doc/index.mdwn | 2 +- doc/logo_small.png | Bin 0 -> 7154 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 doc/logo_small.png diff --git a/doc/index.mdwn b/doc/index.mdwn index add0505d97..c1546dbce8 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -14,7 +14,7 @@ revision control. To get a feel for it, see the [[walkthrough]]. [[!sidebar content=""" -[[!img logo.png size=200x]] +[[!img logo_small.png]] * **[[download]]** * [[install]] diff --git a/doc/logo_small.png b/doc/logo_small.png new file mode 100644 index 0000000000000000000000000000000000000000..38a87e19cd20b6763754cfc89e99ac585880bad2 GIT binary patch literal 7154 zcmVPx#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0GsElL!))Bpe*gh@m}RCwC$ zoq2pz)xE&ax!bIn%#xXeWU`WkY$O;&2(qbB=u@Z^Teo6gedQ@tpY|!OT5Mgg*7vNo zSX!x7eBe?+E4Z)}LCPY8H9#PceVZhcnMr29ckXuHA6KSh62c^zYdZ32QBpey;TsKa zZ)w{qA~8{Eo~ECAn{`Q&GBJ+OhyeAl?#4jNVb{@P-6xCm`63b#Ksdx@Lpt{G0({UB za#~f%1V+5wv{{KOu1%OVNtHP8*V}^aM_k9|nP*#+<}2D&8Q}=>JJD15LF+aVpkEcN z(h!<~{o>aV+E!ORZ=#IP75!38s1QI9Ksh2EWuJ#$tJ|>p>>3PWKgnDk4psi8Is+<} zRh{xo;gb#irdR9M0{{TJ(j{eym58E<5b6%~tg3vfeBXEX+V`^p8~IW6CqW49vmacr z_m;|@vjZP5YhjG_uZwa1^6Ya>!PYmf{^NC)xdbGlQM<{;%DCgK=X?jZrdGx&J0)+i%fHK0^hytgcEqpRtn;G*n zFG?z(rZ0NCd2^5t_Iqc@seQ~!XkhRa>L{hwSO4FYPd-)f^J`4gWrVSp03Y=80kgt{ z!I3g)yvU#IIg_GJN>L@t2xBZF61Um5*ZCVB%vsU51T#8{F8jh^Nl>8w={OWe#*Gla z$jq`i(9%iU`xcfiT0$!$jIpS26_0pc30%Z*Hw4V=*blk*?d`qw` zUpGZY7-PxOW_mcUlXZ<;QX!qT&fj38JFZQbDI<*W;2J4CiIGi#){#ppq|+YnsWedf z6jgHH8%0Jqtie^`s(js)Hrh6#7jlMNd%O1En0UR0Q1`u2k~Fooe`6reR16s&0APLn zhD!I@-<7OUk3e=WT?HW|wE|23{^pmt2W~nV=KUIZ@=n_Rz=_AU*>>ELyeN`iOjJTT z2`h3*mmri{~>`5{?0f)&|aoqIR8e$*6b*?7(ROl{h5C`w-p^7Oj8zaDfR z{&mqW2#gr`gx^erje(Y+Kz|LP$oG0@2>jk{-m;~2TZSg>*`g;3CPwfADwBMe#IN&n~4yESg-h7ghSDc zuvu#iw7hWs<%U4himZEYhVvwLP?U?sLWJGhh7_zT^LjXOT_to0hcX#f+-ZJ+Wd5ugY3b_J z1cfQ`01=6t$Wa&tA*}J$t*%;QAoX{p-?lVmu>z+CTO}YSo_<7wq0gFFgn$x@D$4lM zd30Cjp5xt>rN&}uF1L$uzT2|J&3T?Kd|XRtZ=SSpfZu} zYTlfrw1n5~^%W8${$c+@KtiuFRjHxaqOe@Ho(TcX(~W^368+TKrfYh|u0zX4 zCRl3!YVBX%Z}~fk6L7$~k`#}GRWUrQ1FTpF06~D|>!v)B|Fa~e1&4T}zxnyWH@QH1whEU%!Y2i}qVjUR=Mr^Utw-F&P^0oeY08j^G7m#p8v(|5{c*RSk zYOIgvJrbS*9R5mO8R5T#7MRoSvpB>LIgbPdX5f(}A<-+KfDj5u5HIfw5kS)oC67*d zFp}Sd7?}x!kP(^F?sGWA1AK7f%=LOocex|AxMtbmduV~_RRnn||FM~-=^VOj&!-Yo z#*x~IqI?q%;YDwq)cu|tmu*!it8nFD&k#?=>nVM#st4moCcvz{+;*vB$QyH5uB-~!8>pH2CxoB~cfgg(yy zm-@HdtjD4>I~mtJa~b&8{QI{l^_z$YB@@~dY;lB~Hzr>9LiNk8kZV@LyA-65D2l-# zV(8eqLL`WY4ao>&X+i-2P5$QApzUhow9Ty_nJ5E=Q2+qal4O49QNVLCziE_H3S@+_ z6h>gqdoLzXrrfyf1(vz9E0v!*Jm*UdB_t%DEqVe(c0v$D zSaraqW_W3n4p9^dg1|6LddVJSiT7)yJVzQV;Y zhGFb>`+)-os;jG6mVMxX2Qo4;5JDe+{PEjwzfF>4G?xgu8>N;0PyhV@;i5~X0+gq(&P zk3Skh=;i&D?lXCDIetDE;De=xl3t7vLbbKE8#ZikI-LYT40CyV*pJExV=SLus;F=kozwbx#wX*yEvh9g?7HX$KFHl#6=J@)+;MM8!qeUE)Vg()V- zWdiiI8s>ZNz1Q8{9qH$!=yc$IUkt;ft>waIRun~($+U9i%JlSfAa}6EP%ijttm@=A zBCeC^ni^jK5kL_G+uhxL;J|@!s2PT*P$<&U($wnypW;IhL|Iu`adELip@@Xq+1a^i z(~vRgd||IZw5qB~O5VfyfW=~2xpL)<88a@&PdYivvH*Ztvu3SWu|hh7bV8+4 zSy54O{``3`@_QA0!x2IVJ)HO6V-K#c+W-Kt(H$7X(>1AItJ?JR`2GG!Eo52t_S0*b4<=wY@VN7ZRi5hBU|T?-d691nv`p4!s&HlG zu)18X$mm}bMZI1>b?VeGi6}3$c=6&;C=_XpYin!eq#_2=5@;hKb(LYNoAWjXnseeN zE581igFqk9;$ji&?OAbOv5Jcp*#gaP=8&$1Vo1B~+z8fHf48u4a4mq3|t|PoD z>3Xv=;X2FQI$uL$py{Th>y%%Ara&kZigZ^}JkqaCE-NdeX*wLZDin&0jEwN72qC>* z-`(9EF3La%JsywQY?cuYonJ@@i9je%oDTrdQ}K7EEt_ea-s@3Z1VKc4EGfJ=lVu-UZPY>{Ykc)(2{Q?d2nnI5&%L;> z>kudQZm_UeEJ~#^9HBr6Ns`>OX%o-$5|1lB6&OP3^?G;i+!;BQh!7%4aw3*Vv2mh; z0Dv~Sy&=$et!X9!2}TTgSg%=OA~CWrgY4|=$YLgjVO?EaYuB!oCj9yZ8Ce$%1OjW< zt_>p(f0~(@2_~S77emU5>{-uw4WXG5myJQFz^L0&ZI$K*?{eFKqK@10?>FMdErKJXP?EH%m%Jk|d7f5JEu^c%GLOS4#PN zU7h2&<;#~}@d>c9AstA*bR9KP`kc70)`q=Y^QT>%d;8T1=^c~F^y44@I9%k6JTg29 z%yAsUFf>iGEGvp4yd=F&7z%~H|NZYvj!>Sd7`lAXbue9>I$4_m0Jz|-#Q>J3u@2yw zAcWJWPrvJ~yTTb^WWWM1FOl(!)Niq1!GiMg@?NG@MmRWevE66g=Q1@JNyNolmoHtqRHxH%T(r&Ik;PSlAm-1Xf9IWdN?FPTMK~^e;pg^$Y6x|S zzDRmm<*6nhk)zEXI1iC1E?l@UH8pkr{{1IUo(zRTBuQSe!oqQ!AP6}*ISUpnC@U+& zaeN}W!10A}N5}3sLVJy=RC;;dcagvdsk9ai#e#x@jEsyqbLN~refq?S6J1?hQrRUk z(vfT?&+|&9vZSPB`t<3!xw#gL1x(C-nQ@0ORh^QdNt3Lohx2+^Z>l;4i(**AMK!fr zT~t(*o11&f6(cy6P^z_g)&2gMYqtR$IX0tgpH8ndsTcgn^6pC;KU_$d7 zV}#*JgEdoEhhNtD>v_Z%=<)_t*^z{zD3T=mouKG@-M(L!!tg9;CvEp|-oDekvW&-YMv5tl^+ox*Jo3^OeKw}U z%lT#+XMFQK);CZKA%yk*#;vx`c*KQ)rK(a&4JF@6E|>CYB+C3o4`Jl| zfGgB<%yn`{$8KqVsajwCzOI9HzJ`8Twlkeop7V1O%GCH*3z53n?)O?g_}I2xxco!x zWr<}t6JEU#64;O1x9_zd=wLc&fuRK&1K2#v?6(@;sOFmhGi!#5@_h)k zW~7W$;5I~c)S(CL{(i^lur>oOW zyU+df#G`W)%C1kG-$u88&$?LZl#$M9QWzh}m7kS2!V#%7)?JKaW7FS0Z2LG}lXgq; zO}AT@k~jtW+pD%$cv<7C-_pAEp!0A$-I=4!S~GQZo-SAVloy2)J*Na9USph2KwJhh zf=NY#zv<eC*uxj@!u75fXD-mxQ>0=zAIgAT`~FJ!w0xo&qF7Fo~%w< zoxduM)XHE+(2%~~@P8Gqma#z~-3*+-K2nW0Oa~Ndg)_~GU&bZH!7&%#+DQ_b(B6*?Y|6aWC z`?H?&R&~l$eF41m{!q3L{`z#Qv@PAI?K>?BvssZKBN-8dA%T6b<-@-<|Gg#HHdxJz zM#XW6FEGz>hFlvO-|1o;K%S;!ea3xmsr8oYE%Rbt`jS+6nRUtilkcr?o%mzz z>$E^eztT*WD~wdw3s!q;5|w5nWf+M9v~*5RW;uXWSuE9IhdS5Vp~_-&Vi!NypAe zt2c7LMu{t*E_mFa(7)Zh359gAW9S$Cocp5GzVL?xW=R~%KQdxNFjuN&VSlnWu){AEOWrSlXht9x@@ z-Cr4jkvRfQyiQxWQ0(ozu8C`GBjyp5=uxOF*`16^Xv@~SmmH5 zYG2sH_O#KqVtpY7v9TJpa8kIh>)>Ad0ePpi!D*qF^K~*^rG{zvn6|M;b6LYka2}npS#d0_ktrltt%{0SQ4WSwH-zTYE zZro%>V5He#S)xMN;BWd(^~*N8W6Xb_q;?saG!Ai&kW&U0Ey7Nw>%Eo_T4~!j`IT{m zRzxBrgk)gRB2?i@4X$ysZgG5s4;aJ|00I(ZV9|y%dAi&`T=hbME)N?Y;e(I!VGvUh zN*P#G2oXXWLOmy;j1faI?^hc_I0GSU3ARy?G%L(9u&4~F1gxU_`2Sq^gO~S>Z5%IM zz1CN+!c|F1iwrDUUkCtbr#nAs-*M1+WXu!l=0Hmi=gzpactd6uEyAmfS7m84H#NOK z=0DM-FfFq#`H#%y0LZoO!5~akr;8+DQ?zdQWyb1!%w!up0D2D_(sE~eO(8O^+l@-9G z<*vh6B8(jCk8L{-IS)UayE0joG@MZmf7i`=c65IJVe2+W$aSmr=KsiCt|Q}Qeo@LG zB8BjPQaHJ@^YiSuta~$mh!5#8KjG)Mu>$u=`;LFuzc}Nr%GPGBnYy|xVFm#Sncs*Y z49m5qnR63nH~5=ELP$-hLDcJ>!iP!EZ`)?u@k#p*8{GjAdLVnnttmHa2zB@XnN92s zX{3lq6cHjYyux)NUzZz4XsNZqFhAQ6e2$N>N?$^<*(c%xzC&d$AB zLcM+777yn=-hFC@@hTmu1M(u!(bvK-%dPF79daD0^*7pR8xC=+D%nUGHG~?27%Q>? zK4@ng6h>~H^Ueit?H_CZLSba7vE+y8w?}3z^7O;lAPnmbE3y}Tb*({LJKbSt93IX? z3yg?F5+hZ(DxTD(sZ;JqTLuB_W_xsG9Ep*#&W#r>>}3EELbSkeh!aEs0fa#ehd7Cm oDqL<`8pjCxf*jfbvJ6@NAID37cnRF~@&Et;07*qoM6N<$g4p4e;s5{u literal 0 HcmV?d00001 From a3ddfd8717d5224d2c81e3b6e42d457329c08956 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 21:37:47 -0400 Subject: [PATCH 0499/8313] no link --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index c1546dbce8..439e2da09f 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -14,7 +14,7 @@ revision control. To get a feel for it, see the [[walkthrough]]. [[!sidebar content=""" -[[!img logo_small.png]] +[[!img logo_small.png link=no]] * **[[download]]** * [[install]] From 422cb1e469849b1ce516a6c4e3b2e9edb1caf4b8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 21:45:37 -0400 Subject: [PATCH 0500/8313] change --- doc/logo.png | Bin 15170 -> 16760 bytes doc/logo_small.png | Bin 7154 -> 8163 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index a92ff42f4768ee1f55480dcb5c0f323f2a083577..9667bbd8b034f9dc9f37b881379770281f1a7e44 100644 GIT binary patch literal 16760 zcmX9`1zc3!(_VV%rMnT8mhM=(K~R+D1?dzJkZx&dX=#v@?hfhh2I&-#j_>&W_eWmt z-h0nEb7r1-<{3iORpqcT$T1)g2)2T}^lJzNffanlLJ`4{nN{oO;2VO|YdJWiY=m+T z{DSTv|K14#!6f+ii2zASBLxT1oE21L&=xRp5Ll=`bh%$bAhZw#X-O@&g`;#=PlCTs zed4b3IVy3-Em?7ik+f10WN#h5)6(M*5^3obilfM&XtnwiXux1(+4KYIFv&y|WDMJ^ z)Dh5?QHltGFatdwEu-m z1VW5>tuAVk_e&^Myp1&-lgsoh)t4{SO!X|Qb$`<;+;hJFSS^6j$60JhgnAz6>ojRI zV`w2Stm#Upl3Zp{)Ra(;L4;baVzAe99+(1y*hR%~roYamE*5qFNw+LP+9(nd;MDA% zVIZQ$QKsBY{L_&y$^pq(oq|`kQ|kwdX7v;WI)j&Yk!nW%(6m)b&le55Hk5F@;+ypfzH z&zOyh@eBqzk4Vy8;z$!5eg-f#X>m?}Nghjt$muZB5BcfjVPkSnO14VNiP=X6BBOns z)Z1&6FuJucy*K7b`vcE!BM2(4YoAtUx5`%y9ienQzZf37x?Ml>XCipPzqh@f*Q?2L z7_{tpA2{~m!**r;tc?%0MmBWdm(~ZiVK-O(GiI^y*+~vRB5sMNF6ETEq{oZyo1Sr(jmsJy`O zWn*r7UYiZ`g6#Zx*Y;Q?G9Kaxs$^`_R+}B$iZP?kLCXurZY87)m9O3N4-0(6-fhJ; zxNvCSDF>=g8f_>FI6Ziw@O|<_8zy#A5k2mX7@62f$C?L~>3|S|Gg9Dafa(x7O7Ml(8P3gT6?jtcZ z=i1u6zLRJF7QeObJ`64pJdEycv{^CgA0!#|DP2sYTnJLZpLBDE%!=kba}zVPRgV8K@!*O9`zQOGwD9jfQuJrV{*e@zZO`#S zVHgBKB{aAFFGkm+JclXM9;6q?cI}DKBm*VHgK0yfoy1l(TQiw)5^-)n{z`%MS^Ix7 z`?1Ps)!JH7Bp?@uMvr`JO;ykC^RT|3qve=AI>#HZ;$#wj*V7X|pN&_U74MaOX$T&V zlS9D?RlK8yJDD`}Li|(0<5~ElO<=_^Y04UyAbf%C(pac8e$alqKQs)@lJI*9yXJ2Y zsC!ybE#&;h#Wnr@-~3dt`*^uKJ%fK4-6&Vq+a7T|n5`pV$Ha$XPeL09k925?}yQ2z%1|Xt5#y^ZCOb?rhet|hy0E1B@Bk<~dFZ%tv{#w;_F@BQf zCwHeUZIo1rKwl}T(z=VSyJjJWpOZ^k-(Cbp)&DV{j&Yt&I35Xn2=sl>AlG`DNvGK4 zechWNG+)Zr))*GnfyauqByvB8yb?vBcR>0iG5-2W@m3c&lrgRA@$W?J$Jo0Bh@aLR{=Q=V^n#}g z48`j^Nzw8;H*qrpJ+h5Tq=7~@Th1Z+N?p^!k+sSK&J8m-r&Lhe$B#?a9p^*_C&znO z?SqG=>ZrTm1KV~W-4|E|?+(Spi%q`1D8squN2d##Q=jgvRf1^Y?RI(^=9;~<(QevJ zu0E|C9|-vpfE_fsKzVy*17Q>ce~5Vo~a<= zWy65c8PS9-SV2fiQ1Rf$hcgvaT22sY=$xOG7xsti(`LQX%N|?DZWlAEfA7a;Qye%Y zxNcZGayp#}%0{gMUULqOxsVS2tIPZ4_K_lqZZa58q|Dp`7C~hoxX-X{nu9v#7;NO9#KEBx>Ehx{l*37gJxDHxGmMGP56r(J`260mGJ6k zW)l&xv)=>FZW-X>V%O+8b$GwIFVE_sS}*JZ)nnxl&jrAcUdCiQ47`|q)40&`H0uz< zWp_@FLyvqY8?TET%o(U2T0112qPNpqe8Jn0g7|Mvqc17JN*)G(eFlYH(*-K$zHIxG zaT$^v^fDf#Nx!pNsl!*ZRg;IJkPhr}lw}P~{;Tgk+RSJRhm0{$6 zHx}+=n`emmEKI~UBL2arWII<+I|_zham|fb;3|gjZ|Qn8)SkIBiokdn6e-saTO>CV zH>l#h497%c%RXtq-!egVrD~H7SM-_#emuEG-l}>&)W4VlJEm@T_*Vae$BVl^6%G2# zV9g+2@|eaJbXd?VdB?e-vd;k^U>pX7@+3nEGJ9NuQbl!r#B|g7xgCU@dicO(J}04I zZ%h1{7VfmF=4Ktg!|4F!kwaLN{MrqkpxnGQ0LBFiR$}MY^wFf%=QF~zx(ItCm}&Z> zvM@V>rA2&~+5r<>1y9ae)ib3{(t6@`PsfDh;AY~$!lx@CvaR6+JDZ=m&o$lv_e+z_ zYdO2+qw>7Z%($elYex+K3>Iaq>fZJ=_^=WEg8@J4E)mbr{o@C)_#?S5C&fo@N^0|> zm{FnkiG6I5E;Mt?H$4jzFk=|zLHQqf*My^+oP8)<-gatgpNJM763o~Sb^zMWJGa(Q z9?W;1uC)|1YRs^fhONcy(zXPe+-T+V%zO9+H$3tC;Q6%du&a#t{rkUHAR;zBH?s2a z%ok65X#9EVF+hF9yMfxqo9_1#L)eHh;*U>cY{fDg$-3LgE9*Gz%W$2IFrmAyG&Ppg z;==6sgFgelJti&o2WO_-mMeyi7S{c$%x}I$uq+E&#eHv6p|fLOXl=Q8-q?3%ywP-D zd-_A2K#2Xn+IK^Ey3Fv-A@e^?L(ewUt6{+>Vcyp@o;2f^gJQkbzparBzkU>L6cN{V z*_v!F-&0S6W}yarboz6-laq1GF&aY~%G;1hQzj-$6eof&rtDcDhXF$VObO)Q%r!Ug zdd7WW`%L$FeM+_I*UKM&c@2qe6ibC?Z*oM>yyCo=y6feHAKCbmt55Xwy%drqiR9dI zbn%{iMBmVYB}Ei+I?wO4y<>j*&e9IzUU^^h+!nmltjb$^k3)RWCx`eAec(0gV@_5T z4MywLbi{`#>5U(0HL1Z~+iM34Vb#wD%S(P(Si7ul@&AeZ_N?WA!x!qOy8-XYQ#)(@ zF+N+%TO|{olaaA4-f*7GNdHtM2#}JJ{-Ue_ma*nNlPHy7Cb_()s9Ap}rjqlvSwoqp z$}!1FciVA=>x1@RaqExb1@t%nK~rh?cK)0;ytBW<1fedeFfygQ+)J*H0}hS6B>%TE z`fq(vRDv?M?0>!9nULF9GRcVcW*!j@7Q<(Rx>>e66+6w)1wIyu{`L7;?i__|?8?*I zJ9^gkSa8YnhOiRZzPD9m#N(pW^ErsMufTKuc-%>~*42{bb?FXZa2LDX>u)6x$}Hh2 z(5YP_Lqg9%N`=Nnp8?EqHQl^T40`0x?a7VjCuXI5_@yyb z57(_6xNdf)+QP9T{JCClDneEJLSW>ZydO2DH2or)J(ubB&CnY4f$tDcJet?PEM8fH z&;C|2F(BO%T+(?$0mDalPWbd1!_>Isy(Bj^u1~I0`rC26zaKj4cJ?_Y?@1qp{%Hv)shkxo9#dfa7E6EC0e~G9D){@V@+`=M9A@c-a+YDYi9g`fgeC zB?V#s)$Wd02v+%58GXwFCX5K1!YhWmo>qat-I%Piou!y;xS2p z8&S$L3B~p)gOUIQJj9kj>ED(;^Y$2ZuURURdRk_-x~1w)2ztG_JTqK46bAmzacvZ{ z$p}A|3WvbBV)7q|LU7(bCLG2HEI*CBGRP^ZS-VqyaM$qzcqhdFBz0drImGF1VFwsa zI>=aRiBATsS&ghJ2f48WjiWscvuaK4ewD~Ox)~b23w&YoVWaXK+j5oI4nXoOp|GTQ zGVno}M#UCIz9-0|M@arG>U8t&)2sy#em=98OK9!IYU9&?%V@dRYI_x_HmavcE^L3f z$__JpUrOZ#*w&ttN<(S=8C74r_F-MtY?n7?GYhB2^r#=SI<4L~(yPhk%1C}Y;4fvi9Bu_aU5Px@Z>5?jWf ziJ<@R?J~u;Gx^;ByTho|QpM^Iz~!0d^Mu1f<|kZ8I{{py*%EASZaA~}8#>kaRK`o> zQdsszmRGdM0ZC{zy~S6hg*fsx$W!Q*(ydD`(IxRcSa2NJj%{F!>wQ;LdxC^jD~X*p zF4R>kIcw`n@xDb)Qsyqy-Zos8{3N!khVEZW+qr&Q>FlH=jW)%=^5T$n=0E`C~s>VEW`fh0_hgi1up_c-E zGNj=E=gmW;E{iftaJ}f^F#)47y~;1e(XhxW;obM0j+tZv53TL-BgLabxi=D9>!m%y z)3Vqufk8iy(Z4m)ZfNusDO+5aiB6w^XDlL*4LL_&<;VY75W>6DBAF1U@eVSS$Qp2Xtq{t1q>aI|bi!*i3p45?|k$iNtPJ#(v1 z8%_Db8o<3j8XmIfXK3bV+q_uv1R2$>{P*^!`CrdHgq)zh(55`u^6RYMeO=RiyQ(;m zzCDJdH}x!E#xhT4ZzhR5E2=JbG6yavQddz}AD70^KmZy=jWeT#CrMbU+j2bYNo>)5 z=7&TI_Z@Y7PX;J!_4dDFZ@=f|n~xn^3!ls9q*on*Yr0rgeAQkfJqIWC zPJF5@z99J=^h(9I-ADD;!{j~F)qjuQpJoDwR#W`*JO84r<`tKX$)VQEwD7FG-`;kG zNV_SOFN^aQd~5r|7G6z9C zVjd#zlsdnLLjT0o&2^`fvzD$_Zv9Kod(wm1vj^R9S1$gxRT{U^9Al^MsOlA7f2 zzJogAlk1v$iI8y;vG6v|jh`Nf`GQ8Y1iEK>3&lelV2_bxlb~WENs`jS4;*CDAB;yD z2ez%YC9m`3^w|b+@R6HgEe?#iBPe(}m<+~Cdd-{WHwGKGeSK^EV?t9D$-@!6TEWhSzcvz4#B)ki247H9Q~(1maba z6QQE(q+KhMSt>D;le5&_ky@QX1#V04{SC~-kosq^m^I=%o8NIW3u<4CzlBGu`aw%a z=Wj8Gn06O_Cpdp8qtKk!YbiCIUFGc)9#rcl79;dDRBR_KzDiLJ?4OFeWK*E$uN4>( z?QJ~JmXGZ=lEnHlrn5xA`8OnXIljfVdp0qm!y+P5SkdRUbi-(;*Svz$;?t=Y*gKcw zLa!@#Ql0u|+A8zSB>ZUM)@P>I&49{!g@x_hQ%*uTWAO2CBSPJbWs(;&xq zm=QwXjIJ0kAfMvtW^jRrNx*6jK-BQ!m|(Nww->+rPvv?)NB6>h;X?3MB4)V~4Q=Wp zV4oQ;Itm!5t4_-=%pU4wR!@qrbEK|DlTIGq&FRg(g7`w&K4{NVEBZmR41eEcKz+Jn zcSVyDlEIbBCuTWuG2Y#S(6}8avWDRSMSx|w2O)KHu;)H)?@G=noNQQ^IC`drMj?bQ8D29xfuEa)H|pD0zCd*~k#s>~8Qxa#v{^ zYlGmc8G7n=WGQ5bmf7PWzFzw7%8y^*DW3lC=F(_#@hA*l%VB-W-DcL?#F4Evhw zX6n9lAeGn8FL2Y8P4g-7vD3!?^Lg`1nAe92XPAf!M0!q%={ABJ*30TLqSe;_Cze8n zMo}~WyQ%-^Q8~6J0-O&*+o}5jm|c-~oZaUIK=7G4DqK`uq2#r9p;8j;U^(!N zc<`hjt{2liA)z3u6I|ls$A~8!JrrCzN9bN;7Z$M6sF2aCEb!c)_i zw;56LEJl9{uGNmucv>1h+)Tew48WjsR!DYKbTrf9Hy2WuqwFfUSw-id8M_q|*#59( zarRUurb?0tQ0Ee7MoP|f-aV^6QTmiO z^B_kOA|DsLO>&@FRIf z$=xb0Y@)4S?=OXnNtxu8H?F1XJz&QCo`#j-wQ-{OP2s~)48*uQymV}Uz^sUld;8_q zLNa20slp@V^OFIVYit%?MAsjm6O;aBS~$#D+tm8w-jP2v3%KV!Z#N`e;;RMM9&zaV zP8ki%=B9lfozm%%tqnLru&@%;h>yagYV8OO@6#~HnOC*w7@JT(!#e77+(aB5jtArn z`mu7ylFNjN??QP3QY4vSZiE>y;6`4`{dmek8X2sgFs|%KAALEVNL46OVo?+jf+pcK zbxZj}OmWc(PQ6QceHhENIXrZQ71)u!S?kjM8LEmPp<TtGBhzR}YD$F=g2ysJ$zsC33eqct(|ZUki&wB` zF4Uv$Nk9FEZ27ln{S$Lh^kX-qmv^bzwId{)O<*t1XWGtt15c5ttrUNH3<9%Tc zVij!z4YKL@;f{F_dMY1?Vj2C00w)jKx*RAw%;2zf7hYq{er)MqOSQrs9wk&I#jDNVe700ghQ47P%=VGB}-vA;mrCZ_unJ>wBN!bIDb*YxA=@>7XuYkFAwR= zMK9178)meo*x`xqk4_Tw35xIZCS7XLq(!a^3F*6#=;qt##hb`ThQj*M5^0%Jw6#Vi};%i^z_s-5C$3}25r zQQuB2owhnZKi~T3Zfk2RC55CAd$sE098upCx%AA3OQ~ZZTS?cOJbh$+D^z@qTS6eG9D-#om z13Dz4{qO(u(S#GT?cZPe+#S`)=4dchYTR}!C9y}+h>NkZVu2_q4W+{wndfx21^ijk z3B!6lw>a~3zooD=>F?_m9`eXHZ{9S0IvN-p+;k8Sh++slt(6F6RKQ0b-=ZPMLS3r; zL)g@W-%cd+1-7xVgkYzU9-Xu|U@1K1CnNvHcWY~_?dj1&M~6IWQ;(gbQCGqT-`I7k zb{IwW73DAZu~!T`_9@@nTC`*S(2N%XzLXs+JT1YBUs zTf&DZE1Bns^{GYC(96(BG30?jC7)^GQ*}u|JR&doFqpWHA3qx9;_dd)FKOJK&upYa zvrKpkyeX>E2T2j9Q=F@pjfZ;i2n#3o-j?YVX)!xePPihr^gY_+QZ=YsPZ0AEySS!E z)m(65ag#?L%#`L3LQB=Zuky=THU(K2d*|}j%?2X&rnN2?O9qZz9xW;goM;_ys_pxU@PK0Dwe3x zX>mU{BMzUAmi}c{^`4bDy043qm7boSk+Hk44{gO8X3WBs5ntOg%dwW9`kk~h(B_U` z?iCgT^uLGOzYoPUFYrUnMl+7~_Q+r`T0#ur`+Q&2?}A~+mu=x;ZsNdq0qXH){GwK1 zsq2H3kX%n4CJDRV*TJoaVisVim>3w@+1W2pCa^WsX{Q29r9&BNBbU^3%CSj;98V9* zcCywy&>CfUQ&MDI;$AKG+G+%JoSw8jr6}hcgr5?7{5Rz<&|;tlLk>1hEZ+SU!yrO! zKU4CaRs?1|3rbSUDo85=n&pv;*}w#}%MG?1K2utT(L$}1vQkqQOQ*8M#KfXS&g5le zdh(Q#z_P*Mtcf!3%TW+JfGt4%zlnPA?ZTwzII(JiIX(x-=8P{dd!7C6H|dMo zau6UM$X9yD`c_{4moEZRZv+`b(xB_fioUvf`{p0k8(pzXA8_j0+S+&yodm*{3L1ao zcS=fwH|IOaNEC_HA~&9o_c!N#5o9I_ey~`^qH=J_n3x#*wuj4(05n``5&0Tb`_Gt6 zah9vCUVr}l!4E~wc!@QnCl{x12Y4KsEzh@Ib9|CM7jgG>;y~C*J#TXFv zCw|{^o>7=39%@glj{K{)HO^>NGv2-zu9X>07dCfSny}!BBImVQ?+m)gypmCixE(R> z3T|j<@VZ_PNwnEq``A#Rn%?+uxyZo4uznPhQjt!nLjB>F3(=5hAi0T&x9;}>z@seu zWc%^DC?zF@6%St_)~rg9kcRI&VcEWfhJ?#$`HJVN9G+ zuKogejdc)CYYAvi3y#jV2QFt(jEA7sLlVn;V0ZL>#AAB)?B?cXzH(A#+CJ{NnpqXw zz9A1k|IW@%*CH;x&GUDpbL?N0P_D%}clVi;xv#Nh@MW?2w#>2z=v{z3b z2poo)LS&@zyJ+$`-}7+JKn(&eDLaP1UXSQ3f2u~OC8oQmy1}hv<^14aRLoIjRaM!` zREWH?GGTa6;{8cNLV{Tp8S)xbpqo(UD=nmtlX;|kp>QVkuP1Sv%hv$51%bmVRu#KK z&hGH%RoYHdvGWc;MslSs`tppy3k;l}^7mJuBlw>dAVVcfXArDsj%*|%wgeb69OTpL zV>)E`jPHN_B2)Ad(n_)6KzTi3Wn)wQTiKu1KOn%Pz^Ry*1rBH3Bi;}&(9}digy7Y7 zQTo2LV0L~%gNHo6v}BW&crx2^hewAPR>Hhwn6^}01nE0_0!p;D% zyF{U!w{^OTl8R1AYOx3}GiVXw;n8AF7-*DaPPo0hYdafYVI@B3rLS{HLXFz|*g%O4 z@#_49%GJ5HV<^rS0Heoi$hp{=466!Ybi`z!*%8wIp&rZlo^|GIYmiN3UWZC`ZW5Wk zISSx(fnd$z>Otp^r3!;XZ@v``EP(RfL_r{nJCAd1K=_XntC7?UEQGc1f(uH>tlotc zF7g4i{Y$-w+Xd6Nm#jhN2kdjLY*y|RYI(2c>w^3H`x_G2!hH|Iz;gG=cc z-yvIM(!h@a1<{4;Exy#@#O8TcViq0*?&{c2mWD{eVP{j&#@BQ<E0r~3V!GuHBPPc{D!sewSU4h_?e88h{m6tQ$#jmXyVaj|lCJpJDueOZS zW3P+E*@FoHGfsR;&|)SEm8Jz~QVz8dd#y?*$ATIA2aX4(jLrIy0mI%#$3Pcf6X(x& zhkJXLS=k`yrVeNNJT~Z*)=K)4VRb1gDb?Ap+PN!h#WErxA<;E)eyP9|5D< zQJbzyjv?a~hmt)b>xEiB*mP@44BIlIo*5@F$4rd=H0cw&-OXuiq(;jkzz9M|M?bv` zfHoZv%T6|UB%r?eSzFzOKKXD$Sbfw~&d-zfHm#=PjRuF8q9R7>@NrC7;o)TxhoM2G zY5(KH-L#cU$f;tYC&f8sETfB1x*ogF)5DeP@scGExd*w{fDIom8CT%oR%aL)nqSR- zJs2U4v{+r0Jc7;_4U~K4krh`_Waksb+*61Pkp5I zea0t_F*zZ@Wq+y&V1~ve&>hZo{0ZC6X$*|8@L4v&r& z{1B^}Uz(VJzsc3wI51cW>M)X5P{6|o`kGH7?Zyz?k2?9uSo^1JK#(6mLZAqu(F01S zZ=biCq*{lEhN5F*4-XF?_0)>)=M_Bil1Iws`~<^kZ589-h{lkYe6DYG#Jnc(m&1u; zy7Wbe&a=4KSVW9Kv2_(FNsw%~ZjL?>Ww^MHFl6CQT!O+&5ML;!^2>igRfKvpR)C$j zRtuoJkgjzH0X_DlK3}0<|NShfQ6uT9c&VBHZVnYyuTD+=Rlh4`bWhW;b$8tQ`MY-{ zx>^s{VkGfNyN86#Ikz~OKlvP1NE3826DQYag!&JgaTqk$%W0yDJ<)+JH6-e--N74 zu3mLIi6qwd_yh!-*OqVQY%T$kVxXr7gj6T>r0Ikm%Z($b6|x0(#11%Uj`O9y35L>9 z_9e0Flhe?gRA7r(WsM%rSB2ox;9xkvqjWY3{?D>kIHWi_?gMoyF8_5R zzcx!EcLQ(wJiy6>vhTEXK3(7-W4BB?DVp zxUg^uf;o}rkBC=*+9>zvzSnv&%%^%@<~0X@>ToM?$C>WiWPTDx3LDq0i8Kna85tP> z42w-`p@MAELoK%2E6?Y1fdvUqXl(tfh~d`h&$`bkvRBUjZS|-7^8i$Au4Lu6$pa@S zW>v{O|EZ)2{AsybIX^o?M7hMA!(oFfblPvBy%d&m*+MlJqqymQ_5n3vfM1L7zoc6< zIzlQcDsuAATLTH{!mdB2t#n7WHI2=HDUh(fTRS*V8bRKeqcJ6{uy}+1T(TD|;n%W>|TC6re29N{NmA zxM*8PSIiWNpr;=Q%^H?jbd7%;>YcamZRY7$-K+GV!%{>NN@oZA5WHAeii?Uuhs|`0 z?Q;G7Que5o!%`nI;EZ;Pk*oVAvCKI2h_bj9?WB1?XervLelHLibYY3Z9s zii9^UyicaA*zKat{Dp2eu^q+Fj_{qmiU(PWd!n_qivW}*+0N_f8Wl{A)(feAyH5X1 zEFV7(TWgg@4op6wjcCL2YGhECUkKT(DY*J|wZ(6+__{d&$I=mLj?|;y*L-h7L1tgP zreIA0iMDg43oCe)&fUUw2&-Or#tWilbUkbJixk1cI`?deZ;+ zOvMl+0_%LAYs)TTgjY?=g)*!}`Ygw>U=jq05ZoY(ShgD*3qL+n6gaLfeZ{Bq7_!;i z7jaIFG)n55a{3mp5~bRfG4gZ>upOoA>qlhGDZ|4pj~V1G8?jtn^1cxW7EzZlB!+e@ zii0f_^CBn1QBOJR37K$)S*CL5o%)U{wC_ySgl6Jn>Eq&uKm(#?p&}HhE~&X*aAVjN zfF=Pz-(Ed2Swa?)4~7YyEKtw!@h2xF8Qf~TVh)N8g^pS8P}5nQny!n4Bx*VUzm z1W<6N9i99ckYhD|qYXX!uu5XW5$+K=YDKjPY9OnlIwp+Dr>NkpR0p_YN^OHcQu26f zK_E0BBU(qEgA12SePGiRjUnpqRf9(6_CWv>*!Ec_ZVU-`5lo&AyZ>A~i~ z-3=W;l3A?->s?zv;F!90LWUdumV4Q*VS3-sw5T%Dix1sYk8I>QK-S>&_XU?1X;qko z3MgS{0VmkRs4KVpKoI!Bv3%URiMO}+s%rL|0(Ub83-Xru809xIF@!wuwTY`()ll>;el|4f;bSj8RJ6FOydJ1d{=tF-2F=*1kz+-k(wUB4KW<1k+9XGbgG1r1WR1tfXhsO%+hziF;y%qM zBq*b_ayd9=G6UFrAm#wn0M0+9b!&xoGpXXT~czC{^ri!guEwyO}}+Nx;czDr;@+Fd@{Q+r--^ zv5l_gSo{=g&4ylN^=mEPNij%!aCY%OaYptW)|P`FQ-82NkPVf zqeX4t?eeB^KMYCe8JzI)?P2kSQPIlM$4cTvSELd8PSk+2xMq&MJY$r|ps2m1R|X78 z(seu9m_#UDO^6qGwK$JWeDSf>gK zpfUaNdwdQL9Mfu3c$m_puN{j%(Vf=|0?FXb`$yE5vnC=Jq<)Xnlt(AcRCmlyA$_VX$IquLo(uEKT+YwC~>dc)H)pG zXWO2b`y3)a0@=!0)< z|1;dm1Z5Q2_6vg@d{l|RxA~x~*s_7!C?IYV#j)D=KKn`L~-NMhxluzoJispI>`F|MXv?PzYB%>h$2AeS*c{S`Q*fpnzNhtX6(FP(N*~ zQ9l1f?=)awsm|7L&$J7xD1Z*D`#Ts}7bQYJVlF@k!$x+TlU0iGy{qb(t)KY$;rq6X@+y>0tUe4?l6p@5TN zNVo;(1h+9nY{J6oGp`q!s0s#z4}YEGY&8Lu=!28PrT-WAvQ0(BkSeKT^GC*(l-5JN z2Nbta2t}y6rTQyPa#2byrQU?b6sgQ`&odx6bRL&ZS!q0bT;Df2{ee&mTaTK^blEl= zxLlJr6aHJ>jqN7ld0Z8vHg=$d9(ymaQw4Nqr?m)PES=+YvJ+$NrL=xS-|jydL%lLJ3nR&h*-Sh2R5wgia>@ zCUuMV?(-pN@rTAUAcSwt9fIsYX7JX_CJcA2INy;9^t+1G*B}9HPI*X~N__XY98i(X za<44wdPCBXmVQ8q*jxazo#=7Ts>&ii#7FP*(m(ql?Dc$14wOm4{q40)y>j^vI>Z44 zIMMiI2D-=RQoM7blxgR0mx?hvch*T|X+$*cP9{l`K-KO4&;9<2?@_}gkbML@ z8D)AcvIBN$F_)HrKA0R=_))*d$+Fz@TUC^50@A-Stzi$%u&v5Vjtm>cHta)d6W~4O zIGy$DU&Ong0=a9VSu;}A4S?=rKnL+5f@KIJ1>Gqik$@g+fzx>zh}^%Sos(mR)#QQx zhpfDNeXq2rO-8eDATvQ-ImGiMwznL@1d?!%kGdvO=eW*kwO^l+o`{k!&5{I$!})RUN|d!TUW5je8Uy0B z(BAU<5xQ7i7js{07r>uA=g9vih#^NL1XwZ*ZXhzB_7$z&6=Fp6x%NL<`x)S`A5gsQ zY+hZDpzaQ{L1V^tLATH4I;e~T<*GuU82N==r5wEga!QVG=}no1N7egs{)5#aPc`ca zKdFCGy_b^-lP$iJF6JF{>};Jt4dqy-hwL{6OWsce%f)_`*8&Q3WznpTJSU7{K|Mg%i{m{hkS{UkSC}wDd)^lxgt#~$5 zbf6Y(H>`awu_mXcMT%~cD1_#@0jF;sa9bI-ixEJzUOGhLUh5%p;S=Z%lKxgYl6OQm z!RV0KV)O}FBCzaPC#y;mbTf<1N_=D4#8Ox;PObfzn*`6e>b+rpTwO+>@y}zV_V<_c z??{3FQLmlJWec8%gTJ$HlFUY*u>2R^{{9oA>IC&CHr+Th97l9K18e4N+jm@6u3#ps z(`Vm7{v9OdBXYe#g?#c8l+YP_hy3qQVyz1l4^Ed&dm8T*N*KaTgB@yQ(~BZ4-}hDm z3i>-bG%sa(6D9;xFU-6mTZuh4e$$s&TYhK2M1PNHpF!d_aLnm+m1wcG276ADm8O9W z*PYUkeu1@DSp^X&fU=|%TP%j`VB38}zDI7Y*2j4{nPw0Oy2lo9K2-Zd(2CHDNpYbL zGh)V~wjNG&+@VrygZB2ZTKoafbp61;!I#L&E1s|{sdh?gYu^f2RStNKhSVZ8C9Xpp z!`g`B=2~gdAyE9zEg$@*x3O3p42Q)!N(M&3E55tr>VU2=I-kO-*}$Ws2~mkaoGvE~ z%%!>ia7Nsj&K%`84tShnie@E9BLYJ!L2eM@)S-gY=`g>BWa1h?lh(%64O7J&g;3!_ zDq5g|0v;((=}V-SfEX%HkbCv5XqiuWnS!qHN{LE7!;kttzF#@Dr{cP#sdO`a#; zh(rh|Hz+GQP#(Utfnxt_>1}=kir+Du?QHm-8);tfcH@RIhvZtiUqNxR0A1VJ-*^X( zm8^N}l}Pw5zlt+k2;Kox?J4dt?DLld53bGIK!dA;VD}LwFYOD}&lCKtkUT!~s9^GV zi+CwxVv}wAN*4>q(F`x_U8y@1J$8ze0u0BWW;LW13FUDOW=>2;r3uw<%R#K*073hF z%VU$YmDV$Z1h@8^pi8Fuz~ApVr7v~%yNsM#OkiwL_j|RaYiLP@=*h#>SZToZ$cgqS zo8p3=6^d8uZWte=!x{LopC~CeeuL4E7ml-7G-Ws7Q*J0PS&+!oGP&yZ+tFmq0uUe( zt9}<-)(Br5uW;{_k+xt|E;nypG@YOibl%?Ha#ZSg|LM2#vQKsT?-*d!q1zVeSsAZs z|DuwjGYAYCz!)7hl2!ldNZOyX)4@UK43%rke2JR2K8pE#ILp3`?ILdW7dFMO-TrEN za6npUWOP~Htu|Jzv3fXGZzJ1A{~_=1rKR>C{Bp_{48?sz^~{K?i<%UDvSXlacLU@| zYXl_t>qeINZMHjwew=_q@@umheJR77o@LPKNW-Lrnpyu9>Mn5gsa(VP56R8R8pmQ) z&V~6x>!d4UwT!+JxaUi60?WI$;BaQyD2#U0+~P7qYo(VIQgadQdus$q1ncU1MxW8@ zVmPS^1(4aRo~;tso={UB@zP6^B8xc+;%$&#N{nGOURT|_%7>eC12h7f>zo|{Hd-Ul zCZvN~{CF}Is?g3U0?jEVYrGT^Uw;Fg?d}F`^tH4CWqpr~%nMWW{JT{dl35ZPe=QZh zHTAzS_r37OYv*p0HG)|pI{5DT*Z0nq)j&bnWWym)rZ!GWF}qz_5Wd;cC&9bBPk*t| z|84VQ(}1_}aS1}n$bOm&@;v!Y(&z?Tm<~kD(FKSPfXH_)fFa8UD2Alg%}(`c9P4j) z8*VY3|BR0bwno(Jf&z^C5b;0xK6^&tL()w)WF{NrlT3A9v!2xgAAzWHjTqFlC-0BR zcZpR<#0PFlR>mOI)$}@XmytuRd%TSROSs>^ChkQX^@T2~_Ec!2vFER%Culr4j_f$k zqSVVo1X|uqPY;b;o@-zlf1;M2ivfaz=-`q3aL{&Dn{W?RZFZ5DE*p8Tw6iV=Rr#A5 zkI#uBf<%Al?{19yj5Bz!F5%uy2SFFxK0)ogRm{L6g}t{C*VD3{9(1!*SOZ>0=026H zEMBbge*Xukp4L~+dTQ3Y_o!FylJ6Kc*}jtH0NdJgioJ>1q6k z3*h&Vj5O$2V%YA8DN*>GnSqTc5xC3+x?o%FL;GdywHdN%D#b0Wz6}p=i`z})z&7)#W%L)P=$2yJ$?dcHJiKhq)6il^CF4`j zV-$P0Me{~(mjOu5SVO&umoOlSPD(%VIRjgSPTjdJ#g=N@`i+9IiTOL_NkOm7%b7%+ z1g2F2P9jP~fJgB_2WA`fc`VxO=I9RwZ%-OCB%)mjAiD`GiT>UAF>ow&!6yJ2As{cH zoeg68p%TSxqPfQHIkzbRe0O**DgRalbqC@_)W}l)(@1V_R7~Wg=rJz4r!1rW_uYm* z6(Q-1Q<@_|{1x%<<XUoMjZw{V}>BH3d(}3i`WK!EG2m*qjQrI<)dbcTc!7UCD0wW6-Q^jO)H(4b#{mQGui=V|6+5nlrEuz9-jBthc*-5=gD&K zNDuf%*8y|}?FvF`O-f%)YniS9+(TV3F*$TxsavMuLE_p&e2@s7Pg#<&EO)ty5nsJx zZwm#IQdEKVC~BZG%jN=;x}eb0HmqMG_!8(uuaH%n7N6L+4a^}97GV1`rxt^@gqpT9 zbxmEqOPuaT+;3}McLJHO9z6|p4Ze*mr49yvbyO-qJNw4j74)gPHIIIasr(0qZg=&ytEaZCFv@;ZaWHcvS!`Xf zFCJI(>+zX_bAZ6rXoPC_iawVX&;5O}d*S{GK&AZhExqP{KmkK7mJurv2ZIPjH}AlL zUL|wK$Lagosmdtf&!9T@DAFCTMZl|xDc2*ZGQm@1H79|EN6)0yXOGa^xfxSa_Dfvt z(oN)&*6o4?<2ML~e>F{LZLBETt=?GTdFq$SRk)A?qD7A3cPq;@8>GbS7!>RjTyQ={ z4g688VkF4OOL)p>sjmKoskpEyfEQAGNqS$2NRtxMVNfwsh%7&;nqj{m)2d%zNCYm$ z>D|^p!e!DI(5fVc`vTI0Oc-RWR6KJG^0~i*z`qJO`}cnVUd@|)74E}c`6sNB&*8Hg z;nx!~^!Wqt`jDXEHF*wn@2-Hai??NrxXKsH=G_xY;~qC(68=*@$+k|FC;s%VQLY_d z`I0J=HdN_rw_*b|;NWOG*;C?YQ!pmu$;f^bYfsHD*!604Nz<)HU#CqjMp+jwx;TM1(dNNkR|4d g5u|@Vtu5S(f1(RY$iCP#1*3;3$f!z}!HxX?52@QCRR910 literal 15170 zcmZ8|by!s2_w|4vARS7B4Bg!yKxs+o?(URMC8VWGKtPaA>Fx&U?(Xh-58wB{-#o-K z_s*Pq?mm0(v-aBShJ2QjKz&2-1_FVgev%Ybgg{^n!LJh{EI1POHx3*8gK<)n5QdZt z6YYXGNcNH%P7nw(IDw1}14;Rb2M!`Qf07kLm_tT`A*ApMNvMH9-a|f#epGgwJ4|!+ z!kL`w85z7vSn@0U^^6{&N2ywQcGk@2)-b4Ux#34^KWD_> z7-N#h-mwR!_k;I6Kl*;a%}}IP-`_vAZ4Vk(4|XIk3y$!PxTAGdFu*|k&P6)lAdp`0 z%Ld-bfNO)f{hzmxWC-K}mQ@DQs-Ia?twLzECxdEJ&DB&TYc^RKpc1B{sF{Unxmgq!|GZ{sCFMdo;Q%G@rF%%SkQ*)c7{gQXXJh6eWdGPL*#?t8NYGvD&1&Ip4PO# z!h}w2J|zlbcqTS<+LSeqZBy#mFg(OrF-O%#P@vZ{o+T&Oj?h}8MT-NY~U&=9772SxNqD)O>7b#?H zZGB~fi}~o=Rq12_qJ`vwDJRw6u?(zX{rt%rD36DY&`$_3iz6T0*6t_`!h+>~yq znTT)Bt{(lhDSTPoQKy->m77yrFo9JvR;E7Q3_OyAv`kyr(h0xqo?g7Kvvf`2s+_h9 z^uUJvpsIB7eO$E=jBWg#F6UxpCD+uevWBek+L!G#Z7E^^*x%lCPd1{4SSP1)4~dLE zD}fPN@9VEu5PLL1w$gd1?bcidj;)Eu=DQEzqmJ(awcnP8PzJ!!!+Itvhl0;-_vo{e zSn#ADu#Rc%y_ipzEmfY>MzV>^*#8_=i;w8w4qiK&PwBD4LS6&Us<~ukQ{nmRM<$eR zDnI&)!aH=^Qq-F!J$&g>{oZa~$Gd{rKaf@& ziEMegIjKD55279$g4y&Ed@5It2}z|q3nY+m{&(I+W|KfE$c6LKC#2f)+JUloc;^jR zke`WzTjSbBaS~aDi=Mw0GOR$j00TSAN0gUzjY%P5Dj8De(T^$#$G2%`tLuUM^UE`I zPMVjpKFYpr9$hVs(MQdw;F}EOpFrSYOBU0P6o&9Zr&-vw1Jhe$SU=l5t|@y|q2ws! zrRI9?;|eY8>k_E-21gm)%(P^}v`?;585x`GbKhgPiW%rQelagTJ@_w=AB=xa zW~J9@V8FoScAsOjsZE~eT}cZ3Gi~(m2x7ISGdN6XW1>x-B{1>^_=?%h(8v8D)Do9t zl!2d?!&`IOm^$Foiz|z|w-vJsYnIo#dCb-2T*BXLCI%$7P+v>{i({53xpiqwS>@Df z^DKsle;7THjVny$XN6fDx4`xwtK8O=IcNhi`S!7;B7|DM#$^ zY%r~2NoxYpx#q%>s!npT8v7U516VH7r?}epDJ%51RU2UN;jmOu_HOcIPoF)R;SA$T ziJ_&6YDeMk!W4RVUM9>-Fl;c|(Vj+6ymREc^*;hx_h`18m5_~Ql`rt8o4GdVu60#r zTO&6hJnULwHNJ`$_}d~jsSoA#!A?XcFYyu1wO&0&tbJeoJ=OCMCb-~Fd~72wR#Rcg zzkF+Zx)D+*lO*GgP)Yf?9(yN`xj*|O8pHTXyvc=&MmQh{FL^%5zkXPpQp zWj*8BmR9?_ah%P#>wC@NyNm*t2I&Q^oW^~|%dcSFFcnkBKteB^QXV0GO$$2}?#0l? z_~bQ>yDQxuM+UX5Jz6d=cJ7f^X+RP58#t9zKAh|Q=U%u9T{@pn-~Ti{{@JLg~7*yJypIX?$Is@o1^TdSo_7QFHzgGXge=4CL^}SP6E~r z_JX8Jy*MH7FdnI!3b)U*M*8rBq2VAY?$h=D*>Um}X$#1vBoQskN_(S+mlkt_+)m5u zH6`x@I8sstY3Drj2Gcq=pQ8ewW?pK$=4Nn#lv)7f6rjhnx#CJ7Cm3tWiRac@)e6BpRp6xAI z8P2uhAE^A;|CzE|Gu&L<_h-0KL%XaV*2zhBxCCbP*=^}>_k2y39m9wb$sD7-G;U(L zGN=EGc|u84KQ+7}B&2TS?5wOqzU;DM(bZ<7)e2vUCS(BKRiO9*VKb1B*Q1{8dN1{mA-TF)F(B~=D zC(GeGGi*}Q3fD)g%Z$>k#-YO@5ll<>&WIYZC7DYtWyEAiN4%-@fTx+@7JVC*aPHl2 zut;__a-v{cC~my2OmVtlCtnm_z`Y-dwk5w|Cky?Bu&!D>q$`1VTj{bKi-dliMWXFH zZfN06f7~Qw^}K}^5ADK6Lf6^juhJ930XIO(U;t9a`8W2|zX_&C;lZ7c%`^OF_ag@$ z4zU^yoX{fi@R&y@661t2f z3_Sb!lMkOZ{@ii~GMIjNOm#)g*^5lKMLxM>RR=zyAClTc5|F>}khI z!-{cmcCj9I;|aa1IAvnd<0LyG|9H!e!G?Z&MCLr%S$_Db)LHkmo`D+|hyB zbIXodQ08f<`r9pnRF~Vzph!sd=iT|F6b&S!Q|E@ttew=4@4QTW?GuEwg8)TAJ53>EBjRO)YJRW z4A;2l&(qEC12M1)BXx;gxm!%|UpB>0j-+*zn@z~tz-8Jj;A9ofUzMoids{6hDw|bp zSTiokA>NidPB;3^0yM&FucAT?fD}Ky&u#S@wUehN0p&U5%4MCBe8r5xCkmkoSn8yS@$ysBSar-*r0&zGW& z(e~rYf}GW@9ZFNl(!>hoWj&qwk=sbQ$R8IpV^|bHl!1eM{u8mjNG}~2UqwdMMFIz< znv%}tFTsK|Go6fP=4@8e5$+E9&1Rv~HzB&~ksL(J3Qr=IMg-Ib@47weDYu=aC4Bw_ zcP`2wWn<756~#kwIx6@IOGXzu|Jd1|1`ZeKy=xY|c_LphTqeGr!+1ddDpL!+!u`FP zNh>AFTZ_g&P!Zp~w_NZ%r%@L+!axzwD^_9k6?Xy4=bQL~d%kCR^F`W2lp?%XdG-Y6 zD+|8R&%m$cuF}`msj z;0L2aDA3%sf=rt8XK1VVcTCi&J{$@4c{aYn#Z9vvetU;T#HLyQ-h63jb=oye>Yv6X zfVhvDXSlXa*kqkLYc8G}jL6@%#wYZT$Vbg>rEH`IZ2RcP661CT2!>?06E)}bwg`(* z-lzpdUK6oTQUmBg)VlVq<}`;$Hax^-qL~gS`pRBE+yI~6iV|&)!BBJIlfwd~tr~)! zyUoeh65eU9sp?9{>8-30fb16V!PZM>z7(2gB65N-YtHWYV#V!76nsXd?;A&0-01lY zNdyT`{9hw8!$wUD-WpF!TI{}mvL3@>%pv01V1h#FJ-JprQbv9!`;0SRrp+Dx@JZov z5_#$oZUJy<2f^?-^m8EU&@*S9B#E(vAdfLDPf+6&yO$QN;*O1toRgIoixBt988tS^ z(Nzk8(c(RP#zFq#NCuKB+!C?3ggiSf#JD{f6D_?#dhzl#bzH~?_Yc!8DR}4Gl+X7S z7lh5Nal#ZWybnQOqd^O#fbE6D6aHOuY%GCz`lI$dCo2ufPGdW>U&!*xN4YkqTfK)^ zF`$0oc?@Ks3q2nWKN%D?yo68Frf0$1g{T4xO-QR>!@P9bf*zvA;arLTgt)UD*&6uM z0yd@HKjbT@x_lERMDk(3iZ~0?_yVLME0%Fs^dP3LZ)uOq57XPpvJ$|RW*2oT=`!~u z5z@@`Y~VDy-43f1_O(>=okMmdEjh5j&sbU5g|9iy_#p9X;F@@rS%a6ov1ZQ27F6+apFuO1@1gC9t? z(Pw2dZx-Ze3XCX6Q#yMx0Ydm^l)}i^Ue7@EP$9YfL0G!mjRKw)mRx1^hBR2lPq!9} zdSpFgfC#T?zt|>jkrs(zAk;kir;9qTI4fl{+ezraL+*MDAsdVbCARG1K<9agyd}|J z{%;XgkLNd!S9Y2m7*wBP?OiX3BMZ1cDkztYGiC?K`1!A^yDo{ALG3G|I$Yc<*i^0L zv+xoq3CcZw&m;D1iB>R3ll3|BeCU7^359xpYRb>tzcs!Vl@QDjU}o;P+Ec;3q&*P= zOoG6c`Fe=$2(>3gD-U4C5Xm-f+>N25sBFNbei+zB@1~Mg9AeGeRWi%a0c+-1K`T=0 zL?IvJ`Sq2w<|{}tUN;fHd}nmP_gA1&>bdP^#$E(v8W<>_Z7QxR-c^7(vx|ITO`jBa z!-lztWzDwHgaQpUFt;CuaQzobM6$jkaswi8HUI<5A-yW*6J8SJ8|suZ%G<5ig9Yr9 ziGSmDR()^5etWy8@(F-;V_17h^V7rV8Vq4MPP20Y%-@J7F&^zNvmYj_yyaLvY9X@u zHuOG+(nZ(W#M-o529c{2EJdcj{r*4EYoFN@DbbLr2po!me7=dDoQcn~pU32rA{#-_(FN0?#&gMmWc zdO0N31kXq{s4nX#k8~Y%XckUiHl_az8~}6&EA}0%%>sQ^?G|lRe#GmETw3c_BPqT& zL0M^?b-yTLT76sAp3r;YjA3W|%ObBNuW$4UeE0WcF@(+Y<5cNr<}|o2F~exf5ufkz zQXwv6xP7g_@FA}@4`5NW`oEwJ`U09bPIIKGOGfs-`Y0l=B=41)THD>=0woD-zo;A$%b2#5p! z@Pwte-uwrESY2TdQaM~wJ&TVTyblQohTVK(LBy-j&o#2=8I5}arOF!-mHMlR;eH(! zM1X&WadxF|T;elRFvg6ha=CIilL!rn{C>T{aI}rQLaeC;WJLcuI#+TPU}d_I?Cu91 z^`Q{E_8f~!{IS2u534xGA@ZjGjTDpGmH*APl5Z1*u&wU><}`Uzpz}J2R^-3mM-O`V z?X9AakHLjbzfWub#p|ncHBd$gjoYMf4X|}~^T>L?4$VahmHj}~3Xg#%BXgDhc}_LA zX0J|!JA_Jf*2JAt*2;7^`gJg$Y!i}jm@(_jDOjsvk<8{=kRGc{{yN7=D3<=WhKV4%flr*9Iv!2lDlLQZ) zw;bqKFh7t!L~^F<&s3hLDHI_3)*ZKcl}uVzX%;gK(BXuZki#w{Z*6VCRDM&;kCUXp zO9j?GGRf+QKS`b;B_;E^y72JU)z#I|(6F(wF)|__s0e$<7#b0A$S3|II=b9py*oEI z7sk!jhHHMI&dIKRmnvU5lnJl~+pT2>(cKerudqQrACZ2d&dI3>U{*!kGrUqD4-E}% zetx{OEhzMGA`;e>rHK0ZatNDRZZL_J914YPf_;hJ z2Fk?c(E-`4)v-L1Po7)hB|k6}?JV7(tC@m60bPdq1qE}}Hd?l}C8E$wixdpun#4mh zjRb>w`BRS6p^H}4YoTRsV3NeG4;`4zX=|M6xjSRkjxV$^Z}F|SGqitTpymFkVPZl# zB{xTrw%rHhFCRp9D!@;@4>%$C_)=?L$-S@jGn_8b;dedytImKG0tX9HVMh#skxpiF zlGQlQ%eWYh^>s#$&cC(1j(;Ln&OVGvkmls(wts{C2r)R4LaZpjS`@GS#1VjDfY#MZ zVmsJoAu26B&>cZoSy{gsM|^C>_=_W&XLl2EQ@s;w}4PD3_L& zK7RZtl~WPBml+I5mBoonvVe`h&Kug|h2(XySM~=p>)BES==SGHxJP|7$ zsOBKiIkU5{<}~;t@&6IlNZ?GCxxYTH(JXFyV)G`#{AHoZl<2gMEZ|YN*x>4WIjM#h z_2C~(y+jgGTa;evFEFMd`o4q}Ke41sbDz~c~ z>%5Manldvp!Gy$0K{z_O(b&GUL*k%ad$lSH$sjd`<-}VRb{70Ad3)X9L@7fxKAJ5N zDB^e3i3`4cd$tKiC6B~oGuBA()Cv*BA|`&k7?*#(S&tHWx*_ji!Syx^%_>sezh3su zNJ`o{I*N=05i5$J)LBY;A9^D5$3Ns{8 z))f;IQ(-wt7a`uk%;kC$VSmhP&>j#>>N8`_HP0?g8!vTzIIs5kGb94sG;;K0hXq<_ z)?dwHxj&WLqlOw&rj@|2inZ%1p5d~_sk?i6mOYP~u4H^a<4|ISRNE~_+WrqCWL0q- z=P&9}%uIw~fF2lpJ=8$ba6~IUK0aB7ieE^rSatc!w2tvFENq%kLk*XOS_f<@nX>b{ z`uh5_Gsg>z-hjgFn(AU6kLQ*OU0rvtDtneJNj+;@z1LgR=jD~=DzdT>k8G?S7dw7_ ze*Q9wD_#M!m1bdGE)g#HC_#;l|2boUAXuI>I~hI%@J8Lv^u@ zMjAAVr@Fhv`Adn|O;L886kSR`$;qh}PF{I=n3&`YnTn|S%hWRE%UY;j9rTB>m`MJB zo>S2YkSe_i|-EO0CRbGfar^SvDR9JZ@HtAd;yTKya2_Ai(U#N_U%TU5JC&C$xwOTtOkogM+47`4z>NI#+@}Z@L%SVa~kIno*2379Fq#&L{tkxi!SZ2D3YYm ztTc^C!{xVJd38I7$oGN!r0W2*TeAPdx<4*IZt1zq!p)xHZbbS=eH2=+!>G7hbVu0n zE`+xHQ9k!Ey-~IB^5g`s({pfm_(=K_GFToTI#iX?ts*-X?xlAQ=M2yIX-{;xeX632 z`@5b}6j*wCdc~?%Bh2mw2CKCW>#ynEGHD753f_#ueMPe;vla9*1O1<(V#AK)DB_!& zS!*uObb)BexReyB*R)NcTCY?;f7btV{B0}=pa4<3$@4>Tr`R@A=iYg#oUzu{51u4+ zeZIIuC&7N51f=U?COz%Xr= zg=UW8?Crc&m1YzXFB?q>9GUU;(V`0zmI+?>_&B!rYa?u?9{;<@?bvyFbskfP?NfwH z)h4MT(8}cZcYG!h@b~fYF)W`U$tjz)MnEIN6Y%(}%|xo70p;YZDk&L$`c6(mgXsNw zDZr|&Y%n@#{ei575~0WGkM5Q*R#O$DPU`G9_x1UCWy|T96c+?S(EFkVz}U)H`cqL^ zSr`2kCX5H&Ja1e8cN?V;_~^=HukbTH-8rMt$q7b+`MXzd&w>P z&pnEmgv5LuI>yVc4cb4OK~g1*nUt}OBeWz|M*Jklpin-VWPZ5V9k)oSezw@A6K#4p z8_=c-G+KPqZ0hRnep^`DY{^03dGVx}&%wrqrJ?=s}-Z|zd zCdb7E*s^Q42IQ?HP%)h*kKJZMClIig7PM-FwDA)*(*N!EIJSL-f7$31u5iQ@-i2*$ z^u-pcl{TOC)0+(@ktGfUE;S1rQ#A&eXnbq?_wQes`}x+|nqi3On9`qC)5-`!ZaKPm z!}Zct*ua3<=5gaGT~~YF4z{0A>x_z!dBvI4S1=&9zM%_B%F7>j&Cw;8Fun^@%ny%< z*p$)_4hp(5XC47)c5Q8q-E7#{%xrsmTbd$9_rP;dHn2pmW$9_?GfX3r*@~nQ-5-F3 z)Mb@@&EYIEc7L*)$pK$wwp4Utu2E+IXcxTLlEdl6HV+REkB>>;z3UzlM5Ly5a&-Ky zXaNARmX?-=M)Hs;d$Nr7q*t=c6u(5PH5apb*@t}P_IN*;+IRQ+Xs3tyOR;yCmEz%e zm?R|Wv9aS}6=(}aeUDb<>J0C~{&Tg?W<1(m2 zBJgze5i3iR@SQA`G=pVsWcEDT64q??&xMLur!EDK-Uk$E*C|rP&MMiBk*oroLxJ@P zETkwjUdjZvHWnJ)s}OHW6Pf~WYO7iFtKQD{PFb{RdPsZp=OL5MOH$iIjn~yUHut$+ z+e*pLNB2hE6-DCN8ciNmY5acZ#I-hUx$+s2|H}|)Ttshcv1oC4w_Rf>513V2oDQMh zA)}^_-xWnAB=o#5Cjkb;3&ITRx?b|Mi&7K<{1U!ClIM}~&W0SWGJe5rwc_u%5TYOH zgJN$0c94{m+)*ItdT$w^&E$JAD#7ZAlx#2Ns5Ly1I&3OZ*xAoY(wD}J!La##ywmxN z^KPwGI%jOMP%R3N)o{mdhW=exBC{UJ+qdulE7U>}2>7%^0Jll;_xCuwJ8bgGfF*xt z=KFwu9FH?bQIu3vXd2w<(j+<+#(kY3*e{$6?&fj*H{w^=OSC%7y;h7X^W+}QEEidZ z9~CAFR0`MD*MnD%mQXqXq@A~G@!lFt{)gBb5T1&S0oT*T=p zaEB{b-hIUTpY7Y8v&6!@yuDoyW=5x`y4~@=(O?8%*x4ORkhpjD^dJzxB-=YA^9I@f9_-cyb1#Crj-D|;d@m2?1~n=o0u>#7WN`4|;d~@DH5KfNegZk#yn{0{GjHitZ|?7% zZZThOSK29WH7P-P!@ zO@~tA6aM##kIxz31%*WR1F|hSTjAv1(NX%(pZ@`*#cI@jQpHGy0g72fGRi|IqJSS3 zi(*7ya_<1aVl^OkwlvvtkZqQ!O!r4FjnAMR?pqtEPG3NKEVcaEK$FKQi_=~!3ySPG z5)x9S=}>4$hyo4H!aKq5Y=Da(0uq6jwmQEYw$rMA2;LV80k;wKnhU49i`|8a!ok78 z#>U2nhlik`*Cnc`*BjWvMXK~dIZ_HZ*C$RXP^=IzXQ>=1Il0KtQ1Ew4Y8^2L1}qz7 z+urUV4Jy@%b#=5e-;mbMLv18XEbv`VMLi=jD{tfFWi0%C$kYm$uef+}|8jw&K(%mF zsRaPq#6;W<7P@$;VpUIfcMA@J)dQq!+aIrc;dq$A-s(#&;O>unrp5o7>dy0_;*Uq0 zzy>gU_NX~WN_wP6tE)ZdkHA^IpS;x1Y?gW6O-lkBFW( z-SIzTLENxK!hqVz_AfpX8Op#`_165lY^OQUWw5PEogmExVC3915{{vy&3*>O>?^5= z@`O@xF(v1kFXo73uT#sjyyp#4vWzC8S%^UdzX>LcJU=wP28>6PEi?VUw`_-X;Xp!n zTQ`peC`5Zx<*zi^lUe-BCTU}T4Y*(C9kYmC_xN$0W0bJF#ix1TY@}HHRfGR38kMN0 zU(15)c&95pib{0BNr~8B_>U5p%Op^XImid9*K9|hb!I-6RZKzK2+r{+WC5}D3mY|8 z9C^GP8%V)Wxd1(2kLKg3*WBq6I_-uB#{+|NA|+TT0$7;0xje< zleoQJVk-qgGdi(1>gXS+ev*D0ay#_T(T;RUV%unm74(yZ00f67K@jN#$O8lQ*RR3w z|B{=&Al$grOgaFnt(HSPbr&#tPxiUXl(@~!bLt>TFZOF1@U3vXw_Ae}+5Bf~9>w#p z*$hZhCVL0KianeBXZAN4&;rY(&##LRR%C18d?yap=M(Sb8cqa%70L@5{7ptAn`Nwf z8$|aqt=FW*IqpC-#HZvvEi{=*fb85=tg5pNMOz09P2~5EYba*0{ZJlD-y2|!SHZ&t zKz)!lVY*Shm0`hXU5Q_NJ^;1V#;l8Lv33P*;LYX^fjok8J}F|>eu2}36GD&r-22ZR ztl96^@ODXj{lf?cBObW)*fa?WP{glce}wpFcdK7Z94-O&bgMdj*G1~gH)ojvy+uJd zqbbcMhyRB3T(yu%$CD2H=&9YDysgK=jpPX+bP~-c7mY()ME*c^M~rMF z@Y&bRY_lYG&Ux#rg}#3%L>KTkZK17(ZFlsMK=9Wtl2FE1s4F)R$b7I-NAy;dDDhs; zgH>LFdV=$U5t6*47tS?){Z`0ZV0|Fm6AJg;5connc=A@*d$h@9l#&snc}%0F}J zfYCd0;TF>8^7W6e5{$hCD#vxYa!Vj@AQUcHee(%%R?F4T_0w0-#9dVUZ&cu;a2R8F zeC`ft^@%UT%60sZp5kKPBBgWn);$vv96lC}qca4;glOB-PeIuK(+77I43F?A)=DGe*&r-O#Z8ecMh~~b-eIT8 z>-Kf9kwFnScUz`kIW^|xKR^=Nu)ViJKdJ^UX3sJwvwy~QQ1Vn6hQNAXvNJ!dwz^b4 zqdpJXD?*HSS#1kL_W9unJ_0s{nnm?H5fAE@g>eZFttPmWh{C<&$IBb?CianxVIcgP|mQ{%@4J!J8=vl8;?-ifw&n4jC{YjhKANHs6IzKe5se#BdGbIXxDJ? zSEL9(kb>c!&tIAG2j2pt_Q0{R9vW)7L0E|JzgqxRom_Pq57kZnQOZ2L@%}fqI|w z>9e}p7}DnumDon=y&{jNL8asovzm?n%^v-xal_Zk14?EbdM!%LzMRvnL_&B9@iqUO zoIdDA!dV4l+^&1}1wJ*b<}?^|)%S(p@nyh2W4d)cmeU7*oLb)<(RJ&i1q5s5L7_lz zV$EfoMCoPcmbMI`b`YZfFlr$bns(c7za2lx_{pOq|bg1&A* zh=NPW|8E{*{1s@U;^pIBNDdjD1G0ffR#(b;*TG|Jy1T}4s$59zPZ^xOu;&u~PEteY zXW_66X)j@>7>8ihsXMPJD-Si|3b1tvBi_v- z+^xXz(r4XD38t7*{M#9hj880x_C9uN*Ly8DjE&;8U7)RHFpm7scTwc%7GnJtrjNj?doGy$q%53&7(dp2UgTc<==n8-+}&VZUr8+%hoP@ zggF{0NA%epZRGP2eBOHg8p3&$eDLNDUS6S^%IlN@;H6l`7;%iU+avrdt*6wl1yXT{ z9EtPB-;L83w8cUR%s0GIoxVE%i3Ewo%(KT^C4{#ERWW|QRAtzLag6^IfO&} zLv+s`cGa#Yj{YE4@f{smeSL=c2H8HPdUM&;j;Hn2lD?_EUlW|B4k!!o=Jw5AntPoJ z*^UlT^yXcbvkwa<_S_v%Fw8W5(D6m&f*B^qj_aqX z`cWI*S@Sye_3Au$9s|tb8+SRAtB8L|?FY37%V7_^@?$tesR@qcD&Uc?lYyi&9rgEo ze{s=Umjm+lPbTC>0U!O?djm9!*5(-#RA_wZf0qXkSIMLu}~J?eEMD?7-U27-@Ik@1@@(XjqnfQyrJk`d;7sG5qq#WI9+| zCy$;to4yvv=Ota0OZVu95=@&JiLPPv@dm&ZJp^|@C5rE!|zwocb%Pt4Pq_$gT?kRb7So7dWu;5>x>V zqYo+#q*DjPpicOo=SnWTmZQW5efaJ9hQj9nPS2j*1nrlG9?5H%mlO9o{ros$cQF7B zvY0a?#<+%S1R+T!yjnqFTcy2B+Tn~8TF|{?M43b0LKyx8qVq?s5D=GA93z3ivb?CO zxB{{yd9$Zrp$Er)Dr?x+kj4p&y=3n%Fa1i>~wy81xdn8|rqPR_mD|Kb5FN<94RLr`~h`U1$^sQLqVf zzleo=i|y#0Sr?PH!*XNKPOm7IpZjDGL{I)OLH^W?Nd!$$F?7F9AHKpSVvssIK z(dqA`cS5XdZ2&$0JA^cX{V95Mtg6onXwNrC@NByIql%vY0oIHm`bdeZc4w8R4Btucd9Jjpf{k3=da#_LFB7%FB_b{ zCH2xftvoRXqVF_^Qd+S$)YS|f6?U5~*CJia!sz=EtjrX3QKB`kPN;?-A)!7rN9n=q zNAZX*{}}(hCB^uS$4mM;^|Y?2Dve{UYlHJ2A|#oI8Ts3G3-{9(gUH;(i4F}LO@qAp z=gxH?Q(PON5erQe7oaA}hk<;QxCVCuBvIf2GNs0DMzn?mXxuse1@MT(%4N%{J-Dws zbzHZzAT93Lya*fcY;v!W1$Q~#U;~vVza^?7V_TW{f3>7~W(BxPh9j$_em^Y*9~G}s z)80G)neAbIK=1~X_YNPF5i!6GXB7X+N=ZMZT}*5kXu0W{P1-TtJ?x*622hK|nTt(S zq_&qh^|u1}iuWEbLFeS!u(@|~cy~T{@r)(N#8u_F8Ax)eDplPCMKLcfdL=-eI@$!n zNAROkTp7la1iTh_w#5z)|HxRWTOu&S6#43sJm7PHCf&D| zD8A~bkpD)kZ~C_WYl&~u!tq!`mfdyYeHJ?a)-5dazNY>q=AOIUY|$5>3JCh;?^8H? z{*){a37{%6?DjBHt<0!UUATBEt!%18(W!9|7rma^2TV%8wu&aI2>V@@)AVgc5seZzATB}qGJ)`RFDzsDMn1lLi|9EN%h$J$ zD_*@XV^rgkO_M6KYljK)vFG#Z*2(_z@cpmxK2R`3ai>!RNLpAlQc+0}W71rsD+Sd{ zFD&HP(nSO9sRJeBg{pvvdhF1IwFMZZD%Yb}!xPLhF!Ssbvrxj5;`I3T%j~cm;6i|8 z3}<_4Kp{+JhX%6L`D5MOXb(VGQcIqGo{RlaoJWg(&&)&rj)W{5tk#F=}E!dh2Zg zB6y+!nk9OphtQgA0t;a|`NsnDe;+5+-#L5EPx#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0G-1|aoDCIA2*bxA})RCwC$ zop)dq)z-l8ow7ZxA$>z2y=^wxJ~Q|GV|;5!BiWK1jgF7Ph1$ zRi^w|c=}B7#X?;X0006Y=exJ#$$1!0&*+r~Rk6Mp0Kg#h*PuggJXcN&+m`HmbY$6s zl|o!F(4*hfkblLvMF)!m>pqrT`QU*nttLN{?l!2}oY+nnhEWNl^wSEwy&$mDUh&?cg$H1qHN-yLOQ7g^Mj&Wn77-e!LaT3R>$@+1VnvDiKJwx}FJ z>&xAWq`7xLq8RFv$OX^)41J0vYXU`u`r(ADQ(q4pH#dAb*DthX;|UM|V9%pN@77+RTmH9`1f4aNM#kpIX>>)>T9aA@1qk?T^C0IE1$A^Bvx* z*!ime%m0q}z`FcQGRWf@MRl%*(r^f^-+#DYnHrz={SeQA^E=M85kCMxH@B`3g1gG3 zS{^8e&{ClhJ^Rjo{6&6WcU^2degJ?_NeB+{Ejc@CWx*k|R1Q2oT4_|R?f$Jz`LP!9 zi|GGLR3`7r-Ou$4tt{n+3U{7c%jC5%@w1DiS5L(5lMoX2i{Z4;3esZy5JF3${!1fz z@1r9ojpww`{Mw`tU8*njF*kT-cD@=~BPPV0EwqAV`aEU%J2z&ve7-gLaR@DvQ$^Jrjgxd)2;0RZ{BLJpz%@>rASFZ64z+$Dqr0Hmfg976M@imH~kRCll* zM+P7O00f*Zv_SN^8E$vA}OivZ^v7(;9IC34ic004iXFNe^4apSp` z>C0Oys*qHfLO?u79KazoU;IUWS?Y(amHUN~%Q3E9*uOc1W=bb%c%C-DRa((vT}eiD z_Oo7tID{5SH`gu@z$>NKTR1ucA>5g}pMdyizZW=!7D<$>qW~9ND81T}(HVr0kumOm zba154b3Ba4UgVa=m@n<Zc`Z+j_9RmI|&;OP~=oBW#)}Uk;%~irDg*Zq3;_wf#i* z8tb08AzyP^GYH|ulB+bL<2(nnaPFB^*`Y4&YpOnH3(euTDsA2T#E*{`o^1IS2z6-> zo}A>*HKfgxVtwh%+Y275AH5m$>YVWROxj>;i?}Oyf2gE=Pxo$hZ((+=5cOYg_?Crv z?an>$(#1EGhRTh-zGL~ZcYl4jM@_43XOez>xMx**LQ>`Zdhgn?7OqJD{@0wnY|Or< z$a(Ik2OqI+UP_~C-rYqhm8pX~2Y%J%6AWOi+TBy$UvmGOVtt9C`r#*03vAJ7CuIJ- zS+Qfd&#<>c-f}dALR{#^uV+I687Z@FCGLH6XjOUw4(fW4?Rs6J0wexc;D zuh3^;yM9OWkH73cDoWbXTFoh&%YW}JIa7T8m4Fu)cAV2tTg}2w<_kWLYNes_MZXaXJI-c9 zBnYAPJy-o%c*@&m zT+QRpl9xSDrFD`;Z0z}cpxEE!zd#7(Jhur!;}JjrnYpZ&g?FL!N_^V)h1#O$y@xL8 z{1H1Y-?(rHpQu68&l#apT?k2-O9z&dno^Iv@T#xS=c_KC4fTp;FV=jX{Tl*kM8=8N zrjPb{enQarW@UY%@YK)Q+j6y!gGE7GdM8B6BAVDy;Sl~U3}C)b|H)72J3e3x0HC{D z*P5Kfgm`1G@A}H0zU2ssk#e5f>h$%$q?JSE75c;*_%K<0Z<{yX9Wc8yY*p<>DXZ?2Wy^cVSh@Y^-ljsgI>$fADixw=?aahw;UPs9pnXp0u(S9(erQPgz^klo|nq!M~$@0*o&t1ZF#v001E& zY|6uWTEDJae3Yyc3;tNa$rWioR2i#Z2^cd!Vg?RzVDYXoHAPCg)cdyfhmigB{wzGb zDt%owrJ)fWB=Uc^{iGpYu`M}#3IJdb!-jIQsuiCmf0bFSh;fTv9Q9vcp|>5m2t>=n zORvsalKge9<}m^wP~<;1{QW-ip3U;XtQCNv5hK9)pC*6xN5Nn1`0|=v_*Pl$($*2O z{JuQUGjTi4a1DVIYLv+@e>;d}Z=4>@t>Q>qehdz71JhH!8OA zFfv3Oyej5PIp4h=zqH`g)|?$#)j0y3Z*py#=)cs;t|^5=0C@1_TYD#naG~94UW;lW zgz&TEWv7eJV*pMG`Pbx-33cmt_oIV9XK&L|S_FWUlsxM-gv5x(AwU4`S3bzrJgl1? z50WE%pQ8}n_>0kqni)Dx)HL7K*&EX9<+qdu)!zPFI!eP#dc&&Ak-X#IXZ)yV3=~5J zOM<3^yfxG-u8GO6%lx@6LN*vSC;a^;dNd9ownA8QQpxWF_h1k+DIa_5fhzse z)sH(!f)jeIs2OnCoqJ$a`Z^LLL&QOAx_!+a#lXR9iwHYP!#4L`+gaABCZY7I%oS-r zK#2V-==EtGCULXK+af}Qkcz6hUUutQ@4;PMqiWp^76%RYdG-bGVeDBA)=)kO0jxU- z5poDEF@bMipYceO=O^@Ghe_F1^ylntgFRzIB!Nphe`Lk%0)&1&++yl_pb}i-R0ypZ zo!OOpU~T3`InV9TGuurU7bkyxruaf{_a5KIEVV_a#f-b=J?Ru$xISwW#Tb`%{>W4> zCkjuUEINY$Y#fGR9NG-dOmBSd)nUxlj`1DA65XKqg@nWa&))8>x6vGjuoc3qB8)1C|RcfksnvT z_R)|gS3aN^s=HfPmgw<XFijRvIgL7|(^2008b(+=CG7Bn>xvyVa(H zw>&fmZLqO|{b^yDz5)VF_P?p-_f@GRBzm~VSYKlhf)MI-y6o)i!otGX*jR%2`{i%n zzMYbi(p;r?cXuB?e7FO6Tb3g96tsIgWP)iw6h-wz1~6fg4%V_UrZN58+}!Qkw`XQ% zvR6ow95`?w`^>lBek*N$G{U2q#bT~gfu@H}G5yTZ85tsx2ynABY=Q7{>9zN7&RdhQo~>LhtwjjcIv2`3cnG0_f`Ww$7n-MJT)(iX zqM@{WjIT1P*q?gFfB-PE+uUN4H*k)9Hz<>c>US5HLfwuTQ>xM+l{* zrKP2%ktEqrQjPcL5JD71;W%E`XJHsdEEd1;!V9JaG|nEKfdI~rm}&Ys&VRJ!sskZB zc<>;PSAT1)(P;Yi?Hd#n)JO_^K7YlE6=%<$J#gRvkM~3r4MKSI=uxp)Jbd_Y$MBD= z;8+)Ei*nVEO+VABGA*nZwOW1e-o09S#wu21WaQ?}n`g|J;qUL?h$13{TwGj6j~<UAuObm6bWhTvjV|D&D#_ZQSLF?9Z)=oo`*8_FMi5OKRuh#fuV& zr0&AOK7IPknKQ@apg#3OK?w2r{N>A+*G)|z5UgFh)-k#$R?tFMQiek;Ut5R}av>!` zT#%wlwWM}3GBWD^yFehAHf>t7nm*G)oVj!78VrV-Ull?qCnu+(qQU_b(FzEI#eq1) zAF6X9gdL^f2F93Cm1RkS#l^*SRdCp_VW9a@X?A<8fPjGD;NV(YrePR%_wHTJ76Jf3 zh&YgdL}qn103cEt@kZd-DeWg(lAubZs&z%H)%NMr$80YNAru=MOVdxR`^9lQBO`mD=Xs=0sw%&$d`abo;Kge7)gvQ&=&ps&Z1CB``!IFS?euL%2e+yA0HoIU*E=I zYc!g=-F(g?{YRo*qj(rEQJD+?@DX~636VmbQ(;h=ms;I1JRXmyP$(MG&blOOUSZqV z%*@QXu>!7N_>aWMIunq%U6IKC9O@M(#D%}*|6yh?#kKan_d=oY%9Sfk+)$=or}W&p zb9J?_ZbnXu5FymvJ%&OQix467qVI4cG9Jo1W?q6mK0b9(oj!eFI@q zg?0b2R`ct;2qA<$-C~T4@s2Wy{n^c}ix?N@Y4fwH6=vv7hYlU;M%Qp0|Kf`;nq|FA zlQt3(66!|GX_{`|zCDNVABvr%5je!|Deto@r^f}n$jA7mx1lWsnzOQ=_7>l z^YfQ4U(R+Rny8Kt!ez^rX*8O;57RUq85zkT{73rAdyqeewXM?31O@fJPsr^;B(E<-fCJ!BcArKMfFb}c0( zrL3&1)_5;FqXNhA(9qDBn3%Y@xLT7CIfPBE<+`=DNwNKhjGtC_TNdll-@^23CSF-t zsn_e(YBfSgAQ13)Jef>Jl4QLPJET~7KT-(cCdIEu^8VPJd89u{hILMj0`aB0XY0FwDAtew%Ev>7o7JQsPA?Oth zVrohQ9A(~v10tjtBLaXu{I6cguFiSp^z)x3FXNW%wo!yx)j15Jy&7Jt?J4jO(QTVwA(PuYtRtSI1FI#-5`|- zUy=3$XB1l*A*+9HUY%B9P?8wAtKZMAHAa0K>a-z**7sORG1R_ChtHQ>;z+a_LIe>2p9e}=&(&+3n4@U zT2cKl%%y{^HQtG|5JLE6$_fl%^CM>2{L6xnfCS6$=j?5tBz?&RZmfWCv*K3`tqB$f z4)+<>lCSt@;py`wm#5yC%~3QTAwoz?>o#U@)6x3Hoj+>HdPE4V@3~5V^E0Zl-nl-L zqi7~V2;r*qbtEK*dBufFLR!)$K?q$**`EGegoH3#o%80^ciH8=9Jyw=GYFwlec7lB zuR#d^h}|!H+HkIobWuZVUcWM>imGUh#j4UBuf8 z9~y)rT(qzM)^>b(g+ck!rMJoq6@XL4rXWNJ6=(}j6`eOS##!OhTCoL#5b`kI-hNwx zMS)sM`^x2ssa5G5zo#RF5U$DiQHTrP2zsT}V_j_STl%c)E$=}w^xM~G9L@g&IQ>{Z zYj5`F2Y3$GIxerC{bJdA7$=7;U$ug~1HC1)oF;j>NT&^= zHf9UelxB-!$Hd^*C4{)`*N%k?`5mnxWjCZ06QUVmQ|*G^uH5~xXNF%byWxZu5o^fU zY*PHXGj|`bhZWBVAs50$LP$1b|KfygDOPWd6zYl-GB<6^{>46w0fZxcpF;@Utw>T+ zYR;Vj5W@YBj)-vKaPMdBs3fbpExT5B)ABQD9mFG5Q|cfytilGRMu&=`CXx%2o_V+qW#sKC#6tQ~wfkyW`XtT2W(ZSuh2bQOP=Tur) z&lx!5E=_#<`V4!lh3C8m>uBAnqH|8|kt(Vm=4lJc^cD747$2dxkI-99snabxK+(Pt zUMRi7$9Sm_nf3_@dUzTWeSa$&az0m{(if-Mp1}LtW0u7cP6bW zlS0NwX@p&Wzg=aW4UF-DD%}|oKGNhdi18EI_rZ*OoG&9?G?a$j_0)+F7HW$jguX%_ z2Pjp0NeF=vW}l2VQ-fcf=jNb?+a%Uif_ms(w@$ud) zzpFGO_qhws<&gpaVm$_o@O^H){}==iyZX!C-BBm;OKW?i zj!~+2uFoJb^2eU5oz520h<1Q;lvKv5~=#CgwXkt z%Y2;Q%e}kPAw&pul!oCDKT~|ser+Ly@J#V}08pIA0H>Qf10fvb@r<6)Unsp|$2~&E zc((YGfib+`Gt?R9&JY;USKd>E3->=hVyETVnzK_vhzEPdI>`*g+6WOsZv>8`5&dh< zZhN$XEJ7WlpAzzx^Q_p$Y8jIL?fS?_S&6>vT=7Nwss#w)+Kdetz+zmZ!(2Kz@8UE7 zzz3mI0Dvpg*V$1mAcXR@gyp*lPh}NsN5&=D$(f-h~$exH>)IQt34s&_DHD?Sy+kVGT7+^CCV776)o*&7^D7 z+wzXshw_f?d31SGUkg& zX-D9c@|68ZD-6n4ue`0MH1RQ?4`|n??Yy_rP^nNq?B>=5IK9B9o?c#ItUOV8>W#p0 zZRe#A=Ns%`M#Rd#HZIAQU1AtTI+Fd$8(7zu|r5 zJ-LINKywejyH<8{QPSsXN_}AVqM8*#g4=IzvGZVh=C9e^^6B4-LFg);sORSdPX1P+NHO9_wMdpV_c)z zW)J}c(BFFuCs(F)a*4noW?+mjoFAv7bv%qmBU;ZGg2jQc9{q=T4eBHf2i(b%wnkV> zDvhc?3jR7002ov JPDHLkV1h)u0aE|~ delta 7150 zcmVx{ zK!gyGWg-hE4}pwO78&8UQu_5@Lvw0S_JyN~4MG_Cj)(IG_y7c$p3>DNxzD!K?E(_RpK1tovA*bsX+J2^=MU=4GQv?(I|$($4R3F0+bSY4 zQE8s0pL&~hNs=-#j?jnz^|0>7K+9o&*U@9$CyVs?A`%fmIK*W`I`;4ae9#ecT2;ve zM!em$S&1vIO_()Fl{oO%+k)*!T*u~_XIqr!E80~V;Ry0O(NpoyUfUlpv<5SoGg z;@1({R#!c5qKwZK{ZdV+5I_(>IU*fppNC$n+pzlV8Vq7T$y^=|RsN+q11gq(Rh{xo z;gb#irdR9M0{{TJ(j{eym58E<5b6%~tg3vfeBXEX+V`^p8~IW6CqW49vmacr_m;|@ zvjZP5YhjG_uZwa1^6Ya>!PYmf{^NC)xdbGlQM<|pKgzh{tmk|O-6#M9Inx z`J7VxQf9hFfl)`hPYzJKkrBo~BvpH{`VaPy zDxan=db@dZkPr5IXUM62%t~lr@D=JPrPf#f-<3~3Rq*p`Ow(nAv6lcJ^z#9;!i2$* zGHJZXpX@o4qE1RtCCdn7EFu!O*|yjD8y?JA(YFLMI*Km)!eL2Jp#SMO6i3F55WmRG zvN_PwN!$AxY8m0M^n($9nC3uhkuDz$b9u=M>=i-0JH>KCn+s5!Wc^eWeoCvw4V=*blk*?d`qw` zUpGZY7-PxOW_mcUlXZ<;QX!qT&fj38JFZQbDI<*W;2J4CiIGi#){#ppq|+YnsWedf z6jgHH8%0Jqtie^`s(js)Hrh6#7jlMNd%O1En0UR0Q1`u2yk9JwSQwE&r}Q<9spo{ z{f0{S+256{QI9}&FI@#8B((xd|NiEexd(1K8s_~PdGb!${=kXHw%K;vlDsIAUrba& zIteRsO~Dob00uFM+_z}fzBSm^>+eDUm=y_XA_h}Jw7~3tvF}@7|5m0Z{h{3Za;XgL~dQ(8=u&A?3(`}feqyEzL~!8H`Tvmgiz!=91M$*5oxkDDhPN;J>VA;6^as{k6}W;G*RVq9+RD^8o;-dd~c{ zVPhNJerMWkpWF96U;Wa$(*IT9is4+Sn{_{T?tj|p&L{I9ooSkW#T(+Ai4cQWulQSp zL(z<|S!)cmym0>IhCtJbtb3NG+(O}$kN3ZPaov7@$CopVrEBwEOxL8AC(d79ao_Qt z%6@Ao(ZGWc9_v2&{P~wm3e)q&Pes`eCnpvC4JbeWfvYhVvwLP?U?sLWJGhh7_zT^LjXOT_to0hcX#f+-ZJ+W zd51=JLv@*1>PDgq^PUJh7&M&$Se%QKALuj5UdSZcP4j4>7ql~bhfPxo= zGoGrG-KT$;ak~;%-k*Ih0Du-~6|RC1lp0Ha?nu8)3PgJ?#g_gN;Db9l{?Qm{S(tRg z)y8S*>eK{K$pz z^5U|UxN>XTXB(UUrpA>kvwspzoD!*jLmtlS47uw3jn&?ZU5tYk7zm*XSDF>368+TK zrfYh|u0zX4CRl3!YVBX%Z}~fk6L7$~k`#}GRWUrQ1FTpF06~D|>!v)B|Fa~e1&4T} zzxny)I5OQdS7kLNuSo&p^HN?jS@zl0W;)9$l4#1A=-1O;Z`ktHF~E1-Z73P=zy z?+Xz?(+wq$PI)kr--H;M34@RknbYobIK%^daO2GNdP;Y>Bel3@+2MPCXo2Zf1bHg| zv6-gn9J*}JrxH`fk=lu(d=n4hMQ@$d{hk|_ZB-|$aOGgn5KqPHDSfP}2jfR3z^uLG z!wxzRQy9r3p)fulA5sJSmGck)CLF(sflwOOL(U2kBYDIZ#pe%Dn)w#W#OR&5=&kKw z>x?D6vbdr_Jy&JWgvvhtV;j@K|ZJ=6bX1d18iSSn#R;;Nj+;a!q~?$N4rl3`QQS} zoS#kkshk2%JcK@f&i|MCx7@79qBJ`h*F19>_}BdVw<-0ThzKPU+7xVYgq$}fUiU)v z%dU`XR>Hd!q>w0z!60Jj*t$X_h=>iz2xDnN0RT<@=GLI?YU8xctsj{v1BFol0Me3V ze&|uab1}bZlv4_1gs~JxV9t9lCQzo_xagu8> zN;0PyhV@;i5~X0+gq(&Pk3Skh=;i&D?lXCDIetEW7~q4YhLT>45kj@KwHr2Ua5|j? zK@4+wd)SZ42xBHkTt}6-vM@fcInWXi0$1sadogBN_O;huqiH%)?S>;-tu`ScK{li@ zlRftR7DYmaCVh{6KZPkK$7KTawHoGo@4eUE-5u%Yr08_ueqRj3q^;$`W>yqMlgYGl z<;wJb^mHJ1u*Fa=_-d@`i(bN zLl8t+Sy^#$u|lDUgxcBJxoOiTkH-Thri?o+l=R@?ocqlRlaqDYLtTrLzXL9{!$_a* z+_^KdTNgse^L$oT)=f9vR8mqxQB=RWh~xNwtXZ>ib90XzIkIKT7N5^2ML-ILqNb*1 z^XAPfR;-Yl5M#qnMB<`J<i53aA<006Mj9T>!a z(>1AItJ?JR`2GG!Eo52t_S0*b4<=wY@VN7Z zRi5hBU|T?-d691nv`p4!s&HlGu)18X$mm}bMZI1>b?VeGi6}3$c=6&;C=_XpYin!e zq#_2=5@;hKb(LYNoAWjXnseeNE581Jn1etd5NYlNLCDL?>q`zoNDxHiw#AY=3>#Ig zR-2rh9KIVMgbc$t91c008LlI|Dd~E%GT}PQ+&W)FW1#7#r0bMlf2Kev6pD0LQasYH zO)e`dqiH%ExGEHijEs!%rwAdvUfdsc6N4lc6PrPlKw9RrIDWsf*>2xVaZ3fPwzYaNPrIj z01_0YhjQ-Ajmz#8C92h5dp-t#13(cmTle-Mf1gS0RK3gF!|( zL@NpaXs0`CeD!4sGYO0c38AOYy|}OI5GVF-u&`JxN~JO!p+E>plH9au6VLM!k1IYE z7((dvdUx*J899}R5F$x(B9=+9aiW3%fHu0lA<%fOX(j;)MhtmauUTP#A~CWrgY4|= z$YLgjVO?EaYuB!oCj9yZ8Ce$%1OjWp(f0~(@2_~S77emU5>{-uw4WXG5myJQF zz^L0&ZxZthD0G6h)4&a#}gwv-_zw55M!Wm&?zydEXk@1VvZ?Ry( zg7Wh6UZzz>I5=^!-DlnBGBp`V%0vJ_mFGNx5mVx_2T%*i;+2<|FI~D+r_*s!$mc#etTbaY(2c+uf-^z`)5 zG|h3GMx)VaG-k6oH8nLmJ6ogCC=`lt1zZYm&2K%)YT9|$9pQ90xjV;YkUoVyvP^D=MDLsF*!y?7_Bq~T7Gf* z*MF&hEBsEaubvaRz8(0oj8lZCJyjtwl&#GQD~FA4R}-of)g+nC#6Xy(Nneyy{ypnr z>18Kv_i*06)4Z~b$8bi9DT?()`MNyv(iVL-ro+qmW*TRF^E}o!Pzxc1_5Q}Kw$FIP zg@L7is!~b~CErOdm%$8TB*)5L2}oF5v;Ljt&D-Z~u_(>G^0RM1x;DYIuH zhJIPLGo4kQ^K%l))c98mk-FLL_gX&q*tT7Nxco!xWr<}t6JEU#64;O1x9_zd=wLc& zfuRK&1K2#v?6(@;sOFmhGi!#5@_h)kW~7W$;6jL<)scj1Q0-zwONz3S<)M{K&PwIPP@XebrX;K&;$(5g#H^LFAG}c{=V`I~Q-#%>nI9-!=OY%*(TbGhJ1^U~owpVyr|~b#J1q*cS&<+k84-kkA%T6b<-@-<|Gg#HHdxJzM#XW6FEGz>hFlvO-|1o; zK%S;!ea3xmsr8oYE%Rbt`jS+6nRUtilkcr?o%mzz>$E^eztT*WD~wdw3s!q; z5|w5nWf+M9v~*5RY&tgRP&o?@CfymS@}{Qyi`#g%GyUw%=C2@=3?eNUJw;zeb5GpDuXZpwPeF zyyZl9rEE@P;0ym$`z8U2r74R?I>ZR)8+t1K*}^ATkzHT^7C`dc{ZI|5WWkyPt;f1g zq-m@-B+dh)(g`7{U9rA@@V}=#XlLx<84KB>4$&8GvuzIu^gZc!jOHLp6c;Bi1OPM# zTeY~xq%g|-25m?ognrI{+FfO&^b0I=!00w3N!RLTdtR%1b6wqE8G(^G4Gy8SdE;4c zbvx5hp11&m*qFo=IwCGboxHXEvkKPx##d)&95*F@UI)ga9}po#V8o)t z8?rT7(k&%Z9Gov?gixcu*+>~OG-+cJN=P0tJ1%SU>;b6LYka2}npS#d0_ktrltt%{0SQ4WSwH-zTYEZro%>V5He#S)xMN z;BWd(^~*N8W6Xb_q;?saG!Ai&kW&U0Ey7Nw>%Eo_T4~#VIQf-vgjPf%BZOpN(IQmg zN)4`Yvu<&Ggbx_R5dZ=bWMI*TGkLn)KV0=ffi4dlAK`~NtBqG>X)`x9y+7tZ(WEdfvo86M z%;f;cweGbe|A` z>;eamP`c|^PQJG=KL6A9o!dKhk3PkVzS?fK=NEYoV*nzV;^jyUygiPTldU@aD-Y{= zvf$Bq=Gmi`3jzST*`B6AOL2T5fyob07?LlPbS_P8U0twBi7THu_u}^U-O><`oGwcI z{@(U~$x~I&V*o2Pl*s&sjxcOar^e?$TlA!n(m!AQyO%D!3P)yTMR+IPEv;LBU-ReY zK&xy#V}K9;8+DQ?zdQWyb1!%w!up0D12-}pwurhQ!pi}gjT3m&toljW|%SR#xZ z>yK?a4>=D%oVzkvl{B1D4u99pdUkYv{$cAjN62-n_2&P`T&^SIWPVY~AR>kEfKoWQ zv-9)pxU73Ke~1t1F+btwx3L2EN&Aj}*uOaAuFBSCt(m&IEMW!#37OxBAPmd3rkQhp z6J|H~n?gcJO{hWC>z=}gNzZTFX4~;e`wko30T6m1d&R9OH){xW_yCzr> zA~3wdbs}Gv8%JobIPTJgc*KiHRN#t3jw27BdR$NGznk)J_h#O$Av9w?H~B3>7*?h; z?sJcwek#ZZ7be|sUEdD*TOK%t?i#3avZ7kH`-_$4sok0*+>~Rgc^ewE3yGTXlEQ0 zMsA(+&INDnA8Y?YVPvVX>}3EELbSkeh!aEs0fa#ehd7CmDqL<` k8pjCxf*jfbvJ6@NAID37cnRF~@&Et;07*qoM6N<$f(U%5wEzGB From c54a5ee1ed75d98c3c95dacfdf004ca040258b02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 22:36:32 -0400 Subject: [PATCH 0501/8313] update --- doc/logo.png | Bin 16760 -> 11980 bytes doc/logo_small.png | Bin 8163 -> 7828 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index 9667bbd8b034f9dc9f37b881379770281f1a7e44..ca9bbe4d3cbd1ba93937dc2103a853d2f3affcff 100644 GIT binary patch literal 11980 zcmX9^1ymIM*QP-P0ZHlZ?go*RZt0TlTyjZi;iVe{36U;omhMudW9eGDV}Wn{{(BB* zc7~l_-Foi5PmG3|0yZWoCISKiwvwW(7I3{rKtSZdKmxw8t$NY}7er4j1sR0uN%8~W z=9QbGfhPjOYy5wIhzQxaB)~&-FC|qu^rhEtkmz4oO`4V=AW$PH$-dX|T{_MKnd9#~ zc0=2bekYxDf>=_&Qh$&}ET@hRD-xF0h@_@fgVP|fV#b=`;UXr{u(B$@Uo>C*qQz|D zRVwD3(l}w~(fau0ekHtwWUtiVD;c~Wkm1;RKC-;GcHTiEr-^}(OF-^|hyX@LxWh*9 zK_hlSG@r7=a4zZk_*<5zds!f3h=hgjY= zdO!J<)h;oe;>q7jG=(F1s4S+zNDJh7O2LJk$6ga(_^sQD z%6GwF-5{jML_I0;ww@q$U=NV{V14C)w4MOw2C}i@EsoVA2OlxVEX(b26A)jJDp5c8#_* z?5H=PH4;PNc{vcxjUv|A% z_3|r*)&h^ALD9dkL+>x8ABD?>I}6vD+Qf$gx_et%eb|KVup1L;RYz;sl#CqydqxG`H|nDT~(fcXO0IVaWL^1VnnG^Dk1)I|Fyn*zowk=_y|syl-Q!gX35k5u*L}z>f_A3ts3+=$na!nGpPdiqLWUqBYy$QRT;OO3z%p?3XXk~8oT~VkDQ@4I!>OkQ zV@tE|4^GLHXlr<08mi+8pV4w-!_yS@$r<0QcAnZ_}k9@Tm8)YPYc|2gYr1Nywy=4h_WYf0&w@F9C&hRu4d>(!&M4k*N=yvfW2A`_Ta z>Ws`ya32d7g9en(1{iYz)Ub_uHySvhMefXauh2M8Kv(SxasRy&`OU98CW$``8q&~H z*)whK^l$obCH7BNKLEqC?Q0MTPh8T#`S;6$;SA1g!Cl}?8}L|OmAx)m^7uDVkufcI z1WiONKLd(f2sA{SW;Ys17K}j?M(aW50z7E=J;0IAnOmU>9Ifs5DE2JQVwRbIYw;a! z5bESMEEP4f*9Zk(+!*y#@==$;(2UmNpZ&KW9ySL&&-4ECye8|^A z|8r=<*eG)Qs{X(dWH{$cD8cX%o)-zwuNzum$2jzIF03oRxVPvc{tM+d_BflQK!K$o zVCnh~)K_xx>117%qMHdy?!N-wMth@A@xltKikzO#S+zDHX6QKnq)ZqNl_s;pnE1JJ znmMZ|OtM#Uw#OJH9&VM1Z%lFbB%M;K@*)m{9hQ>%^GRJloG+1yIVGCTYp2cpjn?sjsw(WY;EO?9!6M&i@LlfyJIjP06I-)` z4^)`ohxX0sj?DQ243Y!pDZGchh~C)n#!>6YuMa+|$@54;%Ho_pNnqu%jRPfTT`- z9#1uNan7#h3b1c{+?7?QIuNRy2xq(2eHmN+&(QPNwL<%LK;$D{}uK_xH~fz!A^#PuX|l3ii4#R3vGO-&0}~ zesT9KGs~e!c(G`#)EIbWuHZqI@oehVND=$AVQJ=0X z{){ZL$=?b*?rP`rVU4Qf5zSxfbCgGcPOfQQd-a?K`e}xma-H6@_sP=OD3;9>H?*hW zyQ%Z4q9uKd)aZ>f$I{}$3g1*;D}LCZ+N*#YPL2>)*{Mv@=Z;KAvkEbt148I_l`RA} z@rDW!EJjx5IQe3>>p|zocbZx-eLImk|LA|?5VU>Y*64aEm};k-C4t=WFd3gs#C!qS zgV1h#JladnQS6&yX!l8nR|^U71P8ArmGA1NKp+w6YL4$agFH^DP^%d_Gq3hjZv;MA zLo?B_EXFaHl<*(dyT8%I`#1fQ-*oi5YL2=Ktb^YvA~(#Wobm{4en|rN8DRyWKqv_D ztwqa)Ck9$Tq(UA{$<|8Ry^OjC6-+3JU(@9cw?Bu;B2EyBT!RJ*gV_=#D9`piY4G6@ zUFn^Nz&$wqNkA#dsi6SdkCnl^xYlox`fRzb5ZC*oyN$`Sjep4Bs8_F-oUYsPjpH9b z-d(0kFaQ>-UEnH2==4~#%PtfBExOYh{U+ljfmd90LdOxr3$4_?1xF7u&F* zuUAx@-S?o5PtyZ+!bSlWo*N0dk>bBjoGI<8u>rHY5RjYtq7t1N3A%}qQ|wJ=4D#*b zbPjsng%8q-GDC_xJe2&tM0o;uEl46DQdtH7A7B}%m})+Evh0_xdPQxx*+i@Fd?vvQ z9!DgzIwtsMY583E>A7robbkHGZEgghR%rBcl={Wb#vN>lO*YY~Y<@gPi-K&Y+Od4F z3!$_FR$Q5n;SRbNys%tbs>J+G3IE)R&Xx){_BS$*mbkN{HmXeqzo6H zf~*EgBzJNVY0RK#c(Q$D*@q-~nM;{{*_+cu&#%wFhKP^vtWXadWA6FLNZhct%!w## zVkIAvm`|#{`-&4mzUexV}EAc5R4_4o&2&BG&Si+PJ`XBTh)gt^Z`) z0h8X@AQ~%le3gd}LEuJ&VL%E`2c}#vGg&(=ou3D{W?y&$kmR0g5xFxBRzFq$ zVFR;1+?(&mC7^u|F>ShWMfBxiN;{iR<|~BUvgt`|(Vce8aO)T?$`ZE#&gO zW56u{7@Rqttha2dfl#oZ?Svq9os$nAs?H(-ET1b%o$^F%A(WSC_2u|mp^@_6!3WhZ%QtCM=IV9! zp0ar6iQqkf-Qcu_^woiZ%qOio&%}F4%Vi>CBT1V0%YcKmi7(5G+D?P$lIJoD^yLz& zya&E7V9Kh91#^h77zd->}be~on z!U}AvSm$-?K{8JfOQZsX^q86W5}gxSHX*C`&h%f&UN=9nzJzylB41n7fNK;~u^aUs zA$g-gj}myEvRsi;LXu}O=y3I{zPq6Bb>@rLX$DfQxX`3P;)hrrLe_ zFqQiXaT2=ZD$)2P5)TbdKY8H{rfcVW`(YuoTeC)?i`Hq1*%!O;sqA9-q#m;WZhi7^ zW0*zrAS%a4V+;?mNT`3x^X1X(&`~s2-`T{2-BCPbyI;g7fB#$l5m7k~-dB zT5TlY$G!~zLc5aO1o?r`XNO3u_c4SRwOqekk4Y7 zP+JGCTC3!=5#=nUc~F_Lert){P~lIjs)=;Bfhm+kje#!-><9=zCLV`|g&pT7~PgUQO%-yH^*WKyhkv@Mu z5&V3*B>0gdWvxW<-X-e%G3{*cG2mu^n*io<+VMg!cg*pUBSG_pM&<2=$K7_Q>666@ zs<|>G4J|E+HW0;hoRUw!vBk9DNq2H-P8+9>o~9;3opOJb>VvtUxi*B@*ubmqgjdE6 zaGN#q9d^6w2dWR(s8S~^j}>x1j@hojJVzoVZ0HMLpkg0=etXbGfKyKWL;~wYQslYV z`1C&40H$6CiWTnNlVMcB^gDb{a)~V*0tEn%>oBy}NnVNWgrzx&--UVMA}3HiuDJ~W zt!vR)h@GI2&i~glxf`MQ(zn&VYF83OAK7Qqf|Ku2#WqM}hK<&*P2o9nBaN>^pt~NG z4+JUC8pC)%cKU2W#HWCfr&A2h@62AENWjg;756c`OBI|6MGE0~ve|PtPOYsiW54G_ zf6+Lbx!Hs+@WeEC!=CQ$0W`nSgK)Edt5NVC?N4U;BW`0rzv6w{g1gXP#@>(h6e zmuF!MBHVkJwXoYU027=vZu8{;aqVcL^>@wJ*B?Xo5I+&+s4h`#uNEks-8v_7NxO7Y z!dx*mA*pcp#s{5T<*V@@_PY>*oBzC*rvto`N{bT})Rh@-(GgxhKIz{>fCxz!0c1k5 zihcQ5$ zX8JEDKDY?&pG@y^#zx<&Jp45dSJO@|>OeIW?tRa)cmDVoMwL~}N1Nal@p2}JGn11| zWMWjx{{vShBUN9GRV}ftFUjZ1G;HHLoC;Nna^-fc@?qse+EC_1Mqhy~4&i5Ar){Qr z{m-*|+yNb^`EF3efZG7=4Z%`JMlDiF8atJosHabjbWOYDk4^PMCSpM$%cf8kjD2Uk z0%L7klOOH*fv1RAxT9X$Gu`u z4|i)N$v_TU>OO^T2V{CwDtiIBBuJzMC~c4~X1IMv9P)+ekm@iImSs)Y<8gJXqT~*p z0+^0BYZ@qYnNqGAQE3;mEb4uT-Idod z2PEqLP5pc4p^X--ce}iwGgBleKLrncJQoT$Xuz;vI6frSJs|Wl7{>F8Dy|`duZ6AX zowctmXiSYht#e&j1!wDJOr~0y6roNSlp}QWz->TA0LnE|`PNZ7X*r7A`Pe6cO3waV zZS!^8MIw~9Yy(YOt;6w`Yd|D5Mi@17?AsXGs_&-$iH23bmhPhmEGZMw&zl9{M4qxL z23`S??>o!O+sd;Ev!lf3lyl_2k4#LYjqS|N&U$-$6Ucp~l9Ch^ZSyA2O-ib>=tC>f zvPa!eVVsG^qstGvF_5GR>zuq58{weQBF%?>+8^``;_zU7{LX74GP6rgebM}~o|DvG zTkAHJFTudTpsK2h79Q%C^$4qh*i7aw8dhGz=E=Uw$Mk5- z3IhIk)wqR%#;f>ny8!vug~3<)qY^fw4*#X_6~ii@gFhu7OWqFAT*AYn4b=0LF2B+6 z`(dMkM8~V2udGOH_(}Dv4CThKEnfYFg5uOOJ@6O%pB1gzbc5qQtxDgVskM5;<~BDi zf3y}pccCA3XHTy7`NUkeyNM-{_0{s(P8OGzirO`)CKr!<4%@!b`w2n4tCIA~Q*u=k za5ZY3b+{h{fP6o(;{jZysk^z^X=!Qc>HCL<&{3Kqp_EAPv|Z}Jg7_)s82Fyf zR0ZZF^2yuT0*>TtyXlJb_)Kb7S65gV7`Sl*99`Al?DqzlzCTfY*RC6AlVxgWs+02X z5Q)=uO{A=vXsKO-4q>($W$Qi-bC340N)Rm65@Oi!pw0P^jSx46(kx{`_P?N^rOO z^TNf!K{WVT{PAf#S3{PWKrM^cx;q4s$gki#bL?}II3*V^??pI{8pvu74HJ;Ad>-7?%}|zIQSHV^_hW%EKy(5=jc;A zTGoH`i=6?t)??`gA5Vzt|3LT{7_65X?MWiD@E5H}^mKKb0&ZPU;@S-xo=uaUCY0n@ zWtbZzVP##gg|Xyu^%v^q4K$g3d8?k4{dp&n++2x(Irwz19sdn%-u2$zbV6aecW%N8a(&ZYG?ex zkC>fVykaq(uhmD#B^g=%L^_s^>|B(VGVHp_K5yI-DIF-y+16 zv$to#aA1HS%RZ~L>ga=EO4RWZw)uIxqoX>yx>={7W|vhs9A5cu{ZG@y*+yThaf78y zo?1pu&g(G4kyG>YsxQ;Z2G>IWN;Ewe?bCYa%^yp6XY?r(g#cfrhS_ooPS(`aw6L() zRUv0o;$>rt%U{6pYJ(oN+8#ZS(Gcu-jGeZ}cOB}eFs1-+YV(3Xx6#oUIUV%W5%OP2 zttX-Hu=pvUXK0ZDW@RRh6S`Fk2xy3i;+Sa^?QQMOC zGO1%0{O$pu_9%waZ zIK5Jynr&&ROkpi@y}!Tz8lT}e=Gdt{Xammd)YKF?ga?c7Qu+0#Ue^qxRr~pz_JE@( zp8+xo47GGFGfGlY)|3(PlMofg%iG%msbI;dzShUvq4Dz=;e@dr54XRh=YSy6(+W6l zZ}e3naKHR;9)RR)ba`60mNCcnZLk|Q4v&X(5U%t|8^G=2+{d!GLHz6@nq5- zj10P`{u-(FZ=VbvBAKRnaFvYz*<6^#6RdgId=|8Sg&Tj|6*`0BB0td z*MUWuxOaT)JZVIiu!V{$NG2;?U?C@mibCmg&}>8wBjn3@O3LWw>skb3YASphduKF6 zmNN1=fuH;h69)(O;*a1cO1rdz0@qzLG7=;gT?oF3>-_}1vlYp>q48nMrj(;8TEW!q z?XA!d8fQjMQj$eV$Lp(ulG4&_k?@2fW>;5Nx4-`2sOa#_-?gGCrq!L!2zwgP_v3#Y z46a)_B#&3dy4_=BEp40?aQj<5_nfr+=pvq~UmFT+zpLJQC%{bnYR0Iu(6GeizKCEs%2+qCs2sl?Ec5%FNH=Y6I@Srbn?=v6)pTIGLdW@c0j0PV|CsnD;{ ztC-cx42-e(w`Aq+FqGPQT_-Vec3x!0I=Z>n2! zAfAB)2ZvFQ<1E5pFd!BWV3D83$L6LRm1tENw*skf6fGUygbEhQr*YQ^4bYtOXP90u zP)`c29LWx-5-ta?cZUimNZpJltd6*0VPN<&w)0C_!@nedEQ4z*PYiUr*6GmHHc)Bm2_}0ai7je0tTN^_ ztN%Q5bG-aj-YQ^^H%Fxu@V007f%}jSb>TQ-u9Fa@tZM@mqEu>4RxC#<1cWzQ|F{6s zv=vZFn*O1CZ!`uz>5?|3q+ZNXu_nwtH(`&~4j}THSY7RJ7!b%=+t}ob`_+H@_N}yZ z(3~3s1*uTOE+PAh$Dv7-iHS)-z#G~X)b4Za;N*1FP5RagtxpIVh(A}@87!J<*3cd> zKO?Z`8oB;6zAU`J$-Nt>CrT;b&J-#GU1ou6Z=WHJJ>3m)%G2KpR1;%kubmSyiP>_?OtiQmRQeIvjQ~i99CNJtnh!#%6 zz#zv?)VGN(c-fwz04mLe>P@DYyPE?DXAkyge*)-Gr!xh8&Qp$!i+hcYJux|X(#`F- z#Pk4>|Nh&KUr}8DdTY!|-pP+OW78dH}iA_M0TIl%= zNk{lI8~w(&r^~V~4ze%r^^~4Ij0pujc!f%vPm(L}eIR}l)wgx-D^6K3MO^z0r^jOY-evPnE{yilY8|+Any{of9hWYDL6=s={;2QYxxW8Yv{V7fz)UK8*TnDHx zbu&e$*`JP}o$o;Z`cJC<1=;JEk{?{Ow#9vo*j3XNZV$iA={T(tISnmhj*-0> zBy){_#|MaYL0wKwlOS;@N5s>nKr%?$CE$u|=#1MPW9(XN(#DtLg0M7q+09CMV@j`G zW{He5LktYRd&g%JElcZ3)N|m}z!H0Lb!A-t`4xp&=!y?%Feqh}bSr($p^fxIvQaO> zTu0Q$&Ze27KSdw%ol+FtpLyMda|=I2f#K&v0HUS_?RB+4YoIPB z&+Mz}88ttu*LH^IY0bKdJ7l=da9Ll~DOJVb#wH}Z!N%_Nzdi(Pde)&yKrCdRB@+<4 zcA47s^>xW|>-&?*s`-;kuBJcV{12~Qd#yd%*>6X#Rh)Xb6_(yj{doe~IDoPB^DOVJ zFE_hF%eBi$Nl7un_B(cb_w6EvwnjeJr>3SxQTk&T_;VP1YkICO&~&>wH;I?1iM;$^3E=9?y2j(n4@34F=jfF-g(<11q?{INfBaxhQj`@^ zm1#Qu4-7&Gqvh4fGDTANcV-nUpR}(w?aa=}$;rsjH!@PzO#N+FoF<2XhtjjcgVkr} zc)*1m)wki%`Zk6hFiIdJVqu9)F|xF2A0@=ajg5^Lkga=;(Eqx&{yO^N$+ur0!M66twko50 z#>L8u4Nfqv^r;lDPmvVTS!eMu;3q|ROh0kgdlwg&RUbM1%yk+N06zpNYX7(V>Pb>B z$RQ(8-KFmR zpuRqYqra}Mp~TwNb8O=eQ9MUg3yb6Zf-J;um5^=@o0(qbMh7q(^w$7Z0^Dp>8vA`F@@8hNN_z; zvpl8w4XAdBzOj}mPx2Wy;t`7^Pc5UFN8Ov}Nfqy#-uOiR2th;AR$jkywkcFf9CPAp zA+cQ_>w(SVO}A@z>_Rz=oqvsnM+$MWZd168ZP3|UU$3JKZtYcE8KB-y@k1CZ5xw>` z#PgHmZ;-F}`hLierEyW_gzLpz)29`51gCW&B~5g;MD84ul})ib))03ol~3L5rX_p_ z8K0}UlTpO!yVJo``ni^)fw@zrd1aju(cv}YqHvUn7@R zra7FctZ^7WxkN`8`&SzKTg<0@KqQe3*bEjC`r3;K&??5kC*uB4bNCfoxw?j1lhM~O zBn>WU>>YS;&9x8!u)PY1BnPz0W(PQ)cg{w}Z6aAc0e-_-BZA{yo}s6U-yCalTiQ!` zjctv3Ti|dXM^>i?Pyh!SD!e)Ws<>K7m^fJb#*l_XPs!GzBbZg><;trO%Icu@y2Eyd4Yz#@BLps1fhAa-CNJ@b-3SC zloE=oysfX^`}ycpg!t{*1sS{W{_Ife*UF`pt@qCrj6#4clBGG5IRP-AE-Gs(vrOOl zQo~Deo5W!iNUC@ZwjURYc@V!>y2WlGX;WSYMJ$S!D?XKMHm*5+B?Terr6}Tyik>xgp&!?l)dYo}EMkby z-GmxeyYRV*nPb(P*|0UrRa<#AYe8l2oj5)z+h!dk>DS|iQXD-3YR;9Qz4pmCzfK_H zljXaF85w)0(ALz2tg+AQ{`h+RBp%QgjW)i8#Nfk?!AT z?4fT4nS4-j_Bd4^Lh^WEu$88AfxUGDq~o*4K$E4>Tvv6L$%PH2_RAH?(_f!@3Hbs5 zk@Y?})*~hy-YN|ofU4Qo^ucbwo2(B&CWI4uvWmTHiNSI9YQJ-`)pYvy*}AbHFsxB7 z&*!15&^>|e<-Ky8vqLeV2fgGS4JB@IwQs1-@pdLz4Ur?Zy?uU92C(`iGAm@NjPq5_ zza}S}=?GQK-JMU80At`JDwqL&FP#LDFhlsfkFtbUkZU6#OW!sife$b4GtmNDX`*&G zA~`=$%y*O`;p}x0OBH0H957do8t`5B9_>x3MWND!2yUh@AuGHV>3!>bzznf?UxYB! zMuJ_qYIcqzGc2105bTnoF26z1;VvVJ5Q{hdJa4N3+M`i*g*3pEdJqLw#h*%&6nRFg z=SJdeLM;=Wjs2fq-{fMgzrR|RCIxfe2L+tF`fI0rH&ZV%TKOm1Hy(6K3%11U<_r)P z*-jw#U*G2l5jYumJb5y;6a#%0zCGXRG)rY2=|>^uLDT~jbIL9hn=gcBw@ z{YbOdHTkN?Y6!K2JIBH5i7W%!`deK6otvJp1=k?Evf*~mD8fdnIR*^2*pP2GTs>8C z3p9nTX3N^fk{7f^5N$65=Gr~AkmSWNGCs1cN%PwXumSW`z@K`%E1!9;5yQ||PxC5% z#R(UoZ#(crTwfoR&|QziN|R^-CRw6;3|4*rC~L^7gnQIjuq7UtedU~v?K^FP#p|tI zk&|vE1VEIN$wL4cP`JJRi@x(?dTyK5cCFhYR)I03vjQ!nSN$zt1^tN*+oTgC!7jalvh!Upo< zwcn`?yA2ly0qvWIO4ues64eU%+44K)8C~{{3(c(Ru*I!WPlMb$7aRkj7LoO4zHfqa z!M@%Hx_Z0ToP&Aq1iS;M@!bHTpl!Dzss860NgeOpPS5D6zSF^pa52=+&{Gq=%)`Ga+{9UVv;|bze zK;#g+CDWnsS-ZZbP0g+|y##mtK0s`dZ`Ou0@sve|z>}1;WVHgOPmlr>q8mUbm4S0-Z!|ztnQBlqwg-zD0ri zur7G=AMAjEjrb{AT-%xI*$g*XyQz1(?g4GrxmFx3w6#T)B7k_dyutE2T~LALL0g~}SNQJOSe*EQUpw;432^dO zKc=Kh1M5!ScG!hM)#Ot7!$x`*Wl1?%XEyQ|2w!k&Gq@;QQ%oSJO4jCt7q E0R(INVgLXD literal 16760 zcmX9`1zc3!(_VV%rMnT8mhM=(K~R+D1?dzJkZx&dX=#v@?hfhh2I&-#j_>&W_eWmt z-h0nEb7r1-<{3iORpqcT$T1)g2)2T}^lJzNffanlLJ`4{nN{oO;2VO|YdJWiY=m+T z{DSTv|K14#!6f+ii2zASBLxT1oE21L&=xRp5Ll=`bh%$bAhZw#X-O@&g`;#=PlCTs zed4b3IVy3-Em?7ik+f10WN#h5)6(M*5^3obilfM&XtnwiXux1(+4KYIFv&y|WDMJ^ z)Dh5?QHltGFatdwEu-m z1VW5>tuAVk_e&^Myp1&-lgsoh)t4{SO!X|Qb$`<;+;hJFSS^6j$60JhgnAz6>ojRI zV`w2Stm#Upl3Zp{)Ra(;L4;baVzAe99+(1y*hR%~roYamE*5qFNw+LP+9(nd;MDA% zVIZQ$QKsBY{L_&y$^pq(oq|`kQ|kwdX7v;WI)j&Yk!nW%(6m)b&le55Hk5F@;+ypfzH z&zOyh@eBqzk4Vy8;z$!5eg-f#X>m?}Nghjt$muZB5BcfjVPkSnO14VNiP=X6BBOns z)Z1&6FuJucy*K7b`vcE!BM2(4YoAtUx5`%y9ienQzZf37x?Ml>XCipPzqh@f*Q?2L z7_{tpA2{~m!**r;tc?%0MmBWdm(~ZiVK-O(GiI^y*+~vRB5sMNF6ETEq{oZyo1Sr(jmsJy`O zWn*r7UYiZ`g6#Zx*Y;Q?G9Kaxs$^`_R+}B$iZP?kLCXurZY87)m9O3N4-0(6-fhJ; zxNvCSDF>=g8f_>FI6Ziw@O|<_8zy#A5k2mX7@62f$C?L~>3|S|Gg9Dafa(x7O7Ml(8P3gT6?jtcZ z=i1u6zLRJF7QeObJ`64pJdEycv{^CgA0!#|DP2sYTnJLZpLBDE%!=kba}zVPRgV8K@!*O9`zQOGwD9jfQuJrV{*e@zZO`#S zVHgBKB{aAFFGkm+JclXM9;6q?cI}DKBm*VHgK0yfoy1l(TQiw)5^-)n{z`%MS^Ix7 z`?1Ps)!JH7Bp?@uMvr`JO;ykC^RT|3qve=AI>#HZ;$#wj*V7X|pN&_U74MaOX$T&V zlS9D?RlK8yJDD`}Li|(0<5~ElO<=_^Y04UyAbf%C(pac8e$alqKQs)@lJI*9yXJ2Y zsC!ybE#&;h#Wnr@-~3dt`*^uKJ%fK4-6&Vq+a7T|n5`pV$Ha$XPeL09k925?}yQ2z%1|Xt5#y^ZCOb?rhet|hy0E1B@Bk<~dFZ%tv{#w;_F@BQf zCwHeUZIo1rKwl}T(z=VSyJjJWpOZ^k-(Cbp)&DV{j&Yt&I35Xn2=sl>AlG`DNvGK4 zechWNG+)Zr))*GnfyauqByvB8yb?vBcR>0iG5-2W@m3c&lrgRA@$W?J$Jo0Bh@aLR{=Q=V^n#}g z48`j^Nzw8;H*qrpJ+h5Tq=7~@Th1Z+N?p^!k+sSK&J8m-r&Lhe$B#?a9p^*_C&znO z?SqG=>ZrTm1KV~W-4|E|?+(Spi%q`1D8squN2d##Q=jgvRf1^Y?RI(^=9;~<(QevJ zu0E|C9|-vpfE_fsKzVy*17Q>ce~5Vo~a<= zWy65c8PS9-SV2fiQ1Rf$hcgvaT22sY=$xOG7xsti(`LQX%N|?DZWlAEfA7a;Qye%Y zxNcZGayp#}%0{gMUULqOxsVS2tIPZ4_K_lqZZa58q|Dp`7C~hoxX-X{nu9v#7;NO9#KEBx>Ehx{l*37gJxDHxGmMGP56r(J`260mGJ6k zW)l&xv)=>FZW-X>V%O+8b$GwIFVE_sS}*JZ)nnxl&jrAcUdCiQ47`|q)40&`H0uz< zWp_@FLyvqY8?TET%o(U2T0112qPNpqe8Jn0g7|Mvqc17JN*)G(eFlYH(*-K$zHIxG zaT$^v^fDf#Nx!pNsl!*ZRg;IJkPhr}lw}P~{;Tgk+RSJRhm0{$6 zHx}+=n`emmEKI~UBL2arWII<+I|_zham|fb;3|gjZ|Qn8)SkIBiokdn6e-saTO>CV zH>l#h497%c%RXtq-!egVrD~H7SM-_#emuEG-l}>&)W4VlJEm@T_*Vae$BVl^6%G2# zV9g+2@|eaJbXd?VdB?e-vd;k^U>pX7@+3nEGJ9NuQbl!r#B|g7xgCU@dicO(J}04I zZ%h1{7VfmF=4Ktg!|4F!kwaLN{MrqkpxnGQ0LBFiR$}MY^wFf%=QF~zx(ItCm}&Z> zvM@V>rA2&~+5r<>1y9ae)ib3{(t6@`PsfDh;AY~$!lx@CvaR6+JDZ=m&o$lv_e+z_ zYdO2+qw>7Z%($elYex+K3>Iaq>fZJ=_^=WEg8@J4E)mbr{o@C)_#?S5C&fo@N^0|> zm{FnkiG6I5E;Mt?H$4jzFk=|zLHQqf*My^+oP8)<-gatgpNJM763o~Sb^zMWJGa(Q z9?W;1uC)|1YRs^fhONcy(zXPe+-T+V%zO9+H$3tC;Q6%du&a#t{rkUHAR;zBH?s2a z%ok65X#9EVF+hF9yMfxqo9_1#L)eHh;*U>cY{fDg$-3LgE9*Gz%W$2IFrmAyG&Ppg z;==6sgFgelJti&o2WO_-mMeyi7S{c$%x}I$uq+E&#eHv6p|fLOXl=Q8-q?3%ywP-D zd-_A2K#2Xn+IK^Ey3Fv-A@e^?L(ewUt6{+>Vcyp@o;2f^gJQkbzparBzkU>L6cN{V z*_v!F-&0S6W}yarboz6-laq1GF&aY~%G;1hQzj-$6eof&rtDcDhXF$VObO)Q%r!Ug zdd7WW`%L$FeM+_I*UKM&c@2qe6ibC?Z*oM>yyCo=y6feHAKCbmt55Xwy%drqiR9dI zbn%{iMBmVYB}Ei+I?wO4y<>j*&e9IzUU^^h+!nmltjb$^k3)RWCx`eAec(0gV@_5T z4MywLbi{`#>5U(0HL1Z~+iM34Vb#wD%S(P(Si7ul@&AeZ_N?WA!x!qOy8-XYQ#)(@ zF+N+%TO|{olaaA4-f*7GNdHtM2#}JJ{-Ue_ma*nNlPHy7Cb_()s9Ap}rjqlvSwoqp z$}!1FciVA=>x1@RaqExb1@t%nK~rh?cK)0;ytBW<1fedeFfygQ+)J*H0}hS6B>%TE z`fq(vRDv?M?0>!9nULF9GRcVcW*!j@7Q<(Rx>>e66+6w)1wIyu{`L7;?i__|?8?*I zJ9^gkSa8YnhOiRZzPD9m#N(pW^ErsMufTKuc-%>~*42{bb?FXZa2LDX>u)6x$}Hh2 z(5YP_Lqg9%N`=Nnp8?EqHQl^T40`0x?a7VjCuXI5_@yyb z57(_6xNdf)+QP9T{JCClDneEJLSW>ZydO2DH2or)J(ubB&CnY4f$tDcJet?PEM8fH z&;C|2F(BO%T+(?$0mDalPWbd1!_>Isy(Bj^u1~I0`rC26zaKj4cJ?_Y?@1qp{%Hv)shkxo9#dfa7E6EC0e~G9D){@V@+`=M9A@c-a+YDYi9g`fgeC zB?V#s)$Wd02v+%58GXwFCX5K1!YhWmo>qat-I%Piou!y;xS2p z8&S$L3B~p)gOUIQJj9kj>ED(;^Y$2ZuURURdRk_-x~1w)2ztG_JTqK46bAmzacvZ{ z$p}A|3WvbBV)7q|LU7(bCLG2HEI*CBGRP^ZS-VqyaM$qzcqhdFBz0drImGF1VFwsa zI>=aRiBATsS&ghJ2f48WjiWscvuaK4ewD~Ox)~b23w&YoVWaXK+j5oI4nXoOp|GTQ zGVno}M#UCIz9-0|M@arG>U8t&)2sy#em=98OK9!IYU9&?%V@dRYI_x_HmavcE^L3f z$__JpUrOZ#*w&ttN<(S=8C74r_F-MtY?n7?GYhB2^r#=SI<4L~(yPhk%1C}Y;4fvi9Bu_aU5Px@Z>5?jWf ziJ<@R?J~u;Gx^;ByTho|QpM^Iz~!0d^Mu1f<|kZ8I{{py*%EASZaA~}8#>kaRK`o> zQdsszmRGdM0ZC{zy~S6hg*fsx$W!Q*(ydD`(IxRcSa2NJj%{F!>wQ;LdxC^jD~X*p zF4R>kIcw`n@xDb)Qsyqy-Zos8{3N!khVEZW+qr&Q>FlH=jW)%=^5T$n=0E`C~s>VEW`fh0_hgi1up_c-E zGNj=E=gmW;E{iftaJ}f^F#)47y~;1e(XhxW;obM0j+tZv53TL-BgLabxi=D9>!m%y z)3Vqufk8iy(Z4m)ZfNusDO+5aiB6w^XDlL*4LL_&<;VY75W>6DBAF1U@eVSS$Qp2Xtq{t1q>aI|bi!*i3p45?|k$iNtPJ#(v1 z8%_Db8o<3j8XmIfXK3bV+q_uv1R2$>{P*^!`CrdHgq)zh(55`u^6RYMeO=RiyQ(;m zzCDJdH}x!E#xhT4ZzhR5E2=JbG6yavQddz}AD70^KmZy=jWeT#CrMbU+j2bYNo>)5 z=7&TI_Z@Y7PX;J!_4dDFZ@=f|n~xn^3!ls9q*on*Yr0rgeAQkfJqIWC zPJF5@z99J=^h(9I-ADD;!{j~F)qjuQpJoDwR#W`*JO84r<`tKX$)VQEwD7FG-`;kG zNV_SOFN^aQd~5r|7G6z9C zVjd#zlsdnLLjT0o&2^`fvzD$_Zv9Kod(wm1vj^R9S1$gxRT{U^9Al^MsOlA7f2 zzJogAlk1v$iI8y;vG6v|jh`Nf`GQ8Y1iEK>3&lelV2_bxlb~WENs`jS4;*CDAB;yD z2ez%YC9m`3^w|b+@R6HgEe?#iBPe(}m<+~Cdd-{WHwGKGeSK^EV?t9D$-@!6TEWhSzcvz4#B)ki247H9Q~(1maba z6QQE(q+KhMSt>D;le5&_ky@QX1#V04{SC~-kosq^m^I=%o8NIW3u<4CzlBGu`aw%a z=Wj8Gn06O_Cpdp8qtKk!YbiCIUFGc)9#rcl79;dDRBR_KzDiLJ?4OFeWK*E$uN4>( z?QJ~JmXGZ=lEnHlrn5xA`8OnXIljfVdp0qm!y+P5SkdRUbi-(;*Svz$;?t=Y*gKcw zLa!@#Ql0u|+A8zSB>ZUM)@P>I&49{!g@x_hQ%*uTWAO2CBSPJbWs(;&xq zm=QwXjIJ0kAfMvtW^jRrNx*6jK-BQ!m|(Nww->+rPvv?)NB6>h;X?3MB4)V~4Q=Wp zV4oQ;Itm!5t4_-=%pU4wR!@qrbEK|DlTIGq&FRg(g7`w&K4{NVEBZmR41eEcKz+Jn zcSVyDlEIbBCuTWuG2Y#S(6}8avWDRSMSx|w2O)KHu;)H)?@G=noNQQ^IC`drMj?bQ8D29xfuEa)H|pD0zCd*~k#s>~8Qxa#v{^ zYlGmc8G7n=WGQ5bmf7PWzFzw7%8y^*DW3lC=F(_#@hA*l%VB-W-DcL?#F4Evhw zX6n9lAeGn8FL2Y8P4g-7vD3!?^Lg`1nAe92XPAf!M0!q%={ABJ*30TLqSe;_Cze8n zMo}~WyQ%-^Q8~6J0-O&*+o}5jm|c-~oZaUIK=7G4DqK`uq2#r9p;8j;U^(!N zc<`hjt{2liA)z3u6I|ls$A~8!JrrCzN9bN;7Z$M6sF2aCEb!c)_i zw;56LEJl9{uGNmucv>1h+)Tew48WjsR!DYKbTrf9Hy2WuqwFfUSw-id8M_q|*#59( zarRUurb?0tQ0Ee7MoP|f-aV^6QTmiO z^B_kOA|DsLO>&@FRIf z$=xb0Y@)4S?=OXnNtxu8H?F1XJz&QCo`#j-wQ-{OP2s~)48*uQymV}Uz^sUld;8_q zLNa20slp@V^OFIVYit%?MAsjm6O;aBS~$#D+tm8w-jP2v3%KV!Z#N`e;;RMM9&zaV zP8ki%=B9lfozm%%tqnLru&@%;h>yagYV8OO@6#~HnOC*w7@JT(!#e77+(aB5jtArn z`mu7ylFNjN??QP3QY4vSZiE>y;6`4`{dmek8X2sgFs|%KAALEVNL46OVo?+jf+pcK zbxZj}OmWc(PQ6QceHhENIXrZQ71)u!S?kjM8LEmPp<TtGBhzR}YD$F=g2ysJ$zsC33eqct(|ZUki&wB` zF4Uv$Nk9FEZ27ln{S$Lh^kX-qmv^bzwId{)O<*t1XWGtt15c5ttrUNH3<9%Tc zVij!z4YKL@;f{F_dMY1?Vj2C00w)jKx*RAw%;2zf7hYq{er)MqOSQrs9wk&I#jDNVe700ghQ47P%=VGB}-vA;mrCZ_unJ>wBN!bIDb*YxA=@>7XuYkFAwR= zMK9178)meo*x`xqk4_Tw35xIZCS7XLq(!a^3F*6#=;qt##hb`ThQj*M5^0%Jw6#Vi};%i^z_s-5C$3}25r zQQuB2owhnZKi~T3Zfk2RC55CAd$sE098upCx%AA3OQ~ZZTS?cOJbh$+D^z@qTS6eG9D-#om z13Dz4{qO(u(S#GT?cZPe+#S`)=4dchYTR}!C9y}+h>NkZVu2_q4W+{wndfx21^ijk z3B!6lw>a~3zooD=>F?_m9`eXHZ{9S0IvN-p+;k8Sh++slt(6F6RKQ0b-=ZPMLS3r; zL)g@W-%cd+1-7xVgkYzU9-Xu|U@1K1CnNvHcWY~_?dj1&M~6IWQ;(gbQCGqT-`I7k zb{IwW73DAZu~!T`_9@@nTC`*S(2N%XzLXs+JT1YBUs zTf&DZE1Bns^{GYC(96(BG30?jC7)^GQ*}u|JR&doFqpWHA3qx9;_dd)FKOJK&upYa zvrKpkyeX>E2T2j9Q=F@pjfZ;i2n#3o-j?YVX)!xePPihr^gY_+QZ=YsPZ0AEySS!E z)m(65ag#?L%#`L3LQB=Zuky=THU(K2d*|}j%?2X&rnN2?O9qZz9xW;goM;_ys_pxU@PK0Dwe3x zX>mU{BMzUAmi}c{^`4bDy043qm7boSk+Hk44{gO8X3WBs5ntOg%dwW9`kk~h(B_U` z?iCgT^uLGOzYoPUFYrUnMl+7~_Q+r`T0#ur`+Q&2?}A~+mu=x;ZsNdq0qXH){GwK1 zsq2H3kX%n4CJDRV*TJoaVisVim>3w@+1W2pCa^WsX{Q29r9&BNBbU^3%CSj;98V9* zcCywy&>CfUQ&MDI;$AKG+G+%JoSw8jr6}hcgr5?7{5Rz<&|;tlLk>1hEZ+SU!yrO! zKU4CaRs?1|3rbSUDo85=n&pv;*}w#}%MG?1K2utT(L$}1vQkqQOQ*8M#KfXS&g5le zdh(Q#z_P*Mtcf!3%TW+JfGt4%zlnPA?ZTwzII(JiIX(x-=8P{dd!7C6H|dMo zau6UM$X9yD`c_{4moEZRZv+`b(xB_fioUvf`{p0k8(pzXA8_j0+S+&yodm*{3L1ao zcS=fwH|IOaNEC_HA~&9o_c!N#5o9I_ey~`^qH=J_n3x#*wuj4(05n``5&0Tb`_Gt6 zah9vCUVr}l!4E~wc!@QnCl{x12Y4KsEzh@Ib9|CM7jgG>;y~C*J#TXFv zCw|{^o>7=39%@glj{K{)HO^>NGv2-zu9X>07dCfSny}!BBImVQ?+m)gypmCixE(R> z3T|j<@VZ_PNwnEq``A#Rn%?+uxyZo4uznPhQjt!nLjB>F3(=5hAi0T&x9;}>z@seu zWc%^DC?zF@6%St_)~rg9kcRI&VcEWfhJ?#$`HJVN9G+ zuKogejdc)CYYAvi3y#jV2QFt(jEA7sLlVn;V0ZL>#AAB)?B?cXzH(A#+CJ{NnpqXw zz9A1k|IW@%*CH;x&GUDpbL?N0P_D%}clVi;xv#Nh@MW?2w#>2z=v{z3b z2poo)LS&@zyJ+$`-}7+JKn(&eDLaP1UXSQ3f2u~OC8oQmy1}hv<^14aRLoIjRaM!` zREWH?GGTa6;{8cNLV{Tp8S)xbpqo(UD=nmtlX;|kp>QVkuP1Sv%hv$51%bmVRu#KK z&hGH%RoYHdvGWc;MslSs`tppy3k;l}^7mJuBlw>dAVVcfXArDsj%*|%wgeb69OTpL zV>)E`jPHN_B2)Ad(n_)6KzTi3Wn)wQTiKu1KOn%Pz^Ry*1rBH3Bi;}&(9}digy7Y7 zQTo2LV0L~%gNHo6v}BW&crx2^hewAPR>Hhwn6^}01nE0_0!p;D% zyF{U!w{^OTl8R1AYOx3}GiVXw;n8AF7-*DaPPo0hYdafYVI@B3rLS{HLXFz|*g%O4 z@#_49%GJ5HV<^rS0Heoi$hp{=466!Ybi`z!*%8wIp&rZlo^|GIYmiN3UWZC`ZW5Wk zISSx(fnd$z>Otp^r3!;XZ@v``EP(RfL_r{nJCAd1K=_XntC7?UEQGc1f(uH>tlotc zF7g4i{Y$-w+Xd6Nm#jhN2kdjLY*y|RYI(2c>w^3H`x_G2!hH|Iz;gG=cc z-yvIM(!h@a1<{4;Exy#@#O8TcViq0*?&{c2mWD{eVP{j&#@BQ<E0r~3V!GuHBPPc{D!sewSU4h_?e88h{m6tQ$#jmXyVaj|lCJpJDueOZS zW3P+E*@FoHGfsR;&|)SEm8Jz~QVz8dd#y?*$ATIA2aX4(jLrIy0mI%#$3Pcf6X(x& zhkJXLS=k`yrVeNNJT~Z*)=K)4VRb1gDb?Ap+PN!h#WErxA<;E)eyP9|5D< zQJbzyjv?a~hmt)b>xEiB*mP@44BIlIo*5@F$4rd=H0cw&-OXuiq(;jkzz9M|M?bv` zfHoZv%T6|UB%r?eSzFzOKKXD$Sbfw~&d-zfHm#=PjRuF8q9R7>@NrC7;o)TxhoM2G zY5(KH-L#cU$f;tYC&f8sETfB1x*ogF)5DeP@scGExd*w{fDIom8CT%oR%aL)nqSR- zJs2U4v{+r0Jc7;_4U~K4krh`_Waksb+*61Pkp5I zea0t_F*zZ@Wq+y&V1~ve&>hZo{0ZC6X$*|8@L4v&r& z{1B^}Uz(VJzsc3wI51cW>M)X5P{6|o`kGH7?Zyz?k2?9uSo^1JK#(6mLZAqu(F01S zZ=biCq*{lEhN5F*4-XF?_0)>)=M_Bil1Iws`~<^kZ589-h{lkYe6DYG#Jnc(m&1u; zy7Wbe&a=4KSVW9Kv2_(FNsw%~ZjL?>Ww^MHFl6CQT!O+&5ML;!^2>igRfKvpR)C$j zRtuoJkgjzH0X_DlK3}0<|NShfQ6uT9c&VBHZVnYyuTD+=Rlh4`bWhW;b$8tQ`MY-{ zx>^s{VkGfNyN86#Ikz~OKlvP1NE3826DQYag!&JgaTqk$%W0yDJ<)+JH6-e--N74 zu3mLIi6qwd_yh!-*OqVQY%T$kVxXr7gj6T>r0Ikm%Z($b6|x0(#11%Uj`O9y35L>9 z_9e0Flhe?gRA7r(WsM%rSB2ox;9xkvqjWY3{?D>kIHWi_?gMoyF8_5R zzcx!EcLQ(wJiy6>vhTEXK3(7-W4BB?DVp zxUg^uf;o}rkBC=*+9>zvzSnv&%%^%@<~0X@>ToM?$C>WiWPTDx3LDq0i8Kna85tP> z42w-`p@MAELoK%2E6?Y1fdvUqXl(tfh~d`h&$`bkvRBUjZS|-7^8i$Au4Lu6$pa@S zW>v{O|EZ)2{AsybIX^o?M7hMA!(oFfblPvBy%d&m*+MlJqqymQ_5n3vfM1L7zoc6< zIzlQcDsuAATLTH{!mdB2t#n7WHI2=HDUh(fTRS*V8bRKeqcJ6{uy}+1T(TD|;n%W>|TC6re29N{NmA zxM*8PSIiWNpr;=Q%^H?jbd7%;>YcamZRY7$-K+GV!%{>NN@oZA5WHAeii?Uuhs|`0 z?Q;G7Que5o!%`nI;EZ;Pk*oVAvCKI2h_bj9?WB1?XervLelHLibYY3Z9s zii9^UyicaA*zKat{Dp2eu^q+Fj_{qmiU(PWd!n_qivW}*+0N_f8Wl{A)(feAyH5X1 zEFV7(TWgg@4op6wjcCL2YGhECUkKT(DY*J|wZ(6+__{d&$I=mLj?|;y*L-h7L1tgP zreIA0iMDg43oCe)&fUUw2&-Or#tWilbUkbJixk1cI`?deZ;+ zOvMl+0_%LAYs)TTgjY?=g)*!}`Ygw>U=jq05ZoY(ShgD*3qL+n6gaLfeZ{Bq7_!;i z7jaIFG)n55a{3mp5~bRfG4gZ>upOoA>qlhGDZ|4pj~V1G8?jtn^1cxW7EzZlB!+e@ zii0f_^CBn1QBOJR37K$)S*CL5o%)U{wC_ySgl6Jn>Eq&uKm(#?p&}HhE~&X*aAVjN zfF=Pz-(Ed2Swa?)4~7YyEKtw!@h2xF8Qf~TVh)N8g^pS8P}5nQny!n4Bx*VUzm z1W<6N9i99ckYhD|qYXX!uu5XW5$+K=YDKjPY9OnlIwp+Dr>NkpR0p_YN^OHcQu26f zK_E0BBU(qEgA12SePGiRjUnpqRf9(6_CWv>*!Ec_ZVU-`5lo&AyZ>A~i~ z-3=W;l3A?->s?zv;F!90LWUdumV4Q*VS3-sw5T%Dix1sYk8I>QK-S>&_XU?1X;qko z3MgS{0VmkRs4KVpKoI!Bv3%URiMO}+s%rL|0(Ub83-Xru809xIF@!wuwTY`()ll>;el|4f;bSj8RJ6FOydJ1d{=tF-2F=*1kz+-k(wUB4KW<1k+9XGbgG1r1WR1tfXhsO%+hziF;y%qM zBq*b_ayd9=G6UFrAm#wn0M0+9b!&xoGpXXT~czC{^ri!guEwyO}}+Nx;czDr;@+Fd@{Q+r--^ zv5l_gSo{=g&4ylN^=mEPNij%!aCY%OaYptW)|P`FQ-82NkPVf zqeX4t?eeB^KMYCe8JzI)?P2kSQPIlM$4cTvSELd8PSk+2xMq&MJY$r|ps2m1R|X78 z(seu9m_#UDO^6qGwK$JWeDSf>gK zpfUaNdwdQL9Mfu3c$m_puN{j%(Vf=|0?FXb`$yE5vnC=Jq<)Xnlt(AcRCmlyA$_VX$IquLo(uEKT+YwC~>dc)H)pG zXWO2b`y3)a0@=!0)< z|1;dm1Z5Q2_6vg@d{l|RxA~x~*s_7!C?IYV#j)D=KKn`L~-NMhxluzoJispI>`F|MXv?PzYB%>h$2AeS*c{S`Q*fpnzNhtX6(FP(N*~ zQ9l1f?=)awsm|7L&$J7xD1Z*D`#Ts}7bQYJVlF@k!$x+TlU0iGy{qb(t)KY$;rq6X@+y>0tUe4?l6p@5TN zNVo;(1h+9nY{J6oGp`q!s0s#z4}YEGY&8Lu=!28PrT-WAvQ0(BkSeKT^GC*(l-5JN z2Nbta2t}y6rTQyPa#2byrQU?b6sgQ`&odx6bRL&ZS!q0bT;Df2{ee&mTaTK^blEl= zxLlJr6aHJ>jqN7ld0Z8vHg=$d9(ymaQw4Nqr?m)PES=+YvJ+$NrL=xS-|jydL%lLJ3nR&h*-Sh2R5wgia>@ zCUuMV?(-pN@rTAUAcSwt9fIsYX7JX_CJcA2INy;9^t+1G*B}9HPI*X~N__XY98i(X za<44wdPCBXmVQ8q*jxazo#=7Ts>&ii#7FP*(m(ql?Dc$14wOm4{q40)y>j^vI>Z44 zIMMiI2D-=RQoM7blxgR0mx?hvch*T|X+$*cP9{l`K-KO4&;9<2?@_}gkbML@ z8D)AcvIBN$F_)HrKA0R=_))*d$+Fz@TUC^50@A-Stzi$%u&v5Vjtm>cHta)d6W~4O zIGy$DU&Ong0=a9VSu;}A4S?=rKnL+5f@KIJ1>Gqik$@g+fzx>zh}^%Sos(mR)#QQx zhpfDNeXq2rO-8eDATvQ-ImGiMwznL@1d?!%kGdvO=eW*kwO^l+o`{k!&5{I$!})RUN|d!TUW5je8Uy0B z(BAU<5xQ7i7js{07r>uA=g9vih#^NL1XwZ*ZXhzB_7$z&6=Fp6x%NL<`x)S`A5gsQ zY+hZDpzaQ{L1V^tLATH4I;e~T<*GuU82N==r5wEga!QVG=}no1N7egs{)5#aPc`ca zKdFCGy_b^-lP$iJF6JF{>};Jt4dqy-hwL{6OWsce%f)_`*8&Q3WznpTJSU7{K|Mg%i{m{hkS{UkSC}wDd)^lxgt#~$5 zbf6Y(H>`awu_mXcMT%~cD1_#@0jF;sa9bI-ixEJzUOGhLUh5%p;S=Z%lKxgYl6OQm z!RV0KV)O}FBCzaPC#y;mbTf<1N_=D4#8Ox;PObfzn*`6e>b+rpTwO+>@y}zV_V<_c z??{3FQLmlJWec8%gTJ$HlFUY*u>2R^{{9oA>IC&CHr+Th97l9K18e4N+jm@6u3#ps z(`Vm7{v9OdBXYe#g?#c8l+YP_hy3qQVyz1l4^Ed&dm8T*N*KaTgB@yQ(~BZ4-}hDm z3i>-bG%sa(6D9;xFU-6mTZuh4e$$s&TYhK2M1PNHpF!d_aLnm+m1wcG276ADm8O9W z*PYUkeu1@DSp^X&fU=|%TP%j`VB38}zDI7Y*2j4{nPw0Oy2lo9K2-Zd(2CHDNpYbL zGh)V~wjNG&+@VrygZB2ZTKoafbp61;!I#L&E1s|{sdh?gYu^f2RStNKhSVZ8C9Xpp z!`g`B=2~gdAyE9zEg$@*x3O3p42Q)!N(M&3E55tr>VU2=I-kO-*}$Ws2~mkaoGvE~ z%%!>ia7Nsj&K%`84tShnie@E9BLYJ!L2eM@)S-gY=`g>BWa1h?lh(%64O7J&g;3!_ zDq5g|0v;((=}V-SfEX%HkbCv5XqiuWnS!qHN{LE7!;kttzF#@Dr{cP#sdO`a#; zh(rh|Hz+GQP#(Utfnxt_>1}=kir+Du?QHm-8);tfcH@RIhvZtiUqNxR0A1VJ-*^X( zm8^N}l}Pw5zlt+k2;Kox?J4dt?DLld53bGIK!dA;VD}LwFYOD}&lCKtkUT!~s9^GV zi+CwxVv}wAN*4>q(F`x_U8y@1J$8ze0u0BWW;LW13FUDOW=>2;r3uw<%R#K*073hF z%VU$YmDV$Z1h@8^pi8Fuz~ApVr7v~%yNsM#OkiwL_j|RaYiLP@=*h#>SZToZ$cgqS zo8p3=6^d8uZWte=!x{LopC~CeeuL4E7ml-7G-Ws7Q*J0PS&+!oGP&yZ+tFmq0uUe( zt9}<-)(Br5uW;{_k+xt|E;nypG@YOibl%?Ha#ZSg|LM2#vQKsT?-*d!q1zVeSsAZs z|DuwjGYAYCz!)7hl2!ldNZOyX)4@UK43%rke2JR2K8pE#ILp3`?ILdW7dFMO-TrEN za6npUWOP~Htu|Jzv3fXGZzJ1A{~_=1rKR>C{Bp_{48?sz^~{K?i<%UDvSXlacLU@| zYXl_t>qeINZMHjwew=_q@@umheJR77o@LPKNW-Lrnpyu9>Mn5gsa(VP56R8R8pmQ) z&V~6x>!d4UwT!+JxaUi60?WI$;BaQyD2#U0+~P7qYo(VIQgadQdus$q1ncU1MxW8@ zVmPS^1(4aRo~;tso={UB@zP6^B8xc+;%$&#N{nGOURT|_%7>eC12h7f>zo|{Hd-Ul zCZvN~{CF}Is?g3U0?jEVYrGT^Uw;Fg?d}F`^tH4CWqpr~%nMWW{JT{dl35ZPe=QZh zHTAzS_r37OYv*p0HG)|pI{5DT*Z0nq)j&bnWWym)rZ!GWF}qz_5Wd;cC&9bBPk*t| z|84VQ(}1_}aS1}n$bOm&@;v!Y(&z?Tm<~kD(FKSPfXH_)fFa8UD2Alg%}(`c9P4j) z8*VY3|BR0bwno(Jf&z^C5b;0xK6^&tL()w)WF{NrlT3A9v!2xgAAzWHjTqFlC-0BR zcZpR<#0PFlR>mOI)$}@XmytuRd%TSROSs>^ChkQX^@T2~_Ec!2vFER%Culr4j_f$k zqSVVo1X|uqPY;b;o@-zlf1;M2ivfaz=-`q3aL{&Dn{W?RZFZ5DE*p8Tw6iV=Rr#A5 zkI#uBf<%Al?{19yj5Bz!F5%uy2SFFxK0)ogRm{L6g}t{C*VD3{9(1!*SOZ>0=026H zEMBbge*Xukp4L~+dTQ3Y_o!FylJ6Kc*}jtH0NdJgioJ>1q6k z3*h&Vj5O$2V%YA8DN*>GnSqTc5xC3+x?o%FL;GdywHdN%D#b0Wz6}p=i`z})z&7)#W%L)P=$2yJ$?dcHJiKhq)6il^CF4`j zV-$P0Me{~(mjOu5SVO&umoOlSPD(%VIRjgSPTjdJ#g=N@`i+9IiTOL_NkOm7%b7%+ z1g2F2P9jP~fJgB_2WA`fc`VxO=I9RwZ%-OCB%)mjAiD`GiT>UAF>ow&!6yJ2As{cH zoeg68p%TSxqPfQHIkzbRe0O**DgRalbqC@_)W}l)(@1V_R7~Wg=rJz4r!1rW_uYm* z6(Q-1Q<@_|{1x%<<XUoMjZw{V}>BH3d(}3i`WK!EG2m*qjQrI<)dbcTc!7UCD0wW6-Q^jO)H(4b#{mQGui=V|6+5nlrEuz9-jBthc*-5=gD&K zNDuf%*8y|}?FvF`O-f%)YniS9+(TV3F*$TxsavMuLE_p&e2@s7Pg#<&EO)ty5nsJx zZwm#IQdEKVC~BZG%jN=;x}eb0HmqMG_!8(uuaH%n7N6L+4a^}97GV1`rxt^@gqpT9 zbxmEqOPuaT+;3}McLJHO9z6|p4Ze*mr49yvbyO-qJNw4j74)gPHIIIasr(0qZg=&ytEaZCFv@;ZaWHcvS!`Xf zFCJI(>+zX_bAZ6rXoPC_iawVX&;5O}d*S{GK&AZhExqP{KmkK7mJurv2ZIPjH}AlL zUL|wK$Lagosmdtf&!9T@DAFCTMZl|xDc2*ZGQm@1H79|EN6)0yXOGa^xfxSa_Dfvt z(oN)&*6o4?<2ML~e>F{LZLBETt=?GTdFq$SRk)A?qD7A3cPq;@8>GbS7!>RjTyQ={ z4g688VkF4OOL)p>sjmKoskpEyfEQAGNqS$2NRtxMVNfwsh%7&;nqj{m)2d%zNCYm$ z>D|^p!e!DI(5fVc`vTI0Oc-RWR6KJG^0~i*z`qJO`}cnVUd@|)74E}c`6sNB&*8Hg z;nx!~^!Wqt`jDXEHF*wn@2-Hai??NrxXKsH=G_xY;~qC(68=*@$+k|FC;s%VQLY_d z`I0J=HdN_rw_*b|;NWOG*;C?YQ!pmu$;f^bYfsHD*!604Nz<)HU#CqjMp+jwx;TM1(dNNkR|4d g5u|@Vtu5S(f1(RY$iCP#1*3;3$f!z}!HxX?52@QCRR910 diff --git a/doc/logo_small.png b/doc/logo_small.png index baff7eeeca3ec7247eecc1bee4ccc54923a87938..0c79cf46ca845eda07ebc4c4e9f3bbb0ff4b3216 100644 GIT binary patch literal 7828 zcmX9@1yEG)*ItlV5OtA|mIditLX>9dmPSfiy1QfP5|D1B8|m(j1!?K-rKRD&zwf&< z?|tW<8*}IM^PI3R3X-p|Nw5I`z$v5gMAhx@amIHqRW-s+ zc-u+aeu}iBl++uR+AkU)P#%5v->RwrU`0SUx)TBJPgZQI3Sc9!teJ{lSBwFh{?7-H z$6KkBpxP8U?`=d#;U*}KUD`Bm^mH84$9QLqY4>@&X*Ycl1V;OCX51Br&97@FjczRi z1OvhMSthgRBo+OgvlGec%GkasS<#cTyvoq|;ND^FQcZ6U`Tq4kzoZFv&*jkZy8%Ej znlKOr6(-@t)7HfcW1n;PJC<_+ja1Q1VU1W$vXpv} zsYkUo9HeYeI?BEMPAid z*h%Aw{k|Le($<~%eEt6=vUrXB&T0-bLH|msG4i(_t`K5V>USar-+zFuFerc8 z^9L3(#nZqt8FPm9xyy%(hTa0%kYCw9%nAjVOm9hXD=Q}8f_ngK;eu)T}i&;CxmX^YP1)BuEvt+ye6n2qxqGb+OW%E)k z2a`TFlL4nl_{%dm|I*V*^Z-yy*q6DL2J8gF`~G`+mDCUSpNSKKmj0a2<#D>~;(}f* zh%vBZuR~pg;mAS^SUz&Y-$rZs6^_n=m*Ejdy{vTlD5){`!$Wnki_T`mFA5R-9nb1z zB{QG0M&*~R97yXUC(fHLMpA z*xG8L{ChKv3`X4&VbkA^xgO=VZ;iGKmt2i@8QECnZ`t9hlwd2VVQT`;xbNR*w%C)2 z-+xAIzn(fWkgcd*>#KP#sZwL4@B4Lp@~p9&be!l9GW1*EJ$1A*ZyWPrr2eku?mU*+ z?JXl?|E)#oZFjJ-#JM&x_t&sy#6@!(aW3#J3w&7nyKb^?#X)XL0}d9+jYnZpAc4P2 zOYI^eOZ|u>{9n9RYWvxJ?xR}0g;2k%_{K&>^Z46t-f81!I8rI8k%j!NwTXR<_&r^;V zukwAkHoB7dC{79jb;l1X+lhr!<5DO-#=fjW&3Y5 zc+8%59ND%H-%5yx{SU<^+LM`FPaWjqOpxpE|?hF*_$F$W`c@K zBsXAb*wJjH%cAAn&QtzXfAMQo!@SN0{{Yuxo#RZ>foPGDlTnYvnZg@ks9(%ait<{u zujr(bpD-XN*XgT^Y_WHo>x*$U-nu*QHn5Veb>@Y#6*;Fq+4oud1Is$~{uCZD-F+^m zY8q6I#0?hmVgSR$in?14o1Si5di>EVylKUs?wo8;k#Gm6iSMUkKU(W|@o<;tsP@ z7wV|{b_1E{gDyV$qiDMwBK=VzEXB+>W~7`zRz;BC<1sHz`SSGmP6RG47+GL$)3H7~ zd8SB{|Ap6kmjyQQG^%~!;+@i#A70+?hXurzY06$a@_F*kwhk4B(W~X@7LF)Cdsp3r zrtmaq1bhEzq?aJy_8MIX-ynnmbMUqc=kja^*^N1p*U{QTYW4-JqN-I1M(tFjkyO{!*05rD_2X7C+C8@#QrG}4GcEo0nUAk0IS7R_c@pwBXn9rU z@)$AyYxb@Q4TKg70K$BaZnmgou>zAPbEtP*J*Obxi_q3V6JXzfURt~T_b!TRBXxzL z73pMd%Bn1*7%fuVYO~zj1eJJGJJ{i7+Koh#M(#{3g|SzK7`>olah+Ayw#jZOt8!>A zuD9>a9x-`V$7l3frlMeO)Y{YnAb2BVu=*i`wFZIJ)*}BotiLUK%qm4p{6Xhoje86l3b5fpR`dJ72`Fpr&OlK{nT!^iU@emrv5F~AQ(`6R))hVOtSvS-tVlAj|E0U z>Rw{`0W&P3*`mcQcf0BBdVTA13%1V>gZ_|NujCsW%q&ewI79a)(6{2eyL)7W(MpXg zmq_k@$m{<0+K1!7{_j2&3+kolnZJCK_8}jo3_&%AeQ{93r)LS+V4&I2ncBvVpkK!l zoJ!N;)8@K?C)ohi{4I^z2e!tC_DpU7Ktf7kXxHUdqr-1SUdpuY3Mc(`QyPRvd2I=O z@@S!M<#QXX?C~k8fhp;;YB^gkvzc@ZRWC8l^O_m4Vy?&2O zk_yv}p^#--4xdaA$z{%o0pQ*?=K5z2RzC4%UC&)5lL7YRwv-`hXSs)HqgB7lj(tYY)JRP~m6LcdlDFm!M3_a@o;XzT42f^DH84hqa* z7V>UCcnd9x`L;oh@<>Y1N(lq{abIC;7?}~!yakAb{+6VQV`?P>1IncC(ExWhG_QfJ z{_#Sal?7_YY6eW6ZHu+@zwFk@937NP1)7Cg1?G@+$wrI-@rb~1Bdp#YS6@tcrz8+?-zph2H%;5bxO&kG`QgP zaAMvzDAmm|6&y(Spp8|zkh7C=9`bzNSw-#3Q-y)0r!9G7^3T-&)Iv*O3r-9M<~*Bv zmlvqwcQ;srPNabiyC*r2Wj$ir0f$M0L~SM?W?mrD`~u}pn#H1VAK#6Gph%cd*69Ev zCFGw+uqPFi8MBKT-T9wFl(A~9CG&tLv&Jv{)T)*jZ5n|ygDJ|F=70%g%OpiXxnChq zEe3d3T1g43tyRqC7v^eoGAi&dtv*d`z|*PGpf#}WwE&l^g24fR6#*Ao@hk}-F}_!b zfb|WnD<&X(uD4DlxUo0lP6(1h>io@8;4@fw|DhExCv-Pl@D<=~ZN@XR2{1&~H{R&$ zx98rS-2g+fJo3-Y&hJWx8YCsg(YDi|d9L+WIQPb$@wxr zA_@bw>8;8gm#Q)b8?;guRgfB#@nVDJfLJQ^ctQ;9@2G@Hw2r6N+;rxMFrfgue@6>b zg^IhoyN6dE;ki5G8HXn)Y^dX7W4euYIWrd48~~`4nc3c-d|C6zMoCbQ z{Kn~V%OQBGE&vsD<-E%s^~eB(-8&+F^W9(t_(Akq-Rx~_D)#nBToRNGa#rki;7y=i z=7(>YCCDT%9Lf7vp5;9_udK|fKazN1Vd0;?WWe-g=w(a5`$C#ABF9U}eTeA_)Q`($ zCG3Lbo4ov3+1DC5T0C*3w+TB`XnvN#I5heAK-&$u4LY)QfbhzVnv#uq01T8i_V3@K z?T9glrz5?zBd!GzHwvnsm;)uN9-7~B6*m^tY^Fqwo12?ujUNk4HV_tNV`s@MtYN27H)4N1OW2a7FWoY)Q?svUH>GS9h;?t~E$1rKN)IzVq=4)0 zPqC5|!~!0*@$YD0W_5qpdqQG}AciL^ZHx8R@k7Qu%*$aQfU2sh*WG?0(rd1x^U&uc z5rNZ+6hco<6%5~SNaC5;1Rw7sJh88v?6^BOHa0NiL}=q#v}*U-?-6*VoKYM$(7|Mg zk)GRnP)SKi+Ap2(+_?Doh=_=`&s6AsZQ}P4EHs59@X)Y-a2iRDsD%?9k2$pm5cKx; z*7=Ti@=4G0xNc%%qF~0bH}L-IkjHUHt_AMIFC?zSq+WLF08=`U0t~X7M!?DPqqnfw zdgPkwmel4HWMxaIM6m;YBqtN$;Zb8e%GY>Kx?k>34Zz`*3Y`fF2{be`dOuXBBfjVi zE0Ku=f)297Owb8mB$$sntBDun<>j@%pjAkpAI<*eM1j*2Ny0sB^>GeLllhT_W$a~Y zj|Xml$uGcKoS@nR=VYa4IxX!N#Hw!6T8U*+V&d#=Q%6UKLZRc#zt_}~(PcFuDY^8M z@t)RVB<0aae#v&W2E%?VQ{L5!7St6K4`fMf*08|F#&#$;EA49>7xX^!D=I3|B4m5| zNJlr2wg|OP@FjsrC+r@@Sntj~(25;Zm$J24-h}x3`#ZbLV4$JV<2&Pgd5wcpHWja- zq2XCWP_GjFmJ+AHmrcmg^;G|?x`S=$*>Q-C>Fv9gUc$qg8ZDI~nHbt_JDHS;qvx_DJw zuk>NCAJKyLj51)*xo6*3C32&l6SXhc{>WS#O$C#>z?fPgGnrTGaKA9!<)bG_tn7HY zGgaL%w;jK0!_Fz{vQr}pI?W95_I00I0|QP?#!UfcJFmAO5Qy_fU82qRz7N%{t*t{3 zM?%c>^z;l2O-;RwdJW$9|MU=2D3nlwXaG6e^YYVB)tU%eXhH)y<858aTy#`ad5xMV zkO4oOMmF&i>#JqWcQQa+G=rfynt(6Y7I1!eNJxn1XDDm}S*10*!G5K|PLtNJfk!d8 zi7Myiw=KtK&?y`me*RX$$8dwK^v{kJ-#T8uCO;cb%cca5B(Zfg^&ivQ+uN_$1_i!2 z(MKl@DpKv?p0rjD^`LcmOLv-7s$$hUn@&R8!~RP$re3Su*(UHbBLyk>KAIexnW?ub1%QTXl}Cq$o}$90A8s#5-7*qt&xMD;c;D3xo4jqSky_?^ z+1`Lv4|S~G3^tNCtKfA-c&&B%^A?99+ucINEJ5c3RTC4M9`>iFr>lHCAUJ-=30YID zks0G}tu)!D>UFR$(4OpiBZ3VB6VubtxgXIkf%axOz2y?2<#?)jR>Y$6*OD${et5umss{{g?d3m);XC%bL^dlkt z$V%Yu?k+Aau3oA}R%Gx#QfoB|9RtG!v9!P!EVS&gWSaW)w??{Btfaf6d6V-#q^bWQ zBu1#XKy_-PfFJitDxQr^&-;99$f2oKFp9?o4WDaXRvcKZRgP4foSfWc^z=$`PD2X! z0yvzKAm2fCBJJHyR-;cUbUG(zm7Arhim5p}TmR|dHi|@d(imxQqCR1OyZ$*($1^v=}x5|U;$tuAu;59>gg?81vXBe$s|NXpUnS!u+^$~ z!B>13p{C)(BCzubzbyWvpfZ6!?o;s=($%;nuV6ngDctn`+C{tT_ zq8?mlUFGFdA@>{IR2Q#)wI|ZCQVN%R?FlYpORKyy=n0N`FSH^UGoi*v2=BS@GB)07 zb-(CMAzc%nDyWKQoZt||&m|k-`VS`&gC??|H>K&eY#N1JQ(^%9t-)Bw-SK>S84zq| zPU^UW01eE_YJ*s6`nRyM@?oC%F}#TqwYFhTn1y`o=%%&NTAWk9Ujc%AX5qJqQGXS%iV{A<#T!t1mPD4 zGKEtN3B!JV03o4{wKe^dJzu+*P!y@0CVFIZIdGn4&no+P{YD&9zV)`S#pQN_#8q>P zWjTAZv3T)&j>PwPY}my3gMeaY>OZ|bMMEVQixthRieduYcu!W_=sdhp+Wt+i-VK@cUhbKxeE_Id&s=@3 z-amtH5nUi#P~o`xFm_>tqMxRlrFW$e`gC}2-#Dg7Tx~5ifySOGmgfj{sr4QA{-Mg% z+y@`)mj*%QZ&8$$hqWo1SPvt{niij(eOQ^Q;=&g}-BSeL0hGcnht)|jb-sWX6{fG? zS?&a_CMWiA33`tC)CN*&vcj8V!J-L+>v~dT3UBc_A30)q+m|RRqeG-oV z*0Ux^(b=;M7Ql;W{>Vum4TVH)mdm`opE0a$enx4&{4c+c^qgn4ddka}OT&=lC z&TqNDWQ~0@HH%~^#laS4r`h7mJ5v0q8&bH?G{=T-6$;k3mUvLT@K2Hcwa|) zW%GwKvv2|%_&uVhb_Q^|cyQ}4 z8r~uyqF`R98~f-w=w2hwn<}9@1HWW|(_7>EtAx}cT!IpY^OD%B@v_LDQ-?b*%}vF( z7#|TyAc!wW_*V)V3_!Viu(dSgo&;3UxDIH)Bb1@Inzh(O8*i63D1C{7idGhI)yG*` zNK%38r<5bQ`=M0Fj7y0fL zvJp$zcX@qxFd2$z9Wd6L_8-M|;~q&c%Q2uVyZ^Ky8Y3wPx#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0G-1|aoDCIA2*bxA})RCwC$ zop)dq)z-l8ow7ZxA$>z2y=^wxJ~Q|GV|;5!BiWK1jgF7Ph1$ zRi^w|c=}B7#X?;X0006Y=exJ#$$1!0&*+r~Rk6Mp0Kg#h*PuggJXcN&+m`HmbY$6s zl|o!F(4*hfkblLvMF)!m>pqrT`QU*nttLN{?l!2}oY+nnhEWNl^wSEwy&$mDUh&?cg$H1qHN-yLOQ7g^Mj&Wn77-e!LaT3R>$@+1VnvDiKJwx}FJ z>&xAWq`7xLq8RFv$OX^)41J0vYXU`u`r(ADQ(q4pH#dAb*DthX;|UM|V9%pN@77+RTmH9`1f4aNM#kpIX>>)>T9aA@1qk?T^C0IE1$A^Bvx* z*!ime%m0q}z`FcQGRWf@MRl%*(r^f^-+#DYnHrz={SeQA^E=M85kCMxH@B`3g1gG3 zS{^8e&{ClhJ^Rjo{6&6WcU^2degJ?_NeB+{Ejc@CWx*k|R1Q2oT4_|R?f$Jz`LP!9 zi|GGLR3`7r-Ou$4tt{n+3U{7c%jC5%@w1DiS5L(5lMoX2i{Z4;3esZy5JF3${!1fz z@1r9ojpww`{Mw`tU8*njF*kT-cD@=~BPPV0EwqAV`aEU%J2z&ve7-gLaR@DvQ$^Jrjgxd)2;0RZ{BLJpz%@>rASFZ64z+$Dqr0Hmfg976M@imH~kRCll* zM+P7O00f*Zv_SN^8E$vA}OivZ^v7(;9IC34ic004iXFNe^4apSp` z>C0Oys*qHfLO?u79KazoU;IUWS?Y(amHUN~%Q3E9*uOc1W=bb%c%C-DRa((vT}eiD z_Oo7tID{5SH`gu@z$>NKTR1ucA>5g}pMdyizZW=!7D<$>qW~9ND81T}(HVr0kumOm zba154b3Ba4UgVa=m@n<Zc`Z+j_9RmI|&;OP~=oBW#)}Uk;%~irDg*Zq3;_wf#i* z8tb08AzyP^GYH|ulB+bL<2(nnaPFB^*`Y4&YpOnH3(euTDsA2T#E*{`o^1IS2z6-> zo}A>*HKfgxVtwh%+Y275AH5m$>YVWROxj>;i?}Oyf2gE=Pxo$hZ((+=5cOYg_?Crv z?an>$(#1EGhRTh-zGL~ZcYl4jM@_43XOez>xMx**LQ>`Zdhgn?7OqJD{@0wnY|Or< z$a(Ik2OqI+UP_~C-rYqhm8pX~2Y%J%6AWOi+TBy$UvmGOVtt9C`r#*03vAJ7CuIJ- zS+Qfd&#<>c-f}dALR{#^uV+I687Z@FCGLH6XjOUw4(fW4?Rs6J0wexc;D zuh3^;yM9OWkH73cDoWbXTFoh&%YW}JIa7T8m4Fu)cAV2tTg}2w<_kWLYNes_MZXaXJI-c9 zBnYAPJy-o%c*@&m zT+QRpl9xSDrFD`;Z0z}cpxEE!zd#7(Jhur!;}JjrnYpZ&g?FL!N_^V)h1#O$y@xL8 z{1H1Y-?(rHpQu68&l#apT?k2-O9z&dno^Iv@T#xS=c_KC4fTp;FV=jX{Tl*kM8=8N zrjPb{enQarW@UY%@YK)Q+j6y!gGE7GdM8B6BAVDy;Sl~U3}C)b|H)72J3e3x0HC{D z*P5Kfgm`1G@A}H0zU2ssk#e5f>h$%$q?JSE75c;*_%K<0Z<{yX9Wc8yY*p<>DXZ?2Wy^cVSh@Y^-ljsgI>$fADixw=?aahw;UPs9pnXp0u(S9(erQPgz^klo|nq!M~$@0*o&t1ZF#v001E& zY|6uWTEDJae3Yyc3;tNa$rWioR2i#Z2^cd!Vg?RzVDYXoHAPCg)cdyfhmigB{wzGb zDt%owrJ)fWB=Uc^{iGpYu`M}#3IJdb!-jIQsuiCmf0bFSh;fTv9Q9vcp|>5m2t>=n zORvsalKge9<}m^wP~<;1{QW-ip3U;XtQCNv5hK9)pC*6xN5Nn1`0|=v_*Pl$($*2O z{JuQUGjTi4a1DVIYLv+@e>;d}Z=4>@t>Q>qehdz71JhH!8OA zFfv3Oyej5PIp4h=zqH`g)|?$#)j0y3Z*py#=)cs;t|^5=0C@1_TYD#naG~94UW;lW zgz&TEWv7eJV*pMG`Pbx-33cmt_oIV9XK&L|S_FWUlsxM-gv5x(AwU4`S3bzrJgl1? z50WE%pQ8}n_>0kqni)Dx)HL7K*&EX9<+qdu)!zPFI!eP#dc&&Ak-X#IXZ)yV3=~5J zOM<3^yfxG-u8GO6%lx@6LN*vSC;a^;dNd9ownA8QQpxWF_h1k+DIa_5fhzse z)sH(!f)jeIs2OnCoqJ$a`Z^LLL&QOAx_!+a#lXR9iwHYP!#4L`+gaABCZY7I%oS-r zK#2V-==EtGCULXK+af}Qkcz6hUUutQ@4;PMqiWp^76%RYdG-bGVeDBA)=)kO0jxU- z5poDEF@bMipYceO=O^@Ghe_F1^ylntgFRzIB!Nphe`Lk%0)&1&++yl_pb}i-R0ypZ zo!OOpU~T3`InV9TGuurU7bkyxruaf{_a5KIEVV_a#f-b=J?Ru$xISwW#Tb`%{>W4> zCkjuUEINY$Y#fGR9NG-dOmBSd)nUxlj`1DA65XKqg@nWa&))8>x6vGjuoc3qB8)1C|RcfksnvT z_R)|gS3aN^s=HfPmgw<XFijRvIgL7|(^2008b(+=CG7Bn>xvyVa(H zw>&fmZLqO|{b^yDz5)VF_P?p-_f@GRBzm~VSYKlhf)MI-y6o)i!otGX*jR%2`{i%n zzMYbi(p;r?cXuB?e7FO6Tb3g96tsIgWP)iw6h-wz1~6fg4%V_UrZN58+}!Qkw`XQ% zvR6ow95`?w`^>lBek*N$G{U2q#bT~gfu@H}G5yTZ85tsx2ynABY=Q7{>9zN7&RdhQo~>LhtwjjcIv2`3cnG0_f`Ww$7n-MJT)(iX zqM@{WjIT1P*q?gFfB-PE+uUN4H*k)9Hz<>c>US5HLfwuTQ>xM+l{* zrKP2%ktEqrQjPcL5JD71;W%E`XJHsdEEd1;!V9JaG|nEKfdI~rm}&Ys&VRJ!sskZB zc<>;PSAT1)(P;Yi?Hd#n)JO_^K7YlE6=%<$J#gRvkM~3r4MKSI=uxp)Jbd_Y$MBD= z;8+)Ei*nVEO+VABGA*nZwOW1e-o09S#wu21WaQ?}n`g|J;qUL?h$13{TwGj6j~<UAuObm6bWhTvjV|D&D#_ZQSLF?9Z)=oo`*8_FMi5OKRuh#fuV& zr0&AOK7IPknKQ@apg#3OK?w2r{N>A+*G)|z5UgFh)-k#$R?tFMQiek;Ut5R}av>!` zT#%wlwWM}3GBWD^yFehAHf>t7nm*G)oVj!78VrV-Ull?qCnu+(qQU_b(FzEI#eq1) zAF6X9gdL^f2F93Cm1RkS#l^*SRdCp_VW9a@X?A<8fPjGD;NV(YrePR%_wHTJ76Jf3 zh&YgdL}qn103cEt@kZd-DeWg(lAubZs&z%H)%NMr$80YNAru=MOVdxR`^9lQBO`mD=Xs=0sw%&$d`abo;Kge7)gvQ&=&ps&Z1CB``!IFS?euL%2e+yA0HoIU*E=I zYc!g=-F(g?{YRo*qj(rEQJD+?@DX~636VmbQ(;h=ms;I1JRXmyP$(MG&blOOUSZqV z%*@QXu>!7N_>aWMIunq%U6IKC9O@M(#D%}*|6yh?#kKan_d=oY%9Sfk+)$=or}W&p zb9J?_ZbnXu5FymvJ%&OQix467qVI4cG9Jo1W?q6mK0b9(oj!eFI@q zg?0b2R`ct;2qA<$-C~T4@s2Wy{n^c}ix?N@Y4fwH6=vv7hYlU;M%Qp0|Kf`;nq|FA zlQt3(66!|GX_{`|zCDNVABvr%5je!|Deto@r^f}n$jA7mx1lWsnzOQ=_7>l z^YfQ4U(R+Rny8Kt!ez^rX*8O;57RUq85zkT{73rAdyqeewXM?31O@fJPsr^;B(E<-fCJ!BcArKMfFb}c0( zrL3&1)_5;FqXNhA(9qDBn3%Y@xLT7CIfPBE<+`=DNwNKhjGtC_TNdll-@^23CSF-t zsn_e(YBfSgAQ13)Jef>Jl4QLPJET~7KT-(cCdIEu^8VPJd89u{hILMj0`aB0XY0FwDAtew%Ev>7o7JQsPA?Oth zVrohQ9A(~v10tjtBLaXu{I6cguFiSp^z)x3FXNW%wo!yx)j15Jy&7Jt?J4jO(QTVwA(PuYtRtSI1FI#-5`|- zUy=3$XB1l*A*+9HUY%B9P?8wAtKZMAHAa0K>a-z**7sORG1R_ChtHQ>;z+a_LIe>2p9e}=&(&+3n4@U zT2cKl%%y{^HQtG|5JLE6$_fl%^CM>2{L6xnfCS6$=j?5tBz?&RZmfWCv*K3`tqB$f z4)+<>lCSt@;py`wm#5yC%~3QTAwoz?>o#U@)6x3Hoj+>HdPE4V@3~5V^E0Zl-nl-L zqi7~V2;r*qbtEK*dBufFLR!)$K?q$**`EGegoH3#o%80^ciH8=9Jyw=GYFwlec7lB zuR#d^h}|!H+HkIobWuZVUcWM>imGUh#j4UBuf8 z9~y)rT(qzM)^>b(g+ck!rMJoq6@XL4rXWNJ6=(}j6`eOS##!OhTCoL#5b`kI-hNwx zMS)sM`^x2ssa5G5zo#RF5U$DiQHTrP2zsT}V_j_STl%c)E$=}w^xM~G9L@g&IQ>{Z zYj5`F2Y3$GIxerC{bJdA7$=7;U$ug~1HC1)oF;j>NT&^= zHf9UelxB-!$Hd^*C4{)`*N%k?`5mnxWjCZ06QUVmQ|*G^uH5~xXNF%byWxZu5o^fU zY*PHXGj|`bhZWBVAs50$LP$1b|KfygDOPWd6zYl-GB<6^{>46w0fZxcpF;@Utw>T+ zYR;Vj5W@YBj)-vKaPMdBs3fbpExT5B)ABQD9mFG5Q|cfytilGRMu&=`CXx%2o_V+qW#sKC#6tQ~wfkyW`XtT2W(ZSuh2bQOP=Tur) z&lx!5E=_#<`V4!lh3C8m>uBAnqH|8|kt(Vm=4lJc^cD747$2dxkI-99snabxK+(Pt zUMRi7$9Sm_nf3_@dUzTWeSa$&az0m{(if-Mp1}LtW0u7cP6bW zlS0NwX@p&Wzg=aW4UF-DD%}|oKGNhdi18EI_rZ*OoG&9?G?a$j_0)+F7HW$jguX%_ z2Pjp0NeF=vW}l2VQ-fcf=jNb?+a%Uif_ms(w@$ud) zzpFGO_qhws<&gpaVm$_o@O^H){}==iyZX!C-BBm;OKW?i zj!~+2uFoJb^2eU5oz520h<1Q;lvKv5~=#CgwXkt z%Y2;Q%e}kPAw&pul!oCDKT~|ser+Ly@J#V}08pIA0H>Qf10fvb@r<6)Unsp|$2~&E zc((YGfib+`Gt?R9&JY;USKd>E3->=hVyETVnzK_vhzEPdI>`*g+6WOsZv>8`5&dh< zZhN$XEJ7WlpAzzx^Q_p$Y8jIL?fS?_S&6>vT=7Nwss#w)+Kdetz+zmZ!(2Kz@8UE7 zzz3mI0Dvpg*V$1mAcXR@gyp*lPh}NsN5&=D$(f-h~$exH>)IQt34s&_DHD?Sy+kVGT7+^CCV776)o*&7^D7 z+wzXshw_f?d31SGUkg& zX-D9c@|68ZD-6n4ue`0MH1RQ?4`|n??Yy_rP^nNq?B>=5IK9B9o?c#ItUOV8>W#p0 zZRe#A=Ns%`M#Rd#HZIAQU1AtTI+Fd$8(7zu|r5 zJ-LINKywejyH<8{QPSsXN_}AVqM8*#g4=IzvGZVh=C9e^^6B4-LFg);sORSdPX1P+NHO9_wMdpV_c)z zW)J}c(BFFuCs(F)a*4noW?+mjoFAv7bv%qmBU;ZGg2jQc9{q=T4eBHf2i(b%wnkV> zDvhc?3jR7002ov JPDHLkV1h)u0aE|~ From 9a5356664ec232492c88ac0de0ee3ac5dc9675f7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 22:38:05 -0400 Subject: [PATCH 0502/8313] update --- doc/logo_small.png | Bin 7828 -> 7621 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo_small.png b/doc/logo_small.png index 0c79cf46ca845eda07ebc4c4e9f3bbb0ff4b3216..4ed97ac00dedf917859dd9082dff26a2209345bb 100644 GIT binary patch literal 7621 zcmV;$9XjHPP)Px#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0J%0be7u7XSbqS4l)cRCwC$ zU3p*=RsR3JnaQ1|O`6`NcR5-p2o$*wQM}l75qCiraak3w#pMtXp#maOL=*+l<5dZ%gVnKb9jyx$)Kv9?WVo1|^h`Tmt;CU0imXFlKSecyWs zV+_1>y42hB3V{#C7$Jl){=V!#!Q#N-{>ccT_BMrh2s=|a%r@TmbWh0-(<7$-E&c`P z+obqz?M(2_R%d>AV@t8IR6nKESOWqhOk8xq+1mC}IYD^j-rlsW+J4$cnCc;7Q9$<1!MeZ@Pj;ro+^j58352%5#RY9J`bU5paF`0JcOPqT9Xa{#reeY5PGWU89f3}36(sA zo+>tm1_TsA@(_BeNGS=%U}8)>gq|uYp%MU>7|VDFJypUbAsFC>{w5Ehr^*1|L>9A| zHCa4_9;--2N?(CbiLunmT6qXPRfhQwMSuznMSPCbQ{};+@c@urm&4~sJyI~n$*TLp zq#@zbu+G0aU;gBJI6Ik$JPTa|+pnD96{#nJXJ*VxF0U*w6n4oy@TI`c>dbt7!7CRQ zHqfSiKD}e)Q9Oj6gC77wXj9(SZ%g-?8S|earp$_(-X(Y9G44!T+yo350qUGPDK(a@ z%wBi7Hd`t5O-WolT0Ok+ISgh9Aozm(rgGa{Rb{PqbW}hHV`PzTMUVrJ!OTyZL#wja z$w+BGMeomse%SbE18w{ye_Mf}=(C|8xOG~gM=H`ol^L7!wmITotUPAt{hzrTzj~YD zwTp`j425Dsw5tEoaRH<3hY&(L3%~w%@wa+fZ)53$+Kfp-6L?$r8zB)XB!!OpSAg96 zEC`_=%m2GR=VKbvgZu`(mAE)W63po^gwVyBtPQzaZs_ynlWW;Z|<)-wO%n^)?tk%%0s9JF!p zS|K6y6)1L>e77WP#pWUF-KP?0mj3IxxjcleG7k^{Y{}p8ec68+=!X0Khc52>8b>~k zTbuH??khW>r}dH2UT-EW9O&1d>)%)7p1)9&RnSo6K9z8=B4bnDKWnyX?A_}!cj^7S{LiJCqka7=rt$(>wiEdILqyF=Ox z18q=Hiut|gObMUN9pxeXwVPOAuh-?})aO3c>k01nMBl!4gt&8rq;P4!`3iy0n1GQ@ z3Fla>jJ2?#WcihMqGgdQ5|@NaLnWl-p6LjCtnzK?zBSi2$SA1~C4VB~vDbRP*!U>L zgU7*;U6->qXJd)6w1F{>2^bM14zzdExvET!x$^g6k8-~}9{O;*n1m2ok+`I=q4ryTBK@nzRpthAI-t>0V5-&5!^3flF)vNK67H99jqKE zr({j8-Zj~o)|HPV`L1;T$NB$o)L@)EW_$9-?Y$EVV?A~I?C#R<&sAR_5E`Z$Iwxk< z0KY`;Xqi#7wcwuzD-O#j+0Wy@cmBc8RAuff|L=vG%R0+#3>affARcgQA>h95Ec&|4Sl;$_3LA=Z4f&2P;X}#C z299J{rj4x;h{({MsHf{mM3RW~75J8;D9FmeD%dLlLim~=J*N#C_);Z4!JH{B(m~Ka|2PgoH0Y4u4@RYF0y&Fwx*+P|*0L{pO2EQODGN z>$lqYtB?=?2s+#v1AG&pD++{K z4A)r9vKWU-LO84ZRneX*YfXqaXkPDE#s`jq-&Vx*EC>r5iWg(LxG_XAwoz9 z=EOWZE#k@MA)vj3D$Tb$VMM!?d%lxxAp{!+tw~nh*R7O9`(8(vX#DR=_piIYQA~<4 z;NJ4MZOI=wI@8A5a_VoqoxQHiq!E!K=iRv`K2+Oj2pDXvZAbFvzKVE{PB6KEg$pk& zJ6nB0Op0x+ZOwp`#We}znetoLyoB#|o`)2sI|{lfOKppyWS|-9sWm zLW-McPBUZv{8rkITWNAqMgR(t1iupZ+}MDT7-M@t2E23$gc##$Yt78lbA(91U>3C$ zT@*GHy?lP5kuk|hnT55ymAH6P&;zd8=#3X=Aw&pm$lc5UV`j|@`n=kBxS2y}SMfKW z<$pmS!pN8=28~GCu&FHR{^l}IHdgs;WF)VP+CL+Gum?;h43ClMhe1aI_T z_DIM?Z!$UFqe$n)zcf2)hQCPFL@{2d$vRY#AtFVAqJS+!HmIAOz)M#`h%q+M4Oi=O zhN=cpi1HVynsX~b(RW6S(y9s;s)J&qPj}6Z@CO#*e`) z#w>|Q5g|-XTItq+fx(Q0u{cP!U|UzL`QkAU{!sQ)W=$4FkkQR%vCcUCd{INm>i)|f z3Lfun%lBm1?;Z_)sFl#3hp=T8+Q1k$-uP5V2!h0c;{!%{W0`S>x8W7am%674J*+*(V%Ef%mM6S!FU~;-oxXiGPhS9l#;QjQ zRt{{}3|oW{1~6SbPuJCTS!xDj+3=A_BK3Y2et2UGLTHqFc#>aYV>=fn3H22!5*2;d zB&BrB_t1WjYPy)&HAKLGF&ls2xUg4*mqVCVwA;qo5JJ-;p0uk_4#g;0uOCN$<&6JA zNF;<@-Zg;-#w=!m@CGNAXq|!?mB&d$3JKwG|DpDBE}i+d)?5i4Ez0Jl^B}CV>GKRX zX-uz5Txth3UtjRp@jpF$bZV8Qng_4DL-@t5G>oxAph#8?wilaxe(OsaB^8lEu2_hd zZVlnS@&gn>g-b$&gn%0hEHh~iS02R}&+GlN@b0`wUb>?@gE1~LmN1yHVfw+K33i3K zF6Sd5A*2ZE;ot}P6lk}G5FvESPy_&0*7``u101M&+K_L!i2*11CAK$~=cNmRipzD` z2*6*c@)h{rsT8fxGtdoY#*)%+k!u`#i-k3ry6n+03*KEjlT)9I0Y<8a*~5mfiuWQw z{S>{qQC#P>sCv%d=$fjogAhoizjs2&!IM*;3xE<7eYhWOtnFmg88IOmq8#LWEXesm z@=;^E_Tjz586#RWgeJyppc@cE5z=sO@b5xR7Kun2(?gVloOeXqKIx?KwYbkNbc2ys z)FHO4NQ;dn+}Mqjlyb@IoccTpk!HqnpEAj{iBD&mEjsW=z6z!_D?HED14<}BXIC=7< zPNy@Q%_K>tq@<|TY7XOqf`YYc*U~iIVE{13etv%I)~yo?g`SQg<%kC-A2`tX*)(P? z03_XA3c?-av**yypFh8A*Djq-Cz0G)Mv!3`nx=o%^ClBP5YF^I#@J%9Fbw0JEWGlVNppEt)4YUCOqTB$X z-T0N3md=|uud=d|AP5hdWZO+W(mf?VF1)n-vEzTT|7d0`7_d9`I_03!YPHLkFSlB) z1i|mS;538)P-HBYk`e-t@h%a$C0)d%*yR+&7@JI{HEY(0#bVAJ@~F4~LLW-bg@d-c zi}Bz)rV+V=h3nU^He@-W5+l)wAgIpHA34$XpFJdY840sj&AacFTO}kO_fL_&}fNDFI1!m z&EOk)l#lQ|HW? z6B!w4u~=FJ9%KCW+izPe7LQojspBlXTkOjcK-ddJS{w@7lGCqNwK2rPu4XY}qn@{(SF%(4hyvDtp~i zC!f99PzV?>#=qP?`}FDOzbe`5l4s@`(P%V|OduE9r>3U19ls_xO`JG!!GZGOG~{Y^mGW}G2JiK)|v~|m$)eVRMlCnx$?c7k6h@;l$4aT7)4-=8HQQ3 zXpyh4Z^v>m+{uZFiA$F*Wm&e#e=)`+Nq+d@hu$1i*b^}Y4TW4t5iN^kFe4&Gm1do* zuAQ5k+akAEmQ7Ah4i66}h>oxP#YLP+Nl7Y|s>xfzHCR_yr_pGk=o zP1Eegj-}ZJn-VLaWtkxm?}~e(l;JVC>kjBuO^^3qcU2rKMh= zNFCPBL`r)BU<+%lvDPVtz5wvq&=0vK4qbI^lY`nPe}Dhhak3BJH$JDuG5tH!Kcb?z z(4~nnE-o(iW?L962?78CqR!V`v_G~TKj%#d#@J{yItuSv|6@oTUg)#bYEwK*HziXX!;_Y6o{R0EC< zc?a9t4Iuzb3>wQ|rrKIdV+H`kq&QHdrVw>Pcgn`voWFK+{l-a6O-(Dhx|PIOlS5ow z0RY}%^Ev|Iq~He_%-*(CF)YpX#k5{iEUcBrbg{A2d2{Fp!itItmxOB^mRGM{b)+z5 zGMP6+h%t_lMad{Bi}9(db6j^k95PW(Nm?=ReNV|>$fG>-7!`)3bJ1j9VF)f-dk|clr`DbUi zX#Znzaj{mbZDF4ygfPZoVPRgbNCBXiGzKX#J8P-48yElySBx=jQyOLm>BP8;TZJ@8*3vG zab2IssZE5?OL5Q9EPbv1x`{D4lf$Ga>WL?wXwlc;_*YX?vwZpTBS((dY_`^i@0+|# z8jWV|+_}ZY#jW6{P$;~Eh2+YWD_eP;MWiUb@~DAs5RpP|6H@?)l}CSH_LG&h7B!S` zrm@53Dn33wJw3feE~2|~>C&a+$B%2ZT7f{IR;ydAQsmCm>-A^OoJmVd`|i8%0Km~e z2SUhZv(2ABpUdQVZFa`4z8(x5pH{T{LQR(acM%~R;G1x&>fABiFFMO@wa~uLGd3kwdWu?_>Wf(>v z5ZFhO9kDPdC@3Z-#+%EZ8b9rXz%jdu_XLR4jk|ST@0SjrI3}Z{|J59t74@_;qk7Y) zPd{|%Q0wK}9FSb@)MPT{<>j@kQb@A(N;vL5tyb&InKLPh@{XF-HkIC%nKUY)pO_Fg zuJ3#6>bplPPXa*3gnbMTW@s^Ay}53Fk!-s88f_Qxra7&eW*BCLQ3p0 z9QR>X799flx-}2{U7Mrt2}rlO|1i;e{8RnKIaxNHI7#I3*>8Q`%lr`QKIr z_-f{&mD#Cfrr!*lt{=Dx01UL@ll<*Yc0EF9(3VcaKp1gg!@gcRyf3$iy0mPn?AFtG2bCMzeFlyAORjXEs#bW#9O()51|B#T7 zuxQaDFPPKY5eT_w?(GwY0R2aEut^^+=<}+cHrTMu$$|_!iV+bJsi~w6_)jPlrlh1SSg@d#fktl*D(o`Mf5^$(XZ!`KCjG7!`5zd&r_@-U=-bzMbH#m- zNFSfTCEf) zTj>R^Vjo2$TVwK|=yy1LqCv(?npa5Dv!N~J_1 z2?z*KDwQggDj*;LdXlsQLKfJ3eL=W1OiH$}kimP`KmJK`&`28(jNPjg`nh%jTjP+l zjr8p~3pKXY2SEsZQ}X?s^Yhl`e9&UoJQ4P&g|SFT$vZjkySDG_Hte3xNMq!B&T0$o za3gh=Dh#-=q2*GdK0fiYqyJ1}`burK8(0My#*9VxNrV`dc_ z%RGI9YHx;6B~oI*j&@B#2q8f43|Q87Cy1BMR-__A2mm&G?+VgtOU?T0o3d)I^7wTx z2&*jB2#}PN-1E#Gg*$&N|7q!!Rb~g2<)v#NJX@U!0NmzvvKN3I~=!O^0zXsm2 z_qb<6$YHRzbRU6;SXuNvPY#oWObMUNV5X>{^vj}e_%)rn7(&h^*^s;0SD>)5^w@xr zZE@@8^?tdJA|3&JeB&Q^`kTB_>@o;B_PlfL{k^68FyMHfmy)fYa$mBypkc@Sd22%)sX-G?eN z0OS6?{l2_^n}86sD^%wId!yg{IC&HT^m#$r*{V$5D7su)UbzP$bfzkEQC1296ebP* z^8T&R(exv?Q_!?ivyHTwBFL|XZx0j)@cr&y2mzqhS~un7Oct{wB7YdY%SRx0@-jK~ zxwFr{Dk4PwBGtE}cJhfgw=2@UW&2Ew$%bto4_@aZkUKw5v!9~RyxyxsV{x(QC0GuAd;r`y=w_mZbw6LML#8_5O>qn@Qc`mwMkv5(H z*81AI9pfSZ%sTT@v9XlF?DB+#kA(1>Ai5etD{FoK`leWU%%39ufZVfLIgRNlCuZtt zL%mIZaNM_Qkw1@tD{bN23yTj_r0=-3v)-zA$Hg$l6rw&E^1g+!V!$=lS{?xB<<}wK z(^cnl>+>1Rj8xz6D^$3J@(2Mygf#qz(O=ft^zlA1J*`y9y)4{Q@;wV|usCRC;u3ee zi-F>Rc%K*mFfyhyRhjm!6nJsU7M7VbpWjORYxJMPB%z%~i>FS`(wMdJ^4J~2Ho11~ z_8tUbhvzN0L^kDZ{i=8miOA9F;cJst@OX4Yks<_XMZ2@ySvlB>cQZGB&cwj+1Q16m zPiEC#;qmAQ3wIZP|Iv+4F~G@*2f74B7TE0t|7T*%bK_qm5x?hF`#sXn@^?@KHAFeE zYu+G)2u!>fJKG-nYfRb@Ne~ZAI}|AcEQ6U>;-2p^J6}(2hcel|j&23@d9Rvdg z3|7`U_rj9w`s)Z_LeQ8M2}?*s@=?D-2)p4s&S~|R&o8)EpNkOcujsdF$a}8Zw2gdU3&xrj*y1$NZu?aMLb|F*url5W+6l%VnyQO34vo+%$6F!tZc20|U(!yGejOkik4v(M5u&@aob!Oc`ZN?vaO%Wi9N73_A$$WtC zk&3jrQpFfon6#hdZ!iDtDz`ifJxSJF+ioBQ z0!2@DQ$4;u zg^w{jzHSpCwDsmc=U!OMV5Vm}HQrEqyS-p%R_)bv?XjLoM{j^|sOmlh=-9dmPSfiy1QfP5|D1B8|m(j1!?K-rKRD&zwf&< z?|tW<8*}IM^PI3R3X-p|Nw5I`z$v5gMAhx@amIHqRW-s+ zc-u+aeu}iBl++uR+AkU)P#%5v->RwrU`0SUx)TBJPgZQI3Sc9!teJ{lSBwFh{?7-H z$6KkBpxP8U?`=d#;U*}KUD`Bm^mH84$9QLqY4>@&X*Ycl1V;OCX51Br&97@FjczRi z1OvhMSthgRBo+OgvlGec%GkasS<#cTyvoq|;ND^FQcZ6U`Tq4kzoZFv&*jkZy8%Ej znlKOr6(-@t)7HfcW1n;PJC<_+ja1Q1VU1W$vXpv} zsYkUo9HeYeI?BEMPAid z*h%Aw{k|Le($<~%eEt6=vUrXB&T0-bLH|msG4i(_t`K5V>USar-+zFuFerc8 z^9L3(#nZqt8FPm9xyy%(hTa0%kYCw9%nAjVOm9hXD=Q}8f_ngK;eu)T}i&;CxmX^YP1)BuEvt+ye6n2qxqGb+OW%E)k z2a`TFlL4nl_{%dm|I*V*^Z-yy*q6DL2J8gF`~G`+mDCUSpNSKKmj0a2<#D>~;(}f* zh%vBZuR~pg;mAS^SUz&Y-$rZs6^_n=m*Ejdy{vTlD5){`!$Wnki_T`mFA5R-9nb1z zB{QG0M&*~R97yXUC(fHLMpA z*xG8L{ChKv3`X4&VbkA^xgO=VZ;iGKmt2i@8QECnZ`t9hlwd2VVQT`;xbNR*w%C)2 z-+xAIzn(fWkgcd*>#KP#sZwL4@B4Lp@~p9&be!l9GW1*EJ$1A*ZyWPrr2eku?mU*+ z?JXl?|E)#oZFjJ-#JM&x_t&sy#6@!(aW3#J3w&7nyKb^?#X)XL0}d9+jYnZpAc4P2 zOYI^eOZ|u>{9n9RYWvxJ?xR}0g;2k%_{K&>^Z46t-f81!I8rI8k%j!NwTXR<_&r^;V zukwAkHoB7dC{79jb;l1X+lhr!<5DO-#=fjW&3Y5 zc+8%59ND%H-%5yx{SU<^+LM`FPaWjqOpxpE|?hF*_$F$W`c@K zBsXAb*wJjH%cAAn&QtzXfAMQo!@SN0{{Yuxo#RZ>foPGDlTnYvnZg@ks9(%ait<{u zujr(bpD-XN*XgT^Y_WHo>x*$U-nu*QHn5Veb>@Y#6*;Fq+4oud1Is$~{uCZD-F+^m zY8q6I#0?hmVgSR$in?14o1Si5di>EVylKUs?wo8;k#Gm6iSMUkKU(W|@o<;tsP@ z7wV|{b_1E{gDyV$qiDMwBK=VzEXB+>W~7`zRz;BC<1sHz`SSGmP6RG47+GL$)3H7~ zd8SB{|Ap6kmjyQQG^%~!;+@i#A70+?hXurzY06$a@_F*kwhk4B(W~X@7LF)Cdsp3r zrtmaq1bhEzq?aJy_8MIX-ynnmbMUqc=kja^*^N1p*U{QTYW4-JqN-I1M(tFjkyO{!*05rD_2X7C+C8@#QrG}4GcEo0nUAk0IS7R_c@pwBXn9rU z@)$AyYxb@Q4TKg70K$BaZnmgou>zAPbEtP*J*Obxi_q3V6JXzfURt~T_b!TRBXxzL z73pMd%Bn1*7%fuVYO~zj1eJJGJJ{i7+Koh#M(#{3g|SzK7`>olah+Ayw#jZOt8!>A zuD9>a9x-`V$7l3frlMeO)Y{YnAb2BVu=*i`wFZIJ)*}BotiLUK%qm4p{6Xhoje86l3b5fpR`dJ72`Fpr&OlK{nT!^iU@emrv5F~AQ(`6R))hVOtSvS-tVlAj|E0U z>Rw{`0W&P3*`mcQcf0BBdVTA13%1V>gZ_|NujCsW%q&ewI79a)(6{2eyL)7W(MpXg zmq_k@$m{<0+K1!7{_j2&3+kolnZJCK_8}jo3_&%AeQ{93r)LS+V4&I2ncBvVpkK!l zoJ!N;)8@K?C)ohi{4I^z2e!tC_DpU7Ktf7kXxHUdqr-1SUdpuY3Mc(`QyPRvd2I=O z@@S!M<#QXX?C~k8fhp;;YB^gkvzc@ZRWC8l^O_m4Vy?&2O zk_yv}p^#--4xdaA$z{%o0pQ*?=K5z2RzC4%UC&)5lL7YRwv-`hXSs)HqgB7lj(tYY)JRP~m6LcdlDFm!M3_a@o;XzT42f^DH84hqa* z7V>UCcnd9x`L;oh@<>Y1N(lq{abIC;7?}~!yakAb{+6VQV`?P>1IncC(ExWhG_QfJ z{_#Sal?7_YY6eW6ZHu+@zwFk@937NP1)7Cg1?G@+$wrI-@rb~1Bdp#YS6@tcrz8+?-zph2H%;5bxO&kG`QgP zaAMvzDAmm|6&y(Spp8|zkh7C=9`bzNSw-#3Q-y)0r!9G7^3T-&)Iv*O3r-9M<~*Bv zmlvqwcQ;srPNabiyC*r2Wj$ir0f$M0L~SM?W?mrD`~u}pn#H1VAK#6Gph%cd*69Ev zCFGw+uqPFi8MBKT-T9wFl(A~9CG&tLv&Jv{)T)*jZ5n|ygDJ|F=70%g%OpiXxnChq zEe3d3T1g43tyRqC7v^eoGAi&dtv*d`z|*PGpf#}WwE&l^g24fR6#*Ao@hk}-F}_!b zfb|WnD<&X(uD4DlxUo0lP6(1h>io@8;4@fw|DhExCv-Pl@D<=~ZN@XR2{1&~H{R&$ zx98rS-2g+fJo3-Y&hJWx8YCsg(YDi|d9L+WIQPb$@wxr zA_@bw>8;8gm#Q)b8?;guRgfB#@nVDJfLJQ^ctQ;9@2G@Hw2r6N+;rxMFrfgue@6>b zg^IhoyN6dE;ki5G8HXn)Y^dX7W4euYIWrd48~~`4nc3c-d|C6zMoCbQ z{Kn~V%OQBGE&vsD<-E%s^~eB(-8&+F^W9(t_(Akq-Rx~_D)#nBToRNGa#rki;7y=i z=7(>YCCDT%9Lf7vp5;9_udK|fKazN1Vd0;?WWe-g=w(a5`$C#ABF9U}eTeA_)Q`($ zCG3Lbo4ov3+1DC5T0C*3w+TB`XnvN#I5heAK-&$u4LY)QfbhzVnv#uq01T8i_V3@K z?T9glrz5?zBd!GzHwvnsm;)uN9-7~B6*m^tY^Fqwo12?ujUNk4HV_tNV`s@MtYN27H)4N1OW2a7FWoY)Q?svUH>GS9h;?t~E$1rKN)IzVq=4)0 zPqC5|!~!0*@$YD0W_5qpdqQG}AciL^ZHx8R@k7Qu%*$aQfU2sh*WG?0(rd1x^U&uc z5rNZ+6hco<6%5~SNaC5;1Rw7sJh88v?6^BOHa0NiL}=q#v}*U-?-6*VoKYM$(7|Mg zk)GRnP)SKi+Ap2(+_?Doh=_=`&s6AsZQ}P4EHs59@X)Y-a2iRDsD%?9k2$pm5cKx; z*7=Ti@=4G0xNc%%qF~0bH}L-IkjHUHt_AMIFC?zSq+WLF08=`U0t~X7M!?DPqqnfw zdgPkwmel4HWMxaIM6m;YBqtN$;Zb8e%GY>Kx?k>34Zz`*3Y`fF2{be`dOuXBBfjVi zE0Ku=f)297Owb8mB$$sntBDun<>j@%pjAkpAI<*eM1j*2Ny0sB^>GeLllhT_W$a~Y zj|Xml$uGcKoS@nR=VYa4IxX!N#Hw!6T8U*+V&d#=Q%6UKLZRc#zt_}~(PcFuDY^8M z@t)RVB<0aae#v&W2E%?VQ{L5!7St6K4`fMf*08|F#&#$;EA49>7xX^!D=I3|B4m5| zNJlr2wg|OP@FjsrC+r@@Sntj~(25;Zm$J24-h}x3`#ZbLV4$JV<2&Pgd5wcpHWja- zq2XCWP_GjFmJ+AHmrcmg^;G|?x`S=$*>Q-C>Fv9gUc$qg8ZDI~nHbt_JDHS;qvx_DJw zuk>NCAJKyLj51)*xo6*3C32&l6SXhc{>WS#O$C#>z?fPgGnrTGaKA9!<)bG_tn7HY zGgaL%w;jK0!_Fz{vQr}pI?W95_I00I0|QP?#!UfcJFmAO5Qy_fU82qRz7N%{t*t{3 zM?%c>^z;l2O-;RwdJW$9|MU=2D3nlwXaG6e^YYVB)tU%eXhH)y<858aTy#`ad5xMV zkO4oOMmF&i>#JqWcQQa+G=rfynt(6Y7I1!eNJxn1XDDm}S*10*!G5K|PLtNJfk!d8 zi7Myiw=KtK&?y`me*RX$$8dwK^v{kJ-#T8uCO;cb%cca5B(Zfg^&ivQ+uN_$1_i!2 z(MKl@DpKv?p0rjD^`LcmOLv-7s$$hUn@&R8!~RP$re3Su*(UHbBLyk>KAIexnW?ub1%QTXl}Cq$o}$90A8s#5-7*qt&xMD;c;D3xo4jqSky_?^ z+1`Lv4|S~G3^tNCtKfA-c&&B%^A?99+ucINEJ5c3RTC4M9`>iFr>lHCAUJ-=30YID zks0G}tu)!D>UFR$(4OpiBZ3VB6VubtxgXIkf%axOz2y?2<#?)jR>Y$6*OD${et5umss{{g?d3m);XC%bL^dlkt z$V%Yu?k+Aau3oA}R%Gx#QfoB|9RtG!v9!P!EVS&gWSaW)w??{Btfaf6d6V-#q^bWQ zBu1#XKy_-PfFJitDxQr^&-;99$f2oKFp9?o4WDaXRvcKZRgP4foSfWc^z=$`PD2X! z0yvzKAm2fCBJJHyR-;cUbUG(zm7Arhim5p}TmR|dHi|@d(imxQqCR1OyZ$*($1^v=}x5|U;$tuAu;59>gg?81vXBe$s|NXpUnS!u+^$~ z!B>13p{C)(BCzubzbyWvpfZ6!?o;s=($%;nuV6ngDctn`+C{tT_ zq8?mlUFGFdA@>{IR2Q#)wI|ZCQVN%R?FlYpORKyy=n0N`FSH^UGoi*v2=BS@GB)07 zb-(CMAzc%nDyWKQoZt||&m|k-`VS`&gC??|H>K&eY#N1JQ(^%9t-)Bw-SK>S84zq| zPU^UW01eE_YJ*s6`nRyM@?oC%F}#TqwYFhTn1y`o=%%&NTAWk9Ujc%AX5qJqQGXS%iV{A<#T!t1mPD4 zGKEtN3B!JV03o4{wKe^dJzu+*P!y@0CVFIZIdGn4&no+P{YD&9zV)`S#pQN_#8q>P zWjTAZv3T)&j>PwPY}my3gMeaY>OZ|bMMEVQixthRieduYcu!W_=sdhp+Wt+i-VK@cUhbKxeE_Id&s=@3 z-amtH5nUi#P~o`xFm_>tqMxRlrFW$e`gC}2-#Dg7Tx~5ifySOGmgfj{sr4QA{-Mg% z+y@`)mj*%QZ&8$$hqWo1SPvt{niij(eOQ^Q;=&g}-BSeL0hGcnht)|jb-sWX6{fG? zS?&a_CMWiA33`tC)CN*&vcj8V!J-L+>v~dT3UBc_A30)q+m|RRqeG-oV z*0Ux^(b=;M7Ql;W{>Vum4TVH)mdm`opE0a$enx4&{4c+c^qgn4ddka}OT&=lC z&TqNDWQ~0@HH%~^#laS4r`h7mJ5v0q8&bH?G{=T-6$;k3mUvLT@K2Hcwa|) zW%GwKvv2|%_&uVhb_Q^|cyQ}4 z8r~uyqF`R98~f-w=w2hwn<}9@1HWW|(_7>EtAx}cT!IpY^OD%B@v_LDQ-?b*%}vF( z7#|TyAc!wW_*V)V3_!Viu(dSgo&;3UxDIH)Bb1@Inzh(O8*i63D1C{7idGhI)yG*` zNK%38r<5bQ`=M0Fj7y0fL zvJp$zcX@qxFd2$z9Wd6L_8-M|;~q&c%Q2uVyZ^Ky8Y3w Date: Sat, 13 Nov 2010 22:41:11 -0400 Subject: [PATCH 0503/8313] update --- doc/logo.png | Bin 11980 -> 13171 bytes doc/logo_small.png | Bin 7621 -> 7321 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index ca9bbe4d3cbd1ba93937dc2103a853d2f3affcff..1f51d1890c406ada11721f17f29f8c8061df527a 100644 GIT binary patch literal 13171 zcmX9_1ymIM+g(x`1nDltrMnw}rMnyHF6od|O1it3?nb(iZjhAjZusW?e{;^lVU~&C zGf&=o=ckf_6gmnK3J3&3he%7PfIu)9z=sP77T81nc*F+$f^kui5(8C^lNDZCohX_F+3J^p>RLx`gB-6s@!`#B)+1PDd z?LkKvhnSdD3{_5mB$kAz@&{$5a5|F*`tuxEFgQG%4(Z1SV!ZG`q-iv1WSZ|Wl7~eO zag4+EjpG8$p=jc@$m!Kfw!G{3KBr>}NiFPmO?Ru#DqC_oJaDu4ww zhM@lfRlCee%11S76xmFsa9HOTEtPzH|JgjEnP)KSWViPUF>GO(GU~ zh^3mdCh5_^aK;QX38Sv)h&>{PhDEXFLv1u5Hq_+g{R7FuB_gOpy)j8o*=fcZ z@dl6W?ePj=uRp$V>fP5pS|-r>_;jUUi7H{fFuS!r+V%7MdaNma5g6#^rNHg-YbkBl zD1s6kam#dVt-Sgh8df{`?@$IS2~QN<#c6(XFee-Ui%h7@sdPXsSFDz_gD1^AZ!i1I z8au-Xls~8sY5Z$XyYW9#T{zcnuk=GMpNA6+`N?s@@Z>c95R#yPB_=t&NbkoLt9 ztEjYNQ`+ucRY5Hci^kH$L}$(PLU9%!Sd>S6f@W;`BGo~_zaF%`rrT8CcD%>f&tGzR z{3EsALj){xCKB@xbK0;m`Irrh`!R;VX;#)Z>+C3Y@#wCxWq(C*$NRk)PpJR=ThMPZ zWD49IcO@dVvMOJX+zU3hq287W384$=eerK>53(_RSA?>tX_eH$dHwIx-`N=%GZ2)4 z8b4~PpRb1CTV(>q>?C|%imbM}t&~i3M`_5Pkb1|*c<`;9zvt(u^ z8D+5P;#hSUH|8UU8i9$!-NE2hD(>rk)RgSQL@UO(nE6 z`s`m>rY&}+rsly4I5ifKeCH$0|Jzb`nXsF(cz0a$Tf)gg%6IZYyxd+VKa&fOt~Di_4#F0vE-o=CN5#; zXBCg0zZknnoDA+|I*KSbfK!>6bF`{_K1ByhtUItv9pd2VRJpD3NsmAV7l={7Pt(eo z7qzBTLa?^#?#tqn+gw&jTKQu0SW08^$LKYZ1JyuBjw(rh<+n8Iek~J$(#p;G3v_Y> z*bGPoBlK0}cC1#wWpEeut{K$!9thT~SMybFMks*Qg0Wst5MDn^ zYz|RSg(4{vob*0h+{DWDCq`Go`{Q=0{PXJXym^Z{Cp_#=wR9+lhF>?^lA?3CxD+&<`!J)B$Sb{Jojt+hdLNDjHGJ{)Q;ecF)M9IPU zubZbyD_E(TV)A$wj-?BJ% zuR7VbcT<_Vp`BXOPa8B9n6WFvUW8JXF>goitpR=x;iR{;Z(h5jsSa%TTK^q2&TXy~ z(ss_#eO0#;v^R;JJQYK8p60?mw zf%p5YPpd{ChbuE@2KXVnjE+8Ryug-z_2Xt9adck;AE7kDe&p7?p(_5d3YOUEF>&;y z+5&J6z5+>IS=?t2Ato(fC!0vQC{o{ z-pp>MRVL;*LedB<^ohiwVKKC>-G@!}uPR7OVu1As>o>axw1LwViq8F31#)brXzRbq zExwZ|UK32+d0?YPJlpMgO4+j+Gdar%Q*A$J&4EhK1*DoX2DN zmfMD;mI4V@3b!!a^#sB^)KZy7y<}3YMd$AXr$GUV`vV#x6x2jHz=$GFAXcKLZ%-3d zsd`~;XO~rqMcFs7*xL~XEO~|ED{>$_3=$=^6@z8#DIcz|lIIUF+NBI`&3}*O_WDM; z**J#dYqNABdhkV6nt#cqFUnPrRwg(mYtqm;^=5Dd7Wp(Feb(B{##wkoK1>Jbf#2x2JpOHX-5EG6|v6u^6gG`<>SX2@Z!6?&^G>+Ccl0P?~`)3 zL_Q}az;n71>SN4x#sxT`c&NZt(5m8*>&>ij4298`tuo|OCsKXGJSJ3}&}tT`D9pxU zpEKYnEHRm&QU!WAs1{5djZJ+6Nrvg>jpQ_q(Z$~-`;?$(0vq|(s)(jH4W}3(_Ro&b zTZ<23NnOV`GC_ywTW7E3uTOyAV==G4=IUeP)qQ@+)V!ZTGi}Ff=Pmh3 z;wkYN_s2EC`PZ^lTeYh8*4sFN2tJZsK}+@Nx7Jv8!h$WZr5vMf__yP-$DmWWP!Dt; z#@8=(y1Y7a%YtUYZ!|Aa%Ow9RAc~iP7KWP3D zHq>7)VxfbwXOaB+QZo^CM?Ujv(qd3k=rOpNmS69=`{%}s$?G;a!0E(4^bD0CeL?Ns zrvH2FZ}+8K1%=^pTe|yj6uR$NC^(_*>CYNKGMczTT z+j2XjhuCv(v-3T(P?)31-fDB>_<&N?TuNYm*!4W3=4ww;s9Vn33=;^)Y+S!T8DSnk zVUr95A&rVppz1?4%GvPo=Y0VeF8NvYKA%i9p_WiSCC~k)5#j5KQK5u>PVU3u zpD^3rmf{@J{oh$*i&SgbZ6p6qtyM%zC(=wrr)ZZ|q?7T{YLM6^rX#wE)#A1{%Q*5ZPXTI`1KSHj(2bl=FuhNOw^G|Bk1ZZu0RE?>UiMV}Lz zSDYkT5RK39A>+Xd5%Ia1Ypd{y*QrBvlLubRbq?3nibayUKM zTO<`h?fp2AC-EL=@Uk7o?prOrm0dn_2DH#jw9-s$Z5(SkjVYadkIg$La8d*2B5L+w z5rVP?&cef5|Dy@o<~s;##^eB|iTQr-O)DaERzk$nVPpH%+%Hy;@agm3J>3e-#~3_U9tg(z_=xHtlZpM3x6SuN==9aoFff< zmUg%+1X7F+BVU*5!X|)Fhq46Hq(}XU^5X4i7hlr(BY6|Q$)2)Ken$4xAAo81_ACk} zHEB`BM~^mU7#k6Fl#VN(o`2SNc@~*7_Cg?{?6bHaLs3Y_eMQRTNiR~%pSRpKIM$<@~k|=u*j=PP>c$ z9HsRy3bCP}ZJ|%j^Z#-nJoD+_%ZY9-(LBv7OLHXjHHwnB4tGf3RySS+){Tsnnps?P zXxy47uwrDmfp+~}8Af(O=4MTEgvsugrdI6!1Py(_Zc=+U{SE$Ue2fhV;mDQG1uJjS6DpH$nB z7G+)9w7Dq&6LHq6v)c`Q#pi9UG3 zLxCOa6p2i#6%{YPZO?Mk@)3Kw1gcmYc`l58a-DvHk9;xgQK?%1(p7qRV^+yivJegf zl8?}1YGq|T2!lCMr{cJ6`V1Je@}nGxU?PN>AFVWFWGIAlUW)m|I@^wm$w&PQNpTSv zUbi=Y2@vtkAx5B4;PQ)AY92E@U5Lz4dWA7W@eiKF#(a(sI_tbTA;9>~U3GH)qIEL- zr+fUY+L?Tg23wHZO@Es^h;&x{bP1P=gH@%)U^pI92%qyLM}U%u6z5je|+{; zJ5c(ovIQR)cAOb^q$d5r8-mkymyN?~)Ba{T7-h2>H1DP1(imAyvC*HEw^a=O>BnQ{{K^Hqa zzH-kfJL#MUDN2rgSc&(?l`b5#Vzp{YK7K6R5=pP&&+gC0znKjZjX$3JzLL})bAfW@ z_+z09sO(}P-IG50vS7`Gg|$(A!#R~+0p!lKq1BH=SfZ0#i=N_?QAuTxei4Gkq~k}( zBfZm6^!GO3&<-qi?!~F3lJfF+-Rz$+v>xEg3#)F-R*!c_;VLa_dhNT*gS3aWT)lD0 zkL{Cl|AJ_!>Tr2!51lzpRZH2sdt3O!e9>{bE9nx~!5Dtp^@%Ol(Ui8%=LPtA*#{~l zL=DPUnty86>ECipqIqWi5rqC-&1#}L;GFz93i)C)?HALu>0z@djN<=${pdor^+TIT zQQ5j~O?C@88s)6tm-XDujSRJk-=cwC(#kybY722PLBB6<%GDRrBT_$oLX6ZU;Wzy1 zbl$)gb^TP^F?guZXxKlvWz8@_FA+zBYH{II*5Y2rdJE*yzuVl{Y0y!K znazblk%ENnrmQdCs!ffmN2=tb-EdZXtv|9$G~C^NR6Da#Isb}vZnj*(!BRv7BPN;@ zMP8L|&(+j~)~F#uFAIM5ilaU>!>+lDqg6;jUq0+%Kch4$jkM7FwNh}+fQG8^d}8sFlRp(o#zT>lP#oBS+etr`+9M{Ne^H&a=A($2&3S5BKYdJ^@08HQo$Q+?ZY5&= zy_O26+WT?<5C?B;o#Z20SchUjy!4k!O#5Hk znIqbFX^=(iNNqVTJ~1)Qm2Y;q2g$c7*zu2v z04nr3lOotJ=`pae`ec8JOE8~xV`Ac|crE4`i5G#$0Z)a&} zNswrf1mvX+9%GrN$2KIUgPi=^atQBS(Z6M%*}vR*Xiu%K?<^p1C+-?9WSe(L$YS|N z=K^?>;^jUAmsqHPi{_Hz7`y57sR3RSL52~GCgHpy8UZ(}(W)IYKO_IBfFAbK^K1-< z2W^1wbozHv$p?($AXtCa4e?3jjei{|KZk$>^-_H5+OB&Xm}0?59)THCU`yDZtEMm1 zXOq*Q!Uq%ZBD3?Wlp3g#Qb@QQ`oFNH1U@_m?D!Kip`qi4E}zU;u{Jm|8r*SUe-zA> z`Vp*wpu#E+z#wILcGt%7xzKp}?6&(ny($6egOFL&jxIb>-1W}G z22$962lei5IUMlV!{YgiZFtlrTgkWLYAy`jpaf6Sdd`1dN`;W>SxNQspk;sXVCHz= zh0^<|r{2BJIe5^^8>`5|{hr;={wIJfj;im>SonCZo$tfo$Bi_`>0cij*Yh}g4SlUk z_SJ3u)bUEW^(jfyi8xna8}mkZhfG#!wVJ*{A}`O=?*Q** zJ&~9`ix9&jWI-`$GfSY|4=qLm;`$ZJ4@p)PeDZV_Sk^mtKS9=!~@4fryq9a#c=!`A+Hgd|6gD8py zf&w3SOAUV*sRFHEdO->i!w7>j#eEka9z&Zm8mOC(4S))K`lO}=I0ss$W(_*}N=g)o zGDnApM8w39xC|~k9a~%5Csy%q-)@r_b^kfHpf6Vre@$aC?lJu(IIt7Qd8DKQxs{H@ zlfvUn_HbMUknGXtlG?&>gHa@G_~kc|ka-9MGW^^3X=HTNjW5%ns(5hH|M_At5CL_$ zX{xI8b=ipJhXj|*oAbeR;Z(kql!Sx`yBiJE(Q&-!J9%>H=*xp28HxL@lo`=yeGHt? zMgL56L~%E_n=C=^%d0D15{&3=B+7P98f-`$ylBdcc!$BS6)tBLDRf{`a7J$| z;mS_(`8Ee9O0v%D#=1o>dOL;&XdoBg2qmOcC=7c>0Yy`vxvNzMlrwmTiZL&A3(ZZm?4!q|@9p06hCBUR84gWB4j?k~2bqDh7~9T{v_&5dic0CV|1 zUK6sK1gn8xHp9qKP*FVvvS|HB_pTJP1PF%C#>^)7Jng6Ebj!wZ) z5si^xHpfkpDdfu>?}_L)xp;oKimkw7qC2wb3V;bhM6W&fN|n=4R)(>gy(qw@K`I!< zWmM-j1oxyU>!gU%*~!`2RqUt$i3%ABsd~pK;E;ac7x{#_?$^7<`uYzaK8O&z2|$BU zf*fQ*rphRSP5PD5pb6LWT3>RNv=*;9v&PJ;^NNH!-0b*3n1|f2QMsWM{yA3 z#R#q=Z`ZFYOc`W~+UHjX<0rcmTC6JKo&qrow)n>g_Yler1O==e7-a1N4 zN4~O&vNybIWzi;~Lh50S43946plkTZn8~F+hG$21Z50h>)*}Ge?n6enu{chl(zo1# z>4@z6coPfg`(r$_V#w;-cV6lw*rHZiLH02_O1-@5Ij~k9OZ0`go4{71v(fG zPEKuST2ZXr+}yva`33q1o^sBh{xffPDAIt;cKK0=+u zCu=J!3{)k$AQ1M$e_DX2#$7B(oF3C%BR?W6EG$DTN~kg3A$c`BwE3@|!0-81Dl1{v z;he+Jn}%;E%jf5gKP9&2V6cDEu&_7+g@OUE%;mFAvkT#%6I2c5&usSq7%7)Wl43qk zSLzmt5<3=1wS0{HD-+aC2G=usdot3%AQjQCQ)3bmg0E{s0pE5(ssaXpc%D#IZ>mvF zl?LTMV-0Z?LrXgC@hY+J7Pg1a3BRpfdspJOp7di$3=E^Qz(kNUp%qHqj$N-Jmq#2 zS2peGS1kq&NlC=RLRx9-^{>jxu_3+`&E-6$YH_Q8OGb~&u&2ec-vl2U>dps`?jRx} z>P%NJ+I|DJDtPofduBnwzhcSeaWuZZzPYJ<1!_IPl-Eg5c<~ygEoRC0T5rm(mz#R( zqUnm>jURcKuzxZ#+=&MDcC zmp&?jPg7x1c9ucL1iO2C7RES05FZ>IFfuYWayziGv3+Gr-I>=^Qo`a;4NE|2kj zDv-QU{GH?#bWO?0h+hpl+P4*d6@Fn85n1P|v7l``^5{0&#>x~nDx;n}W+x|GJQzY~ zu+wQd5X+epd76Zt=Vx!6JMyB=!hbNfuUOFedQ=}){{d6uMRas@FxuV`gr>?Bp%Zh_ zCCU)<%$pKuXlOLL>?w?zCdASZl8~UK4v=VpNis6W2rBjBEE-xizULsjKAbJ%$OMg; z-w+Di_DZFSSBXp~)_*m0(& zrZz5|?k=Qi`J4za&|IeOXvV66sdk&2h|3PXiv6yv&!#d3m`rDx1Wx)al=;0=30??v8UXH4u+0kjw|n zf7X6Ae)_i!=+QLA4N~NaVBioPTlBOlnsBtrG%5}a^2d~Srl-2Z;6)*k=*&ozOyhR5 zqX+UPZhYEArl6U8Qq+k4IGI9DPEP&h4%w4N8VK8`PkXaa)&Bneo0}W2j;z7$iw7TB z9bUI673kQ+grcINu;WJ8T8DRfdU{(dZ)CXF^W`YB3l9lP*KzP^*xl*b(wW@L1930z zms{#w%!U>tYsY0kxAA#}K~vpP-7~7?+mZ_@YFKZ}-I_PhhP1su3Vw76>W+#3Z|mK@ zpi7o?aB%SVf2*!$vDOIZ{hpqjJhdtzljqM2}W4&j!C#Z4rl z*7~vO2)@m$htVX)psDr<83am>PnQ?E{CLFuZ$)CxjM}v{LPBe=L0{{ZfR4#R>dTk! z3K<$;pbd(iKD$eJDW|?pa%BLJn#FQcQKU_t*Tb@^kF>N)6%cVdwiH&=0id4}{WQ*7 zUhKhXvjgysgqP)j+AG?``{}aXin}U2sZ`Ve>gQvajtulS2?+_neU+AiQGGjYt=yej zR*SR!y!w8;&%v=cRY!^gSW3x6fV8*eD&;@aN*R5~#$P&8TN1TX$A|UQp96C?7x8eAlkjjmki#@JiDz+%p$5r$W z{*CPA?*Um&*{Y-Gi*d)LZZuYSR1`WUW~cMc2oTMQGKGu_bf51Y7Z(>>T=pKHo>Y%M z<=}*Mq1f*@cWiC4E2?}BH|d|!MReHzw}tJt1cX-*!uManLcyjT| z*@&V9lZoQknt(beK?RYOZ9A%Li%}O3fQzQY4W0y)FhJop**@mpG9+4%MLunv@_{Cm zPFb?C|F^j~I|Bi-WZT&ANC@62oU*(=KbKZePyiY?d3gjw&*UT&G3lZ*3sYOtTd2aIulC(#PA0x29x6s4sJxb5}i`rb_S2${` zF^zxs=k3shvTxJFN_s8Mg*iDn)z#Ixx!-9ZLERz^e!($QmIdTQp~eXyI{g|g2Cj5Q z?4NP*@%edqz)z^-9R+602x-m!w2fxs0uRq%M$Z zTYp!4#Xq50MYfIXD44jq-h^mB3cD3=_<{iR-<)Tj>vK4tiOeJ%@~Q8BFCgmbeF9|| zf`1E;x9SxqYG3=CE&>`aOGQ&w;w>#WCJJ1KN__n!?FDqpJoqL#n9tyIEmjOsc{nJ~8 z&3A#PN85Pfgf{kpp8WpO6g0-Kdg_{RZ?QyQVIm#o?C2upKKx^PYUCFaK^!R z@H{`&6owRL%$UZ_%)^ht#yv4#Sg-bb0rq!&5!^=xx-M4(*fDqK`KhpcU#W9l)fKm2 z^*P7@_F9DLANT6lxGB&M?gG>i`4xqK0o@}{D8Ryc?lmL3)u_z^B&NCf1{`<`U%j}_%nOb3J%*avIKXeHqTbg z;e<;3d*?}9kcQ%67jSamOb`xG1qg&ybZ9b~EW-Q5gOKqVwZ;@|SB zV)$I)b?VZ4_D}ERYrFLo&=uMM<>BLb0>jJTrQzld%r|hNexY#D!kZL%^c{I`M(&}N z!B5=}jL|umo#17^+zae>;jYtyxWeYv%vcg0NfN-+tf98}=iw7u9V>T?MejySGcXI% zWIRFwd?984a3NtR&fo1s#KcbvFMtpS?ecIv!aZaE zyrOJ(uvCIfZP^+u{*FT5U^B4^S&faYQF!7w8!v!VCt_MeAOOsz~G-n7M9N8|Xd zb_7fe3Lh>1mDhYJoO+n1o__q2)!N1ymCT0=^B(%^;FS3Se(iVZpJ7&H3YD4P=-yns zuB44sV|~|N09sVm#KAlZ2|uKV7{Lqa%6nIMd|i6*W@{s3Q%L-z5f<=?WJf_J4ujL9 zSfneX0Qo{^nQx&j-*-}nJ_RCtztww(goSY(4f8FOH8J7e`!~ydaSY9}yxnPQf-N{f z#>D%1IUSOWm(JkA`k=P_Y9asaRn&;Adk3-t7&xzakT=~uj8KGOB{by&3zG!aKdqbp zvB_^@NU72@$qP-+ju;H`u!210*#s=(qDRywr~n8ISoFV94^W%LISNYkN+a$g!;MXO z(r@DKR_@I$2z_nMhv_1Pd{Nx*WPUSqUz*T~AD=1%?yi&Sf|GAB;ycUoUMO7GMS&RA z(ZO8a)E4LOf!h2%Y&uiz%_@)g4xBixqT&Xgs9Zs@Wve@xz z3h)Vg#B6Z_TVY<9f!ye;P_?wEL&h)M;(su2h&DLfWN+jYZsbDU(X^z%0IfaK9S+ok zpYek@G&NO+iY*4Ly!)^Ad$@3VcuDp|3Jwt&03VgrE>*CySM$F zr?cS34V*r~{LJPKux@toVqPb8mxLBI(Mrwsj0baQD~W;dBm#7@Fobo&*)bYNGqDBh z^8$U6o*&#StgANoh4w}AacyT+J(G;j%vzerw}hj_p#AyrX5JAGA~8A&r3_t6Y7PjMwou7hA>M=nk#JJ-r7l%dQgN3B9+ zEwiDz5^?O@>qi%pM!G3#vNQyv0Wzn){Wy)U{yVvfsNAgiWD{uK9)8y|jcAvc0cs>c z!1+3ppI9?RFezQD-UO>YG2}K~z=5Q95COWds=2kOzo2jeEvBIT@14KBEvo2aClqgK(#2WZ@#Z80+Cje+QjSzQLI;L}g z6a+T`|DR0CYyU&5TwxZlo&dsWf}=ZtXjPG?{olWaA}MRtN>TNqsrInLKta3OL`{$~Mk_#6fdPR4-82Y#O{EjBF#3ZM5qQIc zJtMAAQ9k=d=@G}Z+7Wo)BV{vvE0ghK#e9J;_0tJg^gUp-W=Km>-W=LYR;4g1=6|=g zcX2MYe`y6MIRRmGmsV%2>x0QQEx~$_8tAI;NB%z62A`O}au)LcucKsvqSdGI6;=QX zSR|P_+aH%G3S2-c3=_&Qh$&}ET@hRD-xF0h@_@fgVP|fV#b=`;UXr{u(B$@Uo>C*qQz|D zRVwD3(l}w~(fau0ekHtwWUtiVD;c~Wkm1;RKC-;GcHTiEr-^}(OF-^|hyX@LxWh*9 zK_hlSG@r7=a4zZk_*<5zds!f3h=hgjY= zdO!J<)h;oe;>q7jG=(F1s4S+zNDJh7O2LJk$6ga(_^sQD z%6GwF-5{jML_I0;ww@q$U=NV{V14C)w4MOw2C}i@EsoVA2OlxVEX(b26A)jJDp5c8#_* z?5H=PH4;PNc{vcxjUv|A% z_3|r*)&h^ALD9dkL+>x8ABD?>I}6vD+Qf$gx_et%eb|KVup1L;RYz;sl#CqydqxG`H|nDT~(fcXO0IVaWL^1VnnG^Dk1)I|Fyn*zowk=_y|syl-Q!gX35k5u*L}z>f_A3ts3+=$na!nGpPdiqLWUqBYy$QRT;OO3z%p?3XXk~8oT~VkDQ@4I!>OkQ zV@tE|4^GLHXlr<08mi+8pV4w-!_yS@$r<0QcAnZ_}k9@Tm8)YPYc|2gYr1Nywy=4h_WYf0&w@F9C&hRu4d>(!&M4k*N=yvfW2A`_Ta z>Ws`ya32d7g9en(1{iYz)Ub_uHySvhMefXauh2M8Kv(SxasRy&`OU98CW$``8q&~H z*)whK^l$obCH7BNKLEqC?Q0MTPh8T#`S;6$;SA1g!Cl}?8}L|OmAx)m^7uDVkufcI z1WiONKLd(f2sA{SW;Ys17K}j?M(aW50z7E=J;0IAnOmU>9Ifs5DE2JQVwRbIYw;a! z5bESMEEP4f*9Zk(+!*y#@==$;(2UmNpZ&KW9ySL&&-4ECye8|^A z|8r=<*eG)Qs{X(dWH{$cD8cX%o)-zwuNzum$2jzIF03oRxVPvc{tM+d_BflQK!K$o zVCnh~)K_xx>117%qMHdy?!N-wMth@A@xltKikzO#S+zDHX6QKnq)ZqNl_s;pnE1JJ znmMZ|OtM#Uw#OJH9&VM1Z%lFbB%M;K@*)m{9hQ>%^GRJloG+1yIVGCTYp2cpjn?sjsw(WY;EO?9!6M&i@LlfyJIjP06I-)` z4^)`ohxX0sj?DQ243Y!pDZGchh~C)n#!>6YuMa+|$@54;%Ho_pNnqu%jRPfTT`- z9#1uNan7#h3b1c{+?7?QIuNRy2xq(2eHmN+&(QPNwL<%LK;$D{}uK_xH~fz!A^#PuX|l3ii4#R3vGO-&0}~ zesT9KGs~e!c(G`#)EIbWuHZqI@oehVND=$AVQJ=0X z{){ZL$=?b*?rP`rVU4Qf5zSxfbCgGcPOfQQd-a?K`e}xma-H6@_sP=OD3;9>H?*hW zyQ%Z4q9uKd)aZ>f$I{}$3g1*;D}LCZ+N*#YPL2>)*{Mv@=Z;KAvkEbt148I_l`RA} z@rDW!EJjx5IQe3>>p|zocbZx-eLImk|LA|?5VU>Y*64aEm};k-C4t=WFd3gs#C!qS zgV1h#JladnQS6&yX!l8nR|^U71P8ArmGA1NKp+w6YL4$agFH^DP^%d_Gq3hjZv;MA zLo?B_EXFaHl<*(dyT8%I`#1fQ-*oi5YL2=Ktb^YvA~(#Wobm{4en|rN8DRyWKqv_D ztwqa)Ck9$Tq(UA{$<|8Ry^OjC6-+3JU(@9cw?Bu;B2EyBT!RJ*gV_=#D9`piY4G6@ zUFn^Nz&$wqNkA#dsi6SdkCnl^xYlox`fRzb5ZC*oyN$`Sjep4Bs8_F-oUYsPjpH9b z-d(0kFaQ>-UEnH2==4~#%PtfBExOYh{U+ljfmd90LdOxr3$4_?1xF7u&F* zuUAx@-S?o5PtyZ+!bSlWo*N0dk>bBjoGI<8u>rHY5RjYtq7t1N3A%}qQ|wJ=4D#*b zbPjsng%8q-GDC_xJe2&tM0o;uEl46DQdtH7A7B}%m})+Evh0_xdPQxx*+i@Fd?vvQ z9!DgzIwtsMY583E>A7robbkHGZEgghR%rBcl={Wb#vN>lO*YY~Y<@gPi-K&Y+Od4F z3!$_FR$Q5n;SRbNys%tbs>J+G3IE)R&Xx){_BS$*mbkN{HmXeqzo6H zf~*EgBzJNVY0RK#c(Q$D*@q-~nM;{{*_+cu&#%wFhKP^vtWXadWA6FLNZhct%!w## zVkIAvm`|#{`-&4mzUexV}EAc5R4_4o&2&BG&Si+PJ`XBTh)gt^Z`) z0h8X@AQ~%le3gd}LEuJ&VL%E`2c}#vGg&(=ou3D{W?y&$kmR0g5xFxBRzFq$ zVFR;1+?(&mC7^u|F>ShWMfBxiN;{iR<|~BUvgt`|(Vce8aO)T?$`ZE#&gO zW56u{7@Rqttha2dfl#oZ?Svq9os$nAs?H(-ET1b%o$^F%A(WSC_2u|mp^@_6!3WhZ%QtCM=IV9! zp0ar6iQqkf-Qcu_^woiZ%qOio&%}F4%Vi>CBT1V0%YcKmi7(5G+D?P$lIJoD^yLz& zya&E7V9Kh91#^h77zd->}be~on z!U}AvSm$-?K{8JfOQZsX^q86W5}gxSHX*C`&h%f&UN=9nzJzylB41n7fNK;~u^aUs zA$g-gj}myEvRsi;LXu}O=y3I{zPq6Bb>@rLX$DfQxX`3P;)hrrLe_ zFqQiXaT2=ZD$)2P5)TbdKY8H{rfcVW`(YuoTeC)?i`Hq1*%!O;sqA9-q#m;WZhi7^ zW0*zrAS%a4V+;?mNT`3x^X1X(&`~s2-`T{2-BCPbyI;g7fB#$l5m7k~-dB zT5TlY$G!~zLc5aO1o?r`XNO3u_c4SRwOqekk4Y7 zP+JGCTC3!=5#=nUc~F_Lert){P~lIjs)=;Bfhm+kje#!-><9=zCLV`|g&pT7~PgUQO%-yH^*WKyhkv@Mu z5&V3*B>0gdWvxW<-X-e%G3{*cG2mu^n*io<+VMg!cg*pUBSG_pM&<2=$K7_Q>666@ zs<|>G4J|E+HW0;hoRUw!vBk9DNq2H-P8+9>o~9;3opOJb>VvtUxi*B@*ubmqgjdE6 zaGN#q9d^6w2dWR(s8S~^j}>x1j@hojJVzoVZ0HMLpkg0=etXbGfKyKWL;~wYQslYV z`1C&40H$6CiWTnNlVMcB^gDb{a)~V*0tEn%>oBy}NnVNWgrzx&--UVMA}3HiuDJ~W zt!vR)h@GI2&i~glxf`MQ(zn&VYF83OAK7Qqf|Ku2#WqM}hK<&*P2o9nBaN>^pt~NG z4+JUC8pC)%cKU2W#HWCfr&A2h@62AENWjg;756c`OBI|6MGE0~ve|PtPOYsiW54G_ zf6+Lbx!Hs+@WeEC!=CQ$0W`nSgK)Edt5NVC?N4U;BW`0rzv6w{g1gXP#@>(h6e zmuF!MBHVkJwXoYU027=vZu8{;aqVcL^>@wJ*B?Xo5I+&+s4h`#uNEks-8v_7NxO7Y z!dx*mA*pcp#s{5T<*V@@_PY>*oBzC*rvto`N{bT})Rh@-(GgxhKIz{>fCxz!0c1k5 zihcQ5$ zX8JEDKDY?&pG@y^#zx<&Jp45dSJO@|>OeIW?tRa)cmDVoMwL~}N1Nal@p2}JGn11| zWMWjx{{vShBUN9GRV}ftFUjZ1G;HHLoC;Nna^-fc@?qse+EC_1Mqhy~4&i5Ar){Qr z{m-*|+yNb^`EF3efZG7=4Z%`JMlDiF8atJosHabjbWOYDk4^PMCSpM$%cf8kjD2Uk z0%L7klOOH*fv1RAxT9X$Gu`u z4|i)N$v_TU>OO^T2V{CwDtiIBBuJzMC~c4~X1IMv9P)+ekm@iImSs)Y<8gJXqT~*p z0+^0BYZ@qYnNqGAQE3;mEb4uT-Idod z2PEqLP5pc4p^X--ce}iwGgBleKLrncJQoT$Xuz;vI6frSJs|Wl7{>F8Dy|`duZ6AX zowctmXiSYht#e&j1!wDJOr~0y6roNSlp}QWz->TA0LnE|`PNZ7X*r7A`Pe6cO3waV zZS!^8MIw~9Yy(YOt;6w`Yd|D5Mi@17?AsXGs_&-$iH23bmhPhmEGZMw&zl9{M4qxL z23`S??>o!O+sd;Ev!lf3lyl_2k4#LYjqS|N&U$-$6Ucp~l9Ch^ZSyA2O-ib>=tC>f zvPa!eVVsG^qstGvF_5GR>zuq58{weQBF%?>+8^``;_zU7{LX74GP6rgebM}~o|DvG zTkAHJFTudTpsK2h79Q%C^$4qh*i7aw8dhGz=E=Uw$Mk5- z3IhIk)wqR%#;f>ny8!vug~3<)qY^fw4*#X_6~ii@gFhu7OWqFAT*AYn4b=0LF2B+6 z`(dMkM8~V2udGOH_(}Dv4CThKEnfYFg5uOOJ@6O%pB1gzbc5qQtxDgVskM5;<~BDi zf3y}pccCA3XHTy7`NUkeyNM-{_0{s(P8OGzirO`)CKr!<4%@!b`w2n4tCIA~Q*u=k za5ZY3b+{h{fP6o(;{jZysk^z^X=!Qc>HCL<&{3Kqp_EAPv|Z}Jg7_)s82Fyf zR0ZZF^2yuT0*>TtyXlJb_)Kb7S65gV7`Sl*99`Al?DqzlzCTfY*RC6AlVxgWs+02X z5Q)=uO{A=vXsKO-4q>($W$Qi-bC340N)Rm65@Oi!pw0P^jSx46(kx{`_P?N^rOO z^TNf!K{WVT{PAf#S3{PWKrM^cx;q4s$gki#bL?}II3*V^??pI{8pvu74HJ;Ad>-7?%}|zIQSHV^_hW%EKy(5=jc;A zTGoH`i=6?t)??`gA5Vzt|3LT{7_65X?MWiD@E5H}^mKKb0&ZPU;@S-xo=uaUCY0n@ zWtbZzVP##gg|Xyu^%v^q4K$g3d8?k4{dp&n++2x(Irwz19sdn%-u2$zbV6aecW%N8a(&ZYG?ex zkC>fVykaq(uhmD#B^g=%L^_s^>|B(VGVHp_K5yI-DIF-y+16 zv$to#aA1HS%RZ~L>ga=EO4RWZw)uIxqoX>yx>={7W|vhs9A5cu{ZG@y*+yThaf78y zo?1pu&g(G4kyG>YsxQ;Z2G>IWN;Ewe?bCYa%^yp6XY?r(g#cfrhS_ooPS(`aw6L() zRUv0o;$>rt%U{6pYJ(oN+8#ZS(Gcu-jGeZ}cOB}eFs1-+YV(3Xx6#oUIUV%W5%OP2 zttX-Hu=pvUXK0ZDW@RRh6S`Fk2xy3i;+Sa^?QQMOC zGO1%0{O$pu_9%waZ zIK5Jynr&&ROkpi@y}!Tz8lT}e=Gdt{Xammd)YKF?ga?c7Qu+0#Ue^qxRr~pz_JE@( zp8+xo47GGFGfGlY)|3(PlMofg%iG%msbI;dzShUvq4Dz=;e@dr54XRh=YSy6(+W6l zZ}e3naKHR;9)RR)ba`60mNCcnZLk|Q4v&X(5U%t|8^G=2+{d!GLHz6@nq5- zj10P`{u-(FZ=VbvBAKRnaFvYz*<6^#6RdgId=|8Sg&Tj|6*`0BB0td z*MUWuxOaT)JZVIiu!V{$NG2;?U?C@mibCmg&}>8wBjn3@O3LWw>skb3YASphduKF6 zmNN1=fuH;h69)(O;*a1cO1rdz0@qzLG7=;gT?oF3>-_}1vlYp>q48nMrj(;8TEW!q z?XA!d8fQjMQj$eV$Lp(ulG4&_k?@2fW>;5Nx4-`2sOa#_-?gGCrq!L!2zwgP_v3#Y z46a)_B#&3dy4_=BEp40?aQj<5_nfr+=pvq~UmFT+zpLJQC%{bnYR0Iu(6GeizKCEs%2+qCs2sl?Ec5%FNH=Y6I@Srbn?=v6)pTIGLdW@c0j0PV|CsnD;{ ztC-cx42-e(w`Aq+FqGPQT_-Vec3x!0I=Z>n2! zAfAB)2ZvFQ<1E5pFd!BWV3D83$L6LRm1tENw*skf6fGUygbEhQr*YQ^4bYtOXP90u zP)`c29LWx-5-ta?cZUimNZpJltd6*0VPN<&w)0C_!@nedEQ4z*PYiUr*6GmHHc)Bm2_}0ai7je0tTN^_ ztN%Q5bG-aj-YQ^^H%Fxu@V007f%}jSb>TQ-u9Fa@tZM@mqEu>4RxC#<1cWzQ|F{6s zv=vZFn*O1CZ!`uz>5?|3q+ZNXu_nwtH(`&~4j}THSY7RJ7!b%=+t}ob`_+H@_N}yZ z(3~3s1*uTOE+PAh$Dv7-iHS)-z#G~X)b4Za;N*1FP5RagtxpIVh(A}@87!J<*3cd> zKO?Z`8oB;6zAU`J$-Nt>CrT;b&J-#GU1ou6Z=WHJJ>3m)%G2KpR1;%kubmSyiP>_?OtiQmRQeIvjQ~i99CNJtnh!#%6 zz#zv?)VGN(c-fwz04mLe>P@DYyPE?DXAkyge*)-Gr!xh8&Qp$!i+hcYJux|X(#`F- z#Pk4>|Nh&KUr}8DdTY!|-pP+OW78dH}iA_M0TIl%= zNk{lI8~w(&r^~V~4ze%r^^~4Ij0pujc!f%vPm(L}eIR}l)wgx-D^6K3MO^z0r^jOY-evPnE{yilY8|+Any{of9hWYDL6=s={;2QYxxW8Yv{V7fz)UK8*TnDHx zbu&e$*`JP}o$o;Z`cJC<1=;JEk{?{Ow#9vo*j3XNZV$iA={T(tISnmhj*-0> zBy){_#|MaYL0wKwlOS;@N5s>nKr%?$CE$u|=#1MPW9(XN(#DtLg0M7q+09CMV@j`G zW{He5LktYRd&g%JElcZ3)N|m}z!H0Lb!A-t`4xp&=!y?%Feqh}bSr($p^fxIvQaO> zTu0Q$&Ze27KSdw%ol+FtpLyMda|=I2f#K&v0HUS_?RB+4YoIPB z&+Mz}88ttu*LH^IY0bKdJ7l=da9Ll~DOJVb#wH}Z!N%_Nzdi(Pde)&yKrCdRB@+<4 zcA47s^>xW|>-&?*s`-;kuBJcV{12~Qd#yd%*>6X#Rh)Xb6_(yj{doe~IDoPB^DOVJ zFE_hF%eBi$Nl7un_B(cb_w6EvwnjeJr>3SxQTk&T_;VP1YkICO&~&>wH;I?1iM;$^3E=9?y2j(n4@34F=jfF-g(<11q?{INfBaxhQj`@^ zm1#Qu4-7&Gqvh4fGDTANcV-nUpR}(w?aa=}$;rsjH!@PzO#N+FoF<2XhtjjcgVkr} zc)*1m)wki%`Zk6hFiIdJVqu9)F|xF2A0@=ajg5^Lkga=;(Eqx&{yO^N$+ur0!M66twko50 z#>L8u4Nfqv^r;lDPmvVTS!eMu;3q|ROh0kgdlwg&RUbM1%yk+N06zpNYX7(V>Pb>B z$RQ(8-KFmR zpuRqYqra}Mp~TwNb8O=eQ9MUg3yb6Zf-J;um5^=@o0(qbMh7q(^w$7Z0^Dp>8vA`F@@8hNN_z; zvpl8w4XAdBzOj}mPx2Wy;t`7^Pc5UFN8Ov}Nfqy#-uOiR2th;AR$jkywkcFf9CPAp zA+cQ_>w(SVO}A@z>_Rz=oqvsnM+$MWZd168ZP3|UU$3JKZtYcE8KB-y@k1CZ5xw>` z#PgHmZ;-F}`hLierEyW_gzLpz)29`51gCW&B~5g;MD84ul})ib))03ol~3L5rX_p_ z8K0}UlTpO!yVJo``ni^)fw@zrd1aju(cv}YqHvUn7@R zra7FctZ^7WxkN`8`&SzKTg<0@KqQe3*bEjC`r3;K&??5kC*uB4bNCfoxw?j1lhM~O zBn>WU>>YS;&9x8!u)PY1BnPz0W(PQ)cg{w}Z6aAc0e-_-BZA{yo}s6U-yCalTiQ!` zjctv3Ti|dXM^>i?Pyh!SD!e)Ws<>K7m^fJb#*l_XPs!GzBbZg><;trO%Icu@y2Eyd4Yz#@BLps1fhAa-CNJ@b-3SC zloE=oysfX^`}ycpg!t{*1sS{W{_Ife*UF`pt@qCrj6#4clBGG5IRP-AE-Gs(vrOOl zQo~Deo5W!iNUC@ZwjURYc@V!>y2WlGX;WSYMJ$S!D?XKMHm*5+B?Terr6}Tyik>xgp&!?l)dYo}EMkby z-GmxeyYRV*nPb(P*|0UrRa<#AYe8l2oj5)z+h!dk>DS|iQXD-3YR;9Qz4pmCzfK_H zljXaF85w)0(ALz2tg+AQ{`h+RBp%QgjW)i8#Nfk?!AT z?4fT4nS4-j_Bd4^Lh^WEu$88AfxUGDq~o*4K$E4>Tvv6L$%PH2_RAH?(_f!@3Hbs5 zk@Y?})*~hy-YN|ofU4Qo^ucbwo2(B&CWI4uvWmTHiNSI9YQJ-`)pYvy*}AbHFsxB7 z&*!15&^>|e<-Ky8vqLeV2fgGS4JB@IwQs1-@pdLz4Ur?Zy?uU92C(`iGAm@NjPq5_ zza}S}=?GQK-JMU80At`JDwqL&FP#LDFhlsfkFtbUkZU6#OW!sife$b4GtmNDX`*&G zA~`=$%y*O`;p}x0OBH0H957do8t`5B9_>x3MWND!2yUh@AuGHV>3!>bzznf?UxYB! zMuJ_qYIcqzGc2105bTnoF26z1;VvVJ5Q{hdJa4N3+M`i*g*3pEdJqLw#h*%&6nRFg z=SJdeLM;=Wjs2fq-{fMgzrR|RCIxfe2L+tF`fI0rH&ZV%TKOm1Hy(6K3%11U<_r)P z*-jw#U*G2l5jYumJb5y;6a#%0zCGXRG)rY2=|>^uLDT~jbIL9hn=gcBw@ z{YbOdHTkN?Y6!K2JIBH5i7W%!`deK6otvJp1=k?Evf*~mD8fdnIR*^2*pP2GTs>8C z3p9nTX3N^fk{7f^5N$65=Gr~AkmSWNGCs1cN%PwXumSW`z@K`%E1!9;5yQ||PxC5% z#R(UoZ#(crTwfoR&|QziN|R^-CRw6;3|4*rC~L^7gnQIjuq7UtedU~v?K^FP#p|tI zk&|vE1VEIN$wL4cP`JJRi@x(?dTyK5cCFhYR)I03vjQ!nSN$zt1^tN*+oTgC!7jalvh!Upo< zwcn`?yA2ly0qvWIO4ues64eU%+44K)8C~{{3(c(Ru*I!WPlMb$7aRkj7LoO4zHfqa z!M@%Hx_Z0ToP&Aq1iS;M@!bHTpl!Dzss860NgeOpPS5D6zSF^pa52=+&{Gq=%)`Ga+{9UVv;|bze zK;#g+CDWnsS-ZZbP0g+|y##mtK0s`dZ`Ou0@sve|z>}1;WVHgOPmlr>q8mUbm4S0-Z!|ztnQBlqwg-zD0ri zur7G=AMAjEjrb{AT-%xI*$g*XyQz1(?g4GrxmFx3w6#T)B7k_dyutE2T~LALL0g~}SNQJOSe*EQUpw;432^dO zKc=Kh1M5!ScG!hM)#Ot7!$x`*Wl1?%XEyQ|2w!k&Gq@;QQ%oSJO4jCt7q E0R(INVgLXD diff --git a/doc/logo_small.png b/doc/logo_small.png index 4ed97ac00dedf917859dd9082dff26a2209345bb..bb9c2e2fb2f4981045b3816cdecfc545f78ec7f5 100644 GIT binary patch delta 7319 zcmV;I9BAXkJDE9<7YeKh0ssI27J|Zlks&XCDF#O0J$nEE94kpgK~#9!?Ok_#6xH^> zciPsJO|mHvNT*0k2tkS=3QzP0`hxrrl^+6vB8Y-Kfd}&Bfno�fV44F(}BRiXw_s z1Ja}fB#@8>sk`ZAcV=hK?~lQKYqAO3Qg_ZD``Mk{ow;+q_j~R+=bpJ20|1hwEej!k zsi67AwX~rmg*Fdn7-KVd7~^7tKCSXr30qoWs`RG40_57Bp55i7T>RGjdkMnUf*{7Y zlB?R1x&2rDnM$szhOgxWPCx)4AcQc&7-Iqxa#HR=ss7RB)rYhLBo?+J`KjRe7nxhL zs&fSp5Q2fe{bMw}LscPOw1`_6$e!HFKU z+L^O0Pg}DyXV1KDv&M8vYy?Hgkh&5)jPcg2uQ%S_#)IH5(~j&g+?f;3>H6A#iK3L1 zH<27qeUkp)L~m6TxVBEZu?bAxUvx^ zo(@%Y+A(l*{U1mybPB(DmlvHVPU-K{w~6o*iRjB{`up@PG!&P!<@G5iLFl+l zBtz;NNvpWEE`3v|s`IC@%bOZMk?~4W2?)N={|VKb=#U_EgcGwby@?Q7+jm(r;3tZV zWFGX1t>$ZXzRiQ5*l(6azvC~{E>BxiXegF~3TL&+3apHj zH5&!ZQerH9;?y&}y?TCs)^CNxLZ?P187ULrH1nl>WSYbdBhKj0Ni1}bd;$`(s&gaM zp)DJX>Wq3n#@9SmC9%*B@@>vco_uyj%eDa_2Yrc&)6U$zE!&fh7M(0I>K^frH%~ZAW1+3$#dr>X@QpJF#+L0#n=_Nul=2^4 z|2yX;L1+g9l46&mdd~tjX9M=#J$S3~_H!ZsYk6EUq;@c%&YR|!uXA@RDf!H(mn~~8 zq{0hZ<@1cKfM9CGq!w&LEpGr2#BfkDq;`k^!h$PrA1^)?qv_qUEm4qiE9OCBVIyM? z#`t>qjj0!ZW(&Y4XdimN_agD2#jQT)%Kr<*IMR#`g?+e1zq2CPLXk7`dnT4FdB^(_|W4dmm5h5`8 zQ^FpSAe)GMg@)qEXJ4%_RdiBz+!XhzoRpbY{W+9>|MQBQpYk9ad3aw_+kl@6j=p{U zBPFT$E`A4vDXU$UYW1d!AP6gNGe!hz0t$+Xd-wXsKNcLRRe=1whNtD=<4Z>*Wq?X{-EZ+*t* z@$@f+#|Qa6_*25}so@g=;N$3}BqCO)uXp)>c$`s{m3Zp8v`YWxyX zP=PSUspWsnxG)DH)WfU$mi|fRuTsnZn122r1SY&0uXV92LRFms-5U|TX|Ll$54zTJ z8hPWgv^C%7A7%(cZpqrt3T&h*d_&wSIVrdCwC)I@<lvpZVftwIPD8FiD+PVcMf^KP%Vkex|X z6Ywi;tUaMiDPc?HgnU)>2Yr2d+Z8f@xg&%a;~hDB&X!zQ+H0YLRA7v|ctna$iKsP< z@#>fl8G;E`c4!bh3ki^GLsgv*7aToalrpB%lTQRa9_bNYr@L6~MgZ`)V~N$Knh<5D zH@m;_M2E+a{b_WnO_K>>`)J1r+-d!}cdvh3WvcR%`Er6u@n>L}R(W&f&2@8sy1p9a z5rGg2|MLknTeJxP>ta`&EIz#_dbV^`TGEBmONR@N z9?Uxo2m}C(5W)x}Okje>bVvOK5kidd^P$f;YgC#cJ9D(~KkB_iMJYMKG~(nK zBhT6hABo9^x`KcN+j;^}1kDS)=u9qr;ElfpJWOIzq~?mWPY>iB;(;H3=o|M=uZ0Z3 zxMY+SKrn&HB=Gr$f+9mPk9Z}i3Y7c%Fx~_v#rjWIMI%ki!p*mn_vigkY}6wJ+p@ly z7C9LJ2u#|Z7;9)?z!;N=w9$}az!>8-F(3BydVs)$*lWMb|M9c*%_VF}CuQ(QeU^l( zIy;|5FgKOJ1cModKoRPHun2Y7eQ&1~L1=E`*EzekW$mcoDhW)OfEyE>I5BLT1Nj!` zZx|2=NI(dMtHZ>FcyCswFHc*WT7KP&_FCL?{$qi|?wO*dpLJ-8ygcRdCjO?s@aEMe ztiU0J6r^H!;Ly34+24*PAuO6uIU%bsReqMSX;^O3M_t%?Fv&|-y z3&~e()V+LewvN?fgc*Vv9ys)$-DV++6?0q0aYY~o;QR8vUw&h)jAS10?7FG{r{>|X z&BsWtBFD#RYxm#%K~&kp+K1liF_$Lj`Z{8H?#WZnvcNXyO$ve_zF`P@L%&s_s*vWK zux$e@EMZHhUw9*buQnec6etf^75!dEWe4=vh}f*Ji)EKvBo_k!v%9|XWQYGD|9m>B zec+}?Vu=#-LF zn84&G^9@&pHWcmEAI8p?{%##_fLKkmZB8V3Gv5DtU{1Tu7divs!$+UsLFnPx?X!L> zEQc2Hz1~>zuhgYi%dU%nc`=@|yS!pfEZH`O@$tt-SPi!cL>@`lYdfjh&B*5hLu#pu zU4y>Vv-&Q7>c~14Ph9c%Y{^9cATg<7ypv)-4w468jO*-@(zuwQu>_h4@f0F$Z*zM^ zk+K)RTE6zRi}M~1h<`rh89$kipUkI@rv7th|Ave$wY)(9AwaHO)@NzBI>a0oV%LpE zwx7y8|KRQ{1OOA5rktCZZz#yPo&9`B-TZ8|srt=-tM6PWy+mP3LCWWKn>8#Tp~d6E zb}x27h%sK7mXvQO;DP_5-|9Nw{$6_J_21vb7y*JI+ClI1TG*snB}p7Kq$*0u5cKjs zA9nSOvSjDB>~GekZ(;~WK`OuKw>r`zydAp_ZQGE(67k}+h{^R+Q{Uw5mXWe>Rp_Sv zYumSf``0ER6!BPgW6hZ0#4vSd5|Q<7jqp-J^~X7+QF2SJOIXsz#t)M8-!WaIp<0)B7_n< zjhJ{ZV6#S&bG|)X zc$CD%sNg5;QVjqAYqW9J7hLokgfU_mcZ>@G;8NMu+?u--L4DL`iRD*kbmu8dKI}jE zrSJ*%am!@7nb04MhUU9P04gvj>d*;yWFf}*i`&~sOa{pV1LWGj1c)*36&i{<}KoeB3vV*uGduzW%0MxFZ zWNr+??K!)Yq{3h3r(raJ008dP&*;wy0FMSfa?e^~l$5^F zlk!kf3L7Wr=qO(;YQM>M$|?mh#x;BmCvee!-o4Ph z>7=h2g6`)NiR$7-UTV_*vv zblX|9*d7dNYWW`sp-#$R^Zi$G)kc9G7BHlFh*F<49LG7_11O4ey9%~Qb(n!S#A^D8 z)G4ej;((J8j8^8?lx_z;a^%Q=wQJY%^71MwDn^YOHGKGR5ze%|A7z+Rs^{gKk?OIFf3WJM5oh#kt8YNCxv=0 zaA!<{>7yH;hb4PT28s=##Hgq!e}8{J zKfmtXySrUb@nZOdYvq4D7C2P=_;SRg)wkBiYWlcD5de^$o}QAD!Z=sYVs?lKf~emx z0N{Ck;>3w{mg{mCL)t+RxMR@fKHfdVkHZ2I_C2({i$|nO@WU8?@7uSpK7N*aeH)EN z4-b#~vU&RS>GARLEX$ffvivqPGjr+ErRKbhI~HEldHT?WIjYN-FV|P7cz%PyFlWx3 zjT<+P9)174tNs1`Crp^IVZ(;ry?a|FSyooo-o1Oi zE`6g*I&yMyvP>p_v*JSuLQqi9rcIl=bm>CVwC%Q(qNo`&W<37*<0ezxauAXv4;(mP zG#cA+xY8gCGpn;t7N4#*)nJTkOf}cbZ+w=q*(FvqBO{}}BMJaYrE?`Q0Y~-0fCZK%1OC@jF4qjXFKVO0zgSgiB;Dl zF)`6ek7LG+8Fe5OyTaYOciqmAB7_vAoWfK_Ri?Q1j+#=Dn8-I2Ie9&k$;5G7eP}6) z8ai~S!}y7R^z`(MjEt=RZB0#0Wo4xsLMS6Kd9yNI)Xs)}t5!t6H_$iEc?hkhW5UA1 zToTLo?%lh8{)ou)e0qAidqIdXj`5BbkRWcpjWO2B{RaBQId6_Qj^lYNiy(M=dpB}& zj4{V?!e52wc`>tU`Ln2~D5J4%RR=*3w{G2Xuj~v$Xoz;;?z{W5YjVvUyo4*eQIR&- zZ-DbjuZGsAZ0KKu1`X2d^>r|&C@LtZZY>?dF!j5C0>-!!3`}@Wsgo=#CoWRip zjWJ$y^}P$Fzt8SEV{Gs!CkL%Y{MCkzKl$X7HX7>sClUQpw;Kx)LY)*HDNOxTcI^#4l3}aQBdcD4((>8uN%WnbT`t|EpgHCt!ND)Gzst^iO-{l<=t>6Cy z{f!lWSe;Slj7BJmvI^nt+qYd3bf%`JQj`_bqS0vF4nhFv;n`I{LZP8ZjIR3o#(L5o z2ElkJ|A@mxBuSEC7^`I3zkk2OJwwE=q@*MxBg5+GYc<$t=MZ9y2WtlydA6LZG}CT$ z@ZSkUth$-x%q6WlpCm~hJa~|0SqD%w|7HDu`t|kmK4Ly0CML%1AVdg7sKaC=V-$>O zmA6Di4D*lY1kNBBiwwmMyE9QyQGy`UflB1c>#x5qMs@ZAi7`HX`gD4Fy45*khGEod zwL3zH5Q_I3NFX9P`&&^>ksc9#GGBzymds>_E#A0s#fWU}_^(cBM0?LdUk$>P%j+)Jsv(`z9B?7sV51y|wlRpa8~ ztVVBQR4{-3{FN(Lnv*#7gAL2cBbH?sE?oH8XP-3`kcf4pZn~Hy0Ce$)2$i%($buqoV>ie?Ck9P{QO2r7cs^Pg<{mGQOKRWP$tK)s002TlLKZGuh}yLf_hpQnl#BRT_T>caV1&`GJNq1Cp-83=KKMY) zhc(G|xBwXAxpU`=zOMU1C~BO46HL2u_gQjf;RA0l1fw^We4eq@0R@S$`T6;M^2sMI zXQ7B0&-2TcEfa^^_FW`yQzh;br=D4HGwE=_FO~-rJB}bQ@$H>`j?qX|t4gI>ym+ze zccP-GRjXEcdV0D;g}+S@Mtk+7F_oe}Yk6>H)XO9$4ZLCb4F?y%5cwB>6cn^_<;uv& z$eNlO=kKSswzg-_o=Hhb?KqZq1j4bMo)VDooBn*A<8M7M*OT^$(e!pWQzO30WU{$) z=gyrwSEW*kCEgD720;+ya`}P<3uexo*}exq`#e%JSH}E0afo)%%q}lCT~ovhUAuPe z+i$pZ2+ zvTS8#mvzo_ywuj4$gd z3y{BDD<|ZoT=~khBp39ldfS3f6s1zBJUl#neSLj>eLX!r)oQh6xwo4?SZXVT=SnV? zv1Lb#Pu5d8!U&HK9V>uvF#j-;q}j#RO%5W|;Q|O{to7N6$zkK~)Z|%i{v%1NgHS8; z1q4RH*w6sBsK-2iNt>|Pq#dELejir)iTOEWs)psx=l$5IR zpY6SSFuf{s;@N2?fp@F;x(z^hsNiRcAOjj*{RseOc9|-GP-UuKbt_3)fut2d2mrt8 zej^a^fM>VH;{h+)>)$>8DS+^w-2FfQSwk#IOF}3b!?CA-CzY~gRebgI$jSGeA|4iy z(8H@cLoiFPFBd?N%xOzP2mq|Wj{f!eB7=?tZqWntRFuk=^7wVJANw#G02p`X#rFN2 zd#i>}%v!y4epU%vY7~tB=<;e}#}T$?w83-}Pvo6lhuFeI3 zEf206)9LSjwx=R7#*ymqw|dOwK{!%);&{=ol55kf5Q>cX@Q+U}mR{z8U({n>*Lr(q z+ae=^h7a|RM+nWk^6!kQOi9oiLN?{x zYrGUGHm4!on|F{0{vp2suS8C9M(-phcMsY|6EpySOgj6DWKNqxtT-CtnDd>GA1 z-I+W?J^if^0o+8ajIS3(vaD+Onn`f7nQs2mb zpy6^#kzZSocxoI6oCJy85OQEz6Y~*jxw7^y9+4kMFBOnb#+Hvb{&bD0Mnc6t2!{tg zqNY_X*#wOGX`(;xx7sMM9GD(IF{a8?C0SHE^jBgm^`^aALzX17I{UfPFUm<7jp;4@ zlR78@rQD58EYvXGtwd8{s*qiSwi1|s;DDQOcG{(~tCE{)6NIg*HNnabdj}_bQXVG3 zH1qd=o-95sxvBRzq^(Use?AvJq1dPg1TROt@P8psOS8mn2|}~Xy7H$ zacd$iAg1?crEIsksTfb1Qx&4Y=Z5u*K+V)r|%Ax$9`yNhMb8CYHUR#AQxAv}@ zQvFzP1WB`rZ4|;uq2mm^p`5EIW6PyMu}wlq6LdeH7>w}ex4)F|Ym*QngdPhVh7tZD x|FC&8FbTr8BmmIIyC*Af0)kSuOoH(L0TAE;0GL*7(*OVf07*qoM6N;tV1k0<8@&Jk literal 7621 zcmV;$9XjHPP)Px#32;bRa{vGf6951U69E94oEQKA00(qQO+^RV3l0J%0be7u7XSbqS4l)cRCwC$ zU3p*=RsR3JnaQ1|O`6`NcR5-p2o$*wQM}l75qCiraak3w#pMtXp#maOL=*+l<5dZ%gVnKb9jyx$)Kv9?WVo1|^h`Tmt;CU0imXFlKSecyWs zV+_1>y42hB3V{#C7$Jl){=V!#!Q#N-{>ccT_BMrh2s=|a%r@TmbWh0-(<7$-E&c`P z+obqz?M(2_R%d>AV@t8IR6nKESOWqhOk8xq+1mC}IYD^j-rlsW+J4$cnCc;7Q9$<1!MeZ@Pj;ro+^j58352%5#RY9J`bU5paF`0JcOPqT9Xa{#reeY5PGWU89f3}36(sA zo+>tm1_TsA@(_BeNGS=%U}8)>gq|uYp%MU>7|VDFJypUbAsFC>{w5Ehr^*1|L>9A| zHCa4_9;--2N?(CbiLunmT6qXPRfhQwMSuznMSPCbQ{};+@c@urm&4~sJyI~n$*TLp zq#@zbu+G0aU;gBJI6Ik$JPTa|+pnD96{#nJXJ*VxF0U*w6n4oy@TI`c>dbt7!7CRQ zHqfSiKD}e)Q9Oj6gC77wXj9(SZ%g-?8S|earp$_(-X(Y9G44!T+yo350qUGPDK(a@ z%wBi7Hd`t5O-WolT0Ok+ISgh9Aozm(rgGa{Rb{PqbW}hHV`PzTMUVrJ!OTyZL#wja z$w+BGMeomse%SbE18w{ye_Mf}=(C|8xOG~gM=H`ol^L7!wmITotUPAt{hzrTzj~YD zwTp`j425Dsw5tEoaRH<3hY&(L3%~w%@wa+fZ)53$+Kfp-6L?$r8zB)XB!!OpSAg96 zEC`_=%m2GR=VKbvgZu`(mAE)W63po^gwVyBtPQzaZs_ynlWW;Z|<)-wO%n^)?tk%%0s9JF!p zS|K6y6)1L>e77WP#pWUF-KP?0mj3IxxjcleG7k^{Y{}p8ec68+=!X0Khc52>8b>~k zTbuH??khW>r}dH2UT-EW9O&1d>)%)7p1)9&RnSo6K9z8=B4bnDKWnyX?A_}!cj^7S{LiJCqka7=rt$(>wiEdILqyF=Ox z18q=Hiut|gObMUN9pxeXwVPOAuh-?})aO3c>k01nMBl!4gt&8rq;P4!`3iy0n1GQ@ z3Fla>jJ2?#WcihMqGgdQ5|@NaLnWl-p6LjCtnzK?zBSi2$SA1~C4VB~vDbRP*!U>L zgU7*;U6->qXJd)6w1F{>2^bM14zzdExvET!x$^g6k8-~}9{O;*n1m2ok+`I=q4ryTBK@nzRpthAI-t>0V5-&5!^3flF)vNK67H99jqKE zr({j8-Zj~o)|HPV`L1;T$NB$o)L@)EW_$9-?Y$EVV?A~I?C#R<&sAR_5E`Z$Iwxk< z0KY`;Xqi#7wcwuzD-O#j+0Wy@cmBc8RAuff|L=vG%R0+#3>affARcgQA>h95Ec&|4Sl;$_3LA=Z4f&2P;X}#C z299J{rj4x;h{({MsHf{mM3RW~75J8;D9FmeD%dLlLim~=J*N#C_);Z4!JH{B(m~Ka|2PgoH0Y4u4@RYF0y&Fwx*+P|*0L{pO2EQODGN z>$lqYtB?=?2s+#v1AG&pD++{K z4A)r9vKWU-LO84ZRneX*YfXqaXkPDE#s`jq-&Vx*EC>r5iWg(LxG_XAwoz9 z=EOWZE#k@MA)vj3D$Tb$VMM!?d%lxxAp{!+tw~nh*R7O9`(8(vX#DR=_piIYQA~<4 z;NJ4MZOI=wI@8A5a_VoqoxQHiq!E!K=iRv`K2+Oj2pDXvZAbFvzKVE{PB6KEg$pk& zJ6nB0Op0x+ZOwp`#We}znetoLyoB#|o`)2sI|{lfOKppyWS|-9sWm zLW-McPBUZv{8rkITWNAqMgR(t1iupZ+}MDT7-M@t2E23$gc##$Yt78lbA(91U>3C$ zT@*GHy?lP5kuk|hnT55ymAH6P&;zd8=#3X=Aw&pm$lc5UV`j|@`n=kBxS2y}SMfKW z<$pmS!pN8=28~GCu&FHR{^l}IHdgs;WF)VP+CL+Gum?;h43ClMhe1aI_T z_DIM?Z!$UFqe$n)zcf2)hQCPFL@{2d$vRY#AtFVAqJS+!HmIAOz)M#`h%q+M4Oi=O zhN=cpi1HVynsX~b(RW6S(y9s;s)J&qPj}6Z@CO#*e`) z#w>|Q5g|-XTItq+fx(Q0u{cP!U|UzL`QkAU{!sQ)W=$4FkkQR%vCcUCd{INm>i)|f z3Lfun%lBm1?;Z_)sFl#3hp=T8+Q1k$-uP5V2!h0c;{!%{W0`S>x8W7am%674J*+*(V%Ef%mM6S!FU~;-oxXiGPhS9l#;QjQ zRt{{}3|oW{1~6SbPuJCTS!xDj+3=A_BK3Y2et2UGLTHqFc#>aYV>=fn3H22!5*2;d zB&BrB_t1WjYPy)&HAKLGF&ls2xUg4*mqVCVwA;qo5JJ-;p0uk_4#g;0uOCN$<&6JA zNF;<@-Zg;-#w=!m@CGNAXq|!?mB&d$3JKwG|DpDBE}i+d)?5i4Ez0Jl^B}CV>GKRX zX-uz5Txth3UtjRp@jpF$bZV8Qng_4DL-@t5G>oxAph#8?wilaxe(OsaB^8lEu2_hd zZVlnS@&gn>g-b$&gn%0hEHh~iS02R}&+GlN@b0`wUb>?@gE1~LmN1yHVfw+K33i3K zF6Sd5A*2ZE;ot}P6lk}G5FvESPy_&0*7``u101M&+K_L!i2*11CAK$~=cNmRipzD` z2*6*c@)h{rsT8fxGtdoY#*)%+k!u`#i-k3ry6n+03*KEjlT)9I0Y<8a*~5mfiuWQw z{S>{qQC#P>sCv%d=$fjogAhoizjs2&!IM*;3xE<7eYhWOtnFmg88IOmq8#LWEXesm z@=;^E_Tjz586#RWgeJyppc@cE5z=sO@b5xR7Kun2(?gVloOeXqKIx?KwYbkNbc2ys z)FHO4NQ;dn+}Mqjlyb@IoccTpk!HqnpEAj{iBD&mEjsW=z6z!_D?HED14<}BXIC=7< zPNy@Q%_K>tq@<|TY7XOqf`YYc*U~iIVE{13etv%I)~yo?g`SQg<%kC-A2`tX*)(P? z03_XA3c?-av**yypFh8A*Djq-Cz0G)Mv!3`nx=o%^ClBP5YF^I#@J%9Fbw0JEWGlVNppEt)4YUCOqTB$X z-T0N3md=|uud=d|AP5hdWZO+W(mf?VF1)n-vEzTT|7d0`7_d9`I_03!YPHLkFSlB) z1i|mS;538)P-HBYk`e-t@h%a$C0)d%*yR+&7@JI{HEY(0#bVAJ@~F4~LLW-bg@d-c zi}Bz)rV+V=h3nU^He@-W5+l)wAgIpHA34$XpFJdY840sj&AacFTO}kO_fL_&}fNDFI1!m z&EOk)l#lQ|HW? z6B!w4u~=FJ9%KCW+izPe7LQojspBlXTkOjcK-ddJS{w@7lGCqNwK2rPu4XY}qn@{(SF%(4hyvDtp~i zC!f99PzV?>#=qP?`}FDOzbe`5l4s@`(P%V|OduE9r>3U19ls_xO`JG!!GZGOG~{Y^mGW}G2JiK)|v~|m$)eVRMlCnx$?c7k6h@;l$4aT7)4-=8HQQ3 zXpyh4Z^v>m+{uZFiA$F*Wm&e#e=)`+Nq+d@hu$1i*b^}Y4TW4t5iN^kFe4&Gm1do* zuAQ5k+akAEmQ7Ah4i66}h>oxP#YLP+Nl7Y|s>xfzHCR_yr_pGk=o zP1Eegj-}ZJn-VLaWtkxm?}~e(l;JVC>kjBuO^^3qcU2rKMh= zNFCPBL`r)BU<+%lvDPVtz5wvq&=0vK4qbI^lY`nPe}Dhhak3BJH$JDuG5tH!Kcb?z z(4~nnE-o(iW?L962?78CqR!V`v_G~TKj%#d#@J{yItuSv|6@oTUg)#bYEwK*HziXX!;_Y6o{R0EC< zc?a9t4Iuzb3>wQ|rrKIdV+H`kq&QHdrVw>Pcgn`voWFK+{l-a6O-(Dhx|PIOlS5ow z0RY}%^Ev|Iq~He_%-*(CF)YpX#k5{iEUcBrbg{A2d2{Fp!itItmxOB^mRGM{b)+z5 zGMP6+h%t_lMad{Bi}9(db6j^k95PW(Nm?=ReNV|>$fG>-7!`)3bJ1j9VF)f-dk|clr`DbUi zX#Znzaj{mbZDF4ygfPZoVPRgbNCBXiGzKX#J8P-48yElySBx=jQyOLm>BP8;TZJ@8*3vG zab2IssZE5?OL5Q9EPbv1x`{D4lf$Ga>WL?wXwlc;_*YX?vwZpTBS((dY_`^i@0+|# z8jWV|+_}ZY#jW6{P$;~Eh2+YWD_eP;MWiUb@~DAs5RpP|6H@?)l}CSH_LG&h7B!S` zrm@53Dn33wJw3feE~2|~>C&a+$B%2ZT7f{IR;ydAQsmCm>-A^OoJmVd`|i8%0Km~e z2SUhZv(2ABpUdQVZFa`4z8(x5pH{T{LQR(acM%~R;G1x&>fABiFFMO@wa~uLGd3kwdWu?_>Wf(>v z5ZFhO9kDPdC@3Z-#+%EZ8b9rXz%jdu_XLR4jk|ST@0SjrI3}Z{|J59t74@_;qk7Y) zPd{|%Q0wK}9FSb@)MPT{<>j@kQb@A(N;vL5tyb&InKLPh@{XF-HkIC%nKUY)pO_Fg zuJ3#6>bplPPXa*3gnbMTW@s^Ay}53Fk!-s88f_Qxra7&eW*BCLQ3p0 z9QR>X799flx-}2{U7Mrt2}rlO|1i;e{8RnKIaxNHI7#I3*>8Q`%lr`QKIr z_-f{&mD#Cfrr!*lt{=Dx01UL@ll<*Yc0EF9(3VcaKp1gg!@gcRyf3$iy0mPn?AFtG2bCMzeFlyAORjXEs#bW#9O()51|B#T7 zuxQaDFPPKY5eT_w?(GwY0R2aEut^^+=<}+cHrTMu$$|_!iV+bJsi~w6_)jPlrlh1SSg@d#fktl*D(o`Mf5^$(XZ!`KCjG7!`5zd&r_@-U=-bzMbH#m- zNFSfTCEf) zTj>R^Vjo2$TVwK|=yy1LqCv(?npa5Dv!N~J_1 z2?z*KDwQggDj*;LdXlsQLKfJ3eL=W1OiH$}kimP`KmJK`&`28(jNPjg`nh%jTjP+l zjr8p~3pKXY2SEsZQ}X?s^Yhl`e9&UoJQ4P&g|SFT$vZjkySDG_Hte3xNMq!B&T0$o za3gh=Dh#-=q2*GdK0fiYqyJ1}`burK8(0My#*9VxNrV`dc_ z%RGI9YHx;6B~oI*j&@B#2q8f43|Q87Cy1BMR-__A2mm&G?+VgtOU?T0o3d)I^7wTx z2&*jB2#}PN-1E#Gg*$&N|7q!!Rb~g2<)v#NJX@U!0NmzvvKN3I~=!O^0zXsm2 z_qb<6$YHRzbRU6;SXuNvPY#oWObMUNV5X>{^vj}e_%)rn7(&h^*^s;0SD>)5^w@xr zZE@@8^?tdJA|3&JeB&Q^`kTB_>@o;B_PlfL{k^68FyMHfmy)fYa$mBypkc@Sd22%)sX-G?eN z0OS6?{l2_^n}86sD^%wId!yg{IC&HT^m#$r*{V$5D7su)UbzP$bfzkEQC1296ebP* z^8T&R(exv?Q_!?ivyHTwBFL|XZx0j)@cr&y2mzqhS~un7Oct{wB7YdY%SRx0@-jK~ zxwFr{Dk4PwBGtE}cJhfgw=2@UW&2Ew$%bto4_@aZkUKw5v!9~RyxyxsV{x(QC0GuAd;r`y=w_mZbw6LML#8_5O>qn@Qc`mwMkv5(H z*81AI9pfSZ%sTT@v9XlF?DB+#kA(1>Ai5etD{FoK`leWU%%39ufZVfLIgRNlCuZtt zL%mIZaNM_Qkw1@tD{bN23yTj_r0=-3v)-zA$Hg$l6rw&E^1g+!V!$=lS{?xB<<}wK z(^cnl>+>1Rj8xz6D^$3J@(2Mygf#qz(O=ft^zlA1J*`y9y)4{Q@;wV|usCRC;u3ee zi-F>Rc%K*mFfyhyRhjm!6nJsU7M7VbpWjORYxJMPB%z%~i>FS`(wMdJ^4J~2Ho11~ z_8tUbhvzN0L^kDZ{i=8miOA9F;cJst@OX4Yks<_XMZ2@ySvlB>cQZGB&cwj+1Q16m zPiEC#;qmAQ3wIZP|Iv+4F~G@*2f74B7TE0t|7T*%bK_qm5x?hF`#sXn@^?@KHAFeE zYu+G)2u!>fJKG-nYfRb@Ne~ZAI}|AcEQ6U>;-2p^J6}(2hcel|j&23@d9Rvdg z3|7`U_rj9w`s)Z_LeQ8M2}?*s@=?D-2)p4s&S~|R&o8)EpNkOcujsdF$a}8Zw2gdU3&xrj*y1$NZu?aMLb|F*url5W+6l%VnyQO34vo+%$6F!tZc20|U(!yGejOkik4v(M5u&@aob!Oc`ZN?vaO%Wi9N73_A$$WtC zk&3jrQpFfon6#hdZ!iDtDz`ifJxSJF+ioBQ z0!2@DQ$4;u zg^w{jzHSpCwDsmc=U!OMV5Vm}HQrEqyS-p%R_)bv?XjLoM{j^|sOmlh=- Date: Sun, 14 Nov 2010 03:26:31 +0000 Subject: [PATCH 0504/8313] update --- doc/logo.png | Bin 13171 -> 8699 bytes doc/logo_small.png | Bin 7321 -> 4506 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index 1f51d1890c406ada11721f17f29f8c8061df527a..f601656c61a4d215fcd2c05e939766b5b50d9afe 100644 GIT binary patch literal 8699 zcmZ8nby!qiw55?|q-$ttq`PwfsgVZhPU%J@M>+%)RE83eEcYp@gTZo}25)uy4(-Rpf^9wofC8n2(x+3Nr4n8UcL%u)BKEM(i-)5F=~B`o|~YwW?DXl*%X~&S5I+X zJ7YdNf~he$-c_F6loqE&_20pJq2Ez?2fEGJ3X^4jHt)IXSt6OP!B7-5Ts-x|aZI}0 z_i8E3lZJb54%f_A=!TP4PnWw~NSdE3co`|rmzbwIFS^c@oWo}NE;;l0I{z4? zGUS>%?Be06M6S^<7TMLmnQ`<(e+g-N6KSSCJxMv1a!gk8mh&knS(vMolK_z?fz5_QE@j$A;|3!MbsJd_0vb|x(t-?ggX74 z#=P&3j!-0?S&{LLONj{~FA*`nIv56>FBhApk(DHV@kz2(4(TNK)okw^@p$NZIrBz2 z^JYd8UlATT(MmvES5%{fDOWD?_pb5x5)(hcb=`N*f$6vcs`mt3{$f$^RZxy2t(Kz~ zkWP{x$rp+u6}3p32b>%F*}R+Cn7Uw>d_nLnbZ7iEPmjp&JQ6w>t6zhYZ2Sjutnj%p z;ir8l>KTYRa(aKFl%$x*D>S#%!LQ9t*N?GBN};!M>KXypwVQyf4KEvA^d_`p?Z1L@ z=}xghAfx@VetwrHUbeLJZ?wXGD zd+hJyG2Bfff~I05D%Xx|dt?(N1a+3E5;IX(qaM-T%sAB3IFxk9ggAa*OY0r(7$lUcps8ov7a4qA_u!ntzu zg@uRP0~c`n$ATaCA|XE(LpffTA0wRj^I?$j+n^t$vDQ7xY1w%lj+?VA4E2-27fO8A zL7sP*&EDH@CcP$C?Gd~S-|CX|Lt(dVzv_ExX&yzcg$9fX9))Sp1t0u3|0R>Bw3o=_ zYvJkfKU8emnWXs9d)?PE=u|{aE`1L+@ITHjT=?!K#$qc51_u+Up(M*mnq9I>N#6=u zUhsa(79*%xdpEI#S`)~X&t%vsX#5DXI?(053fzfnyBIjfSD57Qn=3hvF01$Q*LXS1 zLo8fqwsn@W)4E-k;0c^x@UBEXqNVvAbA}BI6rP5u|Mis4-G9$w!MKvxcfXZ3FzXWC z2S|Ol)e$8Sk@;1#bwOrbhCw!r_VIJfD{Pfj)<6V!#W`YG1AN znic91!P2yrp0vv@7?m4fFtqSCAS}nS!=l3`>=So94%+!wRS<%SB*W-@p@7gVFT>cu=7Yw#lh&6A1JkeEHr%mCsFg)Qms6EpN^6)CN zq}AM;kz!Lxue|M#Ht@E`qpn`7MJPBj9M45;z$W^tbwD@c#v4bLQ!9dBAKw9qwjdeA zi@~;!AJjITkN*?wL7|0Y{!`F$WkfogPcIx6lwmhF%?w8}w^}4!sJ$0j7bfM+|H9Yx zhgW*|{uLGwtYEH?Oqv~q5UoY+@IqBnk%OOh;ls^ z4(p3uarCHzvY6wFFgfvYNDuG$hko|DUb*+Xzf=7$pTGcQPcV8u0dxL(}@ zDHvpM!lp)7wSq@$P%!svUh_w1K@h}c`V4k!RmmBi1zt<>)ECA1ob@`B0img2^-O^6lU=>b>*Q~HxdMcOqk z)(RQ@uCmZM2t=ywnD~zHsQ| zZcyoW75vwFZs3m!>JqSA1Ir_*c6NT@P@GeOh#=_U*zO4wKf%PSce$pqDhX0A0CRgqs2b6$uP0aEA0 zawHWtO<@?r#`-(y#OEfkaL9h_D^DHrV4viskBxhj4nbCk?}l`Ql5uL^)LgXEzj%Iz zzG{WnT2(4Sciu-624?lMdG~mZsNX0MtV+0)q+0*@>GWeo|LA=HT+(}zK$&<+h7Ou3 zd=p;+EjO1YL5~!R#i~DOCDT3Vlf*A7tftjlHy?)MvLW#p?zx*iKmoi4OWLEQ$KT*E z)PXoRmV`L0Df+~J1h?vPp^kkp0$f4&>`s21sA=fNuXfvwyU+v1JRur~w(l}(@!LC#X9nr_WUmCj{r+=kGhuly{F*QaygqT_GyDEC@5Hh52wxEz ziDr?e?8~(lM%uOE()3VjCzuL_C#*h>AHHCsv-g5vhb@x z&R%&MT&wC5_ftFnf|Hz?{eD89zZj0O(fD&&dV=DiJu(4>D<^`5}HY#7QQv= z$G41%jWpTeLI0B01FO38PnFlvKP@+x?GO~E6Vrw3@$C*+}RW7QklC3H|R&mJ5M0-oto~ZZ6*sWu?i?)+{_GS+r za`k!`!=&v%uL4Dtx#+Faruy~Sv2pkdco;W;C9gr!;;t0suYRHAlKCccrjR^UH+w%9 zJ73UUWc^5R^39sJ9T8`SASM_iEwi zp3AVG#iR6b@WrSUjr6PtXr{n460~B7daK;s6xcFf8`u&=lJHaCe+KtC8lYmRh4bIg zOm>M)c9~ji@w)VpPWXJW`4aJ{{y-~bDj!nQX9BCo zZ;6$ZdiB?S62CW?@#xroST#x~^b?4m#EP+RIqkk;P5k9iI#Rpsgpjk4i@cs6hAX6n z(vFhl7({idqT)%hBy|o3^bo@9&flHCO61Ho!l@Ip`v&`Vb7^Pa{%`!2nwg>vf;ZdD z3Ibtp9c-ikG@+em;;>}mKHt__Ud(9aZ0&I28e<*EAjKQUw)+*0)|R>?ARKzqBE;M!Al1OMjMyl)eWRUt4*Hlqp1!!nrb#rs*n^&6`PN^AZZrV| zO~dGg`<@Pe6uBQ8XPL#PXIu~O{>|_DS{mPLaJ~#iXY8!u5%R!wg)~tbX%7;C;ee9( z>pT}F(501eCNQkh_;)Qy%<~v+;bRCFK*j!H6wt%?c z51=D+2CeFa_vLC@HfrZpGY2c%>ZsoheWKOc>Y~Llf`t)|`Cjgp;Q_(|eQ!_5Dnpdk z=XXNv^O3$cU2c1qpPm2*pw>517jKo;4TdzcqW59icPqU4K_%7FVw)kd++Z5)+!!rVVs ztZ@8V>dwodUE?AqZH$G^cO|4^;bA?!$ z45Q4Y>!cSqk(#irp3EyMM*D2x0NJ-qeua$kRNcmV8=Fjjc@wDv@CLW)HJ45pb_Vl( z`TP|_luq__&L-J^S7R77X!ks_JHG>g{u=kU#5h<`+omXjC~t$H7TlTqQ}ts#xc7a! z$9fiyxE)o4=)KsJy%~aL`t)^G+yr{RcdNhq8&3~6)8Vq&Gz9;Zi%X=8zNpKpoHu#a zKn_6TR|3WIJe5!0)Y7ic`mw3-nL!~X^xC{8V_`~Samr-vyo&sJ%C?93 zCV^lZPd+*HJdgOv1GAmkIY&7kKJ({eWa6xj%0nS2s##ZS1_<~5wnj7B z{#c32!pSjfVfs3Pt-h~&4Een)fG~QJm}J5>5IpA$=8e_rd=n&@td@iB#iD6>k-q`Y zqK!F{Tp=4Or|+96i<&4)WnGrxYEi`ZMnz)8d5lVyUfqyyao?gQ9US2(XiQPvRtA1A zXjLpApo2qs^-4qbMZL^rI~%;hC*@0sqwRkN_IZs^5J{4A3C; zRC4HlR$4+pAl`f!3}|*IF);jJlV0(^CQyQFOD}UiTd(p_7O&0iEwnmFLDl<>9m?@K@?5~i*3{=;F_CdHnu>;DBee$%_wHmN}Y%s7e1292y#b0-DyS0j(vbI#IG6(YI znb&hPWct}PxC&(Tw>d<+%|k@1%~p`I?HVntU)9|X;vfJqxFu!?=}WSNXg}7^)EU8Y z=m`}@w(kEy%c9_$H_tNpMqbPOcgVm3|7-qJ^Z%Lu)C8^-^jBPmx`(FP0N06^?|wWo z!x;SJ$f5bLfJ)f0=SZ{=+vhVCZ_$_R+kDX{5@>_8G$ZaB9mazne{IY2+zqd=!QuBf zLqP2QcT)B;R(+yhJ6-F$TBHI$w_!?E4Z4|YUGP(|GzCUo%>1)rSwueO9J77((^rhN#q`CDa$>7)U~kPW)d>(@kj7VG@D)!VUK0bciHZx>tWyR~Z zBtb11KrHTisNn9-iv(Eje{V!jPjArV!^Nc1x$w)=Fc4Vpxs!Gn-SS>V+ohV>$6?zp zxNJC&(=_L+uVN$tElY*RZxsum^vulYT~T;lwGI=2-=Rik2NZO*Bp2N>LlX6}YoawaKAiO0vs zcifIjusry(tvkak1WP5QIlxSo(7b;B1enh1e0v-QgPC_cW7to{Mv;EFwo3dqn977@ z_r)Cp9JpU)N>DU`9(Q){;pyqV7B4JEh*p94geEKF?JX#U*stO405D;9>pK`}ePiQh zfvINJS>$Ypzv}IZTtsxV@8`LK3a1Dp9`{uRPQz+6y^^Ut__$-;^+tx-+vRrTOLg9S z2%EZO&1_tHI%&nbkJX%AD{tw5L%<>B+^&Bswzx$p6(mB+Wwd_rRa%x14lS;<9i}A3 z2VzfAQ4yV!G*+`ODl&3qGs{g+LF6m70FZ%q*C(Gu+;cViW~QfetE*EY!|4@Owcflr z@Px!u3FDufoW$(4xlhAvhcnUZ<`9A62L}hNTwFFQ0A#&OwV|-BnJr8jw-@*%>iGgq zR?GS^o^!B&Xh^})@yARoI}mrPtE(evZa{F~cdxu{cNRhrBPAy%Lo_w@_kBGvwj#+) zx4qT12_wLQ;UDM~LoDm(wav_qCfr3uMZdl;u%4ZreTIW`cIyci_rb-F4v&lgW?E!A z^;yoV!Po>8d4ukeOX}!s1Q#2d1-%7el<-w%R#uCYMhX)X3k!maHPBd9HD=}>n#-Eo z*qHg>c$E~U+WL9~F((j8FOjVTC|ex$eplm`>FrHDyScvhtB_YEM9W6`@-ce!5WV4> zz|7tFWZ>ON-^uk_VqRU{)WIIm`>RGYvs(I7{N8QHzpUYb<-!1LBPEO?#nl<5Dh4jbHJuojna(+-Lt-Ew!TFDW;lpt^GZ! z)D*dBAu%xivtEg!oLtz9v{Ic>JbXPeB$_;7p%Y7E! zvHUu|!?oUOFr6!=ToIW?FZ{&j1)53FL#3yX=D#xk3sG3MnoDRUi-1`E7?nS!1LyV?AFC+I{o? z?s=zUHuXcpwBUPn_485SUr8}Bv4HajqcTzl2M21Tfg}dzBpZrc*3|{i(a2%jvm0*# zO1vRk{-vNZ_2$z7jyK#2ardnZm&QBjsdpLjW`W#y7YIfYp# zr>0g~5r=yxo!BGdR*rR)WN*Ac-!-%G!#m!pPW+yK4{uvCOSb_cw9A@rrL2CbGedVClsKil(Te%8u~0_k1oOv-dsc34B|}3B0ZR2m1Y86EFPT{oC?_X}lUz3JOa;Bm zEb|eUMdb1B4B&vx3|;7#3Wu6mEghYR++2ojuf`EW!FVM`Aghy7QvUc0AI#L>si~^M zryQTi{pU|19UYx^Aki(Wytsu6CU)-h=Btd#`iF-V?d@5ev;nTn1u-g_Z;$1$vay{E zQokCZ{dLynB18pPZDeF*&iA>}?O*GNL$HmZRKd}}4aJwb{8Z`GHq>s>P6=V*3}g~( zKu9U5OCcFlnj?FOeVy?P3#2m5WtA&{%I1yB0FRO>xxxwWn?%nn)q>rN4|`~`?fbJe zIWv0{TEOy-shWUZiH^29S?#GbDnqsX@qEj%4+sTwSpd@L&~%lQFt}UJ{~&{)BjU-i zq+Gk>2D||E(_*Bt#sK^1=}%@16!2juL{SA81n`EnohXpdWjA?Kj#YjqmT}~~xn~+i zmD(j!4VQCH$=|+>0Q{*9pZXq3lpWE&1bDb7UnP|sunT*=63~M< zWZd0Abl>xZsA8#0#4Cm0oNcXr@f@*`!He)10&R8tJ10Uz284&xOnDDWc2L+D;MO~f z!Rm!smBUgRJ~%m)*xI5lfGc|^r@a%;kl>qH)EIijj#7Xq z_omCb%HK5ITElHLfgH0Jpd9)lO60jY^j@#TMus<^b)d^q4YS$AOJ#G> zZ~)u^C{AWnUTr=bk+-#dAtxs%HUqLg_&^R_ulCu{(b5Y2^9TAdEv>_v&szhwg-f~I z9n;T3#w<%H*40b#7C=}aDFK-Ta1X#!%050mTTH>Y%zzgYQ3qGv!pq9=1O)}{U0e>R z2h4@Cnj}K<#_f6hj?Ez{XyF~90CXuRDA4iooqlZkfXf05tBBNl*NXq7&~It77a5=G hfhR>C_U94xVt+}*bDZCohX_F+3J^p>RLx`gB-6s@!`#B)+1PDd z?LkKvhnSdD3{_5mB$kAz@&{$5a5|F*`tuxEFgQG%4(Z1SV!ZG`q-iv1WSZ|Wl7~eO zag4+EjpG8$p=jc@$m!Kfw!G{3KBr>}NiFPmO?Ru#DqC_oJaDu4ww zhM@lfRlCee%11S76xmFsa9HOTEtPzH|JgjEnP)KSWViPUF>GO(GU~ zh^3mdCh5_^aK;QX38Sv)h&>{PhDEXFLv1u5Hq_+g{R7FuB_gOpy)j8o*=fcZ z@dl6W?ePj=uRp$V>fP5pS|-r>_;jUUi7H{fFuS!r+V%7MdaNma5g6#^rNHg-YbkBl zD1s6kam#dVt-Sgh8df{`?@$IS2~QN<#c6(XFee-Ui%h7@sdPXsSFDz_gD1^AZ!i1I z8au-Xls~8sY5Z$XyYW9#T{zcnuk=GMpNA6+`N?s@@Z>c95R#yPB_=t&NbkoLt9 ztEjYNQ`+ucRY5Hci^kH$L}$(PLU9%!Sd>S6f@W;`BGo~_zaF%`rrT8CcD%>f&tGzR z{3EsALj){xCKB@xbK0;m`Irrh`!R;VX;#)Z>+C3Y@#wCxWq(C*$NRk)PpJR=ThMPZ zWD49IcO@dVvMOJX+zU3hq287W384$=eerK>53(_RSA?>tX_eH$dHwIx-`N=%GZ2)4 z8b4~PpRb1CTV(>q>?C|%imbM}t&~i3M`_5Pkb1|*c<`;9zvt(u^ z8D+5P;#hSUH|8UU8i9$!-NE2hD(>rk)RgSQL@UO(nE6 z`s`m>rY&}+rsly4I5ifKeCH$0|Jzb`nXsF(cz0a$Tf)gg%6IZYyxd+VKa&fOt~Di_4#F0vE-o=CN5#; zXBCg0zZknnoDA+|I*KSbfK!>6bF`{_K1ByhtUItv9pd2VRJpD3NsmAV7l={7Pt(eo z7qzBTLa?^#?#tqn+gw&jTKQu0SW08^$LKYZ1JyuBjw(rh<+n8Iek~J$(#p;G3v_Y> z*bGPoBlK0}cC1#wWpEeut{K$!9thT~SMybFMks*Qg0Wst5MDn^ zYz|RSg(4{vob*0h+{DWDCq`Go`{Q=0{PXJXym^Z{Cp_#=wR9+lhF>?^lA?3CxD+&<`!J)B$Sb{Jojt+hdLNDjHGJ{)Q;ecF)M9IPU zubZbyD_E(TV)A$wj-?BJ% zuR7VbcT<_Vp`BXOPa8B9n6WFvUW8JXF>goitpR=x;iR{;Z(h5jsSa%TTK^q2&TXy~ z(ss_#eO0#;v^R;JJQYK8p60?mw zf%p5YPpd{ChbuE@2KXVnjE+8Ryug-z_2Xt9adck;AE7kDe&p7?p(_5d3YOUEF>&;y z+5&J6z5+>IS=?t2Ato(fC!0vQC{o{ z-pp>MRVL;*LedB<^ohiwVKKC>-G@!}uPR7OVu1As>o>axw1LwViq8F31#)brXzRbq zExwZ|UK32+d0?YPJlpMgO4+j+Gdar%Q*A$J&4EhK1*DoX2DN zmfMD;mI4V@3b!!a^#sB^)KZy7y<}3YMd$AXr$GUV`vV#x6x2jHz=$GFAXcKLZ%-3d zsd`~;XO~rqMcFs7*xL~XEO~|ED{>$_3=$=^6@z8#DIcz|lIIUF+NBI`&3}*O_WDM; z**J#dYqNABdhkV6nt#cqFUnPrRwg(mYtqm;^=5Dd7Wp(Feb(B{##wkoK1>Jbf#2x2JpOHX-5EG6|v6u^6gG`<>SX2@Z!6?&^G>+Ccl0P?~`)3 zL_Q}az;n71>SN4x#sxT`c&NZt(5m8*>&>ij4298`tuo|OCsKXGJSJ3}&}tT`D9pxU zpEKYnEHRm&QU!WAs1{5djZJ+6Nrvg>jpQ_q(Z$~-`;?$(0vq|(s)(jH4W}3(_Ro&b zTZ<23NnOV`GC_ywTW7E3uTOyAV==G4=IUeP)qQ@+)V!ZTGi}Ff=Pmh3 z;wkYN_s2EC`PZ^lTeYh8*4sFN2tJZsK}+@Nx7Jv8!h$WZr5vMf__yP-$DmWWP!Dt; z#@8=(y1Y7a%YtUYZ!|Aa%Ow9RAc~iP7KWP3D zHq>7)VxfbwXOaB+QZo^CM?Ujv(qd3k=rOpNmS69=`{%}s$?G;a!0E(4^bD0CeL?Ns zrvH2FZ}+8K1%=^pTe|yj6uR$NC^(_*>CYNKGMczTT z+j2XjhuCv(v-3T(P?)31-fDB>_<&N?TuNYm*!4W3=4ww;s9Vn33=;^)Y+S!T8DSnk zVUr95A&rVppz1?4%GvPo=Y0VeF8NvYKA%i9p_WiSCC~k)5#j5KQK5u>PVU3u zpD^3rmf{@J{oh$*i&SgbZ6p6qtyM%zC(=wrr)ZZ|q?7T{YLM6^rX#wE)#A1{%Q*5ZPXTI`1KSHj(2bl=FuhNOw^G|Bk1ZZu0RE?>UiMV}Lz zSDYkT5RK39A>+Xd5%Ia1Ypd{y*QrBvlLubRbq?3nibayUKM zTO<`h?fp2AC-EL=@Uk7o?prOrm0dn_2DH#jw9-s$Z5(SkjVYadkIg$La8d*2B5L+w z5rVP?&cef5|Dy@o<~s;##^eB|iTQr-O)DaERzk$nVPpH%+%Hy;@agm3J>3e-#~3_U9tg(z_=xHtlZpM3x6SuN==9aoFff< zmUg%+1X7F+BVU*5!X|)Fhq46Hq(}XU^5X4i7hlr(BY6|Q$)2)Ken$4xAAo81_ACk} zHEB`BM~^mU7#k6Fl#VN(o`2SNc@~*7_Cg?{?6bHaLs3Y_eMQRTNiR~%pSRpKIM$<@~k|=u*j=PP>c$ z9HsRy3bCP}ZJ|%j^Z#-nJoD+_%ZY9-(LBv7OLHXjHHwnB4tGf3RySS+){Tsnnps?P zXxy47uwrDmfp+~}8Af(O=4MTEgvsugrdI6!1Py(_Zc=+U{SE$Ue2fhV;mDQG1uJjS6DpH$nB z7G+)9w7Dq&6LHq6v)c`Q#pi9UG3 zLxCOa6p2i#6%{YPZO?Mk@)3Kw1gcmYc`l58a-DvHk9;xgQK?%1(p7qRV^+yivJegf zl8?}1YGq|T2!lCMr{cJ6`V1Je@}nGxU?PN>AFVWFWGIAlUW)m|I@^wm$w&PQNpTSv zUbi=Y2@vtkAx5B4;PQ)AY92E@U5Lz4dWA7W@eiKF#(a(sI_tbTA;9>~U3GH)qIEL- zr+fUY+L?Tg23wHZO@Es^h;&x{bP1P=gH@%)U^pI92%qyLM}U%u6z5je|+{; zJ5c(ovIQR)cAOb^q$d5r8-mkymyN?~)Ba{T7-h2>H1DP1(imAyvC*HEw^a=O>BnQ{{K^Hqa zzH-kfJL#MUDN2rgSc&(?l`b5#Vzp{YK7K6R5=pP&&+gC0znKjZjX$3JzLL})bAfW@ z_+z09sO(}P-IG50vS7`Gg|$(A!#R~+0p!lKq1BH=SfZ0#i=N_?QAuTxei4Gkq~k}( zBfZm6^!GO3&<-qi?!~F3lJfF+-Rz$+v>xEg3#)F-R*!c_;VLa_dhNT*gS3aWT)lD0 zkL{Cl|AJ_!>Tr2!51lzpRZH2sdt3O!e9>{bE9nx~!5Dtp^@%Ol(Ui8%=LPtA*#{~l zL=DPUnty86>ECipqIqWi5rqC-&1#}L;GFz93i)C)?HALu>0z@djN<=${pdor^+TIT zQQ5j~O?C@88s)6tm-XDujSRJk-=cwC(#kybY722PLBB6<%GDRrBT_$oLX6ZU;Wzy1 zbl$)gb^TP^F?guZXxKlvWz8@_FA+zBYH{II*5Y2rdJE*yzuVl{Y0y!K znazblk%ENnrmQdCs!ffmN2=tb-EdZXtv|9$G~C^NR6Da#Isb}vZnj*(!BRv7BPN;@ zMP8L|&(+j~)~F#uFAIM5ilaU>!>+lDqg6;jUq0+%Kch4$jkM7FwNh}+fQG8^d}8sFlRp(o#zT>lP#oBS+etr`+9M{Ne^H&a=A($2&3S5BKYdJ^@08HQo$Q+?ZY5&= zy_O26+WT?<5C?B;o#Z20SchUjy!4k!O#5Hk znIqbFX^=(iNNqVTJ~1)Qm2Y;q2g$c7*zu2v z04nr3lOotJ=`pae`ec8JOE8~xV`Ac|crE4`i5G#$0Z)a&} zNswrf1mvX+9%GrN$2KIUgPi=^atQBS(Z6M%*}vR*Xiu%K?<^p1C+-?9WSe(L$YS|N z=K^?>;^jUAmsqHPi{_Hz7`y57sR3RSL52~GCgHpy8UZ(}(W)IYKO_IBfFAbK^K1-< z2W^1wbozHv$p?($AXtCa4e?3jjei{|KZk$>^-_H5+OB&Xm}0?59)THCU`yDZtEMm1 zXOq*Q!Uq%ZBD3?Wlp3g#Qb@QQ`oFNH1U@_m?D!Kip`qi4E}zU;u{Jm|8r*SUe-zA> z`Vp*wpu#E+z#wILcGt%7xzKp}?6&(ny($6egOFL&jxIb>-1W}G z22$962lei5IUMlV!{YgiZFtlrTgkWLYAy`jpaf6Sdd`1dN`;W>SxNQspk;sXVCHz= zh0^<|r{2BJIe5^^8>`5|{hr;={wIJfj;im>SonCZo$tfo$Bi_`>0cij*Yh}g4SlUk z_SJ3u)bUEW^(jfyi8xna8}mkZhfG#!wVJ*{A}`O=?*Q** zJ&~9`ix9&jWI-`$GfSY|4=qLm;`$ZJ4@p)PeDZV_Sk^mtKS9=!~@4fryq9a#c=!`A+Hgd|6gD8py zf&w3SOAUV*sRFHEdO->i!w7>j#eEka9z&Zm8mOC(4S))K`lO}=I0ss$W(_*}N=g)o zGDnApM8w39xC|~k9a~%5Csy%q-)@r_b^kfHpf6Vre@$aC?lJu(IIt7Qd8DKQxs{H@ zlfvUn_HbMUknGXtlG?&>gHa@G_~kc|ka-9MGW^^3X=HTNjW5%ns(5hH|M_At5CL_$ zX{xI8b=ipJhXj|*oAbeR;Z(kql!Sx`yBiJE(Q&-!J9%>H=*xp28HxL@lo`=yeGHt? zMgL56L~%E_n=C=^%d0D15{&3=B+7P98f-`$ylBdcc!$BS6)tBLDRf{`a7J$| z;mS_(`8Ee9O0v%D#=1o>dOL;&XdoBg2qmOcC=7c>0Yy`vxvNzMlrwmTiZL&A3(ZZm?4!q|@9p06hCBUR84gWB4j?k~2bqDh7~9T{v_&5dic0CV|1 zUK6sK1gn8xHp9qKP*FVvvS|HB_pTJP1PF%C#>^)7Jng6Ebj!wZ) z5si^xHpfkpDdfu>?}_L)xp;oKimkw7qC2wb3V;bhM6W&fN|n=4R)(>gy(qw@K`I!< zWmM-j1oxyU>!gU%*~!`2RqUt$i3%ABsd~pK;E;ac7x{#_?$^7<`uYzaK8O&z2|$BU zf*fQ*rphRSP5PD5pb6LWT3>RNv=*;9v&PJ;^NNH!-0b*3n1|f2QMsWM{yA3 z#R#q=Z`ZFYOc`W~+UHjX<0rcmTC6JKo&qrow)n>g_Yler1O==e7-a1N4 zN4~O&vNybIWzi;~Lh50S43946plkTZn8~F+hG$21Z50h>)*}Ge?n6enu{chl(zo1# z>4@z6coPfg`(r$_V#w;-cV6lw*rHZiLH02_O1-@5Ij~k9OZ0`go4{71v(fG zPEKuST2ZXr+}yva`33q1o^sBh{xffPDAIt;cKK0=+u zCu=J!3{)k$AQ1M$e_DX2#$7B(oF3C%BR?W6EG$DTN~kg3A$c`BwE3@|!0-81Dl1{v z;he+Jn}%;E%jf5gKP9&2V6cDEu&_7+g@OUE%;mFAvkT#%6I2c5&usSq7%7)Wl43qk zSLzmt5<3=1wS0{HD-+aC2G=usdot3%AQjQCQ)3bmg0E{s0pE5(ssaXpc%D#IZ>mvF zl?LTMV-0Z?LrXgC@hY+J7Pg1a3BRpfdspJOp7di$3=E^Qz(kNUp%qHqj$N-Jmq#2 zS2peGS1kq&NlC=RLRx9-^{>jxu_3+`&E-6$YH_Q8OGb~&u&2ec-vl2U>dps`?jRx} z>P%NJ+I|DJDtPofduBnwzhcSeaWuZZzPYJ<1!_IPl-Eg5c<~ygEoRC0T5rm(mz#R( zqUnm>jURcKuzxZ#+=&MDcC zmp&?jPg7x1c9ucL1iO2C7RES05FZ>IFfuYWayziGv3+Gr-I>=^Qo`a;4NE|2kj zDv-QU{GH?#bWO?0h+hpl+P4*d6@Fn85n1P|v7l``^5{0&#>x~nDx;n}W+x|GJQzY~ zu+wQd5X+epd76Zt=Vx!6JMyB=!hbNfuUOFedQ=}){{d6uMRas@FxuV`gr>?Bp%Zh_ zCCU)<%$pKuXlOLL>?w?zCdASZl8~UK4v=VpNis6W2rBjBEE-xizULsjKAbJ%$OMg; z-w+Di_DZFSSBXp~)_*m0(& zrZz5|?k=Qi`J4za&|IeOXvV66sdk&2h|3PXiv6yv&!#d3m`rDx1Wx)al=;0=30??v8UXH4u+0kjw|n zf7X6Ae)_i!=+QLA4N~NaVBioPTlBOlnsBtrG%5}a^2d~Srl-2Z;6)*k=*&ozOyhR5 zqX+UPZhYEArl6U8Qq+k4IGI9DPEP&h4%w4N8VK8`PkXaa)&Bneo0}W2j;z7$iw7TB z9bUI673kQ+grcINu;WJ8T8DRfdU{(dZ)CXF^W`YB3l9lP*KzP^*xl*b(wW@L1930z zms{#w%!U>tYsY0kxAA#}K~vpP-7~7?+mZ_@YFKZ}-I_PhhP1su3Vw76>W+#3Z|mK@ zpi7o?aB%SVf2*!$vDOIZ{hpqjJhdtzljqM2}W4&j!C#Z4rl z*7~vO2)@m$htVX)psDr<83am>PnQ?E{CLFuZ$)CxjM}v{LPBe=L0{{ZfR4#R>dTk! z3K<$;pbd(iKD$eJDW|?pa%BLJn#FQcQKU_t*Tb@^kF>N)6%cVdwiH&=0id4}{WQ*7 zUhKhXvjgysgqP)j+AG?``{}aXin}U2sZ`Ve>gQvajtulS2?+_neU+AiQGGjYt=yej zR*SR!y!w8;&%v=cRY!^gSW3x6fV8*eD&;@aN*R5~#$P&8TN1TX$A|UQp96C?7x8eAlkjjmki#@JiDz+%p$5r$W z{*CPA?*Um&*{Y-Gi*d)LZZuYSR1`WUW~cMc2oTMQGKGu_bf51Y7Z(>>T=pKHo>Y%M z<=}*Mq1f*@cWiC4E2?}BH|d|!MReHzw}tJt1cX-*!uManLcyjT| z*@&V9lZoQknt(beK?RYOZ9A%Li%}O3fQzQY4W0y)FhJop**@mpG9+4%MLunv@_{Cm zPFb?C|F^j~I|Bi-WZT&ANC@62oU*(=KbKZePyiY?d3gjw&*UT&G3lZ*3sYOtTd2aIulC(#PA0x29x6s4sJxb5}i`rb_S2${` zF^zxs=k3shvTxJFN_s8Mg*iDn)z#Ixx!-9ZLERz^e!($QmIdTQp~eXyI{g|g2Cj5Q z?4NP*@%edqz)z^-9R+602x-m!w2fxs0uRq%M$Z zTYp!4#Xq50MYfIXD44jq-h^mB3cD3=_<{iR-<)Tj>vK4tiOeJ%@~Q8BFCgmbeF9|| zf`1E;x9SxqYG3=CE&>`aOGQ&w;w>#WCJJ1KN__n!?FDqpJoqL#n9tyIEmjOsc{nJ~8 z&3A#PN85Pfgf{kpp8WpO6g0-Kdg_{RZ?QyQVIm#o?C2upKKx^PYUCFaK^!R z@H{`&6owRL%$UZ_%)^ht#yv4#Sg-bb0rq!&5!^=xx-M4(*fDqK`KhpcU#W9l)fKm2 z^*P7@_F9DLANT6lxGB&M?gG>i`4xqK0o@}{D8Ryc?lmL3)u_z^B&NCf1{`<`U%j}_%nOb3J%*avIKXeHqTbg z;e<;3d*?}9kcQ%67jSamOb`xG1qg&ybZ9b~EW-Q5gOKqVwZ;@|SB zV)$I)b?VZ4_D}ERYrFLo&=uMM<>BLb0>jJTrQzld%r|hNexY#D!kZL%^c{I`M(&}N z!B5=}jL|umo#17^+zae>;jYtyxWeYv%vcg0NfN-+tf98}=iw7u9V>T?MejySGcXI% zWIRFwd?984a3NtR&fo1s#KcbvFMtpS?ecIv!aZaE zyrOJ(uvCIfZP^+u{*FT5U^B4^S&faYQF!7w8!v!VCt_MeAOOsz~G-n7M9N8|Xd zb_7fe3Lh>1mDhYJoO+n1o__q2)!N1ymCT0=^B(%^;FS3Se(iVZpJ7&H3YD4P=-yns zuB44sV|~|N09sVm#KAlZ2|uKV7{Lqa%6nIMd|i6*W@{s3Q%L-z5f<=?WJf_J4ujL9 zSfneX0Qo{^nQx&j-*-}nJ_RCtztww(goSY(4f8FOH8J7e`!~ydaSY9}yxnPQf-N{f z#>D%1IUSOWm(JkA`k=P_Y9asaRn&;Adk3-t7&xzakT=~uj8KGOB{by&3zG!aKdqbp zvB_^@NU72@$qP-+ju;H`u!210*#s=(qDRywr~n8ISoFV94^W%LISNYkN+a$g!;MXO z(r@DKR_@I$2z_nMhv_1Pd{Nx*WPUSqUz*T~AD=1%?yi&Sf|GAB;ycUoUMO7GMS&RA z(ZO8a)E4LOf!h2%Y&uiz%_@)g4xBixqT&Xgs9Zs@Wve@xz z3h)Vg#B6Z_TVY<9f!ye;P_?wEL&h)M;(su2h&DLfWN+jYZsbDU(X^z%0IfaK9S+ok zpYek@G&NO+iY*4Ly!)^Ad$@3VcuDp|3Jwt&03VgrE>*CySM$F zr?cS34V*r~{LJPKux@toVqPb8mxLBI(Mrwsj0baQD~W;dBm#7@Fobo&*)bYNGqDBh z^8$U6o*&#StgANoh4w}AacyT+J(G;j%vzerw}hj_p#AyrX5JAGA~8A&r3_t6Y7PjMwou7hA>M=nk#JJ-r7l%dQgN3B9+ zEwiDz5^?O@>qi%pM!G3#vNQyv0Wzn){Wy)U{yVvfsNAgiWD{uK9)8y|jcAvc0cs>c z!1+3ppI9?RFezQD-UO>YG2}K~z=5Q95COWds=2kOzo2jeEvBIT@14KBEvo2aClqgK(#2WZ@#Z80+Cje+QjSzQLI;L}g z6a+T`|DR0CYyU&5TwxZlo&dsWf}=ZtXjPG?{olWaA}MRtN>TNqsrInLKta3OL`{$~Mk_#6fdPR4-82Y#O{EjBF#3ZM5qQIc zJtMAAQ9k=d=@G}Z+7Wo)BV{vvE0ghK#e9J;_0tJg^gUp-W=Km>-W=LYR;4g1=6|=g zcX2MYe`y6MIRRmGmsV%2>x0QQEx~$_8tAI;NB%z62A`O}au)LcucKsvqSdGI6;=QX zSR|P_+aH%G3S2-c3_CX>@2HM@dakSAh-}000pONklSD%0FTRv=#wWdFvAmkK0q886;Ol#vc!OY zfv_7e8nSQQq?2@2SJ!=ikS@EDbkf;7UH5xVPP)6QYq|GRzpZZFCX`Z&M!l2&0k&TG z2LOOliaQf-$A64i3_=K)m^}KjMGPt?jVPtasmTRTRCF-Hn$m24J#{G|1j!)-;ZkIEVX3p6wIq{9gb)-rl&A(r zp$dkF4TG2C(}D#6!$Jo`QY0_1%2-P>HB-4*slIk8G6q@_n|U@3SOO($ph;jyI zM&IBcSi{1Gzzs!xUoF{SlkBWjnE(Jq4W*j+YIxXC|G)x(TVno2eQn?bd{+E9i?f(u zlWefLZ0dVARWKxMu>WA)8aGLQ6K5T+%w};Gi?8r$(LW)OUjp^<{l@uRzeTXpBSvZB ztb)3qSj)2b3g1>9)5KR(<1^a_){xLa;JhOL2DcN(t8+oY+@YRTSbs$mUtJeIvVC9y zz`cogfl>-tkr8c-!qMs+5HNSBOE(CLoa*n<;CB834&gBst~L2_7CqdsB~CnXzm)@me5v;d89HMFP?cB zPPYg`2;$ALm^*M5158JMWrnr+AL6&kvk`5K^4n{1ZEg5+T>)H*3j)W3Cmgn3*}m_BH>k9qqud1y*v~w@R1@Y!MuP<|0l)HpM6-3tRN5@Ynb(Vt=f*O|% zzd1M;m+K1J-N#&ecf98Mmmw=MD5Z!s_ruPK|I3Cg2a&aZ`u+ER&*X+X&qFStho4T=O{NK>u9N#lwv2sIjyF&o@uI78} zDg6e^Mm^Ut>qk53$2^d2`uYEG;c7mXk6y|~aItcO|5;&cbk8~b5GuuL0Ki%MIgAP) z$t+r5fd>?SFP>h7(Gh8wmo!@gQ)l|HQx?lj-I%oRE*OXaLI^0O;0T9rrtDzuvX5A# zvsce!;fX&q2I{f+f<0O~@)^XNbXvB%YQm;%H7%{%f49CDoos>!hvaYw@K*VgHRyuQ;h zmbYYoMaI;udo`o&DW!-qMq>Mfk61t-08l+}Yyqw`nY1|C7=>-OY`pfkUl;XHmx2Xc zd-VGDtXa+cv{eYhj)@;LxOx$)+21~hYNZIcVyO%?2!v>isy*gaV%?Z)1vlw$sO3sF$_ zv*t>1Q#X<<1F`Am_X4&LJfE3vb4M zj)@<`z!|#j7`W~{K|%=rHf}BMO`M_r;stPRHs)ey3GO^F2iaA}1JZhxIxA2iRr&n0 zxvmE6ML`I`gU1%&wF_@GgK9`+&`jnA08T~3vnQ9~ z-@bb?ARS^~a)h1>^H|k!+<9OQt~3;XYZ54hDwvgc7dDJr+at*n*ArI{0NfaT13sIu z8Ai^iu}%Uf;8NX>$lN~@ubh9AMcbW-3m?l_glA8_0B7R^#@?CwYhzx*Q>jn%*zs^Z zdGh^?(bzp{8y*?*fW~?e5D-G}Mafr~mNgr1{qSzP18F_$)$?!R*8O+m2gi?p8W%?? zg^4p^`}j>57oE|g$JC=>d92q%gMW)pZr*@MLxd*w;s^&9bz$3;oygoj6MsJaGDNq? z$nlY}zWu{{n3gpgpO@?gA=-Mmq)3=Q_@(}pqnvJk5wD(KgZ&kUpil)I;WPodLKQ?Cqwv-ZuOKNTQFB2D zt;K4|di^?fJPu@jiG?YT!s!;(_={4CrD@M^zTZu1 z;}Sx!rSLPHw4KJXy#K_MeKYWR$sQHZmO7y`(6trY@f_dnFdeEqil=jz;zUiZ=JP*} zcuI2tif$24q&$KHnO|T^?8IKPQUT!#Tatk@U~bYZ%uSkwb5}3I*re8SqUIzBArOcF z(Jdl1Y?wMvDj@{#{O~@1YFu{Q7<~iMBSs<7l7Ne~`FJk(PtCT~W}?wXTyoNO8gssV zs9oEXs0;ry?p=f%!kcxx$GsCmu%q}(ym8?j$kYvi@UZY^jBT9phf<13apO^&=RgQ= z!S7QZ!|Z`GyS*;er@``8T6n7F{qEAesFmvwB3K%&9FO7FVVZS+c9!gdQxUPdbT8`M z_1HIMCuGHqN~x;dGv#n?7;~A*sB+b`4=lF?iOP+?Qd)}B_Om!qa}rU;NGupOU$vB} zVZ$IPE|~ahw_CESPiR62BQav*xc|m=5orPI#se0trkdXWHY8dSu&?|Ob{6l#jWOC^ zow7-G)JQgj2q6%EI050riU>nEuArn-F<>{Ob+Acx%s4a;p?nBouKv@I88;EXiX7d4rxK4o1(vrJX2u z378T$u}>WD(9pr?XNti{Ybr*D56AV9>CMUt-R_ndSiKwssqauRwE<(2s^a}*ZiW_xuUArycgm0#|TFXig zAB7d^%i0(J5FmuN1YFIDS*(tnAjc!Z*#1DiEo;OR9^nRS`|YPr9)sh+#Nt2(783)D ziGjt$#K2-=Sz%vRgWK8cex$b~#l;$_wIqeIX<%A^)EWD^YeiBYvhX2D4oL)?a8-w3 zu?a|+E9{=Dh7d?4#i-u=cMu@}PF0o>`@|w}A)ySczAkrFC2BTp0B17y`mrdIh?x10 z>jEs6751oaX5|JZCI%K01B;1)#l*z2!d4?@swkj}()&k36$Vz&oM1vC`ak~{5T5US zu-uA&E=mmmyQR}hmMi%02nei`wVmue35oanF1?y{aGObBP!TjVG$1=W8@ai;a5|k> zvSf+6oKlJf3l`w&)vKL`DK#|}FTL~gD zAu=+u)B7zhE`}(IFquq2-^PMop<2IwJq{m#J`A4cRj8O4SRICnQi`QZm!i758hm4> z$e?0ibr>iQK+in$4C?Fadpyq3Tk@@o%DX3C+Prx)>gwux)Jl0n?RL9$1DS4IVSau- z_U_#ap@#?j@jQ=3ixwd@H5DAk>BjZy=3e3Y_3K-Osv}m!lR+x2+%8fS1z&&twI-f_ z(}Tqddax*^C@3gEadC00Kv7CzwOX-i)hfTY-LGFitXsDZFTC&qBuQ!!P)aFIojQdr zTehf{idn26DJ|^Tv&U!c$kWMhzWF9zfBkhtM@ReJ@g9f<4<3vS8#cgfZaxDaAp{)9 zVaJXgC@n2zlY#_G&<+5{k0195hN38cShsE+LPJA4>=HfSS+iyhEEY?P>x6`a;Egxl zV3UFbi^Zy{s)8trK9}Ed#~o^YtD|np6E{&x@!WIIX}0IRuk!M8*lacyV+Di7vfJ%G zVIhRzzWeS|%Mo2s(nv~5g2iHKwmm`!1VO;1OP5%T6%2AX9BP{1R936C%lAirDMfmE zdW)GN&+|Bc{ycMq!48(uX!Of1xm+&Qy*2%7T0S6zAR{BgX9=RHs7M!J>3%#`TW(1Z z1gu`Yxc6p z6HY}@7+AsN4sW{YrdFS_;W!T2+1XgUcri|$I^~!3@?N%e>sCDZ`WY@$vB(G-wb2s6%iH3JOqB zQGueOB8Z}>j<@m!VM9ZI16HnFiKL_?zcNk+Rv)oik|bgN{Q0fk#__I8d78x&hJ2DK1^QgcU1Rz-Tn~sKv0`?bx_+BMb%u8#WV+8DdIG z3Lbv=VThvGV};84`g*Khy;@gL=?W|lNST?Lc;JBt(9qD(BTy}WU%ng(2?@Hp#iP5~ zW;}7!nKNhb>Z`B9Y&Li4Glrh#bh%u3<&{?u7Z(RTO4rjEd19ZOoE(@;rcT2}DTTw~ zz|5I5F>l_yE*U_`!0Llm%cIMeFJsrPT{v>&2n+@T42Jgow64qLf-K9JFku4jx#u4A z@84hdzlx!2i`BGk^WAPY^78VKlaqtIygby`*Q-TO?{b6ZzsY36m@#9Jk&%J1W5;S{ zs?odHjDBsvX1Ckna=B1nUk~8gFdRV;V6|FR$h=3AGN>3>e#6yO>oTyIm^!Kd2dB8X U#Xwo9hyVZp07*qoM6N<$f{9Ole*gdg delta 7320 zcmV;J9B1R2Bbhmn7YeKh0ssI27J|Zlks&UB0x1SY-#vQ(030hxL_t(|+U;F;d=%C8 zzjxZ!lufcJ5J;y;O9(-VAqr3Q2l|5i5S1SSf+C25Jb?%D{{4b=xiovE@gpagGO9AWc}6vES0xBt4_-M}vHQ+}F~Nx* zwAz`oEl*psGiT4dZnMU8N^As0$&k7dJdE+ytgkoT-o}IAFVl|fFx;6F&guGp+KHl+ zl{b?JOpFeGQet6~ONp`cg>y4X*)lbyUKhJOMAg~(a|$4gO_@|^EZ+Cf*BaW(DiI|J zT_Xqc4lhqzV-$?9M@=8w`6(BYJG&-#Y|5l4kBH;}pIbGjQfTjjAU!F4fiVtMb)FFVTyD*O-K^?diG@uv{=Iee-J?Y(Bh+DAA6#cwNN)R8O1QES zC!P*fb=om-bNwGkEOZLLd6yTRC{F3`)3=H66N%`{X!`r~Ei@FDv*qa{LC^vVhTbi-%u!xg)Woi?45h=9_;Pae90Knmb zUnE278cD0TwJv>AsH*d)vCEqpKaufDQV9sY&;JS4o9K`rbc7SLFTIHnTHAM7GvFtR zjAS14iLK^qcI51-b43Y4M}!r)B|YbBWxmaWpV)7fMZe=O(=Jb2Q)nobf(mD~$qKBD zlr=3XwU}y@Ahbtz=I-^8Xc1Rglu3gmwr+FeZU( zC5^dUmWmLPlQIc^LOVo7DJx7>t)wx(>V6{;p`?^ZDh0M(G>muAUuSt*DniYy&SnUD zXh4E^L}H<>60GP@*a{%T7+)y8Okrw9CD>4)BdL2*#G}Nt-j1)s*rd zUH?1hB|&Hh1CnBwqk7K*HfIC&-931#^7eBf|7&?%GNg7epw64-m#=eoD=GQRsFy8k zEu_K=TjleNt$<)^#H1E%LoIIr5X5j$GNg8h0K$SRZyzr{6{G3hvMo`Nax3ORVqqg= z561X<`HiW67iJ5x}w9c|bELAw_p~_U{YA)S=%D+-O}1t(k?Kz$F|VNn^ThqY)x7 z`BTCkk|3Lie1(SM$!A}!FjaI?cH9*AshpIVSN%DEl>hUJo1gL^9C>(OQ`>-_3XZ;g z{Uar*_%412g(<6DmTL8;j35XrZZk#%Y61$1ihK9^$3GSvspSoed(0o%VK@MY$8T1q zFHZd+yXHe)s$-cfG=o*Zs^CYKd69a>n;ePYZW)f zrc642nEzu>ukHuqcZ&F7jLW$4MXB$PKQlGE=FY^h=PnLE9;J@hoA>?Y($uCfV}Q`V zuD-i1Ye#2g@XkS7ylJmSek{d>_rR;=4J*<}Q^`FNaBm6dqvx#PvB`up_#YQP3HscQTZ zQc!^~#;N6h%(yTIA=JaG`_uc?P*x8hu`1h}}c5cbq&I)X#DttrSDmf{)@wDy;q2pIj6&ZDt&Q9;E>GN){w~(Dl zR1@$kZmd0_ODSPX<%E1y^ap)?dfOF$GPxs!7~>r|d(M_zSlVl$f>dCPyLd#3PKl^B zjPdH24;g|9R(5C*JPQesYeQ9?4;LIgUX(JX)00mGJs#;1UZ=ZQ?M49bw_}ObrkW6C zr#HL5@kEEmko{?Ns!fv#Vf$#u3EXM@xp%LBTxF{AllgLjN%3c3nO1po<;`_}bGp77 zcb8oH9OStjMqmu5#XgufqJ%dRa(-8(o2U6 zjvmZA3Olo)~LrV89rYh_um=V!#;VH8CId^m>56gxG7p%m4AS^vxw~Nhf9SM}3xr zsyaKLMKCv&zyyOChCmU2>aYlP*nMxO6+vij;@3I5wq@<8;3^4Bn1CA-oH#LToCEn5 z=WiGg2uMH(g{#BFg?MjPrY}!hn_7O|i}qUFbN*w2!|s`)rk{0aio87K@+SVKzwqYO zC9J?9gcPJ=c;L{v-CjekUfJJ{CLt`EP&py1Fjan*v1wQCJ|(GtOzb#ve)rd#PP5G> zlncpMY}CDcZnloqV}u!k86G(FpWS95ixqQQ#&Ja;2H^YhzF&T0t&C(I@a(#&|EK2R zu+7Ivt|G_BX>0f2{XtaO!`g@5>M@rl==wThdG5(m&$7Ta=S>QNAiiM;dPBcep{kJP zoUm;JEG%J5r(bw~Bd<0eArvSNSQY(VM`Z`}*NE7xuZv}uTO=0)0JFQk@??kqA^&_j zseRz*6ojH`cHcR$?8X`y$(Vqj6*cYI&ZGZy4gR8_+}gX_vUVIU_=Oc%nxJie%B%6z zhK(noG2K}m{A7p63DiztbD1Ii;`W!DGLvac1t|hQiGDwSK;d8iH@7M?Hr(ENtmu@I zRG7fzC-V(gg*FuJ)gQ*rm;P=YZ-7`$v~5l#cr)JrdSFhw%@;ZY;loFt;6do&+3mA_ zD=dc=@x9(y@~_mTSIe%8fO#>Vv%9=vPAu6rhVk*oMpzBE2}B-A*lRnf+Re!40z+!4 zi(P}h)U*13F6ziS7EfI9_-x5V03b1`VZ4)KKMs-yV2tbRlG3=Cps@s+3GoynZEtgX zMUk=>zgoWbwTtr}4v2p~0q zMz){IJpbVCECc`(n5LYYnQth_xSjocNZtHwwW<1l&8zQRD7{2sNB0 z{%hNRxBJ&7Ar$dgc4N($;KVR>XA+V1ZR3)j3o5wEVF3v)ANCsZS)>kP2ZR{oIhWo# zqd%8#C|D8wt|hK3(vqURdkqbU7wZOGPCfz?3EIJwl{^5(h`=BrX&Z!D)j8)%E+T{y zJC3l#b#dwkzZ9J~Qh01=KzviUEdW3flvopg=48O`?s4IQD@zbU5$f;+?O?M;l5@U2 zTzHhk#HipW>{1N?0Bf{y))!p#8-y`p7E)!dr96hVE|XNl!kXLRQ&Og`*C z_@(d(_HoN(x|z@)jE3gBMF1)=DC*D&cVr>P_>0@yNK6LF0|VsRzXXUe?-d$~Ix2&I z?A8kamNU()Ni1ubJqeme8k_Vfs4e`h_1x`F!c@4& zf6G`mXC^C1xv$KpgCa;up|$q05CD!9pQH#X(xMVMMR(Q&rir1?wK4oKM(w({HORtB zu8I@5{(nAkb4GW*f~%C1GRqu|V+v)0Lx648#pozsEo#5XcgiXSF~&7~4JUAa(cZn# zz3HT{8G`QT6XTqF4+Kqzef6J~h%CU3;zEpZ1y^YlSQGGpihHZ$pE#iQa5sX&m*OYDtK62!L$hB+N^78U3Dk?^e8Z~_Qa1qY5w6yo$d(T`~ZdMNt@Ip6B!O z^32fc^?HRu;qX<;%F2)%RmH&Yd3RVQU7C-UTU@$CMvP7qU(~%@8;wOcA zE^uc|g6X3hpNM;_A%ya33oya~a(~yFO@vTIMaAsdvr9`$Ns>fTt(yx&`vq6uK9F~4 zd-g8zV+~)65UR5nt&_j-MhP8Q1N2;glpx0JQg@q{P=Rjq}8|9$7=exL=gaxo}QkPlEOGw&ti6n2!g2J zFaY3re&WQ5b(ZUL7em@X5x8T}=04s%#E-)Q681f`y^BYrOYp;g81LJ+uRea3dwm;? zMh_2<`?7iZ^y%^O@hr=lL9+ZdGc$AP(xv9Sj5`)y)Oq^Qg*mFrmoL{>sCa&Z!7yje zoQ)edjvjsgy{rBG{U=PAuwlc7-o1NUC0SNh*518)+i`XcH8R;W`RuE&|GvO{^Dm#D zy)J#DOFD9LaeX%+REQx2O;G7ow*i5YRLV(zxr~ryRcAZtivmDNNr_e0 zBQY`2NsnX3j2U$x6uZLRyLa8rkRpT>q@2Q3MpdS`_KuoTk(kIg6ghc4lgY$!TzzOM ziW)j}sKfY)fAsY9jEsz||7}f8O=V@J8$u`}F?q8xUDVEoeydhQzc za*Q#@al&7P=Xo)+YWcIMs3@bcZdC_C5VvmKa-BXorYI^XsBSGC!!Y%Ky8_0z9kZ}O2>biSB7};J#hk#= z1dTCXboIRprN7VaI%90`C?^N4M*P)=jz9V2lQtUa`X>?nQnwom5kj359VtxxRCw%O z94+&c5wg8^4m#sBG7Mu?n|i&zq0=^gIm>SW;QICJR)bD=^hgmxp{fuHQ{Uws60P6= z1pSSF6f+W4!bi^QBi^*)PYLm%ImMcE=G0s0*Ns`efo5Idb-s)WQJkX zYPCB;h!BeR8%Q7`Is02tO_3fEellN#(3Z?(hb`W?apTO>Lv_3oK@eumnpNMmsgHMm z9TR%-;>AsyHd!470YIbC$Yiqi>e1W}LhV3=(8=P{0^CcfywhtT0PMc|T?JR+@KxjD z;;cq*VpK4H{`{3ISDKSJ^@9z|$s?9!7cN}**=L_M6p)B@q;9&HB>;5sh~$7{FjLM| zXc%t*==3}&W;PuQIK)%uJr4nQcLBs@GkHa7Okl`B>mCVDqwu&C4N1VIpgvuk2z zjvxrJ;*=ywj4@5q4OvJ4;5csl`0?#~*8V=z*#KbR4d+WQi5At|mxmu8U2CeH8ZpU% zEr-lCFH@&ZCCMh-X#fC1LP8cUT!`AW5cg$_oRo|BS@z`w?O=q_t~>i2W1&c<4?g%n z%!f6}cDMi-L8PD^}mMs&9+xA@~Zc`=h6Q`b8aWm;~!7r8v6FZI|F!Al3eU8ycRI5s*TD*9% z>vy7}s8y?0d3t)fLxsOh5Jr3Tq%oDEKWlk#X4K0hCJnq{`3(mbz!3R=7Zen5!4;?Fja4`QclBC(i)=dr~)ZqdMWvuntiOFH(@6_a3ZvG=ltAkK0 z^92M(!Pw9Mwy4K{JV~3a+K^HNg%IMw)W`sa5V~4+{l#;y>5Zk5W?LPEIn{SCB9xS> z@t^IzdoaB!bK==)CV_XW___^1c&OlKiXa0TUHu6FW_Fn>fKX+sUUe%;T7je$K?nf9 z>V6{-@qlNy#^V7m+Uwsv{waX)pWOXF|5-yUNlQW~8pE-Frze%NWmSCj^vKEgogy9< zkkG@cJ3}x_uP+xskj!aILI?n?z>fa)`67dk18&g+^Hh||mh$*@u^;;|8UPq~=Ee5? zoO`Q=P|RArbbeL|TWS=H|LF2+V#g7-XXBPdzs(A4UTwkK*FKa`Y#9hK#(fWlds7+Go0aLqkB@%#%w(6dB(dX&NulE~!dtVpS8-L6-d!kvnQWSW z?Ep z6Lj0y!Lkh*(xCxEdU^HucaM2;Qr;ArHJ=k>{I}zy9zQXr$f%RT3>PR?5QOGmG&k|o zxN@$7!sK`HJ6yHoxBIdXW1Lx?y)Jzdpk^X}1ORM|TMGb2f!&d_N5a5a78V(Fe?R#w zLomq$J`YxQXtoRMjQSlpyQf7?Au%Dr;1mnjXKZE&MnHn)w&%?telp(~Q7;h)BZPj) z{~5LG1)7=`!XFEM0R*4-`-CRw7BB)BuSolJaq9ceo}MDjNjV51fpCO6tea<-mQvq; z$e`hJN|9e%ka%hw2b=_n-4JqMS`+gTYPquZE*_B|M=uqSP{x*zIR12vsYXJ@J_v^g zKBA^oE!hN&`e~v+@3-10upF2kKQX4tR3%wdJM>p#EcK?nT0@p3vpV~^(=WJ z{gXN<0;SxIO)S(f-mOGaVXBZ_gSHZXnBah$aCX|Iva6DtY7>O5sx`sN4tobDdr}@I z!8G&tf1WHpExD=pH>9miL4Q6MKB3sC2LvxiyzqY^PfN4JZ3#lN%)0o`OJ%7Dp$Lz# zO>t`_6x$etB4FR->|S$g1Bpo`sdz2wrI8&T0|XdJK(-MUiYWf3KfmD0+dS}pCNS-J z=qrDjR>G`RyEA|3To@yS`uoNn9I`Xcr*D`l#D~^MfVCwS)}doVA0=r66Frve(PUI* zO8B)s2+i{Ni7^v?o0g(GE5(Uz6~ZWwa0*k~vcHjN*j6E2+Vfw4;Eeu!rBsP(AygY` zs~7D#L^~+2w!lNGL~UChX&vu>6uJG1U2PjeN!s>UD9WMypZgw8SaWNG1YTQ(Ft_%u zno|8(a0E%SiER|ZNulEmyrG<{C}Yc|L9tCjNE38FpBRkr=eNI<@N1J0B7`0b9EK78 yA^)&>GcXClwj==1$GayhZ~}r-woHQX{{ax-0RWg*ZPNe%002ovPDHK)LSTX`@*8{r From 54d0f73e67135ec675953d608d0780dfbb130f5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 13 Nov 2010 23:32:56 -0400 Subject: [PATCH 0505/8313] update --- doc/logo.png | Bin 8699 -> 9092 bytes doc/logo_small.png | Bin 4506 -> 4713 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/logo.png b/doc/logo.png index f601656c61a4d215fcd2c05e939766b5b50d9afe..38d335a4514173cc21b1049c3de4b75055a502bf 100644 GIT binary patch literal 9092 zcmZX41yqz@(EfsSDBZO*64D^uDF`bFD4o(Eury1vbcb{b2$IrW(k|U05=wX1_wqac z|2f||-?G5o_r3SdotbBznR(BKYp5&YVo_j$KpKPgv17X-oQWu{`RI#=mY(9^!W6X+g6+iT)}iw(sKcUunC`jkwEF0WWYsqS7kMM^abqa z&*+%=8?)SjOXRNdx~{U0_VyMIt{~aZ7ACG1=5+4XuAk@>mDMzK0-lqCKy)BwIcY7A zh5cnu53T7niKELEJ>Ee^MzFVx%=?+(V6`}T{i>4a8^U`DqG}TTDmZbGjloV}-}jdh zs`k?QPO`Sdk#!|*7OEJk12V`+GD>vjU(9Q)>b+sGF=b3}twqAfn-#?A8QT^YpUdGF z_g}9sS6oo^QYbo4u;p(mF|etj%$Tse||Z0@++N=JR+ zZJbra8XyhZlK#>dn`~5ik;+MGfdSh&E0-+%hid4h%SzmkBgc=HJo$QK%>zj5y9Y1= z4gcIOI=K}t%c*ErQQH7st&6*9+;It89cUq@z*Lb*z~? zuB{ylh7M;+d9a*^x3e9bH5*eIx6NDLXvE`q?yCWcS-)^UCjDjp-L*9#(I&l%Eg^hi zn9Z3F-k2PEb)?>>Sk?Jw<{nz6+~zlReVff3`bH{=xkgdr_2-Z{xKGbQkK^xGibA=g zOIs5ck)ItJXdxo9&xRIXdX;nY(yeU$;**`Iw!T3diERF{xJt|FlPN9FkFp^|VtQIy zNhb^Sq%9VCd+C)brK7YzvDrocSeXvB-_yQY`@`8tf5V$0x;pETTjnVw;A%R%s+NHH z(YAe^obklDlG@-f<-YafvF0H-c~uu4O`OsEkXEkNUHRwlGtfjJ#O~lq&Rcu7<1zEv zyum{8L+mjQ!FA(|OS%V^bih<6WASC;#IT^1%(qfJw@@z-pcFxeY8&vFESBTR)wU&6)9V`i!!vB5d2YHr= zxOHa&nWI|KNp8G(9RCw9M9icGjYoj|$GLSsc$C>F6E^Z?KhZ(%y%O5VdB_0P z;BP>k^SFo(5}{tB#kLT)sv+oih|t4O$0O#y|_=7mH}04v6v*!r`jj7O)~+#A4le;p6z6D91*M>lWJSG4M5 zO)`f2aVeM25BnG#T zIl7;c%`llE!<%W>na{!|`QX(lb|+iTq0+dB89Dl!O!RO>@?ur0Rrn{RFfEgu59YpD zY@jr%bPJbj2hpGk!uUgxY0c>LUkhDj%1E6*s@Mcm|Nc7}v)6+qt1G0kjy``~ z$Q<&_}sXID>OoA4&4 zLXffL75}33LSyCIAS3+v%j(%c2$ss5&v}znPDe4PwWC-ac$Ku<5+hX#d0ju&5_#Bt zM81Zpg@6LAqCa5(vsGTuV1JL46?#n#8BbR#L}f0R(TF8qo8*3)7;Guc`OWhyJd}={ z7bU4w~ZY}p;H+UoC#Wbstl z2zZH*!XR8c`g+bVCn)Q@34Vs%-^^x^92VZkXM6>AxoIL;NXjrfw_F z&F!6uln9a0I4UQ&hVT5M;23J6<~6@!=T9T#A?%3yVs$xhsB_lzc4z2clVECw1s`+= zdJ4>qgXzE)&!Z~pUfj6f(snvRU-c-mNQC?v`~4a$;C96rd>b;a|971*=NOob5oy<@ z$nxO4V!^UdR&_Af;&DD8g1}_c#M__w8j+!@IpY;&l{$dFH?<$CCwN&JB9IG41r~`LZ^7s`* z3UdUXAX$9x*I3fGX&~AB)rGzU&6&DuEIW2tR)+_|BfG7PrSIIx`0R)fFXLZW3pJP( z3ephUC;v65z#|%nY3u%8xjkcbDIs#q#ubycdA0$Q+YmP&7?|Gg>QBzOOv5?Lhm-3H z0BHi3jh^CzO2&)z^&!Nu%e}f%O>k;$wlolmfaAqf-TJNXbQ`F#2_2kn+sPqv8{=p` zB_u)Sy{l#IQU#ibg12%|HuRkk^MR|IvrCWLEA*26@}9w%ca1Fe*od$H6 z=*4q@vma7p0tpTck4SrO`j0s9JWCV`XdX-NF=u-XF*~Hw@xK|#?iinf%Ox)yVuzF7 zlPZpMudhFLCP;1M8FG0)Gs?WficiPjP}UFj98Z0I6+M4X0!iRln{9zbzN4NNob`75 zlXGj~flj}Z0gSIk4@6tUi&G1!!$lKFTf{7y*>AqD>l4W>B{U8kcvVvCA*|=a#l~*T zD80p$^jF>opkwQ-t=xqNi~xtH*y7xyL@wzY{U|7I_cD9X!>)5>*gqsmnz#EC51kTmFM-qsle+uN zpUobav_p>}$X`Wk?R#mZNieMZFoX!-v+~de_S05zW-WERqtkv+=*2%udByEYE4?EbbptvJ0L! zc0|CJWD}Umxjd1EFPFm*>JGklshSzZSH^dGB^ub8Q9MSsO1 z|B6aFI+{zyIHh!OUyfy$>}N^NsV&5Bl1g9_(>rOEP0Zy`{t!IXX&#z1?c~P#KH3{k zW&9x*$C1hxTNKfRr6hOpUGySGb~S4-Pp`3>b_;R36Nl`7++|ixfBrmd#<%d3+ib0z zocSuS@wyrM2e64>&oLnJnM6#JPB0P3yx(x`2&d#Nkz{1|*B}mGIp|(FqK{H>wWcK- zqGdYWh|Z%=y`y<(boS$!*GB-a3_j`0=OUb09#u}n0Q2c#g?q&>R*}XMeTRNpCN6b1 zZbes*_~Q&_!lI|4^p;dWEm|;WSYC>eBoYhojYij{2>+!CxEv6;JJt;FiWGE@nCAMm z+*_Y_epr&_>M#_ZhjHkvJzvAxTE1kSlu<84`OK(qwCG77AQw8Dg)f2Iz;wT0CK>s! z)RCk66&2#G2Lbq-amy|R2U__63`5kdMNhu(+htx(%@_76Ra&7Gh6qw(%|<%b2$M|-9nN7Ng7F^#KW+1U8LD@8N=89w+C_T@js z1~G$6N*z_-(~9Y(t+5-S*gIkpmgRq%shZ(8i|We`VvsAAuo+xU)b%=L1j8@MpX-vT zvpBT+3qE9*#A2a>=s`Bns}K*@rTb;eQt*kkQT9Sa>d=bQKhGAEZ*=8@K3Hr4%7(}D ztu|4of4&mj>Zh5PU#-gY%`6Kuw~L@yk(h5W?)^Jy+Pw?8`b2lkP+yWhCkNs#j)m zZpnEQKWH2Y1A#B{qn#Dr$h6J8l;-s8wse4Hiqf@F^if*NJrgRNn?3BY#Naq{HW-Ht z%Bn_sL0taX#%Jre=clpT4K05Db4xb&dOh&4@Uq_eNs$e?e@=-4{3TgnCv8OgT;;3I z#6DR4>!qRjK<%ERyI_{KZ{dFlf!w&C@?6aDjQ@gvOY|WCZ0z=vmWN>vIR(m+L9*VF z9b>%7omww6OfnLl7EW6GUKY&iA!UH&MNXT2MdX;=ad_nK3KTkGLMFrn$O<0&e=lwZ zrAHIUZHFHsUX^t~ZCZ@v3<6swQs|Y)2PO`UmDFw>xB|7gLX*t9MvkmKUao7hie3Gb zep@6YxE3p4mgV(cfRmdX@y3J7WM|Fre!i{!2dg}jpX3eKQnFXeyN@#xc*N(4S(ut+^)WEp%>Q=$9xGRw(fBIgr_`Hh&C4-=USdq zL-GZ@t!{s5whZN%_hnhByh5=Y9C1X)ZKkztSWjtf$;4s}$?Y^KZCJl7I6V2{f6@f7 z36A`o>6Q)0n?|I3LY;V=1boV&)t*1CqVp()((jh_g0%Y(CevMifr{<5_Q9-q#m{qw zisL4;{Ifp7?|4MI>35WGo{#qOF};$uvUBoSOr)L$^muJ2asCQOx*M+FqXvjA2cy)I zbJp%7Vol<5dfmV<+V(8+*#2wU{2a6qdHEz6+Fd(RFv1W%onMdTh?=DFH zMXRaX=ael5=`YjhlL-<(l0H6OdEhpUBKhL4NIW1tZ{#*MOY;(@q42Jhb$QQQ*xI`X zCr`smh?$LT4R_3H{eTQ=arK-l8kYT_f#smqib)!*w#6X0kuZ_M+v;*ByRmc8_~YI} z)1?YvOQ2~9?L9uZ-defdvmfan4qKJd?q^HA)@KbjtbN^S*B0*ZkOyZ+)pBg_}?*V`0Hi)v(Fik{w~x?~q{| zqst(Mgj5lTdFdjCKJJ0*=qOAEiF$ zLtZ<~eTt$KOkb7R(KhUnXoXxfNX*V1lGoWOor&`ldf|Vii9uIbjlOoC*M-?JKt+Z5 zPJn9Bp6iwE>xh$jEWmbd183A4?jMR~HvO?QuGY7DLmDLxY;u};62a5xr2&H;4&8*T zb|UbGVaZNE-+)o$3__mCtp~xJ8`Ab~4e-FqY_jC*%R5Rw{A;|yv2n_UY;xkQx2QM=^902UtyR|ewWX)E+ z@Wj2X+B(`LBTNd#1aCJAY%`b03%qDdwc)8VPUU)`ybv$gZ=Fu-Ue^P5>pB{26u-7C zL}NY$@fGf601i&A$^O~3B|GQ(hjuP+IDm2Zh;X`*`9dXQ@xXd1mc(V(Gc~zBX-(+%q(}Zii-!Zcc_Sw)gL7DGJ2= zNHM#s;=wrGZ5wio=e(-lQX~>e7CFz%XJ^z4?2wr(_7_?3qOyJ7v2u8s8XgGu)J$!* z;BYK3?*%-qnfpq{T2(K0b8!c{w(>*N`@0^yJX1w<3l_k3t7dvYH=n||0>|_@ivVvv zuYTas&GyQW4e?U+jTv>b3{|Uby8GLlb+uUS4$36|>@K4u0zpoU11jHOe|)8JW*Me$ zbb~e&Yz%&x5jZ%c&JExJzv*#FlSG9xMgjeHvV1AwH?|(K7AeC zw7N7QV>HvrNtv!Ck2U)+m&C`Sp&`ThP24MeuQo`9^9&}L3DJ-tlG8Ph9JN|-4Migo zN~IUz#;k68lfWj`3G}yse`YO5=ie)#mW8f;OEZ!JvpN|pz4G?uG*!ZdAKJp|n|y*{ z9Qpk5g0pNkT-8lt(v*m2`x=DWp)$fQ4R_=9>%!xPApr8J>^T(!+DmMI1OP3-(|H%m zuNSaNsn1_?t0-g$jJ8Z3{Khm<@(lG>;Or?Jdt+C&81CbXKT#SEH8hp1wnsSt4}c>by>2^mw7IfH7Qur1H>}375!y(_F*K>RLL_(s))p; zfa~W{Nu)_UC#%CtX%*vythk!{vTU->ui3Tv0`VFcSi0uJo~c3<=fd-v(ZZCz@`*Bh z-oZ%_4SIG)>RJX%?p!j+%3CliRr8eHtHL?jO)(^7f;vp$g%1w3XtwNy%wd*mImHjS z-yxA~d3tHY64aADU+LpseBmn$O=r)4C7TJudEP+PRT?uHhiEc998DB^{mcSyXfXT-^j&92^w>7kaQ9Fwl5r0N3AITktv}rTuh0p zO6OUfPSEtry4LLku>4;v4K!n)Bi*OfLF2P=B5!fE{QY$wKSg1wY%;Lnu;q6pm#aNJ zx>B%Nj|V^&N4w;!0!!v}tVB_Nh4Jv%J4Xo7pMUw!T2!16yT<`M2t(539QMCcC-b&(dF$>Tf-Euz-%<-rMi* z3C?SvNy6+}KVD4wcynJ(BaMgmmo^9_57MHCh3O^ccio0uAExp+fI=U*RV~xJJST&%MSUXwxOXpANqLy7)7B`}i1VWX!9 zhCGFE`2QQ4D&hY(1k20uV+dm*zb@~YK&mVGZ2hs;JV28d`S*a-22~7;{PTEoOrxBK z=PyZRRT3=Zu0j~t8M9tI5CH1?px*-f0UK<*gl2m4=p~Hl>4Vk>U;5vR8K6k}!xWgc zc$~$^aM&ocNU$jc+hV`M{LrM$iyGMrwO;m?dp+;!FjwL!{B*RqYa`GLf??42pFS9s zzL(KB@wj?J{ufO5bmk{R|6dh)f#*H{ONEX1KAm&JgEc6+j7~n&kN9%L)XJRtJ{*!|B^6ccmK z>f&jmmcVhwNr3yX-rLL>4NPcf#o+uC2K!xn|^ktI-XBrtZF zR{LMF)-ub}P-vW=pU+j9JR`#Di^E_mf?$ToAW2F}Vg&kcY?w1LGScz#CWHin<+J7R zP?Qpa-F9cBiTc3!8XvxsV3Tl2>FSc1nwjC_;W@4RW){30klona%&V#SJa8skS67!; zUXD{*T55pg2IY%I0$ljQW``UH&2F`Qd0+PHnc`{;YsAMUZn(SOTM4G#fDy(Z)r(p1Cf#lETKf6rkpBxC_d-YIu6^BykFjs3EGiu{mP0Y=d+Y5 z7;hw^sRtGUS{hH4Xr{e(VSdTZUTrQCe=ZofMZ6m;}j<14%iNvC4QTQ&UsMr(2`TfvIE={sA>{(7@oJbCa?q?-!>oR8r&l z8Vd|8tR2B0!&0*tNXA1+Y`RUKnW7X5CZ!ABmmQxXB-j6q7wlyKP%Gv?Kl2j*@bP2T z17xTMC$cXvI2eV)pxJe-d)w#G_UzxdZo5y@q|O6?ESbE{uWA{(!hr5qScQ$h%pa3&Ilg1OPc~nTTDE#`Bo`+|A zTbYk|d3HpoH!fe=v|2ZJcGm#QKQ0cBk%h&3&#D&fZ?cZzwyT;ZUa*8fP&ioL z!Bj8wg8*qA^ZwkU7xMaRxiqzM$Zu0h&r z)g){Q>wo-63JC_w2Q4qzT3b7Ni7F@H(b68)=k7UTVq&Uas)pyo)6*$P2ttW)nUxd9 z#>dwnRiF4N8c^Qo>ZZLLeW{m{_r5HvSynX;#Lh(h z{W9%OMYFjj23ibq$C5thpOgT;o!=p*zs4C>UZ$rjC@Y6A_WuBw92b zv;LW@HZy!TSyoYDIT;O>Us=$v)#Xr!1rTGQUK}j%rZ5}9W{E8HevTI?8hqpV{Duu! z&s+B)IxUT?xFbrXhyWE8l}3Cs@tkjeuDYklB;G<#K~4_kKLeg`gwy}{@Lg2^t#&?Y zX-Pk}LLSM=GRWD25d|j$ z&JtHu1Nh(2Yn#ax3cc7qpSBT5NJ#K_#wr^j7QitK_&HVoGv*uS#Nk`bQmwJwbC2Ec z4M#V-mF|>Y4U)FnGj!JnEr&;YwFuzI?TA|(3=AnRFA?mpSUmHUlwHD|-QD9c*%#-Y zrl`BgJVlW3vWjM`TFcSE_htLzuAeUy#$0AoEy$^8`9^-SRa2E~}%!a@`a3kx89`5pv`h#DJtZW5!Qpi~)OVZ^X(Vt4RAwq^Z3 z8Gfaxr}yQfrE>Mbq&aV1V^?9fkllwy&cu7>Doe**(c10g}mEe#;TED5}>4MMXsp-2H5&h*`GKdaRP2~#7$;p{C3lya^%x9I!(~v5H04YfN`+Fc#D;FQjhl#M{ zJ6`CsU?6XeW}rta^!Ps!s^GP`o`Qx3-urT0q=Io_^U#;ozo25?7j`5lK$yyE6crZ! zUa+mCCIDNs5j6Q+pGYjTD*z$hnM)v#KR-WDCJMF7ImkK*^`8K$j>GobIlW&%$z%Hf z7^Gac8rke;)OK^imTi&>lSBKpB|v;nRn&%k9oz2_?Yi5;4%+qp*vnGOxO1*sRBRIC zovBhBRzR|gE@q&=#c+xV!08T6upz>94|itA=CBti$~kVIr(jEZ3x6Q?uIwIe^nLtk_$o{ z9uZ+G#Zx$t3NV++?#%bO4%nmr(x-V+l1NlQoXgfo*WL9gK#b%XKvlE3K;-nba_AF( zepMA7t$n22Gk`!S`uRyjL`IG;EMOHu<_`rp zb~#1>h<9~$4GauOSz0pJ)YJ$oyyht^VF6%OUoQxh8YVzF0URCHfF7}ouRCL$z=)a8 zYPvU9-MCjX9G9Ms1yraNo+x_W0Ljjm8|V>}ko-AX>tSVQ4^l7QrNBc_@OOS8k4JE+ z1k$;NL<{^T!a@Zc_TvL5kbmMbGoyf4h4c-X3XsPn&{O(MrMqjrqHbVMurdL#?1{aosG{QH;%IaT$<={ne%Rl;mR>xy7iiQu%MAa&ZugmZM7M!D WD9US+Ndn)80V&I?%T>r22mTMqt$z~$ literal 8699 zcmZ8nby!qiw55?|q-$ttq`PwfsgVZhPU%J@M>+%)RE83eEcYp@gTZo}25)uy4(-Rpf^9wofC8n2(x+3Nr4n8UcL%u)BKEM(i-)5F=~B`o|~YwW?DXl*%X~&S5I+X zJ7YdNf~he$-c_F6loqE&_20pJq2Ez?2fEGJ3X^4jHt)IXSt6OP!B7-5Ts-x|aZI}0 z_i8E3lZJb54%f_A=!TP4PnWw~NSdE3co`|rmzbwIFS^c@oWo}NE;;l0I{z4? zGUS>%?Be06M6S^<7TMLmnQ`<(e+g-N6KSSCJxMv1a!gk8mh&knS(vMolK_z?fz5_QE@j$A;|3!MbsJd_0vb|x(t-?ggX74 z#=P&3j!-0?S&{LLONj{~FA*`nIv56>FBhApk(DHV@kz2(4(TNK)okw^@p$NZIrBz2 z^JYd8UlATT(MmvES5%{fDOWD?_pb5x5)(hcb=`N*f$6vcs`mt3{$f$^RZxy2t(Kz~ zkWP{x$rp+u6}3p32b>%F*}R+Cn7Uw>d_nLnbZ7iEPmjp&JQ6w>t6zhYZ2Sjutnj%p z;ir8l>KTYRa(aKFl%$x*D>S#%!LQ9t*N?GBN};!M>KXypwVQyf4KEvA^d_`p?Z1L@ z=}xghAfx@VetwrHUbeLJZ?wXGD zd+hJyG2Bfff~I05D%Xx|dt?(N1a+3E5;IX(qaM-T%sAB3IFxk9ggAa*OY0r(7$lUcps8ov7a4qA_u!ntzu zg@uRP0~c`n$ATaCA|XE(LpffTA0wRj^I?$j+n^t$vDQ7xY1w%lj+?VA4E2-27fO8A zL7sP*&EDH@CcP$C?Gd~S-|CX|Lt(dVzv_ExX&yzcg$9fX9))Sp1t0u3|0R>Bw3o=_ zYvJkfKU8emnWXs9d)?PE=u|{aE`1L+@ITHjT=?!K#$qc51_u+Up(M*mnq9I>N#6=u zUhsa(79*%xdpEI#S`)~X&t%vsX#5DXI?(053fzfnyBIjfSD57Qn=3hvF01$Q*LXS1 zLo8fqwsn@W)4E-k;0c^x@UBEXqNVvAbA}BI6rP5u|Mis4-G9$w!MKvxcfXZ3FzXWC z2S|Ol)e$8Sk@;1#bwOrbhCw!r_VIJfD{Pfj)<6V!#W`YG1AN znic91!P2yrp0vv@7?m4fFtqSCAS}nS!=l3`>=So94%+!wRS<%SB*W-@p@7gVFT>cu=7Yw#lh&6A1JkeEHr%mCsFg)Qms6EpN^6)CN zq}AM;kz!Lxue|M#Ht@E`qpn`7MJPBj9M45;z$W^tbwD@c#v4bLQ!9dBAKw9qwjdeA zi@~;!AJjITkN*?wL7|0Y{!`F$WkfogPcIx6lwmhF%?w8}w^}4!sJ$0j7bfM+|H9Yx zhgW*|{uLGwtYEH?Oqv~q5UoY+@IqBnk%OOh;ls^ z4(p3uarCHzvY6wFFgfvYNDuG$hko|DUb*+Xzf=7$pTGcQPcV8u0dxL(}@ zDHvpM!lp)7wSq@$P%!svUh_w1K@h}c`V4k!RmmBi1zt<>)ECA1ob@`B0img2^-O^6lU=>b>*Q~HxdMcOqk z)(RQ@uCmZM2t=ywnD~zHsQ| zZcyoW75vwFZs3m!>JqSA1Ir_*c6NT@P@GeOh#=_U*zO4wKf%PSce$pqDhX0A0CRgqs2b6$uP0aEA0 zawHWtO<@?r#`-(y#OEfkaL9h_D^DHrV4viskBxhj4nbCk?}l`Ql5uL^)LgXEzj%Iz zzG{WnT2(4Sciu-624?lMdG~mZsNX0MtV+0)q+0*@>GWeo|LA=HT+(}zK$&<+h7Ou3 zd=p;+EjO1YL5~!R#i~DOCDT3Vlf*A7tftjlHy?)MvLW#p?zx*iKmoi4OWLEQ$KT*E z)PXoRmV`L0Df+~J1h?vPp^kkp0$f4&>`s21sA=fNuXfvwyU+v1JRur~w(l}(@!LC#X9nr_WUmCj{r+=kGhuly{F*QaygqT_GyDEC@5Hh52wxEz ziDr?e?8~(lM%uOE()3VjCzuL_C#*h>AHHCsv-g5vhb@x z&R%&MT&wC5_ftFnf|Hz?{eD89zZj0O(fD&&dV=DiJu(4>D<^`5}HY#7QQv= z$G41%jWpTeLI0B01FO38PnFlvKP@+x?GO~E6Vrw3@$C*+}RW7QklC3H|R&mJ5M0-oto~ZZ6*sWu?i?)+{_GS+r za`k!`!=&v%uL4Dtx#+Faruy~Sv2pkdco;W;C9gr!;;t0suYRHAlKCccrjR^UH+w%9 zJ73UUWc^5R^39sJ9T8`SASM_iEwi zp3AVG#iR6b@WrSUjr6PtXr{n460~B7daK;s6xcFf8`u&=lJHaCe+KtC8lYmRh4bIg zOm>M)c9~ji@w)VpPWXJW`4aJ{{y-~bDj!nQX9BCo zZ;6$ZdiB?S62CW?@#xroST#x~^b?4m#EP+RIqkk;P5k9iI#Rpsgpjk4i@cs6hAX6n z(vFhl7({idqT)%hBy|o3^bo@9&flHCO61Ho!l@Ip`v&`Vb7^Pa{%`!2nwg>vf;ZdD z3Ibtp9c-ikG@+em;;>}mKHt__Ud(9aZ0&I28e<*EAjKQUw)+*0)|R>?ARKzqBE;M!Al1OMjMyl)eWRUt4*Hlqp1!!nrb#rs*n^&6`PN^AZZrV| zO~dGg`<@Pe6uBQ8XPL#PXIu~O{>|_DS{mPLaJ~#iXY8!u5%R!wg)~tbX%7;C;ee9( z>pT}F(501eCNQkh_;)Qy%<~v+;bRCFK*j!H6wt%?c z51=D+2CeFa_vLC@HfrZpGY2c%>ZsoheWKOc>Y~Llf`t)|`Cjgp;Q_(|eQ!_5Dnpdk z=XXNv^O3$cU2c1qpPm2*pw>517jKo;4TdzcqW59icPqU4K_%7FVw)kd++Z5)+!!rVVs ztZ@8V>dwodUE?AqZH$G^cO|4^;bA?!$ z45Q4Y>!cSqk(#irp3EyMM*D2x0NJ-qeua$kRNcmV8=Fjjc@wDv@CLW)HJ45pb_Vl( z`TP|_luq__&L-J^S7R77X!ks_JHG>g{u=kU#5h<`+omXjC~t$H7TlTqQ}ts#xc7a! z$9fiyxE)o4=)KsJy%~aL`t)^G+yr{RcdNhq8&3~6)8Vq&Gz9;Zi%X=8zNpKpoHu#a zKn_6TR|3WIJe5!0)Y7ic`mw3-nL!~X^xC{8V_`~Samr-vyo&sJ%C?93 zCV^lZPd+*HJdgOv1GAmkIY&7kKJ({eWa6xj%0nS2s##ZS1_<~5wnj7B z{#c32!pSjfVfs3Pt-h~&4Een)fG~QJm}J5>5IpA$=8e_rd=n&@td@iB#iD6>k-q`Y zqK!F{Tp=4Or|+96i<&4)WnGrxYEi`ZMnz)8d5lVyUfqyyao?gQ9US2(XiQPvRtA1A zXjLpApo2qs^-4qbMZL^rI~%;hC*@0sqwRkN_IZs^5J{4A3C; zRC4HlR$4+pAl`f!3}|*IF);jJlV0(^CQyQFOD}UiTd(p_7O&0iEwnmFLDl<>9m?@K@?5~i*3{=;F_CdHnu>;DBee$%_wHmN}Y%s7e1292y#b0-DyS0j(vbI#IG6(YI znb&hPWct}PxC&(Tw>d<+%|k@1%~p`I?HVntU)9|X;vfJqxFu!?=}WSNXg}7^)EU8Y z=m`}@w(kEy%c9_$H_tNpMqbPOcgVm3|7-qJ^Z%Lu)C8^-^jBPmx`(FP0N06^?|wWo z!x;SJ$f5bLfJ)f0=SZ{=+vhVCZ_$_R+kDX{5@>_8G$ZaB9mazne{IY2+zqd=!QuBf zLqP2QcT)B;R(+yhJ6-F$TBHI$w_!?E4Z4|YUGP(|GzCUo%>1)rSwueO9J77((^rhN#q`CDa$>7)U~kPW)d>(@kj7VG@D)!VUK0bciHZx>tWyR~Z zBtb11KrHTisNn9-iv(Eje{V!jPjArV!^Nc1x$w)=Fc4Vpxs!Gn-SS>V+ohV>$6?zp zxNJC&(=_L+uVN$tElY*RZxsum^vulYT~T;lwGI=2-=Rik2NZO*Bp2N>LlX6}YoawaKAiO0vs zcifIjusry(tvkak1WP5QIlxSo(7b;B1enh1e0v-QgPC_cW7to{Mv;EFwo3dqn977@ z_r)Cp9JpU)N>DU`9(Q){;pyqV7B4JEh*p94geEKF?JX#U*stO405D;9>pK`}ePiQh zfvINJS>$Ypzv}IZTtsxV@8`LK3a1Dp9`{uRPQz+6y^^Ut__$-;^+tx-+vRrTOLg9S z2%EZO&1_tHI%&nbkJX%AD{tw5L%<>B+^&Bswzx$p6(mB+Wwd_rRa%x14lS;<9i}A3 z2VzfAQ4yV!G*+`ODl&3qGs{g+LF6m70FZ%q*C(Gu+;cViW~QfetE*EY!|4@Owcflr z@Px!u3FDufoW$(4xlhAvhcnUZ<`9A62L}hNTwFFQ0A#&OwV|-BnJr8jw-@*%>iGgq zR?GS^o^!B&Xh^})@yARoI}mrPtE(evZa{F~cdxu{cNRhrBPAy%Lo_w@_kBGvwj#+) zx4qT12_wLQ;UDM~LoDm(wav_qCfr3uMZdl;u%4ZreTIW`cIyci_rb-F4v&lgW?E!A z^;yoV!Po>8d4ukeOX}!s1Q#2d1-%7el<-w%R#uCYMhX)X3k!maHPBd9HD=}>n#-Eo z*qHg>c$E~U+WL9~F((j8FOjVTC|ex$eplm`>FrHDyScvhtB_YEM9W6`@-ce!5WV4> zz|7tFWZ>ON-^uk_VqRU{)WIIm`>RGYvs(I7{N8QHzpUYb<-!1LBPEO?#nl<5Dh4jbHJuojna(+-Lt-Ew!TFDW;lpt^GZ! z)D*dBAu%xivtEg!oLtz9v{Ic>JbXPeB$_;7p%Y7E! zvHUu|!?oUOFr6!=ToIW?FZ{&j1)53FL#3yX=D#xk3sG3MnoDRUi-1`E7?nS!1LyV?AFC+I{o? z?s=zUHuXcpwBUPn_485SUr8}Bv4HajqcTzl2M21Tfg}dzBpZrc*3|{i(a2%jvm0*# zO1vRk{-vNZ_2$z7jyK#2ardnZm&QBjsdpLjW`W#y7YIfYp# zr>0g~5r=yxo!BGdR*rR)WN*Ac-!-%G!#m!pPW+yK4{uvCOSb_cw9A@rrL2CbGedVClsKil(Te%8u~0_k1oOv-dsc34B|}3B0ZR2m1Y86EFPT{oC?_X}lUz3JOa;Bm zEb|eUMdb1B4B&vx3|;7#3Wu6mEghYR++2ojuf`EW!FVM`Aghy7QvUc0AI#L>si~^M zryQTi{pU|19UYx^Aki(Wytsu6CU)-h=Btd#`iF-V?d@5ev;nTn1u-g_Z;$1$vay{E zQokCZ{dLynB18pPZDeF*&iA>}?O*GNL$HmZRKd}}4aJwb{8Z`GHq>s>P6=V*3}g~( zKu9U5OCcFlnj?FOeVy?P3#2m5WtA&{%I1yB0FRO>xxxwWn?%nn)q>rN4|`~`?fbJe zIWv0{TEOy-shWUZiH^29S?#GbDnqsX@qEj%4+sTwSpd@L&~%lQFt}UJ{~&{)BjU-i zq+Gk>2D||E(_*Bt#sK^1=}%@16!2juL{SA81n`EnohXpdWjA?Kj#YjqmT}~~xn~+i zmD(j!4VQCH$=|+>0Q{*9pZXq3lpWE&1bDb7UnP|sunT*=63~M< zWZd0Abl>xZsA8#0#4Cm0oNcXr@f@*`!He)10&R8tJ10Uz284&xOnDDWc2L+D;MO~f z!Rm!smBUgRJ~%m)*xI5lfGc|^r@a%;kl>qH)EIijj#7Xq z_omCb%HK5ITElHLfgH0Jpd9)lO60jY^j@#TMus<^b)d^q4YS$AOJ#G> zZ~)u^C{AWnUTr=bk+-#dAtxs%HUqLg_&^R_ulCu{(b5Y2^9TAdEv>_v&szhwg-f~I z9n;T3#w<%H*40b#7C=}aDFK-Ta1X#!%050mTTH>Y%zzgYQ3qGv!pq9=1O)}{U0e>R z2h4@Cnj}K<#_f6hj?Ez{XyF~90CXuRDA4iooqlZkfXf05tBBNl*NXq7&~It77a5=G hfhR>C_U94xVt+}*bU-_mr`egiOE+RFa8^EpIQl?@Bo0{L=VE!8y0HT z;k*!?bDl?q+o3KXgkW32PAogQg0&>mRZ(F1FYl^Q0l?vkoI0yw*OUpXO}^r4)paIy-;MEYHS*V@p`8>RboP ze!zF7nROTm6+)0vvR`AAH~`pqVGAgw5ET(d&ZsW*P!E2eG#OSQ4Br;-#pa9KSnKLk zOHVAve;ru>6K||{`J&^mLsHyV!BVJ#FABeehbq`KbPaCnKJ;pw1p?6{V*Sw7a49aV zJo^!AVV!Dw!IyvF2nP#q)>tO5%)A-f3wB-w>ul9|oT)m8IoHiVkC1o_in>AFkRu#U zRQ`f6Arvq7n}fuV1lGbj)f2rR#((twJ?^xP2E|pd1j0k16mIH9a@aLjjS{Ve-x=pQ zJeD~PCf;<}fAS&NcH8=^Nr9g@tfeY0E{4r!V?EbJ^h|&DESz$lRatm7Z2>;YU5DeB zPGa_T)A3-!IBoFmRo}2a@IX;NmrHioC2bj^EoHOW>NTxQfvLQjLiPWWA|cLf!Sl$$8rtnQ>i-R#fY0)`w`=FOHLOhSBfDgW-EH?t?iBP1>y1!DDE`oQ zD$G~BoL_&iZnurp#98}GvsmZrT$ECLnzs!C5!CO7bsyq){d&Pli|7wYk<`z*Rp(jD z>Rf~nd{>$Yj&LeyV`9^r2UefZUf_JHt2G`M4wfI*WCKk4(}n7bn)vGa@P5q$3jiKW zxDS+4$cl_;V-&tG&jtZ=hq`owpvW%&5j7syRd9a@k6xjPezy_)`O|yj$DmTK!t~^) z5M%1bT363*5CnV64j{@Hi3-UMp7406=lF)Vp$(*a6qH#hCXI?*C3xrbdw4AAw@9%j z`%UT^1k0Z_V;Q2}T_YMbH(X;7c~v5o;hj_OF{pG{W~l2L8<)^nig}cQM`eZ;r$5BQNfQxmjB2vi;u_oW*{WQ)6*mNq2TwR`ytsuyr6X&# z{?|{j{=%o2@ZBT0D`O(E$__TE{~L9MV`9@GDx&&1zor01t`gSjI?u%^ZWlQ8xkrV$Cu5>*-F4=`j zj}zUE-I~<*EdmRm__aD85rP%r1}lRsh=cc*_N~y|dPb#OiN~{^g#?#p|!4n>Y5ahWEaL#!iy+f0j)e53a@agPX zC~}n`+8Bj3x4er`K15yCRB_EY=yld)#F}kB8*`X>crJq~h^*B)=Xn&mia`iLh1-sM z_fE#ys@!JxG1uG!FTe3s$chX~DPn)j-LZAV|FSXbL1eA|_~VP3fd{c>8~TRzY5u-i z61*=u{yOf7zYBvRZ)moOWD0yu-2WKL#R_<+2OEZcggA3-$AHDk4Z-aW0pN#*b)qgNX3-Ii5i$%(}%6HSRU%Z$Zzk1fe0XkfKm#MaQJ5Q7UnLyh($Vn z=>(=9dhtr29)tBmSGP&uwb~UHx{5G)|5OMZub!-lH(}?EdgGF$e@_XXPV3cjR=A@l+6{Z|`QWJ|Y1y0sCZtoO`S4IVUHLu@n#F}kw^8P5- z7UWY(@z?%y)uF52?~7c;cqH=~bseU_aDP({JPD^;-D4=1DzxoSfiFjGhQJAJdQ7gZFT%(f@#VC ztL9GeP!AFeNHtWftCy zEh9dGfitw*IdJWHfrJozG-L%HOc)2n$2IbVN4~27_w1R3tg?dvX}ttAcWwt%xQT2JyrT&u`n%u28i*Sl)}7mGTGt_rOJ($=m?IrHFXp$b9_g z4>JPNA$BE4=)He2Gs_O*o;{Oru_j-WKq*wggoOLBddP|nNuIclxO)KL=IER7+3
4H5;y^8s?H&O_jtT^;w=_!w<0clI(sHwIPx-FS6(po&D38X_zIp&dA7sOhwI3T z?{197&XJq&WS>Vh){B6E5Q5JOzQkP_6Y=(`cbgqZ>sfzGPrQxWci)dw&T|?UM=6Dg zGhy?vwHOjTxI@pWL&5S|ug7~ojPEc$T?;el89D5a_M`Yyq z$ynbx^&alZn27BKJ3)xHUM?vTruKdWJ4SBol!^0%&T60pOFT9Y{}_8q_T#0Tg%F5A z1jyf|)n_&3g1Gmq*0X6U(u?WFHi&fFqnHKv$@OXk!%KzUejfmqa?Ft+>u= zAn(QA7BdWc(!ap;mlpd%Pv<9Vhs^{7)ycv71 zTz~D1&#kzzaKKC0Jp5zywDjoIbLRq~0Xf29a^imk>`C8_>B&#Qt+?Toofy{r7PWB+ zA=q%?GaRuW!~C4TVf44-u)ScH3TSH9(i|u%hZ+DpXm41)s56{D)io=@k|8(7R znj28`hVZIbSa>rAU77KRQi_qbVW`Y;B80c# zkI7GCV$bpIUYF|9VEHOFJVo<)XW`eVl&gObB3Q0iIUd(q`)bzNTCf8yMa0g+uTkZx z#Q|pr>d4R zC9E$b#jXA}U{?7dO$cEmMywg~4_qIS8nA9WV8N=bv2k}G!4i*ei}zt`{tn#S?IwTC za=YX}g=9yF5CVY{5KgR!FofeG3R)EdwnJJ6yX3&QeNzz1hk)mJSU9r^76C!O@Dy-9 z$FFo(;;W(z+!;Gc{VtH9>QZ09!HLA#y;MN<6w0SiV-Ib{S%D)Qwn@ zwgA6Wp2qRYlbTKnL?u|_P~a-MoCpvehwV8S6?1iX_u{la!*b;syJF@mAV?Hp2*;?H z;TUZj(Iw9Jn$X_pZt8}9))e#$zZN$}rqwAcw7XkoV0Cg_)DBvdHos0r7)*Z(DzN-T z%QE$VSLEh33=tZfDPN(A=5bV0BvTLS8x;|ZPZ8R6pkE+JRy?SZtD0^3hJQ1*!CF>Y zcz-NPo8P?nhX5hGCE#jK%wo0V0y!QL#^wj|jaeg}@CY|pn{PjL@){fmCKd-Wu$UNF zObje0CI%K0%L==)YCNtw&m(_*B`I#!NDU<^luZLuuYPj;e7Q*KM;1N=Ng)Yf6RzqI zEH(iNbA|15*AN1!ARp!H{s|%kz@^GEVxFA|E+mwJ)z#%GD@DcH)!;z~aS=QBhF=j^otXUVLYMk!2Yi z$00H@ven1U&(DV_iZFkfOhMnqf?lCowQ3di@81ue=T)ef7+5Waic*Ss^X8$vyd3JE={^p z;yFE7te^*rQi|N%T;%8HHwYA^6jrMhOO`BY^1i!w?~au#SK{TDUxp+}^#V#M#nGcj zv0=jo)lxBw6(psFUAuPqtsQwg`K`C!!W(bAf#~SyCU?9SqTaoGWA*CQFq`Ynz()uH z$8p%QWeW-m3)z39Ai)x}1Hi$92mOMfC<<1tT#3-o&=$Ky?>5VqFNeiqsdt@_kPy82 z=9_F%kYKS`Wo2a$MbYo_yYIeRt#7r|O?l%cN-5^dnWNdC@3D%Di($9hS&S777R%vq z_=SZKf`=Y@NG(USMM)zuF%cGvrOy5cArJ%sXU?2qF;;&t$mw*dX?|^4t=2X_9;Fm% zX=(LliagKb#EBEk6$U$4Mx(JwZprO-tM0A#U+wY%Aq0a55B6Jv$ji&q1z5TtkJXr4 z5(ELumMyCnz=(*5dLIonR4*$kDq>&-yBf1rUT7jBA{ty?TNf@|(EWq$pm&9#p`m`= zlGEvIl{J5>?-#r%rTG5)@BO-8R>ukwb?ep*vMkrz!J;VQ!i5WM>Q)IMC@CrN3uSzK zJOe8jTwP>jWW6zu7lIWlREQ&AKKRxr84TW-0f!MAKUjzd;f7G}+wg`-E0HpzPVF59?qBWBE)p$_Ep7hfeM zCAG;*8C}9#=;ejgwQJWRBO{~HOiz*|goTBnM~@z`*=)Gxnrjdn8;f4OdI5ks1SdB) z7bSlsCCJOmgD8sXcq@Mp*3{Hs@#4iuOiXN2#>v3yB34V1But$;wZX?YzI7>Yvv>n> zZ^?q=IQ6eNfTX6T;<@LZgI=T?i$y8LnKNgwXwf1Vjm8eO7!HR6Yu2oR!C+v+W`Z$8 zOioV56Hhz=Q4~9@P+47Fjb+Q0=?W@cf#rV%DLp+Mk3RY+YHDga1geD#7a~4BUU!dp zbT`|KH;(%0r=PHN=~9@@<~DuD(A%7Dw;Qj$_8M$98}uk$Ph;ebeX_H&VKSLo4Hu;p zPNx&&$B)O9DO1{H03idb3tBC&&YnGs9Xod5z<~oW7z{8Nn)lPXZnqn#57pMDQ#n83Is@?e>j|Vw9Imph=Movx+s;jHjqNi`U!TYbtWWvCK12K5;U31B;2NmHK~0 W`VnC%q(6TE0000(k&zUCSxG_&30nvR$)LoC3Jeex&=C|7 z9y7=7;8S@XMsSSD%0FTRv=#wWdFvAmkK0q886;Ol#vc!Oaup2NMvTxm_lXO*A z*L{DGF1wO+(%Cy*_j^uGy1S}tx%X4Qt!~{Wlv0XDy_5d|wqE%M0Dw}8I}>ile~efR zLI{|Dm^}KjMGPt?jVPta zsmTRTRCF-Hn$m1NbtxeP$sq&bQe<^usk5B5B$Gyj5EM6*s0K%&3WkRbgO}sef&~D> zLI*=qBrmYaSW7ZBQ@L2FzIG`x23iuEc{UAySOO($ph;jyWA)8aGK3XC1E0W^opaukdNnKOvA`0`>9z#`#>oMX=H%Mrq=#g1Vnr%d+?i-&P*e z#8*?}GusE&kkCQkydwVww-d*!b3ws>+@YRTSbs$mUtJeIvVC9yz`cogfl>-tkr8c- z!qMs+5HNSBOE(CLoa*n<;CB834&gB_3zMD(me{wtQ3<*WzGt`o&O#lO#WA-T2p)`b*+Mb<;$9} z3{mf{5p9|qt}Tc>st~L2_7CqdsB~CnXzm)@me5v;d89HMFP?cBPPYg`2;$ALm^*M5 z158I{hPC-0;^_V9JXHB!JyKSwff}ePw`3NrwaTW9*I?q@uEav7?m8eA7X68|;!HUF#~?ZB`FR&vmvoU#+OZU_2*8DsqF`woEx zQ2bn9j0nMsaD$aW7R14ONBdT2Z#|<{uEj&yPrxSGkzkI;$G3dQdLYPPs+DWukn1tP z65rwN_ze~z1Q%-a@mS6x@Pr2;1Vzpg{OI@zgF};<)e53a@Ufi5D05aI+8Bk6H@%Bc zK15ySuekQ?bc8hp@#Z*xuP<|0l)HpM6-3tRN5@Ynb(Vt=f*O|%zd1M;m+K1J-N#&e zcf98Mmmw=MD5Z!s_ruPK|I3Cg2a&b<{r7*+3_OT8$6;95koIq@Bfqk53 z$2^d2`uYEG;c7mXk6y|~aItcO|5;&cbk8~b5GuuL0Ki%MIgAP)$t+r5fd>>Xo?eB~ z5ows0G+P5xXZo;H7RycDn6&RM7>EEu2q>lC2#0T`>|pM)k65I$SI=YNi9a+3>aqCb zmJMCfcb#^HrOq;c%so6`o!yO63KMU_?n&EN&-DQ}ckBMUp-@FN3^H{iEAyY4`58JQ zu2TmGivH$!EKPeFo+rZyArRdnW*nTydN06B3)2tWr$Pk)3KVP`|MyNq)oHK@NaI|k zQ!zPig65zdavkO!c|;S7F$GT6H{;+uh>fFyJz6^Q8N{1^bXvB%YQm;%H7%{%f49CDoos>!hvaYw@K*VgHRyuQ;hmbYX@#?-8PHKXk* zrHC>{V*7-TSU?{DP(5&L0j@Nev^d%rg>AQNy!NkRS3h5 zi61k#dJ(FB+21~hYN=@EKwI`9{L@27Viq^px)o$ zwJ$`CwXy@#56nhwlMC8S73`U`9Y)UBJy|F1#@?lWlw$sO3sF$_v*t>1Q#X<<1F`Am z_X4&LJfE3vb4bi66tj8M^Hlxb8eb zLJ0mgZY}OjoT2{W1#oRP=3-|F?mRFD*;U5_(t4FTD^MX-`TVoFt_JKyK?uQv#}?qV z3vV@lgK9`+&`jnA08T~3vnQ9~-@bb?ARS^~a)h1> z^H|k!+<9OQt~3;D5-5c#n3Z@JHjG=_Bgqrj6ITxa+!%cWKAW%^M$V|QP68+3Qr(Zp z+&>eqoPU!=+ntCDAIn*UXHUKWXX69L-kJJ;Yhzx*Q>jn%*zs^ZdGh^?(bzp{8y*?* zfW~?e5D-G}Mafr~mNgr1{qSzP18F_$)$?!R*8O+m2gi>Z7e^_Di8EpQ_)QoWozbJm z)T3Z|tk*+>e~V9U-hfC$geLal2nQE+VcV6R$lN~@e?I*(M7PMu@sY8<{lj~hmNgrH zpO@?gA=-Mmq)3=Q_@(}pqnvILuby9n{S}9xPz4;}Gy%Fo6+|1O@YW5lASonKb3q5K#cIiV{W^9$4rG3b zg(;81=@!-ai&BcEY0qk4`mXu}7JvVLM;xd;(ky5VO6%0ZxmkEK4mO^D(~QrhxUhWm z^VmM&BMh_*=+$HA0-*sp!eMUGEF8%E91Bw(g-db4Av-X>-%V=c5<;-0@H3pWoyM}f z|HPDiGw^xI9u?4*I-xVrwH4d(9N+CQ9jZKvr*oI$L`|;d^FNMwN^=2EMqR~cNa?*AhbH073UE7qX3;#3jU4$FL zn{~X$y%R#Pqxeg_ap4`v)D3}u@UZY^jBT9phf<13apO^&=RgQ=!S7QZ!|Z`GyS*;e zr@``8T6n7F{qEAesFmvwB3K%&9FO7FVVZSzmh6I45wW{;FY4U&*f(V-WW|k2sjA&G z<#25nbD7Gha@DjCEVl%S%8kELT8h*5vp7+65>duTEEqOlwUnu0!yqYtE|~ahw_CES zPiR62BQav*xc|m=5orPI#se0trkdXWHY8dSu&?|Ob{6l#jWOC^ow7-G)JQgj2q6$S z0pY}o2tzooprlhVU^k?7ut|2zI5ZETds z)z1PMsy+o4aP9uw=!j8&cyjpTaC%K<4*(}@CpDGkroR*$NQyK@z$#c(c&elt*j-l< zVF*{(85f;_(^t>ouKv@I88;EXiX7di5|2IwmbVpYT87Aox*n_2U&O`Qe4MSlpy{MQ zRHCInN*aH2nWD(9pr? zXNti{Ybr*D56AV9>CMUt-R_ndSiKwPAbWB7*TQLbnd|3k1oE8+CGByDi^@Z>G0e%SsO)g%#<`+86&2AcVIB zT+NADtd5)@$0Ndj*#1DiEo;OR9^nRS`|YPr9)sh+#Nt2(783)DiGjt$#K2-=Sz%vR zgWK8cex$b~#l;$_wIqeIX<%B^8T+|wMN%KK@F7SJNd%j4Rfk})2}qbL?4GNJ5J)A( zsNVc{5Fr3gRhAL^#3FDZp$x3PE_YQWYBp^EXEOKtu_%&%h?x10>jEs6751oaX5|JZ zCI%K01B;1)#l*z2!d4?@swkj}()&k36$Vz&oM1vC`ak~{5T5USu-u9+N(}(JrPE86 zEBNmS2&|K}o$NgciTC?1y_$7!n@M0$5i~S3AUiu7xw*M;I-OXuWQn?*Qi=r&7U1gD ztDS}^H8mA~FTL~gDAu=+u)B7zhE`}(I zFquq2-^PMop<2IwJq{l}44&szsF)a79fpchils}JqPn^od}F4_pkiQk7$^@w&ph)C z>g(%!JkHTu@~w-?yC+`Sym>R~>gsybN_j)=cDr?d1DS4IVSau-_U_#ap@#?j@jQ=3 zixwd@H5DAk>BjZy=3e3Y_3K-Osv}m!lR+x2+%8fS1z&&twI-g^gT)GZuqdS{C@4U2 zadE3aQA%O8TCr-?D!;eguU|i`Tel7`yzl}fNoo;LN-0jAI)yD;wy2hhS*#!_E$rE| z$7k(-$kWMhzWF9zfBkhtM@ReJ@g9f<4<3vS8#cgfZaxDaAp{)9VaJXgC@n2zlY#_G z&<+5{k0195hN37~w{9ImLqj|45WLa*pgGEt9VPRpHx>Z65Dk>^`LYa_|z`zOyR~H!>*<#G&fne?0 zwOzI^N-4H%*#fuQ?b9-)rKK^jf*~+}4+{%ZtLiN+udJ-Z2OoUUDOctBjwp)QvuBUb z5vj5)BRM&lffX#8GiQ$P2H-dxJ9Z53zyE%x_900Uo__jia2)3oPDN1|Si$5DZ@THG zR-dxrI1bs_*;u@IF;1O2<(KvHUbc1XRy_ISlj=Y|U-4CPa&nhEl+h==g`WY@$vB(G-wb2s6%iH3JOqBQGueOB8Z}>j<@m! zVM9X$R<2x$q@*OjGEN3oAF*1JBw_yi`K{i@@vcjGn#B`{drB4@$Eknq0VFLg4NpDw z6!aq9SS(5@E?v5W6)RT2Xf*bJsKv0`?bx_+BMb%u8#WV+8DdIG3Lbv=VThvGV};84 z`g*Khy;@gL=?W|lNST?Lc;JBt(9qD(BTy}0z8nb&3A($*qr2H=JaN>SGiUJXtFOXr zHh1YWhMwkhxm%%$YNPF>l_yE*U_` z!0Llm%cIMeFJsrPT{v>&2n+@T42Jgow64qLf-K9JFku4jx#u4A@84hdzlx!2i`BIA z-EKGX^74?AlY_jxJk;0Mt3^-ma)al;$z;NqF=LRCk%6&e$7*J((Yx4;er>>Jx7*=z zxlmtU58&D`96=CZwOUmv$h=3AGN>3>e#6yO>oTyIm^!Kd2dB8X#Xwo9hyVZp07*qo IM6N<$f-2Bwvj6}9 From a5c4dd974396d1dc9c3e55381215a904fa997cd2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 12:35:05 -0400 Subject: [PATCH 0506/8313] find: New subcommand. --- CmdLine.hs | 3 +++ Command.hs | 7 +++++++ Command/Find.hs | 24 ++++++++++++++++++++++++ debian/changelog | 6 ++++++ doc/git-annex.mdwn | 7 +++++++ 5 files changed, 47 insertions(+) create mode 100644 Command/Find.hs diff --git a/CmdLine.hs b/CmdLine.hs index a683be5c53..caf7279463 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -28,6 +28,7 @@ import qualified Command.Fsck import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit +import qualified Command.Find subCmds :: [SubCommand] subCmds = @@ -61,6 +62,8 @@ subCmds = "fix up symlinks to point to annexed content" , SubCommand "fsck" maybepath Command.Fsck.seek "check for problems" + , SubCommand "find" maybepath Command.Find.seek + "lists available files" ] where path = "PATH ..." diff --git a/Command.hs b/Command.hs index 40a21cacc8..60d82d09fb 100644 --- a/Command.hs +++ b/Command.hs @@ -158,6 +158,13 @@ withAll w a params = do w a [Git.workTree g] else w a params +{- Provides a default parameter to a with search. -} +withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings +withDefault d w a params = do + if null params + then w a [d] + else w a params + {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f diff --git a/Command/Find.hs b/Command/Find.hs new file mode 100644 index 0000000000..db0589feae --- /dev/null +++ b/Command/Find.hs @@ -0,0 +1,24 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Find where + +import Control.Monad (when) +import Control.Monad.State (liftIO) + +import Command +import Core + +seek :: [SubCmdSeek] +seek = [withDefault "." withFilesInGit start] + +{- Output a list of files. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, _) -> do + exists <- inAnnex key + when (exists) $ liftIO $ putStrLn file + return Nothing diff --git a/debian/changelog b/debian/changelog index 64973557ff..07eceae6cd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.07) UNRELEASED; urgency=low + + * find: New subcommand. + + -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 + git-annex (0.06) unstable; urgency=low * fsck: Check if annex.numcopies is satisfied. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 61a5962f1f..c496ca0d8c 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -166,6 +166,13 @@ Many git-annex subcommands will stage changes for later `git commit` by you. With parameters, only the specified files are checked. +* find [path ...] + + Outputs a list of annexed files whose content is currently present. + + With no parameters, defaults to finding all files in the current directory + and its subdirectories. + # OPTIONS * --force From 1af400b98be0033d3c31f68b909368ba9b1ac469 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:07:58 -0400 Subject: [PATCH 0507/8313] add --- doc/forum.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum.mdwn diff --git a/doc/forum.mdwn b/doc/forum.mdwn new file mode 100644 index 0000000000..ce5787adcb --- /dev/null +++ b/doc/forum.mdwn @@ -0,0 +1,3 @@ +This is a place to discuss using git-annex. If you need help, advice, or anything, post about it here. + +[[!inline pages="forum/* and !*/Discussion" archive=yes rootpage=forum postformtext="Add a new thread titled:"]] From f078cea3f62f0424db6d6252373a16c8e48b7179 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:08:20 -0400 Subject: [PATCH 0508/8313] link forum --- doc/contact.mdwn | 2 +- doc/index.mdwn | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/contact.mdwn b/doc/contact.mdwn index 1238ca0403..1f122fdad9 100644 --- a/doc/contact.mdwn +++ b/doc/contact.mdwn @@ -1,4 +1,4 @@ Joey Hess is the author of git-annex. The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home) -is a good place to discuss it. +is a good place to discuss it, or use the [[forum]]. diff --git a/doc/index.mdwn b/doc/index.mdwn index 439e2da09f..d863a9ede9 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -21,6 +21,7 @@ To get a feel for it, see the [[walkthrough]]. * [[news]] * [[bugs]] * [[todo]] +* [[forum]] * [[contact]] """]] From 8542ca49fe724d46fc10609fa424afbb2e12ed95 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 14 Nov 2010 18:09:44 +0000 Subject: [PATCH 0509/8313] --- doc/forum/git-annex_on_OSX.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/git-annex_on_OSX.mdwn diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn new file mode 100644 index 0000000000..85a8413bf9 --- /dev/null +++ b/doc/forum/git-annex_on_OSX.mdwn @@ -0,0 +1,19 @@ +
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum
+
+[waits forever…]
+[finished]
+[realizes missingh isn't working in MacPorts]
+
+sudo cabal  update
+sudo cabal install missingh
+sudo cabal install utf8-string
+
+git clone  git://git.kitenet.net/git-annex
+
+cd git-annex
+make
+sudo install git-annex /usr/local/bin/ # makefile is weird
+
+ +Originally posted by Jon at --[[Joey]] From 1277cd7669445bf12e7ae59982fb2d07cd323409 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 14 Nov 2010 18:12:08 +0000 Subject: [PATCH 0510/8313] --- doc/users/joey.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/users/joey.mdwn diff --git a/doc/users/joey.mdwn b/doc/users/joey.mdwn new file mode 100644 index 0000000000..306e1cc768 --- /dev/null +++ b/doc/users/joey.mdwn @@ -0,0 +1,2 @@ +Joey Hess + From 10f30cf6383167e72ced5c6138ee6857b3fd63eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:14:27 -0400 Subject: [PATCH 0511/8313] add --- doc/users.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/users.mdwn diff --git a/doc/users.mdwn b/doc/users.mdwn new file mode 100644 index 0000000000..b9bab48ecf --- /dev/null +++ b/doc/users.mdwn @@ -0,0 +1,9 @@ +Users of this wiki, feel free to create a subpage of this one and talk +about yourself on it, within reason. You can link to it to sign your +comments. + +List of users +============= +[[!inline pages="users/* and !users/*/* and !*/Discussion" +feeds=no archive=yes sort=title template=titlepage +rootpage="users" postformtext="Add yourself as an git-annex user:"]] From 0e55d6a907a39c3b7239268261edc2d5b5f55caf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:44:24 -0400 Subject: [PATCH 0512/8313] move stuff out of Core --- Command/Init.hs | 40 +++++++++++++++++++++++- Core.hs | 81 +------------------------------------------------ Upgrade.hs | 63 ++++++++++++++++++++++++++++++++++++++ git-annex.hs | 3 +- 4 files changed, 105 insertions(+), 82 deletions(-) create mode 100644 Upgrade.hs diff --git a/Command/Init.hs b/Command/Init.hs index 8110948a41..c928647a50 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -9,14 +9,15 @@ module Command.Init where import Control.Monad.State (liftIO) import Control.Monad (when) +import System.Directory import Command import qualified Annex -import Core import qualified GitRepo as Git import UUID import Version import Messages +import Locations seek :: [SubCmdSeek] seek = [withString start] @@ -46,3 +47,40 @@ cleanup = do liftIO $ Git.run g ["add", logfile] liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] return True + +{- configure git to use union merge driver on state files, if it is not + - already -} +gitAttributes :: Git.Repo -> IO () +gitAttributes repo = do + exists <- doesFileExist attributes + if (not exists) + then do + writeFile attributes $ attrLine ++ "\n" + commit + else do + content <- readFile attributes + when (all (/= attrLine) (lines content)) $ do + appendFile attributes $ attrLine ++ "\n" + commit + where + attrLine = stateLoc ++ "*.log merge=union" + attributes = Git.attributes repo + commit = do + Git.run repo ["add", attributes] + Git.run repo ["commit", "-m", "git-annex setup", + attributes] + +{- set up a git pre-commit hook, if one is not already present -} +gitPreCommitHook :: Git.Repo -> IO () +gitPreCommitHook repo = do + let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ + "/hooks/pre-commit" + exists <- doesFileExist hook + if (exists) + then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + else do + writeFile hook $ "#!/bin/sh\n" ++ + "# automatically configured by git-annex\n" ++ + "git annex pre-commit .\n" + p <- getPermissions hook + setPermissions hook $ p {executable = True} diff --git a/Core.hs b/Core.hs index 9faaada56f..2928dc06df 100644 --- a/Core.hs +++ b/Core.hs @@ -26,7 +26,6 @@ import qualified Annex import qualified Backend import Utility import Messages -import Version {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). @@ -46,11 +45,10 @@ tryRun' state errnum (a:as) = do tryRun' _ errnum [] = when (errnum > 0) $ error $ show errnum ++ " failed" -{- Sets up a git repo for git-annex. -} +{- Actions to perform each time ran. -} startup :: Annex Bool startup = do prepUUID - autoUpgrade return True {- When git-annex is done, it runs this. -} @@ -71,43 +69,6 @@ shutdown = do return True -{- configure git to use union merge driver on state files, if it is not - - already -} -gitAttributes :: Git.Repo -> IO () -gitAttributes repo = do - exists <- doesFileExist attributes - if (not exists) - then do - writeFile attributes $ attrLine ++ "\n" - commit - else do - content <- readFile attributes - when (all (/= attrLine) (lines content)) $ do - appendFile attributes $ attrLine ++ "\n" - commit - where - attrLine = stateLoc ++ "*.log merge=union" - attributes = Git.attributes repo - commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", - attributes] - -{- set up a git pre-commit hook, if one is not already present -} -gitPreCommitHook :: Git.Repo -> IO () -gitPreCommitHook repo = do - let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ - "/hooks/pre-commit" - exists <- doesFileExist hook - if (exists) - then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" - else do - writeFile hook $ "#!/bin/sh\n" ++ - "# automatically configured by git-annex\n" ++ - "git annex pre-commit .\n" - p <- getPermissions hook - setPermissions hook $ p {executable = True} - {- Checks if a given key is currently present in the annexLocation. -} inAnnex :: Key -> Annex Bool inAnnex key = do @@ -237,43 +198,3 @@ getKeysReferenced = do files <- liftIO $ Git.inRepo g $ Git.workTree g keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs - -{- Uses the annex.version git config setting to automate upgrades. -} -autoUpgrade :: Annex () -autoUpgrade = do - version <- getVersion - case version of - Just "0" -> upgradeFrom0 - Nothing -> return () -- repo not initted yet, no version - Just v | v == currentVersion -> return () - Just _ -> error "this version of git-annex is too old for this git repository!" - -upgradeFrom0 :: Annex () -upgradeFrom0 = do - showSideAction "Upgrading object directory layout..." - g <- Annex.gitRepo - - -- do the reorganisation of the files - let olddir = annexDir g - keys <- getKeysPresent' olddir - _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys - - -- update the symlinks to the files - files <- liftIO $ Git.inRepo g $ Git.workTree g - fixlinks files - Annex.queueRun - - setVersion - - where - fixlinks [] = return () - fixlinks (f:fs) = do - r <- Backend.lookupFile f - case r of - Nothing -> return () - Just (k, _) -> do - link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" ["--"] f - fixlinks fs diff --git a/Upgrade.hs b/Upgrade.hs new file mode 100644 index 0000000000..d64d5287d1 --- /dev/null +++ b/Upgrade.hs @@ -0,0 +1,63 @@ +{- git-annex upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade where + +import System.Directory +import Control.Monad.State (liftIO) +import System.Posix.Files + +import Core +import Types +import Locations +import qualified GitRepo as Git +import qualified Annex +import qualified Backend +import Messages +import Version + +{- Uses the annex.version git config setting to automate upgrades. -} +upgrade :: Annex Bool +upgrade = do + version <- getVersion + case version of + Just "0" -> upgradeFrom0 + Nothing -> return True -- repo not initted yet, no version + Just v | v == currentVersion -> return True + Just _ -> error "this version of git-annex is too old for this git repository!" + +upgradeFrom0 :: Annex Bool +upgradeFrom0 = do + showSideAction "Upgrading object directory layout..." + g <- Annex.gitRepo + + -- do the reorganisation of the files + let olddir = annexDir g + keys <- getKeysPresent' olddir + _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys + + -- update the symlinks to the files + files <- liftIO $ Git.inRepo g $ Git.workTree g + fixlinks files + Annex.queueRun + + setVersion + + return True + + where + fixlinks [] = return () + fixlinks (f:fs) = do + r <- Backend.lookupFile f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" ["--"] f + fixlinks fs diff --git a/git-annex.hs b/git-annex.hs index 098ccac2d4..d111156f01 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -9,6 +9,7 @@ import System.Environment import qualified Annex import Core +import Upgrade import CmdLine import qualified GitRepo as Git import BackendList @@ -19,4 +20,4 @@ main = do gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends (configure, actions) <- parseCmd args state - tryRun state $ [startup] ++ configure ++ actions ++ [shutdown] + tryRun state $ [startup, upgrade] ++ configure ++ actions ++ [shutdown] From 27ffa0e86f7c5537888563cdda77be45b2700478 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 14:58:42 -0400 Subject: [PATCH 0513/8313] comment --- Command.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command.hs b/Command.hs index 60d82d09fb..09c935b3bf 100644 --- a/Command.hs +++ b/Command.hs @@ -158,7 +158,7 @@ withAll w a params = do w a [Git.workTree g] else w a params -{- Provides a default parameter to a with search. -} +{- Provides a default parameter to act on if none is specified. -} withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings withDefault d w a params = do if null params From 0b57b68918cc9ba84196df188c76bfb92eb1b549 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 17:56:12 -0400 Subject: [PATCH 0514/8313] deal with OSX case brokenness HELLO, it's 2010.. POSIX calling.. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c94fbbc8f2..77e7a5edb8 100644 --- a/Makefile +++ b/Makefile @@ -32,4 +32,4 @@ clean: rm -rf build git-annex git-annex.1 test rm -rf doc/.ikiwiki html -.PHONY: git-annex test +.PHONY: git-annex test install From e87d77964445bb7045d4a82350ec9410457d62a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 14 Nov 2010 17:56:49 -0400 Subject: [PATCH 0515/8313] should work now --- doc/forum/git-annex_on_OSX.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn index 85a8413bf9..44c275ff1c 100644 --- a/doc/forum/git-annex_on_OSX.mdwn +++ b/doc/forum/git-annex_on_OSX.mdwn @@ -13,7 +13,7 @@ git clone git://git.kitenet.net/git-annex cd git-annex make -sudo install git-annex /usr/local/bin/ # makefile is weird +sudo make install Originally posted by Jon at --[[Joey]] From a51b364a354a3817298a9b0dfef4016850c175de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 13:00:43 -0400 Subject: [PATCH 0516/8313] trim exports --- LocationLog.hs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index 9a9dad1333..1ddbf3c0bb 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -23,9 +23,7 @@ module LocationLog ( LogStatus(..), logChange, - keyLocations, - logFile, - readLog + keyLocations ) where import Data.Time.Clock.POSIX From 0e1e32cb6b6f5fcd9e535d171c639f3c1663a6ce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 13:13:01 -0400 Subject: [PATCH 0517/8313] problem --- doc/todo/branching.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index 81f230a92a..ebcc56aa62 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -46,6 +46,7 @@ Let's use one branch per uuid, named git-annex/$UUID. - TODO: What will need to be done to git to make it push/pull these new branches? - A given repo only ever writes to its UUID branch. So no conflicts. + - **problem**: git annex move needs to update log info for other repos! - (BTW, UUIDs probably don't compress well, and this reduces the bloat of having them repeated lots of times in the tree.) - Per UUID branches mean that if it wants to find a file's location @@ -57,7 +58,7 @@ can cache location log changes and write them all to locationlog in a single git operation on shutdown. - TODO: what if it's ctrl-c'd with changes pending? Perhaps it should - collect them to ,git/annex/locationlog, and inject that file on shutdown? + collect them to .git/annex/locationlog, and inject that file on shutdown? - This will be less overhead than the current staging of all the log files. The log is not appended to, so in git we have a series of commits each of From 40ae21c895b095409ae9af3c44ab9a6b63ea10bb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 14:00:28 -0400 Subject: [PATCH 0518/8313] this is looking more and more problimatic --- doc/todo/branching.mdwn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index ebcc56aa62..c9707c6f9d 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -52,6 +52,11 @@ Let's use one branch per uuid, named git-annex/$UUID. - Per UUID branches mean that if it wants to find a file's location amoung configured remotes, it can examine only their branches, if desired. +- It's important that the per-repo branches propigate beyond immediate + remotes. If there is a central bare repo, that means push --all. Without + one, it means that when repo B pulls from A, and then C pulls from B, + C needs to get A's branch -- which means that B should have a tracking + branch for A's branch. In the branch, only one file is needed. Call it locationlog. git-annex can cache location log changes and write them all to locationlog in @@ -73,3 +78,18 @@ with a indication of the presense/absense of the key is found. - It could get pretty slow when digging deeper. - Only 3 places in git-annex will be affected by any slowdown: move --from, get and drop. + +## alternate + +As above, but use a single git-annex branch, and keep the per-UUID +info in their own log files. Hope that git can auto-merge as long as +each observing repo only writes to its own files. (Well, it can, but for +non-fast-forward merges, the git-annex branch would need to be checked out, +which is problimatic.) + +Use filenames like: + + / + +That allows one repo to record another's state when doing a +`move`. From c305fb2bf318e72ea4f905a38733d93e9a81cc1d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 15:21:11 -0400 Subject: [PATCH 0519/8313] thought --- doc/todo/branching.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index c9707c6f9d..b7441c6e4e 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -93,3 +93,15 @@ Use filenames like: That allows one repo to record another's state when doing a `move`. + +## outside the box approach + +If the problem is limited to only that the `.git-annex/` files make +branching difficult (and not to the related problem that commits to them +and having them in the tree are sorta annoying), then a simple approach +would be to have git-annex look in other branches for location log info +too. + +The problem would then be that any locationlog lookup would need to look in +all other branches (any branch could have more current info after all), +which could get expensive. From 9dc43d25995b085745c16cc31ad11106a74a9973 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 16:35:06 -0400 Subject: [PATCH 0520/8313] unused: New subcommand, finds unused data (the global part of fsck). --- CmdLine.hs | 4 +++ Command/Fsck.hs | 52 ++---------------------------------- Command/Unused.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++++ debian/changelog | 1 + doc/git-annex.mdwn | 5 ++++ 5 files changed, 78 insertions(+), 50 deletions(-) create mode 100644 Command/Unused.hs diff --git a/CmdLine.hs b/CmdLine.hs index caf7279463..cc163fff52 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -25,6 +25,7 @@ import qualified Command.SetKey import qualified Command.Fix import qualified Command.Init import qualified Command.Fsck +import qualified Command.Unused import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit @@ -62,6 +63,8 @@ subCmds = "fix up symlinks to point to annexed content" , SubCommand "fsck" maybepath Command.Fsck.seek "check for problems" + , SubCommand "unused" nothing Command.Unused.seek + "look for unused file content" , SubCommand "find" maybepath Command.Find.seek "lists available files" ] @@ -70,6 +73,7 @@ subCmds = maybepath = "[PATH ...]" key = "KEY ..." desc = "DESCRIPTION" + nothing = "" -- Each dashed command-line option results in generation of an action -- in the Annex monad that performs the necessary setting. diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 5b731a6968..02b66d01ad 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -14,55 +14,7 @@ import Types import Core import Messages import qualified Command.FsckFile +import qualified Command.Unused seek :: [SubCmdSeek] -seek = [withNothing start, withAll withFilesInGit Command.FsckFile.start] - -{- Checks the whole annex for problems, only if specific files were not - - specified. -} -start :: SubCmdStartNothing -start = do - showStart "fsck" "" - return $ Just perform - -perform :: SubCmdPerform -perform = do - ok <- checkUnused - if ok - then return $ Just $ return True - else return Nothing - -checkUnused :: Annex Bool -checkUnused = do - showNote "checking for unused data..." - unused <- unusedKeys - if (null unused) - then return True - else do - showLongNote $ w unused - return False - where - w u = unlines $ [ - "Some annexed data is no longer pointed to by any files in the repository.", - "If this data is no longer needed, it can be removed using git-annex dropkey:" - ] ++ map (\k -> " " ++ show k) u - -{- Finds keys whose content is present, but that do not seem to be used - - by any files in the git repo. -} -unusedKeys :: Annex [Key] -unusedKeys = do - present <- getKeysPresent - referenced <- getKeysReferenced - - -- Constructing a single map, of the set that tends to be smaller, - -- appears more efficient in both memory and CPU than constructing - -- and taking the M.difference of two maps. - let present_m = existsMap present - let unused_m = remove referenced present_m - return $ M.keys unused_m - where - remove [] m = m - remove (x:xs) m = remove xs $ M.delete x m - -existsMap :: Ord k => [k] -> M.Map k Int -existsMap l = M.fromList $ map (\k -> (k, 1)) l +seek = [withNothing Command.Unused.start, withAll withFilesInGit Command.FsckFile.start] diff --git a/Command/Unused.hs b/Command/Unused.hs new file mode 100644 index 0000000000..ed3de5d575 --- /dev/null +++ b/Command/Unused.hs @@ -0,0 +1,66 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Unused where + +import qualified Data.Map as M + +import Command +import Types +import Core +import Messages + +seek :: [SubCmdSeek] +seek = [withNothing start] + +{- Finds unused content in the annex. -} +start :: SubCmdStartNothing +start = do + showStart "unused" "" + return $ Just perform + +perform :: SubCmdPerform +perform = do + ok <- checkUnused + if ok + then return $ Just $ return True + else return Nothing + +checkUnused :: Annex Bool +checkUnused = do + showNote "checking for unused data..." + unused <- unusedKeys + if (null unused) + then return True + else do + showLongNote $ w unused + return False + where + w u = unlines $ [ + "Some annexed data is no longer pointed to by any files in the repository.", + "If this data is no longer needed, it can be removed using git-annex dropkey:" + ] ++ map (\k -> " " ++ show k) u + +{- Finds keys whose content is present, but that do not seem to be used + - by any files in the git repo. -} +unusedKeys :: Annex [Key] +unusedKeys = do + present <- getKeysPresent + referenced <- getKeysReferenced + + -- Constructing a single map, of the set that tends to be smaller, + -- appears more efficient in both memory and CPU than constructing + -- and taking the M.difference of two maps. + let present_m = existsMap present + let unused_m = remove referenced present_m + return $ M.keys unused_m + where + remove [] m = m + remove (x:xs) m = remove xs $ M.delete x m + +existsMap :: Ord k => [k] -> M.Map k Int +existsMap l = M.fromList $ map (\k -> (k, 1)) l diff --git a/debian/changelog b/debian/changelog index 07eceae6cd..dcdbe15e27 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.07) UNRELEASED; urgency=low * find: New subcommand. + * unused: New subcommand, finds unused data (the global part of fsck). -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index c496ca0d8c..a522534da8 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -166,6 +166,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. With parameters, only the specified files are checked. +* unused + + Checks the annex for data that is not used by any files currently + in the annex, and prints a report. + * find [path ...] Outputs a list of annexed files whose content is currently present. From 748a7475bb99e1127dc12bb2cc9d5653e4648200 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:04:19 -0400 Subject: [PATCH 0521/8313] dropunused --- Backend.hs | 13 +++++++--- CmdLine.hs | 4 +++ Command.hs | 2 ++ Command/Fsck.hs | 5 ---- Command/Unused.hs | 24 +++++++++++++---- Locations.hs | 9 +++++-- debian/changelog | 2 ++ doc/git-annex.mdwn | 61 ++++++++++++++++++++++++-------------------- doc/walkthrough.mdwn | 10 +++++--- 9 files changed, 84 insertions(+), 46 deletions(-) diff --git a/Backend.hs b/Backend.hs index 14af56bbfa..2f0f71d749 100644 --- a/Backend.hs +++ b/Backend.hs @@ -25,7 +25,8 @@ module Backend ( hasKey, fsckKey, lookupFile, - chooseBackends + chooseBackends, + keyBackend ) where import Control.Monad.State @@ -111,8 +112,8 @@ removeKey backend key = (Internals.removeKey backend) key {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool hasKey key = do - bs <- Annex.supportedBackends - (Internals.hasKey (lookupBackendName bs $ backendName key)) key + backend <- keyBackend key + (Internals.hasKey backend) key {- Checks a key's backend for problems. -} fsckKey :: Backend -> Key -> Annex Bool @@ -154,3 +155,9 @@ chooseBackends fs = do bs <- Annex.supportedBackends pairs <- liftIO $ Git.checkAttr g "git-annex-backend" fs return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs + +{- Returns the backend to use for a key. -} +keyBackend :: Key -> Annex Backend +keyBackend key = do + bs <- Annex.supportedBackends + return $ lookupBackendName bs $ backendName key diff --git a/CmdLine.hs b/CmdLine.hs index cc163fff52..35e889d7df 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -26,6 +26,7 @@ import qualified Command.Fix import qualified Command.Init import qualified Command.Fsck import qualified Command.Unused +import qualified Command.DropUnused import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit @@ -65,6 +66,8 @@ subCmds = "check for problems" , SubCommand "unused" nothing Command.Unused.seek "look for unused file content" + , SubCommand "dropunused" number Command.DropUnused.seek + "drop unused file content" , SubCommand "find" maybepath Command.Find.seek "lists available files" ] @@ -73,6 +76,7 @@ subCmds = maybepath = "[PATH ...]" key = "KEY ..." desc = "DESCRIPTION" + number = "NUMBER ..." nothing = "" -- Each dashed command-line option results in generation of an action diff --git a/Command.hs b/Command.hs index 09c935b3bf..c6d2d0d5d4 100644 --- a/Command.hs +++ b/Command.hs @@ -129,6 +129,8 @@ backendPairs a files = do return $ map a pairs withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] +withStrings :: SubCmdSeekStrings +withStrings a params = return $ map a params withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 02b66d01ad..a72d753fa0 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,12 +7,7 @@ module Command.Fsck where -import qualified Data.Map as M - import Command -import Types -import Core -import Messages import qualified Command.FsckFile import qualified Command.Unused diff --git a/Command/Unused.hs b/Command/Unused.hs index ed3de5d575..7a34d393cd 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -7,12 +7,15 @@ module Command.Unused where +import Control.Monad.State (liftIO) import qualified Data.Map as M import Command import Types import Core import Messages +import Locations +import qualified Annex seek :: [SubCmdSeek] seek = [withNothing start] @@ -37,13 +40,24 @@ checkUnused = do if (null unused) then return True else do - showLongNote $ w unused + let list = number 1 unused + g <- Annex.gitRepo + liftIO $ writeFile (annexUnusedLog g) $ unlines $ + map (\(n, k) -> show n ++ " " ++ show k) list + showLongNote $ w list return False where - w u = unlines $ [ - "Some annexed data is no longer pointed to by any files in the repository.", - "If this data is no longer needed, it can be removed using git-annex dropkey:" - ] ++ map (\k -> " " ++ show k) u + w u = unlines $ + ["Some annexed data is no longer pointed to by any files in the repository:", + " NUMBER KEY"] + ++ (map (\(n, k) -> " " ++ (pad 6 $ show n) ++ " " ++ show k) u) ++ + ["(To see where data was previously used, try: git log --stat -S'KEY')", + "(To remove unwanted data: git-annex dropunused NUMBER)"] + pad n s = s ++ replicate (n - length s) ' ' + +number :: Integer -> [a] -> [(Integer, a)] +number _ [] = [] +number n (x:xs) = (n, x):(number (n+1) xs) {- Finds keys whose content is present, but that do not seem to be used - by any files in the git repo. -} diff --git a/Locations.hs b/Locations.hs index c3bab285d4..24ccc75c6d 100644 --- a/Locations.hs +++ b/Locations.hs @@ -14,6 +14,7 @@ module Locations ( annexLocationRelative, annexTmpLocation, annexBadLocation, + annexUnusedLog, annexDir, annexObjectDir, @@ -56,14 +57,18 @@ annexDir r = Git.workTree r ++ "/.git/annex" annexObjectDir :: Git.Repo -> FilePath annexObjectDir r = annexDir r ++ "/objects" -{- .git-annex/tmp is used for temp files -} +{- .git-annex/tmp/ is used for temp files -} annexTmpLocation :: Git.Repo -> FilePath annexTmpLocation r = annexDir r ++ "/tmp/" -{- .git-annex/bad is used for bad files found during fsck -} +{- .git-annex/bad/ is used for bad files found during fsck -} annexBadLocation :: Git.Repo -> FilePath annexBadLocation r = annexDir r ++ "/bad/" +{- .git/annex/unused is used to number possibly unused keys -} +annexUnusedLog :: Git.Repo -> FilePath +annexUnusedLog r = annexDir r ++ "/unused" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/debian/changelog b/debian/changelog index dcdbe15e27..cffa7bb261 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ git-annex (0.07) UNRELEASED; urgency=low * find: New subcommand. * unused: New subcommand, finds unused data (the global part of fsck). + * dropunused: New subcommand, provides for easy dropping of unused keys + by number, as listed by unused subcommand. -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a522534da8..618ddf2039 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -112,6 +112,32 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Use this to undo an unlock command if you don't want to modify the files, or have made modifications you want to discard. +* fsck [path ...] + + With no parameters, this subcommand checks the whole annex for consistency, + and warns about any problems found. + + With parameters, only the specified files are checked. + +* unused + + Checks the annex for data that is not used by any files currently + in the annex, and prints a numbered list of the data. + + (This is run as part of `git annex fsck`.) + +* dropunused [number ...] + + Drops the data corresponding to the numbers, as listed by the last + `git annex unused` or `git annex fsck` + +* find [path ...] + + Outputs a list of annexed files whose content is currently present. + + With no parameters, defaults to finding all files in the current directory + and its subdirectories. + * unannex [path ...] Use this to undo an accidental add command. This is not the command you @@ -159,25 +185,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. git annex setkey --key=1287765018:3 /tmp/file -* fsck [path ...] - - With no parameters, this subcommand checks the whole annex for consistency, - and warns about any problems found. - - With parameters, only the specified files are checked. - -* unused - - Checks the annex for data that is not used by any files currently - in the annex, and prints a report. - -* find [path ...] - - Outputs a list of annexed files whose content is currently present. - - With no parameters, defaults to finding all files in the current directory - and its subdirectories. - # OPTIONS * --force @@ -194,14 +201,6 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Enable verbose logging. -* --backend=name - - Specifies the key-value backend to use when adding a file. - -* --key=name - - Specifies a key to operate on, for use with the addkey subcommand. - * --from=repository Specifies a repository that content will be retrieved from. @@ -212,6 +211,14 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a git repository that content will be sent to. It should be specified using the name of a configured git remote. +* --backend=name + + Specifies which key-value backend to use. + +* --key=name + + Specifies a key to operate on. + # CONFIGURATION Like other git commands, git-annex is configured via `git-config`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 9a6ee220c2..1e07053aec 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -287,7 +287,7 @@ setting is satisfied for all files, and it warns about any dangling values in `.git/annex/objects/`. # git annex fsck - fsck (checking for unused data...) ok + unused (checking for unused data...) ok fsck my_cool_big_file (checksum...) ok ...... @@ -304,10 +304,12 @@ Fsck never deletes possibly bad data; instead it will be moved to might say about a badly messed up annex: # git annex fsck - fsck (checking for unused data...) + unused (checking for unused data...) Some annexed data is no longer pointed to by any files in the repository. - If this data is no longer needed, it can be removed using git-annex dropkey: - WORM:1289672605:3:file + NUMBER KEY + 1 WORM:1289672605:3:file + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) failed fsck my_cool_big_file (checksum...) Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 From 3a4e9398a1652b664fb17dc2072f4b85966dcb61 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:06:21 -0400 Subject: [PATCH 0522/8313] add --- Command/DropUnused.hs | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 Command/DropUnused.hs diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs new file mode 100644 index 0000000000..e8b8d43c4c --- /dev/null +++ b/Command/DropUnused.hs @@ -0,0 +1,46 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.DropUnused where + +import Control.Monad.State (liftIO) +import qualified Data.Map as M + +import Command +import Types +import Messages +import Locations +import qualified Annex +import qualified Command.Drop +import Backend + +seek :: [SubCmdSeek] +seek = [withStrings start] + +{- Drops unused content by number. -} +start :: SubCmdStartString +start s = do + m <- readUnusedLog + case M.lookup s m of + Nothing -> return Nothing + Just key -> do + showStart "dropunused" s + backend <- keyBackend key + -- force drop, even if this is the only copy + Annex.flagChange "force" $ FlagBool True + return $ Just $ Command.Drop.perform key backend + +readUnusedLog :: Annex (M.Map String Key) +readUnusedLog = do + g <- Annex.gitRepo + l <- liftIO $ readFile (annexUnusedLog g) + return $ M.fromList $ map parse $ lines l + where + parse line = (head ws, tokey $ unwords $ tail ws) + where + ws = words line + tokey s = read s :: Key From 354be7a00b76b1ee961fa31ba81905e1d8c172bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:13:20 -0400 Subject: [PATCH 0523/8313] on dropunused and unused --- debian/changelog | 4 ++-- doc/walkthrough.mdwn | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index cffa7bb261..b6a426d1ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ git-annex (0.07) UNRELEASED; urgency=low * find: New subcommand. - * unused: New subcommand, finds unused data (the global part of fsck). + * unused: New subcommand, finds unused data. (Split out from fsck.) * dropunused: New subcommand, provides for easy dropping of unused keys - by number, as listed by unused subcommand. + by number, as listed by the unused subcommand. -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 1e07053aec..b6f322c499 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -277,6 +277,34 @@ add something like this to `.gitattributes`: * git-annex-backend=SHA1 +## unused data + +It's possible for data to accumulate in the annex that no files point to +nymore. One way it can happen is if you `git rm` a file without +first calling `git annex drop`. And, when you modify an annexed file, the old +content of the file remains in the annex. + +This might be historical data you want to preserve, so git-annex defaults to +preserving it. So from time to time, you may want to check for such data and +eliminate it to save space. + + # git annex unused + unused (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository. + NUMBER KEY + 1 WORM:1289672605:3:file + 2 WORM:1289672605:14:file + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) + failed + +After running `git annex unused`, you can follow the instructions to examine +the history of files that used the data, and if you decide you don't need that +data anymore, you can easily remove it: + + # git annex dropunused 1 + dropunused 1 ok + ## fsck: verifying your data You can use the fsck subcommand to check for problems in your data. From d6f7ee7db46208d132f305ab651b0bf082be348e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:19:13 -0400 Subject: [PATCH 0524/8313] clearify --- doc/git-annex.mdwn | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 618ddf2039..42f80a3dc2 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -171,19 +171,30 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * dropkey [key ...] - Drops the annexed data for the specified keys from this repository. + This plumbing-level command drops the annexed data for the specified + keys from this repository. This can be used to drop content for arbitrary keys, which do not need to have a file in the git repository pointing at them. -* setkey file - - Sets the annxed data for a key to the content of the specified file, - and then removes the file. + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. Example: - git annex setkey --key=1287765018:3 /tmp/file + git annex dropkey --backend=SHA1 7da006579dd64330eb2456001fd01948430572f2 + +* setkey file + + This plumbing-level command sets the annxed data for a key to the content of + the specified file, and then removes the file. + + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. + + Example: + + git annex setkey --backend=WORM --key=1287765018:3 /tmp/file # OPTIONS From 11096c200f4050c1e4cdb4fbab9410055c6e1c02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:22:50 -0400 Subject: [PATCH 0525/8313] fsck no longer runs unused --- Command/Fsck.hs | 20 +++++++++++++++++--- doc/git-annex.mdwn | 4 +--- doc/walkthrough.mdwn | 12 ++---------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index a72d753fa0..07621ad4ef 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -8,8 +8,22 @@ module Command.Fsck where import Command -import qualified Command.FsckFile -import qualified Command.Unused +import qualified Backend +import Types +import Messages seek :: [SubCmdSeek] -seek = [withNothing Command.Unused.start, withAll withFilesInGit Command.FsckFile.start] +seek = [withFilesInGit start] + +{- Checks a file's backend data for problems. -} +start :: SubCmdStartString +start file = isAnnexed file $ \(key, backend) -> do + showStart "fsck" file + return $ Just $ perform key backend + +perform :: Key -> Backend -> SubCmdPerform +perform key backend = do + success <- Backend.fsckKey backend key + if (success) + then return $ Just $ return True + else return Nothing diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 42f80a3dc2..3df835eac5 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -124,12 +124,10 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Checks the annex for data that is not used by any files currently in the annex, and prints a numbered list of the data. - (This is run as part of `git annex fsck`.) - * dropunused [number ...] Drops the data corresponding to the numbers, as listed by the last - `git annex unused` or `git annex fsck` + `git annex unused` * find [path ...] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b6f322c499..a1a8882425 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -311,8 +311,7 @@ You can use the fsck subcommand to check for problems in your data. What can be checked depends on the [[backend|backends]] you've used to store the data. For example, when you use the SHA1 backend, fsck will verify that the checksums of your files are good. Fsck also checks that the annex.numcopies -setting is satisfied for all files, and it warns about any dangling values -in `.git/annex/objects/`. +setting is satisfied for all files. # git annex fsck unused (checking for unused data...) ok @@ -332,13 +331,6 @@ Fsck never deletes possibly bad data; instead it will be moved to might say about a badly messed up annex: # git annex fsck - unused (checking for unused data...) - Some annexed data is no longer pointed to by any files in the repository. - NUMBER KEY - 1 WORM:1289672605:3:file - (To see where data was previously used, try: git log --stat -S'KEY') - (To remove unwanted data: git-annex dropunused NUMBER) - failed fsck my_cool_big_file (checksum...) Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 ** No known copies of the file exist! @@ -346,4 +338,4 @@ might say about a badly messed up annex: fsck important_file Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. failed - git-annex: 3 failed + git-annex: 2 failed From a5e7f5329f4d9a80a34302e3707f0bd42b91a441 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:24:29 -0400 Subject: [PATCH 0526/8313] fix --- Command/Fsck.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 07621ad4ef..6291d5ba3f 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,7 +13,7 @@ import Types import Messages seek :: [SubCmdSeek] -seek = [withFilesInGit start] +seek = [withAll withFilesInGit start] {- Checks a file's backend data for problems. -} start :: SubCmdStartString From 0893820812c9cc10287c861dccbdae3c287cd7cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:37:49 -0400 Subject: [PATCH 0527/8313] fsck: Print warnings to stderr; --quiet can now be used to only see problems. --- Backend/File.hs | 4 ++-- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- Command/Unused.hs | 6 ++---- Messages.hs | 4 +++- debian/changelog | 2 ++ doc/walkthrough.mdwn | 10 +++++----- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 8351778568..c67fb3ce33 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -175,12 +175,12 @@ checkKeyNumCopies key = do let present = length remotes + if inannex then 1 else 0 if (present < needed) then do - showLongNote $ note present needed + warning $ note present needed return False else return True where note 0 _ = "** No known copies of the file exist!" note present needed = "Only " ++ show present ++ " of " ++ show needed ++ - " copies exist. " ++ + " copies of "++show key++" exist. " ++ "Run git annex get somewhere else to back it up." diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 9e9000ba93..46667c9cda 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -57,5 +57,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - showLongNote $ "Bad file content; moved to "++dest + warning $ "Bad file content; moved to "++dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 3749899968..4e2177fed0 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -66,5 +66,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - showLongNote $ "Bad file size; moved to "++dest + warning $ "Bad file size; moved to "++dest return False diff --git a/Command/Unused.hs b/Command/Unused.hs index 7a34d393cd..ae189550ce 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -28,10 +28,8 @@ start = do perform :: SubCmdPerform perform = do - ok <- checkUnused - if ok - then return $ Just $ return True - else return Nothing + _ <- checkUnused + return $ Just $ return True checkUnused :: Annex Bool checkUnused = do diff --git a/Messages.hs b/Messages.hs index ed4f3b90a1..7dafa8284e 100644 --- a/Messages.hs +++ b/Messages.hs @@ -54,4 +54,6 @@ showErr :: (Show a) => a -> Annex () showErr e = warning $ show e warning :: String -> Annex () -warning s = liftIO $ hPutStrLn stderr $ "git-annex: " ++ s +warning s = do + verbose $ liftIO $ putStr "\n" + liftIO $ hPutStrLn stderr $ "git-annex: " ++ s diff --git a/debian/changelog b/debian/changelog index b6a426d1ff..a6d95ab69e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,8 @@ git-annex (0.07) UNRELEASED; urgency=low * unused: New subcommand, finds unused data. (Split out from fsck.) * dropunused: New subcommand, provides for easy dropping of unused keys by number, as listed by the unused subcommand. + * fsck: Print warnings to stderr; --quiet can now be used to only see + problems. -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index a1a8882425..281f460506 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -296,7 +296,7 @@ eliminate it to save space. 2 WORM:1289672605:14:file (To see where data was previously used, try: git log --stat -S'KEY') (To remove unwanted data: git-annex dropunused NUMBER) - failed + ok After running `git annex unused`, you can follow the instructions to examine the history of files that used the data, and if you decide you don't need that @@ -316,7 +316,7 @@ setting is satisfied for all files. # git annex fsck unused (checking for unused data...) ok fsck my_cool_big_file (checksum...) ok - ...... + ... You can also specifiy the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. @@ -332,10 +332,10 @@ might say about a badly messed up annex: # git annex fsck fsck my_cool_big_file (checksum...) - Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 - ** No known copies of the file exist! + git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 + git-annex: ** No known copies of the file exist! failed fsck important_file - Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. + git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. failed git-annex: 2 failed From 625972d2b3340b6a41d1d47f631c80d6021025dc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:38:41 -0400 Subject: [PATCH 0528/8313] this is kinda weird, but it avoids blank lines after warnings --- Messages.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Messages.hs b/Messages.hs index 7dafa8284e..f9d12c2221 100644 --- a/Messages.hs +++ b/Messages.hs @@ -56,4 +56,4 @@ showErr e = warning $ show e warning :: String -> Annex () warning s = do verbose $ liftIO $ putStr "\n" - liftIO $ hPutStrLn stderr $ "git-annex: " ++ s + liftIO $ hPutStr stderr $ "git-annex: " ++ s ++ " " From 38a75aa26ff70f4ebe5a88eb009d512ff7211163 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:42:32 -0400 Subject: [PATCH 0529/8313] close --- doc/todo/gitrm.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/todo/gitrm.mdwn b/doc/todo/gitrm.mdwn index d771aa32a7..e41c334623 100644 --- a/doc/todo/gitrm.mdwn +++ b/doc/todo/gitrm.mdwn @@ -1,2 +1,5 @@ how to handle git rm file? (should try to drop keys that have no referring file, if it seems safe..) + +[[done]] -- I think that git annex unused and dropunused are the best +solution to this. From 2b7203c5d2a5a22832778d3b3b16bcd8fd7242f1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:43:59 -0400 Subject: [PATCH 0530/8313] releasing version 0.07 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index a6d95ab69e..252d9ae45b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.07) UNRELEASED; urgency=low +git-annex (0.07) unstable; urgency=low * find: New subcommand. * unused: New subcommand, finds unused data. (Split out from fsck.) @@ -7,7 +7,7 @@ git-annex (0.07) UNRELEASED; urgency=low * fsck: Print warnings to stderr; --quiet can now be used to only see problems. - -- Joey Hess Sun, 14 Nov 2010 12:34:49 -0400 + -- Joey Hess Mon, 15 Nov 2010 18:41:50 -0400 git-annex (0.06) unstable; urgency=low From d792024e37895b25f8a00916c97b5cd133582eee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 15 Nov 2010 18:44:06 -0400 Subject: [PATCH 0531/8313] add news item for git-annex 0.07 --- doc/news/version_0.02.mdwn | 18 ------------------ doc/news/version_0.07.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 18 deletions(-) delete mode 100644 doc/news/version_0.02.mdwn create mode 100644 doc/news/version_0.07.mdwn diff --git a/doc/news/version_0.02.mdwn b/doc/news/version_0.02.mdwn deleted file mode 100644 index 689124ab4e..0000000000 --- a/doc/news/version_0.02.mdwn +++ /dev/null @@ -1,18 +0,0 @@ -git-annex 0.02 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Can scp annexed files from remote hosts, and check remote hosts for - file content when dropping files. - * New move subcommand, that makes it easy to move file contents from - or to a remote. - * New fromkey subcommand, for registering urls, etc. - * git-annex init will now set up a pre-commit hook that fixes up symlinks - before they are committed, to ensure that moving symlinks around does not - break them. - * More intelligent and fast staging of modified files; git add coalescing. - * Add remote.annex-ignore git config setting to allow completly disabling - a given remote. - * --from/--to can be used to control the remote repository that git-annex - uses. - * --quiet can be used to avoid verbose output - * New plumbing-level dropkey and addkey subcommands. - * Lots of bug fixes."""]] \ No newline at end of file diff --git a/doc/news/version_0.07.mdwn b/doc/news/version_0.07.mdwn new file mode 100644 index 0000000000..29b4736b92 --- /dev/null +++ b/doc/news/version_0.07.mdwn @@ -0,0 +1,8 @@ +git-annex 0.07 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * find: New subcommand. + * unused: New subcommand, finds unused data. (Split out from fsck.) + * dropunused: New subcommand, provides for easy dropping of unused keys + by number, as listed by the unused subcommand. + * fsck: Print warnings to stderr; --quiet can now be used to only see + problems."""]] \ No newline at end of file From 4dfd134ccb51931e09326d2bd53f2e337638c1ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 16 Nov 2010 18:00:42 -0400 Subject: [PATCH 0532/8313] flattr button --- doc/index.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index d863a9ede9..5cd6b45494 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -23,6 +23,10 @@ To get a feel for it, see the [[walkthrough]]. * [[todo]] * [[forum]] * [[contact]] + + + """]] ## sample use cases From 535755cf12a063ae19dca65d79614913084e499a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 16 Nov 2010 18:46:55 -0400 Subject: [PATCH 0533/8313] remove --- doc/news/prerelease.mdwn | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 doc/news/prerelease.mdwn diff --git a/doc/news/prerelease.mdwn b/doc/news/prerelease.mdwn deleted file mode 100644 index 3a1227ff0a..0000000000 --- a/doc/news/prerelease.mdwn +++ /dev/null @@ -1,9 +0,0 @@ -git-annex is not yet formally released. If you [[download]] it from git, -everything described on this web site should already work, and I hope work -well, with these exceptions: - -* The SHA1 backend is incomplete. - -I reserve the right to change its data formats before the first release, -though I already have enough drives using it that I'd probably have to -write a transition tool anyway! --[[Joey]] From ab55689164a2fa29af61bce692cfe5df2739b790 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 17 Nov 2010 13:46:50 -0400 Subject: [PATCH 0534/8313] simplifiy --- Utility.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Utility.hs b/Utility.hs index 0053c687bb..33db4bb086 100644 --- a/Utility.hs +++ b/Utility.hs @@ -39,8 +39,7 @@ parentDir dir = then slash ++ (join s $ take ((length dirs) - 1) dirs) else "" where - dirs = filter (\x -> length x > 0) $ - split s dir + dirs = filter (\x -> not $ null x) $ split s dir slash = if (not $ isAbsolute dir) then "" else s s = [pathSeparator] From 5c7d1b027916ce3fc207329f926041d2bcad3bcd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 17 Nov 2010 13:55:38 -0400 Subject: [PATCH 0535/8313] Fix `git annex add ../foo` (when ran in a subdir of the repo). There was no reason for Git.relative to be used here. --- Backend.hs | 14 ++++++-------- debian/changelog | 6 ++++++ doc/bugs/dotdot_problem.mdwn | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Backend.hs b/Backend.hs index 2f0f71d749..f24347ca8d 100644 --- a/Backend.hs +++ b/Backend.hs @@ -79,17 +79,15 @@ maybeLookupBackendName bs s = {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend)) storeFileKey file trybackend = do - g <- Annex.gitRepo - let relfile = Git.relative g file bs <- list let bs' = case trybackend of Nothing -> bs Just backend -> backend:bs - storeFileKey' bs' file relfile -storeFileKey' :: [Backend] -> FilePath -> FilePath -> Annex (Maybe (Key, Backend)) -storeFileKey' [] _ _ = return Nothing -storeFileKey' (b:bs) file relfile = do - result <- (Internals.getKey b) relfile + storeFileKey' bs' file +storeFileKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend)) +storeFileKey' [] _ = return Nothing +storeFileKey' (b:bs) file = do + result <- (Internals.getKey b) file case result of Nothing -> nextbackend Just key -> do @@ -98,7 +96,7 @@ storeFileKey' (b:bs) file relfile = do then nextbackend else return $ Just (key, b) where - nextbackend = storeFileKey' bs file relfile + nextbackend = storeFileKey' bs file {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} diff --git a/debian/changelog b/debian/changelog index 252d9ae45b..2a8871c30e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.08) UNRELEASED; urgency=low + + * Fix `git annex add ../foo` (when ran in a subdir of the repo). + + -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 + git-annex (0.07) unstable; urgency=low * find: New subcommand. diff --git a/doc/bugs/dotdot_problem.mdwn b/doc/bugs/dotdot_problem.mdwn index 9d247a9c09..cbefd5dae5 100644 --- a/doc/bugs/dotdot_problem.mdwn +++ b/doc/bugs/dotdot_problem.mdwn @@ -1,2 +1,4 @@ cannot "git annex ../foo" (GitRepo.relative is buggy and git-ls-files also refuses w/o --full-name, which would need other changes) + +[[done]] From 54513c69baffa40f2fcce42eb8651fdd98e05277 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 13:30:42 -0400 Subject: [PATCH 0536/8313] Add configure step to build process. * configure: Check to see if cp -a can be used. * configure: Check to see if cp --reflink=auto can be used. --- .gitignore | 2 ++ Makefile | 8 +++-- configure.hs | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ debian/changelog | 3 ++ 4 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 configure.hs diff --git a/.gitignore b/.gitignore index 6956c49dd2..a4cac10f43 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ build/* test +configure +SysConfig.hs git-annex git-annex.1 doc/.ikiwiki diff --git a/Makefile b/Makefile index 77e7a5edb8..685044146f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,11 @@ all: git-annex docs ghcmake=ghc -Wall -odir build -hidir build -O2 --make -git-annex: +SysConfig.hs: + $(ghcmake) configure + ./configure + +git-annex: SysConfig.hs $(ghcmake) git-annex install: @@ -29,7 +33,7 @@ docs: --disable-plugin=smiley clean: - rm -rf build git-annex git-annex.1 test + rm -rf build git-annex git-annex.1 test configure SysConfig.hs rm -rf doc/.ikiwiki html .PHONY: git-annex test install diff --git a/configure.hs b/configure.hs new file mode 100644 index 0000000000..fa07be3ab5 --- /dev/null +++ b/configure.hs @@ -0,0 +1,78 @@ +{- Checks system configuration and generates SysConfig.hs. + -} + +import System.IO +import System.Cmd +import System.Exit +import System.Directory + +type Test = IO Bool +data TestDesc = TestDesc String String Test +data Config = Config String Bool + +tests :: [TestDesc] +tests = [ + TestDesc "cp -a" "cp_a" cp_a + , TestDesc "cp --reflink" "cp_reflink" cp_reflink + ] + +tmpDir :: String +tmpDir = "tmp" + +testFile :: String +testFile = tmpDir ++ "/testfile" + +quiet :: String -> String +quiet s = s ++ " 2>/dev/null" + +cp_a :: Test +cp_a = testCmd $ quiet $ "cp -a " ++ testFile ++ " " ++ testFile ++ ".new" + +cp_reflink :: Test +cp_reflink = testCmd $ quiet $ "cp --reflink=auto " ++ testFile ++ " " ++ testFile ++ ".new" + +testCmd :: String -> Test +testCmd c = do + ret <- system c + return $ ret == ExitSuccess + +testStart :: String -> IO () +testStart s = do + putStr $ " checking " ++ s ++ "..." + hFlush stdout + +testEnd :: Bool -> IO () +testEnd r = putStrLn $ " " ++ (show r) + +writeSysConfig :: [Config] -> IO () +writeSysConfig config = do + writeFile "SysConfig.hs" $ unlines $ header ++ vars config ++ footer + where + header = [ + "{- Automatically generated by configure. -}" + , "module SysConfig where" + ] + footer = [] + vars [] = [] + vars (c:cs) = showvar c ++ vars cs + showvar (Config name val) = [ + name ++ " :: Bool" + , name ++ " = " ++ show val + ] + +runTests :: [TestDesc] -> IO [Config] +runTests [] = return [] +runTests ((TestDesc tname key t):ts) = do + testStart tname + val <- t + testEnd val + rest <- runTests ts + return $ (Config key val):rest + +main :: IO () +main = do + createDirectoryIfMissing True tmpDir + writeFile testFile "test file contents" + config <- runTests tests + removeDirectoryRecursive tmpDir + writeSysConfig config diff --git a/debian/changelog b/debian/changelog index 2a8871c30e..e6549c1f08 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,9 @@ git-annex (0.08) UNRELEASED; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). + * Add configure step to build process. + * configure: Check to see if cp -a can be used. + * configure: Check to see if cp --reflink=auto can be used. -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 From 161823d6eaff2adb7b99475b0edfe819fde11be1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 13:48:28 -0400 Subject: [PATCH 0537/8313] Only use cp -a if it is supported, falling back to cp -p or plain cp. * cp --reflink=auto is used if supported, and will make git annex unlock much faster on filesystems like btrfs that support copy of write. --- Command/Unlock.hs | 7 ++++--- CopyFile.hs | 24 ++++++++++++++++++++++++ Makefile | 2 +- Remotes.hs | 7 ++++--- configure.hs | 18 +++++++++--------- debian/changelog | 6 ++++-- doc/todo/use_cp_reflink.mdwn | 2 ++ 7 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 CopyFile.hs diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 3ff3023b24..34fde819cb 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -8,7 +8,7 @@ module Command.Unlock where import Control.Monad.State (liftIO) -import System.Directory +import System.Directory hiding (copyFile) import Command import qualified Annex @@ -17,6 +17,7 @@ import Messages import Locations import Utility import Core +import CopyFile seek :: [SubCmdSeek] seek = [withFilesInGit start] @@ -34,9 +35,9 @@ perform dest key = do let src = annexLocation g key liftIO $ removeFile dest showNote "copying..." - ok <- liftIO $ boolSystem "cp" ["-p", src, dest] + ok <- liftIO $ copyFile src dest if ok then do liftIO $ allowWrite dest return $ Just $ return True - else error "cp failed!" + else error "copy failed!" diff --git a/CopyFile.hs b/CopyFile.hs new file mode 100644 index 0000000000..39f1b01833 --- /dev/null +++ b/CopyFile.hs @@ -0,0 +1,24 @@ +{- git-annex file copying + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CopyFile (copyFile) where + +import Utility +import qualified SysConfig + +{- The cp command is used, because I hate reinventing the wheel, + - and because this allows easy access to features like cp --reflink. -} +copyFile :: FilePath -> FilePath -> IO Bool +copyFile src dest = boolSystem "cp" opts + where + opts = if (SysConfig.cp_reflink_auto) + then ["--reflink=auto", src, dest] + else if (SysConfig.cp_a) + then ["-a", src, dest] + else if (SysConfig.cp_p) + then ["-p", src, dest] + else [src, dest] diff --git a/Makefile b/Makefile index 685044146f..17918f9567 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: git-annex docs ghcmake=ghc -Wall -odir build -hidir build -O2 --make -SysConfig.hs: +SysConfig.hs: configure.hs $(ghcmake) configure ./configure diff --git a/Remotes.hs b/Remotes.hs index 7aad6c2a06..bf5ede5729 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -23,7 +23,7 @@ import Control.Monad.State (liftIO) import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils -import System.Directory +import System.Directory hiding (copyFile) import System.Posix.Directory import List import Monad (when, unless) @@ -37,6 +37,7 @@ import UUID import Utility import qualified Core import Messages +import CopyFile {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -204,7 +205,7 @@ copyFromRemote r key file = do then getssh else error "copying from non-ssh repo not supported" where - getlocal = liftIO $ boolSystem "cp" ["-a", keyloc, file] + getlocal = liftIO $ copyFile keyloc file getssh = scp r [sshLocation r keyloc, file] keyloc = annexLocation r key @@ -219,7 +220,7 @@ copyToRemote r key file = do then putssh keyloc else error "copying to non-ssh repo not supported" where - putlocal src = liftIO $ boolSystem "cp" ["-a", src, file] + putlocal src = liftIO $ copyFile src file putssh src = scp r [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath diff --git a/configure.hs b/configure.hs index fa07be3ab5..56daf583a6 100644 --- a/configure.hs +++ b/configure.hs @@ -1,5 +1,4 @@ -{- Checks system configuration and generates SysConfig.hs. - -} +{- Checks system configuration and generates SysConfig.hs. -} import System.IO import System.Cmd @@ -12,8 +11,9 @@ data Config = Config String Bool tests :: [TestDesc] tests = [ - TestDesc "cp -a" "cp_a" cp_a - , TestDesc "cp --reflink" "cp_reflink" cp_reflink + TestDesc "cp -a" "cp_a" $ testCp "-a" + , TestDesc "cp -p" "cp_p" $ testCp "-p" + , TestDesc "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" ] tmpDir :: String @@ -25,11 +25,9 @@ testFile = tmpDir ++ "/testfile" quiet :: String -> String quiet s = s ++ " 2>/dev/null" -cp_a :: Test -cp_a = testCmd $ quiet $ "cp -a " ++ testFile ++ " " ++ testFile ++ ".new" - -cp_reflink :: Test -cp_reflink = testCmd $ quiet $ "cp --reflink=auto " ++ testFile ++ " " ++ testFile ++ ".new" +testCp :: String -> Test +testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ + " " ++ testFile ++ ".new" testCmd :: String -> Test testCmd c = do @@ -51,6 +49,7 @@ writeSysConfig config = do header = [ "{- Automatically generated by configure. -}" , "module SysConfig where" + , "" ] footer = [] vars [] = [] @@ -58,6 +57,7 @@ writeSysConfig config = do showvar (Config name val) = [ name ++ " :: Bool" , name ++ " = " ++ show val + , "" ] runTests :: [TestDesc] -> IO [Config] diff --git a/debian/changelog b/debian/changelog index e6549c1f08..21685ba4d3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,8 +2,10 @@ git-annex (0.08) UNRELEASED; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). * Add configure step to build process. - * configure: Check to see if cp -a can be used. - * configure: Check to see if cp --reflink=auto can be used. + * Only use cp -a if it is supported, falling back to cp -p or plain cp + as needed for portability. + * cp --reflink=auto is used if supported, and will make git annex unlock + much faster on filesystems like btrfs that support copy of write. -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 diff --git a/doc/todo/use_cp_reflink.mdwn b/doc/todo/use_cp_reflink.mdwn index d7974928ce..39518abf18 100644 --- a/doc/todo/use_cp_reflink.mdwn +++ b/doc/todo/use_cp_reflink.mdwn @@ -3,3 +3,5 @@ The unlock command needs to copy a file, and it would be great to use this: O(1) overhead on BTRFS. Needs coreutils 7.6; and remember that git-annex may be used on systems without coreutils.. + +[[done]] From 09964033777ea206db0ff0a27f7928184c70c879 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:07:22 -0400 Subject: [PATCH 0538/8313] tweak --- configure.hs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/configure.hs b/configure.hs index 56daf583a6..d9ce60a14a 100644 --- a/configure.hs +++ b/configure.hs @@ -9,6 +9,12 @@ type Test = IO Bool data TestDesc = TestDesc String String Test data Config = Config String Bool +instance Show Config where + show (Config key value) = unlines $ [ + key ++ " :: Bool" + , key ++ " = " ++ show value + ] + tests :: [TestDesc] tests = [ TestDesc "cp -a" "cp_a" $ testCp "-a" @@ -44,7 +50,7 @@ testEnd r = putStrLn $ " " ++ (show r) writeSysConfig :: [Config] -> IO () writeSysConfig config = do - writeFile "SysConfig.hs" $ unlines $ header ++ vars config ++ footer + writeFile "SysConfig.hs" $ unlines $ header ++ map show config ++ footer where header = [ "{- Automatically generated by configure. -}" @@ -52,13 +58,6 @@ writeSysConfig config = do , "" ] footer = [] - vars [] = [] - vars (c:cs) = showvar c ++ vars cs - showvar (Config name val) = [ - name ++ " :: Bool" - , name ++ " = " ++ show val - , "" - ] runTests :: [TestDesc] -> IO [Config] runTests [] = return [] From b6ecfc916ed46c3d84717ab5097f4731c986c2fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:08:46 -0400 Subject: [PATCH 0539/8313] tweak --- test.hs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test.hs b/test.hs index 2faf579a20..9a6e05a97c 100644 --- a/test.hs +++ b/test.hs @@ -1,5 +1,3 @@ --- TODO find a test harness that is actually in Debian and use it. - import Test.HUnit import Test.HUnit.Tools From fbd1cbf2c915bdb18a09264b356eed55752bb757 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:11:18 -0400 Subject: [PATCH 0540/8313] tweak --- configure.hs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/configure.hs b/configure.hs index d9ce60a14a..b9ea2344fe 100644 --- a/configure.hs +++ b/configure.hs @@ -49,9 +49,9 @@ testEnd :: Bool -> IO () testEnd r = putStrLn $ " " ++ (show r) writeSysConfig :: [Config] -> IO () -writeSysConfig config = do - writeFile "SysConfig.hs" $ unlines $ header ++ map show config ++ footer +writeSysConfig config = writeFile "SysConfig.hs" body where + body = unlines $ header ++ map show config ++ footer header = [ "{- Automatically generated by configure. -}" , "module SysConfig where" @@ -68,10 +68,18 @@ runTests ((TestDesc tname key t):ts) = do rest <- runTests ts return $ (Config key val):rest -main :: IO () -main = do +setup :: IO () +setup = do createDirectoryIfMissing True tmpDir writeFile testFile "test file contents" - config <- runTests tests + +cleanup :: IO () +cleanup = do removeDirectoryRecursive tmpDir + +main :: IO () +main = do + setup + config <- runTests tests writeSysConfig config + cleanup From da794cce7d4630a99d95e39d516485834892c286 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:12:40 -0400 Subject: [PATCH 0541/8313] typo --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 21685ba4d3..7b7ff55ae1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,7 +5,7 @@ git-annex (0.08) UNRELEASED; urgency=low * Only use cp -a if it is supported, falling back to cp -p or plain cp as needed for portability. * cp --reflink=auto is used if supported, and will make git annex unlock - much faster on filesystems like btrfs that support copy of write. + much faster on filesystems like btrfs that support copy on write. -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 From f3f3bc6cae0f907fe023708d2c298c1219b0b603 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 14:29:14 -0400 Subject: [PATCH 0542/8313] mention git-bigfiles --- doc/not.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index affcb57f11..d965ab86f8 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -11,6 +11,10 @@ too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. +* git-annex is more than just a workaround for git limitations that might + eventually be fixed by efforts like + [git-bigfiles](http://caca.zoy.org/wiki/git-bigfiles). + * git-annex is not some flaky script that was quickly thrown together. I wrote it in Haskell because I wanted it to be solid and to compile down to a binary. From 68183b47212b7819b3fc0e7715c37ec38e4a9750 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 18 Nov 2010 15:15:40 -0400 Subject: [PATCH 0543/8313] add tests for uuid and xargs -0 --- configure.hs | 13 ++++++++++++- debian/control | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/configure.hs b/configure.hs index b9ea2344fe..4d0624a422 100644 --- a/configure.hs +++ b/configure.hs @@ -20,6 +20,8 @@ tests = [ TestDesc "cp -a" "cp_a" $ testCp "-a" , TestDesc "cp -p" "cp_p" $ testCp "-p" , TestDesc "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" + , TestDesc "uuid" "uuid" $ requireCommand "uuid" "uuid" + , TestDesc "xargs -0" "xargs_0" $ requireCommand "xargs -0" "xargs -0 String -quiet s = s ++ " 2>/dev/null" +quiet s = s ++ " >/dev/null 2>&1" + +requireCommand :: String -> String -> Test +requireCommand command cmdline = do + ret <- testCmd $ quiet cmdline + if (ret) + then return True + else do + testEnd False + error $ "** the " ++ command ++ " command is required to use git-annex" testCp :: String -> Test testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ diff --git a/debian/control b/debian/control index 3fba367427..0318dafa4e 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From d93e877a3e63501b951e0358d9177c639fd1306a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 21 Nov 2010 13:47:44 -0400 Subject: [PATCH 0544/8313] releasing version 0.08 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7b7ff55ae1..66c4bc94da 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.08) UNRELEASED; urgency=low +git-annex (0.08) unstable; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). * Add configure step to build process. @@ -7,7 +7,7 @@ git-annex (0.08) UNRELEASED; urgency=low * cp --reflink=auto is used if supported, and will make git annex unlock much faster on filesystems like btrfs that support copy on write. - -- Joey Hess Wed, 17 Nov 2010 13:54:49 -0400 + -- Joey Hess Sun, 21 Nov 2010 13:45:44 -0400 git-annex (0.07) unstable; urgency=low From fd11b5a3e5355a68f182a60eda7916d57e141366 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 21 Nov 2010 13:47:52 -0400 Subject: [PATCH 0545/8313] add news item for git-annex 0.08 --- doc/news/version_0.03.mdwn | 16 ---------------- doc/news/version_0.08.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 16 deletions(-) delete mode 100644 doc/news/version_0.03.mdwn create mode 100644 doc/news/version_0.08.mdwn diff --git a/doc/news/version_0.03.mdwn b/doc/news/version_0.03.mdwn deleted file mode 100644 index b22aad91af..0000000000 --- a/doc/news/version_0.03.mdwn +++ /dev/null @@ -1,16 +0,0 @@ -git-annex 0.03 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix support for file:// remotes. - * Add --verbose - * Fix SIGINT handling. - * Fix handling of files with unusual characters in their name. - * Fixed memory leak; git-annex no longer reads the whole file list - from git before starting, and will be much faster with large repos. - * Fix crash on unknown symlinks. - * Added remote.annex-scp-options and remote.annex-ssh-options. - * The backends to use when adding different sets of files can be configured - via gitattributes. - * In .gitattributes, the git-annex-backend attribute can be set to the - names of backends to use when adding different types of files. - * Add fsck subcommand. (For now it only finds unused key contents in the - annex.)"""]] \ No newline at end of file diff --git a/doc/news/version_0.08.mdwn b/doc/news/version_0.08.mdwn new file mode 100644 index 0000000000..8ef7ad7f31 --- /dev/null +++ b/doc/news/version_0.08.mdwn @@ -0,0 +1,8 @@ +git-annex 0.08 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix `git annex add ../foo` (when ran in a subdir of the repo). + * Add configure step to build process. + * Only use cp -a if it is supported, falling back to cp -p or plain cp + as needed for portability. + * cp --reflink=auto is used if supported, and will make git annex unlock + much faster on filesystems like btrfs that support copy on write."""]] \ No newline at end of file From 57adb0347bf4eb71ab846a2947680a20263449a2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 22 Nov 2010 15:46:57 -0400 Subject: [PATCH 0546/8313] hlint tweaks Remotes.hs next, and also Backend/* and Command/* --- Backend.hs | 8 ++++---- Command.hs | 26 +++++++++++--------------- CopyFile.hs | 6 +++--- Core.hs | 12 ++++++------ GitQueue.hs | 4 ++-- GitRepo.hs | 28 ++++++++++++++-------------- LocationLog.hs | 20 ++++++++++---------- Locations.hs | 6 +++--- Messages.hs | 9 +++------ TypeInternals.hs | 6 +++--- UUID.hs | 25 ++++++++++++------------- configure.hs | 6 +++--- 12 files changed, 74 insertions(+), 82 deletions(-) diff --git a/Backend.hs b/Backend.hs index f24347ca8d..01a7298b4e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -45,21 +45,21 @@ import Messages list :: Annex [Backend] list = do l <- Annex.backends -- list is cached here - if (not $ null l) + if not $ null l then return l else do bs <- Annex.supportedBackends g <- Annex.gitRepo let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" - let l' = if (not $ null backendflag) + let l' = if not $ null backendflag then (lookupBackendName bs backendflag):defaults else defaults Annex.backendsChange l' return l' where parseBackendList bs s = - if (null s) + if null s then bs else map (lookupBackendName bs) $ words s @@ -71,7 +71,7 @@ lookupBackendName bs s = Nothing -> error $ "unknown backend " ++ s maybeLookupBackendName :: [Backend] -> String -> Maybe Backend maybeLookupBackendName bs s = - if ((length matches) /= 1) + if 1 /= length matches then Nothing else Just $ head matches where matches = filter (\b -> s == Internals.name b) bs diff --git a/Command.hs b/Command.hs index c6d2d0d5d4..4d10a9e7f1 100644 --- a/Command.hs +++ b/Command.hs @@ -64,17 +64,17 @@ prepSubCmd SubCommand { subcmdseek = seek } state params = do doSubCmd :: SubCmdStart -> SubCmdCleanup doSubCmd start = do s <- start - case (s) of + case s of Nothing -> return True Just perform -> do p <- perform - case (p) of + case p of Nothing -> do showEndFail return False Just cleanup -> do c <- cleanup - if (c) + if c then do showEndOk return True @@ -85,14 +85,14 @@ doSubCmd start = do notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a) notAnnexed file a = do r <- Backend.lookupFile file - case (r) of + case r of Just _ -> return Nothing Nothing -> a isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) isAnnexed file a = do r <- Backend.lookupFile file - case (r) of + case r of Just v -> a v Nothing -> return Nothing @@ -153,19 +153,15 @@ withNothing _ _ = return [] {- Default to acting on all files matching the seek action if - none are specified. -} withAll :: SubCmdSeekStrings -> SubCmdSeekStrings -withAll w a params = do - if null params - then do - g <- Annex.gitRepo - w a [Git.workTree g] - else w a params +withAll w a [] = do + g <- Annex.gitRepo + w a [Git.workTree g] +withAll w a p = w a p {- Provides a default parameter to act on if none is specified. -} withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings -withDefault d w a params = do - if null params - then w a [d] - else w a params +withDefault d w a [] = w a [d] +withDefault _ w a p = w a p {- filter out files from the state directory -} notState :: FilePath -> Bool diff --git a/CopyFile.hs b/CopyFile.hs index 39f1b01833..8bd07dc35a 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -15,10 +15,10 @@ import qualified SysConfig copyFile :: FilePath -> FilePath -> IO Bool copyFile src dest = boolSystem "cp" opts where - opts = if (SysConfig.cp_reflink_auto) + opts = if SysConfig.cp_reflink_auto then ["--reflink=auto", src, dest] - else if (SysConfig.cp_a) + else if SysConfig.cp_a then ["-a", src, dest] - else if (SysConfig.cp_p) + else if SysConfig.cp_p then ["-p", src, dest] else [src, dest] diff --git a/Core.hs b/Core.hs index 2928dc06df..6e0ddd65ff 100644 --- a/Core.hs +++ b/Core.hs @@ -36,7 +36,7 @@ tryRun state actions = tryRun' state 0 actions tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do result <- try $ Annex.run state a - case (result) of + case result of Left err -> do Annex.eval state $ showErr err tryRun' state (errnum + 1) as @@ -64,7 +64,7 @@ shutdown = do g <- Annex.gitRepo let tmp = annexTmpLocation g exists <- liftIO $ doesDirectoryExist tmp - when (exists) $ liftIO $ removeDirectoryRecursive tmp + when exists $ liftIO $ removeDirectoryRecursive tmp liftIO $ createDirectoryIfMissing True tmp return True @@ -81,7 +81,7 @@ calcGitLink :: FilePath -> Key -> Annex FilePath calcGitLink file key = do g <- Annex.gitRepo cwd <- liftIO $ getCurrentDirectory - let absfile = case (absNormPath cwd file) of + let absfile = case absNormPath cwd file of Just f -> f Nothing -> error $ "unable to normalize " ++ file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ @@ -104,7 +104,7 @@ getViaTmp key action = do let tmp = annexTmpLocation g ++ keyFile key liftIO $ createDirectoryIfMissing True (parentDir tmp) success <- action tmp - if (success) + if success then do moveAnnex key tmp logStatus key ValuePresent @@ -125,7 +125,7 @@ preventWrite f = unsetFileMode f writebits allowWrite :: FilePath -> IO () allowWrite f = do s <- getFileStatus f - setFileMode f $ (fileMode s) `unionFileModes` ownerWriteMode + setFileMode f $ fileMode s `unionFileModes` ownerWriteMode {- Moves a file into .git/annex/objects/ -} moveAnnex :: Key -> FilePath -> Annex () @@ -188,7 +188,7 @@ getKeysPresent' dir = do where present d = do s <- getFileStatus $ dir ++ "/" ++ d ++ "/" - ++ (takeFileName d) + ++ takeFileName d return $ isRegularFile s {- List of keys referenced by symlinks in the git repo. -} diff --git a/GitQueue.hs b/GitQueue.hs index 2d44a8f108..d8ba861366 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue {- Runs a queue on a git repository. -} run :: Git.Repo -> Queue -> IO () run repo queue = do - _ <- mapM (\(k, v) -> runAction repo k v) $ M.toList queue + _ <- mapM (uncurry $ runAction repo) $ M.toList queue return () {- Runs an Action on a list of files in a git repository. @@ -56,6 +56,6 @@ runAction repo action files = do unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs - gitcmd = ["git"] ++ Git.gitCommandLine repo + gitcmd = "git" : Git.gitCommandLine repo (getSubcommand action:getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index fa78e51222..f50b3fb2e9 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -127,19 +127,19 @@ repoIsSsh _ = False assertLocal :: Repo -> a -> a assertLocal repo action = - if (not $ repoIsUrl repo) + if not $ repoIsUrl repo then action else error $ "acting on URL git repo " ++ repoDescribe repo ++ " not supported" assertUrl :: Repo -> a -> a assertUrl repo action = - if (repoIsUrl repo) + if repoIsUrl repo then action else error $ "acting on local git repo " ++ repoDescribe repo ++ " not supported" assertSsh :: Repo -> a -> a assertSsh repo action = - if (repoIsSsh repo) + if repoIsSsh repo then action else error $ "unsupported url in repo " ++ repoDescribe repo bare :: Repo -> Bool @@ -199,14 +199,14 @@ urlPath repo = assertUrl repo $ error "internal" gitCommandLine :: Repo -> [String] -> [String] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir="++d++"/"++(gitDir repo), "--work-tree="++d] ++ params + ["--git-dir=" ++ d ++ "/" ++ gitDir repo, "--work-tree=" ++ d] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} run :: Repo -> [String] -> IO () run repo params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo params) - unless (ok) $ error $ "git " ++ show params ++ " failed" + unless ok $ error $ "git " ++ show params ++ " failed" {- Runs a git subcommand and returns its output. -} pipeRead :: Repo -> [String] -> IO String @@ -290,7 +290,7 @@ configRead repo sshopts = assertSsh repo $ do params = case sshopts of Nothing -> [urlHost repo, command] Just l -> l ++ [urlHost repo, command] - command = "cd " ++ (shellEscape $ urlPath repo) ++ + command = "cd " ++ shellEscape (urlPath repo) ++ " && git config --list" hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do @@ -308,8 +308,8 @@ configRemotes repo = map construct remotepairs where remotepairs = Map.toList $ filterremotes $ config repo filterremotes = Map.filterWithKey (\k _ -> isremote k) - isremote k = (startswith "remote." k) && (endswith ".url" k) - remotename k = (split "." k) !! 1 + isremote k = startswith "remote." k && endswith ".url" k + remotename k = split "." k !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } gen v | isURI v = repoFromUrl v | otherwise = repoFromPath v @@ -319,7 +319,7 @@ configParse :: String -> Map.Map String String configParse s = Map.fromList $ map pair $ lines s where pair l = (key l, val l) - key l = (keyval l) !! 0 + key l = head $ keyval l val l = join sep $ drop 1 $ keyval l keyval l = split sep l :: [String] sep = "=" @@ -377,7 +377,7 @@ decodeGitFile f@(c:s) alloctal = isOctDigit n1 && isOctDigit n2 && isOctDigit n3 - fromoctal = [chr $ readoctal (n1:n2:n3:[])] + fromoctal = [chr $ readoctal [n1, n2, n3]] readoctal o = read $ "0o" ++ o :: Int -- \C is used for a few special characters decode (x:nc:rest) @@ -395,9 +395,9 @@ decodeGitFile f@(c:s) {- Should not need to use this, except for testing decodeGitFile. -} encodeGitFile :: FilePath -> String -encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" +encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\"" where - e c = "\\" ++ [c] + e c = '\\' : [c] echar '\a' = e 'a' echar '\b' = e 'b' echar '\f' = e 'f' @@ -413,7 +413,7 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" | ord x > 0x7E = e_num x -- high ascii | otherwise = [x] -- printable ascii where - showoctal i = "\\" ++ (printf "%03o" i) + showoctal i = '\\' : printf "%03o" i e_num c = showoctal $ ord c -- unicode character is decomposed to -- Word8s and each is shown in octal @@ -423,7 +423,7 @@ encodeGitFile s = (foldl (++) "\"" (map echar s)) ++ "\"" {- for quickcheck -} prop_idempotent_deencode :: String -> Bool -prop_idempotent_deencode s = s == (decodeGitFile $ encodeGitFile s) +prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s) {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo diff --git a/LocationLog.hs b/LocationLog.hs index 1ddbf3c0bb..7497d865bb 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -66,8 +66,8 @@ instance Read LogLine where -- read without an exception being thrown. -- Such lines have a status of Undefined. readsPrec _ string = - if (length w == 3) - then case (pdate) of + if length w == 3 + then case pdate of Just v -> good v Nothing -> bad else bad @@ -75,15 +75,16 @@ instance Read LogLine where w = words string s = read $ w !! 1 u = w !! 2 - pdate = (parseTime defaultTimeLocale "%s%Qs" $ w !! 0) :: Maybe UTCTime + pdate :: Maybe UTCTime + pdate = parseTime defaultTimeLocale "%s%Qs" $ head w good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s u - bad = ret $ LogLine (0) Undefined "" + bad = ret $ LogLine 0 Undefined "" ret v = [(v, "")] {- Log a change in the presence of a key's value in a repository, - and returns the filename of the logfile. -} -logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO (FilePath) +logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key u s = do line <- logNow s u ls <- readLog logfile @@ -101,10 +102,9 @@ readLog file = do then do s <- readFile file -- filter out any unparsable lines - return $ filter (\l -> (status l) /= Undefined ) + return $ filter (\l -> status l /= Undefined ) $ map read $ lines s - else do - return [] + else return [] {- Writes a set of lines to a log file -} writeLog :: FilePath -> [LogLine] -> IO () @@ -124,7 +124,7 @@ logNow s u = do {- Returns the filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String logFile repo key = - (gitStateDir repo) ++ (Git.relative repo (keyFile key)) ++ ".log" + gitStateDir repo ++ Git.relative repo (keyFile key) ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} @@ -152,7 +152,7 @@ compactLog' m (l:ls) = compactLog' (mapLog m l) ls - information about a repo than the other logs in the map -} mapLog :: LogMap -> LogLine -> LogMap mapLog m l = - if (better) + if better then Map.insert u l m else m where diff --git a/Locations.hs b/Locations.hs index 24ccc75c6d..6c541e937c 100644 --- a/Locations.hs +++ b/Locations.hs @@ -31,12 +31,12 @@ import qualified GitRepo as Git stateLoc :: String stateLoc = ".git-annex/" gitStateDir :: Git.Repo -> FilePath -gitStateDir repo = (Git.workTree repo) ++ "/" ++ stateLoc +gitStateDir repo = Git.workTree repo ++ "/" ++ stateLoc {- Annexed file's absolute location. -} annexLocation :: Git.Repo -> Key -> FilePath annexLocation r key = - (Git.workTree r) ++ "/" ++ (annexLocationRelative key) + Git.workTree r ++ "/" ++ annexLocationRelative key {- Annexed file's location relative to git's working tree. - @@ -90,5 +90,5 @@ fileKey file = read $ {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool -prop_idempotent_fileKey s = k == (fileKey $ keyFile k) +prop_idempotent_fileKey s = k == fileKey (keyFile k) where k = read $ "test:" ++ s diff --git a/Messages.hs b/Messages.hs index f9d12c2221..e1cf1539a6 100644 --- a/Messages.hs +++ b/Messages.hs @@ -37,17 +37,14 @@ showProgress :: Annex () showProgress = verbose $ liftIO $ putStr "\n" showLongNote :: String -> Annex () -showLongNote s = verbose $ do - liftIO $ putStr $ "\n" ++ indented +showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indented where indented = join "\n" $ map (\l -> " " ++ l) $ lines s showEndOk :: Annex () -showEndOk = verbose $ do - liftIO $ putStrLn "ok" +showEndOk = verbose $ liftIO $ putStrLn "ok" showEndFail :: Annex () -showEndFail = verbose $ do - liftIO $ putStrLn "\nfailed" +showEndFail = verbose $ liftIO $ putStrLn "\nfailed" {- Exception pretty-printing. -} showErr :: (Show a) => a -> Annex () diff --git a/TypeInternals.hs b/TypeInternals.hs index 3078224b15..bcef4ee0a8 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -51,10 +51,10 @@ instance Show Key where show (Key (b, k)) = b ++ ":" ++ k instance Read Key where - readsPrec _ s = [((Key (b,k)) ,"")] + readsPrec _ s = [(Key (b,k), "")] where l = split ":" s - b = l !! 0 + b = head l k = join ":" $ drop 1 l backendName :: Key -> BackendName @@ -81,4 +81,4 @@ data Backend = Backend { } instance Show Backend where - show backend = "Backend { name =\"" ++ (name backend) ++ "\" }" + show backend = "Backend { name =\"" ++ name backend ++ "\" }" diff --git a/UUID.hs b/UUID.hs index 0f8a2173ef..41a35327d9 100644 --- a/UUID.hs +++ b/UUID.hs @@ -20,8 +20,8 @@ module UUID ( ) where import Control.Monad.State -import Maybe -import List +import Data.Maybe +import Data.List import System.Cmd.Utils import System.IO import System.Directory @@ -57,7 +57,7 @@ getUUID r = do let c = cached g let u = uncached - if (c /= u && u /= "") + if c /= u && u /= "" then do updatecache g u return u @@ -66,7 +66,7 @@ getUUID r = do uncached = Git.configGet r "annex.uuid" "" cached g = Git.configGet g cachekey "" updatecache g u = when (g /= r) $ Annex.setConfig cachekey u - cachekey = "remote." ++ (Git.repoRemoteName r) ++ ".annex-uuid" + cachekey = "remote." ++ Git.repoRemoteName r ++ ".annex-uuid" {- Make sure that the repo has an annex.uuid setting. -} prepUUID :: Annex () @@ -79,8 +79,7 @@ prepUUID = do {- Filters a list of repos to ones that have listed UUIDs. -} reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] -reposByUUID repos uuids = do - filterM match repos +reposByUUID repos uuids = filterM match repos where match r = do u <- getUUID r @@ -90,11 +89,11 @@ reposByUUID repos uuids = do prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do m <- uuidMap - return $ unwords $ map (\u -> "\t"++(prettify m u)++"\n") uuids + return $ unwords $ map (\u -> "\t" ++ prettify m u ++ "\n") uuids where prettify m u = - if (not $ null $ findlog m u) - then u ++ " -- " ++ (findlog m u) + if not $ null $ findlog m u + then u ++ " -- " ++ findlog m u else u findlog m u = M.findWithDefault "" u m @@ -117,11 +116,11 @@ uuidMap :: Annex (M.Map UUID String) uuidMap = do logfile <- uuidLog s <- liftIO $ catch (readFile logfile) ignoreerror - return $ M.fromList $ map (\l -> pair l) $ lines s + return $ M.fromList $ map pair $ lines s where pair l = - if (1 < (length $ words l)) - then ((words l) !! 0, unwords $ drop 1 $ words l) + if 1 < length (words l) + then (head $ words l, unwords $ drop 1 $ words l) else ("", "") ignoreerror _ = return "" @@ -129,4 +128,4 @@ uuidMap = do uuidLog :: Annex String uuidLog = do g <- Annex.gitRepo - return $ (gitStateDir g) ++ "uuid.log" + return $ gitStateDir g ++ "uuid.log" diff --git a/configure.hs b/configure.hs index 4d0624a422..7d3fc01271 100644 --- a/configure.hs +++ b/configure.hs @@ -10,7 +10,7 @@ data TestDesc = TestDesc String String Test data Config = Config String Bool instance Show Config where - show (Config key value) = unlines $ [ + show (Config key value) = unlines [ key ++ " :: Bool" , key ++ " = " ++ show value ] @@ -36,7 +36,7 @@ quiet s = s ++ " >/dev/null 2>&1" requireCommand :: String -> String -> Test requireCommand command cmdline = do ret <- testCmd $ quiet cmdline - if (ret) + if ret then return True else do testEnd False @@ -57,7 +57,7 @@ testStart s = do hFlush stdout testEnd :: Bool -> IO () -testEnd r = putStrLn $ " " ++ (show r) +testEnd r = putStrLn $ " " ++ show r writeSysConfig :: [Config] -> IO () writeSysConfig config = writeFile "SysConfig.hs" body From eeae91024285c85a7e77b1b44e501a63bced7154 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 22 Nov 2010 17:51:55 -0400 Subject: [PATCH 0547/8313] finished hlinting --- Backend.hs | 2 +- Backend/File.hs | 78 ++++++++++++++++------------------- Backend/SHA1.hs | 10 ++--- Backend/WORM.hs | 10 ++--- Command/Add.hs | 4 +- Command/Drop.hs | 6 +-- Command/DropKey.hs | 6 +-- Command/Find.hs | 2 +- Command/Fix.hs | 2 +- Command/FromKey.hs | 4 +- Command/Fsck.hs | 2 +- Command/FsckFile.hs | 2 +- Command/Get.hs | 4 +- Command/Init.hs | 10 ++--- Command/Move.hs | 25 ++++++----- Command/PreCommit.hs | 2 +- Command/SetKey.hs | 2 +- Command/Unannex.hs | 2 +- Command/Unused.hs | 8 ++-- Core.hs | 2 +- Remotes.hs | 98 +++++++++++++++++++++----------------------- Utility.hs | 18 ++++---- Version.hs | 4 +- 23 files changed, 144 insertions(+), 159 deletions(-) diff --git a/Backend.hs b/Backend.hs index 01a7298b4e..d5d8efa03e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -30,7 +30,7 @@ module Backend ( ) where import Control.Monad.State -import IO (try) +import System.IO.Error (try) import System.FilePath import System.Posix.Files diff --git a/Backend/File.hs b/Backend/File.hs index c67fb3ce33..c0fc469921 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -34,7 +34,7 @@ backend = Backend { storeFileKey = dummyStore, retrieveKeyFile = copyKeyFile, removeKey = checkRemoveKey, - hasKey = checkKeyFile, + hasKey = inAnnex, fsckKey = mustProvide } @@ -42,19 +42,15 @@ mustProvide :: a mustProvide = error "must provide this field" {- Storing a key is a no-op. -} -dummyStore :: FilePath -> Key -> Annex (Bool) +dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return True -{- Just check if the .git/annex/ file for the key exists. -} -checkKeyFile :: Key -> Annex Bool -checkKeyFile k = inAnnex k - {- Try to find a copy of the file in one of the remotes, - and copy it over to this one. -} -copyKeyFile :: Key -> FilePath -> Annex (Bool) +copyKeyFile :: Key -> FilePath -> Annex Bool copyKeyFile key file = do remotes <- Remotes.keyPossibilities key - if (null remotes) + if null remotes then do showNote "not available" showLocations key @@ -68,76 +64,72 @@ copyKeyFile key file = do return False trycopy full (r:rs) = do probablythere <- probablyPresent r - if (probablythere) + if probablythere then do - showNote $ "copying from " ++ (Git.repoDescribe r) ++ "..." + showNote $ "copying from " ++ Git.repoDescribe r ++ "..." copied <- Remotes.copyFromRemote r key file - if (copied) + if copied then return True else trycopy full rs else trycopy full rs - probablyPresent r = do - -- This check is to avoid an ugly message if a - -- remote is a drive that is not mounted. - -- Avoid checking inAnnex for ssh remotes because - -- that is unnecessarily slow, and the locationlog - -- should be trusted. (If the ssh remote is down - -- or really lacks the file, it's ok to show - -- an ugly message before going on to the next - -- remote.) - if (not $ Git.repoIsUrl r) + -- This check is to avoid an ugly message if a remote is a + -- drive that is not mounted. Avoid checking inAnnex for ssh + -- remotes because that is unnecessarily slow, and the + -- locationlog should be trusted. (If the ssh remote is down + -- or really lacks the file, it's ok to show an ugly message + -- before going on to the next remote.) + probablyPresent r = + if not $ Git.repoIsUrl r then liftIO $ doesFileExist $ annexLocation r key else return True {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an - error if not. -} -checkRemoveKey :: Key -> Annex (Bool) +checkRemoveKey :: Key -> Annex Bool checkRemoveKey key = do force <- Annex.flagIsSet "force" - if (force) + if force then return True else do remotes <- Remotes.keyPossibilities key numcopies <- getNumCopies - if (numcopies > length remotes) + if numcopies > length remotes then notEnoughCopies numcopies (length remotes) [] else findcopies numcopies 0 remotes [] where - findcopies need have [] bad = - if (have >= need) - then return True - else notEnoughCopies need have bad - findcopies need have (r:rs) bad = do - if (have >= need) - then return True - else do - haskey <- Remotes.inAnnex r key - case (haskey) of - Right True -> findcopies need (have+1) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) + findcopies need have [] bad + | have >= need = return True + | otherwise = notEnoughCopies need have bad + findcopies need have (r:rs) bad + | have >= need = return True + | otherwise = do + haskey <- Remotes.inAnnex r key + case haskey of + Right True -> findcopies need (have+1) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) notEnoughCopies need have bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - (show have) ++ " out of " ++ (show need) ++ + show have ++ " out of " ++ show need ++ " necessary copies" showTriedRemotes bad showLocations key hint return False unsafe = showNote "unsafe" - hint = showLongNote $ "(Use --force to override this check, or adjust annex.numcopies.)" + hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)" showLocations :: Key -> Annex () showLocations key = do g <- Annex.gitRepo u <- getUUID g uuids <- liftIO $ keyLocations g key - let uuidsf = filter (\v -> v /= u) uuids + let uuidsf = filter (/= u) uuids ppuuids <- prettyPrintUUIDs uuidsf - if (null uuidsf) + if null uuidsf then showLongNote $ "No other repository is known to contain the file." else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids @@ -145,7 +137,7 @@ showTriedRemotes :: [Git.Repo] -> Annex () showTriedRemotes [] = return () showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ - (Remotes.list remotes) + Remotes.list remotes getNumCopies :: Annex Int getNumCopies = do @@ -173,7 +165,7 @@ checkKeyNumCopies key = do remotes <- Remotes.keyPossibilities key inannex <- inAnnex key let present = length remotes + if inannex then 1 else 0 - if (present < needed) + if present < needed then do warning $ note present needed return False diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 46667c9cda..68f7f683b5 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -33,15 +33,15 @@ sha1 file = do liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do line <- hGetLine h let bits = split " " line - if (null bits) + if null bits then error "sha1sum parse error" - else return $ bits !! 0 + else return $ head bits -- A key is a sha1 of its contents. keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do s <- sha1 file - return $ Just $ Key ((name backend), s) + return $ Just $ Key (name backend, s) -- A key's sha1 is checked during fsck. checkKeySHA1 :: Key -> Annex Bool @@ -49,11 +49,11 @@ checkKeySHA1 key = do g <- Annex.gitRepo let file = annexLocation g key present <- liftIO $ doesFileExist file - if (not present) + if not present then return True else do s <- sha1 file - if (s == keyName key) + if s == keyName key then return True else do dest <- moveBad key diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 4e2177fed0..e9d8c42855 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -37,11 +37,11 @@ backend = Backend.File.backend { keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do stat <- liftIO $ getFileStatus file - return $ Just $ Key ((name backend), key stat) + return $ Just $ Key (name backend, key stat) where key stat = uniqueid stat ++ sep ++ base - uniqueid stat = (show $ modificationTime stat) ++ sep ++ - (show $ fileSize stat) + uniqueid stat = show (modificationTime stat) ++ sep ++ + show (fileSize stat) base = takeFileName file sep = ":" @@ -58,11 +58,11 @@ checkKeySize key = do g <- Annex.gitRepo let file = annexLocation g key present <- liftIO $ doesFileExist file - if (not present) + if not present then return True else do s <- liftIO $ getFileStatus file - if (fileSize s == keySize key) + if fileSize s == keySize key then return True else do dest <- moveBad key diff --git a/Command/Add.hs b/Command/Add.hs index 586807b530..cf32a8d641 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -28,7 +28,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start] start :: SubCmdStartBackendFile start pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file - if ((isSymbolicLink s) || (not $ isRegularFile s)) + if (isSymbolicLink s) || (not $ isRegularFile s) then return Nothing else do showStart "add" file @@ -37,7 +37,7 @@ start pair@(file, _) = notAnnexed file $ do perform :: (FilePath, Maybe Backend) -> SubCmdPerform perform (file, backend) = do stored <- Backend.storeFileKey file backend - case (stored) of + case stored of Nothing -> return Nothing Just (key, _) -> return $ Just $ cleanup file key diff --git a/Command/Drop.hs b/Command/Drop.hs index 1e73d8b821..fbe66f5841 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -24,7 +24,7 @@ seek = [withFilesInGit start] start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key - if (not inbackend) + if not inbackend then return Nothing else do showStart "drop" file @@ -33,13 +33,13 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do success <- Backend.removeKey backend key - if (success) + if success then return $ Just $ cleanup key else return Nothing cleanup :: Key -> SubCmdCleanup cleanup key = do inannex <- inAnnex key - when (inannex) $ removeAnnex key + when inannex $ removeAnnex key logStatus key ValueMissing return True diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 34010481dd..aa72e1bbd6 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -22,12 +22,12 @@ seek = [withKeys start] start :: SubCmdStartString start keyname = do backends <- Backend.list - let key = genKey (backends !! 0) keyname + let key = genKey (head backends) keyname present <- inAnnex key force <- Annex.flagIsSet "force" - if (not present) + if not present then return Nothing - else if (not force) + else if not force then error "dropkey is can cause data loss; use --force if you're sure you want to do this" else do showStart "dropkey" keyname diff --git a/Command/Find.hs b/Command/Find.hs index db0589feae..7b3c8c4636 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -20,5 +20,5 @@ seek = [withDefault "." withFilesInGit start] start :: SubCmdStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when (exists) $ liftIO $ putStrLn file + when exists $ liftIO $ putStrLn file return Nothing diff --git a/Command/Fix.hs b/Command/Fix.hs index 323aca95e3..33630031f6 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -25,7 +25,7 @@ start :: SubCmdStartString start file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file - if (link == l) + if link == l then return Nothing else do showStart "fix" file diff --git a/Command/FromKey.hs b/Command/FromKey.hs index f25de23a2c..eb9ad5e514 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -29,10 +29,10 @@ start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list - let key = genKey (backends !! 0) keyname + let key = genKey (head backends) keyname inbackend <- Backend.hasKey key - unless (inbackend) $ error $ + unless inbackend $ error $ "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ perform file key diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 6291d5ba3f..dc01688015 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -24,6 +24,6 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do success <- Backend.fsckKey backend key - if (success) + if success then return $ Just $ return True else return Nothing diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs index c74e94e62f..e7c3d49158 100644 --- a/Command/FsckFile.hs +++ b/Command/FsckFile.hs @@ -24,6 +24,6 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do success <- Backend.fsckKey backend key - if (success) + if success then return $ Just $ return True else return Nothing diff --git a/Command/Get.hs b/Command/Get.hs index 13d1375377..628ed62935 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -20,7 +20,7 @@ seek = [withFilesInGit start] start :: SubCmdStartString start file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key - if (inannex) + if inannex then return Nothing else do showStart "get" file @@ -29,7 +29,7 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: Key -> Backend -> SubCmdPerform perform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) - if (ok) + if ok then return $ Just $ return True -- no cleanup needed else return Nothing diff --git a/Command/Init.hs b/Command/Init.hs index c928647a50..eb5c58696f 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -25,8 +25,8 @@ seek = [withString start] {- Stores description for the repository etc. -} start :: SubCmdStartString start description = do - when (null description) $ error $ - "please specify a description of this repository\n" + when (null description) $ + error "please specify a description of this repository\n" showStart "init" description return $ Just $ perform description @@ -38,7 +38,7 @@ perform description = do setVersion liftIO $ gitAttributes g liftIO $ gitPreCommitHook g - return $ Just $ cleanup + return $ Just cleanup cleanup :: SubCmdCleanup cleanup = do @@ -53,7 +53,7 @@ cleanup = do gitAttributes :: Git.Repo -> IO () gitAttributes repo = do exists <- doesFileExist attributes - if (not exists) + if not exists then do writeFile attributes $ attrLine ++ "\n" commit @@ -76,7 +76,7 @@ gitPreCommitHook repo = do let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit" exists <- doesFileExist hook - if (exists) + if exists then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" else do writeFile hook $ "#!/bin/sh\n" ++ diff --git a/Command/Move.hs b/Command/Move.hs index 7f8f40737d..c18054c904 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,8 +7,7 @@ module Command.Move where -import Control.Monad.State (liftIO) -import Monad (when) +import Control.Monad.State (liftIO, when) import Command import qualified Command.Drop @@ -53,7 +52,7 @@ start file = do moveToStart :: SubCmdStartString moveToStart file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key - if (not ishere) + if not ishere then return Nothing -- not here, so nothing to do else do showStart "move" file @@ -68,10 +67,10 @@ moveToPerform key = do showNote $ show err return Nothing Right False -> do - showNote $ "moving to " ++ (Git.repoDescribe remote) ++ "..." - let tmpfile = (annexTmpLocation remote) ++ (keyFile key) + showNote $ "moving to " ++ Git.repoDescribe remote ++ "..." + let tmpfile = annexTmpLocation remote ++ keyFile key ok <- Remotes.copyToRemote remote key tmpfile - if (ok) + if ok then return $ Just $ moveToCleanup remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key @@ -79,7 +78,7 @@ moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup moveToCleanup remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", - "--backend=" ++ (backendName key), + "--backend=" ++ backendName key, "--key=" ++ keyName key, tmpfile] if ok @@ -104,7 +103,7 @@ moveFromStart :: SubCmdStartString moveFromStart file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key - if (null $ filter (\r -> Remotes.same r remote) l) + if null $ filter (\r -> Remotes.same r remote) l then return Nothing else do showStart "move" file @@ -113,18 +112,18 @@ moveFromPerform :: Key -> SubCmdPerform moveFromPerform key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key - if (ishere) + if ishere then return $ Just $ moveFromCleanup remote key else do - showNote $ "moving from " ++ (Git.repoDescribe remote) ++ "..." - ok <- getViaTmp key (Remotes.copyFromRemote remote key) - if (ok) + showNote $ "moving from " ++ Git.repoDescribe remote ++ "..." + ok <- getViaTmp key $ Remotes.copyFromRemote remote key + if ok then return $ Just $ moveFromCleanup remote key else return Nothing -- fail moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup moveFromCleanup remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", - "--backend=" ++ (backendName key), + "--backend=" ++ backendName key, keyName key] when ok $ do -- Record locally that the key is not on the remote. diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index a15510bd93..d4e5c04b9c 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -28,7 +28,7 @@ start file = return $ Just $ perform file perform :: FilePath -> SubCmdPerform perform file = do pairs <- Backend.chooseBackends [file] - ok <- doSubCmd $ Command.Add.start $ pairs !! 0 + ok <- doSubCmd $ Command.Add.start $ head pairs if ok then return $ Just $ cleanup file else error $ "failed to add " ++ file ++ "; canceling commit" diff --git a/Command/SetKey.hs b/Command/SetKey.hs index e8d407b832..685872f73d 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -28,7 +28,7 @@ start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" backends <- Backend.list - let key = genKey (backends !! 0) keyname + let key = genKey (head backends) keyname showStart "setkey" file return $ Just $ perform file key perform :: FilePath -> Key -> SubCmdPerform diff --git a/Command/Unannex.hs b/Command/Unannex.hs index e85e8486f5..90ae550581 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -34,7 +34,7 @@ perform file key backend = do -- force backend to always remove Annex.flagChange "force" $ FlagBool True ok <- Backend.removeKey backend key - if (ok) + if ok then return $ Just $ cleanup file key else return Nothing diff --git a/Command/Unused.hs b/Command/Unused.hs index ae189550ce..de34ceae96 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -35,7 +35,7 @@ checkUnused :: Annex Bool checkUnused = do showNote "checking for unused data..." unused <- unusedKeys - if (null unused) + if null unused then return True else do let list = number 1 unused @@ -48,9 +48,10 @@ checkUnused = do w u = unlines $ ["Some annexed data is no longer pointed to by any files in the repository:", " NUMBER KEY"] - ++ (map (\(n, k) -> " " ++ (pad 6 $ show n) ++ " " ++ show k) u) ++ + ++ map cols u ++ ["(To see where data was previously used, try: git log --stat -S'KEY')", "(To remove unwanted data: git-annex dropunused NUMBER)"] + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' number :: Integer -> [a] -> [(Integer, a)] @@ -71,8 +72,7 @@ unusedKeys = do let unused_m = remove referenced present_m return $ M.keys unused_m where - remove [] m = m - remove (x:xs) m = remove xs $ M.delete x m + remove a b = foldl (flip M.delete) b a existsMap :: Ord k => [k] -> M.Map k Int existsMap l = M.fromList $ map (\k -> (k, 1)) l diff --git a/Core.hs b/Core.hs index 6e0ddd65ff..b61d186666 100644 --- a/Core.hs +++ b/Core.hs @@ -7,7 +7,7 @@ module Core where -import IO (try) +import System.IO.Error (try) import System.Directory import Control.Monad.State (liftIO) import System.Path diff --git a/Remotes.hs b/Remotes.hs index bf5ede5729..cb8081d740 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -17,16 +17,14 @@ module Remotes ( runCmd ) where -import IO (bracket_) -import Control.Exception.Extensible hiding (bracket_) +import Control.Exception.Extensible import Control.Monad.State (liftIO) -import Control.Monad (filterM) import qualified Data.Map as Map import Data.String.Utils import System.Directory hiding (copyFile) import System.Posix.Directory -import List -import Monad (when, unless) +import Data.List +import Control.Monad (when, unless, filterM) import Types import qualified GitRepo as Git @@ -55,7 +53,7 @@ keyPossibilities key = do -- But, reading the config of remotes can be expensive, so make -- sure we only do it once per git-annex run. remotesread <- Annex.flagIsSet "remotesread" - if (remotesread) + if remotesread then reposByUUID allremotes uuids else do -- We assume that it's cheap to read the config @@ -65,11 +63,11 @@ keyPossibilities key = do let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive - unless (null doexpensive) $ do + unless (null doexpensive) $ showNote $ "getting UUID for " ++ - (list doexpensive) ++ "..." + list doexpensive ++ "..." let todo = cheap ++ doexpensive - if (not $ null todo) + if not $ null todo then do _ <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True @@ -84,10 +82,9 @@ keyPossibilities key = do - If the remote cannot be accessed, returns a Left error. -} inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool) -inAnnex r key = do - if (not $ Git.repoIsUrl r) - then liftIO $ ((try checklocal)::IO (Either IOException Bool)) - else checkremote +inAnnex r key = if Git.repoIsUrl r + then checkremote + else liftIO (try checklocal ::IO (Either IOException Bool)) where checklocal = do -- run a local check by making an Annex monad @@ -112,12 +109,12 @@ reposByCost :: [Git.Repo] -> Annex [Git.Repo] reposByCost l = do notignored <- filterM repoNotIgnored l costpairs <- mapM costpair notignored - return $ fst $ unzip $ sortBy bycost $ costpairs + return $ fst $ unzip $ sortBy cmpcost costpairs where costpair r = do cost <- repoCost r return (r, cost) - bycost (_, c1) (_, c2) = compare c1 c2 + cmpcost (_, c1) (_, c2) = compare c1 c2 {- Calculates cost for a repo. - @@ -127,9 +124,9 @@ reposByCost l = do repoCost :: Git.Repo -> Annex Int repoCost r = do cost <- repoConfig r "cost" "" - if (not $ null cost) + if not $ null cost then return $ read cost - else if (Git.repoIsUrl r) + else if Git.repoIsUrl r then return 200 else return 100 @@ -141,13 +138,12 @@ repoNotIgnored r = do ignored <- repoConfig r "ignore" "false" fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" - let name = if (not $ null fromName) then fromName else toName - if (not $ null name) + let name = if null fromName then toName else fromName + if not $ null name then return $ match name - else return $ not $ isIgnored ignored + else return $ not $ Git.configTrue ignored where match name = name == Git.repoRemoteName r - isIgnored ignored = Git.configTrue ignored {- Checks if two repos are the same, by comparing their remote names. -} same :: Git.Repo -> Git.Repo -> Bool @@ -158,14 +154,14 @@ commandLineRemote :: Annex Git.Repo commandLineRemote = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" - let name = if (not $ null fromName) then fromName else toName + let name = if null fromName then toName else fromName when (null name) $ error "no remote specified" g <- Annex.gitRepo let match = filter (\r -> name == Git.repoRemoteName r) $ Git.remotes g when (null match) $ error $ "there is no git remote named \"" ++ name ++ "\"" - return $ match !! 0 + return $ head match {- The git configs for the git repo's remotes is not read on startup - because reading it may be expensive. This function tries to read the @@ -174,12 +170,12 @@ commandLineRemote = do tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r = do sshoptions <- repoConfig r "ssh-options" "" - if (Map.null $ Git.configMap r) + if Map.null $ Git.configMap r then do -- configRead can fail due to IO error or -- for other reasons; catch all possible exceptions - result <- liftIO $ (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException (Git.Repo))) - case (result) of + result <- liftIO (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException Git.Repo)) + case result of Left _ -> return $ Left r Right r' -> do g <- Annex.gitRepo @@ -192,18 +188,16 @@ tryGitConfigRead r = do where exchange [] _ = [] exchange (old:ls) new = - if (Git.repoRemoteName old == Git.repoRemoteName new) - then new:(exchange ls new) - else old:(exchange ls new) + if Git.repoRemoteName old == Git.repoRemoteName new + then new : exchange ls new + else old : exchange ls new {- Tries to copy a key's content from a remote to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool -copyFromRemote r key file = do - if (not $ Git.repoIsUrl r) - then getlocal - else if (Git.repoIsSsh r) - then getssh - else error "copying from non-ssh repo not supported" +copyFromRemote r key file + | not $ Git.repoIsUrl r = getlocal + | Git.repoIsSsh r = getssh + | otherwise = error "copying from non-ssh repo not supported" where getlocal = liftIO $ copyFile keyloc file getssh = scp r [sshLocation r keyloc, file] @@ -214,9 +208,9 @@ copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyToRemote r key file = do g <- Annex.gitRepo let keyloc = annexLocation g key - if (not $ Git.repoIsUrl r) + if not $ Git.repoIsUrl r then putlocal keyloc - else if (Git.repoIsSsh r) + else if Git.repoIsSsh r then putssh keyloc else error "copying to non-ssh repo not supported" where @@ -224,7 +218,7 @@ copyToRemote r key file = do putssh src = scp r [src, sshLocation r file] sshLocation :: Git.Repo -> FilePath -> FilePath -sshLocation r file = (Git.urlHost r) ++ ":" ++ shellEscape file +sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file {- Runs scp against a specified remote. (Honors annex-scp-options.) -} scp :: Git.Repo -> [String] -> Annex Bool @@ -238,21 +232,21 @@ scp r params = do runCmd :: Git.Repo -> String -> [String] -> Annex Bool runCmd r command params = do sshoptions <- repoConfig r "ssh-options" "" - if (not $ Git.repoIsUrl r) + if not $ Git.repoIsUrl r then do - cwd <- liftIO $ getCurrentDirectory - liftIO $ bracket_ (changeWorkingDirectory (Git.workTree r)) - (\_ -> changeWorkingDirectory cwd) $ - boolSystem command params - else if (Git.repoIsSsh r) - then do - liftIO $ boolSystem "ssh" $ - (words sshoptions) ++ - [Git.urlHost r, "cd " ++ - (shellEscape $ Git.workTree r) ++ - " && " ++ (shellEscape command) ++ " " ++ - (unwords $ map shellEscape params)] + cwd <- liftIO getCurrentDirectory + liftIO $ bracket_ + (changeWorkingDirectory (Git.workTree r)) + (changeWorkingDirectory cwd) + (boolSystem command params) + else if Git.repoIsSsh r + then liftIO $ boolSystem "ssh" $ + words sshoptions ++ [Git.urlHost r, sshcmd] else error "running command in non-ssh repo not supported" + where + sshcmd = "cd " ++ shellEscape (Git.workTree r) ++ + " && " ++ shellEscape command ++ " " ++ + unwords (map shellEscape params) {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} @@ -262,5 +256,5 @@ repoConfig r key def = do let def' = Git.configGet g global def return $ Git.configGet g local def' where - local = "remote." ++ (Git.repoRemoteName r) ++ ".annex-" ++ key + local = "remote." ++ Git.repoRemoteName r ++ ".annex-" ++ key global = "annex." ++ key diff --git a/Utility.hs b/Utility.hs index 33db4bb086..2bea6e8755 100644 --- a/Utility.hs +++ b/Utility.hs @@ -35,12 +35,12 @@ hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = - if (not $ null dirs) - then slash ++ (join s $ take ((length dirs) - 1) dirs) + if not $ null dirs + then slash ++ join s (take (length dirs - 1) dirs) else "" where - dirs = filter (\x -> not $ null x) $ split s dir - slash = if (not $ isAbsolute dir) then "" else s + dirs = filter (not . null) $ split s dir + slash = if isAbsolute dir then s else "" s = [pathSeparator] {- Constructs a relative path from the CWD to a directory. @@ -58,7 +58,7 @@ relPathCwdToDir dir = do where -- absolute, normalized form of the directory absnorm cwd = - case (absNormPath cwd dir) of + case absNormPath cwd dir of Just d -> d Nothing -> error $ "unable to normalize " ++ dir @@ -70,7 +70,7 @@ relPathCwdToDir dir = do -} relPathDirToDir :: FilePath -> FilePath -> FilePath relPathDirToDir from to = - if (not $ null path) + if not $ null path then addTrailingPathSeparator path else "" where @@ -80,8 +80,8 @@ relPathDirToDir from to = common = map fst $ filter same $ zip pfrom pto same (c,d) = c == d uncommon = drop numcommon pto - dotdots = take ((length pfrom) - numcommon) $ repeat ".." - numcommon = length $ common + dotdots = replicate (length pfrom - numcommon) ".." + numcommon = length common path = join s $ dotdots ++ uncommon {- Run a system command, and returns True or False @@ -124,4 +124,4 @@ shellEscape f = "'" ++ escaped ++ "'" unsetFileMode :: FilePath -> FileMode -> IO () unsetFileMode f m = do s <- getFileStatus f - setFileMode f $ (fileMode s) `intersectFileModes` (complement m) + setFileMode f $ fileMode s `intersectFileModes` complement m diff --git a/Version.hs b/Version.hs index ce39c0c1b1..8394701201 100644 --- a/Version.hs +++ b/Version.hs @@ -25,13 +25,13 @@ getVersion :: Annex (Maybe String) getVersion = do g <- Annex.gitRepo let v = Git.configGet g versionField "" - if (not $ null v) + if not $ null v then return $ Just v else do -- version 0 was not recorded in .git/config; -- such a repo should have an annexDir d <- liftIO $ doesDirectoryExist $ annexDir g - if (d) + if d then return $ Just "0" else return Nothing -- no version yet From e97d13e29b18c4522395996299651334cb221519 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:02:53 -0400 Subject: [PATCH 0548/8313] Add copy subcommand. --- CmdLine.hs | 5 ++- Command/Move.hs | 107 ++++++++++++++++++++++++--------------------- debian/changelog | 6 +++ doc/git-annex.mdwn | 8 ++++ 4 files changed, 74 insertions(+), 52 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 35e889d7df..837420786f 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -18,6 +18,7 @@ import qualified Command.Add import qualified Command.Unannex import qualified Command.Drop import qualified Command.Move +import qualified Command.Copy import qualified Command.Get import qualified Command.FromKey import qualified Command.DropKey @@ -41,7 +42,9 @@ subCmds = , SubCommand "drop" path Command.Drop.seek "indicate content of files not currently wanted" , SubCommand "move" path Command.Move.seek - "transfer content of files to/from another repository" + "move content of files to/from another repository" + , SubCommand "copy" path Command.Copy.seek + "copy content of files to/from another repository" , SubCommand "unlock" path Command.Unlock.seek "unlock files for modification" , SubCommand "edit" path Command.Unlock.seek diff --git a/Command/Move.hs b/Command/Move.hs index c18054c904..cb6525919e 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,7 +7,7 @@ module Command.Move where -import Control.Monad.State (liftIO, when) +import Control.Monad.State (liftIO) import Command import qualified Command.Drop @@ -22,43 +22,55 @@ import UUID import Messages seek :: [SubCmdSeek] -seek = [withFilesInGit start] +seek = [withFilesInGit $ start True] -{- Move a file either --to or --from a repository. +{- Move (or copy) a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -start :: SubCmdStartString -start file = do +start :: Bool -> SubCmdStartString +start move file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" case (fromName, toName) of ("", "") -> error "specify either --from or --to" - ("", _) -> moveToStart file - (_ , "") -> moveFromStart file + ("", _) -> toStart move file + (_ , "") -> fromStart move file (_ , _) -> error "only one of --from or --to can be specified" -{- Moves the content of an annexed file to another repository, - - removing it from the current repository, and updates locationlog - - information on both. +showAction :: Bool -> FilePath -> Annex () +showAction True file = showStart "move" file +showAction False file = showStart "copy" file + +remoteHasKey :: Git.Repo -> Key -> Bool -> Annex () +remoteHasKey remote key present = do + g <- Annex.gitRepo + remoteuuid <- getUUID remote + logfile <- liftIO $ logChange g key remoteuuid status + Annex.queue "add" ["--"] logfile + where + status = if present then ValuePresent else ValueMissing + +{- Moves (or copies) the content of an annexed file to another repository, + - and updates locationlog information on both. - - - If the destination already has the content, it is still removed - - from the current repository. + - When moving, if the destination already has the content, it is + - still removed from the current repository. - - Note that unlike drop, this does not honor annex.numcopies. - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -moveToStart :: SubCmdStartString -moveToStart file = isAnnexed file $ \(key, _) -> do +toStart :: Bool -> SubCmdStartString +toStart move file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if not ishere then return Nothing -- not here, so nothing to do else do - showStart "move" file - return $ Just $ moveToPerform key -moveToPerform :: Key -> SubCmdPerform -moveToPerform key = do + showAction move file + return $ Just $ toPerform move key +toPerform :: Bool -> Key -> SubCmdPerform +toPerform move key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key @@ -67,15 +79,15 @@ moveToPerform key = do showNote $ show err return Nothing Right False -> do - showNote $ "moving to " ++ Git.repoDescribe remote ++ "..." + showNote $ "to " ++ Git.repoDescribe remote ++ "..." let tmpfile = annexTmpLocation remote ++ keyFile key ok <- Remotes.copyToRemote remote key tmpfile if ok - then return $ Just $ moveToCleanup remote key tmpfile + then return $ Just $ toCleanup move remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key -moveToCleanup :: Git.Repo -> Key -> FilePath -> SubCmdCleanup -moveToCleanup remote key tmpfile = do +toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> SubCmdCleanup +toCleanup move remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", "--backend=" ++ backendName key, @@ -83,52 +95,45 @@ moveToCleanup remote key tmpfile = do tmpfile] if ok then do - -- Record that the key is present on the remote. - g <- Annex.gitRepo - remoteuuid <- getUUID remote - logfile <- liftIO $ logChange g key remoteuuid ValuePresent - Annex.queue "add" ["--"] logfile - -- Cleanup on the local side is the same as done for the - -- drop subcommand. - Command.Drop.cleanup key + remoteHasKey remote key True + if move + then Command.Drop.cleanup key + else return True else return False -{- Moves the content of an annexed file from another repository to the current - - repository and updates locationlog information on both. +{- Moves (or copies) the content of an annexed file from another repository + - to the current repository and updates locationlog information on both. - - If the current repository already has the content, it is still removed - - from the other repository. + - from the other repository when moving. -} -moveFromStart :: SubCmdStartString -moveFromStart file = isAnnexed file $ \(key, _) -> do +fromStart :: Bool -> SubCmdStartString +fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote l <- Remotes.keyPossibilities key if null $ filter (\r -> Remotes.same r remote) l then return Nothing else do - showStart "move" file - return $ Just $ moveFromPerform key -moveFromPerform :: Key -> SubCmdPerform -moveFromPerform key = do + showAction move file + return $ Just $ fromPerform move key +fromPerform :: Bool -> Key -> SubCmdPerform +fromPerform move key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key if ishere - then return $ Just $ moveFromCleanup remote key + then return $ Just $ fromCleanup move remote key else do - showNote $ "moving from " ++ Git.repoDescribe remote ++ "..." + showNote $ "from " ++ Git.repoDescribe remote ++ "..." ok <- getViaTmp key $ Remotes.copyFromRemote remote key if ok - then return $ Just $ moveFromCleanup remote key + then return $ Just $ fromCleanup move remote key else return Nothing -- fail -moveFromCleanup :: Git.Repo -> Key -> SubCmdCleanup -moveFromCleanup remote key = do - ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", +fromCleanup :: Bool -> Git.Repo -> Key -> SubCmdCleanup +fromCleanup True remote key = do + ok <- Remotes.runCmd remote "git-annex" + ["dropkey", "--quiet", "--force", "--backend=" ++ backendName key, keyName key] - when ok $ do - -- Record locally that the key is not on the remote. - remoteuuid <- getUUID remote - g <- Annex.gitRepo - logfile <- liftIO $ logChange g key remoteuuid ValueMissing - Annex.queue "add" ["--"] logfile + remoteHasKey remote key False return ok +fromCleanup False _ _ = return True diff --git a/debian/changelog b/debian/changelog index 66c4bc94da..6e5e7ef488 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.09) UNRELEASED; urgency=low + + * Add copy subcommand. + + -- Joey Hess Sat, 27 Nov 2010 16:58:33 -0400 + git-annex (0.08) unstable; urgency=low * Fix `git annex add ../foo` (when ran in a subdir of the repo). diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3df835eac5..27393de503 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -102,6 +102,14 @@ Many git-annex subcommands will stage changes for later `git commit` by you. When used with the --from option, moves the content of annexed files from the specified repository to the current one. +* copy [path ...] + + When used with the --to option, copies the content of annexed files from + the current repository to the specified one. + + When used with the --from option, copies the content of annexed files + from the specified repository to the current one. + * init description Initializes git-annex with a description of the git repository, From e0518a4adc250da1ad088c4362016627e1effb08 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:07:22 -0400 Subject: [PATCH 0549/8313] Fix bug in setkey subcommand triggered by move --to. --- Command/SetKey.hs | 5 ++++- debian/changelog | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 685872f73d..55472ccaed 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -35,7 +35,10 @@ perform :: FilePath -> Key -> SubCmdPerform perform file key = do -- the file might be on a different filesystem, so mv is used -- rather than simply calling moveToObjectDir key file - ok <- getViaTmp key $ \dest -> liftIO $ boolSystem "mv" [file, dest] + ok <- getViaTmp key $ \dest -> do + if dest /= file + then liftIO $ boolSystem "mv" [file, dest] + else return True if ok then return $ Just $ cleanup key else error "mv failed!" diff --git a/debian/changelog b/debian/changelog index 6e5e7ef488..75f4f9f0ae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.09) UNRELEASED; urgency=low * Add copy subcommand. + * Fix bug in setkey subcommand triggered by move --to. -- Joey Hess Sat, 27 Nov 2010 16:58:33 -0400 From 6dfae19b1b1535869fb4fb5ad2aba6ada24ddeef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:09:22 -0400 Subject: [PATCH 0550/8313] add --- Command/Copy.hs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Command/Copy.hs diff --git a/Command/Copy.hs b/Command/Copy.hs new file mode 100644 index 0000000000..aa55731d9c --- /dev/null +++ b/Command/Copy.hs @@ -0,0 +1,15 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Copy where + +import Command +import qualified Command.Move + +-- A copy is just a move that does not delete the source file. +seek :: [SubCmdSeek] +seek = [withFilesInGit $ Command.Move.start False] From 1493601982f438691985deb8bf0f279f6928fb6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:17:14 -0400 Subject: [PATCH 0551/8313] releasing version 0.09 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 75f4f9f0ae..8ab14d64c5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,9 @@ -git-annex (0.09) UNRELEASED; urgency=low +git-annex (0.09) unstable; urgency=low * Add copy subcommand. * Fix bug in setkey subcommand triggered by move --to. - -- Joey Hess Sat, 27 Nov 2010 16:58:33 -0400 + -- Joey Hess Sat, 27 Nov 2010 17:14:59 -0400 git-annex (0.08) unstable; urgency=low From f8d463b1b76fb5865dfffd5ab4a1205595c1a2b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:17:25 -0400 Subject: [PATCH 0552/8313] add news item for git-annex 0.09 --- doc/news/version_0.04.mdwn | 23 ----------------------- doc/news/version_0.09.mdwn | 4 ++++ 2 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 doc/news/version_0.04.mdwn create mode 100644 doc/news/version_0.09.mdwn diff --git a/doc/news/version_0.04.mdwn b/doc/news/version_0.04.mdwn deleted file mode 100644 index c5a9fd3ead..0000000000 --- a/doc/news/version_0.04.mdwn +++ /dev/null @@ -1,23 +0,0 @@ -git-annex 0.04 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add unlock subcommand, which replaces the symlink with a copy of - the file's content in preparation of changing it. The "edit" subcommand - is an alias for unlock. - * Add lock subcommand. - * Unlocked files will now automatically be added back into the annex when - committed (and the updated symlink committed), by some magic in the - pre-commit hook. - * The SHA1 backend is now fully usable. - * Add annex.version, which will be used to automate upgrades - between incompatible versions. - * Reorganised the layout of .git/annex/ - * The new layout will be automatically upgraded to the first time - git-annex is used in a repository with the old layout. - * Note that git-annex 0.04 cannot transfer content from old repositories - that have not yet been upgraded. - * Annexed file contents are now made unwritable and put in unwriteable - directories, to avoid them accidentially being removed or modified. - (Thanks Josh Triplett for the idea.) - * Add build dep on libghc6-testpack-dev. Closes: #[603016](http://bugs.debian.org/603016) - * Avoid using runghc to run test suite as it is not available on all - architectures. Closes: #[603006](http://bugs.debian.org/603006)"""]] \ No newline at end of file diff --git a/doc/news/version_0.09.mdwn b/doc/news/version_0.09.mdwn new file mode 100644 index 0000000000..fff8249f77 --- /dev/null +++ b/doc/news/version_0.09.mdwn @@ -0,0 +1,4 @@ +git-annex 0.09 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add copy subcommand. + * Fix bug in setkey subcommand triggered by move --to."""]] \ No newline at end of file From 1f9ce9e9a5f6d7eaf149f42de559cb9830c7f28e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 27 Nov 2010 17:31:20 -0400 Subject: [PATCH 0553/8313] tweak --- configure.hs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/configure.hs b/configure.hs index 7d3fc01271..92a051c80e 100644 --- a/configure.hs +++ b/configure.hs @@ -6,7 +6,7 @@ import System.Exit import System.Directory type Test = IO Bool -data TestDesc = TestDesc String String Test +data TestCase = TestCase String String Test data Config = Config String Bool instance Show Config where @@ -15,13 +15,13 @@ instance Show Config where , key ++ " = " ++ show value ] -tests :: [TestDesc] +tests :: [TestCase] tests = [ - TestDesc "cp -a" "cp_a" $ testCp "-a" - , TestDesc "cp -p" "cp_p" $ testCp "-p" - , TestDesc "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" - , TestDesc "uuid" "uuid" $ requireCommand "uuid" "uuid" - , TestDesc "xargs -0" "xargs_0" $ requireCommand "xargs -0" "xargs -0 IO [Config] +runTests :: [TestCase] -> IO [Config] runTests [] = return [] -runTests ((TestDesc tname key t):ts) = do +runTests ((TestCase tname key t):ts) = do testStart tname val <- t testEnd val From 92e5d28ca83d057a3d8f5d7d30806642de699172 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 14:19:43 -0400 Subject: [PATCH 0554/8313] precommit: Optimise to avoid calling git-check-attr more than once. --- Command.hs | 32 ++++++++++++++++---------------- Command/Add.hs | 2 +- Command/PreCommit.hs | 14 ++++++-------- Command/Unlock.hs | 1 - debian/changelog | 6 ++++++ 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/Command.hs b/Command.hs index 4d10a9e7f1..7f3063abb9 100644 --- a/Command.hs +++ b/Command.hs @@ -41,8 +41,9 @@ type SubCmdCleanup = Annex Bool - functions. -} type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek type SubCmdStartString = String -> SubCmdStart +type BackendFile = (FilePath, Maybe Backend) type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdStartBackendFile = (FilePath, Maybe Backend) -> SubCmdStart +type SubCmdStartBackendFile = BackendFile -> SubCmdStart type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek type SubCmdStartNothing = SubCmdStart @@ -116,17 +117,6 @@ withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params backendPairs a $ filter notState $ foldl (++) [] newfiles -withFilesUnlocked :: SubCmdSeekBackendFiles -withFilesUnlocked a params = do - -- unlocked files have changed type from a symlink to a regular file - repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedFiles repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - backendPairs a $ filter notState unlockedfiles -backendPairs :: SubCmdSeekBackendFiles -backendPairs a files = do - pairs <- Backend.chooseBackends files - return $ map a pairs withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] withStrings :: SubCmdSeekStrings @@ -136,12 +126,17 @@ withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ mapM (Git.stagedFiles repo) params return $ map a $ filter notState $ foldl (++) [] tocommit -withUnlockedFilesToBeCommitted :: SubCmdSeekStrings -withUnlockedFilesToBeCommitted a params = do +withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles +withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles +withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles +withFilesUnlocked' :: (Git.Repo -> FilePath -> IO [FilePath]) -> SubCmdSeekBackendFiles +withFilesUnlocked' typechanged a params = do + -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (Git.typeChangedStagedFiles repo) params + typechangedfiles <- liftIO $ mapM (typechanged repo) params unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - return $ map a $ filter notState unlockedfiles + backendPairs a $ filter notState unlockedfiles withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings @@ -150,6 +145,11 @@ withNothing :: SubCmdSeekNothing withNothing a [] = return [a] withNothing _ _ = return [] +backendPairs :: SubCmdSeekBackendFiles +backendPairs a files = do + pairs <- Backend.chooseBackends files + return $ map a pairs + {- Default to acting on all files matching the seek action if - none are specified. -} withAll :: SubCmdSeekStrings -> SubCmdSeekStrings diff --git a/Command/Add.hs b/Command/Add.hs index cf32a8d641..d141448a3a 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -34,7 +34,7 @@ start pair@(file, _) = notAnnexed file $ do showStart "add" file return $ Just $ perform pair -perform :: (FilePath, Maybe Backend) -> SubCmdPerform +perform :: BackendFile -> SubCmdPerform perform (file, backend) = do stored <- Backend.storeFileKey file backend case stored of diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index d4e5c04b9c..513d5d43f7 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -11,7 +11,6 @@ import Control.Monad.State (liftIO) import Command import qualified Annex -import qualified Backend import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix @@ -20,15 +19,14 @@ import qualified Command.Fix - And, it needs to inject unlocked files into the annex. -} seek :: [SubCmdSeek] seek = [withFilesToBeCommitted Command.Fix.start, - withUnlockedFilesToBeCommitted start] + withFilesUnlockedToBeCommitted start] -start :: SubCmdStartString -start file = return $ Just $ perform file +start :: SubCmdStartBackendFile +start pair = return $ Just $ perform pair -perform :: FilePath -> SubCmdPerform -perform file = do - pairs <- Backend.chooseBackends [file] - ok <- doSubCmd $ Command.Add.start $ head pairs +perform :: BackendFile -> SubCmdPerform +perform pair@(file, _) = do + ok <- doSubCmd $ Command.Add.start pair if ok then return $ Just $ cleanup file else error $ "failed to add " ++ file ++ "; canceling commit" diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 34fde819cb..ff22fa84b3 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -15,7 +15,6 @@ import qualified Annex import Types import Messages import Locations -import Utility import Core import CopyFile diff --git a/debian/changelog b/debian/changelog index 8ab14d64c5..808087dadc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.10) UNRELEASED; urgency=low + + * precommit: Optimise to avoid calling git-check-attr more than once. + + -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 + git-annex (0.09) unstable; urgency=low * Add copy subcommand. From 653ad35a9f728ed5b3e9b557cdfb15a19b4afe16 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 15:28:20 -0400 Subject: [PATCH 0555/8313] In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. --- Backend.hs | 8 ++++---- Backend/File.hs | 25 +++++++++++++------------ Backend/URL.hs | 10 ++++++++-- Command.hs | 16 +++++++++++++--- Command/Drop.hs | 17 ++++++++++------- Command/DropUnused.hs | 4 +--- Command/Fsck.hs | 17 ++++++++++------- Command/FsckFile.hs | 29 ----------------------------- Command/Unannex.hs | 3 +-- TypeInternals.hs | 9 ++++++--- Utility.hs | 9 ++++++++- debian/changelog | 2 ++ doc/copies.mdwn | 7 +++++-- doc/git-annex.mdwn | 6 ++++++ 14 files changed, 87 insertions(+), 75 deletions(-) delete mode 100644 Command/FsckFile.hs diff --git a/Backend.hs b/Backend.hs index d5d8efa03e..8e17f253f4 100644 --- a/Backend.hs +++ b/Backend.hs @@ -104,8 +104,8 @@ retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} -removeKey :: Backend -> Key -> Annex Bool -removeKey backend key = (Internals.removeKey backend) key +removeKey :: Backend -> Key -> Maybe Int -> Annex Bool +removeKey backend key numcopies = (Internals.removeKey backend) key numcopies {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool @@ -114,8 +114,8 @@ hasKey key = do (Internals.hasKey backend) key {- Checks a key's backend for problems. -} -fsckKey :: Backend -> Key -> Annex Bool -fsckKey backend key = (Internals.fsckKey backend) key +fsckKey :: Backend -> Key -> Maybe Int -> Annex Bool +fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index c0fc469921..5984348b36 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -86,14 +86,14 @@ copyKeyFile key file = do {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an - error if not. -} -checkRemoveKey :: Key -> Annex Bool -checkRemoveKey key = do +checkRemoveKey :: Key -> Maybe Int -> Annex Bool +checkRemoveKey key numcopiesM = do force <- Annex.flagIsSet "force" - if force + if force || numcopiesM == Just 0 then return True else do remotes <- Remotes.keyPossibilities key - numcopies <- getNumCopies + numcopies <- getNumCopies numcopiesM if numcopies > length remotes then notEnoughCopies numcopies (length remotes) [] else findcopies numcopies 0 remotes [] @@ -139,8 +139,9 @@ showTriedRemotes remotes = showLongNote $ "I was unable to access these remotes: " ++ Remotes.list remotes -getNumCopies :: Annex Int -getNumCopies = do +getNumCopies :: Maybe Int -> Annex Int +getNumCopies (Just n) = return n +getNumCopies Nothing = do g <- Annex.gitRepo return $ read $ Git.configGet g config "1" where @@ -153,15 +154,15 @@ getNumCopies = do - The passed action is first run to allow backends deriving this one - to do their own checks. -} -checkKey :: (Key -> Annex Bool) -> Key -> Annex Bool -checkKey a key = do +checkKey :: (Key -> Annex Bool) -> Key -> Maybe Int -> Annex Bool +checkKey a key numcopies = do a_ok <- a key - copies_ok <- checkKeyNumCopies key + copies_ok <- checkKeyNumCopies key numcopies return $ a_ok && copies_ok -checkKeyNumCopies :: Key -> Annex Bool -checkKeyNumCopies key = do - needed <- getNumCopies +checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool +checkKeyNumCopies key numcopies = do + needed <- getNumCopies numcopies remotes <- Remotes.keyPossibilities key inannex <- inAnnex key let present = length remotes + if inannex then 1 else 0 diff --git a/Backend/URL.hs b/Backend/URL.hs index b38ea71c96..3eb7376e01 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -22,11 +22,11 @@ backend = Backend { retrieveKeyFile = downloadUrl, -- allow keys to be removed; presumably they can always be -- downloaded again - removeKey = dummyOk, + removeKey = dummyRemove, -- similarly, keys are always assumed to be out there on the web hasKey = dummyOk, -- and nothing needed to fsck - fsckKey = dummyOk + fsckKey = dummyFsck } -- cannot generate url from filename @@ -37,6 +37,12 @@ keyValue _ = return Nothing dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return False +dummyRemove :: Key -> Maybe a -> Annex Bool +dummyRemove _ _ = return False + +dummyFsck :: Key -> Maybe a -> Annex Bool +dummyFsck _ _ = return True + dummyOk :: Key -> Annex Bool dummyOk _ = return True diff --git a/Command.hs b/Command.hs index 7f3063abb9..059b6e435e 100644 --- a/Command.hs +++ b/Command.hs @@ -44,6 +44,9 @@ type SubCmdStartString = String -> SubCmdStart type BackendFile = (FilePath, Maybe Backend) type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek type SubCmdStartBackendFile = BackendFile -> SubCmdStart +type AttrFile = (FilePath, String) +type SubCmdSeekAttrFiles = SubCmdStartAttrFile -> SubCmdSeek +type SubCmdStartAttrFile = AttrFile -> SubCmdStart type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek type SubCmdStartNothing = SubCmdStart @@ -104,6 +107,13 @@ withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params return $ map a $ filter notState $ foldl (++) [] files +withAttrFilesInGit :: String -> SubCmdSeekAttrFiles +withAttrFilesInGit attr a params = do + repo <- Annex.gitRepo + files <- liftIO $ mapM (Git.inRepo repo) params + pairs <- liftIO $ Git.checkAttr repo attr $ + filter notState $ foldl (++) [] files + return $ map a pairs withFilesMissing :: SubCmdSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params @@ -152,21 +162,21 @@ backendPairs a files = do {- Default to acting on all files matching the seek action if - none are specified. -} -withAll :: SubCmdSeekStrings -> SubCmdSeekStrings +withAll :: (a -> SubCmdSeek) -> a -> SubCmdSeek withAll w a [] = do g <- Annex.gitRepo w a [Git.workTree g] withAll w a p = w a p {- Provides a default parameter to act on if none is specified. -} -withDefault :: String-> SubCmdSeekStrings -> SubCmdSeekStrings +withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek) withDefault d w a [] = w a [d] withDefault _ w a p = w a p {- filter out files from the state directory -} notState :: FilePath -> Bool notState f = stateLoc /= take (length stateLoc) f - + {- filter out symlinks -} notSymlink :: FilePath -> IO Bool notSymlink f = do diff --git a/Command/Drop.hs b/Command/Drop.hs index fbe66f5841..168aa92bd6 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -15,24 +15,27 @@ import LocationLog import Types import Core import Messages +import Utility seek :: [SubCmdSeek] -seek = [withFilesInGit start] +seek = [withAttrFilesInGit "git-annex-numcopies" start] {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -start :: SubCmdStartString -start file = isAnnexed file $ \(key, backend) -> do +start :: SubCmdStartAttrFile +start (file, attr) = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if not inbackend then return Nothing else do showStart "drop" file - return $ Just $ perform key backend + return $ Just $ perform key backend numcopies + where + numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> SubCmdPerform -perform key backend = do - success <- Backend.removeKey backend key +perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform key backend numcopies = do + success <- Backend.removeKey backend key numcopies if success then return $ Just $ cleanup key else return Nothing diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index e8b8d43c4c..016a9faa73 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -30,9 +30,7 @@ start s = do Just key -> do showStart "dropunused" s backend <- keyBackend key - -- force drop, even if this is the only copy - Annex.flagChange "force" $ FlagBool True - return $ Just $ Command.Drop.perform key backend + return $ Just $ Command.Drop.perform key backend (Just 0) readUnusedLog :: Annex (M.Map String Key) readUnusedLog = do diff --git a/Command/Fsck.hs b/Command/Fsck.hs index dc01688015..4341f85cd5 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -11,19 +11,22 @@ import Command import qualified Backend import Types import Messages +import Utility seek :: [SubCmdSeek] -seek = [withAll withFilesInGit start] +seek = [withAll (withAttrFilesInGit "git-annex-numcopies") start] {- Checks a file's backend data for problems. -} -start :: SubCmdStartString -start file = isAnnexed file $ \(key, backend) -> do +start :: SubCmdStartAttrFile +start (file, attr) = isAnnexed file $ \(key, backend) -> do showStart "fsck" file - return $ Just $ perform key backend + return $ Just $ perform key backend numcopies + where + numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> SubCmdPerform -perform key backend = do - success <- Backend.fsckKey backend key +perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform key backend numcopies = do + success <- Backend.fsckKey backend key numcopies if success then return $ Just $ return True else return Nothing diff --git a/Command/FsckFile.hs b/Command/FsckFile.hs deleted file mode 100644 index e7c3d49158..0000000000 --- a/Command/FsckFile.hs +++ /dev/null @@ -1,29 +0,0 @@ -{- git-annex command - - - - Copyright 2010 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Command.FsckFile where - -import Command -import qualified Backend -import Types -import Messages - -seek :: [SubCmdSeek] -seek = [withFilesInGit start] - -{- Checks a file's backend data for problems. -} -start :: SubCmdStartString -start file = isAnnexed file $ \(key, backend) -> do - showStart "fsck" file - return $ Just $ perform key backend - -perform :: Key -> Backend -> SubCmdPerform -perform key backend = do - success <- Backend.fsckKey backend key - if success - then return $ Just $ return True - else return Nothing diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 90ae550581..9580fc5e7c 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -32,8 +32,7 @@ start file = isAnnexed file $ \(key, backend) -> do perform :: FilePath -> Key -> Backend -> SubCmdPerform perform file key backend = do -- force backend to always remove - Annex.flagChange "force" $ FlagBool True - ok <- Backend.removeKey backend key + ok <- Backend.removeKey backend key (Just 0) if ok then return $ Just $ cleanup file key else return Nothing diff --git a/TypeInternals.hs b/TypeInternals.hs index bcef4ee0a8..9acc06bb33 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -72,12 +72,15 @@ data Backend = Backend { storeFileKey :: FilePath -> Key -> Annex Bool, -- retrieves a key's contents to a file retrieveKeyFile :: Key -> FilePath -> Annex Bool, - -- removes a key - removeKey :: Key -> Annex Bool, + -- removes a key, optionally checking that enough copies are stored + -- elsewhere + removeKey :: Key -> Maybe Int -> Annex Bool, -- checks if a backend is storing the content of a key hasKey :: Key -> Annex Bool, -- called during fsck to check a key - fsckKey :: Key -> Annex Bool + -- (second parameter may be the number of copies that there should + -- be of the key) + fsckKey :: Key -> Maybe Int -> Annex Bool } instance Show Backend where diff --git a/Utility.hs b/Utility.hs index 2bea6e8755..882492a2d9 100644 --- a/Utility.hs +++ b/Utility.hs @@ -12,7 +12,8 @@ module Utility ( relPathDirToDir, boolSystem, shellEscape, - unsetFileMode + unsetFileMode, + readMaybe ) where import System.IO @@ -125,3 +126,9 @@ unsetFileMode :: FilePath -> FileMode -> IO () unsetFileMode f m = do s <- getFileStatus f setFileMode f $ fileMode s `intersectFileModes` complement m + +{- Attempts to read a value from a String. -} +readMaybe :: (Read a) => String -> Maybe a +readMaybe s = case reads s of + ((x,_):_) -> Just x + _ -> Nothing diff --git a/debian/changelog b/debian/changelog index 808087dadc..213895c5f7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.10) UNRELEASED; urgency=low * precommit: Optimise to avoid calling git-check-attr more than once. + * In .gitattributes, the git-annex-numcopies attribute can be used + to control the number of copies to retain of different types of files. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 diff --git a/doc/copies.mdwn b/doc/copies.mdwn index aec10ab7af..f647ea622d 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -3,8 +3,11 @@ your git repository's `.git` directory, not in some external data store. It's important that data not get lost by an ill-considered `git annex drop` command. So, then using those backends, git-annex can be configured to try -to keep N copies of a file's content available across all repositories. By -default, N is 1; it is configured by annex.numcopies. +to keep N copies of a file's content available across all repositories. + +By default, N is 1; it is configured by annex.numcopies. This default +can be overridden on a per-file-type basis by the git-annex-numcopies +setting in the `.gitattributes` file. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 27393de503..65cce8cc29 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -275,6 +275,12 @@ but the SHA1 backend for ogg files: * git-annex-backend=WORM *.ogg git-annex-backend=SHA1 +The numcopies setting can also be configured on a per-file-type basis via +the `git-annex-numcopies` attribute. For example, this makes two copies +be needed for ogg files: + + *.ogg git-annex-numcopies=2 + # FILES These files are used, in your git repository: From 52ec6e748d5ef8350e9788d0f19dc5c3d609ab86 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 16:31:20 -0400 Subject: [PATCH 0556/8313] do not need to use Git.relative here (it is a no-op in this case anyway) --- LocationLog.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LocationLog.hs b/LocationLog.hs index 7497d865bb..6d52f4bdd1 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -124,7 +124,7 @@ logNow s u = do {- Returns the filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String logFile repo key = - gitStateDir repo ++ Git.relative repo (keyFile key) ++ ".log" + gitStateDir repo ++ keyFile key ++ ".log" {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} From abf084f628a8c5f5a3685dbb2826739e3e38541e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:17:18 -0400 Subject: [PATCH 0557/8313] Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. --- GitRepo.hs | 26 ++++++++++++++++++-------- Utility.hs | 20 +++++++++++--------- debian/changelog | 2 ++ 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index f50b3fb2e9..539acecb7e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -171,15 +171,17 @@ workTree (Repo { location = Dir d }) = d {- Given a relative or absolute filename in a repository, calculates the - name to use to refer to the file relative to a git repository's top. - This is the same form displayed and used by git. -} -relative :: Repo -> String -> String -relative repo@(Repo { location = Dir d }) file = drop (length absrepo) absfile +relative :: Repo -> FilePath -> IO FilePath +relative repo@(Repo { location = Dir d }) file = do + cwd <- getCurrentDirectory + return $ drop (length absrepo) (absfile cwd) where -- normalize both repo and file, so that repo -- will be substring of file absrepo = case (absNormPath "/" d) of Just f -> f ++ "/" Nothing -> error $ "bad repo" ++ repoDescribe repo - absfile = case (secureAbsNormPath absrepo file) of + absfile c = case (secureAbsNormPath c file) of Just f -> f Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo relative repo _ = assertLocal repo $ error "internal" @@ -333,18 +335,26 @@ configGet repo key defaultValue = configMap :: Repo -> Map.Map String String configMap repo = config repo -{- Looks up a gitattributes value for each file in a list. -} +{- Efficiently looks up a gitattributes value for each file in a list. -} checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)] checkAttr repo attr files = do - (_, s) <- pipeBoth "git" params files0 - return $ map topair $ lines s + -- git check-attr wants files that are absolute (or relative to the + -- top of the repo). But we're passed files relative to the current + -- directory. Convert to absolute, and then convert the filenames + -- in its output back to relative. + absfiles <- mapM absPath files + (_, s) <- pipeBoth "git" params $ join "\0" absfiles + cwd <- getCurrentDirectory + return $ map (topair $ cwd++"/") $ lines s -- XXX handle is left open, this is ok for git-annex, but may need -- to be cleaned up for other uses. where params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] - files0 = join "\0" files - topair l = (file, value) + topair cwd l = (relfile, value) where + relfile + | startswith cwd file = drop (length cwd) file + | otherwise = file file = decodeGitFile $ join sep $ take end bits value = bits !! end end = length bits - 1 diff --git a/Utility.hs b/Utility.hs index 882492a2d9..0f7ce42aa6 100644 --- a/Utility.hs +++ b/Utility.hs @@ -8,6 +8,7 @@ module Utility ( hGetContentsStrict, parentDir, + absPath, relPathCwdToDir, relPathDirToDir, boolSystem, @@ -44,24 +45,25 @@ parentDir dir = slash = if isAbsolute dir then s else "" s = [pathSeparator] +{- Converts a filename into a normalized, absolute path. -} +absPath :: FilePath -> IO FilePath +absPath file = do + cwd <- getCurrentDirectory + case absNormPath cwd file of + Just f -> return f + Nothing -> error $ "unable to normalize " ++ file + {- Constructs a relative path from the CWD to a directory. - - For example, assuming CWD is /tmp/foo/bar: - relPathCwdToDir "/tmp/foo" == "../" - relPathCwdToDir "/tmp/foo/bar" == "" - - relPathCwdToDir "/tmp/foo/bar" == "" -} relPathCwdToDir :: FilePath -> IO FilePath relPathCwdToDir dir = do cwd <- getCurrentDirectory - let absdir = absnorm cwd - return $ relPathDirToDir cwd absdir - where - -- absolute, normalized form of the directory - absnorm cwd = - case absNormPath cwd dir of - Just d -> d - Nothing -> error $ "unable to normalize " ++ dir + a <- absPath dir + return $ relPathDirToDir cwd a {- Constructs a relative path from one directory to another. - diff --git a/debian/changelog b/debian/changelog index 213895c5f7..34e1757dfd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (0.10) UNRELEASED; urgency=low * precommit: Optimise to avoid calling git-check-attr more than once. * In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. + * Bugfix: Always correctly handle gitattributes when in a subdirectory of + the repository. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 From 949e4abc56df1fa23d426f7f86d726ad119fbf54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:26:15 -0400 Subject: [PATCH 0558/8313] fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. --- Backend/File.hs | 6 +++--- debian/changelog | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 5984348b36..e3225a8b85 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -163,9 +163,9 @@ checkKey a key numcopies = do checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool checkKeyNumCopies key numcopies = do needed <- getNumCopies numcopies - remotes <- Remotes.keyPossibilities key - inannex <- inAnnex key - let present = length remotes + if inannex then 1 else 0 + g <- Annex.gitRepo + locations <- liftIO $ keyLocations g key + let present = length locations if present < needed then do warning $ note present needed diff --git a/debian/changelog b/debian/changelog index 34e1757dfd..ad9c00f536 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.10) UNRELEASED; urgency=low to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. + * fsck: Fix warning about not enough copies of a file, when locations + are known, but are not available in currently configured remotes. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 From 7e82d420d80572af2b9f047039ecbafbb2d79e8e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:32:06 -0400 Subject: [PATCH 0559/8313] missing \n in -q mode --- Messages.hs | 2 +- debian/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Messages.hs b/Messages.hs index e1cf1539a6..2934de4287 100644 --- a/Messages.hs +++ b/Messages.hs @@ -53,4 +53,4 @@ showErr e = warning $ show e warning :: String -> Annex () warning s = do verbose $ liftIO $ putStr "\n" - liftIO $ hPutStr stderr $ "git-annex: " ++ s ++ " " + liftIO $ hPutStrLn stderr $ "git-annex: " ++ s ++ " " diff --git a/debian/changelog b/debian/changelog index ad9c00f536..76ee103d0f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,12 @@ git-annex (0.10) UNRELEASED; urgency=low - * precommit: Optimise to avoid calling git-check-attr more than once. * In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. * fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. + * precommit: Optimise to avoid calling git-check-attr more than once. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 From fe4f1aae4b6c5fe3527f3e2462efac6e337f4978 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:33:01 -0400 Subject: [PATCH 0560/8313] include key in message --- Backend/File.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index e3225a8b85..d09e09f56e 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -172,7 +172,7 @@ checkKeyNumCopies key numcopies = do return False else return True where - note 0 _ = "** No known copies of the file exist!" + note 0 _ = "** No known copies of "++show key++" exist!" note present needed = "Only " ++ show present ++ " of " ++ show needed ++ " copies of "++show key++" exist. " ++ From dabfc455c678b073dadda41a4029493716e29c84 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:34:31 -0400 Subject: [PATCH 0561/8313] clarify --- debian/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 76ee103d0f..3f66190e59 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.10) UNRELEASED; urgency=low * In .gitattributes, the git-annex-numcopies attribute can be used to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of - the repository. + the repository. (Had worked ok for ones like "*.mp3", but failed for + ones like "dir/*".) * fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. * precommit: Optimise to avoid calling git-check-attr more than once. From 1d78dc2c8db2550b530a87cf893c2445162e4ea1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:54:42 -0400 Subject: [PATCH 0562/8313] update docs --- doc/copies.mdwn | 2 +- doc/git-annex.mdwn | 12 ++++++------ doc/walkthrough.mdwn | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index f647ea622d..a89376a5e4 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -7,7 +7,7 @@ to keep N copies of a file's content available across all repositories. By default, N is 1; it is configured by annex.numcopies. This default can be overridden on a per-file-type basis by the git-annex-numcopies -setting in the `.gitattributes` file. +setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 65cce8cc29..09282e7213 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -238,7 +238,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. # CONFIGURATION -Like other git commands, git-annex is configured via `git-config`. +Like other git commands, git-annex is configured via `.git/config`. Here are all the supported configuration settings. * `annex.uuid` -- a unique UUID for this repository (automatically set) @@ -267,7 +267,7 @@ Here are all the supported configuration settings. between versions. The backend used when adding a new file to the annex can be configured -on a per-file-type basis via the `.gitattributes` file. In the file, +on a per-file-type basis via `.gitattributes` files. In the file, the `git-annex-backend` attribute can be set to the name of the backend to use. For example, this here's how to use the WORM backend by default, but the SHA1 backend for ogg files: @@ -276,14 +276,14 @@ but the SHA1 backend for ogg files: *.ogg git-annex-backend=SHA1 The numcopies setting can also be configured on a per-file-type basis via -the `git-annex-numcopies` attribute. For example, this makes two copies -be needed for ogg files: +the `git-annex-numcopies` attribute in `.gitattributes` files. +For example, this makes two copies be needed for wav files: - *.ogg git-annex-numcopies=2 + *.wav git-annex-numcopies=2 # FILES -These files are used, in your git repository: +These files are used by git-annex, in your git repository: `.git/annex/objects/` contains the annexed file contents that are currently available. Annexed files in your git repository symlink to that content. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 281f460506..9d4d2ce594 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -339,3 +339,31 @@ might say about a badly messed up annex: git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. failed git-annex: 2 failed + +## backups + +git-annex can be configured to require more than one copy of a file exists, +as a simple backup for your data. This is controled by the "numcopies" +setting, which defaults to 1 copy. Let's change that to require 2 copies, +and send a copy of every file to a USB drive. + + # echo "* git-annex-numcopies=2" >> .gitattributes + # git annex copy . --to usbdrive + +Now when we try to `git annex drop` a file, it will verify that it +knows of 2 other repositories that have a copy before removing its +content from the current repository. + +You can also vary the number of copies needed, depending on the file name. +So, if you want 3 copies of all your flac files, but only 1 copy of oggs: + + # echo "*.ogg git-annex-numcopies=1" >> .gitattributes + # echo "*.flac git-annex-numcopies=3" >> .gitattributes + +Or, you might want to make a directory for important stuff, and configure +it so anything put in there is backed up more thoroughly: + + # mkdir important_stuff + # echo "* git-annex-numcopies=3" > important_stuff/.gitattributes + +For more details about the numcopies setting, see [[copies]]. From dc54214404dbe6e9d602a8a14c08411ca3eaceda Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 17:56:02 -0400 Subject: [PATCH 0563/8313] update --- doc/copies.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index a89376a5e4..f157893ce6 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -30,4 +30,6 @@ to both USB and Server. Note that different repositories can be configured with different values of N. So just because Laptop has N=2, this does not prevent the number of -copies falling to 1, when USB and Server have N=1. +copies falling to 1, when USB and Server have N=1. To avoid this, +configure it in `.gitattributes`, which is shared between repositories +using git. From 9d82e815ff5307187d195fd2529420ef7970c412 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 18:55:49 -0400 Subject: [PATCH 0564/8313] change name of numcopies attribute --- Command/Drop.hs | 2 +- Command/Fsck.hs | 2 +- debian/changelog | 2 +- doc/copies.mdwn | 2 +- doc/git-annex.mdwn | 4 ++-- doc/walkthrough.mdwn | 10 +++++----- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Command/Drop.hs b/Command/Drop.hs index 168aa92bd6..7c4fbea602 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -18,7 +18,7 @@ import Messages import Utility seek :: [SubCmdSeek] -seek = [withAttrFilesInGit "git-annex-numcopies" start] +seek = [withAttrFilesInGit "annex.numcopies" start] {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 4341f85cd5..9acecfce67 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -14,7 +14,7 @@ import Messages import Utility seek :: [SubCmdSeek] -seek = [withAll (withAttrFilesInGit "git-annex-numcopies") start] +seek = [withAll (withAttrFilesInGit "annex.numcopies") start] {- Checks a file's backend data for problems. -} start :: SubCmdStartAttrFile diff --git a/debian/changelog b/debian/changelog index 3f66190e59..767583844e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,6 @@ git-annex (0.10) UNRELEASED; urgency=low - * In .gitattributes, the git-annex-numcopies attribute can be used + * In .gitattributes, the annex.numcopies attribute can be used to control the number of copies to retain of different types of files. * Bugfix: Always correctly handle gitattributes when in a subdirectory of the repository. (Had worked ok for ones like "*.mp3", but failed for diff --git a/doc/copies.mdwn b/doc/copies.mdwn index f157893ce6..165e54b340 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -6,7 +6,7 @@ command. So, then using those backends, git-annex can be configured to try to keep N copies of a file's content available across all repositories. By default, N is 1; it is configured by annex.numcopies. This default -can be overridden on a per-file-type basis by the git-annex-numcopies +can be overridden on a per-file-type basis by the annex.numcopies setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 09282e7213..a2af9d990e 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -276,10 +276,10 @@ but the SHA1 backend for ogg files: *.ogg git-annex-backend=SHA1 The numcopies setting can also be configured on a per-file-type basis via -the `git-annex-numcopies` attribute in `.gitattributes` files. +the `annex.numcopies` attribute in `.gitattributes` files. For example, this makes two copies be needed for wav files: - *.wav git-annex-numcopies=2 + *.wav annex.numcopies=2 # FILES diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 9d4d2ce594..6a1c688b34 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -343,11 +343,11 @@ might say about a badly messed up annex: ## backups git-annex can be configured to require more than one copy of a file exists, -as a simple backup for your data. This is controled by the "numcopies" +as a simple backup for your data. This is controled by the "annex.numcopies" setting, which defaults to 1 copy. Let's change that to require 2 copies, and send a copy of every file to a USB drive. - # echo "* git-annex-numcopies=2" >> .gitattributes + # echo "* annex.numcopies=2" >> .gitattributes # git annex copy . --to usbdrive Now when we try to `git annex drop` a file, it will verify that it @@ -357,13 +357,13 @@ content from the current repository. You can also vary the number of copies needed, depending on the file name. So, if you want 3 copies of all your flac files, but only 1 copy of oggs: - # echo "*.ogg git-annex-numcopies=1" >> .gitattributes - # echo "*.flac git-annex-numcopies=3" >> .gitattributes + # echo "*.ogg annex.numcopies=1" >> .gitattributes + # echo "*.flac annex.numcopies=3" >> .gitattributes Or, you might want to make a directory for important stuff, and configure it so anything put in there is backed up more thoroughly: # mkdir important_stuff - # echo "* git-annex-numcopies=3" > important_stuff/.gitattributes + # echo "* annex.numcopies=3" > important_stuff/.gitattributes For more details about the numcopies setting, see [[copies]]. From ca32c7859bb675c70433f23d7f5b4fb5f569d704 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 18:58:03 -0400 Subject: [PATCH 0565/8313] The git-annex-backend attribute has been renamed to annex.backend. --- Backend.hs | 2 +- debian/changelog | 1 + doc/backends.mdwn | 6 +++--- doc/git-annex.mdwn | 6 +++--- doc/walkthrough.mdwn | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Backend.hs b/Backend.hs index 8e17f253f4..8142e4707a 100644 --- a/Backend.hs +++ b/Backend.hs @@ -151,7 +151,7 @@ chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] chooseBackends fs = do g <- Annex.gitRepo bs <- Annex.supportedBackends - pairs <- liftIO $ Git.checkAttr g "git-annex-backend" fs + pairs <- liftIO $ Git.checkAttr g "annex.backend" fs return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs {- Returns the backend to use for a key. -} diff --git a/debian/changelog b/debian/changelog index 767583844e..fcd768bf7a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.10) UNRELEASED; urgency=low * fsck: Fix warning about not enough copies of a file, when locations are known, but are not available in currently configured remotes. * precommit: Optimise to avoid calling git-check-attr more than once. + * The git-annex-backend attribute has been renamed to annex.backend. -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 diff --git a/doc/backends.mdwn b/doc/backends.mdwn index be4eac7232..3e605e4b15 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -26,12 +26,12 @@ git-annex should use. The first one listed will be used by default when new files are added. For finer control of what backend is used when adding different types of -files, the `.gitattributes` file can be used. The `git-annex-backend` +files, the `.gitattributes` file can be used. The `annex.backend` attribute can be set to the name of the backend to use for matching files. For example, to use the SHA1 backend for sound files, which tend to be smallish and might be modified over time, you could set in `.gitattributes`: - *.mp3 git-annex-backend=SHA1 - *.ogg git-annex-backend=SHA1 + *.mp3 annex.backend=SHA1 + *.ogg annex.backend=SHA1 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a2af9d990e..64ca6a5497 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -268,12 +268,12 @@ Here are all the supported configuration settings. The backend used when adding a new file to the annex can be configured on a per-file-type basis via `.gitattributes` files. In the file, -the `git-annex-backend` attribute can be set to the name of the backend to +the `annex.backend` attribute can be set to the name of the backend to use. For example, this here's how to use the WORM backend by default, but the SHA1 backend for ogg files: - * git-annex-backend=WORM - *.ogg git-annex-backend=SHA1 + * annex.backend=WORM + *.ogg annex.backend=SHA1 The numcopies setting can also be configured on a per-file-type basis via the `annex.numcopies` attribute in `.gitattributes` files. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 6a1c688b34..2aa2478b32 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -275,7 +275,7 @@ files when they're added to the annex, and this can slow things down significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: - * git-annex-backend=SHA1 + * annex.backend=SHA1 ## unused data From 0643b7f4c66831177173654b2617e2ce479cce63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 19:32:00 -0400 Subject: [PATCH 0566/8313] releasing version 0.10 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index fcd768bf7a..b87f835b80 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.10) UNRELEASED; urgency=low +git-annex (0.10) unstable; urgency=low * In .gitattributes, the annex.numcopies attribute can be used to control the number of copies to retain of different types of files. @@ -10,7 +10,7 @@ git-annex (0.10) UNRELEASED; urgency=low * precommit: Optimise to avoid calling git-check-attr more than once. * The git-annex-backend attribute has been renamed to annex.backend. - -- Joey Hess Sun, 28 Nov 2010 14:19:15 -0400 + -- Joey Hess Sun, 28 Nov 2010 19:28:05 -0400 git-annex (0.09) unstable; urgency=low From d5bb2da688fc85d748e0b1f8e3d97a6d0c0a6cb3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 28 Nov 2010 19:32:47 -0400 Subject: [PATCH 0567/8313] add news item for git-annex 0.10 --- doc/news/version_0.05.mdwn | 10 ---------- doc/news/version_0.10.mdwn | 11 +++++++++++ 2 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 doc/news/version_0.05.mdwn create mode 100644 doc/news/version_0.10.mdwn diff --git a/doc/news/version_0.05.mdwn b/doc/news/version_0.05.mdwn deleted file mode 100644 index 88c7391c13..0000000000 --- a/doc/news/version_0.05.mdwn +++ /dev/null @@ -1,10 +0,0 @@ -git-annex 0.05 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Optimize both pre-commit and lock subcommands to not call git diff - on every file being committed/locked. - (This actually also works around a bug in ghc, that caused - git-annex 0.04 pre-commit to sometimes corrupt filename being read - from git ls-files and fail. - The excessive number of calls made by pre-commit exposed the ghc bug. - Thanks Josh Triplett for the debugging.) - * Build with -O2."""]] \ No newline at end of file diff --git a/doc/news/version_0.10.mdwn b/doc/news/version_0.10.mdwn new file mode 100644 index 0000000000..450b7d272f --- /dev/null +++ b/doc/news/version_0.10.mdwn @@ -0,0 +1,11 @@ +git-annex 0.10 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * In .gitattributes, the annex.numcopies attribute can be used + to control the number of copies to retain of different types of files. + * Bugfix: Always correctly handle gitattributes when in a subdirectory of + the repository. (Had worked ok for ones like "*.mp3", but failed for + ones like "dir/*".) + * fsck: Fix warning about not enough copies of a file, when locations + are known, but are not available in currently configured remotes. + * precommit: Optimise to avoid calling git-check-attr more than once. + * The git-annex-backend attribute has been renamed to annex.backend."""]] \ No newline at end of file From ade43d8fab2b42846a06201f03e4a03255ef8fce Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 1 Dec 2010 19:19:36 -0400 Subject: [PATCH 0568/8313] tweak --- BackendList.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BackendList.hs b/BackendList.hs index 59ec026d91..d1180d22f9 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -11,7 +11,7 @@ module BackendList (allBackends) where import qualified Backend.WORM import qualified Backend.SHA1 import qualified Backend.URL -import TypeInternals +import Types allBackends :: [Backend] allBackends = From 37941184f99896a459fd889071e47ed2fa5ceaa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 16:55:21 -0400 Subject: [PATCH 0569/8313] Rsync will now be used to resume interrupted/failed partial file transfers from a remote. --- Remotes.hs | 20 +++++++++++++++++--- configure.hs | 11 ++++++----- debian/changelog | 7 +++++++ debian/control | 4 ++-- doc/install.mdwn | 1 + doc/todo/rsync.mdwn | 2 ++ 6 files changed, 35 insertions(+), 10 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index cb8081d740..6bb67216bf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -36,6 +36,7 @@ import Utility import qualified Core import Messages import CopyFile +import qualified SysConfig {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -199,9 +200,13 @@ copyFromRemote r key file | Git.repoIsSsh r = getssh | otherwise = error "copying from non-ssh repo not supported" where - getlocal = liftIO $ copyFile keyloc file - getssh = scp r [sshLocation r keyloc, file] keyloc = annexLocation r key + getlocal = liftIO $ copyFile keyloc file + getssh = do + exists <- liftIO $ doesFileExist file + if exists && SysConfig.rsync + then rsync r [sshLocation r keyloc, file] + else scp r [sshLocation r keyloc, file] {- Tries to copy a key's content to a file on a remote. -} copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -224,9 +229,18 @@ sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file scp :: Git.Repo -> [String] -> Annex Bool scp r params = do scpoptions <- repoConfig r "scp-options" "" - showProgress -- make way for scp progress bar + showProgress -- make way for progress bar liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params +{- Runs rsync against a specified remote, resuming any interrupted file + - transfer. (Honors annex-rsync-options.) -} +rsync :: Git.Repo -> [String] -> Annex Bool +rsync r params = do + rsyncoptions <- repoConfig r "rsync-options" "" + showProgress -- make way for progress bar + liftIO $ boolSystem "rsync" $ ["--progress", "-a", "--inplace"] ++ + words rsyncoptions ++ params + {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} runCmd :: Git.Repo -> String -> [String] -> Annex Bool diff --git a/configure.hs b/configure.hs index 92a051c80e..2334385a38 100644 --- a/configure.hs +++ b/configure.hs @@ -20,8 +20,9 @@ tests = [ TestCase "cp -a" "cp_a" $ testCp "-a" , TestCase "cp -p" "cp_p" $ testCp "-p" , TestCase "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" - , TestCase "uuid" "uuid" $ requireCommand "uuid" "uuid" - , TestCase "xargs -0" "xargs_0" $ requireCommand "xargs -0" "xargs -0 /dev/null" ] tmpDir :: String @@ -33,14 +34,14 @@ testFile = tmpDir ++ "/testfile" quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" -requireCommand :: String -> String -> Test -requireCommand command cmdline = do +requireCmd :: String -> String -> Test +requireCmd c cmdline = do ret <- testCmd $ quiet cmdline if ret then return True else do testEnd False - error $ "** the " ++ command ++ " command is required to use git-annex" + error $ "** the " ++ c ++ " command is required to use git-annex" testCp :: String -> Test testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ diff --git a/debian/changelog b/debian/changelog index b87f835b80..0bdfbf76ac 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.11) UNRELEASED; urgency=low + + * Rsync will now be used to resume interrupted/failed partial file + transfers from a remote. + + -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 + git-annex (0.10) unstable; urgency=low * In .gitattributes, the annex.numcopies attribute can be used diff --git a/debian/control b/debian/control index 0318dafa4e..88dcab9a3a 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex @@ -10,7 +10,7 @@ Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client +Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/install.mdwn b/doc/install.mdwn index ec9ea92c7f..bca6eb0190 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,6 +5,7 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: +* `rsync` (optional but recommended) * Then just [[download]] git-annex and run: `make; make install` ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, diff --git a/doc/todo/rsync.mdwn b/doc/todo/rsync.mdwn index 75e0175c86..3353f19c43 100644 --- a/doc/todo/rsync.mdwn +++ b/doc/todo/rsync.mdwn @@ -1,2 +1,4 @@ Transferring a file from a ssh:// remote should use rsync to allow resuming of a prior transfer. + +[[done]] From adad12d3374102790cab0cfbb9a5f0721ff88fc9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 16:57:02 -0400 Subject: [PATCH 0570/8313] update --- doc/install.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install.mdwn b/doc/install.mdwn index bca6eb0190..7da45c85b4 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,7 +5,7 @@ To build and use git-annex, you will need: * MissingH: * `uuid`: * `xargs`: -* `rsync` (optional but recommended) +* `rsync`: (optional but recommended) * Then just [[download]] git-annex and run: `make; make install` ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, From b9320ee1d53bfe72b0fbf7e08c927f5b45bbc5c9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:45:28 -0400 Subject: [PATCH 0571/8313] use rsync for all remote file transfers --- Remotes.hs | 45 +++++++++++++++++++++++++-------------------- debian/changelog | 5 +++-- doc/git-annex.mdwn | 10 +++++++--- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 6bb67216bf..6dc5160482 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -202,11 +202,7 @@ copyFromRemote r key file where keyloc = annexLocation r key getlocal = liftIO $ copyFile keyloc file - getssh = do - exists <- liftIO $ doesFileExist file - if exists && SysConfig.rsync - then rsync r [sshLocation r keyloc, file] - else scp r [sshLocation r keyloc, file] + getssh = remoteCopyFile r (sshLocation r keyloc) file {- Tries to copy a key's content to a file on a remote. -} copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -220,26 +216,35 @@ copyToRemote r key file = do else error "copying to non-ssh repo not supported" where putlocal src = liftIO $ copyFile src file - putssh src = scp r [src, sshLocation r file] + putssh src = remoteCopyFile r src (sshLocation r file) sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file -{- Runs scp against a specified remote. (Honors annex-scp-options.) -} -scp :: Git.Repo -> [String] -> Annex Bool -scp r params = do - scpoptions <- repoConfig r "scp-options" "" +{- Copys a file from or to a remote, using rsync (when available) or scp. -} +remoteCopyFile :: Git.Repo -> String -> String -> Annex Bool +remoteCopyFile r src dest = do showProgress -- make way for progress bar - liftIO $ boolSystem "scp" $ "-p":(words scpoptions) ++ params - -{- Runs rsync against a specified remote, resuming any interrupted file - - transfer. (Honors annex-rsync-options.) -} -rsync :: Git.Repo -> [String] -> Annex Bool -rsync r params = do - rsyncoptions <- repoConfig r "rsync-options" "" - showProgress -- make way for progress bar - liftIO $ boolSystem "rsync" $ ["--progress", "-a", "--inplace"] ++ - words rsyncoptions ++ params + o <- repoConfig r configopt "" + res <- liftIO $ boolSystem cmd $ options ++ words o ++ [src, dest] + if res + then return res + else do + when rsync $ + showLongNote "run git annex again to resume file transfer" + return res + where + cmd + | rsync = "rsync" + | otherwise = "scp" + configopt + | rsync = "rsync-options" + | otherwise = "scp-options" + options + -- inplace makes rsync resume partial files + | rsync = ["-p", "--progress", "--inplace"] + | otherwise = ["-p"] + rsync = SysConfig.rsync {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} diff --git a/debian/changelog b/debian/changelog index 0bdfbf76ac..5a9d332bcb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ git-annex (0.11) UNRELEASED; urgency=low - * Rsync will now be used to resume interrupted/failed partial file - transfers from a remote. + * If available, rsync will be used for file transfers from remote + repositories. This allows resuming interrupted transfers. + * Added remote.annex-rsync-options. -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 64ca6a5497..63bc11eb72 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -258,11 +258,15 @@ Here are all the supported configuration settings. here. * `remote..annex-scp-options` -- Options to use when using scp to or from this repository. For example, to force ipv6, and limit - the bandwidth to 100Kbit/s, set it to "-6 -l 100" + the bandwidth to 1000Kbit/s, set it to "-6 -l 1000" * `remote..annex-ssh-options` -- Options to use when using ssh to talk to this repository. -* `annex.scp-options` and `annex.ssh-options` -- Default scp and ssh - options to use if a remote does not have specific options. +* `remote..annex-rsync-options` -- Options to use when using rsync + to or from this repository. For example, to force ipv6, and limit + the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" +* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` -- + Default scp, ssh, and rsync options to use if a remote does not have + specific options. * `annex.version` -- Automatically maintained, and used to automate upgrades between versions. From 2fba1ba40d9c3f07f36e68515ceb1031e7983421 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:51:02 -0400 Subject: [PATCH 0572/8313] Avoid deleting temp files when rsync fails. --- Core.hs | 25 ++++++++++++++----------- debian/changelog | 1 + git-annex.hs | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Core.hs b/Core.hs index b61d186666..8cdb063c79 100644 --- a/Core.hs +++ b/Core.hs @@ -29,7 +29,7 @@ import Messages {- Runs a list of Annex actions. Catches IO errors and continues - (but explicitly thrown errors terminate the whole command). - - Propigates an overall error status at the end. + - Runs shutdown and propigates an overall error status at the end. -} tryRun :: AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions @@ -42,7 +42,8 @@ tryRun' state errnum (a:as) = do tryRun' state (errnum + 1) as Right (True,state') -> tryRun' state' errnum as Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' _ errnum [] = +tryRun' state errnum [] = do + _ <- try $ Annex.run state $ shutdown errnum when (errnum > 0) $ error $ show errnum ++ " failed" {- Actions to perform each time ran. -} @@ -52,20 +53,22 @@ startup = do return True {- When git-annex is done, it runs this. -} -shutdown :: Annex Bool -shutdown = do +shutdown :: Integer -> Annex Bool +shutdown errnum = do q <- Annex.queueGet unless (q == GitQueue.empty) $ do showSideAction "Recording state in git..." Annex.queueRun - -- clean up any files left in the temp directory, but leave - -- the tmp directory itself - g <- Annex.gitRepo - let tmp = annexTmpLocation g - exists <- liftIO $ doesDirectoryExist tmp - when exists $ liftIO $ removeDirectoryRecursive tmp - liftIO $ createDirectoryIfMissing True tmp + -- If nothing failed, clean up any files left in the temp directory, + -- but leave the directory itself. If something failed, temp files + -- are left behind to allow resuming on re-run. + when (errnum == 0) $ do + g <- Annex.gitRepo + let tmp = annexTmpLocation g + exists <- liftIO $ doesDirectoryExist tmp + when exists $ liftIO $ removeDirectoryRecursive tmp + liftIO $ createDirectoryIfMissing True tmp return True diff --git a/debian/changelog b/debian/changelog index 5a9d332bcb..fb2d2aae22 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.11) UNRELEASED; urgency=low * If available, rsync will be used for file transfers from remote repositories. This allows resuming interrupted transfers. * Added remote.annex-rsync-options. + * Avoid deleting temp files when rsync fails. -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 diff --git a/git-annex.hs b/git-annex.hs index d111156f01..417d335e16 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -20,4 +20,4 @@ main = do gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends (configure, actions) <- parseCmd args state - tryRun state $ [startup, upgrade] ++ configure ++ actions ++ [shutdown] + tryRun state $ [startup, upgrade] ++ configure ++ actions From 07648e2daa29fe8d275605c1873efc7912ee8499 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:52:23 -0400 Subject: [PATCH 0573/8313] remove note that looked ugly with resume message --- Backend/File.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index d09e09f56e..afbe38ac0f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -58,7 +58,6 @@ copyKeyFile key file = do else trycopy remotes remotes where trycopy full [] = do - showNote "not available" showTriedRemotes full showLocations key return False From ece7481faad5ce5f040b684c4bd02a37f50a5ff0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:54:08 -0400 Subject: [PATCH 0574/8313] reword --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index 6dc5160482..cf49b624b4 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -231,7 +231,7 @@ remoteCopyFile r src dest = do then return res else do when rsync $ - showLongNote "run git annex again to resume file transfer" + showLongNote "rsync failed -- run git annex again to resume file transfer" return res where cmd From d1b5ef9565cfc5991a473985a36fe00c9384ece5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 17:56:25 -0400 Subject: [PATCH 0575/8313] update for rsync --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 2aa2478b32..9c8106d513 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -198,7 +198,7 @@ to clone the laptop's annex to it: # cd ~/annex # git annex init "my desktop" -Now you can get files and they will be transferred by `scp`: +Now you can get files and they will be transferred (using `rsync` or `scp`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) From 83a87a522903e18a16ae19e1b741ab4e1f2b95a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 21:07:16 -0400 Subject: [PATCH 0576/8313] Improve detection of version 0 repos. --- Version.hs | 6 ++++-- debian/changelog | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Version.hs b/Version.hs index 8394701201..fc1ce3d7ec 100644 --- a/Version.hs +++ b/Version.hs @@ -29,9 +29,11 @@ getVersion = do then return $ Just v else do -- version 0 was not recorded in .git/config; - -- such a repo should have an annexDir + -- such a repo should have an annexDir but no + -- annexObjectDir d <- liftIO $ doesDirectoryExist $ annexDir g - if d + o <- liftIO $ doesDirectoryExist $ annexObjectDir g + if d && not o then return $ Just "0" else return Nothing -- no version yet diff --git a/debian/changelog b/debian/changelog index fb2d2aae22..9e378400c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.11) UNRELEASED; urgency=low repositories. This allows resuming interrupted transfers. * Added remote.annex-rsync-options. * Avoid deleting temp files when rsync fails. + * Improve detection of version 0 repos. -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 From 71d60eb87a492a9561f7142cc897ff5961c21e25 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 21:26:37 -0400 Subject: [PATCH 0577/8313] robustness fix don't crash if an object directory does not contain a file --- Core.hs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Core.hs b/Core.hs index 8cdb063c79..e3702044e8 100644 --- a/Core.hs +++ b/Core.hs @@ -190,9 +190,11 @@ getKeysPresent' dir = do return $ map fileKey files where present d = do - s <- getFileStatus $ dir ++ "/" ++ d ++ "/" - ++ takeFileName d - return $ isRegularFile s + result <- try $ + getFileStatus $ dir ++ "/" ++ d ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False {- List of keys referenced by symlinks in the git repo. -} getKeysReferenced :: Annex [Key] From 6f932a0963341b235f262c43982cc138f443f2de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 2 Dec 2010 21:37:55 -0400 Subject: [PATCH 0578/8313] avoid building news page when building doc wiki for package --- Makefile | 3 ++- doc/news.mdwn | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 17918f9567..ad143e7b69 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,8 @@ docs: $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ --underlaydir=/dev/null --disable-plugin=shortcut \ - --disable-plugin=smiley + --disable-plugin=smiley \ + --exclude='news/.*' clean: rm -rf build git-annex git-annex.1 test configure SysConfig.hs diff --git a/doc/news.mdwn b/doc/news.mdwn index d0ff1ca2c1..3d9ece3bbb 100644 --- a/doc/news.mdwn +++ b/doc/news.mdwn @@ -1,5 +1,11 @@ +[[!if test="news/*" then=""" This is where announcements of new releases, features, and other news is posted. git-annex users are recommended to subscribe to this page's RSS feed. [[!inline pages="./news/* and !*/Discussion" rootpage="news" show="30"]] + +""" +else=""" +(Please see the changelog.) +"""]] From 57305570eb5fce88c743ca4f7ff127c7ef582310 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Dec 2010 00:33:41 -0400 Subject: [PATCH 0579/8313] Add uninit subcommand. Closes: #605749 --- CmdLine.hs | 3 +++ Command/Init.hs | 41 ++++++++++++++++++++------------ Command/Uninit.hs | 59 ++++++++++++++++++++++++++++++++++++++++++++++ Utility.hs | 5 ++++ debian/changelog | 1 + doc/git-annex.mdwn | 6 +++++ 6 files changed, 100 insertions(+), 15 deletions(-) create mode 100644 Command/Uninit.hs diff --git a/CmdLine.hs b/CmdLine.hs index 837420786f..0903cc1fb7 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -32,6 +32,7 @@ import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find +import qualified Command.Uninit subCmds :: [SubCommand] subCmds = @@ -55,6 +56,8 @@ subCmds = "initialize git-annex with repository description" , SubCommand "unannex" path Command.Unannex.seek "undo accidential add command" + , SubCommand "uninit" path Command.Uninit.seek + "de-initialize git-annex and clean out repository" , SubCommand "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" , SubCommand "fromkey" key Command.FromKey.seek diff --git a/Command/Init.hs b/Command/Init.hs index eb5c58696f..e19849ba3b 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -18,6 +18,7 @@ import UUID import Version import Messages import Locations +import Types seek :: [SubCmdSeek] seek = [withString start] @@ -36,8 +37,8 @@ perform description = do u <- getUUID g describeUUID u description setVersion - liftIO $ gitAttributes g - liftIO $ gitPreCommitHook g + liftIO $ gitAttributesWrite g + gitPreCommitHookWrite g return $ Just cleanup cleanup :: SubCmdCleanup @@ -50,8 +51,8 @@ cleanup = do {- configure git to use union merge driver on state files, if it is not - already -} -gitAttributes :: Git.Repo -> IO () -gitAttributes repo = do +gitAttributesWrite :: Git.Repo -> IO () +gitAttributesWrite repo = do exists <- doesFileExist attributes if not exists then do @@ -63,24 +64,34 @@ gitAttributes repo = do appendFile attributes $ attrLine ++ "\n" commit where - attrLine = stateLoc ++ "*.log merge=union" attributes = Git.attributes repo commit = do Git.run repo ["add", attributes] Git.run repo ["commit", "-m", "git-annex setup", attributes] +attrLine :: String +attrLine = stateLoc ++ "*.log merge=union" + {- set up a git pre-commit hook, if one is not already present -} -gitPreCommitHook :: Git.Repo -> IO () -gitPreCommitHook repo = do - let hook = Git.workTree repo ++ "/" ++ Git.gitDir repo ++ - "/hooks/pre-commit" - exists <- doesFileExist hook +gitPreCommitHookWrite :: Git.Repo -> Annex () +gitPreCommitHookWrite repo = do + exists <- liftIO $ doesFileExist hook if exists - then putStrLn $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" - else do - writeFile hook $ "#!/bin/sh\n" ++ - "# automatically configured by git-annex\n" ++ - "git annex pre-commit .\n" + then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring" + else liftIO $ do + writeFile hook preCommitScript p <- getPermissions hook setPermissions hook $ p {executable = True} + where + hook = preCommitHook repo + +preCommitHook :: Git.Repo -> FilePath +preCommitHook repo = + Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit" + +preCommitScript :: String +preCommitScript = + "#!/bin/sh\n" ++ + "# automatically configured by git-annex\n" ++ + "git annex pre-commit .\n" diff --git a/Command/Uninit.hs b/Command/Uninit.hs new file mode 100644 index 0000000000..fcb77a92b0 --- /dev/null +++ b/Command/Uninit.hs @@ -0,0 +1,59 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Uninit where + +import Control.Monad.State (liftIO) +import Control.Monad (when) +import System.Directory + +import Command +import Messages +import Types +import Utility +import qualified GitRepo as Git +import qualified Annex +import qualified Command.Unannex +import qualified Command.Init + +seek :: [SubCmdSeek] +seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] + +start :: SubCmdStartNothing +start = do + showStart "uninit" "" + return $ Just $ perform + +perform :: SubCmdPerform +perform = do + g <- Annex.gitRepo + + gitPreCommitHookUnWrite g + liftIO $ gitAttributesUnWrite g + + return $ Just $ return True + +gitPreCommitHookUnWrite :: Git.Repo -> Annex () +gitPreCommitHookUnWrite repo = do + let hook = Command.Init.preCommitHook repo + hookexists <- liftIO $ doesFileExist hook + when hookexists $ do + c <- liftIO $ readFile hook + if c == Command.Init.preCommitScript + then liftIO $ removeFile hook + else warning $ "pre-commit hook (" ++ hook ++ + ") contents modified; not deleting." ++ + " Edit it to remove call to git annex." + +gitAttributesUnWrite :: Git.Repo -> IO () +gitAttributesUnWrite repo = do + let attributes = Git.attributes repo + attrexists <- doesFileExist attributes + when attrexists $ do + c <- readFileStrict attributes + writeFile attributes $ unlines $ + filter (/= Command.Init.attrLine) $ lines c diff --git a/Utility.hs b/Utility.hs index 0f7ce42aa6..2447c95a0b 100644 --- a/Utility.hs +++ b/Utility.hs @@ -7,6 +7,7 @@ module Utility ( hGetContentsStrict, + readFileStrict, parentDir, absPath, relPathCwdToDir, @@ -34,6 +35,10 @@ import Foreign (complement) hGetContentsStrict :: Handle -> IO String hGetContentsStrict h = hGetContents h >>= \s -> length s `seq` return s +{- A version of readFile that is not lazy. -} +readFileStrict :: FilePath -> IO String +readFileStrict f = readFile f >>= \s -> length s `seq` return s + {- Returns the parent directory of a path. Parent of / is "" -} parentDir :: String -> String parentDir dir = diff --git a/debian/changelog b/debian/changelog index 9e378400c6..b5ef4b5a03 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (0.11) UNRELEASED; urgency=low * Added remote.annex-rsync-options. * Avoid deleting temp files when rsync fails. * Improve detection of version 0 repos. + * Add uninit subcommand. Closes: #605749 -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 63bc11eb72..66e8bbaa8b 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -151,6 +151,12 @@ Many git-annex subcommands will stage changes for later `git commit` by you. any more. In that case you should use `git annex drop` instead, and you can also `git rm` the file. +* uninit + + Use this to stop using git annex. It will unannex every file in the + repository, and remove all of git-annex's other data, leaving you with a + git repository plus the previously annexed files. + * fix [path ...] Fixes up symlinks that have become broken to again point to annexed content. From b34da73ec2eb97001b4d82e214b5a517a8d8d8c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 4 Dec 2010 17:29:28 -0400 Subject: [PATCH 0580/8313] releasing version 0.11 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b5ef4b5a03..5ecf942013 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.11) UNRELEASED; urgency=low +git-annex (0.11) unstable; urgency=low * If available, rsync will be used for file transfers from remote repositories. This allows resuming interrupted transfers. @@ -7,7 +7,7 @@ git-annex (0.11) UNRELEASED; urgency=low * Improve detection of version 0 repos. * Add uninit subcommand. Closes: #605749 - -- Joey Hess Thu, 02 Dec 2010 16:54:12 -0400 + -- Joey Hess Sat, 04 Dec 2010 17:27:42 -0400 git-annex (0.10) unstable; urgency=low From 627a3014376f83d613c448da929231bb9d866435 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 4 Dec 2010 17:29:36 -0400 Subject: [PATCH 0581/8313] add news item for git-annex 0.11 --- doc/news/version_0.06.mdwn | 8 -------- doc/news/version_0.11.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 doc/news/version_0.06.mdwn create mode 100644 doc/news/version_0.11.mdwn diff --git a/doc/news/version_0.06.mdwn b/doc/news/version_0.06.mdwn deleted file mode 100644 index 39689232a7..0000000000 --- a/doc/news/version_0.06.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.06 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * fsck: Check if annex.numcopies is satisfied. - * fsck: Verify the sha1 of files when the SHA1 backend is used. - * fsck: Verify the size of files when the WORM backend is used. - * fsck: Allow specifying individual files if fscking everything - is not desired. - * fsck: Fix bug, introduced in 0.04, in detection of unused data."""]] \ No newline at end of file diff --git a/doc/news/version_0.11.mdwn b/doc/news/version_0.11.mdwn new file mode 100644 index 0000000000..03c4d7d54d --- /dev/null +++ b/doc/news/version_0.11.mdwn @@ -0,0 +1,8 @@ +git-annex 0.11 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * If available, rsync will be used for file transfers from remote + repositories. This allows resuming interrupted transfers. + * Added remote.annex-rsync-options. + * Avoid deleting temp files when rsync fails. + * Improve detection of version 0 repos. + * Add uninit subcommand. Closes: #[605749](http://bugs.debian.org/605749)"""]] \ No newline at end of file From 2099407d8aa1b1e94f29de0d9094ccfa6e05e471 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 8 Dec 2010 14:07:49 -0400 Subject: [PATCH 0582/8313] Add --exclude option to exclude files from processing. Required some lifting so flags are evaled in the Annex monad before file filtering. --- CmdLine.hs | 22 ++++++++++------------ Command.hs | 47 +++++++++++++++++++++++++++++++++------------- debian/changelog | 6 ++++++ doc/git-annex.mdwn | 5 +++++ git-annex.hs | 4 ++-- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 0903cc1fb7..cb164a6ab2 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -9,6 +9,7 @@ module CmdLine (parseCmd) where import System.Console.GetOpt import Control.Monad (when) +import Control.Monad.State (liftIO) import qualified Annex import Types @@ -103,6 +104,8 @@ options = [ "specify to where to transfer content" , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storestring "exclude") "GLOB") + "skip files matching the glob pattern" ] where storebool n b = Annex.flagChange n $ FlagBool b @@ -125,22 +128,17 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs indent l = " " ++ l pad n s = replicate (n - length s) ' ' -{- Parses command line and returns two lists of actions to be - - run in the Annex monad. The first actions configure it - - according to command line options, while the second actions - - handle subcommands. -} -parseCmd :: [String] -> AnnexState -> IO ([Annex Bool], [Annex Bool]) -parseCmd argv state = do - (flags, params) <- getopt +{- Parses command line, stores configure flags, and returns a + - list of actions to be run in the Annex monad. -} +parseCmd :: [String] -> Annex [Annex Bool] +parseCmd argv = do + (flags, params) <- liftIO $ getopt when (null params) $ error usage case lookupCmd (head params) of [] -> error usage [subcommand] -> do - actions <- prepSubCmd subcommand state (drop 1 params) - let configactions = map (\flag -> do - flag - return True) flags - return (configactions, actions) + _ <- sequence flags + prepSubCmd subcommand (drop 1 params) _ -> error "internal error: multiple matching subcommands" where getopt = case getOpt Permute options argv of diff --git a/Command.hs b/Command.hs index 059b6e435e..8edea7622e 100644 --- a/Command.hs +++ b/Command.hs @@ -11,6 +11,8 @@ import Control.Monad.State (liftIO) import System.Directory import System.Posix.Files import Control.Monad (filterM) +import System.Path.WildMatch +import Text.Regex import Types import qualified Backend @@ -59,9 +61,9 @@ data SubCommand = SubCommand { {- Prepares a list of actions to run to perform a subcommand, based on - the parameters passed to it. -} -prepSubCmd :: SubCommand -> AnnexState -> [String] -> IO [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } state params = do - lists <- Annex.eval state $ mapM (\s -> s params) seek +prepSubCmd :: SubCommand -> [String] -> Annex [Annex Bool] +prepSubCmd SubCommand { subcmdseek = seek } params = do + lists <- mapM (\s -> s params) seek return $ map doSubCmd $ foldl (++) [] lists {- Runs a subcommand through the start, perform and cleanup stages -} @@ -106,18 +108,20 @@ withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params - return $ map a $ filter notState $ foldl (++) [] files + files' <- filterFiles $ foldl (++) [] files + return $ map a files' withAttrFilesInGit :: String -> SubCmdSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo files <- liftIO $ mapM (Git.inRepo repo) params - pairs <- liftIO $ Git.checkAttr repo attr $ - filter notState $ foldl (++) [] files + files' <- filterFiles $ foldl (++) [] files + pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs withFilesMissing :: SubCmdSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params - return $ map a $ filter notState files + files' <- filterFiles files + return $ map a files' where missing f = do e <- doesFileExist f @@ -126,7 +130,8 @@ withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ mapM (Git.notInRepo repo) params - backendPairs a $ filter notState $ foldl (++) [] newfiles + newfiles' <- filterFiles $ foldl (++) [] newfiles + backendPairs a newfiles' withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] withStrings :: SubCmdSeekStrings @@ -135,7 +140,8 @@ withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ mapM (Git.stagedFiles repo) params - return $ map a $ filter notState $ foldl (++) [] tocommit + tocommit' <- filterFiles $ foldl (++) [] tocommit + return $ map a tocommit' withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles @@ -146,7 +152,8 @@ withFilesUnlocked' typechanged a params = do repo <- Annex.gitRepo typechangedfiles <- liftIO $ mapM (typechanged repo) params unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles - backendPairs a $ filter notState unlockedfiles + unlockedfiles' <- filterFiles unlockedfiles + backendPairs a unlockedfiles' withKeys :: SubCmdSeekStrings withKeys a params = return $ map a params withTempFile :: SubCmdSeekStrings @@ -173,9 +180,23 @@ withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek) withDefault d w a [] = w a [d] withDefault _ w a p = w a p -{- filter out files from the state directory -} -notState :: FilePath -> Bool -notState f = stateLoc /= take (length stateLoc) f +{- Filter out files from the state directory, and those matching the + - exclude glob pattern, if it was specified. -} +filterFiles :: [FilePath] -> Annex [FilePath] +filterFiles l = do + let l' = filter notState l + exclude <- Annex.flagGet "exclude" + if null exclude + then return l' + else do + let regexp = mkRegex $ "^" ++ wildToRegex exclude + return $ filter (notExcluded regexp) l' + where + notState f = stateLoc /= take stateLocLen f + stateLocLen = length stateLoc + notExcluded r f = case matchRegex r f of + Nothing -> True + Just _ -> False {- filter out symlinks -} notSymlink :: FilePath -> IO Bool diff --git a/debian/changelog b/debian/changelog index 5ecf942013..4b8fb1050b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.12) UNRELEASED; urgency=low + + * Add --exclude option to exclude files from processing. + + -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 + git-annex (0.11) unstable; urgency=low * If available, rsync will be used for file transfers from remote diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 66e8bbaa8b..f6dc2fe5b3 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -234,6 +234,11 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Specifies a git repository that content will be sent to. It should be specified using the name of a configured git remote. +* --exclude=glob + + Skips files matching the glob pattern. The glob is matched relative to + the current directory. + * --backend=name Specifies which key-value backend to use. diff --git a/git-annex.hs b/git-annex.hs index 417d335e16..1173ab9139 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -19,5 +19,5 @@ main = do args <- getArgs gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends - (configure, actions) <- parseCmd args state - tryRun state $ [startup, upgrade] ++ configure ++ actions + (actions, state') <- Annex.run state $ parseCmd args + tryRun state' $ [startup, upgrade] ++ actions From f8851aad66974691f1fef9d118eceb7ec909c8f4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 8 Dec 2010 14:48:10 -0400 Subject: [PATCH 0583/8313] use Text.Regex.PCRE.Light.Char8 rather than Text.Regexp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Text.Regexp does not think that á matches . -- seems to be a unicode problem. --- Command.hs | 6 +++--- debian/control | 2 +- doc/install.mdwn | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Command.hs b/Command.hs index 8edea7622e..4ba6ea4b36 100644 --- a/Command.hs +++ b/Command.hs @@ -12,7 +12,7 @@ import System.Directory import System.Posix.Files import Control.Monad (filterM) import System.Path.WildMatch -import Text.Regex +import Text.Regex.PCRE.Light.Char8 import Types import qualified Backend @@ -189,12 +189,12 @@ filterFiles l = do if null exclude then return l' else do - let regexp = mkRegex $ "^" ++ wildToRegex exclude + let regexp = compile ("^" ++ wildToRegex exclude) [] return $ filter (notExcluded regexp) l' where notState f = stateLoc /= take stateLocLen f stateLocLen = length stateLoc - notExcluded r f = case matchRegex r f of + notExcluded r f = case match r f [] of Nothing -> True Just _ -> False diff --git a/debian/control b/debian/control index 88dcab9a3a..d90041f77e 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/doc/install.mdwn b/doc/install.mdwn index 7da45c85b4..1cff4462e0 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -3,6 +3,7 @@ To build and use git-annex, you will need: * `git`: * The Haskell Platform: * MissingH: +* pcre-light: * `uuid`: * `xargs`: * `rsync`: (optional but recommended) From 939475a6ba50c401398cb624a426a2124fe773b2 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmfO7UdOn_TB4WD13ZBKMTuHW44MBJdgiI" Date: Thu, 9 Dec 2010 00:10:03 +0000 Subject: [PATCH 0584/8313] --- doc/bugs/conflicting_haskell_packages.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/conflicting_haskell_packages.mdwn diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn new file mode 100644 index 0000000000..58e25539d3 --- /dev/null +++ b/doc/bugs/conflicting_haskell_packages.mdwn @@ -0,0 +1,3 @@ +The compilation command should states which packages are used and avoid the default mechnasim that automatically search for them. + +This can be done by the flags -hide-packages and then -package foo From 7b2ed635d3215452b903de20a93bc9f1641d6554 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:32:02 -0400 Subject: [PATCH 0585/8313] typo correction via LWN --- doc/use_case/Alice.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index c42eb3a74c..1dc456d731 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -13,6 +13,6 @@ podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). When she's done, she tells git-annex which to keep and which to remove. -They're all removed from her netbook to save space, and Alice knowns +They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. From 668ccafc93005d02b7881bc88bc9e556bec64a89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:35:07 -0400 Subject: [PATCH 0586/8313] add pointer to lwn article --- doc/news/LWN_article.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/news/LWN_article.mdwn diff --git a/doc/news/LWN_article.mdwn b/doc/news/LWN_article.mdwn new file mode 100644 index 0000000000..c1c0c40472 --- /dev/null +++ b/doc/news/LWN_article.mdwn @@ -0,0 +1,2 @@ +[Linux Weekly News](http://lwn.net/) has a nice +[article on git-annex](http://lwn.net/Articles/418337/) in it this week. From 7c46439de0ff6f099fad29ad09217a402f88607e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:36:10 -0400 Subject: [PATCH 0587/8313] add news to sidebar --- doc/index.mdwn | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 5cd6b45494..cbce1260e4 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -18,12 +18,15 @@ To get a feel for it, see the [[walkthrough]]. * **[[download]]** * [[install]] -* [[news]] * [[bugs]] * [[todo]] * [[forum]] * [[contact]] +[[News]]: + +[[!inlines pages="news/* and !*/discussion" archive=yes show=3]] + Flattr this From 136877264a192d315a28b1778962b0d8c7a65be6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 11:36:38 -0400 Subject: [PATCH 0588/8313] typo --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index cbce1260e4..595db91e01 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -25,7 +25,7 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: -[[!inlines pages="news/* and !*/discussion" archive=yes show=3]] +[[!inline pages="news/* and !*/discussion" archive=yes show=3]] Date: Thu, 9 Dec 2010 11:37:21 -0400 Subject: [PATCH 0589/8313] no feeds --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 595db91e01..a279e3e5fb 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -25,7 +25,7 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: -[[!inline pages="news/* and !*/discussion" archive=yes show=3]] +[[!inline pages="news/* and !*/discussion" archive=yes show=3 feeds=no]] Date: Thu, 9 Dec 2010 11:44:00 -0400 Subject: [PATCH 0590/8313] update --- doc/download.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index ccf21c091b..6c56c8e25a 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,6 +1,7 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -Users of Debian unstable can `apt-get install git-annex` +Users of Debian unstable/testing and Ubuntu natty can +`apt-get install git-annex` Next: [[install]] From 1f22c9ea38b85b2e3e4d25bb56887e55e1c51f40 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 9 Dec 2010 13:45:16 -0400 Subject: [PATCH 0591/8313] response --- doc/bugs/conflicting_haskell_packages.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn index 58e25539d3..6a619de90f 100644 --- a/doc/bugs/conflicting_haskell_packages.mdwn +++ b/doc/bugs/conflicting_haskell_packages.mdwn @@ -1,3 +1,8 @@ The compilation command should states which packages are used and avoid the default mechnasim that automatically search for them. This can be done by the flags -hide-packages and then -package foo + +> My ghc does not have a `--hide-packages` option. +> +> Could you just show the build problem that you are suggesting I work +> around? --[[Joey]] From cdf040e81656b5e8034e4fd2d5e0583c367dff12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 10 Dec 2010 17:27:02 -0400 Subject: [PATCH 0592/8313] mwdn2man: Fix a bug in newline supression. Closes: #606578 --- debian/changelog | 1 + mdwn2man | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 4b8fb1050b..323cd2e75e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.12) UNRELEASED; urgency=low * Add --exclude option to exclude files from processing. + * mwdn2man: Fix a bug in newline supression. Closes: #606578 -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 diff --git a/mdwn2man b/mdwn2man index c212539454..ad6d3c6026 100755 --- a/mdwn2man +++ b/mdwn2man @@ -14,7 +14,7 @@ while (<>) { s/^#\s/.SH /; <>; # blank; } - s/^ +//; + s/^[ \n]+//; s/^\t/ /; s/-/\\-/g; s/^Warning:.*//g; From eef009fa2a94fe8d2934d314898839f7a864c7ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 10 Dec 2010 17:30:13 -0400 Subject: [PATCH 0593/8313] formatting --- doc/git-annex.mdwn | 63 +++++++++++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index f6dc2fe5b3..71a4889ac7 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -252,34 +252,63 @@ Many git-annex subcommands will stage changes for later `git commit` by you. Like other git commands, git-annex is configured via `.git/config`. Here are all the supported configuration settings. -* `annex.uuid` -- a unique UUID for this repository (automatically set) -* `annex.numcopies` -- number of copies of files to keep across all - repositories (default: 1) -* `annex.backends` -- space-separated list of names of - the key-value backends to use. The first listed is used to store - new files by default. (default: "WORM SHA1 URL") -* `remote..annex-cost` -- When determining which repository to +* `annex.uuid` + + A unique UUID for this repository (automatically set). + +* `annex.numcopies` + + Number of copies of files to keep across all repositories. (default: 1) + +* `annex.backends` + + Space-separated list of names of the key-value backends to use. + The first listed is used to store new files by default. + (default: "WORM SHA1 URL") + +* `remote..annex-cost` + + When determining which repository to transfer annexed files from or to, ones with lower costs are preferred. The default cost is 100 for local repositories, and 200 for remote repositories. -* `remote..annex-ignore` -- If set to `true`, prevents git-annex + +* `remote..annex-ignore` + + If set to `true`, prevents git-annex from ever using this remote. This is, for example, useful if the remote is a bare repository, which git-annex does not currently support. -* `remote..annex-uuid` -- git-annex caches UUIDs of repositories - here. -* `remote..annex-scp-options` -- Options to use when using scp + +* `remote..annex-uuid` + + git-annex caches UUIDs of repositories here. + +* `remote..annex-scp-options` + + Options to use when using scp to or from this repository. For example, to force ipv6, and limit the bandwidth to 1000Kbit/s, set it to "-6 -l 1000" -* `remote..annex-ssh-options` -- Options to use when using ssh - to talk to this repository. -* `remote..annex-rsync-options` -- Options to use when using rsync + +* `remote..annex-ssh-options` + + Options to use when using ssh to talk to this repository. + +* `remote..annex-rsync-options` + + Options to use when using rsync to or from this repository. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` -- + +* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` + Default scp, ssh, and rsync options to use if a remote does not have specific options. -* `annex.version` -- Automatically maintained, and used to automate upgrades - between versions. + +* `annex.version` + + Automatically maintained, and used to automate upgrades between versions. + +# CONFIGURATION VIA .gitattributes The backend used when adding a new file to the annex can be configured on a per-file-type basis via `.gitattributes` files. In the file, From 77e52a52dd2f77e5acc0643673169b4e1d2550f9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:14:54 -0400 Subject: [PATCH 0594/8313] Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 --- Command.hs | 4 +++- debian/changelog | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Command.hs b/Command.hs index 4ba6ea4b36..69b9dee9f9 100644 --- a/Command.hs +++ b/Command.hs @@ -151,7 +151,9 @@ withFilesUnlocked' typechanged a params = do -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo typechangedfiles <- liftIO $ mapM (typechanged repo) params - unlockedfiles <- liftIO $ filterM notSymlink $ foldl (++) [] typechangedfiles + unlockedfiles <- liftIO $ filterM notSymlink $ + map (\f -> Git.workTree repo ++ "/" ++ f) $ + foldl (++) [] typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' withKeys :: SubCmdSeekStrings diff --git a/debian/changelog b/debian/changelog index 323cd2e75e..6bc3c788a6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.12) UNRELEASED; urgency=low * Add --exclude option to exclude files from processing. * mwdn2man: Fix a bug in newline supression. Closes: #606578 + * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 From 10484a69b8a7617fea5ce7ba95e31fe1a9eff7b4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:29:50 -0400 Subject: [PATCH 0595/8313] Makefile: Add PREFIX variable. --- Makefile | 15 ++++++++------- debian/changelog | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ad143e7b69..b5f5a720be 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,18 @@ +PREFIX=/usr +GHCMAKE=ghc -Wall -odir build -hidir build -O2 --make + all: git-annex docs -ghcmake=ghc -Wall -odir build -hidir build -O2 --make - SysConfig.hs: configure.hs - $(ghcmake) configure + $(GHCMAKE) configure ./configure git-annex: SysConfig.hs - $(ghcmake) git-annex + $(GHCMAKE) git-annex install: - install -d $(DESTDIR)/usr/bin - install git-annex $(DESTDIR)/usr/bin + install -d $(DESTDIR)$(PREFIX)/bin + install git-annex $(DESTDIR)$(PREFIX)/bin # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. @@ -22,7 +23,7 @@ IKIWIKI=ikiwiki endif test: - $(ghcmake) test + $(GHCMAKE) test ./test docs: diff --git a/debian/changelog b/debian/changelog index 6bc3c788a6..fe15dfdf86 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.12) UNRELEASED; urgency=low * Add --exclude option to exclude files from processing. * mwdn2man: Fix a bug in newline supression. Closes: #606578 * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 + * Makefile: Add PREFIX variable. -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 From 3a252efd9d679a85cca959537af03cbe474167b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:37:24 -0400 Subject: [PATCH 0596/8313] releasing version 0.12 --- Makefile | 8 ++++---- debian/changelog | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b5f5a720be..db34294f7b 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,10 @@ install: install -d $(DESTDIR)$(PREFIX)/bin install git-annex $(DESTDIR)$(PREFIX)/bin +test: + $(GHCMAKE) test + ./test + # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) @@ -22,10 +26,6 @@ else IKIWIKI=ikiwiki endif -test: - $(GHCMAKE) test - ./test - docs: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ diff --git a/debian/changelog b/debian/changelog index fe15dfdf86..c66a536d5b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,11 @@ -git-annex (0.12) UNRELEASED; urgency=low +git-annex (0.12) unstable; urgency=low * Add --exclude option to exclude files from processing. * mwdn2man: Fix a bug in newline supression. Closes: #606578 * Bugfix to git annex add of an unlocked file in a subdir. Closes: #606579 * Makefile: Add PREFIX variable. - -- Joey Hess Wed, 08 Dec 2010 14:06:47 -0400 + -- Joey Hess Sat, 11 Dec 2010 17:32:00 -0400 git-annex (0.11) unstable; urgency=low From f8b59360e386c725dc5e6961beaa78621585d1e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 11 Dec 2010 17:37:40 -0400 Subject: [PATCH 0597/8313] add news item for git-annex 0.12 --- doc/news/version_0.07.mdwn | 8 -------- doc/news/version_0.12.mdwn | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 doc/news/version_0.07.mdwn create mode 100644 doc/news/version_0.12.mdwn diff --git a/doc/news/version_0.07.mdwn b/doc/news/version_0.07.mdwn deleted file mode 100644 index 29b4736b92..0000000000 --- a/doc/news/version_0.07.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.07 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * find: New subcommand. - * unused: New subcommand, finds unused data. (Split out from fsck.) - * dropunused: New subcommand, provides for easy dropping of unused keys - by number, as listed by the unused subcommand. - * fsck: Print warnings to stderr; --quiet can now be used to only see - problems."""]] \ No newline at end of file diff --git a/doc/news/version_0.12.mdwn b/doc/news/version_0.12.mdwn new file mode 100644 index 0000000000..e7fccd1b36 --- /dev/null +++ b/doc/news/version_0.12.mdwn @@ -0,0 +1,6 @@ +git-annex 0.12 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Add --exclude option to exclude files from processing. + * mwdn2man: Fix a bug in newline supression. Closes: #[606578](http://bugs.debian.org/606578) + * Bugfix to git annex add of an unlocked file in a subdir. Closes: #[606579](http://bugs.debian.org/606579) + * Makefile: Add PREFIX variable."""]] \ No newline at end of file From 98ad5402d9b095b07df82f088cf0a19c8298ca0c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 12 Dec 2010 13:15:16 -0400 Subject: [PATCH 0598/8313] Makefile: Install man page and html (when built). --- Makefile | 16 ++++++++++++---- debian/changelog | 6 ++++++ debian/docs | 1 - 3 files changed, 18 insertions(+), 5 deletions(-) delete mode 100644 debian/docs diff --git a/Makefile b/Makefile index db34294f7b..5a39356432 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,27 @@ PREFIX=/usr GHCMAKE=ghc -Wall -odir build -hidir build -O2 --make -all: git-annex docs +all: git-annex git-annex.1 docs SysConfig.hs: configure.hs $(GHCMAKE) configure ./configure +git-annex.1: + ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 + git-annex: SysConfig.hs $(GHCMAKE) git-annex -install: +install: all install -d $(DESTDIR)$(PREFIX)/bin install git-annex $(DESTDIR)$(PREFIX)/bin + install -d $(DESTDIR)$(PREFIX)/share/man/man1 + install -m 0644 git-annex.1 $(DESTDIR)$(PREFIX)/share/man/man1 + install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex + if [ -d html ]; then \ + rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ + fi test: $(GHCMAKE) test @@ -26,8 +35,7 @@ else IKIWIKI=ikiwiki endif -docs: - ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 +docs: git-annex.1 $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ --underlaydir=/dev/null --disable-plugin=shortcut \ diff --git a/debian/changelog b/debian/changelog index c66a536d5b..7e08b07310 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.13) UNRELEASED; urgency=low + + * Makefile: Install man page and html (when built). + + -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 + git-annex (0.12) unstable; urgency=low * Add --exclude option to exclude files from processing. diff --git a/debian/docs b/debian/docs deleted file mode 100644 index 1936cc1d44..0000000000 --- a/debian/docs +++ /dev/null @@ -1 +0,0 @@ -html From b805b9ae66d279c4fee20c433e6c218f29103978 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 12 Dec 2010 13:16:34 -0400 Subject: [PATCH 0599/8313] Makefile: Add GHCOPTS variable. --- Makefile | 3 ++- debian/changelog | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5a39356432..d5f1f652f1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ PREFIX=/usr -GHCMAKE=ghc -Wall -odir build -hidir build -O2 --make +GHCOPTS=-O2 -Wall +GHCMAKE=ghc -odir build -hidir build $(GHCOPTS) --make all: git-annex git-annex.1 docs diff --git a/debian/changelog b/debian/changelog index 7e08b07310..5d088f1bdd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). + * Makefile: Add GHCOPTS variable. -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 67c5036579a4c4af99711cba45ffffb032262d60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 12 Dec 2010 13:17:53 -0400 Subject: [PATCH 0600/8313] Makefile: Add GHCFLAGS variable. --- Makefile | 4 ++-- debian/changelog | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d5f1f652f1..c338427df4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr -GHCOPTS=-O2 -Wall -GHCMAKE=ghc -odir build -hidir build $(GHCOPTS) --make +GHCFLAGS=-O2 -Wall +GHCMAKE=ghc -odir build -hidir build $(GHCFLAGS) --make all: git-annex git-annex.1 docs diff --git a/debian/changelog b/debian/changelog index 5d088f1bdd..19ff29652d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). - * Makefile: Add GHCOPTS variable. + * Makefile: Add GHCFLAGS variable. -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 0210628263f5a9da4f7ceb6b359108174ac01182 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 13 Dec 2010 11:35:00 -0400 Subject: [PATCH 0601/8313] Fix upgrade from 0.03. --- Upgrade.hs | 18 +++++++++++++++++- debian/changelog | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Upgrade.hs b/Upgrade.hs index d64d5287d1..52ecfa07d7 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -7,9 +7,12 @@ module Upgrade where +import System.IO.Error (try) import System.Directory import Control.Monad.State (liftIO) +import Control.Monad (filterM) import System.Posix.Files +import System.FilePath import Core import Types @@ -37,7 +40,7 @@ upgradeFrom0 = do -- do the reorganisation of the files let olddir = annexDir g - keys <- getKeysPresent' olddir + keys <- getKeysPresent0' olddir _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys -- update the symlinks to the files @@ -61,3 +64,16 @@ upgradeFrom0 = do liftIO $ createSymbolicLink link f Annex.queue "add" ["--"] f fixlinks fs + +getKeysPresent0' :: FilePath -> Annex [Key] +getKeysPresent0' dir = do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey files + where + present d = do + result <- try $ + getFileStatus $ dir ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False diff --git a/debian/changelog b/debian/changelog index 19ff29652d..7ac893c13c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. + * Fix upgrade from 0.03. -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 5ec3cea05961e4d9fc3484746860415126db5365 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 11:37:11 -0400 Subject: [PATCH 0602/8313] Support remotes using git+ssh:// as protocol. Closes: #607056 --- GitRepo.hs | 1 + debian/changelog | 1 + 2 files changed, 2 insertions(+) diff --git a/GitRepo.hs b/GitRepo.hs index 539acecb7e..3cbeae192d 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -122,6 +122,7 @@ repoIsUrl _ = False repoIsSsh :: Repo -> Bool repoIsSsh Repo { location = Url url } | uriScheme url == "ssh:" = True + | uriScheme url == "git+ssh:" = True | otherwise = False repoIsSsh _ = False diff --git a/debian/changelog b/debian/changelog index 7ac893c13c..c9f47273f0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. * Fix upgrade from 0.03. + * Support remotes using git+ssh:// as protocol. Closes: #607056 -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From 5d4052d0e05d391d2ebd167b528133ade726eb3b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 12:46:09 -0400 Subject: [PATCH 0603/8313] Support remotes using git+ssh and ssh+git as protocol. Closes: #607056 --- GitRepo.hs | 2 ++ debian/changelog | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 3cbeae192d..1ce6b8aecf 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -122,7 +122,9 @@ repoIsUrl _ = False repoIsSsh :: Repo -> Bool repoIsSsh Repo { location = Url url } | uriScheme url == "ssh:" = True + -- git treats these the same as ssh | uriScheme url == "git+ssh:" = True + | uriScheme url == "ssh+git:" = True | otherwise = False repoIsSsh _ = False diff --git a/debian/changelog b/debian/changelog index c9f47273f0..582c05d2c5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.13) UNRELEASED; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. * Fix upgrade from 0.03. - * Support remotes using git+ssh:// as protocol. Closes: #607056 + * Support remotes using git+ssh and ssh+git as protocol. + Closes: #607056 -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 From dc25c7030a1b7888440ea58b8774073c2c28eb4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 13:12:24 -0400 Subject: [PATCH 0604/8313] releasing version 0.13 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 582c05d2c5..af63601ba8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.13) UNRELEASED; urgency=low +git-annex (0.13) unstable; urgency=low * Makefile: Install man page and html (when built). * Makefile: Add GHCFLAGS variable. @@ -6,7 +6,7 @@ git-annex (0.13) UNRELEASED; urgency=low * Support remotes using git+ssh and ssh+git as protocol. Closes: #607056 - -- Joey Hess Sun, 12 Dec 2010 13:14:58 -0400 + -- Joey Hess Tue, 14 Dec 2010 13:05:10 -0400 git-annex (0.12) unstable; urgency=low From ada2a925195152cde1961eee4b3529f72b51aef8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 14 Dec 2010 13:13:20 -0400 Subject: [PATCH 0605/8313] add news item for git-annex 0.13 --- doc/news/version_0.08.mdwn | 8 -------- doc/news/version_0.13.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 doc/news/version_0.08.mdwn create mode 100644 doc/news/version_0.13.mdwn diff --git a/doc/news/version_0.08.mdwn b/doc/news/version_0.08.mdwn deleted file mode 100644 index 8ef7ad7f31..0000000000 --- a/doc/news/version_0.08.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.08 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix `git annex add ../foo` (when ran in a subdir of the repo). - * Add configure step to build process. - * Only use cp -a if it is supported, falling back to cp -p or plain cp - as needed for portability. - * cp --reflink=auto is used if supported, and will make git annex unlock - much faster on filesystems like btrfs that support copy on write."""]] \ No newline at end of file diff --git a/doc/news/version_0.13.mdwn b/doc/news/version_0.13.mdwn new file mode 100644 index 0000000000..d4de64d3e2 --- /dev/null +++ b/doc/news/version_0.13.mdwn @@ -0,0 +1,7 @@ +git-annex 0.13 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Makefile: Install man page and html (when built). + * Makefile: Add GHCFLAGS variable. + * Fix upgrade from 0.03. + * Support remotes using git+ssh and ssh+git as protocol. + Closes: #[607056](http://bugs.debian.org/607056)"""]] \ No newline at end of file From d0cba98c5b1297b2bebbf2d75eb4960e3ac07779 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Mon, 20 Dec 2010 13:20:00 +0000 Subject: [PATCH 0606/8313] misleading error message on empty repo --- ...git_annex_unused_failes_on_empty_repository.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn diff --git a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn new file mode 100644 index 0000000000..9be98104a0 --- /dev/null +++ b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn @@ -0,0 +1,13 @@ +[[!meta title="`git annex unused` fails on empty repository"]] + +The ``git annex unused`` command fails on a git-annex repository, if there are no objects yet: + + $ git annex unused + unused (checking for unused data...) + git-annex: /tmp/annextest/other_annex/.git/annex/objects: getDirectoryContents: does not exist (No such file or directory) + git-annex: 1 failed + $ + +This can give a user (especially one that wants to try out simple commands with his newly created repo) the impression that something is wrong, while it is not. I'd expect the program either to show the same message ``git annex unused`` shows when everything is ok (since it is, or should be). + +This can be a bug in the ``unused`` subcommand (that fails to accept the absence of an objects directory) or in the ``init`` subcommand (that fails to create it). From 11cf7136ed0fd88ac62cb6d8d16c5eb4c915b392 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Mon, 20 Dec 2010 15:48:58 +0000 Subject: [PATCH 0607/8313] suggestion for out-of-branch location tracking --- doc/forum/working_without_git-annex_commits.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/forum/working_without_git-annex_commits.mdwn diff --git a/doc/forum/working_without_git-annex_commits.mdwn b/doc/forum/working_without_git-annex_commits.mdwn new file mode 100644 index 0000000000..88c2804c5c --- /dev/null +++ b/doc/forum/working_without_git-annex_commits.mdwn @@ -0,0 +1,7 @@ +Is it possible to use git-annex without having [[location tracking]] commits in the style of "got a video I want to rewatch on the plane" or "freed up space" in the main tree? + +I consider these changes to be volatile, and irrelevant to the archive history. While they are unproblematic when it comes to merging, they make the commit tree rather complicated, especially with multiple users (as opposed to a single user managing his files on an external disk, a server and his laptop). Some users might even want to contribute to a shared repository without reporting on what they checked out. + +As a minimal solution, I configured a repository to ``.gitignore`` ``.git-annex/*:*.log``, but even when using modes that do not require that information (``git annex copy --from`` instead of ``git annex get``), that failes when git-annex tried to git-add ignored files. + +A more elaborate solution might be to keep location tracking information in a branch on its own (as suggested in [[todo/branching]]), keeping the main tree clean of such commits. A stealth user could then configure that branch to never be pushed. (Alternatively, if git-annex respects .gitignore and doesn't try to check in changes on ignored files, he could locally ``.gitignore`` ``.git-annex/*:*.log``.) From ea70873b112c9a1a4471bbd263001975e69e6bdf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Dec 2010 14:54:03 -0400 Subject: [PATCH 0608/8313] response --- doc/forum/working_without_git-annex_commits.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/forum/working_without_git-annex_commits.mdwn b/doc/forum/working_without_git-annex_commits.mdwn index 88c2804c5c..00a61f3f0f 100644 --- a/doc/forum/working_without_git-annex_commits.mdwn +++ b/doc/forum/working_without_git-annex_commits.mdwn @@ -5,3 +5,16 @@ I consider these changes to be volatile, and irrelevant to the archive history. As a minimal solution, I configured a repository to ``.gitignore`` ``.git-annex/*:*.log``, but even when using modes that do not require that information (``git annex copy --from`` instead of ``git annex get``), that failes when git-annex tried to git-add ignored files. A more elaborate solution might be to keep location tracking information in a branch on its own (as suggested in [[todo/branching]]), keeping the main tree clean of such commits. A stealth user could then configure that branch to never be pushed. (Alternatively, if git-annex respects .gitignore and doesn't try to check in changes on ignored files, he could locally ``.gitignore`` ``.git-annex/*:*.log``.) + +> A stealth user can simply avoid pushing, and so keep their repository +> in a forked state, that can still pull changes from origin. +> +> Beyond that, [[todo/branching]] is the best solution. +> +> I don't think that gitignoring the log files is a good plan, because +> if the files are left modified and uncommitted, git will not be able to +> merge other changes it pulls. The automerging of log files only works +> if any local changes to them have been committed. +> +> It would be possible to add a knob that +> simply blocks all local modifications to the log files. --[[Joey]] From c4a357d5d13e6c2f09ee98ee6ef8b18104e5d167 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Dec 2010 14:57:43 -0400 Subject: [PATCH 0609/8313] Bugfix to git annex unused in a repository with nothing yet annexed. --- Core.hs | 10 +++++++--- debian/changelog | 6 ++++++ .../git_annex_unused_failes_on_empty_repository.mdwn | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Core.hs b/Core.hs index e3702044e8..9da4f3f2ec 100644 --- a/Core.hs +++ b/Core.hs @@ -185,9 +185,13 @@ getKeysPresent = do getKeysPresent' $ annexObjectDir g getKeysPresent' :: FilePath -> Annex [Key] getKeysPresent' dir = do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ map fileKey files + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey files where present d = do result <- try $ diff --git a/debian/changelog b/debian/changelog index af63601ba8..70d375f30e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.14) UNRELEASED; urgency=low + + * Bugfix to git annex unused in a repository with nothing yet annexed. + + -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 + git-annex (0.13) unstable; urgency=low * Makefile: Install man page and html (when built). diff --git a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn index 9be98104a0..05aa695727 100644 --- a/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn +++ b/doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn @@ -11,3 +11,5 @@ The ``git annex unused`` command fails on a git-annex repository, if there are n This can give a user (especially one that wants to try out simple commands with his newly created repo) the impression that something is wrong, while it is not. I'd expect the program either to show the same message ``git annex unused`` shows when everything is ok (since it is, or should be). This can be a bug in the ``unused`` subcommand (that fails to accept the absence of an objects directory) or in the ``init`` subcommand (that fails to create it). + +> [[fixed|done]] --[[Joey]] From eedebb0057b4d0fc7d0d13214d8f5e3890034374 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 20 Dec 2010 15:01:04 -0400 Subject: [PATCH 0610/8313] Support upgrading from a v0 annex with nothing in it. --- Upgrade.hs | 10 +++++++--- debian/changelog | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Upgrade.hs b/Upgrade.hs index 52ecfa07d7..204e2d555f 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -67,9 +67,13 @@ upgradeFrom0 = do getKeysPresent0' :: FilePath -> Annex [Key] getKeysPresent0' dir = do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ map fileKey files + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey files where present d = do result <- try $ diff --git a/debian/changelog b/debian/changelog index 70d375f30e..ae6c3f0469 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.14) UNRELEASED; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. + * Support upgrading from a v0 annex with nothing in it. -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 From 61b7f3dea3b54ead64cd5e957da28c83e7c25c85 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 22 Dec 2010 11:52:16 +0000 Subject: [PATCH 0611/8313] but report on umlaut handling --- doc/bugs/problems_with_utf8_names.mdwn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/bugs/problems_with_utf8_names.mdwn diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn new file mode 100644 index 0000000000..521f92405e --- /dev/null +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -0,0 +1,20 @@ +There are problems with displaying filenames in UTF8 encoding, as shown here: + + $ echo $LANG + en_GB.UTF-8 + $ git init + $ git annex init test + [...] + $ touch "Umlaut Ü.txt" + $ git annex add Uml* + add Umlaut Ã.txt ok + (Recording state in git...) + $ find -name U\* | hexdump -C + 00000000 2e 2f 55 6d 6c 61 75 74 20 c3 9c 2e 74 78 74 0a |./Umlaut ...txt.| + 00000010 + $ git annex find | hexdump -C + 00000000 55 6d 6c 61 75 74 20 c3 83 c2 9c 2e 74 78 74 0a |Umlaut .....txt.| + 00000010 + $ + +It looks like the common latin1-to-UTF8 encoding. Functionality other than otuput seems not to be affected. From 346c7a025780e011cdfdbb7eb27281f808a8de92 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Dec 2010 21:53:32 -0400 Subject: [PATCH 0612/8313] Avoid multiple calls to git ls-files when passed eg, "*". --- Command.hs | 23 +++++++++++------------ Core.hs | 2 +- GitRepo.hs | 46 ++++++++++++++++++++++------------------------ Upgrade.hs | 2 +- debian/changelog | 1 + 5 files changed, 36 insertions(+), 38 deletions(-) diff --git a/Command.hs b/Command.hs index 69b9dee9f9..e30904d0f7 100644 --- a/Command.hs +++ b/Command.hs @@ -107,14 +107,14 @@ isAnnexed file a = do withFilesInGit :: SubCmdSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - files' <- filterFiles $ foldl (++) [] files + files <- liftIO $ Git.inRepo repo params + files' <- filterFiles files return $ map a files' withAttrFilesInGit :: String -> SubCmdSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo - files <- liftIO $ mapM (Git.inRepo repo) params - files' <- filterFiles $ foldl (++) [] files + files <- liftIO $ Git.inRepo repo params + files' <- filterFiles files pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs withFilesMissing :: SubCmdSeekStrings @@ -129,8 +129,8 @@ withFilesMissing a params = do withFilesNotInGit :: SubCmdSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo - newfiles <- liftIO $ mapM (Git.notInRepo repo) params - newfiles' <- filterFiles $ foldl (++) [] newfiles + newfiles <- liftIO $ Git.notInRepo repo params + newfiles' <- filterFiles newfiles backendPairs a newfiles' withString :: SubCmdSeekStrings withString a params = return [a $ unwords params] @@ -139,21 +139,20 @@ withStrings a params = return $ map a params withFilesToBeCommitted :: SubCmdSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo - tocommit <- liftIO $ mapM (Git.stagedFiles repo) params - tocommit' <- filterFiles $ foldl (++) [] tocommit + tocommit <- liftIO $ Git.stagedFiles repo params + tocommit' <- filterFiles tocommit return $ map a tocommit' withFilesUnlocked :: SubCmdSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles -withFilesUnlocked' :: (Git.Repo -> FilePath -> IO [FilePath]) -> SubCmdSeekBackendFiles +withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> SubCmdSeekBackendFiles withFilesUnlocked' typechanged a params = do -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo - typechangedfiles <- liftIO $ mapM (typechanged repo) params + typechangedfiles <- liftIO $ typechanged repo params unlockedfiles <- liftIO $ filterM notSymlink $ - map (\f -> Git.workTree repo ++ "/" ++ f) $ - foldl (++) [] typechangedfiles + map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' withKeys :: SubCmdSeekStrings diff --git a/Core.hs b/Core.hs index 9da4f3f2ec..d91595a049 100644 --- a/Core.hs +++ b/Core.hs @@ -204,6 +204,6 @@ getKeysPresent' dir = do getKeysReferenced :: Annex [Key] getKeysReferenced = do g <- Annex.gitRepo - files <- liftIO $ Git.inRepo g $ Git.workTree g + files <- liftIO $ Git.inRepo g [Git.workTree g] keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs diff --git a/GitRepo.hs b/GitRepo.hs index 1ce6b8aecf..122b5cc60f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -230,41 +230,39 @@ hPipeRead :: Repo -> [String] -> IO (PipeHandle, String) hPipeRead repo params = assertLocal repo $ do pipeFrom "git" (gitCommandLine repo params) -{- Passed a location, recursively scans for all files that - - are checked into git at that location. -} -inRepo :: Repo -> FilePath -> IO [FilePath] -inRepo repo l = pipeNullSplit repo - ["ls-files", "--cached", "--exclude-standard", "-z", "--", l] +{- Scans for files that are checked into git at the specified locations. -} +inRepo :: Repo -> [FilePath] -> IO [FilePath] +inRepo repo l = pipeNullSplit repo $ + ["ls-files", "--cached", "--exclude-standard", "-z", "--"] ++ l -{- Passed a location, recursively scans for all files that are not checked - - into git, and not gitignored. -} -notInRepo :: Repo -> FilePath -> IO [FilePath] -notInRepo repo l = pipeNullSplit repo - ["ls-files", "--others", "--exclude-standard", "-z", "--", l] +{- Scans for files at the specified locations that are not checked into git, + - and not gitignored. -} +notInRepo :: Repo -> [FilePath] -> IO [FilePath] +notInRepo repo l = pipeNullSplit repo $ + ["ls-files", "--others", "--exclude-standard", "-z", "--"] ++ l -{- Passed a location, returns a list of the files, staged for - - commit, that are being added, moved, or changed (but not deleted). -} -stagedFiles :: Repo -> FilePath -> IO [FilePath] -stagedFiles repo l = pipeNullSplit repo +{- Returns a list of the files, staged for commit, that are being added, + - moved, or changed (but not deleted), from the specified locations. -} +stagedFiles :: Repo -> [FilePath] -> IO [FilePath] +stagedFiles repo l = pipeNullSplit repo $ ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", - "--", l] + "--"] ++ l -{- Passed a location, returns a list of the files, staged for - - commit, whose type has changed. -} -typeChangedStagedFiles :: Repo -> FilePath -> IO [FilePath] +{- Returns a list of the files in the specified locations that are staged + - for commit, and whose type has changed. -} +typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedStagedFiles repo l = typeChangedFiles' repo l ["--cached"] -{- Passed a location, returns a list of the files whose type has changed. - - Files only staged for commit will not be included. -} -typeChangedFiles :: Repo -> FilePath -> IO [FilePath] +{- Returns a list of the files in the specified locations whose type has + - changed. Files only staged for commit will not be included. -} +typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedFiles repo l = typeChangedFiles' repo l [] -typeChangedFiles' :: Repo -> FilePath -> [String] -> IO [FilePath] +typeChangedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath] typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = ["diff", "--name-only", "--diff-filter=T", "-z"] - end = ["--", l] - + end = ["--"] ++ l {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} diff --git a/Upgrade.hs b/Upgrade.hs index 204e2d555f..2e1708439b 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -44,7 +44,7 @@ upgradeFrom0 = do _ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys -- update the symlinks to the files - files <- liftIO $ Git.inRepo g $ Git.workTree g + files <- liftIO $ Git.inRepo g [Git.workTree g] fixlinks files Annex.queueRun diff --git a/debian/changelog b/debian/changelog index ae6c3f0469..bddd256ae5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.14) UNRELEASED; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. * Support upgrading from a v0 annex with nothing in it. + * Avoid multiple calls to git ls-files when passed eg, "*". -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 From 80aaa68e91a8c5d4fb5b426ac24d7de384cf9b8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 17:41:15 -0400 Subject: [PATCH 0613/8313] releasing version 0.14 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index bddd256ae5..f8a3c3129d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,10 @@ -git-annex (0.14) UNRELEASED; urgency=low +git-annex (0.14) unstable; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. * Support upgrading from a v0 annex with nothing in it. * Avoid multiple calls to git ls-files when passed eg, "*". - -- Joey Hess Mon, 20 Dec 2010 14:54:49 -0400 + -- Joey Hess Fri, 24 Dec 2010 17:38:48 -0400 git-annex (0.13) unstable; urgency=low From 36662638df5b3b167f20d1b9e772a2059b96da3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 17:41:39 -0400 Subject: [PATCH 0614/8313] add news item for git-annex 0.14 --- doc/news/version_0.09.mdwn | 4 ---- doc/news/version_0.14.mdwn | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) delete mode 100644 doc/news/version_0.09.mdwn create mode 100644 doc/news/version_0.14.mdwn diff --git a/doc/news/version_0.09.mdwn b/doc/news/version_0.09.mdwn deleted file mode 100644 index fff8249f77..0000000000 --- a/doc/news/version_0.09.mdwn +++ /dev/null @@ -1,4 +0,0 @@ -git-annex 0.09 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add copy subcommand. - * Fix bug in setkey subcommand triggered by move --to."""]] \ No newline at end of file diff --git a/doc/news/version_0.14.mdwn b/doc/news/version_0.14.mdwn new file mode 100644 index 0000000000..fc6c42bc19 --- /dev/null +++ b/doc/news/version_0.14.mdwn @@ -0,0 +1,5 @@ +git-annex 0.14 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Bugfix to git annex unused in a repository with nothing yet annexed. + * Support upgrading from a v0 annex with nothing in it. + * Avoid multiple calls to git ls-files when passed eg, "*"."""]] \ No newline at end of file From 904af72b82ee513748672620dd51bae050494a6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 19:25:59 -0400 Subject: [PATCH 0615/8313] better message --- Backend/File.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index afbe38ac0f..f88bb7c704 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -175,4 +175,4 @@ checkKeyNumCopies key numcopies = do note present needed = "Only " ++ show present ++ " of " ++ show needed ++ " copies of "++show key++" exist. " ++ - "Run git annex get somewhere else to back it up." + "Back it up with git-annex copy." From dd55f21450d5ed9f71b015295c33c0c9bf0060f9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 24 Dec 2010 19:28:02 -0400 Subject: [PATCH 0616/8313] add a newline --- Command/Unused.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Command/Unused.hs b/Command/Unused.hs index de34ceae96..69a16f2547 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -50,7 +50,8 @@ checkUnused = do " NUMBER KEY"] ++ map cols u ++ ["(To see where data was previously used, try: git log --stat -S'KEY')", - "(To remove unwanted data: git-annex dropunused NUMBER)"] + "(To remove unwanted data: git-annex dropunused NUMBER)", + ""] cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' From 35622738aa2ac80e14c9e6248fb123e70ea49e02 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawk_lgZQnQAGL6OznVti_Dc90BJeiH7Ai4U" Date: Tue, 28 Dec 2010 03:51:55 +0000 Subject: [PATCH 0617/8313] --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn new file mode 100644 index 0000000000..5ae8f7a326 --- /dev/null +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -0,0 +1,11 @@ +git-annex does not seem to support all kinds of urls that git does. + +Specifically, if I have ~/bar set up on host foo: + +[remote "foo"] +## this one is not recognized as ssh url at all +# url = foo:bar +## this one makes git-annex try to access '/~/bar' literally +# url = ssh://foo/~/bar +## this one works + url = ssh://foo/home/tv/bar From 1f1d2a2ea7caadb0720427b1a66efb5ffca17c38 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawk_lgZQnQAGL6OznVti_Dc90BJeiH7Ai4U" Date: Tue, 28 Dec 2010 03:53:33 +0000 Subject: [PATCH 0618/8313] Layout. --- .../wishlist:_support_for_more_ssh_urls_.mdwn | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 5ae8f7a326..42107e51a3 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -2,10 +2,10 @@ git-annex does not seem to support all kinds of urls that git does. Specifically, if I have ~/bar set up on host foo: -[remote "foo"] -## this one is not recognized as ssh url at all -# url = foo:bar -## this one makes git-annex try to access '/~/bar' literally -# url = ssh://foo/~/bar -## this one works - url = ssh://foo/home/tv/bar + [remote "foo"] + ## this one is not recognized as ssh url at all + # url = foo:bar + ## this one makes git-annex try to access '/~/bar' literally + # url = ssh://foo/~/bar + ## this one works + url = ssh://foo/home/tv/bar From 70d68212f5283d27bac00812afdf582b54424e47 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 12:55:19 -0400 Subject: [PATCH 0619/8313] let's not use the forum for bug reports --- doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn new file mode 100644 index 0000000000..42107e51a3 --- /dev/null +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -0,0 +1,11 @@ +git-annex does not seem to support all kinds of urls that git does. + +Specifically, if I have ~/bar set up on host foo: + + [remote "foo"] + ## this one is not recognized as ssh url at all + # url = foo:bar + ## this one makes git-annex try to access '/~/bar' literally + # url = ssh://foo/~/bar + ## this one works + url = ssh://foo/home/tv/bar From 0b3cf9bc682e5dfbc78248aa263c2905fd3e79be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 12:56:06 -0400 Subject: [PATCH 0620/8313] moved to bugs --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 42107e51a3..3fd6482219 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -1,11 +1,4 @@ git-annex does not seem to support all kinds of urls that git does. -Specifically, if I have ~/bar set up on host foo: - - [remote "foo"] - ## this one is not recognized as ssh url at all - # url = foo:bar - ## this one makes git-annex try to access '/~/bar' literally - # url = ssh://foo/~/bar - ## this one works - url = ssh://foo/home/tv/bar +> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls]]. +> --[[Joey]] From c7344eff81c7de4f303097816ef26cc74146588a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 12:56:43 -0400 Subject: [PATCH 0621/8313] fix --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 3fd6482219..6ee01d2300 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -1,4 +1,4 @@ git-annex does not seem to support all kinds of urls that git does. -> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls]]. +> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls_]]. > --[[Joey]] From 022e0c7751db23805169c5429851903a3d482cb2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 13:48:11 -0400 Subject: [PATCH 0622/8313] Support scp-style urls for remotes (host:path). --- GitRepo.hs | 15 ++++++++++++++- debian/changelog | 6 ++++++ .../wishlist:_support_for_more_ssh_urls_.mdwn | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 122b5cc60f..2c2ad7b539 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -60,6 +60,7 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf +import Data.List (isInfixOf) import Utility @@ -314,8 +315,20 @@ configRemotes repo = map construct remotepairs isremote k = startswith "remote." k && endswith ".url" k remotename k = split "." k !! 1 construct (k,v) = (gen v) { remoteName = Just $ remotename k } - gen v | isURI v = repoFromUrl v + gen v | scpstyle v = repoFromUrl $ scptourl v + | isURI v = repoFromUrl v | otherwise = repoFromPath v + -- git remotes can be written scp style -- [user@]host:dir + -- where dir is relative to the user's home directory. + scpstyle v = isInfixOf ":" v && (not $ isInfixOf "//" v) + scptourl v = "ssh://" ++ host ++ slash dir + where + bits = split ":" v + host = bits !! 0 + dir = join ":" $ drop 1 bits + slash d | d == "" = "/~/" ++ dir + | d !! 0 == '/' = dir + | otherwise = "/~/" ++ dir {- Parses git config --list output into a config map. -} configParse :: String -> Map.Map String String diff --git a/debian/changelog b/debian/changelog index f8a3c3129d..dba343deff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.15) UNRELEASED; urgency=low + + * Support scp-style urls for remotes (host:path). + + -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 + git-annex (0.14) unstable; urgency=low * Bugfix to git annex unused in a repository with nothing yet annexed. diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn index 42107e51a3..6156ade054 100644 --- a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -9,3 +9,12 @@ Specifically, if I have ~/bar set up on host foo: # url = ssh://foo/~/bar ## this one works url = ssh://foo/home/tv/bar + +> scp-style is now supported. + +> `~` expansions (for the user's home, or other users) +> are somewhat tricky to support as they require running +> code on the remote to lookup homedirs. If git-annex grows a +> `git annex shell` that is run on the remote side +> (something I am considering for other reasons), it +> could handle the expansions there. --[[Joey]] From 6c58a58393a1d6d2257cb9e72e534387561943d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 15:28:23 -0400 Subject: [PATCH 0623/8313] details.. --- doc/bugs/problems_with_utf8_names.mdwn | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 521f92405e..30f3495f40 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -18,3 +18,45 @@ There are problems with displaying filenames in UTF8 encoding, as shown here: $ It looks like the common latin1-to-UTF8 encoding. Functionality other than otuput seems not to be affected. + +> Yes, I believe that git-annex is reading filename data from git +> as a stream of char8s, and not decoding unicode in it into logical +> characters. +> Haskell then I guess, tries to unicode encode it when it's output to +> the console. +> This only seems to matter WRT its output to the console; the data +> does not get mangled internally and so it accesses the right files +> under the hood. +> +> I am too new to haskell to really have a handle on how to handle +> unicode and other encodings issues with it. In general, there are three +> valid approaches: --[[Joey]] +> +> 1. Convert all input data to unicode and be unicode clean end-to-end +> internally. Problimatic here since filenames may not necessarily be +> encoded in utf-8 (an archive could have historical filenames using +> varying encodings), and you don't want which files are accessed to +> depend on locale settings. +> 1. Keep input and internal data un-decoded, but decode it when +> outputting a filename (assuming the filename is encoded using the +> user's configured encoding), and allow haskell's output encoding to then +> encode it according to the user's locale configuration. +> 1. Avoid encodings entirely. Mostly what I'm doing now; probably +> could find a way to disable encoding of console output. Then the raw +> filename would be displayed, which should work ok. git-annex does +> not really need to pull apart filenames; they are almost entirely +> opaque blobs. I guess that the `--exclude` option is the exception +> to that, but it is currently not unicode safe anyway. +> One other possible +> issue would be that this could cause problems if git-annex were +> translated. +> +> BTW, for more fun, try unsetting LANG, and then you can see +> stuff like this: + + joey@gnu:~/tmp/aa>git annex add ./Üa + add add add add git-annex: : commitAndReleaseBuffer: invalid + argument (Invalid or incomplete multibyte or wide character) + +> (Add -q to work around this; once it doesn't need to print the filename, +> it can act on it ok!) From aa4f91b2d67b9f7827b02acebfbf4e67ba33bb80 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:17:02 -0400 Subject: [PATCH 0624/8313] Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking. --- Backend/File.hs | 12 +++++++--- CmdLine.hs | 7 ++++++ Remotes.hs | 6 +++++ UUID.hs | 45 ++++++++++++++++++++++++++++++-------- Utility.hs | 12 +++++++++- debian/changelog | 2 ++ doc/copies.mdwn | 3 ++- doc/git-annex.mdwn | 13 ++++++++++- doc/location_tracking.mdwn | 3 +++ doc/trust.mdwn | 20 +++++++++++++++++ doc/walkthrough.mdwn | 2 +- 11 files changed, 109 insertions(+), 16 deletions(-) create mode 100644 doc/trust.mdwn diff --git a/Backend/File.hs b/Backend/File.hs index f88bb7c704..0b1cdd8e52 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -16,6 +16,7 @@ module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory +import Data.List (intersect) import TypeInternals import LocationLog @@ -91,11 +92,16 @@ checkRemoveKey key numcopiesM = do if force || numcopiesM == Just 0 then return True else do + g <- Annex.gitRepo + locations <- liftIO $ keyLocations g key + trusted <- getTrusted + let trustedlocations = intersect locations trusted remotes <- Remotes.keyPossibilities key + untrustedremotes <- reposWithoutUUID remotes trusted numcopies <- getNumCopies numcopiesM - if numcopies > length remotes - then notEnoughCopies numcopies (length remotes) [] - else findcopies numcopies 0 remotes [] + if numcopies > length untrustedremotes + then notEnoughCopies numcopies (length untrustedremotes) [] + else findcopies numcopies (length trustedlocations) untrustedremotes [] where findcopies need have [] bad | have >= need = return True diff --git a/CmdLine.hs b/CmdLine.hs index cb164a6ab2..7eab0a7e28 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -34,6 +34,8 @@ import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust subCmds :: [SubCommand] subCmds = @@ -61,6 +63,10 @@ subCmds = "de-initialize git-annex and clean out repository" , SubCommand "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" + , SubCommand "trust" remote Command.Trust.seek + "trust a repository" + , SubCommand "untrust" remote Command.Untrust.seek + "do not trust a repository" , SubCommand "fromkey" key Command.FromKey.seek "adds a file using a specific key" , SubCommand "dropkey" key Command.DropKey.seek @@ -84,6 +90,7 @@ subCmds = key = "KEY ..." desc = "DESCRIPTION" number = "NUMBER ..." + remote = "REMOTE ..." nothing = "" -- Each dashed command-line option results in generation of an action diff --git a/Remotes.hs b/Remotes.hs index cf49b624b4..725348a6a3 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -12,6 +12,7 @@ module Remotes ( inAnnex, same, commandLineRemote, + byName, copyFromRemote, copyToRemote, runCmd @@ -156,6 +157,11 @@ commandLineRemote = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" let name = if null fromName then toName else fromName + byName name + +{- Looks up a remote by name. -} +byName :: String -> Annex Git.Repo +byName name = do when (null name) $ error "no remote specified" g <- Annex.gitRepo let match = filter (\r -> name == Git.repoRemoteName r) $ diff --git a/UUID.hs b/UUID.hs index 41a35327d9..3e43c80612 100644 --- a/UUID.hs +++ b/UUID.hs @@ -14,9 +14,13 @@ module UUID ( prepUUID, genUUID, reposByUUID, + reposWithoutUUID, prettyPrintUUIDs, describeUUID, - uuidLog + uuidLog, + trustLog, + getTrusted, + setTrusted ) where import Control.Monad.State @@ -24,9 +28,7 @@ import Data.Maybe import Data.List import System.Cmd.Utils import System.IO -import System.Directory import qualified Data.Map as M -import System.Posix.Process import qualified GitRepo as Git import Types @@ -85,6 +87,14 @@ reposByUUID repos uuids = filterM match repos u <- getUUID r return $ isJust $ elemIndex u uuids +{- Filters a list of repos to ones that do not have the listed UUIDs. -} +reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] +reposWithoutUUID repos uuids = filterM unmatch repos + where + unmatch r = do + u <- getUUID r + return $ not $ isJust $ elemIndex u uuids + {- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do @@ -103,11 +113,7 @@ describeUUID uuid desc = do m <- uuidMap let m' = M.insert uuid desc m logfile <- uuidLog - pid <- liftIO $ getProcessID - let tmplogfile = logfile ++ ".tmp" ++ show pid - liftIO $ createDirectoryIfMissing True (parentDir logfile) - liftIO $ writeFile tmplogfile $ serialize m' - liftIO $ renameFile tmplogfile logfile + liftIO $ safeWriteFile logfile (serialize m') where serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m @@ -125,7 +131,28 @@ uuidMap = do ignoreerror _ = return "" {- Filename of uuid.log. -} -uuidLog :: Annex String +uuidLog :: Annex FilePath uuidLog = do g <- Annex.gitRepo return $ gitStateDir g ++ "uuid.log" + +{- Filename of trust.log. -} +trustLog :: Annex FilePath +trustLog = do + g <- Annex.gitRepo + return $ gitStateDir g ++ "trust.log" + +{- List of trusted UUIDs. -} +getTrusted :: Annex [UUID] +getTrusted = do + logfile <- trustLog + s <- liftIO $ catch (readFile logfile) ignoreerror + return $ map (\l -> head $ words l) $ lines s + where + ignoreerror _ = return "" + +{- Changes the list of trusted UUIDs. -} +setTrusted :: [UUID] -> Annex () +setTrusted u = do + logfile <- trustLog + liftIO $ safeWriteFile logfile $ unlines u diff --git a/Utility.hs b/Utility.hs index 2447c95a0b..3a6c757515 100644 --- a/Utility.hs +++ b/Utility.hs @@ -15,7 +15,8 @@ module Utility ( boolSystem, shellEscape, unsetFileMode, - readMaybe + readMaybe, + safeWriteFile ) where import System.IO @@ -139,3 +140,12 @@ readMaybe :: (Read a) => String -> Maybe a readMaybe s = case reads s of ((x,_):_) -> Just x _ -> Nothing + +{- Writes a file using a temp file that is renamed atomically into place. -} +safeWriteFile :: FilePath -> String -> IO () +safeWriteFile file content = do + pid <- getProcessID + let tmpfile = file ++ ".tmp" ++ show pid + createDirectoryIfMissing True (parentDir file) + writeFile tmpfile content + renameFile tmpfile file diff --git a/debian/changelog b/debian/changelog index dba343deff..9be8b37fd0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.15) UNRELEASED; urgency=low * Support scp-style urls for remotes (host:path). + * Add trust and untrust subcommands, to allow configuring remotes + that are trusted to retain files without explicit checking. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 diff --git a/doc/copies.mdwn b/doc/copies.mdwn index 165e54b340..5b3cbf5154 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -11,7 +11,8 @@ setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have -it, it will retain the file content to avoid data loss. +it, it will retain the file content to avoid data loss. Note that +[[trusted_remotes|trust]] are not explicitly checked. For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex get diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 71a4889ac7..caef49d977 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -171,6 +171,15 @@ Many git-annex subcommands will stage changes for later `git commit` by you. This is meant to be called from git's pre-commit hook. `git annex init` automatically creates a pre-commit hook using this. +* trust [repository ...] + + Records that a repository is [[trusted]] to not unexpectedly lose content. + Use with care. + +* untrust [repository ...] + + Undoes a trust command. + * fromkey file This can be used to maually set up a file to link to a specified key @@ -333,7 +342,9 @@ These files are used by git-annex, in your git repository: available. Annexed files in your git repository symlink to that content. `.git-annex/uuid.log` is used to map between repository UUID and -decscriptions. You may edit it. +decscriptions. + +`.git-annex/trust.log` is used to list the UUIDs of trusted repositories. `.git-annex/*.log` is where git-annex records its content tracking information. These files should be committed to git. diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn index a7d5c150b1..3791029f8f 100644 --- a/doc/location_tracking.mdwn +++ b/doc/location_tracking.mdwn @@ -26,3 +26,6 @@ descriptions to help you with finding them: Try making some of these repositories available: c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 e1938fee-d95b-11df-96cc-002170d25c55 + +In certian cases you may want to configure git-annex to [[trust]] +that location tracking information is always correct for a repository. diff --git a/doc/trust.mdwn b/doc/trust.mdwn new file mode 100644 index 0000000000..b04a112ecf --- /dev/null +++ b/doc/trust.mdwn @@ -0,0 +1,20 @@ +Normally, git-annex does not fully trust its stored [[location_tracking]] +information. When removing content, it will directly check +that other repositories have [[copies]]. + +Generally that explicit checking is a good idea. Consider that the current +[[location_tracking]] information for a remote may not yet have propigated +out. Or, a remote may have suffered a catastrophic loss of data, or itself +been lost. + +Sometimes though, you may have reasons to trust the location tracking +information for a remote repository. For example, it may be an offline +archival drive, from which you rarely or never remove content. Deciding +when it makes sense to trust the tracking info is up to you. + +One way to handle this is just to use `--force` when a command cannot +access a remote you trust. + +Another option is to configure which remotes you trust with the +`git annex trust` command, or by manually adding the UUIDs of trusted remotes +to `.git-annex/trust.log`. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 9c8106d513..b486a4b1ff 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -134,7 +134,7 @@ you'll see something like this. (Use --force to override this check, or adjust annex.numcopies.) failed -Here you might --force it to drop `important_file` if you trust your backup. +Here you might --force it to drop `important_file` if you [[trust]] your backup. But `other.iso` looks to have never been copied to anywhere else, so if it's something you want to hold onto, you'd need to transfer it to some other repository before dropping it. From ee5d81429dcc6c8c2922c4c6696abc17e0440ff6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:19:01 -0400 Subject: [PATCH 0625/8313] tweak --- Backend/File.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 0b1cdd8e52..092cf7e73d 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -95,13 +95,13 @@ checkRemoveKey key numcopiesM = do g <- Annex.gitRepo locations <- liftIO $ keyLocations g key trusted <- getTrusted - let trustedlocations = intersect locations trusted + let trustedcopies = length $ intersect locations trusted remotes <- Remotes.keyPossibilities key untrustedremotes <- reposWithoutUUID remotes trusted numcopies <- getNumCopies numcopiesM if numcopies > length untrustedremotes then notEnoughCopies numcopies (length untrustedremotes) [] - else findcopies numcopies (length trustedlocations) untrustedremotes [] + else findcopies numcopies trustedcopies untrustedremotes [] where findcopies need have [] bad | have >= need = return True From 7329a1b655f7011f09a4519209da748061a014c4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:19:52 -0400 Subject: [PATCH 0626/8313] note --- debian/changelog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 9be8b37fd0..bc2901ef74 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.15) UNRELEASED; urgency=low - * Support scp-style urls for remotes (host:path). + * Support scp-style urls for remotes (host:path). Note that + paths relative to the user's home directory, or containing "~" are + not yet supported. * Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking. From 3714364905a23f3b64c6573a4121bd1cc5acb11e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:44:36 -0400 Subject: [PATCH 0627/8313] design for a git-annex-shell --- .../wishlist:_support_for_more_ssh_urls_.mdwn | 2 +- doc/todo/git-annex-shell.mdwn | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 doc/todo/git-annex-shell.mdwn diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn index 6156ade054..ee3b4a64ad 100644 --- a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -16,5 +16,5 @@ Specifically, if I have ~/bar set up on host foo: > are somewhat tricky to support as they require running > code on the remote to lookup homedirs. If git-annex grows a > `git annex shell` that is run on the remote side -> (something I am considering for other reasons), it +> (something I am [[considering|todo/git-annex-shell]] for other reasons), it > could handle the expansions there. --[[Joey]] diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn new file mode 100644 index 0000000000..a5ff7d6fd7 --- /dev/null +++ b/doc/todo/git-annex-shell.mdwn @@ -0,0 +1,42 @@ +I've been considering adding a `git-annex-shell` command. This would +be similar to `git-shell` (and in fact would pass unknown commands off to +`git-shell`). + +## Reasons + +* Allows locking down an account to only be able to use git-annex (and + git). +* Avoids needing to construct complex shell commands to run on the remote + system. (Mostly already avoided by the plumbing level commands.) +* Could possibly allow multiple things to be done with one ssh connection + in future. +* Allows expanding `~` and `~user` in repopath on the remote system. + +## Design + +`git-annex-shell -c ` + +### options + +Need at least `--quiet`, `--backend`, `--key`, `--force` + +### commands + +* `configlist repopath` + + Returns `git config --list`, for use by `tryGitConfigRead`. + + May filter the listed config to only the options git-annex really needs, + to prevent info disclosure. + +* `inannex repopath key ...` + + Checks if the keys are in the annex; shell exits zero if so. + +* `dropkey repopath key ... ` + + Same as `git annex dropkey`, and taking the same dashed options. + +* `setkey repopath tmpfile` + + Same as `git annex setkey`, and taking the same dashed options. From 1f20277ec7f23fa638422987d312f02f6ae1a5ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:44:55 -0400 Subject: [PATCH 0628/8313] forgot to add these --- Command/Trust.hs | 40 ++++++++++++++++++++++++++++++++++++++++ Command/Untrust.hs | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 Command/Trust.hs create mode 100644 Command/Untrust.hs diff --git a/Command/Trust.hs b/Command/Trust.hs new file mode 100644 index 0000000000..8060ee66f9 --- /dev/null +++ b/Command/Trust.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Trust where + +import Control.Monad.State (liftIO) +import Control.Monad (unless) + +import Command +import qualified Annex +import qualified GitRepo as Git +import qualified Remotes +import UUID +import Messages + +seek :: [SubCmdSeek] +seek = [withString start] + +{- Marks a remote as trusted. -} +start :: SubCmdStartString +start name = do + r <- Remotes.byName name + showStart "trust" name + return $ Just $ perform r + +perform :: Git.Repo -> SubCmdPerform +perform repo = do + uuid <- getUUID repo + trusted <- getTrusted + unless (elem uuid trusted) $ do + setTrusted $ uuid:trusted + g <- Annex.gitRepo + logfile <- trustLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + return $ Just $ return True diff --git a/Command/Untrust.hs b/Command/Untrust.hs new file mode 100644 index 0000000000..5ed8de2451 --- /dev/null +++ b/Command/Untrust.hs @@ -0,0 +1,40 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Untrust where + +import Control.Monad.State (liftIO) +import Control.Monad (when) + +import Command +import qualified Annex +import qualified GitRepo as Git +import qualified Remotes +import UUID +import Messages + +seek :: [SubCmdSeek] +seek = [withString start] + +{- Marks a remote as not trusted. -} +start :: SubCmdStartString +start name = do + r <- Remotes.byName name + showStart "untrust" name + return $ Just $ perform r + +perform :: Git.Repo -> SubCmdPerform +perform repo = do + uuid <- getUUID repo + trusted <- getTrusted + when (elem uuid trusted) $ do + setTrusted $ filter (\u -> u /= uuid) trusted + g <- Annex.gitRepo + logfile <- trustLog + liftIO $ Git.run g ["add", logfile] + liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + return $ Just $ return True From 39d0bcb79332426ac9e20f15614a8eef57a8e7d2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 28 Dec 2010 17:46:38 -0400 Subject: [PATCH 0629/8313] things to do still --- doc/todo/git-annex-shell.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn index a5ff7d6fd7..47db0c1ca7 100644 --- a/doc/todo/git-annex-shell.mdwn +++ b/doc/todo/git-annex-shell.mdwn @@ -40,3 +40,9 @@ Need at least `--quiet`, `--backend`, `--key`, `--force` * `setkey repopath tmpfile` Same as `git annex setkey`, and taking the same dashed options. + +### TODO + +* To be usable as a locked down shell, needs a way to launch the + rsync server, for file receiving. Safely? +* Also needs a way to support receiving files by scp. From d475aac37544d9442fe6ca6af5e949fd48d3bc96 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:21:38 -0400 Subject: [PATCH 0630/8313] refactor --- Backend/File.hs | 17 ++++++----------- Command/Move.hs | 4 ++-- Remotes.hs | 29 +++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 092cf7e73d..f9a3dbfcbf 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -16,7 +16,6 @@ module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory -import Data.List (intersect) import TypeInternals import LocationLog @@ -50,7 +49,8 @@ dummyStore _ _ = return True - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex Bool copyKeyFile key file = do - remotes <- Remotes.keyPossibilities key + (trusted, untrusted) <- Remotes.keyPossibilities key + let remotes = trusted ++ untrusted if null remotes then do showNote "not available" @@ -92,16 +92,11 @@ checkRemoveKey key numcopiesM = do if force || numcopiesM == Just 0 then return True else do - g <- Annex.gitRepo - locations <- liftIO $ keyLocations g key - trusted <- getTrusted - let trustedcopies = length $ intersect locations trusted - remotes <- Remotes.keyPossibilities key - untrustedremotes <- reposWithoutUUID remotes trusted + (trusted, untrusted) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM - if numcopies > length untrustedremotes - then notEnoughCopies numcopies (length untrustedremotes) [] - else findcopies numcopies trustedcopies untrustedremotes [] + if numcopies > length untrusted + then notEnoughCopies numcopies (length untrusted) [] + else findcopies numcopies (length trusted) untrusted [] where findcopies need have [] bad | have >= need = return True diff --git a/Command/Move.hs b/Command/Move.hs index cb6525919e..61242955e3 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -110,8 +110,8 @@ toCleanup move remote key tmpfile = do fromStart :: Bool -> SubCmdStartString fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote - l <- Remotes.keyPossibilities key - if null $ filter (\r -> Remotes.same r remote) l + (trusted, untrusted) <- Remotes.keyPossibilities key + if null $ filter (\r -> Remotes.same r remote) (trusted ++ untrusted) then return Nothing else do showAction move file diff --git a/Remotes.hs b/Remotes.hs index 725348a6a3..3905315508 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -24,7 +24,7 @@ import qualified Data.Map as Map import Data.String.Utils import System.Directory hiding (copyFile) import System.Posix.Directory -import Data.List +import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) import Types @@ -43,11 +43,11 @@ import qualified SysConfig list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes -{- Cost ordered list of remotes that the LocationLog indicate may have a key. -} -keyPossibilities :: Key -> Annex [Git.Repo] +{- Cost ordered lists of remotes that the LocationLog indicate may have a key. + - The first list is of remotes that are trusted to have the key; the + - second is of untrusted remotes that may have the key. -} +keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo]) keyPossibilities key = do - g <- Annex.gitRepo - uuids <- liftIO $ keyLocations g key allremotes <- remotesByCost -- To determine if a remote has a key, its UUID needs to be known. -- The locally cached UUIDs of remotes can fall out of date if @@ -56,7 +56,7 @@ keyPossibilities key = do -- sure we only do it once per git-annex run. remotesread <- Annex.flagIsSet "remotesread" if remotesread - then reposByUUID allremotes uuids + then partition allremotes else do -- We assume that it's cheap to read the config -- of non-URL remotes, so that is done each time. @@ -74,11 +74,24 @@ keyPossibilities key = do _ <- mapM tryGitConfigRead todo Annex.flagChange "remotesread" $ FlagBool True keyPossibilities key - else reposByUUID allremotes uuids + else partition allremotes where cachedUUID r = do u <- getUUID r - return $ null u + return $ null u + partition allremotes = do + g <- Annex.gitRepo + validuuids <- liftIO $ keyLocations g key + alltrusted <- getTrusted + -- get uuids trusted to have the key + -- note that validuuids is assumed to not have dups + let validtrusted = intersect validuuids alltrusted + -- remotes that match uuids that have the key + validremotes <- reposByUUID allremotes validuuids + -- partition out the trusted and untrusted remotes + trustedremotes <- reposByUUID validremotes validtrusted + untrustedremotes <- reposWithoutUUID validremotes alltrusted + return (trustedremotes, untrustedremotes) {- Checks if a given remote has the content for a key inAnnex. - If the remote cannot be accessed, returns a Left error. From 885f7048d5598409476016be3b46c1427532ede3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:31:25 -0400 Subject: [PATCH 0631/8313] Fix bug in numcopies handling when a repoisitory has multiple remotes that point to the same repository. --- Backend/File.hs | 29 +++++++++++++++++------------ debian/changelog | 2 ++ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index f9a3dbfcbf..4773ade9d0 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -94,26 +94,31 @@ checkRemoveKey key numcopiesM = do else do (trusted, untrusted) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM + trusteduuids <- mapM getUUID trusted if numcopies > length untrusted then notEnoughCopies numcopies (length untrusted) [] - else findcopies numcopies (length trusted) untrusted [] + else findcopies numcopies trusteduuids untrusted [] where findcopies need have [] bad - | have >= need = return True - | otherwise = notEnoughCopies need have bad + | length have >= need = return True + | otherwise = notEnoughCopies need (length have) bad findcopies need have (r:rs) bad - | have >= need = return True - | otherwise = do - haskey <- Remotes.inAnnex r key - case haskey of - Right True -> findcopies need (have+1) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) - notEnoughCopies need have bad = do + | length have >= need = return True + | otherwise = do + u <- getUUID r + if not $ elem u have + then do + haskey <- Remotes.inAnnex r key + case haskey of + Right True -> findcopies need (u:have) rs bad + Right False -> findcopies need have rs bad + Left _ -> findcopies need have rs (r:bad) + else findcopies need have rs bad + notEnoughCopies need numhave bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - show have ++ " out of " ++ show need ++ + show numhave ++ " out of " ++ show need ++ " necessary copies" showTriedRemotes bad showLocations key diff --git a/debian/changelog b/debian/changelog index bc2901ef74..3707995160 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,8 @@ git-annex (0.15) UNRELEASED; urgency=low not yet supported. * Add trust and untrust subcommands, to allow configuring remotes that are trusted to retain files without explicit checking. + * Fix bug in numcopies handling when a repoisitory has multiple remotes + that point to the same repository. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 From 1448d8b42dbc8c29477ab75bbc1e5a33d1eae3ba Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:41:27 -0400 Subject: [PATCH 0632/8313] improve list of remotes in error message --- Backend/File.hs | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 4773ade9d0..918ba4716a 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -54,13 +54,13 @@ copyKeyFile key file = do if null remotes then do showNote "not available" - showLocations key + showLocations key [] return False else trycopy remotes remotes where trycopy full [] = do showTriedRemotes full - showLocations key + showLocations key [] return False trycopy full (r:rs) = do probablythere <- probablyPresent r @@ -95,44 +95,40 @@ checkRemoveKey key numcopiesM = do (trusted, untrusted) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM trusteduuids <- mapM getUUID trusted - if numcopies > length untrusted - then notEnoughCopies numcopies (length untrusted) [] - else findcopies numcopies trusteduuids untrusted [] + findcopies numcopies trusteduuids untrusted [] where findcopies need have [] bad | length have >= need = return True - | otherwise = notEnoughCopies need (length have) bad + | otherwise = notEnoughCopies need have bad findcopies need have (r:rs) bad | length have >= need = return True | otherwise = do u <- getUUID r - if not $ elem u have - then do - haskey <- Remotes.inAnnex r key - case haskey of - Right True -> findcopies need (u:have) rs bad - Right False -> findcopies need have rs bad - Left _ -> findcopies need have rs (r:bad) - else findcopies need have rs bad - notEnoughCopies need numhave bad = do + let dup = elem u have + haskey <- Remotes.inAnnex r key + case (dup, haskey) of + (False, Right True) -> findcopies need (u:have) rs bad + (False, Left _) -> findcopies need have rs (r:bad) + _ -> findcopies need have rs bad + notEnoughCopies need have bad = do unsafe showLongNote $ "Could only verify the existence of " ++ - show numhave ++ " out of " ++ show need ++ + show (length have) ++ " out of " ++ show need ++ " necessary copies" showTriedRemotes bad - showLocations key + showLocations key have hint return False unsafe = showNote "unsafe" hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)" -showLocations :: Key -> Annex () -showLocations key = do +showLocations :: Key -> [UUID] -> Annex () +showLocations key exclude = do g <- Annex.gitRepo u <- getUUID g uuids <- liftIO $ keyLocations g key - let uuidsf = filter (/= u) uuids + let uuidsf = filter (\l -> l /= u && (not $ elem l exclude)) uuids ppuuids <- prettyPrintUUIDs uuidsf if null uuidsf then showLongNote $ "No other repository is known to contain the file." From ef85e699babd2aabdc64e048422456ac68244ef2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:48:55 -0400 Subject: [PATCH 0633/8313] simplify --- UUID.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/UUID.hs b/UUID.hs index 3e43c80612..21f2b202e6 100644 --- a/UUID.hs +++ b/UUID.hs @@ -24,8 +24,6 @@ module UUID ( ) where import Control.Monad.State -import Data.Maybe -import Data.List import System.Cmd.Utils import System.IO import qualified Data.Map as M @@ -85,7 +83,7 @@ reposByUUID repos uuids = filterM match repos where match r = do u <- getUUID r - return $ isJust $ elemIndex u uuids + return $ elem u uuids {- Filters a list of repos to ones that do not have the listed UUIDs. -} reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo] @@ -93,7 +91,7 @@ reposWithoutUUID repos uuids = filterM unmatch repos where unmatch r = do u <- getUUID r - return $ not $ isJust $ elemIndex u uuids + return $ not $ elem u uuids {- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String From e64ffc212e394d814ab73a32f750acbccb43dd1f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 16:58:44 -0400 Subject: [PATCH 0634/8313] support trusted repositories that are not configured as remotes --- Backend/File.hs | 7 +++---- Command/Move.hs | 2 +- Remotes.hs | 26 ++++++++++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index 918ba4716a..ee73150211 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -49,7 +49,7 @@ dummyStore _ _ = return True - and copy it over to this one. -} copyKeyFile :: Key -> FilePath -> Annex Bool copyKeyFile key file = do - (trusted, untrusted) <- Remotes.keyPossibilities key + (trusted, untrusted, _) <- Remotes.keyPossibilities key let remotes = trusted ++ untrusted if null remotes then do @@ -92,10 +92,9 @@ checkRemoveKey key numcopiesM = do if force || numcopiesM == Just 0 then return True else do - (trusted, untrusted) <- Remotes.keyPossibilities key + (_, untrusted, have) <- Remotes.keyPossibilities key numcopies <- getNumCopies numcopiesM - trusteduuids <- mapM getUUID trusted - findcopies numcopies trusteduuids untrusted [] + findcopies numcopies have untrusted [] where findcopies need have [] bad | length have >= need = return True diff --git a/Command/Move.hs b/Command/Move.hs index 61242955e3..eb223f5abf 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -110,7 +110,7 @@ toCleanup move remote key tmpfile = do fromStart :: Bool -> SubCmdStartString fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote - (trusted, untrusted) <- Remotes.keyPossibilities key + (trusted, untrusted, _) <- Remotes.keyPossibilities key if null $ filter (\r -> Remotes.same r remote) (trusted ++ untrusted) then return Nothing else do diff --git a/Remotes.hs b/Remotes.hs index 3905315508..99c0930adf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -44,9 +44,15 @@ list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes {- Cost ordered lists of remotes that the LocationLog indicate may have a key. - - The first list is of remotes that are trusted to have the key; the - - second is of untrusted remotes that may have the key. -} -keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo]) + - + - The first list is of remotes that are trusted to have the key. + - + - The second is of untrusted remotes that may have the key. + - + - Also returns a list of all UUIDs that are trusted to have the key + - (some may not have configured remotes). + -} +keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) keyPossibilities key = do allremotes <- remotesByCost -- To determine if a remote has a key, its UUID needs to be known. @@ -79,19 +85,19 @@ keyPossibilities key = do cachedUUID r = do u <- getUUID r return $ null u - partition allremotes = do + partition remotes = do g <- Annex.gitRepo validuuids <- liftIO $ keyLocations g key - alltrusted <- getTrusted + trusted <- getTrusted -- get uuids trusted to have the key -- note that validuuids is assumed to not have dups - let validtrusted = intersect validuuids alltrusted + let validtrusteduuids = intersect validuuids trusted -- remotes that match uuids that have the key - validremotes <- reposByUUID allremotes validuuids + validremotes <- reposByUUID remotes validuuids -- partition out the trusted and untrusted remotes - trustedremotes <- reposByUUID validremotes validtrusted - untrustedremotes <- reposWithoutUUID validremotes alltrusted - return (trustedremotes, untrustedremotes) + trustedremotes <- reposByUUID validremotes validtrusteduuids + untrustedremotes <- reposWithoutUUID validremotes trusted + return (trustedremotes, untrustedremotes, validtrusteduuids) {- Checks if a given remote has the content for a key inAnnex. - If the remote cannot be accessed, returns a Left error. From 25ffa04c5a116dc6ff4266dd738a93f03e7e6017 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 17:00:14 -0400 Subject: [PATCH 0635/8313] wording --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 3707995160..76e16af2ce 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,10 +3,10 @@ git-annex (0.15) UNRELEASED; urgency=low * Support scp-style urls for remotes (host:path). Note that paths relative to the user's home directory, or containing "~" are not yet supported. - * Add trust and untrust subcommands, to allow configuring remotes + * Add trust and untrust subcommands, to allow configuring repositories that are trusted to retain files without explicit checking. - * Fix bug in numcopies handling when a repoisitory has multiple remotes - that point to the same repository. + * Fix bug in numcopies handling when multiple remotes pointed to the + same repository. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 From de2f6137d3f38f822b0bfbf534e8f785459ea3cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 17:01:56 -0400 Subject: [PATCH 0636/8313] wording --- doc/copies.mdwn | 2 +- doc/trust.mdwn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/copies.mdwn b/doc/copies.mdwn index 5b3cbf5154..98f315081f 100644 --- a/doc/copies.mdwn +++ b/doc/copies.mdwn @@ -12,7 +12,7 @@ setting in `.gitattributes` files. `git annex drop` attempts to check with other git remotes, to check that N copies of the file exist. If enough repositories cannot be verified to have it, it will retain the file content to avoid data loss. Note that -[[trusted_remotes|trust]] are not explicitly checked. +[[trusted_repositories|trust]] are not explicitly checked. For example, consider three repositories: Server, Laptop, and USB. Both Server and USB have a copy of a file, and N=1. If on Laptop, you `git annex get diff --git a/doc/trust.mdwn b/doc/trust.mdwn index b04a112ecf..6ce43f30eb 100644 --- a/doc/trust.mdwn +++ b/doc/trust.mdwn @@ -1,6 +1,6 @@ Normally, git-annex does not fully trust its stored [[location_tracking]] information. When removing content, it will directly check -that other repositories have [[copies]]. +that other repositories have enough [[copies]]. Generally that explicit checking is a good idea. Consider that the current [[location_tracking]] information for a remote may not yet have propigated From 5b8f8ced0b6c3b22dcb5425862b9218685fbe394 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 19:09:02 -0400 Subject: [PATCH 0637/8313] don't include the current repo in trusted uuid list --- Remotes.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 99c0930adf..053814d66d 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -49,7 +49,7 @@ list remotes = join ", " $ map Git.repoDescribe remotes - - The second is of untrusted remotes that may have the key. - - - Also returns a list of all UUIDs that are trusted to have the key + - Also returns a list of UUIDs that are trusted to have the key - (some may not have configured remotes). -} keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) @@ -87,11 +87,13 @@ keyPossibilities key = do return $ null u partition remotes = do g <- Annex.gitRepo + u <- getUUID g validuuids <- liftIO $ keyLocations g key trusted <- getTrusted -- get uuids trusted to have the key -- note that validuuids is assumed to not have dups - let validtrusteduuids = intersect validuuids trusted + let validtrusteduuids = filter (/= u) $ + intersect validuuids trusted -- remotes that match uuids that have the key validremotes <- reposByUUID remotes validuuids -- partition out the trusted and untrusted remotes From 36b73714d04ad478c0f06d82cf10b24706cd8c84 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 19:21:13 -0400 Subject: [PATCH 0638/8313] better filtering out of current repo --- Remotes.hs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 053814d66d..78ab010cef 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -88,12 +88,14 @@ keyPossibilities key = do partition remotes = do g <- Annex.gitRepo u <- getUUID g - validuuids <- liftIO $ keyLocations g key trusted <- getTrusted + -- get uuids of other repositories that are + -- believed to have the key + uuids <- liftIO $ keyLocations g key + let validuuids = filter (/= u) uuids -- get uuids trusted to have the key -- note that validuuids is assumed to not have dups - let validtrusteduuids = filter (/= u) $ - intersect validuuids trusted + let validtrusteduuids = intersect validuuids trusted -- remotes that match uuids that have the key validremotes <- reposByUUID remotes validuuids -- partition out the trusted and untrusted remotes From 88edf83b5c0a0be2b7776dcc6a0d16676a716d8a Mon Sep 17 00:00:00 2001 From: praet Date: Wed, 29 Dec 2010 23:47:46 +0000 Subject: [PATCH 0639/8313] --- doc/todo/add_--exclude_option_to_git_annex_find.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/todo/add_--exclude_option_to_git_annex_find.mdwn diff --git a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn new file mode 100644 index 0000000000..3441d262be --- /dev/null +++ b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn @@ -0,0 +1 @@ +Seems pretty self-explanatory. From 80bad5c43507d1a3c79d473287a18b041cfd1ed1 Mon Sep 17 00:00:00 2001 From: praet Date: Thu, 30 Dec 2010 00:01:02 +0000 Subject: [PATCH 0640/8313] --- ...e_repo_description_and__47__or_UUID_in_commit_message.mdwn | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn new file mode 100644 index 0000000000..9802738d12 --- /dev/null +++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn @@ -0,0 +1,4 @@ +Would help alot when having to add large(ish) amounts of remotes. + +Maybe detect this kind of commit message and ask user whether to automatically add them? See [[auto_remotes]]: +> Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? From f6db527af8f4c65d5e8f26abb56408303dc48050 Mon Sep 17 00:00:00 2001 From: praet Date: Thu, 30 Dec 2010 00:12:44 +0000 Subject: [PATCH 0641/8313] --- doc/todo/auto_remotes/discussion.mdwn | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/todo/auto_remotes/discussion.mdwn diff --git a/doc/todo/auto_remotes/discussion.mdwn b/doc/todo/auto_remotes/discussion.mdwn new file mode 100644 index 0000000000..691741cab8 --- /dev/null +++ b/doc/todo/auto_remotes/discussion.mdwn @@ -0,0 +1,2 @@ +Remotes log should probably be stored in ".git/annex/remote.log" +instead of ".git-annex/remote.log" to prevent leaking credentials. From c74347a50879401f4661a16f059b3fc467c28989 Mon Sep 17 00:00:00 2001 From: praet Date: Thu, 30 Dec 2010 00:21:00 +0000 Subject: [PATCH 0642/8313] --- doc/walkthrough.mdwn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b486a4b1ff..3e437e8f34 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -275,7 +275,11 @@ files when they're added to the annex, and this can slow things down significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: - * annex.backend=SHA1 +> annex.backends=SHA1 + +or run: + +> $ git config annex.backends SHA1 ## unused data From 829c8b76605ad3c52eb6147202cc9b867d58cf1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:28:34 -0400 Subject: [PATCH 0643/8313] clarify --- doc/todo/auto_remotes/discussion.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/todo/auto_remotes/discussion.mdwn b/doc/todo/auto_remotes/discussion.mdwn index 691741cab8..b9e1522a8f 100644 --- a/doc/todo/auto_remotes/discussion.mdwn +++ b/doc/todo/auto_remotes/discussion.mdwn @@ -1,2 +1,7 @@ Remotes log should probably be stored in ".git/annex/remote.log" instead of ".git-annex/remote.log" to prevent leaking credentials. + +> The idea is to distribute the info between repositories, which is +> why it'd go in `.git-annex`. Of course that does mean that repository +> location information would be included, and if that'd not desirable +> this feature would need to be turned off. --[[Joey]] From 03e579060b11dcca6115426e7a4255a7ded944e2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:30:03 -0400 Subject: [PATCH 0644/8313] already exists --- doc/todo/add_--exclude_option_to_git_annex_find.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn index 3441d262be..a797e97f58 100644 --- a/doc/todo/add_--exclude_option_to_git_annex_find.mdwn +++ b/doc/todo/add_--exclude_option_to_git_annex_find.mdwn @@ -1 +1,4 @@ Seems pretty self-explanatory. + +> This was already implemented, the --exclude option can be used +> for find as well as most any other subcommand. --[[Joey]] [[done]] From a91e31cef3598f622dd73e95151f96eeaedba327 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:33:55 -0400 Subject: [PATCH 0645/8313] response --- ...epo_description_and__47__or_UUID_in_commit_message.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn index 9802738d12..9ca61bff55 100644 --- a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn +++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn @@ -2,3 +2,10 @@ Would help alot when having to add large(ish) amounts of remotes. Maybe detect this kind of commit message and ask user whether to automatically add them? See [[auto_remotes]]: > Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? + +---- + +I'm not sure that the above suggestion is going down a path that really +makes sense. If you want a list of repository UUIDs and descriptions, +it's there in machine-usable form in `.git-annex/uuid.log`, there is no +need to try to pull this info out of git commit messages. --[[Joey]] From 14d59b40fb4f3a4c9a89266fecae91a0daf08088 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 29 Dec 2010 20:35:51 -0400 Subject: [PATCH 0646/8313] Revert walkthrough annex.backend change This reverts commit c74347a50879401f4661a16f059b3fc467c28989. I'm afraid that change was just all wrong. * The gitattributes setting is `annex.backend`. Not `annex.backends`. * `gitattributes` files need to start with a glob, which this commit removed. * git config's `annex.backends` is NOT the same as the gitattributes setting, and is not a knob that we should be encouraging users tweak. --- doc/walkthrough.mdwn | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 3e437e8f34..b486a4b1ff 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -275,11 +275,7 @@ files when they're added to the annex, and this can slow things down significantly for really big files. To make SHA1 the detault, just add something like this to `.gitattributes`: -> annex.backends=SHA1 - -or run: - -> $ git config annex.backends SHA1 + * annex.backend=SHA1 ## unused data From 6a5be9d53cad9ee2988c6d54001f387dfe1f2716 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 14:19:16 -0400 Subject: [PATCH 0647/8313] rename some stuff and prepare to break out more into Command/* --- Annex.hs | 4 +- CmdLine.hs | 69 +++++++++++++------------- Command.hs | 110 ++++++++++++++++++++++++------------------ Command/Add.hs | 11 +++-- Command/Copy.hs | 2 +- Command/Drop.hs | 8 +-- Command/DropKey.hs | 8 +-- Command/DropUnused.hs | 4 +- Command/Find.hs | 4 +- Command/Fix.hs | 8 +-- Command/FromKey.hs | 8 +-- Command/Fsck.hs | 6 +-- Command/Get.hs | 6 +-- Command/Init.hs | 8 +-- Command/Lock.hs | 6 +-- Command/Move.hs | 16 +++--- Command/PreCommit.hs | 10 ++-- Command/SetKey.hs | 8 +-- Command/Trust.hs | 6 +-- Command/Unannex.hs | 8 +-- Command/Uninit.hs | 6 +-- Command/Unlock.hs | 6 +-- Command/Untrust.hs | 6 +-- Command/Unused.hs | 6 +-- 24 files changed, 176 insertions(+), 158 deletions(-) diff --git a/Annex.hs b/Annex.hs index af761051dc..6e5198e8ef 100644 --- a/Annex.hs +++ b/Annex.hs @@ -110,10 +110,10 @@ flagGet name = do {- Adds a git command to the queue. -} queue :: String -> [String] -> FilePath -> Annex () -queue subcommand params file = do +queue command params file = do state <- get let q = Internals.repoqueue state - put state { Internals.repoqueue = GitQueue.add q subcommand params file } + put state { Internals.repoqueue = GitQueue.add q command params file } {- Returns the queue. -} queueGet :: Annex GitQueue.Queue diff --git a/CmdLine.hs b/CmdLine.hs index 7eab0a7e28..40ce4b1215 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -37,51 +37,50 @@ import qualified Command.Uninit import qualified Command.Trust import qualified Command.Untrust -subCmds :: [SubCommand] -subCmds = - [ SubCommand "add" path Command.Add.seek - "add files to annex" - , SubCommand "get" path Command.Get.seek +cmds :: [Command] +cmds = + [ Command.Add.command + , Command "get" path Command.Get.seek "make content of annexed files available" - , SubCommand "drop" path Command.Drop.seek + , Command "drop" path Command.Drop.seek "indicate content of files not currently wanted" - , SubCommand "move" path Command.Move.seek + , Command "move" path Command.Move.seek "move content of files to/from another repository" - , SubCommand "copy" path Command.Copy.seek + , Command "copy" path Command.Copy.seek "copy content of files to/from another repository" - , SubCommand "unlock" path Command.Unlock.seek + , Command "unlock" path Command.Unlock.seek "unlock files for modification" - , SubCommand "edit" path Command.Unlock.seek + , Command "edit" path Command.Unlock.seek "same as unlock" - , SubCommand "lock" path Command.Lock.seek + , Command "lock" path Command.Lock.seek "undo unlock command" - , SubCommand "init" desc Command.Init.seek + , Command "init" desc Command.Init.seek "initialize git-annex with repository description" - , SubCommand "unannex" path Command.Unannex.seek + , Command "unannex" path Command.Unannex.seek "undo accidential add command" - , SubCommand "uninit" path Command.Uninit.seek + , Command "uninit" path Command.Uninit.seek "de-initialize git-annex and clean out repository" - , SubCommand "pre-commit" path Command.PreCommit.seek + , Command "pre-commit" path Command.PreCommit.seek "run by git pre-commit hook" - , SubCommand "trust" remote Command.Trust.seek + , Command "trust" remote Command.Trust.seek "trust a repository" - , SubCommand "untrust" remote Command.Untrust.seek + , Command "untrust" remote Command.Untrust.seek "do not trust a repository" - , SubCommand "fromkey" key Command.FromKey.seek + , Command "fromkey" key Command.FromKey.seek "adds a file using a specific key" - , SubCommand "dropkey" key Command.DropKey.seek + , Command "dropkey" key Command.DropKey.seek "drops annexed content for specified keys" - , SubCommand "setkey" key Command.SetKey.seek + , Command "setkey" key Command.SetKey.seek "sets annexed content for a key using a temp file" - , SubCommand "fix" path Command.Fix.seek + , Command "fix" path Command.Fix.seek "fix up symlinks to point to annexed content" - , SubCommand "fsck" maybepath Command.Fsck.seek + , Command "fsck" maybepath Command.Fsck.seek "check for problems" - , SubCommand "unused" nothing Command.Unused.seek + , Command "unused" nothing Command.Unused.seek "look for unused file content" - , SubCommand "dropunused" number Command.DropUnused.seek + , Command "dropunused" number Command.DropUnused.seek "drop unused file content" - , SubCommand "find" maybepath Command.Find.seek + , Command "find" maybepath Command.Find.seek "lists available files" ] where @@ -125,13 +124,13 @@ header = "Usage: git-annex subcommand [option ..]" usage :: String usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where - cmddescs = unlines $ map (indent . showcmd) subCmds + cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = - subcmdname c ++ - pad 11 (subcmdname c) ++ - subcmdparams c ++ - pad 13 (subcmdparams c) ++ - subcmddesc c + cmdname c ++ + pad 11 (cmdname c) ++ + cmdparams c ++ + pad 13 (cmdparams c) ++ + cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' @@ -143,12 +142,12 @@ parseCmd argv = do when (null params) $ error usage case lookupCmd (head params) of [] -> error usage - [subcommand] -> do + [command] -> do _ <- sequence flags - prepSubCmd subcommand (drop 1 params) - _ -> error "internal error: multiple matching subcommands" + prepCmd command (drop 1 params) + _ -> error "internal error: multiple matching commands" where getopt = case getOpt Permute options argv of (flags, params, []) -> return (flags, params) (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == subcmdname c) subCmds + lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds diff --git a/Command.hs b/Command.hs index e30904d0f7..2144da353b 100644 --- a/Command.hs +++ b/Command.hs @@ -21,54 +21,54 @@ import qualified Annex import qualified GitRepo as Git import Locations -{- A subcommand runs in four stages. +{- A command runs in four stages. - - - 0. The seek stage takes the parameters passed to the subcommand, + - 0. The seek stage takes the parameters passed to the command, - looks through the repo to find the ones that are relevant - - to that subcommand (ie, new files to add), and generates + - to that command (ie, new files to add), and generates - a list of start stage actions. -} -type SubCmdSeek = [String] -> Annex [SubCmdStart] +type CommandSeek = [String] -> Annex [CommandStart] {- 1. The start stage is run before anything is printed about the - - subcommand, is passed some input, and can early abort it + - command, is passed some input, and can early abort it - if the input does not make sense. It should run quickly and - should not modify Annex state. -} -type SubCmdStart = Annex (Maybe SubCmdPerform) -{- 2. The perform stage is run after a message is printed about the subcommand +type CommandStart = Annex (Maybe CommandPerform) +{- 2. The perform stage is run after a message is printed about the command - being run, and it should be where the bulk of the work happens. -} -type SubCmdPerform = Annex (Maybe SubCmdCleanup) +type CommandPerform = Annex (Maybe CommandCleanup) {- 3. The cleanup stage is run only if the perform stage succeeds, and it - - returns the overall success/fail of the subcommand. -} -type SubCmdCleanup = Annex Bool -{- Some helper functions are used to build up SubCmdSeek and SubCmdStart + - returns the overall success/fail of the command. -} +type CommandCleanup = Annex Bool +{- Some helper functions are used to build up CommandSeek and CommandStart - functions. -} -type SubCmdSeekStrings = SubCmdStartString -> SubCmdSeek -type SubCmdStartString = String -> SubCmdStart +type CommandSeekStrings = CommandStartString -> CommandSeek +type CommandStartString = String -> CommandStart type BackendFile = (FilePath, Maybe Backend) -type SubCmdSeekBackendFiles = SubCmdStartBackendFile -> SubCmdSeek -type SubCmdStartBackendFile = BackendFile -> SubCmdStart +type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek +type CommandStartBackendFile = BackendFile -> CommandStart type AttrFile = (FilePath, String) -type SubCmdSeekAttrFiles = SubCmdStartAttrFile -> SubCmdSeek -type SubCmdStartAttrFile = AttrFile -> SubCmdStart -type SubCmdSeekNothing = SubCmdStart -> SubCmdSeek -type SubCmdStartNothing = SubCmdStart +type CommandSeekAttrFiles = CommandStartAttrFile -> CommandSeek +type CommandStartAttrFile = AttrFile -> CommandStart +type CommandSeekNothing = CommandStart -> CommandSeek +type CommandStartNothing = CommandStart -data SubCommand = SubCommand { - subcmdname :: String, - subcmdparams :: String, - subcmdseek :: [SubCmdSeek], - subcmddesc :: String +data Command = Command { + cmdname :: String, + cmdparams :: String, + cmdseek :: [CommandSeek], + cmddesc :: String } -{- Prepares a list of actions to run to perform a subcommand, based on +{- Prepares a list of actions to run to perform a command, based on - the parameters passed to it. -} -prepSubCmd :: SubCommand -> [String] -> Annex [Annex Bool] -prepSubCmd SubCommand { subcmdseek = seek } params = do +prepCmd :: Command -> [String] -> Annex [Annex Bool] +prepCmd Command { cmdseek = seek } params = do lists <- mapM (\s -> s params) seek - return $ map doSubCmd $ foldl (++) [] lists + return $ map doCommand $ foldl (++) [] lists -{- Runs a subcommand through the start, perform and cleanup stages -} -doSubCmd :: SubCmdStart -> SubCmdCleanup -doSubCmd start = do +{- Runs a command through the start, perform and cleanup stages -} +doCommand :: CommandStart -> CommandCleanup +doCommand start = do s <- start case s of Nothing -> return True @@ -104,20 +104,20 @@ isAnnexed file a = do {- These functions find appropriate files or other things based on a user's parameters, and run a specified action on them. -} -withFilesInGit :: SubCmdSeekStrings +withFilesInGit :: CommandSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ Git.inRepo repo params files' <- filterFiles files return $ map a files' -withAttrFilesInGit :: String -> SubCmdSeekAttrFiles +withAttrFilesInGit :: String -> CommandSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo files <- liftIO $ Git.inRepo repo params files' <- filterFiles files pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs -withFilesMissing :: SubCmdSeekStrings +withFilesMissing :: CommandSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params files' <- filterFiles files @@ -126,27 +126,27 @@ withFilesMissing a params = do missing f = do e <- doesFileExist f return $ not e -withFilesNotInGit :: SubCmdSeekBackendFiles +withFilesNotInGit :: CommandSeekBackendFiles withFilesNotInGit a params = do repo <- Annex.gitRepo newfiles <- liftIO $ Git.notInRepo repo params newfiles' <- filterFiles newfiles backendPairs a newfiles' -withString :: SubCmdSeekStrings +withString :: CommandSeekStrings withString a params = return [a $ unwords params] -withStrings :: SubCmdSeekStrings +withStrings :: CommandSeekStrings withStrings a params = return $ map a params -withFilesToBeCommitted :: SubCmdSeekStrings +withFilesToBeCommitted :: CommandSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ Git.stagedFiles repo params tocommit' <- filterFiles tocommit return $ map a tocommit' -withFilesUnlocked :: SubCmdSeekBackendFiles +withFilesUnlocked :: CommandSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles -withFilesUnlockedToBeCommitted :: SubCmdSeekBackendFiles +withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles -withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> SubCmdSeekBackendFiles +withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles withFilesUnlocked' typechanged a params = do -- unlocked files have changed type from a symlink to a regular file repo <- Annex.gitRepo @@ -155,29 +155,29 @@ withFilesUnlocked' typechanged a params = do map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' -withKeys :: SubCmdSeekStrings +withKeys :: CommandSeekStrings withKeys a params = return $ map a params -withTempFile :: SubCmdSeekStrings +withTempFile :: CommandSeekStrings withTempFile a params = return $ map a params -withNothing :: SubCmdSeekNothing +withNothing :: CommandSeekNothing withNothing a [] = return [a] withNothing _ _ = return [] -backendPairs :: SubCmdSeekBackendFiles +backendPairs :: CommandSeekBackendFiles backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a pairs {- Default to acting on all files matching the seek action if - none are specified. -} -withAll :: (a -> SubCmdSeek) -> a -> SubCmdSeek +withAll :: (a -> CommandSeek) -> a -> CommandSeek withAll w a [] = do g <- Annex.gitRepo w a [Git.workTree g] withAll w a p = w a p {- Provides a default parameter to act on if none is specified. -} -withDefault :: String-> (a -> SubCmdSeek) -> (a -> SubCmdSeek) +withDefault :: String-> (a -> CommandSeek) -> (a -> CommandSeek) withDefault d w a [] = w a [d] withDefault _ w a p = w a p @@ -204,3 +204,19 @@ notSymlink :: FilePath -> IO Bool notSymlink f = do s <- liftIO $ getSymbolicLinkStatus f return $ not $ isSymbolicLink s + +{- descriptions of params used in usage message -} +paramPath :: String +paramPath = "PATH ..." +paramMaybePath :: String +paramMaybePath = "[PATH ...]" +paramKey :: String +paramKey = "KEY ..." +paramDesc :: String +paramDesc = "DESCRIPTION" +paramNumber :: String +paramNumber = "NUMBER ..." +paramRemote :: String +paramRemote = "REMOTE ..." +paramNothing :: String +paramNothing = "" diff --git a/Command/Add.hs b/Command/Add.hs index d141448a3a..08a880206b 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,14 +18,17 @@ import Types import Core import Messages +command :: Command +command = Command "add" paramPath seek "add files to annex" + {- Add acts on both files not checked into git yet, and unlocked files. -} -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesNotInGit start, withFilesUnlocked start] {- The add subcommand annexes a file, storing it in a backend, and then - moving it into the annex directory and setting up the symlink pointing - to its content. -} -start :: SubCmdStartBackendFile +start :: CommandStartBackendFile start pair@(file, _) = notAnnexed file $ do s <- liftIO $ getSymbolicLinkStatus file if (isSymbolicLink s) || (not $ isRegularFile s) @@ -34,14 +37,14 @@ start pair@(file, _) = notAnnexed file $ do showStart "add" file return $ Just $ perform pair -perform :: BackendFile -> SubCmdPerform +perform :: BackendFile -> CommandPerform perform (file, backend) = do stored <- Backend.storeFileKey file backend case stored of Nothing -> return Nothing Just (key, _) -> return $ Just $ cleanup file key -cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup :: FilePath -> Key -> CommandCleanup cleanup file key = do moveAnnex key file logStatus key ValuePresent diff --git a/Command/Copy.hs b/Command/Copy.hs index aa55731d9c..873df7ef2e 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -11,5 +11,5 @@ import Command import qualified Command.Move -- A copy is just a move that does not delete the source file. -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit $ Command.Move.start False] diff --git a/Command/Drop.hs b/Command/Drop.hs index 7c4fbea602..3f27405703 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -17,12 +17,12 @@ import Core import Messages import Utility -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withAttrFilesInGit "annex.numcopies" start] {- Indicates a file's content is not wanted anymore, and should be removed - if it's safe to do so. -} -start :: SubCmdStartAttrFile +start :: CommandStartAttrFile start (file, attr) = isAnnexed file $ \(key, backend) -> do inbackend <- Backend.hasKey key if not inbackend @@ -33,14 +33,14 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform :: Key -> Backend -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.removeKey backend key numcopies if success then return $ Just $ cleanup key else return Nothing -cleanup :: Key -> SubCmdCleanup +cleanup :: Key -> CommandCleanup cleanup key = do inannex <- inAnnex key when inannex $ removeAnnex key diff --git a/Command/DropKey.hs b/Command/DropKey.hs index aa72e1bbd6..870e9a7ab1 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -15,11 +15,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withKeys start] {- Drops cached content for a key. -} -start :: SubCmdStartString +start :: CommandStartString start keyname = do backends <- Backend.list let key = genKey (head backends) keyname @@ -33,12 +33,12 @@ start keyname = do showStart "dropkey" keyname return $ Just $ perform key -perform :: Key -> SubCmdPerform +perform :: Key -> CommandPerform perform key = do removeAnnex key return $ Just $ cleanup key -cleanup :: Key -> SubCmdCleanup +cleanup :: Key -> CommandCleanup cleanup key = do logStatus key ValueMissing return True diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 016a9faa73..9984e49f3f 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -18,11 +18,11 @@ import qualified Annex import qualified Command.Drop import Backend -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withStrings start] {- Drops unused content by number. -} -start :: SubCmdStartString +start :: CommandStartString start s = do m <- readUnusedLog case M.lookup s m of diff --git a/Command/Find.hs b/Command/Find.hs index 7b3c8c4636..9927b692d8 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -13,11 +13,11 @@ import Control.Monad.State (liftIO) import Command import Core -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withDefault "." withFilesInGit start] {- Output a list of files. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key when exists $ liftIO $ putStrLn file diff --git a/Command/Fix.hs b/Command/Fix.hs index 33630031f6..accdadd315 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,11 +17,11 @@ import Utility import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- Fixes the symlink to an annexed file. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do link <- calcGitLink file key l <- liftIO $ readSymbolicLink file @@ -31,14 +31,14 @@ start file = isAnnexed file $ \(key, _) -> do showStart "fix" file return $ Just $ perform file link -perform :: FilePath -> FilePath -> SubCmdPerform +perform :: FilePath -> FilePath -> CommandPerform perform file link = do liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ removeFile file liftIO $ createSymbolicLink link file return $ Just $ cleanup file -cleanup :: FilePath -> SubCmdCleanup +cleanup :: FilePath -> CommandCleanup cleanup file = do Annex.queue "add" ["--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index eb9ad5e514..991428136e 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -20,11 +20,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesMissing start] {- Adds a file pointing at a manually-specified key -} -start :: SubCmdStartString +start :: CommandStartString start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -36,13 +36,13 @@ start file = do "key ("++keyname++") is not present in backend" showStart "fromkey" file return $ Just $ perform file key -perform :: FilePath -> Key -> SubCmdPerform +perform :: FilePath -> Key -> CommandPerform perform file key = do link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ cleanup file -cleanup :: FilePath -> SubCmdCleanup +cleanup :: FilePath -> CommandCleanup cleanup file = do Annex.queue "add" ["--"] file return True diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 9acecfce67..034bdc388b 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,18 +13,18 @@ import Types import Messages import Utility -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withAll (withAttrFilesInGit "annex.numcopies") start] {- Checks a file's backend data for problems. -} -start :: SubCmdStartAttrFile +start :: CommandStartAttrFile start (file, attr) = isAnnexed file $ \(key, backend) -> do showStart "fsck" file return $ Just $ perform key backend numcopies where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> SubCmdPerform +perform :: Key -> Backend -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.fsckKey backend key numcopies if success diff --git a/Command/Get.hs b/Command/Get.hs index 628ed62935..214b689b8a 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -13,11 +13,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- Gets an annexed file from one of the backends. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, backend) -> do inannex <- inAnnex key if inannex @@ -26,7 +26,7 @@ start file = isAnnexed file $ \(key, backend) -> do showStart "get" file return $ Just $ perform key backend -perform :: Key -> Backend -> SubCmdPerform +perform :: Key -> Backend -> CommandPerform perform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if ok diff --git a/Command/Init.hs b/Command/Init.hs index e19849ba3b..806c34c989 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -20,18 +20,18 @@ import Messages import Locations import Types -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withString start] {- Stores description for the repository etc. -} -start :: SubCmdStartString +start :: CommandStartString start description = do when (null description) $ error "please specify a description of this repository\n" showStart "init" description return $ Just $ perform description -perform :: String -> SubCmdPerform +perform :: String -> CommandPerform perform description = do g <- Annex.gitRepo u <- getUUID g @@ -41,7 +41,7 @@ perform description = do gitPreCommitHookWrite g return $ Just cleanup -cleanup :: SubCmdCleanup +cleanup :: CommandCleanup cleanup = do g <- Annex.gitRepo logfile <- uuidLog diff --git a/Command/Lock.hs b/Command/Lock.hs index 27a030bc22..381162536e 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -15,16 +15,16 @@ import Messages import qualified Annex import qualified GitRepo as Git -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesUnlocked start] {- Undo unlock -} -start :: SubCmdStartBackendFile +start :: CommandStartBackendFile start (file, _) = do showStart "lock" file return $ Just $ perform file -perform :: FilePath -> SubCmdPerform +perform :: FilePath -> CommandPerform perform file = do liftIO $ removeFile file g <- Annex.gitRepo diff --git a/Command/Move.hs b/Command/Move.hs index eb223f5abf..8ba8dbfacc 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -21,14 +21,14 @@ import qualified Remotes import UUID import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit $ start True] {- Move (or copy) a file either --to or --from a repository. - - This only operates on the cached file content; it does not involve - moving data in the key-value backend. -} -start :: Bool -> SubCmdStartString +start :: Bool -> CommandStartString start move file = do fromName <- Annex.flagGet "fromrepository" toName <- Annex.flagGet "torepository" @@ -61,7 +61,7 @@ remoteHasKey remote key present = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -toStart :: Bool -> SubCmdStartString +toStart :: Bool -> CommandStartString toStart move file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if not ishere @@ -69,7 +69,7 @@ toStart move file = isAnnexed file $ \(key, _) -> do else do showAction move file return $ Just $ toPerform move key -toPerform :: Bool -> Key -> SubCmdPerform +toPerform :: Bool -> Key -> CommandPerform toPerform move key = do -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote @@ -86,7 +86,7 @@ toPerform move key = do then return $ Just $ toCleanup move remote key tmpfile else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key -toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> SubCmdCleanup +toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> CommandCleanup toCleanup move remote key tmpfile = do -- Tell remote to use the transferred content. ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", @@ -107,7 +107,7 @@ toCleanup move remote key tmpfile = do - If the current repository already has the content, it is still removed - from the other repository when moving. -} -fromStart :: Bool -> SubCmdStartString +fromStart :: Bool -> CommandStartString fromStart move file = isAnnexed file $ \(key, _) -> do remote <- Remotes.commandLineRemote (trusted, untrusted, _) <- Remotes.keyPossibilities key @@ -116,7 +116,7 @@ fromStart move file = isAnnexed file $ \(key, _) -> do else do showAction move file return $ Just $ fromPerform move key -fromPerform :: Bool -> Key -> SubCmdPerform +fromPerform :: Bool -> Key -> CommandPerform fromPerform move key = do remote <- Remotes.commandLineRemote ishere <- inAnnex key @@ -128,7 +128,7 @@ fromPerform move key = do if ok then return $ Just $ fromCleanup move remote key else return Nothing -- fail -fromCleanup :: Bool -> Git.Repo -> Key -> SubCmdCleanup +fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do ok <- Remotes.runCmd remote "git-annex" ["dropkey", "--quiet", "--force", diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 513d5d43f7..8d488514a8 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -17,21 +17,21 @@ import qualified Command.Fix {- The pre-commit hook needs to fix symlinks to all files being committed. - And, it needs to inject unlocked files into the annex. -} -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesToBeCommitted Command.Fix.start, withFilesUnlockedToBeCommitted start] -start :: SubCmdStartBackendFile +start :: CommandStartBackendFile start pair = return $ Just $ perform pair -perform :: BackendFile -> SubCmdPerform +perform :: BackendFile -> CommandPerform perform pair@(file, _) = do - ok <- doSubCmd $ Command.Add.start pair + ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file else error $ "failed to add " ++ file ++ "; canceling commit" -cleanup :: FilePath -> SubCmdCleanup +cleanup :: FilePath -> CommandCleanup cleanup file = do -- git commit will have staged the file's content; -- drop that and run command queued by Add.state to diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 55472ccaed..4c82de3a5b 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -19,11 +19,11 @@ import Types import Core import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withTempFile start] {- Sets cached content for a key. -} -start :: SubCmdStartString +start :: CommandStartString start file = do keyname <- Annex.flagGet "key" when (null keyname) $ error "please specify the key with --key" @@ -31,7 +31,7 @@ start file = do let key = genKey (head backends) keyname showStart "setkey" file return $ Just $ perform file key -perform :: FilePath -> Key -> SubCmdPerform +perform :: FilePath -> Key -> CommandPerform perform file key = do -- the file might be on a different filesystem, so mv is used -- rather than simply calling moveToObjectDir key file @@ -43,7 +43,7 @@ perform file key = do then return $ Just $ cleanup key else error "mv failed!" -cleanup :: Key -> SubCmdCleanup +cleanup :: Key -> CommandCleanup cleanup key = do logStatus key ValuePresent return True diff --git a/Command/Trust.hs b/Command/Trust.hs index 8060ee66f9..3c3ec3b7e5 100644 --- a/Command/Trust.hs +++ b/Command/Trust.hs @@ -17,17 +17,17 @@ import qualified Remotes import UUID import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withString start] {- Marks a remote as trusted. -} -start :: SubCmdStartString +start :: CommandStartString start name = do r <- Remotes.byName name showStart "trust" name return $ Just $ perform r -perform :: Git.Repo -> SubCmdPerform +perform :: Git.Repo -> CommandPerform perform repo = do uuid <- getUUID repo trusted <- getTrusted diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 9580fc5e7c..42354b8c49 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -20,16 +20,16 @@ import Core import qualified GitRepo as Git import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- The unannex subcommand undoes an add. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, backend) -> do showStart "unannex" file return $ Just $ perform file key backend -perform :: FilePath -> Key -> Backend -> SubCmdPerform +perform :: FilePath -> Key -> Backend -> CommandPerform perform file key backend = do -- force backend to always remove ok <- Backend.removeKey backend key (Just 0) @@ -37,7 +37,7 @@ perform file key backend = do then return $ Just $ cleanup file key else return Nothing -cleanup :: FilePath -> Key -> SubCmdCleanup +cleanup :: FilePath -> Key -> CommandCleanup cleanup file key = do g <- Annex.gitRepo diff --git a/Command/Uninit.hs b/Command/Uninit.hs index fcb77a92b0..6001c55cd9 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -20,15 +20,15 @@ import qualified Annex import qualified Command.Unannex import qualified Command.Init -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] -start :: SubCmdStartNothing +start :: CommandStartNothing start = do showStart "uninit" "" return $ Just $ perform -perform :: SubCmdPerform +perform :: CommandPerform perform = do g <- Annex.gitRepo diff --git a/Command/Unlock.hs b/Command/Unlock.hs index ff22fa84b3..21f34d1dba 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -18,17 +18,17 @@ import Locations import Core import CopyFile -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withFilesInGit start] {- The unlock subcommand replaces the symlink with a copy of the file's - content. -} -start :: SubCmdStartString +start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do showStart "unlock" file return $ Just $ perform file key -perform :: FilePath -> Key -> SubCmdPerform +perform :: FilePath -> Key -> CommandPerform perform dest key = do g <- Annex.gitRepo let src = annexLocation g key diff --git a/Command/Untrust.hs b/Command/Untrust.hs index 5ed8de2451..6458040b3f 100644 --- a/Command/Untrust.hs +++ b/Command/Untrust.hs @@ -17,17 +17,17 @@ import qualified Remotes import UUID import Messages -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withString start] {- Marks a remote as not trusted. -} -start :: SubCmdStartString +start :: CommandStartString start name = do r <- Remotes.byName name showStart "untrust" name return $ Just $ perform r -perform :: Git.Repo -> SubCmdPerform +perform :: Git.Repo -> CommandPerform perform repo = do uuid <- getUUID repo trusted <- getTrusted diff --git a/Command/Unused.hs b/Command/Unused.hs index 69a16f2547..dba9aa517a 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -17,16 +17,16 @@ import Messages import Locations import qualified Annex -seek :: [SubCmdSeek] +seek :: [CommandSeek] seek = [withNothing start] {- Finds unused content in the annex. -} -start :: SubCmdStartNothing +start :: CommandStartNothing start = do showStart "unused" "" return $ Just perform -perform :: SubCmdPerform +perform :: CommandPerform perform = do _ <- checkUnused return $ Just $ return True From a89a6f21145966e625b811741d8ae972a11d92b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:06:26 -0400 Subject: [PATCH 0648/8313] refactor in preparation for adding a git-annex-shell command --- CmdLine.hs | 164 ++++++++++-------------------------------- Command.hs | 20 ++++-- Command/Add.hs | 4 +- Command/Copy.hs | 4 ++ Command/Drop.hs | 4 ++ Command/DropKey.hs | 4 ++ Command/DropUnused.hs | 4 ++ Command/Find.hs | 4 ++ Command/Fix.hs | 4 ++ Command/FromKey.hs | 4 ++ Command/Fsck.hs | 4 ++ Command/Get.hs | 4 ++ Command/Init.hs | 4 ++ Command/Lock.hs | 3 + Command/Move.hs | 4 ++ Command/PreCommit.hs | 3 + Command/SetKey.hs | 4 ++ Command/Trust.hs | 4 ++ Command/Unannex.hs | 3 + Command/Uninit.hs | 4 ++ Command/Unlock.hs | 6 ++ Command/Untrust.hs | 4 ++ Command/Unused.hs | 3 + git-annex.hs | 74 ++++++++++++++++++- 24 files changed, 204 insertions(+), 136 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 40ce4b1215..54c2289c61 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -1,11 +1,16 @@ -{- git-annex command line +{- git-annex command line parsing - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module CmdLine (parseCmd) where +module CmdLine ( + parseCmd, + Option, + storeOptBool, + storeOptString, +) where import System.Console.GetOpt import Control.Monad (when) @@ -13,116 +18,41 @@ import Control.Monad.State (liftIO) import qualified Annex import Types - import Command -import qualified Command.Add -import qualified Command.Unannex -import qualified Command.Drop -import qualified Command.Move -import qualified Command.Copy -import qualified Command.Get -import qualified Command.FromKey -import qualified Command.DropKey -import qualified Command.SetKey -import qualified Command.Fix -import qualified Command.Init -import qualified Command.Fsck -import qualified Command.Unused -import qualified Command.DropUnused -import qualified Command.Unlock -import qualified Command.Lock -import qualified Command.PreCommit -import qualified Command.Find -import qualified Command.Uninit -import qualified Command.Trust -import qualified Command.Untrust -cmds :: [Command] -cmds = - [ Command.Add.command - , Command "get" path Command.Get.seek - "make content of annexed files available" - , Command "drop" path Command.Drop.seek - "indicate content of files not currently wanted" - , Command "move" path Command.Move.seek - "move content of files to/from another repository" - , Command "copy" path Command.Copy.seek - "copy content of files to/from another repository" - , Command "unlock" path Command.Unlock.seek - "unlock files for modification" - , Command "edit" path Command.Unlock.seek - "same as unlock" - , Command "lock" path Command.Lock.seek - "undo unlock command" - , Command "init" desc Command.Init.seek - "initialize git-annex with repository description" - , Command "unannex" path Command.Unannex.seek - "undo accidential add command" - , Command "uninit" path Command.Uninit.seek - "de-initialize git-annex and clean out repository" - , Command "pre-commit" path Command.PreCommit.seek - "run by git pre-commit hook" - , Command "trust" remote Command.Trust.seek - "trust a repository" - , Command "untrust" remote Command.Untrust.seek - "do not trust a repository" - , Command "fromkey" key Command.FromKey.seek - "adds a file using a specific key" - , Command "dropkey" key Command.DropKey.seek - "drops annexed content for specified keys" - , Command "setkey" key Command.SetKey.seek - "sets annexed content for a key using a temp file" - , Command "fix" path Command.Fix.seek - "fix up symlinks to point to annexed content" - , Command "fsck" maybepath Command.Fsck.seek - "check for problems" - , Command "unused" nothing Command.Unused.seek - "look for unused file content" - , Command "dropunused" number Command.DropUnused.seek - "drop unused file content" - , Command "find" maybepath Command.Find.seek - "lists available files" - ] +{- Each dashed command-line option results in generation of an action + - in the Annex monad that performs the necessary setting. + -} +type Option = OptDescr (Annex ()) + +storeOptBool :: FlagName -> Bool -> Annex () +storeOptBool name val = Annex.flagChange name $ FlagBool val +storeOptString :: FlagName -> String -> Annex () +storeOptString name val = Annex.flagChange name $ FlagString val + +{- Parses command line, stores configure flags, and returns a + - list of actions to be run in the Annex monad. -} +parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool] +parseCmd argv header cmds options = do + (flags, params) <- liftIO $ getopt + when (null params) $ error usagemsg + case lookupCmd (head params) of + [] -> error usagemsg + [command] -> do + _ <- sequence flags + prepCmd command (drop 1 params) + _ -> error "internal error: multiple matching commands" where - path = "PATH ..." - maybepath = "[PATH ...]" - key = "KEY ..." - desc = "DESCRIPTION" - number = "NUMBER ..." - remote = "REMOTE ..." - nothing = "" + getopt = case getOpt Permute options argv of + (flags, params, []) -> return (flags, params) + (_, _, errs) -> ioError (userError (concat errs ++ usagemsg)) + lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds + usagemsg = usage header cmds options --- Each dashed command-line option results in generation of an action --- in the Annex monad that performs the necessary setting. -options :: [OptDescr (Annex ())] -options = [ - Option ['f'] ["force"] (NoArg (storebool "force" True)) - "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storebool "quiet" True)) - "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storebool "quiet" False)) - "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storestring "backend") "NAME") - "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storestring "key") "KEY") - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storestring "torepository") "REPOSITORY") - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storestring "fromrepository") "REPOSITORY") - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storestring "exclude") "GLOB") - "skip files matching the glob pattern" - ] - where - storebool n b = Annex.flagChange n $ FlagBool b - storestring n s = Annex.flagChange n $ FlagString s - -header :: String -header = "Usage: git-annex subcommand [option ..]" - -{- Usage message with lists of options and subcommands. -} -usage :: String -usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs +{- Usage message with lists of commands and options. -} +usage :: String -> [Command] -> [Option] -> String +usage header cmds options = + usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs where cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = @@ -133,21 +63,3 @@ usage = usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' - -{- Parses command line, stores configure flags, and returns a - - list of actions to be run in the Annex monad. -} -parseCmd :: [String] -> Annex [Annex Bool] -parseCmd argv = do - (flags, params) <- liftIO $ getopt - when (null params) $ error usage - case lookupCmd (head params) of - [] -> error usage - [command] -> do - _ <- sequence flags - prepCmd command (drop 1 params) - _ -> error "internal error: multiple matching commands" - where - getopt = case getOpt Permute options argv of - (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usage)) - lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds diff --git a/Command.hs b/Command.hs index 2144da353b..690dd20ecf 100644 --- a/Command.hs +++ b/Command.hs @@ -205,18 +205,24 @@ notSymlink f = do s <- liftIO $ getSymbolicLinkStatus f return $ not $ isSymbolicLink s -{- descriptions of params used in usage message -} +{- Descriptions of params used in usage messages. -} +paramRepeating :: String -> String +paramRepeating s = s ++ " ..." +paramOptional :: String -> String +paramOptional s = "[" ++ s ++ "]" paramPath :: String -paramPath = "PATH ..." -paramMaybePath :: String -paramMaybePath = "[PATH ...]" +paramPath = "PATH" paramKey :: String -paramKey = "KEY ..." +paramKey = "KEY" paramDesc :: String paramDesc = "DESCRIPTION" paramNumber :: String -paramNumber = "NUMBER ..." +paramNumber = "NUMBER" paramRemote :: String -paramRemote = "REMOTE ..." +paramRemote = "REMOTE" +paramGlob :: String +paramGlob = "GLOB" +paramName :: String +paramName = "NAME" paramNothing :: String paramNothing = "" diff --git a/Command/Add.hs b/Command/Add.hs index 08a880206b..bc869a67de 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,8 +18,8 @@ import Types import Core import Messages -command :: Command -command = Command "add" paramPath seek "add files to annex" +command :: [Command] +command = [Command "add" paramPath seek "add files to annex"] {- Add acts on both files not checked into git yet, and unlocked files. -} seek :: [CommandSeek] diff --git a/Command/Copy.hs b/Command/Copy.hs index 873df7ef2e..93342e11bb 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -10,6 +10,10 @@ module Command.Copy where import Command import qualified Command.Move +command :: [Command] +command = [Command "copy" paramPath seek + "copy content of files to/from another repository"] + -- A copy is just a move that does not delete the source file. seek :: [CommandSeek] seek = [withFilesInGit $ Command.Move.start False] diff --git a/Command/Drop.hs b/Command/Drop.hs index 3f27405703..a425c6138d 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -17,6 +17,10 @@ import Core import Messages import Utility +command :: [Command] +command = [Command "drop" paramPath seek + "indicate content of files not currently wanted"] + seek :: [CommandSeek] seek = [withAttrFilesInGit "annex.numcopies" start] diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 870e9a7ab1..29056139d3 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -15,6 +15,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "dropkey" (paramRepeating paramKey) seek + "drops annexed content for specified keys"] + seek :: [CommandSeek] seek = [withKeys start] diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 9984e49f3f..ea2ff46eba 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -18,6 +18,10 @@ import qualified Annex import qualified Command.Drop import Backend +command :: [Command] +command = [Command "dropunused" (paramRepeating paramNumber) seek + "drop unused file content"] + seek :: [CommandSeek] seek = [withStrings start] diff --git a/Command/Find.hs b/Command/Find.hs index 9927b692d8..7cb781ce8c 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -13,6 +13,10 @@ import Control.Monad.State (liftIO) import Command import Core +command :: [Command] +command = [Command "find" (paramOptional $ paramRepeating paramPath) seek + "lists available files"] + seek :: [CommandSeek] seek = [withDefault "." withFilesInGit start] diff --git a/Command/Fix.hs b/Command/Fix.hs index accdadd315..8b08a26f6d 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -17,6 +17,10 @@ import Utility import Core import Messages +command :: [Command] +command = [Command "fix" paramPath seek + "fix up symlinks to point to annexed content"] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 991428136e..f1cb717fac 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -20,6 +20,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "fromkey" (paramRepeating paramKey) seek + "adds a file using a specific key"] + seek :: [CommandSeek] seek = [withFilesMissing start] diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 034bdc388b..d870bd4198 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -13,6 +13,10 @@ import Types import Messages import Utility +command :: [Command] +command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek + "check for problems"] + seek :: [CommandSeek] seek = [withAll (withAttrFilesInGit "annex.numcopies") start] diff --git a/Command/Get.hs b/Command/Get.hs index 214b689b8a..e3668649ef 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -13,6 +13,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "get" paramPath seek + "make content of annexed files available"] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/Init.hs b/Command/Init.hs index 806c34c989..8ad9f79d70 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -19,6 +19,10 @@ import Version import Messages import Locations import Types + +command :: [Command] +command = [Command "init" paramDesc seek + "initialize git-annex with repository description"] seek :: [CommandSeek] seek = [withString start] diff --git a/Command/Lock.hs b/Command/Lock.hs index 381162536e..00a553e956 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -14,6 +14,9 @@ import Command import Messages import qualified Annex import qualified GitRepo as Git + +command :: [Command] +command = [Command "lock" paramPath seek "undo unlock command"] seek :: [CommandSeek] seek = [withFilesUnlocked start] diff --git a/Command/Move.hs b/Command/Move.hs index 8ba8dbfacc..addeeae8a9 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,6 +20,10 @@ import qualified GitRepo as Git import qualified Remotes import UUID import Messages + +command :: [Command] +command = [Command "move" paramPath seek + "move content of files to/from another repository"] seek :: [CommandSeek] seek = [withFilesInGit $ start True] diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 8d488514a8..12e5ed806d 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -15,6 +15,9 @@ import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix +command :: [Command] +command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"] + {- The pre-commit hook needs to fix symlinks to all files being committed. - And, it needs to inject unlocked files into the annex. -} seek :: [CommandSeek] diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 4c82de3a5b..5048d052f0 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -19,6 +19,10 @@ import Types import Core import Messages +command :: [Command] +command = [Command "setkey" (paramRepeating paramKey) seek + "sets annexed content for a key using a temp file"] + seek :: [CommandSeek] seek = [withTempFile start] diff --git a/Command/Trust.hs b/Command/Trust.hs index 3c3ec3b7e5..35ddefe842 100644 --- a/Command/Trust.hs +++ b/Command/Trust.hs @@ -17,6 +17,10 @@ import qualified Remotes import UUID import Messages +command :: [Command] +command = [Command "trust" (paramRepeating paramRemote) seek + "trust a repository"] + seek :: [CommandSeek] seek = [withString start] diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 42354b8c49..288f9da44a 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -20,6 +20,9 @@ import Core import qualified GitRepo as Git import Messages +command :: [Command] +command = [Command "unannex" paramPath seek "undo accidential add command"] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/Uninit.hs b/Command/Uninit.hs index 6001c55cd9..1a4e9b0d71 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -20,6 +20,10 @@ import qualified Annex import qualified Command.Unannex import qualified Command.Init +command :: [Command] +command = [Command "uninit" paramPath seek + "de-initialize git-annex and clean out repository"] + seek :: [CommandSeek] seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 21f34d1dba..0e55585ae3 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -18,6 +18,12 @@ import Locations import Core import CopyFile +command :: [Command] +command = + [ Command "unlock" paramPath seek "unlock files for modification" + , Command "edit" paramPath seek "same as unlock" + ] + seek :: [CommandSeek] seek = [withFilesInGit start] diff --git a/Command/Untrust.hs b/Command/Untrust.hs index 6458040b3f..f49a2e989a 100644 --- a/Command/Untrust.hs +++ b/Command/Untrust.hs @@ -17,6 +17,10 @@ import qualified Remotes import UUID import Messages +command :: [Command] +command = [Command "untrust" (paramRepeating paramRemote) seek + "do not trust a repository"] + seek :: [CommandSeek] seek = [withString start] diff --git a/Command/Unused.hs b/Command/Unused.hs index dba9aa517a..d2dfc9aa3e 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -17,6 +17,9 @@ import Messages import Locations import qualified Annex +command :: [Command] +command = [Command "unused" paramNothing seek "look for unused file content"] + seek :: [CommandSeek] seek = [withNothing start] diff --git a/git-annex.hs b/git-annex.hs index 1173ab9139..31d90e4fc3 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,6 +6,7 @@ -} import System.Environment +import System.Console.GetOpt import qualified Annex import Core @@ -14,10 +15,81 @@ import CmdLine import qualified GitRepo as Git import BackendList +import Command +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey +import qualified Command.Fix +import qualified Command.Init +import qualified Command.Fsck +import qualified Command.Unused +import qualified Command.DropUnused +import qualified Command.Unlock +import qualified Command.Lock +import qualified Command.PreCommit +import qualified Command.Find +import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust + +cmds :: [Command] +cmds = concat + [ Command.Add.command + , Command.Get.command + , Command.Drop.command + , Command.Move.command + , Command.Copy.command + , Command.Unlock.command + , Command.Lock.command + , Command.Init.command + , Command.Unannex.command + , Command.Uninit.command + , Command.PreCommit.command + , Command.Trust.command + , Command.Untrust.command + , Command.FromKey.command + , Command.DropKey.command + , Command.SetKey.command + , Command.Fix.command + , Command.Fsck.command + , Command.Unused.command + , Command.DropUnused.command + , Command.Find.command + ] + +options :: [Option] +options = [ + Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) + "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + "allow verbose output" + , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + "specify default key-value backend to use" + , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] + +header :: String +header = "Usage: git-annex subcommand [option ..]" + main :: IO () main = do args <- getArgs gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends - (actions, state') <- Annex.run state $ parseCmd args + (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions From c274aadabcd1737e03003834dcf1f6bd9a79b0aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:12:55 -0400 Subject: [PATCH 0649/8313] remove magic numbers --- CmdLine.hs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 54c2289c61..3767fc2406 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -57,9 +57,12 @@ usage header cmds options = cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = cmdname c ++ - pad 11 (cmdname c) ++ + pad (commandlen + 1) (cmdname c) ++ cmdparams c ++ - pad 13 (cmdparams c) ++ + pad (commandparamlen + 2) (cmdparams c) ++ cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' + longest l = foldl max 0 $ map length l + commandlen = longest $ map cmdname cmds + commandparamlen = longest $ map cmdparams cmds From 1c451fe3628f535898e7cf87ccad30270c6d16fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:15:22 -0400 Subject: [PATCH 0650/8313] tweak --- CmdLine.hs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 3767fc2406..a2645f75f5 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -57,12 +57,10 @@ usage header cmds options = cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = cmdname c ++ - pad (commandlen + 1) (cmdname c) ++ + pad (longest cmdname + 1) (cmdname c) ++ cmdparams c ++ - pad (commandparamlen + 2) (cmdparams c) ++ + pad (longest cmdparams + 2) (cmdparams c) ++ cmddesc c indent l = " " ++ l pad n s = replicate (n - length s) ' ' - longest l = foldl max 0 $ map length l - commandlen = longest $ map cmdname cmds - commandparamlen = longest $ map cmdparams cmds + longest f = foldl max 0 $ map (length . f) cmds From 88ff9e82fc3dcb653b2a116f1c162d98a1f6bdcf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 15:44:15 -0400 Subject: [PATCH 0651/8313] factor out a little more --- CmdLine.hs | 15 +++++++++++++++ Core.hs | 2 +- git-annex.hs | 18 ++---------------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index a2645f75f5..b3dfc984df 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -6,19 +6,25 @@ -} module CmdLine ( + cmdLine, parseCmd, Option, storeOptBool, storeOptString, ) where +import System.Environment import System.Console.GetOpt import Control.Monad (when) import Control.Monad.State (liftIO) import qualified Annex +import qualified GitRepo as Git import Types import Command +import BackendList +import Core +import Upgrade {- Each dashed command-line option results in generation of an action - in the Annex monad that performs the necessary setting. @@ -30,6 +36,15 @@ storeOptBool name val = Annex.flagChange name $ FlagBool val storeOptString :: FlagName -> String -> Annex () storeOptString name val = Annex.flagChange name $ FlagString val +{- It all starts here. -} +cmdLine :: [Command] -> [Option] -> String -> IO () +cmdLine cmds options header = do + args <- getArgs + gitrepo <- Git.repoFromCwd + state <- Annex.new gitrepo allBackends + (actions, state') <- Annex.run state $ parseCmd args header cmds options + tryRun state' $ [startup, upgrade] ++ actions + {- Parses command line, stores configure flags, and returns a - list of actions to be run in the Annex monad. -} parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool] diff --git a/Core.hs b/Core.hs index d91595a049..08e2265920 100644 --- a/Core.hs +++ b/Core.hs @@ -45,7 +45,7 @@ tryRun' state errnum (a:as) = do tryRun' state errnum [] = do _ <- try $ Annex.run state $ shutdown errnum when (errnum > 0) $ error $ show errnum ++ " failed" - + {- Actions to perform each time ran. -} startup :: Annex Bool startup = do diff --git a/git-annex.hs b/git-annex.hs index 31d90e4fc3..b8176befaa 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,17 +5,11 @@ - Licensed under the GNU GPL version 3 or higher. -} -import System.Environment import System.Console.GetOpt -import qualified Annex -import Core -import Upgrade import CmdLine -import qualified GitRepo as Git -import BackendList - import Command + import qualified Command.Add import qualified Command.Unannex import qualified Command.Drop @@ -83,13 +77,5 @@ options = [ "skip files matching the glob pattern" ] -header :: String -header = "Usage: git-annex subcommand [option ..]" - main :: IO () -main = do - args <- getArgs - gitrepo <- Git.repoFromCwd - state <- Annex.new gitrepo allBackends - (actions, state') <- Annex.run state $ parseCmd args header cmds options - tryRun state' $ [startup, upgrade] ++ actions +main = cmdLine cmds options "Usage: git-annex subcommand [option ..]" From 7a52b34e0631609d5d862c3ba100cc499b30b5fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 16:52:24 -0400 Subject: [PATCH 0652/8313] add git-annex-shell command This is not yet complete, as it does not allow starting rsync or scp. --- .gitignore | 2 ++ CmdLine.hs | 36 +++++++++-------------- Command/FromKey.hs | 2 +- Makefile | 23 +++++++++------ Options.hs | 44 ++++++++++++++++++++++++++++ doc/git-annex-shell.mdwn | 63 ++++++++++++++++++++++++++++++++++++++++ doc/git-annex.mdwn | 10 +++---- git-annex-shell.hs | 52 +++++++++++++++++++++++++++++++++ git-annex.hs | 27 ++++------------- 9 files changed, 200 insertions(+), 59 deletions(-) create mode 100644 Options.hs create mode 100644 doc/git-annex-shell.mdwn create mode 100644 git-annex-shell.hs diff --git a/.gitignore b/.gitignore index a4cac10f43..d2f4c2b743 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ test configure SysConfig.hs git-annex +git-annex-shell git-annex.1 +git-annex-shell.1 doc/.ikiwiki html diff --git a/CmdLine.hs b/CmdLine.hs index b3dfc984df..34cc22656a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -6,14 +6,13 @@ -} module CmdLine ( - cmdLine, + dispatch, parseCmd, Option, storeOptBool, storeOptString, ) where -import System.Environment import System.Console.GetOpt import Control.Monad (when) import Control.Monad.State (liftIO) @@ -25,21 +24,11 @@ import Command import BackendList import Core import Upgrade +import Options -{- Each dashed command-line option results in generation of an action - - in the Annex monad that performs the necessary setting. - -} -type Option = OptDescr (Annex ()) - -storeOptBool :: FlagName -> Bool -> Annex () -storeOptBool name val = Annex.flagChange name $ FlagBool val -storeOptString :: FlagName -> String -> Annex () -storeOptString name val = Annex.flagChange name $ FlagString val - -{- It all starts here. -} -cmdLine :: [Command] -> [Option] -> String -> IO () -cmdLine cmds options header = do - args <- getArgs +{- Runs the passed command line. -} +dispatch :: [String] -> [Command] -> [Option] -> String -> IO () +dispatch args cmds options header = do gitrepo <- Git.repoFromCwd state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options @@ -50,24 +39,27 @@ cmdLine cmds options header = do parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool] parseCmd argv header cmds options = do (flags, params) <- liftIO $ getopt - when (null params) $ error usagemsg + when (null params) $ error $ "missing command" ++ usagemsg case lookupCmd (head params) of - [] -> error usagemsg + [] -> error $ "unknown command" ++ usagemsg [command] -> do _ <- sequence flags prepCmd command (drop 1 params) _ -> error "internal error: multiple matching commands" where getopt = case getOpt Permute options argv of - (flags, params, []) -> return (flags, params) - (_, _, errs) -> ioError (userError (concat errs ++ usagemsg)) + (flags, params, []) -> + return (flags, params) + (_, _, errs) -> + ioError (userError (concat errs ++ usagemsg)) lookupCmd cmd = filter (\c -> cmd == cmdname c) cmds - usagemsg = usage header cmds options + usagemsg = "\n\n" ++ usage header cmds options {- Usage message with lists of commands and options. -} usage :: String -> [Command] -> [Option] -> String usage header cmds options = - usageInfo header options ++ "\nSubcommands:\n" ++ cmddescs + usageInfo (header ++ "\n\nOptions:") options ++ + "\nCommands:\n" ++ cmddescs where cmddescs = unlines $ map (indent . showcmd) cmds showcmd c = diff --git a/Command/FromKey.hs b/Command/FromKey.hs index f1cb717fac..0a13b8c734 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -21,7 +21,7 @@ import Core import Messages command :: [Command] -command = [Command "fromkey" (paramRepeating paramKey) seek +command = [Command "fromkey" paramPath seek "adds a file using a specific key"] seek :: [CommandSeek] diff --git a/Makefile b/Makefile index c338427df4..2f1fd05b91 100644 --- a/Makefile +++ b/Makefile @@ -2,23 +2,28 @@ PREFIX=/usr GHCFLAGS=-O2 -Wall GHCMAKE=ghc -odir build -hidir build $(GHCFLAGS) --make -all: git-annex git-annex.1 docs +bins=git-annex git-annex-shell +mans=git-annex.1 git-annex-shell.1 + +all: $(bins) $(mans) docs SysConfig.hs: configure.hs $(GHCMAKE) configure ./configure +$(bins): SysConfig.hs + $(GHCMAKE) $@ + git-annex.1: ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 - -git-annex: SysConfig.hs - $(GHCMAKE) git-annex +git-annex-shell.1: + ./mdwn2man git-annex 1 doc/git-annex-shell.mdwn > git-annex-shell.1 install: all install -d $(DESTDIR)$(PREFIX)/bin - install git-annex $(DESTDIR)$(PREFIX)/bin + install $(bins) $(DESTDIR)$(PREFIX)/bin install -d $(DESTDIR)$(PREFIX)/share/man/man1 - install -m 0644 git-annex.1 $(DESTDIR)$(PREFIX)/share/man/man1 + install -m 0644 $(mans) $(DESTDIR)$(PREFIX)/share/man/man1 install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex if [ -d html ]; then \ rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ @@ -36,7 +41,7 @@ else IKIWIKI=ikiwiki endif -docs: git-annex.1 +docs: $(mans) $(IKIWIKI) doc html -v --wikiname git-annex --plugin=goodstuff \ --no-usedirs --disable-plugin=openid --plugin=sidebar \ --underlaydir=/dev/null --disable-plugin=shortcut \ @@ -44,7 +49,7 @@ docs: git-annex.1 --exclude='news/.*' clean: - rm -rf build git-annex git-annex.1 test configure SysConfig.hs + rm -rf build $(bins) $(mans) test configure SysConfig.hs rm -rf doc/.ikiwiki html -.PHONY: git-annex test install +.PHONY: $(bins) test install diff --git a/Options.hs b/Options.hs new file mode 100644 index 0000000000..684aed97d6 --- /dev/null +++ b/Options.hs @@ -0,0 +1,44 @@ +{- git-annex dashed options + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Options where + +import System.Console.GetOpt + +import qualified Annex +import Types +import Command + +{- Each dashed command-line option results in generation of an action + - in the Annex monad that performs the necessary setting. + -} +type Option = OptDescr (Annex ()) + +storeOptBool :: FlagName -> Bool -> Annex () +storeOptBool name val = Annex.flagChange name $ FlagBool val +storeOptString :: FlagName -> String -> Annex () +storeOptString name val = Annex.flagChange name $ FlagString val + +commonOptions :: [Option] +commonOptions = [ + Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) + "allow actions that may lose annexed data" + , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + "avoid verbose output" + , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + "allow verbose output" + , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + "specify default key-value backend to use" + , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn new file mode 100644 index 0000000000..34d9c8afef --- /dev/null +++ b/doc/git-annex-shell.mdwn @@ -0,0 +1,63 @@ +# NAME + +git-annex-shell - Restricted login shell for git-annex only SSH access + +# SYNOPSIS + +git-annex-shell -c command [params ...] + +# DESCRIPTION + +git-annex-shell is a restricted shell, similar to git-shell, which +can be used as a login shell for SSH accounts you want to restrict. + +# COMMANDS + +* git-annex fromkey file + + This can be used to maually set up a file to link to a specified key + in the key-value backend. How you determine an existing key in the backend + varies. For the URL backend, the key is just a URL to the content. + + Example: + + git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + +* git-annex dropkey [key ...] + + This drops the annexed data for the specified + keys from this repository. + + This can be used to drop content for arbitrary keys, which do not need + to have a file in the git repository pointing at them. + + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. + +* git-annex setkey file + + This sets the annxed data for a key to the content of + the specified file, and then removes the file. + + A backend will typically need to be specified with --backend. If none + is specified, the first configured backend is used. + +Any other command is passed through to git-shell. + +# OPTIONS + +Same as git-annex or git-shell, depending on the command being run. + +# SEE ALSO + +[[git-annex]](1) + +git-shell(1) + +# AUTHOR + +Joey Hess + + + +Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD). Edit with care diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index caef49d977..8e6ff2c0cc 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -4,7 +4,7 @@ git-annex - manage files with git, without checking their contents in # SYNOPSIS -git annex subcommand [params ...] +git annex command [params ...] # DESCRIPTION @@ -55,13 +55,13 @@ content from the key-value store. # git annex move iso --to=usbdrive move iso/Debian_5.0.iso (moving to usbdrive...) ok -# SUBCOMMANDS +# COMMANDS Like many git commands, git-annex can be passed a path that is either a file or a directory. In the latter case it acts on all relevant files in the directory. -Many git-annex subcommands will stage changes for later `git commit` by you. +Many git-annex commands will stage changes for later `git commit` by you. * add [path ...] @@ -91,7 +91,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * edit [path ...] - This is an alias for the unlock subcommand. May be easier to remember, + This is an alias for the unlock command. May be easier to remember, if you think of this as allowing you to edit an annexed file. * move [path ...] @@ -122,7 +122,7 @@ Many git-annex subcommands will stage changes for later `git commit` by you. * fsck [path ...] - With no parameters, this subcommand checks the whole annex for consistency, + With no parameters, this command checks the whole annex for consistency, and warns about any problems found. With parameters, only the specified files are checked. diff --git a/git-annex-shell.hs b/git-annex-shell.hs new file mode 100644 index 0000000000..7adb5e7905 --- /dev/null +++ b/git-annex-shell.hs @@ -0,0 +1,52 @@ +{- git-annex-shell main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +import System.Console.GetOpt +import System.Environment +import Control.Monad (when) + +import CmdLine +import Command +import Utility +import Options + +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey + +cmds :: [Command] +cmds = concat + [ Command.FromKey.command + , Command.DropKey.command + , Command.SetKey.command + ] + +options :: [Option] +options = [ Option ['c'] ["command"] (NoArg (storeOptBool "command" True)) + "ignored for compatability with git-shell" + ] ++ commonOptions + +header :: String +header = "Usage:\n" ++ + "\tgit-annex-shell -c git-annex command [option ..]\n" ++ + "\tgit-annex-shell -c shellcommand argument" + +main :: IO () +main = do + args <- getArgs + -- dispatch git-annex commands to builtin versions, + -- and pass everything else to git-shell + case args of + ("git-annex":as) -> builtin as + [] -> builtin [] + _ -> external args + where + builtin l = dispatch l cmds options header + external l = do + ret <- boolSystem "git-shell" l + when (not ret) $ + error "git-shell failed" diff --git a/git-annex.hs b/git-annex.hs index b8176befaa..6c143972a1 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -5,10 +5,11 @@ - Licensed under the GNU GPL version 3 or higher. -} -import System.Console.GetOpt +import System.Environment import CmdLine import Command +import Options import qualified Command.Add import qualified Command.Unannex @@ -57,25 +58,7 @@ cmds = concat , Command.Find.command ] -options :: [Option] -options = [ - Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) - "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) - "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) - "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) - "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) - "skip files matching the glob pattern" - ] - main :: IO () -main = cmdLine cmds options "Usage: git-annex subcommand [option ..]" +main = do + args <- getArgs + dispatch args cmds commonOptions "Usage: git-annex command [option ..]" From a5a302b77d816b189ae5ae55f03b18d2cf6ef45b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 20:08:22 -0400 Subject: [PATCH 0653/8313] git-annex-shell mostly done now, only needs 2 more subcommands --- CmdLine.hs | 9 ++--- Command/ConfigList.hs | 27 +++++++++++++++ Command/InAnnex.hs | 32 ++++++++++++++++++ doc/git-annex-shell.mdwn | 38 +++++++++------------ git-annex-shell.hs | 72 +++++++++++++++++++++++++--------------- git-annex.hs | 7 +++- 6 files changed, 130 insertions(+), 55 deletions(-) create mode 100644 Command/ConfigList.hs create mode 100644 Command/InAnnex.hs diff --git a/CmdLine.hs b/CmdLine.hs index 34cc22656a..fbcfb6405d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -8,9 +8,7 @@ module CmdLine ( dispatch, parseCmd, - Option, - storeOptBool, - storeOptString, + usage, ) where import System.Console.GetOpt @@ -27,9 +25,8 @@ import Upgrade import Options {- Runs the passed command line. -} -dispatch :: [String] -> [Command] -> [Option] -> String -> IO () -dispatch args cmds options header = do - gitrepo <- Git.repoFromCwd +dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () +dispatch gitrepo args cmds options header = do state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs new file mode 100644 index 0000000000..0d9d789b54 --- /dev/null +++ b/Command/ConfigList.hs @@ -0,0 +1,27 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.ConfigList where + +import Control.Monad.State (liftIO) + +import Annex +import Command +import qualified GitRepo as Git + +command :: [Command] +command = [Command "configlist" paramNothing seek + "outputs relevant git configuration"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStartNothing +start = do + g <- Annex.gitRepo + liftIO $ Git.run g ["config", "--list"] + return Nothing diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs new file mode 100644 index 0000000000..d49539513b --- /dev/null +++ b/Command/InAnnex.hs @@ -0,0 +1,32 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.InAnnex where + +import Control.Monad.State (liftIO) +import System.Exit + +import Command +import Types +import Core +import qualified Backend + +command :: [Command] +command = [Command "inannex" (paramRepeating paramKey) seek + "checks if keys are present in the annex"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: CommandStartString +start keyname = do + backends <- Backend.list + let key = genKey (head backends) keyname + present <- inAnnex key + if present + then return Nothing + else liftIO $ exitFailure diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 34d9c8afef..9f51b6813d 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -4,43 +4,37 @@ git-annex-shell - Restricted login shell for git-annex only SSH access # SYNOPSIS -git-annex-shell -c command [params ...] +git-annex-shell [-c] command [params ...] # DESCRIPTION -git-annex-shell is a restricted shell, similar to git-shell, which -can be used as a login shell for SSH accounts you want to restrict. +git-annex-shell is a restricted shell, similar to git-shell, which +can be used as a login shell for SSH accounts. # COMMANDS -* git-annex fromkey file +* configlist directory - This can be used to maually set up a file to link to a specified key - in the key-value backend. How you determine an existing key in the backend - varies. For the URL backend, the key is just a URL to the content. + This outputs the git configuration, in the same form as + `git config --list` - Example: +* inannex directory [key ...] - git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + This checks if all specified keys are present in the annex, + and exits zero if so. -* git-annex dropkey [key ...] +* dropkey directory [key ...] - This drops the annexed data for the specified - keys from this repository. + This drops the annexed data for the specified keys. - This can be used to drop content for arbitrary keys, which do not need - to have a file in the git repository pointing at them. +* recvkey directory key - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. + This runs rsync in server mode to receive the content of a key, + and stores the content in the annex. -* git-annex setkey file +* sendkey directory key - This sets the annxed data for a key to the content of - the specified file, and then removes the file. - - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. + This runs rsync in server mode to transfer out the content of a key. Any other command is passed through to git-shell. diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 7adb5e7905..492d184469 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -5,48 +5,68 @@ - Licensed under the GNU GPL version 3 or higher. -} -import System.Console.GetOpt import System.Environment import Control.Monad (when) +import qualified GitRepo as Git import CmdLine import Command import Utility import Options -import qualified Command.FromKey +import qualified Command.ConfigList +import qualified Command.InAnnex import qualified Command.DropKey -import qualified Command.SetKey +--import qualified Command.RecvKey +--import qualified Command.SendKey cmds :: [Command] -cmds = concat - [ Command.FromKey.command +cmds = map adddirparam $ concat + [ Command.ConfigList.command + , Command.InAnnex.command , Command.DropKey.command - , Command.SetKey.command +-- , Command.RecvKey.command +-- , Command.SendKey.command ] - -options :: [Option] -options = [ Option ['c'] ["command"] (NoArg (storeOptBool "command" True)) - "ignored for compatability with git-shell" - ] ++ commonOptions + where + adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c } header :: String -header = "Usage:\n" ++ - "\tgit-annex-shell -c git-annex command [option ..]\n" ++ - "\tgit-annex-shell -c shellcommand argument" +header = "Usage: git-annex-shell [-c] command [option ..]" main :: IO () main = do args <- getArgs - -- dispatch git-annex commands to builtin versions, - -- and pass everything else to git-shell - case args of - ("git-annex":as) -> builtin as - [] -> builtin [] - _ -> external args - where - builtin l = dispatch l cmds options header - external l = do - ret <- boolSystem "git-shell" l - when (not ret) $ - error "git-shell failed" + main' args + +main' :: [String] -> IO () +main' [] = failure +-- skip leading -c options, passed by eg, ssh +main' ("-c":p) = main' p +-- Since git-annex explicitly runs git-annex-shell, we will be passed +-- a redundant "git-annex-shell" parameter when we're the user's login shell. +main' ("git-annex-shell":p) = main' p +-- a command can be either a builtin or something to pass to git-shell +main' c@(cmd:dir:params) + | elem cmd builtins = builtin cmd dir params + | otherwise = external c +main' c@(cmd:_) + | elem cmd builtins = failure + | otherwise = external c + +builtins :: [String] +builtins = map cmdname cmds + +builtin :: String -> String -> [String] -> IO () +builtin cmd dir params = do + let gitrepo = Git.repoFromPath dir + dispatch gitrepo (cmd:params) cmds commonOptions header + +external :: [String] -> IO () +external l = do + ret <- boolSystem "git-shell" ("-c":l) + when (not ret) $ + error "git-shell failed" + +failure :: IO () +failure = error $ "bad parameters\n\n" ++ usage header cmds commonOptions diff --git a/git-annex.hs b/git-annex.hs index 6c143972a1..110054fd5a 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -7,6 +7,7 @@ import System.Environment +import qualified GitRepo as Git import CmdLine import Command import Options @@ -58,7 +59,11 @@ cmds = concat , Command.Find.command ] +header :: String +header = "Usage: git-annex command [option ..]" + main :: IO () main = do args <- getArgs - dispatch args cmds commonOptions "Usage: git-annex command [option ..]" + gitrepo <- Git.repoFromCwd + dispatch gitrepo args cmds commonOptions header From f38aa3e83abb251a88362dbaf6e8fbddd477fa53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 30 Dec 2010 20:31:52 -0400 Subject: [PATCH 0654/8313] unfinished switch to using git-annex-shell --- Command/Move.hs | 4 ++-- Remotes.hs | 40 +++++++++++++++++++--------------------- configure.hs | 2 +- doc/install.mdwn | 2 +- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index addeeae8a9..e872d86fe7 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -134,8 +134,8 @@ fromPerform move key = do else return Nothing -- fail fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do - ok <- Remotes.runCmd remote "git-annex" - ["dropkey", "--quiet", "--force", + ok <- Remotes.onRemote remote "dropkey" + ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] remoteHasKey remote key False diff --git a/Remotes.hs b/Remotes.hs index 78ab010cef..a775f71d4b 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -15,7 +15,8 @@ module Remotes ( byName, copyFromRemote, copyToRemote, - runCmd + runCmd, + onRemote ) where import Control.Exception.Extensible @@ -37,7 +38,6 @@ import Utility import qualified Core import Messages import CopyFile -import qualified SysConfig {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -118,7 +118,8 @@ inAnnex r key = if Git.repoIsUrl r Annex.eval a (Core.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- runCmd r "test" ["-e", annexLocation r key] + inannex <- onRemote r "inannex" + ["--backend=" ++ backendName key, keyName key] -- XXX Note that ssh failing and the file not existing -- are not currently differentiated. return $ Right inannex @@ -231,7 +232,7 @@ copyFromRemote r key file where keyloc = annexLocation r key getlocal = liftIO $ copyFile keyloc file - getssh = remoteCopyFile r (sshLocation r keyloc) file + getssh = remoteCopyFile True r (sshLocation r keyloc) file {- Tries to copy a key's content to a file on a remote. -} copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -245,35 +246,32 @@ copyToRemote r key file = do else error "copying to non-ssh repo not supported" where putlocal src = liftIO $ copyFile src file - putssh src = remoteCopyFile r src (sshLocation r file) + putssh src = remoteCopyFile False r src (sshLocation r file) sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file -{- Copys a file from or to a remote, using rsync (when available) or scp. -} -remoteCopyFile :: Git.Repo -> String -> String -> Annex Bool -remoteCopyFile r src dest = do +{- Copies a file from or to a remote, using rsync (when available) or scp. -} +remoteCopyFile :: Bool -> Git.Repo -> String -> String -> Annex Bool +remoteCopyFile recv r src dest = do showProgress -- make way for progress bar o <- repoConfig r configopt "" res <- liftIO $ boolSystem cmd $ options ++ words o ++ [src, dest] if res then return res else do - when rsync $ - showLongNote "rsync failed -- run git annex again to resume file transfer" + showLongNote "rsync failed -- run git annex again to resume file transfer" return res where - cmd - | rsync = "rsync" - | otherwise = "scp" - configopt - | rsync = "rsync-options" - | otherwise = "scp-options" - options - -- inplace makes rsync resume partial files - | rsync = ["-p", "--progress", "--inplace"] - | otherwise = ["-p"] - rsync = SysConfig.rsync + cmd = "rsync" + configopt= "rsync-options" + -- inplace makes rsync resume partial files + options = ["-p", "--progress", "--inplace"] + +onRemote :: Git.Repo -> String -> [String] -> Annex Bool +onRemote r command params = runCmd r "git-annex-shell" (command:dir:params) + where + dir = Git.workTree r {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} diff --git a/configure.hs b/configure.hs index 2334385a38..1abdc8914d 100644 --- a/configure.hs +++ b/configure.hs @@ -22,7 +22,7 @@ tests = [ , TestCase "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" , TestCase "uuid" "uuid" $ requireCmd "uuid" "uuid" , TestCase "xargs -0" "xargs_0" $ requireCmd "xargs -0" "xargs -0 /dev/null" + , TestCase "rsync" "rsync" $ requireCmd "rsync" "rsync --version >/dev/null" ] tmpDir :: String diff --git a/doc/install.mdwn b/doc/install.mdwn index 1cff4462e0..bad1d9f258 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -6,7 +6,7 @@ To build and use git-annex, you will need: * pcre-light: * `uuid`: * `xargs`: -* `rsync`: (optional but recommended) +* `rsync`: * Then just [[download]] git-annex and run: `make; make install` ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation, From 60df4e5728b8af804f06c39ef3b897af12247ceb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 13:39:30 -0400 Subject: [PATCH 0655/8313] git-annex-shell is complete still not used --- Command/RecvKey.hs | 38 ++++++++++++++++++++++++++++++++++++++ Command/SendKey.hs | 38 ++++++++++++++++++++++++++++++++++++++ Options.hs | 20 ++++++-------------- Remotes.hs | 2 +- RsyncFile.hs | 33 +++++++++++++++++++++++++++++++++ git-annex-shell.hs | 8 ++++---- git-annex.hs | 15 ++++++++++++++- 7 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 Command/RecvKey.hs create mode 100644 Command/SendKey.hs create mode 100644 RsyncFile.hs diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs new file mode 100644 index 0000000000..3232010d49 --- /dev/null +++ b/Command/RecvKey.hs @@ -0,0 +1,38 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.RecvKey where + +import Control.Monad (when) +import Control.Monad.State (liftIO) +import System.Exit + +import Command +import Types +import Core +import qualified Backend +import RsyncFile + +command :: [Command] +command = [Command "recvkey" paramKey seek + "runs rsync in server mode to receive content"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: CommandStartString +start keyname = do + backends <- Backend.list + let key = genKey (head backends) keyname + present <- inAnnex key + when present $ + error "key is already present in annex" + + ok <- getViaTmp key (liftIO . rsyncServerReceive) + if ok + then return Nothing + else liftIO exitFailure diff --git a/Command/SendKey.hs b/Command/SendKey.hs new file mode 100644 index 0000000000..0ddc0d23b9 --- /dev/null +++ b/Command/SendKey.hs @@ -0,0 +1,38 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.SendKey where + +import Control.Monad (when) +import Control.Monad.State (liftIO) +import System.Exit + +import Locations +import qualified Annex +import Command +import Types +import Core +import qualified Backend +import RsyncFile + +command :: [Command] +command = [Command "sendkey" paramKey seek + "runs rsync in server mode to send content"] + +seek :: [CommandSeek] +seek = [withKeys start] + +start :: CommandStartString +start keyname = do + backends <- Backend.list + let key = genKey (head backends) keyname + present <- inAnnex key + g <- Annex.gitRepo + let file = annexLocation g key + when present $ + liftIO $ rsyncServerSend file + liftIO exitFailure diff --git a/Options.hs b/Options.hs index 684aed97d6..5f367c9dd4 100644 --- a/Options.hs +++ b/Options.hs @@ -24,21 +24,13 @@ storeOptString :: FlagName -> String -> Annex () storeOptString name val = Annex.flagChange name $ FlagString val commonOptions :: [Option] -commonOptions = [ - Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) +commonOptions = + [ Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) "specify default key-value backend to use" - , Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) - "skip files matching the glob pattern" - ] + ] diff --git a/Remotes.hs b/Remotes.hs index a775f71d4b..ca65c99ff0 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -251,7 +251,7 @@ copyToRemote r key file = do sshLocation :: Git.Repo -> FilePath -> FilePath sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file -{- Copies a file from or to a remote, using rsync (when available) or scp. -} +{- Copies a file from or to a remote, using rsync. -} remoteCopyFile :: Bool -> Git.Repo -> String -> String -> Annex Bool remoteCopyFile recv r src dest = do showProgress -- make way for progress bar diff --git a/RsyncFile.hs b/RsyncFile.hs new file mode 100644 index 0000000000..14f6dc926b --- /dev/null +++ b/RsyncFile.hs @@ -0,0 +1,33 @@ +{- git-annex file copying with rsync + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module RsyncFile where + +import Utility +import System.Posix.Process + +{- Runs rsync in server mode to send a file, and exits. -} +rsyncServerSend :: FilePath -> IO () +rsyncServerSend file = rsyncExec $ rsyncServerParams ++ ["--sender", file] + +{- Runs rsync in server mode to receive a file. -} +rsyncServerReceive :: FilePath -> IO Bool +rsyncServerReceive file = rsync $ rsyncServerParams ++ [file] + +rsyncServerParams :: [String] +rsyncServerParams = + [ "--server" + , "-p" -- preserve permissions + , "--inplace" -- allow resuming of transfers of big files + , "-e.Lsf", "." -- other options rsync normally uses in server mode + ] + +rsync :: [String] -> IO Bool +rsync params = boolSystem "rsync" params + +rsyncExec :: [String] -> IO () +rsyncExec params = executeFile "rsync" True params Nothing diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 492d184469..8783e7f60a 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -17,16 +17,16 @@ import Options import qualified Command.ConfigList import qualified Command.InAnnex import qualified Command.DropKey ---import qualified Command.RecvKey ---import qualified Command.SendKey +import qualified Command.RecvKey +import qualified Command.SendKey cmds :: [Command] cmds = map adddirparam $ concat [ Command.ConfigList.command , Command.InAnnex.command , Command.DropKey.command --- , Command.RecvKey.command --- , Command.SendKey.command + , Command.RecvKey.command + , Command.SendKey.command ] where adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c } diff --git a/git-annex.hs b/git-annex.hs index 110054fd5a..dff67f9d84 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,6 +6,7 @@ -} import System.Environment +import System.Console.GetOpt import qualified GitRepo as Git import CmdLine @@ -59,6 +60,18 @@ cmds = concat , Command.Find.command ] +options :: [Option] +options = commonOptions ++ + [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] + header :: String header = "Usage: git-annex command [option ..]" @@ -66,4 +79,4 @@ main :: IO () main = do args <- getArgs gitrepo <- Git.repoFromCwd - dispatch gitrepo args cmds commonOptions header + dispatch gitrepo args cmds options header From eac433a84ad397e371300343b7cd30b7741ee023 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 15:46:33 -0400 Subject: [PATCH 0656/8313] use git-annex-shell configlist --- Annex.hs | 4 +-- Command/Move.hs | 7 +++-- GitRepo.hs | 35 +++++++++++-------------- Remotes.hs | 68 ++++++++++++++++++++++++++++++++----------------- 4 files changed, 66 insertions(+), 48 deletions(-) diff --git a/Annex.hs b/Annex.hs index 6e5198e8ef..55f9edb36e 100644 --- a/Annex.hs +++ b/Annex.hs @@ -47,7 +47,7 @@ new gitrepo allbackends = do where prep = do -- read git config and update state - gitrepo' <- liftIO $ Git.configRead gitrepo Nothing + gitrepo' <- liftIO $ Git.configRead gitrepo Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} @@ -136,5 +136,5 @@ setConfig key value = do g <- Annex.gitRepo liftIO $ Git.run g ["config", key, value] -- re-read git config and update the repo's state - g' <- liftIO $ Git.configRead g Nothing + g' <- liftIO $ Git.configRead g Annex.gitRepoChange g' diff --git a/Command/Move.hs b/Command/Move.hs index e872d86fe7..4291d221a0 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,6 +7,7 @@ module Command.Move where +import Control.Monad (when) import Control.Monad.State (liftIO) import Command @@ -20,6 +21,7 @@ import qualified GitRepo as Git import qualified Remotes import UUID import Messages +import Utility command :: [Command] command = [Command "move" paramPath seek @@ -134,10 +136,11 @@ fromPerform move key = do else return Nothing -- fail fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do - ok <- Remotes.onRemote remote "dropkey" + ok <- Remotes.onRemote remote boolSystem False "dropkey" ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] - remoteHasKey remote key False + when ok $ + remoteHasKey remote key False return ok fromCleanup False _ _ = return True diff --git a/GitRepo.hs b/GitRepo.hs index 2c2ad7b539..9dfce0d35b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -24,6 +24,8 @@ module GitRepo ( configGet, configMap, configRead, + hConfigRead, + configStore, configTrue, gitCommandLine, run, @@ -141,11 +143,7 @@ assertUrl repo action = then action else error $ "acting on local git repo " ++ repoDescribe repo ++ " not supported" -assertSsh :: Repo -> a -> a -assertSsh repo action = - if repoIsSsh repo - then action - else error $ "unsupported url in repo " ++ repoDescribe repo + bare :: Repo -> Bool bare repo = case Map.lookup "core.bare" $ config repo of Just v -> configTrue v @@ -276,11 +274,9 @@ pipeNullSplit repo params = do where split0 s = filter (not . null) $ split "\0" s -{- Runs git config and populates a repo with its config. - - - - For a ssh repository, a list of ssh options may optionally be specified. -} -configRead :: Repo -> Maybe [String] -> IO Repo -configRead repo@(Repo { location = Dir d }) _ = do +{- Runs git config and populates a repo with its config. -} +configRead :: Repo -> IO Repo +configRead repo@(Repo { location = Dir d }) = do {- Cannot use pipeRead because it relies on the config having been already read. Instead, chdir to the repo. -} cwd <- getCurrentDirectory @@ -288,19 +284,18 @@ configRead repo@(Repo { location = Dir d }) _ = do (\_ -> changeWorkingDirectory cwd) $ pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo -configRead repo sshopts = assertSsh repo $ do - pOpen ReadFromPipe "ssh" params $ hConfigRead repo - where - params = case sshopts of - Nothing -> [urlHost repo, command] - Just l -> l ++ [urlHost repo, command] - command = "cd " ++ shellEscape (urlPath repo) ++ - " && git config --list" +configRead r = assertLocal r $ error "internal" + +{- Reads git config from a handle and populates a repo with it. -} hConfigRead :: Repo -> Handle -> IO Repo hConfigRead repo h = do val <- hGetContentsStrict h - let r = repo { config = configParse val } - return r { remotes = configRemotes r } + return $ configStore repo val + +{- Parses a git config and returns a version of the repo using it. -} +configStore :: Repo -> String -> Repo +configStore repo s = r { remotes = configRemotes r } + where r = repo { config = configParse s } {- Checks if a string fron git config is a true value. -} configTrue :: String -> Bool diff --git a/Remotes.hs b/Remotes.hs index ca65c99ff0..841fe947fe 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -25,6 +25,7 @@ import qualified Data.Map as Map import Data.String.Utils import System.Directory hiding (copyFile) import System.Posix.Directory +import System.Cmd.Utils import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) @@ -112,16 +113,14 @@ inAnnex r key = if Git.repoIsUrl r else liftIO (try checklocal ::IO (Either IOException Bool)) where checklocal = do - -- run a local check by making an Annex monad - -- using the remote + -- run a local check inexpensively, + -- by making an Annex monad using the remote a <- Annex.new r [] Annex.eval a (Core.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- onRemote r "inannex" + inannex <- onRemote r boolSystem False "inannex" ["--backend=" ++ backendName key, keyName key] - -- XXX Note that ssh failing and the file not existing - -- are not currently differentiated. return $ Right inannex {- Cost Ordered list of remotes. -} @@ -199,24 +198,29 @@ byName name = do - config for a specified remote, and updates state. If successful, it - returns the updated git repo. -} tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) -tryGitConfigRead r = do - sshoptions <- repoConfig r "ssh-options" "" - if Map.null $ Git.configMap r - then do - -- configRead can fail due to IO error or - -- for other reasons; catch all possible exceptions - result <- liftIO (try (Git.configRead r $ Just $ words sshoptions)::IO (Either SomeException Git.Repo)) +tryGitConfigRead r + | not $ Map.null $ Git.configMap r = return $ Right r -- already read + | Git.repoIsSsh r = store $ onRemote r pipedconfig r "configlist" [] + | Git.repoIsUrl r = return $ Left r + | otherwise = store $ safely $ Git.configRead r + where + -- Reading config can fail due to IO error or + -- for other reasons; catch all possible exceptions. + safely a = do + result <- liftIO (try (a)::IO (Either SomeException Git.Repo)) case result of - Left _ -> return $ Left r - Right r' -> do - g <- Annex.gitRepo - let l = Git.remotes g - let g' = Git.remotesAdd g $ - exchange l r' - Annex.gitRepoChange g' - return $ Right r' - else return $ Right r -- config already read - where + Left _ -> return r + Right r' -> return r' + pipedconfig cmd params = safely $ + pOpen ReadFromPipe cmd params $ + Git.hConfigRead r + store a = do + r' <- a + g <- Annex.gitRepo + let l = Git.remotes g + let g' = Git.remotesAdd g $ exchange l r' + Annex.gitRepoChange g' + return $ Right r' exchange [] _ = [] exchange (old:ls) new = if Git.repoRemoteName old == Git.repoRemoteName new @@ -268,10 +272,26 @@ remoteCopyFile recv r src dest = do -- inplace makes rsync resume partial files options = ["-p", "--progress", "--inplace"] -onRemote :: Git.Repo -> String -> [String] -> Annex Bool -onRemote r command params = runCmd r "git-annex-shell" (command:dir:params) +{- Uses a supplied function to run a git-annex-shell command on a remote. -} +onRemote + :: Git.Repo + -> (String -> [String] -> IO a) + -> a + -> String + -> [String] + -> Annex a +onRemote r with errorval command params + | not $ Git.repoIsUrl r = liftIO $ with shellcmd shellopts + | Git.repoIsSsh r = do + sshoptions <- repoConfig r "ssh-options" "" + liftIO $ with "ssh" $ + words sshoptions ++ [Git.urlHost r, sshcmd] + | otherwise = return errorval where dir = Git.workTree r + shellcmd = "git-annex-shell" + shellopts = command:dir:params + sshcmd = shellcmd ++ " " ++ unwords (map shellEscape shellopts) {- Runs a command in a remote, using ssh if necessary. - (Honors annex-ssh-options.) -} From 30e0065ab97843f866a7fe095b8a18ee6eb4c321 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 15:52:59 -0400 Subject: [PATCH 0657/8313] tuple makes it clearer --- Command/Move.hs | 2 +- Remotes.hs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 4291d221a0..d96d36138c 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -136,7 +136,7 @@ fromPerform move key = do else return Nothing -- fail fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup fromCleanup True remote key = do - ok <- Remotes.onRemote remote boolSystem False "dropkey" + ok <- Remotes.onRemote remote (boolSystem, False) "dropkey" ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] diff --git a/Remotes.hs b/Remotes.hs index 841fe947fe..70356de024 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -119,7 +119,7 @@ inAnnex r key = if Git.repoIsUrl r Annex.eval a (Core.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") - inannex <- onRemote r boolSystem False "inannex" + inannex <- onRemote r (boolSystem, False) "inannex" ["--backend=" ++ backendName key, keyName key] return $ Right inannex @@ -200,7 +200,7 @@ byName name = do tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) tryGitConfigRead r | not $ Map.null $ Git.configMap r = return $ Right r -- already read - | Git.repoIsSsh r = store $ onRemote r pipedconfig r "configlist" [] + | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] | Git.repoIsUrl r = return $ Left r | otherwise = store $ safely $ Git.configRead r where @@ -275,12 +275,11 @@ remoteCopyFile recv r src dest = do {- Uses a supplied function to run a git-annex-shell command on a remote. -} onRemote :: Git.Repo - -> (String -> [String] -> IO a) - -> a + -> ((String -> [String] -> IO a), a) -> String -> [String] -> Annex a -onRemote r with errorval command params +onRemote r (with, errorval) command params | not $ Git.repoIsUrl r = liftIO $ with shellcmd shellopts | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" From 700aed13cff27f9315df1209e0cd37d5e51f5390 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:09:17 -0400 Subject: [PATCH 0658/8313] git-annex-shell now exclusively used for all remote access --- Backend/File.hs | 13 ++-- Command/Move.hs | 33 ++++----- Command/RecvKey.hs | 6 +- Remotes.hs | 124 +++++++++++++++++----------------- RsyncFile.hs | 14 +++- doc/todo/git-annex-shell.mdwn | 37 +--------- 6 files changed, 102 insertions(+), 125 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index ee73150211..9bc5a2aa63 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -65,12 +65,7 @@ copyKeyFile key file = do trycopy full (r:rs) = do probablythere <- probablyPresent r if probablythere - then do - showNote $ "copying from " ++ Git.repoDescribe r ++ "..." - copied <- Remotes.copyFromRemote r key file - if copied - then return True - else trycopy full rs + then docopy r (trycopy full rs) else trycopy full rs -- This check is to avoid an ugly message if a remote is a -- drive that is not mounted. Avoid checking inAnnex for ssh @@ -82,6 +77,12 @@ copyKeyFile key file = do if not $ Git.repoIsUrl r then liftIO $ doesFileExist $ annexLocation r key else return True + docopy r continue = do + showNote $ "copying from " ++ Git.repoDescribe r ++ "..." + copied <- Remotes.copyFromRemote r key file + if copied + then return True + else continue {- Checks remotes to verify that enough copies of a key exist to allow - for a key to be safely removed (with no data loss), and fails with an diff --git a/Command/Move.hs b/Command/Move.hs index d96d36138c..fa847e6bab 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,13 +7,11 @@ module Command.Move where -import Control.Monad (when) import Control.Monad.State (liftIO) import Command import qualified Command.Drop import qualified Annex -import Locations import LocationLog import Types import Core @@ -86,26 +84,17 @@ toPerform move key = do return Nothing Right False -> do showNote $ "to " ++ Git.repoDescribe remote ++ "..." - let tmpfile = annexTmpLocation remote ++ keyFile key - ok <- Remotes.copyToRemote remote key tmpfile + ok <- Remotes.copyToRemote remote key if ok - then return $ Just $ toCleanup move remote key tmpfile + then return $ Just $ toCleanup move remote key else return Nothing -- failed Right True -> return $ Just $ Command.Drop.cleanup key -toCleanup :: Bool -> Git.Repo -> Key -> FilePath -> CommandCleanup -toCleanup move remote key tmpfile = do - -- Tell remote to use the transferred content. - ok <- Remotes.runCmd remote "git-annex" ["setkey", "--quiet", - "--backend=" ++ backendName key, - "--key=" ++ keyName key, - tmpfile] - if ok - then do - remoteHasKey remote key True - if move - then Command.Drop.cleanup key - else return True - else return False +toCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup +toCleanup move remote key = do + remoteHasKey remote key True + if move + then Command.Drop.cleanup key + else return True {- Moves (or copies) the content of an annexed file from another repository - to the current repository and updates locationlog information on both. @@ -140,7 +129,9 @@ fromCleanup True remote key = do ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] - when ok $ - remoteHasKey remote key False + -- better safe than sorry: assume the remote dropped the key + -- even if it seemed to fail; the failure could have occurred + -- after it really dropped it + remoteHasKey remote key False return ok fromCleanup False _ _ = return True diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 3232010d49..840b328613 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -34,5 +34,9 @@ start keyname = do ok <- getViaTmp key (liftIO . rsyncServerReceive) if ok - then return Nothing + then do + -- forcibly quit after receiving one key, + -- and shutdown cleanly so queued git commands run + _ <- shutdown 0 + liftIO exitSuccess else liftIO exitFailure diff --git a/Remotes.hs b/Remotes.hs index 70356de024..19d1bfdd37 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -15,7 +15,6 @@ module Remotes ( byName, copyFromRemote, copyToRemote, - runCmd, onRemote ) where @@ -23,11 +22,10 @@ import Control.Exception.Extensible import Control.Monad.State (liftIO) import qualified Data.Map as Map import Data.String.Utils -import System.Directory hiding (copyFile) -import System.Posix.Directory import System.Cmd.Utils import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) +import Data.Maybe import Types import qualified GitRepo as Git @@ -39,6 +37,7 @@ import Utility import qualified Core import Messages import CopyFile +import RsyncFile {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -227,92 +226,95 @@ tryGitConfigRead r then new : exchange ls new else old : exchange ls new -{- Tries to copy a key's content from a remote to a file. -} +{- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file - | not $ Git.repoIsUrl r = getlocal - | Git.repoIsSsh r = getssh + | not $ Git.repoIsUrl r = liftIO $ copyFile (annexLocation r key) file + | Git.repoIsSsh r = rsynchelper r True key file | otherwise = error "copying from non-ssh repo not supported" - where - keyloc = annexLocation r key - getlocal = liftIO $ copyFile keyloc file - getssh = remoteCopyFile True r (sshLocation r keyloc) file -{- Tries to copy a key's content to a file on a remote. -} -copyToRemote :: Git.Repo -> Key -> FilePath -> Annex Bool -copyToRemote r key file = do - g <- Annex.gitRepo - let keyloc = annexLocation g key - if not $ Git.repoIsUrl r - then putlocal keyloc - else if Git.repoIsSsh r - then putssh keyloc - else error "copying to non-ssh repo not supported" - where - putlocal src = liftIO $ copyFile src file - putssh src = remoteCopyFile False r src (sshLocation r file) +{- Tries to copy a key's content to a remote's annex. -} +copyToRemote :: Git.Repo -> Key -> Annex Bool +copyToRemote r key + | not $ Git.repoIsUrl r = do + g <- Annex.gitRepo + let keysrc = annexLocation g key + let keydest = annexLocation r key + liftIO $ copyFile keysrc keydest + | Git.repoIsSsh r = do + g <- Annex.gitRepo + let keysrc = annexLocation g key + rsynchelper r False key keysrc + | otherwise = error "copying to non-ssh repo not supported" -sshLocation :: Git.Repo -> FilePath -> FilePath -sshLocation r file = Git.urlHost r ++ ":" ++ shellEscape file - -{- Copies a file from or to a remote, using rsync. -} -remoteCopyFile :: Bool -> Git.Repo -> String -> String -> Annex Bool -remoteCopyFile recv r src dest = do +rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool) +rsynchelper r sending key file = do showProgress -- make way for progress bar - o <- repoConfig r configopt "" - res <- liftIO $ boolSystem cmd $ options ++ words o ++ [src, dest] + p <- rsyncParams r sending key file + liftIO $ putStrLn $ unwords p + res <- liftIO $ boolSystem "rsync" p if res then return res else do showLongNote "rsync failed -- run git annex again to resume file transfer" return res + +{- Generates rsync parameters that ssh to the remote and asks it + - to either receive or send the key's content. -} +rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [String] +rsyncParams r sending key file = do + -- Note that the command is terminated with "--", because + -- rsync will tack on its own options to this command, + -- and they need to be ignored. + shellcmd <- git_annex_shell r + (if sending then "sendkey" else "recvkey") + ["--backend=" ++ backendName key, keyName key, "--"] + -- Convert the ssh command into rsync command line. + let eparam = rsyncShell $ fromJust shellcmd + o <- repoConfig r "rsync-options" "" + let base = options ++ words o ++ eparam + if sending + then return $ base ++ [dummy, file] + else return $ base ++ [file, dummy] where - cmd = "rsync" - configopt= "rsync-options" -- inplace makes rsync resume partial files options = ["-p", "--progress", "--inplace"] + -- the rsync shell parameter controls where rsync + -- does, so the source/dest parameter can be a dummy value, + -- that just enables remote rsync mode. + dummy = ":" -{- Uses a supplied function to run a git-annex-shell command on a remote. -} +{- Uses a supplied function to run a git-annex-shell command on a remote. + - + - Or, if the remote does not support running remote commands, returns + - a specified error value. -} onRemote :: Git.Repo - -> ((String -> [String] -> IO a), a) + -> (String -> [String] -> IO a, a) -> String -> [String] -> Annex a -onRemote r (with, errorval) command params - | not $ Git.repoIsUrl r = liftIO $ with shellcmd shellopts +onRemote r (with, errorval) command params = do + s <- git_annex_shell r command params + case s of + Just shellcmd -> liftIO $ with (shellcmd !! 0) (tail shellcmd) + Nothing -> return errorval + +{- Generates parameters to run a git-annex-shell command on a remote. -} +git_annex_shell :: Git.Repo -> String -> [String] -> Annex (Maybe [String]) +git_annex_shell r command params + | not $ Git.repoIsUrl r = return $ Just (shellcmd:shellopts) | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" - liftIO $ with "ssh" $ - words sshoptions ++ [Git.urlHost r, sshcmd] - | otherwise = return errorval + return $ Just $ ["ssh"] ++ words sshoptions ++ + [Git.urlHost r, sshcmd] + | otherwise = return Nothing where dir = Git.workTree r shellcmd = "git-annex-shell" shellopts = command:dir:params sshcmd = shellcmd ++ " " ++ unwords (map shellEscape shellopts) -{- Runs a command in a remote, using ssh if necessary. - - (Honors annex-ssh-options.) -} -runCmd :: Git.Repo -> String -> [String] -> Annex Bool -runCmd r command params = do - sshoptions <- repoConfig r "ssh-options" "" - if not $ Git.repoIsUrl r - then do - cwd <- liftIO getCurrentDirectory - liftIO $ bracket_ - (changeWorkingDirectory (Git.workTree r)) - (changeWorkingDirectory cwd) - (boolSystem command params) - else if Git.repoIsSsh r - then liftIO $ boolSystem "ssh" $ - words sshoptions ++ [Git.urlHost r, sshcmd] - else error "running command in non-ssh repo not supported" - where - sshcmd = "cd " ++ shellEscape (Git.workTree r) ++ - " && " ++ shellEscape command ++ " " ++ - unwords (map shellEscape params) - {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} repoConfig :: Git.Repo -> String -> String -> Annex String diff --git a/RsyncFile.hs b/RsyncFile.hs index 14f6dc926b..274e66151b 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -7,8 +7,20 @@ module RsyncFile where -import Utility import System.Posix.Process +import Data.String.Utils + +import Utility + +{- Generates parameters to make rsync use a specified command as its remote + - shell. -} +rsyncShell :: [String] -> [String] +rsyncShell command = ["-e", unwords $ map escape command] + where + {- rsync requires some weird, non-shell like quoting in + - here. A doubled single quote inside the single quoted + - string is a single quote. -} + escape s = "'" ++ (join "''" $ split "'" s) ++ "'" {- Runs rsync in server mode to send a file, and exits. -} rsyncServerSend :: FilePath -> IO () diff --git a/doc/todo/git-annex-shell.mdwn b/doc/todo/git-annex-shell.mdwn index 47db0c1ca7..a9e3b43ede 100644 --- a/doc/todo/git-annex-shell.mdwn +++ b/doc/todo/git-annex-shell.mdwn @@ -1,3 +1,5 @@ +[[done]] + I've been considering adding a `git-annex-shell` command. This would be similar to `git-shell` (and in fact would pass unknown commands off to `git-shell`). @@ -11,38 +13,3 @@ be similar to `git-shell` (and in fact would pass unknown commands off to * Could possibly allow multiple things to be done with one ssh connection in future. * Allows expanding `~` and `~user` in repopath on the remote system. - -## Design - -`git-annex-shell -c ` - -### options - -Need at least `--quiet`, `--backend`, `--key`, `--force` - -### commands - -* `configlist repopath` - - Returns `git config --list`, for use by `tryGitConfigRead`. - - May filter the listed config to only the options git-annex really needs, - to prevent info disclosure. - -* `inannex repopath key ...` - - Checks if the keys are in the annex; shell exits zero if so. - -* `dropkey repopath key ... ` - - Same as `git annex dropkey`, and taking the same dashed options. - -* `setkey repopath tmpfile` - - Same as `git annex setkey`, and taking the same dashed options. - -### TODO - -* To be usable as a locked down shell, needs a way to launch the - rsync server, for file receiving. Safely? -* Also needs a way to support receiving files by scp. From f48658d4eeaa76062f412d4fec9cdfcecd7b0e9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:11:18 -0400 Subject: [PATCH 0659/8313] Now rsync is exclusively used for copying files to and from remotes. scp is not longer supported. --- debian/changelog | 2 ++ doc/git-annex.mdwn | 10 ++-------- doc/walkthrough.mdwn | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/debian/changelog b/debian/changelog index 76e16af2ce..5b3f2ebfd5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,8 @@ git-annex (0.15) UNRELEASED; urgency=low that are trusted to retain files without explicit checking. * Fix bug in numcopies handling when multiple remotes pointed to the same repository. + * Now rsync is exclusively used for copying files to and from remotes. + scp is not longer supported. -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 8e6ff2c0cc..e99be4e409 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -292,12 +292,6 @@ Here are all the supported configuration settings. git-annex caches UUIDs of repositories here. -* `remote..annex-scp-options` - - Options to use when using scp - to or from this repository. For example, to force ipv6, and limit - the bandwidth to 1000Kbit/s, set it to "-6 -l 1000" - * `remote..annex-ssh-options` Options to use when using ssh to talk to this repository. @@ -308,9 +302,9 @@ Here are all the supported configuration settings. to or from this repository. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.scp-options`, `annex.ssh-options`, `annex.rsync-options` +* `annex.ssh-options`, `annex.rsync-options` - Default scp, ssh, and rsync options to use if a remote does not have + Default ssh and rsync options to use if a remote does not have specific options. * `annex.version` diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index b486a4b1ff..c2ae583f02 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -198,7 +198,7 @@ to clone the laptop's annex to it: # cd ~/annex # git annex init "my desktop" -Now you can get files and they will be transferred (using `rsync` or `scp`): +Now you can get files and they will be transferred (using `rsync`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) From e6af35d20667221074219c8e91655c719c0f8b4a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:19:26 -0400 Subject: [PATCH 0660/8313] update --- debian/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/debian/changelog b/debian/changelog index 5b3f2ebfd5..d74eb8e369 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,11 @@ git-annex (0.15) UNRELEASED; urgency=low that are trusted to retain files without explicit checking. * Fix bug in numcopies handling when multiple remotes pointed to the same repository. + * Introduce the git-annex-shell command. It's now possible to make + a user have it as a restricted login shell, similar to git-shell. + * Note that git-annex will always use git-annex-shell when accessing + a ssh remote, so all of your remotes need to be upgraded to this + version of git-annex at the same time. * Now rsync is exclusively used for copying files to and from remotes. scp is not longer supported. From e153a116bb45eac409a7d4b0a07c5ba10634bd36 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 19:27:34 -0400 Subject: [PATCH 0661/8313] remove debug --- Remotes.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index 19d1bfdd37..297fa8d391 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -251,7 +251,6 @@ rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool) rsynchelper r sending key file = do showProgress -- make way for progress bar p <- rsyncParams r sending key file - liftIO $ putStrLn $ unwords p res <- liftIO $ boolSystem "rsync" p if res then return res From 5c29bb3b7c280b3e2db26dbcb38f063430f731d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 20:33:43 -0400 Subject: [PATCH 0662/8313] git-annex-shell can now be used as a login shell --- Utility.hs | 29 ++++++++++++++++++++++++++++- git-annex-shell.hs | 20 ++++++++++++++------ test.hs | 5 ++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/Utility.hs b/Utility.hs index 3a6c757515..4ab5f09302 100644 --- a/Utility.hs +++ b/Utility.hs @@ -14,9 +14,13 @@ module Utility ( relPathDirToDir, boolSystem, shellEscape, + shellUnEscape, unsetFileMode, readMaybe, - safeWriteFile + safeWriteFile, + + prop_idempotent_shellescape, + prop_idempotent_shellescape_multiword ) where import System.IO @@ -128,6 +132,29 @@ shellEscape f = "'" ++ escaped ++ "'" -- replace ' with '"'"' escaped = join "'\"'\"'" $ split "'" f +{- Unescapes a set of shellEscaped words or filenames. -} +shellUnEscape :: String -> [String] +shellUnEscape [] = [] +shellUnEscape s = word:(shellUnEscape rest) + where + (word, rest) = findword "" s + findword w [] = (w, "") + findword w (c:cs) + | c == ' ' = (w, cs) + | c == '\'' = inquote c w cs + | c == '"' = inquote c w cs + | otherwise = findword (w++[c]) cs + inquote _ w [] = (w, "") + inquote q w (c:cs) + | c == q = findword w cs + | otherwise = inquote q (w++[c]) cs + +{- For quickcheck. -} +prop_idempotent_shellescape :: String -> Bool +prop_idempotent_shellescape s = [s] == (shellUnEscape $ shellEscape s) +prop_idempotent_shellescape_multiword :: [String] -> Bool +prop_idempotent_shellescape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) + {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} unsetFileMode :: FilePath -> FileMode -> IO () diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 8783e7f60a..251acf6132 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -7,6 +7,7 @@ import System.Environment import Control.Monad (when) +import Data.List import qualified GitRepo as Git import CmdLine @@ -43,14 +44,14 @@ main' :: [String] -> IO () main' [] = failure -- skip leading -c options, passed by eg, ssh main' ("-c":p) = main' p --- Since git-annex explicitly runs git-annex-shell, we will be passed --- a redundant "git-annex-shell" parameter when we're the user's login shell. -main' ("git-annex-shell":p) = main' p -- a command can be either a builtin or something to pass to git-shell main' c@(cmd:dir:params) | elem cmd builtins = builtin cmd dir params | otherwise = external c main' c@(cmd:_) + -- Handle the case of being the user's login shell. It will be passed + -- a single string containing all the real parameters. + | isPrefixOf "git-annex-shell " cmd = main' $ drop 1 $ shellUnEscape cmd | elem cmd builtins = failure | otherwise = external c @@ -60,13 +61,20 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do let gitrepo = Git.repoFromPath dir - dispatch gitrepo (cmd:params) cmds commonOptions header + dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header external :: [String] -> IO () -external l = do - ret <- boolSystem "git-shell" ("-c":l) +external params = do + ret <- boolSystem "git-shell" ("-c":(filterparams params)) when (not ret) $ error "git-shell failed" +-- Drop all args after "--". +-- These tend to be passed by rsync and not useful. +filterparams :: [String] -> [String] +filterparams [] = [] +filterparams ("--":_) = [] +filterparams (a:as) = a:filterparams as + failure :: IO () failure = error $ "bad parameters\n\n" ++ usage header cmds commonOptions diff --git a/test.hs b/test.hs index 9a6e05a97c..9d64e92607 100644 --- a/test.hs +++ b/test.hs @@ -3,11 +3,14 @@ import Test.HUnit.Tools import GitRepo import Locations +import Utility alltests :: [Test] alltests = [ qctest "prop_idempotent_deencode" prop_idempotent_deencode, - qctest "prop_idempotent_fileKey" prop_idempotent_fileKey + qctest "prop_idempotent_fileKey" prop_idempotent_fileKey, + qctest "prop_idempotent_shellescape" prop_idempotent_shellescape, + qctest "prop_idempotent_shellescape_multiword" prop_idempotent_shellescape_multiword ] main :: IO (Counts, Int) From ed593f1f3f810ae9e456dc233273da23608cea82 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 20:38:50 -0400 Subject: [PATCH 0663/8313] git-annex-shell makes this more tractable --- doc/bugs/bare_git_repos.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn index e24d7a7ee3..67917db8a6 100644 --- a/doc/bugs/bare_git_repos.mdwn +++ b/doc/bugs/bare_git_repos.mdwn @@ -12,3 +12,15 @@ However, that is not currently supported. Problems include: * `.git-annex/` needs to have state recorded to it and committed, and that is not possible with a bare repo. (If [[todo/branching]] were done, that might be fixed.) + +---- + +Update: Now that git-annex-shell is used for accessing remote repos, +it would be possible to add smarts about bare repos there, and avoid +some of the above problems. Probably only the state recording problem +remains. + +A possible other approach to the state recording repo is to not +record state changes on the remote in that case. Git-annex already +records remote state changes locally whenever it modifies the state of a +remote. --[[Joey]] From 14fe13dc2b0f39857eeccdc36483d0f4a0d81fdc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 21:22:03 -0400 Subject: [PATCH 0664/8313] support ssh urls containing "~", and relative user:path --- GitRepo.hs | 36 ++++++++++++++++++- debian/changelog | 5 ++- .../wishlist:_support_for_more_ssh_urls_.mdwn | 4 ++- git-annex-shell.hs | 3 +- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 9dfce0d35b..07f243b661 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -43,6 +43,7 @@ module GitRepo ( encodeGitFile, typeChangedFiles, typeChangedStagedFiles, + absDir, prop_idempotent_deencode ) where @@ -50,6 +51,7 @@ module GitRepo ( import Control.Monad (unless) import System.Directory import System.Posix.Directory +import System.Posix.User import System.Path import System.Cmd.Utils import IO (bracket_) @@ -62,7 +64,7 @@ import Data.Char import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf -import Data.List (isInfixOf) +import Data.List (isInfixOf, isPrefixOf) import Utility @@ -444,6 +446,38 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\"" prop_idempotent_deencode :: String -> Bool prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s) +{- Git ssh remotes can have a directory that is specified relative + - to a home directory. This converts such a directory to an absolute path. + - Note that it has to run on the remote system. + -} +absDir :: String -> IO String +absDir d + | isPrefixOf "/" d = expandt d + | otherwise = do + h <- myhomedir + return $ h ++ d + where + homedir u = (homeDirectory u) ++ "/" + myhomedir = do + uid <- getEffectiveUserID + u <- getUserEntryForID uid + return $ homedir u + expandt [] = return "" + expandt ('/':'~':'/':cs) = do + h <- myhomedir + return $ h ++ cs + expandt ('/':'~':cs) = do + let (name, rest) = findname "" cs + u <- getUserEntryForName name + return $ homedir u ++ rest + expandt (c:cs) = do + v <- expandt cs + return (c:v) + findname n [] = (n, "") + findname n (c:cs) + | c == '/' = (n, cs) + | otherwise = findname (n++[c]) cs + {- Finds the current git repository, which may be in a parent directory. -} repoFromCwd :: IO Repo repoFromCwd = do diff --git a/debian/changelog b/debian/changelog index d74eb8e369..b5d0058cc8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,7 @@ git-annex (0.15) UNRELEASED; urgency=low - * Support scp-style urls for remotes (host:path). Note that - paths relative to the user's home directory, or containing "~" are - not yet supported. + * Support scp-style urls for remotes (host:path). + * Support ssh urls containing "~". * Add trust and untrust subcommands, to allow configuring repositories that are trusted to retain files without explicit checking. * Fix bug in numcopies handling when multiple remotes pointed to the diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn index ee3b4a64ad..55b8120a75 100644 --- a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn @@ -17,4 +17,6 @@ Specifically, if I have ~/bar set up on host foo: > code on the remote to lookup homedirs. If git-annex grows a > `git annex shell` that is run on the remote side > (something I am [[considering|todo/git-annex-shell]] for other reasons), it -> could handle the expansions there. --[[Joey]] +> could handle the expansions there. --[[Joey]] + +> Update: Now `~` expansions are supported. [[done]] diff --git a/git-annex-shell.hs b/git-annex-shell.hs index 251acf6132..78dd777904 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -60,7 +60,8 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do - let gitrepo = Git.repoFromPath dir + dir' <- Git.absDir dir + let gitrepo = Git.repoFromPath dir' dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header external :: [String] -> IO () From 3902b05b253ffd7a39ccebf996928228bc97732f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 22:22:50 -0400 Subject: [PATCH 0665/8313] releasing version 0.15 --- debian/changelog | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index b5d0058cc8..12a2efaffa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.15) UNRELEASED; urgency=low +git-annex (0.15) unstable; urgency=low * Support scp-style urls for remotes (host:path). * Support ssh urls containing "~". @@ -12,9 +12,9 @@ git-annex (0.15) UNRELEASED; urgency=low a ssh remote, so all of your remotes need to be upgraded to this version of git-annex at the same time. * Now rsync is exclusively used for copying files to and from remotes. - scp is not longer supported. + scp is not longer supported. - -- Joey Hess Tue, 28 Dec 2010 13:13:20 -0400 + -- Joey Hess Fri, 31 Dec 2010 22:00:52 -0400 git-annex (0.14) unstable; urgency=low From 2419d3d50e16113dad69d7b33c14dda43976e83f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 31 Dec 2010 22:22:57 -0400 Subject: [PATCH 0666/8313] add news item for git-annex 0.15 --- doc/news/version_0.10.mdwn | 11 ----------- doc/news/version_0.15.mdwn | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) delete mode 100644 doc/news/version_0.10.mdwn create mode 100644 doc/news/version_0.15.mdwn diff --git a/doc/news/version_0.10.mdwn b/doc/news/version_0.10.mdwn deleted file mode 100644 index 450b7d272f..0000000000 --- a/doc/news/version_0.10.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 0.10 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * In .gitattributes, the annex.numcopies attribute can be used - to control the number of copies to retain of different types of files. - * Bugfix: Always correctly handle gitattributes when in a subdirectory of - the repository. (Had worked ok for ones like "*.mp3", but failed for - ones like "dir/*".) - * fsck: Fix warning about not enough copies of a file, when locations - are known, but are not available in currently configured remotes. - * precommit: Optimise to avoid calling git-check-attr more than once. - * The git-annex-backend attribute has been renamed to annex.backend."""]] \ No newline at end of file diff --git a/doc/news/version_0.15.mdwn b/doc/news/version_0.15.mdwn new file mode 100644 index 0000000000..c3d4d2c0f7 --- /dev/null +++ b/doc/news/version_0.15.mdwn @@ -0,0 +1,15 @@ +git-annex 0.15 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Support scp-style urls for remotes (host:path). + * Support ssh urls containing "~". + * Add trust and untrust subcommands, to allow configuring repositories + that are trusted to retain files without explicit checking. + * Fix bug in numcopies handling when multiple remotes pointed to the + same repository. + * Introduce the git-annex-shell command. It's now possible to make + a user have it as a restricted login shell, similar to git-shell. + * Note that git-annex will always use git-annex-shell when accessing + a ssh remote, so all of your remotes need to be upgraded to this + version of git-annex at the same time. + * Now rsync is exclusively used for copying files to and from remotes. + scp is not longer supported."""]] \ No newline at end of file From 2eb621052599f3a112d57480a5e123bb47835644 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 13:13:27 -0400 Subject: [PATCH 0667/8313] fixed --- doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn index 6ee01d2300..6616758730 100644 --- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn +++ b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn @@ -2,3 +2,5 @@ git-annex does not seem to support all kinds of urls that git does. > Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls_]]. > --[[Joey]] + +>> And now all fixed! --[[Joey]] From 500ddd7f4bf36cdcaa7c0bdf5dae617c36535371 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:03:29 -0400 Subject: [PATCH 0668/8313] add a check for unknown UUID when logging --- LocationLog.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LocationLog.hs b/LocationLog.hs index 6d52f4bdd1..e4dad03f5c 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -32,6 +32,7 @@ import System.Locale import qualified Data.Map as Map import System.Directory import System.Posix.Process +import Control.Monad (when) import qualified GitRepo as Git import Utility @@ -86,6 +87,8 @@ instance Read LogLine where - and returns the filename of the logfile. -} logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key u s = do + when (null u) $ + error $ "bug detected: unknown UUID for " ++ Git.repoDescribe repo line <- logNow s u ls <- readLog logfile writeLog logfile (compactLog $ line:ls) From ca60731e1c9617429b5c04892f165ae5451b0fab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:15:39 -0400 Subject: [PATCH 0669/8313] refactor --- Remotes.hs | 103 +++++++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 297fa8d391..f1df2f6adf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -43,6 +43,39 @@ import RsyncFile list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes +{- Reads the configs of remotes. + - + - This has to be called before things that rely on eg, the UUID of + - remotes. Most such things will take care of running this themselves. + - + - As reading the config of remotes can be expensive, this + - function will only read configs once per git-annex run. It's + - assumed to be cheap to read the config of non-URL remotes, + - so this is done each time git-annex is run. Conversely, + - the config of an URL remote is only read when there is no + - cached UUID value. + - -} +readconfigs :: Annex () +readconfigs = do + g <- Annex.gitRepo + remotesread <- Annex.flagIsSet "remotesread" + unless remotesread $ do + let allremotes = Git.remotes g + let cheap = filter (not . Git.repoIsUrl) allremotes + let expensive = filter Git.repoIsUrl allremotes + doexpensive <- filterM cachedUUID expensive + unless (null doexpensive) $ + showNote $ "getting UUID for " ++ + list doexpensive ++ "..." + let todo = cheap ++ doexpensive + unless (null todo) $ do + _ <- mapM tryGitConfigRead todo + Annex.flagChange "remotesread" $ FlagBool True + where + cachedUUID r = do + u <- getUUID r + return $ null u + {- Cost ordered lists of remotes that the LocationLog indicate may have a key. - - The first list is of remotes that are trusted to have the key. @@ -54,54 +87,30 @@ list remotes = join ", " $ map Git.repoDescribe remotes -} keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) keyPossibilities key = do + readconfigs + allremotes <- remotesByCost - -- To determine if a remote has a key, its UUID needs to be known. - -- The locally cached UUIDs of remotes can fall out of date if - -- eg, a different drive is mounted at the same location. - -- But, reading the config of remotes can be expensive, so make - -- sure we only do it once per git-annex run. - remotesread <- Annex.flagIsSet "remotesread" - if remotesread - then partition allremotes - else do - -- We assume that it's cheap to read the config - -- of non-URL remotes, so that is done each time. - -- But reading the config of an URL remote is - -- only done when there is no cached UUID value. - let cheap = filter (not . Git.repoIsUrl) allremotes - let expensive = filter Git.repoIsUrl allremotes - doexpensive <- filterM cachedUUID expensive - unless (null doexpensive) $ - showNote $ "getting UUID for " ++ - list doexpensive ++ "..." - let todo = cheap ++ doexpensive - if not $ null todo - then do - _ <- mapM tryGitConfigRead todo - Annex.flagChange "remotesread" $ FlagBool True - keyPossibilities key - else partition allremotes - where - cachedUUID r = do - u <- getUUID r - return $ null u - partition remotes = do - g <- Annex.gitRepo - u <- getUUID g - trusted <- getTrusted - -- get uuids of other repositories that are - -- believed to have the key - uuids <- liftIO $ keyLocations g key - let validuuids = filter (/= u) uuids - -- get uuids trusted to have the key - -- note that validuuids is assumed to not have dups - let validtrusteduuids = intersect validuuids trusted - -- remotes that match uuids that have the key - validremotes <- reposByUUID remotes validuuids - -- partition out the trusted and untrusted remotes - trustedremotes <- reposByUUID validremotes validtrusteduuids - untrustedremotes <- reposWithoutUUID validremotes trusted - return (trustedremotes, untrustedremotes, validtrusteduuids) + g <- Annex.gitRepo + u <- getUUID g + trusted <- getTrusted + + -- get uuids of other repositories that are + -- believed to have the key + uuids <- liftIO $ keyLocations g key + let validuuids = filter (/= u) uuids + + -- get uuids trusted to have the key + -- note that validuuids is assumed to not have dups + let validtrusteduuids = intersect validuuids trusted + + -- remotes that match uuids that have the key + validremotes <- reposByUUID allremotes validuuids + + -- partition out the trusted and untrusted remotes + trustedremotes <- reposByUUID validremotes validtrusteduuids + untrustedremotes <- reposWithoutUUID validremotes trusted + + return (trustedremotes, untrustedremotes, validtrusteduuids) {- Checks if a given remote has the content for a key inAnnex. - If the remote cannot be accessed, returns a Left error. From 533419147c3578ae935150b31e1a8a01f8bcfe6f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:20:35 -0400 Subject: [PATCH 0670/8313] reorg --- Remotes.hs | 77 +++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index f1df2f6adf..17d1bd0b0a 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -43,7 +43,42 @@ import RsyncFile list :: [Git.Repo] -> String list remotes = join ", " $ map Git.repoDescribe remotes -{- Reads the configs of remotes. +{- The git configs for the git repo's remotes is not read on startup + - because reading it may be expensive. This function tries to read the + - config for a specified remote, and updates state. If successful, it + - returns the updated git repo. -} +tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) +tryGitConfigRead r + | not $ Map.null $ Git.configMap r = return $ Right r -- already read + | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] + | Git.repoIsUrl r = return $ Left r + | otherwise = store $ safely $ Git.configRead r + where + -- Reading config can fail due to IO error or + -- for other reasons; catch all possible exceptions. + safely a = do + result <- liftIO (try (a)::IO (Either SomeException Git.Repo)) + case result of + Left _ -> return r + Right r' -> return r' + pipedconfig cmd params = safely $ + pOpen ReadFromPipe cmd params $ + Git.hConfigRead r + store a = do + r' <- a + g <- Annex.gitRepo + let l = Git.remotes g + let g' = Git.remotesAdd g $ exchange l r' + Annex.gitRepoChange g' + return $ Right r' + exchange [] _ = [] + exchange (old:ls) new = + if Git.repoRemoteName old == Git.repoRemoteName new + then new : exchange ls new + else old : exchange ls new + + +{- Reads the configs of all remotes. - - This has to be called before things that rely on eg, the UUID of - remotes. Most such things will take care of running this themselves. @@ -55,8 +90,8 @@ list remotes = join ", " $ map Git.repoDescribe remotes - the config of an URL remote is only read when there is no - cached UUID value. - -} -readconfigs :: Annex () -readconfigs = do +readConfigs :: Annex () +readConfigs = do g <- Annex.gitRepo remotesread <- Annex.flagIsSet "remotesread" unless remotesread $ do @@ -87,7 +122,7 @@ readconfigs = do -} keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID]) keyPossibilities key = do - readconfigs + readConfigs allremotes <- remotesByCost g <- Annex.gitRepo @@ -201,40 +236,6 @@ byName name = do "there is no git remote named \"" ++ name ++ "\"" return $ head match -{- The git configs for the git repo's remotes is not read on startup - - because reading it may be expensive. This function tries to read the - - config for a specified remote, and updates state. If successful, it - - returns the updated git repo. -} -tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo) -tryGitConfigRead r - | not $ Map.null $ Git.configMap r = return $ Right r -- already read - | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] - | Git.repoIsUrl r = return $ Left r - | otherwise = store $ safely $ Git.configRead r - where - -- Reading config can fail due to IO error or - -- for other reasons; catch all possible exceptions. - safely a = do - result <- liftIO (try (a)::IO (Either SomeException Git.Repo)) - case result of - Left _ -> return r - Right r' -> return r' - pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd params $ - Git.hConfigRead r - store a = do - r' <- a - g <- Annex.gitRepo - let l = Git.remotes g - let g' = Git.remotesAdd g $ exchange l r' - Annex.gitRepoChange g' - return $ Right r' - exchange [] _ = [] - exchange (old:ls) new = - if Git.repoRemoteName old == Git.repoRemoteName new - then new : exchange ls new - else old : exchange ls new - {- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file From b001e6573c09235f9d22cb4daf034369f6bff11b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:33:24 -0400 Subject: [PATCH 0671/8313] expand --- doc/git-annex-shell.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 9f51b6813d..1231ef668d 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -36,6 +36,10 @@ can be used as a login shell for SSH accounts. This runs rsync in server mode to transfer out the content of a key. +Note that the directory parameter should be an absolute path, otherwise +it is assumed to be relative to the user's home directory. Also the +first "/~/" or "/~user/" is expanded to the specified home directory. + Any other command is passed through to git-shell. # OPTIONS From a857e1f4ee247cae0cf0ce6696a9015460d117de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:34:14 -0400 Subject: [PATCH 0672/8313] git-annex-shell: Avoid exposing any git repo config except for the annex.uuid when doing configlist. --- Command/ConfigList.hs | 5 +++-- debian/changelog | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs index 0d9d789b54..b91c2a9160 100644 --- a/Command/ConfigList.hs +++ b/Command/ConfigList.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Annex import Command -import qualified GitRepo as Git +import UUID command :: [Command] command = [Command "configlist" paramNothing seek @@ -23,5 +23,6 @@ seek = [withNothing start] start :: CommandStartNothing start = do g <- Annex.gitRepo - liftIO $ Git.run g ["config", "--list"] + u <- getUUID g + liftIO $ putStrLn $ "annex.uuid=" ++ u return Nothing diff --git a/debian/changelog b/debian/changelog index 12a2efaffa..1c265f1be4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.16) UNRELEASED; urgency=low + + * git-annex-shell: Avoid exposing any git repo config except for the + annex.uuid when doing configlist. + + -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 + git-annex (0.15) unstable; urgency=low * Support scp-style urls for remotes (host:path). From f1b747e6d9fae2b365f65fd43c6295da503218bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 17:45:27 -0400 Subject: [PATCH 0673/8313] bugfix: Running `move --to` with a remote whose UUID was not yet known * bugfix: Running `move --to` with a remote whose UUID was not yet known could result in git-annex not recording on the local side where the file was moved to. This could not result in data loss, or even a significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. --- Command/Move.hs | 1 + Remotes.hs | 2 +- debian/changelog | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Command/Move.hs b/Command/Move.hs index fa847e6bab..0077618f8b 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -75,6 +75,7 @@ toStart move file = isAnnexed file $ \(key, _) -> do return $ Just $ toPerform move key toPerform :: Bool -> Key -> CommandPerform toPerform move key = do + Remotes.readConfigs -- checking the remote is expensive, so not done in the start step remote <- Remotes.commandLineRemote isthere <- Remotes.inAnnex remote key diff --git a/Remotes.hs b/Remotes.hs index 17d1bd0b0a..da5b3e6229 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -7,8 +7,8 @@ module Remotes ( list, + readConfigs, keyPossibilities, - tryGitConfigRead, inAnnex, same, commandLineRemote, diff --git a/debian/changelog b/debian/changelog index 1c265f1be4..99bcc95657 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,12 @@ git-annex (0.16) UNRELEASED; urgency=low * git-annex-shell: Avoid exposing any git repo config except for the annex.uuid when doing configlist. + * bugfix: Running `move --to` with a remote whose UUID was not yet known + could result in git-annex not recording on the local side where the + file was moved to. This could not result in data loss, or even a + significant problem, since the remote *did* record that it had the file. + * Also, add a general guard to detect attempts to record information + about repositories with missing UUIDs. -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From 759e860e4b6c514b74cafb2c0dd9c52c4d59316b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 21:05:31 -0400 Subject: [PATCH 0674/8313] add testcoverage target using hpc added a test for key read and show --- .gitignore | 2 ++ Makefile | 9 ++++++++- TypeInternals.hs | 18 ++++++++++++++++++ test.hs | 3 +++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d2f4c2b743..f68d1d0adf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ git-annex.1 git-annex-shell.1 doc/.ikiwiki html +*.tix +.hpc diff --git a/Makefile b/Makefile index 2f1fd05b91..8d124f1431 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,13 @@ test: $(GHCMAKE) test ./test +testcoverage: + rm -f test.tix test + ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test + ./test + hpc report test --exclude=Main --exclude=QC + hpc markup test --exclude=Main --exclude=QC --destdir=.hpc + # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. ifeq ($(shell which ikiwiki),) @@ -49,7 +56,7 @@ docs: $(mans) --exclude='news/.*' clean: - rm -rf build $(bins) $(mans) test configure SysConfig.hs + rm -rf build $(bins) $(mans) test configure SysConfig.hs *.tix .hpc rm -rf doc/.ikiwiki html .PHONY: $(bins) test install diff --git a/TypeInternals.hs b/TypeInternals.hs index 9acc06bb33..fe6e562f95 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -12,6 +12,7 @@ module TypeInternals where import Control.Monad.State (StateT) import Data.String.Utils import qualified Data.Map as M +import Test.QuickCheck import qualified GitRepo as Git import qualified GitQueue @@ -57,6 +58,23 @@ instance Read Key where b = head l k = join ":" $ drop 1 l +-- for quickcheck +instance Arbitrary Key where + arbitrary = do + backendname <- arbitrary + keyname <- arbitrary + return $ Key (backendname, keyname) + +prop_idempotent_key_read_show :: Key -> Bool +prop_idempotent_key_read_show k + -- filter out empty key or backend names + -- also backend names will not contain colons + | null kname || null bname || elem ':' bname = True + | otherwise = k == (read $ show k) + where + bname = backendName k + kname = keyName k + backendName :: Key -> BackendName backendName (Key (b,_)) = b keyName :: Key -> KeyName diff --git a/test.hs b/test.hs index 9d64e92607..28b54b78b8 100644 --- a/test.hs +++ b/test.hs @@ -4,14 +4,17 @@ import Test.HUnit.Tools import GitRepo import Locations import Utility +import TypeInternals alltests :: [Test] alltests = [ qctest "prop_idempotent_deencode" prop_idempotent_deencode, qctest "prop_idempotent_fileKey" prop_idempotent_fileKey, + qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show, qctest "prop_idempotent_shellescape" prop_idempotent_shellescape, qctest "prop_idempotent_shellescape_multiword" prop_idempotent_shellescape_multiword ] main :: IO (Counts, Int) main = runVerboseTests (TestList alltests) + From aedc46caca6f24755c914f105d1ac6d9787a7755 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 21:07:58 -0400 Subject: [PATCH 0675/8313] cleanup --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8d124f1431..338aa947e2 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,8 @@ testcoverage: rm -f test.tix test ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test ./test - hpc report test --exclude=Main --exclude=QC - hpc markup test --exclude=Main --exclude=QC --destdir=.hpc + @hpc report test --exclude=Main --exclude=QC + @hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null # If ikiwiki is available, build static html docs suitable for being # shipped in the software package. From a323463726096bba15b6a353d35bc1eb0ae238b9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 21:27:08 -0400 Subject: [PATCH 0676/8313] add more tests --- Utility.hs | 31 ++++++++++++++++++++++++------- test.hs | 15 ++++++++------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/Utility.hs b/Utility.hs index 4ab5f09302..96bbc89ee2 100644 --- a/Utility.hs +++ b/Utility.hs @@ -19,8 +19,10 @@ module Utility ( readMaybe, safeWriteFile, - prop_idempotent_shellescape, - prop_idempotent_shellescape_multiword + prop_idempotent_shellEscape, + prop_idempotent_shellEscape_multiword, + prop_parentDir_basics, + prop_relPathDirToDir_basics ) where import System.IO @@ -45,7 +47,7 @@ readFileStrict :: FilePath -> IO String readFileStrict f = readFile f >>= \s -> length s `seq` return s {- Returns the parent directory of a path. Parent of / is "" -} -parentDir :: String -> String +parentDir :: FilePath -> FilePath parentDir dir = if not $ null dirs then slash ++ join s (take (length dirs - 1) dirs) @@ -55,6 +57,14 @@ parentDir dir = slash = if isAbsolute dir then s else "" s = [pathSeparator] +prop_parentDir_basics :: FilePath -> Bool +prop_parentDir_basics dir + | null dir = True + | dir == "/" = parentDir dir == "" + | otherwise = p /= dir + where + p = parentDir dir + {- Converts a filename into a normalized, absolute path. -} absPath :: FilePath -> IO FilePath absPath file = do @@ -97,6 +107,13 @@ relPathDirToDir from to = numcommon = length common path = join s $ dotdots ++ uncommon +prop_relPathDirToDir_basics :: FilePath -> FilePath -> Bool +prop_relPathDirToDir_basics from to + | from == to = null r + | otherwise = not (null r) && (last r == '/') + where + r = relPathDirToDir from to + {- Run a system command, and returns True or False - if it succeeded or failed. - @@ -150,10 +167,10 @@ shellUnEscape s = word:(shellUnEscape rest) | otherwise = inquote q (w++[c]) cs {- For quickcheck. -} -prop_idempotent_shellescape :: String -> Bool -prop_idempotent_shellescape s = [s] == (shellUnEscape $ shellEscape s) -prop_idempotent_shellescape_multiword :: [String] -> Bool -prop_idempotent_shellescape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) +prop_idempotent_shellEscape :: String -> Bool +prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) +prop_idempotent_shellEscape_multiword :: [String] -> Bool +prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} diff --git a/test.hs b/test.hs index 28b54b78b8..77a8606b3a 100644 --- a/test.hs +++ b/test.hs @@ -7,14 +7,15 @@ import Utility import TypeInternals alltests :: [Test] -alltests = [ - qctest "prop_idempotent_deencode" prop_idempotent_deencode, - qctest "prop_idempotent_fileKey" prop_idempotent_fileKey, - qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show, - qctest "prop_idempotent_shellescape" prop_idempotent_shellescape, - qctest "prop_idempotent_shellescape_multiword" prop_idempotent_shellescape_multiword +alltests = + [ qctest "prop_idempotent_deencode" prop_idempotent_deencode + , qctest "prop_idempotent_fileKey" prop_idempotent_fileKey + , qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show + , qctest "prop_idempotent_shellEscape" prop_idempotent_shellEscape + , qctest "prop_idempotent_shellEscape_multiword" prop_idempotent_shellEscape_multiword + , qctest "prop_parentDir_basics" prop_parentDir_basics + , qctest "prop_relPathDirToDir_basics" prop_relPathDirToDir_basics ] main :: IO (Counts, Int) main = runVerboseTests (TestList alltests) - From 446978c1e6de0996144ba30adc178d358ed877b7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 22:14:24 -0400 Subject: [PATCH 0677/8313] fix reversion in annex-ignored --- Remotes.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remotes.hs b/Remotes.hs index da5b3e6229..5bb2efa554 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -95,7 +95,7 @@ readConfigs = do g <- Annex.gitRepo remotesread <- Annex.flagIsSet "remotesread" unless remotesread $ do - let allremotes = Git.remotes g + allremotes <- filterM repoNotIgnored $ Git.remotes g let cheap = filter (not . Git.repoIsUrl) allremotes let expensive = filter Git.repoIsUrl allremotes doexpensive <- filterM cachedUUID expensive From 27619497ec02685b740c838e512fc61e013c4ec3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 4 Jan 2011 22:17:18 -0400 Subject: [PATCH 0678/8313] remove file before running cp This way, if a temp file was left behind, and permissions don't allow it to be written, cp won't fail. --- CopyFile.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CopyFile.hs b/CopyFile.hs index 8bd07dc35a..e913aa070d 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -7,13 +7,20 @@ module CopyFile (copyFile) where +import Control.Monad (when) +import System.Directory (doesFileExist, removeFile) + import Utility import qualified SysConfig {- The cp command is used, because I hate reinventing the wheel, - and because this allows easy access to features like cp --reflink. -} copyFile :: FilePath -> FilePath -> IO Bool -copyFile src dest = boolSystem "cp" opts +copyFile src dest = do + e <- doesFileExist dest + when e $ + removeFile dest + boolSystem "cp" opts where opts = if SysConfig.cp_reflink_auto then ["--reflink=auto", src, dest] From d134da6dab42d4c99a614fd4081d6d9b8b7b51ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 5 Jan 2011 20:28:50 -0400 Subject: [PATCH 0679/8313] tweak message --- Backend/File.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend/File.hs b/Backend/File.hs index 9bc5a2aa63..073a7c2267 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -137,7 +137,7 @@ showLocations key exclude = do showTriedRemotes :: [Git.Repo] -> Annex () showTriedRemotes [] = return () showTriedRemotes remotes = - showLongNote $ "I was unable to access these remotes: " ++ + showLongNote $ "Unable to access these remotes: " ++ Remotes.list remotes getNumCopies :: Maybe Int -> Annex Int From f936be30c36420072557a80a4baf1fabbeeb7b46 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 5 Jan 2011 20:29:11 -0400 Subject: [PATCH 0680/8313] message tweaked --- doc/walkthrough.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index c2ae583f02..2f868e3030 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -92,7 +92,7 @@ it: # git annex get video/hackity_hack_and_kaxxt.mov get video/_why_hackity_hack_and_kaxxt.mov (not available) - I was unable to access these remotes: usbdrive, server + Unable to access these remotes: usbdrive, server Try making some of these repositories available: 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive @@ -122,7 +122,7 @@ you'll see something like this. # git annex drop important_file other.iso drop important_file (unsafe) Could only verify the existence of 0 out of 1 necessary copies - I was unable to access these remotes: usbdrive + Unable to access these remotes: usbdrive Try making some of these repositories available: 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive From e15385c339ca8bf1d9896b542a866736b6dca924 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 18:12:52 -0400 Subject: [PATCH 0681/8313] git-annex setup --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..b98b07d7d2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git-annex/*.log merge=union From 2772faf921e3c3e68f10876303c158f76ce6fae6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 18:12:52 -0400 Subject: [PATCH 0682/8313] git annex init --- .git-annex/uuid.log | 1 + 1 file changed, 1 insertion(+) create mode 100644 .git-annex/uuid.log diff --git a/.git-annex/uuid.log b/.git-annex/uuid.log new file mode 100644 index 0000000000..8cd9452b64 --- /dev/null +++ b/.git-annex/uuid.log @@ -0,0 +1 @@ +1ac368a4-19e2-11e0-8c0f-8fcd42cf5a8d test repo From 901cdbde78c79a3cc6f5a53b10f925e21b8343b5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 20:09:49 -0400 Subject: [PATCH 0683/8313] added some toplevel git-annex subcommand tests note that test coverage doesn't work for those yet --- debian/changelog | 1 + test.hs | 108 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 99bcc95657..d9aa1e4de3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. + * Test suite improvements. -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/test.hs b/test.hs index 77a8606b3a..b6b0c27405 100644 --- a/test.hs +++ b/test.hs @@ -1,14 +1,38 @@ +{- git-annex test suite + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + import Test.HUnit import Test.HUnit.Tools +import System.Directory +import System.Posix.Directory (changeWorkingDirectory) +import System.Posix.Files +import System.Posix.Env +import IO (bracket_, bracket) +import Control.Monad (unless, when) +import Data.List +import System.IO.Error -import GitRepo +import qualified GitRepo as Git import Locations import Utility import TypeInternals -alltests :: [Test] -alltests = - [ qctest "prop_idempotent_deencode" prop_idempotent_deencode +main :: IO (Counts, Int) +main = do + -- Add current directory to the from of PATH, so git-annex etc will + -- be used, no matter where it is run from. + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd++":"++p) True + runVerboseTests $ TestList [quickchecks, toplevels] + +quickchecks :: Test +quickchecks = TestLabel "quickchecks" $ TestList + [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode , qctest "prop_idempotent_fileKey" prop_idempotent_fileKey , qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show , qctest "prop_idempotent_shellEscape" prop_idempotent_shellEscape @@ -17,5 +41,77 @@ alltests = , qctest "prop_relPathDirToDir_basics" prop_relPathDirToDir_basics ] -main :: IO (Counts, Int) -main = runVerboseTests (TestList alltests) +toplevels :: Test +toplevels = TestLabel "toplevel" $ TestList + [ test_init + , test_add + ] + +test_init :: Test +test_init = TestLabel "git-annex init" $ TestCase $ ingitrepo $ do + git_annex "init" ["-q", reponame] @? "init failed" + e <- doesFileExist annexlog + unless e $ + assertFailure $ annexlog ++ " not created" + c <- readFile annexlog + unless (isInfixOf reponame c) $ + assertFailure $ annexlog ++ " does not contain repo name" + where + annexlog = ".git-annex/uuid.log" + reponame = "test repo" + +test_add :: Test +test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do + writeFile file content + git_annex "add" ["-q", "foo"] @? "add failed" + s <- getSymbolicLinkStatus file + unless (isSymbolicLink s) $ + assertFailure "git-annex add did not create symlink" + c <- readFile file + unless (c == content) $ + assertFailure "file content changed during git-annex add" + r <- try (writeFile file $ content++"bar") + case r of + Left _ -> return () -- expected permission error + Right _ -> assertFailure "was able to modify annexed file content" + where + file = "foo" + content = "foo file content" + +git_annex :: String -> [String] -> IO Bool +git_annex command params = boolSystem "git-annex" (command:params) + +inannex :: Assertion -> Assertion +inannex a = ingitrepo $ do + git_annex "init" ["-q", reponame] @? "init failed" + a + where + reponame = "test repo" + +ingitrepo :: Assertion -> Assertion +ingitrepo a = withgitrepo $ \r -> do + cwd <- getCurrentDirectory + bracket_ (changeWorkingDirectory $ Git.workTree r) + (\_ -> changeWorkingDirectory cwd) + a + +withgitrepo :: (Git.Repo -> Assertion) -> Assertion +withgitrepo = bracket setup cleanup + where + tmpdir = ".t" + repodir = tmpdir ++ "/repo" + setup = do + cleanup True + createDirectory tmpdir + ok <- boolSystem "git" ["init", "-q", repodir] + unless ok $ + assertFailure "git init failed" + return $ Git.repoFromPath repodir + cleanup _ = do + e <- doesDirectoryExist tmpdir + when e $ do + -- git-annex prevents annexed file content + -- from being removed with permissions + -- bits; undo + _ <- boolSystem "chmod" ["+rw", "-R", tmpdir] + removeDirectoryRecursive tmpdir From 2533d826fc265b56556f8a6b9759d98771f79f53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 20:26:57 -0400 Subject: [PATCH 0684/8313] make test suite link in git-annex's commands and run directly this way, test coverage works --- GitAnnex.hs | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ debian/changelog | 2 +- git-annex.hs | 66 +----------------------------------------- test.hs | 42 ++++++++++++++------------- 4 files changed, 99 insertions(+), 86 deletions(-) create mode 100644 GitAnnex.hs diff --git a/GitAnnex.hs b/GitAnnex.hs new file mode 100644 index 0000000000..05e98d3c3a --- /dev/null +++ b/GitAnnex.hs @@ -0,0 +1,75 @@ +{- git-annex main program + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module GitAnnex where + +import System.Console.GetOpt + +import Command +import Options + +import qualified Command.Add +import qualified Command.Unannex +import qualified Command.Drop +import qualified Command.Move +import qualified Command.Copy +import qualified Command.Get +import qualified Command.FromKey +import qualified Command.DropKey +import qualified Command.SetKey +import qualified Command.Fix +import qualified Command.Init +import qualified Command.Fsck +import qualified Command.Unused +import qualified Command.DropUnused +import qualified Command.Unlock +import qualified Command.Lock +import qualified Command.PreCommit +import qualified Command.Find +import qualified Command.Uninit +import qualified Command.Trust +import qualified Command.Untrust + +cmds :: [Command] +cmds = concat + [ Command.Add.command + , Command.Get.command + , Command.Drop.command + , Command.Move.command + , Command.Copy.command + , Command.Unlock.command + , Command.Lock.command + , Command.Init.command + , Command.Unannex.command + , Command.Uninit.command + , Command.PreCommit.command + , Command.Trust.command + , Command.Untrust.command + , Command.FromKey.command + , Command.DropKey.command + , Command.SetKey.command + , Command.Fix.command + , Command.Fsck.command + , Command.Unused.command + , Command.DropUnused.command + , Command.Find.command + ] + +options :: [Option] +options = commonOptions ++ + [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + "specify a key to use" + , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + "specify to where to transfer content" + , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + "specify from where to transfer content" + , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + "skip files matching the glob pattern" + ] + +header :: String +header = "Usage: git-annex command [option ..]" diff --git a/debian/changelog b/debian/changelog index d9aa1e4de3..0aaaa75e60 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,7 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. - * Test suite improvements. + * Test suite improvements. Current top-level test coverage: 43% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/git-annex.hs b/git-annex.hs index dff67f9d84..f951817847 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -6,74 +6,10 @@ -} import System.Environment -import System.Console.GetOpt import qualified GitRepo as Git import CmdLine -import Command -import Options - -import qualified Command.Add -import qualified Command.Unannex -import qualified Command.Drop -import qualified Command.Move -import qualified Command.Copy -import qualified Command.Get -import qualified Command.FromKey -import qualified Command.DropKey -import qualified Command.SetKey -import qualified Command.Fix -import qualified Command.Init -import qualified Command.Fsck -import qualified Command.Unused -import qualified Command.DropUnused -import qualified Command.Unlock -import qualified Command.Lock -import qualified Command.PreCommit -import qualified Command.Find -import qualified Command.Uninit -import qualified Command.Trust -import qualified Command.Untrust - -cmds :: [Command] -cmds = concat - [ Command.Add.command - , Command.Get.command - , Command.Drop.command - , Command.Move.command - , Command.Copy.command - , Command.Unlock.command - , Command.Lock.command - , Command.Init.command - , Command.Unannex.command - , Command.Uninit.command - , Command.PreCommit.command - , Command.Trust.command - , Command.Untrust.command - , Command.FromKey.command - , Command.DropKey.command - , Command.SetKey.command - , Command.Fix.command - , Command.Fsck.command - , Command.Unused.command - , Command.DropUnused.command - , Command.Find.command - ] - -options :: [Option] -options = commonOptions ++ - [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) - "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) - "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) - "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) - "skip files matching the glob pattern" - ] - -header :: String -header = "Usage: git-annex command [option ..]" +import GitAnnex main :: IO () main = do diff --git a/test.hs b/test.hs index b6b0c27405..74cce41427 100644 --- a/test.hs +++ b/test.hs @@ -10,35 +10,30 @@ import Test.HUnit.Tools import System.Directory import System.Posix.Directory (changeWorkingDirectory) import System.Posix.Files -import System.Posix.Env import IO (bracket_, bracket) import Control.Monad (unless, when) import Data.List import System.IO.Error import qualified GitRepo as Git -import Locations -import Utility -import TypeInternals +import qualified Locations +import qualified Utility +import qualified TypeInternals +import qualified GitAnnex +import qualified CmdLine main :: IO (Counts, Int) -main = do - -- Add current directory to the from of PATH, so git-annex etc will - -- be used, no matter where it is run from. - cwd <- getCurrentDirectory - p <- getEnvDefault "PATH" "" - setEnv "PATH" (cwd++":"++p) True - runVerboseTests $ TestList [quickchecks, toplevels] +main = runVerboseTests $ TestList [quickchecks, toplevels] quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode - , qctest "prop_idempotent_fileKey" prop_idempotent_fileKey - , qctest "prop_idempotent_key_read_show" prop_idempotent_key_read_show - , qctest "prop_idempotent_shellEscape" prop_idempotent_shellEscape - , qctest "prop_idempotent_shellEscape_multiword" prop_idempotent_shellEscape_multiword - , qctest "prop_parentDir_basics" prop_parentDir_basics - , qctest "prop_relPathDirToDir_basics" prop_relPathDirToDir_basics + , qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey + , qctest "prop_idempotent_key_read_show" TypeInternals.prop_idempotent_key_read_show + , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape + , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword + , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics + , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics ] toplevels :: Test @@ -79,7 +74,14 @@ test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do content = "foo file content" git_annex :: String -> [String] -> IO Bool -git_annex command params = boolSystem "git-annex" (command:params) +git_annex command params = do + gitrepo <- Git.repoFromCwd + r <- try $ + CmdLine.dispatch gitrepo (command:params) + GitAnnex.cmds GitAnnex.options GitAnnex.header + case r of + Right _ -> return True + Left _ -> return False inannex :: Assertion -> Assertion inannex a = ingitrepo $ do @@ -103,7 +105,7 @@ withgitrepo = bracket setup cleanup setup = do cleanup True createDirectory tmpdir - ok <- boolSystem "git" ["init", "-q", repodir] + ok <- Utility.boolSystem "git" ["init", "-q", repodir] unless ok $ assertFailure "git init failed" return $ Git.repoFromPath repodir @@ -113,5 +115,5 @@ withgitrepo = bracket setup cleanup -- git-annex prevents annexed file content -- from being removed with permissions -- bits; undo - _ <- boolSystem "chmod" ["+rw", "-R", tmpdir] + _ <- Utility.boolSystem "chmod" ["+rw", "-R", tmpdir] removeDirectoryRecursive tmpdir From 87f424eca7f8d9ce7a437fb3756755c042fe9002 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 21:39:26 -0400 Subject: [PATCH 0685/8313] more tests --- Makefile | 1 + debian/changelog | 2 +- test.hs | 139 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 338aa947e2..e499d492c7 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ testcoverage: rm -f test.tix test ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test ./test + @echo "" @hpc report test --exclude=Main --exclude=QC @hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null diff --git a/debian/changelog b/debian/changelog index 0aaaa75e60..edcd349f55 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,7 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. - * Test suite improvements. Current top-level test coverage: 43% + * Test suite improvements. Current top-level test coverage: 53% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/test.hs b/test.hs index 74cce41427..3ad34971aa 100644 --- a/test.hs +++ b/test.hs @@ -14,6 +14,7 @@ import IO (bracket_, bracket) import Control.Monad (unless, when) import Data.List import System.IO.Error +import qualified Control.Exception.Extensible as E import qualified GitRepo as Git import qualified Locations @@ -23,7 +24,10 @@ import qualified GitAnnex import qualified CmdLine main :: IO (Counts, Int) -main = runVerboseTests $ TestList [quickchecks, toplevels] +main = do + r <- runVerboseTests $ TestList [quickchecks, toplevels] + cleanup tmpdir + return r quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList @@ -38,12 +42,15 @@ quickchecks = TestLabel "quickchecks" $ TestList toplevels :: Test toplevels = TestLabel "toplevel" $ TestList + -- test order matters, later tests may rely on state from earlier [ test_init , test_add + , test_unannex + , test_drop ] test_init :: Test -test_init = TestLabel "git-annex init" $ TestCase $ ingitrepo $ do +test_init = TestLabel "git-annex init" $ TestCase $ innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" e <- doesFileExist annexlog unless e $ @@ -56,7 +63,7 @@ test_init = TestLabel "git-annex init" $ TestCase $ ingitrepo $ do reponame = "test repo" test_add :: Test -test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do +test_add = TestLabel "git-annex add" $ TestCase $ inoldrepo $ do writeFile file content git_annex "add" ["-q", "foo"] @? "add failed" s <- getSymbolicLinkStatus file @@ -65,7 +72,7 @@ test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do c <- readFile file unless (c == content) $ assertFailure "file content changed during git-annex add" - r <- try (writeFile file $ content++"bar") + r <- try $ writeFile file $ content++"bar" case r of Left _ -> return () -- expected permission error Right _ -> assertFailure "was able to modify annexed file content" @@ -73,47 +80,113 @@ test_add = TestLabel "git-annex add" $ TestCase $ inannex $ do file = "foo" content = "foo file content" +test_unannex :: Test +test_unannex = TestLabel "git-annex unannex" $ TestCase $ intmpcopyrepo $ do + git_annex "unannex" ["-q", "foo"] @? "unannex failed" + s <- getSymbolicLinkStatus "foo" + when (isSymbolicLink s) $ + assertFailure "git-annex unannex left symlink" + +test_drop :: Test +test_drop = TestLabel "git-annex drop" $ TestCase $ intmpcopyrepo $ do + r <- git_annex "drop" ["-q", "foo"] + (not r) @? "drop wrongly succeeded with no known copy of file" + checklink + git_annex "drop" ["-q", "--force", "foo"] @? "drop --force failed" + checklink + r' <- try $ readFile "foo" + case r' of + Left _ -> return () -- expected; dangling link + Right _ -> assertFailure "drop did not remove file content" + where + checklink = do + s <- getSymbolicLinkStatus "foo" + unless (isSymbolicLink s) $ + assertFailure "git-annex drop killed symlink" + + + + git_annex :: String -> [String] -> IO Bool git_annex command params = do - gitrepo <- Git.repoFromCwd - r <- try $ - CmdLine.dispatch gitrepo (command:params) - GitAnnex.cmds GitAnnex.options GitAnnex.header + -- catch all errors, including normally fatal errors + r <- E.try (run)::IO (Either E.SomeException ()) case r of Right _ -> return True Left _ -> return False + where + run = do + gitrepo <- Git.repoFromCwd + CmdLine.dispatch gitrepo (command:params) + GitAnnex.cmds GitAnnex.options GitAnnex.header -inannex :: Assertion -> Assertion -inannex a = ingitrepo $ do +innewannex :: Assertion -> Assertion +innewannex a = innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" a where reponame = "test repo" -ingitrepo :: Assertion -> Assertion -ingitrepo a = withgitrepo $ \r -> do +innewrepo :: Assertion -> Assertion +innewrepo a = withgitrepo $ \r -> indir r a + +inoldrepo :: Assertion -> Assertion +inoldrepo = indir repodir + +intmpcopyrepo :: Assertion -> Assertion +intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a + +withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion +withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup + +withgitrepo :: (FilePath -> Assertion) -> Assertion +withgitrepo = bracket (setuprepo repodir) return + +indir :: FilePath -> Assertion -> Assertion +indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory $ Git.workTree r) + bracket_ (changeWorkingDirectory $ dir) (\_ -> changeWorkingDirectory cwd) a -withgitrepo :: (Git.Repo -> Assertion) -> Assertion -withgitrepo = bracket setup cleanup - where - tmpdir = ".t" - repodir = tmpdir ++ "/repo" - setup = do - cleanup True - createDirectory tmpdir - ok <- Utility.boolSystem "git" ["init", "-q", repodir] - unless ok $ - assertFailure "git init failed" - return $ Git.repoFromPath repodir - cleanup _ = do - e <- doesDirectoryExist tmpdir - when e $ do - -- git-annex prevents annexed file content - -- from being removed with permissions - -- bits; undo - _ <- Utility.boolSystem "chmod" ["+rw", "-R", tmpdir] - removeDirectoryRecursive tmpdir +setuprepo :: FilePath -> IO FilePath +setuprepo dir = do + cleanup dir + ensuretmpdir + ok <- Utility.boolSystem "git" ["init", "-q", dir] + unless ok $ + assertFailure "git init failed" + return dir + +copyrepo :: FilePath -> FilePath -> IO FilePath +copyrepo old new = do + cleanup new + ensuretmpdir + ok <- Utility.boolSystem "cp" ["-pr", old, new] + unless ok $ + assertFailure "cp -pr failed" + return new + +ensuretmpdir :: IO () +ensuretmpdir = do + e <- doesDirectoryExist tmpdir + unless e $ + createDirectory tmpdir + +cleanup :: FilePath -> IO () +cleanup dir = do + e <- doesDirectoryExist dir + when e $ do + -- git-annex prevents annexed file content from being + -- removed via permissions bits; undo + _ <- Utility.boolSystem "chmod" ["+rw", "-R", dir] + removeDirectoryRecursive dir + +tmpdir :: String +tmpdir = ".t" + +repodir :: String +repodir = tmpdir ++ "/repo" + +tmprepodir :: String +tmprepodir = tmpdir ++ "/tmprepo" From f4a26f01ea0ba956d968bba9b3b948298aa15568 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 6 Jan 2011 22:22:09 -0400 Subject: [PATCH 0686/8313] more tests --- debian/changelog | 2 +- debian/control | 2 +- test.hs | 108 +++++++++++++++++++++++++++++++++-------------- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/debian/changelog b/debian/changelog index edcd349f55..150268c9d9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,7 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. - * Test suite improvements. Current top-level test coverage: 53% + * Test suite improvements. Current top-level test coverage: 57% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/debian/control b/debian/control index d90041f77e..d0f8805d0f 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex diff --git a/test.hs b/test.hs index 3ad34971aa..178842f8cd 100644 --- a/test.hs +++ b/test.hs @@ -47,6 +47,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_add , test_unannex , test_drop + , test_get ] test_init :: Test @@ -64,48 +65,43 @@ test_init = TestLabel "git-annex init" $ TestCase $ innewrepo $ do test_add :: Test test_add = TestLabel "git-annex add" $ TestCase $ inoldrepo $ do - writeFile file content - git_annex "add" ["-q", "foo"] @? "add failed" - s <- getSymbolicLinkStatus file - unless (isSymbolicLink s) $ - assertFailure "git-annex add did not create symlink" - c <- readFile file - unless (c == content) $ - assertFailure "file content changed during git-annex add" - r <- try $ writeFile file $ content++"bar" - case r of - Left _ -> return () -- expected permission error - Right _ -> assertFailure "was able to modify annexed file content" - where - file = "foo" - content = "foo file content" + writeFile foofile foocontent + git_annex "add" ["-q", foofile] @? "add failed" + checklink foofile + checkcontent foofile foocontent + checkunwritable foofile + ok <- Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "added foo"] + unless ok $ + assertFailure "git commit failed" test_unannex :: Test test_unannex = TestLabel "git-annex unannex" $ TestCase $ intmpcopyrepo $ do - git_annex "unannex" ["-q", "foo"] @? "unannex failed" - s <- getSymbolicLinkStatus "foo" + git_annex "unannex" ["-q", foofile] @? "unannex failed" + s <- getSymbolicLinkStatus foofile when (isSymbolicLink s) $ assertFailure "git-annex unannex left symlink" test_drop :: Test test_drop = TestLabel "git-annex drop" $ TestCase $ intmpcopyrepo $ do - r <- git_annex "drop" ["-q", "foo"] + r <- git_annex "drop" ["-q", foofile] (not r) @? "drop wrongly succeeded with no known copy of file" - checklink - git_annex "drop" ["-q", "--force", "foo"] @? "drop --force failed" - checklink - r' <- try $ readFile "foo" - case r' of - Left _ -> return () -- expected; dangling link - Right _ -> assertFailure "drop did not remove file content" - where - checklink = do - s <- getSymbolicLinkStatus "foo" - unless (isSymbolicLink s) $ - assertFailure "git-annex drop killed symlink" - - + checklink foofile + checkcontent foofile foocontent + git_annex "drop" ["-q", "--force", foofile] @? "drop --force failed" + checklink foofile + checkdangling foofile + git_annex "drop" ["-q", foofile] @? "drop of dropped file failed" +test_get :: Test +test_get = TestLabel "git-annex get" $ TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", foofile] @? "get of file failed" + checklink foofile + checkcontent foofile foocontent + checkunwritable foofile + git_annex "get" ["-q", foofile] @? "get of file already here failed" + checklink foofile + checkcontent foofile foocontent + checkunwritable foofile git_annex :: String -> [String] -> IO Bool git_annex command params = do @@ -136,9 +132,15 @@ inoldrepo = indir repodir intmpcopyrepo :: Assertion -> Assertion intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a +intmpclonerepo :: Assertion -> Assertion +intmpclonerepo a = withtmpclonerepo $ \r -> indir r a + withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup +withtmpclonerepo :: (FilePath -> Assertion) -> Assertion +withtmpclonerepo = bracket (clonerepo repodir tmprepodir) cleanup + withgitrepo :: (FilePath -> Assertion) -> Assertion withgitrepo = bracket (setuprepo repodir) return @@ -166,6 +168,16 @@ copyrepo old new = do unless ok $ assertFailure "cp -pr failed" return new + +-- clones are always done as local clones; we cannot test ssh clones +clonerepo :: FilePath -> FilePath -> IO FilePath +clonerepo old new = do + cleanup new + ensuretmpdir + ok <- Utility.boolSystem "git" ["clone", "-q", old, new] + unless ok $ + assertFailure "git clone failed" + return new ensuretmpdir :: IO () ensuretmpdir = do @@ -181,6 +193,32 @@ cleanup dir = do -- removed via permissions bits; undo _ <- Utility.boolSystem "chmod" ["+rw", "-R", dir] removeDirectoryRecursive dir + +checklink :: FilePath -> Assertion +checklink f = do + s <- getSymbolicLinkStatus f + unless (isSymbolicLink s) $ + assertFailure $ f ++ " is not a symlink" + +checkcontent :: FilePath -> String -> Assertion +checkcontent f c = do + c' <- readFile f + unless (c' == c) $ + assertFailure $ f ++ " content unexpected" + +checkunwritable :: FilePath -> Assertion +checkunwritable f = do + r <- try $ writeFile f $ "dummy" + case r of + Left _ -> return () -- expected permission error + Right _ -> assertFailure $ "was able to modify annexed file's " ++ f ++ " content" + +checkdangling :: FilePath -> Assertion +checkdangling f = do + r <- try $ readFile f + case r of + Left _ -> return () -- expected; dangling link + Right _ -> assertFailure $ f ++ " was not a dangling link as expected" tmpdir :: String tmpdir = ".t" @@ -190,3 +228,9 @@ repodir = tmpdir ++ "/repo" tmprepodir :: String tmprepodir = tmpdir ++ "/tmprepo" + +foofile :: String +foofile = "foo" + +foocontent :: String +foocontent = "foo file content" From e29d2376937691a4170498b095191a6ffefc9875 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 01:02:06 -0400 Subject: [PATCH 0687/8313] more tests --- test.hs | 168 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 97 insertions(+), 71 deletions(-) diff --git a/test.hs b/test.hs index 178842f8cd..5a81f1a3e9 100644 --- a/test.hs +++ b/test.hs @@ -48,60 +48,89 @@ toplevels = TestLabel "toplevel" $ TestList , test_unannex , test_drop , test_get + , test_move ] test_init :: Test -test_init = TestLabel "git-annex init" $ TestCase $ innewrepo $ do +test_init = "git-annex init" ~: innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" e <- doesFileExist annexlog - unless e $ - assertFailure $ annexlog ++ " not created" + e @? (annexlog ++ " not created") c <- readFile annexlog - unless (isInfixOf reponame c) $ - assertFailure $ annexlog ++ " does not contain repo name" + isInfixOf reponame c @? annexlog ++ " does not contain repo name" where annexlog = ".git-annex/uuid.log" reponame = "test repo" test_add :: Test -test_add = TestLabel "git-annex add" $ TestCase $ inoldrepo $ do - writeFile foofile foocontent - git_annex "add" ["-q", foofile] @? "add failed" - checklink foofile - checkcontent foofile foocontent - checkunwritable foofile - ok <- Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "added foo"] - unless ok $ - assertFailure "git commit failed" +test_add = "git-annex add" ~: inoldrepo $ do + writeFile annexedfile $ content annexedfile + git_annex "add" ["-q", annexedfile] @? "add failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + writeFile ingitfile $ content ingitfile + Utility.boolSystem "git" ["add", ingitfile] @? "git add failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" + git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" + checkregularfile ingitfile test_unannex :: Test -test_unannex = TestLabel "git-annex unannex" $ TestCase $ intmpcopyrepo $ do - git_annex "unannex" ["-q", foofile] @? "unannex failed" - s <- getSymbolicLinkStatus foofile - when (isSymbolicLink s) $ - assertFailure "git-annex unannex left symlink" +test_unannex = "git-annex unannex" ~: intmpcopyrepo $ do + git_annex "unannex" ["-q", annexedfile] @? "unannex failed" + checkregularfile annexedfile + git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" + checkregularfile annexedfile + git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" test_drop :: Test -test_drop = TestLabel "git-annex drop" $ TestCase $ intmpcopyrepo $ do - r <- git_annex "drop" ["-q", foofile] +test_drop = "git-annex drop" ~: intmpcopyrepo $ do + r <- git_annex "drop" ["-q", annexedfile] (not r) @? "drop wrongly succeeded with no known copy of file" - checklink foofile - checkcontent foofile foocontent - git_annex "drop" ["-q", "--force", foofile] @? "drop --force failed" - checklink foofile - checkdangling foofile - git_annex "drop" ["-q", foofile] @? "drop of dropped file failed" + checklink annexedfile + checkcontent annexedfile + git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" + git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile test_get :: Test -test_get = TestLabel "git-annex get" $ TestCase $ intmpclonerepo $ do - git_annex "get" ["-q", foofile] @? "get of file failed" - checklink foofile - checkcontent foofile foocontent - checkunwritable foofile - git_annex "get" ["-q", foofile] @? "get of file already here failed" - checklink foofile - checkcontent foofile foocontent - checkunwritable foofile +test_get = "git-annex get" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "get" ["-q", annexedfile] @? "get of file already here failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "get" ["-q", ingitfile] @? "get ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + +test_move :: Test +test_move = "git-annex move" ~: intmpclonerepo $ do + git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file already here failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already here failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + git_annex :: String -> [String] -> IO Bool git_annex command params = do @@ -116,24 +145,17 @@ git_annex command params = do CmdLine.dispatch gitrepo (command:params) GitAnnex.cmds GitAnnex.options GitAnnex.header -innewannex :: Assertion -> Assertion -innewannex a = innewrepo $ do - git_annex "init" ["-q", reponame] @? "init failed" - a - where - reponame = "test repo" +innewrepo :: Assertion -> Test +innewrepo a = TestCase $ withgitrepo $ \r -> indir r a -innewrepo :: Assertion -> Assertion -innewrepo a = withgitrepo $ \r -> indir r a +inoldrepo :: Assertion -> Test +inoldrepo a = TestCase $ indir repodir a -inoldrepo :: Assertion -> Assertion -inoldrepo = indir repodir +intmpcopyrepo :: Assertion -> Test +intmpcopyrepo a = TestCase $ withtmpcopyrepo $ \r -> indir r a -intmpcopyrepo :: Assertion -> Assertion -intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a - -intmpclonerepo :: Assertion -> Assertion -intmpclonerepo a = withtmpclonerepo $ \r -> indir r a +intmpclonerepo :: Assertion -> Test +intmpclonerepo a = TestCase $ withtmpclonerepo $ \r -> indir r a withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup @@ -155,18 +177,14 @@ setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir ensuretmpdir - ok <- Utility.boolSystem "git" ["init", "-q", dir] - unless ok $ - assertFailure "git init failed" + Utility.boolSystem "git" ["init", "-q", dir] @? "git init failed" return dir copyrepo :: FilePath -> FilePath -> IO FilePath copyrepo old new = do cleanup new ensuretmpdir - ok <- Utility.boolSystem "cp" ["-pr", old, new] - unless ok $ - assertFailure "cp -pr failed" + Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" return new -- clones are always done as local clones; we cannot test ssh clones @@ -174,9 +192,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath clonerepo old new = do cleanup new ensuretmpdir - ok <- Utility.boolSystem "git" ["clone", "-q", old, new] - unless ok $ - assertFailure "git clone failed" + Utility.boolSystem "git" ["clone", "-q", old, new] @? "git clone failed" return new ensuretmpdir :: IO () @@ -197,14 +213,18 @@ cleanup dir = do checklink :: FilePath -> Assertion checklink f = do s <- getSymbolicLinkStatus f - unless (isSymbolicLink s) $ - assertFailure $ f ++ " is not a symlink" + isSymbolicLink s @? f ++ " is not a symlink" -checkcontent :: FilePath -> String -> Assertion -checkcontent f c = do - c' <- readFile f - unless (c' == c) $ - assertFailure $ f ++ " content unexpected" +checkregularfile :: FilePath -> Assertion +checkregularfile f = do + s <- getSymbolicLinkStatus f + isRegularFile s @? f ++ " is not a normal file" + return () + +checkcontent :: FilePath -> Assertion +checkcontent f = do + c <- readFile f + assertEqual ("checkcontent " ++ f) c (content f) checkunwritable :: FilePath -> Assertion checkunwritable f = do @@ -229,8 +249,14 @@ repodir = tmpdir ++ "/repo" tmprepodir :: String tmprepodir = tmpdir ++ "/tmprepo" -foofile :: String -foofile = "foo" +annexedfile :: String +annexedfile = "foo" -foocontent :: String -foocontent = "foo file content" +ingitfile :: String +ingitfile = "bar" + +content :: FilePath -> String +content f + | f == annexedfile = "annexed file content" + | f == ingitfile = "normal file content" + | otherwise = "unknown file " ++ f From 71a8278f9caea8d3e52a5d43cb60125018ec31fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 01:14:27 -0400 Subject: [PATCH 0688/8313] bugfix: Running `move --to` with a non-ssh remote failed. --- Remotes.hs | 7 +++++-- debian/changelog | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 5bb2efa554..2c8cdfdad1 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -249,8 +249,11 @@ copyToRemote r key | not $ Git.repoIsUrl r = do g <- Annex.gitRepo let keysrc = annexLocation g key - let keydest = annexLocation r key - liftIO $ copyFile keysrc keydest + -- run copy from perspective of remote + liftIO $ do + a <- Annex.new r [] + Annex.eval a $ Core.getViaTmp key $ \f -> + liftIO $ copyFile keysrc f | Git.repoIsSsh r = do g <- Annex.gitRepo let keysrc = annexLocation g key diff --git a/debian/changelog b/debian/changelog index 150268c9d9..969935a4e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.16) UNRELEASED; urgency=low significant problem, since the remote *did* record that it had the file. * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. + * bugfix: Running `move --to` with a non-ssh remote failed. * Test suite improvements. Current top-level test coverage: 57% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From e43d4730c5157acd5ccc421472b33d0f12167a2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:14:22 -0400 Subject: [PATCH 0689/8313] bugfix: Running `copy --to` when both local and remote had the key dropped it from local. --- Command/Move.hs | 2 +- debian/changelog | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 0077618f8b..3e7fde3705 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -89,7 +89,7 @@ toPerform move key = do if ok then return $ Just $ toCleanup move remote key else return Nothing -- failed - Right True -> return $ Just $ Command.Drop.cleanup key + Right True -> return $ Just $ toCleanup move remote key toCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup toCleanup move remote key = do remoteHasKey remote key True diff --git a/debian/changelog b/debian/changelog index 969935a4e9..18aea04670 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,7 +9,9 @@ git-annex (0.16) UNRELEASED; urgency=low * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. - * Test suite improvements. Current top-level test coverage: 57% + * bugfix: Running `copy --to` when both local and remote had the key + dropped it from local. + * Test suite improvements. Current top-level test coverage: 62% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From 3fad3e527ea12aa5cab5fe646e60300444808f0b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:14:48 -0400 Subject: [PATCH 0690/8313] various test fixes --- test.hs | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/test.hs b/test.hs index 5a81f1a3e9..216da2033e 100644 --- a/test.hs +++ b/test.hs @@ -14,6 +14,7 @@ import IO (bracket_, bracket) import Control.Monad (unless, when) import Data.List import System.IO.Error +import System.Posix.Env import qualified Control.Exception.Extensible as E import qualified GitRepo as Git @@ -21,13 +22,18 @@ import qualified Locations import qualified Utility import qualified TypeInternals import qualified GitAnnex -import qualified CmdLine -main :: IO (Counts, Int) +main :: IO () main = do + tweakpath r <- runVerboseTests $ TestList [quickchecks, toplevels] cleanup tmpdir - return r + propigate r + +propigate :: (Counts, Int) -> IO () +propigate (Counts { errors = e }, _) + | e > 0 = error "failed" + | otherwise = return () quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList @@ -49,6 +55,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_drop , test_get , test_move + , test_copy ] test_init :: Test @@ -126,7 +133,26 @@ test_move = "git-annex move" ~: intmpclonerepo $ do checklink annexedfile checkdangling annexedfile checkunwritable annexedfile - git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already here failed" + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" + checklink annexedfile + checkdangling annexedfile + checkunwritable annexedfile + +test_copy :: Test +test_copy = "git-annex copy" ~: intmpclonerepo $ do + git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file already here failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "copy" ["-q", "--to", "origin", annexedfile] @? "copy --to of file already there failed" + checklink annexedfile + checkcontent annexedfile + checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" checklink annexedfile checkdangling annexedfile checkunwritable annexedfile @@ -140,10 +166,7 @@ git_annex command params = do Right _ -> return True Left _ -> return False where - run = do - gitrepo <- Git.repoFromCwd - CmdLine.dispatch gitrepo (command:params) - GitAnnex.cmds GitAnnex.options GitAnnex.header + run = GitAnnex.run (command:params) innewrepo :: Assertion -> Test innewrepo a = TestCase $ withgitrepo $ \r -> indir r a @@ -173,6 +196,14 @@ indir dir a = do (\_ -> changeWorkingDirectory cwd) a +-- While PATH is mostly avoided, the commit hook does run it. Make +-- sure that the just-built git annex is used. +tweakpath :: IO () +tweakpath = do + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd ++ ":" ++ p) True + setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir From 55ca64d8510fc3acd34452844086fbcb4bca122c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:15:10 -0400 Subject: [PATCH 0691/8313] simplify --- GitAnnex.hs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/GitAnnex.hs b/GitAnnex.hs index 05e98d3c3a..24c9ace0ac 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -9,6 +9,8 @@ module GitAnnex where import System.Console.GetOpt +import qualified GitRepo as Git +import CmdLine import Command import Options @@ -73,3 +75,8 @@ options = commonOptions ++ header :: String header = "Usage: git-annex command [option ..]" + +run :: [String] -> IO () +run args = do + gitrepo <- Git.repoFromCwd + dispatch gitrepo args cmds options header From f189929b8b46016a254a6a938f9aba01184fb6c6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:15:23 -0400 Subject: [PATCH 0692/8313] workaround ghc weirdness with -odir The option cause it to always build to build/Main.o, no matter what binary it was building. This caused extra work, and in some cases, could cause the wrong code to be put into the final binary. --- .gitignore | 3 ++- Makefile | 7 ++++--- git-annex.hs | 8 ++------ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index f68d1d0adf..764a1af9d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -build/* +*.hi +*.o test configure SysConfig.hs diff --git a/Makefile b/Makefile index e499d492c7..44d4be02be 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr GHCFLAGS=-O2 -Wall -GHCMAKE=ghc -odir build -hidir build $(GHCFLAGS) --make +GHCMAKE=ghc $(GHCFLAGS) --make bins=git-annex git-annex-shell mans=git-annex.1 git-annex-shell.1 @@ -29,11 +29,11 @@ install: all rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \ fi -test: +test: $(bins) $(GHCMAKE) test ./test -testcoverage: +testcoverage: $(bins) rm -f test.tix test ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test ./test @@ -59,5 +59,6 @@ docs: $(mans) clean: rm -rf build $(bins) $(mans) test configure SysConfig.hs *.tix .hpc rm -rf doc/.ikiwiki html + find . \( -name \*.o -or -name \*.hi \) -exec rm {} \; .PHONY: $(bins) test install diff --git a/git-annex.hs b/git-annex.hs index f951817847..878d8bdbbc 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -1,4 +1,4 @@ -{- git-annex main program +{- git-annex main program stub - - Copyright 2010 Joey Hess - @@ -7,12 +7,8 @@ import System.Environment -import qualified GitRepo as Git -import CmdLine import GitAnnex - main :: IO () main = do args <- getArgs - gitrepo <- Git.repoFromCwd - dispatch gitrepo args cmds options header + run args From 7d15a5795aaad3b291406c1b52b381a1f246d6f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 02:18:39 -0400 Subject: [PATCH 0693/8313] more tests --- test.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test.hs b/test.hs index 216da2033e..18ccf331af 100644 --- a/test.hs +++ b/test.hs @@ -137,6 +137,12 @@ test_move = "git-annex move" ~: intmpclonerepo $ do checklink annexedfile checkdangling annexedfile checkunwritable annexedfile + git_annex "move" ["-q", "--to", "origin", ingitfile] @? "move of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + git_annex "move" ["-q", "--from", "origin", ingitfile] @? "move of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile test_copy :: Test test_copy = "git-annex copy" ~: intmpclonerepo $ do @@ -156,6 +162,12 @@ test_copy = "git-annex copy" ~: intmpclonerepo $ do checklink annexedfile checkdangling annexedfile checkunwritable annexedfile + git_annex "copy" ["-q", "--to", "origin", ingitfile] @? "copy of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile + git_annex "copy" ["-q", "--from", "origin", ingitfile] @? "copy of ingitfile should be no-op" + checkregularfile ingitfile + checkcontent ingitfile git_annex :: String -> [String] -> IO Bool From 2684cbbd23feed32ffd50ccdd8a8303a4dfbfa28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 12:34:44 -0400 Subject: [PATCH 0694/8313] clarify --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 18aea04670..b31a63b287 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,8 +9,7 @@ git-annex (0.16) UNRELEASED; urgency=low * Also, add a general guard to detect attempts to record information about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. - * bugfix: Running `copy --to` when both local and remote had the key - dropped it from local. + * bugfix: Running `copy --to` with a non-ssh remote actually did a move. * Test suite improvements. Current top-level test coverage: 62% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 From 6cb1dff757ffad735e04d6d8134f5fbfdea71650 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 13:57:37 -0400 Subject: [PATCH 0695/8313] quiet git commits --- Command/Init.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/Init.hs b/Command/Init.hs index 8ad9f79d70..47ac8e4c08 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -50,7 +50,7 @@ cleanup = do g <- Annex.gitRepo logfile <- uuidLog liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex init", logfile] + liftIO $ Git.run g ["commit", "-q", "-m", "git annex init", logfile] return True {- configure git to use union merge driver on state files, if it is not @@ -71,7 +71,7 @@ gitAttributesWrite repo = do attributes = Git.attributes repo commit = do Git.run repo ["add", attributes] - Git.run repo ["commit", "-m", "git-annex setup", + Git.run repo ["commit", "-q", "-m", "git-annex setup", attributes] attrLine :: String From f3472d3a5d71ffedc67ed212b87308052bb4c042 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:06:32 -0400 Subject: [PATCH 0696/8313] Test suite improvements. Current top-level test coverage: 65% --- debian/changelog | 2 +- test.hs | 273 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 185 insertions(+), 90 deletions(-) diff --git a/debian/changelog b/debian/changelog index b31a63b287..ead272dbe3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -10,7 +10,7 @@ git-annex (0.16) UNRELEASED; urgency=low about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. * bugfix: Running `copy --to` with a non-ssh remote actually did a move. - * Test suite improvements. Current top-level test coverage: 62% + * Test suite improvements. Current top-level test coverage: 65% -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 diff --git a/test.hs b/test.hs index 18ccf331af..e5563c5148 100644 --- a/test.hs +++ b/test.hs @@ -25,7 +25,7 @@ import qualified GitAnnex main :: IO () main = do - tweakpath + prepare r <- runVerboseTests $ TestList [quickchecks, toplevels] cleanup tmpdir propigate r @@ -56,10 +56,13 @@ toplevels = TestLabel "toplevel" $ TestList , test_get , test_move , test_copy + , test_lock + , test_edit + , test_fix ] test_init :: Test -test_init = "git-annex init" ~: innewrepo $ do +test_init = "git-annex init" ~: TestCase $ innewrepo $ do git_annex "init" ["-q", reponame] @? "init failed" e <- doesFileExist annexlog e @? (annexlog ++ " not created") @@ -70,105 +73,159 @@ test_init = "git-annex init" ~: innewrepo $ do reponame = "test repo" test_add :: Test -test_add = "git-annex add" ~: inoldrepo $ do +test_add = "git-annex add" ~: TestCase $ inmainrepo $ do writeFile annexedfile $ content annexedfile git_annex "add" ["-q", annexedfile] @? "add failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile writeFile ingitfile $ content ingitfile Utility.boolSystem "git" ["add", ingitfile] @? "git add failed" Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" - checkregularfile ingitfile + unannexed ingitfile test_unannex :: Test -test_unannex = "git-annex unannex" ~: intmpcopyrepo $ do +test_unannex = "git-annex unannex" ~: TestCase $ intmpcopyrepo $ do git_annex "unannex" ["-q", annexedfile] @? "unannex failed" - checkregularfile annexedfile + unannexed annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" - checkregularfile annexedfile + unannexed annexedfile git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" + unannexed ingitfile test_drop :: Test -test_drop = "git-annex drop" ~: intmpcopyrepo $ do - r <- git_annex "drop" ["-q", annexedfile] - (not r) @? "drop wrongly succeeded with no known copy of file" - checklink annexedfile - checkcontent annexedfile - git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile - git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" - git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile +test_drop = "git-annex drop" ~: TestList [nocopy, withcopy] + where + nocopy = "no remotes" ~: TestCase $ intmpcopyrepo $ do + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of file" + annexed_present annexedfile + git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed" + annexed_notpresent annexedfile + git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" + git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" + unannexed ingitfile + withcopy = "with remote" ~: TestCase $ intmpclonerepo $ do + git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy" + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile test_get :: Test -test_get = "git-annex get" ~: intmpclonerepo $ do +test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile git_annex "get" ["-q", annexedfile] @? "get of file failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile git_annex "get" ["-q", annexedfile] @? "get of file already here failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_present annexedfile + inmainrepo $ unannexed ingitfile + unannexed ingitfile git_annex "get" ["-q", ingitfile] @? "get ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + inmainrepo $ unannexed ingitfile + unannexed ingitfile test_move :: Test -test_move = "git-annex move" ~: intmpclonerepo $ do +test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file already here failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_notpresent annexedfile git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile + inmainrepo $ annexed_present annexedfile + annexed_notpresent annexedfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "move" ["-q", "--to", "origin", ingitfile] @? "move of ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "move" ["-q", "--from", "origin", ingitfile] @? "move of ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile test_copy :: Test -test_copy = "git-annex copy" ~: intmpclonerepo $ do +test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file already here failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile git_annex "copy" ["-q", "--to", "origin", annexedfile] @? "copy --to of file already there failed" - checklink annexedfile - checkcontent annexedfile - checkunwritable annexedfile + annexed_present annexedfile + inmainrepo $ annexed_present annexedfile git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed" - checklink annexedfile - checkdangling annexedfile - checkunwritable annexedfile + annexed_notpresent annexedfile + inmainrepo $ annexed_present annexedfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "copy" ["-q", "--to", "origin", ingitfile] @? "copy of ingitfile should be no-op" - checkregularfile ingitfile - checkcontent ingitfile + unannexed ingitfile + inmainrepo $ unannexed ingitfile git_annex "copy" ["-q", "--from", "origin", ingitfile] @? "copy of ingitfile should be no-op" checkregularfile ingitfile checkcontent ingitfile +test_lock :: Test +test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "unlock" ["-q", annexedfile] @? "unlock failed" + unannexed annexedfile + -- write different content, to verify that lock + -- throws it away + changecontent annexedfile + writeFile annexedfile $ (content annexedfile) ++ "foo" + git_annex "lock" ["-q", annexedfile] @? "lock failed" + annexed_present annexedfile + git_annex "unlock" ["-q", annexedfile] @? "unlock failed" + unannexed annexedfile + changecontent annexedfile + git_annex "add" ["-q", annexedfile] @? "add of modified file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual ("content of modified file") c (changedcontent annexedfile) + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of modified file" + +test_edit :: Test +test_edit = "git-annex edit/commit" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "edit" ["-q", annexedfile] @? "edit failed" + unannexed annexedfile + changecontent annexedfile + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] + @? "git commit of edited file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual ("content of modified file") c (changedcontent annexedfile) + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of modified file" + +test_fix :: Test +test_fix = "git-annex fix" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + createDirectory subdir + Utility.boolSystem "git" ["mv", annexedfile, subdir] + @? "git mv failed" + git_annex "fix" ["-q", newfile] @? "fix failed" + runchecks [checklink, checkunwritable] newfile + c <- readFile newfile + assertEqual ("content of moved file") c (content annexedfile) + where + subdir = "s" + newfile = subdir ++ "/" ++ annexedfile git_annex :: String -> [String] -> IO Bool git_annex command params = do @@ -180,42 +237,34 @@ git_annex command params = do where run = GitAnnex.run (command:params) -innewrepo :: Assertion -> Test -innewrepo a = TestCase $ withgitrepo $ \r -> indir r a +innewrepo :: Assertion -> Assertion +innewrepo a = withgitrepo $ \r -> indir r a -inoldrepo :: Assertion -> Test -inoldrepo a = TestCase $ indir repodir a +inmainrepo :: Assertion -> Assertion +inmainrepo a = indir mainrepodir a -intmpcopyrepo :: Assertion -> Test -intmpcopyrepo a = TestCase $ withtmpcopyrepo $ \r -> indir r a +intmpcopyrepo :: Assertion -> Assertion +intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a -intmpclonerepo :: Assertion -> Test -intmpclonerepo a = TestCase $ withtmpclonerepo $ \r -> indir r a +intmpclonerepo :: Assertion -> Assertion +intmpclonerepo a = withtmpclonerepo $ \r -> indir r a withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion -withtmpcopyrepo = bracket (copyrepo repodir tmprepodir) cleanup +withtmpcopyrepo = bracket (copyrepo mainrepodir tmprepodir) cleanup withtmpclonerepo :: (FilePath -> Assertion) -> Assertion -withtmpclonerepo = bracket (clonerepo repodir tmprepodir) cleanup +withtmpclonerepo = bracket (clonerepo mainrepodir tmprepodir) cleanup withgitrepo :: (FilePath -> Assertion) -> Assertion -withgitrepo = bracket (setuprepo repodir) return +withgitrepo = bracket (setuprepo mainrepodir) return indir :: FilePath -> Assertion -> Assertion indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeWorkingDirectory $ dir) + bracket_ (changeToTmpDir $ dir) (\_ -> changeWorkingDirectory cwd) a --- While PATH is mostly avoided, the commit hook does run it. Make --- sure that the just-built git annex is used. -tweakpath :: IO () -tweakpath = do - cwd <- getCurrentDirectory - p <- getEnvDefault "PATH" "" - setEnv "PATH" (cwd ++ ":" ++ p) True - setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir @@ -225,6 +274,8 @@ setuprepo dir = do copyrepo :: FilePath -> FilePath -> IO FilePath copyrepo old new = do + _ <- clonerepo old new + indir new $ Utility.boolSystem "git" ["remote", "rm", "origin"] @? "git remote failed" cleanup new ensuretmpdir Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" @@ -236,6 +287,7 @@ clonerepo old new = do cleanup new ensuretmpdir Utility.boolSystem "git" ["clone", "-q", old, new] @? "git clone failed" + indir new $ git_annex "init" ["-q", new] @? "git annex init failed" return new ensuretmpdir :: IO () @@ -271,11 +323,18 @@ checkcontent f = do checkunwritable :: FilePath -> Assertion checkunwritable f = do - r <- try $ writeFile f $ "dummy" + r <- try $ writeFile f $ content f case r of Left _ -> return () -- expected permission error Right _ -> assertFailure $ "was able to modify annexed file's " ++ f ++ " content" +checkwritable :: FilePath -> Assertion +checkwritable f = do + r <- try $ writeFile f $ content f + case r of + Left _ -> assertFailure $ "unable to modify " ++ f + Right _ -> return () + checkdangling :: FilePath -> Assertion checkdangling f = do r <- try $ readFile f @@ -283,15 +342,45 @@ checkdangling f = do Left _ -> return () -- expected; dangling link Right _ -> assertFailure $ f ++ " was not a dangling link as expected" -tmpdir :: String +runchecks :: [FilePath -> Assertion] -> FilePath -> Assertion +runchecks [] _ = return () +runchecks (a:as) f = do + a f + runchecks as f + +annexed_notpresent :: FilePath -> Assertion +annexed_notpresent = runchecks [checklink, checkdangling, checkunwritable] + +annexed_present :: FilePath -> Assertion +annexed_present = runchecks [checklink, checkcontent, checkunwritable] + +unannexed :: FilePath -> Assertion +unannexed = runchecks [checkregularfile, checkcontent, checkwritable] + +prepare :: IO () +prepare = do + -- While PATH is mostly avoided, the commit hook does run it. Make + -- sure that the just-built git annex is used. + cwd <- getCurrentDirectory + p <- getEnvDefault "PATH" "" + setEnv "PATH" (cwd ++ ":" ++ p) True + setEnv "TOPDIR" cwd True + +changeToTmpDir :: FilePath -> IO () +changeToTmpDir t = do + -- Hack alert. Threading state to here was too much bother. + topdir <- getEnvDefault "TOPDIR" "" + changeWorkingDirectory $ topdir ++ "/" ++ t + +tmpdir :: String tmpdir = ".t" -repodir :: String -repodir = tmpdir ++ "/repo" +mainrepodir :: String +mainrepodir = tmpdir ++ "/repo" tmprepodir :: String tmprepodir = tmpdir ++ "/tmprepo" - + annexedfile :: String annexedfile = "foo" @@ -303,3 +392,9 @@ content f | f == annexedfile = "annexed file content" | f == ingitfile = "normal file content" | otherwise = "unknown file " ++ f + +changecontent :: FilePath -> IO () +changecontent f = writeFile f $ changedcontent f + +changedcontent :: FilePath -> String +changedcontent f = (content f) ++ " (modified)" From a14c881f023a55aa325db86fea3a5af1fce3d852 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:08:43 -0400 Subject: [PATCH 0697/8313] more --- test.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test.hs b/test.hs index e5563c5148..852d1b30f5 100644 --- a/test.hs +++ b/test.hs @@ -214,12 +214,17 @@ test_edit = "git-annex edit/commit" ~: intmpclonerepo $ do test_fix :: Test test_fix = "git-annex fix" ~: intmpclonerepo $ do + annexed_notpresent annexedfile + git_annex "fix" ["-q", annexedfile] @? "fix of not present failed" + annexed_notpresent annexedfile git_annex "get" ["-q", annexedfile] @? "get of file failed" annexed_present annexedfile + git_annex "fix" ["-q", annexedfile] @? "fix of present file failed" + annexed_present annexedfile createDirectory subdir Utility.boolSystem "git" ["mv", annexedfile, subdir] @? "git mv failed" - git_annex "fix" ["-q", newfile] @? "fix failed" + git_annex "fix" ["-q", newfile] @? "fix of moved file failed" runchecks [checklink, checkunwritable] newfile c <- readFile newfile assertEqual ("content of moved file") c (content annexedfile) From ad9dfe322894763f7013b7a41e0bf9c905cc597c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:10:38 -0400 Subject: [PATCH 0698/8313] mention test suite --- doc/not.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index d965ab86f8..b138953f91 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -17,4 +17,6 @@ * git-annex is not some flaky script that was quickly thrown together. I wrote it in Haskell because I wanted it to be solid and to compile - down to a binary. + down to a binary. And it has a fairly extensive test suite. (Don't be + fooled by "make test" only showing a few dozen test cases; each test + involves checking dozens to hundreds of assertions.) From d31e61a90da100e188b45fe033337d405c6a2124 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 7 Jan 2011 14:36:52 -0400 Subject: [PATCH 0699/8313] add news item for git-annex 0.16 --- debian/changelog | 6 +++--- doc/news/version_0.11.mdwn | 8 -------- doc/news/version_0.16.mdwn | 13 +++++++++++++ test.hs | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) delete mode 100644 doc/news/version_0.11.mdwn create mode 100644 doc/news/version_0.16.mdwn diff --git a/debian/changelog b/debian/changelog index ead272dbe3..bd986ad24c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.16) UNRELEASED; urgency=low +git-annex (0.16) unstable; urgency=low * git-annex-shell: Avoid exposing any git repo config except for the annex.uuid when doing configlist. @@ -10,9 +10,9 @@ git-annex (0.16) UNRELEASED; urgency=low about repositories with missing UUIDs. * bugfix: Running `move --to` with a non-ssh remote failed. * bugfix: Running `copy --to` with a non-ssh remote actually did a move. - * Test suite improvements. Current top-level test coverage: 65% + * Many test suite improvements. Current top-level test coverage: 65% - -- Joey Hess Tue, 04 Jan 2011 17:33:42 -0400 + -- Joey Hess Fri, 07 Jan 2011 14:33:13 -0400 git-annex (0.15) unstable; urgency=low diff --git a/doc/news/version_0.11.mdwn b/doc/news/version_0.11.mdwn deleted file mode 100644 index 03c4d7d54d..0000000000 --- a/doc/news/version_0.11.mdwn +++ /dev/null @@ -1,8 +0,0 @@ -git-annex 0.11 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * If available, rsync will be used for file transfers from remote - repositories. This allows resuming interrupted transfers. - * Added remote.annex-rsync-options. - * Avoid deleting temp files when rsync fails. - * Improve detection of version 0 repos. - * Add uninit subcommand. Closes: #[605749](http://bugs.debian.org/605749)"""]] \ No newline at end of file diff --git a/doc/news/version_0.16.mdwn b/doc/news/version_0.16.mdwn new file mode 100644 index 0000000000..3247e302dd --- /dev/null +++ b/doc/news/version_0.16.mdwn @@ -0,0 +1,13 @@ +git-annex 0.16 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * git-annex-shell: Avoid exposing any git repo config except for the + annex.uuid when doing configlist. + * bugfix: Running `move --to` with a remote whose UUID was not yet known + could result in git-annex not recording on the local side where the + file was moved to. This could not result in data loss, or even a + significant problem, since the remote *did* record that it had the file. + * Also, add a general guard to detect attempts to record information + about repositories with missing UUIDs. + * bugfix: Running `move --to` with a non-ssh remote failed. + * bugfix: Running `copy --to` with a non-ssh remote actually did a move. + * Many test suite improvements. Current top-level test coverage: 65%"""]] \ No newline at end of file diff --git a/test.hs b/test.hs index 852d1b30f5..a8e0253099 100644 --- a/test.hs +++ b/test.hs @@ -266,7 +266,7 @@ withgitrepo = bracket (setuprepo mainrepodir) return indir :: FilePath -> Assertion -> Assertion indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeToTmpDir $ dir) + bracket_ (changeToTmpDir dir) (\_ -> changeWorkingDirectory cwd) a From 82fe151f8796c5f0a37565134deb55142fdd1a99 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 15:00:04 -0400 Subject: [PATCH 0700/8313] add another test and improve error handling --- test.hs | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/test.hs b/test.hs index a8e0253099..ea1e0728ea 100644 --- a/test.hs +++ b/test.hs @@ -84,18 +84,24 @@ test_add = "git-annex add" ~: TestCase $ inmainrepo $ do unannexed ingitfile test_unannex :: Test -test_unannex = "git-annex unannex" ~: TestCase $ intmpcopyrepo $ do - git_annex "unannex" ["-q", annexedfile] @? "unannex failed" - unannexed annexedfile - git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" - unannexed annexedfile - git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" - unannexed ingitfile +test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] + where + nocopy = "no content" ~: intmpclonerepo $ do + r <- git_annex "unannex" ["-q", annexedfile] + not r @? "unannex incorrectly succeeded with no copy" + unannexed annexedfile + withcopy = "with content" ~: intmpcopyrepo $ do + git_annex "unannex" ["-q", annexedfile] @? "unannex failed" + unannexed annexedfile + git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" + unannexed annexedfile + git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op" + unannexed ingitfile test_drop :: Test -test_drop = "git-annex drop" ~: TestList [nocopy, withcopy] +test_drop = "git-annex drop" ~: TestList [noremote, withremote] where - nocopy = "no remotes" ~: TestCase $ intmpcopyrepo $ do + noremote = "no remotes" ~: TestCase $ intmpcopyrepo $ do r <- git_annex "drop" ["-q", annexedfile] (not r) @? "drop wrongly succeeded with no known copy of file" annexed_present annexedfile @@ -104,7 +110,7 @@ test_drop = "git-annex drop" ~: TestList [nocopy, withcopy] git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed" git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op" unannexed ingitfile - withcopy = "with remote" ~: TestCase $ intmpclonerepo $ do + withremote = "with remote" ~: TestCase $ intmpclonerepo $ do git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy" annexed_notpresent annexedfile inmainrepo $ annexed_present annexedfile @@ -266,9 +272,15 @@ withgitrepo = bracket (setuprepo mainrepodir) return indir :: FilePath -> Assertion -> Assertion indir dir a = do cwd <- getCurrentDirectory - bracket_ (changeToTmpDir dir) + -- Assertion failures throw non-IO errors; catch + -- any type of error and change back to cwd before + -- rethrowing. + r <- bracket_ (changeToTmpDir dir) (\_ -> changeWorkingDirectory cwd) - a + (E.try (a)::IO (Either E.SomeException ())) + case r of + Right () -> return () + Left e -> error $ show e setuprepo :: FilePath -> IO FilePath setuprepo dir = do @@ -279,8 +291,6 @@ setuprepo dir = do copyrepo :: FilePath -> FilePath -> IO FilePath copyrepo old new = do - _ <- clonerepo old new - indir new $ Utility.boolSystem "git" ["remote", "rm", "origin"] @? "git remote failed" cleanup new ensuretmpdir Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" From 32b0e103909035ad0f25427c57a1ff504aefcada Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 15:14:41 -0400 Subject: [PATCH 0701/8313] unannex: Now skips files whose content is not present, rather than it being an error. This allows gradual conversion from one backend to another by running unannex followed by add in each repository. --- Command/Unannex.hs | 8 ++++++-- debian/changelog | 8 ++++++++ doc/walkthrough.mdwn | 13 +++++++++++++ test.hs | 10 ++++++---- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 288f9da44a..2c60a23bb9 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -29,8 +29,12 @@ seek = [withFilesInGit start] {- The unannex subcommand undoes an add. -} start :: CommandStartString start file = isAnnexed file $ \(key, backend) -> do - showStart "unannex" file - return $ Just $ perform file key backend + ishere <- inAnnex key + if ishere + then do + showStart "unannex" file + return $ Just $ perform file key backend + else return Nothing perform :: FilePath -> Key -> Backend -> CommandPerform perform file key backend = do diff --git a/debian/changelog b/debian/changelog index bd986ad24c..7ca74f9945 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (0.17) UNRELEASED; urgency=low + + * unannex: Now skips files whose content is not present, rather than + it being an error. This allows gradual conversion from one backend + to another by running unannex followed by add in each repository. + + -- Joey Hess Sat, 08 Jan 2011 15:04:48 -0400 + git-annex (0.16) unstable; urgency=low * git-annex-shell: Avoid exposing any git repo config except for the diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 2f868e3030..47f05ebcf4 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -277,6 +277,19 @@ add something like this to `.gitattributes`: * annex.backend=SHA1 +## migrating between backends + +Perhaps you had been using the WORM backend, but now have configured +git-annex to use SHA1 for new files. Your old files are still in WORM. How +to migrate that content? A quick and dirty way is to use the unannex +subcommand, which removes a file from git-annex's control, followed by +a re-add of the file, to put it in the new backend. + + # git annex unannex my_cool_big_file + unannex my_cool_big_file ok + # git annex add my_cool_big_file + add my_cool_big_file (checksum ...) ok + ## unused data It's possible for data to accumulate in the annex that no files point to diff --git a/test.hs b/test.hs index ea1e0728ea..35085cc37e 100644 --- a/test.hs +++ b/test.hs @@ -16,6 +16,7 @@ import Data.List import System.IO.Error import System.Posix.Env import qualified Control.Exception.Extensible as E +import Control.Exception (throw) import qualified GitRepo as Git import qualified Locations @@ -87,10 +88,11 @@ test_unannex :: Test test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] where nocopy = "no content" ~: intmpclonerepo $ do - r <- git_annex "unannex" ["-q", annexedfile] - not r @? "unannex incorrectly succeeded with no copy" - unannexed annexedfile + annexed_notpresent annexedfile + git_annex "unannex" ["-q", annexedfile] @? "unannex failed with no copy" + annexed_notpresent annexedfile withcopy = "with content" ~: intmpcopyrepo $ do + annexed_present annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed" unannexed annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" @@ -280,7 +282,7 @@ indir dir a = do (E.try (a)::IO (Either E.SomeException ())) case r of Right () -> return () - Left e -> error $ show e + Left e -> throw e setuprepo :: FilePath -> IO FilePath setuprepo dir = do From a78b0555e1d46c4548cda3aaa1709040f6fa7f33 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 15:54:14 -0400 Subject: [PATCH 0702/8313] New migrate subcommand can be used to switch files to using a different backend, safely and with no duplication of content. --- Command.hs | 6 +++++ Command/Add.hs | 5 ++-- Command/Migrate.hs | 63 ++++++++++++++++++++++++++++++++++++++++++++ GitAnnex.hs | 2 ++ TypeInternals.hs | 3 +++ debian/changelog | 7 ++--- doc/git-annex.mdwn | 8 ++++++ doc/walkthrough.mdwn | 35 +++++++++++++++--------- 8 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 Command/Migrate.hs diff --git a/Command.hs b/Command.hs index 690dd20ecf..b83e640b9b 100644 --- a/Command.hs +++ b/Command.hs @@ -117,6 +117,12 @@ withAttrFilesInGit attr a params = do files' <- filterFiles files pairs <- liftIO $ Git.checkAttr repo attr files' return $ map a pairs +withBackendFilesInGit :: CommandSeekBackendFiles +withBackendFilesInGit a params = do + repo <- Annex.gitRepo + files <- liftIO $ Git.inRepo repo params + files' <- filterFiles files + backendPairs a files' withFilesMissing :: CommandSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params diff --git a/Command/Add.hs b/Command/Add.hs index bc869a67de..c74b726e3f 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -42,11 +42,12 @@ perform (file, backend) = do stored <- Backend.storeFileKey file backend case stored of Nothing -> return Nothing - Just (key, _) -> return $ Just $ cleanup file key + Just (key, _) -> do + moveAnnex key file + return $ Just $ cleanup file key cleanup :: FilePath -> Key -> CommandCleanup cleanup file key = do - moveAnnex key file logStatus key ValuePresent link <- calcGitLink file key diff --git a/Command/Migrate.hs b/Command/Migrate.hs new file mode 100644 index 0000000000..0caded6d13 --- /dev/null +++ b/Command/Migrate.hs @@ -0,0 +1,63 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Migrate where + +import Control.Monad.State (liftIO) +import System.Posix.Files +import System.Directory + +import Command +import qualified Annex +import qualified Backend +import Locations +import Types +import Core +import Messages +import qualified Command.Add + +command :: [Command] +command = [Command "migrate" paramPath seek "switch data to different backend"] + +seek :: [CommandSeek] +seek = [withBackendFilesInGit start] + +start :: CommandStartBackendFile +start (_, Nothing) = return Nothing +start (file, Just newbackend) = isAnnexed file $ \(key, oldbackend) -> do + exists <- inAnnex key + if (newbackend /= oldbackend) && exists + then do + showStart "migrate" file + return $ Just $ perform file key newbackend + else + return Nothing + +perform :: FilePath -> Key -> Backend -> CommandPerform +perform file oldkey newbackend = do + g <- Annex.gitRepo + + -- Store the old backend's cached key in the new backend + -- (the file can't be stored as usual, because it's already a symlink). + -- The old backend's key is not dropped from it, because there may + -- be other files still pointing at that key. + let src = annexLocation g oldkey + stored <- Backend.storeFileKey src $ Just newbackend + case stored of + Nothing -> return Nothing + Just (newkey, _) -> do + ok <- getViaTmp newkey $ \t -> do + -- Make a hard link to the old backend's + -- cached key, to avoid wasting disk space. + liftIO $ createLink src t + return True + if ok + then do + -- Update symlink to use the new key. + liftIO $ removeFile file + return $ Just $ Command.Add.cleanup file newkey + else return Nothing diff --git a/GitAnnex.hs b/GitAnnex.hs index 24c9ace0ac..d9efdad2dd 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -32,6 +32,7 @@ import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find +import qualified Command.Migrate import qualified Command.Uninit import qualified Command.Trust import qualified Command.Untrust @@ -59,6 +60,7 @@ cmds = concat , Command.Unused.command , Command.DropUnused.command , Command.Find.command + , Command.Migrate.command ] options :: [Option] diff --git a/TypeInternals.hs b/TypeInternals.hs index fe6e562f95..12a9080b33 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -103,3 +103,6 @@ data Backend = Backend { instance Show Backend where show backend = "Backend { name =\"" ++ name backend ++ "\" }" + +instance Eq Backend where + a == b = name a == name b diff --git a/debian/changelog b/debian/changelog index 7ca74f9945..85878113e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,11 @@ git-annex (0.17) UNRELEASED; urgency=low * unannex: Now skips files whose content is not present, rather than - it being an error. This allows gradual conversion from one backend - to another by running unannex followed by add in each repository. + it being an error. + * New migrate subcommand can be used to switch files to using a different + backend, safely and with no duplication of content. - -- Joey Hess Sat, 08 Jan 2011 15:04:48 -0400 + -- Joey Hess Sat, 08 Jan 2011 13:45:06 -0400 git-annex (0.16) unstable; urgency=low diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e99be4e409..6d106fea48 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -144,6 +144,14 @@ Many git-annex commands will stage changes for later `git commit` by you. With no parameters, defaults to finding all files in the current directory and its subdirectories. +* migrate [path ...] + + Changes the specified annexed files to store their content in the + default backend (or the one specified with --backend). + + Note that the content is not removed from the backend it was previously in. + Use `git annex unused` to find and remove such content. + * unannex [path ...] Use this to undo an accidental add command. This is not the command you diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 47f05ebcf4..d2231c81e6 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -277,25 +277,32 @@ add something like this to `.gitattributes`: * annex.backend=SHA1 -## migrating between backends +## migrating data to a new backend -Perhaps you had been using the WORM backend, but now have configured -git-annex to use SHA1 for new files. Your old files are still in WORM. How -to migrate that content? A quick and dirty way is to use the unannex -subcommand, which removes a file from git-annex's control, followed by -a re-add of the file, to put it in the new backend. +Maybe you started out using the WORM backend, and have now configured +git-annex to use SHA1. But files you added to the annex before still +use the WORM backend. There is a simple command that can migrate that +data: - # git annex unannex my_cool_big_file - unannex my_cool_big_file ok - # git annex add my_cool_big_file - add my_cool_big_file (checksum ...) ok + # git annex migrate my_cool_big_file + migrate my_cool_big_file (checksum...) ok + +You can only migrate files whose content is currently available. Other +files will be skipped. + +After migrating a file to a new backend, the old content in the old backend +will still be present. That is necessary because multiple files +can point to the same content. The `git annex unused` sucommand can be +used to clear up that detritus later. Note that hard links are used, +to avoid wasting disk space. ## unused data It's possible for data to accumulate in the annex that no files point to -nymore. One way it can happen is if you `git rm` a file without +anymore. One way it can happen is if you `git rm` a file without first calling `git annex drop`. And, when you modify an annexed file, the old -content of the file remains in the annex. +content of the file remains in the annex. Another way is when migrating +between backends. This might be historical data you want to preserve, so git-annex defaults to preserving it. So from time to time, you may want to check for such data and @@ -318,6 +325,10 @@ data anymore, you can easily remove it: # git annex dropunused 1 dropunused 1 ok +Hint: To drop a lot of unused data, use a command like this: + + # git annex dropunused `seq 1 1000` + ## fsck: verifying your data You can use the fsck subcommand to check for problems in your data. From 9de982eab7091aed9f3e75122372bbf0ac141763 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 8 Jan 2011 16:09:17 -0400 Subject: [PATCH 0703/8313] copyright years --- Command/Migrate.hs | 2 +- debian/copyright | 2 +- test.hs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 0caded6d13..3d61214377 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010 Joey Hess + - Copyright 2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} diff --git a/debian/copyright b/debian/copyright index 55638653c1..eceb2d9ffc 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,5 @@ Files: * -Copyright: © 2010 Joey Hess +Copyright: © 2010,2011 Joey Hess License: GPL-3+ The full text of the GPL is distributed as doc/GPL in this package's source, or in /usr/share/common-licenses/GPL on Debian systems. diff --git a/test.hs b/test.hs index 35085cc37e..8c9298e679 100644 --- a/test.hs +++ b/test.hs @@ -1,6 +1,6 @@ {- git-annex test suite - - - Copyright 2010 Joey Hess + - Copyright 2010,2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} From 292c2796a42167378573028b97c4de582d107dc8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmlUJkSWTkiWuwzwilXh1Wd3jg8I33iTQg" Date: Sun, 9 Jan 2011 13:16:53 +0000 Subject: [PATCH 0704/8313] --- doc/bugs/problem_commit_normal_links.mdwn | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 doc/bugs/problem_commit_normal_links.mdwn diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn new file mode 100644 index 0000000000..a669d85fdd --- /dev/null +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -0,0 +1,35 @@ +Dear All, + +thank you for this wonderful tool! + +I am having an issue when I try to commit a normal link + +diokletian*194-> mkdir test +diokletian*195-> cd test +diokletian*196-> git init +Initialized empty Git repository in /home/henrus/test/.git/ +diokletian*197-> git annex init new +init new [master (root-commit) 49f5f91] git-annex setup + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .gitattributes +[master 76496ff] git annex init + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .git-annex/uuid.log +ok +diokletian*198-> mkdir subdir +diokletian*199-> ln -s subdir link +diokletian*200-> git add link +diokletian*201-> git commit -m "ok" +[master f12f62d] ok + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 link +diokletian*202-> ln -s subdir/ link2 +diokletian*203-> git add link2 +diokletian*204-> git commit -m "not ok" +git-annex: Prelude.head: empty list + +The trailing slash seems to make a difference! + +Best Regards, + +Henrik From 51327c242f7087587884903632505552917b844a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmlUJkSWTkiWuwzwilXh1Wd3jg8I33iTQg" Date: Sun, 9 Jan 2011 13:17:59 +0000 Subject: [PATCH 0705/8313] --- doc/bugs/problem_commit_normal_links.mdwn | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn index a669d85fdd..bc96ec9380 100644 --- a/doc/bugs/problem_commit_normal_links.mdwn +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -5,27 +5,49 @@ thank you for this wonderful tool! I am having an issue when I try to commit a normal link diokletian*194-> mkdir test + diokletian*195-> cd test + diokletian*196-> git init + Initialized empty Git repository in /home/henrus/test/.git/ + diokletian*197-> git annex init new + init new [master (root-commit) 49f5f91] git-annex setup + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .gitattributes + [master 76496ff] git annex init + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 100644 .git-annex/uuid.log + ok + diokletian*198-> mkdir subdir + diokletian*199-> ln -s subdir link + diokletian*200-> git add link + diokletian*201-> git commit -m "ok" + [master f12f62d] ok + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 link + diokletian*202-> ln -s subdir/ link2 + diokletian*203-> git add link2 + diokletian*204-> git commit -m "not ok" + git-annex: Prelude.head: empty list The trailing slash seems to make a difference! From f4ddb580c83cba1c0e66fc5deb20bfc49ee4cf1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 10:04:16 -0400 Subject: [PATCH 0706/8313] bugfix: Fix crash caused by a symlink in the repo with link text ending in a "/". (Thanks Henrik for reporting.) --- Backend.hs | 3 ++- debian/changelog | 6 ++++-- doc/bugs/problem_commit_normal_links.mdwn | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Backend.hs b/Backend.hs index 8142e4707a..7e88ff3043 100644 --- a/Backend.hs +++ b/Backend.hs @@ -130,7 +130,8 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey bs l = + makekey _ [] = return Nothing + makekey bs l = do case maybeLookupBackendName bs bname of Nothing -> do unless (null kname || null bname) $ diff --git a/debian/changelog b/debian/changelog index 85878113e9..2fd6d9d81e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,11 +1,13 @@ -git-annex (0.17) UNRELEASED; urgency=low +git-annex (0.17) unstable; urgency=low * unannex: Now skips files whose content is not present, rather than it being an error. * New migrate subcommand can be used to switch files to using a different backend, safely and with no duplication of content. + * bugfix: Fix crash caused by a symlink in the repo with link text ending in + a "/". (Thanks Henrik for reporting.) - -- Joey Hess Sat, 08 Jan 2011 13:45:06 -0400 + -- Joey Hess Sun, 09 Jan 2011 10:04:11 -0400 git-annex (0.16) unstable; urgency=low diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn index bc96ec9380..11c9f7d8fb 100644 --- a/doc/bugs/problem_commit_normal_links.mdwn +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -55,3 +55,5 @@ The trailing slash seems to make a difference! Best Regards, Henrik + +> Thanks for the bug report. This is fixed in 0.17. --[[Joey]] From 53170b1a1496084923531046386794c092cc64b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 10:48:04 -0400 Subject: [PATCH 0707/8313] fix real underlaying cause, and fix test suite to not skip it --- Backend.hs | 1 - TypeInternals.hs | 10 +++------- debian/changelog | 3 +-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Backend.hs b/Backend.hs index 7e88ff3043..a417c7247b 100644 --- a/Backend.hs +++ b/Backend.hs @@ -130,7 +130,6 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey _ [] = return Nothing makekey bs l = do case maybeLookupBackendName bs bname of Nothing -> do diff --git a/TypeInternals.hs b/TypeInternals.hs index 12a9080b33..44db743faa 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -55,7 +55,7 @@ instance Read Key where readsPrec _ s = [(Key (b,k), "")] where l = split ":" s - b = head l + b = if null l then "" else head l k = join ":" $ drop 1 l -- for quickcheck @@ -67,13 +67,9 @@ instance Arbitrary Key where prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k - -- filter out empty key or backend names - -- also backend names will not contain colons - | null kname || null bname || elem ':' bname = True + -- backend names will never contain colons + | elem ':' (backendName k) = True | otherwise = k == (read $ show k) - where - bname = backendName k - kname = keyName k backendName :: Key -> BackendName backendName (Key (b,_)) = b diff --git a/debian/changelog b/debian/changelog index 2fd6d9d81e..9ae31edc7b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,8 +4,7 @@ git-annex (0.17) unstable; urgency=low it being an error. * New migrate subcommand can be used to switch files to using a different backend, safely and with no duplication of content. - * bugfix: Fix crash caused by a symlink in the repo with link text ending in - a "/". (Thanks Henrik for reporting.) + * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.) -- Joey Hess Sun, 09 Jan 2011 10:04:11 -0400 From 5ddabe1bf70cb86be71c4561b88c5d8964fbf515 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 11:01:59 -0400 Subject: [PATCH 0708/8313] release --- doc/news/version_0.12.mdwn | 6 ------ doc/news/version_0.17.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 doc/news/version_0.12.mdwn create mode 100644 doc/news/version_0.17.mdwn diff --git a/doc/news/version_0.12.mdwn b/doc/news/version_0.12.mdwn deleted file mode 100644 index e7fccd1b36..0000000000 --- a/doc/news/version_0.12.mdwn +++ /dev/null @@ -1,6 +0,0 @@ -git-annex 0.12 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Add --exclude option to exclude files from processing. - * mwdn2man: Fix a bug in newline supression. Closes: #[606578](http://bugs.debian.org/606578) - * Bugfix to git annex add of an unlocked file in a subdir. Closes: #[606579](http://bugs.debian.org/606579) - * Makefile: Add PREFIX variable."""]] \ No newline at end of file diff --git a/doc/news/version_0.17.mdwn b/doc/news/version_0.17.mdwn new file mode 100644 index 0000000000..ddb59046eb --- /dev/null +++ b/doc/news/version_0.17.mdwn @@ -0,0 +1,7 @@ +git-annex 0.17 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * unannex: Now skips files whose content is not present, rather than + it being an error. + * New migrate subcommand can be used to switch files to using a different + backend, safely and with no duplication of content. + * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.)"""]] \ No newline at end of file From fd3efc086a98e298faff458aa64c2fe811081c5f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 11:02:04 -0400 Subject: [PATCH 0709/8313] close bug --- doc/bugs/problem_commit_normal_links.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/problem_commit_normal_links.mdwn b/doc/bugs/problem_commit_normal_links.mdwn index 11c9f7d8fb..6dbd41fb4e 100644 --- a/doc/bugs/problem_commit_normal_links.mdwn +++ b/doc/bugs/problem_commit_normal_links.mdwn @@ -56,4 +56,4 @@ Best Regards, Henrik -> Thanks for the bug report. This is fixed in 0.17. --[[Joey]] +> Thanks for the bug report. This is fixed in 0.17. --[[Joey]] [[!tag done]] From 9de1192c574dc304f3e9dbfd01296f74c7ea5969 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 15:12:52 -0400 Subject: [PATCH 0710/8313] enable dpkg-mergechangelogs --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index b98b07d7d2..5d425843f2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -.git-annex/*.log merge=union +debian/changelog merge=dpkg-mergechangelogs From c2b13a88bf2e8bdeb126e4f13a5aeb0fbe09a0c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 9 Jan 2011 18:23:58 -0400 Subject: [PATCH 0711/8313] use concat --- Command.hs | 2 +- GitRepo.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Command.hs b/Command.hs index b83e640b9b..31bb1f1f9b 100644 --- a/Command.hs +++ b/Command.hs @@ -64,7 +64,7 @@ data Command = Command { prepCmd :: Command -> [String] -> Annex [Annex Bool] prepCmd Command { cmdseek = seek } params = do lists <- mapM (\s -> s params) seek - return $ map doCommand $ foldl (++) [] lists + return $ map doCommand $ concat lists {- Runs a command through the start, perform and cleanup stages -} doCommand :: CommandStart -> CommandCleanup diff --git a/GitRepo.hs b/GitRepo.hs index 07f243b661..533038fc86 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -438,7 +438,7 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\"" e_num c = showoctal $ ord c -- unicode character is decomposed to -- Word8s and each is shown in octal - e_utf c = foldl (++) "" $ map showoctal $ + e_utf c = concat $ map showoctal $ (encode [c] :: [Word8]) From 3a844b1f3c5eb85d4571857ac10aa8183996513a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 15:43:29 -0400 Subject: [PATCH 0712/8313] test locationlog contents too --- test.hs | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test.hs b/test.hs index 8c9298e679..20e8bc28ab 100644 --- a/test.hs +++ b/test.hs @@ -18,11 +18,15 @@ import System.Posix.Env import qualified Control.Exception.Extensible as E import Control.Exception (throw) +import qualified Annex +import qualified BackendList +import qualified Backend import qualified GitRepo as Git import qualified Locations import qualified Utility import qualified TypeInternals import qualified GitAnnex +import qualified LocationLog main :: IO () main = do @@ -359,6 +363,26 @@ checkdangling f = do Left _ -> return () -- expected; dangling link Right _ -> assertFailure $ f ++ " was not a dangling link as expected" +checklocationlog :: FilePath -> Bool -> Assertion +checklocationlog f expected = do + g <- Git.repoFromCwd + g' <- Git.configRead g + let thisuuid = Git.configGet g' "annex.uuid" "" + s <- Annex.new g BackendList.allBackends + r <- Annex.eval s $ Backend.lookupFile f + case r of + Just (k, _) -> do + uuids <- LocationLog.keyLocations g' k + assertEqual ("location log for " ++ f ++ " " ++ (show k) ++ " " ++ thisuuid) + expected (elem thisuuid uuids) + _ -> assertFailure $ f ++ " failed to look up key" + +inlocationlog :: FilePath -> Assertion +inlocationlog f = checklocationlog f True + +notinlocationlog :: FilePath -> Assertion +notinlocationlog f = checklocationlog f False + runchecks :: [FilePath -> Assertion] -> FilePath -> Assertion runchecks [] _ = return () runchecks (a:as) f = do @@ -366,10 +390,12 @@ runchecks (a:as) f = do runchecks as f annexed_notpresent :: FilePath -> Assertion -annexed_notpresent = runchecks [checklink, checkdangling, checkunwritable] +annexed_notpresent = runchecks + [checklink, checkdangling, checkunwritable, notinlocationlog] annexed_present :: FilePath -> Assertion -annexed_present = runchecks [checklink, checkcontent, checkunwritable] +annexed_present = runchecks + [checklink, checkcontent, checkunwritable, inlocationlog] unannexed :: FilePath -> Assertion unannexed = runchecks [checkregularfile, checkcontent, checkwritable] From a8ce30401dd69d1d203cfc33b791c1b4d175666e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 16:00:40 -0400 Subject: [PATCH 0713/8313] add checks that location log files are committed currently failing for move --to --- GitRepo.hs | 6 ++++++ LocationLog.hs | 1 + test.hs | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/GitRepo.hs b/GitRepo.hs index 533038fc86..ec363fe73b 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -38,6 +38,7 @@ module GitRepo ( inRepo, notInRepo, stagedFiles, + changedUnstagedFiles, checkAttr, decodeGitFile, encodeGitFile, @@ -249,6 +250,11 @@ stagedFiles repo l = pipeNullSplit repo $ ["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", "--"] ++ l +{- Returns a list of files that have unstaged changes. -} +changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath] +changedUnstagedFiles repo l = pipeNullSplit repo $ + ["diff", "--name-only", "-z", "--"] ++ l + {- Returns a list of the files in the specified locations that are staged - for commit, and whose type has changed. -} typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath] diff --git a/LocationLog.hs b/LocationLog.hs index e4dad03f5c..926939051c 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -23,6 +23,7 @@ module LocationLog ( LogStatus(..), logChange, + logFile, keyLocations ) where diff --git a/test.hs b/test.hs index 20e8bc28ab..0a5a365d97 100644 --- a/test.hs +++ b/test.hs @@ -375,6 +375,18 @@ checklocationlog f expected = do uuids <- LocationLog.keyLocations g' k assertEqual ("location log for " ++ f ++ " " ++ (show k) ++ " " ++ thisuuid) expected (elem thisuuid uuids) + + -- Location log files should always be checked + -- into git, and any modifications staged for + -- commit. This is a regression test, as some + -- commands forgot to. + let lf = LocationLog.logFile g' k + fs <- Git.inRepo g' [lf] + when (null fs) $ + assertFailure $ f ++ " logfile not added to git repo" + ufs <- Git.changedUnstagedFiles g' [lf] + when (not $ null ufs) $ + assertFailure $ f ++ " logfile changes not staged" _ -> assertFailure $ f ++ " failed to look up key" inlocationlog :: FilePath -> Assertion From 196c2fa78633aaaf9a43c8aa6a54b4144ffac3e2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 16:06:19 -0400 Subject: [PATCH 0714/8313] Bugfix: `copy --to` and `move --to` forgot to stage location log changes after transferring the file to the remote repository. (Did not affect ssh remotes.) --- Remotes.hs | 7 +++++-- debian/changelog | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Remotes.hs b/Remotes.hs index 2c8cdfdad1..a7a1db4152 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -252,8 +252,11 @@ copyToRemote r key -- run copy from perspective of remote liftIO $ do a <- Annex.new r [] - Annex.eval a $ Core.getViaTmp key $ \f -> - liftIO $ copyFile keysrc f + Annex.eval a $ do + ok <- Core.getViaTmp key $ + \f -> liftIO $ copyFile keysrc f + Annex.queueRun + return ok | Git.repoIsSsh r = do g <- Annex.gitRepo let keysrc = annexLocation g key diff --git a/debian/changelog b/debian/changelog index 9ae31edc7b..96acc592c9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +git-annex (0.18) UNRELEASED; urgency=low + + * Bugfix: `copy --to` and `move --to` forgot to stage location log changes + after transferring the file to the remote repository. + (Did not affect ssh remotes.) + + -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 + git-annex (0.17) unstable; urgency=low * unannex: Now skips files whose content is not present, rather than From 8d6da87eec87de5317185f7bb8ebf50013e41c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:13:26 -0400 Subject: [PATCH 0715/8313] better types --- Annex.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Annex.hs b/Annex.hs index 55f9edb36e..765c9191fb 100644 --- a/Annex.hs +++ b/Annex.hs @@ -51,9 +51,9 @@ new gitrepo allbackends = do Annex.gitRepoChange gitrepo' {- performs an action in the Annex monad -} -run :: AnnexState -> StateT AnnexState IO a -> IO (a, AnnexState) +run :: AnnexState -> Annex a -> IO (a, AnnexState) run state action = runStateT action state -eval :: AnnexState -> StateT AnnexState IO a -> IO a +eval :: AnnexState -> Annex a -> IO a eval state action = evalStateT action state {- Returns the git repository being acted on -} From 730c273eae055eaed743593b6ab7274e210bf4e3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:49:32 -0400 Subject: [PATCH 0716/8313] robustness fixes for empty or missing trust log --- UUID.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/UUID.hs b/UUID.hs index 21f2b202e6..67695d3101 100644 --- a/UUID.hs +++ b/UUID.hs @@ -145,8 +145,12 @@ getTrusted :: Annex [UUID] getTrusted = do logfile <- trustLog s <- liftIO $ catch (readFile logfile) ignoreerror - return $ map (\l -> head $ words l) $ lines s + return $ parse s where + parse [] = [] + parse s = map firstword $ lines s + firstword [] = "" + firstword l = head $ words l ignoreerror _ = return "" {- Changes the list of trusted UUIDs. -} From caa0b6c0c275ed85567112c273b49e7905bb4b14 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:49:45 -0400 Subject: [PATCH 0717/8313] quiet git commit messages --- Command/Trust.hs | 2 +- Command/Untrust.hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/Trust.hs b/Command/Trust.hs index 35ddefe842..c97d75ee47 100644 --- a/Command/Trust.hs +++ b/Command/Trust.hs @@ -40,5 +40,5 @@ perform repo = do g <- Annex.gitRepo logfile <- trustLog liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile] return $ Just $ return True diff --git a/Command/Untrust.hs b/Command/Untrust.hs index f49a2e989a..01b97b1c1f 100644 --- a/Command/Untrust.hs +++ b/Command/Untrust.hs @@ -40,5 +40,5 @@ perform repo = do g <- Annex.gitRepo logfile <- trustLog liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-m", "git annex untrust", logfile] + liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile] return $ Just $ return True From cebee3740142484f2061ebe323fec7b0f45fbd4c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 18:50:18 -0400 Subject: [PATCH 0718/8313] add tests for trust/untrust --- test.hs | 59 +++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/test.hs b/test.hs index 0a5a365d97..7e56fd65c1 100644 --- a/test.hs +++ b/test.hs @@ -17,6 +17,7 @@ import System.IO.Error import System.Posix.Env import qualified Control.Exception.Extensible as E import Control.Exception (throw) +import Control.Monad.State (liftIO) import qualified Annex import qualified BackendList @@ -27,6 +28,8 @@ import qualified Utility import qualified TypeInternals import qualified GitAnnex import qualified LocationLog +import qualified UUID +import qualified Remotes main :: IO () main = do @@ -64,6 +67,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_lock , test_edit , test_fix + , test_trust ] test_init :: Test @@ -244,6 +248,28 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do subdir = "s" newfile = subdir ++ "/" ++ annexedfile +test_trust :: Test +test_trust = "git-annex trust/untrust" ~: intmpclonerepo $ do + trust False + git_annex "trust" ["-q", "origin"] @? "trust failed" + trust True + git_annex "trust" ["-q", "origin"] @? "trust of trusted failed" + trust True + git_annex "untrust" ["-q", "origin"] @? "untrust failed" + trust False + git_annex "untrust" ["-q", "origin"] @? "untrust of untrusted failed" + trust False + where + trust expected = do + istrusted <- annexeval $ do + uuids <- UUID.getTrusted + r <- Remotes.byName "origin" + u <- UUID.getUUID r + return $ elem u uuids + assertEqual "trust value" expected istrusted + +-- This is equivilant to running git-annex, but it's all run in-process +-- so test coverage collection works. git_annex :: String -> [String] -> IO Bool git_annex command params = do -- catch all errors, including normally fatal errors @@ -254,6 +280,15 @@ git_annex command params = do where run = GitAnnex.run (command:params) +-- Runs an action in the current annex. Note that shutdown actions +-- are not run; this should only be used for actions that query state. +annexeval :: TypeInternals.Annex a -> IO a +annexeval a = do + g <- Git.repoFromCwd + g' <- Git.configRead g + s <- Annex.new g' BackendList.allBackends + Annex.eval s a + innewrepo :: Assertion -> Assertion innewrepo a = withgitrepo $ \r -> indir r a @@ -365,26 +400,30 @@ checkdangling f = do checklocationlog :: FilePath -> Bool -> Assertion checklocationlog f expected = do - g <- Git.repoFromCwd - g' <- Git.configRead g - let thisuuid = Git.configGet g' "annex.uuid" "" - s <- Annex.new g BackendList.allBackends - r <- Annex.eval s $ Backend.lookupFile f + thisuuid <- annexeval $ do + g <- Annex.gitRepo + UUID.getUUID g + r <- annexeval $ Backend.lookupFile f case r of Just (k, _) -> do - uuids <- LocationLog.keyLocations g' k - assertEqual ("location log for " ++ f ++ " " ++ (show k) ++ " " ++ thisuuid) + uuids <- annexeval $ do + g <- Annex.gitRepo + liftIO $ LocationLog.keyLocations g k + assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid) expected (elem thisuuid uuids) -- Location log files should always be checked -- into git, and any modifications staged for -- commit. This is a regression test, as some -- commands forgot to. - let lf = LocationLog.logFile g' k - fs <- Git.inRepo g' [lf] + (fs, ufs) <- annexeval $ do + g <- Annex.gitRepo + let lf = LocationLog.logFile g k + fs <- liftIO $ Git.inRepo g [lf] + ufs <- liftIO $ Git.changedUnstagedFiles g [lf] + return (fs, ufs) when (null fs) $ assertFailure $ f ++ " logfile not added to git repo" - ufs <- Git.changedUnstagedFiles g' [lf] when (not $ null ufs) $ assertFailure $ f ++ " logfile changes not staged" _ -> assertFailure $ f ++ " failed to look up key" From cc7db6f058f69a1a0f08221f46d2d12d3d3764bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 19:34:28 -0400 Subject: [PATCH 0719/8313] test fsck --- test.hs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test.hs b/test.hs index 7e56fd65c1..0c6e018842 100644 --- a/test.hs +++ b/test.hs @@ -30,6 +30,7 @@ import qualified GitAnnex import qualified LocationLog import qualified UUID import qualified Remotes +import qualified Core main :: IO () main = do @@ -68,6 +69,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_edit , test_fix , test_trust + , test_fsck ] test_init :: Test @@ -268,6 +270,20 @@ test_trust = "git-annex trust/untrust" ~: intmpclonerepo $ do return $ elem u uuids assertEqual "trust value" expected istrusted +test_fsck :: Test +test_fsck = "git-annex fsck" ~: intmpclonerepo $ do + git_annex "fsck" ["-q"] @? "fsck failed" + Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" + r <- git_annex "fsck" ["-q"] + not r @? "fsck failed to fail with numcopies unsatisfied" + Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" + + git_annex "get" ["-q", annexedfile] @? "get of file failed" + Core.allowWrite annexedfile + writeFile annexedfile (changedcontent annexedfile) + r <- git_annex "fsck" ["-q"] + not r @? "fsck failed to fail with corrupted file content" + -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. git_annex :: String -> [String] -> IO Bool From e2af0914faf487464046e0a60d20a638add1790d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 19:41:13 -0400 Subject: [PATCH 0720/8313] fsck: Fix bug in moving of corrupted files to .git/annex/bad/ --- Core.hs | 3 ++- debian/changelog | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Core.hs b/Core.hs index 08e2265920..d59120d673 100644 --- a/Core.hs +++ b/Core.hs @@ -173,7 +173,8 @@ moveBad key = do g <- Annex.gitRepo let src = annexLocation g key let dest = annexBadLocation g ++ takeFileName src - liftIO $ createDirectoryIfMissing True dest + liftIO $ createDirectoryIfMissing True (parentDir dest) + liftIO $ allowWrite (parentDir src) liftIO $ renameFile src dest liftIO $ removeDirectory (parentDir src) return dest diff --git a/debian/changelog b/debian/changelog index 96acc592c9..c826fb7d6e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.18) UNRELEASED; urgency=low * Bugfix: `copy --to` and `move --to` forgot to stage location log changes after transferring the file to the remote repository. (Did not affect ssh remotes.) + * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 From 485dbdd1a9b93274d04719b7a218a3a2728b5058 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 19:59:11 -0400 Subject: [PATCH 0721/8313] add tests of setkey/fromkey --- test.hs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/test.hs b/test.hs index 0c6e018842..b6edd8f138 100644 --- a/test.hs +++ b/test.hs @@ -18,6 +18,7 @@ import System.Posix.Env import qualified Control.Exception.Extensible as E import Control.Exception (throw) import Control.Monad.State (liftIO) +import Maybe import qualified Annex import qualified BackendList @@ -31,6 +32,7 @@ import qualified LocationLog import qualified UUID import qualified Remotes import qualified Core +import qualified Backend.SHA1 main :: IO () main = do @@ -60,6 +62,7 @@ toplevels = TestLabel "toplevel" $ TestList -- test order matters, later tests may rely on state from earlier [ test_init , test_add + , test_setkey , test_unannex , test_drop , test_get @@ -94,6 +97,17 @@ test_add = "git-annex add" ~: TestCase $ inmainrepo $ do git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" unannexed ingitfile +test_setkey :: Test +test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do + writeFile tmp $ content sha1annexedfile + r <- annexeval $ TypeInternals.getKey Backend.SHA1.backend tmp + let sha1 = TypeInternals.keyName $ fromJust r + git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" + git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" + annexed_present sha1annexedfile + where + tmp = "tmpfile" + test_unannex :: Test test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] where @@ -281,8 +295,8 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get of file failed" Core.allowWrite annexedfile writeFile annexedfile (changedcontent annexedfile) - r <- git_annex "fsck" ["-q"] - not r @? "fsck failed to fail with corrupted file content" + r' <- git_annex "fsck" ["-q"] + not r' @? "fsck failed to fail with corrupted file content" -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. @@ -494,6 +508,9 @@ tmprepodir = tmpdir ++ "/tmprepo" annexedfile :: String annexedfile = "foo" +sha1annexedfile :: String +sha1annexedfile = "sha1foo" + ingitfile :: String ingitfile = "bar" @@ -501,6 +518,7 @@ content :: FilePath -> String content f | f == annexedfile = "annexed file content" | f == ingitfile = "normal file content" + | f == sha1annexedfile ="sha1 annexed file content" | otherwise = "unknown file " ++ f changecontent :: FilePath -> IO () From 098168559d13d0de4c881015ad251d339ac9b562 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 20:06:15 -0400 Subject: [PATCH 0722/8313] more fsck checks 72% coverage --- test.hs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test.hs b/test.hs index b6edd8f138..e36ab0955b 100644 --- a/test.hs +++ b/test.hs @@ -104,6 +104,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do let sha1 = TypeInternals.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" annexed_present sha1annexedfile where tmp = "tmpfile" @@ -291,12 +292,16 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do r <- git_annex "fsck" ["-q"] not r @? "fsck failed to fail with numcopies unsatisfied" Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" - - git_annex "get" ["-q", annexedfile] @? "get of file failed" - Core.allowWrite annexedfile - writeFile annexedfile (changedcontent annexedfile) - r' <- git_annex "fsck" ["-q"] - not r' @? "fsck failed to fail with corrupted file content" + corrupt annexedfile + corrupt sha1annexedfile + where + corrupt f = do + git_annex "get" ["-q", f] @? "get of file failed" + Core.allowWrite f + writeFile f (changedcontent f) + r <- git_annex "fsck" ["-q"] + not r @? "fsck failed to fail with corrupted file content" + git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem" -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. From 868486c6b6bad4614dcc6cad5a2e29f85833f967 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 20:08:15 -0400 Subject: [PATCH 0723/8313] update --- debian/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/changelog b/debian/changelog index c826fb7d6e..74fa4894a9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.18) UNRELEASED; urgency=low after transferring the file to the remote repository. (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * Test suite improvements. Current top-level test coverage: 72% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 From b13039d62eb3a99606562f2edca7405b96776787 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 20:48:58 -0400 Subject: [PATCH 0724/8313] test precommit; 74% --- test.hs | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/test.hs b/test.hs index e36ab0955b..28209d099e 100644 --- a/test.hs +++ b/test.hs @@ -231,19 +231,29 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do (not r) @? "drop wrongly succeeded with no known copy of modified file" test_edit :: Test -test_edit = "git-annex edit/commit" ~: intmpclonerepo $ do - git_annex "get" ["-q", annexedfile] @? "get of file failed" - annexed_present annexedfile - git_annex "edit" ["-q", annexedfile] @? "edit failed" - unannexed annexedfile - changecontent annexedfile - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] - @? "git commit of edited file failed" - runchecks [checklink, checkunwritable] annexedfile - c <- readFile annexedfile - assertEqual ("content of modified file") c (changedcontent annexedfile) - r <- git_annex "drop" ["-q", annexedfile] - (not r) @? "drop wrongly succeeded with no known copy of modified file" +test_edit = "git-annex edit/commit" ~: TestList [t False, t True] + where t precommit = TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get of file failed" + annexed_present annexedfile + git_annex "edit" ["-q", annexedfile] @? "edit failed" + unannexed annexedfile + changecontent annexedfile + if precommit + then do + -- pre-commit depends on the file being + -- staged, normally git commit does this + Utility.boolSystem "git" ["add", annexedfile] + @? "git add of edited file failed" + git_annex "pre-commit" ["-q"] + @? "pre-commit failed" + else do + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] + @? "git commit of edited file failed" + runchecks [checklink, checkunwritable] annexedfile + c <- readFile annexedfile + assertEqual ("content of modified file") c (changedcontent annexedfile) + r <- git_annex "drop" ["-q", annexedfile] + (not r) @? "drop wrongly succeeded with no known copy of modified file" test_fix :: Test test_fix = "git-annex fix" ~: intmpclonerepo $ do From b557a2ccf4cd634a2c276b9ae03b881814de58be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 21:11:32 -0400 Subject: [PATCH 0725/8313] test migrate; 75% --- debian/changelog | 2 +- test.hs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 74fa4894a9..6950bda20d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ git-annex (0.18) UNRELEASED; urgency=low after transferring the file to the remote repository. (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ - * Test suite improvements. Current top-level test coverage: 72% + * Test suite improvements. Current top-level test coverage: 75% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 28209d099e..716d1a0dee 100644 --- a/test.hs +++ b/test.hs @@ -73,6 +73,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_fix , test_trust , test_fsck + , test_migrate ] test_init :: Test @@ -313,6 +314,35 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do not r @? "fsck failed to fail with corrupted file content" git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem" +test_migrate :: Test +test_migrate = "git-annex migrate" ~: TestList [t False, t True] + where t usegitattributes = TestCase $ intmpclonerepo $ do + annexed_notpresent annexedfile + annexed_notpresent sha1annexedfile + git_annex "migrate" ["-q", annexedfile] @? "migrate of not present failed" + git_annex "migrate" ["-q", sha1annexedfile] @? "migrate of not present failed" + git_annex "get" ["-q", annexedfile] @? "get of file failed" + git_annex "get" ["-q", sha1annexedfile] @? "get of file failed" + annexed_present annexedfile + annexed_present sha1annexedfile + if usegitattributes + then do + writeFile ".gitattributes" "* annex.backend=SHA1" + git_annex "migrate" [sha1annexedfile] @? "migrate to same backend failed" + git_annex "migrate" [annexedfile] @? "migrate to different backend failed" + else do + git_annex "migrate" [sha1annexedfile, "--backend=SHA1"] @? "migrate to same backend failed" + git_annex "migrate" [annexedfile, "--backend=SHA1"] @? "migrate to different backend failed" + annexed_present annexedfile + annexed_present sha1annexedfile + backend annexedfile Backend.SHA1.backend + backend sha1annexedfile Backend.SHA1.backend + where + backend file expected = do + r <- annexeval $ Backend.lookupFile file + let b = snd $ fromJust r + assertEqual ("backend for " ++ file) expected b + -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. git_annex :: String -> [String] -> IO Bool From e18a4d566b4f205c8a60ddf79ce02ba023d34984 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 21:32:38 -0400 Subject: [PATCH 0726/8313] migrate: Fix support for --backend option. --- Command/Migrate.hs | 9 +++++++-- debian/changelog | 1 + test.hs | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 3d61214377..59ad36a2b4 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -27,15 +27,20 @@ seek :: [CommandSeek] seek = [withBackendFilesInGit start] start :: CommandStartBackendFile -start (_, Nothing) = return Nothing -start (file, Just newbackend) = isAnnexed file $ \(key, oldbackend) -> do +start (file, b) = isAnnexed file $ \(key, oldbackend) -> do exists <- inAnnex key + newbackend <- choosebackend b if (newbackend /= oldbackend) && exists then do showStart "migrate" file return $ Just $ perform file key newbackend else return Nothing + where + choosebackend Nothing = do + backends <- Backend.list + return $ head backends + choosebackend (Just backend) = return backend perform :: FilePath -> Key -> Backend -> CommandPerform perform file oldkey newbackend = do diff --git a/debian/changelog b/debian/changelog index 6950bda20d..510091b7bc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.18) UNRELEASED; urgency=low after transferring the file to the remote repository. (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * migrate: Fix support for --backend option. * Test suite improvements. Current top-level test coverage: 75% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 716d1a0dee..417d830e86 100644 --- a/test.hs +++ b/test.hs @@ -328,8 +328,8 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True] if usegitattributes then do writeFile ".gitattributes" "* annex.backend=SHA1" - git_annex "migrate" [sha1annexedfile] @? "migrate to same backend failed" - git_annex "migrate" [annexedfile] @? "migrate to different backend failed" + git_annex "migrate" ["-q", sha1annexedfile] @? "migrate to same backend failed" + git_annex "migrate" ["-q", annexedfile] @? "migrate to different backend failed" else do git_annex "migrate" [sha1annexedfile, "--backend=SHA1"] @? "migrate to same backend failed" git_annex "migrate" [annexedfile, "--backend=SHA1"] @? "migrate to different backend failed" From 611018e4bd1a79becdbb58846f8faa7464a57a01 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Jan 2011 23:02:21 -0400 Subject: [PATCH 0727/8313] I thought that reversion a migration might fail. It didn't. :) --- test.hs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/test.hs b/test.hs index 417d830e86..6a8784d38c 100644 --- a/test.hs +++ b/test.hs @@ -33,6 +33,7 @@ import qualified UUID import qualified Remotes import qualified Core import qualified Backend.SHA1 +import qualified Backend.WORM main :: IO () main = do @@ -327,18 +328,34 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True] annexed_present sha1annexedfile if usegitattributes then do - writeFile ".gitattributes" "* annex.backend=SHA1" - git_annex "migrate" ["-q", sha1annexedfile] @? "migrate to same backend failed" - git_annex "migrate" ["-q", annexedfile] @? "migrate to different backend failed" + writeFile ".gitattributes" $ "* annex.backend=SHA1" + git_annex "migrate" ["-q", sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" ["-q", annexedfile] + @? "migrate annexedfile failed" else do - git_annex "migrate" [sha1annexedfile, "--backend=SHA1"] @? "migrate to same backend failed" - git_annex "migrate" [annexedfile, "--backend=SHA1"] @? "migrate to different backend failed" + git_annex "migrate" ["-q", sha1annexedfile, "--backend", "SHA1"] + @? "migrate sha1annexedfile failed" + git_annex "migrate" ["-q", annexedfile, "--backend", "SHA1"] + @? "migrate annexedfile failed" annexed_present annexedfile annexed_present sha1annexedfile - backend annexedfile Backend.SHA1.backend - backend sha1annexedfile Backend.SHA1.backend + checkbackend annexedfile Backend.SHA1.backend + checkbackend sha1annexedfile Backend.SHA1.backend + + -- check that reversing a migration works + writeFile ".gitattributes" $ "* annex.backend=WORM" + git_annex "migrate" ["-q", sha1annexedfile] + @? "migrate sha1annexedfile failed" + git_annex "migrate" ["-q", annexedfile] + @? "migrate annexedfile failed" + annexed_present annexedfile + annexed_present sha1annexedfile + checkbackend annexedfile Backend.WORM.backend + checkbackend sha1annexedfile Backend.WORM.backend + where - backend file expected = do + checkbackend file expected = do r <- annexeval $ Backend.lookupFile file let b = snd $ fromJust r assertEqual ("backend for " ++ file) expected b From ba6727f66342e59b7e420f102548db5c9a49c6b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Jan 2011 01:57:32 -0400 Subject: [PATCH 0728/8313] always write log, so it's empty if nothing is unused --- Command/Unused.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Command/Unused.hs b/Command/Unused.hs index d2dfc9aa3e..62bc5d023d 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -38,13 +38,13 @@ checkUnused :: Annex Bool checkUnused = do showNote "checking for unused data..." unused <- unusedKeys + let list = number 1 unused + g <- Annex.gitRepo + liftIO $ writeFile (annexUnusedLog g) $ unlines $ + map (\(n, k) -> show n ++ " " ++ show k) list if null unused then return True else do - let list = number 1 unused - g <- Annex.gitRepo - liftIO $ writeFile (annexUnusedLog g) $ unlines $ - map (\(n, k) -> show n ++ " " ++ show k) list showLongNote $ w list return False where From bb4a45f9ce9ea3b5b024bc6e46ab61b7b493de9b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Jan 2011 01:57:49 -0400 Subject: [PATCH 0729/8313] avoid crashing if run before unused log is present --- Command/DropUnused.hs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index ea2ff46eba..cfd6fc3d0e 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -9,6 +9,7 @@ module Command.DropUnused where import Control.Monad.State (liftIO) import qualified Data.Map as M +import System.Directory import Command import Types @@ -39,8 +40,13 @@ start s = do readUnusedLog :: Annex (M.Map String Key) readUnusedLog = do g <- Annex.gitRepo - l <- liftIO $ readFile (annexUnusedLog g) - return $ M.fromList $ map parse $ lines l + let f = annexUnusedLog g + e <- liftIO $ doesFileExist f + if e + then do + l <- liftIO $ readFile f + return $ M.fromList $ map parse $ lines l + else return $ M.empty where parse line = (head ws, tokey $ unwords $ tail ws) where From 5869e7ccd532735f991c4f386aecf86c74ec7fc4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Jan 2011 01:58:23 -0400 Subject: [PATCH 0730/8313] test unused et al --- debian/changelog | 2 +- test.hs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 510091b7bc..ed0b8923e9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,7 +5,7 @@ git-annex (0.18) UNRELEASED; urgency=low (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ * migrate: Fix support for --backend option. - * Test suite improvements. Current top-level test coverage: 75% + * Test suite improvements. Current top-level test coverage: 80% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 6a8784d38c..326fa549f8 100644 --- a/test.hs +++ b/test.hs @@ -19,6 +19,7 @@ import qualified Control.Exception.Extensible as E import Control.Exception (throw) import Control.Monad.State (liftIO) import Maybe +import qualified Data.Map as M import qualified Annex import qualified BackendList @@ -34,6 +35,7 @@ import qualified Remotes import qualified Core import qualified Backend.SHA1 import qualified Backend.WORM +import qualified Command.DropUnused main :: IO () main = do @@ -75,6 +77,7 @@ toplevels = TestLabel "toplevel" $ TestList , test_trust , test_fsck , test_migrate + , test_unused ] test_init :: Test @@ -360,6 +363,39 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True] let b = snd $ fromJust r assertEqual ("backend for " ++ file) expected b +test_unused :: Test +test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do + -- keys have to be looked up before files are removed + annexedfilekey <- annexeval $ findkey annexedfile + sha1annexedfilekey <- annexeval $ findkey sha1annexedfile + git_annex "get" ["-q", annexedfile] @? "get of file failed" + git_annex "get" ["-q", sha1annexedfile] @? "get of file failed" + checkunused [] + Utility.boolSystem "git" ["rm", "-q", annexedfile] @? "git rm failed" + checkunused [annexedfilekey] + Utility.boolSystem "git" ["rm", "-q", sha1annexedfile] @? "git rm failed" + checkunused [annexedfilekey, sha1annexedfilekey] + + -- good opportunity to test dropkey also + git_annex "dropkey" ["-q", "--force", TypeInternals.keyName annexedfilekey] + @? "dropkey failed" + checkunused [sha1annexedfilekey] + + git_annex "dropunused" ["-q", "1", "2"] @? "dropunused failed" + checkunused [] + git_annex "dropunused" ["-q", "10", "501"] @? "dropunused failed on bogus numbers" + + where + checkunused expectedkeys = do + git_annex "unused" ["-q"] @? "unused failed" + unusedmap <- annexeval $ Command.DropUnused.readUnusedLog + let unusedkeys = M.elems unusedmap + assertEqual "unused keys differ" + (sort expectedkeys) (sort unusedkeys) + findkey f = do + r <- Backend.lookupFile f + return $ fst $ fromJust r + -- This is equivilant to running git-annex, but it's all run in-process -- so test coverage collection works. git_annex :: String -> [String] -> IO Bool From 2a67721d51918339570f9db2b41d413205307e97 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Jan 2011 18:53:15 -0400 Subject: [PATCH 0731/8313] DEP5 --- debian/copyright | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index eceb2d9ffc..692616a7c1 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,8 @@ +Format: http://dep.debian.net/deps/dep5/ +Source: native package + Files: * -Copyright: © 2010,2011 Joey Hess +Copyright: © 2010-2011 Joey Hess License: GPL-3+ The full text of the GPL is distributed as doc/GPL in this package's source, or in /usr/share/common-licenses/GPL on Debian systems. From 23686cc63d38faf7fb4d66ece8cbeb5214b07e5f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Jan 2011 23:36:58 -0400 Subject: [PATCH 0732/8313] clarify default values when no path is specified Much of the code to handle this was unnecessary, as git ls-files is used, and defaults to returning all files of the desired type. --- Command.hs | 13 ------------- Command/Find.hs | 2 +- Command/Fsck.hs | 2 +- Command/Uninit.hs | 2 +- doc/git-annex.mdwn | 4 +++- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Command.hs b/Command.hs index 31bb1f1f9b..9fafb18ef9 100644 --- a/Command.hs +++ b/Command.hs @@ -174,19 +174,6 @@ backendPairs a files = do pairs <- Backend.chooseBackends files return $ map a pairs -{- Default to acting on all files matching the seek action if - - none are specified. -} -withAll :: (a -> CommandSeek) -> a -> CommandSeek -withAll w a [] = do - g <- Annex.gitRepo - w a [Git.workTree g] -withAll w a p = w a p - -{- Provides a default parameter to act on if none is specified. -} -withDefault :: String-> (a -> CommandSeek) -> (a -> CommandSeek) -withDefault d w a [] = w a [d] -withDefault _ w a p = w a p - {- Filter out files from the state directory, and those matching the - exclude glob pattern, if it was specified. -} filterFiles :: [FilePath] -> Annex [FilePath] diff --git a/Command/Find.hs b/Command/Find.hs index 7cb781ce8c..6d94ea3f49 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -18,7 +18,7 @@ command = [Command "find" (paramOptional $ paramRepeating paramPath) seek "lists available files"] seek :: [CommandSeek] -seek = [withDefault "." withFilesInGit start] +seek = [withFilesInGit start] {- Output a list of files. -} start :: CommandStartString diff --git a/Command/Fsck.hs b/Command/Fsck.hs index d870bd4198..662c281c27 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -18,7 +18,7 @@ command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek "check for problems"] seek :: [CommandSeek] -seek = [withAll (withAttrFilesInGit "annex.numcopies") start] +seek = [withAttrFilesInGit "annex.numcopies" start] {- Checks a file's backend data for problems. -} start :: CommandStartAttrFile diff --git a/Command/Uninit.hs b/Command/Uninit.hs index 1a4e9b0d71..93465df373 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -25,7 +25,7 @@ command = [Command "uninit" paramPath seek "de-initialize git-annex and clean out repository"] seek :: [CommandSeek] -seek = [withAll withFilesInGit Command.Unannex.start, withNothing start] +seek = [withFilesInGit Command.Unannex.start, withNothing start] start :: CommandStartNothing start = do diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 6d106fea48..51a31ae159 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -59,7 +59,9 @@ content from the key-value store. Like many git commands, git-annex can be passed a path that is either a file or a directory. In the latter case it acts on all relevant -files in the directory. +files in the directory. If no path is specified, most git-annex commands +default to acting on all relevant files in the current directory (and +subdirectories). Many git-annex commands will stage changes for later `git commit` by you. From c1839fdccb286cb5e83f0cf2d1d2d8e15226b0eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 13 Jan 2011 23:46:02 -0400 Subject: [PATCH 0733/8313] unlock: Fix behavior when file content is not present. --- Command/Unlock.hs | 9 +++++++-- debian/changelog | 1 + test.hs | 10 ++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 0e55585ae3..7c1625bf06 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -12,6 +12,7 @@ import System.Directory hiding (copyFile) import Command import qualified Annex +import qualified Backend import Types import Messages import Locations @@ -31,8 +32,12 @@ seek = [withFilesInGit start] - content. -} start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do - showStart "unlock" file - return $ Just $ perform file key + inbackend <- Backend.hasKey key + if not inbackend + then return Nothing + else do + showStart "unlock" file + return $ Just $ perform file key perform :: FilePath -> Key -> CommandPerform perform dest key = do diff --git a/debian/changelog b/debian/changelog index ed0b8923e9..b899fc4b41 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (0.18) UNRELEASED; urgency=low (Did not affect ssh remotes.) * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ * migrate: Fix support for --backend option. + * unlock: Fix behavior when file content is not present. * Test suite improvements. Current top-level test coverage: 80% -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 diff --git a/test.hs b/test.hs index 326fa549f8..ee29855b48 100644 --- a/test.hs +++ b/test.hs @@ -215,6 +215,12 @@ test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do test_lock :: Test test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do + -- regression test: unlock of not present file should skip it + annexed_notpresent annexedfile + r <- git_annex "unlock" ["-q", annexedfile] + r @? "unlock failed with not present file" + annexed_notpresent annexedfile + git_annex "get" ["-q", annexedfile] @? "get of file failed" annexed_present annexedfile git_annex "unlock" ["-q", annexedfile] @? "unlock failed" @@ -232,8 +238,8 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do runchecks [checklink, checkunwritable] annexedfile c <- readFile annexedfile assertEqual ("content of modified file") c (changedcontent annexedfile) - r <- git_annex "drop" ["-q", annexedfile] - (not r) @? "drop wrongly succeeded with no known copy of modified file" + r' <- git_annex "drop" ["-q", annexedfile] + not r' @? "drop wrongly succeeded with no known copy of modified file" test_edit :: Test test_edit = "git-annex edit/commit" ~: TestList [t False, t True] From 59c9eda9624a9d35f44400104e8f022afb0f2f7e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Jan 2011 00:02:33 -0400 Subject: [PATCH 0734/8313] on second thought, unlock should fail if content is not present --- Command/Unlock.hs | 13 +++++++------ test.hs | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 7c1625bf06..4bd6e85998 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -7,6 +7,7 @@ module Command.Unlock where +import Control.Monad (when) import Control.Monad.State (liftIO) import System.Directory hiding (copyFile) @@ -32,15 +33,15 @@ seek = [withFilesInGit start] - content. -} start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do - inbackend <- Backend.hasKey key - if not inbackend - then return Nothing - else do - showStart "unlock" file - return $ Just $ perform file key + showStart "unlock" file + return $ Just $ perform file key perform :: FilePath -> Key -> CommandPerform perform dest key = do + inbackend <- Backend.hasKey key + when (not inbackend) $ + error "content not present" + g <- Annex.gitRepo let src = annexLocation g key liftIO $ removeFile dest diff --git a/test.hs b/test.hs index ee29855b48..2504bc7974 100644 --- a/test.hs +++ b/test.hs @@ -218,7 +218,7 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do -- regression test: unlock of not present file should skip it annexed_notpresent annexedfile r <- git_annex "unlock" ["-q", annexedfile] - r @? "unlock failed with not present file" + not r @? "unlock failed to fail with not present file" annexed_notpresent annexedfile git_annex "get" ["-q", annexedfile] @? "get of file failed" From 818111e0c2501a4e68419941655c10d657cc67d0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Jan 2011 15:10:13 -0400 Subject: [PATCH 0735/8313] releasing version 0.18 --- debian/changelog | 4 ++-- debian/copyright | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index b899fc4b41..9ca3ea82b0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.18) UNRELEASED; urgency=low +git-annex (0.18) unstable; urgency=low * Bugfix: `copy --to` and `move --to` forgot to stage location log changes after transferring the file to the remote repository. @@ -8,7 +8,7 @@ git-annex (0.18) UNRELEASED; urgency=low * unlock: Fix behavior when file content is not present. * Test suite improvements. Current top-level test coverage: 80% - -- Joey Hess Tue, 11 Jan 2011 16:05:25 -0400 + -- Joey Hess Fri, 14 Jan 2011 14:17:44 -0400 git-annex (0.17) unstable; urgency=low diff --git a/debian/copyright b/debian/copyright index 692616a7c1..90e26b7524 100644 --- a/debian/copyright +++ b/debian/copyright @@ -4,5 +4,6 @@ Source: native package Files: * Copyright: © 2010-2011 Joey Hess License: GPL-3+ - The full text of the GPL is distributed as doc/GPL in this package's - source, or in /usr/share/common-licenses/GPL on Debian systems. + The full text of version 3 of the GPL is distributed as doc/GPL in + this package's source, or in /usr/share/common-licenses/GPL-3 on + Debian systems. From 84836ed804633fa3d8ff50064331b8b90bb160dd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 14 Jan 2011 15:10:30 -0400 Subject: [PATCH 0736/8313] add news item for git-annex 0.18 --- doc/news/version_0.13.mdwn | 7 ------- doc/news/version_0.18.mdwn | 9 +++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) delete mode 100644 doc/news/version_0.13.mdwn create mode 100644 doc/news/version_0.18.mdwn diff --git a/doc/news/version_0.13.mdwn b/doc/news/version_0.13.mdwn deleted file mode 100644 index d4de64d3e2..0000000000 --- a/doc/news/version_0.13.mdwn +++ /dev/null @@ -1,7 +0,0 @@ -git-annex 0.13 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Makefile: Install man page and html (when built). - * Makefile: Add GHCFLAGS variable. - * Fix upgrade from 0.03. - * Support remotes using git+ssh and ssh+git as protocol. - Closes: #[607056](http://bugs.debian.org/607056)"""]] \ No newline at end of file diff --git a/doc/news/version_0.18.mdwn b/doc/news/version_0.18.mdwn new file mode 100644 index 0000000000..f2e0edb424 --- /dev/null +++ b/doc/news/version_0.18.mdwn @@ -0,0 +1,9 @@ +git-annex 0.18 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Bugfix: `copy --to` and `move --to` forgot to stage location log changes + after transferring the file to the remote repository. + (Did not affect ssh remotes.) + * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ + * migrate: Fix support for --backend option. + * unlock: Fix behavior when file content is not present. + * Test suite improvements. Current top-level test coverage: 80%"""]] \ No newline at end of file From e7b557ef5d347831142fd98eac901d79c7e1305d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 16 Jan 2011 16:05:05 -0400 Subject: [PATCH 0737/8313] got rid of Core module Most of it was to do with managing annexed Content, so put there --- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- CmdLine.hs | 56 +++++++++++++++++++++++++++++--- Command/Add.hs | 2 +- Command/Drop.hs | 2 +- Command/DropKey.hs | 2 +- Command/Find.hs | 2 +- Command/Fix.hs | 2 +- Command/FromKey.hs | 2 +- Command/Get.hs | 2 +- Command/InAnnex.hs | 2 +- Command/Migrate.hs | 2 +- Command/Move.hs | 2 +- Command/RecvKey.hs | 3 +- Command/SendKey.hs | 2 +- Command/SetKey.hs | 2 +- Command/Unannex.hs | 2 +- Command/Unlock.hs | 2 +- Command/Unused.hs | 13 +++++++- Core.hs => Content.hs | 75 +++++++++---------------------------------- Remotes.hs | 6 ++-- Upgrade.hs | 2 +- test.hs | 4 +-- 24 files changed, 104 insertions(+), 89 deletions(-) rename Core.hs => Content.hs (68%) diff --git a/Backend/File.hs b/Backend/File.hs index 073a7c2267..27b2a69015 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -22,7 +22,7 @@ import LocationLog import Locations import qualified Remotes import qualified GitRepo as Git -import Core +import Content import qualified Annex import UUID import Messages diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 68f7f683b5..2f3e2cf534 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -18,7 +18,7 @@ import TypeInternals import Messages import qualified Annex import Locations -import Core +import Content backend :: Backend backend = Backend.File.backend { diff --git a/Backend/WORM.hs b/Backend/WORM.hs index e9d8c42855..0c93012380 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -18,7 +18,7 @@ import qualified Backend.File import TypeInternals import Locations import qualified Annex -import Core +import Content import Messages backend :: Backend diff --git a/CmdLine.hs b/CmdLine.hs index fbcfb6405d..6772282c50 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -1,4 +1,4 @@ -{- git-annex command line parsing +{- git-annex command line parsing and dispatch - - Copyright 2010 Joey Hess - @@ -7,22 +7,27 @@ module CmdLine ( dispatch, - parseCmd, usage, + shutdown ) where +import System.IO.Error (try) import System.Console.GetOpt -import Control.Monad (when) import Control.Monad.State (liftIO) +import Control.Monad (when, unless) +import System.Directory import qualified Annex import qualified GitRepo as Git +import qualified GitQueue import Types import Command import BackendList -import Core import Upgrade import Options +import Messages +import UUID +import Locations {- Runs the passed command line. -} dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () @@ -68,3 +73,46 @@ usage header cmds options = indent l = " " ++ l pad n s = replicate (n - length s) ' ' longest f = foldl max 0 $ map (length . f) cmds + +{- Runs a list of Annex actions. Catches IO errors and continues + - (but explicitly thrown errors terminate the whole command). + - Runs shutdown and propigates an overall error status at the end. + -} +tryRun :: AnnexState -> [Annex Bool] -> IO () +tryRun state actions = tryRun' state 0 actions +tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () +tryRun' state errnum (a:as) = do + result <- try $ Annex.run state a + case result of + Left err -> do + Annex.eval state $ showErr err + tryRun' state (errnum + 1) as + Right (True,state') -> tryRun' state' errnum as + Right (False,state') -> tryRun' state' (errnum + 1) as +tryRun' state errnum [] = do + _ <- try $ Annex.run state $ shutdown errnum + when (errnum > 0) $ error $ show errnum ++ " failed" + +{- Actions to perform each time ran. -} +startup :: Annex Bool +startup = do + prepUUID + return True + +{- Cleanup actions. -} +shutdown :: Integer -> Annex () +shutdown errnum = do + q <- Annex.queueGet + unless (q == GitQueue.empty) $ do + showSideAction "Recording state in git..." + Annex.queueRun + + -- If nothing failed, clean up any files left in the temp directory, + -- but leave the directory itself. If something failed, temp files + -- are left behind to allow resuming on re-run. + when (errnum == 0) $ do + g <- Annex.gitRepo + let tmp = annexTmpLocation g + exists <- liftIO $ doesDirectoryExist tmp + when exists $ liftIO $ removeDirectoryRecursive tmp + liftIO $ createDirectoryIfMissing True tmp diff --git a/Command/Add.hs b/Command/Add.hs index c74b726e3f..4b49297fc6 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -15,7 +15,7 @@ import qualified Annex import qualified Backend import LocationLog import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Drop.hs b/Command/Drop.hs index a425c6138d..065e1743a1 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -13,7 +13,7 @@ import Command import qualified Backend import LocationLog import Types -import Core +import Content import Messages import Utility diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 29056139d3..6ba5c117c4 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -12,7 +12,7 @@ import qualified Annex import qualified Backend import LocationLog import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Find.hs b/Command/Find.hs index 6d94ea3f49..3ed15c1537 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -11,7 +11,7 @@ import Control.Monad (when) import Control.Monad.State (liftIO) import Command -import Core +import Content command :: [Command] command = [Command "find" (paramOptional $ paramRepeating paramPath) seek diff --git a/Command/Fix.hs b/Command/Fix.hs index 8b08a26f6d..d67eca164c 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -14,7 +14,7 @@ import System.Directory import Command import qualified Annex import Utility -import Core +import Content import Messages command :: [Command] diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 0a13b8c734..9c4a3cfdcb 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -17,7 +17,7 @@ import qualified Annex import Utility import qualified Backend import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Get.hs b/Command/Get.hs index e3668649ef..e0af6c4078 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -10,7 +10,7 @@ module Command.Get where import Command import qualified Backend import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index d49539513b..68ac9a2c67 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -12,7 +12,7 @@ import System.Exit import Command import Types -import Core +import Content import qualified Backend command :: [Command] diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 59ad36a2b4..5bc54ceab5 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -16,7 +16,7 @@ import qualified Annex import qualified Backend import Locations import Types -import Core +import Content import Messages import qualified Command.Add diff --git a/Command/Move.hs b/Command/Move.hs index 3e7fde3705..2920c06616 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -14,7 +14,7 @@ import qualified Command.Drop import qualified Annex import LocationLog import Types -import Core +import Content import qualified GitRepo as Git import qualified Remotes import UUID diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 840b328613..0abea07f20 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -13,7 +13,8 @@ import System.Exit import Command import Types -import Core +import CmdLine +import Content import qualified Backend import RsyncFile diff --git a/Command/SendKey.hs b/Command/SendKey.hs index 0ddc0d23b9..aaa0b48369 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -15,7 +15,7 @@ import Locations import qualified Annex import Command import Types -import Core +import Content import qualified Backend import RsyncFile diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 5048d052f0..412504b2ee 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -16,7 +16,7 @@ import Utility import qualified Backend import LocationLog import Types -import Core +import Content import Messages command :: [Command] diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 2c60a23bb9..cdd577ba8b 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -16,7 +16,7 @@ import Utility import qualified Backend import LocationLog import Types -import Core +import Content import qualified GitRepo as Git import Messages diff --git a/Command/Unlock.hs b/Command/Unlock.hs index 4bd6e85998..645fac8a25 100644 --- a/Command/Unlock.hs +++ b/Command/Unlock.hs @@ -17,7 +17,7 @@ import qualified Backend import Types import Messages import Locations -import Core +import Content import CopyFile command :: [Command] diff --git a/Command/Unused.hs b/Command/Unused.hs index 62bc5d023d..9fdf4cda65 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -9,13 +9,16 @@ module Command.Unused where import Control.Monad.State (liftIO) import qualified Data.Map as M +import Data.Maybe import Command import Types -import Core +import Content import Messages import Locations import qualified Annex +import qualified GitRepo as Git +import qualified Backend command :: [Command] command = [Command "unused" paramNothing seek "look for unused file content"] @@ -80,3 +83,11 @@ unusedKeys = do existsMap :: Ord k => [k] -> M.Map k Int existsMap l = M.fromList $ map (\k -> (k, 1)) l + +{- List of keys referenced by symlinks in the git repo. -} +getKeysReferenced :: Annex [Key] +getKeysReferenced = do + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g [Git.workTree g] + keypairs <- mapM Backend.lookupFile files + return $ map fst $ catMaybes keypairs diff --git a/Core.hs b/Content.hs similarity index 68% rename from Core.hs rename to Content.hs index d59120d673..0cbd6905cb 100644 --- a/Core.hs +++ b/Content.hs @@ -1,19 +1,30 @@ -{- git-annex core functions +{- git-annex file content managing - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Core where +module Content ( + inAnnex, + calcGitLink, + logStatus, + getViaTmp, + preventWrite, + allowWrite, + moveAnnex, + removeAnnex, + fromAnnex, + moveBad, + getKeysPresent +) where import System.IO.Error (try) import System.Directory import Control.Monad.State (liftIO) import System.Path -import Control.Monad (when, unless, filterM) +import Control.Monad (when, filterM) import System.Posix.Files -import Data.Maybe import System.FilePath import Types @@ -21,56 +32,8 @@ import Locations import LocationLog import UUID import qualified GitRepo as Git -import qualified GitQueue import qualified Annex -import qualified Backend import Utility -import Messages - -{- Runs a list of Annex actions. Catches IO errors and continues - - (but explicitly thrown errors terminate the whole command). - - Runs shutdown and propigates an overall error status at the end. - -} -tryRun :: AnnexState -> [Annex Bool] -> IO () -tryRun state actions = tryRun' state 0 actions -tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () -tryRun' state errnum (a:as) = do - result <- try $ Annex.run state a - case result of - Left err -> do - Annex.eval state $ showErr err - tryRun' state (errnum + 1) as - Right (True,state') -> tryRun' state' errnum as - Right (False,state') -> tryRun' state' (errnum + 1) as -tryRun' state errnum [] = do - _ <- try $ Annex.run state $ shutdown errnum - when (errnum > 0) $ error $ show errnum ++ " failed" - -{- Actions to perform each time ran. -} -startup :: Annex Bool -startup = do - prepUUID - return True - -{- When git-annex is done, it runs this. -} -shutdown :: Integer -> Annex Bool -shutdown errnum = do - q <- Annex.queueGet - unless (q == GitQueue.empty) $ do - showSideAction "Recording state in git..." - Annex.queueRun - - -- If nothing failed, clean up any files left in the temp directory, - -- but leave the directory itself. If something failed, temp files - -- are left behind to allow resuming on re-run. - when (errnum == 0) $ do - g <- Annex.gitRepo - let tmp = annexTmpLocation g - exists <- liftIO $ doesDirectoryExist tmp - when exists $ liftIO $ removeDirectoryRecursive tmp - liftIO $ createDirectoryIfMissing True tmp - - return True {- Checks if a given key is currently present in the annexLocation. -} inAnnex :: Key -> Annex Bool @@ -200,11 +163,3 @@ getKeysPresent' dir = do case result of Right s -> return $ isRegularFile s Left _ -> return False - -{- List of keys referenced by symlinks in the git repo. -} -getKeysReferenced :: Annex [Key] -getKeysReferenced = do - g <- Annex.gitRepo - files <- liftIO $ Git.inRepo g [Git.workTree g] - keypairs <- mapM Backend.lookupFile files - return $ map fst $ catMaybes keypairs diff --git a/Remotes.hs b/Remotes.hs index a7a1db4152..9004b33d00 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -34,7 +34,7 @@ import LocationLog import Locations import UUID import Utility -import qualified Core +import qualified Content import Messages import CopyFile import RsyncFile @@ -159,7 +159,7 @@ inAnnex r key = if Git.repoIsUrl r -- run a local check inexpensively, -- by making an Annex monad using the remote a <- Annex.new r [] - Annex.eval a (Core.inAnnex key) + Annex.eval a (Content.inAnnex key) checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" @@ -253,7 +253,7 @@ copyToRemote r key liftIO $ do a <- Annex.new r [] Annex.eval a $ do - ok <- Core.getViaTmp key $ + ok <- Content.getViaTmp key $ \f -> liftIO $ copyFile keysrc f Annex.queueRun return ok diff --git a/Upgrade.hs b/Upgrade.hs index 2e1708439b..596d525db2 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -14,7 +14,7 @@ import Control.Monad (filterM) import System.Posix.Files import System.FilePath -import Core +import Content import Types import Locations import qualified GitRepo as Git diff --git a/test.hs b/test.hs index 2504bc7974..b8b264f0cf 100644 --- a/test.hs +++ b/test.hs @@ -32,7 +32,7 @@ import qualified GitAnnex import qualified LocationLog import qualified UUID import qualified Remotes -import qualified Core +import qualified Content import qualified Backend.SHA1 import qualified Backend.WORM import qualified Command.DropUnused @@ -318,7 +318,7 @@ test_fsck = "git-annex fsck" ~: intmpclonerepo $ do where corrupt f = do git_annex "get" ["-q", f] @? "get of file failed" - Core.allowWrite f + Content.allowWrite f writeFile f (changedcontent f) r <- git_annex "fsck" ["-q"] not r @? "fsck failed to fail with corrupted file content" From d5c18d71ef643f8ab55f1d0d4444621ba364b13d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 18 Jan 2011 13:31:22 -0400 Subject: [PATCH 0738/8313] bug --- doc/bugs/tmp_file_handling.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/tmp_file_handling.mdwn diff --git a/doc/bugs/tmp_file_handling.mdwn b/doc/bugs/tmp_file_handling.mdwn new file mode 100644 index 0000000000..3c6c2a5976 --- /dev/null +++ b/doc/bugs/tmp_file_handling.mdwn @@ -0,0 +1,11 @@ +git-annex deletes all tmp files on shutdown, if everything succeeded. +This presents 2 problems: + +1. If git-annex is rsyncing something and another one is run, it will + delete the running instance's tmp files. +2. If a long-running rsync transfer is interrupted partway through, the + tmp file was expensive to obtain, and one needs to avoid running + git-annex to do anything else until that transfer can be resumed and + finished. + +--[[Joey]] From 1cc064d1a2b1686b0a45b38c53f8f9f922dd49e6 Mon Sep 17 00:00:00 2001 From: "http://jcftang.myopenid.com/" Date: Wed, 19 Jan 2011 11:00:17 +0000 Subject: [PATCH 0739/8313] --- ...figure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn new file mode 100644 index 0000000000..b7ec0d3883 --- /dev/null +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -0,0 +1 @@ +On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? From a2428e7bdc993478cc197497f4735a11076de4e9 Mon Sep 17 00:00:00 2001 From: "http://jcftang.myopenid.com/" Date: Wed, 19 Jan 2011 12:12:22 +0000 Subject: [PATCH 0740/8313] --- ...igure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn index b7ec0d3883..83d1ae6641 100644 --- a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -1 +1 @@ -On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? +On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? - also uuidgen behaves differently from uuid on debian. From 1881b4d7fcb50d563d0e6890fc39dde0663d87a3 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 19 Jan 2011 17:43:16 +0000 Subject: [PATCH 0741/8313] small helper script for reflecting git-annex-add on rsync-style backups --- doc/forum/migration_to_git-annex_and_rsync.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/migration_to_git-annex_and_rsync.mdwn diff --git a/doc/forum/migration_to_git-annex_and_rsync.mdwn b/doc/forum/migration_to_git-annex_and_rsync.mdwn new file mode 100644 index 0000000000..51e03de0d9 --- /dev/null +++ b/doc/forum/migration_to_git-annex_and_rsync.mdwn @@ -0,0 +1,13 @@ +When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached [[migrate.py]], and it saved me roughly a day of backuping. + +A note on terminology: "migrating" here means migrating from not using git-annex at all to using it, not to the ``git annex migrate`` command, for which a similar but different solution may be created. + +**WARNING**: This is a quickly hacked-together script. It worked for me, but is untested apart from that. It's just a dozen lines of code, so have a look at it and make sure you understand what it does, and what migrate.sh looks like. Take special care as this tampers with your backups, and if something goes wrong, well... + +First, have an up-to-date backup; then, git annex init / add etc as described in the [[walkthrough]]. In the directory in which you use git-annex, run: + + $ python migrate.py > migrate.sh + +Then copy the resulting migrate.sh to the equivalent location inside your backups and run it there. It will move all files that are now symlinked on the master to their new positions according to the symlinks (inside .git/annex/objects), but not create the symlinks (you will do a backup later anyway). + +After that, do a backup as usual. As rsync sees the moved files at their new locations, it will accept them and not duplicate the data. From 27325f212bfdf915d16eadfa9fc51b416d4177c0 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 19 Jan 2011 17:46:08 +0000 Subject: [PATCH 0742/8313] no attachments w/o admin, copy/pasting instead --- .../migration_to_git-annex_and_rsync.mdwn | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/forum/migration_to_git-annex_and_rsync.mdwn b/doc/forum/migration_to_git-annex_and_rsync.mdwn index 51e03de0d9..d99dab8728 100644 --- a/doc/forum/migration_to_git-annex_and_rsync.mdwn +++ b/doc/forum/migration_to_git-annex_and_rsync.mdwn @@ -1,4 +1,4 @@ -When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached [[migrate.py]], and it saved me roughly a day of backuping. +When migrating large file repositories to git-annex that are backuped in a way that uses an rsync-style mechanism (e.g. [dirvish](http://www.dirvish.org/)) and thus keeps incremental backups small by using hardlinks, space can be saved by manually reflecting the migration on the backup. So, instead of making a last pre-git-annex backup, migrating, and duplicating all backupped data with the next backup, I used the attached migrate.py file below, and it saved me roughly a day of backuping. A note on terminology: "migrating" here means migrating from not using git-annex at all to using it, not to the ``git annex migrate`` command, for which a similar but different solution may be created. @@ -11,3 +11,23 @@ First, have an up-to-date backup; then, git annex init / add etc as described in Then copy the resulting migrate.sh to the equivalent location inside your backups and run it there. It will move all files that are now symlinked on the master to their new positions according to the symlinks (inside .git/annex/objects), but not create the symlinks (you will do a backup later anyway). After that, do a backup as usual. As rsync sees the moved files at their new locations, it will accept them and not duplicate the data. + +**migrate.py**: + + #!/usr/bin/env python + + import os + from pipes import quote + + print "#!/bin/sh" + print "set -e" + print "" + + for (dirpath, dirnames, filenames) in os.walk("."): + for f in filenames: + fn = os.path.join(dirpath, f) + if os.path.islink(fn): + link = os.path.normpath(os.path.join(dirpath, os.readlink(fn))) + assert link.startswith(".git/annex/objects/") + print "mkdir -p %s"%quote(os.path.dirname(link)) + print "mv %s %s"%(quote(fn), quote(link)) From dbb76c22d0f4f979fe90eeeff233dbbbfcf2346d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 19 Jan 2011 18:08:50 -0400 Subject: [PATCH 0743/8313] Support using the uuidgen command if the uuid command is not available. --- UUID.hs | 10 +- configure.hs | 96 ++++++++++++------- debian/changelog | 6 ++ ...d_detect_uuidgen_instead_of_just_uuid.mdwn | 7 +- doc/install.mdwn | 1 + 5 files changed, 82 insertions(+), 38 deletions(-) diff --git a/UUID.hs b/UUID.hs index 67695d3101..26a64523fc 100644 --- a/UUID.hs +++ b/UUID.hs @@ -33,6 +33,7 @@ import Types import Locations import qualified Annex import Utility +import qualified SysConfig type UUID = String @@ -42,7 +43,14 @@ configkey="annex.uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} genUUID :: IO UUID -genUUID = liftIO $ pOpen ReadFromPipe "uuid" ["-m"] $ \h -> hGetLine h +genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h + where + command = SysConfig.uuid + params = if (command == "uuid") + -- request a random uuid be generated + then ["-m"] + -- uuidgen generates random uuid by default + else [] {- Looks up a repo's UUID. May return "" if none is known. - diff --git a/configure.hs b/configure.hs index 1abdc8914d..9f50328d33 100644 --- a/configure.hs +++ b/configure.hs @@ -5,24 +5,33 @@ import System.Cmd import System.Exit import System.Directory -type Test = IO Bool -data TestCase = TestCase String String Test -data Config = Config String Bool +type ConfigKey = String +data ConfigValue = BoolConfig Bool | StringConfig String +data Config = Config ConfigKey ConfigValue + +type Test = IO Config +type TestName = String +data TestCase = TestCase TestName Test instance Show Config where show (Config key value) = unlines [ - key ++ " :: Bool" - , key ++ " = " ++ show value + key ++ " :: " ++ valuetype value + , key ++ " = " ++ showvalue value ] + where + valuetype (BoolConfig _) = "Bool" + valuetype (StringConfig _) = "String" + showvalue (BoolConfig b) = show b + showvalue (StringConfig s) = show s tests :: [TestCase] tests = [ - TestCase "cp -a" "cp_a" $ testCp "-a" - , TestCase "cp -p" "cp_p" $ testCp "-p" - , TestCase "cp --reflink=auto" "cp_reflink_auto" $ testCp "--reflink=auto" - , TestCase "uuid" "uuid" $ requireCmd "uuid" "uuid" - , TestCase "xargs -0" "xargs_0" $ requireCmd "xargs -0" "xargs -0 /dev/null" + TestCase "cp -a" $ testCp "cp_a" "-a" + , TestCase "cp -p" $ testCp "cp_p" "-p" + , TestCase "cp --reflink=auto" $ testCp "cp_reflink_auto" "--reflink=auto" + , TestCase "uuid" $ selectCmd "uuid" ["uuid", "uuidgen"] + , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0" "xargs -0 /dev/null" ] tmpDir :: String @@ -31,34 +40,49 @@ tmpDir = "tmp" testFile :: String testFile = tmpDir ++ "/testfile" +requireCmd :: ConfigKey -> String -> String -> Test +requireCmd k c cmdline = do + ret <- testCmd k cmdline + handle ret + where + handle r@(Config _ (BoolConfig True)) = return r + handle r = do + testEnd r + error $ "** the " ++ c ++ " command is required to use git-annex" + +testCp :: ConfigKey -> String -> Test +testCp k option = testCmd k $ + "cp " ++ option ++ " " ++ testFile ++ " " ++ testFile ++ ".new" + +testCmd :: ConfigKey -> String -> Test +testCmd k c = do + ret <- system $ quiet c + return $ Config k (BoolConfig $ ret == ExitSuccess) + +selectCmd :: ConfigKey -> [String] -> Test +selectCmd k cmds = search cmds + where + search [] = do + testEnd $ Config k (BoolConfig False) + error $ "* need one of these commands, but none are available: " ++ show cmds + search (c:cs) = do + ret <- system $ quiet c + if (ret == ExitSuccess) + then return $ Config k (StringConfig c) + else search cs + quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" -requireCmd :: String -> String -> Test -requireCmd c cmdline = do - ret <- testCmd $ quiet cmdline - if ret - then return True - else do - testEnd False - error $ "** the " ++ c ++ " command is required to use git-annex" - -testCp :: String -> Test -testCp option = testCmd $ quiet $ "cp " ++ option ++ " " ++ testFile ++ - " " ++ testFile ++ ".new" - -testCmd :: String -> Test -testCmd c = do - ret <- system c - return $ ret == ExitSuccess - -testStart :: String -> IO () +testStart :: TestName -> IO () testStart s = do putStr $ " checking " ++ s ++ "..." hFlush stdout -testEnd :: Bool -> IO () -testEnd r = putStrLn $ " " ++ show r +testEnd :: Config -> IO () +testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" +testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" +testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s writeSysConfig :: [Config] -> IO () writeSysConfig config = writeFile "SysConfig.hs" body @@ -73,12 +97,12 @@ writeSysConfig config = writeFile "SysConfig.hs" body runTests :: [TestCase] -> IO [Config] runTests [] = return [] -runTests ((TestCase tname key t):ts) = do +runTests ((TestCase tname t):ts) = do testStart tname - val <- t - testEnd val + c <- t + testEnd c rest <- runTests ts - return $ (Config key val):rest + return $ c:rest setup :: IO () setup = do diff --git a/debian/changelog b/debian/changelog index 9ca3ea82b0..23358486d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.19) UNRELEASED; urgency=low + + * Support using the uuidgen command if the uuid command is not available. + + -- Joey Hess Wed, 19 Jan 2011 18:07:51 -0400 + git-annex (0.18) unstable; urgency=low * Bugfix: `copy --to` and `move --to` forgot to stage location log changes diff --git a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn index 83d1ae6641..2b9c773678 100644 --- a/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn +++ b/doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn @@ -1 +1,6 @@ -On RHEL5 (and clones) systems uuidgen is available as an alternative to uuid, the configure script fails, it should probably detect either uuid or uuidgen, or let the user decide? - also uuidgen behaves differently from uuid on debian. +On RHEL5 (and clones) systems uuidgen is available as an alternative to +uuid, the configure script fails, it should probably detect either uuid or +uuidgen, or let the user decide? - also uuidgen behaves differently from +uuid on debian. + +> uuidgen is now supported. --[[Joey]] [[done]] diff --git a/doc/install.mdwn b/doc/install.mdwn index bad1d9f258..732660c507 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -5,6 +5,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * `uuid`: + (or uuidgen from util-linux) * `xargs`: * `rsync`: * Then just [[download]] git-annex and run: `make; make install` From 4465689cc22564e650be4bad759006d587b41307 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 19 Jan 2011 20:02:48 -0400 Subject: [PATCH 0744/8313] refactor --- Makefile | 2 +- TestConfig.hs | 93 ++++++++++++++++++++++++++++++++++++++++++++++ configure.hs | 100 ++++++-------------------------------------------- 3 files changed, 105 insertions(+), 90 deletions(-) create mode 100644 TestConfig.hs diff --git a/Makefile b/Makefile index 44d4be02be..831c004dc5 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ mans=git-annex.1 git-annex-shell.1 all: $(bins) $(mans) docs -SysConfig.hs: configure.hs +SysConfig.hs: configure.hs TestConfig.hs $(GHCMAKE) configure ./configure diff --git a/TestConfig.hs b/TestConfig.hs new file mode 100644 index 0000000000..5e59681ddf --- /dev/null +++ b/TestConfig.hs @@ -0,0 +1,93 @@ +{- Tests the system and generates SysConfig.hs. -} + +module TestConfig where + +import System.IO +import System.Cmd +import System.Exit + +type ConfigKey = String +data ConfigValue = BoolConfig Bool | StringConfig String +data Config = Config ConfigKey ConfigValue + +type Test = IO Config +type TestName = String +data TestCase = TestCase TestName Test + +instance Show ConfigValue where + show (BoolConfig b) = show b + show (StringConfig s) = show s + +instance Show Config where + show (Config key value) = unlines + [ key ++ " :: " ++ valuetype value + , key ++ " = " ++ show value + ] + where + valuetype (BoolConfig _) = "Bool" + valuetype (StringConfig _) = "String" + +writeSysConfig :: [Config] -> IO () +writeSysConfig config = writeFile "SysConfig.hs" body + where + body = unlines $ header ++ map show config ++ footer + header = [ + "{- Automatically generated. -}" + , "module SysConfig where" + , "" + ] + footer = [] + +runTests :: [TestCase] -> IO [Config] +runTests [] = return [] +runTests ((TestCase tname t):ts) = do + testStart tname + c <- t + testEnd c + rest <- runTests ts + return $ c:rest + +{- Tests that a command is available, aborting if not. -} +requireCmd :: ConfigKey -> String -> Test +requireCmd k cmdline = do + ret <- testCmd k cmdline + handle ret + where + handle r@(Config _ (BoolConfig True)) = return r + handle r = do + testEnd r + error $ "** the " ++ c ++ " command is required" + c = (words cmdline) !! 0 + +{- Checks if a command is available by running a command line. -} +testCmd :: ConfigKey -> String -> Test +testCmd k cmdline = do + ret <- system $ quiet cmdline + return $ Config k (BoolConfig $ ret == ExitSuccess) + +{- Ensures that one of a set of commands is available by running each in + - turn. The Config is set to the first one found. -} +selectCmd :: ConfigKey -> [String] -> Test +selectCmd k cmds = search cmds + where + search [] = do + testEnd $ Config k (BoolConfig False) + error $ "* need one of these commands, but none are available: " ++ show cmds + search (c:cs) = do + ret <- system $ quiet c + if (ret == ExitSuccess) + then return $ Config k (StringConfig c) + else search cs + +quiet :: String -> String +quiet s = s ++ " >/dev/null 2>&1" + +testStart :: TestName -> IO () +testStart s = do + putStr $ " checking " ++ s ++ "..." + hFlush stdout + +testEnd :: Config -> IO () +testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" +testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" +testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s diff --git a/configure.hs b/configure.hs index 9f50328d33..8d1c117a71 100644 --- a/configure.hs +++ b/configure.hs @@ -1,37 +1,17 @@ {- Checks system configuration and generates SysConfig.hs. -} -import System.IO -import System.Cmd -import System.Exit import System.Directory -type ConfigKey = String -data ConfigValue = BoolConfig Bool | StringConfig String -data Config = Config ConfigKey ConfigValue - -type Test = IO Config -type TestName = String -data TestCase = TestCase TestName Test - -instance Show Config where - show (Config key value) = unlines [ - key ++ " :: " ++ valuetype value - , key ++ " = " ++ showvalue value - ] - where - valuetype (BoolConfig _) = "Bool" - valuetype (StringConfig _) = "String" - showvalue (BoolConfig b) = show b - showvalue (StringConfig s) = show s +import TestConfig tests :: [TestCase] tests = [ - TestCase "cp -a" $ testCp "cp_a" "-a" - , TestCase "cp -p" $ testCp "cp_p" "-p" - , TestCase "cp --reflink=auto" $ testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid" $ selectCmd "uuid" ["uuid", "uuidgen"] - , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0" "xargs -0 /dev/null" + testCp "cp_a" "-a" + , testCp "cp_p" "-p" + , testCp "cp_reflink_auto" "--reflink=auto" + , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] + , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" ] tmpDir :: String @@ -40,69 +20,11 @@ tmpDir = "tmp" testFile :: String testFile = tmpDir ++ "/testfile" -requireCmd :: ConfigKey -> String -> String -> Test -requireCmd k c cmdline = do - ret <- testCmd k cmdline - handle ret +testCp :: ConfigKey -> String -> TestCase +testCp k option = TestCase cmd $ testCmd k run where - handle r@(Config _ (BoolConfig True)) = return r - handle r = do - testEnd r - error $ "** the " ++ c ++ " command is required to use git-annex" - -testCp :: ConfigKey -> String -> Test -testCp k option = testCmd k $ - "cp " ++ option ++ " " ++ testFile ++ " " ++ testFile ++ ".new" - -testCmd :: ConfigKey -> String -> Test -testCmd k c = do - ret <- system $ quiet c - return $ Config k (BoolConfig $ ret == ExitSuccess) - -selectCmd :: ConfigKey -> [String] -> Test -selectCmd k cmds = search cmds - where - search [] = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds - search (c:cs) = do - ret <- system $ quiet c - if (ret == ExitSuccess) - then return $ Config k (StringConfig c) - else search cs - -quiet :: String -> String -quiet s = s ++ " >/dev/null 2>&1" - -testStart :: TestName -> IO () -testStart s = do - putStr $ " checking " ++ s ++ "..." - hFlush stdout - -testEnd :: Config -> IO () -testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" -testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" -testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s - -writeSysConfig :: [Config] -> IO () -writeSysConfig config = writeFile "SysConfig.hs" body - where - body = unlines $ header ++ map show config ++ footer - header = [ - "{- Automatically generated by configure. -}" - , "module SysConfig where" - , "" - ] - footer = [] - -runTests :: [TestCase] -> IO [Config] -runTests [] = return [] -runTests ((TestCase tname t):ts) = do - testStart tname - c <- t - testEnd c - rest <- runTests ts - return $ c:rest + cmd = "cp " ++ option + run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new" setup :: IO () setup = do From 778966b4f41407078372185c57ba18ae0c578ef7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 18:54:27 -0400 Subject: [PATCH 0745/8313] improve man building --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 831c004dc5..565edfb0e0 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,10 @@ SysConfig.hs: configure.hs TestConfig.hs $(bins): SysConfig.hs $(GHCMAKE) $@ -git-annex.1: +git-annex.1: doc/git-annex.mdwn ./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1 -git-annex-shell.1: - ./mdwn2man git-annex 1 doc/git-annex-shell.mdwn > git-annex-shell.1 +git-annex-shell.1: doc/git-annex-shell.mdwn + ./mdwn2man git-annex-shell 1 doc/git-annex-shell.mdwn > git-annex-shell.1 install: all install -d $(DESTDIR)$(PREFIX)/bin From f8e303e1c9a9d976a4dc339295aa0f09eb997b7f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 18:54:34 -0400 Subject: [PATCH 0746/8313] document interaction of annex-ignore with --from/--to --- doc/git-annex.mdwn | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 51a31ae159..9d89413b96 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -295,8 +295,13 @@ Here are all the supported configuration settings. * `remote..annex-ignore` If set to `true`, prevents git-annex - from ever using this remote. This is, for example, useful if the - remote is a bare repository, which git-annex does not currently support. + from using this remote by default. (You can still request it be used + by the --from and --to options.) + + This is, for example, useful if the remote is a bare repository, + which git-annex does not currently support. Or, it could be used + if the network connection between two repositories is too slow + to be used normally. * `remote..annex-uuid` From 109a719b03dbeb70eb317be17f7e18567efa9dac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 21:02:34 -0400 Subject: [PATCH 0747/8313] parameterize Backend type This allows the Backend type to not depend on the Annex type, and so the Annex type can later be moved out of TypeInternals. --- Annex.hs | 11 ++++++----- Backend.hs | 22 +++++++++++----------- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- BackendList.hs | 2 +- Command.hs | 4 ++-- Command/Drop.hs | 2 +- Command/Fsck.hs | 2 +- Command/Get.hs | 2 +- Command/Migrate.hs | 2 +- Command/Unannex.hs | 2 +- TypeInternals.hs | 27 ++++++++++++++------------- 14 files changed, 43 insertions(+), 41 deletions(-) diff --git a/Annex.hs b/Annex.hs index 765c9191fb..a0de630874 100644 --- a/Annex.hs +++ b/Annex.hs @@ -33,14 +33,15 @@ import Types import qualified TypeInternals as Internals {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [Backend] -> IO AnnexState +new :: Git.Repo -> [Backend Annex] -> IO AnnexState new gitrepo allbackends = do let s = Internals.AnnexState { Internals.repo = gitrepo, Internals.backends = [], Internals.supportedBackends = allbackends, Internals.flags = M.empty, - Internals.repoqueue = GitQueue.empty + Internals.repoqueue = GitQueue.empty, + Internals.quiet = False } (_,s') <- Annex.run s prep return s' @@ -69,19 +70,19 @@ gitRepoChange r = do put state { Internals.repo = r } {- Returns the backends being used. -} -backends :: Annex [Backend] +backends :: Annex [Backend Annex] backends = do state <- get return (Internals.backends state) {- Sets the backends to use. -} -backendsChange :: [Backend] -> Annex () +backendsChange :: [Backend Annex] -> Annex () backendsChange b = do state <- get put state { Internals.backends = b } {- Returns the full list of supported backends. -} -supportedBackends :: Annex [Backend] +supportedBackends :: Annex [Backend Annex] supportedBackends = do state <- get return (Internals.supportedBackends state) diff --git a/Backend.hs b/Backend.hs index a417c7247b..caf50005ad 100644 --- a/Backend.hs +++ b/Backend.hs @@ -42,7 +42,7 @@ import qualified TypeInternals as Internals import Messages {- List of backends in the order to try them when storing a new key. -} -list :: Annex [Backend] +list :: Annex [Backend Annex] list = do l <- Annex.backends -- list is cached here if not $ null l @@ -64,12 +64,12 @@ list = do else map (lookupBackendName bs) $ words s {- Looks up a backend in a list. May fail if unknown. -} -lookupBackendName :: [Backend] -> String -> Backend +lookupBackendName :: [Backend Annex] -> String -> Backend Annex lookupBackendName bs s = case maybeLookupBackendName bs s of Just b -> b Nothing -> error $ "unknown backend " ++ s -maybeLookupBackendName :: [Backend] -> String -> Maybe Backend +maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex) maybeLookupBackendName bs s = if 1 /= length matches then Nothing @@ -77,14 +77,14 @@ maybeLookupBackendName bs s = where matches = filter (\b -> s == Internals.name b) bs {- Attempts to store a file in one of the backends. -} -storeFileKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend)) +storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex)) storeFileKey file trybackend = do bs <- list let bs' = case trybackend of Nothing -> bs Just backend -> backend:bs storeFileKey' bs' file -storeFileKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend)) +storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex)) storeFileKey' [] _ = return Nothing storeFileKey' (b:bs) file = do result <- (Internals.getKey b) file @@ -100,11 +100,11 @@ storeFileKey' (b:bs) file = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} -retrieveKeyFile :: Backend -> Key -> FilePath -> Annex Bool +retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} -removeKey :: Backend -> Key -> Maybe Int -> Annex Bool +removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool removeKey backend key numcopies = (Internals.removeKey backend) key numcopies {- Checks if a key is present in its backend. -} @@ -114,12 +114,12 @@ hasKey key = do (Internals.hasKey backend) key {- Checks a key's backend for problems. -} -fsckKey :: Backend -> Key -> Maybe Int -> Annex Bool +fsckKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} -lookupFile :: FilePath -> Annex (Maybe (Key, Backend)) +lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex)) lookupFile file = do bs <- Annex.supportedBackends tl <- liftIO $ try getsymlink @@ -147,7 +147,7 @@ lookupFile file = do {- Looks up the backends that should be used for each file in a list. - That can be configured on a per-file basis in the gitattributes file. -} -chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe Backend)] +chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))] chooseBackends fs = do g <- Annex.gitRepo bs <- Annex.supportedBackends @@ -155,7 +155,7 @@ chooseBackends fs = do return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs {- Returns the backend to use for a key. -} -keyBackend :: Key -> Annex Backend +keyBackend :: Key -> Annex (Backend Annex) keyBackend key = do bs <- Annex.supportedBackends return $ lookupBackendName bs $ backendName key diff --git a/Backend/File.hs b/Backend/File.hs index 27b2a69015..c8ddd59381 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -27,7 +27,7 @@ import qualified Annex import UUID import Messages -backend :: Backend +backend :: Backend Annex backend = Backend { name = mustProvide, getKey = mustProvide, diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 2f3e2cf534..e665e5da75 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -20,7 +20,7 @@ import qualified Annex import Locations import Content -backend :: Backend +backend :: Backend Annex backend = Backend.File.backend { name = "SHA1", getKey = keyValue, diff --git a/Backend/URL.hs b/Backend/URL.hs index 3eb7376e01..8ed354aed8 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -14,7 +14,7 @@ import TypeInternals import Utility import Messages -backend :: Backend +backend :: Backend Annex backend = Backend { name = "URL", getKey = keyValue, diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 0c93012380..cd4254e2bf 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -21,7 +21,7 @@ import qualified Annex import Content import Messages -backend :: Backend +backend :: Backend Annex backend = Backend.File.backend { name = "WORM", getKey = keyValue, diff --git a/BackendList.hs b/BackendList.hs index d1180d22f9..5ae78bcc7f 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -13,7 +13,7 @@ import qualified Backend.SHA1 import qualified Backend.URL import Types -allBackends :: [Backend] +allBackends :: [Backend Annex] allBackends = [ Backend.WORM.backend , Backend.SHA1.backend diff --git a/Command.hs b/Command.hs index 9fafb18ef9..06fc704bd9 100644 --- a/Command.hs +++ b/Command.hs @@ -43,7 +43,7 @@ type CommandCleanup = Annex Bool - functions. -} type CommandSeekStrings = CommandStartString -> CommandSeek type CommandStartString = String -> CommandStart -type BackendFile = (FilePath, Maybe Backend) +type BackendFile = (FilePath, Maybe (Backend Annex)) type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek type CommandStartBackendFile = BackendFile -> CommandStart type AttrFile = (FilePath, String) @@ -95,7 +95,7 @@ notAnnexed file a = do Just _ -> return Nothing Nothing -> a -isAnnexed :: FilePath -> ((Key, Backend) -> Annex (Maybe a)) -> Annex (Maybe a) +isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a) isAnnexed file a = do r <- Backend.lookupFile file case r of diff --git a/Command/Drop.hs b/Command/Drop.hs index 065e1743a1..fdc55969f0 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -37,7 +37,7 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> CommandPerform +perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.removeKey backend key numcopies if success diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 662c281c27..fc9bd7f527 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -28,7 +28,7 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do where numcopies = readMaybe attr :: Maybe Int -perform :: Key -> Backend -> Maybe Int -> CommandPerform +perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform perform key backend numcopies = do success <- Backend.fsckKey backend key numcopies if success diff --git a/Command/Get.hs b/Command/Get.hs index e0af6c4078..2aa3c0c150 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -30,7 +30,7 @@ start file = isAnnexed file $ \(key, backend) -> do showStart "get" file return $ Just $ perform key backend -perform :: Key -> Backend -> CommandPerform +perform :: Key -> Backend Annex -> CommandPerform perform key backend = do ok <- getViaTmp key (Backend.retrieveKeyFile backend key) if ok diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 5bc54ceab5..566b508c0c 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -42,7 +42,7 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do return $ head backends choosebackend (Just backend) = return backend -perform :: FilePath -> Key -> Backend -> CommandPerform +perform :: FilePath -> Key -> Backend Annex -> CommandPerform perform file oldkey newbackend = do g <- Annex.gitRepo diff --git a/Command/Unannex.hs b/Command/Unannex.hs index cdd577ba8b..4134439697 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -36,7 +36,7 @@ start file = isAnnexed file $ \(key, backend) -> do return $ Just $ perform file key backend else return Nothing -perform :: FilePath -> Key -> Backend -> CommandPerform +perform :: FilePath -> Key -> Backend Annex -> CommandPerform perform file key backend = do -- force backend to always remove ok <- Backend.removeKey backend key (Just 0) diff --git a/TypeInternals.hs b/TypeInternals.hs index 44db743faa..99f3049730 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -28,10 +28,11 @@ data Flag = -- but it uses Backend, so has to be here to avoid a depends loop. data AnnexState = AnnexState { repo :: Git.Repo, - backends :: [Backend], - supportedBackends :: [Backend], + backends :: [Backend Annex], + supportedBackends :: [Backend Annex], flags :: M.Map FlagName Flag, - repoqueue :: GitQueue.Queue + repoqueue :: GitQueue.Queue, + quiet :: Bool } deriving (Show) -- git-annex's monad @@ -43,7 +44,7 @@ type BackendName = String data Key = Key (BackendName, KeyName) deriving (Eq, Ord) -- constructs a key in a backend -genKey :: Backend -> KeyName -> Key +genKey :: Backend a -> KeyName -> Key genKey b f = Key (name b,f) -- show a key to convert it to a string; the string includes the @@ -77,28 +78,28 @@ keyName :: Key -> KeyName keyName (Key (_,k)) = k -- this structure represents a key-value backend -data Backend = Backend { +data Backend a = Backend { -- name of this backend name :: String, -- converts a filename to a key - getKey :: FilePath -> Annex (Maybe Key), + getKey :: FilePath -> a (Maybe Key), -- stores a file's contents to a key - storeFileKey :: FilePath -> Key -> Annex Bool, + storeFileKey :: FilePath -> Key -> a Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> Annex Bool, + retrieveKeyFile :: Key -> FilePath -> a Bool, -- removes a key, optionally checking that enough copies are stored -- elsewhere - removeKey :: Key -> Maybe Int -> Annex Bool, + removeKey :: Key -> Maybe Int -> a Bool, -- checks if a backend is storing the content of a key - hasKey :: Key -> Annex Bool, + hasKey :: Key -> a Bool, -- called during fsck to check a key -- (second parameter may be the number of copies that there should -- be of the key) - fsckKey :: Key -> Maybe Int -> Annex Bool + fsckKey :: Key -> Maybe Int -> a Bool } -instance Show Backend where +instance Show (Backend a) where show backend = "Backend { name =\"" ++ name backend ++ "\" }" -instance Eq Backend where +instance Eq (Backend a) where a == b = name a == name b From 082b022f9ae56b1446b6607cf7851cd4f1d4f904 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 25 Jan 2011 21:49:04 -0400 Subject: [PATCH 0748/8313] successfully split Annex and AnnexState out of TypeInternals --- Annex.hs | 91 +++++++++++++++++++++++++++--------------------- Backend.hs | 10 +++--- Backend/File.hs | 1 + Backend/SHA1.hs | 1 + Backend/URL.hs | 1 + Backend/WORM.hs | 1 + CmdLine.hs | 4 +-- Options.hs | 8 ++--- Remotes.hs | 2 +- TypeInternals.hs | 21 ----------- Types.hs | 6 ++-- test.hs | 3 +- 12 files changed, 72 insertions(+), 77 deletions(-) diff --git a/Annex.hs b/Annex.hs index a0de630874..a67ea48631 100644 --- a/Annex.hs +++ b/Annex.hs @@ -6,18 +6,20 @@ -} module Annex ( + Annex, + AnnexState(..), + getState, new, run, eval, gitRepo, gitRepoChange, - backends, backendsChange, - supportedBackends, + FlagName, + Flag(..), flagIsSet, flagChange, flagGet, - Flag(..), queue, queueGet, queueRun, @@ -29,19 +31,38 @@ import qualified Data.Map as M import qualified GitRepo as Git import qualified GitQueue -import Types -import qualified TypeInternals as Internals +import qualified TypeInternals + +-- git-annex's monad +type Annex = StateT AnnexState IO + +-- internal state storage +data AnnexState = AnnexState { + repo :: Git.Repo, + backends :: [TypeInternals.Backend Annex], + supportedBackends :: [TypeInternals.Backend Annex], + flags :: M.Map FlagName Flag, + repoqueue :: GitQueue.Queue, + quiet :: Bool +} deriving (Show) + +-- command-line flags +type FlagName = String +data Flag = + FlagBool Bool | + FlagString String + deriving (Eq, Read, Show) {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [Backend Annex] -> IO AnnexState +new :: Git.Repo -> [TypeInternals.Backend Annex] -> IO AnnexState new gitrepo allbackends = do - let s = Internals.AnnexState { - Internals.repo = gitrepo, - Internals.backends = [], - Internals.supportedBackends = allbackends, - Internals.flags = M.empty, - Internals.repoqueue = GitQueue.empty, - Internals.quiet = False + let s = AnnexState { + repo = gitrepo, + backends = [], + supportedBackends = allbackends, + flags = M.empty, + repoqueue = GitQueue.empty, + quiet = False } (_,s') <- Annex.run s prep return s' @@ -57,41 +78,33 @@ run state action = runStateT action state eval :: AnnexState -> Annex a -> IO a eval state action = evalStateT action state +{- gets a value from the internal Annex state -} +getState :: (AnnexState -> a) -> Annex a +getState a = do + state <- get + return (a state) + {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo -gitRepo = do - state <- get - return (Internals.repo state) +gitRepo = getState repo {- Changes the git repository being acted on. -} gitRepoChange :: Git.Repo -> Annex () gitRepoChange r = do state <- get - put state { Internals.repo = r } - -{- Returns the backends being used. -} -backends :: Annex [Backend Annex] -backends = do - state <- get - return (Internals.backends state) + put state { repo = r } {- Sets the backends to use. -} -backendsChange :: [Backend Annex] -> Annex () +backendsChange :: [TypeInternals.Backend Annex] -> Annex () backendsChange b = do state <- get - put state { Internals.backends = b } - -{- Returns the full list of supported backends. -} -supportedBackends :: Annex [Backend Annex] -supportedBackends = do - state <- get - return (Internals.supportedBackends state) + put state { backends = b } {- Return True if a Bool flag is set. -} flagIsSet :: FlagName -> Annex Bool flagIsSet name = do state <- get - case (M.lookup name $ Internals.flags state) of + case (M.lookup name $ flags state) of Just (FlagBool True) -> return True _ -> return False @@ -99,13 +112,13 @@ flagIsSet name = do flagChange :: FlagName -> Flag -> Annex () flagChange name val = do state <- get - put state { Internals.flags = M.insert name val $ Internals.flags state } + put state { flags = M.insert name val $ flags state } {- Gets the value of a String flag (or "" if there is no such String flag) -} flagGet :: FlagName -> Annex String flagGet name = do state <- get - case (M.lookup name $ Internals.flags state) of + case (M.lookup name $ flags state) of Just (FlagString s) -> return s _ -> return "" @@ -113,23 +126,23 @@ flagGet name = do queue :: String -> [String] -> FilePath -> Annex () queue command params file = do state <- get - let q = Internals.repoqueue state - put state { Internals.repoqueue = GitQueue.add q command params file } + let q = repoqueue state + put state { repoqueue = GitQueue.add q command params file } {- Returns the queue. -} queueGet :: Annex GitQueue.Queue queueGet = do state <- get - return (Internals.repoqueue state) + return (repoqueue state) {- Runs (and empties) the queue. -} queueRun :: Annex () queueRun = do state <- get - let q = Internals.repoqueue state + let q = repoqueue state g <- gitRepo liftIO $ GitQueue.run g q - put state { Internals.repoqueue = GitQueue.empty } + put state { repoqueue = GitQueue.empty } {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () diff --git a/Backend.hs b/Backend.hs index caf50005ad..551c041a80 100644 --- a/Backend.hs +++ b/Backend.hs @@ -44,11 +44,11 @@ import Messages {- List of backends in the order to try them when storing a new key. -} list :: Annex [Backend Annex] list = do - l <- Annex.backends -- list is cached here + l <- Annex.getState Annex.backends -- list is cached here if not $ null l then return l else do - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends g <- Annex.gitRepo let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" backendflag <- Annex.flagGet "backend" @@ -121,7 +121,7 @@ fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies - by examining what the file symlinks to. -} lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex)) lookupFile file = do - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends tl <- liftIO $ try getsymlink case tl of Left _ -> return Nothing @@ -150,12 +150,12 @@ lookupFile file = do chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))] chooseBackends fs = do g <- Annex.gitRepo - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends pairs <- liftIO $ Git.checkAttr g "annex.backend" fs return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs {- Returns the backend to use for a key. -} keyBackend :: Key -> Annex (Backend Annex) keyBackend key = do - bs <- Annex.supportedBackends + bs <- Annex.getState Annex.supportedBackends return $ lookupBackendName bs $ backendName key diff --git a/Backend/File.hs b/Backend/File.hs index c8ddd59381..962d09909b 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -24,6 +24,7 @@ import qualified Remotes import qualified GitRepo as Git import Content import qualified Annex +import Types import UUID import Messages diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index e665e5da75..be41264b0e 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -19,6 +19,7 @@ import Messages import qualified Annex import Locations import Content +import Types backend :: Backend Annex backend = Backend.File.backend { diff --git a/Backend/URL.hs b/Backend/URL.hs index 8ed354aed8..d67b7db847 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -10,6 +10,7 @@ module Backend.URL (backend) where import Control.Monad.State (liftIO) import Data.String.Utils +import Types import TypeInternals import Utility import Messages diff --git a/Backend/WORM.hs b/Backend/WORM.hs index cd4254e2bf..0110183938 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -20,6 +20,7 @@ import Locations import qualified Annex import Content import Messages +import Types backend :: Backend Annex backend = Backend.File.backend { diff --git a/CmdLine.hs b/CmdLine.hs index 6772282c50..39dd61e99a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -78,9 +78,9 @@ usage header cmds options = - (but explicitly thrown errors terminate the whole command). - Runs shutdown and propigates an overall error status at the end. -} -tryRun :: AnnexState -> [Annex Bool] -> IO () +tryRun :: Annex.AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions -tryRun' :: AnnexState -> Integer -> [Annex Bool] -> IO () +tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do result <- try $ Annex.run state a case result of diff --git a/Options.hs b/Options.hs index 5f367c9dd4..2d4bb85fb2 100644 --- a/Options.hs +++ b/Options.hs @@ -18,10 +18,10 @@ import Command -} type Option = OptDescr (Annex ()) -storeOptBool :: FlagName -> Bool -> Annex () -storeOptBool name val = Annex.flagChange name $ FlagBool val -storeOptString :: FlagName -> String -> Annex () -storeOptString name val = Annex.flagChange name $ FlagString val +storeOptBool :: Annex.FlagName -> Bool -> Annex () +storeOptBool name val = Annex.flagChange name $ Annex.FlagBool val +storeOptString :: Annex.FlagName -> String -> Annex () +storeOptString name val = Annex.flagChange name $ Annex.FlagString val commonOptions :: [Option] commonOptions = diff --git a/Remotes.hs b/Remotes.hs index 9004b33d00..e5aa80e1c2 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -105,7 +105,7 @@ readConfigs = do let todo = cheap ++ doexpensive unless (null todo) $ do _ <- mapM tryGitConfigRead todo - Annex.flagChange "remotesread" $ FlagBool True + Annex.flagChange "remotesread" $ Annex.FlagBool True where cachedUUID r = do u <- getUUID r diff --git a/TypeInternals.hs b/TypeInternals.hs index 99f3049730..abafe8711c 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -17,27 +17,6 @@ import Test.QuickCheck import qualified GitRepo as Git import qualified GitQueue --- command-line flags -type FlagName = String -data Flag = - FlagBool Bool | - FlagString String - deriving (Eq, Read, Show) - --- git-annex's runtime state type doesn't really belong here, --- but it uses Backend, so has to be here to avoid a depends loop. -data AnnexState = AnnexState { - repo :: Git.Repo, - backends :: [Backend Annex], - supportedBackends :: [Backend Annex], - flags :: M.Map FlagName Flag, - repoqueue :: GitQueue.Queue, - quiet :: Bool -} deriving (Show) - --- git-annex's monad -type Annex = StateT AnnexState IO - -- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String diff --git a/Types.hs b/Types.hs index b94a4170af..8c19bbbb39 100644 --- a/Types.hs +++ b/Types.hs @@ -7,14 +7,12 @@ module Types ( Annex, - AnnexState, Backend, Key, genKey, backendName, - keyName, - FlagName, - Flag(..) + keyName ) where import TypeInternals +import Annex diff --git a/test.hs b/test.hs index b8b264f0cf..2528e6398e 100644 --- a/test.hs +++ b/test.hs @@ -28,6 +28,7 @@ import qualified GitRepo as Git import qualified Locations import qualified Utility import qualified TypeInternals +import qualified Types import qualified GitAnnex import qualified LocationLog import qualified UUID @@ -416,7 +417,7 @@ git_annex command params = do -- Runs an action in the current annex. Note that shutdown actions -- are not run; this should only be used for actions that query state. -annexeval :: TypeInternals.Annex a -> IO a +annexeval :: Types.Annex a -> IO a annexeval a = do g <- Git.repoFromCwd g' <- Git.configRead g From 6a97b10fcb3e1fa6a230d92a25b42ded587ff743 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 00:17:38 -0400 Subject: [PATCH 0749/8313] rework config storage Moved away from a map of flags to storing config directly in the AnnexState structure. Got rid of most accessor functions in Annex. This allowed supporting multiple --exclude flags. --- Annex.hs | 134 +++++++++++++++++---------------------------- Backend.hs | 28 ++++++---- Backend/File.hs | 2 +- CmdLine.hs | 2 +- Command.hs | 21 ++++++- Command/DropKey.hs | 2 +- Command/FromKey.hs | 19 +++---- Command/Move.hs | 77 +++++++++++++------------- Command/SetKey.hs | 26 ++++----- GitAnnex.hs | 14 +++-- Messages.hs | 2 +- Options.hs | 17 +++--- Remotes.hs | 30 ++++------ debian/changelog | 1 + doc/git-annex.mdwn | 2 + 15 files changed, 179 insertions(+), 198 deletions(-) diff --git a/Annex.hs b/Annex.hs index a67ea48631..d47d449677 100644 --- a/Annex.hs +++ b/Annex.hs @@ -8,26 +8,18 @@ module Annex ( Annex, AnnexState(..), - getState, new, run, eval, + getState, + changeState, gitRepo, - gitRepoChange, - backendsChange, - FlagName, - Flag(..), - flagIsSet, - flagChange, - flagGet, queue, - queueGet, queueRun, setConfig ) where import Control.Monad.State -import qualified Data.Map as M import qualified GitRepo as Git import qualified GitQueue @@ -37,40 +29,42 @@ import qualified TypeInternals type Annex = StateT AnnexState IO -- internal state storage -data AnnexState = AnnexState { - repo :: Git.Repo, - backends :: [TypeInternals.Backend Annex], - supportedBackends :: [TypeInternals.Backend Annex], - flags :: M.Map FlagName Flag, - repoqueue :: GitQueue.Queue, - quiet :: Bool -} deriving (Show) +data AnnexState = AnnexState + { repo :: Git.Repo + , backends :: [TypeInternals.Backend Annex] + , supportedBackends :: [TypeInternals.Backend Annex] + , repoqueue :: GitQueue.Queue + , quiet :: Bool + , force :: Bool + , defaultbackend :: Maybe String + , defaultkey :: Maybe String + , toremote :: Maybe String + , fromremote :: Maybe String + , exclude :: [String] + , remotesread :: Bool + } deriving (Show) --- command-line flags -type FlagName = String -data Flag = - FlagBool Bool | - FlagString String - deriving (Eq, Read, Show) +newState :: Git.Repo -> [TypeInternals.Backend Annex] -> AnnexState +newState gitrepo allbackends = AnnexState + { repo = gitrepo + , backends = [] + , supportedBackends = allbackends + , repoqueue = GitQueue.empty + , quiet = False + , force = False + , defaultbackend = Nothing + , defaultkey = Nothing + , toremote = Nothing + , fromremote = Nothing + , exclude = [] + , remotesread = False + } {- Create and returns an Annex state object for the specified git repo. -} new :: Git.Repo -> [TypeInternals.Backend Annex] -> IO AnnexState new gitrepo allbackends = do - let s = AnnexState { - repo = gitrepo, - backends = [], - supportedBackends = allbackends, - flags = M.empty, - repoqueue = GitQueue.empty, - quiet = False - } - (_,s') <- Annex.run s prep - return s' - where - prep = do - -- read git config and update state - gitrepo' <- liftIO $ Git.configRead gitrepo - Annex.gitRepoChange gitrepo' + gitrepo' <- liftIO $ Git.configRead gitrepo + return $ newState gitrepo' allbackends {- performs an action in the Annex monad -} run :: AnnexState -> Annex a -> IO (a, AnnexState) @@ -78,50 +72,26 @@ run state action = runStateT action state eval :: AnnexState -> Annex a -> IO a eval state action = evalStateT action state -{- gets a value from the internal Annex state -} +{- Gets a value from the internal state, selected by the passed value + - constructor. -} getState :: (AnnexState -> a) -> Annex a -getState a = do +getState c = do state <- get - return (a state) + return (c state) + +{- Applies a state mutation function to change the internal state. + - + - Example: changeState (\s -> s { quiet = True }) + -} +changeState :: (AnnexState -> AnnexState) -> Annex () +changeState a = do + state <- get + put (a state) {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo gitRepo = getState repo -{- Changes the git repository being acted on. -} -gitRepoChange :: Git.Repo -> Annex () -gitRepoChange r = do - state <- get - put state { repo = r } - -{- Sets the backends to use. -} -backendsChange :: [TypeInternals.Backend Annex] -> Annex () -backendsChange b = do - state <- get - put state { backends = b } - -{- Return True if a Bool flag is set. -} -flagIsSet :: FlagName -> Annex Bool -flagIsSet name = do - state <- get - case (M.lookup name $ flags state) of - Just (FlagBool True) -> return True - _ -> return False - -{- Sets the value of a flag. -} -flagChange :: FlagName -> Flag -> Annex () -flagChange name val = do - state <- get - put state { flags = M.insert name val $ flags state } - -{- Gets the value of a String flag (or "" if there is no such String flag) -} -flagGet :: FlagName -> Annex String -flagGet name = do - state <- get - case (M.lookup name $ flags state) of - Just (FlagString s) -> return s - _ -> return "" - {- Adds a git command to the queue. -} queue :: String -> [String] -> FilePath -> Annex () queue command params file = do @@ -129,12 +99,6 @@ queue command params file = do let q = repoqueue state put state { repoqueue = GitQueue.add q command params file } -{- Returns the queue. -} -queueGet :: Annex GitQueue.Queue -queueGet = do - state <- get - return (repoqueue state) - {- Runs (and empties) the queue. -} queueRun :: Annex () queueRun = do @@ -146,9 +110,9 @@ queueRun = do {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () -setConfig key value = do +setConfig k value = do g <- Annex.gitRepo - liftIO $ Git.run g ["config", key, value] + liftIO $ Git.run g ["config", k, value] -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g - Annex.gitRepoChange g' + Annex.changeState $ \s -> s { Annex.repo = g' } diff --git a/Backend.hs b/Backend.hs index 551c041a80..055c5b8ab3 100644 --- a/Backend.hs +++ b/Backend.hs @@ -48,20 +48,24 @@ list = do if not $ null l then return l else do + s <- getstandard + d <- Annex.getState Annex.defaultbackend + handle d s + where + parseBackendList l [] = l + parseBackendList bs s = map (lookupBackendName bs) $ words s + handle Nothing s = return s + handle (Just "") s = return s + handle (Just name) s = do + bs <- Annex.getState Annex.supportedBackends + let l' = (lookupBackendName bs name):s + Annex.changeState $ \state -> state { Annex.backends = l' } + return l' + getstandard = do bs <- Annex.getState Annex.supportedBackends g <- Annex.gitRepo - let defaults = parseBackendList bs $ Git.configGet g "annex.backends" "" - backendflag <- Annex.flagGet "backend" - let l' = if not $ null backendflag - then (lookupBackendName bs backendflag):defaults - else defaults - Annex.backendsChange l' - return l' - where - parseBackendList bs s = - if null s - then bs - else map (lookupBackendName bs) $ words s + return $ parseBackendList bs $ + Git.configGet g "annex.backends" "" {- Looks up a backend in a list. May fail if unknown. -} lookupBackendName :: [Backend Annex] -> String -> Backend Annex diff --git a/Backend/File.hs b/Backend/File.hs index 962d09909b..d0c1e0e22a 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -90,7 +90,7 @@ copyKeyFile key file = do - error if not. -} checkRemoveKey :: Key -> Maybe Int -> Annex Bool checkRemoveKey key numcopiesM = do - force <- Annex.flagIsSet "force" + force <- Annex.getState Annex.force if force || numcopiesM == Just 0 then return True else do diff --git a/CmdLine.hs b/CmdLine.hs index 39dd61e99a..1b5daadeb0 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -102,7 +102,7 @@ startup = do {- Cleanup actions. -} shutdown :: Integer -> Annex () shutdown errnum = do - q <- Annex.queueGet + q <- Annex.getState Annex.repoqueue unless (q == GitQueue.empty) $ do showSideAction "Recording state in git..." Annex.queueRun diff --git a/Command.hs b/Command.hs index 06fc704bd9..cbfb265002 100644 --- a/Command.hs +++ b/Command.hs @@ -179,11 +179,11 @@ backendPairs a files = do filterFiles :: [FilePath] -> Annex [FilePath] filterFiles l = do let l' = filter notState l - exclude <- Annex.flagGet "exclude" + exclude <- Annex.getState Annex.exclude if null exclude then return l' else do - let regexp = compile ("^" ++ wildToRegex exclude) [] + let regexp = compile (toregex exclude) [] return $ filter (notExcluded regexp) l' where notState f = stateLoc /= take stateLocLen f @@ -191,6 +191,10 @@ filterFiles l = do notExcluded r f = case match r f [] of Nothing -> True Just _ -> False + toregex exclude = "^(" ++ toregex' exclude "" ++ ")" + toregex' [] c = c + toregex' (w:ws) "" = toregex' ws (wildToRegex w) + toregex' (w:ws) c = toregex' ws (c ++ "|" ++ wildToRegex w) {- filter out symlinks -} notSymlink :: FilePath -> IO Bool @@ -219,3 +223,16 @@ paramName :: String paramName = "NAME" paramNothing :: String paramNothing = "" + +{- The Key specified by the --key and --backend parameters. -} +cmdlineKey :: Annex Key +cmdlineKey = do + k <- Annex.getState Annex.defaultkey + backends <- Backend.list + return $ genKey (head backends) (keyname' k) + where + keyname' Nothing = badkey + keyname' (Just "") = badkey + keyname' (Just n) = n + badkey = error "please specify the key with --key" + diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 6ba5c117c4..8c7566df84 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -28,7 +28,7 @@ start keyname = do backends <- Backend.list let key = genKey (head backends) keyname present <- inAnnex key - force <- Annex.flagIsSet "force" + force <- Annex.getState Annex.force if not present then return Nothing else if not force diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 9c4a3cfdcb..8817942580 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -10,7 +10,7 @@ module Command.FromKey where import Control.Monad.State (liftIO) import System.Posix.Files import System.Directory -import Control.Monad (when, unless) +import Control.Monad (unless) import Command import qualified Annex @@ -30,22 +30,21 @@ seek = [withFilesMissing start] {- Adds a file pointing at a manually-specified key -} start :: CommandStartString start file = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (head backends) keyname - + key <- cmdlineKey inbackend <- Backend.hasKey key unless inbackend $ error $ - "key ("++keyname++") is not present in backend" + "key ("++keyName key++") is not present in backend" showStart "fromkey" file - return $ Just $ perform file key -perform :: FilePath -> Key -> CommandPerform -perform file key = do + return $ Just $ perform file + +perform :: FilePath -> CommandPerform +perform file = do + key <- cmdlineKey link <- calcGitLink file key liftIO $ createDirectoryIfMissing True (parentDir file) liftIO $ createSymbolicLink link file return $ Just $ cleanup file + cleanup :: FilePath -> CommandCleanup cleanup file = do Annex.queue "add" ["--"] file diff --git a/Command/Move.hs b/Command/Move.hs index 2920c06616..4416134c04 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -34,12 +34,16 @@ seek = [withFilesInGit $ start True] - moving data in the key-value backend. -} start :: Bool -> CommandStartString start move file = do - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - case (fromName, toName) of - ("", "") -> error "specify either --from or --to" - ("", _) -> toStart move file - (_ , "") -> fromStart move file + to <- Annex.getState Annex.toremote + from <- Annex.getState Annex.fromremote + case (from, to) of + (Nothing, Nothing) -> error "specify either --from or --to" + (Nothing, Just name) -> do + dest <- Remotes.byName name + toStart dest move file + (Just name, Nothing) -> do + src <- Remotes.byName name + fromStart src move file (_ , _) -> error "only one of --from or --to can be specified" showAction :: Bool -> FilePath -> Annex () @@ -65,34 +69,33 @@ remoteHasKey remote key present = do - A file's content can be moved even if there are insufficient copies to - allow it to be dropped. -} -toStart :: Bool -> CommandStartString -toStart move file = isAnnexed file $ \(key, _) -> do +toStart :: Git.Repo -> Bool -> CommandStartString +toStart dest move file = isAnnexed file $ \(key, _) -> do ishere <- inAnnex key if not ishere then return Nothing -- not here, so nothing to do else do showAction move file - return $ Just $ toPerform move key -toPerform :: Bool -> Key -> CommandPerform -toPerform move key = do + return $ Just $ toPerform dest move key +toPerform :: Git.Repo -> Bool -> Key -> CommandPerform +toPerform dest move key = do Remotes.readConfigs -- checking the remote is expensive, so not done in the start step - remote <- Remotes.commandLineRemote - isthere <- Remotes.inAnnex remote key + isthere <- Remotes.inAnnex dest key case isthere of Left err -> do showNote $ show err return Nothing Right False -> do - showNote $ "to " ++ Git.repoDescribe remote ++ "..." - ok <- Remotes.copyToRemote remote key + showNote $ "to " ++ Git.repoDescribe dest ++ "..." + ok <- Remotes.copyToRemote dest key if ok - then return $ Just $ toCleanup move remote key + then return $ Just $ toCleanup dest move key else return Nothing -- failed - Right True -> return $ Just $ toCleanup move remote key -toCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup -toCleanup move remote key = do - remoteHasKey remote key True + Right True -> return $ Just $ toCleanup dest move key +toCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup +toCleanup dest move key = do + remoteHasKey dest key True if move then Command.Drop.cleanup key else return True @@ -103,36 +106,34 @@ toCleanup move remote key = do - If the current repository already has the content, it is still removed - from the other repository when moving. -} -fromStart :: Bool -> CommandStartString -fromStart move file = isAnnexed file $ \(key, _) -> do - remote <- Remotes.commandLineRemote +fromStart :: Git.Repo -> Bool -> CommandStartString +fromStart src move file = isAnnexed file $ \(key, _) -> do (trusted, untrusted, _) <- Remotes.keyPossibilities key - if null $ filter (\r -> Remotes.same r remote) (trusted ++ untrusted) + if null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted) then return Nothing else do showAction move file - return $ Just $ fromPerform move key -fromPerform :: Bool -> Key -> CommandPerform -fromPerform move key = do - remote <- Remotes.commandLineRemote + return $ Just $ fromPerform src move key +fromPerform :: Git.Repo -> Bool -> Key -> CommandPerform +fromPerform src move key = do ishere <- inAnnex key if ishere - then return $ Just $ fromCleanup move remote key + then return $ Just $ fromCleanup src move key else do - showNote $ "from " ++ Git.repoDescribe remote ++ "..." - ok <- getViaTmp key $ Remotes.copyFromRemote remote key + showNote $ "from " ++ Git.repoDescribe src ++ "..." + ok <- getViaTmp key $ Remotes.copyFromRemote src key if ok - then return $ Just $ fromCleanup move remote key + then return $ Just $ fromCleanup src move key else return Nothing -- fail -fromCleanup :: Bool -> Git.Repo -> Key -> CommandCleanup -fromCleanup True remote key = do - ok <- Remotes.onRemote remote (boolSystem, False) "dropkey" +fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup +fromCleanup src True key = do + ok <- Remotes.onRemote src (boolSystem, False) "dropkey" ["--quiet", "--force", "--backend=" ++ backendName key, keyName key] - -- better safe than sorry: assume the remote dropped the key + -- better safe than sorry: assume the src dropped the key -- even if it seemed to fail; the failure could have occurred -- after it really dropped it - remoteHasKey remote key False + remoteHasKey src key False return ok -fromCleanup False _ _ = return True +fromCleanup _ False _ = return True diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 412504b2ee..388392cd60 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -8,14 +8,10 @@ module Command.SetKey where import Control.Monad.State (liftIO) -import Control.Monad (when) import Command -import qualified Annex import Utility -import qualified Backend import LocationLog -import Types import Content import Messages @@ -29,26 +25,24 @@ seek = [withTempFile start] {- Sets cached content for a key. -} start :: CommandStartString start file = do - keyname <- Annex.flagGet "key" - when (null keyname) $ error "please specify the key with --key" - backends <- Backend.list - let key = genKey (head backends) keyname showStart "setkey" file - return $ Just $ perform file key -perform :: FilePath -> Key -> CommandPerform -perform file key = do + return $ Just $ perform file + +perform :: FilePath -> CommandPerform +perform file = do + key <- cmdlineKey -- the file might be on a different filesystem, so mv is used - -- rather than simply calling moveToObjectDir key file + -- rather than simply calling moveToObjectDir ok <- getViaTmp key $ \dest -> do if dest /= file then liftIO $ boolSystem "mv" [file, dest] else return True if ok - then return $ Just $ cleanup key + then return $ Just $ cleanup else error "mv failed!" -cleanup :: Key -> CommandCleanup -cleanup key = do +cleanup :: CommandCleanup +cleanup = do + key <- cmdlineKey logStatus key ValuePresent return True - diff --git a/GitAnnex.hs b/GitAnnex.hs index d9efdad2dd..378b6e538d 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -13,6 +13,7 @@ import qualified GitRepo as Git import CmdLine import Command import Options +import qualified Annex import qualified Command.Add import qualified Command.Unannex @@ -65,15 +66,20 @@ cmds = concat options :: [Option] options = commonOptions ++ - [ Option ['k'] ["key"] (ReqArg (storeOptString "key") paramKey) + [ Option ['k'] ["key"] (ReqArg setkey paramKey) "specify a key to use" - , Option ['t'] ["to"] (ReqArg (storeOptString "torepository") paramRemote) + , Option ['t'] ["to"] (ReqArg setto paramRemote) "specify to where to transfer content" - , Option ['f'] ["from"] (ReqArg (storeOptString "fromrepository") paramRemote) + , Option ['f'] ["from"] (ReqArg setfrom paramRemote) "specify from where to transfer content" - , Option ['x'] ["exclude"] (ReqArg (storeOptString "exclude") paramGlob) + , Option ['x'] ["exclude"] (ReqArg addexclude paramGlob) "skip files matching the glob pattern" ] + where + setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v } + setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v } + setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v } + addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) } header :: String header = "Usage: git-annex command [option ..]" diff --git a/Messages.hs b/Messages.hs index 2934de4287..2b98622309 100644 --- a/Messages.hs +++ b/Messages.hs @@ -17,7 +17,7 @@ import qualified Annex verbose :: Annex () -> Annex () verbose a = do - q <- Annex.flagIsSet "quiet" + q <- Annex.getState Annex.quiet unless q a showSideAction :: String -> Annex () diff --git a/Options.hs b/Options.hs index 2d4bb85fb2..4cd62c2222 100644 --- a/Options.hs +++ b/Options.hs @@ -18,19 +18,18 @@ import Command -} type Option = OptDescr (Annex ()) -storeOptBool :: Annex.FlagName -> Bool -> Annex () -storeOptBool name val = Annex.flagChange name $ Annex.FlagBool val -storeOptString :: Annex.FlagName -> String -> Annex () -storeOptString name val = Annex.flagChange name $ Annex.FlagString val - commonOptions :: [Option] commonOptions = - [ Option ['f'] ["force"] (NoArg (storeOptBool "force" True)) + [ Option ['f'] ["force"] (NoArg (setforce True)) "allow actions that may lose annexed data" - , Option ['q'] ["quiet"] (NoArg (storeOptBool "quiet" True)) + , Option ['q'] ["quiet"] (NoArg (setquiet True)) "avoid verbose output" - , Option ['v'] ["verbose"] (NoArg (storeOptBool "quiet" False)) + , Option ['v'] ["verbose"] (NoArg (setquiet False)) "allow verbose output" - , Option ['b'] ["backend"] (ReqArg (storeOptString "backend") paramName) + , Option ['b'] ["backend"] (ReqArg setdefaultbackend paramName) "specify default key-value backend to use" ] + where + setforce v = Annex.changeState $ \s -> s { Annex.force = v } + setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v } + setdefaultbackend v = Annex.changeState $ \s -> s { Annex.defaultbackend = Just v } diff --git a/Remotes.hs b/Remotes.hs index e5aa80e1c2..e04874f7dc 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -11,7 +11,6 @@ module Remotes ( keyPossibilities, inAnnex, same, - commandLineRemote, byName, copyFromRemote, copyToRemote, @@ -69,7 +68,7 @@ tryGitConfigRead r g <- Annex.gitRepo let l = Git.remotes g let g' = Git.remotesAdd g $ exchange l r' - Annex.gitRepoChange g' + Annex.changeState $ \s -> s { Annex.repo = g' } return $ Right r' exchange [] _ = [] exchange (old:ls) new = @@ -93,7 +92,7 @@ tryGitConfigRead r readConfigs :: Annex () readConfigs = do g <- Annex.gitRepo - remotesread <- Annex.flagIsSet "remotesread" + remotesread <- Annex.getState Annex.remotesread unless remotesread $ do allremotes <- filterM repoNotIgnored $ Git.remotes g let cheap = filter (not . Git.repoIsUrl) allremotes @@ -105,7 +104,7 @@ readConfigs = do let todo = cheap ++ doexpensive unless (null todo) $ do _ <- mapM tryGitConfigRead todo - Annex.flagChange "remotesread" $ Annex.FlagBool True + Annex.changeState $ \s -> s { Annex.remotesread = True } where cachedUUID r = do u <- getUUID r @@ -204,27 +203,22 @@ repoCost r = do repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do ignored <- repoConfig r "ignore" "false" - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - let name = if null fromName then toName else fromName - if not $ null name - then return $ match name + to <- match Annex.toremote + from <- match Annex.fromremote + if to || from + then return True else return $ not $ Git.configTrue ignored where - match name = name == Git.repoRemoteName r + match a = do + name <- Annex.getState a + case name of + Nothing -> return False + Just n -> return $ n == Git.repoRemoteName r {- Checks if two repos are the same, by comparing their remote names. -} same :: Git.Repo -> Git.Repo -> Bool same a b = Git.repoRemoteName a == Git.repoRemoteName b -{- Returns the remote specified by --from or --to, may fail with error. -} -commandLineRemote :: Annex Git.Repo -commandLineRemote = do - fromName <- Annex.flagGet "fromrepository" - toName <- Annex.flagGet "torepository" - let name = if null fromName then toName else fromName - byName name - {- Looks up a remote by name. -} byName :: String -> Annex Git.Repo byName name = do diff --git a/debian/changelog b/debian/changelog index 23358486d1..53b6e20a6c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.19) UNRELEASED; urgency=low * Support using the uuidgen command if the uuid command is not available. + * Allow --exclude to be specified more than once. -- Joey Hess Wed, 19 Jan 2011 18:07:51 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 9d89413b96..8e9418fd13 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -258,6 +258,8 @@ Many git-annex commands will stage changes for later `git commit` by you. Skips files matching the glob pattern. The glob is matched relative to the current directory. + This option can be specified multiple times. + * --backend=name Specifies which key-value backend to use. From aa2ca533bcef1848a9dc94bfea8d33a99e8f2df0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 00:29:32 -0400 Subject: [PATCH 0750/8313] trim cruft --- TypeInternals.hs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/TypeInternals.hs b/TypeInternals.hs index abafe8711c..d3592f4822 100644 --- a/TypeInternals.hs +++ b/TypeInternals.hs @@ -9,14 +9,9 @@ module TypeInternals where -import Control.Monad.State (StateT) import Data.String.Utils -import qualified Data.Map as M import Test.QuickCheck -import qualified GitRepo as Git -import qualified GitQueue - -- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String From 616d1d4a208693c46f41781d9099c1f04ae067e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 00:37:50 -0400 Subject: [PATCH 0751/8313] rename TypeInternals to BackendTypes Now that it only contains types used by the backends --- Annex.hs | 10 ++-- Backend.hs | 16 +++---- Backend/File.hs | 2 +- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- TypeInternals.hs => BackendTypes.hs | 71 ++++++++++++++--------------- Types.hs | 2 +- test.hs | 10 ++-- 9 files changed, 58 insertions(+), 59 deletions(-) rename TypeInternals.hs => BackendTypes.hs (89%) diff --git a/Annex.hs b/Annex.hs index d47d449677..4a1b89dcf2 100644 --- a/Annex.hs +++ b/Annex.hs @@ -23,7 +23,7 @@ import Control.Monad.State import qualified GitRepo as Git import qualified GitQueue -import qualified TypeInternals +import qualified BackendTypes -- git-annex's monad type Annex = StateT AnnexState IO @@ -31,8 +31,8 @@ type Annex = StateT AnnexState IO -- internal state storage data AnnexState = AnnexState { repo :: Git.Repo - , backends :: [TypeInternals.Backend Annex] - , supportedBackends :: [TypeInternals.Backend Annex] + , backends :: [BackendTypes.Backend Annex] + , supportedBackends :: [BackendTypes.Backend Annex] , repoqueue :: GitQueue.Queue , quiet :: Bool , force :: Bool @@ -44,7 +44,7 @@ data AnnexState = AnnexState , remotesread :: Bool } deriving (Show) -newState :: Git.Repo -> [TypeInternals.Backend Annex] -> AnnexState +newState :: Git.Repo -> [BackendTypes.Backend Annex] -> AnnexState newState gitrepo allbackends = AnnexState { repo = gitrepo , backends = [] @@ -61,7 +61,7 @@ newState gitrepo allbackends = AnnexState } {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [TypeInternals.Backend Annex] -> IO AnnexState +new :: Git.Repo -> [BackendTypes.Backend Annex] -> IO AnnexState new gitrepo allbackends = do gitrepo' <- liftIO $ Git.configRead gitrepo return $ newState gitrepo' allbackends diff --git a/Backend.hs b/Backend.hs index 055c5b8ab3..d9bf35f0dd 100644 --- a/Backend.hs +++ b/Backend.hs @@ -38,7 +38,7 @@ import Locations import qualified GitRepo as Git import qualified Annex import Types -import qualified TypeInternals as Internals +import qualified BackendTypes as B import Messages {- List of backends in the order to try them when storing a new key. -} @@ -78,7 +78,7 @@ maybeLookupBackendName bs s = if 1 /= length matches then Nothing else Just $ head matches - where matches = filter (\b -> s == Internals.name b) bs + where matches = filter (\b -> s == B.name b) bs {- Attempts to store a file in one of the backends. -} storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex)) @@ -91,11 +91,11 @@ storeFileKey file trybackend = do storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex)) storeFileKey' [] _ = return Nothing storeFileKey' (b:bs) file = do - result <- (Internals.getKey b) file + result <- (B.getKey b) file case result of Nothing -> nextbackend Just key -> do - stored <- (Internals.storeFileKey b) file key + stored <- (B.storeFileKey b) file key if (not stored) then nextbackend else return $ Just (key, b) @@ -105,21 +105,21 @@ storeFileKey' (b:bs) file = do {- Attempts to retrieve an key from one of the backends, saving it to - a specified location. -} retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool -retrieveKeyFile backend key dest = (Internals.retrieveKeyFile backend) key dest +retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest {- Removes a key from a backend. -} removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool -removeKey backend key numcopies = (Internals.removeKey backend) key numcopies +removeKey backend key numcopies = (B.removeKey backend) key numcopies {- Checks if a key is present in its backend. -} hasKey :: Key -> Annex Bool hasKey key = do backend <- keyBackend key - (Internals.hasKey backend) key + (B.hasKey backend) key {- Checks a key's backend for problems. -} fsckKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool -fsckKey backend key numcopies = (Internals.fsckKey backend) key numcopies +fsckKey backend key numcopies = (B.fsckKey backend) key numcopies {- Looks up the key and backend corresponding to an annexed file, - by examining what the file symlinks to. -} diff --git a/Backend/File.hs b/Backend/File.hs index d0c1e0e22a..ac6e4a910f 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -17,7 +17,7 @@ module Backend.File (backend, checkKey) where import Control.Monad.State import System.Directory -import TypeInternals +import BackendTypes import LocationLog import Locations import qualified Remotes diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index be41264b0e..f8dbea4b03 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -14,7 +14,7 @@ import System.IO import System.Directory import qualified Backend.File -import TypeInternals +import BackendTypes import Messages import qualified Annex import Locations diff --git a/Backend/URL.hs b/Backend/URL.hs index d67b7db847..45a204b07d 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Data.String.Utils import Types -import TypeInternals +import BackendTypes import Utility import Messages diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 0110183938..56f243396e 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -15,7 +15,7 @@ import System.Directory import Data.String.Utils import qualified Backend.File -import TypeInternals +import BackendTypes import Locations import qualified Annex import Content diff --git a/TypeInternals.hs b/BackendTypes.hs similarity index 89% rename from TypeInternals.hs rename to BackendTypes.hs index d3592f4822..e4b155f98f 100644 --- a/TypeInternals.hs +++ b/BackendTypes.hs @@ -1,22 +1,53 @@ -{- git-annex internal data types +{- git-annex key/value backend data types - - - Most things should not need this, using Types and/or Annex instead. + - Most things should not need this, using Types instead - - Copyright 2010 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module TypeInternals where +module BackendTypes where import Data.String.Utils import Test.QuickCheck --- annexed filenames are mapped through a backend into keys type KeyName = String type BackendName = String data Key = Key (BackendName, KeyName) deriving (Eq, Ord) +data Backend a = Backend { + -- name of this backend + name :: String, + -- converts a filename to a key + getKey :: FilePath -> a (Maybe Key), + -- stores a file's contents to a key + storeFileKey :: FilePath -> Key -> a Bool, + -- retrieves a key's contents to a file + retrieveKeyFile :: Key -> FilePath -> a Bool, + -- removes a key, optionally checking that enough copies are stored + -- elsewhere + removeKey :: Key -> Maybe Int -> a Bool, + -- checks if a backend is storing the content of a key + hasKey :: Key -> a Bool, + -- called during fsck to check a key + -- (second parameter may be the number of copies that there should + -- be of the key) + fsckKey :: Key -> Maybe Int -> a Bool +} + +instance Show (Backend a) where + show backend = "Backend { name =\"" ++ name backend ++ "\" }" + +instance Eq (Backend a) where + a == b = name a == name b + +-- accessors for the parts of a key +keyName :: Key -> KeyName +keyName (Key (_,k)) = k +backendName :: Key -> BackendName +backendName (Key (b,_)) = b + -- constructs a key in a backend genKey :: Backend a -> KeyName -> Key genKey b f = Key (name b,f) @@ -45,35 +76,3 @@ prop_idempotent_key_read_show k -- backend names will never contain colons | elem ':' (backendName k) = True | otherwise = k == (read $ show k) - -backendName :: Key -> BackendName -backendName (Key (b,_)) = b -keyName :: Key -> KeyName -keyName (Key (_,k)) = k - --- this structure represents a key-value backend -data Backend a = Backend { - -- name of this backend - name :: String, - -- converts a filename to a key - getKey :: FilePath -> a (Maybe Key), - -- stores a file's contents to a key - storeFileKey :: FilePath -> Key -> a Bool, - -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> a Bool, - -- removes a key, optionally checking that enough copies are stored - -- elsewhere - removeKey :: Key -> Maybe Int -> a Bool, - -- checks if a backend is storing the content of a key - hasKey :: Key -> a Bool, - -- called during fsck to check a key - -- (second parameter may be the number of copies that there should - -- be of the key) - fsckKey :: Key -> Maybe Int -> a Bool -} - -instance Show (Backend a) where - show backend = "Backend { name =\"" ++ name backend ++ "\" }" - -instance Eq (Backend a) where - a == b = name a == name b diff --git a/Types.hs b/Types.hs index 8c19bbbb39..0890efd5e3 100644 --- a/Types.hs +++ b/Types.hs @@ -14,5 +14,5 @@ module Types ( keyName ) where -import TypeInternals +import BackendTypes import Annex diff --git a/test.hs b/test.hs index 2528e6398e..0c47da310c 100644 --- a/test.hs +++ b/test.hs @@ -27,7 +27,7 @@ import qualified Backend import qualified GitRepo as Git import qualified Locations import qualified Utility -import qualified TypeInternals +import qualified BackendTypes import qualified Types import qualified GitAnnex import qualified LocationLog @@ -54,7 +54,7 @@ quickchecks :: Test quickchecks = TestLabel "quickchecks" $ TestList [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode , qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey - , qctest "prop_idempotent_key_read_show" TypeInternals.prop_idempotent_key_read_show + , qctest "prop_idempotent_key_read_show" BackendTypes.prop_idempotent_key_read_show , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics @@ -106,8 +106,8 @@ test_add = "git-annex add" ~: TestCase $ inmainrepo $ do test_setkey :: Test test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do writeFile tmp $ content sha1annexedfile - r <- annexeval $ TypeInternals.getKey Backend.SHA1.backend tmp - let sha1 = TypeInternals.keyName $ fromJust r + r <- annexeval $ BackendTypes.getKey Backend.SHA1.backend tmp + let sha1 = BackendTypes.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" @@ -384,7 +384,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also - git_annex "dropkey" ["-q", "--force", TypeInternals.keyName annexedfilekey] + git_annex "dropkey" ["-q", "--force", BackendTypes.keyName annexedfilekey] @? "dropkey failed" checkunused [sha1annexedfilekey] From ff11803c8983173b00ca8a74e3d323679a9e128c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 12:34:15 -0400 Subject: [PATCH 0752/8313] add --- doc/todo/S3.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/todo/S3.mdwn diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn new file mode 100644 index 0000000000..ec2d403ced --- /dev/null +++ b/doc/todo/S3.mdwn @@ -0,0 +1 @@ +Support Amazon S3 as a file storage backend. From b4179f5081414f3ac903a878538b976931442913 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 12:43:21 -0400 Subject: [PATCH 0753/8313] mention git-media --- doc/not.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index b138953f91..c9c5754d7e 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -20,3 +20,13 @@ down to a binary. And it has a fairly extensive test suite. (Don't be fooled by "make test" only showing a few dozen test cases; each test involves checking dozens to hundreds of assertions.) + +* git-annex is not [git-media](https://github.com/schacon/git-media), + although they both approach the same problem from a similar direction. + I only learned of git-media after writing git-annex, but I probably + would have still written git-annex instead of using it. Currently, + git-media has the advantage of using git smudge filters rather than + git-annex's pile of symlinks, and it may be a tighter fit for certian + situations. It lacks git-annex's support for widely distributed storage, + using only a single backend data store. It also does not support + partial checkouts of file contents, like git-annex does. From 06ca13b103eb22afe5c514a9f26f97862cb37384 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 26 Jan 2011 13:21:51 -0400 Subject: [PATCH 0754/8313] add --- doc/todo/smudge.mdwn | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 doc/todo/smudge.mdwn diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn new file mode 100644 index 0000000000..65cfb0fdab --- /dev/null +++ b/doc/todo/smudge.mdwn @@ -0,0 +1,65 @@ +git-annex should use smudge/clean filters. + +The trick is doing it efficiently. Since git a2b665d, 2011-01-05, +something like this works to provide a filename to the clean script: + + git config --global filter.huge.clean huge-clean %f + +This avoids it needing to read all the current file content from stdin +when doing eg, a git status or git commit. Instead it is passed the +filename that git is operating on, I think that's from the working +directory. + +So, WORM could just look at that file and easily tell if it is one +it already knows (same mtime and size). If so, it can short-circuit and +do nothing, file content is already cached. + +SHA1 has a harder job. Would not want to re-sha1 the file every time, +probably. So it'd need a cache of file stat info, mapped to known objects. + +On the smudge side, I have not heard of a way to have the smudge filter +point to an existing file, it probably still needs to cat it out. Luckily +that is only done at checkout anyway. + +---- + +The other trick may be doing it with partial content availability. +When a smudge filter fails, git leaves the tree and index in a very weird +state. More investigation needed. + +### test files + +huge-smudge: + +
+#!/bin/sh
+read sha1
+echo "smudging $sha1" >&2
+cat ~/$sha1
+
+ +huge-clean: + +
+#!/bin/sh
+cat >temp
+sha1=`sha1sum temp | cut -d' ' -f1`
+echo "cleaning $sha1" >&2
+ls -l temp >&2
+mv temp ~/$sha1
+echo $sha1
+
+ +.gitattributes: + +
+*.huge filter=huge
+
+ +in .git/config: + +
+[filter "huge"]
+        clean = huge-clean
+        smudge = huge-smudge
+

From 758019cc18e03c203b023efb3c0d76c053265362 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 13:34:39 -0400
Subject: [PATCH 0755/8313] update

---
 doc/todo/smudge.mdwn | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 65cfb0fdab..855d9c7f89 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -21,11 +21,19 @@ On the smudge side, I have not heard of a way to have the smudge filter
 point to an existing file, it probably still needs to cat it out. Luckily
 that is only done at checkout anyway.
 
-----
+### dealing with partial content availability
+
+The smudge filter cannot be allowed to fail, that leaves the tree and
+index in a weird state. So if a file's content is requested by calling
+the smudge filter, the trick is to instead provide dummy content,
+indicating it is not available (and perhaps saying to run "git-annex get").
+
+Then, in the clean filter, it has to detect that it's cleaning a file
+with that dummy content, and make sure to provide the same identifier as
+it would if the file content was there. 
+
+I've a demo implementation of this technique in the scripts below.
 
-The other trick may be doing it with partial content availability.
-When a smudge filter fails, git leaves the tree and index in a very weird
-state. More investigation needed.
 
 ### test files
 
@@ -35,7 +43,11 @@ huge-smudge:
 #!/bin/sh
 read sha1
 echo "smudging $sha1" >&2
-cat ~/$sha1
+if [ -e ~/$sha1 ]; then
+	cat ~/$sha1
+else
+	echo "$sha1 not available"
+fi
 

 
 huge-clean:
@@ -43,6 +55,11 @@ huge-clean:
 
 #!/bin/sh
 cat >temp
+if grep -q 'not available' temp; then
+	awk '{print $1}' temp # provide what we would if the content were avail!
+	rm temp
+	exit 0
+fi
 sha1=`sha1sum temp | cut -d' ' -f1`
 echo "cleaning $sha1" >&2
 ls -l temp >&2

From 07769fc94968ab65a828c0c001c2fa443a328d99 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 13:40:11 -0400
Subject: [PATCH 0756/8313] more

---
 doc/todo/smudge.mdwn | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 855d9c7f89..f51f45a391 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -17,10 +17,6 @@ do nothing, file content is already cached.
 SHA1 has a harder job. Would not want to re-sha1 the file every time,
 probably. So it'd need a cache of file stat info, mapped to known objects.
 
-On the smudge side, I have not heard of a way to have the smudge filter
-point to an existing file, it probably still needs to cat it out. Luckily
-that is only done at checkout anyway.
-
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -34,6 +30,13 @@ it would if the file content was there.
 
 I've a demo implementation of this technique in the scripts below.
 
+----
+
+It may further be possible to use the %f with the smudge filter
+(docs say it's supported), and instead of outputting the dummy content, 
+it could create a dangling symlink, which would be more like git-annex's
+behavior now, and makes it easy to tell what content is not available
+with `ls`.
 
 ### test files
 

From f7e3d6eea2f71efe14c3ccb29ef4e88840384d02 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 14:09:06 -0400
Subject: [PATCH 0757/8313] document 3-level trust

---
 debian/changelog   |  3 +++
 doc/copies.mdwn    |  1 +
 doc/git-annex.mdwn |  7 ++++---
 doc/trust.mdwn     | 39 ++++++++++++++++++++++++++++++++++-----
 4 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 53b6e20a6c..529a0896ba 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,9 @@ git-annex (0.19) UNRELEASED; urgency=low
 
   * Support using the uuidgen command if the uuid command is not available.
   * Allow --exclude to be specified more than once.
+  * There are now three levels of repository trust.
+  * untrust: Now marks the current repository as untrusted.
+  * semitrust: Now restores the default trust level. (What untrust used to do.)
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 98f315081f..39a714d3bb 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -4,6 +4,7 @@ your git repository's `.git` directory, not in some external data store.
 It's important that data not get lost by an ill-considered `git annex drop`
 command.  So, then using those backends, git-annex can be configured to try
 to keep N copies of a file's content available across all repositories. 
+(Although [[untrusted_repositories|trust]] don't count toward this total.)
 
 By default, N is 1; it is configured by annex.numcopies. This default
 can be overridden on a per-file-type basis by the annex.numcopies
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8e9418fd13..49e67a83ea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -183,12 +183,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * trust [repository ...]
 
-  Records that a repository is [[trusted]] to not unexpectedly lose content.
-  Use with care.
+  Records that a repository is [[trusted|trust]] to not unexpectedly lose
+  content. Use with care.
 
 * untrust [repository ...]
 
-  Undoes a trust command.
+  Records that a repository is [[not trusted|trusted]] and could lose content
+  at any time.
 
 * fromkey file
 
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index 6ce43f30eb..edec6568e7 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -1,3 +1,11 @@
+Git-annex supports three levels of trust of a repository:
+
+* semitrusted (default)
+* untrusted
+* trusted
+
+## semi-trusted
+
 Normally, git-annex does not fully trust its stored [[location_tracking]]
 information. When removing content, it will directly check
 that other repositories have enough [[copies]].
@@ -7,14 +15,35 @@ Generally that explicit checking is a good idea. Consider that the current
 out. Or, a remote may have suffered a catastrophic loss of data, or itself
 been lost.
 
-Sometimes though, you may have reasons to trust the location tracking
-information for a remote repository. For example, it may be an offline
+There is still some trust involved here. A semi-trusted repository is
+dependended on to retain a copy of the file content; possibly the only
+[[copy|copies]].
+
+(Being semitrusted is the default. The `git annex semitrust` command
+restores a repository to this default, when it has been overridden.)
+
+## untrusted
+
+An untrusted repository is not trusted to retain data at all. Git-annex
+will not count data in such a repository as a of the data, and will
+retain sufficient [[copies]] elsewhere.
+
+This is a good choice for eg, portable drives that could get lost. Or,
+if a disk is known to be dying, you can set it to untrusted and let
+`git annex fsck` warn about data that needs to be copied off it.
+
+To configure a repository as untrusted, use the `git annex untrust`
+command.
+
+## trusted
+
+Sometimes, you may have reasons to fully trust the location tracking
+information for a repository. For example, it may be an offline
 archival drive, from which you rarely or never remove content. Deciding
 when it makes sense to trust the tracking info is up to you.
 
 One way to handle this is just to use `--force` when a command cannot
 access a remote you trust.
 
-Another option is to configure which remotes you trust with the 
-`git annex trust` command, or by manually adding the UUIDs of trusted remotes
-to `.git-annex/trust.log`.
+To configure a repository as fully trusted, use the `git annex trust`
+command.

From 268cb35e644754093db003aee08d050a1f3f9466 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 15:37:16 -0400
Subject: [PATCH 0758/8313] implement 3 level trust storage in trust.log

---
 Command/Semitrust.hs | 35 ++++++++++++++++++
 Command/Trust.hs     | 13 ++-----
 Command/Untrust.hs   | 13 ++-----
 GitAnnex.hs          |  2 ++
 Remotes.hs           |  3 +-
 Trust.hs             | 86 ++++++++++++++++++++++++++++++++++++++++++++
 UUID.hs              | 30 +---------------
 doc/git-annex.mdwn   |  9 +++--
 test.hs              | 27 ++++++++------
 9 files changed, 153 insertions(+), 65 deletions(-)
 create mode 100644 Command/Semitrust.hs
 create mode 100644 Trust.hs

diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
new file mode 100644
index 0000000000..8ed95f5a3c
--- /dev/null
+++ b/Command/Semitrust.hs
@@ -0,0 +1,35 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Semitrust where
+
+import Command
+import qualified GitRepo as Git
+import qualified Remotes
+import UUID
+import Trust
+import Messages
+
+command :: [Command]
+command = [Command "semitrust" (paramRepeating paramRemote) seek
+	"return repository to default trust level"]
+
+seek :: [CommandSeek]
+seek = [withString start]
+
+{- Marks a remote as not trusted. -}
+start :: CommandStartString
+start name = do
+	r <- Remotes.byName name
+	showStart "untrust" name
+	return $ Just $ perform r
+
+perform :: Git.Repo -> CommandPerform
+perform repo = do
+	uuid <- getUUID repo
+	trustSet uuid SemiTrusted
+	return $ Just $ return True
diff --git a/Command/Trust.hs b/Command/Trust.hs
index c97d75ee47..a40c8dccce 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,13 +7,10 @@
 
 module Command.Trust where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (unless)
-
 import Command
-import qualified Annex
 import qualified GitRepo as Git
 import qualified Remotes
+import Trust
 import UUID
 import Messages
 
@@ -34,11 +31,5 @@ start name = do
 perform :: Git.Repo -> CommandPerform
 perform repo = do
 	uuid <- getUUID repo
-	trusted <- getTrusted
-	unless (elem uuid trusted) $ do
-		setTrusted $ uuid:trusted
-		g <- Annex.gitRepo
-		logfile <- trustLog
-		liftIO $ Git.run g ["add", logfile]
-		liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile]
+	trustSet uuid Trusted
 	return $ Just $ return True
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 01b97b1c1f..9e884e812f 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,14 +7,11 @@
 
 module Command.Untrust where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when)
-
 import Command
-import qualified Annex
 import qualified GitRepo as Git
 import qualified Remotes
 import UUID
+import Trust
 import Messages
 
 command :: [Command]
@@ -34,11 +31,5 @@ start name = do
 perform :: Git.Repo -> CommandPerform
 perform repo = do
 	uuid <- getUUID repo
-	trusted <- getTrusted
-	when (elem uuid trusted) $ do
-		setTrusted $ filter (\u -> u /= uuid) trusted
-		g <- Annex.gitRepo
-		logfile <- trustLog
-		liftIO $ Git.run g ["add", logfile]
-		liftIO $ Git.run g ["commit", "-q", "-m", "git annex untrust", logfile]
+	trustSet uuid UnTrusted
 	return $ Just $ return True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 378b6e538d..b09ec82ffc 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -37,6 +37,7 @@ import qualified Command.Migrate
 import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
+import qualified Command.Semitrust
 
 cmds :: [Command]
 cmds = concat
@@ -53,6 +54,7 @@ cmds = concat
 	, Command.PreCommit.command
 	, Command.Trust.command
 	, Command.Untrust.command
+	, Command.Semitrust.command
 	, Command.FromKey.command
 	, Command.DropKey.command
 	, Command.SetKey.command
diff --git a/Remotes.hs b/Remotes.hs
index e04874f7dc..e8a78a59ed 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -32,6 +32,7 @@ import qualified Annex
 import LocationLog
 import Locations
 import UUID
+import Trust
 import Utility
 import qualified Content
 import Messages
@@ -126,7 +127,7 @@ keyPossibilities key = do
 	allremotes <- remotesByCost
 	g <- Annex.gitRepo
 	u <- getUUID g
-	trusted <- getTrusted
+	trusted <- trustGet Trusted
 
 	-- get uuids of other repositories that are
 	-- believed to have the key
diff --git a/Trust.hs b/Trust.hs
new file mode 100644
index 0000000000..9474d47d7a
--- /dev/null
+++ b/Trust.hs
@@ -0,0 +1,86 @@
+{- git-annex trust levels
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Trust (
+	TrustLevel(..),
+	trustLog,
+	trustGet,
+	trustMap,
+	trustMapParse,
+	trustSet
+) where
+
+import Control.Monad.State
+import qualified Data.Map as M
+
+import qualified GitRepo as Git
+import Types
+import UUID
+import Locations
+import qualified Annex
+import Utility
+
+data TrustLevel = SemiTrusted | UnTrusted | Trusted
+	deriving Eq
+
+instance Show TrustLevel where
+        show SemiTrusted = "?"
+        show UnTrusted = "0"
+        show Trusted = "1"
+
+instance Read TrustLevel where
+        readsPrec _ "1" = [(Trusted, "")]
+        readsPrec _ "0" = [(UnTrusted, "")]
+	readsPrec _ _ = [(SemiTrusted, "")]
+
+{- Filename of trust.log. -}
+trustLog :: Annex FilePath
+trustLog = do
+	g <- Annex.gitRepo
+	return $ gitStateDir g ++ "trust.log"
+
+{- Returns a list of UUIDs at the specified trust level. -}
+trustGet :: TrustLevel -> Annex [UUID]
+trustGet level = do
+	m <- trustMap
+	return $ M.keys $ M.filter (== level) m
+
+{- Read the trustLog into a map. -}
+trustMap :: Annex (M.Map UUID TrustLevel)
+trustMap = do
+	logfile <- trustLog
+	s <- liftIO $ catch (readFile logfile) ignoreerror
+	return $ trustMapParse s
+	where
+                ignoreerror _ = return ""
+
+{- Trust map parser. -}
+trustMapParse :: String -> M.Map UUID TrustLevel
+trustMapParse s = M.fromList $ map pair $ filter (not . null) $ lines s
+	where
+		pair l
+			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)
+			-- for back-compat; the trust log used to only
+			-- list trusted uuids
+			| otherwise = (w !! 0, Trusted)
+			where
+				w = words l
+
+{- Changes the trust level for a uuid in the trustLog, and commits it. -}
+trustSet :: UUID -> TrustLevel -> Annex ()
+trustSet uuid level = do
+        m <- trustMap
+	when (M.lookup uuid m /= Just level) $ do
+		let m' = M.insert uuid level m
+	        logfile <- trustLog
+	        liftIO $ safeWriteFile logfile (serialize m')
+		g <- Annex.gitRepo
+		liftIO $ Git.run g ["add", logfile]
+		liftIO $ Git.run g ["commit", "-q", "-m", "git annex trust change", logfile]
+        where
+                serialize m = unlines $ map showpair $ M.toList m
+		showpair (u, t) = u ++ " " ++ show t
diff --git a/UUID.hs b/UUID.hs
index 26a64523fc..ec67026894 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -17,10 +17,7 @@ module UUID (
 	reposWithoutUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidLog,
-	trustLog,
-	getTrusted,
-	setTrusted
+	uuidLog
 ) where
 
 import Control.Monad.State
@@ -141,28 +138,3 @@ uuidLog :: Annex FilePath
 uuidLog = do
 	g <- Annex.gitRepo
 	return $ gitStateDir g ++ "uuid.log"
-
-{- Filename of trust.log. -}
-trustLog :: Annex FilePath
-trustLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "trust.log"
-
-{- List of trusted UUIDs. -}
-getTrusted :: Annex [UUID]
-getTrusted = do
-	logfile <- trustLog
-	s <- liftIO $ catch (readFile logfile) ignoreerror
-	return $ parse s
-	where
-		parse [] = []
-		parse s = map firstword $ lines s
-		firstword [] = ""
-		firstword l = head $ words l
-		ignoreerror _ = return ""
-
-{- Changes the list of trusted UUIDs. -}
-setTrusted :: [UUID] -> Annex ()
-setTrusted u = do
-	logfile <- trustLog
-	liftIO $ safeWriteFile logfile $ unlines u
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 49e67a83ea..9c2be8dc90 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -188,9 +188,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * untrust [repository ...]
 
-  Records that a repository is [[not trusted|trusted]] and could lose content
+  Records that a repository is [[not trusted|trust]] and could lose content
   at any time.
 
+* semitrust [repository ...]
+
+  Returns a repository to the default [[semi trusted|trust]] state.
+
 * fromkey file
 
   This can be used to maually set up a file to link to a specified key
@@ -356,7 +360,8 @@ available. Annexed files in your git repository symlink to that content.
 `.git-annex/uuid.log` is used to map between repository UUID and
 decscriptions.
 
-`.git-annex/trust.log` is used to list the UUIDs of trusted repositories.
+`.git-annex/trust.log` is used to indicate which repositories are trusted
+and untrusted.
 
 `.git-annex/*.log` is where git-annex records its content tracking
 information. These files should be committed to git.
diff --git a/test.hs b/test.hs
index 0c47da310c..c5e755966c 100644
--- a/test.hs
+++ b/test.hs
@@ -32,6 +32,7 @@ import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
 import qualified UUID
+import qualified Trust
 import qualified Remotes
 import qualified Content
 import qualified Backend.SHA1
@@ -288,24 +289,28 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 		newfile = subdir ++ "/" ++ annexedfile
 
 test_trust :: Test
-test_trust = "git-annex trust/untrust" ~: intmpclonerepo $ do
-	trust False
+test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
+	trustcheck Trust.SemiTrusted
 	git_annex "trust" ["-q", "origin"] @? "trust failed"
-	trust True
+	trustcheck Trust.Trusted
 	git_annex "trust" ["-q", "origin"] @? "trust of trusted failed"
-	trust True
+	trustcheck Trust.Trusted
 	git_annex "untrust" ["-q", "origin"] @? "untrust failed"
-	trust False
+	trustcheck Trust.UnTrusted
 	git_annex "untrust" ["-q", "origin"] @? "untrust of untrusted failed"
-	trust False
+	trustcheck Trust.UnTrusted
+	git_annex "semitrust" ["-q", "origin"] @? "semitrust failed"
+	trustcheck Trust.SemiTrusted
+	git_annex "semitrust" ["-q", "origin"] @? "semitrust of semitrusted failed"
+	trustcheck Trust.SemiTrusted
 	where
-		trust expected = do
-			istrusted <- annexeval $ do
-				uuids <- UUID.getTrusted
+		trustcheck expected = do
+			present <- annexeval $ do
+				l <- Trust.trustGet expected
 				r <- Remotes.byName "origin"
 				u <- UUID.getUUID r
-				return $ elem u uuids
-			assertEqual "trust value" expected istrusted
+				return $ elem u l
+			assertEqual ("trust value " ++ show expected) True present
 
 test_fsck :: Test
 test_fsck = "git-annex fsck" ~: intmpclonerepo $ do

From 7b2da21ab7bc51785203a69cc05ab811a8629ecb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 15:59:10 -0400
Subject: [PATCH 0759/8313] avoid moving if src and dest are the same

---
 Command/Move.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 4416134c04..d52ca07df8 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -71,8 +71,9 @@ remoteHasKey remote key present	= do
  -}
 toStart :: Git.Repo -> Bool -> CommandStartString
 toStart dest move file = isAnnexed file $ \(key, _) -> do
+	g <- Annex.gitRepo
 	ishere <- inAnnex key
-	if not ishere
+	if not ishere || g == dest
 		then return Nothing -- not here, so nothing to do
 		else do
 			showAction move file
@@ -108,8 +109,9 @@ toCleanup dest move key = do
  -}
 fromStart :: Git.Repo -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
+	g <- Annex.gitRepo
 	(trusted, untrusted, _) <- Remotes.keyPossibilities key
-	if null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted)
+	if (g == src) || (null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted))
 		then return Nothing
 		else do
 			showAction move file

From 7f6af79232eee685daf58e72737c5284f80cf482 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 16:20:28 -0400
Subject: [PATCH 0760/8313] trust setting improvements

---
 Command/Semitrust.hs |  4 +--
 Command/Trust.hs     |  4 +--
 Command/Untrust.hs   |  4 +--
 Remotes.hs           |  1 +
 Trust.hs             |  2 ++
 debian/changelog     |  1 +
 doc/git-annex.mdwn   |  2 ++
 test.hs              | 64 ++++++++++++++++++++++++--------------------
 8 files changed, 47 insertions(+), 35 deletions(-)

diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 8ed95f5a3c..a91d25359c 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -21,11 +21,11 @@ command = [Command "semitrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withString start]
 
-{- Marks a remote as not trusted. -}
 start :: CommandStartString
 start name = do
+	showStart "semitrust" name
+	Remotes.readConfigs
 	r <- Remotes.byName name
-	showStart "untrust" name
 	return $ Just $ perform r
 
 perform :: Git.Repo -> CommandPerform
diff --git a/Command/Trust.hs b/Command/Trust.hs
index a40c8dccce..3fbff68b89 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -21,11 +21,11 @@ command = [Command "trust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withString start]
 
-{- Marks a remote as trusted. -}
 start :: CommandStartString
 start name = do
-	r <- Remotes.byName name
 	showStart "trust" name
+	Remotes.readConfigs
+	r <- Remotes.byName name
 	return $ Just $ perform r
 
 perform :: Git.Repo -> CommandPerform
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 9e884e812f..69d0ab391a 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -21,11 +21,11 @@ command = [Command "untrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withString start]
 
-{- Marks a remote as not trusted. -}
 start :: CommandStartString
 start name = do
-	r <- Remotes.byName name
 	showStart "untrust" name
+	Remotes.readConfigs
+	r <- Remotes.byName name
 	return $ Just $ perform r
 
 perform :: Git.Repo -> CommandPerform
diff --git a/Remotes.hs b/Remotes.hs
index e8a78a59ed..fe5b7f767e 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -222,6 +222,7 @@ same a b = Git.repoRemoteName a == Git.repoRemoteName b
 
 {- Looks up a remote by name. -}
 byName :: String -> Annex Git.Repo
+byName "." = Annex.gitRepo -- special case to refer to current repository
 byName name = do
 	when (null name) $ error "no remote specified"
 	g <- Annex.gitRepo
diff --git a/Trust.hs b/Trust.hs
index 9474d47d7a..695059a932 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -73,6 +73,8 @@ trustMapParse s = M.fromList $ map pair $ filter (not . null) $ lines s
 {- Changes the trust level for a uuid in the trustLog, and commits it. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
+	when (null uuid) $
+		error "unknown UUID; cannot modify trust level"
         m <- trustMap
 	when (M.lookup uuid m /= Just level) $ do
 		let m' = M.insert uuid level m
diff --git a/debian/changelog b/debian/changelog
index 529a0896ba..8b863b947b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
+  * fsck: Warn if content is only in untrusted repositories.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9c2be8dc90..d2cde35a0d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -186,6 +186,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Records that a repository is [[trusted|trust]] to not unexpectedly lose
   content. Use with care.
 
+  To trust the current repository, use "."
+
 * untrust [repository ...]
 
   Records that a repository is [[not trusted|trust]] and could lose content
diff --git a/test.hs b/test.hs
index c5e755966c..817b1bafbe 100644
--- a/test.hs
+++ b/test.hs
@@ -42,7 +42,7 @@ import qualified Command.DropUnused
 main :: IO ()
 main = do
 	prepare
-	r <- runVerboseTests $ TestList [quickchecks, toplevels]
+	r <- runVerboseTests $ TestList [quickcheck, blackbox]
 	cleanup tmpdir
 	propigate r
 
@@ -51,8 +51,8 @@ propigate (Counts { errors = e }, _)
 	| e > 0 = error "failed"
 	| otherwise = return ()
 
-quickchecks :: Test
-quickchecks = TestLabel "quickchecks" $ TestList
+quickcheck :: Test
+quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" BackendTypes.prop_idempotent_key_read_show
@@ -62,8 +62,8 @@ quickchecks = TestLabel "quickchecks" $ TestList
 	, qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics
 	]
 
-toplevels :: Test
-toplevels = TestLabel "toplevel" $ TestList
+blackbox :: Test
+blackbox = TestLabel "blackbox" $ TestList
 	-- test order matters, later tests may rely on state from earlier
 	[ test_init
 	, test_add
@@ -290,38 +290,44 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 
 test_trust :: Test
 test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
-	trustcheck Trust.SemiTrusted
-	git_annex "trust" ["-q", "origin"] @? "trust failed"
-	trustcheck Trust.Trusted
-	git_annex "trust" ["-q", "origin"] @? "trust of trusted failed"
-	trustcheck Trust.Trusted
-	git_annex "untrust" ["-q", "origin"] @? "untrust failed"
-	trustcheck Trust.UnTrusted
-	git_annex "untrust" ["-q", "origin"] @? "untrust of untrusted failed"
-	trustcheck Trust.UnTrusted
-	git_annex "semitrust" ["-q", "origin"] @? "semitrust failed"
-	trustcheck Trust.SemiTrusted
-	git_annex "semitrust" ["-q", "origin"] @? "semitrust of semitrusted failed"
-	trustcheck Trust.SemiTrusted
+	git_annex "trust" ["-q", repo] @? "trust failed"
+	trustcheck Trust.Trusted "trusted 1"
+	git_annex "trust" ["-q", repo] @? "trust of trusted failed"
+	trustcheck Trust.Trusted "trusted 2"
+	git_annex "untrust" ["-q", repo] @? "untrust failed"
+	trustcheck Trust.UnTrusted "untrusted 1"
+	git_annex "untrust" ["-q", repo] @? "untrust of untrusted failed"
+	trustcheck Trust.UnTrusted "untrusted 2"
+	git_annex "semitrust" ["-q", repo] @? "semitrust failed"
+	trustcheck Trust.SemiTrusted "semitrusted 1"
+	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
+	trustcheck Trust.SemiTrusted "semitrusted 2"
 	where
-		trustcheck expected = do
+		trustcheck expected msg = do
 			present <- annexeval $ do
+				Remotes.readConfigs
 				l <- Trust.trustGet expected
-				r <- Remotes.byName "origin"
+				r <- Remotes.byName repo
 				u <- UUID.getUUID r
 				return $ elem u l
-			assertEqual ("trust value " ++ show expected) True present
+			assertBool msg present
+		repo = "origin"
 
 test_fsck :: Test
-test_fsck = "git-annex fsck" ~: intmpclonerepo $ do
-	git_annex "fsck" ["-q"] @? "fsck failed"
-	Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
-	r <- git_annex "fsck" ["-q"]
-	not r @? "fsck failed to fail with numcopies unsatisfied"
-	Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed"
-	corrupt annexedfile
-	corrupt sha1annexedfile
+test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted]
 	where
+		basicfsck = TestCase $ intmpclonerepo $ do
+			git_annex "fsck" ["-q"] @? "fsck failed"
+			Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
+			r <- git_annex "fsck" ["-q"]
+			not r @? "fsck failed to fail with numcopies unsatisfied"
+			Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed"
+			corrupt annexedfile
+			corrupt sha1annexedfile
+		withlocaluntrusted = TestCase $ intmpcopyrepo $ do
+			git_annex "untrust" ["-q", "."] @? "untrust of current repo failed"
+			r <- git_annex "fsck" ["-q"]
+			not r @? "fsck failed to fail with content only available in untrusted (current) repository"
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
 			Content.allowWrite f

From b7903eb2d149ceb164d7422fb56573735d83ebde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 16:44:14 -0400
Subject: [PATCH 0761/8313] move partitioning out of keyPossibilities

And a bug fix in passing.
---
 Backend/File.hs  |  8 ++++----
 Command/Move.hs  |  4 ++--
 Remotes.hs       | 16 +++-------------
 debian/changelog |  2 ++
 4 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index ac6e4a910f..358bc4b7cc 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -50,8 +50,7 @@ dummyStore _ _ = return True
  - and copy it over to this one. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(trusted, untrusted, _) <- Remotes.keyPossibilities key
-	let remotes = trusted ++ untrusted
+	(remotes, _) <- Remotes.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -94,9 +93,10 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(_, untrusted, have) <- Remotes.keyPossibilities key
+			(remotes, trusteduuids) <- Remotes.keyPossibilities key
+			untrusted <- reposWithoutUUID remotes trusteduuids
 			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies have untrusted []
+			findcopies numcopies trusteduuids untrusted []
 	where
 		findcopies need have [] bad
 			| length have >= need = return True
diff --git a/Command/Move.hs b/Command/Move.hs
index d52ca07df8..3eadf30c6f 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -110,8 +110,8 @@ toCleanup dest move key = do
 fromStart :: Git.Repo -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
-	(trusted, untrusted, _) <- Remotes.keyPossibilities key
-	if (g == src) || (null $ filter (\r -> Remotes.same r src) (trusted ++ untrusted))
+	(remotes, _) <- Remotes.keyPossibilities key
+	if (g == src) || (null $ filter (\r -> Remotes.same r src) remotes)
 		then return Nothing
 		else do
 			showAction move file
diff --git a/Remotes.hs b/Remotes.hs
index fe5b7f767e..be4c1383af 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -112,15 +112,11 @@ readConfigs = do
 			return $ null u
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - The first list is of remotes that are trusted to have the key.
- -
- - The second is of untrusted remotes that may have the key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilities :: Key -> Annex ([Git.Repo], [Git.Repo], [UUID])
+keyPossibilities :: Key -> Annex ([Git.Repo], [UUID])
 keyPossibilities key = do
 	readConfigs
 
@@ -129,23 +125,17 @@ keyPossibilities key = do
 	u <- getUUID g
 	trusted <- trustGet Trusted
 
-	-- get uuids of other repositories that are
-	-- believed to have the key
+	-- get uuids of all repositories that are recorded to have the key
 	uuids <- liftIO $ keyLocations g key
 	let validuuids = filter (/= u) uuids
 
-	-- get uuids trusted to have the key
 	-- note that validuuids is assumed to not have dups
 	let validtrusteduuids = intersect validuuids trusted
 
 	-- remotes that match uuids that have the key
 	validremotes <- reposByUUID allremotes validuuids
 
-	-- partition out the trusted and untrusted remotes
-	trustedremotes <- reposByUUID validremotes validtrusteduuids
-	untrustedremotes <- reposWithoutUUID validremotes trusted
-
-	return (trustedremotes, untrustedremotes, validtrusteduuids)
+	return (validremotes, validtrusteduuids)
 
 {- Checks if a given remote has the content for a key inAnnex.
  - If the remote cannot be accessed, returns a Left error.
diff --git a/debian/changelog b/debian/changelog
index 8b863b947b..01e99ae2b3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.19) UNRELEASED; urgency=low
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
   * fsck: Warn if content is only in untrusted repositories.
+  * bugfix: Files were copied from trusted remotes first even if their
+    annex.cost was higher than other remotes.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 

From ba748a11989f6ba7ea1f306be99b8b3f259ede68 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 17:44:40 -0400
Subject: [PATCH 0762/8313] fsck: handle untrusted repos

---
 Backend/File.hs      | 36 +++++++++++++++++++++++++++---------
 debian/changelog     |  2 +-
 doc/trust.mdwn       |  3 +--
 doc/walkthrough.mdwn | 27 +++++++++++++++++++++++++++
 4 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 358bc4b7cc..dea69ef9b6 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -16,6 +16,7 @@ module Backend.File (backend, checkKey) where
 
 import Control.Monad.State
 import System.Directory
+import Data.List
 
 import BackendTypes
 import LocationLog
@@ -27,6 +28,7 @@ import qualified Annex
 import Types
 import UUID
 import Messages
+import Trust
 
 backend :: Backend Annex
 backend = Backend {
@@ -150,8 +152,8 @@ getNumCopies Nothing = do
 		config = "annex.numcopies"
 
 {- This is used to check that numcopies is satisfied for the key on fsck.
- - This trusts the location log, and so checks all keys, even those with
- - data not present in the current annex.
+ - This trusts data in the the location log, and so can check all keys, even
+ - those with data not present in the current annex.
  -
  - The passed action is first run to allow backends deriving this one
  - to do their own checks.
@@ -167,15 +169,31 @@ checkKeyNumCopies key numcopies = do
 	needed <- getNumCopies numcopies
 	g <- Annex.gitRepo
 	locations <- liftIO $ keyLocations g key
-	let present = length locations
+	untrusted <- trustGet UnTrusted
+	let untrustedlocations = intersect untrusted locations
+	let safelocations = filter (\l -> not $ l `elem` untrusted) locations
+	let present = length safelocations
 	if present < needed
 		then do
-			warning $ note present needed
+			ppuuids <- prettyPrintUUIDs untrustedlocations
+			missingNote present needed ppuuids
 			return False
 		else return True
 	where
-		note 0 _ = "** No known copies of "++show key++" exist!"
-		note present needed = 
-			"Only " ++ show present ++ " of " ++ show needed ++ 
-			" copies of "++show key++" exist. " ++
-			"Back it up with git-annex copy."
+
+missingNote :: Int -> Int -> String -> Annex ()
+missingNote 0 _ [] = showLongNote $ "** No known copies of this file exist!"
+missingNote 0 _ untrusted = do
+	showLongNote $
+		"Only these untrusted locations may have copies of this file!" ++
+		"\n" ++ untrusted ++
+		"Back it up to trusted locations with git-annex copy."
+missingNote present needed untrusted = do
+	showLongNote $
+		"Only " ++ show present ++ " of " ++ show needed ++ 
+		" trustworthy copies of this file exist." ++
+		"\nBack it up with git-annex copy."
+	when (not $ null untrusted) $ do
+		showLongNote $
+			"\nThe following untrusted copies may also exist: " ++
+			"\n" ++ untrusted
diff --git a/debian/changelog b/debian/changelog
index 01e99ae2b3..fcd986e1b6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
-  * fsck: Warn if content is only in untrusted repositories.
+  * fsck: Take untrusted repositories into account.
   * bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
 
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index edec6568e7..dbddfe4269 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -25,8 +25,7 @@ restores a repository to this default, when it has been overridden.)
 ## untrusted
 
 An untrusted repository is not trusted to retain data at all. Git-annex
-will not count data in such a repository as a of the data, and will
-retain sufficient [[copies]] elsewhere.
+will retain sufficient [[copies]] of data elsewhere.
 
 This is a good choice for eg, portable drives that could get lost. Or,
 if a disk is known to be dying, you can set it to untrusted and let
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index d2231c81e6..f375d4e444 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -391,3 +391,30 @@ it so anything put in there is backed up more thoroughly:
 	# echo "* annex.numcopies=3" > important_stuff/.gitattributes
 
 For more details about the numcopies setting, see [[copies]].
+
+## untrusted repositories
+
+Suppose you have a portable USB drive and are using it as a git annex
+repository. You don't trust the drive, because you could lose it, or
+just because portable USB drives don't tend to last very long. You can
+let git-annex know about this, and it will adjust its behavior to avoid
+relying on that drive's continued availability.
+	
+	# cd /media/usb
+	# git annex untrust .
+	untrust . ok
+
+Now when you do a fsck, you'll be warned appropriately:
+
+	# git annex fsck .
+	fsck my_big_file
+	  Only these untrusted locations may have copies of this file!
+	  	05e296c4-2989-11e0-bf40-bad1535567fe  -- portable USB drive
+	  Back it up to trusted locations with git-annex copy.
+	failed
+
+Also, git-annex will refuse to drop a file from elsewhere just because
+it can see a copy on the untrusted drive.
+
+It's also possible to tell git-annex that you have an unusually high
+level of trust for a repository. See [[trust]] for details.

From 6b48f740f1e55b8461757bed670f29bc87e186a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 17:47:02 -0400
Subject: [PATCH 0763/8313] rework note

---
 Backend/File.hs | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index dea69ef9b6..8c39c3f803 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -176,24 +176,22 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			missingNote present needed ppuuids
+			showLongNote $ missingNote present needed ppuuids
 			return False
 		else return True
 	where
 
-missingNote :: Int -> Int -> String -> Annex ()
-missingNote 0 _ [] = showLongNote $ "** No known copies of this file exist!"
-missingNote 0 _ untrusted = do
-	showLongNote $
+missingNote :: Int -> Int -> String -> String
+missingNote 0 _ [] = 
+		"** No known copies of this file exist!"
+missingNote 0 _ untrusted =
 		"Only these untrusted locations may have copies of this file!" ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
-missingNote present needed untrusted = do
-	showLongNote $
+missingNote present needed [] =
 		"Only " ++ show present ++ " of " ++ show needed ++ 
 		" trustworthy copies of this file exist." ++
 		"\nBack it up with git-annex copy."
-	when (not $ null untrusted) $ do
-		showLongNote $
-			"\nThe following untrusted copies may also exist: " ++
-			"\n" ++ untrusted
+missingNote present needed untrusted = missingNote present needed [] ++
+		"\nThe following untrusted copies may also exist: " ++
+		"\n" ++ untrusted

From ff3c12725502451c8ab7da72797b2d69c4a66900 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 19:35:29 -0400
Subject: [PATCH 0764/8313] wording

---
 doc/walkthrough.mdwn | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index f375d4e444..231c3e543c 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -394,15 +394,19 @@ For more details about the numcopies setting, see [[copies]].
 
 ## untrusted repositories
 
-Suppose you have a portable USB drive and are using it as a git annex
+Suppose you have a USB thunb drive and are using it as a git annex
 repository. You don't trust the drive, because you could lose it, or
-just because portable USB drives don't tend to last very long. You can
-let git-annex know about this, and it will adjust its behavior to avoid
-relying on that drive's continued availability.
+accidentially run it through the laundry. Or, maybe you have a drive that
+you know is dying, and you'd like to be warned if there are any files
+on it not backed up somewhere else. Maybe the drive has already died
+or been lost.
+
+You can let git-annex know that you don't trust a repository, and it will
+adjust its behavior to avoid relying on that repositories's continued
+availability.
 	
-	# cd /media/usb
-	# git annex untrust .
-	untrust . ok
+	# git annex untrust usbdrive
+	untrust usbdrive ok
 
 Now when you do a fsck, you'll be warned appropriately:
 
@@ -414,7 +418,7 @@ Now when you do a fsck, you'll be warned appropriately:
 	failed
 
 Also, git-annex will refuse to drop a file from elsewhere just because
-it can see a copy on the untrusted drive.
+it can see a copy on the untrusted repository.
 
 It's also possible to tell git-annex that you have an unusually high
 level of trust for a repository. See [[trust]] for details.

From 1a11085a50e79fbcce829d4ea89539107f5b306c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 19:35:35 -0400
Subject: [PATCH 0765/8313] drop: suppprt untrusted repos

---
 Backend/File.hs  | 22 +++++++++++++++-------
 debian/changelog |  2 +-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 8c39c3f803..ab14b82339 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -96,9 +96,10 @@ checkRemoveKey key numcopiesM = do
 		then return True
 		else do
 			(remotes, trusteduuids) <- Remotes.keyPossibilities key
-			untrusted <- reposWithoutUUID remotes trusteduuids
+			untrusteduuids <- trustGet UnTrusted
+			tocheck <- reposWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies trusteduuids untrusted []
+			findcopies numcopies trusteduuids tocheck []
 	where
 		findcopies need have [] bad
 			| length have >= need = return True
@@ -131,11 +132,18 @@ showLocations key exclude = do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	uuids <- liftIO $ keyLocations g key
-	let uuidsf = filter (\l -> l /= u && (not $ elem l exclude)) uuids
-	ppuuids <- prettyPrintUUIDs uuidsf
-	if null uuidsf
-		then showLongNote $ "No other repository is known to contain the file."
-		else showLongNote $ "Try making some of these repositories available:\n" ++ ppuuids
+	untrusteduuids <- trustGet UnTrusted
+	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
+	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
+	ppuuidswanted <- prettyPrintUUIDs uuidswanted
+	ppuuidsskipped <- prettyPrintUUIDs uuidsskipped
+	showLongNote $ message ppuuidswanted ppuuidsskipped
+	where
+		filteruuids list x = filter (\l -> not $ elem l x) list
+		message [] [] = "No other repository is known to contain the file."
+		message rs [] = "Try making some of these repositories available:\n" ++ rs
+		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
+		message rs us = message rs [] ++ message [] us
 
 showTriedRemotes :: [Git.Repo] -> Annex ()
 showTriedRemotes [] = return ()	
diff --git a/debian/changelog b/debian/changelog
index fcd986e1b6..c675d1456d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
-  * fsck: Take untrusted repositories into account.
+  * fsck, drop: Take untrusted repositories into account.
   * bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
 

From 15d27232bdaea0cf4f70cbbe6fa925d593d18e73 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:03:03 -0400
Subject: [PATCH 0766/8313] test untrusted repo handling

---
 test.hs | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/test.hs b/test.hs
index 817b1bafbe..4e02c2a6f2 100644
--- a/test.hs
+++ b/test.hs
@@ -133,11 +133,11 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy]
 			unannexed ingitfile
 
 test_drop :: Test
-test_drop = "git-annex drop" ~: TestList [noremote, withremote]
+test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpcopyrepo $ do
 			r <- git_annex "drop" ["-q", annexedfile]
-			(not r) @? "drop wrongly succeeded with no known copy of file"
+			not r @? "drop wrongly succeeded with no known copy of file"
 			annexed_present annexedfile
 			git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed"
 			annexed_notpresent annexedfile
@@ -145,9 +145,19 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote]
 			git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op"
 			unannexed ingitfile
 		withremote = "with remote" ~: TestCase $ intmpclonerepo $ do
+			git_annex "get" ["-q", annexedfile] @? "get failed"
+			annexed_present annexedfile
 			git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy"
 			annexed_notpresent annexedfile
 			inmainrepo $ annexed_present annexedfile
+		untrustedremote = "untrusted remote" ~: TestCase $ intmpclonerepo $ do
+			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
+			git_annex "get" ["-q", annexedfile] @? "get failed"
+			annexed_present annexedfile
+			r <- git_annex "drop" ["-q", annexedfile]
+			not r @? "drop wrongly suceeded with only an untrusted copy of the file"
+			annexed_present annexedfile
+			inmainrepo $ annexed_present annexedfile
 
 test_get :: Test
 test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
@@ -314,20 +324,27 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 		repo = "origin"
 
 test_fsck :: Test
-test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted]
+test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" ["-q"] @? "fsck failed"
 			Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
-			r <- git_annex "fsck" ["-q"]
-			not r @? "fsck failed to fail with numcopies unsatisfied"
+			fsck_should_fail "numcopies unsatisfied"
 			Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpcopyrepo $ do
 			git_annex "untrust" ["-q", "."] @? "untrust of current repo failed"
-			r <- git_annex "fsck" ["-q"]
-			not r @? "fsck failed to fail with content only available in untrusted (current) repository"
+			fsck_should_fail "content only available in untrusted (current) repository"
+			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
+			git_annex "fsck" ["-q"] @? "fsck failed on trusted repo"
+		withremoteuntrusted = TestCase $ intmpclonerepo $ do
+			Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed"
+			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
+			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
+			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
+			fsck_should_fail "content not replicated to enough non-untrusted repositories"
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
 			Content.allowWrite f
@@ -335,6 +352,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted]
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with corrupted file content"
 			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem"
+		fsck_should_fail m = do
+			r <- git_annex "fsck" ["-q"]
+			not r @? "fsck failed to fail with " ++ m
 
 test_migrate :: Test
 test_migrate = "git-annex migrate" ~: TestList [t False, t True]

From ee2e94f08703ee1b5f51bf5c2d26de141b58298c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:03:12 -0400
Subject: [PATCH 0767/8313] this should be a warning

---
 Backend/File.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index ab14b82339..f31752d90d 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -184,7 +184,7 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			showLongNote $ missingNote present needed ppuuids
+			warning $ missingNote present needed ppuuids
 			return False
 		else return True
 	where

From 3cb5cb6bf6f03ad3ef574a14ed35275916ac44b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:08:37 -0400
Subject: [PATCH 0768/8313] bring back display of keys

in fsck -q, that's the only way to know what file it means
---
 Backend/File.hs | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index f31752d90d..10304dd91c 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -184,22 +184,24 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			warning $ missingNote present needed ppuuids
+			warning $ missingNote (show key) present needed ppuuids
 			return False
 		else return True
 	where
 
-missingNote :: Int -> Int -> String -> String
-missingNote 0 _ [] = 
-		"** No known copies of this file exist!"
-missingNote 0 _ untrusted =
-		"Only these untrusted locations may have copies of this file!" ++
+missingNote :: String -> Int -> Int -> String -> String
+missingNote file 0 _ [] = 
+		"** No known copies of " ++ file ++ " exist!"
+missingNote file 0 _ untrusted =
+		"Only these untrusted locations may have copies of " ++ file ++
+		"\n-- they could lose it at any time!" ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
-missingNote present needed [] =
+missingNote file present needed [] =
 		"Only " ++ show present ++ " of " ++ show needed ++ 
-		" trustworthy copies of this file exist." ++
+		" trustworthy copies of " ++ file ++ " exist." ++
 		"\nBack it up with git-annex copy."
-missingNote present needed untrusted = missingNote present needed [] ++
+missingNote file present needed untrusted = 
+		missingNote file present needed [] ++
 		"\nThe following untrusted copies may also exist: " ++
 		"\n" ++ untrusted

From c30d38e108ade7eb4fa57552631b64dd3b9582c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:30:07 -0400
Subject: [PATCH 0769/8313] better warnings display

---
 CmdLine.hs       |  1 -
 Messages.hs      | 15 +++++++++------
 debian/changelog |  2 +-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 1b5daadeb0..76402a8218 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -70,7 +70,6 @@ usage header cmds options =
 			cmdparams c ++
 			pad (longest cmdparams + 2) (cmdparams c) ++
 			cmddesc c
-		indent l = "  " ++ l
 		pad n s = replicate (n - length s) ' '
 		longest f = foldl max 0 $ map (length . f) cmds
 
diff --git a/Messages.hs b/Messages.hs
index 2b98622309..6f4ec1e626 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -37,9 +37,8 @@ showProgress :: Annex ()
 showProgress = verbose $ liftIO $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indented
-	where
-		indented = join "\n" $ map (\l -> "  " ++ l) $ lines s 
+showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indent s
+
 showEndOk :: Annex ()
 showEndOk = verbose $ liftIO $ putStrLn "ok"
 
@@ -48,9 +47,13 @@ showEndFail = verbose $ liftIO $ putStrLn "\nfailed"
 
 {- Exception pretty-printing. -}
 showErr :: (Show a) => a -> Annex ()
-showErr e = warning $ show e
+showErr e = warning $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning s = do
+warning w = do
 	verbose $ liftIO $ putStr "\n"
-	liftIO $ hPutStrLn stderr $ "git-annex: " ++ s ++ " "
+	liftIO $ hFlush stdout
+	liftIO $ hPutStrLn stderr $ indent w
+
+indent :: String -> String
+indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/debian/changelog b/debian/changelog
index c675d1456d..92b21173c8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,7 +6,7 @@ git-annex (0.19) UNRELEASED; urgency=low
   * untrust: Now marks the current repository as untrusted.
   * semitrust: Now restores the default trust level. (What untrust used to do.)
   * fsck, drop: Take untrusted repositories into account.
-  * bugfix: Files were copied from trusted remotes first even if their
+  * Bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400

From e1d213d6e3c9fe0cda6e2b80c4abeb17db5d3a16 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 20:37:46 -0400
Subject: [PATCH 0770/8313] make filename available to fsck messages

---
 Backend.hs      |  4 ++--
 Backend/File.hs | 16 +++++++++-------
 Backend/URL.hs  |  4 ++--
 BackendTypes.hs |  5 +++--
 Command/Fsck.hs |  8 ++++----
 5 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index d9bf35f0dd..bdceea2f5b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -118,8 +118,8 @@ hasKey key = do
 	(B.hasKey backend) key
 
 {- Checks a key's backend for problems. -}
-fsckKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool
-fsckKey backend key numcopies = (B.fsckKey backend) key numcopies
+fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+fsckKey backend key file numcopies = (B.fsckKey backend) key file numcopies
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
diff --git a/Backend/File.hs b/Backend/File.hs
index 10304dd91c..26a6add1a4 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -166,14 +166,14 @@ getNumCopies Nothing = do
  - The passed action is first run to allow backends deriving this one
  - to do their own checks.
  -}
-checkKey :: (Key -> Annex Bool) -> Key -> Maybe Int -> Annex Bool
-checkKey a key numcopies = do
+checkKey :: (Key -> Annex Bool) -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKey a key file numcopies = do
 	a_ok <- a key
-	copies_ok <- checkKeyNumCopies key numcopies
+	copies_ok <- checkKeyNumCopies key file numcopies
 	return $ a_ok && copies_ok
 
-checkKeyNumCopies :: Key -> Maybe Int -> Annex Bool
-checkKeyNumCopies key numcopies = do
+checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
 	g <- Annex.gitRepo
 	locations <- liftIO $ keyLocations g key
@@ -184,10 +184,12 @@ checkKeyNumCopies key numcopies = do
 	if present < needed
 		then do
 			ppuuids <- prettyPrintUUIDs untrustedlocations
-			warning $ missingNote (show key) present needed ppuuids
+			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
 	where
+		filename Nothing k = show k
+		filename (Just f) _ = f
 
 missingNote :: String -> Int -> Int -> String -> String
 missingNote file 0 _ [] = 
@@ -203,5 +205,5 @@ missingNote file present needed [] =
 		"\nBack it up with git-annex copy."
 missingNote file present needed untrusted = 
 		missingNote file present needed [] ++
-		"\nThe following untrusted copies may also exist: " ++
+		"\nThe following untrusted locations may also have copies: " ++
 		"\n" ++ untrusted
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 45a204b07d..38954e5a31 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -41,8 +41,8 @@ dummyStore _ _ = return False
 dummyRemove :: Key -> Maybe a -> Annex Bool
 dummyRemove _ _ = return False
 
-dummyFsck :: Key -> Maybe a -> Annex Bool
-dummyFsck _ _ = return True
+dummyFsck :: Key -> Maybe FilePath -> Maybe a -> Annex Bool
+dummyFsck _ _ _ = return True
 
 dummyOk :: Key -> Annex Bool
 dummyOk _ = return True
diff --git a/BackendTypes.hs b/BackendTypes.hs
index e4b155f98f..fd4a61b989 100644
--- a/BackendTypes.hs
+++ b/BackendTypes.hs
@@ -31,9 +31,10 @@ data Backend a = Backend {
 	-- checks if a backend is storing the content of a key
 	hasKey :: Key -> a Bool,
 	-- called during fsck to check a key
-	-- (second parameter may be the number of copies that there should
+	-- (second parameter may be the filename associated with it)
+	-- (third parameter may be the number of copies that there should
 	-- be of the key)
-	fsckKey :: Key -> Maybe Int -> a Bool
+	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool
 }
 
 instance Show (Backend a) where
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index fc9bd7f527..b6f330d4c2 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -24,13 +24,13 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 start :: CommandStartAttrFile
 start (file, attr) = isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	return $ Just $ perform key backend numcopies
+	return $ Just $ perform key file backend numcopies
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
-perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform
-perform key backend numcopies = do
-	success <- Backend.fsckKey backend key numcopies
+perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
+perform key file backend numcopies = do
+	success <- Backend.fsckKey backend key (Just file) numcopies
 	if success
 		then return $ Just $ return True
 		else return Nothing

From 2e0cbcdf5db31ec6f092ba81fbb7fc6685c051a1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 01:29:08 +0000
Subject: [PATCH 0771/8313] Feature request for a sort of "dedup" for filenames

---
 ...4___command_that_will_skip_duplicates.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
new file mode 100644
index 0000000000..31734e01b9
--- /dev/null
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -0,0 +1,22 @@
+(Hi, this is paulproteus/Asheesh Laroia).
+
+I've been enjoying using git-annex to archive my data.
+
+It's great that, by using git-annex and the SHA1 backend, I get a space-saving kind of deduplication through the symbolic links.
+
+My question is, is there a frontend by which I can look at some files and only store the ones that are not already in the repository? That would help me in terms of personal file organization.
+
+It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
+
+ $ git annex add --no-add-if-already-present .
+ $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
+
+And then I'd do something like:
+
+ $ git clean -f
+
+to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
+
+I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks.
+
+(I would be even happier for "git annex add --unlink-duplicates .")

From 8dc7af780b8432d95dd6421bf7d54f11a2d312ba Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 01:29:35 +0000
Subject: [PATCH 0772/8313] Fix some formatting.

---
 ...git_annex__34___command_that_will_skip_duplicates.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index 31734e01b9..f4f0b7bced 100644
--- a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -1,4 +1,4 @@
-(Hi, this is paulproteus/Asheesh Laroia).
+(Hi, this is paulproteus@debian, AKA Asheesh).
 
 I've been enjoying using git-annex to archive my data.
 
@@ -8,12 +8,12 @@ My question is, is there a frontend by which I can look at some files and only s
 
 It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
 
- $ git annex add --no-add-if-already-present .
- $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
+* $ git annex add --no-add-if-already-present .
+* $ git commit -m "Slurping in some photos I found on my old laptop hard drive"
 
 And then I'd do something like:
 
- $ git clean -f
+* $ git clean -f
 
 to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
 

From e51b4d172b3ad2d6053bcf99ac72f8f1d72707d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 01:30:34 +0000
Subject: [PATCH 0773/8313]

---
 ...__34__git_annex__34___command_that_will_skip_duplicates.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index f4f0b7bced..68efc0d1ac 100644
--- a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -17,6 +17,6 @@ And then I'd do something like:
 
 to remove the files that didn't get annexed in this run. That way, only one filename would ever point to a particular SHA1.
 
-I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks.
+I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks. If this feature existed, I could comfortably toss them all into one git annex that grew, bit by bit, to store all of these files exactly once.
 
 (I would be even happier for "git annex add --unlink-duplicates .")

From 7ab481448c4cdc39bae746423f535ae742b8784a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 26 Jan 2011 21:52:51 -0400
Subject: [PATCH 0774/8313] wording

---
 doc/trust.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index dbddfe4269..f843d3858c 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -4,7 +4,7 @@ Git-annex supports three levels of trust of a repository:
 * untrusted
 * trusted
 
-## semi-trusted
+## semitrusted
 
 Normally, git-annex does not fully trust its stored [[location_tracking]]
 information. When removing content, it will directly check
@@ -15,7 +15,7 @@ Generally that explicit checking is a good idea. Consider that the current
 out. Or, a remote may have suffered a catastrophic loss of data, or itself
 been lost.
 
-There is still some trust involved here. A semi-trusted repository is
+There is still some trust involved here. A semitrusted repository is
 dependended on to retain a copy of the file content; possibly the only
 [[copy|copies]].
 

From 346ade8de55a527b5d8576ca11cc5cc03d8147cf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Thu, 27 Jan 2011 05:59:26 +0000
Subject: [PATCH 0775/8313]

---
 ...__git_annex__34___command_that_will_skip_duplicates.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index 68efc0d1ac..eda17aea66 100644
--- a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -4,7 +4,7 @@ I've been enjoying using git-annex to archive my data.
 
 It's great that, by using git-annex and the SHA1 backend, I get a space-saving kind of deduplication through the symbolic links.
 
-My question is, is there a frontend by which I can look at some files and only store the ones that are not already in the repository? That would help me in terms of personal file organization.
+I'm looking for the ability to filter files, before they get added to the annex, so that I don't add new files whose content is already in the annex.look  That would help me in terms of personal file organization.
 
 It seems there is not, so this is a wishlist bug filed so that maybe such a thing might exist. What I would really like to do is:
 
@@ -20,3 +20,7 @@ to remove the files that didn't get annexed in this run. That way, only one file
 I want this because I have copies of various of mine (photos, in particular) scattered across various hard disks. If this feature existed, I could comfortably toss them all into one git annex that grew, bit by bit, to store all of these files exactly once.
 
 (I would be even happier for "git annex add --unlink-duplicates .")
+
+(Another way to do this would be to "git annex add" them all, and then use a "git annex remove-duplicates" that could prompt me about which files are duplicates of each other, and then I could pipe that command's output into xargs git rm.)
+
+(As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)

From 9d3ba00a15ecc67eb98a9cf99561f2a54ade15c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 14:01:30 -0400
Subject: [PATCH 0776/8313] add test of adding a second file with the same sha1

---
 test.hs | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/test.hs b/test.hs
index 4e02c2a6f2..d7a6bd152f 100644
--- a/test.hs
+++ b/test.hs
@@ -94,15 +94,27 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 		reponame = "test repo"
 
 test_add :: Test
-test_add = "git-annex add" ~: TestCase $ inmainrepo $ do
-	writeFile annexedfile $ content annexedfile
-	git_annex "add" ["-q", annexedfile] @? "add failed"
-	annexed_present annexedfile
-	writeFile ingitfile $ content ingitfile
-	Utility.boolSystem "git" ["add", ingitfile] @? "git add failed"
-	Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed"
-	git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
-	unannexed ingitfile
+test_add = "git-annex add" ~: TestList [basic, sha1dup]
+	where
+		-- this test case runs in the main repo, to set up a basic
+		-- annexed file that later tests will use
+		basic = TestCase $ inmainrepo $ do
+			writeFile annexedfile $ content annexedfile
+			git_annex "add" ["-q", annexedfile] @? "add failed"
+			annexed_present annexedfile
+			writeFile ingitfile $ content ingitfile
+			Utility.boolSystem "git" ["add", ingitfile] @? "git add failed"
+			Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed"
+			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
+			unannexed ingitfile
+		sha1dup = TestCase $ intmpclonerepo $ do
+			writeFile sha1annexedfile $ content sha1annexedfile
+			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
+			annexed_present sha1annexedfile
+			writeFile sha1annexedfiledup $ content sha1annexedfiledup
+			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
+			annexed_present sha1annexedfiledup
+			annexed_present sha1annexedfile
 
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
@@ -647,6 +659,9 @@ annexedfile = "foo"
 sha1annexedfile :: String
 sha1annexedfile = "sha1foo"
 
+sha1annexedfiledup :: String
+sha1annexedfiledup = "sha1foodup"
+
 ingitfile :: String
 ingitfile = "bar"
 
@@ -655,6 +670,7 @@ content f
 	| f == annexedfile = "annexed file content"
 	| f == ingitfile = "normal file content"
 	| f == sha1annexedfile ="sha1 annexed file content"
+	| f == sha1annexedfiledup = content sha1annexedfile
 	| otherwise = "unknown file " ++ f
 
 changecontent :: FilePath -> IO ()

From 2087c8ef5e21d3a6c9427b08b59c6814be767baf Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 27 Jan 2011 18:29:44 +0000
Subject: [PATCH 0777/8313] Added a comment

---
 ..._fd213310ee548d8726791d2b02237fde._comment | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment
new file mode 100644
index 0000000000..094e4526eb
--- /dev/null
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-01-27T18:29:44Z"
+ content="""
+Hey Asheesh, I'm happy you're finding git-annex useful.
+
+So, there are two forms of duplication going on here. There's duplication of the content, and duplication of the filenames 
+pointing at that content.
+
+Duplication of the filenames is probably not a concern, although it's what I thought you were talking about at first. It's probably info worth recording that backup-2010/some_dir/foo and backup-2009/other_dir/foo are two names you've used for the same content in the past. If you really wanted to remove backup-2009/foo, you could do it by writing a script that looks at the basenames of the symlink targets and removes files that point to the same content as other files.
+
+Using SHA1 ensures that the same key is used for identical files, so generally avoids duplication of content. But if you have 2 disks with an identical file on each, and make them both into annexes, then git-annex will happily retain both
+copies of the content, one per disk. It generally considers keeping copies of content a good thing. :)
+
+So, what if you want to remove the unnecessary copies? Well, there's a really simple way:
+
+
+cd /media/usb-1
+git remote add other-disk /media/usb-0
+git annex add
+git annex drop
+

+
+This asks git-annex to add everything to the annex, but then remove any file contents that it can safely remove. What can it safely remove? Well, anything that it can verify is on another repository such as \"other-disk\"! So, this will happily drop any duplicated file contents, while leaving all the rest alone.
+
+In practice, you might not want to have all your old backup disks mounted at the same time and configured as remotes. Look into configuring [[trust]] to avoid needing do to that. If usb-0 is already a trusted disk, all you need is a simple \"git annex drop\" on usb-1.
+"""]]

From 5e54eb79b8dbb58f9e772fc4c7c9fe900d045217 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 15:12:38 -0400
Subject: [PATCH 0778/8313] less verbose

---
 Backend/File.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 26a6add1a4..35cbc01911 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -196,7 +196,6 @@ missingNote file 0 _ [] =
 		"** No known copies of " ++ file ++ " exist!"
 missingNote file 0 _ untrusted =
 		"Only these untrusted locations may have copies of " ++ file ++
-		"\n-- they could lose it at any time!" ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
 missingNote file present needed [] =

From dee9655237d753cfeffc5cb7ffccaa86b464dcc2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 15:45:22 -0400
Subject: [PATCH 0779/8313] bugfix to move --to

Due to recent changes, the remotes config was not read before the remote
to act on was picked.
---
 Command/Move.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 3eadf30c6f..6dc2e48746 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -34,6 +34,7 @@ seek = [withFilesInGit $ start True]
  - moving data in the key-value backend. -}
 start :: Bool -> CommandStartString
 start move file = do
+	Remotes.readConfigs
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
 	case (from, to) of
@@ -80,7 +81,6 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 			return $ Just $ toPerform dest move key
 toPerform :: Git.Repo -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
-	Remotes.readConfigs
 	-- checking the remote is expensive, so not done in the start step
 	isthere <- Remotes.inAnnex dest key
 	case isthere of

From 96404170673e4ff42b68de1ba967a09a88dd550e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 16:10:45 -0400
Subject: [PATCH 0780/8313] avoid warning when symlink in the repo contains a
 colon but is not a pointer to annexed content

---
 Backend.hs   | 3 ++-
 Locations.hs | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/Backend.hs b/Backend.hs
index bdceea2f5b..74f71f8bea 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -137,7 +137,8 @@ lookupFile file = do
 		makekey bs l = do
 			case maybeLookupBackendName bs bname of
 				Nothing -> do
-					unless (null kname || null bname) $
+					unless (null kname || null bname ||
+					        not (isLinkToAnnex l)) $
 						warning skip
 					return Nothing
 				Just backend -> return $ Just (k, backend)
diff --git a/Locations.hs b/Locations.hs
index 6c541e937c..327c099e38 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -15,6 +15,7 @@ module Locations (
 	annexTmpLocation,
 	annexBadLocation,
 	annexUnusedLog,
+	isLinkToAnnex,
 	annexDir,
 	annexObjectDir,
 
@@ -22,6 +23,7 @@ module Locations (
 ) where
 
 import Data.String.Utils
+import Data.List
 
 import Types
 import qualified GitRepo as Git
@@ -69,6 +71,10 @@ annexBadLocation r = annexDir r ++ "/bad/"
 annexUnusedLog :: Git.Repo -> FilePath
 annexUnusedLog r = annexDir r ++ "/unused"
 
+{- Checks a symlink target to see if it appears to point to annexed content. -}
+isLinkToAnnex :: FilePath -> Bool
+isLinkToAnnex s = isInfixOf "/.git/annex/objects/" s
+
 {- Converts a key into a filename fragment.
  -
  - Escape "/" in the key name, to keep a flat tree of files and avoid

From 6be516ae3bddb8f05ea62661019836e03be12a2c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 16:31:29 -0400
Subject: [PATCH 0781/8313] use isPrefixOf

---
 Command.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command.hs b/Command.hs
index cbfb265002..bedb18cc97 100644
--- a/Command.hs
+++ b/Command.hs
@@ -13,6 +13,7 @@ import System.Posix.Files
 import Control.Monad (filterM)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
+import Data.List
 
 import Types
 import qualified Backend
@@ -186,8 +187,7 @@ filterFiles l = do
 			let regexp = compile (toregex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
-		notState f = stateLoc /= take stateLocLen f
-		stateLocLen = length stateLoc
+		notState f = not $ isPrefixOf stateLoc f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False

From 167523f09d48777f3a5931fdcbc21b9d363e0e6c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 17:00:32 -0400
Subject: [PATCH 0782/8313] better directory handling

Rename Locations functions for better consitency, and make their values
more consistent too.

Used  rather than manually building paths. There are still more places
that manually do so, but are tricky, due to the behavior of  when
the second FilePath is absolute. So I only changed places where
it obviously was relative.
---
 Backend/File.hs       |  2 +-
 Backend/SHA1.hs       |  2 +-
 Backend/WORM.hs       |  2 +-
 CmdLine.hs            |  2 +-
 Command.hs            |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Init.hs       |  3 +-
 Command/Migrate.hs    |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/Unlock.hs     |  2 +-
 Command/Unused.hs     |  2 +-
 Content.hs            | 20 ++++++------
 Locations.hs          | 74 +++++++++++++++++++++++++++----------------
 Remotes.hs            |  6 ++--
 Upgrade.hs            |  4 +--
 Version.hs            |  8 ++---
 16 files changed, 78 insertions(+), 57 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 35cbc01911..68dd4db271 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -77,7 +77,7 @@ copyKeyFile key file = do
 		-- before going on to the next remote.)
 		probablyPresent r =
 			if not $ Git.repoIsUrl r
-				then liftIO $ doesFileExist $ annexLocation r key
+				then liftIO $ doesFileExist $ gitAnnexLocation r key
 				else return True
 		docopy r continue = do
 			showNote $ "copying from " ++ Git.repoDescribe r ++ "..."
diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs
index f8dbea4b03..3d868dbd19 100644
--- a/Backend/SHA1.hs
+++ b/Backend/SHA1.hs
@@ -48,7 +48,7 @@ keyValue file = do
 checkKeySHA1 :: Key -> Annex Bool
 checkKeySHA1 key = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
 	if not present
 		then return True
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 56f243396e..20a81d8411 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -57,7 +57,7 @@ keySize key = read $ section !! 1
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
 	if not present
 		then return True
diff --git a/CmdLine.hs b/CmdLine.hs
index 76402a8218..68d1e0dd31 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -111,7 +111,7 @@ shutdown errnum = do
 	-- are left behind to allow resuming on re-run.
 	when (errnum == 0) $ do
 		g <- Annex.gitRepo
-		let tmp = annexTmpLocation g
+		let tmp = gitAnnexTmpDir g
 		exists <- liftIO $ doesDirectoryExist tmp
 		when exists $ liftIO $ removeDirectoryRecursive tmp
 		liftIO $ createDirectoryIfMissing True tmp
diff --git a/Command.hs b/Command.hs
index bedb18cc97..bebbf3f1be 100644
--- a/Command.hs
+++ b/Command.hs
@@ -187,7 +187,7 @@ filterFiles l = do
 			let regexp = compile (toregex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
-		notState f = not $ isPrefixOf stateLoc f
+		notState f = not $ isPrefixOf stateDir f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index cfd6fc3d0e..9427f81035 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -40,7 +40,7 @@ start s = do
 readUnusedLog :: Annex (M.Map String Key)
 readUnusedLog = do
 	g <- Annex.gitRepo
-	let f = annexUnusedLog g
+	let f = gitAnnexUnusedLog g
 	e <- liftIO $ doesFileExist f
 	if e
 		then do
diff --git a/Command/Init.hs b/Command/Init.hs
index 47ac8e4c08..e780c88634 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -10,6 +10,7 @@ module Command.Init where
 import Control.Monad.State (liftIO)
 import Control.Monad (when)
 import System.Directory
+import System.FilePath
 
 import Command
 import qualified Annex
@@ -75,7 +76,7 @@ gitAttributesWrite repo = do
 					attributes]
 
 attrLine :: String		
-attrLine = stateLoc ++ "*.log merge=union"
+attrLine = stateDir  "*.log merge=union"
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 566b508c0c..c0e80c5b47 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -50,7 +50,7 @@ perform file oldkey newbackend = do
 	-- (the file can't be stored as usual, because it's already a symlink).
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
-	let src = annexLocation g oldkey
+	let src = gitAnnexLocation g oldkey
 	stored <- Backend.storeFileKey src $ Just newbackend
 	case stored of
 		Nothing -> return Nothing
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index aaa0b48369..cb883b53aa 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -32,7 +32,7 @@ start keyname = do
 	let key = genKey (head backends) keyname
 	present <- inAnnex key
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	when present $
 		liftIO $ rsyncServerSend file
 	liftIO exitFailure
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 645fac8a25..bd1021cc3c 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -43,7 +43,7 @@ perform dest key = do
 		error "content not present"
 
 	g <- Annex.gitRepo
-	let src = annexLocation g key
+	let src = gitAnnexLocation g key
 	liftIO $ removeFile dest
 	showNote "copying..."
 	ok <- liftIO $ copyFile src dest
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 9fdf4cda65..5e5698e38d 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -43,7 +43,7 @@ checkUnused = do
 	unused <- unusedKeys
 	let list = number 1 unused
 	g <- Annex.gitRepo
-	liftIO $ writeFile (annexUnusedLog g) $ unlines $ 
+	liftIO $ writeFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
 	if null unused
 		then return True
diff --git a/Content.hs b/Content.hs
index 0cbd6905cb..d0ed8d861c 100644
--- a/Content.hs
+++ b/Content.hs
@@ -35,12 +35,12 @@ import qualified GitRepo as Git
 import qualified Annex
 import Utility
 
-{- Checks if a given key is currently present in the annexLocation. -}
+{- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
 inAnnex key = do
 	g <- Annex.gitRepo
 	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"
-	liftIO $ doesFileExist $ annexLocation g key
+	liftIO $ doesFileExist $ gitAnnexLocation g key
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
@@ -51,7 +51,7 @@ calcGitLink file key = do
 		Just f -> f
 		Nothing -> error $ "unable to normalize " ++ file
 	return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++
-		annexLocationRelative key
+		annexLocation key
 
 {- Updates the LocationLog when a key's presence changes. -}
 logStatus :: Key -> LogStatus -> Annex ()
@@ -67,7 +67,7 @@ logStatus key status = do
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
 	g <- Annex.gitRepo
-	let tmp = annexTmpLocation g ++ keyFile key
+	let tmp = gitAnnexTmpDir g  keyFile key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	success <- action tmp
 	if success
@@ -97,7 +97,7 @@ allowWrite f = do
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
 	g <- Annex.gitRepo
-	let dest = annexLocation g key
+	let dest = gitAnnexLocation g key
 	let dir = parentDir dest
 	liftIO $ do
 		createDirectoryIfMissing True dir
@@ -110,7 +110,7 @@ moveAnnex key src = do
 removeAnnex :: Key -> Annex ()
 removeAnnex key = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	let dir = parentDir file
 	liftIO $ do
 		allowWrite dir
@@ -121,7 +121,7 @@ removeAnnex key = do
 fromAnnex :: Key -> FilePath -> Annex ()
 fromAnnex key dest = do
 	g <- Annex.gitRepo
-	let file = annexLocation g key
+	let file = gitAnnexLocation g key
 	let dir = parentDir file
 	liftIO $ do
 		allowWrite dir
@@ -134,8 +134,8 @@ fromAnnex key dest = do
 moveBad :: Key -> Annex FilePath
 moveBad key = do
 	g <- Annex.gitRepo
-	let src = annexLocation g key
-	let dest = annexBadLocation g ++ takeFileName src
+	let src = gitAnnexLocation g key
+	let dest = gitAnnexBadDir g  takeFileName src
 	liftIO $ createDirectoryIfMissing True (parentDir dest)
 	liftIO $ allowWrite (parentDir src)
 	liftIO $ renameFile src dest
@@ -146,7 +146,7 @@ moveBad key = do
 getKeysPresent :: Annex [Key]
 getKeysPresent = do
 	g <- Annex.gitRepo
-	getKeysPresent' $ annexObjectDir g
+	getKeysPresent' $ gitAnnexObjectDir g
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
diff --git a/Locations.hs b/Locations.hs
index 327c099e38..b2624754ec 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -7,73 +7,93 @@
 
 module Locations (
 	gitStateDir,
-	stateLoc,
+	stateDir,
 	keyFile,
 	fileKey,
+	gitAnnexLocation,
 	annexLocation,
-	annexLocationRelative,
-	annexTmpLocation,
-	annexBadLocation,
-	annexUnusedLog,
+	gitAnnexDir,
+	gitAnnexObjectDir,
+	gitAnnexTmpDir,
+	gitAnnexBadDir,
+	gitAnnexUnusedLog,
 	isLinkToAnnex,
-	annexDir,
-	annexObjectDir,
 
 	prop_idempotent_fileKey
 ) where
 
+import System.FilePath
 import Data.String.Utils
 import Data.List
 
 import Types
 import qualified GitRepo as Git
 
+{- Conventions:
+ -
+ - Functions ending in "Dir" should always return values ending with a
+ - trailing path separator. Most code does not rely on that, but a few
+ - things do. 
+ -
+ - Everything else should not end in a trailing path sepatator. 
+ -
+ - Only functions (with names starting with "git") that build a path
+ - based on a git repository should return an absolute path.
+ - Everything else should use relative paths.
+ -}
+
 {- Long-term, cross-repo state is stored in files inside the .git-annex
  - directory, in the git repository's working tree. -}
-stateLoc :: String
-stateLoc = ".git-annex/"
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
-gitStateDir repo = Git.workTree repo ++ "/" ++ stateLoc
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
 
-{- Annexed file's absolute location. -}
-annexLocation :: Git.Repo -> Key -> FilePath
-annexLocation r key = 
-	Git.workTree r ++ "/" ++ annexLocationRelative key
+{- Annexed content is stored in .git/annex/objects; .git/annex is used
+ - for other temporary storage also. -}
+annexDir :: FilePath
+annexDir = addTrailingPathSeparator $ ".git/annex"
+objectDir :: FilePath
+objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
 {- Annexed file's location relative to git's working tree. 
  -
  - Note: Assumes repo is NOT bare.-}
-annexLocationRelative :: Key -> FilePath
-annexLocationRelative key = ".git/annex/objects/" ++ f ++ "/" ++ f
+annexLocation :: Key -> FilePath
+annexLocation key = ".git/annex/objects"  f  f
 	where
 		f = keyFile key
 
+{- Annexed file's absolute location in a repository. -}
+gitAnnexLocation :: Git.Repo -> Key -> FilePath
+gitAnnexLocation r key = Git.workTree r  annexLocation key
+
 {- The annex directory of a repository.
  -
  - Note: Assumes repo is NOT bare. -}
-annexDir :: Git.Repo -> FilePath
-annexDir r = Git.workTree r ++ "/.git/annex"
+gitAnnexDir :: Git.Repo -> FilePath
+gitAnnexDir r = addTrailingPathSeparator $ Git.workTree r  annexDir
 
 {- The part of the annex directory where file contents are stored.
  -}
-annexObjectDir :: Git.Repo -> FilePath
-annexObjectDir r = annexDir r ++ "/objects"
+gitAnnexObjectDir :: Git.Repo -> FilePath
+gitAnnexObjectDir r = addTrailingPathSeparator $ Git.workTree r  objectDir
 
 {- .git-annex/tmp/ is used for temp files -}
-annexTmpLocation :: Git.Repo -> FilePath
-annexTmpLocation r = annexDir r ++ "/tmp/"
+gitAnnexTmpDir :: Git.Repo -> FilePath
+gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
 {- .git-annex/bad/ is used for bad files found during fsck -}
-annexBadLocation :: Git.Repo -> FilePath
-annexBadLocation r = annexDir r ++ "/bad/"
+gitAnnexBadDir :: Git.Repo -> FilePath
+gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 
 {- .git/annex/unused is used to number possibly unused keys -}
-annexUnusedLog :: Git.Repo -> FilePath
-annexUnusedLog r = annexDir r ++ "/unused"
+gitAnnexUnusedLog :: Git.Repo -> FilePath
+gitAnnexUnusedLog r = gitAnnexDir r  "unused"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = isInfixOf "/.git/annex/objects/" s
+isLinkToAnnex s = isInfixOf ("/" ++ objectDir) s
 
 {- Converts a key into a filename fragment.
  -
diff --git a/Remotes.hs b/Remotes.hs
index be4c1383af..616db225ef 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -225,7 +225,7 @@ byName name = do
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
-	| not $ Git.repoIsUrl r = liftIO $ copyFile (annexLocation r key) file
+	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsynchelper r True key file
 	| otherwise = error "copying from non-ssh repo not supported"
 
@@ -234,7 +234,7 @@ copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
 		g <- Annex.gitRepo
-		let keysrc = annexLocation g key
+		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ do
 			a <- Annex.new r []
@@ -245,7 +245,7 @@ copyToRemote r key
 				return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
-		let keysrc = annexLocation g key
+		let keysrc = gitAnnexLocation g key
 		rsynchelper r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
diff --git a/Upgrade.hs b/Upgrade.hs
index 596d525db2..1e70e68d56 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -39,9 +39,9 @@ upgradeFrom0 = do
 	g <- Annex.gitRepo
 
 	-- do the reorganisation of the files
-	let olddir = annexDir g
+	let olddir = gitAnnexDir g
 	keys <- getKeysPresent0' olddir
-	_ <- mapM (\k -> moveAnnex k $ olddir ++ "/" ++ keyFile k) keys
+	_ <- mapM (\k -> moveAnnex k $ olddir  keyFile k) keys
 
 	-- update the symlinks to the files
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
diff --git a/Version.hs b/Version.hs
index fc1ce3d7ec..9e31d3c9eb 100644
--- a/Version.hs
+++ b/Version.hs
@@ -29,10 +29,10 @@ getVersion = do
 		then return $ Just v
 		else do
 			-- version 0 was not recorded in .git/config;
-			-- such a repo should have an annexDir but no
-			-- annexObjectDir
-			d <- liftIO $ doesDirectoryExist $ annexDir g
-			o <- liftIO $ doesDirectoryExist $ annexObjectDir g
+			-- such a repo should have an gitAnnexDir but no
+			-- gitAnnexObjectDir
+			d <- liftIO $ doesDirectoryExist $ gitAnnexDir g
+			o <- liftIO $ doesDirectoryExist $ gitAnnexObjectDir g
 			if d && not o
 				then return $ Just "0"
 				else return Nothing -- no version yet

From 4f9336bb60ca4dc08ca271605de71415c98ec3ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Jan 2011 17:58:30 -0400
Subject: [PATCH 0783/8313] idiom

---
 Command.hs         | 2 +-
 GitRepo.hs         | 4 ++--
 Locations.hs       | 2 +-
 git-annex-shell.hs | 2 +-
 test.hs            | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command.hs b/Command.hs
index bebbf3f1be..298234fa05 100644
--- a/Command.hs
+++ b/Command.hs
@@ -187,7 +187,7 @@ filterFiles l = do
 			let regexp = compile (toregex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
-		notState f = not $ isPrefixOf stateDir f
+		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
diff --git a/GitRepo.hs b/GitRepo.hs
index ec363fe73b..4e69544d43 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -323,7 +323,7 @@ configRemotes repo = map construct remotepairs
 			| otherwise = repoFromPath v
 		-- git remotes can be written scp style -- [user@]host:dir
 		-- where dir is relative to the user's home directory.
-		scpstyle v = isInfixOf ":" v && (not $ isInfixOf "//" v)
+		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
 				bits = split ":" v
@@ -458,7 +458,7 @@ prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
  -}
 absDir :: String -> IO String
 absDir d
-	| isPrefixOf "/" d = expandt d
+	| "/" `isPrefixOf` d = expandt d
 	| otherwise = do
 		h <- myhomedir
 		return $ h ++ d
diff --git a/Locations.hs b/Locations.hs
index b2624754ec..75843c29eb 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -93,7 +93,7 @@ gitAnnexUnusedLog r = gitAnnexDir r  "unused"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = isInfixOf ("/" ++ objectDir) s
+isLinkToAnnex s = ("/" ++ objectDir) `isInfixOf` s
 
 {- Converts a key into a filename fragment.
  -
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 78dd777904..fa2a7f606a 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -51,7 +51,7 @@ main' c@(cmd:dir:params)
 main' c@(cmd:_)
 	-- Handle the case of being the user's login shell. It will be passed
 	-- a single string containing all the real parameters.
-	| isPrefixOf "git-annex-shell " cmd = main' $ drop 1 $ shellUnEscape cmd
+	| "git-annex-shell " `isPrefixOf` cmd = main' $ drop 1 $ shellUnEscape cmd
 	| elem cmd builtins = failure
 	| otherwise = external c
 
diff --git a/test.hs b/test.hs
index d7a6bd152f..5dda3b835a 100644
--- a/test.hs
+++ b/test.hs
@@ -88,7 +88,7 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 	e <- doesFileExist annexlog
 	e @? (annexlog ++ " not created")
 	c <- readFile annexlog
-	isInfixOf reponame c @? annexlog ++ " does not contain repo name"
+	reponame `isInfixOf` c @? annexlog ++ " does not contain repo name"
 	where
 		annexlog = ".git-annex/uuid.log"
 		reponame = "test repo"

From 5ab922ce0e47e200f943e8a8c9aeb77d2a863b3e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
 
Date: Fri, 28 Jan 2011 07:30:06 +0000
Subject: [PATCH 0784/8313] Added a comment: I actually *do* want to avoid
 duplication of filenames

---
 ...2_4394bde1c6fd44acae649baffe802775._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment

diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment
new file mode 100644
index 0000000000..04d58a4598
--- /dev/null
+++ b/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I"
+ nickname="Asheesh"
+ subject="I actually *do* want to avoid duplication of filenames"
+ date="2011-01-28T07:30:05Z"
+ content="""
+I really do want just one filename per file, at least for some cases.
+
+For my photos, there's no benefit to having a few filenames point to the same file. As I'm putting them all into the git-annex, that is a good time to remove the pure duplicates so that I don't e.g. see them twice when browsing the directory as a gallery. Also, I am uploading my photos to the web, and I want to avoid uploading the same photo (by content) twice.
+
+I hope that makes things clearer!
+
+For now I'm just doing this:
+
+* paulproteus@renaissance:/mnt/backups-terabyte/paulproteus/sd-card-from-2011-01-06/sd-cards/DCIM/100CANON $ for file in *; do hash=$(sha1sum \"$file\"); if ls /home/paulproteus/Photos/in-flickr/.git-annex | grep -q \"$hash\"; then echo already annexed ; else flickr_upload \"$file\" && mv \"$file\" \"/home/paulproteus/Photos/in-flickr/2011-01-28/from-some-nested-sd-card-bk\" && (cd /home/paulproteus/Photos/in-flickr/2011-01-28/from-some-nested-sd-card-bk && git annex add . && git commit -m ...) ; fi; done
+
+(Yeah, Flickr for my photos for now. I feel sad about betraying the principle of autonomo.us-ness.)
+"""]]

From 04fe906ac6e611fd59ef44244a01e8fe61abec6f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 12:35:51 -0400
Subject: [PATCH 0785/8313] use safewritefile

---
 Command/Init.hs   | 5 +++--
 Command/Uninit.hs | 2 +-
 Command/Unused.hs | 3 ++-
 LocationLog.hs    | 8 +-------
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index e780c88634..2976b988d1 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -20,6 +20,7 @@ import Version
 import Messages
 import Locations
 import Types
+import Utility
 	
 command :: [Command]
 command = [Command "init" paramDesc seek
@@ -61,7 +62,7 @@ gitAttributesWrite repo = do
 	exists <- doesFileExist attributes
 	if not exists
 		then do
-			writeFile attributes $ attrLine ++ "\n"
+			safeWriteFile attributes $ attrLine ++ "\n"
 			commit
 		else do
 			content <- readFile attributes
@@ -85,7 +86,7 @@ gitPreCommitHookWrite repo = do
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
 		else liftIO $ do
-			writeFile hook preCommitScript
+			safeWriteFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
 	where
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 93465df373..e9406ce3af 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -59,5 +59,5 @@ gitAttributesUnWrite repo = do
 	attrexists <- doesFileExist attributes
 	when attrexists $ do
 		c <- readFileStrict attributes
-		writeFile attributes $ unlines $
+		safeWriteFile attributes $ unlines $
 			filter (/= Command.Init.attrLine) $ lines c
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5e5698e38d..b9dc62a324 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,6 +16,7 @@ import Types
 import Content
 import Messages
 import Locations
+import Utility
 import qualified Annex
 import qualified GitRepo as Git
 import qualified Backend
@@ -43,7 +44,7 @@ checkUnused = do
 	unused <- unusedKeys
 	let list = number 1 unused
 	g <- Annex.gitRepo
-	liftIO $ writeFile (gitAnnexUnusedLog g) $ unlines $ 
+	liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
 	if null unused
 		then return True
diff --git a/LocationLog.hs b/LocationLog.hs
index 926939051c..56953bc029 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -32,7 +32,6 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as Map
 import System.Directory
-import System.Posix.Process
 import Control.Monad (when)
 
 import qualified GitRepo as Git
@@ -112,12 +111,7 @@ readLog file = do
 
 {- Writes a set of lines to a log file -}
 writeLog :: FilePath -> [LogLine] -> IO ()
-writeLog file ls = do
-	pid <- getProcessID
-	let tmpfile = file ++ ".tmp" ++ show pid
-	createDirectoryIfMissing True (parentDir file)
-	writeFile tmpfile $ unlines $ map show ls
-	renameFile tmpfile file
+writeLog file ls = safeWriteFile file (unlines $ map show ls)
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> UUID -> IO LogLine

From e6da7eb1770e506661d8c6755736ed285a17554a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 14:10:50 -0400
Subject: [PATCH 0786/8313] Improved temp file handling

* Improved temp file handling. Transfers of content can now be resumed
  from temp files later; the resume does not have to be the immediate
  next git-annex run.
* unused: Include partially transferred content in the list.
---
 CmdLine.hs                      | 10 ----
 Command/DropUnused.hs           |  7 +++
 Command/Unused.hs               | 95 +++++++++++++++++++++++----------
 Content.hs                      |  2 +-
 Locations.hs                    |  5 ++
 debian/changelog                |  4 ++
 doc/bugs/tmp_file_handling.mdwn |  2 +
 doc/git-annex.mdwn              |  2 +-
 8 files changed, 87 insertions(+), 40 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 68d1e0dd31..82a21d0fc7 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -105,13 +105,3 @@ shutdown errnum = do
 	unless (q == GitQueue.empty) $ do
 		showSideAction "Recording state in git..."
 		Annex.queueRun
-
-	-- If nothing failed, clean up any files left in the temp directory,
-	-- but leave the directory itself. If something failed, temp files
-	-- are left behind to allow resuming on re-run.
-	when (errnum == 0) $ do
-		g <- Annex.gitRepo
-		let tmp = gitAnnexTmpDir g
-		exists <- liftIO $ doesDirectoryExist tmp
-		when exists $ liftIO $ removeDirectoryRecursive tmp
-		liftIO $ createDirectoryIfMissing True tmp
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 9427f81035..63216ce4f1 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,6 +7,7 @@
 
 module Command.DropUnused where
 
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Directory
@@ -33,8 +34,14 @@ start s = do
 	case M.lookup s m of
 		Nothing -> return Nothing
 		Just key -> do
+			g <- Annex.gitRepo
 			showStart "dropunused" s
 			backend <- keyBackend key
+			-- drop both content in the backend and any tmp
+			-- file for the key
+			let tmp = gitAnnexTmpLocation g key
+			tmp_exists <- liftIO $ doesFileExist tmp
+			when tmp_exists $ liftIO $ removeFile tmp
 			return $ Just $ Command.Drop.perform key backend (Just 0)
 
 readUnusedLog :: Annex (M.Map String Key)
diff --git a/Command/Unused.hs b/Command/Unused.hs
index b9dc62a324..4ffa6a17f8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -7,9 +7,12 @@
 
 module Command.Unused where
 
+import Control.Monad (filterM, unless)
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import Data.Maybe
+import System.FilePath
+import System.Directory
 
 import Command
 import Types
@@ -41,49 +44,71 @@ perform = do
 checkUnused :: Annex Bool
 checkUnused = do
 	showNote "checking for unused data..."
-	unused <- unusedKeys
-	let list = number 1 unused
+	(unused, staletmp) <- unusedKeys
+	let unusedlist = number 0 unused
+	let staletmplist = number (length unused) staletmp
+	let list = unusedlist ++ staletmplist
 	g <- Annex.gitRepo
 	liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
-	if null unused
-		then return True
-		else do
-			showLongNote $ w list
-			return False
+	unless (null unused) $
+		showLongNote $ unusedmsg unusedlist
+	unless (null staletmp) $
+		showLongNote $ staletmpmsg staletmplist
+	unless (null list) $
+		showLongNote $ "\n"
+	return $ null list
+
 	where
-		w u = unlines $
-			["Some annexed data is no longer pointed to by any files in the repository:",
-			 "  NUMBER  KEY"]
-			++ map cols u ++
-			["(To see where data was previously used, try: git log --stat -S'KEY')",
-			 "(To remove unwanted data: git-annex dropunused NUMBER)",
-			 ""]
+		unusedmsg u = unlines $
+			["Some annexed data is no longer pointed to by any files in the repository:"]
+			++ table u ++
+			["(To see where data was previously used, try: git log --stat -S'KEY')"] ++
+			dropmsg
+		staletmpmsg t = unlines $ 
+			["Some partially transferred data exists in temporary files:"]
+			++ table t ++ dropmsg
+		dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"]
+			
+		table l = ["  NUMBER  KEY"] ++ map cols l
 		cols (n,k) = "  " ++ pad 6 (show n) ++ "  " ++ show k
 		pad n s = s ++ replicate (n - length s) ' '
 
-number :: Integer -> [a] -> [(Integer, a)]
+number :: Int -> [a] -> [(Int, a)]
 number _ [] = []
-number n (x:xs) = (n, x):(number (n+1) xs)
+number n (x:xs) = (n+1, x):(number (n+1) xs)
 
 {- Finds keys whose content is present, but that do not seem to be used
- - by any files in the git repo. -}
-unusedKeys :: Annex [Key]
+ - by any files in the git repo, or that are only present as tmp files. -}
+unusedKeys :: Annex ([Key], [Key])
 unusedKeys = do
+	g <- Annex.gitRepo
 	present <- getKeysPresent
 	referenced <- getKeysReferenced
 	
-	-- Constructing a single map, of the set that tends to be smaller,
-	-- appears more efficient in both memory and CPU than constructing
-	-- and taking the M.difference of two maps.
-	let present_m = existsMap present
-	let unused_m = remove referenced present_m
-	return $ M.keys unused_m
-	where
-		remove a b = foldl (flip M.delete) b a
+	let unused = present `exclude` referenced
 
-existsMap :: Ord k => [k] -> M.Map k Int
-existsMap l = M.fromList $ map (\k -> (k, 1)) l
+	-- Some tmp files may be dups copies of content that is fully present.
+	-- Simply delete those, while including the keys for the rest of
+	-- the temp files in the returned list for the user to deal with.
+	tmps <- tmpKeys
+	let staletmp = tmps `exclude` present
+	let duptmp = tmps `exclude` staletmp
+	_ <- liftIO $ mapM (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
+
+	return (unused, staletmp)
+
+	where
+		-- Constructing a single map, of the set that tends to be
+		-- smaller, appears more efficient in both memory and CPU
+		-- than constructing and taking the M.difference of two maps.
+		exclude [] _ = [] -- optimisation
+		exclude smaller larger = M.keys $ remove larger $ existsMap smaller
+
+		existsMap :: Ord k => [k] -> M.Map k Int
+		existsMap l = M.fromList $ map (\k -> (k, 1)) l
+
+		remove a b = foldl (flip M.delete) b a
 
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]
@@ -92,3 +117,17 @@ getKeysReferenced = do
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
+
+{- List of keys that have temp files in the git repo. -}
+tmpKeys :: Annex [Key]
+tmpKeys = do
+	g <- Annex.gitRepo
+	let tmp = gitAnnexTmpDir g
+	exists <- liftIO $ doesDirectoryExist tmp
+	if (not exists)
+		then return []
+		else do
+			contents <- liftIO $ getDirectoryContents tmp
+			files <- liftIO $ filterM doesFileExist $
+				map (tmp ) contents
+			return $ map (fileKey . takeFileName) files
diff --git a/Content.hs b/Content.hs
index d0ed8d861c..e16ad883c2 100644
--- a/Content.hs
+++ b/Content.hs
@@ -67,7 +67,7 @@ logStatus key status = do
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
 	g <- Annex.gitRepo
-	let tmp = gitAnnexTmpDir g  keyFile key
+	let tmp = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	success <- action tmp
 	if success
diff --git a/Locations.hs b/Locations.hs
index 75843c29eb..d30ceb1367 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -15,6 +15,7 @@ module Locations (
 	gitAnnexDir,
 	gitAnnexObjectDir,
 	gitAnnexTmpDir,
+	gitAnnexTmpLocation,
 	gitAnnexBadDir,
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
@@ -83,6 +84,10 @@ gitAnnexObjectDir r = addTrailingPathSeparator $ Git.workTree r  objectDir
 gitAnnexTmpDir :: Git.Repo -> FilePath
 gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
+{- The temp file to use for a given key. -}
+gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath
+gitAnnexTmpLocation r key = gitAnnexTmpDir r  keyFile key
+
 {- .git-annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath
 gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
diff --git a/debian/changelog b/debian/changelog
index 92b21173c8..cf17e8a00f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,10 @@ git-annex (0.19) UNRELEASED; urgency=low
   * fsck, drop: Take untrusted repositories into account.
   * Bugfix: Files were copied from trusted remotes first even if their
     annex.cost was higher than other remotes.
+  * Improved temp file handling. Transfers of content can now be resumed
+    from temp files later; the resume does not have to be the immediate
+    next git-annex run.
+  * unused: Include partially transferred content in the list.
 
  -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
 
diff --git a/doc/bugs/tmp_file_handling.mdwn b/doc/bugs/tmp_file_handling.mdwn
index 3c6c2a5976..9db932e571 100644
--- a/doc/bugs/tmp_file_handling.mdwn
+++ b/doc/bugs/tmp_file_handling.mdwn
@@ -9,3 +9,5 @@ This presents 2 problems:
    finished.
 
 --[[Joey]] 
+
+[[done]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d2cde35a0d..83a286b0e1 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,7 +132,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * unused
 
   Checks the annex for data that is not used by any files currently
-  in the annex, and prints a numbered list of the data. 
+  in the annex, and prints a numbered list of the data.
 
 * dropunused [number ...]
 

From 3c491130239dba5a4c1d882a6e91f1e138938f94 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 14:42:06 -0400
Subject: [PATCH 0787/8313] releasing version 0.19

---
 debian/changelog | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index cf17e8a00f..7abdee85cb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
-git-annex (0.19) UNRELEASED; urgency=low
+git-annex (0.19) unstable; urgency=low
 
-  * Support using the uuidgen command if the uuid command is not available.
+  * configure: Support using the uuidgen command if the uuid command is
+    not available.
   * Allow --exclude to be specified more than once.
   * There are now three levels of repository trust.
   * untrust: Now marks the current repository as untrusted.
@@ -12,8 +13,10 @@ git-annex (0.19) UNRELEASED; urgency=low
     from temp files later; the resume does not have to be the immediate
     next git-annex run.
   * unused: Include partially transferred content in the list.
+  * Bugfix: Running a second git-annex while a first has a transfer in
+    progress no longer deletes the first processes's temp file.
 
- -- Joey Hess   Wed, 19 Jan 2011 18:07:51 -0400
+ -- Joey Hess   Fri, 28 Jan 2011 14:31:37 -0400
 
 git-annex (0.18) unstable; urgency=low
 

From f88fd8d9fcc70365bc562f62b569b89109bb7e9a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Jan 2011 14:42:24 -0400
Subject: [PATCH 0788/8313] add news item for git-annex 0.19

---
 doc/news/version_0.14.mdwn |  5 -----
 doc/news/version_0.19.mdwn | 17 +++++++++++++++++
 2 files changed, 17 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_0.14.mdwn
 create mode 100644 doc/news/version_0.19.mdwn

diff --git a/doc/news/version_0.14.mdwn b/doc/news/version_0.14.mdwn
deleted file mode 100644
index fc6c42bc19..0000000000
--- a/doc/news/version_0.14.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 0.14 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Bugfix to git annex unused in a repository with nothing yet annexed.
-   * Support upgrading from a v0 annex with nothing in it.
-   * Avoid multiple calls to git ls-files when passed eg, "*"."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.19.mdwn b/doc/news/version_0.19.mdwn
new file mode 100644
index 0000000000..5d6ab47be0
--- /dev/null
+++ b/doc/news/version_0.19.mdwn
@@ -0,0 +1,17 @@
+git-annex 0.19 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * configure: Support using the uuidgen command if the uuid command is
+     not available.
+   * Allow --exclude to be specified more than once.
+   * There are now three levels of repository trust.
+   * untrust: Now marks the current repository as untrusted.
+   * semitrust: Now restores the default trust level. (What untrust used to do.)
+   * fsck, drop: Take untrusted repositories into account.
+   * Bugfix: Files were copied from trusted remotes first even if their
+     annex.cost was higher than other remotes.
+   * Improved temp file handling. Transfers of content can now be resumed
+     from temp files later; the resume does not have to be the immediate
+     next git-annex run.
+   * unused: Include partially transferred content in the list.
+   * Bugfix: Running a second git-annex while a first has a transfer in
+     progress no longer deletes the first processes's temp file."""]]
\ No newline at end of file

From b8318384858c59d73f231d225323a8a0a4a42868 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:27:17 -0400
Subject: [PATCH 0789/8313] cleanup

---
 CmdLine.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 82a21d0fc7..308b04d1a9 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -15,7 +15,6 @@ import System.IO.Error (try)
 import System.Console.GetOpt
 import Control.Monad.State (liftIO)
 import Control.Monad (when, unless)
-import System.Directory
 
 import qualified Annex
 import qualified GitRepo as Git
@@ -27,7 +26,6 @@ import Upgrade
 import Options
 import Messages
 import UUID
-import Locations
 
 {- Runs the passed command line. -}
 dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO ()

From 5584ccc8adbbcd629b3809fd39b5c00479fd0a25 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:30:08 -0400
Subject: [PATCH 0790/8313] factor out pure code

---
 Command/Unused.hs | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 4ffa6a17f8..32d70aa19d 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -51,12 +51,9 @@ checkUnused = do
 	g <- Annex.gitRepo
 	liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ 
 		map (\(n, k) -> show n ++ " " ++ show k) list
-	unless (null unused) $
-		showLongNote $ unusedmsg unusedlist
-	unless (null staletmp) $
-		showLongNote $ staletmpmsg staletmplist
-	unless (null list) $
-		showLongNote $ "\n"
+	unless (null unused) $ showLongNote $ unusedmsg unusedlist
+	unless (null staletmp) $ showLongNote $ staletmpmsg staletmplist
+	unless (null list) $ showLongNote $ "\n"
 	return $ null list
 
 	where
@@ -85,20 +82,23 @@ unusedKeys = do
 	g <- Annex.gitRepo
 	present <- getKeysPresent
 	referenced <- getKeysReferenced
-	
-	let unused = present `exclude` referenced
-
-	-- Some tmp files may be dups copies of content that is fully present.
-	-- Simply delete those, while including the keys for the rest of
-	-- the temp files in the returned list for the user to deal with.
 	tmps <- tmpKeys
-	let staletmp = tmps `exclude` present
-	let duptmp = tmps `exclude` staletmp
+	
+	let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps
+
+	-- Tmp files that are dups of content already present can simply
+	-- be removed.
 	_ <- liftIO $ mapM (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
 
 	return (unused, staletmp)
 
+calcUnusedKeys :: [Key] -> [Key] -> [Key] -> ([Key], [Key], [Key])
+calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp)
 	where
+		unused = present `exclude` referenced
+		staletmp = tmps `exclude` present
+		duptmp = tmps `exclude` staletmp
+
 		-- Constructing a single map, of the set that tends to be
 		-- smaller, appears more efficient in both memory and CPU
 		-- than constructing and taking the M.difference of two maps.

From c64b50a0ce725df2d5317b7ae4918b61aafa25ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:32:32 -0400
Subject: [PATCH 0791/8313] shutdown no longer a special case

---
 CmdLine.hs         | 12 ++++++------
 Command/RecvKey.hs |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 308b04d1a9..030637a738 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -32,7 +32,7 @@ dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO ()
 dispatch gitrepo args cmds options header = do
 	state <- Annex.new gitrepo allBackends
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
-	tryRun state' $ [startup, upgrade] ++ actions
+	tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown]
 
 {- Parses command line, stores configure flags, and returns a 
  - list of actions to be run in the Annex monad. -}
@@ -73,7 +73,6 @@ usage header cmds options =
 
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
- - Runs shutdown and propigates an overall error status at the end.
  -}
 tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun state actions = tryRun' state 0 actions
@@ -86,8 +85,7 @@ tryRun' state errnum (a:as) = do
 			tryRun' state (errnum + 1) as
 		Right (True,state') -> tryRun' state' errnum as
 		Right (False,state') -> tryRun' state' (errnum + 1) as
-tryRun' state errnum [] = do
-	_ <- try $ Annex.run state $ shutdown errnum
+tryRun' _ errnum [] = do
 	when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
@@ -97,9 +95,11 @@ startup = do
 	return True
 
 {- Cleanup actions. -}
-shutdown :: Integer -> Annex ()
-shutdown errnum = do
+shutdown :: Annex Bool
+shutdown = do
 	q <- Annex.getState Annex.repoqueue
 	unless (q == GitQueue.empty) $ do
 		showSideAction "Recording state in git..."
 		Annex.queueRun
+
+	return True
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 0abea07f20..8a96730503 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -38,6 +38,6 @@ start keyname = do
 		then do
 			-- forcibly quit after receiving one key,
 			-- and shutdown cleanly so queued git commands run
-			_ <- shutdown 0
+			_ <- shutdown
 			liftIO exitSuccess
 		else liftIO exitFailure

From 6d3df8a0833fdef290b76681e8ba85ec94b76e36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Jan 2011 23:47:10 -0400
Subject: [PATCH 0792/8313] more pure code refactoring

---
 CmdLine.hs     |  1 -
 Command.hs     | 13 ++++++++-----
 LocationLog.hs | 10 +++++++---
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 030637a738..d65739791f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -101,5 +101,4 @@ shutdown = do
 	unless (q == GitQueue.empty) $ do
 		showSideAction "Recording state in git..."
 		Annex.queueRun
-
 	return True
diff --git a/Command.hs b/Command.hs
index 298234fa05..e54add707b 100644
--- a/Command.hs
+++ b/Command.hs
@@ -184,17 +184,20 @@ filterFiles l = do
 	if null exclude
 		then return l'
 		else do
-			let regexp = compile (toregex exclude) []
+			let regexp = compile (wildsRegex exclude) []
 			return $ filter (notExcluded regexp) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
-		toregex exclude = "^(" ++ toregex' exclude "" ++ ")"
-		toregex' [] c = c
-		toregex' (w:ws) "" = toregex' ws (wildToRegex w)
-		toregex' (w:ws) c = toregex' ws (c ++ "|" ++ wildToRegex w)
+
+wildsRegex :: [String] -> String
+wildsRegex ws = "^(" ++ wildsRegex' ws "" ++ ")"
+wildsRegex' :: [String] -> String -> String
+wildsRegex' [] c = c
+wildsRegex' (w:ws) "" = wildsRegex' ws (wildToRegex w)
+wildsRegex' (w:ws) c = wildsRegex' ws (c ++ "|" ++ wildToRegex w)
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
diff --git a/LocationLog.hs b/LocationLog.hs
index 56953bc029..f778df3864 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -104,11 +104,15 @@ readLog file = do
 	if exists
 		then do
 			s <- readFile file
-			-- filter out any unparsable lines
-			return $ filter (\l -> status l /= Undefined )
-				$ map read $ lines s
+			return $ parseLog s
 		else return []
 
+parseLog :: String -> [LogLine]
+parseLog s = filter parsable $ map read $ lines s
+	where
+		-- some lines may be unparseable, avoid them
+		parsable l = status l /= Undefined
+
 {- Writes a set of lines to a log file -}
 writeLog :: FilePath -> [LogLine] -> IO ()
 writeLog file ls = safeWriteFile file (unlines $ map show ls)

From 53677d764710d522394f59e494fb6b632aca2326 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Jan 2011 00:08:17 -0400
Subject: [PATCH 0793/8313] tweak

---
 Command.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index e54add707b..1c1ff6bbcf 100644
--- a/Command.hs
+++ b/Command.hs
@@ -183,17 +183,16 @@ filterFiles l = do
 	exclude <- Annex.getState Annex.exclude
 	if null exclude
 		then return l'
-		else do
-			let regexp = compile (wildsRegex exclude) []
-			return $ filter (notExcluded regexp) l'
+		else return $ filter (notExcluded $ wildsRegex exclude) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = case match r f [] of
 			Nothing -> True
 			Just _ -> False
 
-wildsRegex :: [String] -> String
-wildsRegex ws = "^(" ++ wildsRegex' ws "" ++ ")"
+wildsRegex :: [String] -> Regex
+wildsRegex ws = compile regex []
+	where regex = "^(" ++ wildsRegex' ws "" ++ ")"
 wildsRegex' :: [String] -> String -> String
 wildsRegex' [] c = c
 wildsRegex' (w:ws) "" = wildsRegex' ws (wildToRegex w)

From 96e561bc477bd0a4bbffb769369fabe1e1e1982f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Jan 2011 01:41:15 -0400
Subject: [PATCH 0794/8313] use Set instead of existence Map

more efficient and idiomatic

I did try using Set.difference, it's still slower than my method.
---
 Command/Unused.hs | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 32d70aa19d..28a26f82c6 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -9,7 +9,7 @@ module Command.Unused where
 
 import Control.Monad (filterM, unless)
 import Control.Monad.State (liftIO)
-import qualified Data.Map as M
+import qualified Data.Set as S
 import Data.Maybe
 import System.FilePath
 import System.Directory
@@ -99,16 +99,12 @@ calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp)
 		staletmp = tmps `exclude` present
 		duptmp = tmps `exclude` staletmp
 
-		-- Constructing a single map, of the set that tends to be
+		-- Constructing a single set, of the list that tends to be
 		-- smaller, appears more efficient in both memory and CPU
-		-- than constructing and taking the M.difference of two maps.
+		-- than constructing and taking the S.difference of two sets.
 		exclude [] _ = [] -- optimisation
-		exclude smaller larger = M.keys $ remove larger $ existsMap smaller
-
-		existsMap :: Ord k => [k] -> M.Map k Int
-		existsMap l = M.fromList $ map (\k -> (k, 1)) l
-
-		remove a b = foldl (flip M.delete) b a
+		exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
+		remove a b = foldl (flip S.delete) b a
 
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]

From 1b0edc1ab2f3516dc532b0cf4ea39a0af2f6392f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Jan 2011 12:01:56 -0400
Subject: [PATCH 0795/8313] idiomatic elem

---
 Backend/File.hs    | 6 +++---
 BackendTypes.hs    | 2 +-
 UUID.hs            | 4 ++--
 git-annex-shell.hs | 4 ++--
 test.hs            | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 68dd4db271..d76cd29391 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -108,7 +108,7 @@ checkRemoveKey key numcopiesM = do
 			| length have >= need = return True
 			| otherwise = do
 				u <- getUUID r
-				let dup = elem u have
+				let dup = u `elem` have
 				haskey <- Remotes.inAnnex r key
 				case (dup, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
@@ -139,7 +139,7 @@ showLocations key exclude = do
 	ppuuidsskipped <- prettyPrintUUIDs uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
-		filteruuids list x = filter (\l -> not $ elem l x) list
+		filteruuids list x = filter (`notElem` x) list
 		message [] [] = "No other repository is known to contain the file."
 		message rs [] = "Try making some of these repositories available:\n" ++ rs
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
@@ -179,7 +179,7 @@ checkKeyNumCopies key file numcopies = do
 	locations <- liftIO $ keyLocations g key
 	untrusted <- trustGet UnTrusted
 	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (\l -> not $ l `elem` untrusted) locations
+	let safelocations = filter (`notElem` untrusted) locations
 	let present = length safelocations
 	if present < needed
 		then do
diff --git a/BackendTypes.hs b/BackendTypes.hs
index fd4a61b989..c4927ab20c 100644
--- a/BackendTypes.hs
+++ b/BackendTypes.hs
@@ -75,5 +75,5 @@ instance Arbitrary Key where
 prop_idempotent_key_read_show :: Key -> Bool
 prop_idempotent_key_read_show k
 	-- backend names will never contain colons
-	| elem ':' (backendName k) = True
+	| ':' `elem` (backendName k) = True
 	| otherwise = k == (read $ show k)
diff --git a/UUID.hs b/UUID.hs
index ec67026894..a654424b4f 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -88,7 +88,7 @@ reposByUUID repos uuids = filterM match repos
 	where
 		match r = do
 			u <- getUUID r
-			return $ elem u uuids
+			return $ u `elem` uuids
 
 {- Filters a list of repos to ones that do not have the listed UUIDs. -}
 reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
@@ -96,7 +96,7 @@ reposWithoutUUID repos uuids = filterM unmatch repos
 	where
 		unmatch r = do
 			u <- getUUID r
-			return $ not $ elem u uuids
+			return $ u `notElem` uuids
 
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index fa2a7f606a..fee4091ef8 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -46,13 +46,13 @@ main' [] = failure
 main' ("-c":p) = main' p
 -- a command can be either a builtin or something to pass to git-shell
 main' c@(cmd:dir:params)
-	| elem cmd builtins = builtin cmd dir params
+	| cmd `elem` builtins = builtin cmd dir params
 	| otherwise = external c
 main' c@(cmd:_)
 	-- Handle the case of being the user's login shell. It will be passed
 	-- a single string containing all the real parameters.
 	| "git-annex-shell " `isPrefixOf` cmd = main' $ drop 1 $ shellUnEscape cmd
-	| elem cmd builtins = failure
+	| cmd `elem` builtins = failure
 	| otherwise = external c
 
 builtins :: [String]
diff --git a/test.hs b/test.hs
index 5dda3b835a..f7d3f7c53d 100644
--- a/test.hs
+++ b/test.hs
@@ -331,7 +331,7 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 				l <- Trust.trustGet expected
 				r <- Remotes.byName repo
 				u <- UUID.getUUID r
-				return $ elem u l
+				return $ u `elem` l
 			assertBool msg present
 		repo = "origin"
 
@@ -588,7 +588,7 @@ checklocationlog f expected = do
 				g <- Annex.gitRepo
 				liftIO $ LocationLog.keyLocations g k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
-				expected (elem thisuuid uuids)
+				expected (thisuuid `elem` uuids)
 
 			-- Location log files should always be checked
 			-- into git, and any modifications staged for

From 4a0fe24f394e9c5489bacaa5630b3cabefdf95e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 11:42:22 -0400
Subject: [PATCH 0796/8313] use a newtype for Key

should be more efficient
---
 BackendTypes.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/BackendTypes.hs b/BackendTypes.hs
index c4927ab20c..c0705a550f 100644
--- a/BackendTypes.hs
+++ b/BackendTypes.hs
@@ -14,7 +14,7 @@ import Test.QuickCheck
 
 type KeyName = String
 type BackendName = String
-data Key = Key (BackendName, KeyName) deriving (Eq, Ord)
+newtype Key = Key (BackendName, KeyName) deriving (Eq, Ord)
 
 data Backend a = Backend {
 	-- name of this backend

From d007e58a5437daa72062453fc6f44c5beb7d271d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 13:52:11 -0400
Subject: [PATCH 0797/8313] use mapM_

---
 Command/Unused.hs | 2 +-
 GitQueue.hs       | 2 +-
 Remotes.hs        | 2 +-
 Upgrade.hs        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 28a26f82c6..fecfec7422 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -88,7 +88,7 @@ unusedKeys = do
 
 	-- Tmp files that are dups of content already present can simply
 	-- be removed.
-	_ <- liftIO $ mapM (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
+	liftIO $ mapM_ (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
 
 	return (unused, staletmp)
 
diff --git a/GitQueue.hs b/GitQueue.hs
index d8ba861366..0cb64caefb 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue
 {- Runs a queue on a git repository. -}
 run :: Git.Repo -> Queue -> IO ()
 run repo queue = do
-	_ <- mapM (uncurry $ runAction repo) $ M.toList queue
+	mapM_ (uncurry $ runAction repo) $ M.toList queue
 	return ()
 
 {- Runs an Action on a list of files in a git repository.
diff --git a/Remotes.hs b/Remotes.hs
index 616db225ef..9f1e2ee507 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -104,7 +104,7 @@ readConfigs = do
 				list doexpensive ++ "..."
 		let todo = cheap ++ doexpensive
 		unless (null todo) $ do
-			_ <- mapM tryGitConfigRead todo
+			mapM_ tryGitConfigRead todo
 			Annex.changeState $ \s -> s { Annex.remotesread = True }
 	where
 		cachedUUID r = do
diff --git a/Upgrade.hs b/Upgrade.hs
index 1e70e68d56..9c5a57a0c2 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -41,7 +41,7 @@ upgradeFrom0 = do
 	-- do the reorganisation of the files
 	let olddir = gitAnnexDir g
 	keys <- getKeysPresent0' olddir
-	_ <- mapM (\k -> moveAnnex k $ olddir  keyFile k) keys
+	mapM_ (\k -> moveAnnex k $ olddir  keyFile k) keys
 
 	-- update the symlinks to the files
 	files <- liftIO $ Git.inRepo g [Git.workTree g]

From 9fe5865a07ac5a66c7ecaad4a5682b205939735d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 16:06:27 -0400
Subject: [PATCH 0798/8313] annoyance

---
 doc/bugs/ordering.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/ordering.mdwn

diff --git a/doc/bugs/ordering.mdwn b/doc/bugs/ordering.mdwn
new file mode 100644
index 0000000000..902cf7676c
--- /dev/null
+++ b/doc/bugs/ordering.mdwn
@@ -0,0 +1,10 @@
+One would expect "git annex get foo bar" to first retrieve foo, and then
+bar. Actually though, it will operate on them in alphabetical order
+(probably). This is annoying when you wanted to 1st list the most important
+files to get. Maybe you'll run out of time before all can be gotten. The
+workaround of course is to run "git annex get" twice.
+
+This ordering comes from "git ls-files". git-annex passes it all the files
+the user specified. This is a useful optimisation -- earlier it would
+run "git ls-files" once per parameter, and so "git annex get *" could be
+rather slow. But, it produces this ordering problem.

From 37c62eebb72fa0c216336435669cf8a05c2dbc88 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 20:06:34 -0400
Subject: [PATCH 0799/8313] Preserve specified file ordering when instructed to
 act on multiple files or directories.

---
 Command.hs             | 47 ++++++++++++++++++++++++++++++++++++------
 debian/changelog       |  7 +++++++
 doc/bugs/ordering.mdwn |  2 ++
 3 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/Command.hs b/Command.hs
index 1c1ff6bbcf..859f713a0c 100644
--- a/Command.hs
+++ b/Command.hs
@@ -14,6 +14,7 @@ import Control.Monad (filterM)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
+import System.Path
 
 import Types
 import qualified Backend
@@ -108,20 +109,20 @@ isAnnexed file a = do
 withFilesInGit :: CommandSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo repo params
+	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	return $ map a files'
 withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo repo params
+	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	pairs <- liftIO $ Git.checkAttr repo attr files'
 	return $ map a pairs
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo repo params
+	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
 withFilesMissing :: CommandSeekStrings
@@ -136,7 +137,7 @@ withFilesMissing a params = do
 withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
-	newfiles <- liftIO $ Git.notInRepo repo params
+	newfiles <- liftIO $ runPreserverOrder (Git.notInRepo repo) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withString :: CommandSeekStrings
@@ -146,7 +147,7 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ Git.stagedFiles repo params
+	tocommit <- liftIO $ runPreserverOrder (Git.stagedFiles repo) params
 	tocommit' <- filterFiles tocommit
 	return $ map a tocommit'
 withFilesUnlocked :: CommandSeekBackendFiles
@@ -157,7 +158,7 @@ withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBa
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	repo <- Annex.gitRepo
-	typechangedfiles <- liftIO $ typechanged repo params
+	typechangedfiles <- liftIO $ runPreserverOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
 	unlockedfiles' <- filterFiles unlockedfiles
@@ -238,3 +239,37 @@ cmdlineKey  = do
 		keyname' (Just n) = n
 		badkey = error "please specify the key with --key"
 
+{- Given an original list of files, and an expanded list derived from it,
+ - ensures that the original list's ordering is preserved. 
+ -
+ - The input list may contain a directory, like "dir" or "dir/". Any
+ - items in the expanded list that are contained in that directory will
+ - appear at the same position as it did in the input list.
+ -}
+preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
+-- optimisation, only one item in original list, so no reordering needed
+preserveOrder [_] new = new
+preserveOrder orig new = collect orig new
+	where
+		collect [] n = n
+		collect [_] n = n -- optimisation
+		collect (l:ls) n = found ++ collect ls rest
+			where (found, rest)=partition (l `dirContains`) n
+
+runPreserverOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
+runPreserverOrder a files = do
+	r <- a files
+	return $ preserveOrder files r
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = case (absNormPath p ".") of
+			Just r -> r
+			Nothing -> ""
+		a' = norm a
+		b' = norm b
diff --git a/debian/changelog b/debian/changelog
index 7abdee85cb..eee71a5e94 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (0.20) UNRELEASED; urgency=low
+
+  * Preserve specified file ordering when instructed to act on multiple
+    files or directories.
+
+ -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
+
 git-annex (0.19) unstable; urgency=low
 
   * configure: Support using the uuidgen command if the uuid command is
diff --git a/doc/bugs/ordering.mdwn b/doc/bugs/ordering.mdwn
index 902cf7676c..536bfce36a 100644
--- a/doc/bugs/ordering.mdwn
+++ b/doc/bugs/ordering.mdwn
@@ -8,3 +8,5 @@ This ordering comes from "git ls-files". git-annex passes it all the files
 the user specified. This is a useful optimisation -- earlier it would
 run "git ls-files" once per parameter, and so "git annex get *" could be
 rather slow. But, it produces this ordering problem.
+
+[[done]]

From 27056daccd1a2f541cd104a835a32523a532d4da Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 20:14:08 -0400
Subject: [PATCH 0800/8313] cleanup last change

---
 Command.hs       | 38 ++++++++++++++++----------------------
 Utility.hs       | 15 +++++++++++++++
 debian/changelog |  3 ++-
 3 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/Command.hs b/Command.hs
index 859f713a0c..0bbc6088c5 100644
--- a/Command.hs
+++ b/Command.hs
@@ -14,7 +14,6 @@ import Control.Monad (filterM)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
-import System.Path
 
 import Types
 import qualified Backend
@@ -22,6 +21,7 @@ import Messages
 import qualified Annex
 import qualified GitRepo as Git
 import Locations
+import Utility
 
 {- A command runs in four stages.
  -
@@ -109,20 +109,20 @@ isAnnexed file a = do
 withFilesInGit :: CommandSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	return $ map a files'
 withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	pairs <- liftIO $ Git.checkAttr repo attr files'
 	return $ map a pairs
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserverOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
 withFilesMissing :: CommandSeekStrings
@@ -137,7 +137,7 @@ withFilesMissing a params = do
 withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
-	newfiles <- liftIO $ runPreserverOrder (Git.notInRepo repo) params
+	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withString :: CommandSeekStrings
@@ -147,7 +147,7 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserverOrder (Git.stagedFiles repo) params
+	tocommit <- liftIO $ runPreserveOrder (Git.stagedFiles repo) params
 	tocommit' <- filterFiles tocommit
 	return $ map a tocommit'
 withFilesUnlocked :: CommandSeekBackendFiles
@@ -158,7 +158,7 @@ withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBa
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	repo <- Annex.gitRepo
-	typechangedfiles <- liftIO $ runPreserverOrder (typechanged repo) params
+	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
 	unlockedfiles' <- filterFiles unlockedfiles
@@ -256,20 +256,14 @@ preserveOrder orig new = collect orig new
 		collect (l:ls) n = found ++ collect ls rest
 			where (found, rest)=partition (l `dirContains`) n
 
-runPreserverOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserverOrder a files = do
+{- Runs an action that takes a list of FilePaths, and ensures that 
+ - its return list preserves order.
+ -
+ - This assumes that it's cheaper to call preserveOrder on the result,
+ - than it would be to run the action once with each param. In the case
+ - of git file list commands, that assumption tends to hold.
+ -}
+runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
+runPreserveOrder a files = do
 	r <- a files
 	return $ preserveOrder files r
-
-{- Checks if the first FilePath is, or could be said to contain the second.
- - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
- - are all equivilant.
- -}
-dirContains :: FilePath -> FilePath -> Bool
-dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
-	where
-		norm p = case (absNormPath p ".") of
-			Just r -> r
-			Nothing -> ""
-		a' = norm a
-		b' = norm b
diff --git a/Utility.hs b/Utility.hs
index 96bbc89ee2..2bb623532d 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -18,6 +18,7 @@ module Utility (
 	unsetFileMode,
 	readMaybe,
 	safeWriteFile,
+	dirContains,
 	
 	prop_idempotent_shellEscape,
 	prop_idempotent_shellEscape_multiword,
@@ -36,6 +37,7 @@ import System.Path
 import System.FilePath
 import System.Directory
 import Foreign (complement)
+import Data.List
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -65,6 +67,19 @@ prop_parentDir_basics dir
 	where
 		p = parentDir dir
 
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = case (absNormPath p ".") of
+			Just r -> r
+			Nothing -> ""
+		a' = norm a
+		b' = norm b
+
 {- Converts a filename into a normalized, absolute path. -}
 absPath :: FilePath -> IO FilePath
 absPath file = do
diff --git a/debian/changelog b/debian/changelog
index eee71a5e94..d7edc1733f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,8 @@
 git-annex (0.20) UNRELEASED; urgency=low
 
   * Preserve specified file ordering when instructed to act on multiple
-    files or directories.
+    files or directories. For example, "git annex get a b" will now always
+    get "a" before "b". Previously it could operate in either order.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 

From 755029ae0e46e9d4ca2b7416d5a67b2be34eec0a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Jan 2011 23:27:53 -0400
Subject: [PATCH 0801/8313] use forM_ in a few places

---
 Command/Unused.hs | 4 ++--
 GitQueue.hs       | 4 ++--
 Upgrade.hs        | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index fecfec7422..d9f4e39783 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -7,7 +7,7 @@
 
 module Command.Unused where
 
-import Control.Monad (filterM, unless)
+import Control.Monad (filterM, unless, forM_)
 import Control.Monad.State (liftIO)
 import qualified Data.Set as S
 import Data.Maybe
@@ -88,7 +88,7 @@ unusedKeys = do
 
 	-- Tmp files that are dups of content already present can simply
 	-- be removed.
-	liftIO $ mapM_ (\t -> removeFile $ gitAnnexTmpLocation g t) duptmp
+	liftIO $ forM_ duptmp $ \t -> removeFile $ gitAnnexTmpLocation g t
 
 	return (unused, staletmp)
 
diff --git a/GitQueue.hs b/GitQueue.hs
index 0cb64caefb..4a777af4db 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -16,7 +16,7 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Control.Monad (unless)
+import Control.Monad (unless, forM_)
 
 import qualified GitRepo as Git
 
@@ -45,7 +45,7 @@ add queue subcommand params file = M.insertWith (++) action [file] queue
 {- Runs a queue on a git repository. -}
 run :: Git.Repo -> Queue -> IO ()
 run repo queue = do
-	mapM_ (uncurry $ runAction repo) $ M.toList queue
+	forM_ (M.toList queue) $ uncurry $ runAction repo
 	return ()
 
 {- Runs an Action on a list of files in a git repository.
diff --git a/Upgrade.hs b/Upgrade.hs
index 9c5a57a0c2..b584b2666f 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -10,7 +10,7 @@ module Upgrade where
 import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
-import Control.Monad (filterM)
+import Control.Monad (filterM, forM_)
 import System.Posix.Files
 import System.FilePath
 
@@ -41,7 +41,7 @@ upgradeFrom0 = do
 	-- do the reorganisation of the files
 	let olddir = gitAnnexDir g
 	keys <- getKeysPresent0' olddir
-	mapM_ (\k -> moveAnnex k $ olddir  keyFile k) keys
+	forM_ keys $ \k -> moveAnnex k $ olddir  keyFile k
 
 	-- update the symlinks to the files
 	files <- liftIO $ Git.inRepo g [Git.workTree g]

From 3c13f62f11d1aca82ea48cfa9cf66c4bd88a7b7b Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Tue, 1 Feb 2011 23:39:12 +0000
Subject: [PATCH 0802/8313] getting git-annex files back to git

---
 doc/forum/unannex_alternatives.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives.mdwn

diff --git a/doc/forum/unannex_alternatives.mdwn b/doc/forum/unannex_alternatives.mdwn
new file mode 100644
index 0000000000..efd05838e6
--- /dev/null
+++ b/doc/forum/unannex_alternatives.mdwn
@@ -0,0 +1,9 @@
+what is the work flow to get a file that is in git-annex out of there and into git? (current situation: `git-annex add`ed a bunch of pictures, later found make files in there which i'd rather have in git for proper source code control)
+
+the most intuitive thing to do is `git unannex`, which at first seemed to do the right thing, but when committing there came the hook and everything was back to where it was before.
+
+i could disable the hook as a workaround, but that doesn't smell like a good work flow.
+
+the [[man page|git-annex]] does warn that `unannex` is only supposed to be used against unintentional `git annex add`s (probably meaning that it should be used before something is committed), but the alternatives it suggests (`git rm` and `git annex drop`) don't to what i want to do.
+
+am i missing something or is there really no work flow for this? --[[chrysn]]

From 97d1cba498c12c0caac919d9763884646de04aec Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:39:10 +0000
Subject: [PATCH 0803/8313] Added a comment

---
 ..._dcd4cd41280b41512bbdffafaf307993._comment | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment

diff --git a/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment b/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment
new file mode 100644
index 0000000000..7f278d2bc9
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment
@@ -0,0 +1,46 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-02-02T00:39:10Z"
+ content="""
+Git-annex's commit hook does not prevent unannex being used. The file you unannex will not be checked into git anymore and will be a regular file again, not a git-annex symlink. 
+
+For example, here's a transcript:
+
+
+joey@gnu:~/tmp>mkdir demo
+joey@gnu:~/tmp>cd demo
+joey@gnu:~/tmp/demo>git init
+Initialized empty Git repository in /home/joey/tmp/demo/.git/
+joey@gnu:~/tmp/demo>git annex init demo
+init demo ok
+joey@gnu:~/tmp/demo>echo hi > file
+joey@gnu:~/tmp/demo>git annex add file 
+add file ok
+(Recording state in git...)
+joey@gnu:~/tmp/demo>git commit -m add
+[master 64cf267] add
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/WORM:1296607093:3:file.log
+ create mode 120000 file
+joey@gnu:~/tmp/demo>git annex unannex file
+unannex file ok
+(Recording state in git...)
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git commit
+[master 78a09cc] unannex
+ 2 files changed, 1 insertions(+), 2 deletions(-)
+ delete mode 120000 file
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git status
+# On branch master
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	file
+nothing added to commit but untracked files present (use \"git add\" to track)
+

+"""]]

From b9f40099866201a23b998fcc9f15fce859907228 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:41:24 +0000
Subject: [PATCH 0804/8313] Added a comment

---
 ..._58a72a9fe0f58c7af0b4d7927a2dd21d._comment | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment

diff --git a/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment b/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment
new file mode 100644
index 0000000000..91ddadf8c6
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-02-02T00:41:24Z"
+ content="""
+And following on to my transcript, you can then add the file to git in the regular git way, and it works fine:
+
+
+joey@gnu:~/tmp/demo>git add file
+joey@gnu:~/tmp/demo>git commit
+[master 225ffc0] added as regular git file, not in annex
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 100644 file
+joey@gnu:~/tmp/demo>ls -l file
+-rw-r--r-- 1 joey joey 3 Feb  1 20:38 file
+joey@gnu:~/tmp/demo>git log file
+commit 225ffc048f5af7c0466b3b1fe549a6d5e9a9e9fe
+Author: Joey Hess 
+Date:   Tue Feb 1 20:43:13 2011 -0400
+
+    added as regular git file, not in annex
+
+commit 78a09cc791b875c3b859ca9401e5b6472bf19d08
+Author: Joey Hess 
+Date:   Tue Feb 1 20:38:30 2011 -0400
+
+    unannex
+
+commit 64cf267734adae05c020d9fd4d5a7ff7c64390db
+Author: Joey Hess 
+Date:   Tue Feb 1 20:38:18 2011 -0400
+
+    add
+

+"""]]

From 513da0f1ef4a1e05f4b4f9461b2eb1fad31fcb43 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:46:00 +0000
Subject: [PATCH 0805/8313] Added a comment

---
 ...mment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment

diff --git a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
new file mode 100644
index 0000000000..548f7aacbb
--- /dev/null
+++ b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-02-02T00:46:00Z"
+ content="""
+Sorry for all the followups, but I see now that if you unannex, then add the file to git normally, and commit, the hook *does* misbehave.
+
+This seems to be a bug. git-annex's hook thinks that you have used git annex unlock (or \"git annex edit\") on the file and are now committing a changed version, and the right thing to do there is to add the new content to the annex and update the symlink accordingly. I'll track this bug over at [[bugs/unannex_vs_unlock_hook_confusion]].
+
+So, committing after unannex, and before checking the file into git in the usual way, is a workaround.
+"""]]

From 3ceef4410c49100b603bc41e2c8f9487ea093274 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 2 Feb 2011 00:47:43 +0000
Subject: [PATCH 0806/8313]

---
 doc/bugs/unannex_vs_unlock_hook_confusion._comment | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/unannex_vs_unlock_hook_confusion._comment

diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion._comment b/doc/bugs/unannex_vs_unlock_hook_confusion._comment
new file mode 100644
index 0000000000..8a63303775
--- /dev/null
+++ b/doc/bugs/unannex_vs_unlock_hook_confusion._comment
@@ -0,0 +1,9 @@
+See [[forum/unannex_alternatives]] for problem description.
+
+If an unannex is followed by a "git add; git commit", git-annex's hook thinks
+that you have used git annex unlock on the file and are
+now committing a changed version, and the right thing to do there is to add the
+new content to the annex and update the symlink accordingly.
+
+Can we tell the difference between an unannexed file that has yet to be committed
+and an unlocked file? --[[Joey||

From ed089c22554ca5b3cd8c8f532db4a9fcc65385d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:01:43 -0400
Subject: [PATCH 0807/8313] rename (caused by ikiwiki bug, now fixed)

---
 ...k_confusion._comment => unannex_vs_unlock_hook_confusion.mdwn} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/bugs/{unannex_vs_unlock_hook_confusion._comment => unannex_vs_unlock_hook_confusion.mdwn} (100%)

diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion._comment b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
similarity index 100%
rename from doc/bugs/unannex_vs_unlock_hook_confusion._comment
rename to doc/bugs/unannex_vs_unlock_hook_confusion.mdwn

From 9aecf4110afb2fb1f6b10a9a9234ad0afafda56e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:04:43 -0400
Subject: [PATCH 0808/8313] followup

---
 doc/bugs/unannex_vs_unlock_hook_confusion.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
index 8a63303775..7e4a9f2914 100644
--- a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
+++ b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
@@ -6,4 +6,8 @@ now committing a changed version, and the right thing to do there is to add the
 new content to the annex and update the symlink accordingly.
 
 Can we tell the difference between an unannexed file that has yet to be committed
-and an unlocked file? --[[Joey||
+and has been re-added as a normal file, vs an unlocked file? --[[Joey||
+
+> Hmm, not really. An unannexed file's content will have been dropped from
+> the backend, but that's about the only difference. Perhaps unannex should
+> just commit the removal of the file itself? --[[Joey]]

From c77ac11acc10efc23acfa3d81e1deaac11cb724f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:26:19 -0400
Subject: [PATCH 0809/8313] unannex: Commit staged changes at end

to avoid some confusing behavior with the pre-commit hook, which would see
some types of commits after an unannex as checking in of an unlocked file.
---
 Command/Unannex.hs                                    |  7 ++++++-
 debian/changelog                                      |  3 +++
 doc/bugs/unannex_vs_unlock_hook_confusion.mdwn        |  2 ++
 ...omment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment |  6 +++++-
 doc/git-annex.mdwn                                    | 11 +++++++----
 5 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 4134439697..3810cca1c2 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -55,5 +55,10 @@ cleanup file key = do
 
 	fromAnnex key file
 	logStatus key ValueMissing
-
+	
+	-- Commit staged changes at end to avoid confusing the
+	-- pre-commit hook if this file is later added back to
+	-- git as a normal, non-annexed file.
+	Annex.queue "commit" ["-m", "content removed from git annex"] "--"
+	
 	return True
diff --git a/debian/changelog b/debian/changelog
index d7edc1733f..6bfc946442 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (0.20) UNRELEASED; urgency=low
   * Preserve specified file ordering when instructed to act on multiple
     files or directories. For example, "git annex get a b" will now always
     get "a" before "b". Previously it could operate in either order.
+  * unannex: Commit staged changes at end, to avoid some confusing behavior
+    with the pre-commit hook, which would see some types of commits after
+    an unannex as checking in of an unlocked file.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 
diff --git a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
index 7e4a9f2914..c03990c203 100644
--- a/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
+++ b/doc/bugs/unannex_vs_unlock_hook_confusion.mdwn
@@ -11,3 +11,5 @@ and has been re-added as a normal file, vs an unlocked file? --[[Joey||
 > Hmm, not really. An unannexed file's content will have been dropped from
 > the backend, but that's about the only difference. Perhaps unannex should
 > just commit the removal of the file itself? --[[Joey]]
+
+> [[done]], staged changes committed at end.
diff --git a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
index 548f7aacbb..9f3223578b 100644
--- a/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
+++ b/doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment
@@ -8,5 +8,9 @@ Sorry for all the followups, but I see now that if you unannex, then add the fil
 
 This seems to be a bug. git-annex's hook thinks that you have used git annex unlock (or \"git annex edit\") on the file and are now committing a changed version, and the right thing to do there is to add the new content to the annex and update the symlink accordingly. I'll track this bug over at [[bugs/unannex_vs_unlock_hook_confusion]].
 
-So, committing after unannex, and before checking the file into git in the usual way, is a workaround.
+So, committing after unannex, and before checking the file into git in the
+usual way, is a workaround. But only if you do a "git commit" to commit
+staged changes.
+
+Anyway, this confusing point is fixed in git now!
 """]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 83a286b0e1..68a1672df0 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -156,10 +156,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * unannex [path ...]
 
-  Use this to undo an accidental add command. This is not the command you
-  should use if you intentionally annexed a file and don't want its contents
-  any more. In that case you should use `git annex drop` instead, and you
-  can also `git rm` the file.
+  Use this to undo an accidental `git annex add` command. You can use
+  `git annex unannex` to move content out of the annex at any point,
+  even if you've already committed it.
+
+  This is not the command you should use if you intentionally annexed a
+  file and don't want its contents any more. In that case you should use
+  `git annex drop` instead, and you can also `git rm` the file.
 
 * uninit
 

From 0e7984a79354135f265d2342608953104d15db2e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 Feb 2011 21:58:47 -0400
Subject: [PATCH 0810/8313] add check for unclean tree

---
 .git-annex/uuid.log |  1 -
 Command.hs          |  2 +-
 Command/Unannex.hs  |  6 ++++++
 GitRepo.hs          | 17 +++++++++++++----
 4 files changed, 20 insertions(+), 6 deletions(-)
 delete mode 100644 .git-annex/uuid.log

diff --git a/.git-annex/uuid.log b/.git-annex/uuid.log
deleted file mode 100644
index 8cd9452b64..0000000000
--- a/.git-annex/uuid.log
+++ /dev/null
@@ -1 +0,0 @@
-1ac368a4-19e2-11e0-8c0f-8fcd42cf5a8d test repo
diff --git a/Command.hs b/Command.hs
index 0bbc6088c5..601b584642 100644
--- a/Command.hs
+++ b/Command.hs
@@ -147,7 +147,7 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserveOrder (Git.stagedFiles repo) params
+	tocommit <- liftIO $ runPreserveOrder (Git.stagedFilesNotDeleted repo) params
 	tocommit' <- filterFiles tocommit
 	return $ map a tocommit'
 withFilesUnlocked :: CommandSeekBackendFiles
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 3810cca1c2..c663b29ab0 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -8,6 +8,7 @@
 module Command.Unannex where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (unless)
 import System.Directory
 
 import Command
@@ -32,6 +33,11 @@ start file = isAnnexed file $ \(key, backend) -> do
 	ishere <- inAnnex key
 	if ishere
 		then do
+			g <- Annex.gitRepo
+			staged <- liftIO $ Git.stagedFiles g [Git.workTree g]
+			unless (null staged) $
+				error "This command cannot be run when there are already files staged for commit."
+
 			showStart "unannex" file
 			return $ Just $ perform file key backend
 		else return Nothing
diff --git a/GitRepo.hs b/GitRepo.hs
index 4e69544d43..031a9cbe21 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -38,6 +38,7 @@ module GitRepo (
 	inRepo,
 	notInRepo,
 	stagedFiles,
+	stagedFilesNotDeleted,
 	changedUnstagedFiles,
 	checkAttr,
 	decodeGitFile,
@@ -243,12 +244,20 @@ notInRepo :: Repo -> [FilePath] -> IO [FilePath]
 notInRepo repo l = pipeNullSplit repo $
 	["ls-files", "--others", "--exclude-standard", "-z", "--"] ++ l
 
+{- Returns a list of all files that are staged for commit. -}
+stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
+stagedFiles repo l = stagedFiles' repo l []
+
 {- Returns a list of the files, staged for commit, that are being added,
  - moved, or changed (but not deleted), from the specified locations. -}
-stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-stagedFiles repo l = pipeNullSplit repo $
-	["diff", "--cached", "--name-only", "--diff-filter=ACMRT", "-z", 
-		"--"] ++ l
+stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
+stagedFilesNotDeleted repo l = stagedFiles' repo l ["--diff-filter=ACMRT"]
+
+stagedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath]
+stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+	where
+		start = ["diff", "--cached", "--name-only", "-z"]
+		end = ["--"] ++ l
 
 {- Returns a list of files that have unstaged changes. -}
 changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath]

From 8f0b86dab5db2882aefd2b3653de186b94fec4f4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Thu, 3 Feb 2011 14:07:52 +0000
Subject: [PATCH 0811/8313]

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 29 ++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx.mdwn

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
new file mode 100644
index 0000000000..6708981b90
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -0,0 +1,29 @@
+Followed the instructions over here: http://git-annex.branchable.com/forum/git-annex_on_OSX/
+
+and had to install the following extra packages to be able to get make to start:
+
+[realizes pcre-light is needed but pcre not installed on my mac]
+sudo port install pcre
+sudo cabal install pcre-light
+
+But then I got the following error:
+
+ghc -O2 -Wall --make git-annex
+[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o 
+
+BackendTypes.hs:71:17:
+    No instance for (Arbitrary Char)
+      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25
+    Possible fix: add an instance declaration for (Arbitrary Char)
+    In a stmt of a 'do' expression: backendname <- arbitrary
+    In the expression:
+        do backendname <- arbitrary
+           keyname <- arbitrary
+             return $ Key (backendname, keyname)
+    In the definition of `arbitrary':
+        arbitrary = do backendname <- arbitrary
+                       keyname <- arbitrary
+                         return $ Key (backendname, keyname)
+make: *** [git-annex] Error 1
+
+My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.

From 41656301ef734a9dd810ec77adb1fd1ff350fd39 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Thu, 3 Feb 2011 14:08:38 +0000
Subject: [PATCH 0812/8313]

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
index 6708981b90..d14c058e41 100644
--- a/doc/bugs/Problems_running_make_on_osx.mdwn
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -5,7 +5,6 @@ and had to install the following extra packages to be able to get make to start:
 [realizes pcre-light is needed but pcre not installed on my mac]
 sudo port install pcre
 sudo cabal install pcre-light
-
 But then I got the following error:
 
 ghc -O2 -Wall --make git-annex

From 3595abe1ca77c7942bcee420e9058b99b198708c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Thu, 3 Feb 2011 14:10:28 +0000
Subject: [PATCH 0813/8313]

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 40 +++++++++++-----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
index d14c058e41..96626cc376 100644
--- a/doc/bugs/Problems_running_make_on_osx.mdwn
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -2,27 +2,27 @@ Followed the instructions over here: http://git-annex.branchable.com/forum/git-a
 
 and had to install the following extra packages to be able to get make to start:
 
-[realizes pcre-light is needed but pcre not installed on my mac]
-sudo port install pcre
-sudo cabal install pcre-light
-But then I got the following error:
+[realizes pcre-light is needed but pcre not installed on my mac]  
+sudo port install pcre  
+sudo cabal install pcre-light  
+But then I got the following error:  
 
-ghc -O2 -Wall --make git-annex
-[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o 
+ghc -O2 -Wall --make git-annex  
+[ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o   
 
-BackendTypes.hs:71:17:
-    No instance for (Arbitrary Char)
-      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25
-    Possible fix: add an instance declaration for (Arbitrary Char)
-    In a stmt of a 'do' expression: backendname <- arbitrary
-    In the expression:
-        do backendname <- arbitrary
-           keyname <- arbitrary
-             return $ Key (backendname, keyname)
-    In the definition of `arbitrary':
-        arbitrary = do backendname <- arbitrary
-                       keyname <- arbitrary
-                         return $ Key (backendname, keyname)
-make: *** [git-annex] Error 1
+BackendTypes.hs:71:17:  
+    No instance for (Arbitrary Char)  
+      arising from a use of `arbitrary' at BackendTypes.hs:71:17-25  
+    Possible fix: add an instance declaration for (Arbitrary Char)  
+    In a stmt of a 'do' expression: backendname <- arbitrary  
+    In the expression:  
+        do backendname <- arbitrary  
+           keyname <- arbitrary  
+             return $ Key (backendname, keyname)  
+    In the definition of `arbitrary':  
+        arbitrary = do backendname <- arbitrary  
+                       keyname <- arbitrary  
+                         return $ Key (backendname, keyname)  
+make: *** [git-annex] Error 1  
 
 My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.

From d29187b1518e5821a8367b74315442f8557d5617 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 14:03:24 -0400
Subject: [PATCH 0814/8313] update instructions for pcre

---
 doc/forum/git-annex_on_OSX.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn
index 44c275ff1c..302ad10994 100644
--- a/doc/forum/git-annex_on_OSX.mdwn
+++ b/doc/forum/git-annex_on_OSX.mdwn
@@ -8,6 +8,8 @@ sudo port install haskell-platform git-core ossp-uuid md5sha1sum
 sudo cabal  update
 sudo cabal install missingh
 sudo cabal install utf8-string
+sudo port install pcre
+sudo cabal install pcre-light
 
 git clone  git://git.kitenet.net/git-annex
 

From b1caa49248a10a54b3c5c38acda11dd81ce60d11 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 14:11:28 -0400
Subject: [PATCH 0815/8313] workaround

---
 doc/bugs/Problems_running_make_on_osx.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn
index 96626cc376..bd0f2e64a8 100644
--- a/doc/bugs/Problems_running_make_on_osx.mdwn
+++ b/doc/bugs/Problems_running_make_on_osx.mdwn
@@ -5,8 +5,14 @@ and had to install the following extra packages to be able to get make to start:
 [realizes pcre-light is needed but pcre not installed on my mac]  
 sudo port install pcre  
 sudo cabal install pcre-light  
+
+> Ah right, that is a new dependency. I've updated the forum page
+> with this info.
+> --[[Joey]] 
+
 But then I got the following error:  
 
+
 ghc -O2 -Wall --make git-annex  
 [ 7 of 52] Compiling BackendTypes     ( BackendTypes.hs, BackendTypes.o   
 
@@ -24,5 +30,14 @@ BackendTypes.hs:71:17:
                        keyname <- arbitrary  
                          return $ Key (backendname, keyname)  
 make: *** [git-annex] Error 1  
+

 
 My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary so any help would be appreciated.
+
+> Hmm, it seems you may be missing part of the quickcheck haskell
+> library, or have a different version than me.
+> 
+> The easy fix is probably to just edit BackendTypes.hs and delete the
+> entire end of the file from line 68, "for quickcheck" down. This code
+> is only used by the test suite (so "make test" will fail), 
+> but it should get it to build. --[[Joey]]

From 14bc885de96dd3ec52ab33ec6bbb02974d0a381c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 18:47:14 -0400
Subject: [PATCH 0816/8313] more accessor functions and better bad url handling

---
 GitRepo.hs | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 031a9cbe21..b5a94d4263 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -16,11 +16,13 @@ module GitRepo (
 	repoIsUrl,
 	repoIsSsh,
 	repoDescribe,
+	repoLocation,
 	workTree,
 	gitDir,
 	relative,
 	urlPath,
 	urlHost,
+	urlScheme,
 	configGet,
 	configMap,
 	configRead,
@@ -101,7 +103,10 @@ repoFromUrl :: String -> Repo
 repoFromUrl url
 	| startswith "file://" url = repoFromPath $ uriPath u
 	| otherwise = newFrom $ Url u
-		where u = fromJust $ parseURI url
+		where
+			u = case (parseURI url) of
+				Just v -> v
+				Nothing -> error $ "bad url " ++ url
 
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
@@ -109,6 +114,11 @@ repoDescribe Repo { remoteName = Just name } = name
 repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 
+{- Location of the repo, either as a path or url. -}
+repoLocation :: Repo -> String
+repoLocation Repo { location = Url url } = show url
+repoLocation Repo { location = Dir dir } = dir
+
 {- Constructs and returns an updated version of a repo with
  - different remotes list. -}
 remotesAdd :: Repo -> [Repo] -> Repo
@@ -192,10 +202,16 @@ relative repo@(Repo { location = Dir d }) file = do
 			Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo
 relative repo _ = assertLocal repo $ error "internal"
 
+{- Scheme of an URL repo. -}
+urlScheme :: Repo -> String
+urlScheme Repo { location = Url u } = uriScheme u
+urlScheme repo = assertUrl repo $ error "internal"
+
 {- Hostname of an URL repo. (May include a username and/or port too.) -}
 urlHost :: Repo -> String
 urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a
-	where a = fromJust $ uriAuthority u
+	where
+		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
 urlHost repo = assertUrl repo $ error "internal"
 
 {- Path of an URL repo. -}

From 0c7d17ae062c136e549cc9800dae85f3e3793237 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 18:55:12 -0400
Subject: [PATCH 0817/8313] new map subcommand, basically working

Still todo:

- add repos from uuid.log that were not directly found
- group repos into their respective hosts
- display inaccessible repos and broken remote connections in red
- anonymize the url display somewhat, so the maps can be shared
- use uuid info to tell when two apparently different repos are actually
  the same repo accessed in different ways
---
 Command/Map.hs     | 153 +++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |   2 +
 Remotes.hs         |   7 ++-
 UUID.hs            |  10 ++-
 debian/changelog   |   2 +
 doc/git-annex.mdwn |  17 +++++
 6 files changed, 185 insertions(+), 6 deletions(-)
 create mode 100644 Command/Map.hs

diff --git a/Command/Map.hs b/Command/Map.hs
new file mode 100644
index 0000000000..753d6ebdcb
--- /dev/null
+++ b/Command/Map.hs
@@ -0,0 +1,153 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Map where
+
+import Control.Monad.State (liftIO)
+import Control.Exception.Extensible
+import System.Cmd.Utils
+
+import Command
+import qualified Annex
+import qualified GitRepo as Git
+import qualified Remotes
+import Messages
+import Types
+import Utility
+
+-- a link from the first repository to the second (its remote)
+data Link = Link Git.Repo Git.Repo
+
+command :: [Command]
+command = [Command "map" paramNothing seek "generate map of repositories"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStartNothing
+start = do
+	g <- Annex.gitRepo
+	rs <- spider g
+
+	liftIO $ writeFile file (dotGraph rs)
+	showLongNote $ "running: dot -Tx11 " ++ file ++ "\n"
+	r <- liftIO $ boolSystem "dot" ["-Tx11", file]
+	return $ Just $ return $ Just $ return r
+	where
+		file = "map.dot"
+
+{- Generates a graph for dot(1). Each repository is displayed
+ - as a node, and each of its remotes is represented as an edge
+ - pointing at the node for the remote. -}
+dotGraph :: [Git.Repo] -> String
+dotGraph rs = unlines $ [header] ++ map dotGraphRepo rs ++ [footer]
+	where
+		header = "digraph map {"
+		footer= "}"
+
+dotGraphRepo :: Git.Repo -> String
+dotGraphRepo r = unlines $ map dotline (node:edges)
+	where
+		node = nodename r ++
+			" [ label=" ++ dotquote (Git.repoDescribe r) ++ " ]"
+		edges = map edge (Git.remotes r)
+		edge e = nodename r ++ " -> " ++ nodename (makeabs r e)
+		nodename n = dotquote (Git.repoLocation n)
+		dotquote s = "\"" ++ s ++ "\""
+		dotline s = "\t" ++ s ++ ";"
+
+{- Recursively searches out remotes starting with the specified repo. -}
+spider :: Git.Repo -> Annex [Git.Repo]
+spider r = spider' [r] []
+spider' :: [Git.Repo] -> [Git.Repo] -> Annex [Git.Repo]
+spider' [] known = return known
+spider' (r:rs) known
+	| any (same r) known = spider' rs known
+	| otherwise = do
+		r' <- scan r
+		let remotes = map (makeabs r') (Git.remotes r')
+		spider' (rs ++ remotes) (r':known)
+
+{- Makes a remote have an absolute url, rather than a host-local path. -}
+makeabs :: Git.Repo -> Git.Repo -> Git.Repo
+makeabs repo remote
+	| Git.repoIsUrl remote = remote
+	| not $ Git.repoIsUrl repo = remote
+	| otherwise = Git.repoFromUrl combinedurl
+		where
+			combinedurl =
+				Git.urlScheme repo ++ "//" ++
+				Git.urlHost repo ++
+				Git.workTree remote
+
+{- Checks if two repos are the same. -}
+same :: Git.Repo -> Git.Repo -> Bool
+same a b
+	| both Git.repoIsSsh = matching Git.urlHost && matching Git.workTree
+	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
+	| otherwise = False
+		
+	where
+		matching t = t a == t b
+		both t = t a && t b
+		neither t = not (t a) && not (t b)
+
+{- reads the config of a remote, with progress display -}
+scan :: Git.Repo -> Annex Git.Repo
+scan r = do
+	showStart "map" (Git.repoDescribe r)
+	v <- tryScan r
+	case v of
+		Just r' -> do
+			showEndOk
+			return r'
+		Nothing -> do
+			showEndFail
+			return r
+
+{- tries to read the config of a remote, returning it only if it can
+ - be accessed -}
+tryScan :: Git.Repo -> Annex (Maybe Git.Repo)
+tryScan r
+	| Git.repoIsSsh r = sshscan
+	| Git.repoIsUrl r = return Nothing
+	| otherwise = safely $ Git.configRead r
+	where
+		safely a = do
+			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			case result of
+				Left _ -> return Nothing
+				Right r' -> return $ Just r'
+		pipedconfig cmd params = safely $
+			pOpen ReadFromPipe cmd params $
+				Git.hConfigRead r
+
+		configlist =
+			Remotes.onRemote r (pipedconfig, Nothing) "configlist" []
+		manualconfiglist = do
+			sshoptions <- Remotes.repoConfig r "ssh-options" ""
+			let sshcmd =
+				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
+				"git config --list"
+			liftIO $ pipedconfig "ssh" $
+				words sshoptions ++ [Git.urlHost r, sshcmd]
+
+		-- First, try sshing and running git config manually,
+		-- only fall back to git-annex-shell configlist if that
+		-- fails.
+		-- 
+		-- This is done for two reasons, first I'd like this
+		-- subcommand to be usable on non-git-annex repos.
+		-- Secondly, configlist doesn't include information about
+		-- the remote's remotes.
+		sshscan = do
+			showNote "sshing..."
+			showProgress
+			v <- manualconfiglist
+			case v of
+				Nothing -> configlist
+				ok -> return ok
diff --git a/GitAnnex.hs b/GitAnnex.hs
index b09ec82ffc..3be222874f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -38,6 +38,7 @@ import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
+import qualified Command.Map
 
 cmds :: [Command]
 cmds = concat
@@ -64,6 +65,7 @@ cmds = concat
 	, Command.DropUnused.command
 	, Command.Find.command
 	, Command.Migrate.command
+	, Command.Map.command
 	]
 
 options :: [Option]
diff --git a/Remotes.hs b/Remotes.hs
index 9f1e2ee507..89b4032475 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -7,6 +7,7 @@
 
 module Remotes (
 	list,
+	tryGitConfigRead,
 	readConfigs,
 	keyPossibilities,
 	inAnnex,
@@ -14,7 +15,8 @@ module Remotes (
 	byName,
 	copyFromRemote,
 	copyToRemote,
-	onRemote
+	onRemote,
+	repoConfig
 ) where
 
 import Control.Exception.Extensible
@@ -77,7 +79,6 @@ tryGitConfigRead r
 				then new : exchange ls new
 				else old : exchange ls new
 
-
 {- Reads the configs of all remotes.
  -
  - This has to be called before things that rely on eg, the UUID of
@@ -92,9 +93,9 @@ tryGitConfigRead r
  - -}
 readConfigs :: Annex ()
 readConfigs = do
-	g <- Annex.gitRepo
 	remotesread <- Annex.getState Annex.remotesread
 	unless remotesread $ do
+		g <- Annex.gitRepo
 		allremotes <- filterM repoNotIgnored $ Git.remotes g
 		let cheap = filter (not . Git.repoIsUrl) allremotes
 		let expensive = filter Git.repoIsUrl allremotes
diff --git a/UUID.hs b/UUID.hs
index a654424b4f..6c719b41ef 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -11,13 +11,15 @@
 module UUID (
 	UUID,
 	getUUID,
+	getUncachedUUID,
 	prepUUID,
 	genUUID,
 	reposByUUID,
 	reposWithoutUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidLog
+	uuidLog,
+	uuidMap
 ) where
 
 import Control.Monad.State
@@ -60,7 +62,7 @@ getUUID r = do
 	g <- Annex.gitRepo
 
 	let c = cached g
-	let u = uncached
+	let u = getUncachedUUID r
 	
 	if c /= u && u /= ""
 		then do
@@ -68,11 +70,13 @@ getUUID r = do
 			return u
 		else return c
 	where
-		uncached = Git.configGet r "annex.uuid" ""
 		cached g = Git.configGet g cachekey ""
 		updatecache g u = when (g /= r) $ Annex.setConfig cachekey u
 		cachekey = "remote." ++ Git.repoRemoteName r ++ ".annex-uuid"
 
+getUncachedUUID :: Git.Repo -> UUID
+getUncachedUUID r = Git.configGet r "annex.uuid" ""
+
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = do
diff --git a/debian/changelog b/debian/changelog
index 6bfc946442..42d45c3a3e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.20) UNRELEASED; urgency=low
   * unannex: Commit staged changes at end, to avoid some confusing behavior
     with the pre-commit hook, which would see some types of commits after
     an unannex as checking in of an unlocked file.
+  * map: New subcommand that uses graphviz to display a nice map of
+    the git repository network.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 68a1672df0..d670d626ee 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -154,6 +154,23 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Note that the content is not removed from the backend it was previously in.
   Use `git annex unused` to find and remove such content. 
 
+* map
+
+  Helps you keep track of your repositories, and the connections between them,
+  by going out and looking at all the ones it can get to, and generating a
+  Graphviz file displaying it all. If the `dot` command is available, it is
+  used to display the file to your screen (using x11 backend).
+
+  Note that this only connects to hosts that the host it's run on can
+  directly connect to. It does not try to tunnel through intermediate hosts.
+  So it might not show all connections between the repositories in the network.
+
+  Also, if connecting to a host requires a password, you might have to enter
+  it several times as the map is being built.
+
+  Note that this subcommand can be used to graph any git repository; it
+  is not limited to git-annex repositories.
+
 * unannex [path ...]
 
   Use this to undo an accidental `git annex add` command. You can use

From 17829be0fd2ec090c2854f05856e91ca4359e71c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 22:20:55 -0400
Subject: [PATCH 0818/8313] map improvements

added uuid.log repos

group repos by host

avoid displaying most urls

display remote names on edges

still some bugs
---
 Command/Map.hs | 117 +++++++++++++++++++++++++++++++++++++++++--------
 GitRepo.hs     |  20 ++++++---
 Remotes.hs     |   8 ++--
 UUID.hs        |   3 +-
 4 files changed, 119 insertions(+), 29 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 753d6ebdcb..b89f8f89b4 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -10,6 +10,8 @@ module Command.Map where
 import Control.Monad.State (liftIO)
 import Control.Exception.Extensible
 import System.Cmd.Utils
+import qualified Data.Map as M
+import Data.List.Utils
 
 import Command
 import qualified Annex
@@ -18,6 +20,7 @@ import qualified Remotes
 import Messages
 import Types
 import Utility
+import UUID
 
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
@@ -33,32 +36,110 @@ start = do
 	g <- Annex.gitRepo
 	rs <- spider g
 
-	liftIO $ writeFile file (dotGraph rs)
-	showLongNote $ "running: dot -Tx11 " ++ file ++ "\n"
+	umap <- uuidMap
+
+	liftIO $ writeFile file (drawMap rs umap)
+	showLongNote $ "running: dot -Tx11 " ++ file
+	showProgress
 	r <- liftIO $ boolSystem "dot" ["-Tx11", file]
 	return $ Just $ return $ Just $ return r
 	where
 		file = "map.dot"
 
-{- Generates a graph for dot(1). Each repository is displayed
- - as a node, and each of its remotes is represented as an edge
+{- Generates a graph for dot(1). Each repository, and any other uuids, are
+ - displayed as a node, and each of its remotes is represented as an edge
  - pointing at the node for the remote. -}
-dotGraph :: [Git.Repo] -> String
-dotGraph rs = unlines $ [header] ++ map dotGraphRepo rs ++ [footer]
+drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
+drawMap rs umap = dotGraph $ repos ++ others
+	where
+		repos = map (dotGraphRepo umap rs) rs
+		others = map uuidnode (M.keys umap)
+		uuidnode u = dotGraphNode u $ M.findWithDefault "" u umap
+
+dotGraphRepo :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
+dotGraphRepo umap fullinfo r = unlines $ node:edges
+	where
+		node = inhost $ dotGraphNode (nodeid r) (repoName umap r)
+		edges = map edge (Git.remotes r)
+
+		inhost a
+			| Git.repoIsUrl r = dotSubGraph hostname a
+			| otherwise = a
+
+		hostname = head $ split "." $ Git.urlHost r
+
+		edge to =
+			-- get the full info for the repo since its UUID
+			-- is in there
+			let to' = findfullinfo to
+			in dotGraphEdge
+				(nodeid r)
+				(nodeid $ makeabs r to')
+				(edgename to to')
+		
+		-- Only name an edge if the name is different than the name
+		-- that will be used for the destination node. (This
+		-- reduces visual clutter.)
+		edgename to to' =
+			case (Git.repoRemoteName to) of
+				Nothing -> Nothing
+				Just n ->
+					if (n == repoName umap to')
+						then Nothing
+						else Just n
+
+		nodeid n =
+			case (getUncachedUUID n) of
+				"" -> Git.repoLocation n
+				u -> u
+		findfullinfo n =
+			case (filter (same n) fullinfo) of
+				[] -> n
+				(n':_) -> n'
+
+repoName :: (M.Map UUID String) -> Git.Repo -> String
+repoName umap r
+	| null repouuid = fallback
+	| otherwise = M.findWithDefault fallback repouuid umap
+	where
+		repouuid = getUncachedUUID r
+		fallback =
+			case (Git.repoRemoteName r) of
+				Just n -> n
+				Nothing -> "unknown"
+
+dotGraphNode :: String -> String -> String
+dotGraphNode nodeid desc = dotLineLabeled desc $ dotQuote nodeid
+
+dotGraphEdge :: String -> String -> Maybe String -> String
+dotGraphEdge fromid toid d =
+	case d of
+		Nothing -> dotLine edge
+		Just desc -> dotLineLabeled desc edge
+	where
+		edge = dotQuote fromid ++ " -> " ++ dotQuote toid
+
+dotGraph :: [String] -> String
+dotGraph s = unlines $ [header] ++ s ++ [footer]
 	where
 		header = "digraph map {"
 		footer= "}"
 
-dotGraphRepo :: Git.Repo -> String
-dotGraphRepo r = unlines $ map dotline (node:edges)
+dotQuote :: String -> String
+dotQuote s = "\"" ++ s ++ "\""
+
+dotLine :: String -> String
+dotLine s = "\t" ++ s ++ ";"
+
+dotLineLabeled :: String -> String -> String
+dotLineLabeled label s = dotLine $ s ++ " [ label=" ++ dotQuote label ++ " ]"
+
+dotSubGraph :: String -> String -> String
+dotSubGraph label s = "subgraph " ++ name ++ "{ " ++ setlabel ++ s ++ " }"
 	where
-		node = nodename r ++
-			" [ label=" ++ dotquote (Git.repoDescribe r) ++ " ]"
-		edges = map edge (Git.remotes r)
-		edge e = nodename r ++ " -> " ++ nodename (makeabs r e)
-		nodename n = dotquote (Git.repoLocation n)
-		dotquote s = "\"" ++ s ++ "\""
-		dotline s = "\t" ++ s ++ ";"
+		-- the "cluster_" makes dot draw a box
+		name = dotQuote ("cluster_ " ++ label)
+		setlabel = dotLine $ "label=" ++ dotQuote label
 
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
@@ -81,13 +162,13 @@ makeabs repo remote
 		where
 			combinedurl =
 				Git.urlScheme repo ++ "//" ++
-				Git.urlHost repo ++
+				Git.urlHostFull repo ++
 				Git.workTree remote
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
 same a b
-	| both Git.repoIsSsh = matching Git.urlHost && matching Git.workTree
+	| both Git.repoIsSsh = matching Git.urlHostFull && matching Git.workTree
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
 	| otherwise = False
 		
@@ -134,7 +215,7 @@ tryScan r
 				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
 				"git config --list"
 			liftIO $ pipedconfig "ssh" $
-				words sshoptions ++ [Git.urlHost r, sshcmd]
+				words sshoptions ++ [Git.urlHostFull r, sshcmd]
 
 		-- First, try sshing and running git config manually,
 		-- only fall back to git-annex-shell configlist if that
diff --git a/GitRepo.hs b/GitRepo.hs
index b5a94d4263..4b252868ff 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -22,6 +22,7 @@ module GitRepo (
 	relative,
 	urlPath,
 	urlHost,
+	urlHostFull,
 	urlScheme,
 	configGet,
 	configMap,
@@ -124,11 +125,11 @@ repoLocation Repo { location = Dir dir } = dir
 remotesAdd :: Repo -> [Repo] -> Repo
 remotesAdd repo rs = repo { remotes = rs }
 
-{- Returns the name of the remote that corresponds to the repo, if 
- - it is a remote. Otherwise, "" -}
-repoRemoteName :: Repo -> String
-repoRemoteName Repo { remoteName = Just name } = name
-repoRemoteName _ = ""
+{- Returns the name of the remote that corresponds to the repo, if
+ - it is a remote. -}
+repoRemoteName :: Repo -> Maybe String
+repoRemoteName Repo { remoteName = Just name } = Just name
+repoRemoteName _ = Nothing
 
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
@@ -209,11 +210,18 @@ urlScheme repo = assertUrl repo $ error "internal"
 
 {- Hostname of an URL repo. (May include a username and/or port too.) -}
 urlHost :: Repo -> String
-urlHost Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a
+urlHost Repo { location = Url u } = uriRegName a
 	where
 		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
 urlHost repo = assertUrl repo $ error "internal"
 
+{- Full hostname of an URL repo. (May include a username and/or port too.) -}
+urlHostFull :: Repo -> String
+urlHostFull Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a
+	where
+		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
+urlHostFull repo = assertUrl repo $ error "internal"
+
 {- Path of an URL repo. -}
 urlPath :: Repo -> String
 urlPath Repo { location = Url u } = uriPath u
diff --git a/Remotes.hs b/Remotes.hs
index 89b4032475..15f5185b91 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -205,7 +205,7 @@ repoNotIgnored r = do
 			name <- Annex.getState a
 			case name of
 				Nothing -> return False
-				Just n -> return $ n == Git.repoRemoteName r
+				n -> return $ n == Git.repoRemoteName r
 
 {- Checks if two repos are the same, by comparing their remote names. -}
 same :: Git.Repo -> Git.Repo -> Bool
@@ -217,7 +217,7 @@ byName "." = Annex.gitRepo -- special case to refer to current repository
 byName name = do
 	when (null name) $ error "no remote specified"
 	g <- Annex.gitRepo
-	let match = filter (\r -> name == Git.repoRemoteName r) $
+	let match = filter (\r -> Just name == Git.repoRemoteName r) $
 		Git.remotes g
 	when (null match) $ error $
 		"there is no git remote named \"" ++ name ++ "\""
@@ -309,7 +309,7 @@ git_annex_shell r command params
 	| Git.repoIsSsh r = do
 		sshoptions <- repoConfig r "ssh-options" ""
 		return $ Just $ ["ssh"] ++ words sshoptions ++ 
-			[Git.urlHost r, sshcmd]
+			[Git.urlHostFull r, sshcmd]
 	| otherwise = return Nothing
 	where
 		dir = Git.workTree r
@@ -325,5 +325,5 @@ repoConfig r key def = do
 	let def' = Git.configGet g global def
 	return $ Git.configGet g local def'
 	where
-		local = "remote." ++ Git.repoRemoteName r ++ ".annex-" ++ key
+		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 		global = "annex." ++ key
diff --git a/UUID.hs b/UUID.hs
index 6c719b41ef..67cba30313 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -26,6 +26,7 @@ import Control.Monad.State
 import System.Cmd.Utils
 import System.IO
 import qualified Data.Map as M
+import Data.Maybe
 
 import qualified GitRepo as Git
 import Types
@@ -72,7 +73,7 @@ getUUID r = do
 	where
 		cached g = Git.configGet g cachekey ""
 		updatecache g u = when (g /= r) $ Annex.setConfig cachekey u
-		cachekey = "remote." ++ Git.repoRemoteName r ++ ".annex-uuid"
+		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
 getUncachedUUID r = Git.configGet r "annex.uuid" ""

From 1b1a37b7b1c9ee5c7f7d5c176222fc3c7e5e8ab4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 22:44:17 -0400
Subject: [PATCH 0819/8313] refactor

---
 Command/Map.hs | 52 ++++++++++----------------------------------------
 Dot.hs         | 47 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 42 deletions(-)
 create mode 100644 Dot.hs

diff --git a/Command/Map.hs b/Command/Map.hs
index b89f8f89b4..5b035e283f 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -21,6 +21,7 @@ import Messages
 import Types
 import Utility
 import UUID
+import qualified Dot
 
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
@@ -50,41 +51,41 @@ start = do
  - displayed as a node, and each of its remotes is represented as an edge
  - pointing at the node for the remote. -}
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = dotGraph $ repos ++ others
+drawMap rs umap = Dot.graph $ repos ++ others
 	where
 		repos = map (dotGraphRepo umap rs) rs
 		others = map uuidnode (M.keys umap)
-		uuidnode u = dotGraphNode u $ M.findWithDefault "" u umap
+		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
 
 dotGraphRepo :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 dotGraphRepo umap fullinfo r = unlines $ node:edges
 	where
-		node = inhost $ dotGraphNode (nodeid r) (repoName umap r)
+		node = inhost $ Dot.graphNode (nodeid r) (repoName umap r)
 		edges = map edge (Git.remotes r)
 
 		inhost a
-			| Git.repoIsUrl r = dotSubGraph hostname a
+			| Git.repoIsUrl r = Dot.subGraph (Git.urlHost r) (hostname r) a
 			| otherwise = a
 
-		hostname = head $ split "." $ Git.urlHost r
+		hostname n = head $ split "." $ Git.urlHost n
 
 		edge to =
 			-- get the full info for the repo since its UUID
 			-- is in there
 			let to' = findfullinfo to
-			in dotGraphEdge
+			in Dot.graphEdge
 				(nodeid r)
 				(nodeid $ makeabs r to')
 				(edgename to to')
 		
 		-- Only name an edge if the name is different than the name
-		-- that will be used for the destination node. (This
-		-- reduces visual clutter.)
+		-- that will be used for the destination node, and is
+		-- different from its hostname. (This reduces visual clutter.)
 		edgename to to' =
 			case (Git.repoRemoteName to) of
 				Nothing -> Nothing
 				Just n ->
-					if (n == repoName umap to')
+					if (n == repoName umap to' || n == hostname to')
 						then Nothing
 						else Just n
 
@@ -108,39 +109,6 @@ repoName umap r
 				Just n -> n
 				Nothing -> "unknown"
 
-dotGraphNode :: String -> String -> String
-dotGraphNode nodeid desc = dotLineLabeled desc $ dotQuote nodeid
-
-dotGraphEdge :: String -> String -> Maybe String -> String
-dotGraphEdge fromid toid d =
-	case d of
-		Nothing -> dotLine edge
-		Just desc -> dotLineLabeled desc edge
-	where
-		edge = dotQuote fromid ++ " -> " ++ dotQuote toid
-
-dotGraph :: [String] -> String
-dotGraph s = unlines $ [header] ++ s ++ [footer]
-	where
-		header = "digraph map {"
-		footer= "}"
-
-dotQuote :: String -> String
-dotQuote s = "\"" ++ s ++ "\""
-
-dotLine :: String -> String
-dotLine s = "\t" ++ s ++ ";"
-
-dotLineLabeled :: String -> String -> String
-dotLineLabeled label s = dotLine $ s ++ " [ label=" ++ dotQuote label ++ " ]"
-
-dotSubGraph :: String -> String -> String
-dotSubGraph label s = "subgraph " ++ name ++ "{ " ++ setlabel ++ s ++ " }"
-	where
-		-- the "cluster_" makes dot draw a box
-		name = dotQuote ("cluster_ " ++ label)
-		setlabel = dotLine $ "label=" ++ dotQuote label
-
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
 spider r = spider' [r] []
diff --git a/Dot.hs b/Dot.hs
new file mode 100644
index 0000000000..1d9c29c532
--- /dev/null
+++ b/Dot.hs
@@ -0,0 +1,47 @@
+{- a simple graphviz / dot(1) digraph description generator library
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Dot where -- import qualified
+
+{- generates a graph description from a list of lines -}
+graph :: [String] -> String
+graph s = unlines $ [header] ++ s ++ [footer]
+	where
+		header = "digraph map {"
+		footer= "}"
+
+{- a node in the graph -}
+graphNode :: String -> String -> String
+graphNode nodeid desc = lineLabeled desc $ quote nodeid
+
+{- an edge between two nodes -}
+graphEdge :: String -> String -> Maybe String -> String
+graphEdge fromid toid d =
+	case d of
+		Nothing -> line edge
+		Just desc -> lineLabeled desc edge
+	where
+		edge = quote fromid ++ " -> " ++ quote toid
+
+quote :: String -> String
+quote s = "\"" ++ s ++ "\""
+
+line :: String -> String
+line s = "\t" ++ s ++ ";"
+
+{- a line with a label -}
+lineLabeled :: String -> String -> String
+lineLabeled label s = line $ s ++ " [ label=" ++ quote label ++ " ]"
+
+{- apply to graphNode to put the node in a labeled box -}
+subGraph :: String -> String -> String -> String
+subGraph subid label s = line $
+	"subgraph " ++ name ++ "{\n" ++ setlabel ++ "\n" ++ s ++ "\n}"
+	where
+		-- the "cluster_" makes dot draw a box
+		name = quote ("cluster_" ++ subid)
+		setlabel = line $ "label=" ++ quote label

From dff47d51e65fcf14566a06ebaae112c859d1824c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 23:23:16 -0400
Subject: [PATCH 0820/8313] cleanup

---
 Command/Map.hs | 88 +++++++++++++++++++++++++++-----------------------
 1 file changed, 47 insertions(+), 41 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 5b035e283f..1d38bc42fe 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -53,51 +53,20 @@ start = do
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
 drawMap rs umap = Dot.graph $ repos ++ others
 	where
-		repos = map (dotGraphRepo umap rs) rs
+		repos = map (node umap rs) rs
 		others = map uuidnode (M.keys umap)
 		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
 
-dotGraphRepo :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
-dotGraphRepo umap fullinfo r = unlines $ node:edges
-	where
-		node = inhost $ Dot.graphNode (nodeid r) (repoName umap r)
-		edges = map edge (Git.remotes r)
+hostname :: Git.Repo -> String
+hostname r
+	| Git.repoIsUrl r = Git.urlHost r
+	| otherwise = "localhost"
 
-		inhost a
-			| Git.repoIsUrl r = Dot.subGraph (Git.urlHost r) (hostname r) a
-			| otherwise = a
-
-		hostname n = head $ split "." $ Git.urlHost n
-
-		edge to =
-			-- get the full info for the repo since its UUID
-			-- is in there
-			let to' = findfullinfo to
-			in Dot.graphEdge
-				(nodeid r)
-				(nodeid $ makeabs r to')
-				(edgename to to')
-		
-		-- Only name an edge if the name is different than the name
-		-- that will be used for the destination node, and is
-		-- different from its hostname. (This reduces visual clutter.)
-		edgename to to' =
-			case (Git.repoRemoteName to) of
-				Nothing -> Nothing
-				Just n ->
-					if (n == repoName umap to' || n == hostname to')
-						then Nothing
-						else Just n
-
-		nodeid n =
-			case (getUncachedUUID n) of
-				"" -> Git.repoLocation n
-				u -> u
-		findfullinfo n =
-			case (filter (same n) fullinfo) of
-				[] -> n
-				(n':_) -> n'
+basehostname :: Git.Repo -> String
+basehostname r = head $ split "." $ hostname r
 
+{- A name to display for a repo. Uses the name from uuid.log if available,
+ - or the remote name if not. -}
 repoName :: (M.Map UUID String) -> Git.Repo -> String
 repoName umap r
 	| null repouuid = fallback
@@ -109,6 +78,43 @@ repoName umap r
 				Just n -> n
 				Nothing -> "unknown"
 
+{- A unique id for the node. Uses the annex.uuid if available. -}
+nodeId :: Git.Repo -> String
+nodeId r =
+	case (getUncachedUUID r) of
+		"" -> Git.repoLocation r
+		u -> u
+
+{- A node representing a repo. -}
+node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
+node umap fullinfo r = unlines $ n:edges
+	where
+		n = Dot.subGraph (hostname r) (basehostname r) $
+			Dot.graphNode (nodeId r) (repoName umap r)
+		edges = map (edge umap fullinfo r) (Git.remotes r)
+
+{- An edge between two repos. The second repo is a remote of the first. -}
+edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
+edge umap fullinfo from to =
+	Dot.graphEdge (nodeId from) (nodeId $ makeabs from fullto) edgename
+	where
+		-- get the full info for the remote, to get its UUID
+		fullto = findfullinfo to
+		findfullinfo n =
+			case (filter (same n) fullinfo) of
+				[] -> n
+				(n':_) -> n'
+		{- Only name an edge if the name is different than the name
+		 - that will be used for the destination node, and is
+		 - different from its hostname. (This reduces visual clutter.) -}
+		edgename =
+			case (Git.repoRemoteName to) of
+				Nothing -> Nothing
+				Just n ->
+					if (n == repoName umap fullto || n == hostname fullto)
+						then Nothing
+						else Just n
+
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
 spider r = spider' [r] []
@@ -148,7 +154,7 @@ same a b
 {- reads the config of a remote, with progress display -}
 scan :: Git.Repo -> Annex Git.Repo
 scan r = do
-	showStart "map" (Git.repoDescribe r)
+	showStart "map" $ Git.repoDescribe r
 	v <- tryScan r
 	case v of
 		Just r' -> do

From 67c1facad150cfbc706721cbd9b482be22a31f4c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Feb 2011 23:23:36 -0400
Subject: [PATCH 0821/8313] fix infinite loop

Local repos with the same path are not different. :)
---
 Command/Map.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Map.hs b/Command/Map.hs
index 1d38bc42fe..fa2a412536 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -144,6 +144,7 @@ same :: Git.Repo -> Git.Repo -> Bool
 same a b
 	| both Git.repoIsSsh = matching Git.urlHostFull && matching Git.workTree
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
+	| neither Git.repoIsSsh = matching Git.workTree
 	| otherwise = False
 		
 	where

From 0fd0e414ec593dcb965ca9a348798857be2bb3e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 00:06:23 -0400
Subject: [PATCH 0822/8313] color unreachable nodes

---
 Command/Map.hs | 17 +++++++++++++----
 Dot.hs         | 43 +++++++++++++++++++++++++++----------------
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index fa2a412536..bc117d4790 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -51,11 +51,12 @@ start = do
  - displayed as a node, and each of its remotes is represented as an edge
  - pointing at the node for the remote. -}
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = Dot.graph $ repos ++ others
+drawMap rs umap = Dot.graph $ others ++ repos
 	where
 		repos = map (node umap rs) rs
 		others = map uuidnode (M.keys umap)
-		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
+		uuidnode u = unreachable $
+			Dot.graphNode u $ M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
@@ -78,7 +79,7 @@ repoName umap r
 				Just n -> n
 				Nothing -> "unknown"
 
-{- A unique id for the node. Uses the annex.uuid if available. -}
+{- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
 nodeId r =
 	case (getUncachedUUID r) of
@@ -90,8 +91,11 @@ node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
 		n = Dot.subGraph (hostname r) (basehostname r) $
-			Dot.graphNode (nodeId r) (repoName umap r)
+			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
+		decorate
+			| Git.configMap r == M.empty = unreachable
+			| otherwise = reachable
 
 {- An edge between two repos. The second repo is a remote of the first. -}
 edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
@@ -115,6 +119,11 @@ edge umap fullinfo from to =
 						then Nothing
 						else Just n
 
+unreachable :: String -> String
+unreachable s = Dot.fillColor "red" s
+reachable :: String -> String
+reachable s = Dot.fillColor "white" s
+
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]
 spider r = spider' [r] []
diff --git a/Dot.hs b/Dot.hs
index 1d9c29c532..0507c638ce 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -9,39 +9,50 @@ module Dot where -- import qualified
 
 {- generates a graph description from a list of lines -}
 graph :: [String] -> String
-graph s = unlines $ [header] ++ s ++ [footer]
+graph s = unlines $ [header] ++ map formatLine s ++ [footer]
 	where
 		header = "digraph map {"
 		footer= "}"
 
 {- a node in the graph -}
 graphNode :: String -> String -> String
-graphNode nodeid desc = lineLabeled desc $ quote nodeid
+graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid d =
-	case d of
-		Nothing -> line edge
-		Just desc -> lineLabeled desc edge
+graphEdge fromid toid desc =
+	case desc of
+		Nothing -> edge
+		Just d -> label d edge
 	where
 		edge = quote fromid ++ " -> " ++ quote toid
 
-quote :: String -> String
-quote s = "\"" ++ s ++ "\""
+{- adds a label to a node or edge -}
+label :: String -> String -> String
+label l s = attr "label" l s
 
-line :: String -> String
-line s = "\t" ++ s ++ ";"
+{- adds an attribute to a node or edge
+ - (can be called multiple times for multiple attributes) -}
+attr :: String -> String -> String -> String
+attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]"
 
-{- a line with a label -}
-lineLabeled :: String -> String -> String
-lineLabeled label s = line $ s ++ " [ label=" ++ quote label ++ " ]"
+{- fills a node with a color -}
+fillColor :: String -> String -> String
+fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
 
 {- apply to graphNode to put the node in a labeled box -}
 subGraph :: String -> String -> String -> String
-subGraph subid label s = line $
-	"subgraph " ++ name ++ "{\n" ++ setlabel ++ "\n" ++ s ++ "\n}"
+subGraph subid l s =
+	"subgraph " ++ name ++ "{\n\t" ++ setlabel ++ "\n\t\t" ++ s ++ "\n\t}"
 	where
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
-		setlabel = line $ "label=" ++ quote label
+		setlabel = formatLine $ "label=" ++ quote l
+
+formatLine :: String -> String	
+formatLine s = "\t" ++ s ++ ";"
+
+quote :: String -> String
+quote s = "\"" ++ s' ++ "\""
+	where
+		s' = filter (/= '"') s

From 926df3d91ec29f35e2cdf3df0d286721f40b9cbf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 00:13:47 -0400
Subject: [PATCH 0823/8313] node ordering

---
 Command/Map.hs | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index bc117d4790..83207d551c 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -49,12 +49,19 @@ start = do
 
 {- Generates a graph for dot(1). Each repository, and any other uuids, are
  - displayed as a node, and each of its remotes is represented as an edge
- - pointing at the node for the remote. -}
+ - pointing at the node for the remote.
+ -
+ - The order nodes are added to the graph matters, since dot will draw
+ - the first ones near to the top and left. So it looks better to put
+ - the repositories first, followed by uuids that were not matched
+ - to a repository.
+ -}
 drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = Dot.graph $ others ++ repos
+drawMap rs umap = Dot.graph $ repos ++ others
 	where
 		repos = map (node umap rs) rs
-		others = map uuidnode (M.keys umap)
+		ruuids = map getUncachedUUID rs
+		others = map uuidnode $ filter (`notElem` ruuids) (M.keys umap)
 		uuidnode u = unreachable $
 			Dot.graphNode u $ M.findWithDefault "" u umap
 
@@ -120,9 +127,9 @@ edge umap fullinfo from to =
 						else Just n
 
 unreachable :: String -> String
-unreachable s = Dot.fillColor "red" s
+unreachable = Dot.fillColor "red"
 reachable :: String -> String
-reachable s = Dot.fillColor "white" s
+reachable = Dot.fillColor "white"
 
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]

From 30869187f0890f9e742b4a5dbb4579b0fca6f7e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 00:36:36 -0400
Subject: [PATCH 0824/8313] improve output

---
 Dot.hs | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Dot.hs b/Dot.hs
index 0507c638ce..fcd0c19cc2 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -9,7 +9,7 @@ module Dot where -- import qualified
 
 {- generates a graph description from a list of lines -}
 graph :: [String] -> String
-graph s = unlines $ [header] ++ map formatLine s ++ [footer]
+graph s = unlines $ [header] ++ map indent s ++ [footer]
 	where
 		header = "digraph map {"
 		footer= "}"
@@ -20,7 +20,7 @@ graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid desc =
+graphEdge fromid toid desc = indent $
 	case desc of
 		Nothing -> edge
 		Just d -> label d edge
@@ -43,14 +43,15 @@ fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
 {- apply to graphNode to put the node in a labeled box -}
 subGraph :: String -> String -> String -> String
 subGraph subid l s =
-	"subgraph " ++ name ++ "{\n\t" ++ setlabel ++ "\n\t\t" ++ s ++ "\n\t}"
+	"subgraph " ++ name ++ " {\n" ++ ii setlabel ++ ii s ++ indent "}"
 	where
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
-		setlabel = formatLine $ "label=" ++ quote l
+		setlabel = "label=" ++ quote l
+		ii x = (indent $ indent x) ++ "\n"
 
-formatLine :: String -> String	
-formatLine s = "\t" ++ s ++ ";"
+indent ::String -> String
+indent s = "\t" ++ s
 
 quote :: String -> String
 quote s = "\"" ++ s' ++ "\""

From ef2d4f650edff99b67554be9288face87159131e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Feb 2011 01:56:45 -0400
Subject: [PATCH 0825/8313] fix absrepo data loss

it was dropping the config map for the repos it changed
---
 Command/Map.hs | 30 ++++++++++++++----------------
 GitRepo.hs     | 16 +++++++++++++++-
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 83207d551c..d8dd0e94cf 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -107,7 +107,7 @@ node umap fullinfo r = unlines $ n:edges
 {- An edge between two repos. The second repo is a remote of the first. -}
 edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
 edge umap fullinfo from to =
-	Dot.graphEdge (nodeId from) (nodeId $ makeabs from fullto) edgename
+	Dot.graphEdge (nodeId from) (nodeId $ absRepo from fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
 		fullto = findfullinfo to
@@ -140,20 +140,13 @@ spider' (r:rs) known
 	| any (same r) known = spider' rs known
 	| otherwise = do
 		r' <- scan r
-		let remotes = map (makeabs r') (Git.remotes r')
+		let remotes = map (absRepo r') (Git.remotes r')
 		spider' (rs ++ remotes) (r':known)
 
-{- Makes a remote have an absolute url, rather than a host-local path. -}
-makeabs :: Git.Repo -> Git.Repo -> Git.Repo
-makeabs repo remote
-	| Git.repoIsUrl remote = remote
-	| not $ Git.repoIsUrl repo = remote
-	| otherwise = Git.repoFromUrl combinedurl
-		where
-			combinedurl =
-				Git.urlScheme repo ++ "//" ++
-				Git.urlHostFull repo ++
-				Git.workTree remote
+absRepo :: Git.Repo -> Git.Repo -> Git.Repo
+absRepo reference r
+	| Git.repoIsUrl reference = Git.localToUrl reference r
+	| otherwise = r
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
@@ -217,9 +210,14 @@ tryScan r
 		-- Secondly, configlist doesn't include information about
 		-- the remote's remotes.
 		sshscan = do
-			showNote "sshing..."
-			showProgress
+			sshnote
 			v <- manualconfiglist
 			case v of
-				Nothing -> configlist
+				Nothing -> do
+					sshnote
+					configlist
 				ok -> return ok
+
+		sshnote = do
+			showNote "sshing..."
+			showProgress
diff --git a/GitRepo.hs b/GitRepo.hs
index 4b252868ff..7bb20fc53c 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -13,6 +13,7 @@ module GitRepo (
 	repoFromCwd,
 	repoFromPath,
 	repoFromUrl,
+	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
 	repoDescribe,
@@ -109,6 +110,19 @@ repoFromUrl url
 				Just v -> v
 				Nothing -> error $ "bad url " ++ url
 
+{- Converts a Local Repo into a remote repo, using the reference repo
+ - which is assumed to be on the same host. -}
+localToUrl :: Repo -> Repo -> Repo
+localToUrl reference r
+	| not $ repoIsUrl reference = error "internal error; reference repo not url"
+	| repoIsUrl r = r
+	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
+	where
+		absurl =
+			urlScheme reference ++ "//" ++
+			urlHostFull reference ++
+			workTree r
+
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
 repoDescribe Repo { remoteName = Just name } = name
@@ -338,7 +352,7 @@ configStore :: Repo -> String -> Repo
 configStore repo s = r { remotes = configRemotes r }
 	where r = repo { config = configParse s }
 
-{- Checks if a string fron git config is a true value. -}
+{- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 

From c975384195467b5e5bca32d8199330e96be9feba Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
 
Date: Sun, 6 Feb 2011 06:02:59 +0000
Subject: [PATCH 0826/8313] Added a comment: Got it going!

---
 ..._1_34120e82331ace01a6a4960862d38f2d._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment b/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment
new file mode 100644
index 0000000000..a33fef7d99
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmd3qri1pXEYktlxYGwj37wCnrM4FMEJCc"
+ nickname="Antoine"
+ subject="Got it going!"
+ date="2011-02-06T06:02:57Z"
+ content="""
+Thanks to your feedback, I got it going.  
+
+Maybe those two should be added to the 'OSX how-to' in the forum   
+
+[realizes pcre-light is needed but pcre not installed on my mac]  
+sudo port install pcre  
+sudo cabal install pcre-light  
+
+[tests are failing, need haskell's quickcheck]  
+sudo cabal install quickcheck  
+"""]]

From 413f50b82db47ca231fd36b73551b2633488a06e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 6 Feb 2011 17:39:53 +0000
Subject: [PATCH 0827/8313] Added a comment

---
 .../comment_2_cc53d1681d576186dbc868dd9801d551._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment b/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment
new file mode 100644
index 0000000000..91d3e89f06
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-02-06T17:39:52Z"
+ content="""
+Yes, I've moved it to [[install/OSX]] page where anyone can update it in this wiki, and added your improvements.
+"""]]

From 6c45f1123d0709787c7df15c46b8d97e9813eb83 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Feb 2011 13:43:31 -0400
Subject: [PATCH 0828/8313] reord install pages; per-OS instructions

---
 doc/download.mdwn               |  6 ++----
 doc/forum/git-annex_on_OSX.mdwn | 22 +---------------------
 doc/install.mdwn                |  6 ++++++
 doc/install/Debian.mdwn         |  9 +++++++++
 doc/install/OSX.mdwn            | 17 +++++++++++++++++
 doc/install/Ubuntu.mdwn         |  5 +++++
 6 files changed, 40 insertions(+), 25 deletions(-)
 create mode 100644 doc/install/Debian.mdwn
 create mode 100644 doc/install/OSX.mdwn
 create mode 100644 doc/install/Ubuntu.mdwn

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 6c56c8e25a..748ac0ca34 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -1,7 +1,5 @@
 The main git repository for git-annex is `git://git.kitenet.net/git-annex`
 [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
 
-Users of Debian unstable/testing and Ubuntu natty can
-`apt-get install git-annex`
-
-Next: [[install]]
+Some operating systems include git-annex in easily prepackaged form and
+others need some manual work. See [[install]] for details.
diff --git a/doc/forum/git-annex_on_OSX.mdwn b/doc/forum/git-annex_on_OSX.mdwn
index 302ad10994..a00548366a 100644
--- a/doc/forum/git-annex_on_OSX.mdwn
+++ b/doc/forum/git-annex_on_OSX.mdwn
@@ -1,21 +1 @@
-
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum
-
-[waits forever…]
-[finished]
-[realizes missingh isn't working in MacPorts]
-
-sudo cabal  update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo port install pcre
-sudo cabal install pcre-light
-
-git clone  git://git.kitenet.net/git-annex
-
-cd git-annex
-make
-sudo make install
-

-
-Originally posted by Jon at  --[[Joey]]
+See [[install/OSX]].
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 732660c507..3c7025fa60 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -12,3 +12,9 @@ To build and use git-annex, you will need:
 
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
 but that will be skipped if it is not installed.)
+
+OS-specific instructions:
+
+* [[OSX]]
+* [[Debian]]
+* [[Ubuntu]]
diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn
new file mode 100644
index 0000000000..90c9658a9c
--- /dev/null
+++ b/doc/install/Debian.mdwn
@@ -0,0 +1,9 @@
+If using Debian testing or unstable:
+
+	sudo apt-get install git-annex
+
+git-annex is not yet in a Debian stable release, but the source
+can be built from Debian stable. Just [[download]] it from git, and
+then in the `git-annex` directory, run `dpkg-checkbuilddeps` and install
+the necessary packages. You can also run `sudo debian/rules binary` to build
+a `git-annex.deb`.
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
new file mode 100644
index 0000000000..e107e48eb9
--- /dev/null
+++ b/doc/install/OSX.mdwn
@@ -0,0 +1,17 @@
+
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum
+sudo cabal update
+sudo cabal install missingh
+sudo cabal install utf8-string
+sudo port install pcre
+sudo cabal install pcre-light
+sudo cabal install quickcheck  
+
+git clone  git://git.kitenet.net/git-annex
+
+cd git-annex
+make
+sudo make install
+

+
+Originally posted by Jon at  --[[Joey]]
diff --git a/doc/install/Ubuntu.mdwn b/doc/install/Ubuntu.mdwn
new file mode 100644
index 0000000000..ffc763ff31
--- /dev/null
+++ b/doc/install/Ubuntu.mdwn
@@ -0,0 +1,5 @@
+If using Ubuntu natty or newer:
+
+	sudo apt-get install git-annex
+
+Otherwise, see [[Debian]] manual installation instructions.

From 3093d2fad7c2dfd62b9e016a8e5a895568f4ce06 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 7 Feb 2011 12:43:44 +0000
Subject: [PATCH 0829/8313] Added a comment: tests fail with more recent
 installs of haskell platform

---
 ..._68f0f8ae953589ae26d57310b40c878d._comment | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment b/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment
new file mode 100644
index 0000000000..39f32c244f
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment
@@ -0,0 +1,57 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="tests fail with more recent installs of haskell platform"
+ date="2011-02-07T12:43:43Z"
+ content="""
+I'm running ghc 6.12.3 with the corresponding haskell-platform package from the HP site which I installed in preference to the macports version of haskell-platform (it's quite old). it seems when you install quickcheck, the version that is installed is of version 2.4.0.1 and not 1.2.0 which git-annex depends on for its tests.
+
+
+jtang@x00:~ $ cabal install quickcheck --reinstall               
+Resolving dependencies...
+Configuring QuickCheck-2.4.0.1...
+Preprocessing library QuickCheck-2.4.0.1...
+
+..
+and so on..
+..
+
+

+
+it fails with this
+
+
+[54 of 54] Compiling Main             ( test.hs, test.o )
+
+test.hs:56:3:
+    No instance for (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)
+      arising from a use of `qctest' at test.hs:56:3-64
+    Possible fix:
+      add an instance declaration for
+      (QuickCheck-1.2.0.1:Test.QuickCheck.Arbitrary Char)
+    In the expression:
+        qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode
+    In the first argument of `TestList', namely
+        `[qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,
+          qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,
+          qctest
+            \"prop_idempotent_key_read_show\"
+            BackendTypes.prop_idempotent_key_read_show,
+          qctest
+            \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,
+          ....]'
+    In the second argument of `($)', namely
+        `TestList
+           [qctest \"prop_idempotent_deencode\" Git.prop_idempotent_deencode,
+            qctest \"prop_idempotent_fileKey\" Locations.prop_idempotent_fileKey,
+            qctest
+              \"prop_idempotent_key_read_show\"
+              BackendTypes.prop_idempotent_key_read_show,
+            qctest
+              \"prop_idempotent_shellEscape\" Utility.prop_idempotent_shellEscape,
+            ....]'
+

+
+I'd imagine if I could downgrade, it would compile and pass the tests (I hope)
+
+"""]]

From 82ff914492636869079a33b67b4d918e93213dc7 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Mon, 7 Feb 2011 14:12:44 +0000
Subject: [PATCH 0830/8313] Added a comment: how to reproduce the package
 conflict issue

---
 ..._e552a6cc6d7d1882e14130edfc2d6b3b._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment

diff --git a/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment b/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment
new file mode 100644
index 0000000000..42f44bf9cf
--- /dev/null
+++ b/doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="how to reproduce the package conflict issue"
+ date="2011-02-07T14:12:43Z"
+ content="""
+If you install the monads-fd package (with cabal install for instance), then you can no longer build git-annex:
+
+
+./configure
+  checking cp -a... yes
+  checking cp -p... yes
+  checking cp --reflink=auto... yes
+  checking uuid generator... uuid
+  checking xargs -0... yes
+  checking rsync... yes
+ghc -O2 -Wall --make git-annex
+
+Annex.hs:22:7:
+    Ambiguous module name `Control.Monad.State':
+      it was found in multiple packages: monads-fd-0.2.0.0 mtl-2.0.1.0
+make: *** [git-annex] Error 1
+

+"""]]

From 217e5de24483e30d8dc7f9167da1531ddc0c94a4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 8 Feb 2011 19:00:14 +0000
Subject: [PATCH 0831/8313] Added a comment

---
 ...mment_4_c52be386f79f14c8570a8f1397c68581._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment b/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment
new file mode 100644
index 0000000000..e245e139fb
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-02-08T19:00:14Z"
+ content="""
+I doubt that git-annex can be used with QuickCheck 1.2.0. The QuickCheck I've tested it with is 2.1.0.3 actually.
+
+I suspect you have an old version of the TestPack haskell library on your system, that is linked against QuickCheck 1.2.0. Git-annex has been tested with TestPack 2.0.0, which uses QuickCheck 2.x.
+
+In any case, you don't have to run 'make test' to build git-annex, and my comments above should make the main program compile, I expect.
+"""]]

From 45387b3fcb0785d0e9957a638fdd8ffccb3eb40c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 15:11:49 -0400
Subject: [PATCH 0832/8313] Deal with the mtl/monads-fd conflict.

---
 Makefile                                   | 2 +-
 debian/changelog                           | 1 +
 doc/bugs/conflicting_haskell_packages.mdwn | 9 +++++++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 565edfb0e0..c888fc2151 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 PREFIX=/usr
-GHCFLAGS=-O2 -Wall
+GHCFLAGS=-O2 -Wall -ignore-package monads-fd
 GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell
diff --git a/debian/changelog b/debian/changelog
index 42d45c3a3e..3a5f60fefe 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,7 @@ git-annex (0.20) UNRELEASED; urgency=low
     an unannex as checking in of an unlocked file.
   * map: New subcommand that uses graphviz to display a nice map of
     the git repository network.
+  * Deal with the mtl/monads-fd conflict.
 
  -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
 
diff --git a/doc/bugs/conflicting_haskell_packages.mdwn b/doc/bugs/conflicting_haskell_packages.mdwn
index 6a619de90f..5528fad824 100644
--- a/doc/bugs/conflicting_haskell_packages.mdwn
+++ b/doc/bugs/conflicting_haskell_packages.mdwn
@@ -6,3 +6,12 @@ This can be done by the flags -hide-packages and then -package foo
 > 
 > Could you just show the build problem that you are suggesting I work
 > around? --[[Joey]]
+
+
+> Thanks npouillard, I see the problem now.
+> 
+> 
+> I've added "-ignore-package monads-fd" to GHCFLAGS. I hope I don't
+> really have to hide all packages and individually turn them back on;
+> surely this monads-fd/mtl conflict is an exception, and Haskell's module
+> system is not a mess of conflicting modules? --[[Joey]] [[done]]

From 3775dc96073d52dcae13a020a95186acad789d74 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 8 Feb 2011 19:56:56 +0000
Subject: [PATCH 0833/8313] Added a comment

---
 ..._7f1330a1e541b0f3e2192e596d7f7bee._comment | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment b/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment
new file mode 100644
index 0000000000..9c83feb32f
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment
@@ -0,0 +1,107 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2011-02-08T19:56:55Z"
+ content="""
+Ah, that gave me a good clue, my system just got pretty confused with a mixture of quickcheck and testpack installs. Would it be possible to put up a list of versions of the software you are using on your development environment? (at least the minimum tested version) 
+
+I guess it shouldn't matter to most users who are going to rely on packagers to sort these dependancy issues, but it's nice to know. 
+
+Anyway, the tests build now, and they seem to fail on my (rather messy) install of haskell platform + ghc 6.12 on osx 10.6.6.
+
+
+< output that passed some tests >
+Testing 1:blackbox:0:git-annex init
+Testing 1:blackbox:1:git-annex add:0
+Testing 1:blackbox:1:git-annex add:1
+Cases: 30  Tried: 9  Errors: 0  Failures: 0test: sha1sum: executeFile: does not exist (No such file or directory)
+  git-annex: : hGetLine: end of file
+### Failure in: 1:blackbox:1:git-annex add:1
+add with SHA1 failed
+Testing 1:blackbox:2:git-annex setkey/fromkey
+Cases: 30  Tried: 10  Errors: 0  Failures: 1(checksum...) test: sha1sum: executeFile: does not exist (No such file or directory)
+### Error in:   1:blackbox:2:git-annex setkey/fromkey
+: hGetLine: end of file
+Testing 1:blackbox:3:git-annex unannex:0:no content
+Cases: 30  Tried: 11  Errors: 1  Failures: 1chmod: -R: No such file or directory
+chmod: -R: No such file or directory
+Testing 1:blackbox:3:git-annex unannex:1:with content
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes
+Cases: 30  Tried: 13  Errors: 1  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:0:no remotes
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:4:git-annex drop:1:with remote
+Cases: 30  Tried: 14  Errors: 2  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:1:with remote
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote
+Cases: 30  Tried: 15  Errors: 3  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:4:git-annex drop:2:untrusted remote
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:5:git-annex get
+Cases: 30  Tried: 16  Errors: 4  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:5:git-annex get
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:6:git-annex move
+Cases: 30  Tried: 17  Errors: 5  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:6:git-annex move
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:7:git-annex copy
+Cases: 30  Tried: 18  Errors: 6  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:7:git-annex copy
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:8:git-annex unlock/lock
+Cases: 30  Tried: 19  Errors: 7  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:8:git-annex unlock/lock
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:9:git-annex edit/commit:0
+Cases: 30  Tried: 20  Errors: 8  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:9:git-annex edit/commit:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:9:git-annex edit/commit:1
+Cases: 30  Tried: 21  Errors: 9  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:9:git-annex edit/commit:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:10:git-annex fix
+Cases: 30  Tried: 22  Errors: 10  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:10:git-annex fix
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust
+Cases: 30  Tried: 23  Errors: 11  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:0
+Cases: 30  Tried: 24  Errors: 12  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:1
+Cases: 30  Tried: 25  Errors: 13  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:12:git-annex fsck:2
+Cases: 30  Tried: 26  Errors: 14  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:12:git-annex fsck:2
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:13:git-annex migrate:0
+Cases: 30  Tried: 27  Errors: 15  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:13:git-annex migrate:0
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:13:git-annex migrate:1
+Cases: 30  Tried: 28  Errors: 16  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:13:git-annex migrate:1
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:14:git-annex unused/dropunused
+Cases: 30  Tried: 29  Errors: 17  Failures: 2chmod: -R: No such file or directory
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+.t/tmprepo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+Cases: 30  Tried: 30  Errors: 18  Failures: 2
+chmod: -R: No such file or directory
+test: .t/repo/.git/annex/objects/WORM:1297194705:20:foo/WORM:1297194705:20:foo: removeLink: permission denied (Permission denied)
+make: *** [test] Error 1
+

+
+I assumed that since the tests built, then running them shouldn't be a problem. It looks like some argument isn't being passed about for the location of the .t directory that gets created. I will check the dependancies on my system again.
+"""]]

From 97d5e23430748487d9b6dea842cdba4e66ba39d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 17:46:52 -0400
Subject: [PATCH 0834/8313] map bugfix

Need to find the absolute repo path before looking up the full info for the
repo. Otherwise, it doesn't find the right full info.
---
 Command/Map.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index d8dd0e94cf..1b15e34f6a 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -107,10 +107,10 @@ node umap fullinfo r = unlines $ n:edges
 {- An edge between two repos. The second repo is a remote of the first. -}
 edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
 edge umap fullinfo from to =
-	Dot.graphEdge (nodeId from) (nodeId $ absRepo from fullto) edgename
+	Dot.graphEdge (nodeId from) (nodeId fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
-		fullto = findfullinfo to
+		fullto = findfullinfo (absRepo from to)
 		findfullinfo n =
 			case (filter (same n) fullinfo) of
 				[] -> n

From 3ae654254da0449c5ddc4fe16600b76c82e828fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 17:52:32 -0400
Subject: [PATCH 0835/8313] make remotes absolute while spidering

---
 Command/Map.hs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 1b15e34f6a..05c03feaf0 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -110,7 +110,7 @@ edge umap fullinfo from to =
 	Dot.graphEdge (nodeId from) (nodeId fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
-		fullto = findfullinfo (absRepo from to)
+		fullto = findfullinfo to
 		findfullinfo n =
 			case (filter (same n) fullinfo) of
 				[] -> n
@@ -140,8 +140,13 @@ spider' (r:rs) known
 	| any (same r) known = spider' rs known
 	| otherwise = do
 		r' <- scan r
+
+		-- The remotes will be relative to r', and need to be
+		-- made absolute for later use.
 		let remotes = map (absRepo r') (Git.remotes r')
-		spider' (rs ++ remotes) (r':known)
+		let r'' = Git.remotesAdd r' remotes
+
+		spider' (rs ++ remotes) (r'':known)
 
 absRepo :: Git.Repo -> Git.Repo -> Git.Repo
 absRepo reference r

From c0ec5a35db54f45b14971ae61b42ccc64cb92f62 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 18:04:19 -0400
Subject: [PATCH 0836/8313] show trusted repos in green

---
 Command/Map.hs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 05c03feaf0..05fa258857 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -21,6 +21,7 @@ import Messages
 import Types
 import Utility
 import UUID
+import Trust
 import qualified Dot
 
 -- a link from the first repository to the second (its remote)
@@ -38,8 +39,9 @@ start = do
 	rs <- spider g
 
 	umap <- uuidMap
+	trusted <- trustGet Trusted
 
-	liftIO $ writeFile file (drawMap rs umap)
+	liftIO $ writeFile file (drawMap rs umap trusted)
 	showLongNote $ "running: dot -Tx11 " ++ file
 	showProgress
 	r <- liftIO $ boolSystem "dot" ["-Tx11", file]
@@ -56,14 +58,15 @@ start = do
  - the repositories first, followed by uuids that were not matched
  - to a repository.
  -}
-drawMap :: [Git.Repo] -> (M.Map UUID String) -> String
-drawMap rs umap = Dot.graph $ repos ++ others
+drawMap :: [Git.Repo] -> (M.Map UUID String) -> [UUID] -> String
+drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 	where
 		repos = map (node umap rs) rs
-		ruuids = map getUncachedUUID rs
-		others = map uuidnode $ filter (`notElem` ruuids) (M.keys umap)
-		uuidnode u = unreachable $
-			Dot.graphNode u $ M.findWithDefault "" u umap
+		ruuids = ts ++ map getUncachedUUID rs
+		others = map (unreachable . uuidnode) $
+			filter (`notElem` ruuids) (M.keys umap)
+		trusted = map (trustworthy . uuidnode) ts
+		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
@@ -130,6 +133,8 @@ unreachable :: String -> String
 unreachable = Dot.fillColor "red"
 reachable :: String -> String
 reachable = Dot.fillColor "white"
+trustworthy :: String -> String
+trustworthy = Dot.fillColor "green"
 
 {- Recursively searches out remotes starting with the specified repo. -}
 spider :: Git.Repo -> Annex [Git.Repo]

From c1b69d1511cd9a6d63981f74f6d926d59dba7c8c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 18:17:46 -0400
Subject: [PATCH 0837/8313] fill color for host boxes

---
 Command/Map.hs |  2 +-
 Dot.hs         | 11 ++++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 05fa258857..74005b521d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -100,7 +100,7 @@ nodeId r =
 node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
-		n = Dot.subGraph (hostname r) (basehostname r) $
+		n = Dot.subGraph (hostname r) (basehostname r) "grey" $
 			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
 		decorate
diff --git a/Dot.hs b/Dot.hs
index fcd0c19cc2..a21d705365 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -41,13 +41,18 @@ fillColor :: String -> String -> String
 fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
 
 {- apply to graphNode to put the node in a labeled box -}
-subGraph :: String -> String -> String -> String
-subGraph subid l s =
-	"subgraph " ++ name ++ " {\n" ++ ii setlabel ++ ii s ++ indent "}"
+subGraph :: String -> String -> String -> String -> String
+subGraph subid l color s =
+	"subgraph " ++ name ++ " {\n" ++
+		ii setlabel ++
+		ii setcolor ++
+		ii s ++
+		indent "}"
 	where
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
 		setlabel = "label=" ++ quote l
+		setcolor = "fillcolor=" ++ quote color
 		ii x = (indent $ indent x) ++ "\n"
 
 indent ::String -> String

From 81e045a539e5877d445abaa53783f94f4e17513a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 18:26:38 -0400
Subject: [PATCH 0838/8313] tweak

---
 Command/Map.hs | 2 +-
 Dot.hs         | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 74005b521d..0a3bb9fff0 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -100,7 +100,7 @@ nodeId r =
 node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
-		n = Dot.subGraph (hostname r) (basehostname r) "grey" $
+		n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $
 			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
 		decorate
diff --git a/Dot.hs b/Dot.hs
index a21d705365..592b21f691 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -45,6 +45,7 @@ subGraph :: String -> String -> String -> String -> String
 subGraph subid l color s =
 	"subgraph " ++ name ++ " {\n" ++
 		ii setlabel ++
+		ii setfilled ++
 		ii setcolor ++
 		ii s ++
 		indent "}"
@@ -52,6 +53,7 @@ subGraph subid l color s =
 		-- the "cluster_" makes dot draw a box
 		name = quote ("cluster_" ++ subid)
 		setlabel = "label=" ++ quote l
+		setfilled = "style=" ++ quote "filled"
 		setcolor = "fillcolor=" ++ quote color
 		ii x = (indent $ indent x) ++ "\n"
 

From be8b42b40dba9f6ffebf4c3818fefb8dc5464137 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 8 Feb 2011 23:20:08 +0000
Subject: [PATCH 0839/8313] Added a comment

---
 .../comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment b/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment
new file mode 100644
index 0000000000..afc3088d4f
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-02-08T23:20:08Z"
+ content="""
+You're missing the sha1sum command, everything else is a followon error from that. Added a hint about this to [[install]],
+and in the next version configure will check for sha1sum.
+"""]]

From dd90d4a70e6f8e155902cd40dc21c138d4e1282c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 19:31:27 -0400
Subject: [PATCH 0840/8313] configure: Check for sha1sum.

---
 configure.hs     | 1 +
 debian/changelog | 5 +++--
 doc/install.mdwn | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/configure.hs b/configure.hs
index 8d1c117a71..1451d7eaa3 100644
--- a/configure.hs
+++ b/configure.hs
@@ -10,6 +10,7 @@ tests = [
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
 	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
+	, TestCase "sha1sum" $ requireCmd "sha1sum" "sha1sum /dev/null"
 	]
diff --git a/debian/changelog b/debian/changelog
index 3a5f60fefe..b8d27f77e4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20) UNRELEASED; urgency=low
+git-annex (0.20) unstable; urgency=low
 
   * Preserve specified file ordering when instructed to act on multiple
     files or directories. For example, "git annex get a b" will now always
@@ -9,8 +9,9 @@ git-annex (0.20) UNRELEASED; urgency=low
   * map: New subcommand that uses graphviz to display a nice map of
     the git repository network.
   * Deal with the mtl/monads-fd conflict.
+  * configure: Check for sha1sum.
 
- -- Joey Hess   Mon, 31 Jan 2011 20:06:02 -0400
+ -- Joey Hess   Tue, 08 Feb 2011 18:57:24 -0400
 
 git-annex (0.19) unstable; urgency=low
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 3c7025fa60..fc9f28ae7b 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -8,6 +8,7 @@ To build and use git-annex, you will need:
   (or uuidgen from util-linux)
 * `xargs`: 
 * `rsync`: 
+* `sha1sum`: 
 * Then just [[download]] git-annex and run: `make; make install`
 
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,

From de1e33010232a8abad59a54758ec00808caa8a5c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Feb 2011 19:32:57 -0400
Subject: [PATCH 0841/8313] add news item for git-annex 0.20

---
 doc/news/version_0.15.mdwn | 15 ---------------
 doc/news/version_0.20.mdwn | 12 ++++++++++++
 2 files changed, 12 insertions(+), 15 deletions(-)
 delete mode 100644 doc/news/version_0.15.mdwn
 create mode 100644 doc/news/version_0.20.mdwn

diff --git a/doc/news/version_0.15.mdwn b/doc/news/version_0.15.mdwn
deleted file mode 100644
index c3d4d2c0f7..0000000000
--- a/doc/news/version_0.15.mdwn
+++ /dev/null
@@ -1,15 +0,0 @@
-git-annex 0.15 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Support scp-style urls for remotes (host:path).
-   * Support ssh urls containing "~".
-   * Add trust and untrust subcommands, to allow configuring repositories
-     that are trusted to retain files without explicit checking.
-   * Fix bug in numcopies handling when multiple remotes pointed to the
-     same repository.
-   * Introduce the git-annex-shell command. It's now possible to make
-     a user have it as a restricted login shell, similar to git-shell.
-   * Note that git-annex will always use git-annex-shell when accessing
-     a ssh remote, so all of your remotes need to be upgraded to this
-     version of git-annex at the same time.
-   * Now rsync is exclusively used for copying files to and from remotes.
-     scp is not longer supported."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20.mdwn b/doc/news/version_0.20.mdwn
new file mode 100644
index 0000000000..9b95b652e5
--- /dev/null
+++ b/doc/news/version_0.20.mdwn
@@ -0,0 +1,12 @@
+git-annex 0.20 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Preserve specified file ordering when instructed to act on multiple
+     files or directories. For example, "git annex get a b" will now always
+     get "a" before "b". Previously it could operate in either order.
+   * unannex: Commit staged changes at end, to avoid some confusing behavior
+     with the pre-commit hook, which would see some types of commits after
+     an unannex as checking in of an unlocked file.
+   * map: New subcommand that uses graphviz to display a nice map of
+     the git repository network.
+   * Deal with the mtl/monads-fd conflict.
+   * configure: Check for sha1sum."""]]
\ No newline at end of file

From 01a45b6be4063eddb980b87c816281f952338482 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 00:45:32 +0000
Subject: [PATCH 0842/8313] Added a comment

---
 ..._237a137cce58a28abcc736cbf2c420b0._comment | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment b/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment
new file mode 100644
index 0000000000..8d8aefcb20
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 7"
+ date="2011-02-09T00:45:31Z"
+ content="""
+That's odd, I have the md5sha1sum package installed and it still fails with pretty much the same error
+
+
+Testing 1:blackbox:0:git-annex init
+Cases: 30  Tried: 7  Errors: 0  Failures: 0chmod: -R: No such file or directory
+### Error in:   1:blackbox:0:git-annex init
+.t/repo/.git/annex/objects/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa: removeLink: permission denied (Permission denied)
+Testing 1:blackbox:1:git-annex add:0
+### Error in:   1:blackbox:1:git-annex add:0
+foo: openFile: permission denied (Permission denied)
+
+< and so on >
+

+
+the configure script finds sha1sum, builds and starts to run.
+"""]]

From 7a0826293e0ac6c0000f49a1618c1c613b909aa1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Feb 2011 00:12:45 -0400
Subject: [PATCH 0843/8313] test: Don't rely on chmod -R working.

Sometimes I wish I could give all my users a real OS.
---
 debian/changelog | 6 ++++++
 test.hs          | 8 +++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index b8d27f77e4..52a749178f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.21) UNRELEASED; urgency=low
+
+  * test: Don't rely on chmod -R working.
+
+ -- Joey Hess   Wed, 09 Feb 2011 00:12:11 -0400
+
 git-annex (0.20) unstable; urgency=low
 
   * Preserve specified file ordering when instructed to act on multiple
diff --git a/test.hs b/test.hs
index f7d3f7c53d..b68360e9dd 100644
--- a/test.hs
+++ b/test.hs
@@ -20,6 +20,8 @@ import Control.Exception (throw)
 import Control.Monad.State (liftIO)
 import Maybe
 import qualified Data.Map as M
+import System.Path (recurseDir)
+import System.IO.HVFS (SystemFS(..))
 
 import qualified Annex
 import qualified BackendList
@@ -536,8 +538,12 @@ cleanup dir = do
 	when e $ do
 		-- git-annex prevents annexed file content from being
 		-- removed via permissions bits; undo
-		_ <- Utility.boolSystem "chmod" ["+rw", "-R", dir]
+		recurseDir SystemFS dir >>= mapM_ fixmodes
 		removeDirectoryRecursive dir
+	where
+		fixmodes f = do
+			s <- getSymbolicLinkStatus f
+			unless (isSymbolicLink s) $ Content.allowWrite f
 	
 checklink :: FilePath -> Assertion
 checklink f = do

From c1bc8f31c50bc9d4dea180346adc874629c76b24 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 9 Feb 2011 04:10:27 +0000
Subject: [PATCH 0844/8313] Added a comment

---
 .../comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment b/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment
new file mode 100644
index 0000000000..9401bd453e
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-02-09T04:10:27Z"
+ content="""
+The chmod errors are because your chmod does not understand the -R argument. Only the test suite uses chmod -R. I've fixed it to modify modes manually.
+"""]]

From 2bd16f84ae3c6a26bff5eb5e4b39221577191def Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Feb 2011 00:25:22 -0400
Subject: [PATCH 0845/8313] more install notes

---
 doc/install.mdwn | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index fc9f28ae7b..f1305777c0 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -1,3 +1,11 @@
+## OS-specific instructions
+
+* [[OSX]]
+* [[Debian]]
+* [[Ubuntu]]
+
+## Generic instructions
+
 To build and use git-annex, you will need:
 
 * `git`: 
@@ -14,8 +22,7 @@ To build and use git-annex, you will need:
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
 but that will be skipped if it is not installed.)
 
-OS-specific instructions:
+Additionally, to run the test suite (via `make test`), you will need:
 
-* [[OSX]]
-* [[Debian]]
-* [[Ubuntu]]
+* `TestPack` 
+* `QuickCheck` 2 

From 62ebeb00d9f079f079c38798f8496dd666ae1fd5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Feb 2011 01:01:06 -0400
Subject: [PATCH 0846/8313] simpler approach

---
 test.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/test.hs b/test.hs
index b68360e9dd..b32a4abb04 100644
--- a/test.hs
+++ b/test.hs
@@ -11,7 +11,7 @@ import System.Directory
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
 import IO (bracket_, bracket)
-import Control.Monad (unless, when)
+import Control.Monad (unless, when, filterM)
 import Data.List
 import System.IO.Error
 import System.Posix.Env
@@ -537,13 +537,11 @@ cleanup dir = do
 	e <- doesDirectoryExist dir
 	when e $ do
 		-- git-annex prevents annexed file content from being
-		-- removed via permissions bits; undo
-		recurseDir SystemFS dir >>= mapM_ fixmodes
+		-- removed via directory permissions; undo
+		recurseDir SystemFS dir >>=
+			filterM doesDirectoryExist >>=
+			mapM_ Content.allowWrite
 		removeDirectoryRecursive dir
-	where
-		fixmodes f = do
-			s <- getSymbolicLinkStatus f
-			unless (isSymbolicLink s) $ Content.allowWrite f
 	
 checklink :: FilePath -> Assertion
 checklink f = do

From 0fb84912b611e03c47ebd4a3cdd3008dbb5fe417 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 09:09:44 +0000
Subject: [PATCH 0847/8313]

---
 ..._is_no_global_.gitconfig_for_the_user.mdwn | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn

diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
new file mode 100644
index 0000000000..2a26a67267
--- /dev/null
+++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
@@ -0,0 +1,25 @@
+Make test fails when git doesn't know what identity to give to commits
+
+
+
+Testing 1:blackbox:0:git-annex init
+Cases: 30  Tried: 7  Errors: 0  Failures: 0
+*** Please tell me who you are.
+
+Run
+
+  git config --global user.email "you@example.com"
+  git config --global user.name "Your Name"
+
+to set your account's default identity.
+Omit --global to set the identity only in this repository.
+
+fatal: empty ident   not allowed
+### Failure in: 1:blackbox:0:git-annex init
+init failed
+Testing 1:blackbox:1:git-annex add:0
+Cases: 30  Tried: 8  Errors: 0  Failures: 1
+*** Please tell me who you are.
+

+
+I guess most users or people testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.

From 23ab9a228d31543a8f2ad8084d73dbc489b90054 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 09:12:53 +0000
Subject: [PATCH 0848/8313] Added a comment

---
 ..._cc283b485b3c95ba7eebc8f0c96969b3._comment | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment

diff --git a/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment b/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment
new file mode 100644
index 0000000000..da6d7ca178
--- /dev/null
+++ b/doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment
@@ -0,0 +1,66 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 9"
+ date="2011-02-09T09:12:52Z"
+ content="""
+[a0826293][] fixed the last problem, there is coreutils available in macports, if they are installed you get the gnu equivalents but they are prefixed with a g (e.g. gchmod instead of chmod), I guess not everyone will have these install or prefer these on [[install/OSX]]
+
+Some more tests fail now...
+
+
+Testing 1:blackbox:3:git-annex unannex:1:with content
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Testing 1:blackbox:4:git-annex drop:1:with remote
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote
+Testing 1:blackbox:5:git-annex get
+Testing 1:blackbox:6:git-annex move
+Testing 1:blackbox:7:git-annex copy
+### Failure in: 1:blackbox:7:git-annex copy
+move --to of file already there failed
+Testing 1:blackbox:8:git-annex unlock/lock
+### Error in:   1:blackbox:8:git-annex unlock/lock
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:9:git-annex edit/commit:0
+### Error in:   1:blackbox:9:git-annex edit/commit:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:9:git-annex edit/commit:1
+### Error in:   1:blackbox:9:git-annex edit/commit:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:10:git-annex fix
+### Error in:   1:blackbox:10:git-annex fix
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust
+### Error in:   1:blackbox:11:git-annex trust/untrust/semitrust
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:0
+### Error in:   1:blackbox:12:git-annex fsck:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:1
+### Error in:   1:blackbox:12:git-annex fsck:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:12:git-annex fsck:2
+### Error in:   1:blackbox:12:git-annex fsck:2
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:13:git-annex migrate:0
+### Error in:   1:blackbox:13:git-annex migrate:0
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:13:git-annex migrate:1
+### Error in:   1:blackbox:13:git-annex migrate:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:14:git-annex unused/dropunused
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Cases: 30  Tried: 30  Errors: 11  Failures: 3
+test: failed
+make: *** [test] Error 1
+

+
+On a side note, I think I found another bug in the testing. I had tested in a virtual machine in archlinux (a very recent updated version) Please see the report here [[tests fail when there is no global .gitconfig for the user]]
+
+[a0826293]: http://git.kitenet.net/?p=git-annex;a=commit;h=7a0826293e0ac6c0000f49a1618c1c613b909aa1
+"""]]

From 077b46abc08ca93ef3cfbd1bd23aed8ebc3494bc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 09:13:37 +0000
Subject: [PATCH 0849/8313]

---
 ...ts_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
index 2a26a67267..855f19a67e 100644
--- a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
+++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
@@ -22,4 +22,4 @@ Cases: 30  Tried: 8  Errors: 0  Failures: 1
 *** Please tell me who you are.
 

 
-I guess most users or people testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.
+I guess most users testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop.

From fdb2896958438e1069fa92e69c478f91b182b414 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 9 Feb 2011 10:29:19 +0000
Subject: [PATCH 0850/8313]

---
 ..._is_no_global_.gitconfig_for_the_user.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
index 855f19a67e..e9dda99167 100644
--- a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
+++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn
@@ -23,3 +23,26 @@ Cases: 30  Tried: 8  Errors: 0  Failures: 1
I guess most users testing git-annex probably have a .gitconfig sitting in their home directories already so the above never cropped up. This failure was initially found in a clean and fresh install of a virtual machine with archlinux and repeated again on my archlinux laptop. + +Update: I pulled the master on my rhel5 test machine and moved my .gitconfig out of the way, the tests passes and continues but I still get a "warning message" from git. + +
+Testing 1:blackbox:3:git-annex unannex:1:with content                         
+Cases: 30  Tried: 12  Errors: 0  Failures: 0[master fce0cde] content removed from git annex
+ Committer: Jimmy Tang 
+Your name and email address were configured automatically based
+on your username and hostname. Please check that they are accurate.
+You can suppress this message by setting them explicitly:
+
+    git config --global user.name "Your Name"
+    git config --global user.email you@example.com
+
+After doing this, you may fix the identity used for this commit with:
+
+    git commit --amend --reset-author
+
+ 2 files changed, 1 insertions(+), 2 deletions(-)
+ delete mode 120000 foo
+
+ +I guess it also depends a bit on how git figures out who it is is committing and how the machine in question is configured with hostnames and domain names. From 35a65068123f0215929e06e98bf4a6a67d2ae7dd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Feb 2011 11:02:21 -0400 Subject: [PATCH 0851/8313] unannex: Fix recently introduced bug when attempting to unannex more than one file at a time. --- Command/Unannex.hs | 13 ++++++++----- debian/changelog | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Command/Unannex.hs b/Command/Unannex.hs index c663b29ab0..19cb1624e5 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -33,10 +33,13 @@ start file = isAnnexed file $ \(key, backend) -> do ishere <- inAnnex key if ishere then do - g <- Annex.gitRepo - staged <- liftIO $ Git.stagedFiles g [Git.workTree g] - unless (null staged) $ - error "This command cannot be run when there are already files staged for commit." + force <- Annex.getState Annex.force + unless force $ do + g <- Annex.gitRepo + staged <- liftIO $ Git.stagedFiles g [Git.workTree g] + unless (null staged) $ + error "This command cannot be run when there are already files staged for commit." + Annex.changeState $ \s -> s { Annex.force = True } showStart "unannex" file return $ Just $ perform file key backend @@ -65,6 +68,6 @@ cleanup file key = do -- Commit staged changes at end to avoid confusing the -- pre-commit hook if this file is later added back to -- git as a normal, non-annexed file. - Annex.queue "commit" ["-m", "content removed from git annex"] "--" + Annex.queue "commit" ["-m", "content removed from git annex"] "-a" return True diff --git a/debian/changelog b/debian/changelog index 52a749178f..072e5fc232 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.21) UNRELEASED; urgency=low * test: Don't rely on chmod -R working. + * unannex: Fix recently introduced bug when attempting to unannex more + than one file at a time. -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 From df2312c165d5b019e761bbd80c062f6ff4629618 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 9 Feb 2011 15:04:50 +0000 Subject: [PATCH 0852/8313] Added a comment --- .../comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment b/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment new file mode 100644 index 0000000000..95a9773e2b --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-02-09T15:04:50Z" + content=""" +I don't know what these problems forking could be. Can you strace it? +"""]] From 82bc10d5eafa76ce06b294382e1597d67b06e911 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Feb 2011 11:17:26 -0400 Subject: [PATCH 0853/8313] test: Set git user name and email in case git can't guess values. --- debian/changelog | 1 + ...s_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn | 2 ++ test.hs | 3 +++ 3 files changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 072e5fc232..2e5b97c0f9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.21) UNRELEASED; urgency=low * test: Don't rely on chmod -R working. * unannex: Fix recently introduced bug when attempting to unannex more than one file at a time. + * test: Set git user name and email in case git can't guess values. -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 diff --git a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn index e9dda99167..b90b501e31 100644 --- a/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn +++ b/doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn @@ -46,3 +46,5 @@ After doing this, you may fix the identity used for this commit with: I guess it also depends a bit on how git figures out who it is is committing and how the machine in question is configured with hostnames and domain names. + +> Fixed that. [[done]] --[[Joey]] diff --git a/test.hs b/test.hs index b32a4abb04..309b541762 100644 --- a/test.hs +++ b/test.hs @@ -508,6 +508,9 @@ setuprepo dir = do cleanup dir ensuretmpdir Utility.boolSystem "git" ["init", "-q", dir] @? "git init failed" + indir dir $ do + Utility.boolSystem "git" ["config", "user.name", "Test User"] @? "git config failed" + Utility.boolSystem "git" ["config", "user.email", "test@example.com"] @? "git config failed" return dir copyrepo :: FilePath -> FilePath -> IO FilePath From cf27ae27532b525b41a1d79f2d6888fa6d5f7f97 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Feb 2011 11:19:19 -0400 Subject: [PATCH 0854/8313] regression test for multi-file unannex --- test.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.hs b/test.hs index 309b541762..1951b44260 100644 --- a/test.hs +++ b/test.hs @@ -139,7 +139,7 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] annexed_notpresent annexedfile withcopy = "with content" ~: intmpcopyrepo $ do annexed_present annexedfile - git_annex "unannex" ["-q", annexedfile] @? "unannex failed" + git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed" unannexed annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file" unannexed annexedfile From 8cc289dca5fc89cc81a2c84531fffa3442a9a48b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 9 Feb 2011 19:35:48 +0000 Subject: [PATCH 0855/8313] Added a comment --- .../comment_11_56f1143fa191361d63b441741699e17f._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment b/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment new file mode 100644 index 0000000000..3fbe57ecd5 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 11" + date="2011-02-09T19:35:47Z" + content=""" +I got dtruss to give me a trace, the output is quite big to post here (~560kb gzip'd), do you mind if I emailed it or posted it somewhere else for you? +"""]] From 0885c886d05c9ce885096c43121a96f9a2e4c2aa Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 9 Feb 2011 19:47:30 +0000 Subject: [PATCH 0856/8313] Added a comment --- .../comment_12_ec5131624d0d2285d3b6880e47033f97._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment b/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment new file mode 100644 index 0000000000..beba5dc42c --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-02-09T19:47:30Z" + content=""" +joey@kitenet.net (hope I can make sense of dtruss output) +"""]] From fe84d45423b51ec69f7e82ebcff1947e278a14f5 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 9 Feb 2011 21:59:47 +0000 Subject: [PATCH 0857/8313] Added a comment --- ..._13_88ed095a448096bf8a69015a04e64df1._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment b/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment new file mode 100644 index 0000000000..dd25c3d0cb --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 13" + date="2011-02-09T21:59:47Z" + content=""" +The dtrace puzzlingly does not have the same errors shown above, but a set of mostly new errors. I don't know what to make of that. + +> git-annex: git-annex/.t/repo/.git/hooks/pre-commit: fileAccess: permission denied (Operation not permitted) + +This seems to be caused by it setting the execute bit on the file. I don't know why that would fail; it's just written the file and renamed it into place so clearly should be able to write to it. + +> was able to modify annexed file's sha1foo content + +This also suggests something breaking with permissions. +"""]] From fe55b4644e67bba60b35e07abcdd312b65c9d6f3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 14:21:44 -0400 Subject: [PATCH 0858/8313] Fix display of unicode filenames. Internally, the filenames are stored as un-decoded unicode. I tried decoding them, but then haskell tries to access the wrong files. Hmm. So, I've unhappily chosen option "B", which is to decode filenames before they are displayed. --- Backend/File.hs | 6 ++--- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- Command/Find.hs | 3 ++- Command/PreCommit.hs | 2 +- Command/Unused.hs | 2 +- Content.hs | 2 +- Messages.hs | 9 +++++-- debian/changelog | 1 + doc/bugs/problems_with_utf8_names.mdwn | 22 ++++++++------- doc/bugs/unhappy_without_UTF8_locale.mdwn | 33 +++++++++++++++++++++++ 11 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 doc/bugs/unhappy_without_UTF8_locale.mdwn diff --git a/Backend/File.hs b/Backend/File.hs index d76cd29391..fca385a1e9 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -193,14 +193,14 @@ checkKeyNumCopies key file numcopies = do missingNote :: String -> Int -> Int -> String -> String missingNote file 0 _ [] = - "** No known copies of " ++ file ++ " exist!" + "** No known copies of " ++ showFile file ++ " exist!" missingNote file 0 _ untrusted = - "Only these untrusted locations may have copies of " ++ file ++ + "Only these untrusted locations may have copies of " ++ showFile file ++ "\n" ++ untrusted ++ "Back it up to trusted locations with git-annex copy." missingNote file present needed [] = "Only " ++ show present ++ " of " ++ show needed ++ - " trustworthy copies of " ++ file ++ " exist." ++ + " trustworthy copies of " ++ showFile file ++ " exist." ++ "\nBack it up with git-annex copy." missingNote file present needed untrusted = missingNote file present needed [] ++ diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 3d868dbd19..f1092492ed 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -58,5 +58,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - warning $ "Bad file content; moved to "++dest + warning $ "Bad file content; moved to " ++ showFile dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 20a81d8411..7f40a2acb5 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -67,5 +67,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - warning $ "Bad file size; moved to "++dest + warning $ "Bad file size; moved to " ++ showFile dest return False diff --git a/Command/Find.hs b/Command/Find.hs index 3ed15c1537..45156af051 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -12,6 +12,7 @@ import Control.Monad.State (liftIO) import Command import Content +import Messages command :: [Command] command = [Command "find" (paramOptional $ paramRepeating paramPath) seek @@ -24,5 +25,5 @@ seek = [withFilesInGit start] start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when exists $ liftIO $ putStrLn file + when exists $ liftIO $ putStrLn $ showFile file return Nothing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 12e5ed806d..f22300a030 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -32,7 +32,7 @@ perform pair@(file, _) = do ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file - else error $ "failed to add " ++ file ++ "; canceling commit" + else error $ "failed to add " ++ showFile file ++ "; canceling commit" cleanup :: FilePath -> CommandCleanup cleanup file = do diff --git a/Command/Unused.hs b/Command/Unused.hs index d9f4e39783..2b390b9563 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -68,7 +68,7 @@ checkUnused = do dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] table l = [" NUMBER KEY"] ++ map cols l - cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (showFile . show) k pad n s = s ++ replicate (n - length s) ' ' number :: Int -> [a] -> [(Int, a)] diff --git a/Content.hs b/Content.hs index e16ad883c2..fae980bae0 100644 --- a/Content.hs +++ b/Content.hs @@ -49,7 +49,7 @@ calcGitLink file key = do cwd <- liftIO $ getCurrentDirectory let absfile = case absNormPath cwd file of Just f -> f - Nothing -> error $ "unable to normalize " ++ file + Nothing -> error $ "unable to normalize " ++ showFile file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ annexLocation key diff --git a/Messages.hs b/Messages.hs index 6f4ec1e626..12a836d3c5 100644 --- a/Messages.hs +++ b/Messages.hs @@ -11,6 +11,7 @@ import Control.Monad.State (liftIO) import System.IO import Control.Monad (unless) import Data.String.Utils +import Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex @@ -25,7 +26,7 @@ showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" showStart :: String -> String -> Annex () showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ file ++ " " + liftIO $ putStr $ command ++ " " ++ showFile file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () @@ -45,7 +46,6 @@ showEndOk = verbose $ liftIO $ putStrLn "ok" showEndFail :: Annex () showEndFail = verbose $ liftIO $ putStrLn "\nfailed" -{- Exception pretty-printing. -} showErr :: (Show a) => a -> Annex () showErr e = warning $ "git-annex: " ++ show e @@ -57,3 +57,8 @@ warning w = do indent :: String -> String indent s = join "\n" $ map (\l -> " " ++ l) $ lines s + +{- Prepares a filename for display. This is needed because strings are + - internally represented in git-annex is non-decoded form. -} +showFile :: String -> String +showFile = decodeString diff --git a/debian/changelog b/debian/changelog index 2e5b97c0f9..277869b020 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (0.21) UNRELEASED; urgency=low * unannex: Fix recently introduced bug when attempting to unannex more than one file at a time. * test: Set git user name and email in case git can't guess values. + * Fix display of unicode filenames. -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 30f3495f40..257f8dff21 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -37,10 +37,22 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > encoded in utf-8 (an archive could have historical filenames using > varying encodings), and you don't want which files are accessed to > depend on locale settings. +> > I tried to do this by making parts of GitRepo call +> > Codec.Binary.UTF8.String.decodeString when reading filenames from +> > git. This seemed to break attempts to operate on the files, +> > weirdly encoded strings were seen in syscalls in strace. > 1. Keep input and internal data un-decoded, but decode it when > outputting a filename (assuming the filename is encoded using the > user's configured encoding), and allow haskell's output encoding to then > encode it according to the user's locale configuration. +> > This is now [[implemented|done]]. I'm not very happy that I have to watch +> > out for any place that a filename is output and call `showFile` +> > on it, but there are really not too many such places in git-annex. +> > +> > Note that this only affects filenames apparently. +> > (Names of files in the annex, and also some places where names +> > of keys are displayed.) Utf-8 in the uuid.map file etc seems +> > to be handled cleanly. > 1. Avoid encodings entirely. Mostly what I'm doing now; probably > could find a way to disable encoding of console output. Then the raw > filename would be displayed, which should work ok. git-annex does @@ -50,13 +62,3 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > One other possible > issue would be that this could cause problems if git-annex were > translated. -> -> BTW, for more fun, try unsetting LANG, and then you can see -> stuff like this: - - joey@gnu:~/tmp/aa>git annex add ./Üa - add add add add git-annex: : commitAndReleaseBuffer: invalid - argument (Invalid or incomplete multibyte or wide character) - -> (Add -q to work around this; once it doesn't need to print the filename, -> it can act on it ok!) diff --git a/doc/bugs/unhappy_without_UTF8_locale.mdwn b/doc/bugs/unhappy_without_UTF8_locale.mdwn new file mode 100644 index 0000000000..6f1df4fab7 --- /dev/null +++ b/doc/bugs/unhappy_without_UTF8_locale.mdwn @@ -0,0 +1,33 @@ +Try unsetting LANG and passing git-annex unicode filenames. + + joey@gnu:~/tmp/aa>git annex add ./Üa + add add add add git-annex: : commitAndReleaseBuffer: invalid + argument (Invalid or incomplete multibyte or wide character) + +The same problem can be seen with a simple haskell program: + + import System.Environment + import Codec.Binary.UTF8.String + main = do + args <- getArgs + putStrLn $ decodeString $ args !! 0 + + joey@gnu:~/src/git-annex>LANG= runghc ~/foo.hs Ü + foo.hs: : hPutChar: invalid argument (Invalid or incomplete multibyte or wide character) + +(The call to `decodeString` is necessary to make the input +unicode string be displayed properly in a utf8 locale, but +does not contribute to this problem.) + +I guess that haskell is setting the IO encoding to latin1, which +is [documented](http://haskell.org/ghc/docs/latest/html/libraries/base/System-IO.html#v:latin1) +to error out on characters > 255. + +So this program doesn't have the problem -- but may output garbage +on non-utf-8 capable terminals: + + import System.IO + main = do + hSetEncoding stdout utf8 + args <- getArgs + putStrLn $ decodeString $ args !! 0 From 52fa424faef63690d5b44cc4b6fd11aa67f1e619 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 14:45:35 -0400 Subject: [PATCH 0859/8313] update --- doc/bugs/problems_with_utf8_names.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 257f8dff21..58eb5ad1c0 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -58,7 +58,8 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > filename would be displayed, which should work ok. git-annex does > not really need to pull apart filenames; they are almost entirely > opaque blobs. I guess that the `--exclude` option is the exception -> to that, but it is currently not unicode safe anyway. +> to that, but it is currently not unicode safe anyway. (Update: tried +> `--exclude` again, seems it is unicode clean..) > One other possible > issue would be that this could cause problems if git-annex were > translated. From 0fef480bcaee9ae87f17d008a3787871d3bc9786 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 14:58:09 -0400 Subject: [PATCH 0860/8313] update --- doc/bugs/problems_with_utf8_names.mdwn | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 58eb5ad1c0..29213e1ec9 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -63,3 +63,39 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > One other possible > issue would be that this could cause problems if git-annex were > translated. + +---- + +Simpler test case: + +
+import Codec.Binary.UTF8.String
+import System.Environment
+
+main = do
+        args <- getArgs
+        let file = decodeString $ head args
+        putStrLn $ "file is: " ++ file
+        putStr =<< readFile file
+
+ +If I pass this a filename like 'ü', it will fail, and notice +the bad encoding of the filename in the error message: + +
+$ echo hi > ü; runghc foo.hs ü
+file is: ü
+foo.hs: �: openFile: does not exist (No such file or directory)
+
+ +On the other hand, if I remove the decodeString, it prints the filename +wrong, while accessing it right: + +
+$ runghc foo.hs ü
+file is: üa
+hi
+
+ +The only way that seems to consistently work is to delay decoding the +filename to places where it's output. But then it's easy to miss some. From 75e507d0af707f9773f4b93c17c76c5161d2a195 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 10 Feb 2011 16:53:35 -0400 Subject: [PATCH 0861/8313] foo --- Messages.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Messages.hs b/Messages.hs index 12a836d3c5..ab5992a6c6 100644 --- a/Messages.hs +++ b/Messages.hs @@ -60,5 +60,5 @@ indent s = join "\n" $ map (\l -> " " ++ l) $ lines s {- Prepares a filename for display. This is needed because strings are - internally represented in git-annex is non-decoded form. -} -showFile :: String -> String +showFile :: FilePath -> String showFile = decodeString From 285fb2bb08c7da534c111ebfeee5911e850570cc Mon Sep 17 00:00:00 2001 From: Michael Kenney Date: Thu, 10 Feb 2011 15:45:34 -0800 Subject: [PATCH 0862/8313] Fixed missing import of Messages module --- Command/PreCommit.hs | 1 + Content.hs | 1 + 2 files changed, 2 insertions(+) diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index f22300a030..76aab3855a 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -14,6 +14,7 @@ import qualified Annex import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix +import Messages command :: [Command] command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"] diff --git a/Content.hs b/Content.hs index fae980bae0..188a38787e 100644 --- a/Content.hs +++ b/Content.hs @@ -34,6 +34,7 @@ import UUID import qualified GitRepo as Git import qualified Annex import Utility +import Messages {- Checks if a given key is currently present in the gitAnnexLocation. -} inAnnex :: Key -> Annex Bool From 5a50a7cf137997a9d940b9a89a0968452a1ac411 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 15:37:37 -0400 Subject: [PATCH 0863/8313] update unicode FilePath handling Based on http://hackage.haskell.org/trac/ghc/ticket/3307 , whether FilePath contains decoded unicode varies by OS. So, add a configure check for it. Also, renamed showFile to filePathToString --- Backend/File.hs | 6 +++--- Backend/SHA1.hs | 2 +- Backend/WORM.hs | 2 +- Command/Find.hs | 2 +- Command/PreCommit.hs | 2 +- Command/Unused.hs | 2 +- Content.hs | 2 +- Messages.hs | 14 ++++++++------ configure.hs | 15 +++++++++++++++ doc/bugs/problems_with_utf8_names.mdwn | 2 +- testdata/unicode-test-ö | 1 + 11 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 testdata/unicode-test-ö diff --git a/Backend/File.hs b/Backend/File.hs index fca385a1e9..d5691595a8 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -193,14 +193,14 @@ checkKeyNumCopies key file numcopies = do missingNote :: String -> Int -> Int -> String -> String missingNote file 0 _ [] = - "** No known copies of " ++ showFile file ++ " exist!" + "** No known copies of " ++ filePathToString file ++ " exist!" missingNote file 0 _ untrusted = - "Only these untrusted locations may have copies of " ++ showFile file ++ + "Only these untrusted locations may have copies of " ++ filePathToString file ++ "\n" ++ untrusted ++ "Back it up to trusted locations with git-annex copy." missingNote file present needed [] = "Only " ++ show present ++ " of " ++ show needed ++ - " trustworthy copies of " ++ showFile file ++ " exist." ++ + " trustworthy copies of " ++ filePathToString file ++ " exist." ++ "\nBack it up with git-annex copy." missingNote file present needed untrusted = missingNote file present needed [] ++ diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index f1092492ed..9636787f0d 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -58,5 +58,5 @@ checkKeySHA1 key = do then return True else do dest <- moveBad key - warning $ "Bad file content; moved to " ++ showFile dest + warning $ "Bad file content; moved to " ++ filePathToString dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 7f40a2acb5..92fe5a2d4c 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -67,5 +67,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - warning $ "Bad file size; moved to " ++ showFile dest + warning $ "Bad file size; moved to " ++ filePathToString dest return False diff --git a/Command/Find.hs b/Command/Find.hs index 45156af051..3e9125b9a6 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -25,5 +25,5 @@ seek = [withFilesInGit start] start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when exists $ liftIO $ putStrLn $ showFile file + when exists $ liftIO $ putStrLn $ filePathToString file return Nothing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 76aab3855a..750997f540 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -33,7 +33,7 @@ perform pair@(file, _) = do ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file - else error $ "failed to add " ++ showFile file ++ "; canceling commit" + else error $ "failed to add " ++ filePathToString file ++ "; canceling commit" cleanup :: FilePath -> CommandCleanup cleanup file = do diff --git a/Command/Unused.hs b/Command/Unused.hs index 2b390b9563..67a2272371 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -68,7 +68,7 @@ checkUnused = do dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] table l = [" NUMBER KEY"] ++ map cols l - cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (showFile . show) k + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (filePathToString . show) k pad n s = s ++ replicate (n - length s) ' ' number :: Int -> [a] -> [(Int, a)] diff --git a/Content.hs b/Content.hs index 188a38787e..345599dbad 100644 --- a/Content.hs +++ b/Content.hs @@ -50,7 +50,7 @@ calcGitLink file key = do cwd <- liftIO $ getCurrentDirectory let absfile = case absNormPath cwd file of Just f -> f - Nothing -> error $ "unable to normalize " ++ showFile file + Nothing -> error $ "unable to normalize " ++ filePathToString file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++ annexLocation key diff --git a/Messages.hs b/Messages.hs index ab5992a6c6..80b53e5cd5 100644 --- a/Messages.hs +++ b/Messages.hs @@ -11,10 +11,11 @@ import Control.Monad.State (liftIO) import System.IO import Control.Monad (unless) import Data.String.Utils -import Codec.Binary.UTF8.String as UTF8 +import qualified Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex +import SysConfig verbose :: Annex () -> Annex () verbose a = do @@ -26,7 +27,7 @@ showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" showStart :: String -> String -> Annex () showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ showFile file ++ " " + liftIO $ putStr $ command ++ " " ++ filePathToString file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () @@ -58,7 +59,8 @@ warning w = do indent :: String -> String indent s = join "\n" $ map (\l -> " " ++ l) $ lines s -{- Prepares a filename for display. This is needed because strings are - - internally represented in git-annex is non-decoded form. -} -showFile :: FilePath -> String -showFile = decodeString +{- Prepares a filename for display. This is needed because on many + - platforms (eg, unix), FilePaths are internally stored in + - non-decoded form. -} +filePathToString :: FilePath -> String +filePathToString = if unicodefilepath then id else UTF8.decodeString diff --git a/configure.hs b/configure.hs index 1451d7eaa3..b5437ec1a5 100644 --- a/configure.hs +++ b/configure.hs @@ -1,6 +1,7 @@ {- Checks system configuration and generates SysConfig.hs. -} import System.Directory +import Data.List import TestConfig @@ -13,6 +14,7 @@ tests = [ , TestCase "sha1sum" $ requireCmd "sha1sum" "sha1sum /dev/null" + , TestCase "unicode FilePath support" $ unicodeFilePath ] tmpDir :: String @@ -27,6 +29,19 @@ testCp k option = TestCase cmd $ testCmd k run cmd = "cp " ++ option run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new" +{- Checks if FilePaths contain decoded unicode, or not. The testdata + - directory contains a "unicode-test-ü" file; try to find the file, + - and see if the "ü" is encoded correctly. + - + - Note that the file is shipped with git-annex, rather than created, + - to avoid other potential unicode issues. + -} +unicodeFilePath :: Test +unicodeFilePath = do + fs <- getDirectoryContents "testdata" + let file = head $ filter (isInfixOf "unicode-test") fs + return $ Config "unicodefilepath" (BoolConfig $ isInfixOf "ü" file) + setup :: IO () setup = do createDirectoryIfMissing True tmpDir diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index 29213e1ec9..efde1c9a3a 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -46,7 +46,7 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > user's configured encoding), and allow haskell's output encoding to then > encode it according to the user's locale configuration. > > This is now [[implemented|done]]. I'm not very happy that I have to watch -> > out for any place that a filename is output and call `showFile` +> > out for any place that a filename is output and call `filePathToString` > > on it, but there are really not too many such places in git-annex. > > > > Note that this only affects filenames apparently. diff --git a/testdata/unicode-test-ö b/testdata/unicode-test-ö new file mode 100644 index 0000000000..45b983be36 --- /dev/null +++ b/testdata/unicode-test-ö @@ -0,0 +1 @@ +hi From 735076c767f09c740e0c99201d341a12d7b7df2c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 15:49:59 -0400 Subject: [PATCH 0864/8313] qualified import --- Messages.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Messages.hs b/Messages.hs index 80b53e5cd5..90857280a5 100644 --- a/Messages.hs +++ b/Messages.hs @@ -15,7 +15,7 @@ import qualified Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex -import SysConfig +import qualified SysConfig verbose :: Annex () -> Annex () verbose a = do @@ -63,4 +63,4 @@ indent s = join "\n" $ map (\l -> " " ++ l) $ lines s - platforms (eg, unix), FilePaths are internally stored in - non-decoded form. -} filePathToString :: FilePath -> String -filePathToString = if unicodefilepath then id else UTF8.decodeString +filePathToString = if SysConfig.unicodefilepath then id else UTF8.decodeString From ddd305aa10b2ee63c493aee05ab30d06b9424139 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 23:23:36 -0400 Subject: [PATCH 0865/8313] releasing version 0.21 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 277869b020..11a745f00d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.21) UNRELEASED; urgency=low +git-annex (0.21) unstable; urgency=low * test: Don't rely on chmod -R working. * unannex: Fix recently introduced bug when attempting to unannex more @@ -6,7 +6,7 @@ git-annex (0.21) UNRELEASED; urgency=low * test: Set git user name and email in case git can't guess values. * Fix display of unicode filenames. - -- Joey Hess Wed, 09 Feb 2011 00:12:11 -0400 + -- Joey Hess Fri, 11 Feb 2011 23:21:08 -0400 git-annex (0.20) unstable; urgency=low From 9806af73686259bf400222df1636cf3987b1ff76 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 11 Feb 2011 23:23:53 -0400 Subject: [PATCH 0866/8313] add news item for git-annex 0.21 --- doc/news/version_0.16.mdwn | 13 ------------- doc/news/version_0.21.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 doc/news/version_0.16.mdwn create mode 100644 doc/news/version_0.21.mdwn diff --git a/doc/news/version_0.16.mdwn b/doc/news/version_0.16.mdwn deleted file mode 100644 index 3247e302dd..0000000000 --- a/doc/news/version_0.16.mdwn +++ /dev/null @@ -1,13 +0,0 @@ -git-annex 0.16 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * git-annex-shell: Avoid exposing any git repo config except for the - annex.uuid when doing configlist. - * bugfix: Running `move --to` with a remote whose UUID was not yet known - could result in git-annex not recording on the local side where the - file was moved to. This could not result in data loss, or even a - significant problem, since the remote *did* record that it had the file. - * Also, add a general guard to detect attempts to record information - about repositories with missing UUIDs. - * bugfix: Running `move --to` with a non-ssh remote failed. - * bugfix: Running `copy --to` with a non-ssh remote actually did a move. - * Many test suite improvements. Current top-level test coverage: 65%"""]] \ No newline at end of file diff --git a/doc/news/version_0.21.mdwn b/doc/news/version_0.21.mdwn new file mode 100644 index 0000000000..996474910e --- /dev/null +++ b/doc/news/version_0.21.mdwn @@ -0,0 +1,7 @@ +git-annex 0.21 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * test: Don't rely on chmod -R working. + * unannex: Fix recently introduced bug when attempting to unannex more + than one file at a time. + * test: Set git user name and email in case git can't guess values. + * Fix display of unicode filenames."""]] \ No newline at end of file From 4f5f29c146a162c10a29c7744a923b9b4499846c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sat, 12 Feb 2011 21:19:26 +0000 Subject: [PATCH 0867/8313] Added a comment --- .../comment_14_89a960b6706ed703b390a81a8bc4e311._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment b/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment new file mode 100644 index 0000000000..724fe5505a --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 14" + date="2011-02-12T21:19:24Z" + content=""" +I've been trying to dig around the trace and code, and used google to see if the forkProcess issue was a haskell thing or an OSX thing. It seems that someone may have ran into a similar issue, though I am not sure if its related. +"""]] From 44d015974911ed1d5beb3ddde49ff6d59d533ca1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 02:45:52 +0000 Subject: [PATCH 0868/8313] Added a comment --- ..._6b8867b8e48bf807c955779c9f8f0909._comment | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment b/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment new file mode 100644 index 0000000000..733ec997a6 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment @@ -0,0 +1,71 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 15" + date="2011-02-13T02:45:51Z" + content=""" +It may be possible that OSX has some low resource limits, for user processes (266 per user I think) doing a + + sudo sysctl -w kern.maxproc=2048 + sudo sysctl -w kern.maxprocperuid=1024 + sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf + sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf + +seems to change the behaviour of the tests abit... + +
+Testing 1:blackbox:3:git-annex unannex:1:with content                         
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+Testing 1:blackbox:4:git-annex drop:0:no remotes                              
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Testing 1:blackbox:4:git-annex drop:1:with remote                             
+Testing 1:blackbox:4:git-annex drop:2:untrusted remote                        
+Testing 1:blackbox:5:git-annex get                                            
+Testing 1:blackbox:6:git-annex move                                           
+Testing 1:blackbox:7:git-annex copy                                           
+Testing 1:blackbox:8:git-annex unlock/lock                                    
+Testing 1:blackbox:9:git-annex edit/commit:0                                  
+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok
+ok
+Testing 1:blackbox:9:git-annex edit/commit:1                                  
+Testing 1:blackbox:10:git-annex fix                                           
+Testing 1:blackbox:11:git-annex trust/untrust/semitrust                       
+Testing 1:blackbox:12:git-annex fsck:0                                        
+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297565141:20:foo
+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa
+Testing 1:blackbox:12:git-annex fsck:1                                        
+### Failure in: 1:blackbox:12:git-annex fsck:1
+fsck failed to fail with content only available in untrusted (current) repository
+Testing 1:blackbox:12:git-annex fsck:2                                        
+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	58e831c2-371b-11e0-bc1f-47d738dc52ee  -- test repo
+Testing 1:blackbox:13:git-annex migrate:0                                     
+Cases: 30  Tried: 27  Errors: 0  Failures: 3  git-annex: user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))
+### Failure in: 1:blackbox:13:git-annex migrate:0
+migrate annexedfile failed
+Testing 1:blackbox:13:git-annex migrate:1                                     
+### Error in:   1:blackbox:13:git-annex migrate:1
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Testing 1:blackbox:14:git-annex unused/dropunused                             
+### Error in:   1:blackbox:14:git-annex unused/dropunused
+forkProcess: resource exhausted (Resource temporarily unavailable)
+Cases: 30  Tried: 30  Errors: 2  Failures: 4
+test: failed
+
+ + +the number of failures vary as I change the values of the maxprocs, I think I have narrowed it down to OSX just being stupid with limits thus causing the tests to fail. + +"""]] From c319a336a3829b2f613a86725603c4b2e8837476 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Feb 2011 00:50:09 -0400 Subject: [PATCH 0869/8313] Fix test suite to reap zombies. I had not taken into account that the code was written to run git and leave zombies, for performance/laziness reasons, when I wrote the test suite. So rather than the typical 1 zombie process that git-annex develops, test developed dozens. Caused problems on system with low process limits. Added a reap function to GitRepo, that waits for any zombie child processes. --- CmdLine.hs | 3 +++ GitRepo.hs | 37 ++++++++++++++++++------------------- debian/changelog | 6 ++++++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index d65739791f..475ca99e78 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -101,4 +101,7 @@ shutdown = do unless (q == GitQueue.empty) $ do showSideAction "Recording state in git..." Annex.queueRun + + liftIO $ Git.reap + return True diff --git a/GitRepo.hs b/GitRepo.hs index 7bb20fc53c..7cf0891eda 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -34,7 +34,6 @@ module GitRepo ( gitCommandLine, run, pipeRead, - hPipeRead, attributes, remotes, remotesAdd, @@ -50,6 +49,7 @@ module GitRepo ( typeChangedFiles, typeChangedStagedFiles, absDir, + reap, prop_idempotent_deencode ) where @@ -58,6 +58,7 @@ import Control.Monad (unless) import System.Directory import System.Posix.Directory import System.Posix.User +import System.Posix.Process import System.Path import System.Cmd.Utils import IO (bracket_) @@ -254,22 +255,24 @@ run repo params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo params) unless ok $ error $ "git " ++ show params ++ " failed" -{- Runs a git subcommand and returns its output. -} +{- Runs a git subcommand and returns it output, lazily. + - + - Note that this leaves the git process running, and so zombies will + - result unless reap is called. + -} pipeRead :: Repo -> [String] -> IO String pipeRead repo params = assertLocal repo $ do - pOpen ReadFromPipe "git" (gitCommandLine repo params) $ \h -> do - hGetContentsStrict h + (_, s) <- pipeFrom "git" (gitCommandLine repo params) + return s -{- Like pipeRead, but does not read output strictly; recommended - - for git commands that produce a lot of output that will be processed - - lazily. - - - - ONLY AFTER the string has been read completely, You must call either - - getProcessStatus or forceSuccess on the PipeHandle. Zombies will result - - otherwise.-} -hPipeRead :: Repo -> [String] -> IO (PipeHandle, String) -hPipeRead repo params = assertLocal repo $ do - pipeFrom "git" (gitCommandLine repo params) +{- Reaps any zombie git processes. -} +reap :: IO () +reap = do + -- throws an exception when there are no child processes + r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing) + case r of + Nothing -> return () + Just _ -> reap {- Scans for files that are checked into git at the specified locations. -} inRepo :: Repo -> [FilePath] -> IO [FilePath] @@ -322,9 +325,7 @@ typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end - parameter), and splits it into a list of files. -} pipeNullSplit :: Repo -> [String] -> IO [FilePath] pipeNullSplit repo params = do - -- XXX handle is left open, this is ok for git-annex, but may need - -- to be cleaned up for other uses. - (_, fs0) <- hPipeRead repo params + fs0 <- pipeRead repo params return $ split0 fs0 where split0 s = filter (not . null) $ split "\0" s @@ -410,8 +411,6 @@ checkAttr repo attr files = do (_, s) <- pipeBoth "git" params $ join "\0" absfiles cwd <- getCurrentDirectory return $ map (topair $ cwd++"/") $ lines s - -- XXX handle is left open, this is ok for git-annex, but may need - -- to be cleaned up for other uses. where params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] topair cwd l = (relfile, value) diff --git a/debian/changelog b/debian/changelog index 11a745f00d..9fc2e559a7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.22) UNRELEASED; urgency=low + + * Fix test suite to reap zombies. + + -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 + git-annex (0.21) unstable; urgency=low * test: Don't rely on chmod -R working. From 123f6a571d2b6aff097cd03e64769756a900bc6a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Feb 2011 00:51:20 -0400 Subject: [PATCH 0870/8313] heh --- debian/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/changelog b/debian/changelog index 9fc2e559a7..fd038a2dab 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.22) UNRELEASED; urgency=low * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 From c52bd2f10b2e4317c4c9cbc451f273dc4dfec79e Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 13 Feb 2011 04:52:26 +0000 Subject: [PATCH 0871/8313] Added a comment --- .../comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment b/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment new file mode 100644 index 0000000000..ca1b8e8cd5 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 16" + date="2011-02-13T04:52:26Z" + content=""" +I've fixed the test suite to not accumulate all those zombie processes. Now only 2 or 3 processes should run max. Am curious to see if that clears up all the problems. +"""]] From ed15a4e184a7d962d8f45dbadf640c24e397af0e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 10:46:55 +0000 Subject: [PATCH 0872/8313] Added a comment --- ..._62fccb04b0e4b695312f7a3f32fb96ee._comment | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment b/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment new file mode 100644 index 0000000000..7c7200fb9e --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment @@ -0,0 +1,43 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 17" + date="2011-02-13T10:46:54Z" + content=""" +Yeap, that did the trick. I just tested a few separate OSX 10.6.6 systems and the tests are better behaved now, only 3 failures now. + +So the tests behave better (at least we don't get resource fork errors any more) + + * after the commit c319a3 without modifying the system limits (of 266 procs per user) + * without the commit c319a3 and when I increase the system process limits to as much as OSX allows + +On all the systems I tested on, I'm down to 3 failures now. + +
+### Failure in: 1:blackbox:3:git-annex unannex:1:with content
+foo is not a symlink
+### Failure in: 1:blackbox:4:git-annex drop:0:no remotes
+drop wrongly succeeded with no known copy of file
+Cases: 30  Tried: 20  Errors: 0  Failures: 2add foo ok
+ok
+Cases: 30  Tried: 24  Errors: 0  Failures: 2  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  Bad file size; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/WORM:1297594011:20:foo
+  Bad file content; moved to /Users/jtang/develop/git-annex/.t/tmprepo/.git/annex/bad/SHA1:ee80d2cec57a3810db83b80e1b320df3a3721ffa
+### Failure in: 1:blackbox:12:git-annex fsck:1
+fsck failed to fail with content only available in untrusted (current) repository
+Cases: 30  Tried: 26  Errors: 0  Failures: 3  Only 1 of 2 trustworthy copies of foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo
+  Only 1 of 2 trustworthy copies of sha1foo exist.
+  Back it up with git-annex copy.
+  The following untrusted locations may also have copies: 
+  	90d63906-375e-11e0-8867-abb8a6368269  -- test repo
+Cases: 30  Tried: 30  Errors: 0  Failures: 3
+
+ +It's the same set of failures across all the OSX systems that I have tested on. Now I just need to figure out why there are still these three failures. +"""]] From 65d30ffc375259a0d09c914abf15057e8ef639c5 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 15:12:11 +0000 Subject: [PATCH 0873/8313] Added a comment: maybe killed another osx bug in the test. --- ..._64fab50d95de619eb2e8f08f90237de1._comment | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment b/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment new file mode 100644 index 0000000000..df76bb3017 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment @@ -0,0 +1,24 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="maybe killed another osx bug in the test." + date="2011-02-13T15:12:10Z" + content=""" +I think I have figured out why + + ### Failure in: 1:blackbox:3:git-annex unannex:1:with content + foo is not a symlink + +It goes back to the this piece of code (in test.hs) + + copyrepo :: FilePath -> FilePath -> IO FilePath + copyrepo old new = do + cleanup new + ensuretmpdir + Utility.boolSystem \"cp\" [\"-pr\", old, new] @? \"cp -pr failed\" + +It seems that on OSX it does not preserve the symbolic link information, basically cp is not gnu cp on OSX, doing a \"cp -a SOURCE DEST\" seem's to the right thing on OSX. I tried it out on my archlinux workstation by replacing *-pr* with just *-a* and all the tests passed on archlinux. + +I'm not sure what the implications would be with changing the test with changing the cp command. + +"""]] From 02711df66e77c8eb1686e5d4d50c93da475daa7c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 13 Feb 2011 15:55:48 +0000 Subject: [PATCH 0874/8313] Added a comment --- ...mment_19_4253988ed178054c8b6400beeed68a29._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment b/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment new file mode 100644 index 0000000000..090c991c3a --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 19" + date="2011-02-13T15:55:47Z" + content=""" +On second thought and after some messing (trying most of the options and combinations of options on OSX for).... I tried replacing cp with gnu cp from coreutils on my OSX install, and all the tests passed. *sigh* cp -a is preserving some permissions and attributes but not all, its not behaving in the same way as the gnu cp does... the closet thing that I have found on OSX that behaves in the same way as gnu \"cp -pr\" is to use \"ditto\". + +Just doing a \"ditto SOURCE DEST\" in the tests passes everything. I'm not sure if its a good idea to use this even though it works. Though this is just the tests, does it affect CopyFile.hs where \"cp\" is called? + +"""]] From e2cf7389be5158b1016ef81d41d235b4e7cf7c2d Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 13 Feb 2011 17:54:09 +0000 Subject: [PATCH 0875/8313] Added a comment --- ...omment_20_7db27d1a22666c831848bc6c06d66a84._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment diff --git a/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment b/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment new file mode 100644 index 0000000000..b617da9261 --- /dev/null +++ b/doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 20" + date="2011-02-13T17:54:09Z" + content=""" +Outside the test suite, git-annex's actual use of cp puts fairly low demands on it. It tries to use cp -a or cp -p if available just to preserve whatever attributes it can preserve, but the worst case if that you have a symlink pointing to a file that doesn't have the original timestamp or whatever. And there's little expectation git preserves that stuff anyway. + +I will probably try to make the test suite entirely use git clone rather than cp. +"""]] From 33901834008bef4e260d0be3b9e344b8f82b5856 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Feb 2011 14:17:02 -0400 Subject: [PATCH 0876/8313] Make test suite not rely on a working cp -pr. (The Unix wars are still ON!) --- debian/changelog | 2 ++ doc/bugs/Problems_running_make_on_osx.mdwn | 6 +++++ test.hs | 31 ++++++++++------------ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/debian/changelog b/debian/changelog index fd038a2dab..173c441b03 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (0.22) UNRELEASED; urgency=low * Fix test suite to reap zombies. (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 diff --git a/doc/bugs/Problems_running_make_on_osx.mdwn b/doc/bugs/Problems_running_make_on_osx.mdwn index bd0f2e64a8..83b75fb544 100644 --- a/doc/bugs/Problems_running_make_on_osx.mdwn +++ b/doc/bugs/Problems_running_make_on_osx.mdwn @@ -41,3 +41,9 @@ My knowledge of Haskell (had to lookup the spelling...) is more than rudimentary > entire end of the file from line 68, "for quickcheck" down. This code > is only used by the test suite (so "make test" will fail), > but it should get it to build. --[[Joey]] + +--- + +Closing this bug because the above problem now has a solution documented on +the install page, and the below test suite failure problems should all be +resolved on OSX. [[done]] --[[Joey]] diff --git a/test.hs b/test.hs index 1951b44260..9b50bcb2ef 100644 --- a/test.hs +++ b/test.hs @@ -137,7 +137,10 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] annexed_notpresent annexedfile git_annex "unannex" ["-q", annexedfile] @? "unannex failed with no copy" annexed_notpresent annexedfile - withcopy = "with content" ~: intmpcopyrepo $ do + withcopy = "with content" ~: intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + @? "git commit of state failed" annexed_present annexedfile git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed" unannexed annexedfile @@ -149,7 +152,12 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] test_drop :: Test test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote] where - noremote = "no remotes" ~: TestCase $ intmpcopyrepo $ do + noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get failed" + Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + @? "git commit of state failed" + Utility.boolSystem "git" ["remote", "rm", "origin"] + @? "git remote rm origin failed" r <- git_annex "drop" ["-q", annexedfile] not r @? "drop wrongly succeeded with no known copy of file" annexed_present annexedfile @@ -347,11 +355,13 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" corrupt annexedfile corrupt sha1annexedfile - withlocaluntrusted = TestCase $ intmpcopyrepo $ do + withlocaluntrusted = TestCase $ intmpclonerepo $ do + git_annex "get" ["-q", annexedfile] @? "get failed" + git_annex "untrust" ["-q", "origin"] @? "untrust of origin repo failed" git_annex "untrust" ["-q", "."] @? "untrust of current repo failed" fsck_should_fail "content only available in untrusted (current) repository" git_annex "trust" ["-q", "."] @? "trust of current repo failed" - git_annex "fsck" ["-q"] @? "fsck failed on trusted repo" + git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo" withremoteuntrusted = TestCase $ intmpclonerepo $ do Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" git_annex "get" ["-q", annexedfile] @? "get failed" @@ -475,15 +485,9 @@ innewrepo a = withgitrepo $ \r -> indir r a inmainrepo :: Assertion -> Assertion inmainrepo a = indir mainrepodir a -intmpcopyrepo :: Assertion -> Assertion -intmpcopyrepo a = withtmpcopyrepo $ \r -> indir r a - intmpclonerepo :: Assertion -> Assertion intmpclonerepo a = withtmpclonerepo $ \r -> indir r a -withtmpcopyrepo :: (FilePath -> Assertion) -> Assertion -withtmpcopyrepo = bracket (copyrepo mainrepodir tmprepodir) cleanup - withtmpclonerepo :: (FilePath -> Assertion) -> Assertion withtmpclonerepo = bracket (clonerepo mainrepodir tmprepodir) cleanup @@ -513,13 +517,6 @@ setuprepo dir = do Utility.boolSystem "git" ["config", "user.email", "test@example.com"] @? "git config failed" return dir -copyrepo :: FilePath -> FilePath -> IO FilePath -copyrepo old new = do - cleanup new - ensuretmpdir - Utility.boolSystem "cp" ["-pr", old, new] @? "cp -pr failed" - return new - -- clones are always done as local clones; we cannot test ssh clones clonerepo :: FilePath -> FilePath -> IO FilePath clonerepo old new = do From 68db7bba4390fb54ffeb9ce5d72644d2e8f4790a Mon Sep 17 00:00:00 2001 From: "http://dieter-be.myopenid.com/" Date: Mon, 14 Feb 2011 21:06:55 +0000 Subject: [PATCH 0877/8313] --- doc/forum/can_git-annex_replace_ddm__63__.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/can_git-annex_replace_ddm__63__.mdwn diff --git a/doc/forum/can_git-annex_replace_ddm__63__.mdwn b/doc/forum/can_git-annex_replace_ddm__63__.mdwn new file mode 100644 index 0000000000..8d49652c3d --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__.mdwn @@ -0,0 +1,13 @@ +Hi, +a few years ago I wrote a tool called 'ddm'. The code is overengineered and the script is more complicated then it should be, +but I think it demonstrates some good use cases, and I wonder how well git-annex can fulfill the requirements for those use cases - maybe I should remove ddm and start hacking with git-annex instead. + +To answer this question, you should read the section about the possible dataset types on http://dieter.plaetinck.be/ddm_a_distributed_data_manager.html, and the example at the bottom of that page. it demonstrates the idea behind the "selection" dataset to always try to keep a subset (the most appropriate, based on the output of some script) of files "checked out". +the introduction section on https://github.com/Dieterbe/ddm/raw/358f7cf92c0ba7b336dc97638351d4e324461afa/MANUAL should further clarify things, as well as give some more good use cases (as you can see it's a bit more about [semi-]automated workflows then purely tracking what's where) + +So I'm not sure, maybe the way to go for me is to make git-annex my "housekeeping about which data is where" backend and make ddm into a set of policies and tools on top of git-annex. + +Any input? + +Thanks, +Dieter From 208fb142d40e80da4f3dd9744e06027b3c2bbc46 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Feb 2011 22:08:54 +0000 Subject: [PATCH 0878/8313] Added a comment --- ...mment_1_aa05008dfe800474ff76678a400099e1._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment new file mode 100644 index 0000000000..eb824971f6 --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-14T22:08:54Z" + content=""" +Yes, there is value in layering something over git-annex to use a policy to choose what goes where. + +I use [mr](http://kitenet.net/~joey/code/mr/) to update and manage all my repositories, and since mr can be made to run arbitrary commands when doing eg, an update, I use its config file as such a policy layer. For example, my podcasts are pulled into my sound repository in a subdirectory; boxes that consume podcasts run \"git pull; git annex get podcasts --exclude=\"*/out/*\"; git annex drop podcasts/*/out\". I move podcasts to \"out\" directories once done with them (I have yet to teach mpd to do that for me..), and the next time I run \"mr update\" to update everything, it pulls down new ones and removes old ones. + +I don't see any obstacle to doing what you want. May be that you'd need better querying facilities in git-annex (so the policy layer can know what is available where), or finer control (--exclude is a good enough hammer for me, but maybe not for you). +"""]] From 1760ad71d85bcb66841dfcd45ba64f29fd809637 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:25:05 +0000 Subject: [PATCH 0879/8313] Typo fixes in index.mdwn --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index a279e3e5fb..503c858018 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -51,7 +51,7 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage * [[location_tracking]] reminds you where git-annex has seen files -* git-annex prevents accidential data loss by [[tracking copies|copies]] +* git-annex prevents accidental data loss by [[tracking copies|copies]] of your files * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. From 399be585380839eda2ccd77cffaeab289ecc9ffa Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:42:20 +0000 Subject: [PATCH 0880/8313] Typo fixes in git-annex.mdwn --- doc/git-annex.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d670d626ee..2f25f9bf81 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -219,7 +219,7 @@ Many git-annex commands will stage changes for later `git commit` by you. * fromkey file - This can be used to maually set up a file to link to a specified key + This can be used to manually set up a file to link to a specified key in the key-value backend. How you determine an existing key in the backend varies. For the URL backend, the key is just a URL to the content. @@ -244,7 +244,7 @@ Many git-annex commands will stage changes for later `git commit` by you. * setkey file - This plumbing-level command sets the annxed data for a key to the content of + This plumbing-level command sets the annexed data for a key to the content of the specified file, and then removes the file. A backend will typically need to be specified with --backend. If none @@ -380,7 +380,7 @@ These files are used by git-annex, in your git repository: available. Annexed files in your git repository symlink to that content. `.git-annex/uuid.log` is used to map between repository UUID and -decscriptions. +descriptions. `.git-annex/trust.log` is used to indicate which repositories are trusted and untrusted. From cf595181748bac6791970f3a3bf2e43196c7422a Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:47:47 +0000 Subject: [PATCH 0881/8313] Typo fixes in location_tracking.mdwn --- doc/location_tracking.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn index 3791029f8f..301282b6ff 100644 --- a/doc/location_tracking.mdwn +++ b/doc/location_tracking.mdwn @@ -27,5 +27,5 @@ descriptions to help you with finding them: c0a28e06-d7ef-11df-885c-775af44f8882 -- USB archive drive 1 e1938fee-d95b-11df-96cc-002170d25c55 -In certian cases you may want to configure git-annex to [[trust]] +In certain cases you may want to configure git-annex to [[trust]] that location tracking information is always correct for a repository. From c9348f8e55f4fd2c5f63e10262434771d6aff359 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:50:54 +0000 Subject: [PATCH 0882/8313] Typo fixes in walkthrough.mdwn --- doc/walkthrough.mdwn | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 231c3e543c..d08b247f73 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -84,7 +84,7 @@ can get them. ## transferring files: When things go wrong -After a while, you'll have serveral annexes, with different file contents. +After a while, you'll have several annexes, with different file contents. You don't have to try to keep all that straight; git-annex does [[location_tracking]] for you. If you ask it to get a file and the drive or file server is not accessible, it will let you know what it needs to get @@ -146,7 +146,7 @@ That's a good thing, because it might be the only copy, you wouldn't want to lose it in a fumblefingered mistake. # echo oops > my_cool_big_file - bash: my_cool_big_file: Permission deined + bash: my_cool_big_file: Permission denied In order to modify a file, it should first be unlocked. @@ -176,7 +176,7 @@ There is one problem with using `git commit` like this: Git wants to first stage the entire contents of the file in its index. That can be slow for big files (sorta why git-annex exists in the first place). So, the automatic handling on commit is a nice safety feature, since it prevents -the file content being accidentially commited into git. But when working with +the file content being accidentally committed into git. But when working with big files, it's faster to explicitly add them to the annex yourself before committing. @@ -267,12 +267,12 @@ that the URL is stable; no local backup is kept. Another handy alternative to the default [[backend|backends]] is the SHA1 backend. This backend provides more git-style assurance that your data -has not been damanged. And the checksum means that when you add the same +has not been damaged. And the checksum means that when you add the same content to the annex twice, only one copy need be stored in the backend. The only reason it's not the default is that it needs to checksum files when they're added to the annex, and this can slow things down -significantly for really big files. To make SHA1 the detault, just +significantly for really big files. To make SHA1 the default, just add something like this to `.gitattributes`: * annex.backend=SHA1 @@ -292,7 +292,7 @@ files will be skipped. After migrating a file to a new backend, the old content in the old backend will still be present. That is necessary because multiple files -can point to the same content. The `git annex unused` sucommand can be +can point to the same content. The `git annex unused` subcommand can be used to clear up that detritus later. Note that hard links are used, to avoid wasting disk space. @@ -342,7 +342,7 @@ setting is satisfied for all files. fsck my_cool_big_file (checksum...) ok ... -You can also specifiy the files to check. This is particularly useful if +You can also specify the files to check. This is particularly useful if you're using sha1 and don't want to spend a long time checksumming everything. # git annex fsck my_cool_big_file @@ -367,7 +367,7 @@ might say about a badly messed up annex: ## backups git-annex can be configured to require more than one copy of a file exists, -as a simple backup for your data. This is controled by the "annex.numcopies" +as a simple backup for your data. This is controlled by the "annex.numcopies" setting, which defaults to 1 copy. Let's change that to require 2 copies, and send a copy of every file to a USB drive. @@ -394,9 +394,9 @@ For more details about the numcopies setting, see [[copies]]. ## untrusted repositories -Suppose you have a USB thunb drive and are using it as a git annex +Suppose you have a USB thumb drive and are using it as a git annex repository. You don't trust the drive, because you could lose it, or -accidentially run it through the laundry. Or, maybe you have a drive that +accidentally run it through the laundry. Or, maybe you have a drive that you know is dying, and you'd like to be warned if there are any files on it not backed up somewhere else. Maybe the drive has already died or been lost. From 36e13cfce64b43a47c2812be6b17b7a8af22f786 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:54:03 +0000 Subject: [PATCH 0883/8313] Typo fix in not.mdwn --- doc/not.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index c9c5754d7e..80c0acafaf 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -26,7 +26,7 @@ I only learned of git-media after writing git-annex, but I probably would have still written git-annex instead of using it. Currently, git-media has the advantage of using git smudge filters rather than - git-annex's pile of symlinks, and it may be a tighter fit for certian + git-annex's pile of symlinks, and it may be a tighter fit for certain situations. It lacks git-annex's support for widely distributed storage, using only a single backend data store. It also does not support partial checkouts of file contents, like git-annex does. From 000d9fcf334436dc085d52655dc07f69cd465980 Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:56:24 +0000 Subject: [PATCH 0884/8313] Typo fixes in trust.mdwn --- doc/trust.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/trust.mdwn b/doc/trust.mdwn index f843d3858c..317e4b541f 100644 --- a/doc/trust.mdwn +++ b/doc/trust.mdwn @@ -11,12 +11,12 @@ information. When removing content, it will directly check that other repositories have enough [[copies]]. Generally that explicit checking is a good idea. Consider that the current -[[location_tracking]] information for a remote may not yet have propigated +[[location_tracking]] information for a remote may not yet have propagated out. Or, a remote may have suffered a catastrophic loss of data, or itself been lost. There is still some trust involved here. A semitrusted repository is -dependended on to retain a copy of the file content; possibly the only +depended on to retain a copy of the file content; possibly the only [[copy|copies]]. (Being semitrusted is the default. The `git annex semitrust` command From 982d0962b3ff4e82e83b715d8cc3fc93ea71783c Mon Sep 17 00:00:00 2001 From: "http://sunny256.sunbase.org/" Date: Tue, 15 Feb 2011 05:58:23 +0000 Subject: [PATCH 0885/8313] Typo fixes in use_cases/Bob.mdwn --- doc/use_case/Bob.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index a5dc01b373..573d9cde97 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -6,13 +6,13 @@ safe place. With git-annex, Bob has a single directory tree that includes all his files, even if their content is being stored offline. He can reorganize his files using that tree, committing new versions to git, -without worry about accidentially deleting anything. +without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -Run in a cron job, git-annex adds new files to achival drives at night. It +Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. From dd0f662849fa24ded0d9ecb43000ac0ab8b1f7e7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 19 Feb 2011 17:00:40 -0400 Subject: [PATCH 0886/8313] hello, liftM --- Annex.hs | 4 +--- Command.hs | 28 +++++++++------------------- Utility.hs | 6 ++---- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/Annex.hs b/Annex.hs index 4a1b89dcf2..5496ada67c 100644 --- a/Annex.hs +++ b/Annex.hs @@ -75,9 +75,7 @@ eval state action = evalStateT action state {- Gets a value from the internal state, selected by the passed value - constructor. -} getState :: (AnnexState -> a) -> Annex a -getState c = do - state <- get - return (c state) +getState c = liftM c get {- Applies a state mutation function to change the internal state. - diff --git a/Command.hs b/Command.hs index 601b584642..86da454195 100644 --- a/Command.hs +++ b/Command.hs @@ -10,7 +10,7 @@ module Command where import Control.Monad.State (liftIO) import System.Directory import System.Posix.Files -import Control.Monad (filterM) +import Control.Monad (filterM, liftM) import System.Path.WildMatch import Text.Regex.PCRE.Light.Char8 import Data.List @@ -110,15 +110,13 @@ withFilesInGit :: CommandSeekStrings withFilesInGit a params = do repo <- Annex.gitRepo files <- liftIO $ runPreserveOrder (Git.inRepo repo) params - files' <- filterFiles files - return $ map a files' + liftM (map a) $ filterFiles files withAttrFilesInGit :: String -> CommandSeekAttrFiles withAttrFilesInGit attr a params = do repo <- Annex.gitRepo files <- liftIO $ runPreserveOrder (Git.inRepo repo) params files' <- filterFiles files - pairs <- liftIO $ Git.checkAttr repo attr files' - return $ map a pairs + liftM (map a) $ liftIO $ Git.checkAttr repo attr files' withBackendFilesInGit :: CommandSeekBackendFiles withBackendFilesInGit a params = do repo <- Annex.gitRepo @@ -128,8 +126,7 @@ withBackendFilesInGit a params = do withFilesMissing :: CommandSeekStrings withFilesMissing a params = do files <- liftIO $ filterM missing params - files' <- filterFiles files - return $ map a files' + liftM (map a) $ filterFiles files where missing f = do e <- doesFileExist f @@ -148,8 +145,7 @@ withFilesToBeCommitted :: CommandSeekStrings withFilesToBeCommitted a params = do repo <- Annex.gitRepo tocommit <- liftIO $ runPreserveOrder (Git.stagedFilesNotDeleted repo) params - tocommit' <- filterFiles tocommit - return $ map a tocommit' + liftM (map a) $ filterFiles tocommit withFilesUnlocked :: CommandSeekBackendFiles withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles @@ -172,9 +168,7 @@ withNothing a [] = return [a] withNothing _ _ = return [] backendPairs :: CommandSeekBackendFiles -backendPairs a files = do - pairs <- Backend.chooseBackends files - return $ map a pairs +backendPairs a files = liftM (map a) $ Backend.chooseBackends files {- Filter out files from the state directory, and those matching the - exclude glob pattern, if it was specified. -} @@ -201,9 +195,7 @@ wildsRegex' (w:ws) c = wildsRegex' ws (c ++ "|" ++ wildToRegex w) {- filter out symlinks -} notSymlink :: FilePath -> IO Bool -notSymlink f = do - s <- liftIO $ getSymbolicLinkStatus f - return $ not $ isSymbolicLink s +notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f {- Descriptions of params used in usage messages. -} paramRepeating :: String -> String @@ -260,10 +252,8 @@ preserveOrder orig new = collect orig new - its return list preserves order. - - This assumes that it's cheaper to call preserveOrder on the result, - - than it would be to run the action once with each param. In the case + - than it would be to run the action separately with each param. In the case - of git file list commands, that assumption tends to hold. -} runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath] -runPreserveOrder a files = do - r <- a files - return $ preserveOrder files r +runPreserveOrder a files = liftM (preserveOrder files) (a files) diff --git a/Utility.hs b/Utility.hs index 2bb623532d..89e129181a 100644 --- a/Utility.hs +++ b/Utility.hs @@ -38,6 +38,7 @@ import System.FilePath import System.Directory import Foreign (complement) import Data.List +import Control.Monad (liftM2) {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} @@ -95,10 +96,7 @@ absPath file = do - relPathCwdToDir "/tmp/foo/bar" == "" -} relPathCwdToDir :: FilePath -> IO FilePath -relPathCwdToDir dir = do - cwd <- getCurrentDirectory - a <- absPath dir - return $ relPathDirToDir cwd a +relPathCwdToDir dir = liftM2 relPathDirToDir getCurrentDirectory (absPath dir) {- Constructs a relative path from one directory to another. - From d8f65ced83a8ad35e72ba9eff3539638cc0c9f6c Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Sun, 20 Feb 2011 15:38:22 +0000 Subject: [PATCH 0887/8313] describe myself and what i think of git-annex --- doc/forum/chrysn.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/chrysn.mdwn diff --git a/doc/forum/chrysn.mdwn b/doc/forum/chrysn.mdwn new file mode 100644 index 0000000000..f5c07b88b3 --- /dev/null +++ b/doc/forum/chrysn.mdwn @@ -0,0 +1,5 @@ +* **name**: chrysn +* **website**: +* **uses git-annex for**: managing the family's photos (and possibly videos and music in the future) +* **likes git-annex because**: it adds a layer of commit semantics over a regular file system without keeping everything in duplicate locally +* **would like git-annex to**: not be required any more as git itself learns to use cow filesystems to avoid abundant disk usage and gets better with sparser checkouts (git-annex might then still be a simpler tool that watches over what can be safely dropped for a sparser checkout) From c215c68aaa16cfbd173881e24db49e920749bbb9 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Sun, 20 Feb 2011 16:24:41 +0000 Subject: [PATCH 0888/8313] feeling lucky with proposing deeper git-annex changes --- doc/forum/relying_on_git_for_numcopies.mdwn | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies.mdwn diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn new file mode 100644 index 0000000000..5097c9e7e8 --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies.mdwn @@ -0,0 +1,45 @@ +This is a rough sketch of a modification of git-annex to rely more on git commit semantics. It might be flawed due to my lack of understanding of git-annex internals. --[[chrysn]] + +Summary +========= + +Currently, the location tracking is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file without any promise that it might not be dropped if another repository takes over responsibility. + +git's semantics for atomic commits are proposed to be used, which makes sure that before files are actually deleted, another repository has accepted the deletion. + +Modified git-annex-drop behavior +========================== + +The most important (if not only) git-annex command that is affected by this is `git annex drop`. Currently, for dropping a large number of files, every file is checked with another (or multiple, if so configured) host if it's safe to delete. + +The new behavior would be to + +* decrement the location tracking counter for all files to be dropped, +* commit that change, +* try to push it to at least as many repositories that the numcopies constraints are met, +* revert if that fails, +* otherwise really drop the files from the backend. + +Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.) + +Implications for trust +============== + +The proposed change also changes the semantics of trust. Trust can now be controlled in a finer-grained way between untrusted and semi-trusted, as best illustrated by a use case: + +> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home. +> +> She tells her netbook's repository to dereference the old images (but not other parts of the repository she has not copied anywhere yet) and pushes to the server before leaving. When she adds pictures from her camera to the repository, git-annex can now free up space as needed. + +Dereferencing could be implemented as `git annex drop --not-yet`, freeing space is similar to `dropunused`. + +A trusted repository with the new semantics would mean that the repository would not accept dropping anything, just as before. + +Advantages / Disadvantages +===================== + +The advantage of this proposal is that the round trips required for dropping something could be greatly reduced. + +There should also be simplifications in the `git annex drop` command as it doesn't need to take care of locking any more (git should already do that between checking if HEAD is a parent of the pushed commit and replacing HEAD). + +Besides being a major change in git-annex (with the requirement to track hosts' git-annex versions for migration, as the new trust system is incompatible with the old one), no disadvantages of that stragegy are known to the author (hoping for discussion below). From 39dfdea642a3d0d0a6a8d5b51f1f1bc41b16e961 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Tue, 22 Feb 2011 10:17:46 +0000 Subject: [PATCH 0889/8313] better wording at some places --- doc/forum/relying_on_git_for_numcopies.mdwn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn index 5097c9e7e8..b7ebba805a 100644 --- a/doc/forum/relying_on_git_for_numcopies.mdwn +++ b/doc/forum/relying_on_git_for_numcopies.mdwn @@ -3,7 +3,7 @@ This is a rough sketch of a modification of git-annex to rely more on git commit Summary ========= -Currently, the location tracking is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file without any promise that it might not be dropped if another repository takes over responsibility. +Currently, [[location tracking]] is only used for informational purposes unless a repository is [[trust]]ed, in which case there is no checking at all. It is proposed to use the location tracking information as a commitment to keep track of a file until another repository takes over responsibility. git's semantics for atomic commits are proposed to be used, which makes sure that before files are actually deleted, another repository has accepted the deletion. @@ -20,18 +20,18 @@ The new behavior would be to * revert if that fails, * otherwise really drop the files from the backend. -Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.) +Unlike explicit checking, this never looks at the remote backend if the file is really present -- otoh, git-annex already relies on the files in the backend to not be touched by anyone but git-annex itself, and git-annex would only drop them if they were derefed and committed, in which case git would not accept the push. (git by itself would accept a merged push, but even if the reverting step failed due to a power outage or similar, git-annex would, before really deleting files from the backend, check again if the numcopies restraint is still met, and revert its own delete commit as the files are still present anyway.) Implications for trust ============== The proposed change also changes the semantics of trust. Trust can now be controlled in a finer-grained way between untrusted and semi-trusted, as best illustrated by a use case: -> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home. +> Alice takes her netbook with her on a trip through Spain, and will fill most of its disk up with pictures she takes. As she expects to meet some old friends during the first days, she wants to take older pictures with her, which are safely backed up at home, so they can be deleted on demand. > > She tells her netbook's repository to dereference the old images (but not other parts of the repository she has not copied anywhere yet) and pushes to the server before leaving. When she adds pictures from her camera to the repository, git-annex can now free up space as needed. -Dereferencing could be implemented as `git annex drop --not-yet`, freeing space is similar to `dropunused`. +Dereferencing could be implemented as `git annex drop --no-rm` (or `move --no-rm`), freeing space is similar to `dropunused`. A trusted repository with the new semantics would mean that the repository would not accept dropping anything, just as before. From 1f44ccb7026e0ef033f9415130f62d35604590d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 22 Feb 2011 14:44:17 -0400 Subject: [PATCH 0890/8313] add --- doc/todo/hidden_files.mdwn | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/todo/hidden_files.mdwn diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn new file mode 100644 index 0000000000..2e5ec4d9e8 --- /dev/null +++ b/doc/todo/hidden_files.mdwn @@ -0,0 +1,20 @@ +Add a `git annex hide $file` that behaves like drop, checking counter info +and updating location log to say the current repo no longer has a file -- +but does not actually remove the content. + +Then `git annex unused` can be used to clean it up later. And in the +meantime, it's still locally accessible. This can be useful if you're +planning to need to free up space later, but want to hold onto the content +for a while. Possibly you'll be disconnected later, so it's easier to push +out that intent now. + +-- + +TODO: + +* Make 100% sure this is safe. Drop, etc should never check content files + are present on other repos if the location log doesn't say the repo + has the content. + +* What will `git annex get` do if it's asked to get a file that has been + hidden? From 725c9b1ecb3ac126bbfb238730a11d9d474ddd53 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 22 Feb 2011 18:44:28 +0000 Subject: [PATCH 0891/8313] Added a comment --- ..._8ad3cccd7f66f6423341d71241ba89fc._comment | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment diff --git a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment new file mode 100644 index 0000000000..219bb60b9a --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment @@ -0,0 +1,36 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-22T18:44:28Z" + content=""" +I see the following problems with this scheme: + +- Disallows removal of files when disconnected. It's currently safe to force that, as long as + git-annex tells you enough other repos are belived to have the file. Just as long as you + only force on one machine (say your laptop). With your scheme, if you drop a file while + disconnected, any other host could see that the counter is still at N, because your + laptop had the file last time it was online, and can decide to drop the file, and lose the last +version. + +- pushing a changed counter commit to other repos is tricky, because they're not bare, and + the network topology to get the commit pulled into the other repo could vary. + +- Merging counter files issues. If the counter file doesn't automerge, two repos dropping the same file will conflict. But, if it does automerge, it breaks the counter conflict detection. + +- Needing to revert commits is going to be annoying. An actual git revert + could probably not reliably be done. It's need to construct a revert + and commit it as a new commit. And then try to push that to remotes, and + what if *that* push conflicts? + +- I do like the pre-removal dropping somewhat as an alternative to + trust checking. I think that can be done with current git-annex though, + just remove the files from the location log, but keep them in-annex. + Dropping a file only looks at repos that the location log says have a + file; so other repos can have retained a copy of a file secretly like + this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. + +- I don't see any reduced round trips. It still has to contact N other + repos on drop. Now, rather than checking that they have a file, it needs + to push a change to them. +"""]] From 0d699b7a322bfa2977cba1f29bd8f44fabb88aa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 22 Feb 2011 14:49:36 -0400 Subject: [PATCH 0892/8313] meh --- .../comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment index 219bb60b9a..83a908da8c 100644 --- a/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment +++ b/doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment @@ -28,7 +28,7 @@ version. just remove the files from the location log, but keep them in-annex. Dropping a file only looks at repos that the location log says have a file; so other repos can have retained a copy of a file secretly like - this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. + this, and can safely remove it at any time. I'd need to look into this a bit more to be 100% sure it's safe, but have started [[todo/hidden_files]]. - I don't see any reduced round trips. It still has to contact N other repos on drop. Now, rather than checking that they have a file, it needs From 4c761b0b08f904e3bce78e979bb855f4c78507aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 22 Feb 2011 14:52:22 -0400 Subject: [PATCH 0893/8313] move --- doc/{forum => users}/chrysn.mdwn | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{forum => users}/chrysn.mdwn (100%) diff --git a/doc/forum/chrysn.mdwn b/doc/users/chrysn.mdwn similarity index 100% rename from doc/forum/chrysn.mdwn rename to doc/users/chrysn.mdwn From 5b96961957678765480a4e540f3f82e28d921d5d Mon Sep 17 00:00:00 2001 From: tyger Date: Wed, 23 Feb 2011 09:49:34 +0000 Subject: [PATCH 0894/8313] --- .../migrate_existing_git_repository_to_git-annex.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex.mdwn diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn new file mode 100644 index 0000000000..fe26f69523 --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn @@ -0,0 +1,7 @@ +I have a large git repository with binary files scattered over different branches. I want to switch to git-annex mainly for performance reasons, but I don't want to loose my history. + +I tried to rewrite the (cloned) repository with git-filter-branch but failed miserably for several reasons: + * --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/). + * annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them + +Any suggestions how to proceed? From 0573ccecc51792c68d3d179601361c3ac97d1a0b Mon Sep 17 00:00:00 2001 From: tyger Date: Wed, 23 Feb 2011 15:15:03 +0000 Subject: [PATCH 0895/8313] reformat list --- doc/forum/migrate_existing_git_repository_to_git-annex.mdwn | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn index fe26f69523..94c5f1d133 100644 --- a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn +++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn @@ -1,7 +1,8 @@ I have a large git repository with binary files scattered over different branches. I want to switch to git-annex mainly for performance reasons, but I don't want to loose my history. I tried to rewrite the (cloned) repository with git-filter-branch but failed miserably for several reasons: - * --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/). - * annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them + +* --tree-filter performs its operations in a temporary directory (.git-rewrite/t/) so the symlinks point to the wrong destination (../../.git/annex/). +* annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them Any suggestions how to proceed? From c5ffefcde1e2056d0c0257cbc84c5a6e8e046b62 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 23 Feb 2011 16:44:01 +0000 Subject: [PATCH 0896/8313] Added a comment --- ...2_be6acbc26008a9cb54e7b8f498f2c2a2._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment diff --git a/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment b/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment new file mode 100644 index 0000000000..d9ce8b50e0 --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="comment 2" + date="2011-02-23T16:43:59Z" + content=""" +i'll comment on each of the points separately, well aware that even a single little leftover issue can show that my plan is faulty: + +* force removal: well, yes -- but the file that is currently force-removed on the laptop could just as well be the last of its kind itself. i see the problem, but am not sure if it's fatal (after all, if we rely on out-of-band knowledge when forcing something, we could just as well ask a little more) +* non-bare repos: pushing is tricky with non-bare repos now just as well; a post-commit hook could auto-accept counter changes. (but pushing causes problems with counters anyway, doesn't it?) +* merging: i'd have them auto-merge. git-annex will have to check the validity of the current state anyway, and a situation in which a counter-decrementing commit is not a fast-forward one would be reverted in the next step (or upon discovery, in case the next step never took place). +* reverting: my wording was bad as \"revert\" is already taken in git-lingo. the correct term for what i was thinking of is \"reset\". (as the commit could not be pushed, it would be rolled back completely). + * we might have to resort to reverting, though, if the commit has already been pused to a first server of many. +* [[todo/hidden files]]: yes, this solves pre-removal dropping :-) +* round trips: it's not the number of servers, it's the number of files (up to 30k in my case). it seems to me that an individual request was made for every single file i wanted to drop (that would be N*M roundtrips for N affected servers and M files, and N roundtrips with git managed numcopies) + +all together, it seems to be a bit more complicated than i imagined, although not completely impossible. a combination of [[todo/hidden files]] and maybe a simpler reduction of the number of requests might though achieve the important goals as well. +"""]] From bcd3a9ea34763b4b8f40eef75c05c6e9ea27d178 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 23 Feb 2011 21:48:15 +0000 Subject: [PATCH 0897/8313] Added a comment: relation to [[todo/branching]] --- .../comment_3_43d8e1513eb9947f8a503f094c03f307._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment diff --git a/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment b/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment new file mode 100644 index 0000000000..27076a877f --- /dev/null +++ b/doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="relation to [[todo/branching]]" + date="2011-02-23T21:48:14Z" + content=""" +the non-bare repository issue would go away if this was combined with the \"alternate\" approach to [[todo/branching]]. (with the \"fleshed out proposal\" of branching, this would not work at all for lack of shared commits.) +"""]] From e61b47bc8a51447f9fbde6463e09d4beef8d5a7f Mon Sep 17 00:00:00 2001 From: "http://jandd.myopenid.com/" Date: Thu, 24 Feb 2011 10:11:36 +0000 Subject: [PATCH 0898/8313] --- .../error_with_file_names_starting_with_dash.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/error_with_file_names_starting_with_dash.mdwn diff --git a/doc/bugs/error_with_file_names_starting_with_dash.mdwn b/doc/bugs/error_with_file_names_starting_with_dash.mdwn new file mode 100644 index 0000000000..7a4d33911f --- /dev/null +++ b/doc/bugs/error_with_file_names_starting_with_dash.mdwn @@ -0,0 +1,12 @@ +git annex add has problems if items start with dashes, example: + +-wut-a-directory-name-/file1 + +leads to + +[[!format bash """ +add -wut-a-directory-name-/file1 (checksum...) sha1sum: invalid option -- 'u' +„sha1sum --help“ gibt weitere Informationen. + + git-annex: : hGetLine: end of file +"""]] From 836e71297b8e3b5bd6f89f7eb1198f59af985b0b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 25 Feb 2011 01:13:01 -0400 Subject: [PATCH 0899/8313] Support filenames that start with a dash; when such a file is passed to a utility it will be escaped to avoid it being interpreted as an option. --- Backend/SHA1.hs | 3 ++- Backend/URL.hs | 2 +- Command/SetKey.hs | 3 ++- CopyFile.hs | 10 ++++++---- Remotes.hs | 2 +- RsyncFile.hs | 5 +++-- Utility.hs | 8 ++++++++ debian/changelog | 3 +++ doc/bugs/error_with_file_names_starting_with_dash.mdwn | 3 +++ 9 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 9636787f0d..e1830bc134 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -20,6 +20,7 @@ import qualified Annex import Locations import Content import Types +import Utility backend :: Backend Annex backend = Backend.File.backend { @@ -31,7 +32,7 @@ backend = Backend.File.backend { sha1 :: FilePath -> Annex String sha1 file = do showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" [file] $ \h -> do + liftIO $ pOpen ReadFromPipe "sha1sum" [utilityEscape file] $ \h -> do line <- hGetLine h let bits = split " " line if null bits diff --git a/Backend/URL.hs b/Backend/URL.hs index 38954e5a31..15cc88d644 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -51,6 +51,6 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" showProgress -- make way for curl progress bar - liftIO $ boolSystem "curl" ["-#", "-o", file, url] + liftIO $ boolSystem "curl" ["-#", "-o", utilityEscape file, url] where url = join ":" $ drop 1 $ split ":" $ show key diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 388392cd60..025fb74d62 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -35,7 +35,8 @@ perform file = do -- rather than simply calling moveToObjectDir ok <- getViaTmp key $ \dest -> do if dest /= file - then liftIO $ boolSystem "mv" [file, dest] + then liftIO $ + boolSystem "mv" [utilityEscape file, utilityEscape dest] else return True if ok then return $ Just $ cleanup diff --git a/CopyFile.hs b/CopyFile.hs index e913aa070d..73d911a291 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -23,9 +23,11 @@ copyFile src dest = do boolSystem "cp" opts where opts = if SysConfig.cp_reflink_auto - then ["--reflink=auto", src, dest] + then ["--reflink=auto", src', dest'] else if SysConfig.cp_a - then ["-a", src, dest] + then ["-a", src', dest'] else if SysConfig.cp_p - then ["-p", src, dest] - else [src, dest] + then ["-p", src', dest'] + else [src', dest'] + src' = utilityEscape src + dest' = utilityEscape dest diff --git a/Remotes.hs b/Remotes.hs index 15f5185b91..c7e69aad8b 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -282,7 +282,7 @@ rsyncParams r sending key file = do -- inplace makes rsync resume partial files options = ["-p", "--progress", "--inplace"] -- the rsync shell parameter controls where rsync - -- does, so the source/dest parameter can be a dummy value, + -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. dummy = ":" diff --git a/RsyncFile.hs b/RsyncFile.hs index 274e66151b..9de2e2c594 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -24,11 +24,12 @@ rsyncShell command = ["-e", unwords $ map escape command] {- Runs rsync in server mode to send a file, and exits. -} rsyncServerSend :: FilePath -> IO () -rsyncServerSend file = rsyncExec $ rsyncServerParams ++ ["--sender", file] +rsyncServerSend file = rsyncExec $ + rsyncServerParams ++ ["--sender", utilityEscape file] {- Runs rsync in server mode to receive a file. -} rsyncServerReceive :: FilePath -> IO Bool -rsyncServerReceive file = rsync $ rsyncServerParams ++ [file] +rsyncServerReceive file = rsync $ rsyncServerParams ++ [utilityEscape file] rsyncServerParams :: [String] rsyncServerParams = diff --git a/Utility.hs b/Utility.hs index 89e129181a..6a90e3cd50 100644 --- a/Utility.hs +++ b/Utility.hs @@ -15,6 +15,7 @@ module Utility ( boolSystem, shellEscape, shellUnEscape, + utilityEscape, unsetFileMode, readMaybe, safeWriteFile, @@ -179,6 +180,13 @@ shellUnEscape s = word:(shellUnEscape rest) | c == q = findword w cs | otherwise = inquote q (w++[c]) cs +{- Ensures that a filename is safe to pass to a utility program. In particular + - since utilities tend to interpret things starting with a dash as + - an option, relative filenames starting with a dash are escaped. -} +utilityEscape :: FilePath -> FilePath +utilityEscape ('-':s) = "./-" ++ s +utilityEscape s = s + {- For quickcheck. -} prop_idempotent_shellEscape :: String -> Bool prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) diff --git a/debian/changelog b/debian/changelog index 173c441b03..d634223245 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,9 @@ git-annex (0.22) UNRELEASED; urgency=low for his help eliminating the infestation... for now.) * Make test suite not rely on a working cp -pr. (The Unix wars are still ON!) + * Support filenames that start with a dash; when such a file is passed + to a utility it will be escaped to avoid it being interpreted as an + option. -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 diff --git a/doc/bugs/error_with_file_names_starting_with_dash.mdwn b/doc/bugs/error_with_file_names_starting_with_dash.mdwn index 7a4d33911f..84bf1cfa07 100644 --- a/doc/bugs/error_with_file_names_starting_with_dash.mdwn +++ b/doc/bugs/error_with_file_names_starting_with_dash.mdwn @@ -10,3 +10,6 @@ add -wut-a-directory-name-/file1 (checksum...) sha1sum: invalid option -- 'u' git-annex: : hGetLine: end of file """]] + +> This is fixed in git, at least I think I've found all cases where +> filenames are passed to programs and escaped them. --[[Joey]] [[done]] From d35c11de7a83e135636749deb563de2ef3fee3a6 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 05:16:48 +0000 Subject: [PATCH 0900/8313] Added a comment --- ...comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment new file mode 100644 index 0000000000..e88794d621 --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-25T05:16:48Z" + content=""" +I don't know how to approach this yet, but I support the idea -- it would be great if there was a tool that could punch files out of git history and put them in the annex. (Of course with typical git history rewriting caveats.) + +Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be? +"""]] From fa37ae1d9669a76d8e8e9a08cf044836f7eb8c0b Mon Sep 17 00:00:00 2001 From: jbd Date: Fri, 25 Feb 2011 09:12:45 +0000 Subject: [PATCH 0901/8313] --- ...list:___34__git_annex_add__34___multiple_processes.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn new file mode 100644 index 0000000000..f3d3837015 --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn @@ -0,0 +1,7 @@ +Hello, + +i'm in the process of managing my music collection with git-annex. The initial "git annex add" using the sha1 banckend is quite long an i was wondering that it could be nice to launch multiple "sha1sum" processes in parallel to speed up things. + +Anyway, thanks for this wonderful piece of software. + +Jean-Baptiste From 5cb24bebf4479f5fc83980e328bce511b36b05fd Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:12:42 +0000 Subject: [PATCH 0902/8313] Added a comment --- ...comment_1_85b14478411a33e6186a64bd41f0910d._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment new file mode 100644 index 0000000000..2364b7fb83 --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-02-25T19:12:42Z" + content=""" +I'd expect the checksumming to be disk bound, not CPU bound, on most systems. + +I suggest you start off on the WORM backend, and then you can run a job later to [[migrate|walkthrough#index14h2]] to the SHA1 backend. +"""]] From 3f7c0b6970f58c1484615df7b3ec40f3ee44c9df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 25 Feb 2011 15:50:17 -0400 Subject: [PATCH 0903/8313] further investigation --- doc/todo/smudge.mdwn | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn index f51f45a391..c51662b282 100644 --- a/doc/todo/smudge.mdwn +++ b/doc/todo/smudge.mdwn @@ -1,14 +1,34 @@ git-annex should use smudge/clean filters. -The trick is doing it efficiently. Since git a2b665d, 2011-01-05, +The clean filter is run when files are staged for commit. So a user could copy +any file into the annex, git add it, and git-annex's clean filter causes +the file's key to be staged, while its value is added to the annex. + +The smudge filter is run when files are checked out. Since git annex +repos have partial content, this would not git annex get the file content. +Instead, if the content is not currently available, it would need to do +something like return empty file content. (Sadly, it cannot create a +symlink, as git still wants to write the file afterwards. + +So the nice current behavior of unavailable files being clearly missing due +to dangling symlinks, would be lost when using smudge/clean filters. +(Contact git developers to get an interface to do this?) + +Instead, we get the nice behavior of not having to remeber to `git annex +add` files, and just being able to use `git add` or `git commit -a`, +and have it use git-annex when .gitattributes says to. Also, annexed +files can be directly modified without having to `git annex unlock`. + +### efficiency + +The trick is doing it efficiently. Since git a2b665d, v1.7.4.1, something like this works to provide a filename to the clean script: git config --global filter.huge.clean huge-clean %f This avoids it needing to read all the current file content from stdin when doing eg, a git status or git commit. Instead it is passed the -filename that git is operating on, I think that's from the working -directory. +filename that git is operating on, in the working directory. So, WORM could just look at that file and easily tell if it is one it already knows (same mtime and size). If so, it can short-circuit and @@ -32,12 +52,6 @@ I've a demo implementation of this technique in the scripts below. ---- -It may further be possible to use the %f with the smudge filter -(docs say it's supported), and instead of outputting the dummy content, -it could create a dangling symlink, which would be more like git-annex's -behavior now, and makes it easy to tell what content is not available -with `ls`. - ### test files huge-smudge: From 42b181a04b6e3cc4f2019aa8b5b7dfa7ffb5ce51 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:54:29 +0000 Subject: [PATCH 0904/8313] Added a comment --- .../comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment new file mode 100644 index 0000000000..9b8240658b --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-02-25T19:54:28Z" + content=""" +But, see [[todo/parallel_possibilities]] +"""]] From 8aebf9a337e26328c5cb406300fcef5df42bc7b3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 25 Feb 2011 15:54:29 -0400 Subject: [PATCH 0905/8313] update --- doc/todo/parallel_possibilities.mdwn | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/todo/parallel_possibilities.mdwn b/doc/todo/parallel_possibilities.mdwn index 15e5171ca3..9c0e69e294 100644 --- a/doc/todo/parallel_possibilities.mdwn +++ b/doc/todo/parallel_possibilities.mdwn @@ -6,7 +6,8 @@ Anyway, each git-annex command is broken down into a series of independant actions, which has some potential for parallelism. Each action has 3 distinct phases, basically "check", "perform", and -"cleanup". The perform actions are not parellizable; the cleanup may be, -and the check should be easily parallelizable, although they may access the -disk or run minor git query commands, so would probably not want to run -too many of them at once. +"cleanup". The perform actions are probably parellizable; the cleanup may be +(but not if it has to run git commands to stage state; it can queue +commands though); the check should be easily parallelizable, although they +may access the disk or run minor git query commands, so would probably not +want to run too many of them at once. From 2d1e1e414d4c2fbaa5c1a43cc751cebad937d898 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:54:31 +0000 Subject: [PATCH 0906/8313] Added a comment --- .../comment_3_bd17c490fce8de85352938976eeb92d7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment new file mode 100644 index 0000000000..e8ac2d266b --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-02-25T19:54:31Z" + content=""" +But, see [[todo/parallel_possibilities]] +"""]] From 82d6545b6f92792f81f2de33a99b5cb5403485d1 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 25 Feb 2011 19:54:48 +0000 Subject: [PATCH 0907/8313] removed --- .../comment_3_bd17c490fce8de85352938976eeb92d7._comment | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment deleted file mode 100644 index e8ac2d266b..0000000000 --- a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_bd17c490fce8de85352938976eeb92d7._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="http://joey.kitenet.net/" - nickname="joey" - subject="comment 3" - date="2011-02-25T19:54:31Z" - content=""" -But, see [[todo/parallel_possibilities]] -"""]] From 307685c3494e59c6e92b378e3a171bb0f6a11c95 Mon Sep 17 00:00:00 2001 From: jbd Date: Sat, 26 Feb 2011 10:26:13 +0000 Subject: [PATCH 0908/8313] Added a comment --- .../comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment new file mode 100644 index 0000000000..ee769f0ddd --- /dev/null +++ b/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="jbd" + ip="89.158.228.148" + subject="comment 3" + date="2011-02-26T10:26:12Z" + content=""" +Thank your for your answer and the link ! +"""]] From 9e74d3bfc341c142b6ea14a173640831dca14450 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Sat, 26 Feb 2011 21:43:22 +0000 Subject: [PATCH 0909/8313] Added a comment: git-add instead of git-annex-add --- .../comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment diff --git a/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment new file mode 100644 index 0000000000..a4eb3cf235 --- /dev/null +++ b/doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="git-add instead of git-annex-add" + date="2011-02-26T21:43:21Z" + content=""" +would, with these modifications in place, there still be a way to *really* git-add a file? (my main repository contains both normal git and git-annex files.) +"""]] From 715faa86afcaca5ffc8f0af24a06cf9cabf54619 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkju1sxeJoVVa04plfuhH4Dp8KJOA-Gu_g" Date: Sun, 27 Feb 2011 13:59:42 +0000 Subject: [PATCH 0910/8313] --- ...ge_argument_to___34__git_annex_dropunused__34___.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn new file mode 100644 index 0000000000..97b04e2926 --- /dev/null +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -0,0 +1,9 @@ +The command `git annex dropunused` currently takes a number, as referenced in output of last `git annex unused` command. + +When you want to drop all, or a range, this may be annoying, as you have to specify each number on the command line. + +A range argument, such as `1-1845`, possibly combined with other argument types (Cf. many print dialogues: `1,3,5-7,9`) would be great. + +I work around this lack as I want to drop all unused files anyway by something like this: + + git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused From 22c8dfcd3fb8e052b2c3ae10da6ebc862d6e6efc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkju1sxeJoVVa04plfuhH4Dp8KJOA-Gu_g" Date: Sun, 27 Feb 2011 14:01:37 +0000 Subject: [PATCH 0911/8313] --- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn new file mode 100644 index 0000000000..2fed00be7e --- /dev/null +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -0,0 +1 @@ +`git annex migrate` leaves old, unlinked backend versions lying around. It would be great if these were purged automatically somehow. From b14ad6338150243b7f60d0c59aeebf7b0c6e1d03 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:17:19 -0400 Subject: [PATCH 0912/8313] responses --- ...t_to___34__git_annex_dropunused__34___.mdwn | 9 +++++++++ ...ate_leaves_old_backend_versions_around.mdwn | 18 +++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn index 97b04e2926..dc9b7acda5 100644 --- a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -7,3 +7,12 @@ A range argument, such as `1-1845`, possibly combined with other argument types I work around this lack as I want to drop all unused files anyway by something like this: git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused + +> It's designed to be used with `seq`. There's an example in the +> [[walkthrough]], and of course multiple seq calls can be used to +> specifiy multiple ranges. So: + + git annex dropunused `seq 1 9` `seq 11 1845` + +> I don't see adding my own range operations to be an improvement worth +> making; it'd arguably only be a complication. --[[Joey]] [[done]] diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index 2fed00be7e..c32b763a2d 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -1 +1,17 @@ -`git annex migrate` leaves old, unlinked backend versions lying around. It would be great if these were purged automatically somehow. +`git annex migrate` leaves old, unlinked backend versions lying around. It +would be great if these were purged automatically somehow. + +> Yes, this is an issue mentioned in the [[walkthrough#index14h2]]. +> +> Since multiple files can point to the same content, it could be that +> only one file has been migrated, and the content is still used. So +> the content either has to be retained, or an operation as expensive +> as `git annex unused` used to find it something else still uses it. +> +> Rather than adding such an +> expensive operation to each call to migrate, I focused on hard-linking +> the values for the old and new keys, so that the old keys don't actually +> use any additional resources (beyond an extra inode). +> +> This way a lot of migrations can be done, and only when you're done you +> can do the more expensive cleanup pass if you want to. --[[Joey]] From 9e49a71282def0b6d6f7507d59eb0f805c6e0073 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:19:28 -0400 Subject: [PATCH 0913/8313] typo --- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index c32b763a2d..fc8551ddb4 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -6,7 +6,7 @@ would be great if these were purged automatically somehow. > Since multiple files can point to the same content, it could be that > only one file has been migrated, and the content is still used. So > the content either has to be retained, or an operation as expensive -> as `git annex unused` used to find it something else still uses it. +> as `git annex unused` used to find if something else still uses it. > > Rather than adding such an > expensive operation to each call to migrate, I focused on hard-linking From 98e246b49b3c4fed319fe7bc1e900ba20ebfc9e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:45:48 -0400 Subject: [PATCH 0914/8313] split the walkthrough and inline back together --- doc/walkthrough.mdwn | 441 +----------------- doc/walkthrough/adding_a_remote.mdwn | 19 + doc/walkthrough/adding_files.mdwn | 11 + doc/walkthrough/backups.mdwn | 25 + doc/walkthrough/creating_a_repository.mdwn | 6 + .../fsck:_verifying_your_data.mdwn | 16 + .../fsck:_when_things_go_wrong.mdwn | 13 + doc/walkthrough/getting_file_content.mdwn | 16 + .../migrating_data_to_a_new_backend.mdwn | 16 + doc/walkthrough/modifying_annexed_files.mdwn | 43 ++ ...ing_file_content_between_repositories.mdwn | 13 + doc/walkthrough/removing_files.mdwn | 6 + .../removing_files:_When_things_go_wrong.mdwn | 24 + doc/walkthrough/renaming_files.mdwn | 13 + ...nsferring_files:_When_things_go_wrong.mdwn | 18 + doc/walkthrough/untrusted_repositories.mdwn | 28 ++ doc/walkthrough/unused_data.mdwn | 30 ++ doc/walkthrough/using_ssh_remotes.mdwn | 33 ++ doc/walkthrough/using_the_SHA1_backend.mdwn | 11 + doc/walkthrough/using_the_URL_backend.mdwn | 24 + 20 files changed, 386 insertions(+), 420 deletions(-) create mode 100644 doc/walkthrough/adding_a_remote.mdwn create mode 100644 doc/walkthrough/adding_files.mdwn create mode 100644 doc/walkthrough/backups.mdwn create mode 100644 doc/walkthrough/creating_a_repository.mdwn create mode 100644 doc/walkthrough/fsck:_verifying_your_data.mdwn create mode 100644 doc/walkthrough/fsck:_when_things_go_wrong.mdwn create mode 100644 doc/walkthrough/getting_file_content.mdwn create mode 100644 doc/walkthrough/migrating_data_to_a_new_backend.mdwn create mode 100644 doc/walkthrough/modifying_annexed_files.mdwn create mode 100644 doc/walkthrough/moving_file_content_between_repositories.mdwn create mode 100644 doc/walkthrough/removing_files.mdwn create mode 100644 doc/walkthrough/removing_files:_When_things_go_wrong.mdwn create mode 100644 doc/walkthrough/renaming_files.mdwn create mode 100644 doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn create mode 100644 doc/walkthrough/untrusted_repositories.mdwn create mode 100644 doc/walkthrough/unused_data.mdwn create mode 100644 doc/walkthrough/using_ssh_remotes.mdwn create mode 100644 doc/walkthrough/using_the_SHA1_backend.mdwn create mode 100644 doc/walkthrough/using_the_URL_backend.mdwn diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index d08b247f73..896b560ec6 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -2,423 +2,24 @@ A walkthrough of the basic features of git-annex. [[!toc]] -## creating a repository - -This is very straightforward. Just tell it a description of the repository. - - # mkdir ~/annex - # cd ~/annex - # git init - # git annex init "my laptop" - -## adding a remote - -Like any other git repository, git-annex repositories have remotes. -Let's start by adding a USB drive as a remote. - - # sudo mount /media/usb - # cd /media/usb - # git clone ~/annex - # cd annex - # git annex init "portable USB drive" - # git remote add laptop ~/annex - # cd ~/annex - # git remote add usbdrive /media/usb - -This is all standard ad-hoc distributed git repository setup. -The only git-annex specific part is telling it the name -of the new repository created on the USB drive. - -Notice that both repos are set up as remotes of one another. This lets -either get annexed files from the other. You'll want to do that even -if you are using git in a more centralized fashion. - -## adding files - - # cd ~/annex - # cp /tmp/big_file . - # cp /tmp/debian.iso . - # git annex add . - add big_file ok - add debian.iso ok - # git commit -a -m added - -When you add a file to the annex and commit it, only a symlink to -the annexed content is committed. The content itself is stored in -git-annex's backend. - -## renaming files - - # cd ~/annex - # git mv big_file my_cool_big_file - # mkdir iso - # git mv debian.iso iso/ - # git commit -m moved - -You can use any normal git operations to move files around, or even -make copies or delete them. - -Notice that, since annexed files are represented by symlinks, -the symlink will break when the file is moved into a subdirectory. -But, git-annex will fix this up for you when you commit -- -it has a pre-commit hook that watches for and corrects broken symlinks. - -## getting file content - -A repository does not always have all annexed file contents available. -When you need the content of a file, you can use "git annex get" to -make it available. - -We can use this to copy everything in the laptop's annex to the -USB drive. - - # cd /media/usb/annex - # git pull laptop master - # git annex get . - get my_cool_big_file (copying from laptop...) ok - get iso/debian.iso (copying from laptop...) ok - -Notice that you had to git pull from laptop first, this lets git-annex know -what has changed in laptop, and so it knows about the files present there and -can get them. - -## transferring files: When things go wrong - -After a while, you'll have several annexes, with different file contents. -You don't have to try to keep all that straight; git-annex does -[[location_tracking]] for you. If you ask it to get a file and the drive -or file server is not accessible, it will let you know what it needs to get -it: - - # git annex get video/hackity_hack_and_kaxxt.mov - get video/_why_hackity_hack_and_kaxxt.mov (not available) - Unable to access these remotes: usbdrive, server - Try making some of these repositories available: - 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server - 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive - ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive - failed - # sudo mount /media/usb - # git annex get video/hackity_hack_and_kaxxt.mov - get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok - # git commit -a -m "got a video I want to rewatch on the plane" - -## removing files - -You can always drop files safely. Git-annex checks that some other annex -has the file before removing it. - - # git annex drop iso/debian.iso - drop iso/Debian_5.0.iso ok - # git commit -a -m "freed up space" - -## removing files: When things go wrong - -Before dropping a file, git-annex wants to be able to look at other -remotes, and verify that they still have a file. After all, it could -have been dropped from them too. If the remotes are not mounted/available, -you'll see something like this. - - # git annex drop important_file other.iso - drop important_file (unsafe) - Could only verify the existence of 0 out of 1 necessary copies - Unable to access these remotes: usbdrive - Try making some of these repositories available: - 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive - ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive - (Use --force to override this check, or adjust annex.numcopies.) - failed - drop other.iso (unsafe) - Could only verify the existence of 0 out of 1 necessary copies - No other repository is known to contain the file. - (Use --force to override this check, or adjust annex.numcopies.) - failed - -Here you might --force it to drop `important_file` if you [[trust]] your backup. -But `other.iso` looks to have never been copied to anywhere else, so if -it's something you want to hold onto, you'd need to transfer it to -some other repository before dropping it. - -## modifying annexed files - -Normally, the content of files in the annex is prevented from being modified. -That's a good thing, because it might be the only copy, you wouldn't -want to lose it in a fumblefingered mistake. - - # echo oops > my_cool_big_file - bash: my_cool_big_file: Permission denied - -In order to modify a file, it should first be unlocked. - - # git annex unlock my_cool_big_file - unlock my_cool_big_file (copying...) ok - -That replaces the symlink that normally points at its content with a copy -of the content. You can then modify the file like any regular file. Because -it is a regular file. - -(If you decide you don't need to modify the file after all, or want to discard -modifications, just use `git annex lock`.) - -When you `git commit`, git-annex's pre-commit hook will automatically -notice that you are committing an unlocked file, and add its new content -to the annex. The file will be replaced with a symlink to the new content, -and this symlink is what gets committed to git in the end. - - # echo "now smaller, but even cooler" > my_cool_big_file - # git commit my_cool_big_file -m "changed an annexed file" - add my_cool_big_file ok - [master 64cda67] changed an annexed file - 2 files changed, 2 insertions(+), 1 deletions(-) - create mode 100644 .git-annex/WORM:1289672605:30:file.log - -There is one problem with using `git commit` like this: Git wants to first -stage the entire contents of the file in its index. That can be slow for -big files (sorta why git-annex exists in the first place). So, the -automatic handling on commit is a nice safety feature, since it prevents -the file content being accidentally committed into git. But when working with -big files, it's faster to explicitly add them to the annex yourself -before committing. - - # echo "now smaller, but even cooler yet" > my_cool_big_file - # git annex add my_cool_big_file - add my_cool_big_file ok - # git commit my_cool_big_file -m "changed an annexed file" - -## using ssh remotes - -So far in this walkthrough, git-annex has been used with a remote -repository on a USB drive. But it can also be used with a git remote -that is truely remote, a host accessed by ssh. - -Say you have a desktop on the same network as your laptop and want -to clone the laptop's annex to it: - - # git clone ssh://mylaptop/home/me/annex ~/annex - # cd ~/annex - # git annex init "my desktop" - -Now you can get files and they will be transferred (using `rsync`): - - # git annex get my_cool_big_file - get my_cool_big_file (getting UUID for origin...) (copying from origin...) - WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 - ok - -When you drop files, git-annex will ssh over to the remote and make -sure the file's content is still there before removing it locally: - - # git annex drop my_cool_big_file - drop my_cool_big_file (checking origin..) ok - -Note that normally git-annex prefers to use non-ssh remotes, like -a USB drive, before ssh remotes. They are assumed to be faster/cheaper to -access, if available. There is a annex-cost setting you can configure in -`.git/config` to adjust which repositories it prefers. See -[[the_man_page|git-annex]] for details. - -Also, note that you need full shell access for this to work -- -git-annex needs to be able to ssh in and run commands. - -## moving file content between repositories - -Often you will want to move some file contents from a repository to some -other one. For example, your laptop's disk is getting full; time to move -some files to an external disk before moving another file from a file -server to your laptop. Doing that by hand (by using `git annex get` and -`git annex drop`) is possible, but a bit of a pain. `git annex move` -makes it very easy. - - # git annex move my_cool_big_file --to usbdrive - move my_cool_big_file (moving to usbdrive...) ok - # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver - move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) - WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 - ok - -## using the URL backend - -git-annex has multiple key-value [[backends]]. So far this walkthrough has -demonstrated the default, WORM (Write Once, Read Many) backend. - -Another handy backend is the URL backend, which can fetch file's content -from remote URLs. Here's how to set up some files in your repository -that use this backend: - - # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile - fromkey somefile ok - # git commit -m "added a file from the Internet Archive" - -Now you if you ask git-annex to get that file, it will download it, -and cache it locally. - - # git annex get somefile - get somefile (downloading) - #########################################################################100.0% - ok - -You can always drop files downloaded by the URL backend. It is assumed -that the URL is stable; no local backup is kept. - - # git annex drop somefile - drop somefile (ok) - -## using the SHA1 backend - -Another handy alternative to the default [[backend|backends]] is the -SHA1 backend. This backend provides more git-style assurance that your data -has not been damaged. And the checksum means that when you add the same -content to the annex twice, only one copy need be stored in the backend. - -The only reason it's not the default is that it needs to checksum -files when they're added to the annex, and this can slow things down -significantly for really big files. To make SHA1 the default, just -add something like this to `.gitattributes`: - - * annex.backend=SHA1 - -## migrating data to a new backend - -Maybe you started out using the WORM backend, and have now configured -git-annex to use SHA1. But files you added to the annex before still -use the WORM backend. There is a simple command that can migrate that -data: - - # git annex migrate my_cool_big_file - migrate my_cool_big_file (checksum...) ok - -You can only migrate files whose content is currently available. Other -files will be skipped. - -After migrating a file to a new backend, the old content in the old backend -will still be present. That is necessary because multiple files -can point to the same content. The `git annex unused` subcommand can be -used to clear up that detritus later. Note that hard links are used, -to avoid wasting disk space. - -## unused data - -It's possible for data to accumulate in the annex that no files point to -anymore. One way it can happen is if you `git rm` a file without -first calling `git annex drop`. And, when you modify an annexed file, the old -content of the file remains in the annex. Another way is when migrating -between backends. - -This might be historical data you want to preserve, so git-annex defaults to -preserving it. So from time to time, you may want to check for such data and -eliminate it to save space. - - # git annex unused - unused (checking for unused data...) - Some annexed data is no longer pointed to by any files in the repository. - NUMBER KEY - 1 WORM:1289672605:3:file - 2 WORM:1289672605:14:file - (To see where data was previously used, try: git log --stat -S'KEY') - (To remove unwanted data: git-annex dropunused NUMBER) - ok - -After running `git annex unused`, you can follow the instructions to examine -the history of files that used the data, and if you decide you don't need that -data anymore, you can easily remove it: - - # git annex dropunused 1 - dropunused 1 ok - -Hint: To drop a lot of unused data, use a command like this: - - # git annex dropunused `seq 1 1000` - -## fsck: verifying your data - -You can use the fsck subcommand to check for problems in your data. -What can be checked depends on the [[backend|backends]] you've used to store -the data. For example, when you use the SHA1 backend, fsck will verify that -the checksums of your files are good. Fsck also checks that the annex.numcopies -setting is satisfied for all files. - - # git annex fsck - unused (checking for unused data...) ok - fsck my_cool_big_file (checksum...) ok - ... - -You can also specify the files to check. This is particularly useful if -you're using sha1 and don't want to spend a long time checksumming everything. - - # git annex fsck my_cool_big_file - fsck my_cool_big_file (checksum...) ok - -## fsck: When things go wrong - -Fsck never deletes possibly bad data; instead it will be moved to -`.git/annex/bad/` for you to recover. Here is a sample of what fsck -might say about a badly messed up annex: - - # git annex fsck - fsck my_cool_big_file (checksum...) - git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 - git-annex: ** No known copies of the file exist! - failed - fsck important_file - git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. - failed - git-annex: 2 failed - -## backups - -git-annex can be configured to require more than one copy of a file exists, -as a simple backup for your data. This is controlled by the "annex.numcopies" -setting, which defaults to 1 copy. Let's change that to require 2 copies, -and send a copy of every file to a USB drive. - - # echo "* annex.numcopies=2" >> .gitattributes - # git annex copy . --to usbdrive - -Now when we try to `git annex drop` a file, it will verify that it -knows of 2 other repositories that have a copy before removing its -content from the current repository. - -You can also vary the number of copies needed, depending on the file name. -So, if you want 3 copies of all your flac files, but only 1 copy of oggs: - - # echo "*.ogg annex.numcopies=1" >> .gitattributes - # echo "*.flac annex.numcopies=3" >> .gitattributes - -Or, you might want to make a directory for important stuff, and configure -it so anything put in there is backed up more thoroughly: - - # mkdir important_stuff - # echo "* annex.numcopies=3" > important_stuff/.gitattributes - -For more details about the numcopies setting, see [[copies]]. - -## untrusted repositories - -Suppose you have a USB thumb drive and are using it as a git annex -repository. You don't trust the drive, because you could lose it, or -accidentally run it through the laundry. Or, maybe you have a drive that -you know is dying, and you'd like to be warned if there are any files -on it not backed up somewhere else. Maybe the drive has already died -or been lost. - -You can let git-annex know that you don't trust a repository, and it will -adjust its behavior to avoid relying on that repositories's continued -availability. - - # git annex untrust usbdrive - untrust usbdrive ok - -Now when you do a fsck, you'll be warned appropriately: - - # git annex fsck . - fsck my_big_file - Only these untrusted locations may have copies of this file! - 05e296c4-2989-11e0-bf40-bad1535567fe -- portable USB drive - Back it up to trusted locations with git-annex copy. - failed - -Also, git-annex will refuse to drop a file from elsewhere just because -it can see a copy on the untrusted repository. - -It's also possible to tell git-annex that you have an unusually high -level of trust for a repository. See [[trust]] for details. +[[!inline feeds=no pagenames=""" + creating_a_repository + adding_a_remote + adding_files + renaming_files + getting_file_content + transferring_files:_When_things_go_wrong + removing_files + removing_files:_When_things_go_wrong + modifying_annexed_files + using_ssh_remotes + moving_file_content_between_repositories + using_the_URL_backend + using_the_SHA1_backend + migrating_data_to_a_new_backend + unused_data + fsck:_verifying_your_data + fsck:_when_things_go_wrong + backups + untrusted_repositories +"""]] diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn new file mode 100644 index 0000000000..be8e8e7fe5 --- /dev/null +++ b/doc/walkthrough/adding_a_remote.mdwn @@ -0,0 +1,19 @@ +Like any other git repository, git-annex repositories have remotes. +Let's start by adding a USB drive as a remote. + + # sudo mount /media/usb + # cd /media/usb + # git clone ~/annex + # cd annex + # git annex init "portable USB drive" + # git remote add laptop ~/annex + # cd ~/annex + # git remote add usbdrive /media/usb + +This is all standard ad-hoc distributed git repository setup. +The only git-annex specific part is telling it the name +of the new repository created on the USB drive. + +Notice that both repos are set up as remotes of one another. This lets +either get annexed files from the other. You'll want to do that even +if you are using git in a more centralized fashion. diff --git a/doc/walkthrough/adding_files.mdwn b/doc/walkthrough/adding_files.mdwn new file mode 100644 index 0000000000..77a7fbc154 --- /dev/null +++ b/doc/walkthrough/adding_files.mdwn @@ -0,0 +1,11 @@ + # cd ~/annex + # cp /tmp/big_file . + # cp /tmp/debian.iso . + # git annex add . + add big_file ok + add debian.iso ok + # git commit -a -m added + +When you add a file to the annex and commit it, only a symlink to +the annexed content is committed. The content itself is stored in +git-annex's backend. diff --git a/doc/walkthrough/backups.mdwn b/doc/walkthrough/backups.mdwn new file mode 100644 index 0000000000..9723022b4c --- /dev/null +++ b/doc/walkthrough/backups.mdwn @@ -0,0 +1,25 @@ +git-annex can be configured to require more than one copy of a file exists, +as a simple backup for your data. This is controlled by the "annex.numcopies" +setting, which defaults to 1 copy. Let's change that to require 2 copies, +and send a copy of every file to a USB drive. + + # echo "* annex.numcopies=2" >> .gitattributes + # git annex copy . --to usbdrive + +Now when we try to `git annex drop` a file, it will verify that it +knows of 2 other repositories that have a copy before removing its +content from the current repository. + +You can also vary the number of copies needed, depending on the file name. +So, if you want 3 copies of all your flac files, but only 1 copy of oggs: + + # echo "*.ogg annex.numcopies=1" >> .gitattributes + # echo "*.flac annex.numcopies=3" >> .gitattributes + +Or, you might want to make a directory for important stuff, and configure +it so anything put in there is backed up more thoroughly: + + # mkdir important_stuff + # echo "* annex.numcopies=3" > important_stuff/.gitattributes + +For more details about the numcopies setting, see [[copies]]. diff --git a/doc/walkthrough/creating_a_repository.mdwn b/doc/walkthrough/creating_a_repository.mdwn new file mode 100644 index 0000000000..51ff1c72b3 --- /dev/null +++ b/doc/walkthrough/creating_a_repository.mdwn @@ -0,0 +1,6 @@ +This is very straightforward. Just tell it a description of the repository. + + # mkdir ~/annex + # cd ~/annex + # git init + # git annex init "my laptop" diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn new file mode 100644 index 0000000000..cd3a47a8a9 --- /dev/null +++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn @@ -0,0 +1,16 @@ +You can use the fsck subcommand to check for problems in your data. +What can be checked depends on the [[backend|backends]] you've used to store +the data. For example, when you use the SHA1 backend, fsck will verify that +the checksums of your files are good. Fsck also checks that the annex.numcopies +setting is satisfied for all files. + + # git annex fsck + unused (checking for unused data...) ok + fsck my_cool_big_file (checksum...) ok + ... + +You can also specify the files to check. This is particularly useful if +you're using sha1 and don't want to spend a long time checksumming everything. + + # git annex fsck my_cool_big_file + fsck my_cool_big_file (checksum...) ok diff --git a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn new file mode 100644 index 0000000000..05b9f385c0 --- /dev/null +++ b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn @@ -0,0 +1,13 @@ +Fsck never deletes possibly bad data; instead it will be moved to +`.git/annex/bad/` for you to recover. Here is a sample of what fsck +might say about a badly messed up annex: + + # git annex fsck + fsck my_cool_big_file (checksum...) + git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2 + git-annex: ** No known copies of the file exist! + failed + fsck important_file + git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up. + failed + git-annex: 2 failed diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn new file mode 100644 index 0000000000..a863303cef --- /dev/null +++ b/doc/walkthrough/getting_file_content.mdwn @@ -0,0 +1,16 @@ +A repository does not always have all annexed file contents available. +When you need the content of a file, you can use "git annex get" to +make it available. + +We can use this to copy everything in the laptop's annex to the +USB drive. + + # cd /media/usb/annex + # git pull laptop master + # git annex get . + get my_cool_big_file (copying from laptop...) ok + get iso/debian.iso (copying from laptop...) ok + +Notice that you had to git pull from laptop first, this lets git-annex know +what has changed in laptop, and so it knows about the files present there and +can get them. diff --git a/doc/walkthrough/migrating_data_to_a_new_backend.mdwn b/doc/walkthrough/migrating_data_to_a_new_backend.mdwn new file mode 100644 index 0000000000..b9acb8bd15 --- /dev/null +++ b/doc/walkthrough/migrating_data_to_a_new_backend.mdwn @@ -0,0 +1,16 @@ +Maybe you started out using the WORM backend, and have now configured +git-annex to use SHA1. But files you added to the annex before still +use the WORM backend. There is a simple command that can migrate that +data: + + # git annex migrate my_cool_big_file + migrate my_cool_big_file (checksum...) ok + +You can only migrate files whose content is currently available. Other +files will be skipped. + +After migrating a file to a new backend, the old content in the old backend +will still be present. That is necessary because multiple files +can point to the same content. The `git annex unused` subcommand can be +used to clear up that detritus later. Note that hard links are used, +to avoid wasting disk space. diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn new file mode 100644 index 0000000000..3ad4e82eab --- /dev/null +++ b/doc/walkthrough/modifying_annexed_files.mdwn @@ -0,0 +1,43 @@ +Normally, the content of files in the annex is prevented from being modified. +That's a good thing, because it might be the only copy, you wouldn't +want to lose it in a fumblefingered mistake. + + # echo oops > my_cool_big_file + bash: my_cool_big_file: Permission denied + +In order to modify a file, it should first be unlocked. + + # git annex unlock my_cool_big_file + unlock my_cool_big_file (copying...) ok + +That replaces the symlink that normally points at its content with a copy +of the content. You can then modify the file like any regular file. Because +it is a regular file. + +(If you decide you don't need to modify the file after all, or want to discard +modifications, just use `git annex lock`.) + +When you `git commit`, git-annex's pre-commit hook will automatically +notice that you are committing an unlocked file, and add its new content +to the annex. The file will be replaced with a symlink to the new content, +and this symlink is what gets committed to git in the end. + + # echo "now smaller, but even cooler" > my_cool_big_file + # git commit my_cool_big_file -m "changed an annexed file" + add my_cool_big_file ok + [master 64cda67] changed an annexed file + 2 files changed, 2 insertions(+), 1 deletions(-) + create mode 100644 .git-annex/WORM:1289672605:30:file.log + +There is one problem with using `git commit` like this: Git wants to first +stage the entire contents of the file in its index. That can be slow for +big files (sorta why git-annex exists in the first place). So, the +automatic handling on commit is a nice safety feature, since it prevents +the file content being accidentally committed into git. But when working with +big files, it's faster to explicitly add them to the annex yourself +before committing. + + # echo "now smaller, but even cooler yet" > my_cool_big_file + # git annex add my_cool_big_file + add my_cool_big_file ok + # git commit my_cool_big_file -m "changed an annexed file" diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn new file mode 100644 index 0000000000..d7150f109a --- /dev/null +++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn @@ -0,0 +1,13 @@ +Often you will want to move some file contents from a repository to some +other one. For example, your laptop's disk is getting full; time to move +some files to an external disk before moving another file from a file +server to your laptop. Doing that by hand (by using `git annex get` and +`git annex drop`) is possible, but a bit of a pain. `git annex move` +makes it very easy. + + # git annex move my_cool_big_file --to usbdrive + move my_cool_big_file (moving to usbdrive...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver + move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) + WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 + ok diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn new file mode 100644 index 0000000000..85a7d50a6b --- /dev/null +++ b/doc/walkthrough/removing_files.mdwn @@ -0,0 +1,6 @@ +You can always drop files safely. Git-annex checks that some other annex +has the file before removing it. + + # git annex drop iso/debian.iso + drop iso/Debian_5.0.iso ok + # git commit -a -m "freed up space" diff --git a/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn b/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn new file mode 100644 index 0000000000..2d3c0cde08 --- /dev/null +++ b/doc/walkthrough/removing_files:_When_things_go_wrong.mdwn @@ -0,0 +1,24 @@ +Before dropping a file, git-annex wants to be able to look at other +remotes, and verify that they still have a file. After all, it could +have been dropped from them too. If the remotes are not mounted/available, +you'll see something like this. + + # git annex drop important_file other.iso + drop important_file (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + Unable to access these remotes: usbdrive + Try making some of these repositories available: + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + (Use --force to override this check, or adjust annex.numcopies.) + failed + drop other.iso (unsafe) + Could only verify the existence of 0 out of 1 necessary copies + No other repository is known to contain the file. + (Use --force to override this check, or adjust annex.numcopies.) + failed + +Here you might --force it to drop `important_file` if you [[trust]] your backup. +But `other.iso` looks to have never been copied to anywhere else, so if +it's something you want to hold onto, you'd need to transfer it to +some other repository before dropping it. diff --git a/doc/walkthrough/renaming_files.mdwn b/doc/walkthrough/renaming_files.mdwn new file mode 100644 index 0000000000..85964d1ea5 --- /dev/null +++ b/doc/walkthrough/renaming_files.mdwn @@ -0,0 +1,13 @@ + # cd ~/annex + # git mv big_file my_cool_big_file + # mkdir iso + # git mv debian.iso iso/ + # git commit -m moved + +You can use any normal git operations to move files around, or even +make copies or delete them. + +Notice that, since annexed files are represented by symlinks, +the symlink will break when the file is moved into a subdirectory. +But, git-annex will fix this up for you when you commit -- +it has a pre-commit hook that watches for and corrects broken symlinks. diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn new file mode 100644 index 0000000000..d8f0a19bd6 --- /dev/null +++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn @@ -0,0 +1,18 @@ +After a while, you'll have several annexes, with different file contents. +You don't have to try to keep all that straight; git-annex does +[[location_tracking]] for you. If you ask it to get a file and the drive +or file server is not accessible, it will let you know what it needs to get +it: + + # git annex get video/hackity_hack_and_kaxxt.mov + get video/_why_hackity_hack_and_kaxxt.mov (not available) + Unable to access these remotes: usbdrive, server + Try making some of these repositories available: + 5863d8c0-d9a9-11df-adb2-af51e6559a49 -- my home file server + 58d84e8a-d9ae-11df-a1aa-ab9aa8c00826 -- portable USB drive + ca20064c-dbb5-11df-b2fe-002170d25c55 -- backup SATA drive + failed + # sudo mount /media/usb + # git annex get video/hackity_hack_and_kaxxt.mov + get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok + # git commit -a -m "got a video I want to rewatch on the plane" diff --git a/doc/walkthrough/untrusted_repositories.mdwn b/doc/walkthrough/untrusted_repositories.mdwn new file mode 100644 index 0000000000..cdb5da7c3d --- /dev/null +++ b/doc/walkthrough/untrusted_repositories.mdwn @@ -0,0 +1,28 @@ +Suppose you have a USB thumb drive and are using it as a git annex +repository. You don't trust the drive, because you could lose it, or +accidentally run it through the laundry. Or, maybe you have a drive that +you know is dying, and you'd like to be warned if there are any files +on it not backed up somewhere else. Maybe the drive has already died +or been lost. + +You can let git-annex know that you don't trust a repository, and it will +adjust its behavior to avoid relying on that repositories's continued +availability. + + # git annex untrust usbdrive + untrust usbdrive ok + +Now when you do a fsck, you'll be warned appropriately: + + # git annex fsck . + fsck my_big_file + Only these untrusted locations may have copies of this file! + 05e296c4-2989-11e0-bf40-bad1535567fe -- portable USB drive + Back it up to trusted locations with git-annex copy. + failed + +Also, git-annex will refuse to drop a file from elsewhere just because +it can see a copy on the untrusted repository. + +It's also possible to tell git-annex that you have an unusually high +level of trust for a repository. See [[trust]] for details. diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn new file mode 100644 index 0000000000..69a581fe1e --- /dev/null +++ b/doc/walkthrough/unused_data.mdwn @@ -0,0 +1,30 @@ +It's possible for data to accumulate in the annex that no files point to +anymore. One way it can happen is if you `git rm` a file without +first calling `git annex drop`. And, when you modify an annexed file, the old +content of the file remains in the annex. Another way is when migrating +between backends. + +This might be historical data you want to preserve, so git-annex defaults to +preserving it. So from time to time, you may want to check for such data and +eliminate it to save space. + + # git annex unused + unused (checking for unused data...) + Some annexed data is no longer pointed to by any files in the repository. + NUMBER KEY + 1 WORM:1289672605:3:file + 2 WORM:1289672605:14:file + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused NUMBER) + ok + +After running `git annex unused`, you can follow the instructions to examine +the history of files that used the data, and if you decide you don't need that +data anymore, you can easily remove it: + + # git annex dropunused 1 + dropunused 1 ok + +Hint: To drop a lot of unused data, use a command like this: + + # git annex dropunused `seq 1 1000` diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn new file mode 100644 index 0000000000..831746ac0e --- /dev/null +++ b/doc/walkthrough/using_ssh_remotes.mdwn @@ -0,0 +1,33 @@ +So far in this walkthrough, git-annex has been used with a remote +repository on a USB drive. But it can also be used with a git remote +that is truely remote, a host accessed by ssh. + +Say you have a desktop on the same network as your laptop and want +to clone the laptop's annex to it: + + # git clone ssh://mylaptop/home/me/annex ~/annex + # cd ~/annex + # git annex init "my desktop" + +Now you can get files and they will be transferred (using `rsync`): + + # git annex get my_cool_big_file + get my_cool_big_file (getting UUID for origin...) (copying from origin...) + WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 + ok + +When you drop files, git-annex will ssh over to the remote and make +sure the file's content is still there before removing it locally: + + # git annex drop my_cool_big_file + drop my_cool_big_file (checking origin..) ok + +Note that normally git-annex prefers to use non-ssh remotes, like +a USB drive, before ssh remotes. They are assumed to be faster/cheaper to +access, if available. There is a annex-cost setting you can configure in +`.git/config` to adjust which repositories it prefers. See +[[the_man_page|git-annex]] for details. + +Also, note that you need full shell access for this to work -- +git-annex needs to be able to ssh in and run commands. Or at least, +your shell needs to be able to run the [[git-annex-shell]] command. diff --git a/doc/walkthrough/using_the_SHA1_backend.mdwn b/doc/walkthrough/using_the_SHA1_backend.mdwn new file mode 100644 index 0000000000..c04729e2cd --- /dev/null +++ b/doc/walkthrough/using_the_SHA1_backend.mdwn @@ -0,0 +1,11 @@ +Another handy alternative to the default [[backend|backends]] is the +SHA1 backend. This backend provides more git-style assurance that your data +has not been damaged. And the checksum means that when you add the same +content to the annex twice, only one copy need be stored in the backend. + +The only reason it's not the default is that it needs to checksum +files when they're added to the annex, and this can slow things down +significantly for really big files. To make SHA1 the default, just +add something like this to `.gitattributes`: + + * annex.backend=SHA1 diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn new file mode 100644 index 0000000000..fe79a6be2e --- /dev/null +++ b/doc/walkthrough/using_the_URL_backend.mdwn @@ -0,0 +1,24 @@ +git-annex has multiple key-value [[backends]]. So far this walkthrough has +demonstrated the default, WORM (Write Once, Read Many) backend. + +Another handy backend is the URL backend, which can fetch file's content +from remote URLs. Here's how to set up some files in your repository +that use this backend: + + # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + fromkey somefile ok + # git commit -m "added a file from the Internet Archive" + +Now you if you ask git-annex to get that file, it will download it, +and cache it locally. + + # git annex get somefile + get somefile (downloading) + #########################################################################100.0% + ok + +You can always drop files downloaded by the URL backend. It is assumed +that the URL is stable; no local backup is kept. + + # git annex drop somefile + drop somefile (ok) From 78fa8c1b439ef6a5ea4da04b6cff1d7174818c7e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:47:43 -0400 Subject: [PATCH 0915/8313] walkthrough deep links --- ...dd_range_argument_to___34__git_annex_dropunused__34___.mdwn | 2 +- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn index dc9b7acda5..bbe6007a87 100644 --- a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn +++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn @@ -9,7 +9,7 @@ I work around this lack as I want to drop all unused files anyway by something l git annex unused | grep -o -P "^ [0-9]+" | xargs git annex dropunused > It's designed to be used with `seq`. There's an example in the -> [[walkthrough]], and of course multiple seq calls can be used to +> [[walkthrough|walkthrough/unused_data]], and of course multiple seq calls can be used to > specifiy multiple ranges. So: git annex dropunused `seq 1 9` `seq 11 1845` diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index fc8551ddb4..e37ee06bb3 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -1,7 +1,8 @@ `git annex migrate` leaves old, unlinked backend versions lying around. It would be great if these were purged automatically somehow. -> Yes, this is an issue mentioned in the [[walkthrough#index14h2]]. +> Yes, this is an issue mentioned in the +> [[walkthrough|walkthrough/migrating_data_to_a_new_backend]]. > > Since multiple files can point to the same content, it could be that > only one file has been migrated, and the content is still used. So From 9a4e54a5e7200a0ff6e3b3741174b4c10a3a3b44 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:52:27 -0400 Subject: [PATCH 0916/8313] custom template for walkthrough inline --- doc/templates/walkthrough.tmpl | 2 ++ doc/walkthrough.mdwn | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 doc/templates/walkthrough.tmpl diff --git a/doc/templates/walkthrough.tmpl b/doc/templates/walkthrough.tmpl new file mode 100644 index 0000000000..a500a5a865 --- /dev/null +++ b/doc/templates/walkthrough.tmpl @@ -0,0 +1,2 @@ +

+ diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 896b560ec6..ea2ee63546 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -2,7 +2,7 @@ A walkthrough of the basic features of git-annex. [[!toc]] -[[!inline feeds=no pagenames=""" +[[!inline feeds=no template=walkthrough pagenames=""" creating_a_repository adding_a_remote adding_files From 1bf6ed740e2f9fce319ec822585a60a114029b3d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 27 Feb 2011 12:55:13 -0400 Subject: [PATCH 0917/8313] show all --- doc/walkthrough.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ea2ee63546..f400191c61 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -2,7 +2,7 @@ A walkthrough of the basic features of git-annex. [[!toc]] -[[!inline feeds=no template=walkthrough pagenames=""" +[[!inline feeds=no show=0 template=walkthrough pagenames=""" creating_a_repository adding_a_remote adding_files From 7e5678bcf7cd78bd04520117201be37dc9d4d544 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 28 Feb 2011 16:09:17 -0400 Subject: [PATCH 0918/8313] add ShellParam type, for type-checked shell params --- Utility.hs | 145 +++++++++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 66 deletions(-) diff --git a/Utility.hs b/Utility.hs index 6a90e3cd50..b5c0dd617e 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,6 +6,8 @@ -} module Utility ( + ShellParam(..), + toShell, hGetContentsStrict, readFileStrict, parentDir, @@ -15,7 +17,6 @@ module Utility ( boolSystem, shellEscape, shellUnEscape, - utilityEscape, unsetFileMode, readMaybe, safeWriteFile, @@ -41,6 +42,83 @@ import Foreign (complement) import Data.List import Control.Monad (liftM2) +{- A type for parameters passed to a shell command. A command can + - be passed either some Params (multiple parameters can be included, + - whitespace-separated, or a single Param (for when parameters contain + - whitespace), or a File. + -} +data ShellParam = Params String | Param String | File FilePath + deriving (Eq, Show, Ord) + +{- When converting ShellParam to a String in preparation for passing to + - a shell command, Files that start with a dash are modified to avoid + - the shell command interpreting them as options. -} +toShell :: [ShellParam] -> [String] +toShell l = concat $ map unwrap l + where + unwrap (Param s) = [s] + unwrap (Params s) = filter (not . null) (split " " s) + unwrap (File ('-':s)) = ["./-" ++ s] + unwrap (File (s)) = [s] + +{- Run a system command, and returns True or False + - if it succeeded or failed. + - + - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. + -} +boolSystem :: FilePath -> [ShellParam] -> IO Bool +boolSystem command params = do + -- Going low-level because all the high-level system functions + -- block SIGINT etc. We need to block SIGCHLD, but allow + -- SIGINT to do its default program termination. + let sigset = addSignal sigCHLD emptySignalSet + oldint <- installHandler sigINT Default Nothing + oldset <- getSignalMask + blockSignals sigset + childpid <- forkProcess $ childaction oldint oldset + mps <- getProcessStatus True False childpid + restoresignals oldint oldset + case mps of + Just (Exited ExitSuccess) -> return True + _ -> return False + where + restoresignals oldint oldset = do + _ <- installHandler sigINT oldint Nothing + setSignalMask oldset + childaction oldint oldset = do + restoresignals oldint oldset + executeFile command True (toShell params) Nothing + +{- Escapes a filename to be safely able to be exposed to the shell. -} +shellEscape :: FilePath -> String +shellEscape f = "'" ++ escaped ++ "'" + where + -- replace ' with '"'"' + escaped = join "'\"'\"'" $ split "'" f + +{- Unescapes a set of shellEscaped words or filenames. -} +shellUnEscape :: String -> [String] +shellUnEscape [] = [] +shellUnEscape s = word:(shellUnEscape rest) + where + (word, rest) = findword "" s + findword w [] = (w, "") + findword w (c:cs) + | c == ' ' = (w, cs) + | c == '\'' = inquote c w cs + | c == '"' = inquote c w cs + | otherwise = findword (w++[c]) cs + inquote _ w [] = (w, "") + inquote q w (c:cs) + | c == q = findword w cs + | otherwise = inquote q (w++[c]) cs + +{- For quickcheck. -} +prop_idempotent_shellEscape :: String -> Bool +prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) +prop_idempotent_shellEscape_multiword :: [String] -> Bool +prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) + {- A version of hgetContents that is not lazy. Ensures file is - all read before it gets closed. -} hGetContentsStrict :: Handle -> IO String @@ -128,71 +206,6 @@ prop_relPathDirToDir_basics from to where r = relPathDirToDir from to -{- Run a system command, and returns True or False - - if it succeeded or failed. - - - - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. - -} -boolSystem :: FilePath -> [String] -> IO Bool -boolSystem command params = do - -- Going low-level because all the high-level system functions - -- block SIGINT etc. We need to block SIGCHLD, but allow - -- SIGINT to do its default program termination. - let sigset = addSignal sigCHLD emptySignalSet - oldint <- installHandler sigINT Default Nothing - oldset <- getSignalMask - blockSignals sigset - childpid <- forkProcess $ childaction oldint oldset - mps <- getProcessStatus True False childpid - restoresignals oldint oldset - case mps of - Just (Exited ExitSuccess) -> return True - _ -> return False - where - restoresignals oldint oldset = do - _ <- installHandler sigINT oldint Nothing - setSignalMask oldset - childaction oldint oldset = do - restoresignals oldint oldset - executeFile command True params Nothing - -{- Escapes a filename to be safely able to be exposed to the shell. -} -shellEscape :: FilePath -> String -shellEscape f = "'" ++ escaped ++ "'" - where - -- replace ' with '"'"' - escaped = join "'\"'\"'" $ split "'" f - -{- Unescapes a set of shellEscaped words or filenames. -} -shellUnEscape :: String -> [String] -shellUnEscape [] = [] -shellUnEscape s = word:(shellUnEscape rest) - where - (word, rest) = findword "" s - findword w [] = (w, "") - findword w (c:cs) - | c == ' ' = (w, cs) - | c == '\'' = inquote c w cs - | c == '"' = inquote c w cs - | otherwise = findword (w++[c]) cs - inquote _ w [] = (w, "") - inquote q w (c:cs) - | c == q = findword w cs - | otherwise = inquote q (w++[c]) cs - -{- Ensures that a filename is safe to pass to a utility program. In particular - - since utilities tend to interpret things starting with a dash as - - an option, relative filenames starting with a dash are escaped. -} -utilityEscape :: FilePath -> FilePath -utilityEscape ('-':s) = "./-" ++ s -utilityEscape s = s - -{- For quickcheck. -} -prop_idempotent_shellEscape :: String -> Bool -prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s) -prop_idempotent_shellEscape_multiword :: [String] -> Bool -prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s) - {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} unsetFileMode :: FilePath -> FileMode -> IO () From fcdc4797a9ab2b792a9bb20f2ca9802b8f6d5a1e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 28 Feb 2011 16:10:16 -0400 Subject: [PATCH 0919/8313] use ShellParam type So, I have a type checked safe handling of filenames starting with dashes, throughout the code. --- Annex.hs | 5 +++-- Backend/SHA1.hs | 2 +- Backend/URL.hs | 2 +- Command/Add.hs | 3 ++- Command/Fix.hs | 2 +- Command/FromKey.hs | 2 +- Command/Init.hs | 17 ++++++++++----- Command/Lock.hs | 5 +++-- Command/Map.hs | 9 ++++---- Command/Move.hs | 9 ++++---- Command/PreCommit.hs | 3 ++- Command/SetKey.hs | 2 +- Command/Unannex.hs | 4 ++-- Content.hs | 2 +- CopyFile.hs | 14 ++++++------- GitQueue.hs | 11 +++++----- GitRepo.hs | 44 +++++++++++++++++++------------------- Remotes.hs | 50 ++++++++++++++++++++++++-------------------- RsyncFile.hs | 29 +++++++++++++------------ Trust.hs | 8 +++++-- Upgrade.hs | 3 ++- Utility.hs | 11 +++++----- git-annex-shell.hs | 2 +- test.hs | 36 +++++++++++++++---------------- 24 files changed, 151 insertions(+), 124 deletions(-) diff --git a/Annex.hs b/Annex.hs index 5496ada67c..cb662a1307 100644 --- a/Annex.hs +++ b/Annex.hs @@ -24,6 +24,7 @@ import Control.Monad.State import qualified GitRepo as Git import qualified GitQueue import qualified BackendTypes +import Utility -- git-annex's monad type Annex = StateT AnnexState IO @@ -91,7 +92,7 @@ gitRepo :: Annex Git.Repo gitRepo = getState repo {- Adds a git command to the queue. -} -queue :: String -> [String] -> FilePath -> Annex () +queue :: String -> [ShellParam] -> FilePath -> Annex () queue command params file = do state <- get let q = repoqueue state @@ -110,7 +111,7 @@ queueRun = do setConfig :: String -> String -> Annex () setConfig k value = do g <- Annex.gitRepo - liftIO $ Git.run g ["config", k, value] + liftIO $ Git.run g "config" [Param k, Param value] -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g Annex.changeState $ \s -> s { Annex.repo = g' } diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index e1830bc134..a7f592b73e 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -32,7 +32,7 @@ backend = Backend.File.backend { sha1 :: FilePath -> Annex String sha1 file = do showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" [utilityEscape file] $ \h -> do + liftIO $ pOpen ReadFromPipe "sha1sum" (toShell [File file]) $ \h -> do line <- hGetLine h let bits = split " " line if null bits diff --git a/Backend/URL.hs b/Backend/URL.hs index 15cc88d644..864c793010 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -51,6 +51,6 @@ downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do showNote "downloading" showProgress -- make way for curl progress bar - liftIO $ boolSystem "curl" ["-#", "-o", utilityEscape file, url] + liftIO $ boolSystem "curl" [Params "-# -o", File file, File url] where url = join ":" $ drop 1 $ split ":" $ show key diff --git a/Command/Add.hs b/Command/Add.hs index 4b49297fc6..26e7fa258d 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -17,6 +17,7 @@ import LocationLog import Types import Content import Messages +import Utility command :: [Command] command = [Command "add" paramPath seek "add files to annex"] @@ -52,5 +53,5 @@ cleanup file key = do link <- calcGitLink file key liftIO $ createSymbolicLink link file - Annex.queue "add" ["--"] file + Annex.queue "add" [Param "--"] file return True diff --git a/Command/Fix.hs b/Command/Fix.hs index d67eca164c..0047548715 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -44,5 +44,5 @@ perform file link = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" ["--"] file + Annex.queue "add" [Param "--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 8817942580..d16eff8466 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -47,5 +47,5 @@ perform file = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" ["--"] file + Annex.queue "add" [Param "--"] file return True diff --git a/Command/Init.hs b/Command/Init.hs index 2976b988d1..1074d100ea 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -51,8 +51,12 @@ cleanup :: CommandCleanup cleanup = do g <- Annex.gitRepo logfile <- uuidLog - liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-q", "-m", "git annex init", logfile] + liftIO $ Git.run g "add" [File logfile] + liftIO $ Git.run g "commit" + [ Params "-q -m" + , Param "git annex init" + , File logfile + ] return True {- configure git to use union merge driver on state files, if it is not @@ -72,9 +76,12 @@ gitAttributesWrite repo = do where attributes = Git.attributes repo commit = do - Git.run repo ["add", attributes] - Git.run repo ["commit", "-q", "-m", "git-annex setup", - attributes] + Git.run repo "add" [Param attributes] + Git.run repo "commit" + [ Params "-q -m" + , Param "git-annex setup" + , Param attributes + ] attrLine :: String attrLine = stateDir "*.log merge=union" diff --git a/Command/Lock.hs b/Command/Lock.hs index 00a553e956..a3a39a9078 100644 --- a/Command/Lock.hs +++ b/Command/Lock.hs @@ -14,6 +14,7 @@ import Command import Messages import qualified Annex import qualified GitRepo as Git +import Utility command :: [Command] command = [Command "lock" paramPath seek "undo unlock command"] @@ -32,7 +33,7 @@ perform file = do liftIO $ removeFile file g <- Annex.gitRepo -- first reset the file to drop any changes checked into the index - liftIO $ Git.run g ["reset", "-q", "--", file] + liftIO $ Git.run g "reset" [Params "-q --", File file] -- checkout the symlink - liftIO $ Git.run g ["checkout", "--", file] + liftIO $ Git.run g "checkout" [Param "--", File file] return $ Just $ return True -- no cleanup needed diff --git a/Command/Map.hs b/Command/Map.hs index 0a3bb9fff0..00b5fc21b2 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -44,7 +44,7 @@ start = do liftIO $ writeFile file (drawMap rs umap trusted) showLongNote $ "running: dot -Tx11 " ++ file showProgress - r <- liftIO $ boolSystem "dot" ["-Tx11", file] + r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file] return $ Just $ return $ Just $ return r where file = "map.dot" @@ -198,7 +198,7 @@ tryScan r Left _ -> return Nothing Right r' -> return $ Just r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd params $ + pOpen ReadFromPipe cmd (toShell params) $ Git.hConfigRead r configlist = @@ -208,8 +208,9 @@ tryScan r let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ "git config --list" - liftIO $ pipedconfig "ssh" $ - words sshoptions ++ [Git.urlHostFull r, sshcmd] + liftIO $ pipedconfig "ssh" $ map Param $ + words sshoptions ++ + [Git.urlHostFull r, sshcmd] -- First, try sshing and running git config manually, -- only fall back to git-annex-shell configlist if that diff --git a/Command/Move.hs b/Command/Move.hs index 6dc2e48746..8c19539fba 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -56,7 +56,7 @@ remoteHasKey remote key present = do g <- Annex.gitRepo remoteuuid <- getUUID remote logfile <- liftIO $ logChange g key remoteuuid status - Annex.queue "add" ["--"] logfile + Annex.queue "add" [Param "--"] logfile where status = if present then ValuePresent else ValueMissing @@ -130,9 +130,10 @@ fromPerform src move key = do fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup fromCleanup src True key = do ok <- Remotes.onRemote src (boolSystem, False) "dropkey" - ["--quiet", "--force", - "--backend=" ++ backendName key, - keyName key] + [ Params "--quiet --force" + , Param $ "--backend=" ++ backendName key + , Param $ keyName key + ] -- better safe than sorry: assume the src dropped the key -- even if it seemed to fail; the failure could have occurred -- after it really dropped it diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 750997f540..d2f6964343 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -15,6 +15,7 @@ import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix import Messages +import Utility command :: [Command] command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"] @@ -41,6 +42,6 @@ cleanup file = do -- drop that and run command queued by Add.state to -- stage the symlink g <- Annex.gitRepo - liftIO $ Git.run g ["reset", "-q", "--", file] + liftIO $ Git.run g "reset" [Params "-q --", File file] Annex.queueRun return True diff --git a/Command/SetKey.hs b/Command/SetKey.hs index 025fb74d62..fdda1c3bee 100644 --- a/Command/SetKey.hs +++ b/Command/SetKey.hs @@ -36,7 +36,7 @@ perform file = do ok <- getViaTmp key $ \dest -> do if dest /= file then liftIO $ - boolSystem "mv" [utilityEscape file, utilityEscape dest] + boolSystem "mv" [File file, File dest] else return True if ok then return $ Just $ cleanup diff --git a/Command/Unannex.hs b/Command/Unannex.hs index 19cb1624e5..42dc1fb0ab 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -58,7 +58,7 @@ cleanup file key = do g <- Annex.gitRepo liftIO $ removeFile file - liftIO $ Git.run g ["rm", "--quiet", "--", file] + liftIO $ Git.run g "rm" [Params "--quiet --", File file] -- git rm deletes empty directories; put them back liftIO $ createDirectoryIfMissing True (parentDir file) @@ -68,6 +68,6 @@ cleanup file key = do -- Commit staged changes at end to avoid confusing the -- pre-commit hook if this file is later added back to -- git as a normal, non-annexed file. - Annex.queue "commit" ["-m", "content removed from git annex"] "-a" + Annex.queue "commit" [Params "-a -m", Param "content removed from git annex"] "-a" return True diff --git a/Content.hs b/Content.hs index 345599dbad..cb954f4a0b 100644 --- a/Content.hs +++ b/Content.hs @@ -60,7 +60,7 @@ logStatus key status = do g <- Annex.gitRepo u <- getUUID g logfile <- liftIO $ logChange g key u status - Annex.queue "add" ["--"] logfile + Annex.queue "add" [Param "--"] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/CopyFile.hs b/CopyFile.hs index 73d911a291..4575fb08ad 100644 --- a/CopyFile.hs +++ b/CopyFile.hs @@ -20,14 +20,12 @@ copyFile src dest = do e <- doesFileExist dest when e $ removeFile dest - boolSystem "cp" opts + boolSystem "cp" [params, File src, File dest] where - opts = if SysConfig.cp_reflink_auto - then ["--reflink=auto", src', dest'] + params = if SysConfig.cp_reflink_auto + then Params "--reflink=auto" else if SysConfig.cp_a - then ["-a", src', dest'] + then Params "-a" else if SysConfig.cp_p - then ["-p", src', dest'] - else [src', dest'] - src' = utilityEscape src - dest' = utilityEscape dest + then Params "-p" + else Params "" diff --git a/GitQueue.hs b/GitQueue.hs index 4a777af4db..328243fa00 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -17,6 +17,7 @@ import System.IO import System.Cmd.Utils import Data.String.Utils import Control.Monad (unless, forM_) +import Utility import qualified GitRepo as Git @@ -24,7 +25,7 @@ import qualified GitRepo as Git - is not included, and must be able to be appended after the params. -} data Action = Action { getSubcommand :: String, - getParams :: [String] + getParams :: [ShellParam] } deriving (Show, Eq, Ord) {- A queue of actions to perform (in any order) on a git repository, @@ -37,7 +38,7 @@ empty :: Queue empty = M.empty {- Adds an action to a queue. -} -add :: Queue -> String -> [String] -> FilePath -> Queue +add :: Queue -> String -> [ShellParam] -> FilePath -> Queue add queue subcommand params file = M.insertWith (++) action [file] queue where action = Action subcommand params @@ -55,7 +56,7 @@ runAction :: Git.Repo -> Action -> [FilePath] -> IO () runAction repo action files = do unless (null files) runxargs where - runxargs = pOpen WriteToPipe "xargs" ("-0":gitcmd) feedxargs - gitcmd = "git" : Git.gitCommandLine repo - (getSubcommand action:getParams action) + runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs + params = toShell $ Git.gitCommandLine repo + (Param (getSubcommand action):getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index 7cf0891eda..3f2acdcf4f 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -243,16 +243,18 @@ urlPath Repo { location = Url u } = uriPath u urlPath repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} -gitCommandLine :: Repo -> [String] -> [String] +gitCommandLine :: Repo -> [ShellParam] -> [ShellParam] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree - ["--git-dir=" ++ d ++ "/" ++ gitDir repo, "--work-tree=" ++ d] ++ params + [ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo) + , Param ("--work-tree=" ++ d) + ] ++ params gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} -run :: Repo -> [String] -> IO () -run repo params = assertLocal repo $ do - ok <- boolSystem "git" (gitCommandLine repo params) +run :: Repo -> String -> [ShellParam] -> IO () +run repo subcommand params = assertLocal repo $ do + ok <- boolSystem "git" (gitCommandLine repo ((Param subcommand):params)) unless ok $ error $ "git " ++ show params ++ " failed" {- Runs a git subcommand and returns it output, lazily. @@ -260,9 +262,9 @@ run repo params = assertLocal repo $ do - Note that this leaves the git process running, and so zombies will - result unless reap is called. -} -pipeRead :: Repo -> [String] -> IO String +pipeRead :: Repo -> [ShellParam] -> IO String pipeRead repo params = assertLocal repo $ do - (_, s) <- pipeFrom "git" (gitCommandLine repo params) + (_, s) <- pipeFrom "git" $ toShell $ gitCommandLine repo params return s {- Reaps any zombie git processes. -} @@ -277,13 +279,13 @@ reap = do {- Scans for files that are checked into git at the specified locations. -} inRepo :: Repo -> [FilePath] -> IO [FilePath] inRepo repo l = pipeNullSplit repo $ - ["ls-files", "--cached", "--exclude-standard", "-z", "--"] ++ l + [Params "ls-files --cached --exclude-standard -z --"] ++ map File l {- Scans for files at the specified locations that are not checked into git, - and not gitignored. -} notInRepo :: Repo -> [FilePath] -> IO [FilePath] notInRepo repo l = pipeNullSplit repo $ - ["ls-files", "--others", "--exclude-standard", "-z", "--"] ++ l + [Params "ls-files --others --exclude-standard -z --"] ++ map File l {- Returns a list of all files that are staged for commit. -} stagedFiles :: Repo -> [FilePath] -> IO [FilePath] @@ -292,38 +294,38 @@ stagedFiles repo l = stagedFiles' repo l [] {- Returns a list of the files, staged for commit, that are being added, - moved, or changed (but not deleted), from the specified locations. -} stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath] -stagedFilesNotDeleted repo l = stagedFiles' repo l ["--diff-filter=ACMRT"] +stagedFilesNotDeleted repo l = stagedFiles' repo l [Param "--diff-filter=ACMRT"] -stagedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath] +stagedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where - start = ["diff", "--cached", "--name-only", "-z"] - end = ["--"] ++ l + start = [Params "diff --cached --name-only -z"] + end = [Param "--"] ++ map File l {- Returns a list of files that have unstaged changes. -} changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath] changedUnstagedFiles repo l = pipeNullSplit repo $ - ["diff", "--name-only", "-z", "--"] ++ l + [Params "diff --name-only -z --"] ++ map File l {- Returns a list of the files in the specified locations that are staged - for commit, and whose type has changed. -} typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath] -typeChangedStagedFiles repo l = typeChangedFiles' repo l ["--cached"] +typeChangedStagedFiles repo l = typeChangedFiles' repo l [Param "--cached"] {- Returns a list of the files in the specified locations whose type has - changed. Files only staged for commit will not be included. -} typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedFiles repo l = typeChangedFiles' repo l [] -typeChangedFiles' :: Repo -> [FilePath] -> [String] -> IO [FilePath] +typeChangedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where - start = ["diff", "--name-only", "--diff-filter=T", "-z"] - end = ["--"] ++ l + start = [Params "diff --name-only --diff-filter=T -z"] + end = [Param "--"] ++ map File l {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} -pipeNullSplit :: Repo -> [String] -> IO [FilePath] +pipeNullSplit :: Repo -> [ShellParam] -> IO [FilePath] pipeNullSplit repo params = do fs0 <- pipeRead repo params return $ split0 fs0 @@ -408,11 +410,11 @@ checkAttr repo attr files = do -- directory. Convert to absolute, and then convert the filenames -- in its output back to relative. absfiles <- mapM absPath files - (_, s) <- pipeBoth "git" params $ join "\0" absfiles + (_, s) <- pipeBoth "git" (toShell params) $ join "\0" absfiles cwd <- getCurrentDirectory return $ map (topair $ cwd++"/") $ lines s where - params = gitCommandLine repo ["check-attr", attr, "-z", "--stdin"] + params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"] topair cwd l = (relfile, value) where relfile diff --git a/Remotes.hs b/Remotes.hs index c7e69aad8b..1523e67509 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -64,7 +64,7 @@ tryGitConfigRead r Left _ -> return r Right r' -> return r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd params $ + pOpen ReadFromPipe cmd (toShell params) $ Git.hConfigRead r store a = do r' <- a @@ -154,7 +154,7 @@ inAnnex r key = if Git.repoIsUrl r checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" - ["--backend=" ++ backendName key, keyName key] + [Param ("--backend=" ++ backendName key), Param (keyName key)] return $ Right inannex {- Cost Ordered list of remotes. -} @@ -263,28 +263,31 @@ rsynchelper r sending key file = do {- Generates rsync parameters that ssh to the remote and asks it - to either receive or send the key's content. -} -rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [String] +rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [ShellParam] rsyncParams r sending key file = do - -- Note that the command is terminated with "--", because - -- rsync will tack on its own options to this command, - -- and they need to be ignored. - shellcmd <- git_annex_shell r + Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - ["--backend=" ++ backendName key, keyName key, "--"] + [ Param $ "--backend=" ++ backendName key + , Param $ keyName key + -- Command is terminated with "--", because + -- rsync will tack on its own options afterwards, + -- and they need to be ignored. + , Param "--" + ] -- Convert the ssh command into rsync command line. - let eparam = rsyncShell $ fromJust shellcmd + let eparam = rsyncShell (Param shellcmd:shellparams) o <- repoConfig r "rsync-options" "" - let base = options ++ words o ++ eparam + let base = options ++ map Param (words o) ++ eparam if sending - then return $ base ++ [dummy, file] - else return $ base ++ [file, dummy] + then return $ base ++ [dummy, File file] + else return $ base ++ [File file, dummy] where -- inplace makes rsync resume partial files - options = ["-p", "--progress", "--inplace"] + options = [Params "-p --progress --inplace"] -- the rsync shell parameter controls where rsync -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. - dummy = ":" + dummy = Param ":" {- Uses a supplied function to run a git-annex-shell command on a remote. - @@ -292,30 +295,31 @@ rsyncParams r sending key file = do - a specified error value. -} onRemote :: Git.Repo - -> (String -> [String] -> IO a, a) + -> (FilePath -> [ShellParam] -> IO a, a) -> String - -> [String] + -> [ShellParam] -> Annex a onRemote r (with, errorval) command params = do s <- git_annex_shell r command params case s of - Just shellcmd -> liftIO $ with (shellcmd !! 0) (tail shellcmd) + Just (c, ps) -> liftIO $ with c ps Nothing -> return errorval {- Generates parameters to run a git-annex-shell command on a remote. -} -git_annex_shell :: Git.Repo -> String -> [String] -> Annex (Maybe [String]) +git_annex_shell :: Git.Repo -> String -> [ShellParam] -> Annex (Maybe (FilePath, [ShellParam])) git_annex_shell r command params - | not $ Git.repoIsUrl r = return $ Just (shellcmd:shellopts) + | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" - return $ Just $ ["ssh"] ++ words sshoptions ++ - [Git.urlHostFull r, sshcmd] + return $ Just ("ssh", map Param (words sshoptions) ++ + [Param (Git.urlHostFull r), Param sshcmd]) | otherwise = return Nothing where dir = Git.workTree r shellcmd = "git-annex-shell" - shellopts = command:dir:params - sshcmd = shellcmd ++ " " ++ unwords (map shellEscape shellopts) + shellopts = (Param command):(File dir):params + sshcmd = shellcmd ++ " " ++ + unwords (map shellEscape $ toShell shellopts) {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} diff --git a/RsyncFile.hs b/RsyncFile.hs index 9de2e2c594..149b45b11b 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -14,8 +14,8 @@ import Utility {- Generates parameters to make rsync use a specified command as its remote - shell. -} -rsyncShell :: [String] -> [String] -rsyncShell command = ["-e", unwords $ map escape command] +rsyncShell :: [ShellParam] -> [ShellParam] +rsyncShell command = [Param "-e", Param $ unwords $ map escape (toShell command)] where {- rsync requires some weird, non-shell like quoting in - here. A doubled single quote inside the single quoted @@ -25,22 +25,25 @@ rsyncShell command = ["-e", unwords $ map escape command] {- Runs rsync in server mode to send a file, and exits. -} rsyncServerSend :: FilePath -> IO () rsyncServerSend file = rsyncExec $ - rsyncServerParams ++ ["--sender", utilityEscape file] + rsyncServerParams ++ [Param "--sender", File file] {- Runs rsync in server mode to receive a file. -} rsyncServerReceive :: FilePath -> IO Bool -rsyncServerReceive file = rsync $ rsyncServerParams ++ [utilityEscape file] +rsyncServerReceive file = rsync $ rsyncServerParams ++ [File file] -rsyncServerParams :: [String] +rsyncServerParams :: [ShellParam] rsyncServerParams = - [ "--server" - , "-p" -- preserve permissions - , "--inplace" -- allow resuming of transfers of big files - , "-e.Lsf", "." -- other options rsync normally uses in server mode + [ Param "--server" + -- preserve permissions + , Param "-p" + -- allow resuming of transfers of big files + , Param "--inplace" + -- other options rsync normally uses in server mode + , Params "-e.Lsf ." ] -rsync :: [String] -> IO Bool -rsync params = boolSystem "rsync" params +rsync :: [ShellParam] -> IO Bool +rsync = boolSystem "rsync" -rsyncExec :: [String] -> IO () -rsyncExec params = executeFile "rsync" True params Nothing +rsyncExec :: [ShellParam] -> IO () +rsyncExec params = executeFile "rsync" True (toShell params) Nothing diff --git a/Trust.hs b/Trust.hs index 695059a932..7b2cf9ff88 100644 --- a/Trust.hs +++ b/Trust.hs @@ -81,8 +81,12 @@ trustSet uuid level = do logfile <- trustLog liftIO $ safeWriteFile logfile (serialize m') g <- Annex.gitRepo - liftIO $ Git.run g ["add", logfile] - liftIO $ Git.run g ["commit", "-q", "-m", "git annex trust change", logfile] + liftIO $ Git.run g "add" [File logfile] + liftIO $ Git.run g "commit" + [ Params "-q -m" + , Param "git annex trust change" + , File logfile + ] where serialize m = unlines $ map showpair $ M.toList m showpair (u, t) = u ++ " " ++ show t diff --git a/Upgrade.hs b/Upgrade.hs index b584b2666f..3c16bcc862 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -22,6 +22,7 @@ import qualified Annex import qualified Backend import Messages import Version +import Utility {- Uses the annex.version git config setting to automate upgrades. -} upgrade :: Annex Bool @@ -62,7 +63,7 @@ upgradeFrom0 = do link <- calcGitLink f k liftIO $ removeFile f liftIO $ createSymbolicLink link f - Annex.queue "add" ["--"] f + Annex.queue "add" [Param "--"] f fixlinks fs getKeysPresent0' :: FilePath -> Annex [Key] diff --git a/Utility.hs b/Utility.hs index b5c0dd617e..90494a0c44 100644 --- a/Utility.hs +++ b/Utility.hs @@ -1,6 +1,6 @@ {- git-annex utility functions - - - Copyright 2010 Joey Hess + - Copyright 2010-2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -50,16 +50,17 @@ import Control.Monad (liftM2) data ShellParam = Params String | Param String | File FilePath deriving (Eq, Show, Ord) -{- When converting ShellParam to a String in preparation for passing to - - a shell command, Files that start with a dash are modified to avoid - - the shell command interpreting them as options. -} +{- Used to pass a list of ShellParams to a function that runs + - a shell command and expects Strings. -} toShell :: [ShellParam] -> [String] toShell l = concat $ map unwrap l where unwrap (Param s) = [s] unwrap (Params s) = filter (not . null) (split " " s) + -- Files that start with a dash are modified to avoid + -- the shell command interpreting them as options. unwrap (File ('-':s)) = ["./-" ++ s] - unwrap (File (s)) = [s] + unwrap (File s) = [s] {- Run a system command, and returns True or False - if it succeeded or failed. diff --git a/git-annex-shell.hs b/git-annex-shell.hs index fee4091ef8..aeaadcbf85 100644 --- a/git-annex-shell.hs +++ b/git-annex-shell.hs @@ -66,7 +66,7 @@ builtin cmd dir params = do external :: [String] -> IO () external params = do - ret <- boolSystem "git-shell" ("-c":(filterparams params)) + ret <- boolSystem "git-shell" $ map Param $ ("-c":filterparams params) when (not ret) $ error "git-shell failed" diff --git a/test.hs b/test.hs index 9b50bcb2ef..1bae3bd83e 100644 --- a/test.hs +++ b/test.hs @@ -105,8 +105,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup] git_annex "add" ["-q", annexedfile] @? "add failed" annexed_present annexedfile writeFile ingitfile $ content ingitfile - Utility.boolSystem "git" ["add", ingitfile] @? "git add failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" + Utility.boolSystem "git" [Utility.Param "add", Utility.File ingitfile] @? "git add failed" + Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op" unannexed ingitfile sha1dup = TestCase $ intmpclonerepo $ do @@ -125,7 +125,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do let sha1 = BackendTypes.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "commit"] @? "git commit failed" + Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" annexed_present sha1annexedfile where tmp = "tmpfile" @@ -139,7 +139,7 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy] annexed_notpresent annexedfile withcopy = "with content" ~: intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"] @? "git commit of state failed" annexed_present annexedfile git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed" @@ -154,9 +154,9 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote] where noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get failed" - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "state changed"] + Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"] @? "git commit of state failed" - Utility.boolSystem "git" ["remote", "rm", "origin"] + Utility.boolSystem "git" [Utility.Params "remote rm origin"] @? "git remote rm origin failed" r <- git_annex "drop" ["-q", annexedfile] not r @? "drop wrongly succeeded with no known copy of file" @@ -287,12 +287,12 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True] then do -- pre-commit depends on the file being -- staged, normally git commit does this - Utility.boolSystem "git" ["add", annexedfile] + Utility.boolSystem "git" [Utility.Param "add", Utility.File annexedfile] @? "git add of edited file failed" git_annex "pre-commit" ["-q"] @? "pre-commit failed" else do - Utility.boolSystem "git" ["commit", "-q", "-a", "-m", "content changed"] + Utility.boolSystem "git" [Utility.Params "commit -q -a -m contentchanged"] @? "git commit of edited file failed" runchecks [checklink, checkunwritable] annexedfile c <- readFile annexedfile @@ -310,7 +310,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do git_annex "fix" ["-q", annexedfile] @? "fix of present file failed" annexed_present annexedfile createDirectory subdir - Utility.boolSystem "git" ["mv", annexedfile, subdir] + Utility.boolSystem "git" [Utility.Param "mv", Utility.File annexedfile, Utility.File subdir] @? "git mv failed" git_annex "fix" ["-q", newfile] @? "fix of moved file failed" runchecks [checklink, checkunwritable] newfile @@ -350,9 +350,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem where basicfsck = TestCase $ intmpclonerepo $ do git_annex "fsck" ["-q"] @? "fsck failed" - Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed" fsck_should_fail "numcopies unsatisfied" - Utility.boolSystem "git" ["config", "annex.numcopies", "1"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config annex.numcopies 1"] @? "git config failed" corrupt annexedfile corrupt sha1annexedfile withlocaluntrusted = TestCase $ intmpclonerepo $ do @@ -363,7 +363,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem git_annex "trust" ["-q", "."] @? "trust of current repo failed" git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo" withremoteuntrusted = TestCase $ intmpclonerepo $ do - Utility.boolSystem "git" ["config", "annex.numcopies", "2"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed" git_annex "get" ["-q", annexedfile] @? "get failed" git_annex "get" ["-q", sha1annexedfile] @? "get failed" git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies" @@ -433,9 +433,9 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do git_annex "get" ["-q", annexedfile] @? "get of file failed" git_annex "get" ["-q", sha1annexedfile] @? "get of file failed" checkunused [] - Utility.boolSystem "git" ["rm", "-q", annexedfile] @? "git rm failed" + Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File annexedfile] @? "git rm failed" checkunused [annexedfilekey] - Utility.boolSystem "git" ["rm", "-q", sha1annexedfile] @? "git rm failed" + Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File sha1annexedfile] @? "git rm failed" checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also @@ -511,10 +511,10 @@ setuprepo :: FilePath -> IO FilePath setuprepo dir = do cleanup dir ensuretmpdir - Utility.boolSystem "git" ["init", "-q", dir] @? "git init failed" + Utility.boolSystem "git" [Utility.Params "init -q", Utility.File dir] @? "git init failed" indir dir $ do - Utility.boolSystem "git" ["config", "user.name", "Test User"] @? "git config failed" - Utility.boolSystem "git" ["config", "user.email", "test@example.com"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config user.name", Utility.Param "Test User"] @? "git config failed" + Utility.boolSystem "git" [Utility.Params "config user.email test@example.com"] @? "git config failed" return dir -- clones are always done as local clones; we cannot test ssh clones @@ -522,7 +522,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath clonerepo old new = do cleanup new ensuretmpdir - Utility.boolSystem "git" ["clone", "-q", old, new] @? "git clone failed" + Utility.boolSystem "git" [Utility.Params "clone -q", Utility.File old, Utility.File new] @? "git clone failed" indir new $ git_annex "init" ["-q", new] @? "git annex init failed" return new From 4cd96ad2db0867ef7450215d3de7afcf748d7088 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 28 Feb 2011 16:25:31 -0400 Subject: [PATCH 0920/8313] rename --- Annex.hs | 2 +- Backend/SHA1.hs | 2 +- Command/Map.hs | 2 +- GitQueue.hs | 6 +++--- GitRepo.hs | 16 ++++++++-------- Remotes.hs | 12 ++++++------ RsyncFile.hs | 12 ++++++------ Utility.hs | 20 ++++++++++---------- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Annex.hs b/Annex.hs index cb662a1307..62e7e023d5 100644 --- a/Annex.hs +++ b/Annex.hs @@ -92,7 +92,7 @@ gitRepo :: Annex Git.Repo gitRepo = getState repo {- Adds a git command to the queue. -} -queue :: String -> [ShellParam] -> FilePath -> Annex () +queue :: String -> [CommandParam] -> FilePath -> Annex () queue command params file = do state <- get let q = repoqueue state diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index a7f592b73e..22bc493b77 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -32,7 +32,7 @@ backend = Backend.File.backend { sha1 :: FilePath -> Annex String sha1 file = do showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" (toShell [File file]) $ \h -> do + liftIO $ pOpen ReadFromPipe "sha1sum" (toCommand [File file]) $ \h -> do line <- hGetLine h let bits = split " " line if null bits diff --git a/Command/Map.hs b/Command/Map.hs index 00b5fc21b2..4d0f900038 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -198,7 +198,7 @@ tryScan r Left _ -> return Nothing Right r' -> return $ Just r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd (toShell params) $ + pOpen ReadFromPipe cmd (toCommand params) $ Git.hConfigRead r configlist = diff --git a/GitQueue.hs b/GitQueue.hs index 328243fa00..07cf9f62fc 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -25,7 +25,7 @@ import qualified GitRepo as Git - is not included, and must be able to be appended after the params. -} data Action = Action { getSubcommand :: String, - getParams :: [ShellParam] + getParams :: [CommandParam] } deriving (Show, Eq, Ord) {- A queue of actions to perform (in any order) on a git repository, @@ -38,7 +38,7 @@ empty :: Queue empty = M.empty {- Adds an action to a queue. -} -add :: Queue -> String -> [ShellParam] -> FilePath -> Queue +add :: Queue -> String -> [CommandParam] -> FilePath -> Queue add queue subcommand params file = M.insertWith (++) action [file] queue where action = Action subcommand params @@ -57,6 +57,6 @@ runAction repo action files = do unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs - params = toShell $ Git.gitCommandLine repo + params = toCommand $ Git.gitCommandLine repo (Param (getSubcommand action):getParams action) feedxargs h = hPutStr h $ join "\0" files diff --git a/GitRepo.hs b/GitRepo.hs index 3f2acdcf4f..04a0c2d540 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -243,7 +243,7 @@ urlPath Repo { location = Url u } = uriPath u urlPath repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} -gitCommandLine :: Repo -> [ShellParam] -> [ShellParam] +gitCommandLine :: Repo -> [CommandParam] -> [CommandParam] gitCommandLine repo@(Repo { location = Dir d} ) params = -- force use of specified repo via --git-dir and --work-tree [ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo) @@ -252,7 +252,7 @@ gitCommandLine repo@(Repo { location = Dir d} ) params = gitCommandLine repo _ = assertLocal repo $ error "internal" {- Runs git in the specified repo, throwing an error if it fails. -} -run :: Repo -> String -> [ShellParam] -> IO () +run :: Repo -> String -> [CommandParam] -> IO () run repo subcommand params = assertLocal repo $ do ok <- boolSystem "git" (gitCommandLine repo ((Param subcommand):params)) unless ok $ error $ "git " ++ show params ++ " failed" @@ -262,9 +262,9 @@ run repo subcommand params = assertLocal repo $ do - Note that this leaves the git process running, and so zombies will - result unless reap is called. -} -pipeRead :: Repo -> [ShellParam] -> IO String +pipeRead :: Repo -> [CommandParam] -> IO String pipeRead repo params = assertLocal repo $ do - (_, s) <- pipeFrom "git" $ toShell $ gitCommandLine repo params + (_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params return s {- Reaps any zombie git processes. -} @@ -296,7 +296,7 @@ stagedFiles repo l = stagedFiles' repo l [] stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath] stagedFilesNotDeleted repo l = stagedFiles' repo l [Param "--diff-filter=ACMRT"] -stagedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] +stagedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath] stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = [Params "diff --cached --name-only -z"] @@ -317,7 +317,7 @@ typeChangedStagedFiles repo l = typeChangedFiles' repo l [Param "--cached"] typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath] typeChangedFiles repo l = typeChangedFiles' repo l [] -typeChangedFiles' :: Repo -> [FilePath] -> [ShellParam] -> IO [FilePath] +typeChangedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath] typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end where start = [Params "diff --name-only --diff-filter=T -z"] @@ -325,7 +325,7 @@ typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it into a list of files. -} -pipeNullSplit :: Repo -> [ShellParam] -> IO [FilePath] +pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath] pipeNullSplit repo params = do fs0 <- pipeRead repo params return $ split0 fs0 @@ -410,7 +410,7 @@ checkAttr repo attr files = do -- directory. Convert to absolute, and then convert the filenames -- in its output back to relative. absfiles <- mapM absPath files - (_, s) <- pipeBoth "git" (toShell params) $ join "\0" absfiles + (_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles cwd <- getCurrentDirectory return $ map (topair $ cwd++"/") $ lines s where diff --git a/Remotes.hs b/Remotes.hs index 1523e67509..4dcc4c9adf 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -64,7 +64,7 @@ tryGitConfigRead r Left _ -> return r Right r' -> return r' pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd (toShell params) $ + pOpen ReadFromPipe cmd (toCommand params) $ Git.hConfigRead r store a = do r' <- a @@ -263,7 +263,7 @@ rsynchelper r sending key file = do {- Generates rsync parameters that ssh to the remote and asks it - to either receive or send the key's content. -} -rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [ShellParam] +rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] rsyncParams r sending key file = do Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") @@ -295,9 +295,9 @@ rsyncParams r sending key file = do - a specified error value. -} onRemote :: Git.Repo - -> (FilePath -> [ShellParam] -> IO a, a) + -> (FilePath -> [CommandParam] -> IO a, a) -> String - -> [ShellParam] + -> [CommandParam] -> Annex a onRemote r (with, errorval) command params = do s <- git_annex_shell r command params @@ -306,7 +306,7 @@ onRemote r (with, errorval) command params = do Nothing -> return errorval {- Generates parameters to run a git-annex-shell command on a remote. -} -git_annex_shell :: Git.Repo -> String -> [ShellParam] -> Annex (Maybe (FilePath, [ShellParam])) +git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) git_annex_shell r command params | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do @@ -319,7 +319,7 @@ git_annex_shell r command params shellcmd = "git-annex-shell" shellopts = (Param command):(File dir):params sshcmd = shellcmd ++ " " ++ - unwords (map shellEscape $ toShell shellopts) + unwords (map shellEscape $ toCommand shellopts) {- Looks up a per-remote config option in git config. - Failing that, tries looking for a global config option. -} diff --git a/RsyncFile.hs b/RsyncFile.hs index 149b45b11b..afff46c0ce 100644 --- a/RsyncFile.hs +++ b/RsyncFile.hs @@ -14,8 +14,8 @@ import Utility {- Generates parameters to make rsync use a specified command as its remote - shell. -} -rsyncShell :: [ShellParam] -> [ShellParam] -rsyncShell command = [Param "-e", Param $ unwords $ map escape (toShell command)] +rsyncShell :: [CommandParam] -> [CommandParam] +rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand command)] where {- rsync requires some weird, non-shell like quoting in - here. A doubled single quote inside the single quoted @@ -31,7 +31,7 @@ rsyncServerSend file = rsyncExec $ rsyncServerReceive :: FilePath -> IO Bool rsyncServerReceive file = rsync $ rsyncServerParams ++ [File file] -rsyncServerParams :: [ShellParam] +rsyncServerParams :: [CommandParam] rsyncServerParams = [ Param "--server" -- preserve permissions @@ -42,8 +42,8 @@ rsyncServerParams = , Params "-e.Lsf ." ] -rsync :: [ShellParam] -> IO Bool +rsync :: [CommandParam] -> IO Bool rsync = boolSystem "rsync" -rsyncExec :: [ShellParam] -> IO () -rsyncExec params = executeFile "rsync" True (toShell params) Nothing +rsyncExec :: [CommandParam] -> IO () +rsyncExec params = executeFile "rsync" True (toCommand params) Nothing diff --git a/Utility.hs b/Utility.hs index 90494a0c44..e63fa1f6be 100644 --- a/Utility.hs +++ b/Utility.hs @@ -6,8 +6,8 @@ -} module Utility ( - ShellParam(..), - toShell, + CommandParam(..), + toCommand, hGetContentsStrict, readFileStrict, parentDir, @@ -47,18 +47,18 @@ import Control.Monad (liftM2) - whitespace-separated, or a single Param (for when parameters contain - whitespace), or a File. -} -data ShellParam = Params String | Param String | File FilePath +data CommandParam = Params String | Param String | File FilePath deriving (Eq, Show, Ord) -{- Used to pass a list of ShellParams to a function that runs - - a shell command and expects Strings. -} -toShell :: [ShellParam] -> [String] -toShell l = concat $ map unwrap l +{- Used to pass a list of CommandParams to a function that runs + - a command and expects Strings. -} +toCommand :: [CommandParam] -> [String] +toCommand l = concat $ map unwrap l where unwrap (Param s) = [s] unwrap (Params s) = filter (not . null) (split " " s) -- Files that start with a dash are modified to avoid - -- the shell command interpreting them as options. + -- the command interpreting them as options. unwrap (File ('-':s)) = ["./-" ++ s] unwrap (File s) = [s] @@ -67,7 +67,7 @@ toShell l = concat $ map unwrap l - - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} -boolSystem :: FilePath -> [ShellParam] -> IO Bool +boolSystem :: FilePath -> [CommandParam] -> IO Bool boolSystem command params = do -- Going low-level because all the high-level system functions -- block SIGINT etc. We need to block SIGCHLD, but allow @@ -88,7 +88,7 @@ boolSystem command params = do setSignalMask oldset childaction oldint oldset = do restoresignals oldint oldset - executeFile command True (toShell params) Nothing + executeFile command True (toCommand params) Nothing {- Escapes a filename to be safely able to be exposed to the shell. -} shellEscape :: FilePath -> String From e2e4096a28042327ccb905b80ec0cbf550f88c61 Mon Sep 17 00:00:00 2001 From: tyger Date: Tue, 1 Mar 2011 14:07:52 +0000 Subject: [PATCH 0921/8313] Added a comment --- ..._5f08da5e21c0b3b5a8d1e4408c0d6405._comment | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment new file mode 100644 index 0000000000..71a40ad8cb --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment @@ -0,0 +1,60 @@ +[[!comment format=mdwn + username="tyger" + ip="80.66.20.180" + subject="comment 2" + date="2011-03-01T14:07:50Z" + content=""" +My current workflow looks like this (I'm still experimenting): + +### Create backup clone for migration + + git clone original migrate + cd migrate + for branch in $(git branch -a | grep remotes/origin | grep -v HEAD); do git checkout --track $branch; done + +### Inject git annex initialization at repository base + + git symbolic-ref HEAD refs/heads/newroot + git rm --cached *.rpm + git clean -f -d + git annex init master + git cherry-pick $(git rev-list --reverse master | head -1) + git rebase --onto newroot newroot master + git rebase master mybranch # how to automate this for all branches? + git branch -d newroot + +### Start migration with tree filter + + echo \*.rpm annex.backend=SHA1 > .git/info/attributes + MYWORKDIR=$(pwd) git filter-branch --tree-filter ' \ + if [ ! -d .git-annex ]; then \ + mkdir .git-annex; \ + cp ${MYWORKDIR}/.git-annex/uuid.log .git-annex/; \ + cp ${MYWORKDIR}/.gitattributes .; \ + fi + for rpm in $(git ls-files | grep \"\.rpm$\"); do \ + echo; \ + git annex add $rpm; \ + annexdest=$(readlink $rpm); \ + if [ -e .git-annex/$(basename $annexdest).log ]; then \ + echo \"FOUND $(basename $annexdest).log\"; \ + else \ + echo \"COPY $(basename $annexdest).log\"; \ + cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/; \ + fi; \ + ln -sf ${annexdest#../../} $rpm; \ + done; \ + git reset HEAD .git-rewrite; \ + : \ + ' -- $(git branch | cut -c 3-) + rm -rf .temp + git reset --hard + + +There are still some drawbacks: + +* git history shows that git annex log files are modified with each checkin +* branches have to be rebased manually before starting migration + + +"""]] From be4af85f98c45888a2ab3436642850bdfb3db721 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 13:23:44 -0400 Subject: [PATCH 0922/8313] a slightly annoying thing --- doc/bugs/git_rename_detection_on_file_move.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git_rename_detection_on_file_move.mdwn diff --git a/doc/bugs/git_rename_detection_on_file_move.mdwn b/doc/bugs/git_rename_detection_on_file_move.mdwn new file mode 100644 index 0000000000..76f1e098e5 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move.mdwn @@ -0,0 +1,13 @@ +It's unfortunate that git-annex sorta defeats git's rename detection. + +When an annexed file is moved to a different directory (specifically, a +directory that is shallower or deeper than the old directory), +the symlink often has to change. And so git log cannot --follow back +through the rename history, since all it has to go on is that symlink, +which it effectively sees as a one line file containing the symlink target. + +One way to fix this might be to do the `git annex fix` *after* the rename +is committed. This would mean that a commit would result in new staged +changes for another commit, which is perhaps startling behavior. + +The other way to fix it is to stop using symlinks, see [[todo/smudge]]. From d140c01bfd09669c632c3973b7e5e342f0ad7de5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 13:37:24 -0400 Subject: [PATCH 0923/8313] bug --- doc/bugs/weird_local_clone_confuses.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/weird_local_clone_confuses.mdwn diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn new file mode 100644 index 0000000000..fc652ae5eb --- /dev/null +++ b/doc/bugs/weird_local_clone_confuses.mdwn @@ -0,0 +1,9 @@ +See + + +If a local repo is cloned with "git clone orig/.git new", then git-annex in +new cannot see origin. + +the .git/config has "url=/.../orig/.git". Apparently git is ok with that +weird construction; probably it treats it as a bare git repo. But git-annex +just sees a directory w/o a .git subdir, and gives up. From b7f4801801aa8b8e20ea82261b193a73b7fec799 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 16:50:53 -0400 Subject: [PATCH 0924/8313] generic SHA size support --- Backend/SHA.hs | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ Backend/SHA1.hs | 55 +++----------------------------------- 2 files changed, 74 insertions(+), 52 deletions(-) create mode 100644 Backend/SHA.hs diff --git a/Backend/SHA.hs b/Backend/SHA.hs new file mode 100644 index 0000000000..d779e80553 --- /dev/null +++ b/Backend/SHA.hs @@ -0,0 +1,71 @@ +{- git-annex SHA abstract backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA (genBackend) where + +import Control.Monad.State +import Data.String.Utils +import System.Cmd.Utils +import System.IO +import System.Directory + +import qualified Backend.File +import BackendTypes +import Messages +import qualified Annex +import Locations +import Content +import Types +import Utility + +type SHASize = Int + +-- Constructor for Backends using a given SHASize. +genBackend :: SHASize -> Backend Annex +genBackend size = Backend.File.backend + { name = shaName size + , getKey = keyValue size + , fsckKey = Backend.File.checkKey $ checkKeyChecksum size + } + +shaName :: SHASize -> String +shaName size = "SHA" ++ show size + +shaN :: SHASize -> FilePath -> Annex String +shaN size file = do + showNote "checksum..." + liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do + line <- hGetLine h + let bits = split " " line + if null bits + then error $ command ++ " parse error" + else return $ head bits + where + command = "sha" ++ (show size) ++ "sum" + +-- A key is a checksum of its contents. +keyValue :: SHASize -> FilePath -> Annex (Maybe Key) +keyValue size file = do + s <- shaN size file + return $ Just $ Key (shaName size, s) + +-- A key's checksum is checked during fsck. +checkKeyChecksum :: SHASize -> Key -> Annex Bool +checkKeyChecksum size key = do + g <- Annex.gitRepo + let file = gitAnnexLocation g key + present <- liftIO $ doesFileExist file + if not present + then return True + else do + s <- shaN size file + if s == keyName key + then return True + else do + dest <- moveBad key + warning $ "Bad file content; moved to " ++ filePathToString dest + return False diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs index 22bc493b77..76d2af69e3 100644 --- a/Backend/SHA1.hs +++ b/Backend/SHA1.hs @@ -1,63 +1,14 @@ {- git-annex "SHA1" backend - - - Copyright 2010 Joey Hess + - Copyright 2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} module Backend.SHA1 (backend) where -import Control.Monad.State -import Data.String.Utils -import System.Cmd.Utils -import System.IO -import System.Directory - -import qualified Backend.File -import BackendTypes -import Messages -import qualified Annex -import Locations -import Content import Types -import Utility +import Backend.SHA backend :: Backend Annex -backend = Backend.File.backend { - name = "SHA1", - getKey = keyValue, - fsckKey = Backend.File.checkKey checkKeySHA1 -} - -sha1 :: FilePath -> Annex String -sha1 file = do - showNote "checksum..." - liftIO $ pOpen ReadFromPipe "sha1sum" (toCommand [File file]) $ \h -> do - line <- hGetLine h - let bits = split " " line - if null bits - then error "sha1sum parse error" - else return $ head bits - --- A key is a sha1 of its contents. -keyValue :: FilePath -> Annex (Maybe Key) -keyValue file = do - s <- sha1 file - return $ Just $ Key (name backend, s) - --- A key's sha1 is checked during fsck. -checkKeySHA1 :: Key -> Annex Bool -checkKeySHA1 key = do - g <- Annex.gitRepo - let file = gitAnnexLocation g key - present <- liftIO $ doesFileExist file - if not present - then return True - else do - s <- sha1 file - if s == keyName key - then return True - else do - dest <- moveBad key - warning $ "Bad file content; moved to " ++ filePathToString dest - return False +backend = genBackend 1 From 1b9c4477fb542cddbb05012a52c602eb203b2d83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 17:07:15 -0400 Subject: [PATCH 0925/8313] New backends: SHA512 SHA384 SHA256 SHA224 --- Backend/SHA224.hs | 14 ++++++++++++++ Backend/SHA256.hs | 14 ++++++++++++++ Backend/SHA384.hs | 14 ++++++++++++++ Backend/SHA512.hs | 14 ++++++++++++++ BackendList.hs | 8 ++++++++ configure.hs | 10 ++++++++-- debian/changelog | 1 + doc/backends.mdwn | 3 +++ 8 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 Backend/SHA224.hs create mode 100644 Backend/SHA256.hs create mode 100644 Backend/SHA384.hs create mode 100644 Backend/SHA512.hs diff --git a/Backend/SHA224.hs b/Backend/SHA224.hs new file mode 100644 index 0000000000..614f31c4b8 --- /dev/null +++ b/Backend/SHA224.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA224" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA224 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 224 diff --git a/Backend/SHA256.hs b/Backend/SHA256.hs new file mode 100644 index 0000000000..42d3d48c7d --- /dev/null +++ b/Backend/SHA256.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA256" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA256 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 256 diff --git a/Backend/SHA384.hs b/Backend/SHA384.hs new file mode 100644 index 0000000000..0cf77ea647 --- /dev/null +++ b/Backend/SHA384.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA384" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA384 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 384 diff --git a/Backend/SHA512.hs b/Backend/SHA512.hs new file mode 100644 index 0000000000..aed8bbcedf --- /dev/null +++ b/Backend/SHA512.hs @@ -0,0 +1,14 @@ +{- git-annex "SHA512" backend + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Backend.SHA512 (backend) where + +import Types +import Backend.SHA + +backend :: Backend Annex +backend = genBackend 512 diff --git a/BackendList.hs b/BackendList.hs index 5ae78bcc7f..4279b69fc9 100644 --- a/BackendList.hs +++ b/BackendList.hs @@ -10,6 +10,10 @@ module BackendList (allBackends) where -- When adding a new backend, import it here and add it to the list. import qualified Backend.WORM import qualified Backend.SHA1 +import qualified Backend.SHA256 +import qualified Backend.SHA512 +import qualified Backend.SHA224 +import qualified Backend.SHA384 import qualified Backend.URL import Types @@ -17,5 +21,9 @@ allBackends :: [Backend Annex] allBackends = [ Backend.WORM.backend , Backend.SHA1.backend + , Backend.SHA256.backend + , Backend.SHA512.backend + , Backend.SHA224.backend + , Backend.SHA384.backend , Backend.URL.backend ] diff --git a/configure.hs b/configure.hs index b5437ec1a5..772ba54899 100644 --- a/configure.hs +++ b/configure.hs @@ -11,11 +11,17 @@ tests = [ , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] - , TestCase "sha1sum" $ requireCmd "sha1sum" "sha1sum /dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath - ] + ] ++ shaTestCases [1, 256, 512, 224, 384] + +shaTestCases :: [Int] -> [TestCase] +shaTestCases l = map make l + where + make n = + let cmd = "sha" ++ show n ++ "sum" + in TestCase cmd $ requireCmd cmd (cmd ++ " Sun, 13 Feb 2011 00:48:02 -0400 diff --git a/doc/backends.mdwn b/doc/backends.mdwn index 3e605e4b15..5d02ad3a1c 100644 --- a/doc/backends.mdwn +++ b/doc/backends.mdwn @@ -19,6 +19,9 @@ can use different backends for different files. allows modifications of files to be tracked. Its need to generate checksums can make it slower for large files. for use. +* `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger + checksums. Mostly useful for the very paranoid, or anyone who is + researching checksum collisions and wants to annex their colliding data. ;) * `URL` -- This backend downloads the file's content from an external URL. The `annex.backends` git-config setting can be used to list the backends From fa92c29764d0deaf0f016729b90c6fdc4d4b7390 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 21:32:28 -0400 Subject: [PATCH 0926/8313] add internals page --- doc/index.mdwn | 1 + doc/internals.mdwn | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 doc/internals.mdwn diff --git a/doc/index.mdwn b/doc/index.mdwn index 503c858018..a6d072e093 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -53,6 +53,7 @@ files with git. * [[location_tracking]] reminds you where git-annex has seen files * git-annex prevents accidental data loss by [[tracking copies|copies]] of your files +* [[internals]] * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. diff --git a/doc/internals.mdwn b/doc/internals.mdwn new file mode 100644 index 0000000000..fe50681870 --- /dev/null +++ b/doc/internals.mdwn @@ -0,0 +1,56 @@ +In the world of git, we're not scared about internal implementation +details, and sometimes we like to dive in and tweak things by hand. Here's +some documentation to that end. + +## .git/annex/objects/*/* + +This is where locally available file contents are actually stored. +Files added to the annex get a symlink checked into git that points +to the file content. + +Each subdirectory has the name of a key. The file inside also has the name +of the key. This two-level structure is used because it allows the write +bit to be removed from the subdirectories as well as from the files. +That prevents accidentially deleting or changing the file contents. + +## .git-annex/uuid.log + +Records the UUIDs of known repositories, and associates them with a +description of the repository. This allows git-annex to display something +more useful than a UUID when it refers to a repository that does not have +a configured git remote pointing at it. + +The file format is simply one line per repository, with the uuid followed by a +space and then the description through to the end of the line. Example: + + e605dca6-446a-11e0-8b2a-002170d25c55 laptop + 26339d22-446b-11e0-9101-002170d25c55 usb disk + +## .git-annex/trust.log + +Records the [[trust]] information for repositories. Does not exist unless +[[trust]] values are configured. + +The file format is one line per repository, with the uuid followed by a +space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted). +Repositories not listed are semi-trusted. + +Example: + + e605dca6-446a-11e0-8b2a-002170d25c55 1 + 26339d22-446b-11e0-9101-002170d25c55 ? + +## .git-annex/*.log + +The remainder of the log files record [[location_tracking]] information +for file contents. The name of the key is the filename, and the content +consists of a timestamp, either 1 (present) or 0 (not present), and +the UUID of the repository that has or lacks the file content. + +Example: + + 1287290776.765152s 1 e605dca6-446a-11e0-8b2a-002170d25c55 + 1287290767.478634s 0 26339d22-446b-11e0-9101-002170d25c55 + +These files are designed to be auto-merged using git's union merge driver. +The timestamps allow the most recent information to be identified. From d6be2e222f2c8ffa0f6cf5e46a19795846d5ba93 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 21:37:27 -0400 Subject: [PATCH 0927/8313] format --- doc/internals.mdwn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/internals.mdwn b/doc/internals.mdwn index fe50681870..f69ee6f8b9 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -2,7 +2,7 @@ In the world of git, we're not scared about internal implementation details, and sometimes we like to dive in and tweak things by hand. Here's some documentation to that end. -## .git/annex/objects/*/* +## `.git/annex/objects/*/*` This is where locally available file contents are actually stored. Files added to the annex get a symlink checked into git that points @@ -13,7 +13,7 @@ of the key. This two-level structure is used because it allows the write bit to be removed from the subdirectories as well as from the files. That prevents accidentially deleting or changing the file contents. -## .git-annex/uuid.log +## `.git-annex/uuid.log` Records the UUIDs of known repositories, and associates them with a description of the repository. This allows git-annex to display something @@ -26,7 +26,7 @@ space and then the description through to the end of the line. Example: e605dca6-446a-11e0-8b2a-002170d25c55 laptop 26339d22-446b-11e0-9101-002170d25c55 usb disk -## .git-annex/trust.log +## `.git-annex/trust.log` Records the [[trust]] information for repositories. Does not exist unless [[trust]] values are configured. @@ -40,7 +40,7 @@ Example: e605dca6-446a-11e0-8b2a-002170d25c55 1 26339d22-446b-11e0-9101-002170d25c55 ? -## .git-annex/*.log +## `.git-annex/*.log` The remainder of the log files record [[location_tracking]] information for file contents. The name of the key is the filename, and the content From 1072683a47efe384a85280fa68afa4a8d31cfd7a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 1 Mar 2011 21:38:47 -0400 Subject: [PATCH 0928/8313] link --- doc/internals.mdwn | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/internals.mdwn b/doc/internals.mdwn index f69ee6f8b9..3f680dd8f2 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -8,10 +8,11 @@ This is where locally available file contents are actually stored. Files added to the annex get a symlink checked into git that points to the file content. -Each subdirectory has the name of a key. The file inside also has the name -of the key. This two-level structure is used because it allows the write -bit to be removed from the subdirectories as well as from the files. -That prevents accidentially deleting or changing the file contents. +Each subdirectory has the name of a key in one of the +[[key-value_backends|backends]]. The file inside also has the name of the key. +This two-level structure is used because it allows the write bit to be removed +from the subdirectories as well as from the files. That prevents accidentially +deleting or changing the file contents. ## `.git-annex/uuid.log` From 3db25aaa3bcb243b3ba5c5638af1894e1aae9ed0 Mon Sep 17 00:00:00 2001 From: tyger Date: Wed, 2 Mar 2011 08:15:37 +0000 Subject: [PATCH 0929/8313] Added a comment --- ...mment_3_f483038c006cf7dcccf1014fa771744f._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment new file mode 100644 index 0000000000..90bf23b6cf --- /dev/null +++ b/doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="tyger" + ip="80.66.20.180" + subject="comment 3" + date="2011-03-02T08:15:37Z" + content=""" +> Sounds like it might be enough to add a switch to git-annex that overrides where it considers the top of the git repository to be? + +It should sufficient to honor GIT_DIR/GIT_WORK_TREE/GIT_INDEX_FILE environment variables. git filter-branch sets GIT_WORK_TREE to ., but this can be mitigated by starting the filter script with 'GIT_WORK_TREE=$(pwd $GIT_WORK_TREE)'. E.g. GIT_DIR=/home/tyger/repo/.git, GIT_WORK_TREE=/home/tyger/repo/.git-rewrite/t, then git annex should be able to compute the correct relative path or maybe use absolute pathes in symlinks. + +Another problem I observed is that git annex add automatically commits the symlink; this behaviour doesn't work well with filter-tree. git annex commits the wrong path (.git-rewrite/t/LINK instead of LINK). Also filter-tree doesn't expect that the filter script commmits anything; new files in the temporary work tree will be committed by filter-tree on each iteration of the filter script (missing files will be removed). +"""]] From 1c08b8bf8aae73238f3571b643d4ed9d8301cfae Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 2 Mar 2011 10:36:33 +0000 Subject: [PATCH 0930/8313] --- doc/install/OSX.mdwn | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn index e107e48eb9..af078e363d 100644 --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -1,5 +1,5 @@ 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
@@ -7,6 +7,9 @@ sudo port install pcre
 sudo cabal install pcre-light
 sudo cabal install quickcheck  
 
+# this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
+export PATH=$PATH:/opt/local/libexec/gnubin
+
 git clone  git://git.kitenet.net/git-annex
 
 cd git-annex

From af6a7c4b81527d3798c48c0ef0f3a317882bc651 Mon Sep 17 00:00:00 2001
From: tyger 
Date: Wed, 2 Mar 2011 13:08:46 +0000
Subject: [PATCH 0931/8313] Found temporary solution

---
 ..._existing_git_repository_to_git-annex.mdwn | 56 +++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
index 94c5f1d133..93ff4972de 100644
--- a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
@@ -6,3 +6,59 @@ I tried to rewrite the (cloned) repository with git-filter-branch but failed mis
 * annex log files are stored in .git-annex/ instead of .git-rewrite/t/.git-annex/ so the filter operation misses them
 
 Any suggestions how to proceed?
+
+EDIT 3/2/2010
+I finally got it working for my purposes. Hardest part was preserving the branches while injecting the new `git annex setup` base commit.
+
+#### Clone repository
+    git clone original migrate
+    cd migrate
+    git checkout mybranch
+    git checkout master
+    git remote rm origin
+
+#### Inject `git annex setup` base commit and repair branches
+    git symbolic-ref HEAD refs/heads/newroot
+    git rm --cached *
+    git clean -f -d
+    git annex init master
+    echo \*.rpm annex.backend=SHA1 >> .gitattributes
+    git commit -m "store rpms in git annex" .gitattributes
+    git cherry-pick $(git rev-list --reverse master | head -1)
+    git rebase --onto newroot newroot master
+    git rebase --onto master mybranch~1 mybranch
+    git branch -d newroot
+
+#### Migrate repository
+    mkdir .temp
+    cp .git-annex/* .temp/
+    MYWORKDIR=$(pwd) git filter-branch --tree-filter '
+        mkdir -p .git-annex;
+        cp ${MYWORKDIR}/.temp/* .git-annex/;
+        for rpm in $(git ls-files | grep "\.rpm$"); do
+            echo;
+            git annex add $rpm;
+            annexdest=$(readlink $rpm);
+            if [ -e .git-annex/$(basename $annexdest).log ]; then
+                echo "FOUND $(basename $annexdest).log";
+            else
+                echo "COPY $(basename $annexdest).log";
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log .git-annex/;
+                cp ${MYWORKDIR}/.git-annex/$(basename $annexdest).log ${MYWORKDIR}/.temp/;
+            fi;
+            ln -sf ${annexdest#../../} $rpm;
+        done;
+        git reset HEAD .git-rewrite;
+        :
+        ' -- $(git branch | cut -c 3-)
+    rm -rf .temp
+    git reset --hard
+
+
+TODO:
+
+* Find a way to repair branches automatically (detect branch points and run appropriate `git rebase` commands)
+
+I'll be happy to try any suggestions to improve this migration script.
+
+P.S. Is there a way to edit comments?

From 70a6eb6d73b3e302b9b9c02342581fada25dcad9 Mon Sep 17 00:00:00 2001
From: tyger 
Date: Wed, 2 Mar 2011 13:18:10 +0000
Subject: [PATCH 0932/8313] don't forget to rewrite tags

---
 doc/forum/migrate_existing_git_repository_to_git-annex.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
index 93ff4972de..f673de765b 100644
--- a/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
+++ b/doc/forum/migrate_existing_git_repository_to_git-annex.mdwn
@@ -32,7 +32,9 @@ I finally got it working for my purposes. Hardest part was preserving the branch
 #### Migrate repository
     mkdir .temp
     cp .git-annex/* .temp/
-    MYWORKDIR=$(pwd) git filter-branch --tree-filter '
+    MYWORKDIR=$(pwd) git filter-branch \
+     --tag-name-filter cat \
+     --tree-filter '
         mkdir -p .git-annex;
         cp ${MYWORKDIR}/.temp/* .git-annex/;
         for rpm in $(git ls-files | grep "\.rpm$"); do

From a3daac8a8b06bbe2f35ca16cc1b27e21cad8a0e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Mar 2011 13:47:45 -0400
Subject: [PATCH 0933/8313] only enable SHA backends that configure finds
 support for

---
 Backend.hs           |  3 ++-
 Backend/SHA.hs       | 32 ++++++++++++++++++++++++--------
 Backend/SHA1.hs      | 14 --------------
 Backend/SHA224.hs    | 14 --------------
 Backend/SHA384.hs    | 14 --------------
 Backend/SHA512.hs    | 14 --------------
 Backend/URL.hs       |  5 ++++-
 Backend/WORM.hs      |  5 ++++-
 BackendList.hs       | 18 +++++-------------
 doc/install.mdwn     |  2 +-
 doc/install/OSX.mdwn |  2 +-
 test.hs              | 23 ++++++++++++++++-------
 12 files changed, 57 insertions(+), 89 deletions(-)
 delete mode 100644 Backend/SHA1.hs
 delete mode 100644 Backend/SHA224.hs
 delete mode 100644 Backend/SHA384.hs
 delete mode 100644 Backend/SHA512.hs

diff --git a/Backend.hs b/Backend.hs
index 74f71f8bea..df23e80a33 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -26,7 +26,8 @@ module Backend (
 	fsckKey,
 	lookupFile,
 	chooseBackends,
-	keyBackend
+	keyBackend,
+	lookupBackendName
 ) where
 
 import Control.Monad.State
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index d779e80553..c074ab48a2 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -5,13 +5,14 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Backend.SHA (genBackend) where
+module Backend.SHA (backends) where
 
 import Control.Monad.State
 import Data.String.Utils
 import System.Cmd.Utils
 import System.IO
 import System.Directory
+import Data.Maybe
 
 import qualified Backend.File
 import BackendTypes
@@ -21,16 +22,31 @@ import Locations
 import Content
 import Types
 import Utility
+import qualified SysConfig
 
 type SHASize = Int
 
--- Constructor for Backends using a given SHASize.
-genBackend :: SHASize -> Backend Annex
-genBackend size = Backend.File.backend 
-	{ name = shaName size
-	, getKey = keyValue size
-	, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
-	}
+backends :: [Backend Annex]
+-- order is slightly significant; want sha1 first ,and more general
+-- sizes earlier
+backends = catMaybes $ map genBackend [1, 256, 512, 224, 384]
+
+genBackend :: SHASize -> Maybe (Backend Annex)
+genBackend size
+	| supported size = Just b 
+	| otherwise = Nothing
+	where
+		b = Backend.File.backend 
+			{ name = shaName size
+			, getKey = keyValue size
+			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
+			}
+		supported 1 = SysConfig.sha1sum
+		supported 256 = SysConfig.sha256sum
+		supported 224 = SysConfig.sha224sum
+		supported 384 = SysConfig.sha384sum
+		supported 512 = SysConfig.sha512sum
+		supported _ = False
 
 shaName :: SHASize -> String
 shaName size = "SHA" ++ show size
diff --git a/Backend/SHA1.hs b/Backend/SHA1.hs
deleted file mode 100644
index 76d2af69e3..0000000000
--- a/Backend/SHA1.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA1" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA1 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 1
diff --git a/Backend/SHA224.hs b/Backend/SHA224.hs
deleted file mode 100644
index 614f31c4b8..0000000000
--- a/Backend/SHA224.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA224" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA224 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 224
diff --git a/Backend/SHA384.hs b/Backend/SHA384.hs
deleted file mode 100644
index 0cf77ea647..0000000000
--- a/Backend/SHA384.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA384" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA384 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 384
diff --git a/Backend/SHA512.hs b/Backend/SHA512.hs
deleted file mode 100644
index aed8bbcedf..0000000000
--- a/Backend/SHA512.hs
+++ /dev/null
@@ -1,14 +0,0 @@
-{- git-annex "SHA512" backend
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.SHA512 (backend) where
-
-import Types
-import Backend.SHA
-
-backend :: Backend Annex
-backend = genBackend 512
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 864c793010..29dc8fefa7 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Backend.URL (backend) where
+module Backend.URL (backends) where
 
 import Control.Monad.State (liftIO)
 import Data.String.Utils
@@ -15,6 +15,9 @@ import BackendTypes
 import Utility
 import Messages
 
+backends :: [Backend Annex]
+backends = [backend]
+
 backend :: Backend Annex
 backend = Backend {
 	name = "URL",
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 92fe5a2d4c..8a6412eb11 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Backend.WORM (backend) where
+module Backend.WORM (backends) where
 
 import Control.Monad.State
 import System.FilePath
@@ -22,6 +22,9 @@ import Content
 import Messages
 import Types
 
+backends :: [Backend Annex]
+backends = [backend]
+
 backend :: Backend Annex
 backend = Backend.File.backend {
 	name = "WORM",
diff --git a/BackendList.hs b/BackendList.hs
index 4279b69fc9..bc3fd83142 100644
--- a/BackendList.hs
+++ b/BackendList.hs
@@ -9,21 +9,13 @@ module BackendList (allBackends) where
 
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
-import qualified Backend.SHA1
-import qualified Backend.SHA256
-import qualified Backend.SHA512
-import qualified Backend.SHA224
-import qualified Backend.SHA384
+import qualified Backend.SHA
 import qualified Backend.URL
 import Types
 
 allBackends :: [Backend Annex]
-allBackends = 
-	[ Backend.WORM.backend
-	, Backend.SHA1.backend
-	, Backend.SHA256.backend
-	, Backend.SHA512.backend
-	, Backend.SHA224.backend
-	, Backend.SHA384.backend
-	, Backend.URL.backend
+allBackends = concat 
+	[ Backend.WORM.backends
+	, Backend.SHA.backends
+	, Backend.URL.backends
 	]
diff --git a/doc/install.mdwn b/doc/install.mdwn
index f1305777c0..9eb8bbacf7 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -16,7 +16,7 @@ To build and use git-annex, you will need:
   (or uuidgen from util-linux)
 * `xargs`: 
 * `rsync`: 
-* `sha1sum`: 
+* `sha1sum`:  (optional, but recommended)
 * Then just [[download]] git-annex and run: `make; make install`
 
 ([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index af078e363d..3c29fc101c 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,7 +7,7 @@ sudo port install pcre
 sudo cabal install pcre-light
 sudo cabal install quickcheck  
 
-# this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
+# optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
 git clone  git://git.kitenet.net/git-annex
diff --git a/test.hs b/test.hs
index 1bae3bd83e..6e35eaee59 100644
--- a/test.hs
+++ b/test.hs
@@ -37,8 +37,8 @@ import qualified UUID
 import qualified Trust
 import qualified Remotes
 import qualified Content
-import qualified Backend.SHA1
-import qualified Backend.WORM
+import qualified BackendList
+import qualified Backend
 import qualified Command.DropUnused
 
 main :: IO ()
@@ -121,7 +121,7 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup]
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	writeFile tmp $ content sha1annexedfile
-	r <- annexeval $ BackendTypes.getKey Backend.SHA1.backend tmp
+	r <- annexeval $ BackendTypes.getKey backendSHA1 tmp
 	let sha1 = BackendTypes.keyName $ fromJust r
 	git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed"
@@ -405,8 +405,8 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 					@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
-		checkbackend annexedfile Backend.SHA1.backend
-		checkbackend sha1annexedfile Backend.SHA1.backend
+		checkbackend annexedfile backendSHA1
+		checkbackend sha1annexedfile backendSHA1
 
 		-- check that reversing a migration works
 		writeFile ".gitattributes" $ "* annex.backend=WORM"
@@ -416,8 +416,8 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 			@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
-		checkbackend annexedfile Backend.WORM.backend
-		checkbackend sha1annexedfile Backend.WORM.backend
+		checkbackend annexedfile backendWORM
+		checkbackend sha1annexedfile backendWORM
 		
 		where
 			checkbackend file expected = do
@@ -682,3 +682,12 @@ changecontent f = writeFile f $ changedcontent f
 
 changedcontent :: FilePath -> String
 changedcontent f = (content f) ++ " (modified)"
+
+backendSHA1 :: Types.Backend Types.Annex
+backendSHA1 = backend_ "SHA1"
+
+backendWORM :: Types.Backend Types.Annex
+backendWORM = backend_ "WORM"
+
+backend_ :: String -> Types.Backend Types.Annex
+backend_ name = Backend.lookupBackendName BackendList.allBackends name

From 6206b46e60803b9d15c08062780d153ebfe4a9ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Mar 2011 14:30:36 -0400
Subject: [PATCH 0934/8313] fsck: Check for and repair location log damage.

---
 Command/Fsck.hs  | 44 +++++++++++++++++++++++++++++++++++++++++---
 debian/changelog |  1 +
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index b6f330d4c2..f8c957053d 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,11 +7,17 @@
 
 module Command.Fsck where
 
+import Control.Monad.State (liftIO)
+
 import Command
 import qualified Backend
+import qualified Annex
+import UUID
 import Types
 import Messages
 import Utility
+import Content
+import LocationLog
 
 command :: [Command]
 command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek
@@ -20,7 +26,6 @@ command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek
 seek :: [CommandSeek]
 seek = [withAttrFilesInGit "annex.numcopies" start]
 
-{- Checks a file's backend data for problems. -}
 start :: CommandStartAttrFile
 start (file, attr) = isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
@@ -30,7 +35,40 @@ start (file, attr) = isAnnexed file $ \(key, backend) -> do
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
 perform key file backend numcopies = do
-	success <- Backend.fsckKey backend key (Just file) numcopies
-	if success
+	-- the location log is checked first, so that if it has bad data
+	-- that gets corrected
+	locationlogok <- verifyLocationLog key file
+	backendok <- Backend.fsckKey backend key (Just file) numcopies
+	if locationlogok && backendok
 		then return $ Just $ return True
 		else return Nothing
+
+{- Checks that the location log reflects the current status of the key,
+   in this repository only. -}
+verifyLocationLog :: Key -> FilePath -> Annex Bool
+verifyLocationLog key file = do
+	present <- inAnnex key
+	
+	g <- Annex.gitRepo
+	u <- getUUID g
+        uuids <- liftIO $ keyLocations g key
+
+	case (present, u `elem` uuids) of
+		(True, False) -> do
+				fix g u ValuePresent
+				-- There is no data loss, so do not fail.
+				return True
+		(False, True) -> do
+				fix g u ValueMissing
+				warning $
+					"** Based on the location log, " ++ file
+					++ "\n** was expected to be present, " ++
+					"but its content is missing."
+				return False
+		_ -> return True
+	
+	where
+		fix g u s = do
+			showNote "fixing location log"
+			_ <- liftIO $ logChange g key u s
+			return ()
diff --git a/debian/changelog b/debian/changelog
index 8d7dd46e5c..29f2fd134c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,7 @@ git-annex (0.22) UNRELEASED; urgency=low
     to a utility it will be escaped to avoid it being interpreted as an
     option.
   * New backends: SHA512 SHA384 SHA256 SHA224
+  * fsck: Check for and repair location log damage.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 

From f9da8625003b29ec47baaed272bca1c772186b10 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Thu, 3 Mar 2011 10:31:47 +0000
Subject: [PATCH 0935/8313]

---
 ...nused_seems_to_check_for_current_path.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
new file mode 100644
index 0000000000..14caee022c
--- /dev/null
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -0,0 +1,21 @@
+When I run `git annex unused` from my repository's root it shows everything ok:
+
+    ~/annex$ git annex unused
+    unused  (checking for unused data...) ok
+
+But... When I run it from a subdirectory, it shows a lot:
+
+    ~/annex/Software$ git annex unused
+    unused  (checking for unused data...) 
+      Some annexed data is no longer pointed to by any files in the repository:
+        NUMBER  KEY
+        1       SHA1:########################################
+    ...
+        921     SHA1:########################################
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      (To remove unwanted data: git-annex dropunused NUMBER)
+      ok
+
+Is this a bug or by design? By removing these "unused" files with `dropunused` I've just lost the only copy of 160 files.
+
+I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b

From 1bf3be5d6cf24e6a04066e68cfe8d92dd58859f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 13:25:17 -0400
Subject: [PATCH 0936/8313] doesn't seem possible..

---
 ...git_annex_unused_seems_to_check_for_current_path.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index 14caee022c..76a0fc1ec7 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -19,3 +19,12 @@ But... When I run it from a subdirectory, it shows a lot:
 Is this a bug or by design? By removing these "unused" files with `dropunused` I've just lost the only copy of 160 files.
 
 I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
+
+> I'm very sorry you lost data.
+> 
+> But, git annex unused absolutely does not let the current directory
+> influence what it does. It always scans the entire repo from the top.
+> And I've tested it just now to make sure that in a subdirectory
+> it does the same thing as at the top. The only way I could explain
+> what you show above is if "Software" were a separate git repository
+> than "~/annex". --[[Joey]] 

From f93b8a66971b9a8cb036258e14da6614264c800d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 13:26:53 -0400
Subject: [PATCH 0937/8313] amend

---
 .../git_annex_unused_seems_to_check_for_current_path.mdwn     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index 76a0fc1ec7..ff318d616e 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -27,4 +27,6 @@ I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
 > And I've tested it just now to make sure that in a subdirectory
 > it does the same thing as at the top. The only way I could explain
 > what you show above is if "Software" were a separate git repository
-> than "~/annex". --[[Joey]] 
+> than "~/annex". Or if the symlinks to the content are somehow broken
+> when looked at from within Software, but unbroken when looked at from the
+> parent directory. I can't think how that would happen. --[[Joey]] 

From 7ea9f52c2859a72f8e46522338c1c8a112549d84 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 13:37:46 -0400
Subject: [PATCH 0938/8313] cannot be broken symlinks after all.. one other
 idea

---
 ...ex_unused_seems_to_check_for_current_path.mdwn | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index ff318d616e..df0fb50cc1 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -25,8 +25,13 @@ I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
 > But, git annex unused absolutely does not let the current directory
 > influence what it does. It always scans the entire repo from the top.
 > And I've tested it just now to make sure that in a subdirectory
-> it does the same thing as at the top. The only way I could explain
-> what you show above is if "Software" were a separate git repository
-> than "~/annex". Or if the symlinks to the content are somehow broken
-> when looked at from within Software, but unbroken when looked at from the
-> parent directory. I can't think how that would happen. --[[Joey]] 
+> it does the same thing as at the top. 
+> 
+> There are only two ways this could happen that I can think of:
+> 
+> 1. If "Software" were a separate git repository than "~/annex".
+> 2. If gitignores or something made `git ls-files`
+>    not list the files when ran in the subdir. This seems *possible*,
+>    but I don't know how to construct such an ignore.
+> 
+> --[[Joey]] 

From d651d4985b9fca2fbeeab2f1b700e6423994c400 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 14:51:57 -0400
Subject: [PATCH 0939/8313] some initial support for local bare repos

This relies on git-annex's behavior of reading the config of local repos.
That allows repoIsLocalBare to examine the git config for core.bare.

Hopefully, gitAnnexLocation, gitAnnexDir, and gitAnnexObjectDir
are only used on local repos. But, I have not audited fully, since
they're probably not (see for example copyToRemote). And so,
the functions fall back to their old non-bare-aware behavior for
non-local repos.
---
 Content.hs   |  4 ++--
 GitRepo.hs   | 18 ++++++++++++++----
 Locations.hs | 33 +++++++++++++++++++--------------
 3 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/Content.hs b/Content.hs
index cb954f4a0b..658d9a8611 100644
--- a/Content.hs
+++ b/Content.hs
@@ -51,8 +51,8 @@ calcGitLink file key = do
 	let absfile = case absNormPath cwd file of
 		Just f -> f
 		Nothing -> error $ "unable to normalize " ++ filePathToString file
-	return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ++
-		annexLocation key
+	return $ relPathDirToDir (parentDir absfile) 
+			(Git.workTree g)  ".git"  annexLocation key
 
 {- Updates the LocationLog when a key's presence changes. -}
 logStatus :: Key -> LogStatus -> Annex ()
diff --git a/GitRepo.hs b/GitRepo.hs
index 04a0c2d540..ba01251218 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -16,6 +16,8 @@ module GitRepo (
 	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
+	repoIsLocalBare,
+	repoIsLocalFull,
 	repoDescribe,
 	repoLocation,
 	workTree,
@@ -161,6 +163,14 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+repoIsLocalBare :: Repo -> Bool
+repoIsLocalBare r@(Repo { location = Dir _ }) = configBare r
+repoIsLocalBare _ = False
+
+repoIsLocalFull :: Repo -> Bool
+repoIsLocalFull r@(Repo { location = Dir _ }) = not $ configBare r
+repoIsLocalFull _ = False
+
 assertLocal :: Repo -> a -> a
 assertLocal repo action = 
 	if not $ repoIsUrl repo
@@ -174,8 +184,8 @@ assertUrl repo action =
 		else error $ "acting on local git repo " ++  repoDescribe repo ++ 
 				" not supported"
 
-bare :: Repo -> Bool
-bare repo = case Map.lookup "core.bare" $ config repo of
+configBare :: Repo -> Bool
+configBare repo = case Map.lookup "core.bare" $ config repo of
 	Just v -> configTrue v
 	Nothing -> error $ "it is not known if git repo " ++
 			repoDescribe repo ++
@@ -184,13 +194,13 @@ bare repo = case Map.lookup "core.bare" $ config repo of
 {- Path to a repository's gitattributes file. -}
 attributes :: Repo -> String
 attributes repo
-	| bare repo = workTree repo ++ "/info/.gitattributes"
+	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
 {- Path to a repository's .git directory, relative to its workTree. -}
 gitDir :: Repo -> String
 gitDir repo
-	| bare repo = ""
+	| configBare repo = ""
 	| otherwise = ".git"
 
 {- Path to a repository's --work-tree, that is, its top.
diff --git a/Locations.hs b/Locations.hs
index d30ceb1367..908d5b74ed 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -50,35 +50,40 @@ stateDir = addTrailingPathSeparator $ ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
 
-{- Annexed content is stored in .git/annex/objects; .git/annex is used
- - for other temporary storage also. -}
+{- The directory git annex uses for local state, relative to the .git
+ - directory -}
 annexDir :: FilePath
-annexDir = addTrailingPathSeparator $ ".git/annex"
+annexDir = addTrailingPathSeparator $ "annex"
+
+{- The directory git annex uses for locally available object content,
+ - relative to the .git directory -}
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
-{- Annexed file's location relative to git's working tree. 
- -
- - Note: Assumes repo is NOT bare.-}
+{- Annexed file's location relative to the .git directory. -}
 annexLocation :: Key -> FilePath
-annexLocation key = ".git/annex/objects"  f  f
+annexLocation key = objectDir  f  f
 	where
 		f = keyFile key
 
 {- Annexed file's absolute location in a repository. -}
 gitAnnexLocation :: Git.Repo -> Key -> FilePath
-gitAnnexLocation r key = Git.workTree r  annexLocation key
+gitAnnexLocation r key
+	| Git.repoIsLocalBare r = Git.workTree r  annexLocation key
+	| otherwise = Git.workTree r  ".git"  annexLocation key
 
-{- The annex directory of a repository.
- -
- - Note: Assumes repo is NOT bare. -}
+{- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
-gitAnnexDir r = addTrailingPathSeparator $ Git.workTree r  annexDir
+gitAnnexDir r
+	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  annexDir
+	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  annexDir
 
 {- The part of the annex directory where file contents are stored.
  -}
 gitAnnexObjectDir :: Git.Repo -> FilePath
-gitAnnexObjectDir r = addTrailingPathSeparator $ Git.workTree r  objectDir
+gitAnnexObjectDir r
+	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
+	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  objectDir
 
 {- .git-annex/tmp/ is used for temp files -}
 gitAnnexTmpDir :: Git.Repo -> FilePath
@@ -98,7 +103,7 @@ gitAnnexUnusedLog r = gitAnnexDir r  "unused"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = ("/" ++ objectDir) `isInfixOf` s
+isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
 {- Converts a key into a filename fragment.
  -

From d25a8540854fed30567868799322bbdf4e947c2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 14:59:45 -0400
Subject: [PATCH 0940/8313] remove redundant imports

---
 test.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test.hs b/test.hs
index 6e35eaee59..936e4da175 100644
--- a/test.hs
+++ b/test.hs
@@ -37,8 +37,6 @@ import qualified UUID
 import qualified Trust
 import qualified Remotes
 import qualified Content
-import qualified BackendList
-import qualified Backend
 import qualified Command.DropUnused
 
 main :: IO ()

From 9f20aee2192bcc5f2c0ae1f59db88f6eadeb7335 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:22:53 -0400
Subject: [PATCH 0941/8313] avoid logging to location log when in a bare repo

This assumes that changes to content in bare repos are made from some
non-bare repo, and that the location log is updated on that side.

That's true for move --from and move --to.

It's *not* true for dropkey and setkey and recvkey. But those are plumbing
level commands, so I guess it's ok to assume that someone running those
in a bare repo knows what they're doing. And git-annex-shell is used to
run those, and if the bare repo is non-local, it needs to be able to use
them even though they cannot update the location log. So this seems
unavoidable.
---
 Command/Move.hs |  5 +++++
 Content.hs      | 16 +++++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 8c19539fba..3774ccbe9d 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -51,6 +51,11 @@ showAction :: Bool -> FilePath -> Annex ()
 showAction True file = showStart "move" file
 showAction False file = showStart "copy" file
 
+{- Used to log a change in a remote's having a key. The change is logged
+ - in the local repo, not on the remote. The process of transferring the
+ - key to the remote, or removing the key from it *may* log the change
+ - on the remote, but this cannot be relied on. For example, it's not done
+ - for bare repos. -}
 remoteHasKey :: Git.Repo -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	g <- Annex.gitRepo
diff --git a/Content.hs b/Content.hs
index 658d9a8611..6b8316c082 100644
--- a/Content.hs
+++ b/Content.hs
@@ -23,7 +23,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, filterM)
+import Control.Monad (when, unless, filterM)
 import System.Posix.Files
 import System.FilePath
 
@@ -54,13 +54,19 @@ calcGitLink file key = do
 	return $ relPathDirToDir (parentDir absfile) 
 			(Git.workTree g)  ".git"  annexLocation key
 
-{- Updates the LocationLog when a key's presence changes. -}
+{- Updates the LocationLog when a key's presence changes.
+ -
+ - Note that the LocationLog is not updated in bare repositories.
+ - Operations that change a bare repository should be done from
+ - a non-bare repository, and the LocationLog in that repository be
+ - updated instead. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
 	g <- Annex.gitRepo
-	u <- getUUID g
-	logfile <- liftIO $ logChange g key u status
-	Annex.queue "add" [Param "--"] logfile
+	unless (Git.repoIsLocalBare g) $ do
+		u <- getUUID g
+		logfile <- liftIO $ logChange g key u status
+		Annex.queue "add" [Param "--"] logfile
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 

From d28d659a241fb1780264416e4317e65691201f50 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:44:01 -0400
Subject: [PATCH 0942/8313] initial documantation/todo list for bare
 repositories

---
 doc/bare_repositories.mdwn | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bare_repositories.mdwn

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
new file mode 100644
index 0000000000..eb48dfa10b
--- /dev/null
+++ b/doc/bare_repositories.mdwn
@@ -0,0 +1,29 @@
+Due to popular demand, git-annex can now be used with bare repositories.
+
+**This is still an experimental feature!** 
+
+Known to work ok, so far for local bare repositories only:
+
+* `git annex move --to` and `--from`, when pointed at a bare repository.
+* `git annex copy` ditto.
+* `git annex drop` can check that a bare repository has a copy of data
+  that is being dropped.
+* `git annex get` can transfer data from a bare repository.
+
+There are a few caveats to keep in mind:
+
+* Using non-local bare repositories is not tested and probably broken.
+* `git annex init` can be run in a bare repository, but it cannot
+  store the name you gave the repository in .git-annex/uuid.log (because
+  the bare repository has no such file to commit to).
+* `git annex trust` cannot be used in a bare repository, and currently
+  does something pointless. Same for `untrust` and `semitrust`.
+* `git annex fromkey` does something pointless in a bare repository.
+* `git annex fsck` cannot detect any problems in a bare repository.
+* `git annex unused` will think everything stored in a bare repository
+  is unused.
+* `git annex setkey` is a plumbing-level command, and using it manually
+  to add content to a bare repository is not recommended, since there
+  will be no record accessible by other repositories that the content
+  is stored there.
+* `git-annex-shell inannex` fails in a bare repository

From 486f882471829f7438b49f8f3d9791e4b8b926d6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:49:53 -0400
Subject: [PATCH 0943/8313] wow, non-local bare repos just worked

---
 doc/bare_repositories.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index eb48dfa10b..dee2ba69e7 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -2,7 +2,7 @@ Due to popular demand, git-annex can now be used with bare repositories.
 
 **This is still an experimental feature!** 
 
-Known to work ok, so far for local bare repositories only:
+Known to work ok:
 
 * `git annex move --to` and `--from`, when pointed at a bare repository.
 * `git annex copy` ditto.
@@ -26,4 +26,3 @@ There are a few caveats to keep in mind:
   to add content to a bare repository is not recommended, since there
   will be no record accessible by other repositories that the content
   is stored there.
-* `git-annex-shell inannex` fails in a bare repository

From a9d0538da559c29509c50b664e31ea3b23b7bc14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 15:59:16 -0400
Subject: [PATCH 0944/8313] updates for bare repo support

---
 debian/changelog                         | 2 ++
 doc/bare_repositories.mdwn               | 7 +++++++
 doc/bugs/bare_git_repos.mdwn             | 3 +++
 doc/bugs/fat_support.mdwn                | 4 ++--
 doc/bugs/weird_local_clone_confuses.mdwn | 5 +++++
 doc/git-annex.mdwn                       | 8 ++++----
 6 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 29f2fd134c..3d96934a55 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,8 @@ git-annex (0.22) UNRELEASED; urgency=low
     option.
   * New backends: SHA512 SHA384 SHA256 SHA224
   * fsck: Check for and repair location log damage.
+  * Git annexes can now be attached to bare git repositories. Due to popular
+    demand.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index dee2ba69e7..f08cf605d1 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -1,5 +1,12 @@
 Due to popular demand, git-annex can now be used with bare repositories.
 
+So, for example, you can stash a file away in your
+repos's origin: `git annex move mybigfile --to origin`
+
+Of course, for that to work, the bare repository has to be on a system with
+[[git-annex-shell]] installed. If "origin" is on gitweb, you still can't
+use git-annex to there.
+
 **This is still an experimental feature!** 
 
 Known to work ok:
diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn
index 67917db8a6..f219840e71 100644
--- a/doc/bugs/bare_git_repos.mdwn
+++ b/doc/bugs/bare_git_repos.mdwn
@@ -24,3 +24,6 @@ A possible other approach to the state recording repo is to not
 record state changes on the remote in that case. Git-annex already
 records remote state changes locally whenever it modifies the state of a
 remote. --[[Joey]]
+
+> And... [[done]]! See [[/bare_repositories]] for current status
+> and gotchas. --[[Joey]] 
diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn
index 2a998cf1ad..2a7187a7f6 100644
--- a/doc/bugs/fat_support.mdwn
+++ b/doc/bugs/fat_support.mdwn
@@ -3,8 +3,8 @@ git-annex from being used on USB keys, that would typically
 be VFAT formatted:
 
 - Use of symlinks, which VFAT does not support. Very hard to fix.
-  One possibility is to add [[bare_git_repos]] support, then
-  a git repo on a thumb drive could be used to transfer data.
+  Instead, just use [[/bare_repositories]] on the key,
+  they're supported now.
 - Use of ":" in filenames of object files, also not supported.
   Could easily be fixed by reorganizing the object directory.
 
diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn
index fc652ae5eb..d209dd80de 100644
--- a/doc/bugs/weird_local_clone_confuses.mdwn
+++ b/doc/bugs/weird_local_clone_confuses.mdwn
@@ -7,3 +7,8 @@ new cannot see origin.
 the .git/config has "url=/.../orig/.git". Apparently git is ok with that
 weird construction; probably it treats it as a bare git repo. But git-annex
 just sees a directory w/o a .git subdir, and gives up.
+
+---
+
+Just tested, and the new support for bare repositories didn't solve this.
+--[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2f25f9bf81..5a0c710593 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -327,10 +327,10 @@ Here are all the supported configuration settings.
   from using this remote by default. (You can still request it be used
   by the --from and --to options.)
 
-  This is, for example, useful if the remote is a bare repository,
-  which git-annex does not currently support. Or, it could be used
-  if the network connection between two repositories is too slow
-  to be used normally.
+  This is, for example, useful if the remote is located somewhere
+  without [[git-annex-shell]]. (For example, if it's on GitHub).
+  Or, it could be used if the network connection between two
+  repositories is too slow to be used normally.
 
 * `remote..annex-uuid`
 

From b88637fff10d4d845404882e4ec95cfc071dcac0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 16:22:53 -0400
Subject: [PATCH 0945/8313] prevent trust commands from trying to do things in
 a bare repo

Since they need to stage changes, they would actually, if allowed to run,
succeed, but wipe out existing trust.log content.
---
 Command.hs                 | 9 ++++++++-
 Command/Semitrust.hs       | 2 +-
 Command/Trust.hs           | 2 +-
 Command/Untrust.hs         | 2 +-
 doc/bare_repositories.mdwn | 3 ---
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/Command.hs b/Command.hs
index 86da454195..d54a7052e9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -10,7 +10,7 @@ module Command where
 import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
-import Control.Monad (filterM, liftM)
+import Control.Monad (filterM, liftM, when)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
@@ -104,6 +104,13 @@ isAnnexed file a = do
 		Just v -> a v
 		Nothing -> return Nothing
 
+notBareRepo :: Annex a -> Annex a
+notBareRepo a = do
+	g <- Annex.gitRepo
+	when (Git.repoIsLocalBare g) $ do
+		error "You cannot run this subcommand in a bare repository."
+	a
+
 {- These functions find appropriate files or other things based on a
    user's parameters, and run a specified action on them. -}
 withFilesInGit :: CommandSeekStrings
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index a91d25359c..13c6847e17 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withString start]
 
 start :: CommandStartString
-start name = do
+start name = notBareRepo $ do
 	showStart "semitrust" name
 	Remotes.readConfigs
 	r <- Remotes.byName name
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 3fbff68b89..ea661da2a6 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withString start]
 
 start :: CommandStartString
-start name = do
+start name = notBareRepo $ do
 	showStart "trust" name
 	Remotes.readConfigs
 	r <- Remotes.byName name
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 69d0ab391a..fdf9a83dec 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withString start]
 
 start :: CommandStartString
-start name = do
+start name = notBareRepo $ do
 	showStart "untrust" name
 	Remotes.readConfigs
 	r <- Remotes.byName name
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index f08cf605d1..e3c82d0194 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -19,12 +19,9 @@ Known to work ok:
 
 There are a few caveats to keep in mind:
 
-* Using non-local bare repositories is not tested and probably broken.
 * `git annex init` can be run in a bare repository, but it cannot
   store the name you gave the repository in .git-annex/uuid.log (because
   the bare repository has no such file to commit to).
-* `git annex trust` cannot be used in a bare repository, and currently
-  does something pointless. Same for `untrust` and `semitrust`.
 * `git annex fromkey` does something pointless in a bare repository.
 * `git annex fsck` cannot detect any problems in a bare repository.
 * `git annex unused` will think everything stored in a bare repository

From b5b78f26ecabdb74c05f8200de5f9d054da5cbae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 16:40:55 -0400
Subject: [PATCH 0946/8313] fix up commands that are trouble on bare repos

Most will just abort. init does a basic init and gives a command to
run elsewhere to finish it.
---
 Command/DropUnused.hs      |  2 +-
 Command/FromKey.hs         |  3 +--
 Command/Fsck.hs            |  2 +-
 Command/Init.hs            | 16 ++++++++++++----
 Command/Unused.hs          |  2 +-
 debian/changelog           |  3 ++-
 doc/bare_repositories.mdwn | 25 ++++++++++++-------------
 doc/index.mdwn             |  1 +
 8 files changed, 31 insertions(+), 23 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 63216ce4f1..594564cb72 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -29,7 +29,7 @@ seek = [withStrings start]
 
 {- Drops unused content by number. -} 
 start :: CommandStartString
-start s = do
+start s = notBareRepo $ do
 	m <- readUnusedLog
 	case M.lookup s m of
 		Nothing -> return Nothing
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index d16eff8466..717d528bc9 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -27,9 +27,8 @@ command = [Command "fromkey" paramPath seek
 seek :: [CommandSeek]
 seek = [withFilesMissing start]
 
-{- Adds a file pointing at a manually-specified key -}
 start :: CommandStartString
-start file = do
+start file = notBareRepo $ do
 	key <- cmdlineKey
 	inbackend <- Backend.hasKey key
 	unless inbackend $ error $
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index f8c957053d..76d0e38b4b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -27,7 +27,7 @@ seek :: [CommandSeek]
 seek = [withAttrFilesInGit "annex.numcopies" start]
 
 start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, backend) -> do
+start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
 	return $ Just $ perform key file backend numcopies
 	where
diff --git a/Command/Init.hs b/Command/Init.hs
index 1074d100ea..509c9e51c0 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -41,11 +41,19 @@ perform :: String -> CommandPerform
 perform description = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	describeUUID u description
 	setVersion
-	liftIO $ gitAttributesWrite g
-	gitPreCommitHookWrite g
-	return $ Just cleanup
+	if Git.repoIsLocalBare g
+		then do
+			showLongNote $
+				"This is a bare repository, so its description cannot be committed.\n" ++
+				"To record the description, run this command in a clone of this repository:\n" ++
+				"   git annex describe " ++ (show u) ++ " '" ++ description ++ "'\n\n"
+			return $ Just $ return True
+		else do
+			describeUUID u description
+			liftIO $ gitAttributesWrite g
+			gitPreCommitHookWrite g
+			return $ Just cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 67a2272371..9f3881d595 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -32,7 +32,7 @@ seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
 start :: CommandStartNothing
-start = do
+start = notBareRepo $ do
 	showStart "unused" ""
 	return $ Just perform
 
diff --git a/debian/changelog b/debian/changelog
index 3d96934a55..4405ee2a4a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,7 +11,8 @@ git-annex (0.22) UNRELEASED; urgency=low
   * New backends: SHA512 SHA384 SHA256 SHA224
   * fsck: Check for and repair location log damage.
   * Git annexes can now be attached to bare git repositories. Due to popular
-    demand.
+    demand. Both the local and remote host must have this version of git-annex
+    installed for it to work.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index e3c82d0194..b7ae0b316f 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -1,13 +1,13 @@
+**This is still an experimental feature!** Use with caution.
+
 Due to popular demand, git-annex can now be used with bare repositories.
 
 So, for example, you can stash a file away in your
 repos's origin: `git annex move mybigfile --to origin`
 
 Of course, for that to work, the bare repository has to be on a system with
-[[git-annex-shell]] installed. If "origin" is on gitweb, you still can't
-use git-annex to there.
-
-**This is still an experimental feature!** 
+[[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't
+use git-annex to store stuff there.
 
 Known to work ok:
 
@@ -17,16 +17,15 @@ Known to work ok:
   that is being dropped.
 * `git annex get` can transfer data from a bare repository.
 
-There are a few caveats to keep in mind:
+There are a few caveats to keep in mind when using bare repositories:
 
 * `git annex init` can be run in a bare repository, but it cannot
   store the name you gave the repository in .git-annex/uuid.log (because
-  the bare repository has no such file to commit to).
-* `git annex fromkey` does something pointless in a bare repository.
-* `git annex fsck` cannot detect any problems in a bare repository.
-* `git annex unused` will think everything stored in a bare repository
-  is unused.
-* `git annex setkey` is a plumbing-level command, and using it manually
+  the bare repository has no such file to commit to). Instead, it will
+  tell you a command to run in some non-bare clone of the repository.
+* Some subcommands, like `fsck`, `trust`, `unused` and `fromkey`, 
+  cannot be run in a bare repository. Those subcommands will
+  refuse to do anything.
+* `git annex setkey` is a plumbing-level command; using it manually
   to add content to a bare repository is not recommended, since there
-  will be no record accessible by other repositories that the content
-  is stored there.
+  will be no record that the content is stored there.
diff --git a/doc/index.mdwn b/doc/index.mdwn
index a6d072e093..00a3315cbb 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -54,6 +54,7 @@ files with git.
 * git-annex prevents accidental data loss by [[tracking copies|copies]]
   of your files
 * [[internals]]
+* [[bare_repositories]]
 * [[what git annex is not|not]]
 * git-annex is Free Software, licensed under the [[GPL]].
 

From 1de12a291891463c6d532a10c74cbda1872c8b9b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 16:58:52 -0400
Subject: [PATCH 0947/8313] document describe command

---
 debian/changelog                              |  5 ++-
 doc/git-annex.mdwn                            | 40 +++++++++++--------
 ...what_to_do_when_you_lose_a_repository.mdwn | 18 +++++++++
 3 files changed, 46 insertions(+), 17 deletions(-)
 create mode 100644 doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn

diff --git a/debian/changelog b/debian/changelog
index 4405ee2a4a..ca1c51c4bf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,7 +12,10 @@ git-annex (0.22) UNRELEASED; urgency=low
   * fsck: Check for and repair location log damage.
   * Git annexes can now be attached to bare git repositories. Due to popular
     demand. Both the local and remote host must have this version of git-annex
-    installed for it to work.
+    installed for it to work. This is still a semi-experimental feature;
+    use caution!
+  * describe: New subcommand that can set or change the description of
+    a repository.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 5a0c710593..76f1a577b9 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -83,19 +83,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
   git-annex may refuse to drop content if the backend does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
-* unlock [path ...]
-
-  Normally, the content of annexed files is protected from being changed.
-  Unlocking a annexed file allows it to be modified. This replaces the
-  symlink for each specified file with a copy of the file's content.
-  You can then modify it and `git annex add` (or `git commit`) to inject
-  it back into the annex.
-
-* edit [path ...]
-
-  This is an alias for the unlock command. May be easier to remember,
-  if you think of this as allowing you to edit an annexed file.
-
 * move [path ...]
 
   When used with the --to option, moves the content of annexed files from
@@ -112,16 +99,37 @@ Many git-annex commands will stage changes for later `git commit` by you.
   When used with the --from option, copies the content of annexed files
   from the specified repository to the current one.
 
-* init description
+* unlock [path ...]
 
-  Initializes git-annex with a description of the git repository,
-  and sets up `.gitattributes` and the pre-commit hook.
+  Normally, the content of annexed files is protected from being changed.
+  Unlocking a annexed file allows it to be modified. This replaces the
+  symlink for each specified file with a copy of the file's content.
+  You can then modify it and `git annex add` (or `git commit`) to inject
+  it back into the annex.
+
+* edit [path ...]
+
+  This is an alias for the unlock command. May be easier to remember,
+  if you think of this as allowing you to edit an annexed file.
 
 * lock [path ...]
 
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
+* init description
+
+  Initializes git-annex with a description of the git repository,
+  and sets up `.gitattributes` and the pre-commit hook.
+
+* describe repository description
+
+  Changes the description of a git repository.
+
+  The repository to describe can be specified by git remote name or
+  by uuid. To change the description of the current repository, use
+  "."
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
new file mode 100644
index 0000000000..c914c1bb3d
--- /dev/null
+++ b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
@@ -0,0 +1,18 @@
+So you lost a thumb drive containing a git-annex repository. Or a hard
+drive died or some other misfortune has befallen your data.
+
+Unless you configured backups, git-annex can't get your data back. But it
+can help you deal with the loss.
+
+First, go somewhere that knows about the lost repository, and mark it as
+untrusted.
+
+# git annex untrust usbdrive
+
+To remind yourself later what happened, you can change its description, too:
+
+# git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
+
+This retains the [[location_tracking]] information for the repository.
+Maybe you'll find the drive later. Maybe that's impossible. Either way,
+this lets git-annex tell you why a file is no longer accessible.

From 657395b628b46b378eda77d5dc6edcdde7183b38 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:21:00 -0400
Subject: [PATCH 0948/8313] add describe subcommand

---
 Command.hs          |  2 ++
 Command/Describe.hs | 41 +++++++++++++++++++++++++++++++++++++++++
 Command/Init.hs     |  2 +-
 GitAnnex.hs         |  2 ++
 Remotes.hs          | 12 +++++++++---
 5 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 Command/Describe.hs

diff --git a/Command.hs b/Command.hs
index d54a7052e9..09adc09491 100644
--- a/Command.hs
+++ b/Command.hs
@@ -209,6 +209,8 @@ paramRepeating :: String -> String
 paramRepeating s = s ++ " ..."
 paramOptional :: String -> String
 paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b
 paramPath :: String
 paramPath = "PATH"
 paramKey :: String
diff --git a/Command/Describe.hs b/Command/Describe.hs
new file mode 100644
index 0000000000..643ca04718
--- /dev/null
+++ b/Command/Describe.hs
@@ -0,0 +1,41 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Describe where
+
+
+import Command
+import qualified GitRepo as Git
+import qualified Remotes
+import UUID
+import Messages
+import qualified Command.Init
+
+command :: [Command]
+command = [Command "describe" (paramPair paramRemote paramDesc) seek
+	"change description of a repository"]
+
+seek :: [CommandSeek]
+seek = [withString start]
+
+start :: CommandStartString
+start params = notBareRepo $ do
+	let (name, description) =
+		case (words params) of
+			(n:d) -> (n,unwords d)
+			_ -> error "Specify a repository and a description."
+	
+	showStart "describe" name
+	Remotes.readConfigs
+	r <- Remotes.byName name
+	return $ Just $ perform r description
+
+perform :: Git.Repo -> String -> CommandPerform
+perform repo description = do
+	u <- getUUID repo
+	describeUUID u description
+	return $ Just $ Command.Init.cleanup
diff --git a/Command/Init.hs b/Command/Init.hs
index 509c9e51c0..2c5fdc2fc9 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -62,7 +62,7 @@ cleanup = do
 	liftIO $ Git.run g "add" [File logfile]
 	liftIO $ Git.run g "commit" 
 		[ Params "-q -m"
-		, Param "git annex init"
+		, Param "git annex repository description"
 		, File logfile
 		]
 	return True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 3be222874f..b26714a598 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -26,6 +26,7 @@ import qualified Command.DropKey
 import qualified Command.SetKey
 import qualified Command.Fix
 import qualified Command.Init
+import qualified Command.Describe
 import qualified Command.Fsck
 import qualified Command.Unused
 import qualified Command.DropUnused
@@ -50,6 +51,7 @@ cmds = concat
 	, Command.Unlock.command
 	, Command.Lock.command
 	, Command.Init.command
+	, Command.Describe.command
 	, Command.Unannex.command
 	, Command.Uninit.command
 	, Command.PreCommit.command
diff --git a/Remotes.hs b/Remotes.hs
index 4dcc4c9adf..a7d6be67df 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -211,17 +211,23 @@ repoNotIgnored r = do
 same :: Git.Repo -> Git.Repo -> Bool
 same a b = Git.repoRemoteName a == Git.repoRemoteName b
 
-{- Looks up a remote by name. -}
+{- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex Git.Repo
 byName "." = Annex.gitRepo -- special case to refer to current repository
 byName name = do
 	when (null name) $ error "no remote specified"
 	g <- Annex.gitRepo
-	let match = filter (\r -> Just name == Git.repoRemoteName r) $
-		Git.remotes g
+	match <- filterM matching $ Git.remotes g
 	when (null match) $ error $
 		"there is no git remote named \"" ++ name ++ "\""
 	return $ head match
+	where
+		matching r = do
+			if Just name == Git.repoRemoteName r
+				then return True
+				else do
+					u <- getUUID r
+					return $ (name == u)
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool

From 7b2762fb9226ca76dce6510dd821bfe432c1cfa2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:22:17 -0400
Subject: [PATCH 0949/8313] better quoting of description via show

---
 Command/Init.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 2c5fdc2fc9..6618351699 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -47,7 +47,7 @@ perform description = do
 			showLongNote $
 				"This is a bare repository, so its description cannot be committed.\n" ++
 				"To record the description, run this command in a clone of this repository:\n" ++
-				"   git annex describe " ++ (show u) ++ " '" ++ description ++ "'\n\n"
+				"   git annex describe " ++ show u ++ " " ++ show description ++ "\n\n"
 			return $ Just $ return True
 		else do
 			describeUUID u description

From ea4ea466f550b71ab389d6651c90bf521e30a1c1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:23:44 -0400
Subject: [PATCH 0950/8313] update

---
 doc/walkthrough.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index f400191c61..1e8ad7e988 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -22,4 +22,5 @@ A walkthrough of the basic features of git-annex.
 	fsck:_when_things_go_wrong
 	backups
 	untrusted_repositories
+	what_to_do_when_you_lose_a_repository
 """]]

From eeadc2e3e041b77e409bfe258ebe6a778f70ccb5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:24:46 -0400
Subject: [PATCH 0951/8313] formatting

---
 doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
index c914c1bb3d..1159b22171 100644
--- a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
+++ b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
@@ -7,11 +7,11 @@ can help you deal with the loss.
 First, go somewhere that knows about the lost repository, and mark it as
 untrusted.
 
-# git annex untrust usbdrive
+	git annex untrust usbdrive
 
 To remind yourself later what happened, you can change its description, too:
 
-# git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
+	git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
 
 This retains the [[location_tracking]] information for the repository.
 Maybe you'll find the drive later. Maybe that's impossible. Either way,

From e49e6a403b16cd12594239e18fa708aab61243a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:33:15 -0400
Subject: [PATCH 0952/8313] avoid failing if a local repo is not currently
 available

The added check if a repo is bare means its config needs to be read, but
in this case it cannot be. That means that a repo currently not available
is assumed to be non-bare.
---
 GitRepo.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index ba01251218..92539eb826 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -163,12 +163,15 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+configAvail ::Repo -> Bool
+configAvail Repo { config = c } = c /= Map.empty
+
 repoIsLocalBare :: Repo -> Bool
-repoIsLocalBare r@(Repo { location = Dir _ }) = configBare r
+repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
 repoIsLocalBare _ = False
 
 repoIsLocalFull :: Repo -> Bool
-repoIsLocalFull r@(Repo { location = Dir _ }) = not $ configBare r
+repoIsLocalFull r@(Repo { location = Dir _ }) = configAvail r && not (configBare r)
 repoIsLocalFull _ = False
 
 assertLocal :: Repo -> a -> a

From b27b0d5cd463afcce8a95730a49288edd89eb8b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 17:35:44 -0400
Subject: [PATCH 0953/8313] remove unused code

---
 GitRepo.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 92539eb826..8e3f14ee5a 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -17,7 +17,6 @@ module GitRepo (
 	repoIsUrl,
 	repoIsSsh,
 	repoIsLocalBare,
-	repoIsLocalFull,
 	repoDescribe,
 	repoLocation,
 	workTree,
@@ -170,10 +169,6 @@ repoIsLocalBare :: Repo -> Bool
 repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
 repoIsLocalBare _ = False
 
-repoIsLocalFull :: Repo -> Bool
-repoIsLocalFull r@(Repo { location = Dir _ }) = configAvail r && not (configBare r)
-repoIsLocalFull _ = False
-
 assertLocal :: Repo -> a -> a
 assertLocal repo action = 
 	if not $ repoIsUrl repo

From 42259eee9200588f69af2b56557d5d191d426ad0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:02:29 -0400
Subject: [PATCH 0954/8313] support git funky remote syntaxes

* Look for dir.git directories the same as git does.
* Support remote urls specified as relative paths.
* Support non-ssh remote paths that contain tilde expansions.
---
 GitRepo.hs         | 114 ++++++++++++++++++++++++++++-----------------
 debian/changelog   |   3 ++
 git-annex-shell.hs |   4 +-
 3 files changed, 77 insertions(+), 44 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 8e3f14ee5a..bb0e3d5b7a 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -11,7 +11,7 @@
 module GitRepo (
 	Repo,
 	repoFromCwd,
-	repoFromPath,
+	repoFromAbsPath,
 	repoFromUrl,
 	localToUrl,
 	repoIsUrl,
@@ -49,7 +49,7 @@ module GitRepo (
 	encodeGitFile,
 	typeChangedFiles,
 	typeChangedStagedFiles,
-	absDir,
+	repoAbsPath,
 	reap,
 
 	prop_idempotent_deencode
@@ -57,6 +57,7 @@ module GitRepo (
 
 import Control.Monad (unless)
 import System.Directory
+import System.FilePath
 import System.Posix.Directory
 import System.Posix.User
 import System.Posix.Process
@@ -98,15 +99,23 @@ newFrom l =
 		remoteName = Nothing
 	}
 
-{- Local Repo constructor. -}
-repoFromPath :: FilePath -> Repo
-repoFromPath dir = newFrom $ Dir dir
+{- Local Repo constructor, requires an absolute path to the repo be
+ - specified. -}
+repoFromAbsPath :: FilePath -> IO Repo
+repoFromAbsPath dir
+	| "/" `isPrefixOf` dir = do
+ 		-- Git always looks for "dir.git" in preference to
+		-- to "dir", even if dir ends in a "/".
+		let dir' = (dropTrailingPathSeparator dir) ++ ".git"
+		e <- doesDirectoryExist dir'
+		return $ newFrom $ Dir $ if e then dir' else dir
+	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
 
 {- Remote Repo constructor. Throws exception on invalid url. -}
-repoFromUrl :: String -> Repo
+repoFromUrl :: String -> IO Repo
 repoFromUrl url
-	| startswith "file://" url = repoFromPath $ uriPath u
-	| otherwise = newFrom $ Url u
+	| startswith "file://" url = repoFromAbsPath $ uriPath u
+	| otherwise = return $ newFrom $ Url u
 		where
 			u = case (parseURI url) of
 				Just v -> v
@@ -356,31 +365,35 @@ configRead r = assertLocal r $ error "internal"
 hConfigRead :: Repo -> Handle -> IO Repo
 hConfigRead repo h = do
 	val <- hGetContentsStrict h
-	return $ configStore repo val
+	configStore repo val
 
 {- Parses a git config and returns a version of the repo using it. -}
-configStore :: Repo -> String -> Repo
-configStore repo s = r { remotes = configRemotes r }
-	where r = repo { config = configParse s }
+configStore :: Repo -> String -> IO Repo
+configStore repo s = do
+	rs <- configRemotes r
+	return $ r { remotes = rs }
+	where
+		r = repo { config = configParse s }
 
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
-configRemotes :: Repo -> [Repo]
-configRemotes repo = map construct remotepairs
+configRemotes :: Repo -> IO [Repo]
+configRemotes repo = mapM construct remotepairs
 	where
 		remotepairs = Map.toList $ filterremotes $ config repo
 		filterremotes = Map.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
 		remotename k = split "." k !! 1
-		construct (k,v) = (gen v) { remoteName = Just $ remotename k }
+		construct (k,v) = do
+			r <- gen v
+			return $ r { remoteName = Just $ remotename k }
 		gen v	| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
-			| otherwise = repoFromPath v
+			| otherwise = repoFromRemotePath v repo
 		-- git remotes can be written scp style -- [user@]host:dir
-		-- where dir is relative to the user's home directory.
 		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
@@ -389,6 +402,7 @@ configRemotes repo = map construct remotepairs
 				dir = join ":" $ drop 1 bits
 				slash d	| d == "" = "/~/" ++ dir
 					| d !! 0 == '/' = dir
+					| d !! 0 == '~' = '/':dir
 					| otherwise = "/~/" ++ dir
 
 {- Parses git config --list output into a config map. -}
@@ -503,37 +517,51 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
 				e_utf c = concat $ map showoctal $
 						(encode [c] :: [Word8])
 
-
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
 prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
 
-{- Git ssh remotes can have a directory that is specified relative
- - to a home directory. This converts such a directory to an absolute path.
- - Note that it has to run on the remote system.
+{- Constructs a Repo from the path specified in the git remotes of
+ - another Repo. -}
+repoFromRemotePath :: FilePath -> Repo -> IO Repo
+repoFromRemotePath dir repo = do
+	dir' <- expandTilde dir
+	repoFromAbsPath $ workTree repo  dir'
+
+{- Git remotes can have a directory that is specified relative
+ - to the user's home directory, or that contains tilde expansions.
+ - This converts such a directory to an absolute path.
+ - Note that it has to run on the system where the remote is.
  -}
-absDir :: String -> IO String
-absDir d
-	| "/" `isPrefixOf` d = expandt d
-	| otherwise = do
-		h <- myhomedir
-		return $ h ++ d
+repoAbsPath :: FilePath -> IO FilePath
+repoAbsPath d = do
+	d' <- expandTilde d
+	h <- myHomeDir
+	hPutStrLn stderr $ "repoAbsPath " ++ d
+	return $ h  d'
+
+myHomeDir :: IO FilePath
+myHomeDir = do
+	uid <- getEffectiveUserID
+	u <- getUserEntryForID uid
+	return $ homeDirectory u
+
+expandTilde :: FilePath -> IO FilePath
+expandTilde = expandt True
 	where
-		homedir u = (homeDirectory u) ++ "/"
-		myhomedir = do
-			uid <- getEffectiveUserID
-			u <- getUserEntryForID uid
-			return $ homedir u
-		expandt [] = return ""
-		expandt ('/':'~':'/':cs) = do
-			h <- myhomedir
-			return $ h ++ cs
-		expandt ('/':'~':cs) = do
+		expandt _ [] = return ""
+		expandt _ ('/':cs) = do
+			v <- expandt True cs
+			return ('/':v)
+		expandt True ('~':'/':cs) = do
+			h <- myHomeDir
+			return $ h  cs
+		expandt True ('~':cs) = do
 			let (name, rest) = findname "" cs
 			u <- getUserEntryForName name
-			return $ homedir u ++ rest
-		expandt (c:cs) = do
-			v <- expandt cs
+			return $ homeDirectory u  rest
+		expandt _ (c:cs) = do
+			v <- expandt False cs
 			return (c:v)
 		findname n [] = (n, "")
 		findname n (c:cs)
@@ -546,10 +574,12 @@ repoFromCwd = do
 	cwd <- getCurrentDirectory
 	top <- seekUp cwd isRepoTop
 	case top of
-		(Just dir) -> return $ repoFromPath dir
+		-- repoFromAbsPath is not used to avoid looking for
+		-- "dir.git" directories.
+		(Just dir) -> return $ newFrom $ Dir dir
 		Nothing -> error "Not in a git repository."
 
-seekUp :: String -> (String -> IO Bool) -> IO (Maybe String)
+seekUp :: FilePath -> (FilePath -> IO Bool) -> IO (Maybe FilePath)
 seekUp dir want = do
 	ok <- want dir
 	if ok
diff --git a/debian/changelog b/debian/changelog
index ca1c51c4bf..a854235ce4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,9 @@ git-annex (0.22) UNRELEASED; urgency=low
     use caution!
   * describe: New subcommand that can set or change the description of
     a repository.
+  * Look for dir.git directories the same as git does.
+  * Support remote urls specified as relative paths.
+  * Support non-ssh remote paths that contain tilde expansions.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index aeaadcbf85..e8a744748b 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -60,8 +60,8 @@ builtins = map cmdname cmds
 
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
-	dir' <- Git.absDir dir
-	let gitrepo = Git.repoFromPath dir'
+	dir' <- Git.repoAbsPath dir
+	gitrepo <- Git.repoFromAbsPath dir'
 	dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header
 
 external :: [String] -> IO ()

From 14e0d01b58c504e889b636e952e165f3cf8ab6c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:19:01 -0400
Subject: [PATCH 0955/8313] remove debugging printf

---
 GitRepo.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index bb0e3d5b7a..ef8ad25baa 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -537,7 +537,6 @@ repoAbsPath :: FilePath -> IO FilePath
 repoAbsPath d = do
 	d' <- expandTilde d
 	h <- myHomeDir
-	hPutStrLn stderr $ "repoAbsPath " ++ d
 	return $ h  d'
 
 myHomeDir :: IO FilePath

From bc2df77642490be830cde9a46428d25051120893 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:34:30 -0400
Subject: [PATCH 0956/8313] Bugfix: When fsck detected and moved away corrupt
 file content, it did not update the location log.

---
 Content.hs       | 1 +
 debian/changelog | 2 ++
 test.hs          | 3 ++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Content.hs b/Content.hs
index 6b8316c082..bcd4ac0e13 100644
--- a/Content.hs
+++ b/Content.hs
@@ -147,6 +147,7 @@ moveBad key = do
 	liftIO $ allowWrite (parentDir src)
 	liftIO $ renameFile src dest
 	liftIO $ removeDirectory (parentDir src)
+	logStatus key ValueMissing
 	return dest
 
 {- List of keys whose content exists in .git/annex/objects/ -}
diff --git a/debian/changelog b/debian/changelog
index a854235ce4..ab9aa3a272 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,8 @@ git-annex (0.22) UNRELEASED; urgency=low
   * Look for dir.git directories the same as git does.
   * Support remote urls specified as relative paths.
   * Support non-ssh remote paths that contain tilde expansions.
+  * Bugfix: When fsck detected and moved away corrupt file content, it did
+    not update the location log.
 
  -- Joey Hess   Sun, 13 Feb 2011 00:48:02 -0400
 
diff --git a/test.hs b/test.hs
index 936e4da175..31960bb2e2 100644
--- a/test.hs
+++ b/test.hs
@@ -367,13 +367,14 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
 			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
 			fsck_should_fail "content not replicated to enough non-untrusted repositories"
+
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
 			Content.allowWrite f
 			writeFile f (changedcontent f)
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with corrupted file content"
-			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem"
+			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
 		fsck_should_fail m = do
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with " ++ m

From 8514a2a98783d520e7115635a27428923b81ffb9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:55:56 -0400
Subject: [PATCH 0957/8313] updates

---
 doc/git-annex-shell.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 1231ef668d..b32114a74d 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -11,11 +11,15 @@ git-annex-shell [-c] command [params ...]
 git-annex-shell is a restricted shell, similar to git-shell, which
 can be used as a login shell for SSH accounts.
 
+Since its syntax is identical to git-shell's, it can be used as a drop-in
+replacement anywhere git-shell is used. For example it can be used as a 
+user's restricted login shell.
+
 # COMMANDS
 
 * configlist directory
 
-  This outputs the git configuration, in the same form as
+  This outputs a subset of the git configuration, in the same form as
   `git config --list`
 
 * inannex directory [key ...]

From c5c7eaf00917d8654c82de5a4b17465b354f8fbf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Mar 2011 21:56:03 -0400
Subject: [PATCH 0958/8313] prep for release

---
 debian/changelog           | 13 +++++++------
 doc/bare_repositories.mdwn |  6 ++----
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ab9aa3a272..79be983679 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,20 +5,21 @@ git-annex (0.22) UNRELEASED; urgency=low
     for his help eliminating the infestation... for now.)
   * Make test suite not rely on a working cp -pr.
     (The Unix wars are still ON!)
+  * Git annexes can now be attached to bare git repositories.
+    (Both the local and remote host must have this version of git-annex
+    installed for it to work.)
   * Support filenames that start with a dash; when such a file is passed
     to a utility it will be escaped to avoid it being interpreted as an
-    option.
+    option. (I went a little overboard and got the type checker involved
+    in this, so such files are rather comprehensively supported now.)
   * New backends: SHA512 SHA384 SHA256 SHA224
-  * fsck: Check for and repair location log damage.
-  * Git annexes can now be attached to bare git repositories. Due to popular
-    demand. Both the local and remote host must have this version of git-annex
-    installed for it to work. This is still a semi-experimental feature;
-    use caution!
+    (Supported on systems where corresponding shaNsum commands are available.)
   * describe: New subcommand that can set or change the description of
     a repository.
   * Look for dir.git directories the same as git does.
   * Support remote urls specified as relative paths.
   * Support non-ssh remote paths that contain tilde expansions.
+  * fsck: Check for and repair location log damage.
   * Bugfix: When fsck detected and moved away corrupt file content, it did
     not update the location log.
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index b7ae0b316f..a9ccab8d1b 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -1,9 +1,7 @@
-**This is still an experimental feature!** Use with caution.
-
 Due to popular demand, git-annex can now be used with bare repositories.
 
-So, for example, you can stash a file away in your
-repos's origin: `git annex move mybigfile --to origin`
+So, for example, you can stash a file away in the origin:
+`git annex move mybigfile --to origin`
 
 Of course, for that to work, the bare repository has to be on a system with
 [[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't

From 69c14d130bc7a754e3a4fa184ff317690ad48ca6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Mar 2011 12:31:01 -0400
Subject: [PATCH 0959/8313] update

---
 doc/distributed_version_control.mdwn |  13 +++++++++++++
 doc/future_proofing.mdwn             |  24 ++++++++++++++++++++++++
 doc/not.mdwn                         |   8 ++++++++
 doc/repomap.png                      | Bin 0 -> 129065 bytes
 doc/transferring_data.mdwn           |  14 ++++++++++++++
 doc/use_case/Alice.mdwn              |   6 ++++--
 doc/use_case/Bob.mdwn                |   7 +++++++
 7 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/distributed_version_control.mdwn
 create mode 100644 doc/future_proofing.mdwn
 create mode 100644 doc/repomap.png
 create mode 100644 doc/transferring_data.mdwn

diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
new file mode 100644
index 0000000000..f9cdb7e99e
--- /dev/null
+++ b/doc/distributed_version_control.mdwn
@@ -0,0 +1,13 @@
+In git, there can be multiple clones of a repository, each clone can 
+be independently modified, and clones can push or pull changes to
+one-another to get back in sync.
+
+git-annex preserves that fundamental distributed nature of git, while
+dropping the requirement that, once in sync, each clone contains all the data
+that was committed to each other clone. Instead of storing the content
+of a file in the repository, git-annex stores a pointer to the content.
+
+Each git-annex repository is responsible for storing some of the content,
+and can copy it to or from other repositories. [[Location_tracking]]
+information is committed to git, to let repositories inform other
+repositories what file contents they have available.
diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn
new file mode 100644
index 0000000000..4d4939b5a9
--- /dev/null
+++ b/doc/future_proofing.mdwn
@@ -0,0 +1,24 @@
+Imagine putting a git-annex drive in a time capsule. In 20, or 50, or 100
+years, you'd like its contents to be as accessible as possible to whoever
+digs it up.
+
+This is a hard problem. git-annex cannot completly solve it, but it does
+its best to not contribute to the problem. Here are some aspects of the
+problem:
+
+* How are files accessed? Git-annex carefully adds minimal complexity
+  to access files in a repository. Nothing needs to be done to extract
+  files from the repository; they are there on disk in the usual way,
+  with just some symlinks pointing at the annexed file contents.
+  Neither git-annex nor git is needed to get at the file contents.
+
+* What file formats are used? Will they still be readable? To deal with
+  this, it's best to stick to plain text files, and the most common
+  image, sound, etc formats. Consider storing the same content in multiple
+  formats.
+
+* What filesystem is used on the drive? Will that filesystem still be
+  available?
+
+* What is the hardware interface of the drive? Will hardware still exist
+  to talk to it?
diff --git a/doc/not.mdwn b/doc/not.mdwn
index 80c0acafaf..fe6e1b37d9 100644
--- a/doc/not.mdwn
+++ b/doc/not.mdwn
@@ -30,3 +30,11 @@
   situations. It lacks git-annex's support for widely distributed storage,
   using only a single backend data store. It also does not support
   partial checkouts of file contents, like git-annex does.
+
+* git-annex is also not [boar](http://code.google.com/p/boar/),
+  although it shares many of its goals and characteristics. Boar implements
+  its own version control system, rather than simply embarcing and
+  extending git. And while boar supports distributed clones of a repository,
+  it does not support keeping different files in different clones of the
+  same repository, which git-annex does, and is an important feature for
+  large-scale archiving.
diff --git a/doc/repomap.png b/doc/repomap.png
new file mode 100644
index 0000000000000000000000000000000000000000..4d334aec947e33900920351ccbd094b22c4aed3a
GIT binary patch
literal 129065
zcmce8c{G;o+wMcANM;!`Bua#Y2J=uPN~uhlr9{S3GLMNAl~h7Xip(h#GAAOEF(Q>Q
zQOXqdarM5x{e5eHYyY>`w%&K8Jf7#iui-q8>AEBL8}4RiJg{4nlHDXQg1^w&>hIRZpZJ>7dLDp3R=Vk$dQvD%9OQp0%B@5m
z{3E@Wo`DX1&k7n=?v=EcXkSq%!W2Di4db)jtF{^vK^|Ic3C
zrdeM=GrdJqOKV_gs7{lW%_uYPc#G&+AD?_yZEbvVR`X!pW%tjo*v_6mU%;cS-S&x=
zp5AVHxJ8PFjZH4};Mmh+Z}_-_E?h8E_CLN?TU(mBE|i1q@0Z;F_=Zhu+dn`0@89UF
z|JQA{eST3|YTg#7)4U@&{!9${!QZ=o6DmZ_ieLSoZ`l$R>$vj5@KFi&NH<(14IQ1R
zii#k`+R5o+&>v;cuV7TQT!>U!Cmz2kkrI=sI
zcrKYgb=FWH_NxbmzoZxm3nwFN9{YQ?}#jV%xpO{!p
zDLV6w@2N{qc+WG}LEKWpHh0mb*+G^MFV1mDN=lmKT2sAv@dEE39U4mW=SPD~t6fQP
zvCW4f_uOaCBJ%UMT)cSE4IkwEu}qMjWv%Z3>sk(u5NY$LDJMI!H%Lo|B_|67hlEVk
z2?gkC%{C+kSX&EIteXAoo80PyxKl7Zzq3^tRphx3XkTk
zOYG~}r^h<>m6Qf7FI~|Un;p%}aT{q}zc+E4QMY6%e!5Cv+y0WA(o&VO`O!@y&8n)Z
z+kB=swC7k|{`ASV?frYb@8@SH2bhw39FzURvyVQfrg(XI<+=@M($Led+rEA7dXv@e
z-SnoWrbV7Zs~H&?txud-DIp1bSG
zhKAXC&2Q@KSC%jS=Ku8RliT+`73H&&>U^)1T>GMP%9n1j2;;qsgmU
zFh}=V{5o>1Tyys=cI4*f4z0~8EA*ZCP%JAi9}*l)P3dpCv(t6)&sa$5y?g7#4HGZO
zl+99~|2m6LP&aI+eIO-@d(S=n%r9xJY^w>equ^83sqN}Uk}Y5!SyrQAX4fmp-+uVUBz
zyT|SA?MZQ=T0v)x;=;;6v(HlY%a=U&!Cil_wu{^bs9RfGg*R-(5)7@
zkY`u7=kmzd7?X~U4yJI$+}xZqvxKOqsFUWFxFLf9{pnRpOG^n`92m{3F)D{9fB&u>
zsEUv~_rsn$1O=HUW+izwzHAx(&u>J--TwK}fBz;vkW4SWP$K^M`rqG>Isg0hKfn3^
z^6H1*uAt0c7(R6U#*KT~*{dBK99Sc@A7*Fo($S%HI(ag!2W#;D0=Cq>2M-=p$c>JU
z?l(1EN!gF8l9*@_vcSd3ney=AhDB2>I_e+_D)p5cHzF=xWaQ-HO6&Rj`EyD}#%uZr
z+#|WehYsvfH5)c;xYEbS#AIY_9NOA?;M(=;dgbi_R%T`wRW}Mr@&)BIJTh
zOc?S^q!tAa9Xd2TI;yF!&$MI5j<+o>8g(&(ceAp3`yV`dv@%GYA}uYAn@~S=hzlQy
zm4%Nu)jlSP7N;wwX==**x}m}7;K7j8RM89CqMU2i=;w@$n4{brICzkJ#@+oFa4U2b
zrAsBFIlXV+vd+)X%WmCz&0j{Qd3gs}5Lh3>cf{EvCugVZMx@uRTd{rn_E+`w8kUx8
zF#{9|8V6a;Qc_Z+-5xx6aCmI2>eVZT@bGY1S=qq6Jon7urlUeXe*OAoHOKSi%a>P;
zjoOBWtMdyAVy|Bh8y|Nbd5Y$E|G|SjRV9gu0@Ksed~F$i3%^O>DlXnmLrs+&{Vzyp
zJKWaRCcJ$+FN=_>Giymfe!ehwln5&WspBV4o;-W*Tt#K2rq4-SkJstbrq$^WGcqc>
zyZPGN+XJ3k;fitBvA1qrympQA{{8#?E%$aaFfmo#)rt^ij}(@WpkKFc-3ehOC8db#
z*SVymrSGPsytE~C0&``gf7sG8=JRKJF)=aWjT=`+Mn>xD>1iB2D!6s)R%;iROPi0D
zSbu&Mn_E~Ih6aV-?&9lNg_}u9Pmfru;mNUUZ0+Dc%g@j6HQpI}wGJOmI#Nr|wVO91J3CFPYimVSRo8CZxDo9U
z)1-|zlRMEW=JmZ#(9qB@_sJ9c3Nr?*3KXW!t}ca=@3=+M>#$JW7Jj=301{yOu3W#a
zv+(;{OPc^-s?
z@Bv3lyn+e}DT7v-BL0vYu2nO
zDk`#``c#9#C4<(4Zs9r9G!P|I7Kqp3a^7MP0Fv)vD@zIdJ6u%iC^wp#nrJzp!^6i>
z9c&sC3U81EZsiKt0^i|Lo_7-y}C}(o$Ea
z7$zx40!^;s+sc5kz6#u`Xr_T%zW
zL~_6quU7b~iqG{|x1T{HdPr+qt{|FvV?VZ7>G>Jk(Y8k@OO+1{ZeNLw4RYyuDi`?9
z+|JH!g_@cgY0iuDzsRuQ=iw+6cT-b?Y5#43PDe4vhIe*^Z9Mp(e|qGd$@4SPYn9Ii
z(lT+Y8yH-oP30zAwydm7ch8>6(f66|KR)a2O;YAfH@p+9EouXj^S3#emHqkS``9Fp
z+TY(lLFK$6mRD$+{tb`D8P8X*Ub%G@KC@s#eUdWe%OY@x{Nk=j657GQ*
z{@Ogk!omhZSX2qVtE{Zd1%kl($dg_5BP;mnQ$Uv$qIS`&Y86S{mmZ&
zDn6W>nD65Zrm@Nf9Lpd?j|>ZcX&DTeoh__J^?SH#3`Woap)0gFc++^UEnAA%Wb}ogJqu
zvEn`qcLsFO{&6UYD4!@+UVQ$meCDy;+8tgDeSLjzTU&$Cd{EhPedj!|^3{I2-oT=`
zcUe{Q`I&FRGBV4rPct$z*E~7#4xoz>VC>AdZkx9$8o*hanP>eM=I3p3CzC&aR^g`R
zSdst{n6Jj>HVc>11Ox=^y}5<9qM|~Iu24@wK_NlmBscJ3#J8Sj&mMYmFoq3x7jDAs
z@;M91+0qy$t1?rVOM*Hg+E2WDFwkESaw%qJ;d{9qM)}h3fc5~U0Pl}c;|p7cet6rI
zmt`)rJvun>{rhsG-}tw01j8Obethe;K5Qs=tWdf-pF2BL#!5iT`ch8MIgI{(AZ_-9
zL1JH0bzkXubfxa`@)NGEt^%sQJXa<5ePu$$C}eO3VDow1Rp2PX9hK+5IM3~UieJ%L
z{qSLK3=!+vwK1ckPN5MI%XTNf0;#%BebPbk71^{&pKrtBpFe%G3NKz2D+`*8Y0bPF
zY()8I!WfvCnDkdia)7y#DHC0{ZeU;_2t>>G=NqLVW>w@y6++K)iqm?`+$P7Om>SS%}y&wHP7k#S9p59cUg
zV_OUWj3&()M8NFxT-yEn6pFHn3cvI6SaZOaiHW{`M-C1Siu#EYBmCvzxE6F)ih7P^
zDId*;bn}L>lfRy*`pv)iUA*|a$2DQ=iB?ao(|}=_K20;SMdjSX^W@d(%NQIk7onnpF5jDE^
z@t~2>xo@84C`~9<+#p0C(<%A+?Cg;=Si@OAifxV^Yws`K_ph4Zy4O#4VQjox8r6iB
zA}l5KxZ-#D(w_`qA&T-y_~H|=@6xH0S>J1=xuf`d9bH{n`t_n_t?HtePyac4P=D`U
znn`=*!b_K!uU)%FNqN7zWuv4dH3it{2Q8rk1mV5eqNo`D;DI=R0)~uPNL7F$8?fZ7
zy=#{yU<<$i6SvgyxWSz}c2HAPmuBq5OnDi}A;DMZ)FH!<
zB_u1b&5d&6#0hXZozthu6HjL#gAOAGjbvtqufifHKh74e7Tm!}kv;Wc8$NNXiG=Uh#+#GBe$_xC!j-Q|xP4no
zn3eSKr%#_I$UCmiwkq$F8u~R5Aw+$)Rk{?76x_Z7dw$vd#X0@cll>axit_V)cY@**
zrBWMxFqjoAm$$=6>3sxEe`~!3->-lxB?@X^Zh2rA!tz*X$N_wvAM^ykc
z4Pn-ojg6}^kL1*%WZ+s)L0N$=q=kl?TpV5qJq55bbZW4!s=As6oaXiG*N|fB{>=Sk
zTe}vQS%_Nv*tCESgb3IleR;*<9Lr09m(NOu=D(LO$3W?^`&1qICjyIUn$A6Pi_b=j
z)aX9i-7x|cXxeOie2nz;5!?Y~YRiA*kG;7?i-KiQ=6jPxc<~;#1w=wyfIA+gQx><&
z4sSLBFp*|92YXr~vYVTm+e;Ktwl!-yQkz^L7?KtSx^HA*5nUC*GB|iyjV@48=)uDi
z)z#J3DCxuPS^Tvzf(^m{nc<^bwrrXG^RpQXo({!=hKA-zg6Gb2Kg5)kl~eBDrvbYG
zi{=75b~|&XHgUV>Y=6tg9iCrT%}%&!i3;Pp_^V@ba`NTm
zKotRc#)l7={rb7%{BJ4nZ3e(Gtb!D~x)_tE&b-0FP;^sX(#Iq{N=>EyIn-Q7u+qp#
zEbvu6QO
zm?6p7P*8RWp?4C5l#qAs=P`0q1d^6`jRap-ooB$ll3HE3fhdCGT?H+LCy#&{eCly6
zuL1hLo1R|x>E979+-NdM28zw8RU-F}2Pt8f@UCrV!Q!
z)(Fl{&@cI{Cr^Cu9(DouBB+jO&E{&15vzbeG
zl!B2YJC_j7%*=)EfFt-cf{xLr_nVkp=ud6j1Lo%ZglnL@?}s=-q;oI;lYIM?zJ7i!
zceUtOtyShpN=hP><>pOZp|{?hSXZbg7vH|!pSaDP8n|LVga?qm!;ejZuz=LHv_iuE
zh3P)>U?6EC3(&Gq&8xAPFkZCh&YgSW`f)AT`sCD95D)<%QJ>TCjp!8G+BEo8R?tKU
zGs7bzHk17oZeCv6XU-@R8IZ65v?G+k;dkjQC?5S^zJ!4WXc!vCu=w&DA2|~7-23|{
z0DeOFF|k*Yl2)UtcujmTofkQP#fP0h7&3?$I?)P{Y0ztXHV7bprO=rD8=T+f-??@z
z6lex2>BC2lDqg>41hT5Vz_yNXc76Q@I*znbX38}wEtHy8DkzYr6%3$61>VWEwy>t(iey9k9>R8)lAwfoSa>(k!2h1j^bVDuHyty?((
zEBN^M2oi?6|F-nJcQzGlr?!i)h)>m^F7q1u=HP?98c=zoGz=bKVX&-KKJJ$3x^nB5
zEt(Ol1#y4tOQC!fCU`c#dUfTg*GCAM0AaFFN^oNzK78o^{CT?dYM!1LyR!cu9>CI*
zve~7NAFqPOKK!9r9u=VF(LssM?(S6VQ?l4^-1ljKU`%)wfGW6QKN=wJs}?s8y=nz&
zgV$)=O03FI;1*{W7n@V3R`c@ST(jdhx|{o0yR=2QzaX{~B*NAOpr(QT{y@CNz4J2@
z23yKC%m==GyM%gY_v3TD6y4v7qqVK{{KcrK*eh287iPZeukeN*M98GAtthhuntyJ7
zzLWcy%aOx}MR)Ar12DzzC891UYzqdhsJOV|{rf`G5=MP7cKn{OoE#ewfea0I`S|Pr
z$M{EzdW)^m+0&zc($U6-iiwFS;pWY}UoTdMoPl?A;J|?^n3ntZ#Xgh=2zh_+tHBkD
zKzc&M0Pq16scLOy$MP>a_fs5lJw0G5%nvqTYX7V{Z7J?`_)KUBnkFVZ>>M0MN00Uj
zU+d}VxreuMb9dJOpW>wr|wC*=sr7Y2`w
zvhP@>l7d3mqemMd1*Iv$yCA4+hd)NgfAQB53or;!P98OXOVux!oUyVGmb#IgR!^`ANObgRiR2M3YO
zn^$9lqPQ-WFR!wHH}R0+Hu
z2rOhJhxpbxv0yac3(?Wq$B%EA=zSgmv`*K7%cEssVSzW6YY^c^fSP3VKp@&aFW&>j27%aof9WE?%A`4FZ8z$
z{@4#M6EhCr1n2KAY60{T@Mm>l)~#mEW|##d=`poy8#ix0HEppl_7wdP0yPKN`sm?9b(At<&;cL7SA?mQ
zl9?GrJSi|6EL*;(@7vl^?%t(xaQMPK*9k}rO29D-y^xG86upYNy7Vsd^cq_^VmCr_
z(1tB7FMn`eBMS4fckkY}sNTdz&(6-yI%7%oUyG~-Eu{6`JO0oK6bm#)l)H+)zO_&W
zKn?=a(i}3K1Ywfb{u=8b0t(*u9yAA3TQ_g-i099DqQhGre=yqq6(Fj!^t=+q8kXhR
z<;9tvw$ppW*r&hstf9z)OiCRsd2-0o3do44TLfHUjQ%3QeMujjr3ahiAnQZ1z#NFI
zTX(Ob=*PU=BoUqQ84z#G$LaYA2n1hSHPDv&arPq$FzM6TU$Axwz{$#z2Dqk%uaD|o
z-Xi;!6Whnu&dz)4EnVH>ar?r=bScHUx|k_ToDii{-k_rTa#6b#8Sj)v2{upA*C00_}SAjvz`
zTcX(Y+G-y>a_m5q3Ic2KS-I8~!H3UDV-b?VOmtxosDTcvfEo~fS@;jpf;*pf)n%v|
zU~R6TUtJBXn%3jt>6uolhYnAJsy99D1;#VD+-LEsK?~~|W)|@;z;ki`w6H69059g}
zxc))R!^P+ttjHEXQ2$WuNe|PMSb0E%C1<`}tiLL8Pyfbd43JA+)LZUU2|wG`55U
zp6a@~b<)zzxKQGa?b*YKOKI;XN-KCdN?}|~R|l$9%wUeyr>wGHR*(xy3{b@=_sgw>
z=Et7bHsIGVdZ;VS7Lqa0I9=?#y?i!~$MogD&*1`qk}mx+G(>#%)vH$%EQf!xIe8Ms
zam``BEC3+@TP?H&;DWm#&TvV=CsRP`q%(Go-}XR^V*Eq#!5wnl0ZTuHpkREzQ}i3c
z`T*$TrTC+_e5+)+kHj?e=*7#kb+JM@)d0cv1jm6VhaTd=H5
z72ept`~x(ts1`Bpbq;8<9d+jj1}
zcO=Jha(dc%N}n(^t!+qJ$!~%m9XWEp6%_w@2oYKjMXI*_h)wSGTM9)h}o&W7m=>zi`*(c(`VD2+>t_X?#kCG
znfPTTZIVmj5Z`8;j%V$?0+VjlKs9R<>k{1|v?L_6h#Lr42JS))9{>oLs0>6j+HrccZPdJE
z?0S6sC(>f!FXF~lLXAMdhDb#?8_1vUf&O}vWvIpLfY*=?dieNp^-X!l+%wBHJNO3V2H#gyKN3ty{U}m7F
zg#hr`wWR7m+d6()|DHlRmi`|I)Scx4YAANO=Vwkqt^Gu*@)|4ODMLCpicFhBXLB
R0`k<2u)YQVZ7(r_m8Xf2&VJ*
zFMD!*am3t@={&f09Kvj1Fu;?j8$-nbK+_z+>o9qoCJUDq=6ec$4-|R*^>=+%0K2ep
zafM;yL4v`qVZm^Lrd0g+(lGGpQ|s5@Y+%xY=r`{)zrE&8ZvZ5Ds5hY@AzHUf{J}g{LGi^V4l6HL
z^PeA)NK`q`4aW}(;#ak#fcdwOn|z@x
zO)nsv0GCA1wRu#Lg@R>&N%OqlrUk2K9RXHCe+sexv$K~BrK)tb+ioGRRLXf~x9bjk
zers1L%t{0h32gd@H*$fVyJVZ&flm^p;n(OPLIKGR_x9cmof-~p_XD$;+Xv7aR|{+l
zwDO;$CF%mk3fgEmi8<9=brb-ggO73{*FpieOqaTVIJ}ei(Awd}rMX44S-QDxxEBp!@|VC9RE@v5H|)
z`!gA4_N1YqfuH8r{P(ePZfh;WL<;s-|%*;Aqr&uS5b5VJF6WV+io7ly}H0#Net&o;^AyT!T{5?7OS2`RY
zZ&W(BEDRosn3pz+%dzI9OF*O=FvQShh#-%}FxBOlTzJtT<5tf@qkE`5Qj5i9n|?zd
zlSL2;Vr03e0!Z&m6pYxbSBcpSS1|3p%)ZW6&DnQCzRrCwc*#abCS>Z%4J%tLrZ@98
zPA)E4TNB~>-$TFrhrzhWZeAL3A)z#3tL2~X^s7}2fZ6$4lUEPzqo1GWo^uSFoj*0>
z>xY2W1=v2fVojuKu|0|Gg|0%1Hj)(Bp@BI$GJq=G;u2H!uU}VwX}H0LcmRn1@slU-
z?MqfAzAUP4UzdA^%stQvwZ&-;0Na(g8gj98baaS1@GZ{`B@@9GBI6MMhvghqi`PI2
z*d|0B1oY}39i7KM@2@{Q1r2%6b3>~Vua)}oQbBM>l4T7V;Tb@zuEHA1LxQ(4NrfMF
zHjLDe^mNCcFCwqr*iE`6kV?mn7+ohdrN9gToBq1B-m;wxD3qqoiPEn)AJw
zqZ_RWi-@qpUL{o-S|TEHFX?7nyYhiGX=g{$lRK~WZ^ZBcS=68z%i2~mJbd_280rHO
zy??!&t`Y_sYRJqf0n1CBd3N+fk*7jP59=FL@GD0;Z&C!-@QS6y?*Vr8aqEYBpDU^U
znbdl0RbEz;y?rw)!~K(4<);#ITy1i`?;$g6HTL{n`u+P`yNAvl9S2ED;%yt#nlG4m
zhuZxQqu)dZ8Svrs^dh~epO>G>3@BOfGcsa>Jiy-z!K;0^
z2-U4seaUNE!2H5cZ!=45Bs8Rlw0ipbRgi>8wgt72*aY`>UDEtm=GQas(~kDYuySPu
zS=@l%xIKU_>(={vsPQ%pp=yvOUb}MY0W<#F!k6{
z9Ba20swNveYB+Max)kjAmMn9nKQM+m%lvjO{rS1i_eZ@1i8s05s(=W8StEqbW~4QZ
zNB-EVV~q(^YHH=)OImk_%H*!!2>5pR9H;=+CZ;4MH+O%@2aCm-eXi%HhmArOAh+-d
z?n6q=$54jj(;;Ky*+BLdd9-Dpjfn}&See?ox(v9&2w-}A{WYyRay~N2xW41d*^Y+1
zM#fSbpcJwOxyom*Rw!_#o*XI)>wz2zpawkQL3&1Yes~=mR@I*dUl50mMgRlV_NuhJ
z%)o8eG7Tt(s17x@e{?)NwvZSIO2gjUip;3sCpZXxZBon83C-!Z?Dou3>V?wqO((?AW~2xQB^K3{~1yvEb8Ni
z7vcy8$U8RmW7s3&J?06aOhXodbl4_=h~)7dg6b-#)$%W93R)wgltf&L{ufsVb1*
z;Y)Ax_{@MdWP@lP8XGHAW7mSCD}|5+BJpEcSyr%KqD_Box)X){n73fhQ4jSCnIdvj
z0H%)}l1u?Tw}}l2CQ$>Ml}FS{185vdN)Tc@C`7r!0WkZbpv}GP?G+sB$f*Ncl>-Zh
zBYblc@75In!~}gnuTbEYu_R$109F|NC1MHYP;U
zrv{d;Xr`p>fHV)_#t^Kmf~|wFd7$^16X{<;Tp|FpZLzIo6%=aF4=jhb7kYoUB>WF_
zS?bfK@Vyv!#8Tu#mzlA7X6NR}k%-Rc-ZBKg!B6pcd{$mX0w%I2DFVpTklKrmR^QR%
zeFg9Y;e4yJU#?+-aX&=wCQKQ0?Z1Mc)z#WWRmWH$Hrx8~AhayZY4oK_HZZ9GJkaK=
z+S)j4Yio(@j96*Zor+{qq?q;h?W+UD3CFw=47qN1Of@d9=d;8s)Fje~$h~7E2{l0t
zCDALg0Xam?@mW|{kVSzoH8~|Dyi7t0fDU9!
zAx1``h@T3YRYQOqquy2TI5U0$OP|CHNIacvd6efo+3D_4iIA}m#Xmnzpv8m({gL@a
zfEeaAnR-b6&{SiQY#kpTUuFVL0Z6?@_Y*vjfF6KP0_wxUYUCw&)|132Wnamig@uJd
zXqO-yl`yMikH1Y(oLTuD&Z9dlP~sjS`U79E8b5>uSaZ1InAgn2QRfduY*5Y8gCL{zIAYORa+<{KWF1~}!n(g}P)G-HasamjfS}>x
zA`M5DFgd6mo~t_eiH%E64ZV0F0BBox9yhqB4gzZha-k%9k@wJ`8a^nF$*7|~tXj3o
zW59^>;=e;U~l_)G^!p8!%xH90fh`8RS1oR(;n8cC9n9v&XhzIC8-
zsxFMnbBA?5nOA65QC8;by%8HL!aPK72~U
zYu1Omr7CTNdPY)`u)&EJhK!Adj!xPkD+KA-sHv&lzBckga=Z(cOrkDWY~5`MzuS@R
zp%A=`9HpGs@czczN-Q`dVesPx0Ow}3<3>>>ImA7beP?qk6cOv7sKWtgnhwTW$(D#0P^uNCKRQwQz^^=|Sz0m6Ovz1Lb#K6IFBe&#WBg
zh?q?zb$~$XEY_-#nHgh$y%P&)eRxPn$n;6Ld)u}PU<)Zkkah#tTvk!B7OctKZ_EuW
zi#Q#SMj)IHl+N^3L4QV1sHI^BQX>nWBnY3MUC`kg_U!(Q@0~M@7hQnhke4)v{X%DT&pXc6*}>OKOlW
zMn0d~C-Y4rR`&`#74%JVOhHxEvZPy=gbQXy%^RM>DS^-lKR=(I4>wSH5yvi^J9CMG
zn1Dlt$C?tmoXjskNpOdmL|_RHY-rMBiUb9z>7T#Mao>AHiYL4f
zLaiaq%DVO`bB7JPLW0z-nnU@!-!nEkI*C*UNlIamakQZd2%Dsi|xp
zLRTT`1ylS2CYwYa-u!q5Un>MVE`Rt6%pDRIfIZcGJ3@Fl9XfweCULMh82#8
znI!YqUrKbIFoG+uD&&icoP2Ox(7R@1HD2xLo=(Qx}PRFvg
z(Ouuw6xAQ8nhQW=B-x1QkIU0OE~p|vsF&Jpv%cpGu(Adsn$eGuflEZ9d6i*I{8Bya
z|Cd9y@2sEx0th5d3Lq=#ijcW5N!Ez35c{#KYcA~fxvrlak=g`fP_I~l_`zxlL8Gve
zD_4)X2;r)5bj){5RF@!BZW&v9O_17Uq417=a
z(|x_Gvi7g8&@Ge>&z0+bv;4wmwgsCVc+B?8>+59sl+As;U4HaPf@M>Bvj4>oA1wAJ
zs|sQcssgE2m~f_w3$rqi)C^(rFp2z+$(AGLNSZ4wJtc;(B(8(>1VBtt+4fRd-bkDy
z;2qYPTUfAz7bQLVW&q%N5~6I%lP8>;4nJ0h_h5d=0O>{d`RNnnFf3srfXlmL1juPr
z)q%b(5Vn!)7C?j^9STuJ(}HJ7C^EJ{U(oMl;mGpQZiVR&c%(2Zh$O;z@AD%i0=-sM
ziPN-JyXdU^eqHza+I5LvNmN+4zj%mUK;Vz&!i&Z^TpX-aPMiyFR<;C}$;Fa8amBCC
z9Th40{GDY^YjoWUU=DWuFN2tuYEv{qaQHZE*FuhI!LE{KoMnt^n>dtnH*FKzHR4pU
z2?>7>l0vw3{S^}QZ{o(~(`h}?SR&ePJ2TD$)(L05cFoCDrP(qNKSRDoXkNtLL)IOK
z+freKkozaGE$D$OJx7X>IM3=v-^n{Yb{NJ7QH_Y*r!bU+^B@?`PvBW_1nUsKBGE~l
z8~PdKLWNx&VQ^cKcrBQyTEqZ8zT)|YGi4|O$}{IChg;T-_m%QvNLx3TqtMMl3`9*_
zZ>O{2KUn|^O5p9={N#u!jD-!GHr;#pP`4zy8(ESx5oM6Zmsh7AKz$>^K0I1hA)(7q
z&r2s&Wjy^6snziE+KPUkhoj2y8&*(2rNW6O0JufM!S^O+A?M5njQ5_iFqm6@Io0v)
zsRUR4_mfY>?C<35`kgg?vU2iG-1xO!XGFWa--uPpir1Fe8!*_6=X1Io*!XQ`X-T$M
zDCFjzXQzIhQ(dd_sn7HKT;j}7;>5X+%};T#XtX0Ir|8k#*2p(|DxK`@r{Bj<0`wrC
zg`M;T+XCE-B>(*vzA4rC=O0*Vtls>*E@kv0xAbcu4M@?BU^LM}YHQ(8O+~DyM`=4V
z@j(x57mIudvYsvbZjD##5xv;z+<66&Qd>vIblpbRz8CdySjoXyP|Wqn>VWsco8r_v
zhyK*k(XmE@Q=5JBXDc{hB(03FD9%O&!ow@$z0JYNNzO|YgQwv=pytS#n3_iFr5bAk
z>7dv<;uJDSP@Z)Kby}IJEPkO2
zrU-;k5kYDbeYVJx8d5h)#VmBDq%L*T*paDf*;Z;|h@PxIxCe1za6_Wi*Z
z*>DeDeevLyGI@z2U#44`zV~Kuaq+pB>dsfMEPTUd7-39Ij2C>Ljymtf
z4u-qYVpGY>0YASoQxKd*yzSTR?^omEQfFtPdF4q{!WWq2hPBUfbIHLjnDh=#w;x~o
z4oF3La=hhJRKCNLw~o#~AJH-MeKv`X;@Y&!*?D};<&G@#nSy1<>Bn=E10kRC9$!<-
z%-i%3XTlztD^ypWDHjRq~OMTGMzA^CGA
z2SIoHj3$FNmQ;AMBQ7b)@DiWG@pliH1$TI*V7fSEEG=mEk-@p8*NS=Ga43Tl4@kHR
zCqVLOA5aWptJ}{`3Ilwkfnp>Vax!Kd+y_ue2-AIshIq8P?N-b#vu)(~oWLr`gCkq+%q?fN6}yqEXSUQ6X=3iz|Kpyu}UT8w_dRsk4SN
z4~(vNKB7ijg3eZl=*Lavb6c=KxTK$Lf7#iYfLi&pibV}Quo<0RXFu-v`o@EFK%@Zj
zftP=hGn0|fG(3QA$ze)dlTB4P(O}U!%$_>m9Mn%?6kWL{RB9)BCeDhbVi=&RW=tzB
zfs(=7P9~%?R+kv4NXp$cesmyJ^iU(ps=K$hgo?@?EdIqYZPQ{6$?Kw5>&xbR_wRq}
zD4&S%$41qDsv75B)V9TOzG94*(o9E+#tI$%pWyyH+uVdjPcetIL0
z4_Z~v0}otu*`hBRD}19Nw%-2c#G%y8OntEFr=t>?Uz+!h=X8%i^op#KrAC357>x_
zS?$vO~35?Cwg~6^GMj~y%{aD_T4-AW)8PN(Eh6l39pdKDW89V91*yigtYV}
z{CyA))A$oGa~!f11tmxa`w9sRm|Hh7xB*j^fK{Jywh?|T#+CD9W7V>ERCA-E=|(Fs
zD=H#Dp<=AtxG@s4F9H@-8HX|{`A^0V)J|ExI2#LKv{z!H>-zN{j*4fxwuk@8u&<9J
zvKsItip{C@}c+8_{_Jv)^Z_XT-Qdyo+_Orkm$5mnob
zkJ45Wj0c_=FCPRzmM=9p{Mr#73b-Ro6DSwHQ$C1;Lb$v=a#q2m
z`x+MC!1(x|uEAx6cV}UmVhwHj((F8*e}2{6AsFHAo}Pp7J&wU~xM;&tIdc=l!+YX`
z&8sUS2y7i4Y1qgZoT7|HP~FtD8VMt$Z;8kat?w&By5KR-rd+3MknM5*vBC_(IJNhj|8Z3
zMl>YK<4#Wg@+*_!UIMucqHZC>+WOFF9Y8+U$xsvyfk^F29I-q*GagovEiz*B?^z+VVu0rYilF1Q4^yt?&WcESS
zgObtIT9ScGhdLe?0Q}fj=9f9$9)#*!kk|0dZa*!aLjt)W5L-?jA~mW`dP1k+t1=W+
z4~jGIHJcccF;>SA&fnvD7fGdK-A_)SX^`g)LCHNNxfTd-#F61`Ya9qlW|*N(4TW84IxAYr>FfVVi>Y=-fT2^i`CMD^(P
z8;@iQfEm@awr=_O*vOKn^z2kH#`PzT_2Zz?D-cfOrG_}v9(QaSqjQ=;;M2t9B#+}k
z<2w;~*5<&Y1M886F5ZeM-6xn-g>qfMIn|o^^
zN!DYTPW}G<03sP_P&nfg8dZ2KoIQC+=3_b(A#Kgn)LobOYVg63Dg1-(2&z8QO*;Uq7!}6>BPS{xykGawW0P2%
zPbP^06i1|)9HAXtlQxfstWGFnHKz@V6{84PYE*!%VoaR6}3($laLmVX>
zDQ2-jYUM02&rNI{?4YK1`5(#yl98&EKiMuNY5Xt{lY;_K3uVUm+C>RtKo@dd#*a2e~-}?t?#Hi^x~{LIc2-FxBx7Xn(8^5$ky9(s0TV2udFo%4htswVxVsXVyln{
z&XR$}>eN=J(MZva6(ueV&g5h0v`~X@?l>)p?OBKHJL=}ezgPJXTEZK0od`cd&1fv}
zc;UAY4=ZvD-VFg%lV_&{!E3L{SXmm!UHpl~Regbt=qMoR+?y?QBiYw8Hcm0~ZV81-
z2L)eBawmNah9G{@*BUXaCoVnQcqN1&aq1s}2-p}tI*`UAXB#?vMv|e@5j(dI@B^6{
z8(gD+^4TkBH?2TFe-;;Y0zsEJ#mG@W$YY>HAvnwQMUNN$Y8NU$hI-wG4HvLI3m{D5
zDEc}{-I|)}YUFgOA-zQ+_6L`N91DIa=m>x|-1AHZv}Zl2x3TZ)qcf1rQX}+L0S!4j
zX0L5Lh|ms~M!mS8pdi3Z4g7wSf@7C3+YDB+TegsgG2l>70_Y>h+O>utbSLn56=-?v
zRP1_1`&rp1u|kJVQ2V2UkSV}oA)it%eEKf?v9R?-O!|WWk~U+((Ik?HK|qn0t1x{oOk|%m7q4tWgekQQvgf
zYn0f~p>?F20!1JIDj+DRr>o0~F~k!VM%%L*Am&$JdLqoQ{0Q_Pr{RJTO($I%3h{~V
zCy$QvuF)&Hl1zR8yh&~l1=hoF8T^f;Ixum_gS|#0`0;DJD=f@n^AHdV8A9|Q#17ou
z+%_27`RqJB8j46ayhz))r{6ibxpBxR+=O2txdoRD0|dJ5z~|4}K0YN3dlXS-$X3Sc
zhL#ycCif!t)ydKKo5c4eMFVQW>9@yT+lsdPwnjrVRu&YPbRyuT>c}ZUgfusr=G&Jg
ztCY*)xcM<$B96fYVzeUz?GNIa9k9Q^x0lji2jfJ|?noHz)G
z0#w0W!KPP~;slux(A5D#6^P@|Xc?Z5Ku!dpt5m&ta|snRGA;5)5coeHIG9&ZuueoI
zh?!pzdQpO=igSw7`;2IQr5My|4;*}PkF!6BC$ezP$2#K#=rt5oqk>}wY&7crKWdijsgvLBiNI+*x$JXWIdmI4#Rs$?gRVgWQb
ztgO8N03s!zF3V8Z<{vQ9-?`!sO{N9pn_b0<&8L~D9z{%kWUsHiM1
z&O_hHNfHu184o-~2H}sp(|)yeb@DJ%K+Br|0Ekk8t(UsR_ygb?v7oMs8>Av#Bb`8yH
za9}{Bsm1jjK4WJSpXlK@ArJbX|4cDj|yAyYP0OS%&S&W+O0aGBFVyJ;w
z6P0i%Mi!n!-TVFfHvut#y8F`&1&vlpqJzhfZi}Zd=p0!OlEaqZOhUd0a)Yt%j*fa0
zillPXR6p$%j2#~6@DiL18w6Kc1>RPKJqUY^yMmQQqz=XCA}|;N6Iwvaq>v+J@X!3U
z0(s2+>acS6@F3u8&qoyD1w!?q1524d-Rq^+OixHS5)j2WcS%lu!+gehOtysaC8{F_
z?I5;6V3x;aq79XGisJwTRwkR2YIghnEHeeR3)HW$m7sm#xjcKEoH7`6x!?bnCW)~j
zkmQvdh&Y}AO3p4_2jObPsWm*QPFc0@^$|tPHsp}(DbiCOF$n2kUXR;aiifT{2ZVm@eLau?yuX3
zO$*p_;Q1LB_f}4IM0mGt-;U7gYoH$;^?Y*30|z}Lr+x(sZ>S+?cVPO{_xCy*59GAWcnP%f@EAjvWFi8Y23gEc=%uGm2s)vDxL1p7b`V9|SbOc8OHTjFg
zuZDpAnZjU3DMaSrdQs7~{=vyBz$8?;D6N{kwBwj0Vyg`JC41nGocHp!yIAiS3FM3Lhtt9J;zp(JW}v2seP{
zL!PFKvurnT3=Wscn*X}D_YPnLLL`o9n|0!E-6E)n0f9->ZSd=;{-IdJ<{{P?`uNN%
zad;up45V@n@P%ktP#lnDitOKLiY*TE;S9^~m>by!rqc;G97
z0jY1-^v8f+h6vqLZUc~q5?#Y|%G$O9#AOM&?
z4s)B9cs;^g4kT|Wc@{5u=pv?5^0XBseRQTUWEL-cdHkaj1>Q8zw#M-?{9SjF==}Cg
z4%iF|9U${=sWmrtV+?EYt7Y2{tA*?p$Abam&@|TE>IC~bfI==Oqak8vZT%8%ThQf#
zK_*jZ`^rcKzDBM?Tb4#+{|=I|P*J&wiVIFkWVU^C^YhfO`X0AMb|Q3zGd3bn@mU+L
ze96Ouhtb4;zUOWE;;|id3X9+paA=hEm%F%*PYC2-P;ue}&*pUX7l94X1g24mVPVv9
zCLI#7!fkqtBd4iyv4S+;FRYWmNxl#e4I(Ah{9z9kW_T@ocG8-t3_xd0f8K6KdAoTA
zK0e}fEWXYwk8q>(f&t5I-3q*7M5D@rvPS5hv2mM{xd|={1)8Jc^_bdMC#S7WZ5M!A
zJ%-54_)xTqYZu)TOH!UNm@Y_8iVNSjXQ$78*b*aCRX
zUM!X$e&YxQtYl%4P*zR^iNXLDr=y`wJc%fsHL}W%q8Ih4lh*A+sL=qZgj}D56
zWMza@GD-_s*)x?$h$17qG(<+jsI;Ef&G|jgKfmWVM}5Da&wE_=bzS#$J3zRzeEi;<
z@d_ju6pX%MR2OLg%vvy3nF{wX?_YQ~PF|syVL|QeRcbywY$u2civT74sCd7Lkipz~
z1;EdG!&aN!JwL~Hw3$kEO?7Qn`H7fWnt?V=x@!baH0rg($$Qw*<}MGk997K~^k0QK
zE56L~H?@e4n>4Yjx<|L*@81G^Ek{ob$k4K!aLGnX*Y1?%#d*(;*~X}FuRK^5vE=&)
zt0QAW_Czn*f=*~#xNLb+nI|ziFWuH1r2)m_9r^NQEG^4;`63%ww)Wk$$tKdQsj#6>j`~Y=dXb;lu0=T_5k@u_Hl*&`YkPO?O5|EFu6c7UdBf)BTgO|P
z)UnNYbl7p#;ES&Mx^)8A-5+k6IJ}{&#XpXUdh6A^YmIiBUx_@Ow&BOWHZJ;4qiv4c
zo!mL{#qunX*Be+>T3dS
zbUOCy-`}>1=aw)$xqnwfp)NpAc>XX>M3yh>LN!Yzs8drrHx58MhF{AaSvgOC@ywaS
zInY)6_J-Q*Qo~OUOHSx~>7P6^m3T~Jw0uwt=W_5kJuL6JpN2A84Bvw}{+vPkS>4VH
z7d`}_9-EPX@Vzk!rU}s1SV|peOW6I%_9|=XF%dsI*VX-AT6;N#xe^*SXpsFodW_N9
zk~#`4moK-EI^v|*U!hRJ)b`k$xrr^an=Pq(CF*qA@{sMCAx)g?)>SuHJnqTthi(~B
zTO)=nrpOZ246?y*QEJ;z<(S51&)VJlo|~#?BSs5
z5#RY<`BpzaR-F$&nZlcfvTTZJ&TBj_<+Yt&s5Io>zlGVFA_JUgib~Nc0h56R@6dEt
z!fxN@ZThSum}d*O^TgF>PoZj@$P@9PrE!=i4
zv%eG=dRuXQqE`04Yl911^)uG}*4FO7_^mo+SoqW=q+V9we@H*7h6ExpIOXDz~qWP@CXf95H
z9XXAIX=6a}N(NW+sFgVNxOp7_}CrgGMu&~l)F`6Ey`XK8DN9*t~MaN?w9
z70s;zv?$c`Kkakkv|*hNT2rVRoEFwiOKaoKdGmf)j!$XSeTKs4in>c$7YCXbPo9{&
zX4PPot-drkOA6J@*|U%A>pbqRAx@k)G@D?(8EbfZx+lX>v1ajSm%@i(wjJz%G>A2q6rJ~OvqPngyRRI*6c1zgIJ#GO{P
z{TtR7DQ^1c!GiF1t=3O%ZKwG@Ne#?`
zMJbxNf|HInexEPcixN8Z{3Z~#7+q?~F_2K=4ntC*^
zs&mrXuE5$}{lo0pP1+gm;jGw_T{R}D122i>r1H;*s
z5oauD`_xjCPzy`T9Fzt*AE?Eb5e3Gwy9F{(5Y~vH2GJZvg2AXZofj-P7#px;9hQ})
z<;An4koj=ot`8;~b%R9@PEEVacWEV6Y9>{k%%Ldd$sv4n-m@e;8?B}u7+e;`*+r+%
z?J34iq(3uJfsqX6sP67x+hpBXokCZySMv@Ls(ps`+VTF_^Qd{pO`0842(pMiXfgii
z*^su8cXmcY)^9W(2+8B;D?w`bn0H~>#4gW~+7JiuQ%Yn`A8XcJ-+|Na8bm}!Hf+>L
zysz>tq5k;)Msx0>SO2xzm>~Az9I^g1>eqY=4=rF^7KPr=5<-imYDjDw*uW{Ypi@ydP63MrR-E5<=H0p>7MXC}O)m?XX>N8Cjh&6bjcU6l@i-1PY7CgYrN_nvir~|0Cpx=&Tj1bw3lT-WQ
zE5j!xD;4E?X;K3vOAsY=m}ckB>-bKnCosfB=j;F`ICJDd9dXL*2JZFl)v!M0tTB8!
zd3uyAoplE_E^7$yoXgH;+nNba>EI5r$Z2UtA0T%pCv>At+}%Rz!@)ye-?O`X@1CX+
zv}WICaooz5!{Wsea>=EkMyzAs(b~6KJZkYQwAbeKTMG+E&2{Uu>dhd?s>z+N+a|0o
z+sPV%w#HG@8Xggty_aJ7N&K4CXg`~97wzli*!=?dnrQHBdO2}U`>_PO1gD~*Hp2!c
zwmV*F{#?CXi?!Zkh{4mNMBRY+Im&hRp0V1IRNN#%?cy74Ti3v6*L-v=!H$sFsJD(t
zA~$8`NL5u8Y{TylFTNZco78Xi$SfA2+S_;Tzc<}XP2F~A$cjg1Ry%fl`t(%!S@m0$
z^j3XStgDF=vouqK);s5mE*gB3Au@c
zUE>Q}Sy|MP2mqR_eEF2Z+>mZLsY}E9kJUq`6PPJo{YKt34o%Lpb?mr%{j=6q?qD}k
z^53l`DAtOS!w?TJy3#F!%c(J%l$D3rv`=WWr?2PK({@S8r%#I?brLE
zs5+>HCnA^AgPZQPrajvMw>Rl%8@cE|yfXH0Z)n1ez4v(|6`V7mH=P!3EoR!Q?=T(E
zBTyr|M?ha|?OVw&Luhb0;bO?M$Nv4Nt@p(X8P)>qOaZb>3Me;X1ntyLHUmaXsDpi!
zW9Jra!f{uy7*D?#GC?Hv
zL(4H+5S6M*c}1NEf`0I0>imuwNLByI{^Gj}r$k|>ymv1LTQzzKuyf7cs#m<$t}
z%Fa0bt&DpEc65Z!XP?S1FK@j~9NsMIh|L8BtsBSpy3Dzwa`oUQ3yUj*4K)Jpl}0sz
z_ejQHf*_D_)qPH=k!KP|W6bpHm#&nX=JYt$ImXtK>rrgN-Nqdxd~r7-kVMf_(jI=g
zrk%RF{?eRLcpqrLR}0*2!BYi1>%)dl7uC)oGCX_)*Bcigs`ah}aRNgo>SM5Fg$K{C
z+_7`#2VCCLAZA$ejsqb#xWkYayFW!1_K6&Jb-!=Hhbj#zPhN9yzc@dqIa_Y@^yx$G
z%GN-YiG07`J}q>W9F)eZR5KmSe`(g1mmM8e>s2)^HrOlotVedY!-EcoZmeFjP3_Sp
z|MS7~6`!sDG<-qtXJu~deCt6kNBmfG&qW`P^>D6Y*b5ho28L@xb*kU|zW(|rck0=T
z7n2hc6=0{(7pjwAyub{r&${Ns_Gd~4r2A2H${9b5XebfI7t|3M8!P_`h4D_
zvk#mM(sNE$LP{1s;@Pw02M@NxT5+cTewMq_-@kyKj|jG{UxHQGijnCrdY^y$q;}zT
zZ{y&XZCvh-Z4$3FXwW{Y9qTCb7EPKqrFO2It@g2~SHV$9=4M;l#HcoJ?(gM3eJ>O%
zcfjhFCcQ{M
zrA%A+ou^L)7R>ov_sQ$m2UwKuK+&EF16lJFFc2pPb3gAc!rq%TaA6nC0h2yu^{jq<
zDxs>=>$|o2KTBhcp4TnU?Nk(esQTV|jdf)Pj&^ESnpk!)2%8l#DyhQ$;*=XpBW=oZ
z!P4Tp#}WxinI7i7t9INaQIKiiEqUqAGb*4^#q#)NTi%bbB1uAn1#9TFAo};^lKsoB
z8ch__JiH#GYDx}_QM(8_6n^r<#`5Qt_w!4=tuEDm&h3$VamMR!Vd?q*Hi^&Ye8$P&
zg0K5BH!MEuXnys(WwWblamr#_WdK1EfbGO0EzMucbvwW8@Fbg{Yc=X3^WhdCLsZ9Bmg|rqmTRek@EPNu7{Jtzmy?
zy50B3_=!d%18H{eZUF}PK!pYP?1&D@qBJDkoSi%)wD$5MF^EA2x4M^mwBg)4)L_Z%D&k=)i)iFoqSjIt9h
z{WHf1FNh(wKNB~%k8{$A7bOATg6G%oA`b1Nk&z=QozV{9B-i{3mH+4SXK44xDJyhy
z*R)hop)ad?w!^vKO9KPHeKhQ_Ve#YV&$SH=D-Oig&hbpRe2|93r^hFW9oh86h!H|_
zi3^3sts3{&Cs&NFFMZHJlMi?koLo1KT}32=wzjsYiWJ;isbLP=wk>PPhffRM(I-MH
zZkJ5{!JL7>qMO;xd4qpI|Dg;=)Hxq9XRKRPSo6a72_IfRdl+8r@+RqMZFSem%7Q1h
zngi7QUk}%vRQT#TMH5tOtNqfIQ$dptb@yGk9E-7INA(W&Uf%jQR^npn0%({NHVxs2
zodC(S>1`*R#%0w3hl+D%b_Od?-->#R@0c0UK7^$=2T801rOSi6#%}`tWwy%x?blu`_YvNUm-Sa
z6d92g)}_x5NqZoj9r$e&%fF`&OYry#yU@zuONeo8?V@K73kTmO?!9)wfOtgpOSyYr
zw(%Ns=X=(wHM3o!4`NxQo5X4H;@GE4pI)sFfzs@UfAqwKHK;Wr$@6qajUBuF%I>>S
zEK^CXOqsD}jhREun8~RxkB;vo4X(Tuj$Y;vbjg|_
zBMmfQrFn4Omk-04{ns#@yknI1%NH+p?b=oROfS3M2xZ^eZg+gumT+>&b|W*ky+zXb
zA=+;MMz{v41!DH*I`r{2zP5I4d&}|Zfme3cTd_93Wc#+cogNkSNN-*_M5pH8Y2`(I
zvuZCU#NJrj<;;g+JG&b6u9>mE#NNKWk=K;7nv=#6oGrdFOd$E6f~~Tu>sJ)D&f?nk
z?t3%yXzC0p+NsmLOMa8fbp|>wUFDqjw4C`EDDl4N{0}nVGhF-l)q!v{IqGbOzKXBC
zcT{vVT$X3o&Fgope=Gv$VDh&F*^iv<#FGVMoH$PbC@@C9pm<+m^~aC*#~Hi~w}I)h0uQX5Q!B}Hqx33#M7UmLV;PGJ=e3D^)?
zNOZ;&?8faSMi)Zi0RLjcCPV~EAa(M=b1EAa>wtVNmZwa20d7RQg7q_zmHomaNv*?(8L=SE|{XKmf~jpQ#W_O
zgh^IAOo5{J7hIJR0k}+is&TruM%C$IcV8Yfv;*mM1TG3$v+
zvzfDIjd*))d9tr_$dC88>RSf)eh3<9Gx*K}}@QGXg6Hb3o*63F8Fxv^OJ48tA7R`;{Yo_
zzNDvA`no`$vvm~HWm?}XYDr+q*oWs(8H<>`dk^ykpxqBla4C7&-t@E
zK=6v<5kCTy>rE>A@o6Hfd+wvdis(8tKs(dk9~3k>bnj8YS?c}G`+ho9oF#cdBs)PO
zDF5%1$m9(ko^*-K(|x4do-Awvo=)$LrtsR{!CUGL~d&eOI
z)9-5pT5_2SVM+O&gVPxWCWL-|Dg)blFHr3RJ1#y|nBTJWGSIdk3zD|^L2G{ZM|W_B
zVu4To;Q5kO4t%3aez>izHtDC2Xz#!Yq6fq6gE=!v%=r?hCzC?7?ze2+I_QLRIX2|E
zPvVCjin2-D_C&+(hSIB;6D`?5X^mSSyB$0nynwa18Q}asqURV`g@t%4Gc)*ygZ=Yy
z%D#H0qes(4I1Wl;?%Qhv=ztTKKcqzlTxMW-yE>2X8!;l`Wt`{vV;9C2GSvjyDHo_N
zrDFpS_Y0VAL$Uw}jhzspkKNU2=FD9uDi02H*FM#&*IAp1x9kqKzM5nB$ZYzdm0DxY
z&3ikT#y;EQUa;D96N!l@Hz&sgevNG$K|=@&Dp;Hlma4uE2-&x(18+ikV?gf7I205D
zEHwkEOE<%a-PgT~i(`u(Wu@0tZxZQ0-Cc|aVm;?bex
zb9;TMmvj@b=A{{qQ(W@A#%i~0)uVx-CJeKf;}$Fs3p-t|@8N^#-q}il)X%dt&nRSA
zmmaKr-?7@cm$TNkY_-236f_nW)9sDb>WwIWReh|%h?`59uRiYJp!Ep}36_C9^!6+%
zzp^6xcCyj)|IY=m2sk$NMC%@J-(2Yua2D42eva+Ez7~mIGh;@X9BtNN$&!CZO`O;-
zYi|Pl3awRskW(c?ay+TIkSJ=`!M`VDKn0
zGv5Bi#smQiklTb_KxP#Qt!*>6Q?X+Zu0Eul7s!Fi}Azf+!&T7
z>Xs{~9;D`eDp3iig;+;skH*^0QI~OP;WRo!doAj&^j_w>OSvh{htPYdFtT9A-&2hf9jQ$XLmSiK?<<1R!!0*x!1edcB
z^OYP!9ZbqgJ{T&=kg6
zS&8D$GG)4*=8PQ&)EWgNwGkI0N)gZCKQUHPWoS5s3>7~qb+I+wMy
zSznwtX=z54g!4!q2nU2Y(8l!7SBtmqBfKae^$9KPfjz^HpT6Y~>p1kyo3~e)3X<^k
zZR)zzV4Yrp`o_I>%o<){;x$gIPQtC)vnvZ`4fs&oDJATyhN;cg*5M20T6e8XoAQq{
zhLX)ww=ZnH{pS$p)F%DfIQMb3(l-h;aIH};E?X@*3&ob@m2N*Tl=p$YK
zB(;mtjyWb0I~!xm@xG4ipTP%dWuACr)pB-;V@^t{-yXpmm_^+&iE!K;ZuW*6T>+lW
zn*RvvuiJ|=8n7~0vFCk#csYg6YE|0&q;U|tZ=TEj&EP`FqXn-oS(#i>HZgCI*@Wet
zgWoz286bJpkZtZ|qv%J$P)dzV%+%hZ%L``hF&Qy&=uj{BWgivYMjp*+-?_<@6Okjg
zobDZ>+saCPpo^n_$K+rCj9*Fv{jp~KzZ1O2})|t&r>+V9XfjMZQ`17SJN?lQOq+G;|ra(qy
zZg%!?%m|-QoQb*s%m=dC%oS%8mpwpwrUk9xd!?s@%c8cXF*Q!^NGj-*eRNqa);;LC@`v>o@CT)NQl&wQK+0z4v~4qUnaTFwL0;
zXGYouAG&r;v*1IH{n2KfE-oI=`p+sn`78O5O-vp`80DZWY+VH7#=HYT-A&c|&!ou|
zN43!;uCvjSv4T^~GI|fR;rE;MJM9Vzx4PW0SIwS72YSH2**Tx_Wv}fOq
zL!~40&ob?R$)97B-B_1UOEiQwDgGUt(`I;=JgcsJq2A_^Znw>5x~+Dsdo|o5WK;K!
zgQ|*B6zkRkL**dTu@GJx`p1p8^Jg|R=H3ZYeAe7_iS2qZ_afH}zap&TI}1ORj9)Qu)9>Gd>`s_1
zG%)z>v5F!^x8=s6GH?D_z(j2NVnf7tOt2>2;Ej@tOIO(^Ju-@N7uJ(F%urQQ+^?lDINBzC@owvmqY37*^+6~llqWC
z8*Q`IB%FZWZN#9eHd2Dv9J)yM9Q`pk2AHa>F?48$jV2>FQX~(B&qhPX&jKzjvLU-I
zTAPRctqF!3sw)m=A3h5&GZOk!+Ne8RA?2t$350Jt_qZcfKkBv^Iv@jf`{YGJ2T{X1
z1nIUkY0&V!Rwk>1?v|tcaq`+}V*+)5KG^nnPj9`$FJHfYLc;XV%*Ai7nR3=)b!Xt4
z%7I;pCFLF}0ej5Xlo!Wj_I>tbq+0h=C&s+Xztr|{w)M74jm)ieJ8jg7-gD6VdZ&WE
zPcu`lnJD$_F)8WzGS~g~Qy(9-Z1r>BtgA0P_j7Qw_zEX~Bl%joUJzMuptD)~TkGk}
zcCslpZpxh|eH3ghM91;mAJV$U&E;{Taz0<2XUHqCVhN
z?Z;?NSN=nYXHTZUGH-E0?TqjVs^9!~$Jg|mKOO11AoP~!dSkso!%|_#L$AafwuzY#
zV@r-m>v@$(na`H@C)|!0A+fn$`**H%E&*CONF?%z%hj^OC6ICD?SzcT;qHW#GaxaM
z3qi|Qt&cCUEYj5{9Z^axxboGs#t>lx;)32eGHO9AA`>6!(f(Y#z&n__gxIau#K6CP
z`#{TNKFw2SqbGkmwSW7(Wtny8(1Bkcy=bXd+yJY%DFim+BExsm0tYm{ts1=U+C{^D{Q@{#)KED)c6p@LOzBZ_Ke)prQv%~>ZlmQdFVE~yrlIJ6HVzugDmsDzI;^Ed(y}weZ6>u
zEN`Ei;i*x@*}Say_WS@+U2#}E4X@<85XTHdCKJ*myPX3n{m{m^KB9X_>A${ieQ~!#
zoJMg*CU{t6aZY)IgEE+NNGe&PuPwo2
z1Dq1Lp>Kn)rV*cq
zHfV+c>zLTgp7kfhT|ki%jK2IMaiuaUtPEo&=z
zD)N-xc@~IIysL%-xYc2A6PPxDUq7eYTU0K-CC798sWm-rZ|knw2rT`qWelkhyGR)a
zE6N4*^;IjN2o6fe)O2so9Rkwcvjw|xzC#>l=jH|!+05KOlWO|GLBjyJZ<
z%X&RI{d$nJ&(BO$@VvFvPWo4|@4cj19JW4?(;E?$Ep4vxcL_@VDnDRSj`mMLqR6A?
z1|IV`O_z^+Fz0KKblWYX@g#O+NFufaTA$m&@Wd{coF-{|W&0A0Ui@x#(PKg};HwXp
zqzsR?5zCKcY0wQ&{^fkS<&W|rHJ$0#`gigwR>aOs_uKW`X-;~6=@sCwAuK^njW~r@
z(A?51;_t6_PGh7Xeuuc$RKe0m0N@P1vhebS3!|cKcJJLA)vDLlF5X;cw0qD}U`U8N
zB94Iq9Emj5OYZ=+BbP&&*(G^yWRc1cZGam;Pr{JqYd0eq(uuUZDnfs^cO1H(qSUR*
z7{qgkM?}}HZolYlcF*DYIF3gHX_$`M9IlXye?3v+0k#*81MVhg#a&n<$)w#DL)%!%jl;CdG_Z
zw&B2m*Qn4k2aC|)^*j|t2`8`L$?Nd)w|in8t|nph-+{M=8mJ$C%l=|5#}H85mlEwe
zRaU@H=!tarph5IHe;<7!zk5>Bo3&1d<1?Vx|(eLKSR6X1yW^$7&FC&NXHoq>?cpsZH47q
zf;qqt!VLQR`%nI7{5l>DV5Pk*b>@GqkB|RVql4Bt3xPp85W!2H&f9{Y<24qK32o}n
zYZmErhs75I3D#|@oB!5b9NSn3V6Fj&h%!bvX;PTRL~=)%Eh#QO3~iKrsqtcj6A1-1GHMT>Ax{A+;=(TvjvalT
zoCJIETgJojqGFi=Qs@Jp1kANS7uA$-v+Rym^)Jz2D4P9
zE1lIy9&tOM+3{y7zt|z5xsANoKEw~uMx5{KEj^!ZFVAVA!-R7x*=G$I9v6r4_bhU}
z&L~A$Z`_~4xFLA;`tK%_wZ;t^Np-2-A_)PIJJip&*vTZuR%SyAT?*L%7JoCb-@fbD
zuLmDHyF)4}3)4CKU$T?AQWC*@AAEhbM;
z3W?^F>w|@M7%4#-$Qo(Lr+Q}`ZF5hu5O8)&RFc`oB_SHsKW$=crSC${+p6-Va^8ZK
zDRr@=^<&-}u*WVU_Xu?d=!}F!0o~W7QA~U9h#mL6Ljgv;MU$
z68nX%nFPNy7MB=M(N$BPJPF_nw1+c+?;bGQyHz74R0kJ;l6+b$(=ajlqsr}2ro>7O
zAm+?*;-5@)Bovz-Obdz%!b5D7t=*^%T~W>)uCF*Xs;vL2mTVCVOUo|W+W#=lTW8Ck
zPq`Wzzzo4}z#_wF4h7!n12%}3Z$cfOC1_YOs%dX3pf=!fFq+VW6QAfjY28ydyyecX
zreVkB?^FO#T<>nTooVoUzy&~jUSNrVDAW>J?vZT
zB_#l1wq!mE1*GY_?Q%C7|8Qn-;zcd3JOXWw7PysG0QjtBXBun7UaX`+ojo8a{M2AY
zao=~YTCj5IjKNdm)mKWNGD
zX8#(8X1@AL3qGlIP?(SQN52Ew+-Aja8aQFMW0Z{JxLvhk+5{DJr6`d`g>z{?d(1-<
z<~Bpr_*15g7!@uFpWMrq8jZ@=zK&mWUG*<28%oV8m{gjNWjKWQi7}KR=
z-l7CwARc4M|If~rH`MGs3k>{Zy>@Z_t+#Iv!PSPsbA;Ub`&m43-n_?OA)avc>;7=SI9u3QpPJhhJyO!
zSnZ=wL;mBu1w|p3qbc#=D`hGlb%1=7QeMJL7nj>qT_0bbKdPFigq591K_B2i08M{U
zL-K)(WW)e^=DxXd_Rcrw5PjEUlTNh881eMJaww%88)XoZupnE^!VurtU%y&$HeWn_
zv2rS{t^2SY!Oh@+-3OU>U$1_{?8OI3bLD<_1+Zd;WcF1%9l!2yah7hHu}#1wib2-u|`FRIDV~9Bg>v+M@9AiQ|LNBk3xDfid-Xq%hXSE6su0bY=}p4NHW+?JHFiL
z>e!r^c2|0>fGv+Iw)!{ZHm}xt=r*V09UE_+pmUF8;)^pUCF#A;=0L%6!WqJzb
z$dv!ed7mwHzL%Er>{{Av@#18zPaGICw>Oz(7-4xp)@B5ONzK#@tz%wPy5+6~9`hKYRR``PrKR{gRzesRaU(k)G?weG(83
zI^|E8zVtrC`3YZsE@ZOs=yyg1+`$zF!$)>(<(be}+qYRarAhy|d
zjG=R_?0y(QwS!@n(bo!NH|@)%BK|hV|2lbD)boh8bGbjL767esD|agzbH%&Xm|L(L
z7D&3R!>4qt%n|Jn4xuHwIX1B9&Wo*?=tA?26AO$+<{6E;G`fv(2hbzwdjNs3RnjF+
z_nduo1tqzZtmxQ<(GQW7q^pn%%Ohtsb0cstIe`_#k5;$thHf3p$mQhHC`{)M$DsTX
zLoi{y&yfk;F{J&2Kt>2^6OX5ZvI`1E6K_7zTTP~+NSXns0Zq!iGVJd)`DD^yVV8&F
z7f+>%gS@)^@S)Y`UvEM9L@}$L$gU>)={+zF%=PlBdpDRA@yc*_O92nC&{~%e1?n-b
z{xOle3CI~hH>Ny0Tz(Pp%rgL}NWaKQEj+<-*kMiPh)lsTjNJ33?pHpK@Fg+w6
zgq&D#mlu-^7ysK(?ohd>SsX&XK%KzAV^)d)>M=&pUm7)HGUMqbm0nExL^#LP*YUrR
zCr(s8%?$1}?MO|$^e}Q%Q(cnsnJ*Fz&ne6B?8-n~!1s8@t~_5=V8Tp7hvl2J0ZZjHP)CLsdwR?NlbG;iL0g09Ip^;6AP
z67ec{#@c&BisEiTHT3>`SFI&K0i(obDgn#Bqt><)v;3msQO4aI8tcGK40vuju9Zqy&S7At>T{s<7&WYxk>&CjpNJUKN0
zO0@-@jKsRUb1Qzw6bZ}=1-xcl2hy-gJd6Y
zmcm97*|x0iu{JER|4}Q=jmw9xO5Re(^Ksq+m^81P?{Th{csQTVV4jkS0`$S!!b%yf
zG@O)>iT+^r@zu2{1wB2VXLbtv+tjQinR$rS2^WthYcKf>F(#fm#5=@!2V@6
zuDx&{C99wtNe*0p{`~pL-xA2^BXG|Py3nWW^;truFw06)VA}!n_SU=a1+84d$uNs%
zHDFw*dalm9cwr$`RHRAGCBn!$TiF;GG-rnIr-b;_K1A}EtQk7q@lo2thiksM*9>Xv
zU@a^$C20=*%5X;GFns3_A3>ik<%Zc66S}Kz?=Yk_z5mSlib@&YcT&8UC#5Lq1;z6l
zb}{)5IARLRcpICHF>Acv?U|0n)w}R4tbZNVPYUf^^N!l6w7=g`fv5|1Sf&*w}DLh?hdPG}0>8Fb;bXseL{dGZU4<5Xq57Fv(T6bJ^{&55KbY%U?9@?BY8xqs7M{zWofaoMUR#Vv`2eZ_EdKHfB2CiwkoD
zp06}be}aXT#ntGUl2weYP;lILyn+G_SS_q&3o>rlnZbwDRlatSJNs
z@(KbJ9ET%OEqHq^P=@lsA?cVm6j20+=153M)scCm{pCkbOWLu0yJV7sXo)~*{$E@R
znlX@OCf5*HtM#W}G*g9fKa1a!a&`Oq^@`%q!Lc|ZT|c>T-_1keGj^|g`9@3(us79E>oy{8;+&i5wWnkHlGq*e3c#fz%B
zGuVPIF#FvD?;q_bgHHt-0$hH4&h$q|c|y}Io)|Gh^Fo3ZYJz%^Dap+t1&U(aI&m_h
z$IzwIoJ_kwpS{Crka&YT_3noY%KJMcr5_qn_8?(tPA!8DC0!@vpjCtLk|?dJrr=y>
zc(?~)!)ExE?Bl_d@V2E*ZTYrrZlD+3sG_Lx`XQ5u?C%Vcb&ZI1!riCESw_={ufe-{
zHZ~B-63h(^nody{)mh8-KO#(~;6fVN-%4&g@`_2gkSr1d2;|2I3XWh!h2c&g0&a5LO03oS~{0xY;l}`Gi1_2XD|*-KRCcXRcU38g8Qm)<
zemIS?$WH`L%VYtrWgVNobZ`!oL`t*mZAS3IqB|>Ha7q>(2}p(e(4hBWL0TOD6-
ze{9k(?tYU`ucCONRMe#m9MReI2ShBV_UVE!E3(J(4kWwCj5R^RkRdQU=VfLhO}|)-
zc=bmwbZ_MhpN&OoGj@~-i@LW;qC-;Z{uy2+8+)1_nT63!+^-y7d!4ov94C;A3opi|Hj7zadD>PN-njdWwG~<{?{0L^fQ&>C;p?p@NLTJ#
zpv^e@#(ptUj1c3O^ZE0(*w|RrsCArC`*4J@6U2RBquiA8ahPlY5M6YeoDf+j++dWP
zyPe|JL0sZ~hJ&*5)9`&XMZfRsc0%v7`ODEEF3wAU;^IJI3ie)LB80Xy*E>2orXSpO
z90)|UvYE{Z7d+H(so!qFQ@rL382gaN*OfUvrC6XJ$m_@B(R>s*-(LI-iBmfTCc=d?c`2Y|mx;^o<7GFAQ5i!93YQp(E{*AK4t^gPS^
zhzAlQ@)Rev4d+yN00|*VZOfzkP|WP~Vg@ipN5@=4co1>?2pvbgt@L;%cS~1YT|ek)
z^S-wymim-Y9Y(d00la&I{zyb6g>;p}ef9>JKlxY3f%=uy{F
zJ=YQ;B62?;E~w?vHd0f_6uX`=KUE~a9o{))@bcqhZy!;jAQ{bR&b1JzI?@P*(-%3;
zor)hb5I8qV>yI=VrygiYe&;c>@lLHebxNm>+8h?vi6L$8DI#6D=IA-zrjM3Wco7*I
z?{rfXu)v0|-ksaNy@{a8MAoFM#VHfXPT;Pbi71KqY&qh>sD!oE1R(nMJgXTR!w$?)
zaa4e7{p%A*J|Sg3jugb!Nx8q!B7*qYR1f1(n`j5oz_$cAjT8H=lv^g`=KyZr&sm}Glw
z{8RVq=^-Ov5=GSNuj|FMpb$Bs$jFbc`8}WAYKpz{QKt9LiW(jklye@RBe>#_n|gEr1NR*fVaa|BMzbKbc6XQ43Yd5g-Yoxya0t2_kzjC?a6
z)gm6!COgfX7JJD^~IoU(8e*O9ipZ5IW>mAeB
zkQ}?cOQw^5|3so1dEAOK;Uy&3*5$?sX`F#cat0L9S~Z2Q
zObByhDn^_uh$pJKO469c{qa|tDp|%-rziL*
z5{uRf@yUy0h|L_uaist<{&q2a&nenz5w8fUD_}t{)4!qUIe&fGmzNW}7WDBrMD>LQ
zQ_L=81=}@>$lu?8u7t6m^z3$OFaX5E&EmT3QdRG9Ogg@NIT)ql(UjQ!9%Ya?82T()
z&6o*?CdNZhRO`xx3!z;pfeX|19OLeG3-e0idkNtLoy{P`${mijmvEp^GzC)3GX1k>
zx2N;}ElBGi1r4?kfw(@Z8Qg3HJm%9r5#ai!9&x?gA${FHj1|K8yKVyo5~rAk?3v$D
z<<*kvNqCV6A|Y%$>}i^)7TZa`VIHp+@d>P1z@H5?9{PT)gbEHxsAH!!q6^q6-%-he(thQ#E#`;@`u
zyu@<=L!|fo^(zX!<@=;XoI-duAIl3$(p3L-#fgs@(TzbHa1YzK-R^GvsV=QS(yPoh
zW2)VxlwnicSJohq~2u(d7dW+7YZ>Q=nQNRl#S}zT46LbTY
zIkh^=D?dkCwY)pVL90gXBD7XIn=6n9xU+Ov^Ng}GS+!$pHWyQ
z=lHNPT|#7gI+s;P8JW^b!LaEw#4ZERyDGJyUU4TVMQ4;T_7m!d^*sx;MbK*WjVl26
zUh7JW0ep!2psxmrEH!bb`2BaN4lfv9Es|@QJ6R?vQKz~
z1w|{DH~{5t(*%hzLuXSWLM&|#+{V#u^d`{Hpm5bw>(8%MTf{ge%FfAW-^9bCO5TJ(
zbCS~%TlR}o8;uQv9VJr+3~%~%mVOf9)B7xXGYG$FRQHWP0I|Ed;sA+6g*%?_V*o5%{Byo?fhzVd@WXM;ltFU?iw)gOE!Dx9orQ@}*2g;B?*y
zG7R{l6=SPQps*FDK^6s}lT-X`%wc(4*pd|Lo2VOL6yv23hH!xcy`efX_2tW#m`%Hm
ziTiL1*8pw}Y_)%(Q5SF0TQzlck5AcI-yi-ARR;F_JbqIZg9GQe7GW=d2ouI#2>!vV
zKTkB1Y&CGMJmCWeQiX-YszfirAKD7!D25F145XCKme%51##$;DAsl3eiX6y}Hlk8t
z@`3=spL=mt#VNR>W3{z|mm>(!cIRZ!(FN5W4zguq(t6`7WOgjwk0Nrv~MH%Xj!$FEe&;|H37SGSGwSexKg#Zef
zcx}qKE8Q*s!B#9G8tl`1ckk}zuHo_Xk|_kS!&cv3{t_R*w6aN^&jI4QdGiVH$%M9WK2k6CVA)&qX!t@1rL`?@
zeQmUQM>k*P#eSN>f3nEH+rn5da~+h3UTM_g1%d#x6Why2hWSC$*?6>kWs$UEntdsDb9;uPP6S13YhPJYOT^d#|`tpZih``6<_A_094O
zQ_BnfLW#AH5kG&uzsHI~jg7Z`<(*)oI7~;TC@$iMCtX(J*4YjCoyYqY`^3$*9)iye
zGT@$H;ii729fGINLB686mXzhYZ
zVujI$N{r(oJnm7<+*9unTM9pi8L|AOF=q&>*TJB%-|Kf=9(g|?GiNZQ
zCX}Qvu6X1)sr%%Cwrh8yqrc;$%$|h@lmQMhlh_O13z}+P)zWsrHH`Mb7CS@VgNL&5
z1h7#Xa{KGt9b=F9CD8^H#$6_I@`w>_!h~IKMm;rQf!FcNG}1Sf33)PGEBmq41LAN{
zb&E?&y9^#||F*Jgk)dbS_ZjkFU%zhW@$^LFnGhce;y8lxo`_rl-UP|X2rvaSpzP?=
zCBDizRvE!6MnwM~>SoMbU`h~I86qE`?JLZMNvO+LJHzX}XY3;*^bA(?C`oa5M
zP62uG$g?MOGQx$gL*L^AAA8)ZcB3@l$`#DdVpHt)iHpk1bCDi3j0_BM5Axa45YI-vnZO(@F8^RJ`RQI7UodsO@nVMxtOyF_
z+-!>F*vG;=PQ#za29=)NAAe64Y4-+m8`j!cKZxkqwr$1j4sQ(x^jnQdn^8ft(4>S9
zw9ou{1^EFh6u3Tr)Q89C{*`+f(gu)YKEiPX4g`n@`ZEXGG_lSjC|}0n0d}#lQa63d
z8Dpmg54Pj;&PD7y8d_iuB&QPGMDfX2W@RfuREkxGEjyCBNp_s0Vy!CRSGK^ZCtRF+I8@R;yk-Ze%nJn1GOKFvTqk?@K1GqY}8h)@2$p}BD)RXy+t0tpN%
zT7;NfVkU(9#Lb;C0`h@c&(7aMHD@_PPmiFEV<^~j;6M+h$6a`Cz{z=U*5)tQ%Obc=
zQXE&B)|W*;aNy({Z(e-A7BMW18wF*j?jr~h1s+lgCAqkwfC(;h=D6qHY06UjZELrj
z#j(?6tcM1c!$ZuS|4UgXPV~nM)^YXfN&h
zlbSlFdWvJ`VPNs~Bd>6BU(<@?Wy+*sq!qLroo>68BW~{rDo%QH_EF{>evTI`A0(vJ
z(8&}G_!Z}6bC{W3tKEb*Tfv)2U4w6LEHA`l_+i3&4e8
zE!Ogjmd&K%F|=!{f@>ddEyP99m9GdS4s7qURUq^lRhLmUB6%$9p}z
zbt`iCty;%|pZk$?
zH2%klKfKL6I*Mr9oi%4p*{ii?DC;OH`8Uem870-Cgeqkrpp&4qo(-s3*
zsGWR&CKi|9zLsLrFXX@hS_z{D77~+6`f)cjse4MaFc?q0yot7aac&QAFs94xaHy3<(K**I8h@4E59ROJ_F>R<3d4%PC~2|6!hLvN#sG
zrH;TooBqBA6^2UV#C%)`L<@Gpq$yCZHWGB>r
z&1B`plfchcSA10$BdU|z8EQ@iw>*8<*si{*jLAXZ4R1|86+3m?g+TilTm6Nv`imu(
zLxzdO2UUM-!PH4;5b{g^WEkN^JRZ)
zfCJB^JC4#jy(dn#Ee(46U*#Iw;JxYFR%>8wB_Dt!PZz?gwYj#ldyhQwTKaP-pgv&Zd9*4H%%+tCv-yk~)72#nZ{;FcPfY
z5U)dL9i4_A{de{-HjX#%=E}HmCsBkcllYt#@3$o){NQVJg#$|WiqpKJU|DpMAI-$i8^~&j3tG~)CB&KLZyO&UX&vF3E=;GL
zi`fi>jPCo>__*xMCBW
zEVvY_ugW7lKke}FEhI+w)Gaqxk>PXcdypjT1i*g_y;=ZR1
zf!lBTe*dE}C%TJ}frlCDA8rTt5M1N~_ABs7s!dHTt=*`->=!<9U_?V4ar1R)#s+mr
zpn(G7Gk$3XuSuiFy3wB}4jD8^Y-vb=B@Z$s4j8|59zV-$Ms01XH_WhnA*urxhr;=f
zj##x5cMP10yc2$w@S)tWio162wqBr}fVEi$QcG3Kp{&nR-rs8YlWS@zQJPAR35q~$w*hX6Z5WT@M90~OfNx`+VEr(?JqB2$`9
zY*t6o|pUl{y2|w&V5VY@8|P=U*mPXUa#wO;YA1V{5U?)VrptO-g($ZswVYr
z+h^{P$x$CO5CqK1JMyrJk3NjefQ8MP6yI;VCpIC$0ZJZYk0x;N}k
zzgsIme>DbUE8I29EA?qw^FN5EasE4ymg?=5>V~f;8tk$O(LM~^p_SF1u!Cl-D}-Mz
zvHC~HSa3TruhlS_TVV)sv*YTbqyhHZ`koiRyl*4=VH+6&&?YmQj`6|c{(-fDA&r{B}h9W_|xqk{WP
zP@zoCV3Pnqvcu)o0pydGYHHe(9tJv-4Vu7#?sX);{nZ;xU?LCBS|HwL{PXYM59OT;
zWt2fzpx#+KK)1M{`$3=HW^JLoW>^}4j8W8IS~@WKuM}}V*3G6+4Em+V)(7;u^!t4`
z)(XI^yycbns{cM=FNNZRb@UH+HSqDamT2W&j?T#Rua|6HVBpOKQ_<{UET8BoHfhZ5
zLLOBZL+bdQxU_!Az#-2PmL`ElI!p;r4Gr+LXrQ9;AT5>+Tm<$XZqsUyG4;cP|L66R
z2?w1j=*wCn1*Le?%`7|408bJvM8JzXgCMbC-4QY3nMLB*+cJ!gTFzkwRJkn53^8hI
zP!$Z1y2i7-m3C8S;<{{WxbOAefvY)!PRZI+N^4#qOGfZe;WzP
zIU%107qfAZ>(+hkx8VJzB$uNL*}lkQ7j8JEx$^vmMq`esk`-nBh{@8Ct_k_O&nZ2i
zO#kliB(7r@E(aggy2k@wD@Kpnefy5PuAy>h4Cp9}n2bJb@Kj?UrGS>t*0eAaj|T$x
z!LbP-NF2buB%S{~o*TLYaTeqYvX!+i#p|Tap<uL~A?%2(?DOt1lty_sJvO4fD
zA;)YQwUvTw%P8u){{_?-^}i%IbzH0--)Jm5Dj+Y5SCm`gb$;0!JA@tT^WPDS{^vK)
z+1PjsLqD)a*?I-Ti=Cm7>POeK)X)fkz}9c^8(n7QFyhckFyK8@`v#pp*t&m7>k69X
zO4Qu;rA3*-RHi$~1-eD@ev{Y%3#}0tqCV1EiGKY3KEK1yvV`u{t02GHy=S+-vz&Eo
z?DTn4*QWmG>rwF3yX(y0({vSL1%NjslnzZsD8Dxkdi8n|$|Me&UFU#@5Lggvw;_?s
zqQ``U1f^@Xar!&T-F})b1HVK4B6S#O+e*kybZ%Hrjd9oflV(=Qfb+Z8&++5O(_8VS
z`bnk2V2JA!V&ZPe$-fmnDC4-Hj)bc&EkAR(7cr_5BOREo;8ymd{6~zGV~>R|BH(5m
z9PhXH5;J!1+9k3vur6ezfa3ct{Zs}MT_t-%9tPZD=*S+i&O8f1a2~BSFfm80=vtct
zG`PIn3%$yA-n?L+Nx75`bGb&749#hrHh
z_;Ra-LeEU|yL~?T_wBs&-ZSgYOZA?34qoD-{?IX6zsSGw%dRhvja^)tS+Du$7q>Tu
zk6onTGhu>?hM|AhW`{~g0|T^bsO4AB{?+|+(P6Pzy~%yO^4
zC%$C#EQ?ubh@1GMd+*h@8F1I{&80k7mW79M1_j}f_@QY5rYj3`Ay~8TAV8fBh{*T)
zt&s>-H;y0MJ7D=XW+MwRtOiX8q|L;+f6Vc5A$+9K@a#CB8^XeJhx>g&8dc$bUYT`6
zi}99}ogv^_1UO-`!wQ=+f4*q5#9ah`8p&>>6wBT*q=3Z39CiM}*YE$Guhi(PZd`KM
z@WlAOPFtE|)|vt_v^EBWwd+x6|tip|J^aIzW2Khf37Is&ImTq0>BH_eY*`<3CbY)*?V%Yfi|h{r9RtT1i?z=WpFbKjsdY|{=h`_GPeN)&&Y
z9QNe>lK#L*XBm5mbRA&rErE!E{~msVHETop<}{1#as6`X)RMSRn?zg?-zJ1Z`Hn{f
z;emj-Ju*_0-{KhAJFV`+)v0{q|2jksin4vRwY)g6q^mqup|Ejh5WRLpGC1dPQuo(m
z1`W}BiG7lg;5Wa$zqlB+`t)h0qq`a%a&gu+3mN4<;AO)nZ5}*5zQ8TeM;*WubaTt_KzWQ%1OX6gh?kkk)>umX-yW
z+VDDK8M}c1BBeB`XJ&P@=(Eah0TbDik{=({jJG2D^0eS->~>zDg<
z9BruayLhdT1&#V;Ec+C7sTMb)`AiSDK)xXI%IYtn#1uIULDiz%h`FkyRGDi5^$TJL
z@GDO5@HHxPlsay}pI(ijKgx%C@)odJn979$XH!I8MV;=*c$0%3)P7&NP;g;cj%q*X
z-q8b{W1|1LG9)oRe(Y_n#GP(~`z5ws8kO{7{8~GE@5!y-?W(@4z#7Gm*P3jbgs)^b
z*Kum9{`N;>eG8Ju{P=d2@@id!CLSMjF90)DURE1V{T&O4G=kYRAsbU+;~{2Q<|9Tn
z-5S@tHC$ApHIfV_KQGw@!Egb#R}1d5dDYJYPKJi#lXV%r{$;O_xK8^0S+@GmCkyOR
z8tXb{en418OBc7+x^akKSOw*m(0GW{*LdXA*lJv&ScuU!&$3;yItO;lCVwk8YbMMq
zsbwheWxybWdYYNI{Li9l|Hx8{*Be_bF8#f9L@Sl|Mj9g?%{tN~-1I=}&s)YCmOiT2
z*(}8N<&m+se{9w)SXHV*(!6Tq*zwq-W5cLPaVnjAX7}0}QZY0&*K;wOjjy3G0hJJ%
z1(O8b&NkO~v}>NT3hN8(W~KqP09%SI80IUkd6!ub`R=w1nTQ0kYlwg@E7FCHC_@rl
zZy764VQqy++Wxdv(8rK~L94z6Avie2;~{!!(Jt^wZhN~{(>S*2Ggk|^?ln-QNbX=j
zAq^c&?E=T;-lN+KSrWPsoz0K(r`LX=vpC^4a!;&oyyDWO{bN(U?!tSgB>>Z51}ZjC
zv31(=fReJQcZP^GK{@4uqjvQW1~`rF@ZmR%{ienOUx+}D+brx~I({DFLC8|VdHE~5
zbkn|iV_!zIr8kHi<4;bh*P;KCm(XQ|rVj*-f-kJ}{B!o*@8P1TwQoOyVrv(xp4qk~
z&$nRyvhS45m9MV`pPpS18kpkY{JC=GGQ}35p}U%_Sg-rC+_v5dg+u;5?d&EtTxD&&
zaLFe(!l6}N;l%=Q9J}$S4I~NU%2ske+P!(hkT{`q(|`*fNPND*0`g&54CnQ|J~_?hH1UC5#d+W4t(3%Zp5$RMo-h`k0iMQ^p1aD6Q6W=@1%1@
zn?DYXb9s1hXd{^cz*khV_C3kl5Pukm88~~j|7TS=kU}n~=-hu6pXE;Tq_IukOY4I3
zjTPg~T`Z4);SpYcc;l(h|8DN1
zsiIWV{Um+^6m`iX)Fd``Be6326apF$7aYgxTN<{W!Ek?G2kD
z+h-G*BR}^4ATeb`wtD#|PUN@Gh5M8Ng}M#g+pT|x52QLHgBGO?bWq!Wrh3bZrhdq
z5r1$h+t7f`#)+Is>3qG;sztRuC-BD3Ok|*Ik
znAMAaAIYh38`)~E;(NbtzV^fHuj;DZm;lqHGg_8zt@mdD8;>b%mh##7JQ
zRe|tb<+S`Y7W(}yBxi*qy?Zqy7|i|ZQ~Yxnrfd93d^l^@0MF(vuW!F$(PP_fJG&~|
zMQu-KWp>=UA+CRiKRA>k%!0t6^{;w!TZgkZEBQsYcK0+8D&>@*w5Mkmb&>HdA}#8M
zY4dx0*y3x9^#)PpE*v|75h>z#9%!uH)I`}m#iJPk;w6-zE6<)yuzyt_b`~tzq4b9!
z24ca~@L~o4mG1!Gz61Ex^-Wz5Um3gUp`oGw{7j>*4VT(B_Vf8>8uhu1F1_T=?$KSN
zbZ+-FV`akQNb2f#`GMvki&odwEPulcluGXx-&U`4bmhRKPNvhYHij*qNdZ@YN?x8(
zU%M)xk@08dwh;#qXu&Qj(V2|AXJNTZl_5I}8m4cjTJ{~V#Bh6$$t7oPhI$$GIW$J8
zs;5zKlw-8{l8Pxw6Vsg`Ui-$`9Wa1hw*~L
z;2b+T>Ros|6I5Qp8&X%*wtiz!%$F9n0F$E}ett=TNRHvC9H1irD0vsDKx5w~xxKB;
zJcI7=n+38?Ntw^y*8x$QXb(CoUww>XH7jDS#$s
zm{n1*FgDwwM4^&ySH;s6aUN%>?95hfw<+Wr{F^W^$nFz3+rvZF-0=WgfXSL4@!K$_B)D*Eck|2ydg94wkY<
zSF4R|Gb_0rxgDyGt!aSj?#O_R8#c(O7cRt}B!?+@Ep@cr|BF+gwr^A;|51l?>-PW4
zBuYf*lq<7M{~*IM`tk9_?K4-cH{J9c<$eG&qdh=
zRd#M4u}=Y8i*0U+_m&0rM_!#Y`QjYalsKBFo{MvIFXfgkjJ!YL(eW7y^@4-v7M)s8
z1Jky|(sE^`(u4`S6`f|7Fpd%o^ieYl#j={3(%;5$*Qw#7zqz#BBGDO{yOb|
zn^n345nRGq!IYtB$upbx<38p+n$H{{r|P4{*8EWyO#tQCwDYp0L})U=#bOI^VqO>i
zu{aCCdYekm4TXaDzF=a@%1Q2DN5h}WdiLyDXOB_}KZ{^9vn_W5IyS!HKWcEjOr?{J
z+ScXo%%qA;x?GM;xYGoSf4e8#%xGr*Dytm^o
zpPF_Z&s&zRIWhU>xQOIV{S1De@2Y6aN8DN{e(
z*|9sPciU!SUjYe(^%wpG!4#tJY0o&^0;c`C
z^o&92kb_P;x(siA+AXoRmMrz*gBuOtBvzF5mFuC-zkrmG;hcpw*Ft>6X|!&iw1n^+l&}{U;$hF(n!6_EH#x%z8o*ItV9j_zHtAg-{1
zR#fP?HD9vpT-;B1Wjl6wptfS^>9-@Nv^`nBZ}T?iswbEv?(E8F6V}wi4~rQ4;f+x;9si#T0I{#~&0QHlgk=~k?Jp|~{3FLM
zVxXcAgpQJ{16A9=(LT38KV$#&sinvPMjFZI%3V4DlNF7wV}g+15IZq*2s3F1KtG{K
zA=&eOM_gT7WKz4!rdV3tvuDp+L?v`CXEa?x8s_N;)Ms@$j?;hW&KAMiJ
zz+3W>w>xTE|JXUTpn#0Q%--tUsBPcg?<&vBD4N;63Y}E}%eQEJz|P7t!gQ|$S9;@J7YKx!u`DNX6n+5W0q^zy4k2M42Dt9`TkJd0nsZ%wXt
z>9Hq0I-B3EQOlekI^pYp*yleN+be!noNKpoyNUnk!OdPSczL7gw&i*Ub_FXVVd{O>
z7wJc9rc?B1^2%TgOvNVXy{PUgPqqAf*vTi@dhFoWQ8O8yDs+t05Gef&t9a
z{Pz7U50e_>(aTq_gxge@KB8xeWy4$Dp2&V@VnKFfsH)ENunO=P=aW;0tZ#OeA&cx)
z;r2^XVGOT$VaYYi?_J+h%3&B|0OCfpcfl}ZHRWZCRd>JPw-SJjEiZX@Yw^LGmgwT#
zKkUq)rn_2%E8SUJJ@|c$@q%{g7gUclGD&@Yc=7;ug>Sm)3FnNQ_La2${y4kCtij$t
zD||EWg#W2a$om>Uv9GPtYP-dCCPqJ3)?exJc|z2iHQg1?hwEOP7WH=iy?|Y=4)(8(
z)k_LKmRn}Z5N3gINOv*$LCA!}?AVfL*J*=gFamP|bKT7lpA~L(4?oJXl@>DBKY|z&
zP7K$P%FRz^FA9D+)j7+5ot)ZtzyLOR+aSS-)Dg|@^*_1Gyb90E+f(}!#BZ9X=SE^M
z9Hf#+_s|R@?(g2x+k-IU;7EAhJmV5p!3_D`ZLB$`i;xO1XAuEYhG99=U827}{k5Xl
z3bxD79H*E#hp_vrW!VUeo>#mzzkBERxMTb+AKn_PTLm?B$ijc3Y0~C7ZYP%IG8I;Q
z$+i0Z$L0LR)L9Q-g>gAR7)yxzlO|Z!IsgQ(49^}hzR4fW1KPKypXn6(Tzx<4l8%LA
zTAkfrnR4xXM~4%WRqcNL>JeAiyVUdPvC3nvwUIw`)*0MNe0wZ>PiTo*g@X2O)#V2J
zwyiP$vdd==C-xxklrqSHrzm9*nJ4yHKe7BZso`YLukkd==L&>oqz(MX-DfrYugR-M
zxB;mCE?Qi2^UH~!hxJz|ka-`~6=FcifbYL~%38$+lO3L>90RehBwc7fv!9ngYF;V3
zbxlbrzyGc~kM^x|b*S1C`Ze#?AndgHs^t?U-Z+%LduKQuliW7Z0qxFx+coTks)vVT
z`Ao;A`71)thqX{ls&{{M?dmU`?x$31l-8=vZp;2XC`t#rFHQmmdm@E%*oVWPg&AJm7web^dd=5D6
z=;qk*mq)9t%-DI&?jP@Q3euFqUNFSex*8^ZCjYz{Z9AagF^AAS%HWOPn=N*ZN4uXY
z>o@rsahx@#O?@0^-v}%(NZAN}Ic3hAO#ttN8lma`9$S&Nbl93bc^x&W*?qlMdMw+0
z>3en6%OUZPZUv|o^ip2FaN$eNq)mt8qxOX!)`I%kiS|+qgo{DS_!R31L2dIc;dap}
zajXNydCz&B7}nVxx4BX};>of(ZTkk7jDHv%_PqDS+Z)4uvhT0fHR=7M#jWn|4g9|u
zeos0Yah*5!$2P@e$MgP|)Xxt0j{Kt(dj7tNc6!1pn*j&Mjy^WApL~0Tl|DY~go>X1
zVFRqpPX07c3sFupTMK3pFNBHTX6N9FRLywiFktF?B9~ak@dC(~0^mRpuDqV~Ww-V}
zkxq9aO^O^Qea)RP=Vv7#^-Emu?_YNK
zQA~!J=x8hLSnCk}?-3_)6@ufjFR3Rq8_1i<2$qOQj+8mgR=NND^lan4efutbx9myI
zbvkCNrnun6H2a1N&KumClrwt4(Wmu46sA45n_THN=$6CQxU@}!$9G=3D#d$npliq6
zW__F7`IXtENm;S(LOQQ6&Dv#KB&aqtTpIsPDp_G-zI?gk;Ijqzg^JH1fkwn#sde}-
z<}wv_T=kIo=`N7cl!e7saN=hzoXLc^5mpVu&doc-;2zTmm8~{m+KZ>ho|)eX0Z;Ub
z!y~im)pTuJ{!G0PZmaATqBTUfw)vs&4`wf2=Nq5-eM^E~k
zQMtn@*^TNMm>tWe>Fk?`jEuaz)VT;c4|C>wdA+mh!Rg&ws;{p*qhi?&h1a(GWA7*#
zhI$umYGJfIG_wDmhG#>br@m0B(5X3hWckY5<+JbVue|%UkFH~_hvK*$&;PyHNj3bk
znfviEE(a&3H{GLP+xSZ@mIZrZ;uFv$e?tO&{r>&2@|chB-`fkc#RuYFMK9Td8b|Q}
z<$vFP{rh+=yT!|v#j|t$cU=UV
zi0U#xMlbYAFFYTG`7KDS6zdJg$1kYF%(4OyX^DHnq{dSRnjhDDj$J-8Ng7b{?oMm
z;W)+X@y{oBTfh85dA_ypitJB`Gym+b`;&H~?(rFye8=>eXL1KCn5MXz{2JRT%Rg>j
zv!CC;*S!0D3Xa*T6Lm}(amPiNUfw9<$DhI$0FW=2KlSj?D=sS-YKjO&^qv?guH@6?b=Zrfi&D02-q}Nt
z8(+SNRkobd+afk7Io$fdMf+jl`#M~#sn+dmrByH0Z6q}_^d1YrHvP2}7BBv~#Lth7
z6|kDEAvDC*My#ctdRKUH00hm0)VF_O3Eg4^Sr7rO5{{U|w?U_<7f~IFRbtbaRs$2)
z4&Cr_%(-4$M~paYGAu&xoqv+O-VI~pU47@dof~DW8&GvxTr?7k8+KkA3`00&Nzqtz
z&ukZbi(gzwrst00D;vL7eVJZeqLZqslM=R&p$;^(%K&NP@SzOPG>CaPOLm_zpUXEX
zo(lXhVprdpDm#1h$a6~Q*KL@`^H#%OE{+Q=8Zn}3(bHbS8X2ON6%>1BL;2IxZbKjM
zXni93e9x}RX;iIn4~0ohN$yaQTYx{m0Cyn2*HOGG>IB?;uG>EJyn&cY1M@FLu4okK
zI0XiiXjJ+7?dEQd#n~my^G;{?UzK@dj*c_A<`F($;otl@wWm`{*Q9t{?DaKKHERXZ
zjR01**?X^0aJO}@S>$r
zrD;lG3}O`AB9`y}Uz+i(uKL0(_v(S4k0F%b%8mar{==iwjeZ+ySUZ@tZP1~jr;~G>
zRoVOOb#V@Q^)$1BfYwkoirhc0_rNd+4g)87+N=r7OJ4~$ZP&n+-W%?z?=QU{*1yA`
z>WcaT=A-(;Dsap?odDD0y6vYx$APT53re!-np$6@tv@x`FvNm^>y(Ouleg+R#&p~I
zQAe+WiC^E=?uLz*mwiJ1+=y4bW~KT6?%K*P-2TF|(^DTkPfyOHDAWSky?t+gM1h-Sp0s{0
zaVkXq>xe37*gh9U&Yl&h+Au2MFvHpocwmZ2-P!Z!zkM+x;HcUL*q`KYZ
z0vj>7et4-9Yoji8=oczkJ7wY2u7(=EPymgG?K?VcZ6|-7uB|p7Pb@2bdm}4)O+5S)
z_4eAyyI_chLa`76?&X_P`2JU@*K*IeGppwX1-K=~6VQ2n;1_
zfM6+ZOF&Hk!h%IoMY~l$acsBC>Q-8l)tkS>xt0TZbKd|9Y+J<4lse5;IZh3h1ubPXbLT7n{Krx
zOZs<^!6)T7jPlJQ2J!p7L|6&9NrTp6RlFMarRJ@Dw2}Wk_%kKW+rN1fBewL_(BN7Xcpm^WC_gA
zwMqWB_&FqBR8&rkiCGbMnHRnroSBwo{GS8xw=fS$(KMdIw7w)^Lgb}6y_Ay(tJ`oX4}LRuGne7@&(kJ4$W1NOiv_A`c(}$z}qg
zUn#VQrdp={2y#U$e>--C(5uEVQj_KL;6tJ&gjw92WW_HeJCEAgU6ol4KPPU64A#Z)
zn4PFWsNkskBR@5N05<*g#S!PZg~@@nAm8b8$6JmzTsS77EWxQ_K*b2l8|Btb_9|U!
z^gM4)cD=PLzq&9!;HTdhPqZc9#Xd_{Ef_a;x~BI5EVSBNI!g-EQr
z@$_kwc+tZolg$nU*~od>;zS|VE|{ALtBXB>cM0>~CfzUENSjeuSSW)qU~DIL0?X6`
zav%@zRMX2EI(y8jo-W%zaHV)9a9PE^&Dia95XOWd*gi0R#-H6c*?M^az>U~iiNud`
zR#pVjcQVCgJJFs`o`>-G2mWk?)nCU!tHpk%>g!Flos-kowXJY5oE87jsfA$!!;+L{
zgZnPjc!GP{&^VoeM#`Rz8XbFlbbWNBCJ^b)&dS>Ubqktd$7)U^eErrfyLF-W(9jx=
zIkMyOXLrb;Z3Zke#;ya>U?3HbIi@zUlZN{slPPoGeP6d2hJj_uPDoypprt}wXGDY_
z=IYRS?9|yr&Ha$PAv#2|omfV|@_ma$c}I&4usYjOQh7rxca;>zAPvN!5yE9XW)?Uf
zQPX?LhHC!M7X2zZptrSo-DUSn?Br?pRNR9m<1OHaGaQRwJDyHD^)(}nH(9&=h~ov7c}
zOl@?;rpnUnhi5!Sa;tl^^3uLKAs<6rs`_LYg<}7sY}-)R7bc%YRgD-NP*#7#@_`8*
zGtJ=y?uT=@C%Aduj}3`wThTw5_prbj^`8=d
zRCnIA#qjj~fgBk2&qOv!Htsjn;EqfU85GcGh?yJ}C#qZpw7%2$MQFHmZjZd)4VJUb
z9mHlhfjo${Ye{?EkBzMyDVed)WG*$h@@%}+-t1P}xUmt{N*u<0Wl>ixeSXk@&KDgx
zQ`SR8C%{t>cL-OEHt8A@TS8PuuS0M4bhYLm#GzUeS7Nf%DWvnX8^ie|C5=bP>OP2j
z)V0C}QIN`+5O=F|QK`zaBzFUI{e}uYsclJ1>cPPpjW#JZSgSbL(md_^#6|-ZJ1Q>s
zZ)cP?C1TT*xX?d#cC(iBHy$~1QTK^jk6q&4{d?LF%;GS{72e6Mt$a_OPAtalAk-fY
zn+@QhbV5ADL-}*WnU|TVaOSAW_91*`g^(d}hr@*$!5QiiS!i8Y*ywZ7hdFbk*q5AS
z5mpi>Wr*Ly{n7{~Gq9DzySa+O&FMda;m)$RqsY3vxg&s?b5h%aFJWplRAgoFfZcH|
zyYozkcd!AL?_R#QxHv07)DW==qhd*S|7nIA>bfs79(S^{&>an({HeY}4IDc%lDS1q
z!`T7~g{Qd9>R@Je_9mErH6P~mY6~UF9>%7yC$TgGo`(o|jhKq1kYn>Cx2?F)^Y=-d
z;9X&mdbGtk1su@>;gayDJRz?8Vp}Rnj>+-GpCz3Of~jUmab&~cYl=X^uQ08n#jFPe
z@*024_US&64{mI?8`1NR*W}cwfhr9e^ln=@EA`3R+MQiWJ@19zIOc!4euKHJ#rvMW
zzy24aj2ns%7n~hts1a~9|9Sn626`d8JhqPAxVPV;u1*R!tO94fDhXhl0vGV!j!j!s
zb$pz=MhIs~vIYo*gV&1wt13&BSx}L!eq%(ix}I!yCX5VC$Y)&A0VdmZzKOVONkv6<
zt02htfx-1yzl2y_RtmvH^_v7V*wCLfM)#ACUQ8F*eNp`tzrPs?GkA6NE|5;R&%(LI
z076z5{??lhBfu_1XBeX9I9y{Xz!WKs*3477HJ1+~^CLMPG{6tz95gAAuZSxO(*jf0
z0JYd5;lCp#vR~TwF*JTsAwa=?zd-P)HPv4lA@>Us;0yp
z2op_~%)ppwiEZS-|7h3KUPHV7vE@lsRg1z#JzKUs-re}*nbTLZpSJDRuHDCvpNw|(
zG-&m;L>rQg1Xa-b5Q_zq%SQDGxaq_>c7)g3=p240wNxkS0
zHnK_i!q>$vC*CtUymrV
zYJk&X&i(fDr%u&w5Opu7qk;FAp^ZZ#V93b|Y}vU(F|F4*G+@mR-ia){p+GZoJN<5r
z>xle0HR=D34Olq|=wB8=31N&*FqWe$dP|wWfB4XCXkLJ+dZk^BHFZliDY)=_yZpmg
z&ceHaYcm$6;)M#`c8C9JF@O6
z0k}Oh@q}zLc9W*bJ*+!B|DzL#e&Pd+A4Kx^sSC1MUz5o~l=yO(=3V
zXzW+D3Q9XR`fzg?$DxlZ91Xjr1>DC4lqXZn$!oB0cZ6?TD7Bv0hFz-pF+m)uV6Kbr
z&wuYLRd61PD+XI?>t759g_D({oJA`d_I`f9dnw4+o$;-CLOj3hao4Y$hBwfVp)=bK
z)>U4Uz=2=L56iHz=3Jr9;_>X|LC{Kyv4Pw?8*^RSAM?o)e^R!Aqf0%ya8gduRkWFK
ztktx%8fQX=!4^v!(ENM15lR2g1t83f@ff3N)%c(*R~E@dZ10Jo6=|zr^wIUkbBz9a
zEEZ%E^piQ%BCBKh8HN&VyEgvHL63xMbmUL|!a7qe&95kvl$elkvE7+6ftn;yfWNw{EM-z>xI}
z09Nc=A&YQC-oxvO8JiI2zx57f%n;@l;wAzE!xMDUWt-Le>FRWaQ--dLCx>}uz=^@6)FPD2O
zvI~-0j`E(fqz4KYcH=U|4Ci<*vV{N;$50U!Bb4H}jynW#?jlI|{vhpH}Y8OSD
z@V^_H9}A^}Mr(%Bzs6Vtiy
z4n}jr$=o+!BbK_{ndy*X2m1UN@i(?zKg7CI>1dcYdLI25N8VtNAjS*)0D{6DM0+PZ
z{P83b2gK9cnrK_MX4T48@&TtVVv4W7aMK{wczr9MNi)oh$82_vuF(@?>6&n?6I?P
z{8BX~*Q2qZ@X`-~pBdLpzIgFs0i#qFJ5`zf+)A|=4DzM%0;&Ly`2;T*`H^EVF9-z6+2MnK7VO7qHn
z7aBzfiv?U{F$!r3+_dC@tnZO7l^ckU&miv|Job=d&}hU&WOq(cEHRk
zCV{@N3Zf&$*HA^t)En4b*3!ZvZEiu^+OBkfv?O9aM$sz(83|I%?a|p~l40V`pHsEc
z3gK1fa($@>){iF3z9v)uXnGP$E
zJn)<~U*QK3>)#Y~l3wEoo4ez@kO>k)#px-~&+@K@j?@=8&M!-succ
zvkF~!1+AnE^v1IvWy#||nhV}1;kRYPJ2pM$cWLq4
z=5#_+$aWe%CXQ2az90>NC}U60dnR@|I4hfImZ02xZi+4KBA^R=^$jntXf*qL!bbA^
zax9!or$8gk0xhM#2ni3@t+#SBjkA!~mvk=%G0mbn;(NO!q=nO?gO9d|K*Xq$fsRi{NH1l0=?`xR;i>VJJZ5fglPqvhyeWg
zBe?qJEW>4{5vzJF<2?^Y7?NBsS*wLhsq!ZI%X@Iwe$%CY|1G;L#cIF;?dy%ayt$rSLt<^xMi+`*Z~SL$OFeS$o~p}dB_$w;kHA_
z42gB^+7kj)FOmxxI+MAJuGRw8jva;k3b@^8Q_Vg`YxG3SLniTiFOn;w=?@=pC4hh#
z;zvPUx6A6e5l$~v`*fbMh?gmwd6GMLs$X!S7TAR{QC$3J2;&K|GG0Lpqa3bk6`R8U
zmn}K>drnzsvMDIl(lBR7kJMN1}KD6cbBeU_4ndNwb8nyPKk3fNGiS%@Avl6DQJi
ziy;&OljfBz;lvfQ4-a{V#@;GJzV8eQHe&EL4yIJ?%A0oRPMmR^dzQnzefNRx7hBAa
z3nkts=3vNJ{W0`>LGP`n1#!oDs0DTG`obH8j(4Q|gs)$}
zhJ6jKDnadiwAgRRdTFS2u3q0eg9mvv9hm!IwWp|0-r_WJFWhFG`J
zQCSz`zp3-i_+9Iw`eg?<8F}p=yG16ZKeLBz(-?g_P-C~xflk?LyR{l}Ewe$7F-=F#
zi2nNLkjY!k*?Nk}lYc(jI$_zp;)#z}?%%vP>GAi;V*Df><@Et$f&XBnDU&BHEyX$(
z6v3LVpYZKLwM12L>c}MH+h58vYD@AtzT%V4>cXhEK6^gR1N~;QFcrHlZY^R1oqzt>
zN9`sHzQEvFkg&hKzg4(8uBhVfOWSX^`L#R#!%*EP#r|9bOjkvYwz-UpJ#r^HV*Y89jLi&`LvmU08~+2r(|xyLX?wYc}yqYjR}
zy)HXYkChOdwFhAQ1=v
zw=Z?a?}wQ4W~C1nWEXsYx%6CCaUxN(pl
z?%m^$-g`x(|KDn#d#sptD*0j3t=nc=QMQ*|r8B5qd?A%jU)@)eGVcvMAlZ`A%j?ky
zE48>+)uS-D*Ws~JE{ZN5-$r#?wH)lH*++!0T1PlKl0NvIg7eD0x}7~H*%R|l{aW|O
z;>3W5E87(q^a5GHy$0?kmZtqi?Zk+uws>VZ;jl>Txk8wrV
z(D1rRFdOO8h)=?N;G<4nTkh~fImAA0)u$R9i4;zJyiJFvGQ($$w5ST8q3&%?L`F|<
z5U6@>Gqywwb*1ksFG)#0qFIdX`!Mg|WHttxlI+=!upLtf9@}!82^02E$BU{O(fEBq
zQ#7=Poi!W>ZiZ9pMrw`QnnTnG8WVYmnjo3v=KLbN3lTW6n
z;hcPHM{U+}WPiO6K!80N;M_DWKK|I6-(?0%X|eNxQ$~BSVuZDq`GixKL6BsN&d4&W
zHIt%dGG;k(F@w-s*T3GqXY6DwsmXw)HbX=
zwEk3F>|F8#J>r;VM<~X)<}sX;uHWf0^M3#S{T^MWedm~~`-^TFII0zY{+#sr$*Dt1
z&UfN)1RZK+l?NqyOuM#ikFn<`AL6nT9(rSIC+lAAYgIz2x2ufq$vVwXKbsQvRuVJ0qr!lR;&ok>e`UHgX1&Nl1CR;`MroOHnw-StY&i`+OFFK@K^=SVM%9yvtAUZw|evm
zuSu2an+%;}8=L4bT@_LR&kPV#Q{7e>o8Z9!rLxC*1uX~FyN#oxRG#gbEWydcLOagm
zS6iG&Lkv2&xrrnXV#^|0DVOh0sbkn=*~4X66bC6q&pTz^9>MO_RnVYve&22$0FpPT^Dcn2I;s
zUI-iExb}8-O<5SJRG>M2v;F49@M$sao5Gv`9HyV?u#hn%N$I|9Y#PFAGBR<$6#n=3
zu4&V*UC)It1(P1);SsOoqJp-3C%I7EfvI=Ivky)*sF_BiMvX?M-K7JyA7>S$KAp({
zuI#&DE2rnRDjs%FubQVOp7HWlVVzBEZtEP237R~6#)?ewYQe#i%p}fF0@7k+n2AN{SaSQU{9~;MgWY?d_(g>XDV8Q}zXL+X
z?)E`3PsCt>12l~jDvhFg&<>r6B_|*pcoVhQqR@3}V?4XDxVNOpFu>f0e6pcMr+bvh
zzgTw;5GZ^paRkAPS9wHZ=eeVf;MzS}d;n*U>)spFN(Psj?=4Yq^K;u6Dh}PyG-SP#
z#sF^Jbjlr2%DuQR*#7;pYp}G~&Lo+aw$t=#H&_+7iTvsKm0QezK84~1qk{+R7b6%8
zR})bUK96=;>Q)-F<@++ngb1x(=SFcd1jT~N(oSOZ&xak4$jHsjTg7nGVG^utv?FNf
z^o(y}lr^jtvki4RDU60(vQ+DifVMNd7RL`V!e13F#Y-8(1BxoyoFvE$>MC1KmClV@
zw}J%{DKB@A+4zVr$$0B1UH8FHLur>OtEORKD=k!WOXs<>y}V=t$LZ6jXT306bOXcFI@Qd4kw5=s{h(H9w9iltq`Gm_rZZs6a5^UZ_uB>SgTgxQl5cu|c45?i
z1_bgrDaIF<<>-mI2mCyHvbJf@o>#CT!F7EnJVj2{mwR`43`ix3IIcSG?kM^ttY&4s
z+P7c;6(LzgdPKg9UZXvR&ir>EzM&*MeQoU_e}Daj-2D7&|1q5);PqG=5hQBkyVOMQ*(AAu`01>)fri?N05tQag2
zxtw<)J|VJUe#Q*j&M}$zzB6aI#UGTy2b(UA8jO{70-<$t^?=*U{6fZ_Jl>EKMEEw?
z=?Y@Uh0%MTdhO~}4SR$F6r!GC3b!r(MnnkB5#M!~8p
zdErAjzSPESoymBTu^)5pWadu;2M%O3?`QDrPImSy6scP6+T~KX_d8-8{O0xRq4fRr
z&SI}5VN$#Ra6widWoI&@=;5zPLmq^romj!{&uiocj@K79=Q(tad4Q<>0c2j;B!EjN
z)sF`axwzH;p(!%OyLnmHzSr{QKV2^s
zVd*U_2>>yY^Ej)nw*u#y%~y^zE53GXd(Gyxif>$$n3o9jAv#cfY^
zn8}(T!ik=$1Y;;dvI~yxJ3xCA?c+b9Jm;3fDibaO9Zu!T%k^<{+0H5kl)T!K5wWaS
zY-v?p7uK?ScOho;tfAvUUZu6K-PfuZl=3fE3hXy9=YwrfObj2HN?i$4r}TFwv5hrX
z9wH?7U)q(7B7;*!wQHN8(Qs+n0(m%w~qxafxt+SS&Rz=6vk-Ms%1<#(^+Q5yLbqi&mxAP|HCJO
zPuHsI3@O)k)vEq5V8hlAZ=8t{GM$@jIR1;t{>}3nEk-zOChn!c_QLr^SJhc()aiD;
zW@Sn=Mg%K{kA{XpuxDl9D@P9L0m5=|ej!ua&jR9LFmvH&p^x-wuljEP$j4@>Oy>3L
z@q^P^f^U)exNg^|SDLrit`!-U&}I2hU9gP)n-9_+vj_rK8kU&OS|?BeJM1SQTO%)608?cyh|b$nC-O6CiVN&5fV~i>&Z|NF*Rf0>WV8ZNJZ@_
z&3O`6!C@Faol(-aRz~|K&R>M)h75YI%_*Wy+{&?}^Js8pE3!>*M%}oAZ?-zPqrOd0
z20808*B*Y|B5W}J2JOQAq({X2GGES8jk94OE)R(8AzR1@kF&=%5tRdsT86Ap|E|Ci
zSoy0mE`Z-}iaumodJ&PP$AhGI2SJ9T0Y+uXiET^Ru
zF<3Hspd&6h@7}+E_32Z0!gd|@m&qJ<8N6Yiq${_1g%9U<2;C2f&R#1fPf|auLwNv*
z;<+UEaT(w@)tkaAjZ!X@GYb^VcQeCte^9H`r{r1~8Z_0ig
zK9V<~T0!vm;hp>fxX}Ik*VSnl`W*kogAjrWz56ULuUx=x3Zh{w0b{9M=Ya#;WAZI9
z1!Fw-HtYRhF0hu(0iqJ%#|*fr6#w*#W~Qb$)0!e&8qY#UkeOH>pSM%vFB>jZS32O&
zE;D^RveknOd$F+V&R;Brc{^{sdNbF+Y-*O$U8MyOrA2qa{FRb>9WXy9`wnb6Zq1I}
zyNwPiJ1bpW@ns!P=qrOMep@i^MX(Ejr%DYoiGcTMlF0=P`D`q%r7>zI`&`fkO#WkG
z$%+iBQ(I~`X3B4Yp7p77=wB*j=t56IrFRci>XQ`D?QeeO&FL-qnT8jI2YuJ^m0~_9
z15Qf2OEXp5x4%OendzILnl3)G3@XQ+oU(RAjy`dE!F>W5MdnFF3ob4$oU%gy{rfxe
z!e&4c5PL#uQ|R3GoJ|?mN`@ep$5P-h15)kKp*!C+pmz25Z{MnzK_Z5JczRkLH@x4b
zJZ4B#bh1^HR7qJ@pX*bfwQD_Cw#YQ2E_?AHzu)1g0Jag*Wc8|oNH=!weAF$iQ{RPy
zzxqICHegpLzC!{W@b0y6uil|=14tCZQFR#tp
zw{H*7mM^)0L?sK|h`P!{gVIn4+Y>p*vmFR>BZB)4)LBsS*7H$828s8@pFZ8?OxoJo
z?syPjsLOF+V>oUlql9UIy@3L6Zo?ZhklB3SL1$(?RP^)nV;9hDh!9AF>ebFYSbLsV
zip9n-mP)*1yrMv_@B8V~nn~)5s4SxV@Cv??ndy)84&$C(2s#vy)jM3ePzx!9iW03;
zD5Xal3J-A~p&O2{dun|!#H?5^)?d~$2s;Ddig;mfXjQV77MrXmT_^VZvfF^W#+|%|
z%_=Jw$#}yly_g3a;IPB(8nv$Fs}543$Vh%dT63mZ%q@Fg^FH%2SaT09joIbr)6+{=
z+;Oj2y4z&>^w(=H)||L_Klywrc)*5rQVeL6K)^ZzA1$q
zMr0HlZHSFDgF7<}zT9IVp=<_(APNR*cfF?Cl_6)&^r4@f#u7NLMFHM;#dUGQco1;}+~Q7Ze-5b@y^}ag1GkwLD%fo&0$?jolcZlvYHn(mOJfJx*Ci^VN|qbO|~CRX}^wP
zosu6b7Ga>p)Dh`zy*0}^v}n>Kx4PsSH+BWg1`n&T+SY|2?&cnMv}iqtO`8^@q$B6l
zlQM>in3UO^bGrO0q&}!Or(nmS3{kA^my`!(8Z=W#Se7hW5Xu*um1sMjc}+mgo-Q+N
zkQIa%SGxU7om-&p&!mM_DmFCJJQ|ozTWGLnuO5^^@KW}d9*x`#Q`&!!=d^9xw|jCn
z`<*-Xb}Y@GZ(FLqx*BvLwplEBIv(I)7hT
zTje?ff;;@DX(rTEfjGn|i+YhkenYBz`e&Jb6ez=OO?
zZtC6gP))?<%}qe^?xy+8hBHOMN?#|M#k-#-om+D2>*9p`IXuw~WL!o%J1r&$`O+a?
zrJfVVG_=Pz9zJxo5`5tK9z-t?3uSn5=W)xXaQKeG4xs~yBk}-m0F2!t#mUD7d?r#%
zpEhkB4$AR{*sCWi8C-2AoI(0b$ai`-K7oJx3LBh*=es&SflDnc>d4Scv
zfA$0!n*Gn^FQfK%HSJTNfJO$iBJAx{od_6S}?OCNsttr+RiX~a=;SN;B-y{G%t
zBIJ&5Z~ye6@~1G6*)Qv0ZBKfyc6}MEcR`6S*Ot0n5HjwgeP=5TXE!%Dsi@w)3xa6M
z9Bd+6<7t|_**nc!vT<{33wFvVs11@YPQl63r^71(ss@2-{*MkrW&hqPqlyz@!|5aA
z-v7F|LBobIE=MaT-f5y;dwy3r+yKZ92l)OuAEZV12F+Z>BIyVWUvjXBoZU<_W|vdf
z2pC(e1Zg@Lm=$i2>~Wk3*Oj6vG%9Kn7-4+Gb4*MT;@GZ;8PkrWB5eT%QodEi8w1ao
zPyj+Rg1~fl;6L{{sv@xl^va^@E}WsQ@`QOc>s?-*FTZpM9ngR1;(mtQr#CclR@_v{
z_i%=e{`t21){*b>I?()5R_cHkkf3#5Bew_1X
zOl|lXR6T9@ngA5+0JuyDsCu7pLJr2&t4cinkq+sMLIMozFJLkw-j_hq6OT;q55CCe
z_j&t!D9MXu6vfga!=fUq3qzS?&VWZFxX_!as$8kG*Kgm-lJO_0UXjLLI@Wg83XdNi
zBv?sqCW?n4tvh5W&VcmUpaL&He%!(b<84zw-&|qg{Ax_1TLd@i6k9Iwjoe+H68$#5
zF|*d|6Y6T&@{EaAEaiH#??Kh_UBqllOUpZZEVk3UyMUo957TJeuSvJdl(|F6u(*J#zhJWY`xn=pe
z8mcXp@iO!ANhr#I&q|EZe1t|Ks(?$PKvvAA8N*>yfcwzmuE-
z4X**kVk3y-+%^yq1y$f)el!@f1;8VEO=ZTlsA$lF#BhDEi*QCerR=29gjSU3xe>70
z)h#T4PH)*z*r(Wq0vcPi#GM8d=u>E@d@C+DLL+pmosqS6TO>X3l4j7Q2qpAy2l^al
z!XYLFzI8P}*$X#GQO#-=HI)=YAXxQ~^eQu-60HcwN)mZ2h_|1lV-JQsR*-F_um>A-=FxZW-d!F#<&spvv@JK-tFS78?ts}B
z$j}0jX26tM(gwWE`q8+Nv_A7Wc+E9c4
zCvv~LejGk%oOkPxUEG&8lF7^6Yl_;|4|
zi@7W;9UkXWdi#Mf9n1ew^&U_?|L^~QGArJZ$WAE=Wo1i5sE}+$wrr`4NW)AaQ7DwM
zM=~RlqLf)e8L6Z)id2%6`rWR4zW;On-*Z0ad+?^$>-l_)>$sxhkb>YH2S8fCZ
z2S4MycO23g=$}PGOs!~2=_yZun3rr&#C~v_r)Vu}v8js;l+9N(k
zJ}YH^p4HZ-^=Y-$_z6Q9!KlEoB9v-5X~_AukiL`!ZP8JCc2eV6C!a7Cm%xu)dfEfzeim&-Ng+J|{bBdnF+%K#
z5t?#v3qy7#FarN#7**h|HhsNGgp)CX(oic|MfU;?gmh*l|4G)G288nP;0lV6>iHUu
z%q9ZQcFI@_^XmT1F?iN7O&qjfUypERHrxBYO$=?2Yh__mQ=5C1tvNIpVpKowk@l&UHkU4zua?U>(Eq1e7R@G^gnS$(+;0S8pY6@VA5+qItYC(Ti#sxU2Nr&A%I`IkQ=56R-o6cJuY!waaxydRbvq
zGm62joE%RAI*zpNb_gDx_6<=0(7GmFfRdTqH>dH&QzKqk@K8-gi}h)#qUC)Ajwt8r
zS1QZ%ryRU}d_Hm}Q7LZ&9FfC_J;6|!K9O?wiH>bipkbbnCe$R%98?2YPJV84G$0@W
z=}Ad=qng117TZYk>DK(NuC8t|SkJT$5#iyxC!UQ5>_ZXw_;8nz&GP8b#nm9GCB_>g
z|LxWOOfp1wG@Y1s5D~v&Tbr5%Axh`M5G^Sctb+Ua-zo?b?B!`8Bl+5#iN_r{GKOKnMyQROo68Yvrz-3{)@?1>V
zfs#9j=2%M{ZsLlmrQOO!rU
z8A?#%B8_~^vivvskKaBI^TJvDxhFFiIjN;mGHQPOeU#Gml`yv-3;!{EGLMdPL;{AF(~PS3MhiVuGF+DLD#*
z*0>SHRC=;e1kK@(Pg@;mGhM#T2lU2AU>ZL4czF0z@HwOVU3nPfP4$q%hl+;rjaL7*
zs!VwUXSt6e9g24q24$=gc{_sN!tly$tG9V5qyrZ}Zsj4F$qIGQ?${G$0`C{uT>&
zCRb2iMU#RR5R#j_05EclxjEyHk8*71|6a#!MN`QiZ>EoerLW-2W-ira;H|!GWOpum
z5toDJ=HC3971Qmi&s@^65O?LrAe^Ek6!@lVS7i@s%Dm|UO||C;B*NyD?$8@quej1q
zqB9KI^eR1l^x4T1BbU!$Z!t+T0U&i!EZQ-a)-Ws+J&lZ>6i;fir@xkRDl~;k5$DcJ!8Pc-4
zU1CM~^&8+GXE+#wOK(E^phrd?pQowm^z*ZIMn;BF<^JH{@nF&lRCiq{fT>WPCT!zn
z@jm+2cmMS<#Ke94(xvA*zu+B>Bdpz|%g~|0@-o4YfRnpv)V&hGAS5v0ah@w#8Tzj|
zTYN8H?!~7#ErW)7%(g4-lyJMLJJEzgxlrX&owyY_OTs&tIjNQb7Tpl8BcV17#f+IH
zy(JQJ$p0Vq{eRuOIRG42?4lHTRE>OF5>bF|A?Aa2Zo-r7X1LvqRWM)fMj
zDd^TONAE@=7De-KipB*+SU%_&rsu6EPUrx)kC(;Ng+-HC^tGTs@F9pPg7+kZ0I4yL
zlnJ-biucdTURa}^-?*&8+N18OYuBc89L$dTz4!h5XueA`9ZE_IA4)$z*{oI1}Z0I*SPBQg)MVv
zxw$f13uu8eyfpk(FEqu#yCpw9jDXvY=g&n+Bavd}Q$3vbNWQB|a|P;g%kW}_R%qzfR8+PimeEMg?1yQjm$
z$5Gq#>eJ^qkBAA+OO)5}PmvWQF}VlUSYKH4CAB_fEG2LPiip~7XTA8<6A>R!8a}6h
zR~p$!%keCd3-E{ji5K2M-pN-51uUf!OkbUzeD`;u;IL#Q!W>9Vo2Vut?eYh94v#IK_SH5g=2);VG@O4RMT!ACL7&Zg3-_C
zU-i9Dp|u)0o$zpwm68#GfLU-bQ2wX!E?jhy8##Bb4Ea1WGpaZLe2^wYz>;wqX1Bdk
zpU(ve68}AYam%I29cWDT9^kdWrazCq-}TRz)2oI&-;3$deGyBJyPBRJjHgRt_jeJ@
zlwQyK0j3Irc;I53zz4)aF$OG++V}$sOS0b&vO(yUoYdcNkxK>>ip60sCWidoZ`^r?
zSJhN3cu!aVS|xLRDC+TOM&^H_Bf#3bof~tRw=>WIht)1#I#VZjHp{A)0S|ckXwT)r
ziS$w^L3|4T$k=DJz&Ok#J`Ih;816$aS4WDbn(nbRHePj^BaO3m510)XTWm;ywG83~
z@W)2*FAbq3rn4_?485bN@&NtK80z!mZ6)qZql5|+HGR9OgG>(Vvs`9o1Z@`6E$Gq&
zItJ4~tKt4D**KzD1;jGDNskn5O#Fn1(LhviyxedOm(O!->Ul=ctac>UB>mBQs<
zxlrcvmMF9qXWZ^~h4GYlOG>^(Dj=t|v%wR-@w6vJx3b28!sIQC*T}DVZJQs8AQ=
zvQHvnvT;_f`icUq#mw!@>?~u>cK$E*Q)r0vq(sfSqA3SJ~T(wFQqp#B!^`g+`XAYAo?6qX~U~yK{Xpsg16x&9&
zcy$8j(7=CbJ)N)>Q{aU%$9ZGXozZX%OHxlL(L)^AZ0ogdB+V8N-Wr&Jisiqmf!%e`
zG|-VqE)++l4BlZwvIp8rU?y4&pz=lS*J)*rM6wi!bOmQB;$O?!%I*f&W+J;lfO?Qf
zfNof~M#b*HR5B|X)J*Cq^ovp_AdHpPoKJ#sc1qgtmSK&!y#9gXX97oxQ=649;y954
zgBMwxRHHn@;$e);c55v(TOB~Qm_&vlCaApt%`(L-KshD`EQ*6po{h9Du5-8U(KLAm
zGkg>70;pA{KT&Z=;0{N@0!2M7??9o^DA>h)kP+qNLdFYv$Pc_)0q7(mm$OwK01XW$
zsNNWOWjM2NE?hH?A4W;r5{&2h;XYj^GCH7ZAl^=k+t(rhZjlIG%f*ZJ$a!H<+K-1_
zLIkPd$?IBbD_JZ0lyG^7_W_bx3ONpAD&A5Q6mo#FY>^O8Q~%}qe_DVRtZcPIDM{7i
z8HF=XBxj=CjW(vl5@w(BTV#zh|F$soSEi1noHLeiEY^EUn3;;GpF|K(`BM*r12-IH
z>I3X2NDO5F3Y6~aTWh+Z6OAQ!f&1h+)L+-hXF7TYI5OKz^bdd-=FJSN_fMXjV`6}B
zl>$ZNJW3-PS#ck&BDVYILGBr3jOe?q(u<>PZgZDXWn2
z78c$hd3x47KUVmB(mN&Yl$+d%SbvywnMubza`M(^gPq$Bsh4O+$T|`#mSNA2sW7F*
z;>MTWnIqhC*vZK(DbU9G;_$7NOugqkboo`-n&LG8NJe)QB2o&?0oG4?<40|x|YK)N6N?`c^^1y`%V0ZQ>msGd{%9gaD*oHJ8TU72RE;*rz0jHu?@Og(*kEMqz}H2W2AV<`cdB6(tSCgE@m{!~6*
zy@w#6q`zTaP#RIyDX{&l=w9@<5?%ut)@q<*ZnxQIOdDCh0v54>
zID~^g#uTO_sI6JIww-!?YG=vd1&Bkf(HIHQg0txpMOPxZ3bi~-&UJiyxe|UOEZz-4heSNqzfsvB1w#wg&0$E3;L#X{?EzWL
zyw*VUjJA1Kh~<${>O53Tpp8-RNF~#{bznhxb%^&GY9tZw!0PA!!mKF~?jYAxgl_hU
z!>vWD4tq)kj8R*c$PTNaTP>}dRli$xm!Nfn+5@24HB`V*u(#&z+I8oo_ukD(4}K+c
zE|6^QsaLWcwU8w@4NdCP7o`BKiZCf7IU@9#Cj>zamd9_Lh#Ip`Bg?JQ0Ig*h)iK?Q
zD18)Q{FPlzYPy~7|NaK$;$?JZlmRI(UL2LabY={IKIh3k+p6MPEi=w}+`_2yZn0EVCf5@?{%*w*=
z`S2*uGY_T%SrZgZ9j
zC;||{FW}U+IWEu5zf<#16ew;Mt^r5%Uq~NhRx*dC`)L*mkfi}6i{q}x8oIWe9$b9w
zAPx@p6j{7GZ&ECJ6iglGAK^iolJjW!Q_bfM!7r1~rDv{MytvViNA|z|XJI^OT#p!*
zl7P#vD+X>ij;ORdRN1jcJuDLAZvO;?SCFey^TEk@Vtc*l;vr!EeYzUyKPXFQT){ijT@D61
zAj6qAC*W}Sb?6;E+nq?==V$>G05Yj7Mdgh@3m`3GSY-J+LVOgCfuD%1D?TxCKUF22
z8Y-8z{0B1jX)7xjhXr3E?1vxV^_Qo%sjdGb#WH`+10s2;potmsn7G(CjO>
zkNz)eApd5#UWbN76TvGvN$LB|hwiczMwF5zA#}tJoyNt=+ivODtyC|10@GPycP^}qkz7FOta
zAYm)PKajtE`LC>}S*spD{bg5P$?`x-S5;8(JL1K@bmOAC-wF!8S4M0~e(F{|ZG6<5
z{*(`>gvuU$-0-VV2_DmW3_Q2s!e{WPCJ#vH+J+-Qm4kltq
zXOlf!a-=pT{#$y=oc#RGn(-A$@8ADD>U#HXI5G-Jxur4Rb>P6YO)b+)Z*T56dP~xN
zu7c;7PDC)0yJ1h$urqDJUO9Kk+lq-Fw^!zu;d&Cr2|PR^Ba4<>b~`5szZUh}r%#Do
ztypIxt`eV01~*o(6BtHd0#V7aB>&?T%eyhtK7a466=rdQ2+VOL5_KpQIls{lv&!jD
zeNoMl1kHQ-5BhEbEuaWi=Br(=DoLMO>Ap0sM_1OOLv@vlRJ@cEClz{ZL*f3EHla2M
ztFd3H;^u3V2@IDELrYFsV7{n*OS94~<8AIG%w~1!M^dwZF+|PTiUhP9^tmjvtuA-$p
z%G6X5-6IY4C5RAy{A(yy)ahz_Vu`Ne6VgcO|GTY7uRcXlQxTK96OWz_eA?4`3ju}b
zSQ^s7Bi^)}IrB(8ovrJ)*@WlAy0g($32K3=`AnccWhq;1XzJgYi@~{?0f^tbe;<7y
zyu3TzID$w){7?ac{J@paSW}1bmlF9LXj}LV8KTB$0R#S(fW2Gw@@TQed;s_*OCL(K
zH^dNCmN~5*^54CC8CS|QL4{q4(
z*&*0aKn3DN6DYQ_Uxk77bVuq#Qa^?{lweIr6^XQ_RblZyp+u*J3cu#+Q`7w(RiqD>
za?g4SgVa2eR`H1z(FsT;PgNqXW1czvDtjjN(%ccCb>bG_E|-`(sA@^{1LGZw0*4`2
zo?~NlMky7!BjRB}+K($BF!oI)w;si*-}9yuDa2YGksJ2q%MKSo}eEZ7D9q
zpNX1qf$NQ={;V9hLg}(0kh*$GUlo^X>q`X
zb5SBwUCYckPUDlHGqM8+v8khM?SW`qupBaRrNxEdY1*>ocHD(9u;NyvK?EqHMfZyx
z=_zD1VLMmriQc*XV`xW&nwvqKN}&zruUsi$xy3KdDI-S6K~ho@#;c3nva!)@-Zm|6
zDY-&Dsa8n(1)Hp$426Y{?r|a#HKdK(0d?!?$6I>&_;f;|k8Z~SU7xi62n7FS{PJ03
z0vl!S1A-kjr=uX_XQL^=^JmTq1Pl$MMwQ_UB3fo0j@9xY{Fh563jRbQR{2VR
z!_8Z?s6XWMAC4JIPCePn5-mv$oLV|EdMYauM$d^a0kYc3lQVK;+-e7hIk-?|DlodK
zY-|`jw4Xs7ofzl=67z?ie(?oju`?x`FrOF&#cGB4l;jVtHVQ*a;6cZOI})Ox!qqjx
z;veFyV`~av29t!PS`2l3M!FzP1;0CL@D(h!0MI_*T{zXM2T#3PW%Z&D!v
zdk*6}EZp3;*oO7t2mUC25B+<1Zo;w&T{z0c#JteivZpkB0H!q275vj#f?%%xUJ6x6
zJ4e+|x1P=ECDCv!xV+5s8LwZr;=}^D9|?z{H#?@PlMmC|>vi{T2X=@21~_uVf+&m%
z%#8y=Yjbq`4%Q&g5r=w9@B!%#U|(pQiCY|$T)q}nIsfzmdwVB%PI*As=i@lq7#^J$
zej_uCI29nsX`RQaZyN>IMk)wjx=Za%@?qvop1eOtr`WvEz6#0|Us~nY&nTXt+8e)r
z-=GhYuK^JGKSUem$?m!jw&W&5@DcvLCv`iD!IV1CAbjDzaq4
z?jV9VHg+(ykjp}z3a^R$;k@AEpec}Z4R{%nb3}2>U&blO*;<{t?Fy>_Md)ct3Gs=^
zG608>Z%MHTU-3!4;XeRAX}~P&YzegEh}W&XX4)Y(n&W_I|eQz5O;
z|3}&0@uxc(8t$RcK*WeXy%X*RZiWo)!N8?}wmh|im77=|I7lJI^=I8O!ty|51g_y-
z6aWZIYkGTk+q7X;`<>=L~-veYNoUV*`>9#*|r=m43MjFD-
z!{wCyAgX^PaCw}x^H|xk$JLvjoESJ*D`UUdaEdaG1xrS|Md^DZiMX~l@FZ|Ad;R>1
zlggRrl-qI=NF*p2QpCNs7v8;n%hZM4yLXTK&i^+E-(`l4NEwRfA9Hr7@;7?8s86w+
zJNC+tKz$AjnEDjp(#)SyP2pB7MMCjb-w=WQ7)
zr%fYv!W^+3P*9tct%2EyO0cmrdqEWC*8|0^DjU(F;PL8`AQjyW|uE@)1!0b>G2N7pelw7=gX6oIhbnDZ(hlhsbw`5S_OYiWkg8%e?n3mRZeb3B_tXUPMZ`TD_
zl)WzP8sH80oITv~n#D=Jc<%U12jQ%dPAGs-MQM6e-RNR!hxU>mG2O-{|9b(ow74LF
ziBnU1muIOSbg!}fZdG&F>FXBNp{TcpX0x_L|IXqP%Oq=P9MvYplG(~$y74~i2P6)k
zKK+pT2T*$Jn{eK~$lrzOzy~A(q_C|n(_h~?$FU`&UjQw}@SG$!16&D?HuTj}C)@W^
zx=va7JovzY?c-062%q|?6vs`q1U&3m(H
z#@WgFZWCnUVN{gM_Y1#&%$Zqze+4JO$j}LM7A%kn(bRl`XhL9Sq#L_Nb5x2(31EwD
zfPVHlj8tIRU`7&7U;m(ncpSeyC#rRA50;RTH3FbSOIgD7
zk=BA2?W?K0TO$+q)V!f5N0VoPbbJiu8Cdg2;)|Z#D376(VR_o4eqt}F7aYNd_M6CP
z34-O0nMVku^!xIWI7%{Fx)y&;H>3f9{gF6U0#0gEy{Yo${H4Kxlp7YY-Q4S7>4Ubb
z7;D^h_~(JGUu)a>a5l?G)%O2o>MKOX_7u8hzLX{Y=aU58jgGllVeH<{>
zBA6*omtGrxEf>0m1Rpe~`}WpT;9AilD{wO~pjWzd%5cYh_!zXfT0dU`-iSvIVo=Om
z>oQ$V$EIB;p@3QyiH(7E>ohdKR0Sr*k#mWCWlj*gaXQncDT-YSW3se04KD;($XUeO
zNn-g3Y(uhA4r)gq-9_Kcy=@
zG<4G-@Pqn8uR22yqVfSmKaeyQz1djE?chVl5zTTUN-syZJWxrcV4m&ST#zMTlz}2U
zmkNp{IFtd*LgNA{>gpJm{D|C|syj=*`E5i#z}c;hH(UeaY^r;Rk!C2_z@dZQ|1HByQ&X-1o)!Um#
z615^ASZ~TXbv|dRnxWavrV%0BF9Lhhx)ORkQq%xydZ7u)
z>FZB+nAO<@P%-phfwX60)E;Q=(~(vT4RbtaniR~GO&?|F-N
zD@ZF6`AOr`w9ld26Wc50l?)sNIUt)yg}5odm0xv_H9GkHvwZgxuxw;&i52RfS{TIK
z%O-pglpj*l&FoT1kU|zXyPzNEv=eB&goIB^6g7v-&{Zo}J`>$Kc{6%?jgF4*(q(0{
zMQjvA_#&O4nVd)C$W7%08lwN;WFrlJsGmDcm5d;gyaB`nwRLb^3VTWR<_q;gAssqS;-tX{Xyp6*flVJbTF0SRXs9SrR#k5h-dYf)n%+p8eO
z4Y|fvz0GLg0huL@_!#sZH*Gz#P_(b_W<#YaSJxBzNE`w@sNUALy(Zz}lEa#_h6IyE
zsD~t-YVbGrZbkp61t5%SK!Qico@w&UE^xsLQ7C_hA6vV6O^G?vM6vdm5={(RX!;(!
zetk1(g*_En=I(~>&1AS3odh;q^W>}ZHiR3o{sn5LucfpQ#FT1L>9ki@<@?UEQ-_r1
zWJ?Ff$&4*j6er#uT;QohY5;_Z6Pt}^xbGe0Dox~Czhy^zNSzwU8^-(FduVj4^C)4s
zfx}=v#YD&6DP=1>ZrII??xq<>`M(ok7Vs1=S1KD0EcrI;cT*jU=mGMYB1ld|tbN{v
zzaAtc_#HiJ@G&kVj>C`_e1!~LK|}Y>dxV1=%;;RGTPBm_aem)#Zs2@sh35G8>GZm!1g1#Ry5N+z%4DHNKey)qi=f&qhxe0nF+`EqBtElyCLkT-VIM
z7)EL+Wwy-iK%^2eefdUO1UeV-1A4mtC!r%q@4;v+TIqUFSmM^>$?U*DC3h%y`U#&6
z&Bzea(-N@e`RNII5*`DV@;_6Am#IX~!6Hdf#>*9n{ouj3=Ux3Pq?uw>eM;yTWE073
zIx0xry=r0b9LQh1`G1$WEJ34j%m>V|`&cwZ&Z~q`+-E9I5t;m-^Gk~gyu9!mvgygIY*f#dT
z7Kv%QET2id!Vvt1<9VYC6^
z|*isJy1QEvg|Au-4(aL8>Gg#jU
zMd{grBi|Rx0G0#-ABgX2s;wyn0oC0;#GVv`j8fTjIqNhL1xu9N&AGI@fKIT#c;)_h
zs;F(|Q9ue>223Hqiqa{Mlc!H>+zvIC<4r;G*AeB1QfbirVmoeBybpQ4WHWJriqe4h
zh^WZoNlXh}+E_6(*Z=jWYBG8bskv#vW_MpT)toa4g0L@u!~ZfQt#dc{
ztFHq$6_If5TKeTXGp2FTewkY7
zO+3#?8=IgfabcrH4MR~XK4}OLZE6KJrc!53ABQDM>8rM@dFm-i0l=#!i_4zZ%!R9D
z)<~(#WC+=rNKH*XN}7APPjH@nB=wrHS~~sU+mbQmS=u`F8(e
zG6`q%&`Ahj$x+Sdc~uW42vMs_!-9wY*o!Y@VJTw)4Fmf3jXubqa!(eLn~co?@xgDB
zZz{xwHS^Q`v>&L^98lCq%{aIyx-@B-YP$iVM_^W%8prmdO>Tf~5|j%Bz5nP@fs3Tm
z;_W8)U2>E)%5(n669~gH;1ZLc_5NPXJF}_}ku~dP2SuymtnAK+?8IURNWl_F}^3Y3V1KQ-fM6ubyu6#yia
z{?GYEa)C+^6q;ZVVcsBwN%0fKlDW;IQL`n+{DNgt$Zc6N=;$6wLBR5R{8gB#5w)Y(
zB)>w&R?|l(2h8WiG*VId$o;*|t#Rq2%7wZ19sa;cZ^Nbg1FKJObc!tvXHsPyn8Ps?
zG0f(k*-q)prT)YVD)sc4so_BanGJ*97tBO93Z(`oX2H`Z(Et?ivkWUNjn>_Z%nhbH
z_T4*?nImDA@FMEW`2Y+qZW9<`*z;nBaWqS`6JV$a`dBA&8Pmr7ebNrK1We03)+(B?vda;g=!ni5ypPb1h@QP@
zQ$^s8%odFan1u_a)D{9=MnrI2;Ga@^W;tg3_){=FYFj((Kj2^;CeC&Ck;H@s4ufO%
zKmaCHAeDR|qo8@cDd%lJ3MOAGxm_MSyNG5KoTCp1=7;y|vJx;QaYxh{a=C3KXFnYF
z_?5#O)hyfAf%XldjC@u0Jv0tWc0-}DoN~-_-$~WNiy-Yu(}Nn-WR1HF{wgjW4oC3Y
zoy*-_RnJCf_JyORFh@=_9H|Kk3jeJoYM7=)4A}AKA=xix47H``0N#j$-*oD60zf6}
zv-6;sn(B?j9GbKJTN!U~Ub*^|6@`fccz*1|hhO|6zJcg+x;YP;6`WOhV#X_ix+-2f
zk?+tTn{L_d;=NPJ#`2Y9HC{Cxyxk@5Z|{=9s+;l%6eswWHlrWtgvc%d(}pAm
zch+9TW1F}YCUoROnRO!qF2DlGZ$ts$$395uM#m`LA^6wY$n7;>Puyre6`68Qc6Kc9
zvh?-HWxRyZOcg}uLX*FPOa)4eGq1j87`}HQQD9Tvw+#WAjaJ<$YYD_D9(x!L1vH$S
zIaY|@^Ho0>J1NonZqjQzUUufD@>yWKdn3$ljymr0HNCV`(dMKPA%Q_b5=|omuF(f!
zdr}q`D0c39E)Ae32t0u@P}@$u?BAW0vnbZ14nF^*^!C|!>Vz~i84S3%1Br061V)*
zg|zD#ON`n`yF;puoG`#J{)$6J)Af}cc~v>LyowtF*l;W;Gq?OgV51(7apAdYPHfxFu}>3*oQn
z>4o;izkV^USwDH>c9SRKW^swsn|5xpk_Uf&fMS7Ge*pciW!~Q*|GuMeg)^|gA~--h
z0qLmYe;VSpF5skb7lRLB@xA-?TLdvHMgR(=7D;A!O4RG)M1s}Z#?vLsU;Dm8HOFQf
zfnUv;KL)KpvUPH;>6
zQ}QQU`)+AdRmyi+OHyS3o3tX+m8gKdbAo?V{+YYT)E)3u?>W|#Cv5A-z_SBMrU%f|
zTTNhExMIDzO4^X}ys5qIRIwej$lc-Di~CE7$}Q
zd()+~v#_YS72bJU#Q|dZWsRW?diIGXPg*aydhW-4eanv-9OG(AG
zUlxxYi_|CAyQ42Qbzvcj+l2J-oh)fa<9WCqjyzfPBp0gJ}*e1#uTe&Y5|+l)PfE1?6M{!@C!qe%r&15-
zSu&i0Qbu?U95~}%QC7c@+P6Ym_&xv(unqUFN
zhJ;13S0o^axoKlo-u%mu0>qGi{800J!iL3A9-1-QfbFg842$|?#ISlx0IX;iG9VcNZGDEH(Hz8
zhYOn7jlj0SkLdbjz8mZ;ClhX2Kkb>`4*K_W6+^AqZK#Dh(NS{*$TP}akrg8Y2Px#G
zZ{{jcdt`YfGV&D610zbUhk88~J3DZ}5HNBLh%75OR14hXrKaxZCz_7M6#X&goj
zL1WNdqGqv?Qhe>-yZ6|-)*RwtFTS8bh(01sOQ^dD-w~0u9^lXsLIMCSY>ecuQ0&O7
zV@*p!`J5ZSpZDjo6Y)J@;5LK~O1UlInC8%d@!z26oXPH#cL888S-r?b?!2v$rntIL
z-g=59=))<_H|N6F)~jxtDQS7P44?X##VS9UYJ34t1|!F$%@6HETuaPXobemi+PL*6#Ngu%(7#|Frr3#1I$Wijh$R3V6JL20;IQdc((M#
zlK6&1AJ9q+DLkf}>!L#?C{8?2wDA(p40oz?NO3ZH9^o~JIKeq`aF|_aBVRACy|z|6
zH5T!>IL)6gHrn7+d4E73>zG~pbEJe2
z^xD0q+?{ij=zK3l4NNyO0Rx2uD35D{!b0Vw6_`slH0YaKLR
z2c=q~8xUp@6lwAGw`x%#^gA4`LeT+;x!aO=Kn)|7O1_JHr8#J4MgpEOx8@iJysjB9Jm
z*1(>5!(`L{j^GJN{1Ww)Df1Y(T-Pte?{SLG5zy!o;~UUaYKN-#q5~TnJS32JsAg<<
zz2oFF6Y8*=C|^6x^_rH({U@=%uv8W=x4LuVhoO6M^pGVmxZG9fsVo%wmEk@q!3u+RVwVQK_ZPIso3B7;e*d;7xf)!4I&S@zI@r
zWDf_t*7F2-@quomXqDCv?GE>_8P}7}p?S2sJA@4bTIM%(J%{X|q8ej)Gie&4Bq`3Y
zpY&Xn89nb#4@xV#a`Sc0f>tFTN4WX0yVtRa+>
zM6HiSMair~;HsaK7c9t~bi2#!n+~2}KWH+aak2>eYH_kE(BPR!s0k2H!h(pBZ(MN1
z&OLh$tSk_Z=gp1f7mpZeXHf^HFfU#%9A3cXj2+YyL?G|x-fTCa_RB4!ffU}TxAozs
z8!V1LmqkcP@&Gtavm$8fc^Pvp;p0F@EZ^ar)wc_!0Y}|xr33|OSdDarn$NP+IO1m+KuH<&Ksu8%3t4%dn4+7GnoKl
z)Mhu0S_eT+NuLJf5;g&umI5`IFn}`Y384VFM3*`GV=WG$W^CT7Rg!4nNX>98+Hr@_
zN19t0L)o;{?*%JH!QCmq%&L!#+tBp=9DRW{r%(FteO=mW(Y5di$#xpHbetw
z6a)=MKE)|$i{xaG`9y4h%dhh}-T7mX%e{5=Wc@AVOYdQSNLDbKV=d1{)|!7GWNh`Qh6=WgP>XvH1(c`3fanF?S4
z?b5ZM*-e~g5FCy_Y8iTOd`v%{kJfz7CYo1jexYejf+$O+G1=)f43?I63?#Sz(oYc_
z$bGx?`k>^9Wkzj4b5KA;DWWz
z)_OFxqfw#N@phi?HU9KRt}F2HN5h+Q9r;Pn5E_r!vfNw0*rqpE?1D3AX7uslhdc$x
zkc3;EYifpwA3#x!(~CB6VzX&)Vs5uvxM2SL^_QjL0Ke<@w=BJj-AklHWEnekc`NrRINIK9i$C@EPkadP
zy`HniDRFI^Hf@HMZ6wl`5Vh39yAWC#ynk*MT0QpOmN$tEDw(8nqnJb#A`~WZlnk7c
z(+t?TX~ZeA;6Rn@px*=eM@DjEtM<=*qmC2E?gs)a?xL|{Mb!a(48z9#-jPt5UAr2m
zGF~EOsnLnkr}ATVOg*n_VBCIwr%nV@IfnG_)i*)!Qiiw@b^NXP}z5C3v}=JN|s
zV4MZ$3fr`8dw6u_M;Czqo+L{8TjX^L&ql<;mXru*hS$+uV1-IDjw?f$DBg6@?11c3
zHOl-$@J{XCz12_~f>3>TIsSaH)GR_D;JwF^9HE!vsrNJgyF2*%?cYB_Q*V#38_wSpfiH8map$Ge!
zJ7o9Ro_%|5s&bj%{K+oOv1=F4o$HCBBPFHhI>nKqHB`L^>F?;f8=WiPII5<9pFWQ_
zEnm9y&&w6f5^U*p$nfhe{UH>g+0u8?T?4m=RCpexuxeDSCdDOpjTq$#j{P?#PTd~?<
zb?7c16%n!GE(4-Pnxq*uAcXO*E7!faN|`*)
z*>()nqBj9~#w;={e5xp`Bzhw0+sXYc%SM`r+8_&p>TS8ig2edru_dTvO&1${L;Uk)<
zUG&ScANAg*K1MOPUr?tON$2+u8}uXFsZL`x$4FyaPwl?;3HlywQat+Q#VI>~8FcN9
zmFA$Z33@#Z7J6Ua*$Zf5`hhF)@vrMyI$2&YxN!N>rPzO&f3T1NmA|umm@vVAR8n_B
zJe_0^ZLFx0ur5zO5&4bXBn_?O-dd50KO46>QPC_)!C*7`Pgl2WvwJ)Ep~NFozTJOX
zkiIi|^%k!)dRH75ZRt0)>+1eJk8Pf6xng$%jo3O44ca-iw2!qpUHAH}IsGP|-*l;J
zbY}-uuLkoxH@Us5O`#f@iV_zS_LRA-$
z^?2+a>p`80f9@=6HRu1d06$&7)>~nUWOE{6EOgeSnsp$QvkPq?vNJM*TS=#OezHz4
zeZQan&b=q~DRtGXt5VlL&Eis>qr3kL-`#SBi^25bIlUT=b@HBj^+;mmz?%M#^@GRN
z@i~&-)?<6l;MosVR&8{A*2DAVk$J{_KGmE(_Bhqg&wu!+4Gk8UE}z$Z!I0k_%=Rg6
zscfKEZ9PnT@&0Rz_ZjU?tlGG}^Y-b}XUti+V%yQ{?SJ+A1?l$5P=eB4zaCPOaew8D
zusNtTr6Wf2C(!`?u(Dy@hoYJ0tL$WNKjZ+g|I>@$L@dN-{YD
z65&P9YZy9o(pC%@wI$SaxXXwU=av|`hP|5D`iz2q!!b}T<4kVKQ@snKmK_q)g4ns
zP)G=^h{yShc2&mna^`#a`gAeq)nvC#DIie9)}UiOCmgd(Uw!;%lh6P8Khr<3!clqg
z;UfucjJ5~On`Lx&t*Xfd2iME1UWASq*Fm#Y=Q=wFT0Wbc}3L&R#SsxX)(j3>{#+I;WmZ_|!$Cr0ud!0rHhzk-2p}DR{!8e)ebohG@?^IO)jjI)(`k
z9CK2%+=e=CP3YR9frV}hQo@#!pF+n=xy|TVgDz8hk#k&@l;TYzq_(Ml(Cy-NJAHgR
z=^K9W`<~h0Km)y50iS0&pXqk)!-`{#=Y(7x@VDslgUQ~QdIs6+mn+tfZt1nr+1$sx
zlR@X*X9ipA9rtfFv*$34<0Cc|&CWWl&~0$H!oqJ3UB8WNomkW4u**#mOZ0N_26LQn
zd+iFs6=k-hUF9$IboHpVs0DJfYjLa}ci*I?NqR1R$sdg410ZVf6+U;+)3H#noXGJej9@>owNt2
zwMcl`e(;1Z&g*a9p7c1e%MHDQ2TH%-2KF*mdV2();!3|U1OJD)o{q6&pCz;ZN0YV4t}UJ
zx{Fh{uPwLNvUjU4RW44mUG*?o9Oqv5az1>BV=pbHDnv7OpmNR
zarfrILt_w@*_?UR`1zN`4F-I9Zm_?^vh}eQ&oj2;!rk6Er%^+vr0k{G
zC|JmJU`EVED
zJPriVm@#@`vDL2;iIygQf8TDf>mh%pL4a;g4nJj_t#7t2rcYg)-0qVmcl}=X?%B!9
zO;&GqN%Za0{LHL7gT`fTItXC#Dbvn8IvSJ!0b&9Lcq3KSE>$58eBUv9tQrhkaxOl*
zN!^!rOPzN+t3H@Axk0AF^#JuZVXbDLH7N|6TtbT1_=uTDdMBqG4(nJG{d#
ztA=B{k9An>6nFn=U6w7l33U7DorReaNG$m!V!O5R`9aH0+o1C?uD1914|UC4^fNM*
zVA`~F?dQf8{_W7li_Xf#t>);V-|I8`87&C188>Wu|mV}
z(&rqrmOZ8q+rKU|bm94_u)ia=pB}Pf5)~+A$A^m>ThL2sJH+%lsXi(z|O(wjY
zaO~HVE$-t(XSKfe-R7D__|`5zeR^dFUah(@%(tPbR>9Ji{qL_VNzQX0aPwowbMI7#
z9=g8zTIj@W7LzYd4(y+~UcMUHNIrj;e}`7cIl3-WOGl?(j~)@}SDL`x$1w*-a+n5o
zmEL9z;-HRLDj6AIbi?OX(+F*Rb_Q9S63U){UsvBNgwtYSwf7iq_}Tio#4i!Z;Y
zp8i-_xIC?@qRWg^c1p1m(?t&+HbYk~FiV;+I!v3io?BP)S$ZX%o4%b$;4d`P!
z$#zTHY?W7@Hpk9({CH*Cdc%{R4F;;zwDEP*9H6@MliEP1>c$$+W(~VDZnvTBj?00)
zf~O2QJ1e{&4#~K8?{2@oeyIPF)UABEEi8qY$nV+WQcF+QpoAT$TKkp^Ie2v`YN^Mi
z+oY{F4E55WR~1Rpn=5ncYJK=%)VF8$w4-|tEbUv;CFb*$1{><$dA0PSV(Gkjzt?6C
zNxL|6XI7Vx6mT)D8D(Afa~~Pbj(I}+Ghkg_yYvwe$Dd7|KRK<@nofOh{dcS3i?sB5
z0RfFqHr390qd3l~<*;qalVe*X4GnpEviZSAe+R3Vb?!T8_kUg1by+$k+^la=Z5S*b
z!WLiB90^<@ff_DA#&AaDrL7XzE#JWLqLOs}@7KUfd{YDLYdgSD?@pFR}k9ZuK->YzN
zjIYVGnw@`Q+F06Jx_}5tzfI)v<
zSy=e{OP4=Ci>z3`7V$$RErtrxsqkkn!{EDKqr&4ZUDP^f*mb=IF`Q?focP=_Y)m(^
z?~VP*@UDB&^lRMY|Gp1Rird!k`H5wEMKd~Ny^WFR8dO&rJ5|*g)+FhcuSpsR)%buc
zfG*@xeAUp}o4gR)`=`ti8Hg4D10UX;P~3`+l;u7Bj-&1?e~bSV#$E6IVX}F2OcTWo
z%dWe&Z~OT7>W_=F3m$t}ANS5{=!|jW6U6(1WKZMx(TVeO=WV?e(B}Pw_R%U`2kYP3
zrkAEZpkmy
z-Y%QJ4zGjtLNh$a*%-Ljxk0D|LOo#WQ%E5}kQfr*KE%CvAmm2&j{PQ0v$}P7w&m*{
z<@d_%ofQjn2dG%*w5mJKLrEIclIMwO)b8WoaANn2u^Myz*0jQ@cK6j$Jw|{+Xc=Ue1xEa@j2C40|yV4+X@paepFD}n}
z(|c<8%oz(J4QT6;Y@~ds8Ps$@-@Ox(C%jhqI;QWF`d=RxZ``GAuYV?8*OM-H
zcS|f?<4}M5=0Dfkm}*rTi@JxU&{NJp?`;~JRSbsuglN;hLfV(SG90RX9lH*zlszrm!=fRBdKCg@d?95WroKbumdacW+{xcsyi
zSFOKw?$w2~r6+@%MGdO+?mC=8<`ZDS*^m2)oWI7dy8Em<{x64~U0nS<)_K_XOVfKg
z545y?*I`z(jD7=y6MXek>b{)za&h|NHyc;?2wC%Y7f@-7*N2MH-vX>j4t8i^4lnjZ
zw_8$M<*?9WL`Nl);D9zPI{tCobm~Q~PCfT3fBDk=<(CtlqlcX8tGa!9a${xfe*GdR
zBtL~?>$U##oUrKQDR!y%jE-Gv*IX;UTkm?eJpP;ux-fS}ce|KFA7-oe*jt=K{enxg
z<)hr?;+KxubeNjpQLmO#=Fb%Ct;7t01J>G`MyggJpqM#m-JYT;UG36$f9>$=``9`$
z*>Q(*Cl1?~o#t-0@ps&s<_qf_KZ3F;+qWiqe3K^)zUD_CY?iI)T<_PN*kyLkl~+7l
z4>O1zVt=;2Yuz)!m$J@cAD|mWgxnMm3WH9hdcL$;ktE8aR!=c`A0#kW9-ayFSGYm@&UYM*fng
zlhP_CgdcHV_4MO)Tc?<*1t!Vv2d}p8=b@BcXP3f0<$(pC{%W~r&Mz5aa;S%+Ye;VA
z6vI!4-YOV7J*#7v(@x(!@oL9Ib@mV2IP&qf-zxPQD|hVNxpUUEQgq5KAGPlyj!hJ~
zXJ*gtoYXYAbGed6SR+AX5uUr2JnVuja?&lMfH?q5YaSoyHTHO@YqQtE$)2-o%8Eif
zU&OjpuUmI^R82)NKA>l1Ki5(Qmu~WSRpYY4=*RqLVH=;NM<@MSm^X8o=AY#9qscSY
z*4gNI@o`Ofse9%Py@`sYlh4)I7scEU8uYH=9`_OV>^5tADX&a0b~x72d!z5IhB@&z
zw$qlCt$AhqC#}pW?c5rzw`o&a+!(NW!U~_XJ1NRuDhto-%`G3TeKIY5^Y#|slh@=a
zZ~fu$Y*Vn)fOWxUd0^Zw1uZ#blV9GwvHs1x|8}$=b=9hF+3v!4%T>D7H!SuwzUF4>
zxNc<4`m2iNb<<8(R9~s?+xA%L;K&=QJtu~Et#SEiAMsda{U+<@A$r4CMHg+a-hbwO
zta+?q-QshNXDS}?JaB94PSY<{?|%m87CDV=?Be=j!`6$>vMfvAT4ZMp3O!M7y6=5r
zpVK8<{rl+0_w!vCf3VZ^>&pX%Cr&=7KdG~&%{^p_TYY`4A}{VXbW~lUq@le-1exMa
zVmpx*Jyvnv2@dPnqn{W5$=kaor-$O{vWJEG$8J^+bzU1(el4ZTRjcQH9nI?al$CqG
zcoDtoc(qICrcKV2l}DNsG|&4P)1}k9{USz;d%=2hJ`z)#JJol64UO%^MEB{jK(6*W?`c^ExVJN;eHIDXu?LQ(^8L
zerDC6Q&tUTf38{d$tbVN*73CBug_L{(>t81n0(egXhE+=dZ%jqzv|!muJvJ<@n3VJ
zPFsq-Qxd;e{dG&Lmpb_DnhjPZKRfLIdE`)$lk1D1<99Flk9CSudXe~HR_4^vjyc6o
z9=6k3=&pSE)aC9oV&adPA*(ndH_+F7z-JOLyBBOxF~7GhDICoL`bi=Gt}TAP8MFD6
z&GWVr1*3m{V&0*O@B_ze=RRtDCcLIYb>`Kz{?%oUpYzl8>$&dBxBJ^y<8Ns*XQ%by
zqtZLRvT+Xy%q(B9pwg~?ON08iLYK|CR_9hhQQV7|F5wG3LSizGT%6(9`%~|VEm1{j
zn|92*loMj${XqLp!0)WC4{sT@SgCMpWzr%Kjax0#`-j(WbG%@D;pUa*{ZrPwSbuQy
zpZMObU;F2mjr)^sbIkT|>+IV`Cd+oLEZnX?q0`f%mFD(G-hMTCwV}?C;ptP}P6^p%
zWntg?i}Ay(PlsBS{62O4qk+beK}!q=EI76&_r%9Q+t(vXt}IylEn{KIj2AvFx{bno
z!Jxn=*T&r?&l83zb&K^FHN%lBPfhMxfeTm2QA9%!$7Y-7Ef63z%HO};RpV9pGb7p#
zUv)A{b$OW8hU3rs_C6V~w)o(b&)Z|Z4^RIyL_sh7?p@ctp7K;Kehsm$@pV
zQ9suRl;y`7-wF=?Toh3=?b!zv
z>H*KM(bZ*tTf16qxDm0dx?;s{=fP#k3;UGB?XDi!<+_qdy$esD_W8E`enLY1Nh*`x
zpP2hV?|8uPS$5M-T{!eAx$*m($|=_m#jNc6EiUV}$+Q%c(La8b-FdMm#4V)k?(7~@
ziuT2+YG$t4XaDn1bV0mPe6c}w4~-*7HVoNqu6CyA3{J5A8TL9;Z&{hc@*dDXe1Upk
z!_PC$X36s+d_NU@?8XbbcMP7r5#gpIcdB7vGS{x|fGP$QG1>av5n-q*SO)t2bZ2Rb3#NCMK9T)3-Qo5BHej>_#Z1Tq5
zS2h{{OxCPxqM-CAvGbgywngT?>6N#4A9?Zmr^gS4*ZEfy
z9zT9O$L#I9cXb(Rdr1GV>z|)XNX+hPzP{x~;|E!3|F5w(f#$kxzemwTDv}UNGLIRO
zF(ee3LMmj+5GgVg5)vXwA~J+RNTeiVQHsnJk}?mOG7~ABeb@6l|KB=it+UR%*0WYm
z?^C|t&*#3cVPE^&`?Jw)XHM1cJiUY+frs&F*K*=it-Xc^hWx=tv#JGVx-KP;o?Z0X
z@gROEXuD75$}ftg8Ai94yxFt!Sx#*0J++cj7}Z+ozf<4v;JT>s_Zd`t+}P(GkkdLk
z8q!ugJh8+^Q6WJe9=$jiKI%NUpp~%
z%Eso7{k~olh6f-r0nz=0c4wRWAkkA`3duvmtYUp`*ER+Q6_l<-Ttn+~G$k@RQLvJA
zh+oen91Nwtx<<~?ket>*hD}42I|>f8u9}%+?b$Ln->Eh`e}!lENw(+V6PC)pZu{hq
zE>0_atgnC0y=tQR{md
z8uS|^<|Y{V_54Ck@mWph(_D{Of0Fj$)AxsXmZcvKMZZ~;(+Jxxo~VB
z1A>K%Lf!{36a6tbN2~nc_*1K*bHCGgOoL4;zkZFweivjX1nf8KreiJ41?*e)f|H2(
zvm9c6%Yt`~Mpac7j9AsF72bp
zc$6V#a=LP8x4NXu+xJ@WClxq3DX|59HKqhY0kECDv959Lr|l=HBhH3+;71WjEs({d
zlS>bPs10fS^AB@2FTN_iFslCR(NX&x{qwq;?s#xfa-6n$vm0cxcnO}`m{$Tv9;_CM
z$$ndwr}VH-`^X)GVP4L|5vRBU7AP!}9diPB^=dPj&K%3GpM_($fIzIvMtsTLot>RU
z7hH1Ne_(=!h;U2E*9Qd+{fYQspo@A4eVF@jZ}wdWf(jyHQ-`*K5Ef`FAtZ6`oJW>n
zdsGF_2PwIEF>%YhOo2$dhE$8!{%y~;WBv*RH`iy(EAHs&pz57l4AYK^F?Xa|LnVF(
ztA#xOQj{KZjryGdtZ7(QHgC%u^{|Vsi&HbnYIz3dLo5{vR!OsgR0+C?KNxp8?c`&<
ze+MQZ3De*cVNXNs;Dw?F8UTj)5}13+96XnF@7{LTm5G(y-3CEkf?E$hn{*nSO|uPi
z7ihm2LTMa#H@E7I|4KuWF0`(mU5${|qzs*vY6LQy5kM=^#R4#5Si3#xLw8VT8GrD0ioa>)>=)ra-4A&)~p^
z0UF&ax?9&9Q?;<}wZ?Ib53U-qj#1$DzFOf@oi~8hHX5J9U=eff{rC0W
zPc^Oum5Ncscu(uOGR;l*=_?8F4Ag|4o}Nh6e}StvmN=rzhl+}edtqt`!5^@LWxY(k
zK~fW|1hJB*)+Qc%!a$+bPE9VUrc4rpGOItH8WF5MxRViAw-QD5yYo{O%
zYzt%<%5v3%B!P;L601WMV?L(r;2l}Lzs0G~j6=~_FVcf2ZmCN4unW#@2M?D9j7Mw0
znjnszq)KUp8$1zy#a^zYO(w$u7i_OX#T1K2!CMbUPc}8Bd*3&jp}D2I$tEYA@&(0~
zqk=hS^4|}sTNQFQTAZioyo3j(
z`xOR#=79H78W-=5-#8G
z5{V~*Ok}^(D|8)Pbj;*>MU7s+ve>P25K=euf3}KllK#mdB7NuFdj+N
zldZ*0&HI?$9AsA%Z7=?fx3TIiB%{H4m%tDlYd1*)qoTg=FRWX=no$3UBF^VE>+-}z
zZ=N$2S@qo5n_^o;bY2I=XNs;zh6-!q_Bu6ON{w3jA1*-Dct@DTFCR9D3Ifo?9WTd{
z6&Cq$?}&~kuF%bp
zjgW!CnxC4VAH+)}D;-d9v+Il4ZLUakfw4fVjM1}8-COZ(5V4H=S}%<`DE2wNPqbGb
zo0@7i5FEaBTjf%6q2Y?~JUcbz3*?Q%_q%!Yo}4>r3$`7)TYLN`~qx_mehjMOhu=%i0;X|0Dbfm|1g&(5DKwwJ-s>K5k|6QA6{L)^!N0dfcS_N8sf}LoY=8+SXVbaesTl>y#c!TRrn`nw5qlQ
z+iFn|e^puvoS5!n}l0jQ@)XnPZJ?j)G(-PLl_VPMYe0gZ-qL;NM(0`=m
zk@)*oW+=E|34{C4M5f+-6-GhBVBD>=SJBvz2Z&1eUkKOj#U7#WUq&e0@Cr6cQM<37
z>~ieBXu5je{VSh_Y;AV0E#x}Ypf;Yqzkq4}YixN}
zsN~3lLp0Z42w#UdBPhyZgld7IRA
z>;O8W)-nnTnr^we?u+1^1*)(X2Ue;(x6b!Z2(6ddz;4Jdlc9f}Pj;L9;lmD}Gb_F<
z&JpoaBO{4m**s8Z$R+{tLn~;-ibxT5hDOMQJ-)-gM2bmC
z729OTp)rBY<5P&$>iJFxoIpC``KOaGrZp~%VlyjV!K@n}XRtA_aIT*9GH#%PF8
z=hW2T4p6@&KVNk5`wqGXJ0^E6zcbhH+P`%@^O})24tsJ~`8O1fa0JfrU6-5|i1tuA
zU!Jtc{d?J*?p@M)L*GZ;J5m)EgsP3=1gj@*a~!ifwqBmOGVMMsNb@*@)L5RF#>8cC
z(91Ihs9=E+UDb*tv~U(Go-xTc$~+h67N1RNL*^bdV>IDf%8*)eea9wGBY++%P%
zt^j5+)pMaab7r(zs;=uh#7sakL}EAqaS?Ppy*(Az-u_{&xO$b&gZ=ly6V>f{cLnui
zGn1(^?lqrI*rTVMlpufqcQJ*Y-pb1`_Ejl!>pe1j-%>oae-*w)?FN<2V+OV7RC7rV
zdo<$8&b_HWD#PfUtbB#4u>T^bZ^R#&qpoj0e%#E=9MjBgqS$>&iMWYju!YUaRtp=D
zVz%SSoQM-<1OLB~|Fc~Tw`s6K#-WH^L>2?2>LX=7mx<6hpWUxn=s-a>3`P#U0|SAP
zkp_uhoG>$k^p<@`u@nV_61q{45(Y9Bfb+%{-M@F0RMqxdN3CBoF1U4|*+KL4o|PdG
zA(+SJX_rUZJZI)w*RN?kDkz%SQu^Vv*b#pIAN>KBs&5+#9lbt2P(wVw@y2&EZ{o5A
z#|kw+NRz+1<_6!LcsX(U&#>!lbMBR~i`sSC
zdQ5u$TTZV(ttX$s$x(c5-^w4kvAkLShPcoQ1GnsE`=l=Kw^ypSY>Cp1qNj~p$J*T3
z;;ZyobBiOpV{W6Ju40L|%z*2g&gvc4BsRg{j~?p~SetsEq*Ug{+-MCM5jcad#Y4Or
zmHzQGx)#f~!J8S{9r2{LV8*@B=)25C;?>1x`S~{3BPN3(bg{WudcfMj-N9Zl=(ytu
zVwrs(+#z%m6Ha4+sF&lEBXsvL_aml}I7Ix(05Nqm#D9s02S{@lK
z(>r;1=H|z;vjq%ucQ&Y9?>_RJU*!7M6tu^X=s0ywRPtg-K@uOzG@aJ3<9M?lcm~5c
zw%`;kdcd3sEj^dw-o*5R#A)H4ZIIpPR?yukph)BXY%wV?*6))}NJuu#@gW`V?0Zbv
zX-xUq#uxQZq{>zc&+^4e6;F*_K!z|J?m7^cBrwazzt~l)(P5@P%O^Ed8CXj7e9?Nr
zz4tN4h%S5axUlZ0(!`&m=@!#(R;&3_MW*u>w{B7XZ5qmJcJ+JO_w0BJd%KYQY$@13
ze=+`MWq0=4y|C?j)f{u5ScF3ugZR9D`E;CqY_ecu987-;?AbsYTU&5#@%e!#
zTQ)g4O_X)9rKdvoWEBzWMS#VXR=B(v1q{TMYj1QRpRUJjKTYH9+sk&pKxB9#fBb9Q
z-PDHmWjVe>tIZyIu1IDxg(*&jx1K)q(RYd0)xco-TU*avC8eLgX8LX^DJ9;#*^gbe
z{7dPO#L`OOmDp!lkemB{Qn(p!W+x}-6Z8<3Tc+KSA$rdDht786ufPy*#zwh;m5+cf);_m!LgPA
zUj<7Wo4|kMbKgru*+s449TxFWZuUWHSK@f=lfjRfhMBil%fw?8zucfUON8~ah{
z%Za@-06h2&9MWp$S&!w3)8>3F_OFbxTCk^OX>0qlKn`wovBg1WzsB34L;w<;`~3M^
zP?6Eg$tv$^<(mj%Uxwu)#9KIdcmlbX)}4ZWFNpH^0yyFnrQW}8g($bYM(v*xG`5+N=kcQ
z7aWDHi}wkAVAc9JU
zWLK{qNNign3{#jw5MesBEEy+DUKn1dOEnby=SgVf&#AgsjI-~dj
z)+j}=QoUe)sYioBG~oRmNMIkrLK{FNvcQQxVlehHOa+PH6+R;5YiYsp{rdxW^Kx^O
zac!}rdj|pr8p?0iL8+J>g|Ou5&|$)!Y%hGO)gG&j_fOCQ?8xnnyL_7)bbmT*0kF$o
z*Z{H@(XfJo&`jN{_}JK|V9bNNWe@vA81h7|hk{b+shLBSwk1}t8X
z-FrBg2yexsl8C@n!KWkM3MiQHMJ{2R6@GI~So67h#}iGNS*gc<(1>`-+pMwQ
zonf>$f_xPv0?9#H5#JY_cn(Mz*Gfts#BNgNibp&a8&7fv((j7QcWfhym{_yrdQCrJ
z10E*o4eY}?a
zstg+5M92m=0Ba)7AZ}%D0Jx`tttX(XHs~&Z-f1eU23?--xQL$e3e}qREPH62;46WG
ztPXT>yh>6;kSp#Pe(l;i;#Gqp4cwLQvC-Ed`*t37eE|G2jy>5y;Lw_#coV6>svx&9
z1uEEKF?5b^-xBR1sM%QLN|22&bcCq;8M+}maQ0gesJwVIkrZ}U5mQgu*&PJSF6>;Fp^FE)P;B%QbsO;3(tKBzO-nr@
zK#Bok80J+5Wym%_Z$!5a7YR$+e;U&y-~y2L0Fv>SJ~`x*)(S*v7B_zp%TkoF#1$U9
zp0eu$-&hQcgDwX)8S8Vch~-no(3QYZ4Aivo7RXY`YH=74qvhf8bQG?LQ?2k4B!616PJ2k-O6y8PQ}ZaS)DjpV$ZLafp@Cs!F_P89Hw_{3r|)Qi9J&wSa64OYK#GOk{|I{d9*H{OE{
zATDGmUGh9xZozgO8>T>tG{xS^AH`}NKKK;KFKmzNuw+M!=And7jB|I|abisj7%kbx
z+`^}`r{;2$INl(gey+uS3}-5^j!1T@A(POH4qv^j3lXFbA0k2i;!TvAaAp;~v!9xC
z!)XK~plF9~J5TAr5SaHLNwqn702^%)_#)U`*+_JFNZE+C@WurWOaQ1YKllcK%7$$e
ziAFcwaA>!W(d0r?4SaO&t>mvw#VRey;}Bq_3Gxs^1i`Lr)K`%XdaRoeRXKuDqdjHl
z7mkAFIW|U#jwbfG)k~5fC>tPl7xUCy&>;e4
z7}%DFZ18GU5!*G~kMC&Zqb-7cW&U}D
z!65}_M-~$wGcz-WA>lm`T!v-jdr%o~_?Z74#XV|0^)Q@W_6<}r>VZ_wy!DF5_Ok85
zWDgMID55=x9cmJkrV{@wE#W$Cz)G-Kjnq}O%3q6%y+3}`0pc>UwVliA_TD%Rv@RY3p*;$NpB1chEnY;kEEhrDcgQ3#2I=+S_(Nh*y@
zysxPpMC%1oAAo!Btv*1c^@y7zG*}!yxNsoTi|lt3byQnhUdrke
zR6cmT4#;<9H(YxDo}gCjDdX32udiXg2WXBIUghch-=wqFXJH4
z_iAxO=xTI4oXwyogGlg%yZZ!g@?o;fi`9pL%)-Oudzz3P58&0P!78Ci1FErk;ukp3
z70f_{gLuW0V1-O+^TM1n;7~d$6$W3
ziGI@m-WqTw2-Ov>p(Rd|r$g63gD!Q12<-mIP7(pTvCl|Sv!_*^8*CQxY6e0O-VGUK1Ijl$lpPE=6iy?b6O&;mN?eUbxQ)flLhy-Ptw5`NSL_PDJzj0h
z;Lnp}j{XAHO1+pjTHynb6?a?}R}9=!x}0Dw1*^V#gz0J|CCn@!-w^>-4~;bZ$cy~P
zb9WMm2B8iQOb4^$hTz7N%mxrU&!4Ee*78A
zQ@eKU;@3~5MouH@%8>C->6$>G$^|rno~IDnSV&ugk-CbcFVPnWpzrl%k#bySc
zjXvf?>EJ)HI3(yBtL16O=b)~o1D8YW3Gm;kQP{Q)R5g`;7AT8i10f3NC$T2NstJSx
z_n_WaFPngz6fVPgFj&-Yu)M~OCr111c-(3!9}f-SiSI!?4#K)2Y~&*>Sc;G;BrsmY
zZ`w|-HpSLHtmcR$5RC^DV)=TA4S^iY*~r^X=4dXdWI~MFO~m+6aS&B(L}X8^uCtDg
zcVWXyx-Jl(hzuR3sCcJbTZwW!>cegF7lettHY5zFtlnZkeiO>ufUu-I+Hei~fW}BL
zDTES5K;pTCtpx}-M5mu36r){sW(itaMC!i?)R|C3+B#{kuh{`#z5ykLNnFAs
z�QF7N_nV86ULYY7n{(i)GurfI^lUIMiw77XeUkx5~QfMW}hc?D!9-5m3EF>#V*n1>P_uPFd!3y>N6esc^yMe_V9*D-E86ktf4gW;Kuf`}rpM(OZW%ug_
zYaobZcMi2J5q6`YmJpAJqc=J*OHjN?Aj82X!!wBMp=PO;6ep6Bv9M4hy6)&3wQw!*
zhjCt#GS_oXe_sj2(d_B#BMPcKJJq?NNP=>QXbS_{-eCL_kC}`d5SWRx8^aH+S+oYt
z+dd%G_!kuwtyUv9`y3<6OJs~#X{Q~iY4E&YdVEVkD$)bj5>)F!;y_VRaTOp`6Z$5N
zO4PUwI8Xm=%HR`dDBmc@wIhz`c)qEKi)(k6AtnQ}a>433;aAXk>uVRlx|Xkl>V-{SNi#(O7i6+R$jl@6`h
ztimFKtyGjU&gakfkaEheUmwna#IuuBxlo^AEBymv3r67L1XRa0<>`*KhORg{YZ%*p
zK*LQFOd5*PZZcjnf0ckpH3L)+u<<*bLb%v^Cx-0D4@QKsfp7y^ekF}LluO_iC*0Q+
zK(9__@|bTXpeoa>H4#Nv-S@~STN91ziBgZJuWRX0L862qv<3rP6J|Mt36hzh2L2J}
z>A#=<9r2rhAW+~8!g)n^%|@IBKr^^)$8ZEoCwN9gI|^82GHwFV6afd$%F>eajw*Eo
z53yf|NTTG@L_kmw7PGmlYxYAClE`#*{o0Xl9ERl|&;)zX@2y=~f$;J{sQcKX{K1F0
zef74+UndQmVPXveMG2mrBpgmlU}TFWMe=hIpvf6Tj{ghFhfCL1BPZR(ok8D$ews-B
zqNomtkB>KG;8cCM9_#@$^O=XY+C*am1rP)wYB)PP-zvAu!*((H6J?;GoxffYIbcYA
z#39&19=Qbu=4hXOp(&hce)jN$J%TJ&F`_Xm(xehav{d=ifnSrw10sinX(EC5VQ3?x
z>wfY^NC=4}L@f~ga8s$LE4~X^nbJ(t7Yw1_)pUFE!-o%AD<(v^Ur{j;1F1vZc_&aR
zEKAWSVa5VAV{!^13V;r#+tS284Z6uh_7}mBoHH``P6@w4
z|7HQ2Mr(oBLG&k3Jte}gUk~*I?vIs&Lk()ajb_U@nZuALB)gwT5Dd<%44`JSO(KAt
zS~Mg?5ryVDX&)kd@Ikx*7ZCdvbc6&{05n6UfJE8_?@Mfv1u(QEvI7BNi84D}FHkY2
zdrb9`>xr?Th8hH`hz4CBpurQLmQI!?q4a$UQV}3B7<5Q`{37)S0CbcA>PBnCbFrd_
z!F40(TVGyXmd6c`N(OH^fPMo0WQmYHbcW;(+7xwkGQ>0-(X$+Sk!S%yv^di&j_sB-
z#5j~N*YJdiBs!XNbaexzINC)egMNu-GMO5F8GFftjg^+j=t>E-sjUtJdDV~R_}+fa-hI;P91
zdUGpyHK1|hG2XH0-${6J_$4`5o4M3kO8>x;AP~o#7_Dg~g(*Gp+!w9`88~oV9^l|G
zFcK)|%_N3s|3q{Wpud2ZfV{K~Eeb{?w-8Nj%dbB}5E6q4lbQ
zd&O^7Duc$N8{LEci2!=p>%dtcyJCSV8s=Vym0kQGj)GSfUQ%-D;f+<{H*XRpz~^Vb
zMI#R%D&IJSrU5$rbiis5;%=6dl>ANF0q?zmMMJ^!4pERq3xgxOX2`bJ{lbNZ05gHL
zLCVkKxwtbtX-QEGxf@b;qZ1?gy`XDk!2B1XuNO)E@8|Nz`X%C{CF|%Z>W>KCf@*dG_0^44(#a^UKOxFs5C0vaF^O*f&fAGbsBHm>
z$b~mEfDjU`z|#Ybt{V=M75xMds-=bR?X3z
z0te-q=GfB)pp1M>S%({)isMA|6nWjYs91;MyMl%M9!DhaEF0qY%&gWJ2KSaw&-8};
zIQA{Ox(6E3G!khrD1Zpi_?Wp3Hf%@XRt1SPBCh-JMi&N?aLU>bn2F98HdlyCJJ4VQ
zx8OG|6odcKeax>fO(tH-Q0Gf|QVIj;TDzOzqC__mdAf(xCDML4hCRc>G|DL0kQEph
zR~cYQAUgn2%kteJDkUyE9R68(UWLUbqED+cWznUOW)g
zv-fdBNV3BBAVQ6NdtX_Q;Sg8+Jl~jc7=r
zk|y}`tEHbNfW&#kVxfc%D473D^0lqlp9B5gq+1~eleLRDes14$zf
zPC;;r+`UzedTKw!l7=9=MoMy&4;+S{aSJGkF$%N_`7A|ROY4ztdDiXVB60so7LFSrQNWa7R_8Tr8G#B0>h{yuGMICH~
z)J`rO()S1A$+~?z0lX0Mdhp+-{PDA8Y{iH?Mamklo5NJo|OGsDk&XTKP$|^4L$M@(6ERBEZ`8!56>o`
z`Hdz4lO_mRu$h>cWWJx^gxVVp2XR~>YI~?QRFNKd#7J`xaA-FwPcm-9@;GrBK4
zFB50xCcwu;JfxStdZiAxfeZ)z)OFjclAB4g*20J~dfngq5RaJFO!lnjnAR_zEE!O6#
z#5GF3-}&n|Zg68U9j^qojx|7l0fi@nE`zDoCjhoM&}a7?z6G&NqN#~9{{?d(NZ+M6
z_q-vSgSewj=r1vYPPL+r*!b+yy-8^(xPy19i;D8@9((;D-xb_bfJrMHFP+f1!^q-5
zrlAW+2F9KcH?;|-E25Kg`~+^a<=
zo90g)5#!{itwq8E(Evf!Qc*SmL0mMcICh&rM+bEPWGq^uKJDk`9rLT
z+%PZu&E$}u?J9EMgjc*AOIgpcY?eGDA{FzTLml_QVtU{FA((53=*APBrD#X
zJ1Ni%643+PLEI$Y91i*20SgyV)}bMSmwP38Xvsgn@n!q)a!{3{bS@{^4m~Xe;B58n
zCD@e}gWV<>t>Ln7SmgYUK{=qD&7GZ{sDchKOl}}g5L=U$_*e1};FWliVKTvMDIyw8
zD!o%AP9XxbEOfHKnBhZQ0VqI-y+40GMD0#yF0gt3J`&;r1`FXrBBc+a(15iL4!{*`
zGCK%)qDC{YlVDP@Hq~*Y@jVOAJFK~gL5Oz4Qs?OTPA5T3eW5T~+1%XBA8vzok~C9f
zSb=R11|H2)N%d{^G;{zaFfk?%3&q~R*K>;}#t^HBj5De=!g@diIza5C{)M^sW8j8h
zxsUCBz>!zpo)B%cv&#aUhM{}XB0Sn*m=FSvb5r5@
zQ%Dc|0MHGoId5>&Kx(q<*N?nLBBhO2raM1@`R8>~?t|_SdSY>OmlE}VRsFyOdmi_-M%35q6Xpt
z;fNeLL_G}D?#i9N1+E)!59xFNgHhhUUgQA2h@za(1z^R%b&}(FDjyBhZi%xYfR}>t$xNLIfHhRTzeBC@Ij7C(p9~t%_95QjM|**BsDcKjd8l0t3m;Ern*Za0vFh
zz|A5ZJAw{o^f6gKccZ#RGIR!~0!20rEiIRTKnQ$K9`g2+FxJv?6!YpLews2M>u6jF
z_Kxd)xKnQ*nDD}4i
zYezRrsFw^0rs|5Kn^mYS#SmH`9Yw%bVsuJ@99)M7Ou!f-c!Nj6BPh5YWKiP!ot|zt
zl7iGP>m}j@vu%8jgXq!7NSV;#@qO7L`iQKQcdkRu?3K3$W)!DCBvT@|!DpCU}aQ$ZD1OEa|syirJR7
zqFK%i3?a_%(xppmsqH9io4|a>p_hTDKwaJxF@YdlF);a2VQ8WUBjk+ozgUin^$mas
zpv|P9y8u~VT^)MQiE($)Psie{-q+8BbicQQz8PXJ+%p06d*1xE8#{Wqf%;C}coDJ~
zY#=ES_A9AW0Nm+6KYfJkcHvzRy;QW}5;PI?ZEzK^A_fuoLYS$B>Sso+_I^!;TSENj
zH%9xd1B8o+N^*ssp5D#zrKo$*nLS3#AyLW`(;-|2`~}DV717j|GN8JF@uJ4cGxtN8Dw_yFoFfjp7k*t8Z#+g^oRWC%Rnr^7p3jM-B5b%Zax?Y>8kx#Q8Fk
ziX}B;GmZZtdf_vc7w@S3uo20)c&znVNsXCX#6ilr2h6KCZ$7@6jcR<203|-{ZM{!4
zoSYU6RyEeP;~W|dyo_oN42n+%Wg9u|Y#ZtlT-669yUZg#|FuR#0sy**Bj<-+7WD-&
zGA75FOp9m7JAU7$?j{2{Sey)wWIa(Fy&N4CVlJGvkm9SHe1DsqbgY71*LVDq
z4KI}os)~w6R;yyQ4WM~DMiR5q@M})NS
ztA9Pb4~L4izP>9=Q|}%QD_;m#n5L#68IgP(JSO;~+(p=LyBm5L>=FXNBAX)q%?)Vv
zD=aKc_%=#+d6S-5KveWQ%HZJS5i?(8xy6Y~CM$|50yV0tsUqMC+5x=+-;JdfRa5S_F@k3R8sv&P
zsHXN2e#J~}(oa)U1JD2y12S-+-(tR<@%V8+wp*kh^@nMHk%ybqmfa%yhm#&W&V2Sv
zBJt9=EcAuMW3`%FS|}~@jXlA8`}Sjx@uq9U0h_G8n0fRBx_(-{Pf>9PT4rn{9|toA
zC%NU?{hs4bjz9HpHMPhR*s=9XGV2
z(RN5;2kJ7#R&sp4KE#j1%|8gApuDlRwl;NsT})3qOFgY^Yy6KuOr>UxH|EFdd3ezUr
zhg-L9O?~)~_2>)=vFX?AIGi!J6K0}C-Uj~=L0$&~StJHXC3cUDc)d3F1>J|BrLYB)
zer1TF?jG9STyQ?g_#VfeTG_(FB2RAz8{4IiK6>=_0vH?Uj`|=JWlt~NiHVtMke{HK
ztPy%L2|ljX-uTQ%k9vy+6S-gpgIYDoogbL7EXWQJ$G(lsxi@~SS3fE)t_GZl<8bak
zEN}N+SyEW&zKSqDbl_r$L6S3QI8FV|GNf=qnS-x$J36`&z#b6J-8fOWaTrO3>5(^t
zZ-dD$Gf#rFJH`@RQ4k!sgD;?GYU-#pzZWl^>+ZFnAhT&#?%JXj*2MsH=rMeW9{zy5
zhXWoCh2haaquaqfDZHT5D-_k}se*6f@%8Ngic=9RcvjYFO7@OR^aZ)3=K{&-J>?g7agPYm*7L3=>y)9SlH_3z6v
z&A=2Dj-42^v%OXj5eWmvwyj5jpFiV-7&fHKhI$nE_^#nLa@{3wq^3p_(_bwFoG{x7
z;_`}V^<}v0zV}1Yhao{;F&!#I$ooVn92%LQIU>9jT(VIY@zCVcVTzAC59Vl}4bKR?@
zX=Xo(GbW6+;!l_@qSU`wH70sT6rni35Jt|o6O;}a0R8$4BR;lz<$S-AYd1+7~utzmWq)v)k5^_b%Y(9
zo!+NaI|%EPZ)5M8HEXg30~%u`2LtF~8c1R@7`}=&6ubHPu?RLA^_vjK(eryxy<;Zm
zWz~*LKW-=7Cq|;={s7l>X|stD2f1<_-FK*BMmj%&T(IeMPpSCO-k=$xCg0Sgp{>mW
zK{|}s?t*b^i~ld`Ag&LrB%GZmJ;%zF9nWK
z*7kFD)@_GJH|6r6@A9?s^71V^V!$24UVx^y_C{8ejszkWLrzXP4AL=FOY!IUUZN(a_c9+qbVHBVmGs?m^qUZKx}Owh$XFZtl0o
z!A99VH0sD&2NrK7s;;MPj;OG}kJHf7+5j^-vWSI$0iD;~f460}ed44#;*7#vvw?c;
z){K?=4#1Er>sE`zHVyY*S+~Oi!~FCgedqSZqd6ad3AeH%
zd?F8!EI2SQa9o%PaV>2AGs_1lmE7mn1la_|L1Dh#72(IK;9F)^78m4Rk3zaA2ZgMQo-{6oca&4xPvg774Q&TN3FS*#*STah*vBZ}|5lCxLqo$oCaoBa=pn(7u)nP4XOFvdJ@+ga>OuTIhV!#LD#bQ8J^XJdcFizOvHN**~R+nv+x6Fkf
zEI8YJsFJJ@14x;NWFAWjEi58!R1sL;1Nz7G43052kNFf7{sequ%^E~43peMPfDu_g
zH0wy*;PNnF#)W7yHQ%;&7BPc?gF_c6{L^~dyr)>J)i*E*`~HL66wp~IfP`({vgMRv
z-6<<8tUfe_M9(}bElogE8W|sd7=sDYz2b@tM4#`I@%r(FW`4r@DYO&CVo$fy8?h;g
zC`mm&lc4h`LC4{qj*p9s&Xh`aWQ
zjQqoy6ckKz8mjvZkyB7^ek|>D={#+*7swIivcbMCz+*TpXI}IV4zih##IK$0i$SnT
zYHD@sE~iYf>O8)$tH6n4)3sHYbfcP9S7&W1U20c8;&T3Z(3Lha8a6g&#S6CbwdHL)
zaUv=@rghz^tUIrGx4d$9@0Y*#mMX>XT=~Gp!_wt*#}w^0I;U3aukWXnRZgr{z1gC9
zvq{6gqTIQoqkWQ=x?D--E;1cnVBw$X+W0Y7jE4zlPx*4BYe|O=a&;^K2{iwT_aB(R
z>K(KYB4;1BX8vKFtt3WDhyWAEj*gi0mOH{ijuiU{B!sSn&-n)2)w`$}(1Fa*R2lre
zIjW-9hUr82c6oJtJ2Hy~ob%5hg%zA1%6}>@7VetzJPHQ7Pkw|a*6N+uHxY-ysM>ZlKu9pqbgN|vQl!@RQOryPG(K<
zTkC&Nee$zrX4W4W$(6n5@pEXus@tBWb(`K|M+1gabbbh*#K{tv$fcTJud=eT2pJUh
zG2gu{OA9-F8u!gle(PF((Guuf($N4$b7R@^+-^b!F*DnN2CNSuz4xa4czIRTK@R;R
z)ClPjpV>adi+Xx`s&ARE^z-qSnV*@F=W7}D_opH#6Hb0yHrqijBuE9$glmC;J>%m!
zksdSw&go4W`0^M+_Fmh__9Zg%0CQ*{&Tx<%`v%IjRM|GAqk`S*EVy!CJARjs3^}zi
zDIb%J^Po5Z$#AXV?Z+EnHuJaw3QD5O`aFYswO~0sU!XTLFsMK_J@LYE8`vtzg@tzk
z>JYXS(6@h@a24G-|+)UX84a$PsK^uV_uPh
ztR`1<_Aj)Ih<2uIIqm`1LF$X=IX~L=t+lw@(dkgO;UYzj!*OSs3<@TSwsK|+@T*d}
zy)KScP98}q94{WlT+-?10T|KNx{jPAT5gO>vv1zn44TRbi+{ZK7YO^#U&9g(fBC@AFqGod1}o|V6$?bWuF3b
z0+6ip&_Ww|FADsgo~C|oT$Gh_Q{L||Pmm2os9&I{=#PRn>XC0T=L=BL
zu)5457K`Xzd3qeA1URA96m~BC`P1NGJxP0nX19pQ0BB}*;Nm_u&R>si5-F%cTe{;d
z03qErM)QBytX~EYoT9xSHxDXvag9M$N
z!nffoOx-6VL)P$a5yj?RR?gay-(d%l#kMI4_LrsBO&53
zi8U88^TQ7xJ;3)jiR>D@U2ls5Wg@`2ljSShpo(;Sy@#hKL{Gw>JbCgIq$Z+Of$>~BekUt*GUPi4V#Ll3fwuxYMejfBJ?Qu9u2TV|BG92;=?z{dygGG|!I<
zdw6*z(l@ZJi~)xLYH6F-#&QS?OXuqovobQU;kE(L2WSFx_%>vPGa-Tf#|3%RxcJ8S
z7V$?L52&c9Ss}{a|VGY1-&SRt{9ixD=HTfGMa==
zQoHxJv#4FB66!*6W`6;9X&hO{oZeK8kQaBfLF5)uWeIvhMtDLZW#9^6Hr&p4RE@CvN|&IHAKk5Uf)3hijP{5&zD-sHNV1=Hm9
zcPi<%y9cMzlXyW8xiKFT6H^7~8Xi)TFYcY=`Gd9zjDbq(P6)D~!zN4gSTj>oQStvi
z$rOlIdvnB#N2#fAsfV;HykJ*{;9L3JjyEy{=qDTHOJs1YGs0}~?4FBCZGZZKh{)ad
zdW947EH5u?#1mH{Os~_;LE7BBtBFVypn2}*G5nPw1b?=c>;|(Ti?{y)DV^)YR-X@iC3q9l-;ej
zCG5?O((UEgN%OgIU25~c&wcoiz5TGjDlap$#E*R2N3uG!5B!?q?0_5ybxVGJKI+vc
zCzPEMzGfE|vSe_Gyux=;=XY|9=3rzDC@b>~{o(7AqvC<@ynA_A*N{W*?aJ4&8=ulV
zbS~3qJu|lN%Hw-d`jZuaYuUnG6r6;%fstKYpi@?h=E;-yHopP3j9Ks1&BbK-5qC;Z
z@Hz@Rq%}EG*F*2Yeb>L3qUv)GjJgOMigV{gfBpJJ3K(BzM#fu7NxG)hxyNO93)6pR3WQ02nIMonZ4}g
zEW@fF7Zjg<`*?hIu!^$#LTKO^6YcKOKN82xJO?x;j+vP?ppp0ma8fM3p07^2v+yk9
z2w2R-5(_B{7vBbqYBGu+%WrkycKw@?H%H7g4!h|a8d?WTSFZPJDq|Dsq5viSw2|yQ
zyv#Arz+uMm2rDW1EpWhV|9;9a^LezBx~v~SqZ@87n(_2}aduZtW}N6
za4V*#r)9uYK|Ew+WYjx;{E&~2BFZHq#)8g?OE|XP7IQUH5s;B(Q&SU*U)GBk(buo9
zC#5TjCPa{^n3z>_21QbPBb*XkU)oy$&zjdTPwF^V_3qt3K<;OjN(4LbA{anGlN;^>
zS<$n*^OEPI7tt}s70jbS`7^urEqgCWilv;C4>G+hg$JDdvGuoG2imk!RCSBF*Z<-n
z!KSQ^d)aXKa8zAMb$#2%`}mX0Oz|Ckvj~P|XU4-70JlV)4Ru2>z0E8Oj?1_Bo7Qo;eK84WQGO6g%OPs)_GxoHLK^y3aVjqj4H{xq
zj+z|+1QuCRukt^jCplx+pmEdp38I4su=o46iBZ{Zp7H^OSDnV)N#(mNbSqw1bc=RR
z^(B@(E%I11$(XyEf7Qs%l9$6Vqf6x+SMvSV6t1~TS$MDi)x)GV+f`<3zfmD
z#*u!<{N4n$Mt#gZTfIi1AoA?q&yUSI9RE;k{_n{#S)nH?V=2xy5&{qD56a>}I39On
z6qJ6it&VCW>*Zk628-<pa(RkH52hNl2yX^MYf5h-YwZHWEOD`$Rb+DJt;0GCw
zq>bGaO3_gt(gVZU+NX9Y*d5MHm^&{gAn$#gxgc1DR9V5#$z~$ohQgnR<=n>W!kLHW
z)<{1tQ>Z>G#P*rf?x}BA&|Fa83#ZfX24|~+T)F?}x~+AaXbGhX9q?B;*7x=VonBX2
z@iWU48x%a+FE^;3lAK?6g(HKb<KCU;Tml=xg(`Gj|v5
zNcYLj)9>D!>{{U>@5~syZijs6l5Uz0eg4bxt$t@Jik}vDeqC$lLLq+z)W?$<|MLNu
zOx;_KvcLG&=0Ta5wSBJ1=jB>e%CRe(Hm)SD^=_h1Xf)IvNmph&a9PPkx_d`=ebBd(
z7tdVtcF0&2NMid5f05n(loYfm!G}UWg5vz(&70#15wCM;#k5bzcgF?|#K<&u|J=FU
zR95td3q-oN@}vptrrtXNpsY9l{??m9gqX~93I-LZQW
zufEco1qusI{2ATj*{0$a6j=lm?XQCZ-ml4&Z;5ha4mdrVJaIoZd{?9A;yzkFY
zim7{tyQ$OOJoRgR{=e3;oZY)c$%UIPb6fF$=gV}Lxlz6CL&bUa)-cw0{E;Ejg_8!Z
zCp8nArDMuIT3hvpNl!O7XWp&9z5$)ecY@w-j$o>{mYC{q((UUTt!(u`{vZ;w+#Tg`d25obCrC~hL84#=0j@SLK3wi`0h=?1$poTrn~Q5B)gxOU%G7$`JcanbW(7YP6Sz&NVY_an$_Lyw=R8bG@g;w
zr?YdBL7rFMFWgVfNNC;2NZ~HIxMNNwVHkO9%k%!1gj{|CnL@S5L;gXPioPL8&HVB^
zC0N_my$dvP+*?U;?88sN{tFhgb4Mq1PnsSoXq%jxw18FwI8fcg!||9>a!ec)DBR!#6@=oBU?P?nJ0L=iUWX+0N-J5o|^1;*QcIE%gQ
z*~t`4QEL3<())E6R#lg)=f!H{R5
zY*UU4r7dCKR5NygqSR}4r-EEuNJIoNgGbddH^uZX_ieWSg(pi%Nr_IEpeK|lB2Xz3
zyaZkDtx{hlb7m%eL*wh&oYW@Tyv4@CZc?#@>_fH%O&vYSCo7nLRJOhNCjB^z?ekXU
zlNDWQwvuT~v
zY-D#915;eS`WN!-n@Cl8
Date: Fri, 4 Mar 2011 12:45:09 -0400
Subject: [PATCH 0960/8313] format

---
 doc/use_case/Alice.mdwn | 4 ++--
 doc/use_case/Bob.mdwn   | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn
index 836d935726..cd5955b3f4 100644
--- a/doc/use_case/Alice.mdwn
+++ b/doc/use_case/Alice.mdwn
@@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her
 USB drive. High in the sky or in a remote cabin, she catches up on
 podcasts, videos, and games, first letting git-annex copy them from
 her USB drive to the netbook (this saves battery power). 
-([[more about transferring data|transferring_data]])
+[[more about transferring data|transferring_data]]
 
 When she's done, she tells git-annex which to keep and which to remove.
 They're all removed from her netbook to save space, and Alice knows
 that next time she syncs up to the net, her changes will be synced back 
 to her server. 
-([more about distributed version control|distributed_version_control])
+[[more about distributed version control|distributed_version_control]]
diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn
index 53c09bc18a..5982b5fcb2 100644
--- a/doc/use_case/Bob.mdwn
+++ b/doc/use_case/Bob.mdwn
@@ -11,15 +11,15 @@ without worry about accidentally deleting anything.
 When Bob needs access to some files, git-annex can tell him which drive(s)
 they're on, and easily make them available. Indeed, every drive knows what
 is on every other drive.
-([[more about location tracking|location_tracking]])
+[[more about location tracking|location_tracking]]
 
 Bob thinks long-term, and so he's glad that git-annex uses a simple
 repository format. He knows his files will be accessible in the future
 even if the world has forgotten about git-annex and git.
-([[more about future-proofing|future_proofing]])
+[[more about future-proofing|future_proofing]]
 
 Run in a cron job, git-annex adds new files to archival drives at night. It
 also helps Bob keep track of intentional, and unintentional copies of
 files, and logs information he can use to decide when it's time to duplicate
 the content of old drives.
-([[more about backup copies|copies]])
+[[more about backup copies|copies]]

From 88898d427d57f27d26d50d5bf10d8d22e50c5e63 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Mar 2011 12:48:21 -0400
Subject: [PATCH 0961/8313] format

---
 doc/index.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 00a3315cbb..b2b1bb257b 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -36,8 +36,8 @@ alt="Flattr this" title="Flattr this" />
- - + +
[[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]][[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]]
From fbb588b486c6fac122e3b1dfcdec63d40c0eaf83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:04:49 -0400 Subject: [PATCH 0962/8313] format --- doc/index.mdwn | 4 ++-- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index b2b1bb257b..a5a2043273 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -36,8 +36,8 @@ alt="Flattr this" title="Flattr this" /> - - + +
[[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]][[!inline feeds=no template=bare pages=use_case/bob]][[!inline feeds=no template=bare pages=use_case/alice]]
diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index cd5955b3f4..57862c83c9 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -10,11 +10,11 @@ When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from -her USB drive to the netbook (this saves battery power). +her USB drive to the netbook (this saves battery power). [[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back -to her server. +to her server. [[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 5982b5fcb2..5e6d93461b 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -10,16 +10,16 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what -is on every other drive. +is on every other drive. [[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future -even if the world has forgotten about git-annex and git. +even if the world has forgotten about git-annex and git. [[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate -the content of old drives. +the content of old drives. [[more about backup copies|copies]] From dceea4d5f3c462a8c6806c2290936ce99acb64ee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:07:48 -0400 Subject: [PATCH 0963/8313] format --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 57862c83c9..f47dc7d016 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). -[[more about transferring data|transferring_data]] +[[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. -[[more about distributed version control|distributed_version_control]] +[[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 5e6d93461b..8fb8e8aaea 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -11,15 +11,15 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -[[more about location tracking|location_tracking]] +[[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. -[[more about future-proofing|future_proofing]] +[[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -[[more about backup copies|copies]] +[[more about backup copies|copies]] From 400ef405ca5aeaebcf7c22411ae3a29388986715 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:09:48 -0400 Subject: [PATCH 0964/8313] format --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index f47dc7d016..c7b70468d5 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). -[[more about transferring data|transferring_data]] +[[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. -[[more about distributed version control|distributed_version_control]] +[[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 8fb8e8aaea..21cce21dec 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -11,15 +11,15 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -[[more about location tracking|location_tracking]] +[[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. -[[more about future-proofing|future_proofing]] +[[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -[[more about backup copies|copies]] +[[more about backup copies|copies]] From 6072399dc225e4ca333fe3d424b587d151b24065 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:12:48 -0400 Subject: [PATCH 0965/8313] format --- doc/use_case/Alice.mdwn | 4 ++-- doc/use_case/Bob.mdwn | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index c7b70468d5..4e07e22813 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -11,10 +11,10 @@ server for later. At a coffee shop, she has git-annex download them to her USB drive. High in the sky or in a remote cabin, she catches up on podcasts, videos, and games, first letting git-annex copy them from her USB drive to the netbook (this saves battery power). -[[more about transferring data|transferring_data]] +[[more about transferring data|transferring_data]] When she's done, she tells git-annex which to keep and which to remove. They're all removed from her netbook to save space, and Alice knows that next time she syncs up to the net, her changes will be synced back to her server. -[[more about distributed version control|distributed_version_control]] +[[more about distributed version control|distributed_version_control]] diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 21cce21dec..915fa1e0b1 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -11,15 +11,15 @@ without worry about accidentally deleting anything. When Bob needs access to some files, git-annex can tell him which drive(s) they're on, and easily make them available. Indeed, every drive knows what is on every other drive. -[[more about location tracking|location_tracking]] +[[more about location tracking|location_tracking]] Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. -[[more about future-proofing|future_proofing]] +[[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It also helps Bob keep track of intentional, and unintentional copies of files, and logs information he can use to decide when it's time to duplicate the content of old drives. -[[more about backup copies|copies]] +[[more about backup copies|copies]] From 1cd4393c7fef81a28e4f57adb8aac49458d18182 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:15:04 -0400 Subject: [PATCH 0966/8313] format --- doc/use_case/Bob.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 915fa1e0b1..847f5d16e5 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -15,7 +15,7 @@ is on every other drive. Bob thinks long-term, and so he's glad that git-annex uses a simple repository format. He knows his files will be accessible in the future -even if the world has forgotten about git-annex and git. +even if the world has forgotten about git-annex and git. [[more about future-proofing|future_proofing]] Run in a cron job, git-annex adds new files to archival drives at night. It From d73a6e2e7294d6778ba43d20d643521952c96646 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:16:32 -0400 Subject: [PATCH 0967/8313] remove links llinked to above --- doc/index.mdwn | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index a5a2043273..e70c5cbb13 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -50,9 +50,6 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage -* [[location_tracking]] reminds you where git-annex has seen files -* git-annex prevents accidental data loss by [[tracking copies|copies]] - of your files * [[internals]] * [[bare_repositories]] * [[what git annex is not|not]] From 354402e6b2be7317da3ad681873666f62cc8167b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 13:19:26 -0400 Subject: [PATCH 0968/8313] headings --- doc/index.mdwn | 2 -- doc/use_case/Alice.mdwn | 2 +- doc/use_case/Bob.mdwn | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index e70c5cbb13..0838505e84 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -32,8 +32,6 @@ To get a feel for it, see the [[walkthrough]]. alt="Flattr this" title="Flattr this" /> """]] -## sample use cases - diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 4e07e22813..ccc0727bce 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -1,4 +1,4 @@ -### The Nomad +### use case: The Nomad Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 847f5d16e5..18cdd4d8f4 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -1,4 +1,4 @@ -### The Archivist +### use case: The Archivist Bob has many drives to archive his data, most of them kept offline, in a safe place. From 38f1771866cd539b6f9c089eaacd53d1ce8f2b5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 15:15:55 -0400 Subject: [PATCH 0969/8313] scaled --- doc/repomap.png | Bin 129065 -> 67316 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/repomap.png b/doc/repomap.png index 4d334aec947e33900920351ccbd094b22c4aed3a..dcd777e1254698aed16dd2f6a2804b69b5e9533e 100644 GIT binary patch literal 67316 zcmb@uby!sG*EYOo=k(-lsB%|t^BDD_M}B8X7fe&3y$?vGACk`l^pEx*coH7Z0` z!L@un9Wj?2{-1}^Gi8xEH0J+2posr|#{B=s)4-(Mz$@j~$6Xi&;6vbBZqS_px;_yR z5dw zp}T&WyL`)j)4)I0xA52!AP<~Ba0BGoVO|{t{WW@C3D**KoaW{oVi5Bv984D~L2tah zzNm1Tt1r`k@mkpC*s$Ht`+1|o^Z)B~R3M!7@XvPcOng|!<$BbW@CW!We0s^!l`o0? zj;f0Qea#8|&?EBCX29jVW6f@~hV+x>nVA;n3v&)&zyQ00RiW1(A zOAGJIWAk=nvEqO3Ah>}d3e2IaIvP?%e!q9}W$}tc7M>DF6|VUINVokyp8xcDwejcQ z{?79a6G>-bcr?0=4pYA!+e9sT!c}__zqWulL|W z2;s)9FRSiw$81Nl%48NwwI19qv8trlt#TPQl)gNwHw}o*NHY55^1MQ?N*f%+gAmmo3*5_4?cPrqXF`Aa;Shj`1=$MW?YS!KK!&&ZBnWF6@5Os(YY%G z&0d+dPadZd5}~dFUh_~86~_IVn+EZPSSb-RcdH#7hW#oE^k@6U@}mmV-N~XeKMOKc z$~uMmXmRho()mV572Ldk#ts??Xd*0tQX2!(dlebnZ~+lD72@v;+PDk_t z%HeXmM3ofd&$Z!1m}=XL{3maJ```i4Ae%Xx*Yv)tkh9Gs4TlP=cldO_9a(~ERUt44 z3|vIuDuTc!H~_*Zw?qUN22x)~KlJ9uQk>2~$_WX}MHpAx|>~-cnl%No5xX)nMR{=7B z6uc6+0Z8#sfOJH}mQVyaN8G|E*BW+f<1cd`ZD+`3PU%;e)W_Qt7EL<>{xCz4Cma2_ zXA3Y5X%ez5XmBy+0SedD|L zj7bRzuWBWO-sfxypK>e`HoIde=AY4y*Vn(dC#3w(?JCx~-{PG4e_Hmd2!p{$9)Ib6 z*BE;NUrrOS^ulEtdzHPJSZCS4x6-#~yN;*KZ!Tp(E`Wg38?dLe+D$M!^Uw-fDT!Y+ zmi}v=rKSI04uCxH{Qv%d|2GH!+Q@&O|Bu1^KOTVZ1x?9ePv-6Rz=s2QToZJwB6YFoxd+0EK405$h~I8e>h~eO&Pvl)TV%KMAIGFC=wEFT zc^|c3l!LZh*627>e!r5y>8{QYune2dbZJqJVURIh^4*BBuUQH>9dQ6n%uv6^v`WNz zzVPMG#j>&M&f7!Rs|x}UtAgTIUZ3<*oQc;;|Gm5D6#k0I^1q-r%&ZXw$TI8Shb{hG zYMttfp_u}g{!|DW&ZFA1!v+0D*A>(IMPII*#8w`7dsPmiZn@vtUb(|mdAZFw(}c4aJLqy1STr7yuXL{)*lA(uBJha0om5AiNP#GpF(x6}z+^iZs5yetCvOAt)`$ z4K#GsXXL=XcNF*D1>6;r9z&kMhzQ;Gt3H({JWzBaZHRbPqO=tPNul&o~`4@!K-LpS8r(SpA*tvpnHqW#+Tk>|U-@ zqV9KH$6U`>>$D!$w?42v>xyKG-_++P{SP((q-O@qeU@=j#l@=W1xp{{$VhUYa&%nE zJj>JCG>Dq|8FZ%?8~=^7#Pm2g0~OT54m-Ec%XK;LQNPi$AAhAK`rj`sh3${P!6IT& zB~-*XHlJ(RE3f#TA?)~jGq!a)&W(akY4Z<_Yg;~vW?w8_h2bZcs;>BEISVGx0P-%x zuBV5lW&c6PdvMf>Gq^+$8aIIh)I-A|*M0x+`Q!k#%+2u|glQJ?xU3{+k}!=+pBHZq z5UCN1i%Lak0-E>S1KfQ_Sye21UuE=Z?_kzrYp^W;>AQ0F+yHHM80PO@3e@cDWERrc zUJ!@8MLa-f<7Ggzj;{73;$KLy(@lD0t*WFBh>S`cNoMb65M_n4EQ?=W?blS`(}|Y~ zS`U`exHQYYC@Bta%;X9A4={q>nwPsxHJ$ib3MrSKY7CW-Li^dYQ432X{A!W&MZn41 zuv5=5DgD$j0qd8G*unqnv=A9Kt9tJ;OGT#5i)H^Y&+Ui8S2@1x5#d!15h;ocuOAib z$#Mrb@9qY!Pjg%RJ4>lZ2E7L@2Jy2edlfZ)(}RjqvV&!z%snGy9GZ-u-@uz@aVF^KU!O#NhAcO;?t*3QOUV`V?g10 zD_Eo@j*{sBrK(<)VG+2nizLc0|7(Fog8O36Yu*~(QET{Kl`R3(>rn)r9wR8 zaX8;Nl_ln>8%56Z=<;Bt^F?P62)X6plHB)nT5fx_7lOk(juxLiz7_h`|L14+r*7h2 zJL3ib{G*MUuz5GCu>B;~daDRW3+TU#D#;1b zW7r6SYjchaJQv6&VOQVTz9B$GUY+fieAej%X~RdxQ6yB8gcn%f6Mo`kywu`-X1Trr zO+_iRVC{K^L27yUWl|I*Vhvk$D_MwMN3SFPkY0j3`5g>gTnnm4hx>2ebw)DX@AkXu2yWiR5F%c$W zbOo;%^h2YBv%WFpE;hFyIyW_pdGAij+ip#^qbRXVuq6QDH+R=Z;N5v5E@Ake75mSk zP4$PEu5+qhB1SV^JIZoTWgY;y=n9g{_0}r$p6ff^5SVR2-+O)2G6}oKJW1;2L60|3 zW6TDIaDg~_sf_bQEW}Oces$~Q@&{Fw0?6_2-EY*uoauj73dpy#gslfjg`H+=!D(#P ziV%{6aEnI)(pmcX!0P5e(wS$6QPDH9!{Qh;fL_c)t}{+0t+|Iz;$^9Z%&qR$P^w1% zpA&zzj7JkEuciEcgWly&d)uqiJrMMX&4~}8f!obw>6HD!pwk!j$-w4l zx&5BDnoKdk|HWNa!e*yAL1!#Uw^f+&C=<6mTYd=c+*=%q!M9`>0;`Hk^4!^=aekZY zblF2^1~|Zo72p6k_Lzm`-(V2X(vQuDdQ_ILQ=xa=C=WwIK27)E+ot*V1Y2^PxS5LGLW75}^ycG4e zWAxqwtf_XXhdiGa3uO7f&x$7W0J#a zdPmSqBOTD(L@U6Vls^xKFiX(GJ{s5WXwd@}MKXNyF}vLfWbxrW3^!j3=_Ot|&AaEv z2{K|Df2O(v=lg!r!8N^+JZ{craF_I576UVuiQflgLOa9{Su8Yj@e!1QR^hxg-98+3 zAvCkUV}yrLX*v6FX%o9}NVO+Je{qb8`!kBktTJ05Po(k`O|dqO`kUD zL;kd}6RG5!4iHduJ}AabYZ?^DVVNwywIXVXFiU@^T^KH{U#mM|zp)V~Go9X<4``GR z)^y&LSNW4}u#senM|d%C8aLPSYMazq$ls>a#p zz7?wpKek-^V6)O-|LwQ*Iy^D{i}cpt`@lUvE@b{d#VkI>8_D!G=MVmN1jYKjS^SrE z^w!(Hp|5_vSQjUCp4L>qw)cS+2Z3bK7^;n>lW^E4IPSI55X4xCu<7<`Nidj3A&|@q zCZ-TP2}WX>JqcO>3A0vf#>XqE(UaSkUt7Q~kqh_W!cg2>SKCaUcBj2B?nK^#t#NhmI2ZhyY9qNz5)EZCong7@SFe03*1ii~4=ze{ddoba2d$EFgVHT*rC z0uenEoCaqte*EH#-`8I;T>hW|&Bx#}*3**;vMCqAf}!PxjShat)@evGupC!4n%tmvm+i|CzQwJ_~OLFmBH0lFS;|r#ZRE=EjvsygiUG@ucsiVhIMzahA%>Gr8$T^)rlnNoV$)G|uJn~lq6Y;{lc_do_XT^A2us7+hX>Gg9= zbg#9d-Z$d7PP`D)zocKl{{73S$327Qt!7G>fDIKq{FCcS^_Xx|>YRYYJN~`B@6O82xSnaP&q*Bq(V*rT2+hrkER+adp1!$96P*$~z=xv@W(9-6E( zg~z*nF&AJavN%7O6K}$CBqT5JeSgHi+U;6v+N{Dkr46R*oF}^dnp~E|Zw;iAhOgX5 zuR=g-I>R5*_)ynTpSe<(rHzs+Z?`t#>#b!h>q#Cwibm9>a%lj5z`T33Z(OW=od4pu zCxYZTNG=+#(+_)N@ba_*!{Fm!Fk5yXZ_h}OKjqHK0>i#-?zoLG4@oV{3E?dV)x>zM z|8f2rWN?w`N8G-u;mwe`a&zS_zqkG|_%tV|?PmMEmW;ovTYpXpyii@Qq)>iP91y+W z(6=Vo`xeKG@b8b}gXhy|6_op777-|EiQaeaH`CQ76_yVmI-{L%gI!!os1y2T>FL)I z9M-ph+2SvF?a19%Fsgj0Qykq(i301Y6iW@C#}_5c$G;tDa;&Z^N?{lqo$VO+<(J@$hI zJv3ITZIpz)Rya!8L3e-V6Tesu2?)vOETK;#1*mEPR7Tm#j561qVr^FBM+iMiaC3Q5 zV@{t%tJ~U>*Og)v%7l?y9iG!!k6?D(0nOp=Dk%Fbya`LgpD}LCv#qjKMe4PPwH}Z5 z*bu*j-LTD#CJQjfCQib@5OBIKt$DF7=aklN(+330e|#y4o)Z;X<`#1*m_bF1sNTX3_#;s^s%g)5_vi0ch&H&j0 zF{RHB@}lHO5*y9I=J=7!O7E9!Bk`Q-i<={&E0tpdR&dQ2Zz|Yk#;Oe#>pm41^S%Nc zb(#77Ew(sjldH3tl^IGPeF=Mgdp5HN5{qe8pKh`` zmPn8@f&mkH<1YdGwT^Bnc^qM5XSmVJqvfil=cDwWBQg&DNKNBug1O+-rmXSxm|x75 zsUImlS=`Mdw3~u`<<7a#;vp1l-l7ydHn|v3#%GnZton)V%d|ezD4)2j?P@cq92a#|T21*_Md z+4$6tl-@i5240~Rl2lwyC^Fbz)FTW2D^G4_xMt0!v1kD2P@3m3YO0LQ{csSD<)Xoo z_t>&74Rd!6%-g`*T^e3l7O+2fuPmL8MZ3naQ18B>kZm71lr`g)PlEGF@6T-QuY&-S$cujNamAxI5Jo!GLpm; zc%2*GNfEk6%fp-dFjkX%SLT%*xbb7i~HOBt!m+rSW2E6 z7B%;4(li^1bW~*kDQ-EK*8JKZKrVjvbi81W^KI7yqT@DoTpZ9?QvULfF|Eps5;PK?8Zd8 z>f_SxPm_NmJ>IbLz~iRmd@#D4(P$x*s-{+y?4C;&yQZRf&V$9r)Z|3sUcUC3aMzZP zZpLn97N1cq+O_M$Z{u)CXPH%&c(!#bv1Kv3anYvhz7pwB^36tgr3b_ji*m$AJ}15f zs{{>alN{3npH>2mj@Vs{v9$7&@wbW*a)~xOMQXtwmhX#E$>wHyo+t97+ z8}XT;@HjEdkdJQ)r$tzq1b?mRxOxVbDE#Lw)DHMpmC!{55)h32bocPOwEqL??S%*K>-kl0FhXu8RvTKc-Jn?zZPh zm$!Sz`R(a2E4kXa4EdNDS_c|6IFV?;gZe=?{b$)Ci;?i})tz^KbX;HLQgO%K$i;rh zm=K0vbP+4p83A7#`hN*tijmC_!&J;9^@?F!TqHM;Ah#w_dfkGA|91`An{C;#ZmAjd0a6e zpZA|xjYS?&hwn-&$|&Wgp90V_k-Oot0aQygAqHQfty2m*8~RR%nvz%}S$hsyo$3ci z*;1k{-$n6Fh_XaLVWHeFMr8wa5X^yQY43p;T7L_NGV^D?_}hR$rmstB)qt3C;w-o- zh1)u>_A2(ks7GDjkNqQb)x!x45CYHT1&J05NcVPHbwRb&w5+2mw|Hx&7a-2Eho!ilC zw5=lX&%ubgp6sOOW0tFwGX&eQx#t-Dxw9P4ynS!eJZ{op>ALQ7M4jp3nJIFHKsF(w zH^a$476etL2B0VUfbwR4Jn6nyiGVDpk+FQ*8W9 zBxd|gBW&RRD>>;x{AnnoZOby{Q1U!IFv#7QCT-zt0YT3c`jcM<_VDVAC54h=dnCb>1YB_O zLgj#17Cdu%(kSp7DTP4dBjQ>`)}`ET*(v~j7gO{wFx|PgccXgT2i?_o%+;y85>Q%S z@0+9=_2CpBKKX)o(Cf^8ndL#@Wr2Avy$vZ6c&r~`u>SG8Z}imBx6v5udisbabdLkK zQym(7j_CeZIoyC3b_&d5SWcS)^D|YRF5dLhbvdeyUDl;q^QYN6@6_IV{{$T`;!OED!=D@VR|7f zJixtnps`AYlNT{n2&<}d5QwL)bt4}rE{v{g5hGE*3|RH2B`{UYzIFKnW`!@ZhM*#X zS9GUR9|L>vAZGBMOG#?@O2R3MXd($-63jV8GQIbtWqR!0Y)K{eV4xi1WGB5%=%qJ6yBo$|y=F9lS2;g3m97C#S zYrIgp-j13G{}fGH{jwxnt1Q9w>{|0#f14=C%}m%rE|fU(Ls9$Dd>$3OwENIcCvI({ z8HholQD@QHpiQ_)tC;w5XLqPx2|dp!Bv`%!qXEiV$*|pG|IwOs+%rdrg>{WyCg8F* zFn3m|(uOVW=V zer|8j##JYqf7uUlkbF`q4e5>dbz4X7f>E-D(_<`!-}WLrf1zXSVx9MZj9`Oas3e$P=3OnSXfk8JQ zR+Ku+SND)?108roBKS6DPMTXvP&rD@f``SFum)YU)QZiMZbQC1N&Yd0GA9(Q0YMRv z<9B3X;#R7wCy}Dq+_Sh=8BZovuaLp%i#h}AwoPRF756Ji|eK%$qiaWy)2S*=J%r834Wforx zm3EC4D$)S>xFee3f9SV^#W2)x8NJzs=GD!}%_KoNz#Vt+7Ml~qQ6xpqcp%FXQRu!= z-%%-ZMuY`p?<3|%BOSS0sWty%px1Bp>G})Z@XNg?)S@m;1YHf~=Z zD|ldS(45kWeq6L0Z~X)21?#_{0y+dhOfMD=L2sx#MIX%_fQrI)V*?gazxjRhYfn~u zWKa#yo@My`F!s3z&h@L=T61b3n;ceOHezc>ChvnDn0wLeSe(m&v<3*d80*5bd{~l? z#plY}FfO&i(2yTG1*=U_5J2|8WS_Q@Erj@0?XcKVCX2h-E0KMS05rDXT4}070&a_D zrx!uD-nVmL9=rrXk|J&D50-g<2Z_@>z95)>GAjE7&+XlYxDDN_>`Z2Mpk=EVHC4o2 z6FwaCevtsMV{_*Itoof@BZm|xMrn8ptO-q>684^)c)5L1yTrD{-Od85_BD`kdof1o zXq)v>vp>G@Pibau`-#I{tFdI^5zD_}YVyaHxrFTfaJn=zfTlmA;}# zZ^9v#>1U>*hfqAPb@U$s3(n^;Mx%6#-)P_2jnaHF0Pv2@srtlum=iH^m}c-T38!r> z-f(92rpYd}4tKCK13St%j43E=LE^SO-S3m<}9k;Xby^UHuiU z{|g{|`pt))2IyIeSa@%RnS){u@ASn*wb(-8#*G%>w_vT*ElX5iUiIdV!cBn>s@`O| zuYjJ;CwzCkwXfcGS!m@8T9n{gCMAp=fLd|CTd(`!6Ky@gp(n+7100MrmhIY_lpzi^!o=)voypv_TxcTEaTBGRsS%V#jSX`ptwC<$BOSa zqlJGX1PZxPBe<A{*+TbKtzp7b&Fak(I_&krlM$NnjgZtL7Gk^;ZO z(c&-p*6UOaVtwByny>deJheqn1a1}!c)kOHYzCF~@LM3k*T6Kp6X(GEGxQD@NIY|3 z`-?R!=gX42If`RLpe}Hb0)BNPip`uOkpCw)IQ}L=t$&APrd1Yc0@rYTz8}vF? zpp!G%$zi2eB^yW!f34c(b`@D%@15?6dm$l}o6SUSiUuo*?}|Bliv_PlFuT;_YMX{# zy&v{B)+lozBL%!Zqq*H)or$l~oq9{I-+VPK`a!*W7MxA3O?3xYqGLH6tPyMU1$J3S zIVKO};7sMTaTz7wmYxoWi>&t7a~5?mW|teZcg^o=iKAU`MTl6X zu~mG)dP5?RoBt$uCOH$I*a2~ox+z__!Ul3&zsNa{SjZcYym&f}=UlBPJmRx~sc*xM&+l?!vO z=gFs~F??_@uIV@^QbmYli)+`#S}8oU!1QK~=o~q?IgLac$E;oj3m;l+P@i8l2}w95Daue zA*|XgII07cMTL-)+`u<34%bAMyL~J%R)-`o2F$`N4V&JAb|}K8n`I}*4ZRK;=_}t% zK0seQ?Pw-O1%V}`LGvKOd!0magf4n!HtFrE^$kM>q2bCu36HNUqH*HYB1pdtJxWm) zsjG()XFlO-lIwx65x=aJlW2Vxe#a-0+AP*mY2{VrHv$beGqDXI!6ZFNd>%h3keq1I z)(v<2F-*0R^#vRa1cv_0r1j-)NPAuFP5M#Nu5M@G>m_;>i>gT@!<}4J17Mv=?bn+H zV94shH>r?}z{*byjPbW>DMEU@pQ*%*KI?(>T$1>V1mz-u0z#-y?fobSlmw*{%1NRH zV>)xT`W#ia+gUYVL%;2qi))Na@ z5DS%oUDGra)Mr$jSex%GA;9{N-e4m#Q1s_K z6APAh?<$z*0uw0{20+AVx4a5_s8W((RyFhR)S`Wo)I(+ZU7-Moh4Dep_uke#&q|Xn z)tm;AQ@P=O<-6u7j+z7m;8l4rL%b{of$`d-^8Sn;!u%rrzM$lY`nWv{ z`K5z>F<&u`J`o`9R-=<7`ZFay>(gmtP2EcGw_0ptxmIjF1#i zWd?{Bp5}cD?zCIXK-Jd~35$3hDONI8-lv-@>S}5dHLRl78NCkKeHBI4TQO~dpj9~l z>-K2BpP&W_UbtsxxS{c(Fbdj^(hB(b56^w^F_M z$o5DfQh7V|VBAe*(TY?@`JCm1%K!Y}#DunRjy&?cg|hke<2#nyc>dLMCs!c;{7vy} zliiMc5?F;&t;pW-1`5COdQZ#A-Jphmg~e}g$2zZnO3WCcfW`$Nz-s0=PX|8jW5T*E z0vw8;mX$$e$vfsvB;l9`Cb+0;4=q=~5$hhkkU64RSKQrsVr&N1#dMpT<{Db)DwSRq z993BX?-(RrjY(bX;L4q6r$XYDPr&;7xcg@-iKy^P`4w1jH!Ya(=A29k(;8np2-mYL^w$V}l zd9f;0b}f-IGc-5o&sOTxR_cpt5*ls1J5}dLEf#>d*HkG|LCK`I7&j4-j3M-%{N8Kv?rXT=X#DT-r6Z}Dtp*bBD}k}TlOH+=4RUTY zzW3Ny#schT+i7jD)Din&HKqK}XQ@cUX})l}(x^nQ(zGh2#<+}}50q0H)IE_gb#4#b z55sd9YI5HDDdq3A>Fk9lnApYsi0;mersm`kvq6%{^ z(cmvK&mxzl&n%cZc$#cE5V-B@fvf2$smi?tX0xi}$n>5k;?H)HKIp4bwWoWyLef`j zGeTX(vb^Xq#N!7QZLu04ZnT>y@Ty6=32h~C6Iih}psDmk^^xxEa{qC$V{!|sJa;A) z8un3s8Td?3Rq_qKtd(l(KNlNAkwIZwP|kHV?gG0O%J}+JMx#J^rV@8vINw`=DsZB! z>!H{H8{+Q1%IDu~^BtQvd4RrR01RhH1na1FEDMUF2k~@@ck*5j##)V~YFH0t>XQxR zzL=_S_<&|MV>MQ-U?ko(nNIGj+Q4vn^JV0Jkd$7NO0bjAP!JhEII34Qje8K(+P06%~2` zC_T5Ep%En)tK0|`zpxM7Pn#%+4{T5!waWhXG##!t=o268lAIAr!ltUE^NeXYLs$ir zB5^M5f3N%E(Cf1U{~|2Q`Ji&6k^uImHT3Kyz2`cuflOYCggy7#y=YXvPC1hCj^ryU zv)GFp{C7?`CP9Fc3-{6w!$_+v;});7j$J~c6D}OWnpk!XZUmu;yc9Z7U4TeQWKYYFWc;>mY2I3 zM2)rv32lcuw)+>nOu&qGEe?xjADtKrD1Zwpy)^5N?X@C>OT-I#4ksy+ePVRX zLDOeYzd&H)!Nax5O5F{>%ZYV9@1> zs{m_cyrp?d1**hYqW4cK0QF`$;8gATP>cCBI4M#=w4XbUnm6#%<>UgmqMt6-Mxl-O z{{rR8djnhGHWnMGvomW9f38e`AQoLy!l5rW^-9>YQVZ}gyfP(sOR=N8nV$4lvZ`%x z%70RB-5`7SJ%*@LOJ+O}TfF^8&>0dQ70frxWnD%b!tv?S+1z zW^*uwBfkkwc-Ht1IaX%NeOpqLx;0qjAmUJH(gpv%2n3J%KCAS%n(#OPBn6Rm4w`K= zzP%2i!WxLNg(t?$RXKq4x|eb^WRT-#zD$Qb`z+c!UNauf&WW~^Z@-mQtLNQ9FYhx%7~IvZ4`3NBgE;{zG76v>a7@Hw@S<{q}jn`@gcUlCmi zPGWHLm~$#~1(T;*S*<_=dN$1Im?e_1oHoBE$Vv!fZvHW9Ze9hUAeqOZ%+P!dWiiM@BN-B&=B!34g(ay)kJb&zj zfWUzdsDqb?IB=DpFsmjJdr#W7QolDCk35Oau!3Oj8j&FWV>^!@|3pm{NQ z5)pOAk_;2|P$b>*l7pd0HT@#2i!DSGGEVw}t%6T`h&<^@?sEGSnRH0UFw8BcXzHQf zcWc>oEy8&##CqR@jg+1k&>r@%(tbT`4nRAXd-Ap$l$JYS_82Pq&|tstba_j%M>Gum zdPGYYJWIAqnvN5(v8bZBIw4?xfF;)ej9vOX%WT9DbrB zTKmU3wJd?0^ma|#EUI6S0G{#5tSimunZQuD;Zq{(XLY|1=5#K6yqeAg$juTiyLAdz z@T|p7qyvNZPwxqIcRx$ACQbD6XntRV%_dZVgHG-pS9$s>sElUO1z7B8XO7fFmq>7 z;CE@j*_3{{{kP)uKsH^#%B+7-F3gdIP-h|o85r5Ue%_8}KhZtQK*jxTCboZ~`*9>z zz*=|OceNfgg3E-7-KB&l!!Utx-D zoOQB2Bw;w*JmOyCy-6Q}O9eyuOP=0yp^LAN zeehM`MRs)}uFGu=)=Z?b#XVS!^KKJs*dHIjk7q!R@6+mrKfb5w(k6HF5ERotGbHkX zAUQA`f7X{BrMwSyfE1=sBlt&45ze%DFz{j)$p(3 zZ8ejJgL6BUmhCPbA3e9#Z|Rr#FL_zjJ+?=yEaz0?Z05^OBml3xaq|b`&%S!Dizu6# zN!13PypFWW1iLut`&mSfTO~1KQQn3^rr&MVj#e2qKEpsr{y>kqdR=K-``Q8^#QfmH zpKU;#t!W+@#SLeBXI6(yhvfYU1m)A;%;qYf$VsgMDpxM%b@sKe+)N_~i~xJCC4bTS z9TWj$aEHOIyBp!=y;RgiXoniDvmOf73zdM|i})j}P3Yab+e?|)iyf2zQ(WV&8Uw$P zoe84P!D8s}x3yNRLwjIrvpBqGSPlKw0Ya7M1#0E4gjA0axUYlpR>#HTM;~M*p;Dp< zwy9$fiw`owQ}F<8*O)QJ;zB$oynueHZ;@Nox@Fq-fYJ<>VF^{&vy}HZkA;B30Q|UY zJv9rl5!>2usxD?Li^VKxGbG+g^ohot8K4xS?wp+?@WA;6B@R3Ab^pGJt9m1?l3ON6F^{b27l2(4Yl91{tuK$nK!_y&9ys!j0pZQjmm=0KDid2JY$(fZ{)6V&Hw-t ze2OBAHALpk7ED{OP`3pokjc%H;)7u5w-#dFNF0Jk;JpsCULZ4iRV-uj+WQ7Y1d0c+>Dq$F7S)iv78232Bw&(5zG*n@hrSv=I@QCM%*VS3{i%~(r_Ah zk8Ae`pXeDBK@Wu)!lpne`R0)o8UlAm)gw6wq%uA~m#b|0sDMr{$)j`#a7V=qEIZ0b z+XelVzdqR)m<_ApK3x20+XG<21?t!w>mdL{HuH(0Kkocm@)PoF)))~B>S7aioA1!<^_+M0cjI~U#8ZK**{8+7Vk7C)fsCs z5bUQyL~YM`3Hw_xu(41YmV~ZH0B1&kI$ues+7{avWdt4IJhfb>O2P|<`^=py0fbB+ z(k@kNR-8~S)T2Izhv&jWGQB`+##8FCYCq|t5kP?%$2G6It;r`g93$^;_Z(VpR2q~? z=MD;Eg64nfn%3Ctvc8O7mQ*fl4O4WybU;T}nzMI1@4|HFPr4 z%oO}9Ok!rgcP5lGZy+#6=k=Zl@pLpth7|MAX5mpOi&iBEq_zzZZ&ld(vqhz||LS8+ zFcV_}Aq1#-`VN0S#14qx0)4o8=U6?!hm1bh1~G1beV#kX8jnhl)fB{lXO3mHG&f^!r^Zeep+X&uqFvU!xyJ?e>72=xd>!heDB|+x9Q|>zGlR z<_IAVh|l5t^BSRj2qO^=c(GfE*~zO!H{#dyX)6VrgehjJGZ>f$rnpppQ@(`m6Q7pe&b0YaN>8uZ#m&Xmz}0iCq+_ocj2p zUr>KJ(7CPX_5hV&*`Em$faZTP%oqziYGd>U?6*X^!jw&ARua5v)rgg97C%oN(ugW# zL54RYw|ZHmSp2#nK%Es4L4RP^WOUJBmOmR26zGw+Tg>T6IF?$roWyVeU@qK%RCcLl z_9r%+v@J8?{l(880|_{~q3_1W$j3Ll^THv6JuQjZhM>UOYajO!)U7WjS4&NGdIHFX zJjo-t5HWcUyfFRn4Li>U8MEcC>Z@8n+DV~`P$ZTG#L=gb;s}=>6=rTVntgqk|~ODNJxwgWSpj0r+Lm*h%T*wQH>#P`Mov~Fw;h7W|L)` zaa$rXfH3}i#wH)K`(VWQ(DQ7HdYO2xYhw)X9meK;YT)vSd{~2GIZuI4mnEN)RAJn3&3doJcNg9_80VphFnRETOz-pQ6R_AfRjJ*D-zf*0`e6B z%RZ}FdR|=|#{HPHxI*%ps5+~tfBH<>M_m_@ahvsNhr%MS*P99uE-cE7ecrG3>3mAbwHiWB1#fshH7Krh zFkR^2$q%4+fNh7~QjaaB!+D}UJrC#Q?_ugQBIh&BN+IG-guCZRfT^QK_1}4bq$uqo zuFVvNq%pAB2DaK4*{%&#iq_>P_Y+~>H;x0UrG&mN&^1eE$Yw0V*YH9VB{RW+Fcv8``(qOVnJUZu+xQTJ~sNviYJm4Rwz;Tk~KE3V-ev-=Il+-?jxtymyzqFc%}TW zRv)JGzp5`+86eDkvYH#~T%bm0LbDDD3?|n*=SeSRLFdo6c+OOZSRY_!HV9_eCZL*6 zHTH;5cUUVkknVs^7j;!{($}jEyJL61rj>^u8&lWUCQ5|#3bf^Yi(eMKGE|0SM-|ks=g($0l z{{rA%1`~=&wo{fJHpFI14zoaTyri!Dp9^#Ih|j#~$CBt6+BXETb#%1p6I?+OW*$&2$~{ViKufwcxr_Qh?MYQiqXP3fZq=g*WkA|hC^PhD3h6hp+^$U92x+<=op7t~_Yg^JOh zbV}vg$Y9B(5<;z3=SR1&)dwxF~agdIAEna&fNhZE8$Fu~Rb(BvYILw^#s;xbuZ&& zYPa{&bEWWG)}ld!Kd3X)8T)}7V_a0?r@sFn?+1GVG7~-JOZcfhw^I4-TnLeSs!`rb zpKKk1P=KS>v(4{p999uyTTbUjGzRFw z%99GL6J~ivUL3akdu`CGOz5QP=ig2@*}c}oI>jejQ}3Fw=_~K^ggw1j`sn9n%v@=m zq_!!4zr!`@5vQj5U2+DNJ_T%2n8L#t1*@Ik&Gmem5rY_C`j%|XNb_*U)<*NpILOC> zaJYFY_U%cO_{~q)dV{S(bw{1V8OD*&ZHtiTxk34f1*Z>-?8zzr9&Z1TcBe_y-6hKQ zV%4-#F!So>!DCJO&5jy8H*(RvUvX@2-=Vt5rTaLGUUCXS&_`+1JUC&;8_+xP-bvNu zZM)L9Z#Q?^I{p|n&rDROK6@Ranjy4RSfZ}`J_uv_Evwz}-T zDp}KeED=wJc1ImXp9(k2j6EdOXBxfD-?g`&#Y%hdJW5VyIrBVOj=dejzV;HWdkg?$ z6R8J@zr~JlZcuO)kp#ZX;uCt?^s`e4D2Df*{NcX1L0uGshv=0tYjQ8n6t;__l?`m& zy+0(=>wv?Fmav99r(4s{MQB-{HE&l=>fJq&bT(?pSlRxi;}K(-`1-poWc_IZt7mX| z^!dZ@PygGk8gZQ)W6leEXrXHA@1!w`H0k!RUdaQ&@dTB-ekV`S=pEfZh0%`-F3+yr z^=(Hhuc<~}slEbMZr!@NxZ+p573NP8vn6$J$XIcwmJx94YUfCK&&-T~Ld+Hy>Fl*v z*yTHZBLOdBr&m$Jb5oOue1y`|)`-xW3}x06Y%&nEY)klN-=(e!Wocl|0F!NxGz_EQ zdOWr(Vo3@sJ{*-bGoq}W?-gU|h8w(ACaGX;H194)R$FG)xV-i1{XOJ}FyYcw{y2n?@5CS5w&51oz?m`m z?;@n*Q<9)K^f0XT022pJK%|HJ;oi+;qAE)sq#C{2-NwGEK4P-^)ykHshMoCohAi>2 zikTOTdkv->e~BC^2EY3ut5L?~Ce@p>bS?x;W0={0wr4e3@Tx1dE;B+Hc)Fg_&3Cm~ z_yP6RcFJEwYCL7|)blQPoAQcCOwvQ6@>MiMDJYkfd?tJ~sK&Hhcs@3?8~M2^$*ugW z-f|ipygZEFy9mjW)BI$T{o~W+p);x7UNm+E=o`6RfNFD{0+>Kh*hAXArD|V!U!A-b1sXD0`2$H-s4h=>IeOoy;13k{emUeo9 z=r#VYsgF;}suv>JZ$QJ;O7^J3rr8jQuSnL-NVGTRHBlInGJ+#&Z8@C+(iW}+7QeaQ zx5$hyUs(;c*}{u)y*Ea3>gj4hOZHrlPuqA3*@kKxTBz@N_>hOD;vLetny`^1ofl%UV+o^f9GMI?q?k60xU&yp!6=|*EU*?&LM zu(IvkVW&B|9-J}NhNjp)YqO9U|4`v#rGO|>F6MNU*C))>9};4;h2F({20#CPdh&-?G4M~ zODg-%npH%rR`vILQdBtj#CC!MP3!CdOsY(H08+5;VKE_r(=;cY{IK{5iwa%IraMm( zr7Mw@dQljy;>&(xFRMtB+RAK~DK?Rd_6ka|Bx~2T=fxlM@}LCxDsrqf6ZdyFmD#4F z=|9wm02echTYnPU6S;I{$hxy6o!5lHI(U%!1VwF}L2g>|j6sEHDyxWC2rE^+;|H>C zL+BgMNHv!>$4yOU+hM0Q-|YM--B`;$%sw)^jT5Uu!O!@8qPLuvyWoh86M%q_mUbqa zfB4<~zlT4f{6!z=C($1Mh}VQA1ZA(Z=SX=asR$@XQW%zKrDv_YxBd#8x_Z**n?-gM zm2t;=R?-;foXcwZH~Fo_lVjv-!c}bD{M?*I0uUj5Y|eYfzr>-}R6QR zEE35)gr$vxD{SCE^HA*ec7g+0{^Z1i7@#|!e4{a2F+r{oAGhzArG`Xl_R_p{GpGp|4wF|Zknumd1w0IqGhjbGUMvM zm2YtqA(H0;w4if`)&@`z(Rt2!h08k+zlcA+p!qvpY0^?R2-aUQXM}#ysZd4dY59}; zsc*e7gj8smt_c78T86N|g9K*WR9X#`w=r6V;Q zb=%qeUwNJI{bt96Lf(WAu(I~$KPvKL_xyAuZgZ#VccS^{XW#X@)$ywK(}iw${!o6< znprDNYhzNC4@Qo|WTHQZ&J(jceXZ^tpT}Yd+&^QC3+jS{UF!v)66^%NIeAC5h89WS z8Rq%_jvNd5S8e#npC!c0oO4!s#}_~dvZY^NYrS~o5nW|^GLyEknPH8`Ra*&-_ih@R z(|H&7|58uw9-!IQ2UgW-fVeQ>9_0;9y5FB&9LjL<>iDE)ZS=XCg!o%^JHzS79x7-! zT`8N_DqlbJHAhs-|8E{o-StBSVoy#eVg=qUzh0teX#isy>rx1zZ~&YAlYTOC1fzs& zejI=2j${4|QNCqOWi$nMiIHWZ{|be&<>JqxXbREXjIn;WQH{1z3^g-lJEj3dGQ#1! z(8?9!A_8i8d-;FW_)*I@;5lZqUM8S`XvS`P1rJW`kP)kJTe~E+22s4yb8Kg-wnyPr z>WbGt@aP%VnhUKFT7Z1|z3>r&24r_sKFWUTiJqt#jh|oFgZy{uXQzPe`R<+6WZX_7 z5`*FY)`vkS)5Bj8lN3o$Ae)WDjaEo+nuIJr5rqhNFH1FRvAX{Ry^ZO)F7 zl(>ONWSMSR$?UhT^XD+3p>v&;wbsLt$=;55S8LAAi55^9LwNMHP?z4Kd?) ztg?KXmL`&bi=7W_WXOM)(XnF~RDcB`gwzF~^+qv}DDabm_AN>ppV#U(owRS=Zk&Al z3Q$yRi#?Y!D>tlQ8}RM_+a+pJrpLSU`#4EA^0>D&YAxPf68Tpf=2|5-ITClO25URT zKy;>mu6GLgA^x{WV`IPK}@#`sCJb$i4c4eB2V<0UyUlI7f(N?1ujI*|nrjrnaB&>P} zO`<39P-t(xo7bny-_hrQoHf}Lem3g3E-zKxip;=+Hn-50vBF;pSiq z@ML-_FN>S!`izkIEP6I}LW|YOvO3qQUTjY=br=XTcFw>(oRk?PdIXSnfEPoc*$DO7 zy~Ef%ZtMB?j&^JK%Y3kVNx}RoA1*I0vH~BKSHa}w3~$kiJslebD8!09;U&#&l7cX{JwcgU zXjzMCIFO5kf7}J!poeeumC&5cm%jl_hvV{*pfS`X-@B}uRbpSE+}HwR-m)gyT&y&? zBdw|9nfB-7=E|_WAI!Gl)edl&z1e8*K=Q+o#s_Lu?+9fV(#>CWTCzgavXF>)XAFs1 zz_BhZw1mdT<#lvomoukm3Z+W1+z%rhpK-uI8Am^gMid(o8@gn!5odNt^w#vrKWI8w zTNg=k%0J;GzWv|f^6_TfJ$CzBwcVXqa@eEaKZkFpXDhEP<@#>jMg!ZVIX#rrEo_a2 zTJ0XzpFWWnEp||TY$0#)!cWxM?OR*G+?FDH`Bdk{n;X;l#$*DV#MEX$qAu>7Km?KM z*!A=SoC^=pf-?0_;45w3o0zBb5pGvZZp|E}46`EP?@Z!QCuw11jXE`?gm0dBj7EtY&s~N>2jdWIw;fh^CJjz#%D-9p5$SZ>nd#|8m*z8Bc=QH-~PJ)lnR!S0;odD zr!GoxR5`bxZwI@y)dBp*pI={#aHJnL>^~2#?cc=0y}CitvonaLz>HiTjh=TDs_ysg z-r5cox)b?$Io=vmw*I3B9V8^9Jrn?>m`}z4(WS2Lk6Q(;>;28c9X0sR=`ww(xe@v9rxjp`kW2lst=pRkb(3==x{0z_ed zTVxT|{>z&(V;3~cwxg^eh!nfC^(>bOX`J;_`s>kb$>ftIpr6RvmHg(B8ZS5~Xk*T) zpiYW2*eSnx=`7Ldjpv=1>iX}=wCh)-GD*=J9RN~n?gW_@Ian%R&7OU-;vL}?ld2dA zVIQ4-(Y|LGFB)-fL)*H8OucdCPvOfjKUnrM{$(O@@GGwAKU%F&0w$5r=*O+^_rCC> z01g)2NzP25zIWmxWQe7Rk5YTqzEnSJT2?FX=Ali+`>NcZrs-3^ChO+HHt-$+N=A!+ z`cfC%uY+P+%ti(@y?7|wC142oogv^Yef}()qxuJs)TI9ZOJ64s^0la?>8gZlC{VIATbH<)L-X;$^HeIBQTLl2WcWCaMQwMq9AgwPa&SxSm z^P3K^_Aht=eFt}abnPuv`f2}_q;dfq%4+x-)%{6B(AbmXf-4HIUsX@F26tzST&7*+ z%JXIoGrGA|DG4~eI#IIkn*-ES#WPaDtedi(!0nR-$(wdpkz+wDNSMm`VT9swDKWcM zP4a(>z)K|+gq) zS8@(lq3^r=-9L`i8uFm7`T16L!Q}PUNSAuU*ja?-^?&cabJ^o*d$*MG7DEn*v{#9) z0gZPB9^z8)3~V!g%&50!VJDuYa{l}nQ?5OYm72f7o1o4zsY#vRLCy@tSMxdQ*eBbf=z8aJ6$?a5H z{IPwS(ghu6*;fxkt8$S3Z+Gee0%O!@2zpVj?`cOeXy(c1S^0!N9Nzd9u3V4l0Qrs+ zsbLG`0%Q(o49kgHcHw$~rXK4wTqT-~Yk}<30~!LII*F+fZ2z!+J}~Rg<@KR|5_`9t z1jrnZJiQ4$vY+Vos-qeMi}X9C0AMUFsbA{=Th3VocAFnx{CrC2XzTo6lfB1P0BX*r z*dc=jU|&*)psA0l0RjdVO%a~Ial#@P=pSYZS zc`9vpt-5mV{-duz?CF>MZv<-a$hu=1IaMv=Pmg>7E+{m2d%A4nd7Ak4h@>XK#F#OC zP~a9!Ia1%9smMY?Ry?f6yoPn0uSatw+&XowWWG~OX%tJBWNonXV>Bq^!C_wy}hU-{E_w`ht`n)i0m7!VRK$}vRNktsjW6C}mEG$hW9%-LDZ zKJ=Zbb+elM`SRD=W8PJ=_W{wO#{k!M0Ex^i z5;9)}xpANOXIeLe-um?G{(GtmQIx!#^dcVS+)5ES@A{mj83EMuKsmd}U7j0^znCj=@(-w0ETb^s}Qixyl58ZG3q2;Q7`p1P^EmuwBn|^8C%7ekKw*%@6eL%mP-8ECY=wMBSB1n@}o))DETfMaDUuo`m zwHwZ35J%+9`16qNy5AFYdFQY)xV}!K*B*fH8QS|2LBl>P>$&5=sQ?U;dLlyB=mJCf zErXON=N9z`K9lCRG{UxBY-{my7}>JP$sgyL_N#FyK2@7 zHr&Rlni|lxyosaAE|{M(86kF{BqG9%5Vv@@f*mIg-2#1HJpc)7-hdJggPNkQSOCKa z+H{Ak1?H=*!nq7;URSqaEOyD+xjJ5S&~~3KyjCGI$$M1(B2s6B1;5yX@hl*Z0}i3NTi|{ELj~$0UC6(O zF-tj2g@$X@t-iJ$&8@H?N~2+5db_GXB(W>={Kf7@JcR4XU*=ZR<6;+W;oaer@V2d&oL~lUIJdno2m>!0e=eNh&}h zz8@Ucj((6iq*0B3u`!MZY~h4?r2y95l=P z?qewh3ZyOY$|E`tEQF0g*ia*&YHD=DGdo_*C6PNF`@XqZtF9J?6xJy=$(v94ts0Va zx_F$63;pQv1<*K^50E=qp0wXHlVhkI<77x;)Qz1yY=b>4yBF*{%5;afpWHXm+qf&2 z@eVXJj7Fb4jzZ{sJCpe?eGsEa`R^uh45NoLW}e8P&DU-~D_ILA_>=;@@pcEw8n(1i z3t_S$BABhN8o9f)?fI;uf`tSu)U71}IWub0(k8=ZFjcoq|NY2VRJuJRG7xAv--vO7 zuOR3K>L>NjElidTY8`e-!R#8jzX(tEom2G?uV%YDtwSR=v{hl zLWhH(=u2CdCPn|A7!N`O6)jb^;NmCXjj)3u7Wrek3DQ>sghC^OT7D#{j&P-@3A@Tz z&1BqB!Y0H{bgg*I6Q6_&fa?QO6={T!waFJH9zP(n$&Hl6nDPMnR^~DO{E+U~%89pp z=P~~#y=mN&xY1(+W@=65Dym{oO+}1T%*I_i9x?H=5+)IG6aNtiA@~D{i`Ppn>HfpO zl)b1oj$sn|^6ABOT{YQ#jJ;|e0NR4dQ9=2W?sibZD9KY7Y zBrHc~YqRi$$hR1ql+cgUL`|Dkg_vICZ+n<>A^v;eM~9LasG~Wyzc;DG4(|blzgLXJ zd(bvgPs3qZiXX*5Vc^_QyQA@MF9k-u##?Y$0>KokpDT`%`Cby6s%I-M9uqrXm$578 z0;rstYj!;BcE6vYkc5XS=XWMGHgnz;9}EEnL#Ietg&DhDxjIk;#Hr3S*`Kz{xjozR zIIrh3FXTZjyk(<LR$!dSUOP7zMU*6Tgq>&?7>=IQnFJ_dVJ<}RX9)b)ihbqf((QVfzXm9a_t?#*Mr&HUJiXz#?jsV>&lQJrS+W<-roS#RI2$dEKX#OIQ3$S z6Ko{r8Gm|;LlfG?UM!D;|9(21RRmK%|681TA+Wxe((#}YIaT~ptc5J(v~dl=iM|)= zGokO61#ojpN^|N#CT9%ex4>i@%NfV_6|y&G0}Ux1P{DdDGFu*Aq*!qR-dou!84K@? zN59N$Q*Fk7Fff&$f+B$M!uAog&(D0e3n+HEm~uTHLScX-1FZWHbo=u?I4)Ke%b857 zY?P*wltCj56c6(IVz2g+iHv)WAN?VRS6x|OeJ)kVz-#VQ9u)WHvS6DE$&zp@gv>GB zK?Bbr?Hv_tjFq5y{G1#8(xbLwdlsMD`F-IaLnUaXFoisVU2K>a>N~e`D{@p3ioT!q zt+-UCS|PBuQ=od;`OS;^_B_8=+ezE=8f+G%V9YPZ!95cVDgWYtl(!}RJ=l4$d^fku zdpXT!U6&!k#NWkAL{$=6X>hO2vTPCv@aD zFDTnis*Aaax0^;A3(z|Q#mf{2_>fdDqDStoJoNNI-ajnm5ho;sTVtx$i82GuID{OoYT??BS7n1frJ4G^s|0f=fodp*QQA-g?w=M^ z2*~e(1k2z=mKo^+qAN<}L-A<`K+Q!qn@)|2cjs)(jr?OZR;=`fDf&PNH5sO9dEHuI zyxIlVW>YDU#7T4N(;Q@OezQ|e@z5T%Hy2el} zC-wCpu7B4;_VTpAP=|&HnK6Ie z;SD9nkF_yxoQD$rcBgO(&O4H;DwkKFE*Y!k>qZFb`n0pgV}rho6E00r73mhBUVSj` z2#Mw~Yg;(1MuA)0BO-6ziis-M0Fd#~Ofpv+RZ{e^UIQANO7k%NA%_$D5kvS<&__Aq zPI*P0ePnl66CPCk%=!nm;3A3TRhEd+=%)By^ERJZ*FXm0HV`nh$Z(q;J;j=9+Ff<~ zdqD!x(EXg>VS_L$&yRj4J70EStdu+r__s|_cl%55ds>b`f_9DwY{6zxYfKLEM{%Kw z?dp)dc^T-O+d1)j_cq#ENm2{td+jYqzo)F|)@N|uP;VI%cbfC096HD_Usei7Ew*^~ z)-9!H8Mm(s-dhNoca9S3-TDwwW?ATJXc^gLjX2RM#1to+5@CHUmp_m-P>%#1Wulj+ z()?stD1fM|Y_m8bh1}I(Jn|+(KAr$u*iwq9`?n@zYA&dUn)S;t-S3+= zpGm<}&J=iB(3WwSiABsNlU<#Au$f_(nO7b-SDcVeY zN^-sal!8iC&W#0jkW;YypEUCRa!+;tlK|6o+hb2u55iSdTGJ%SsUB@cV5TH zO{e8_1;aY~07|L2HQF~n`r$4 z|H~xS9xMSsfy_CTQ_HNSLz(fQMpAtq1erCY!F+d`y&OPET>8A|L8)gDoJ%&tEj9H+ zT}A!-UgY(LLG$3GVMP;7&`T{r_^W4`LcsdtZMGkEh-lwCSy@6VA&CI9nFgk%+XsiU zW-rufA@_1>acWuDc{4yl@b&L)R+=yarT5YBHJI^wb<(Ud$^jy+ptQcgiGUg8oq~gL zUZRngD>iFRkA|v_JY$sdd@IH%kcwj%Zj~bfp2Z|>$Q}_I1TQwZxobdMkD+9}?iS?! z{Vy=+34WDJ?e6X|zo`uR9h$waT3qWul{4*~R6U@njtylV+q&L4-70H#-6onWc~>&C z=+b0Z|F|fLO;^?tGIRKOl`3QR^CI=H`A-^EUHT&p)lUsa)YamtyVXbRsx3wWrY-VV zy6>D#rkm_E>h1eUrK_xBBWZWaRcMJ}ha2oRff|hC|T#e*E5Wm19+ek@c?n5Lt-d zm`w6P|IuY!cRl~VpWUYzi-pC_xUVQL55_?;+y+maR3%jc1H<IZWc-3IG6yspwe8*#%A6xCL+JL?Eoiy4nJhNPgOII6Pij6@ruor^5Sxa zHe=Y>TXL(I%6+?|C?>^shGZktS&+`lH3#`sPNDb|+CYh~3(h4IR*&oY@1xf_25Lca z+&fK(xy!~ChK97Nd^O*|8nJpN^VPCWmBnMaD?>quri%_T;K25e`@ALUIK>uAw=LyI z=Jes09bZtJD{5a~_4@wZ!r}n;ADZ3e9!nL=w zpqO9-n7T~r&~!v%w}Xd=oe#3#s`nW2qa`c?b+Oo5ilLY1R)It zT7EzkTk(T_z?suS-C2y+{Vp(zC+Kc&?S zWgE6%qA~GbeDG{4rZP1A^vtP#?%5eDU32;ZIzVjuLDa>p8L{$^K6-Ly><>rTVW80k zR@kVXtC42ml5w4^EldDec3-?CRs%AEJEGQo#2{o$Qrga(jgSfxO(G;fm7LI1)&ueS zW=j(Y!1-KH)!>m6;Zrob)}lDQ2gDXK@AZ)T1&B zkFBr(^+b+%>k9&0nLCik%iGn)o7#s?zad{l>A6K3X_89!$k;f#e`% zx1efB?vT!!lZyAT@?yGl%Q;Kv_q|cn&`LBfwX;J`bQckudnel+2FTM$2{!&^+Na#? zQ(N2fV|gBP5LW>z$yi#5W*!%>(4XMzwQctC*4^*M-hXv$qo1y6I^P0ilRt2XC|jRu zr)13en|`Xw*&fX?0Zp7*PaGuzRF9X8-wcngw2SHSD0ygG`NHQiUS9pS*0!$^x_Eiz zp5QgsY#qUhI-R9ZJ9z?TsM3jRJED-abbr204w@riHUxfW6u|ZW^wC~{3)7|j_lfN{ zziI)QY2+#V9B4p1JG54?BJm<^QOhkOjY;;Qw2!TWY@c~k5fCgNJ(-+3n_>7p$_M*o zCS(vVyYnX*7VJVoXYVbn5;E_cinV5a8y`B?hfycBb1E|pyH`1PU-%!Eg^Uuta6qIi{j?Rvdw5Y$f_B~D*}33`P1FA(-LF&Htu z$e{Uf7Q(LaK|*f#culM1M2TF_*o|tx_EUE(UGxQ-v5~;J$$3k22UeW#P$usH%)*g6 z3d;Xv1-p$i3H}wKdXI#AK=j~?RXiRfwT?5lOVaS&ThN2a#!{Ksf)7MN9Moea%pk4D zNyg~-E;B}T?2}V1Xg=mm|H0_46TtU=o`Kgf2$Z^g#PC8`dw97G)lioH73G_i!Hkc8 z>?omAEy}!WmZ;-MTe|ByqZ!d!_Fq2DJLylK$SUSN;h&Boi!J{_D(+jC7V`s12eBIT zS?}LT+u`x0{}ph!aF%=6q!K6u;VO4I_=BhuK$fVD0LPK}^>aKud&#LY=7|l1on)V1 z{HguM%F&W1>Q!`j;L?GPPj^o2F+XVm3+_uo6G;#aSG0(UK2z)-M~?+qfF7E!<}M&! zjqsJ|X47bbLdj0lYwPi1n;nStYYu86gU;>}b;73Y`F+`O7UDYiwv=JSj`y#b?@y@; zVzL{y5!KWs7uWK5HK%MFore~S8_jhdv zmm1ZEY`0d~;faH|HU8?l+F}fz08#9Dhqn3QiO^`u?qU2s)-cuZJH9Zv>-+x{#5^7k zX0m`6tWs$z>T_n_@(ub7F1!zm&$6)D=ggBN%$KY#5hE#kzhzdj;?Y2PbZi)irMmVU zESQjgZ~F15vDYPC+vot2Gb~mFW7Q21P+8`8@PI4ix9h4CA|*-WXy*!qRZO%>4^fqv zFzg-7fl<}l4KfyVq5}~8(l4Q3l;FL7%?~(X$jGd zDMOq!_YXrJGm^9g$lJ841PF``MM?KAk==Nw3cDvPs)O_U>;CWm$!0A+J*zI3+_@Ir z`7>mfWn|d@_r|Ephi?~C-{FBhf&K$py&6XHzczGvc`crp3^aY&V~8GafY%=L%0<7y zO32Okp_4F-Q)bW)9d%Lpn3U_3?^8#k(l1M(9XT3yy$cCUNTD7J=l6NM7GGMw zQ+K17k6S)RZw7SnogW?!Rq>hIY>c0{?>(}IE`!(Nt{5vQ5}NQ< zA5=wQftPX{B**s`^!XnI1bN(Z@BPslkptoOWpu<6%p(8TQ;vjQpjCextVv@Hy%1tM zJi)lryr02a1nq1Muww%85qJ=sd?BMD-pzmE!E#!E*F+7h+Sga!{qBpdz#YYK)jtHi zcKT8>cREc?5fB&MzSw;HUPE2}l}|yj%~MX1ypLbQhLKJ#WR6KO9FX=c%zY|IEfy&! zG*msN<$srcum*smJd+C^{U#<>2&T89V6chtiV=(-^_X;k5@I}nMk{>7Gw*w?e|&WU zkD!fO91Z2Q*HlA7s52X^fs?~cWjQ3T}Eci?Zu z-cR59H6*w{aB9fs4;=&UhPV&*6+m z{&Tk0ZZvS_SRvctF0^0vz6~__^~;e3eicgG>m_;Bc{?TI)weZ=U+@%;9DA`>Xnth= ze9(*CPBTOOR6c4*sEA?(s+P;lT}LmGI;a6eK<+}D^j{(lgsC_KSexjciQO0_gf5P^ zsiu8}B5qO(AWA6Z3^l!?nF*(B3I>tAe+2UM+hk6N%MoX*Ke0PpG7O)coNd*s4lv~w9yIQOUYSz$>56EC2woTHMGryC<3&oM zPtAvgszNSE^E(uuLi-?)YW$n%uBDXve)dBG2(fAj)TqaP$g)lK!E|Luq4G!7_&nS`-|gm?Pvs<2ozC2 z?a%0<%>yM^-FsSy{8x98x3UJ_GJyI?vKrAEADMz5o_hkg^rj z)bg>fgcol);frDMzM3%?()_;oZ6QOyWx* zXxJTKL4{qC0)p(FSoF9kT<>sWd&0PKj?q;t)Ku*)E1pz?c1%3)-5GTZ-SQgL`x;nT zkx99vFeIWv7qGK3F#YQ{7ckr{sX_ZUF6wn*ZoaYxQZo~UCx7OV=khTt*X$NHb?j|E zwlD&yq-4dAHCa5Jf`cC70qCbVjvr@38N#|?Fz8tWTcr>mXU}HmhJM_M`J5eol;`xN zsvr!}qfDxx#^YJuHKjAm)5KaPKgPP#ZTYe4w3-jOmp#)K&DXqH-qeMM@~6B>a=|co z`+}JJ*2UsacpMmVUdazVuM)fuX?BT2$~hb!BoC5tExQax+Rsd@GBpZj@P?jB7T`N$ z?|!i|!KHQm8t`Ya)ORFXuq-}p-0ww*)|8u`S%*&jAdYw-h?%{ zYT1d|+UjE4b_8S1&Cv{h$-o9aP~2(2pPMyN$q8>kCE9 zOzk}o^iD`2Z9k`qe}o{!WvRGxStid}&Jz%HeXcRwQ984D_n2Wdhk^@AZ+=Kf!>3>_ zKoam~aS9KumYIRlUlntgJwl|I^_QpS=dI(1bWvG$zkG#4-}_fsW1bky6@LNpuI;X} z$NJFtx%!wwb`nVWz@{dJ1xv2>=2!1VGnMvz@K+5PKW^$^D}#&w2kJw0VXDaEddCJppM;befOS>tWkVOZu>43T=Fd@(-DJ;j z8C8?P9=Hys->xPPSj+B78bTr$iTA93{$49>EUKz$|7JE9zsok$OX=&4+iGDApB{K}udIGe9}WmQ-&EFs;AI<$uKV)ygipb)Wyp+=T}+T}ifV6sr2;Q)wshI-CihL{@j z_4wSBJa7NJcl*r2#i$BtrmQ}yF#^(DGT&V=h^qF{15HH-$TRrnuKC=&mNxgq4UwZ@C8-4Fp!8gR}E(Ci$#J>pYxD zxLn{MxoI30l<7tkk6*F#wsikw-o$is49_X?onwjfg!FY-gY0?d3FbvDrPl2&dDs4}_gOE=KL3C|j$^&cF~yTF_(DZZ96?vCKudNvPuKuJ zj&8L925DBv9TvF+cs6*{)b;D74XBzAy_+112JTXAVcfi%(6IC!t^~?4SF_*OAaKe7 z%mhjm5^t40jiYZ)+hbzHytm;S=-|y;pzK+}r8@=k&n6gNS~o9%JwAs`{VhSG=g*1P zdC8HGTq`q}TF2jpuqSzq8aCyxkeJmsnn^I9>^zu%gm0vC5`+9y-os&PX9~IA<{PGO zKjlUDDG1E-`{z-JomWW4ooM@~xg_b`s;(-@#pw2dbhY>Pj5;_WDfhZ8fH>kO8G+8Q z_L!56L;lu$F{h>Suy~rHEm5b^Or(Is&~Ep|Gs0-T-ic;Ad^K5@pS*QlHFa8`xuLi* zvK0pM)I#pN%?~}~1b!j*L*w@I`#!3-gvOGIP(U&}9q9^Q-N}f~Kh;$r>rmHdgj&_D zT3d>p#6zK~2R@|1>OJO=;#D}0-<_il+tYmIgmDQ{wqBcj_Dr#4JPPjC0r_5F2^-d^ z1%C!SVqLKP>7uhtZ>nY%Pj(WK-3d*za=VA&S0`8)fDS}H4(-jMUc=?wTj80CBxnQ> zD|GPjoH0}7@$E#2dh?cTo>1S|hdTYerBc_#TU>s*cS}QyRe#O6$bJp*cu^b8c}=6V zdB^IH7<wV-A;Y`P&Buy-dV7wq&oy!HkVII>vnAMY~GPNVo!Q z8Ng5Kr@4W%vHgIskzEDzk8tSQvd8ZVqq&6*dfek&N666XY*Vq-ad5`<{ANMES>!~& zO&%-W`E@+BDg1c^9iAm(xFKX>5%_J#k|AmzX4e8YL06@Z0K&KUCM_L*H-_3A#?BLV zgv(x(E|bONzaO+aeLAgg_6`k;ou$77C#NIDE+GTTVtZFn48KONpKJXL^a+uUiLmH9 z5^rAhKX;q3XK+Mp%Li?1u8tEGDnm@xkzYA z$AnC3L!s=O;^W_II_l+D-%3=eMBO_v>$@_Q8w_4HFJ1GuZTjwmQ~dCz;i_@|!mA*- zEF6Il^4>KLSk2~y2p1mo2$FtB-J4khys^h=CeJc*nbMP+kyeA?Ra3u4ya%(wXR}SO zO{c>ZUs=yP3!bn}sqg6a2O*nfK(+Dnc_;D@(iTkh>g;gKfN=TZXfF%7kBrq}PeJ*3 zqz;P-YFHSMN9RNNdoY38367F)Y~IG?=O9?Lyp7-oyg2DDjXwDttExEq=q3B!Lgfcv zB0*#eSJET<`LH`wUkLu(62s%+$!oa>O{x7o*u*4tvH4bW^9vQf!A06VR{(%^Xu}F; zDc{+isxRYv?qLDepih%~DN3}d;A})%-!Yez?dbZF9%K7BI5o8rsWn|{VkBBS63ZBH z{??6*!@&_|kVLO^!i3sE}b5~(ip8&z9+A7))wa2t7UEU~=TFBSQ z45#z!LFXry0!Y$k6G}3u8gw!LW<4(i!jAJsm@hRwT8+fp6r(jnk@3novKYGuN+txgEz!D!gzndhR(5|dd^KBw`)Id^nk|W11@GJf#k#;@#JAe zN_q@6v=}TKae*j;bq+)6U;ldTTGi$~ziIOVguqt|WnMY8efCJTYD@bCk{;y7eYhz( z*qxk9Ndqc5fS)f6tLF&^287{adwU=@5#bsqV3kPIg3_NFA{!Z%=q*>syYmS0gfF2J zXE%tGEO1pw!JRT6fSK1}up9)WPg=QI&=D*2U!o0i=&bt@dNBb9>=*ArP&^29f6Al1 z@RYPLY4h-w7WCz#pZ76H{BD9n!$>s~`MdW`+d7802&gxEk`<{ZL6QnP;$Koksdh?o z<0n_)dim<~M0z0*&$**bE$(dG9AkmjevVIR2cO+O-_|EE2iB~bOEn-9JF?Mu#?a#f zOe3x6#Bp?}^jv+c)@SSc1eoZwbKAo7%O~bHJiD#TZTAbX$tmwS1f$7&Em3a5|HYuk zQT#iqkcn)BF~NulQj7org^VTKyLWrzQKJ{6Vpt%B^<5e5GC+Z)3YkB6_;hZ$lPJG`cSS04#*A$&ci) z<&A8QxMw87)Yx!y&~cHjzE+szCi4R{$Y~HJnwYA~$K{aXm;)y3N^X^`TiIQ-lrE6< z!c8pbj@K6XY<21wone%_$6mMx_TT{k#WIcy7F^QQ<3*Bm9kK$6f!s_2 z?>&`6j-9*-_Q-Sf-OUGa^zT#^5{e%*!2sNzRj-fxLPO*uCD&tRqvzIU5!gBoeOjqe zZX8;lBH3^Ukw9vdWfnyX5D-#)^N|!uwy>%2BLLXkR+~?_9KvyO?>2WYnPm- z|KQn&NFV*iu%B`rL~rs~vlT*p`atOM>w#3>vPXdS#$iAyf73>eCO!v<6dq#a2XQ%^ zXjM%U$s5YL33*a?T)vpr)qjW9zX9&PW*5h*`f5)s+Rx`-tzNg#<&ijaCHdm7|KglZ zPwyBZQCHDQMZ1H;Qt}l$lM=j^?@4z*NsEw(s1m&lm#??L9*f3LhcexcD#;8O*>V8mr~-U=fQTh7gz8Jzao zGQg{}_$KMjE^vyd{rg@;WCCLfzcOcSfP|*`ZJ^Kz%fo z?%=o?)o8R?Kydf@nc)r?T6V6!`Dz9{KH6aDoMIhDJ$(D}jUJ25%eVJ9J-`hB|7U7F zkWkJhbI3Yvq@&{k-;_Mc;SxX(jNByy&g(|{j(&>{#M-rP6JldfKC~lY;w7EG!naQ- zeqOmf)>@d^ecaz-D$#M^pTtOFBIT9&cg)Ea>a$)pmQ@_(bCH0DSC^9 zB~hL<_I5GLs-gcnNpnP_sGv3kE_kJPoyOgR5dMbH27fSNoC=x%$=@?>pi}(p6M>2h zxP8vqPu(n5o_(g>tlv&NF8=hql|cQ@i+CT=xsT%;5?UABTC3snhy=_$kSwf$-qL_z z0QHW!QJC}Tfa2aU*;3Qr@S99CJE1_-^}Z#~l}g<)-}*B`(Kym3MlW6Nr`R&NZ^1+$ zmlhy%kVdLyO9&3<{45p>-{#etP5a7k)bLab@UGP7wvSq4XHProv-=J{$9mw1bBa97 zkB)wB0v5&CEQfTcA&g8mcN4|Lw}J+i{)wOmPCQ@GNK3*FPWcQyEP4fS(s`oDNi`F& zxOc$A^l4M*u_=a}ye+5dE6$E{Zy+*oL<$S}>@TO6W4KKQSz%RNXj4Er8@Cwr-X>TK zT%ot%b3-)Mbdc`Uq{^Mophh9B$bRZ=;#`aO`*YBioitfKX#HYv0?MljoPBh7y4>|ZL*)7X4=|YL065CIuPqN$@(JYA z)%M)XJ(TK19)euZH*~V-EfI)d-fUM01)4*yaP#HU4k1hFz(xkluh*>KsM8wJ?+Jd5 zXU0IL_Y(Du-@(If7;3iojLl%IE=kYG0hgmO&%v*g1`~0+#pj0$T>?TsHCt0ysZ8yz zFw#B?!Ff?r&F+CE03j*3$Or-eltZ%^En&&i$#+SgZ@qgn(PXuH8w~$yb`y8W<&p6B zU~u`uWarx_+0E0S4GfTf9_L(Z01(1gQWtkzB}eZb4jE9&t-Q_2V{VX|wg$4y=D+1Y z0rcJm=wP2*vFWG-5c+F?!+FC6X5?XnG=(_#?flhvI5>S7J#hBOx3sY6+SMZ&<)TkC zMzqDj3{?AnBwcqrm3$V|quWsmGlbc{sy-jT9}BYR|Ig@o*g zNLKdDe1G@z{`I`i=kvTA_ql(+>$<+zclhr9Rz3oTl7OPFO)F?(WU53|z6}VRYg{0( zfJ$N}>T%fXy~$hSv^PnC&%`y(R?p%4^u^%j+%KB|PNfyFQgpNC6`LRL%JB{`BVFrB z>bq7Rz0=+cyUjfK_#_MJJ&mHD;k{l)vPg_dvR~%2fIG>3wMg*61z#%k!wVZ5f<+%9 zI?YEil!dSS-vd7ID#2IuUwWfD()Yh`IPAv}#+7D4wYipfn3( zUt-gSI0daMNWeQA=23X>sg=R)h_f1v`o*nD>^nA8guWjj9D5F!h@ps}g+9-~oF!{~ z%uJ9pJ|sqm`pCiqC2VVbFc9~g^_*B$#=U`nz4pLuP)>!yD{Z0=HE>0bEb<}mew;2V z3qhdVXMxipsPK0jTA3lSepeh0PaWk^VwSs34NECWStOKya3M4@2nX|t1{KAU=y63*j00d3a6pkqC1T+_0Q5&Y1d z5bl^blq*g>R?Nh63}(BOCjiP|PIA=JgH?d)Hb%|(L`eZPzZNJL&%ze_K(c@*b^1pd z(){N7@j74!+(RM502KM-s!kkgf-@-S{ch z?rnf>?t03A+2+t!Z`L1_x9$CMKu)g1%5l0_eAV5=}KR0{ixG;Q3!@oJK)L{zENt@;y3LL z@&a-r)$a6Ah{Q$Uisk&;0iJ75E}q|54a43wk-!CXEaj^xbFc_3XOEf{qT~ST?nLxr z_^edmnU6t3Xxld%u}YiT9MPMPudaDt^EHJux#B0Nu^7uD=?{;cntx*7WP7C8YaHgJZv-_hm=HvhlE zrp~S4`YN*og|q|!9_4S880lTH-S?atSGhrofJhKAMf7PB`8(|x_<*ZBnGb0RPfF+t zh0lO)VcId$$^wx38^w*yC>~c(?1L{G+w4_uvXE^_^)*zH+p(-dKjjs3YA9(Lc%Obs z+bH+Fm8`3d0ye$TnpxsNTs19zotl-XN-*ceg|9eBB-;?-i2Mt}ns@Pu0L<5^MO0Y7 z@cNdKubQT|FVLmCGTy3>;ax#4sQn^%@w3_YQ++WYRg&ahX!KoPkCnM*V;&V0N7@P+ zqP0c?%ijt54>P$1ao}t|F)aOfH?M=Xtdg9b%4d7?Slx)%Q%`M~ zRw@Pu^M*cwQT5g}skass!DziNpoNl4k-t)|HQlwJz26ZCeqBV(kaHc_0hGHJQ{~uJ z*;|OSX?E~|)xBR?vheti;L|sGe}bOMf$#QB3p3ByIoO3JX2(Omz3##3}f1mfB8#H|NJ(d&4TyfWHFlas@9z z_`2r@-uO=9`-Ob`m>L9B*i0(^3FwExKo*}w*}+wmyh`^}p-|is$>V9DKC(4ow?n@p zho{w)%!D1B0E$yte+A%>H0kNRSX{Lg4@aL)WL^6ng-5;c(no!Yad|mWcoN_}ohtSJ zIOPV(5?5?;0t!xlb(?D~pL&dOj6ZI^b-_}{M`qC|83Ftg4Qxdf9XP4HAx>75BpaGw z6|aMpr{)U|KXF8wB)Ak8Zg|Z63hN=8_;2p}f$Cn9{t47UfEoL|qx?yH=NrN5f0oa! z%Tg?m(8_9Jx|hVhoQFg5g*Lu?C__7t7?r^K3OX&ZXN5WLc_<+RC@JVMm*`vB1rT%a zX8(XgE)_YCCffW>B2bSaFaP$k*00m8Uy=t}@qMds9>;ml@?kv-5te{jPx0{-kixIa zufQRkt}h3->WgVvtXv@U40m3hdj64stSuOAlBzx1cJ&I?S!W4=`LRGLP>}oi4hyJ^ zW?$K-5@}=bUrN?1Nxr4mr+5Qe2`JY6kfE*~*Ti@f!XXe&Z{i~lhj08BRL=tU4WJCt zJRvbcLM|ZW)P5*e-X8R$d3mcz!K*Z3f6h1zZ!@lAno^i)kQ>9uR|kc&%p6~$GgwS7 zSHTOsPe!sGOu4SiQv}MK_8&C_%DEp@#8H5Sb-mw^=k2_r)Y48O*sLj$l>^{j0>!o1 z`?#T4s0}6y!Zb_iMI-Ie;>+WnyQ6cL7v2GbTISao*V{Zt^6rxu|CRufaus6``B4Mh z{acLPo}U+n{fZ2xJT;R8U?&QgNo;OQ>?I)eV_u*8Al{`P$#C0I+ z{(fLaHl~IIc$qWw`);#x`YL5IC|B>*%h+OT(mL!aX3 zs{zfhrN*1Ulhtsf$p}Hb2Z+M8G8gnkc+R^sD!K+m1rBiuyD^4n{P@gU z(x;MdHO0iCp{Uf(kLn=c@Q*rp3WhNZp|~p1;m6;lm5$=OBI(yTe9R~$WS(ft0Gg02U;0_~{Z^-v+M1#l$bs2*!cR7&Bt7 zCvM|q`ZhInefGd`mxhcGYFKT%sV1`v3$RJ3wEfvvD8xtkp2)tf*DsQ)%Y zRzN;}lx`5w=gt8ptPx^`QI-MFrLBPxj{GO+g10Juc%86pzkI^jsY{DV1*(I=k z`NyJb&p}Q?`VD(+S088J_hLZwTM;M-t39LV77ZTb?IlWIxTvKST+hwVukVC-s%kgq z6`dl6dI6X>3ZR~QyK(>JMpNT3N&*2WKMxi%QxaGNBZX;GWO_4btI7?zTEH{%qt3bQ zXfUm4H{&bvHk<&r4QPG;6IK75K(AxO%Z5g;a5-d840r?I+S0if|0Occv}752bPDUr zCKfwDaS`b~?fc;CB`KcY`zkS4_Vh_rAkd&Q(M!dyP6DK6Mc?j6?FTuq@WqEdGHpj? z$psh$Y_}6Oj&rygSa$!4S)XmHt0iEQ@zIp|JZf`Ne0#fP)Vdf4DJtT9@s_HsHE)*qAw%`zdWbt%IK+bk=hHm@{r_=7_#Qd zVlQ}~0h$V)C7|6MA*<}V0t9O1_*1D2g zO;DL{S^AB`{?r#(a3cgkE2=oc!Wa}A?#-*~B|4?mw_=FdZ)pN{t(ch8bhUERnG5k1 znv+%rF|6O9I>!v}C~!#*N%xID;;SXuPoH>fBDxb~d5RM>SpTFfba3t+;2aTGV=U6M zOc~`_E7&DnP_IN>)cba*A<`~PFdSVK>-^tUG$UX+O$|V&KF?D zuyY`xV-gXtqwTib$QrfM+LfSmb4R5z8GXVI*5+>sEm=y=RT)qOX6`wITDO{@sR-UF z>S^bNrChh*fgErMX^h&EUQ#`&h0Kr$AyHn-Jgc@F4=5py`J5BSdTm}+;LBm_*~98> ztK?!pgim^oA4K!MFom729?kXx-T-i%YnuOK^IPGZWqh@b52xbZ!$*ngQA?v--P7-a z7KzVhFl)HxXFUX39pE6FF3(xn6WZcDCzq`qOcFco0${oEe5r6iM`oGKP~mpmx+(k? zNIE7s5m=cN^d6X16YOk*Tm{{~2jOdqOyEPnL>=e3_5)tiPt7sK^i_c<9xDB=Fq6C`Jq!d1DaY@|4Jl37kADiym=MCzvzajzUCcCv-w z#6IQfN!-%3{!{zIsh@H!iiGKwI}k-3G;lA+KxSZ63~empVuQOYb<%CmwNbu}-}FSl z9heTXw&cXy#mKJpL^Fy$ut}zVanFwG&7wLU1D0B(A@hqVx7SfqFcybe(;4YS!`8FF zye1NU&;;uFm_@8?K~NP8NG|+~s)@DA*b8=NpAgKC`prTWO#o4Kzq`CRY8AnPU;3n; zq@X_p0MogdkPriD)lZ1i)sW`jjX#lIo=H?vAWosC#s$!>8}k_$^9cwn^SleGap7I^ ztWzA)H6bLNjO^gFGhvrC<{UT_RrL1?AzYo(=v?4-doxhXs*gr?>d9V(X=`9YkN^`c zl+lpDtYMyNbpd9%GD;Gq+XdiKU`7a)c2aZu=f9<@%);`Z9T{clfIfjxF`EoPAg5^_Dt_(r)IoE*0+;>M!*t_H~4 zQG+!wiirdT;U|sig863{kpZYe;g49y*+!jv z*CDEPEv~$`t3c^n(;sYdFfkE^GyNSf%p9&%D!_hp zh-j04PSelc)gBVWERh$)ILF90Zd%qH)X#xzOQHfGq~6fnTN4gC1t~Th%rKOjes{Ip zbDzs>ZnJ~JD8EKMYEl>?eoUgnVjx@Kio_=V6&?$OL761<&OhPlHuJ~AY$PLpi=(c z0tf-n3Sav<2$Ta~R}LgpiWW0!uMZ89qgJWBV$RBC9i=d)XY84ramTyd>j0u1`5_7Z zF;%B7dHr50*Beb0LU5xPuYS^~V|0I3NlDN#=L-yn0dN;zPMci>nTZLQVYWa9)Htyk zyefg8WD6g>Xx;qI2ky-);OeZQY-`hc8Oh+F^G3$kBi??RcQ9S_q0QUfz{*hSpva-4 z1Mvx?MDmfzieIC#Vsr1d#5C>J&SwExU^T_?tpdK|zhsN+On|~(1!HMH?EMH}B*yQ- z0I`a{fPFbQfd$tf0$dv8BQ){w>1LBg>dm+NxK=hRcB+g_P<%T@B+ZVEDl(p?#OuQP zCg#cyqUx_{!q>@0)0FNh4+`f#lwnVlyq}xB`LwSk%?ftpHm{+Lz6D+LA=4To0|O5( z(Q`mBsmaF$Xe%C|LWw|OSHfg^JXO8;gl#P|78JN~)8nozgoI0o(Uq2!h0+L^02IzZ zB=iSLDCt1>ena-!ncR;DoHj_ef-!bo9kXck%{Plz$hE1JqIPF!Fx#fr%B6Z zzHK*pUeDc&9=lz75aq=lUcd&4ke31Rkx?5s3$%}T07c%!D-_;GrQI)i{AlReRMmZc zqk2u7w+Ws`U`%l2@0?X$h$uE?C_{@+u+1y*-#Zk~QKoUhbE3dFAT!Fg+CvD(z>1|{ z?z4`H5!ck1>T}q;>?#C3Y+x1f;EQdSBFx3gjyp6^~g$i-;f7QKuu?$ghapt5|@$ufq@)4e8{#+>TGVDI5{*f3GH54Z7lp-EidP;qhBF<{LT&fVBe5QHo{_%kBuh^!xy5 zca6A&W|4PT=HCsF>yKi-gDJ<8Y7+pJN|)S2``ygWMZm{UDd2F*W(~A|dN4e!zdQjl z)#%6n^|~p0u;Cr42 zLHV{BN9P|7ntdI$cn+>m=7cLV?FFI%6MAAX2fouupr`U3;51F zQ{2JqEG`Mtyt9Gt+sU!kRwdw}*aPXJJy0Xr13rMAuC9d|jY^&bSQWM;X zl-zG#rAx1%5$jG?Jy)MeY2z4g1hHqgEpKHTO>AQBO}j9yU!!{u5R7AMKr=i#piRjP-riY6T2t}fRmFb#N_$&1$L=te>+vK~ss=)5|ioE59Gqs(+ z@p0G_zM}J&`dRN3x61viMMczn6+(M?p!Rrlbh z@=mi%n9F*`{hMs#0}fZe>Wn5Z{EaPiMS40mnYrG>3A`4MCRLv2Y|2tj=VCAGwG{sb zEZ*ik;#@k=aeXKT4Ty^-q(M{%tbE zDUu>&;}wHBJxPO5Yy2aJiO;MSh@OlofFB~`{wGQxTk%ixe??pt6F07Y_!iVU$n4Mj zE9!3#-&)##d3+~Encjgk+pC!&vTnb_2e}K$ zZ-|@m8L&G2az(U#1i(n={slNypK>Jdp#Ud*Be278It8O2FXVIk?3c;)@~wqP!EqBd`3W=;#O=O@^jE|F@c~k^fAaOWv==t1zNp1f@KKEZx4Xee9fyCtjm<;E|vh zW&Qo<<*q1;#NB(qe{ATlp1r%)>0_|99QsbFwB&F8u;}p(UGUciwYJX3GChMG7xA-J zBQkAQ+=J=H3Cd#8vqg&$&ESMfBf9viQ?5TCMi>Pp!Pe_NA%GF;>}R+VOVXRMgone? z`l>XuW8@V%YoALCgW!P3fb;cznmHKuse^eB&BrR=FoFZ3D$Mh7Qwrj1x{>NQ}V7#ynREc8)zZXY^CkgdK_Sq3c%%l|_Qj1%N{t zhxYqI#h38wF>m_Xsu?81Hu9rIUcPAx632@NUK{J%(H^)32W*MWzB3@kq8g}I2JgtH zF5;o=DRs-x!m~CjgeNbGW$bxY6=wIpa~8h*da{%D zo_X0Dk`Gl~Z<~ATWWq~_I)O5`(&jwMdoT#xkqyy{t9PQCK)oPbcD zm0XqvzA5~xbn=HJ{C_bh5NW$ehWV>fPAgKG==ub}ZPa@~LEYQM<9{B&JLfbUTD)Oq z2ZO4Y3<4ri*$VvifPf&d+%!9^Ee>!rSZWk&9iOWMIZGm#&*Wrnc+$K{?m z`PO4m!CO#g4f%>i@x7IO%6el|<(`oc6t3Buxrj-Q3(zn%Gic$Ync*S+OeD?$XvxJN zLcg$WVSkR)O`>_gq&63u<^QE>4PF~OM->5R_Ae!(ElHPK@$fUC#r(-=@-I{Ebh5(3 zShM0HhB^;3js%;x6&$5JUIPUd`7aI-5<8J{i;X$b7__!CdQckQZH-rnC`fcx6X!+b zf$&yliud4#Xt^=oE&hJgB>kn#}|<^PCkBs4~fBLZ5_5`NvxzU@IP<^4IgX0$fwmf9aRW zJeT^CJ+;Vog3$zEOT6`()V$m8^#fd$1)LbtT&hNFoYLxqB&Z8CRPH#7_HcT`fA5UI zWrth~xO~1gbJ7d9ppV4^ZObbbISmF30*5({`Y z6ncP#%;-0SH8K)-S&M*StNNVijKE2cLrHQSs(OY>p9sGct@)*z2Pxlb3!i#m&XBGD z`3uuBh@~ky%Bjb^yIM9F8Xb-g?p%6>8M4iEc*OMz0}PuGj&j+Oi_E-l*5(@Bp{&OS z*6Z=Z-Xi||BaZJx!~U7fUk;=&_nq9io2XA1qj_{+a=m@MJy0|&;5S`qoV#r0g^Aac zmGl6jk{*h3YSrrQ?L?dm1DGx?SQll)izYbp)B10BZI0jBC+;$jA`2c3asSaNaE+S$ zjOrb_(ZkNMX4l=JUeN$@T~2&|gANpeIZ_5Rf;gt2s1y)vZiaB^Qd27wH9SNZo47Y_ z;}P7SL3WB$e^$=o;l$1Q7RV1}lR}E&m+$8r;Wy1W^yEP&YraD3J-C{el?9!(F2C{j z_#hK}{)Hs~%u@Jx_l@i}-eDxMaUL(yr9Df^mu%ZK{1835y+eg#Z{VL%L688Oq>)Ob z&jU*Kr5<;ft__WKK^cf*dE{StsqEEjt3XE;LI0Q2^5zq~%=cHX%QbMkYR`>S`4KLiNU3%T2!g=S4l*K1N7l7xqtnU z{PmL4iCQ;t}NlQ(&;5LXK72eXllfr-{X<0fRSd zw#)qnDjTWfHV4gesY`>MtO|E>v-^2}DVK+V<%fr(;$S8W_z&`p7KkmUw7v01V)om~ zNfAIV0`0%lM2$~*!C^vT9ybWqU(%Zl(J@WuQDt`JPJb|zaJ=yh&~NO4aOcf@L{nGZ zXL0(K2ZL9xDK;bgiDs`$O00w{>;yPnS^3)EfXbFLI3G@q2)*7xa?E0OZ&qqvqwSk9 zr1aQ*dJP68Pqd0&-3E-}bPTMSGP|Le-n)9uY~BSB&Fn!mT?tTCm%xJ}^}8m#@q4eo zjs)PO$+#|rI^{f&Vv>F_6e(8fpjAF~+@3{+^g-QmJcI+1?*z?DapIMi*~N5MW4@AFx=sJLPu%8`g?pcp~lN@`h4tJ`{!M_>JILCfFc{4c$RI zwPFfK5gO>>f%jo}vN1Y+Km9CyAJGBy(+0BJZMcnvTX04c@)x{rv2cT7{>8Rl#RqmR z<6wmNZ~JY+eSII;UpDqQAvt-H5fq5+m4^d6oWF1?nMEwa?LlZ*%qiX|MkeMVV9iwP zq}I#67QCvV#=?%WwO4g3Yw!xuKNxTaCU`7QkZJ)Z`^u-R$Zl=|9h8tS$h{H+o)s`J zc4ZEc@P%kS6iovjVj^w?0YN2`$S^Cw_&Y4c7_?kgD-!*O3FQPB!^t3!6hTq9N=0;0YG`_ITQz1sE_aI)s z)ei2gO=mctn63d!1$g?S@q9W$+}8l%ZRco-qp9tz9=e(>Bw?}uE4&EEE{bSx>B6R% zE0WbS#N(&$^6DlpK;HC#3g+v>ADIE(o-WB)?Li-=)ajYR) za6k_}F@*~`*_7#`v8o-2aU=@ITLejMfkyy(oAZ5sHeMKE9YgV|wz)$r@T&s{@HIWY{*fE7SXUl(9rQdL5gm_k6tdf7sXv_mgIeU^@dm{%_; zh!nwAy_7#4ed1Dh${!fxhzP_so0D|~N8U)bSoJ<~se)X9c-)T#ka?heS-gbyFe?1SJ%`P;mG$b2N$l+82CEZU+(6%klr+&{-9La$-upddw3zi+)F9XoSC+iBkW)&<}Xg*`X%#+n>f7l4U3f9O4V)4sLZ|gNTDoJ$fH&b zew3<=w>VreH1hEOraAD+>-qCF)3XX%Mx<2PA~5a=cz7c2v(+Dx+`xufmFX z>`!~z9xsaOMj*JND^T3x1LJd@O_PYuP8H={?w+R2?F`mmOf=*GR@v>7rFP#7O^;>MvNR zKcwATDU_5)=5^jr=nT=JjLe;qaQb-{p3Scjd*LtY?ifO4;HTyA_%zQI~0Nx6oi-kJr*wPc4_A}UJwaY6f3cv@d_ zsA1w)-e&e_`VeU8OL;s_Cc57V2?=#M&j3*T&GcOj*@L9bx2i!awuxqEx>rZgcwg2E z1i118X3*Nw>5vjb(0^a+BQW0qIKR;HAvi0ImApd2fz`wyIn;gTQQ)>A7e$;hshOXn zX*`u`F@w$RvX#rS0(2yn={rS6jAl}-Yhsr+yb8IdT!jOpnbIMu@bNGfm;(%#65u}# zUyzF0lTs}C(x~`r_dV{2RBQ+yjsZ5`fh9Q?37&Ns7sb**`jaDe?!-y5)0^oKuay~) za88%3p2Zs?CXvjj zZb*<`kH7?LUPW>Klaq_C{AmXX<^p)Z>oY`mjcZ#a3pNcU~z=Bb@ZE05lZ{Lq#5vSUf%F0qaNb; zq0B5X&$}&)d{dqCd6!1T`@|~MxZszxi$Cv+s=#wU zy}}LqBPu#Ti%=sj^mDlYzQ*nwaS$Ab%L6a9`1)Y*uZ#0}zqv`g1;+^`XqK0bK+p8{ zbRJK-&d@jPg;NaES?#~@XapxF#^?3%EX&)_s9E^6#a!A#OafH}m|w_R{m}EynEoq>#De=TbAk<=O@m zZ)bP)J+_p)$T%?KjGFZwMcugK{To)C_3+3sBNrz+aTe^oH{EglY3_C3h#nh+*wTm6 z5gbca^-li)FR^C?Gl?X$6$d{E9CdAY%f1x}x2aAyFfpP|LnsHWEGaV@ueQ-;NU&R@B`X(-ndNTiDMg%k?nEsH{{U}Y9=5)7f`<`~6n;{?(}tP&z( zzRdus45JhMDO(-Z^(wQ==j>-OwEcuG*zt5E+WZ^`L0Kf)GlX0Qy}!mOKM!Qy@;hK= z#i-CaK>rB%IQyK;Jm6#029L@EKfi?l(|U1I>VX6L*Drn4L}jDp#)j~wK993g_16M! zfl^5u+3L#}ntB~R)8HDA(U+FM&Gj4-Q5^wZr)_L>VkWqKO20XrqAq_y975o$B~J!! z{H!4ZTW%k=rP^)+>ajiWFP+v3PmbXtU7#187MLfm6PqCE3G##v2`JdgjQ6;tAg4C| zRze0GPwsIcO6aSbKKc4cp+-*i(zcWhwGtfjIKUIZ`}dyFTqn6Se}u_qqZ=GXD)Qi& zp;$_rC1M_C2F51CU5FB0xtV~M2t7V5`w7u_t!il*N`1>_zFTFZ7os&PXyf4M7I&UW z;X=(5LkW`tB8FrfQamq^m~P|e00Fd}Bre0XEwiz_tU?~rdoE{96>_KJafs{kKv8Ly z2~-1qPx(Epw+KonUfuBp4v?#Fmch2hkrANfBPkk2nKBh&W^}vAnuf}f{(1RL7v0M)=8q> zzsSk)>k|>}EVd)M^xO7j_> zCI5Y<2m%~>Bs$ml0eH0nb;=&l9Ou8up6F{*eD_RHHcIUuO@%YXc?{o~Lt}6__n$uz zBT7J}6c8q%_60(T9D%(sunGlVKN+6+`R5u1Tps|!+^uoPeZI1fIwHA3U8scF zC=KSim9a)^4_A(`Vla$;Qr#!wVWZR&avYGfX(0{r2sAU~?L{v8E&Mu#y=;MpyG;M< zH>41k%0JN70@VP}4ZOU?Kq3!#yzpr2=a<(Zt;qQs=#$ol2t@VsU;mIu8{rJa+4eu$ zJlOx#MvV{QTzC7#^QrqcxTy;rtf@6JpOJ7#c)=IrRuotoUQ7RQ&*NN_*1lqy9eHW{?ciVv6CZ@ESF1aYZa%B$ji08NMG$XGhe@ z@cbAtpHg+aTZQOP;(N#Zc@c+ho8;3a?|KmaE2f}WnO!AmNv_fnl89^Q%gPHSZWlLa z?zM@Eip0o-r-+%4HnDI7jo>mJ81>$hP%3GKv#b3DoDQ9`?@8{$_p^%C#yyzx<>MWQuB9svr%LFW@d9opEv@m*(Lu-VI}KEyR0ByGwc1zNR^ z79X-k^PV_--zYXBr7p9!UoBku2@;CX7uU2C8MAKbm47_s zKTnutvUsT01Hd?R9X4rkC8l1>iAmCVX|_wv zBzQFM04Y7*jf3jul3C?@ydM>4J>@H|*Nv_oDU(T>BbFo^adC<%dVO(8yZr+tUFI>H zO4ne>KORpX0k^BO>LjHYC3`?vJ4PU5?l_?de7Ma-98LZHSvf}g{!7;LkRqZv%JLmp zZ`Qy_<}CX^&snN$E^CXPbO1_UNnIKeA- z!_pW!{5{}arNBPfYP!*Q>I9;LKCF%3+0=BW6VQIw9jg~`a-?V6sJ(rOIU)P zFs-2DH!N_PtAFfRgZjNG>pvPoA^RTpbgF`YE?m$}fw+Ax2}%bXyTF_;M&{6U?+=-8 z8dXbRqWaI)PPg=PwKEV!d*B=(Gawa=AGR0D(0oquSMPLlws7;OXWcY5;MKI>vTlv< zqIJzqZM6MHUyc1vbC-R0wCL!8MTOpVf>=%5Q!?#yzk?@w9!(0K!0V;Pe&Uh1!`9onMGQA94{7-1liIC$SdPbkHq( z>^#a*+LPU zEdx{nptTw;Cu^my;t#6r#~x+N_>R==#R9>V1Amaw3Z7Xc{*XUpE3IJmug#C}8NVr3 z!PpU@|20xT3%@_L^`Lje8+m?LrA}Y^+X&V8R`o!RX>xU7>YSkEhljEx45ucZ z$zha@8>5~OiKG5ceM6F$(YW+ zyd|A<94qZF%{O#*Sh{p}aJqD`ue-G0s*n`Rus|r`6+UHZhGsHNHWuF-xuU9c}>e#@Y+hSVDPJZ z`Ay^8>YIA+vl=URUzR8@a+het%Gv*2!P;+TH8eU4MBHxUOi?}*m*5Z5b*aedDIg_X zN?)O8`9|l+z^R@i$N%3YVc83|^aWcTT2oh6tRXS{1pT|*OeQ~S{Ob(d`uW!~fJe|? zf!bJCgL|nTosqHK>NS;|s{N}fI@~8VbXz@-=05Jrt5nof*X64r9W8^N_TP3KEmBS# z-PI`_1z>D}KJgP-0-?0hivoZPr1m+uP{7G&u>LDtrpQ!t#_Pn9y*#eWYOOV!Abju3Qg2X0N4E<1}S(us*J#*6fv+@nY|J z%|Z&T{r51nwKuXyD`|Q$@2=9QIX<>!TvW%9C znx}S4&hGD(%A_v;>zA&Z#))s$2bMr{h9~dx?;4(z=C>N2JiPMRsQKce@@S*t@YBVY zpGR%`bNnr`nSFr)Udg3T1U%IlPt%qLDsF0*50UwPa;#`fy)@y?x9X$AgfNLoqY0!+mD6N8mG+s%gX&$9GCL*e*3u>)=lrd z4W;Kss8VVN3wFi6zAKR^DxVg!e@Ae{ROZxt@qo2L$p>#)#_72Qt;X+$Ej-kp;2>~o z-`T9c{%27DzaKVx{tvW~K$!vl+bCe+r?^q3sY0YIlYC)Dmmq#XfF!iQFl5_X4%4EP z8^`eZt=F!?#9;Oq$nZ>aGHswDz@7Mlc71GA2b@lH)iT9(r1qAyr`#-`NHObQ+Dz*< zZ{e?L(a~YKFZxGrOlik>Nom*4Olcn)P3dg-)p#~3mMsjr^L=y|7zfldA*Bg6gV){m zCdaV+N~Ut|K0q);DV2QFfP9i;$sUiJ1d5JKy>sSzmAi;Upjnfw3bVWeG3?$swjv0= zi}n+ww6#lC5!Cs(JUEs@RcA%w&Jx`fruu!yqPvt_O}fZn%%K>N-%6joyKtkhzGK$( ztm`$;i}dq=r#+lbH6981*;U0Pf{mIrB|jcTGl(dBFl{oC1wPA6Tm}ToE$|V+bi7ZZr(m>uLVyEugb0knaScAX<=;#aEo|(ISys6} zPonb$u$$!HBw4GHlRNhc#?zX|?ikY9T5utS8tq}h14!kf zy*1XZM{e)nAynrIl9P{P`)dtq?1gdLsIEWg{Ll?%E`Tf~Q7XlDgVezlBuKF7CWo+0 z@&CNa@rAw-@88e&HZ&L=IRNzkSkq=?5chRoK$Nv#`(0dOJC=9T(9>aIc7ck1Ge^H{ zF=vXJe(TRcXUtvWTF0Pu`HjyH=qOYA-##E^ed{1^&PTyhDwl>&`BrEj`1nyJ3fBRI zws9Z|{s`eJoN*U<)tj$vIESVFzMvFP*buPFng=*O)qR%Xh z!(Kfb_`%;+=4YalTwx-Yyb*Y(^t;205F~5Dm(yha08a8&!*(w{CgWwdNh7CKEB)sM z^@K4;sV}Tzm;a_cNfy*KX^QxI5I+{J^#H23{>mf{be%;#KVZTzq>2x^;TB~StFzGS+&p8uC8WChv`z*aA@s=J_^%-NK z!vB5Cx4PW)8Czbh`)je7v+;!)XA1Blg0|z^*i#bL%bV1fcysPq`ClIh?u&4{cft6_ zTLiaF^94`B+}9N4ad%09iKy}DM!)6GA%!$r=z=lt)~utz!re>z{M44mUY@?C{hk^t zQBFxBI4HbW${oz`03R6Fjsni_0yNoFp2O7vM9r!lWk6=*z@_5FL8i)nuS}- zaM(>YQ|sblX34IR=EL7Ff_c^~H@3s$F^>bS+0 zOq_A&w-)Db$c|^93IAkAUUq=ISoJHyDXl>P1SQXAH^$ArycwW3H0Z^C>y{iR={FGm zQI+!5OYM7KS<~X48ViJ;7y)2XEdgfws@Gj(w!3RS&Q7(hRhj?%w*(!urbS%VKV`hc zcL~X7-k|eH-}t76_+gJpHlMt~T5(j@__l}Kp zR8FO#ZMD+_{=-YQn7QB1`4+92j`kaq%iknZqJbfAF;}+6;9ORQns`f@ok4<5(_8g$AH$r^XJ|>-GKk7?DVr)|DIl%ajWXhfBdr6-XpzR zE)`xM_xYO+rtkj!HX^+letsNyD*Nfj_z3=Op-<+m7lJB1cLEE8$s!0A#?dEf=RPm< zTc`Vf|69Wx-1%**qt`Jae7Gm->LP4vl#7{e@b~VlISW7PAMd$e-W=pru*`U;KkeMF z^_)#8b(Ww|G-d)*le#hP@{R>c`#zF=8%17OZ3Y#wRf^%Ge<#yQO{1qM-Ep0+I&AN9MYPJ4ET8wE%7XJPUN(Okz7*NqGE6T(Cfln zurPqzHK6Q;or=cfthAdHwC(7hEj>AZ_z;3JdSaylbC$R6picZwo7naJf?DQg&1jpY zG8_Hc#^h()0luE%Y7@}rvIIcr5}R*t{@bXUgrXKEHSZd?0T;>5D*uDkcbif7W^xe1 zduMX5bT+0r#%p=_&pSr0rKD6;~T zAH3K=@;@`Aef49=qr2Q+kHd^Ts=8HFG@RrZV;njB@83*1YMH8M@2NbrJ;8B0d1kzf zQ875usYaaMW*$V0+8qWm(JcCO@qP&Mb^9}X|MD}U4J`lcZu!0kY)pZG#AG0y26-Dq ze*ud>L(09txwh2qd%?68Jw5dN!gmaCFKYzVA)msSS$#+*r&3W_9oQ!Q6>Hq_$6o{QS>07=F`npSvM0Kl_ZkU@ zIKI10|9R@t02$;nN)VQf*v9@uU}}P)uc@5fkTo9W&p(yhL&;dv)9%2QT(a;7AAhW; ze*M+)b)YXbJ#x2l!%e{Ea#Qfodt9n4%8F%No(IaX`u%b4ayal)XV^;A`119hJ*+$= zH1u+9CM@>vvZ=no_{i|#dMV54=i;du3*$jk;nqg~quHkj4t_@9{X(}RIaQ#>%xaN& z)3^Y()rEz>;GTLR&#{BK{xD|$<&C%$MGDz{O(TeszP}d)IrT{*%*a$;)b3L-cMH`0 ziaC<2yIa?bz%+ZcVWS=&`qw{?l{Trc@4?HMTQ$FX(OJu3zS=Y!xV3fY@BG8_UpWbc zT9(eSF(kLh#xqY5Wu)J4^rX^<`2HzWC2oFQCxD)h5riH6U1t7YPiGw#RrkI90g+ZI z38hOyKw7#>x*MdsyBq10E7&CEG7d!N16Uh7_; zTLdClYBp?r_13Ss6G5>0_u5+<1Cik;msW)g46nMPz4Q?o|M1wTS5_I6#u8PhjxAeC z`_+>?{xMD%;;P?4h4O9bgQO$NZY z0)dGiNNV8$rV8(bTcs^a<(*Ea9Nx^(Q;Yn;2e4w5jfb|yx$@!Fo>!f={miP zA`&s@*B-@ueWQj8F@K<*`=N?9s~#EDaOBiXN^!jf#R>x_2@*B~I#x)RC#T!tVi-D7 zR1h%0lP~5q5* zPg#;3M^M|Db=GB8D1mu_*TTac)X4fi!)`|vaK(&1=k1(Gm$BJA^&FwXxQ@{M8XJBq zCLzAJ0C!?#$|paMTdi*WJ-I6J%@VrCN)_y~H$ zulgs`X?(HDuLl|8M)k$riKBTWJEJ+WCH$h&QuV+wybgm6(g-vcs9SOpi)j?hM-7_LNuw#QPRwOZUAL=l5v_3Xt($reh7M3Clgt3`MK zsI(H07|*UlNPerSbPtXJ4se3XhU?`-JSxS6T+Ty!rR~<*sDPL#@HYJE+uvqFt~6!k z!nFtqI8}Ga8+7s~ynfvaSy&jK1ztQvKMt1p+~@C2O(12DoWAO(LySwdj~~JYXy~(= zLK>_zW~0y;22epgX;q*&2ARN9$}T;ihTH|vnHYC_)B?PBX+7W7%ao{qa+^vsFCa%C z;TLo!nlGeYTdUI9;sP|~Na#SgzdsC9p2}pu*PJR=*kZz1#Ej#j6QuF8t-ACvd2@I^ z`mIE*q$KqDtCDQ*T`3>pPmWHX2_Q;68~l6X^g6kVGF1b)l**FHynv4aV(dqTfId|K zUk81>YmGd7L+7CVEjr`ruVVDjtt6%&h?+tBIkZc!-z#0;PK+qEl}ce>>{pNWbe*Q8 zv14qw!Dvre_u0*Ywn<-IpC{Sm_O&!6V-RGtOlCc_a@_fMN~Q?UzZRL zmxa3pvZDx?BRve}+;wALwpCZx(X##K$S84gAaYTy|2{b`CVg?y?t-}^t7r<5#P$bA zFw#=ygiE{Ncu{P_HC0;R9{v}!c7ljo2@QUSNCs~GqndF22bd%!>CF#w7q`4TE6JR5Ea7(9EAnvK+Bq8y z7U){_L`L*Cy)%^&04OhwGq}P0S>KX>;te3p$*x%jB_Sy~gmRF=QT?H(i}$e3UN1uW z&66}PKiPhUT?gX)e0s5L+3Bd&GS00RF;TUI{2`cF2Acae;}_tE6@7d^3^kd)&pnob?(qxf0$7=VCsly7*X{2gEKY(sv5;EHa!=?g=X7fCj`E+F7FUp{<1^86O1Z0;wTg^6|B)%hF)!dIl zwL^^icZo;|tcGxY%L_iA>}qO00y@rRH{N%kJl(xdu#VKv@5OOr-jnCW)y4UHaac&F zUQ`;fPLqe4!cgj{nw?wl4QCiXTR#L{%kFpx|uT9?UqVN z364@oHJ+M@^k-FF9yisqt>KyOL@u|6ofVxAT{pmn7Utf)3$jf!oP8@%#a)kCTu#&* ztB*ZV1YbGqmH@^mVLrJXdVgS70tz)}4r z^Y2dK6sSM^sI!^-HeIGwXp8R_w7XQ>T(VTuG@v#GA9TjhIyvq;_W<98bkH%NshNcE zJkjbWtKVd-_xN}cV0F8(H6H&E9!h$pqC~{wtgtPzY(NO4-?t~sl>OYD2njRzJ~_y7PaQ>6OvS#prn>FlGRdYrax zBUR}|)JurzI(ZMO{en$>G2`#%@2k_g*TB~EdwC_|&Vg%vY^?QO+Ja7l-2L<+hcG85 zmcFNUg?gO~foju-ms5U!1VL@atvKjMTl(he1^0Q#cbRKzp5DJxK|gj63ghcRbE2|#p7*f-Mo*{ zo9h-kw~P1Y^*vwNm)EW7%%GVi_7fc?&Nt&Fi(DHjNeQYd_uLn%IoUpwHMA3xM6VBj zl_BcajyRAG2BS3`@D3rvu*LbrKIdZwu5TmoEQ3vuo$%RnvFN^51$*akF0hkYkK z1_sVBr{5+*{{^zRyaIv8JEsl?|6D=%aJpEHC(I5(ktqzyd|u}Ycwohi&`?kv8MjSl ztT@Mfjh?7@bVMY5RO=jCV8g&*WZ@vqRHmh>`x^?aJ63E+OXbg!TtPOJXFnb+xkJZQ zF#95q+}TUwrNejrM`X7RfCl-RuZnnVn?;73ERtVwyBAr`6lXORorslaTo}?VG`hZ6 zN=VK&v+$Aj(&io;r%JZ1Ja=fmpObKHZWPt*OK2Tp<@iNZcTU?Zf5MQw^s(1c&(uOp zTH56HJNw5<>k@@2;S$fF-4c`LWJ|$EI9LwI@;?3IFaADAg!45#3i-$hTudrkOnZM` z!^3CYopxfvha~h5&JvX?X#R}4DyAHAiPw+@3#|9E00a#6TGEd%yA7z`)@hX;;GWDUc@)j=@?9OhoHR+*xuSNCBkz(O)GxLmgF)AJM>bhc;!#(cXMM2n)#tax zj72({N#&Z#Q-CL;?Y5gAs^?XvRbw391e)ik=%=~IOQJ6>g0yFBDvERmNBIg9pS<^* zTh_>!UDmLzs3_NL6(7!K)|PFuO?b8F2h!mMpDevJwOur`233O4UKeq1O({-EtEsEf zsVF9WS5w>A{j9vj!K?n#T2o=;AXVMuC{5R-c_1)1H#z2E@U%cWey9?7Kz@Jw+ZbTiBA_UeG1WVWo5Z{p&sJD>|r8bt~h{Tt7B$y9tz~ zd0>JomTtD>3*A_LFrAl!rfQ{@m}-IHOVu(Jb{ECQNR@lKizyR4-EUf)mSyGJ&LOYt zry%T?^Ocr8lLVGGRU7m0x4`9RXKx?qLzR*Axr+3BcQ$GocrP25M>Yf@#&N7XWzOxM^lhNUACqj$UB_ zff_|ARu?|sLl-u8n=Pn;TVABITOPJ}Sgz(vahrQIxXcW2ym<9q;o@*g`WZ2sB>e@{ z{V|kQdGhWl+T^^;-XvI%cPUBWr!*M{1wdU`&b80l2Vp+JPtW*VGlpmjt@osT0%?5Y zwN475QOpH)tUxY99IUW6;=Bk7l>CphLDiNUbhY+BgC>sXOR92P~WB2M7p-|0B~nY0EbvgA6TAX&WMmkcS|SHC8;s|j&_TXO=laMe#)s0 z5fi(&?(TvIlFnS`vSCqdBRFGgMvaOuj2l>o4L9KS3o23V!SDeMmSJdJ zd~4LmY~9~2EF|Q6L6fjagD((w<()*kP-WDBhNdb3EOh@!JSH{Jorz|(PicDGfyOjx zWL7e41`>&G-Mlb+qO{0RPpL`N{GdSXr;<i7@6642xK<0sB(&LZk#8pe;uQEYNah0ZxN|$A$={{{WTRXj3O> z<#`$Hr~a}%@Fd`^PWVNn0K^vn&31Rc|9C!_*S!UE$b#9xe9jmI?Xd;)L=6T@pq=C- zUj~qFGRAhu_Nh}UFsE$*D#8qoRA|?jC?X=FM}oFr*djiQ$8F#tED=5cLYkEUG}GVL zm^uTX;2`uJlttQ;jNEvm5&V~dDc*j%3+yu&#f%Zr2RLv5rx@JskW2E1s$>Fursn;s znyc;ccm{0_pr2<{R2+M60mK9sE`gR==yfjEB@IW36e5|(csJO6A_r$4q%AWTuPk|0Qc!k7Pl61@_t-YegLY{ zaup@)<(L@cASAO0K8bTbRC$Fa^*OhwAp@31;0zjrb*cF?h<*GGuVqTqc6TcMo5y{! zNOAXkX97NuoE!zC%x12f`F;2g`k)hd=Y`r<|I&nM&!4%AEi0E%Dj z@n+OdY2f`BNNE6Usy+~-gmv8-c0OCIx6_}3Wr*BX>LL#TOOG2H=3Z`A)*KclY2Eu| zwLI)4ZuoD0U6|BS*7dz3;3Dxc7LKP%5MBd7oKB-H>_ADB5BjqCvrr^^(maM`gPy}p z8Z-0UnxP_ttGK@X;)TJ!wf;9E3&S#qohfsisSmgSJNKumqy^EqDYtsM%#;^`aN!4& zpiUW#W8CU_p#&YWJe ztIh0k=WCIhR29hwds80exW%$H(B3NfK_^UO0U6nC8r@mh`w#G8$Th-q1FZls$51YN za&l&zzax6SHyc$Hd7n9#+(o7UEiUi%(reoWq2VkZPLs_{AbTzX=+GulUD!EDoTgU^ zpiFgHdZM2o0;|5hBt~8Cvby99OY!h9cNbyta6aj$S4Ow{qy-L+f^4q>`xs z=H6_+6gJRXLH<129qF{&yq)0BHmb9pN5v!;jbFz|IJ*&{VtJV!UL`FhL;GzgpOJ2g z)QFtR*pQ8SJXL{~-HAZ|`Htj-1)U{b{Aj)@_2|`Iwr1>J*3`V$kJ_vGz3WHa8J~mN zD?O{$Bkq>bTfd{PecXK_7NAk|^5U@=(A@0@JC^v3QbNS0J>dV?`Q2umAy$2HX#bYp zZm18O!TBaYLLV$+uiv0{>IY$vbKrN^f|iC>Oy+Vi&|_gAj{;TeIh_TEoiwo4rUA{o z+4=m2{%fu+Af1wljC4g2j88&K|J3FUJeJ;pV!XZ2UnK{Uel$vN7Vm5OYVgLeD@X%>>&?uUMB?PnEkPcbxGP=?cEc%>Gt>;P~4hk`fM}VG8 zR{|o=vYSflJxr#b;LV{`k7K4=28@#DfCXU>bQETYX?2)mD+xixr2g_=2w2=NO_piT zT-c*-w&o|`=LfJov&~?$Fa*W>3w<){tl40i>)BkQ@E6j%8no9wPuGV(GE4 z7oSW|y}ITH-Q`ayCk~n^;b=sIgI?kPk$phTS~MF6-z+d)HP%uo7O6|Ke80B?~0YtRw@i*g8<_64HfF-`&(S?I2d8{Wn+dmtTg zkqxZ!OEhVN%IGlI(<$1ZG6ysx*Sye?UvE7DFr9!Qb!R*-BV%tKt$-fOUs%+`MaD@U zD-VcCw-S7UWKER&Df*_aK%&(E0H6*M>Vbr``171R+3HU~vNez4Ieeeu4mBUkGbHF` z(QB6=@u`^|jUVaBkGQ+LBPpfGZ;7L51d0CCX>>m*c*iU#tWtfQdHuKglsG;#apZ?G zyh4~^SgFAMjy$7$-0{0N7;=)u4LQM|cBmvTj==^w3_dgeg;GA>`H3iGd?*XyZWX^9 z1?Xd2_aN;KN{d@k>R#aVI}F0Z+aQ;qcpHJ+xzUJ=M$#~l)!7eM%`6Tra z47VbDfEbPC&Uit2G21=D30?lG@AzYs4+Wcu{7(QmCR@D^?!$%aV?OQ`co_T4pgkQ< z<~wQ%hYWhKojm+1zeU;Wi!K)zX75gS$>^Sim%fwj(LXM{ou>uEp83T*n1cn!D2j+@ zI!j^OHwRUizTp5Q0@6XhR--HDIx1rCFiJcFJVs|CEOFwr5`ssJ9WqA;p*06gkNHaC zua%B~v1n#&;!;tP7HKw*WP%FDq>G?D7}z&FV|CtE*wV!Z>d>1#{l|1H;dc!$RD{Q` z)4j!UA7I!mlI%V@xUey4>?0qGu($J)1tmr3c>Xv(*DiF^El{wQ{)7Q=3FseENq}Z` zt68{N#xojR;PZGM95^*QsK=-fRryyatOiJjECv2NU}?B36u2*@w}*8?c-)=z>Qa5d zNMnRlX4p3MtG?`I#fyuizjOd{#X}EFNt~F1JP4(HYF^xGsf5a{l47P*2hA*Y9Z(wD zhB{M|gI@?fWEvD3twF2st{l2Mq7PbtN(uS5`LwpceG0>4uovD7$;F)_iczBow)iK} zsj>iYVEEwK(^lU_(@@UaW)P@`T}dTFx~fCANZ<$u9S)A_fett^F>#GL#JD3JLuU)N zr6;DOY@Ld-zC;luOMs{+*O=wn>4Rp0%qs)HcH9J+eklU7YY;Iu=G&K07Q0i+M^J7D z9x{nGp$ak%9s7@w#4e15u^x(0oq(ud7}OhvfZl$XB9_+>cyE$#op=Goq%6?C4}mho z5ZFH#A1-$$deMn_XLU7Bu8hM4E`2~NHRJtP&~3*^j=5JUBN4zl^1gHbcQfZ&n?7jm zb;`o-+Z$PTkpr_I2SfYqwdkfgL)FS@T~?xwqu?S1JDw7$kF?ZOwA7psH#nCnYWbE zca=x6e3z*Ug{!4V`raoYXLp!OBl$+z7yE#W#Icp-mi-zMxAW|r3KAkoMuxn;rYOq^ z)^(?8O|vvui0HCk_|u7QpW^leP^%ifdjOxTH*3-SVHZ1}3PG$1MbOS|(c#mX<;v5y zyKUCZAY?QB(;mPW997YwOw0&rBq=6d)T^~jI=eIIM~7owt_Mof@)w4sz4TWmx*daQ zQhP@^k9TG>ofu5Eb);a9M4$=F;YQ_seZ)!R!#9RHZONqoC?w&iXds^dC8rA3)zR|u z1}^4+>GL9t9dZb!F*0XQCmDQ16LfTsLTCF?oJ_R=vhMm0>D^qTAMo>t+MDV1cA8i1 zuA&B6-@aCX^$*`}L$a6Mlyvb=!c*Ur#LMDUfQKL-1(`1RrFsda&4gWC%=z2ZunJ?5 zhV;X08N#3ReW?tmNEYFV*70dEQz3T;c77Q( z@ECRd?wr%;jN~(=nrA`czr+HzUcjZ%;?1!+i(kVuzU4mZb!GCfYp!&>RY#+8>>7|&D#i_NTLFv!ozL-jcv29<_4^gF98h5eXnwCeRMy@JcZU2#Ab_6_Uyw``MCHz= z)e^b8(aYWO9Qu3z&5~g3)0@w=EZQ?<;BjFQ#ZPE&0Y#0JE;)7IgyEBp-@t0obf#Pv z<)!XQ2-({Pcce7tfjsH=w;+G$*1s;C9|1WRA7RL5;WZ8U9h;r4a$L=nZ$?^#5wM?f z^d#(B#G5R;ME%F%!)Bxi*2ay9b)CgTPi^+8U!*t$KAU+7O;4U$hnK?lCJJNa++uh$ zQUP`wGRTOzpGeS%qKiNw;9y>15y&LV`vAh7PmI5<=A${U(e|+<1|xNE{I3PTN>2KG zkBQ~K(&!I76NK!7#xV`p(I{PJDkuX=Yf^%-@1Wt!E_pBm&H?bbm)fYl?Nc@^LB z4Eoy~fz)Va`$Gi?iE0Fh1+M=*dE;;4dgOFpMFj4nN)&&5+W&*D9Q^vTZ1>f0#!ul7 zzA_KT>Yd>@`o7TjV+sM`g}ET9Rz8frsv2gDEa(fol!sIis-pwD((MX5`g&PR1{#hS zql1(U#y{a@<%f`(cDAO4pa{lD_jH_idWW$Q&G6dkSf6SkAMLFw!7CtRHgyz|v9F@N z5CUj%nXQqm-@*Q{DCqm9B$`zV-tKWC&l`Plvq}N{sifdK6sI>@`4RhA{z6vMrD=c) zV;S%$p%6DA@j~7alBl_k$@q{u9-otQ6q6&_7!{B3MIgIRzw7K166^1n>OXB*s)Ake z>1_QlXnqqLiM<0Q4^Oa6K@!s#XF0@vpy9D&q}*ehzVI*lgi zm-(x%7;HS{=JZ6l>@T={pCF>!6DyQGfn8APV{Z%*V?82VXi8ih<{OO7r`X)zo<$v% zy_8qPeQ>Yq+~$RnxTDf~lQEMpg_KB z`$RqU<1KKaV;n|AO(I~2QfSV`XP z=s$YR1+AHA1oc+kY!Yl)XQ4N6G{KSv%*gJj(7(I%oZ~U?UPb2uh}NNSeefLUxv~N| z029!oLNs;O-Som8dG-Osdh4giJ80rgdasmOrwl-X-(jv?(hn|7vg20b!xLp!^R)Gh z_v+<3b3T3vn%gVUb#gkMneUA9|jO?Lrp_7%M6v8O; zi&4LAkq@bN8bk(4r_F?mBzaGU@CxH$vMMFR!$$Cw34dNif!Gldcm9Ar%*$#wobd!D z6#D~?5?F`#Yqy8>FhqXS2X98T2FY;h&%<}xdT4k&V=;l!xHxvY0MI*fHn6vO^>O7> z90W($xB(*yVM&;pQQ>~j-m8n$XyZEk<5v%+?B~3pZE)67_0qG~SJ21SMu0k4l;nf* z)jfXKzaKyulUx|@r}KIGW%P##LMWQxx2d?!^r8wTTI{HFv_p{EBZ`U6I!hD?%ocp) zOfOnHF(Q7~=-2X1<)we{qSRBed_W%BHyU1kxtRp+B8)ub)l29(N-416;!)h(*l0Xv zG~k(^`Ljadhis?sPO;OG%`?7?K?Z0W4rEq!-5gSVW^!*AekS(2I-h3m|Bt2>qHi+v$Vhse=*!0w3p_$`!HS-N|8I*nPAKu045-c8q{1 z@rrANRo@YXga$Ylkcoy129GfY?nCSHj;>GP5UKm;4;M{lEttRRSXI|{wYv4_L! zrw~ewSU(5-7tI)CNfGX-bI;4n@f$yJhEBg83zLrbhQqf;m8BW~6a=W<>k+hAl{Vtl zPa_oqlR=EyVjApnzbvWy#{v8b&Uyl-J{06ZT!H-<{wvETV7hTJQTHO>lJH(S^ZvV~ z;!JN|oFvj&b5)TSwcyy78u5}o9k4@ZLT)gk!PIHGxm|v=eNN1~T|$lKS>ECJSyB%Q zQUCXlBN{Hh$f?jAe@<}@%Ft6p;pj-@7HB@r zF}AkdW1GR6xO#y!%}jlAINMjwaSmF^2$ir7wh)tya!l>44pfOi36=3358ZbJRd5y-A*UC1Fc8l^pC2fKZ1mUhcs( z3gIeL`#I1r3M|!8{fhauPY>)O%R=y!5G^4^|g7`cq@^B5~vlXba&uu(T zkbX!EnrW=&I@qt4E&C=bt9sj*r2}m%R<18EwkvZunE#f0UvA&5*6K+;`tZYZt!Hzx zdW0oftCoeY^t#b<2%mKr^#iZ_dEMICrGW|4JVumG96~n<-W#3w$G?UCj6E{xkS2uw zIeQYBj-WK(I}J$BnqGfyKS-6i8B!ym z3DsLp(WTh+`J5bz@i3*K8l^9=cm%o<0biLatAL-*FXzYYl_WAxGuvKhvE9Cf!_WHk zBK}jE$1ZKP`;HQ;$F9mmp_%*y8|Tbq)q=H&O>nJM8K1FDeQnUR436vMydLkT>13@8 zqYk+z1I2h~gMS7Sfu>(LYQMZ_R){r$yiqYK)*_gezuJ37Zk z;A1%Rgi!F~2ZeapoVYMCLzb&v&bjCVh0qZ|UYdcH)f zrw9Tk=kdSf-yZ`%GBMjTfEZxT@wrUPaQhGzZ|pli_U@_%=(Nk(pD&O_5$)WtoQP__ zg2%99TA627!^s-2$SsafF%wT=uu@w%Xt6s$vyvc3ZtC#mWJc~3nZDHY>YRTv?YX(0 z3c9fxXSe|WcKA#U9kDeiOn~ke1&V-ONOURvGq}F3z^(Z6PynytsT^?Qye9g5oCsSn zO0mLKc*xOrDBMAcyjcw|s=#VIKb&4v2;WiI6bqGDQ0~(GZ(wefeIG%WbeFf5@Hks# zkCi0VhP>}7w)%yP+^dCX759YlH|q(gf=m(au@(nTdqi%jmk0A{xR+dSG!`^v$kdh4 z@Z!QolBN+gt+dKB31pm38=Z6yNr`LiM`7Gi9Z&ci0U!7irCx0aB9uud^O#s^QIr`8 zj{-;FX)pbUV=1tr1S1Dsz6&0dj451USB+Caal$IFd3NKUx!w;4i`52mR=ebp<9fxh z(p2~;(78~Q3IA-pQx+RIbKafoUj#r`IebT`b3P^fudRy}*@NlUX4;fQqr+z&e;!^^ z_ry?3A%{l({7QDTW;N}(Fj}c~vHMNu=Jb%Y0FwcO|=TVA^G?G-6596sY}!*)BG ztQZ>}3-6odl|ODu9rCbeRg_t-N(lVITM73T8A78%Si2{$uuLhCDgmm?)|j}Pj;%oa zTCoP>@InM90q?T?$%}?#vKx~JE*f3}PTNfER2;k(0L&Qa#NKQBRsunf5#AGUui#qX z_thipsPD%J-YebE<|!0IbGprPYeEamql*aQdseKF&1R86z~gGIlEX9_o08^?c&d0) ze!`E)L`r#03HE7WN+FsAzXpdjako5iQYFQGkzkwzQY@~WiTC&pTN%o?TWfA|=V2lp zV>q?Pv#_HsaVRen`R5Mo$A*q?te?M;hKWJ`V$j@FB$d9wjZb5g4dWy_%cCdfCFYjE`|RMV&Dm}D6jwe ztMRrc@|bI*MIXiY$uO=3UJeB(VtOG@^>FBKtyI3;2P%Ho3UK1yK#;DjvN!1M~zgHx_7)+h-TNfSr}XBgOxzrH{sMc`_b$}p8_(rYB!2jvoUYqf5RJ0KIm1Re!!%epT%yvH{Isf*YsMNkDPvBk|VG14MH5D_~7t7FEz(qym*`HZEOQB zY_B5|9bg5ih$4hP>olznPv?+oMp3**a&sSvknj25o>z;I*4*lDyRNmbF{0mlbDl#7 za=F?U+j_JLDsJ6SQ2G)=1ADb8Ua%;|qdz5a*ApTZgzY*Q z3yETdP_9fc`L=!xAkeF`)}iy9LGXQa+qOIFMEEm?Ox#W|9rNPuet{g}~W5J>Zv7K!gQyCg&aKP;5oGyR9{wmwn#%TNT77f$C z?Zuy0-w&K)EC6%YQ{Af#glD`mfvNTXIvaQ1;6I>MrG{D)c#u6_q9 zrSChiMJ&wX1uWpWey+UK#EE_6kGY3iO<1+d zM!5c#%3VKMV{g8>-oxQ+Gwdk#e_zu9-X2)Ar;VYcLtpo&3Omy3y|MA15$MxOr_$fYZfl@*PyhUb3zxI<5OjS#9ed z_-c@?4`a{=YumpctrX-|?C_1&nw)oy<>-`qtfhs0^`!WGB%ajAYGjz}OU!*p8UolM z-M)I7A36WK6L$D_=GqUZcxT&rV&9}6A2tbZ30ceTS!Fv~X1`A}R=f6uze#`nMve_I z!rkrX!q-NdRdQt4lxF10vp zIg_hqKc6>x2@6pFjh7l~`xkpd$3BvWV6~ z$k*Vb|E#r`psoffgR|??jm_-aBTn@wd;{)mpE|BG;ipb{zaNUH`zZG94lJ#d%YNI~ z)YaZhZ))9|NY4Jo8u-0uScW;AUUougrWG*vL!F7$wgZ9HZ$csN-F$gL1L; zFl56*n}6dR2Z>_QiNfy`X#gdHYJla!&>2<81wfM$XhBmc42kT+$%v+6$c3>C_`ieb(N2X4v<#IzKTUYjf zV@(E^Dx6I76Kq5OwnIsXX(I^#{!L#j|NjOiI1cjpzsETk;miNtm_Q4;+MhE5#z18B zp|D<2Y5@U#eNg;>t(AP&>xuF*phPDN3L!GKw&kU%94@;mK57515_bLv)CxGkc{?HL zR}3Ln9#bN0wjz^(D}hA^e89i818j`2c>(_X`$@nH{(U%j!M|+`ne5+(ffxMu<}l93 Z7b6D)2y7pzKSRJjDKRQ zQOXqdarM5x{e5eHYyY>`w%&K8Jf7#iui-q8>AEBL8}4RiJg{4nlHDXQg1^w&>hIRZpZJ>7dLDp3R=Vk$dQvD%9OQp0%B@5m z{3E@Wo`DX1&k7n=?v=EcXkSq%!W2Di4db)jtF{^vK^|Ic3C zrdeM=GrdJqOKV_gs7{lW%_uYPc#G&+AD?_yZEbvVR`X!pW%tjo*v_6mU%;cS-S&x= zp5AVHxJ8PFjZH4};Mmh+Z}_-_E?h8E_CLN?TU(mBE|i1q@0Z;F_=Zhu+dn`0@89UF z|JQA{eST3|YTg#7)4U@&{!9${!QZ=o6DmZ_ieLSoZ`l$R>$vj5@KFi&NH<(14IQ1R zii#k`+R5o+&>v;cuV7TQT!>U!Cmz2kkrI=sI zcrKYgb=FWH_NxbmzoZxm3nwFN9{YQ?}#jV%xpO{!p zDLV6w@2N{qc+WG}LEKWpHh0mb*+G^MFV1mDN=lmKT2sAv@dEE39U4mW=SPD~t6fQP zvCW4f_uOaCBJ%UMT)cSE4IkwEu}qMjWv%Z3>sk(u5NY$LDJMI!H%Lo|B_|67hlEVk z2?gkC%{C+kSX&EIteXAoo80PyxKl7Zzq3^tRphx3XkTk zOYG~}r^h<>m6Qf7FI~|Un;p%}aT{q}zc+E4QMY6%e!5Cv+y0WA(o&VO`O!@y&8n)Z z+kB=swC7k|{`ASV?frYb@8@SH2bhw39FzURvyVQfrg(XI<+=@M($Led+rEA7dXv@e z-SnoWrbV7Zs~H&?txud-DIp1bSG zhKAXC&2Q@KSC%jS=Ku8RliT+`73H&&>U^)1T>GMP%9n1j2;;qsgmU zFh}=V{5o>1Tyys=cI4*f4z0~8EA*ZCP%JAi9}*l)P3dpCv(t6)&sa$5y?g7#4HGZO zl+99~|2m6LP&aI+eIO-@d(S=n%r9xJY^w>equ^83sqN}Uk}Y5!SyrQAX4fmp-+uVUBz zyT|SA?MZQ=T0v)x;=;;6v(HlY%a=U&!Cil_wu{^bs9RfGg*R-(5)7@ zkY`u7=kmzd7?X~U4yJI$+}xZqvxKOqsFUWFxFLf9{pnRpOG^n`92m{3F)D{9fB&u> zsEUv~_rsn$1O=HUW+izwzHAx(&u>J--TwK}fBz;vkW4SWP$K^M`rqG>Isg0hKfn3^ z^6H1*uAt0c7(R6U#*KT~*{dBK99Sc@A7*Fo($S%HI(ag!2W#;D0=Cq>2M-=p$c>JU z?l(1EN!gF8l9*@_vcSd3ney=AhDB2>I_e+_D)p5cHzF=xWaQ-HO6&Rj`EyD}#%uZr z+#|WehYsvfH5)c;xYEbS#AIY_9NOA?;M(=;dgbi_R%T`wRW}Mr@&)BIJTh zOc?S^q!tAa9Xd2TI;yF!&$MI5j<+o>8g(&(ceAp3`yV`dv@%GYA}uYAn@~S=hzlQy zm4%Nu)jlSP7N;wwX==**x}m}7;K7j8RM89CqMU2i=;w@$n4{brICzkJ#@+oFa4U2b zrAsBFIlXV+vd+)X%WmCz&0j{Qd3gs}5Lh3>cf{EvCugVZMx@uRTd{rn_E+`w8kUx8 zF#{9|8V6a;Qc_Z+-5xx6aCmI2>eVZT@bGY1S=qq6Jon7urlUeXe*OAoHOKSi%a>P; zjoOBWtMdyAVy|Bh8y|Nbd5Y$E|G|SjRV9gu0@Ksed~F$i3%^O>DlXnmLrs+&{Vzyp zJKWaRCcJ$+FN=_>Giymfe!ehwln5&WspBV4o;-W*Tt#K2rq4-SkJstbrq$^WGcqc> zyZPGN+XJ3k;fitBvA1qrympQA{{8#?E%$aaFfmo#)rt^ij}(@WpkKFc-3ehOC8db# z*SVymrSGPsytE~C0&``gf7sG8=JRKJF)=aWjT=`+Mn>xD>1iB2D!6s)R%;iROPi0D zSbu&Mn_E~Ih6aV-?&9lNg_}u9Pmfru;mNUUZ0+Dc%g@j6HQpI}wGJOmI#Nr|wVO91J3CFPYimVSRo8CZxDo9U z)1-|zlRMEW=JmZ#(9qB@_sJ9c3Nr?*3KXW!t}ca=@3=+M>#$JW7Jj=301{yOu3W#a zv+(;{OPc^-s? z@Bv3lyn+e}DT7v-BL0vYu2nO zDk`#``c#9#C4<(4Zs9r9G!P|I7Kqp3a^7MP0Fv)vD@zIdJ6u%iC^wp#nrJzp!^6i> z9c&sC3U81EZsiKt0^i|Lo_7-y}C}(o$Ea z7$zx40!^;s+sc5kz6#u`Xr_T%zW zL~_6quU7b~iqG{|x1T{HdPr+qt{|FvV?VZ7>G>Jk(Y8k@OO+1{ZeNLw4RYyuDi`?9 z+|JH!g_@cgY0iuDzsRuQ=iw+6cT-b?Y5#43PDe4vhIe*^Z9Mp(e|qGd$@4SPYn9Ii z(lT+Y8yH-oP30zAwydm7ch8>6(f66|KR)a2O;YAfH@p+9EouXj^S3#emHqkS``9Fp z+TY(lLFK$6mRD$+{tb`D8P8X*Ub%G@KC@s#eUdWe%OY@x{Nk=j657GQ* z{@Ogk!omhZSX2qVtE{Zd1%kl($dg_5BP;mnQ$Uv$qIS`&Y86S{mmZ& zDn6W>nD65Zrm@Nf9Lpd?j|>ZcX&DTeoh__J^?SH#3`Woap)0gFc++^UEnAA%Wb}ogJqu zvEn`qcLsFO{&6UYD4!@+UVQ$meCDy;+8tgDeSLjzTU&$Cd{EhPedj!|^3{I2-oT=` zcUe{Q`I&FRGBV4rPct$z*E~7#4xoz>VC>AdZkx9$8o*hanP>eM=I3p3CzC&aR^g`R zSdst{n6Jj>HVc>11Ox=^y}5<9qM|~Iu24@wK_NlmBscJ3#J8Sj&mMYmFoq3x7jDAs z@;M91+0qy$t1?rVOM*Hg+E2WDFwkESaw%qJ;d{9qM)}h3fc5~U0Pl}c;|p7cet6rI zmt`)rJvun>{rhsG-}tw01j8Obethe;K5Qs=tWdf-pF2BL#!5iT`ch8MIgI{(AZ_-9 zL1JH0bzkXubfxa`@)NGEt^%sQJXa<5ePu$$C}eO3VDow1Rp2PX9hK+5IM3~UieJ%L z{qSLK3=!+vwK1ckPN5MI%XTNf0;#%BebPbk71^{&pKrtBpFe%G3NKz2D+`*8Y0bPF zY()8I!WfvCnDkdia)7y#DHC0{ZeU;_2t>>G=NqLVW>w@y6++K)iqm?`+$P7Om>SS%}y&wHP7k#S9p59cUg zV_OUWj3&()M8NFxT-yEn6pFHn3cvI6SaZOaiHW{`M-C1Siu#EYBmCvzxE6F)ih7P^ zDId*;bn}L>lfRy*`pv)iUA*|a$2DQ=iB?ao(|}=_K20;SMdjSX^W@d(%NQIk7onnpF5jDE^ z@t~2>xo@84C`~9<+#p0C(<%A+?Cg;=Si@OAifxV^Yws`K_ph4Zy4O#4VQjox8r6iB zA}l5KxZ-#D(w_`qA&T-y_~H|=@6xH0S>J1=xuf`d9bH{n`t_n_t?HtePyac4P=D`U znn`=*!b_K!uU)%FNqN7zWuv4dH3it{2Q8rk1mV5eqNo`D;DI=R0)~uPNL7F$8?fZ7 zy=#{yU<<$i6SvgyxWSz}c2HAPmuBq5OnDi}A;DMZ)FH!< zB_u1b&5d&6#0hXZozthu6HjL#gAOAGjbvtqufifHKh74e7Tm!}kv;Wc8$NNXiG=Uh#+#GBe$_xC!j-Q|xP4no zn3eSKr%#_I$UCmiwkq$F8u~R5Aw+$)Rk{?76x_Z7dw$vd#X0@cll>axit_V)cY@** zrBWMxFqjoAm$$=6>3sxEe`~!3->-lxB?@X^Zh2rA!tz*X$N_wvAM^ykc z4Pn-ojg6}^kL1*%WZ+s)L0N$=q=kl?TpV5qJq55bbZW4!s=As6oaXiG*N|fB{>=Sk zTe}vQS%_Nv*tCESgb3IleR;*<9Lr09m(NOu=D(LO$3W?^`&1qICjyIUn$A6Pi_b=j z)aX9i-7x|cXxeOie2nz;5!?Y~YRiA*kG;7?i-KiQ=6jPxc<~;#1w=wyfIA+gQx><& z4sSLBFp*|92YXr~vYVTm+e;Ktwl!-yQkz^L7?KtSx^HA*5nUC*GB|iyjV@48=)uDi z)z#J3DCxuPS^Tvzf(^m{nc<^bwrrXG^RpQXo({!=hKA-zg6Gb2Kg5)kl~eBDrvbYG zi{=75b~|&XHgUV>Y=6tg9iCrT%}%&!i3;Pp_^V@ba`NTm zKotRc#)l7={rb7%{BJ4nZ3e(Gtb!D~x)_tE&b-0FP;^sX(#Iq{N=>EyIn-Q7u+qp# zEbvu6QO zm?6p7P*8RWp?4C5l#qAs=P`0q1d^6`jRap-ooB$ll3HE3fhdCGT?H+LCy#&{eCly6 zuL1hLo1R|x>E979+-NdM28zw8RU-F}2Pt8f@UCrV!Q! z)(Fl{&@cI{Cr^Cu9(DouBB+jO&E{&15vzbeG zl!B2YJC_j7%*=)EfFt-cf{xLr_nVkp=ud6j1Lo%ZglnL@?}s=-q;oI;lYIM?zJ7i! zceUtOtyShpN=hP><>pOZp|{?hSXZbg7vH|!pSaDP8n|LVga?qm!;ejZuz=LHv_iuE zh3P)>U?6EC3(&Gq&8xAPFkZCh&YgSW`f)AT`sCD95D)<%QJ>TCjp!8G+BEo8R?tKU zGs7bzHk17oZeCv6XU-@R8IZ65v?G+k;dkjQC?5S^zJ!4WXc!vCu=w&DA2|~7-23|{ z0DeOFF|k*Yl2)UtcujmTofkQP#fP0h7&3?$I?)P{Y0ztXHV7bprO=rD8=T+f-??@z z6lex2>BC2lDqg>41hT5Vz_yNXc76Q@I*znbX38}wEtHy8DkzYr6%3$61>VWEwy>t(iey9k9>R8)lAwfoSa>(k!2h1j^bVDuHyty?(( zEBN^M2oi?6|F-nJcQzGlr?!i)h)>m^F7q1u=HP?98c=zoGz=bKVX&-KKJJ$3x^nB5 zEt(Ol1#y4tOQC!fCU`c#dUfTg*GCAM0AaFFN^oNzK78o^{CT?dYM!1LyR!cu9>CI* zve~7NAFqPOKK!9r9u=VF(LssM?(S6VQ?l4^-1ljKU`%)wfGW6QKN=wJs}?s8y=nz& zgV$)=O03FI;1*{W7n@V3R`c@ST(jdhx|{o0yR=2QzaX{~B*NAOpr(QT{y@CNz4J2@ z23yKC%m==GyM%gY_v3TD6y4v7qqVK{{KcrK*eh287iPZeukeN*M98GAtthhuntyJ7 zzLWcy%aOx}MR)Ar12DzzC891UYzqdhsJOV|{rf`G5=MP7cKn{OoE#ewfea0I`S|Pr z$M{EzdW)^m+0&zc($U6-iiwFS;pWY}UoTdMoPl?A;J|?^n3ntZ#Xgh=2zh_+tHBkD zKzc&M0Pq16scLOy$MP>a_fs5lJw0G5%nvqTYX7V{Z7J?`_)KUBnkFVZ>>M0MN00Uj zU+d}VxreuMb9dJOpW>wr|wC*=sr7Y2`w zvhP@>l7d3mqemMd1*Iv$yCA4+hd)NgfAQB53or;!P98OXOVux!oUyVGmb#IgR!^`ANObgRiR2M3YO zn^$9lqPQ-WFR!wHH}R0+Hu z2rOhJhxpbxv0yac3(?Wq$B%EA=zSgmv`*K7%cEssVSzW6YY^c^fSP3VKp@&aFW&>j27%aof9WE?%A`4FZ8z$ z{@4#M6EhCr1n2KAY60{T@Mm>l)~#mEW|##d=`poy8#ix0HEppl_7wdP0yPKN`sm?9b(At<&;cL7SA?mQ zl9?GrJSi|6EL*;(@7vl^?%t(xaQMPK*9k}rO29D-y^xG86upYNy7Vsd^cq_^VmCr_ z(1tB7FMn`eBMS4fckkY}sNTdz&(6-yI%7%oUyG~-Eu{6`JO0oK6bm#)l)H+)zO_&W zKn?=a(i}3K1Ywfb{u=8b0t(*u9yAA3TQ_g-i099DqQhGre=yqq6(Fj!^t=+q8kXhR z<;9tvw$ppW*r&hstf9z)OiCRsd2-0o3do44TLfHUjQ%3QeMujjr3ahiAnQZ1z#NFI zTX(Ob=*PU=BoUqQ84z#G$LaYA2n1hSHPDv&arPq$FzM6TU$Axwz{$#z2Dqk%uaD|o z-Xi;!6Whnu&dz)4EnVH>ar?r=bScHUx|k_ToDii{-k_rTa#6b#8Sj)v2{upA*C00_}SAjvz` zTcX(Y+G-y>a_m5q3Ic2KS-I8~!H3UDV-b?VOmtxosDTcvfEo~fS@;jpf;*pf)n%v| zU~R6TUtJBXn%3jt>6uolhYnAJsy99D1;#VD+-LEsK?~~|W)|@;z;ki`w6H69059g} zxc))R!^P+ttjHEXQ2$WuNe|PMSb0E%C1<`}tiLL8Pyfbd43JA+)LZUU2|wG`55U zp6a@~b<)zzxKQGa?b*YKOKI;XN-KCdN?}|~R|l$9%wUeyr>wGHR*(xy3{b@=_sgw> z=Et7bHsIGVdZ;VS7Lqa0I9=?#y?i!~$MogD&*1`qk}mx+G(>#%)vH$%EQf!xIe8Ms zam``BEC3+@TP?H&;DWm#&TvV=CsRP`q%(Go-}XR^V*Eq#!5wnl0ZTuHpkREzQ}i3c z`T*$TrTC+_e5+)+kHj?e=*7#kb+JM@)d0cv1jm6VhaTd=H5 z72ept`~x(ts1`Bpbq;8<9d+jj1} zcO=Jha(dc%N}n(^t!+qJ$!~%m9XWEp6%_w@2oYKjMXI*_h)wSGTM9)h}o&W7m=>zi`*(c(`VD2+>t_X?#kCG znfPTTZIVmj5Z`8;j%V$?0+VjlKs9R<>k{1|v?L_6h#Lr42JS))9{>oLs0>6j+HrccZPdJE z?0S6sC(>f!FXF~lLXAMdhDb#?8_1vUf&O}vWvIpLfY*=?dieNp^-X!l+%wBHJNO3V2H#gyKN3ty{U}m7F zg#hr`wWR7m+d6()|DHlRmi`|I)Scx4YAANO=Vwkqt^Gu*@)|4ODMLCpicFhBXLB

R0`k<2u)YQVZ7(r_m8Xf2&VJ* zFMD!*am3t@={&f09Kvj1Fu;?j8$-nbK+_z+>o9qoCJUDq=6ec$4-|R*^>=+%0K2ep zafM;yL4v`qVZm^Lrd0g+(lGGpQ|s5@Y+%xY=r`{)zrE&8ZvZ5Ds5hY@AzHUf{J}g{LGi^V4l6HL z^PeA)NK`q`4aW}(;#ak#fcdwOn|z@x zO)nsv0GCA1wRu#Lg@R>&N%OqlrUk2K9RXHCe+sexv$K~BrK)tb+ioGRRLXf~x9bjk zers1L%t{0h32gd@H*$fVyJVZ&flm^p;n(OPLIKGR_x9cmof-~p_XD$;+Xv7aR|{+l zwDO;$CF%mk3fgEmi8<9=brb-ggO73{*FpieOqaTVIJ}ei(Awd}rMX44S-QDxxEBp!@|VC9RE@v5H|) z`!gA4_N1YqfuH8r{P(ePZfh;WL<;s-|%*;Aqr&uS5b5VJF6WV+io7ly}H0#Net&o;^AyT!T{5?7OS2`RY zZ&W(BEDRosn3pz+%dzI9OF*O=FvQShh#-%}FxBOlTzJtT<5tf@qkE`5Qj5i9n|?zd zlSL2;Vr03e0!Z&m6pYxbSBcpSS1|3p%)ZW6&DnQCzRrCwc*#abCS>Z%4J%tLrZ@98 zPA)E4TNB~>-$TFrhrzhWZeAL3A)z#3tL2~X^s7}2fZ6$4lUEPzqo1GWo^uSFoj*0> z>xY2W1=v2fVojuKu|0|Gg|0%1Hj)(Bp@BI$GJq=G;u2H!uU}VwX}H0LcmRn1@slU- z?MqfAzAUP4UzdA^%stQvwZ&-;0Na(g8gj98baaS1@GZ{`B@@9GBI6MMhvghqi`PI2 z*d|0B1oY}39i7KM@2@{Q1r2%6b3>~Vua)}oQbBM>l4T7V;Tb@zuEHA1LxQ(4NrfMF zHjLDe^mNCcFCwqr*iE`6kV?mn7+ohdrN9gToBq1B-m;wxD3qqoiPEn)AJw zqZ_RWi-@qpUL{o-S|TEHFX?7nyYhiGX=g{$lRK~WZ^ZBcS=68z%i2~mJbd_280rHO zy??!&t`Y_sYRJqf0n1CBd3N+fk*7jP59=FL@GD0;Z&C!-@QS6y?*Vr8aqEYBpDU^U znbdl0RbEz;y?rw)!~K(4<);#ITy1i`?;$g6HTL{n`u+P`yNAvl9S2ED;%yt#nlG4m zhuZxQqu)dZ8Svrs^dh~epO>G>3@BOfGcsa>Jiy-z!K;0^ z2-U4seaUNE!2H5cZ!=45Bs8Rlw0ipbRgi>8wgt72*aY`>UDEtm=GQas(~kDYuySPu zS=@l%xIKU_>(={vsPQ%pp=yvOUb}MY0W<#F!k6{ z9Ba20swNveYB+Max)kjAmMn9nKQM+m%lvjO{rS1i_eZ@1i8s05s(=W8StEqbW~4QZ zNB-EVV~q(^YHH=)OImk_%H*!!2>5pR9H;=+CZ;4MH+O%@2aCm-eXi%HhmArOAh+-d z?n6q=$54jj(;;Ky*+BLdd9-Dpjfn}&See?ox(v9&2w-}A{WYyRay~N2xW41d*^Y+1 zM#fSbpcJwOxyom*Rw!_#o*XI)>wz2zpawkQL3&1Yes~=mR@I*dUl50mMgRlV_NuhJ z%)o8eG7Tt(s17x@e{?)NwvZSIO2gjUip;3sCpZXxZBon83C-!Z?Dou3>V?wqO((?AW~2xQB^K3{~1yvEb8Ni z7vcy8$U8RmW7s3&J?06aOhXodbl4_=h~)7dg6b-#)$%W93R)wgltf&L{ufsVb1* z;Y)Ax_{@MdWP@lP8XGHAW7mSCD}|5+BJpEcSyr%KqD_Box)X){n73fhQ4jSCnIdvj z0H%)}l1u?Tw}}l2CQ$>Ml}FS{185vdN)Tc@C`7r!0WkZbpv}GP?G+sB$f*Ncl>-Zh zBYblc@75In!~}gnuTbEYu_R$109F|NC1MHYP;U zrv{d;Xr`p>fHV)_#t^Kmf~|wFd7$^16X{<;Tp|FpZLzIo6%=aF4=jhb7kYoUB>WF_ zS?bfK@Vyv!#8Tu#mzlA7X6NR}k%-Rc-ZBKg!B6pcd{$mX0w%I2DFVpTklKrmR^QR% zeFg9Y;e4yJU#?+-aX&=wCQKQ0?Z1Mc)z#WWRmWH$Hrx8~AhayZY4oK_HZZ9GJkaK= z+S)j4Yio(@j96*Zor+{qq?q;h?W+UD3CFw=47qN1Of@d9=d;8s)Fje~$h~7E2{l0t zCDALg0Xam?@mW|{kVSzoH8~|Dyi7t0fDU9! zAx1``h@T3YRYQOqquy2TI5U0$OP|CHNIacvd6efo+3D_4iIA}m#Xmnzpv8m({gL@a zfEeaAnR-b6&{SiQY#kpTUuFVL0Z6?@_Y*vjfF6KP0_wxUYUCw&)|132Wnamig@uJd zXqO-yl`yMikH1Y(oLTuD&Z9dlP~sjS`U79E8b5>uSaZ1InAgn2QRfduY*5Y8gCL{zIAYORa+<{KWF1~}!n(g}P)G-HasamjfS}>x zA`M5DFgd6mo~t_eiH%E64ZV0F0BBox9yhqB4gzZha-k%9k@wJ`8a^nF$*7|~tXj3o zW59^>;=e;U~l_)G^!p8!%xH90fh`8RS1oR(;n8cC9n9v&XhzIC8- zsxFMnbBA?5nOA65QC8;by%8HL!aPK72~U zYu1Omr7CTNdPY)`u)&EJhK!Adj!xPkD+KA-sHv&lzBckga=Z(cOrkDWY~5`MzuS@R zp%A=`9HpGs@czczN-Q`dVesPx0Ow}3<3>>>ImA7beP?qk6cOv7sKWtgnhwTW$(D#0P^uNCKRQwQz^^=|Sz0m6Ovz1Lb#K6IFBe&#WBg zh?q?zb$~$XEY_-#nHgh$y%P&)eRxPn$n;6Ld)u}PU<)Zkkah#tTvk!B7OctKZ_EuW zi#Q#SMj)IHl+N^3L4QV1sHI^BQX>nWBnY3MUC`kg_U!(Q@0~M@7hQnhke4)v{X%DT&pXc6*}>OKOlW zMn0d~C-Y4rR`&`#74%JVOhHxEvZPy=gbQXy%^RM>DS^-lKR=(I4>wSH5yvi^J9CMG zn1Dlt$C?tmoXjskNpOdmL|_RHY-rMBiUb9z>7T#Mao>AHiYL4f zLaiaq%DVO`bB7JPLW0z-nnU@!-!nEkI*C*UNlIamakQZd2%Dsi|xp zLRTT`1ylS2CYwYa-u!q5Un>MVE`Rt6%pDRIfIZcGJ3@Fl9XfweCULMh82#8 znI!YqUrKbIFoG+uD&&icoP2Ox(7R@1HD2xLo=(Qx}PRFvg z(Ouuw6xAQ8nhQW=B-x1QkIU0OE~p|vsF&Jpv%cpGu(Adsn$eGuflEZ9d6i*I{8Bya z|Cd9y@2sEx0th5d3Lq=#ijcW5N!Ez35c{#KYcA~fxvrlak=g`fP_I~l_`zxlL8Gve zD_4)X2;r)5bj){5RF@!BZW&v9O_17Uq417=a z(|x_Gvi7g8&@Ge>&z0+bv;4wmwgsCVc+B?8>+59sl+As;U4HaPf@M>Bvj4>oA1wAJ zs|sQcssgE2m~f_w3$rqi)C^(rFp2z+$(AGLNSZ4wJtc;(B(8(>1VBtt+4fRd-bkDy z;2qYPTUfAz7bQLVW&q%N5~6I%lP8>;4nJ0h_h5d=0O>{d`RNnnFf3srfXlmL1juPr z)q%b(5Vn!)7C?j^9STuJ(}HJ7C^EJ{U(oMl;mGpQZiVR&c%(2Zh$O;z@AD%i0=-sM ziPN-JyXdU^eqHza+I5LvNmN+4zj%mUK;Vz&!i&Z^TpX-aPMiyFR<;C}$;Fa8amBCC z9Th40{GDY^YjoWUU=DWuFN2tuYEv{qaQHZE*FuhI!LE{KoMnt^n>dtnH*FKzHR4pU z2?>7>l0vw3{S^}QZ{o(~(`h}?SR&ePJ2TD$)(L05cFoCDrP(qNKSRDoXkNtLL)IOK z+freKkozaGE$D$OJx7X>IM3=v-^n{Yb{NJ7QH_Y*r!bU+^B@?`PvBW_1nUsKBGE~l z8~PdKLWNx&VQ^cKcrBQyTEqZ8zT)|YGi4|O$}{IChg;T-_m%QvNLx3TqtMMl3`9*_ zZ>O{2KUn|^O5p9={N#u!jD-!GHr;#pP`4zy8(ESx5oM6Zmsh7AKz$>^K0I1hA)(7q z&r2s&Wjy^6snziE+KPUkhoj2y8&*(2rNW6O0JufM!S^O+A?M5njQ5_iFqm6@Io0v) zsRUR4_mfY>?C<35`kgg?vU2iG-1xO!XGFWa--uPpir1Fe8!*_6=X1Io*!XQ`X-T$M zDCFjzXQzIhQ(dd_sn7HKT;j}7;>5X+%};T#XtX0Ir|8k#*2p(|DxK`@r{Bj<0`wrC zg`M;T+XCE-B>(*vzA4rC=O0*Vtls>*E@kv0xAbcu4M@?BU^LM}YHQ(8O+~DyM`=4V z@j(x57mIudvYsvbZjD##5xv;z+<66&Qd>vIblpbRz8CdySjoXyP|Wqn>VWsco8r_v zhyK*k(XmE@Q=5JBXDc{hB(03FD9%O&!ow@$z0JYNNzO|YgQwv=pytS#n3_iFr5bAk z>7dv<;uJDSP@Z)Kby}IJEPkO2 zrU-;k5kYDbeYVJx8d5h)#VmBDq%L*T*paDf*;Z;|h@PxIxCe1za6_Wi*Z z*>DeDeevLyGI@z2U#44`zV~Kuaq+pB>dsfMEPTUd7-39Ij2C>Ljymtf z4u-qYVpGY>0YASoQxKd*yzSTR?^omEQfFtPdF4q{!WWq2hPBUfbIHLjnDh=#w;x~o z4oF3La=hhJRKCNLw~o#~AJH-MeKv`X;@Y&!*?D};<&G@#nSy1<>Bn=E10kRC9$!<- z%-i%3XTlztD^ypWDHjRq~OMTGMzA^CGA z2SIoHj3$FNmQ;AMBQ7b)@DiWG@pliH1$TI*V7fSEEG=mEk-@p8*NS=Ga43Tl4@kHR zCqVLOA5aWptJ}{`3Ilwkfnp>Vax!Kd+y_ue2-AIshIq8P?N-b#vu)(~oWLr`gCkq+%q?fN6}yqEXSUQ6X=3iz|Kpyu}UT8w_dRsk4SN z4~(vNKB7ijg3eZl=*Lavb6c=KxTK$Lf7#iYfLi&pibV}Quo<0RXFu-v`o@EFK%@Zj zftP=hGn0|fG(3QA$ze)dlTB4P(O}U!%$_>m9Mn%?6kWL{RB9)BCeDhbVi=&RW=tzB zfs(=7P9~%?R+kv4NXp$cesmyJ^iU(ps=K$hgo?@?EdIqYZPQ{6$?Kw5>&xbR_wRq} zD4&S%$41qDsv75B)V9TOzG94*(o9E+#tI$%pWyyH+uVdjPcetIL0 z4_Z~v0}otu*`hBRD}19Nw%-2c#G%y8OntEFr=t>?Uz+!h=X8%i^op#KrAC357>x_ zS?$vO~35?Cwg~6^GMj~y%{aD_T4-AW)8PN(Eh6l39pdKDW89V91*yigtYV} z{CyA))A$oGa~!f11tmxa`w9sRm|Hh7xB*j^fK{Jywh?|T#+CD9W7V>ERCA-E=|(Fs zD=H#Dp<=AtxG@s4F9H@-8HX|{`A^0V)J|ExI2#LKv{z!H>-zN{j*4fxwuk@8u&<9J zvKsItip{C@}c+8_{_Jv)^Z_XT-Qdyo+_Orkm$5mnob zkJ45Wj0c_=FCPRzmM=9p{Mr#73b-Ro6DSwHQ$C1;Lb$v=a#q2m z`x+MC!1(x|uEAx6cV}UmVhwHj((F8*e}2{6AsFHAo}Pp7J&wU~xM;&tIdc=l!+YX` z&8sUS2y7i4Y1qgZoT7|HP~FtD8VMt$Z;8kat?w&By5KR-rd+3MknM5*vBC_(IJNhj|8Z3 zMl>YK<4#Wg@+*_!UIMucqHZC>+WOFF9Y8+U$xsvyfk^F29I-q*GagovEiz*B?^z+VVu0rYilF1Q4^yt?&WcESS zgObtIT9ScGhdLe?0Q}fj=9f9$9)#*!kk|0dZa*!aLjt)W5L-?jA~mW`dP1k+t1=W+ z4~jGIHJcccF;>SA&fnvD7fGdK-A_)SX^`g)LCHNNxfTd-#F61`Ya9qlW|*N(4TW84IxAYr>FfVVi>Y=-fT2^i`CMD^(P z8;@iQfEm@awr=_O*vOKn^z2kH#`PzT_2Zz?D-cfOrG_}v9(QaSqjQ=;;M2t9B#+}k z<2w;~*5<&Y1M886F5ZeM-6xn-g>qfMIn|o^^ zN!DYTPW}G<03sP_P&nfg8dZ2KoIQC+=3_b(A#Kgn)LobOYVg63Dg1-(2&z8QO*;Uq7!}6>BPS{xykGawW0P2% zPbP^06i1|)9HAXtlQxfstWGFnHKz@V6{84PYE*!%VoaR6}3($laLmVX> zDQ2-jYUM02&rNI{?4YK1`5(#yl98&EKiMuNY5Xt{lY;_K3uVUm+C>RtKo@dd#*a2e~-}?t?#Hi^x~{LIc2-FxBx7Xn(8^5$ky9(s0TV2udFo%4htswVxVsXVyln{ z&XR$}>eN=J(MZva6(ueV&g5h0v`~X@?l>)p?OBKHJL=}ezgPJXTEZK0od`cd&1fv} zc;UAY4=ZvD-VFg%lV_&{!E3L{SXmm!UHpl~Regbt=qMoR+?y?QBiYw8Hcm0~ZV81- z2L)eBawmNah9G{@*BUXaCoVnQcqN1&aq1s}2-p}tI*`UAXB#?vMv|e@5j(dI@B^6{ z8(gD+^4TkBH?2TFe-;;Y0zsEJ#mG@W$YY>HAvnwQMUNN$Y8NU$hI-wG4HvLI3m{D5 zDEc}{-I|)}YUFgOA-zQ+_6L`N91DIa=m>x|-1AHZv}Zl2x3TZ)qcf1rQX}+L0S!4j zX0L5Lh|ms~M!mS8pdi3Z4g7wSf@7C3+YDB+TegsgG2l>70_Y>h+O>utbSLn56=-?v zRP1_1`&rp1u|kJVQ2V2UkSV}oA)it%eEKf?v9R?-O!|WWk~U+((Ik?HK|qn0t1x{oOk|%m7q4tWgekQQvgf zYn0f~p>?F20!1JIDj+DRr>o0~F~k!VM%%L*Am&$JdLqoQ{0Q_Pr{RJTO($I%3h{~V zCy$QvuF)&Hl1zR8yh&~l1=hoF8T^f;Ixum_gS|#0`0;DJD=f@n^AHdV8A9|Q#17ou z+%_27`RqJB8j46ayhz))r{6ibxpBxR+=O2txdoRD0|dJ5z~|4}K0YN3dlXS-$X3Sc zhL#ycCif!t)ydKKo5c4eMFVQW>9@yT+lsdPwnjrVRu&YPbRyuT>c}ZUgfusr=G&Jg ztCY*)xcM<$B96fYVzeUz?GNIa9k9Q^x0lji2jfJ|?noHz)G z0#w0W!KPP~;slux(A5D#6^P@|Xc?Z5Ku!dpt5m&ta|snRGA;5)5coeHIG9&ZuueoI zh?!pzdQpO=igSw7`;2IQr5My|4;*}PkF!6BC$ezP$2#K#=rt5oqk>}wY&7crKWdijsgvLBiNI+*x$JXWIdmI4#Rs$?gRVgWQb ztgO8N03s!zF3V8Z<{vQ9-?`!sO{N9pn_b0<&8L~D9z{%kWUsHiM1 z&O_hHNfHu184o-~2H}sp(|)yeb@DJ%K+Br|0Ekk8t(UsR_ygb?v7oMs8>Av#Bb`8yH za9}{Bsm1jjK4WJSpXlK@ArJbX|4cDj|yAyYP0OS%&S&W+O0aGBFVyJ;w z6P0i%Mi!n!-TVFfHvut#y8F`&1&vlpqJzhfZi}Zd=p0!OlEaqZOhUd0a)Yt%j*fa0 zillPXR6p$%j2#~6@DiL18w6Kc1>RPKJqUY^yMmQQqz=XCA}|;N6Iwvaq>v+J@X!3U z0(s2+>acS6@F3u8&qoyD1w!?q1524d-Rq^+OixHS5)j2WcS%lu!+gehOtysaC8{F_ z?I5;6V3x;aq79XGisJwTRwkR2YIghnEHeeR3)HW$m7sm#xjcKEoH7`6x!?bnCW)~j zkmQvdh&Y}AO3p4_2jObPsWm*QPFc0@^$|tPHsp}(DbiCOF$n2kUXR;aiifT{2ZVm@eLau?yuX3 zO$*p_;Q1LB_f}4IM0mGt-;U7gYoH$;^?Y*30|z}Lr+x(sZ>S+?cVPO{_xCy*59GAWcnP%f@EAjvWFi8Y23gEc=%uGm2s)vDxL1p7b`V9|SbOc8OHTjFg zuZDpAnZjU3DMaSrdQs7~{=vyBz$8?;D6N{kwBwj0Vyg`JC41nGocHp!yIAiS3FM3Lhtt9J;zp(JW}v2seP{ zL!PFKvurnT3=Wscn*X}D_YPnLLL`o9n|0!E-6E)n0f9->ZSd=;{-IdJ<{{P?`uNN% zad;up45V@n@P%ktP#lnDitOKLiY*TE;S9^~m>by!rqc;G97 z0jY1-^v8f+h6vqLZUc~q5?#Y|%G$O9#AOM&? z4s)B9cs;^g4kT|Wc@{5u=pv?5^0XBseRQTUWEL-cdHkaj1>Q8zw#M-?{9SjF==}Cg z4%iF|9U${=sWmrtV+?EYt7Y2{tA*?p$Abam&@|TE>IC~bfI==Oqak8vZT%8%ThQf# zK_*jZ`^rcKzDBM?Tb4#+{|=I|P*J&wiVIFkWVU^C^YhfO`X0AMb|Q3zGd3bn@mU+L ze96Ouhtb4;zUOWE;;|id3X9+paA=hEm%F%*PYC2-P;ue}&*pUX7l94X1g24mVPVv9 zCLI#7!fkqtBd4iyv4S+;FRYWmNxl#e4I(Ah{9z9kW_T@ocG8-t3_xd0f8K6KdAoTA zK0e}fEWXYwk8q>(f&t5I-3q*7M5D@rvPS5hv2mM{xd|={1)8Jc^_bdMC#S7WZ5M!A zJ%-54_)xTqYZu)TOH!UNm@Y_8iVNSjXQ$78*b*aCRX zUM!X$e&YxQtYl%4P*zR^iNXLDr=y`wJc%fsHL}W%q8Ih4lh*A+sL=qZgj}D56 zWMza@GD-_s*)x?$h$17qG(<+jsI;Ef&G|jgKfmWVM}5Da&wE_=bzS#$J3zRzeEi;< z@d_ju6pX%MR2OLg%vvy3nF{wX?_YQ~PF|syVL|QeRcbywY$u2civT74sCd7Lkipz~ z1;EdG!&aN!JwL~Hw3$kEO?7Qn`H7fWnt?V=x@!baH0rg($$Qw*<}MGk997K~^k0QK zE56L~H?@e4n>4Yjx<|L*@81G^Ek{ob$k4K!aLGnX*Y1?%#d*(;*~X}FuRK^5vE=&) zt0QAW_Czn*f=*~#xNLb+nI|ziFWuH1r2)m_9r^NQEG^4;`63%ww)Wk$$tKdQsj#6>j`~Y=dXb;lu0=T_5k@u_Hl*&`YkPO?O5|EFu6c7UdBf)BTgO|P z)UnNYbl7p#;ES&Mx^)8A-5+k6IJ}{&#XpXUdh6A^YmIiBUx_@Ow&BOWHZJ;4qiv4c zo!mL{#qunX*Be+>T3dS zbUOCy-`}>1=aw)$xqnwfp)NpAc>XX>M3yh>LN!Yzs8drrHx58MhF{AaSvgOC@ywaS zInY)6_J-Q*Qo~OUOHSx~>7P6^m3T~Jw0uwt=W_5kJuL6JpN2A84Bvw}{+vPkS>4VH z7d`}_9-EPX@Vzk!rU}s1SV|peOW6I%_9|=XF%dsI*VX-AT6;N#xe^*SXpsFodW_N9 zk~#`4moK-EI^v|*U!hRJ)b`k$xrr^an=Pq(CF*qA@{sMCAx)g?)>SuHJnqTthi(~B zTO)=nrpOZ246?y*QEJ;z<(S51&)VJlo|~#?BSs5 z5#RY<`BpzaR-F$&nZlcfvTTZJ&TBj_<+Yt&s5Io>zlGVFA_JUgib~Nc0h56R@6dEt z!fxN@ZThSum}d*O^TgF>PoZj@$P@9PrE!=i4 zv%eG=dRuXQqE`04Yl911^)uG}*4FO7_^mo+SoqW=q+V9we@H*7h6ExpIOXDz~qWP@CXf95H z9XXAIX=6a}N(NW+sFgVNxOp7_}CrgGMu&~l)F`6Ey`XK8DN9*t~MaN?w9 z70s;zv?$c`Kkakkv|*hNT2rVRoEFwiOKaoKdGmf)j!$XSeTKs4in>c$7YCXbPo9{& zX4PPot-drkOA6J@*|U%A>pbqRAx@k)G@D?(8EbfZx+lX>v1ajSm%@i(wjJz%G>A2q6rJ~OvqPngyRRI*6c1zgIJ#GO{P z{TtR7DQ^1c!GiF1t=3O%ZKwG@Ne#?` zMJbxNf|HInexEPcixN8Z{3Z~#7+q?~F_2K=4ntC*^ zs&mrXuE5$}{lo0pP1+gm;jGw_T{R}D122i>r1H;*s z5oauD`_xjCPzy`T9Fzt*AE?Eb5e3Gwy9F{(5Y~vH2GJZvg2AXZofj-P7#px;9hQ}) z<;An4koj=ot`8;~b%R9@PEEVacWEV6Y9>{k%%Ldd$sv4n-m@e;8?B}u7+e;`*+r+% z?J34iq(3uJfsqX6sP67x+hpBXokCZySMv@Ls(ps`+VTF_^Qd{pO`0842(pMiXfgii z*^su8cXmcY)^9W(2+8B;D?w`bn0H~>#4gW~+7JiuQ%Yn`A8XcJ-+|Na8bm}!Hf+>L zysz>tq5k;)Msx0>SO2xzm>~Az9I^g1>eqY=4=rF^7KPr=5<-imYDjDw*uW{Ypi@ydP63MrR-E5<=H0p>7MXC}O)m?XX>N8Cjh&6bjcU6l@i-1PY7CgYrN_nvir~|0Cpx=&Tj1bw3lT-WQ zE5j!xD;4E?X;K3vOAsY=m}ckB>-bKnCosfB=j;F`ICJDd9dXL*2JZFl)v!M0tTB8! zd3uyAoplE_E^7$yoXgH;+nNba>EI5r$Z2UtA0T%pCv>At+}%Rz!@)ye-?O`X@1CX+ zv}WICaooz5!{Wsea>=EkMyzAs(b~6KJZkYQwAbeKTMG+E&2{Uu>dhd?s>z+N+a|0o z+sPV%w#HG@8Xggty_aJ7N&K4CXg`~97wzli*!=?dnrQHBdO2}U`>_PO1gD~*Hp2!c zwmV*F{#?CXi?!Zkh{4mNMBRY+Im&hRp0V1IRNN#%?cy74Ti3v6*L-v=!H$sFsJD(t zA~$8`NL5u8Y{TylFTNZco78Xi$SfA2+S_;Tzc<}XP2F~A$cjg1Ry%fl`t(%!S@m0$ z^j3XStgDF=vouqK);s5mE*gB3Au@c zUE>Q}Sy|MP2mqR_eEF2Z+>mZLsY}E9kJUq`6PPJo{YKt34o%Lpb?mr%{j=6q?qD}k z^53l`DAtOS!w?TJy3#F!%c(J%l$D3rv`=WWr?2PK({@S8r%#I?brLE zs5+>HCnA^AgPZQPrajvMw>Rl%8@cE|yfXH0Z)n1ez4v(|6`V7mH=P!3EoR!Q?=T(E zBTyr|M?ha|?OVw&Luhb0;bO?M$Nv4Nt@p(X8P)>qOaZb>3Me;X1ntyLHUmaXsDpi! zW9Jra!f{uy7*D?#GC?Hv zL(4H+5S6M*c}1NEf`0I0>imuwNLByI{^Gj}r$k|>ymv1LTQzzKuyf7cs#m<$t} z%Fa0bt&DpEc65Z!XP?S1FK@j~9NsMIh|L8BtsBSpy3Dzwa`oUQ3yUj*4K)Jpl}0sz z_ejQHf*_D_)qPH=k!KP|W6bpHm#&nX=JYt$ImXtK>rrgN-Nqdxd~r7-kVMf_(jI=g zrk%RF{?eRLcpqrLR}0*2!BYi1>%)dl7uC)oGCX_)*Bcigs`ah}aRNgo>SM5Fg$K{C z+_7`#2VCCLAZA$ejsqb#xWkYayFW!1_K6&Jb-!=Hhbj#zPhN9yzc@dqIa_Y@^yx$G z%GN-YiG07`J}q>W9F)eZR5KmSe`(g1mmM8e>s2)^HrOlotVedY!-EcoZmeFjP3_Sp z|MS7~6`!sDG<-qtXJu~deCt6kNBmfG&qW`P^>D6Y*b5ho28L@xb*kU|zW(|rck0=T z7n2hc6=0{(7pjwAyub{r&${Ns_Gd~4r2A2H${9b5XebfI7t|3M8!P_`h4D_ zvk#mM(sNE$LP{1s;@Pw02M@NxT5+cTewMq_-@kyKj|jG{UxHQGijnCrdY^y$q;}zT zZ{y&XZCvh-Z4$3FXwW{Y9qTCb7EPKqrFO2It@g2~SHV$9=4M;l#HcoJ?(gM3eJ>O% zcfjhFCcQ{M zrA%A+ou^L)7R>ov_sQ$m2UwKuK+&EF16lJFFc2pPb3gAc!rq%TaA6nC0h2yu^{jq< zDxs>=>$|o2KTBhcp4TnU?Nk(esQTV|jdf)Pj&^ESnpk!)2%8l#DyhQ$;*=XpBW=oZ z!P4Tp#}WxinI7i7t9INaQIKiiEqUqAGb*4^#q#)NTi%bbB1uAn1#9TFAo};^lKsoB z8ch__JiH#GYDx}_QM(8_6n^r<#`5Qt_w!4=tuEDm&h3$VamMR!Vd?q*Hi^&Ye8$P& zg0K5BH!MEuXnys(WwWblamr#_WdK1EfbGO0EzMucbvwW8@Fbg{Yc=X3^WhdCLsZ9Bmg|rqmTRek@EPNu7{Jtzmy? zy50B3_=!d%18H{eZUF}PK!pYP?1&D@qBJDkoSi%)wD$5MF^EA2x4M^mwBg)4)L_Z%D&k=)i)iFoqSjIt9h z{WHf1FNh(wKNB~%k8{$A7bOATg6G%oA`b1Nk&z=QozV{9B-i{3mH+4SXK44xDJyhy z*R)hop)ad?w!^vKO9KPHeKhQ_Ve#YV&$SH=D-Oig&hbpRe2|93r^hFW9oh86h!H|_ zi3^3sts3{&Cs&NFFMZHJlMi?koLo1KT}32=wzjsYiWJ;isbLP=wk>PPhffRM(I-MH zZkJ5{!JL7>qMO;xd4qpI|Dg;=)Hxq9XRKRPSo6a72_IfRdl+8r@+RqMZFSem%7Q1h zngi7QUk}%vRQT#TMH5tOtNqfIQ$dptb@yGk9E-7INA(W&Uf%jQR^npn0%({NHVxs2 zodC(S>1`*R#%0w3hl+D%b_Od?-->#R@0c0UK7^$=2T801rOSi6#%}`tWwy%x?blu`_YvNUm-Sa z6d92g)}_x5NqZoj9r$e&%fF`&OYry#yU@zuONeo8?V@K73kTmO?!9)wfOtgpOSyYr zw(%Ns=X=(wHM3o!4`NxQo5X4H;@GE4pI)sFfzs@UfAqwKHK;Wr$@6qajUBuF%I>>S zEK^CXOqsD}jhREun8~RxkB;vo4X(Tuj$Y;vbjg|_ zBMmfQrFn4Omk-04{ns#@yknI1%NH+p?b=oROfS3M2xZ^eZg+gumT+>&b|W*ky+zXb zA=+;MMz{v41!DH*I`r{2zP5I4d&}|Zfme3cTd_93Wc#+cogNkSNN-*_M5pH8Y2`(I zvuZCU#NJrj<;;g+JG&b6u9>mE#NNKWk=K;7nv=#6oGrdFOd$E6f~~Tu>sJ)D&f?nk z?t3%yXzC0p+NsmLOMa8fbp|>wUFDqjw4C`EDDl4N{0}nVGhF-l)q!v{IqGbOzKXBC zcT{vVT$X3o&Fgope=Gv$VDh&F*^iv<#FGVMoH$PbC@@C9pm<+m^~aC*#~Hi~w}I)h0uQX5Q!B}Hqx33#M7UmLV;PGJ=e3D^)? zNOZ;&?8faSMi)Zi0RLjcCPV~EAa(M=b1EAa>wtVNmZwa20d7RQg7q_zmHomaNv*?(8L=SE|{XKmf~jpQ#W_O zgh^IAOo5{J7hIJR0k}+is&TruM%C$IcV8Yfv;*mM1TG3$v+ zvzfDIjd*))d9tr_$dC88>RSf)eh3<9Gx*K}}@QGXg6Hb3o*63F8Fxv^OJ48tA7R`;{Yo_ zzNDvA`no`$vvm~HWm?}XYDr+q*oWs(8H<>`dk^ykpxqBla4C7&-t@E zK=6v<5kCTy>rE>A@o6Hfd+wvdis(8tKs(dk9~3k>bnj8YS?c}G`+ho9oF#cdBs)PO zDF5%1$m9(ko^*-K(|x4do-Awvo=)$LrtsR{!CUGL~d&eOI z)9-5pT5_2SVM+O&gVPxWCWL-|Dg)blFHr3RJ1#y|nBTJWGSIdk3zD|^L2G{ZM|W_B zVu4To;Q5kO4t%3aez>izHtDC2Xz#!Yq6fq6gE=!v%=r?hCzC?7?ze2+I_QLRIX2|E zPvVCjin2-D_C&+(hSIB;6D`?5X^mSSyB$0nynwa18Q}asqURV`g@t%4Gc)*ygZ=Yy z%D#H0qes(4I1Wl;?%Qhv=ztTKKcqzlTxMW-yE>2X8!;l`Wt`{vV;9C2GSvjyDHo_N zrDFpS_Y0VAL$Uw}jhzspkKNU2=FD9uDi02H*FM#&*IAp1x9kqKzM5nB$ZYzdm0DxY z&3ikT#y;EQUa;D96N!l@Hz&sgevNG$K|=@&Dp;Hlma4uE2-&x(18+ikV?gf7I205D zEHwkEOE<%a-PgT~i(`u(Wu@0tZxZQ0-Cc|aVm;?bex zb9;TMmvj@b=A{{qQ(W@A#%i~0)uVx-CJeKf;}$Fs3p-t|@8N^#-q}il)X%dt&nRSA zmmaKr-?7@cm$TNkY_-236f_nW)9sDb>WwIWReh|%h?`59uRiYJp!Ep}36_C9^!6+% zzp^6xcCyj)|IY=m2sk$NMC%@J-(2Yua2D42eva+Ez7~mIGh;@X9BtNN$&!CZO`O;- zYi|Pl3awRskW(c?ay+TIkSJ=`!M`VDKn0 zGv5Bi#smQiklTb_KxP#Qt!*>6Q?X+Zu0Eul7s!Fi}Azf+!&T7 z>Xs{~9;D`eDp3iig;+;skH*^0QI~OP;WRo!doAj&^j_w>OSvh{htPYdFtT9A-&2hf9jQ$XLmSiK?<<1R!!0*x!1edcB z^OYP!9ZbqgJ{T&=kg6 zS&8D$GG)4*=8PQ&)EWgNwGkI0N)gZCKQUHPWoS5s3>7~qb+I+wMy zSznwtX=z54g!4!q2nU2Y(8l!7SBtmqBfKae^$9KPfjz^HpT6Y~>p1kyo3~e)3X<^k zZR)zzV4Yrp`o_I>%o<){;x$gIPQtC)vnvZ`4fs&oDJATyhN;cg*5M20T6e8XoAQq{ zhLX)ww=ZnH{pS$p)F%DfIQMb3(l-h;aIH};E?X@*3&ob@m2N*Tl=p$YK zB(;mtjyWb0I~!xm@xG4ipTP%dWuACr)pB-;V@^t{-yXpmm_^+&iE!K;ZuW*6T>+lW zn*RvvuiJ|=8n7~0vFCk#csYg6YE|0&q;U|tZ=TEj&EP`FqXn-oS(#i>HZgCI*@Wet zgWoz286bJpkZtZ|qv%J$P)dzV%+%hZ%L``hF&Qy&=uj{BWgivYMjp*+-?_<@6Okjg zobDZ>+saCPpo^n_$K+rCj9*Fv{jp~KzZ1O2})|t&r>+V9XfjMZQ`17SJN?lQOq+G;|ra(qy zZg%!?%m|-QoQb*s%m=dC%oS%8mpwpwrUk9xd!?s@%c8cXF*Q!^NGj-*eRNqa);;LC@`v>o@CT)NQl&wQK+0z4v~4qUnaTFwL0; zXGYouAG&r;v*1IH{n2KfE-oI=`p+sn`78O5O-vp`80DZWY+VH7#=HYT-A&c|&!ou| zN43!;uCvjSv4T^~GI|fR;rE;MJM9Vzx4PW0SIwS72YSH2**Tx_Wv}fOq zL!~40&ob?R$)97B-B_1UOEiQwDgGUt(`I;=JgcsJq2A_^Znw>5x~+Dsdo|o5WK;K! zgQ|*B6zkRkL**dTu@GJx`p1p8^Jg|R=H3ZYeAe7_iS2qZ_afH}zap&TI}1ORj9)Qu)9>Gd>`s_1 zG%)z>v5F!^x8=s6GH?D_z(j2NVnf7tOt2>2;Ej@tOIO(^Ju-@N7uJ(F%urQQ+^?lDINBzC@owvmqY37*^+6~llqWC z8*Q`IB%FZWZN#9eHd2Dv9J)yM9Q`pk2AHa>F?48$jV2>FQX~(B&qhPX&jKzjvLU-I zTAPRctqF!3sw)m=A3h5&GZOk!+Ne8RA?2t$350Jt_qZcfKkBv^Iv@jf`{YGJ2T{X1 z1nIUkY0&V!Rwk>1?v|tcaq`+}V*+)5KG^nnPj9`$FJHfYLc;XV%*Ai7nR3=)b!Xt4 z%7I;pCFLF}0ej5Xlo!Wj_I>tbq+0h=C&s+Xztr|{w)M74jm)ieJ8jg7-gD6VdZ&WE zPcu`lnJD$_F)8WzGS~g~Qy(9-Z1r>BtgA0P_j7Qw_zEX~Bl%joUJzMuptD)~TkGk} zcCslpZpxh|eH3ghM91;mAJV$U&E;{Taz0<2XUHqCVhN z?Z;?NSN=nYXHTZUGH-E0?TqjVs^9!~$Jg|mKOO11AoP~!dSkso!%|_#L$AafwuzY# zV@r-m>v@$(na`H@C)|!0A+fn$`**H%E&*CONF?%z%hj^OC6ICD?SzcT;qHW#GaxaM z3qi|Qt&cCUEYj5{9Z^axxboGs#t>lx;)32eGHO9AA`>6!(f(Y#z&n__gxIau#K6CP z`#{TNKFw2SqbGkmwSW7(Wtny8(1Bkcy=bXd+yJY%DFim+BExsm0tYm{ts1=U+C{^D{Q@{#)KED)c6p@LOzBZ_Ke)prQv%~>ZlmQdFVE~yrlIJ6HVzugDmsDzI;^Ed(y}weZ6>u zEN`Ei;i*x@*}Say_WS@+U2#}E4X@<85XTHdCKJ*myPX3n{m{m^KB9X_>A${ieQ~!# zoJMg*CU{t6aZY)IgEE+NNGe&PuPwo2 z1Dq1Lp>Kn)rV*cq zHfV+c>zLTgp7kfhT|ki%jK2IMaiuaUtPEo&=z zD)N-xc@~IIysL%-xYc2A6PPxDUq7eYTU0K-CC798sWm-rZ|knw2rT`qWelkhyGR)a zE6N4*^;IjN2o6fe)O2so9Rkwcvjw|xzC#>l=jH|!+05KOlWO|GLBjyJZ< z%X&RI{d$nJ&(BO$@VvFvPWo4|@4cj19JW4?(;E?$Ep4vxcL_@VDnDRSj`mMLqR6A? z1|IV`O_z^+Fz0KKblWYX@g#O+NFufaTA$m&@Wd{coF-{|W&0A0Ui@x#(PKg};HwXp zqzsR?5zCKcY0wQ&{^fkS<&W|rHJ$0#`gigwR>aOs_uKW`X-;~6=@sCwAuK^njW~r@ z(A?51;_t6_PGh7Xeuuc$RKe0m0N@P1vhebS3!|cKcJJLA)vDLlF5X;cw0qD}U`U8N zB94Iq9Emj5OYZ=+BbP&&*(G^yWRc1cZGam;Pr{JqYd0eq(uuUZDnfs^cO1H(qSUR* z7{qgkM?}}HZolYlcF*DYIF3gHX_$`M9IlXye?3v+0k#*81MVhg#a&n<$)w#DL)%!%jl;CdG_Z zw&B2m*Qn4k2aC|)^*j|t2`8`L$?Nd)w|in8t|nph-+{M=8mJ$C%l=|5#}H85mlEwe zRaU@H=!tarph5IHe;<7!zk5>Bo3&1d<1?Vx|(eLKSR6X1yW^$7&FC&NXHoq>?cpsZH47q zf;qqt!VLQR`%nI7{5l>DV5Pk*b>@GqkB|RVql4Bt3xPp85W!2H&f9{Y<24qK32o}n zYZmErhs75I3D#|@oB!5b9NSn3V6Fj&h%!bvX;PTRL~=)%Eh#QO3~iKrsqtcj6A1-1GHMT>Ax{A+;=(TvjvalT zoCJIETgJojqGFi=Qs@Jp1kANS7uA$-v+Rym^)Jz2D4P9 zE1lIy9&tOM+3{y7zt|z5xsANoKEw~uMx5{KEj^!ZFVAVA!-R7x*=G$I9v6r4_bhU} z&L~A$Z`_~4xFLA;`tK%_wZ;t^Np-2-A_)PIJJip&*vTZuR%SyAT?*L%7JoCb-@fbD zuLmDHyF)4}3)4CKU$T?AQWC*@AAEhbM; z3W?^F>w|@M7%4#-$Qo(Lr+Q}`ZF5hu5O8)&RFc`oB_SHsKW$=crSC${+p6-Va^8ZK zDRr@=^<&-}u*WVU_Xu?d=!}F!0o~W7QA~U9h#mL6Ljgv;MU$ z68nX%nFPNy7MB=M(N$BPJPF_nw1+c+?;bGQyHz74R0kJ;l6+b$(=ajlqsr}2ro>7O zAm+?*;-5@)Bovz-Obdz%!b5D7t=*^%T~W>)uCF*Xs;vL2mTVCVOUo|W+W#=lTW8Ck zPq`Wzzzo4}z#_wF4h7!n12%}3Z$cfOC1_YOs%dX3pf=!fFq+VW6QAfjY28ydyyecX zreVkB?^FO#T<>nTooVoUzy&~jUSNrVDAW>J?vZT zB_#l1wq!mE1*GY_?Q%C7|8Qn-;zcd3JOXWw7PysG0QjtBXBun7UaX`+ojo8a{M2AY zao=~YTCj5IjKNdm)mKWNGD zX8#(8X1@AL3qGlIP?(SQN52Ew+-Aja8aQFMW0Z{JxLvhk+5{DJr6`d`g>z{?d(1-< z<~Bpr_*15g7!@uFpWMrq8jZ@=zK&mWUG*<28%oV8m{gjNWjKWQi7}KR= z-l7CwARc4M|If~rH`MGs3k>{Zy>@Z_t+#Iv!PSPsbA;Ub`&m43-n_?OA)avc>;7=SI9u3QpPJhhJyO! zSnZ=wL;mBu1w|p3qbc#=D`hGlb%1=7QeMJL7nj>qT_0bbKdPFigq591K_B2i08M{U zL-K)(WW)e^=DxXd_Rcrw5PjEUlTNh881eMJaww%88)XoZupnE^!VurtU%y&$HeWn_ zv2rS{t^2SY!Oh@+-3OU>U$1_{?8OI3bLD<_1+Zd;WcF1%9l!2yah7hHu}#1wib2-u|`FRIDV~9Bg>v+M@9AiQ|LNBk3xDfid-Xq%hXSE6su0bY=}p4NHW+?JHFiL z>e!r^c2|0>fGv+Iw)!{ZHm}xt=r*V09UE_+pmUF8;)^pUCF#A;=0L%6!WqJzb z$dv!ed7mwHzL%Er>{{Av@#18zPaGICw>Oz(7-4xp)@B5ONzK#@tz%wPy5+6~9`hKYRR``PrKR{gRzesRaU(k)G?weG(83 zI^|E8zVtrC`3YZsE@ZOs=yyg1+`$zF!$)>(<(be}+qYRarAhy|d zjG=R_?0y(QwS!@n(bo!NH|@)%BK|hV|2lbD)boh8bGbjL767esD|agzbH%&Xm|L(L z7D&3R!>4qt%n|Jn4xuHwIX1B9&Wo*?=tA?26AO$+<{6E;G`fv(2hbzwdjNs3RnjF+ z_nduo1tqzZtmxQ<(GQW7q^pn%%Ohtsb0cstIe`_#k5;$thHf3p$mQhHC`{)M$DsTX zLoi{y&yfk;F{J&2Kt>2^6OX5ZvI`1E6K_7zTTP~+NSXns0Zq!iGVJd)`DD^yVV8&F z7f+>%gS@)^@S)Y`UvEM9L@}$L$gU>)={+zF%=PlBdpDRA@yc*_O92nC&{~%e1?n-b z{xOle3CI~hH>Ny0Tz(Pp%rgL}NWaKQEj+<-*kMiPh)lsTjNJ33?pHpK@Fg+w6 zgq&D#mlu-^7ysK(?ohd>SsX&XK%KzAV^)d)>M=&pUm7)HGUMqbm0nExL^#LP*YUrR zCr(s8%?$1}?MO|$^e}Q%Q(cnsnJ*Fz&ne6B?8-n~!1s8@t~_5=V8Tp7hvl2J0ZZjHP)CLsdwR?NlbG;iL0g09Ip^;6AP z67ec{#@c&BisEiTHT3>`SFI&K0i(obDgn#Bqt><)v;3msQO4aI8tcGK40vuju9Zqy&S7At>T{s<7&WYxk>&CjpNJUKN0 zO0@-@jKsRUb1Qzw6bZ}=1-xcl2hy-gJd6Y zmcm97*|x0iu{JER|4}Q=jmw9xO5Re(^Ksq+m^81P?{Th{csQTVV4jkS0`$S!!b%yf zG@O)>iT+^r@zu2{1wB2VXLbtv+tjQinR$rS2^WthYcKf>F(#fm#5=@!2V@6 zuDx&{C99wtNe*0p{`~pL-xA2^BXG|Py3nWW^;truFw06)VA}!n_SU=a1+84d$uNs% zHDFw*dalm9cwr$`RHRAGCBn!$TiF;GG-rnIr-b;_K1A}EtQk7q@lo2thiksM*9>Xv zU@a^$C20=*%5X;GFns3_A3>ik<%Zc66S}Kz?=Yk_z5mSlib@&YcT&8UC#5Lq1;z6l zb}{)5IARLRcpICHF>Acv?U|0n)w}R4tbZNVPYUf^^N!l6w7=g`fv5|1Sf&*w}DLh?hdPG}0>8Fb;bXseL{dGZU4<5Xq57Fv(T6bJ^{&55KbY%U?9@?BY8xqs7M{zWofaoMUR#Vv`2eZ_EdKHfB2CiwkoD zp06}be}aXT#ntGUl2weYP;lILyn+G_SS_q&3o>rlnZbwDRlatSJNs z@(KbJ9ET%OEqHq^P=@lsA?cVm6j20+=153M)scCm{pCkbOWLu0yJV7sXo)~*{$E@R znlX@OCf5*HtM#W}G*g9fKa1a!a&`Oq^@`%q!Lc|ZT|c>T-_1keGj^|g`9@3(us79E>oy{8;+&i5wWnkHlGq*e3c#fz%B zGuVPIF#FvD?;q_bgHHt-0$hH4&h$q|c|y}Io)|Gh^Fo3ZYJz%^Dap+t1&U(aI&m_h z$IzwIoJ_kwpS{Crka&YT_3noY%KJMcr5_qn_8?(tPA!8DC0!@vpjCtLk|?dJrr=y> zc(?~)!)ExE?Bl_d@V2E*ZTYrrZlD+3sG_Lx`XQ5u?C%Vcb&ZI1!riCESw_={ufe-{ zHZ~B-63h(^nody{)mh8-KO#(~;6fVN-%4&g@`_2gkSr1d2;|2I3XWh!h2c&g0&a5LO03oS~{0xY;l}`Gi1_2XD|*-KRCcXRcU38g8Qm)< zemIS?$WH`L%VYtrWgVNobZ`!oL`t*mZAS3IqB|>Ha7q>(2}p(e(4hBWL0TOD6- ze{9k(?tYU`ucCONRMe#m9MReI2ShBV_UVE!E3(J(4kWwCj5R^RkRdQU=VfLhO}|)- zc=bmwbZ_MhpN&OoGj@~-i@LW;qC-;Z{uy2+8+)1_nT63!+^-y7d!4ov94C;A3opi|Hj7zadD>PN-njdWwG~<{?{0L^fQ&>C;p?p@NLTJ# zpv^e@#(ptUj1c3O^ZE0(*w|RrsCArC`*4J@6U2RBquiA8ahPlY5M6YeoDf+j++dWP zyPe|JL0sZ~hJ&*5)9`&XMZfRsc0%v7`ODEEF3wAU;^IJI3ie)LB80Xy*E>2orXSpO z90)|UvYE{Z7d+H(so!qFQ@rL382gaN*OfUvrC6XJ$m_@B(R>s*-(LI-iBmfTCc=d?c`2Y|mx;^o<7GFAQ5i!93YQp(E{*AK4t^gPS^ zhzAlQ@)Rev4d+yN00|*VZOfzkP|WP~Vg@ipN5@=4co1>?2pvbgt@L;%cS~1YT|ek) z^S-wymim-Y9Y(d00la&I{zyb6g>;p}ef9>JKlxY3f%=uy{F zJ=YQ;B62?;E~w?vHd0f_6uX`=KUE~a9o{))@bcqhZy!;jAQ{bR&b1JzI?@P*(-%3; zor)hb5I8qV>yI=VrygiYe&;c>@lLHebxNm>+8h?vi6L$8DI#6D=IA-zrjM3Wco7*I z?{rfXu)v0|-ksaNy@{a8MAoFM#VHfXPT;Pbi71KqY&qh>sD!oE1R(nMJgXTR!w$?) zaa4e7{p%A*J|Sg3jugb!Nx8q!B7*qYR1f1(n`j5oz_$cAjT8H=lv^g`=KyZr&sm}Glw z{8RVq=^-Ov5=GSNuj|FMpb$Bs$jFbc`8}WAYKpz{QKt9LiW(jklye@RBe>#_n|gEr1NR*fVaa|BMzbKbc6XQ43Yd5g-Yoxya0t2_kzjC?a6 z)gm6!COgfX7JJD^~IoU(8e*O9ipZ5IW>mAeB zkQ}?cOQw^5|3so1dEAOK;Uy&3*5$?sX`F#cat0L9S~Z2Q zObByhDn^_uh$pJKO469c{qa|tDp|%-rziL* z5{uRf@yUy0h|L_uaist<{&q2a&nenz5w8fUD_}t{)4!qUIe&fGmzNW}7WDBrMD>LQ zQ_L=81=}@>$lu?8u7t6m^z3$OFaX5E&EmT3QdRG9Ogg@NIT)ql(UjQ!9%Ya?82T() z&6o*?CdNZhRO`xx3!z;pfeX|19OLeG3-e0idkNtLoy{P`${mijmvEp^GzC)3GX1k> zx2N;}ElBGi1r4?kfw(@Z8Qg3HJm%9r5#ai!9&x?gA${FHj1|K8yKVyo5~rAk?3v$D z<<*kvNqCV6A|Y%$>}i^)7TZa`VIHp+@d>P1z@H5?9{PT)gbEHxsAH!!q6^q6-%-he(thQ#E#`;@`u zyu@<=L!|fo^(zX!<@=;XoI-duAIl3$(p3L-#fgs@(TzbHa1YzK-R^GvsV=QS(yPoh zW2)VxlwnicSJohq~2u(d7dW+7YZ>Q=nQNRl#S}zT46LbTY zIkh^=D?dkCwY)pVL90gXBD7XIn=6n9xU+Ov^Ng}GS+!$pHWyQ z=lHNPT|#7gI+s;P8JW^b!LaEw#4ZERyDGJyUU4TVMQ4;T_7m!d^*sx;MbK*WjVl26 zUh7JW0ep!2psxmrEH!bb`2BaN4lfv9Es|@QJ6R?vQKz~ z1w|{DH~{5t(*%hzLuXSWLM&|#+{V#u^d`{Hpm5bw>(8%MTf{ge%FfAW-^9bCO5TJ( zbCS~%TlR}o8;uQv9VJr+3~%~%mVOf9)B7xXGYG$FRQHWP0I|Ed;sA+6g*%?_V*o5%{Byo?fhzVd@WXM;ltFU?iw)gOE!Dx9orQ@}*2g;B?*y zG7R{l6=SPQps*FDK^6s}lT-X`%wc(4*pd|Lo2VOL6yv23hH!xcy`efX_2tW#m`%Hm ziTiL1*8pw}Y_)%(Q5SF0TQzlck5AcI-yi-ARR;F_JbqIZg9GQe7GW=d2ouI#2>!vV zKTkB1Y&CGMJmCWeQiX-YszfirAKD7!D25F145XCKme%51##$;DAsl3eiX6y}Hlk8t z@`3=spL=mt#VNR>W3{z|mm>(!cIRZ!(FN5W4zguq(t6`7WOgjwk0Nrv~MH%Xj!$FEe&;|H37SGSGwSexKg#Zef zcx}qKE8Q*s!B#9G8tl`1ckk}zuHo_Xk|_kS!&cv3{t_R*w6aN^&jI4QdGiVH$%M9WK2k6CVA)&qX!t@1rL`?@ zeQmUQM>k*P#eSN>f3nEH+rn5da~+h3UTM_g1%d#x6Why2hWSC$*?6>kWs$UEntdsDb9;uPP6S13YhPJYOT^d#|`tpZih``6<_A_094O zQ_BnfLW#AH5kG&uzsHI~jg7Z`<(*)oI7~;TC@$iMCtX(J*4YjCoyYqY`^3$*9)iye zGT@$H;ii729fGINLB686mXzhYZ zVujI$N{r(oJnm7<+*9unTM9pi8L|AOF=q&>*TJB%-|Kf=9(g|?GiNZQ zCX}Qvu6X1)sr%%Cwrh8yqrc;$%$|h@lmQMhlh_O13z}+P)zWsrHH`Mb7CS@VgNL&5 z1h7#Xa{KGt9b=F9CD8^H#$6_I@`w>_!h~IKMm;rQf!FcNG}1Sf33)PGEBmq41LAN{ zb&E?&y9^#||F*Jgk)dbS_ZjkFU%zhW@$^LFnGhce;y8lxo`_rl-UP|X2rvaSpzP?= zCBDizRvE!6MnwM~>SoMbU`h~I86qE`?JLZMNvO+LJHzX}XY3;*^bA(?C`oa5M zP62uG$g?MOGQx$gL*L^AAA8)ZcB3@l$`#DdVpHt)iHpk1bCDi3j0_BM5Axa45YI-vnZO(@F8^RJ`RQI7UodsO@nVMxtOyF_ z+-!>F*vG;=PQ#za29=)NAAe64Y4-+m8`j!cKZxkqwr$1j4sQ(x^jnQdn^8ft(4>S9 zw9ou{1^EFh6u3Tr)Q89C{*`+f(gu)YKEiPX4g`n@`ZEXGG_lSjC|}0n0d}#lQa63d z8Dpmg54Pj;&PD7y8d_iuB&QPGMDfX2W@RfuREkxGEjyCBNp_s0Vy!CRSGK^ZCtRF+I8@R;yk-Ze%nJn1GOKFvTqk?@K1GqY}8h)@2$p}BD)RXy+t0tpN% zT7;NfVkU(9#Lb;C0`h@c&(7aMHD@_PPmiFEV<^~j;6M+h$6a`Cz{z=U*5)tQ%Obc= zQXE&B)|W*;aNy({Z(e-A7BMW18wF*j?jr~h1s+lgCAqkwfC(;h=D6qHY06UjZELrj z#j(?6tcM1c!$ZuS|4UgXPV~nM)^YXfN&h zlbSlFdWvJ`VPNs~Bd>6BU(<@?Wy+*sq!qLroo>68BW~{rDo%QH_EF{>evTI`A0(vJ z(8&}G_!Z}6bC{W3tKEb*Tfv)2U4w6LEHA`l_+i3&4e8 zE!Ogjmd&K%F|=!{f@>ddEyP99m9GdS4s7qURUq^lRhLmUB6%$9p}z zbt`iCty;%|pZk$? zH2%klKfKL6I*Mr9oi%4p*{ii?DC;OH`8Uem870-Cgeqkrpp&4qo(-s3* zsGWR&CKi|9zLsLrFXX@hS_z{D77~+6`f)cjse4MaFc?q0yot7aac&QAFs94xaHy3<(K**I8h@4E59ROJ_F>R<3d4%PC~2|6!hLvN#sG zrH;TooBqBA6^2UV#C%)`L<@Gpq$yCZHWGB>r z&1B`plfchcSA10$BdU|z8EQ@iw>*8<*si{*jLAXZ4R1|86+3m?g+TilTm6Nv`imu( zLxzdO2UUM-!PH4;5b{g^WEkN^JRZ) zfCJB^JC4#jy(dn#Ee(46U*#Iw;JxYFR%>8wB_Dt!PZz?gwYj#ldyhQwTKaP-pgv&Zd9*4H%%+tCv-yk~)72#nZ{;FcPfY z5U)dL9i4_A{de{-HjX#%=E}HmCsBkcllYt#@3$o){NQVJg#$|WiqpKJU|DpMAI-$i8^~&j3tG~)CB&KLZyO&UX&vF3E=;GL zi`fi>jPCo>__*xMCBW zEVvY_ugW7lKke}FEhI+w)Gaqxk>PXcdypjT1i*g_y;=ZR1 zf!lBTe*dE}C%TJ}frlCDA8rTt5M1N~_ABs7s!dHTt=*`->=!<9U_?V4ar1R)#s+mr zpn(G7Gk$3XuSuiFy3wB}4jD8^Y-vb=B@Z$s4j8|59zV-$Ms01XH_WhnA*urxhr;=f zj##x5cMP10yc2$w@S)tWio162wqBr}fVEi$QcG3Kp{&nR-rs8YlWS@zQJPAR35q~$w*hX6Z5WT@M90~OfNx`+VEr(?JqB2$`9 zY*t6o|pUl{y2|w&V5VY@8|P=U*mPXUa#wO;YA1V{5U?)VrptO-g($ZswVYr z+h^{P$x$CO5CqK1JMyrJk3NjefQ8MP6yI;VCpIC$0ZJZYk0x;N}k zzgsIme>DbUE8I29EA?qw^FN5EasE4ymg?=5>V~f;8tk$O(LM~^p_SF1u!Cl-D}-Mz zvHC~HSa3TruhlS_TVV)sv*YTbqyhHZ`koiRyl*4=VH+6&&?YmQj`6|c{(-fDA&r{B}h9W_|xqk{WP zP@zoCV3Pnqvcu)o0pydGYHHe(9tJv-4Vu7#?sX);{nZ;xU?LCBS|HwL{PXYM59OT; zWt2fzpx#+KK)1M{`$3=HW^JLoW>^}4j8W8IS~@WKuM}}V*3G6+4Em+V)(7;u^!t4` z)(XI^yycbns{cM=FNNZRb@UH+HSqDamT2W&j?T#Rua|6HVBpOKQ_<{UET8BoHfhZ5 zLLOBZL+bdQxU_!Az#-2PmL`ElI!p;r4Gr+LXrQ9;AT5>+Tm<$XZqsUyG4;cP|L66R z2?w1j=*wCn1*Le?%`7|408bJvM8JzXgCMbC-4QY3nMLB*+cJ!gTFzkwRJkn53^8hI zP!$Z1y2i7-m3C8S;<{{WxbOAefvY)!PRZI+N^4#qOGfZe;WzP zIU%107qfAZ>(+hkx8VJzB$uNL*}lkQ7j8JEx$^vmMq`esk`-nBh{@8Ct_k_O&nZ2i zO#kliB(7r@E(aggy2k@wD@Kpnefy5PuAy>h4Cp9}n2bJb@Kj?UrGS>t*0eAaj|T$x z!LbP-NF2buB%S{~o*TLYaTeqYvX!+i#p|Tap<uL~A?%2(?DOt1lty_sJvO4fD zA;)YQwUvTw%P8u){{_?-^}i%IbzH0--)Jm5Dj+Y5SCm`gb$;0!JA@tT^WPDS{^vK) z+1PjsLqD)a*?I-Ti=Cm7>POeK)X)fkz}9c^8(n7QFyhckFyK8@`v#pp*t&m7>k69X zO4Qu;rA3*-RHi$~1-eD@ev{Y%3#}0tqCV1EiGKY3KEK1yvV`u{t02GHy=S+-vz&Eo z?DTn4*QWmG>rwF3yX(y0({vSL1%NjslnzZsD8Dxkdi8n|$|Me&UFU#@5Lggvw;_?s zqQ``U1f^@Xar!&T-F})b1HVK4B6S#O+e*kybZ%Hrjd9oflV(=Qfb+Z8&++5O(_8VS z`bnk2V2JA!V&ZPe$-fmnDC4-Hj)bc&EkAR(7cr_5BOREo;8ymd{6~zGV~>R|BH(5m z9PhXH5;J!1+9k3vur6ezfa3ct{Zs}MT_t-%9tPZD=*S+i&O8f1a2~BSFfm80=vtct zG`PIn3%$yA-n?L+Nx75`bGb&749#hrHh z_;Ra-LeEU|yL~?T_wBs&-ZSgYOZA?34qoD-{?IX6zsSGw%dRhvja^)tS+Du$7q>Tu zk6onTGhu>?hM|AhW`{~g0|T^bsO4AB{?+|+(P6Pzy~%yO^4 zC%$C#EQ?ubh@1GMd+*h@8F1I{&80k7mW79M1_j}f_@QY5rYj3`Ay~8TAV8fBh{*T) zt&s>-H;y0MJ7D=XW+MwRtOiX8q|L;+f6Vc5A$+9K@a#CB8^XeJhx>g&8dc$bUYT`6 zi}99}ogv^_1UO-`!wQ=+f4*q5#9ah`8p&>>6wBT*q=3Z39CiM}*YE$Guhi(PZd`KM z@WlAOPFtE|)|vt_v^EBWwd+x6|tip|J^aIzW2Khf37Is&ImTq0>BH_eY*`<3CbY)*?V%Yfi|h{r9RtT1i?z=WpFbKjsdY|{=h`_GPeN)&&Y z9QNe>lK#L*XBm5mbRA&rErE!E{~msVHETop<}{1#as6`X)RMSRn?zg?-zJ1Z`Hn{f z;emj-Ju*_0-{KhAJFV`+)v0{q|2jksin4vRwY)g6q^mqup|Ejh5WRLpGC1dPQuo(m z1`W}BiG7lg;5Wa$zqlB+`t)h0qq`a%a&gu+3mN4<;AO)nZ5}*5zQ8TeM;*WubaTt_KzWQ%1OX6gh?kkk)>umX-yW z+VDDK8M}c1BBeB`XJ&P@=(Eah0TbDik{=({jJG2D^0eS->~>zDg< z9BruayLhdT1&#V;Ec+C7sTMb)`AiSDK)xXI%IYtn#1uIULDiz%h`FkyRGDi5^$TJL z@GDO5@HHxPlsay}pI(ijKgx%C@)odJn979$XH!I8MV;=*c$0%3)P7&NP;g;cj%q*X z-q8b{W1|1LG9)oRe(Y_n#GP(~`z5ws8kO{7{8~GE@5!y-?W(@4z#7Gm*P3jbgs)^b z*Kum9{`N;>eG8Ju{P=d2@@id!CLSMjF90)DURE1V{T&O4G=kYRAsbU+;~{2Q<|9Tn z-5S@tHC$ApHIfV_KQGw@!Egb#R}1d5dDYJYPKJi#lXV%r{$;O_xK8^0S+@GmCkyOR z8tXb{en418OBc7+x^akKSOw*m(0GW{*LdXA*lJv&ScuU!&$3;yItO;lCVwk8YbMMq zsbwheWxybWdYYNI{Li9l|Hx8{*Be_bF8#f9L@Sl|Mj9g?%{tN~-1I=}&s)YCmOiT2 z*(}8N<&m+se{9w)SXHV*(!6Tq*zwq-W5cLPaVnjAX7}0}QZY0&*K;wOjjy3G0hJJ% z1(O8b&NkO~v}>NT3hN8(W~KqP09%SI80IUkd6!ub`R=w1nTQ0kYlwg@E7FCHC_@rl zZy764VQqy++Wxdv(8rK~L94z6Avie2;~{!!(Jt^wZhN~{(>S*2Ggk|^?ln-QNbX=j zAq^c&?E=T;-lN+KSrWPsoz0K(r`LX=vpC^4a!;&oyyDWO{bN(U?!tSgB>>Z51}ZjC zv31(=fReJQcZP^GK{@4uqjvQW1~`rF@ZmR%{ienOUx+}D+brx~I({DFLC8|VdHE~5 zbkn|iV_!zIr8kHi<4;bh*P;KCm(XQ|rVj*-f-kJ}{B!o*@8P1TwQoOyVrv(xp4qk~ z&$nRyvhS45m9MV`pPpS18kpkY{JC=GGQ}35p}U%_Sg-rC+_v5dg+u;5?d&EtTxD&& zaLFe(!l6}N;l%=Q9J}$S4I~NU%2ske+P!(hkT{`q(|`*fNPND*0`g&54CnQ|J~_?hH1UC5#d+W4t(3%Zp5$RMo-h`k0iMQ^p1aD6Q6W=@1%1@ zn?DYXb9s1hXd{^cz*khV_C3kl5Pukm88~~j|7TS=kU}n~=-hu6pXE;Tq_IukOY4I3 zjTPg~T`Z4);SpYcc;l(h|8DN1 zsiIWV{Um+^6m`iX)Fd``Be6326apF$7aYgxTN<{W!Ek?G2kD z+h-G*BR}^4ATeb`wtD#|PUN@Gh5M8Ng}M#g+pT|x52QLHgBGO?bWq!Wrh3bZrhdq z5r1$h+t7f`#)+Is>3qG;sztRuC-BD3Ok|*Ik znAMAaAIYh38`)~E;(NbtzV^fHuj;DZm;lqHGg_8zt@mdD8;>b%mh##7JQ zRe|tb<+S`Y7W(}yBxi*qy?Zqy7|i|ZQ~Yxnrfd93d^l^@0MF(vuW!F$(PP_fJG&~| zMQu-KWp>=UA+CRiKRA>k%!0t6^{;w!TZgkZEBQsYcK0+8D&>@*w5Mkmb&>HdA}#8M zY4dx0*y3x9^#)PpE*v|75h>z#9%!uH)I`}m#iJPk;w6-zE6<)yuzyt_b`~tzq4b9! z24ca~@L~o4mG1!Gz61Ex^-Wz5Um3gUp`oGw{7j>*4VT(B_Vf8>8uhu1F1_T=?$KSN zbZ+-FV`akQNb2f#`GMvki&odwEPulcluGXx-&U`4bmhRKPNvhYHij*qNdZ@YN?x8( zU%M)xk@08dwh;#qXu&Qj(V2|AXJNTZl_5I}8m4cjTJ{~V#Bh6$$t7oPhI$$GIW$J8 zs;5zKlw-8{l8Pxw6Vsg`Ui-$`9Wa1hw*~L z;2b+T>Ros|6I5Qp8&X%*wtiz!%$F9n0F$E}ett=TNRHvC9H1irD0vsDKx5w~xxKB; zJcI7=n+38?Ntw^y*8x$QXb(CoUww>XH7jDS#$s zm{n1*FgDwwM4^&ySH;s6aUN%>?95hfw<+Wr{F^W^$nFz3+rvZF-0=WgfXSL4@!K$_B)D*Eck|2ydg94wkY< zSF4R|Gb_0rxgDyGt!aSj?#O_R8#c(O7cRt}B!?+@Ep@cr|BF+gwr^A;|51l?>-PW4 zBuYf*lq<7M{~*IM`tk9_?K4-cH{J9c<$eG&qdh= zRd#M4u}=Y8i*0U+_m&0rM_!#Y`QjYalsKBFo{MvIFXfgkjJ!YL(eW7y^@4-v7M)s8 z1Jky|(sE^`(u4`S6`f|7Fpd%o^ieYl#j={3(%;5$*Qw#7zqz#BBGDO{yOb| zn^n345nRGq!IYtB$upbx<38p+n$H{{r|P4{*8EWyO#tQCwDYp0L})U=#bOI^VqO>i zu{aCCdYekm4TXaDzF=a@%1Q2DN5h}WdiLyDXOB_}KZ{^9vn_W5IyS!HKWcEjOr?{J z+ScXo%%qA;x?GM;xYGoSf4e8#%xGr*Dytm^o zpPF_Z&s&zRIWhU>xQOIV{S1De@2Y6aN8DN{e( z*|9sPciU!SUjYe(^%wpG!4#tJY0o&^0;c`C z^o&92kb_P;x(siA+AXoRmMrz*gBuOtBvzF5mFuC-zkrmG;hcpw*Ft>6X|!&iw1n^+l&}{U;$hF(n!6_EH#x%z8o*ItV9j_zHtAg-{1 zR#fP?HD9vpT-;B1Wjl6wptfS^>9-@Nv^`nBZ}T?iswbEv?(E8F6V}wi4~rQ4;f+x;9si#T0I{#~&0QHlgk=~k?Jp|~{3FLM zVxXcAgpQJ{16A9=(LT38KV$#&sinvPMjFZI%3V4DlNF7wV}g+15IZq*2s3F1KtG{K zA=&eOM_gT7WKz4!rdV3tvuDp+L?v`CXEa?x8s_N;)Ms@$j?;hW&KAMiJ zz+3W>w>xTE|JXUTpn#0Q%--tUsBPcg?<&vBD4N;63Y}E}%eQEJz|P7t!gQ|$S9;@J7YKx!u`DNX6n+5W0q^zy4k2M42Dt9`TkJd0nsZ%wXt z>9Hq0I-B3EQOlekI^pYp*yleN+be!noNKpoyNUnk!OdPSczL7gw&i*Ub_FXVVd{O> z7wJc9rc?B1^2%TgOvNVXy{PUgPqqAf*vTi@dhFoWQ8O8yDs+t05Gef&t9a z{Pz7U50e_>(aTq_gxge@KB8xeWy4$Dp2&V@VnKFfsH)ENunO=P=aW;0tZ#OeA&cx) z;r2^XVGOT$VaYYi?_J+h%3&B|0OCfpcfl}ZHRWZCRd>JPw-SJjEiZX@Yw^LGmgwT# zKkUq)rn_2%E8SUJJ@|c$@q%{g7gUclGD&@Yc=7;ug>Sm)3FnNQ_La2${y4kCtij$t zD||EWg#W2a$om>Uv9GPtYP-dCCPqJ3)?exJc|z2iHQg1?hwEOP7WH=iy?|Y=4)(8( z)k_LKmRn}Z5N3gINOv*$LCA!}?AVfL*J*=gFamP|bKT7lpA~L(4?oJXl@>DBKY|z& zP7K$P%FRz^FA9D+)j7+5ot)ZtzyLOR+aSS-)Dg|@^*_1Gyb90E+f(}!#BZ9X=SE^M z9Hf#+_s|R@?(g2x+k-IU;7EAhJmV5p!3_D`ZLB$`i;xO1XAuEYhG99=U827}{k5Xl z3bxD79H*E#hp_vrW!VUeo>#mzzkBERxMTb+AKn_PTLm?B$ijc3Y0~C7ZYP%IG8I;Q z$+i0Z$L0LR)L9Q-g>gAR7)yxzlO|Z!IsgQ(49^}hzR4fW1KPKypXn6(Tzx<4l8%LA zTAkfrnR4xXM~4%WRqcNL>JeAiyVUdPvC3nvwUIw`)*0MNe0wZ>PiTo*g@X2O)#V2J zwyiP$vdd==C-xxklrqSHrzm9*nJ4yHKe7BZso`YLukkd==L&>oqz(MX-DfrYugR-M zxB;mCE?Qi2^UH~!hxJz|ka-`~6=FcifbYL~%38$+lO3L>90RehBwc7fv!9ngYF;V3 zbxlbrzyGc~kM^x|b*S1C`Ze#?AndgHs^t?U-Z+%LduKQuliW7Z0qxFx+coTks)vVT z`Ao;A`71)thqX{ls&{{M?dmU`?x$31l-8=vZp;2XC`t#rFHQmmdm@E%*oVWPg&AJm7web^dd=5D6 z=;qk*mq)9t%-DI&?jP@Q3euFqUNFSex*8^ZCjYz{Z9AagF^AAS%HWOPn=N*ZN4uXY z>o@rsahx@#O?@0^-v}%(NZAN}Ic3hAO#ttN8lma`9$S&Nbl93bc^x&W*?qlMdMw+0 z>3en6%OUZPZUv|o^ip2FaN$eNq)mt8qxOX!)`I%kiS|+qgo{DS_!R31L2dIc;dap} zajXNydCz&B7}nVxx4BX};>of(ZTkk7jDHv%_PqDS+Z)4uvhT0fHR=7M#jWn|4g9|u zeos0Yah*5!$2P@e$MgP|)Xxt0j{Kt(dj7tNc6!1pn*j&Mjy^WApL~0Tl|DY~go>X1 zVFRqpPX07c3sFupTMK3pFNBHTX6N9FRLywiFktF?B9~ak@dC(~0^mRpuDqV~Ww-V} zkxq9aO^O^Qea)RP=Vv7#^-Emu?_YNK zQA~!J=x8hLSnCk}?-3_)6@ufjFR3Rq8_1i<2$qOQj+8mgR=NND^lan4efutbx9myI zbvkCNrnun6H2a1N&KumClrwt4(Wmu46sA45n_THN=$6CQxU@}!$9G=3D#d$npliq6 zW__F7`IXtENm;S(LOQQ6&Dv#KB&aqtTpIsPDp_G-zI?gk;Ijqzg^JH1fkwn#sde}- z<}wv_T=kIo=`N7cl!e7saN=hzoXLc^5mpVu&doc-;2zTmm8~{m+KZ>ho|)eX0Z;Ub z!y~im)pTuJ{!G0PZmaATqBTUfw)vs&4`wf2=Nq5-eM^E~k zQMtn@*^TNMm>tWe>Fk?`jEuaz)VT;c4|C>wdA+mh!Rg&ws;{p*qhi?&h1a(GWA7*# zhI$umYGJfIG_wDmhG#>br@m0B(5X3hWckY5<+JbVue|%UkFH~_hvK*$&;PyHNj3bk znfviEE(a&3H{GLP+xSZ@mIZrZ;uFv$e?tO&{r>&2@|chB-`fkc#RuYFMK9Td8b|Q} z<$vFP{rh+=yT!|v#j|t$cU=UV zi0U#xMlbYAFFYTG`7KDS6zdJg$1kYF%(4OyX^DHnq{dSRnjhDDj$J-8Ng7b{?oMm z;W)+X@y{oBTfh85dA_ypitJB`Gym+b`;&H~?(rFye8=>eXL1KCn5MXz{2JRT%Rg>j zv!CC;*S!0D3Xa*T6Lm}(amPiNUfw9<$DhI$0FW=2KlSj?D=sS-YKjO&^qv?guH@6?b=Zrfi&D02-q}Nt z8(+SNRkobd+afk7Io$fdMf+jl`#M~#sn+dmrByH0Z6q}_^d1YrHvP2}7BBv~#Lth7 z6|kDEAvDC*My#ctdRKUH00hm0)VF_O3Eg4^Sr7rO5{{U|w?U_<7f~IFRbtbaRs$2) z4&Cr_%(-4$M~paYGAu&xoqv+O-VI~pU47@dof~DW8&GvxTr?7k8+KkA3`00&Nzqtz z&ukZbi(gzwrst00D;vL7eVJZeqLZqslM=R&p$;^(%K&NP@SzOPG>CaPOLm_zpUXEX zo(lXhVprdpDm#1h$a6~Q*KL@`^H#%OE{+Q=8Zn}3(bHbS8X2ON6%>1BL;2IxZbKjM zXni93e9x}RX;iIn4~0ohN$yaQTYx{m0Cyn2*HOGG>IB?;uG>EJyn&cY1M@FLu4okK zI0XiiXjJ+7?dEQd#n~my^G;{?UzK@dj*c_A<`F($;otl@wWm`{*Q9t{?DaKKHERXZ zjR01**?X^0aJO}@S>$r zrD;lG3}O`AB9`y}Uz+i(uKL0(_v(S4k0F%b%8mar{==iwjeZ+ySUZ@tZP1~jr;~G> zRoVOOb#V@Q^)$1BfYwkoirhc0_rNd+4g)87+N=r7OJ4~$ZP&n+-W%?z?=QU{*1yA` z>WcaT=A-(;Dsap?odDD0y6vYx$APT53re!-np$6@tv@x`FvNm^>y(Ouleg+R#&p~I zQAe+WiC^E=?uLz*mwiJ1+=y4bW~KT6?%K*P-2TF|(^DTkPfyOHDAWSky?t+gM1h-Sp0s{0 zaVkXq>xe37*gh9U&Yl&h+Au2MFvHpocwmZ2-P!Z!zkM+x;HcUL*q`KYZ z0vj>7et4-9Yoji8=oczkJ7wY2u7(=EPymgG?K?VcZ6|-7uB|p7Pb@2bdm}4)O+5S) z_4eAyyI_chLa`76?&X_P`2JU@*K*IeGppwX1-K=~6VQ2n;1_ zfM6+ZOF&Hk!h%IoMY~l$acsBC>Q-8l)tkS>xt0TZbKd|9Y+J<4lse5;IZh3h1ubPXbLT7n{Krx zOZs<^!6)T7jPlJQ2J!p7L|6&9NrTp6RlFMarRJ@Dw2}Wk_%kKW+rN1fBewL_(BN7Xcpm^WC_gA zwMqWB_&FqBR8&rkiCGbMnHRnroSBwo{GS8xw=fS$(KMdIw7w)^Lgb}6y_Ay(tJ`oX4}LRuGne7@&(kJ4$W1NOiv_A`c(}$z}qg zUn#VQrdp={2y#U$e>--C(5uEVQj_KL;6tJ&gjw92WW_HeJCEAgU6ol4KPPU64A#Z) zn4PFWsNkskBR@5N05<*g#S!PZg~@@nAm8b8$6JmzTsS77EWxQ_K*b2l8|Btb_9|U! z^gM4)cD=PLzq&9!;HTdhPqZc9#Xd_{Ef_a;x~BI5EVSBNI!g-EQr z@$_kwc+tZolg$nU*~od>;zS|VE|{ALtBXB>cM0>~CfzUENSjeuSSW)qU~DIL0?X6` zav%@zRMX2EI(y8jo-W%zaHV)9a9PE^&Dia95XOWd*gi0R#-H6c*?M^az>U~iiNud` zR#pVjcQVCgJJFs`o`>-G2mWk?)nCU!tHpk%>g!Flos-kowXJY5oE87jsfA$!!;+L{ zgZnPjc!GP{&^VoeM#`Rz8XbFlbbWNBCJ^b)&dS>Ubqktd$7)U^eErrfyLF-W(9jx= zIkMyOXLrb;Z3Zke#;ya>U?3HbIi@zUlZN{slPPoGeP6d2hJj_uPDoypprt}wXGDY_ z=IYRS?9|yr&Ha$PAv#2|omfV|@_ma$c}I&4usYjOQh7rxca;>zAPvN!5yE9XW)?Uf zQPX?LhHC!M7X2zZptrSo-DUSn?Br?pRNR9m<1OHaGaQRwJDyHD^)(}nH(9&=h~ov7c} zOl@?;rpnUnhi5!Sa;tl^^3uLKAs<6rs`_LYg<}7sY}-)R7bc%YRgD-NP*#7#@_`8* zGtJ=y?uT=@C%Aduj}3`wThTw5_prbj^`8=d zRCnIA#qjj~fgBk2&qOv!Htsjn;EqfU85GcGh?yJ}C#qZpw7%2$MQFHmZjZd)4VJUb z9mHlhfjo${Ye{?EkBzMyDVed)WG*$h@@%}+-t1P}xUmt{N*u<0Wl>ixeSXk@&KDgx zQ`SR8C%{t>cL-OEHt8A@TS8PuuS0M4bhYLm#GzUeS7Nf%DWvnX8^ie|C5=bP>OP2j z)V0C}QIN`+5O=F|QK`zaBzFUI{e}uYsclJ1>cPPpjW#JZSgSbL(md_^#6|-ZJ1Q>s zZ)cP?C1TT*xX?d#cC(iBHy$~1QTK^jk6q&4{d?LF%;GS{72e6Mt$a_OPAtalAk-fY zn+@QhbV5ADL-}*WnU|TVaOSAW_91*`g^(d}hr@*$!5QiiS!i8Y*ywZ7hdFbk*q5AS z5mpi>Wr*Ly{n7{~Gq9DzySa+O&FMda;m)$RqsY3vxg&s?b5h%aFJWplRAgoFfZcH| zyYozkcd!AL?_R#QxHv07)DW==qhd*S|7nIA>bfs79(S^{&>an({HeY}4IDc%lDS1q z!`T7~g{Qd9>R@Je_9mErH6P~mY6~UF9>%7yC$TgGo`(o|jhKq1kYn>Cx2?F)^Y=-d z;9X&mdbGtk1su@>;gayDJRz?8Vp}Rnj>+-GpCz3Of~jUmab&~cYl=X^uQ08n#jFPe z@*024_US&64{mI?8`1NR*W}cwfhr9e^ln=@EA`3R+MQiWJ@19zIOc!4euKHJ#rvMW zzy24aj2ns%7n~hts1a~9|9Sn626`d8JhqPAxVPV;u1*R!tO94fDhXhl0vGV!j!j!s zb$pz=MhIs~vIYo*gV&1wt13&BSx}L!eq%(ix}I!yCX5VC$Y)&A0VdmZzKOVONkv6< zt02htfx-1yzl2y_RtmvH^_v7V*wCLfM)#ACUQ8F*eNp`tzrPs?GkA6NE|5;R&%(LI z076z5{??lhBfu_1XBeX9I9y{Xz!WKs*3477HJ1+~^CLMPG{6tz95gAAuZSxO(*jf0 z0JYd5;lCp#vR~TwF*JTsAwa=?zd-P)HPv4lA@>Us;0yp z2op_~%)ppwiEZS-|7h3KUPHV7vE@lsRg1z#JzKUs-re}*nbTLZpSJDRuHDCvpNw|( zG-&m;L>rQg1Xa-b5Q_zq%SQDGxaq_>c7)g3=p240wNxkS0 zHnK_i!q>$vC*CtUymrV zYJk&X&i(fDr%u&w5Opu7qk;FAp^ZZ#V93b|Y}vU(F|F4*G+@mR-ia){p+GZoJN<5r z>xle0HR=D34Olq|=wB8=31N&*FqWe$dP|wWfB4XCXkLJ+dZk^BHFZliDY)=_yZpmg z&ceHaYcm$6;)M#`c8C9JF@O6 z0k}Oh@q}zLc9W*bJ*+!B|DzL#e&Pd+A4Kx^sSC1MUz5o~l=yO(=3V zXzW+D3Q9XR`fzg?$DxlZ91Xjr1>DC4lqXZn$!oB0cZ6?TD7Bv0hFz-pF+m)uV6Kbr z&wuYLRd61PD+XI?>t759g_D({oJA`d_I`f9dnw4+o$;-CLOj3hao4Y$hBwfVp)=bK z)>U4Uz=2=L56iHz=3Jr9;_>X|LC{Kyv4Pw?8*^RSAM?o)e^R!Aqf0%ya8gduRkWFK ztktx%8fQX=!4^v!(ENM15lR2g1t83f@ff3N)%c(*R~E@dZ10Jo6=|zr^wIUkbBz9a zEEZ%E^piQ%BCBKh8HN&VyEgvHL63xMbmUL|!a7qe&95kvl$elkvE7+6ftn;yfWNw{EM-z>xI} z09Nc=A&YQC-oxvO8JiI2zx57f%n;@l;wAzE!xMDUWt-Le>FRWaQ--dLCx>}uz=^@6)FPD2O zvI~-0j`E(fqz4KYcH=U|4Ci<*vV{N;$50U!Bb4H}jynW#?jlI|{vhpH}Y8OSD z@V^_H9}A^}Mr(%Bzs6Vtiy z4n}jr$=o+!BbK_{ndy*X2m1UN@i(?zKg7CI>1dcYdLI25N8VtNAjS*)0D{6DM0+PZ z{P83b2gK9cnrK_MX4T48@&TtVVv4W7aMK{wczr9MNi)oh$82_vuF(@?>6&n?6I?P z{8BX~*Q2qZ@X`-~pBdLpzIgFs0i#qFJ5`zf+)A|=4DzM%0;&Ly`2;T*`H^EVF9-z6+2MnK7VO7qHn z7aBzfiv?U{F$!r3+_dC@tnZO7l^ckU&miv|Job=d&}hU&WOq(cEHRk zCV{@N3Zf&$*HA^t)En4b*3!ZvZEiu^+OBkfv?O9aM$sz(83|I%?a|p~l40V`pHsEc z3gK1fa($@>){iF3z9v)uXnGP$E zJn)<~U*QK3>)#Y~l3wEoo4ez@kO>k)#px-~&+@K@j?@=8&M!-succ zvkF~!1+AnE^v1IvWy#||nhV}1;kRYPJ2pM$cWLq4 z=5#_+$aWe%CXQ2az90>NC}U60dnR@|I4hfImZ02xZi+4KBA^R=^$jntXf*qL!bbA^ zax9!or$8gk0xhM#2ni3@t+#SBjkA!~mvk=%G0mbn;(NO!q=nO?gO9d|K*Xq$fsRi{NH1l0=?`xR;i>VJJZ5fglPqvhyeWg zBe?qJEW>4{5vzJF<2?^Y7?NBsS*wLhsq!ZI%X@Iwe$%CY|1G;L#cIF;?dy%ayt$rSLt<^xMi+`*Z~SL$OFeS$o~p}dB_$w;kHA_ z42gB^+7kj)FOmxxI+MAJuGRw8jva;k3b@^8Q_Vg`YxG3SLniTiFOn;w=?@=pC4hh# z;zvPUx6A6e5l$~v`*fbMh?gmwd6GMLs$X!S7TAR{QC$3J2;&K|GG0Lpqa3bk6`R8U zmn}K>drnzsvMDIl(lBR7kJMN1}KD6cbBeU_4ndNwb8nyPKk3fNGiS%@Avl6DQJi ziy;&OljfBz;lvfQ4-a{V#@;GJzV8eQHe&EL4yIJ?%A0oRPMmR^dzQnzefNRx7hBAa z3nkts=3vNJ{W0`>LGP`n1#!oDs0DTG`obH8j(4Q|gs)$} zhJ6jKDnadiwAgRRdTFS2u3q0eg9mvv9hm!IwWp|0-r_WJFWhFG`J zQCSz`zp3-i_+9Iw`eg?<8F}p=yG16ZKeLBz(-?g_P-C~xflk?LyR{l}Ewe$7F-=F# zi2nNLkjY!k*?Nk}lYc(jI$_zp;)#z}?%%vP>GAi;V*Df><@Et$f&XBnDU&BHEyX$( z6v3LVpYZKLwM12L>c}MH+h58vYD@AtzT%V4>cXhEK6^gR1N~;QFcrHlZY^R1oqzt> zN9`sHzQEvFkg&hKzg4(8uBhVfOWSX^`L#R#!%*EP#r|9bOjkvYwz-UpJ#r^HV*Y89jLi&`LvmU08~+2r(|xyLX?wYc}yqYjR} zy)HXYkChOdwFhAQ1=v zw=Z?a?}wQ4W~C1nWEXsYx%6CCaUxN(pl z?%m^$-g`x(|KDn#d#sptD*0j3t=nc=QMQ*|r8B5qd?A%jU)@)eGVcvMAlZ`A%j?ky zE48>+)uS-D*Ws~JE{ZN5-$r#?wH)lH*++!0T1PlKl0NvIg7eD0x}7~H*%R|l{aW|O z;>3W5E87(q^a5GHy$0?kmZtqi?Zk+uws>VZ;jl>Txk8wrV z(D1rRFdOO8h)=?N;G<4nTkh~fImAA0)u$R9i4;zJyiJFvGQ($$w5ST8q3&%?L`F|< z5U6@>Gqywwb*1ksFG)#0qFIdX`!Mg|WHttxlI+=!upLtf9@}!82^02E$BU{O(fEBq zQ#7=Poi!W>ZiZ9pMrw`QnnTnG8WVYmnjo3v=KLbN3lTW6n z;hcPHM{U+}WPiO6K!80N;M_DWKK|I6-(?0%X|eNxQ$~BSVuZDq`GixKL6BsN&d4&W zHIt%dGG;k(F@w-s*T3GqXY6DwsmXw)HbX= zwEk3F>|F8#J>r;VM<~X)<}sX;uHWf0^M3#S{T^MWedm~~`-^TFII0zY{+#sr$*Dt1 z&UfN)1RZK+l?NqyOuM#ikFn<`AL6nT9(rSIC+lAAYgIz2x2ufq$vVwXKbsQvRuVJ0qr!lR;&ok>e`UHgX1&Nl1CR;`MroOHnw-StY&i`+OFFK@K^=SVM%9yvtAUZw|evm zuSu2an+%;}8=L4bT@_LR&kPV#Q{7e>o8Z9!rLxC*1uX~FyN#oxRG#gbEWydcLOagm zS6iG&Lkv2&xrrnXV#^|0DVOh0sbkn=*~4X66bC6q&pTz^9>MO_RnVYve&22$0FpPT^Dcn2I;s zUI-iExb}8-O<5SJRG>M2v;F49@M$sao5Gv`9HyV?u#hn%N$I|9Y#PFAGBR<$6#n=3 zu4&V*UC)It1(P1);SsOoqJp-3C%I7EfvI=Ivky)*sF_BiMvX?M-K7JyA7>S$KAp({ zuI#&DE2rnRDjs%FubQVOp7HWlVVzBEZtEP237R~6#)?ewYQe#i%p}fF0@7k+n2AN{SaSQU{9~;MgWY?d_(g>XDV8Q}zXL+X z?)E`3PsCt>12l~jDvhFg&<>r6B_|*pcoVhQqR@3}V?4XDxVNOpFu>f0e6pcMr+bvh zzgTw;5GZ^paRkAPS9wHZ=eeVf;MzS}d;n*U>)spFN(Psj?=4Yq^K;u6Dh}PyG-SP# z#sF^Jbjlr2%DuQR*#7;pYp}G~&Lo+aw$t=#H&_+7iTvsKm0QezK84~1qk{+R7b6%8 zR})bUK96=;>Q)-F<@++ngb1x(=SFcd1jT~N(oSOZ&xak4$jHsjTg7nGVG^utv?FNf z^o(y}lr^jtvki4RDU60(vQ+DifVMNd7RL`V!e13F#Y-8(1BxoyoFvE$>MC1KmClV@ zw}J%{DKB@A+4zVr$$0B1UH8FHLur>OtEORKD=k!WOXs<>y}V=t$LZ6jXT306bOXcFI@Qd4kw5=s{h(H9w9iltq`Gm_rZZs6a5^UZ_uB>SgTgxQl5cu|c45?i z1_bgrDaIF<<>-mI2mCyHvbJf@o>#CT!F7EnJVj2{mwR`43`ix3IIcSG?kM^ttY&4s z+P7c;6(LzgdPKg9UZXvR&ir>EzM&*MeQoU_e}Daj-2D7&|1q5);PqG=5hQBkyVOMQ*(AAu`01>)fri?N05tQag2 zxtw<)J|VJUe#Q*j&M}$zzB6aI#UGTy2b(UA8jO{70-<$t^?=*U{6fZ_Jl>EKMEEw? z=?Y@Uh0%MTdhO~}4SR$F6r!GC3b!r(MnnkB5#M!~8p zdErAjzSPESoymBTu^)5pWadu;2M%O3?`QDrPImSy6scP6+T~KX_d8-8{O0xRq4fRr z&SI}5VN$#Ra6widWoI&@=;5zPLmq^romj!{&uiocj@K79=Q(tad4Q<>0c2j;B!EjN z)sF`axwzH;p(!%OyLnmHzSr{QKV2^s zVd*U_2>>yY^Ej)nw*u#y%~y^zE53GXd(Gyxif>$$n3o9jAv#cfY^ zn8}(T!ik=$1Y;;dvI~yxJ3xCA?c+b9Jm;3fDibaO9Zu!T%k^<{+0H5kl)T!K5wWaS zY-v?p7uK?ScOho;tfAvUUZu6K-PfuZl=3fE3hXy9=YwrfObj2HN?i$4r}TFwv5hrX z9wH?7U)q(7B7;*!wQHN8(Qs+n0(m%w~qxafxt+SS&Rz=6vk-Ms%1<#(^+Q5yLbqi&mxAP|HCJO zPuHsI3@O)k)vEq5V8hlAZ=8t{GM$@jIR1;t{>}3nEk-zOChn!c_QLr^SJhc()aiD; zW@Sn=Mg%K{kA{XpuxDl9D@P9L0m5=|ej!ua&jR9LFmvH&p^x-wuljEP$j4@>Oy>3L z@q^P^f^U)exNg^|SDLrit`!-U&}I2hU9gP)n-9_+vj_rK8kU&OS|?BeJM1SQTO%)608?cyh|b$nC-O6CiVN&5fV~i>&Z|NF*Rf0>WV8ZNJZ@_ z&3O`6!C@Faol(-aRz~|K&R>M)h75YI%_*Wy+{&?}^Js8pE3!>*M%}oAZ?-zPqrOd0 z20808*B*Y|B5W}J2JOQAq({X2GGES8jk94OE)R(8AzR1@kF&=%5tRdsT86Ap|E|Ci zSoy0mE`Z-}iaumodJ&PP$AhGI2SJ9T0Y+uXiET^Ru zF<3Hspd&6h@7}+E_32Z0!gd|@m&qJ<8N6Yiq${_1g%9U<2;C2f&R#1fPf|auLwNv* z;<+UEaT(w@)tkaAjZ!X@GYb^VcQeCte^9H`r{r1~8Z_0ig zK9V<~T0!vm;hp>fxX}Ik*VSnl`W*kogAjrWz56ULuUx=x3Zh{w0b{9M=Ya#;WAZI9 z1!Fw-HtYRhF0hu(0iqJ%#|*fr6#w*#W~Qb$)0!e&8qY#UkeOH>pSM%vFB>jZS32O& zE;D^RveknOd$F+V&R;Brc{^{sdNbF+Y-*O$U8MyOrA2qa{FRb>9WXy9`wnb6Zq1I} zyNwPiJ1bpW@ns!P=qrOMep@i^MX(Ejr%DYoiGcTMlF0=P`D`q%r7>zI`&`fkO#WkG z$%+iBQ(I~`X3B4Yp7p77=wB*j=t56IrFRci>XQ`D?QeeO&FL-qnT8jI2YuJ^m0~_9 z15Qf2OEXp5x4%OendzILnl3)G3@XQ+oU(RAjy`dE!F>W5MdnFF3ob4$oU%gy{rfxe z!e&4c5PL#uQ|R3GoJ|?mN`@ep$5P-h15)kKp*!C+pmz25Z{MnzK_Z5JczRkLH@x4b zJZ4B#bh1^HR7qJ@pX*bfwQD_Cw#YQ2E_?AHzu)1g0Jag*Wc8|oNH=!weAF$iQ{RPy zzxqICHegpLzC!{W@b0y6uil|=14tCZQFR#tp zw{H*7mM^)0L?sK|h`P!{gVIn4+Y>p*vmFR>BZB)4)LBsS*7H$828s8@pFZ8?OxoJo z?syPjsLOF+V>oUlql9UIy@3L6Zo?ZhklB3SL1$(?RP^)nV;9hDh!9AF>ebFYSbLsV zip9n-mP)*1yrMv_@B8V~nn~)5s4SxV@Cv??ndy)84&$C(2s#vy)jM3ePzx!9iW03; zD5Xal3J-A~p&O2{dun|!#H?5^)?d~$2s;Ddig;mfXjQV77MrXmT_^VZvfF^W#+|%| z%_=Jw$#}yly_g3a;IPB(8nv$Fs}543$Vh%dT63mZ%q@Fg^FH%2SaT09joIbr)6+{= z+;Oj2y4z&>^w(=H)||L_Klywrc)*5rQVeL6K)^ZzA1$q zMr0HlZHSFDgF7<}zT9IVp=<_(APNR*cfF?Cl_6)&^r4@f#u7NLMFHM;#dUGQco1;}+~Q7Ze-5b@y^}ag1GkwLD%fo&0$?jolcZlvYHn(mOJfJx*Ci^VN|qbO|~CRX}^wP zosu6b7Ga>p)Dh`zy*0}^v}n>Kx4PsSH+BWg1`n&T+SY|2?&cnMv}iqtO`8^@q$B6l zlQM>in3UO^bGrO0q&}!Or(nmS3{kA^my`!(8Z=W#Se7hW5Xu*um1sMjc}+mgo-Q+N zkQIa%SGxU7om-&p&!mM_DmFCJJQ|ozTWGLnuO5^^@KW}d9*x`#Q`&!!=d^9xw|jCn z`<*-Xb}Y@GZ(FLqx*BvLwplEBIv(I)7hT zTje?ff;;@DX(rTEfjGn|i+YhkenYBz`e&Jb6ez=OO? zZtC6gP))?<%}qe^?xy+8hBHOMN?#|M#k-#-om+D2>*9p`IXuw~WL!o%J1r&$`O+a? zrJfVVG_=Pz9zJxo5`5tK9z-t?3uSn5=W)xXaQKeG4xs~yBk}-m0F2!t#mUD7d?r#% zpEhkB4$AR{*sCWi8C-2AoI(0b$ai`-K7oJx3LBh*=es&SflDnc>d4Scv zfA$0!n*Gn^FQfK%HSJTNfJO$iBJAx{od_6S}?OCNsttr+RiX~a=;SN;B-y{G%t zBIJ&5Z~ye6@~1G6*)Qv0ZBKfyc6}MEcR`6S*Ot0n5HjwgeP=5TXE!%Dsi@w)3xa6M z9Bd+6<7t|_**nc!vT<{33wFvVs11@YPQl63r^71(ss@2-{*MkrW&hqPqlyz@!|5aA z-v7F|LBobIE=MaT-f5y;dwy3r+yKZ92l)OuAEZV12F+Z>BIyVWUvjXBoZU<_W|vdf z2pC(e1Zg@Lm=$i2>~Wk3*Oj6vG%9Kn7-4+Gb4*MT;@GZ;8PkrWB5eT%QodEi8w1ao zPyj+Rg1~fl;6L{{sv@xl^va^@E}WsQ@`QOc>s?-*FTZpM9ngR1;(mtQr#CclR@_v{ z_i%=e{`t21){*b>I?()5R_cHkkf3#5Bew_1X zOl|lXR6T9@ngA5+0JuyDsCu7pLJr2&t4cinkq+sMLIMozFJLkw-j_hq6OT;q55CCe z_j&t!D9MXu6vfga!=fUq3qzS?&VWZFxX_!as$8kG*Kgm-lJO_0UXjLLI@Wg83XdNi zBv?sqCW?n4tvh5W&VcmUpaL&He%!(b<84zw-&|qg{Ax_1TLd@i6k9Iwjoe+H68$#5 zF|*d|6Y6T&@{EaAEaiH#??Kh_UBqllOUpZZEVk3UyMUo957TJeuSvJdl(|F6u(*J#zhJWY`xn=pe z8mcXp@iO!ANhr#I&q|EZe1t|Ks(?$PKvvAA8N*>yfcwzmuE- z4X**kVk3y-+%^yq1y$f)el!@f1;8VEO=ZTlsA$lF#BhDEi*QCerR=29gjSU3xe>70 z)h#T4PH)*z*r(Wq0vcPi#GM8d=u>E@d@C+DLL+pmosqS6TO>X3l4j7Q2qpAy2l^al z!XYLFzI8P}*$X#GQO#-=HI)=YAXxQ~^eQu-60HcwN)mZ2h_|1lV-JQsR*-F_um>A-=FxZW-d!F#<&spvv@JK-tFS78?ts}B z$j}0jX26tM(gwWE`q8+Nv_A7Wc+E9c4 zCvv~LejGk%oOkPxUEG&8lF7^6Yl_;|4| zi@7W;9UkXWdi#Mf9n1ew^&U_?|L^~QGArJZ$WAE=Wo1i5sE}+$wrr`4NW)AaQ7DwM zM=~RlqLf)e8L6Z)id2%6`rWR4zW;On-*Z0ad+?^$>-l_)>$sxhkb>YH2S8fCZ z2S4MycO23g=$}PGOs!~2=_yZun3rr&#C~v_r)Vu}v8js;l+9N(k zJ}YH^p4HZ-^=Y-$_z6Q9!KlEoB9v-5X~_AukiL`!ZP8JCc2eV6C!a7Cm%xu)dfEfzeim&-Ng+J|{bBdnF+%K# z5t?#v3qy7#FarN#7**h|HhsNGgp)CX(oic|MfU;?gmh*l|4G)G288nP;0lV6>iHUu z%q9ZQcFI@_^XmT1F?iN7O&qjfUypERHrxBYO$=?2Yh__mQ=5C1tvNIpVpKowk@l&UHkU4zua?U>(Eq1e7R@G^gnS$(+;0S8pY6@VA5+qItYC(Ti#sxU2Nr&A%I`IkQ=56R-o6cJuY!waaxydRbvq zGm62joE%RAI*zpNb_gDx_6<=0(7GmFfRdTqH>dH&QzKqk@K8-gi}h)#qUC)Ajwt8r zS1QZ%ryRU}d_Hm}Q7LZ&9FfC_J;6|!K9O?wiH>bipkbbnCe$R%98?2YPJV84G$0@W z=}Ad=qng117TZYk>DK(NuC8t|SkJT$5#iyxC!UQ5>_ZXw_;8nz&GP8b#nm9GCB_>g z|LxWOOfp1wG@Y1s5D~v&Tbr5%Axh`M5G^Sctb+Ua-zo?b?B!`8Bl+5#iN_r{GKOKnMyQROo68Yvrz-3{)@?1>V zfs#9j=2%M{ZsLlmrQOO!rU z8A?#%B8_~^vivvskKaBI^TJvDxhFFiIjN;mGHQPOeU#Gml`yv-3;!{EGLMdPL;{AF(~PS3MhiVuGF+DLD#* z*0>SHRC=;e1kK@(Pg@;mGhM#T2lU2AU>ZL4czF0z@HwOVU3nPfP4$q%hl+;rjaL7* zs!VwUXSt6e9g24q24$=gc{_sN!tly$tG9V5qyrZ}Zsj4F$qIGQ?${G$0`C{uT>& zCRb2iMU#RR5R#j_05EclxjEyHk8*71|6a#!MN`QiZ>EoerLW-2W-ira;H|!GWOpum z5toDJ=HC3971Qmi&s@^65O?LrAe^Ek6!@lVS7i@s%Dm|UO||C;B*NyD?$8@quej1q zqB9KI^eR1l^x4T1BbU!$Z!t+T0U&i!EZQ-a)-Ws+J&lZ>6i;fir@xkRDl~;k5$DcJ!8Pc-4 zU1CM~^&8+GXE+#wOK(E^phrd?pQowm^z*ZIMn;BF<^JH{@nF&lRCiq{fT>WPCT!zn z@jm+2cmMS<#Ke94(xvA*zu+B>Bdpz|%g~|0@-o4YfRnpv)V&hGAS5v0ah@w#8Tzj| zTYN8H?!~7#ErW)7%(g4-lyJMLJJEzgxlrX&owyY_OTs&tIjNQb7Tpl8BcV17#f+IH zy(JQJ$p0Vq{eRuOIRG42?4lHTRE>OF5>bF|A?Aa2Zo-r7X1LvqRWM)fMj zDd^TONAE@=7De-KipB*+SU%_&rsu6EPUrx)kC(;Ng+-HC^tGTs@F9pPg7+kZ0I4yL zlnJ-biucdTURa}^-?*&8+N18OYuBc89L$dTz4!h5XueA`9ZE_IA4)$z*{oI1}Z0I*SPBQg)MVv zxw$f13uu8eyfpk(FEqu#yCpw9jDXvY=g&n+Bavd}Q$3vbNWQB|a|P;g%kW}_R%qzfR8+PimeEMg?1yQjm$ z$5Gq#>eJ^qkBAA+OO)5}PmvWQF}VlUSYKH4CAB_fEG2LPiip~7XTA8<6A>R!8a}6h zR~p$!%keCd3-E{ji5K2M-pN-51uUf!OkbUzeD`;u;IL#Q!W>9Vo2Vut?eYh94v#IK_SH5g=2);VG@O4RMT!ACL7&Zg3-_C zU-i9Dp|u)0o$zpwm68#GfLU-bQ2wX!E?jhy8##Bb4Ea1WGpaZLe2^wYz>;wqX1Bdk zpU(ve68}AYam%I29cWDT9^kdWrazCq-}TRz)2oI&-;3$deGyBJyPBRJjHgRt_jeJ@ zlwQyK0j3Irc;I53zz4)aF$OG++V}$sOS0b&vO(yUoYdcNkxK>>ip60sCWidoZ`^r? zSJhN3cu!aVS|xLRDC+TOM&^H_Bf#3bof~tRw=>WIht)1#I#VZjHp{A)0S|ckXwT)r ziS$w^L3|4T$k=DJz&Ok#J`Ih;816$aS4WDbn(nbRHePj^BaO3m510)XTWm;ywG83~ z@W)2*FAbq3rn4_?485bN@&NtK80z!mZ6)qZql5|+HGR9OgG>(Vvs`9o1Z@`6E$Gq& zItJ4~tKt4D**KzD1;jGDNskn5O#Fn1(LhviyxedOm(O!->Ul=ctac>UB>mBQs< zxlrcvmMF9qXWZ^~h4GYlOG>^(Dj=t|v%wR-@w6vJx3b28!sIQC*T}DVZJQs8AQ= zvQHvnvT;_f`icUq#mw!@>?~u>cK$E*Q)r0vq(sfSqA3SJ~T(wFQqp#B!^`g+`XAYAo?6qX~U~yK{Xpsg16x&9& zcy$8j(7=CbJ)N)>Q{aU%$9ZGXozZX%OHxlL(L)^AZ0ogdB+V8N-Wr&Jisiqmf!%e` zG|-VqE)++l4BlZwvIp8rU?y4&pz=lS*J)*rM6wi!bOmQB;$O?!%I*f&W+J;lfO?Qf zfNof~M#b*HR5B|X)J*Cq^ovp_AdHpPoKJ#sc1qgtmSK&!y#9gXX97oxQ=649;y954 zgBMwxRHHn@;$e);c55v(TOB~Qm_&vlCaApt%`(L-KshD`EQ*6po{h9Du5-8U(KLAm zGkg>70;pA{KT&Z=;0{N@0!2M7??9o^DA>h)kP+qNLdFYv$Pc_)0q7(mm$OwK01XW$ zsNNWOWjM2NE?hH?A4W;r5{&2h;XYj^GCH7ZAl^=k+t(rhZjlIG%f*ZJ$a!H<+K-1_ zLIkPd$?IBbD_JZ0lyG^7_W_bx3ONpAD&A5Q6mo#FY>^O8Q~%}qe_DVRtZcPIDM{7i z8HF=XBxj=CjW(vl5@w(BTV#zh|F$soSEi1noHLeiEY^EUn3;;GpF|K(`BM*r12-IH z>I3X2NDO5F3Y6~aTWh+Z6OAQ!f&1h+)L+-hXF7TYI5OKz^bdd-=FJSN_fMXjV`6}B zl>$ZNJW3-PS#ck&BDVYILGBr3jOe?q(u<>PZgZDXWn2 z78c$hd3x47KUVmB(mN&Yl$+d%SbvywnMubza`M(^gPq$Bsh4O+$T|`#mSNA2sW7F* z;>MTWnIqhC*vZK(DbU9G;_$7NOugqkboo`-n&LG8NJe)QB2o&?0oG4?<40|x|YK)N6N?`c^^1y`%V0ZQ>msGd{%9gaD*oHJ8TU72RE;*rz0jHu?@Og(*kEMqz}H2W2AV<`cdB6(tSCgE@m{!~6* zy@w#6q`zTaP#RIyDX{&l=w9@<5?%ut)@q<*ZnxQIOdDCh0v54> zID~^g#uTO_sI6JIww-!?YG=vd1&Bkf(HIHQg0txpMOPxZ3bi~-&UJiyxe|UOEZz-4heSNqzfsvB1w#wg&0$E3;L#X{?EzWL zyw*VUjJA1Kh~<${>O53Tpp8-RNF~#{bznhxb%^&GY9tZw!0PA!!mKF~?jYAxgl_hU z!>vWD4tq)kj8R*c$PTNaTP>}dRli$xm!Nfn+5@24HB`V*u(#&z+I8oo_ukD(4}K+c zE|6^QsaLWcwU8w@4NdCP7o`BKiZCf7IU@9#Cj>zamd9_Lh#Ip`Bg?JQ0Ig*h)iK?Q zD18)Q{FPlzYPy~7|NaK$;$?JZlmRI(UL2LabY={IKIh3k+p6MPEi=w}+`_2yZn0EVCf5@?{%*w*= z`S2*uGY_T%SrZgZ9j zC;||{FW}U+IWEu5zf<#16ew;Mt^r5%Uq~NhRx*dC`)L*mkfi}6i{q}x8oIWe9$b9w zAPx@p6j{7GZ&ECJ6iglGAK^iolJjW!Q_bfM!7r1~rDv{MytvViNA|z|XJI^OT#p!* zl7P#vD+X>ij;ORdRN1jcJuDLAZvO;?SCFey^TEk@Vtc*l;vr!EeYzUyKPXFQT){ijT@D61 zAj6qAC*W}Sb?6;E+nq?==V$>G05Yj7Mdgh@3m`3GSY-J+LVOgCfuD%1D?TxCKUF22 z8Y-8z{0B1jX)7xjhXr3E?1vxV^_Qo%sjdGb#WH`+10s2;potmsn7G(CjO> zkNz)eApd5#UWbN76TvGvN$LB|hwiczMwF5zA#}tJoyNt=+ivODtyC|10@GPycP^}qkz7FOta zAYm)PKajtE`LC>}S*spD{bg5P$?`x-S5;8(JL1K@bmOAC-wF!8S4M0~e(F{|ZG6<5 z{*(`>gvuU$-0-VV2_DmW3_Q2s!e{WPCJ#vH+J+-Qm4kltq zXOlf!a-=pT{#$y=oc#RGn(-A$@8ADD>U#HXI5G-Jxur4Rb>P6YO)b+)Z*T56dP~xN zu7c;7PDC)0yJ1h$urqDJUO9Kk+lq-Fw^!zu;d&Cr2|PR^Ba4<>b~`5szZUh}r%#Do ztypIxt`eV01~*o(6BtHd0#V7aB>&?T%eyhtK7a466=rdQ2+VOL5_KpQIls{lv&!jD zeNoMl1kHQ-5BhEbEuaWi=Br(=DoLMO>Ap0sM_1OOLv@vlRJ@cEClz{ZL*f3EHla2M ztFd3H;^u3V2@IDELrYFsV7{n*OS94~<8AIG%w~1!M^dwZF+|PTiUhP9^tmjvtuA-$p z%G6X5-6IY4C5RAy{A(yy)ahz_Vu`Ne6VgcO|GTY7uRcXlQxTK96OWz_eA?4`3ju}b zSQ^s7Bi^)}IrB(8ovrJ)*@WlAy0g($32K3=`AnccWhq;1XzJgYi@~{?0f^tbe;<7y zyu3TzID$w){7?ac{J@paSW}1bmlF9LXj}LV8KTB$0R#S(fW2Gw@@TQed;s_*OCL(K zH^dNCmN~5*^54CC8CS|QL4{q4( z*&*0aKn3DN6DYQ_Uxk77bVuq#Qa^?{lweIr6^XQ_RblZyp+u*J3cu#+Q`7w(RiqD> za?g4SgVa2eR`H1z(FsT;PgNqXW1czvDtjjN(%ccCb>bG_E|-`(sA@^{1LGZw0*4`2 zo?~NlMky7!BjRB}+K($BF!oI)w;si*-}9yuDa2YGksJ2q%MKSo}eEZ7D9q zpNX1qf$NQ={;V9hLg}(0kh*$GUlo^X>q`X zb5SBwUCYckPUDlHGqM8+v8khM?SW`qupBaRrNxEdY1*>ocHD(9u;NyvK?EqHMfZyx z=_zD1VLMmriQc*XV`xW&nwvqKN}&zruUsi$xy3KdDI-S6K~ho@#;c3nva!)@-Zm|6 zDY-&Dsa8n(1)Hp$426Y{?r|a#HKdK(0d?!?$6I>&_;f;|k8Z~SU7xi62n7FS{PJ03 z0vl!S1A-kjr=uX_XQL^=^JmTq1Pl$MMwQ_UB3fo0j@9xY{Fh563jRbQR{2VR z!_8Z?s6XWMAC4JIPCePn5-mv$oLV|EdMYauM$d^a0kYc3lQVK;+-e7hIk-?|DlodK zY-|`jw4Xs7ofzl=67z?ie(?oju`?x`FrOF&#cGB4l;jVtHVQ*a;6cZOI})Ox!qqjx z;veFyV`~av29t!PS`2l3M!FzP1;0CL@D(h!0MI_*T{zXM2T#3PW%Z&D!v zdk*6}EZp3;*oO7t2mUC25B+<1Zo;w&T{z0c#JteivZpkB0H!q275vj#f?%%xUJ6x6 zJ4e+|x1P=ECDCv!xV+5s8LwZr;=}^D9|?z{H#?@PlMmC|>vi{T2X=@21~_uVf+&m% z%#8y=Yjbq`4%Q&g5r=w9@B!%#U|(pQiCY|$T)q}nIsfzmdwVB%PI*As=i@lq7#^J$ zej_uCI29nsX`RQaZyN>IMk)wjx=Za%@?qvop1eOtr`WvEz6#0|Us~nY&nTXt+8e)r z-=GhYuK^JGKSUem$?m!jw&W&5@DcvLCv`iD!IV1CAbjDzaq4 z?jV9VHg+(ykjp}z3a^R$;k@AEpec}Z4R{%nb3}2>U&blO*;<{t?Fy>_Md)ct3Gs=^ zG608>Z%MHTU-3!4;XeRAX}~P&YzegEh}W&XX4)Y(n&W_I|eQz5O; z|3}&0@uxc(8t$RcK*WeXy%X*RZiWo)!N8?}wmh|im77=|I7lJI^=I8O!ty|51g_y- z6aWZIYkGTk+q7X;`<>=L~-veYNoUV*`>9#*|r=m43MjFD- z!{wCyAgX^PaCw}x^H|xk$JLvjoESJ*D`UUdaEdaG1xrS|Md^DZiMX~l@FZ|Ad;R>1 zlggRrl-qI=NF*p2QpCNs7v8;n%hZM4yLXTK&i^+E-(`l4NEwRfA9Hr7@;7?8s86w+ zJNC+tKz$AjnEDjp(#)SyP2pB7MMCjb-w=WQ7) zr%fYv!W^+3P*9tct%2EyO0cmrdqEWC*8|0^DjU(F;PL8`AQjyW|uE@)1!0b>G2N7pelw7=gX6oIhbnDZ(hlhsbw`5S_OYiWkg8%e?n3mRZeb3B_tXUPMZ`TD_ zl)WzP8sH80oITv~n#D=Jc<%U12jQ%dPAGs-MQM6e-RNR!hxU>mG2O-{|9b(ow74LF ziBnU1muIOSbg!}fZdG&F>FXBNp{TcpX0x_L|IXqP%Oq=P9MvYplG(~$y74~i2P6)k zKK+pT2T*$Jn{eK~$lrzOzy~A(q_C|n(_h~?$FU`&UjQw}@SG$!16&D?HuTj}C)@W^ zx=va7JovzY?c-062%q|?6vs`q1U&3m(H z#@WgFZWCnUVN{gM_Y1#&%$Zqze+4JO$j}LM7A%kn(bRl`XhL9Sq#L_Nb5x2(31EwD zfPVHlj8tIRU`7&7U;m(ncpSeyC#rRA50;RTH3FbSOIgD7 zk=BA2?W?K0TO$+q)V!f5N0VoPbbJiu8Cdg2;)|Z#D376(VR_o4eqt}F7aYNd_M6CP z34-O0nMVku^!xIWI7%{Fx)y&;H>3f9{gF6U0#0gEy{Yo${H4Kxlp7YY-Q4S7>4Ubb z7;D^h_~(JGUu)a>a5l?G)%O2o>MKOX_7u8hzLX{Y=aU58jgGllVeH<{> zBA6*omtGrxEf>0m1Rpe~`}WpT;9AilD{wO~pjWzd%5cYh_!zXfT0dU`-iSvIVo=Om z>oQ$V$EIB;p@3QyiH(7E>ohdKR0Sr*k#mWCWlj*gaXQncDT-YSW3se04KD;($XUeO zNn-g3Y(uhA4r)gq-9_Kcy=@ zG<4G-@Pqn8uR22yqVfSmKaeyQz1djE?chVl5zTTUN-syZJWxrcV4m&ST#zMTlz}2U zmkNp{IFtd*LgNA{>gpJm{D|C|syj=*`E5i#z}c;hH(UeaY^r;Rk!C2_z@dZQ|1HByQ&X-1o)!Um# z615^ASZ~TXbv|dRnxWavrV%0BF9Lhhx)ORkQq%xydZ7u) z>FZB+nAO<@P%-phfwX60)E;Q=(~(vT4RbtaniR~GO&?|F-N zD@ZF6`AOr`w9ld26Wc50l?)sNIUt)yg}5odm0xv_H9GkHvwZgxuxw;&i52RfS{TIK z%O-pglpj*l&FoT1kU|zXyPzNEv=eB&goIB^6g7v-&{Zo}J`>$Kc{6%?jgF4*(q(0{ zMQjvA_#&O4nVd)C$W7%08lwN;WFrlJsGmDcm5d;gyaB`nwRLb^3VTWR<_q;gAssqS;-tX{Xyp6*flVJbTF0SRXs9SrR#k5h-dYf)n%+p8eO z4Y|fvz0GLg0huL@_!#sZH*Gz#P_(b_W<#YaSJxBzNE`w@sNUALy(Zz}lEa#_h6IyE zsD~t-YVbGrZbkp61t5%SK!Qico@w&UE^xsLQ7C_hA6vV6O^G?vM6vdm5={(RX!;(! zetk1(g*_En=I(~>&1AS3odh;q^W>}ZHiR3o{sn5LucfpQ#FT1L>9ki@<@?UEQ-_r1 zWJ?Ff$&4*j6er#uT;QohY5;_Z6Pt}^xbGe0Dox~Czhy^zNSzwU8^-(FduVj4^C)4s zfx}=v#YD&6DP=1>ZrII??xq<>`M(ok7Vs1=S1KD0EcrI;cT*jU=mGMYB1ld|tbN{v zzaAtc_#HiJ@G&kVj>C`_e1!~LK|}Y>dxV1=%;;RGTPBm_aem)#Zs2@sh35G8>GZm!1g1#Ry5N+z%4DHNKey)qi=f&qhxe0nF+`EqBtElyCLkT-VIM z7)EL+Wwy-iK%^2eefdUO1UeV-1A4mtC!r%q@4;v+TIqUFSmM^>$?U*DC3h%y`U#&6 z&Bzea(-N@e`RNII5*`DV@;_6Am#IX~!6Hdf#>*9n{ouj3=Ux3Pq?uw>eM;yTWE073 zIx0xry=r0b9LQh1`G1$WEJ34j%m>V|`&cwZ&Z~q`+-E9I5t;m-^Gk~gyu9!mvgygIY*f#dT z7Kv%QET2id!Vvt1<9VYC6^ z|*isJy1QEvg|Au-4(aL8>Gg#jU zMd{grBi|Rx0G0#-ABgX2s;wyn0oC0;#GVv`j8fTjIqNhL1xu9N&AGI@fKIT#c;)_h zs;F(|Q9ue>223Hqiqa{Mlc!H>+zvIC<4r;G*AeB1QfbirVmoeBybpQ4WHWJriqe4h zh^WZoNlXh}+E_6(*Z=jWYBG8bskv#vW_MpT)toa4g0L@u!~ZfQt#dc{ ztFHq$6_If5TKeTXGp2FTewkY7 zO+3#?8=IgfabcrH4MR~XK4}OLZE6KJrc!53ABQDM>8rM@dFm-i0l=#!i_4zZ%!R9D z)<~(#WC+=rNKH*XN}7APPjH@nB=wrHS~~sU+mbQmS=u`F8(e zG6`q%&`Ahj$x+Sdc~uW42vMs_!-9wY*o!Y@VJTw)4Fmf3jXubqa!(eLn~co?@xgDB zZz{xwHS^Q`v>&L^98lCq%{aIyx-@B-YP$iVM_^W%8prmdO>Tf~5|j%Bz5nP@fs3Tm z;_W8)U2>E)%5(n669~gH;1ZLc_5NPXJF}_}ku~dP2SuymtnAK+?8IURNWl_F}^3Y3V1KQ-fM6ubyu6#yia z{?GYEa)C+^6q;ZVVcsBwN%0fKlDW;IQL`n+{DNgt$Zc6N=;$6wLBR5R{8gB#5w)Y( zB)>w&R?|l(2h8WiG*VId$o;*|t#Rq2%7wZ19sa;cZ^Nbg1FKJObc!tvXHsPyn8Ps? zG0f(k*-q)prT)YVD)sc4so_BanGJ*97tBO93Z(`oX2H`Z(Et?ivkWUNjn>_Z%nhbH z_T4*?nImDA@FMEW`2Y+qZW9<`*z;nBaWqS`6JV$a`dBA&8Pmr7ebNrK1We03)+(B?vda;g=!ni5ypPb1h@QP@ zQ$^s8%odFan1u_a)D{9=MnrI2;Ga@^W;tg3_){=FYFj((Kj2^;CeC&Ck;H@s4ufO% zKmaCHAeDR|qo8@cDd%lJ3MOAGxm_MSyNG5KoTCp1=7;y|vJx;QaYxh{a=C3KXFnYF z_?5#O)hyfAf%XldjC@u0Jv0tWc0-}DoN~-_-$~WNiy-Yu(}Nn-WR1HF{wgjW4oC3Y zoy*-_RnJCf_JyORFh@=_9H|Kk3jeJoYM7=)4A}AKA=xix47H``0N#j$-*oD60zf6} zv-6;sn(B?j9GbKJTN!U~Ub*^|6@`fccz*1|hhO|6zJcg+x;YP;6`WOhV#X_ix+-2f zk?+tTn{L_d;=NPJ#`2Y9HC{Cxyxk@5Z|{=9s+;l%6eswWHlrWtgvc%d(}pAm zch+9TW1F}YCUoROnRO!qF2DlGZ$ts$$395uM#m`LA^6wY$n7;>Puyre6`68Qc6Kc9 zvh?-HWxRyZOcg}uLX*FPOa)4eGq1j87`}HQQD9Tvw+#WAjaJ<$YYD_D9(x!L1vH$S zIaY|@^Ho0>J1NonZqjQzUUufD@>yWKdn3$ljymr0HNCV`(dMKPA%Q_b5=|omuF(f! zdr}q`D0c39E)Ae32t0u@P}@$u?BAW0vnbZ14nF^*^!C|!>Vz~i84S3%1Br061V)* zg|zD#ON`n`yF;puoG`#J{)$6J)Af}cc~v>LyowtF*l;W;Gq?OgV51(7apAdYPHfxFu}>3*oQn z>4o;izkV^USwDH>c9SRKW^swsn|5xpk_Uf&fMS7Ge*pciW!~Q*|GuMeg)^|gA~--h z0qLmYe;VSpF5skb7lRLB@xA-?TLdvHMgR(=7D;A!O4RG)M1s}Z#?vLsU;Dm8HOFQf zfnUv;KL)KpvUPH;>6 zQ}QQU`)+AdRmyi+OHyS3o3tX+m8gKdbAo?V{+YYT)E)3u?>W|#Cv5A-z_SBMrU%f| zTTNhExMIDzO4^X}ys5qIRIwej$lc-Di~CE7$}Q zd()+~v#_YS72bJU#Q|dZWsRW?diIGXPg*aydhW-4eanv-9OG(AG zUlxxYi_|CAyQ42Qbzvcj+l2J-oh)fa<9WCqjyzfPBp0gJ}*e1#uTe&Y5|+l)PfE1?6M{!@C!qe%r&15- zSu&i0Qbu?U95~}%QC7c@+P6Ym_&xv(unqUFN zhJ;13S0o^axoKlo-u%mu0>qGi{800J!iL3A9-1-QfbFg842$|?#ISlx0IX;iG9VcNZGDEH(Hz8 zhYOn7jlj0SkLdbjz8mZ;ClhX2Kkb>`4*K_W6+^AqZK#Dh(NS{*$TP}akrg8Y2Px#G zZ{{jcdt`YfGV&D610zbUhk88~J3DZ}5HNBLh%75OR14hXrKaxZCz_7M6#X&goj zL1WNdqGqv?Qhe>-yZ6|-)*RwtFTS8bh(01sOQ^dD-w~0u9^lXsLIMCSY>ecuQ0&O7 zV@*p!`J5ZSpZDjo6Y)J@;5LK~O1UlInC8%d@!z26oXPH#cL888S-r?b?!2v$rntIL z-g=59=))<_H|N6F)~jxtDQS7P44?X##VS9UYJ34t1|!F$%@6HETuaPXobemi+PL*6#Ngu%(7#|Frr3#1I$Wijh$R3V6JL20;IQdc((M# zlK6&1AJ9q+DLkf}>!L#?C{8?2wDA(p40oz?NO3ZH9^o~JIKeq`aF|_aBVRACy|z|6 zH5T!>IL)6gHrn7+d4E73>zG~pbEJe2 z^xD0q+?{ij=zK3l4NNyO0Rx2uD35D{!b0Vw6_`slH0YaKLR z2c=q~8xUp@6lwAGw`x%#^gA4`LeT+;x!aO=Kn)|7O1_JHr8#J4MgpEOx8@iJysjB9Jm z*1(>5!(`L{j^GJN{1Ww)Df1Y(T-Pte?{SLG5zy!o;~UUaYKN-#q5~TnJS32JsAg<< zz2oFF6Y8*=C|^6x^_rH({U@=%uv8W=x4LuVhoO6M^pGVmxZG9fsVo%wmEk@q!3u+RVwVQK_ZPIso3B7;e*d;7xf)!4I&S@zI@r zWDf_t*7F2-@quomXqDCv?GE>_8P}7}p?S2sJA@4bTIM%(J%{X|q8ej)Gie&4Bq`3Y zpY&Xn89nb#4@xV#a`Sc0f>tFTN4WX0yVtRa+> zM6HiSMair~;HsaK7c9t~bi2#!n+~2}KWH+aak2>eYH_kE(BPR!s0k2H!h(pBZ(MN1 z&OLh$tSk_Z=gp1f7mpZeXHf^HFfU#%9A3cXj2+YyL?G|x-fTCa_RB4!ffU}TxAozs z8!V1LmqkcP@&Gtavm$8fc^Pvp;p0F@EZ^ar)wc_!0Y}|xr33|OSdDarn$NP+IO1m+KuH<&Ksu8%3t4%dn4+7GnoKl z)Mhu0S_eT+NuLJf5;g&umI5`IFn}`Y384VFM3*`GV=WG$W^CT7Rg!4nNX>98+Hr@_ zN19t0L)o;{?*%JH!QCmq%&L!#+tBp=9DRW{r%(FteO=mW(Y5di$#xpHbetw z6a)=MKE)|$i{xaG`9y4h%dhh}-T7mX%e{5=Wc@AVOYdQSNLDbKV=d1{)|!7GWNh`Qh6=WgP>XvH1(c`3fanF?S4 z?b5ZM*-e~g5FCy_Y8iTOd`v%{kJfz7CYo1jexYejf+$O+G1=)f43?I63?#Sz(oYc_ z$bGx?`k>^9Wkzj4b5KA;DWWz z)_OFxqfw#N@phi?HU9KRt}F2HN5h+Q9r;Pn5E_r!vfNw0*rqpE?1D3AX7uslhdc$x zkc3;EYifpwA3#x!(~CB6VzX&)Vs5uvxM2SL^_QjL0Ke<@w=BJj-AklHWEnekc`NrRINIK9i$C@EPkadP zy`HniDRFI^Hf@HMZ6wl`5Vh39yAWC#ynk*MT0QpOmN$tEDw(8nqnJb#A`~WZlnk7c z(+t?TX~ZeA;6Rn@px*=eM@DjEtM<=*qmC2E?gs)a?xL|{Mb!a(48z9#-jPt5UAr2m zGF~EOsnLnkr}ATVOg*n_VBCIwr%nV@IfnG_)i*)!Qiiw@b^NXP}z5C3v}=JN|s zV4MZ$3fr`8dw6u_M;Czqo+L{8TjX^L&ql<;mXru*hS$+uV1-IDjw?f$DBg6@?11c3 zHOl-$@J{XCz12_~f>3>TIsSaH)GR_D;JwF^9HE!vsrNJgyF2*%?cYB_Q*V#38_wSpfiH8map$Ge! zJ7o9Ro_%|5s&bj%{K+oOv1=F4o$HCBBPFHhI>nKqHB`L^>F?;f8=WiPII5<9pFWQ_ zEnm9y&&w6f5^U*p$nfhe{UH>g+0u8?T?4m=RCpexuxeDSCdDOpjTq$#j{P?#PTd~?< zb?7c16%n!GE(4-Pnxq*uAcXO*E7!faN|`*) z*>()nqBj9~#w;={e5xp`Bzhw0+sXYc%SM`r+8_&p>TS8ig2edru_dTvO&1${L;Uk)< zUG&ScANAg*K1MOPUr?tON$2+u8}uXFsZL`x$4FyaPwl?;3HlywQat+Q#VI>~8FcN9 zmFA$Z33@#Z7J6Ua*$Zf5`hhF)@vrMyI$2&YxN!N>rPzO&f3T1NmA|umm@vVAR8n_B zJe_0^ZLFx0ur5zO5&4bXBn_?O-dd50KO46>QPC_)!C*7`Pgl2WvwJ)Ep~NFozTJOX zkiIi|^%k!)dRH75ZRt0)>+1eJk8Pf6xng$%jo3O44ca-iw2!qpUHAH}IsGP|-*l;J zbY}-uuLkoxH@Us5O`#f@iV_zS_LRA-$ z^?2+a>p`80f9@=6HRu1d06$&7)>~nUWOE{6EOgeSnsp$QvkPq?vNJM*TS=#OezHz4 zeZQan&b=q~DRtGXt5VlL&Eis>qr3kL-`#SBi^25bIlUT=b@HBj^+;mmz?%M#^@GRN z@i~&-)?<6l;MosVR&8{A*2DAVk$J{_KGmE(_Bhqg&wu!+4Gk8UE}z$Z!I0k_%=Rg6 zscfKEZ9PnT@&0Rz_ZjU?tlGG}^Y-b}XUti+V%yQ{?SJ+A1?l$5P=eB4zaCPOaew8D zusNtTr6Wf2C(!`?u(Dy@hoYJ0tL$WNKjZ+g|I>@$L@dN-{YD z65&P9YZy9o(pC%@wI$SaxXXwU=av|`hP|5D`iz2q!!b}T<4kVKQ@snKmK_q)g4ns zP)G=^h{yShc2&mna^`#a`gAeq)nvC#DIie9)}UiOCmgd(Uw!;%lh6P8Khr<3!clqg z;UfucjJ5~On`Lx&t*Xfd2iME1UWASq*Fm#Y=Q=wFT0Wbc}3L&R#SsxX)(j3>{#+I;WmZ_|!$Cr0ud!0rHhzk-2p}DR{!8e)ebohG@?^IO)jjI)(`k z9CK2%+=e=CP3YR9frV}hQo@#!pF+n=xy|TVgDz8hk#k&@l;TYzq_(Ml(Cy-NJAHgR z=^K9W`<~h0Km)y50iS0&pXqk)!-`{#=Y(7x@VDslgUQ~QdIs6+mn+tfZt1nr+1$sx zlR@X*X9ipA9rtfFv*$34<0Cc|&CWWl&~0$H!oqJ3UB8WNomkW4u**#mOZ0N_26LQn zd+iFs6=k-hUF9$IboHpVs0DJfYjLa}ci*I?NqR1R$sdg410ZVf6+U;+)3H#noXGJej9@>owNt2 zwMcl`e(;1Z&g*a9p7c1e%MHDQ2TH%-2KF*mdV2();!3|U1OJD)o{q6&pCz;ZN0YV4t}UJ zx{Fh{uPwLNvUjU4RW44mUG*?o9Oqv5az1>BV=pbHDnv7OpmNR zarfrILt_w@*_?UR`1zN`4F-I9Zm_?^vh}eQ&oj2;!rk6Er%^+vr0k{G zC|JmJU`EVED zJPriVm@#@`vDL2;iIygQf8TDf>mh%pL4a;g4nJj_t#7t2rcYg)-0qVmcl}=X?%B!9 zO;&GqN%Za0{LHL7gT`fTItXC#Dbvn8IvSJ!0b&9Lcq3KSE>$58eBUv9tQrhkaxOl* zN!^!rOPzN+t3H@Axk0AF^#JuZVXbDLH7N|6TtbT1_=uTDdMBqG4(nJG{d# ztA=B{k9An>6nFn=U6w7l33U7DorReaNG$m!V!O5R`9aH0+o1C?uD1914|UC4^fNM* zVA`~F?dQf8{_W7li_Xf#t>);V-|I8`87&C188>Wu|mV} z(&rqrmOZ8q+rKU|bm94_u)ia=pB}Pf5)~+A$A^m>ThL2sJH+%lsXi(z|O(wjY zaO~HVE$-t(XSKfe-R7D__|`5zeR^dFUah(@%(tPbR>9Ji{qL_VNzQX0aPwowbMI7# z9=g8zTIj@W7LzYd4(y+~UcMUHNIrj;e}`7cIl3-WOGl?(j~)@}SDL`x$1w*-a+n5o zmEL9z;-HRLDj6AIbi?OX(+F*Rb_Q9S63U){UsvBNgwtYSwf7iq_}Tio#4i!Z;Y zp8i-_xIC?@qRWg^c1p1m(?t&+HbYk~FiV;+I!v3io?BP)S$ZX%o4%b$;4d`P! z$#zTHY?W7@Hpk9({CH*Cdc%{R4F;;zwDEP*9H6@MliEP1>c$$+W(~VDZnvTBj?00) zf~O2QJ1e{&4#~K8?{2@oeyIPF)UABEEi8qY$nV+WQcF+QpoAT$TKkp^Ie2v`YN^Mi z+oY{F4E55WR~1Rpn=5ncYJK=%)VF8$w4-|tEbUv;CFb*$1{><$dA0PSV(Gkjzt?6C zNxL|6XI7Vx6mT)D8D(Afa~~Pbj(I}+Ghkg_yYvwe$Dd7|KRK<@nofOh{dcS3i?sB5 z0RfFqHr390qd3l~<*;qalVe*X4GnpEviZSAe+R3Vb?!T8_kUg1by+$k+^la=Z5S*b z!WLiB90^<@ff_DA#&AaDrL7XzE#JWLqLOs}@7KUfd{YDLYdgSD?@pFR}k9ZuK->YzN zjIYVGnw@`Q+F06Jx_}5tzfI)v< zSy=e{OP4=Ci>z3`7V$$RErtrxsqkkn!{EDKqr&4ZUDP^f*mb=IF`Q?focP=_Y)m(^ z?~VP*@UDB&^lRMY|Gp1Rird!k`H5wEMKd~Ny^WFR8dO&rJ5|*g)+FhcuSpsR)%buc zfG*@xeAUp}o4gR)`=`ti8Hg4D10UX;P~3`+l;u7Bj-&1?e~bSV#$E6IVX}F2OcTWo z%dWe&Z~OT7>W_=F3m$t}ANS5{=!|jW6U6(1WKZMx(TVeO=WV?e(B}Pw_R%U`2kYP3 zrkAEZpkmy z-Y%QJ4zGjtLNh$a*%-Ljxk0D|LOo#WQ%E5}kQfr*KE%CvAmm2&j{PQ0v$}P7w&m*{ z<@d_%ofQjn2dG%*w5mJKLrEIclIMwO)b8WoaANn2u^Myz*0jQ@cK6j$Jw|{+Xc=Ue1xEa@j2C40|yV4+X@paepFD}n} z(|c<8%oz(J4QT6;Y@~ds8Ps$@-@Ox(C%jhqI;QWF`d=RxZ``GAuYV?8*OM-H zcS|f?<4}M5=0Dfkm}*rTi@JxU&{NJp?`;~JRSbsuglN;hLfV(SG90RX9lH*zlszrm!=fRBdKCg@d?95WroKbumdacW+{xcsyi zSFOKw?$w2~r6+@%MGdO+?mC=8<`ZDS*^m2)oWI7dy8Em<{x64~U0nS<)_K_XOVfKg z545y?*I`z(jD7=y6MXek>b{)za&h|NHyc;?2wC%Y7f@-7*N2MH-vX>j4t8i^4lnjZ zw_8$M<*?9WL`Nl);D9zPI{tCobm~Q~PCfT3fBDk=<(CtlqlcX8tGa!9a${xfe*GdR zBtL~?>$U##oUrKQDR!y%jE-Gv*IX;UTkm?eJpP;ux-fS}ce|KFA7-oe*jt=K{enxg z<)hr?;+KxubeNjpQLmO#=Fb%Ct;7t01J>G`MyggJpqM#m-JYT;UG36$f9>$=``9`$ z*>Q(*Cl1?~o#t-0@ps&s<_qf_KZ3F;+qWiqe3K^)zUD_CY?iI)T<_PN*kyLkl~+7l z4>O1zVt=;2Yuz)!m$J@cAD|mWgxnMm3WH9hdcL$;ktE8aR!=c`A0#kW9-ayFSGYm@&UYM*fng zlhP_CgdcHV_4MO)Tc?<*1t!Vv2d}p8=b@BcXP3f0<$(pC{%W~r&Mz5aa;S%+Ye;VA z6vI!4-YOV7J*#7v(@x(!@oL9Ib@mV2IP&qf-zxPQD|hVNxpUUEQgq5KAGPlyj!hJ~ zXJ*gtoYXYAbGed6SR+AX5uUr2JnVuja?&lMfH?q5YaSoyHTHO@YqQtE$)2-o%8Eif zU&OjpuUmI^R82)NKA>l1Ki5(Qmu~WSRpYY4=*RqLVH=;NM<@MSm^X8o=AY#9qscSY z*4gNI@o`Ofse9%Py@`sYlh4)I7scEU8uYH=9`_OV>^5tADX&a0b~x72d!z5IhB@&z zw$qlCt$AhqC#}pW?c5rzw`o&a+!(NW!U~_XJ1NRuDhto-%`G3TeKIY5^Y#|slh@=a zZ~fu$Y*Vn)fOWxUd0^Zw1uZ#blV9GwvHs1x|8}$=b=9hF+3v!4%T>D7H!SuwzUF4> zxNc<4`m2iNb<<8(R9~s?+xA%L;K&=QJtu~Et#SEiAMsda{U+<@A$r4CMHg+a-hbwO zta+?q-QshNXDS}?JaB94PSY<{?|%m87CDV=?Be=j!`6$>vMfvAT4ZMp3O!M7y6=5r zpVK8<{rl+0_w!vCf3VZ^>&pX%Cr&=7KdG~&%{^p_TYY`4A}{VXbW~lUq@le-1exMa zVmpx*Jyvnv2@dPnqn{W5$=kaor-$O{vWJEG$8J^+bzU1(el4ZTRjcQH9nI?al$CqG zcoDtoc(qICrcKV2l}DNsG|&4P)1}k9{USz;d%=2hJ`z)#JJol64UO%^MEB{jK(6*W?`c^ExVJN;eHIDXu?LQ(^8L zerDC6Q&tUTf38{d$tbVN*73CBug_L{(>t81n0(egXhE+=dZ%jqzv|!muJvJ<@n3VJ zPFsq-Qxd;e{dG&Lmpb_DnhjPZKRfLIdE`)$lk1D1<99Flk9CSudXe~HR_4^vjyc6o z9=6k3=&pSE)aC9oV&adPA*(ndH_+F7z-JOLyBBOxF~7GhDICoL`bi=Gt}TAP8MFD6 z&GWVr1*3m{V&0*O@B_ze=RRtDCcLIYb>`Kz{?%oUpYzl8>$&dBxBJ^y<8Ns*XQ%by zqtZLRvT+Xy%q(B9pwg~?ON08iLYK|CR_9hhQQV7|F5wG3LSizGT%6(9`%~|VEm1{j zn|92*loMj${XqLp!0)WC4{sT@SgCMpWzr%Kjax0#`-j(WbG%@D;pUa*{ZrPwSbuQy zpZMObU;F2mjr)^sbIkT|>+IV`Cd+oLEZnX?q0`f%mFD(G-hMTCwV}?C;ptP}P6^p% zWntg?i}Ay(PlsBS{62O4qk+beK}!q=EI76&_r%9Q+t(vXt}IylEn{KIj2AvFx{bno z!Jxn=*T&r?&l83zb&K^FHN%lBPfhMxfeTm2QA9%!$7Y-7Ef63z%HO};RpV9pGb7p# zUv)A{b$OW8hU3rs_C6V~w)o(b&)Z|Z4^RIyL_sh7?p@ctp7K;Kehsm$@pV zQ9suRl;y`7-wF=?Toh3=?b!zv z>H*KM(bZ*tTf16qxDm0dx?;s{=fP#k3;UGB?XDi!<+_qdy$esD_W8E`enLY1Nh*`x zpP2hV?|8uPS$5M-T{!eAx$*m($|=_m#jNc6EiUV}$+Q%c(La8b-FdMm#4V)k?(7~@ ziuT2+YG$t4XaDn1bV0mPe6c}w4~-*7HVoNqu6CyA3{J5A8TL9;Z&{hc@*dDXe1Upk z!_PC$X36s+d_NU@?8XbbcMP7r5#gpIcdB7vGS{x|fGP$QG1>av5n-q*SO)t2bZ2Rb3#NCMK9T)3-Qo5BHej>_#Z1Tq5 zS2h{{OxCPxqM-CAvGbgywngT?>6N#4A9?Zmr^gS4*ZEfy z9zT9O$L#I9cXb(Rdr1GV>z|)XNX+hPzP{x~;|E!3|F5w(f#$kxzemwTDv}UNGLIRO zF(ee3LMmj+5GgVg5)vXwA~J+RNTeiVQHsnJk}?mOG7~ABeb@6l|KB=it+UR%*0WYm z?^C|t&*#3cVPE^&`?Jw)XHM1cJiUY+frs&F*K*=it-Xc^hWx=tv#JGVx-KP;o?Z0X z@gROEXuD75$}ftg8Ai94yxFt!Sx#*0J++cj7}Z+ozf<4v;JT>s_Zd`t+}P(GkkdLk z8q!ugJh8+^Q6WJe9=$jiKI%NUpp~% z%Eso7{k~olh6f-r0nz=0c4wRWAkkA`3duvmtYUp`*ER+Q6_l<-Ttn+~G$k@RQLvJA zh+oen91Nwtx<<~?ket>*hD}42I|>f8u9}%+?b$Ln->Eh`e}!lENw(+V6PC)pZu{hq zE>0_atgnC0y=tQR{md z8uS|^<|Y{V_54Ck@mWph(_D{Of0Fj$)AxsXmZcvKMZZ~;(+Jxxo~VB z1A>K%Lf!{36a6tbN2~nc_*1K*bHCGgOoL4;zkZFweivjX1nf8KreiJ41?*e)f|H2( zvm9c6%Yt`~Mpac7j9AsF72bp zc$6V#a=LP8x4NXu+xJ@WClxq3DX|59HKqhY0kECDv959Lr|l=HBhH3+;71WjEs({d zlS>bPs10fS^AB@2FTN_iFslCR(NX&x{qwq;?s#xfa-6n$vm0cxcnO}`m{$Tv9;_CM z$$ndwr}VH-`^X)GVP4L|5vRBU7AP!}9diPB^=dPj&K%3GpM_($fIzIvMtsTLot>RU z7hH1Ne_(=!h;U2E*9Qd+{fYQspo@A4eVF@jZ}wdWf(jyHQ-`*K5Ef`FAtZ6`oJW>n zdsGF_2PwIEF>%YhOo2$dhE$8!{%y~;WBv*RH`iy(EAHs&pz57l4AYK^F?Xa|LnVF( ztA#xOQj{KZjryGdtZ7(QHgC%u^{|Vsi&HbnYIz3dLo5{vR!OsgR0+C?KNxp8?c`&< ze+MQZ3De*cVNXNs;Dw?F8UTj)5}13+96XnF@7{LTm5G(y-3CEkf?E$hn{*nSO|uPi z7ihm2LTMa#H@E7I|4KuWF0`(mU5${|qzs*vY6LQy5kM=^#R4#5Si3#xLw8VT8GrD0ioa>)>=)ra-4A&)~p^ z0UF&ax?9&9Q?;<}wZ?Ib53U-qj#1$DzFOf@oi~8hHX5J9U=eff{rC0W zPc^Oum5Ncscu(uOGR;l*=_?8F4Ag|4o}Nh6e}StvmN=rzhl+}edtqt`!5^@LWxY(k zK~fW|1hJB*)+Qc%!a$+bPE9VUrc4rpGOItH8WF5MxRViAw-QD5yYo{O% zYzt%<%5v3%B!P;L601WMV?L(r;2l}Lzs0G~j6=~_FVcf2ZmCN4unW#@2M?D9j7Mw0 znjnszq)KUp8$1zy#a^zYO(w$u7i_OX#T1K2!CMbUPc}8Bd*3&jp}D2I$tEYA@&(0~ zqk=hS^4|}sTNQFQTAZioyo3j( z`xOR#=79H78W-=5-#8G z5{V~*Ok}^(D|8)Pbj;*>MU7s+ve>P25K=euf3}KllK#mdB7NuFdj+N zldZ*0&HI?$9AsA%Z7=?fx3TIiB%{H4m%tDlYd1*)qoTg=FRWX=no$3UBF^VE>+-}z zZ=N$2S@qo5n_^o;bY2I=XNs;zh6-!q_Bu6ON{w3jA1*-Dct@DTFCR9D3Ifo?9WTd{ z6&Cq$?}&~kuF%bp zjgW!CnxC4VAH+)}D;-d9v+Il4ZLUakfw4fVjM1}8-COZ(5V4H=S}%<`DE2wNPqbGb zo0@7i5FEaBTjf%6q2Y?~JUcbz3*?Q%_q%!Yo}4>r3$`7)TYLN`~qx_mehjMOhu=%i0;X|0Dbfm|1g&(5DKwwJ-s>K5k|6QA6{L)^!N0dfcS_N8sf}LoY=8+SXVbaesTl>y#c!TRrn`nw5qlQ z+iFn|e^puvoS5!n}l0jQ@)XnPZJ?j)G(-PLl_VPMYe0gZ-qL;NM(0`=m zk@)*oW+=E|34{C4M5f+-6-GhBVBD>=SJBvz2Z&1eUkKOj#U7#WUq&e0@Cr6cQM<37 z>~ieBXu5je{VSh_Y;AV0E#x}Ypf;Yqzkq4}YixN} zsN~3lLp0Z42w#UdBPhyZgld7IRA z>;O8W)-nnTnr^we?u+1^1*)(X2Ue;(x6b!Z2(6ddz;4Jdlc9f}Pj;L9;lmD}Gb_F< z&JpoaBO{4m**s8Z$R+{tLn~;-ibxT5hDOMQJ-)-gM2bmC z729OTp)rBY<5P&$>iJFxoIpC``KOaGrZp~%VlyjV!K@n}XRtA_aIT*9GH#%PF8 z=hW2T4p6@&KVNk5`wqGXJ0^E6zcbhH+P`%@^O})24tsJ~`8O1fa0JfrU6-5|i1tuA zU!Jtc{d?J*?p@M)L*GZ;J5m)EgsP3=1gj@*a~!ifwqBmOGVMMsNb@*@)L5RF#>8cC z(91Ihs9=E+UDb*tv~U(Go-xTc$~+h67N1RNL*^bdV>IDf%8*)eea9wGBY++%P% zt^j5+)pMaab7r(zs;=uh#7sakL}EAqaS?Ppy*(Az-u_{&xO$b&gZ=ly6V>f{cLnui zGn1(^?lqrI*rTVMlpufqcQJ*Y-pb1`_Ejl!>pe1j-%>oae-*w)?FN<2V+OV7RC7rV zdo<$8&b_HWD#PfUtbB#4u>T^bZ^R#&qpoj0e%#E=9MjBgqS$>&iMWYju!YUaRtp=D zVz%SSoQM-<1OLB~|Fc~Tw`s6K#-WH^L>2?2>LX=7mx<6hpWUxn=s-a>3`P#U0|SAP zkp_uhoG>$k^p<@`u@nV_61q{45(Y9Bfb+%{-M@F0RMqxdN3CBoF1U4|*+KL4o|PdG zA(+SJX_rUZJZI)w*RN?kDkz%SQu^Vv*b#pIAN>KBs&5+#9lbt2P(wVw@y2&EZ{o5A z#|kw+NRz+1<_6!LcsX(U&#>!lbMBR~i`sSC zdQ5u$TTZV(ttX$s$x(c5-^w4kvAkLShPcoQ1GnsE`=l=Kw^ypSY>Cp1qNj~p$J*T3 z;;ZyobBiOpV{W6Ju40L|%z*2g&gvc4BsRg{j~?p~SetsEq*Ug{+-MCM5jcad#Y4Or zmHzQGx)#f~!J8S{9r2{LV8*@B=)25C;?>1x`S~{3BPN3(bg{WudcfMj-N9Zl=(ytu zVwrs(+#z%m6Ha4+sF&lEBXsvL_aml}I7Ix(05Nqm#D9s02S{@lK z(>r;1=H|z;vjq%ucQ&Y9?>_RJU*!7M6tu^X=s0ywRPtg-K@uOzG@aJ3<9M?lcm~5c zw%`;kdcd3sEj^dw-o*5R#A)H4ZIIpPR?yukph)BXY%wV?*6))}NJuu#@gW`V?0Zbv zX-xUq#uxQZq{>zc&+^4e6;F*_K!z|J?m7^cBrwazzt~l)(P5@P%O^Ed8CXj7e9?Nr zz4tN4h%S5axUlZ0(!`&m=@!#(R;&3_MW*u>w{B7XZ5qmJcJ+JO_w0BJd%KYQY$@13 ze=+`MWq0=4y|C?j)f{u5ScF3ugZR9D`E;CqY_ecu987-;?AbsYTU&5#@%e!# zTQ)g4O_X)9rKdvoWEBzWMS#VXR=B(v1q{TMYj1QRpRUJjKTYH9+sk&pKxB9#fBb9Q z-PDHmWjVe>tIZyIu1IDxg(*&jx1K)q(RYd0)xco-TU*avC8eLgX8LX^DJ9;#*^gbe z{7dPO#L`OOmDp!lkemB{Qn(p!W+x}-6Z8<3Tc+KSA$rdDht786ufPy*#zwh;m5+cf);_m!LgPA zUj<7Wo4|kMbKgru*+s449TxFWZuUWHSK@f=lfjRfhMBil%fw?8zucfUON8~ah{ z%Za@-06h2&9MWp$S&!w3)8>3F_OFbxTCk^OX>0qlKn`wovBg1WzsB34L;w<;`~3M^ zP?6Eg$tv$^<(mj%Uxwu)#9KIdcmlbX)}4ZWFNpH^0yyFnrQW}8g($bYM(v*xG`5+N=kcQ z7aWDHi}wkAVAc9JU zWLK{qNNign3{#jw5MesBEEy+DUKn1dOEnby=SgVf&#AgsjI-~dj z)+j}=QoUe)sYioBG~oRmNMIkrLK{FNvcQQxVlehHOa+PH6+R;5YiYsp{rdxW^Kx^O zac!}rdj|pr8p?0iL8+J>g|Ou5&|$)!Y%hGO)gG&j_fOCQ?8xnnyL_7)bbmT*0kF$o z*Z{H@(XfJo&`jN{_}JK|V9bNNWe@vA81h7|hk{b+shLBSwk1}t8X z-FrBg2yexsl8C@n!KWkM3MiQHMJ{2R6@GI~So67h#}iGNS*gc<(1>`-+pMwQ zonf>$f_xPv0?9#H5#JY_cn(Mz*Gfts#BNgNibp&a8&7fv((j7QcWfhym{_yrdQCrJ z10E*o4eY}?a zstg+5M92m=0Ba)7AZ}%D0Jx`tttX(XHs~&Z-f1eU23?--xQL$e3e}qREPH62;46WG ztPXT>yh>6;kSp#Pe(l;i;#Gqp4cwLQvC-Ed`*t37eE|G2jy>5y;Lw_#coV6>svx&9 z1uEEKF?5b^-xBR1sM%QLN|22&bcCq;8M+}maQ0gesJwVIkrZ}U5mQgu*&PJSF6>;Fp^FE)P;B%QbsO;3(tKBzO-nr@ zK#Bok80J+5Wym%_Z$!5a7YR$+e;U&y-~y2L0Fv>SJ~`x*)(S*v7B_zp%TkoF#1$U9 zp0eu$-&hQcgDwX)8S8Vch~-no(3QYZ4Aivo7RXY`YH=74qvhf8bQG?LQ?2k4B!616PJ2k-O6y8PQ}ZaS)DjpV$ZLafp@Cs!F_P89Hw_{3r|)Qi9J&wSa64OYK#GOk{|I{d9*H{OE{ zATDGmUGh9xZozgO8>T>tG{xS^AH`}NKKK;KFKmzNuw+M!=And7jB|I|abisj7%kbx z+`^}`r{;2$INl(gey+uS3}-5^j!1T@A(POH4qv^j3lXFbA0k2i;!TvAaAp;~v!9xC z!)XK~plF9~J5TAr5SaHLNwqn702^%)_#)U`*+_JFNZE+C@WurWOaQ1YKllcK%7$$e ziAFcwaA>!W(d0r?4SaO&t>mvw#VRey;}Bq_3Gxs^1i`Lr)K`%XdaRoeRXKuDqdjHl z7mkAFIW|U#jwbfG)k~5fC>tPl7xUCy&>;e4 z7}%DFZ18GU5!*G~kMC&Zqb-7cW&U}D z!65}_M-~$wGcz-WA>lm`T!v-jdr%o~_?Z74#XV|0^)Q@W_6<}r>VZ_wy!DF5_Ok85 zWDgMID55=x9cmJkrV{@wE#W$Cz)G-Kjnq}O%3q6%y+3}`0pc>UwVliA_TD%Rv@RY3p*;$NpB1chEnY;kEEhrDcgQ3#2I=+S_(Nh*y@ zysxPpMC%1oAAo!Btv*1c^@y7zG*}!yxNsoTi|lt3byQnhUdrke zR6cmT4#;<9H(YxDo}gCjDdX32udiXg2WXBIUghch-=wqFXJH4 z_iAxO=xTI4oXwyogGlg%yZZ!g@?o;fi`9pL%)-Oudzz3P58&0P!78Ci1FErk;ukp3 z70f_{gLuW0V1-O+^TM1n;7~d$6$W3 ziGI@m-WqTw2-Ov>p(Rd|r$g63gD!Q12<-mIP7(pTvCl|Sv!_*^8*CQxY6e0O-VGUK1Ijl$lpPE=6iy?b6O&;mN?eUbxQ)flLhy-Ptw5`NSL_PDJzj0h z;Lnp}j{XAHO1+pjTHynb6?a?}R}9=!x}0Dw1*^V#gz0J|CCn@!-w^>-4~;bZ$cy~P zb9WMm2B8iQOb4^$hTz7N%mxrU&!4Ee*78A zQ@eKU;@3~5MouH@%8>C->6$>G$^|rno~IDnSV&ugk-CbcFVPnWpzrl%k#bySc zjXvf?>EJ)HI3(yBtL16O=b)~o1D8YW3Gm;kQP{Q)R5g`;7AT8i10f3NC$T2NstJSx z_n_WaFPngz6fVPgFj&-Yu)M~OCr111c-(3!9}f-SiSI!?4#K)2Y~&*>Sc;G;BrsmY zZ`w|-HpSLHtmcR$5RC^DV)=TA4S^iY*~r^X=4dXdWI~MFO~m+6aS&B(L}X8^uCtDg zcVWXyx-Jl(hzuR3sCcJbTZwW!>cegF7lettHY5zFtlnZkeiO>ufUu-I+Hei~fW}BL zDTES5K;pTCtpx}-M5mu36r){sW(itaMC!i?)R|C3+B#{kuh{`#z5ykLNnFAs z�QF7N_nV86ULYY7n{(i)GurfI^lUIMiw77XeUkx5~QfMW}hc?D!9-5m3EF>#V*n1>P_uPFd!3y>N6esc^yMe_V9*D-E86ktf4gW;Kuf`}rpM(OZW%ug_ zYaobZcMi2J5q6`YmJpAJqc=J*OHjN?Aj82X!!wBMp=PO;6ep6Bv9M4hy6)&3wQw!* zhjCt#GS_oXe_sj2(d_B#BMPcKJJq?NNP=>QXbS_{-eCL_kC}`d5SWRx8^aH+S+oYt z+dd%G_!kuwtyUv9`y3<6OJs~#X{Q~iY4E&YdVEVkD$)bj5>)F!;y_VRaTOp`6Z$5N zO4PUwI8Xm=%HR`dDBmc@wIhz`c)qEKi)(k6AtnQ}a>433;aAXk>uVRlx|Xkl>V-{SNi#(O7i6+R$jl@6`h ztimFKtyGjU&gakfkaEheUmwna#IuuBxlo^AEBymv3r67L1XRa0<>`*KhORg{YZ%*p zK*LQFOd5*PZZcjnf0ckpH3L)+u<<*bLb%v^Cx-0D4@QKsfp7y^ekF}LluO_iC*0Q+ zK(9__@|bTXpeoa>H4#Nv-S@~STN91ziBgZJuWRX0L862qv<3rP6J|Mt36hzh2L2J} z>A#=<9r2rhAW+~8!g)n^%|@IBKr^^)$8ZEoCwN9gI|^82GHwFV6afd$%F>eajw*Eo z53yf|NTTG@L_kmw7PGmlYxYAClE`#*{o0Xl9ERl|&;)zX@2y=~f$;J{sQcKX{K1F0 zef74+UndQmVPXveMG2mrBpgmlU}TFWMe=hIpvf6Tj{ghFhfCL1BPZR(ok8D$ews-B zqNomtkB>KG;8cCM9_#@$^O=XY+C*am1rP)wYB)PP-zvAu!*((H6J?;GoxffYIbcYA z#39&19=Qbu=4hXOp(&hce)jN$J%TJ&F`_Xm(xehav{d=ifnSrw10sinX(EC5VQ3?x z>wfY^NC=4}L@f~ga8s$LE4~X^nbJ(t7Yw1_)pUFE!-o%AD<(v^Ur{j;1F1vZc_&aR zEKAWSVa5VAV{!^13V;r#+tS284Z6uh_7}mBoHH``P6@w4 z|7HQ2Mr(oBLG&k3Jte}gUk~*I?vIs&Lk()ajb_U@nZuALB)gwT5Dd<%44`JSO(KAt zS~Mg?5ryVDX&)kd@Ikx*7ZCdvbc6&{05n6UfJE8_?@Mfv1u(QEvI7BNi84D}FHkY2 zdrb9`>xr?Th8hH`hz4CBpurQLmQI!?q4a$UQV}3B7<5Q`{37)S0CbcA>PBnCbFrd_ z!F40(TVGyXmd6c`N(OH^fPMo0WQmYHbcW;(+7xwkGQ>0-(X$+Sk!S%yv^di&j_sB- z#5j~N*YJdiBs!XNbaexzINC)egMNu-GMO5F8GFftjg^+j=t>E-sjUtJdDV~R_}+fa-hI;P91 zdUGpyHK1|hG2XH0-${6J_$4`5o4M3kO8>x;AP~o#7_Dg~g(*Gp+!w9`88~oV9^l|G zFcK)|%_N3s|3q{Wpud2ZfV{K~Eeb{?w-8Nj%dbB}5E6q4lbQ zd&O^7Duc$N8{LEci2!=p>%dtcyJCSV8s=Vym0kQGj)GSfUQ%-D;f+<{H*XRpz~^Vb zMI#R%D&IJSrU5$rbiis5;%=6dl>ANF0q?zmMMJ^!4pERq3xgxOX2`bJ{lbNZ05gHL zLCVkKxwtbtX-QEGxf@b;qZ1?gy`XDk!2B1XuNO)E@8|Nz`X%C{CF|%Z>W>KCf@*dG_0^44(#a^UKOxFs5C0vaF^O*f&fAGbsBHm> z$b~mEfDjU`z|#Ybt{V=M75xMds-=bR?X3z z0te-q=GfB)pp1M>S%({)isMA|6nWjYs91;MyMl%M9!DhaEF0qY%&gWJ2KSaw&-8}; zIQA{Ox(6E3G!khrD1Zpi_?Wp3Hf%@XRt1SPBCh-JMi&N?aLU>bn2F98HdlyCJJ4VQ zx8OG|6odcKeax>fO(tH-Q0Gf|QVIj;TDzOzqC__mdAf(xCDML4hCRc>G|DL0kQEph zR~cYQAUgn2%kteJDkUyE9R68(UWLUbqED+cWznUOW)g zv-fdBNV3BBAVQ6NdtX_Q;Sg8+Jl~jc7=r zk|y}`tEHbNfW&#kVxfc%D473D^0lqlp9B5gq+1~eleLRDes14$zf zPC;;r+`UzedTKw!l7=9=MoMy&4;+S{aSJGkF$%N_`7A|ROY4ztdDiXVB60so7LFSrQNWa7R_8Tr8G#B0>h{yuGMICH~ z)J`rO()S1A$+~?z0lX0Mdhp+-{PDA8Y{iH?Mamklo5NJo|OGsDk&XTKP$|^4L$M@(6ERBEZ`8!56>o` z`Hdz4lO_mRu$h>cWWJx^gxVVp2XR~>YI~?QRFNKd#7J`xaA-FwPcm-9@;GrBK4 zFB50xCcwu;JfxStdZiAxfeZ)z)OFjclAB4g*20J~dfngq5RaJFO!lnjnAR_zEE!O6# z#5GF3-}&n|Zg68U9j^qojx|7l0fi@nE`zDoCjhoM&}a7?z6G&NqN#~9{{?d(NZ+M6 z_q-vSgSewj=r1vYPPL+r*!b+yy-8^(xPy19i;D8@9((;D-xb_bfJrMHFP+f1!^q-5 zrlAW+2F9KcH?;|-E25Kg`~+^a<= zo90g)5#!{itwq8E(Evf!Qc*SmL0mMcICh&rM+bEPWGq^uKJDk`9rLT z+%PZu&E$}u?J9EMgjc*AOIgpcY?eGDA{FzTLml_QVtU{FA((53=*APBrD#X zJ1Ni%643+PLEI$Y91i*20SgyV)}bMSmwP38Xvsgn@n!q)a!{3{bS@{^4m~Xe;B58n zCD@e}gWV<>t>Ln7SmgYUK{=qD&7GZ{sDchKOl}}g5L=U$_*e1};FWliVKTvMDIyw8 zD!o%AP9XxbEOfHKnBhZQ0VqI-y+40GMD0#yF0gt3J`&;r1`FXrBBc+a(15iL4!{*` zGCK%)qDC{YlVDP@Hq~*Y@jVOAJFK~gL5Oz4Qs?OTPA5T3eW5T~+1%XBA8vzok~C9f zSb=R11|H2)N%d{^G;{zaFfk?%3&q~R*K>;}#t^HBj5De=!g@diIza5C{)M^sW8j8h zxsUCBz>!zpo)B%cv&#aUhM{}XB0Sn*m=FSvb5r5@ zQ%Dc|0MHGoId5>&Kx(q<*N?nLBBhO2raM1@`R8>~?t|_SdSY>OmlE}VRsFyOdmi_-M%35q6Xpt z;fNeLL_G}D?#i9N1+E)!59xFNgHhhUUgQA2h@za(1z^R%b&}(FDjyBhZi%xYfR}>t$xNLIfHhRTzeBC@Ij7C(p9~t%_95QjM|**BsDcKjd8l0t3m;Ern*Za0vFh zz|A5ZJAw{o^f6gKccZ#RGIR!~0!20rEiIRTKnQ$K9`g2+FxJv?6!YpLews2M>u6jF z_Kxd)xKnQ*nDD}4i zYezRrsFw^0rs|5Kn^mYS#SmH`9Yw%bVsuJ@99)M7Ou!f-c!Nj6BPh5YWKiP!ot|zt zl7iGP>m}j@vu%8jgXq!7NSV;#@qO7L`iQKQcdkRu?3K3$W)!DCBvT@|!DpCU}aQ$ZD1OEa|syirJR7 zqFK%i3?a_%(xppmsqH9io4|a>p_hTDKwaJxF@YdlF);a2VQ8WUBjk+ozgUin^$mas zpv|P9y8u~VT^)MQiE($)Psie{-q+8BbicQQz8PXJ+%p06d*1xE8#{Wqf%;C}coDJ~ zY#=ES_A9AW0Nm+6KYfJkcHvzRy;QW}5;PI?ZEzK^A_fuoLYS$B>Sso+_I^!;TSENj zH%9xd1B8o+N^*ssp5D#zrKo$*nLS3#AyLW`(;-|2`~}DV717j|GN8JF@uJ4cGxtN8Dw_yFoFfjp7k*t8Z#+g^oRWC%Rnr^7p3jM-B5b%Zax?Y>8kx#Q8Fk ziX}B;GmZZtdf_vc7w@S3uo20)c&znVNsXCX#6ilr2h6KCZ$7@6jcR<203|-{ZM{!4 zoSYU6RyEeP;~W|dyo_oN42n+%Wg9u|Y#ZtlT-669yUZg#|FuR#0sy**Bj<-+7WD-& zGA75FOp9m7JAU7$?j{2{Sey)wWIa(Fy&N4CVlJGvkm9SHe1DsqbgY71*LVDq z4KI}os)~w6R;yyQ4WM~DMiR5q@M})NS ztA9Pb4~L4izP>9=Q|}%QD_;m#n5L#68IgP(JSO;~+(p=LyBm5L>=FXNBAX)q%?)Vv zD=aKc_%=#+d6S-5KveWQ%HZJS5i?(8xy6Y~CM$|50yV0tsUqMC+5x=+-;JdfRa5S_F@k3R8sv&P zsHXN2e#J~}(oa)U1JD2y12S-+-(tR<@%V8+wp*kh^@nMHk%ybqmfa%yhm#&W&V2Sv zBJt9=EcAuMW3`%FS|}~@jXlA8`}Sjx@uq9U0h_G8n0fRBx_(-{Pf>9PT4rn{9|toA zC%NU?{hs4bjz9HpHMPhR*s=9XGV2 z(RN5;2kJ7#R&sp4KE#j1%|8gApuDlRwl;NsT})3qOFgY^Yy6KuOr>UxH|EFdd3ezUr zhg-L9O?~)~_2>)=vFX?AIGi!J6K0}C-Uj~=L0$&~StJHXC3cUDc)d3F1>J|BrLYB) zer1TF?jG9STyQ?g_#VfeTG_(FB2RAz8{4IiK6>=_0vH?Uj`|=JWlt~NiHVtMke{HK ztPy%L2|ljX-uTQ%k9vy+6S-gpgIYDoogbL7EXWQJ$G(lsxi@~SS3fE)t_GZl<8bak zEN}N+SyEW&zKSqDbl_r$L6S3QI8FV|GNf=qnS-x$J36`&z#b6J-8fOWaTrO3>5(^t zZ-dD$Gf#rFJH`@RQ4k!sgD;?GYU-#pzZWl^>+ZFnAhT&#?%JXj*2MsH=rMeW9{zy5 zhXWoCh2haaquaqfDZHT5D-_k}se*6f@%8Ngic=9RcvjYFO7@OR^aZ)3=K{&-J>?g7agPYm*7L3=>y)9SlH_3z6v z&A=2Dj-42^v%OXj5eWmvwyj5jpFiV-7&fHKhI$nE_^#nLa@{3wq^3p_(_bwFoG{x7 z;_`}V^<}v0zV}1Yhao{;F&!#I$ooVn92%LQIU>9jT(VIY@zCVcVTzAC59Vl}4bKR?@ zX=Xo(GbW6+;!l_@qSU`wH70sT6rni35Jt|o6O;}a0R8$4BR;lz<$S-AYd1+7~utzmWq)v)k5^_b%Y(9 zo!+NaI|%EPZ)5M8HEXg30~%u`2LtF~8c1R@7`}=&6ubHPu?RLA^_vjK(eryxy<;Zm zWz~*LKW-=7Cq|;={s7l>X|stD2f1<_-FK*BMmj%&T(IeMPpSCO-k=$xCg0Sgp{>mW zK{|}s?t*b^i~ld`Ag&LrB%GZmJ;%zF9nWK z*7kFD)@_GJH|6r6@A9?s^71V^V!$24UVx^y_C{8ejszkWLrzXP4AL=FOY!IUUZN(a_c9+qbVHBVmGs?m^qUZKx}Owh$XFZtl0o z!A99VH0sD&2NrK7s;;MPj;OG}kJHf7+5j^-vWSI$0iD;~f460}ed44#;*7#vvw?c; z){K?=4#1Er>sE`zHVyY*S+~Oi!~FCgedqSZqd6ad3AeH% zd?F8!EI2SQa9o%PaV>2AGs_1lmE7mn1la_|L1Dh#72(IK;9F)^78m4Rk3zaA2ZgMQo-{6oca&4xPvg774Q&TN3FS*#*STah*vBZ}|5lCxLqo$oCaoBa=pn(7u)nP4XOFvdJ@+ga>OuTIhV!#LD#bQ8J^XJdcFizOvHN**~R+nv+x6Fkf zEI8YJsFJJ@14x;NWFAWjEi58!R1sL;1Nz7G43052kNFf7{sequ%^E~43peMPfDu_g zH0wy*;PNnF#)W7yHQ%;&7BPc?gF_c6{L^~dyr)>J)i*E*`~HL66wp~IfP`({vgMRv z-6<<8tUfe_M9(}bElogE8W|sd7=sDYz2b@tM4#`I@%r(FW`4r@DYO&CVo$fy8?h;g zC`mm&lc4h`LC4{qj*p9s&Xh`aWQ zjQqoy6ckKz8mjvZkyB7^ek|>D={#+*7swIivcbMCz+*TpXI}IV4zih##IK$0i$SnT zYHD@sE~iYf>O8)$tH6n4)3sHYbfcP9S7&W1U20c8;&T3Z(3Lha8a6g&#S6CbwdHL) zaUv=@rghz^tUIrGx4d$9@0Y*#mMX>XT=~Gp!_wt*#}w^0I;U3aukWXnRZgr{z1gC9 zvq{6gqTIQoqkWQ=x?D--E;1cnVBw$X+W0Y7jE4zlPx*4BYe|O=a&;^K2{iwT_aB(R z>K(KYB4;1BX8vKFtt3WDhyWAEj*gi0mOH{ijuiU{B!sSn&-n)2)w`$}(1Fa*R2lre zIjW-9hUr82c6oJtJ2Hy~ob%5hg%zA1%6}>@7VetzJPHQ7Pkw|a*6N+uHxY-ysM>ZlKu9pqbgN|vQl!@RQOryPG(K< zTkC&Nee$zrX4W4W$(6n5@pEXus@tBWb(`K|M+1gabbbh*#K{tv$fcTJud=eT2pJUh zG2gu{OA9-F8u!gle(PF((Guuf($N4$b7R@^+-^b!F*DnN2CNSuz4xa4czIRTK@R;R z)ClPjpV>adi+Xx`s&ARE^z-qSnV*@F=W7}D_opH#6Hb0yHrqijBuE9$glmC;J>%m! zksdSw&go4W`0^M+_Fmh__9Zg%0CQ*{&Tx<%`v%IjRM|GAqk`S*EVy!CJARjs3^}zi zDIb%J^Po5Z$#AXV?Z+EnHuJaw3QD5O`aFYswO~0sU!XTLFsMK_J@LYE8`vtzg@tzk z>JYXS(6@h@a24G-|+)UX84a$PsK^uV_uPh ztR`1<_Aj)Ih<2uIIqm`1LF$X=IX~L=t+lw@(dkgO;UYzj!*OSs3<@TSwsK|+@T*d} zy)KScP98}q94{WlT+-?10T|KNx{jPAT5gO>vv1zn44TRbi+{ZK7YO^#U&9g(fBC@AFqGod1}o|V6$?bWuF3b z0+6ip&_Ww|FADsgo~C|oT$Gh_Q{L||Pmm2os9&I{=#PRn>XC0T=L=BL zu)5457K`Xzd3qeA1URA96m~BC`P1NGJxP0nX19pQ0BB}*;Nm_u&R>si5-F%cTe{;d z03qErM)QBytX~EYoT9xSHxDXvag9M$N z!nffoOx-6VL)P$a5yj?RR?gay-(d%l#kMI4_LrsBO&53 zi8U88^TQ7xJ;3)jiR>D@U2ls5Wg@`2ljSShpo(;Sy@#hKL{Gw>JbCgIq$Z+Of$>~BekUt*GUPi4V#Ll3fwuxYMejfBJ?Qu9u2TV|BG92;=?z{dygGG|!I< zdw6*z(l@ZJi~)xLYH6F-#&QS?OXuqovobQU;kE(L2WSFx_%>vPGa-Tf#|3%RxcJ8S z7V$?L52&c9Ss}{a|VGY1-&SRt{9ixD=HTfGMa== zQoHxJv#4FB66!*6W`6;9X&hO{oZeK8kQaBfLF5)uWeIvhMtDLZW#9^6Hr&p4RE@CvN|&IHAKk5Uf)3hijP{5&zD-sHNV1=Hm9 zcPi<%y9cMzlXyW8xiKFT6H^7~8Xi)TFYcY=`Gd9zjDbq(P6)D~!zN4gSTj>oQStvi z$rOlIdvnB#N2#fAsfV;HykJ*{;9L3JjyEy{=qDTHOJs1YGs0}~?4FBCZGZZKh{)ad zdW947EH5u?#1mH{Os~_;LE7BBtBFVypn2}*G5nPw1b?=c>;|(Ti?{y)DV^)YR-X@iC3q9l-;ej zCG5?O((UEgN%OgIU25~c&wcoiz5TGjDlap$#E*R2N3uG!5B!?q?0_5ybxVGJKI+vc zCzPEMzGfE|vSe_Gyux=;=XY|9=3rzDC@b>~{o(7AqvC<@ynA_A*N{W*?aJ4&8=ulV zbS~3qJu|lN%Hw-d`jZuaYuUnG6r6;%fstKYpi@?h=E;-yHopP3j9Ks1&BbK-5qC;Z z@Hz@Rq%}EG*F*2Yeb>L3qUv)GjJgOMigV{gfBpJJ3K(BzM#fu7NxG)hxyNO93)6pR3WQ02nIMonZ4}g zEW@fF7Zjg<`*?hIu!^$#LTKO^6YcKOKN82xJO?x;j+vP?ppp0ma8fM3p07^2v+yk9 z2w2R-5(_B{7vBbqYBGu+%WrkycKw@?H%H7g4!h|a8d?WTSFZPJDq|Dsq5viSw2|yQ zyv#Arz+uMm2rDW1EpWhV|9;9a^LezBx~v~SqZ@87n(_2}aduZtW}N6 za4V*#r)9uYK|Ew+WYjx;{E&~2BFZHq#)8g?OE|XP7IQUH5s;B(Q&SU*U)GBk(buo9 zC#5TjCPa{^n3z>_21QbPBb*XkU)oy$&zjdTPwF^V_3qt3K<;OjN(4LbA{anGlN;^> zS<$n*^OEPI7tt}s70jbS`7^urEqgCWilv;C4>G+hg$JDdvGuoG2imk!RCSBF*Z<-n z!KSQ^d)aXKa8zAMb$#2%`}mX0Oz|Ckvj~P|XU4-70JlV)4Ru2>z0E8Oj?1_Bo7Qo;eK84WQGO6g%OPs)_GxoHLK^y3aVjqj4H{xq zj+z|+1QuCRukt^jCplx+pmEdp38I4su=o46iBZ{Zp7H^OSDnV)N#(mNbSqw1bc=RR z^(B@(E%I11$(XyEf7Qs%l9$6Vqf6x+SMvSV6t1~TS$MDi)x)GV+f`<3zfmD z#*u!<{N4n$Mt#gZTfIi1AoA?q&yUSI9RE;k{_n{#S)nH?V=2xy5&{qD56a>}I39On z6qJ6it&VCW>*Zk628-<pa(RkH52hNl2yX^MYf5h-YwZHWEOD`$Rb+DJt;0GCw zq>bGaO3_gt(gVZU+NX9Y*d5MHm^&{gAn$#gxgc1DR9V5#$z~$ohQgnR<=n>W!kLHW z)<{1tQ>Z>G#P*rf?x}BA&|Fa83#ZfX24|~+T)F?}x~+AaXbGhX9q?B;*7x=VonBX2 z@iWU48x%a+FE^;3lAK?6g(HKb<KCU;Tml=xg(`Gj|v5 zNcYLj)9>D!>{{U>@5~syZijs6l5Uz0eg4bxt$t@Jik}vDeqC$lLLq+z)W?$<|MLNu zOx;_KvcLG&=0Ta5wSBJ1=jB>e%CRe(Hm)SD^=_h1Xf)IvNmph&a9PPkx_d`=ebBd( z7tdVtcF0&2NMid5f05n(loYfm!G}UWg5vz(&70#15wCM;#k5bzcgF?|#K<&u|J=FU zR95td3q-oN@}vptrrtXNpsY9l{??m9gqX~93I-LZQW zufEco1qusI{2ATj*{0$a6j=lm?XQCZ-ml4&Z;5ha4mdrVJaIoZd{?9A;yzkFY zim7{tyQ$OOJoRgR{=e3;oZY)c$%UIPb6fF$=gV}Lxlz6CL&bUa)-cw0{E;Ejg_8!Z zCp8nArDMuIT3hvpNl!O7XWp&9z5$)ecY@w-j$o>{mYC{q((UUTt!(u`{vZ;w+#Tg`d25obCrC~hL84#=0j@SLK3wi`0h=?1$poTrn~Q5B)gxOU%G7$`JcanbW(7YP6Sz&NVY_an$_Lyw=R8bG@g;w zr?YdBL7rFMFWgVfNNC;2NZ~HIxMNNwVHkO9%k%!1gj{|CnL@S5L;gXPioPL8&HVB^ zC0N_my$dvP+*?U;?88sN{tFhgb4Mq1PnsSoXq%jxw18FwI8fcg!||9>a!ec)DBR!#6@=oBU?P?nJ0L=iUWX+0N-J5o|^1;*QcIE%gQ z*~t`4QEL3<())E6R#lg)=f!H{R5 zY*UU4r7dCKR5NygqSR}4r-EEuNJIoNgGbddH^uZX_ieWSg(pi%Nr_IEpeK|lB2Xz3 zyaZkDtx{hlb7m%eL*wh&oYW@Tyv4@CZc?#@>_fH%O&vYSCo7nLRJOhNCjB^z?ekXU zlNDWQwvuT~v zY-D#915;eS`WN!-n@Cl8 Date: Fri, 4 Mar 2011 15:23:04 -0400 Subject: [PATCH 0970/8313] releasing version 0.22 --- debian/changelog | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/debian/changelog b/debian/changelog index 79be983679..56f3bbdaae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,5 @@ -git-annex (0.22) UNRELEASED; urgency=low +git-annex (0.22) unstable; urgency=low - * Fix test suite to reap zombies. - (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang - for his help eliminating the infestation... for now.) - * Make test suite not rely on a working cp -pr. - (The Unix wars are still ON!) * Git annexes can now be attached to bare git repositories. (Both the local and remote host must have this version of git-annex installed for it to work.) @@ -16,6 +11,11 @@ git-annex (0.22) UNRELEASED; urgency=low (Supported on systems where corresponding shaNsum commands are available.) * describe: New subcommand that can set or change the description of a repository. + * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) * Look for dir.git directories the same as git does. * Support remote urls specified as relative paths. * Support non-ssh remote paths that contain tilde expansions. @@ -23,7 +23,7 @@ git-annex (0.22) UNRELEASED; urgency=low * Bugfix: When fsck detected and moved away corrupt file content, it did not update the location log. - -- Joey Hess Sun, 13 Feb 2011 00:48:02 -0400 + -- Joey Hess Fri, 04 Mar 2011 15:10:57 -0400 git-annex (0.21) unstable; urgency=low From b4bc6b6385f503c16f9472a6fbec8a98afb74c83 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 4 Mar 2011 15:23:51 -0400 Subject: [PATCH 0971/8313] add news item for git-annex 0.22 --- doc/news/version_0.17.mdwn | 7 ------- doc/news/version_0.22.mdwn | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 doc/news/version_0.17.mdwn create mode 100644 doc/news/version_0.22.mdwn diff --git a/doc/news/version_0.17.mdwn b/doc/news/version_0.17.mdwn deleted file mode 100644 index ddb59046eb..0000000000 --- a/doc/news/version_0.17.mdwn +++ /dev/null @@ -1,7 +0,0 @@ -git-annex 0.17 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * unannex: Now skips files whose content is not present, rather than - it being an error. - * New migrate subcommand can be used to switch files to using a different - backend, safely and with no duplication of content. - * bugfix: Fix crash caused by empty key name. (Thanks Henrik for reporting.)"""]] \ No newline at end of file diff --git a/doc/news/version_0.22.mdwn b/doc/news/version_0.22.mdwn new file mode 100644 index 0000000000..4848e861e1 --- /dev/null +++ b/doc/news/version_0.22.mdwn @@ -0,0 +1,24 @@ +git-annex 0.22 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Git annexes can now be attached to bare git repositories. + (Both the local and remote host must have this version of git-annex + installed for it to work.) + * Support filenames that start with a dash; when such a file is passed + to a utility it will be escaped to avoid it being interpreted as an + option. (I went a little overboard and got the type checker involved + in this, so such files are rather comprehensively supported now.) + * New backends: SHA512 SHA384 SHA256 SHA224 + (Supported on systems where corresponding shaNsum commands are available.) + * describe: New subcommand that can set or change the description of + a repository. + * Fix test suite to reap zombies. + (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang + for his help eliminating the infestation... for now.) + * Make test suite not rely on a working cp -pr. + (The Unix wars are still ON!) + * Look for dir.git directories the same as git does. + * Support remote urls specified as relative paths. + * Support non-ssh remote paths that contain tilde expansions. + * fsck: Check for and repair location log damage. + * Bugfix: When fsck detected and moved away corrupt file content, it did + not update the location log."""]] \ No newline at end of file From c4dd0e377301b8b2e1f7637bbd68dcad860a0646 Mon Sep 17 00:00:00 2001 From: "http://batchyx.myopenid.com/" Date: Sat, 5 Mar 2011 07:30:19 +0000 Subject: [PATCH 0972/8313] initial bug creation. --- ...t_choke_when_remote_is_an_ssh_url_with_a_port.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn new file mode 100644 index 0000000000..031956b9a4 --- /dev/null +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -0,0 +1,11 @@ +when i want to + + git annex get file + +on repo ssh://host-without-port/annex, it works, but if i want to get a file from ssh://host:5122/annex, it tries to run command +ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port. +this is confusing because git can handle this url correctly, and will happily clone/push/pull to/from these url. + +temporary workaround is to use remote.name.annex-ssh-options, but we need to workaround when doing get and remove the workaround when pushing/cloning. + +if i had more time, i would have learned haskell and provided a patch ;) From 5c4f90b2d0188abf2aa40e1e5f6d3ecc41e5aa5e Mon Sep 17 00:00:00 2001 From: "http://batchyx.myopenid.com/" Date: Sat, 5 Mar 2011 07:33:05 +0000 Subject: [PATCH 0973/8313] clarif --- ...annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn index 031956b9a4..0746417fa2 100644 --- a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -3,9 +3,9 @@ when i want to git annex get file on repo ssh://host-without-port/annex, it works, but if i want to get a file from ssh://host:5122/annex, it tries to run command -ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port. +ssh ["host:5122", "git-annex-shell 'configlist' '/annex/file'"] and fails. ssh needs the -p option to set the default port, it doesn't support host:port notation. this is confusing because git can handle this url correctly, and will happily clone/push/pull to/from these url. -temporary workaround is to use remote.name.annex-ssh-options, but we need to workaround when doing get and remove the workaround when pushing/cloning. +temporary workaround is to use ssh://host/annex as url and define remote.name.annex-ssh-options to "-p 5122", but we need to use this workaround when doing annex get and undo the workaround when pushing/cloning. if i had more time, i would have learned haskell and provided a patch ;) From acde7a1736fdee58be0af0773da6e2d9e0c2d220 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:13:16 -0400 Subject: [PATCH 0974/8313] improve GitRepos functions for pulling apart URL to repo --- Command/Map.hs | 4 ++-- GitRepo.hs | 48 +++++++++++++++++++++++++++++++++--------------- Remotes.hs | 2 +- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/Command/Map.hs b/Command/Map.hs index 4d0f900038..b3005e482d 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -161,7 +161,7 @@ absRepo reference r {- Checks if two repos are the same. -} same :: Git.Repo -> Git.Repo -> Bool same a b - | both Git.repoIsSsh = matching Git.urlHostFull && matching Git.workTree + | both Git.repoIsSsh = matching Git.urlAuthority && matching Git.workTree | both Git.repoIsUrl && neither Git.repoIsSsh = matching show | neither Git.repoIsSsh = matching Git.workTree | otherwise = False @@ -210,7 +210,7 @@ tryScan r "git config --list" liftIO $ pipedconfig "ssh" $ map Param $ words sshoptions ++ - [Git.urlHostFull r, sshcmd] + [Git.urlAuthority r, sshcmd] -- First, try sshing and running git config manually, -- only fall back to git-annex-shell configlist if that diff --git a/GitRepo.hs b/GitRepo.hs index ef8ad25baa..a62d765961 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -24,7 +24,9 @@ module GitRepo ( relative, urlPath, urlHost, - urlHostFull, + urlPort, + urlHostUser, + urlAuthority, urlScheme, configGet, configMap, @@ -131,7 +133,7 @@ localToUrl reference r where absurl = urlScheme reference ++ "//" ++ - urlHostFull reference ++ + urlAuthority reference ++ workTree r {- User-visible description of a git repo. -} @@ -235,29 +237,45 @@ relative repo@(Repo { location = Dir d }) file = do Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo relative repo _ = assertLocal repo $ error "internal" +{- Path of an URL repo. -} +urlPath :: Repo -> String +urlPath Repo { location = Url u } = uriPath u +urlPath repo = assertUrl repo $ error "internal" + {- Scheme of an URL repo. -} urlScheme :: Repo -> String urlScheme Repo { location = Url u } = uriScheme u urlScheme repo = assertUrl repo $ error "internal" -{- Hostname of an URL repo. (May include a username and/or port too.) -} +{- Hostname of an URL repo. -} urlHost :: Repo -> String -urlHost Repo { location = Url u } = uriRegName a +urlHost = urlAuthPart uriRegName + +{- Port of an URL repo, if it has a nonstandard one. -} +urlPort :: Repo -> Maybe Integer +urlPort r = + case urlAuthPart uriPort r of + ":" -> Nothing + (':':p) -> Just (read p) + _ -> Nothing + +{- Hostname of an URL repo, including any username (ie, "user@host") -} +urlHostUser :: Repo -> String +urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName r + +{- The full authority portion an URL repo. (ie, "user@host:port") -} +urlAuthority :: Repo -> String +urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a where a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) -urlHost repo = assertUrl repo $ error "internal" +urlAuthority repo = assertUrl repo $ error "internal" -{- Full hostname of an URL repo. (May include a username and/or port too.) -} -urlHostFull :: Repo -> String -urlHostFull Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a +{- Applies a function to extract part of the uriAuthority of an URL repo. -} +urlAuthPart :: (URIAuth -> a) -> Repo -> a +urlAuthPart a Repo { location = Url u } = a auth where - a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) -urlHostFull repo = assertUrl repo $ error "internal" - -{- Path of an URL repo. -} -urlPath :: Repo -> String -urlPath Repo { location = Url u } = uriPath u -urlPath repo = assertUrl repo $ error "internal" + auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) +urlAuthPart _ repo = assertUrl repo $ error "internal" {- Constructs a git command line operating on the specified repo. -} gitCommandLine :: Repo -> [CommandParam] -> [CommandParam] diff --git a/Remotes.hs b/Remotes.hs index a7d6be67df..aeaa5874f3 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -318,7 +318,7 @@ git_annex_shell r command params | Git.repoIsSsh r = do sshoptions <- repoConfig r "ssh-options" "" return $ Just ("ssh", map Param (words sshoptions) ++ - [Param (Git.urlHostFull r), Param sshcmd]) + [Param (Git.urlAuthority r), Param sshcmd]) | otherwise = return Nothing where dir = Git.workTree r From aad1372880ba32f1161a0d05422008cba38bb412 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:31:46 -0400 Subject: [PATCH 0975/8313] move repoConfig out of Remotes --- Annex.hs | 15 ++++++++++++++- Command/Map.hs | 2 +- Remotes.hs | 25 ++++++------------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Annex.hs b/Annex.hs index 62e7e023d5..dd3362b29d 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,10 +16,12 @@ module Annex ( gitRepo, queue, queueRun, - setConfig + setConfig, + repoConfig ) where import Control.Monad.State +import Data.Maybe import qualified GitRepo as Git import qualified GitQueue @@ -115,3 +117,14 @@ setConfig k value = do -- re-read git config and update the repo's state g' <- liftIO $ Git.configRead g Annex.changeState $ \s -> s { Annex.repo = g' } + +{- Looks up a per-remote config option in git config. + - Failing that, tries looking for a global config option. -} +repoConfig :: Git.Repo -> String -> String -> Annex String +repoConfig r key def = do + g <- Annex.gitRepo + let def' = Git.configGet g global def + return $ Git.configGet g local def' + where + local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key + global = "annex." ++ key diff --git a/Command/Map.hs b/Command/Map.hs index b3005e482d..fbc48392ae 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -204,7 +204,7 @@ tryScan r configlist = Remotes.onRemote r (pipedconfig, Nothing) "configlist" [] manualconfiglist = do - sshoptions <- Remotes.repoConfig r "ssh-options" "" + sshoptions <- Annex.repoConfig r "ssh-options" "" let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ "git config --list" diff --git a/Remotes.hs b/Remotes.hs index aeaa5874f3..faee8ace5c 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -15,8 +15,7 @@ module Remotes ( byName, copyFromRemote, copyToRemote, - onRemote, - repoConfig + onRemote ) where import Control.Exception.Extensible @@ -26,7 +25,6 @@ import Data.String.Utils import System.Cmd.Utils import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) -import Data.Maybe import Types import qualified GitRepo as Git @@ -182,7 +180,7 @@ reposByCost l = do -} repoCost :: Git.Repo -> Annex Int repoCost r = do - cost <- repoConfig r "cost" "" + cost <- Annex.repoConfig r "cost" "" if not $ null cost then return $ read cost else if Git.repoIsUrl r @@ -194,7 +192,7 @@ repoCost r = do - annex-ignore. -} repoNotIgnored :: Git.Repo -> Annex Bool repoNotIgnored r = do - ignored <- repoConfig r "ignore" "false" + ignored <- Annex.repoConfig r "ignore" "false" to <- match Annex.toremote from <- match Annex.fromremote if to || from @@ -282,7 +280,7 @@ rsyncParams r sending key file = do ] -- Convert the ssh command into rsync command line. let eparam = rsyncShell (Param shellcmd:shellparams) - o <- repoConfig r "rsync-options" "" + o <- Annex.repoConfig r "rsync-options" "" let base = options ++ map Param (words o) ++ eparam if sending then return $ base ++ [dummy, File file] @@ -316,9 +314,9 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat git_annex_shell r command params | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do - sshoptions <- repoConfig r "ssh-options" "" + sshoptions <- Annex.repoConfig r "ssh-options" "" return $ Just ("ssh", map Param (words sshoptions) ++ - [Param (Git.urlAuthority r), Param sshcmd]) + [Param (Git.urlHostUser r), Param sshcmd]) | otherwise = return Nothing where dir = Git.workTree r @@ -326,14 +324,3 @@ git_annex_shell r command params shellopts = (Param command):(File dir):params sshcmd = shellcmd ++ " " ++ unwords (map shellEscape $ toCommand shellopts) - -{- Looks up a per-remote config option in git config. - - Failing that, tries looking for a global config option. -} -repoConfig :: Git.Repo -> String -> String -> Annex String -repoConfig r key def = do - g <- Annex.gitRepo - let def' = Git.configGet g global def - return $ Git.configGet g local def' - where - local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key - global = "annex." ++ key From 6c1607ce66fb456880495d9026fa368ad48eda3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:47:00 -0400 Subject: [PATCH 0976/8313] Support ssh remotes with a port specified. --- Command/Map.hs | 7 +++-- Remotes.hs | 6 ++--- Ssh.hs | 26 +++++++++++++++++++ debian/changelog | 6 +++++ ...when_remote_is_an_ssh_url_with_a_port.mdwn | 2 ++ 5 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 Ssh.hs diff --git a/Command/Map.hs b/Command/Map.hs index fbc48392ae..6c3e0b3df5 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -22,6 +22,7 @@ import Types import Utility import UUID import Trust +import Ssh import qualified Dot -- a link from the first repository to the second (its remote) @@ -204,13 +205,11 @@ tryScan r configlist = Remotes.onRemote r (pipedconfig, Nothing) "configlist" [] manualconfiglist = do - sshoptions <- Annex.repoConfig r "ssh-options" "" let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ "git config --list" - liftIO $ pipedconfig "ssh" $ map Param $ - words sshoptions ++ - [Git.urlAuthority r, sshcmd] + sshparams <- sshToRepo r [Param sshcmd] + liftIO $ pipedconfig "ssh" sshparams -- First, try sshing and running git config manually, -- only fall back to git-annex-shell configlist if that diff --git a/Remotes.hs b/Remotes.hs index faee8ace5c..3c9db314c8 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -38,6 +38,7 @@ import qualified Content import Messages import CopyFile import RsyncFile +import Ssh {- Human visible list of remotes. -} list :: [Git.Repo] -> String @@ -314,9 +315,8 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat git_annex_shell r command params | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) | Git.repoIsSsh r = do - sshoptions <- Annex.repoConfig r "ssh-options" "" - return $ Just ("ssh", map Param (words sshoptions) ++ - [Param (Git.urlHostUser r), Param sshcmd]) + sshparams <- sshToRepo r [Param sshcmd] + return $ Just ("ssh", sshparams) | otherwise = return Nothing where dir = Git.workTree r diff --git a/Ssh.hs b/Ssh.hs new file mode 100644 index 0000000000..04cd9bec83 --- /dev/null +++ b/Ssh.hs @@ -0,0 +1,26 @@ +{- git-annex repository access with ssh + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Ssh where + +import qualified Annex +import qualified GitRepo as Git +import Utility +import Types + +{- Generates parameters to ssh to a repository's host and run a command. + - Caller is responsible for doing any neccessary shellEscaping of the + - passed command. -} +sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam] +sshToRepo repo sshcmd = do + s <- Annex.repoConfig repo "ssh-options" "" + let sshoptions = map Param (words s) + let sshport = case Git.urlPort repo of + Nothing -> [] + Just p -> [Param "-p", Param (show p)] + let sshhost = Param $ Git.urlHostUser repo + return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd diff --git a/debian/changelog b/debian/changelog index 56f3bbdaae..6d77caf4cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.23) UNRELEASED; urgency=low + + * Support ssh remotes with a port specified. + + -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 + git-annex (0.22) unstable; urgency=low * Git annexes can now be attached to bare git repositories. diff --git a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn index 0746417fa2..92cc9170f9 100644 --- a/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn +++ b/doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn @@ -9,3 +9,5 @@ this is confusing because git can handle this url correctly, and will happily cl temporary workaround is to use ssh://host/annex as url and define remote.name.annex-ssh-options to "-p 5122", but we need to use this workaround when doing annex get and undo the workaround when pushing/cloning. if i had more time, i would have learned haskell and provided a patch ;) + +> Fixed in git! --[[Joey]] [[done]] From 598027d82489aded053154f7d8f7b5dcf4d79ebb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 15:58:57 -0400 Subject: [PATCH 0977/8313] update --- doc/future_proofing.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn index 4d4939b5a9..d9a36ff733 100644 --- a/doc/future_proofing.mdwn +++ b/doc/future_proofing.mdwn @@ -11,6 +11,9 @@ problem: files from the repository; they are there on disk in the usual way, with just some symlinks pointing at the annexed file contents. Neither git-annex nor git is needed to get at the file contents. + + (Also, git-annex provides an "uninit" command that moves everything out + of the annex, if you should ever want to stop using it.) * What file formats are used? Will they still be readable? To deal with this, it's best to stick to plain text files, and the most common From 41d5c4acf6810966bdef25e2fc7a4cfb78b6f972 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:05:19 -0400 Subject: [PATCH 0978/8313] shorten a help string to avoid column getting too wide --- Command.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Command.hs b/Command.hs index 09adc09491..eba7f2cef5 100644 --- a/Command.hs +++ b/Command.hs @@ -216,7 +216,7 @@ paramPath = "PATH" paramKey :: String paramKey = "KEY" paramDesc :: String -paramDesc = "DESCRIPTION" +paramDesc = "DESC" paramNumber :: String paramNumber = "NUMBER" paramRemote :: String From 0de3005c648400e67ce4bfe88ac7999e56e3b56e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:23:55 -0400 Subject: [PATCH 0979/8313] whereis: New subcommand to show where a file's content has gotten to. --- Command/Whereis.hs | 41 +++++++++++++++++++++++++++++++++++++++++ GitAnnex.hs | 2 ++ debian/changelog | 1 + doc/git-annex.mdwn | 5 +++++ 4 files changed, 49 insertions(+) create mode 100644 Command/Whereis.hs diff --git a/Command/Whereis.hs b/Command/Whereis.hs new file mode 100644 index 0000000000..de51923855 --- /dev/null +++ b/Command/Whereis.hs @@ -0,0 +1,41 @@ +{- git-annex command + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Whereis where + +import Control.Monad.State (liftIO) + +import qualified Annex +import LocationLog +import Command +import Content +import Messages +import UUID +import Types + +command :: [Command] +command = [Command "whereis" (paramOptional $ paramRepeating paramPath) seek + "lists repositories that have file content"] + +seek :: [CommandSeek] +seek = [withFilesInGit start] + +start :: CommandStartString +start file = isAnnexed file $ \(key, _) -> do + showStart "whereis" file + return $ Just $ perform key + +perform :: Key -> CommandPerform +perform key = do + g <- Annex.gitRepo + uuids <- liftIO $ keyLocations g key + pp <- prettyPrintUUIDs uuids + showLongNote $ pp + showProgress + if null $ uuids + then return Nothing + else return $ Just $ return True diff --git a/GitAnnex.hs b/GitAnnex.hs index b26714a598..da91f6e74e 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -34,6 +34,7 @@ import qualified Command.Unlock import qualified Command.Lock import qualified Command.PreCommit import qualified Command.Find +import qualified Command.Whereis import qualified Command.Migrate import qualified Command.Uninit import qualified Command.Trust @@ -66,6 +67,7 @@ cmds = concat , Command.Unused.command , Command.DropUnused.command , Command.Find.command + , Command.Whereis.command , Command.Migrate.command , Command.Map.command ] diff --git a/debian/changelog b/debian/changelog index 6d77caf4cb..e8b094607c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.23) UNRELEASED; urgency=low * Support ssh remotes with a port specified. + * whereis: New subcommand to show where a file's content has gotten to. -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 76f1a577b9..4998a64911 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -154,6 +154,11 @@ Many git-annex commands will stage changes for later `git commit` by you. With no parameters, defaults to finding all files in the current directory and its subdirectories. +* whereis [path ...] + + Displays a list of repositories known to contain the content of the + specified file or files. + * migrate [path ...] Changes the specified annexed files to store their content in the From 80fdfdb72bde67977da950f70a7a058f53322ba1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:33:57 -0400 Subject: [PATCH 0980/8313] note current repo when prettifying uuis list --- UUID.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/UUID.hs b/UUID.hs index 67cba30313..239d373f14 100644 --- a/UUID.hs +++ b/UUID.hs @@ -106,13 +106,17 @@ reposWithoutUUID repos uuids = filterM unmatch repos {- Pretty-prints a list of UUIDs -} prettyPrintUUIDs :: [UUID] -> Annex String prettyPrintUUIDs uuids = do + g <- Annex.gitRepo + here <- getUUID g m <- uuidMap - return $ unwords $ map (\u -> "\t" ++ prettify m u ++ "\n") uuids + return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids where - prettify m u = - if not $ null $ findlog m u - then u ++ " -- " ++ findlog m u - else u + prettify m u here = base ++ ishere + where + base = if not $ null $ findlog m u + then u ++ " -- " ++ findlog m u + else u + ishere = if here == u then " <-- here" else "" findlog m u = M.findWithDefault "" u m {- Records a description for a uuid in the uuidLog. -} From ef92bd2b0bc2a85f42594e92d295230421186b72 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 5 Mar 2011 17:41:36 -0400 Subject: [PATCH 0981/8313] add copy count --- Command/Whereis.hs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Command/Whereis.hs b/Command/Whereis.hs index de51923855..5b0bcbbd26 100644 --- a/Command/Whereis.hs +++ b/Command/Whereis.hs @@ -12,7 +12,6 @@ import Control.Monad.State (liftIO) import qualified Annex import LocationLog import Command -import Content import Messages import UUID import Types @@ -33,9 +32,15 @@ perform :: Key -> CommandPerform perform key = do g <- Annex.gitRepo uuids <- liftIO $ keyLocations g key - pp <- prettyPrintUUIDs uuids - showLongNote $ pp - showProgress + let num = length uuids + showNote $ show num ++ " " ++ copiesplural num if null $ uuids then return Nothing - else return $ Just $ return True + else do + pp <- prettyPrintUUIDs uuids + showLongNote $ pp + showProgress + return $ Just $ return True + where + copiesplural 1 = "copy" + copiesplural _ = "copies" From e082bc9aab5dc7424f2a8e79397681af96603555 Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 02:34:51 +0000 Subject: [PATCH 0982/8313] --- doc/forum/rsync_over_ssh__63__.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/rsync_over_ssh__63__.mdwn diff --git a/doc/forum/rsync_over_ssh__63__.mdwn b/doc/forum/rsync_over_ssh__63__.mdwn new file mode 100644 index 0000000000..dab8e0fe8d --- /dev/null +++ b/doc/forum/rsync_over_ssh__63__.mdwn @@ -0,0 +1 @@ +[Walkthrough](http://git-annex.branchable.com/walkthrough/using_ssh_remotes/) says that when using ssh remotes rsync is used for transfering files. Is rsync used via ssh or unsecure? From cf25bb296459549843eaae1759dce5126c07df0c Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 02:36:05 +0000 Subject: [PATCH 0983/8313] + signature --- doc/forum/rsync_over_ssh__63__.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/forum/rsync_over_ssh__63__.mdwn b/doc/forum/rsync_over_ssh__63__.mdwn index dab8e0fe8d..9c0c9add63 100644 --- a/doc/forum/rsync_over_ssh__63__.mdwn +++ b/doc/forum/rsync_over_ssh__63__.mdwn @@ -1 +1,2 @@ [Walkthrough](http://git-annex.branchable.com/walkthrough/using_ssh_remotes/) says that when using ssh remotes rsync is used for transfering files. Is rsync used via ssh or unsecure? +-- Michael K. From 8617847ecaf924ab00f53e3e7d2be6badcdf1067 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 6 Mar 2011 15:59:37 +0000 Subject: [PATCH 0984/8313] Added a comment --- .../comment_1_ee21f32e90303e20339e0a568321bbbe._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment diff --git a/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment b/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment new file mode 100644 index 0000000000..2b9fc9552d --- /dev/null +++ b/doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-06T15:59:37Z" + content=""" +Everything is done over ssh unless both repos are on the same system (or unless you NFS mount a repo) +"""]] From 9c98901f1622e9d70b5279d7e886d53d59b6220b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 6 Mar 2011 12:22:56 -0400 Subject: [PATCH 0985/8313] split out summary and inline raw --- doc/index.mdwn | 13 +------------ doc/summary.mdwn | 12 ++++++++++++ 2 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 doc/summary.mdwn diff --git a/doc/index.mdwn b/doc/index.mdwn index 0838505e84..47682349ff 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -1,15 +1,4 @@ -git-annex allows managing files with git, without checking the file -contents into git. While that may seem paradoxical, it is useful when -dealing with files larger than git can currently easily handle, whether due -to limitations in memory, checksumming time, or disk space. - -Even without file content tracking, being able to manage files with git, -move files around and delete files with versioned directory trees, and use -branches and distributed clones, are all very handy reasons to use git. And -annexed files can co-exist in the same git repository with regularly -versioned files, which is convenient for maintaining documents, Makefiles, -etc that are associated with annexed files but that benefit from full -revision control. +[[!inline raw=yes pages="summary"]] To get a feel for it, see the [[walkthrough]]. diff --git a/doc/summary.mdwn b/doc/summary.mdwn new file mode 100644 index 0000000000..458eaab56d --- /dev/null +++ b/doc/summary.mdwn @@ -0,0 +1,12 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +Even without file content tracking, being able to manage files with git, +move files around and delete files with versioned directory trees, and use +branches and distributed clones, are all very handy reasons to use git. And +annexed files can co-exist in the same git repository with regularly +versioned files, which is convenient for maintaining documents, Makefiles, +etc that are associated with annexed files but that benefit from full +revision control. From 232553af01f4194c9e0e2f3f603740aa1e1c16cc Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 16:32:16 +0000 Subject: [PATCH 0986/8313] be more clear about the rsync usage --- doc/walkthrough/using_ssh_remotes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn index 831746ac0e..6af9e1f477 100644 --- a/doc/walkthrough/using_ssh_remotes.mdwn +++ b/doc/walkthrough/using_ssh_remotes.mdwn @@ -9,7 +9,7 @@ to clone the laptop's annex to it: # cd ~/annex # git annex init "my desktop" -Now you can get files and they will be transferred (using `rsync`): +Now you can get files and they will be transferred (using `rsync` via `ssh`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) From b945beede9f3e88efb55283da19ec8fb40117ff2 Mon Sep 17 00:00:00 2001 From: "http://m-f-k.myopenid.com/" Date: Sun, 6 Mar 2011 16:33:31 +0000 Subject: [PATCH 0987/8313] Added a comment --- .../comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment diff --git a/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment b/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment new file mode 100644 index 0000000000..49003937b6 --- /dev/null +++ b/doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://m-f-k.myopenid.com/" + ip="92.194.43.135" + subject="comment 2" + date="2011-03-06T16:33:19Z" + content=""" +Great! This was the only thing about git-annex which could have kept me from using it. --Michael +"""]] From 2deb35e4b03f21d33a56b437bdcb7715f369f50b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA" Date: Mon, 7 Mar 2011 17:21:54 +0000 Subject: [PATCH 0988/8313] --- .../Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn new file mode 100644 index 0000000000..f458ba72e0 --- /dev/null +++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn @@ -0,0 +1,3 @@ +FAT32 does not support symlinks, so I wonder if there's going to be a problem with that. + +Generally speaking, I am wondering about portability of git annex on windows and on android... From ea4932d2bfe2c99bd120fd810a15e6c3952ceb6b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 7 Mar 2011 19:13:14 +0000 Subject: [PATCH 0989/8313] Added a comment --- .../comment_1_426482e6eb3a27687a48f24f6ef2332f._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment new file mode 100644 index 0000000000..7b9935f7cc --- /dev/null +++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-07T19:13:14Z" + content=""" +See [[fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames. +"""]] From cd02f05eb11fd6a8eff345c76deeed6c378cc2af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 7 Mar 2011 15:16:53 -0400 Subject: [PATCH 0990/8313] fix --- .../comment_1_426482e6eb3a27687a48f24f6ef2332f._comment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment index 7b9935f7cc..119c9e535a 100644 --- a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment +++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment @@ -4,5 +4,5 @@ subject="comment 1" date="2011-03-07T19:13:14Z" content=""" -See [[fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames. +See [[bugs/fat_support]]. A bare git repo will have to be used to avoid symlink problems, at least for now. The other problem is that git-annex key files have colons in their filenames. """]] From 4cfcdfa9ba1f88e7aa9a6e7878cd86a2705541ab Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 14:20:11 -0400 Subject: [PATCH 0991/8313] mention mercurial bfiles extn --- doc/not.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index fe6e1b37d9..1d38b848c6 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -38,3 +38,9 @@ it does not support keeping different files in different clones of the same repository, which git-annex does, and is an important feature for large-scale archiving. + +* git-annex is not the [Mercurial bfiles extension](http://mercurial.selenic.com/wiki/BfilesExtension). + Although mercurial and git have some of the same problems around large + files, and both try to solve them in similar ways (standin files using + mostly hashes of the real content). + From b85c91743c73597f41cae73b042ffa86be929ed3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 17:48:25 -0400 Subject: [PATCH 0992/8313] update --- doc/bugs/unhappy_without_UTF8_locale.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/bugs/unhappy_without_UTF8_locale.mdwn b/doc/bugs/unhappy_without_UTF8_locale.mdwn index 6f1df4fab7..7b0e5be661 100644 --- a/doc/bugs/unhappy_without_UTF8_locale.mdwn +++ b/doc/bugs/unhappy_without_UTF8_locale.mdwn @@ -4,6 +4,12 @@ Try unsetting LANG and passing git-annex unicode filenames. add add add add git-annex: : commitAndReleaseBuffer: invalid argument (Invalid or incomplete multibyte or wide character) +> Interestingly, I can get the same crash in the de_DE.UTF-8 locale +> with certian input filenames, while in en_US.UTF-8, it's ok. +> The workaround below avoided the problem in de_DE.UTF-8. --[[Joey]] + +## underlying haskell problem and workaround + The same problem can be seen with a simple haskell program: import System.Environment From 26544de9463291b8185fdd1a7c1b33710ef7db3c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 18:05:20 -0400 Subject: [PATCH 0993/8313] put in utf8 forcing workaround Haskell's IO layer crashes on characters > 255 when in a non-unicode (latin1) locale. Until Haskell gets better behavior, put in an admittedly ugly workaround for that: git-annex forces utf8 output mode no matter what locale is selected. So if you use a non-utf8 locale, your filenames with characters > 127 will not be displayed as you'd expect. But at least it won't crash. --- CmdLine.hs | 2 ++ Messages.hs | 9 +++++++++ debian/changelog | 6 ++++++ doc/bugs/unhappy_without_UTF8_locale.mdwn | 2 ++ git-annex.hs | 1 + 5 files changed, 20 insertions(+) diff --git a/CmdLine.hs b/CmdLine.hs index 475ca99e78..1c01aa75f6 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,6 +11,7 @@ module CmdLine ( shutdown ) where +import System.IO import System.IO.Error (try) import System.Console.GetOpt import Control.Monad.State (liftIO) @@ -30,6 +31,7 @@ import UUID {- Runs the passed command line. -} dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () dispatch gitrepo args cmds options header = do + forceUtf8 state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown] diff --git a/Messages.hs b/Messages.hs index 90857280a5..83b3ecf239 100644 --- a/Messages.hs +++ b/Messages.hs @@ -64,3 +64,12 @@ indent s = join "\n" $ map (\l -> " " ++ l) $ lines s - non-decoded form. -} filePathToString :: FilePath -> String filePathToString = if SysConfig.unicodefilepath then id else UTF8.decodeString + +{- Workaround to avoid crashes displaying filenames containing + - characters > 255 in non-utf8 locales. Force encodings to utf-8, + - even though this may mean some characters in the encoding + - are mangled. -} +forceUtf8 :: IO () +forceUtf8 = do + hSetEncoding stdout utf8 + hSetEncoding stderr utf8 diff --git a/debian/changelog b/debian/changelog index e8b094607c..a414b3befa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,12 @@ git-annex (0.23) UNRELEASED; urgency=low * Support ssh remotes with a port specified. * whereis: New subcommand to show where a file's content has gotten to. + * Haskell's IO layer crashes on characters > 255 when in a non-unicode + locale. Until Haskell gets better behavior, put in an admittedly + ugly workaround for that: git-annex forces utf8 output mode no matter + what locale is selected. So if you use a non-utf8 locale, your + filenames with characters > 127 will not be displayed as you'd expect. + But at least it won't crash. -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 diff --git a/doc/bugs/unhappy_without_UTF8_locale.mdwn b/doc/bugs/unhappy_without_UTF8_locale.mdwn index 7b0e5be661..8d22b9ee44 100644 --- a/doc/bugs/unhappy_without_UTF8_locale.mdwn +++ b/doc/bugs/unhappy_without_UTF8_locale.mdwn @@ -8,6 +8,8 @@ Try unsetting LANG and passing git-annex unicode filenames. > with certian input filenames, while in en_US.UTF-8, it's ok. > The workaround below avoided the problem in de_DE.UTF-8. --[[Joey]] +> Put in the utf-8 forcing workaround for now. [[done]] --[[Joey]] + ## underlying haskell problem and workaround The same problem can be seen with a simple haskell program: diff --git a/git-annex.hs b/git-annex.hs index 878d8bdbbc..9d6012f2c1 100644 --- a/git-annex.hs +++ b/git-annex.hs @@ -8,6 +8,7 @@ import System.Environment import GitAnnex + main :: IO () main = do args <- getArgs From 876f0c6fbce125b946aa37eb1c409d6030785d72 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 18:19:52 -0400 Subject: [PATCH 0994/8313] add new todo item --- doc/todo/support-non-utf8-locales.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/todo/support-non-utf8-locales.mdwn diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn new file mode 100644 index 0000000000..26f600be4a --- /dev/null +++ b/doc/todo/support-non-utf8-locales.mdwn @@ -0,0 +1,19 @@ +Currenty, git-annex forces output, particularly of filenames, in a utf-8 +locale. + +Note that this does not mean it cannot be used with filenames in other +encodings. It just displays their names always converted to utf-8, which +may not look right when you have a non-utf8 locale. + +This had to be done to work around some bugs with haskell's handling +of filename encodings. In particular, + +* [[bugs/unhappy_without_UTF8_locale]]: haskell crashes when told to output + a string with characters > 255 in a non-utf8 locale. +* [[bugs/problems_with_utf8_names]]: On many OSs, haskell expects + non-decoded raw char8 in FilePaths. In order to display a filename, + though, it needs to first be decoded, and git-annex currently assumes + it was encoded as utf8. + +git-annex's behavior is unlikely to improve much until haskell's +support for utf8 filenames improves. --[[Joey]] From d7b4c8372b3901e09a0268d55b0a567a878166f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 8 Mar 2011 18:23:13 -0400 Subject: [PATCH 0995/8313] update --- doc/todo/support-non-utf8-locales.mdwn | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn index 26f600be4a..60f35eec81 100644 --- a/doc/todo/support-non-utf8-locales.mdwn +++ b/doc/todo/support-non-utf8-locales.mdwn @@ -2,8 +2,9 @@ Currenty, git-annex forces output, particularly of filenames, in a utf-8 locale. Note that this does not mean it cannot be used with filenames in other -encodings. It just displays their names always converted to utf-8, which -may not look right when you have a non-utf8 locale. +encodings. git-annex is entirely encoding agnostic when it comes to +manipulating filenames. It just *displays* their names always converted to +utf-8, which may not look right when you have a non-utf8 locale. This had to be done to work around some bugs with haskell's handling of filename encodings. In particular, From 42b7f244060bb9f49f9cbe9e93ee8024a678771d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Mar 2011 01:56:24 -0400 Subject: [PATCH 0996/8313] update --- doc/future_proofing.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn index d9a36ff733..4fc8246b6c 100644 --- a/doc/future_proofing.mdwn +++ b/doc/future_proofing.mdwn @@ -25,3 +25,12 @@ problem: * What is the hardware interface of the drive? Will hardware still exist to talk to it? + +* What if some of the data is damaged? git-annex facilitates storing a + configurable number of [[copies]] of the file contents. The metadata + about your files is stored in git, and so every clone of the repository + means another copy of that is stored. Also, git-annex uses filenames + for the data that encode everything needed to match it back to the + metadata. So if a filesystem is badly corrupted and all your annexed + files end up in `lost+found`, they can easily be lifted back out into + another clone of the repository. From b5134b4716c266147a35353316686cf29658350d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Mar 2011 01:57:30 -0400 Subject: [PATCH 0997/8313] tweak --- doc/use_case/Bob.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Bob.mdwn b/doc/use_case/Bob.mdwn index 18cdd4d8f4..42d10ea975 100644 --- a/doc/use_case/Bob.mdwn +++ b/doc/use_case/Bob.mdwn @@ -13,7 +13,7 @@ they're on, and easily make them available. Indeed, every drive knows what is on every other drive. [[more about location tracking|location_tracking]] -Bob thinks long-term, and so he's glad that git-annex uses a simple +Bob thinks long-term, and so he appreciates that git-annex uses a simple repository format. He knows his files will be accessible in the future even if the world has forgotten about git-annex and git. [[more about future-proofing|future_proofing]] From 9229d182d32570f6829ced655aa673ceddfe7693 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 9 Mar 2011 15:59:44 -0400 Subject: [PATCH 0998/8313] update --- doc/future_proofing.mdwn | 3 ++- doc/walkthrough.mdwn | 1 + .../recover_data_from_lost+found.mdwn | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 doc/walkthrough/recover_data_from_lost+found.mdwn diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn index 4fc8246b6c..3937e92656 100644 --- a/doc/future_proofing.mdwn +++ b/doc/future_proofing.mdwn @@ -33,4 +33,5 @@ problem: for the data that encode everything needed to match it back to the metadata. So if a filesystem is badly corrupted and all your annexed files end up in `lost+found`, they can easily be lifted back out into - another clone of the repository. + another clone of the repository. Even if the filenames are lost, + it's possible to [[walkthrough/recover_data_from_lost+found]]. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 1e8ad7e988..3b4f7d56a4 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -23,4 +23,5 @@ A walkthrough of the basic features of git-annex. backups untrusted_repositories what_to_do_when_you_lose_a_repository + recover_data_from_lost+found """]] diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/walkthrough/recover_data_from_lost+found.mdwn new file mode 100644 index 0000000000..6e2c241485 --- /dev/null +++ b/doc/walkthrough/recover_data_from_lost+found.mdwn @@ -0,0 +1,17 @@ +Suppose something goes wrong, and fsck puts all the files in lost+found. +It's actually very easy to recover from this disaster. + +First, check out the git repository again. Then, in the new checkout: + + mkdir recovered-content + sudo mv ../lost+found/* recovered-content + git annex add recovered-content + git rm recovered-content + git commit -m "recovered some content" + git annex fsck + +The way that works is that when git-annex adds the same content that was in +the repository before, all the old links to that content start working +again. This works particularly well if the SHA1 backend is used, but even +with the default backend it will work pretty well, as long as fsck +preserved the modification time of the files. From 4163341c9d09f92f4bd5dfb5d2f34538bee51bdb Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 9 Mar 2011 23:47:49 +0000 Subject: [PATCH 0999/8313] Added a comment: use mini-branches --- ..._0531dcfa833b0321a7009526efe3df33._comment | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment new file mode 100644 index 0000000000..8fec6bad72 --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment @@ -0,0 +1,26 @@ +[[!comment format=mdwn + username="http://christian.amsuess.com/chrysn" + nickname="chrysn" + subject="use mini-branches" + date="2011-03-09T23:47:48Z" + content=""" +if you go for the two-commits version, small intermediate branches (or git-commit-tree) could be used to create a tree like this: + + + * commit 106eef2 + |\ Merge: 436e46f 9395665 + | | + | | the main commit + | | + | * commit 9395665 + |/ + | intermediate move + | + * commit 436e46f + | + | ... + +while the first commit (436e46f) has a \"`/subdir/foo → ../.git-annex/where_foo_is`\", the intermediate (9395665) has \"`/subdir/deeper/foo → ../.git-annex/where_foo_is`\", and the inal commit (106eef2) has \"`/subdir/deeper/foo → ../../.git-annex/where_foo_is`\". + +`--follow` uses the intermediate commit to find the history, but the intermediate commit would neither show up in `git log --first-parent` nor affect `git diff HEAD^..` & co. (there could still be confusion over `git show`, though). +"""]] From 72d268401604fbac93ca4701ab53d32880483686 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 12 Mar 2011 15:30:17 -0400 Subject: [PATCH 1000/8313] Rethink filename encoding handling for display. Since filename encoding may or may not match locale settings, any attempt to decode filenames will fail for some files. So instead, do all output in binary mode. --- Backend/File.hs | 6 +++--- Backend/SHA.hs | 2 +- Backend/WORM.hs | 2 +- CmdLine.hs | 3 +-- Command/Find.hs | 2 +- Command/PreCommit.hs | 2 +- Command/Unused.hs | 2 +- Content.hs | 2 +- Messages.hs | 30 +++++++++++--------------- debian/changelog | 13 +++++------ doc/bugs/problems_with_utf8_names.mdwn | 3 +++ doc/todo/support-non-utf8-locales.mdwn | 6 ++++++ 12 files changed, 37 insertions(+), 36 deletions(-) diff --git a/Backend/File.hs b/Backend/File.hs index d5691595a8..d76cd29391 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -193,14 +193,14 @@ checkKeyNumCopies key file numcopies = do missingNote :: String -> Int -> Int -> String -> String missingNote file 0 _ [] = - "** No known copies of " ++ filePathToString file ++ " exist!" + "** No known copies of " ++ file ++ " exist!" missingNote file 0 _ untrusted = - "Only these untrusted locations may have copies of " ++ filePathToString file ++ + "Only these untrusted locations may have copies of " ++ file ++ "\n" ++ untrusted ++ "Back it up to trusted locations with git-annex copy." missingNote file present needed [] = "Only " ++ show present ++ " of " ++ show needed ++ - " trustworthy copies of " ++ filePathToString file ++ " exist." ++ + " trustworthy copies of " ++ file ++ " exist." ++ "\nBack it up with git-annex copy." missingNote file present needed untrusted = missingNote file present needed [] ++ diff --git a/Backend/SHA.hs b/Backend/SHA.hs index c074ab48a2..4eea890ce4 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -83,5 +83,5 @@ checkKeyChecksum size key = do then return True else do dest <- moveBad key - warning $ "Bad file content; moved to " ++ filePathToString dest + warning $ "Bad file content; moved to " ++ dest return False diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 8a6412eb11..a0d814aa08 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -70,5 +70,5 @@ checkKeySize key = do then return True else do dest <- moveBad key - warning $ "Bad file size; moved to " ++ filePathToString dest + warning $ "Bad file size; moved to " ++ dest return False diff --git a/CmdLine.hs b/CmdLine.hs index 1c01aa75f6..b8fd6af7ce 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -11,7 +11,6 @@ module CmdLine ( shutdown ) where -import System.IO import System.IO.Error (try) import System.Console.GetOpt import Control.Monad.State (liftIO) @@ -31,7 +30,7 @@ import UUID {- Runs the passed command line. -} dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO () dispatch gitrepo args cmds options header = do - forceUtf8 + setupConsole state <- Annex.new gitrepo allBackends (actions, state') <- Annex.run state $ parseCmd args header cmds options tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown] diff --git a/Command/Find.hs b/Command/Find.hs index 3e9125b9a6..1ca6ff1e7c 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -25,5 +25,5 @@ seek = [withFilesInGit start] start :: CommandStartString start file = isAnnexed file $ \(key, _) -> do exists <- inAnnex key - when exists $ liftIO $ putStrLn $ filePathToString file + when exists $ liftIO $ putStrLn file return Nothing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index d2f6964343..6f9adb79a5 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -34,7 +34,7 @@ perform pair@(file, _) = do ok <- doCommand $ Command.Add.start pair if ok then return $ Just $ cleanup file - else error $ "failed to add " ++ filePathToString file ++ "; canceling commit" + else error $ "failed to add " ++ file ++ "; canceling commit" cleanup :: FilePath -> CommandCleanup cleanup file = do diff --git a/Command/Unused.hs b/Command/Unused.hs index 9f3881d595..a614ce5d94 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -68,7 +68,7 @@ checkUnused = do dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] table l = [" NUMBER KEY"] ++ map cols l - cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ (filePathToString . show) k + cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' number :: Int -> [a] -> [(Int, a)] diff --git a/Content.hs b/Content.hs index bcd4ac0e13..895a8812c2 100644 --- a/Content.hs +++ b/Content.hs @@ -50,7 +50,7 @@ calcGitLink file key = do cwd <- liftIO $ getCurrentDirectory let absfile = case absNormPath cwd file of Just f -> f - Nothing -> error $ "unable to normalize " ++ filePathToString file + Nothing -> error $ "unable to normalize " ++ file return $ relPathDirToDir (parentDir absfile) (Git.workTree g) ".git" annexLocation key diff --git a/Messages.hs b/Messages.hs index 83b3ecf239..733638ce12 100644 --- a/Messages.hs +++ b/Messages.hs @@ -11,11 +11,9 @@ import Control.Monad.State (liftIO) import System.IO import Control.Monad (unless) import Data.String.Utils -import qualified Codec.Binary.UTF8.String as UTF8 import Types import qualified Annex -import qualified SysConfig verbose :: Annex () -> Annex () verbose a = do @@ -27,7 +25,7 @@ showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")" showStart :: String -> String -> Annex () showStart command file = verbose $ do - liftIO $ putStr $ command ++ " " ++ filePathToString file ++ " " + liftIO $ putStr $ command ++ " " ++ file ++ " " liftIO $ hFlush stdout showNote :: String -> Annex () @@ -59,17 +57,15 @@ warning w = do indent :: String -> String indent s = join "\n" $ map (\l -> " " ++ l) $ lines s -{- Prepares a filename for display. This is needed because on many - - platforms (eg, unix), FilePaths are internally stored in - - non-decoded form. -} -filePathToString :: FilePath -> String -filePathToString = if SysConfig.unicodefilepath then id else UTF8.decodeString - -{- Workaround to avoid crashes displaying filenames containing - - characters > 255 in non-utf8 locales. Force encodings to utf-8, - - even though this may mean some characters in the encoding - - are mangled. -} -forceUtf8 :: IO () -forceUtf8 = do - hSetEncoding stdout utf8 - hSetEncoding stderr utf8 +{- By default, haskell honors the user's locale in its output to stdout + - and stderr. While that's great for proper unicode support, for git-annex + - all that's really needed is the ability to display simple messages + - (currently untranslated), and importantly, to display filenames exactly + - as they are written on disk, no matter what their encoding. So, force + - raw mode. + - + - NB: Once git-annex gets localized, this will need a rethink. -} +setupConsole :: IO () +setupConsole = do + hSetBinaryMode stdout True + hSetBinaryMode stderr True diff --git a/debian/changelog b/debian/changelog index a414b3befa..90b4cf6b25 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,15 +1,12 @@ -git-annex (0.23) UNRELEASED; urgency=low +git-annex (0.23) unstable; urgency=low * Support ssh remotes with a port specified. * whereis: New subcommand to show where a file's content has gotten to. - * Haskell's IO layer crashes on characters > 255 when in a non-unicode - locale. Until Haskell gets better behavior, put in an admittedly - ugly workaround for that: git-annex forces utf8 output mode no matter - what locale is selected. So if you use a non-utf8 locale, your - filenames with characters > 127 will not be displayed as you'd expect. - But at least it won't crash. + * Rethink filename encoding handling for display. Since filename encoding + may or may not match locale settings, any attempt to decode filenames + will fail for some files. So instead, do all output in binary mode. - -- Joey Hess Sat, 05 Mar 2011 15:39:13 -0400 + -- Joey Hess Sat, 12 Mar 2011 15:02:49 -0400 git-annex (0.22) unstable; urgency=low diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn index efde1c9a3a..d6dc6ca3c3 100644 --- a/doc/bugs/problems_with_utf8_names.mdwn +++ b/doc/bugs/problems_with_utf8_names.mdwn @@ -63,6 +63,9 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu > One other possible > issue would be that this could cause problems if git-annex were > translated. +> > On second thought, I switched to this. Any decoding of a filename +> > is going to make someone unhappy; the previous approach broke +> > non-utf8 filenames. ---- diff --git a/doc/todo/support-non-utf8-locales.mdwn b/doc/todo/support-non-utf8-locales.mdwn index 60f35eec81..da40118d52 100644 --- a/doc/todo/support-non-utf8-locales.mdwn +++ b/doc/todo/support-non-utf8-locales.mdwn @@ -18,3 +18,9 @@ of filename encodings. In particular, git-annex's behavior is unlikely to improve much until haskell's support for utf8 filenames improves. --[[Joey]] + +> [[done]] -- I just turned off all encoding handling on stdout and stderr, +> which avoids these problems nicely. Git-annex now displays just what it +> input, at least on platforms where haskell does not decode unicode in +> FilePaths. This will later be a problem when it gets localized, but for +> now works great. --[[Joey]] From cf6a13a05710e53a9a1b28b7474a5a31494349a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 12 Mar 2011 15:41:48 -0400 Subject: [PATCH 1001/8313] add news item for git-annex 0.23 --- doc/news/version_0.18.mdwn | 9 --------- doc/news/version_0.23.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 doc/news/version_0.18.mdwn create mode 100644 doc/news/version_0.23.mdwn diff --git a/doc/news/version_0.18.mdwn b/doc/news/version_0.18.mdwn deleted file mode 100644 index f2e0edb424..0000000000 --- a/doc/news/version_0.18.mdwn +++ /dev/null @@ -1,9 +0,0 @@ -git-annex 0.18 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Bugfix: `copy --to` and `move --to` forgot to stage location log changes - after transferring the file to the remote repository. - (Did not affect ssh remotes.) - * fsck: Fix bug in moving of corrupted files to .git/annex/bad/ - * migrate: Fix support for --backend option. - * unlock: Fix behavior when file content is not present. - * Test suite improvements. Current top-level test coverage: 80%"""]] \ No newline at end of file diff --git a/doc/news/version_0.23.mdwn b/doc/news/version_0.23.mdwn new file mode 100644 index 0000000000..e045352ad4 --- /dev/null +++ b/doc/news/version_0.23.mdwn @@ -0,0 +1,7 @@ +git-annex 0.23 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Support ssh remotes with a port specified. + * whereis: New subcommand to show where a file's content has gotten to. + * Rethink filename encoding handling for display. Since filename encoding + may or may not match locale settings, any attempt to decode filenames + will fail for some files. So instead, do all output in binary mode."""]] \ No newline at end of file From 175d055d4d632c0fc0d2080c7721bf7b3121ddc1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Mar 2011 14:25:32 -0400 Subject: [PATCH 1002/8313] Add Suggests on graphviz. Closes: #618039 --- Content.hs | 1 - debian/changelog | 6 ++++++ debian/control | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Content.hs b/Content.hs index 895a8812c2..dc675389f1 100644 --- a/Content.hs +++ b/Content.hs @@ -34,7 +34,6 @@ import UUID import qualified GitRepo as Git import qualified Annex import Utility -import Messages {- Checks if a given key is currently present in the gitAnnexLocation. -} inAnnex :: Key -> Annex Bool diff --git a/debian/changelog b/debian/changelog index 90b4cf6b25..6e70ac67e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.24) UNRELEASED; urgency=low + + * Add Suggests on graphviz. Closes: #618039 + + -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 + git-annex (0.23) unstable; urgency=low * Support ssh remotes with a port specified. diff --git a/debian/control b/debian/control index d0f8805d0f..4b31a295f9 100644 --- a/debian/control +++ b/debian/control @@ -11,6 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync +Suggests: graphviz Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when From 4ddcf26051f6fed9e846f15bee9e1c02b1b9a9d6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 13 Mar 2011 20:32:12 +0000 Subject: [PATCH 1003/8313] --- doc/bugs/softlink_atime.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/softlink_atime.mdwn diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn new file mode 100644 index 0000000000..ad91cf8a9d --- /dev/null +++ b/doc/bugs/softlink_atime.mdwn @@ -0,0 +1,3 @@ +When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. + +Optionally, editing the meta-data should change the times in all annexes. From 61afcd2d16c27430f1a6c6ae5cd468b16243c966 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 13 Mar 2011 17:42:07 -0400 Subject: [PATCH 1004/8313] response --- doc/bugs/softlink_atime.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index ad91cf8a9d..e60b8ec98b 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -1,3 +1,10 @@ When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. Optionally, editing the meta-data should change the times in all annexes. + +> Thing is, git does not preserve file timestamps much at all. +> It's not uncommon for a `git checkout` to or `git update` to +> mess up timestamps. This is why things like metastore exist (and +> metastore should work ok with git annexed files too). Trying to +> make annexed file symlinks have better timestamp handling than regular +> files in git seems pointless. --[[Joey]] From 7dd8cde63c6e41c78a06f0f03bd555cfa780864f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkbfH595UQMdPtWbUKZtjsr-nR6AR-cxek" Date: Mon, 14 Mar 2011 08:10:08 +0000 Subject: [PATCH 1005/8313] thoughts? --- doc/forum/hashing_objects_directories.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/hashing_objects_directories.mdwn diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn new file mode 100644 index 0000000000..4e5f4b1c98 --- /dev/null +++ b/doc/forum/hashing_objects_directories.mdwn @@ -0,0 +1,17 @@ +I'm wondering how easy the addition of hashing to the directories of the objects would be. + +Currently a tree directory structure becomes a flat two level tree under the .git/annex/objects directory ([[internals]]). This, through the 555 mode on the directory prevents the accidental destruction of content, which is _good_. However file and directory numbers soon add up in there and as such any file-systems with sub directory limitations will quickly realize the limit (certainly quicker than maybe expected). + +Suggestion is therefore to change from + + `.git/annex/objects/SHA1:123456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678` + +to + + `.git/annex/objects/SHA1:1/2/3456789abcdef0123456789abcdef012345678/SHA1:123456789abcdef0123456789abcdef012345678` + +or anything in between to a paranoid + + `.git/annex/objects/SHA1:123/456/789/abc/def/012/345/678/9ab/cde/f01/234/5678/SHA1:123456789abcdef0123456789abcdef012345678` + +Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...` From 1033b4a555df340dcf80ce2e614540263a728fdb Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkbfH595UQMdPtWbUKZtjsr-nR6AR-cxek" Date: Mon, 14 Mar 2011 09:40:08 +0000 Subject: [PATCH 1006/8313] init --- doc/forum/hashing_objects_directories.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn index 4e5f4b1c98..715e972ca4 100644 --- a/doc/forum/hashing_objects_directories.mdwn +++ b/doc/forum/hashing_objects_directories.mdwn @@ -15,3 +15,5 @@ or anything in between to a paranoid `.git/annex/objects/SHA1:123/456/789/abc/def/012/345/678/9ab/cde/f01/234/5678/SHA1:123456789abcdef0123456789abcdef012345678` Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...` + +`git annex init` could also create all but the last level directory on initialization. I'm thinking `SHA1/1/1, SHA1/1/2, ..., SHA256/f/f, ..., URL/f/f, ..., WORM/f/f` From 5aac014a677ff636f61f4ccc70cc61bf920a14c6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 13:47:38 +0000 Subject: [PATCH 1007/8313] --- doc/bugs/softlink_atime.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index e60b8ec98b..ab13863f1a 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -1,4 +1,4 @@ -When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. +When adding files to git annex, softlinks are created with current atime (and ctime, etc). Instead, the atime of the added file should be used and added to the meta-data, restoring it everywhere an annex is cloned to. -- RichiH Optionally, editing the meta-data should change the times in all annexes. @@ -8,3 +8,5 @@ Optionally, editing the meta-data should change the times in all annexes. > metastore should work ok with git annexed files too). Trying to > make annexed file symlinks have better timestamp handling than regular > files in git seems pointless. --[[Joey]] + +> > Improving an area where git is (not yet?) good at still makes sense, imo. Photos and the like need absolute timestamps more than source code which is fine with relative timestamps (local builds & updates). Maintaining global timestamps for source code could even cause a lot of unwanted effects. As it is, this issue is the only, but a major, blocker for me before I can start adapting git-annex. As I have three different use cases for it, this is a shame. Unfortunately, I don't speak any Haskell so scratching my own itch isn't do-able (without major effort and not soon, at least). Is there a realistic chance that you will tackle this nonetheless or is this WONTFIX? -- RichiH From d86357a4166e9fdfd45ea2b47d62e3c09ced13fd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 13:52:07 +0000 Subject: [PATCH 1008/8313] --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 91a5001cc9..6020db86dc 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -4,3 +4,5 @@ repository! This way, you know your annexed content is safe & versioned, but you only have to deal with the pain of git with large files in one place, and can use all of git-annex's features everywhere else. + +> Speaking as a future user, do very, very much want. -- RichiH From 9b353aa60e3828d354868d2024b858b7155622be Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 13:54:07 +0000 Subject: [PATCH 1009/8313] --- doc/todo/auto_remotes.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn index 2ac4790af9..c12878950a 100644 --- a/doc/todo/auto_remotes.mdwn +++ b/doc/todo/auto_remotes.mdwn @@ -19,3 +19,5 @@ otherwise ssh:// ones.) Question: When should git-annex update the remote.log? (If not just on init.) Whenever it reads in a repo's remotes? + +> This sounds useful and the log should be updated every time any remote is being accessed. A counter or timestamp (yes, distributed times may be wrong/different) could be used to auto-prune old entries via a global and per-remote config setting. -- RichiH From d7ea568f50026ea581b0c91ae97495a4e89609b2 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 14:02:47 +0000 Subject: [PATCH 1010/8313] --- doc/todo/hidden_files.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn index 2e5ec4d9e8..14c4f0da5a 100644 --- a/doc/todo/hidden_files.mdwn +++ b/doc/todo/hidden_files.mdwn @@ -18,3 +18,7 @@ TODO: * What will `git annex get` do if it's asked to get a file that has been hidden? + +> Unless I am missing something: Make sure the data is correct (for SHA1 or other tracking) and restore locally. If that's not the case, delete and restore from remote. -- RichiH + +* Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH From 1a2c9b61f748760176bb61193b2aec2babf0e169 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:08:38 -0400 Subject: [PATCH 1011/8313] note --- doc/bugs/fat_support.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 2a7187a7f6..8088e965a7 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -8,4 +8,8 @@ be VFAT formatted: - Use of ":" in filenames of object files, also not supported. Could easily be fixed by reorganizing the object directory. + (If the object directory is reorganized, should consider adding hashing + at the same time.) + [[!tag wishlist]] + From 769d2a780d33fda9dd6e3b4242d31edf5ff2e232 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Mar 2011 16:12:49 +0000 Subject: [PATCH 1012/8313] Added a comment --- ...mment_1_c55c56076be4f54251b0b7f79f28a607._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment diff --git a/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment b/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment new file mode 100644 index 0000000000..3a19310b63 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-14T16:12:49Z" + content=""" +My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. + +I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.) + +There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning.. +"""]] From 5e9f90240367e9b1006402fe776c24f2488876b2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Mar 2011 16:12:51 +0000 Subject: [PATCH 1013/8313] Added a comment --- ...mment_2_d095c769febedca2d69fa5d099bcb520._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment diff --git a/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment b/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment new file mode 100644 index 0000000000..41c6ad5d96 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-14T16:12:51Z" + content=""" +My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. + +I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.) + +There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning.. +"""]] From f130c413a9c8d82b7813e001c7e561609cce4caa Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 14 Mar 2011 16:13:30 +0000 Subject: [PATCH 1014/8313] removed --- ...mment_2_d095c769febedca2d69fa5d099bcb520._comment | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment diff --git a/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment b/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment deleted file mode 100644 index 41c6ad5d96..0000000000 --- a/doc/forum/hashing_objects_directories/comment_2_d095c769febedca2d69fa5d099bcb520._comment +++ /dev/null @@ -1,12 +0,0 @@ -[[!comment format=mdwn - username="http://joey.kitenet.net/" - nickname="joey" - subject="comment 2" - date="2011-03-14T16:12:51Z" - content=""" -My experience is that modern filesystems are not going to have many issues with tens to hundreds of thousands of items in the directory. However, if a transition does happen for FAT support I will consider adding hashing. Although getting a good balanced hash in general without, say, checksumming the filename and taking part of the checksum, is difficult. - -I prefer to keep all the metadata in the filename, as this eases recovery if the files end up in lost+found. So while \"SHA/\" is a nice workaround for the FAT colon problem, I'll be doing something else. (What I'm not sure yet.) - -There is no point in creating unused hash directories on initialization. If anything, with a bad filesystem that just guarantees worst performance from the beginning.. -"""]] From 90eb1185679b70f27a4659155e5ec22944ff2783 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:17:34 -0400 Subject: [PATCH 1015/8313] response --- doc/bugs/softlink_atime.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index ab13863f1a..7ab0881df3 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -10,3 +10,12 @@ Optionally, editing the meta-data should change the times in all annexes. > files in git seems pointless. --[[Joey]] > > Improving an area where git is (not yet?) good at still makes sense, imo. Photos and the like need absolute timestamps more than source code which is fine with relative timestamps (local builds & updates). Maintaining global timestamps for source code could even cause a lot of unwanted effects. As it is, this issue is the only, but a major, blocker for me before I can start adapting git-annex. As I have three different use cases for it, this is a shame. Unfortunately, I don't speak any Haskell so scratching my own itch isn't do-able (without major effort and not soon, at least). Is there a realistic chance that you will tackle this nonetheless or is this WONTFIX? -- RichiH + +>>> Not quite WONTFIX. git-annex should at least, when adding new files, +>>> preserve their timestamp in the symlink it creates. +>>> +>>> Since it doesn't have anything to do with maintaining the symlinks +>>> during an update, or a clone, etc, maintaining the permissions of them +>>> is also out of scope, and it's best to just use metastore if you need +>>> it. Otherwise, git-annex would have to reimplement metastore, and is +>>> unlikely to do it better. From 5ea4f0c0f36fc70c905b452ecd1aa35201d8554c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:18:23 -0400 Subject: [PATCH 1016/8313] response --- doc/todo/add_a_git_backend.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 6020db86dc..25e780269f 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -6,3 +6,7 @@ have to deal with the pain of git with large files in one place, and can use all of git-annex's features everywhere else. > Speaking as a future user, do very, very much want. -- RichiH + +>> Might also be interesting to use `bup` in the git backend, to work +>> around git's big file issues there. So git-annex would pull data out +>> of the git backend using bup. --[[Joey]] From 16845a1f663a38b7d82750db48424011c9ca5a5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:21:58 -0400 Subject: [PATCH 1017/8313] response --- doc/todo/hidden_files.mdwn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn index 14c4f0da5a..97eb71e7be 100644 --- a/doc/todo/hidden_files.mdwn +++ b/doc/todo/hidden_files.mdwn @@ -21,4 +21,8 @@ TODO: > Unless I am missing something: Make sure the data is correct (for SHA1 or other tracking) and restore locally. If that's not the case, delete and restore from remote. -- RichiH -* Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH +---- + +Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH + +> Nah, `git annex unused/dropunused` already exist. --[[Joey]] From a021ae92913c0eaca1509fe09e3560871480478e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 12:33:15 -0400 Subject: [PATCH 1018/8313] design --- doc/bugs/fat_support.mdwn | 3 --- doc/todo/object_dir_reorg_v2.mdwn | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 doc/todo/object_dir_reorg_v2.mdwn diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 8088e965a7..2c6c973856 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -8,8 +8,5 @@ be VFAT formatted: - Use of ":" in filenames of object files, also not supported. Could easily be fixed by reorganizing the object directory. - (If the object directory is reorganized, should consider adding hashing - at the same time.) - [[!tag wishlist]] diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn new file mode 100644 index 0000000000..db18856995 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -0,0 +1,19 @@ +Several things suggest now would be a good time to reorgaize the object +directory. This would be annex.version=2. It will be slightly painful for +all users, so this should be the *last* reorg in the forseeable future. + +1. Remove colons from filenames, for [[bugs/fat_support]] + +2. Add hashing, since some filesystems do suck (like er, fat at least :) + [[forum/hashing_objects_directories]] + +3. Add filesize metadata for [[bugs/free_space_checking]]. (Currently only + present in WORM, and in an ad-hoc way.) + +4. Perhaps use a generic format that will allow further metadata to be + added later. For example, + "bSHA1,s101111,kf3101c30bb23467deaec5d78c6daa71d395d1879" + + (Probably everything after ",k" should be part of the key, even if it + contains the "," separator character. Otherwise an escaping mechanism + would be needed.) From 61396e34b5c9cb8b94940e9e83baf4e1a9466205 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 17:00:38 +0000 Subject: [PATCH 1019/8313] --- doc/bugs/softlink_atime.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index 7ab0881df3..49e26013ff 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -19,3 +19,5 @@ Optionally, editing the meta-data should change the times in all annexes. >>> is also out of scope, and it's best to just use metastore if you need >>> it. Otherwise, git-annex would have to reimplement metastore, and is >>> unlikely to do it better. + +>>>> OK, thanks for the clarification. Would it be acceptable for you to put the timestamps into the metastore with vanilla git? If such an option existed, everyone would be able to benefit and not just me. -- RichiH From a4993f9321702d75d93c76227a8b8ee2fd863fb1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 17:06:30 +0000 Subject: [PATCH 1020/8313] --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 25e780269f..34322d9811 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -10,3 +10,5 @@ use all of git-annex's features everywhere else. >> Might also be interesting to use `bup` in the git backend, to work >> around git's big file issues there. So git-annex would pull data out >> of the git backend using bup. --[[Joey]] + +>>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH From 96e074bb0333b6952fb9fbce2f0a810ebafd3d2c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 14 Mar 2011 17:07:31 +0000 Subject: [PATCH 1021/8313] --- doc/todo/hidden_files.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/hidden_files.mdwn b/doc/todo/hidden_files.mdwn index 97eb71e7be..191e9c3286 100644 --- a/doc/todo/hidden_files.mdwn +++ b/doc/todo/hidden_files.mdwn @@ -26,3 +26,5 @@ TODO: Is 'unused' a good name? 'clean' and 'autoclean' would make more sense, imo. 'clean' deletes everything, whereas an optional 'autoclean' could try to be smart based on disk usage and/or SHA1, etc. -- RichiH > Nah, `git annex unused/dropunused` already exist. --[[Joey]] + +>> OK, in that case forget what I said. No idea about your internal policy, but feel free to delete this part of the page, then. -- RichiH From bc5c54c987f548505a3877e8a0e460abe0b2a081 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 14 Mar 2011 23:00:23 -0400 Subject: [PATCH 1022/8313] symlink touching fun When adding files to the annex, the symlinks pointing at the annexed content are made to have the same mtime as the original file. While git does not preserve that information, this allows a tool like metastore to be used with annexed files. --- .gitignore | 1 + Command/Add.hs | 7 ++++ Command/Find.hs | 1 - Command/PreCommit.hs | 1 - Makefile | 7 ++-- Touch.hsc | 70 ++++++++++++++++++++++++++++++++++++ debian/changelog | 4 +++ doc/bugs/softlink_atime.mdwn | 14 ++++++++ 8 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 Touch.hsc diff --git a/.gitignore b/.gitignore index 764a1af9d5..69d2c80709 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ doc/.ikiwiki html *.tix .hpc +Touch.hs diff --git a/Command/Add.hs b/Command/Add.hs index 26e7fa258d..09fff7cff7 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -18,6 +18,7 @@ import Types import Content import Messages import Utility +import Touch command :: [Command] command = [Command "add" paramPath seek "add files to annex"] @@ -53,5 +54,11 @@ cleanup file key = do link <- calcGitLink file key liftIO $ createSymbolicLink link file + + -- touch the symlink to have the same mtime as the file it points to + s <- liftIO $ getFileStatus file + let mtime = modificationTime s + _ <- liftIO $ touch file (TimeSpec mtime 0) False + Annex.queue "add" [Param "--"] file return True diff --git a/Command/Find.hs b/Command/Find.hs index 1ca6ff1e7c..3ed15c1537 100644 --- a/Command/Find.hs +++ b/Command/Find.hs @@ -12,7 +12,6 @@ import Control.Monad.State (liftIO) import Command import Content -import Messages command :: [Command] command = [Command "find" (paramOptional $ paramRepeating paramPath) seek diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 6f9adb79a5..1465ebc615 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -14,7 +14,6 @@ import qualified Annex import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix -import Messages import Utility command :: [Command] diff --git a/Makefile b/Makefile index c888fc2151..c381ae986d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,10 @@ SysConfig.hs: configure.hs TestConfig.hs $(GHCMAKE) configure ./configure -$(bins): SysConfig.hs +Touch.hs: Touch.hsc + hsc2hs $< + +$(bins): SysConfig.hs Touch.hs $(GHCMAKE) $@ git-annex.1: doc/git-annex.mdwn @@ -57,7 +60,7 @@ docs: $(mans) --exclude='news/.*' clean: - rm -rf build $(bins) $(mans) test configure SysConfig.hs *.tix .hpc + rm -rf build $(bins) $(mans) test configure Touch.hs SysConfig.hs *.tix .hpc rm -rf doc/.ikiwiki html find . \( -name \*.o -or -name \*.hi \) -exec rm {} \; diff --git a/Touch.hsc b/Touch.hsc new file mode 100644 index 0000000000..ad36761c4b --- /dev/null +++ b/Touch.hsc @@ -0,0 +1,70 @@ +{- More control over touching a file. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Touch ( + TimeSpec(..), + now, + omit, + touchBoth, + touch +) where + +import Foreign +import Foreign.C + +#include +#include +#include + +#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__) + +data TimeSpec = TimeSpec CTime CLong + +instance Storable TimeSpec where + alignment _ = #{alignment struct timespec} + sizeOf _ = #{size struct timespec} + peek ptr = do + sec <- #{peek struct timespec, tv_sec} ptr + nsec <- #{peek struct timespec, tv_nsec} ptr + return $ TimeSpec sec nsec + poke ptr (TimeSpec sec nsec) = do + #{poke struct timespec, tv_sec} ptr sec + #{poke struct timespec, tv_nsec} ptr nsec + +{- special timespecs -} +omit :: TimeSpec +omit = TimeSpec 0 #const UTIME_OMIT +now :: TimeSpec +now = TimeSpec 0 #const UTIME_NOW + +{- While its interface is beastly, utimensat is in recent + POSIX standards, unlike futimes. -} +foreign import ccall "utimensat" + c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt + +{- Changes the access and/or modification times of a file. + Can follow symlinks, or not. -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO Bool +touchBoth file atime mtime follow = + allocaArray 2 $ \ptr -> + withCString file $ \f -> do + pokeArray ptr [atime, mtime] + r <- c_utimensat at_fdcwd f ptr flags + putStrLn $ "ret " ++ (show r) + return (r == 0) + where + at_fdcwd = #const AT_FDCWD + at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW + + flags = if follow + then 0 + else at_symlink_nofollow + +touch :: FilePath -> TimeSpec -> Bool -> IO Bool +touch file mtime follow = touchBoth file omit mtime follow diff --git a/debian/changelog b/debian/changelog index 6e70ac67e3..e7017a26d0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,10 @@ git-annex (0.24) UNRELEASED; urgency=low * Add Suggests on graphviz. Closes: #618039 + * When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index 49e26013ff..ebb040dd1e 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -21,3 +21,17 @@ Optionally, editing the meta-data should change the times in all annexes. >>> unlikely to do it better. >>>> OK, thanks for the clarification. Would it be acceptable for you to put the timestamps into the metastore with vanilla git? If such an option existed, everyone would be able to benefit and not just me. -- RichiH + +>>>>> I've now committed to git changes to make git-annex add make +>>>>> symlinks that reflect the original file's mtime. (It's not possible +>>>>> to set the ctime of a symlink; nor would you want to as messing with +>>>>> ctimes can break backup software ... and atime doesn't much matter.) +>>>>> +>>>>> So all you have to do is make the pre-commit hook call +>>>>> [metastore](http://david.hardeman.nu/software.php). The hook +>>>>> would look like this: ---[[Joey]] [[!tag done]] + + #!/bin/sh + git annex pre-commit . + metastore --save + git add .metadata From 88ef64e566cac30f552982354642feb993d13823 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 01:16:27 -0400 Subject: [PATCH 1023/8313] calculate alignment in a way I can understand See http://therning.org/magnus/archives/315 --- Touch.hsc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Touch.hsc b/Touch.hsc index ad36761c4b..f85873125c 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -22,12 +22,14 @@ import Foreign.C #include #include -#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__) - data TimeSpec = TimeSpec CTime CLong instance Storable TimeSpec where - alignment _ = #{alignment struct timespec} + -- use the larger alignment of the two types in the struct + alignment _ = max sec_alignment nsec_alignment + where + sec_alignment = alignment $ undefined::CTime + nsec_alignment = alignment $ undefined::CLong sizeOf _ = #{size struct timespec} peek ptr = do sec <- #{peek struct timespec, tv_sec} ptr From 199ffa6c36178ac7360ad957feccba188e0dad02 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 02:31:47 -0400 Subject: [PATCH 1024/8313] remove debug --- Touch.hsc | 1 - 1 file changed, 1 deletion(-) diff --git a/Touch.hsc b/Touch.hsc index f85873125c..4e1052b94a 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -58,7 +58,6 @@ touchBoth file atime mtime follow = withCString file $ \f -> do pokeArray ptr [atime, mtime] r <- c_utimensat at_fdcwd f ptr flags - putStrLn $ "ret " ++ (show r) return (r == 0) where at_fdcwd = #const AT_FDCWD From bca68c263da765acedad5643a37f09f142555683 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 02:36:24 -0400 Subject: [PATCH 1025/8313] tweaks --- Touch.hsc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Touch.hsc b/Touch.hsc index 4e1052b94a..34b34be515 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -9,8 +9,8 @@ module Touch ( TimeSpec(..), - now, - omit, + nowTime, + omitTime, touchBoth, touch ) where @@ -28,8 +28,8 @@ instance Storable TimeSpec where -- use the larger alignment of the two types in the struct alignment _ = max sec_alignment nsec_alignment where - sec_alignment = alignment $ undefined::CTime - nsec_alignment = alignment $ undefined::CLong + sec_alignment = alignment (undefined::CTime) + nsec_alignment = alignment (undefined::CLong) sizeOf _ = #{size struct timespec} peek ptr = do sec <- #{peek struct timespec, tv_sec} ptr @@ -40,10 +40,10 @@ instance Storable TimeSpec where #{poke struct timespec, tv_nsec} ptr nsec {- special timespecs -} -omit :: TimeSpec -omit = TimeSpec 0 #const UTIME_OMIT -now :: TimeSpec -now = TimeSpec 0 #const UTIME_NOW +omitTime :: TimeSpec +omitTime = TimeSpec 0 #const UTIME_OMIT +nowTime :: TimeSpec +nowTime = TimeSpec 0 #const UTIME_NOW {- While its interface is beastly, utimensat is in recent POSIX standards, unlike futimes. -} @@ -68,4 +68,4 @@ touchBoth file atime mtime follow = else at_symlink_nofollow touch :: FilePath -> TimeSpec -> Bool -> IO Bool -touch file mtime follow = touchBoth file omit mtime follow +touch file mtime follow = touchBoth file omitTime mtime follow From 58b3e53ff6ee8af6e7a1d67d21c7ce7ae8c6cfda Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Tue, 15 Mar 2011 08:38:53 +0000 Subject: [PATCH 1026/8313] --- doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn diff --git a/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn b/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn new file mode 100644 index 0000000000..60014a7f53 --- /dev/null +++ b/doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn @@ -0,0 +1 @@ +This is just a comment on git-annex building on haskell platform 2011.2.0.0 on archlinux. It just works. From 7b0c6177ff9e0d0f2c23f798b14f1e49128f3589 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 15 Mar 2011 10:49:10 +0000 Subject: [PATCH 1027/8313] --- doc/bugs/softlink_atime.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn index ebb040dd1e..0d26d1ce6a 100644 --- a/doc/bugs/softlink_atime.mdwn +++ b/doc/bugs/softlink_atime.mdwn @@ -35,3 +35,5 @@ Optionally, editing the meta-data should change the times in all annexes. git annex pre-commit . metastore --save git add .metadata + +>>>>>> Thanks a lot. Doing this in a new git-annex repo from the start should at least ensure local consistency and I assume I can simply add a post-pull hook to restore the mtimes on all all other repositories? -- RichiH From 83a9bb624bcd7c5b4eee69bd91150d46c82146d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 11:50:40 -0400 Subject: [PATCH 1028/8313] fix error throwing --- Command/Add.hs | 2 +- Touch.hsc | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Command/Add.hs b/Command/Add.hs index 09fff7cff7..a577203bf4 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -58,7 +58,7 @@ cleanup file key = do -- touch the symlink to have the same mtime as the file it points to s <- liftIO $ getFileStatus file let mtime = modificationTime s - _ <- liftIO $ touch file (TimeSpec mtime 0) False + liftIO $ touch file (TimeSpec mtime 0) False Annex.queue "add" [Param "--"] file return True diff --git a/Touch.hsc b/Touch.hsc index 34b34be515..689c58765f 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -50,15 +50,17 @@ nowTime = TimeSpec 0 #const UTIME_NOW foreign import ccall "utimensat" c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt -{- Changes the access and/or modification times of a file. - Can follow symlinks, or not. -} -touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO Bool +{- Changes the access and/or modification times of an existing file. + Can follow symlinks, or not. Throws IO error on failure. -} +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () touchBoth file atime mtime follow = allocaArray 2 $ \ptr -> withCString file $ \f -> do pokeArray ptr [atime, mtime] r <- c_utimensat at_fdcwd f ptr flags - return (r == 0) + if (r /= 0) + then throwErrno "touchBoth" + else return () where at_fdcwd = #const AT_FDCWD at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW @@ -67,5 +69,5 @@ touchBoth file atime mtime follow = then 0 else at_symlink_nofollow -touch :: FilePath -> TimeSpec -> Bool -> IO Bool +touch :: FilePath -> TimeSpec -> Bool -> IO () touch file mtime follow = touchBoth file omitTime mtime follow From 0e0f85e09d975a6062fb417f8bbae5fbadb6f79f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 17:47:00 -0400 Subject: [PATCH 1029/8313] add hash directory stuff, not used yet --- Locations.hs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Locations.hs b/Locations.hs index 908d5b74ed..91a61ddd7b 100644 --- a/Locations.hs +++ b/Locations.hs @@ -26,6 +26,9 @@ module Locations ( import System.FilePath import Data.String.Utils import Data.List +import Bits +import Word +import Data.Hash.MD5 import Types import qualified GitRepo as Git @@ -128,3 +131,29 @@ fileKey file = read $ prop_idempotent_fileKey :: String -> Bool prop_idempotent_fileKey s = k == fileKey (keyFile k) where k = read $ "test:" ++ s + +{- Given a filename, generates a short directory name to put it in, + - to do hashing to protect against filesystems that dislike having + - many items in a single directory. -} +hashDir :: FilePath -> FilePath +hashDir s = take 2 $ abcd_to_dir $ md5 (Str s) + +abcd_to_dir :: ABCD -> String +abcd_to_dir (ABCD (a,b,c,d)) = concat $ map display_32bits_as_dir [a,b,c,d] + +{- modified version of display_32bits_as_hex from Data.Hash.MD5 + - Copyright (C) 2001 Ian Lynagh + - License: Either BSD or GPL + -} +display_32bits_as_dir :: Word32 -> String +display_32bits_as_dir w = trim $ swap_pairs cs + where + -- Need 32 characters to use. To avoid inaverdently making + -- a real word, use the alphabet without vowels. + chars = ['0'..'9'] ++ "bcdfghjklnmpqrstvwxyzZ" + cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7] + getc n = chars !! (fromIntegral n) + swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs + swap_pairs _ = [] + -- Last 2 will always be 00, so omit. + trim s = take 6 s From fe09c2b7231485afced594cd27582bc6bd32f250 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 17:47:29 -0400 Subject: [PATCH 1030/8313] a new Key data type with metadata --- Key.hs | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ test.hs | 3 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 Key.hs diff --git a/Key.hs b/Key.hs new file mode 100644 index 0000000000..cc4effb11f --- /dev/null +++ b/Key.hs @@ -0,0 +1,88 @@ +{- git-annex Key data type + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Key where + +import Data.String.Utils +import Test.QuickCheck +import Data.Maybe +import Data.List + +{- A Key has a unique name, is associated with a backend, + - and may contain other metadata. -} +data Field = KeyName | KeyBackend | KeySize | KeyModTime + deriving (Eq, Ord, Show) +newtype Key = Key [(Field, String)] + deriving (Eq, Ord) + +{- Generates a Key given a name, a backend and a list of other metadata. -} +keyGen :: String -> String -> [(Field, String)] -> Key +keyGen name backend meta = Key $ (KeyName, name):(KeyBackend, backend):meta + +{- Gets the name of a Key. -} +keyName :: Key -> String +keyName key = fromJust $ keyField key KeyName + +{- Gets the backend associated with a Key. -} +keyBackend :: Key -> String +keyBackend key = fromJust $ keyField key KeyBackend + +{- Looks up a given Field of a Key's metadata. -} +keyField :: Key -> Field -> Maybe String +keyField (Key meta) field = + if null matches + then Nothing + else Just $ snd $ head matches + where + matches = filter match meta + match (f, _) = f == field + +fieldSep :: Char +fieldSep = ',' + +{- Keys show as strings that are suitable for use as filenames. + - The name field is always shown last, and is the only field + - allowed to contain the fieldSep. -} +instance Show Key where + show k@(Key meta) = join [fieldSep] $ map showp meta' ++ [name] + where + name = 'n':keyName k + meta' = sort $ (filter (\(f, _) -> f /= KeyName)) meta + showp (f, v) = (field f) : v + + field KeyBackend = 'b' + field KeySize = 's' + field KeyModTime = 'm' + field f = error $ "unknown key field" ++ show f + +instance Read Key where + readsPrec _ s = [(Key (meta s []), "")] + where + meta (c:r) m = findfield c r m + meta [] m = m + + findfield 'n' v m = (KeyName, v):m -- rest is name + findfield c v m = let (v', _:r) = span (/= fieldSep) v in + meta r (field c v' m) + + field 'b' v m = (KeyBackend, v):m + field 's' v m = (KeySize, v):m + field 'm' v m = (KeyModTime, v):m + field _ _ m = m -- just ignore unparseable fields + +-- for quickcheck +instance Arbitrary Key where + arbitrary = do + backendname <- arbitrary + value <- arbitrary + return $ keyGen value backendname [] + +prop_idempotent_key_read_show :: Key -> Bool +prop_idempotent_key_read_show k + -- backend names will never contain the fieldSep + | fieldSep `elem` (keyBackend k) = True + | otherwise = k == (read $ show k) diff --git a/test.hs b/test.hs index 31960bb2e2..bc849dadc5 100644 --- a/test.hs +++ b/test.hs @@ -38,6 +38,7 @@ import qualified Trust import qualified Remotes import qualified Content import qualified Command.DropUnused +import qualified Key main :: IO () main = do @@ -55,7 +56,7 @@ quickcheck :: Test quickcheck = TestLabel "quickcheck" $ TestList [ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode , qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey - , qctest "prop_idempotent_key_read_show" BackendTypes.prop_idempotent_key_read_show + , qctest "prop_idempotent_key_read_show" Key.prop_idempotent_key_read_show , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics From 940c4e361dd5149d52c773a0e020150d1b5fed56 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 18:15:44 -0400 Subject: [PATCH 1031/8313] cleanup --- Key.hs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Key.hs b/Key.hs index cc4effb11f..cc089104e3 100644 --- a/Key.hs +++ b/Key.hs @@ -60,19 +60,19 @@ instance Show Key where field f = error $ "unknown key field" ++ show f instance Read Key where - readsPrec _ s = [(Key (meta s []), "")] + readsPrec _ s = [(Key (findfields s []), "")] where - meta (c:r) m = findfield c r m - meta [] m = m - - findfield 'n' v m = (KeyName, v):m -- rest is name - findfield c v m = let (v', _:r) = span (/= fieldSep) v in - meta r (field c v' m) + findfields ('n':v) m = (KeyName, v):m -- rest is name + findfields (c:v) m = + case span (/= fieldSep) v of + (v', _:r) -> findfields r (field c v' m) + _ -> m + findfields [] m = m field 'b' v m = (KeyBackend, v):m field 's' v m = (KeySize, v):m field 'm' v m = (KeyModTime, v):m - field _ _ m = m -- just ignore unparseable fields + field _ _ m = m -- for quickcheck instance Arbitrary Key where From 675ee89749ba2272d37b763078020b6e5f4cd380 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 19:11:21 -0400 Subject: [PATCH 1032/8313] redo using record syntax --- Key.hs | 100 ++++++++++++++++++++++++--------------------------------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/Key.hs b/Key.hs index cc089104e3..c542b46ed8 100644 --- a/Key.hs +++ b/Key.hs @@ -7,39 +7,17 @@ module Key where -import Data.String.Utils import Test.QuickCheck -import Data.Maybe -import Data.List +import Utility {- A Key has a unique name, is associated with a backend, - and may contain other metadata. -} -data Field = KeyName | KeyBackend | KeySize | KeyModTime - deriving (Eq, Ord, Show) -newtype Key = Key [(Field, String)] - deriving (Eq, Ord) - -{- Generates a Key given a name, a backend and a list of other metadata. -} -keyGen :: String -> String -> [(Field, String)] -> Key -keyGen name backend meta = Key $ (KeyName, name):(KeyBackend, backend):meta - -{- Gets the name of a Key. -} -keyName :: Key -> String -keyName key = fromJust $ keyField key KeyName - -{- Gets the backend associated with a Key. -} -keyBackend :: Key -> String -keyBackend key = fromJust $ keyField key KeyBackend - -{- Looks up a given Field of a Key's metadata. -} -keyField :: Key -> Field -> Maybe String -keyField (Key meta) field = - if null matches - then Nothing - else Just $ snd $ head matches - where - matches = filter match meta - match (f, _) = f == field +data Key = Key { + keyName :: String, + keyBackend :: String, + keySize :: Maybe Int, + keyMtime :: Maybe Int +} deriving (Eq, Ord) fieldSep :: Char fieldSep = ',' @@ -48,41 +26,47 @@ fieldSep = ',' - The name field is always shown last, and is the only field - allowed to contain the fieldSep. -} instance Show Key where - show k@(Key meta) = join [fieldSep] $ map showp meta' ++ [name] + show Key { keyBackend = b, keySize = s, keyMtime = m, keyName = n } = + ('b' : b) +++ ('s' ?: s) +++ ('m' ?: m) +++ ('n' : n) where - name = 'n':keyName k - meta' = sort $ (filter (\(f, _) -> f /= KeyName)) meta - showp (f, v) = (field f) : v + "" +++ y = y + x +++ "" = x + x +++ y = x ++ fieldSep:y + c ?: (Just v) = c:(show v) + _ ?: _ = "" - field KeyBackend = 'b' - field KeySize = 's' - field KeyModTime = 'm' - field f = error $ "unknown key field" ++ show f +readKey :: String -> Maybe Key +readKey s = if key == stub then Nothing else key + where + key = findfields s stub -instance Read Key where - readsPrec _ s = [(Key (findfields s []), "")] - where - findfields ('n':v) m = (KeyName, v):m -- rest is name - findfields (c:v) m = - case span (/= fieldSep) v of - (v', _:r) -> findfields r (field c v' m) - _ -> m - findfields [] m = m - - field 'b' v m = (KeyBackend, v):m - field 's' v m = (KeySize, v):m - field 'm' v m = (KeyModTime, v):m - field _ _ m = m + stub = Just Key { + keyName = "", + keyBackend = "", + keySize = Nothing, + keyMtime = Nothing + } + + findfields ('n':v) (Just k) = Just $ k { keyName = v } + findfields (c:v) (Just k) = + case span (/= fieldSep) v of + (v', _:r) -> findfields r $ addfield k c v' + _ -> Nothing + findfields _ v = v + + addfield k 'b' v = Just k { keyBackend = v } + addfield k 's' v = Just k { keySize = readMaybe v } + addfield k 'm' v = Just k { keyMtime = readMaybe v } + addfield _ _ _ = Nothing -- for quickcheck instance Arbitrary Key where arbitrary = do - backendname <- arbitrary - value <- arbitrary - return $ keyGen value backendname [] + n <- arbitrary + b <- elements ['A'..'Z'] + s <- arbitrary + m <- arbitrary + return $ Key { keyName = n, keyBackend = [b] , keySize = s, keyMtime = m } prop_idempotent_key_read_show :: Key -> Bool -prop_idempotent_key_read_show k - -- backend names will never contain the fieldSep - | fieldSep `elem` (keyBackend k) = True - | otherwise = k == (read $ show k) +prop_idempotent_key_read_show k = Just k == (readKey $ show k) From 2d5339d7cb887abb8e5baee8ba9244636492f023 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 16 Mar 2011 01:16:48 +0000 Subject: [PATCH 1033/8313] Added a comment --- .../comment_1_ba03333dc76ff49eccaba375e68cb525._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment b/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment new file mode 100644 index 0000000000..261c2a51f3 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-16T01:16:48Z" + content=""" +If you support generic meta-data, keep in mind that you will need to do conflict resolution. Timestamps may not be synched across all systems, so keeping a log of old metadata could be used, sorting by history and using the latest. Which leaves the situation of two incompatible changes. This would probably mean manual conflict resolution. You will probably have thought of this already, but I still wanted to make sure this is recorded. -- RichiH +"""]] From 7d316d0f0de9daced582eb957b0d2b9127a9c6dc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 16 Mar 2011 01:19:26 +0000 Subject: [PATCH 1034/8313] Added a comment --- .../comment_2_81276ac309959dc741bc90101c213ab7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment b/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment new file mode 100644 index 0000000000..9785f1989e --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-03-16T01:19:25Z" + content=""" +Hmm, I added quite a few comments at work, but they are stuck in moderation. Maybe I forgot to log in before adding them. I am surprised this one appeared immediately. -- RichiH +"""]] From 9d49fe2c172b135a1a3735827df014b5f45d99a2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 21:34:13 -0400 Subject: [PATCH 1035/8313] first pass at using new keys It compiles. It sorta works. Several subcommands are FIXME marked and broken, because things that used to accept separate --backend and --key params need to be changed to accept just a --key that encodes all the key info, now that there is metadata in keys. --- Backend.hs | 24 ++++++++++++----------- Backend/SHA.hs | 11 +++++++++-- Backend/WORM.hs | 40 ++++++++++++++++---------------------- BackendTypes.hs | 44 ++---------------------------------------- Command.hs | 13 +++++++++---- Command/DropKey.hs | 6 ++++-- Command/DropUnused.hs | 5 +++-- Command/FromKey.hs | 2 +- Command/InAnnex.hs | 9 +++++++-- Command/Move.hs | 5 +++-- Command/RecvKey.hs | 3 +++ Command/SendKey.hs | 3 +++ Command/Unused.hs | 2 +- Content.hs | 3 ++- Key.hs | 45 +++++++++++++++++++++++++------------------ Locations.hs | 9 +++++---- Makefile | 1 + Remotes.hs | 5 +++-- Types.hs | 6 ++---- Upgrade.hs | 3 ++- 20 files changed, 116 insertions(+), 123 deletions(-) diff --git a/Backend.hs b/Backend.hs index df23e80a33..94755e8d6b 100644 --- a/Backend.hs +++ b/Backend.hs @@ -39,6 +39,7 @@ import Locations import qualified GitRepo as Git import qualified Annex import Types +import Key import qualified BackendTypes as B import Messages @@ -135,18 +136,19 @@ lookupFile file = do getsymlink = do l <- readSymbolicLink file return $ takeFileName l - makekey bs l = do + makekey bs l = + case fileKey l of + Just k -> makeret k l bs + Nothing -> return Nothing + makeret k l bs = case maybeLookupBackendName bs bname of - Nothing -> do - unless (null kname || null bname || - not (isLinkToAnnex l)) $ - warning skip - return Nothing - Just backend -> return $ Just (k, backend) + Just backend -> return $ Just (k, backend) + Nothing -> do + when (isLinkToAnnex l) $ + warning skip + return Nothing where - k = fileKey l - bname = backendName k - kname = keyName k + bname = keyBackendName k skip = "skipping " ++ file ++ " (unknown backend " ++ bname ++ ")" @@ -164,4 +166,4 @@ chooseBackends fs = do keyBackend :: Key -> Annex (Backend Annex) keyBackend key = do bs <- Annex.getState Annex.supportedBackends - return $ lookupBackendName bs $ backendName key + return $ lookupBackendName bs $ keyBackendName key diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 4eea890ce4..3cdc3bf808 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -13,6 +13,7 @@ import System.Cmd.Utils import System.IO import System.Directory import Data.Maybe +import System.Posix.Files import qualified Backend.File import BackendTypes @@ -23,6 +24,7 @@ import Content import Types import Utility import qualified SysConfig +import Key type SHASize = Int @@ -63,11 +65,16 @@ shaN size file = do where command = "sha" ++ (show size) ++ "sum" --- A key is a checksum of its contents. +{- A key is a checksum of its contents. -} keyValue :: SHASize -> FilePath -> Annex (Maybe Key) keyValue size file = do s <- shaN size file - return $ Just $ Key (shaName size, s) + stat <- liftIO $ getFileStatus file + return $ Just $ stubKey { + keyName = s, + keyBackendName = shaName size, + keySize = Just $ fromIntegral $ fileSize stat + } -- A key's checksum is checked during fsck. checkKeyChecksum :: SHASize -> Key -> Annex Bool diff --git a/Backend/WORM.hs b/Backend/WORM.hs index a0d814aa08..324aee76be 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,9 +10,8 @@ module Backend.WORM (backends) where import Control.Monad.State import System.FilePath import System.Posix.Files -import System.Posix.Types import System.Directory -import Data.String.Utils +import Data.Maybe import qualified Backend.File import BackendTypes @@ -21,6 +20,7 @@ import qualified Annex import Content import Messages import Types +import Key backends :: [Backend Annex] backends = [backend] @@ -32,31 +32,25 @@ backend = Backend.File.backend { fsckKey = Backend.File.checkKey checkKeySize } --- The key is formed from the file size, modification time, and the --- basename of the filename. --- --- That allows multiple files with the same names to have different keys, --- while also allowing a file to be moved around while retaining the --- same key. +{- The key includes the file size, modification time, and the + - basename of the filename. + - + - That allows multiple files with the same names to have different keys, + - while also allowing a file to be moved around while retaining the + - same key. + -} keyValue :: FilePath -> Annex (Maybe Key) keyValue file = do stat <- liftIO $ getFileStatus file - return $ Just $ Key (name backend, key stat) - where - key stat = uniqueid stat ++ sep ++ base - uniqueid stat = show (modificationTime stat) ++ sep ++ - show (fileSize stat) - base = takeFileName file - sep = ":" - -{- Extracts the file size from a key. -} -keySize :: Key -> FileOffset -keySize key = read $ section !! 1 - where - section = split ":" (keyName key) + return $ Just $ Key { + keyName = takeFileName file, + keyBackendName = name backend, + keySize = Just $ fromIntegral $ fileSize stat, + keyMtime = Just $ modificationTime stat + } {- The size of the data for a key is checked against the size encoded in - - the key. Note that the modification time is not checked. -} + - the key's metadata. -} checkKeySize :: Key -> Annex Bool checkKeySize key = do g <- Annex.gitRepo @@ -66,7 +60,7 @@ checkKeySize key = do then return True else do s <- liftIO $ getFileStatus file - if fileSize s == keySize key + if fromIntegral (fileSize s) == fromJust (keySize key) then return True else do dest <- moveBad key diff --git a/BackendTypes.hs b/BackendTypes.hs index c0705a550f..48b208a9b4 100644 --- a/BackendTypes.hs +++ b/BackendTypes.hs @@ -1,4 +1,4 @@ -{- git-annex key/value backend data types +{- git-annex key/value backend data type - - Most things should not need this, using Types instead - @@ -9,12 +9,7 @@ module BackendTypes where -import Data.String.Utils -import Test.QuickCheck - -type KeyName = String -type BackendName = String -newtype Key = Key (BackendName, KeyName) deriving (Eq, Ord) +import Key data Backend a = Backend { -- name of this backend @@ -42,38 +37,3 @@ instance Show (Backend a) where instance Eq (Backend a) where a == b = name a == name b - --- accessors for the parts of a key -keyName :: Key -> KeyName -keyName (Key (_,k)) = k -backendName :: Key -> BackendName -backendName (Key (b,_)) = b - --- constructs a key in a backend -genKey :: Backend a -> KeyName -> Key -genKey b f = Key (name b,f) - --- show a key to convert it to a string; the string includes the --- name of the backend to avoid collisions between key strings -instance Show Key where - show (Key (b, k)) = b ++ ":" ++ k - -instance Read Key where - readsPrec _ s = [(Key (b,k), "")] - where - l = split ":" s - b = if null l then "" else head l - k = join ":" $ drop 1 l - --- for quickcheck -instance Arbitrary Key where - arbitrary = do - backendname <- arbitrary - keyname <- arbitrary - return $ Key (backendname, keyname) - -prop_idempotent_key_read_show :: Key -> Bool -prop_idempotent_key_read_show k - -- backend names will never contain colons - | ':' `elem` (backendName k) = True - | otherwise = k == (read $ show k) diff --git a/Command.hs b/Command.hs index eba7f2cef5..38c63bd77e 100644 --- a/Command.hs +++ b/Command.hs @@ -17,11 +17,13 @@ import Data.List import Types import qualified Backend +import qualified BackendTypes import Messages import qualified Annex import qualified GitRepo as Git import Locations import Utility +import Key {- A command runs in four stages. - @@ -233,11 +235,14 @@ cmdlineKey :: Annex Key cmdlineKey = do k <- Annex.getState Annex.defaultkey backends <- Backend.list - return $ genKey (head backends) (keyname' k) + return $ stubKey { + keyName = kname k, + keyBackendName = BackendTypes.name $ head backends + } where - keyname' Nothing = badkey - keyname' (Just "") = badkey - keyname' (Just n) = n + kname Nothing = badkey + kname (Just "") = badkey + kname (Just n) = n badkey = error "please specify the key with --key" {- Given an original list of files, and an expanded list derived from it, diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 8c7566df84..f0450eea34 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -26,8 +26,10 @@ seek = [withKeys start] start :: CommandStartString start keyname = do backends <- Backend.list - let key = genKey (head backends) keyname - present <- inAnnex key + let key = error "fixme!!" + --let key = genKey (head backends) keyname --TODO FIXME + let present = error "fixme!!" + --present <- inAnnex key force <- Annex.getState Annex.force if not present then return Nothing diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 594564cb72..8ed61ba65b 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -11,6 +11,7 @@ import Control.Monad (when) import Control.Monad.State (liftIO) import qualified Data.Map as M import System.Directory +import Data.Maybe import Command import Types @@ -19,6 +20,7 @@ import Locations import qualified Annex import qualified Command.Drop import Backend +import Key command :: [Command] command = [Command "dropunused" (paramRepeating paramNumber) seek @@ -55,7 +57,6 @@ readUnusedLog = do return $ M.fromList $ map parse $ lines l else return $ M.empty where - parse line = (head ws, tokey $ unwords $ tail ws) + parse line = (head ws, fromJust $ readKey $ unwords $ tail ws) where ws = words line - tokey s = read s :: Key diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 717d528bc9..176d2cd54d 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -16,9 +16,9 @@ import Command import qualified Annex import Utility import qualified Backend -import Types import Content import Messages +import Key command :: [Command] command = [Command "fromkey" paramPath seek diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index 68ac9a2c67..4a4102754a 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -11,9 +11,10 @@ import Control.Monad.State (liftIO) import System.Exit import Command -import Types import Content import qualified Backend +import qualified BackendTypes +import Key command :: [Command] command = [Command "inannex" (paramRepeating paramKey) seek @@ -25,7 +26,11 @@ seek = [withKeys start] start :: CommandStartString start keyname = do backends <- Backend.list - let key = genKey (head backends) keyname + let key = stubKey { + keyName = keyname, + keyBackendName = BackendTypes.name (head backends) + } + error "BROKEN. fixme!" present <- inAnnex key if present then return Nothing diff --git a/Command/Move.hs b/Command/Move.hs index 3774ccbe9d..1b14813089 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,7 +20,8 @@ import qualified Remotes import UUID import Messages import Utility - +import Key + command :: [Command] command = [Command "move" paramPath seek "move content of files to/from another repository"] @@ -136,7 +137,7 @@ fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup fromCleanup src True key = do ok <- Remotes.onRemote src (boolSystem, False) "dropkey" [ Params "--quiet --force" - , Param $ "--backend=" ++ backendName key + , Param $ "--backend=" ++ keyBackendName key , Param $ keyName key ] -- better safe than sorry: assume the src dropped the key diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 8a96730503..488bab62d6 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -27,6 +27,8 @@ seek = [withKeys start] start :: CommandStartString start keyname = do + error "BROKEN FIXME!" + {- backends <- Backend.list let key = genKey (head backends) keyname present <- inAnnex key @@ -41,3 +43,4 @@ start keyname = do _ <- shutdown liftIO exitSuccess else liftIO exitFailure + -} diff --git a/Command/SendKey.hs b/Command/SendKey.hs index cb883b53aa..ff269f21fe 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -28,6 +28,8 @@ seek = [withKeys start] start :: CommandStartString start keyname = do + error "BROKEN FIXME!" + {- backends <- Backend.list let key = genKey (head backends) keyname present <- inAnnex key @@ -36,3 +38,4 @@ start keyname = do when present $ liftIO $ rsyncServerSend file liftIO exitFailure + -} diff --git a/Command/Unused.hs b/Command/Unused.hs index a614ce5d94..52e483d870 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -126,4 +126,4 @@ tmpKeys = do contents <- liftIO $ getDirectoryContents tmp files <- liftIO $ filterM doesFileExist $ map (tmp ) contents - return $ map (fileKey . takeFileName) files + return $ catMaybes $ map (fileKey . takeFileName) files diff --git a/Content.hs b/Content.hs index dc675389f1..1a5a80a9f1 100644 --- a/Content.hs +++ b/Content.hs @@ -26,6 +26,7 @@ import System.Path import Control.Monad (when, unless, filterM) import System.Posix.Files import System.FilePath +import Data.Maybe import Types import Locations @@ -162,7 +163,7 @@ getKeysPresent' dir = do else do contents <- liftIO $ getDirectoryContents dir files <- liftIO $ filterM present contents - return $ map fileKey files + return $ catMaybes $ map fileKey files where present d = do result <- try $ diff --git a/Key.hs b/Key.hs index c542b46ed8..178f1ca69e 100644 --- a/Key.hs +++ b/Key.hs @@ -5,20 +5,35 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Key where +module Key ( + Key(..), + stubKey, + readKey, + + prop_idempotent_key_read_show +) where import Test.QuickCheck import Utility +import System.Posix.Types -{- A Key has a unique name, is associated with a backend, - - and may contain other metadata. -} +{- A Key has a unique name, is associated with a key/value backend, + - and may contain other optional metadata. -} data Key = Key { keyName :: String, - keyBackend :: String, - keySize :: Maybe Int, - keyMtime :: Maybe Int + keyBackendName :: String, + keySize :: Maybe Integer, + keyMtime :: Maybe EpochTime } deriving (Eq, Ord) +stubKey :: Key +stubKey = Key { + keyName = "", + keyBackendName = "", + keySize = Nothing, + keyMtime = Nothing +} + fieldSep :: Char fieldSep = ',' @@ -26,7 +41,7 @@ fieldSep = ',' - The name field is always shown last, and is the only field - allowed to contain the fieldSep. -} instance Show Key where - show Key { keyBackend = b, keySize = s, keyMtime = m, keyName = n } = + show Key { keyBackendName = b, keySize = s, keyMtime = m, keyName = n } = ('b' : b) +++ ('s' ?: s) +++ ('m' ?: m) +++ ('n' : n) where "" +++ y = y @@ -36,16 +51,9 @@ instance Show Key where _ ?: _ = "" readKey :: String -> Maybe Key -readKey s = if key == stub then Nothing else key +readKey s = if key == Just stubKey then Nothing else key where - key = findfields s stub - - stub = Just Key { - keyName = "", - keyBackend = "", - keySize = Nothing, - keyMtime = Nothing - } + key = findfields s $ Just stubKey findfields ('n':v) (Just k) = Just $ k { keyName = v } findfields (c:v) (Just k) = @@ -54,7 +62,7 @@ readKey s = if key == stub then Nothing else key _ -> Nothing findfields _ v = v - addfield k 'b' v = Just k { keyBackend = v } + addfield k 'b' v = Just k { keyBackendName = v } addfield k 's' v = Just k { keySize = readMaybe v } addfield k 'm' v = Just k { keyMtime = readMaybe v } addfield _ _ _ = Nothing @@ -65,8 +73,7 @@ instance Arbitrary Key where n <- arbitrary b <- elements ['A'..'Z'] s <- arbitrary - m <- arbitrary - return $ Key { keyName = n, keyBackend = [b] , keySize = s, keyMtime = m } + return $ Key { keyName = n, keyBackendName = [b] , keySize = s } prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k = Just k == (readKey $ show k) diff --git a/Locations.hs b/Locations.hs index 91a61ddd7b..6cff910880 100644 --- a/Locations.hs +++ b/Locations.hs @@ -31,6 +31,7 @@ import Word import Data.Hash.MD5 import Types +import Key import qualified GitRepo as Git {- Conventions: @@ -123,14 +124,14 @@ keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} -fileKey :: FilePath -> Key -fileKey file = read $ +fileKey :: FilePath -> Maybe Key +fileKey file = readKey $ replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool -prop_idempotent_fileKey s = k == fileKey (keyFile k) - where k = read $ "test:" ++ s +prop_idempotent_fileKey s = Just k == fileKey (keyFile k) + where k = stubKey { keyName = s, keyBackendName = "test" } {- Given a filename, generates a short directory name to put it in, - to do hashing to protect against filesystems that dislike having diff --git a/Makefile b/Makefile index c381ae986d..c60e19b311 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ SysConfig.hs: configure.hs TestConfig.hs Touch.hs: Touch.hsc hsc2hs $< + perl -i -pe 's/^{-# INCLUDE.*//' $@ $(bins): SysConfig.hs Touch.hs $(GHCMAKE) $@ diff --git a/Remotes.hs b/Remotes.hs index 3c9db314c8..dd733e4545 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -27,6 +27,7 @@ import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) import Types +import Key import qualified GitRepo as Git import qualified Annex import LocationLog @@ -153,7 +154,7 @@ inAnnex r key = if Git.repoIsUrl r checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" - [Param ("--backend=" ++ backendName key), Param (keyName key)] + [Param ("--backend=" ++ keyBackendName key), Param (keyName key)] return $ Right inannex {- Cost Ordered list of remotes. -} @@ -272,7 +273,7 @@ rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] rsyncParams r sending key file = do Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - [ Param $ "--backend=" ++ backendName key + [ Param $ "--backend=" ++ keyBackendName key , Param $ keyName key -- Command is terminated with "--", because -- rsync will tack on its own options afterwards, diff --git a/Types.hs b/Types.hs index 0890efd5e3..f48d4079be 100644 --- a/Types.hs +++ b/Types.hs @@ -8,11 +8,9 @@ module Types ( Annex, Backend, - Key, - genKey, - backendName, - keyName + Key ) where import BackendTypes import Annex +import Key diff --git a/Upgrade.hs b/Upgrade.hs index 3c16bcc862..7469d9ba75 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -13,6 +13,7 @@ import Control.Monad.State (liftIO) import Control.Monad (filterM, forM_) import System.Posix.Files import System.FilePath +import Data.Maybe import Content import Types @@ -74,7 +75,7 @@ getKeysPresent0' dir = do else do contents <- liftIO $ getDirectoryContents dir files <- liftIO $ filterM present contents - return $ map fileKey files + return $ catMaybes $ map fileKey files where present d = do result <- try $ From f27df5e6584ed6a09be4a9bb1030be132c2d8051 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 21:54:38 -0400 Subject: [PATCH 1036/8313] improve key filenames --- Key.hs | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Key.hs b/Key.hs index 178f1ca69e..3385148707 100644 --- a/Key.hs +++ b/Key.hs @@ -35,14 +35,14 @@ stubKey = Key { } fieldSep :: Char -fieldSep = ',' +fieldSep = '-' {- Keys show as strings that are suitable for use as filenames. - - The name field is always shown last, and is the only field - - allowed to contain the fieldSep. -} + - The name field is always shown last, separated by doubled fieldSeps, + - and is the only field allowed to contain the fieldSep. -} instance Show Key where show Key { keyBackendName = b, keySize = s, keyMtime = m, keyName = n } = - ('b' : b) +++ ('s' ?: s) +++ ('m' ?: m) +++ ('n' : n) + b +++ ('s' ?: s) +++ ('m' ?: m) +++ (fieldSep : n) where "" +++ y = y x +++ "" = x @@ -53,18 +53,22 @@ instance Show Key where readKey :: String -> Maybe Key readKey s = if key == Just stubKey then Nothing else key where - key = findfields s $ Just stubKey + key = startbackend stubKey s - findfields ('n':v) (Just k) = Just $ k { keyName = v } - findfields (c:v) (Just k) = - case span (/= fieldSep) v of - (v', _:r) -> findfields r $ addfield k c v' - _ -> Nothing - findfields _ v = v + startbackend k v = sepfield k v addbackend - addfield k 'b' v = Just k { keyBackendName = v } - addfield k 's' v = Just k { keySize = readMaybe v } - addfield k 'm' v = Just k { keyMtime = readMaybe v } + sepfield k v a = case span (/= fieldSep) v of + (v', _:r) -> findfields r $ a k v' + _ -> Nothing + + findfields (c:v) (Just k) + | c == fieldSep = Just $ k { keyName = v } + | otherwise = sepfield k v $ addfield c + findfields _ v = v + + addbackend k v = Just k { keyBackendName = v } + addfield 's' k v = Just k { keySize = readMaybe v } + addfield 'm' k v = Just k { keyMtime = readMaybe v } addfield _ _ _ = Nothing -- for quickcheck @@ -73,7 +77,12 @@ instance Arbitrary Key where n <- arbitrary b <- elements ['A'..'Z'] s <- arbitrary - return $ Key { keyName = n, keyBackendName = [b] , keySize = s } + return $ Key { + keyName = n, + keyBackendName = [b], + keySize = s, + keyMtime = Nothing + } prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k = Just k == (readKey $ show k) From 4594bd51c1754bc7e1fdf03d83569aeea163e761 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:04:50 -0400 Subject: [PATCH 1037/8313] rename file --- Annex.hs | 10 +++++----- Backend.hs | 2 +- Backend/File.hs | 2 +- Backend/SHA.hs | 2 +- Backend/URL.hs | 2 +- Backend/WORM.hs | 2 +- BackendTypes.hs => BackendClass.hs | 2 +- Command.hs | 4 ++-- Command/InAnnex.hs | 4 ++-- Key.hs | 3 +-- Types.hs | 2 +- test.hs | 8 ++++---- 12 files changed, 21 insertions(+), 22 deletions(-) rename BackendTypes.hs => BackendClass.hs (97%) diff --git a/Annex.hs b/Annex.hs index dd3362b29d..f8cfd0ec92 100644 --- a/Annex.hs +++ b/Annex.hs @@ -25,7 +25,7 @@ import Data.Maybe import qualified GitRepo as Git import qualified GitQueue -import qualified BackendTypes +import qualified BackendClass import Utility -- git-annex's monad @@ -34,8 +34,8 @@ type Annex = StateT AnnexState IO -- internal state storage data AnnexState = AnnexState { repo :: Git.Repo - , backends :: [BackendTypes.Backend Annex] - , supportedBackends :: [BackendTypes.Backend Annex] + , backends :: [BackendClass.Backend Annex] + , supportedBackends :: [BackendClass.Backend Annex] , repoqueue :: GitQueue.Queue , quiet :: Bool , force :: Bool @@ -47,7 +47,7 @@ data AnnexState = AnnexState , remotesread :: Bool } deriving (Show) -newState :: Git.Repo -> [BackendTypes.Backend Annex] -> AnnexState +newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState newState gitrepo allbackends = AnnexState { repo = gitrepo , backends = [] @@ -64,7 +64,7 @@ newState gitrepo allbackends = AnnexState } {- Create and returns an Annex state object for the specified git repo. -} -new :: Git.Repo -> [BackendTypes.Backend Annex] -> IO AnnexState +new :: Git.Repo -> [BackendClass.Backend Annex] -> IO AnnexState new gitrepo allbackends = do gitrepo' <- liftIO $ Git.configRead gitrepo return $ newState gitrepo' allbackends diff --git a/Backend.hs b/Backend.hs index 94755e8d6b..e1f8f388b9 100644 --- a/Backend.hs +++ b/Backend.hs @@ -40,7 +40,7 @@ import qualified GitRepo as Git import qualified Annex import Types import Key -import qualified BackendTypes as B +import qualified BackendClass as B import Messages {- List of backends in the order to try them when storing a new key. -} diff --git a/Backend/File.hs b/Backend/File.hs index d76cd29391..a5e2431998 100644 --- a/Backend/File.hs +++ b/Backend/File.hs @@ -18,7 +18,7 @@ import Control.Monad.State import System.Directory import Data.List -import BackendTypes +import BackendClass import LocationLog import Locations import qualified Remotes diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 3cdc3bf808..0563851076 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -16,7 +16,7 @@ import Data.Maybe import System.Posix.Files import qualified Backend.File -import BackendTypes +import BackendClass import Messages import qualified Annex import Locations diff --git a/Backend/URL.hs b/Backend/URL.hs index 29dc8fefa7..b40ff39599 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import Data.String.Utils import Types -import BackendTypes +import BackendClass import Utility import Messages diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 324aee76be..a011995da3 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -14,7 +14,7 @@ import System.Directory import Data.Maybe import qualified Backend.File -import BackendTypes +import BackendClass import Locations import qualified Annex import Content diff --git a/BackendTypes.hs b/BackendClass.hs similarity index 97% rename from BackendTypes.hs rename to BackendClass.hs index 48b208a9b4..909ae8f96e 100644 --- a/BackendTypes.hs +++ b/BackendClass.hs @@ -7,7 +7,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module BackendTypes where +module BackendClass where import Key diff --git a/Command.hs b/Command.hs index 38c63bd77e..18b4a167aa 100644 --- a/Command.hs +++ b/Command.hs @@ -17,7 +17,7 @@ import Data.List import Types import qualified Backend -import qualified BackendTypes +import qualified BackendClass import Messages import qualified Annex import qualified GitRepo as Git @@ -237,7 +237,7 @@ cmdlineKey = do backends <- Backend.list return $ stubKey { keyName = kname k, - keyBackendName = BackendTypes.name $ head backends + keyBackendName = BackendClass.name $ head backends } where kname Nothing = badkey diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index 4a4102754a..a2beda4a59 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -13,7 +13,7 @@ import System.Exit import Command import Content import qualified Backend -import qualified BackendTypes +import qualified BackendClass import Key command :: [Command] @@ -28,7 +28,7 @@ start keyname = do backends <- Backend.list let key = stubKey { keyName = keyname, - keyBackendName = BackendTypes.name (head backends) + keyBackendName = BackendClass.name (head backends) } error "BROKEN. fixme!" present <- inAnnex key diff --git a/Key.hs b/Key.hs index 3385148707..f52aea31b7 100644 --- a/Key.hs +++ b/Key.hs @@ -76,11 +76,10 @@ instance Arbitrary Key where arbitrary = do n <- arbitrary b <- elements ['A'..'Z'] - s <- arbitrary return $ Key { keyName = n, keyBackendName = [b], - keySize = s, + keySize = Nothing, keyMtime = Nothing } diff --git a/Types.hs b/Types.hs index f48d4079be..503e27d312 100644 --- a/Types.hs +++ b/Types.hs @@ -11,6 +11,6 @@ module Types ( Key ) where -import BackendTypes +import BackendClass import Annex import Key diff --git a/test.hs b/test.hs index bc849dadc5..2bc0c37a37 100644 --- a/test.hs +++ b/test.hs @@ -29,7 +29,7 @@ import qualified Backend import qualified GitRepo as Git import qualified Locations import qualified Utility -import qualified BackendTypes +import qualified BackendClass import qualified Types import qualified GitAnnex import qualified LocationLog @@ -120,8 +120,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup] test_setkey :: Test test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do writeFile tmp $ content sha1annexedfile - r <- annexeval $ BackendTypes.getKey backendSHA1 tmp - let sha1 = BackendTypes.keyName $ fromJust r + r <- annexeval $ BackendClass.getKey backendSHA1 tmp + let sha1 = Key.keyName $ fromJust r git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" @@ -439,7 +439,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also - git_annex "dropkey" ["-q", "--force", BackendTypes.keyName annexedfilekey] + git_annex "dropkey" ["-q", "--force", Key.keyName annexedfilekey] @? "dropkey failed" checkunused [sha1annexedfilekey] From 4651688290086d70275e3d0b2976dbc57f8e4df1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:08:19 -0400 Subject: [PATCH 1038/8313] increase repo version --- Upgrade.hs | 6 ++++++ Version.hs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Upgrade.hs b/Upgrade.hs index 7469d9ba75..eba75bf587 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -31,10 +31,16 @@ upgrade = do version <- getVersion case version of Just "0" -> upgradeFrom0 + Just "1" -> upgradeFrom1 Nothing -> return True -- repo not initted yet, no version Just v | v == currentVersion -> return True Just _ -> error "this version of git-annex is too old for this git repository!" +upgradeFrom1 :: Annex Bool +upgradeFrom1 = do + showSideAction "Upgrading object directory layout..." + error "upgradeFrom1 TODO FIXME" + upgradeFrom0 :: Annex Bool upgradeFrom0 = do showSideAction "Upgrading object directory layout..." diff --git a/Version.hs b/Version.hs index 9e31d3c9eb..7fdbd1a49a 100644 --- a/Version.hs +++ b/Version.hs @@ -16,7 +16,7 @@ import qualified GitRepo as Git import Locations currentVersion :: String -currentVersion = "1" +currentVersion = "2" versionField :: String versionField = "annex.version" From 27472710c73320449ce96bc409fa82e0a46be1cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:19:44 -0400 Subject: [PATCH 1039/8313] initial pass at doc update --- debian/changelog | 3 +++ doc/git-annex.mdwn | 14 ++++---------- doc/walkthrough/modifying_annexed_files.mdwn | 2 +- .../moving_file_content_between_repositories.mdwn | 2 +- doc/walkthrough/unused_data.mdwn | 4 ++-- doc/walkthrough/using_ssh_remotes.mdwn | 2 +- doc/walkthrough/using_the_URL_backend.mdwn | 2 +- 7 files changed, 13 insertions(+), 16 deletions(-) diff --git a/debian/changelog b/debian/changelog index e7017a26d0..0d1832374d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,8 @@ git-annex (0.24) UNRELEASED; urgency=low + * Reorganized annexed object store. annex.version=2 + * The setkey, fromkey, and dropkey subcommands have changed how + the key is specified. --backend is no longer used with these. * Add Suggests on graphviz. Closes: #618039 * When adding files to the annex, the symlinks pointing at the annexed content are made to have the same mtime as the original file. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 4998a64911..f2ec5fd15e 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -234,11 +234,11 @@ Many git-annex commands will stage changes for later `git commit` by you. This can be used to manually set up a file to link to a specified key in the key-value backend. How you determine an existing key in the backend - varies. For the URL backend, the key is just a URL to the content. + varies. For the URL backend, the key is based on an URL to the content. Example: - git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + git annex fromkey --key=URL--http://www.archive.org/somefile somefile * dropkey [key ...] @@ -248,24 +248,18 @@ Many git-annex commands will stage changes for later `git commit` by you. This can be used to drop content for arbitrary keys, which do not need to have a file in the git repository pointing at them. - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. - Example: - git annex dropkey --backend=SHA1 7da006579dd64330eb2456001fd01948430572f2 + git annex dropkey --key=SHA1-s10-7da006579dd64330eb2456001fd01948430572f2 * setkey file This plumbing-level command sets the annexed data for a key to the content of the specified file, and then removes the file. - A backend will typically need to be specified with --backend. If none - is specified, the first configured backend is used. - Example: - git annex setkey --backend=WORM --key=1287765018:3 /tmp/file + git annex setkey --key=WORM-s3-m1287765018--file /tmp/file # OPTIONS diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn index 3ad4e82eab..f75b73a24c 100644 --- a/doc/walkthrough/modifying_annexed_files.mdwn +++ b/doc/walkthrough/modifying_annexed_files.mdwn @@ -27,7 +27,7 @@ and this symlink is what gets committed to git in the end. add my_cool_big_file ok [master 64cda67] changed an annexed file 2 files changed, 2 insertions(+), 1 deletions(-) - create mode 100644 .git-annex/WORM:1289672605:30:file.log + create mode 100644 .git-annex/WORM-s30-m1289672605--file.log There is one problem with using `git commit` like this: Git wants to first stage the entire contents of the file in its index. That can be slow for diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn index d7150f109a..6b3e3f4e80 100644 --- a/doc/walkthrough/moving_file_content_between_repositories.mdwn +++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn @@ -9,5 +9,5 @@ makes it very easy. move my_cool_big_file (moving to usbdrive...) ok # git annex move video/hackity_hack_and_kaxxt.mov --from fileserver move video/hackity_hack_and_kaxxt.mov (moving from fileserver...) - WORM:1274316523:86050597:hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 + WORM-s86050597-m1274316523--hackity_hack_and_kax 100% 82MB 199.1KB/s 07:02 ok diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn index 69a581fe1e..9be32577c3 100644 --- a/doc/walkthrough/unused_data.mdwn +++ b/doc/walkthrough/unused_data.mdwn @@ -12,8 +12,8 @@ eliminate it to save space. unused (checking for unused data...) Some annexed data is no longer pointed to by any files in the repository. NUMBER KEY - 1 WORM:1289672605:3:file - 2 WORM:1289672605:14:file + 1 WORM-s3-m1289672605--file + 2 WORM-s14-m1289672605--file (To see where data was previously used, try: git log --stat -S'KEY') (To remove unwanted data: git-annex dropunused NUMBER) ok diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn index 6af9e1f477..4c2f830de8 100644 --- a/doc/walkthrough/using_ssh_remotes.mdwn +++ b/doc/walkthrough/using_ssh_remotes.mdwn @@ -13,7 +13,7 @@ Now you can get files and they will be transferred (using `rsync` via `ssh`): # git annex get my_cool_big_file get my_cool_big_file (getting UUID for origin...) (copying from origin...) - WORM:1285650548:2159:my_cool_big_file 100% 2159 2.1KB/s 00:00 + WORM-s2159-m1285650548--my_cool_big_file 100% 2159 2.1KB/s 00:00 ok When you drop files, git-annex will ssh over to the remote and make diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn index fe79a6be2e..585fd0668a 100644 --- a/doc/walkthrough/using_the_URL_backend.mdwn +++ b/doc/walkthrough/using_the_URL_backend.mdwn @@ -5,7 +5,7 @@ Another handy backend is the URL backend, which can fetch file's content from remote URLs. Here's how to set up some files in your repository that use this backend: - # git annex fromkey --backend=URL --key=http://www.archive.org/somefile somefile + # git annex fromkey --key=URL--http://www.archive.org/somefile somefile fromkey somefile ok # git commit -m "added a file from the Internet Archive" From da504f647fdbec7aa3a3c08244520de2c00898ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:28:18 -0400 Subject: [PATCH 1040/8313] fromkey, and url backend download work now --- Backend/URL.hs | 6 +++--- Command.hs | 19 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index b40ff39599..02ce3563cd 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -8,12 +8,12 @@ module Backend.URL (backends) where import Control.Monad.State (liftIO) -import Data.String.Utils import Types import BackendClass import Utility import Messages +import Key backends :: [Backend Annex] backends = [backend] @@ -52,8 +52,8 @@ dummyOk _ = return True downloadUrl :: Key -> FilePath -> Annex Bool downloadUrl key file = do - showNote "downloading" + showNote $ "downloading" showProgress -- make way for curl progress bar liftIO $ boolSystem "curl" [Params "-# -o", File file, File url] where - url = join ":" $ drop 1 $ split ":" $ show key + url = keyName key diff --git a/Command.hs b/Command.hs index 18b4a167aa..27598fff0a 100644 --- a/Command.hs +++ b/Command.hs @@ -17,7 +17,6 @@ import Data.List import Types import qualified Backend -import qualified BackendClass import Messages import qualified Annex import qualified GitRepo as Git @@ -230,20 +229,18 @@ paramName = "NAME" paramNothing :: String paramNothing = "" -{- The Key specified by the --key and --backend parameters. -} +{- The Key specified by the --key parameter. -} cmdlineKey :: Annex Key cmdlineKey = do k <- Annex.getState Annex.defaultkey - backends <- Backend.list - return $ stubKey { - keyName = kname k, - keyBackendName = BackendClass.name $ head backends - } + case k of + Nothing -> nokey + Just "" -> nokey + Just kstring -> case readKey kstring of + Nothing -> error "bad key" + Just key -> return key where - kname Nothing = badkey - kname (Just "") = badkey - kname (Just n) = n - badkey = error "please specify the key with --key" + nokey = error "please specify the key with --key" {- Given an original list of files, and an expanded list derived from it, - ensures that the original list's ordering is preserved. From 2e1cc2f8b98aaf9e01b620d557c42d5b1ae2aaa6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:42:34 -0400 Subject: [PATCH 1041/8313] fixed dropkey, setkey, and git-annex-shell subcommands key is now specified as the full key, no --backend needed --- Command.hs | 7 +++++-- Command/DropKey.hs | 14 +++++--------- Command/InAnnex.hs | 13 ++----------- Command/RecvKey.hs | 11 ++--------- Command/SendKey.hs | 11 ++--------- doc/git-annex.mdwn | 2 +- 6 files changed, 17 insertions(+), 41 deletions(-) diff --git a/Command.hs b/Command.hs index 27598fff0a..c3cb612ee0 100644 --- a/Command.hs +++ b/Command.hs @@ -14,6 +14,7 @@ import Control.Monad (filterM, liftM, when) import System.Path.WildMatch import Text.Regex.PCRE.Light.Char8 import Data.List +import Data.Maybe import Types import qualified Backend @@ -46,6 +47,8 @@ type CommandCleanup = Annex Bool - functions. -} type CommandSeekStrings = CommandStartString -> CommandSeek type CommandStartString = String -> CommandStart +type CommandSeekKeys = CommandStartKey -> CommandSeek +type CommandStartKey = Key -> CommandStart type BackendFile = (FilePath, Maybe (Backend Annex)) type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek type CommandStartBackendFile = BackendFile -> CommandStart @@ -167,8 +170,8 @@ withFilesUnlocked' typechanged a params = do map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' -withKeys :: CommandSeekStrings -withKeys a params = return $ map a params +withKeys :: CommandSeekKeys +withKeys a params = return $ map a $ catMaybes $ map readKey params withTempFile :: CommandSeekStrings withTempFile a params = return $ map a params withNothing :: CommandSeekNothing diff --git a/Command/DropKey.hs b/Command/DropKey.hs index f0450eea34..419d9caa43 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -14,6 +14,7 @@ import LocationLog import Types import Content import Messages +import Key command :: [Command] command = [Command "dropkey" (paramRepeating paramKey) seek @@ -22,21 +23,16 @@ command = [Command "dropkey" (paramRepeating paramKey) seek seek :: [CommandSeek] seek = [withKeys start] -{- Drops cached content for a key. -} -start :: CommandStartString -start keyname = do - backends <- Backend.list - let key = error "fixme!!" - --let key = genKey (head backends) keyname --TODO FIXME - let present = error "fixme!!" - --present <- inAnnex key +start :: CommandStartKey +start key = do + present <- inAnnex key force <- Annex.getState Annex.force if not present then return Nothing else if not force then error "dropkey is can cause data loss; use --force if you're sure you want to do this" else do - showStart "dropkey" keyname + showStart "dropkey" (show key) return $ Just $ perform key perform :: Key -> CommandPerform diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs index a2beda4a59..fa81fc9a4c 100644 --- a/Command/InAnnex.hs +++ b/Command/InAnnex.hs @@ -12,9 +12,6 @@ import System.Exit import Command import Content -import qualified Backend -import qualified BackendClass -import Key command :: [Command] command = [Command "inannex" (paramRepeating paramKey) seek @@ -23,14 +20,8 @@ command = [Command "inannex" (paramRepeating paramKey) seek seek :: [CommandSeek] seek = [withKeys start] -start :: CommandStartString -start keyname = do - backends <- Backend.list - let key = stubKey { - keyName = keyname, - keyBackendName = BackendClass.name (head backends) - } - error "BROKEN. fixme!" +start :: CommandStartKey +start key = do present <- inAnnex key if present then return Nothing diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 488bab62d6..c7c37d1e31 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -12,10 +12,8 @@ import Control.Monad.State (liftIO) import System.Exit import Command -import Types import CmdLine import Content -import qualified Backend import RsyncFile command :: [Command] @@ -25,12 +23,8 @@ command = [Command "recvkey" paramKey seek seek :: [CommandSeek] seek = [withKeys start] -start :: CommandStartString -start keyname = do - error "BROKEN FIXME!" - {- - backends <- Backend.list - let key = genKey (head backends) keyname +start :: CommandStartKey +start key = do present <- inAnnex key when present $ error "key is already present in annex" @@ -43,4 +37,3 @@ start keyname = do _ <- shutdown liftIO exitSuccess else liftIO exitFailure - -} diff --git a/Command/SendKey.hs b/Command/SendKey.hs index ff269f21fe..56974bda96 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -14,9 +14,7 @@ import System.Exit import Locations import qualified Annex import Command -import Types import Content -import qualified Backend import RsyncFile command :: [Command] @@ -26,16 +24,11 @@ command = [Command "sendkey" paramKey seek seek :: [CommandSeek] seek = [withKeys start] -start :: CommandStartString -start keyname = do - error "BROKEN FIXME!" - {- - backends <- Backend.list - let key = genKey (head backends) keyname +start :: CommandStartKey +start key = do present <- inAnnex key g <- Annex.gitRepo let file = gitAnnexLocation g key when present $ liftIO $ rsyncServerSend file liftIO exitFailure - -} diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index f2ec5fd15e..bfec527d9a 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -250,7 +250,7 @@ Many git-annex commands will stage changes for later `git commit` by you. Example: - git annex dropkey --key=SHA1-s10-7da006579dd64330eb2456001fd01948430572f2 + git annex dropkey SHA1-s10-7da006579dd64330eb2456001fd01948430572f2 * setkey file From 9d24cc7bdb011d66e41229a3b96401808be47268 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:46:47 -0400 Subject: [PATCH 1042/8313] make commands that take a key as a parameter error if it's bad --- Command.hs | 7 +++++-- Command/DropKey.hs | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Command.hs b/Command.hs index c3cb612ee0..41ad884a93 100644 --- a/Command.hs +++ b/Command.hs @@ -14,7 +14,6 @@ import Control.Monad (filterM, liftM, when) import System.Path.WildMatch import Text.Regex.PCRE.Light.Char8 import Data.List -import Data.Maybe import Types import qualified Backend @@ -171,7 +170,11 @@ withFilesUnlocked' typechanged a params = do unlockedfiles' <- filterFiles unlockedfiles backendPairs a unlockedfiles' withKeys :: CommandSeekKeys -withKeys a params = return $ map a $ catMaybes $ map readKey params +withKeys a params = return $ map a $ map parse params + where + parse p = case readKey p of + Just k -> k + Nothing -> error "bad key" withTempFile :: CommandSeekStrings withTempFile a params = return $ map a params withNothing :: CommandSeekNothing diff --git a/Command/DropKey.hs b/Command/DropKey.hs index 419d9caa43..b3cc60961c 100644 --- a/Command/DropKey.hs +++ b/Command/DropKey.hs @@ -9,12 +9,10 @@ module Command.DropKey where import Command import qualified Annex -import qualified Backend import LocationLog import Types import Content import Messages -import Key command :: [Command] command = [Command "dropkey" (paramRepeating paramKey) seek From 49b7f5918341c30140779ea1f376b4d9f81d8a30 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 22:53:14 -0400 Subject: [PATCH 1043/8313] test suite passes again doesn't test remote functionality.. but that may be working too now --- Command/Move.hs | 4 +--- Remotes.hs | 6 ++---- doc/git-annex.mdwn | 5 ++++- test.hs | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 1b14813089..2d6c973fe0 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -20,7 +20,6 @@ import qualified Remotes import UUID import Messages import Utility -import Key command :: [Command] command = [Command "move" paramPath seek @@ -137,8 +136,7 @@ fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup fromCleanup src True key = do ok <- Remotes.onRemote src (boolSystem, False) "dropkey" [ Params "--quiet --force" - , Param $ "--backend=" ++ keyBackendName key - , Param $ keyName key + , Param $ show key ] -- better safe than sorry: assume the src dropped the key -- even if it seemed to fail; the failure could have occurred diff --git a/Remotes.hs b/Remotes.hs index dd733e4545..8b760ac957 100644 --- a/Remotes.hs +++ b/Remotes.hs @@ -27,7 +27,6 @@ import Data.List (intersect, sortBy) import Control.Monad (when, unless, filterM) import Types -import Key import qualified GitRepo as Git import qualified Annex import LocationLog @@ -154,7 +153,7 @@ inAnnex r key = if Git.repoIsUrl r checkremote = do showNote ("checking " ++ Git.repoDescribe r ++ "...") inannex <- onRemote r (boolSystem, False) "inannex" - [Param ("--backend=" ++ keyBackendName key), Param (keyName key)] + [Param (show key)] return $ Right inannex {- Cost Ordered list of remotes. -} @@ -273,8 +272,7 @@ rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] rsyncParams r sending key file = do Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - [ Param $ "--backend=" ++ keyBackendName key - , Param $ keyName key + [ Param $ show key -- Command is terminated with "--", because -- rsync will tack on its own options afterwards, -- and they need to be ignored. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index bfec527d9a..e559e8cba6 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -296,7 +296,10 @@ Many git-annex commands will stage changes for later `git commit` by you. * --backend=name - Specifies which key-value backend to use. + Specifies which key-value backend to use. This can be used when + adding a file to the annex, or migrating a file. Once files + are in the annex, their backend is known and this option is not + necessary. * --key=name diff --git a/test.hs b/test.hs index 2bc0c37a37..49f7f2ab99 100644 --- a/test.hs +++ b/test.hs @@ -121,9 +121,9 @@ test_setkey :: Test test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do writeFile tmp $ content sha1annexedfile r <- annexeval $ BackendClass.getKey backendSHA1 tmp - let sha1 = Key.keyName $ fromJust r - git_annex "setkey" ["-q", "--backend", "SHA1", "--key", sha1, tmp] @? "setkey failed" - git_annex "fromkey" ["-q", "--backend", "SHA1", "--key", sha1, sha1annexedfile] @? "fromkey failed" + let key = show $ fromJust r + git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed" + git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed" Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed" annexed_present sha1annexedfile where @@ -439,7 +439,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do checkunused [annexedfilekey, sha1annexedfilekey] -- good opportunity to test dropkey also - git_annex "dropkey" ["-q", "--force", Key.keyName annexedfilekey] + git_annex "dropkey" ["-q", "--force", show annexedfilekey] @? "dropkey failed" checkunused [sha1annexedfilekey] From fe4e482a9633e977beb1c132c3705b5e103846fa Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 02:58:44 +0000 Subject: [PATCH 1044/8313] Comment moderation --- ..._a868e805be43c5a7c19c41f1af8e41e6._comment | 10 +++++++ ..._7101d07400ad5935f880dc00d89bf90e._comment | 27 +++++++++++++++++++ ..._008554306dd082d7f543baf283510e92._comment | 19 +++++++++++++ ..._504c96959c779176f991f4125ea22009._comment | 14 ++++++++++ ..._79bdf9c51dec9f52372ce95b53233bb2._comment | 12 +++++++++ 5 files changed, 82 insertions(+) create mode 100644 doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment create mode 100644 doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment create mode 100644 doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment create mode 100644 doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment diff --git a/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment b/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment new file mode 100644 index 0000000000..954433deb4 --- /dev/null +++ b/doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-15T14:11:27Z" + content=""" +Keep in mind that lots of small files may have significant overhead, so a warning that it's not possible to make sure there's enough space would make sense for certain corner cases. Actually finding out the exact overhead is beyond git-annex' scope and, given transparent compression etc, ability, but a warning, optionally with a \"do you want to continue\" prompt can't hurt. + +-- RichiH +"""]] diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment new file mode 100644 index 0000000000..7d50c58d1b --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment @@ -0,0 +1,27 @@ +[[!comment format=mdwn + username="praet" + ip="81.240.159.215" + subject="Use variable symlinks, relative to the repo's root ?" + date="2011-03-10T16:50:28Z" + content=""" +It all boils down to the fact that the path to a relative symlink's target is determined relative to the symlink itself. + +Now, if we define the symlink's target relative to the git repo's root (eg. using the $GIT_DIR environment variable, which can be a relative or absolute path itself), this unfortunately results in an absolute symlink, which would -for obvious reasons- only be usable locally: + + user@host:~$ mkdir -p tmp/{.git/annex,somefolder} + user@host:~$ export GIT_DIR=~/tmp + user@host:~$ touch $GIT_DIR/.git/annex/realfile + user@host:~$ ln -s $GIT_DIR/.git/annex/realfile $GIT_DIR/somefolder/file + user@host:~$ ls -al $GIT_DIR/somefolder/ + total 12 + drwxr-x--- 2 user group 4096 2011-03-10 16:54 . + drwxr-x--- 4 user group 4096 2011-03-10 16:53 .. + lrwxrwxrwx 1 user group 33 2011-03-10 16:54 file -> /home/user/tmp/.git/annex/realfile + user@host:~$ + +So, what we need is the ability to record the actual variable name (instead of it's value) in our symlinks. + +It *is* possible, using [variable/variant symlinks](http://en.wikipedia.org/wiki/Symbolic_link#Variable_symbolic_links), yet I'm unsure as to whether or not this is available on Linux systems, and even if it is, it would introduce compatibility issues in multi-OS environments. + +Thoughts on this? +"""]] diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment new file mode 100644 index 0000000000..ab114bb1c8 --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://dieter-be.myopenid.com/" + nickname="dieter" + subject="comment 2" + date="2011-02-16T21:32:04Z" + content=""" +thanks Joey, + +is it possible to run some git annex command that tells me, for a specific directory, which files are available in an other remote? (and which remote, and which filenames?) +I guess I could run that, do my own policy thingie, and run `git annex get` for the files I want. + +For your podcast use case (and some of my use cases) don't you think git [annex] might actually be overkill? For example your podcasts use case, what value does git annex give over a simple rsync/rm script? +such a script wouldn't even need a data store to store its state, unlike git. it seems simpler and cleaner to me. + +for the mpd thing, check http://alip.github.com/mpdcron/ (bad project name, it's a plugin based \"event handler\") +you should be able to write a simple plugin for mpdcron that does what you want (or even interface with mpd yourself from perl/python/.. to use its idle mode to get events) + +Dieter +"""]] diff --git a/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment b/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment new file mode 100644 index 0000000000..64f1e16b50 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-03-15T13:52:16Z" + content=""" +Can't you just use an underscore instead of a colon? + +Would it be feasible to split directories dynamically? I.e. start with SHA1_123456789abcdef0123456789abcdef012345678/SHA1_123456789abcdef0123456789abcdef012345678 and, at a certain cut-off point, switch to shorter directory names? This could even be done per subdirectory and based purely on a locally-configured number. Different annexes on different file systems or with different file subsets might even have different thresholds. This would ensure scale while not forcing you to segment from the start. Also, while segmenting with longer directory names means a flatter tree, segments longer than four characters might not make too much sense. Segmenting too often could lead to some directories becoming too populated, bringing us back to the dynamic segmentation. + +All of the above would make merging annexes by hand a _lot_ harder, but I don't know if this is a valid use case. And if all else fails, one could merge everything with the unsegemented directory names and start again from there. + +-- RichiH +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment b/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment new file mode 100644 index 0000000000..886941be72 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-03-15T14:08:41Z" + content=""" +What is the potential time-frame for this change? As I am not using git-annex for production yet, I can see myself waiting to avoid any potential hassle. + +Supporting generic metadata seems like a great idea. Though if you are going this path, wouldn't it make sense to avoid metastore for mtime etc and support this natively without outside dependencies? + +-- RichiH +"""]] From 795502624dd8bdbcd59767232e9e3d4d8906ca9a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:01:17 +0000 Subject: [PATCH 1045/8313] Added a comment --- ...mment_3_4c69097fe2ee81359655e59a03a9bb8d._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment diff --git a/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment b/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment new file mode 100644 index 0000000000..5cdd6aa0c6 --- /dev/null +++ b/doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:01:17Z" + content=""" +Whups, the comment above got stuck in moderation queue for 27 days. I will try to check that more frequently. + +In the meantime, I've implemented \"git annex whereis\" -- enjoy! + +I find keeping my podcasts in the annex useful because it allows me to download individual episodes or poscasts easily when low bandwidth is available (ie, dialup), or over sneakernet. And generally keeps everything organised. +"""]] From 9741c2026898ef453716d0368091bd17dbebc13a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:03:19 +0000 Subject: [PATCH 1046/8313] Added a comment --- .../comment_3_57010bcaca42089b451ad8659a1e018e._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment new file mode 100644 index 0000000000..534723254a --- /dev/null +++ b/doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:03:19Z" + content=""" +Interesting, I had not heard of variable symlinks before. AFAIK linux does not have them. +"""]] From cf5e002daf481fa4d209ac10d3a991a09e55e13a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:04:50 +0000 Subject: [PATCH 1047/8313] Added a comment --- .../comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment diff --git a/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment b/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment new file mode 100644 index 0000000000..9a43fe3f27 --- /dev/null +++ b/doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-03-16T03:04:50Z" + content=""" +Right. You probably don't want git-annex to fill up your entire drive anyway, so if it tries to reseve 10 mb or 1% or whatever (probably configurable) for overhead, that should be good enough. +"""]] From 682db07d308118064725e57ee90e1aed4323b08c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:13:39 +0000 Subject: [PATCH 1048/8313] Added a comment --- ...mment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment diff --git a/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment b/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment new file mode 100644 index 0000000000..51deb2f959 --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T03:13:39Z" + content=""" +It is unfortunatly not possible to do system-dependant hashing, so long as git-annex stores symlinks to the content in git. + +It might be possible to start without hashing, and add hashing for new files after a cutoff point. It would add complexity. + +I'm currently looking at a 2 character hash directory segment, based on an md5sum of the key, which splits it into 1024 buckets. git uses just 256 buckets for its object directory, but then its objects tend to get packed away. I sorta hope that one level is enough, but guess I could go to 2 levels (objects/ab/cd/key), which would provide 1048576 buckets, probably plenty, as if you are storing more than a million files, you are probably using a modern enough system to have a filesystem that doesn't need hashing. +"""]] From 0ead8b59c247c6208a7223fe5ab013ae79b6a1e2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 03:22:46 +0000 Subject: [PATCH 1049/8313] Added a comment --- ...mment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment b/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment new file mode 100644 index 0000000000..475359abbf --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-03-16T03:22:45Z" + content=""" +Well, I spent a few hours playing this evening in the 'reorg' branch in git. It seems to be shaping up pretty well; type-based refactoring in haskell makes these kind of big systematic changes a matter of editing until it compiles. And it compiles and test suite passes. But, so far I've only covered 1. 3. and 4. on the list, and have yet to deal with upgrades. + +I'd recommend you not wait before using git-annex. I am committed to provide upgradability between annexes created with all versions of git-annex, going forward. This is important because we can have offline archival drives that sit unused for years. Git-annex will upgrade a repository to current standard the first time it sees it, and I hope the upgrade will be pretty smooth. It was not bad for the annex.version 0 to 1 upgrade earlier. The only annoyance with upgrades is that it will result in some big commits to git, as every symlink in the repo gets changed, and log files get moved to new names. + +(The metadata being stored with keys is data that a particular backend can use, and is static to a given key, so there are no merge issues (and it won't be used to preserve mtimes, etc).) +"""]] From 955bda7803b5d8dcbe103d986d53a35186b34ab0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:25:12 -0400 Subject: [PATCH 1050/8313] update --- doc/todo/object_dir_reorg_v2.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn index db18856995..1c2d2f21b7 100644 --- a/doc/todo/object_dir_reorg_v2.mdwn +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -6,6 +6,8 @@ all users, so this should be the *last* reorg in the forseeable future. 2. Add hashing, since some filesystems do suck (like er, fat at least :) [[forum/hashing_objects_directories]] + (Also, may as well hash .git-annex/* while at it -- that's what + really gets big.) 3. Add filesize metadata for [[bugs/free_space_checking]]. (Currently only present in WORM, and in an ad-hoc way.) From fd2f04694f8ba52d9b67e35c95f1bddc33bbe292 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:25:16 -0400 Subject: [PATCH 1051/8313] add comments page --- doc/comments.mdwn | 9 +++++++++ doc/index.mdwn | 1 + 2 files changed, 10 insertions(+) create mode 100644 doc/comments.mdwn diff --git a/doc/comments.mdwn b/doc/comments.mdwn new file mode 100644 index 0000000000..e19962b92a --- /dev/null +++ b/doc/comments.mdwn @@ -0,0 +1,9 @@ +[[!sidebar content=""" +[[!inline pages="comment_pending(*)" feedfile=pendingmoderation +description="comments pending moderation" show=-1]] +Comments in the [[!commentmoderation desc="moderation queue"]]: +[[!pagecount pages="comment_pending(*)"]] +"""]] + +Recent comments posted to this site: +[[!inline pages="comment(*)" template="comment"]] diff --git a/doc/index.mdwn b/doc/index.mdwn index 47682349ff..4f967b71f5 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -10,6 +10,7 @@ To get a feel for it, see the [[walkthrough]]. * [[bugs]] * [[todo]] * [[forum]] +* [[comments]] * [[contact]] [[News]]: From 6c412fb9f55155b0b7bf58d578e51640514ec562 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:39:04 -0400 Subject: [PATCH 1052/8313] escape colons in key files --- Locations.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index 6cff910880..9ffcd9f42b 100644 --- a/Locations.hs +++ b/Locations.hs @@ -118,15 +118,19 @@ isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s - a slash - "%" is escaped to "&s", and "&" to "&a"; this ensures that the mapping - is one to one. + - ":" is escaped to "&c", because despite it being 2011, people still care + - about FAT. - -} keyFile :: Key -> FilePath -keyFile key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key +keyFile key = replace "/" "%" $ replace ":" "&c" $ + replace "%" "&s" $ replace "&" "&a" $ show key {- Reverses keyFile, converting a filename fragment (ie, the basename of - the symlink target) into a key. -} fileKey :: FilePath -> Maybe Key fileKey file = readKey $ - replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + replace "&a" "&" $ replace "&s" "%" $ + replace "&c" ":" $ replace "%" "/" file {- for quickcheck -} prop_idempotent_fileKey :: String -> Bool From dd5448eb075c3774aa173cb9f2e4344ce62b3e13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 15 Mar 2011 23:58:27 -0400 Subject: [PATCH 1053/8313] added 2 level hashing This means there can be 1024 subdirs, each with up to 1024 sub-subdirs. So with hundreds of millions of annexed objects, each leaf directory will have only a few files on average. --- LocationLog.hs | 5 ----- Locations.hs | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index f778df3864..a939af825d 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -123,11 +123,6 @@ logNow s u = do now <- getPOSIXTime return $ LogLine now s u -{- Returns the filename of the log file for a given key. -} -logFile :: Git.Repo -> Key -> String -logFile repo key = - gitStateDir repo ++ keyFile key ++ ".log" - {- Returns a list of repository UUIDs that, according to the log, have - the value of a key. -} keyLocations :: Git.Repo -> Key -> IO [UUID] diff --git a/Locations.hs b/Locations.hs index 9ffcd9f42b..b2d31a1bf8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -19,6 +19,7 @@ module Locations ( gitAnnexBadDir, gitAnnexUnusedLog, isLinkToAnnex, + logFile, prop_idempotent_fileKey ) where @@ -66,7 +67,7 @@ objectDir = addTrailingPathSeparator $ annexDir "objects" {- Annexed file's location relative to the .git directory. -} annexLocation :: Key -> FilePath -annexLocation key = objectDir f f +annexLocation key = objectDir hashDir key f f where f = keyFile key @@ -109,6 +110,11 @@ gitAnnexUnusedLog r = gitAnnexDir r "unused" isLinkToAnnex :: FilePath -> Bool isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s +{- The filename of the log file for a given key. -} +logFile :: Git.Repo -> Key -> String +logFile repo key = + gitStateDir repo ++ hashDir key ++ keyFile key ++ ".log" + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid @@ -137,11 +143,13 @@ prop_idempotent_fileKey :: String -> Bool prop_idempotent_fileKey s = Just k == fileKey (keyFile k) where k = stubKey { keyName = s, keyBackendName = "test" } -{- Given a filename, generates a short directory name to put it in, +{- Given a key, generates a short directory name to put it in, - to do hashing to protect against filesystems that dislike having - many items in a single directory. -} -hashDir :: FilePath -> FilePath -hashDir s = take 2 $ abcd_to_dir $ md5 (Str s) +hashDir :: Key -> FilePath +hashDir k = addTrailingPathSeparator $ take 2 dir drop 2 dir + where + dir = take 4 $ abcd_to_dir $ md5 $ Str $ show k abcd_to_dir :: ABCD -> String abcd_to_dir (ABCD (a,b,c,d)) = concat $ map display_32bits_as_dir [a,b,c,d] From 2e26caa8568001b33a969efc46f6911278686e0e Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 04:06:19 +0000 Subject: [PATCH 1054/8313] Added a comment --- ...mment_4_0de9170e429cbfea66f5afa8980d45ac._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment diff --git a/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment b/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment new file mode 100644 index 0000000000..b29eea1b2b --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-03-16T04:06:19Z" + content=""" +The .git-annex/ directory is what really needs hashing. + +Consider that when git looks for changes in there, it has to scan every file in the directory. With hashing, it should be able to more quickly identify just the subdirectories that contained changed files, by the directory mtimes. + +And the real kicker is that when committing there, git has to create a tree object containing every single file, even if only 1 file changed. That will be a lot of extra work; with hashed subdirs it will instead create just 2 or 3 small tree objects leading down to the changed file. (Probably these trees both pack down to similar size pack files, not sure.) +"""]] From 09a7689bc30faaf938a0b32a417d38ac093a6f7a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 00:08:02 -0400 Subject: [PATCH 1055/8313] update and bug closures for v2 layout --- debian/changelog | 7 +++++++ doc/bugs/fat_support.mdwn | 3 +++ doc/forum/hashing_objects_directories.mdwn | 8 ++++++++ doc/internals.mdwn | 10 +++++++--- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0d1832374d..ac7c854ff5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,13 @@ git-annex (0.24) UNRELEASED; urgency=low * Reorganized annexed object store. annex.version=2 + * Colons are now avoided in filenames, so bare clones of git repos + can be put on USB thumb drives formatted with vFAT or similar + filesystems. + * Added two levels of hashing to object directory and .git-annex logs, + to improve scalability with enormous numbers of annexed + objects. (With one hundred million annexed objects, each + directory would contain fewer than 1024 files.) * The setkey, fromkey, and dropkey subcommands have changed how the key is specified. --backend is no longer used with these. * Add Suggests on graphviz. Closes: #618039 diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn index 2c6c973856..60633c29bf 100644 --- a/doc/bugs/fat_support.mdwn +++ b/doc/bugs/fat_support.mdwn @@ -10,3 +10,6 @@ be VFAT formatted: [[!tag wishlist]] +[[Done]]; in annex.version 2 repos, colons are entirely avoided in +filenames. So a bare git clone can be put on VFAT, and git-annex +used to move stuff --to and --from it, for sneakernet. diff --git a/doc/forum/hashing_objects_directories.mdwn b/doc/forum/hashing_objects_directories.mdwn index 715e972ca4..5b7708fb58 100644 --- a/doc/forum/hashing_objects_directories.mdwn +++ b/doc/forum/hashing_objects_directories.mdwn @@ -17,3 +17,11 @@ or anything in between to a paranoid Also the use of a colon specifically breaks FAT32 ([[bugs/fat_support]]), must it be a colon or could an extra directory be used? i.e. `.git/annex/objects/SHA1/*/...` `git annex init` could also create all but the last level directory on initialization. I'm thinking `SHA1/1/1, SHA1/1/2, ..., SHA256/f/f, ..., URL/f/f, ..., WORM/f/f` + +> This is done now with a 2-level hash. It also hashes .git-annex/ log +> files which were the worse problem really. Scales to hundreds of millions +> of files with each dir having 1024 or fewer contents. Example: +> +> `me -> .git/annex/objects/71/9t/WORM-s3-m1300247299--me/WORM-s3-m1300247299--me` +> +> --[[Joey]] diff --git a/doc/internals.mdwn b/doc/internals.mdwn index 3f680dd8f2..a133320b4b 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -2,12 +2,15 @@ In the world of git, we're not scared about internal implementation details, and sometimes we like to dive in and tweak things by hand. Here's some documentation to that end. -## `.git/annex/objects/*/*` +## `.git/annex/objects/aa/bb/*/*` This is where locally available file contents are actually stored. Files added to the annex get a symlink checked into git that points to the file content. +First there are two levels of directories used for hashing, to prevent +too many things ending up in any one directory. + Each subdirectory has the name of a key in one of the [[key-value_backends|backends]]. The file inside also has the name of the key. This two-level structure is used because it allows the write bit to be removed @@ -41,10 +44,11 @@ Example: e605dca6-446a-11e0-8b2a-002170d25c55 1 26339d22-446b-11e0-9101-002170d25c55 ? -## `.git-annex/*.log` +## `.git-annex/aa/bb/*.log` The remainder of the log files record [[location_tracking]] information -for file contents. The name of the key is the filename, and the content +for file contents. Again these are placed in two levels of subdirectories +for hashing. The name of the key is the filename, and the content consists of a timestamp, either 1 (present) or 0 (not present), and the UUID of the repository that has or lacks the file content. From f1e010f42e373bd0658c3b9c6ab67cd84715ad60 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 00:32:15 -0400 Subject: [PATCH 1056/8313] upgrade thoughts long comments :) --- Upgrade.hs | 47 +++++++++++++++++++++++++++++++++++++++++++++-- debian/changelog | 1 + 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/Upgrade.hs b/Upgrade.hs index eba75bf587..d63397ce0f 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -38,12 +38,52 @@ upgrade = do upgradeFrom1 :: Annex Bool upgradeFrom1 = do - showSideAction "Upgrading object directory layout..." + showSideAction "Upgrading object directory layout v1 to v2..." error "upgradeFrom1 TODO FIXME" + -- v2 adds hashing of filenames of content and location log files. + -- + -- Key information is encoded in filenames differently. + -- + -- When upgrading a v1 key to v2, file size metadata needs to be + -- added to the key (unless it is a WORM key, which encoded + -- mtime:size in v1). This can only be done when the file content + -- is present. + -- + -- So there are two approaches -- either upgrade + -- everything, leaving out file size information for files not + -- present in the current repo; or upgrade peicemeil, only + -- upgrading keys whose content is present. + -- + -- The latter approach would mean that, until every clone of an + -- annex is upgraded, git annex would refuse to operate on annexed + -- files that had not yet been committed. Unless it were taught to + -- work with both v1 and v2 keys in the same repo. + -- + -- Another problem with the latter approach might involve content + -- being moved between repos while the conversion is still + -- incomplete. If repo A has already upgraded, and B has not, and B + -- has K, moving K from B -> A would result in it lurking + -- unconverted on A. Unless A upgraded it in passing. But that's + -- getting really complex, and would mean a constant trickle of + -- upgrade commits, which users would find annoying. + -- + -- So, the former option it is! Note that file size metadata + -- will only be used for detecting situations where git-annex + -- would run out of disk space, so if some keys don't have it, + -- the impact is small. At least initially. It could be used in the + -- future by smart auto-repo balancing code, etc. + -- + -- Anyway, since v2 plans ahead for other metadata being included + -- in keys, there should probably be a way to update a key. + -- Something similar to the migrate subcommand could be used, + -- and users could then run that at their leisure. Or, this upgrade + -- could to that key update for all keys that have been converted + -- and have content in the repo. + upgradeFrom0 :: Annex Bool upgradeFrom0 = do - showSideAction "Upgrading object directory layout..." + showSideAction "Upgrading object directory layout v0 to v1..." g <- Annex.gitRepo -- do the reorganisation of the files @@ -56,6 +96,9 @@ upgradeFrom0 = do fixlinks files Annex.queueRun + -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 + upgradeFrom1 + setVersion return True diff --git a/debian/changelog b/debian/changelog index ac7c854ff5..738faf916e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,6 @@ git-annex (0.24) UNRELEASED; urgency=low + * TODO: upgrade v1 -> v2 * Reorganized annexed object store. annex.version=2 * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar From e227c210ec817eca6b4409cb4cc893f791d51c00 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 01:23:20 -0400 Subject: [PATCH 1057/8313] upgrade groundwork pulled in old versions of functions for working with keys Wrote a parser from old key filenames to new keys. --- Backend.hs | 3 +- Upgrade.hs | 118 ++------------------------------------ Upgrade/V0.hs | 80 ++++++++++++++++++++++++++ Upgrade/V1.hs | 155 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 241 insertions(+), 115 deletions(-) create mode 100644 Upgrade/V0.hs create mode 100644 Upgrade/V1.hs diff --git a/Backend.hs b/Backend.hs index e1f8f388b9..cd14ce50e1 100644 --- a/Backend.hs +++ b/Backend.hs @@ -27,7 +27,8 @@ module Backend ( lookupFile, chooseBackends, keyBackend, - lookupBackendName + lookupBackendName, + maybeLookupBackendName ) where import Control.Monad.State diff --git a/Upgrade.hs b/Upgrade.hs index d63397ce0f..a152582043 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -7,128 +7,18 @@ module Upgrade where -import System.IO.Error (try) -import System.Directory -import Control.Monad.State (liftIO) -import Control.Monad (filterM, forM_) -import System.Posix.Files -import System.FilePath -import Data.Maybe - -import Content import Types -import Locations -import qualified GitRepo as Git -import qualified Annex -import qualified Backend -import Messages import Version -import Utility +import qualified Upgrade.V0 +import qualified Upgrade.V1 {- Uses the annex.version git config setting to automate upgrades. -} upgrade :: Annex Bool upgrade = do version <- getVersion case version of - Just "0" -> upgradeFrom0 - Just "1" -> upgradeFrom1 + Just "0" -> Upgrade.V0.upgrade + Just "1" -> Upgrade.V1.upgrade Nothing -> return True -- repo not initted yet, no version Just v | v == currentVersion -> return True Just _ -> error "this version of git-annex is too old for this git repository!" - -upgradeFrom1 :: Annex Bool -upgradeFrom1 = do - showSideAction "Upgrading object directory layout v1 to v2..." - error "upgradeFrom1 TODO FIXME" - - -- v2 adds hashing of filenames of content and location log files. - -- - -- Key information is encoded in filenames differently. - -- - -- When upgrading a v1 key to v2, file size metadata needs to be - -- added to the key (unless it is a WORM key, which encoded - -- mtime:size in v1). This can only be done when the file content - -- is present. - -- - -- So there are two approaches -- either upgrade - -- everything, leaving out file size information for files not - -- present in the current repo; or upgrade peicemeil, only - -- upgrading keys whose content is present. - -- - -- The latter approach would mean that, until every clone of an - -- annex is upgraded, git annex would refuse to operate on annexed - -- files that had not yet been committed. Unless it were taught to - -- work with both v1 and v2 keys in the same repo. - -- - -- Another problem with the latter approach might involve content - -- being moved between repos while the conversion is still - -- incomplete. If repo A has already upgraded, and B has not, and B - -- has K, moving K from B -> A would result in it lurking - -- unconverted on A. Unless A upgraded it in passing. But that's - -- getting really complex, and would mean a constant trickle of - -- upgrade commits, which users would find annoying. - -- - -- So, the former option it is! Note that file size metadata - -- will only be used for detecting situations where git-annex - -- would run out of disk space, so if some keys don't have it, - -- the impact is small. At least initially. It could be used in the - -- future by smart auto-repo balancing code, etc. - -- - -- Anyway, since v2 plans ahead for other metadata being included - -- in keys, there should probably be a way to update a key. - -- Something similar to the migrate subcommand could be used, - -- and users could then run that at their leisure. Or, this upgrade - -- could to that key update for all keys that have been converted - -- and have content in the repo. - -upgradeFrom0 :: Annex Bool -upgradeFrom0 = do - showSideAction "Upgrading object directory layout v0 to v1..." - g <- Annex.gitRepo - - -- do the reorganisation of the files - let olddir = gitAnnexDir g - keys <- getKeysPresent0' olddir - forM_ keys $ \k -> moveAnnex k $ olddir keyFile k - - -- update the symlinks to the files - files <- liftIO $ Git.inRepo g [Git.workTree g] - fixlinks files - Annex.queueRun - - -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 - upgradeFrom1 - - setVersion - - return True - - where - fixlinks [] = return () - fixlinks (f:fs) = do - r <- Backend.lookupFile f - case r of - Nothing -> return () - Just (k, _) -> do - link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f - fixlinks fs - -getKeysPresent0' :: FilePath -> Annex [Key] -getKeysPresent0' dir = do - exists <- liftIO $ doesDirectoryExist dir - if (not exists) - then return [] - else do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ catMaybes $ map fileKey files - where - present d = do - result <- try $ - getFileStatus $ dir ++ "/" ++ takeFileName d - case result of - Right s -> return $ isRegularFile s - Left _ -> return False diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs new file mode 100644 index 0000000000..25b6f27635 --- /dev/null +++ b/Upgrade/V0.hs @@ -0,0 +1,80 @@ +{- git-annex v0 -> v1 upgrade support + - + - Copyright 2010 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V0 where + +import System.IO.Error (try) +import System.Directory +import Control.Monad.State (liftIO) +import Control.Monad (filterM, forM_) +import System.Posix.Files +import System.FilePath + +import Content +import Types +import Locations +import qualified GitRepo as Git +import qualified Annex +import Messages +import Utility +import qualified Upgrade.V1 + +upgrade :: Annex Bool +upgrade = do + showSideAction "Upgrading object directory layout v0 to v1..." + g <- Annex.gitRepo + + -- do the reorganisation of the key files + let olddir = gitAnnexDir g + keys <- getKeysPresent0 olddir + forM_ keys $ \k -> moveAnnex k $ olddir keyFile0 k + + -- update the symlinks to the key files + files <- liftIO $ Git.inRepo g [Git.workTree g] + fixlinks files + Annex.queueRun + + -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 + Upgrade.V1.upgrade + + where + fixlinks [] = return () + fixlinks (f:fs) = do + r <- lookupFile0 f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [Param "--"] f + fixlinks fs + +-- these stayed unchanged between v0 and v1 +keyFile0 :: Key -> FilePath +keyFile0 = Upgrade.V1.keyFile1 +fileKey0 :: FilePath -> Key +fileKey0 = Upgrade.V1.fileKey1 +lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend Annex)) +lookupFile0 = Upgrade.V1.lookupFile1 + +getKeysPresent0 :: FilePath -> Annex [Key] +getKeysPresent0 dir = do + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey0 files + where + present d = do + result <- try $ + getFileStatus $ dir ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs new file mode 100644 index 0000000000..dd51206b30 --- /dev/null +++ b/Upgrade/V1.hs @@ -0,0 +1,155 @@ +{- git-annex v1 -> v2 upgrade support + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Upgrade.V1 where + +import System.IO.Error (try) +import System.Directory +import Control.Monad.State (liftIO) +import Control.Monad (filterM, forM_, unless) +import System.Posix.Files +import System.FilePath +import Data.String.Utils +import Key +import System.Posix.Types + +import Content +import Types +import Locations +import qualified Annex +import Backend +import Messages +import Version + +upgrade :: Annex Bool +upgrade = do + showSideAction "Upgrading object directory layout v1 to v2..." + error "upgradeFrom1 TODO FIXME" + + -- v2 adds hashing of filenames of content and location log files. + -- + -- Key information is encoded in filenames differently. + -- + -- When upgrading a v1 key to v2, file size metadata needs to be + -- added to the key (unless it is a WORM key, which encoded + -- mtime:size in v1). This can only be done when the file content + -- is present. + -- + -- So there are two approaches -- either upgrade + -- everything, leaving out file size information for files not + -- present in the current repo; or upgrade peicemeil, only + -- upgrading keys whose content is present. + -- + -- The latter approach would mean that, until every clone of an + -- annex is upgraded, git annex would refuse to operate on annexed + -- files that had not yet been committed. Unless it were taught to + -- work with both v1 and v2 keys in the same repo. + -- + -- Another problem with the latter approach might involve content + -- being moved between repos while the conversion is still + -- incomplete. If repo A has already upgraded, and B has not, and B + -- has K, moving K from B -> A would result in it lurking + -- unconverted on A. Unless A upgraded it in passing. But that's + -- getting really complex, and would mean a constant trickle of + -- upgrade commits, which users would find annoying. + -- + -- So, the former option it is! Note that file size metadata + -- will only be used for detecting situations where git-annex + -- would run out of disk space, so if some keys don't have it, + -- the impact is small. At least initially. It could be used in the + -- future by smart auto-repo balancing code, etc. + -- + -- Anyway, since v2 plans ahead for other metadata being included + -- in keys, there should probably be a way to update a key. + -- Something similar to the migrate subcommand could be used, + -- and users could then run that at their leisure. Or, this upgrade + -- could to that key update for all keys that have been converted + -- and have content in the repo. + + -- do the reorganisation of the log files + + -- do the reorganisation of the key files + g <- Annex.gitRepo + let olddir = gitAnnexDir g + keys <- getKeysPresent1 + forM_ keys $ \k -> moveAnnex k $ olddir keyFile1 k + + -- update the symlinks to the key files + + Annex.queueRun + + setVersion + + return True + +keyFile1 :: Key -> FilePath +keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key + +fileKey1 :: FilePath -> Key +fileKey1 file = readKey1 $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +readKey1 :: String -> Key +readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } + where + bits = split ":" v + b = head bits + n = join ":" $ drop (if wormy then 3 else 1) bits + t = if wormy + then Just (read (bits !! 1) :: EpochTime) + else Nothing + s = if wormy + then Just (read (bits !! 2) :: Integer) + else Nothing + wormy = b == "WORM" + +lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex)) +lookupFile1 file = do + bs <- Annex.getState Annex.supportedBackends + tl <- liftIO $ try getsymlink + case tl of + Left _ -> return Nothing + Right l -> makekey bs l + where + getsymlink = do + l <- readSymbolicLink file + return $ takeFileName l + makekey bs l = do + case maybeLookupBackendName bs bname of + Nothing -> do + unless (null kname || null bname || + not (isLinkToAnnex l)) $ + warning skip + return Nothing + Just backend -> return $ Just (k, backend) + where + k = fileKey1 l + bname = keyBackendName k + kname = keyName k + skip = "skipping " ++ file ++ + " (unknown backend " ++ bname ++ ")" + +getKeysPresent1 :: Annex [Key] +getKeysPresent1 = do + g <- Annex.gitRepo + getKeysPresent1' $ gitAnnexObjectDir g +getKeysPresent1' :: FilePath -> Annex [Key] +getKeysPresent1' dir = do + exists <- liftIO $ doesDirectoryExist dir + if (not exists) + then return [] + else do + contents <- liftIO $ getDirectoryContents dir + files <- liftIO $ filterM present contents + return $ map fileKey1 files + where + present d = do + result <- try $ + getFileStatus $ dir ++ "/" ++ d ++ "/" ++ takeFileName d + case result of + Right s -> return $ isRegularFile s + Left _ -> return False From a4d0250298a503c7d9093e6dd1618ff5b07b19e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 02:15:54 -0400 Subject: [PATCH 1058/8313] slways set current version in new repos detect v1 repos that don't have a version set --- Upgrade.hs | 9 ++++----- Version.hs | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Upgrade.hs b/Upgrade.hs index a152582043..76dd156f83 100644 --- a/Upgrade.hs +++ b/Upgrade.hs @@ -17,8 +17,7 @@ upgrade :: Annex Bool upgrade = do version <- getVersion case version of - Just "0" -> Upgrade.V0.upgrade - Just "1" -> Upgrade.V1.upgrade - Nothing -> return True -- repo not initted yet, no version - Just v | v == currentVersion -> return True - Just _ -> error "this version of git-annex is too old for this git repository!" + "0" -> Upgrade.V0.upgrade + "1" -> Upgrade.V1.upgrade + v | v == currentVersion -> return True + _ -> error "this version of git-annex is too old for this git repository!" diff --git a/Version.hs b/Version.hs index 7fdbd1a49a..5f414e93b8 100644 --- a/Version.hs +++ b/Version.hs @@ -21,21 +21,28 @@ currentVersion = "2" versionField :: String versionField = "annex.version" -getVersion :: Annex (Maybe String) +getVersion :: Annex String getVersion = do g <- Annex.gitRepo let v = Git.configGet g versionField "" if not $ null v - then return $ Just v + then return v else do -- version 0 was not recorded in .git/config; -- such a repo should have an gitAnnexDir but no - -- gitAnnexObjectDir + -- gitAnnexObjectDir. + -- + -- version 1 may not be recorded if the user + -- forgot to init. Such a repo should have a + -- gitAnnexObjectDir already. d <- liftIO $ doesDirectoryExist $ gitAnnexDir g o <- liftIO $ doesDirectoryExist $ gitAnnexObjectDir g - if d && not o - then return $ Just "0" - else return Nothing -- no version yet + case (d, o) of + (True, False) -> return "0" + (True, True) -> return "1" + _ -> do + setVersion + return currentVersion setVersion :: Annex () setVersion = Annex.setConfig versionField currentVersion From 500c4e44c560a04aaa30e165b70d4d8491ad9c32 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 02:35:48 -0400 Subject: [PATCH 1059/8313] v1 -> v2 upgrade partially working still need to move location log files, and auto-commit --- Upgrade/V0.hs | 19 +------ Upgrade/V1.hs | 136 +++++++++++++++++++++++++++-------------------- debian/changelog | 2 +- 3 files changed, 81 insertions(+), 76 deletions(-) diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs index 25b6f27635..5ba305817b 100644 --- a/Upgrade/V0.hs +++ b/Upgrade/V0.hs @@ -17,10 +17,8 @@ import System.FilePath import Content import Types import Locations -import qualified GitRepo as Git import qualified Annex import Messages -import Utility import qualified Upgrade.V1 upgrade :: Annex Bool @@ -34,26 +32,11 @@ upgrade = do forM_ keys $ \k -> moveAnnex k $ olddir keyFile0 k -- update the symlinks to the key files - files <- liftIO $ Git.inRepo g [Git.workTree g] - fixlinks files - Annex.queueRun + -- No longer needed here; V1.upgrade does the same thing -- Few people had v0 repos, so go the long way around from 0 -> 1 -> 2 Upgrade.V1.upgrade - where - fixlinks [] = return () - fixlinks (f:fs) = do - r <- lookupFile0 f - case r of - Nothing -> return () - Just (k, _) -> do - link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f - fixlinks fs - -- these stayed unchanged between v0 and v1 keyFile0 :: Key -> FilePath keyFile0 = Upgrade.V1.keyFile1 diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index dd51206b30..850080436f 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -21,78 +21,83 @@ import Content import Types import Locations import qualified Annex +import qualified GitRepo as Git import Backend import Messages import Version +import Utility + +-- v2 adds hashing of filenames of content and location log files. +-- Key information is encoded in filenames differently, so +-- both content and location log files move around, and symlinks +-- to content need to be changed. +-- +-- When upgrading a v1 key to v2, file size metadata ought to be +-- added to the key (unless it is a WORM key, which encoded +-- mtime:size in v1). This can only be done when the file content +-- is present. Since upgrades need to happen consistently, +-- (so that two repos get changed the same way by the upgrade, and +-- will merge), that metadata cannot be added on upgrade. +-- +-- Note that file size metadata +-- will only be used for detecting situations where git-annex +-- would run out of disk space, so if some keys don't have it, +-- the impact is minor. At least initially. It could be used in the +-- future by smart auto-repo balancing code, etc. +-- +-- Anyway, since v2 plans ahead for other metadata being included +-- in keys, there should probably be a way to update a key. +-- Something similar to the migrate subcommand could be used, +-- and users could then run that at their leisure. upgrade :: Annex Bool upgrade = do showSideAction "Upgrading object directory layout v1 to v2..." - error "upgradeFrom1 TODO FIXME" - -- v2 adds hashing of filenames of content and location log files. - -- - -- Key information is encoded in filenames differently. - -- - -- When upgrading a v1 key to v2, file size metadata needs to be - -- added to the key (unless it is a WORM key, which encoded - -- mtime:size in v1). This can only be done when the file content - -- is present. - -- - -- So there are two approaches -- either upgrade - -- everything, leaving out file size information for files not - -- present in the current repo; or upgrade peicemeil, only - -- upgrading keys whose content is present. - -- - -- The latter approach would mean that, until every clone of an - -- annex is upgraded, git annex would refuse to operate on annexed - -- files that had not yet been committed. Unless it were taught to - -- work with both v1 and v2 keys in the same repo. - -- - -- Another problem with the latter approach might involve content - -- being moved between repos while the conversion is still - -- incomplete. If repo A has already upgraded, and B has not, and B - -- has K, moving K from B -> A would result in it lurking - -- unconverted on A. Unless A upgraded it in passing. But that's - -- getting really complex, and would mean a constant trickle of - -- upgrade commits, which users would find annoying. - -- - -- So, the former option it is! Note that file size metadata - -- will only be used for detecting situations where git-annex - -- would run out of disk space, so if some keys don't have it, - -- the impact is small. At least initially. It could be used in the - -- future by smart auto-repo balancing code, etc. - -- - -- Anyway, since v2 plans ahead for other metadata being included - -- in keys, there should probably be a way to update a key. - -- Something similar to the migrate subcommand could be used, - -- and users could then run that at their leisure. Or, this upgrade - -- could to that key update for all keys that have been converted - -- and have content in the repo. - - -- do the reorganisation of the log files - - -- do the reorganisation of the key files - g <- Annex.gitRepo - let olddir = gitAnnexDir g - keys <- getKeysPresent1 - forM_ keys $ \k -> moveAnnex k $ olddir keyFile1 k - - -- update the symlinks to the key files + moveContent + updateSymlinks + moveLocationLogs Annex.queueRun - setVersion - return True -keyFile1 :: Key -> FilePath -keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ show key +moveContent :: Annex () +moveContent = do + keys <- getKeysPresent1 + forM_ keys move + where + move k = do + g <- Annex.gitRepo + let f = gitAnnexObjectDir g keyFile1 k keyFile1 k + let d = parentDir f + liftIO $ allowWrite d + liftIO $ allowWrite f + moveAnnex k f + liftIO $ removeDirectory d -fileKey1 :: FilePath -> Key -fileKey1 file = readKey1 $ - replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file +updateSymlinks :: Annex () +updateSymlinks = do + g <- Annex.gitRepo + files <- liftIO $ Git.inRepo g [Git.workTree g] + forM_ files $ fixlink + where + fixlink f = do + r <- lookupFile1 f + case r of + Nothing -> return () + Just (k, _) -> do + link <- calcGitLink f k + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [Param "--"] f +moveLocationLogs :: Annex () +moveLocationLogs = do + warning "TODO location log move" + +-- WORM backend keys: "WORM:mtime:size:filename" +-- all the rest: "backend:key" readKey1 :: String -> Key readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } where @@ -107,6 +112,23 @@ readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } else Nothing wormy = b == "WORM" +showKey1 :: Key -> String +showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } = + join ":" $ filter (not . null) [b, showifhere t, showifhere s, n] + where + showifhere Nothing = "" + showifhere (Just v) = show v + +keyFile1 :: Key -> FilePath +keyFile1 key = replace "/" "%" $ replace "%" "&s" $ replace "&" "&a" $ showKey1 key + +fileKey1 :: FilePath -> Key +fileKey1 file = readKey1 $ + replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file + +logFile1 :: Git.Repo -> Key -> String +logFile1 repo key = gitStateDir repo ++ keyFile1 key ++ ".log" + lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex)) lookupFile1 file = do bs <- Annex.getState Annex.supportedBackends diff --git a/debian/changelog b/debian/changelog index 738faf916e..6cd8a9326b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.24) UNRELEASED; urgency=low +git-annex (0.20110316) UNRELEASED; urgency=low * TODO: upgrade v1 -> v2 * Reorganized annexed object store. annex.version=2 From 137257ded1acb1b15c0413a8998af1e999cd1a53 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 02:50:13 -0400 Subject: [PATCH 1060/8313] better letter choice for hashing --- Locations.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Locations.hs b/Locations.hs index b2d31a1bf8..3cce4c2611 100644 --- a/Locations.hs +++ b/Locations.hs @@ -162,8 +162,8 @@ display_32bits_as_dir :: Word32 -> String display_32bits_as_dir w = trim $ swap_pairs cs where -- Need 32 characters to use. To avoid inaverdently making - -- a real word, use the alphabet without vowels. - chars = ['0'..'9'] ++ "bcdfghjklnmpqrstvwxyzZ" + -- a real word, use letters that appear less frequently. + chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF" cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7] getc n = chars !! (fromIntegral n) swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs From ea81da347f952bedb8d793c7fa190c229bce04ee Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 16 Mar 2011 14:27:45 +0000 Subject: [PATCH 1061/8313] --- ...has_problems_on_non-linux_based_systems.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn new file mode 100644 index 0000000000..20c16e3cfa --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn @@ -0,0 +1,17 @@ +It seems that commit bc5c54c987f548505a3877e8a0e460abe0b2a081 introduced some linux specific things... + +

+hsc2hs Touch.hsc
+Touch.hsc: In function ‘main’:
+Touch.hsc:46: error: ‘UTIME_OMIT’ undeclared (first use in this function)
+Touch.hsc:46: error: (Each undeclared identifier is reported only once
+Touch.hsc:46: error: for each function it appears in.)
+Touch.hsc:48: error: ‘UTIME_NOW’ undeclared (first use in this function)
+Touch.hsc:67: error: ‘AT_FDCWD’ undeclared (first use in this function)
+Touch.hsc:68: error: ‘AT_SYMLINK_NOFOLLOW’ undeclared (first use in this function)
+compiling Touch_hsc_make.c failed
+command was: /usr/bin/gcc -c -m32 -I/Library/Frameworks/GHC.framework/Versions/612/usr/lib/ghc-6.12.3/include/ Touch_hsc_make.c -o Touch_hsc_make.o
+make: *** [Touch.hs] Error 1
+
+ +I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex. From a0807999001017bd6897bad82e747c16e18af6bc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 10:56:59 -0400 Subject: [PATCH 1062/8313] upgrades seem to fully work --- Upgrade/V1.hs | 34 ++++++++++++++++++++++++++++++++-- debian/changelog | 9 +++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 850080436f..602ba41c59 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -14,9 +14,10 @@ import Control.Monad (filterM, forM_, unless) import System.Posix.Files import System.FilePath import Data.String.Utils -import Key import System.Posix.Types +import Data.Maybe +import Key import Content import Types import Locations @@ -94,7 +95,36 @@ updateSymlinks = do moveLocationLogs :: Annex () moveLocationLogs = do - warning "TODO location log move" + logkeys <- oldlocationlogs + forM_ logkeys move + where + oldlocationlogs = do + g <- Annex.gitRepo + let dir = gitStateDir g + contents <- liftIO $ getDirectoryContents dir + return $ catMaybes $ map oldlog2key contents + move (l, k) = do + g <- Annex.gitRepo + let dest = logFile g k + let dir = gitStateDir g + let f = dir l + liftIO $ createDirectoryIfMissing True (parentDir dest) + -- could just git mv, but this way deals with + -- log files that are not checked into git + liftIO $ copyFile f dest + Annex.queue "add" [Param "--"] dest + Annex.queue "add" [Param "--"] f + Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f + +oldlog2key :: FilePath -> Maybe (FilePath, Key) +oldlog2key l = + let len = length l - 4 in + if drop len l == ".log" + then let k = readKey1 (take len l) in + if null (keyName k) || null (keyBackendName k) + then Nothing + else Just (l, k) + else Nothing -- WORM backend keys: "WORM:mtime:size:filename" -- all the rest: "backend:key" diff --git a/debian/changelog b/debian/changelog index 6cd8a9326b..c40385bebe 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,12 @@ git-annex (0.20110316) UNRELEASED; urgency=low - * TODO: upgrade v1 -> v2 - * Reorganized annexed object store. annex.version=2 + * Reorganized .git/annex/objects and .git-annex/; annex.version=2 + * The first time git-annex is run in an old format repository, it + will automatically upgrade it to the new format, staging all + necessary changes to git. + * Note that remotes must be running this version of git-annex, + and must also have been upgraded, in order for git-annex to + communicate with them. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. From 744638197f51811fca13a37c7bbc51dfb626793b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 11:27:29 -0400 Subject: [PATCH 1063/8313] fix getKeyspresent to work with hashed dirs --- Content.hs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Content.hs b/Content.hs index 1a5a80a9f1..a59484b5ab 100644 --- a/Content.hs +++ b/Content.hs @@ -161,13 +161,21 @@ getKeysPresent' dir = do if (not exists) then return [] else do - contents <- liftIO $ getDirectoryContents dir - files <- liftIO $ filterM present contents - return $ catMaybes $ map fileKey files + -- 2 levels of hashing + levela <- liftIO $ subdirContent dir + levelb <- liftIO $ mapM subdirContent levela + contents <- liftIO $ mapM subdirContent (concat levelb) + files <- liftIO $ filterM present (concat contents) + return $ catMaybes $ map (fileKey . takeFileName) files where present d = do result <- try $ - getFileStatus $ dir ++ "/" ++ d ++ "/" ++ takeFileName d + getFileStatus $ d takeFileName d + liftIO $ putStrLn $ "trying " ++ (d takeFileName d) case result of Right s -> return $ isRegularFile s Left _ -> return False + subdirContent d = do + c <- getDirectoryContents d + return $ map (d ) $ filter notcruft c + notcruft f = f /= "." && f /= ".." From 5eb76d2b033436973e0732215fa9d0227a2187bf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 11:53:46 -0400 Subject: [PATCH 1064/8313] improve upgrade --- Command/Init.hs | 17 +++++++++++------ Command/Uninit.hs | 2 +- LocationLog.hs | 2 ++ Upgrade/V1.hs | 18 +++++++++++++++--- debian/changelog | 5 +---- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/Command/Init.hs b/Command/Init.hs index 6618351699..d9ea394a33 100644 --- a/Command/Init.hs +++ b/Command/Init.hs @@ -8,7 +8,7 @@ module Command.Init where import Control.Monad.State (liftIO) -import Control.Monad (when) +import Control.Monad (when, unless) import System.Directory import System.FilePath @@ -74,12 +74,14 @@ gitAttributesWrite repo = do exists <- doesFileExist attributes if not exists then do - safeWriteFile attributes $ attrLine ++ "\n" + safeWriteFile attributes $ unlines attrLines commit else do content <- readFile attributes - when (all (/= attrLine) (lines content)) $ do - appendFile attributes $ attrLine ++ "\n" + let present = lines content + let missing = filter (\l -> not $ l `elem` present) attrLines + unless (null missing) $ do + appendFile attributes $ unlines missing commit where attributes = Git.attributes repo @@ -91,8 +93,11 @@ gitAttributesWrite repo = do , Param attributes ] -attrLine :: String -attrLine = stateDir "*.log merge=union" +attrLines :: [String] +attrLines = + [ stateDir "*.log merge=union" + , stateDir "*/*/*.log merge=union" + ] {- set up a git pre-commit hook, if one is not already present -} gitPreCommitHookWrite :: Git.Repo -> Annex () diff --git a/Command/Uninit.hs b/Command/Uninit.hs index e9406ce3af..e8ac1bbd5e 100644 --- a/Command/Uninit.hs +++ b/Command/Uninit.hs @@ -60,4 +60,4 @@ gitAttributesUnWrite repo = do when attrexists $ do c <- readFileStrict attributes safeWriteFile attributes $ unlines $ - filter (/= Command.Init.attrLine) $ lines c + filter (\l -> not $ l `elem` Command.Init.attrLines) $ lines c diff --git a/LocationLog.hs b/LocationLog.hs index a939af825d..f1e54432ca 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -24,6 +24,8 @@ module LocationLog ( LogStatus(..), logChange, logFile, + readLog, + writeLog, keyLocations ) where diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 602ba41c59..797bdee0dc 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -21,13 +21,15 @@ import Key import Content import Types import Locations +import LocationLog import qualified Annex import qualified GitRepo as Git import Backend import Messages import Version import Utility - +import qualified Command.Init + -- v2 adds hashing of filenames of content and location log files. -- Key information is encoded in filenames differently, so -- both content and location log files move around, and symlinks @@ -61,6 +63,12 @@ upgrade = do Annex.queueRun setVersion + + -- add new line to auto-merge hashed location logs + -- this commits, so has to come after the upgrade + g <- Annex.gitRepo + liftIO $ Command.Init.gitAttributesWrite g + return True moveContent :: Annex () @@ -110,8 +118,12 @@ moveLocationLogs = do let f = dir l liftIO $ createDirectoryIfMissing True (parentDir dest) -- could just git mv, but this way deals with - -- log files that are not checked into git - liftIO $ copyFile f dest + -- log files that are not checked into git, + -- as well as merging with already upgraded + -- logs that have been pulled from elsewhere + old <- liftIO $ readLog f + new <- liftIO $ readLog dest + liftIO $ writeLog dest (old++new) Annex.queue "add" [Param "--"] dest Annex.queue "add" [Param "--"] f Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f diff --git a/debian/changelog b/debian/changelog index c40385bebe..68afb12933 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,12 +1,9 @@ git-annex (0.20110316) UNRELEASED; urgency=low - * Reorganized .git/annex/objects and .git-annex/; annex.version=2 + * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it will automatically upgrade it to the new format, staging all necessary changes to git. - * Note that remotes must be running this version of git-annex, - and must also have been upgraded, in order for git-annex to - communicate with them. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. From c93cd81a3372f50fb5b5c8b8668536c125dce463 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 12:06:32 -0400 Subject: [PATCH 1065/8313] define feature test macro --- Touch.hsc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Touch.hsc b/Touch.hsc index 689c58765f..f0c4debb33 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -22,6 +22,10 @@ import Foreign.C #include #include +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE >= 200809L +#endif + data TimeSpec = TimeSpec CTime CLong instance Storable TimeSpec where From 0512c7dcfa9e4e95044b9717fc864380b4c08214 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 16:07:26 +0000 Subject: [PATCH 1066/8313] Added a comment --- ...comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment new file mode 100644 index 0000000000..f26239c3e9 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-03-16T16:07:26Z" + content=""" +Hmm.. is utimensat available at all? + +I've committed an update that may convince at least some compilers to expose this newer POSIX stuff. I don't know if it will help, please let me know. +"""]] From e044d40cbda1a2740fca373922f29f2d57174de8 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 16:29:33 +0000 Subject: [PATCH 1067/8313] Comment moderation --- ...mment_3_0fc6ff79a357b1619d13018ccacc7c10._comment | 8 ++++++++ ...mment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment | 12 ++++++++++++ ...mment_5_821c382987f105da72a50e0a5ce61fdc._comment | 12 ++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment create mode 100644 doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment create mode 100644 doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment diff --git a/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment b/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment new file mode 100644 index 0000000000..ea4fb6c23e --- /dev/null +++ b/doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-03-16T15:40:56Z" + content=""" +Sometimes, I might want to fill up the disk as much as possible. Thus, a warning is preferable to erroring out too early, imo -- Richard +"""]] diff --git a/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment b/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment new file mode 100644 index 0000000000..c558ee65ee --- /dev/null +++ b/doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-03-16T15:47:17Z" + content=""" +If you can't segment the names retroactively, it's better to start with segmenting, imo. + +As subdirectories are cheap, going with ab/cd/rest or even ab/cd/ef/rest by default wouldn't hurt. + +Your point about git not needing to create as many tree objects is a kicker indeed. If I were you, I would default to segmentation. +"""]] diff --git a/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment b/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment new file mode 100644 index 0000000000..2032bce3c0 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 5" + date="2011-03-16T15:51:30Z" + content=""" +Hashing & segmenting seems to be around the corner, which is nice :) + +Is there a chance that you will optionally add mtime to your native metadata store? If yes, I'd rather wait for v2 to start with the native system from the start. If not, I will probably set it up tonight. + +PS: While posting from work, my comments are held for moderation once again. I am somewhat confused as to why this happens when I can just submit directly from home. And yes, I am using the same auth provider and user in both cases. +"""]] From ba584005b04a33d2defcfec391f4ea3d94eec1ab Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 16:32:52 +0000 Subject: [PATCH 1068/8313] Added a comment --- ...comment_6_8834c3a3f1258c4349d23aff8549bf35._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment b/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment new file mode 100644 index 0000000000..ff86e3970b --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-03-16T16:32:52Z" + content=""" +The mtime cannot be stored for all keys. Consider a SHA1 key. The mtime is irrelevant; 2 files with different mtimes, when added to the SHA1 backend, should get the same key. + +Probably our spam filter doesn't like your work IP. +"""]] From f655cecfedfa17d6d59415da0e787d5c63629f34 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 16 Mar 2011 16:49:20 +0000 Subject: [PATCH 1069/8313] Added a comment --- .../comment_2_bf112edd075fbebe4fc959a387946eb9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment new file mode 100644 index 0000000000..0222e645b9 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2011-03-16T16:49:18Z" + content=""" +Just pulled the changes, it still fails to build. utimensat doesn't seem to exist on OSX 10.6.6. +"""]] From 1443fcfe022028c2c074fc555d1e0d595fd4db95 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 12:52:30 -0400 Subject: [PATCH 1070/8313] don't use queue when upgrading In a large repo, just queuing the things to do used a lot of ram. --- Upgrade/V1.hs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 797bdee0dc..ffb774f7de 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -61,7 +61,6 @@ upgrade = do updateSymlinks moveLocationLogs - Annex.queueRun setVersion -- add new line to auto-merge hashed location logs @@ -89,17 +88,18 @@ updateSymlinks :: Annex () updateSymlinks = do g <- Annex.gitRepo files <- liftIO $ Git.inRepo g [Git.workTree g] - forM_ files $ fixlink + forM_ files $ (fixlink g) where - fixlink f = do + fixlink g f = do r <- lookupFile1 f case r of Nothing -> return () Just (k, _) -> do link <- calcGitLink f k - liftIO $ removeFile f - liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f + liftIO $ do + removeFile f + createSymbolicLink link f + Git.run g "add" [Param "--", File f] moveLocationLogs :: Annex () moveLocationLogs = do @@ -123,10 +123,11 @@ moveLocationLogs = do -- logs that have been pulled from elsewhere old <- liftIO $ readLog f new <- liftIO $ readLog dest - liftIO $ writeLog dest (old++new) - Annex.queue "add" [Param "--"] dest - Annex.queue "add" [Param "--"] f - Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f + liftIO $ do + writeLog dest (old++new) + Git.run g "add" [Param "--", File dest] + Git.run g "add" [Param "--", File f] + Git.run g "rm" [Param "--quiet", Param "-f", Param "--", File f] oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = From a4bc3d6f38e45d982fc20c7c7e84207b5140b24f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 13:16:52 -0400 Subject: [PATCH 1071/8313] bare repo upgrade support --- Upgrade/V1.hs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index ffb774f7de..1bf3cc0e85 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -57,17 +57,21 @@ upgrade :: Annex Bool upgrade = do showSideAction "Upgrading object directory layout v1 to v2..." - moveContent - updateSymlinks - moveLocationLogs + g <- Annex.gitRepo + if Git.repoIsLocalBare g + then do + moveContent + else do + moveContent + updateSymlinks + moveLocationLogs + + -- add new line to auto-merge hashed location logs + -- this commits, so has to come after the upgrade + g <- Annex.gitRepo + liftIO $ Command.Init.gitAttributesWrite g setVersion - - -- add new line to auto-merge hashed location logs - -- this commits, so has to come after the upgrade - g <- Annex.gitRepo - liftIO $ Command.Init.gitAttributesWrite g - return True moveContent :: Annex () From 35cbd107d5107eb96f52860098b1036c73281715 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 13:46:08 -0400 Subject: [PATCH 1072/8313] detect systems w/o utmensat and ifdef out code that needs it --- Touch.hsc | 37 +++++++++++++++++++++++++------------ debian/changelog | 1 + 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Touch.hsc b/Touch.hsc index f0c4debb33..456175182e 100644 --- a/Touch.hsc +++ b/Touch.hsc @@ -28,6 +28,25 @@ import Foreign.C data TimeSpec = TimeSpec CTime CLong +touch :: FilePath -> TimeSpec -> Bool -> IO () +touch file mtime follow = touchBoth file omitTime mtime follow + +touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () + +omitTime :: TimeSpec +nowTime :: TimeSpec + +#if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW) + +at_fdcwd :: CInt +at_fdcwd = #const AT_FDCWD + +at_symlink_nofollow :: CInt +at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW + +omitTime = TimeSpec 0 #const UTIME_OMIT +nowTime = TimeSpec 0 #const UTIME_NOW + instance Storable TimeSpec where -- use the larger alignment of the two types in the struct alignment _ = max sec_alignment nsec_alignment @@ -43,12 +62,6 @@ instance Storable TimeSpec where #{poke struct timespec, tv_sec} ptr sec #{poke struct timespec, tv_nsec} ptr nsec -{- special timespecs -} -omitTime :: TimeSpec -omitTime = TimeSpec 0 #const UTIME_OMIT -nowTime :: TimeSpec -nowTime = TimeSpec 0 #const UTIME_NOW - {- While its interface is beastly, utimensat is in recent POSIX standards, unlike futimes. -} foreign import ccall "utimensat" @@ -56,7 +69,6 @@ foreign import ccall "utimensat" {- Changes the access and/or modification times of an existing file. Can follow symlinks, or not. Throws IO error on failure. -} -touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO () touchBoth file atime mtime follow = allocaArray 2 $ \ptr -> withCString file $ \f -> do @@ -66,12 +78,13 @@ touchBoth file atime mtime follow = then throwErrno "touchBoth" else return () where - at_fdcwd = #const AT_FDCWD - at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW - flags = if follow then 0 else at_symlink_nofollow -touch :: FilePath -> TimeSpec -> Bool -> IO () -touch file mtime follow = touchBoth file omitTime mtime follow +#else +#warning "utimensat not available; building without symlink timestamp preservation support" +omitTime = TimeSpec 0 (-1) +nowTime = TimeSpec 0 (-2) +touchBoth _ _ _ _ = return () +#endif diff --git a/debian/changelog b/debian/changelog index e7017a26d0..110fc2b4c0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (0.24) UNRELEASED; urgency=low content are made to have the same mtime as the original file. While git does not preserve that information, this allows a tool like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.) -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 From a9111ff7e04161afcdf2e335a03d528d5afc780b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Wed, 16 Mar 2011 17:46:40 +0000 Subject: [PATCH 1073/8313] Added a comment --- .../comment_3_a46080fbe82adf0986c5dc045e382501._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment new file mode 100644 index 0000000000..7e79dea881 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-03-16T17:46:40Z" + content=""" +Alright, I've added #idefs and the symlink timestamp mirroring feature will be unavailable on OSX until I get a version that works there. +"""]] From bc21502b9a640e798dc6bbbb255aa9742a1c6187 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 15:10:15 -0400 Subject: [PATCH 1074/8313] use queue when upgrading, flushing every so often Added a cheap way to query the size of a queue. runQueueAt is not the default yet only because there may be some code that expects to be able to queue some suff, do something else, and run the whole queue at the end. 10240 is an arbitrary size for the queue. If we assume annexed filenames are between 10 and 255 characters long, then the queue will build up between 100kb and 2550kb long commands. The max command line length on linux is somewhere above 20k, so this is a fairly good balance -- the queue will buffer only a few megabytes of stuff and a minimal number of commands will be run by xargs. Also, insert queue items strictly, this should save memory. --- Annex.hs | 8 ++++++++ CmdLine.hs | 2 +- GitQueue.hs | 17 ++++++++++++----- Upgrade/V1.hs | 25 +++++++++++++------------ 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Annex.hs b/Annex.hs index f8cfd0ec92..608151d824 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,6 +16,7 @@ module Annex ( gitRepo, queue, queueRun, + queueRunAt, setConfig, repoConfig ) where @@ -109,6 +110,13 @@ queueRun = do liftIO $ GitQueue.run g q put state { repoqueue = GitQueue.empty } +{- Runs the queue if the specified number of items have been queued. -} +queueRunAt :: Integer -> Annex () +queueRunAt n = do + state <- get + let q = repoqueue state + when (GitQueue.size q >= n) queueRun + {- Changes a git config setting in both internal state and .git/config -} setConfig :: String -> String -> Annex () setConfig k value = do diff --git a/CmdLine.hs b/CmdLine.hs index b8fd6af7ce..0698f2f5ea 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -99,7 +99,7 @@ startup = do shutdown :: Annex Bool shutdown = do q <- Annex.getState Annex.repoqueue - unless (q == GitQueue.empty) $ do + unless (0 == GitQueue.size q) $ do showSideAction "Recording state in git..." Annex.queueRun diff --git a/GitQueue.hs b/GitQueue.hs index 07cf9f62fc..097516c195 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -9,6 +9,7 @@ module GitQueue ( Queue, empty, add, + size, run ) where @@ -31,22 +32,28 @@ data Action = Action { {- A queue of actions to perform (in any order) on a git repository, - with lists of files to perform them on. This allows coalescing - similar git commands. -} -type Queue = M.Map Action [FilePath] +data Queue = Queue Integer (M.Map Action [FilePath]) + deriving (Show, Eq) {- Constructor for empty queue. -} empty :: Queue -empty = M.empty +empty = Queue 0 M.empty {- Adds an action to a queue. -} add :: Queue -> String -> [CommandParam] -> FilePath -> Queue -add queue subcommand params file = M.insertWith (++) action [file] queue +add (Queue n m) subcommand params file = Queue (n + 1) m' where action = Action subcommand params + m' = M.insertWith' (++) action [file] m + +{- Number of items in a queue. -} +size :: Queue -> Integer +size (Queue n _) = n {- Runs a queue on a git repository. -} run :: Git.Repo -> Queue -> IO () -run repo queue = do - forM_ (M.toList queue) $ uncurry $ runAction repo +run repo (Queue _ m) = do + forM_ (M.toList m) $ uncurry $ runAction repo return () {- Runs an Action on a list of files in a git repository. diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 1bf3cc0e85..64ca298eb4 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -66,9 +66,10 @@ upgrade = do updateSymlinks moveLocationLogs + Annex.queueRun + -- add new line to auto-merge hashed location logs -- this commits, so has to come after the upgrade - g <- Annex.gitRepo liftIO $ Command.Init.gitAttributesWrite g setVersion @@ -92,18 +93,18 @@ updateSymlinks :: Annex () updateSymlinks = do g <- Annex.gitRepo files <- liftIO $ Git.inRepo g [Git.workTree g] - forM_ files $ (fixlink g) + forM_ files $ fixlink where - fixlink g f = do + fixlink f = do r <- lookupFile1 f case r of Nothing -> return () Just (k, _) -> do link <- calcGitLink f k - liftIO $ do - removeFile f - createSymbolicLink link f - Git.run g "add" [Param "--", File f] + liftIO $ removeFile f + liftIO $ createSymbolicLink link f + Annex.queue "add" [Param "--"] f + Annex.queueRunAt 1024 moveLocationLogs :: Annex () moveLocationLogs = do @@ -127,11 +128,11 @@ moveLocationLogs = do -- logs that have been pulled from elsewhere old <- liftIO $ readLog f new <- liftIO $ readLog dest - liftIO $ do - writeLog dest (old++new) - Git.run g "add" [Param "--", File dest] - Git.run g "add" [Param "--", File f] - Git.run g "rm" [Param "--quiet", Param "-f", Param "--", File f] + liftIO $ writeLog dest (old++new) + Annex.queue "add" [Param "--"] dest + Annex.queue "add" [Param "--"] f + Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f + Annex.queueRunAt 1024 oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = From d7ef5fd2941fa66aa7f9c998fe4acfda60e63295 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 15:48:26 -0400 Subject: [PATCH 1075/8313] add explicit upgrade command --- Command/Upgrade.hs | 22 ++++++++++++++++++++++ GitAnnex.hs | 2 ++ Upgrade/V1.hs | 4 ++-- debian/changelog | 3 ++- doc/git-annex.mdwn | 6 ++++++ 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 Command/Upgrade.hs diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs new file mode 100644 index 0000000000..3c9fa3eebf --- /dev/null +++ b/Command/Upgrade.hs @@ -0,0 +1,22 @@ +{- git-annex command + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Upgrade where + +import Command + +command :: [Command] +command = [Command "upgrade" paramNothing seek "upgrade repository layout"] + +seek :: [CommandSeek] +seek = [withNothing start] + +start :: CommandStartNothing +start = do + -- The actual upgrading is handled by just running any command, + -- so nothing extra needs to be done. + return $ Just $ return $ Just $ return True diff --git a/GitAnnex.hs b/GitAnnex.hs index da91f6e74e..b9c22bdfb4 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -41,6 +41,7 @@ import qualified Command.Trust import qualified Command.Untrust import qualified Command.Semitrust import qualified Command.Map +import qualified Command.Upgrade cmds :: [Command] cmds = concat @@ -70,6 +71,7 @@ cmds = concat , Command.Whereis.command , Command.Migrate.command , Command.Map.command + , Command.Upgrade.command ] options :: [Option] diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 64ca298eb4..f1c3e6143c 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -104,7 +104,7 @@ updateSymlinks = do liftIO $ removeFile f liftIO $ createSymbolicLink link f Annex.queue "add" [Param "--"] f - Annex.queueRunAt 1024 + Annex.queueRunAt 10240 moveLocationLogs :: Annex () moveLocationLogs = do @@ -132,7 +132,7 @@ moveLocationLogs = do Annex.queue "add" [Param "--"] dest Annex.queue "add" [Param "--"] f Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f - Annex.queueRunAt 1024 + Annex.queueRunAt 10240 oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = diff --git a/debian/changelog b/debian/changelog index e9fdceee6f..a9b9249c1b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.20110316) UNRELEASED; urgency=low * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it will automatically upgrade it to the new format, staging all - necessary changes to git. + necessary changes to git. See + for details. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index e559e8cba6..ee40190682 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -261,6 +261,12 @@ Many git-annex commands will stage changes for later `git commit` by you. git annex setkey --key=WORM-s3-m1287765018--file /tmp/file +* upgrade + + Upgrades the repository to current layout. Upgrades are done automatically + whenever a newer git annex encounters an old repository; this command + allows explcitly starting an upgrade. + # OPTIONS * --force From 49da5d1a7b6d86b6795399322ce7ff6be921855f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 15:51:42 -0400 Subject: [PATCH 1076/8313] upgrade documentation --- debian/NEWS | 11 ++++++++ doc/upgrades.mdwn | 67 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 debian/NEWS create mode 100644 doc/upgrades.mdwn diff --git a/debian/NEWS b/debian/NEWS new file mode 100644 index 0000000000..19c277eb0c --- /dev/null +++ b/debian/NEWS @@ -0,0 +1,11 @@ +git-annex (0.20110316) unstable; urgency=low + + This version reorganises the layout of git-annex's files in your repository. + There is an upgrade process to convert a repository from the old git-annex + to this version. While git-annex will attempt to transparently handle + upgrades, you may want to drive the upgrade process by hand. + + See or + /usr/share/doc/git-annex/html/upgrades.html + + -- Joey Hess Wed, 16 Mar 2011 15:49:15 -0400 diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn new file mode 100644 index 0000000000..245212124a --- /dev/null +++ b/doc/upgrades.mdwn @@ -0,0 +1,67 @@ +Occasionally improvments are made to how git-annex stores its data, +that require an upgrade process to convert repositories made with an older +version to be used by a newer version. It's annoying, it should happen +rarely, but sometimes, it's worth it. + +There's a committment that git-annex will always support upgrades from all +past versions. After all, you may have offline drives from an earlier +git-annex, and might want to use them with a newer git-annex. + +## Upgrade process + +git-annex will automatically notice if it is run in a repository that +needs an upgrade, and perform the upgrade before running whatever it +was asked to do. Or you can use the "git annex upgrade" command to +explicitly do an upgrade. The upgrade can tend to take a while, +if you have a lot of files. + +Each clone of a repository should be individually upgraded. +Until a repository's remotes have been upgraded, git-annex +may refuse to communicate with them. + +Generally, start by upgrading one repository, and then you can commit +the changes git-annex staged during upgrade, and push them out to other +repositories. And then upgrade those other repositories. Doing it this +way avoids git-annex doing some duplicate work during the upgrade. + +The upgrade process is guaranteed to be conflict-free. Unless you +already have git conflicts in your repository or between repositories. +Upgrading a repository with conflicts is not recommended; resolve the +conflicts first before upgrading git-annex. + +Example upgrade process: + + cd localrepo + git pull + git annex upgrade + (Upgrading object directory layout v1 to v2...) + git commit -a -m "upgrade v1 to v2" + git push + + ssh remote + cd remoterepo + git pull + git annex upgrade + ... + +## Upgrade events, so far + +### v1 -> v2 (git-annex version 0.23 to version 0.20110316) + +Involved adding hashing to .git/annex/ and changing the names of all keys. +Symlinks changed. + +Also, hashing was added to location log files in .git-annex/. +And .gitattributes needed to have another line added to it. + +Handled transparently. + +### v0 -> v1 (git-annex version 0.03 to version 0.04) + +Involved a reogranisation of the layout of .git/annex/. Symlinks changed. + +Handled more or less transparently, although git-annex was just 2 weeks +old at the time, and had few users other than Joey. + +This upgrade is belived to still be supported, but has not been tested +lately. From 8ad3fd26570e8d6a8066f35353b535e4af81b7ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:07:28 -0400 Subject: [PATCH 1077/8313] remove debugging --- Content.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Content.hs b/Content.hs index a59484b5ab..4bd8265c2d 100644 --- a/Content.hs +++ b/Content.hs @@ -171,7 +171,6 @@ getKeysPresent' dir = do present d = do result <- try $ getFileStatus $ d takeFileName d - liftIO $ putStrLn $ "trying " ++ (d takeFileName d) case result of Right s -> return $ isRegularFile s Left _ -> return False From b7a49283fb94a9fc1d5b4d66f0f992e329ef1ee7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:07:33 -0400 Subject: [PATCH 1078/8313] set version before running operation that can commit --- Upgrade/V1.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index f1c3e6143c..270de5f74b 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -61,18 +61,19 @@ upgrade = do if Git.repoIsLocalBare g then do moveContent + setVersion else do moveContent updateSymlinks moveLocationLogs Annex.queueRun + setVersion -- add new line to auto-merge hashed location logs -- this commits, so has to come after the upgrade liftIO $ Command.Init.gitAttributesWrite g - setVersion return True moveContent :: Annex () From 6255865c6c0a2fee75ff120cd30211674044c1fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:14:50 -0400 Subject: [PATCH 1079/8313] update --- doc/bugs/free_space_checking.mdwn | 10 ++++++++++ doc/todo/object_dir_reorg_v2.mdwn | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn index 34528a7b35..eaa3294d60 100644 --- a/doc/bugs/free_space_checking.mdwn +++ b/doc/bugs/free_space_checking.mdwn @@ -6,3 +6,13 @@ file around. * And, need a way to tell the size of a file before copying it from a remote, to check local disk space. + + As of annex.version 2, this metadata can be available for any type + of backend. Newly added files will always have file size metadata, + while files that used a SHA backend and were added before the upgrade + won't. + + So, need a migration process from eg SHA1 to SHA1+filesize. It will + find files that lack size info, and rename their keys to add the size + info. Users with old repos can run this on them, to get the missing + info recorded. diff --git a/doc/todo/object_dir_reorg_v2.mdwn b/doc/todo/object_dir_reorg_v2.mdwn index 1c2d2f21b7..49666ddc79 100644 --- a/doc/todo/object_dir_reorg_v2.mdwn +++ b/doc/todo/object_dir_reorg_v2.mdwn @@ -19,3 +19,7 @@ all users, so this should be the *last* reorg in the forseeable future. (Probably everything after ",k" should be part of the key, even if it contains the "," separator character. Otherwise an escaping mechanism would be needed.) + +[[done]] now! + +Although [[bugs/free_space_checking]] is not quite there --[[Joey]] From 00eb8ae82981b36d1f96740a703d92ed3c913877 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 16:25:20 -0400 Subject: [PATCH 1080/8313] prepping experimental release --- debian/NEWS | 2 +- debian/changelog | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/NEWS b/debian/NEWS index 19c277eb0c..df8518cef9 100644 --- a/debian/NEWS +++ b/debian/NEWS @@ -1,4 +1,4 @@ -git-annex (0.20110316) unstable; urgency=low +git-annex (0.20110316) experimental; urgency=low This version reorganises the layout of git-annex's files in your repository. There is an upgrade process to convert a repository from the old git-annex diff --git a/debian/changelog b/debian/changelog index a9b9249c1b..c572a4e768 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110316) UNRELEASED; urgency=low +git-annex (0.20110316) experimental; urgency=low * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it @@ -21,7 +21,7 @@ git-annex (0.20110316) UNRELEASED; urgency=low like metastore to be used with annexed files. (Currently this is only done on systems supporting POSIX 200809.) - -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 + -- Joey Hess Wed, 16 Mar 2011 16:20:23 -0400 git-annex (0.23) unstable; urgency=low From b630705981ba4be8d54896e9418ead43bc4c4630 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 16 Mar 2011 20:32:03 +0000 Subject: [PATCH 1081/8313] Added a comment --- .../comment_4_760437bf3ba972a775bb190fb4b38202._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment new file mode 100644 index 0000000000..6b1e03b026 --- /dev/null +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2011-03-16T20:32:01Z" + content=""" +Just tried it out on my mac and it's working again. I guess this issue could be closed for now. +"""]] From a5b48c7b66b16aee4d624325d4cae7e2b473224a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 16 Mar 2011 21:05:39 +0000 Subject: [PATCH 1082/8313] Added a comment --- ...mment_7_42501404c82ca07147e2cce0cff59474._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment diff --git a/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment b/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment new file mode 100644 index 0000000000..fc866c57a6 --- /dev/null +++ b/doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 7" + date="2011-03-16T21:05:38Z" + content=""" +Ah, OK. I assumed the metadata would be attached to a key, not part of the key. This seems to make upgrades/extensions down the line harder than they need to be, but you are right that this way, merges are not, and never will be, an issue. + +Though with the SHA1 backend, changing files can be tracked. This means that tracking changes in mtime or other is possible. It also means that there are potential merge issues. But I won't argue the point endlessly. I can accept design decisions :) + +The prefix at work is from a university netblock so yes, it might be on a few hundred proxy lists etc. +"""]] From 67be8f13fbd36d70083d56d2f038af7c085a9b89 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 17:37:56 -0400 Subject: [PATCH 1083/8313] tweak --- doc/upgrades.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn index 245212124a..1371dc0330 100644 --- a/doc/upgrades.mdwn +++ b/doc/upgrades.mdwn @@ -35,7 +35,7 @@ Example upgrade process: git pull git annex upgrade (Upgrading object directory layout v1 to v2...) - git commit -a -m "upgrade v1 to v2" + git commit -m "upgrade v1 to v2" git push ssh remote From 63360f776783d2527af8c8ed9a4dd79d3159b9a1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:33:28 -0400 Subject: [PATCH 1084/8313] update --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index c572a4e768..c71c5ee25a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,7 @@ git-annex (0.20110316) experimental; urgency=low * New repository format, annex.version=2. * The first time git-annex is run in an old format repository, it will automatically upgrade it to the new format, staging all - necessary changes to git. See - for details. + necessary changes to git. Also added a "git annex upgrade" command. * Colons are now avoided in filenames, so bare clones of git repos can be put on USB thumb drives formatted with vFAT or similar filesystems. From 1079ade2083a7294cb2ce266d8f32bf70821cf1b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:41:02 -0400 Subject: [PATCH 1085/8313] releasing version 0.24 --- debian/changelog | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 110fc2b4c0..f5fc4eebe4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,8 @@ -git-annex (0.24) UNRELEASED; urgency=low +git-annex (0.24) unstable; urgency=low + + Branched the 0.24 series, which will be maintained for a while to + support v1 git-annex repos, while main development moves to the 0.2011 + series, with v2 git-annex repos. * Add Suggests on graphviz. Closes: #618039 * When adding files to the annex, the symlinks pointing at the annexed @@ -7,7 +11,7 @@ git-annex (0.24) UNRELEASED; urgency=low like metastore to be used with annexed files. (Currently this is only done on systems supporting POSIX 200809.) - -- Joey Hess Sun, 13 Mar 2011 14:25:17 -0400 + -- Joey Hess Wed, 16 Mar 2011 18:35:13 -0400 git-annex (0.23) unstable; urgency=low From 7e99ac0222b3b3e69cce06d7de720a78be11b209 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:41:32 -0400 Subject: [PATCH 1086/8313] add news item for git-annex 0.24 --- doc/news/version_0.19.mdwn | 17 ----------------- doc/news/version_0.24.mdwn | 11 +++++++++++ 2 files changed, 11 insertions(+), 17 deletions(-) delete mode 100644 doc/news/version_0.19.mdwn create mode 100644 doc/news/version_0.24.mdwn diff --git a/doc/news/version_0.19.mdwn b/doc/news/version_0.19.mdwn deleted file mode 100644 index 5d6ab47be0..0000000000 --- a/doc/news/version_0.19.mdwn +++ /dev/null @@ -1,17 +0,0 @@ -git-annex 0.19 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * configure: Support using the uuidgen command if the uuid command is - not available. - * Allow --exclude to be specified more than once. - * There are now three levels of repository trust. - * untrust: Now marks the current repository as untrusted. - * semitrust: Now restores the default trust level. (What untrust used to do.) - * fsck, drop: Take untrusted repositories into account. - * Bugfix: Files were copied from trusted remotes first even if their - annex.cost was higher than other remotes. - * Improved temp file handling. Transfers of content can now be resumed - from temp files later; the resume does not have to be the immediate - next git-annex run. - * unused: Include partially transferred content in the list. - * Bugfix: Running a second git-annex while a first has a transfer in - progress no longer deletes the first processes's temp file."""]] \ No newline at end of file diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn new file mode 100644 index 0000000000..b240820408 --- /dev/null +++ b/doc/news/version_0.24.mdwn @@ -0,0 +1,11 @@ +git-annex 0.24 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + Branched the 0.24 series, which will be maintained for a while to + support v1 git-annex repos, while main development moves to the 0.2011 + series, with v2 git-annex repos. + * Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039) + * When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.)"""]] \ No newline at end of file From 40652e0eaf019ce076fcee24c25fb1f8c950ac5d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 18:43:32 -0400 Subject: [PATCH 1087/8313] format --- doc/news/version_0.24.mdwn | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn index b240820408..2d94a0e9bb 100644 --- a/doc/news/version_0.24.mdwn +++ b/doc/news/version_0.24.mdwn @@ -1,11 +1,12 @@ +Branched the 0.24 series, which will be maintained for a while to +support v1 git-annex repos, while main development moves to the 0.2011 +series, with v2 git-annex repos. + git-annex 0.24 released with [[!toggle text="these changes"]] [[!toggleable text=""" - Branched the 0.24 series, which will be maintained for a while to - support v1 git-annex repos, while main development moves to the 0.2011 - series, with v2 git-annex repos. - * Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039) - * When adding files to the annex, the symlinks pointing at the annexed - content are made to have the same mtime as the original file. - While git does not preserve that information, this allows a tool - like metastore to be used with annexed files. - (Currently this is only done on systems supporting POSIX 200809.)"""]] \ No newline at end of file +* Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039) +* When adding files to the annex, the symlinks pointing at the annexed + content are made to have the same mtime as the original file. + While git does not preserve that information, this allows a tool + like metastore to be used with annexed files. + (Currently this is only done on systems supporting POSIX 200809.)"""]] From 2f515fb57f1316bd5732faa4ce03d30ed099b74a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 17 Mar 2011 00:04:38 +0000 Subject: [PATCH 1088/8313] --- doc/bugs/check_for_curl_in_configure.hs.mdwn | 39 ++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/bugs/check_for_curl_in_configure.hs.mdwn diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn new file mode 100644 index 0000000000..ee24769ce7 --- /dev/null +++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn @@ -0,0 +1,39 @@ +I thought this might be useful, since curl is being used for the URL backend, it might be worth checking for it's existence. + +
+diff --git a/configure.hs b/configure.hs
+index 772ba54..1a563e0 100644
+--- a/configure.hs
++++ b/configure.hs
+@@ -13,6 +13,7 @@ tests = [
+        , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
+        , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
++       , TestCase "curl" $ requireCmd "curl" "curl --version >/dev/null"
+        , TestCase "unicode FilePath support" $ unicodeFilePath
+        ] ++ shaTestCases [1, 256, 512, 224, 384]
+
+ +also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not) + +
+diff --git a/Backend/URL.hs b/Backend/URL.hs
+index 29dc8fe..4afcf86 100644
+--- a/Backend/URL.hs
++++ b/Backend/URL.hs
+@@ -50,10 +50,13 @@ dummyFsck _ _ _ = return True
+ dummyOk :: Key -> Annex Bool
+ dummyOk _ = return True
+ 
++curl :: [CommandParam] -> IO Bool
++curl = boolSystem "curl"
++
+ downloadUrl :: Key -> FilePath -> Annex Bool
+ downloadUrl key file = do
+        showNote "downloading"
+        showProgress -- make way for curl progress bar
+-       liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
++       liftIO $ curl [Params "-# -o", File file, File url]
+        where
+                url = join ":" $ drop 1 $ split ":" $ show key 
+
From 14d7049886314cc73148c61921c2193be92c835f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 21:04:25 -0400 Subject: [PATCH 1089/8313] releasing version 0.20110316 --- doc/news/version_0.24.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn index 2d94a0e9bb..81b013a26f 100644 --- a/doc/news/version_0.24.mdwn +++ b/doc/news/version_0.24.mdwn @@ -1,6 +1,6 @@ -Branched the 0.24 series, which will be maintained for a while to -support v1 git-annex repos, while main development moves to the 0.2011 -series, with v2 git-annex repos. +Branched the 0.24 series, which will be maintained for a while (in the +stable branch in git) to support v1 git-annex repos, while main development +moves to the 0.2011 series, with v2 git-annex repos. git-annex 0.24 released with [[!toggle text="these changes"]] [[!toggleable text=""" From 455721597b4ed8d9e2379e66ccc47a4c9459867c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 16 Mar 2011 21:05:00 -0400 Subject: [PATCH 1090/8313] add news item for git-annex 0.20110316 --- doc/news/version_0.20.mdwn | 12 ------------ doc/news/version_0.20110316.mdwn | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 12 deletions(-) delete mode 100644 doc/news/version_0.20.mdwn create mode 100644 doc/news/version_0.20110316.mdwn diff --git a/doc/news/version_0.20.mdwn b/doc/news/version_0.20.mdwn deleted file mode 100644 index 9b95b652e5..0000000000 --- a/doc/news/version_0.20.mdwn +++ /dev/null @@ -1,12 +0,0 @@ -git-annex 0.20 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Preserve specified file ordering when instructed to act on multiple - files or directories. For example, "git annex get a b" will now always - get "a" before "b". Previously it could operate in either order. - * unannex: Commit staged changes at end, to avoid some confusing behavior - with the pre-commit hook, which would see some types of commits after - an unannex as checking in of an unlocked file. - * map: New subcommand that uses graphviz to display a nice map of - the git repository network. - * Deal with the mtl/monads-fd conflict. - * configure: Check for sha1sum."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110316.mdwn b/doc/news/version_0.20110316.mdwn new file mode 100644 index 0000000000..5654c15bcf --- /dev/null +++ b/doc/news/version_0.20110316.mdwn @@ -0,0 +1,24 @@ +News for git-annex 0.20110316: + + This version reorganises the layout of git-annex's files in your repository. + There is an upgrade process to convert a repository from the old git-annex + to this version. While git-annex will attempt to transparently handle + upgrades, you may want to drive the upgrade process by hand. + See <http://git-annex.branchable.com/upgrades/> or + /usr/share/doc/git-annex/html/upgrades.html + +git-annex 0.20110316 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * New repository format, annex.version=2. + * The first time git-annex is run in an old format repository, it + will automatically upgrade it to the new format, staging all + necessary changes to git. Also added a "git annex upgrade" command. + * Colons are now avoided in filenames, so bare clones of git repos + can be put on USB thumb drives formatted with vFAT or similar + filesystems. + * Added two levels of hashing to object directory and .git-annex logs, + to improve scalability with enormous numbers of annexed + objects. (With one hundred million annexed objects, each + directory would contain fewer than 1024 files.) + * The setkey, fromkey, and dropkey subcommands have changed how + the key is specified. --backend is no longer used with these."""]] \ No newline at end of file From cb1c68de5c26fcaa83ccdbc3d6ef79e400a3e297 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 17 Mar 2011 00:47:03 -0400 Subject: [PATCH 1091/8313] format --- doc/news/version_0.20110316.mdwn | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/news/version_0.20110316.mdwn b/doc/news/version_0.20110316.mdwn index 5654c15bcf..968722c0e6 100644 --- a/doc/news/version_0.20110316.mdwn +++ b/doc/news/version_0.20110316.mdwn @@ -1,11 +1,8 @@ -News for git-annex 0.20110316: - - This version reorganises the layout of git-annex's files in your repository. - There is an upgrade process to convert a repository from the old git-annex - to this version. While git-annex will attempt to transparently handle - upgrades, you may want to drive the upgrade process by hand. - See <http://git-annex.branchable.com/upgrades/> or - /usr/share/doc/git-annex/html/upgrades.html +This version reorganises the layout of git-annex's files in your repository. +There is an upgrade process to convert a repository from the old git-annex +to this version. While git-annex will attempt to transparently handle +upgrades, you may want to drive the upgrade process by hand. +See [[upgrades]] for details. git-annex 0.20110316 released with [[!toggle text="these changes"]] [[!toggleable text=""" @@ -21,4 +18,4 @@ git-annex 0.20110316 released with [[!toggle text="these changes"]] objects. (With one hundred million annexed objects, each directory would contain fewer than 1024 files.) * The setkey, fromkey, and dropkey subcommands have changed how - the key is specified. --backend is no longer used with these."""]] \ No newline at end of file + the key is specified. --backend is no longer used with these."""]] From a24a04ff7e417f588bb627af8a8a139c46834fc6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 17 Mar 2011 09:59:05 +0000 Subject: [PATCH 1092/8313] --- doc/install.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/install.mdwn b/doc/install.mdwn index 9eb8bbacf7..a531015782 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -16,6 +16,7 @@ To build and use git-annex, you will need: (or uuidgen from util-linux) * `xargs`: * `rsync`: +* `curl` : (optional, but recommended) * `sha1sum`: (optional, but recommended) * Then just [[download]] git-annex and run: `make; make install` From ba91851632a5756c96e47d6036713fdb417e9ce9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 17 Mar 2011 10:14:19 +0000 Subject: [PATCH 1093/8313] --- doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn new file mode 100644 index 0000000000..8ce7e84ad8 --- /dev/null +++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn @@ -0,0 +1,11 @@ +I was trying out the example with the walkthrough [[walkthrough/using_the_URL_backend]]. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with + + +
+[jtang@lenny gc]$ git annex drop -v curl-7.21.4.tar.gz
+drop curl-7.21.4.tar.gz
+failed
+git-annex: 1 failed
+
+ +At first I thought it was just my OSX machine not having the coreutils stuff load up before the BSD utils, but I then tried the same thing on my archlinux machine and it showed the same behaviour, that is I could not drop a file with the URL backend as shown in the walkthrough. From 9aada06745d138c953d28929ac561449ddc8fabe Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 17 Mar 2011 15:44:27 +0000 Subject: [PATCH 1094/8313] --- ...magic_upgrade_of_the_object_directory_safe__63__.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn new file mode 100644 index 0000000000..5643f6b7a0 --- /dev/null +++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn @@ -0,0 +1,9 @@ +Consider the following two use cases: + +* I have a git-annex repo on a portable medium and carry it around between several machines. I use it on a non-important system with the most current git-annex installed, automagic upgrade happens. I am now forced to upgrade git-annex on all other machines. Bonus points if this happens in the background and I don't even notice it until it's too late. + +* My system crashes and I use a rescue CD to access local data, including git-annex. The rescue CD includes a newer version of git-annex and once my system is restored, I am forced to upgrade git-annex locally. + +My suggestion would be not to upgrade automatically, but to either ask the user if this is OK or to error out and request that they run git annex update by hand. + +Optionally, this could be done via a local config variable which should default to error or ask, not upgrade. From 7b5b1276085be3d2c12c8c28b7be1aceccae44f9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 17 Mar 2011 11:49:21 -0400 Subject: [PATCH 1095/8313] Fix dropping of files using the URL backend. --- Backend/URL.hs | 2 +- debian/changelog | 6 ++++++ doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Backend/URL.hs b/Backend/URL.hs index 02ce3563cd..210c7c5b48 100644 --- a/Backend/URL.hs +++ b/Backend/URL.hs @@ -42,7 +42,7 @@ dummyStore :: FilePath -> Key -> Annex Bool dummyStore _ _ = return False dummyRemove :: Key -> Maybe a -> Annex Bool -dummyRemove _ _ = return False +dummyRemove _ _ = return True dummyFsck :: Key -> Maybe FilePath -> Maybe a -> Annex Bool dummyFsck _ _ _ = return True diff --git a/debian/changelog b/debian/changelog index 47a9148124..81257a24be 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110317) UNRELEASED; urgency=low + + * Fix dropping of files using the URL backend. + + -- Joey Hess Thu, 17 Mar 2011 11:46:53 -0400 + git-annex (0.20110316) experimental; urgency=low * New repository format, annex.version=2. diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn index 8ce7e84ad8..e88bf07f4d 100644 --- a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn +++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn @@ -9,3 +9,5 @@ git-annex: 1 failed At first I thought it was just my OSX machine not having the coreutils stuff load up before the BSD utils, but I then tried the same thing on my archlinux machine and it showed the same behaviour, that is I could not drop a file with the URL backend as shown in the walkthrough. + +> Whoops, got some logic backwards. [[fixed|done]]! --[[Joey]] From 8543d9add4c717987ce4b8bbd44b285b4963c125 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 17 Mar 2011 11:57:03 -0400 Subject: [PATCH 1096/8313] check for curl in configure, thanks Jimmy --- configure.hs | 1 + doc/bugs/check_for_curl_in_configure.hs.mdwn | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/configure.hs b/configure.hs index 772ba54899..f5c2956488 100644 --- a/configure.hs +++ b/configure.hs @@ -13,6 +13,7 @@ tests = [ , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" + , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn index ee24769ce7..18edabe5e0 100644 --- a/doc/bugs/check_for_curl_in_configure.hs.mdwn +++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn @@ -14,8 +14,21 @@ index 772ba54..1a563e0 100644 ] ++ shaTestCases [1, 256, 512, 224, 384] +> Well, curl is an optional extra, so requireCmd is too strong. Changed +> to testCmd and applied, thank you! +> +> I thought about actually *using* the resulting SysConfig.curl +> to disable the URL backend if False.. but probably it's better +> to just let it fail if curl is not available. Although, if we wanted +> to add a check for wget or something and use it when curl was not +> available, that might be worth doing. --[[Joey]] + also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not) +> It's correct, typewise, but I don't see any real reason to bother +> with the change. But I do appreciate patches, which have been rare +> so far, probaby because of Haskell.. :) --[[Joey]] + 
 diff --git a/Backend/URL.hs b/Backend/URL.hs
 index 29dc8fe..4afcf86 100644

From c196875932d7b51083cc835ce0518d7a3d961198 Mon Sep 17 00:00:00 2001
From: Richard Hartmann 
Date: Thu, 17 Mar 2011 12:20:55 +0100
Subject: [PATCH 1097/8313] Fix typos

---
 doc/upgrades.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 1371dc0330..1bd97c46e2 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -58,10 +58,10 @@ Handled transparently.
 
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
-Involved a reogranisation of the layout of .git/annex/. Symlinks changed.
+Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
 
 Handled more or less transparently, although git-annex was just 2 weeks
 old at the time, and had few users other than Joey.
 
-This upgrade is belived to still be supported, but has not been tested
+This upgrade is believed to still be supported, but has not been tested
 lately.

From 98364839ca5367bdf21bc2a2cbfab6e8189aafaf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Thu, 17 Mar 2011 19:11:21 +0000
Subject: [PATCH 1098/8313]

---
 doc/bugs/No_version_information_from_cli.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/No_version_information_from_cli.mdwn

diff --git a/doc/bugs/No_version_information_from_cli.mdwn b/doc/bugs/No_version_information_from_cli.mdwn
new file mode 100644
index 0000000000..c432cf99b7
--- /dev/null
+++ b/doc/bugs/No_version_information_from_cli.mdwn
@@ -0,0 +1,16 @@
+git-annex does not listen to -v, --version or version.
+
+At the very least, it should return both the version of the binary and the version of the object store it supports.
+If it supports several annex versions, they should be listed in a comma-separated fashion.
+If git-annex is called from within an annex, it should print the version of the local object store.
+
+Sample:
+
+    % git annex version
+    git-annex version               : 0.24
+    default object store version    : 3
+    supported object store versions : 2,3
+    local object store version      : 2
+    % 
+
+The above might look like overkill, but it's in a form that will, most likely, never need to be extended.

From e31be22c12a0c69e17042138663cc6e9fa539db2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 17 Mar 2011 20:27:54 +0000
Subject: [PATCH 1099/8313]

---
 doc/bugs/check_for_curl_in_configure.hs.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn
index 18edabe5e0..bba2b81fcb 100644
--- a/doc/bugs/check_for_curl_in_configure.hs.mdwn
+++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn
@@ -23,12 +23,23 @@ index 772ba54..1a563e0 100644
 > to add a check for wget or something and use it when curl was not
 > available, that might be worth doing. --[[Joey]] 
 
+>> I was thinking that is it worth doing a generic "stat", "delete", "get" 
+>> and "put" options, I do like the idea of having the possibility of 
+>> being about to use completely arbitrary storage systems or arbitrary 
+>> transfer systems. If there was the capability of doing so it would be 
+>> interesting to see possibilities of using aria2 for using something 
+>> like bittorrent as backend, or using something like irods or some 
+>> grid storage system as the storage archive. It's just an idea as 
+>> I have seen it implemented quite well in irods.
+
 also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)
 
 > It's correct, typewise, but I don't see any real reason to bother
 > with the change. But I do appreciate patches, which have been rare
 > so far, probaby because of Haskell.. :) --[[Joey]] 
 
+>> heh agreed
+
 
 diff --git a/Backend/URL.hs b/Backend/URL.hs
 index 29dc8fe..4afcf86 100644

From be27fa41fa3aff7fb0c455d51c7d67d7b4114b69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Mar 2011 20:36:25 -0400
Subject: [PATCH 1100/8313] response

---
 doc/bugs/check_for_curl_in_configure.hs.mdwn | 21 ++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn
index bba2b81fcb..2a5227491a 100644
--- a/doc/bugs/check_for_curl_in_configure.hs.mdwn
+++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn
@@ -1,3 +1,9 @@
+[[!meta title="arbitrary/configurable backends"]]
+
+(Retitling as this has drifted..)
+
+---
+
 I thought this might be useful, since curl is being used for the URL backend, it might be worth checking for it's existence.
 
 
@@ -32,6 +38,21 @@ index 772ba54..1a563e0 100644
 >> grid storage system as the storage archive. It's just an idea as 
 >> I have seen it implemented quite well in irods.
 
+>>> I'm unsure about the idea of having a backend where that is
+>>> parameterized. It would mean that one annex's GENERIC-foo key
+>>> might be entirely different from another's key with the same backend
+>>> and details. And a misconfiguration could get data the wrong
+>>> way and get the wrong data, etc.
+>>>
+>>> I mostly look at the URL backend as an example that can be modified to
+>>> make this kind of custom backend. You already probably know enough to
+>>> make a TORRENT backend where keys are the urls to torrents to download
+>>> with `aria2c --follow-torrent=mem`.
+>>>
+>>> I am also interested in doing backends that use eg, cloud storage.
+>>> A S3 backend that could upload files to S3 in addition to downloading
+>>> them, for example, would be handy. --[[Joey]]
+
 also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)
 
 > It's correct, typewise, but I don't see any real reason to bother

From d235677926109bad7567bda101364087fd121223 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 18 Mar 2011 00:38:51 +0000
Subject: [PATCH 1101/8313] Added a comment

---
 ...comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment

diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
new file mode 100644
index 0000000000..6a34becd37
--- /dev/null
+++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-18T00:38:51Z"
+ content="""
+These are good examples; I think you've convinced me at least for upgrades going forward after v2. I'm not sure we have enough users and outdated git-annex installations to worry about it for v1.
+
+(Hoping such upgrades are rare anyway.. Part of the point of changes made in v2 was to allow lots of changes to be made later w/o needing a v3.)
+"""]]

From 028eb96f408cddab03a4b77709fb6a24806a5e57 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 09:39:27 -0400
Subject: [PATCH 1102/8313] design

---
 doc/todo/S3.mdwn | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index ec2d403ced..cb5186b095 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -1 +1,38 @@
 Support Amazon S3 as a file storage backend.
+
+There's a haskell library that looks good. Not yet in Debian.
+
+Multiple ways of using S3 are possible. Current plan is to have a S3BUCKET
+backend, that is derived from Backend.File, so it caches files locally and
+can transfer files between systems too, without involving S3.
+
+get will try to get it from S3 or from a remote. A annex.s3.cost can
+configure the cost of S3 vs the cost of other remotes.
+
+add will always upload a copy to S3.
+
+Each file in the S3 bucket is assumed to be in the annex. So unused
+will show files in the bucket that nothing points to, and dropunused remove
+them.
+
+For numcopies counting, S3 will count as 1 copy (or maybe more?), so if
+numcopies=2, then you don't fully trust S3 and request git-annex assure
+one other copy.
+
+drop will remove a file locally, but keep it in S3. drop --force *might*
+remove it from S3. TBD.
+
+annex.s3.bucket would configure the bucket the use. (And an env var or
+something configure the password.) Although the bucket
+would also be encoded in the keys. So, the configured bucket would be used
+when adding new files. A system could move from one bucket to another over
+time while still having legacy files in an earlier one; 
+perhaps you move to Europe and want new files to be put in that region.
+
+And git annex `migrate --backend=S3BUCKET --force` could move files
+between datacenters!
+
+Problem: Then the only way for unused to know what buckets are in use
+is to see what keys point to them -- but if the last file from a bucket is
+deleted, it would then not be able to say that the files in that bucket are
+all unused. Need cached list of recently seen S3 buckets?

From 70a8a3ab711fb266657653177ec58f8234bb716c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 09:47:12 -0400
Subject: [PATCH 1103/8313] update

---
 doc/todo/S3.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index cb5186b095..3d18527d48 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -36,3 +36,17 @@ Problem: Then the only way for unused to know what buckets are in use
 is to see what keys point to them -- but if the last file from a bucket is
 deleted, it would then not be able to say that the files in that bucket are
 all unused. Need cached list of recently seen S3 buckets?
+
+-----
+
+One problem with this is what key metadata to include. Should it be like
+WORM? Or like SHA1? Or just a new unique identifier for each file? It might
+be worth having S3 variants of *all* the Backend.File derived backends.
+
+More blue-sky, it might be nice to be able to union or stack together
+multiple backends, so S3BUCKET+SHA1 or S3BUCKET+WORM. That would likely
+be hard to get right.
+
+Less blue-sky, if the S3 capability were added directly to Backend.File,
+and bucket name was configured by annex.s3.bucket, then any existing
+annexed file could be upgraded to also store on S3.

From 8fed2539a3056bafb92dc55b1bd393be03fef758 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 17:54:32 +0000
Subject: [PATCH 1104/8313] new: bugs/fsck output

---
 doc/bugs/fsck_output.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/fsck_output.mdwn

diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
new file mode 100644
index 0000000000..81fecceb3f
--- /dev/null
+++ b/doc/bugs/fsck_output.mdwn
@@ -0,0 +1,16 @@
+When you check several files and the fsck fails, you get confusing output:
+
+
+fsck test1 (checksum...) 
+  Only 1 of 2 trustworthy copies of test1 exist.
+  Back it up with git-annex copy.
+
+failed
+fsck test2 (checksum...) 
+  Only 1 of 2 trustworthy copies of test2 exist.
+  Back it up with git-annex copy.
+
+failed
+

+
+The newline is in the wrong place and confuses the user. It should be printed _after_ "failed".

From 12a2f366c235dc7b7280411622c8462f0afaae3c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 18:07:52 +0000
Subject: [PATCH 1105/8313]

---
 doc/walkthrough/adding_a_remote.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn
index be8e8e7fe5..013ac51100 100644
--- a/doc/walkthrough/adding_a_remote.mdwn
+++ b/doc/walkthrough/adding_a_remote.mdwn
@@ -17,3 +17,5 @@ of the new repository created on the USB drive.
 Notice that both repos are set up as remotes of one another. This lets
 either get annexed files from the other. You'll want to do that even
 if you are using git in a more centralized fashion.
+
+Note that you need to pull from the new remote at least once to make your local annex aware of the remote annex.

From bc72c0bab6bb2d17293f7549aa031b4d439ea954 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 18:37:05 +0000
Subject: [PATCH 1106/8313]

---
 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
new file mode 100644
index 0000000000..8399ff9844
--- /dev/null
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -0,0 +1 @@
+I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. The name scheme for an annex should be the same as for git repos themselves.

From 482782ad05d0a8a8ce4375c437e092d40aa574cb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 18:38:30 +0000
Subject: [PATCH 1107/8313]

---
 ...bout_remotes_with_dots_in_their_names.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn

diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
new file mode 100644
index 0000000000..e6dee3e0b7
--- /dev/null
+++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
@@ -0,0 +1,22 @@
+For test.com//test, I get this:
+
+    % git annex copy . --to test.com//test
+    (getting UUID for test...) git-annex: there is no git remote named "test.com//test"
+
+And my .git/config changes from
+
+    [remote "test.com//test"]
+    	url = richih@test.com:/test
+    	fetch = +refs/heads/*:refs/remotes/test.com//test/*
+
+to
+
+    [remote "test.com//test"]
+    	url = richih@test.com:/test
+    	fetch = +refs/heads/*:refs/remotes/test.com//test/*
+    	annex-uuid = xyz
+    [remote "test"]
+    	annex-uuid = xyz
+
+
+Unless I am misunderstanding something, git annex gets confused about what the name of the remote it supposed to be, truncates at the dot for some operations and uses the full name for others.

From ca48255495e1b8ef4bda5f7f019c482d2a59b431 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 19:24:10 +0000
Subject: [PATCH 1108/8313]

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
new file mode 100644
index 0000000000..06e940a104
--- /dev/null
+++ b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
@@ -0,0 +1 @@
+* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git repos themselves.

From ba7970c644a2583d0cd50242f170770c4a0ff1b7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 19:24:40 +0000
Subject: [PATCH 1109/8313]

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
index 06e940a104..17b3c65b16 100644
--- a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
+++ b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
@@ -1 +1 @@
-* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git repos themselves.
+* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git remotes themselves.

From b7ed29f3c07c7025c203813e4ecc5b6c6908279f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 18 Mar 2011 19:24:56 +0000
Subject: [PATCH 1110/8313]

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
index 17b3c65b16..7862c44e93 100644
--- a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
+++ b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
@@ -1 +1 @@
-* I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. Name schemes for an annex should be the same as for git remotes themselves.
+I can create an annex remote named 'test:/test'. git itself does not allow colons in remote names, though. Name schemes for an annex should be the same as for git remotes themselves.

From efd555848d042450d867761799ebd1b23459ea4e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:13:54 -0400
Subject: [PATCH 1111/8313] remove bad information

You do not need to pull from a remote to make git-annex aware of it.
Pulling only makes git know about remote/master branches, but git-annex
does not care about that. When git-annex encounters a remote in .git/config
that has no recorded annex.uuid, it connects to it to get the uuid
automatically, and will immediately start using it.

A later part of this walkthrough shows how to pull from a remote
in order to get its location tracking info, so that git-annex can retrieve
files from it. That's the right place to document the need to pull from
a remote periodically.
---
 doc/walkthrough/adding_a_remote.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn
index 013ac51100..be8e8e7fe5 100644
--- a/doc/walkthrough/adding_a_remote.mdwn
+++ b/doc/walkthrough/adding_a_remote.mdwn
@@ -17,5 +17,3 @@ of the new repository created on the USB drive.
 Notice that both repos are set up as remotes of one another. This lets
 either get annexed files from the other. You'll want to do that even
 if you are using git in a more centralized fashion.
-
-Note that you need to pull from the new remote at least once to make your local annex aware of the remote annex.

From 0cc05e0c19d7e0a6112167440a9ba48629407625 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:29:14 -0400
Subject: [PATCH 1112/8313] response

---
 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
index 8399ff9844..aa658819b4 100644
--- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -1 +1,6 @@
 I can create an annex remote named 'test:/test'. git itself does not allow colons in names, though. The name scheme for an annex should be the same as for git repos themselves.
+
+> What do you mean by "an annex remote"? git-annex uses the same
+> remotes configuration as does git. If you put invalid
+> stuff in .git/config it might handle it slightly different than 
+> git, I don't know. Examples needed. --[[Joey]] 

From 0663f14cf7cb49189352a9c35ce649f3ad10de8b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:29:42 -0400
Subject: [PATCH 1113/8313] Fix support for remotes with '.' in their names.

---
 GitRepo.hs       | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index a62d765961..34a59a10d4 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -404,7 +404,7 @@ configRemotes repo = mapM construct remotepairs
 		remotepairs = Map.toList $ filterremotes $ config repo
 		filterremotes = Map.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
-		remotename k = split "." k !! 1
+		remotename k = join "." $ reverse $ drop 1 $ reverse $ drop 1 $ split "." k
 		construct (k,v) = do
 			r <- gen v
 			return $ r { remoteName = Just $ remotename k }
diff --git a/debian/changelog b/debian/changelog
index 81257a24be..e1c0576e8b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (0.20110317) UNRELEASED; urgency=low
 
   * Fix dropping of files using the URL backend.
+  * Fix support for remotes with '.' in their names.
 
  -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
 

From 69841b67c9dfc58d33735b6e15d81099b5f502e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:32:31 -0400
Subject: [PATCH 1114/8313] done

---
 ...ets_confused_about_remotes_with_dots_in_their_names.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
index e6dee3e0b7..6f35cef8d1 100644
--- a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
+++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
@@ -20,3 +20,9 @@ to
 
 
 Unless I am misunderstanding something, git annex gets confused about what the name of the remote it supposed to be, truncates at the dot for some operations and uses the full name for others.
+
+> I've fixed this bug. [[done]]
+> 
+> However, using "/" in a remote name seems likely to me to confuse 
+> git's own remote branch handling. Although I've never tried it.
+> --[[Joey]] 

From 75ad0f22d84603fa609acb5a62f4cc4dcd86ee72 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 16:47:20 -0400
Subject: [PATCH 1115/8313] analysis

---
 doc/bugs/fsck_output.mdwn | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 81fecceb3f..413336b313 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -1,16 +1,28 @@
 When you check several files and the fsck fails, you get confusing output:
 
 
-fsck test1 (checksum...) 
-  Only 1 of 2 trustworthy copies of test1 exist.
-  Back it up with git-annex copy.
-
-failed
-fsck test2 (checksum...) 
-  Only 1 of 2 trustworthy copies of test2 exist.
-  Back it up with git-annex copy.
-
-failed
+O fsck test1 (checksum...) 
+E  Only 1 of 2 trustworthy copies of test1 exist.
+E  Back it up with git-annex copy.
+O
+O failed
+O fsck test2 (checksum...) 
+E  Only 1 of 2 trustworthy copies of test2 exist.
+E  Back it up with git-annex copy.
+O 
+O failed
 

 
 The newline is in the wrong place and confuses the user. It should be printed _after_ "failed".
+
+> This is a consequence of part of the output being printed to stderr, and
+> part to stdout. I've marked the lines above with E and O.
+> 
+> Normally a "failed" is preceeded by a message output to stdout desribing
+> the problem; such a message will not be "\n" terminated, so a newline
+> is always displayed before "failed". In this case, since the message
+> is sent to stderr, it is newline terminated.
+> 
+> Fixing this properly would involve storing state, or rethinking 
+> when git-annex displays newlines (and I rather like its behavior
+> otherwise). --[[Joey]]

From 885215508ce8e07676fd92cd8bbb4ac68bd5c1b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Mar 2011 17:05:03 -0400
Subject: [PATCH 1116/8313] add

---
 doc/bugs/fsck_output.mdwn | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 413336b313..3ded1b409e 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -25,4 +25,12 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > 
 > Fixing this properly would involve storing state, or rethinking 
 > when git-annex displays newlines (and I rather like its behavior
-> otherwise). --[[Joey]]
+> otherwise).
+> 
+> A related problem occurs if an error message is unexpetedly printed.
+> Dummying up an example:
+> 
+> O get test1 (copying from foo...) E git-annex: failed to run ssh
+> failed
+> 
+> --[[Joey]] 

From 3dbff3356d14eadc3463693ae44921b98d7e03f4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:09:22 +0000
Subject: [PATCH 1117/8313]

---
 ...ets_confused_about_remotes_with_dots_in_their_names.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
index 6f35cef8d1..d35282e750 100644
--- a/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
+++ b/doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn
@@ -26,3 +26,9 @@ Unless I am misunderstanding something, git annex gets confused about what the n
 > However, using "/" in a remote name seems likely to me to confuse 
 > git's own remote branch handling. Although I've never tried it.
 > --[[Joey]] 
+
+>> From what I can see, git handles / just fine, but would get upset about : which is why it's not allowed in a remote's name.
+>> My naming scheme is host//path/to/annex. It sorts nicely and gives all important information left to right with the most specific parts at the beginning and end.
+>> If you have any other ideas or scheme, I am all ears :)
+>> Either way, thanks for fixing this so quickly.
+>> -- RichiH

From debc7b27445da764561abd83044e7865024b39f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:10:26 +0000
Subject: [PATCH 1118/8313] removed

---
 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 doc/bugs/names_for_remotes_are_not_handled_properly.mdwn

diff --git a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn b/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
deleted file mode 100644
index 7862c44e93..0000000000
--- a/doc/bugs/names_for_remotes_are_not_handled_properly.mdwn
+++ /dev/null
@@ -1 +0,0 @@
-I can create an annex remote named 'test:/test'. git itself does not allow colons in remote names, though. Name schemes for an annex should be the same as for git remotes themselves.

From 9e8d5373365281f67b9539e0443572e92bde5b60 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:16:09 +0000
Subject: [PATCH 1119/8313]

---
 ...me_scheme_does_not_follow_git__39__s_rules.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
index aa658819b4..5971888891 100644
--- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -4,3 +4,17 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon
 > remotes configuration as does git. If you put invalid
 > stuff in .git/config it might handle it slightly different than 
 > git, I don't know. Examples needed. --[[Joey]] 
+
+>> What I mean is this:
+
+    % cd 1
+    % git init
+    % git annex init "my:colon"
+    % [...]
+    % cd ../2
+    % git init
+    % git annex init "second"
+    % git remote add "my:colon" ../1
+    fatal: 'my:colon' is not a valid remote name
+
+>> -- RichiH

From 5e76dab6febf727a37421ab124d6f380114f5deb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 19 Mar 2011 01:18:49 +0000
Subject: [PATCH 1120/8313] Added a comment

---
 .../comment_1_0a59355bd33a796aec97173607e6adc9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment

diff --git a/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
new file mode 100644
index 0000000000..4b0b9c0fd2
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-19T01:18:49Z"
+ content="""
+After doing the above with two required copy per file, `git annex fsck` complained that I had only one copy per file even though I had created my clone, already. Once I `git pull`ed from the second repo, not getting any changes for obvious reasons, `git annex fsck` was happy. So I am not sure how my addition was incorrect. -- RichiH
+"""]]

From a3ee0da1e0706834d3e8da8d2abf1f134e26f625 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 11:31:23 +0000
Subject: [PATCH 1121/8313]

---
 ...askell-platform_statically_links_things.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn

diff --git a/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn
new file mode 100644
index 0000000000..f656427589
--- /dev/null
+++ b/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn
@@ -0,0 +1,17 @@
+This isn't really a bug of git-annex, but a problem with haskell-platform/ghc6.12.x so this post might need to be moved to a better place (maybe tips).
+
+OSX's haskell-platform doesn't have the dynamic libraries available, as far as I know it just isn't supported therefore git-annex will always be statically built on OSX, so wrappers like  or [[!google dsocks]] for preloading connect() calls won't work. 
+
+
+jtang@x00:~/annex $ tsocks git annex get .
+dyld: could not load inserted library: /opt/local/lib/libtsocks.dylib
+
+error: git-annex died of signal 5
+

+
+The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like "http://bent.latency.net/bent/git/goto-san-connect-1.85/src/connect.html".
+
+
+host remotemyhost
+        ProxyCommand connect -S proxy.mydomain:1080 -R local %h %p
+


From 4395429ab1d8165240c6c76dc90e22fd0c023532 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 11:39:28 +0000
Subject: [PATCH 1122/8313]

---
 ...default_sshd_behaviour_has_limited_paths_set.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn

diff --git a/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
new file mode 100644
index 0000000000..99b7092c7f
--- /dev/null
+++ b/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
@@ -0,0 +1,12 @@
+This is a tip for users who wish to use remotes which are based on OSX systems and have used macports to install some of the required utilities for git-annex to work.
+
+The default behaviour of OSX's sshd is to have a "highly restricted" restricted environment. The defaults that it allows is
+
+    jtang@x00:~ $ ssh x00 echo \$PATH
+    /usr/bin:/bin:/usr/sbin:/sbin
+
+One solution is to enable *PermitUserEnvironment yes* in `/etc/sshd_config` and then in your own `~/.ssh/environment` file you could add something like (the below is an example)
+
+    PATH=/Users/jtang/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/X11/bin:/usr/X11/bin:/Users/jtang/.cabal/bin:/opt/local/libexec/gnubin
+
+If the above is not done, cloning from the OSX host will fail if git is not installed isn't /usr/bin (which it probably won't be).

From 065a3e4d2bfa2340b57830baf4e3899a8bc25217 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 11:31:08 -0400
Subject: [PATCH 1123/8313] move OSX tips to forum and link to from install/OSX

---
 ...__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn | 0
 .../OSX__39__s_haskell-platform_statically_links_things.mdwn | 0
 doc/install/OSX.mdwn                                         | 5 +++++
 3 files changed, 5 insertions(+)
 rename doc/{bugs => forum}/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn (100%)
 rename doc/{bugs => forum}/OSX__39__s_haskell-platform_statically_links_things.mdwn (100%)

diff --git a/doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
similarity index 100%
rename from doc/bugs/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
rename to doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
diff --git a/doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
similarity index 100%
rename from doc/bugs/OSX__39__s_haskell-platform_statically_links_things.mdwn
rename to doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 3c29fc101c..c8c381486b 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -18,3 +18,8 @@ sudo make install
 

 
 Originally posted by Jon at  --[[Joey]]
+
+See also:
+
+* [[forum/OSX__39__s_haskell-platform_statically_links_things]]
+* [[forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set]]

From 6783c31ba36cbebfb6c6f256d0562e7a0d0195da Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 11:33:04 -0400
Subject: [PATCH 1124/8313] notabug

---
 doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
index 5971888891..f90eb5ae3c 100644
--- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
+++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn
@@ -18,3 +18,9 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon
     fatal: 'my:colon' is not a valid remote name
 
 >> -- RichiH
+
+>>> I see.. Git annex init does not specifiy a remote's name, it specifies
+>>> an arbitrary human-readable description of the repository, which will
+>>> be displayed when there is no configured remote corresponding to the
+>>> repository. So this is not a bug unless some documentation of that is
+>>> unclear. --[[Joey]] 

From 9d96ef4ddada7c015264db72fb3fceca0ad87c3d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 19 Mar 2011 15:35:38 +0000
Subject: [PATCH 1125/8313] Added a comment

---
 .../comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment

diff --git a/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
new file mode 100644
index 0000000000..015417a4f7
--- /dev/null
+++ b/doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:35:38Z"
+ content="""
+Yes, you have to pull down location tracking information in order for fsck to be satisfied in that situation. But since this is a walkthrough, and neither fsck or numcopies settings are mentioned until later, it's ok for this pull to be described a few steps along in [[getting file content]].
+
+"""]]

From f3f57f0c5ea50ab0da7c102d0041f1f056a6408c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 19 Mar 2011 15:37:22 +0000
Subject: [PATCH 1126/8313] Added a comment

---
 .../comment_2_af4f8b52526d8bea2904c95406fd2796._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment

diff --git a/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment
new file mode 100644
index 0000000000..ca599b2857
--- /dev/null
+++ b/doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-19T15:37:22Z"
+ content="""
+Now it's fully supported, so long as you put a bare git repo on your key.
+"""]]

From d6e1d0680421152541e74d6372bfcc2b46bebfed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 12:49:37 -0400
Subject: [PATCH 1127/8313] alt approach

---
 doc/todo/S3.mdwn | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 3d18527d48..56023e71e1 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -50,3 +50,24 @@ be hard to get right.
 Less blue-sky, if the S3 capability were added directly to Backend.File,
 and bucket name was configured by annex.s3.bucket, then any existing
 annexed file could be upgraded to also store on S3.
+
+## alternate approach
+
+The above assumes S3 should be a separate backend somehow. What if,
+instead a S3 bucket is treated as a separate **remote**.
+
+* Could "git annex add" while offline, and "git annex push --to S3" when
+  online.
+* No need to choose whether a file goes to S3 at add time; no need to
+  migrate to move files there.
+* numcopies counting Just Works
+* Could have multiple S3 buckets as desired.
+
+The bucket name could 1:1 map with its annex.uuid, so not much
+configuration would be needed when cloning a repo to get it using S3 --
+just configure the S3 access token(s) to use for various UUIDs.
+
+Implementing this might not be as conceptually nice as making S3 a separate
+backend. It would need some changes to the remotes code, perhaps lifting
+some of it into backend-specific hooks. Then the S3 backend could be
+implicitly stacked in front of a backend like WORM.

From c0eabcc19165d5ba7c03f53002fb1b186f9f43c5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 17:33:21 +0000
Subject: [PATCH 1128/8313] link to page

---
 .../OSX__39__s_haskell-platform_statically_links_things.mdwn    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
index f656427589..537c85d019 100644
--- a/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
+++ b/doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn
@@ -9,7 +9,7 @@ dyld: could not load inserted library: /opt/local/lib/libtsocks.dylib
 error: git-annex died of signal 5
 

 
-The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like "http://bent.latency.net/bent/git/goto-san-connect-1.85/src/connect.html".
+The side effect of this means that users who are behind restrictive firewalls that allow only ssh via a socks proxy, they will need to configure ssh to use something like .
 
 
 host remotemyhost

From 942480c47f69e13cf053b8f50c98c2ce4eaa256e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 19 Mar 2011 17:33:32 +0000
Subject: [PATCH 1129/8313] fix typo

---
 ...OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
index 99b7092c7f..5e417b14c9 100644
--- a/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
+++ b/doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn
@@ -9,4 +9,4 @@ One solution is to enable *PermitUserEnvironment yes* in `/etc/sshd_config` and
 
     PATH=/Users/jtang/bin:/opt/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/opt/X11/bin:/usr/X11/bin:/Users/jtang/.cabal/bin:/opt/local/libexec/gnubin
 
-If the above is not done, cloning from the OSX host will fail if git is not installed isn't /usr/bin (which it probably won't be).
+If the above is not done, cloning from the OSX host will fail if git is not installed in /usr/bin (which it probably won't be).

From 828a84ba3341d4b7a84292d8b9002a8095dd2382 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 14:33:24 -0400
Subject: [PATCH 1130/8313] Add version command to show git-annex version as
 well as repository version information.

---
 Command/Version.hs | 34 ++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  2 ++
 Upgrade.hs         |  5 ++++-
 Version.hs         | 15 ++++++++++-----
 configure.hs       | 13 ++++++++++++-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  4 ++++
 7 files changed, 68 insertions(+), 7 deletions(-)
 create mode 100644 Command/Version.hs

diff --git a/Command/Version.hs b/Command/Version.hs
new file mode 100644
index 0000000000..480f2166b1
--- /dev/null
+++ b/Command/Version.hs
@@ -0,0 +1,34 @@
+{- git-annex command
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Version where
+
+import Control.Monad.State (liftIO)
+import Data.String.Utils
+
+import Command
+import qualified SysConfig
+import Version
+import Upgrade
+
+command :: [Command]
+command = [Command "version" paramNothing seek "show versions"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStartNothing
+start = do
+	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
+	v <- getVersion
+	liftIO $ putStrLn $ "local repository version: " ++ v
+	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
+	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
+	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
+	return Nothing
+	where
+		vs l = join " " l
diff --git a/GitAnnex.hs b/GitAnnex.hs
index b9c22bdfb4..adf07e5b3e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -42,6 +42,7 @@ import qualified Command.Untrust
 import qualified Command.Semitrust
 import qualified Command.Map
 import qualified Command.Upgrade
+import qualified Command.Version
 
 cmds :: [Command]
 cmds = concat
@@ -72,6 +73,7 @@ cmds = concat
 	, Command.Migrate.command
 	, Command.Map.command
 	, Command.Upgrade.command
+	, Command.Version.command
 	]
 
 options :: [Option]
diff --git a/Upgrade.hs b/Upgrade.hs
index 76dd156f83..d201cc73ed 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -12,6 +12,9 @@ import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
 
+upgradableVersions :: [Version]
+upgradableVersions = ["0", "1"]
+
 {- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
 upgrade = do
@@ -19,5 +22,5 @@ upgrade = do
 	case version of
 		"0" -> Upgrade.V0.upgrade
 		"1" -> Upgrade.V1.upgrade
-		v | v == currentVersion -> return True
+		v | v `elem` supportedVersions -> return True
 		_ -> error "this version of git-annex is too old for this git repository!"
diff --git a/Version.hs b/Version.hs
index 5f414e93b8..d4a58d77a2 100644
--- a/Version.hs
+++ b/Version.hs
@@ -15,13 +15,18 @@ import qualified Annex
 import qualified GitRepo as Git
 import Locations
 
-currentVersion :: String
-currentVersion = "2"
+type Version = String
+
+defaultVersion :: Version
+defaultVersion = "2"
+
+supportedVersions :: [Version]
+supportedVersions = [defaultVersion]
 
 versionField :: String
 versionField = "annex.version"
 
-getVersion :: Annex String
+getVersion :: Annex Version
 getVersion = do
 	g <- Annex.gitRepo
 	let v = Git.configGet g versionField ""
@@ -42,7 +47,7 @@ getVersion = do
 				(True, True) -> return "1"
 				_ -> do
 					setVersion
-					return currentVersion
+					return defaultVersion
 
 setVersion :: Annex ()
-setVersion = Annex.setConfig versionField currentVersion
+setVersion = Annex.setConfig versionField defaultVersion
diff --git a/configure.hs b/configure.hs
index f5c2956488..f8cd577e99 100644
--- a/configure.hs
+++ b/configure.hs
@@ -7,7 +7,8 @@ import TestConfig
 
 tests :: [TestCase]
 tests = [
-	  testCp "cp_a" "-a"
+	  TestCase "version" $ getVersion
+	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
 	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"]
@@ -49,6 +50,16 @@ unicodeFilePath = do
 	let file = head $ filter (isInfixOf "unicode-test") fs
 	return $ Config "unicodefilepath" (BoolConfig $ isInfixOf "ü" file)
 
+{- Pulls package version out of the changelog. -}
+getVersion :: Test
+getVersion = do
+	changelog <- readFile "debian/changelog"
+	let verline = head $ lines changelog
+	let version = middle (words verline !! 1)
+	return $ Config "packageversion" (StringConfig version)
+	where
+		middle s = drop 1 $ take (length s - 1) s
+
 setup :: IO ()
 setup = do
 	createDirectoryIfMissing True tmpDir
diff --git a/debian/changelog b/debian/changelog
index e1c0576e8b..751fcaff96 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (0.20110317) UNRELEASED; urgency=low
 
   * Fix dropping of files using the URL backend.
   * Fix support for remotes with '.' in their names.
+  * Add version command to show git-annex version as well as repository
+    version information.
 
  -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ee40190682..1e4af022f1 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -267,6 +267,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
   whenever a newer git annex encounters an old repository; this command
   allows explcitly starting an upgrade.
 
+* version
+
+  Shows the version of git-annex, as well as repository version information.
+
 # OPTIONS
 
 * --force

From 5022a69e45a073046a2b14b6a4e798910c920ee9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 14:39:03 -0400
Subject: [PATCH 1131/8313] implemented

---
 doc/bugs/No_version_information_from_cli.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/No_version_information_from_cli.mdwn b/doc/bugs/No_version_information_from_cli.mdwn
index c432cf99b7..a0d30db414 100644
--- a/doc/bugs/No_version_information_from_cli.mdwn
+++ b/doc/bugs/No_version_information_from_cli.mdwn
@@ -14,3 +14,5 @@ Sample:
     % 
 
 The above might look like overkill, but it's in a form that will, most likely, never need to be extended.
+
+> Great idea, [[done]] --[[Joey]] 

From 43b3675d4298ffe8cfaefed0d70f0ef1fce25e37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 14:46:44 -0400
Subject: [PATCH 1132/8313] don't crash on upgrade if .git-annex DNE

---
 Upgrade/V1.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 270de5f74b..43f279ad08 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -115,8 +115,12 @@ moveLocationLogs = do
 			oldlocationlogs = do
 				g <- Annex.gitRepo
 				let dir = gitStateDir g
-				contents <- liftIO $ getDirectoryContents dir
-				return $ catMaybes $ map oldlog2key contents
+				exists <- liftIO $ doesDirectoryExist dir
+				if exists
+					then do
+						contents <- liftIO $ getDirectoryContents dir
+						return $ catMaybes $ map oldlog2key contents
+					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
 				let dest = logFile g k

From 6a2a17658c1d508cec022132768e9bc1957098d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:33:39 -0400
Subject: [PATCH 1133/8313] No longer auto-upgrade to repository format 2, to
 avoid accidental upgrades, etc. Use git-annex upgrade when you're ready to
 run this version.

---
 CmdLine.hs                                       |  5 +++--
 Command/Version.hs                               |  1 -
 Upgrade.hs                                       |  6 +-----
 Version.hs                                       | 16 ++++++++++++++++
 debian/NEWS                                      |  5 +----
 debian/changelog                                 |  3 +++
 ...t_1_c25900b9d2d62cc0b8c77150bcfebadf._comment |  3 +++
 doc/upgrades.mdwn                                | 13 +++++--------
 8 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0698f2f5ea..6b4fd0f368 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,7 @@ import qualified GitQueue
 import Types
 import Command
 import BackendList
-import Upgrade
+import Version
 import Options
 import Messages
 import UUID
@@ -33,7 +33,7 @@ dispatch gitrepo args cmds options header = do
 	setupConsole
 	state <- Annex.new gitrepo allBackends
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
-	tryRun state' $ [startup, upgrade] ++ actions ++ [shutdown]
+	tryRun state' $ [startup] ++ actions ++ [shutdown]
 
 {- Parses command line, stores configure flags, and returns a 
  - list of actions to be run in the Annex monad. -}
@@ -93,6 +93,7 @@ tryRun' _ errnum [] = do
 startup :: Annex Bool
 startup = do
 	prepUUID
+	checkVersion
 	return True
 
 {- Cleanup actions. -}
diff --git a/Command/Version.hs b/Command/Version.hs
index 480f2166b1..ac8fdd48c9 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,6 @@ import Data.String.Utils
 import Command
 import qualified SysConfig
 import Version
-import Upgrade
 
 command :: [Command]
 command = [Command "version" paramNothing seek "show versions"]
diff --git a/Upgrade.hs b/Upgrade.hs
index d201cc73ed..08481755f3 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -12,9 +12,6 @@ import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
 
-upgradableVersions :: [Version]
-upgradableVersions = ["0", "1"]
-
 {- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
 upgrade = do
@@ -22,5 +19,4 @@ upgrade = do
 	case version of
 		"0" -> Upgrade.V0.upgrade
 		"1" -> Upgrade.V1.upgrade
-		v | v `elem` supportedVersions -> return True
-		_ -> error "this version of git-annex is too old for this git repository!"
+		_ -> return True
diff --git a/Version.hs b/Version.hs
index d4a58d77a2..d061a2eab2 100644
--- a/Version.hs
+++ b/Version.hs
@@ -8,6 +8,7 @@
 module Version where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (unless)
 import System.Directory
 
 import Types
@@ -23,6 +24,9 @@ defaultVersion = "2"
 supportedVersions :: [Version]
 supportedVersions = [defaultVersion]
 
+upgradableVersions :: [Version]
+upgradableVersions = ["0", "1"]
+
 versionField :: String
 versionField = "annex.version"
 
@@ -51,3 +55,15 @@ getVersion = do
 
 setVersion :: Annex ()
 setVersion = Annex.setConfig versionField defaultVersion
+
+checkVersion :: Annex ()
+checkVersion = do
+	v <- getVersion
+	unless (v `elem` supportedVersions) $ do
+		error $ "Repository version " ++ v ++ 
+			" is not supported. " ++
+			msg v
+	where
+		msg v
+			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
+			| otherwise = "Upgrade git-annex."
diff --git a/debian/NEWS b/debian/NEWS
index df8518cef9..a8b258bfe4 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -2,10 +2,7 @@ git-annex (0.20110316) experimental; urgency=low
 
   This version reorganises the layout of git-annex's files in your repository.
   There is an upgrade process to convert a repository from the old git-annex
-  to this version. While git-annex will attempt to transparently handle
-  upgrades, you may want to drive the upgrade process by hand.
-
-  See  or
+  to this version. See  or
   /usr/share/doc/git-annex/html/upgrades.html
 
  -- Joey Hess   Wed, 16 Mar 2011 15:49:15 -0400
diff --git a/debian/changelog b/debian/changelog
index 751fcaff96..032dedfb9e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,9 @@ git-annex (0.20110317) UNRELEASED; urgency=low
   * Fix support for remotes with '.' in their names.
   * Add version command to show git-annex version as well as repository
     version information.
+  * No longer auto-upgrade to repository format 2, to avoid accidental
+    upgrades, etc. Use git-annex upgrade when you're ready to run this
+    version.
 
  -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
 
diff --git a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
index 6a34becd37..8420d7bb3a 100644
--- a/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
+++ b/doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment
@@ -7,4 +7,7 @@
 These are good examples; I think you've convinced me at least for upgrades going forward after v2. I'm not sure we have enough users and outdated git-annex installations to worry about it for v1.
 
 (Hoping such upgrades are rare anyway.. Part of the point of changes made in v2 was to allow lots of changes to be made later w/o needing a v3.)
+
+Update: Upgrades from v1 to v2 will no longer be handled automatically
+now.
 """]]
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 1bd97c46e2..5c2187c891 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -9,15 +9,14 @@ git-annex, and might want to use them with a newer git-annex.
 
 ## Upgrade process
 
-git-annex will automatically notice if it is run in a repository that
-needs an upgrade, and perform the upgrade before running whatever it
-was asked to do. Or you can use the "git annex upgrade" command to
-explicitly do an upgrade. The upgrade can tend to take a while,
-if you have a lot of files.
+git-annex will notice if it is run in a repository that
+needs an upgrade, and refuse to do anything. To upgrade,
+use the "git annex upgrade" command. The upgrade can tend
+to take a while, if you have a lot of files.
 
 Each clone of a repository should be individually upgraded.
 Until a repository's remotes have been upgraded, git-annex
-may refuse to communicate with them.
+will refuse to communicate with them.
 
 Generally, start by upgrading one repository, and then you can commit
 the changes git-annex staged during upgrade, and push them out to other
@@ -54,8 +53,6 @@ Symlinks changed.
 Also, hashing was added to location log files in .git-annex/.
 And .gitattributes needed to have another line added to it.
 
-Handled transparently.
-
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
 Involved a reorganisation of the layout of .git/annex/. Symlinks changed.

From 91d1d935dd034850c152455bfab8f9df227a682f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:56:36 -0400
Subject: [PATCH 1134/8313] git-annex setup

---
 .gitattributes | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitattributes b/.gitattributes
index 5d425843f2..b83edcae09 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1,3 @@
 debian/changelog merge=dpkg-mergechangelogs
+.git-annex/*.log merge=union
+.git-annex/*/*/*.log merge=union

From 54c39d09ba0185cea398876e609e59d3059b18e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:58:10 -0400
Subject: [PATCH 1135/8313] need to explicitly run upgrade now

---
 Command/Upgrade.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 3c9fa3eebf..94398c70a1 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -8,15 +8,18 @@
 module Command.Upgrade where
 
 import Command
+import Upgrade
+import Version
 
 command :: [Command]
-command = [Command "upgrade" paramNothing seek "upgrade repository layout"]
+command = [standaloneCommand "upgrade" paramNothing seek
+	"upgrade repository layout"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
 
 start :: CommandStartNothing
 start = do
-	-- The actual upgrading is handled by just running any command,
-	-- so nothing extra needs to be done.
-	return $ Just $ return $ Just $ return True
+	r <- upgrade
+	checkVersion
+	return $ Just $ return $ Just $ return r

From 140a351fc535235ae5714122f0c98174cbdb19ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Mar 2011 18:58:49 -0400
Subject: [PATCH 1136/8313] avoid version check before running version and
 upgrade commands

There are two types of commands; those that access the repository and those
that don't. Sorted.
---
 CmdLine.hs            |  5 +++--
 Command.hs            | 13 ++++++++++---
 Command/Add.hs        |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Copy.hs       |  2 +-
 Command/Describe.hs   |  2 +-
 Command/Drop.hs       |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/Fsck.hs       |  2 +-
 Command/Get.hs        |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  2 +-
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Move.hs       |  2 +-
 Command/PreCommit.hs  |  2 +-
 Command/RecvKey.hs    |  2 +-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/SetKey.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  2 +-
 Command/Uninit.hs     |  2 +-
 Command/Unlock.hs     |  4 ++--
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  3 ++-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 33 files changed, 46 insertions(+), 37 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 6b4fd0f368..de03d96ed4 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -45,7 +45,9 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			prepCmd command (drop 1 params)
+			when (cmdusesrepo command) $
+				checkVersion
+			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
 		getopt = case getOpt Permute options argv of
@@ -93,7 +95,6 @@ tryRun' _ errnum [] = do
 startup :: Annex Bool
 startup = do
 	prepUUID
-	checkVersion
 	return True
 
 {- Cleanup actions. -}
diff --git a/Command.hs b/Command.hs
index 41ad884a93..1449d7eedf 100644
--- a/Command.hs
+++ b/Command.hs
@@ -61,13 +61,20 @@ data Command = Command {
 	cmdname :: String,
 	cmdparams :: String,
 	cmdseek :: [CommandSeek],
-	cmddesc :: String
+	cmddesc :: String,
+	cmdusesrepo :: Bool
 }
 
+repoCommand :: String -> String -> [CommandSeek] -> String -> Command
+repoCommand n p s d = Command n p s d True
+
+standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
+standaloneCommand n p s d = Command n p s d False
+
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
-prepCmd :: Command -> [String] -> Annex [Annex Bool]
-prepCmd Command { cmdseek = seek } params = do
+prepCommand :: Command -> [String] -> Annex [Annex Bool]
+prepCommand Command { cmdseek = seek } params = do
 	lists <- mapM (\s -> s params) seek
 	return $ map doCommand $ concat lists
 
diff --git a/Command/Add.hs b/Command/Add.hs
index a577203bf4..f6ccf0fb88 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -21,7 +21,7 @@ import Utility
 import Touch
 
 command :: [Command]
-command = [Command "add" paramPath seek "add files to annex"]
+command = [repoCommand "add" paramPath seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index b91c2a9160..476d73cfbb 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -14,7 +14,7 @@ import Command
 import UUID
 
 command :: [Command]
-command = [Command "configlist" paramNothing seek
+command = [standaloneCommand "configlist" paramNothing seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 93342e11bb..46d49dd058 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -11,7 +11,7 @@ import Command
 import qualified Command.Move
 
 command :: [Command]
-command = [Command "copy" paramPath seek
+command = [repoCommand "copy" paramPath seek
 	"copy content of files to/from another repository"]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 643ca04718..32aef4f245 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -16,7 +16,7 @@ import Messages
 import qualified Command.Init
 
 command :: [Command]
-command = [Command "describe" (paramPair paramRemote paramDesc) seek
+command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index fdc55969f0..52b724e62b 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -18,7 +18,7 @@ import Messages
 import Utility
 
 command :: [Command]
-command = [Command "drop" paramPath seek
+command = [repoCommand "drop" paramPath seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index b3cc60961c..4c6f1ab2e1 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -15,7 +15,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "dropkey" (paramRepeating paramKey) seek
+command = [repoCommand "dropkey" (paramRepeating paramKey) seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 8ed61ba65b..c6a28663ec 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -23,7 +23,7 @@ import Backend
 import Key
 
 command :: [Command]
-command = [Command "dropunused" (paramRepeating paramNumber) seek
+command = [repoCommand "dropunused" (paramRepeating paramNumber) seek
 	"drop unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 3ed15c1537..6a6ae29787 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -14,7 +14,7 @@ import Command
 import Content
 
 command :: [Command]
-command = [Command "find" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
 	"lists available files"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 0047548715..513e07a310 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -18,7 +18,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "fix" paramPath seek
+command = [repoCommand "fix" paramPath seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 176d2cd54d..8c1a1028fe 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -21,7 +21,7 @@ import Messages
 import Key
 
 command :: [Command]
-command = [Command "fromkey" paramPath seek
+command = [repoCommand "fromkey" paramPath seek
 	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 76d0e38b4b..216c87493b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,7 +20,7 @@ import Content
 import LocationLog
 
 command :: [Command]
-command = [Command "fsck" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
 	"check for problems"]
 
 seek :: [CommandSeek]
diff --git a/Command/Get.hs b/Command/Get.hs
index 2aa3c0c150..0463dccb05 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -14,7 +14,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "get" paramPath seek
+command = [repoCommand "get" paramPath seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index fa81fc9a4c..a7e2ecff60 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -14,7 +14,7 @@ import Command
 import Content
 
 command :: [Command]
-command = [Command "inannex" (paramRepeating paramKey) seek
+command = [repoCommand "inannex" (paramRepeating paramKey) seek
 		"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Init.hs b/Command/Init.hs
index d9ea394a33..cca2e8faef 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -23,7 +23,7 @@ import Types
 import Utility
 	
 command :: [Command]
-command = [Command "init" paramDesc seek
+command = [repoCommand "init" paramDesc seek
 		"initialize git-annex with repository description"]
 
 seek :: [CommandSeek]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index a3a39a9078..cdbc560194 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -17,7 +17,7 @@ import qualified GitRepo as Git
 import Utility
 	
 command :: [Command]
-command = [Command "lock" paramPath seek "undo unlock command"]
+command = [repoCommand "lock" paramPath seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 6c3e0b3df5..4ae947b15e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -29,7 +29,7 @@ import qualified Dot
 data Link = Link Git.Repo Git.Repo
 
 command :: [Command]
-command = [Command "map" paramNothing seek "generate map of repositories"]
+command = [repoCommand "map" paramNothing seek "generate map of repositories"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index c0e80c5b47..584f6e34e1 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -21,7 +21,7 @@ import Messages
 import qualified Command.Add
 
 command :: [Command]
-command = [Command "migrate" paramPath seek "switch data to different backend"]
+command = [repoCommand "migrate" paramPath seek "switch data to different backend"]
 
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
diff --git a/Command/Move.hs b/Command/Move.hs
index 2d6c973fe0..8056e95dbe 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -22,7 +22,7 @@ import Messages
 import Utility
 
 command :: [Command]
-command = [Command "move" paramPath seek
+command = [repoCommand "move" paramPath seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 1465ebc615..727a637285 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -17,7 +17,7 @@ import qualified Command.Fix
 import Utility
 
 command :: [Command]
-command = [Command "pre-commit" paramPath seek "run by git pre-commit hook"]
+command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index c7c37d1e31..126608f614 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -17,7 +17,7 @@ import Content
 import RsyncFile
 
 command :: [Command]
-command = [Command "recvkey" paramKey seek
+command = [repoCommand "recvkey" paramKey seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 13c6847e17..351336b899 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -15,7 +15,7 @@ import Trust
 import Messages
 
 command :: [Command]
-command = [Command "semitrust" (paramRepeating paramRemote) seek
+command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 56974bda96..871a530af7 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -18,7 +18,7 @@ import Content
 import RsyncFile
 
 command :: [Command]
-command = [Command "sendkey" paramKey seek
+command = [repoCommand "sendkey" paramKey seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index fdda1c3bee..af46fe06e4 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -16,7 +16,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [Command "setkey" (paramRepeating paramKey) seek
+command = [repoCommand "setkey" (paramRepeating paramKey) seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index ea661da2a6..f7dba56485 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -15,7 +15,7 @@ import UUID
 import Messages
 
 command :: [Command]
-command = [Command "trust" (paramRepeating paramRemote) seek
+command = [repoCommand "trust" (paramRepeating paramRemote) seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 42dc1fb0ab..b0ce31ceed 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -22,7 +22,7 @@ import qualified GitRepo as Git
 import Messages
 
 command :: [Command]
-command = [Command "unannex" paramPath seek "undo accidential add command"]
+command = [repoCommand "unannex" paramPath seek "undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index e8ac1bbd5e..ee0cbde6b3 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -21,7 +21,7 @@ import qualified Command.Unannex
 import qualified Command.Init
 
 command :: [Command]
-command = [Command "uninit" paramPath seek 
+command = [repoCommand "uninit" paramPath seek 
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index bd1021cc3c..ac7b22ac72 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -22,8 +22,8 @@ import CopyFile
 
 command :: [Command]
 command =
-	[ Command "unlock" paramPath seek "unlock files for modification"
-	, Command "edit" paramPath seek "same as unlock"
+	[ repoCommand "unlock" paramPath seek "unlock files for modification"
+	, repoCommand "edit" paramPath seek "same as unlock"
 	]
 
 seek :: [CommandSeek]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index fdf9a83dec..9c11efe465 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -15,7 +15,7 @@ import Trust
 import Messages
 
 command :: [Command]
-command = [Command "untrust" (paramRepeating paramRemote) seek
+command = [repoCommand "untrust" (paramRepeating paramRemote) seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 52e483d870..a1c4ee03c9 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -25,7 +25,8 @@ import qualified GitRepo as Git
 import qualified Backend
 
 command :: [Command]
-command = [Command "unused" paramNothing seek "look for unused file content"]
+command = [repoCommand "unused" paramNothing seek
+	"look for unused file content"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Version.hs b/Command/Version.hs
index ac8fdd48c9..2b294c80be 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -15,7 +15,7 @@ import qualified SysConfig
 import Version
 
 command :: [Command]
-command = [Command "version" paramNothing seek "show versions"]
+command = [standaloneCommand "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 5b0bcbbd26..599df44676 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -17,7 +17,7 @@ import UUID
 import Types
 
 command :: [Command]
-command = [Command "whereis" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]

From 6634b6a6b84a924f6f6059b5bea61f449d056eee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 14:04:39 -0400
Subject: [PATCH 1137/8313] imcomplete attempt at supporting lutimes(3) for BSD
 compat

---
 Command/Add.hs |  2 +-
 Touch.hsc      | 84 ++++++++++++++++++++++++++++++++++----------------
 2 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index f6ccf0fb88..da98bffa4f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -58,7 +58,7 @@ cleanup file key = do
 	-- touch the symlink to have the same mtime as the file it points to
 	s <- liftIO $ getFileStatus file
 	let mtime = modificationTime s
-	liftIO $ touch file (TimeSpec mtime 0) False
+	liftIO $ touch file (TimeSpec mtime) False
 
 	Annex.queue "add" [Param "--"] file
 	return True
diff --git a/Touch.hsc b/Touch.hsc
index 456175182e..8a3e31b571 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -9,8 +9,6 @@
 
 module Touch (
 	TimeSpec(..),
-	nowTime,
-	omitTime,
 	touchBoth,
 	touch
 ) where
@@ -18,24 +16,24 @@ module Touch (
 import Foreign
 import Foreign.C
 
+data TimeSpec = TimeSpec CTime
+
+{- Changes the access and modification times of an existing file.
+   Can follow symlinks, or not. Throws IO error on failure. -}
+touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()
+
+touch :: FilePath -> TimeSpec -> Bool -> IO ()
+touch file mtime follow = touchBoth file mtime mtime follow
+
 #include 
 #include 
 #include 
+#include 
 
-#ifndef _POSIX_C_SOURCE
-#define _POSIX_C_SOURCE >= 200809L
+#ifndef _BSD_SOURCE
+#define _BSD_SOURCE
 #endif
 
-data TimeSpec = TimeSpec CTime CLong
-
-touch :: FilePath -> TimeSpec -> Bool -> IO ()
-touch file mtime follow = touchBoth file omitTime mtime follow
-
-touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()
-
-omitTime :: TimeSpec
-nowTime :: TimeSpec
-
 #if (defined UTIME_OMIT && defined UTIME_NOW && defined AT_FDCWD && defined AT_SYMLINK_NOFOLLOW)
 
 at_fdcwd :: CInt
@@ -44,9 +42,6 @@ at_fdcwd = #const AT_FDCWD
 at_symlink_nofollow :: CInt
 at_symlink_nofollow = #const AT_SYMLINK_NOFOLLOW
 
-omitTime = TimeSpec 0 #const UTIME_OMIT
-nowTime = TimeSpec 0 #const UTIME_NOW
-
 instance Storable TimeSpec where
 	-- use the larger alignment of the two types in the struct
 	alignment _ = max sec_alignment nsec_alignment
@@ -56,19 +51,16 @@ instance Storable TimeSpec where
 	sizeOf _ = #{size struct timespec}
 	peek ptr = do
 		sec <- #{peek struct timespec, tv_sec} ptr
-		nsec <- #{peek struct timespec, tv_nsec} ptr
-		return $ TimeSpec sec nsec
-	poke ptr (TimeSpec sec nsec) = do
+		return $ TimeSpec sec
+	poke ptr (TimeSpec sec) = do
 		#{poke struct timespec, tv_sec} ptr sec
-		#{poke struct timespec, tv_nsec} ptr nsec
+		#{poke struct timespec, tv_nsec} ptr (0 :: CLong)
 
 {- While its interface is beastly, utimensat is in recent
-   POSIX standards, unlike futimes. -}
+   POSIX standards, unlike lutimes. -}
 foreign import ccall "utimensat" 
 	c_utimensat :: CInt -> CString -> Ptr TimeSpec -> CInt -> IO CInt
 
-{- Changes the access and/or modification times of an existing file.
-   Can follow symlinks, or not. Throws IO error on failure. -}
 touchBoth file atime mtime follow = 
 	allocaArray 2 $ \ptr ->
 	withCString file $ \f -> do
@@ -83,8 +75,46 @@ touchBoth file atime mtime follow =
 			else at_symlink_nofollow 
 
 #else
-#warning "utimensat not available; building without symlink timestamp preservation support"
-omitTime = TimeSpec 0 (-1)
-nowTime = TimeSpec 0 (-2)
+#if 0
+{- Using lutimes is needed for BSD.
+ - 
+ - TODO: test if lutimes is available. May have to do it in configure.
+ - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation
+ - dependent whether these are the same; need to find a cast that works.
+ - (The cast below fails.. without the cast it works on linux i386, but
+ - maybe not elsewhere.)
+ -}
+
+instance Storable TimeSpec where
+	alignment _ = alignment (undefined::CLong)
+	sizeOf _ = #{size struct timeval}
+	peek ptr = do
+		sec <- #{peek struct timeval, tv_sec} ptr
+		return $ TimeSpec sec
+	poke ptr (TimeSpec sec) = do
+		#{poke struct timeval, tv_sec} ptr (sec :: CLong)
+		#{poke struct timeval, tv_usec} ptr (0 :: CLong) 
+
+foreign import ccall "utimes" 
+	c_utimes :: CString -> Ptr TimeSpec -> IO CInt
+foreign import ccall "lutimes" 
+	c_lutimes :: CString -> Ptr TimeSpec -> IO CInt
+
+touchBoth file atime mtime follow = 
+	allocaArray 2 $ \ptr ->
+	withCString file $ \f -> do
+		pokeArray ptr [atime, mtime]
+		r <- syscall f ptr
+		if (r /= 0)
+			then throwErrno "touchBoth"
+			else return ()
+	where
+		syscall = if follow
+			then c_lutimes
+			else c_utimes
+
+#else
+warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
 touchBoth _ _ _ _ = return ()
 #endif
+#endif

From 05ae9cb1bb55c95ff4436d15f66903eb757cac2a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 20 Mar 2011 18:12:59 +0000
Subject: [PATCH 1138/8313] Added a comment

---
 ...comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment
new file mode 100644
index 0000000000..aeb576be37
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-03-20T18:12:59Z"
+ content="""
+I'm leaving this bug open because this feature, however minor is not available on OSX and BSD. 
+
+I have added a partial implementation using lutimes(3), which should be available on the BSDs. However, it's ifdefed out due to a casting problem: The TimeSpec uses a CTime, while lutimes uses a CLong. These data types may be internally the same on some or all platforms, so if you want this feature you can try changing the \"ifdef 0\" in Touch.hsc to 1 and try it, see if \"git annex add\" mirrors file modification time in created symlinks, and let me know.
+"""]]

From 23cf9dac8633f110e900e7b79d0e1c12ce9d3b5f Mon Sep 17 00:00:00 2001
From: praet 
Date: Sun, 20 Mar 2011 20:11:28 +0000
Subject: [PATCH 1139/8313] Added a comment: Brainfart

---
 ..._79d96599f757757f34d7b784e6c0e81c._comment | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment
new file mode 100644
index 0000000000..c265b58995
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.240.27.89"
+ subject="Brainfart"
+ date="2011-03-20T20:11:27Z"
+ content="""
+Haven't given these any serious thought (which will become apparent in a moment) but hoping they will give birth to some less retarded ideas:
+
+---
+
+### Bait'n'switch
+
+- pre-commit: Replace all staged symlinks (when pointing to annexed files) with plaintext files containing the key of their respective annexed content, re-stage, and add their paths (relative to repo root) to .gitignore.
+- post-commit: Replace the plaintext files with (git annex fix'ed) symlinks.
+
+In doing so, the blobs to be committed can remain unaltered, irrespective of their related files' depth in the directory hierarchy.
+
+To prevent git from reporting ALL annexed files as unstaged changes after running post-commit hook, their paths would need to be added to .gitignore.
+
+This wouldn't cause any issues when adding files, very little when modifying files (would need some alterations to \"git annex unlock\"), BUT would make git totally oblivious to removals...
+
+---
+
+### Manifest-based (re)population
+- Keep a manifest of all annexed files (key + relative path)
+- DON'T track the symlinks (.gitignore)
+- Populate/update the directory structure using a post-commit hook.
+
+... thus circumventing the issue entirely, yet diffstats (et al.) would be rather uninformative.
+
+---
+
+***Wide open to suggestions, criticism, mocking laughter and finger-pointing :)***
+"""]]

From cb4ed36f13b05ca9baa64820f64bc84448a7af46 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 20 Mar 2011 20:48:42 +0000
Subject: [PATCH 1140/8313] Added a comment

---
 ..._548303d6ffb21a9370b6904f41ff49c1._comment | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment
new file mode 100644
index 0000000000..cd116c232d
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2011-03-20T20:48:41Z"
+ content="""
+ok, pulling the latest master and building on OSX now does this...
+
+
+ghc -O2 -Wall -ignore-package monads-fd --make git-annex
+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )
+
+Touch.hsc:24:0:
+    The type signature for `touchBoth' lacks an accompanying binding
+
+Touch.hsc:27:26: Not in scope: `touchBoth'
+make: *** [git-annex] Error 1
+

+
+changing the #if 0 to 1 gives this...
+
+
+ghc -O2 -Wall -ignore-package monads-fd --make git-annex
+[ 1 of 63] Compiling Touch            ( Touch.hs, Touch.o )
+
+Touch.hsc:95:43:
+    Couldn't match expected type `CLong' against inferred type `CTime'
+    In the second argument of `(\ hsc_ptr
+                                    -> pokeByteOff hsc_ptr 0)', namely
+        `(sec :: CLong)'
+    In a stmt of a 'do' expression:
+        (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong)
+    In the expression:
+        do { (\ hsc_ptr -> pokeByteOff hsc_ptr 0) ptr (sec :: CLong);
+             (\ hsc_ptr -> pokeByteOff hsc_ptr 4) ptr (0 :: CLong) }
+make: *** [git-annex] Error 1
+

+
+
+it seems that commit 6634b6a6b84a924f6f6059b5bea61f449d056eee has broken support for OSX.
+
+"""]]

From 08d1c13628c1f927be58f8454058f458cca22817 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 18:06:06 -0400
Subject: [PATCH 1141/8313] fix typo

---
 Touch.hsc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Touch.hsc b/Touch.hsc
index 8a3e31b571..fd3500f869 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -81,7 +81,7 @@ touchBoth file atime mtime follow =
  - TODO: test if lutimes is available. May have to do it in configure.
  - TODO: TimeSpec uses a CTime, while tv_sec is a CLong. It is implementation
  - dependent whether these are the same; need to find a cast that works.
- - (The cast below fails.. without the cast it works on linux i386, but
+ - (Without the cast it works on linux i386, but
  - maybe not elsewhere.)
  -}
 
@@ -92,7 +92,7 @@ instance Storable TimeSpec where
 		sec <- #{peek struct timeval, tv_sec} ptr
 		return $ TimeSpec sec
 	poke ptr (TimeSpec sec) = do
-		#{poke struct timeval, tv_sec} ptr (sec :: CLong)
+		#{poke struct timeval, tv_sec} ptr sec
 		#{poke struct timeval, tv_usec} ptr (0 :: CLong) 
 
 foreign import ccall "utimes" 
@@ -114,7 +114,7 @@ touchBoth file atime mtime follow =
 			else c_utimes
 
 #else
-warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
+#warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
 touchBoth _ _ _ _ = return ()
 #endif
 #endif

From fccd30cacde588ca36dd066b1df5ec586a4647cb Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 20 Mar 2011 22:06:26 +0000
Subject: [PATCH 1142/8313] Added a comment

---
 .../comment_7_7ca00527ab5db058aadec4fe813e51fd._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment
new file mode 100644
index 0000000000..e35dc8a827
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-20T22:06:25Z"
+ content="""
+Fixed that, and removed the impossible cast so it can be built with #if 1
+"""]]

From 09b16afe02882485f60953f01726ea65d38af920 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 18:11:00 -0400
Subject: [PATCH 1143/8313] releasing version 0.20110320

---
 debian/changelog           | 4 ++--
 doc/news/version_0.25.mdwn | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)
 create mode 100644 doc/news/version_0.25.mdwn

diff --git a/debian/changelog b/debian/changelog
index 032dedfb9e..eb8d735047 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110317) UNRELEASED; urgency=low
+git-annex (0.20110320) experimental; urgency=low
 
   * Fix dropping of files using the URL backend.
   * Fix support for remotes with '.' in their names.
@@ -8,7 +8,7 @@ git-annex (0.20110317) UNRELEASED; urgency=low
     upgrades, etc. Use git-annex upgrade when you're ready to run this
     version.
 
- -- Joey Hess   Thu, 17 Mar 2011 11:46:53 -0400
+ -- Joey Hess   Sun, 20 Mar 2011 16:36:33 -0400
 
 git-annex (0.20110316) experimental; urgency=low
 
diff --git a/doc/news/version_0.25.mdwn b/doc/news/version_0.25.mdwn
new file mode 100644
index 0000000000..13abd66cd5
--- /dev/null
+++ b/doc/news/version_0.25.mdwn
@@ -0,0 +1,6 @@
+git-annex 0.25 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix dropping of files using the URL backend.
+   * Fix support for remotes with '.' in their names.
+   * Add version command to show git-annex version as well as repository
+     version information."""]]
\ No newline at end of file

From 162047b8da7ede5da44d031d4265890669ed8e78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Mar 2011 18:11:24 -0400
Subject: [PATCH 1144/8313] add news item for git-annex 0.20110320

---
 doc/news/version_0.20110320.mdwn |  9 +++++++++
 doc/news/version_0.21.mdwn       |  7 -------
 doc/news/version_0.22.mdwn       | 24 ------------------------
 3 files changed, 9 insertions(+), 31 deletions(-)
 create mode 100644 doc/news/version_0.20110320.mdwn
 delete mode 100644 doc/news/version_0.21.mdwn
 delete mode 100644 doc/news/version_0.22.mdwn

diff --git a/doc/news/version_0.20110320.mdwn b/doc/news/version_0.20110320.mdwn
new file mode 100644
index 0000000000..84475829c3
--- /dev/null
+++ b/doc/news/version_0.20110320.mdwn
@@ -0,0 +1,9 @@
+git-annex 0.20110320 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix dropping of files using the URL backend.
+   * Fix support for remotes with '.' in their names.
+   * Add version command to show git-annex version as well as repository
+     version information.
+   * No longer auto-upgrade to repository format 2, to avoid accidental
+     upgrades, etc. Use git-annex upgrade when you're ready to run this
+     version."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.21.mdwn b/doc/news/version_0.21.mdwn
deleted file mode 100644
index 996474910e..0000000000
--- a/doc/news/version_0.21.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.21 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * test: Don't rely on chmod -R working.
-   * unannex: Fix recently introduced bug when attempting to unannex more
-     than one file at a time.
-   * test: Set git user name and email in case git can't guess values.
-   * Fix display of unicode filenames."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.22.mdwn b/doc/news/version_0.22.mdwn
deleted file mode 100644
index 4848e861e1..0000000000
--- a/doc/news/version_0.22.mdwn
+++ /dev/null
@@ -1,24 +0,0 @@
-git-annex 0.22 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Git annexes can now be attached to bare git repositories.
-     (Both the local and remote host must have this version of git-annex
-     installed for it to work.)
-   * Support filenames that start with a dash; when such a file is passed
-     to a utility it will be escaped to avoid it being interpreted as an
-     option. (I went a little overboard and got the type checker involved
-     in this, so such files are rather comprehensively supported now.)
-   * New backends: SHA512 SHA384 SHA256 SHA224
-     (Supported on systems where corresponding shaNsum commands are available.)
-   * describe: New subcommand that can set or change the description of
-     a repository.
-   * Fix test suite to reap zombies.
-     (Zombies can be particularly annoying on OSX; thanks to Jimmy Tang
-     for his help eliminating the infestation... for now.)
-   * Make test suite not rely on a working cp -pr.
-     (The Unix wars are still ON!)
-   * Look for dir.git directories the same as git does.
-   * Support remote urls specified as relative paths.
-   * Support non-ssh remote paths that contain tilde expansions.
-   * fsck: Check for and repair location log damage.
-   * Bugfix: When fsck detected and moved away corrupt file content, it did
-     not update the location log."""]]
\ No newline at end of file

From d8dfd2c103606e29cd16fc33680f53650144836c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmSbJHbvlxbCjtPXk_Io3qP3MFqJr3pUgQ"
 
Date: Mon, 21 Mar 2011 04:27:46 +0000
Subject: [PATCH 1145/8313]

---
 .../Unfortunate_interaction_with_Calibre.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/Unfortunate_interaction_with_Calibre.mdwn

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn b/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn
new file mode 100644
index 0000000000..d00a6720cd
--- /dev/null
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre.mdwn
@@ -0,0 +1,21 @@
+# Calibre 
+
+Calibre is a somewhat popular eBook management package that's also free software.    
+
+Install via
+    # apt-get install calibre
+
+There is a somewhat unfortunate interaction between Calibre and git-annex...
+
+* git-annex makes its files become read-only.  By the way, that's not quite obvious from the documentation; I suggest making that more prominent.
+* Calibre modifies files (not quite sure of semantics, how, or why) when doing various operations, notably such as when copying a book from one's library to one's portable reading device.
+
+These don't play well together, sadly.
+
+I'd expect most of the issue to sit on the Calibre side, and have reported it as a bug.
+[Calibre bug #739045](https://bugs.launchpad.net/calibre/+bug/739045)
+Preliminary indication is that they're treating it as a functionality change they'll decline to fix.  Which isn't entirely unreasonable - I anticipated as much, and I don't want to treat that as a bad/wrong decision.
+
+However, I think it's:
+* Unfortunate, as fitting Calibre together with git-annex seems like a neat idea.
+* Useful to make sure that this kind of "doesn't play well together" condition is documented, even if only as a bug report.

From 6fe02b24fa8c5f9a11e4deb319acad671285f1a5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 21 Mar 2011 08:52:19 +0000
Subject: [PATCH 1146/8313] Added a comment

---
 .../comment_8_881aecb9ae671689453f6d5d780d844b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment
new file mode 100644
index 0000000000..56a7eb360e
--- /dev/null
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-03-21T08:52:18Z"
+ content="""
+Just tried building both of the code paths, and they seem to build and somewhat function on OSX. I have yet to confirm the functionality is working correctly, but so far it's looking good. (I somewhat care less about the utimes/mtimes of my files since I care more about the content :) )
+"""]]

From e1147b4454109dddff0b6e79decda710fd71cbb2 Mon Sep 17 00:00:00 2001
From: praet 
Date: Mon, 21 Mar 2011 19:58:36 +0000
Subject: [PATCH 1147/8313] Added a comment

---
 ...mment_5_d61f5693d947b9736b29fca1dbc7ad76._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment
new file mode 100644
index 0000000000..93db97e704
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="praet"
+ ip="81.242.56.203"
+ subject="comment 5"
+ date="2011-03-21T19:58:34Z"
+ content="""
+In the meantime, would it be acceptable to split the pre-commit hook
+into two discrete parts?
+
+This would allow to (if preferred) defer \"git annex fix\" until
+post-commit while still keeping the safety net for unlocked files.
+"""]]

From c048905dc4c2ce155dbb03d7e60014568faa9553 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 21 Mar 2011 20:46:43 -0400
Subject: [PATCH 1148/8313] upgrade messages

---
 Command/Upgrade.hs | 2 ++
 Upgrade/V0.hs      | 2 +-
 Upgrade/V1.hs      | 5 ++++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 94398c70a1..880a5324f7 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -10,6 +10,7 @@ module Command.Upgrade where
 import Command
 import Upgrade
 import Version
+import Messages
 
 command :: [Command]
 command = [standaloneCommand "upgrade" paramNothing seek
@@ -20,6 +21,7 @@ seek = [withNothing start]
 
 start :: CommandStartNothing
 start = do
+	showStart "upgrade" ""
 	r <- upgrade
 	checkVersion
 	return $ Just $ return $ Just $ return r
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index 5ba305817b..eabd030098 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -23,7 +23,7 @@ import qualified Upgrade.V1
 
 upgrade :: Annex Bool
 upgrade = do
-	showSideAction "Upgrading object directory layout v0 to v1..."
+	showNote "v0 to v1..."
 	g <- Annex.gitRepo
 
 	-- do the reorganisation of the key files
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 43f279ad08..f18a76c875 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -55,7 +55,7 @@ import qualified Command.Init
 
 upgrade :: Annex Bool
 upgrade = do
-	showSideAction "Upgrading object directory layout v1 to v2..."
+	showNote "v1 to v2"
 
 	g <- Annex.gitRepo
 	if Git.repoIsLocalBare g
@@ -78,6 +78,7 @@ upgrade = do
 
 moveContent :: Annex ()
 moveContent = do
+	showNote "moving content..."
 	keys <- getKeysPresent1
 	forM_ keys move
 	where
@@ -92,6 +93,7 @@ moveContent = do
 
 updateSymlinks :: Annex ()
 updateSymlinks = do
+	showNote "updating symlinks content..."
 	g <- Annex.gitRepo
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
 	forM_ files $ fixlink
@@ -109,6 +111,7 @@ updateSymlinks = do
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
+	showNote "moving location logs..."
 	logkeys <- oldlocationlogs
 	forM_ logkeys move
 		where

From c94261020f29887b46fffc5652f700a00fefb1a2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 21 Mar 2011 23:37:46 -0400
Subject: [PATCH 1149/8313] typo

---
 Upgrade/V1.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index f18a76c875..c0470a3bc8 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -93,7 +93,7 @@ moveContent = do
 
 updateSymlinks :: Annex ()
 updateSymlinks = do
-	showNote "updating symlinks content..."
+	showNote "updating symlinks..."
 	g <- Annex.gitRepo
 	files <- liftIO $ Git.inRepo g [Git.workTree g]
 	forM_ files $ fixlink

From e6dfcbf32b051547b78d68f289d5a505341e9d85 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 22 Mar 2011 13:06:59 +0000
Subject: [PATCH 1150/8313] Added a comment: Please provide stable tarballs or
 zipfiles

---
 .../comment_1_fbd8b6d39e9d3c71791551358c863966._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment

diff --git a/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment b/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
new file mode 100644
index 0000000000..488e005278
--- /dev/null
+++ b/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="84.189.2.244"
+ subject="Please provide stable tarballs or zipfiles"
+ date="2011-03-22T13:06:58Z"
+ content="""
+I'm trying to package git annex for ArchLinux and NixOS. That task would be a *lot* easier, if there were proper release archives available for download. The Gitweb site offers to create snapshot tarballs on the fly, but those tarballs have a different SHA hash every time they're generated, so they cannot be used for the purposes of a distribution. A simple solution for this problem would be to enable snapshots in zip format (because zip files look the same every time they're generated).
+"""]]

From 90a15fc214d913d9e8aa3a937ca214af1d8d66e6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 22 Mar 2011 14:01:38 +0000
Subject: [PATCH 1151/8313] Added a comment

---
 .../comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment

diff --git a/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment b/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
new file mode 100644
index 0000000000..5441c3e4ce
--- /dev/null
+++ b/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-22T14:01:37Z"
+ content="""
+maybe snag tarballs from  ? 
+"""]]

From 6815a9974febe2b872523672772aeb1c3fea9689 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Mar 2011 18:09:21 +0000
Subject: [PATCH 1152/8313] Added a comment

---
 ...comment_3_cf6044ebe99f71158034e21197228abd._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment

diff --git a/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment b/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
new file mode 100644
index 0000000000..b72b848f80
--- /dev/null
+++ b/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-22T18:09:21Z"
+ content="""
+The tarballs produced by gitweb are actually stable. They are wrapped in a gz file with a varying timestamp however. It might be nice if gitweb passed --no-name to gzip to avoid that inconsistency.
+
+git-annex also has a [pristine-tar](http://kitenet.net/~joey/code/pristine-tar/) branch in git that can be used to recreate the tarballs I upload to Debian.
+"""]]

From 25605d18fd0a9404dba92e0c7d9f438fbd70fcd4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 15:39:36 -0400
Subject: [PATCH 1153/8313] generalize

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c60e19b311..08e2f59fb0 100644
--- a/Makefile
+++ b/Makefile
@@ -11,7 +11,7 @@ SysConfig.hs: configure.hs TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
 
-Touch.hs: Touch.hsc
+%.hs: %.hsc
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 

From 8ede2e255ffd2397f0714df066d078bc45d839ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 15:48:14 -0400
Subject: [PATCH 1154/8313] add StatFS.hsc, copied from xmobar

---
 .gitignore       |   1 +
 StatFS.hsc       | 109 +++++++++++++++++++++++++++++++++++++++++++++++
 debian/copyright |   4 ++
 3 files changed, 114 insertions(+)
 create mode 100644 StatFS.hsc

diff --git a/.gitignore b/.gitignore
index 69d2c80709..aa677c1335 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ html
 *.tix
 .hpc
 Touch.hs
+StatFS.hs
diff --git a/StatFS.hsc b/StatFS.hsc
new file mode 100644
index 0000000000..f91a6912ce
--- /dev/null
+++ b/StatFS.hsc
@@ -0,0 +1,109 @@
+-----------------------------------------------------------------------------
+-- |
+--
+-- (This code comes from xmobar)
+-- 
+-- Module      :  StatFS
+-- Copyright   :  (c) Jose A Ortega Ruiz
+-- License     :  BSD-3-clause
+--
+-- All rights reserved.
+-- 
+-- Redistribution and use in source and binary forms, with or without
+-- modification, are permitted provided that the following conditions
+-- are met:
+-- 
+-- 1. Redistributions of source code must retain the above copyright
+--    notice, this list of conditions and the following disclaimer.
+-- 2. Redistributions in binary form must reproduce the above copyright
+--    notice, this list of conditions and the following disclaimer in the
+--    documentation and/or other materials provided with the distribution.
+-- 3. Neither the name of the author nor the names of his contributors
+--    may be used to endorse or promote products derived from this software
+--    without specific prior written permission.
+-- 
+-- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+-- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+-- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+-- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+-- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+-- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+-- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+-- SUCH DAMAGE.
+--
+-- Maintainer  :  Jose A Ortega Ruiz 
+-- Stability   :  unstable
+-- Portability :  unportable
+--
+--  A binding to C's statvfs(2)
+--
+-----------------------------------------------------------------------------
+
+{-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
+
+
+module StatFS ( FileSystemStats(..), getFileSystemStats ) where
+
+import Foreign
+import Foreign.C.Types
+import Foreign.C.String
+import Data.ByteString (useAsCString)
+import Data.ByteString.Char8 (pack)
+
+#if defined (__FreeBSD__)
+# include 
+# include 
+#else
+#include 
+#endif
+
+data FileSystemStats = FileSystemStats {
+  fsStatBlockSize :: Integer
+  -- ^ Optimal transfer block size.
+  , fsStatBlockCount :: Integer
+  -- ^ Total data blocks in file system.
+  , fsStatByteCount :: Integer
+  -- ^ Total bytes in file system.
+  , fsStatBytesFree :: Integer
+  -- ^ Free bytes in file system.
+  , fsStatBytesAvailable :: Integer
+  -- ^ Free bytes available to non-superusers.
+  , fsStatBytesUsed :: Integer
+  -- ^ Bytes used.
+  } deriving (Show, Eq)
+
+data CStatfs
+
+#if defined(__FreeBSD__)
+foreign import ccall unsafe "sys/mount.h statfs"
+#else
+foreign import ccall unsafe "sys/vfs.h statfs64"
+#endif
+  c_statfs :: CString -> Ptr CStatfs -> IO CInt
+
+toI :: CLong -> Integer
+toI = toInteger
+
+getFileSystemStats :: String -> IO (Maybe FileSystemStats)
+getFileSystemStats path =
+  allocaBytes (#size struct statfs) $ \vfs ->
+  useAsCString (pack path) $ \cpath -> do
+    res <- c_statfs cpath vfs
+    if res == -1 then return Nothing
+      else do
+        bsize <- (#peek struct statfs, f_bsize) vfs
+        bcount <- (#peek struct statfs, f_blocks) vfs
+        bfree <- (#peek struct statfs, f_bfree) vfs
+        bavail <- (#peek struct statfs, f_bavail) vfs
+        let bpb = toI bsize
+        return $ Just FileSystemStats
+                       { fsStatBlockSize = bpb
+                       , fsStatBlockCount = toI bcount
+                       , fsStatByteCount = toI bcount * bpb
+                       , fsStatBytesFree = toI bfree * bpb
+                       , fsStatBytesAvailable = toI bavail * bpb
+                       , fsStatBytesUsed = toI (bcount - bfree) * bpb
+                       }
diff --git a/debian/copyright b/debian/copyright
index 90e26b7524..2144501e11 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -7,3 +7,7 @@ License: GPL-3+
  The full text of version 3 of the GPL is distributed as doc/GPL in
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
+
+Files: StatFS.hsc
+Copyright: Jose A Ortega Ruiz 
+License: BSD-3-clause

From aa1bc31e0aede63a1e68d2ec3e2653a7f5be0ae7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 15:58:47 -0400
Subject: [PATCH 1155/8313] be a no-op on non-linux, non-freebsd systems

Todo later: use POSIX statvfs

Note: Re OSX, see http://code.google.com/p/xmobar/issues/detail?id=28
Apparently xmobar's code will work on OSX, probably __FreeBSD__ is defined
there.
---
 StatFS.hsc | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index f91a6912ce..8b453dc199 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -1,7 +1,7 @@
 -----------------------------------------------------------------------------
 -- |
 --
--- (This code comes from xmobar)
+-- (This code originally comes from xmobar)
 -- 
 -- Module      :  StatFS
 -- Copyright   :  (c) Jose A Ortega Ruiz
@@ -57,7 +57,11 @@ import Data.ByteString.Char8 (pack)
 # include 
 # include 
 #else
+#if defined (__linux__)
 #include 
+#else
+#define UNKNOWN
+#endif
 #endif
 
 data FileSystemStats = FileSystemStats {
@@ -77,18 +81,25 @@ data FileSystemStats = FileSystemStats {
 
 data CStatfs
 
+#ifdef UNKNOWN
+#warning free space checking code not available for this OS
+#else
 #if defined(__FreeBSD__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"
 #endif
   c_statfs :: CString -> Ptr CStatfs -> IO CInt
+#endif
 
 toI :: CLong -> Integer
 toI = toInteger
 
 getFileSystemStats :: String -> IO (Maybe FileSystemStats)
 getFileSystemStats path =
+#ifdef UNKNOWN
+  return Nothing
+#else
   allocaBytes (#size struct statfs) $ \vfs ->
   useAsCString (pack path) $ \cpath -> do
     res <- c_statfs cpath vfs
@@ -107,3 +118,4 @@ getFileSystemStats path =
                        , fsStatBytesAvailable = toI bavail * bpb
                        , fsStatBytesUsed = toI (bcount - bfree) * bpb
                        }
+#endif

From 66f1d7dc5b63d1f017e557eb354407f1d8454207 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 21:25:29 +0000
Subject: [PATCH 1156/8313]

---
 doc/bugs/softlink_atime.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index 0d26d1ce6a..c48d3a4de4 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -37,3 +37,13 @@ Optionally, editing the meta-data should change the times in all annexes.
 	git add .metadata
 
 >>>>>> Thanks a lot. Doing this in a new git-annex repo from the start should at least ensure local consistency and I assume I can simply add a post-pull hook to restore the mtimes on all all other repositories? -- RichiH
+
+>>>>>>> This is even better:
+
+    #!/bin/sh
+    git annex pre-commit .
+    which metastore || echo "$0: metastore is not installed; exiting" ; exit 99
+    metastore --save
+    git add .metadata
+
+>>>>>>> -- RichiH

From 0fe3ff8e1404f9893f4340281fedb70de2622de7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 21:25:47 +0000
Subject: [PATCH 1157/8313]

---
 doc/bugs/softlink_atime.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index c48d3a4de4..69d6f6600e 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -42,7 +42,7 @@ Optionally, editing the meta-data should change the times in all annexes.
 
     #!/bin/sh
     git annex pre-commit .
-    which metastore || echo "$0: metastore is not installed; exiting" ; exit 99
+    which metastore || echo "$0: metastore is not installed; exiting"; exit 99
     metastore --save
     git add .metadata
 

From aa2d8e33df3fc6ba204e28001ab0d1d231c9c58e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 17:27:04 -0400
Subject: [PATCH 1158/8313] free space checking

Free space checking is now done, for transfers of data for keys that have free space metadata.
(Notably, not for SHA* keys generated with git-annex 0.24 or earlier.)

The code is believed to work on Linux, FreeBSD, and OSX; check compile-time
messages to see if it is not enabled for your OS.
---
 Command/Migrate.hs                |  2 +-
 Command/SetKey.hs                 |  3 +-
 Command/Unlock.hs                 |  2 ++
 Content.hs                        | 51 +++++++++++++++++++++++++++++++
 debian/changelog                  | 10 ++++++
 doc/bugs/free_space_checking.mdwn |  3 ++
 6 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 584f6e34e1..56147113b9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -55,7 +55,7 @@ perform file oldkey newbackend = do
 	case stored of
 		Nothing -> return Nothing
 		Just (newkey, _) -> do
-			ok <- getViaTmp newkey $ \t -> do
+			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
 				liftIO $ createLink src t
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index af46fe06e4..6f6078e4ba 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -32,7 +32,8 @@ perform :: FilePath -> CommandPerform
 perform file = do
 	key <- cmdlineKey
 	-- the file might be on a different filesystem, so mv is used
-	-- rather than simply calling moveToObjectDir
+	-- rather than simply calling moveToObjectDir; disk space is also
+	-- checked this way.
 	ok <- getViaTmp key $ \dest -> do
 		if dest /= file
 			then liftIO $
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index ac7b22ac72..bf593e1e99 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -41,6 +41,8 @@ perform dest key = do
 	inbackend <- Backend.hasKey key
 	when (not inbackend) $
 		error "content not present"
+	
+	checkDiskSpace key
 
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g key
diff --git a/Content.hs b/Content.hs
index 4bd8265c2d..596274ad09 100644
--- a/Content.hs
+++ b/Content.hs
@@ -10,6 +10,8 @@ module Content (
 	calcGitLink,
 	logStatus,
 	getViaTmp,
+	getViaTmpUnchecked,
+	checkDiskSpace,
 	preventWrite,
 	allowWrite,
 	moveAnnex,
@@ -35,6 +37,8 @@ import UUID
 import qualified GitRepo as Git
 import qualified Annex
 import Utility
+import StatFS
+import Key
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
@@ -75,6 +79,27 @@ getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
 	g <- Annex.gitRepo
 	let tmp = gitAnnexTmpLocation g key
+
+	-- Check that there is enough free disk space.
+	-- When the temp file already exists, count the space
+	-- it is using as free.
+	e <- liftIO $ doesFileExist tmp
+	if e
+		then do
+			stat <- liftIO $ getFileStatus tmp
+			checkDiskSpace' (fromIntegral $ fileSize stat) key
+		else checkDiskSpace key
+
+	getViaTmpUnchecked key action
+
+{- Like getViaTmp, but does not check that there is enough disk space
+ - for the incoming key. For use when the key content is already on disk
+ - and not being copied into place. -}
+getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool
+getViaTmpUnchecked key action = do
+	g <- Annex.gitRepo
+	let tmp = gitAnnexTmpLocation g key
+
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	success <- action tmp
 	if success
@@ -87,6 +112,32 @@ getViaTmp key action = do
 			-- to resume its transfer
 			return False
 
+{- Checks that there is disk space available to store a given key,
+ - throwing an error if not. -}
+checkDiskSpace :: Key -> Annex ()
+checkDiskSpace = checkDiskSpace' 0
+
+checkDiskSpace' :: Integer -> Key -> Annex ()
+checkDiskSpace' adjustment key = do
+	liftIO $ putStrLn $ "adjust " ++ show adjustment
+	g <- Annex.gitRepo
+	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
+	case (stats, keySize key) of
+		(Nothing, _) -> return ()
+		(_, Nothing) -> return ()
+		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
+			if (need + overhead >= have + adjustment)
+				then error $ "not enough free space (have " ++ 
+					showsize (have + adjustment) ++ "; need " ++
+					showsize (need + overhead) ++ ")"
+				else return ()
+	where
+		showsize i = show i
+		-- Adding a file to the annex requires some overhead beyond
+		-- just the file size; the git index must be updated, etc. 
+		-- This is an arbitrary value.
+		overhead = 1024 * 1024 -- 1 mb
+
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
 preventWrite f = unsetFileMode f writebits
diff --git a/debian/changelog b/debian/changelog
index eb8d735047..a5830884ab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+git-annex (0.20110321) UNRELEASED; urgency=low
+
+  * Free space checking is now done, for transfers of data for keys
+    that have free space metadata. (Notably, not for SHA* keys generated
+    with git-annex 0.24 or earlier.) The code is believed to work on
+    Linux, FreeBSD, and OSX; check compile-time messages to see if it
+    is not enabled for your OS.
+
+ -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
+
 git-annex (0.20110320) experimental; urgency=low
 
   * Fix dropping of files using the URL backend.
diff --git a/doc/bugs/free_space_checking.mdwn b/doc/bugs/free_space_checking.mdwn
index eaa3294d60..92e8be40d1 100644
--- a/doc/bugs/free_space_checking.mdwn
+++ b/doc/bugs/free_space_checking.mdwn
@@ -16,3 +16,6 @@ file around.
   find files that lack size info, and rename their keys to add the size
   info. Users with old repos can run this on them, to get the missing
   info recorded.
+
+> [[done]]; no migtation process for old SHA1 keys from v1 repo though.
+> --[[Joey]] 

From c21998722cb6a65993a3b72e66b225443cfce48b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 17:41:06 -0400
Subject: [PATCH 1159/8313] fast mode

Add --fast flag, that can enable less expensive, but also less thurough versions of some commands.

* Add --fast flag, that can enable less expensive, but also less thurough
  versions of some commands.
* fsck: In fast mode, avoid checking checksums.
* unused: In fast mode, just show all existing temp files as unused,
  and avoid expensive scan for other unused content.
---
 Annex.hs           |  2 ++
 Backend/SHA.hs     |  3 ++-
 Command/Unused.hs  | 27 ++++++++++++++++++---------
 Options.hs         |  3 +++
 debian/changelog   |  5 +++++
 doc/git-annex.mdwn |  6 ++++++
 6 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 608151d824..f45415a72f 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -40,6 +40,7 @@ data AnnexState = AnnexState
 	, repoqueue :: GitQueue.Queue
 	, quiet :: Bool
 	, force :: Bool
+	, fast :: Bool
 	, defaultbackend :: Maybe String
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
@@ -56,6 +57,7 @@ newState gitrepo allbackends = AnnexState
 	, repoqueue = GitQueue.empty
 	, quiet = False
 	, force = False
+	, fast = False
 	, defaultbackend = Nothing
 	, defaultkey = Nothing
 	, toremote = Nothing
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 0563851076..0ec555ce3f 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -80,9 +80,10 @@ keyValue size file = do
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	g <- Annex.gitRepo
+	fast <- Annex.getState Annex.fast
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
-	if not present
+	if (not present || fast)
 		then return True
 		else do
 			s <- shaN size file
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a1c4ee03c9..518e986561 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -44,7 +44,6 @@ perform = do
 
 checkUnused :: Annex Bool
 checkUnused = do
-	showNote "checking for unused data..."
 	(unused, staletmp) <- unusedKeys
 	let unusedlist = number 0 unused
 	let staletmplist = number (length unused) staletmp
@@ -81,17 +80,27 @@ number n (x:xs) = (n+1, x):(number (n+1) xs)
 unusedKeys :: Annex ([Key], [Key])
 unusedKeys = do
 	g <- Annex.gitRepo
-	present <- getKeysPresent
-	referenced <- getKeysReferenced
-	tmps <- tmpKeys
 	
-	let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps
+	fast <- Annex.getState Annex.fast
+	if fast
+		then do
+			showNote "fast mode enabled; assuming all temporary files are unused"
+			tmps <- tmpKeys
+			return ([], tmps)
+		else do
+			showNote "checking for unused data..."
+			present <- getKeysPresent
+			referenced <- getKeysReferenced
+			tmps <- tmpKeys
+	
+			let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps
 
-	-- Tmp files that are dups of content already present can simply
-	-- be removed.
-	liftIO $ forM_ duptmp $ \t -> removeFile $ gitAnnexTmpLocation g t
+			-- Tmp files that are dups of content already present
+			-- can simply be removed.
+			liftIO $ forM_ duptmp $ \t -> removeFile $
+				gitAnnexTmpLocation g t
 
-	return (unused, staletmp)
+			return (unused, staletmp)
 
 calcUnusedKeys :: [Key] -> [Key] -> [Key] -> ([Key], [Key], [Key])
 calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp)
diff --git a/Options.hs b/Options.hs
index 4cd62c2222..10c3714e41 100644
--- a/Options.hs
+++ b/Options.hs
@@ -22,6 +22,8 @@ commonOptions :: [Option]
 commonOptions =
 	[ Option ['f'] ["force"] (NoArg (setforce True))
 		"allow actions that may lose annexed data"
+	, Option ['F'] ["fast"] (NoArg (setfast True))
+		"avoid slow operations"
 	, Option ['q'] ["quiet"] (NoArg (setquiet True))
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setquiet False))
@@ -31,5 +33,6 @@ commonOptions =
 	]
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
+		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
 		setdefaultbackend v = Annex.changeState $ \s -> s { Annex.defaultbackend = Just v }
diff --git a/debian/changelog b/debian/changelog
index a5830884ab..e0927817a7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,11 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     with git-annex 0.24 or earlier.) The code is believed to work on
     Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
+  * Add --fast flag, that can enable less expensive, but also less thurough
+    versions of some commands.
+  * fsck: In fast mode, avoid checking checksums.
+  * unused: In fast mode, just show all existing temp files as unused,
+    and avoid expensive scan for other unused content.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1e4af022f1..6168ebae2f 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -278,6 +278,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Force unsafe actions, such as dropping a file's content when no other
   source of it can be verified to still exist. Use with care.
 
+* --fast
+
+  Enables less expensive, but also less thorough versions of some commands.
+  What is avoided depends on the command. A fast fsck avoids calculating
+  checksums; a fast unused only shows temp files and not other unused files.
+
 * --quiet
 
   Avoid the default verbose logging of what is done; only show errors

From 368e20eb84fac8224a2ab33616cdd31f2c4d5ff1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 17:53:40 -0400
Subject: [PATCH 1160/8313] diskreserve setting

Add annex.diskreserve config setting, to control how much free space to
reserve for other purposes and avoid using (defaults to 1 mb).
---
 Content.hs         | 13 ++++++-------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  8 ++++++++
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/Content.hs b/Content.hs
index 596274ad09..39a3addccf 100644
--- a/Content.hs
+++ b/Content.hs
@@ -119,24 +119,23 @@ checkDiskSpace = checkDiskSpace' 0
 
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
-	liftIO $ putStrLn $ "adjust " ++ show adjustment
 	g <- Annex.gitRepo
+	r <- Annex.repoConfig g "diskreserve" ""
+	let reserve = if null r then megabyte else (read r :: Integer)
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
-			if (need + overhead >= have + adjustment)
+			if (need + reserve > have + adjustment)
 				then error $ "not enough free space (have " ++ 
 					showsize (have + adjustment) ++ "; need " ++
-					showsize (need + overhead) ++ ")"
+					showsize (need + reserve) ++ ")"
 				else return ()
 	where
 		showsize i = show i
-		-- Adding a file to the annex requires some overhead beyond
-		-- just the file size; the git index must be updated, etc. 
-		-- This is an arbitrary value.
-		overhead = 1024 * 1024 -- 1 mb
+		megabyte :: Integer
+		megabyte = 1024 * 1024
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
diff --git a/debian/changelog b/debian/changelog
index e0927817a7..88e0986a94 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     with git-annex 0.24 or earlier.) The code is believed to work on
     Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
+  * Add annex.diskreserve config setting, to control how much free space
+    to reserve for other purposes and avoid using (defaults to 1 mb).
   * Add --fast flag, that can enable less expensive, but also less thurough
     versions of some commands.
   * fsck: In fast mode, avoid checking checksums.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 6168ebae2f..3cf408939e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -377,6 +377,14 @@ Here are all the supported configuration settings.
   Default ssh and rsync options to use if a remote does not have
   specific options.
 
+* `annex.diskreserve`
+
+  Amount of disk space to reserve. Disk space is checked when transferring
+  content to avoid running out, and additional free space can be reserved
+  via this option, to make space for more important content (such as git
+  commit logs). The units are bytes.
+  The default reserve is 1048576 (1 megabyte).
+
 * `annex.version`
 
   Automatically maintained, and used to automate upgrades between versions.

From 4ca3a3a9b5d38f1ac239b441c75ccb3a461e35cb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 22 Mar 2011 22:25:27 +0000
Subject: [PATCH 1161/8313]

---
 .../Makefile_is_missing_dependancies.mdwn     | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies.mdwn

diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn
new file mode 100644
index 0000000000..5a04ba49f6
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn
@@ -0,0 +1,31 @@
+
+
+From e45c73e66fc18d27bdf5797876fbeb07786a4af1 Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Tue, 22 Mar 2011 22:24:07 +0000
+Subject: [PATCH] Touch up Makefile to depend on StatFS.hs
+
+---
+ Makefile |    2 +-
+ 1 files changed, 1 insertions(+), 1 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 08e2f59..4ae8392 100644
+--- a/Makefile
++++ b/Makefile
+@@ -15,7 +15,7 @@ SysConfig.hs: configure.hs TestConfig.hs
+        hsc2hs $<
+        perl -i -pe 's/^{-# INCLUDE.*//' $@
+ 
+-$(bins): SysConfig.hs Touch.hs
++$(bins): SysConfig.hs Touch.hs StatFS.hs
+        $(GHCMAKE) $@
+ 
+ git-annex.1: doc/git-annex.mdwn
+-- 
+1.7.4.1
+
+

+
+
+StatFS.hs never gets depended on and compiled, the makefile was just missing something

From c44c318eafbfcd55a568c2761721520a95dfe530 Mon Sep 17 00:00:00 2001
From: Jimmy Tang 
Date: Tue, 22 Mar 2011 22:24:07 +0000
Subject: [PATCH 1162/8313] Touch up Makefile to depend on StatFS.hs

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 08e2f59fb0..4ae8392c71 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@ SysConfig.hs: configure.hs TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-$(bins): SysConfig.hs Touch.hs
+$(bins): SysConfig.hs Touch.hs StatFS.hs
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn

From fea20d260c7c095775b4667a95501f4e9cb7b741 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:45:11 -0400
Subject: [PATCH 1163/8313] applied

---
 doc/bugs/Makefile_is_missing_dependancies.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn
index 5a04ba49f6..4bda91f17d 100644
--- a/doc/bugs/Makefile_is_missing_dependancies.mdwn
+++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn
@@ -29,3 +29,6 @@ index 08e2f59..4ae8392 100644
 
 
 StatFS.hs never gets depended on and compiled, the makefile was just missing something
+
+> Thanks, [done]]! Interested to hear if StatFS.hs works on OSX (no warning) or
+> is a no-op (with warning). --[[Joey]] 

From ee6fabb7cac48f0f2eaec999164b5f40186ca92b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:50:36 -0400
Subject: [PATCH 1164/8313] update

---
 doc/upgrades.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 5c2187c891..dbea5e9c4a 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -53,6 +53,13 @@ Symlinks changed.
 Also, hashing was added to location log files in .git-annex/.
 And .gitattributes needed to have another line added to it.
 
+Previously, files added to the SHA [[backends]] did not have their file
+size tracked, while files added to the WORM backend did. Files added to
+the SHA backends after the conversion will have their file size tracked,
+and that information will be used by git-annex for disk space checking.
+There is not yet a way to add file size tracking information to old files
+in the SHA backend.
+
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
 Involved a reorganisation of the layout of .git/annex/. Symlinks changed.

From 9f9325aa3da7df99e09b8b1d393814e4590b9358 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:51:14 -0400
Subject: [PATCH 1165/8313] clean too

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4ae8392c71..e1aaf8ec37 100644
--- a/Makefile
+++ b/Makefile
@@ -61,7 +61,8 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure Touch.hs SysConfig.hs *.tix .hpc
+	rm -rf build $(bins) $(mans) test configure \
+		StatFS.hs Touch.hs SysConfig.hs *.tix .hpc
 	rm -rf doc/.ikiwiki html
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 

From 1ea373e2466e8a0f665a42392c71cb2db46f23e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:52:52 -0400
Subject: [PATCH 1166/8313] clarify

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3cf408939e..d7b57675d0 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -133,7 +133,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
-  and warns about any problems found.
+  and warns about or fixes any problems found.
 
   With parameters, only the specified files are checked.
 

From 3a419237895f1d8c5c598134330830b10d0176e6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:53:16 -0400
Subject: [PATCH 1167/8313] clarify wording

temp files are always assumed to be unused, in either mode
---
 Command/Unused.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 518e986561..83d8757cff 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -84,7 +84,7 @@ unusedKeys = do
 	fast <- Annex.getState Annex.fast
 	if fast
 		then do
-			showNote "fast mode enabled; assuming all temporary files are unused"
+			showNote "fast mode enabled; only finding temporary files"
 			tmps <- tmpKeys
 			return ([], tmps)
 		else do

From 5d759195618a96ce745b8ee559b439c86426a0f3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 18:55:29 -0400
Subject: [PATCH 1168/8313] update

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 88e0986a94..3c5c91b1b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,7 +7,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space
     to reserve for other purposes and avoid using (defaults to 1 mb).
-  * Add --fast flag, that can enable less expensive, but also less thurough
+  * Add --fast flag, that can enable less expensive, but also less thorough
     versions of some commands.
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,

From d2a8e511776ac84793ab70fcd33e7498bafe83cf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 22:59:55 +0000
Subject: [PATCH 1169/8313]

---
 doc/bugs/softlink_atime.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index 69d6f6600e..c62610be02 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -41,8 +41,8 @@ Optionally, editing the meta-data should change the times in all annexes.
 >>>>>>> This is even better:
 
     #!/bin/sh
+    if ! type metastore >/dev/null; then echo "$0: metastore is not installed; exiting"; exit 1; fi
     git annex pre-commit .
-    which metastore || echo "$0: metastore is not installed; exiting"; exit 99
     metastore --save
     git add .metadata
 

From 686b12ae303a5ddc69b51beb467113f0e8e6d1b5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 22 Mar 2011 23:41:53 +0000
Subject: [PATCH 1170/8313] Added a comment

---
 ..._4c30ade91fc7113a95960aa3bd1d5427._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
new file mode 100644
index 0000000000..b3dc8fe7a2
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-22T23:41:51Z"
+ content="""
+I may be missing something obvious, but when I copy to a remote repository, the object files are created, but no softlinks are created. When I pull everything from the remote, it pulls only files the local repo knows about already.
+
+        A
+       / \
+      B   C
+
+Moving from B to A creates no symlinks in A but the object files are moved to A. Copying back from A to B restores the object files in B and keeps them in A.
+
+Copying from A to an empty C does not create any object files nor symlinks. Copying from C to A creates no symlinks in A but the object files are copied to A.
+
+-- RichiH
+
+"""]]

From c1dc4079419cff94cca72441d5e67a866110ec7e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 20:31:22 -0400
Subject: [PATCH 1171/8313] Fix space leak in fsck and drop commands.

The space leak was somehow caused by this line:

	absfiles <- mapM absPath files

I confess, I don't quite understand why this caused bad buffering,
but apparently the whole pipeline from git-ls-files backed up at that
point.

Happily, rewriting the code to only get the cwd once and use a pure
function to calculate absfiles clears it up, and should be a little more
efficient in syscalls too.
---
 Command.hs       | 3 +--
 GitRepo.hs       | 4 ++--
 Utility.hs       | 9 ++++++++-
 debian/changelog | 1 +
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index 1449d7eedf..446b1b55fe 100644
--- a/Command.hs
+++ b/Command.hs
@@ -132,8 +132,7 @@ withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
-	files' <- filterFiles files
-	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
+	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
diff --git a/GitRepo.hs b/GitRepo.hs
index 34a59a10d4..4e4a063e82 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -449,9 +449,9 @@ checkAttr repo attr files = do
 	-- top of the repo). But we're passed files relative to the current
 	-- directory. Convert to absolute, and then convert the filenames
 	-- in its output back to relative.
-	absfiles <- mapM absPath files
-	(_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles
 	cwd <- getCurrentDirectory
+	let absfiles = map  (absPathFrom cwd) files
+	(_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles
 	return $ map (topair $ cwd++"/") $ lines s
 	where
 		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
diff --git a/Utility.hs b/Utility.hs
index e63fa1f6be..8312335f86 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -12,6 +12,7 @@ module Utility (
 	readFileStrict,
 	parentDir,
 	absPath,
+	absPathFrom,
 	relPathCwdToDir,
 	relPathDirToDir,
 	boolSystem,
@@ -165,8 +166,14 @@ dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 absPath :: FilePath -> IO FilePath
 absPath file = do
 	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file =
 	case absNormPath cwd file of
-		Just f -> return f
+		Just f -> f
 		Nothing -> error $ "unable to normalize " ++ file
 
 {- Constructs a relative path from the CWD to a directory.
diff --git a/debian/changelog b/debian/changelog
index 3c5c91b1b0..3738c77589 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
+  * Fix space leak in fsck and drop commands.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 

From 0e8f304dec2a0c2c1c20ce4c1cfd95b936d2e494 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 00:38:10 +0000
Subject: [PATCH 1172/8313] Added a comment

---
 ...comment_2_7d90e1e150e7524ba31687108fcc38d6._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
new file mode 100644
index 0000000000..a6f8e9cf97
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T00:38:10Z"
+ content="""
+`git annex move` only moves content. All symlink management is handled by git, so you have to keep repositories in sync using git as you would any other repo. When you `git pull B` in A, it will get whatever symlinks were added to B.
+
+(It can be useful to use a central bare repo and avoid needing to git pull from one repo to another, then you can just always push commits to the central repo, and pull down all changes from other repos.)
+"""]]

From 7051763b5b9ef4c230169a432384488d66078cee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 21:00:18 -0400
Subject: [PATCH 1173/8313] tweak

---
 GitRepo.hs       | 2 +-
 debian/changelog | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 4e4a063e82..ad58b28a00 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -450,7 +450,7 @@ checkAttr repo attr files = do
 	-- directory. Convert to absolute, and then convert the filenames
 	-- in its output back to relative.
 	cwd <- getCurrentDirectory
-	let absfiles = map  (absPathFrom cwd) files
+	let absfiles = map (absPathFrom cwd) files
 	(_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles
 	return $ map (topair $ cwd++"/") $ lines s
 	where
diff --git a/debian/changelog b/debian/changelog
index 3738c77589..f124296cc8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,7 +2,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
 
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
-    with git-annex 0.24 or earlier.) The code is believed to work on
+    with git-annex 0.2x or earlier.) The code is believed to work on
     Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space

From 376fc65d3e5a16e7d3b5dc8bf2dd35d72a28c2c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Mar 2011 21:23:35 -0400
Subject: [PATCH 1174/8313] add license text

---
 debian/copyright | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/debian/copyright b/debian/copyright
index 2144501e11..112ad54aaa 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -11,3 +11,29 @@ License: GPL-3+
 Files: StatFS.hsc
 Copyright: Jose A Ortega Ruiz 
 License: BSD-3-clause
+ -- All rights reserved.
+ -- 
+ -- Redistribution and use in source and binary forms, with or without
+ -- modification, are permitted provided that the following conditions
+ -- are met:
+ -- 
+ -- 1. Redistributions of source code must retain the above copyright
+ --    notice, this list of conditions and the following disclaimer.
+ -- 2. Redistributions in binary form must reproduce the above copyright
+ --    notice, this list of conditions and the following disclaimer in the
+ --    documentation and/or other materials provided with the distribution.
+ -- 3. Neither the name of the author nor the names of his contributors
+ --    may be used to endorse or promote products derived from this software
+ --    without specific prior written permission.
+ -- 
+ -- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ -- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ -- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ -- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ -- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ -- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ -- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ -- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ -- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ -- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ -- SUCH DAMAGE.

From 745f48297d46ada2242c8a698fa1e133d6d7d3f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 23 Mar 2011 02:07:49 +0000
Subject: [PATCH 1175/8313] Added a comment

---
 ...mment_3_558d80384434207b9cfc033763863de3._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
new file mode 100644
index 0000000000..9a128f1ed6
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-23T02:07:49Z"
+ content="""
+Ah yes, I feel kinda stupid in hindsight.
+
+As the central server is most likely a common use case, would you object if I added that to the walkthrough? If you have any best practices on how to automate a push with every copy to a bare remote? AFAIK, git does not store information about bare/non-bare remotes, but this could easily be put into .git/config by git annex.
+
+-- RichiH
+"""]]

From 1161fa8bdffbf2c83351024fdd653beefe050f82 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 01:06:14 -0400
Subject: [PATCH 1176/8313] add units to disk size check message

---
 Content.hs   |  6 ++---
 DataUnits.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 DataUnits.hs

diff --git a/Content.hs b/Content.hs
index 39a3addccf..2a3106c108 100644
--- a/Content.hs
+++ b/Content.hs
@@ -39,6 +39,7 @@ import qualified Annex
 import Utility
 import StatFS
 import Key
+import DataUnits
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
@@ -129,11 +130,10 @@ checkDiskSpace' adjustment key = do
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			if (need + reserve > have + adjustment)
 				then error $ "not enough free space (have " ++ 
-					showsize (have + adjustment) ++ "; need " ++
-					showsize (need + reserve) ++ ")"
+					roughSize True (have + adjustment) ++ "; need " ++
+					roughSize True (need + reserve) ++ ")"
 				else return ()
 	where
-		showsize i = show i
 		megabyte :: Integer
 		megabyte = 1024 * 1024
 
diff --git a/DataUnits.hs b/DataUnits.hs
new file mode 100644
index 0000000000..3404e0ab18
--- /dev/null
+++ b/DataUnits.hs
@@ -0,0 +1,66 @@
+{- data size display
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module DataUnits (roughSize) where
+
+{- And now a rant: 
+ -
+ - In the beginning, we had powers of two, and they were good.
+ -
+ - Disk drive manufacturers noticed that some powers of two were
+ - sorta close to some powers of ten, and that rounding down to the nearest
+ - power of ten allowed them to advertise their drives were bigger. This
+ - was sorta annoying.
+ -
+ - Then drives got big. Really, really big. This was good.
+ -
+ - Except that the small rounding error perpretrated by the drive
+ - manufacturers suffered the fate of a small error, and became a large
+ - error. This was bad.
+ -
+ - So, a committee was formed. And it arrived at a committee-like decision,
+ - which satisfied noone, confused everyone, and made the world an uglier
+ - place. As with all committees, this was meh.
+ -
+ - And the drive manufacturers happily continued selling drives that are
+ - increasingly smaller than you'd expect, if you don't count on your
+ - fingers. But that are increasingly bigger.
+ -
+ - Thus, I use units here that I loathe. Because if I didn't, people would
+ - be confused that their drives seem the wrong size, and other people would
+ - complain at me for not being standards compliant. And we call this
+ - progress?
+ -}
+
+{- approximate display of a particular number of bytes -}
+roughSize :: Bool -> Integer -> String
+roughSize short i
+	| i < 0 = "-" ++ roughSize short (negate i)
+	| i >= at 8 = units 8 "yottabyte" "YB"
+	| i >= at 7 = units 7 "zettabyte" "ZB"
+	| i >= at 6 = units 6 "exabyte" "EB"
+	| i >= at 5 = units 5 "petabyte" "PB"
+	| i >= at 4 = units 4 "terabyte" "TB"
+	| i >= at 3 = units 3 "gigabyte" "GB"
+	| i >= at 2 = units 2 "megabyte" "MB"
+	| i >= at 1 = units 1 "kilobyte" "kB"
+	| otherwise = units 0 "byte" "B"
+	where
+		at :: Integer -> Integer
+		at n = 1000^n
+
+		chop :: Integer -> Integer
+		chop d = round ((fromInteger i :: Double) / fromInteger (at d))
+
+		units :: Integer -> String -> String -> String
+		units d u u' = let num = chop d in
+			show num ++ " " ++
+			(if short then u' else plural num u)
+
+		plural n u
+			| n == 1 = u
+			| otherwise = u ++ "s"

From 12cdc96216e0c516ceeee922f9ca1568e9d4d592 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 01:13:13 -0400
Subject: [PATCH 1177/8313] tweak

---
 DataUnits.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index 3404e0ab18..329bf7e580 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -54,9 +54,8 @@ roughSize short i
 		at n = 1000^n
 
 		chop :: Integer -> Integer
-		chop d = round ((fromInteger i :: Double) / fromInteger (at d))
+		chop d = round $ (fromInteger i :: Double) / fromInteger (at d)
 
-		units :: Integer -> String -> String -> String
 		units d u u' = let num = chop d in
 			show num ++ " " ++
 			(if short then u' else plural num u)

From c43e3b5c787050664089fa4498e660a475addcd9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 02:10:59 -0400
Subject: [PATCH 1178/8313] check key size when available, no matter the
 backend

Now that SHA and other backends can have size info, fsck should check it
whenever available.
---
 Backend.hs      | 40 ++++++++++++++++++++++++++++++++++++++--
 Backend/File.hs |  5 ++++-
 Backend/WORM.hs | 27 +--------------------------
 3 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index cd14ce50e1..d7334f144e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -35,6 +35,7 @@ import Control.Monad.State
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
+import System.Directory
 
 import Locations
 import qualified GitRepo as Git
@@ -43,6 +44,8 @@ import Types
 import Key
 import qualified BackendClass as B
 import Messages
+import Content
+import DataUnits
 
 {- List of backends in the order to try them when storing a new key. -}
 list :: Annex [Backend Annex]
@@ -120,9 +123,12 @@ hasKey key = do
 	backend <- keyBackend key
 	(B.hasKey backend) key
 
-{- Checks a key's backend for problems. -}
+{- Checks a key for problems. -}
 fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-fsckKey backend key file numcopies = (B.fsckKey backend) key file numcopies
+fsckKey backend key file numcopies = do
+	size_ok <- checkKeySize key
+	backend_ok <-(B.fsckKey backend) key file numcopies
+	return $ size_ok && backend_ok
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
@@ -168,3 +174,33 @@ keyBackend :: Key -> Annex (Backend Annex)
 keyBackend key = do
 	bs <- Annex.getState Annex.supportedBackends
 	return $ lookupBackendName bs $ keyBackendName key
+
+{- The size of the data for a key is checked against the size encoded in
+ - the key's metadata, if available. -}
+checkKeySize :: Key -> Annex Bool
+checkKeySize key = do
+	g <- Annex.gitRepo
+	let file = gitAnnexLocation g key
+	present <- liftIO $ doesFileExist file
+	case (present, keySize key) of
+		(_, Nothing) -> return True
+		(False, _) -> return True
+		(True, Just size) -> do
+			stat <- liftIO $ getFileStatus file
+			let size' = fromIntegral (fileSize stat)
+			if size == size'
+				then return True
+				else do
+					dest <- moveBad key
+					warning $ badsizeNote dest size size'
+					return False
+
+badsizeNote :: FilePath -> Integer -> Integer -> String
+badsizeNote dest expected got = "Bad file size (" ++ aside ++ "); moved to " ++ dest
+	where
+		expected' = roughSize True expected
+		got' = roughSize True got
+		aside = 
+			if expected' == got'
+				then show expected ++ " not " ++ show got
+				else expected' ++ " not " ++ got'
diff --git a/Backend/File.hs b/Backend/File.hs
index a5e2431998..a6d42eabde 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -38,7 +38,7 @@ backend = Backend {
 	retrieveKeyFile = copyKeyFile,
 	removeKey = checkRemoveKey,
 	hasKey = inAnnex,
-	fsckKey = mustProvide
+	fsckKey = checkKeyOnly
 }
 
 mustProvide :: a
@@ -172,6 +172,9 @@ checkKey a key file numcopies = do
 	copies_ok <- checkKeyNumCopies key file numcopies
 	return $ a_ok && copies_ok
 
+checkKeyOnly :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyOnly = checkKey (\_ -> return True)
+
 checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index a011995da3..b33c607632 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -10,15 +10,9 @@ module Backend.WORM (backends) where
 import Control.Monad.State
 import System.FilePath
 import System.Posix.Files
-import System.Directory
-import Data.Maybe
 
 import qualified Backend.File
 import BackendClass
-import Locations
-import qualified Annex
-import Content
-import Messages
 import Types
 import Key
 
@@ -28,8 +22,7 @@ backends = [backend]
 backend :: Backend Annex
 backend = Backend.File.backend {
 	name = "WORM",
-	getKey = keyValue,
-	fsckKey = Backend.File.checkKey checkKeySize
+	getKey = keyValue
 }
 
 {- The key includes the file size, modification time, and the
@@ -48,21 +41,3 @@ keyValue file = do
 		keySize = Just $ fromIntegral $ fileSize stat,
 		keyMtime = Just $ modificationTime stat
 	}
-
-{- The size of the data for a key is checked against the size encoded in
- - the key's metadata. -}
-checkKeySize :: Key -> Annex Bool
-checkKeySize key = do
-	g <- Annex.gitRepo
-	let file = gitAnnexLocation g key
-	present <- liftIO $ doesFileExist file
-	if not present
-		then return True
-		else do
-			s <- liftIO $ getFileStatus file
-			if fromIntegral (fileSize s) == fromJust (keySize key)
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file size; moved to " ++ dest
-					return False

From cd1cb526522029cbc9c6c8fe396da1685bfb603b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 02:15:26 -0400
Subject: [PATCH 1179/8313] tweak

---
 Backend.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index d7334f144e..fe61739e83 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -202,5 +202,5 @@ badsizeNote dest expected got = "Bad file size (" ++ aside ++ "); moved to " ++
 		got' = roughSize True got
 		aside = 
 			if expected' == got'
-				then show expected ++ " not " ++ show got
-				else expected' ++ " not " ++ got'
+				then show got ++ " not " ++ show expected
+				else got' ++ " not " ++ expected'

From 04539d16718265441c607da08d3e27d959f749c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 02:42:14 -0400
Subject: [PATCH 1180/8313] improve size change display

---
 Backend.hs   | 14 +++-----------
 DataUnits.hs |  8 +++++++-
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index fe61739e83..0ee56d2623 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -192,15 +192,7 @@ checkKeySize key = do
 				then return True
 				else do
 					dest <- moveBad key
-					warning $ badsizeNote dest size size'
+					warning $ "Bad file size (" ++
+						compareSizes True size size' ++ 
+						"); moved to " ++ dest
 					return False
-
-badsizeNote :: FilePath -> Integer -> Integer -> String
-badsizeNote dest expected got = "Bad file size (" ++ aside ++ "); moved to " ++ dest
-	where
-		expected' = roughSize True expected
-		got' = roughSize True got
-		aside = 
-			if expected' == got'
-				then show got ++ " not " ++ show expected
-				else got' ++ " not " ++ expected'
diff --git a/DataUnits.hs b/DataUnits.hs
index 329bf7e580..c2845affea 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module DataUnits (roughSize) where
+module DataUnits (roughSize, compareSizes) where
 
 {- And now a rant: 
  -
@@ -63,3 +63,9 @@ roughSize short i
 		plural n u
 			| n == 1 = u
 			| otherwise = u ++ "s"
+
+compareSizes :: Bool -> Integer -> Integer -> String
+compareSizes short old new
+	| old > new = roughSize short (old - new) ++ " smaller"
+	| old < new = roughSize short (new - old) ++ " larger"
+	| otherwise = "same"

From ba064654a3707ac9407cf17ea3c0a7e7fb573382 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 07:27:14 +0000
Subject: [PATCH 1181/8313]

---
 doc/bugs/Makefile_is_missing_dependancies.mdwn | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/Makefile_is_missing_dependancies.mdwn b/doc/bugs/Makefile_is_missing_dependancies.mdwn
index 4bda91f17d..3e9d6e903c 100644
--- a/doc/bugs/Makefile_is_missing_dependancies.mdwn
+++ b/doc/bugs/Makefile_is_missing_dependancies.mdwn
@@ -1,4 +1,3 @@
-
 
 From e45c73e66fc18d27bdf5797876fbeb07786a4af1 Mon Sep 17 00:00:00 2001
 From: Jimmy Tang 
@@ -30,5 +29,19 @@ index 08e2f59..4ae8392 100644
 
 StatFS.hs never gets depended on and compiled, the makefile was just missing something
 
-> Thanks, [done]]! Interested to hear if StatFS.hs works on OSX (no warning) or
+> Thanks, [[done]]! Interested to hear if StatFS.hs works on OSX (no warning) or
 > is a no-op (with warning). --[[Joey]] 
+
+>> 
+>> for now it gives a warning, it looks like it should be easy enough to add OSX
+>> support, I guess it's a case of just digging around documentation to find the equivalent
+>> calls/headers. I'll give it a go at making this feature work on OSX and get back to you.
+>> 
+
+
+jtang@exia:~/develop/git-annex $ make
+hsc2hs StatFS.hsc
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+StatFS.hsc:85:2: warning: #warning free space checking code not available for this OS
+


From 847629ee69e891185484704bd20b137c7ad06a25 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 08:21:32 +0000
Subject: [PATCH 1182/8313] Added a comment

---
 ..._5a3da5f79c8563c7a450aa29728abe7c._comment | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment
new file mode 100644
index 0000000000..ab8493a7a8
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment
@@ -0,0 +1,47 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-03-23T08:21:30Z"
+ content="""
+Just did some minor digging around and checking, this seems to satisfy the compilers etc... I have yet to confirm that it *really* is working as expected. Also it might be better to check for a darwin operating system instead of apple I think, though I don't know of any one really using a pure darwin OS. But for now it works (I think)
+
+
+From fbfe27c2e19906ac02e3673b91bffa920f6dae5d Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Wed, 23 Mar 2011 08:15:39 +0000
+Subject: [PATCH] Define (__APPLE__) in StatFS
+
+At least on OSX 10.6.6 it appears to have the same defintions as
+FreeBSD. The build process doesn't complain and the code is enabled,
+this needs to be tested and checked more.
+---
+ StatFS.hsc |    4 ++--
+ 1 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/StatFS.hsc b/StatFS.hsc
+index 8b453dc..45fd7e4 100644
+--- a/StatFS.hsc
++++ b/StatFS.hsc
+@@ -53,7 +53,7 @@ import Foreign.C.String
+ import Data.ByteString (useAsCString)
+ import Data.ByteString.Char8 (pack)
+ 
+-#if defined (__FreeBSD__)
++#if defined (__FreeBSD__) || defined(__APPLE__)
+ # include 
+ # include 
+ #else
+@@ -84,7 +84,7 @@ data CStatfs
+ #ifdef UNKNOWN
+ #warning free space checking code not available for this OS
+ #else
+-#if defined(__FreeBSD__)
++#if defined(__FreeBSD__) || defined(__APPLE__)
+ foreign import ccall unsafe \"sys/mount.h statfs\"
+ #else
+ foreign import ccall unsafe \"sys/vfs.h statfs64\"
+-- 
+1.7.4.1
+

+"""]]

From 002db413a53ec83c93f389b09fe9926cebfaa0ec Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 23 Mar 2011 09:53:16 +0000
Subject: [PATCH 1183/8313]

---
 doc/bugs/softlink_atime.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_atime.mdwn
index c62610be02..fe09ade38b 100644
--- a/doc/bugs/softlink_atime.mdwn
+++ b/doc/bugs/softlink_atime.mdwn
@@ -29,7 +29,7 @@ Optionally, editing the meta-data should change the times in all annexes.
 >>>>> 
 >>>>> So all you have to do is make the pre-commit hook call
 >>>>> [metastore](http://david.hardeman.nu/software.php). The hook
->>>>> would look like this: ---[[Joey]] [[!tag done]]
+>>>>> would look like this: ---[[Joey]]
 
 	#!/bin/sh
 	git annex pre-commit .
@@ -47,3 +47,5 @@ Optionally, editing the meta-data should change the times in all annexes.
     git add .metadata
 
 >>>>>>> -- RichiH
+
+>>>>>>>> After getting to actually play with this from different machines with a bare git as central instance for several distributed repos, the metastore trick does not work. The .metadata is causing merge conflicts for every pull. I removed the "done" tag from this issue. -- RichiH

From 6d857b540aab2e80394984572ebd26be31ececff Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 23 Mar 2011 11:31:06 +0000
Subject: [PATCH 1184/8313] Added a comment: Why isn't this package built with
 Cabal?

---
 .../comment_1_d9f7b851567445c7aa7ebbb440781819._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment

diff --git a/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment b/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
new file mode 100644
index 0000000000..616b3c4dd5
--- /dev/null
+++ b/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="84.189.1.247"
+ subject="Why isn't this package built with Cabal?"
+ date="2011-03-23T11:31:06Z"
+ content="""
+It would be a lot easier to compile this package, if it had a Cabal file to describe the build; especially the build-time dependencies. Why isn't Cabal used?
+"""]]

From b0eccb003ae70792e89eda54d0c0bea1f38847ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 10:52:40 -0400
Subject: [PATCH 1185/8313] improve free space needed display

---
 Content.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Content.hs b/Content.hs
index 2a3106c108..bc43e4b443 100644
--- a/Content.hs
+++ b/Content.hs
@@ -129,9 +129,9 @@ checkDiskSpace' adjustment key = do
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			if (need + reserve > have + adjustment)
-				then error $ "not enough free space (have " ++ 
-					roughSize True (have + adjustment) ++ "; need " ++
-					roughSize True (need + reserve) ++ ")"
+				then error $ "not enough free space, need " ++ 
+					roughSize True (need + reserve - have - adjustment) ++
+					" more"
 				else return ()
 	where
 		megabyte :: Integer

From 5b4ae90cdb27b0024f5caf129ffa6f05488869a4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:05:12 +0000
Subject: [PATCH 1186/8313] Added a comment

---
 .../comment_2_416f12dbd0c2b841fac8164645b81df5._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment
new file mode 100644
index 0000000000..d355514a31
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T15:05:12Z"
+ content="""
+There's a simple test -- just configure annex.diskreserve to be say, 10 megabytes less than the total free space on your disk. Then try to git annex get a 11 mb file, and a 9 mb file. :)
+"""]]

From 4440ecf4a74b85341d5ecc1ecb1a9349b6fc5d3b Mon Sep 17 00:00:00 2001
From: Jimmy Tang 
Date: Wed, 23 Mar 2011 08:15:39 +0000
Subject: [PATCH 1187/8313] Define (__APPLE__) in StatFS

At least on OSX 10.6.6 it appears to have the same defintions as
FreeBSD. The build process doesn't complain and the code is enabled,
this needs to be tested and checked more.
---
 StatFS.hsc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 8b453dc199..45fd7e4db9 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__)
+#if defined (__FreeBSD__) || defined(__APPLE__)
 # include 
 # include 
 #else
@@ -84,7 +84,7 @@ data CStatfs
 #ifdef UNKNOWN
 #warning free space checking code not available for this OS
 #else
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined(__APPLE__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"

From cf70075c4be00781e87b5b9a4774c2bdb95d949e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:13:33 +0000
Subject: [PATCH 1188/8313] Added a comment

---
 ..._c38b6f4abc9b9ad413c3b83ca04386c3._comment | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment
new file mode 100644
index 0000000000..6b4cf5789c
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-23T15:13:33Z"
+ content="""
+Alternatively, you can just load it up in ghci and see if it reports numbers that make sense:
+
+
+joey@gnu:~/src/git-annex>make StatFS.hs
+hsc2hs StatFS.hsc
+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs
+joey@gnu:~/src/git-annex>ghci StatFS.hs
+GHCi, version 6.12.1: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.5 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 7427989, fsStatByteCount = 30425042944, fsStatBytesFree = 2528489472, fsStatBytesAvailable = 2219384832, fsStatBytesUsed = 27896553472})
+

+"""]]

From 7808de3d471dffff3ee2360f3bbf1627c492cb0f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:18:29 +0000
Subject: [PATCH 1189/8313] Added a comment

---
 ...mment_2_cf0f829536744098d6846500db998b6a._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment

diff --git a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
new file mode 100644
index 0000000000..134024908e
--- /dev/null
+++ b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-23T15:18:29Z"
+ content="""
+Because I haven't learned Cabal yet.
+
+But also because I've had bad experiences with both a) tying a particular program to a particular language's pet build system and then having to add ugliness when I later need to do something in the build that has nothing to do with that language and b) as a user, needing to deal with the pet build systems of languages when I just need to make some small change to the build process that is trivial in a Makefile.
+
+With that said, I do have a configure program written in Haskell, so at least it doesn't use autotools. :)
+"""]]

From d0a7513bbbe8b13c99ed3af986aef82e5d0f0248 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:28:00 +0000
Subject: [PATCH 1190/8313] Added a comment

---
 .../comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment

diff --git a/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
new file mode 100644
index 0000000000..8b4d9a0538
--- /dev/null
+++ b/doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-03-23T15:28:00Z"
+ content="""
+I would not mind if the walkthrough documented the central git repo case. But I don't want to complicate it unduely (it's long enough), and it's important that the fully distributed case be shown to work, and I assume that people already have basic git knowledge, so documenting the details of set up of a bare git repo is sorta out of scope. (There are also a lot of way to do it, using github, or gitosis, or raw git, etc.)
+"""]]

From da198e87fb87430a8d5325d3f793ceeb8f746677 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 15:31:54 +0000
Subject: [PATCH 1191/8313] bit about bare repos

---
 doc/walkthrough/getting_file_content.mdwn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index a863303cef..5c899ee3c5 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -13,4 +13,7 @@ USB drive.
 
 Notice that you had to git pull from laptop first, this lets git-annex know
 what has changed in laptop, and so it knows about the files present there and
-can get them.
+can get them. The alternate approach is to set up a central bare repository,
+and always push changes to it after committing them, then in the above,
+you can just pull from the central repository to get synced up to all
+repositories.

From 328b023c5381513cb0b9fb00f832cb8251799b4a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 16:02:35 +0000
Subject: [PATCH 1192/8313] Added a comment

---
 ..._cc13873175edf191047282700315beee._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment
new file mode 100644
index 0000000000..c3ad2dafd6
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2011-03-23T16:02:34Z"
+ content="""
+Ok, well it looks like it isn't doing anything useful at all.
+
+
+jtang@x00:~/develop/git-annex $ make StatFS.hs                                                                                                                                    
+hsc2hs StatFS.hsc
+perl -i -pe 's/^{-# INCLUDE.*//' StatFS.hs
+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    
+GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+Loading package ffi-1.0 ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.7 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})
+*StatFS> s <- getFileSystemStats \"/\"
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 0, fsStatBlockCount = 1048576, fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0, fsStatBytesUsed = 0})
+*StatFS> 
+

+"""]]

From 48a8f811fa85f94912bd5009728d054cf6e8ac48 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 16:14:22 +0000
Subject: [PATCH 1193/8313] Added a comment

---
 ..._0a1c52e2c96d19b9c3eb7e99b8c2434f._comment | 59 +++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment
new file mode 100644
index 0000000000..149aeeb75a
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment
@@ -0,0 +1,59 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2011-03-23T16:14:22Z"
+ content="""
+Actually I may have just been stupid and should have read the man page on statfs...
+
+
+jtang@x00:~/develop/git-annex $ git diff
+diff --git a/StatFS.hsc b/StatFS.hsc
+index 8b453dc..e10b2dd 100644
+--- a/StatFS.hsc
++++ b/StatFS.hsc
+@@ -53,7 +53,7 @@ import Foreign.C.String
+ import Data.ByteString (useAsCString)
+ import Data.ByteString.Char8 (pack)
+ 
+-#if defined (__FreeBSD__)
++#if defined (__FreeBSD__) || defined (__APPLE__)
+ # include 
+ # include 
+ #else
+@@ -84,8 +84,8 @@ data CStatfs
+ #ifdef UNKNOWN
+ #warning free space checking code not available for this OS
+ #else
+-#if defined(__FreeBSD__)
+-foreign import ccall unsafe \"sys/mount.h statfs\"
++#if defined(__FreeBSD__) || defined (__APPLE__)
++foreign import ccall unsafe \"sys/mount.h statfs64\"
+ #else
+ foreign import ccall unsafe \"sys/vfs.h statfs64\"
+ #endif
+

+
+yields this...
+
+
+jtang@x00:~/develop/git-annex $ ghci StatFS.hs                                                                                                                                    
+GHCi, version 6.12.3: http://www.haskell.org/ghc/  :? for help
+Loading package ghc-prim ... linking ... done.
+Loading package integer-gmp ... linking ... done.
+Loading package base ... linking ... done.
+Loading package ffi-1.0 ... linking ... done.
+[1 of 1] Compiling StatFS           ( StatFS.hs, interpreted )
+Ok, modules loaded: StatFS.
+*StatFS> s <- getFileSystemStats \".\"
+Loading package bytestring-0.9.1.7 ... linking ... done.
+*StatFS> s
+Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 244106668, fsStatByteCount = 999860912128, fsStatBytesFree = 423097798656, fsStatBytesAvailable = 422835654656, fsStatBytesUsed = 576763113472})
+*StatFS> 
+

+
+
+we could just stick another if defined (__APPLE__) instead of what I previously had and it looks like it will do the right thing on OSX.
+
+
+"""]]

From 0dac2b4f6ce0a616711a24e964c138055607e35e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:23:13 -0400
Subject: [PATCH 1194/8313] Revert "Define (__APPLE__) in StatFS"

This reverts commit 4440ecf4a74b85341d5ecc1ecb1a9349b6fc5d3b.

Turns out that it is reporting a block size of 0 and so all bogus.
---
 StatFS.hsc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 45fd7e4db9..8b453dc199 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__) || defined(__APPLE__)
+#if defined (__FreeBSD__)
 # include 
 # include 
 #else
@@ -84,7 +84,7 @@ data CStatfs
 #ifdef UNKNOWN
 #warning free space checking code not available for this OS
 #else
-#if defined(__FreeBSD__) || defined(__APPLE__)
+#if defined(__FreeBSD__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"

From 7ffc3a72ae05dde951cee7bdf972545ac461a184 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 16:23:57 +0000
Subject: [PATCH 1195/8313] Added a comment

---
 ...comment_6_24119fc5d5963ce9dd669f7dcf006859._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment
new file mode 100644
index 0000000000..714459fbe8
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2011-03-23T16:23:56Z"
+ content="""
+I forgot to mention that the statfs64 stuff in OSX seems to be deprecated, see http://developer.apple.com/library/mac/#documentation/Darwin/Reference/ManPages/man2/statfs64.2.html
+
+on a slightly different note, is anonymous pushing to the \"wiki\" over git allowed? I'd prefer to be able to edit stuff inline for updating some of my own comments if I can :P
+"""]]

From 68d90b3328f7b8e7c9aa0bd997bef20724d7cafc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:45:34 -0400
Subject: [PATCH 1196/8313] allow force overriding the disk space check

---
 Content.hs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Content.hs b/Content.hs
index bc43e4b443..633d827e6f 100644
--- a/Content.hs
+++ b/Content.hs
@@ -129,13 +129,17 @@ checkDiskSpace' adjustment key = do
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			if (need + reserve > have + adjustment)
-				then error $ "not enough free space, need " ++ 
-					roughSize True (need + reserve - have - adjustment) ++
-					" more"
+				then needmorespace (need + reserve - have - adjustment)
 				else return ()
 	where
 		megabyte :: Integer
 		megabyte = 1024 * 1024
+		needmorespace n = do
+			force <- Annex.getState Annex.force
+			unless force $ 
+				error $ "not enough free space, need " ++ 
+					roughSize True n ++
+					" more (use --force to override this check or adjust annex.diskreserve)"
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()

From 7400c8318aea1cc5f43352ae97454fd34e1126b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:46:51 -0400
Subject: [PATCH 1197/8313] correct

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index f124296cc8..eef1d7e29f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
     with git-annex 0.2x or earlier.) The code is believed to work on
-    Linux, FreeBSD, and OSX; check compile-time messages to see if it
+    Linux and FreeBSD; check compile-time messages to see if it
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space
     to reserve for other purposes and avoid using (defaults to 1 mb).

From b9c4c006bdc8f0317d0598f05b45e4fdc1102a6c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 12:53:57 -0400
Subject: [PATCH 1198/8313] use statfs64 on apple, while retaining statfs on
 freebsd

http://git-annex.branchable.com/bugs/Makefile_is_missing_dependancies/#comment-3196b43b7d745ab206435d0a69686815
indicates statfs64 works on apple. Probably on freebsd too, but I
have not tested it and so will stick with the old code there.
---
 StatFS.hsc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 8b453dc199..dd0c90bebd 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__)
+#if defined (__FreeBSD__) || defined (__APPLE__)
 # include 
 # include 
 #else
@@ -84,10 +84,14 @@ data CStatfs
 #ifdef UNKNOWN
 #warning free space checking code not available for this OS
 #else
+#if defined(__APPLE__)
+foreign import ccall unsafe "sys/mount.h statfs64"
+#else
 #if defined(__FreeBSD__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"
+#endif
 #endif
   c_statfs :: CString -> Ptr CStatfs -> IO CInt
 #endif

From 4727ebab53d52e5126238e64ca03d03cf309d599 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 23 Mar 2011 16:57:56 +0000
Subject: [PATCH 1199/8313] Added a comment

---
 ...mment_7_96fd4725df4b54e670077a18d3ac4943._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment
new file mode 100644
index 0000000000..8ba8e8d1f6
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-23T16:57:56Z"
+ content="""
+Try the changes I've pushed to use statfs64 on apple.
+
+There is actually a standardized statvfs that I'd rather use, but after the last time that I tried going with the POSIX option first only to find it was not broadly implemented, I was happy to find some already existing code that worked for some OSs.
+
+(While ikiwiki supports anonymous git push, it's a feature we have not rolled out on Branchable.com yet, and anyway, ikiwiki disallows editing existing comments that way. I would, however, be happy to git pull changes from somewhere.)
+"""]]

From 443bf2ade72a2117fc5ca77f7b9ee2dda2bb4c45 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 23 Mar 2011 17:03:53 +0000
Subject: [PATCH 1200/8313] Added a comment

---
 .../comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment

diff --git a/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment b/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment
new file mode 100644
index 0000000000..63d188bcce
--- /dev/null
+++ b/doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-03-23T17:03:51Z"
+ content="""
+The latest change looks good, it seems to be returning sensible numbers for me. Just tried it out on a few different mount points and it appears to be working.
+"""]]

From 0cd70cb5c0c33ce5db2a68fb0c76341f9d71f400 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 13:10:20 -0400
Subject: [PATCH 1201/8313] kFreeBSD support

Tested on Debian kfreebsd-amd64. The BSD #includes worked. Both statfs64
and statfs worked. Using statfs to keep the same as general freebsd, and
because I didn't try it on 32 bit.
---
 StatFS.hsc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index dd0c90bebd..0828f1378c 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -53,7 +53,7 @@ import Foreign.C.String
 import Data.ByteString (useAsCString)
 import Data.ByteString.Char8 (pack)
 
-#if defined (__FreeBSD__) || defined (__APPLE__)
+#if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
 # include 
 # include 
 #else
@@ -87,7 +87,7 @@ data CStatfs
 #if defined(__APPLE__)
 foreign import ccall unsafe "sys/mount.h statfs64"
 #else
-#if defined(__FreeBSD__)
+#if defined(__FreeBSD__) || defined (__FreeBSD_kernel__)
 foreign import ccall unsafe "sys/mount.h statfs"
 #else
 foreign import ccall unsafe "sys/vfs.h statfs64"

From af45a62980b6b231225e4456e602eec4dfc4e04b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 13:13:51 -0400
Subject: [PATCH 1202/8313] update

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index eef1d7e29f..f124296cc8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
     with git-annex 0.2x or earlier.) The code is believed to work on
-    Linux and FreeBSD; check compile-time messages to see if it
+    Linux, FreeBSD, and OSX; check compile-time messages to see if it
     is not enabled for your OS.
   * Add annex.diskreserve config setting, to control how much free space
     to reserve for other purposes and avoid using (defaults to 1 mb).

From 8beb72e20604b0d2cc6359e75ed4eac45fcd1081 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 17:25:28 -0400
Subject: [PATCH 1203/8313] migrate: Bugfix for case when migrating a file
 results in a key that is already present in .git/annex/objects.

For example, this could happen if using SHA1 and a file with content
"foo" were added to that backend. Then a file with "content" foo were
migrated from the WORM backend.

Assume that, if a backend assigned the same key, the already annexed
content must be the same. So, the "old" content can be reused.
---
 Command/Migrate.hs | 4 +++-
 debian/changelog   | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 56147113b9..38dfd06b23 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,6 +8,7 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (unless)
 import System.Posix.Files
 import System.Directory
 
@@ -58,7 +59,8 @@ perform file oldkey newbackend = do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
-				liftIO $ createLink src t
+				exists <- liftIO $ doesFileExist t
+				unless exists $ liftIO $ createLink src t
 				return True
 			if ok
 				then do
diff --git a/debian/changelog b/debian/changelog
index f124296cc8..9a3ffbafaf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
   * Fix space leak in fsck and drop commands.
+  * migrate: Bugfix for case when migrating a file results in a key that
+    is already present in .git/annex/objects.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 

From ad08273ac5118f1faac539b53f1fa63908dc5656 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 17:29:54 -0400
Subject: [PATCH 1204/8313] refactor

---
 Command/Migrate.hs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 38dfd06b23..a7014b9bcf 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,7 +31,8 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	if (newbackend /= oldbackend) && exists
+	force <- Annex.getState Annex.force
+	if (newbackend /= oldbackend || force) && exists
 		then do
 			showStart "migrate" file
 			return $ Just $ perform file key newbackend
@@ -59,8 +60,9 @@ perform file oldkey newbackend = do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
-				exists <- liftIO $ doesFileExist t
-				unless exists $ liftIO $ createLink src t
+				liftIO $ do
+					exists <- doesFileExist t
+					unless exists $ createLink src t
 				return True
 			if ok
 				then do

From 6246b807f7df32877a87d906cfbe1ae26c51dd8e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 17:57:10 -0400
Subject: [PATCH 1205/8313] migrate: Support migrating v1 SHA keys to v2 SHA
 keys with size information that can be used for free space checking.

---
 Backend.hs                 |  5 +++++
 Backend/File.hs            | 10 +++++++++-
 Backend/URL.hs             |  4 +++-
 BackendClass.hs            |  4 +++-
 Command/Migrate.hs         |  4 ++--
 debian/changelog           |  2 ++
 doc/git-annex.mdwn         |  9 +++++++--
 doc/upgrades.mdwn          |  6 +++---
 doc/upgrades/SHA_size.mdwn | 20 ++++++++++++++++++++
 9 files changed, 54 insertions(+), 10 deletions(-)
 create mode 100644 doc/upgrades/SHA_size.mdwn

diff --git a/Backend.hs b/Backend.hs
index 0ee56d2623..4b0095214c 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -24,6 +24,7 @@ module Backend (
 	removeKey,
 	hasKey,
 	fsckKey,
+	upgradableKey,
 	lookupFile,
 	chooseBackends,
 	keyBackend,
@@ -130,6 +131,10 @@ fsckKey backend key file numcopies = do
 	backend_ok <-(B.fsckKey backend) key file numcopies
 	return $ size_ok && backend_ok
 
+{- Checks if a key is upgradable to a newer representation. -}
+upgradableKey :: Backend Annex -> Key -> Annex Bool
+upgradableKey backend key = (B.upgradableKey backend) key
+
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))
diff --git a/Backend/File.hs b/Backend/File.hs
index a6d42eabde..fb8a052553 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -29,6 +29,7 @@ import Types
 import UUID
 import Messages
 import Trust
+import Key
 
 backend :: Backend Annex
 backend = Backend {
@@ -38,7 +39,8 @@ backend = Backend {
 	retrieveKeyFile = copyKeyFile,
 	removeKey = checkRemoveKey,
 	hasKey = inAnnex,
-	fsckKey = checkKeyOnly
+	fsckKey = checkKeyOnly,
+	upgradableKey = checkUpgradableKey
 }
 
 mustProvide :: a
@@ -159,6 +161,12 @@ getNumCopies Nothing = do
 	where
 		config = "annex.numcopies"
 
+{- Ideally, all keys have file size metadata. Old keys may not. -}
+checkUpgradableKey :: Key -> Annex Bool
+checkUpgradableKey key
+	| keySize key == Nothing = return True
+	| otherwise = return False
+
 {- This is used to check that numcopies is satisfied for the key on fsck.
  - This trusts data in the the location log, and so can check all keys, even
  - those with data not present in the current annex.
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 210c7c5b48..3068c30270 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -30,7 +30,9 @@ backend = Backend {
 	-- similarly, keys are always assumed to be out there on the web
 	hasKey = dummyOk,
 	-- and nothing needed to fsck
-	fsckKey = dummyFsck
+	fsckKey = dummyFsck,
+	-- and key upgrade not needed
+	upgradableKey = \_ -> return False
 }
 
 -- cannot generate url from filename
diff --git a/BackendClass.hs b/BackendClass.hs
index 909ae8f96e..b2d8879c2f 100644
--- a/BackendClass.hs
+++ b/BackendClass.hs
@@ -29,7 +29,9 @@ data Backend a = Backend {
 	-- (second parameter may be the filename associated with it)
 	-- (third parameter may be the number of copies that there should
 	-- be of the key)
-	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool
+	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool,
+	-- Is a newer repesentation possible for a key?
+	upgradableKey :: Key -> a Bool
 }
 
 instance Show (Backend a) where
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a7014b9bcf..0d21fcbdf9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,8 +31,8 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	force <- Annex.getState Annex.force
-	if (newbackend /= oldbackend || force) && exists
+	upgradable <- Backend.upgradableKey oldbackend key
+	if (newbackend /= oldbackend || upgradable) && exists
 		then do
 			showStart "migrate" file
 			return $ Just $ perform file key newbackend
diff --git a/debian/changelog b/debian/changelog
index 9a3ffbafaf..7bb5bd8d21 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,8 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * fsck: In fast mode, avoid checking checksums.
   * unused: In fast mode, just show all existing temp files as unused,
     and avoid expensive scan for other unused content.
+  * migrate: Support migrating v1 SHA keys to v2 SHA keys with
+    size information that can be used for free space checking.
   * Fix space leak in fsck and drop commands.
   * migrate: Bugfix for case when migrating a file results in a key that
     is already present in .git/annex/objects.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d7b57675d0..81cea04cd7 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -162,10 +162,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * migrate [path ...]
 
   Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend).
+  default backend (or the one specified with --backend). Only files whose
+  content is currently available are migrated.
 
   Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content. 
+  Use `git annex unused` to find and remove such content.
+
+  Normally, nothing will be done to files already in the backend.
+  However, if a backend changes the information it uses to construct a key,
+  this can also be used to migrate files to use the new key format.
 
 * map
 
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index dbea5e9c4a..0a07ef7aa8 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -56,9 +56,9 @@ And .gitattributes needed to have another line added to it.
 Previously, files added to the SHA [[backends]] did not have their file
 size tracked, while files added to the WORM backend did. Files added to
 the SHA backends after the conversion will have their file size tracked,
-and that information will be used by git-annex for disk space checking.
-There is not yet a way to add file size tracking information to old files
-in the SHA backend.
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
 
 ### v0 -> v1 (git-annex version 0.03 to version 0.04)
 
diff --git a/doc/upgrades/SHA_size.mdwn b/doc/upgrades/SHA_size.mdwn
new file mode 100644
index 0000000000..319b91108a
--- /dev/null
+++ b/doc/upgrades/SHA_size.mdwn
@@ -0,0 +1,20 @@
+Before version 2 of the git-annex repository, files added to the SHA
+[[backends]] did not have their file size tracked, while files added to the
+WORM backend did. The file size information is used for disk free space
+checking.
+
+Files added to the SHA backends after the conversion will have their file
+size tracked automatically. This disk free space checking is an optional
+feature and since you're more likely to be using more recently added files,
+you're unlikely to see any bad effect if you do nothing.
+
+That said, if you have old files added to SHA backends that lack file size
+tracking info, here's how you can add that info. After [[upgrading|upgrades]]
+to repository version 2, in each repository run:
+
+	git annex migrate
+	git commit -m 'migrated keys for v2'
+
+The usual caveats about [[walkthrough/migrating_data_to_a_new_backend]]
+apply; you will end up with unused keys that you can later clean up with
+`git annex unused`.

From 03fdd0d56e554c65946b9eadd32c5be5d6d0c806 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 Mar 2011 23:47:02 -0400
Subject: [PATCH 1206/8313] dropunused: Significantly sped up; only read unused
 log file once.

---
 Command/DropUnused.hs | 12 ++++++++----
 debian/changelog      |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index c6a28663ec..932a8b8635 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -27,12 +27,16 @@ command = [repoCommand "dropunused" (paramRepeating paramNumber) seek
 	"drop unused file content"]
 
 seek :: [CommandSeek]
-seek = [withStrings start]
+seek = [withUnusedMap]
 
-{- Drops unused content by number. -} 
-start :: CommandStartString
-start s = notBareRepo $ do
+{- Read unusedlog once, and pass the map to each start action. -}
+withUnusedMap :: CommandSeek
+withUnusedMap params = do
 	m <- readUnusedLog
+	return $ map (start m) params
+
+start :: M.Map String Key -> CommandStartString
+start m s = notBareRepo $ do
 	case M.lookup s m of
 		Nothing -> return Nothing
 		Just key -> do
diff --git a/debian/changelog b/debian/changelog
index 7bb5bd8d21..9235c3ca33 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
   * Fix space leak in fsck and drop commands.
   * migrate: Bugfix for case when migrating a file results in a key that
     is already present in .git/annex/objects.
+  * dropunused: Significantly sped up; only read unused log file once.
 
  -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
 

From 31bf31583be78b589c4b78a0dc8705f54fd9946e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 Mar 2011 14:32:22 +0000
Subject: [PATCH 1207/8313] Did a fresh install of haskell-platform on a rhel5
 32bit host and iinstalling HP, pcre-light, missingh wasnt enough to build
 git-annex

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index a531015782..0501663231 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -12,6 +12,7 @@ To build and use git-annex, you will need:
 * The Haskell Platform: 
 * MissingH: 
 * pcre-light: 
+* utf8-string: 
 * `uuid`: 
   (or uuidgen from util-linux)
 * `xargs`: 

From e691727f24e8d42f2bd402f08e89d5f29af96cb0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 Mar 2011 16:25:44 +0000
Subject: [PATCH 1208/8313]

---
 doc/forum/Behaviour_of_fsck.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck.mdwn

diff --git a/doc/forum/Behaviour_of_fsck.mdwn b/doc/forum/Behaviour_of_fsck.mdwn
new file mode 100644
index 0000000000..29aa3ec645
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck.mdwn
@@ -0,0 +1,13 @@
+The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
+
+Would it be better for the default behaviour of fsck when it has not been given a path to only output errors and not bother to show that a file is ok for every single file in a repo. i.e.
+
+    git annex fsck
+
+should show only 'errors' and maybe a simple indicator showing the status (show a spinner or dots?) and when 
+
+    git annex fsck PATH/FILE 
+
+it should have the current behaviour? 
+
+Right now the current fsck behaviour might get annoying for anyone who would want to run fsck with repos with lots of big files.

From 0ed90ae99246c44a36c8956124c1c645cb845809 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 24 Mar 2011 17:45:08 +0000
Subject: [PATCH 1209/8313] Added a comment

---
 .../comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment b/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment
new file mode 100644
index 0000000000..dc48e2f943
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-24T17:45:08Z"
+ content="""
+I tend to agree that the default output of fsck is not quite right. I often use git annex fsck -q. A progress spinner display is a good idea.
+"""]]

From bc5f5428d39459228f61ec2e164bcbd00e110593 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 Mar 2011 18:38:38 +0000
Subject: [PATCH 1210/8313]

---
 doc/forum/Behaviour_of_fsck.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/Behaviour_of_fsck.mdwn b/doc/forum/Behaviour_of_fsck.mdwn
index 29aa3ec645..cd27d49f76 100644
--- a/doc/forum/Behaviour_of_fsck.mdwn
+++ b/doc/forum/Behaviour_of_fsck.mdwn
@@ -1,4 +1,4 @@
-The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
+The current behaviour of 'fsck' is a bit verbose. I have an annex'd directory of tarballs for my own build system for "science" applications, there's about ~600 or so blobs in my repo, I do occassionally like to run fsck across all my data to see what files don't meet the min num copies requirement that I have set.
 
 Would it be better for the default behaviour of fsck when it has not been given a path to only output errors and not bother to show that a file is ok for every single file in a repo. i.e.
 

From bc80ace96b2251262d1836c312f3924fe32e7cac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 Mar 2011 00:51:12 -0400
Subject: [PATCH 1211/8313] releasing version 0.20110325

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 9235c3ca33..c664566a9e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110321) UNRELEASED; urgency=low
+git-annex (0.20110325) experimental; urgency=low
 
   * Free space checking is now done, for transfers of data for keys
     that have free space metadata. (Notably, not for SHA* keys generated
@@ -19,7 +19,7 @@ git-annex (0.20110321) UNRELEASED; urgency=low
     is already present in .git/annex/objects.
   * dropunused: Significantly sped up; only read unused log file once.
 
- -- Joey Hess   Tue, 22 Mar 2011 16:52:00 -0400
+ -- Joey Hess   Fri, 25 Mar 2011 00:47:37 -0400
 
 git-annex (0.20110320) experimental; urgency=low
 

From 25f842f58fbdf7e52b93e6e0a79cf3fed77dc78b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 Mar 2011 00:52:46 -0400
Subject: [PATCH 1212/8313] add news item for git-annex 0.20110325

---
 doc/news/version_0.20110325.mdwn | 20 ++++++++++++++++++++
 doc/news/version_0.23.mdwn       |  7 -------
 2 files changed, 20 insertions(+), 7 deletions(-)
 create mode 100644 doc/news/version_0.20110325.mdwn
 delete mode 100644 doc/news/version_0.23.mdwn

diff --git a/doc/news/version_0.20110325.mdwn b/doc/news/version_0.20110325.mdwn
new file mode 100644
index 0000000000..d5a59afe65
--- /dev/null
+++ b/doc/news/version_0.20110325.mdwn
@@ -0,0 +1,20 @@
+git-annex 0.20110325 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Free space checking is now done, for transfers of data for keys
+     that have free space metadata. (Notably, not for SHA* keys generated
+     with git-annex 0.2x or earlier.) The code is believed to work on
+     Linux, FreeBSD, and OSX; check compile-time messages to see if it
+     is not enabled for your OS.
+   * Add annex.diskreserve config setting, to control how much free space
+     to reserve for other purposes and avoid using (defaults to 1 mb).
+   * Add --fast flag, that can enable less expensive, but also less thorough
+     versions of some commands.
+   * fsck: In fast mode, avoid checking checksums.
+   * unused: In fast mode, just show all existing temp files as unused,
+     and avoid expensive scan for other unused content.
+   * migrate: Support migrating v1 SHA keys to v2 SHA keys with
+     size information that can be used for free space checking.
+   * Fix space leak in fsck and drop commands.
+   * migrate: Bugfix for case when migrating a file results in a key that
+     is already present in .git/annex/objects.
+   * dropunused: Significantly sped up; only read unused log file once."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.23.mdwn b/doc/news/version_0.23.mdwn
deleted file mode 100644
index e045352ad4..0000000000
--- a/doc/news/version_0.23.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.23 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Support ssh remotes with a port specified.
-   * whereis: New subcommand to show where a file's content has gotten to.
-   * Rethink filename encoding handling for display. Since filename encoding
-     may or may not match locale settings, any attempt to decode filenames
-     will fail for some files. So instead, do all output in binary mode."""]]
\ No newline at end of file

From 1bae07641eab449f74a411fe6f134dda83eaa42c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 26 Mar 2011 10:57:44 +0000
Subject: [PATCH 1213/8313] Added a comment

---
 ..._ead36a23c3e6efa1c41e4555f93e014e._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment b/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment
new file mode 100644
index 0000000000..357b48a234
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-26T10:57:41Z"
+ content="""
+After some thought, perhaps the default fsck output should be at least machine readable and copy and pasteable i.e.
+
+
+$ git annex fsck
+Files with errors
+
+    file1
+    file2
+
+

+
+so I can then copy the list of borked files and then just paste it into a for loop in my shell to recover the files. it's just an idea.
+"""]]

From ceb9593a9cbd39b00daf57ce52724eb40d85f1e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:24:29 -0400
Subject: [PATCH 1214/8313] added dat unit parsing

Also added all 3 existing kinds of data units. And even more of my opinions
to this opinionated piece of code.
---
 DataUnits.hs | 132 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 106 insertions(+), 26 deletions(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index c2845affea..d8c8d31491 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -1,11 +1,27 @@
-{- data size display
+{- data size display and parsing
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module DataUnits (roughSize, compareSizes) where
+module DataUnits (
+	dataUnits,
+	storageUnits,
+	memoryUnits,
+	oldSchoolUnits,
+	roughSize,
+	compareSizes
+) where
+
+import Data.List
+import Data.Char
+
+type ByteSize = Integer
+type Name = String
+type Abbrev = String
+data Unit = Unit ByteSize Abbrev Name
+	deriving (Ord, Show, Eq)
 
 {- And now a rant: 
  -
@@ -28,7 +44,8 @@ module DataUnits (roughSize, compareSizes) where
  -
  - And the drive manufacturers happily continued selling drives that are
  - increasingly smaller than you'd expect, if you don't count on your
- - fingers. But that are increasingly bigger.
+ - fingers. But that are increasingly too big for anyone to much notice.
+ - This caused me to need git-annex.
  -
  - Thus, I use units here that I loathe. Because if I didn't, people would
  - be confused that their drives seem the wrong size, and other people would
@@ -36,36 +53,99 @@ module DataUnits (roughSize, compareSizes) where
  - progress?
  -}
 
-{- approximate display of a particular number of bytes -}
-roughSize :: Bool -> Integer -> String
-roughSize short i
-	| i < 0 = "-" ++ roughSize short (negate i)
-	| i >= at 8 = units 8 "yottabyte" "YB"
-	| i >= at 7 = units 7 "zettabyte" "ZB"
-	| i >= at 6 = units 6 "exabyte" "EB"
-	| i >= at 5 = units 5 "petabyte" "PB"
-	| i >= at 4 = units 4 "terabyte" "TB"
-	| i >= at 3 = units 3 "gigabyte" "GB"
-	| i >= at 2 = units 2 "megabyte" "MB"
-	| i >= at 1 = units 1 "kilobyte" "kB"
-	| otherwise = units 0 "byte" "B"
+dataUnits = storageUnits ++ memoryUnits
+
+{- Storage units are (stupidly) powers of ten. -}
+storageUnits :: [Unit]
+storageUnits =
+	[ Unit (p 8) "YB" "yottabyte"
+	, Unit (p 7) "ZB" "zettabyte"
+	, Unit (p 6) "EB" "exabyte"
+	, Unit (p 5) "PB" "petabyte"
+	, Unit (p 4) "TB" "terabyte"
+	, Unit (p 3) "GB" "gigabyte"
+	, Unit (p 2) "MB" "megabyte"
+	, Unit (p 1) "kB" "kilobyte" -- weird capitalization thanks to committe
+	, Unit (p 0) "B" "byte"
+	]
 	where
-		at :: Integer -> Integer
-		at n = 1000^n
+		p n = 1000^n
 
-		chop :: Integer -> Integer
-		chop d = round $ (fromInteger i :: Double) / fromInteger (at d)
+{- Memory units are (stupidly named) powers of 2. -}
+memoryUnits :: [Unit]
+memoryUnits =
+	[ Unit (p 8) "YiB" "yobibyte"
+	, Unit (p 7) "ZiB" "zebibyte"
+	, Unit (p 6) "EiB" "exbibyte"
+	, Unit (p 5) "PiB" "pebibyte"
+	, Unit (p 4) "TiB" "tebibyte"
+	, Unit (p 3) "GiB" "gigabyte"
+	, Unit (p 2) "MiB" "mebibyte"
+	, Unit (p 1) "kiB" "kibibyte"
+	, Unit (p 0) "B" "byte"
+	]
+	where
+		p n = 2^(n*10)
 
-		units d u u' = let num = chop d in
+{- Do you yearn for the days when men were men and megabytes were megabytes? -}
+oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
+	where
+		mingle (Unit s a n, Unit s' a' n') = Unit s' a n
+
+{- approximate display of a particular number of bytes -}
+roughSize :: [Unit] -> Bool -> ByteSize -> String
+roughSize units abbrev i
+	| i < 0 = "-" ++ findUnit units' (negate i)
+	| otherwise = findUnit units' i
+	where
+		units' = reverse $ sort units -- largest first
+
+		findUnit (u@(Unit s _ _):us) i'
+			| i' >= s = showUnit i' u
+			| otherwise = findUnit us i'
+		findUnit [] i' = showUnit i' (last units') -- bytes
+
+		showUnit i' (Unit s a n) = let num = chop i' s in
 			show num ++ " " ++
-			(if short then u' else plural num u)
+			(if abbrev then a else plural num n)
+
+		chop :: Integer -> Integer -> Integer
+		chop i' d = round $ (fromInteger i' :: Double) / fromInteger d
 
 		plural n u
 			| n == 1 = u
 			| otherwise = u ++ "s"
 
-compareSizes :: Bool -> Integer -> Integer -> String
-compareSizes short old new
-	| old > new = roughSize short (old - new) ++ " smaller"
-	| old < new = roughSize short (new - old) ++ " larger"
+{- displays comparison of two sizes -}
+compareSizes :: [Unit] -> Bool -> ByteSize -> ByteSize -> String
+compareSizes units abbrev old new
+	| old > new = roughSize units abbrev (old - new) ++ " smaller"
+	| old < new = roughSize units abbrev (new - old) ++ " larger"
 	| otherwise = "same"
+
+{- Parses strings like "10 kilobytes" or "0.5tb". -}
+readSize :: String -> [Unit] -> Maybe ByteSize
+readSize s units
+	| null parsednum = Nothing
+	| null parsedunit = Nothing
+	| otherwise = Just $ round $ number * (fromIntegral multiplier)
+	where
+		(number, rest) = head parsednum
+		multiplier = head $ parsedunit
+
+		parsednum = reads s :: [(Double, String)]
+		parsedunit = lookupUnit units unit
+
+		unit = takeWhile isAlpha $ dropWhile isSpace rest
+
+		lookupUnit _ [] = [1] -- no unit given, assume bytes
+		lookupUnit [] _ = []
+		lookupUnit (u@(Unit s a n):us) v
+			| a ~~ v || n ~~ v = [s]
+			| plural n ~~ v || a ~~ byteabbrev v = [s]
+			| otherwise = lookupUnit us v
+		
+		a ~~ b = map toLower a == map toLower b
+		
+		plural n = n ++ "s"
+		byteabbrev a = a ++ "b"

From 8bcdf42b99675d507813205f097ab7b64b30f514 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:37:39 -0400
Subject: [PATCH 1215/8313] annex.diskreserve can be given in arbitrary units
 (ie "0.5 gigabytes")

---
 Backend.hs         |  2 +-
 Content.hs         |  8 +++++---
 DataUnits.hs       | 17 +++++++++++------
 debian/changelog   |  6 ++++++
 doc/git-annex.mdwn |  6 ++++--
 5 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 4b0095214c..c0e93acc23 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -198,6 +198,6 @@ checkKeySize key = do
 				else do
 					dest <- moveBad key
 					warning $ "Bad file size (" ++
-						compareSizes True size size' ++ 
+						compareSizes storageUnits True size size' ++ 
 						"); moved to " ++ dest
 					return False
diff --git a/Content.hs b/Content.hs
index 633d827e6f..7aa30f7ffc 100644
--- a/Content.hs
+++ b/Content.hs
@@ -122,7 +122,9 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
 	r <- Annex.repoConfig g "diskreserve" ""
-	let reserve = if null r then megabyte else (read r :: Integer)
+	let reserve = case readSize dataUnits r of
+		Nothing -> megabyte
+		Just v -> v
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
@@ -133,12 +135,12 @@ checkDiskSpace' adjustment key = do
 				else return ()
 	where
 		megabyte :: Integer
-		megabyte = 1024 * 1024
+		megabyte = 1000000
 		needmorespace n = do
 			force <- Annex.getState Annex.force
 			unless force $ 
 				error $ "not enough free space, need " ++ 
-					roughSize True n ++
+					roughSize storageUnits True n ++
 					" more (use --force to override this check or adjust annex.diskreserve)"
 
 {- Removes the write bits from a file. -}
diff --git a/DataUnits.hs b/DataUnits.hs
index d8c8d31491..37b0fa4295 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -11,7 +11,8 @@ module DataUnits (
 	memoryUnits,
 	oldSchoolUnits,
 	roughSize,
-	compareSizes
+	compareSizes,
+	readSize
 ) where
 
 import Data.List
@@ -53,6 +54,7 @@ data Unit = Unit ByteSize Abbrev Name
  - progress?
  -}
 
+dataUnits :: [Unit]
 dataUnits = storageUnits ++ memoryUnits
 
 {- Storage units are (stupidly) powers of ten. -}
@@ -69,6 +71,7 @@ storageUnits =
 	, Unit (p 0) "B" "byte"
 	]
 	where
+		p :: Integer -> Integer
 		p n = 1000^n
 
 {- Memory units are (stupidly named) powers of 2. -}
@@ -85,12 +88,14 @@ memoryUnits =
 	, Unit (p 0) "B" "byte"
 	]
 	where
+		p :: Integer -> Integer
 		p n = 2^(n*10)
 
 {- Do you yearn for the days when men were men and megabytes were megabytes? -}
+oldSchoolUnits :: [Unit]
 oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
 	where
-		mingle (Unit s a n, Unit s' a' n') = Unit s' a n
+		mingle (Unit _ a n, Unit s' _ _) = Unit s' a n
 
 {- approximate display of a particular number of bytes -}
 roughSize :: [Unit] -> Bool -> ByteSize -> String
@@ -124,8 +129,8 @@ compareSizes units abbrev old new
 	| otherwise = "same"
 
 {- Parses strings like "10 kilobytes" or "0.5tb". -}
-readSize :: String -> [Unit] -> Maybe ByteSize
-readSize s units
+readSize :: [Unit] -> String -> Maybe ByteSize
+readSize units input
 	| null parsednum = Nothing
 	| null parsedunit = Nothing
 	| otherwise = Just $ round $ number * (fromIntegral multiplier)
@@ -133,14 +138,14 @@ readSize s units
 		(number, rest) = head parsednum
 		multiplier = head $ parsedunit
 
-		parsednum = reads s :: [(Double, String)]
+		parsednum = reads input :: [(Double, String)]
 		parsedunit = lookupUnit units unit
 
 		unit = takeWhile isAlpha $ dropWhile isSpace rest
 
 		lookupUnit _ [] = [1] -- no unit given, assume bytes
 		lookupUnit [] _ = []
-		lookupUnit (u@(Unit s a n):us) v
+		lookupUnit (Unit s a n:us) v
 			| a ~~ v || n ~~ v = [s]
 			| plural n ~~ v || a ~~ byteabbrev v = [s]
 			| otherwise = lookupUnit us v
diff --git a/debian/changelog b/debian/changelog
index c664566a9e..e9bc896bf0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110326) UNRELEASED; urgency=low
+
+  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+
+ -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
+
 git-annex (0.20110325) experimental; urgency=low
 
   * Free space checking is now done, for transfers of data for keys
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 81cea04cd7..32f190e75d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -387,8 +387,10 @@ Here are all the supported configuration settings.
   Amount of disk space to reserve. Disk space is checked when transferring
   content to avoid running out, and additional free space can be reserved
   via this option, to make space for more important content (such as git
-  commit logs). The units are bytes.
-  The default reserve is 1048576 (1 megabyte).
+  commit logs). Can be specified with any commonly used units, for example,
+  "0.5 gb" or "100 KiloBytes"
+
+  The default reserve is 1 megabyte.
 
 * `annex.version`
 

From a5c1c2770e967ec16d64d015fff9ab3eecb314df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:47:55 -0400
Subject: [PATCH 1216/8313] add more pointless opinion

---
 DataUnits.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/DataUnits.hs b/DataUnits.hs
index 37b0fa4295..fb8ce3d89c 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -91,6 +91,10 @@ memoryUnits =
 		p :: Integer -> Integer
 		p n = 2^(n*10)
 
+{- Bandwidth units are only measured in bits if you're some crazy telco. -}
+bandwidthUnits :: [Unit]
+bandwidthUnits = error "stop trying to rip people off"
+
 {- Do you yearn for the days when men were men and megabytes were megabytes? -}
 oldSchoolUnits :: [Unit]
 oldSchoolUnits = map mingle $ zip storageUnits memoryUnits

From 70fe1ee70b53fe5df766abcf38646f70dcea6c75 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:49:23 -0400
Subject: [PATCH 1217/8313] update

---
 DataUnits.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/DataUnits.hs b/DataUnits.hs
index fb8ce3d89c..fa706ef068 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -9,6 +9,7 @@ module DataUnits (
 	dataUnits,
 	storageUnits,
 	memoryUnits,
+	bandwidthUnits,
 	oldSchoolUnits,
 	roughSize,
 	compareSizes,

From fd8cba164c180149127656070ac4a68bbe13b9cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 14:54:11 -0400
Subject: [PATCH 1218/8313] thanks liw

---
 DataUnits.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index fa706ef068..a175b5f6aa 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -85,7 +85,7 @@ memoryUnits =
 	, Unit (p 4) "TiB" "tebibyte"
 	, Unit (p 3) "GiB" "gigabyte"
 	, Unit (p 2) "MiB" "mebibyte"
-	, Unit (p 1) "kiB" "kibibyte"
+	, Unit (p 1) "KiB" "kibibyte"
 	, Unit (p 0) "B" "byte"
 	]
 	where

From 849c183f23a68d6204d5aacd0b9750374a5f59b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 15:10:12 -0400
Subject: [PATCH 1219/8313] tweak

---
 DataUnits.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/DataUnits.hs b/DataUnits.hs
index a175b5f6aa..c81c6e42e5 100644
--- a/DataUnits.hs
+++ b/DataUnits.hs
@@ -11,6 +11,7 @@ module DataUnits (
 	memoryUnits,
 	bandwidthUnits,
 	oldSchoolUnits,
+
 	roughSize,
 	compareSizes,
 	readSize
@@ -142,11 +143,10 @@ readSize units input
 	where
 		(number, rest) = head parsednum
 		multiplier = head $ parsedunit
+		unitname = takeWhile isAlpha $ dropWhile isSpace rest
 
 		parsednum = reads input :: [(Double, String)]
-		parsedunit = lookupUnit units unit
-
-		unit = takeWhile isAlpha $ dropWhile isSpace rest
+		parsedunit = lookupUnit units unitname
 
 		lookupUnit _ [] = [1] -- no unit given, assume bytes
 		lookupUnit [] _ = []

From 433c1780cff5673591be94deae02e775f804ab7c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Mar 2011 18:26:36 -0400
Subject: [PATCH 1220/8313] more thoughts

---
 Remotes.hs       |  1 -
 doc/todo/S3.mdwn | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/Remotes.hs b/Remotes.hs
index 8b760ac957..5a65e4fc7d 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -7,7 +7,6 @@
 
 module Remotes (
 	list,
-	tryGitConfigRead,
 	readConfigs,
 	keyPossibilities,
 	inAnnex,
diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 56023e71e1..946fa68170 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -71,3 +71,21 @@ Implementing this might not be as conceptually nice as making S3 a separate
 backend. It would need some changes to the remotes code, perhaps lifting
 some of it into backend-specific hooks. Then the S3 backend could be
 implicitly stacked in front of a backend like WORM.
+
+---
+
+Maybe the right way to look at this is that a list of Stores
+should be a property of the Backend. Backend.File is a Backend, that
+uses various Stores, which can be of different types (the local
+git repo, remote git repos, S3, etc). Backend.URL is a backend that uses
+other Stores (the local git repo, and the web).
+
+Operations on Stores are:
+
+* uuid -- each store has a unique uuid value
+* cost -- each store has a use cost value
+* getConfig -- attempts to look up values (uuid, possibly cost)
+* copyToStore -- store a file's contents to a key
+* copyFromStore -- retrieve a key's contents to a file
+* removeFromStore -- removes a key's contents from the store
+* hasKey -- checks if the key's content is available

From 2f32b7afc0ba361a4b92b74357115284634af529 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 10:21:18 +0000
Subject: [PATCH 1221/8313]

---
 ...has_issues_with_git_when_staging__47__commiting_logs.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
new file mode 100644
index 0000000000..89b6f9bee2
--- /dev/null
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -0,0 +1,5 @@
+After a series of pretty convoluted copying files around between annex'd repos and pulling changes around between repos. I noticed that occassionally when git-annex tries to stage files (the `.git-annex/*/*/*logs`) git some times gets wedged and doing a "git commit -a" doesn't seem to work or files might not get added thus leaving a bunch of untracked files or modified files that aren't staged for a commit.
+
+I tried running a *git add -u .git-annex/* and also just the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
+
+For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.

From 07d15ab2eae88059f2a90e1b389e5ce34deeafe6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 10:33:08 +0000
Subject: [PATCH 1222/8313]

---
 ...ex_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 89b6f9bee2..d4e2635f56 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -1,5 +1,5 @@
 After a series of pretty convoluted copying files around between annex'd repos and pulling changes around between repos. I noticed that occassionally when git-annex tries to stage files (the `.git-annex/*/*/*logs`) git some times gets wedged and doing a "git commit -a" doesn't seem to work or files might not get added thus leaving a bunch of untracked files or modified files that aren't staged for a commit.
 
-I tried running a *git add -u .git-annex/* and also just the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
+I tried running a *`git rm --cached -f -r *`* then *git add -u .git-annex/* or the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
 
 For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.

From ca5a6f4f42237136550347539f7d51376beef997 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 12:58:15 +0000
Subject: [PATCH 1223/8313]

---
 ...git-annex_directory_hashing_problems_on_osx.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
new file mode 100644
index 0000000000..9320b30a99
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -0,0 +1,13 @@
+Currently the hashed directories in .git-annex allow for upper and lower case directory names... on linux (or any case sensitive filesystem) the directory names such as 'Gg' and 'GG' are different and unique. However on systems like OSX (and probably windows if it is ever supported) the directory names 'Gg' is the same as 'GG'
+
+In one of the annex'd repos that I have this has occured...
+
+
+$ git add -i                                                                                          
+           staged     unstaged path
+  1:    unchanged        +1/-1 .git-annex/GM/GV/WORM-s183630166-m1301072171--somefile.log
+  2:    unchanged        +1/-1 .git-annex/Gm/GV/WORM-s183630166-m1301072171--somefile.log
+

+
+
+this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.

From 3aaf1fcc549791c19688a8d91cacc7c037506de5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 13:02:00 +0000
Subject: [PATCH 1224/8313]

---
 doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index 9320b30a99..b11cfca427 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -11,3 +11,5 @@ $ git add -i
 
 
 this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.
+
+Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.

From 3d84ec1bda71771c5d04c4c8ade37977eecf5a65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 12:41:53 -0400
Subject: [PATCH 1225/8313] response

---
 ...annex_directory_hashing_problems_on_osx.mdwn | 17 +++++++++++++++++
 ...th_git_when_staging__47__commiting_logs.mdwn |  6 ++++++
 2 files changed, 23 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index b11cfca427..d1a3c06077 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -12,4 +12,21 @@ $ git add -i
 
 this has somewhat confused git when it tries to stage/merge files, I didn't notice this at first, but it is definately a problem for someone using case insensitive filesystems  like the default OSX HFS+ formats or vfat/fat32.
 
+> I feel a bit stupid to not have considered case-insensative filesystems.
+> They are just so far from where I have lived for 20 years that it's hard
+> to keep them in mind.
+> 
+> I guess that
+> [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]] is
+> somehow a consequence (or cause?) of this, but I don't quite understand
+> how this is causing git to fail to stage files, or stage the same file
+> twice under different capitalizations. git-annex always will run git add
+> on the path with the "correct" capitalization. So unless something else
+> has added the path with the other capitalization (perhaps git add
+> .git-annex manually?) I don't understand how you get to this state.
+> --[[Joey]]
+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
+
+> There is a guard against `git annex add .git-annex/foo`, but it doesn't
+> notice `cd .git-annex; git annex add foo`. --[[Joey]]
diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index d4e2635f56..5c654ff2ba 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -3,3 +3,9 @@ After a series of pretty convoluted copying files around between annex'd repos a
 I tried running a *`git rm --cached -f -r *`* then *git add -u .git-annex/* or the usual *git add* then a commit fixes things for me. If I don't do that then my subsequent merges/pulls will fail and result in *no known copies of files* I suspect git-annex might have just touched some file modes and git picked up the changes but got confused since there was no content change. It might also just be a git on OSX thing and it doesn't affect linux/bsd users.
 
 For now it's just a bit of extra work for me when it does occur but it does not seem to occur often.
+
+> What do you mean when you say that git "got wedged"? It hung somehow?
+>
+> If git-annex runs concurrently with another git command that locks
+> the repository its git add of log files can fail.
+> --[[Joey]] 

From 5daf33d557291f1467a995f6c26f193e9323dd95 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 16:49:10 +0000
Subject: [PATCH 1226/8313]

---
 ...ex_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 5c654ff2ba..4b61950dfe 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -9,3 +9,5 @@ For now it's just a bit of extra work for me when it does occur but it does not
 > If git-annex runs concurrently with another git command that locks
 > the repository its git add of log files can fail.
 > --[[Joey]] 
+
+>> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.

From bb7688233bd10773ab75eba7135497ae10af9061 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 16:53:51 +0000
Subject: [PATCH 1227/8313]

---
 doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index d1a3c06077..2315bbb83f 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -26,6 +26,8 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 > .git-annex manually?) I don't understand how you get to this state.
 > --[[Joey]]
 
+>> I think I got myself into this situation when I copied some files over from a HFS+ partition to a GPFS network share (which is pretty posix compliant) over samba. It probably is related to the [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. I thought they were unique enough to have two bug reports logged as one is a git behavioural thing and the other is git-annex specific.
+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't

From 41d660c88f60e982bf3f954d70b074ad1dd0c13a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 13:15:25 -0400
Subject: [PATCH 1228/8313] response

---
 ...x_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 4b61950dfe..f1290a8186 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -11,3 +11,6 @@ For now it's just a bit of extra work for me when it does occur but it does not
 > --[[Joey]] 
 
 >> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.
+
+>>> I've never seen git refuse to commit staged files. There would have to
+>>> be some error message? --[[Joey]] 

From 2821effce9ae95a2ef12a083ce0806fe058ac987 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 13:30:18 -0400
Subject: [PATCH 1229/8313] response

---
 ...nex_directory_hashing_problems_on_osx.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index 2315bbb83f..2e6b138c16 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -28,6 +28,29 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 
 >> I think I got myself into this situation when I copied some files over from a HFS+ partition to a GPFS network share (which is pretty posix compliant) over samba. It probably is related to the [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]]. I thought they were unique enough to have two bug reports logged as one is a git behavioural thing and the other is git-annex specific.
 
+>>> If you copied `.git/` over, perhaps you got a git repo without
+>>> core.ignorecase set right for the filesystem it landed on?
+>>> 
+>>> Something like this might reproduce it:
+
+
+# mkdir test; cd test; git init
+# git config core.ignorecase false
+# mkdir Foo
+# touch Foo/bar
+# git add Foo/bar
+# git add foo/bar
+# git add fOo/bar
+# git status
+# touch foo/other
+# git add fOo/other
+# git status
+

+
+>>>> And then either git commit or git clone would probably get confused
+>>>> if it thought 3 distinct files had been committed.
+>>>> --[[Joey]]
+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't

From 3ded849532216d97ac6004146445fbcc85769054 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 17:37:07 +0000
Subject: [PATCH 1230/8313]

---
 ...ex_has_issues_with_git_when_staging__47__commiting_logs.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index f1290a8186..b7944b418c 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -14,3 +14,5 @@ For now it's just a bit of extra work for me when it does occur but it does not
 
 >>> I've never seen git refuse to commit staged files. There would have to
 >>> be some error message? --[[Joey]] 
+
+>>>> there were no error messages at all

From 1fccea5154ac5b8e10fa387e2230e3a4103cef3b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 27 Mar 2011 17:48:51 +0000
Subject: [PATCH 1231/8313]

---
 ...nex_directory_hashing_problems_on_osx.mdwn | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
index 2e6b138c16..e5937fe83f 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn
@@ -30,6 +30,9 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 
 >>> If you copied `.git/` over, perhaps you got a git repo without
 >>> core.ignorecase set right for the filesystem it landed on?
+
+>>>> I usually git clone or do a fresh repository and pull things in, I was also unaware of this ignorecase setting as well.
+
 >>> 
 >>> Something like this might reproduce it:
 
@@ -51,6 +54,47 @@ this has somewhat confused git when it tries to stage/merge files, I didn't noti
 >>>> if it thought 3 distinct files had been committed.
 >>>> --[[Joey]]
 
+>>>>> Doing the above test on a HFS+ partition yields this
+
+
+## with ignorecase=false
+commit bb024c6fd7482b2d10f60ae899cb7a949aca1ad8
+Author: Jimmy Tang 
+Date:   Sun Mar 27 18:40:24 2011 +0100
+
+    commit
+
+diff --git a/Foo/bar b/Foo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/fOo/bar b/fOo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/fOo/other b/fOo/other
+new file mode 100644
+index 0000000..e69de29
+diff --git a/foo/bar b/foo/bar
+new file mode 100644
+index 0000000..e69de29
+

+
+>>>>> and without changing ignorecase
+
+
+commit 909a089158ffb98f8e91f98905e2bfdc7234666f
+Author: Jimmy Tang 
+Date:   Sun Mar 27 18:46:57 2011 +0100
+
+    commit
+
+diff --git a/Foo/bar b/Foo/bar
+new file mode 100644
+index 0000000..e69de29
+diff --git a/Foo/other b/Foo/other
+new file mode 100644
+index 0000000..e69de29
+

+
 Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories.
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't

From b40f253d6e126d699e9f298bf670fc5e875bfd86 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 15:56:43 -0400
Subject: [PATCH 1232/8313] start of generalizing remotes

Goal is to support multiple different types of remotes, some of which
are not git repositories. To that end, added a Remote class, and moved
git remote specific code into Remote.GitRemote.

Remotes.hs is still present as some code has not been converted to use the
new Remote class yet.
---
 Backend/File.hs     |  42 +++----
 Remote.hs           |  66 +++++++++++
 Remote/GitRemote.hs | 263 ++++++++++++++++++++++++++++++++++++++++++++
 RemoteClass.hs      |  46 ++++++++
 Remotes.hs          |   9 ++
 UUID.hs             |  18 ---
 6 files changed, 405 insertions(+), 39 deletions(-)
 create mode 100644 Remote.hs
 create mode 100644 Remote/GitRemote.hs
 create mode 100644 RemoteClass.hs

diff --git a/Backend/File.hs b/Backend/File.hs
index fb8a052553..743d8d6278 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -14,14 +14,14 @@
 
 module Backend.File (backend, checkKey) where
 
-import Control.Monad.State
-import System.Directory
+import Control.Monad.State (liftIO)
 import Data.List
+import Data.String.Utils
 
 import BackendClass
 import LocationLog
-import Locations
-import qualified Remotes
+import qualified Remote
+import qualified RemoteClass
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -51,10 +51,10 @@ dummyStore :: FilePath -> Key -> Annex Bool
 dummyStore _ _ = return True
 
 {- Try to find a copy of the file in one of the remotes,
- - and copy it over to this one. -}
+ - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- Remotes.keyPossibilities key
+	(remotes, _) <- Remote.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -72,18 +72,18 @@ copyKeyFile key file = do
 				then docopy r (trycopy full rs)
 				else trycopy full rs
 		-- This check is to avoid an ugly message if a remote is a
-		-- drive that is not mounted. Avoid checking inAnnex for ssh
-		-- remotes because that is unnecessarily slow, and the
-		-- locationlog should be trusted. (If the ssh remote is down
-		-- or really lacks the file, it's ok to show an ugly message
-		-- before going on to the next remote.)
+		-- drive that is not mounted.
 		probablyPresent r =
-			if not $ Git.repoIsUrl r
-				then liftIO $ doesFileExist $ gitAnnexLocation r key
+			if RemoteClass.hasKeyCheap r
+				then do
+					res <- RemoteClass.hasKey r key
+					case res of
+						Right b -> return b
+						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "copying from " ++ Git.repoDescribe r ++ "..."
-			copied <- Remotes.copyFromRemote r key file
+			showNote $ "copying from " ++ RemoteClass.name r ++ "..."
+			copied <- RemoteClass.retrieveKeyFile r key file
 			if copied
 				then return True
 				else continue
@@ -97,9 +97,9 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remotes.keyPossibilities key
+			(remotes, trusteduuids) <- Remote.keyPossibilities key
 			untrusteduuids <- trustGet UnTrusted
-			tocheck <- reposWithoutUUID remotes (trusteduuids++untrusteduuids)
+			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
 			findcopies numcopies trusteduuids tocheck []
 	where
@@ -109,9 +109,9 @@ checkRemoveKey key numcopiesM = do
 		findcopies need have (r:rs) bad
 			| length have >= need = return True
 			| otherwise = do
-				u <- getUUID r
+				let u = RemoteClass.uuid r
 				let dup = u `elem` have
-				haskey <- Remotes.inAnnex r key
+				haskey <- (RemoteClass.hasKey r) key
 				case (dup, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
 					(False, Left _)		-> findcopies need have rs (r:bad)
@@ -147,11 +147,11 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [Git.Repo] -> Annex ()
+showTriedRemotes :: [RemoteClass.Remote] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
-		Remotes.list remotes
+		(join ", " $ map RemoteClass.name remotes)
 
 getNumCopies :: Maybe Int -> Annex Int
 getNumCopies (Just n) = return n
diff --git a/Remote.hs b/Remote.hs
new file mode 100644
index 0000000000..9eff5556c9
--- /dev/null
+++ b/Remote.hs
@@ -0,0 +1,66 @@
+{- git-annex remotes
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote (
+	generate,
+	keyPossibilities,
+	remotesWithUUID,
+	remotesWithoutUUID
+) where
+
+import Control.Monad.State (liftIO)
+import Data.List
+
+import RemoteClass
+import qualified Remote.GitRemote
+import Types
+import UUID
+import qualified Annex
+import Trust
+import LocationLog
+
+{- add generators for new Remotes here -}
+generators :: [Annex [Remote]]
+generators = [Remote.GitRemote.generate]
+
+{- generates a list of all available Remotes -}
+generate :: Annex [Remote]
+generate = do
+	lists <- sequence generators
+	return $ concat lists
+
+{- Filters a list of remotes to ones that have the listed uuids. -}
+remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
+
+{- Filters a list of remotes to ones that do not have the listed uuids. -}
+remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
+
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -
+ - Also returns a list of UUIDs that are trusted to have the key
+ - (some may not have configured remotes).
+ -}
+keyPossibilities :: Key -> Annex ([Remote], [UUID])
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	trusted <- trustGet Trusted
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- liftIO $ keyLocations g key
+	let validuuids = filter (/= u) uuids
+
+	-- note that validuuids is assumed to not have dups
+	let validtrusteduuids = intersect validuuids trusted
+
+	-- remotes that match uuids that have the key
+	allremotes <- generate
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return (sort validremotes, validtrusteduuids)
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
new file mode 100644
index 0000000000..ccc5f7b42f
--- /dev/null
+++ b/Remote/GitRemote.hs
@@ -0,0 +1,263 @@
+{- Standard git remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.GitRemote (generate) where
+
+import Control.Exception.Extensible
+import Control.Monad.State (liftIO)
+import qualified Data.Map as Map
+import Data.String.Utils
+import System.Cmd.Utils
+import Control.Monad (unless, filterM)
+
+import RemoteClass
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import Locations
+import UUID
+import Utility
+import qualified Content
+import Messages
+import CopyFile
+import RsyncFile
+import Ssh
+
+generate :: Annex [Remote]
+generate = do
+	readConfigs
+	g <- Annex.gitRepo
+	rs <- filterM repoNotIgnored (Git.remotes g)
+	mapM genRemote rs
+	
+genRemote :: Git.Repo -> Annex Remote
+genRemote r = do
+	u <- getUUID r
+	c <- repoCost r
+	return Remote {
+		uuid = u,
+		cost = c,
+		name = Git.repoDescribe r,
+		storeKey = copyToRemote r,
+		retrieveKeyFile = copyFromRemote r,
+		removeKey = error "TODO Remote.GitRemote.removeKey",
+		hasKey = inAnnex r,
+		hasKeyCheap = not (Git.repoIsUrl r)
+	}
+
+{- Reads the configs of all remotes.
+ -
+ - As reading the config of remotes can be expensive, this
+ - function will only read configs once per git-annex run. It's
+ - assumed to be cheap to read the config of non-URL remotes,
+ - so this is done each time git-annex is run. Conversely,
+ - the config of an URL remote is only read when there is no
+ - cached UUID value.
+ - -}
+readConfigs :: Annex ()
+readConfigs = do
+	remotesread <- Annex.getState Annex.remotesread
+	unless remotesread $ do
+		g <- Annex.gitRepo
+		allremotes <- filterM repoNotIgnored $ Git.remotes g
+		let cheap = filter (not . Git.repoIsUrl) allremotes
+		let expensive = filter Git.repoIsUrl allremotes
+		doexpensive <- filterM cachedUUID expensive
+		unless (null doexpensive) $
+			showNote $ "getting UUID for " ++
+				list doexpensive ++ "..."
+		let todo = cheap ++ doexpensive
+		unless (null todo) $ do
+			mapM_ tryGitConfigRead todo
+			Annex.changeState $ \s -> s { Annex.remotesread = True }
+	where
+		cachedUUID r = do
+			u <- getUUID r
+			return $ null u
+
+{- The git configs for the git repo's remotes is not read on startup
+ - because reading it may be expensive. This function tries to read the
+ - config for a specified remote, and updates state. If successful, it
+ - returns the updated git repo. -}
+tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo)
+tryGitConfigRead r 
+	| not $ Map.null $ Git.configMap r = return $ Right r -- already read
+	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
+	| Git.repoIsUrl r = return $ Left r
+	| otherwise = store $ safely $ Git.configRead r
+	where
+		-- Reading config can fail due to IO error or
+		-- for other reasons; catch all possible exceptions.
+		safely a = do
+			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			case result of
+				Left _ -> return r
+				Right r' -> return r'
+		pipedconfig cmd params = safely $
+			pOpen ReadFromPipe cmd (toCommand params) $
+				Git.hConfigRead r
+		store a = do
+			r' <- a
+			g <- Annex.gitRepo
+			let l = Git.remotes g
+			let g' = Git.remotesAdd g $ exchange l r'
+			Annex.changeState $ \s -> s { Annex.repo = g' }
+			return $ Right r'
+		exchange [] _ = []
+		exchange (old:ls) new =
+			if Git.repoRemoteName old == Git.repoRemoteName new
+				then new : exchange ls new
+				else old : exchange ls new
+
+{- Calculates cost for a repo.
+ -
+ - The default cost is 100 for local repositories, and 200 for remote
+ - repositories; it can also be configured by remote..annex-cost
+ -}
+repoCost :: Git.Repo -> Annex Int
+repoCost r = do
+	c <- Annex.repoConfig r "cost" ""
+	if not $ null c
+		then return $ read c
+		else if Git.repoIsUrl r
+			then return 200
+			else return 100
+
+{- Checks if a repo should be ignored, based either on annex-ignore
+ - setting, or on command-line options. Allows command-line to override
+ - annex-ignore. -}
+repoNotIgnored :: Git.Repo -> Annex Bool
+repoNotIgnored r = do
+	ignored <- Annex.repoConfig r "ignore" "false"
+	to <- match Annex.toremote
+	from <- match Annex.fromremote
+	if to || from
+		then return True
+		else return $ not $ Git.configTrue ignored
+	where
+		match a = do
+			n <- Annex.getState a
+			return $ n == Git.repoRemoteName r
+
+{- Checks if a given remote has the content for a key inAnnex.
+ - If the remote cannot be accessed, returns a Left error.
+ -}
+inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
+inAnnex r key = if Git.repoIsUrl r
+		then checkremote
+		else liftIO (try checklocal ::IO (Either IOException Bool))
+	where
+		checklocal = do
+			-- run a local check inexpensively,
+			-- by making an Annex monad using the remote
+			a <- Annex.new r []
+			Annex.eval a (Content.inAnnex key)
+		checkremote = do
+			showNote ("checking " ++ Git.repoDescribe r ++ "...")
+			inannex <- onRemote r (boolSystem, False) "inannex" 
+				[Param (show key)]
+			return $ Right inannex
+
+{- Tries to copy a key's content from a remote's annex to a file. -}
+copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
+copyFromRemote r key file
+	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
+	| Git.repoIsSsh r = rsynchelper r True key file
+	| otherwise = error "copying from non-ssh repo not supported"
+
+{- Tries to copy a key's content to a remote's annex. -}
+copyToRemote :: Git.Repo -> Key -> Annex Bool
+copyToRemote r key
+	| not $ Git.repoIsUrl r = do
+		g <- Annex.gitRepo
+		let keysrc = gitAnnexLocation g key
+		-- run copy from perspective of remote
+		liftIO $ do
+			a <- Annex.new r []
+			Annex.eval a $ do
+				ok <- Content.getViaTmp key $
+					\f -> liftIO $ copyFile keysrc f
+				Annex.queueRun
+				return ok
+	| Git.repoIsSsh r = do
+		g <- Annex.gitRepo
+		let keysrc = gitAnnexLocation g key
+		rsynchelper r False key keysrc
+	| otherwise = error "copying to non-ssh repo not supported"
+
+rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool)
+rsynchelper r sending key file = do
+	showProgress -- make way for progress bar
+	p <- rsyncParams r sending key file
+	res <- liftIO $ boolSystem "rsync" p
+	if res
+		then return res
+		else do
+			showLongNote "rsync failed -- run git annex again to resume file transfer"
+			return res
+
+{- Generates rsync parameters that ssh to the remote and asks it
+ - to either receive or send the key's content. -}
+rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
+rsyncParams r sending key file = do
+	Just (shellcmd, shellparams) <- git_annex_shell r
+		(if sending then "sendkey" else "recvkey")
+		[ Param $ show key
+		-- Command is terminated with "--", because
+		-- rsync will tack on its own options afterwards,
+		-- and they need to be ignored.
+		, Param "--"
+		]
+	-- Convert the ssh command into rsync command line.
+	let eparam = rsyncShell (Param shellcmd:shellparams)
+	o <- Annex.repoConfig r "rsync-options" ""
+	let base = options ++ map Param (words o) ++ eparam
+	if sending
+		then return $ base ++ [dummy, File file]
+		else return $ base ++ [File file, dummy]
+	where
+		-- inplace makes rsync resume partial files
+		options = [Params "-p --progress --inplace"]
+		-- the rsync shell parameter controls where rsync
+		-- goes, so the source/dest parameter can be a dummy value,
+		-- that just enables remote rsync mode.
+		dummy = Param ":"
+
+{- Uses a supplied function to run a git-annex-shell command on a remote.
+ -
+ - Or, if the remote does not support running remote commands, returns
+ - a specified error value. -}
+onRemote 
+	:: Git.Repo
+	-> (FilePath -> [CommandParam] -> IO a, a)
+	-> String
+	-> [CommandParam]
+	-> Annex a
+onRemote r (with, errorval) command params = do
+	s <- git_annex_shell r command params
+	case s of
+		Just (c, ps) -> liftIO $ with c ps
+		Nothing -> return errorval
+
+{- Generates parameters to run a git-annex-shell command on a remote. -}
+git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam]))
+git_annex_shell r command params
+	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
+	| Git.repoIsSsh r = do
+		sshparams <- sshToRepo r [Param sshcmd]
+		return $ Just ("ssh", sshparams)
+	| otherwise = return Nothing
+	where
+		dir = Git.workTree r
+		shellcmd = "git-annex-shell"
+		shellopts = (Param command):(File dir):params
+		sshcmd = shellcmd ++ " " ++ 
+			unwords (map shellEscape $ toCommand shellopts)
+
+{- Human visible list of remotes. -}
+list :: [Git.Repo] -> String
+list remotes = join ", " $ map Git.repoDescribe remotes
diff --git a/RemoteClass.hs b/RemoteClass.hs
new file mode 100644
index 0000000000..df2aefb715
--- /dev/null
+++ b/RemoteClass.hs
@@ -0,0 +1,46 @@
+{- git-annex remotes class
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteClass where
+
+import Control.Exception
+
+import Annex
+import UUID
+import Key
+
+data Remote = Remote {
+	-- each Remote has a unique uuid
+	uuid :: UUID,
+	-- each Remote has a human visible name
+	name :: String,
+	-- Remotes have a use cost; higher is more expensive
+	cost :: Int,
+	-- Transfers a key to the remote.
+	storeKey :: Key -> Annex Bool,
+	-- retrieves a key's contents to a file
+	retrieveKeyFile :: Key -> FilePath -> Annex Bool,
+	-- removes a key's contents
+	removeKey :: Key -> Annex Bool,
+	-- Checks if a key is present in the remote; if the remote
+	-- cannot be accessed returns a Left error.
+	hasKey :: Key -> Annex (Either IOException Bool),
+	-- Some remotes can check hasKey without an expensive network
+	-- operation.
+	hasKeyCheap :: Bool
+}
+
+instance Show Remote where
+	show remote = "Remote { uuid =\"" ++ uuid remote ++ "\" }"
+
+-- two remotes are the same if they have the same uuid
+instance Eq Remote where
+	a == b = uuid a == uuid b
+
+-- order remotes by cost
+instance Ord Remote where
+	compare a b = compare (cost a) (cost b)
diff --git a/Remotes.hs b/Remotes.hs
index 5a65e4fc7d..5fc594ee29 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -322,3 +322,12 @@ git_annex_shell r command params
 		shellopts = (Param command):(File dir):params
 		sshcmd = shellcmd ++ " " ++ 
 			unwords (map shellEscape $ toCommand shellopts)
+
+{- Filters a list of repos to ones that have listed UUIDs. -}
+reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
+reposByUUID repos uuids = filterM match repos
+	where
+		match r = do
+			u <- getUUID r
+			return $ u `elem` uuids
+
diff --git a/UUID.hs b/UUID.hs
index 239d373f14..42afd7ba86 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -14,8 +14,6 @@ module UUID (
 	getUncachedUUID,
 	prepUUID,
 	genUUID,
-	reposByUUID,
-	reposWithoutUUID,
 	prettyPrintUUIDs,
 	describeUUID,
 	uuidLog,
@@ -87,22 +85,6 @@ prepUUID = do
 		uuid <- liftIO $ genUUID
 		Annex.setConfig configkey uuid
 
-{- Filters a list of repos to ones that have listed UUIDs. -}
-reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
-reposByUUID repos uuids = filterM match repos
-	where
-		match r = do
-			u <- getUUID r
-			return $ u `elem` uuids
-
-{- Filters a list of repos to ones that do not have the listed UUIDs. -}
-reposWithoutUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
-reposWithoutUUID repos uuids = filterM unmatch repos
-	where
-		unmatch r = do
-			u <- getUUID r
-			return $ u `notElem` uuids
-
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
 prettyPrintUUIDs uuids = do

From cd451760c33b702616727f91b0364601609491c5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 20:11:20 +0000
Subject: [PATCH 1233/8313]

---
 doc/forum/batch_check_on_remote_when_using_copy.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/batch_check_on_remote_when_using_copy.mdwn

diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
new file mode 100644
index 0000000000..355d98f04b
--- /dev/null
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -0,0 +1,5 @@
+When I copy my local repository with SHA* to a remote repo with SHA*, every single file is checked by itself which seems rather inefficient. When my remote is accessed via ssh, git-annex opens a new connections for every check. If you are not using a ssh key or key agent, this gets tedious...
+
+For all locked files, either git's built-in mechanisms should be used or, if that's not possible, a few hundred checksums (assuming SHA* backend) should be transfered at once and then checked locally before deciding that to transfer.
+
+-- RichiH

From bec63186ea0b9b25e2387685c209578bc316397d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 20:14:47 +0000
Subject: [PATCH 1234/8313]

---
 doc/forum/batch_check_on_remote_when_using_copy.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
index 355d98f04b..0f20ab6454 100644
--- a/doc/forum/batch_check_on_remote_when_using_copy.mdwn
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -2,4 +2,7 @@ When I copy my local repository with SHA* to a remote repo with SHA*, every sing
 
 For all locked files, either git's built-in mechanisms should be used or, if that's not possible, a few hundred checksums (assuming SHA* backend) should be transfered at once and then checked locally before deciding that to transfer.
 
+Once all checks are done, one single transfer session should be started. Creating new sessions and waiting for TCP's slowstart to get going is a lot less than efficient.
+
+
 -- RichiH

From f30320aa75d6fa590f60030f13df6b3899816196 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:17:56 -0400
Subject: [PATCH 1235/8313] add remotes slot to Annex

This required parameterizing the type for Remote, to avoid a cycle.
---
 Annex.hs            |  5 +++--
 Backend/File.hs     |  2 +-
 Remote.hs           | 10 +++++-----
 Remote/GitRemote.hs | 37 ++++++++++++++++---------------------
 RemoteClass.hs      | 24 +++++++++++-------------
 Remotes.hs          |  5 +++--
 UUID.hs             |  4 ++--
 7 files changed, 41 insertions(+), 46 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f45415a72f..bb26608f4a 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -27,6 +27,7 @@ import Data.Maybe
 import qualified GitRepo as Git
 import qualified GitQueue
 import qualified BackendClass
+import qualified RemoteClass
 import Utility
 
 -- git-annex's monad
@@ -37,6 +38,7 @@ data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [BackendClass.Backend Annex]
 	, supportedBackends :: [BackendClass.Backend Annex]
+	, remotes :: [RemoteClass.Remote Annex]
 	, repoqueue :: GitQueue.Queue
 	, quiet :: Bool
 	, force :: Bool
@@ -46,13 +48,13 @@ data AnnexState = AnnexState
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, exclude :: [String]
-	, remotesread :: Bool
 	} deriving (Show)
 
 newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState
 newState gitrepo allbackends = AnnexState
 	{ repo = gitrepo
 	, backends = []
+	, remotes = []
 	, supportedBackends = allbackends
 	, repoqueue = GitQueue.empty
 	, quiet = False
@@ -63,7 +65,6 @@ newState gitrepo allbackends = AnnexState
 	, toremote = Nothing
 	, fromremote = Nothing
 	, exclude = []
-	, remotesread = False
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
diff --git a/Backend/File.hs b/Backend/File.hs
index 743d8d6278..9c102cf509 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -147,7 +147,7 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [RemoteClass.Remote] -> Annex ()
+showTriedRemotes :: [RemoteClass.Remote Annex] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
diff --git a/Remote.hs b/Remote.hs
index 9eff5556c9..078a603bbf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -24,21 +24,21 @@ import Trust
 import LocationLog
 
 {- add generators for new Remotes here -}
-generators :: [Annex [Remote]]
+generators :: [Annex [Remote Annex]]
 generators = [Remote.GitRemote.generate]
 
 {- generates a list of all available Remotes -}
-generate :: Annex [Remote]
+generate :: Annex [Remote Annex]
 generate = do
 	lists <- sequence generators
 	return $ concat lists
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
-remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 
 {- Filters a list of remotes to ones that do not have the listed uuids. -}
-remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote]
+remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
@@ -46,7 +46,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilities :: Key -> Annex ([Remote], [UUID])
+keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
 keyPossibilities key = do
 	g <- Annex.gitRepo
 	u <- getUUID g
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
index ccc5f7b42f..0ec0c70e8d 100644
--- a/Remote/GitRemote.hs
+++ b/Remote/GitRemote.hs
@@ -27,14 +27,14 @@ import CopyFile
 import RsyncFile
 import Ssh
 
-generate :: Annex [Remote]
+generate :: Annex [Remote Annex]
 generate = do
 	readConfigs
 	g <- Annex.gitRepo
 	rs <- filterM repoNotIgnored (Git.remotes g)
 	mapM genRemote rs
 	
-genRemote :: Git.Repo -> Annex Remote
+genRemote :: Git.Repo -> Annex (Remote Annex)
 genRemote r = do
 	u <- getUUID r
 	c <- repoCost r
@@ -49,31 +49,26 @@ genRemote r = do
 		hasKeyCheap = not (Git.repoIsUrl r)
 	}
 
-{- Reads the configs of all remotes.
+{- Reads the configs of git remotes.
  -
- - As reading the config of remotes can be expensive, this
- - function will only read configs once per git-annex run. It's
- - assumed to be cheap to read the config of non-URL remotes,
+ - It's assumed to be cheap to read the config of non-URL remotes,
  - so this is done each time git-annex is run. Conversely,
  - the config of an URL remote is only read when there is no
  - cached UUID value.
- - -}
+ -}
 readConfigs :: Annex ()
 readConfigs = do
-	remotesread <- Annex.getState Annex.remotesread
-	unless remotesread $ do
-		g <- Annex.gitRepo
-		allremotes <- filterM repoNotIgnored $ Git.remotes g
-		let cheap = filter (not . Git.repoIsUrl) allremotes
-		let expensive = filter Git.repoIsUrl allremotes
-		doexpensive <- filterM cachedUUID expensive
-		unless (null doexpensive) $
-			showNote $ "getting UUID for " ++
-				list doexpensive ++ "..."
-		let todo = cheap ++ doexpensive
-		unless (null todo) $ do
-			mapM_ tryGitConfigRead todo
-			Annex.changeState $ \s -> s { Annex.remotesread = True }
+	g <- Annex.gitRepo
+	allremotes <- filterM repoNotIgnored $ Git.remotes g
+	let cheap = filter (not . Git.repoIsUrl) allremotes
+	let expensive = filter Git.repoIsUrl allremotes
+	doexpensive <- filterM cachedUUID expensive
+	unless (null doexpensive) $
+		showNote $ "getting UUID for " ++
+			list doexpensive ++ "..."
+	let todo = cheap ++ doexpensive
+	unless (null todo) $ do
+		mapM_ tryGitConfigRead todo
 	where
 		cachedUUID r = do
 			u <- getUUID r
diff --git a/RemoteClass.hs b/RemoteClass.hs
index df2aefb715..9fef0e44a0 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -9,38 +9,36 @@ module RemoteClass where
 
 import Control.Exception
 
-import Annex
-import UUID
 import Key
 
-data Remote = Remote {
+data Remote a = Remote {
 	-- each Remote has a unique uuid
-	uuid :: UUID,
+	uuid :: String,
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
 	cost :: Int,
 	-- Transfers a key to the remote.
-	storeKey :: Key -> Annex Bool,
+	storeKey :: Key -> a Bool,
 	-- retrieves a key's contents to a file
-	retrieveKeyFile :: Key -> FilePath -> Annex Bool,
+	retrieveKeyFile :: Key -> FilePath -> a Bool,
 	-- removes a key's contents
-	removeKey :: Key -> Annex Bool,
+	removeKey :: Key -> a Bool,
 	-- Checks if a key is present in the remote; if the remote
 	-- cannot be accessed returns a Left error.
-	hasKey :: Key -> Annex (Either IOException Bool),
+	hasKey :: Key -> a (Either IOException Bool),
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool
 }
 
-instance Show Remote where
+instance Show (Remote a) where
 	show remote = "Remote { uuid =\"" ++ uuid remote ++ "\" }"
 
 -- two remotes are the same if they have the same uuid
-instance Eq Remote where
-	a == b = uuid a == uuid b
+instance Eq (Remote a) where
+	x == y = uuid x == uuid y
 
 -- order remotes by cost
-instance Ord Remote where
-	compare a b = compare (cost a) (cost b)
+instance Ord (Remote a) where
+	compare x y = compare (cost x) (cost y)
diff --git a/Remotes.hs b/Remotes.hs
index 5fc594ee29..7f6a6718b3 100644
--- a/Remotes.hs
+++ b/Remotes.hs
@@ -91,7 +91,8 @@ tryGitConfigRead r
  - -}
 readConfigs :: Annex ()
 readConfigs = do
-	remotesread <- Annex.getState Annex.remotesread
+--	remotesread <- Annex.getState Annex.remotesread
+	let remotesread = False
 	unless remotesread $ do
 		g <- Annex.gitRepo
 		allremotes <- filterM repoNotIgnored $ Git.remotes g
@@ -104,7 +105,7 @@ readConfigs = do
 		let todo = cheap ++ doexpensive
 		unless (null todo) $ do
 			mapM_ tryGitConfigRead todo
-			Annex.changeState $ \s -> s { Annex.remotesread = True }
+--			Annex.changeState $ \s -> s { Annex.remotesread = True }
 	where
 		cachedUUID r = do
 			u <- getUUID r
diff --git a/UUID.hs b/UUID.hs
index 42afd7ba86..3f28434851 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -36,7 +36,7 @@ import qualified SysConfig
 type UUID = String
 
 configkey :: String
-configkey="annex.uuid"
+configkey = "annex.uuid"
 
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
@@ -74,7 +74,7 @@ getUUID r = do
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = Git.configGet r "annex.uuid" ""
+getUncachedUUID r = Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()

From 3470260a8500b42f805b8263af9c73b99706bb92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:24:46 -0400
Subject: [PATCH 1236/8313] clean up remote list generation to only run once

---
 Remote.hs | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 078a603bbf..f4b56846b1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -6,7 +6,6 @@
  -}
 
 module Remote (
-	generate,
 	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID
@@ -27,11 +26,19 @@ import LocationLog
 generators :: [Annex [Remote Annex]]
 generators = [Remote.GitRemote.generate]
 
-{- generates a list of all available Remotes -}
-generate :: Annex [Remote Annex]
-generate = do
-	lists <- sequence generators
-	return $ concat lists
+{- Builds a list of all available Remotes.
+ - Since doing so can be expensive, the list is cached in the Annex. -}
+genList :: Annex [Remote Annex]
+genList = do
+	liftIO $ putStrLn "Remote.genList"
+	rs <- Annex.getState Annex.remotes
+	if null rs
+		then do
+			lists <- sequence generators
+			let rs' = concat lists
+			Annex.changeState $ \s -> s { Annex.remotes = rs' }
+			return rs'
+		else return rs
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
@@ -60,7 +67,7 @@ keyPossibilities key = do
 	let validtrusteduuids = intersect validuuids trusted
 
 	-- remotes that match uuids that have the key
-	allremotes <- generate
+	allremotes <- genList
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)

From 834309d1d0c96f5f1ca9530c9622a24234242e28 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 20:39:14 +0000
Subject: [PATCH 1237/8313]

---
 ..._version_upgrade_leaves_repo_unusable.mdwn | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
new file mode 100644
index 0000000000..16a0ca3744
--- /dev/null
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -0,0 +1,28 @@
+foo is a local repo, bar is a bare remote.
+
+I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend to version 2. I then ran `git annex copy . --to bar` and checked the remote. This created WORM:SHA512--123123 files in annex/objects. Understandable but unwanted. So I upgraded git-annex on bar's machine, as well.
+
+    % git annex copy . --to bar
+    copy quux (checking bar) git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade (to bar)
+    git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade
+    rsync: connection unexpectedly closed (0 bytes received so far) [sender]
+    rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.7]
+    
+      rsync failed -- run git annex again to resume file transfer
+    failed
+
+Running `git annex upgrade` on bar's machine I get:
+
+    % git annex upgrade
+    upgrade  (v1 to v2) (moving content...) git-annex: Prelude.read: no parse
+
+Again, bar is a bare repo.
+Running the copy job again, I am still getting the same error as above (as expected). Partial contents of annex/objects on bar:
+
+    [...]
+    SHA512:123
+    WORM:SHA512--234
+    [...]
+
+
+-- RichiH

From 30f427700f9e9d59a83775a049e670cc05f2dee6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:55:43 -0400
Subject: [PATCH 1238/8313] converted several commands to use Remote

only move and map still to convert
---
 Command/Describe.hs  | 14 +++++---------
 Command/Semitrust.hs | 13 +++++--------
 Command/Trust.hs     | 13 +++++--------
 Command/Untrust.hs   | 13 +++++--------
 Remote.hs            | 40 +++++++++++++++++++++++++++++++++-------
 UUID.hs              |  9 ++++-----
 test.hs              |  2 +-
 7 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/Command/Describe.hs b/Command/Describe.hs
index 32aef4f245..9e98a81437 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,10 +7,8 @@
 
 module Command.Describe where
 
-
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Messages
 import qualified Command.Init
@@ -30,12 +28,10 @@ start params = notBareRepo $ do
 			_ -> error "Specify a repository and a description."
 	
 	showStart "describe" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r description
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u description
 
-perform :: Git.Repo -> String -> CommandPerform
-perform repo description = do
-	u <- getUUID repo
+perform :: UUID -> String -> CommandPerform
+perform u description = do
 	describeUUID u description
 	return $ Just $ Command.Init.cleanup
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 351336b899..e64d418f83 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -8,8 +8,7 @@
 module Command.Semitrust where
 
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Trust
 import Messages
@@ -24,12 +23,10 @@ seek = [withString start]
 start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "semitrust" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u
 
-perform :: Git.Repo -> CommandPerform
-perform repo = do
-	uuid <- getUUID repo
+perform :: UUID -> CommandPerform
+perform uuid = do
 	trustSet uuid SemiTrusted
 	return $ Just $ return True
diff --git a/Command/Trust.hs b/Command/Trust.hs
index f7dba56485..05505cd045 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -8,8 +8,7 @@
 module Command.Trust where
 
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import Trust
 import UUID
 import Messages
@@ -24,12 +23,10 @@ seek = [withString start]
 start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "trust" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u
 
-perform :: Git.Repo -> CommandPerform
-perform repo = do
-	uuid <- getUUID repo
+perform :: UUID -> CommandPerform
+perform uuid = do
 	trustSet uuid Trusted
 	return $ Just $ return True
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 9c11efe465..311ec6eeb7 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -8,8 +8,7 @@
 module Command.Untrust where
 
 import Command
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Trust
 import Messages
@@ -24,12 +23,10 @@ seek = [withString start]
 start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "untrust" name
-	Remotes.readConfigs
-	r <- Remotes.byName name
-	return $ Just $ perform r
+	u <- Remote.nameToUUID name
+	return $ Just $ perform u
 
-perform :: Git.Repo -> CommandPerform
-perform repo = do
-	uuid <- getUUID repo
+perform :: UUID -> CommandPerform
+perform uuid = do
 	trustSet uuid UnTrusted
 	return $ Just $ return True
diff --git a/Remote.hs b/Remote.hs
index f4b56846b1..64ad85d62e 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -6,12 +6,15 @@
  -}
 
 module Remote (
+	byName,
+	nameToUUID,
 	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID
 ) where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (when, liftM)
 import Data.List
 
 import RemoteClass
@@ -21,6 +24,7 @@ import UUID
 import qualified Annex
 import Trust
 import LocationLog
+import Messages
 
 {- add generators for new Remotes here -}
 generators :: [Annex [Remote Annex]]
@@ -30,7 +34,9 @@ generators = [Remote.GitRemote.generate]
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
 genList = do
-	liftIO $ putStrLn "Remote.genList"
+	g <- Annex.gitRepo
+	u <- getUUID g
+	showNote $ "Remote.genList " ++ u
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
@@ -40,13 +46,24 @@ genList = do
 			return rs'
 		else return rs
 
-{- Filters a list of remotes to ones that have the listed uuids. -}
-remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
-remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
+{- Looks up a remote by name. (Or by UUID.) -}
+byName :: String -> Annex (Remote Annex)
+byName "" = error "no remote specified"
+byName n = do
+	allremotes <- genList
+	let match = filter matching allremotes
+	when (null match) $ error $
+		"there is no git remote named \"" ++ n ++ "\""
+	return $ head match
+	where
+		matching r = n == name r || n == uuid r
 
-{- Filters a list of remotes to ones that do not have the listed uuids. -}
-remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
-remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
+{- Looks up a remote by name (or by UUID), and returns its UUID. -}
+nameToUUID :: String -> Annex UUID
+nameToUUID "." = do -- special case for current repo
+	g <- Annex.gitRepo
+	getUUID g
+nameToUUID n = liftM uuid (byName n)
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
@@ -71,3 +88,12 @@ keyPossibilities key = do
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)
+
+{- Filters a list of remotes to ones that have the listed uuids. -}
+remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
+remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
+
+{- Filters a list of remotes to ones that do not have the listed uuids. -}
+remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
+remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
+
diff --git a/UUID.hs b/UUID.hs
index 3f28434851..5caf110453 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -3,6 +3,9 @@
  - Each git repository used by git-annex has an annex.uuid setting that
  - uniquely identifies that repository.
  -
+ - UUIDs of remotes are cached in git config, using keys named
+ - remote..annex-uuid
+ -
  - Copyright 2010 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
@@ -51,11 +54,7 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 			else []
 
 {- Looks up a repo's UUID. May return "" if none is known.
- -
- - UUIDs of remotes are cached in git config, using keys named
- - remote..annex-uuid
- -
- - -}
+ -}
 getUUID :: Git.Repo -> Annex UUID
 getUUID r = do
 	g <- Annex.gitRepo
diff --git a/test.hs b/test.hs
index 49f7f2ab99..57eb5e664e 100644
--- a/test.hs
+++ b/test.hs
@@ -334,6 +334,7 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
 	trustcheck Trust.SemiTrusted "semitrusted 2"
 	where
+		repo = "origin"
 		trustcheck expected msg = do
 			present <- annexeval $ do
 				Remotes.readConfigs
@@ -342,7 +343,6 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 				u <- UUID.getUUID r
 				return $ u `elem` l
 			assertBool msg present
-		repo = "origin"
 
 test_fsck :: Test
 test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]

From 688e94fd30ca1aa908f5387e488451b9ed2f35e5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 16:58:28 -0400
Subject: [PATCH 1239/8313] convert test to Remote

---
 test.hs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test.hs b/test.hs
index 57eb5e664e..30ebe6e593 100644
--- a/test.hs
+++ b/test.hs
@@ -35,7 +35,7 @@ import qualified GitAnnex
 import qualified LocationLog
 import qualified UUID
 import qualified Trust
-import qualified Remotes
+import qualified Remote
 import qualified Content
 import qualified Command.DropUnused
 import qualified Key
@@ -337,10 +337,8 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 		repo = "origin"
 		trustcheck expected msg = do
 			present <- annexeval $ do
-				Remotes.readConfigs
 				l <- Trust.trustGet expected
-				r <- Remotes.byName repo
-				u <- UUID.getUUID r
+				u <- Remote.nameToUUID repo
 				return $ u `elem` l
 			assertBool msg present
 

From 0d83d17f0455e664f06e3e46b994abadceb7b879 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:00:05 -0400
Subject: [PATCH 1240/8313] convert map to use new code

---
 Command/Map.hs      | 4 ++--
 Remote/GitRemote.hs | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 4ae947b15e..6206aeeb5e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,7 +16,7 @@ import Data.List.Utils
 import Command
 import qualified Annex
 import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote.GitRemote
 import Messages
 import Types
 import Utility
@@ -203,7 +203,7 @@ tryScan r
 				Git.hConfigRead r
 
 		configlist =
-			Remotes.onRemote r (pipedconfig, Nothing) "configlist" []
+			Remote.GitRemote.onRemote r (pipedconfig, Nothing) "configlist" []
 		manualconfiglist = do
 			let sshcmd =
 				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
index 0ec0c70e8d..8671ef7fa2 100644
--- a/Remote/GitRemote.hs
+++ b/Remote/GitRemote.hs
@@ -5,7 +5,10 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.GitRemote (generate) where
+module Remote.GitRemote (
+	generate,
+	onRemote
+) where
 
 import Control.Exception.Extensible
 import Control.Monad.State (liftIO)

From 48418cb92bd2548d333350ac0b7bf2a04540d621 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:12:32 -0400
Subject: [PATCH 1241/8313] reexport RemoteClass from Remote for cleanliness

---
 Backend/File.hs | 17 ++++++++---------
 Remote.hs       |  9 +++++++++
 RemoteClass.hs  |  2 ++
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 9c102cf509..b86413e400 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -21,7 +21,6 @@ import Data.String.Utils
 import BackendClass
 import LocationLog
 import qualified Remote
-import qualified RemoteClass
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -74,16 +73,16 @@ copyKeyFile key file = do
 		-- This check is to avoid an ugly message if a remote is a
 		-- drive that is not mounted.
 		probablyPresent r =
-			if RemoteClass.hasKeyCheap r
+			if Remote.hasKeyCheap r
 				then do
-					res <- RemoteClass.hasKey r key
+					res <- Remote.hasKey r key
 					case res of
 						Right b -> return b
 						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "copying from " ++ RemoteClass.name r ++ "..."
-			copied <- RemoteClass.retrieveKeyFile r key file
+			showNote $ "copying from " ++ Remote.name r ++ "..."
+			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
 				else continue
@@ -109,9 +108,9 @@ checkRemoveKey key numcopiesM = do
 		findcopies need have (r:rs) bad
 			| length have >= need = return True
 			| otherwise = do
-				let u = RemoteClass.uuid r
+				let u = Remote.uuid r
 				let dup = u `elem` have
-				haskey <- (RemoteClass.hasKey r) key
+				haskey <- Remote.hasKey r key
 				case (dup, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
 					(False, Left _)		-> findcopies need have rs (r:bad)
@@ -147,11 +146,11 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [RemoteClass.Remote Annex] -> Annex ()
+showTriedRemotes :: [Remote.Remote Annex] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
-		(join ", " $ map RemoteClass.name remotes)
+		(join ", " $ map Remote.name remotes)
 
 getNumCopies :: Maybe Int -> Annex Int
 getNumCopies (Just n) = return n
diff --git a/Remote.hs b/Remote.hs
index 64ad85d62e..1d9a828d50 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -6,6 +6,15 @@
  -}
 
 module Remote (
+	Remote,
+	uuid,
+	name,
+	storeKey,
+	retrieveKeyFile,
+	removeKey,
+	hasKey,
+	hasKeyCheap,
+
 	byName,
 	nameToUUID,
 	keyPossibilities,
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 9fef0e44a0..38e8407a54 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -1,4 +1,6 @@
 {- git-annex remotes class
+ -
+ - Most things should not need this, using Remote instead
  -
  - Copyright 2011 Joey Hess 
  -

From a70035e981d44a3d5f1dd224be643f4eebb0f243 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:24:20 -0400
Subject: [PATCH 1242/8313] converted move to use Remote

Drop old Remotes.hs, now unused!
---
 Command/Move.hs     |  60 ++++----
 Remote/GitRemote.hs |   9 +-
 Remotes.hs          | 334 --------------------------------------------
 3 files changed, 36 insertions(+), 367 deletions(-)
 delete mode 100644 Remotes.hs

diff --git a/Command/Move.hs b/Command/Move.hs
index 8056e95dbe..907bbf00ef 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -15,8 +15,7 @@ import qualified Annex
 import LocationLog
 import Types
 import Content
-import qualified GitRepo as Git
-import qualified Remotes
+import qualified Remote
 import UUID
 import Messages
 import Utility
@@ -34,16 +33,15 @@ seek = [withFilesInGit $ start True]
  - moving data in the key-value backend. -}
 start :: Bool -> CommandStartString
 start move file = do
-	Remotes.readConfigs
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
 	case (from, to) of
 		(Nothing, Nothing) -> error "specify either --from or --to"
 		(Nothing, Just name) -> do
-			dest <- Remotes.byName name
+			dest <- Remote.byName name
 			toStart dest move file
 		(Just name, Nothing) -> do
-			src <- Remotes.byName name
+			src <- Remote.byName name
 			fromStart src move file
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 
@@ -56,88 +54,86 @@ showAction False file = showStart "copy" file
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot be relied on. For example, it's not done
  - for bare repos. -}
-remoteHasKey :: Git.Repo -> Key -> Bool -> Annex ()
+remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	g <- Annex.gitRepo
-	remoteuuid <- getUUID remote
+	let remoteuuid = Remote.uuid remote
 	logfile <- liftIO $ logChange g key remoteuuid status
 	Annex.queue "add" [Param "--"] logfile
 	where
 		status = if present then ValuePresent else ValueMissing
 
-{- Moves (or copies) the content of an annexed file to another repository,
- - and updates locationlog information on both.
+{- Moves (or copies) the content of an annexed file to a remote.
  -
- - When moving, if the destination already has the content, it is
- - still removed from the current repository.
+ - If the remote already has the content, it is still removed from
+ - the current repository.
  -
  - Note that unlike drop, this does not honor annex.numcopies.
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Git.Repo -> Bool -> CommandStartString
+toStart :: Remote.Remote Annex -> Bool -> CommandStartString
 toStart dest move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
+	u <- getUUID g
 	ishere <- inAnnex key
-	if not ishere || g == dest
+	if not ishere || u == Remote.uuid dest
 		then return Nothing -- not here, so nothing to do
 		else do
 			showAction move file
 			return $ Just $ toPerform dest move key
-toPerform :: Git.Repo -> Bool -> Key -> CommandPerform
+toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
 	-- checking the remote is expensive, so not done in the start step
-	isthere <- Remotes.inAnnex dest key
+	isthere <- Remote.hasKey dest key
 	case isthere of
 		Left err -> do
 			showNote $ show err
 			return Nothing
 		Right False -> do
-			showNote $ "to " ++ Git.repoDescribe dest ++ "..."
-			ok <- Remotes.copyToRemote dest key
+			showNote $ "to " ++ Remote.name dest ++ "..."
+			ok <- Remote.storeKey dest key
 			if ok
 				then return $ Just $ toCleanup dest move key
 				else return Nothing -- failed
 		Right True -> return $ Just $ toCleanup dest move key
-toCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup
+toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
 	remoteHasKey dest key True
 	if move
 		then Command.Drop.cleanup key
 		else return True
 
-{- Moves (or copies) the content of an annexed file from another repository
- - to the current repository and updates locationlog information on both.
+{- Moves (or copies) the content of an annexed file from a remote
+ - to the current repository.
  -
  - If the current repository already has the content, it is still removed
- - from the other repository when moving.
+ - from the remote.
  -}
-fromStart :: Git.Repo -> Bool -> CommandStartString
+fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
-	(remotes, _) <- Remotes.keyPossibilities key
-	if (g == src) || (null $ filter (\r -> Remotes.same r src) remotes)
+	u <- getUUID g
+	(remotes, _) <- Remote.keyPossibilities key
+	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then return Nothing
 		else do
 			showAction move file
 			return $ Just $ fromPerform src move key
-fromPerform :: Git.Repo -> Bool -> Key -> CommandPerform
+fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = do
 	ishere <- inAnnex key
 	if ishere
 		then return $ Just $ fromCleanup src move key
 		else do
-			showNote $ "from " ++ Git.repoDescribe src ++ "..."
-			ok <- getViaTmp key $ Remotes.copyFromRemote src key
+			showNote $ "from " ++ Remote.name src ++ "..."
+			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			if ok
 				then return $ Just $ fromCleanup src move key
 				else return Nothing -- fail
-fromCleanup :: Git.Repo -> Bool -> Key -> CommandCleanup
+fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 fromCleanup src True key = do
-	ok <- Remotes.onRemote src (boolSystem, False) "dropkey" 
-		[ Params "--quiet --force"
-		, Param $ show key
-		]
+	ok <- Remote.removeKey src key
 	-- better safe than sorry: assume the src dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
diff --git a/Remote/GitRemote.hs b/Remote/GitRemote.hs
index 8671ef7fa2..43e75b97bd 100644
--- a/Remote/GitRemote.hs
+++ b/Remote/GitRemote.hs
@@ -47,7 +47,7 @@ genRemote r = do
 		name = Git.repoDescribe r,
 		storeKey = copyToRemote r,
 		retrieveKeyFile = copyFromRemote r,
-		removeKey = error "TODO Remote.GitRemote.removeKey",
+		removeKey = dropKey r,
 		hasKey = inAnnex r,
 		hasKeyCheap = not (Git.repoIsUrl r)
 	}
@@ -159,6 +159,13 @@ inAnnex r key = if Git.repoIsUrl r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
+	
+dropKey :: Git.Repo -> Key -> Annex Bool
+dropKey r key = 
+	onRemote r (boolSystem, False) "dropkey"
+		[ Params "--quiet --force"
+		, Param $ show key
+		]
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
diff --git a/Remotes.hs b/Remotes.hs
deleted file mode 100644
index 7f6a6718b3..0000000000
--- a/Remotes.hs
+++ /dev/null
@@ -1,334 +0,0 @@
-{- git-annex remote repositories
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Remotes (
-	list,
-	readConfigs,
-	keyPossibilities,
-	inAnnex,
-	same,
-	byName,
-	copyFromRemote,
-	copyToRemote,
-	onRemote
-) where
-
-import Control.Exception.Extensible
-import Control.Monad.State (liftIO)
-import qualified Data.Map as Map
-import Data.String.Utils
-import System.Cmd.Utils
-import Data.List (intersect, sortBy)
-import Control.Monad (when, unless, filterM)
-
-import Types
-import qualified GitRepo as Git
-import qualified Annex
-import LocationLog
-import Locations
-import UUID
-import Trust
-import Utility
-import qualified Content
-import Messages
-import CopyFile
-import RsyncFile
-import Ssh
-
-{- Human visible list of remotes. -}
-list :: [Git.Repo] -> String
-list remotes = join ", " $ map Git.repoDescribe remotes 
-
-{- The git configs for the git repo's remotes is not read on startup
- - because reading it may be expensive. This function tries to read the
- - config for a specified remote, and updates state. If successful, it
- - returns the updated git repo. -}
-tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo)
-tryGitConfigRead r 
-	| not $ Map.null $ Git.configMap r = return $ Right r -- already read
-	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsUrl r = return $ Left r
-	| otherwise = store $ safely $ Git.configRead r
-	where
-		-- Reading config can fail due to IO error or
-		-- for other reasons; catch all possible exceptions.
-		safely a = do
-			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
-			case result of
-				Left _ -> return r
-				Right r' -> return r'
-		pipedconfig cmd params = safely $
-			pOpen ReadFromPipe cmd (toCommand params) $
-				Git.hConfigRead r
-		store a = do
-			r' <- a
-			g <- Annex.gitRepo
-			let l = Git.remotes g
-			let g' = Git.remotesAdd g $ exchange l r'
-			Annex.changeState $ \s -> s { Annex.repo = g' }
-			return $ Right r'
-		exchange [] _ = []
-		exchange (old:ls) new =
-			if Git.repoRemoteName old == Git.repoRemoteName new
-				then new : exchange ls new
-				else old : exchange ls new
-
-{- Reads the configs of all remotes.
- -
- - This has to be called before things that rely on eg, the UUID of
- - remotes. Most such things will take care of running this themselves.
- -
- - As reading the config of remotes can be expensive, this
- - function will only read configs once per git-annex run. It's
- - assumed to be cheap to read the config of non-URL remotes,
- - so this is done each time git-annex is run. Conversely,
- - the config of an URL remote is only read when there is no
- - cached UUID value.
- - -}
-readConfigs :: Annex ()
-readConfigs = do
---	remotesread <- Annex.getState Annex.remotesread
-	let remotesread = False
-	unless remotesread $ do
-		g <- Annex.gitRepo
-		allremotes <- filterM repoNotIgnored $ Git.remotes g
-		let cheap = filter (not . Git.repoIsUrl) allremotes
-		let expensive = filter Git.repoIsUrl allremotes
-		doexpensive <- filterM cachedUUID expensive
-		unless (null doexpensive) $
-			showNote $ "getting UUID for " ++
-				list doexpensive ++ "..."
-		let todo = cheap ++ doexpensive
-		unless (null todo) $ do
-			mapM_ tryGitConfigRead todo
---			Annex.changeState $ \s -> s { Annex.remotesread = True }
-	where
-		cachedUUID r = do
-			u <- getUUID r
-			return $ null u
-
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - Also returns a list of UUIDs that are trusted to have the key
- - (some may not have configured remotes).
- -}
-keyPossibilities :: Key -> Annex ([Git.Repo], [UUID])
-keyPossibilities key = do
-	readConfigs
-
-	allremotes <- remotesByCost
-	g <- Annex.gitRepo
-	u <- getUUID g
-	trusted <- trustGet Trusted
-
-	-- get uuids of all repositories that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
-	let validuuids = filter (/= u) uuids
-
-	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
-
-	-- remotes that match uuids that have the key
-	validremotes <- reposByUUID allremotes validuuids
-
-	return (validremotes, validtrusteduuids)
-
-{- Checks if a given remote has the content for a key inAnnex.
- - If the remote cannot be accessed, returns a Left error.
- -}
-inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
-inAnnex r key = if Git.repoIsUrl r
-		then checkremote
-		else liftIO (try checklocal ::IO (Either IOException Bool))
-	where
-		checklocal = do
-			-- run a local check inexpensively,
-			-- by making an Annex monad using the remote
-			a <- Annex.new r []
-			Annex.eval a (Content.inAnnex key)
-		checkremote = do
-			showNote ("checking " ++ Git.repoDescribe r ++ "...")
-			inannex <- onRemote r (boolSystem, False) "inannex" 
-				[Param (show key)]
-			return $ Right inannex
-
-{- Cost Ordered list of remotes. -}
-remotesByCost :: Annex [Git.Repo]
-remotesByCost = do
-	g <- Annex.gitRepo
-	reposByCost $ Git.remotes g
-
-{- Orders a list of git repos by cost. Throws out ignored ones. -}
-reposByCost :: [Git.Repo] -> Annex [Git.Repo]
-reposByCost l = do
-	notignored <- filterM repoNotIgnored l
-	costpairs <- mapM costpair notignored
-	return $ fst $ unzip $ sortBy cmpcost costpairs
-	where
-		costpair r = do
-			cost <- repoCost r
-			return (r, cost)
-		cmpcost (_, c1) (_, c2) = compare c1 c2
-
-{- Calculates cost for a repo.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
- -}
-repoCost :: Git.Repo -> Annex Int
-repoCost r = do
-	cost <- Annex.repoConfig r "cost" ""
-	if not $ null cost
-		then return $ read cost
-		else if Git.repoIsUrl r
-			then return 200
-			else return 100
-
-{- Checks if a repo should be ignored, based either on annex-ignore
- - setting, or on command-line options. Allows command-line to override
- - annex-ignore. -}
-repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = do
-	ignored <- Annex.repoConfig r "ignore" "false"
-	to <- match Annex.toremote
-	from <- match Annex.fromremote
-	if to || from
-		then return True
-		else return $ not $ Git.configTrue ignored
-	where
-		match a = do
-			name <- Annex.getState a
-			case name of
-				Nothing -> return False
-				n -> return $ n == Git.repoRemoteName r
-
-{- Checks if two repos are the same, by comparing their remote names. -}
-same :: Git.Repo -> Git.Repo -> Bool
-same a b = Git.repoRemoteName a == Git.repoRemoteName b
-
-{- Looks up a remote by name. (Or by UUID.) -}
-byName :: String -> Annex Git.Repo
-byName "." = Annex.gitRepo -- special case to refer to current repository
-byName name = do
-	when (null name) $ error "no remote specified"
-	g <- Annex.gitRepo
-	match <- filterM matching $ Git.remotes g
-	when (null match) $ error $
-		"there is no git remote named \"" ++ name ++ "\""
-	return $ head match
-	where
-		matching r = do
-			if Just name == Git.repoRemoteName r
-				then return True
-				else do
-					u <- getUUID r
-					return $ (name == u)
-
-{- Tries to copy a key's content from a remote's annex to a file. -}
-copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
-copyFromRemote r key file
-	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
-	| Git.repoIsSsh r = rsynchelper r True key file
-	| otherwise = error "copying from non-ssh repo not supported"
-
-{- Tries to copy a key's content to a remote's annex. -}
-copyToRemote :: Git.Repo -> Key -> Annex Bool
-copyToRemote r key
-	| not $ Git.repoIsUrl r = do
-		g <- Annex.gitRepo
-		let keysrc = gitAnnexLocation g key
-		-- run copy from perspective of remote
-		liftIO $ do
-			a <- Annex.new r []
-			Annex.eval a $ do
-				ok <- Content.getViaTmp key $
-					\f -> liftIO $ copyFile keysrc f
-				Annex.queueRun
-				return ok
-	| Git.repoIsSsh r = do
-		g <- Annex.gitRepo
-		let keysrc = gitAnnexLocation g key
-		rsynchelper r False key keysrc
-	| otherwise = error "copying to non-ssh repo not supported"
-
-rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool)
-rsynchelper r sending key file = do
-	showProgress -- make way for progress bar
-	p <- rsyncParams r sending key file
-	res <- liftIO $ boolSystem "rsync" p
-	if res
-		then return res
-		else do
-			showLongNote "rsync failed -- run git annex again to resume file transfer"
-			return res
-
-{- Generates rsync parameters that ssh to the remote and asks it
- - to either receive or send the key's content. -}
-rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
-rsyncParams r sending key file = do
-	Just (shellcmd, shellparams) <- git_annex_shell r
-		(if sending then "sendkey" else "recvkey")
-		[ Param $ show key
-		-- Command is terminated with "--", because
-		-- rsync will tack on its own options afterwards,
-		-- and they need to be ignored.
-		, Param "--"
-		]
-	-- Convert the ssh command into rsync command line.
-	let eparam = rsyncShell (Param shellcmd:shellparams)
-	o <- Annex.repoConfig r "rsync-options" ""
-	let base = options ++ map Param (words o) ++ eparam
-	if sending
-		then return $ base ++ [dummy, File file]
-		else return $ base ++ [File file, dummy]
-	where
-		-- inplace makes rsync resume partial files
-		options = [Params "-p --progress --inplace"]
-		-- the rsync shell parameter controls where rsync
-		-- goes, so the source/dest parameter can be a dummy value,
-		-- that just enables remote rsync mode.
-		dummy = Param ":"
-
-{- Uses a supplied function to run a git-annex-shell command on a remote.
- -
- - Or, if the remote does not support running remote commands, returns
- - a specified error value. -}
-onRemote 
-	:: Git.Repo
-	-> (FilePath -> [CommandParam] -> IO a, a)
-	-> String
-	-> [CommandParam]
-	-> Annex a
-onRemote r (with, errorval) command params = do
-	s <- git_annex_shell r command params
-	case s of
-		Just (c, ps) -> liftIO $ with c ps
-		Nothing -> return errorval
-
-{- Generates parameters to run a git-annex-shell command on a remote. -}
-git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam]))
-git_annex_shell r command params
-	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
-	| Git.repoIsSsh r = do
-		sshparams <- sshToRepo r [Param sshcmd]
-		return $ Just ("ssh", sshparams)
-	| otherwise = return Nothing
-	where
-		dir = Git.workTree r
-		shellcmd = "git-annex-shell"
-		shellopts = (Param command):(File dir):params
-		sshcmd = shellcmd ++ " " ++ 
-			unwords (map shellEscape $ toCommand shellopts)
-
-{- Filters a list of repos to ones that have listed UUIDs. -}
-reposByUUID :: [Git.Repo] -> [UUID] -> Annex [Git.Repo]
-reposByUUID repos uuids = filterM match repos
-	where
-		match r = do
-			u <- getUUID r
-			return $ u `elem` uuids
-

From 45dbfbd02f20c211f70061a93d382ee333ef27ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:26:46 -0400
Subject: [PATCH 1243/8313] remove debug

---
 Remote.hs | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 1d9a828d50..a7136ea651 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -33,7 +33,6 @@ import UUID
 import qualified Annex
 import Trust
 import LocationLog
-import Messages
 
 {- add generators for new Remotes here -}
 generators :: [Annex [Remote Annex]]
@@ -43,9 +42,6 @@ generators = [Remote.GitRemote.generate]
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
 genList = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	showNote $ "Remote.genList " ++ u
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do

From f8693facabdaa81cbebab7141151d306615fd6a5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:30:44 -0400
Subject: [PATCH 1244/8313] doc update

---
 debian/changelog  | 2 ++
 doc/backends.mdwn | 9 +++------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e9bc896bf0..e995009db0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (0.20110326) UNRELEASED; urgency=low
 
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+  * Generalized remotes handling, laying groundwork for remotes that are
+    not regular git remotes.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 5d02ad3a1c..22164850ab 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -9,16 +9,13 @@ to retrieve the file's content (its value).
 Multiple pluggable backends are supported, and a single repository
 can use different backends for different files.
 
-* `WORM` ("Write Once, Read Many") This backend stores the file's content
-  only in `.git/annex/objects/`, and assumes that any file with the same
-  basename, size, and modification time has the same content. So with
+* `WORM` ("Write Once, Read Many") This backend assumes that any file with
+  the same basename, size, and modification time has the same content. So with
   this backend, files can be moved around, but should never be added to
   or changed. This is the default, and the least expensive backend.
-* `SHA1` -- This backend stores the file's content in
-  `.git/annex/objects/`, with a name based on its sha1 checksum. This backend
+* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend
   allows modifications of files to be tracked. Its need to generate checksums
   can make it slower for large files.
-  for use.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
   researching checksum collisions and wants to annex their colliding data. ;)

From fb47e884040c3d95ceaa9e9bbc442fdf14abdd3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 17:45:11 -0400
Subject: [PATCH 1245/8313] revamp s3 design

looking very doable now
---
 doc/todo/S3.mdwn | 103 ++++++++++++-----------------------------------
 1 file changed, 25 insertions(+), 78 deletions(-)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 946fa68170..09a64f1a7a 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -2,90 +2,37 @@ Support Amazon S3 as a file storage backend.
 
 There's a haskell library that looks good. Not yet in Debian.
 
-Multiple ways of using S3 are possible. Current plan is to have a S3BUCKET
-backend, that is derived from Backend.File, so it caches files locally and
-can transfer files between systems too, without involving S3.
+Multiple ways of using S3 are possible. Current plan is to 
+have a special type of git remote (though git won't know how to use it;
+only git-annex will) that uses a S3 bucket.
 
-get will try to get it from S3 or from a remote. A annex.s3.cost can
-configure the cost of S3 vs the cost of other remotes.
+Something like:
 
-add will always upload a copy to S3.
+	[remote "s3"]
+		annex-s3bucket = mybucket
+		annex-s3datacenter = Europe
+		annex-uuid = 1a586cf6-45e9-11e0-ba9c-3b0a3397aec2
+		annex-cost = 500
 
-Each file in the S3 bucket is assumed to be in the annex. So unused
-will show files in the bucket that nothing points to, and dropunused remove
-them.
+The UUID will be stored as a special file in the S3 bucket.
 
-For numcopies counting, S3 will count as 1 copy (or maybe more?), so if
-numcopies=2, then you don't fully trust S3 and request git-annex assure
-one other copy.
+Using a different type of remote like this will allow S3 to be used
+anywhere a regular remote would be used. `git annex get` will transparently
+download a file from S3 if S3 has it and is the cheapest remote.
 
-drop will remove a file locally, but keep it in S3. drop --force *might*
-remove it from S3. TBD.
+	git annex copy --to s3
+	git annex move --from s3
+	git annex drop --from s3 # not currently allowed, will need adding
 
-annex.s3.bucket would configure the bucket the use. (And an env var or
-something configure the password.) Although the bucket
-would also be encoded in the keys. So, the configured bucket would be used
-when adding new files. A system could move from one bucket to another over
-time while still having legacy files in an earlier one; 
-perhaps you move to Europe and want new files to be put in that region.
+Each s3 remote will count as one copy for numcopies handling, just like
+any other remote.
 
-And git annex `migrate --backend=S3BUCKET --force` could move files
-between datacenters!
+## unused checking
 
-Problem: Then the only way for unused to know what buckets are in use
-is to see what keys point to them -- but if the last file from a bucket is
-deleted, it would then not be able to say that the files in that bucket are
-all unused. Need cached list of recently seen S3 buckets?
+One problem is `git annex unused`. Currently it only looks at the local
+repository, not remotes. But if something is dropped from the local repo,
+and you forget to drop it from S3, cruft can build up there.
 
------
-
-One problem with this is what key metadata to include. Should it be like
-WORM? Or like SHA1? Or just a new unique identifier for each file? It might
-be worth having S3 variants of *all* the Backend.File derived backends.
-
-More blue-sky, it might be nice to be able to union or stack together
-multiple backends, so S3BUCKET+SHA1 or S3BUCKET+WORM. That would likely
-be hard to get right.
-
-Less blue-sky, if the S3 capability were added directly to Backend.File,
-and bucket name was configured by annex.s3.bucket, then any existing
-annexed file could be upgraded to also store on S3.
-
-## alternate approach
-
-The above assumes S3 should be a separate backend somehow. What if,
-instead a S3 bucket is treated as a separate **remote**.
-
-* Could "git annex add" while offline, and "git annex push --to S3" when
-  online.
-* No need to choose whether a file goes to S3 at add time; no need to
-  migrate to move files there.
-* numcopies counting Just Works
-* Could have multiple S3 buckets as desired.
-
-The bucket name could 1:1 map with its annex.uuid, so not much
-configuration would be needed when cloning a repo to get it using S3 --
-just configure the S3 access token(s) to use for various UUIDs.
-
-Implementing this might not be as conceptually nice as making S3 a separate
-backend. It would need some changes to the remotes code, perhaps lifting
-some of it into backend-specific hooks. Then the S3 backend could be
-implicitly stacked in front of a backend like WORM.
-
----
-
-Maybe the right way to look at this is that a list of Stores
-should be a property of the Backend. Backend.File is a Backend, that
-uses various Stores, which can be of different types (the local
-git repo, remote git repos, S3, etc). Backend.URL is a backend that uses
-other Stores (the local git repo, and the web).
-
-Operations on Stores are:
-
-* uuid -- each store has a unique uuid value
-* cost -- each store has a use cost value
-* getConfig -- attempts to look up values (uuid, possibly cost)
-* copyToStore -- store a file's contents to a key
-* copyFromStore -- retrieve a key's contents to a file
-* removeFromStore -- removes a key's contents from the store
-* hasKey -- checks if the key's content is available
+This could be fixed by adding a hook to list all keys present in a remote.
+Then unused could scan remotes for keys, and if they were not used locally,
+offer the possibility to drop them from the remote.

From 9a4127f0fed9fa9cd684fff78b3a2af9da3c62ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 18:02:42 -0400
Subject: [PATCH 1246/8313] response

---
 Upgrade/V1.hs                                              | 1 +
 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c0470a3bc8..a876142220 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -226,6 +226,7 @@ getKeysPresent1' dir = do
 			return $ map fileKey1 files
 	where
 		present d = do
+			liftIO $ putStrLn $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
 			result <- try $
 				getFileStatus $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
 			case result of
diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 16a0ca3744..1c18e585c7 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -26,3 +26,8 @@ Running the copy job again, I am still getting the same error as above (as expec
 
 
 -- RichiH
+
+> Upgrading bare repos to v2 generally works fine, so I actually need
+> to see the full content of annex/, not a fragment, in order to debug this.
+> (Filename contents I don't need to see.) Feel free to email me the details at
+> joey@kitenet.net if you don't want to post them here. --[[Joey]]

From 4868b64868747455a9c5d512650f9e7074e6009e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 18:34:30 -0400
Subject: [PATCH 1247/8313] Provide a less expensive version of `git annex copy
 --to`, enabled via --fast. This assumes that location tracking information is
 correct, rather than contacting the remote for every file.

---
 Command/Move.hs                               | 10 ++++++++--
 debian/changelog                              |  3 +++
 ...batch_check_on_remote_when_using_copy.mdwn | 19 +++++++++++++++++++
 doc/git-annex.mdwn                            | 19 ++++++++++++-------
 4 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 907bbf00ef..3ac5a7ab2c 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -84,8 +84,14 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 			return $ Just $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
-	-- checking the remote is expensive, so not done in the start step
-	isthere <- Remote.hasKey dest key
+	-- Checking the remote is expensive, so not done in the start step.
+	-- In fast mode, location tracking is assumed to be correct,
+	-- and an explicit check is not done, when copying. When moving,
+	-- it has to be done, to avoid inaverdent data loss.
+	fast <- Annex.getState Annex.fast
+	isthere <- if fast && not move
+		then return $ Right True
+		else Remote.hasKey dest key
 	case isthere of
 		Left err -> do
 			showNote $ show err
diff --git a/debian/changelog b/debian/changelog
index e995009db0..2f532784d4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (0.20110326) UNRELEASED; urgency=low
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
   * Generalized remotes handling, laying groundwork for remotes that are
     not regular git remotes.
+  * Provide a less expensive version of `git annex copy --to`, enabled
+    via --fast. This assumes that location tracking information is correct,
+    rather than contacting the remote for every file.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
index 0f20ab6454..b08c33b8ba 100644
--- a/doc/forum/batch_check_on_remote_when_using_copy.mdwn
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -6,3 +6,22 @@ Once all checks are done, one single transfer session should be started. Creatin
 
 
 -- RichiH
+
+> (Use of SHA is irrelevant here, copy does not checksum anything.)
+> 
+> I think what you're seeing is
+> that `git annex copy --to remote` is slow, going to the remote repository
+> every time to see if it has the file, while `git annex copy --from remote`
+> is fast, since it looks at what files are locally present.
+> 
+> That is something I mean to improve. At least `git annex copy --fast --to remote`
+> could easily do a fast copy of all files that are known to be missing from
+> the remote repository. When local and remote git repos are not 100% in sync,
+> relying on that data could miss some files that the remote doesn't have anymore,
+> but local doesn't know it dropped. That's why it's a candidate for `--fast`.
+> 
+> I've just implemented that.
+> 
+> While I do hope to improve ssh usage so that it sshs once, and feeds
+> `git-annex-shell` a series of commands to run, that is a much longer-term
+> thing. --[[Joey]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 32f190e75d..8afe93c109 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -84,20 +84,22 @@ Many git-annex commands will stage changes for later `git commit` by you.
   it is safe to do so, typically because of the setting of annex.numcopies.
 
 * move [path ...]
+  
+  When used with the --from option, moves the content of annexed files
+  from the specified repository to the current one.
 
   When used with the --to option, moves the content of annexed files from
   the current repository to the specified one.
 
-  When used with the --from option, moves the content of annexed files
-  from the specified repository to the current one.
-
 * copy [path ...]
 
+  When used with the --from option, copies the content of annexed files
+  from the specified repository to the current one.
+
   When used with the --to option, copies the content of annexed files from
   the current repository to the specified one.
 
-  When used with the --from option, copies the content of annexed files
-  from the specified repository to the current one.
+  To avoid contacting the remote to check if it has every file, specify --fast
 
 * unlock [path ...]
 
@@ -137,11 +139,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   With parameters, only the specified files are checked.
 
+  To avoid expensive checksum calculations, specify --fast
+
 * unused
 
   Checks the annex for data that is not used by any files currently
   in the annex, and prints a numbered list of the data.
 
+  To only show unused temp files, specify --fast
+
 * dropunused [number ...]
 
   Drops the data corresponding to the numbers, as listed by the last
@@ -286,8 +292,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * --fast
 
   Enables less expensive, but also less thorough versions of some commands.
-  What is avoided depends on the command. A fast fsck avoids calculating
-  checksums; a fast unused only shows temp files and not other unused files.
+  What is avoided depends on the command.
 
 * --quiet
 

From be3f6a9acf6cfea9fcba861bf51e10a59039d575 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 27 Mar 2011 23:19:45 +0000
Subject: [PATCH 1248/8313]

---
 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 1c18e585c7..1218b0e517 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -31,3 +31,5 @@ Running the copy job again, I am still getting the same error as above (as expec
 > to see the full content of annex/, not a fragment, in order to debug this.
 > (Filename contents I don't need to see.) Feel free to email me the details at
 > joey@kitenet.net if you don't want to post them here. --[[Joey]]
+
+>> Sent. -- RichiH

From 28bf28a73c503c7c2d9add38e964149355bb9e50 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 19:23:00 -0400
Subject: [PATCH 1249/8313] rename

---
 Command/Map.hs                  | 4 ++--
 Remote.hs                       | 5 +++--
 Remote/{GitRemote.hs => Git.hs} | 0
 3 files changed, 5 insertions(+), 4 deletions(-)
 rename Remote/{GitRemote.hs => Git.hs} (100%)

diff --git a/Command/Map.hs b/Command/Map.hs
index 6206aeeb5e..dc3acb56e4 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,7 +16,7 @@ import Data.List.Utils
 import Command
 import qualified Annex
 import qualified GitRepo as Git
-import qualified Remote.GitRemote
+import qualified Remote.Git
 import Messages
 import Types
 import Utility
@@ -203,7 +203,7 @@ tryScan r
 				Git.hConfigRead r
 
 		configlist =
-			Remote.GitRemote.onRemote r (pipedconfig, Nothing) "configlist" []
+			Remote.Git.onRemote r (pipedconfig, Nothing) "configlist" []
 		manualconfiglist = do
 			let sshcmd =
 				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
diff --git a/Remote.hs b/Remote.hs
index a7136ea651..b3f1a0c6b8 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -27,7 +27,8 @@ import Control.Monad (when, liftM)
 import Data.List
 
 import RemoteClass
-import qualified Remote.GitRemote
+import qualified Remote.Git
+import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
@@ -36,7 +37,7 @@ import LocationLog
 
 {- add generators for new Remotes here -}
 generators :: [Annex [Remote Annex]]
-generators = [Remote.GitRemote.generate]
+generators = [Remote.Git.generate]
 
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached in the Annex. -}
diff --git a/Remote/GitRemote.hs b/Remote/Git.hs
similarity index 100%
rename from Remote/GitRemote.hs
rename to Remote/Git.hs

From 221251b4584824637db2c97b97efb919c9dd57fe Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 01:19:22 +0000
Subject: [PATCH 1250/8313] Added a comment

---
 ..._97848f9a3db89c0427cfb671ba13300e._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment b/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment
new file mode 100644
index 0000000000..be34473c0a
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-28T01:16:21Z"
+ content="""
+Another nice thing would be a summary of _what_ is wrong. I.e.
+
+    % git fsck
+    [...]
+    git-annex: 100 total failed
+      50 checksum failed
+      50 not enough copies exit
+
+And the same/similar for all other failure modes.
+
+
+-- RichiH
+"""]]

From 029a50a7ce4ff8032fd1d24d94f4e97c667455ce Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 01:22:25 +0000
Subject: [PATCH 1251/8313] Added a comment

---
 ..._ea3c828c5f51c3fcca2de16bce4a7561._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment b/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
new file mode 100644
index 0000000000..dced87287a
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-03-28T01:19:25Z"
+ content="""
+Another nice thing would be a summary of _what_ is wrong. I.e.
+
+    % git fsck
+    [...]
+    git-annex: 100 total failed
+      50 checksum failed
+      50 not enough copies exit
+
+And the same/similar for all other failure modes.
+
+
+-- RichiH
+"""]]

From 68005f25f9446603ae84cae109ea76a69a2bd156 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 01:27:05 +0000
Subject: [PATCH 1252/8313] removed

---
 ..._ea3c828c5f51c3fcca2de16bce4a7561._comment | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment

diff --git a/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment b/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
deleted file mode 100644
index dced87287a..0000000000
--- a/doc/forum/Behaviour_of_fsck/comment_4_ea3c828c5f51c3fcca2de16bce4a7561._comment
+++ /dev/null
@@ -1,19 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
- nickname="Richard"
- subject="comment 4"
- date="2011-03-28T01:19:25Z"
- content="""
-Another nice thing would be a summary of _what_ is wrong. I.e.
-
-    % git fsck
-    [...]
-    git-annex: 100 total failed
-      50 checksum failed
-      50 not enough copies exit
-
-And the same/similar for all other failure modes.
-
-
--- RichiH
-"""]]

From 6b5918c295715d0599005c9367f5dab5468169c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 21:43:25 -0400
Subject: [PATCH 1253/8313] some reorg and further remote generalization

---
 Annex.hs       |  23 -----------
 Config.hs      |  68 +++++++++++++++++++++++++++++++
 Content.hs     |   3 +-
 GitRepo.hs     |  30 ++++++++++----
 Remote.hs      |  24 ++++++++---
 Remote/Git.hs  | 106 ++++++++++++++-----------------------------------
 RemoteClass.hs |   5 +++
 Ssh.hs         |   4 +-
 UUID.hs        |   5 ++-
 Version.hs     |   3 +-
 10 files changed, 154 insertions(+), 117 deletions(-)
 create mode 100644 Config.hs

diff --git a/Annex.hs b/Annex.hs
index bb26608f4a..2723c6a008 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,12 +17,9 @@ module Annex (
 	queue,
 	queueRun,
 	queueRunAt,
-	setConfig,
-	repoConfig
 ) where
 
 import Control.Monad.State
-import Data.Maybe
 
 import qualified GitRepo as Git
 import qualified GitQueue
@@ -119,23 +116,3 @@ queueRunAt n = do
 	state <- get
 	let q = repoqueue state
 	when (GitQueue.size q >= n) queueRun
-
-{- Changes a git config setting in both internal state and .git/config -}
-setConfig :: String -> String -> Annex ()
-setConfig k value = do
-	g <- Annex.gitRepo
-	liftIO $ Git.run g "config" [Param k, Param value]
-	-- re-read git config and update the repo's state
-	g' <- liftIO $ Git.configRead g
-	Annex.changeState $ \s -> s { Annex.repo = g' }
-
-{- Looks up a per-remote config option in git config.
- - Failing that, tries looking for a global config option. -}
-repoConfig :: Git.Repo -> String -> String -> Annex String
-repoConfig r key def = do
-	g <- Annex.gitRepo
-	let def' = Git.configGet g global def
-	return $ Git.configGet g local def'
-	where
-		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
-		global = "annex." ++ key
diff --git a/Config.hs b/Config.hs
new file mode 100644
index 0000000000..aae7d82915
--- /dev/null
+++ b/Config.hs
@@ -0,0 +1,68 @@
+{- Git configuration
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Config where
+
+import Data.Maybe
+import Control.Monad.State (liftIO)
+
+import qualified GitRepo as Git
+import qualified Annex
+import Types
+import Utility
+
+type ConfigKey = String
+
+{- Changes a git config setting in both internal state and .git/config -}
+setConfig :: ConfigKey -> String -> Annex ()
+setConfig k value = do
+	g <- Annex.gitRepo
+	liftIO $ Git.run g "config" [Param k, Param value]
+	-- re-read git config and update the repo's state
+	g' <- liftIO $ Git.configRead g
+	Annex.changeState $ \s -> s { Annex.repo = g' }
+
+{- Looks up a per-remote config setting in git config.
+ - Failing that, tries looking for a global config option. -}
+getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
+getConfig r key def = do
+	g <- Annex.gitRepo
+	let def' = Git.configGet g global def
+	return $ Git.configGet g local def'
+	where
+		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
+		global = "annex." ++ key
+
+{- Calculates cost for a remote.
+ -
+ - The default cost is 100 for local repositories, and 200 for remote
+ - repositories; it can also be configured by remote..annex-cost
+ -}
+remoteCost :: Git.Repo -> Annex Int
+remoteCost r = do
+	c <- getConfig r "cost" ""
+	if not $ null c
+		then return $ read c
+		else if not $ Git.repoIsUrl r
+			then return 100
+			else return 200
+
+{- Checks if a repo should be ignored, based either on annex-ignore
+ - setting, or on command-line options. Allows command-line to override
+ - annex-ignore. -}
+remoteNotIgnored :: Git.Repo -> Annex Bool
+remoteNotIgnored r = do
+	ignored <- getConfig r "ignore" "false"
+	to <- match Annex.toremote
+	from <- match Annex.fromremote
+	if to || from
+		then return True
+		else return $ not $ Git.configTrue ignored
+	where
+		match a = do
+			n <- Annex.getState a
+			return $ n == Git.repoRemoteName r
diff --git a/Content.hs b/Content.hs
index 7aa30f7ffc..88e8dbc007 100644
--- a/Content.hs
+++ b/Content.hs
@@ -40,6 +40,7 @@ import Utility
 import StatFS
 import Key
 import DataUnits
+import Config
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
@@ -121,7 +122,7 @@ checkDiskSpace = checkDiskSpace' 0
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
-	r <- Annex.repoConfig g "diskreserve" ""
+	r <- getConfig g "diskreserve" ""
 	let reserve = case readSize dataUnits r of
 		Nothing -> megabyte
 		Just v -> v
diff --git a/GitRepo.hs b/GitRepo.hs
index ad58b28a00..1b14e4a636 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -12,6 +12,7 @@ module GitRepo (
 	Repo,
 	repoFromCwd,
 	repoFromAbsPath,
+	repoFromUnknown,
 	repoFromUrl,
 	localToUrl,
 	repoIsUrl,
@@ -41,6 +42,7 @@ module GitRepo (
 	remotes,
 	remotesAdd,
 	repoRemoteName,
+	repoRemoteNameSet,
 	inRepo,
 	notInRepo,
 	stagedFiles,
@@ -81,7 +83,7 @@ import Utility
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
-data RepoLocation = Dir FilePath | Url URI
+data RepoLocation = Dir FilePath | Url URI | Unknown
 	deriving (Show, Eq)
 
 data Repo = Repo {
@@ -123,6 +125,10 @@ repoFromUrl url
 				Just v -> v
 				Nothing -> error $ "bad url " ++ url
 
+{- Creates a repo that has an unknown location. -}
+repoFromUnknown :: Repo
+repoFromUnknown = newFrom Unknown
+
 {- Converts a Local Repo into a remote repo, using the reference repo
  - which is assumed to be on the same host. -}
 localToUrl :: Repo -> Repo -> Repo
@@ -141,11 +147,13 @@ repoDescribe :: Repo -> String
 repoDescribe Repo { remoteName = Just name } = name
 repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
+repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url
 repoLocation Repo { location = Dir dir } = dir
+repoLocation Repo { location = Unknown } = undefined
 
 {- Constructs and returns an updated version of a repo with
  - different remotes list. -}
@@ -158,6 +166,14 @@ repoRemoteName :: Repo -> Maybe String
 repoRemoteName Repo { remoteName = Just name } = Just name
 repoRemoteName _ = Nothing
 
+{- Sets the name of a remote based on the git config key, such as
+   "remote.foo.url". -}
+repoRemoteNameSet :: Repo -> String -> Repo
+repoRemoteNameSet r k = r { remoteName = Just basename }
+	where
+		basename = join "." $ reverse $ drop 1 $
+				reverse $ drop 1 $ split "." k
+
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
 repoIsUrl :: Repo -> Bool
@@ -218,6 +234,7 @@ gitDir repo
 workTree :: Repo -> FilePath
 workTree r@(Repo { location = Url _ }) = urlPath r
 workTree (Repo { location = Dir d }) = d
+workTree Repo { location = Unknown } = undefined
 
 {- Given a relative or absolute filename in a repository, calculates the
  - name to use to refer to the file relative to a git repository's top.
@@ -393,10 +410,6 @@ configStore repo s = do
 	where
 		r = repo { config = configParse s }
 
-{- Checks if a string from git config is a true value. -}
-configTrue :: String -> Bool
-configTrue s = map toLower s == "true"
-
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 configRemotes :: Repo -> IO [Repo]
 configRemotes repo = mapM construct remotepairs
@@ -404,10 +417,9 @@ configRemotes repo = mapM construct remotepairs
 		remotepairs = Map.toList $ filterremotes $ config repo
 		filterremotes = Map.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
-		remotename k = join "." $ reverse $ drop 1 $ reverse $ drop 1 $ split "." k
 		construct (k,v) = do
 			r <- gen v
-			return $ r { remoteName = Just $ remotename k }
+			return $ repoRemoteNameSet r k
 		gen v	| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
@@ -423,6 +435,10 @@ configRemotes repo = mapM construct remotepairs
 					| d !! 0 == '~' = '/':dir
 					| otherwise = "/~/" ++ dir
 
+{- Checks if a string from git config is a true value. -}
+configTrue :: String -> Bool
+configTrue s = map toLower s == "true"
+
 {- Parses git config --list output into a config map. -}
 configParse :: String -> Map.Map String String
 configParse s = Map.fromList $ map pair $ lines s
diff --git a/Remote.hs b/Remote.hs
index b3f1a0c6b8..5508e0d123 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -25,20 +25,35 @@ module Remote (
 import Control.Monad.State (liftIO)
 import Control.Monad (when, liftM)
 import Data.List
+import Data.String.Utils
 
 import RemoteClass
 import qualified Remote.Git
-import qualified Remote.S3
+--import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
 import Trust
 import LocationLog
+import Messages
 
-{- add generators for new Remotes here -}
-generators :: [Annex [Remote Annex]]
+{- Add generators for new Remotes here. -}
+generators :: [Annex (RemoteGenerator Annex)]
 generators = [Remote.Git.generate]
 
+{- Runs a list of generators. -}
+runGenerators :: [Annex (RemoteGenerator Annex)] -> Annex [Remote Annex]
+runGenerators gs = do
+	(actions, expensive) <- collect ([], []) gs
+	when (not $ null expensive) $
+		showNote $ "getting UUID for " ++ join ", " expensive
+	sequence actions
+	where
+		collect v [] = return v
+		collect (actions, expensive) (x:xs) = do
+			(a, e) <- x
+			collect (a++actions, e++expensive) xs
+
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
@@ -46,8 +61,7 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			lists <- sequence generators
-			let rs' = concat lists
+			rs' <- runGenerators generators
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 43e75b97bd..9021a2230c 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.GitRemote (
+module Remote.Git (
 	generate,
 	onRemote
 ) where
@@ -13,9 +13,8 @@ module Remote.GitRemote (
 import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as Map
-import Data.String.Utils
 import System.Cmd.Utils
-import Control.Monad (unless, filterM)
+import Control.Monad (filterM, liftM)
 
 import RemoteClass
 import Types
@@ -29,18 +28,34 @@ import Messages
 import CopyFile
 import RsyncFile
 import Ssh
+import Config
 
-generate :: Annex [Remote Annex]
+generate :: Annex (RemoteGenerator Annex)
 generate = do
-	readConfigs
 	g <- Annex.gitRepo
-	rs <- filterM repoNotIgnored (Git.remotes g)
-	mapM genRemote rs
+	allremotes <- filterM remoteNotIgnored $ Git.remotes g
+
+ 	{- It's assumed to be cheap to read the config of non-URL remotes,
+	 - so this is done each time git-annex is run. Conversely,
+	 - the config of an URL remote is only read when there is no
+	 - cached UUID value. -}
+	let cheap = filter (not . Git.repoIsUrl) allremotes
+	let expensive = filter Git.repoIsUrl allremotes
+	expensive_todo <- filterM cachedUUID expensive
+	let skip = filter (`notElem` expensive_todo) expensive
+	let todo = cheap++expensive_todo
+
+	let actions = map genRemote skip ++ 
+		map (\r -> genRemote =<< tryGitConfigRead r) todo
+	return (actions, map Git.repoDescribe expensive_todo)
+
+	where
+		cachedUUID r = liftM null $ getUUID r
 	
 genRemote :: Git.Repo -> Annex (Remote Annex)
 genRemote r = do
 	u <- getUUID r
-	c <- repoCost r
+	c <- remoteCost r
 	return Remote {
 		uuid = u,
 		cost = c,
@@ -52,40 +67,13 @@ genRemote r = do
 		hasKeyCheap = not (Git.repoIsUrl r)
 	}
 
-{- Reads the configs of git remotes.
- -
- - It's assumed to be cheap to read the config of non-URL remotes,
- - so this is done each time git-annex is run. Conversely,
- - the config of an URL remote is only read when there is no
- - cached UUID value.
- -}
-readConfigs :: Annex ()
-readConfigs = do
-	g <- Annex.gitRepo
-	allremotes <- filterM repoNotIgnored $ Git.remotes g
-	let cheap = filter (not . Git.repoIsUrl) allremotes
-	let expensive = filter Git.repoIsUrl allremotes
-	doexpensive <- filterM cachedUUID expensive
-	unless (null doexpensive) $
-		showNote $ "getting UUID for " ++
-			list doexpensive ++ "..."
-	let todo = cheap ++ doexpensive
-	unless (null todo) $ do
-		mapM_ tryGitConfigRead todo
-	where
-		cachedUUID r = do
-			u <- getUUID r
-			return $ null u
-
-{- The git configs for the git repo's remotes is not read on startup
- - because reading it may be expensive. This function tries to read the
- - config for a specified remote, and updates state. If successful, it
- - returns the updated git repo. -}
-tryGitConfigRead :: Git.Repo -> Annex (Either Git.Repo Git.Repo)
+{- Tries to read the config for a specified remote, updates state, and
+ - returns the updated repo. -}
+tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
-	| not $ Map.null $ Git.configMap r = return $ Right r -- already read
+	| not $ Map.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsUrl r = return $ Left r
+	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
@@ -104,43 +92,13 @@ tryGitConfigRead r
 			let l = Git.remotes g
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
-			return $ Right r'
+			return r'
 		exchange [] _ = []
 		exchange (old:ls) new =
 			if Git.repoRemoteName old == Git.repoRemoteName new
 				then new : exchange ls new
 				else old : exchange ls new
 
-{- Calculates cost for a repo.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
- -}
-repoCost :: Git.Repo -> Annex Int
-repoCost r = do
-	c <- Annex.repoConfig r "cost" ""
-	if not $ null c
-		then return $ read c
-		else if Git.repoIsUrl r
-			then return 200
-			else return 100
-
-{- Checks if a repo should be ignored, based either on annex-ignore
- - setting, or on command-line options. Allows command-line to override
- - annex-ignore. -}
-repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = do
-	ignored <- Annex.repoConfig r "ignore" "false"
-	to <- match Annex.toremote
-	from <- match Annex.fromremote
-	if to || from
-		then return True
-		else return $ not $ Git.configTrue ignored
-	where
-		match a = do
-			n <- Annex.getState a
-			return $ n == Git.repoRemoteName r
-
 {- Checks if a given remote has the content for a key inAnnex.
  - If the remote cannot be accessed, returns a Left error.
  -}
@@ -219,7 +177,7 @@ rsyncParams r sending key file = do
 		]
 	-- Convert the ssh command into rsync command line.
 	let eparam = rsyncShell (Param shellcmd:shellparams)
-	o <- Annex.repoConfig r "rsync-options" ""
+	o <- getConfig r "rsync-options" ""
 	let base = options ++ map Param (words o) ++ eparam
 	if sending
 		then return $ base ++ [dummy, File file]
@@ -262,7 +220,3 @@ git_annex_shell r command params
 		shellopts = (Param command):(File dir):params
 		sshcmd = shellcmd ++ " " ++ 
 			unwords (map shellEscape $ toCommand shellopts)
-
-{- Human visible list of remotes. -}
-list :: [Git.Repo] -> String
-list remotes = join ", " $ map Git.repoDescribe remotes
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 38e8407a54..eb4a017486 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -13,6 +13,11 @@ import Control.Exception
 
 import Key
 
+{- A remote generator identifies configured remotes, and returns an action
+ - that can be run to set up each remote, and a list of names of remotes
+ - that are not cheap to set up. -}
+type RemoteGenerator a = ([a (Remote a)], [String])
+
 data Remote a = Remote {
 	-- each Remote has a unique uuid
 	uuid :: String,
diff --git a/Ssh.hs b/Ssh.hs
index 04cd9bec83..6d01a56423 100644
--- a/Ssh.hs
+++ b/Ssh.hs
@@ -7,17 +7,17 @@
 
 module Ssh where
 
-import qualified Annex
 import qualified GitRepo as Git
 import Utility
 import Types
+import Config
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
  - passed command. -}
 sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
-	s <- Annex.repoConfig repo "ssh-options" ""
+	s <- getConfig repo "ssh-options" ""
 	let sshoptions = map Param (words s)
 	let sshport = case Git.urlPort repo of
 		Nothing -> []
diff --git a/UUID.hs b/UUID.hs
index 5caf110453..eb1fb319c6 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -35,6 +35,7 @@ import Locations
 import qualified Annex
 import Utility
 import qualified SysConfig
+import Config
 
 type UUID = String
 
@@ -69,7 +70,7 @@ getUUID r = do
 		else return c
 	where
 		cached g = Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ Annex.setConfig cachekey u
+		updatecache g u = when (g /= r) $ setConfig cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
@@ -82,7 +83,7 @@ prepUUID = do
 	u <- getUUID g
 	when ("" == u) $ do
 		uuid <- liftIO $ genUUID
-		Annex.setConfig configkey uuid
+		setConfig configkey uuid
 
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
diff --git a/Version.hs b/Version.hs
index d061a2eab2..947af8cef5 100644
--- a/Version.hs
+++ b/Version.hs
@@ -15,6 +15,7 @@ import Types
 import qualified Annex
 import qualified GitRepo as Git
 import Locations
+import Config
 
 type Version = String
 
@@ -54,7 +55,7 @@ getVersion = do
 					return defaultVersion
 
 setVersion :: Annex ()
-setVersion = Annex.setConfig versionField defaultVersion
+setVersion = setConfig versionField defaultVersion
 
 checkVersion :: Annex ()
 checkVersion = do

From 65b72604d73b1d92dea1d81984964394235834bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 22:00:44 -0400
Subject: [PATCH 1254/8313] skeleton of S3 remote

---
 Remote.hs    |  7 ++++--
 Remote/S3.hs | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 Remote/S3.hs

diff --git a/Remote.hs b/Remote.hs
index 5508e0d123..6aab4a741c 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,7 +29,7 @@ import Data.String.Utils
 
 import RemoteClass
 import qualified Remote.Git
---import qualified Remote.S3
+import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
@@ -39,7 +39,10 @@ import Messages
 
 {- Add generators for new Remotes here. -}
 generators :: [Annex (RemoteGenerator Annex)]
-generators = [Remote.Git.generate]
+generators =
+	[ Remote.Git.generate
+	, Remote.S3.generate
+	]
 
 {- Runs a list of generators. -}
 runGenerators :: [Annex (RemoteGenerator Annex)] -> Annex [Remote Annex]
diff --git a/Remote/S3.hs b/Remote/S3.hs
new file mode 100644
index 0000000000..818cde2030
--- /dev/null
+++ b/Remote/S3.hs
@@ -0,0 +1,61 @@
+{- Amazon S3 remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.S3 (generate) where
+
+import qualified Data.Map as Map
+import Data.String.Utils
+import Control.Monad (filterM, liftM)
+
+import RemoteClass
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import UUID
+import Config
+
+generate :: Annex (RemoteGenerator Annex)
+generate = do
+	g <- Annex.gitRepo
+	remotes <- filterM remoteNotIgnored $ findS3Remotes g
+	todo <- filterM cachedUUID remotes
+	let ok = filter (`notElem` todo) remotes
+	
+	let actions = map genRemote ok ++
+		map (\r -> genRemote =<< tryS3ConfigRead r) todo
+	return (actions, map Git.repoDescribe todo)
+
+	where
+		cachedUUID r = liftM null $ getUUID r
+
+genRemote :: Git.Repo -> Annex (Remote Annex)
+genRemote r = do
+	return Remote {
+		uuid = error "TODO",
+		cost = error "TODO",
+		name = Git.repoDescribe r,
+		storeKey = error "TODO",
+		retrieveKeyFile = error "TODO",
+		removeKey = error "TODO",
+		hasKey = error "TODO",
+		hasKeyCheap = False
+	}
+
+{- S3 remotes have a remote..annex-s3bucket config setting.
+ - Git.Repo does not normally generate remotes for things that
+ - have no configured url, so the Git.Repo objects have to be
+ - constructed as coming from an unknown location. -}
+findS3Remotes :: Git.Repo -> [Git.Repo]
+findS3Remotes r = map construct remotepairs
+	where
+		remotepairs = Map.toList $ filterremotes $ Git.configMap r
+		filterremotes = Map.filterWithKey (\k _ -> s3remote k)
+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		s3remote k = startswith "remote." k && endswith ".annex-s3bucket" k
+
+tryS3ConfigRead :: Git.Repo -> Annex Git.Repo
+tryS3ConfigRead r = error "TODO"

From c0fd38bfa9ade9dc4515140cf5cf5619582c5a89 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 22:52:13 -0400
Subject: [PATCH 1255/8313] document S3 remotes

---
 Remote/S3.hs                                  |  4 +-
 doc/backends.mdwn                             |  7 ++++
 doc/special_remotes.mdwn                      | 38 +++++++++++++++++++
 doc/walkthrough.mdwn                          |  1 +
 ...ing_file_content_between_repositories.mdwn |  4 +-
 doc/walkthrough/using_Amazon_S3.mdwn          | 15 ++++++++
 6 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 doc/special_remotes.mdwn
 create mode 100644 doc/walkthrough/using_Amazon_S3.mdwn

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 818cde2030..bc010bf0bb 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -45,7 +45,7 @@ genRemote r = do
 		hasKeyCheap = False
 	}
 
-{- S3 remotes have a remote..annex-s3bucket config setting.
+{- S3 remotes have a remote..annex-s3-bucket config setting.
  - Git.Repo does not normally generate remotes for things that
  - have no configured url, so the Git.Repo objects have to be
  - constructed as coming from an unknown location. -}
@@ -55,7 +55,7 @@ findS3Remotes r = map construct remotepairs
 		remotepairs = Map.toList $ filterremotes $ Git.configMap r
 		filterremotes = Map.filterWithKey (\k _ -> s3remote k)
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		s3remote k = startswith "remote." k && endswith ".annex-s3bucket" k
+		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
 
 tryS3ConfigRead :: Git.Repo -> Annex Git.Repo
 tryS3ConfigRead r = error "TODO"
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 22164850ab..b0a2c882aa 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -9,6 +9,10 @@ to retrieve the file's content (its value).
 Multiple pluggable backends are supported, and a single repository
 can use different backends for different files.
 
+These backends can transfer file contents in configured git remotes.
+It's also possible to use [[special_remotes]], such as Amazon S3 with
+these backends.
+
 * `WORM` ("Write Once, Read Many") This backend assumes that any file with
   the same basename, size, and modification time has the same content. So with
   this backend, files can be moved around, but should never be added to
@@ -19,6 +23,9 @@ can use different backends for different files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
   researching checksum collisions and wants to annex their colliding data. ;)
+
+These backends store file contents in other key/value stores.
+
 * `URL` -- This backend downloads the file's content from an external URL.
 
 The `annex.backends` git-config setting can be used to list the backends
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
new file mode 100644
index 0000000000..a62d55f5bb
--- /dev/null
+++ b/doc/special_remotes.mdwn
@@ -0,0 +1,38 @@
+Most [[backends]] can transfer data to and from configured git remotes.
+Normally those remotes are normal git repositories (bare and non-bare),
+that store the file contents in their own git annex directory.
+
+But, git-annex also extends git's concept of remotes, with these special
+types of remotes. These can be used just like any normal remote by git-annex.
+They cannot be used by other git commands though.
+
+## Amazon S3
+
+Stores file contents in a bucket in Amazon S3 or a similar service.
+
+Example of configuring such a remote:
+
+	git config remote.mys3.annex-s3-bucket myannex
+	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+
+That creates a remote named "mys3" using the bucket named "myannex",
+which will be created if it doesn't already exist.
+
+Here is the full set of configurable settings for Amazon S3.
+Each setting can be configured on a per-remote basis in git-config,
+or globally in an environment variable.
+
+* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
+  Your S3 password. Usually stored in the environment variable
+  to avoid it being exposed.
+* `remote.$name.annex-s3-access-key-id` `ANNEX_S3_ACCESS_KEY_ID` 
+  Your S3 access key. For example, "". Does not need to be kept
+  private.
+* `remote.$name.annex-s3-host` `ANNEX_S3_HOST` 
+  Host to connect to. Default is s3.amazonaws.com.
+* `remote.$name.annex-s3-port` `ANNEX_S3_PORT` 
+  Port to connect to. Default is 80.
+* `remote.$name.annex-s3-datacenter` `ANNEX_S3_DATACENTER` 
+  Name of the datacenter to use. Default is "US";
+  other valid values include "EU", "us-west-1", and "ap-southeast-1".
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index 3b4f7d56a4..53f0be6bb4 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -14,6 +14,7 @@ A walkthrough of the basic features of git-annex.
 	modifying_annexed_files
 	using_ssh_remotes
 	moving_file_content_between_repositories
+	using_Amazon_S3
 	using_the_URL_backend
 	using_the_SHA1_backend
 	migrating_data_to_a_new_backend
diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn
index 6b3e3f4e80..27dffe9138 100644
--- a/doc/walkthrough/moving_file_content_between_repositories.mdwn
+++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn
@@ -6,8 +6,8 @@ server to your laptop. Doing that by hand (by using `git annex get` and
 makes it very easy.
 
 	# git annex move my_cool_big_file --to usbdrive
-	move my_cool_big_file (moving to usbdrive...) ok
+	move my_cool_big_file (to usbdrive...) ok
 	# git annex move video/hackity_hack_and_kaxxt.mov --from fileserver
-	move video/hackity_hack_and_kaxxt.mov (moving from fileserver...)
+	move video/hackity_hack_and_kaxxt.mov (from fileserver...)
 	WORM-s86050597-m1274316523--hackity_hack_and_kax 100%   82MB 199.1KB/s   07:02
 	ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
new file mode 100644
index 0000000000..d7222731b3
--- /dev/null
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -0,0 +1,15 @@
+git-annex extends git's usual remotes with some [[special_remotes]], that
+are not git repositories. This way you can set up a remote using say,
+Amazon S3, and use git-annex to transfer files into the cloud.
+
+	# git config remote.mys3.annex-s3-bucket myannex
+	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+	# git annex copy my_cool_big_file --to mys3
+	copy my_cool_big_file (to mys3...) ok
+	# git annex move video/hackity_hack_and_kaxxt.mov --to mys3
+	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
+
+An Amazon S3 remote works just like a ssh remote, except it does not have
+a git repository at the other end, and it costs you money. :) For full
+details about setting them up, see [[special_remotes]].

From 026c76914e21c768a38e86461849213e33b70046 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Mar 2011 23:11:09 -0400
Subject: [PATCH 1256/8313] update

---
 doc/special_remotes.mdwn | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index a62d55f5bb..3849b0e841 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -23,16 +23,20 @@ Here is the full set of configurable settings for Amazon S3.
 Each setting can be configured on a per-remote basis in git-config,
 or globally in an environment variable.
 
-* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
-  Your S3 password. Usually stored in the environment variable
-  to avoid it being exposed.
 * `remote.$name.annex-s3-access-key-id` `ANNEX_S3_ACCESS_KEY_ID` 
-  Your S3 access key. For example, "". Does not need to be kept
-  private.
+  Your S3 Access Key ID. Does not need to be kept private.
+* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
+  Your S3 Secret Access Key. This is a password. Usually stored in the
+  environment variable to avoid it being exposed.
 * `remote.$name.annex-s3-host` `ANNEX_S3_HOST` 
   Host to connect to. Default is s3.amazonaws.com.
 * `remote.$name.annex-s3-port` `ANNEX_S3_PORT` 
   Port to connect to. Default is 80.
 * `remote.$name.annex-s3-datacenter` `ANNEX_S3_DATACENTER` 
-  Name of the datacenter to use. Default is "US";
+  Name of the datacenter to use when creating a new bucket. Default is "US";
   other valid values include "EU", "us-west-1", and "ap-southeast-1".
+* `remote.$name.annex-s3-storageclass` `ANNEX_S3_STORAGECLASS` 
+  Name of storage class to use when adding new content to the bucket.
+  Default is "STANDARD". If you have configured git-annex to preserve
+  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY" to
+  save money.

From a7bd63eb0100fd282da9058acc28935bdfdf25df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 01:32:47 -0400
Subject: [PATCH 1257/8313] basic s3 remote start

But bucket name is not handled right; it needs to be globally unique.
---
 Config.hs        |  10 ++---
 Remote/S3.hs     | 100 ++++++++++++++++++++++++++++++++++++++---------
 debian/changelog |   3 +-
 3 files changed, 89 insertions(+), 24 deletions(-)

diff --git a/Config.hs b/Config.hs
index aae7d82915..c821364ced 100644
--- a/Config.hs
+++ b/Config.hs
@@ -31,11 +31,11 @@ setConfig k value = do
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
 	g <- Annex.gitRepo
-	let def' = Git.configGet g global def
-	return $ Git.configGet g local def'
-	where
-		local = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
-		global = "annex." ++ key
+	let def' = Git.configGet g ("annex." ++ key) def
+	return $ Git.configGet g (remoteConfig r key) def'
+
+remoteConfig :: Git.Repo -> ConfigKey -> String
+remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
 {- Calculates cost for a remote.
  -
diff --git a/Remote/S3.hs b/Remote/S3.hs
index bc010bf0bb..23ec33bb59 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -7,9 +7,18 @@
 
 module Remote.S3 (generate) where
 
+import Network.AWS.AWSConnection
+import Network.AWS.S3Object
+import Network.AWS.S3Bucket
+import Network.AWS.AWSResult
+import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as Map
 import Data.String.Utils
-import Control.Monad (filterM, liftM)
+import Control.Monad (filterM, liftM, when)
+import Control.Monad.State (liftIO)
+import System.Environment
+import Data.Char
+import Messages
 
 import RemoteClass
 import Types
@@ -25,26 +34,13 @@ generate = do
 	todo <- filterM cachedUUID remotes
 	let ok = filter (`notElem` todo) remotes
 	
-	let actions = map genRemote ok ++
-		map (\r -> genRemote =<< tryS3ConfigRead r) todo
+	let actions = map (\r -> genRemote r =<< getUUID r) ok ++
+		map (\r -> genRemote r =<< getS3UUID r) todo
 	return (actions, map Git.repoDescribe todo)
 
 	where
 		cachedUUID r = liftM null $ getUUID r
 
-genRemote :: Git.Repo -> Annex (Remote Annex)
-genRemote r = do
-	return Remote {
-		uuid = error "TODO",
-		cost = error "TODO",
-		name = Git.repoDescribe r,
-		storeKey = error "TODO",
-		retrieveKeyFile = error "TODO",
-		removeKey = error "TODO",
-		hasKey = error "TODO",
-		hasKeyCheap = False
-	}
-
 {- S3 remotes have a remote..annex-s3-bucket config setting.
  - Git.Repo does not normally generate remotes for things that
  - have no configured url, so the Git.Repo objects have to be
@@ -57,5 +53,73 @@ findS3Remotes r = map construct remotepairs
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
 		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
 
-tryS3ConfigRead :: Git.Repo -> Annex Git.Repo
-tryS3ConfigRead r = error "TODO"
+genRemote :: Git.Repo -> UUID -> Annex (Remote Annex)
+genRemote r u = do
+	c <- remoteCost r
+	return Remote {
+		uuid = u,
+		cost = c,
+		name = Git.repoDescribe r,
+		storeKey = error "TODO",
+		retrieveKeyFile = error "TODO",
+		removeKey = error "TODO",
+		hasKey = error "TODO",
+		hasKeyCheap = False
+	}
+
+s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
+s3Connection r = do
+	host <- getS3Config r "s3-host" (Just defaultAmazonS3Host)
+	port <- getS3Config r "s3-port" (Just $ show defaultAmazonS3Port)
+	accesskey <- getS3Config r "s3-access-key-id" Nothing
+	secretkey <- getS3Config r "s3-secret-access-key" Nothing
+	case reads port of
+		[(p, _)] -> return $ Just $ AWSConnection host p accesskey secretkey
+		_ -> error $ "bad S3 port value: " ++ port
+
+withS3Connection :: Git.Repo -> Annex a -> ((AWSConnection, String) -> Annex a) -> Annex a
+withS3Connection r def a = do
+	c <- s3Connection r
+	case c of
+		Nothing -> def
+		Just c' -> do
+			b <- getConfig r "s3-bucket" ""
+			a (c', b)
+
+getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
+getS3Config r s def = do
+	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
+	v <- case e of
+		Nothing -> getConfig r s ""
+		Just d -> getConfig r s d
+	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
+	return v
+	where
+		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
+
+{- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
+ - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
+ - also creating the bucket. -}
+getS3UUID :: Git.Repo -> Annex UUID
+getS3UUID r = withS3Connection r disable $ \(c, b) -> do
+	res <- liftIO $
+		getObject c $ S3Object b uuidfile "" [] L.empty
+	case res of
+		Right o -> return $ L.unpack $ obj_data o
+		Left _ -> do
+			location <- getS3Config r "s3-datacenter" (Just "EU")
+			-- bucket may already exist, or not
+			_ <- liftIO $ createBucketIn c b location
+			u <- getUUID r
+			res' <- liftIO $ sendObject c $
+				S3Object b uuidfile "" [] $
+					L.pack u
+			case res' of
+				Right _ -> return u
+				Left e -> do
+					warning $ prettyReqError e
+					disable
+					
+	where
+		uuidfile = "git-annex-uuid"
+		disable = return "" -- empty uuid will disable this remote
diff --git a/debian/changelog b/debian/changelog
index 2f532784d4..0469f2242b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,12 @@
 git-annex (0.20110326) UNRELEASED; urgency=low
 
-  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+  * Amazon is S3 now supported as a special type of remote.
   * Generalized remotes handling, laying groundwork for remotes that are
     not regular git remotes.
   * Provide a less expensive version of `git annex copy --to`, enabled
     via --fast. This assumes that location tracking information is correct,
     rather than contacting the remote for every file.
+  * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 

From 1878745a4693913417f65c7c5182e939512e9b22 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 02:12:05 -0400
Subject: [PATCH 1258/8313] more s3 docs

---
 debian/changelog                     |  2 +-
 doc/git-annex.mdwn                   | 31 +++++++++++++++++++++++++++
 doc/internals.mdwn                   | 11 ++++++++++
 doc/special_remotes.mdwn             | 32 +---------------------------
 doc/todo/S3.mdwn                     |  2 ++
 doc/walkthrough/using_Amazon_S3.mdwn | 22 ++++++++++++++-----
 6 files changed, 63 insertions(+), 37 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 0469f2242b..945a4ed6b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (0.20110326) UNRELEASED; urgency=low
 
-  * Amazon is S3 now supported as a special type of remote.
+  * Amazon S3 is now supported as a special type of remote.
   * Generalized remotes handling, laying groundwork for remotes that are
     not regular git remotes.
   * Provide a less expensive version of `git annex copy --to`, enabled
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8afe93c109..6960c19662 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,6 +132,16 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
+* s3bucket name description [datacenter host port]
+
+  Creates a bucket in Amazon S3. The bucket's name can be used
+  to configure git remote using the bucket.
+
+  The datacenter defaults to "US". Other values include "EU",
+  "us-west-1", and "ap-southeast-1".
+
+  To use a different, S3-compatable service, specify a host and port.
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
@@ -387,6 +397,25 @@ Here are all the supported configuration settings.
   Default ssh and rsync options to use if a remote does not have
   specific options.
 
+* `remote..annex-s3-access-key-id`
+
+  Your S3 Access Key ID. Does not need to be kept private.
+  If not set, the environment variable `AWS_ACCESS_KEY_ID`
+  will be used.
+
+* `remote..annex-s3-secret-access-key`
+
+  Your S3 Secret Access Key. This is a password.
+  If not set, the environment variable `AWS_SECRET_ACCESS_KEY`
+  will be used.
+
+* `remote..annex-s3-storageclass`
+
+  Storage class to use when adding new content to S3. The default
+  is "STANDARD". If you have configured git-annex to preserve
+  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"
+  to save money.
+
 * `annex.diskreserve`
 
   Amount of disk space to reserve. Disk space is checked when transferring
@@ -401,6 +430,8 @@ Here are all the supported configuration settings.
 
   Automatically maintained, and used to automate upgrades between versions.
 
+  
+
 # CONFIGURATION VIA .gitattributes
 
 The backend used when adding a new file to the annex can be configured
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index a133320b4b..55b1045a11 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -30,6 +30,17 @@ space and then the description through to the end of the line. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
 	26339d22-446b-11e0-9101-002170d25c55 usb disk
 
+## `git-annex/s3.log`
+
+Associates the UUIDs of Amazon S3 buckets with a bucket nickname and connection
+information. Example:
+
+	be72acb8-5901-11e0-b600-002170d25c55 mybucket s3.amazonaws.com 80
+
+Note that the actual bucket name used on S3 in the above example 
+is "mybucket-be72acb8-5901-11e0-b600-002170d25c55". The UUID is included
+in the bucket name to ensure it is globally unique.
+
 ## `.git-annex/trust.log`
 
 Records the [[trust]] information for repositories. Does not exist unless
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 3849b0e841..7dc54fd9bf 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -9,34 +9,4 @@ They cannot be used by other git commands though.
 ## Amazon S3
 
 Stores file contents in a bucket in Amazon S3 or a similar service.
-
-Example of configuring such a remote:
-
-	git config remote.mys3.annex-s3-bucket myannex
-	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
-
-That creates a remote named "mys3" using the bucket named "myannex",
-which will be created if it doesn't already exist.
-
-Here is the full set of configurable settings for Amazon S3.
-Each setting can be configured on a per-remote basis in git-config,
-or globally in an environment variable.
-
-* `remote.$name.annex-s3-access-key-id` `ANNEX_S3_ACCESS_KEY_ID` 
-  Your S3 Access Key ID. Does not need to be kept private.
-* `remote.$name.annex-s3-secret-access-key` `ANNEX_S3_SECRET_ACCESS_KEY` 
-  Your S3 Secret Access Key. This is a password. Usually stored in the
-  environment variable to avoid it being exposed.
-* `remote.$name.annex-s3-host` `ANNEX_S3_HOST` 
-  Host to connect to. Default is s3.amazonaws.com.
-* `remote.$name.annex-s3-port` `ANNEX_S3_PORT` 
-  Port to connect to. Default is 80.
-* `remote.$name.annex-s3-datacenter` `ANNEX_S3_DATACENTER` 
-  Name of the datacenter to use when creating a new bucket. Default is "US";
-  other valid values include "EU", "us-west-1", and "ap-southeast-1".
-* `remote.$name.annex-s3-storageclass` `ANNEX_S3_STORAGECLASS` 
-  Name of storage class to use when adding new content to the bucket.
-  Default is "STANDARD". If you have configured git-annex to preserve
-  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY" to
-  save money.
+See [[walkthrough/using_Amazon_S3]] for examples.
diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 09a64f1a7a..356b2af2eb 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -1,3 +1,5 @@
+[[done]]
+
 Support Amazon S3 as a file storage backend.
 
 There's a haskell library that looks good. Not yet in Debian.
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index d7222731b3..cadd78582c 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -2,14 +2,26 @@ git-annex extends git's usual remotes with some [[special_remotes]], that
 are not git repositories. This way you can set up a remote using say,
 Amazon S3, and use git-annex to transfer files into the cloud.
 
-	# git config remote.mys3.annex-s3-bucket myannex
-	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+First, export your S3 credentials:
+
+	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+
+Next, create a bucket, giving it a name and a description:
+
+	git annex s3bucket mybucket "my Amazon S3 bucket"
+	s3bucket (creating mybucket...) ok
+
+Finally, configure a git remote to use the bucket you created:
+
+	git config remote.mys3.annex-s3-bucket mybucket
+
+Now the remote can be used like any other remote.
+
 	# git annex copy my_cool_big_file --to mys3
 	copy my_cool_big_file (to mys3...) ok
 	# git annex move video/hackity_hack_and_kaxxt.mov --to mys3
 	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
 
 An Amazon S3 remote works just like a ssh remote, except it does not have
-a git repository at the other end, and it costs you money. :) For full
-details about setting them up, see [[special_remotes]].
+a git repository at the other end, and it costs you money. :)

From 61063dee6cc50652b393979358142f246894de58 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 07:26:41 +0000
Subject: [PATCH 1259/8313] Added a comment

---
 .../comment_1_9a7b09de132097100c1a68ea7b846727._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment
new file mode 100644
index 0000000000..aa5e46ca2b
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-03-28T07:23:41Z"
+ content="""
+One possible work around is to just create a loopback file system with a case sensitive filesystem. I think I might do that for anything that I really care about for now.
+"""]]

From ee9973019c933a58568040f8fd12fb8736dbc965 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Mon, 28 Mar 2011 12:32:30 +0000
Subject: [PATCH 1260/8313]

---
 ...d_check__47__fix_the_permissions_of_.git__47__annex.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn

diff --git a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
new file mode 100644
index 0000000000..ec8b6d2330
--- /dev/null
+++ b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
@@ -0,0 +1,6 @@
+git annex carefully setup restrictive permissions of .git/annex directories and files.
+
+The fsck command should check that they are still correct.
+The fix command should fix them.
+
+PS: Thanks for this nice tool!

From 3bdc5eb29077add9f2de18ba587ca88bb98cb63e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:40:05 -0400
Subject: [PATCH 1261/8313] will need gpg encryption for s3

---
 doc/git-annex.mdwn                   | 6 +++++-
 doc/special_remotes.mdwn             | 1 +
 doc/walkthrough/using_Amazon_S3.mdwn | 4 +++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 6960c19662..c01f4fbc59 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,7 +132,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* s3bucket name description [datacenter host port]
+* s3bucket name description [datacenter host port] [--key=gpgkey]
 
   Creates a bucket in Amazon S3. The bucket's name can be used
   to configure git remote using the bucket.
@@ -142,6 +142,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   To use a different, S3-compatable service, specify a host and port.
 
+  By default, data (including filenames) is encrypted using gpg.
+  To use a key other than the default gpg key, specify it with
+  the --key option. To disable encryption, specify "none".
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 7dc54fd9bf..717ec48404 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -9,4 +9,5 @@ They cannot be used by other git commands though.
 ## Amazon S3
 
 Stores file contents in a bucket in Amazon S3 or a similar service.
+Content is stored encrypted by gpg.
 See [[walkthrough/using_Amazon_S3]] for examples.
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index cadd78582c..8cb77ab6cd 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -24,4 +24,6 @@ Now the remote can be used like any other remote.
 	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
 
 An Amazon S3 remote works just like a ssh remote, except it does not have
-a git repository at the other end, and it costs you money. :)
+a git repository at the other end, and it costs you money. :) In particular,
+all data is stored encrypted with gpg, so neither Amazon nor anyone in
+between can see it.

From 02601c6b9f2129e80323abea5ad76bf64c27b574 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:41:23 -0400
Subject: [PATCH 1262/8313] analysis; workaround

---
 ..._version_upgrade_leaves_repo_unusable.mdwn | 26 ++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 1218b0e517..964a176bd1 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -1,6 +1,10 @@
 foo is a local repo, bar is a bare remote.
 
-I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend to version 2. I then ran `git annex copy . --to bar` and checked the remote. This created WORM:SHA512--123123 files in annex/objects. Understandable but unwanted. So I upgraded git-annex on bar's machine, as well.
+I upgraded foo's git-annex to 0.20110325 and upgraded a local repo backend
+to version 2. I then ran `git annex copy . --to bar` and checked the
+remote. This created WORM:SHA512--123123 files in annex/objects.
+Understandable but unwanted. So I upgraded git-annex on bar's machine, as
+well.
 
     % git annex copy . --to bar
     copy quux (checking bar) git-annex-shell: Repository version 1 is not supported. Upgrade this repository: git-annex upgrade (to bar)
@@ -33,3 +37,23 @@ Running the copy job again, I am still getting the same error as above (as expec
 > joey@kitenet.net if you don't want to post them here. --[[Joey]]
 
 >> Sent. -- RichiH
+
+>>> Ok, I'm going to go work on my reading comprehension. I see now
+>>> that you
+>>> explained the problem pretty well. The problem is caused by these
+>>> few weird v1 mixed with v2 keys in the annex.
+>>> Ones like "annex/objects/WORM:SHA512--$sha512".
+>>>
+>>> That's a v1 key, but a corrupt form of the key; it's missing the 
+>>> size and mtime fields that all WORM keys have in v1. And 
+>>> the filename is itself a key, a v2 SHA512 key. My guess at what
+>>> happened is that these were created when you did the `git annex copy`
+>>> to the v1 bare repo.
+>>>
+>>> So, assuming none of these are the only copy of your data
+>>> (which you should check), the best thing to do is delete those
+>>> keys, and re-run the copy now that it's upgraded. Not sure 
+>>> it even makes sense to fix the crash. I should probably focus
+>>> on fixing what let these mixed keys be created by the mixed-version
+>>> copy.
+>>> --[[Joey]]

From d2ed1b3a99da45ae25e84cbc832693442bab7de6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:52:42 -0400
Subject: [PATCH 1263/8313] second thought

---
 doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 964a176bd1..7bbd02364f 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -56,4 +56,7 @@ Running the copy job again, I am still getting the same error as above (as expec
 >>> it even makes sense to fix the crash. I should probably focus
 >>> on fixing what let these mixed keys be created by the mixed-version
 >>> copy.
+>>>
+>>> On second thought, you shouldn't delete anything. I'll simply
+>>> make the v2 upgrade detect and work around this bug.
 >>> --[[Joey]]

From 9d86d02b3db23f0b8848f4a9a044befa58e1ecbb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 08:55:00 -0400
Subject: [PATCH 1264/8313] update

---
 ..._version_upgrade_leaves_repo_unusable.mdwn | 20 ++++++++-----------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 7bbd02364f..64c18f202c 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -46,17 +46,13 @@ Running the copy job again, I am still getting the same error as above (as expec
 >>>
 >>> That's a v1 key, but a corrupt form of the key; it's missing the 
 >>> size and mtime fields that all WORM keys have in v1. And 
->>> the filename is itself a key, a v2 SHA512 key. My guess at what
->>> happened is that these were created when you did the `git annex copy`
->>> to the v1 bare repo.
+>>> the filename is itself a key, a v2 SHA512 key. These were
+>>> created when you did the `git annex copy to the v1 bare repo.
+>>> In v2, git-annex-shell takes a full key object, while in v1,
+>>> it takes a key name and a backend name. This incompatability
+>>> leads to the weird behavior seen.
 >>>
->>> So, assuming none of these are the only copy of your data
->>> (which you should check), the best thing to do is delete those
->>> keys, and re-run the copy now that it's upgraded. Not sure 
->>> it even makes sense to fix the crash. I should probably focus
->>> on fixing what let these mixed keys be created by the mixed-version
->>> copy.
->>>
->>> On second thought, you shouldn't delete anything. I'll simply
->>> make the v2 upgrade detect and work around this bug.
+>>> I had suggested you delete data.. don't. On second thought,
+>>> you shouldn't delete anything. I'll simply make the v2 upgrade
+>>> detect and work around this bug.
 >>> --[[Joey]]

From 016eea028086f2e8c1733ac77612f4397297d1a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 09:27:28 -0400
Subject: [PATCH 1265/8313] Bugfix: Keys could be received into v1 annexes from
 v2 annexes, via v1 git-annex-shell. This results in some oddly named keys in
 the v1 annex. Recognise and fix those keys when upgrading, instead of
 crashing.

---
 Upgrade/V1.hs                                 | 44 +++++++++++--------
 debian/changelog                              |  3 ++
 ..._version_upgrade_leaves_repo_unusable.mdwn |  3 ++
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index a876142220..4ce2612d60 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -16,6 +16,7 @@ import System.FilePath
 import Data.String.Utils
 import System.Posix.Types
 import Data.Maybe
+import Data.Char
 
 import Key
 import Content
@@ -79,12 +80,11 @@ upgrade = do
 moveContent :: Annex ()
 moveContent = do
 	showNote "moving content..."
-	keys <- getKeysPresent1
-	forM_ keys move
+	files <- getKeyFilesPresent1
+	forM_ files move
 	where
-		move k = do
-			g <- Annex.gitRepo
-			let f = gitAnnexObjectDir g  keyFile1 k  keyFile1 k
+		move f = do
+			let k = fileKey1 (takeFileName f)
 			let d = parentDir f
 			liftIO $ allowWrite d
 			liftIO $ allowWrite f
@@ -154,8 +154,15 @@ oldlog2key l =
 
 -- WORM backend keys: "WORM:mtime:size:filename"
 -- all the rest: "backend:key"
+--
+-- If the file looks like "WORM:XXX-...", then it was created by mixing
+-- v2 and v1; that infelicity is worked around by treating the value
+-- as the v2 key that it is.
 readKey1 :: String -> Key
-readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
+readKey1 v = 
+	if mixup
+		then fromJust $ readKey $ join ":" $ tail bits
+		else Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
 	where
 		bits = split ":" v
 		b = head bits
@@ -166,7 +173,8 @@ readKey1 v = Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
 		s = if wormy
 			then Just (read (bits !! 2) :: Integer)
 			else Nothing
-		wormy = b == "WORM"
+		wormy = head bits == "WORM"
+		mixup = wormy && (isUpper $ head $ bits !! 1)
 
 showKey1 :: Key -> String
 showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =
@@ -211,24 +219,22 @@ lookupFile1 file = do
 				skip = "skipping " ++ file ++ 
 					" (unknown backend " ++ bname ++ ")"
 
-getKeysPresent1 :: Annex [Key]
-getKeysPresent1  = do
+getKeyFilesPresent1 :: Annex [FilePath]
+getKeyFilesPresent1  = do
 	g <- Annex.gitRepo
-	getKeysPresent1' $ gitAnnexObjectDir g
-getKeysPresent1' :: FilePath -> Annex [Key]
-getKeysPresent1' dir = do
+	getKeyFilesPresent1' $ gitAnnexObjectDir g
+getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
+getKeyFilesPresent1' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
 	if (not exists)
 		then return []
 		else do
-			contents <- liftIO $ getDirectoryContents dir
-			files <- liftIO $ filterM present contents
-			return $ map fileKey1 files
+			dirs <- liftIO $ getDirectoryContents dir
+			let files = map (\d -> dir ++ "/" ++ d ++ "/" ++ takeFileName d) dirs
+			liftIO $ filterM present files
 	where
-		present d = do
-			liftIO $ putStrLn $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
-			result <- try $
-				getFileStatus $ dir ++ "/" ++ d ++ "/"  ++ takeFileName d
+		present f = do
+			result <- try $ getFileStatus f
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
diff --git a/debian/changelog b/debian/changelog
index 2f532784d4..517cf59695 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,9 @@ git-annex (0.20110326) UNRELEASED; urgency=low
   * Provide a less expensive version of `git annex copy --to`, enabled
     via --fast. This assumes that location tracking information is correct,
     rather than contacting the remote for every file.
+  * Bugfix: Keys could be received into v1 annexes from v2 annexes, via
+    v1 git-annex-shell. This results in some oddly named keys in the v1
+    annex. Recognise and fix those keys when upgrading, instead of crashing.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 64c18f202c..1eebd9ecd4 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -56,3 +56,6 @@ Running the copy job again, I am still getting the same error as above (as expec
 >>> you shouldn't delete anything. I'll simply make the v2 upgrade
 >>> detect and work around this bug.
 >>> --[[Joey]]
+
+>>>> This should be fixed in current git. The scambled keys will be 
+>>>> fixed up on upgrade. Thanks for your patience! [[done]] --[[Joey]]

From 2933860847c8092cd2f447e29f3da8b00a15cb18 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Mon, 28 Mar 2011 14:01:00 +0000
Subject: [PATCH 1266/8313] not actively pursuing this issue any more

---
 doc/forum/relying_on_git_for_numcopies.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/relying_on_git_for_numcopies.mdwn b/doc/forum/relying_on_git_for_numcopies.mdwn
index b7ebba805a..37b46cf4e7 100644
--- a/doc/forum/relying_on_git_for_numcopies.mdwn
+++ b/doc/forum/relying_on_git_for_numcopies.mdwn
@@ -1,3 +1,5 @@
+**<out-of-date-warning>**The main problems this is supposed to solve are addressed in a different way with [[todo/hidden files]] and the `--fast` option introduced in [[batch check on remote when using copy]], so while this is not technically obsolete, the main reasons for it are gone. --[[chrysn]]**</out-of-date-warning>**
+
 This is a rough sketch of a modification of git-annex to rely more on git commit semantics. It might be flawed due to my lack of understanding of git-annex internals. --[[chrysn]]
 
 Summary

From fc363b9ca39f264566ec9299358f201c348b792a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 14:44:26 +0000
Subject: [PATCH 1267/8313]

---
 .../backend_version_upgrade_leaves_repo_unusable.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
index 1eebd9ecd4..122224a8f3 100644
--- a/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
+++ b/doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn
@@ -59,3 +59,14 @@ Running the copy job again, I am still getting the same error as above (as expec
 
 >>>> This should be fixed in current git. The scambled keys will be 
 >>>> fixed up on upgrade. Thanks for your patience! [[done]] --[[Joey]]
+
+>>>>> I should stop reading your answers via git; by the time I got to
+>>>>> "second thoughts", I had already deleted the files & directories
+>>>>> in question, upgraded the bare repo and was busy uploading from my
+>>>>> local repo. I agree that taking care of this in the upgrade code
+>>>>> is the cleanest approach, by the way.
+>>>>> No need to thank me for my patience; thank you for your quickness!
+>>>>> RichiH
+>>>>> 
+>>>>> PS: If I get a handle on the mtime issue in the SHA backend, git
+>>>>> annex will be pretty much perfect :)

From 1b6927995de5e25ec6c5c464c6444d2ba61ec748 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:12:32 -0400
Subject: [PATCH 1268/8313] releasing version 0.20110328

---
 debian/changelog | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 517cf59695..7251ab6653 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,8 @@
-git-annex (0.20110326) UNRELEASED; urgency=low
+git-annex (0.20110328) experimental; urgency=low
 
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
   * Generalized remotes handling, laying groundwork for remotes that are
-    not regular git remotes.
+    not regular git remotes. (Think Amazon S3.)
   * Provide a less expensive version of `git annex copy --to`, enabled
     via --fast. This assumes that location tracking information is correct,
     rather than contacting the remote for every file.
@@ -10,7 +10,7 @@ git-annex (0.20110326) UNRELEASED; urgency=low
     v1 git-annex-shell. This results in some oddly named keys in the v1
     annex. Recognise and fix those keys when upgrading, instead of crashing.
 
- -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
+ -- Joey Hess   Mon, 28 Mar 2011 10:47:29 -0400
 
 git-annex (0.20110325) experimental; urgency=low
 

From 659f0fe980b6bff6900037d9ae52024b989613e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:13:15 -0400
Subject: [PATCH 1269/8313] add news item for git-annex 0.20110328

---
 doc/news/version_0.20110328.mdwn | 11 +++++++++++
 doc/news/version_0.24.mdwn       | 12 ------------
 2 files changed, 11 insertions(+), 12 deletions(-)
 create mode 100644 doc/news/version_0.20110328.mdwn
 delete mode 100644 doc/news/version_0.24.mdwn

diff --git a/doc/news/version_0.20110328.mdwn b/doc/news/version_0.20110328.mdwn
new file mode 100644
index 0000000000..512ce4647a
--- /dev/null
+++ b/doc/news/version_0.20110328.mdwn
@@ -0,0 +1,11 @@
+git-annex 0.20110328 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
+   * Generalized remotes handling, laying groundwork for remotes that are
+     not regular git remotes. (Think Amazon S3.)
+   * Provide a less expensive version of `git annex copy --to`, enabled
+     via --fast. This assumes that location tracking information is correct,
+     rather than contacting the remote for every file.
+   * Bugfix: Keys could be received into v1 annexes from v2 annexes, via
+     v1 git-annex-shell. This results in some oddly named keys in the v1
+     annex. Recognise and fix those keys when upgrading, instead of crashing."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.24.mdwn b/doc/news/version_0.24.mdwn
deleted file mode 100644
index 81b013a26f..0000000000
--- a/doc/news/version_0.24.mdwn
+++ /dev/null
@@ -1,12 +0,0 @@
-Branched the 0.24 series, which will be maintained for a while (in the
-stable branch in git) to support v1 git-annex repos, while main development
-moves to the 0.2011 series, with v2 git-annex repos.
-
-git-annex 0.24 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-* Add Suggests on graphviz. Closes: #[618039](http://bugs.debian.org/618039)
-* When adding files to the annex, the symlinks pointing at the annexed
-  content are made to have the same mtime as the original file.
-  While git does not preserve that information, this allows a tool
-  like metastore to be used with annexed files.
-  (Currently this is only done on systems supporting POSIX 200809.)"""]]

From b6d40c119a9982f05db4134e182e762cb1953697 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 15:13:54 +0000
Subject: [PATCH 1270/8313] Added a comment

---
 ..._174952fc3e3be12912e5fcfe78f2dd13._comment | 185 ++++++++++++++++++
 1 file changed, 185 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment
new file mode 100644
index 0000000000..6e6e5dc6be
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment
@@ -0,0 +1,185 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-28T15:09:45Z"
+ content="""
+I think I know how I got myself into this mess... I was on my mac workstation and I had just pulled in a change set from another repo on a linux workstation after I had a made a bunch of moves. here's a bit of a log of what happened...
+
+
+
+jtang@x00:~/sources $ git pull cports-devel master
+Warning: untrusted X11 forwarding setup failed: xauth key data not generated
+Warning: No xauth data; using fake authentication data for X11 forwarding.
+remote: Counting objects: 4195, done.
+remote: Compressing objects: 100% (1135/1135), done.
+remote: Total 2582 (delta 866), reused 2576 (delta 860)
+Receiving objects: 100% (2582/2582), 229.42 KiB | 111 KiB/s, done.
+Resolving deltas: 100% (866/866), completed with 9 local objects.
+From cports-devel:/home/people/jtang/sources
+ * branch            master     -> FETCH_HEAD
+Updating 319df99..ab0a98c
+error: Your local changes to the following files would be overwritten by merge:
+	.git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log
+	.git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log
+	.git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log
+	.git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log
+	.git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log
+	.git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log
+	.git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log
+	.git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log
+	.git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log
+	.git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log
+	.git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log
+	.git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log
+	.git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log
+	.git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log
+	.git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log
+	.git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log
+Please, commit your changes or stash them before you can merge.
+error: Your local changes to the following files would be overwritten by merge:
+	.git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log
+	.git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log
+	.git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log
+	.git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log
+	.git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log
+	.git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log
+	.git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log
+	.git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log
+	.git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log
+	.git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log
+	.git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log
+	.git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log
+	.git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log
+	.git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log
+	.git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log
+	.git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log
+	.git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log
+	.git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log
+	.git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log
+	.git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log
+	.git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log
+	.git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log
+	.git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log
+	.git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log
+	.git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log
+	.git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log
+	.git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log
+	.git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log
+	.git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log
+	.git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log
+Please, commit your changes or stash them before you can merge.
+error: The following untracked working tree files would be overwritten by merge:
+	.git-annex/1g/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log
+	.git-annex/3w/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log
+	.git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log
+	.git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log
+	.git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log
+	.git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log
+	.git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log
+	.git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log
+	.git-annex/jK/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log
+	.git-annex/vK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log
+	.git-annex/vw/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log
+	.git-annex/zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+Please move or remove them before you can merge.
+Aborting
+1|jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 2 commits.
+#
+# Changes to be committed:
+#   (use \"git reset HEAD ...\" to unstage)
+#
+#	modified:   .git-annex/09/5X/WORM-s361516678-m1301310614--l_fcompxe_intel64_2011.2.137.tgz.log
+#	modified:   .git-annex/43/2g/WORM-s19509673-m1301310496--l_fcompxe_2011.2.137_redist.tgz.log
+#	modified:   .git-annex/4J/qF/WORM-s18891115-m1301310934--w_flm_p_1.0.011_ia64.zip.log
+#	modified:   .git-annex/87/w1/WORM-s12212473-m1301310909--w_flm_p_1.0.011_ia32.zip.log
+#	modified:   .git-annex/99/Jq/WORM-s194345957-m1301310926--l_mkl_10.3.2.137_ia32.log
+#	modified:   .git-annex/99/kf/WORM-s9784531-m1301311680--l_ccompxe_2011.2.137_redist.log
+#	modified:   .git-annex/FF/f3/WORM-s93033394-m1301311706--l_gen_ipp_7.0.2.137.log
+#	modified:   .git-annex/MF/xZ/WORM-s515140733-m1301310936--l_cprof_p_11.1.075.log
+#	modified:   .git-annex/XW/X8/WORM-s355559731-m1301310797--l_mkl_10.3.2.137.log
+#	modified:   .git-annex/fJ/mZ/WORM-s1372886477-m1301313368--l_cproc_p_11.1.075.log
+#	modified:   .git-annex/j7/Q9/WORM-s44423202-m1301310622--l_cprof_p_11.1.075_redist.log
+#	modified:   .git-annex/k4/K7/WORM-s239539070-m1301310760--l_mkl_10.3.2.137_intel64.log
+#	modified:   .git-annex/kz/01/WORM-s279573314-m1301310783--l_cprof_p_11.1.075_ia32.log
+#	modified:   .git-annex/p6/Kq/WORM-s31199343-m1301311829--l_cproc_p_11.1.075_redist.log
+#	modified:   .git-annex/pz/J5/WORM-s626995277-m1301312301--l_ccompxe_ia32_2011.2.137.log
+#	modified:   .git-annex/v3/kX/WORM-s339693045-m1301310851--l_cprof_p_11.1.075_intel64.log
+#
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   .git-annex/12/3W/WORM-s3058814-m1276699694--Botan-1.8.9.tgz.log
+#	modified:   .git-annex/1G/qV/WORM-s9122-m1251558854--Array-Compare-2.01.tar.gz.log
+#	modified:   .git-annex/3W/W5/WORM-s231523-m1270740744--DBD-Pg-2.17.1.tar.gz.log
+#	modified:   .git-annex/3x/PX/WORM-s380310-m1293025187--HTSeq-0.4.7.tar.gz.log
+#	modified:   .git-annex/45/gk/WORM-s67337-m1248732018--ExtUtils-Install-1.54.tar.gz.log
+#	modified:   .git-annex/4J/7Q/WORM-s8608-m1224694862--Algorithm-Munkres-0.08.tar.gz.log
+#	modified:   .git-annex/4g/XQ/WORM-s89208-m1278682033--HTML-Parser-3.66.tar.gz.log
+#	modified:   .git-annex/54/jw/WORM-s300163-m1226422051--AcePerl-1.92.tar.gz.log
+#	modified:   .git-annex/63/kj/WORM-s1213460-m1262942058--DBD-SQLite-1.29.tar.gz.log
+#	modified:   .git-annex/6Z/42/WORM-s4074-m943766010--File-Sync-0.09.tar.gz.log
+#	modified:   .git-annex/8F/M5/WORM-s6989-m1263161127--Digest-HMAC-1.02.tar.gz.log
+#	modified:   .git-annex/G2/FK/WORM-s3309-m1163872981--Bundle-BioPerl-2.1.8.tar.gz.log
+#	modified:   .git-annex/Gk/XF/WORM-s23572243-m1279546902--EMBOSS-6.3.1.tar.gz.log
+#	modified:   .git-annex/Jk/X6/WORM-s566429-m1279309002--DBI-1.612.tar.gz.log
+#	modified:   .git-annex/K6/fV/WORM-s1561451-m1240055295--Convert-Binary-C-0.74.tar.gz.log
+#	modified:   .git-annex/KM/4q/WORM-s146959-m1268515086--Graph-0.94.tar.gz.log
+#	modified:   .git-annex/MF/m2/WORM-s425766-m1212514609--Data-Stag-0.11.tar.gz.log
+#	modified:   .git-annex/QJ/P6/WORM-s1045868-m1282215033--9base-6.tar.gz.log
+#	modified:   .git-annex/Qm/WG/WORM-s39078-m1278163547--Digest-SHA1-2.13.tar.gz.log
+#	modified:   .git-annex/Wq/Fj/WORM-s45680640-m1297862101--BclConverter-1.7.1.tar.log
+#	modified:   .git-annex/Wq/Wm/WORM-s263536640-m1295025537--CASAVA_v1.7.0.tar.log
+#	modified:   .git-annex/XW/qm/WORM-s36609-m1276050470--Bio-ASN1-EntrezGene-1.10-withoutworldwriteables.tar.gz.log
+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+#	modified:   .git-annex/f7/g0/WORM-s40872-m1278273227--ExtUtils-ParseXS-2.2206.tar.gz.log
+#	modified:   .git-annex/j3/JF/WORM-s11753-m1232427595--Clone-0.31.tar.gz.log
+#	modified:   .git-annex/kX/9g/WORM-s84690-m1229117599--GraphViz-2.04.tar.gz.log
+#	modified:   .git-annex/km/z5/WORM-s44634-m1275505134--Authen-SASL-2.15.tar.gz.log
+#	modified:   .git-annex/kw/J3/WORM-s132396-m1278780649--DBD-mysql-4.016.tar.gz.log
+#	modified:   .git-annex/p5/1P/WORM-s53736-m1278673485--Archive-Tar-1.64.tar.gz.log
+#	modified:   .git-annex/wv/zG/WORM-s30584-m1268774021--ExtUtils-CBuilder-0.2703.tar.gz.log
+#	modified:   .git-annex/x5/7v/WORM-s10462526-m1254242591--BioPerl-1.6.1.tar.gz.log
+#
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	.git-annex/1G/X3/
+#	.git-annex/3W/Xf/
+#	.git-annex/9q/Wz/
+#	.git-annex/Fq/4z/
+#	.git-annex/Jk/zK/
+#	.git-annex/Kx/w1/
+#	.git-annex/VK/kv/
+#	.git-annex/fv/0P/
+#	.git-annex/jX/qM/
+#	.git-annex/vW/jK/
+#	.git-annex/vW/v1/
+jtang@x00:~/sources $ git commit -a -m \"snap\"
+[master 45f254a] snap
+ 47 files changed, 64 insertions(+), 30 deletions(-)
+jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 3 commits.
+#
+# Untracked files:
+#   (use \"git add ...\" to include in what will be committed)
+#
+#	.git-annex/1G/X3/
+#	.git-annex/3W/Xf/
+#	.git-annex/9q/Wz/
+#	.git-annex/Fq/4z/
+#	.git-annex/Jk/zK/
+#	.git-annex/Kx/w1/
+#	.git-annex/VK/kv/
+#	.git-annex/fv/0P/
+#	.git-annex/jX/qM/
+#	.git-annex/vW/jK/
+#	.git-annex/vW/v1/
+nothing added to commit but untracked files present (use \"git add\" to track)
+jtang@x00:~/sources $ git pull
+

+"""]]

From 8db6b4f655096d141af5b8a25cf76826c02d9fdd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:16:36 -0400
Subject: [PATCH 1271/8313] response

---
 ...has_issues_with_git_when_staging__47__commiting_logs.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index b7944b418c..554cfa41e2 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -16,3 +16,8 @@ For now it's just a bit of extra work for me when it does occur but it does not
 >>> be some error message? --[[Joey]] 
 
 >>>> there were no error messages at all
+
+>>>>> Can I see a transcript? I'm having difficulty getting my head around
+>>>>> what git is doing. Sounds like the files could just not be `git
+>>>>> added` yet, but I get the impression from other things that you say
+>>>>> that it's not so simple. --[[Joey]] 

From 73b4fe5d61e492fe537f04d3971f25970b7ebf3f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 15:20:59 +0000
Subject: [PATCH 1272/8313]

---
 doc/forum/wishlist:_git_backend_for_git-annex.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex.mdwn

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex.mdwn b/doc/forum/wishlist:_git_backend_for_git-annex.mdwn
new file mode 100644
index 0000000000..63ae83097e
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex.mdwn
@@ -0,0 +1,7 @@
+Preamble: Obviously, the core feature of git-annex is the ability to keep a subset of files in a local repo. The main trade-off is that you don't get version tracking.
+
+Use case: On my laptop, I might not have enough disk space to store everything. Not so for my main box nor my backup server. And I would _really_ like to have proper version tracking for many of my files. Thus...
+
+Wish: ...why not use git as a version backend? That way, I could just push all my stuff to the central instance(s) and have the best of both worlds. Depending on what backend is used in the local repos, it might make sense to define a list of supported client backends with pre-computed keys.
+
+-- RichiH

From 70d4df79ee1ada9a773d03579a86b798f3e1ff1d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 15:25:18 +0000
Subject: [PATCH 1273/8313] Added a comment

---
 ...3_a18ada7ac74c63be5753fdb2fe68dae5._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
new file mode 100644
index 0000000000..4c2f609d9c
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-28T15:25:18Z"
+ content="""
+So, there is evidence here of a circumstance caused by the [[other_bug|git-annex_has_issues_with_git_when_staging__47__commiting_logs]], as I suspected.
+
+I don't think that manual `git commit -a` caused the problem. I suspect it was a subsequent `git add` that caused git to follow the wrong case paths and add the files in the wrong place. Ie, when you run \"git add .git-annex\", it recurses into `.git-annex/Gm/`, and adds files using that case, that were previously added from `.git-annex/GM/`.
+
+For completeness, can you verify this repo's core.ignorecase setting?
+
+---
+
+I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a look filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.
+
+It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd murder myself -- upgrading all my offline disks from v1 to v2 took me many days.
+"""]]

From dd4004c43b95389440d9eadbcefc72dd726269b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 11:28:34 -0400
Subject: [PATCH 1274/8313] wording

---
 .../comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
index 4c2f609d9c..00988ab58c 100644
--- a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment
@@ -12,7 +12,7 @@ For completeness, can you verify this repo's core.ignorecase setting?
 
 ---
 
-I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a look filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.
+I hate that you are stuck using loop filesystems to work around this bug. If my guess is correct, you don't need to, as long as you avoid manually running \"git add .git-annex\". I take this bug seriously. While I'm currently very involved in adding Amazon S3 support to git-annex (which will take days more of solid work), I do plan to make a loop filesystem of my own, probably vfat, so I can try and reproduce this on a case-insensative filesystem. If you could confirm my above hypothesis, that would speed things up for me.
 
-It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd murder myself -- upgrading all my offline disks from v1 to v2 took me many days.
+It's possible I will have to tweak the hash directories. Hopefully if so, I will only tweak them for *new* keys; if I had to do a v3 backend just to fix this stupid thing, I'd be sad -- upgrading all my offline disks from v1 to v2 took me many days.
 """]]

From 2b8abd920101be74da8559694125a19f062e2c71 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 15:31:52 +0000
Subject: [PATCH 1275/8313]

---
 doc/forum/git-annex_communication_channels.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels.mdwn

diff --git a/doc/forum/git-annex_communication_channels.mdwn b/doc/forum/git-annex_communication_channels.mdwn
new file mode 100644
index 0000000000..8c56ac36a2
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels.mdwn
@@ -0,0 +1,10 @@
+Thought I'd ask how y'all are finding the current communication by this forum/website/git repo only.
+
+Would there be a benefit to having an irc channel for git-annex?
+
+Maybe a mailing list? (Any persuasive reason why it would be better than this forum?)
+
+Are the existing RSS feeds on this site, for eg, new [[comments]] and posts to this forum, sufficient to keep up with
+things?
+
+--[[Joey]]

From 06342e016930562f5e233b6521fad3a2997fd700 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 15:44:56 +0000
Subject: [PATCH 1276/8313] Added a comment

---
 .../comment_4_039e945617a6c1852c96974a402db29c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment
new file mode 100644
index 0000000000..d045f71205
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2011-03-28T15:41:56Z"
+ content="""
+In my \"sources\" repo on x00, the current setting is this \"ignorecase = true\" it was the first repo that I created before I clone it elsewhere and pull my changes back, it is on a HFS+ partition which is case insensitive and it is replicated on a portable hdd with a bare repo on a exfat partition. I wonder if my portable disk has a partially borked repo :P
+"""]]

From 62cf351e5afc96a94a4a1ec010649a40b339bf62 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 15:51:08 +0000
Subject: [PATCH 1277/8313] Added a comment

---
 ..._1_198325d2e9337c90f026396de89eec0e._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment b/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment
new file mode 100644
index 0000000000..163aae02cc
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-28T15:48:08Z"
+ content="""
+No matter what you end up doing, I would appreciate a git-annex-announce@ list.
+
+I really like the persistence of ikiwiki, but it's not ideal for quick communication. I would be fine with IRC and/or ML. The advantage of a ML over ikiwiki is that it doesn't seem to be as \"wasteful\" to mix normal chat with actual problem-solving.  But maybe that's merely my own perception.
+
+Speaking of RSS: I thought I had added a wishlist item to ikiwiki about providing per-subsite RSS feeds. For example there is no (obvious) way to subscribe to changes in http://git-annex.branchable.com/forum/git-annex_communication_channels/ .
+
+FWIW, I resorted to tagging my local clone of git-annex to keep track of what I've read, already.
+
+
+-- RichiH
+"""]]

From 8a20d0d7440c7668bbc977d74bb99e14c79cd606 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 15:51:32 +0000
Subject: [PATCH 1278/8313] Added a comment

---
 ..._eacd0b18475c05ab9feed8cf7290b79a._comment | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment
new file mode 100644
index 0000000000..7127a6eef8
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2011-03-28T15:51:11Z"
+ content="""
+I also failed to mention, that in the case when i have stray log files after what has happened in comment 2, I get this left over after a commit when git is confused...
+
+
+
+jtang@x00:~/sources $ git status
+# On branch master
+# Your branch is ahead of 'origin/master' by 1 commit.
+#
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   .git-annex/1G/X3/WORM-s309910751-m1301311322--l_fcompxe_ia32_2011.2.137.tgz.log
+#	modified:   .git-annex/3W/Xf/WORM-s805764902-m1301312756--l_cproc_p_11.1.075_intel64.log
+#	modified:   .git-annex/9Q/Wz/WORM-s1234430253-m1301311891--l_ccompxe_2011.2.137.log
+#	modified:   .git-annex/FQ/4z/WORM-s318168323-m1301310848--l_cprof_p_11.1.075_ia64.log
+#	modified:   .git-annex/FV/0P/WORM-s710135470-m1301311835--l_ccompxe_intel64_2011.2.137.log
+#	modified:   .git-annex/Jk/zK/WORM-s374617670-m1301312705--l_ipp_7.0.2.137_intel64.log
+#	modified:   .git-annex/Jx/qM/WORM-s599386592-m1301310731--l_fcompxe_2011.2.137.tgz.log
+#	modified:   .git-annex/KX/w1/WORM-s35976002-m1301312193--l_tbb_3.0.6.174.log
+#	modified:   .git-annex/VK/kv/WORM-s584342291-m1301312669--l_cproc_p_11.1.075_ia64.log
+#	modified:   .git-annex/Vw/jK/WORM-s15795178-m1301310913--w_flm_p_1.0.011_intel64.zip.log
+#	modified:   .git-annex/Zq/7X/WORM-s343075585-m1301312233--l_ipp_7.0.2.137_ia32.log
+#	modified:   .git-annex/vW/v1/WORM-s736986678-m1301312794--l_cproc_p_11.1.075_ia32.log
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+

+
+
+Up until now I have just been updating the status of the staged files by hand and commiting it on my mac x00, this probably isn't helping. I'd rather not lose the tracking information.
+"""]]

From d8ac58a15ea48bfed8338000ac44833c37a11e82 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 16:01:30 +0000
Subject: [PATCH 1279/8313] Added a comment

---
 ...comment_1_04319051fedc583e6c326bb21fcce5a5._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment
new file mode 100644
index 0000000000..a691393b1a
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-28T16:01:30Z"
+ content="""
+Indeed, see [[todo/add_a_git_backend]], where you and I have already discussed this idea. :)
+
+With the new support for special remotes, which will be used by S3, it would be possible to make such a git repo, using bup, be a special remote. I think it would be pretty easy to implement now. Not a priority for me though.
+"""]]

From cf99575d74d2a68004efa152ea9e7ac1c2866935 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 12:09:47 -0400
Subject: [PATCH 1280/8313] update

---
 Makefile                                                  | 1 +
 ..._issues_with_git_when_staging__47__commiting_logs.mdwn | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e1aaf8ec37..8e16645034 100644
--- a/Makefile
+++ b/Makefile
@@ -58,6 +58,7 @@ docs: $(mans)
 		--no-usedirs --disable-plugin=openid --plugin=sidebar \
 		--underlaydir=/dev/null --disable-plugin=shortcut \
 		--disable-plugin=smiley \
+		--plugin=comments --set comments_pagespec="*" \
 		--exclude='news/.*'
 
 clean:
diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 554cfa41e2..774ca6a16c 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -7,7 +7,13 @@ For now it's just a bit of extra work for me when it does occur but it does not
 > What do you mean when you say that git "got wedged"? It hung somehow?
 >
 > If git-annex runs concurrently with another git command that locks
-> the repository its git add of log files can fail.
+> the repository, its git add of log files can fail.
+> 
+> Update: Also, of course, if you are running a "got annex get" or
+> similar, and ctrl-c it after it has gotten some files, it can
+> end up with unstaged or in some cases un-added log files that git-annex
+> wrote -- since git-annex only stages log files in git on shutdown, and
+> ctrl-c bypasses that.
 > --[[Joey]] 
 
 >> It "got wedged" as in git doesn't let me commit anything, even though it tells me that there is stuff to be committed in the staging area.

From 0cb63c0737a0a4896c78a45ab8bfd56125c95c50 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 16:57:50 +0000
Subject: [PATCH 1281/8313] Comment moderation

---
 .../comment_1_7cb5561f11dfc7726a537ddde2477489._comment   | 8 ++++++++
 .../comment_4_e4911dc6793f98fb81151daacbe49968._comment   | 8 ++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment
 create mode 100644 doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment
new file mode 100644
index 0000000000..35a2cdb3fe
--- /dev/null
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-03-21T13:15:03Z"
+ content="""
+Maybe I will run into issues myself somewhere down the road, but generally speaking, I really really like the fact that files are immutable by default.
+"""]]
diff --git a/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment b/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment
new file mode 100644
index 0000000000..e8c9837462
--- /dev/null
+++ b/doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-25T11:23:04Z"
+ content="""
+FWIW, I wanted to suggest exactly the same thing.
+"""]]

From 0591ecb2a108ca8d3d1cb6b649c4a2913c96b593 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 13:20:12 -0400
Subject: [PATCH 1282/8313] docs

---
 doc/transferring_data.mdwn | 3 ++-
 doc/use_case/Alice.mdwn    | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index 9526a3e48c..f6ae9bfcde 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -1,7 +1,8 @@
 git-annex can transfer data to or from any of a repository's git remotes.
 Depending on where the remote is, the data transfer is done using rsync
 (over ssh, with automatic resume), or plain cp (with copy-on-write
-optimisations on supported filesystems).
+optimisations on supported filesystems). Some [[special_remotes]]
+are also supported that are not traditional git remotes.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever
diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn
index ccc0727bce..ee97efa43a 100644
--- a/doc/use_case/Alice.mdwn
+++ b/doc/use_case/Alice.mdwn
@@ -2,9 +2,9 @@
 
 Alice is always on the move, often with her trusty netbook and a small
 handheld terabyte USB drive, or a smaller USB keydrive. She has a server
-out there on the net. All these things can have different files on them,
-but Alice no longer has to deal with the tedious process of keeping them
-manually in sync.
+out there on the net. She stores data in the Cloud. All these things can
+have different files on them, but Alice no longer has to deal with the
+tedious process of keeping them manually in sync.
 
 When she has 1 bar on her cell, Alice queues up interesting files on her
 server for later. At a coffee shop, she has git-annex download them to her

From 3162a724f1bfdc15efadd939a49ba8740d553d69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 13:47:29 -0400
Subject: [PATCH 1283/8313] S3 updates; gpg keys

---
 debian/changelog                     |  2 ++
 doc/git-annex.mdwn                   | 16 +++++-----
 doc/special_remotes.mdwn             |  6 +---
 doc/special_remotes/Amazon_S3.mdwn   | 45 ++++++++++++++++++++++++++++
 doc/walkthrough/using_Amazon_S3.mdwn | 12 ++++----
 5 files changed, 63 insertions(+), 18 deletions(-)
 create mode 100644 doc/special_remotes/Amazon_S3.mdwn

diff --git a/debian/changelog b/debian/changelog
index 03a0091cbf..825382256d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (0.20110329) UNRELEASED; urgency=low
 
   * Amazon S3 is now supported as a special type of remote.
+    Warning: Encrypting data before sending it to S3 is not currently
+    supported.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c01f4fbc59..3985addc6c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,20 +132,22 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* s3bucket name description [datacenter host port] [--key=gpgkey]
+* s3bucket name gpgkey [datacenter host port]
 
-  Creates a bucket in Amazon S3. The bucket's name can be used
-  to configure git remote using the bucket.
+  Create or updates the key of a bucket in Amazon S3. The bucket's
+  name can be used to configure git remote using the bucket.
+
+  The gpgkey is a value that can be looked up (using gpg -k) to
+  find a gpg encryption key that will be given access to the bucket.
+  To disable encryption, specify "unencrypted". Note that additional gpg
+  keys can be given access to a bucket by running s3bucket on an existing
+  bucket, with a new key.
 
   The datacenter defaults to "US". Other values include "EU",
   "us-west-1", and "ap-southeast-1".
 
   To use a different, S3-compatable service, specify a host and port.
 
-  By default, data (including filenames) is encrypted using gpg.
-  To use a key other than the default gpg key, specify it with
-  the --key option. To disable encryption, specify "none".
-
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 717ec48404..651b24afa4 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -6,8 +6,4 @@ But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
 They cannot be used by other git commands though.
 
-## Amazon S3
-
-Stores file contents in a bucket in Amazon S3 or a similar service.
-Content is stored encrypted by gpg.
-See [[walkthrough/using_Amazon_S3]] for examples.
+* [[Amazon_S3]]
diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
new file mode 100644
index 0000000000..dce0a9241f
--- /dev/null
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -0,0 +1,45 @@
+This special remote type stores file contents in a bucket in Amazon S3
+or a similar service.
+
+See [[walkthrough/using_Amazon_S3]] for usage examples.
+
+## bucket names
+
+When `git annex s3bucket` is used to create a new bucket, it generates a
+UUID, and the name of the bucket includes that UUID, as well as the name
+specified by the user. This makes for some unweidly bucket names, but
+since S3 requires that bucket names be globally unique, it avoids needing
+to hunt for a unused bucket name.
+
+## data security
+
+When `git annex s3bucket` is used to create an unencrypted bucket,
+there is **no** protection against your data being read as it is sent
+to/from S3, or by Amazon when it is stored in S3. This should only be used
+for public data.
+
+** Encryption is not yet supported. **
+
+When an encrypted bucket is created, all files stored in the bucket are
+encrypted with gpg. Additionally, the filenames themselves are hashed
+to obfuscate them. The size of the encrypted files, and access patterns of
+the data, should be the only clues to what type of data you are storing in
+S3.
+
+[[!template id=note text="""
+This scheme was originally developed by Lars Wirzenius at al [for Obnam](http://braawi.org/obnam/encryption/).
+"""]]
+The data stored in S3 is encrypted by gpg with a symmetric cipher. The
+passphrase of the cipher is itself checked into your git repository,
+encrypted using one or more gpg public keys. This scheme allows new public
+keys to be given access to a bucket's content, after the bucket is created
+and is in use. It also allows revoking compromised public keys without
+having to throw out the contents of the bucket. The symmetric cipher
+is also hashed together with filenames used in the bucket, obfuscate
+the filenames.
+
+To add a new gpg key to an existing bucket, just re-run `git annex
+s3bucket`, specifying the new key id. For example:
+
+	# git annex s3bucket mybucket 16D0B8EF
+	s3bucket (adding gpg key 16D0B8EF) ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 8cb77ab6cd..2833a9c5a4 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -9,8 +9,11 @@ First, export your S3 credentials:
 
 Next, create a bucket, giving it a name and a description:
 
-	git annex s3bucket mybucket "my Amazon S3 bucket"
-	s3bucket (creating mybucket...) ok
+	git annex s3bucket mybucket unencrypted
+	s3bucket (creating mybucket...) (no encryption!) ok
+
+**Note that encrypted buckets are not (yet) supported. Data sent to S3
+is susceptible to snooping.**
 
 Finally, configure a git remote to use the bucket you created:
 
@@ -23,7 +26,4 @@ Now the remote can be used like any other remote.
 	# git annex move video/hackity_hack_and_kaxxt.mov --to mys3
 	move video/hackity_hack_and_kaxxt.mov (to mys3...) ok
 
-An Amazon S3 remote works just like a ssh remote, except it does not have
-a git repository at the other end, and it costs you money. :) In particular,
-all data is stored encrypted with gpg, so neither Amazon nor anyone in
-between can see it.
+See [[special_remotes/Amazon_S3]] for details.

From bd40e0c777ec164a0edbb716f12587acdfcd09e8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 13:49:48 -0400
Subject: [PATCH 1284/8313] thinko

---
 doc/special_remotes/Amazon_S3.mdwn | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index dce0a9241f..67bea3b1c0 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -27,13 +27,14 @@ the data, should be the only clues to what type of data you are storing in
 S3.
 
 [[!template id=note text="""
-This scheme was originally developed by Lars Wirzenius at al [for Obnam](http://braawi.org/obnam/encryption/).
+This scheme was originally developed by Lars Wirzenius at al
+[for Obnam](http://braawi.org/obnam/encryption/).
 """]]
 The data stored in S3 is encrypted by gpg with a symmetric cipher. The
 passphrase of the cipher is itself checked into your git repository,
-encrypted using one or more gpg public keys. This scheme allows new public
+encrypted using one or more gpg public keys. This scheme allows new private
 keys to be given access to a bucket's content, after the bucket is created
-and is in use. It also allows revoking compromised public keys without
+and is in use. It also allows revoking compromised private keys without
 having to throw out the contents of the bucket. The symmetric cipher
 is also hashed together with filenames used in the bucket, obfuscate
 the filenames.

From 6ba8a53de19263c7dcfd2e5e4683a716ac358f24 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 17:50:38 +0000
Subject: [PATCH 1285/8313] Added a comment

---
 ...ent_2_7f529f19a47e10b571f65ab382e97fd5._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment
new file mode 100644
index 0000000000..14798e7a71
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-28T17:47:38Z"
+ content="""
+On the plus side, the past me wanted exactly what I had in mind.
+
+On the meh side, I really forgot about this conversation :/
+
+When you say this todo is not a priority, does that mean there's no ETA at all and that it will most likely sleep for a long time? Or the almost usual \"what the heck, I will just wizard it up in two lines of haskell\"?
+
+-- RichiH
+"""]]

From 0212126f67660ce2753aa2fb32c643107243f2af Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 28 Mar 2011 18:35:51 +0000
Subject: [PATCH 1286/8313] Added a comment

---
 .../comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment b/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment
new file mode 100644
index 0000000000..09a2b8c1a5
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2011-03-28T18:35:50Z"
+ content="""
+I think the forums/website currently is sufficient, I do at times wish there was a mailing list or anonymous git push to the wiki as I find editing posts through the web browser is some times tedious (the lack of !fmt or alt-q bugs me at times ;) ). The main advantage of keeping stuff on the site/forum is that everything gets saved and passed on to anyone who checks out the git repo of the code base.
+"""]]

From 974b1e18d14fa0a311a7379d0ce6eb34ca3c2cf9 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 28 Mar 2011 20:05:13 +0000
Subject: [PATCH 1287/8313] Added a comment

---
 ...comment_3_a077bbad3e4b07cce019eb55a45330e7._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment
new file mode 100644
index 0000000000..8c3286d27b
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-28T20:05:13Z"
+ content="""
+Probably more like 150 lines of haskell. Maybe just 50 lines if the bup repository is required to be on the same computer as the git-annex repository.
+
+Since I do have some repositories where I'd appreciate this level of assurance that data not be lost, it's mostly a matter of me finding a free day.
+"""]]

From 0956f0dd15f3114d7227787578499210f0c17db8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 16:19:20 -0400
Subject: [PATCH 1288/8313] fsck: Ensure that files and directories in
 .git/annex/objects have proper permissions.

---
 Command/Fsck.hs                                       | 11 ++++++++++-
 debian/changelog                                      |  7 +++++++
 ...k__47__fix_the_permissions_of_.git__47__annex.mdwn |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 216c87493b..bedb9fb992 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,6 +7,7 @@
 
 module Command.Fsck where
 
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 
 import Command
@@ -18,6 +19,7 @@ import Messages
 import Utility
 import Content
 import LocationLog
+import Locations
 
 command :: [Command]
 command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
@@ -47,9 +49,16 @@ perform key file backend numcopies = do
    in this repository only. -}
 verifyLocationLog :: Key -> FilePath -> Annex Bool
 verifyLocationLog key file = do
+	g <- Annex.gitRepo
 	present <- inAnnex key
 	
-	g <- Annex.gitRepo
+	-- Since we're checking that a key's file is present, throw
+	-- in a permission fixup here too.
+	when present $ liftIO $ do
+		let f = gitAnnexLocation g key
+		preventWrite f
+		preventWrite (parentDir f)
+
 	u <- getUUID g
         uuids <- liftIO $ keyLocations g key
 
diff --git a/debian/changelog b/debian/changelog
index 7251ab6653..1b09165538 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (0.20110329) UNRELEASED; urgency=low
+
+  * fsck: Ensure that files and directories in .git/annex/objects
+    have proper permissions.
+
+ -- Joey Hess   Mon, 28 Mar 2011 16:17:59 -0400
+
 git-annex (0.20110328) experimental; urgency=low
 
   * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes")
diff --git a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
index ec8b6d2330..c649ff9f7c 100644
--- a/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
+++ b/doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn
@@ -4,3 +4,5 @@ The fsck command should check that they are still correct.
 The fix command should fix them.
 
 PS: Thanks for this nice tool!
+
+> Good idea, [[done]] (actually, fsck just fixes them too)! --[[Joey]]

From caef7c82213e00695679bd9c934a4edef0a04eaa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 16:35:59 -0400
Subject: [PATCH 1289/8313] nix on revocation

---
 doc/special_remotes/Amazon_S3.mdwn | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index 67bea3b1c0..ae3990a76e 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -34,10 +34,8 @@ The data stored in S3 is encrypted by gpg with a symmetric cipher. The
 passphrase of the cipher is itself checked into your git repository,
 encrypted using one or more gpg public keys. This scheme allows new private
 keys to be given access to a bucket's content, after the bucket is created
-and is in use. It also allows revoking compromised private keys without
-having to throw out the contents of the bucket. The symmetric cipher
-is also hashed together with filenames used in the bucket, obfuscate
-the filenames.
+and is in use. The symmetric cipher is also hashed together with filenames
+used in the bucket, in order to obfuscate the filenames.
 
 To add a new gpg key to an existing bucket, just re-run `git annex
 s3bucket`, specifying the new key id. For example:

From 216817e0d42cb69768b42c79cbdb7f6072f5a47e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 20:45:36 +0000
Subject: [PATCH 1290/8313] Added a comment

---
 .../comment_4_ecca429e12d734b509c671166a676c9d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment

diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment
new file mode 100644
index 0000000000..cf649a8a25
--- /dev/null
+++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-03-28T20:45:35Z"
+ content="""
+Personally, I would not mind a requirement to keep a local bup repo. I wouldn't want my data to to unncessarily complex setups, anyway. -- RichiH
+"""]]

From 42483ba19856aa1e51054d100364e45fdaf8696c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 28 Mar 2011 20:47:23 +0000
Subject: [PATCH 1291/8313] Added a comment

---
 .../comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment b/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment
new file mode 100644
index 0000000000..72a48445ed
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-03-28T20:47:23Z"
+ content="""
+Push access to the non-code bits of git-annex' ikiwiki would be very welcome indeed. Given the choice, I would rather edit everything in Vim than in a browser. -- RichiH
+"""]]

From 58af57493418d80eb9fb3c4719f20442725aa7f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 19:08:12 -0400
Subject: [PATCH 1292/8313] generalize special remote configuration storage

---
 doc/git-annex.mdwn                   | 39 ++++++---------------------
 doc/internals.mdwn                   | 14 +++++-----
 doc/special_remotes/Amazon_S3.mdwn   | 40 ++++++++++++++++------------
 doc/walkthrough/using_Amazon_S3.mdwn | 10 +++----
 4 files changed, 40 insertions(+), 63 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3985addc6c..ce5b380d0e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,21 +132,17 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* s3bucket name gpgkey [datacenter host port]
+* initremote name [type=value param=value ...]
 
-  Create or updates the key of a bucket in Amazon S3. The bucket's
-  name can be used to configure git remote using the bucket.
+  Sets up a [[special_remote|special_remotes]] of some type. The remote's
+  type and configuration is specified by the parameters. If a remote
+  with the specified name has already been configured, its configuration
+  is modified by any values specified. In either case, the remote will be
+  added added to `.git/config`.
 
-  The gpgkey is a value that can be looked up (using gpg -k) to
-  find a gpg encryption key that will be given access to the bucket.
-  To disable encryption, specify "unencrypted". Note that additional gpg
-  keys can be given access to a bucket by running s3bucket on an existing
-  bucket, with a new key.
+  Example Amazon S3 remote:
 
-  The datacenter defaults to "US". Other values include "EU",
-  "us-west-1", and "ap-southeast-1".
-
-  To use a different, S3-compatable service, specify a host and port.
+	initremote mys3 type=S3 encryption=none datacenter=EU
 
 * fsck [path ...]
 
@@ -403,25 +399,6 @@ Here are all the supported configuration settings.
   Default ssh and rsync options to use if a remote does not have
   specific options.
 
-* `remote..annex-s3-access-key-id`
-
-  Your S3 Access Key ID. Does not need to be kept private.
-  If not set, the environment variable `AWS_ACCESS_KEY_ID`
-  will be used.
-
-* `remote..annex-s3-secret-access-key`
-
-  Your S3 Secret Access Key. This is a password.
-  If not set, the environment variable `AWS_SECRET_ACCESS_KEY`
-  will be used.
-
-* `remote..annex-s3-storageclass`
-
-  Storage class to use when adding new content to S3. The default
-  is "STANDARD". If you have configured git-annex to preserve
-  multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"
-  to save money.
-
 * `annex.diskreserve`
 
   Amount of disk space to reserve. Disk space is checked when transferring
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 55b1045a11..6296095035 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -30,16 +30,14 @@ space and then the description through to the end of the line. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
 	26339d22-446b-11e0-9101-002170d25c55 usb disk
 
-## `git-annex/s3.log`
+## `git-annex/remotes.log`
 
-Associates the UUIDs of Amazon S3 buckets with a bucket nickname and connection
-information. Example:
+Holds persistent configuration settings for [[special_remotes]] such as
+Amazon S3.
 
-	be72acb8-5901-11e0-b600-002170d25c55 mybucket s3.amazonaws.com 80
-
-Note that the actual bucket name used on S3 in the above example 
-is "mybucket-be72acb8-5901-11e0-b600-002170d25c55". The UUID is included
-in the bucket name to ensure it is globally unique.
+The file format is one line per remote, starting with the uuid of the
+remote, followed by a space, the name of the remote, a space, and then
+a series of key=value pairs, each separated by whitespace.
 
 ## `.git-annex/trust.log`
 
diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index ae3990a76e..42c4a54534 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -3,24 +3,36 @@ or a similar service.
 
 See [[walkthrough/using_Amazon_S3]] for usage examples.
 
-## bucket names
+## initremote parameters
 
-When `git annex s3bucket` is used to create a new bucket, it generates a
-UUID, and the name of the bucket includes that UUID, as well as the name
-specified by the user. This makes for some unweidly bucket names, but
-since S3 requires that bucket names be globally unique, it avoids needing
-to hunt for a unused bucket name.
+A number of parameters can be passed to `git annex initremote` to configure
+the S3 remote.
+
+* `encryption` - Either "none" to disable encryption,
+  or a value that can be looked up (using gpg -k) to find a gpg encryption
+  key that will be given access to the remote. Note that additional gpg
+  keys can be given access to a remote by rerunning initremote with
+  the new key id.
+
+* `datacenter` - Defaults to "US". Other values include "EU",
+  "us-west-1", and "ap-southeast-1".
+
+* `storageclass` - Default is "STANDARD". If you have configured git-annex
+  to preserve multiple [[copies]], consider setting this to "REDUCED_REDUNDANCY"
+  to save money.
+
+* `host` and `port` - Specify in order to use a different, S3 compatable
+  service.
 
 ## data security
 
-When `git annex s3bucket` is used to create an unencrypted bucket,
-there is **no** protection against your data being read as it is sent
-to/from S3, or by Amazon when it is stored in S3. This should only be used
-for public data.
+When encryption=none, there is **no** protection against your data being read
+as it is sent to/from S3, or by Amazon when it is stored in S3. This should
+only be used for public data.
 
 ** Encryption is not yet supported. **
 
-When an encrypted bucket is created, all files stored in the bucket are
+When encryption is enabled, all files stored in the bucket are
 encrypted with gpg. Additionally, the filenames themselves are hashed
 to obfuscate them. The size of the encrypted files, and access patterns of
 the data, should be the only clues to what type of data you are storing in
@@ -36,9 +48,3 @@ encrypted using one or more gpg public keys. This scheme allows new private
 keys to be given access to a bucket's content, after the bucket is created
 and is in use. The symmetric cipher is also hashed together with filenames
 used in the bucket, in order to obfuscate the filenames.
-
-To add a new gpg key to an existing bucket, just re-run `git annex
-s3bucket`, specifying the new key id. For example:
-
-	# git annex s3bucket mybucket 16D0B8EF
-	s3bucket (adding gpg key 16D0B8EF) ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 2833a9c5a4..b87238a327 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -7,18 +7,14 @@ First, export your S3 credentials:
 	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
 	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
 
-Next, create a bucket, giving it a name and a description:
+Next, create the remote.
 
-	git annex s3bucket mybucket unencrypted
-	s3bucket (creating mybucket...) (no encryption!) ok
+	git annex initremote mys3 encryption=none
+	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
 
 **Note that encrypted buckets are not (yet) supported. Data sent to S3
 is susceptible to snooping.**
 
-Finally, configure a git remote to use the bucket you created:
-
-	git config remote.mys3.annex-s3-bucket mybucket
-
 Now the remote can be used like any other remote.
 
 	# git annex copy my_cool_big_file --to mys3

From 235720d27e5c1044ddd8904d7140c9e8841e5715 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 22:05:11 -0400
Subject: [PATCH 1293/8313] tweak

---
 doc/git-annex.mdwn                   | 20 ++++++++++----------
 doc/walkthrough/using_Amazon_S3.mdwn |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ce5b380d0e..4d14623942 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -126,23 +126,23 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * describe repository description
 
-  Changes the description of a git repository.
+  Changes the description of a repository.
 
   The repository to describe can be specified by git remote name or
   by uuid. To change the description of the current repository, use
   "."
 
-* initremote name [type=value param=value ...]
+* initremote type name [param=value ...]
 
   Sets up a [[special_remote|special_remotes]] of some type. The remote's
-  type and configuration is specified by the parameters. If a remote
+  configuration is configured by the parameters. If a remote
   with the specified name has already been configured, its configuration
   is modified by any values specified. In either case, the remote will be
   added added to `.git/config`.
 
   Example Amazon S3 remote:
 
-	initremote mys3 type=S3 encryption=none datacenter=EU
+	initremote s3 mys3 type=S3 encryption=none datacenter=EU
 
 * fsck [path ...]
 
@@ -318,12 +318,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * --from=repository
 
   Specifies a repository that content will be retrieved from.
-  It should be specified using the name of a configured git remote.
+  It should be specified using the name of a configured remote.
 
 * --to=repository
 
-  Specifies a git repository that content will be sent to. 
-  It should be specified using the name of a configured git remote.
+  Specifies a repository that content will be sent to. 
+  It should be specified using the name of a configured remote.
 
 * --exclude=glob
 
@@ -382,16 +382,16 @@ Here are all the supported configuration settings.
 
 * `remote..annex-uuid`
 
-  git-annex caches UUIDs of repositories here.
+  git-annex caches UUIDs of remote repositories here.
 
 * `remote..annex-ssh-options`
 
-  Options to use when using ssh to talk to this repository.
+  Options to use when using ssh to talk to this remote.
 
 * `remote..annex-rsync-options`
 
   Options to use when using rsync
-  to or from this repository. For example, to force ipv6, and limit
+  to or from this remote. For example, to force ipv6, and limit
   the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100"
 
 * `annex.ssh-options`, `annex.rsync-options`
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index b87238a327..38a6a6de59 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -9,11 +9,11 @@ First, export your S3 credentials:
 
 Next, create the remote.
 
-	git annex initremote mys3 encryption=none
+	git annex initremote s3 mys3 encryption=none
 	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
 
 **Note that encrypted buckets are not (yet) supported. Data sent to S3
-is susceptible to snooping.**
+is without encryption susceptible to snooping.**
 
 Now the remote can be used like any other remote.
 

From b1db436816b6b70ff0b9891bbc4a5468d9b895b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 23:22:31 -0400
Subject: [PATCH 1294/8313] started on initremote

---
 Command.hs                           |  4 ++
 Command/InitRemote.hs                | 48 ++++++++++++++++++
 GitAnnex.hs                          |  2 +
 Remote.hs                            | 74 +++++++++++++++++++++++++++-
 doc/git-annex.mdwn                   |  6 +--
 doc/walkthrough/using_Amazon_S3.mdwn |  2 +-
 6 files changed, 131 insertions(+), 5 deletions(-)
 create mode 100644 Command/InitRemote.hs

diff --git a/Command.hs b/Command.hs
index 446b1b55fe..9c908b8004 100644
--- a/Command.hs
+++ b/Command.hs
@@ -238,6 +238,10 @@ paramGlob :: String
 paramGlob = "GLOB"
 paramName :: String
 paramName = "NAME"
+paramType :: String
+paramType = "TYPE"
+paramKeyValue :: String
+paramKeyValue = "K=V"
 paramNothing :: String
 paramNothing = ""
 
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
new file mode 100644
index 0000000000..cf6a341c53
--- /dev/null
+++ b/Command/InitRemote.hs
@@ -0,0 +1,48 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.InitRemote where
+
+import qualified Data.Map as M
+import Control.Monad (when)
+import Control.Monad.State (liftIO)
+
+import Command
+import qualified Remote
+import UUID
+import Messages
+
+command :: [Command]
+command = [repoCommand "initremote"
+	(paramPair paramName $
+		paramOptional $ paramRepeating $ paramKeyValue) seek
+	"sets up a special (non-git) remote"]
+
+seek :: [CommandSeek]
+seek = [withString start]
+
+start :: CommandStartString
+start params = notBareRepo $ do
+	when (null ws) $ error "Specify a name for the remote"
+	showStart "initremote" name
+	r <- Remote.configGet name
+	(u, c) <- case r of
+		Just t -> return t
+		Nothing -> do
+			uuid <- liftIO $ genUUID
+			return $ (uuid, M.empty)
+	return $ Just $ perform name u $ M.union config c
+
+	where
+		ws = words params
+		name = head ws
+		config = Remote.keyValToMap $ tail ws
+
+perform :: String -> UUID -> M.Map String String -> CommandPerform
+perform name uuid config = do
+	liftIO $ putStrLn $ show $ (uuid, config)
+	return Nothing
diff --git a/GitAnnex.hs b/GitAnnex.hs
index adf07e5b3e..736b430e60 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -27,6 +27,7 @@ import qualified Command.SetKey
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Describe
+import qualified Command.InitRemote
 import qualified Command.Fsck
 import qualified Command.Unused
 import qualified Command.DropUnused
@@ -55,6 +56,7 @@ cmds = concat
 	, Command.Lock.command
 	, Command.Init.command
 	, Command.Describe.command
+	, Command.InitRemote.command
 	, Command.Unannex.command
 	, Command.Uninit.command
 	, Command.PreCommit.command
diff --git a/Remote.hs b/Remote.hs
index 6aab4a741c..f281d565a4 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -19,13 +19,19 @@ module Remote (
 	nameToUUID,
 	keyPossibilities,
 	remotesWithUUID,
-	remotesWithoutUUID
+	remotesWithoutUUID,
+
+	configGet,
+	configSet,
+	keyValToMap
 ) where
 
 import Control.Monad.State (liftIO)
 import Control.Monad (when, liftM)
 import Data.List
 import Data.String.Utils
+import qualified Data.Map as M
+import Data.Maybe
 
 import RemoteClass
 import qualified Remote.Git
@@ -35,6 +41,7 @@ import UUID
 import qualified Annex
 import Trust
 import LocationLog
+import Locations
 import Messages
 
 {- Add generators for new Remotes here. -}
@@ -120,3 +127,68 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
+{- Filename of remote.log. -}
+remoteLog :: Annex FilePath
+remoteLog = do
+	g <- Annex.gitRepo
+	return $ gitStateDir g ++ "remote.log"
+
+{- Reads the uuid and config of the specified remote from the remoteLog. -}
+configGet :: String -> Annex (Maybe (UUID, M.Map String String))
+configGet n = do
+	rs <- readRemoteLog
+	let matches = filter (matchName n) rs
+	case matches of
+		[] -> return Nothing
+		((u, _, c):_) -> return $ Just (u, c)
+
+{- Changes or adds a remote's config in the remoteLog. -}
+configSet :: String -> UUID -> M.Map String String -> Annex ()
+configSet n u c = do
+	rs <- readRemoteLog
+	let others = filter (not . matchName n) rs
+	writeRemoteLog $ (u, n, c):others
+
+matchName :: String -> (UUID, String, M.Map String String) -> Bool
+matchName n (_, n', _) = n == n'
+
+readRemoteLog :: Annex [(UUID, String, M.Map String String)]
+readRemoteLog = do
+	l <- remoteLog
+	s <- liftIO $ catch (readFile l) ignoreerror
+	return $ remoteLogParse s
+	where
+		ignoreerror _ = return []
+
+writeRemoteLog :: [(UUID, String, M.Map String String)] -> Annex ()
+writeRemoteLog rs = do
+	l <- remoteLog
+	liftIO $ writeFile l $ unlines $ map toline rs
+	where
+		toline (u, n, c) = u ++ " " ++ n ++ (unwords $ mapToKeyVal c)
+
+remoteLogParse :: String -> [(UUID, String, M.Map String String)]
+remoteLogParse s = catMaybes $ map parseline $ filter (not . null) $ lines s
+	where
+		parseline l
+			| length w > 2 = Just (u, n, c)
+			| otherwise = Nothing
+			where
+				w = words l
+				u = w !! 0
+				n = w !! 1
+				c = keyValToMap $ drop 2 w
+
+{- Given Strings like "key=value", generates a Map. -}
+keyValToMap :: [String] -> M.Map String String
+keyValToMap ws = M.fromList $ map (/=/) ws
+	where
+		(/=/) s = (k, v)
+			where
+				k = takeWhile (/= '=') s
+				v = drop (1 + length k) s
+
+mapToKeyVal :: M.Map String String -> [String]
+mapToKeyVal m = map toword $ M.toList m
+	where
+		toword (k, v) = k ++ "=" ++ v
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 4d14623942..0f548fa8a5 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -132,17 +132,17 @@ Many git-annex commands will stage changes for later `git commit` by you.
   by uuid. To change the description of the current repository, use
   "."
 
-* initremote type name [param=value ...]
+* initremote name [param=value ...]
 
   Sets up a [[special_remote|special_remotes]] of some type. The remote's
-  configuration is configured by the parameters. If a remote
+  type and configuration is specified by the parameters. If a remote
   with the specified name has already been configured, its configuration
   is modified by any values specified. In either case, the remote will be
   added added to `.git/config`.
 
   Example Amazon S3 remote:
 
-	initremote s3 mys3 type=S3 encryption=none datacenter=EU
+	initremote mys3 type=S3 encryption=none datacenter=EU
 
 * fsck [path ...]
 
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 38a6a6de59..a99746c955 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -9,7 +9,7 @@ First, export your S3 credentials:
 
 Next, create the remote.
 
-	git annex initremote s3 mys3 encryption=none
+	git annex initremote mys3 type=S3 encryption=none
 	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
 
 **Note that encrypted buckets are not (yet) supported. Data sent to S3

From a3b6586902d6689b07c050b1fc50e19f4115c42e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Mar 2011 23:51:07 -0400
Subject: [PATCH 1295/8313] update

---
 Remote.hs      | 19 +++++++++----------
 Remote/Git.hs  | 14 ++++++++++----
 Remote/S3.hs   | 14 ++++++++++----
 RemoteClass.hs | 17 ++++++++++++++++-
 4 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index f281d565a4..71bc08c8ae 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -44,17 +44,16 @@ import LocationLog
 import Locations
 import Messages
 
-{- Add generators for new Remotes here. -}
-generators :: [Annex (RemoteGenerator Annex)]
-generators =
-	[ Remote.Git.generate
-	, Remote.S3.generate
+remoteTypes :: [RemoteType Annex]
+remoteTypes =
+	[ Remote.Git.remote
+	, Remote.S3.remote
 	]
 
-{- Runs a list of generators. -}
-runGenerators :: [Annex (RemoteGenerator Annex)] -> Annex [Remote Annex]
-runGenerators gs = do
-	(actions, expensive) <- collect ([], []) gs
+{- Runs the generators of each type of Remote -}
+runGenerators :: Annex [Remote Annex]
+runGenerators = do
+	(actions, expensive) <- collect ([], []) $ map generator remoteTypes
 	when (not $ null expensive) $
 		showNote $ "getting UUID for " ++ join ", " expensive
 	sequence actions
@@ -71,7 +70,7 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			rs' <- runGenerators generators
+			rs' <- runGenerators
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9021a2230c..68bd172e91 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -6,7 +6,7 @@
  -}
 
 module Remote.Git (
-	generate,
+	remote,
 	onRemote
 ) where
 
@@ -30,8 +30,11 @@ import RsyncFile
 import Ssh
 import Config
 
-generate :: Annex (RemoteGenerator Annex)
-generate = do
+remote :: RemoteType Annex
+remote = RemoteType { typename = "git", generator = gen }
+
+gen :: Annex (RemoteGenerator Annex)
+gen = do
 	g <- Annex.gitRepo
 	allremotes <- filterM remoteNotIgnored $ Git.remotes g
 
@@ -64,7 +67,10 @@ genRemote r = do
 		retrieveKeyFile = copyFromRemote r,
 		removeKey = dropKey r,
 		hasKey = inAnnex r,
-		hasKeyCheap = not (Git.repoIsUrl r)
+		hasKeyCheap = not (Git.repoIsUrl r),
+		hasConfig = False,
+		config = Nothing,
+		setup = \_ -> return ()
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 23ec33bb59..4aa1bc639b 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.S3 (generate) where
+module Remote.S3 (remote) where
 
 import Network.AWS.AWSConnection
 import Network.AWS.S3Object
@@ -27,8 +27,11 @@ import qualified Annex
 import UUID
 import Config
 
-generate :: Annex (RemoteGenerator Annex)
-generate = do
+remote :: RemoteType Annex
+remote = RemoteType { typename = "S3", generator = gen }
+
+gen :: Annex (RemoteGenerator Annex)
+gen = do
 	g <- Annex.gitRepo
 	remotes <- filterM remoteNotIgnored $ findS3Remotes g
 	todo <- filterM cachedUUID remotes
@@ -64,7 +67,10 @@ genRemote r u = do
 		retrieveKeyFile = error "TODO",
 		removeKey = error "TODO",
 		hasKey = error "TODO",
-		hasKeyCheap = False
+		hasKeyCheap = False,
+		hasConfig = True,
+		config = Nothing,
+		setup = \_ -> return ()
 	}
 
 s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
diff --git a/RemoteClass.hs b/RemoteClass.hs
index eb4a017486..f3cc9379b0 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -10,6 +10,7 @@
 module RemoteClass where
 
 import Control.Exception
+import Data.Map as M
 
 import Key
 
@@ -18,6 +19,15 @@ import Key
  - that are not cheap to set up. -}
 type RemoteGenerator a = ([a (Remote a)], [String])
 
+{- There are different types of remotes. -}
+data RemoteType a = RemoteType {
+	-- human visible type name
+	typename :: String,
+	-- generates remotes of this type
+	generator :: a (RemoteGenerator a)
+}
+
+{- An individual remote. -}
 data Remote a = Remote {
 	-- each Remote has a unique uuid
 	uuid :: String,
@@ -36,7 +46,12 @@ data Remote a = Remote {
 	hasKey :: Key -> a (Either IOException Bool),
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
-	hasKeyCheap :: Bool
+	hasKeyCheap :: Bool,
+	-- a Remote may have a persistent configuration store
+	hasConfig :: Bool,
+	config :: Maybe (M.Map String String),
+	-- initializes or changes the config of a remote
+	setup :: M.Map String String -> a ()
 }
 
 instance Show (Remote a) where

From 97e3486f15eee4ac038b3e21d295b789091b0aef Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 29 Mar 2011 12:11:38 +0000
Subject: [PATCH 1296/8313]

---
 doc/forum/batch_check_on_remote_when_using_copy.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/forum/batch_check_on_remote_when_using_copy.mdwn b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
index b08c33b8ba..633b61d6f9 100644
--- a/doc/forum/batch_check_on_remote_when_using_copy.mdwn
+++ b/doc/forum/batch_check_on_remote_when_using_copy.mdwn
@@ -25,3 +25,10 @@ Once all checks are done, one single transfer session should be started. Creatin
 > While I do hope to improve ssh usage so that it sshs once, and feeds
 > `git-annex-shell` a series of commands to run, that is a much longer-term
 > thing. --[[Joey]]
+
+>> FYI, in a repo with 1228 files, all small, repos _completely in sync_.
+
+    % git annex copy . --to foo # 1200 seconds
+    % git annex copy . --to foo --fast # 20 seconds
+
+>> RichiH

From 05751d55cd8002e6a2a2afc520622fb6697472e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 13:49:54 -0400
Subject: [PATCH 1297/8313] clean up remote.log handling

---
 Command/InitRemote.hs | 20 +++++++++++---
 Remote.hs             | 63 +++++++++++++++++++++----------------------
 Remote/Git.hs         |  5 ++--
 Remote/S3.hs          |  7 +++--
 RemoteClass.hs        |  3 +--
 doc/internals.mdwn    |  4 +--
 6 files changed, 55 insertions(+), 47 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index cf6a341c53..0d9a40cd33 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -29,12 +29,12 @@ start :: CommandStartString
 start params = notBareRepo $ do
 	when (null ws) $ error "Specify a name for the remote"
 	showStart "initremote" name
-	r <- Remote.configGet name
-	(u, c) <- case r of
+	m <- Remote.readRemoteLog
+	(u, c) <- case findByName name m of
 		Just t -> return t
 		Nothing -> do
 			uuid <- liftIO $ genUUID
-			return $ (uuid, M.empty)
+			return $ (uuid, M.insert nameKey name M.empty)
 	return $ Just $ perform name u $ M.union config c
 
 	where
@@ -46,3 +46,17 @@ perform :: String -> UUID -> M.Map String String -> CommandPerform
 perform name uuid config = do
 	liftIO $ putStrLn $ show $ (uuid, config)
 	return Nothing
+
+findByName :: String ->  M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String)
+findByName n m = if null matches then Nothing else Just $ head matches
+	where
+		matches = filter (matching . snd) $ M.toList m
+		matching c = case M.lookup nameKey c of
+			Nothing -> False
+			Just n'
+				| n' == n -> True
+				| otherwise -> False
+
+{- The name of a configured remote is stored in its config using this key. -}
+nameKey :: String
+nameKey = "name"
diff --git a/Remote.hs b/Remote.hs
index 71bc08c8ae..f79b512625 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -21,7 +21,7 @@ module Remote (
 	remotesWithUUID,
 	remotesWithoutUUID,
 
-	configGet,
+	readRemoteLog,
 	configSet,
 	keyValToMap
 ) where
@@ -71,7 +71,8 @@ genList = do
 	if null rs
 		then do
 			rs' <- runGenerators
-			Annex.changeState $ \s -> s { Annex.remotes = rs' }
+			rs'' <- getConfigs rs'
+			Annex.changeState $ \s -> s { Annex.remotes = rs'' }
 			return rs'
 		else return rs
 
@@ -132,51 +133,47 @@ remoteLog = do
 	g <- Annex.gitRepo
 	return $ gitStateDir g ++ "remote.log"
 
-{- Reads the uuid and config of the specified remote from the remoteLog. -}
-configGet :: String -> Annex (Maybe (UUID, M.Map String String))
-configGet n = do
-	rs <- readRemoteLog
-	let matches = filter (matchName n) rs
-	case matches of
-		[] -> return Nothing
-		((u, _, c):_) -> return $ Just (u, c)
+{- Load stored config into remotes.
+ -
+ - This way, the log is read once, lazily, so if no remotes access
+ - their config, no work is done.
+ -}
+getConfigs :: [Remote Annex] -> Annex [Remote Annex]
+getConfigs rs = do
+	m <- readRemoteLog
+	return $ map (get m) rs	
+	where
+		get m r = r { config = M.lookup (uuid r) m }
 
-{- Changes or adds a remote's config in the remoteLog. -}
-configSet :: String -> UUID -> M.Map String String -> Annex ()
-configSet n u c = do
-	rs <- readRemoteLog
-	let others = filter (not . matchName n) rs
-	writeRemoteLog $ (u, n, c):others
+{- Adds or updates a remote's config in the log. -}
+configSet :: UUID -> M.Map String String -> Annex ()
+configSet u c = do
+	m <- readRemoteLog
+	l <- remoteLog
+	liftIO $ writeFile l $ unlines $ map toline $ M.toList $ M.insert u c m
+	where
+		toline (u', c') = u' ++ " " ++ (unwords $ mapToKeyVal c')
 
-matchName :: String -> (UUID, String, M.Map String String) -> Bool
-matchName n (_, n', _) = n == n'
-
-readRemoteLog :: Annex [(UUID, String, M.Map String String)]
+{- Map of remotes by uuid containing key/value config maps. -}
+readRemoteLog :: Annex (M.Map UUID (M.Map String String))
 readRemoteLog = do
 	l <- remoteLog
 	s <- liftIO $ catch (readFile l) ignoreerror
 	return $ remoteLogParse s
 	where
-		ignoreerror _ = return []
+		ignoreerror _ = return ""
 
-writeRemoteLog :: [(UUID, String, M.Map String String)] -> Annex ()
-writeRemoteLog rs = do
-	l <- remoteLog
-	liftIO $ writeFile l $ unlines $ map toline rs
-	where
-		toline (u, n, c) = u ++ " " ++ n ++ (unwords $ mapToKeyVal c)
-
-remoteLogParse :: String -> [(UUID, String, M.Map String String)]
-remoteLogParse s = catMaybes $ map parseline $ filter (not . null) $ lines s
+remoteLogParse :: String -> M.Map UUID (M.Map String String)
+remoteLogParse s =
+	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
 	where
 		parseline l
-			| length w > 2 = Just (u, n, c)
+			| length w > 2 = Just (u, c)
 			| otherwise = Nothing
 			where
 				w = words l
 				u = w !! 0
-				n = w !! 1
-				c = keyValToMap $ drop 2 w
+				c = keyValToMap $ tail w
 
 {- Given Strings like "key=value", generates a Map. -}
 keyValToMap :: [String] -> M.Map String String
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 68bd172e91..b686e47af5 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,7 +12,7 @@ module Remote.Git (
 
 import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
-import qualified Data.Map as Map
+import qualified Data.Map as M
 import System.Cmd.Utils
 import Control.Monad (filterM, liftM)
 
@@ -68,7 +68,6 @@ genRemote r = do
 		removeKey = dropKey r,
 		hasKey = inAnnex r,
 		hasKeyCheap = not (Git.repoIsUrl r),
-		hasConfig = False,
 		config = Nothing,
 		setup = \_ -> return ()
 	}
@@ -77,7 +76,7 @@ genRemote r = do
  - returns the updated repo. -}
 tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
-	| not $ Map.null $ Git.configMap r = return r -- already read
+	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 4aa1bc639b..7971faa8fb 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -12,7 +12,7 @@ import Network.AWS.S3Object
 import Network.AWS.S3Bucket
 import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
-import qualified Data.Map as Map
+import qualified Data.Map as M
 import Data.String.Utils
 import Control.Monad (filterM, liftM, when)
 import Control.Monad.State (liftIO)
@@ -51,8 +51,8 @@ gen = do
 findS3Remotes :: Git.Repo -> [Git.Repo]
 findS3Remotes r = map construct remotepairs
 	where
-		remotepairs = Map.toList $ filterremotes $ Git.configMap r
-		filterremotes = Map.filterWithKey (\k _ -> s3remote k)
+		remotepairs = M.toList $ filterremotes $ Git.configMap r
+		filterremotes = M.filterWithKey (\k _ -> s3remote k)
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
 		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
 
@@ -68,7 +68,6 @@ genRemote r u = do
 		removeKey = error "TODO",
 		hasKey = error "TODO",
 		hasKeyCheap = False,
-		hasConfig = True,
 		config = Nothing,
 		setup = \_ -> return ()
 	}
diff --git a/RemoteClass.hs b/RemoteClass.hs
index f3cc9379b0..0482faac70 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -47,8 +47,7 @@ data Remote a = Remote {
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool,
-	-- a Remote may have a persistent configuration store
-	hasConfig :: Bool,
+	-- a Remote can have a persistent configuration store
 	config :: Maybe (M.Map String String),
 	-- initializes or changes the config of a remote
 	setup :: M.Map String String -> a ()
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 6296095035..2e9f253831 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -36,8 +36,8 @@ Holds persistent configuration settings for [[special_remotes]] such as
 Amazon S3.
 
 The file format is one line per remote, starting with the uuid of the
-remote, followed by a space, the name of the remote, a space, and then
-a series of key=value pairs, each separated by whitespace.
+remote, followed by a space, and then a series of key=value pairs,
+each separated by whitespace.
 
 ## `.git-annex/trust.log`
 

From 0a4c610b4fd78f7d1589117cb723d7d8c15c120c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 14:55:59 -0400
Subject: [PATCH 1298/8313] initremote works

---
 Command/InitRemote.hs | 66 ++++++++++++++++++++++++++++++++++---------
 Remote.hs             | 13 ++++++---
 Remote/Git.hs         |  9 ++++--
 Remote/S3.hs          | 13 +++++++--
 RemoteClass.hs        |  8 +++---
 doc/git-annex.mdwn    |  6 ++--
 6 files changed, 85 insertions(+), 30 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 0d9a40cd33..39ec366539 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -12,7 +12,12 @@ import Control.Monad (when)
 import Control.Monad.State (liftIO)
 
 import Command
+import qualified Annex
 import qualified Remote
+import qualified RemoteClass
+import qualified GitRepo as Git
+import Utility
+import Types
 import UUID
 import Messages
 
@@ -28,27 +33,49 @@ seek = [withString start]
 start :: CommandStartString
 start params = notBareRepo $ do
 	when (null ws) $ error "Specify a name for the remote"
+
+	(u, c) <- findByName name
+	let fullconfig = M.union config c	
+	t <- findType fullconfig
+	
 	showStart "initremote" name
-	m <- Remote.readRemoteLog
-	(u, c) <- case findByName name m of
-		Just t -> return t
-		Nothing -> do
-			uuid <- liftIO $ genUUID
-			return $ (uuid, M.insert nameKey name M.empty)
-	return $ Just $ perform name u $ M.union config c
+	return $ Just $ perform t u $ M.union config c
 
 	where
 		ws = words params
 		name = head ws
 		config = Remote.keyValToMap $ tail ws
 
-perform :: String -> UUID -> M.Map String String -> CommandPerform
-perform name uuid config = do
-	liftIO $ putStrLn $ show $ (uuid, config)
-	return Nothing
+perform :: RemoteClass.RemoteType Annex -> UUID -> M.Map String String -> CommandPerform
+perform t u c = do
+	c' <- RemoteClass.setup t u c
+	return $ Just $ cleanup u c'
 
-findByName :: String ->  M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String)
-findByName n m = if null matches then Nothing else Just $ head matches
+cleanup :: UUID -> M.Map String String -> CommandCleanup
+cleanup u c = do
+	Remote.configSet u c
+	g <- Annex.gitRepo
+	logfile <- Remote.remoteLog
+	liftIO $ Git.run g "add" [File logfile]
+        liftIO $ Git.run g "commit"
+                [ Params "-q --allow-empty -m"
+                , Param "git annex initremote"
+                , File logfile
+                ]
+        return True
+
+{- Look up existing remote's UUID and config by name, or generate a new one -}
+findByName :: String -> Annex (UUID, M.Map String String)
+findByName name = do
+	m <- Remote.readRemoteLog
+	case findByName' name m of
+		Just i -> return i
+		Nothing -> do
+			uuid <- liftIO $ genUUID
+			return $ (uuid, M.insert nameKey name M.empty)
+
+findByName' :: String ->  M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String)
+findByName' n m = if null matches then Nothing else Just $ head matches
 	where
 		matches = filter (matching . snd) $ M.toList m
 		matching c = case M.lookup nameKey c of
@@ -57,6 +84,19 @@ findByName n m = if null matches then Nothing else Just $ head matches
 				| n' == n -> True
 				| otherwise -> False
 
+{- find the specified remote type -}
+findType :: M.Map String String -> Annex (RemoteClass.RemoteType Annex)
+findType config = 
+	case M.lookup typeKey config of
+		Nothing -> error "Specify the type of remote with type="
+		Just s -> case filter (\i -> RemoteClass.typename i == s) Remote.remoteTypes of
+			[] -> error $ "Unknown remote type " ++ s
+			(t:_) -> return t
+
 {- The name of a configured remote is stored in its config using this key. -}
 nameKey :: String
 nameKey = "name"
+
+{- The type of a remote is stored in its config using this key. -}
+typeKey :: String
+typeKey = "type"
diff --git a/Remote.hs b/Remote.hs
index f79b512625..1ca05d77ba 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,12 +15,14 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 
+	remoteTypes,
 	byName,
 	nameToUUID,
 	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID,
 
+	remoteLog,
 	readRemoteLog,
 	configSet,
 	keyValToMap
@@ -34,8 +36,6 @@ import qualified Data.Map as M
 import Data.Maybe
 
 import RemoteClass
-import qualified Remote.Git
-import qualified Remote.S3
 import Types
 import UUID
 import qualified Annex
@@ -43,6 +43,10 @@ import Trust
 import LocationLog
 import Locations
 import Messages
+import Utility
+
+import qualified Remote.Git
+import qualified Remote.S3
 
 remoteTypes :: [RemoteType Annex]
 remoteTypes =
@@ -150,7 +154,8 @@ configSet :: UUID -> M.Map String String -> Annex ()
 configSet u c = do
 	m <- readRemoteLog
 	l <- remoteLog
-	liftIO $ writeFile l $ unlines $ map toline $ M.toList $ M.insert u c m
+	liftIO $ safeWriteFile l $ unlines $ sort $
+		map toline $ M.toList $ M.insert u c m
 	where
 		toline (u', c') = u' ++ " " ++ (unwords $ mapToKeyVal c')
 
@@ -185,6 +190,6 @@ keyValToMap ws = M.fromList $ map (/=/) ws
 				v = drop (1 + length k) s
 
 mapToKeyVal :: M.Map String String -> [String]
-mapToKeyVal m = map toword $ M.toList m
+mapToKeyVal m = map toword $ sort $ M.toList m
 	where
 		toword (k, v) = k ++ "=" ++ v
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b686e47af5..2d7a0c8ff7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -31,7 +31,11 @@ import Ssh
 import Config
 
 remote :: RemoteType Annex
-remote = RemoteType { typename = "git", generator = gen }
+remote = RemoteType {
+	typename = "git",
+	generator = gen,
+	setup = error "not supported"
+}
 
 gen :: Annex (RemoteGenerator Annex)
 gen = do
@@ -68,8 +72,7 @@ genRemote r = do
 		removeKey = dropKey r,
 		hasKey = inAnnex r,
 		hasKeyCheap = not (Git.repoIsUrl r),
-		config = Nothing,
-		setup = \_ -> return ()
+		config = Nothing
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 7971faa8fb..489114b126 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -28,7 +28,11 @@ import UUID
 import Config
 
 remote :: RemoteType Annex
-remote = RemoteType { typename = "S3", generator = gen }
+remote = RemoteType {
+	typename = "S3",
+	generator = gen,
+	setup = s3Setup
+}
 
 gen :: Annex (RemoteGenerator Annex)
 gen = do
@@ -68,8 +72,7 @@ genRemote r u = do
 		removeKey = error "TODO",
 		hasKey = error "TODO",
 		hasKeyCheap = False,
-		config = Nothing,
-		setup = \_ -> return ()
+		config = Nothing
 	}
 
 s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
@@ -102,6 +105,10 @@ getS3Config r s def = do
 	where
 		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
 
+s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String)
+s3Setup u c = do
+	return c
+
 {- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
  - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
  - also creating the bucket. -}
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 0482faac70..825197a4bc 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -24,7 +24,9 @@ data RemoteType a = RemoteType {
 	-- human visible type name
 	typename :: String,
 	-- generates remotes of this type
-	generator :: a (RemoteGenerator a)
+	generator :: a (RemoteGenerator a),
+	-- initializes or changes a remote
+	setup :: String -> M.Map String String -> a (M.Map String String)
 }
 
 {- An individual remote. -}
@@ -48,9 +50,7 @@ data Remote a = Remote {
 	-- operation.
 	hasKeyCheap :: Bool,
 	-- a Remote can have a persistent configuration store
-	config :: Maybe (M.Map String String),
-	-- initializes or changes the config of a remote
-	setup :: M.Map String String -> a ()
+	config :: Maybe (M.Map String String)
 }
 
 instance Show (Remote a) where
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0f548fa8a5..d890b518b8 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -134,11 +134,11 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * initremote name [param=value ...]
 
-  Sets up a [[special_remote|special_remotes]] of some type. The remote's
-  type and configuration is specified by the parameters. If a remote
+  Sets up a [[special_remote|special_remotes]]. The remote's
+  configuration is specified by the parameters. If a remote
   with the specified name has already been configured, its configuration
   is modified by any values specified. In either case, the remote will be
-  added added to `.git/config`.
+  added to `.git/config`.
 
   Example Amazon S3 remote:
 

From e62f9816ab29dceb5489d520b9cf569ede2ffb52 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 15:12:07 -0400
Subject: [PATCH 1299/8313] doc update for S3

---
 doc/special_remotes/Amazon_S3.mdwn   |  6 +++++-
 doc/walkthrough/using_Amazon_S3.mdwn | 28 ++++++++++++++++++----------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index 42c4a54534..c8e44b609c 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -8,7 +8,7 @@ See [[walkthrough/using_Amazon_S3]] for usage examples.
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.
 
-* `encryption` - Either "none" to disable encryption,
+* `encryption` - Required. Either "none" to disable encryption,
   or a value that can be looked up (using gpg -k) to find a gpg encryption
   key that will be given access to the remote. Note that additional gpg
   keys can be given access to a remote by rerunning initremote with
@@ -24,6 +24,10 @@ the S3 remote.
 * `host` and `port` - Specify in order to use a different, S3 compatable
   service.
 
+* `bucket` - S3 requires that buckets have a globally unique name, 
+  so by default, a bucket name is chosen based on the remote name
+  and UUID. This can be specified to pick a bucket name.
+
 ## data security
 
 When encryption=none, there is **no** protection against your data being read
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index a99746c955..34c843b18f 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -2,19 +2,27 @@ git-annex extends git's usual remotes with some [[special_remotes]], that
 are not git repositories. This way you can set up a remote using say,
 Amazon S3, and use git-annex to transfer files into the cloud.
 
-First, export your S3 credentials:
-
-	export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
-
-Next, create the remote.
-
-	git annex initremote mys3 type=S3 encryption=none
-	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
-
 **Note that encrypted buckets are not (yet) supported. Data sent to S3
 is without encryption susceptible to snooping.**
 
+First, export your S3 credentials:
+
+	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+
+Next, create the S3 remote.
+
+	# git annex initremote mys3 type=S3 encryption=none
+	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
+
+The configuration for the S3 remote is stored in git. So to make a different
+repository use the same S3 remote is easy:
+
+	# cd /media/usb/annex
+        # git pull laptop master
+	# git annex initremote mys3
+	initremote ok
+
 Now the remote can be used like any other remote.
 
 	# git annex copy my_cool_big_file --to mys3

From 475f7073613b7164302e3f826f60929cf4cd38f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 16:21:21 -0400
Subject: [PATCH 1300/8313] initremote now creates buckets

---
 Remote.hs                            |  21 +---
 Remote/Git.hs                        |  16 +--
 Remote/S3.hs                         | 147 ++++++++++++++++-----------
 RemoteClass.hs                       |   7 +-
 doc/walkthrough/using_Amazon_S3.mdwn |   4 +-
 5 files changed, 106 insertions(+), 89 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 1ca05d77ba..03615ac6ef 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -31,7 +31,6 @@ module Remote (
 import Control.Monad.State (liftIO)
 import Control.Monad (when, liftM)
 import Data.List
-import Data.String.Utils
 import qualified Data.Map as M
 import Data.Maybe
 
@@ -42,7 +41,6 @@ import qualified Annex
 import Trust
 import LocationLog
 import Locations
-import Messages
 import Utility
 
 import qualified Remote.Git
@@ -54,19 +52,6 @@ remoteTypes =
 	, Remote.S3.remote
 	]
 
-{- Runs the generators of each type of Remote -}
-runGenerators :: Annex [Remote Annex]
-runGenerators = do
-	(actions, expensive) <- collect ([], []) $ map generator remoteTypes
-	when (not $ null expensive) $
-		showNote $ "getting UUID for " ++ join ", " expensive
-	sequence actions
-	where
-		collect v [] = return v
-		collect (actions, expensive) (x:xs) = do
-			(a, e) <- x
-			collect (a++actions, e++expensive) xs
-
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached in the Annex. -}
 genList :: Annex [Remote Annex]
@@ -74,9 +59,9 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			rs' <- runGenerators
-			rs'' <- getConfigs rs'
-			Annex.changeState $ \s -> s { Annex.remotes = rs'' }
+			l <- mapM generator remoteTypes
+			rs' <- getConfigs (concat l)
+			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2d7a0c8ff7..85bd04a23a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -15,6 +15,8 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 import Control.Monad (filterM, liftM)
+import Data.String.Utils
+import Maybe
 
 import RemoteClass
 import Types
@@ -37,7 +39,7 @@ remote = RemoteType {
 	setup = error "not supported"
 }
 
-gen :: Annex (RemoteGenerator Annex)
+gen :: Annex [Remote Annex]
 gen = do
 	g <- Annex.gitRepo
 	allremotes <- filterM remoteNotIgnored $ Git.remotes g
@@ -52,18 +54,20 @@ gen = do
 	let skip = filter (`notElem` expensive_todo) expensive
 	let todo = cheap++expensive_todo
 
-	let actions = map genRemote skip ++ 
-		map (\r -> genRemote =<< tryGitConfigRead r) todo
-	return (actions, map Git.repoDescribe expensive_todo)
+	showNote $ "getting UUID for " ++ (join ", " $
+		map Git.repoDescribe expensive_todo)
+	done <- mapM tryGitConfigRead todo
 
+	generated <- mapM genRemote $ skip ++ done
+	return $ catMaybes generated
 	where
 		cachedUUID r = liftM null $ getUUID r
 	
-genRemote :: Git.Repo -> Annex (Remote Annex)
+genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
 genRemote r = do
 	u <- getUUID r
 	c <- remoteCost r
-	return Remote {
+	return $ Just $ Remote {
 		uuid = u,
 		cost = c,
 		name = Git.repoDescribe r,
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 489114b126..16b3992da8 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -13,12 +13,12 @@ import Network.AWS.S3Bucket
 import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
+import Data.Maybe
 import Data.String.Utils
 import Control.Monad (filterM, liftM, when)
 import Control.Monad.State (liftIO)
 import System.Environment
 import Data.Char
-import Messages
 
 import RemoteClass
 import Types
@@ -26,6 +26,8 @@ import qualified GitRepo as Git
 import qualified Annex
 import UUID
 import Config
+import Utility
+import Messages
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -34,21 +36,14 @@ remote = RemoteType {
 	setup = s3Setup
 }
 
-gen :: Annex (RemoteGenerator Annex)
+gen :: Annex [Remote Annex]
 gen = do
 	g <- Annex.gitRepo
-	remotes <- filterM remoteNotIgnored $ findS3Remotes g
-	todo <- filterM cachedUUID remotes
-	let ok = filter (`notElem` todo) remotes
-	
-	let actions = map (\r -> genRemote r =<< getUUID r) ok ++
-		map (\r -> genRemote r =<< getS3UUID r) todo
-	return (actions, map Git.repoDescribe todo)
+	l <- filterM remoteNotIgnored $ findS3Remotes g
+	generated <- mapM genRemote l
+	return $ catMaybes generated
 
-	where
-		cachedUUID r = liftM null $ getUUID r
-
-{- S3 remotes have a remote..annex-s3-bucket config setting.
+{- S3 remotes have a remote..annex-s3 config setting.
  - Git.Repo does not normally generate remotes for things that
  - have no configured url, so the Git.Repo objects have to be
  - constructed as coming from an unknown location. -}
@@ -58,56 +53,81 @@ findS3Remotes r = map construct remotepairs
 		remotepairs = M.toList $ filterremotes $ Git.configMap r
 		filterremotes = M.filterWithKey (\k _ -> s3remote k)
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		s3remote k = startswith "remote." k && endswith ".annex-s3-bucket" k
+		s3remote k = startswith "remote." k && endswith ".annex-s3" k
 
-genRemote :: Git.Repo -> UUID -> Annex (Remote Annex)
-genRemote r u = do
-	c <- remoteCost r
-	return Remote {
-		uuid = u,
-		cost = c,
-		name = Git.repoDescribe r,
-		storeKey = error "TODO",
-		retrieveKeyFile = error "TODO",
-		removeKey = error "TODO",
-		hasKey = error "TODO",
-		hasKeyCheap = False,
-		config = Nothing
-	}
+genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
+genRemote r = do
+	u <- getUUID r
+	if (u == "")
+		then return Nothing
+		else do
+			c <- remoteCost r
+			return $ Just $ Remote {
+				uuid = u,
+				cost = c,
+				name = Git.repoDescribe r,
+				storeKey = error "TODO",
+				retrieveKeyFile = error "TODO",
+				removeKey = error "TODO",
+				hasKey = error "TODO",
+				hasKeyCheap = False,
+				config = Nothing
+			}
 
-s3Connection :: Git.Repo -> Annex (Maybe AWSConnection)
-s3Connection r = do
-	host <- getS3Config r "s3-host" (Just defaultAmazonS3Host)
-	port <- getS3Config r "s3-port" (Just $ show defaultAmazonS3Port)
-	accesskey <- getS3Config r "s3-access-key-id" Nothing
-	secretkey <- getS3Config r "s3-secret-access-key" Nothing
-	case reads port of
-		[(p, _)] -> return $ Just $ AWSConnection host p accesskey secretkey
-		_ -> error $ "bad S3 port value: " ++ port
-
-withS3Connection :: Git.Repo -> Annex a -> ((AWSConnection, String) -> Annex a) -> Annex a
-withS3Connection r def a = do
-	c <- s3Connection r
-	case c of
-		Nothing -> def
-		Just c' -> do
-			b <- getConfig r "s3-bucket" ""
-			a (c', b)
-
-getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
-getS3Config r s def = do
-	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
-	v <- case e of
-		Nothing -> getConfig r s ""
-		Just d -> getConfig r s d
-	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
-	return v
+s3Connection :: M.Map String String -> IO AWSConnection
+s3Connection c = do
+	ak <- getEnvKey "AWS_ACCESS_KEY_ID"
+	sk <- getEnvKey "AWS_SECRET_ACCESS_KEY"
+	return $ AWSConnection host port ak sk
 	where
-		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
+		host = fromJust $ (M.lookup "host" c)
+		port = let s = fromJust $ (M.lookup "port" c) in
+			case reads s of
+			[(p, _)] -> p
+			_ -> error $ "bad S3 port value: " ++ s
+		getEnvKey s = catch (getEnv s) (error $ "Set " ++ s)
 
 s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String)
 s3Setup u c = do
-	return c
+	-- verify configuration is sane
+	case M.lookup "encryption" c of
+		Nothing -> error "Specify encryption=key or encryption=none"
+		Just "none" -> return ()
+		Just k -> error "encryption keys not yet supported"
+	let fullconfig = M.union c defaults
+
+	-- check bucket location to see if the bucket exists
+	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
+	conn <- liftIO $ s3Connection fullconfig
+	showNote "checking bucket"
+	loc <- liftIO $ getBucketLocation conn bucket 
+	case loc of
+		Right _ -> return ()
+		Left err@(NetworkError _) -> error $ prettyReqError err
+		Left (AWSError _ _) -> do
+			showNote "creating bucket"
+			res <- liftIO $ createBucketIn conn bucket datacenter
+			case res of
+				Right _ -> return ()
+				Left err -> error $ prettyReqError err
+
+	g <- Annex.gitRepo
+	liftIO $ do
+		Git.run g "config" [Param ("remote." ++ name ++ ".annex-s3"), Param "true"]
+		Git.run g "config" [Param ("remote." ++ name ++ ".annex-uuid"), Param u]
+	return fullconfig
+	where
+		name = fromJust (M.lookup "name" c)
+		bucket = name ++ "-" ++ u
+		defaults = M.fromList
+			[ ("datacenter", "US")
+			, ("storageclass", "STANDARD")
+			, ("host", defaultAmazonS3Host)
+			, ("port", show defaultAmazonS3Port)
+			, ("bucket", bucket)
+			]
+
+{-
 
 {- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
  - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
@@ -135,3 +155,16 @@ getS3UUID r = withS3Connection r disable $ \(c, b) -> do
 	where
 		uuidfile = "git-annex-uuid"
 		disable = return "" -- empty uuid will disable this remote
+
+getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
+getS3Config r s def = do
+	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
+	v <- case e of
+		Nothing -> getConfig r s ""
+		Just d -> getConfig r s d
+	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
+	return v
+	where
+		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
+
+-}
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 825197a4bc..e16cbdbb01 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -14,17 +14,12 @@ import Data.Map as M
 
 import Key
 
-{- A remote generator identifies configured remotes, and returns an action
- - that can be run to set up each remote, and a list of names of remotes
- - that are not cheap to set up. -}
-type RemoteGenerator a = ([a (Remote a)], [String])
-
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
 	typename :: String,
 	-- generates remotes of this type
-	generator :: a (RemoteGenerator a),
+	generator :: a [Remote a],
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 34c843b18f..5f2766868b 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -13,7 +13,7 @@ First, export your S3 credentials:
 Next, create the S3 remote.
 
 	# git annex initremote mys3 type=S3 encryption=none
-	initremote (creating bucket mys3-291d2fdc-5990-11e0-909a-002170d25c55...) ok
+	initremote mys3 (checking bucket) (creating bucket) ok
 
 The configuration for the S3 remote is stored in git. So to make a different
 repository use the same S3 remote is easy:
@@ -21,7 +21,7 @@ repository use the same S3 remote is easy:
 	# cd /media/usb/annex
         # git pull laptop master
 	# git annex initremote mys3
-	initremote ok
+	initremote mys3 (checking bucket) ok
 
 Now the remote can be used like any other remote.
 

From 72f94cc42eca1a6aaa7cc95daf423915761805ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 17:20:22 -0400
Subject: [PATCH 1301/8313] progress

---
 Remote.hs                            |   4 +-
 Remote/Git.hs                        |  11 +--
 Remote/S3.hs                         | 114 +++++++++++++--------------
 doc/walkthrough/using_Amazon_S3.mdwn |   2 +-
 4 files changed, 66 insertions(+), 65 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 03615ac6ef..1474811853 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -53,7 +53,7 @@ remoteTypes =
 	]
 
 {- Builds a list of all available Remotes.
- - Since doing so can be expensive, the list is cached in the Annex. -}
+ - Since doing so can be expensive, the list is cached. -}
 genList :: Annex [Remote Annex]
 genList = do
 	rs <- Annex.getState Annex.remotes
@@ -130,7 +130,7 @@ remoteLog = do
 getConfigs :: [Remote Annex] -> Annex [Remote Annex]
 getConfigs rs = do
 	m <- readRemoteLog
-	return $ map (get m) rs	
+	return $ map (get m) rs
 	where
 		get m r = r { config = M.lookup (uuid r) m }
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 85bd04a23a..e5f2aa62d0 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -14,7 +14,7 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
-import Control.Monad (filterM, liftM)
+import Control.Monad (filterM, liftM, when)
 import Data.String.Utils
 import Maybe
 
@@ -50,18 +50,19 @@ gen = do
 	 - cached UUID value. -}
 	let cheap = filter (not . Git.repoIsUrl) allremotes
 	let expensive = filter Git.repoIsUrl allremotes
-	expensive_todo <- filterM cachedUUID expensive
+	expensive_todo <- filterM noCachedUUID expensive
 	let skip = filter (`notElem` expensive_todo) expensive
 	let todo = cheap++expensive_todo
 
-	showNote $ "getting UUID for " ++ (join ", " $
-		map Git.repoDescribe expensive_todo)
+	when (not $ null expensive_todo) $
+		showNote $ "getting UUID for " ++ (join ", " $
+			map Git.repoDescribe expensive_todo)
 	done <- mapM tryGitConfigRead todo
 
 	generated <- mapM genRemote $ skip ++ done
 	return $ catMaybes generated
 	where
-		cachedUUID r = liftM null $ getUUID r
+		noCachedUUID r = liftM null $ getUUID r
 	
 genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
 genRemote r = do
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 16b3992da8..887b19e731 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -7,6 +7,7 @@
 
 module Remote.S3 (remote) where
 
+import Control.Exception.Extensible (IOException)
 import Network.AWS.AWSConnection
 import Network.AWS.S3Object
 import Network.AWS.S3Bucket
@@ -15,10 +16,9 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
 import Data.String.Utils
-import Control.Monad (filterM, liftM, when)
+import Control.Monad (filterM, when)
 import Control.Monad.State (liftIO)
 import System.Environment
-import Data.Char
 
 import RemoteClass
 import Types
@@ -62,17 +62,21 @@ genRemote r = do
 		then return Nothing
 		else do
 			c <- remoteCost r
-			return $ Just $ Remote {
-				uuid = u,
-				cost = c,
-				name = Git.repoDescribe r,
-				storeKey = error "TODO",
-				retrieveKeyFile = error "TODO",
-				removeKey = error "TODO",
-				hasKey = error "TODO",
-				hasKeyCheap = False,
-				config = Nothing
-			}
+			return $ Just $ newremote u c
+	where
+		newremote u c = this
+			where
+				this = Remote {
+					uuid = u,
+					cost = c,
+					name = Git.repoDescribe r,
+			 		storeKey = s3Store this,
+					retrieveKeyFile = error "TODO retrievekey",
+					removeKey = error "TODO removekey",
+					hasKey = s3CheckPresent this,
+					hasKeyCheap = False,
+					config = Nothing
+				}
 
 s3Connection :: M.Map String String -> IO AWSConnection
 s3Connection c = do
@@ -93,10 +97,10 @@ s3Setup u c = do
 	case M.lookup "encryption" c of
 		Nothing -> error "Specify encryption=key or encryption=none"
 		Just "none" -> return ()
-		Just k -> error "encryption keys not yet supported"
+		Just _ -> error "encryption keys not yet supported"
 	let fullconfig = M.union c defaults
 
-	-- check bucket location to see if the bucket exists
+	-- check bucket location to see if the bucket exists, and create it
 	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
 	conn <- liftIO $ s3Connection fullconfig
 	showNote "checking bucket"
@@ -105,7 +109,7 @@ s3Setup u c = do
 		Right _ -> return ()
 		Left err@(NetworkError _) -> error $ prettyReqError err
 		Left (AWSError _ _) -> do
-			showNote "creating bucket"
+			showNote $ "creating bucket in " ++ datacenter
 			res <- liftIO $ createBucketIn conn bucket datacenter
 			case res of
 				Right _ -> return ()
@@ -113,12 +117,13 @@ s3Setup u c = do
 
 	g <- Annex.gitRepo
 	liftIO $ do
-		Git.run g "config" [Param ("remote." ++ name ++ ".annex-s3"), Param "true"]
-		Git.run g "config" [Param ("remote." ++ name ++ ".annex-uuid"), Param u]
+		Git.run g "config" [Param (configsetting "annex-s3"), Param "true"]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
 	return fullconfig
 	where
-		name = fromJust (M.lookup "name" c)
-		bucket = name ++ "-" ++ u
+		remotename = fromJust (M.lookup "name" c)
+		bucket = remotename ++ "-" ++ u
+		configsetting s = "remote." ++ remotename ++ "." ++ s
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
@@ -127,44 +132,39 @@ s3Setup u c = do
 			, ("bucket", bucket)
 			]
 
-{-
+s3Action :: Remote Annex -> ((AWSConnection, String) -> Annex a) -> Annex a
+s3Action r a = do
+	when (config r == Nothing) $
+		error $ "Missing configuration for special remote " ++ name r
+	conn <- liftIO $ s3Connection (fromJust $ config r)
+	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
+	a (conn, bucket)
 
-{- The UUID of a S3 bucket is stored in a file "git-annex-uuid" in the
- - bucket. Gets the UUID, or if there is none, sets a new UUID, possibly
- - also creating the bucket. -}
-getS3UUID :: Git.Repo -> Annex UUID
-getS3UUID r = withS3Connection r disable $ \(c, b) -> do
-	res <- liftIO $
-		getObject c $ S3Object b uuidfile "" [] L.empty
+s3File :: Key -> FilePath
+s3File k = show k
+
+s3CheckPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
+	let object = S3Object bucket (s3File k) "" [] L.empty
+	showNote ("checking " ++ name r ++ "...")
+	res <- liftIO $ getObjectInfo conn object
 	case res of
-		Right o -> return $ L.unpack $ obj_data o
-		Left _ -> do
-			location <- getS3Config r "s3-datacenter" (Just "EU")
-			-- bucket may already exist, or not
-			_ <- liftIO $ createBucketIn c b location
-			u <- getUUID r
-			res' <- liftIO $ sendObject c $
-				S3Object b uuidfile "" [] $
-					L.pack u
-			case res' of
-				Right _ -> return u
-				Left e -> do
-					warning $ prettyReqError e
-					disable
-					
-	where
-		uuidfile = "git-annex-uuid"
-		disable = return "" -- empty uuid will disable this remote
+		Right _ -> return $ Right True
+		Left (AWSError _ _) -> return $ Right False
+		Left e -> return $ Left (error $ prettyReqError e)
 
-getS3Config :: Git.Repo -> String -> Maybe String-> Annex String
-getS3Config r s def = do
-	e <- liftIO $ catch (liftM Just $ getEnv envvar) (const $ return def)
-	v <- case e of
-		Nothing -> getConfig r s ""
-		Just d -> getConfig r s d
-	when (null v) $ error $ "set " ++ envvar ++ " or " ++ remoteConfig r s
-	return v
+s3Store :: Remote Annex -> Key -> Annex Bool
+s3Store r k = s3Action r $ \(conn, bucket) -> do
+	let object = setStorageClass storageclass $
+		S3Object bucket (s3File k) "" [] (error "read content here")
+	res <- liftIO $ sendObject conn object
+	case res of
+		Right _ -> return True
+		Left e -> do
+			warning $ prettyReqError e
+			return False
 	where
-		envvar = "ANNEX_" ++ map (\c -> if c == '-' then '_' else toUpper c) s
-
--}
+		storageclass =
+			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
+				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
+				_ -> STANDARD
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 5f2766868b..e0d2296620 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -13,7 +13,7 @@ First, export your S3 credentials:
 Next, create the S3 remote.
 
 	# git annex initremote mys3 type=S3 encryption=none
-	initremote mys3 (checking bucket) (creating bucket) ok
+	initremote mys3 (checking bucket) (creating bucket in US) ok
 
 The configuration for the S3 remote is stored in git. So to make a different
 repository use the same S3 remote is easy:

From 0782d7006365e82c0040b25364fa452b0e00e527 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 17:57:20 -0400
Subject: [PATCH 1302/8313] copy --to S3 works

---
 Remote.hs      | 24 +++++++++-------------
 Remote/Git.hs  | 41 ++++++++++++++++---------------------
 Remote/S3.hs   | 55 +++++++++++++++++++++++++-------------------------
 RemoteClass.hs |  7 +++++--
 4 files changed, 60 insertions(+), 67 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 1474811853..9fd53a2f27 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -59,11 +59,19 @@ genList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
-			l <- mapM generator remoteTypes
-			rs' <- getConfigs (concat l)
+			m <- readRemoteLog
+			l <- mapM (process m) remoteTypes
+			let rs' = concat l
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
+	where
+		process m t = do
+			l <- enumerate t
+			mapM (gen m t) l
+		gen m t r = do
+			u <- getUUID r
+			generate t r (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
@@ -122,18 +130,6 @@ remoteLog = do
 	g <- Annex.gitRepo
 	return $ gitStateDir g ++ "remote.log"
 
-{- Load stored config into remotes.
- -
- - This way, the log is read once, lazily, so if no remotes access
- - their config, no work is done.
- -}
-getConfigs :: [Remote Annex] -> Annex [Remote Annex]
-getConfigs rs = do
-	m <- readRemoteLog
-	return $ map (get m) rs
-	where
-		get m r = r { config = M.lookup (uuid r) m }
-
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> M.Map String String -> Annex ()
 configSet u c = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e5f2aa62d0..984f9c88fe 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -14,9 +14,7 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
-import Control.Monad (filterM, liftM, when)
-import Data.String.Utils
-import Maybe
+import Control.Monad (filterM)
 
 import RemoteClass
 import Types
@@ -35,40 +33,35 @@ import Config
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "git",
-	generator = gen,
+	enumerate = list,
+	generate = gen,
 	setup = error "not supported"
 }
 
-gen :: Annex [Remote Annex]
-gen = do
+list :: Annex [Git.Repo]
+list = do
 	g <- Annex.gitRepo
-	allremotes <- filterM remoteNotIgnored $ Git.remotes g
+	filterM remoteNotIgnored $ Git.remotes g
 
+gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen repo _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
 	 - the config of an URL remote is only read when there is no
 	 - cached UUID value. -}
-	let cheap = filter (not . Git.repoIsUrl) allremotes
-	let expensive = filter Git.repoIsUrl allremotes
-	expensive_todo <- filterM noCachedUUID expensive
-	let skip = filter (`notElem` expensive_todo) expensive
-	let todo = cheap++expensive_todo
-
-	when (not $ null expensive_todo) $
-		showNote $ "getting UUID for " ++ (join ", " $
-			map Git.repoDescribe expensive_todo)
-	done <- mapM tryGitConfigRead todo
-
-	generated <- mapM genRemote $ skip ++ done
-	return $ catMaybes generated
-	where
-		noCachedUUID r = liftM null $ getUUID r
+	let cheap = not $ Git.repoIsUrl repo
+	u <- getUUID repo
+	repo' <- case (cheap, u) of
+		(True, _) -> tryGitConfigRead repo
+		(False, "") -> tryGitConfigRead repo
+		_ -> return repo
+	genRemote repo'
 	
-genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
+genRemote :: Git.Repo -> Annex (Remote Annex)
 genRemote r = do
 	u <- getUUID r
 	c <- remoteCost r
-	return $ Just $ Remote {
+	return $ Remote {
 		uuid = u,
 		cost = c,
 		name = Git.repoDescribe r,
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 887b19e731..4e151e22f3 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -28,20 +28,20 @@ import UUID
 import Config
 import Utility
 import Messages
+import Locations
 
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "S3",
-	generator = gen,
+	enumerate = s3List,
+	generate = s3Gen,
 	setup = s3Setup
 }
 
-gen :: Annex [Remote Annex]
-gen = do
+s3List :: Annex [Git.Repo]
+s3List = do
 	g <- Annex.gitRepo
-	l <- filterM remoteNotIgnored $ findS3Remotes g
-	generated <- mapM genRemote l
-	return $ catMaybes generated
+	filterM remoteNotIgnored $ findS3Remotes g
 
 {- S3 remotes have a remote..annex-s3 config setting.
  - Git.Repo does not normally generate remotes for things that
@@ -55,28 +55,27 @@ findS3Remotes r = map construct remotepairs
 		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
 		s3remote k = startswith "remote." k && endswith ".annex-s3" k
 
-genRemote :: Git.Repo -> Annex (Maybe (Remote Annex))
-genRemote r = do
+s3Gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
+s3Gen r c = do
 	u <- getUUID r
-	if (u == "")
-		then return Nothing
-		else do
-			c <- remoteCost r
-			return $ Just $ newremote u c
+	cst <- remoteCost r
+	return $ genRemote r u c cst
 	where
-		newremote u c = this
-			where
-				this = Remote {
-					uuid = u,
-					cost = c,
-					name = Git.repoDescribe r,
-			 		storeKey = s3Store this,
-					retrieveKeyFile = error "TODO retrievekey",
-					removeKey = error "TODO removekey",
-					hasKey = s3CheckPresent this,
-					hasKeyCheap = False,
-					config = Nothing
-				}
+
+genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
+genRemote r u c cst = this
+	where
+		this = Remote {
+			uuid = u,
+			cost = cst,
+			name = Git.repoDescribe r,
+	 		storeKey = s3Store this,
+			retrieveKeyFile = error "TODO retrievekey",
+			removeKey = error "TODO removekey",
+			hasKey = s3CheckPresent this,
+			hasKeyCheap = False,
+			config = c
+		}
 
 s3Connection :: M.Map String String -> IO AWSConnection
 s3Connection c = do
@@ -155,8 +154,10 @@ s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
 
 s3Store :: Remote Annex -> Key -> Annex Bool
 s3Store r k = s3Action r $ \(conn, bucket) -> do
+	g <- Annex.gitRepo
+	content <- liftIO $ L.readFile $ gitAnnexLocation g k
 	let object = setStorageClass storageclass $
-		S3Object bucket (s3File k) "" [] (error "read content here")
+		S3Object bucket (s3File k) "" [] content
 	res <- liftIO $ sendObject conn object
 	case res of
 		Right _ -> return True
diff --git a/RemoteClass.hs b/RemoteClass.hs
index e16cbdbb01..de4c281f44 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -12,14 +12,17 @@ module RemoteClass where
 import Control.Exception
 import Data.Map as M
 
+import qualified GitRepo as Git
 import Key
 
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
 	typename :: String,
-	-- generates remotes of this type
-	generator :: a [Remote a],
+	-- enumerates remotes of this type
+	enumerate :: a [Git.Repo],
+	-- generates a remote of this type
+	generate :: Git.Repo -> Maybe (M.Map String String) -> a (Remote a),
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }

From d8154eaad3f39e045d7abba187a7d2c1399b89dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:09:22 -0400
Subject: [PATCH 1303/8313] transfering content back from s3 works!

---
 Remote/S3.hs | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 4e151e22f3..3265ced78c 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -70,7 +70,7 @@ genRemote r u c cst = this
 			cost = cst,
 			name = Git.repoDescribe r,
 	 		storeKey = s3Store this,
-			retrieveKeyFile = error "TODO retrievekey",
+			retrieveKeyFile = s3Retrieve this,
 			removeKey = error "TODO removekey",
 			hasKey = s3CheckPresent this,
 			hasKeyCheap = False,
@@ -139,14 +139,13 @@ s3Action r a = do
 	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
 	a (conn, bucket)
 
-s3File :: Key -> FilePath
-s3File k = show k
+bucketKey :: String -> Key -> L.ByteString -> S3Object
+bucketKey bucket k content = S3Object bucket (show k) "" [] content
 
 s3CheckPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
 s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
-	let object = S3Object bucket (s3File k) "" [] L.empty
 	showNote ("checking " ++ name r ++ "...")
-	res <- liftIO $ getObjectInfo conn object
+	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return $ Right True
 		Left (AWSError _ _) -> return $ Right False
@@ -156,8 +155,7 @@ s3Store :: Remote Annex -> Key -> Annex Bool
 s3Store r k = s3Action r $ \(conn, bucket) -> do
 	g <- Annex.gitRepo
 	content <- liftIO $ L.readFile $ gitAnnexLocation g k
-	let object = setStorageClass storageclass $
-		S3Object bucket (s3File k) "" [] content
+	let object = setStorageClass storageclass $ bucketKey bucket k content
 	res <- liftIO $ sendObject conn object
 	case res of
 		Right _ -> return True
@@ -169,3 +167,14 @@ s3Store r k = s3Action r $ \(conn, bucket) -> do
 			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
+
+s3Retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
+	res <- liftIO $ getObject conn $ bucketKey bucket k L.empty
+	case res of
+		Right o -> do
+			liftIO $ L.writeFile f (obj_data o)
+			return True
+		Left e -> do
+			warning $ prettyReqError e
+			return False

From 3adb48f46a553e8de926e2ce93ea5162dd589111 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:21:05 -0400
Subject: [PATCH 1304/8313] more S3 docs

---
 Remote/S3.hs                         | 11 ++++++++++-
 doc/install.mdwn                     |  1 +
 doc/walkthrough/using_Amazon_S3.mdwn |  6 ++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 3265ced78c..b3a9106396 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -71,7 +71,7 @@ genRemote r u c cst = this
 			name = Git.repoDescribe r,
 	 		storeKey = s3Store this,
 			retrieveKeyFile = s3Retrieve this,
-			removeKey = error "TODO removekey",
+			removeKey = s3Remove this,
 			hasKey = s3CheckPresent this,
 			hasKeyCheap = False,
 			config = c
@@ -178,3 +178,12 @@ s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
 		Left e -> do
 			warning $ prettyReqError e
 			return False
+
+s3Remove :: Remote Annex -> Key -> Annex Bool
+s3Remove r k = s3Action r $ \(conn, bucket) -> do
+	res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty
+	case res of
+		Right _ -> return True
+		Left e -> do
+			warning $ prettyReqError e
+			return False
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 0501663231..7b2c536c9d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -13,6 +13,7 @@ To build and use git-annex, you will need:
 * MissingH: 
 * pcre-light: 
 * utf8-string: 
+* hS3: 
 * `uuid`: 
   (or uuidgen from util-linux)
 * `xargs`: 
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index e0d2296620..b8eb7da530 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -10,12 +10,14 @@ First, export your S3 credentials:
 	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
 	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
 
-Next, create the S3 remote.
+Next, create the S3 remote, and describe it.
 
 	# git annex initremote mys3 type=S3 encryption=none
 	initremote mys3 (checking bucket) (creating bucket in US) ok
+	# git annex describe mys3 "at Amazon's US datacenter"
+	describe mys3 ok
 
-The configuration for the S3 remote is stored in git. So to make a different
+The configuration for the S3 remote is stored in git. So to make another
 repository use the same S3 remote is easy:
 
 	# cd /media/usb/annex

From 43bdebbc2d2c22623a0114dca51b8339bf7231c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:24:26 -0400
Subject: [PATCH 1305/8313] update

---
 debian/changelog | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4c0c9f7411..faf2833a84 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,7 @@
 git-annex (0.20110329) UNRELEASED; urgency=low
 
   * Amazon S3 is now supported as a special type of remote.
-    Warning: Encrypting data before sending it to S3 is not currently
-    supported.
+    Warning: Encrypting data before sending it to S3 is not yet supported.
   * fsck: Ensure that files and directories in .git/annex/objects
     have proper permissions.
 

From 8f9951369d5e85e3a1bf323760f0c873a3f21b97 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:28:37 -0400
Subject: [PATCH 1306/8313] refactor

---
 Remote.hs     | 6 ++++--
 Remote/Git.hs | 3 +--
 Remote/S3.hs  | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 9fd53a2f27..914c69abe5 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,7 +29,7 @@ module Remote (
 ) where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when, liftM)
+import Control.Monad (when, liftM, filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
@@ -42,6 +42,7 @@ import Trust
 import LocationLog
 import Locations
 import Utility
+import Config
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -68,7 +69,8 @@ genList = do
 	where
 		process m t = do
 			l <- enumerate t
-			mapM (gen m t) l
+			l' <- filterM remoteNotIgnored l
+			mapM (gen m t) l'
 		gen m t r = do
 			u <- getUUID r
 			generate t r (M.lookup u m)
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 984f9c88fe..d0dedd4fde 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -14,7 +14,6 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
-import Control.Monad (filterM)
 
 import RemoteClass
 import Types
@@ -41,7 +40,7 @@ remote = RemoteType {
 list :: Annex [Git.Repo]
 list = do
 	g <- Annex.gitRepo
-	filterM remoteNotIgnored $ Git.remotes g
+	return $ Git.remotes g
 
 gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
 gen repo _ = do
diff --git a/Remote/S3.hs b/Remote/S3.hs
index b3a9106396..260c1eee8e 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -16,7 +16,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
 import Data.String.Utils
-import Control.Monad (filterM, when)
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Environment
 
@@ -41,7 +41,7 @@ remote = RemoteType {
 s3List :: Annex [Git.Repo]
 s3List = do
 	g <- Annex.gitRepo
-	filterM remoteNotIgnored $ findS3Remotes g
+	return $ findS3Remotes g
 
 {- S3 remotes have a remote..annex-s3 config setting.
  - Git.Repo does not normally generate remotes for things that

From e7c1332fa2f447f46781641821d1ff2c71716657 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:30:41 -0400
Subject: [PATCH 1307/8313] updat

---
 doc/todo/S3.mdwn | 28 +++++-----------------------
 1 file changed, 5 insertions(+), 23 deletions(-)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index 356b2af2eb..dd0174cc55 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -1,33 +1,15 @@
-[[done]]
-
 Support Amazon S3 as a file storage backend.
 
 There's a haskell library that looks good. Not yet in Debian.
 
-Multiple ways of using S3 are possible. Current plan is to 
-have a special type of git remote (though git won't know how to use it;
-only git-annex will) that uses a S3 bucket.
+Multiple ways of using S3 are possible. Currently implemented as 
+a special type of git remote.
 
-Something like:
+Before this can be close, I need to fix:
 
-	[remote "s3"]
-		annex-s3bucket = mybucket
-		annex-s3datacenter = Europe
-		annex-uuid = 1a586cf6-45e9-11e0-ba9c-3b0a3397aec2
-		annex-cost = 500
+## encryption
 
-The UUID will be stored as a special file in the S3 bucket.
-
-Using a different type of remote like this will allow S3 to be used
-anywhere a regular remote would be used. `git annex get` will transparently
-download a file from S3 if S3 has it and is the cheapest remote.
-
-	git annex copy --to s3
-	git annex move --from s3
-	git annex drop --from s3 # not currently allowed, will need adding
-
-Each s3 remote will count as one copy for numcopies handling, just like
-any other remote.
+TODO
 
 ## unused checking
 

From def137b0cc0c86d9cd976c11b59f7ba0669c0735 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 Mar 2011 18:41:57 -0400
Subject: [PATCH 1308/8313] use HMAC

---
 doc/special_remotes/Amazon_S3.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index c8e44b609c..ad73e0a10d 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -37,10 +37,10 @@ only be used for public data.
 ** Encryption is not yet supported. **
 
 When encryption is enabled, all files stored in the bucket are
-encrypted with gpg. Additionally, the filenames themselves are hashed
-to obfuscate them. The size of the encrypted files, and access patterns of
-the data, should be the only clues to what type of data you are storing in
-S3.
+encrypted with gpg. Additionally, the filenames themselves are encrypted
+(using HMAC). The size of the encrypted files, and
+access patterns of the data, should be the only clues to what type of
+data you are storing in S3.
 
 [[!template id=note text="""
 This scheme was originally developed by Lars Wirzenius at al

From 9c96d86502c521cf78228f816e33ac456fb2ee59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 01:32:05 -0400
Subject: [PATCH 1309/8313] nasty hack to build when hS3 is not available

So, it would be nicer to just use Cabal and take advantage
of its conditional compilation support. But, Cabal seems to
lack good support for a package with an internal library that is used by
multiple executables. It wants to build everything twice or more.
That's too slow for me.

Anyway, fairly soon, I expect to upgrade hS3 to a requirment, and I
can just revert this.
---
 Makefile                    | 13 ++++++++++---
 Remote/{S3.hs => S3real.hs} |  0
 Remote/S3stub.hs            | 13 +++++++++++++
 debian/changelog            |  2 ++
 doc/install.mdwn            |  8 ++++----
 5 files changed, 29 insertions(+), 7 deletions(-)
 rename Remote/{S3.hs => S3real.hs} (100%)
 create mode 100644 Remote/S3stub.hs

diff --git a/Makefile b/Makefile
index 8e16645034..6a1531a3c8 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,14 @@ SysConfig.hs: configure.hs TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-$(bins): SysConfig.hs Touch.hs StatFS.hs
+Remote/S3.o:
+	@ln -sf S3real.hs Remote/S3.hs
+	@if ! $(GHCMAKE) Remote/S3.hs; then \
+		ln -sf S3stub.hs Remote/S3.hs; \
+		echo "** building without S3 support"; \
+	fi
+
+$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
@@ -62,8 +69,8 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure \
-		StatFS.hs Touch.hs SysConfig.hs *.tix .hpc
+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
+		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
 	rm -rf doc/.ikiwiki html
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
diff --git a/Remote/S3.hs b/Remote/S3real.hs
similarity index 100%
rename from Remote/S3.hs
rename to Remote/S3real.hs
diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
new file mode 100644
index 0000000000..0d6ec47de3
--- /dev/null
+++ b/Remote/S3stub.hs
@@ -0,0 +1,13 @@
+-- stub for when hS3 is not available
+module Remote.S3 (remote) where
+
+import RemoteClass
+import Types
+
+remote :: RemoteType Annex
+remote = RemoteType {
+	typename = "S3",
+	enumerate = return [],
+	generate = error "S3 not enabled",
+	setup = error "S3 not enabled"
+}
diff --git a/debian/changelog b/debian/changelog
index faf2833a84..b03bc1d1b5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (0.20110329) UNRELEASED; urgency=low
 
   * Amazon S3 is now supported as a special type of remote.
     Warning: Encrypting data before sending it to S3 is not yet supported.
+  * Note that Amazon S3 support is not built in by default on Debian yet,
+    as hS3 is not packaged.
   * fsck: Ensure that files and directories in .git/annex/objects
     have proper permissions.
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7b2c536c9d..70ab8e30b6 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -13,17 +13,17 @@ To build and use git-annex, you will need:
 * MissingH: 
 * pcre-light: 
 * utf8-string: 
-* hS3: 
+* hS3  (optional, but recommended)
 * `uuid`: 
   (or uuidgen from util-linux)
 * `xargs`: 
 * `rsync`: 
 * `curl` :  (optional, but recommended)
 * `sha1sum`:  (optional, but recommended)
-* Then just [[download]] git-annex and run: `make; make install`
+* [Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
+  but that will be skipped if it is not installed.
 
-([Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
-but that will be skipped if it is not installed.)
+Then just [[download]] git-annex and run: `make; make install`
 
 Additionally, to run the test suite (via `make test`), you will need:
 

From 5eb5b8721521ee7cde756716563fe89374d2c023 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 01:37:02 -0400
Subject: [PATCH 1310/8313] typo

---
 doc/special_remotes/Amazon_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index ad73e0a10d..b625d9b3aa 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -43,7 +43,7 @@ access patterns of the data, should be the only clues to what type of
 data you are storing in S3.
 
 [[!template id=note text="""
-This scheme was originally developed by Lars Wirzenius at al
+This scheme was originally developed by Lars Wirzenius et al
 [for Obnam](http://braawi.org/obnam/encryption/).
 """]]
 The data stored in S3 is encrypted by gpg with a symmetric cipher. The

From 320a4102d6dfff193fc501e53859b2b3edc397d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 01:45:39 -0400
Subject: [PATCH 1311/8313] update

---
 doc/special_remotes/Amazon_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn
index b625d9b3aa..e4e57b3689 100644
--- a/doc/special_remotes/Amazon_S3.mdwn
+++ b/doc/special_remotes/Amazon_S3.mdwn
@@ -3,7 +3,7 @@ or a similar service.
 
 See [[walkthrough/using_Amazon_S3]] for usage examples.
 
-## initremote parameters
+## configuration
 
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.

From ee84c75de066826c9fbd8351b456bd7c9980bab6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 10:37:27 +0000
Subject: [PATCH 1312/8313]

---
 ..._bucket_uses_the_same_key_for_encryption_and_hashing.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn
new file mode 100644
index 0000000000..0ec66652e2
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn
@@ -0,0 +1,5 @@
+While using HMAC instead of "plain" hash functions is inherently more secure, it's still a bad idea to re-use keys for different purposes.
+
+Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice?
+
+Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing.

From 8e32c3a596a8860ad71e51aafddfe8e401e8d364 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 30 Mar 2011 14:32:34 +0000
Subject: [PATCH 1313/8313] Added a comment

---
 ...1_dc5ae7af499203cfd903e866595b8fea._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment
new file mode 100644
index 0000000000..320fb5ef08
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-03-30T14:32:34Z"
+ content="""
+S3 doesn't support encryption at all, yet.
+
+It certainly makes sense to use a different portion of the encrypted secret key for HMAC than is uses as the gpg symmetric encryption key.
+
+The two keys used in HMAC would be the secret key and the key/value key for the content being stored.
+
+There is a difficult problem with encrypting filenames in S3 buckets, and that is determining when some data in the bucket is unused for dropunused. I've considered two choices:
+
+1. gpg encrypt the filenames. This would allow dropunused to recover the original filenames, and is probably more robust encryption. But it would double the number of times gpg is run when moving content in/out, and to check for unused content, gpg would have to be run once for every item in the bucket, which just feels way excessive, even though it would not be prompting for a passphrase. Still, haven't ruled this out.
+
+2. HMAC or other hash. To determine what data was unused the same hash and secret key would have to be used to hash all filenames currently used, and then that set of hashes could be interested with the set in the bucket. But then git-annex could only say \"here are some opaque hashes of content that appears unused by anything in your current git repository, but there's no way, short of downloading it and examining it to tell what it is\". (This could be improved by keeping a local mapping between filenames and S3 keys, but maintaining and committing that would bring pain of its own.)
+"""]]

From 564d013628c7ef5eab23880be6e6dabff5bc85d4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 17:01:41 +0000
Subject: [PATCH 1314/8313] Added a comment

---
 ...mment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment
new file mode 100644
index 0000000000..dec06c89ff
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-03-30T17:01:40Z"
+ content="""
+After mulling this over, I think actually encrypting the filenames is preferable.
+
+Did you consider encrypting the symmetric key with an asymmetric one? That's what TrueCrypt etc are using to allow different people access to a shared volume. This has the added benefit that you could, potentially, add new keys for data that new people should have access to while making access to old data impossible. Or keys per subdirectory, or, or, or.
+
+As an aside, could the same mechanism be extended to transparently encrypt data for a remote annex repo? A friend of mine is interested to host his data with me, but he wants to encrypt his data for obvious reasons.
+"""]]

From a47ed922e1302480d79f54f553532e85eebae872 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 13:18:46 -0400
Subject: [PATCH 1315/8313] add Remote.Directory

---
 .gitignore                         |   1 +
 Remote.hs                          |   2 +
 Remote/Directory.hs                | 110 +++++++++++++++++++++++++++++
 debian/changelog                   |   2 +
 doc/special_remotes.mdwn           |   1 +
 doc/special_remotes/directory.mdwn |  10 +++
 6 files changed, 126 insertions(+)
 create mode 100644 Remote/Directory.hs
 create mode 100644 doc/special_remotes/directory.mdwn

diff --git a/.gitignore b/.gitignore
index aa677c1335..b73167c925 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ html
 .hpc
 Touch.hs
 StatFS.hs
+Remote/S3.hs
diff --git a/Remote.hs b/Remote.hs
index 914c69abe5..0cfec3c282 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -46,11 +46,13 @@ import Config
 
 import qualified Remote.Git
 import qualified Remote.S3
+import qualified Remote.Directory
 
 remoteTypes :: [RemoteType Annex]
 remoteTypes =
 	[ Remote.Git.remote
 	, Remote.S3.remote
+	, Remote.Directory.remote
 	]
 
 {- Builds a list of all available Remotes.
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
new file mode 100644
index 0000000000..697de5ea7c
--- /dev/null
+++ b/Remote/Directory.hs
@@ -0,0 +1,110 @@
+{- A "remote" that is just a local directory.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Directory (remote) where
+
+import IO
+import Control.Exception.Extensible (IOException)
+import qualified Data.Map as M
+import Data.Maybe
+import Data.String.Utils
+import Control.Monad (when)
+import Control.Monad.State (liftIO)
+import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
+import System.FilePath
+
+import RemoteClass
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import UUID
+import Config
+import Utility
+import Locations
+import CopyFile
+
+remote :: RemoteType Annex
+remote = RemoteType {
+	typename = "directory",
+	enumerate = list,
+	generate = gen,
+	setup = dosetup
+}
+
+list :: Annex [Git.Repo]
+list = do
+	g <- Annex.gitRepo
+	return $ findDirectoryRemotes g
+
+findDirectoryRemotes :: Git.Repo -> [Git.Repo]
+findDirectoryRemotes r = map construct remotepairs
+	where
+		remotepairs = M.toList $ filterremotes $ Git.configMap r
+		filterremotes = M.filterWithKey (\k _ -> directoryremote k)
+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		directoryremote k = startswith "remote." k && endswith ".annex-directory" k
+
+gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r c = do
+	u <- getUUID r
+	cst <- remoteCost r
+	return $ genRemote r u c cst
+	where
+
+genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
+genRemote r u c cst = this
+	where
+		this = Remote {
+			uuid = u,
+			cost = cst,
+			name = Git.repoDescribe r,
+	 		storeKey = store this,
+			retrieveKeyFile = retrieve this,
+			removeKey = remove this,
+			hasKey = checkPresent this,
+			hasKeyCheap = True,
+			config = c
+		}
+
+dosetup :: UUID -> M.Map String String -> Annex (M.Map String String)
+dosetup u c = do
+	-- verify configuration is sane
+	let dir = case M.lookup "directory" c of
+		Nothing -> error "Specify directory="
+		Just d -> d
+	e <- liftIO $ doesDirectoryExist dir
+	when (not e) $ error $ "Directory does not exist: " ++ dir
+
+	g <- Annex.gitRepo
+	liftIO $ do
+		Git.run g "config" [Param (configsetting "annex-directory"), Param "true"]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
+	return c
+	where
+		remotename = fromJust (M.lookup "name" c)
+		configsetting s = "remote." ++ remotename ++ "." ++ s
+
+dirKey :: Remote Annex -> Key -> FilePath
+dirKey r k = dir  show k
+	where
+		dir = fromJust $ M.lookup "directory" $ fromJust $ config r
+
+store :: Remote Annex -> Key -> Annex Bool
+store r k = do
+	g <- Annex.gitRepo
+	liftIO $ copyFile (gitAnnexLocation g k) (dirKey r k)
+
+retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+retrieve r k f = liftIO $ copyFile (dirKey r k) f
+
+remove :: Remote Annex -> Key -> Annex Bool
+remove r k = liftIO $ catch
+	(removeFile (dirKey r k) >> return True)
+	(const $ return False)
+
+checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+checkPresent r k = liftIO $ try $ doesFileExist (dirKey r k)
diff --git a/debian/changelog b/debian/changelog
index b03bc1d1b5..0a232220f0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.20110329) UNRELEASED; urgency=low
     as hS3 is not packaged.
   * fsck: Ensure that files and directories in .git/annex/objects
     have proper permissions.
+  * Added a special type of remote called a directory remote, which
+    simply stores files in an arbitrary local directory.
 
  -- Joey Hess   Sat, 26 Mar 2011 14:36:16 -0400
 
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 651b24afa4..09b751d0f4 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -7,3 +7,4 @@ types of remotes. These can be used just like any normal remote by git-annex.
 They cannot be used by other git commands though.
 
 * [[Amazon_S3]]
+* [[directory]]
diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn
new file mode 100644
index 0000000000..42dbc5749e
--- /dev/null
+++ b/doc/special_remotes/directory.mdwn
@@ -0,0 +1,10 @@
+This special remote type stores file contents in directory on the system.
+
+One use case for this would be if you have a removable drive, that you
+cannot put a git repository on for some reason, and you want to use it
+to sneakernet files between systems. Just set up both systems to use
+the drive's mountpoint as a directory remote.
+
+Setup example:
+
+	# git annex initremote usbdrive directory=/media/usbdrive/

From 619f07ee6a0ad875f365886096b112b6c18b0606 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 14:00:54 -0400
Subject: [PATCH 1316/8313] boilerplate reduction

---
 Remote.hs           |  3 ++-
 Remote/Directory.hs | 36 ++++++-------------------
 Remote/Git.hs       | 36 +++++++++++--------------
 Remote/S3real.hs    | 66 +++++++++++++--------------------------------
 Remote/Special.hs   | 43 +++++++++++++++++++++++++++++
 RemoteClass.hs      |  6 +++--
 6 files changed, 90 insertions(+), 100 deletions(-)
 create mode 100644 Remote/Special.hs

diff --git a/Remote.hs b/Remote.hs
index 0cfec3c282..4e401ddcc0 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -75,7 +75,8 @@ genList = do
 			mapM (gen m t) l'
 		gen m t r = do
 			u <- getUUID r
-			generate t r (M.lookup u m)
+			cst <- remoteCost r
+			generate t r u cst (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 697de5ea7c..12736e0507 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -11,7 +11,6 @@ import IO
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Data.Maybe
-import Data.String.Utils
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
@@ -22,41 +21,21 @@ import Types
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
-import Config
 import Utility
 import Locations
 import CopyFile
+import Remote.Special
 
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "directory",
-	enumerate = list,
+	enumerate = findSpecialRemotes "directory",
 	generate = gen,
-	setup = dosetup
+	setup = directorySetup
 }
 
-list :: Annex [Git.Repo]
-list = do
-	g <- Annex.gitRepo
-	return $ findDirectoryRemotes g
-
-findDirectoryRemotes :: Git.Repo -> [Git.Repo]
-findDirectoryRemotes r = map construct remotepairs
-	where
-		remotepairs = M.toList $ filterremotes $ Git.configMap r
-		filterremotes = M.filterWithKey (\k _ -> directoryremote k)
-		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		directoryremote k = startswith "remote." k && endswith ".annex-directory" k
-
-gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r c = do
-	u <- getUUID r
-	cst <- remoteCost r
-	return $ genRemote r u c cst
-	where
-
-genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
-genRemote r u c cst = this
+gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u cst c = return this
 	where
 		this = Remote {
 			uuid = u,
@@ -70,8 +49,8 @@ genRemote r u c cst = this
 			config = c
 		}
 
-dosetup :: UUID -> M.Map String String -> Annex (M.Map String String)
-dosetup u c = do
+directorySetup :: UUID -> M.Map String String -> Annex (M.Map String String)
+directorySetup u c = do
 	-- verify configuration is sane
 	let dir = case M.lookup "directory" c of
 		Nothing -> error "Specify directory="
@@ -79,6 +58,7 @@ dosetup u c = do
 	e <- liftIO $ doesDirectoryExist dir
 	when (not e) $ error $ "Directory does not exist: " ++ dir
 
+	gitConfigSpecialRemote "directory" u c
 	g <- Annex.gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting "annex-directory"), Param "true"]
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d0dedd4fde..286a8c645c 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -42,33 +42,27 @@ list = do
 	g <- Annex.gitRepo
 	return $ Git.remotes g
 
-gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen repo _ = do
+gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u cst _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
 	 - the config of an URL remote is only read when there is no
 	 - cached UUID value. -}
-	let cheap = not $ Git.repoIsUrl repo
-	u <- getUUID repo
-	repo' <- case (cheap, u) of
-		(True, _) -> tryGitConfigRead repo
-		(False, "") -> tryGitConfigRead repo
-		_ -> return repo
-	genRemote repo'
-	
-genRemote :: Git.Repo -> Annex (Remote Annex)
-genRemote r = do
-	u <- getUUID r
-	c <- remoteCost r
+	let cheap = not $ Git.repoIsUrl r
+	r' <- case (cheap, u) of
+		(True, _) -> tryGitConfigRead r
+		(False, "") -> tryGitConfigRead r
+		_ -> return r
+
 	return $ Remote {
 		uuid = u,
-		cost = c,
-		name = Git.repoDescribe r,
-		storeKey = copyToRemote r,
-		retrieveKeyFile = copyFromRemote r,
-		removeKey = dropKey r,
-		hasKey = inAnnex r,
-		hasKeyCheap = not (Git.repoIsUrl r),
+		cost = cst,
+		name = Git.repoDescribe r',
+		storeKey = copyToRemote r',
+		retrieveKeyFile = copyFromRemote r',
+		removeKey = dropKey r',
+		hasKey = inAnnex r',
+		hasKeyCheap = not (Git.repoIsUrl r'),
 		config = Nothing
 	}
 
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 260c1eee8e..4380231fdc 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -15,7 +15,6 @@ import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
-import Data.String.Utils
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Environment
@@ -25,54 +24,29 @@ import Types
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
-import Config
-import Utility
 import Messages
 import Locations
+import Remote.Special
 
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "S3",
-	enumerate = s3List,
-	generate = s3Gen,
+	enumerate = findSpecialRemotes "s3",
+	generate = gen,
 	setup = s3Setup
 }
 
-s3List :: Annex [Git.Repo]
-s3List = do
-	g <- Annex.gitRepo
-	return $ findS3Remotes g
-
-{- S3 remotes have a remote..annex-s3 config setting.
- - Git.Repo does not normally generate remotes for things that
- - have no configured url, so the Git.Repo objects have to be
- - constructed as coming from an unknown location. -}
-findS3Remotes :: Git.Repo -> [Git.Repo]
-findS3Remotes r = map construct remotepairs
-	where
-		remotepairs = M.toList $ filterremotes $ Git.configMap r
-		filterremotes = M.filterWithKey (\k _ -> s3remote k)
-		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
-		s3remote k = startswith "remote." k && endswith ".annex-s3" k
-
-s3Gen :: Git.Repo -> Maybe (M.Map String String) -> Annex (Remote Annex)
-s3Gen r c = do
-	u <- getUUID r
-	cst <- remoteCost r
-	return $ genRemote r u c cst
-	where
-
-genRemote :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Int -> Remote Annex
-genRemote r u c cst = this
+gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u cst c = return this
 	where
 		this = Remote {
 			uuid = u,
 			cost = cst,
 			name = Git.repoDescribe r,
-	 		storeKey = s3Store this,
-			retrieveKeyFile = s3Retrieve this,
-			removeKey = s3Remove this,
-			hasKey = s3CheckPresent this,
+	 		storeKey = store this,
+			retrieveKeyFile = retrieve this,
+			removeKey = remove this,
+			hasKey = checkPresent this,
 			hasKeyCheap = False,
 			config = c
 		}
@@ -114,15 +88,11 @@ s3Setup u c = do
 				Right _ -> return ()
 				Left err -> error $ prettyReqError err
 
-	g <- Annex.gitRepo
-	liftIO $ do
-		Git.run g "config" [Param (configsetting "annex-s3"), Param "true"]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
+	gitConfigSpecialRemote "s3" u fullconfig
 	return fullconfig
 	where
 		remotename = fromJust (M.lookup "name" c)
 		bucket = remotename ++ "-" ++ u
-		configsetting s = "remote." ++ remotename ++ "." ++ s
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
@@ -142,8 +112,8 @@ s3Action r a = do
 bucketKey :: String -> Key -> L.ByteString -> S3Object
 bucketKey bucket k content = S3Object bucket (show k) "" [] content
 
-s3CheckPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
-s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
+checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+checkPresent r k = s3Action r $ \(conn, bucket) -> do
 	showNote ("checking " ++ name r ++ "...")
 	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty
 	case res of
@@ -151,8 +121,8 @@ s3CheckPresent r k = s3Action r $ \(conn, bucket) -> do
 		Left (AWSError _ _) -> return $ Right False
 		Left e -> return $ Left (error $ prettyReqError e)
 
-s3Store :: Remote Annex -> Key -> Annex Bool
-s3Store r k = s3Action r $ \(conn, bucket) -> do
+store :: Remote Annex -> Key -> Annex Bool
+store r k = s3Action r $ \(conn, bucket) -> do
 	g <- Annex.gitRepo
 	content <- liftIO $ L.readFile $ gitAnnexLocation g k
 	let object = setStorageClass storageclass $ bucketKey bucket k content
@@ -168,8 +138,8 @@ s3Store r k = s3Action r $ \(conn, bucket) -> do
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
 
-s3Retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
-s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
+retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+retrieve r k f = s3Action r $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right o -> do
@@ -179,8 +149,8 @@ s3Retrieve r k f = s3Action r $ \(conn, bucket) -> do
 			warning $ prettyReqError e
 			return False
 
-s3Remove :: Remote Annex -> Key -> Annex Bool
-s3Remove r k = s3Action r $ \(conn, bucket) -> do
+remove :: Remote Annex -> Key -> Annex Bool
+remove r k = s3Action r $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return True
diff --git a/Remote/Special.hs b/Remote/Special.hs
new file mode 100644
index 0000000000..d985eef6fe
--- /dev/null
+++ b/Remote/Special.hs
@@ -0,0 +1,43 @@
+{- common functions for special remotes
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Special where
+
+import qualified Data.Map as M
+import Data.Maybe
+import Data.String.Utils
+import Control.Monad.State (liftIO)
+
+import Types
+import qualified GitRepo as Git
+import qualified Annex
+import UUID
+import Utility
+
+{- Special remotes don't have a configured url, so Git.Repo does not
+ - automatically generate remotes for them. This looks for a different
+ - configuration key instead.
+ -}
+findSpecialRemotes :: String -> Annex [Git.Repo]
+findSpecialRemotes s = do
+	g <- Annex.gitRepo
+	return $ map construct $ remotepairs g
+	where
+		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
+		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		match k _ = startswith "remote." k && endswith (".annex-"++s) k
+
+{- Sets up configuration for a special remote in .git/config. -}
+gitConfigSpecialRemote :: String -> UUID -> M.Map String String -> Annex ()
+gitConfigSpecialRemote s u c = do
+	g <- Annex.gitRepo
+	liftIO $ do
+		Git.run g "config" [Param (configsetting $ "annex-"++s), Param "true"]
+		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]
+	where
+		remotename = fromJust (M.lookup "name" c)
+		configsetting v = "remote." ++ remotename ++ "." ++ v
diff --git a/RemoteClass.hs b/RemoteClass.hs
index de4c281f44..43bf403de4 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -15,6 +15,8 @@ import Data.Map as M
 import qualified GitRepo as Git
 import Key
 
+type Cost = Int
+
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
@@ -22,7 +24,7 @@ data RemoteType a = RemoteType {
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> Maybe (M.Map String String) -> a (Remote a),
+	generate :: Git.Repo -> String -> Cost -> Maybe (M.Map String String) -> a (Remote a),
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }
@@ -34,7 +36,7 @@ data Remote a = Remote {
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
-	cost :: Int,
+	cost :: Cost,
 	-- Transfers a key to the remote.
 	storeKey :: Key -> a Bool,
 	-- retrieves a key's contents to a file

From 30801372e2c0fbce198154370990a7c463bc5c4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 30 Mar 2011 18:15:18 +0000
Subject: [PATCH 1317/8313] Added a comment

---
 ...comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment
new file mode 100644
index 0000000000..c5bb26f595
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-03-30T18:15:18Z"
+ content="""
+Yes, encrypting the symmetric key with users' regular gpg keys is the plan.
+
+I don't think that encryption of content in a git annex remote makes much sense; the filenames obviously cannot be encrypted there. It's more likely that the same encryption would get used for a bup remote, or with the [[special_remotes/directory]] remote I threw in today.
+"""]]

From b7a48de30451e18d13686e3f864a712f0d83b720 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 18:20:57 +0000
Subject: [PATCH 1318/8313] Added a comment

---
 .../comment_4_bb6b814ab961818d514f6553455d2bf3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment
new file mode 100644
index 0000000000..09b7a8b1ab
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2011-03-30T18:20:56Z"
+ content="""
+Picking up the automagic encryption idea for annex remotes, this would allow you to host a branchable-esque git-annex hosting service. (Nexenta with ZFS is a cheap and reliable option until btrfs becomes stable in a year or five).
+"""]]

From 8b6ef15835087c2b266df624bb24f5e30154dddb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 14:32:08 -0400
Subject: [PATCH 1319/8313] allow directory remotes to be in different
 locations

Two machines might have access to the same directory remote on different
paths, so don't include the path in its persistent config, instead use
the git config to record it.
---
 Remote/Directory.hs | 72 ++++++++++++++++++++-------------------------
 Remote/S3real.hs    |  2 +-
 Remote/Special.hs   |  8 ++---
 3 files changed, 37 insertions(+), 45 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 12736e0507..919dcc295f 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -10,7 +10,6 @@ module Remote.Directory (remote) where
 import IO
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Data.Maybe
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
@@ -21,9 +20,9 @@ import Types
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
-import Utility
 import Locations
 import CopyFile
+import Config
 import Remote.Special
 
 remote :: RemoteType Annex
@@ -35,19 +34,19 @@ remote = RemoteType {
 }
 
 gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst c = return this
-	where
-		this = Remote {
-			uuid = u,
-			cost = cst,
-			name = Git.repoDescribe r,
-	 		storeKey = store this,
-			retrieveKeyFile = retrieve this,
-			removeKey = remove this,
-			hasKey = checkPresent this,
-			hasKeyCheap = True,
-			config = c
-		}
+gen r u cst _ = do
+	dir <- getConfig r "directory" (error "missing directory")
+	return $ Remote {
+		uuid = u,
+		cost = cst,
+		name = Git.repoDescribe r,
+ 		storeKey = store dir,
+		retrieveKeyFile = retrieve dir,
+		removeKey = remove dir,
+		hasKey = checkPresent dir,
+		hasKeyCheap = True,
+		config = Nothing
+	}
 
 directorySetup :: UUID -> M.Map String String -> Annex (M.Map String String)
 directorySetup u c = do
@@ -58,33 +57,26 @@ directorySetup u c = do
 	e <- liftIO $ doesDirectoryExist dir
 	when (not e) $ error $ "Directory does not exist: " ++ dir
 
-	gitConfigSpecialRemote "directory" u c
+	-- The directory is stored in git config, not in this remote's
+	-- persistant state, so it can vary between hosts.
+	gitConfigSpecialRemote u c "directory" dir
+	return $ M.delete "directory" c
+
+dirKey :: FilePath -> Key -> FilePath
+dirKey d k = d  show k
+
+store :: FilePath -> Key -> Annex Bool
+store d k = do
 	g <- Annex.gitRepo
-	liftIO $ do
-		Git.run g "config" [Param (configsetting "annex-directory"), Param "true"]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
-	return c
-	where
-		remotename = fromJust (M.lookup "name" c)
-		configsetting s = "remote." ++ remotename ++ "." ++ s
+	liftIO $ copyFile (gitAnnexLocation g k) (dirKey d k)
 
-dirKey :: Remote Annex -> Key -> FilePath
-dirKey r k = dir  show k
-	where
-		dir = fromJust $ M.lookup "directory" $ fromJust $ config r
+retrieve :: FilePath -> Key -> FilePath -> Annex Bool
+retrieve d k f = liftIO $ copyFile (dirKey d k) f
 
-store :: Remote Annex -> Key -> Annex Bool
-store r k = do
-	g <- Annex.gitRepo
-	liftIO $ copyFile (gitAnnexLocation g k) (dirKey r k)
-
-retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
-retrieve r k f = liftIO $ copyFile (dirKey r k) f
-
-remove :: Remote Annex -> Key -> Annex Bool
-remove r k = liftIO $ catch
-	(removeFile (dirKey r k) >> return True)
+remove :: FilePath -> Key -> Annex Bool
+remove d k = liftIO $ catch
+	(removeFile (dirKey d k) >> return True)
 	(const $ return False)
 
-checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
-checkPresent r k = liftIO $ try $ doesFileExist (dirKey r k)
+checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)
+checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 4380231fdc..0827c4fbff 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -88,7 +88,7 @@ s3Setup u c = do
 				Right _ -> return ()
 				Left err -> error $ prettyReqError err
 
-	gitConfigSpecialRemote "s3" u fullconfig
+	gitConfigSpecialRemote u fullconfig "s3" "true"
 	return fullconfig
 	where
 		remotename = fromJust (M.lookup "name" c)
diff --git a/Remote/Special.hs b/Remote/Special.hs
index d985eef6fe..b5d5a137fe 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -32,12 +32,12 @@ findSpecialRemotes s = do
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
-gitConfigSpecialRemote :: String -> UUID -> M.Map String String -> Annex ()
-gitConfigSpecialRemote s u c = do
+gitConfigSpecialRemote :: UUID -> M.Map String String -> String -> String -> Annex ()
+gitConfigSpecialRemote u c k v = do
 	g <- Annex.gitRepo
 	liftIO $ do
-		Git.run g "config" [Param (configsetting $ "annex-"++s), Param "true"]
+		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
 		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]
 	where
 		remotename = fromJust (M.lookup "name" c)
-		configsetting v = "remote." ++ remotename ++ "." ++ v
+		configsetting s = "remote." ++ remotename ++ "." ++ s

From fdd455e913964200177530df085f2a7ad7c7f8b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 14:56:31 -0400
Subject: [PATCH 1320/8313] use same directory structure as .git/annex/objects
 for directory remotes

And same file perms.
---
 Locations.hs        |  1 +
 Remote/Directory.hs | 36 +++++++++++++++++++++++++++++-------
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 3cce4c2611..8e10c36b41 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
 	logFile,
+	hashDir,
 
 	prop_idempotent_fileKey
 ) where
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 919dcc295f..cc37e496ea 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -1,4 +1,4 @@
-{- A "remote" that is just a local directory.
+{- A "remote" that is just a filesystem directory.
  -
  - Copyright 2011 Joey Hess 
  -
@@ -12,7 +12,7 @@ import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
-import System.Directory (doesDirectoryExist, doesFileExist, removeFile)
+import System.Directory hiding (copyFile)
 import System.FilePath
 
 import RemoteClass
@@ -23,6 +23,8 @@ import UUID
 import Locations
 import CopyFile
 import Config
+import Content
+import Utility
 import Remote.Special
 
 remote :: RemoteType Annex
@@ -63,20 +65,40 @@ directorySetup u c = do
 	return $ M.delete "directory" c
 
 dirKey :: FilePath -> Key -> FilePath
-dirKey d k = d  show k
+dirKey d k = d  hashDir k  f  f
+	where
+		f = keyFile k
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
 	g <- Annex.gitRepo
-	liftIO $ copyFile (gitAnnexLocation g k) (dirKey d k)
+	let src = gitAnnexLocation g k
+	liftIO $ catch (copy src) (const $ return False)
+	where
+		dest = dirKey d k
+		dir = parentDir dest
+		copy src = do
+			createDirectoryIfMissing True dir
+			allowWrite dir
+			ok <- copyFile src dest
+			when ok $ do
+				preventWrite dest
+				preventWrite dir
+			return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
 retrieve d k f = liftIO $ copyFile (dirKey d k) f
 
 remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ catch
-	(removeFile (dirKey d k) >> return True)
-	(const $ return False)
+remove d k = liftIO $ catch del (const $ return False)
+	where
+		file = dirKey d k
+		dir = parentDir file
+		del = do
+			allowWrite dir
+			removeFile file
+			removeDirectory dir
+			return True
 
 checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)
 checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)

From 8f47b38dbc97d2062c21c8b6f883187ffc9f2ef0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 18:59:19 +0000
Subject: [PATCH 1321/8313] Added a comment

---
 .../comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment
new file mode 100644
index 0000000000..49d43ffc63
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 5"
+ date="2011-03-30T18:59:19Z"
+ content="""
+This is brain-storming only so the idea might be crap, but a branch could keep encrypted filenames while master keeps the real deal. This might fit into the whole scheme just nicely or break future stuff in a dozen places, I am not really sure yet. But at least I can't forget the idea, now.
+"""]]

From 07070e6dd938a6d389fa7334e415472b797c61b2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 30 Mar 2011 19:02:20 +0000
Subject: [PATCH 1322/8313] Added a comment

---
 .../comment_6_63fb74da342751fc35e1850409c506f6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment

diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment
new file mode 100644
index 0000000000..d994ca77f3
--- /dev/null
+++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 6"
+ date="2011-03-30T19:02:20Z"
+ content="""
+OTOH, if encryption makes a bup backend more likely disregard the idea above ;)
+"""]]

From 0c73c08c1c0929f0ba53dcfb6d5d32a73a5f28d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 15:15:46 -0400
Subject: [PATCH 1323/8313] cost bugfixes

---
 Config.hs           | 13 ++++++++-----
 Remote.hs           |  3 +--
 Remote/Directory.hs |  5 +++--
 Remote/Git.hs       |  9 +++++++--
 Remote/S3real.hs    | 26 +++++++++++++++-----------
 RemoteClass.hs      |  8 +++-----
 6 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/Config.hs b/Config.hs
index c821364ced..17a1fa9856 100644
--- a/Config.hs
+++ b/Config.hs
@@ -42,14 +42,17 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
  - The default cost is 100 for local repositories, and 200 for remote
  - repositories; it can also be configured by remote..annex-cost
  -}
-remoteCost :: Git.Repo -> Annex Int
-remoteCost r = do
+remoteCost :: Git.Repo -> Int -> Annex Int
+remoteCost r def = do
 	c <- getConfig r "cost" ""
 	if not $ null c
 		then return $ read c
-		else if not $ Git.repoIsUrl r
-			then return 100
-			else return 200
+		else return def
+
+cheapRemoteCost :: Int
+cheapRemoteCost = 100
+expensiveRemoteCost :: Int
+expensiveRemoteCost = 200
 
 {- Checks if a repo should be ignored, based either on annex-ignore
  - setting, or on command-line options. Allows command-line to override
diff --git a/Remote.hs b/Remote.hs
index 4e401ddcc0..26097da747 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -75,8 +75,7 @@ genList = do
 			mapM (gen m t) l'
 		gen m t r = do
 			u <- getUUID r
-			cst <- remoteCost r
-			generate t r u cst (M.lookup u m)
+			generate t r u (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index cc37e496ea..f97449eaa7 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -35,9 +35,10 @@ remote = RemoteType {
 	setup = directorySetup
 }
 
-gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst _ = do
+gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u _ = do
 	dir <- getConfig r "directory" (error "missing directory")
+	cst <- remoteCost r cheapRemoteCost
 	return $ Remote {
 		uuid = u,
 		cost = cst,
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 286a8c645c..c1423bef7a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -42,8 +42,8 @@ list = do
 	g <- Annex.gitRepo
 	return $ Git.remotes g
 
-gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst _ = do
+gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
 	 - the config of an URL remote is only read when there is no
@@ -54,6 +54,11 @@ gen r u cst _ = do
 		(False, "") -> tryGitConfigRead r
 		_ -> return r
 
+	let defcst = if not $ Git.repoIsUrl r
+		then cheapRemoteCost
+		else expensiveRemoteCost
+	cst <- remoteCost r' defcst
+
 	return $ Remote {
 		uuid = u,
 		cost = cst,
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 0827c4fbff..d7a6d507b1 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -26,6 +26,7 @@ import qualified Annex
 import UUID
 import Messages
 import Locations
+import Config
 import Remote.Special
 
 remote :: RemoteType Annex
@@ -36,25 +37,28 @@ remote = RemoteType {
 	setup = s3Setup
 }
 
-gen :: Git.Repo -> UUID -> Cost -> Maybe (M.Map String String) -> Annex (Remote Annex)
-gen r u cst c = return this
+gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex)
+gen r u c = do
+	cst <- remoteCost r expensiveRemoteCost
+	return $ this cst
 	where
-		this = Remote {
+		this cst = Remote {
 			uuid = u,
 			cost = cst,
 			name = Git.repoDescribe r,
-	 		storeKey = store this,
-			retrieveKeyFile = retrieve this,
-			removeKey = remove this,
-			hasKey = checkPresent this,
+	 		storeKey = store (this cst),
+			retrieveKeyFile = retrieve (this cst),
+			removeKey = remove (this cst),
+			hasKey = checkPresent (this cst),
 			hasKeyCheap = False,
 			config = c
 		}
 
-s3Connection :: M.Map String String -> IO AWSConnection
+s3Connection :: M.Map String String -> Annex AWSConnection
 s3Connection c = do
 	ak <- getEnvKey "AWS_ACCESS_KEY_ID"
 	sk <- getEnvKey "AWS_SECRET_ACCESS_KEY"
+	when (null ak || null sk) $ warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3"
 	return $ AWSConnection host port ak sk
 	where
 		host = fromJust $ (M.lookup "host" c)
@@ -62,7 +66,7 @@ s3Connection c = do
 			case reads s of
 			[(p, _)] -> p
 			_ -> error $ "bad S3 port value: " ++ s
-		getEnvKey s = catch (getEnv s) (error $ "Set " ++ s)
+		getEnvKey s = liftIO $ catch (getEnv s) (const $ return "")
 
 s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String)
 s3Setup u c = do
@@ -75,7 +79,7 @@ s3Setup u c = do
 
 	-- check bucket location to see if the bucket exists, and create it
 	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
-	conn <- liftIO $ s3Connection fullconfig
+	conn <- s3Connection fullconfig
 	showNote "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
@@ -105,7 +109,7 @@ s3Action :: Remote Annex -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r a = do
 	when (config r == Nothing) $
 		error $ "Missing configuration for special remote " ++ name r
-	conn <- liftIO $ s3Connection (fromJust $ config r)
+	conn <- s3Connection (fromJust $ config r)
 	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
 	a (conn, bucket)
 
diff --git a/RemoteClass.hs b/RemoteClass.hs
index 43bf403de4..8055c16b06 100644
--- a/RemoteClass.hs
+++ b/RemoteClass.hs
@@ -15,8 +15,6 @@ import Data.Map as M
 import qualified GitRepo as Git
 import Key
 
-type Cost = Int
-
 {- There are different types of remotes. -}
 data RemoteType a = RemoteType {
 	-- human visible type name
@@ -24,7 +22,7 @@ data RemoteType a = RemoteType {
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> String -> Cost -> Maybe (M.Map String String) -> a (Remote a),
+	generate :: Git.Repo -> String -> Maybe (M.Map String String) -> a (Remote a),
 	-- initializes or changes a remote
 	setup :: String -> M.Map String String -> a (M.Map String String)
 }
@@ -36,7 +34,7 @@ data Remote a = Remote {
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
-	cost :: Cost,
+	cost :: Int,
 	-- Transfers a key to the remote.
 	storeKey :: Key -> a Bool,
 	-- retrieves a key's contents to a file
@@ -54,7 +52,7 @@ data Remote a = Remote {
 }
 
 instance Show (Remote a) where
-	show remote = "Remote { uuid =\"" ++ uuid remote ++ "\" }"
+	show remote = "Remote { name =\"" ++ name remote ++ "\" }"
 
 -- two remotes are the same if they have the same uuid
 instance Eq (Remote a) where

From 2c7ceceba64a75deb69033199acff8ccbcb49bdf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 15:25:59 -0400
Subject: [PATCH 1324/8313] improve robustness when S3 access tokens are is not
 configured

---
 Remote/S3real.hs | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index d7a6d507b1..bb82d54e0e 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -54,12 +54,22 @@ gen r u c = do
 			config = c
 		}
 
-s3Connection :: M.Map String String -> Annex AWSConnection
+s3ConnectionRequired :: M.Map String String -> Annex AWSConnection
+s3ConnectionRequired c = do
+	conn <- s3Connection c
+	case conn of
+		Nothing -> error "Cannot connect to S3"
+		Just conn' -> return conn'
+
+s3Connection :: M.Map String String -> Annex (Maybe AWSConnection)
 s3Connection c = do
 	ak <- getEnvKey "AWS_ACCESS_KEY_ID"
 	sk <- getEnvKey "AWS_SECRET_ACCESS_KEY"
-	when (null ak || null sk) $ warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3"
-	return $ AWSConnection host port ak sk
+	if (null ak || null sk)
+		then do
+			warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3"
+			return Nothing
+		else return $ Just $ AWSConnection host port ak sk
 	where
 		host = fromJust $ (M.lookup "host" c)
 		port = let s = fromJust $ (M.lookup "port" c) in
@@ -79,7 +89,8 @@ s3Setup u c = do
 
 	-- check bucket location to see if the bucket exists, and create it
 	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
-	conn <- s3Connection fullconfig
+	conn <- s3ConnectionRequired fullconfig
+	
 	showNote "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
@@ -105,28 +116,32 @@ s3Setup u c = do
 			, ("bucket", bucket)
 			]
 
-s3Action :: Remote Annex -> ((AWSConnection, String) -> Annex a) -> Annex a
-s3Action r a = do
+s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
+s3Action r noconn action = do
 	when (config r == Nothing) $
 		error $ "Missing configuration for special remote " ++ name r
+	let bucket = M.lookup "bucket" $ fromJust $ config r
 	conn <- s3Connection (fromJust $ config r)
-	let bucket = fromJust $ M.lookup "bucket" $ fromJust $ config r
-	a (conn, bucket)
+	case (bucket, conn) of
+		(Just b, Just c) -> action (c, b)
+		_ -> return noconn
 
 bucketKey :: String -> Key -> L.ByteString -> S3Object
 bucketKey bucket k content = S3Object bucket (show k) "" [] content
 
 checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
-checkPresent r k = s3Action r $ \(conn, bucket) -> do
+checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showNote ("checking " ++ name r ++ "...")
 	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return $ Right True
 		Left (AWSError _ _) -> return $ Right False
 		Left e -> return $ Left (error $ prettyReqError e)
+	where
+		noconn = Left $ error "S3 not configured"
 
 store :: Remote Annex -> Key -> Annex Bool
-store r k = s3Action r $ \(conn, bucket) -> do
+store r k = s3Action r False $ \(conn, bucket) -> do
 	g <- Annex.gitRepo
 	content <- liftIO $ L.readFile $ gitAnnexLocation g k
 	let object = setStorageClass storageclass $ bucketKey bucket k content
@@ -143,7 +158,7 @@ store r k = s3Action r $ \(conn, bucket) -> do
 				_ -> STANDARD
 
 retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
-retrieve r k f = s3Action r $ \(conn, bucket) -> do
+retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right o -> do
@@ -154,7 +169,7 @@ retrieve r k f = s3Action r $ \(conn, bucket) -> do
 			return False
 
 remove :: Remote Annex -> Key -> Annex Bool
-remove r k = s3Action r $ \(conn, bucket) -> do
+remove r k = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty
 	case res of
 		Right _ -> return True

From 56491a7e134e6c67b08fdc0200835ecc5b95b074 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Mar 2011 15:32:34 -0400
Subject: [PATCH 1325/8313] tweak

---
 doc/special_remotes/directory.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn
index 42dbc5749e..daa0b74128 100644
--- a/doc/special_remotes/directory.mdwn
+++ b/doc/special_remotes/directory.mdwn
@@ -1,4 +1,4 @@
-This special remote type stores file contents in directory on the system.
+This special remote type stores file contents in directory.
 
 One use case for this would be if you have a removable drive, that you
 cannot put a git repository on for some reason, and you want to use it

From 95f5247ee6d412eaa9b20fb2fa383142fb7f7f69 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 18:02:42 +0000
Subject: [PATCH 1326/8313] Added a comment

---
 ...ent_6_e55117cb628dc532e468519252571474._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment
new file mode 100644
index 0000000000..aae020972c
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-03-31T18:02:42Z"
+ content="""
+Alright, I have created a case-insensative HFS+ filesystem here on my linux laptop. 
+
+I have not been able to trick git into staging the same file with 2 different capitalizations yet.
+
+It might be helpful if you can send me a copy of a git repository where 'git add -i' shows the same file staged with two capitalizations. Leaving out .git/annex of course. (joey@kitenet.net; a tarball would probably work)
+
+It seems that `git add` only started properly working on case insensative filesystems quite recently. The commit in question is 5e738ae820ec53c45895b029baa3a1f63e654b1b, \"Support case folding for git add when core.ignorecase=true\", which was first released in git 1.7.4, January 30, 2011. If you don't yet have that version, that could explain the problem entirely. In about half an hour (dialup!) I will have downloaded an older git and will see if I can reproduce the problem with it.
+"""]]

From 6e835afc4e624203ad2d8e4e43c6dd8fee528bd8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:03:28 -0400
Subject: [PATCH 1327/8313] update

---
 .../sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
new file mode 100644
index 0000000000..cc4488371c
--- /dev/null
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -0,0 +1,15 @@
+Christophe-Marie Duquesne has just announced 
+[Sharebox|(https://github.com/chmduquesne/sharebox), a FUSE filesystem
+relying on git-annex:
+
+

+What are your goals?
+Seamless synchronization "à la dropbox".
+Ability to use with big binary files such as mp3/movies.
+Entirely decentralized.
+Don't use unnecessary space
+Keep it simple: avoid special VCS commands and keep a filesystem
+interface as much as possible.
+

+
+While still alpha, this is promising. --[[Joey]]

From ea91f32651e3a6ac7e1a4bc94fa6cb408516eb93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:07:28 -0400
Subject: [PATCH 1328/8313] format

---
 .../sharebox_a_FUSE_filesystem_for_git-annex.mdwn  | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index cc4488371c..63d71a3632 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -1,14 +1,16 @@
+[[!meta title="sharebox: a FUSE filesystem for git-annex"]]
+
 Christophe-Marie Duquesne has just announced 
 [Sharebox|(https://github.com/chmduquesne/sharebox), a FUSE filesystem
 relying on git-annex:
 
 

-What are your goals?
-Seamless synchronization "à la dropbox".
-Ability to use with big binary files such as mp3/movies.
-Entirely decentralized.
-Don't use unnecessary space
-Keep it simple: avoid special VCS commands and keep a filesystem
+What are your goals?  
+Seamless synchronization "à la dropbox".  
+Ability to use with big binary files such as mp3/movies.  
+Entirely decentralized.  
+Don't use unnecessary space  
+Keep it simple: avoid special VCS commands and keep a filesystem  
 interface as much as possible.
 

 

From 29738265f238e0a8782db0d6690aa98d9be1368a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:08:20 -0400
Subject: [PATCH 1329/8313] link

---
 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index 63d71a3632..f8d1a0ad7c 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -1,7 +1,7 @@
 [[!meta title="sharebox: a FUSE filesystem for git-annex"]]
 
 Christophe-Marie Duquesne has just announced 
-[Sharebox|(https://github.com/chmduquesne/sharebox), a FUSE filesystem
+[Sharebox](https://github.com/chmduquesne/sharebox), a FUSE filesystem
 relying on git-annex:
 
 


From 48c102f5225ea35fb12b47830bb4bfa404ca1110 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:10:08 -0400
Subject: [PATCH 1330/8313] format

---
 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index f8d1a0ad7c..7386841b2b 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -5,6 +5,7 @@ Christophe-Marie Duquesne has just announced
 relying on git-annex:
 
 

+
 What are your goals?  
 Seamless synchronization "à la dropbox".  
 Ability to use with big binary files such as mp3/movies.  
@@ -12,6 +13,7 @@ Entirely decentralized.
 Don't use unnecessary space  
 Keep it simple: avoid special VCS commands and keep a filesystem  
 interface as much as possible.
+

 

 
 While still alpha, this is promising. --[[Joey]]

From 4e8fc2c5b91065e70605b408aba96fec779b18ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 14:12:07 -0400
Subject: [PATCH 1331/8313] show more news

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 4f967b71f5..e84c56b449 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -15,7 +15,7 @@ To get a feel for it, see the [[walkthrough]].
 
 [[News]]:
 
-[[!inline pages="news/* and !*/discussion" archive=yes show=3 feeds=no]]
+[[!inline pages="news/* and !*/discussion" archive=yes show=6 feeds=no]]
 
 
 
Date: Thu, 31 Mar 2011 14:12:48 -0400
Subject: [PATCH 1332/8313] more news yet

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index e84c56b449..5fc642ecd4 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -15,7 +15,7 @@ To get a feel for it, see the [[walkthrough]].
 
 [[News]]:
 
-[[!inline pages="news/* and !*/discussion" archive=yes show=6 feeds=no]]
+[[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]]
 
 
 
Date: Thu, 31 Mar 2011 19:08:01 +0000
Subject: [PATCH 1333/8313] Added a comment

---
 ..._0f4f471102e394ebb01da40e4d0fd9f6._comment | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment
new file mode 100644
index 0000000000..c3aee6c579
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment
@@ -0,0 +1,68 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-03-31T19:08:01Z"
+ content="""
+git 1.7.4 does not make things better. With it, if I add first \"X/foo\" and then \"x/bar\", it commits \"X/bar\".
+
+That will *certianly* cause problems when interoperating with a repo clone on a case-sensative filesystem, since
+git-annex there will not see the location log that git committed to the wrong case directory.
+
+It's possible there is some interoperability problem when pulling from linux like you did, onto HFS+, too. I am not quite sure. Ah, I did find one.. if I clone the repo with \"X/foo\" in it to a case-sensative filesystem, and add a \"x/foo\" there,
+and pull that commit back to HFS+, git says:
+
+
+ * branch            master     -> FETCH_HEAD
+Updating 8754149..e3d4640
+Fast-forward
+ x/foo |    1 +
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 100644 x/foo
+joey@gnu:/mnt/r4>ls
+X/
+joey@gnu:/mnt/r4>git st
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory
+
+#	modified:   X/foo
+

+
+Aha -- that lets me reproduce your problem with the same file being staged twice with different capitalizations, too:
+
+
+joey@gnu:/mnt/r4>echo haaai >| x/foo
+joey@gnu:/mnt/r4>git st
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   X/bar
+#	modified:   X/foo
+#	modified:   x/foo
+#
+joey@gnu:/mnt/r4>git commit -a
+fatal: Will not add file alias 'X/Bar' ('x/Bar' already exists in index)
+

+
+And modified files that git refuses to commit, which entirely explains [[git-annex_has_issues_with_git_when_staging__47__commiting_logs]].
+
+
+joey@gnu:/mnt/r4>git add X/foo
+joey@gnu:/mnt/r4>git commit X/foo
+# On branch master
+# Changes not staged for commit:
+#   (use \"git add ...\" to update what will be committed)
+#   (use \"git checkout -- ...\" to discard changes in working directory)
+#
+#	modified:   X/bar
+#	modified:   X/foo
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+

+
+I think git is frankly, buggy. It seems I will need to work around this by stopping using mixed case hashing for location logs.
+"""]]

From d512a86da9170d8c95e3aa0c6c736a77c8a32ea5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 19:28:02 +0000
Subject: [PATCH 1334/8313] Added a comment

---
 ...mment_8_68e2d6ccdb9622b879e4bc7005804623._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment
new file mode 100644
index 0000000000..05fe4658df
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-03-31T19:28:02Z"
+ content="""
+I've posted about this on the git mailing list. It's possible that these bugs, which can be shown to affect things other than just git-annex, will be fixed in git.
+
+I will wait a while to see. But am considering making git-annex use all-lowercase hash dirs for the log files. Maybe it could first look for .git-annex/aaaa/bbbb/foo.log, but also look for, read, and merge in any info from 
+.git-annex/Aa/Bb/foo.log. And always write to the new style filenames. This would avoid confusing git with changes to
+mixed-case files, and avoid another massive transition.
+"""]]

From ad2b8759dfa0fa599685b7cb26812872afe9d3a0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 19:30:30 +0000
Subject: [PATCH 1335/8313]

---
 ...has_issues_with_git_when_staging__47__commiting_logs.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
index 774ca6a16c..ed629c4240 100644
--- a/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
+++ b/doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn
@@ -27,3 +27,8 @@ For now it's just a bit of extra work for me when it does occur but it does not
 >>>>> what git is doing. Sounds like the files could just not be `git
 >>>>> added` yet, but I get the impression from other things that you say
 >>>>> that it's not so simple. --[[Joey]] 
+
+This turns out to be a bug in git, and I have posted a bug report on the mailing list.
+The git-annex behavior that causes this situation is being handled as
+another bug, [[git-annex directory hashing problems on osx]].
+So, closing this bug report. [[done]] --[[Joey]]

From a7680717f6600469c6920e22cc3dffac06993b1b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 31 Mar 2011 19:32:25 +0000
Subject: [PATCH 1336/8313] Added a comment

---
 .../comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
new file mode 100644
index 0000000000..e46f2388f6
--- /dev/null
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-03-31T19:32:25Z"
+ content="""
+One option would be to use the new [[sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`
+"""]]

From a76d1cda912ebdc31211ef92e1ddfef4b4295e41 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 15:40:08 -0400
Subject: [PATCH 1337/8313] update

---
 .../comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment   | 2 +-
 doc/bugs/check_for_curl_in_configure.hs.mdwn              | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
index e46f2388f6..719451976b 100644
--- a/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
+++ b/doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment
@@ -4,5 +4,5 @@
  subject="comment 2"
  date="2011-03-31T19:32:25Z"
  content="""
-One option would be to use the new [[sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`
+One option would be to use the new [[news/sharebox_a_FUSE_filesystem_for_git-annex]], which would hide the immutable file details from Calibre, and proxy any changes it made through to git-annex as a series of `git annex unlock; modify; git-annex lock`
 """]]
diff --git a/doc/bugs/check_for_curl_in_configure.hs.mdwn b/doc/bugs/check_for_curl_in_configure.hs.mdwn
index 2a5227491a..a880392bf7 100644
--- a/doc/bugs/check_for_curl_in_configure.hs.mdwn
+++ b/doc/bugs/check_for_curl_in_configure.hs.mdwn
@@ -53,6 +53,14 @@ index 772ba54..1a563e0 100644
 >>> A S3 backend that could upload files to S3 in addition to downloading
 >>> them, for example, would be handy. --[[Joey]]
 
+>>>> So, rather than use backends to do this, it instead made more sense
+>>>> to make them [[special_remotes]]. The URL backend remains a bit
+>>>> of a special case, and a bittorrent backend that downloaded a file
+>>>> from a bittorrent url would still be a good use of backend, but for
+>>>> storing files in external data stores like S3, making it a remote
+>>>> makes better sense. I think I can close this bug now, [[done]]
+>>>> --[[Joey]] 
+
 also in Backend/URL.hs is it worth making a minor change to the way curl is called (I'm not sure if the following is correct or not)
 
 > It's correct, typewise, but I don't see any real reason to bother

From 70572b24813a38761c480dfa3de8f96466747d79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 16:00:33 -0400
Subject: [PATCH 1338/8313] add feed aggregator

---
 doc/feeds.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/feeds.mdwn

diff --git a/doc/feeds.mdwn b/doc/feeds.mdwn
new file mode 100644
index 0000000000..25a3190497
--- /dev/null
+++ b/doc/feeds.mdwn
@@ -0,0 +1,3 @@
+Aggregating git-annex mentions from elsewhere on the net..
+
+* [[!aggregate expirecount=25 name="identica" feedurl="http://identi.ca/api/statusnet/tags/timeline/gitannex.rss" url="http://identi.ca/tag/gitannex"]]

From 779de890c2c09c9bee0589db37d5fd79c23f7aea Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 Mar 2011 16:02:36 -0400
Subject: [PATCH 1339/8313] add feeds to sidebar

---
 doc/index.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 5fc642ecd4..60477d0db1 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -22,6 +22,10 @@ To get a feel for it, see the [[walkthrough]].
 alt="Flattr this" title="Flattr this" />
 """]]
 
+[[Feeds]]:
+
+[[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]]
+
[[!inline feeds=no template=bare pages=use_case/bob]]
From 6d4506ab775247a15344d5f6bedec38fa9a0a8c1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:05:51 -0400 Subject: [PATCH 1340/8313] layout --- doc/index.mdwn | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 60477d0db1..a7813f99fb 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -12,19 +12,16 @@ To get a feel for it, see the [[walkthrough]]. * [[forum]] * [[comments]] * [[contact]] +* Flattr this [[News]]: [[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]] - - -"""]] - [[Feeds]]: [[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] +"""]]
[[!inline feeds=no template=bare pages=use_case/bob]]
From 1e0fbab24e96e3e92845e6825f836d883cca2ae0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:08:25 -0400 Subject: [PATCH 1341/8313] smaller --- doc/index.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index a7813f99fb..4f2a6d0123 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -16,11 +16,15 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: + [[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]] + [[Feeds]]: + [[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] + """]]
From d71eb6bec6e4c910acc99de0f029f7e8856d9988 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:09:12 -0400 Subject: [PATCH 1342/8313] fix --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 4f2a6d0123..63c8e41661 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -22,7 +22,7 @@ To get a feel for it, see the [[walkthrough]]. [[Feeds]]: - + [[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] """]] From 80b52afc728c05fb97ae29e1d05819e51f9fb612 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:10:17 -0400 Subject: [PATCH 1343/8313] layout --- doc/index.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index 63c8e41661..96b7f0cf0f 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -17,7 +17,7 @@ To get a feel for it, see the [[walkthrough]]. [[News]]: -[[!inline pages="news/* and !*/discussion" archive=yes show=7 feeds=no]] +[[!inline pages="news/* and !*/discussion" archive=yes show=3 feeds=no]] [[Feeds]]: @@ -48,7 +48,7 @@ files with git. * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. ----- +
git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and hosted by [Branchable](http://branchable.com/). From cc17e6ced2e9ae19da363bef49e3db580af13b5e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:11:00 -0400 Subject: [PATCH 1344/8313] layout --- doc/index.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index 96b7f0cf0f..25456b2b55 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -50,5 +50,7 @@ files with git.
+---- + git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and hosted by [Branchable](http://branchable.com/). From 2fb7eb1e6272b17b1fb2c1e41e66ead612dbee9a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 31 Mar 2011 20:12:09 +0000 Subject: [PATCH 1345/8313] rename bugs/softlink_atime.mdwn to bugs/softlink_mtime.mdwn --- doc/bugs/{softlink_atime.mdwn => softlink_mtime.mdwn} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/bugs/{softlink_atime.mdwn => softlink_mtime.mdwn} (100%) diff --git a/doc/bugs/softlink_atime.mdwn b/doc/bugs/softlink_mtime.mdwn similarity index 100% rename from doc/bugs/softlink_atime.mdwn rename to doc/bugs/softlink_mtime.mdwn From 8e2f5ae50418a54a7e1c79c7a8bdcd7022a839e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:12:26 -0400 Subject: [PATCH 1346/8313] remove author (always the same..) --- doc/templates/microblog.tmpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/templates/microblog.tmpl diff --git a/doc/templates/microblog.tmpl b/doc/templates/microblog.tmpl new file mode 100644 index 0000000000..d42e1fae2a --- /dev/null +++ b/doc/templates/microblog.tmpl @@ -0,0 +1,12 @@ +
+ +
+ +
+ +
+ +— + +
+
From cbabee36e1e794096ad0693d6f65467fc7936894 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 31 Mar 2011 16:13:24 -0400 Subject: [PATCH 1347/8313] layout --- doc/index.mdwn | 2 +- doc/templates/microblog.tmpl | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 doc/templates/microblog.tmpl diff --git a/doc/index.mdwn b/doc/index.mdwn index 25456b2b55..cb2f485c92 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -23,7 +23,7 @@ To get a feel for it, see the [[walkthrough]]. [[Feeds]]: -[[!inline pages="internal(feeds/*)" template="microblog" show=5 feeds=no]] +[[!inline pages="internal(feeds/*)" archive=yes show=5 feeds=no]] """]] diff --git a/doc/templates/microblog.tmpl b/doc/templates/microblog.tmpl deleted file mode 100644 index d42e1fae2a..0000000000 --- a/doc/templates/microblog.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -
- -
- -
- -
- -— - -
-
From ff97607879817ce441637751f21462ac769fe099 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 31 Mar 2011 20:25:07 +0000 Subject: [PATCH 1348/8313] --- doc/bugs/softlink_mtime.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/softlink_mtime.mdwn b/doc/bugs/softlink_mtime.mdwn index fe09ade38b..1427fc7147 100644 --- a/doc/bugs/softlink_mtime.mdwn +++ b/doc/bugs/softlink_mtime.mdwn @@ -49,3 +49,6 @@ Optionally, editing the meta-data should change the times in all annexes. >>>>>>> -- RichiH >>>>>>>> After getting to actually play with this from different machines with a bare git as central instance for several distributed repos, the metastore trick does not work. The .metadata is causing merge conflicts for every pull. I removed the "done" tag from this issue. -- RichiH + +>>>>>>>>> softbox sounds _really_ nice. File systems need to preserve mtimes. Oviously, it would be nice if git-annex exposed this to the upper layer instead of relying on this FUSE implementation, or the next, or the other totally cool thing around the corner to implement it again and again. +>>>>>>>>> I talked to the author of metastore; he is aware that the format is merge-unfriendly but never needed merges for himself. He is aware that this is not ideal for something like git. He does not have the time to implement a text storage instead of binary and I lack the skills to do it. If metastore is used, all it would need to do is introduce a new version of the store (it's versioned, apparently) and save metadata in text, one file per line. xattr would need to be ASCII-armoured, the rest could be plain text. I still think storing this directly in git-annex would make the most sense. Introducing a metadata storage file per storage object in .git/annex and using the object file's name as index is impossible because several softlinks might point to one object so it would need to be done per-softlink :/ -- RichiH From ee3be251a6480ea997c1e7c2c0aa86a046332aff Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 31 Mar 2011 21:32:11 +0000 Subject: [PATCH 1349/8313] Added a comment --- .../comment_9_45b11ddd200261115b653c7a14d28aa9._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment new file mode 100644 index 0000000000..8dfe746420 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 9" + date="2011-03-31T21:32:10Z" + content=""" +I'm was running git 1.7.4.1 at the time when I came across it, I have just upgraded to 1.7.4.2. I've also just moved to using a loopback fs for the stuff i care about. Do you still want a repo that exhibits the problem (excluding the .git/annex data) ??? I'm also not sure if 1.7.4.2 has corrected the problem yet as I haven't done much with my repos since. I suspect just making all the .git-annex hashed directories seems to be lower case might be better in the long run. +"""]] From f003603e8cc191efbba31b1f19b0086cd374635f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 10:46:23 +0000 Subject: [PATCH 1350/8313] --- ...it_annex_copy_--fast_does_not_copy_files.mdwn | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn new file mode 100644 index 0000000000..1a59538360 --- /dev/null +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -0,0 +1,16 @@ +Workflow: + + % git annex add + # list new files + % git commit -a -m "foo" + # commit summary + % git annex copy . --to remote --fast + # all files listed with "ok + % git annex copy . --to remote + # again, lists all files, _but the new ones are actually copied, this time_. + +This happens no matter if I + + % git push + +before copy or not. From 2f814eb0be5bb8c9d4e99e00e7ca2998649a4d9f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 10:47:49 +0000 Subject: [PATCH 1351/8313] --- doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn index 1a59538360..ada844a14c 100644 --- a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -5,7 +5,7 @@ Workflow: % git commit -a -m "foo" # commit summary % git annex copy . --to remote --fast - # all files listed with "ok + # all files listed with "ok" % git annex copy . --to remote # again, lists all files, _but the new ones are actually copied, this time_. @@ -14,3 +14,5 @@ This happens no matter if I % git push before copy or not. + +PS: Arguably, a copy should push automagically. From 5a6c69c8388dc80154b50000e608468ea36377d1 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 1 Apr 2011 16:11:52 +0000 Subject: [PATCH 1352/8313] Added a comment --- .../comment_10_f3594de3ba2ab17771a4b116031511bb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment new file mode 100644 index 0000000000..c3e6b5e598 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 10" + date="2011-04-01T16:11:52Z" + content=""" +No, I don't need a copy of your repo now. +"""]] From bf1e2205a4af4b490d5dd284a411bd180d8d6c9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 12:19:26 -0400 Subject: [PATCH 1353/8313] bugfix for uuid lookup --- GitQueue.hs | 3 +-- Remote/Git.hs | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GitQueue.hs b/GitQueue.hs index 097516c195..dfe2976da1 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -60,8 +60,7 @@ run repo (Queue _ m) = do - - Complicated by commandline length limits. -} runAction :: Git.Repo -> Action -> [FilePath] -> IO () -runAction repo action files = do - unless (null files) runxargs +runAction repo action files = unless (null files) runxargs where runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs params = toCommand $ Git.gitCommandLine repo diff --git a/Remote/Git.hs b/Remote/Git.hs index c1423bef7a..a458455109 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -54,13 +54,15 @@ gen r u _ = do (False, "") -> tryGitConfigRead r _ -> return r + u' <- getUUID r' + let defcst = if not $ Git.repoIsUrl r then cheapRemoteCost else expensiveRemoteCost cst <- remoteCost r' defcst return $ Remote { - uuid = u, + uuid = u', cost = cst, name = Git.repoDescribe r', storeKey = copyToRemote r', From ed7fc4fce919af6b10a4ab098f72862060ed750f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 12:34:06 -0400 Subject: [PATCH 1354/8313] Bugfix: copy --to --fast never really copied, fixed. --- Command/Move.hs | 6 ++++-- debian/changelog | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Command/Move.hs b/Command/Move.hs index 3ac5a7ab2c..951695d66e 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -89,8 +89,10 @@ toPerform dest move key = do -- and an explicit check is not done, when copying. When moving, -- it has to be done, to avoid inaverdent data loss. fast <- Annex.getState Annex.fast - isthere <- if fast && not move - then return $ Right True + isthere <- if fast && not move && not (Remote.hasKeyCheap dest) + then do + (remotes, _) <- Remote.keyPossibilities key + return $ Right $ dest `elem` remotes else Remote.hasKey dest key case isthere of Left err -> do diff --git a/debian/changelog b/debian/changelog index 0a232220f0..1f951064b1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -8,6 +8,7 @@ git-annex (0.20110329) UNRELEASED; urgency=low have proper permissions. * Added a special type of remote called a directory remote, which simply stores files in an arbitrary local directory. + * Bugfix: copy --to --fast never really copied, fixed. -- Joey Hess Sat, 26 Mar 2011 14:36:16 -0400 From dd5591781db3d47123c51ad5935c651f6a8ecf86 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 12:35:02 -0400 Subject: [PATCH 1355/8313] fixed --- doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn index ada844a14c..9b84c21fdb 100644 --- a/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn +++ b/doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn @@ -16,3 +16,7 @@ This happens no matter if I before copy or not. PS: Arguably, a copy should push automagically. + +> Whups, not supposed to be that fast! [[Fixed|done]], and +> you should run `git annex fsck --fast` on the repo you ran the +> copy in. --[[Joey]] From 3e33e489e8256787e847616730802657ea8cab38 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 23:00:53 +0000 Subject: [PATCH 1356/8313] --- ...ce_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn new file mode 100644 index 0000000000..a04c8b040b --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn @@ -0,0 +1,3 @@ +This works with bind-mount, I might try with softlinks as well. + +Going through git's data on push/pull can take ages on a spindle disk even if the repo is rather small in size. This is especially true if you are used to ssd speeds, but ssd storage is expensive. Storing the annex objects on a cheap spindle disk and everything else on a ssd makes things a _lot_ faster. From fd9b90fea1bd9b0052d1cd6205ee3406b5032b24 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 1 Apr 2011 23:06:06 +0000 Subject: [PATCH 1357/8313] --- doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn diff --git a/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn new file mode 100644 index 0000000000..3ec4f68aae --- /dev/null +++ b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn @@ -0,0 +1 @@ +As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo creates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH From 6ab63c438d3628dbcad201f775a18f4b6c8e758f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 21:24:06 -0400 Subject: [PATCH 1358/8313] not really a bug This can occur if a local remote repo has not been initted, so has no uuid yet. --- LocationLog.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/LocationLog.hs b/LocationLog.hs index f1e54432ca..d989cad611 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -90,7 +90,8 @@ instance Read LogLine where logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath logChange repo key u s = do when (null u) $ - error $ "bug detected: unknown UUID for " ++ Git.repoDescribe repo + error $ "unknown UUID for " ++ Git.repoDescribe repo ++ + " (have you run git annex init there?)" line <- logNow s u ls <- readLog logfile writeLog logfile (compactLog $ line:ls) From 1283ef73f8f5de109b350a52c07d2e8f52736679 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 21:31:37 -0400 Subject: [PATCH 1359/8313] releasing version 0.20110401 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 1f951064b1..6d56eeb460 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110329) UNRELEASED; urgency=low +git-annex (0.20110401) experimental; urgency=low * Amazon S3 is now supported as a special type of remote. Warning: Encrypting data before sending it to S3 is not yet supported. @@ -10,7 +10,7 @@ git-annex (0.20110329) UNRELEASED; urgency=low simply stores files in an arbitrary local directory. * Bugfix: copy --to --fast never really copied, fixed. - -- Joey Hess Sat, 26 Mar 2011 14:36:16 -0400 + -- Joey Hess Fri, 01 Apr 2011 21:27:22 -0400 git-annex (0.20110328) experimental; urgency=low From 5ec33dc6f90e81f6cc139a25104e96753203ae0d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 1 Apr 2011 21:32:05 -0400 Subject: [PATCH 1360/8313] add news item for git-annex 0.20110401 --- doc/news/version_0.20110401.mdwn | 11 +++++++++++ doc/news/version_0.25.mdwn | 6 ------ 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 doc/news/version_0.20110401.mdwn delete mode 100644 doc/news/version_0.25.mdwn diff --git a/doc/news/version_0.20110401.mdwn b/doc/news/version_0.20110401.mdwn new file mode 100644 index 0000000000..7c9ca6f5ca --- /dev/null +++ b/doc/news/version_0.20110401.mdwn @@ -0,0 +1,11 @@ +git-annex 0.20110401 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Amazon S3 is now supported as a special type of remote. + Warning: Encrypting data before sending it to S3 is not yet supported. + * Note that Amazon S3 support is not built in by default on Debian yet, + as hS3 is not packaged. + * fsck: Ensure that files and directories in .git/annex/objects + have proper permissions. + * Added a special type of remote called a directory remote, which + simply stores files in an arbitrary local directory. + * Bugfix: copy --to --fast never really copied, fixed."""]] \ No newline at end of file diff --git a/doc/news/version_0.25.mdwn b/doc/news/version_0.25.mdwn deleted file mode 100644 index 13abd66cd5..0000000000 --- a/doc/news/version_0.25.mdwn +++ /dev/null @@ -1,6 +0,0 @@ -git-annex 0.25 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix dropping of files using the URL backend. - * Fix support for remotes with '.' in their names. - * Add version command to show git-annex version as well as repository - version information."""]] \ No newline at end of file From 66ab18325e93b244b27b1c80269c943388622716 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 13:35:57 -0400 Subject: [PATCH 1361/8313] mention archive.org's S3 server git-annex + archive.org could be an interesting combo for public archivists --- doc/special_remotes/Amazon_S3.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index e4e57b3689..384110d1df 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -1,5 +1,6 @@ This special remote type stores file contents in a bucket in Amazon S3 -or a similar service. +or a similar service, such as +[Archive.org's S3 API](http://www.archive.org/help/abouts3.txt). See [[walkthrough/using_Amazon_S3]] for usage examples. From b159c10d9c839dc6dffc10ce4a7c51b456b65b28 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 2 Apr 2011 17:48:29 +0000 Subject: [PATCH 1362/8313] Added a comment --- .../comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment new file mode 100644 index 0000000000..124993bcf1 --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-02T17:48:29Z" + content=""" +Either option should work fine, but git gc --aggressive will probably avoid most of git's seeking. +"""]] From 616e6f8a840ef4d99632d12a2e7ea15c3cfb1805 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 13:49:03 -0400 Subject: [PATCH 1363/8313] Use lowercase hash directories for locationlog files to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still read; new information is written to the new directories. --- LocationLog.hs | 23 ++++++++--------------- Locations.hs | 33 ++++++++++++++++++++++++--------- Remote/Directory.hs | 2 +- debian/changelog | 9 +++++++++ doc/internals.mdwn | 2 +- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index d989cad611..8a47db2da3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -23,7 +23,6 @@ module LocationLog ( LogStatus(..), logChange, - logFile, readLog, writeLog, keyLocations @@ -33,7 +32,6 @@ import Data.Time.Clock.POSIX import Data.Time import System.Locale import qualified Data.Map as Map -import System.Directory import Control.Monad (when) import qualified GitRepo as Git @@ -93,22 +91,16 @@ logChange repo key u s = do error $ "unknown UUID for " ++ Git.repoDescribe repo ++ " (have you run git annex init there?)" line <- logNow s u - ls <- readLog logfile - writeLog logfile (compactLog $ line:ls) - return logfile - where - logfile = logFile repo key + let f = logFile repo key + ls' <- readLog $ logFileOld repo key + ls <- readLog f + writeLog f (compactLog $ line:ls'++ls) + return f {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> IO [LogLine] -readLog file = do - exists <- doesFileExist file - if exists - then do - s <- readFile file - return $ parseLog s - else return [] +readLog file = catch (return . parseLog =<< readFile file) (const $ return []) parseLog :: String -> [LogLine] parseLog s = filter parsable $ map read $ lines s @@ -131,7 +123,8 @@ logNow s u = do keyLocations :: Git.Repo -> Key -> IO [UUID] keyLocations thisrepo key = do ls <- readLog $ logFile thisrepo key - return $ map uuid $ filterPresent ls + ls' <- readLog $ logFileOld thisrepo key + return $ map uuid $ filterPresent $ ls'++ls {- Filters the list of LogLines to find ones where the value - is (or should still be) present. -} diff --git a/Locations.hs b/Locations.hs index 8e10c36b41..6c413a2183 100644 --- a/Locations.hs +++ b/Locations.hs @@ -20,7 +20,8 @@ module Locations ( gitAnnexUnusedLog, isLinkToAnnex, logFile, - hashDir, + logFileOld, + hashDirMixed, prop_idempotent_fileKey ) where @@ -68,7 +69,7 @@ objectDir = addTrailingPathSeparator $ annexDir "objects" {- Annexed file's location relative to the .git directory. -} annexLocation :: Key -> FilePath -annexLocation key = objectDir hashDir key f f +annexLocation key = objectDir hashDirMixed key f f where f = keyFile key @@ -113,8 +114,18 @@ isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s {- The filename of the log file for a given key. -} logFile :: Git.Repo -> Key -> String -logFile repo key = - gitStateDir repo ++ hashDir key ++ keyFile key ++ ".log" +logFile = logFile' hashDirLower + +{- The old filename of the log file for a key. These can have mixed + - case, which turned out to be a bad idea for directories whose contents + - are checked into git. There was no conversion, so these have to be checked + - for and merged in at runtime. -} +logFileOld :: Git.Repo -> Key -> String +logFileOld = logFile' hashDirMixed + +logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String +logFile' hasher repo key = + gitStateDir repo ++ hasher key ++ keyFile key ++ ".log" {- Converts a key into a filename fragment. - @@ -147,13 +158,17 @@ prop_idempotent_fileKey s = Just k == fileKey (keyFile k) {- Given a key, generates a short directory name to put it in, - to do hashing to protect against filesystems that dislike having - many items in a single directory. -} -hashDir :: Key -> FilePath -hashDir k = addTrailingPathSeparator $ take 2 dir drop 2 dir +hashDirMixed :: Key -> FilePath +hashDirMixed k = addTrailingPathSeparator $ take 2 dir drop 2 dir where - dir = take 4 $ abcd_to_dir $ md5 $ Str $ show k + dir = take 4 $ concat $ map display_32bits_as_dir [a,b,c,d] + ABCD (a,b,c,d) = md5 $ Str $ show k -abcd_to_dir :: ABCD -> String -abcd_to_dir (ABCD (a,b,c,d)) = concat $ map display_32bits_as_dir [a,b,c,d] +{- Generates a hash directory that is all lower case. -} +hashDirLower :: Key -> FilePath +hashDirLower k = addTrailingPathSeparator $ take 3 dir drop 3 dir + where + dir = take 6 $ md5s $ Str $ show k {- modified version of display_32bits_as_hex from Data.Hash.MD5 - Copyright (C) 2001 Ian Lynagh diff --git a/Remote/Directory.hs b/Remote/Directory.hs index f97449eaa7..0d3478b79d 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -66,7 +66,7 @@ directorySetup u c = do return $ M.delete "directory" c dirKey :: FilePath -> Key -> FilePath -dirKey d k = d hashDir k f f +dirKey d k = d hashDirMixed k f f where f = keyFile k diff --git a/debian/changelog b/debian/changelog index 6d56eeb460..29f60063e8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +git-annex (0.20110402) UNRELEASED; urgency=low + + * Use lowercase hash directories for locationlog files, to avoid + some issues with git on OSX with the mixed-case directories. + No migration is needed; the old mixed case hash directories are still + read; new information is written to the new directories. + + -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 + git-annex (0.20110401) experimental; urgency=low * Amazon S3 is now supported as a special type of remote. diff --git a/doc/internals.mdwn b/doc/internals.mdwn index 2e9f253831..b362e68e1e 100644 --- a/doc/internals.mdwn +++ b/doc/internals.mdwn @@ -53,7 +53,7 @@ Example: e605dca6-446a-11e0-8b2a-002170d25c55 1 26339d22-446b-11e0-9101-002170d25c55 ? -## `.git-annex/aa/bb/*.log` +## `.git-annex/aaa/bbb/*.log` The remainder of the log files record [[location_tracking]] information for file contents. Again these are placed in two levels of subdirectories From 623a071fdba9061caed2fa39d0353508df593d06 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 2 Apr 2011 17:53:58 +0000 Subject: [PATCH 1364/8313] Added a comment --- ...ent_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment new file mode 100644 index 0000000000..db605f9650 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 11" + date="2011-04-02T17:53:58Z" + content=""" +I have pushed out a preliminary fix. The old mixed-case directories will be left where they are, and still read from by git-annex. New data will be written to new, lower-case directories. I think that once git stops seeing changes being made +to mixed-case, colliding directories, the bugs you ran into won't manifest any more. + +You will need to find a way to get your git repository out of the state where it complains about uncommitted files (and won't let you commit them). I have not found a reliable way to do that; git reset --hard worked in one case but not in another. May need to clone a fresh git repository. + +Let me know how it works out. +"""]] From ba423c5c0b07161a819ee6b21a37c0f7d983b83a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 2 Apr 2011 17:58:24 +0000 Subject: [PATCH 1365/8313] Added a comment --- .../comment_12_f1c53c3058a587185e7a78d84987539d._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment new file mode 100644 index 0000000000..5f9a0ae275 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 12" + date="2011-04-02T17:58:24Z" + content=""" +Also, you can delete `.git-annex/??` if you want to, then running `git annex fsck --fast` in each of your clones would regenerate the data using only the lower-case hash directories. +"""]] From 7048a126e2c9a5aa1c5a7781c00e627190183bd1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 14:26:47 -0400 Subject: [PATCH 1366/8313] tweak --- doc/use_case/Alice.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index ee97efa43a..7199258a6f 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -4,7 +4,8 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. She stores data in the Cloud. All these things can have different files on them, but Alice no longer has to deal with the -tedious process of keeping them manually in sync. +tedious process of keeping them manually in sync, or rembering where +she put a file. When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her From c4b3081f7c972fe8fd45c4dfa34d37a659b207e3 Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sat, 2 Apr 2011 19:11:11 +0000 Subject: [PATCH 1367/8313] --- ...tic__41__:_strange_closure_type_30799.mdwn | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn new file mode 100644 index 0000000000..60f7d9ea9b --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -0,0 +1,45 @@ +I ran git-annex (git version) on three machines with ghc-7.0.2 for about a month, but recently (no more than a week ago) I've started getting this error for every file on "git annex get": + + git-annex-shell: internal error: evacuate(static): strange closure type 30799 + (GHC version 7.0.2 for i386_unknown_linux) + Please report this as a GHC bug: http://www.haskell.org/ghc/reportabug + +There were no changes to ghc or it's modules, so I assume something has changed in git-annex itself. + +strace shows "git annnex get" (on "host1") performing following exec's: + + [pid 9481] execve("/usr/bin/rsync", ["rsync", "-p", "--progress", "--inplace", "-e", "'ssh' 'user@host2' 'git-annex-shell ''sendkey'' ''/remote/path'' ''SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b'' ''--'''", ":", "/local/path/.git/annex/tmp/SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b"], [/* 41 vars */]) = 0 + [pid 9482] execve("/usr/bin/ssh", ["ssh", "user@host2", "git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--'", "", "rsync", "--server", "--sender", "-vpe.Lsf", "--inplace", ".", ""], [/* 41 vars */] + +I've tried running the second command directly from the shell and got the same error message from a remote GHC. +Adding strace before git-annex-shell to remote command yielded something like this in the end: + + stat64("/local/path.git", 0xb727d610) = -1 ENOENT (No such file or directory) + stat64("/local/path.git", 0xb727d6b0) = -1 ENOENT (No such file or directory) + waitpid(7525, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0) = 7525 + chdir("/home/user") = 0 + rt_sigprocmask(SIG_BLOCK, [INT], [], 8) = 0 + write(2, "git-annex-shell: internal error: ", 33git-annex-shell: internal error: ) = 33 + ... + +Note that "/local/path" here is not what's specified in rsync arguments at all, and git repo with files-to-be-fetched on "host2" is in "/remote/path", but "/local/path" is present in git remotes there since I mount it via nfs from "host1" (yes, to the same path as it's there): + + [remote "nfs"] + url = /local/path + fetch = +refs/heads/*:refs/remotes/nfs/* + push = refs/heads/*:refs/remotes/host2/* + annex-uuid = 0a4e14ba-5236-11e0-9004-7f24452c0f05 + +If I comment that remote out from "/remote/path/.git/config", "git annex get" works fine. +The only git-command git-annex-shell seem to exec there (on "host2") is "git config --list", so it's shouldn't be git trying to do something with it's remotes - it's git-annex itself, right? + +Anyways, looks like a simple path-joining error, if "/local/path.git" should be "/local/path/.git" there. + +I'm actually quite confused about what it's trying to do with that path. +Connect from "host1" to "host2" just to connect back to "host1"? +What for, when it should just fetch files from "host2"? + +Not sure if it's a bug or I'm doing something wrong, but if git-annex really need to check something in git remotes' paths, error message (the one at the top of this post) can be a more descriptive, I guess. +Something like "error: failed to do something with git remote X on a remote host" would've been a lot less confusing than that GHC thing. + +Thanks! From f005a84e5675cb3e551b2922ad42642df28264d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 15:50:51 -0400 Subject: [PATCH 1368/8313] add loggedKeys --- Content.hs | 10 +++------- LocationLog.hs | 32 ++++++++++++++++++++++++++------ Locations.hs | 9 +++++++++ Utility.hs | 12 ++++++++++++ 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/Content.hs b/Content.hs index 88e8dbc007..ba265c9307 100644 --- a/Content.hs +++ b/Content.hs @@ -219,9 +219,9 @@ getKeysPresent' dir = do then return [] else do -- 2 levels of hashing - levela <- liftIO $ subdirContent dir - levelb <- liftIO $ mapM subdirContent levela - contents <- liftIO $ mapM subdirContent (concat levelb) + levela <- liftIO $ dirContents dir + levelb <- liftIO $ mapM dirContents levela + contents <- liftIO $ mapM dirContents (concat levelb) files <- liftIO $ filterM present (concat contents) return $ catMaybes $ map (fileKey . takeFileName) files where @@ -231,7 +231,3 @@ getKeysPresent' dir = do case result of Right s -> return $ isRegularFile s Left _ -> return False - subdirContent d = do - c <- getDirectoryContents d - return $ map (d ) $ filter notcruft c - notcruft f = f /= "." && f /= ".." diff --git a/LocationLog.hs b/LocationLog.hs index 8a47db2da3..c2d956a291 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -1,9 +1,7 @@ {- git-annex location log - - - git-annex keeps track of on which repository it last saw a value. - - This can be useful when using it for archiving with offline storage. - - When you indicate you --want a file, git-annex will tell you which - - repositories have the value. + - git-annex keeps track of which repositories have the contents of annexed + - files. - - Location tracking information is stored in `.git-annex/key.log`. - Repositories record their UUID and the date when they --get or --drop @@ -15,7 +13,7 @@ - Git is configured to use a union merge for this file, - so the lines may be in arbitrary order, but it will never conflict. - - - Copyright 2010 Joey Hess + - Copyright 2010-2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -25,14 +23,19 @@ module LocationLog ( logChange, readLog, writeLog, - keyLocations + keyLocations, + loggedKeys, + logFile ) where import Data.Time.Clock.POSIX import Data.Time import System.Locale +import System.Directory +import System.FilePath import qualified Data.Map as Map import Control.Monad (when) +import Data.Maybe import qualified GitRepo as Git import Utility @@ -153,3 +156,20 @@ mapLog m l = Just l' -> (date l' <= date l) Nothing -> True u = uuid l + +{- Finds all keys that have location log information. -} +loggedKeys :: Git.Repo -> IO [Key] +loggedKeys repo = do + let dir = gitStateDir repo + exists <- doesDirectoryExist dir + if exists + then do + -- 2 levels of hashing + levela <- dirContents dir + levelb <- mapM tryDirContents levela + files <- mapM tryDirContents (concat levelb) + return $ catMaybes $ + map (logFileKey . takeFileName) (concat files) + else return [] + where + tryDirContents d = catch (dirContents d) (return . const []) diff --git a/Locations.hs b/Locations.hs index 6c413a2183..f263ea526b 100644 --- a/Locations.hs +++ b/Locations.hs @@ -21,6 +21,7 @@ module Locations ( isLinkToAnnex, logFile, logFileOld, + logFileKey, hashDirMixed, prop_idempotent_fileKey @@ -127,6 +128,14 @@ logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String logFile' hasher repo key = gitStateDir repo ++ hasher key ++ keyFile key ++ ".log" +{- Converts a log filename into a key. -} +logFileKey :: FilePath -> Maybe Key +logFileKey file + | end == ".log" = readKey beginning + | otherwise = Nothing + where + (beginning, end) = splitAt (length file - 4) file + {- Converts a key into a filename fragment. - - Escape "/" in the key name, to keep a flat tree of files and avoid diff --git a/Utility.hs b/Utility.hs index 8312335f86..72f5c50638 100644 --- a/Utility.hs +++ b/Utility.hs @@ -22,6 +22,7 @@ module Utility ( readMaybe, safeWriteFile, dirContains, + dirContents, prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, @@ -235,3 +236,14 @@ safeWriteFile file content = do createDirectoryIfMissing True (parentDir file) writeFile tmpfile content renameFile tmpfile file + +{- Lists the contents of a directory. + - Unlike getDirectoryContents, paths are not relative to the directory. -} +dirContents :: FilePath -> IO [FilePath] +dirContents d = do + c <- getDirectoryContents d + return $ map (d ) $ filter notcruft c + where + notcruft "." = False + notcruft ".." = False + notcruft _ = True From c5f8284af1518799bbb76d1ea189a286f03b9bd0 Mon Sep 17 00:00:00 2001 From: "http://dieter-be.myopenid.com/" Date: Sat, 2 Apr 2011 20:24:34 +0000 Subject: [PATCH 1369/8313] Added a comment --- .../comment_3_60691af4400521b5a8c8d75efe3b44cb._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment diff --git a/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment new file mode 100644 index 0000000000..9280f2dccf --- /dev/null +++ b/doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://dieter-be.myopenid.com/" + nickname="dieter" + subject="comment 3" + date="2011-04-02T20:24:33Z" + content=""" + * why the `git remote add laptop ~/annex` ? this remote already exists under the name origin. + * doesn't the last command need to be `git remote add usbdrive /media/usb/annex`? because the actual repo would be in /media/usb/annex, not /media/usb? +"""]] From d1b18047501c8c815ad8cc910e96fae2cb64d4ca Mon Sep 17 00:00:00 2001 From: gernot Date: Sat, 2 Apr 2011 20:54:43 +0000 Subject: [PATCH 1370/8313] --- ..._untracked_.git-annex__47____42___directories.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn new file mode 100644 index 0000000000..ab8b255a0f --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn @@ -0,0 +1,11 @@ +I upgraded another one of my git-annex clones. The upgrade worked fine (i.e. +according to the manual) on two other clones before, but this time something is +different. + +After 'git pull' and 'git annex upgrade', which took a long time and seemed to +have succeeded, there are no staged changes in git. Instead there are lots of +untracked directories in .git-annex. Aside from that, nothing seems to be +wrong. + +At the time I had git-annex version 0.20110329 and I've been using the SHA1 +backend since version 1. From 61a22d3fee29f4c8d46ab7996391f1b539974944 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 2 Apr 2011 21:29:56 +0000 Subject: [PATCH 1371/8313] typo --- doc/use_case/Alice.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 7199258a6f..80280580e3 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -4,7 +4,7 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server out there on the net. She stores data in the Cloud. All these things can have different files on them, but Alice no longer has to deal with the -tedious process of keeping them manually in sync, or rembering where +tedious process of keeping them manually in sync, or remembering where she put a file. When she has 1 bar on her cell, Alice queues up interesting files on her From f69a4c56e970ccc991c6970d9ead225da2d4200b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 2 Apr 2011 21:34:24 +0000 Subject: [PATCH 1372/8313] Added a comment --- ..._f94abce32ef818176b42a3cc860691ae._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment new file mode 100644 index 0000000000..eddc8c6315 --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-02T21:34:24Z" + content=""" +I'll give it a try as soon as I get rid of this: + + % git annex fsck +fatal: index file smaller than expected +fatal: index file smaller than expected + % git status +fatal: index file smaller than expected + % + +And no, I am not sure where that is coming from all of a sudden... (it might have to do with a hard lockup of the whole system due to a faulty hdd I tested, but I didn't do anything to it for ages before that lock-up. So meh. Also, this is prolly off topic in here) + + +Richard +"""]] From 2a08107ce46a0ebd444af274b7923f0b3043de9d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 2 Apr 2011 21:53:18 +0000 Subject: [PATCH 1373/8313] --- doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn index 3ec4f68aae..4b661101d7 100644 --- a/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn +++ b/doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn @@ -1 +1,3 @@ -As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo creates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH +As git annex keeps logs about file transfers anyway, it should be relatively easy to add traffic accounting to a repo. That would allow me to monitor how much traffic a given repo generates. As I might end up hosting git-annex repos for a few personal friends, I need/want a way to track the heavy hitters. -- RichiH + +PS: If you ever plan to host git-annex similar branchable, this would probably be of interest to you, as well :) From 6c320804395b3bcb043e2d73322465c2ab7212fc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 00:27:30 +0000 Subject: [PATCH 1374/8313] --- .../No_easy_way_to_re-inject_a_file_into_an_annex.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn new file mode 100644 index 0000000000..ced4fc5a0f --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn @@ -0,0 +1,10 @@ +My local git index got corrupted and I needed to clone and annex get all data from my main repo. + +Some files were never copied anywhere so I am stuck with symlinks to nowhere. + +I tried to copy over the symlink with a copy of the actual file, which did not work. Trying to unlock, copying over the symlink, and relock did not work, either. + +Then, I copied the annex object to the correct place in .git/annex/objects/..., set all modes, re-ran fsck and the file re-appeared. + + +Long story short, I think there should be a `git annex reinject $file` or similar which will take a file, either one replacing the symlink or with an arbitrary path, and put it into the correct place in the object store. Called normally, it should reject all reinjects where the checksum does not match. With --force, this should be overridden. For reasons of safety, WORM should always require --force. From 09a16176dea5ef2a51e3a3d00d77180966c597d9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 20:36:01 -0400 Subject: [PATCH 1375/8313] read log files strictly This avoids leaking fds when an operation needs to read a lot of log files, as unused will. --- LocationLog.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/LocationLog.hs b/LocationLog.hs index c2d956a291..e0ccb642b3 100644 --- a/LocationLog.hs +++ b/LocationLog.hs @@ -103,7 +103,7 @@ logChange repo key u s = do {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> IO [LogLine] -readLog file = catch (return . parseLog =<< readFile file) (const $ return []) +readLog file = catch (return . parseLog =<< readFileStrict file) (const $ return []) parseLog :: String -> [LogLine] parseLog s = filter parsable $ map read $ lines s @@ -157,7 +157,8 @@ mapLog m l = Nothing -> True u = uuid l -{- Finds all keys that have location log information. -} +{- Finds all keys that have location log information. + - (There may be duplicate keys in the list.) -} loggedKeys :: Git.Repo -> IO [Key] loggedKeys repo = do let dir = gitStateDir repo From 5eaa90ab3514e4417b0e002ee47da0a0c3c1d716 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 00:36:04 +0000 Subject: [PATCH 1376/8313] --- doc/bugs/Displayed_copy_speed_is_wrong.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/bugs/Displayed_copy_speed_is_wrong.mdwn diff --git a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn new file mode 100644 index 0000000000..c1209c73c0 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn @@ -0,0 +1,5 @@ +When copying data to my remote, I regularly see speeds in excess of 100 MB/s on my home DSL line. + + 2073939 100% 176.96MB/s 0:00:00 (xfer#1, to-check=0/1) + +This is definitely not correct. From a31fb09eafb73897d63d7527f2e66bdd11f6802f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 00:50:48 +0000 Subject: [PATCH 1377/8313] --- ...ninit_do_not_work_when_git_index_is_broken.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn new file mode 100644 index 0000000000..509e12aebb --- /dev/null +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn @@ -0,0 +1,14 @@ +git's index broke and I was unable to restore it. While this is not git-annex' problem, it should still be possible to get my data in an un-annexed state. + + % git status + fatal: index file smaller than expected + % git annex unannex foo + fatal: index file smaller than expected + % git annex uninit + fatal: index file smaller than expected + uninit + pre-commit hook (/path/to/git-annex/.git/hooks/pre-commit) contents modified; not deleting. Edit it to remove call to git annex. + ok + % + +Ttbomk, the softlinks and objects are enough to un-annex the files; side-stepping git's index if necessary. From 868300d4c1dafd2c4b91ad3f369cfb48f14bb82a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 20:59:41 -0400 Subject: [PATCH 1378/8313] unused/dropunused: support --from --- Command/DropUnused.hs | 31 ++++++--- Command/Unused.hs | 116 ++++++++++++++++++++++--------- debian/changelog | 3 + doc/git-annex.mdwn | 16 +++-- doc/special_remotes.mdwn | 23 ++++++ doc/walkthrough/unused_data.mdwn | 2 +- 6 files changed, 147 insertions(+), 44 deletions(-) diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 932a8b8635..1eec688202 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -19,6 +19,8 @@ import Messages import Locations import qualified Annex import qualified Command.Drop +import qualified Command.Move +import qualified Remote import Backend import Key @@ -40,15 +42,28 @@ start m s = notBareRepo $ do case M.lookup s m of Nothing -> return Nothing Just key -> do - g <- Annex.gitRepo showStart "dropunused" s - backend <- keyBackend key - -- drop both content in the backend and any tmp - -- file for the key - let tmp = gitAnnexTmpLocation g key - tmp_exists <- liftIO $ doesFileExist tmp - when tmp_exists $ liftIO $ removeFile tmp - return $ Just $ Command.Drop.perform key backend (Just 0) + from <- Annex.getState Annex.fromremote + case from of + Just name -> do + r <- Remote.byName name + return $ Just $ performRemote r key + _ -> return $ Just $ perform key + +{- drop both content in the backend and any tmp file for the key -} +perform :: Key -> CommandPerform +perform key = do + g <- Annex.gitRepo + let tmp = gitAnnexTmpLocation g key + tmp_exists <- liftIO $ doesFileExist tmp + when tmp_exists $ liftIO $ removeFile tmp + backend <- keyBackend key + Command.Drop.perform key backend (Just 0) -- force drop + +performRemote :: Remote.Remote Annex -> Key -> CommandPerform +performRemote r key = do + showNote $ "from " ++ Remote.name r ++ "..." + return $ Just $ Command.Move.fromCleanup r True key readUnusedLog :: Annex (M.Map String Key) readUnusedLog = do diff --git a/Command/Unused.hs b/Command/Unused.hs index 83d8757cff..a3fb6fe232 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -20,9 +20,11 @@ import Content import Messages import Locations import Utility +import LocationLog import qualified Annex import qualified GitRepo as Git import qualified Backend +import qualified Remote command :: [Command] command = [repoCommand "unused" paramNothing seek @@ -39,35 +41,54 @@ start = notBareRepo $ do perform :: CommandPerform perform = do - _ <- checkUnused + from <- Annex.getState Annex.fromremote + case from of + Just name -> do + r <- Remote.byName name + checkRemoteUnused r + _ -> checkUnused return $ Just $ return True -checkUnused :: Annex Bool +checkUnused :: Annex () checkUnused = do (unused, staletmp) <- unusedKeys let unusedlist = number 0 unused let staletmplist = number (length unused) staletmp let list = unusedlist ++ staletmplist - g <- Annex.gitRepo - liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ unlines $ - map (\(n, k) -> show n ++ " " ++ show k) list - unless (null unused) $ showLongNote $ unusedmsg unusedlist - unless (null staletmp) $ showLongNote $ staletmpmsg staletmplist + writeUnusedFile list + unless (null unused) $ showLongNote $ unusedMsg unusedlist + unless (null staletmp) $ showLongNote $ staleTmpMsg staletmplist unless (null list) $ showLongNote $ "\n" - return $ null list +checkRemoteUnused :: Remote.Remote Annex -> Annex () +checkRemoteUnused r = do + g <- Annex.gitRepo + showNote $ "checking for unused data on " ++ Remote.name r ++ "..." + referenced <- getKeysReferenced + logged <- liftIO $ loggedKeys g + remotehas <- filterM isthere logged + let remoteunused = remotehas `exclude` referenced + let list = number 0 remoteunused + writeUnusedFile list + unless (null remoteunused) $ do + showLongNote $ remoteUnusedMsg r list + showLongNote $ "\n" + where + isthere k = do + g <- Annex.gitRepo + us <- liftIO $ keyLocations g k + return $ uuid `elem` us + uuid = Remote.uuid r + +writeUnusedFile :: [(Int, Key)] -> Annex () +writeUnusedFile l = do + g <- Annex.gitRepo + liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ + unlines $ map (\(n, k) -> show n ++ " " ++ show k) l + +table :: [(Int, Key)] -> [String] +table l = [" NUMBER KEY"] ++ map cols l where - unusedmsg u = unlines $ - ["Some annexed data is no longer pointed to by any files in the repository:"] - ++ table u ++ - ["(To see where data was previously used, try: git log --stat -S'KEY')"] ++ - dropmsg - staletmpmsg t = unlines $ - ["Some partially transferred data exists in temporary files:"] - ++ table t ++ dropmsg - dropmsg = ["(To remove unwanted data: git-annex dropunused NUMBER)"] - - table l = [" NUMBER KEY"] ++ map cols l cols (n,k) = " " ++ pad 6 (show n) ++ " " ++ show k pad n s = s ++ replicate (n - length s) ' ' @@ -75,6 +96,39 @@ number :: Int -> [a] -> [(Int, a)] number _ [] = [] number n (x:xs) = (n+1, x):(number (n+1) xs) +staleTmpMsg :: [(Int, Key)] -> String +staleTmpMsg t = unlines $ + ["Some partially transferred data exists in temporary files:"] + ++ table t ++ [dropMsg Nothing] + +unusedMsg :: [(Int, Key)] -> String +unusedMsg u = unusedMsg' u + ["Some annexed data is no longer used by any files in the repository:"] + [dropMsg Nothing] + +remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String +remoteUnusedMsg r u = unusedMsg' u + ["Some annexed data on " ++ name ++ + " is not used by any files in this repository."] + [dropMsg $ Just r, + "Please be cautious -- are you sure that the remote repository", + "does not use this data?"] + where + name = Remote.name r + +unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String +unusedMsg' u header trailer = unlines $ + header ++ + table u ++ + ["(To see where data was previously used, try: git log --stat -S'KEY')"] ++ + trailer + +dropMsg :: Maybe (Remote.Remote Annex) -> String +dropMsg Nothing = dropMsg' "" +dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r +dropMsg' :: String -> String +dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)" + {- Finds keys whose content is present, but that do not seem to be used - by any files in the git repo, or that are only present as tmp files. -} unusedKeys :: Annex ([Key], [Key]) @@ -93,7 +147,9 @@ unusedKeys = do referenced <- getKeysReferenced tmps <- tmpKeys - let (unused, staletmp, duptmp) = calcUnusedKeys present referenced tmps + let unused = present `exclude` referenced + let staletmp = tmps `exclude` present + let duptmp = tmps `exclude` staletmp -- Tmp files that are dups of content already present -- can simply be removed. @@ -102,18 +158,16 @@ unusedKeys = do return (unused, staletmp) -calcUnusedKeys :: [Key] -> [Key] -> [Key] -> ([Key], [Key], [Key]) -calcUnusedKeys present referenced tmps = (unused, staletmp, duptmp) +{- Finds items in the first, smaller list, that are not + - present in the second, larger list. + - + - Constructing a single set, of the list that tends to be + - smaller, appears more efficient in both memory and CPU + - than constructing and taking the S.difference of two sets. -} +exclude :: Ord a => [a] -> [a] -> [a] +exclude [] _ = [] -- optimisation +exclude smaller larger = S.toList $ remove larger $ S.fromList smaller where - unused = present `exclude` referenced - staletmp = tmps `exclude` present - duptmp = tmps `exclude` staletmp - - -- Constructing a single set, of the list that tends to be - -- smaller, appears more efficient in both memory and CPU - -- than constructing and taking the S.difference of two sets. - exclude [] _ = [] -- optimisation - exclude smaller larger = S.toList $ remove larger $ S.fromList smaller remove a b = foldl (flip S.delete) b a {- List of keys referenced by symlinks in the git repo. -} diff --git a/debian/changelog b/debian/changelog index 29f60063e8..e504bd8f60 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (0.20110402) UNRELEASED; urgency=low some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still read; new information is written to the new directories. + * Unused files on remotes, particulary special remotes, can now be + identified and dropped, by using "--from remote" with git annex unused + and git annex dropunused. -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index d890b518b8..7d0fb3e792 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -155,16 +155,21 @@ Many git-annex commands will stage changes for later `git commit` by you. * unused - Checks the annex for data that is not used by any files currently - in the annex, and prints a numbered list of the data. + Checks the annex for data that does not correspond to any files currently + in the respository, and prints a numbered list of the data. To only show unused temp files, specify --fast + To check data on a remote that does not correspond to any files currently + in the local repository, specify --from. + * dropunused [number ...] Drops the data corresponding to the numbers, as listed by the last `git annex unused` + To drop the data from a remote, specify --from. + * find [path ...] Outputs a list of annexed files whose content is currently present. @@ -317,12 +322,15 @@ Many git-annex commands will stage changes for later `git commit` by you. * --from=repository - Specifies a repository that content will be retrieved from. + Specifies a repository that content will be retrieved from, or that + should otherwise be acted on. + It should be specified using the name of a configured remote. * --to=repository - Specifies a repository that content will be sent to. + Specifies a repository that content will be sent to. + It should be specified using the name of a configured remote. * --exclude=glob diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index 09b751d0f4..f4d479aa9c 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -8,3 +8,26 @@ They cannot be used by other git commands though. * [[Amazon_S3]] * [[directory]] + +## Unused content on special remotes + +Over time, special remotes can accumulate file content that is no longer +referred to by files in git. Normally, unused content in the current +repository is found by running `git annex unused`. To detect unused content +on special remotes, instead use `git annex unused --from`. Example: + + $ git annex unused --from mys3 + unused (checking for unused data on mys3...) + Some annexed data on mys3 is not used by any files in this repository. + NUMBER KEY + 1 WORM-s3-m1301674316--foo + (To see where data was previously used, try: git log --stat -S'KEY') + (To remove unwanted data: git-annex dropunused --from mys3 NUMBER) + Please be cautious -- are you sure that the remote repository + does not use this data? + $ git annex dropunused --from mys3 1 + dropunused 12948 (from mys3...) ok + +Do be cautious when using this; it cannot detect if content in a remote +is used by that remote, or is the last copy of data that is used by +some *other* remote. diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn index 9be32577c3..2f8edcd388 100644 --- a/doc/walkthrough/unused_data.mdwn +++ b/doc/walkthrough/unused_data.mdwn @@ -10,7 +10,7 @@ eliminate it to save space. # git annex unused unused (checking for unused data...) - Some annexed data is no longer pointed to by any files in the repository. + Some annexed data is no longer used by any files in the repository. NUMBER KEY 1 WORM-s3-m1289672605--file 2 WORM-s14-m1289672605--file From dd64c46804be06ae3a1dda3ab353a6241566d5a8 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:37:29 +0000 Subject: [PATCH 1379/8313] Added a comment --- .../comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment diff --git a/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment b/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment new file mode 100644 index 0000000000..62a595be77 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:37:29Z" + content=""" +That is displayed by rsync. It's not unheard of for rsync to resume a transfer and display extremely high speeds. +"""]] From 9026289e9a1cc20a014945e9d529e7d602a35432 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:40:50 +0000 Subject: [PATCH 1380/8313] Added a comment --- .../comment_1_1931e733f0698af5603a8b92267203d4._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment new file mode 100644 index 0000000000..84b68bb7ba --- /dev/null +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:40:50Z" + content=""" +They rely on git-ls-files to get a list of files that are checked into git, in order to tell what to unannex. +"""]] From 192e988a4422afef08142ab40008bdce6aada8d2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:46:16 +0000 Subject: [PATCH 1381/8313] Added a comment --- .../comment_1_c871605e187f539f3bfe7478433e7fb5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment new file mode 100644 index 0000000000..9688012a47 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T01:46:16Z" + content=""" +Have you seen [[walkthrough/recover_data_from_lost+found]]? The method described there will also work in this scenario. +"""]] From efefe1397d1cb88b42242bc05c85be1ea7450826 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 01:48:57 +0000 Subject: [PATCH 1382/8313] Added a comment --- ...comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment new file mode 100644 index 0000000000..fc29236c6d --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-03T01:48:57Z" + content=""" +For future reference, git can recover from a corrupted index file with `rm .git/index; git reset --mixed`. + +Of course, you lose any staged changes that were in the old index file, and may need to re-stage some files. +"""]] From 59923c6cf735260d96f99b0a914ce84df122aece Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 21:52:57 -0400 Subject: [PATCH 1383/8313] tweak --- doc/walkthrough/recover_data_from_lost+found.mdwn | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/walkthrough/recover_data_from_lost+found.mdwn index 6e2c241485..f101b3caf6 100644 --- a/doc/walkthrough/recover_data_from_lost+found.mdwn +++ b/doc/walkthrough/recover_data_from_lost+found.mdwn @@ -3,12 +3,14 @@ It's actually very easy to recover from this disaster. First, check out the git repository again. Then, in the new checkout: - mkdir recovered-content - sudo mv ../lost+found/* recovered-content - git annex add recovered-content - git rm recovered-content - git commit -m "recovered some content" - git annex fsck + $ mkdir recovered-content + $ sudo mv ../lost+found/* recovered-content + $ sudo chown you:you recovered-content + $ chmod -R u+w recovered-content + $ git annex add recovered-content + $ git rm recovered-content + $ git commit -m "recovered some content" + $ git annex fsck The way that works is that when git-annex adds the same content that was in the repository before, all the old links to that content start working From 31912a9b3bd0ee31aea0cc3be6acacf230278968 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 02:26:20 +0000 Subject: [PATCH 1384/8313] Added a comment --- ...comment_1_9ca2da52f3c8add0276b72d6099516a6._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment new file mode 100644 index 0000000000..78309df872 --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T02:26:20Z" + content=""" +I'm not sure how this happened, as far as I can see, and based on my testing, `git annex upgrade` does stage the location log files. OTOH, I vaguely rememeber needing to stage some of them when I was doing my own upgrades, but that was a while ago, and I don't remember the details. + +Your upgrade seems to have gone ok from the file lists you sent, so you can just: `git add .git-annex; git commit` +"""]] From e08fa92dfa1c2bc791b940790c9075d400c518b9 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 02:32:17 +0000 Subject: [PATCH 1385/8313] Added a comment --- ...comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment diff --git a/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment new file mode 100644 index 0000000000..b4dcb6422a --- /dev/null +++ b/doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-03T02:32:17Z" + content=""" +Good spotting on the last line, fixed. + +The laptop remote is indeed redundant, but it leads to clearer views of what is going on later in the walkthrough (\"git pull laptop master\", \"(copying from laptop...)\"). And if the original clone is made from a central bare repo, this reinforces that you'll want to set up remotes for other repos on the computer. +"""]] From 248bbf76ca408fe7c89f47917e1cfae5ffc079c7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 22:34:05 -0400 Subject: [PATCH 1386/8313] typo --- doc/walkthrough/adding_a_remote.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/adding_a_remote.mdwn b/doc/walkthrough/adding_a_remote.mdwn index be8e8e7fe5..97690dfcdf 100644 --- a/doc/walkthrough/adding_a_remote.mdwn +++ b/doc/walkthrough/adding_a_remote.mdwn @@ -8,7 +8,7 @@ Let's start by adding a USB drive as a remote. # git annex init "portable USB drive" # git remote add laptop ~/annex # cd ~/annex - # git remote add usbdrive /media/usb + # git remote add usbdrive /media/usb/annex This is all standard ad-hoc distributed git repository setup. The only git-annex specific part is telling it the name From 757a465b53fb5bf613dc7808e8c29819059d1ff4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 2 Apr 2011 22:55:57 -0400 Subject: [PATCH 1387/8313] analysis --- ...tic__41__:_strange_closure_type_30799.mdwn | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn index 60f7d9ea9b..87a0435208 100644 --- a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -39,7 +39,33 @@ I'm actually quite confused about what it's trying to do with that path. Connect from "host1" to "host2" just to connect back to "host1"? What for, when it should just fetch files from "host2"? +> git-annex (and git-annex shell) always start up by learning what git +> remotes are locally configured, and this includes checking them to +> try to look up their annex.uuid setting. +> +> Since git will, given a remote like "url = /foo", first look in +> "/foo.git" for a bare git repository, so too does git-annex. +> I do not think this is a path joining error. That seems likely to +> be a red herring. --[[Joey]] + Not sure if it's a bug or I'm doing something wrong, but if git-annex really need to check something in git remotes' paths, error message (the one at the top of this post) can be a more descriptive, I guess. Something like "error: failed to do something with git remote X on a remote host" would've been a lot less confusing than that GHC thing. Thanks! + +> I've never seen anything like this error message. I don't know if the +> problem is caused by building with GHC 7, or what. You didn't say what +> OS you're using. Searching for the error message, it seems to involve +> Mac OS X. + +> For example: +>> The error "strange closure type" indicates some kind of memory corruption, which can have many different causes, from bugs in the GC to hardware failures. +> +> You said that you'd been using git-annex built with that version of GHC +> successfully before. Perhaps you could use `git bisect` to see if you can +> identify a point in git-annex's history where this started happening? +> Since you can reproduce the problem by just running git-annex-shell at +> the command line with the right parameters, it should be easy to bisect it. +> +> Probably your best bet will be changing to a different version or build of +> GHC.. --[[Joey]] From 38598e6f234ba44d997e59ac57e5436063eb1fd0 Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sun, 3 Apr 2011 04:45:50 +0000 Subject: [PATCH 1388/8313] Added a comment: Bisect it is, then --- ...comment_1_1c19e716069911f17bbebd196d9e4b61._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment new file mode 100644 index 0000000000..98f0adc3db --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Bisect it is, then" + date="2011-04-03T04:45:49Z" + content=""" +Hm, if path's ok, guess there's no way around git-bisect indeed. Wonder if there's some kind of ccache for haskell... + +OS is linux, amd64 on \"host1\" and i386 on \"host2\" where git-annex-shell is crashing. +I'll try to come up with a commit, thanks for clarifications. +"""]] From 702ab4b008cae422bf737106a4f1b8eed105717b Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sun, 3 Apr 2011 06:22:16 +0000 Subject: [PATCH 1389/8313] Added a comment: Bisect results --- ..._a4d66f29d257044e548313e014ca3dc3._comment | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment new file mode 100644 index 0000000000..fb36581912 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment @@ -0,0 +1,66 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Bisect results" + date="2011-04-03T06:22:15Z" + content=""" +Completed git-bisect twice, getting roughly the same results: + + 828a84ba3341d4b7a84292d8b9002a8095dd2382 is the first bad commit + commit 828a84ba3341d4b7a84292d8b9002a8095dd2382 + Author: Joey Hess + Date: Sat Mar 19 14:33:24 2011 -0400 + + Add version command to show git-annex version as well as repository version information. + + :040000 040000 ed849b7b6e9b177d6887ecebd6a0f146357824f3 1c98699dfd3fc3a3e2ce6b55150c4ef917de96e9 M Command + :100644 100644 b9c22bdfb403b0bdb1999411ccfd34e934f45f5c adf07e5b3e6260b296c982a01a73116b8a9a023c M GitAnnex.hs + :100644 100644 76dd156f83f3d757e1c20c80d689d24d0c533e16 d201cc73edb31f833b6d00edcbe4cf3f48eaecb0 M Upgrade.hs + :100644 100644 5f414e93b84589473af5b093381694090c278e50 d4a58d77a29a6a02daf13cec0df08b5aab74f65e M Version.hs + :100644 100644 f5c2956488a7afafd20374873d79579fb09b1677 f8cd577e992d38c7ec1438ce5c141eb0eb410243 M configure.hs + :040000 040000 f9b7295e997c0a5b1dda352f151417564458bd6e a30008475c1889f4fd8d60d4d9c982563380a692 M debian + :040000 040000 9d87a5d8b9b9fe7b722df303252ffd5760d66f75 08834f61a10d36651b3cdcc38389f45991acdf5e M doc + +contents of final refs/bisect: + + bad (828a84ba3341d4b7a84292d8b9002a8095dd2382) + good-33cb114be5135ce02671d8ce80440d40e97ca824 + good-942480c47f69e13cf053b8f50c98c2ce4eaa256e + good-ca48255495e1b8ef4bda5f7f019c482d2a59b431 + +\"roughly\" because second bisect gave two commits as a result, failing to build one of them (missing .o file on link, guess it's because of -j4 and bad deps in that version's build system): + + There are only 'skip'ped commits left to test. + The first bad commit could be any of: + 828a84ba3341d4b7a84292d8b9002a8095dd2382 + 5022a69e45a073046a2b14b6a4e798910c920ee9 + We cannot bisect more! + +Also noticed that \"git-annex-shell ...\" command succeeds if ran as root user, while failing from unprivileged one. +There are no permission/access errors in \"strace -f git-annex-shell ...\", so I guess it could be some bug in the GHC indeed. + +JIC, logged a whole second bisect operation. +Resulting log: [http://fraggod.net/static/share/git-annex-bisect.log](http://fraggod.net/static/share/git-annex-bisect.log) + +Bisect script I've used (git-annex-shell dies with error code 134 - SIGABRT on GHC error): + + res= + while true; do + if [[ -n \"$res\" ]]; then + cd /var/tmp/paludis/build/dev-scm-git-annex-scm.bak/work/git-annex-scm + echo \"---=== BISECT ($res) ===---\"; git bisect \"$res\" 2>&1; echo '---=== /BISECT ===---' + cd + rm -Rf /var/tmp/paludis/build/dev-scm-git-annex-scm + cp -a --reflink=auto /var/tmp/paludis/build/dev-scm-git-annex-scm{.bak,} + chown -R paludisbuild: /var/tmp/paludis/build/dev-scm-git-annex-scm + fi + res= + cave resolve -zx1 git-annex --skip-until-phase configure || res=skip + if [[ -z \"$res\" ]]; then + cd /remote/path + sudo -u user git-annex-shell 'sendkey' '/remote/path' 'SHA1-s6654080--abd8edec20648ade69351d68ae1c64c8074a6f0b' '--' rsync --server --sender -vpe.Lsf --inplace . '' + if [[ $? -eq 134 ]]; then res=bad; else res=good; fi + cd + fi + done 2>&1 | tee ~/git-annex-bisect.log + +"""]] From 263524e2c45e985eb1f8c75caa4f74218c164d7d Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Sun, 3 Apr 2011 06:57:02 +0000 Subject: [PATCH 1390/8313] Added a comment --- ..._f5f1081eb18143383b2fb1f57d8640f5._comment | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment new file mode 100644 index 0000000000..491b537862 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment @@ -0,0 +1,38 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="comment 3" + date="2011-04-03T06:57:02Z" + content=""" +Repeated bisect with -j1, just to be sure it's not a random error, and it gave me 828a84ba3341d4b7a84292d8b9002a8095dd2382 again. +Guess I'll look through the changes there a bit later and try to revert these until it works. + +Not sure if it's repeatable by anyone but me (and hence worth fixing), but here's a bit more of info about the system: + + Exherbo linux + Linux sacrilege 2.6.38.2-fg.roam #4 SMP PREEMPT Mon Mar 28 21:08:47 YEKST 2011 i686 GNU/Linux + + dev-lang/ghc-7.0.2:7.0.2::installed + dev-haskell/HUnit-1.2.2.3:1.2.2.3::installed + dev-haskell/MissingH-1.1.0.3:1.1.0.3::installed + dev-haskell/QuickCheck-2.4.0.1:2.4.0.1::installed + dev-haskell/array-0.3.0.2:0.3.0.2::installed + dev-haskell/bytestring-0.9.1.7:0.9.1.7::installed + dev-haskell/containers-0.4.0.0:0.4.0.0::installed + dev-haskell/extensible-exceptions-0.1.1.2:0.1.1.2::installed + dev-haskell/filepath-1.2.0.0:1.2.0.0::installed + dev-haskell/hslogger-1.1.3:0::installed + dev-haskell/mtl-2.0.1.0:2.0.1.0::installed + dev-haskell/network-2.3.0.1:2.3.0.1::installed + dev-haskell/old-locale-1.0.0.2:1.0.0.2::installed + dev-haskell/parsec-3.1.0:3.1.0::installed + dev-haskell/pcre-light-0.4:0::installed + dev-haskell/regex-base-0.93.2:0.93.2::installed + dev-haskell/regex-compat-0.93.1:0.93.1::installed + dev-haskell/regex-posix-0.94.4:0.94.4::installed + dev-haskell/syb-0.3:0.3::installed + dev-haskell/transformers-0.2.2.0:0.2.2.0::installed + dev-haskell/utf8-string-0.3.6:0.3.6::installed + +(some stuff listed here as ::installed, but contains no files, since these packages detect whether ghc-7.0.2 already comes with the same/newer package version) + +"""]] From ed40974ed9e388639be0352946f3cab38c621552 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 07:43:37 +0000 Subject: [PATCH 1391/8313] Added a comment --- .../comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment new file mode 100644 index 0000000000..b4a5a72d01 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 13" + date="2011-04-03T07:43:37Z" + content=""" +Ok, thanks for the fix. It seems the fix isn't too reliable with my repos, I get different numbers of \"** No known copies of...\" in the various cloned repos that I have. After all the \"messing\" that I have done to my repos I think git-annex has gotten very confused. I will just leave things as they are and let git-annex slowly migrate over to the new format or re-clone from a linux source and see how things go. I will report back on this issue in abit after I use it more to see. +"""]] From d204b882afc310886b8d490163796a4c392f30ad Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 08:23:43 +0000 Subject: [PATCH 1392/8313] --- ..._-f_REMOTE_._doesn__39__t_work_as_expected.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn diff --git a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn new file mode 100644 index 0000000000..b57471e5f8 --- /dev/null +++ b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn @@ -0,0 +1,14 @@ +I was testing out the fix/workaround for [[git-annex directory hashing problems on osx]] and I tried using the short forms of some of the commands i.e. + + git annex copy -f externalusb . + +which gives me + + git-annex: user error (option `-f' is ambiguous; could be one of: + -f --force allow actions that may lose annexed data + -f REMOTE --from=REMOTE specify from where to transfer content + + +I would have expected that since *--to* is the same as *-t* and *--from* is the same as *-f* as the in program documentation suggests. But *-f* clashes with the force command, I would suggest that the short form of *--force* be changed to *-F* and possibly rename the *Fast* commands to *Quick* and use *-Q* as the short form of the *Quick* operations. I didn't try the *-f* option with the move command, but it probably suffers from the same issue. It's probably better to avoid clashing short forms of command options. + +I guess this issue is just a documentation issue and a minor interface change if needed and not a bug of git-annex, but a quirk. From 529393a9e2f2954cc549204f501f4425c27e820f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 08:24:18 +0000 Subject: [PATCH 1393/8313] Added a comment --- ...omment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment new file mode 100644 index 0000000000..b92c3ab4ab --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 14" + date="2011-04-03T08:24:17Z" + content=""" +I meant to say in it wasn't reliable when I was following the instructions for \"Comment 12\". I did find that just doing a \"git annex copy -t externalusb .\" then a \"git annex drop .\" from the root of my cloned and \"none trusted\" annexed repos to be more reliable, it just means I temporarily need a load of space to get myself out of my earlier mess. + +On testing this bug fix, I found a minor behavioural issue with [[git annex copy -f REMOTE . doesn't work as expected]] +"""]] From c737e86b62bc4afcb9be105b0304dc50fe88783b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 08:55:18 +0000 Subject: [PATCH 1394/8313] Added a comment --- .../comment_2_40920b88537b7715395808d8aa94bf03._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment diff --git a/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment new file mode 100644 index 0000000000..215619043a --- /dev/null +++ b/doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T08:55:18Z" + content=""" +Given that the softlinks contain all needed information (if the object exists, locally), an emergency way to get files \"out\" of git-annex would be nice. I am aware that one can script it, but a canonical way is always better, especially when things go south. +"""]] From 4881a1833a110137f1320643ac01b33af6ea5e4c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 08:56:48 +0000 Subject: [PATCH 1395/8313] Added a comment --- .../comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment diff --git a/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment b/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment new file mode 100644 index 0000000000..28305d3ac8 --- /dev/null +++ b/doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T08:56:48Z" + content=""" +Pity. Mark as done/upstream (or similar) for house-keeping? +"""]] From e8796f439d4d9d043c87479d1c2680d92c0d1fc8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 08:58:20 +0000 Subject: [PATCH 1396/8313] --- doc/walkthrough/recover_data_from_lost+found.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/walkthrough/recover_data_from_lost+found.mdwn index f101b3caf6..48ef2a1d73 100644 --- a/doc/walkthrough/recover_data_from_lost+found.mdwn +++ b/doc/walkthrough/recover_data_from_lost+found.mdwn @@ -14,6 +14,6 @@ First, check out the git repository again. Then, in the new checkout: The way that works is that when git-annex adds the same content that was in the repository before, all the old links to that content start working -again. This works particularly well if the SHA1 backend is used, but even +again. This works particularly well if the SHA* backends are used, but even with the default backend it will work pretty well, as long as fsck preserved the modification time of the files. From 0e0341ff02bcfa18e250a6124c8d78c1cc3f07dd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 09:00:17 +0000 Subject: [PATCH 1397/8313] Added a comment --- ...comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment new file mode 100644 index 0000000000..c9b74d98f0 --- /dev/null +++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-03T09:00:17Z" + content=""" +I did not. Thanks :) + +This still means that you can't re-inject a new version of a file unless you have the old one if you are using a SHA* backend, but that might be a corner case anyway. +"""]] From 51eb3db90a2011c736729a645c0e7191f4a44573 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 09:03:22 +0000 Subject: [PATCH 1398/8313] Added a comment --- .../comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment new file mode 100644 index 0000000000..ec0f88d13c --- /dev/null +++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-03T09:03:22Z" + content=""" +Thanks a lot. I tried various howtos around the net, but none of them worked; yours did. (I tried it in one of the copies of the broken repo which I keep around for obvious reasons). +"""]] From 25fb248277a7d970e301806d2bcbec71e987b20b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 12:39:50 +0000 Subject: [PATCH 1399/8313] --- ...getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn new file mode 100644 index 0000000000..f5d5a66bfc --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn @@ -0,0 +1,9 @@ +I'm not sure if this is my stupidity or if it's a bug, but + + git annex copy --force --to REMOTE . + +just zip's through really quickly and doesn't actually force a copy to a remote location. This is just following up on the [[git-annex directory hashing problems on osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a + + git annex copy --force --from REMOTE . + +to pull down files from a remote. From 7ee30bd1ac68c47c009c703ce967f1362af96302 Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 3 Apr 2011 15:35:53 +0000 Subject: [PATCH 1400/8313] Added a comment --- ...2_e14e84b770305893f2fc6e4938359f47._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment new file mode 100644 index 0000000000..4fc9647e81 --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 2" + date="2011-04-03T15:35:52Z" + content=""" +'git add .git-annex' didn't do anything. That's when I noticed that this +repository is on a case-insensitive HFS+ file system. + +So, if I get this right it's not a new bug, but similar to this situation: +[[git-annex_directory_hashing_problems_on_osx]] + +Assuming that it was the file system's fault, I went ahead and upgraded yet +another clone. That one (on an ext3 file system) had neither staged changes +nor left-over untracked files. Everything seems to just have fallen right into +place. Is that possible or still weird? + +"""]] From 5f3385ce3b8aca1c06981a6c9e0684232ad59be6 Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 3 Apr 2011 15:41:01 +0000 Subject: [PATCH 1401/8313] Added a comment --- ...15_37f1d669c1fa53ee371f781c7bb820ae._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment new file mode 100644 index 0000000000..d722d546a3 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 15" + date="2011-04-03T15:41:00Z" + content=""" +I also ran into problems on a case-insensitive HFS+ file system, it seems. I +tried following the instructions in comment 12: + + 1. Remove everything in .git-annex besides uuid.log and trust.log + 2. git annex fsck --fast + 3. Commit + +However, I still see upper and lower case directories in .git-annex. Did I +misunderstand that they should all be lower case now? + +"""]] From 862df8aa214da146fd93ddff69dfd3aeb6e4d7b6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:02:33 +0000 Subject: [PATCH 1402/8313] Added a comment --- ..._8a4ab1af59098f4950726cf53636c2b3._comment | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment new file mode 100644 index 0000000000..97eab78c91 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 16" + date="2011-04-03T16:02:33Z" + content=""" +I think the correct steps should be, make a backup first :) then ... + +1. git pull # update your clone, and commit everything so you don't lose anything +2. git annex fsck --fast # check the repo first, just in case +3. rm -rf .git-annex/?? # remove the old metadata +4. git annex fsck --fast # get git annex to regenerate it all +5. push your changes out to your other repos, you will need to make sure git-annex is updated everywhere if there are remotes in your setup. + +I eventually migrated all of my own annex'd repos and I no longer have the old hashed directories but the new ones in the form + + .git/annex/aaa/bbb/foo.log + +I did lose some tracking information but not data (as far as I can see for now), but that was quickly fixed by pushing and pulling to my bare repo which tracks most of my data. + +I also found that it worked a bit more reliably for me on the copies of repos that were located on case sensitive filesystems, but I guess that was expected. +"""]] From 58493c064ba49158c7209adddea3558e4fa3ddd0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:05:39 +0000 Subject: [PATCH 1403/8313] Added a comment --- .../comment_3_ec04e306c96fd20ab912aea54a8340aa._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment new file mode 100644 index 0000000000..99095c1569 --- /dev/null +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-03T16:05:39Z" + content=""" +Yes you seem to have come across the same bug that I had initially reported :P +"""]] From c90344908dfdab933e92d2008784298087aaaaa5 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:06:34 +0000 Subject: [PATCH 1404/8313] Added a comment --- ...comment_4_b1f818b85c3540591c48e7ba8560d070._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment new file mode 100644 index 0000000000..45d3d8bac4 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-03T16:06:34Z" + content=""" +Nice work on the bisection. It's obviously a compiler bug. Having two test cases that differ in only as trivial and innocous a commit as 828a84ba3341d4b7a84292d8b9002a8095dd2382 might help a GHC developer track it down. + +We should probably forward this as a GHC bug. I hope you can find a different version or build of GHC to build git-annex with. +"""]] From 623d612ebca0d4a231652918155c794afa43162b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:13:02 +0000 Subject: [PATCH 1405/8313] --- doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn diff --git a/doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn b/doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn new file mode 100644 index 0000000000..6299899e4f --- /dev/null +++ b/doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn @@ -0,0 +1,5 @@ +Given that git/config will have information on remotes and maybe costs, it might be a good idea to do a simple round robin selection of remotes to download files where the costs are the same. + +This of course assumes that we like the idea of "parallel" launching and running of curl/rsync processes... + +This wish item is probably only useful for the paranoid people who store more than 1 copy of their data. From 216ad1a4d3434306cf3338217fbcf5cbe6e1c8d6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 12:18:38 -0400 Subject: [PATCH 1406/8313] Clear up short option confusion between --from and --force (-f is now --from, and there is no short option for --force). --- Options.hs | 2 +- debian/changelog | 2 ++ ..._annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Options.hs b/Options.hs index 10c3714e41..31b73e25a8 100644 --- a/Options.hs +++ b/Options.hs @@ -20,7 +20,7 @@ type Option = OptDescr (Annex ()) commonOptions :: [Option] commonOptions = - [ Option ['f'] ["force"] (NoArg (setforce True)) + [ Option [] ["force"] (NoArg (setforce True)) "allow actions that may lose annexed data" , Option ['F'] ["fast"] (NoArg (setfast True)) "avoid slow operations" diff --git a/debian/changelog b/debian/changelog index e504bd8f60..71ea599cd9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Unused files on remotes, particulary special remotes, can now be identified and dropped, by using "--from remote" with git annex unused and git annex dropunused. + * Clear up short option confusion between --from and --force (-f is now + --from, and there is no short option for --force). -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn index b57471e5f8..3bda451499 100644 --- a/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn +++ b/doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn @@ -12,3 +12,7 @@ which gives me I would have expected that since *--to* is the same as *-t* and *--from* is the same as *-f* as the in program documentation suggests. But *-f* clashes with the force command, I would suggest that the short form of *--force* be changed to *-F* and possibly rename the *Fast* commands to *Quick* and use *-Q* as the short form of the *Quick* operations. I didn't try the *-f* option with the move command, but it probably suffers from the same issue. It's probably better to avoid clashing short forms of command options. I guess this issue is just a documentation issue and a minor interface change if needed and not a bug of git-annex, but a quirk. + +> Yeah, -f needs to be from; -F was already --fast. I have made --force not +> have any short option abbreviation, I think it's entirely reasonable to +> avoid fat-fingering an option that can lose data. [[done]] --[[Joey]] From c93ce86b008ef7d93180b64ded12443e6a0c115f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 12:21:41 -0400 Subject: [PATCH 1407/8313] close --- doc/bugs/Displayed_copy_speed_is_wrong.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn index c1209c73c0..cf3b31cf48 100644 --- a/doc/bugs/Displayed_copy_speed_is_wrong.mdwn +++ b/doc/bugs/Displayed_copy_speed_is_wrong.mdwn @@ -3,3 +3,6 @@ When copying data to my remote, I regularly see speeds in excess of 100 MB/s on 2073939 100% 176.96MB/s 0:00:00 (xfer#1, to-check=0/1) This is definitely not correct. + +> Closing, as rsync does this to show you when it's making your life +> faster than it would be w/o rsync. [[done]] --[[Joey]] From ee313074ed65145692acff1402e6262c2ea4b437 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 12:27:13 -0400 Subject: [PATCH 1408/8313] close --- ...de_left_untracked_.git-annex__47____42___directories.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn index ab8b255a0f..7fdbc3ca4e 100644 --- a/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn +++ b/doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn @@ -9,3 +9,8 @@ wrong. At the time I had git-annex version 0.20110329 and I've been using the SHA1 backend since version 1. + +> Yes, I agree with Jimmy, it's the same bug. So I'll be closing this one. +> Please keep us informed how the workaround committed to git-annex +> yesterday for the case insensativity issue works out. [[dup|done]] +> --[[Joey]] From 5c2e08c3f6211720a5be0e288744fd13f8f16d36 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:39:35 +0000 Subject: [PATCH 1409/8313] Added a comment --- .../comment_1_460335b0e59ad03871c524f1fe812357._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment diff --git a/doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment b/doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment new file mode 100644 index 0000000000..6a5fd3d530 --- /dev/null +++ b/doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T16:39:35Z" + content=""" +I dunno about parrallel downloads -- eek! -- but there is at least room for improvement of what \"git annex get\" does when there are multiple remotes that have a file, and the one it decides to use is not available, or very slow, or whatever. +"""]] From 199e897b15aef307073d49199b561344f17a62e2 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:49:01 +0000 Subject: [PATCH 1410/8313] Added a comment --- ...ent_1_3deb2c31cad37a49896f00d600253ee3._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment new file mode 100644 index 0000000000..d2692f26f0 --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-03T16:49:01Z" + content=""" +How remote is REMOTE? If it's a directory on the same computer, then git-annex copy --to is actually quickly checking that each file is present on the remote, and when it is, skipping copying it again. + +If the remote is ssh, git-annex copy talks to the remote to see if it has the file. This makes copy --to slow, as Rich [[complained_before|forum/batch_check_on_remote_when_using_copy]]. :) + +So, copy --to does not trust location tracking information (unless --fast is specified), which means that it should be doing exactly what you want it to do in your situation -- transferring every file that is really not present in the destination repository already. + +Neither does copy --from, by the way. It always checks if each file is present in the current repository's annex before trying to download it. +"""]] From 3c0835e542070da7dd4b401e54d38a4c4961639b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 16:53:51 +0000 Subject: [PATCH 1411/8313] Added a comment --- .../comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment new file mode 100644 index 0000000000..f7feac67cf --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 17" + date="2011-04-03T16:53:51Z" + content=""" +@gernot step 0 is to upgrade git-annex to current git, on all systems where you use it, in case that wasn't clear. + +"""]] From 54b903cb54adb6b0cac4308c5974342f0bea1e67 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 16:59:48 +0000 Subject: [PATCH 1412/8313] Added a comment --- ...mment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment new file mode 100644 index 0000000000..1079303197 --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2011-04-03T16:59:47Z" + content=""" +Remote as in \"another physical machine\". I assumed that + + git annex copy --force --to REMOTE . + +would have not trusted the contents in the current directory (or the remote that is being copied to) and then just go off and re-download/upload all the files and overwrite what is already there. I expected the combination of *--force* and copy *--to* that it would not bother to check if the files are there or not and just copy it regardless of the outcome. +"""]] From 8345a90e66474b9458c9d11d4b66ef01bdf1cfdd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sun, 3 Apr 2011 17:12:36 +0000 Subject: [PATCH 1413/8313] Added a comment --- .../comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment new file mode 100644 index 0000000000..c3df214988 --- /dev/null +++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-03T17:12:35Z" + content=""" +On second thought maybe the current behaviour is better than what I am suggesting that the force command should do. I guess it's better to be safe than sorry. +"""]] From 83acc9ba52ecba85180355a8c08311bd4826ed0f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:34:00 -0400 Subject: [PATCH 1414/8313] encryption design document --- ...e_same_key_for_encryption_and_hashing.mdwn | 3 + doc/design.mdwn | 4 + doc/design/encryption.mdwn | 108 ++++++++++++++++++ doc/special_remotes/Amazon_S3.mdwn | 17 +-- 4 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 doc/design.mdwn create mode 100644 doc/design/encryption.mdwn diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn index 0ec66652e2..1980a8f444 100644 --- a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -3,3 +3,6 @@ While using HMAC instead of "plain" hash functions is inherently more secure, it Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice? Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing. + +> See [[design/encryption]]. I don't think this bug needs to be kept +> open. [[done]] --[[Joey]] diff --git a/doc/design.mdwn b/doc/design.mdwn new file mode 100644 index 0000000000..dc66d5c80a --- /dev/null +++ b/doc/design.mdwn @@ -0,0 +1,4 @@ +git-annex's high-level design is mostly inherent in the data that it +stores in git, and alongside git. See [[internals]] for details. + +See [[encryption]] for design of encryption elements. diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn new file mode 100644 index 0000000000..003336dd3e --- /dev/null +++ b/doc/design/encryption.mdwn @@ -0,0 +1,108 @@ +git-annex mostly does not use encryption. Anyone with access to a git +repository can see all the filenames in it, its history, and can access +any annexed file contents. + +Encryption is needed when using [[special_remotes]] like Amazon S3, where +file content is sent to an untrusted party who does not have access to the +git repository. + +Such an encrypted remote uses strong encryption on the contents of files, +as well as the filenames. The size of the encrypted files, and access +patterns of the data, should be the only clues to what type of is stored in +such a remote. + +## encryption backends + +It makes sense to support multiple encryption backends. So, there +should be a way to tell what backend is responsible for a given filename +in an encrypted remote. (And since special remotes can also store files +unencrypted, differentiate from those as well.) + +At a high level, an encryption backend needs to support these operations: + +* Given a key/value backend key, produce and return an encrypted key. + + The same naming scheme git-annex uses for keys in regular key/value + [[backends]] can be used. So a filename for a key might be + "GPG-s12345--armoureddatahere" + +* Given a streaming source of file content, encrypt it, and send it in + a stream to an action that consumes the encrypted content. + +* Given a streaming source of encrypted content, decrypt it, and send + it in a stream to an anction that consumes the decrypted content. + +* Initialize itself. + +* Clean up. + +* Configure an encryption key to use. + +The rest of this page will describe a single encryption backend using GPG. +Probably only one will be needed, but who knows? Maybe that backend will +turn out badly designed, or some other encryptor needed. Designing +with more than one encryption backend in mind helps future-proofing. + +## encryption key management + +[[!template id=note text=""" +The basis of this scheme was originally developed by Lars Wirzenius et al +[for Obnam](http://braawi.org/obnam/encryption/). +"""]] + +Data is encrypted by gpg, using a symmetric cipher. The passphrase of the +cipher is itself checked into your git repository, encrypted using one or +more gpg public keys. This scheme allows new gpg private keys to be given +access to content that has already been stored in the remote. + +Different encrypted remotes need to be able to each use different ciphers. +There does not seem to be a benefit to allowing multiple cipers to be +used within a single remote, and it would add a lot of complexity. +Instead, if you want a new cipher, create a new S3 bucket, or whatever. +There does not seem to be much benefit to using the same cipher for +two different enrypted remotes. + +So, the encrypted cipher could just be stored with the rest of a remote's +configuration in `.git-annex/remotes.log` (see [[internals]]). When `git +annex intiremote` makes a remote, it can generate a random symmetric +cipher, and encrypt it with the specified gpg key. To allow another gpg +public key access, update the encrypted cipher to be encrypted to both gpg +keys. + +## filename enumeration + +If the names of files are encrypted, this makes it harder for +git-annex (let alone untrusted third parties!) to get a list +of the files that are stored on a given enrypted remote. This has been +a concern, and it has been considered to use a hash like HMAC, rather +than gpg encrypting filenames, to make it easier. (For git-annex, but +possibly also for attackers!) But, does git-annex really ever need to do +such an enumeration? + +Apparently not. `git annex unused --from remote` can now check for +unused data that is stored on a remote, and it does so based only on +location log data for the remote. This assumes that the location log is +kept accurately. + +What about `git annex fsck --from remote`? Such a command should be able to, +for each file in the repository, contact the encrypted remote to check +if it has the file. This can be done without enumeration, although it will +mean running gpg once per file fscked, to get the encrypted filename. + +### risks + +A risk of this scheme is that, once the symmetric cipher has been obtained, it +allows full access to all the encrypted content. This scheme does not allow +revoking a given gpg key access to the cipher, since anyone with such a key +could have already decrypted the cipher and stored a copy. + +If git-annex stores the decrypted symmetric cipher in memory, then there +is a risk that it could be intercepted from there by an attacker. Gpg +amelorates these type of risks by using locked memory. + +This design does not support obfuscating the size of files by chunking +them, as that would have added a lot of complexity, for dubious benefits. +If the untrusted party running the encrypted remote wants to know file sizes, +they could correlate chunks that are accessed together. Enctypting data +changes the original file size enough to avoid it being used as a direct +fingerprint at least. diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index 384110d1df..2cf23187d1 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -37,19 +37,4 @@ only be used for public data. ** Encryption is not yet supported. ** -When encryption is enabled, all files stored in the bucket are -encrypted with gpg. Additionally, the filenames themselves are encrypted -(using HMAC). The size of the encrypted files, and -access patterns of the data, should be the only clues to what type of -data you are storing in S3. - -[[!template id=note text=""" -This scheme was originally developed by Lars Wirzenius et al -[for Obnam](http://braawi.org/obnam/encryption/). -"""]] -The data stored in S3 is encrypted by gpg with a symmetric cipher. The -passphrase of the cipher is itself checked into your git repository, -encrypted using one or more gpg public keys. This scheme allows new private -keys to be given access to a bucket's content, after the bucket is created -and is in use. The symmetric cipher is also hashed together with filenames -used in the bucket, in order to obfuscate the filenames. +See [[design/encryption]]. From dbe41e667bba1096de8d60b75f932efcbf674f85 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:43:38 -0400 Subject: [PATCH 1415/8313] update --- doc/design/encryption.mdwn | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 003336dd3e..43d8119e3b 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -20,6 +20,13 @@ unencrypted, differentiate from those as well.) At a high level, an encryption backend needs to support these operations: +* Create a new encrypted cipher, or update the cipher. Some input + parameters will specifiy things like the gpg public keys that + can access the cipher. + +* Initialize an instance of the encryption backend, that will use a + specified encrypted cipher. + * Given a key/value backend key, produce and return an encrypted key. The same naming scheme git-annex uses for keys in regular key/value @@ -32,8 +39,6 @@ At a high level, an encryption backend needs to support these operations: * Given a streaming source of encrypted content, decrypt it, and send it in a stream to an anction that consumes the decrypted content. -* Initialize itself. - * Clean up. * Configure an encryption key to use. From 8c9d9eb8af88035a05378214e86b679fce091acf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:47:43 -0400 Subject: [PATCH 1416/8313] update --- doc/design/encryption.mdwn | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 43d8119e3b..c9b1bdb5dc 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -11,6 +11,8 @@ as well as the filenames. The size of the encrypted files, and access patterns of the data, should be the only clues to what type of is stored in such a remote. +[[!toc]] + ## encryption backends It makes sense to support multiple encryption backends. So, there @@ -94,7 +96,7 @@ for each file in the repository, contact the encrypted remote to check if it has the file. This can be done without enumeration, although it will mean running gpg once per file fscked, to get the encrypted filename. -### risks +## risks A risk of this scheme is that, once the symmetric cipher has been obtained, it allows full access to all the encrypted content. This scheme does not allow @@ -108,6 +110,6 @@ amelorates these type of risks by using locked memory. This design does not support obfuscating the size of files by chunking them, as that would have added a lot of complexity, for dubious benefits. If the untrusted party running the encrypted remote wants to know file sizes, -they could correlate chunks that are accessed together. Enctypting data +they could correlate chunks that are accessed together. Encrypting data changes the original file size enough to avoid it being used as a direct fingerprint at least. From 0d1f2023340dd30e81bc003144a37e0fe03c333b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 14:53:12 -0400 Subject: [PATCH 1417/8313] update --- doc/design/encryption.mdwn | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index c9b1bdb5dc..72a7ad286e 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -78,13 +78,10 @@ keys. ## filename enumeration -If the names of files are encrypted, this makes it harder for -git-annex (let alone untrusted third parties!) to get a list -of the files that are stored on a given enrypted remote. This has been -a concern, and it has been considered to use a hash like HMAC, rather -than gpg encrypting filenames, to make it easier. (For git-annex, but -possibly also for attackers!) But, does git-annex really ever need to do -such an enumeration? +If the names of files are encrypted or securely hashed, or whatever is +chosen, this makes it harder for git-annex (let alone untrusted third parties!) +to get a list of the files that are stored on a given enrypted remote. +But, does git-annex really ever need to do such an enumeration? Apparently not. `git annex unused --from remote` can now check for unused data that is stored on a remote, and it does so based only on From 9c4285406b465d324f7b48d81e48614c27bc48ff Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 19:40:13 +0000 Subject: [PATCH 1418/8313] Added link to design --- doc/index.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index cb2f485c92..ea0c830bc6 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -10,6 +10,7 @@ To get a feel for it, see the [[walkthrough]]. * [[bugs]] * [[todo]] * [[forum]] +* [[design]] * [[comments]] * [[contact]] * Flattr this From fef63846b9e990c2b3d9c7f3e88af5a05fa3b543 Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 3 Apr 2011 19:46:17 +0000 Subject: [PATCH 1419/8313] Added a comment --- ...nt_18_db64c91dd1322a0ab168190686db494f._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment new file mode 100644 index 0000000000..550558ec16 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="gernot" + ip="213.168.117.192" + subject="comment 18" + date="2011-04-03T19:46:16Z" + content=""" +Joey, sorry, I got it wrong. I thought upgrading git didn't help and you +adjusted things in git-annex instead. + +Anyway, can I get around upgrading on all hosts by reformatting the drive to +case-sensitive HFS+? Or will I have to upgrade git (currently version 1.7.2.5) +eventually anyway? + +"""]] From 261b1e6310885fcad3b50c8cd7240ccdc5ed54a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 15:51:24 -0400 Subject: [PATCH 1420/8313] update --- doc/design/encryption.mdwn | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 72a7ad286e..0242aabebb 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -102,8 +102,11 @@ could have already decrypted the cipher and stored a copy. If git-annex stores the decrypted symmetric cipher in memory, then there is a risk that it could be intercepted from there by an attacker. Gpg -amelorates these type of risks by using locked memory. - +amelorates these type of risks by using locked memory. For git-annex, note +that an attacker with local machine access can tell at least all the +filenames and metadata of files stored in the encrypted remote anyway, +and can access whatever content is stored locally. + This design does not support obfuscating the size of files by chunking them, as that would have added a lot of complexity, for dubious benefits. If the untrusted party running the encrypted remote wants to know file sizes, From 3e0119a89f8968128f5436a68171279e5c6e2295 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 3 Apr 2011 15:53:40 -0400 Subject: [PATCH 1421/8313] move --- doc/index.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index ea0c830bc6..b5880823e1 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -10,7 +10,6 @@ To get a feel for it, see the [[walkthrough]]. * [[bugs]] * [[todo]] * [[forum]] -* [[design]] * [[comments]] * [[contact]] * Flattr this @@ -46,6 +45,7 @@ files with git. * [[key-value backends|backends]] for data storage * [[internals]] * [[bare_repositories]] +* [[design]] * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. From 6fd8efbc3cc754f574c022f325e8a442fe5cb02c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sun, 3 Apr 2011 19:53:44 +0000 Subject: [PATCH 1422/8313] Added a comment --- .../comment_19_ff555c271637af065203ca99c9eeaf89._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment new file mode 100644 index 0000000000..2676b35897 --- /dev/null +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 19" + date="2011-04-03T19:53:44Z" + content=""" +Git does not need to be upgraded. Git-annex needs to be upgraded to git rev 616e6f8a840ef4d99632d12a2e7ea15c3cfb1805 or newer, on all machines. +"""]] From 218c58f3c9d08e981a4dbade8aa9c9acc0facaf1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 20:03:14 +0000 Subject: [PATCH 1423/8313] Added a comment --- ...mment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment diff --git a/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment b/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment new file mode 100644 index 0000000000..f2ecc46d0a --- /dev/null +++ b/doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-03T20:03:14Z" + content=""" +New encryption keys could be used for different directories/files/patterns/times/whatever. One could then encrypt this new key for the public keys of other people/machines and push them out along with the actual data. This would allow some level of access restriction or future revocation. git-annex would need to keep track of which files can be decrypted with which keys. I am undecided if that information needs to be encrypted or not. + +Encrypted object files should be checksummed in encrypted form so that it's possible to verify integrity without knowing any keys. Same goes for encrypted keys, etc. + +Chunking files in this context seems like needless overkill. This might make sense to store a DVD image on CDs or similar, at some point. But not for encryption, imo. Coming up with sane chunk sizes for all use cases is literally impossible and as you pointed out, correlation by the remote admin is trivial. +"""]] From b73accb4831adab09d4059d05e68b40fac7fa482 Mon Sep 17 00:00:00 2001 From: "http://dieter-be.myopenid.com/" Date: Sun, 3 Apr 2011 20:30:22 +0000 Subject: [PATCH 1424/8313] Added a comment: symlinks --- ...mment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment diff --git a/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment new file mode 100644 index 0000000000..3a223e1c7b --- /dev/null +++ b/doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://dieter-be.myopenid.com/" + nickname="dieter" + subject="symlinks" + date="2011-04-03T20:30:21Z" + content=""" +> (Sadly, it cannot create a symlink, as git still wants to write the file afterwards. +> So the nice current behavior of unavailable files being clearly missing due to dangling symlinks, would be lost when using smudge/clean filters. (Contact git developers to get an interface to do this?) + +Have you checked what the smudge filter sees when the input is a symlink? Because git supports tracking symlinks, so it should also support pushing symlinks through a smudge filter, right? +Either way: yes, contact the git devs, one can only ask and hope. And if you can demonstrate the awesomeness of git-annex they might get more 1interested :) +"""]] From 6ee8c9ecaa7b6a3567b3d225db041c3832b4f977 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 3 Apr 2011 22:59:46 +0000 Subject: [PATCH 1425/8313] --- ...t_--_same_as_get__44___but_for_defaults.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn new file mode 100644 index 0000000000..9cd56749e8 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn @@ -0,0 +1,17 @@ +I am running centralized git-annex exclusively. + +Similar to + + git annex get + +I'd like to have a + + git annex put + +which would put all files on the default remote(s). + +My main reason for not wanting to use copy --to is that I need to specify the remote's name in this case which makes writing a wrapper unnecessarily hard. Also, this would allow + + mr push + +to do the right thing all by itself. From 81ae895aeb054bc9efbad15732046b182cda9d19 Mon Sep 17 00:00:00 2001 From: fmarier Date: Mon, 4 Apr 2011 07:40:43 +0000 Subject: [PATCH 1426/8313] Added a comment: Exporting to a FAT filesystem? --- ...mment_1_04bcc4795d431e8cb32293aab29bbfe2._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment diff --git a/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment new file mode 100644 index 0000000000..510e449842 --- /dev/null +++ b/doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="fmarier" + ip="121.73.248.43" + subject="Exporting to a FAT filesystem?" + date="2011-04-04T07:40:41Z" + content=""" +I'm using git-annex to keep my music in sync between all of my different machines. What I'd love to be able to do is to also keep it in sync with my iRiver player. Unfortunately, the firmware, Rockbox, doesn't support ext3, so I'm stuck with a FAT filesystem. + +I can see how the design of git-annex makes it rather difficult to get rid of the symlinks, so how about taking a different approach: something like a \"git annex export DEST\" which would take a destination (not a git remote) and rsync the content over to there as regular files. + +Maybe \"git annex sync DEST\" or \"git annex rsync DEST\" would be better names if we want to convey the idea that the destination will be made to look like the source repo, including performing the necessary deletions. +"""]] From 19782240b9c493cff0121487a103a7bfc3d325b9 Mon Sep 17 00:00:00 2001 From: fmarier Date: Mon, 4 Apr 2011 07:43:08 +0000 Subject: [PATCH 1427/8313] --- doc/users/fmarier.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 doc/users/fmarier.mdwn diff --git a/doc/users/fmarier.mdwn b/doc/users/fmarier.mdwn new file mode 100644 index 0000000000..ecf3426978 --- /dev/null +++ b/doc/users/fmarier.mdwn @@ -0,0 +1,6 @@ +# François Marier + +Free Software and Debian Developer. Lead developer of [Libravatar](http://www.libravatar.org) + +* [Blog](http://feeding.cloud.geek.nz) +* [Identica](http://identi.ca/fmarier) / [Twitter](http://twitter.com/fmarier) From 8527643324102de36bb43785a14cf0a5020004e7 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 4 Apr 2011 10:19:02 +0000 Subject: [PATCH 1428/8313] --- ...or_bug:_errors_are_not_verbose_enough.mdwn | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn diff --git a/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn new file mode 100644 index 0000000000..8def2e8c3f --- /dev/null +++ b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn @@ -0,0 +1,24 @@ +Current: + + % git annex status + git-annex: unknown command + +Better: + + % git annex status + git-annex: status: unknown command + +Current: + + % git annex fsck + [...] + git-annex: 18 failed + +Better: + + % git annex fsck + [...] + git-annex: fsck: 18 failed + + +etc pp. From 27eaab346903aa26c7639014dd26520b8d8da95c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 4 Apr 2011 18:13:46 +0000 Subject: [PATCH 1429/8313] Added a comment --- ...comment_1_d5413c8acce308505e4e2bec82fb1261._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment new file mode 100644 index 0000000000..fe1d5520f4 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-04T18:13:46Z" + content=""" +This begs the question: What is the default remote? It's probably *not* the same repository that git's master branch is tracking (ie, origin/master). It seems there would have to be an annex.defaultremote setting. + +BTW, mr can easily be configured on a per-repo basis so that \"mr push\" copies to somewhere: `push = git push; git annex push wherever` +"""]] From 8528a39f77477dd96c71ce92b87395920c5b7e2b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Mon, 4 Apr 2011 18:20:45 +0000 Subject: [PATCH 1430/8313] Added a comment --- ...ent_2_bb4a97ebadb5c53809fc78431eabd7c8._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment diff --git a/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment new file mode 100644 index 0000000000..7618c9a7b6 --- /dev/null +++ b/doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-04T18:20:45Z" + content=""" +Hey @fmarier. Well, this bug report is closed because you can already get rid of the symlinks. Just put a bare git repo on your fat filesystem, and use git-annex copy --to/--from there. + +Now, that puts all the files that are on the device in .git/annex/objects/xx/yy/blah.mp3 -- how well rockbox would support that I don't know. And if it tries to modify or delete those files, git annex also can't help you manage those changes. + +Another recent option is the [[special_remotes/directory]] special remote type, which again uses \"xx/yy/blah.mp3\" and can't track changes made to the files. This could perhaps be extended in the direction you suggest, although trying to fit this into the special remote infrastructure might not be a good fit really. + +The most likely way this has to get dealt with is really by using [[todo/smudge]] filters, which would eliminate the symlinks and allow copying a non-bare git repo onto vfat. +"""]] From 9a8861c08f4856594952a4a68f7024d67f958187 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 4 Apr 2011 20:45:31 +0000 Subject: [PATCH 1431/8313] Added a comment --- ...mment_2_0aa227c85d34dfff4e94febca44abea8._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment new file mode 100644 index 0000000000..3090b575b7 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-04T20:45:30Z" + content=""" +In my case, the remotes are the same, but adding a new option could make sense. + +And while I can tell mr what to do explicitly, I would prefer if it did the right thing all by itself. Having to change configs in two separate places is less than ideal. + +I am not sure what you mean by `git annex push` as that does not exist. Did you mean copy? +"""]] From 708ae036819b9efe69e31cfacea78f8933779885 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 4 Apr 2011 23:15:05 +0000 Subject: [PATCH 1432/8313] --- ...stormning:_git_annex_push___38___pull.mdwn | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/bainstormning:_git_annex_push___38___pull.mdwn diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn b/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn new file mode 100644 index 0000000000..a2d320e352 --- /dev/null +++ b/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn @@ -0,0 +1,21 @@ +Wouldn't it make sense to offer + + git annex pull + +which would basically do + + git pull + git annex get + +and + + git annex push + +which would do + + git annex commit . + git annex put # (the proposed "send to default annex" command) + git commit -a -m "$HOST $(date +%F-%H-%M-%S)" # or similar + git push + +Resulting in commands that are totally analogous to git push & pull: Sync all data from/to a remote. From 5dea16889e42042a0841662d9e3b2c7be2ca4923 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" Date: Tue, 5 Apr 2011 04:37:52 +0000 Subject: [PATCH 1433/8313] --- doc/forum/Problems_with_large_numbers_of_files.mdwn | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files.mdwn diff --git a/doc/forum/Problems_with_large_numbers_of_files.mdwn b/doc/forum/Problems_with_large_numbers_of_files.mdwn new file mode 100644 index 0000000000..943b979d6e --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files.mdwn @@ -0,0 +1,8 @@ +I'm trying to use git-annex to archive scientific data. I'm often dealing with large numbers of files, sometimes 10k or more. When I try to git-annex add these files I get this error: + + +Stack space overflow: current size 8388608 bytes. +Use `+RTS -Ksize' to increase it. + + +This is with the latest version of git-annex and a current version of git on OS 10.6.7. After this error occurs, I am unable to un-annex the files and I'm forced to recover from a backup. From 43dac0b4c96e04144e125430d7bbabc80dc4b2d8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" Date: Tue, 5 Apr 2011 04:38:18 +0000 Subject: [PATCH 1434/8313] --- doc/forum/Problems_with_large_numbers_of_files.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/forum/Problems_with_large_numbers_of_files.mdwn b/doc/forum/Problems_with_large_numbers_of_files.mdwn index 943b979d6e..1dbddd3e28 100644 --- a/doc/forum/Problems_with_large_numbers_of_files.mdwn +++ b/doc/forum/Problems_with_large_numbers_of_files.mdwn @@ -1,8 +1,8 @@ I'm trying to use git-annex to archive scientific data. I'm often dealing with large numbers of files, sometimes 10k or more. When I try to git-annex add these files I get this error: -Stack space overflow: current size 8388608 bytes. -Use `+RTS -Ksize' to increase it. + Stack space overflow: current size 8388608 bytes. + Use `+RTS -Ksize' to increase it. This is with the latest version of git-annex and a current version of git on OS 10.6.7. After this error occurs, I am unable to un-annex the files and I'm forced to recover from a backup. From ef67b19c808d373783b9d74bdc61e69dff3c88ca Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Tue, 5 Apr 2011 07:27:47 +0000 Subject: [PATCH 1435/8313] Added a comment --- ..._08791cb78b982087c2a07316fe3ed46c._comment | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment new file mode 100644 index 0000000000..94043a7001 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment @@ -0,0 +1,22 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-04-05T07:27:46Z" + content=""" +Heh, cool, I was thinking throwing about 28million files at git-annex. Let me know how it goes, I suspect you have just run into a default limits OSX problem. + +You probably just need to up some system limits (you will need to read the error messages that first appear) then do something like + +
+# this is really for the run time, you can set these settings in /etc/sysctl.conf
+sudo sysctl -w kern.maxproc=2048
+sudo sysctl -w kern.maxprocperuid=1024
+
+# tell launchd about having higher limits
+sudo echo \"limit maxfiles 1024 unlimited\" >> /etc/launchd.conf
+sudo echo \"limit maxproc 1024 2048\" >> /etc/launchd.conf
+
+ +There are other system limits which you can check by doing a \"ulimit -a\", once you make the above changes, you will need to reboot to make the changes take affect. I am unsure if the above will help as it is an example of what I did on 10.6.6 a few months ago to fix some forking issues. From the error you got you will probably need to increase the stacksize to something bigger or even make it unlimited if you feel lucky, the default stacksize on OSX is 8192, try making it say 10times that size first and see what happens. +"""]] From 24e2c13387179d3ca1ed2dd50f5d3fbafbc8f32e Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 17:46:03 +0000 Subject: [PATCH 1436/8313] Added a comment --- ..._0392a11219463e40c53bae73c8188b69._comment | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment new file mode 100644 index 0000000000..8ea5531f43 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment @@ -0,0 +1,25 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-05T17:46:03Z" + content=""" +This message comes from ghc's runtime memory manager. Apparently your ghc defaults to limiting the stack to 80 mb. +Mine seems to limit it slightly higher -- I have seen haskell programs successfully grow as large as 350 mb, although generally not intentionally. :) + +Here's how to adjust the limit at runtime, obviously you'd want a larger number: + +
+# git-annex +RTS -K100 -RTS find
+Stack space overflow: current size 100 bytes.
+Use `+RTS -Ksize -RTS' to increase it.
+
+ +I've tried to avoid git-annex using quantities of memory that scale with the number of files in the repo, and I think in general successfully -- I run it on 32 mb and 128 mb machines, FWIW. There are some tricky cases, and haskell makes it easy to accidentally write code that uses much more memory than would be expected. + +One well known case is `git annex unused`, which *has* to build a structure of every annexed file. I have been considering using a bloom filter or something to avoid that. + +Another possible case is when running a command like `git annex add`, and passing it a lot of files/directories. Some code tries to preserve the order of your input after passing it through `git ls-files` (which destroys ordering), and to do so it needs to buffer both the input and the result in ram. + +It's possible to build git-annex with memory profiling and generate some quite helpful profiling data. Edit the Makefile and add this to GHCFLAGS: `-prof -auto-all -caf-all -fforce-recomp` then when running git-annex, add the parameters: `+RTS -p -RTS` , and look for the git-annex.prof file. +"""]] From 094983a2bdbb7cbf2aea96addd3afe1ddedc6c69 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 5 Apr 2011 14:00:51 -0400 Subject: [PATCH 1437/8313] support PROFILE=1 to enable profiling --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6a1531a3c8..24f1b1e1f9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ PREFIX=/usr -GHCFLAGS=-O2 -Wall -ignore-package monads-fd +IGNORE=-ignore-package monads-fd +GHCFLAGS=-O2 -Wall -ignore-package $(IGNORE) +ifdef PROFILE +GHCFLAGS=-prof -auto-all -caf-all -fforce-recomp $(IGNORE) +endif GHCMAKE=ghc $(GHCFLAGS) --make bins=git-annex git-annex-shell From 2bd955bc65f0fdd2f48a5514bcf343d880295068 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 18:02:05 +0000 Subject: [PATCH 1438/8313] Added a comment --- .../comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment new file mode 100644 index 0000000000..8e4101e37e --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-05T18:02:05Z" + content=""" +Oh, you'll need profiling builds of various haskell libraries to build with profiling support. If that's not easily accomplished, if you could show me the form of the command you're running, and also how git annex unannex fails, that would be helpful for investigating. +"""]] From 237e96bc6a3504525e82eadb8fc9c5eea6e94d66 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 18:05:00 +0000 Subject: [PATCH 1439/8313] Added a comment --- ...omment_1_3a0bf74b51586354b7a91f8b43472376._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment b/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment new file mode 100644 index 0000000000..3d69e8f290 --- /dev/null +++ b/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-05T18:05:00Z" + content=""" +Maybe, otoh, part of the point of git-annex is that the data may be too large to pull down all of it. + +I find mr useful as a policy layer over top of git-annex, so \"mr update\" can pull down appropriate quantities of data from +appropriate locations. +"""]] From a301a38d9969febdea3a4f3d3eb2d98077d3d66f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 5 Apr 2011 14:29:44 -0400 Subject: [PATCH 1440/8313] redundancy --- doc/design/encryption.mdwn | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 0242aabebb..f8f8656a74 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -43,8 +43,6 @@ At a high level, an encryption backend needs to support these operations: * Clean up. -* Configure an encryption key to use. - The rest of this page will describe a single encryption backend using GPG. Probably only one will be needed, but who knows? Maybe that backend will turn out badly designed, or some other encryptor needed. Designing From 08a23997dd5068218e7fd05bfb23cf52dd6299b0 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 5 Apr 2011 18:41:49 +0000 Subject: [PATCH 1441/8313] Added a comment --- ...comment_2_a610b3d056a059899178859a3a821ea5._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment diff --git a/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment b/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment new file mode 100644 index 0000000000..d5461e23c0 --- /dev/null +++ b/doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-05T18:41:49Z" + content=""" +I see no use case for verifying encrypted object files w/o access to the encryption key. And possible use cases for not allowing anyone to verify your data. + +If there are to be multiple encryption keys usable within a single encrypted remote, than they would need to be given some kind of name (a since symmetric key is used, there is no pubkey to provide a name), and the name encoded in the files stored in the remote. While certainly doable I'm not sold that adding a layer of indirection is worthwhile. It only seems it would be worthwhile if setting up a new encrypted remote was expensive to do. Perhaps that could be the case for some type of remote other than S3 buckets. +"""]] From 421df56d6c2932043afb78147eada388127ea91f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 5 Apr 2011 20:09:53 +0000 Subject: [PATCH 1442/8313] rename forum/bainstormning:_git_annex_push___38___pull.mdwn to forum/bainstorming:_git_annex_push___38___pull.mdwn --- ...___pull.mdwn => bainstorming:_git_annex_push___38___pull.mdwn} | 0 .../comment_1_3a0bf74b51586354b7a91f8b43472376._comment | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename doc/forum/{bainstormning:_git_annex_push___38___pull.mdwn => bainstorming:_git_annex_push___38___pull.mdwn} (100%) rename doc/forum/{bainstormning:_git_annex_push___38___pull => bainstorming:_git_annex_push___38___pull}/comment_1_3a0bf74b51586354b7a91f8b43472376._comment (100%) diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull.mdwn b/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn similarity index 100% rename from doc/forum/bainstormning:_git_annex_push___38___pull.mdwn rename to doc/forum/bainstorming:_git_annex_push___38___pull.mdwn diff --git a/doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment b/doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment similarity index 100% rename from doc/forum/bainstormning:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment rename to doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment From b4d6c52b712fb1ffa4de91842e9be9380b2015cb Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 5 Apr 2011 20:52:53 +0000 Subject: [PATCH 1443/8313] Added a comment --- ...ent_2_b02ca09914e788393c01196686f95831._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment diff --git a/doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment b/doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment new file mode 100644 index 0000000000..e0ecc1a819 --- /dev/null +++ b/doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-05T20:52:52Z" + content=""" +No-so-subtle sarcasm taken and acknowledged :) + +Arguably, git-annex should know about any local limits and not have them implemented via mr from the outside. I guess my concern boils down to having git-annex do the right thing all by itself with minimal user interaction. And while I really do appreciate the flexibility of chaining commands, I am a firm believer in exposing the common use cases as easily as possible. + +And yes, I am fully aware that not all annexes are created equal. Point in case, I would never use git annex pull on my laptop, but I would git annex push extensively. + + +"""]] From fbec3aa7512d6c77e655c118692950eeb3e789ab Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" Date: Tue, 5 Apr 2011 21:14:12 +0000 Subject: [PATCH 1444/8313] Added a comment --- ..._7cb65d013e72bd2b7e90452079d42ac9._comment | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment new file mode 100644 index 0000000000..bac9fd7cad --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment @@ -0,0 +1,29 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkhdKAhe3l_UyGt5SdfRBPYVwe-9f8P2dM" + nickname="Justin" + subject="comment 4" + date="2011-04-05T21:14:12Z" + content=""" +@joey + +OK, I'll try increasing the stack size and see if that helps. + +For reference, I was running: + +git annex add . + +on a directory containing about 100k files spread over many nested subdirectories. I actually have more than a dozen projects like this that I plan to keep in git annex, possibly in separate repositories if necessary. I could probably tar the data and then archive that, but I like the idea of being able to see the structure of my data even though the contents of the files are on a different machine. + +After the crash, running: + +git annex unannex + +does nothing and returns instantly. What exactly is 'git annex add' doing? I know that it's moving files into the key-value store and adding symlinks, but I don't know what else it does. + +--Justin + + + +If + +"""]] From 711d48f32a205ad2023489f131e9a3b70080e900 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Tue, 5 Apr 2011 23:24:18 +0000 Subject: [PATCH 1445/8313] Added a comment --- ...mment_3_cca186a9536cd3f6e86994631b14231c._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment diff --git a/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment b/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment new file mode 100644 index 0000000000..d3c483fdf3 --- /dev/null +++ b/doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-04-05T23:24:17Z" + content=""" +Assuming you're storing your encrypted annex with me and I with you, our regular cron jobs to verify all data will catch corruption in each other's annexes. + +Checksums of the encrypted objects could be optional, mitigating any potential attack scenarios. + +It's not only about the cost of setting up new remotes. It would also be a way to keep data in one annex while making it accessible only in a subset of them. For example, I might need some private letters at work, but I don't want my work machine to be able to access them all. +"""]] From 788fcc2bb498b582b5c6f73ff38ca200bdd18c05 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Apr 2011 21:41:13 -0400 Subject: [PATCH 1446/8313] typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 24f1b1e1f9..13adfe0a2d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr IGNORE=-ignore-package monads-fd -GHCFLAGS=-O2 -Wall -ignore-package $(IGNORE) +GHCFLAGS=-O2 -Wall $(IGNORE) ifdef PROFILE GHCFLAGS=-prof -auto-all -caf-all -fforce-recomp $(IGNORE) endif From c1bbe434224cce9a1afab0f358c6df83fa014e9c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Apr 2011 21:53:06 -0400 Subject: [PATCH 1447/8313] Add build depend on perlmagick so docs are consistently built. Closes: #621410 --- debian/changelog | 2 ++ debian/control | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 71ea599cd9..87ad9d01b5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low and git annex dropunused. * Clear up short option confusion between --from and --force (-f is now --from, and there is no short option for --force). + * Add build depend on perlmagick so docs are consistently built. + Closes: #621410 -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/debian/control b/debian/control index 4b31a295f9..37e6220437 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core +Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core, perlmagick Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex From ab0e03498f8a52bdf140587a4c7e13b8fe949ce7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 6 Apr 2011 21:57:22 -0400 Subject: [PATCH 1448/8313] Add doc-base file. Closes: #621408 --- debian/changelog | 1 + debian/doc-base | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 debian/doc-base diff --git a/debian/changelog b/debian/changelog index 87ad9d01b5..6ccb9eac96 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,7 @@ git-annex (0.20110402) UNRELEASED; urgency=low --from, and there is no short option for --force). * Add build depend on perlmagick so docs are consistently built. Closes: #621410 + * Add doc-base file. Closes: #621408 -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 diff --git a/debian/doc-base b/debian/doc-base new file mode 100644 index 0000000000..f71a233333 --- /dev/null +++ b/debian/doc-base @@ -0,0 +1,9 @@ +Document: git-annex +Title: git-annex documentation +Author: Joey Hess +Abstract: All the documentation from git-annex's website. +Section: File Management + +Format: HTML +Index: /usr/share/doc/git-annex/html/index.html +Files: /usr/share/doc/git-annex/html/*.html From a4d37c4550b911567d0dfb29ed5ba89f619065a7 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Thu, 7 Apr 2011 08:03:12 +0000 Subject: [PATCH 1449/8313] --- .../sparse_git_checkouts_with_annex.mdwn | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex.mdwn diff --git a/doc/forum/sparse_git_checkouts_with_annex.mdwn b/doc/forum/sparse_git_checkouts_with_annex.mdwn new file mode 100644 index 0000000000..32e6e4bbc4 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex.mdwn @@ -0,0 +1,28 @@ +I checked in my music collection into git annex (about 25000 files) and i'm really impressed by the performance of git annex (after i've done an git-repack). Now i'm also moving my movies into the same git-annex, but i have the following layout of my disk drives: + +* small raid-1 for important stuff (music, documents), which is also backupped (aka: raid) +* big bulk data store (aka: media) + +In the git-annex the following layout of files is used: + +* documents/ <- on raid +* music/ <- on raid +* videos/ <- on media + +Now i didn't simply clone the raid-annex to media, but did an sparse-checkout (possible since version 1.7.0) + +* raid: .git-annex/, documents/ and music +* media: .git-annex/, videos/ + +As you can see i have to checkout the .git-annex directory with the file-logs twice which slows down git operations. Everything else works fine until now. git-annex doesn't have any problem, that only a part of the symlinks are present, which is really great. Is there a possibility to sparse checkout the .git-annex directory also? Perhaps splitting the log files in .git-annex/ into N subfolders, corresponding to the toplevel subfolders, like this? + +* Before: + $ ls .git-annex + 00 01 02.... +* After: + $ ls .git-annex + documents/ music/ videos/ + $ ls .git-annex/documents + 00 01 02.... + +This would make it possible to checkout only the part of the log files which i'm interested in. From eca9914be1213f8110afef35e647ce14ae22bfb7 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Thu, 7 Apr 2011 08:04:39 +0000 Subject: [PATCH 1450/8313] (sorry for noise, had to format the code blocks) --- .../sparse_git_checkouts_with_annex.mdwn | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/forum/sparse_git_checkouts_with_annex.mdwn b/doc/forum/sparse_git_checkouts_with_annex.mdwn index 32e6e4bbc4..97d2f445d3 100644 --- a/doc/forum/sparse_git_checkouts_with_annex.mdwn +++ b/doc/forum/sparse_git_checkouts_with_annex.mdwn @@ -16,13 +16,16 @@ Now i didn't simply clone the raid-annex to media, but did an sparse-checkout (p As you can see i have to checkout the .git-annex directory with the file-logs twice which slows down git operations. Everything else works fine until now. git-annex doesn't have any problem, that only a part of the symlinks are present, which is really great. Is there a possibility to sparse checkout the .git-annex directory also? Perhaps splitting the log files in .git-annex/ into N subfolders, corresponding to the toplevel subfolders, like this? -* Before: - $ ls .git-annex - 00 01 02.... -* After: - $ ls .git-annex - documents/ music/ videos/ - $ ls .git-annex/documents - 00 01 02.... +Before: + + $ ls .git-annex + 00 01 02.... + +After: + + $ ls .git-annex + documents/ music/ videos/ + $ ls .git-annex/documents + 00 01 02.... This would make it possible to checkout only the part of the log files which i'm interested in. From 7634f92e83f8a789df1d660ed95a08d5c136f70f Mon Sep 17 00:00:00 2001 From: "http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" Date: Thu, 7 Apr 2011 13:44:38 +0000 Subject: [PATCH 1451/8313] Added a comment: Reported the issue to GHC --- ...ment_5_67406dd8d9bd4944202353508468c907._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment new file mode 100644 index 0000000000..bffa9bb868 --- /dev/null +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="http://fraggod.pip.verisignlabs.com.pip.verisignlabs.com/" + subject="Reported the issue to GHC" + date="2011-04-07T13:44:36Z" + content=""" +Finally got around to [report the issue to GHC tracker](http://hackage.haskell.org/trac/ghc/ticket/5085#comment:7). + +Looks quite alike (at least to the haskell-illiterate person like me) to a highest-priority issue that's hanging right at the top of the list. +There are other similar reports, but they seem to be either related to PowerPC Macs, closed as invalid or due to needinfo inactivity. + +Guess any further discussion belongs there, unless ghc developers will bounce it back. +Thanks a lot for your help, Joey, and for sharing a great thing that git-annex is. +"""]] From 00f1c720eddf95ce21e6d2c35623a82e97ed604c Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 16:32:04 +0000 Subject: [PATCH 1452/8313] Added a comment --- ...mment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment new file mode 100644 index 0000000000..7adf4fc4d6 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-07T16:32:04Z" + content=""" +That's awesome, I had not heard of git sparse checkouts before. + +It does not make sense to tie the log files to the directory of the corresponding files, as then the logs would have to move when the files are moved, which would be a PITA and likely make merging log file changes very complex. Also, of course, multiple files in different locations can point at the same content, which has the same log file. And, to cap it off, git-annex can need to access the log file for a given key without having the slightest idea what file in the repository might point to it, and it would be very expensive to scan the whole repository to find out what that file is in order to lookup the filename of the log file. + +The most likely change in git-annex that will make this better is in [[this_todo_item|todo/branching]] -- but it's unknown how to do it yet. +"""]] From 7e76c60a0c6e89c88fcfdaed2fcac6f3c2368d7b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 16:33:30 +0000 Subject: [PATCH 1453/8313] Added a comment --- .../comment_2_e357db3ccc4079f07a291843975535eb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment new file mode 100644 index 0000000000..d8088a2d82 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-07T16:33:30Z" + content=""" +BTW, git-annex unused *will* have a problem that not all the symlinks are present. It will suggest dropping content belonging to the excluded symlinks. +"""]] From 079e57adf77de3601d9fe0df63aa472317889c11 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 12:33:48 -0400 Subject: [PATCH 1454/8313] current thoughts --- doc/todo/branching.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index b7441c6e4e..06bd50beed 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -105,3 +105,13 @@ too. The problem would then be that any locationlog lookup would need to look in all other branches (any branch could have more current info after all), which could get expensive. + +## way outside the box approach + +Another approach I have been mulling over is keeping the log file +branch checked out in .git-annex/logs/ -- this would be a checkout of a git +repository inside a git repository, using "git fake bare" techniques. This +would solve the merge problem, since git auto merge could be used. It would +still mean all the log files are on-disk, which annoys some. It would +require some tighter integration with git, so that after a pull, the log +repo is updated with the data pulled. --[[Joey]] From df65f0c77d335644a64abe81d554eedfbb157b5d Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 16:41:00 +0000 Subject: [PATCH 1455/8313] Added a comment --- ...ent_5_86a42ee3173a5d38f803e64b79496ab3._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment new file mode 100644 index 0000000000..7dcccef2e5 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-04-07T16:41:00Z" + content=""" +I think what is happening with \"git annex unannex\" is that \"git annex add\" crashes before it can \"git add\" the symlinks. unannex only looks at files that \"git ls-files\" shows, and so files that are not added to git are not seen. So, this can be recovered from by looking at git status and manually adding the symlinks to git, and then unannex. + +That also suggests that \"git annex add .\" has done something before crashing. That's consistent with you passing it < 2 parameters; it's not just running out of memory trying to expand and preserve order of its parameters (like it might if you ran \"git annex add experiment-1/ experiment-2/\") + +I'm pretty sure I know where the space leak is now. git-annex builds up a queue of git commands, so that it can run git a minimum number of times. Currently, this queue is only flushed at the end. I had been meaning to work on having it flush the queue periodically to avoid it growing without bounds, and I will prioritize doing that. + +(The only other thing that \"git annex add\" does is record location log information.) +"""]] From bc51387e6dd426f46f9ab0ef23e6e3eefe7a4417 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 13:59:31 -0400 Subject: [PATCH 1456/8313] Periodically flush git command queue, to avoid boating memory usage too much. Since the queue is flushed in between subcommand actions being run, there should be no issues with actions that expect to queue up some stuff and have it run after they do other stuff. So I didn't have to audit for such assumptions. --- Annex.hs | 29 +-------------------------- AnnexQueue.hs | 47 ++++++++++++++++++++++++++++++++++++++++++++ CmdLine.hs | 13 ++++++------ Command/Add.hs | 4 ++-- Command/Fix.hs | 4 ++-- Command/FromKey.hs | 4 ++-- Command/Move.hs | 3 ++- Command/PreCommit.hs | 3 ++- Command/Unannex.hs | 3 ++- Content.hs | 3 ++- GitQueue.hs | 29 +++++++++++++++++++++------ Remote/Git.hs | 3 ++- Upgrade/V1.hs | 13 ++++++------ debian/changelog | 2 ++ 14 files changed, 101 insertions(+), 59 deletions(-) create mode 100644 AnnexQueue.hs diff --git a/Annex.hs b/Annex.hs index 2723c6a008..f4e5d599d0 100644 --- a/Annex.hs +++ b/Annex.hs @@ -13,10 +13,7 @@ module Annex ( eval, getState, changeState, - gitRepo, - queue, - queueRun, - queueRunAt, + gitRepo ) where import Control.Monad.State @@ -25,7 +22,6 @@ import qualified GitRepo as Git import qualified GitQueue import qualified BackendClass import qualified RemoteClass -import Utility -- git-annex's monad type Annex = StateT AnnexState IO @@ -93,26 +89,3 @@ changeState a = do {- Returns the git repository being acted on -} gitRepo :: Annex Git.Repo gitRepo = getState repo - -{- Adds a git command to the queue. -} -queue :: String -> [CommandParam] -> FilePath -> Annex () -queue command params file = do - state <- get - let q = repoqueue state - put state { repoqueue = GitQueue.add q command params file } - -{- Runs (and empties) the queue. -} -queueRun :: Annex () -queueRun = do - state <- get - let q = repoqueue state - g <- gitRepo - liftIO $ GitQueue.run g q - put state { repoqueue = GitQueue.empty } - -{- Runs the queue if the specified number of items have been queued. -} -queueRunAt :: Integer -> Annex () -queueRunAt n = do - state <- get - let q = repoqueue state - when (GitQueue.size q >= n) queueRun diff --git a/AnnexQueue.hs b/AnnexQueue.hs new file mode 100644 index 0000000000..58e77a6e85 --- /dev/null +++ b/AnnexQueue.hs @@ -0,0 +1,47 @@ +{- git-annex command queue + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module AnnexQueue ( + add, + flush, + flushWhenFull +) where + +import Control.Monad.State (liftIO) +import Control.Monad (when, unless) + +import Annex +import Messages +import qualified GitQueue +import Utility + +{- Adds a git command to the queue, possibly running previously queued + - actions if enough have accumulated. -} +add :: String -> [CommandParam] -> FilePath -> Annex () +add command params file = do + q <- getState repoqueue + store $ GitQueue.add q command params file + +{- Runs the queue if it is full. Should be called periodically. -} +flushWhenFull :: Annex () +flushWhenFull = do + q <- getState repoqueue + when (GitQueue.full q) $ flush False + +{- Runs (and empties) the queue. -} +flush :: Bool -> Annex () +flush silent = do + q <- getState repoqueue + unless (0 == GitQueue.size q) $ do + unless silent $ + showSideAction "Recording state in git..." + g <- gitRepo + q' <- liftIO $ GitQueue.flush g q + store q' + +store :: GitQueue.Queue -> Annex () +store q = changeState $ \s -> s { repoqueue = q } diff --git a/CmdLine.hs b/CmdLine.hs index de03d96ed4..684ebf979a 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -14,11 +14,11 @@ module CmdLine ( import System.IO.Error (try) import System.Console.GetOpt import Control.Monad.State (liftIO) -import Control.Monad (when, unless) +import Control.Monad (when) import qualified Annex +import qualified AnnexQueue import qualified GitRepo as Git -import qualified GitQueue import Types import Command import BackendList @@ -81,7 +81,9 @@ tryRun :: Annex.AnnexState -> [Annex Bool] -> IO () tryRun state actions = tryRun' state 0 actions tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO () tryRun' state errnum (a:as) = do - result <- try $ Annex.run state a + result <- try $ Annex.run state $ do + AnnexQueue.flushWhenFull + a case result of Left err -> do Annex.eval state $ showErr err @@ -100,10 +102,7 @@ startup = do {- Cleanup actions. -} shutdown :: Annex Bool shutdown = do - q <- Annex.getState Annex.repoqueue - unless (0 == GitQueue.size q) $ do - showSideAction "Recording state in git..." - Annex.queueRun + AnnexQueue.flush False liftIO $ Git.reap diff --git a/Command/Add.hs b/Command/Add.hs index da98bffa4f..b532ab045d 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -11,7 +11,7 @@ import Control.Monad.State (liftIO) import System.Posix.Files import Command -import qualified Annex +import qualified AnnexQueue import qualified Backend import LocationLog import Types @@ -60,5 +60,5 @@ cleanup file key = do let mtime = modificationTime s liftIO $ touch file (TimeSpec mtime) False - Annex.queue "add" [Param "--"] file + AnnexQueue.add "add" [Param "--"] file return True diff --git a/Command/Fix.hs b/Command/Fix.hs index 513e07a310..d898ce517d 100644 --- a/Command/Fix.hs +++ b/Command/Fix.hs @@ -12,7 +12,7 @@ import System.Posix.Files import System.Directory import Command -import qualified Annex +import qualified AnnexQueue import Utility import Content import Messages @@ -44,5 +44,5 @@ perform file link = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" [Param "--"] file + AnnexQueue.add "add" [Param "--"] file return True diff --git a/Command/FromKey.hs b/Command/FromKey.hs index 8c1a1028fe..eadaa13e1f 100644 --- a/Command/FromKey.hs +++ b/Command/FromKey.hs @@ -13,7 +13,7 @@ import System.Directory import Control.Monad (unless) import Command -import qualified Annex +import qualified AnnexQueue import Utility import qualified Backend import Content @@ -46,5 +46,5 @@ perform file = do cleanup :: FilePath -> CommandCleanup cleanup file = do - Annex.queue "add" [Param "--"] file + AnnexQueue.add "add" [Param "--"] file return True diff --git a/Command/Move.hs b/Command/Move.hs index 951695d66e..e5e78d2495 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -12,6 +12,7 @@ import Control.Monad.State (liftIO) import Command import qualified Command.Drop import qualified Annex +import qualified AnnexQueue import LocationLog import Types import Content @@ -59,7 +60,7 @@ remoteHasKey remote key present = do g <- Annex.gitRepo let remoteuuid = Remote.uuid remote logfile <- liftIO $ logChange g key remoteuuid status - Annex.queue "add" [Param "--"] logfile + AnnexQueue.add "add" [Param "--"] logfile where status = if present then ValuePresent else ValueMissing diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs index 727a637285..1db40f75fa 100644 --- a/Command/PreCommit.hs +++ b/Command/PreCommit.hs @@ -11,6 +11,7 @@ import Control.Monad.State (liftIO) import Command import qualified Annex +import qualified AnnexQueue import qualified GitRepo as Git import qualified Command.Add import qualified Command.Fix @@ -42,5 +43,5 @@ cleanup file = do -- stage the symlink g <- Annex.gitRepo liftIO $ Git.run g "reset" [Params "-q --", File file] - Annex.queueRun + AnnexQueue.flush True return True diff --git a/Command/Unannex.hs b/Command/Unannex.hs index b0ce31ceed..94db500c68 100644 --- a/Command/Unannex.hs +++ b/Command/Unannex.hs @@ -13,6 +13,7 @@ import System.Directory import Command import qualified Annex +import qualified AnnexQueue import Utility import qualified Backend import LocationLog @@ -68,6 +69,6 @@ cleanup file key = do -- Commit staged changes at end to avoid confusing the -- pre-commit hook if this file is later added back to -- git as a normal, non-annexed file. - Annex.queue "commit" [Params "-a -m", Param "content removed from git annex"] "-a" + AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] "-a" return True diff --git a/Content.hs b/Content.hs index ba265c9307..f63c02311f 100644 --- a/Content.hs +++ b/Content.hs @@ -36,6 +36,7 @@ import LocationLog import UUID import qualified GitRepo as Git import qualified Annex +import qualified AnnexQueue import Utility import StatFS import Key @@ -72,7 +73,7 @@ logStatus key status = do unless (Git.repoIsLocalBare g) $ do u <- getUUID g logfile <- liftIO $ logChange g key u status - Annex.queue "add" [Param "--"] logfile + AnnexQueue.add "add" [Param "--"] logfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/GitQueue.hs b/GitQueue.hs index dfe2976da1..480027fa0c 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -10,7 +10,8 @@ module GitQueue ( empty, add, size, - run + full, + flush ) where import qualified Data.Map as M @@ -32,9 +33,21 @@ data Action = Action { {- A queue of actions to perform (in any order) on a git repository, - with lists of files to perform them on. This allows coalescing - similar git commands. -} -data Queue = Queue Integer (M.Map Action [FilePath]) +data Queue = Queue Int (M.Map Action [FilePath]) deriving (Show, Eq) +{- A recommended maximum size for the queue, after which it should be + - run. + - + - 10240 is semi-arbitrary. If we assume git filenames are between 10 and + - 255 characters long, then the queue will build up between 100kb and + - 2550kb long commands. The max command line length on linux is somewhere + - above 20k, so this is a fairly good balance -- the queue will buffer + - only a few megabytes of stuff and a minimal number of commands will be + - run by xargs. -} +maxSize :: Int +maxSize = 10240 + {- Constructor for empty queue. -} empty :: Queue empty = Queue 0 M.empty @@ -47,14 +60,18 @@ add (Queue n m) subcommand params file = Queue (n + 1) m' m' = M.insertWith' (++) action [file] m {- Number of items in a queue. -} -size :: Queue -> Integer +size :: Queue -> Int size (Queue n _) = n +{- Is a queue large enough that it should be flushed? -} +full :: Queue -> Bool +full (Queue n _) = n > maxSize + {- Runs a queue on a git repository. -} -run :: Git.Repo -> Queue -> IO () -run repo (Queue _ m) = do +flush :: Git.Repo -> Queue -> IO Queue +flush repo (Queue _ m) = do forM_ (M.toList m) $ uncurry $ runAction repo - return () + return empty {- Runs an Action on a list of files in a git repository. - diff --git a/Remote/Git.hs b/Remote/Git.hs index a458455109..2936beaf7d 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -19,6 +19,7 @@ import RemoteClass import Types import qualified GitRepo as Git import qualified Annex +import qualified AnnexQueue import Locations import UUID import Utility @@ -150,7 +151,7 @@ copyToRemote r key Annex.eval a $ do ok <- Content.getViaTmp key $ \f -> liftIO $ copyFile keysrc f - Annex.queueRun + AnnexQueue.flush True return ok | Git.repoIsSsh r = do g <- Annex.gitRepo diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs index 4ce2612d60..9278bce603 100644 --- a/Upgrade/V1.hs +++ b/Upgrade/V1.hs @@ -24,6 +24,7 @@ import Types import Locations import LocationLog import qualified Annex +import qualified AnnexQueue import qualified GitRepo as Git import Backend import Messages @@ -68,7 +69,7 @@ upgrade = do updateSymlinks moveLocationLogs - Annex.queueRun + AnnexQueue.flush True setVersion -- add new line to auto-merge hashed location logs @@ -106,8 +107,7 @@ updateSymlinks = do link <- calcGitLink f k liftIO $ removeFile f liftIO $ createSymbolicLink link f - Annex.queue "add" [Param "--"] f - Annex.queueRunAt 10240 + AnnexQueue.add "add" [Param "--"] f moveLocationLogs :: Annex () moveLocationLogs = do @@ -137,10 +137,9 @@ moveLocationLogs = do old <- liftIO $ readLog f new <- liftIO $ readLog dest liftIO $ writeLog dest (old++new) - Annex.queue "add" [Param "--"] dest - Annex.queue "add" [Param "--"] f - Annex.queue "rm" [Param "--quiet", Param "-f", Param "--"] f - Annex.queueRunAt 10240 + AnnexQueue.add "add" [Param "--"] dest + AnnexQueue.add "add" [Param "--"] f + AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f oldlog2key :: FilePath -> Maybe (FilePath, Key) oldlog2key l = diff --git a/debian/changelog b/debian/changelog index 6ccb9eac96..fdc740cb85 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,6 +12,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Add build depend on perlmagick so docs are consistently built. Closes: #621410 * Add doc-base file. Closes: #621408 + * Periodically flush git command queue, to avoid boating memory usage + too much. -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 From d804062f8df64219a781b06f27e609464d6e3159 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 18:09:13 +0000 Subject: [PATCH 1457/8313] Added a comment --- ...omment_6_4551274288383c9cc27cbf85b122d307._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment new file mode 100644 index 0000000000..fff8f7cdde --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 6" + date="2011-04-07T18:09:13Z" + content=""" +I've committed the queue flush improvements, so it will buffer up to 10240 git actions, and then flush the queue. + +There may be other memory leaks at scale (besides the two I mentioned earlier), but this seems promising. I'm well into running `git annex add` on a half million files and it's using 18 mb ram and has flushed the queue several times. This run +will fail due to running out of inodes for the log files, not due to memory. :) +"""]] From 17e974d32d956fd99416da706e7ae008ed7af308 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 18:13:46 +0000 Subject: [PATCH 1458/8313] Comment moderation --- ...omment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment | 11 +++++++++++ ...omment_3_2082f4d708a584a1403cc1d4d005fb56._comment | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment create mode 100644 doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment diff --git a/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment new file mode 100644 index 0000000000..f3db75c2f6 --- /dev/null +++ b/doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="fmarier" + subject="comment 3" + date="2011-04-05T10:00:21Z" + content=""" +Thanks for the reply @joey. + +While it would certainly be possible for a bare repo to exist on my iRiver, the problem is that the music player uses the filesystem to organize files into directories like \"Artist/Album/Track.ogg\". So replacing that with \"..../xx/yy/Track.ogg\" would make it fairly difficult to browse my music collection and select the album/track I want to listen to :) + +So unless I have the files physically organized like the symlinks, then it's probably not going to work very for that particular workflow. Smudge filters are interesting though. In the meantime, I'll look into rsyncing from another box which has the right filesystem layout onto my iRiver directly. +"""]] diff --git a/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment new file mode 100644 index 0000000000..01dc7813ff --- /dev/null +++ b/doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-04T10:28:01Z" + content=""" +Going one step further, a --min-copy could put all files so that numcopies is satisfied. --all could push to all available ones. + +To take everything another step further, if it was possible to group remotes, one could act on the groups. \"all\" would be an obvious choice for a group that always exists, everything else would be set up by the user. +"""]] From 4ea0b7c28850eb703562cd9dc84a02c49b5fda00 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 14:45:10 -0400 Subject: [PATCH 1459/8313] add --- doc/todo/git-annex_unused_eats_memory.mdwn | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/todo/git-annex_unused_eats_memory.mdwn diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn new file mode 100644 index 0000000000..6ce7140045 --- /dev/null +++ b/doc/todo/git-annex_unused_eats_memory.mdwn @@ -0,0 +1,25 @@ +`git-annex unused` has to compare large sets of data +(all keys with content present in the repository, +with all keys used by files in the repository), and so +uses more memory than git-annex typically needs; around +60-80 mb when run in a repository with 80 thousand files. + +I would like to reduce this. One idea is to use a bloom filter. +For example, construct a bloom filter of all keys used by files in +the repository. Then for each key with content present, check if it's +in the bloom filter. Since there can be false negatives, this might +miss finding some unused keys. The probability/size of filter +could be tunable. + +Another way might be to scan the git log for files that got removed +or changed what key they pointed to. Correlate with keys with content +currently present in the repository (possibly using a bloom filter again), +and that would yield a shortlist of keys that are probably not used. +Then scan thru all files in the repo to make sure that none point to keys +on the shortlist. + +---- + +`git annex unused --from remote` is much worse, using hundreds of mb of +memory. It has not been profiled at all yet, and can probably be improved +somewhat by fixing whatever memory leak it (probably) has. From 135d75f2b9c0f77b837da4243dea32a97d4088a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 15:00:06 -0400 Subject: [PATCH 1460/8313] avoid list traverse on queue I wanted to use M.insertWith' (\_ l -> file:l) action [] m , but the order of the parameters and which to ignore is not clear, and seems unsafe to rely on. --- GitQueue.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/GitQueue.hs b/GitQueue.hs index 480027fa0c..be0fcfc4af 100644 --- a/GitQueue.hs +++ b/GitQueue.hs @@ -57,7 +57,11 @@ add :: Queue -> String -> [CommandParam] -> FilePath -> Queue add (Queue n m) subcommand params file = Queue (n + 1) m' where action = Action subcommand params - m' = M.insertWith' (++) action [file] m + -- There are probably few items in the map, but there + -- can be a lot of files per item. So, optimise adding + -- files. + m' = M.insertWith' const action files m + files = file:(M.findWithDefault [] action m) {- Number of items in a queue. -} size :: Queue -> Int From 36894655f34eb46b386f708b889f05ffd269f0cf Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 7 Apr 2011 19:59:30 +0000 Subject: [PATCH 1461/8313] Added a comment --- ...comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment diff --git a/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment b/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment new file mode 100644 index 0000000000..14eb1acac1 --- /dev/null +++ b/doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 4" + date="2011-04-07T19:59:30Z" + content=""" +@Richard the easy way to deal with that scenario is to set up a remote that work can access, and only put in it files work should be able to see. Needing to specify which key a file should be encrypted to when putting it in a remote that supported multiple keys would add another level of complexity which that avoids. + +Of course, the right approach is probably to have a separate repository for work. If you don't trust it with seeing file contents, you probably also don't trust it with the contents of your git repository. +"""]] From bd1bbc21fac4b36975d03638d15e9aab915927cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 16:05:30 -0400 Subject: [PATCH 1462/8313] update --- doc/design/encryption.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index f8f8656a74..8a8f38108e 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -61,8 +61,8 @@ more gpg public keys. This scheme allows new gpg private keys to be given access to content that has already been stored in the remote. Different encrypted remotes need to be able to each use different ciphers. -There does not seem to be a benefit to allowing multiple cipers to be -used within a single remote, and it would add a lot of complexity. +Allowing multiple cipers to be used within a single remote would add a lot +of complexity, so is not planned to be supported. Instead, if you want a new cipher, create a new S3 bucket, or whatever. There does not seem to be much benefit to using the same cipher for two different enrypted remotes. From f5b2d650bb75dc7ca2f77dae59fb1ab7f7405e03 Mon Sep 17 00:00:00 2001 From: Fraser Tweedale Date: Fri, 8 Apr 2011 10:08:11 +1000 Subject: [PATCH 1463/8313] recognise differently-named shaN programs --- Backend/SHA.hs | 18 ++++++++++-------- TestConfig.hs | 14 +++++++++++++- configure.hs | 9 +++++---- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 0ec555ce3f..42b5efeff0 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -35,7 +35,7 @@ backends = catMaybes $ map genBackend [1, 256, 512, 224, 384] genBackend :: SHASize -> Maybe (Backend Annex) genBackend size - | supported size = Just b + | shaCommand size /= "" = Just b | otherwise = Nothing where b = Backend.File.backend @@ -43,12 +43,14 @@ genBackend size , getKey = keyValue size , fsckKey = Backend.File.checkKey $ checkKeyChecksum size } - supported 1 = SysConfig.sha1sum - supported 256 = SysConfig.sha256sum - supported 224 = SysConfig.sha224sum - supported 384 = SysConfig.sha384sum - supported 512 = SysConfig.sha512sum - supported _ = False + +shaCommand :: SHASize -> String +shaCommand 1 = SysConfig.sha1 +shaCommand 256 = SysConfig.sha256 +shaCommand 224 = SysConfig.sha224 +shaCommand 384 = SysConfig.sha384 +shaCommand 512 = SysConfig.sha512 +shaCommand _ = "" shaName :: SHASize -> String shaName size = "SHA" ++ show size @@ -63,7 +65,7 @@ shaN size file = do then error $ command ++ " parse error" else return $ head bits where - command = "sha" ++ (show size) ++ "sum" + command = shaCommand size {- A key is a checksum of its contents. -} keyValue :: SHASize -> FilePath -> Annex (Maybe Key) diff --git a/TestConfig.hs b/TestConfig.hs index 5e59681ddf..d1560b660f 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,13 +72,25 @@ selectCmd k cmds = search cmds where search [] = do testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds + error $ "* need one of these commands, but none are available: " ++ show (map (head . words) cmds) search (c:cs) = do ret <- system $ quiet c if (ret == ExitSuccess) then return $ Config k (StringConfig c) else search cs +whichCmd :: ConfigKey -> [String] -> Test +whichCmd k cmds = search cmds + where + search [] = do + testEnd $ Config k (StringConfig "") + return $ Config k (StringConfig "") + search (c:cs) = do + ret <- system $ quiet c + if (ret == ExitSuccess) + then return $ Config k (StringConfig $ head $ words c) + else search cs + quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index f8cd577e99..c0e3d8106e 100644 --- a/configure.hs +++ b/configure.hs @@ -20,10 +20,11 @@ tests = [ shaTestCases :: [Int] -> [TestCase] shaTestCases l = map make l - where - make n = - let cmd = "sha" ++ show n ++ "sum" - in TestCase cmd $ requireCmd cmd (cmd ++ " "sha" ++ show n ++ x ++ " Date: Fri, 8 Apr 2011 01:13:06 +0000 Subject: [PATCH 1464/8313] --- doc/forum/wishlist:_git_annex_status.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_status.mdwn diff --git a/doc/forum/wishlist:_git_annex_status.mdwn b/doc/forum/wishlist:_git_annex_status.mdwn new file mode 100644 index 0000000000..add865410e --- /dev/null +++ b/doc/forum/wishlist:_git_annex_status.mdwn @@ -0,0 +1,19 @@ +Ideally, it would look similar to this. And yes, I put "put" in there ;) + + non-annex % git annex status + git annex status: error: not a git annex repository + annex % git annex status + annex object storage version: A + annex backend engine: {WORM,SHA512,...} + Estimated local annex size: B MiB + Estimated total annex size: C MiB + Files without file size information in local annex: D + Files without file size information in total annex: E + Last fsck: datetime + Last git pull: datetime - $annex_name + Last git push: datetime - $annex_name + Last git annex get: datetime - $annex_name + Last git annex put: datetime - $annex_name + annex % + +Datetime could be ISO's YYYY-MM-DDThh:mm:ss or, personal preference, YYYY-MM-DD--hh-mm-ss. I prefer the latter as it's DNS-, tag- and filename-safe which is why I am using it for everything. In a perfect world, ISO would standardize YYYY-MM-DD-T-hh-mm-ss-Z[-SSSSSSSS][--$timezone], but meh. From b889543507959c0a311bc8387f78ec4c0e93f7a8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 21:47:56 -0400 Subject: [PATCH 1465/8313] let's use Maybe String for commands that may not be avilable --- Backend/SHA.hs | 12 ++++++------ TestConfig.hs | 18 +++++++++++++----- debian/changelog | 1 + 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 42b5efeff0..d9aeb72aa4 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -27,7 +27,7 @@ import qualified SysConfig import Key type SHASize = Int - + backends :: [Backend Annex] -- order is slightly significant; want sha1 first ,and more general -- sizes earlier @@ -35,8 +35,8 @@ backends = catMaybes $ map genBackend [1, 256, 512, 224, 384] genBackend :: SHASize -> Maybe (Backend Annex) genBackend size - | shaCommand size /= "" = Just b - | otherwise = Nothing + | shaCommand size == Nothing = Nothing + | otherwise = Just b where b = Backend.File.backend { name = shaName size @@ -44,13 +44,13 @@ genBackend size , fsckKey = Backend.File.checkKey $ checkKeyChecksum size } -shaCommand :: SHASize -> String +shaCommand :: SHASize -> Maybe String shaCommand 1 = SysConfig.sha1 shaCommand 256 = SysConfig.sha256 shaCommand 224 = SysConfig.sha224 shaCommand 384 = SysConfig.sha384 shaCommand 512 = SysConfig.sha512 -shaCommand _ = "" +shaCommand _ = Nothing shaName :: SHASize -> String shaName size = "SHA" ++ show size @@ -65,7 +65,7 @@ shaN size file = do then error $ command ++ " parse error" else return $ head bits where - command = shaCommand size + command = fromJust $ shaCommand size {- A key is a checksum of its contents. -} keyValue :: SHASize -> FilePath -> Annex (Maybe Key) diff --git a/TestConfig.hs b/TestConfig.hs index d1560b660f..f6ca2afcd8 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -7,7 +7,10 @@ import System.Cmd import System.Exit type ConfigKey = String -data ConfigValue = BoolConfig Bool | StringConfig String +data ConfigValue = + BoolConfig Bool | + StringConfig String | + MaybeStringConfig (Maybe String) data Config = Config ConfigKey ConfigValue type Test = IO Config @@ -17,15 +20,17 @@ data TestCase = TestCase TestName Test instance Show ConfigValue where show (BoolConfig b) = show b show (StringConfig s) = show s + show (MaybeStringConfig s) = show s instance Show Config where - show (Config key value) = unlines + show (Config key value) = unlines [ key ++ " :: " ++ valuetype value , key ++ " = " ++ show value ] where valuetype (BoolConfig _) = "Bool" valuetype (StringConfig _) = "String" + valuetype (MaybeStringConfig _) = "Maybe String" writeSysConfig :: [Config] -> IO () writeSysConfig config = writeFile "SysConfig.hs" body @@ -83,12 +88,13 @@ whichCmd :: ConfigKey -> [String] -> Test whichCmd k cmds = search cmds where search [] = do - testEnd $ Config k (StringConfig "") - return $ Config k (StringConfig "") + let r = Config k (MaybeStringConfig Nothing) + testEnd r + return r search (c:cs) = do ret <- system $ quiet c if (ret == ExitSuccess) - then return $ Config k (StringConfig $ head $ words c) + then return $ Config k (MaybeStringConfig $ Just $ head $ words c) else search cs quiet :: String -> String @@ -103,3 +109,5 @@ testEnd :: Config -> IO () testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes" testEnd (Config _ (BoolConfig False)) = putStrLn $ " no" testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s +testEnd (Config _ (MaybeStringConfig (Just s))) = putStrLn $ " " ++ s +testEnd (Config _ (MaybeStringConfig Nothing)) = putStrLn $ " not available" diff --git a/debian/changelog b/debian/changelog index fdc740cb85..3072ac476a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -14,6 +14,7 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Add doc-base file. Closes: #621408 * Periodically flush git command queue, to avoid boating memory usage too much. + * Support "sha1" and "sha512" commands on FreeBSD. Thanks, Fraser Tweedale -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 From e2404ca40933ccdc046d08f7040b2d86e32d5f88 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 Apr 2011 22:03:31 -0400 Subject: [PATCH 1466/8313] refactor away whichCmd and some other cleanup --- TestConfig.hs | 36 ++++++++++++++++-------------------- configure.hs | 6 +++--- debian/changelog | 3 ++- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/TestConfig.hs b/TestConfig.hs index f6ca2afcd8..9b2759e199 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,30 +72,26 @@ testCmd k cmdline = do {- Ensures that one of a set of commands is available by running each in - turn. The Config is set to the first one found. -} -selectCmd :: ConfigKey -> [String] -> Test -selectCmd k cmds = search cmds +selectCmd :: Bool -> ConfigKey -> [String] -> String -> Test +selectCmd required k cmds param = search cmds where - search [] = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show (map (head . words) cmds) + search [] = failure search (c:cs) = do - ret <- system $ quiet c + ret <- system $ quiet c ++ " " ++ param if (ret == ExitSuccess) - then return $ Config k (StringConfig c) - else search cs - -whichCmd :: ConfigKey -> [String] -> Test -whichCmd k cmds = search cmds - where - search [] = do - let r = Config k (MaybeStringConfig Nothing) - testEnd r - return r - search (c:cs) = do - ret <- system $ quiet c - if (ret == ExitSuccess) - then return $ Config k (MaybeStringConfig $ Just $ head $ words c) + then success c else search cs + success c + | required == True = return $ Config k (StringConfig c) + | otherwise = return $ Config k (MaybeStringConfig $ Just c) + failure + | required == True = do + testEnd $ Config k (BoolConfig False) + error $ "* need one of these commands, but none are available: " ++ show cmds + | otherwise = do + let r = Config k (MaybeStringConfig Nothing) + testEnd r + return r quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index c0e3d8106e..0661813aef 100644 --- a/configure.hs +++ b/configure.hs @@ -11,7 +11,7 @@ tests = [ , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] + , TestCase "uuid generator" $ selectCmd True "uuid" ["uuid", "uuidgen"] "" , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" @@ -22,9 +22,9 @@ shaTestCases :: [Int] -> [TestCase] shaTestCases l = map make l where make n = let - cmds = map (\x -> "sha" ++ show n ++ x ++ " "sha" ++ show n ++ x) ["", "sum"] key = "sha" ++ show n - in TestCase key $ whichCmd key cmds + in TestCase key $ selectCmd False key cmds " Sat, 02 Apr 2011 13:45:54 -0400 From a77c34d2b4795a5d41f82a78e999ce33f43d8862 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 00:12:00 -0400 Subject: [PATCH 1467/8313] refactor --- TestConfig.hs | 30 ++++++++++++++++-------------- configure.hs | 8 ++++---- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/TestConfig.hs b/TestConfig.hs index 9b2759e199..bab297003a 100644 --- a/TestConfig.hs +++ b/TestConfig.hs @@ -72,26 +72,28 @@ testCmd k cmdline = do {- Ensures that one of a set of commands is available by running each in - turn. The Config is set to the first one found. -} -selectCmd :: Bool -> ConfigKey -> [String] -> String -> Test -selectCmd required k cmds param = search cmds +selectCmd :: ConfigKey -> [String] -> String -> Test +selectCmd k = searchCmd + (\match -> return $ Config k $ StringConfig match) + (\cmds -> do + testEnd $ Config k $ BoolConfig False + error $ "* need one of these commands, but none are available: " ++ show cmds + ) + +maybeSelectCmd :: ConfigKey -> [String] -> String -> Test +maybeSelectCmd k = searchCmd + (\match -> return $ Config k $ MaybeStringConfig $ Just match) + (\_ -> return $ Config k $ MaybeStringConfig Nothing) + +searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test +searchCmd success failure cmds param = search cmds where - search [] = failure + search [] = failure cmds search (c:cs) = do ret <- system $ quiet c ++ " " ++ param if (ret == ExitSuccess) then success c else search cs - success c - | required == True = return $ Config k (StringConfig c) - | otherwise = return $ Config k (MaybeStringConfig $ Just c) - failure - | required == True = do - testEnd $ Config k (BoolConfig False) - error $ "* need one of these commands, but none are available: " ++ show cmds - | otherwise = do - let r = Config k (MaybeStringConfig Nothing) - testEnd r - return r quiet :: String -> String quiet s = s ++ " >/dev/null 2>&1" diff --git a/configure.hs b/configure.hs index 0661813aef..d340f937d4 100644 --- a/configure.hs +++ b/configure.hs @@ -6,12 +6,12 @@ import Data.List import TestConfig tests :: [TestCase] -tests = [ - TestCase "version" $ getVersion +tests = + [ TestCase "version" $ getVersion , testCp "cp_a" "-a" , testCp "cp_p" "-p" , testCp "cp_reflink_auto" "--reflink=auto" - , TestCase "uuid generator" $ selectCmd True "uuid" ["uuid", "uuidgen"] "" + , TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] "" , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" @@ -24,7 +24,7 @@ shaTestCases l = map make l let cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"] key = "sha" ++ show n - in TestCase key $ selectCmd False key cmds " Date: Fri, 8 Apr 2011 07:23:09 +0000 Subject: [PATCH 1468/8313] Added a comment --- .../comment_1_994bfd12c5d82e08040d6116915c5090._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment diff --git a/doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment b/doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment new file mode 100644 index 0000000000..7b5e7bd449 --- /dev/null +++ b/doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-04-08T07:23:08Z" + content=""" ++1 for this feature, I've been longing for something like this other than rolling my own perl/shell scripts to parse the outputs of \"git annex whereis .\" to see how many files are on my machine or not. +"""]] From 2aae340e98341a4c6681ff8ec722117f8155aca3 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Fri, 8 Apr 2011 07:31:04 +0000 Subject: [PATCH 1469/8313] Added a comment --- .../comment_3_fcfafca994194d57dccf5319c7c9e646._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment new file mode 100644 index 0000000000..1b849ef891 --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 3" + date="2011-04-08T07:31:03Z" + content=""" +So perhaps checking if git-status (or similar) complains about missing files is a possible solution for this? +"""]] From 275f1bd5c5a72c9a0f8fe0e21282278a6adbf6d9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Fri, 8 Apr 2011 07:54:38 +0000 Subject: [PATCH 1470/8313] Added a comment --- ..._04dc14880f31eee2b6d767d4d4258c5a._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment diff --git a/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment b/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment new file mode 100644 index 0000000000..9280fc51da --- /dev/null +++ b/doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 4" + date="2011-04-08T07:54:37Z" + content=""" +And something else i've done is, that i symlinked the video/ directory from the media annex to the normal raid annex + + ln -s ~/media/annex/video ~/annex + +And it's working out great. + + ~annex $ git annex whereis video/series/episode1.avi + whereis video/series/episode1.avi(1 copy) + f210b45a-60d3-11e0-b593-3318d96f2520 -- Trantor - Media + ok + +I really like this, perhaps it is a good idea to store all log files in every repo, but maybe there is a possibilitiy to to pack multiple log files into one single file, where not only the time, the present bit and the annex-repository is stored, but also the file key. I don't know if this format would also be merged correctly by the union merge driver. + +"""]] From 784f47d2822a9f0e4cfa507cfbd01f99f4da60e6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Fri, 8 Apr 2011 12:19:05 +0000 Subject: [PATCH 1471/8313] --- doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn new file mode 100644 index 0000000000..608704d9d7 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn @@ -0,0 +1 @@ +I have files with very long filenames on an xfs at home. On my laptop the annex should have been checked out on an encfs, but there filenames can't be as long as on the xfs. So perhaps it would be good to limit the keysize to a sane substring of the filename e.g. use only the first 120 characters. From 72e6bed8bab48a77aa3b0211dcf801d21a7b886a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 8 Apr 2011 17:14:25 +0000 Subject: [PATCH 1472/8313] Added a comment --- ...comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment new file mode 100644 index 0000000000..41d3afb3eb --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-08T17:14:25Z" + content=""" +Seems like you probably have files in git with nearly as long filenames as the key files. Course, you can rename those yourself. + +This couldn't be changed directly in WORM without some ugly transition, but it would be possible to implement it as a WORM100 or so. OTOH, if you're going to git annex migrate, you might as well use SHA1. +"""]] From fbd77e02177463efa1f546ec78e158de5475e793 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawn5vmWzuHG-guyTurgCKCyIOl-uRTWpFyw" Date: Fri, 8 Apr 2011 17:49:27 +0000 Subject: [PATCH 1473/8313] --- doc/not.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/not.mdwn b/doc/not.mdwn index 1d38b848c6..fea8b3e969 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -33,7 +33,7 @@ * git-annex is also not [boar](http://code.google.com/p/boar/), although it shares many of its goals and characteristics. Boar implements - its own version control system, rather than simply embarcing and + its own version control system, rather than simply embracing and extending git. And while boar supports distributed clones of a repository, it does not support keeping different files in different clones of the same repository, which git-annex does, and is an important feature for From f3cf20d22a5c27b83138c4ee062edb7532fecbb3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 14:56:57 -0400 Subject: [PATCH 1474/8313] document bup special remotes --- doc/git-annex.mdwn | 10 ++++++++-- doc/not.mdwn | 6 ++---- doc/special_remotes.mdwn | 1 + doc/special_remotes/bup.mdwn | 34 ++++++++++++++++++++++++++++++++++ doc/walkthrough.mdwn | 1 + doc/walkthrough/using_bup.mdwn | 18 ++++++++++++++++++ 6 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 doc/special_remotes/bup.mdwn create mode 100644 doc/walkthrough/using_bup.mdwn diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 7d0fb3e792..3514002a49 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -402,9 +402,15 @@ Here are all the supported configuration settings. to or from this remote. For example, to force ipv6, and limit the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100" -* `annex.ssh-options`, `annex.rsync-options` +* `remote..annex-bup-split-options` - Default ssh and rsync options to use if a remote does not have + Options to pass to bup split when storing content in this remote. + For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k" + (There is no corresponding option for bup join.) + +* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options` + + Default ssh, rsync, and bup options to use if a remote does not have specific options. * `annex.diskreserve` diff --git a/doc/not.mdwn b/doc/not.mdwn index 1d38b848c6..8d5dbd0b52 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -2,10 +2,8 @@ * git-annex is not a backup system. It may be a useful component of an [[archival|use_case/bob]] system, or a way to deliver files to a backup - system. - - For a backup system that uses git, take a look at - [bup](http://github.com/apenwarr/bup). + system. For a backup system that uses git and that git-annex supports + storing data in, see [[special_remotes/bup]]. * git-annex is not unison, but if you're finding unison's checksumming too slow, or its strict mirroring of everything to both places too diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index f4d479aa9c..a33d3f6127 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -7,6 +7,7 @@ types of remotes. These can be used just like any normal remote by git-annex. They cannot be used by other git commands though. * [[Amazon_S3]] +* [[bup]] * [[directory]] ## Unused content on special remotes diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn new file mode 100644 index 0000000000..09eadd0085 --- /dev/null +++ b/doc/special_remotes/bup.mdwn @@ -0,0 +1,34 @@ +This special remote type stores file contents in a +[bup](http://github.com/apenwarr/bup) repository. By using git-annex +in the front-end, and bup as a remote, you get an easy git-style +interface to large files, and easy backups of the file contents using git. + +See [[walkthrough/using_bup]] for usage examples. + +## configuration + +These parameters can be passed to `git annex initremote` to configure bup: + +* `encryption` - Required. Either "none" to disable encryption, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. + +* `remote` - Required. This is passed to `bup` as the `--remote` + to use to store data. `bup init` will be run to create the + repository. Example: "remote=example.com:/big/mybup" + +Options to pass to `bup split` when sending content to bup can also +be specified, by using `git config annex.bup-split-options`. This +can be used to, for example, limit its bandwidth. + +## data security + +When encryption=none, there is **no** protection against your data being read +by anyone who can access the bup remote. However, bup does transfer data +using ssh, and if you trust the security of the remote, that's fine. + +** Encryption is not yet supported. ** + +See [[design/encryption]]. diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index 53f0be6bb4..c648807494 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -15,6 +15,7 @@ A walkthrough of the basic features of git-annex. using_ssh_remotes moving_file_content_between_repositories using_Amazon_S3 + using_bup using_the_URL_backend using_the_SHA1_backend migrating_data_to_a_new_backend diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn new file mode 100644 index 0000000000..1a506c2811 --- /dev/null +++ b/doc/walkthrough/using_bup.mdwn @@ -0,0 +1,18 @@ +Another [[special_remote|special_remotes]] that git-annex can use is +a [[special_remotes/bup]] repository. Bup stores large file contents +in a git repository of its own, with deduplication. Combined with +git-annex, you can have git on both the frontend and the backend. + +Here's how to create a bup remote, and describe it. + + # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup + initremote bup (init) ok + # git annex describe mybup "my bup repository at example.com" + describe mybup ok + +Now the remote can be used like any other remote. + + # git annex move my_cool_big_file --to mybup + move my_cool_big_file (to mybup...) ok + +See [[special_remotes/bup]] for details. From 44c65f40b7f67ee5d53769c6e5fc87f2c7849425 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:44:43 -0400 Subject: [PATCH 1475/8313] bup is now supported as a special type of remote. --- Remote.hs | 2 + Remote/Bup.hs | 133 +++++++++++++++++++++++++++++++++ configure.hs | 1 + debian/changelog | 1 + debian/control | 2 +- doc/walkthrough/using_bup.mdwn | 8 +- 6 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 Remote/Bup.hs diff --git a/Remote.hs b/Remote.hs index 26097da747..bb661c5a90 100644 --- a/Remote.hs +++ b/Remote.hs @@ -46,12 +46,14 @@ import Config import qualified Remote.Git import qualified Remote.S3 +import qualified Remote.Bup import qualified Remote.Directory remoteTypes :: [RemoteType Annex] remoteTypes = [ Remote.Git.remote , Remote.S3.remote + , Remote.Bup.remote , Remote.Directory.remote ] diff --git a/Remote/Bup.hs b/Remote/Bup.hs new file mode 100644 index 0000000000..ef34e2c635 --- /dev/null +++ b/Remote/Bup.hs @@ -0,0 +1,133 @@ +{- Using bup as a remote. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Bup (remote) where + +import IO +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad (unless) +import Control.Monad.State (liftIO) +import System.Process +import System.Exit + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import LocationLog +import Config +import Utility +import Messages +import Remote.Special + +remote :: RemoteType Annex +remote = RemoteType { + typename = "bup", + enumerate = findSpecialRemotes "bupremote", + generate = gen, + setup = bupSetup +} + +gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen r u c = do + cst <- remoteCost r expensiveRemoteCost + bupremote <- getConfig r "bupremote" (error "missing bupremote") + return $ this cst bupremote + where + this cst bupremote = Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store r bupremote, + retrieveKeyFile = retrieve bupremote, + removeKey = remove, + hasKey = checkPresent u, + hasKeyCheap = True, + config = c + } + +bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) +bupSetup u c = do + -- verify configuration is sane + let bupremote = case M.lookup "remote" c of + Nothing -> error "Specify remote=" + Just r -> r + case M.lookup "encryption" c of + Nothing -> error "Specify encryption=key or encryption=none" + Just "none" -> return () + Just _ -> error "encryption keys not yet supported" + + -- bup init will create the repository. + -- (If the repository already exists, bup init again appears safe.) + showNote "bup init" + ok <- bup "init" bupremote [] + unless ok $ error "bup init failed" + + -- The bup remote is stored in git config, as well as this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c "bupremote" bupremote + + return $ M.delete "directory" c + +bupParams :: String -> String -> [CommandParam] -> [CommandParam] +bupParams command bupremote params = + (Param command) : [Param "-r", Param bupremote] ++ params + +bup :: String -> String -> [CommandParam] -> Annex Bool +bup command bupremote params = do + showProgress -- make way for bup output + liftIO $ boolSystem "bup" $ bupParams command bupremote params + +store :: Git.Repo -> String -> Key -> Annex Bool +store r bupremote k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + o <- getConfig r "bup-split-options" "" + let os = map Param $ words o + bup "split" bupremote $ os ++ [Param "-n", Param (show k), File src] + +retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve bupremote k f = do + let params = bupParams "join" bupremote [Param $ show k] + ret <- liftIO $ try $ do + -- pipe bup's stdout directly to file + tofile <- openFile f WriteMode + p <- runProcess "bup" (toCommand params) + Nothing Nothing Nothing (Just tofile) Nothing + r <- waitForProcess p + case r of + ExitSuccess -> return True + _ -> return False + case ret of + Right r -> return r + Left e -> return False + +remove :: Key -> Annex Bool +remove _ = do + warning "content cannot be removed from bup remote" + return False + +{- Bup does not provide a way to tell if a given dataset is present + - in a bup repository. One way it to check if the git repository has + - a branch matching the name (as created by bup split -n). + - + - However, git-annex's ususal reasons for checking if a remote really + - has a key also don't really apply in the case of bup, since, short + - of deleting bup's git repository, data cannot be removed from it. + - + - So, trust git-annex's location log; if it says a bup repository has + - content, assume it's right. + -} +checkPresent :: UUID -> Key -> Annex (Either IOException Bool) +checkPresent u k = do + g <- Annex.gitRepo + liftIO $ try $ do + uuids <- keyLocations g k + return $ u `elem` uuids diff --git a/configure.hs b/configure.hs index d340f937d4..4ab3052395 100644 --- a/configure.hs +++ b/configure.hs @@ -15,6 +15,7 @@ tests = , TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" + , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] diff --git a/debian/changelog b/debian/changelog index 7f104be101..91c0c8f4b3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,6 @@ git-annex (0.20110402) UNRELEASED; urgency=low + * bup is now supported as a special type of remote. * Use lowercase hash directories for locationlog files, to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still diff --git a/debian/control b/debian/control index 37e6220437..15155b9b43 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync -Suggests: graphviz +Suggests: graphviz, bup Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 1a506c2811..7e1562d12e 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -6,13 +6,17 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup - initremote bup (init) ok + initremote bup (bup init) + Initialized empty Git repository in /big/mybup/ + ok # git annex describe mybup "my bup repository at example.com" describe mybup ok Now the remote can be used like any other remote. # git annex move my_cool_big_file --to mybup - move my_cool_big_file (to mybup...) ok + move my_cool_big_file (to mybup...) + Receiving index from server: 1100/1100, done. + ok See [[special_remotes/bup]] for details. From f2324c941567bb5ac67835fd3c3a82109bbac4be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:51:49 -0400 Subject: [PATCH 1476/8313] bupdates --- doc/special_remotes/bup.mdwn | 6 ++++++ doc/walkthrough/using_bup.mdwn | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 09eadd0085..950b64e9c0 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -5,6 +5,12 @@ interface to large files, and easy backups of the file contents using git. See [[walkthrough/using_bup]] for usage examples. +Each individual key is stored in a bup remote using `bup split`, with +a git branch named the same as the key name. Content is retrieved from +bup using `bup join`. All other bup operations are up to you -- consider +running `bup fsck --generate` in a cron job to generate recovery blocks, +for example; or clone bup's git repository to further back it up. + ## configuration These parameters can be passed to `git annex initremote` to configure bup: diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 7e1562d12e..12aa610372 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -19,4 +19,13 @@ Now the remote can be used like any other remote. Receiving index from server: 1100/1100, done. ok +Note that, unlike other remotes, bup does not really support removing +content from its git repositories. This is a feature. :) + + # git annex move my_cool_big_file --from mybup + move my_cool_big_file + content cannot be removed from bup remote + failed + git-annex: 1 failed + See [[special_remotes/bup]] for details. From 80b6c6d4bee93def8c398353ea45cb65155806aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:54:53 -0400 Subject: [PATCH 1477/8313] tweak --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 12aa610372..e84ff613b7 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -23,7 +23,7 @@ Note that, unlike other remotes, bup does not really support removing content from its git repositories. This is a feature. :) # git annex move my_cool_big_file --from mybup - move my_cool_big_file + move my_cool_big_file (from mybup...) content cannot be removed from bup remote failed git-annex: 1 failed From 1861a065b28160586811c88b571e4ce80b0e806c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 16:59:13 -0400 Subject: [PATCH 1478/8313] close --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index 34322d9811..d84e9b69b5 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -12,3 +12,5 @@ use all of git-annex's features everywhere else. >> of the git backend using bup. --[[Joey]] >>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH + +[[done]], the bup special remote type is written! --[[Joey]] From c90db9c93fbecc491498509ded7f3ab8249b8094 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 8 Apr 2011 20:59:37 +0000 Subject: [PATCH 1479/8313] Added a comment --- .../comment_5_3459f0b41d818c23c8fb33edb89df634._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment diff --git a/doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment b/doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment new file mode 100644 index 0000000000..a1300f2e64 --- /dev/null +++ b/doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-04-08T20:59:37Z" + content=""" +My estimates were pretty close -- the new bup special remote type took 133 lines of code, and 2 hours to write. A testament to the flexibility of the special remote infrastructure. :) +"""]] From f66b6f6360a448adc3e2ad501bd1e7673ebe0192 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:07:44 -0400 Subject: [PATCH 1480/8313] move esoteric stuff from walkthrough to cheatsheet --- doc/cheatsheet.mdwn | 14 ++++++++++++++ doc/walkthrough.mdwn | 15 +++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 doc/cheatsheet.mdwn diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn new file mode 100644 index 0000000000..4287756a6c --- /dev/null +++ b/doc/cheatsheet.mdwn @@ -0,0 +1,14 @@ +A suppliment to the [[walkthrough]]. + +[[!toc]] + +[[!inline feeds=no show=0 template=walkthrough pagenames=""" + walkthrough/using_Amazon_S3 + walkthrough/using_bup + walkthrough/using_the_URL_backend + walkthrough/using_the_SHA1_backend + walkthrough/migrating_data_to_a_new_backend + walkthrough/untrusted_repositories + walkthrough/what_to_do_when_you_lose_a_repository + walkthrough/recover_data_from_lost+found +"""]] diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index c648807494..ae305cb1be 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -14,16 +14,15 @@ A walkthrough of the basic features of git-annex. modifying_annexed_files using_ssh_remotes moving_file_content_between_repositories - using_Amazon_S3 - using_bup - using_the_URL_backend - using_the_SHA1_backend - migrating_data_to_a_new_backend unused_data fsck:_verifying_your_data fsck:_when_things_go_wrong backups - untrusted_repositories - what_to_do_when_you_lose_a_repository - recover_data_from_lost+found """]] + +---- + +So ends the walkthrough. By now you should be able to use git-annex. + +Want more? See the [[cheatsheet]] for info about all of git-annex's hidden +features. From dacf53dacc63c20f09ccf5089b543d7b17e4a99e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:09:50 -0400 Subject: [PATCH 1481/8313] reorg --- doc/walkthrough.mdwn | 8 +------- doc/walkthrough/more.mdwn | 4 ++++ 2 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 doc/walkthrough/more.mdwn diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn index ae305cb1be..eaae6b455c 100644 --- a/doc/walkthrough.mdwn +++ b/doc/walkthrough.mdwn @@ -18,11 +18,5 @@ A walkthrough of the basic features of git-annex. fsck:_verifying_your_data fsck:_when_things_go_wrong backups + more """]] - ----- - -So ends the walkthrough. By now you should be able to use git-annex. - -Want more? See the [[cheatsheet]] for info about all of git-annex's hidden -features. diff --git a/doc/walkthrough/more.mdwn b/doc/walkthrough/more.mdwn new file mode 100644 index 0000000000..1eaf9009f6 --- /dev/null +++ b/doc/walkthrough/more.mdwn @@ -0,0 +1,4 @@ +So ends the walkthrough. By now you should be able to use git-annex. + +Want more? See the [[cheatsheet]] for info about all of git-annex's hidden +features. From f3953d5d4f22932822a7c5dbd5085c03c5232b21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:12:29 -0400 Subject: [PATCH 1482/8313] reword --- doc/walkthrough/using_the_URL_backend.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn index 585fd0668a..50da4dad89 100644 --- a/doc/walkthrough/using_the_URL_backend.mdwn +++ b/doc/walkthrough/using_the_URL_backend.mdwn @@ -1,5 +1,5 @@ -git-annex has multiple key-value [[backends]]. So far this walkthrough has -demonstrated the default, WORM (Write Once, Read Many) backend. +git-annex has multiple key-value [[backends]]. So far you have been using +the default, WORM (Write Once, Read Many) backend. Another handy backend is the URL backend, which can fetch file's content from remote URLs. Here's how to set up some files in your repository From 280b9677d60c0f9c9b83063fa7edea72e0774421 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:13:50 -0400 Subject: [PATCH 1483/8313] expand --- doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn index 1159b22171..16a55b37b3 100644 --- a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn +++ b/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn @@ -15,4 +15,5 @@ To remind yourself later what happened, you can change its description, too: This retains the [[location_tracking]] information for the repository. Maybe you'll find the drive later. Maybe that's impossible. Either way, -this lets git-annex tell you why a file is no longer accessible. +this lets git-annex tell you why a file is no longer accessible, and +it avoids it relying on that drive to hold any content. From 9de1f5d9cfec3492ebee55fabc64f36a595d5be5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 17:15:14 -0400 Subject: [PATCH 1484/8313] typo --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index e84ff613b7..7ba9d1fce9 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -5,7 +5,7 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. - # git annex initremote mybup type=bup encryption=none remote=example.com/big/mybup + # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup initremote bup (bup init) Initialized empty Git repository in /big/mybup/ ok From 5bca5733fc674ed8244b5589855dde5c92df2934 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 21:51:17 +0000 Subject: [PATCH 1485/8313] typo --- doc/design/encryption.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 8a8f38108e..90b722b9b8 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -61,7 +61,7 @@ more gpg public keys. This scheme allows new gpg private keys to be given access to content that has already been stored in the remote. Different encrypted remotes need to be able to each use different ciphers. -Allowing multiple cipers to be used within a single remote would add a lot +Allowing multiple ciphers to be used within a single remote would add a lot of complexity, so is not planned to be supported. Instead, if you want a new cipher, create a new S3 bucket, or whatever. There does not seem to be much benefit to using the same cipher for From 59ffe8e12713b66b1efa303a6ca7ba51d8dee535 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 21:55:37 +0000 Subject: [PATCH 1486/8313] Added a comment --- .../comment_7_d18cf944352f8303799c86f2c0354e8e._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment diff --git a/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment b/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment new file mode 100644 index 0000000000..7d2ad5eba7 --- /dev/null +++ b/doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 7" + date="2011-04-08T21:55:36Z" + content=""" +http://xfs.org/index.php/XFS_FAQ#Q:_Performance:_mkfs.xfs_-n_size.3D64k_option +"""]] From bed6cc80b9bffeab677fc1d2124a2d76a230e031 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 22:02:41 +0000 Subject: [PATCH 1487/8313] Added a comment --- .../comment_2_fe735d728878d889ccd34ec12b3a7dea._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment new file mode 100644 index 0000000000..d00191f9d8 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 2" + date="2011-04-08T22:02:41Z" + content=""" +What if your files have the same prefix and it happens to be 100 chars long? This can not be solved within WORM, but as Joey pointed out, SHA* exists. +"""]] From 939c88787ad785a9d00e376b8417ab6a5c40b339 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 8 Apr 2011 22:05:26 +0000 Subject: [PATCH 1488/8313] --- doc/todo/add_a_git_backend.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todo/add_a_git_backend.mdwn b/doc/todo/add_a_git_backend.mdwn index d84e9b69b5..2b224710ed 100644 --- a/doc/todo/add_a_git_backend.mdwn +++ b/doc/todo/add_a_git_backend.mdwn @@ -14,3 +14,5 @@ use all of git-annex's features everywhere else. >>> Very much so. Generally speaking, having one or more versioned storage back-ends with current data in the local annexes sounds incredibly useful. Still being able to get at old data in via the back-end and/or making offline backups of the full history are excellent use cases. -- RichiH [[done]], the bup special remote type is written! --[[Joey]] + +> Yay! -- RichiH From 5d7ef05d95dc2a8e926a87cf552b119457231db0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:44:52 -0400 Subject: [PATCH 1489/8313] local bup remotes --- doc/special_remotes/bup.mdwn | 4 ++-- doc/walkthrough/using_bup.mdwn | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 950b64e9c0..4872072a03 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -22,8 +22,8 @@ These parameters can be passed to `git annex initremote` to configure bup: the new key id. * `remote` - Required. This is passed to `bup` as the `--remote` - to use to store data. `bup init` will be run to create the - repository. Example: "remote=example.com:/big/mybup" + to use to store data. To create the repository,`bup init` will be run. + Example: "remote=example.com:/big/mybup" or "remote=/big/mybup" Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 7ba9d1fce9..4ce189480f 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -5,6 +5,12 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. +[[!template id=note test=""" +Instead of specifying a remote system, you could choose to make a bup +remote that is only accessible on the current system, by passing +"remote=/big/mybup". +"""] + # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup initremote bup (bup init) Initialized empty Git repository in /big/mybup/ From 1bfd8d659ae23c3f8996974deda59a277a857b63 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:46:40 -0400 Subject: [PATCH 1490/8313] typo --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 4ce189480f..4ee2bbf088 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -9,7 +9,7 @@ Here's how to create a bup remote, and describe it. Instead of specifying a remote system, you could choose to make a bup remote that is only accessible on the current system, by passing "remote=/big/mybup". -"""] +"""]] # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup initremote bup (bup init) From 3650f42bcfb631e1ef6d53be725b72cbbfc249cd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:47:23 -0400 Subject: [PATCH 1491/8313] typo --- doc/walkthrough/using_bup.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index 4ee2bbf088..c3200dd320 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -5,7 +5,7 @@ git-annex, you can have git on both the frontend and the backend. Here's how to create a bup remote, and describe it. -[[!template id=note test=""" +[[!template id=note text=""" Instead of specifying a remote system, you could choose to make a bup remote that is only accessible on the current system, by passing "remote=/big/mybup". From c253d07a82f1ffac4aa256c09a6fd7dcbc780923 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 20:55:22 -0400 Subject: [PATCH 1492/8313] minor --- Remote/Bup.hs | 2 +- Remote/Git.hs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index ef34e2c635..5fbe9a8b5d 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -107,7 +107,7 @@ retrieve bupremote k f = do _ -> return False case ret of Right r -> return r - Left e -> return False + Left _ -> return False remove :: Key -> Annex Bool remove _ = do diff --git a/Remote/Git.hs b/Remote/Git.hs index 2936beaf7d..c315d457d6 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -57,9 +57,7 @@ gen r u _ = do u' <- getUUID r' - let defcst = if not $ Git.repoIsUrl r - then cheapRemoteCost - else expensiveRemoteCost + let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost cst <- remoteCost r' defcst return $ Remote { @@ -70,7 +68,7 @@ gen r u _ = do retrieveKeyFile = copyFromRemote r', removeKey = dropKey r', hasKey = inAnnex r', - hasKeyCheap = not (Git.repoIsUrl r'), + hasKeyCheap = cheap, config = Nothing } From 1bfd3922c06e6a6982a6ec78ea21432652e471d9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 21:37:59 -0400 Subject: [PATCH 1493/8313] set cost for local bup repos to cheap --- Remote/Bup.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 5fbe9a8b5d..6a7609aad9 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -37,8 +37,10 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do - cst <- remoteCost r expensiveRemoteCost bupremote <- getConfig r "bupremote" (error "missing bupremote") + let local = ':' `notElem` bupremote + cst <- remoteCost r (if local then cheapRemoteCost else expensiveRemoteCost) + return $ this cst bupremote where this cst bupremote = Remote { From c5174f0cb8a234dbd3656f108194a4e3dec6fec5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 8 Apr 2011 23:08:21 -0400 Subject: [PATCH 1494/8313] make local bup repos a bit more expensive than local git repos does have to run bup and reassemble files, after all --- Config.hs | 2 ++ Remote/Bup.hs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Config.hs b/Config.hs index 17a1fa9856..53f1a455fd 100644 --- a/Config.hs +++ b/Config.hs @@ -51,6 +51,8 @@ remoteCost r def = do cheapRemoteCost :: Int cheapRemoteCost = 100 +semiCheapRemoteCost :: Int +semiCheapRemoteCost = 150 expensiveRemoteCost :: Int expensiveRemoteCost = 200 diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 6a7609aad9..8d92792e19 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -39,7 +39,7 @@ gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do bupremote <- getConfig r "bupremote" (error "missing bupremote") let local = ':' `notElem` bupremote - cst <- remoteCost r (if local then cheapRemoteCost else expensiveRemoteCost) + cst <- remoteCost r (if local then semiCheapRemoteCost else expensiveRemoteCost) return $ this cst bupremote where From fdf1c648129042c4d61fedb93830a6d00e06ac21 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 11:13:01 -0400 Subject: [PATCH 1495/8313] notes --- doc/todo/branching.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn index 06bd50beed..79f278480e 100644 --- a/doc/todo/branching.mdwn +++ b/doc/todo/branching.mdwn @@ -115,3 +115,14 @@ would solve the merge problem, since git auto merge could be used. It would still mean all the log files are on-disk, which annoys some. It would require some tighter integration with git, so that after a pull, the log repo is updated with the data pulled. --[[Joey]] + +## notes + +Another approach could be to use git-notes. It supports merging branches +of notes, with union merge strategy (a hook would have to do this after +a pull, it's not done automatically). + +Problem: Notes are usually attached to git +objects, and there are no git objects corresponding to git-annex keys. + +Problem: Notes are not normally copied when cloning. From e7d30fe3da0530bce6e8498ecb9020bbbabccf43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 11:41:10 -0400 Subject: [PATCH 1496/8313] mention how to use default bup repo --- doc/special_remotes/bup.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 4872072a03..19f2320f8c 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -24,6 +24,7 @@ These parameters can be passed to `git annex initremote` to configure bup: * `remote` - Required. This is passed to `bup` as the `--remote` to use to store data. To create the repository,`bup init` will be run. Example: "remote=example.com:/big/mybup" or "remote=/big/mybup" + (To use the default `~/.bup` repository on the local host, specify "remote=") Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This From 141e55ff11394e2f162397957c96c02ad3f0bd37 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:34:49 -0400 Subject: [PATCH 1497/8313] store annex.uuid in bup repos --- GitRepo.hs | 6 ------ Remote/Bup.hs | 57 ++++++++++++++++++++++++++++++++++++++++++++++++--- Utility.hs | 9 ++++++++ 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 1b14e4a636..543ad801a4 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -573,12 +573,6 @@ repoAbsPath d = do h <- myHomeDir return $ h d' -myHomeDir :: IO FilePath -myHomeDir = do - uid <- getEffectiveUserID - u <- getUserEntryForID uid - return $ homeDirectory u - expandTilde :: FilePath -> IO FilePath expandTilde = expandt True where diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 8d92792e19..d43b03a92d 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -10,10 +10,12 @@ module Remote.Bup (remote) where import IO import Control.Exception.Extensible (IOException) import qualified Data.Map as M -import Control.Monad (unless) +import Control.Monad (unless, when) import Control.Monad.State (liftIO) import System.Process import System.Exit +import System.FilePath +import Data.List.Utils import RemoteClass import Types @@ -26,6 +28,7 @@ import Config import Utility import Messages import Remote.Special +import Ssh remote :: RemoteType Annex remote = RemoteType { @@ -38,8 +41,7 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do bupremote <- getConfig r "bupremote" (error "missing bupremote") - let local = ':' `notElem` bupremote - cst <- remoteCost r (if local then semiCheapRemoteCost else expensiveRemoteCost) + cst <- remoteCost r (if bupLocal bupremote then semiCheapRemoteCost else expensiveRemoteCost) return $ this cst bupremote where @@ -72,6 +74,8 @@ bupSetup u c = do ok <- bup "init" bupremote [] unless ok $ error "bup init failed" + storeBupUUID u bupremote + -- The bup remote is stored in git config, as well as this remote's -- persistant state, so it can vary between hosts. gitConfigSpecialRemote u c "bupremote" bupremote @@ -133,3 +137,50 @@ checkPresent u k = do liftIO $ try $ do uuids <- keyLocations g k return $ u `elem` uuids + +{- Store UUID in the annex.uuid setting of the bup repository. -} +storeBupUUID :: UUID -> FilePath -> Annex () +storeBupUUID u bupremote = do + r <- liftIO $ bup2GitRemote bupremote + if Git.repoIsUrl r + then do + showNote "storing uuid" + let dir = shellEscape (Git.workTree r) + sshparams <- sshToRepo r + [Param $ "cd " ++ dir ++ + " && git config annex.uuid " ++ u] + ok <- liftIO $ boolSystem "ssh" sshparams + unless ok $ do error "ssh failed" + else liftIO $ do + r' <- Git.configRead r + let olduuid = Git.configGet r' "annex.uuid" "" + when (olduuid == "") $ + Git.run r' "config" [Param "annex.uuid", Param u] + +{- Converts a bup remote path spec into a Git.Repo. There are some + - differences in path representation between git and bup. -} +bup2GitRemote :: FilePath -> IO Git.Repo +bup2GitRemote "" = do + -- bup -r "" operates on ~/.bup + h <- myHomeDir + Git.repoFromAbsPath $ h ".bup" +bup2GitRemote r + | bupLocal r = + if r !! 0 == '/' + then Git.repoFromAbsPath r + else error "please specify an absolute path" + | otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir + where + bits = split ":" r + host = bits !! 0 + dir = join ":" $ drop 1 bits + -- "host:~user/dir" is not supported specially by bup; + -- "host:dir" is relative to the home directory; + -- "host:" goes in ~/.bup + slash d + | d == "" = "/~/.bup" + | d !! 0 == '/' = d + | otherwise = "/~/" ++ d + +bupLocal :: FilePath -> Bool +bupLocal = notElem ':' diff --git a/Utility.hs b/Utility.hs index 72f5c50638..1c6b4d21e6 100644 --- a/Utility.hs +++ b/Utility.hs @@ -23,6 +23,7 @@ module Utility ( safeWriteFile, dirContains, dirContents, + myHomeDir, prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, @@ -36,6 +37,7 @@ import System.Posix.Process import System.Posix.Signals import System.Posix.Files import System.Posix.Types +import System.Posix.User import Data.String.Utils import System.Path import System.FilePath @@ -247,3 +249,10 @@ dirContents d = do notcruft "." = False notcruft ".." = False notcruft _ = True + +{- Current user's home directory. -} +myHomeDir :: IO FilePath +myHomeDir = do + uid <- getEffectiveUserID + u <- getUserEntryForID uid + return $ homeDirectory u From c739c7d7872557ca0c5d3518fb0a3bdbade871eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:41:17 -0400 Subject: [PATCH 1498/8313] change name of buprepo Instead of remote=, use buprepo= Anyone already using bup will need to re-run git annex initremote. --- Remote/Bup.hs | 54 +++++++++++++++++++--------------- doc/special_remotes/bup.mdwn | 6 ++-- doc/walkthrough/using_bup.mdwn | 4 +-- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index d43b03a92d..b6d08d89bc 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -33,26 +33,27 @@ import Ssh remote :: RemoteType Annex remote = RemoteType { typename = "bup", - enumerate = findSpecialRemotes "bupremote", + enumerate = findSpecialRemotes "buprepo", generate = gen, setup = bupSetup } gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do - bupremote <- getConfig r "bupremote" (error "missing bupremote") - cst <- remoteCost r (if bupLocal bupremote then semiCheapRemoteCost else expensiveRemoteCost) + buprepo <- getConfig r "buprepo" (error "missing buprepo") + cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) +-- u' <- getBupUUID r u - return $ this cst bupremote + return $ this cst buprepo u' where - this cst bupremote = Remote { - uuid = u, + this cst buprepo u' = Remote { + uuid = u', cost = cst, name = Git.repoDescribe r, - storeKey = store r bupremote, - retrieveKeyFile = retrieve bupremote, + storeKey = store r buprepo, + retrieveKeyFile = retrieve buprepo, removeKey = remove, - hasKey = checkPresent u, + hasKey = checkPresent u', hasKeyCheap = True, config = c } @@ -60,7 +61,7 @@ gen r u c = do bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) bupSetup u c = do -- verify configuration is sane - let bupremote = case M.lookup "remote" c of + let buprepo = case M.lookup "remote" c of Nothing -> error "Specify remote=" Just r -> r case M.lookup "encryption" c of @@ -71,37 +72,37 @@ bupSetup u c = do -- bup init will create the repository. -- (If the repository already exists, bup init again appears safe.) showNote "bup init" - ok <- bup "init" bupremote [] + ok <- bup "init" buprepo [] unless ok $ error "bup init failed" - storeBupUUID u bupremote + storeBupUUID u buprepo - -- The bup remote is stored in git config, as well as this remote's + -- The buprepo is stored in git config, as well as this repo's -- persistant state, so it can vary between hosts. - gitConfigSpecialRemote u c "bupremote" bupremote + gitConfigSpecialRemote u c "buprepo" buprepo return $ M.delete "directory" c bupParams :: String -> String -> [CommandParam] -> [CommandParam] -bupParams command bupremote params = - (Param command) : [Param "-r", Param bupremote] ++ params +bupParams command buprepo params = + (Param command) : [Param "-r", Param buprepo] ++ params bup :: String -> String -> [CommandParam] -> Annex Bool -bup command bupremote params = do +bup command buprepo params = do showProgress -- make way for bup output - liftIO $ boolSystem "bup" $ bupParams command bupremote params + liftIO $ boolSystem "bup" $ bupParams command buprepo params store :: Git.Repo -> String -> Key -> Annex Bool -store r bupremote k = do +store r buprepo k = do g <- Annex.gitRepo let src = gitAnnexLocation g k o <- getConfig r "bup-split-options" "" let os = map Param $ words o - bup "split" bupremote $ os ++ [Param "-n", Param (show k), File src] + bup "split" buprepo $ os ++ [Param "-n", Param (show k), File src] retrieve :: String -> Key -> FilePath -> Annex Bool -retrieve bupremote k f = do - let params = bupParams "join" bupremote [Param $ show k] +retrieve buprepo k f = do + let params = bupParams "join" buprepo [Param $ show k] ret <- liftIO $ try $ do -- pipe bup's stdout directly to file tofile <- openFile f WriteMode @@ -140,8 +141,8 @@ checkPresent u k = do {- Store UUID in the annex.uuid setting of the bup repository. -} storeBupUUID :: UUID -> FilePath -> Annex () -storeBupUUID u bupremote = do - r <- liftIO $ bup2GitRemote bupremote +storeBupUUID u buprepo = do + r <- liftIO $ bup2GitRemote buprepo if Git.repoIsUrl r then do showNote "storing uuid" @@ -157,6 +158,11 @@ storeBupUUID u bupremote = do when (olduuid == "") $ Git.run r' "config" [Param "annex.uuid", Param u] +{- Allow for bup repositories on removable media by checking + - local bup repositories -} +--getBupUUID :: UUID -> FilePath -> Annex () +--getBupUUID u buprepo = do + {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} bup2GitRemote :: FilePath -> IO Git.Repo diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 19f2320f8c..c74bdaf7e2 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -21,10 +21,10 @@ These parameters can be passed to `git annex initremote` to configure bup: keys can be given access to a remote by rerunning initremote with the new key id. -* `remote` - Required. This is passed to `bup` as the `--remote` +* `buprepo` - Required. This is passed to `bup` as the `--remote` to use to store data. To create the repository,`bup init` will be run. - Example: "remote=example.com:/big/mybup" or "remote=/big/mybup" - (To use the default `~/.bup` repository on the local host, specify "remote=") + Example: "buprepo=example.com:/big/mybup" or "buprepo=/big/mybup" + (To use the default `~/.bup` repository on the local host, specify "buprepo=") Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This diff --git a/doc/walkthrough/using_bup.mdwn b/doc/walkthrough/using_bup.mdwn index c3200dd320..3a6a8776aa 100644 --- a/doc/walkthrough/using_bup.mdwn +++ b/doc/walkthrough/using_bup.mdwn @@ -8,10 +8,10 @@ Here's how to create a bup remote, and describe it. [[!template id=note text=""" Instead of specifying a remote system, you could choose to make a bup remote that is only accessible on the current system, by passing -"remote=/big/mybup". +"buprepo=/big/mybup". """]] - # git annex initremote mybup type=bup encryption=none remote=example.com:/big/mybup + # git annex initremote mybup type=bup encryption=none buprepo=example.com:/big/mybup initremote bup (bup init) Initialized empty Git repository in /big/mybup/ ok From f808a8335064d22fac1f9b7ee6c32a6680dd6cbc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:45:30 -0400 Subject: [PATCH 1499/8313] more buprepo fixes --- Remote/Bup.hs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index b6d08d89bc..fdc3919e23 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -42,7 +42,7 @@ gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do buprepo <- getConfig r "buprepo" (error "missing buprepo") cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) --- u' <- getBupUUID r u + u' <- getBupUUID buprepo u return $ this cst buprepo u' where @@ -61,8 +61,8 @@ gen r u c = do bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) bupSetup u c = do -- verify configuration is sane - let buprepo = case M.lookup "remote" c of - Nothing -> error "Specify remote=" + let buprepo = case M.lookup "buprepo" c of + Nothing -> error "Specify buprepo=" Just r -> r case M.lookup "encryption" c of Nothing -> error "Specify encryption=key or encryption=none" @@ -160,8 +160,9 @@ storeBupUUID u buprepo = do {- Allow for bup repositories on removable media by checking - local bup repositories -} ---getBupUUID :: UUID -> FilePath -> Annex () ---getBupUUID u buprepo = do +getBupUUID :: FilePath -> UUID -> Annex UUID +getBupUUID buprepo u = do + return u -- TODO {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} From 54286c993dab913515b12ae8d87c2944ea11e6b2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 12:59:18 -0400 Subject: [PATCH 1500/8313] support bup repositories on removable media --- Remote/Bup.hs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index fdc3919e23..dc653631d5 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -159,10 +159,23 @@ storeBupUUID u buprepo = do Git.run r' "config" [Param "annex.uuid", Param u] {- Allow for bup repositories on removable media by checking - - local bup repositories -} + - local bup repositories to see if they are available, and getting their + - uuid (which may be different from the stored uuid for the bup remote). + - + - If a bup repository is not available, returns a dummy uuid of "". + - This will cause checkPresent to indicate nothing from the bup remote + - is known to be present. + -} getBupUUID :: FilePath -> UUID -> Annex UUID -getBupUUID buprepo u = do - return u -- TODO +getBupUUID buprepo u = liftIO $ do + r <- bup2GitRemote buprepo + if Git.repoIsUrl r + then return u + else do + ret <- try $ Git.configRead r + case ret of + Right r' -> return $ Git.configGet r' "annex.uuid" "" + Left _ -> return "" {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} From 40a7ed859c55df12d5da99e596171d27bfcb5ee2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 13:10:01 -0400 Subject: [PATCH 1501/8313] link --- doc/index.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/index.mdwn b/doc/index.mdwn index b5880823e1..224d25d2cc 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -43,8 +43,9 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage -* [[internals]] +* [[special_remotes]] (including [[special_remotes/Amazon_S3]] and [[special_remotes/bup]]) * [[bare_repositories]] +* [[internals]] * [[design]] * [[what git annex is not|not]] * git-annex is Free Software, licensed under the [[GPL]]. From cdea1a6b3a5310287479e0ee580dccbf352822bb Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 9 Apr 2011 17:15:01 +0000 Subject: [PATCH 1502/8313] Comment moderation --- .../comment_1_d8e34fc2bc4e5cf761574608f970d496._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment diff --git a/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment new file mode 100644 index 0000000000..4aceb3abd3 --- /dev/null +++ b/doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 1" + date="2011-04-08T12:41:43Z" + content=""" +I also think, that fetching keys via rsync can be done by one rsync process, when the keys are fetched from one host. This would avoid establishing a new TCP connection for every file. +"""]] From 1e7ad2ee7c63c02c8f6b9df465576c8245bba8a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 14:15:38 -0400 Subject: [PATCH 1503/8313] reformat to work around man page conversion bugs --- doc/git-annex-shell.mdwn | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index b32114a74d..060d23ca18 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -17,6 +17,8 @@ user's restricted login shell. # COMMANDS +Any command not listed below is passed through to git-shell. + * configlist directory This outputs a subset of the git configuration, in the same form as @@ -40,16 +42,14 @@ user's restricted login shell. This runs rsync in server mode to transfer out the content of a key. +## OPTIONS + +Same as git-annex or git-shell, depending on the command being run. + Note that the directory parameter should be an absolute path, otherwise it is assumed to be relative to the user's home directory. Also the first "/~/" or "/~user/" is expanded to the specified home directory. -Any other command is passed through to git-shell. - -# OPTIONS - -Same as git-annex or git-shell, depending on the command being run. - # SEE ALSO [[git-annex]](1) From 8ad901a647a9c7cf179dc2dd73d121adc43a28fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 14:26:32 -0400 Subject: [PATCH 1504/8313] refactor --- Command/Map.hs | 3 +-- Remote/Git.hs | 36 +----------------------------------- Ssh.hs | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Command/Map.hs b/Command/Map.hs index dc3acb56e4..2325c87e14 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -16,7 +16,6 @@ import Data.List.Utils import Command import qualified Annex import qualified GitRepo as Git -import qualified Remote.Git import Messages import Types import Utility @@ -203,7 +202,7 @@ tryScan r Git.hConfigRead r configlist = - Remote.Git.onRemote r (pipedconfig, Nothing) "configlist" [] + onRemote r (pipedconfig, Nothing) "configlist" [] manualconfiglist = do let sshcmd = "cd " ++ shellEscape(Git.workTree r) ++ " && " ++ diff --git a/Remote/Git.hs b/Remote/Git.hs index c315d457d6..7724df79af 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -5,10 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Git ( - remote, - onRemote -) where +module Remote.Git (remote) where import Control.Exception.Extensible import Control.Monad.State (liftIO) @@ -194,34 +191,3 @@ rsyncParams r sending key file = do -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. dummy = Param ":" - -{- Uses a supplied function to run a git-annex-shell command on a remote. - - - - Or, if the remote does not support running remote commands, returns - - a specified error value. -} -onRemote - :: Git.Repo - -> (FilePath -> [CommandParam] -> IO a, a) - -> String - -> [CommandParam] - -> Annex a -onRemote r (with, errorval) command params = do - s <- git_annex_shell r command params - case s of - Just (c, ps) -> liftIO $ with c ps - Nothing -> return errorval - -{- Generates parameters to run a git-annex-shell command on a remote. -} -git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) -git_annex_shell r command params - | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) - | Git.repoIsSsh r = do - sshparams <- sshToRepo r [Param sshcmd] - return $ Just ("ssh", sshparams) - | otherwise = return Nothing - where - dir = Git.workTree r - shellcmd = "git-annex-shell" - shellopts = (Param command):(File dir):params - sshcmd = shellcmd ++ " " ++ - unwords (map shellEscape $ toCommand shellopts) diff --git a/Ssh.hs b/Ssh.hs index 6d01a56423..0cf2919c25 100644 --- a/Ssh.hs +++ b/Ssh.hs @@ -7,6 +7,8 @@ module Ssh where +import Control.Monad.State (liftIO) + import qualified GitRepo as Git import Utility import Types @@ -24,3 +26,36 @@ sshToRepo repo sshcmd = do Just p -> [Param "-p", Param (show p)] let sshhost = Param $ Git.urlHostUser repo return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd + +{- Generates parameters to run a git-annex-shell command on a remote + - repository. -} +git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) +git_annex_shell r command params + | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) + | Git.repoIsSsh r = do + sshparams <- sshToRepo r [Param sshcmd] + return $ Just ("ssh", sshparams) + | otherwise = return Nothing + where + dir = Git.workTree r + shellcmd = "git-annex-shell" + shellopts = (Param command):(File dir):params + sshcmd = shellcmd ++ " " ++ + unwords (map shellEscape $ toCommand shellopts) + +{- Uses a supplied function (such as boolSystem) to run a git-annex-shell + - command on a remote. + - + - Or, if the remote does not support running remote commands, returns + - a specified error value. -} +onRemote + :: Git.Repo + -> (FilePath -> [CommandParam] -> IO a, a) + -> String + -> [CommandParam] + -> Annex a +onRemote r (with, errorval) command params = do + s <- git_annex_shell r command params + case s of + Just (c, ps) -> liftIO $ with c ps + Nothing -> return errorval From ede234136b38e2039f9f056a6c05b10c65a07b51 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 14:40:14 -0400 Subject: [PATCH 1505/8313] bup and git-annex-shell Looked at having git-annex shell support running bup server.. Then looked at how bup runs bup server over ssh, and fled away, screaming.. --- doc/special_remotes/bup.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index c74bdaf7e2..90b84e9f47 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -30,6 +30,11 @@ Options to pass to `bup split` when sending content to bup can also be specified, by using `git config annex.bup-split-options`. This can be used to, for example, limit its bandwidth. +## notes + +[[git-annex-shell]] does not support bup, due to the wacky way that bup +starts its server. So, to use bup, you need full shell access to the server. + ## data security When encryption=none, there is **no** protection against your data being read From 66950189fcfc9d005c5c2d13ce2060a815362b6e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 15:36:54 -0400 Subject: [PATCH 1506/8313] actually check that bup has keys I don't trust the location log, even for bup. Too many things could go wrong. --- Remote/Bup.hs | 84 +++++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index dc653631d5..916afeb406 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -23,13 +23,14 @@ import qualified GitRepo as Git import qualified Annex import UUID import Locations -import LocationLog import Config import Utility import Messages import Remote.Special import Ssh +type BupRepo = String + remote :: RemoteType Annex remote = RemoteType { typename = "bup", @@ -42,18 +43,19 @@ gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) gen r u c = do buprepo <- getConfig r "buprepo" (error "missing buprepo") cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) - u' <- getBupUUID buprepo u + bupr <- liftIO $ bup2GitRemote buprepo + (u', bupr') <- getBupUUID bupr u - return $ this cst buprepo u' + return $ this cst buprepo u' bupr' where - this cst buprepo u' = Remote { + this cst buprepo u' bupr = Remote { uuid = u', cost = cst, name = Git.repoDescribe r, storeKey = store r buprepo, retrieveKeyFile = retrieve buprepo, removeKey = remove, - hasKey = checkPresent u', + hasKey = checkPresent r bupr, hasKeyCheap = True, config = c } @@ -83,16 +85,16 @@ bupSetup u c = do return $ M.delete "directory" c -bupParams :: String -> String -> [CommandParam] -> [CommandParam] +bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] bupParams command buprepo params = (Param command) : [Param "-r", Param buprepo] ++ params -bup :: String -> String -> [CommandParam] -> Annex Bool +bup :: String -> BupRepo -> [CommandParam] -> Annex Bool bup command buprepo params = do showProgress -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params -store :: Git.Repo -> String -> Key -> Annex Bool +store :: Git.Repo -> BupRepo -> Key -> Annex Bool store r buprepo k = do g <- Annex.gitRepo let src = gitAnnexLocation g k @@ -100,7 +102,7 @@ store r buprepo k = do let os = map Param $ words o bup "split" buprepo $ os ++ [Param "-n", Param (show k), File src] -retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do let params = bupParams "join" buprepo [Param $ show k] ret <- liftIO $ try $ do @@ -124,33 +126,28 @@ remove _ = do {- Bup does not provide a way to tell if a given dataset is present - in a bup repository. One way it to check if the git repository has - a branch matching the name (as created by bup split -n). - - - - However, git-annex's ususal reasons for checking if a remote really - - has a key also don't really apply in the case of bup, since, short - - of deleting bup's git repository, data cannot be removed from it. - - - - So, trust git-annex's location log; if it says a bup repository has - - content, assume it's right. -} -checkPresent :: UUID -> Key -> Annex (Either IOException Bool) -checkPresent u k = do - g <- Annex.gitRepo - liftIO $ try $ do - uuids <- keyLocations g k - return $ u `elem` uuids +checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool) +checkPresent r bupr k + | Git.repoIsUrl bupr = do + showNote ("checking " ++ Git.repoDescribe r ++ "...") + ok <- onBupRemote bupr boolSystem "git" params + return $ Right ok + | otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params + where + params = + [ Params "show-ref --quiet --verify" + , Param $ "refs/heads/" ++ show k] {- Store UUID in the annex.uuid setting of the bup repository. -} -storeBupUUID :: UUID -> FilePath -> Annex () +storeBupUUID :: UUID -> BupRepo -> Annex () storeBupUUID u buprepo = do r <- liftIO $ bup2GitRemote buprepo if Git.repoIsUrl r then do showNote "storing uuid" - let dir = shellEscape (Git.workTree r) - sshparams <- sshToRepo r - [Param $ "cd " ++ dir ++ - " && git config annex.uuid " ++ u] - ok <- liftIO $ boolSystem "ssh" sshparams + ok <- onBupRemote r boolSystem "git" + [Params $ "config annex.uuid " ++ u] unless ok $ do error "ssh failed" else liftIO $ do r' <- Git.configRead r @@ -158,6 +155,13 @@ storeBupUUID u buprepo = do when (olduuid == "") $ Git.run r' "config" [Param "annex.uuid", Param u] +onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a +onBupRemote r a command params = do + let dir = shellEscape (Git.workTree r) + sshparams <- sshToRepo r [Param $ + "cd " ++ dir ++ " && " ++ (unwords $ command : toCommand params)] + liftIO $ a "ssh" sshparams + {- Allow for bup repositories on removable media by checking - local bup repositories to see if they are available, and getting their - uuid (which may be different from the stored uuid for the bup remote). @@ -165,21 +169,21 @@ storeBupUUID u buprepo = do - If a bup repository is not available, returns a dummy uuid of "". - This will cause checkPresent to indicate nothing from the bup remote - is known to be present. + - + - Also, returns a version of the repo with config read, if it is local. -} -getBupUUID :: FilePath -> UUID -> Annex UUID -getBupUUID buprepo u = liftIO $ do - r <- bup2GitRemote buprepo - if Git.repoIsUrl r - then return u - else do - ret <- try $ Git.configRead r - case ret of - Right r' -> return $ Git.configGet r' "annex.uuid" "" - Left _ -> return "" +getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo) +getBupUUID r u + | Git.repoIsUrl r = return (u, r) + | otherwise = liftIO $ do + ret <- try $ Git.configRead r + case ret of + Right r' -> return (Git.configGet r' "annex.uuid" "", r') + Left _ -> return ("", r) {- Converts a bup remote path spec into a Git.Repo. There are some - differences in path representation between git and bup. -} -bup2GitRemote :: FilePath -> IO Git.Repo +bup2GitRemote :: BupRepo -> IO Git.Repo bup2GitRemote "" = do -- bup -r "" operates on ~/.bup h <- myHomeDir @@ -202,5 +206,5 @@ bup2GitRemote r | d !! 0 == '/' = d | otherwise = "/~/" ++ d -bupLocal :: FilePath -> Bool +bupLocal :: BupRepo -> Bool bupLocal = notElem ':' From dbea472f06b806248a6501163eccf4279c058f75 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 15:57:45 -0400 Subject: [PATCH 1507/8313] update --- doc/git-annex.mdwn | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3514002a49..3e91e7ad92 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -134,7 +134,7 @@ Many git-annex commands will stage changes for later `git commit` by you. * initremote name [param=value ...] - Sets up a [[special_remote|special_remotes]]. The remote's + Sets up a special remote. The remote's configuration is specified by the parameters. If a remote with the specified name has already been configured, its configuration is modified by any values specified. In either case, the remote will be @@ -244,19 +244,19 @@ Many git-annex commands will stage changes for later `git commit` by you. * trust [repository ...] - Records that a repository is [[trusted|trust]] to not unexpectedly lose + Records that a repository is trusted to not unexpectedly lose content. Use with care. To trust the current repository, use "." * untrust [repository ...] - Records that a repository is [[not trusted|trust]] and could lose content + Records that a repository is not trusted and could lose content at any time. * semitrust [repository ...] - Returns a repository to the default [[semi trusted|trust]] state. + Returns a repository to the default semi trusted state. * fromkey file @@ -384,7 +384,7 @@ Here are all the supported configuration settings. by the --from and --to options.) This is, for example, useful if the remote is located somewhere - without [[git-annex-shell]]. (For example, if it's on GitHub). + without git-annex-shell. (For example, if it's on GitHub). Or, it could be used if the network connection between two repositories is too slow to be used normally. @@ -427,7 +427,23 @@ Here are all the supported configuration settings. Automatically maintained, and used to automate upgrades between versions. - +* `remote..buprepo` + + Used by bup special remotes, this configures + the location of the bup repository to use. Normally this is automaticaly + set up by `git annex initremote`, but you can change it if needed. + +* `remote..directory` + + Used by directory special remotes, this configures + the location of the directory where annexed files are stored for this + remote. Normally this is automaticaly set up by `git annex initremote`, + but you can change it if needed. + +* `remote..s3` + + Used to identify Amazon S3 special remotes. + Normally this is automaticaly set up by `git annex initremote`. # CONFIGURATION VIA .gitattributes From c4bdc59da013edb5723447bcdc9bfa7241cba337 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 16:04:12 -0400 Subject: [PATCH 1508/8313] forwarded --- ...vacuate__40__static__41__:_strange_closure_type_30799.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn index 87a0435208..f9a61a8590 100644 --- a/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn +++ b/doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn @@ -69,3 +69,7 @@ Thanks! > > Probably your best bet will be changing to a different version or build of > GHC.. --[[Joey]] + +--- + +forwarded to GHC upstream; closing [[done]] --[[Joey]] From de14252f780fb0259dbd9e6f1fa71c1f092a2135 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 16:08:07 -0400 Subject: [PATCH 1509/8313] rainy day bug maintenance Sitting on the porch, enjoying a thunderstorm, what else to do? --- doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 3 +++ doc/bugs/annex_add_in_annex.mdwn | 4 ++++ doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn | 7 +++---- .../touch.hsc_has_problems_on_non-linux_based_systems.mdwn | 2 ++ 4 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 doc/bugs/annex_add_in_annex.mdwn diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn index f90eb5ae3c..f256b0ed1f 100644 --- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn +++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn @@ -24,3 +24,6 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon >>> be displayed when there is no configured remote corresponding to the >>> repository. So this is not a bug unless some documentation of that is >>> unclear. --[[Joey]] + +>>>> Nobody spoke up to say it's unclear, so closing as PEBKAC :) +>>>> [[done]] --[[Joey]] diff --git a/doc/bugs/annex_add_in_annex.mdwn b/doc/bugs/annex_add_in_annex.mdwn new file mode 100644 index 0000000000..80084cec12 --- /dev/null +++ b/doc/bugs/annex_add_in_annex.mdwn @@ -0,0 +1,4 @@ +I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories. + +> There is a guard against `git annex add .git-annex/foo`, but it doesn't +> notice `cd .git-annex; git annex add foo`. --[[Joey]] diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn index e5937fe83f..db6a35293c 100644 --- a/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn +++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn @@ -95,7 +95,6 @@ new file mode 100644 index 0000000..e69de29 -Also I came across this when I accidentally annexed some files in the .git-annex directory and it cause git-annex/git to be very unhappy when i pulled the repo to somewhere else. It might be worth teaching git-annex to disallow annex'ing of files inside the .git-annex/.git directories. - -> There is a guard against `git annex add .git-annex/foo`, but it doesn't -> notice `cd .git-annex; git annex add foo`. --[[Joey]] +> Closing this bug, as it seems I have dealt with it adequately now. +> [[done]] +> --[[Joey]] diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn index 20c16e3cfa..c447f75f0c 100644 --- a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn +++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn @@ -15,3 +15,5 @@ make: *** [Touch.hs] Error 1 I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex. + +[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"] From ad7f87880e0aca651162e8960b398f6c8d52e7a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 9 Apr 2011 16:12:32 -0400 Subject: [PATCH 1510/8313] move wishlists to todo --- ...a___34__git_annex__34___command_that_will_skip_duplicates.mdwn | 0 .../comment_1_fd213310ee548d8726791d2b02237fde._comment | 0 .../comment_2_4394bde1c6fd44acae649baffe802775._comment | 0 .../wishlist:___34__git_annex_add__34___multiple_processes.mdwn | 0 .../comment_1_85b14478411a33e6186a64bd41f0910d._comment | 0 .../comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment | 0 .../comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment | 0 doc/{bugs => todo}/wishlist:_support_for_more_ssh_urls_.mdwn | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename doc/{bugs => todo}/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn (100%) rename doc/{bugs => todo}/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment (100%) rename doc/{bugs => todo}/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes.mdwn (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment (100%) rename doc/{bugs => todo}/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment (100%) rename doc/{bugs => todo}/wishlist:_support_for_more_ssh_urls_.mdwn (100%) diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn similarity index 100% rename from doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn rename to doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment similarity index 100% rename from doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment rename to doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment diff --git a/doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment similarity index 100% rename from doc/bugs/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment rename to doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes.mdwn rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment diff --git a/doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment similarity index 100% rename from doc/bugs/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment rename to doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment diff --git a/doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn similarity index 100% rename from doc/bugs/wishlist:_support_for_more_ssh_urls_.mdwn rename to doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn From 74e877357d1f03827dc1fac438bf8ed8af0d47ad Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 9 Apr 2011 20:11:59 +0000 Subject: [PATCH 1511/8313] Added a comment --- ...comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment new file mode 100644 index 0000000000..d9c291b178 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-09T20:11:59Z" + content=""" +I wouldn't say it's completly impossible for a WORM100 to work. It would just have the contract that the pair of mtime+100chars has to be unique for each unique piece of data. + +But, I have yet to be convinced there's any point, since SHA1 exists. +"""]] From 29022a4143888bcae9b35cba427568f5da33c7da Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 9 Apr 2011 23:43:15 +0000 Subject: [PATCH 1512/8313] --- doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn index f256b0ed1f..722dac50b8 100644 --- a/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn +++ b/doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn @@ -27,3 +27,5 @@ I can create an annex remote named 'test:/test'. git itself does not allow colon >>>> Nobody spoke up to say it's unclear, so closing as PEBKAC :) >>>> [[done]] --[[Joey]] + +>>>>> I still think git-annex should follow the same rules as git in this regard, but if your design decision is different, I won't try to argue the point :) -- RichiH From 1880563b02b2101fd5d6cc23a2b02c169e91a506 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sat, 9 Apr 2011 23:45:29 +0000 Subject: [PATCH 1513/8313] Added a comment --- .../comment_4_8f7ba9372463863dda5aae13205861bf._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment new file mode 100644 index 0000000000..5c08cad6e0 --- /dev/null +++ b/doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-09T23:45:28Z" + content=""" +mtime+100chars can still get collisions and a _lot_ easier than even SHA1. This introduces more problems that it solves, imo. +"""]] From da05f77f7a0873cad2a380461c6e46832624fd3d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" Date: Wed, 13 Apr 2011 17:53:27 +0000 Subject: [PATCH 1514/8313] Added a comment --- .../comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment diff --git a/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment b/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment new file mode 100644 index 0000000000..d6bba93651 --- /dev/null +++ b/doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnx8kHW66N3BqmkVpgtXDlYMvr8TJ5VvfY" + nickname="Yaroslav" + subject="comment 4" + date="2011-04-13T17:53:26Z" + content=""" +.1 cents: Having IRC would be really nice for seeking quick help. E.g. like I was trying to do now, google lead me to this page. +"""]] From 279a8549fcbfd7945b4b87c62f85bfb9058c1ac2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 13 Apr 2011 16:00:17 -0400 Subject: [PATCH 1515/8313] analized and closed --- doc/bugs/weird_local_clone_confuses.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn index d209dd80de..371f6d9a5e 100644 --- a/doc/bugs/weird_local_clone_confuses.mdwn +++ b/doc/bugs/weird_local_clone_confuses.mdwn @@ -11,4 +11,8 @@ just sees a directory w/o a .git subdir, and gives up. --- Just tested, and the new support for bare repositories didn't solve this. +(Because config.bare is not set.) + +I think this is not something git-annex should go out of its way to +support. [[done]] --[[Joey]] From 268f2dc710ad433f3ea622ff1422a86250312571 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 13 Apr 2011 22:21:47 +0000 Subject: [PATCH 1516/8313] --- doc/forum/bainstorming:_git_annex_push___38___pull.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn b/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn index a2d320e352..8a6c552b80 100644 --- a/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn +++ b/doc/forum/bainstorming:_git_annex_push___38___pull.mdwn @@ -19,3 +19,10 @@ which would do git push Resulting in commands that are totally analogous to git push & pull: Sync all data from/to a remote. + +> Update: + +This is useful: + + git config [--global] alias.annex-push '!git pull && git annex add . && git annex copy . --to $REMOTE --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push' + From a9ecebedf144c2e2190baab0f895624a144d6a36 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 13 Apr 2011 22:38:55 +0000 Subject: [PATCH 1517/8313] --- doc/forum/wishlist:alias_system.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/wishlist:alias_system.mdwn diff --git a/doc/forum/wishlist:alias_system.mdwn b/doc/forum/wishlist:alias_system.mdwn new file mode 100644 index 0000000000..1f5012966e --- /dev/null +++ b/doc/forum/wishlist:alias_system.mdwn @@ -0,0 +1 @@ +To implement things like my custom `git annex-push` without the dash, i.e. `git annex push`, an alias system for git-annex would be nice. From 35a4039166a72192e49a1b0f9ddbe95ef518c162 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" Date: Thu, 14 Apr 2011 11:25:00 +0000 Subject: [PATCH 1518/8313] Added a comment --- .../comment_5_404b723a681eb93fee015cea8024b6bc._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment diff --git a/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment b/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment new file mode 100644 index 0000000000..042dcc1f38 --- /dev/null +++ b/doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkptNW1PzrVjYlJWP_9e499uH0mjnBV6GQ" + nickname="Christian" + subject="comment 5" + date="2011-04-14T11:24:59Z" + content=""" +I would also like an git-annex channel. Would be #git-annex@OFTC ok? +"""]] From f7018e47e48cc61ef6e84adcff89f892cee2c8db Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Apr 2011 15:09:30 -0400 Subject: [PATCH 1519/8313] typo --- doc/design/encryption.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 90b722b9b8..915eee1a13 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -39,7 +39,7 @@ At a high level, an encryption backend needs to support these operations: a stream to an action that consumes the encrypted content. * Given a streaming source of encrypted content, decrypt it, and send - it in a stream to an anction that consumes the decrypted content. + it in a stream to an action that consumes the decrypted content. * Clean up. From 1e84dab4c8def55699fc1b673bd0abd0f5dc4aee Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Apr 2011 15:09:36 -0400 Subject: [PATCH 1520/8313] RemoteConfig type --- Command/InitRemote.hs | 12 ++++++------ Remote.hs | 22 +++++++++++----------- Remote/Bup.hs | 4 ++-- Remote/Directory.hs | 4 ++-- Remote/Git.hs | 2 +- Remote/S3real.hs | 8 ++++---- Remote/Special.hs | 3 ++- RemoteClass.hs | 8 +++++--- 8 files changed, 33 insertions(+), 30 deletions(-) diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs index 39ec366539..4c2fc3a078 100644 --- a/Command/InitRemote.hs +++ b/Command/InitRemote.hs @@ -44,14 +44,14 @@ start params = notBareRepo $ do where ws = words params name = head ws - config = Remote.keyValToMap $ tail ws + config = Remote.keyValToConfig $ tail ws -perform :: RemoteClass.RemoteType Annex -> UUID -> M.Map String String -> CommandPerform +perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform perform t u c = do c' <- RemoteClass.setup t u c return $ Just $ cleanup u c' -cleanup :: UUID -> M.Map String String -> CommandCleanup +cleanup :: UUID -> RemoteClass.RemoteConfig -> CommandCleanup cleanup u c = do Remote.configSet u c g <- Annex.gitRepo @@ -65,7 +65,7 @@ cleanup u c = do return True {- Look up existing remote's UUID and config by name, or generate a new one -} -findByName :: String -> Annex (UUID, M.Map String String) +findByName :: String -> Annex (UUID, RemoteClass.RemoteConfig) findByName name = do m <- Remote.readRemoteLog case findByName' name m of @@ -74,7 +74,7 @@ findByName name = do uuid <- liftIO $ genUUID return $ (uuid, M.insert nameKey name M.empty) -findByName' :: String -> M.Map UUID (M.Map String String) -> Maybe (UUID, M.Map String String) +findByName' :: String -> M.Map UUID RemoteClass.RemoteConfig -> Maybe (UUID, RemoteClass.RemoteConfig) findByName' n m = if null matches then Nothing else Just $ head matches where matches = filter (matching . snd) $ M.toList m @@ -85,7 +85,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches | otherwise -> False {- find the specified remote type -} -findType :: M.Map String String -> Annex (RemoteClass.RemoteType Annex) +findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex) findType config = case M.lookup typeKey config of Nothing -> error "Specify the type of remote with type=" diff --git a/Remote.hs b/Remote.hs index bb661c5a90..8d2ab0399a 100644 --- a/Remote.hs +++ b/Remote.hs @@ -25,7 +25,7 @@ module Remote ( remoteLog, readRemoteLog, configSet, - keyValToMap + keyValToConfig ) where import Control.Monad.State (liftIO) @@ -137,17 +137,17 @@ remoteLog = do return $ gitStateDir g ++ "remote.log" {- Adds or updates a remote's config in the log. -} -configSet :: UUID -> M.Map String String -> Annex () +configSet :: UUID -> RemoteConfig -> Annex () configSet u c = do m <- readRemoteLog l <- remoteLog liftIO $ safeWriteFile l $ unlines $ sort $ map toline $ M.toList $ M.insert u c m where - toline (u', c') = u' ++ " " ++ (unwords $ mapToKeyVal c') + toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c') {- Map of remotes by uuid containing key/value config maps. -} -readRemoteLog :: Annex (M.Map UUID (M.Map String String)) +readRemoteLog :: Annex (M.Map UUID RemoteConfig) readRemoteLog = do l <- remoteLog s <- liftIO $ catch (readFile l) ignoreerror @@ -155,7 +155,7 @@ readRemoteLog = do where ignoreerror _ = return "" -remoteLogParse :: String -> M.Map UUID (M.Map String String) +remoteLogParse :: String -> M.Map UUID RemoteConfig remoteLogParse s = M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s where @@ -165,18 +165,18 @@ remoteLogParse s = where w = words l u = w !! 0 - c = keyValToMap $ tail w + c = keyValToConfig $ tail w -{- Given Strings like "key=value", generates a Map. -} -keyValToMap :: [String] -> M.Map String String -keyValToMap ws = M.fromList $ map (/=/) ws +{- Given Strings like "key=value", generates a RemoteConfig. -} +keyValToConfig :: [String] -> RemoteConfig +keyValToConfig ws = M.fromList $ map (/=/) ws where (/=/) s = (k, v) where k = takeWhile (/= '=') s v = drop (1 + length k) s -mapToKeyVal :: M.Map String String -> [String] -mapToKeyVal m = map toword $ sort $ M.toList m +configToKeyVal :: M.Map String String -> [String] +configToKeyVal m = map toword $ sort $ M.toList m where toword (k, v) = k ++ "=" ++ v diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 916afeb406..66c78970c9 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -39,7 +39,7 @@ remote = RemoteType { setup = bupSetup } -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do buprepo <- getConfig r "buprepo" (error "missing buprepo") cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost) @@ -60,7 +60,7 @@ gen r u c = do config = c } -bupSetup :: UUID -> M.Map String String -> Annex (M.Map String String) +bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig bupSetup u c = do -- verify configuration is sane let buprepo = case M.lookup "buprepo" c of diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 0d3478b79d..2313f79a05 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -35,7 +35,7 @@ remote = RemoteType { setup = directorySetup } -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u _ = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost @@ -51,7 +51,7 @@ gen r u _ = do config = Nothing } -directorySetup :: UUID -> M.Map String String -> Annex (M.Map String String) +directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig directorySetup u c = do -- verify configuration is sane let dir = case M.lookup "directory" c of diff --git a/Remote/Git.hs b/Remote/Git.hs index 7724df79af..bab452a331 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -40,7 +40,7 @@ list = do g <- Annex.gitRepo return $ Git.remotes g -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u _ = do {- It's assumed to be cheap to read the config of non-URL remotes, - so this is done each time git-annex is run. Conversely, diff --git a/Remote/S3real.hs b/Remote/S3real.hs index bb82d54e0e..af4e48048a 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -37,7 +37,7 @@ remote = RemoteType { setup = s3Setup } -gen :: Git.Repo -> UUID -> Maybe (M.Map String String) -> Annex (Remote Annex) +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost return $ this cst @@ -54,14 +54,14 @@ gen r u c = do config = c } -s3ConnectionRequired :: M.Map String String -> Annex AWSConnection +s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection s3ConnectionRequired c = do conn <- s3Connection c case conn of Nothing -> error "Cannot connect to S3" Just conn' -> return conn' -s3Connection :: M.Map String String -> Annex (Maybe AWSConnection) +s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) s3Connection c = do ak <- getEnvKey "AWS_ACCESS_KEY_ID" sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" @@ -78,7 +78,7 @@ s3Connection c = do _ -> error $ "bad S3 port value: " ++ s getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") -s3Setup :: UUID -> M.Map String String -> Annex (M.Map String String) +s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane case M.lookup "encryption" c of diff --git a/Remote/Special.hs b/Remote/Special.hs index b5d5a137fe..53ac2c6eed 100644 --- a/Remote/Special.hs +++ b/Remote/Special.hs @@ -13,6 +13,7 @@ import Data.String.Utils import Control.Monad.State (liftIO) import Types +import RemoteClass import qualified GitRepo as Git import qualified Annex import UUID @@ -32,7 +33,7 @@ findSpecialRemotes s = do match k _ = startswith "remote." k && endswith (".annex-"++s) k {- Sets up configuration for a special remote in .git/config. -} -gitConfigSpecialRemote :: UUID -> M.Map String String -> String -> String -> Annex () +gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex () gitConfigSpecialRemote u c k v = do g <- Annex.gitRepo liftIO $ do diff --git a/RemoteClass.hs b/RemoteClass.hs index 8055c16b06..f954e4ff8f 100644 --- a/RemoteClass.hs +++ b/RemoteClass.hs @@ -15,6 +15,8 @@ import Data.Map as M import qualified GitRepo as Git import Key +type RemoteConfig = M.Map String String + {- There are different types of remotes. -} data RemoteType a = RemoteType { -- human visible type name @@ -22,9 +24,9 @@ data RemoteType a = RemoteType { -- enumerates remotes of this type enumerate :: a [Git.Repo], -- generates a remote of this type - generate :: Git.Repo -> String -> Maybe (M.Map String String) -> a (Remote a), + generate :: Git.Repo -> String -> Maybe RemoteConfig -> a (Remote a), -- initializes or changes a remote - setup :: String -> M.Map String String -> a (M.Map String String) + setup :: String -> RemoteConfig -> a RemoteConfig } {- An individual remote. -} @@ -48,7 +50,7 @@ data Remote a = Remote { -- operation. hasKeyCheap :: Bool, -- a Remote can have a persistent configuration store - config :: Maybe (M.Map String String) + config :: Maybe RemoteConfig } instance Show (Remote a) where From 452c63035af391e109ad37c0722c2f32cbf77428 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 15 Apr 2011 19:32:09 +0000 Subject: [PATCH 1521/8313] Added a comment --- .../comment_6_0d87d0e26461494b1d7f8a701a924729._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment diff --git a/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment b/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment new file mode 100644 index 0000000000..8dfd0f8203 --- /dev/null +++ b/doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 6" + date="2011-04-15T19:32:08Z" + content=""" +We seem to be using #vcs-home @ OFTC for now. madduck is fine with it and joeyh pokes his head in there, as well. I just added a CIA bot to #vcs-home and this comment is a test if pushing works. -- RichiH +"""]] From 467020e56015d5760b1aa580f8cb2a2c5f91a18c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 15 Apr 2011 19:40:30 +0000 Subject: [PATCH 1522/8313] --- ...from_the_website__39__s_repo__44___not_your_personal_one.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn diff --git a/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn new file mode 100644 index 0000000000..6926e3cca2 --- /dev/null +++ b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn @@ -0,0 +1 @@ +I just added a CIA bot to #vcs-home and tracking commits immediately would be nice. -- RichiH From 480d780297dac12576a90c25cca5cb989e1a1e4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 15 Apr 2011 18:18:39 -0400 Subject: [PATCH 1523/8313] add --- Crypto.hs | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 Crypto.hs diff --git a/Crypto.hs b/Crypto.hs new file mode 100644 index 0000000000..4ea43838a2 --- /dev/null +++ b/Crypto.hs @@ -0,0 +1,111 @@ +{- git-annex crypto + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Crypto ( + genCipher, + updateCipher, + storeCipher, + extractCipher, + decryptCipher, + encryptKey, + encryptContent, + decryptContent +) where + +import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.Map as M +import System.IO +import System.Cmd.Utils + +import Types +import RemoteClass +import Utility + +data Cipher = Cipher String -- XXX ideally, this would be a locked memory region +data EncryptedCipher = EncryptedCipher String + deriving Show + +{- Creates a new Cipher, encrypted as specified in the remote's configuration -} +genCipher :: RemoteConfig -> IO EncryptedCipher +genCipher config = do + random <- genrandom + encryptCipher config $ Cipher random + where + genrandom = gpgPipeRead + [ Params "--armor --gen-random" + , Param $ show randomquality + , Param $ show ciphersize + ] + randomquality = 1 -- 1 is /dev/urandom; 2 is /dev/random + ciphersize = 1024 + +{- Updates an existing Cipher, re-encrypting it as specified in the + - remote's configuration -} +updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher +updateCipher config encipher = do + cipher <- decryptCipher config encipher + encryptCipher config cipher + +{- Stores an EncryptedCipher in a remote's configuration. -} +storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig +storeCipher config (EncryptedCipher c) = M.insert "cipher" c config + +{- Extracts an EncryptedCipher from a remote's configuration. -} +extractCipher :: RemoteConfig -> EncryptedCipher +extractCipher config = case M.lookup "cipher" config of + Just c -> EncryptedCipher c + Nothing -> error "missing cipher in remote config" + +{- Encryptes a Cipher as specified by a remote's configuration. -} +encryptCipher :: RemoteConfig -> Cipher -> IO EncryptedCipher +encryptCipher config (Cipher c) = do + encipher <- gpgPipeBoth (encrypt++recipient) c + return $ EncryptedCipher encipher + where + encrypt = + [ Params "--encrypt --armor" + , Params "--trust-model always" + ] + recipient = case M.lookup "encryption" config of + Nothing -> [ Param "--default-recipient-self" ] + Just r -> + -- Force gpg to only encrypt to the specified + -- recipients, not configured defaults. + [ Params "--no-encrypt-to --no-default-recipient" + , Param "--recipient" + , Param r + ] + +{- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} +decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher +decryptCipher = error "TODO" + +{- Genetates an encrypted form of a Key. The enctyption does not need to be + - reversable, nor does it need to be the same type of encryption used + - on content. -} +encryptKey :: Cipher -> Key -> IO Key +encryptKey = error "TODO" + +{- Streams content, encrypting. -} +encryptContent :: Cipher -> L.ByteString -> IO L.ByteString +encryptContent = error "TODO" + +{- Streams encrypted content, decrypting. -} +decryptContent :: Cipher -> L.ByteString -> IO L.ByteString +decryptContent = error "TODO" + + +gpgParams :: [CommandParam] -> [String] +gpgParams params = ["--batch", "--quiet"] ++ toCommand params + +gpgPipeRead :: [CommandParam] -> IO String +gpgPipeRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict + +gpgPipeBoth :: [CommandParam] -> String -> IO String +gpgPipeBoth params input = do + (_, s) <- pipeBoth "gpg" (gpgParams params) input + return s From 7fdf20f577f63f8437c63d7d83e70d34de89269f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 13:25:27 -0400 Subject: [PATCH 1524/8313] encryption key management working Encrypted remotes don't yet encrypt data, but git annex initremote can be used to generate a cipher and add additional gpg keys that can use it. --- Crypto.hs | 120 ++++++++++++++++++++++++++++++-------------- Remote/Bup.hs | 12 ++--- Remote/Encrypted.hs | 31 ++++++++++++ Remote/S3real.hs | 8 ++- 4 files changed, 120 insertions(+), 51 deletions(-) create mode 100644 Remote/Encrypted.hs diff --git a/Crypto.hs b/Crypto.hs index 4ea43838a2..f32d429c3b 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -1,4 +1,7 @@ {- git-annex crypto + - + - Currently using gpg; could later be modified to support different + - crypto backends if neccessary. - - Copyright 2011 Joey Hess - @@ -18,71 +21,91 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M -import System.IO +import qualified Codec.Binary.Base64 as B64 import System.Cmd.Utils +import Data.String.Utils +import Data.List +import Data.Bits.Utils import Types import RemoteClass import Utility data Cipher = Cipher String -- XXX ideally, this would be a locked memory region -data EncryptedCipher = EncryptedCipher String - deriving Show + +data EncryptedCipher = EncryptedCipher String KeyIds + +data KeyIds = KeyIds [String] + +instance Show KeyIds where + show (KeyIds ks) = join "," ks + +instance Read KeyIds where + readsPrec _ s = [(KeyIds (split "," s), "")] {- Creates a new Cipher, encrypted as specified in the remote's configuration -} genCipher :: RemoteConfig -> IO EncryptedCipher -genCipher config = do +genCipher c = do + ks <- configKeyIds c random <- genrandom - encryptCipher config $ Cipher random + encryptCipher (Cipher random) ks where genrandom = gpgPipeRead - [ Params "--armor --gen-random" + [ Params "--gen-random" , Param $ show randomquality , Param $ show ciphersize ] - randomquality = 1 -- 1 is /dev/urandom; 2 is /dev/random - ciphersize = 1024 + randomquality = 1 :: Int -- 1 is /dev/urandom; 2 is /dev/random + ciphersize = 1024 :: Int -{- Updates an existing Cipher, re-encrypting it as specified in the - - remote's configuration -} +{- Updates an existing Cipher, re-encrypting it to add KeyIds specified in + - the remote's configuration. -} updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher -updateCipher config encipher = do - cipher <- decryptCipher config encipher - encryptCipher config cipher +updateCipher c encipher@(EncryptedCipher _ ks) = do + ks' <- configKeyIds c + cipher <- decryptCipher c encipher + encryptCipher cipher (combine ks ks') + where + combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b {- Stores an EncryptedCipher in a remote's configuration. -} storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig -storeCipher config (EncryptedCipher c) = M.insert "cipher" c config +storeCipher c (EncryptedCipher t ks) = + M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c + where + toB64 = B64.encode . s2w8 {- Extracts an EncryptedCipher from a remote's configuration. -} -extractCipher :: RemoteConfig -> EncryptedCipher -extractCipher config = case M.lookup "cipher" config of - Just c -> EncryptedCipher c - Nothing -> error "missing cipher in remote config" - -{- Encryptes a Cipher as specified by a remote's configuration. -} -encryptCipher :: RemoteConfig -> Cipher -> IO EncryptedCipher -encryptCipher config (Cipher c) = do - encipher <- gpgPipeBoth (encrypt++recipient) c - return $ EncryptedCipher encipher +extractCipher :: RemoteConfig -> Maybe EncryptedCipher +extractCipher c = + case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of + (Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks) + _ -> Nothing where - encrypt = - [ Params "--encrypt --armor" - , Params "--trust-model always" - ] - recipient = case M.lookup "encryption" config of - Nothing -> [ Param "--default-recipient-self" ] - Just r -> - -- Force gpg to only encrypt to the specified - -- recipients, not configured defaults. - [ Params "--no-encrypt-to --no-default-recipient" - , Param "--recipient" - , Param r - ] + fromB64 s = case B64.decode s of + Nothing -> error "bad base64 encoded cipher in remote config" + Just ws -> w82s ws + +{- Encrypts a Cipher to the specified KeyIds. -} +encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher +encryptCipher (Cipher c) (KeyIds ks) = do + let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids + encipher <- gpgPipeBoth (encrypt++recipients ks') c + return $ EncryptedCipher encipher (KeyIds ks') + where + encrypt = [ Params "--encrypt" ] + recipients l = + -- Force gpg to only encrypt to the specified + -- recipients, not configured defaults. + [ Params "--no-encrypt-to --no-default-recipient"] ++ + (concat $ map (\k -> [Param "--recipient", Param k]) l) {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher -decryptCipher = error "TODO" +decryptCipher _ (EncryptedCipher encipher _) = + return . Cipher =<< gpgPipeBoth decrypt encipher + where + decrypt = [ Params "--decrypt" ] {- Genetates an encrypted form of a Key. The enctyption does not need to be - reversable, nor does it need to be the same type of encryption used @@ -100,7 +123,10 @@ decryptContent = error "TODO" gpgParams :: [CommandParam] -> [String] -gpgParams params = ["--batch", "--quiet"] ++ toCommand params +gpgParams params = + -- avoid console IO, and be quiet, even about checking the trustdb + ["--batch", "--quiet", "--trust-model", "always"] ++ + toCommand params gpgPipeRead :: [CommandParam] -> IO String gpgPipeRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict @@ -109,3 +135,19 @@ gpgPipeBoth :: [CommandParam] -> String -> IO String gpgPipeBoth params input = do (_, s) <- pipeBoth "gpg" (gpgParams params) input return s + +configKeyIds :: RemoteConfig -> IO KeyIds +configKeyIds c = do + let k = configGet c "encryption" + s <- gpgPipeRead [Params "--with-colons --list-public-keys", Param k] + return $ KeyIds $ parseWithColons s + where + parseWithColons s = map keyIdField $ filter pubKey $ lines s + pubKey = isPrefixOf "pub:" + keyIdField s = (split ":" s) !! 4 + +configGet :: RemoteConfig -> String -> String +configGet c key = + case M.lookup key c of + Just v -> v + Nothing -> error $ "missing " ++ key ++ " in remote config" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 66c78970c9..b4403bb03e 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -26,8 +26,9 @@ import Locations import Config import Utility import Messages -import Remote.Special import Ssh +import Remote.Special +import Remote.Encrypted type BupRepo = String @@ -66,10 +67,7 @@ bupSetup u c = do let buprepo = case M.lookup "buprepo" c of Nothing -> error "Specify buprepo=" Just r -> r - case M.lookup "encryption" c of - Nothing -> error "Specify encryption=key or encryption=none" - Just "none" -> return () - Just _ -> error "encryption keys not yet supported" + c' <- encryptionSetup c -- bup init will create the repository. -- (If the repository already exists, bup init again appears safe.) @@ -81,9 +79,9 @@ bupSetup u c = do -- The buprepo is stored in git config, as well as this repo's -- persistant state, so it can vary between hosts. - gitConfigSpecialRemote u c "buprepo" buprepo + gitConfigSpecialRemote u c' "buprepo" buprepo - return $ M.delete "directory" c + return c' bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam] bupParams command buprepo params = diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs new file mode 100644 index 0000000000..ae40446209 --- /dev/null +++ b/Remote/Encrypted.hs @@ -0,0 +1,31 @@ +{- common functions for encrypted remotes + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Encrypted where + +import qualified Data.Map as M +import Control.Monad.State (liftIO) + +import Types +import RemoteClass +import Crypto + +{- Encryption setup for a remote. The user must specify whether to use + - an encryption key, or not encrypt. An encrypted cipher is created, or is + - updated to be accessible to an additional encryption key. -} +encryptionSetup :: RemoteConfig -> Annex RemoteConfig +encryptionSetup c = + case (M.lookup "encryption" c, extractCipher c) of + (Nothing, Nothing) -> error "Specify encryption=key or encryption=none" + (Just "none", _) -> return c + (Nothing, Just _) -> return c + (Just _, Nothing) -> use $ genCipher c + (Just _, Just v) -> use $ updateCipher c v + where + use a = do + cipher <- liftIO a + return $ M.delete "encryption" $ storeCipher c cipher diff --git a/Remote/S3real.hs b/Remote/S3real.hs index af4e48048a..0f6327f575 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -28,6 +28,7 @@ import Messages import Locations import Config import Remote.Special +import Remote.Encrypted remote :: RemoteType Annex remote = RemoteType { @@ -81,11 +82,8 @@ s3Connection c = do s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane - case M.lookup "encryption" c of - Nothing -> error "Specify encryption=key or encryption=none" - Just "none" -> return () - Just _ -> error "encryption keys not yet supported" - let fullconfig = M.union c defaults + c' <- encryptionSetup c + let fullconfig = M.union c' defaults -- check bucket location to see if the bucket exists, and create it let datacenter = fromJust $ M.lookup "datacenter" fullconfig From 669851454cd3032d2097842f7b6027b3464da032 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 16:26:47 -0400 Subject: [PATCH 1525/8313] crypto library almost complete Piping data through gpg with symmetric cipher is working. Only Key encryption is not done. --- Crypto.hs | 111 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index f32d429c3b..2e20dddb10 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -15,8 +15,8 @@ module Crypto ( extractCipher, decryptCipher, encryptKey, - encryptContent, - decryptContent + withEncryptedContent, + withDecryptedContent, ) where import qualified Data.ByteString.Lazy.Char8 as L @@ -26,8 +26,14 @@ import System.Cmd.Utils import Data.String.Utils import Data.List import Data.Bits.Utils +import System.IO +import System.Posix.IO +import System.Posix.Types +import Control.Concurrent +import Control.Exception import Types +import Key import RemoteClass import Utility @@ -50,13 +56,16 @@ genCipher c = do random <- genrandom encryptCipher (Cipher random) ks where - genrandom = gpgPipeRead - [ Params "--gen-random" + genrandom = gpgRead + -- Armor the random data, to avoid newlines, + -- since gpg only reads ciphers up to the first + -- newline. + [ Params "--gen-random --armor" , Param $ show randomquality , Param $ show ciphersize ] randomquality = 1 :: Int -- 1 is /dev/urandom; 2 is /dev/random - ciphersize = 1024 :: Int + ciphersize = 256 :: Int {- Updates an existing Cipher, re-encrypting it to add KeyIds specified in - the remote's configuration. -} @@ -72,8 +81,6 @@ updateCipher c encipher@(EncryptedCipher _ ks) = do storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig storeCipher c (EncryptedCipher t ks) = M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c - where - toB64 = B64.encode . s2w8 {- Extracts an EncryptedCipher from a remote's configuration. -} extractCipher :: RemoteConfig -> Maybe EncryptedCipher @@ -81,16 +88,12 @@ extractCipher c = case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of (Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks) _ -> Nothing - where - fromB64 s = case B64.decode s of - Nothing -> error "bad base64 encoded cipher in remote config" - Just ws -> w82s ws {- Encrypts a Cipher to the specified KeyIds. -} encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher encryptCipher (Cipher c) (KeyIds ks) = do let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids - encipher <- gpgPipeBoth (encrypt++recipients ks') c + encipher <- gpgPipeStrict (encrypt++recipients ks') c return $ EncryptedCipher encipher (KeyIds ks') where encrypt = [ Params "--encrypt" ] @@ -103,43 +106,80 @@ encryptCipher (Cipher c) (KeyIds ks) = do {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -} decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher decryptCipher _ (EncryptedCipher encipher _) = - return . Cipher =<< gpgPipeBoth decrypt encipher + return . Cipher =<< gpgPipeStrict decrypt encipher where - decrypt = [ Params "--decrypt" ] + decrypt = [ Param "--decrypt" ] -{- Genetates an encrypted form of a Key. The enctyption does not need to be +{- Generates an encrypted form of a Key. The encryption does not need to be - reversable, nor does it need to be the same type of encryption used - - on content. -} + - on content. It does need to be repeatable. -} encryptKey :: Cipher -> Key -> IO Key -encryptKey = error "TODO" +encryptKey c k = + return Key { + -- FIXME: should use HMAC with the cipher; I don't + -- have Data.Crypto in Debian yet though. + keyName = show k, + keyBackendName = "INSECURE", + keySize = Nothing, -- size and mtime omitted + keyMtime = Nothing -- to avoid leaking data + } -{- Streams content, encrypting. -} -encryptContent :: Cipher -> L.ByteString -> IO L.ByteString -encryptContent = error "TODO" +{- Streams encrypted content to an action. -} +withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withEncryptedContent = gpgCipher [Params "--symmetric --force-mdc"] -{- Streams encrypted content, decrypting. -} -decryptContent :: Cipher -> L.ByteString -> IO L.ByteString -decryptContent = error "TODO" +{- Streams decrypted content to an action. -} +withDecryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withDecryptedContent = gpgCipher [Param "--decrypt"] gpgParams :: [CommandParam] -> [String] gpgParams params = - -- avoid console IO, and be quiet, even about checking the trustdb + -- avoid prompting, and be quiet, even about checking the trustdb ["--batch", "--quiet", "--trust-model", "always"] ++ toCommand params -gpgPipeRead :: [CommandParam] -> IO String -gpgPipeRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict +gpgRead :: [CommandParam] -> IO String +gpgRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict -gpgPipeBoth :: [CommandParam] -> String -> IO String -gpgPipeBoth params input = do - (_, s) <- pipeBoth "gpg" (gpgParams params) input - return s +gpgPipeStrict :: [CommandParam] -> String -> IO String +gpgPipeStrict params input = do + (_, output) <- pipeBoth "gpg" (gpgParams params) input + return output + +gpgPipeBytes :: [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString) +gpgPipeBytes params input = do + (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) + _ <- forkIO $ finally (L.hPut toh input) (hClose toh) + output <- L.hGetContents fromh + return (pid, output) + +{- Runs gpg with a cipher and some parameters, feeding it an input, + - and piping its output lazily to an action. -} +gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +gpgCipher params (Cipher c) input a = do + -- pipe the passphrase into gpg on a fd + (frompipe, topipe) <- createPipe + toh <- fdToHandle topipe + let Fd fromno = frompipe + _ <- forkIO $ do + hPutStrLn toh c + hClose toh + let passphrase = [Param "--passphrase-fd", Param $ show fromno] + (pid, output) <- gpgPipeBytes (passphrase ++ params) input + + ret <- a output + + -- cleanup + forceSuccess pid + closeFd frompipe + + return ret configKeyIds :: RemoteConfig -> IO KeyIds configKeyIds c = do let k = configGet c "encryption" - s <- gpgPipeRead [Params "--with-colons --list-public-keys", Param k] + s <- gpgRead [Params "--with-colons --list-public-keys", Param k] return $ KeyIds $ parseWithColons s where parseWithColons s = map keyIdField $ filter pubKey $ lines s @@ -151,3 +191,12 @@ configGet c key = case M.lookup key c of Just v -> v Nothing -> error $ "missing " ++ key ++ " in remote config" + +toB64 :: String -> String +toB64 = B64.encode . s2w8 + +fromB64 :: String -> String +fromB64 s = + case B64.decode s of + Nothing -> error "bad base64 encoded data" + Just ws -> w82s ws From 5efd41327045f8da55c972b7391309c99dee5afc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 16:29:28 -0400 Subject: [PATCH 1526/8313] add encryption support to directory special remotes --- Remote/Directory.hs | 6 ++++-- doc/special_remotes/directory.mdwn | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 2313f79a05..bb1ef60e49 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -26,6 +26,7 @@ import Config import Content import Utility import Remote.Special +import Remote.Encrypted remote :: RemoteType Annex remote = RemoteType { @@ -59,11 +60,12 @@ directorySetup u c = do Just d -> d e <- liftIO $ doesDirectoryExist dir when (not e) $ error $ "Directory does not exist: " ++ dir + c' <- encryptionSetup c -- The directory is stored in git config, not in this remote's -- persistant state, so it can vary between hosts. - gitConfigSpecialRemote u c "directory" dir - return $ M.delete "directory" c + gitConfigSpecialRemote u c' "directory" dir + return $ M.delete "directory" c' dirKey :: FilePath -> Key -> FilePath dirKey d k = d hashDirMixed k f f diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index daa0b74128..18d30e3110 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -7,4 +7,4 @@ the drive's mountpoint as a directory remote. Setup example: - # git annex initremote usbdrive directory=/media/usbdrive/ + # git annex initremote usbdrive directory=/media/usbdrive/ encryption=none From 9fe7e6be7064d9c47e6c6fd4f1b3a70da604727d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 16:41:46 -0400 Subject: [PATCH 1527/8313] add cipher field to AnnexState --- Annex.hs | 5 ++++- Crypto.hs | 13 +------------ CryptoTypes.hs | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 CryptoTypes.hs diff --git a/Annex.hs b/Annex.hs index f4e5d599d0..9086db9bf7 100644 --- a/Annex.hs +++ b/Annex.hs @@ -22,6 +22,7 @@ import qualified GitRepo as Git import qualified GitQueue import qualified BackendClass import qualified RemoteClass +import qualified CryptoTypes -- git-annex's monad type Annex = StateT AnnexState IO @@ -41,7 +42,8 @@ data AnnexState = AnnexState , toremote :: Maybe String , fromremote :: Maybe String , exclude :: [String] - } deriving (Show) + , cipher :: Maybe CryptoTypes.Cipher + } newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState newState gitrepo allbackends = AnnexState @@ -58,6 +60,7 @@ newState gitrepo allbackends = AnnexState , toremote = Nothing , fromremote = Nothing , exclude = [] + , cipher = Nothing } {- Create and returns an Annex state object for the specified git repo. -} diff --git a/Crypto.hs b/Crypto.hs index 2e20dddb10..337aedff65 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -36,18 +36,7 @@ import Types import Key import RemoteClass import Utility - -data Cipher = Cipher String -- XXX ideally, this would be a locked memory region - -data EncryptedCipher = EncryptedCipher String KeyIds - -data KeyIds = KeyIds [String] - -instance Show KeyIds where - show (KeyIds ks) = join "," ks - -instance Read KeyIds where - readsPrec _ s = [(KeyIds (split "," s), "")] +import CryptoTypes {- Creates a new Cipher, encrypted as specified in the remote's configuration -} genCipher :: RemoteConfig -> IO EncryptedCipher diff --git a/CryptoTypes.hs b/CryptoTypes.hs new file mode 100644 index 0000000000..944a9d34e0 --- /dev/null +++ b/CryptoTypes.hs @@ -0,0 +1,22 @@ +{- git-annex crypto types + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module CryptoTypes where + +import Data.String.Utils + +data Cipher = Cipher String -- XXX ideally, this would be a locked memory region + +data EncryptedCipher = EncryptedCipher String KeyIds + +data KeyIds = KeyIds [String] + +instance Show KeyIds where + show (KeyIds ks) = join "," ks + +instance Read KeyIds where + readsPrec _ s = [(KeyIds (split "," s), "")] From 4f9fafa02354d275d6fa83ff42ada4ebd1bc83d8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 18:22:52 -0400 Subject: [PATCH 1528/8313] full encryption support for directory special remotes --- Crypto.hs | 7 ++++++- Remote/Directory.hs | 48 ++++++++++++++++++++++++++++++++------------- Remote/Encrypted.hs | 43 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 337aedff65..9f404c1b17 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -9,6 +9,8 @@ -} module Crypto ( + Cipher, + EncryptedCipher, genCipher, updateCipher, storeCipher, @@ -133,7 +135,10 @@ gpgRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict gpgPipeStrict :: [CommandParam] -> String -> IO String gpgPipeStrict params input = do - (_, output) <- pipeBoth "gpg" (gpgParams params) input + (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) + _ <- forkIO $ finally (hPutStr toh input) (hClose toh) + output <- hGetContentsStrict fromh + forceSuccess pid return output gpgPipeBytes :: [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index bb1ef60e49..5ea0a1e6b3 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -7,6 +7,7 @@ module Remote.Directory (remote) where +import qualified Data.ByteString.Lazy.Char8 as L import IO import Control.Exception.Extensible (IOException) import qualified Data.Map as M @@ -27,6 +28,7 @@ import Content import Utility import Remote.Special import Remote.Encrypted +import Crypto remote :: RemoteType Annex remote = RemoteType { @@ -37,17 +39,17 @@ remote = RemoteType { } gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) -gen r u _ = do +gen r u c = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost return $ Remote { uuid = u, cost = cst, name = Git.repoDescribe r, - storeKey = store dir, - retrieveKeyFile = retrieve dir, - removeKey = remove dir, - hasKey = checkPresent dir, + storeKey = storeKeyEncrypted c $ store dir, + retrieveKeyFile = retrieveKeyFileEncrypted c $ retrieve dir, + removeKey = removeKeyEncrypted c $ remove dir, + hasKey = hasKeyEncrypted c $ checkPresent dir, hasKeyCheap = True, config = Nothing } @@ -72,25 +74,43 @@ dirKey d k = d hashDirMixed k f f where f = keyFile k -store :: FilePath -> Key -> Annex Bool -store d k = do +store :: FilePath -> Key -> Maybe (Cipher, Key) -> Annex Bool +store d k c = do g <- Annex.gitRepo - let src = gitAnnexLocation g k + let src = gitAnnexLocation g k liftIO $ catch (copy src) (const $ return False) where - dest = dirKey d k - dir = parentDir dest - copy src = do + copy src = case c of + Just (cipher, enckey) -> do + content <- L.readFile src + let dest = dirKey d enckey + prep dest + withEncryptedContent cipher content $ \s -> do + L.writeFile dest s + cleanup True dest + _ -> do + let dest = dirKey d k + prep dest + ok <- copyFile src dest + cleanup ok dest + prep dest = liftIO $ do + let dir = parentDir dest createDirectoryIfMissing True dir allowWrite dir - ok <- copyFile src dest + cleanup ok dest = do when ok $ do + let dir = parentDir dest preventWrite dest preventWrite dir return ok -retrieve :: FilePath -> Key -> FilePath -> Annex Bool -retrieve d k f = liftIO $ copyFile (dirKey d k) f +retrieve :: FilePath -> Key -> FilePath -> Maybe (Cipher, Key) -> Annex Bool +retrieve d k f Nothing = liftIO $ copyFile (dirKey d k) f +retrieve d k f (Just (cipher, enckey)) = + liftIO $ flip catch (const $ return False) $ do + content <- L.readFile (dirKey d enckey) + withDecryptedContent cipher content $ L.writeFile f + return True remove :: FilePath -> Key -> Annex Bool remove d k = liftIO $ catch del (const $ return False) diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs index ae40446209..2a0fb13bca 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encrypted.hs @@ -13,6 +13,8 @@ import Control.Monad.State (liftIO) import Types import RemoteClass import Crypto +import qualified Annex +import Messages {- Encryption setup for a remote. The user must specify whether to use - an encryption key, or not encrypt. An encrypted cipher is created, or is @@ -29,3 +31,44 @@ encryptionSetup c = use a = do cipher <- liftIO a return $ M.delete "encryption" $ storeCipher c cipher + +{- Helpers that can be applied to a Remote's normal actions to + - add crypto support. -} +storeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Maybe (Cipher, Key) -> Annex a) -> Key -> Annex a +storeKeyEncrypted c a k = a k =<< cipherKey c k +retrieveKeyFileEncrypted :: Maybe RemoteConfig -> (Key -> FilePath -> Maybe (Cipher, Key) -> Annex a) -> Key -> FilePath -> Annex a +retrieveKeyFileEncrypted c a k f = a k f =<< cipherKey c k +removeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a +removeKeyEncrypted = withEncryptedKey +hasKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a +hasKeyEncrypted = withEncryptedKey + +{- Gets encryption Cipher, and encrypted version of Key. + - + - The decrypted Cipher is cached in the Annex state. -} +cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) +cipherKey Nothing _ = return Nothing +cipherKey (Just c) k = do + cache <- Annex.getState Annex.cipher + case cache of + Just cipher -> ret cipher + Nothing -> case extractCipher c of + Nothing -> return Nothing + Just encipher -> do + showNote "getting encryption key" + cipher <- liftIO $ decryptCipher c encipher + Annex.changeState (\s -> s { Annex.cipher = Just cipher }) + ret cipher + where + ret cipher = do + k' <- liftIO $ encryptKey cipher k + return $ Just (cipher, k') + +{- Passes the encrypted version of the key to the action when encryption + - is enabled, and the non-encrypted version otherwise. -} +withEncryptedKey :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a +withEncryptedKey c a k = do + v <- cipherKey c k + case v of + Nothing -> a k + Just (_, k') -> a k' From 1247bfeaa7356e766d3ea09fa50bd300650f78af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 19:13:05 -0400 Subject: [PATCH 1529/8313] gpg recommended --- configure.hs | 1 + debian/changelog | 2 ++ debian/control | 2 +- doc/install.mdwn | 10 ++++------ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/configure.hs b/configure.hs index 4ab3052395..c81aa17e69 100644 --- a/configure.hs +++ b/configure.hs @@ -16,6 +16,7 @@ tests = , TestCase "rsync" $ requireCmd "rsync" "rsync --version >/dev/null" , TestCase "curl" $ testCmd "curl" "curl --version >/dev/null" , TestCase "bup" $ testCmd "bup" "bup --version >/dev/null" + , TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null" , TestCase "unicode FilePath support" $ unicodeFilePath ] ++ shaTestCases [1, 256, 512, 224, 384] diff --git a/debian/changelog b/debian/changelog index 91c0c8f4b3..04a6896e32 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (0.20110402) UNRELEASED; urgency=low * bup is now supported as a special type of remote. + * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted + using GPG for privacy. * Use lowercase hash directories for locationlog files, to avoid some issues with git on OSX with the mixed-case directories. No migration is needed; the old mixed case hash directories are still diff --git a/debian/control b/debian/control index 15155b9b43..42c79c91d8 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Package: git-annex Architecture: any Section: utils Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync -Suggests: graphviz, bup +Suggests: graphviz, bup, gnupg Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file contents into git. While that may seem paradoxical, it is useful when diff --git a/doc/install.mdwn b/doc/install.mdwn index 70ab8e30b6..746352cb8a 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,19 +13,17 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: +* TestPack +* QuickCheck 2 * hS3 (optional, but recommended) * `uuid`: - (or uuidgen from util-linux) + (or `uuidgen` from util-linux) * `xargs`: * `rsync`: * `curl` : (optional, but recommended) * `sha1sum`: (optional, but recommended) +* `gpg`: (optional; needed for encryption) * [Ikiwiki](http://ikiwiki.info) is needed to build the documentation, but that will be skipped if it is not installed. Then just [[download]] git-annex and run: `make; make install` - -Additionally, to run the test suite (via `make test`), you will need: - -* `TestPack` -* `QuickCheck` 2 From d2e74efdb2e5b819d5c56f167291b006badd94cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 19:30:31 -0400 Subject: [PATCH 1530/8313] document encryption --- doc/design/encryption.mdwn | 37 ++-------------------------- doc/encryption.mdwn | 35 ++++++++++++++++++++++++++ doc/special_remotes/Amazon_S3.mdwn | 15 +++-------- doc/special_remotes/bup.mdwn | 15 +++-------- doc/special_remotes/directory.mdwn | 6 ++--- doc/walkthrough/using_Amazon_S3.mdwn | 12 +++++---- 6 files changed, 53 insertions(+), 67 deletions(-) create mode 100644 doc/encryption.mdwn diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 915eee1a13..5a4bc8bbdf 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -1,15 +1,5 @@ -git-annex mostly does not use encryption. Anyone with access to a git -repository can see all the filenames in it, its history, and can access -any annexed file contents. - -Encryption is needed when using [[special_remotes]] like Amazon S3, where -file content is sent to an untrusted party who does not have access to the -git repository. - -Such an encrypted remote uses strong encryption on the contents of files, -as well as the filenames. The size of the encrypted files, and access -patterns of the data, should be the only clues to what type of is stored in -such a remote. +This was the design doc for [[encryption]] and is preserved for +the curious. [[!toc]] @@ -20,29 +10,6 @@ should be a way to tell what backend is responsible for a given filename in an encrypted remote. (And since special remotes can also store files unencrypted, differentiate from those as well.) -At a high level, an encryption backend needs to support these operations: - -* Create a new encrypted cipher, or update the cipher. Some input - parameters will specifiy things like the gpg public keys that - can access the cipher. - -* Initialize an instance of the encryption backend, that will use a - specified encrypted cipher. - -* Given a key/value backend key, produce and return an encrypted key. - - The same naming scheme git-annex uses for keys in regular key/value - [[backends]] can be used. So a filename for a key might be - "GPG-s12345--armoureddatahere" - -* Given a streaming source of file content, encrypt it, and send it in - a stream to an action that consumes the encrypted content. - -* Given a streaming source of encrypted content, decrypt it, and send - it in a stream to an action that consumes the decrypted content. - -* Clean up. - The rest of this page will describe a single encryption backend using GPG. Probably only one will be needed, but who knows? Maybe that backend will turn out badly designed, or some other encryptor needed. Designing diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn new file mode 100644 index 0000000000..0f83bb7f90 --- /dev/null +++ b/doc/encryption.mdwn @@ -0,0 +1,35 @@ +git-annex mostly does not use encryption. Anyone with access to a git +repository can see all the filenames in it, its history, and can access +any annexed file contents. + +Encryption is needed when using [[special_remotes]] like Amazon S3, where +file content is sent to an untrusted party who does not have access to the +git repository. + +Such an encrypted remote uses strong GPG encryption on the contents of files, +as well as HMAC hashing of the filenames. The size of the encrypted files, +and access patterns of the data, should be the only clues to what is +stored in such a remote. + +You should decide whether to use encryption with a special remote before +any data is stored in it. So, `git annex initremote` requires you +to specify "encryption=none" when first setting up a remote in order +to disable encryption. + +If you want to use encryption, run `git annex initremote` with +"encryption=USERID". The value will be passed to `gpg` to find encryption keys. +Typically, you will say "encryption=2512E3C7" to use a specific gpg key. +Or, you might say "encryption=joey@kitenet.net" to search for matching keys. + +The [[encryption_design|design/encryption]] allows additional encryption keys +to be added on to a special remote later. Once a key is added, it is able +to access content that has already been stored in the special remote. +To add a new key, just run `git annex initremote` again, specifying the +new encryption key: + + git annex initremote myremote encryption=788A3F4C + +Note that once a key has been given access to a remote, it's not +possible to revoke that access, short of deleting the remote. See +[[encryption_design|design/encryption]] for other security risks +associated with encryption. diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index 2cf23187d1..87cde32993 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -9,11 +9,12 @@ See [[walkthrough/using_Amazon_S3]] for usage examples. A number of parameters can be passed to `git annex initremote` to configure the S3 remote. -* `encryption` - Required. Either "none" to disable encryption, +* `encryption` - Required. Either "none" to disable encryption + (not recommended), or a value that can be looked up (using gpg -k) to find a gpg encryption key that will be given access to the remote. Note that additional gpg keys can be given access to a remote by rerunning initremote with - the new key id. + the new key id. See [[encryption]]. * `datacenter` - Defaults to "US". Other values include "EU", "us-west-1", and "ap-southeast-1". @@ -28,13 +29,3 @@ the S3 remote. * `bucket` - S3 requires that buckets have a globally unique name, so by default, a bucket name is chosen based on the remote name and UUID. This can be specified to pick a bucket name. - -## data security - -When encryption=none, there is **no** protection against your data being read -as it is sent to/from S3, or by Amazon when it is stored in S3. This should -only be used for public data. - -** Encryption is not yet supported. ** - -See [[design/encryption]]. diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 90b84e9f47..5bc1fb7a2e 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -15,11 +15,12 @@ for example; or clone bup's git repository to further back it up. These parameters can be passed to `git annex initremote` to configure bup: -* `encryption` - Required. Either "none" to disable encryption, +* `encryption` - Required. Either "none" to disable encryption of content + stored in bup (ssh will still be used to transport it securely), or a value that can be looked up (using gpg -k) to find a gpg encryption key that will be given access to the remote. Note that additional gpg keys can be given access to a remote by rerunning initremote with - the new key id. + the new key id. See [[encryption]]. * `buprepo` - Required. This is passed to `bup` as the `--remote` to use to store data. To create the repository,`bup init` will be run. @@ -34,13 +35,3 @@ can be used to, for example, limit its bandwidth. [[git-annex-shell]] does not support bup, due to the wacky way that bup starts its server. So, to use bup, you need full shell access to the server. - -## data security - -When encryption=none, there is **no** protection against your data being read -by anyone who can access the bup remote. However, bup does transfer data -using ssh, and if you trust the security of the remote, that's fine. - -** Encryption is not yet supported. ** - -See [[design/encryption]]. diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index 18d30e3110..8006c44fc9 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -1,8 +1,8 @@ This special remote type stores file contents in directory. -One use case for this would be if you have a removable drive, that you -cannot put a git repository on for some reason, and you want to use it -to sneakernet files between systems. Just set up both systems to use +One use case for this would be if you have a removable drive that +you want to use it to sneakernet files between systems (possibly with +[[encrypted|encryption]] contents). Just set up both systems to use the drive's mountpoint as a directory remote. Setup example: diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index b8eb7da530..7f84f2ac2a 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -2,17 +2,19 @@ git-annex extends git's usual remotes with some [[special_remotes]], that are not git repositories. This way you can set up a remote using say, Amazon S3, and use git-annex to transfer files into the cloud. -**Note that encrypted buckets are not (yet) supported. Data sent to S3 -is without encryption susceptible to snooping.** - First, export your S3 credentials: # export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91" # export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t" -Next, create the S3 remote, and describe it. +Now, create a gpg key, if you don't already have one. This will be used +to encrypt everything stored in S3, for your privacy. Once you have +a gpg key, run `gpg --list-secret-keys` to look up its key id, something +like "2512E3C7" - # git annex initremote mys3 type=S3 encryption=none +Next, create the S3 remote, and describe it. + + # git annex initremote mys3 type=S3 encryption=2512E3C7 initremote mys3 (checking bucket) (creating bucket in US) ok # git annex describe mys3 "at Amazon's US datacenter" describe mys3 ok From 98e3817466130209d88d5061be9a590cdd609e78 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 19:45:59 -0400 Subject: [PATCH 1531/8313] don't let users change encryption type once remote is initted --- Remote/Encrypted.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs index 2a0fb13bca..0ff2833b37 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encrypted.hs @@ -23,7 +23,8 @@ encryptionSetup :: RemoteConfig -> Annex RemoteConfig encryptionSetup c = case (M.lookup "encryption" c, extractCipher c) of (Nothing, Nothing) -> error "Specify encryption=key or encryption=none" - (Just "none", _) -> return c + (Just "none", Nothing) -> return c + (Just "none", Just _) -> error "Cannot change encryption type of existing remote." (Nothing, Just _) -> return c (Just _, Nothing) -> use $ genCipher c (Just _, Just v) -> use $ updateCipher c v From 991efddfa1333839885c9bc5490ff79d7dfc046c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 21:41:14 -0400 Subject: [PATCH 1532/8313] refactor --- Remote/Directory.hs | 99 ++++++++++++++++++++++++--------------------- Remote/Encrypted.hs | 52 +++++++++++++++--------- 2 files changed, 86 insertions(+), 65 deletions(-) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 5ea0a1e6b3..2d31d12b2d 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -42,17 +42,20 @@ gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost - return $ Remote { - uuid = u, - cost = cst, - name = Git.repoDescribe r, - storeKey = storeKeyEncrypted c $ store dir, - retrieveKeyFile = retrieveKeyFileEncrypted c $ retrieve dir, - removeKey = removeKeyEncrypted c $ remove dir, - hasKey = hasKeyEncrypted c $ checkPresent dir, - hasKeyCheap = True, - config = Nothing - } + return $ encryptedRemote c + (storeEncrypted dir) + (retrieveEncrypted dir) + Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store dir, + retrieveKeyFile = retrieve dir, + removeKey = remove dir, + hasKey = checkPresent dir, + hasKeyCheap = True, + config = Nothing + } directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig directorySetup u c = do @@ -74,43 +77,47 @@ dirKey d k = d hashDirMixed k f f where f = keyFile k -store :: FilePath -> Key -> Maybe (Cipher, Key) -> Annex Bool -store d k c = do +store :: FilePath -> Key -> Annex Bool +store d k = do g <- Annex.gitRepo - let src = gitAnnexLocation g k - liftIO $ catch (copy src) (const $ return False) - where - copy src = case c of - Just (cipher, enckey) -> do - content <- L.readFile src - let dest = dirKey d enckey - prep dest - withEncryptedContent cipher content $ \s -> do - L.writeFile dest s - cleanup True dest - _ -> do - let dest = dirKey d k - prep dest - ok <- copyFile src dest - cleanup ok dest - prep dest = liftIO $ do - let dir = parentDir dest - createDirectoryIfMissing True dir - allowWrite dir - cleanup ok dest = do - when ok $ do - let dir = parentDir dest - preventWrite dest - preventWrite dir - return ok + let src = gitAnnexLocation g k + let dest = dirKey d k + liftIO $ catch (storeHelper dest $ copyFile src dest) (const $ return False) -retrieve :: FilePath -> Key -> FilePath -> Maybe (Cipher, Key) -> Annex Bool -retrieve d k f Nothing = liftIO $ copyFile (dirKey d k) f -retrieve d k f (Just (cipher, enckey)) = - liftIO $ flip catch (const $ return False) $ do - content <- L.readFile (dirKey d enckey) - withDecryptedContent cipher content $ L.writeFile f - return True +storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted d (cipher, enck) k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + let dest = dirKey d enck + liftIO $ catch (storeHelper dest $ encrypt src dest) (const $ return False) + where + encrypt src dest = do + content <- L.readFile src + withEncryptedContent cipher content $ L.writeFile dest + return True + +storeHelper :: FilePath -> IO Bool -> IO Bool +storeHelper dest a = do + let dir = parentDir dest + createDirectoryIfMissing True dir + allowWrite dir + ok <- a + when ok $ do + preventWrite dest + preventWrite dir + return ok + +retrieve :: FilePath -> Key -> FilePath -> Annex Bool +retrieve d k f = liftIO $ copyFile (dirKey d k) f + +retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted d (cipher, enck) f = + liftIO $ catch decrypt (const $ return False) + where + decrypt = do + content <- L.readFile (dirKey d enck) + withDecryptedContent cipher content $ L.writeFile f + return True remove :: FilePath -> Key -> Annex Bool remove d k = liftIO $ catch del (const $ return False) diff --git a/Remote/Encrypted.hs b/Remote/Encrypted.hs index 0ff2833b37..255b41d730 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encrypted.hs @@ -33,16 +33,39 @@ encryptionSetup c = cipher <- liftIO a return $ M.delete "encryption" $ storeCipher c cipher -{- Helpers that can be applied to a Remote's normal actions to - - add crypto support. -} -storeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Maybe (Cipher, Key) -> Annex a) -> Key -> Annex a -storeKeyEncrypted c a k = a k =<< cipherKey c k -retrieveKeyFileEncrypted :: Maybe RemoteConfig -> (Key -> FilePath -> Maybe (Cipher, Key) -> Annex a) -> Key -> FilePath -> Annex a -retrieveKeyFileEncrypted c a k f = a k f =<< cipherKey c k -removeKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a -removeKeyEncrypted = withEncryptedKey -hasKeyEncrypted :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a -hasKeyEncrypted = withEncryptedKey +{- Modifies a Remote to support encryption. + - + - Two additional functions must be provided by the remote, + - to support storing and retrieving encrypted content. -} +encryptedRemote + :: Maybe RemoteConfig + -> ((Cipher, Key) -> Key -> Annex Bool) + -> ((Cipher, Key) -> FilePath -> Annex Bool) + -> Remote Annex + -> Remote Annex +encryptedRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = + r { + storeKey = store, + retrieveKeyFile = retrieve, + removeKey = withkey $ removeKey r, + hasKey = withkey $ hasKey r + } + where + store k = do + v <- cipherKey c k + case v of + Nothing -> (storeKey r) k + Just x -> storeKeyEncrypted x k + retrieve k f = do + v <- cipherKey c k + case v of + Nothing -> (retrieveKeyFile r) k f + Just x -> retrieveKeyFileEncrypted x f + withkey a k = do + v <- cipherKey c k + case v of + Nothing -> a k + Just (_, k') -> a k' {- Gets encryption Cipher, and encrypted version of Key. - @@ -64,12 +87,3 @@ cipherKey (Just c) k = do ret cipher = do k' <- liftIO $ encryptKey cipher k return $ Just (cipher, k') - -{- Passes the encrypted version of the key to the action when encryption - - is enabled, and the non-encrypted version otherwise. -} -withEncryptedKey :: Maybe RemoteConfig -> (Key -> Annex a) -> Key -> Annex a -withEncryptedKey c a k = do - v <- cipherKey c k - case v of - Nothing -> a k - Just (_, k') -> a k' From 480cc353c46d88c55b252fbb6c5dc4feff08995c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 23:01:29 -0400 Subject: [PATCH 1533/8313] incomplete and buggy encryption support for bup Some kind of laziness issue that I don't want to debug right now, and decryption is not implemented. --- Remote/Bup.hs | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index b4403bb03e..6f4c9278e8 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -7,6 +7,7 @@ module Remote.Bup (remote) where +import qualified Data.ByteString.Lazy.Char8 as L import IO import Control.Exception.Extensible (IOException) import qualified Data.Map as M @@ -16,6 +17,7 @@ import System.Process import System.Exit import System.FilePath import Data.List.Utils +import System.Cmd.Utils import RemoteClass import Types @@ -29,6 +31,7 @@ import Messages import Ssh import Remote.Special import Remote.Encrypted +import Crypto type BupRepo = String @@ -47,16 +50,17 @@ gen r u c = do bupr <- liftIO $ bup2GitRemote buprepo (u', bupr') <- getBupUUID bupr u - return $ this cst buprepo u' bupr' - where - this cst buprepo u' bupr = Remote { + return $ encryptedRemote c + (storeEncrypted r buprepo) + (retrieveEncrypted buprepo) + Remote { uuid = u', cost = cst, name = Git.repoDescribe r, storeKey = store r buprepo, retrieveKeyFile = retrieve buprepo, removeKey = remove, - hasKey = checkPresent r bupr, + hasKey = checkPresent r bupr', hasKeyCheap = True, config = c } @@ -92,13 +96,34 @@ bup command buprepo params = do showProgress -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params +bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandParam] +bupSplitParams r buprepo k src = do + o <- getConfig r "bup-split-options" "" + let os = map Param $ words o + showProgress -- make way for bup output + return $ bupParams "split" buprepo + (os ++ [Param "-n", Param (show k), src]) + store :: Git.Repo -> BupRepo -> Key -> Annex Bool store r buprepo k = do g <- Annex.gitRepo let src = gitAnnexLocation g k - o <- getConfig r "bup-split-options" "" - let os = map Param $ words o - bup "split" buprepo $ os ++ [Param "-n", Param (show k), File src] + params <- bupSplitParams r buprepo k (File src) + liftIO $ boolSystem "bup" params + +storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted r buprepo (cipher, enck) k = do + g <- Annex.gitRepo + let src = gitAnnexLocation g k + params <- bupSplitParams r buprepo enck (Param "-") + liftIO $ flip catch (const $ return False) $ do + content <- L.readFile src + -- FIXME hangs after a while + (pid, h) <- hPipeTo "bup" (toCommand params) + withEncryptedContent cipher content $ L.hPut h + hClose h + forceSuccess pid + return True retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do @@ -116,6 +141,10 @@ retrieve buprepo k f = do Right r -> return r Left _ -> return False +retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted bupreoo (cipher, enck) f = do + error "TODO" + remove :: Key -> Annex Bool remove _ = do warning "content cannot be removed from bup remote" From d82898841581bab7785a0010f49e21c5eec5b51b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 23:02:09 -0400 Subject: [PATCH 1534/8313] proper encrypted keys For HMAC, using the Data.Digest.Pure.SHA library. I have been avoiding this library for checksumming generally, since it's (probably) not as fast as external utilities, but it's fine to use it for HMAC. --- Crypto.hs | 17 +++++++++-------- doc/install.mdwn | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 9f404c1b17..25d9a11573 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -24,6 +24,8 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import qualified Codec.Binary.Base64 as B64 +import Data.ByteString.Lazy.UTF8 (fromString) +import Data.Digest.Pure.SHA import System.Cmd.Utils import Data.String.Utils import Data.List @@ -105,12 +107,11 @@ decryptCipher _ (EncryptedCipher encipher _) = - reversable, nor does it need to be the same type of encryption used - on content. It does need to be repeatable. -} encryptKey :: Cipher -> Key -> IO Key -encryptKey c k = +encryptKey (Cipher c) k = return Key { - -- FIXME: should use HMAC with the cipher; I don't - -- have Data.Crypto in Debian yet though. - keyName = show k, - keyBackendName = "INSECURE", + keyName = showDigest $ + hmacSha1 (fromString $ show k) (fromString c), + keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data } @@ -154,12 +155,12 @@ gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) gpgCipher params (Cipher c) input a = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe - toh <- fdToHandle topipe - let Fd fromno = frompipe _ <- forkIO $ do + toh <- fdToHandle topipe hPutStrLn toh c hClose toh - let passphrase = [Param "--passphrase-fd", Param $ show fromno] + let Fd passphrasefd = frompipe + let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] (pid, output) <- gpgPipeBytes (passphrase ++ params) input ret <- a output diff --git a/doc/install.mdwn b/doc/install.mdwn index 746352cb8a..7a0c6020ba 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,6 +13,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: +* SHA: * TestPack * QuickCheck 2 * hS3 (optional, but recommended) From 11da36e48fb0a9de35b8b386a0c4156b6dfd0ead Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 16 Apr 2011 23:05:26 -0400 Subject: [PATCH 1535/8313] build dep update --- debian/control | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 42c79c91d8..ae0ca0c598 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,18 @@ Source: git-annex Section: utils Priority: optional -Build-Depends: debhelper (>= 7.0.50), ghc6, libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, ikiwiki, uuid, rsync, git | git-core, perlmagick +Build-Depends: + debhelper (>= 7.0.50), + ghc6, + libghc6-missingh-dev, + libghc6-pcre-light-dev, + libghc6-testpack-dev, + libghc6-sha-dev + ikiwiki, + perlmagick, + git | git-core, + uuid, + rsync, Maintainer: Joey Hess Standards-Version: 3.9.1 Vcs-Git: git://git.kitenet.net/git-annex @@ -10,7 +21,11 @@ Homepage: http://git-annex.branchable.com/ Package: git-annex Architecture: any Section: utils -Depends: ${misc:Depends}, ${shlibs:Depends}, git | git-core, uuid, openssh-client, rsync +Depends: ${misc:Depends}, ${shlibs:Depends}, + git | git-core, + uuid, + rsync, + openssh-client Suggests: graphviz, bup, gnupg Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file From d996637fd68430b4236d2899c49827cbf457471f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:34:38 -0400 Subject: [PATCH 1536/8313] fix stall while storing encrypted data in bup Forking a new process rather than relying on a thread to feed gpg. The feeder thread was stalling, probably when the main thread got to the point it was wait()ing on the gpg to exit. --- Crypto.hs | 42 ++++++++++++++++++++++++++++-------------- Remote/Bup.hs | 31 +++++++++++++------------------ 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 25d9a11573..4ec186ea22 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -17,6 +17,7 @@ module Crypto ( extractCipher, decryptCipher, encryptKey, + withEncryptedContentHandle, withEncryptedContent, withDecryptedContent, ) where @@ -33,8 +34,10 @@ import Data.Bits.Utils import System.IO import System.Posix.IO import System.Posix.Types +import System.Posix.Process import Control.Concurrent import Control.Exception +import System.Exit import Types import Key @@ -116,6 +119,11 @@ encryptKey (Cipher c) k = keyMtime = Nothing -- to avoid leaking data } +{- Runs an action passing it a handle from which it can + - stream encrypted content. -} +withEncryptedContentHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withEncryptedContentHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] + {- Streams encrypted content to an action. -} withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a withEncryptedContent = gpgCipher [Params "--symmetric --force-mdc"] @@ -142,17 +150,10 @@ gpgPipeStrict params input = do forceSuccess pid return output -gpgPipeBytes :: [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString) -gpgPipeBytes params input = do - (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) - _ <- forkIO $ finally (L.hPut toh input) (hClose toh) - output <- L.hGetContents fromh - return (pid, output) - {- Runs gpg with a cipher and some parameters, feeding it an input, - - and piping its output lazily to an action. -} -gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -gpgCipher params (Cipher c) input a = do + - and passing a handle to its output to an action. -} +gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +gpgCipherHandle params (Cipher c) input a = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe _ <- forkIO $ do @@ -161,16 +162,29 @@ gpgCipher params (Cipher c) input a = do hClose toh let Fd passphrasefd = frompipe let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] - (pid, output) <- gpgPipeBytes (passphrase ++ params) input - - ret <- a output + + (pid, fromh, toh) <- hPipeBoth "gpg" $ + gpgParams $ passphrase ++ params + _ <- forkProcess $ do + L.hPut toh input + hClose toh + exitSuccess + hClose toh + ret <- a fromh -- cleanup forceSuccess pid closeFd frompipe - return ret +{- Runs gpg with a cipher and some parameters, feeding it an input, + - and piping its output lazily to an action. -} +gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +gpgCipher params c input a = do + gpgCipherHandle params c input $ \h -> do + content <- L.hGetContents h + a content + configKeyIds :: RemoteConfig -> IO KeyIds configKeyIds c = do let k = configGet c "encryption" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 6f4c9278e8..771212372f 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -17,7 +17,6 @@ import System.Process import System.Exit import System.FilePath import Data.List.Utils -import System.Cmd.Utils import RemoteClass import Types @@ -96,6 +95,15 @@ bup command buprepo params = do showProgress -- make way for bup output liftIO $ boolSystem "bup" $ bupParams command buprepo params +pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool +pipeBup params inh outh = do + p <- runProcess "bup" (toCommand params) + Nothing Nothing inh outh Nothing + ok <- waitForProcess p + case ok of + ExitSuccess -> return True + _ -> return False + bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandParam] bupSplitParams r buprepo k src = do o <- getConfig r "bup-split-options" "" @@ -118,28 +126,15 @@ storeEncrypted r buprepo (cipher, enck) k = do params <- bupSplitParams r buprepo enck (Param "-") liftIO $ flip catch (const $ return False) $ do content <- L.readFile src - -- FIXME hangs after a while - (pid, h) <- hPipeTo "bup" (toCommand params) - withEncryptedContent cipher content $ L.hPut h - hClose h - forceSuccess pid - return True + withEncryptedContentHandle cipher content $ \h -> do + pipeBup params (Just h) Nothing retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do let params = bupParams "join" buprepo [Param $ show k] - ret <- liftIO $ try $ do - -- pipe bup's stdout directly to file + liftIO $ flip catch (const $ return False) $ do tofile <- openFile f WriteMode - p <- runProcess "bup" (toCommand params) - Nothing Nothing Nothing (Just tofile) Nothing - r <- waitForProcess p - case r of - ExitSuccess -> return True - _ -> return False - case ret of - Right r -> return r - Left _ -> return False + pipeBup params Nothing (Just tofile) retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted bupreoo (cipher, enck) f = do From b6b04642c8d513aaa75b924e1ef8480fa39f3109 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:40:23 -0400 Subject: [PATCH 1537/8313] rename --- Remote/Bup.hs | 4 ++-- Remote/Directory.hs | 4 ++-- Remote/{Encrypted.hs => Encryptable.hs} | 8 ++++---- Remote/S3real.hs | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) rename Remote/{Encrypted.hs => Encryptable.hs} (93%) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 771212372f..698d1b188a 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -29,7 +29,7 @@ import Utility import Messages import Ssh import Remote.Special -import Remote.Encrypted +import Remote.Encryptable import Crypto type BupRepo = String @@ -49,7 +49,7 @@ gen r u c = do bupr <- liftIO $ bup2GitRemote buprepo (u', bupr') <- getBupUUID bupr u - return $ encryptedRemote c + return $ encryptableRemote c (storeEncrypted r buprepo) (retrieveEncrypted buprepo) Remote { diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 2d31d12b2d..a84a1f45a7 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -27,7 +27,7 @@ import Config import Content import Utility import Remote.Special -import Remote.Encrypted +import Remote.Encryptable import Crypto remote :: RemoteType Annex @@ -42,7 +42,7 @@ gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do dir <- getConfig r "directory" (error "missing directory") cst <- remoteCost r cheapRemoteCost - return $ encryptedRemote c + return $ encryptableRemote c (storeEncrypted dir) (retrieveEncrypted dir) Remote { diff --git a/Remote/Encrypted.hs b/Remote/Encryptable.hs similarity index 93% rename from Remote/Encrypted.hs rename to Remote/Encryptable.hs index 255b41d730..a9a7472fb5 100644 --- a/Remote/Encrypted.hs +++ b/Remote/Encryptable.hs @@ -1,11 +1,11 @@ -{- common functions for encrypted remotes +{- common functions for encryptable remotes - - Copyright 2011 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Encrypted where +module Remote.Encryptable where import qualified Data.Map as M import Control.Monad.State (liftIO) @@ -37,13 +37,13 @@ encryptionSetup c = - - Two additional functions must be provided by the remote, - to support storing and retrieving encrypted content. -} -encryptedRemote +encryptableRemote :: Maybe RemoteConfig -> ((Cipher, Key) -> Key -> Annex Bool) -> ((Cipher, Key) -> FilePath -> Annex Bool) -> Remote Annex -> Remote Annex -encryptedRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = +encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = r { storeKey = store, retrieveKeyFile = retrieve, diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 0f6327f575..1fa387d68e 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -28,7 +28,7 @@ import Messages import Locations import Config import Remote.Special -import Remote.Encrypted +import Remote.Encryptable remote :: RemoteType Annex remote = RemoteType { From 9606409b9dd5218b0540418170ec86e9e1cec038 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:57:11 -0400 Subject: [PATCH 1538/8313] bup encryption support 100% working --- Remote/Bup.hs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 698d1b188a..0edb33ba8a 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -17,6 +17,7 @@ import System.Process import System.Exit import System.FilePath import Data.List.Utils +import System.Cmd.Utils import RemoteClass import Types @@ -124,7 +125,7 @@ storeEncrypted r buprepo (cipher, enck) k = do g <- Annex.gitRepo let src = gitAnnexLocation g k params <- bupSplitParams r buprepo enck (Param "-") - liftIO $ flip catch (const $ return False) $ do + liftIO $ catchBool $ do content <- L.readFile src withEncryptedContentHandle cipher content $ \h -> do pipeBup params (Just h) Nothing @@ -132,13 +133,19 @@ storeEncrypted r buprepo (cipher, enck) k = do retrieve :: BupRepo -> Key -> FilePath -> Annex Bool retrieve buprepo k f = do let params = bupParams "join" buprepo [Param $ show k] - liftIO $ flip catch (const $ return False) $ do + liftIO $ catchBool $ do tofile <- openFile f WriteMode pipeBup params Nothing (Just tofile) retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool -retrieveEncrypted bupreoo (cipher, enck) f = do - error "TODO" +retrieveEncrypted buprepo (cipher, enck) f = do + let params = bupParams "join" buprepo [Param $ show enck] + liftIO $ catchBool $ do + (pid, h) <- hPipeFrom "bup" $ toCommand params + content <- L.hGetContents h + withDecryptedContent cipher content $ L.writeFile f + forceSuccess pid + return True remove :: Key -> Annex Bool remove _ = do From 89fab6c7b8955ef26e653d539f7be3b70129c15e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 00:57:29 -0400 Subject: [PATCH 1539/8313] refactor --- Remote/Directory.hs | 25 +++++++++++-------------- Utility.hs | 5 +++++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Remote/Directory.hs b/Remote/Directory.hs index a84a1f45a7..d9bee80c3f 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -82,14 +82,14 @@ store d k = do g <- Annex.gitRepo let src = gitAnnexLocation g k let dest = dirKey d k - liftIO $ catch (storeHelper dest $ copyFile src dest) (const $ return False) + liftIO $ catchBool $ storeHelper dest $ copyFile src dest storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted d (cipher, enck) k = do g <- Annex.gitRepo let src = gitAnnexLocation g k let dest = dirKey d enck - liftIO $ catch (storeHelper dest $ encrypt src dest) (const $ return False) + liftIO $ catchBool $ storeHelper dest $ encrypt src dest where encrypt src dest = do content <- L.readFile src @@ -112,23 +112,20 @@ retrieve d k f = liftIO $ copyFile (dirKey d k) f retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted d (cipher, enck) f = - liftIO $ catch decrypt (const $ return False) - where - decrypt = do - content <- L.readFile (dirKey d enck) - withDecryptedContent cipher content $ L.writeFile f - return True + liftIO $ catchBool $ do + content <- L.readFile (dirKey d enck) + withDecryptedContent cipher content $ L.writeFile f + return True remove :: FilePath -> Key -> Annex Bool -remove d k = liftIO $ catch del (const $ return False) +remove d k = liftIO $ catchBool $ do + allowWrite dir + removeFile file + removeDirectory dir + return True where file = dirKey d k dir = parentDir file - del = do - allowWrite dir - removeFile file - removeDirectory dir - return True checkPresent :: FilePath -> Key -> Annex (Either IOException Bool) checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k) diff --git a/Utility.hs b/Utility.hs index 1c6b4d21e6..5639a8799a 100644 --- a/Utility.hs +++ b/Utility.hs @@ -24,6 +24,7 @@ module Utility ( dirContains, dirContents, myHomeDir, + catchBool, prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, @@ -256,3 +257,7 @@ myHomeDir = do uid <- getEffectiveUserID u <- getUserEntryForID uid return $ homeDirectory u + +{- Catches IO errors and returns a Bool -} +catchBool :: IO Bool -> IO Bool +catchBool = flip catch (const $ return False) From 50cfcdf54b828fbeab532b712e00063ae9e82581 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 01:13:21 -0400 Subject: [PATCH 1540/8313] make encrypted remotes have slightly higher costs --- Config.hs | 17 ++++++++++++++++- Remote/Encryptable.hs | 4 +++- test.hs | 2 ++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Config.hs b/Config.hs index 53f1a455fd..a324427d44 100644 --- a/Config.hs +++ b/Config.hs @@ -52,10 +52,25 @@ remoteCost r def = do cheapRemoteCost :: Int cheapRemoteCost = 100 semiCheapRemoteCost :: Int -semiCheapRemoteCost = 150 +semiCheapRemoteCost = 110 expensiveRemoteCost :: Int expensiveRemoteCost = 200 +{- Adjust's a remote's cost to reflect it being encrypted. -} +encryptedRemoteCostAdj :: Int +encryptedRemoteCostAdj = 50 + +{- Make sure the remote cost numbers work out. -} +prop_cost_sane :: Bool +prop_cost_sane = False `notElem` + [ expensiveRemoteCost > 0 + , cheapRemoteCost < semiCheapRemoteCost + , semiCheapRemoteCost < expensiveRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj > semiCheapRemoteCost + , cheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + , semiCheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost + ] + {- Checks if a repo should be ignored, based either on annex-ignore - setting, or on command-line options. Allows command-line to override - annex-ignore. -} diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index a9a7472fb5..aa7c2a5691 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -15,6 +15,7 @@ import RemoteClass import Crypto import qualified Annex import Messages +import Config {- Encryption setup for a remote. The user must specify whether to use - an encryption key, or not encrypt. An encrypted cipher is created, or is @@ -48,7 +49,8 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = storeKey = store, retrieveKeyFile = retrieve, removeKey = withkey $ removeKey r, - hasKey = withkey $ hasKey r + hasKey = withkey $ hasKey r, + cost = cost r + encryptedRemoteCostAdj } where store k = do diff --git a/test.hs b/test.hs index 30ebe6e593..cdec4ea614 100644 --- a/test.hs +++ b/test.hs @@ -39,6 +39,7 @@ import qualified Remote import qualified Content import qualified Command.DropUnused import qualified Key +import qualified Config main :: IO () main = do @@ -61,6 +62,7 @@ quickcheck = TestLabel "quickcheck" $ TestList , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics + , qctest "prop_cost_sane" Config.prop_cost_sane ] blackbox :: Test From 4d136e1ef5a3c06bbc8e10a5aa7ac20e17a39c4f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 01:34:28 -0400 Subject: [PATCH 1541/8313] use different parts of cipher for hmac and gpg Per bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing It may be paranoid to worry about the cipher being recovered from hmac keys, but yes.. let's be paranoid. --- Crypto.hs | 35 ++++++++++++++----- ...e_same_key_for_encryption_and_hashing.mdwn | 4 +-- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 4ec186ea22..e42a21e282 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -45,6 +45,24 @@ import RemoteClass import Utility import CryptoTypes +{- The first half of a Cipher is used for HMAC; the remainder + - is used as the GPG symmetric encryption passphrase. + - + - 256 is enough for gpg's symetric cipher; unlike weaker public key + - crypto, the key does not need to be too large. + -} +cipherHalf :: Int +cipherHalf = 256 + +cipherSize :: Int +cipherSize = cipherHalf * 2 + +cipherPassphrase :: Cipher -> String +cipherPassphrase (Cipher c) = drop cipherHalf c + +cipherHmac :: Cipher -> String +cipherHmac (Cipher c) = take cipherHalf c + {- Creates a new Cipher, encrypted as specified in the remote's configuration -} genCipher :: RemoteConfig -> IO EncryptedCipher genCipher c = do @@ -58,10 +76,10 @@ genCipher c = do -- newline. [ Params "--gen-random --armor" , Param $ show randomquality - , Param $ show ciphersize + , Param $ show cipherSize ] - randomquality = 1 :: Int -- 1 is /dev/urandom; 2 is /dev/random - ciphersize = 256 :: Int + -- 1 is /dev/urandom; 2 is /dev/random + randomquality = 1 :: Int {- Updates an existing Cipher, re-encrypting it to add KeyIds specified in - the remote's configuration. -} @@ -110,10 +128,11 @@ decryptCipher _ (EncryptedCipher encipher _) = - reversable, nor does it need to be the same type of encryption used - on content. It does need to be repeatable. -} encryptKey :: Cipher -> Key -> IO Key -encryptKey (Cipher c) k = +encryptKey c k = return Key { - keyName = showDigest $ - hmacSha1 (fromString $ show k) (fromString c), + keyName = showDigest $ hmacSha1 + (fromString $ show k) + (fromString $ cipherHmac c), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data @@ -153,12 +172,12 @@ gpgPipeStrict params input = do {- Runs gpg with a cipher and some parameters, feeding it an input, - and passing a handle to its output to an action. -} gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a -gpgCipherHandle params (Cipher c) input a = do +gpgCipherHandle params c input a = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe _ <- forkIO $ do toh <- fdToHandle topipe - hPutStrLn toh c + hPutStrLn toh $ cipherPassphrase c hClose toh let Fd passphrasefd = frompipe let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn index 1980a8f444..9fc31fa485 100644 --- a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -4,5 +4,5 @@ Also, ttbomk, HMAC needs two keys, not one. Are you re-using the same key twice? Compability for old buckets and support for different ones can be maintained by introducing a new option and simply copying over the encryption key's identifier into this new option should it be missing. -> See [[design/encryption]]. I don't think this bug needs to be kept -> open. [[done]] --[[Joey]] +> Bug was filed prematurely, but was a good bit of paranoia, and gpg and +> hmac are given different secret keys [[done]] --[[Joey]] From 67cced26dc3407a749f01010515e2d2827af2a10 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:01:34 -0400 Subject: [PATCH 1542/8313] S3 crypto support Untested, I will need to dust off my S3 keys, and plug the modem back in that was unplugged last night due to very low battery bank power. But it compiles, so it's probably perfect. :) --- Remote/S3real.hs | 50 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 1fa387d68e..b88b22037f 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -29,6 +29,7 @@ import Locations import Config import Remote.Special import Remote.Encryptable +import Crypto remote :: RemoteType Annex remote = RemoteType { @@ -41,16 +42,22 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost - return $ this cst + return $ gen' r u c cst +gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex +gen' r u c cst = + encryptableRemote c + (storeEncrypted this) + (retrieveEncrypted this) + this where - this cst = Remote { + this = Remote { uuid = u, cost = cst, name = Git.repoDescribe r, - storeKey = store (this cst), - retrieveKeyFile = retrieve (this cst), - removeKey = remove (this cst), - hasKey = checkPresent (this cst), + storeKey = store this, + retrieveKeyFile = retrieve this, + removeKey = remove this, + hasKey = checkPresent this, hasKeyCheap = False, config = c } @@ -139,9 +146,21 @@ checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do noconn = Left $ error "S3 not configured" store :: Remote Annex -> Key -> Annex Bool -store r k = s3Action r False $ \(conn, bucket) -> do +store r k = storeHelper r k =<< lazyKeyContent k + +storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted r (cipher, enck) k = do + content <- lazyKeyContent k + content' <- liftIO $ withEncryptedContent cipher content return + storeHelper r enck content' + +lazyKeyContent :: Key -> Annex L.ByteString +lazyKeyContent k = do g <- Annex.gitRepo - content <- liftIO $ L.readFile $ gitAnnexLocation g k + liftIO $ L.readFile $ gitAnnexLocation g k + +storeHelper :: Remote Annex -> Key -> L.ByteString -> Annex Bool +storeHelper r k content = s3Action r False $ \(conn, bucket) -> do let object = setStorageClass storageclass $ bucketKey bucket k content res <- liftIO $ sendObject conn object case res of @@ -156,16 +175,25 @@ store r k = s3Action r False $ \(conn, bucket) -> do _ -> STANDARD retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool -retrieve r k f = s3Action r False $ \(conn, bucket) -> do +retrieve = retrieveHelper (return . obj_data) + +retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) f = retrieveHelper decrypt r enck f + where + decrypt o = withDecryptedContent cipher (obj_data o) return + +retrieveHelper :: (S3Object -> IO L.ByteString) -> Remote Annex -> Key -> FilePath -> Annex Bool +retrieveHelper a r k f = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey bucket k L.empty case res of Right o -> do - liftIO $ L.writeFile f (obj_data o) + content <- liftIO $ a o + liftIO $ L.writeFile f content return True Left e -> do warning $ prettyReqError e return False - + remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty From 3d2a0f68b53dd6045558c232f5e5d860dde3ff91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:08:11 -0400 Subject: [PATCH 1543/8313] note --- doc/special_remotes/bup.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/special_remotes/bup.mdwn b/doc/special_remotes/bup.mdwn index 5bc1fb7a2e..e59ff240de 100644 --- a/doc/special_remotes/bup.mdwn +++ b/doc/special_remotes/bup.mdwn @@ -3,6 +3,10 @@ This special remote type stores file contents in a in the front-end, and bup as a remote, you get an easy git-style interface to large files, and easy backups of the file contents using git. +This is particularly well suited to collaboration on projects involving +large files, since both the git-annex and bup repositories can be +accessed like any other git repository. + See [[walkthrough/using_bup]] for usage examples. Each individual key is stored in a bup remote using `bup split`, with From eafb51959225d871b94136b7b02c5cf7752d49be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:09:33 -0400 Subject: [PATCH 1544/8313] hmacSha1 wants the secret key as first parameter I was offline last night and going by function signatures, and unable to tell which was which. Note sure it matters to HMAC which comes first; better safe than sorry. --- Crypto.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index e42a21e282..12c70ef048 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -131,8 +131,8 @@ encryptKey :: Cipher -> Key -> IO Key encryptKey c k = return Key { keyName = showDigest $ hmacSha1 - (fromString $ show k) - (fromString $ cipherHmac c), + (fromString $ cipherHmac c) + (fromString $ show k), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data From 80981600a0994ba36d3867d5ada55b5ea5a3d9cb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:13:54 -0400 Subject: [PATCH 1545/8313] looked up HMAC block size details --- Crypto.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Crypto.hs b/Crypto.hs index 12c70ef048..25a87c3ad9 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -48,6 +48,9 @@ import CryptoTypes {- The first half of a Cipher is used for HMAC; the remainder - is used as the GPG symmetric encryption passphrase. - + - HMAC SHA1 needs only 64 bytes. The remainder is for expansion, + - perhaps to HMAC SHA512, which needs 128 bytes (ideally). + - - 256 is enough for gpg's symetric cipher; unlike weaker public key - crypto, the key does not need to be too large. -} From 83423211a21d061b4f0d62c925dae7aa4cc62f98 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:27:24 -0400 Subject: [PATCH 1546/8313] design wrapup --- doc/design/encryption.mdwn | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 5a4bc8bbdf..b30e01cdda 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -22,8 +22,8 @@ The basis of this scheme was originally developed by Lars Wirzenius et al [for Obnam](http://braawi.org/obnam/encryption/). """]] -Data is encrypted by gpg, using a symmetric cipher. The passphrase of the -cipher is itself checked into your git repository, encrypted using one or +Data is encrypted by gpg, using a symmetric cipher. +The cipher is itself checked into your git repository, encrypted using one or more gpg public keys. This scheme allows new gpg private keys to be given access to content that has already been stored in the remote. @@ -58,6 +58,33 @@ for each file in the repository, contact the encrypted remote to check if it has the file. This can be done without enumeration, although it will mean running gpg once per file fscked, to get the encrypted filename. +So, the files stored in the remote should be encrypted. But, it needs +to be a repeatable encryption, so they cannot just be gpg encrypted, +that would yeild a new name each time. Instead, HMAC is used. Any hash +could be used with HMAC; currently SHA1 is used. + +It was suggested that it might not be wise to use the same cipher for both +gpg and HMAC. Being paranoid, it's best not to tie the security of one +to the security of the other. So, the encrypted cipher described above is +actually split in two; half is used for HMAC, and half for gpg. + +---- + +Does the HMAC cipher need to be gpg encrypted? Imagine if it were +stored in plainext in the git repository. Anyone who can access +the git repository already knows the actual filenames, and typically also +the content hashes of annexed content. Having access to the HMAC cipher +could perhaps be said to only let them verify that data they already +know. + +While this seems a pretty persuasive argument, I'm not 100% convinced, and +anyway, most times that the HMAC cipher is needed, the gpg cipher is also +needed. Keeping the HMAC cipher encrypted does slow down two things: +dropping content from encrypted remotes, and checking if encrypted remotes +really have content. If it's later determined to be safe to not encrypt the +HMAC cipher, the current design allows changing that, even for existing +remotes. + ## risks A risk of this scheme is that, once the symmetric cipher has been obtained, it From f486768b169ec392448af75dc3a5f90de9d5a353 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:31:56 -0400 Subject: [PATCH 1547/8313] tweak wording --- Remote/Encryptable.hs | 2 +- doc/walkthrough/using_Amazon_S3.mdwn | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index aa7c2a5691..9e4f58ed40 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -81,7 +81,7 @@ cipherKey (Just c) k = do Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do - showNote "getting encryption key" + showNote "unlocking" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) ret cipher diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 7f84f2ac2a..0a13900b8d 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -30,8 +30,8 @@ repository use the same S3 remote is easy: Now the remote can be used like any other remote. # git annex copy my_cool_big_file --to mys3 - copy my_cool_big_file (to mys3...) ok + copy my_cool_big_file (unlocking) (checking mys3...) (to mys3...) ok # git annex move video/hackity_hack_and_kaxxt.mov --to mys3 - move video/hackity_hack_and_kaxxt.mov (to mys3...) ok + move video/hackity_hack_and_kaxxt.mov (unlocking) (checking mys3...) (to mys3...) ok See [[special_remotes/Amazon_S3]] for details. From 808040d72a9a3322096a2bb2448da4265c62a751 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:38:28 -0400 Subject: [PATCH 1548/8313] update to mention encryption --- doc/use_case/Alice.mdwn | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 80280580e3..24ed7c2a92 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -2,10 +2,13 @@ Alice is always on the move, often with her trusty netbook and a small handheld terabyte USB drive, or a smaller USB keydrive. She has a server -out there on the net. She stores data in the Cloud. All these things can -have different files on them, but Alice no longer has to deal with the -tedious process of keeping them manually in sync, or remembering where -she put a file. +out there on the net. She stores data, encrypted in the Cloud. + +All these things can have different files on them, but Alice no longer +has to deal with the tedious process of keeping them manually in sync, +or remembering where she put a file. git-annex manages all these data +sources as if they were git remotes. +[[more about special remotes|special_remotes]] When she has 1 bar on her cell, Alice queues up interesting files on her server for later. At a coffee shop, she has git-annex download them to her From d93e2f52c337f1d8016dbd887d69a5eec74e5a08 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:41:52 -0400 Subject: [PATCH 1549/8313] update --- doc/not.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/not.mdwn b/doc/not.mdwn index 009e1a79d7..2827dd12d0 100644 --- a/doc/not.mdwn +++ b/doc/not.mdwn @@ -5,6 +5,12 @@ system. For a backup system that uses git and that git-annex supports storing data in, see [[special_remotes/bup]]. +* git-annex is not a filesystem or DropBox clone. But there + is a FUSE filesystem built on top of git-annex, called + [ShareBox](https://github.com/chmduquesne/sharebox), and there is + interest in making it easy to use and covering some of the use + cases supported by DropBox. + * git-annex is not unison, but if you're finding unison's checksumming too slow, or its strict mirroring of everything to both places too limiting, then git-annex could be a useful alternative. From dcbe94b3a5c132470635bca522527c3c6c586948 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:44:32 -0400 Subject: [PATCH 1550/8313] layout --- doc/use_case/Alice.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/use_case/Alice.mdwn b/doc/use_case/Alice.mdwn index 24ed7c2a92..cdd3ea546d 100644 --- a/doc/use_case/Alice.mdwn +++ b/doc/use_case/Alice.mdwn @@ -7,7 +7,7 @@ out there on the net. She stores data, encrypted in the Cloud. All these things can have different files on them, but Alice no longer has to deal with the tedious process of keeping them manually in sync, or remembering where she put a file. git-annex manages all these data -sources as if they were git remotes. +sources as if they were git remotes. [[more about special remotes|special_remotes]] When she has 1 bar on her cell, Alice queues up interesting files on her From 416c5e38e7e8966dbf232692c0c5ab6b6430cea5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 11:45:34 -0400 Subject: [PATCH 1551/8313] link --- doc/index.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/index.mdwn b/doc/index.mdwn index 224d25d2cc..eb6307f327 100644 --- a/doc/index.mdwn +++ b/doc/index.mdwn @@ -44,6 +44,7 @@ files with git. * [[git-annex man page|git-annex]] * [[key-value backends|backends]] for data storage * [[special_remotes]] (including [[special_remotes/Amazon_S3]] and [[special_remotes/bup]]) +* [[encryption]] * [[bare_repositories]] * [[internals]] * [[design]] From db348896c3164d468bb137eaba77a9990aa2d6ac Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 17 Apr 2011 16:02:22 +0000 Subject: [PATCH 1552/8313] --- .../S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn index 9fc31fa485..2c0037c903 100644 --- a/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn +++ b/doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn @@ -6,3 +6,5 @@ Compability for old buckets and support for different ones can be maintained by > Bug was filed prematurely, but was a good bit of paranoia, and gpg and > hmac are given different secret keys [[done]] --[[Joey]] + +>> Thanks :) -- RIchiH From dd207994bc8026f22bb366e654a3945d5e995f87 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 12:36:12 -0400 Subject: [PATCH 1553/8313] reword again On second thought, "unlocking" is confusable with git-annex unlock. --- Remote/Encryptable.hs | 2 +- doc/walkthrough/using_Amazon_S3.mdwn | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 9e4f58ed40..11df2673cf 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -81,7 +81,7 @@ cipherKey (Just c) k = do Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do - showNote "unlocking" + showNote "gpg" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) ret cipher diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 0a13900b8d..a0ce951a84 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -14,24 +14,24 @@ like "2512E3C7" Next, create the S3 remote, and describe it. - # git annex initremote mys3 type=S3 encryption=2512E3C7 - initremote mys3 (checking bucket) (creating bucket in US) ok - # git annex describe mys3 "at Amazon's US datacenter" - describe mys3 ok + # git annex initremote cloud type=S3 encryption=2512E3C7 + initremote cloud (checking bucket) (creating bucket in US) ok + # git annex describe cloud "at Amazon's US datacenter" + describe cloud ok The configuration for the S3 remote is stored in git. So to make another repository use the same S3 remote is easy: # cd /media/usb/annex # git pull laptop master - # git annex initremote mys3 - initremote mys3 (checking bucket) ok + # git annex initremote cloud + initremote cloud (checking bucket) ok Now the remote can be used like any other remote. - # git annex copy my_cool_big_file --to mys3 - copy my_cool_big_file (unlocking) (checking mys3...) (to mys3...) ok - # git annex move video/hackity_hack_and_kaxxt.mov --to mys3 - move video/hackity_hack_and_kaxxt.mov (unlocking) (checking mys3...) (to mys3...) ok + # git annex copy my_cool_big_file --to cloud + copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok + # git annex move video/hackity_hack_and_kaxxt.mov --to cloud + move video/hackity_hack_and_kaxxt.mov (gpg) (checking cloud...) (to cloud...) ok See [[special_remotes/Amazon_S3]] for details. From 36f048979ff897bc631f499f6f2fe63803e0c1fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 12:43:36 -0400 Subject: [PATCH 1554/8313] releasing version 0.20110417 --- debian/changelog | 4 ++-- debian/control | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 04a6896e32..750f467d58 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110402) UNRELEASED; urgency=low +git-annex (0.20110417) unstable; urgency=low * bup is now supported as a special type of remote. * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted @@ -20,7 +20,7 @@ git-annex (0.20110402) UNRELEASED; urgency=low * Support "sha1" and "sha512" commands on FreeBSD, and allow building if any/all SHA commands are not available. Thanks, Fraser Tweedale - -- Joey Hess Sat, 02 Apr 2011 13:45:54 -0400 + -- Joey Hess Sun, 17 Apr 2011 12:00:24 -0400 git-annex (0.20110401) experimental; urgency=low diff --git a/debian/control b/debian/control index ae0ca0c598..ebc2487c5e 100644 --- a/debian/control +++ b/debian/control @@ -7,7 +7,7 @@ Build-Depends: libghc6-missingh-dev, libghc6-pcre-light-dev, libghc6-testpack-dev, - libghc6-sha-dev + libghc6-sha-dev, ikiwiki, perlmagick, git | git-core, From 8e53d0032e174fa613230568fe8c1f6347b75735 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 12:44:29 -0400 Subject: [PATCH 1555/8313] add news item for git-annex 0.20110417 --- doc/news/version_0.20110316.mdwn | 21 --------------------- doc/news/version_0.20110417.mdwn | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 doc/news/version_0.20110316.mdwn create mode 100644 doc/news/version_0.20110417.mdwn diff --git a/doc/news/version_0.20110316.mdwn b/doc/news/version_0.20110316.mdwn deleted file mode 100644 index 968722c0e6..0000000000 --- a/doc/news/version_0.20110316.mdwn +++ /dev/null @@ -1,21 +0,0 @@ -This version reorganises the layout of git-annex's files in your repository. -There is an upgrade process to convert a repository from the old git-annex -to this version. While git-annex will attempt to transparently handle -upgrades, you may want to drive the upgrade process by hand. -See [[upgrades]] for details. - -git-annex 0.20110316 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * New repository format, annex.version=2. - * The first time git-annex is run in an old format repository, it - will automatically upgrade it to the new format, staging all - necessary changes to git. Also added a "git annex upgrade" command. - * Colons are now avoided in filenames, so bare clones of git repos - can be put on USB thumb drives formatted with vFAT or similar - filesystems. - * Added two levels of hashing to object directory and .git-annex logs, - to improve scalability with enormous numbers of annexed - objects. (With one hundred million annexed objects, each - directory would contain fewer than 1024 files.) - * The setkey, fromkey, and dropkey subcommands have changed how - the key is specified. --backend is no longer used with these."""]] diff --git a/doc/news/version_0.20110417.mdwn b/doc/news/version_0.20110417.mdwn new file mode 100644 index 0000000000..7e28ea2138 --- /dev/null +++ b/doc/news/version_0.20110417.mdwn @@ -0,0 +1,21 @@ +git-annex 0.20110417 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * bup is now supported as a special type of remote. + * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted + using GPG for privacy. + * Use lowercase hash directories for locationlog files, to avoid + some issues with git on OSX with the mixed-case directories. + No migration is needed; the old mixed case hash directories are still + read; new information is written to the new directories. + * Unused files on remotes, particulary special remotes, can now be + identified and dropped, by using "--from remote" with git annex unused + and git annex dropunused. + * Clear up short option confusion between --from and --force (-f is now + --from, and there is no short option for --force). + * Add build depend on perlmagick so docs are consistently built. + Closes: #[621410](http://bugs.debian.org/621410) + * Add doc-base file. Closes: #[621408](http://bugs.debian.org/621408) + * Periodically flush git command queue, to avoid boating memory usage + too much. + * Support "sha1" and "sha512" commands on FreeBSD, and allow building + if any/all SHA commands are not available. Thanks, Fraser Tweedale"""]] \ No newline at end of file From d9690a9b5d6e706abe41fd76800ce9c526ad0b4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 13:11:38 -0400 Subject: [PATCH 1556/8313] cleanup --- Crypto.hs | 29 +++++++++++++++-------------- Remote/Bup.hs | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 25a87c3ad9..bbe8a6f4ce 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -17,7 +17,8 @@ module Crypto ( extractCipher, decryptCipher, encryptKey, - withEncryptedContentHandle, + withEncryptedHandle, + withDecryptedHandle, withEncryptedContent, withDecryptedContent, ) where @@ -141,19 +142,27 @@ encryptKey c k = keyMtime = Nothing -- to avoid leaking data } -{- Runs an action passing it a handle from which it can +{- Runs an action, passing it a handle from which it can - stream encrypted content. -} -withEncryptedContentHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a -withEncryptedContentHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] +withEncryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] + +{- Runs an action, passing it a handle from which it can + - stream decrypted content. -} +withDecryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withDecryptedHandle = gpgCipherHandle [Param "--decrypt"] {- Streams encrypted content to an action. -} withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -withEncryptedContent = gpgCipher [Params "--symmetric --force-mdc"] +withEncryptedContent = pass withEncryptedHandle {- Streams decrypted content to an action. -} withDecryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -withDecryptedContent = gpgCipher [Param "--decrypt"] +withDecryptedContent = pass withDecryptedHandle +pass :: (Cipher -> L.ByteString -> (Handle -> IO a) -> IO a) + -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +pass to c i a = to c i $ \h -> a =<< L.hGetContents h gpgParams :: [CommandParam] -> [String] gpgParams params = @@ -199,14 +208,6 @@ gpgCipherHandle params c input a = do closeFd frompipe return ret -{- Runs gpg with a cipher and some parameters, feeding it an input, - - and piping its output lazily to an action. -} -gpgCipher :: [CommandParam] -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a -gpgCipher params c input a = do - gpgCipherHandle params c input $ \h -> do - content <- L.hGetContents h - a content - configKeyIds :: RemoteConfig -> IO KeyIds configKeyIds c = do let k = configGet c "encryption" diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 0edb33ba8a..16e1bbdcb5 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -127,7 +127,7 @@ storeEncrypted r buprepo (cipher, enck) k = do params <- bupSplitParams r buprepo enck (Param "-") liftIO $ catchBool $ do content <- L.readFile src - withEncryptedContentHandle cipher content $ \h -> do + withEncryptedHandle cipher content $ \h -> do pipeBup params (Just h) Nothing retrieve :: BupRepo -> Key -> FilePath -> Annex Bool From 7aa668f4b488cc29fe0722f8f01071540ed56434 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 14:30:22 -0400 Subject: [PATCH 1557/8313] Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. --- Crypto.hs | 2 +- debian/changelog | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Crypto.hs b/Crypto.hs index bbe8a6f4ce..6b5d1218a3 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -167,7 +167,7 @@ pass to c i a = to c i $ \h -> a =<< L.hGetContents h gpgParams :: [CommandParam] -> [String] gpgParams params = -- avoid prompting, and be quiet, even about checking the trustdb - ["--batch", "--quiet", "--trust-model", "always"] ++ + ["--quiet", "--trust-model", "always"] ++ toCommand params gpgRead :: [CommandParam] -> IO String diff --git a/debian/changelog b/debian/changelog index 750f467d58..c837da8763 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.20110418) UNRELEASED; urgency=low + + * Don't run gpg in batch mode, so it can prompt for passphrase when + there is no agent. + + -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 + git-annex (0.20110417) unstable; urgency=low * bup is now supported as a special type of remote. From a91a51fc03a68f2a5ede6df5182471f6ebfcc037 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 14:41:24 -0400 Subject: [PATCH 1558/8313] Add missing build dep on dataenc. --- debian/changelog | 1 + debian/control | 2 ++ doc/install.mdwn | 1 + 3 files changed, 4 insertions(+) diff --git a/debian/changelog b/debian/changelog index c837da8763..aa00d3fca3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.20110418) UNRELEASED; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. + * Add missing build dep on dataenc. -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 diff --git a/debian/control b/debian/control index ebc2487c5e..00740b7403 100644 --- a/debian/control +++ b/debian/control @@ -8,6 +8,7 @@ Build-Depends: libghc6-pcre-light-dev, libghc6-testpack-dev, libghc6-sha-dev, + libghc6-dataenc-dev, ikiwiki, perlmagick, git | git-core, @@ -40,3 +41,4 @@ Description: manage files with git, without checking their contents into git versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. + diff --git a/doc/install.mdwn b/doc/install.mdwn index 7a0c6020ba..3d15eac604 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -14,6 +14,7 @@ To build and use git-annex, you will need: * pcre-light: * utf8-string: * SHA: +* dataenc: * TestPack * QuickCheck 2 * hS3 (optional, but recommended) From 1d943233185dc17466a985d0bd408389ee27de7b Mon Sep 17 00:00:00 2001 From: praet Date: Sun, 17 Apr 2011 20:22:24 +0000 Subject: [PATCH 1559/8313] --- doc/forum/wishlist:_command_options_changes.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes.mdwn diff --git a/doc/forum/wishlist:_command_options_changes.mdwn b/doc/forum/wishlist:_command_options_changes.mdwn new file mode 100644 index 0000000000..1d1b7087b8 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes.mdwn @@ -0,0 +1,14 @@ +Some suggestions for changes to command options: + + * --verbose: + * add alternate: -v + + * --from: + * replace with: -s $SOURCE || --source=$SOURCE + + * --to: + * replace with: -d $DESTINATION || --destination=$DESTINATION + + * --force: + * re-add alternate: -f (was removed in v0.20110417) + From f5845ce97b67d98495e42dec35e9821a561d5884 Mon Sep 17 00:00:00 2001 From: praet Date: Sun, 17 Apr 2011 20:26:56 +0000 Subject: [PATCH 1560/8313] --- doc/forum/wishlist:_command_options_changes.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_command_options_changes.mdwn b/doc/forum/wishlist:_command_options_changes.mdwn index 1d1b7087b8..250fd75c81 100644 --- a/doc/forum/wishlist:_command_options_changes.mdwn +++ b/doc/forum/wishlist:_command_options_changes.mdwn @@ -10,5 +10,5 @@ Some suggestions for changes to command options: * replace with: -d $DESTINATION || --destination=$DESTINATION * --force: - * re-add alternate: -f (was removed in v0.20110417) + * add alternate: -F (-f was removed in v0.20110417) From 0c856a99254b323cc5a8840a522092014f77cc38 Mon Sep 17 00:00:00 2001 From: praet Date: Sun, 17 Apr 2011 20:29:34 +0000 Subject: [PATCH 1561/8313] --- doc/forum/wishlist:_command_options_changes.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/wishlist:_command_options_changes.mdwn b/doc/forum/wishlist:_command_options_changes.mdwn index 250fd75c81..4509f815d7 100644 --- a/doc/forum/wishlist:_command_options_changes.mdwn +++ b/doc/forum/wishlist:_command_options_changes.mdwn @@ -10,5 +10,7 @@ Some suggestions for changes to command options: * replace with: -d $DESTINATION || --destination=$DESTINATION * --force: - * add alternate: -F (-f was removed in v0.20110417) + * add alternate: -F + * "-f" was removed in v0.20110417 + * since it forces unsafe operations, should be capitalized to reduce chance of accidental usage. From 86f7feb2784221459d4015b0d70f537dbf2e9bb7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 17 Apr 2011 18:18:27 -0400 Subject: [PATCH 1562/8313] initremote: show gpg keys --- Crypto.hs | 8 ++++++++ Remote/Encryptable.hs | 7 ++++--- doc/walkthrough/using_Amazon_S3.mdwn | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 6b5d1218a3..1617f5aadb 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -13,6 +13,7 @@ module Crypto ( EncryptedCipher, genCipher, updateCipher, + describeCipher, storeCipher, extractCipher, decryptCipher, @@ -95,6 +96,13 @@ updateCipher c encipher@(EncryptedCipher _ ks) = do where combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b +describeCipher :: EncryptedCipher -> String +describeCipher (EncryptedCipher _ (KeyIds ks)) = + "with gpg " ++ keys ks ++ " " ++ unwords ks + where + keys [_] = "key" + keys _ = "keys" + {- Stores an EncryptedCipher in a remote's configuration. -} storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig storeCipher c (EncryptedCipher t ks) = diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 11df2673cf..493ff12143 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -27,11 +27,12 @@ encryptionSetup c = (Just "none", Nothing) -> return c (Just "none", Just _) -> error "Cannot change encryption type of existing remote." (Nothing, Just _) -> return c - (Just _, Nothing) -> use $ genCipher c - (Just _, Just v) -> use $ updateCipher c v + (Just _, Nothing) -> use "encryption setup" $ genCipher c + (Just _, Just v) -> use "encryption updated" $ updateCipher c v where - use a = do + use m a = do cipher <- liftIO a + showNote $ m ++ " " ++ describeCipher cipher return $ M.delete "encryption" $ storeCipher c cipher {- Modifies a Remote to support encryption. diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index a0ce951a84..c842583546 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -15,7 +15,7 @@ like "2512E3C7" Next, create the S3 remote, and describe it. # git annex initremote cloud type=S3 encryption=2512E3C7 - initremote cloud (checking bucket) (creating bucket in US) ok + initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) ok # git annex describe cloud "at Amazon's US datacenter" describe cloud ok From 684ad747100ccf5023415ea5e6996bc0e0d97583 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Sun, 17 Apr 2011 23:46:38 +0000 Subject: [PATCH 1563/8313] Added a comment --- ..._1_bfba72a696789bf21b2435dea15f967a._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment b/doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment new file mode 100644 index 0000000000..0ab113211e --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-17T23:46:37Z" + content=""" +--to and --from seem to have different semantics than --source and --destination. Subtle, but still different. + +That being said, I am not sure --from and --to are needed at all. Calling the local repo . and all remotes by their name, they are arguably redundant and removing them would make the syntax a lot prettier; mv and cp don't need them, either. + +I am not sure changing syntax at this point is considered good style though personally, I wouldn't mind adapting and would actually prefer it over using --to and --from. + +-v and -q would be nice. + + +Richard +"""]] From 4cbd71b05771479061c3b1a029dc4aabe748d1fb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 13:40:02 -0400 Subject: [PATCH 1564/8313] enable gpg batch mode when GPG_AGENT_INFO is set --- Crypto.hs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 1617f5aadb..1f4493b948 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -38,8 +38,9 @@ import System.Posix.IO import System.Posix.Types import System.Posix.Process import Control.Concurrent -import Control.Exception +import Control.Exception (finally) import System.Exit +import System.Environment import Types import Key @@ -172,18 +173,26 @@ pass :: (Cipher -> L.ByteString -> (Handle -> IO a) -> IO a) -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a pass to c i a = to c i $ \h -> a =<< L.hGetContents h -gpgParams :: [CommandParam] -> [String] -gpgParams params = - -- avoid prompting, and be quiet, even about checking the trustdb - ["--quiet", "--trust-model", "always"] ++ - toCommand params +gpgParams :: [CommandParam] -> IO [String] +gpgParams params = do + -- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous + -- gpg output about password prompts. + e <- catch (getEnv "GPG_AGENT_INFO") (const $ return "") + let batch = if null e then [] else ["--batch"] + return $ batch ++ defaults ++ toCommand params + where + -- be quiet, even about checking the trustdb + defaults = ["--quiet", "--trust-model", "always"] gpgRead :: [CommandParam] -> IO String -gpgRead params = pOpen ReadFromPipe "gpg" (gpgParams params) hGetContentsStrict +gpgRead params = do + params' <- gpgParams params + pOpen ReadFromPipe "gpg" params' hGetContentsStrict gpgPipeStrict :: [CommandParam] -> String -> IO String gpgPipeStrict params input = do - (pid, fromh, toh) <- hPipeBoth "gpg" (gpgParams params) + params' <- gpgParams params + (pid, fromh, toh) <- hPipeBoth "gpg" params' _ <- forkIO $ finally (hPutStr toh input) (hClose toh) output <- hGetContentsStrict fromh forceSuccess pid @@ -202,8 +211,8 @@ gpgCipherHandle params c input a = do let Fd passphrasefd = frompipe let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd] - (pid, fromh, toh) <- hPipeBoth "gpg" $ - gpgParams $ passphrase ++ params + params' <- gpgParams $ passphrase ++ params + (pid, fromh, toh) <- hPipeBoth "gpg" params' _ <- forkProcess $ do L.hPut toh input hClose toh From 1687fecd33ff73a71b2084532e9731796758047a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 13:45:32 -0400 Subject: [PATCH 1565/8313] bug --- doc/bugs/encrypted_S3_stalls.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/encrypted_S3_stalls.mdwn diff --git a/doc/bugs/encrypted_S3_stalls.mdwn b/doc/bugs/encrypted_S3_stalls.mdwn new file mode 100644 index 0000000000..c4484b9c4f --- /dev/null +++ b/doc/bugs/encrypted_S3_stalls.mdwn @@ -0,0 +1,7 @@ +Sending large-ish (few megabytes) files to encrypted S3 remotes stalls out. +It works for the tiny files I was using to test while developing it, on +dialup. + +There was a similar issue with bup, which I fixed by forking a process +rather than using a thread to do some IO. Probably need the same here. +--[[Joey]] From a441e08da1e6305f36db782ec9eda44f213ffa29 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 14:45:19 -0400 Subject: [PATCH 1566/8313] Fix stalls in S3 when transferring encrypted data. Stalls were caused by code that did approximatly: content' <- liftIO $ withEncryptedContent cipher content return store content' The return evaluated without actually reading content from S3, and so the cleanup code began waiting on gpg to exit before gpg could send all its data. Fixing it involved moving the `store` type action into the IO monad: liftIO $ withEncryptedContent cipher content store Which was a bit of a pain to do, thank you type system, but avoids the problem as now the whole content is consumed, and stored, before cleanup. --- Crypto.hs | 5 +- Remote/S3real.hs | 76 +++++++++++++++++-------------- debian/changelog | 1 + doc/bugs/encrypted_S3_stalls.mdwn | 2 + 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 1f4493b948..41f6b999ba 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -199,7 +199,10 @@ gpgPipeStrict params input = do return output {- Runs gpg with a cipher and some parameters, feeding it an input, - - and passing a handle to its output to an action. -} + - and passing a handle to its output to an action. + - + - Note that to avoid deadlock with the cleanup stage, + - the action must fully consume gpg's input before returning. -} gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a gpgCipherHandle params c input a = do -- pipe the passphrase into gpg on a fd diff --git a/Remote/S3real.hs b/Remote/S3real.hs index b88b22037f..fe68a7f5b7 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -100,13 +100,13 @@ s3Setup u c = do loc <- liftIO $ getBucketLocation conn bucket case loc of Right _ -> return () - Left err@(NetworkError _) -> error $ prettyReqError err + Left err@(NetworkError _) -> s3Error err Left (AWSError _ _) -> do showNote $ "creating bucket in " ++ datacenter res <- liftIO $ createBucketIn conn bucket datacenter case res of Right _ -> return () - Left err -> error $ prettyReqError err + Left err -> s3Error err gitConfigSpecialRemote u fullconfig "s3" "true" return fullconfig @@ -141,33 +141,32 @@ checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do case res of Right _ -> return $ Right True Left (AWSError _ _) -> return $ Right False - Left e -> return $ Left (error $ prettyReqError e) + Left e -> return $ Left (s3Error e) where noconn = Left $ error "S3 not configured" store :: Remote Annex -> Key -> Annex Bool -store r k = storeHelper r k =<< lazyKeyContent k +store r k = s3Action r False $ \(conn, bucket) -> do + content <- lazyKeyContent k + res <- liftIO $ storeHelper (conn, bucket) r k content + s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool -storeEncrypted r (cipher, enck) k = do +storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do content <- lazyKeyContent k - content' <- liftIO $ withEncryptedContent cipher content return - storeHelper r enck content' + res <- liftIO $ withEncryptedContent cipher content $ \s -> do + storeHelper (conn, bucket) r enck s + s3Bool res lazyKeyContent :: Key -> Annex L.ByteString lazyKeyContent k = do g <- Annex.gitRepo liftIO $ L.readFile $ gitAnnexLocation g k -storeHelper :: Remote Annex -> Key -> L.ByteString -> Annex Bool -storeHelper r k content = s3Action r False $ \(conn, bucket) -> do +storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> L.ByteString -> IO (AWSResult ()) +storeHelper (conn, bucket) r k content = do let object = setStorageClass storageclass $ bucketKey bucket k content - res <- liftIO $ sendObject conn object - case res of - Right _ -> return True - Left e -> do - warning $ prettyReqError e - return False + sendObject conn object where storageclass = case fromJust $ M.lookup "storageclass" $ fromJust $ config r of @@ -175,30 +174,41 @@ storeHelper r k content = s3Action r False $ \(conn, bucket) -> do _ -> STANDARD retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool -retrieve = retrieveHelper (return . obj_data) - -retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool -retrieveEncrypted r (cipher, enck) f = retrieveHelper decrypt r enck f - where - decrypt o = withDecryptedContent cipher (obj_data o) return - -retrieveHelper :: (S3Object -> IO L.ByteString) -> Remote Annex -> Key -> FilePath -> Annex Bool -retrieveHelper a r k f = s3Action r False $ \(conn, bucket) -> do +retrieve r k f = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey bucket k L.empty case res of Right o -> do - content <- liftIO $ a o - liftIO $ L.writeFile f content + liftIO $ L.writeFile f $ obj_data o return True - Left e -> do - warning $ prettyReqError e - return False - + Left e -> s3Warning e + +retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do + res <- liftIO $ getObject conn $ bucketKey bucket enck L.empty + case res of + Right o -> liftIO $ + withDecryptedContent cipher (obj_data o) $ \content -> do + L.writeFile f content + return True + Left e -> s3Warning e + remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty case res of Right _ -> return True - Left e -> do - warning $ prettyReqError e - return False + Left e -> s3Warning e + +s3Warning :: ReqError -> Annex Bool +s3Warning e = do + warning $ prettyReqError e + return False + +s3Error :: ReqError -> a +s3Error e = error $ prettyReqError e + +s3Bool :: AWSResult () -> Annex Bool +s3Bool res = do + case res of + Right _ -> return True + Left e -> s3Warning e diff --git a/debian/changelog b/debian/changelog index aa00d3fca3..60ccace7af 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.20110418) UNRELEASED; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. * Add missing build dep on dataenc. + * Fix stalls in S3 when transferring encrypted data. -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 diff --git a/doc/bugs/encrypted_S3_stalls.mdwn b/doc/bugs/encrypted_S3_stalls.mdwn index c4484b9c4f..109e6e793a 100644 --- a/doc/bugs/encrypted_S3_stalls.mdwn +++ b/doc/bugs/encrypted_S3_stalls.mdwn @@ -5,3 +5,5 @@ dialup. There was a similar issue with bup, which I fixed by forking a process rather than using a thread to do some IO. Probably need the same here. --[[Joey]] + +[[done]] --[[Joey]] From b1274b637863ebb4e14d39ca2cf00a27c9d1f142 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 14:50:09 -0400 Subject: [PATCH 1567/8313] refactor --- Remote/S3real.hs | 98 ++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index fe68a7f5b7..5d8435932b 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -62,30 +62,6 @@ gen' r u c cst = config = c } -s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection -s3ConnectionRequired c = do - conn <- s3Connection c - case conn of - Nothing -> error "Cannot connect to S3" - Just conn' -> return conn' - -s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) -s3Connection c = do - ak <- getEnvKey "AWS_ACCESS_KEY_ID" - sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" - if (null ak || null sk) - then do - warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3" - return Nothing - else return $ Just $ AWSConnection host port ak sk - where - host = fromJust $ (M.lookup "host" c) - port = let s = fromJust $ (M.lookup "port" c) in - case reads s of - [(p, _)] -> p - _ -> error $ "bad S3 port value: " ++ s - getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") - s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane @@ -121,30 +97,6 @@ s3Setup u c = do , ("bucket", bucket) ] -s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a -s3Action r noconn action = do - when (config r == Nothing) $ - error $ "Missing configuration for special remote " ++ name r - let bucket = M.lookup "bucket" $ fromJust $ config r - conn <- s3Connection (fromJust $ config r) - case (bucket, conn) of - (Just b, Just c) -> action (c, b) - _ -> return noconn - -bucketKey :: String -> Key -> L.ByteString -> S3Object -bucketKey bucket k content = S3Object bucket (show k) "" [] content - -checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool) -checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do - showNote ("checking " ++ name r ++ "...") - res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty - case res of - Right _ -> return $ Right True - Left (AWSError _ _) -> return $ Right False - Left e -> return $ Left (s3Error e) - where - noconn = Left $ error "S3 not configured" - store :: Remote Annex -> Key -> Annex Bool store r k = s3Action r False $ \(conn, bucket) -> do content <- lazyKeyContent k @@ -195,9 +147,18 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty + s3Bool res + +checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool) +checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do + showNote ("checking " ++ name r ++ "...") + res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty case res of - Right _ -> return True - Left e -> s3Warning e + Right _ -> return $ Right True + Left (AWSError _ _) -> return $ Right False + Left e -> return $ Left (s3Error e) + where + noconn = Left $ error "S3 not configured" s3Warning :: ReqError -> Annex Bool s3Warning e = do @@ -212,3 +173,40 @@ s3Bool res = do case res of Right _ -> return True Left e -> s3Warning e + +s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection +s3ConnectionRequired c = do + conn <- s3Connection c + case conn of + Nothing -> error "Cannot connect to S3" + Just conn' -> return conn' + +s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) +s3Connection c = do + ak <- getEnvKey "AWS_ACCESS_KEY_ID" + sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" + if (null ak || null sk) + then do + warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3" + return Nothing + else return $ Just $ AWSConnection host port ak sk + where + host = fromJust $ (M.lookup "host" c) + port = let s = fromJust $ (M.lookup "port" c) in + case reads s of + [(p, _)] -> p + _ -> error $ "bad S3 port value: " ++ s + getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") + +s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a +s3Action r noconn action = do + when (config r == Nothing) $ + error $ "Missing configuration for special remote " ++ name r + let bucket = M.lookup "bucket" $ fromJust $ config r + conn <- s3Connection (fromJust $ config r) + case (bucket, conn) of + (Just b, Just c) -> action (c, b) + _ -> return noconn + +bucketKey :: String -> Key -> L.ByteString -> S3Object +bucketKey bucket k content = S3Object bucket (show k) "" [] content From 5985acdfad8a6791f0b2fc54a1e116cee9c12479 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 15:26:50 -0400 Subject: [PATCH 1568/8313] bup: Avoid memory leak when transferring encrypted data. This was a most surprising leak. It occurred in the process that is forked off to feed data to gpg. That process was passed a lazy ByteString of input, and ghc seemed to not GC the ByteString as it was lazily read and consumed, so memory slowly leaked as the file was read and passed through gpg to bup. To fix it, I simply changed the feeder to take an IO action that returns the lazy bytestring, and fed the result directly to hPut. AFAICS, this should change nothing WRT buffering. But somehow it makes ghc's GC do the right thing. Probably I triggered some weakness in ghc's GC (version 6.12.1). (Note that S3 still has this leak, and others too. Fixing it will involve another dance with the type system.) Update: One theory I have is that this has something to do with the forking of the feeder process. Perhaps, when the ByteString is produced before the fork, ghc decides it need to hold a pointer to the start of it, for some reason -- maybe it doesn't realize that it is only used in the forked process. --- Crypto.hs | 20 ++++++++++---------- Remote/Bup.hs | 6 ++---- Remote/Directory.hs | 6 ++---- Remote/S3real.hs | 4 ++-- debian/changelog | 3 ++- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 41f6b999ba..478d837615 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -153,24 +153,24 @@ encryptKey c k = {- Runs an action, passing it a handle from which it can - stream encrypted content. -} -withEncryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withEncryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"] {- Runs an action, passing it a handle from which it can - stream decrypted content. -} -withDecryptedHandle :: Cipher -> L.ByteString -> (Handle -> IO a) -> IO a +withDecryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a withDecryptedHandle = gpgCipherHandle [Param "--decrypt"] {- Streams encrypted content to an action. -} -withEncryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withEncryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a withEncryptedContent = pass withEncryptedHandle {- Streams decrypted content to an action. -} -withDecryptedContent :: Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +withDecryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a withDecryptedContent = pass withDecryptedHandle -pass :: (Cipher -> L.ByteString -> (Handle -> IO a) -> IO a) - -> Cipher -> L.ByteString -> (L.ByteString -> IO a) -> IO a +pass :: (Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a) + -> Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a pass to c i a = to c i $ \h -> a =<< L.hGetContents h gpgParams :: [CommandParam] -> IO [String] @@ -203,8 +203,8 @@ gpgPipeStrict params input = do - - Note that to avoid deadlock with the cleanup stage, - the action must fully consume gpg's input before returning. -} -gpgCipherHandle :: [CommandParam] -> Cipher -> L.ByteString -> (Handle -> IO a) -> IO a -gpgCipherHandle params c input a = do +gpgCipherHandle :: [CommandParam] -> Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a +gpgCipherHandle params c a b = do -- pipe the passphrase into gpg on a fd (frompipe, topipe) <- createPipe _ <- forkIO $ do @@ -217,11 +217,11 @@ gpgCipherHandle params c input a = do params' <- gpgParams $ passphrase ++ params (pid, fromh, toh) <- hPipeBoth "gpg" params' _ <- forkProcess $ do - L.hPut toh input + L.hPut toh =<< a hClose toh exitSuccess hClose toh - ret <- a fromh + ret <- b fromh -- cleanup forceSuccess pid diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 16e1bbdcb5..6ae002c3b9 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -126,8 +126,7 @@ storeEncrypted r buprepo (cipher, enck) k = do let src = gitAnnexLocation g k params <- bupSplitParams r buprepo enck (Param "-") liftIO $ catchBool $ do - content <- L.readFile src - withEncryptedHandle cipher content $ \h -> do + withEncryptedHandle cipher (L.readFile src) $ \h -> do pipeBup params (Just h) Nothing retrieve :: BupRepo -> Key -> FilePath -> Annex Bool @@ -142,8 +141,7 @@ retrieveEncrypted buprepo (cipher, enck) f = do let params = bupParams "join" buprepo [Param $ show enck] liftIO $ catchBool $ do (pid, h) <- hPipeFrom "bup" $ toCommand params - content <- L.hGetContents h - withDecryptedContent cipher content $ L.writeFile f + withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f forceSuccess pid return True diff --git a/Remote/Directory.hs b/Remote/Directory.hs index d9bee80c3f..c680d61212 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -92,8 +92,7 @@ storeEncrypted d (cipher, enck) k = do liftIO $ catchBool $ storeHelper dest $ encrypt src dest where encrypt src dest = do - content <- L.readFile src - withEncryptedContent cipher content $ L.writeFile dest + withEncryptedContent cipher (L.readFile src) $ L.writeFile dest return True storeHelper :: FilePath -> IO Bool -> IO Bool @@ -113,8 +112,7 @@ retrieve d k f = liftIO $ copyFile (dirKey d k) f retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted d (cipher, enck) f = liftIO $ catchBool $ do - content <- L.readFile (dirKey d enck) - withDecryptedContent cipher content $ L.writeFile f + withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f return True remove :: FilePath -> Key -> Annex Bool diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 5d8435932b..f40deaf17e 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -106,7 +106,7 @@ store r k = s3Action r False $ \(conn, bucket) -> do storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do content <- lazyKeyContent k - res <- liftIO $ withEncryptedContent cipher content $ \s -> do + res <- liftIO $ withEncryptedContent cipher (return content) $ \s -> do storeHelper (conn, bucket) r enck s s3Bool res @@ -139,7 +139,7 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey bucket enck L.empty case res of Right o -> liftIO $ - withDecryptedContent cipher (obj_data o) $ \content -> do + withDecryptedContent cipher (return $ obj_data o) $ \content -> do L.writeFile f content return True Left e -> s3Warning e diff --git a/debian/changelog b/debian/changelog index 60ccace7af..4e9ea441d1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,8 @@ git-annex (0.20110418) UNRELEASED; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. * Add missing build dep on dataenc. - * Fix stalls in S3 when transferring encrypted data. + * S3: Fix stalls when transferring encrypted data. + * bup: Avoid memory leak when transferring encrypted data. -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 From 030c7a056b0795adf037464d608da5170638f4ac Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 16:03:27 -0400 Subject: [PATCH 1569/8313] update --- doc/bugs/S3_memory_leaks.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/S3_memory_leaks.mdwn diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn new file mode 100644 index 0000000000..d7cae2933a --- /dev/null +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -0,0 +1,13 @@ +S3 has two memory leaks. + +One only occurs with encryption. It was was fixed for bup, but +not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. + +The other occurs independant of encryption use. Copying a 100 mb +file to S3 causes an immediate sharp memory spike to 119 mb. +Copying the file back from S3 causes a slow memory increase toward 119 mb. +It's likely that this memory is used by the hS3 library, if it does not +construct the message to Amazon lazily. (And it may not be possible to +construct it lazily, if it includes checksum headers..) I have +emailed the hS3 author about this. +--[[Joey]] From ef6d265af410634ce720a5e190d9014d1b55a538 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 19 Apr 2011 20:13:10 +0000 Subject: [PATCH 1570/8313] Added a comment --- ..._f6a637c78c989382e3c22d41b7fb4cc2._comment | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment b/doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment new file mode 100644 index 0000000000..0072ae1d71 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-19T20:13:10Z" + content=""" +Let's see.. + +* -v is already an alias for --verbose + +* I don't find --source and --destination as easy to type or as clear as --from or --to. + +* -F is fast, so it cannot be used for --force. And I have no desire to make it easy to mistype a short option and enable --force; it can lose data. + +@richard while it would be possible to support some syntax like \"git annex copy . remote\"; what is it supposed to do if there are local files named foo and bar, and a remotes named foo and bar? Does \"git annex copy foo bar\" copy file foo to remote bar, or file bar from remote foo? I chose to use --from/--to to specify remotes independant of files to avoid such +ambiguity, which plain old `cp` doesn't have since it's operating entirely on filesystem objects, not both filesystem objects and abstract remotes. + +Seems like nothing to do here. [[done]] --[[Joey]] +"""]] From 4837176897ae5ade15b23de4999c370d3ac2ef3e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 16:31:35 -0400 Subject: [PATCH 1571/8313] update on memory leak Finished applying to S3 the change that fixed the memory leak in bup, but it didn't seem to help S3.. with encryption it still grows to 2x file size. --- Remote/S3real.hs | 13 +++++-------- doc/bugs/S3_memory_leaks.mdwn | 1 + 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index f40deaf17e..5095b40392 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -99,22 +99,19 @@ s3Setup u c = do store :: Remote Annex -> Key -> Annex Bool store r k = s3Action r False $ \(conn, bucket) -> do - content <- lazyKeyContent k + g <- Annex.gitRepo + content <- liftIO $ L.readFile $ gitAnnexLocation g k res <- liftIO $ storeHelper (conn, bucket) r k content s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do - content <- lazyKeyContent k - res <- liftIO $ withEncryptedContent cipher (return content) $ \s -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + res <- liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> do storeHelper (conn, bucket) r enck s s3Bool res -lazyKeyContent :: Key -> Annex L.ByteString -lazyKeyContent k = do - g <- Annex.gitRepo - liftIO $ L.readFile $ gitAnnexLocation g k - storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> L.ByteString -> IO (AWSResult ()) storeHelper (conn, bucket) r k content = do let object = setStorageClass storageclass $ bucketKey bucket k content diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index d7cae2933a..f0522304c2 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -2,6 +2,7 @@ S3 has two memory leaks. One only occurs with encryption. It was was fixed for bup, but not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. +(The fix I used for bup doesn't seem to work with S3.) The other occurs independant of encryption use. Copying a 100 mb file to S3 causes an immediate sharp memory spike to 119 mb. From 936ad63cf47287549412fa0013bcc975c11026d7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 17:56:56 -0400 Subject: [PATCH 1572/8313] heard from hS3 author --- doc/bugs/S3_memory_leaks.mdwn | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index f0522304c2..4182601253 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -1,14 +1,21 @@ S3 has two memory leaks. +## with encryption + One only occurs with encryption. It was was fixed for bup, but not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. (The fix I used for bup doesn't seem to work with S3.) +## always + The other occurs independant of encryption use. Copying a 100 mb file to S3 causes an immediate sharp memory spike to 119 mb. Copying the file back from S3 causes a slow memory increase toward 119 mb. It's likely that this memory is used by the hS3 library, if it does not construct the message to Amazon lazily. (And it may not be possible to -construct it lazily, if it includes checksum headers..) I have -emailed the hS3 author about this. +construct it lazily, if it includes checksum headers..) + +I have emailed the hS3 author about this. He wrote back quickly, seems +only getting the size of the file is causing it to be buffered, and a quick +fix should be forthcoming. --[[Joey]] From 143fc7b6923c6b6b39a175332eebbaa7645970c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 21:40:21 -0400 Subject: [PATCH 1573/8313] finalize release --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 4e9ea441d1..91ab3a4024 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110418) UNRELEASED; urgency=low +git-annex (0.20110419) unstable; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when there is no agent. @@ -6,7 +6,7 @@ git-annex (0.20110418) UNRELEASED; urgency=low * S3: Fix stalls when transferring encrypted data. * bup: Avoid memory leak when transferring encrypted data. - -- Joey Hess Sun, 17 Apr 2011 14:29:49 -0400 + -- Joey Hess Tue, 19 Apr 2011 21:26:51 -0400 git-annex (0.20110417) unstable; urgency=low From 0c9896114e8ef125b04ff368efde82a88136bdcf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Apr 2011 22:26:40 -0400 Subject: [PATCH 1574/8313] add news item for git-annex 0.20110419 --- doc/news/version_0.20110320.mdwn | 9 --------- doc/news/version_0.20110419.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 doc/news/version_0.20110320.mdwn create mode 100644 doc/news/version_0.20110419.mdwn diff --git a/doc/news/version_0.20110320.mdwn b/doc/news/version_0.20110320.mdwn deleted file mode 100644 index 84475829c3..0000000000 --- a/doc/news/version_0.20110320.mdwn +++ /dev/null @@ -1,9 +0,0 @@ -git-annex 0.20110320 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Fix dropping of files using the URL backend. - * Fix support for remotes with '.' in their names. - * Add version command to show git-annex version as well as repository - version information. - * No longer auto-upgrade to repository format 2, to avoid accidental - upgrades, etc. Use git-annex upgrade when you're ready to run this - version."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110419.mdwn b/doc/news/version_0.20110419.mdwn new file mode 100644 index 0000000000..2506ad0a87 --- /dev/null +++ b/doc/news/version_0.20110419.mdwn @@ -0,0 +1,7 @@ +git-annex 0.20110419 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Don't run gpg in batch mode, so it can prompt for passphrase when + there is no agent. + * Add missing build dep on dataenc. + * S3: Fix stalls when transferring encrypted data. + * bup: Avoid memory leak when transferring encrypted data."""]] \ No newline at end of file From 130252d9382d1c52174ac2dc3ff72edad2d8befa Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 20 Apr 2011 21:30:13 +0000 Subject: [PATCH 1575/8313] Added a comment --- .../comment_3_bf1114533d2895804e531e76eb6b8095._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment b/doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment new file mode 100644 index 0000000000..9fcbae6d20 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 3" + date="2011-04-20T21:28:06Z" + content=""" +Good point. scp fixes this by using a colon, but as colons aren't needed in git-annex remotes' names... -- RichiH +"""]] From 8d6b4f6f7d898f76cb82ac2dfee8a2bb12ed04bf Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 20 Apr 2011 21:33:17 +0000 Subject: [PATCH 1576/8313] Added a comment --- .../comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment b/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment new file mode 100644 index 0000000000..0fa5cfac51 --- /dev/null +++ b/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 4" + date="2011-04-20T21:30:17Z" + content=""" +Good point. scp fixes this by using a colon, but as colons aren't needed in git-annex remotes' names... -- RichiH +"""]] From 156f6c7aa658d4ab3705bb1fc40d9eca659cfc03 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Wed, 20 Apr 2011 21:46:33 +0000 Subject: [PATCH 1577/8313] removed --- .../comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment diff --git a/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment b/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment deleted file mode 100644 index 0fa5cfac51..0000000000 --- a/doc/forum/wishlist:_command_options_changes/comment_4_71bd5838c11c2a06d21ad2afdac6aee2._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" - nickname="Richard" - subject="comment 4" - date="2011-04-20T21:30:17Z" - content=""" -Good point. scp fixes this by using a colon, but as colons aren't needed in git-annex remotes' names... -- RichiH -"""]] From e3651b1285311f547e167531ec2a14108f3d491c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 01:51:15 -0400 Subject: [PATCH 1578/8313] update --- doc/bugs/S3_memory_leaks.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index 4182601253..86e3dd1a42 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -17,5 +17,7 @@ construct it lazily, if it includes checksum headers..) I have emailed the hS3 author about this. He wrote back quickly, seems only getting the size of the file is causing it to be buffered, and a quick -fix should be forthcoming. +fix should be forthcoming. Update: 0.5.6 has been released which will +allow providing file size out of band to avoid buffering when uploading. +Downloading will take further work in hS3. --[[Joey]] From 43639f69f665fca39df675668f1e1f3138d310ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 02:07:40 -0400 Subject: [PATCH 1579/8313] ghc7 * Update Debian build dependencies for ghc 7. * Debian package is now built with S3 support. Thanks Joachim Breitner for making this possible, also thanks Greg Heartsfield for working to improve the hS3 library for git-annex. Also hid a conflicting new symbol from Control.Monad.State --- Annex.hs | 2 +- Backend.hs | 2 +- debian/changelog | 9 +++++++++ debian/control | 11 ++++++----- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Annex.hs b/Annex.hs index 9086db9bf7..03f9130b6a 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,7 +16,7 @@ module Annex ( gitRepo ) where -import Control.Monad.State +import Control.Monad.State hiding (state) import qualified GitRepo as Git import qualified GitQueue diff --git a/Backend.hs b/Backend.hs index c0e93acc23..0edff2ab5e 100644 --- a/Backend.hs +++ b/Backend.hs @@ -32,7 +32,7 @@ module Backend ( maybeLookupBackendName ) where -import Control.Monad.State +import Control.Monad.State hiding (state) import System.IO.Error (try) import System.FilePath import System.Posix.Files diff --git a/debian/changelog b/debian/changelog index 91ab3a4024..cdb530c247 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +git-annex (0.20110420) UNRELEASED; urgency=low + + * Update Debian build dependencies for ghc 7. + * Debian package is now built with S3 support. Thanks Joachim Breitner for + making this possible, also thanks Greg Heartsfield for working to improve + the hS3 library for git-annex. + + -- Joey Hess Thu, 21 Apr 2011 02:00:00 -0400 + git-annex (0.20110419) unstable; urgency=low * Don't run gpg in batch mode, so it can prompt for passphrase when diff --git a/debian/control b/debian/control index 00740b7403..7aba047173 100644 --- a/debian/control +++ b/debian/control @@ -4,11 +4,12 @@ Priority: optional Build-Depends: debhelper (>= 7.0.50), ghc6, - libghc6-missingh-dev, - libghc6-pcre-light-dev, - libghc6-testpack-dev, - libghc6-sha-dev, - libghc6-dataenc-dev, + libghc-missingh-dev, + libghc-pcre-light-dev, + libghc-testpack-dev, + libghc-sha-dev, + libghc-dataenc-dev, + libghc-hs3-dev, ikiwiki, perlmagick, git | git-core, From d8329731c617b278967304389c300f2c832db28d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 09:58:32 -0400 Subject: [PATCH 1580/8313] missing build dep --- debian/control | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 7aba047173..444d712611 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,8 @@ Build-Depends: libghc-testpack-dev, libghc-sha-dev, libghc-dataenc-dev, - libghc-hs3-dev, + libghc-utf8-string-dev, + libghc-hs3-dev (>= 0.5.6), ikiwiki, perlmagick, git | git-core, From 6fcd3e1ef77d3dc99da30cdf2b82489d4bd3d7df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 10:31:54 -0400 Subject: [PATCH 1581/8313] fix S3 upload buffering problem Provide file size to new version of hS3. --- Remote/S3real.hs | 45 ++++++++++++++++++++++++----------- debian/changelog | 9 ++++--- debian/control | 1 - doc/bugs/S3_memory_leaks.mdwn | 18 +++++--------- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 5095b40392..2b0234dc27 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -10,7 +10,7 @@ module Remote.S3 (remote) where import Control.Exception.Extensible (IOException) import Network.AWS.AWSConnection import Network.AWS.S3Object -import Network.AWS.S3Bucket +import Network.AWS.S3Bucket hiding (size) import Network.AWS.AWSResult import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M @@ -18,6 +18,8 @@ import Data.Maybe import Control.Monad (when) import Control.Monad.State (liftIO) import System.Environment +import System.Posix.Files +import System.Directory import RemoteClass import Types @@ -30,6 +32,7 @@ import Config import Remote.Special import Remote.Encryptable import Crypto +import Key remote :: RemoteType Annex remote = RemoteType { @@ -100,21 +103,35 @@ s3Setup u c = do store :: Remote Annex -> Key -> Annex Bool store r k = s3Action r False $ \(conn, bucket) -> do g <- Annex.gitRepo - content <- liftIO $ L.readFile $ gitAnnexLocation g k - res <- liftIO $ storeHelper (conn, bucket) r k content + res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do g <- Annex.gitRepo let f = gitAnnexLocation g k - res <- liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> do - storeHelper (conn, bucket) r enck s + -- To get file size of the encrypted content, have to use a temp file. + -- (An alternative would be chunking to to a constant size.) + let tmp = gitAnnexTmpLocation g enck + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + res <- liftIO $ storeHelper (conn, bucket) r enck tmp + tmp_exists <- liftIO $ doesFileExist tmp + when tmp_exists $ liftIO $ removeFile tmp s3Bool res -storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> L.ByteString -> IO (AWSResult ()) -storeHelper (conn, bucket) r k content = do - let object = setStorageClass storageclass $ bucketKey bucket k content +storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> FilePath -> IO (AWSResult ()) +storeHelper (conn, bucket) r k file = do + content <- liftIO $ L.readFile file + -- size is provided to S3 so the whole content does not need to be + -- buffered to calculate it + size <- case keySize k of + Just s -> return $ fromIntegral s + Nothing -> do + s <- liftIO $ getFileStatus file + return $ fileSize s + let object = setStorageClass storageclass $ + S3Object bucket (show k) "" + [("Content-Length",(show size))] content sendObject conn object where storageclass = @@ -124,7 +141,7 @@ storeHelper (conn, bucket) r k content = do retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool retrieve r k f = s3Action r False $ \(conn, bucket) -> do - res <- liftIO $ getObject conn $ bucketKey bucket k L.empty + res <- liftIO $ getObject conn $ bucketKey bucket k case res of Right o -> do liftIO $ L.writeFile f $ obj_data o @@ -133,7 +150,7 @@ retrieve r k f = s3Action r False $ \(conn, bucket) -> do retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do - res <- liftIO $ getObject conn $ bucketKey bucket enck L.empty + res <- liftIO $ getObject conn $ bucketKey bucket enck case res of Right o -> liftIO $ withDecryptedContent cipher (return $ obj_data o) $ \content -> do @@ -143,13 +160,13 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do remove :: Remote Annex -> Key -> Annex Bool remove r k = s3Action r False $ \(conn, bucket) -> do - res <- liftIO $ deleteObject conn $ bucketKey bucket k L.empty + res <- liftIO $ deleteObject conn $ bucketKey bucket k s3Bool res checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool) checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do showNote ("checking " ++ name r ++ "...") - res <- liftIO $ getObjectInfo conn $ bucketKey bucket k L.empty + res <- liftIO $ getObjectInfo conn $ bucketKey bucket k case res of Right _ -> return $ Right True Left (AWSError _ _) -> return $ Right False @@ -205,5 +222,5 @@ s3Action r noconn action = do (Just b, Just c) -> action (c, b) _ -> return noconn -bucketKey :: String -> Key -> L.ByteString -> S3Object -bucketKey bucket k content = S3Object bucket (show k) "" [] content +bucketKey :: String -> Key -> S3Object +bucketKey bucket k = S3Object bucket (show k) "" [] L.empty diff --git a/debian/changelog b/debian/changelog index cdb530c247..30a3e7cc5a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,12 @@ git-annex (0.20110420) UNRELEASED; urgency=low * Update Debian build dependencies for ghc 7. - * Debian package is now built with S3 support. Thanks Joachim Breitner for - making this possible, also thanks Greg Heartsfield for working to improve - the hS3 library for git-annex. + * Debian package is now built with S3 support. + Thanks Joachim Breitner for making this possible. + * No longer needs to buffer entire files when sending them to S3. + (However, getting files from S3 still requires buffering.) + Thanks Greg Heartsfield for ongoing work to improve the hS3 library + for git-annex. -- Joey Hess Thu, 21 Apr 2011 02:00:00 -0400 diff --git a/debian/control b/debian/control index 444d712611..be2e359110 100644 --- a/debian/control +++ b/debian/control @@ -43,4 +43,3 @@ Description: manage files with git, without checking their contents into git versioned files, which is convenient for maintaining documents, Makefiles, etc that are associated with annexed files but that benefit from full revision control. - diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index 86e3dd1a42..bfb3e1ec58 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -1,4 +1,4 @@ -S3 has two memory leaks. +S3 has memory leaks ## with encryption @@ -8,16 +8,10 @@ not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. ## always -The other occurs independant of encryption use. Copying a 100 mb -file to S3 causes an immediate sharp memory spike to 119 mb. Copying the file back from S3 causes a slow memory increase toward 119 mb. -It's likely that this memory is used by the hS3 library, if it does not -construct the message to Amazon lazily. (And it may not be possible to -construct it lazily, if it includes checksum headers..) -I have emailed the hS3 author about this. He wrote back quickly, seems -only getting the size of the file is causing it to be buffered, and a quick -fix should be forthcoming. Update: 0.5.6 has been released which will -allow providing file size out of band to avoid buffering when uploading. -Downloading will take further work in hS3. ---[[Joey]] +The author of hS3 is aware of the problem, and working on it. + +## fixed + +memory leak while uploading content to S3 From 45bdb2d4136f2faf61a3fde63477ad5d935583b1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 10:53:29 -0400 Subject: [PATCH 1582/8313] ensure tmp dir exists --- Remote/S3real.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 2b0234dc27..07e33368ea 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -33,6 +33,7 @@ import Remote.Special import Remote.Encryptable import Crypto import Key +import Utility remote :: RemoteType Annex remote = RemoteType { @@ -113,6 +114,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do -- To get file size of the encrypted content, have to use a temp file. -- (An alternative would be chunking to to a constant size.) let tmp = gitAnnexTmpLocation g enck + liftIO $ createDirectoryIfMissing True (parentDir tmp) liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s res <- liftIO $ storeHelper (conn, bucket) r enck tmp tmp_exists <- liftIO $ doesFileExist tmp From 2467c567713321b061c3daf92df39d4e35226c7b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 11:06:29 -0400 Subject: [PATCH 1583/8313] update on S3 memory leaks The remaining leaks are in hS3. The leak with encryption was worked around by the use of the temp file. (And was probably originally caused by gpgCipherHandle sparking a thread which kept a reference to the start of the byte string.) --- debian/changelog | 3 +-- doc/bugs/S3_memory_leaks.mdwn | 15 +++------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/debian/changelog b/debian/changelog index 30a3e7cc5a..7b0bdd103b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,7 @@ git-annex (0.20110420) UNRELEASED; urgency=low * Update Debian build dependencies for ghc 7. * Debian package is now built with S3 support. Thanks Joachim Breitner for making this possible. - * No longer needs to buffer entire files when sending them to S3. - (However, getting files from S3 still requires buffering.) + * Somewhat improved memory usage of S3, still work to do. Thanks Greg Heartsfield for ongoing work to improve the hS3 library for git-annex. diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index bfb3e1ec58..d8b10b075c 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -1,17 +1,8 @@ S3 has memory leaks -## with encryption +Sending a file to S3 causes a slow memory increase toward the file size. -One only occurs with encryption. It was was fixed for bup, but -not yet for S3, in 5985acdfad8a6791f0b2fc54a1e116cee9c12479. -(The fix I used for bup doesn't seem to work with S3.) - -## always - -Copying the file back from S3 causes a slow memory increase toward 119 mb. +Copying the file back from S3 causes a slow memory increase toward the +file size. The author of hS3 is aware of the problem, and working on it. - -## fixed - -memory leak while uploading content to S3 From 82347fc5ab16823a2ad956e4cf44daaca710e48f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 11:23:44 -0400 Subject: [PATCH 1584/8313] seems -rtsopts is needed now for profiling --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13adfe0a2d..a8eccc5bfd 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX=/usr IGNORE=-ignore-package monads-fd GHCFLAGS=-O2 -Wall $(IGNORE) ifdef PROFILE -GHCFLAGS=-prof -auto-all -caf-all -fforce-recomp $(IGNORE) +GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE) endif GHCMAKE=ghc $(GHCFLAGS) --make From dda812583f214dd9835c826b7b9bcc2e8685333e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 12:13:17 -0400 Subject: [PATCH 1585/8313] root caused --- doc/bugs/S3_memory_leaks.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/S3_memory_leaks.mdwn b/doc/bugs/S3_memory_leaks.mdwn index d8b10b075c..f612de3960 100644 --- a/doc/bugs/S3_memory_leaks.mdwn +++ b/doc/bugs/S3_memory_leaks.mdwn @@ -5,4 +5,6 @@ Sending a file to S3 causes a slow memory increase toward the file size. Copying the file back from S3 causes a slow memory increase toward the file size. -The author of hS3 is aware of the problem, and working on it. +The author of hS3 is aware of the problem, and working on it. I think I +have identified the root cause of the buffering; it's done by hS3 so it can +resend the data if S3 sends it a 307 redirect. --[[Joey]] From 6668a061a8bab5360a5af4e61ead823f9af93369 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 14:53:07 -0400 Subject: [PATCH 1586/8313] typo --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index be2e359110..7686389972 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: utils Priority: optional Build-Depends: debhelper (>= 7.0.50), - ghc6, + ghc, libghc-missingh-dev, libghc-pcre-light-dev, libghc-testpack-dev, From 24feee25c9ea92ef90e9ea44f50ec26e321a23a4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 15:11:51 -0400 Subject: [PATCH 1587/8313] releasing version 0.20110420 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7b0bdd103b..26e9e81721 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110420) UNRELEASED; urgency=low +git-annex (0.20110420) unstable; urgency=low * Update Debian build dependencies for ghc 7. * Debian package is now built with S3 support. @@ -7,7 +7,7 @@ git-annex (0.20110420) UNRELEASED; urgency=low Thanks Greg Heartsfield for ongoing work to improve the hS3 library for git-annex. - -- Joey Hess Thu, 21 Apr 2011 02:00:00 -0400 + -- Joey Hess Thu, 21 Apr 2011 15:00:48 -0400 git-annex (0.20110419) unstable; urgency=low From 66d951c3fd1a2aa19543d4148be8de734f54fd5c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 15:12:22 -0400 Subject: [PATCH 1588/8313] add news item for git-annex 0.20110420 --- doc/news/version_0.20110325.mdwn | 20 -------------------- doc/news/version_0.20110420.mdwn | 8 ++++++++ 2 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 doc/news/version_0.20110325.mdwn create mode 100644 doc/news/version_0.20110420.mdwn diff --git a/doc/news/version_0.20110325.mdwn b/doc/news/version_0.20110325.mdwn deleted file mode 100644 index d5a59afe65..0000000000 --- a/doc/news/version_0.20110325.mdwn +++ /dev/null @@ -1,20 +0,0 @@ -git-annex 0.20110325 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Free space checking is now done, for transfers of data for keys - that have free space metadata. (Notably, not for SHA* keys generated - with git-annex 0.2x or earlier.) The code is believed to work on - Linux, FreeBSD, and OSX; check compile-time messages to see if it - is not enabled for your OS. - * Add annex.diskreserve config setting, to control how much free space - to reserve for other purposes and avoid using (defaults to 1 mb). - * Add --fast flag, that can enable less expensive, but also less thorough - versions of some commands. - * fsck: In fast mode, avoid checking checksums. - * unused: In fast mode, just show all existing temp files as unused, - and avoid expensive scan for other unused content. - * migrate: Support migrating v1 SHA keys to v2 SHA keys with - size information that can be used for free space checking. - * Fix space leak in fsck and drop commands. - * migrate: Bugfix for case when migrating a file results in a key that - is already present in .git/annex/objects. - * dropunused: Significantly sped up; only read unused log file once."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110420.mdwn b/doc/news/version_0.20110420.mdwn new file mode 100644 index 0000000000..bb7fee219b --- /dev/null +++ b/doc/news/version_0.20110420.mdwn @@ -0,0 +1,8 @@ +git-annex 0.20110420 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Update Debian build dependencies for ghc 7. + * Debian package is now built with S3 support. + Thanks Joachim Breitner for making this possible. + * Somewhat improved memory usage of S3, still work to do. + Thanks Greg Heartsfield for ongoing work to improve the hS3 library + for git-annex."""]] \ No newline at end of file From 892593c5efacbc084d19af4b5d7164ededaea7ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 16:37:14 -0400 Subject: [PATCH 1589/8313] Use haskell Crypto library instead of haskell SHA library.a Since hS3 needs Crypto anyway, this actually reduces dependencies. --- Crypto.hs | 24 ++++++++++++++++-------- debian/changelog | 6 ++++++ debian/control | 2 +- doc/install.mdwn | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 478d837615..ef7b49d8ff 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -27,20 +27,21 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import qualified Codec.Binary.Base64 as B64 -import Data.ByteString.Lazy.UTF8 (fromString) -import Data.Digest.Pure.SHA -import System.Cmd.Utils import Data.String.Utils import Data.List import Data.Bits.Utils +import Data.HMAC +import Data.Array +import Codec.Utils +import System.Cmd.Utils import System.IO import System.Posix.IO import System.Posix.Types import System.Posix.Process -import Control.Concurrent -import Control.Exception (finally) import System.Exit import System.Environment +import Control.Concurrent +import Control.Exception (finally) import Types import Key @@ -143,9 +144,9 @@ decryptCipher _ (EncryptedCipher encipher _) = encryptKey :: Cipher -> Key -> IO Key encryptKey c k = return Key { - keyName = showDigest $ hmacSha1 - (fromString $ cipherHmac c) - (fromString $ show k), + keyName = showOctets $ hmac_sha1 + (s2w8 $ cipherHmac c) + (s2w8 $ show k), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data @@ -252,3 +253,10 @@ fromB64 s = case B64.decode s of Nothing -> error "bad base64 encoded data" Just ws -> w82s ws + +showOctets :: [Octet] -> String +showOctets = concat . map hexChars + where + hexChars c = [arr ! (c `div` 16), arr ! (c `mod` 16)] + arr = listArray (0, 15) "0123456789abcdef" + diff --git a/debian/changelog b/debian/changelog index 26e9e81721..5955f29589 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110421) UNRELEASED; urgency=low + + * Use haskell Crypto library instead of haskell SHA library. + + -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 + git-annex (0.20110420) unstable; urgency=low * Update Debian build dependencies for ghc 7. diff --git a/debian/control b/debian/control index 7686389972..97b04ea5b6 100644 --- a/debian/control +++ b/debian/control @@ -7,8 +7,8 @@ Build-Depends: libghc-missingh-dev, libghc-pcre-light-dev, libghc-testpack-dev, - libghc-sha-dev, libghc-dataenc-dev, + libghc-crypto-dev, libghc-utf8-string-dev, libghc-hs3-dev (>= 0.5.6), ikiwiki, diff --git a/doc/install.mdwn b/doc/install.mdwn index 3d15eac604..a6b86891d5 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,7 +13,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: -* SHA: +* crypto: * dataenc: * TestPack * QuickCheck 2 From b72de39ba469fb7f39be0728a10fe949619c7be0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Apr 2011 16:56:24 -0400 Subject: [PATCH 1590/8313] add test to ensure hmac remains stable --- Crypto.hs | 16 +++++++++++++--- test.hs | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index ef7b49d8ff..4d6e38bac8 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -22,6 +22,8 @@ module Crypto ( withDecryptedHandle, withEncryptedContent, withDecryptedContent, + + prop_hmacWithCipher_sane ) where import qualified Data.ByteString.Lazy.Char8 as L @@ -144,9 +146,7 @@ decryptCipher _ (EncryptedCipher encipher _) = encryptKey :: Cipher -> Key -> IO Key encryptKey c k = return Key { - keyName = showOctets $ hmac_sha1 - (s2w8 $ cipherHmac c) - (s2w8 $ show k), + keyName = hmacWithCipher c (show k), keyBackendName = "GPGHMACSHA1", keySize = Nothing, -- size and mtime omitted keyMtime = Nothing -- to avoid leaking data @@ -259,4 +259,14 @@ showOctets = concat . map hexChars where hexChars c = [arr ! (c `div` 16), arr ! (c `mod` 16)] arr = listArray (0, 15) "0123456789abcdef" + +hmacWithCipher :: Cipher -> String -> String +hmacWithCipher c = hmacWithCipher' (cipherHmac c) +hmacWithCipher' :: String -> String -> String +hmacWithCipher' c s = showOctets $ hmac_sha1 (s2w8 c) (s2w8 s) +{- Ensure that hmacWithCipher' returns the same thing forevermore. -} +prop_hmacWithCipher_sane :: Bool +prop_hmacWithCipher_sane = known_good == hmacWithCipher' "foo" "bar" + where + known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51" diff --git a/test.hs b/test.hs index cdec4ea614..9304eee83f 100644 --- a/test.hs +++ b/test.hs @@ -40,6 +40,7 @@ import qualified Content import qualified Command.DropUnused import qualified Key import qualified Config +import qualified Crypto main :: IO () main = do @@ -63,6 +64,7 @@ quickcheck = TestLabel "quickcheck" $ TestList , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics , qctest "prop_cost_sane" Config.prop_cost_sane + , qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane ] blackbox :: Test From ca3f05fd6cf0f12dc29193a5c5b3fc01c31097dc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Fri, 22 Apr 2011 14:24:17 +0000 Subject: [PATCH 1591/8313] --- doc/forum/wishlist:_git-annex_replicate.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate.mdwn diff --git a/doc/forum/wishlist:_git-annex_replicate.mdwn b/doc/forum/wishlist:_git-annex_replicate.mdwn new file mode 100644 index 0000000000..0d926b3375 --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate.mdwn @@ -0,0 +1,12 @@ +I'd like to be able to do something like the following: + + * Create encrypted git-annex remotes on a couple of semi-trusted machines - ones that have good connectivity, but non-redundant hardware + * set numcopies=3 + * run `git-annex replicate` and have git-annex run the appropriate copy commands to make sure every file is on at least 3 machines + +There would also likely be a `git annex rebalance` command which could be used if remotes were added or removed. If possible, it should copy files between servers directly, rather than proxy through a potentially slow client. + +There might be the need to have a 'replication_priority' option for each remote that configures which machines would be preferred. That way you could set your local server to a high priority to ensure that it is always 1 of the 3 machines used and files are distributed across 2 of the remaining remotes. Other than priority, other options that might help: + + * maxspace - A self imposed quota per remote machine. git-annex replicate should try to replicate files first to machines with more free space. maxspace would change the free space calculation to be `min(actual_free_space, maxspace - space_used_by_git_annex) + * bandwidth - when replication files, copies should be done between machines with the highest available bandwidth. ( I think this option could be useful for git-annex get in general) From 028b338c29fa26a3a821fd79172bc8a42a23387a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 22 Apr 2011 18:27:01 +0000 Subject: [PATCH 1592/8313] Added a comment --- ...comment_1_9926132ec6052760cdf28518a24e2358._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment b/doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment new file mode 100644 index 0000000000..cec971ee3b --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-04-22T18:27:00Z" + content=""" +While having remotes redistribute introduces some obvious security concerns, I might use it. + +As remotes support a cost factor already, you can basically implement bandwidth through that. +"""]] From a03dc49bb21190bd2823df9d6b99b04158a955ee Mon Sep 17 00:00:00 2001 From: gernot Date: Sat, 23 Apr 2011 16:02:42 +0000 Subject: [PATCH 1593/8313] --- ...efine_remotes_that_must_have_all_files.mdwn | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn diff --git a/doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn b/doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn new file mode 100644 index 0000000000..156cfb0090 --- /dev/null +++ b/doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn @@ -0,0 +1,18 @@ +I would like to be able to name a few remotes that must retain *all* annexed +files. `git-annex fsck` should warn me if any files are missing from those +remotes, even if `annex.numcopies` has been satisfied by other remotes. + +I imagine this could also be useful for bup remotes, but I haven't actually +looked at those yet. + +Based on existing output, this is what a warning message could look like: + + fsck FILE + 3 of 3 trustworthy copies of FILE exist. + FILE is, however, still missing from these required remotes: + UUID -- Backup Drive 1 + UUID -- Backup Drive 2 + Back it up with git-annex copy. + Warning + +What do you think? From 65adb9240f45c265dc19d28976823186a8c4b7b0 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 23 Apr 2011 16:22:07 +0000 Subject: [PATCH 1594/8313] Added a comment --- ...mment_2_c43932f4194aba8fb2470b18e0817599._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment b/doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment new file mode 100644 index 0000000000..9d50d15310 --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-23T16:22:07Z" + content=""" +Besides the cost values, annex.diskreserve was recently added. (But is not available for special remotes.) + +I have held off on adding high-level management stuff like this to git-annex, as it's hard to make it generic enough to cover use cases. + +A low-level way to accomplish this would be to have a way for `git annex get` and/or `copy` to skip files when `numcopies` is already satisfied. Then cron jobs could be used. +"""]] From 96a7b7926ed09aa264207907ea4e0a5e31a031cb Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Sat, 23 Apr 2011 16:27:13 +0000 Subject: [PATCH 1595/8313] Added a comment --- ...comment_1_cceccc1a1730ac688d712b81a44e31c3._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment diff --git a/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment new file mode 100644 index 0000000000..1f65fd982f --- /dev/null +++ b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-23T16:27:13Z" + content=""" +Seems to have a scalability problem, what happens when such a repository becomes full? + +Another way to accomplish I think the same thing is to pick the repositories that you would include in such a set, and make all other repositories untrusted. And set numcopies as desired. Then git-annex will never remove files from the set of non-untrusted repositories, and fsck will warn if a file is present on only an untrusted repository. +"""]] From aa820623dc9ea648fb9fa8e9263557529155a7a9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Sat, 23 Apr 2011 17:54:43 +0000 Subject: [PATCH 1596/8313] Added a comment --- ...comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment b/doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment new file mode 100644 index 0000000000..e7eb06b3b1 --- /dev/null +++ b/doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 3" + date="2011-04-23T17:54:42Z" + content=""" +Hmm, so it seems there is almost a way to do this already. + +I think the one thing that isn't currently possible is to have 'plain' ssh remotes.. basically something just like the directory remote, but able to take a ssh user@host/path url. something like sshfs could be used to fake this, but for things like fsck you would want to do the sha1 calculations on the remote host. +"""]] From 9715f3132c5fa69e8edf2bc7c41c1a4e9c0602be Mon Sep 17 00:00:00 2001 From: gernot Date: Sun, 24 Apr 2011 11:20:06 +0000 Subject: [PATCH 1597/8313] Added a comment --- ...t_2_eec848fcf3979c03cbff2b7407c75a7a._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment diff --git a/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment new file mode 100644 index 0000000000..1855cdda01 --- /dev/null +++ b/doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="gernot" + ip="87.79.209.169" + subject="comment 2" + date="2011-04-24T11:20:05Z" + content=""" +Right, I have thought about untrusting all but a few remotes to achieve +something similar before and I'm sure it would kind of work. It would be more +of an ugly workaround, however, because I would have to untrust remotes that +are, in reality, at least semi-trusted. That's why an extra option/attribute +for that kind of purpose/remote would be nice. + +Obviously I didn't see the scalability problem though. Good Point. Maybe I can +achieve the same thing by writing a log parsing script for myself? + +"""]] From 8512a4a1a1f5367249cdb12aab75ed5d1bb42c8a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 12:43:22 -0400 Subject: [PATCH 1598/8313] Remove testpack from build depends, as it is not available on all architectures. The test suite will not be run if it cannot be compiled. It may be possible later to split off the quickcheck using tests into a separate program and keep most of the tests using just hunit. --- Makefile | 7 +++++-- debian/changelog | 3 +++ debian/control | 1 - 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a8eccc5bfd..69fb518074 100644 --- a/Makefile +++ b/Makefile @@ -45,8 +45,11 @@ install: all fi test: $(bins) - $(GHCMAKE) test - ./test + if ! $(GHCMAKE) test; then \ + echo "** not running test suite" >&2; \ + else \ + ./test; \ + fi testcoverage: $(bins) rm -f test.tix test diff --git a/debian/changelog b/debian/changelog index 5955f29589..872277d0f5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,9 @@ git-annex (0.20110421) UNRELEASED; urgency=low * Use haskell Crypto library instead of haskell SHA library. + * Remove testpack from build depends, as it is not available + on all architectures. The test suite will not be run if it + cannot be compiled. -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 diff --git a/debian/control b/debian/control index 97b04ea5b6..f9e6196aba 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,6 @@ Build-Depends: ghc, libghc-missingh-dev, libghc-pcre-light-dev, - libghc-testpack-dev, libghc-dataenc-dev, libghc-crypto-dev, libghc-utf8-string-dev, From b0b413c69f76bcfa46d01ff1623027707483c63c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 13:02:54 -0400 Subject: [PATCH 1599/8313] fix relative Not currently used, but was buggy. --- GitRepo.hs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 543ad801a4..3e177cf1be 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -59,7 +59,7 @@ module GitRepo ( prop_idempotent_deencode ) where -import Control.Monad (unless) +import Control.Monad (unless, when) import System.Directory import System.FilePath import System.Posix.Directory @@ -242,7 +242,11 @@ workTree Repo { location = Unknown } = undefined relative :: Repo -> FilePath -> IO FilePath relative repo@(Repo { location = Dir d }) file = do cwd <- getCurrentDirectory - return $ drop (length absrepo) (absfile cwd) + let file' = absfile cwd + let len = length absrepo + when (take len file' /= absrepo) $ + error $ file ++ " is not located inside git repository " ++ absrepo + return $ drop (length absrepo) file' where -- normalize both repo and file, so that repo -- will be substring of file @@ -251,7 +255,7 @@ relative repo@(Repo { location = Dir d }) file = do Nothing -> error $ "bad repo" ++ repoDescribe repo absfile c = case (secureAbsNormPath c file) of Just f -> f - Nothing -> error $ file ++ " is not located inside git repository " ++ absrepo + Nothing -> file relative repo _ = assertLocal repo $ error "internal" {- Path of an URL repo. -} From e433c6f0bb2ee5f03217b85e3b677b961f5d391a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 13:36:39 -0400 Subject: [PATCH 1600/8313] generalized relPathDirTo functions --- Content.hs | 2 +- Utility.hs | 37 ++++++++++++++++--------------------- test.hs | 2 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Content.hs b/Content.hs index f63c02311f..576eecb319 100644 --- a/Content.hs +++ b/Content.hs @@ -58,7 +58,7 @@ calcGitLink file key = do let absfile = case absNormPath cwd file of Just f -> f Nothing -> error $ "unable to normalize " ++ file - return $ relPathDirToDir (parentDir absfile) + return $ relPathDirToFile (parentDir absfile) (Git.workTree g) ".git" annexLocation key {- Updates the LocationLog when a key's presence changes. diff --git a/Utility.hs b/Utility.hs index 5639a8799a..13ebbfccba 100644 --- a/Utility.hs +++ b/Utility.hs @@ -13,8 +13,8 @@ module Utility ( parentDir, absPath, absPathFrom, - relPathCwdToDir, - relPathDirToDir, + relPathCwdToFile, + relPathDirToFile, boolSystem, shellEscape, shellUnEscape, @@ -29,7 +29,7 @@ module Utility ( prop_idempotent_shellEscape, prop_idempotent_shellEscape_multiword, prop_parentDir_basics, - prop_relPathDirToDir_basics + prop_relPathDirToFile_basics ) where import System.IO @@ -180,26 +180,21 @@ absPathFrom cwd file = Just f -> f Nothing -> error $ "unable to normalize " ++ file -{- Constructs a relative path from the CWD to a directory. +{- Constructs a relative path from the CWD to a file. - - For example, assuming CWD is /tmp/foo/bar: - - relPathCwdToDir "/tmp/foo" == "../" - - relPathCwdToDir "/tmp/foo/bar" == "" + - relPathCwdToFile "/tmp/foo" == ".." + - relPathCwdToFile "/tmp/foo/bar" == "" -} -relPathCwdToDir :: FilePath -> IO FilePath -relPathCwdToDir dir = liftM2 relPathDirToDir getCurrentDirectory (absPath dir) +relPathCwdToFile :: FilePath -> IO FilePath +relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f) -{- Constructs a relative path from one directory to another. +{- Constructs a relative path from a directory to a file. - - - Both directories must be absolute, and normalized (eg with absNormpath). - - - - The path will end with "/", unless it is empty. + - Both must be absolute, and normalized (eg with absNormpath). -} -relPathDirToDir :: FilePath -> FilePath -> FilePath -relPathDirToDir from to = - if not $ null path - then addTrailingPathSeparator path - else "" +relPathDirToFile :: FilePath -> FilePath -> FilePath +relPathDirToFile from to = path where s = [pathSeparator] pfrom = split s from @@ -211,12 +206,12 @@ relPathDirToDir from to = numcommon = length common path = join s $ dotdots ++ uncommon -prop_relPathDirToDir_basics :: FilePath -> FilePath -> Bool -prop_relPathDirToDir_basics from to +prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool +prop_relPathDirToFile_basics from to | from == to = null r - | otherwise = not (null r) && (last r == '/') + | otherwise = not (null r) where - r = relPathDirToDir from to + r = relPathDirToFile from to {- Removes a FileMode from a file. - For example, call with otherWriteMode to chmod o-w -} diff --git a/test.hs b/test.hs index 9304eee83f..7775fb8b5b 100644 --- a/test.hs +++ b/test.hs @@ -62,7 +62,7 @@ quickcheck = TestLabel "quickcheck" $ TestList , qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape , qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword , qctest "prop_parentDir_basics" Utility.prop_parentDir_basics - , qctest "prop_relPathDirToDir_basics" Utility.prop_relPathDirToDir_basics + , qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics , qctest "prop_cost_sane" Config.prop_cost_sane , qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane ] From 76911a446a7156ffb23679c6325fa8aab1edce13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 14:54:24 -0400 Subject: [PATCH 1601/8313] Avoid using absolute paths when staging location log, as that can confuse git when a remote's path contains a symlink. Closes: #621386 This was a real PITA to fix, since location logs can be staged in both the current repo, as well as in local remote's repos, in which case the cwd will not be in the repo. And git add needs different params in both cases, when absolute paths are not used. In passing, git annex fsck now stages location log fixes. --- Command/Fsck.hs | 9 ++++----- Command/Move.hs | 8 +------- Content.hs | 16 +++++++++++++--- GitRepo.hs | 35 +++++++++++++++++++++++------------ debian/changelog | 2 ++ 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index bedb9fb992..20ef2c8083 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -64,11 +64,11 @@ verifyLocationLog key file = do case (present, u `elem` uuids) of (True, False) -> do - fix g u ValuePresent + fix u ValuePresent -- There is no data loss, so do not fail. return True (False, True) -> do - fix g u ValueMissing + fix u ValueMissing warning $ "** Based on the location log, " ++ file ++ "\n** was expected to be present, " ++ @@ -77,7 +77,6 @@ verifyLocationLog key file = do _ -> return True where - fix g u s = do + fix u s = do showNote "fixing location log" - _ <- liftIO $ logChange g key u s - return () + logStatusFor u key s diff --git a/Command/Move.hs b/Command/Move.hs index e5e78d2495..476bf866a0 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -7,19 +7,15 @@ module Command.Move where -import Control.Monad.State (liftIO) - import Command import qualified Command.Drop import qualified Annex -import qualified AnnexQueue import LocationLog import Types import Content import qualified Remote import UUID import Messages -import Utility command :: [Command] command = [repoCommand "move" paramPath seek @@ -57,10 +53,8 @@ showAction False file = showStart "copy" file - for bare repos. -} remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex () remoteHasKey remote key present = do - g <- Annex.gitRepo let remoteuuid = Remote.uuid remote - logfile <- liftIO $ logChange g key remoteuuid status - AnnexQueue.add "add" [Param "--"] logfile + logStatusFor remoteuuid key status where status = if present then ValuePresent else ValueMissing diff --git a/Content.hs b/Content.hs index 576eecb319..bf94562218 100644 --- a/Content.hs +++ b/Content.hs @@ -9,6 +9,7 @@ module Content ( inAnnex, calcGitLink, logStatus, + logStatusFor, getViaTmp, getViaTmpUnchecked, checkDiskSpace, @@ -61,7 +62,8 @@ calcGitLink file key = do return $ relPathDirToFile (parentDir absfile) (Git.workTree g) ".git" annexLocation key -{- Updates the LocationLog when a key's presence changes. +{- Updates the LocationLog when a key's presence changes in the current + - repository. - - Note that the LocationLog is not updated in bare repositories. - Operations that change a bare repository should be done from @@ -69,11 +71,19 @@ calcGitLink file key = do - updated instead. -} logStatus :: Key -> LogStatus -> Annex () logStatus key status = do + g <- Annex.gitRepo + u <- getUUID g + logStatusFor u key status + +{- Updates the LocationLog when a key's presence changes in a repository + - identified by UUID. -} +logStatusFor :: UUID -> Key -> LogStatus -> Annex () +logStatusFor u key status = do g <- Annex.gitRepo unless (Git.repoIsLocalBare g) $ do - u <- getUUID g logfile <- liftIO $ logChange g key u status - AnnexQueue.add "add" [Param "--"] logfile + rellogfile <- liftIO $ Git.workTreeFile g logfile + AnnexQueue.add "add" [Param "--"] rellogfile {- Runs an action, passing it a temporary filename to download, - and if the action succeeds, moves the temp file into diff --git a/GitRepo.hs b/GitRepo.hs index 3e177cf1be..2bf320eda2 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -21,8 +21,8 @@ module GitRepo ( repoDescribe, repoLocation, workTree, + workTreeFile, gitDir, - relative, urlPath, urlHost, urlPort, @@ -59,7 +59,7 @@ module GitRepo ( prop_idempotent_deencode ) where -import Control.Monad (unless, when) +import Control.Monad (unless) import System.Directory import System.FilePath import System.Posix.Directory @@ -236,27 +236,38 @@ workTree r@(Repo { location = Url _ }) = urlPath r workTree (Repo { location = Dir d }) = d workTree Repo { location = Unknown } = undefined -{- Given a relative or absolute filename in a repository, calculates the - - name to use to refer to the file relative to a git repository's top. - - This is the same form displayed and used by git. -} -relative :: Repo -> FilePath -> IO FilePath -relative repo@(Repo { location = Dir d }) file = do +{- Given a relative or absolute filename inside a git repository's + - workTree, calculates the name to use to refer to that file to git. + - + - This is complicated because the best choice can vary depending on + - whether the cwd is in a subdirectory of the git repository, or not. + - + - For example, when adding a file "/tmp/repo/foo", it's best to refer + - to it as "foo" if the cwd is outside the repository entirely + - (this avoids a gotcha with using the full path name when /tmp/repo + - is itself a symlink). But, if the cwd is "/tmp/repo/subdir", + - it's best to refer to "../foo". + -} +workTreeFile :: Repo -> FilePath -> IO FilePath +workTreeFile repo@(Repo { location = Dir d }) file = do cwd <- getCurrentDirectory let file' = absfile cwd - let len = length absrepo - when (take len file' /= absrepo) $ + unless (inrepo file') $ error $ file ++ " is not located inside git repository " ++ absrepo - return $ drop (length absrepo) file' + if (inrepo $ addTrailingPathSeparator cwd) + then return $ relPathDirToFile cwd file' + else return $ drop (length absrepo) file' where -- normalize both repo and file, so that repo -- will be substring of file absrepo = case (absNormPath "/" d) of - Just f -> f ++ "/" + Just f -> addTrailingPathSeparator f Nothing -> error $ "bad repo" ++ repoDescribe repo absfile c = case (secureAbsNormPath c file) of Just f -> f Nothing -> file -relative repo _ = assertLocal repo $ error "internal" + inrepo f = absrepo `isPrefixOf` f +workTreeFile repo _ = assertLocal repo $ error "internal" {- Path of an URL repo. -} urlPath :: Repo -> String diff --git a/debian/changelog b/debian/changelog index 872277d0f5..c6dfb1ff32 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,8 @@ git-annex (0.20110421) UNRELEASED; urgency=low * Remove testpack from build depends, as it is not available on all architectures. The test suite will not be run if it cannot be compiled. + * Avoid using absolute paths when staging location log, as that can + confuse git when a remote's path contains a symlink. Closes: #621386 -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 From 3d3abab6797d49679bfcfa4d3bb919cfc7b5f811 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 15:28:41 -0400 Subject: [PATCH 1602/8313] move quickcheck Arbitrary declaration into test suite So git-annex can build w/o quickcheck installed. --- Key.hs | 13 ------------- test.hs | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Key.hs b/Key.hs index f52aea31b7..e1d8ee34d0 100644 --- a/Key.hs +++ b/Key.hs @@ -13,7 +13,6 @@ module Key ( prop_idempotent_key_read_show ) where -import Test.QuickCheck import Utility import System.Posix.Types @@ -71,17 +70,5 @@ readKey s = if key == Just stubKey then Nothing else key addfield 'm' k v = Just k { keyMtime = readMaybe v } addfield _ _ _ = Nothing --- for quickcheck -instance Arbitrary Key where - arbitrary = do - n <- arbitrary - b <- elements ['A'..'Z'] - return $ Key { - keyName = n, - keyBackendName = [b], - keySize = Nothing, - keyMtime = Nothing - } - prop_idempotent_key_read_show :: Key -> Bool prop_idempotent_key_read_show k = Just k == (readKey $ show k) diff --git a/test.hs b/test.hs index 7775fb8b5b..ab4766b428 100644 --- a/test.hs +++ b/test.hs @@ -7,6 +7,8 @@ import Test.HUnit import Test.HUnit.Tools +import Test.QuickCheck + import System.Directory import System.Posix.Directory (changeWorkingDirectory) import System.Posix.Files @@ -42,6 +44,18 @@ import qualified Key import qualified Config import qualified Crypto +-- for quickcheck +instance Arbitrary Key.Key where + arbitrary = do + n <- arbitrary + b <- elements ['A'..'Z'] + return $ Key.Key { + Key.keyName = n, + Key.keyBackendName = [b], + Key.keySize = Nothing, + Key.keyMtime = Nothing + } + main :: IO () main = do prepare From 7d71f8770bd4686c024071c0ca593ed8f16001e1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 16:02:57 -0400 Subject: [PATCH 1603/8313] releasing version 0.20110425 --- debian/changelog | 9 ++++----- debian/control | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index c6dfb1ff32..0ba7588f47 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,13 +1,12 @@ -git-annex (0.20110421) UNRELEASED; urgency=low +git-annex (0.20110425) unstable; urgency=low * Use haskell Crypto library instead of haskell SHA library. - * Remove testpack from build depends, as it is not available - on all architectures. The test suite will not be run if it - cannot be compiled. + * Remove testpack from build depends for non x86 architectures where it + is not available. The test suite will not be run if it cannot be compiled. * Avoid using absolute paths when staging location log, as that can confuse git when a remote's path contains a symlink. Closes: #621386 - -- Joey Hess Thu, 21 Apr 2011 16:35:27 -0400 + -- Joey Hess Mon, 25 Apr 2011 15:47:00 -0400 git-annex (0.20110420) unstable; urgency=low diff --git a/debian/control b/debian/control index f9e6196aba..c6b3e7d0d4 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Build-Depends: libghc-crypto-dev, libghc-utf8-string-dev, libghc-hs3-dev (>= 0.5.6), + libghc-testpack-dev [any-i386 any-amd64], ikiwiki, perlmagick, git | git-core, From 4ea9579b42aff232090e8238e27d5eec3001bd69 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 16:03:26 -0400 Subject: [PATCH 1604/8313] add news item for git-annex 0.20110425 --- doc/news/version_0.20110328.mdwn | 11 ----------- doc/news/version_0.20110425.mdwn | 7 +++++++ 2 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 doc/news/version_0.20110328.mdwn create mode 100644 doc/news/version_0.20110425.mdwn diff --git a/doc/news/version_0.20110328.mdwn b/doc/news/version_0.20110328.mdwn deleted file mode 100644 index 512ce4647a..0000000000 --- a/doc/news/version_0.20110328.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 0.20110328 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * annex.diskreserve can be given in arbitrary units (ie "0.5 gigabytes") - * Generalized remotes handling, laying groundwork for remotes that are - not regular git remotes. (Think Amazon S3.) - * Provide a less expensive version of `git annex copy --to`, enabled - via --fast. This assumes that location tracking information is correct, - rather than contacting the remote for every file. - * Bugfix: Keys could be received into v1 annexes from v2 annexes, via - v1 git-annex-shell. This results in some oddly named keys in the v1 - annex. Recognise and fix those keys when upgrading, instead of crashing."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110425.mdwn b/doc/news/version_0.20110425.mdwn new file mode 100644 index 0000000000..8f5c6515a0 --- /dev/null +++ b/doc/news/version_0.20110425.mdwn @@ -0,0 +1,7 @@ +git-annex 0.20110425 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Use haskell Crypto library instead of haskell SHA library. + * Remove testpack from build depends for non x86 architectures where it + is not available. The test suite will not be run if it cannot be compiled. + * Avoid using absolute paths when staging location log, as that can + confuse git when a remote's path contains a symlink. Closes: #[621386](http://bugs.debian.org/621386)"""]] \ No newline at end of file From 70d4e7349bd7ed04a9cc8c1b309c1fdfb375a7af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Apr 2011 22:04:12 -0400 Subject: [PATCH 1605/8313] ensure tmp file is writable, so rsync can resume It's possible that rsync finishes transferring a file and sets its mode, but the file transfer to the annex then fails. When resuming, rsync would then not be able to write to the tmp file. --- Content.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Content.hs b/Content.hs index bf94562218..dd0ea5ca1f 100644 --- a/Content.hs +++ b/Content.hs @@ -103,6 +103,8 @@ getViaTmp key action = do checkDiskSpace' (fromIntegral $ fileSize stat) key else checkDiskSpace key + when e $ liftIO $ allowWrite tmp + getViaTmpUnchecked key action {- Like getViaTmp, but does not check that there is enough disk space From ebd6ea1abb80c617f2bf1b693a54ec8b653db623 Mon Sep 17 00:00:00 2001 From: gernot Date: Tue, 26 Apr 2011 13:06:58 +0000 Subject: [PATCH 1606/8313] --- doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn b/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn new file mode 100644 index 0000000000..7db19697c3 --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn @@ -0,0 +1,5 @@ +The instructions for building git-annex on [[install/Debian]] stable don't seem to be valid anymore. + +1. `dpkg-checkbuilddeps` is looking for the wrong packages, e.g. libghc-missingh-dev instead of libghc6-missingh-dev. + +2. Not all dependencies are available in the Squeeze repositories anymore (at least not Crypto and hS3), if I am not mistaken. From 27774bdd56e05dabf98deeb09b78433fd374465c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 11:24:23 -0400 Subject: [PATCH 1607/8313] Revert "Use haskell Crypto library instead of haskell SHA library.a" This reverts commit 892593c5efacbc084d19af4b5d7164ededaea7ff. Conflicts: Crypto.hs debian/control --- Crypto.hs | 20 ++++++-------------- debian/changelog | 7 +++++++ debian/control | 2 +- doc/install.mdwn | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 4d6e38bac8..20b999f65a 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -29,21 +29,20 @@ module Crypto ( import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import qualified Codec.Binary.Base64 as B64 +import Data.ByteString.Lazy.UTF8 (fromString) +import Data.Digest.Pure.SHA +import System.Cmd.Utils import Data.String.Utils import Data.List import Data.Bits.Utils -import Data.HMAC -import Data.Array -import Codec.Utils -import System.Cmd.Utils import System.IO import System.Posix.IO import System.Posix.Types import System.Posix.Process -import System.Exit -import System.Environment import Control.Concurrent import Control.Exception (finally) +import System.Exit +import System.Environment import Types import Key @@ -253,17 +252,10 @@ fromB64 s = case B64.decode s of Nothing -> error "bad base64 encoded data" Just ws -> w82s ws - -showOctets :: [Octet] -> String -showOctets = concat . map hexChars - where - hexChars c = [arr ! (c `div` 16), arr ! (c `mod` 16)] - arr = listArray (0, 15) "0123456789abcdef" - hmacWithCipher :: Cipher -> String -> String hmacWithCipher c = hmacWithCipher' (cipherHmac c) hmacWithCipher' :: String -> String -> String -hmacWithCipher' c s = showOctets $ hmac_sha1 (s2w8 c) (s2w8 s) +hmacWithCipher' c s = showDigest $ hmacSha1 (fromString c) (fromString s) {- Ensure that hmacWithCipher' returns the same thing forevermore. -} prop_hmacWithCipher_sane :: Bool diff --git a/debian/changelog b/debian/changelog index 0ba7588f47..ce5e651b06 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (0.20110426) UNRELEASED; urgency=low + + * Switch back to haskell SHA library, so git-annex remains buildable on + Debian stable. + + -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 + git-annex (0.20110425) unstable; urgency=low * Use haskell Crypto library instead of haskell SHA library. diff --git a/debian/control b/debian/control index c6b3e7d0d4..b75b35afbc 100644 --- a/debian/control +++ b/debian/control @@ -6,8 +6,8 @@ Build-Depends: ghc, libghc-missingh-dev, libghc-pcre-light-dev, + libghc-sha-dev, libghc-dataenc-dev, - libghc-crypto-dev, libghc-utf8-string-dev, libghc-hs3-dev (>= 0.5.6), libghc-testpack-dev [any-i386 any-amd64], diff --git a/doc/install.mdwn b/doc/install.mdwn index a6b86891d5..3d15eac604 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -13,7 +13,7 @@ To build and use git-annex, you will need: * MissingH: * pcre-light: * utf8-string: -* crypto: +* SHA: * dataenc: * TestPack * QuickCheck 2 From de35104b41409e0b776b544589af9ca7fd15d1df Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 26 Apr 2011 15:27:49 +0000 Subject: [PATCH 1608/8313] Added a comment --- .../comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment new file mode 100644 index 0000000000..e464c84da1 --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 1" + date="2011-04-26T15:27:49Z" + content=""" +I have updated the instructions. +"""]] From 5535f91d917d1c48e1d5590420af84929353bba7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 11:30:49 -0400 Subject: [PATCH 1609/8313] update instructions for stable --- doc/install/Debian.mdwn | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index 90c9658a9c..64cbf32414 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -2,8 +2,23 @@ If using Debian testing or unstable: sudo apt-get install git-annex +## on Debian stable + git-annex is not yet in a Debian stable release, but the source -can be built from Debian stable. Just [[download]] it from git, and -then in the `git-annex` directory, run `dpkg-checkbuilddeps` and install -the necessary packages. You can also run `sudo debian/rules binary` to build +can be built from Debian stable. + +First, install build dependencies. Package names have changed since +Debian stable, so this must be done by hand, rather than by using +`dpkg-checkbuilddeps`: + + sudo apt-get install debhelper ghc6 libghc-missingh-dev \ + libghc6-pcre-light-dev libghc6-sha-dev \ + libghc6-dataenc-dev libghc6-utf8-string-dev \ + libghc6-testpack-dev + +Then [[download]] git-annex it from git. + +In the `git-annex` directory, run `sudo debian/rules binary` to build a `git-annex.deb`. + +Note that S3 support will not be available when building on stable. From 920d736fa9626be88a193bd981c04f44b08ff143 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 11:33:59 -0400 Subject: [PATCH 1610/8313] update --- doc/install/Debian.mdwn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index 64cbf32414..6bed4825ba 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -2,10 +2,10 @@ If using Debian testing or unstable: sudo apt-get install git-annex -## on Debian stable +### Debian stable -git-annex is not yet in a Debian stable release, but the source -can be built from Debian stable. +git-annex is not yet in a Debian stable release, and nobody has backported it +(yet?), but the source can be built from Debian stable. First, install build dependencies. Package names have changed since Debian stable, so this must be done by hand, rather than by using From 7a3b45db29cad8efcf0754cb9eafd1ca9874d057 Mon Sep 17 00:00:00 2001 From: gernot Date: Tue, 26 Apr 2011 18:56:45 +0000 Subject: [PATCH 1611/8313] Added a comment --- ..._f6ff8306c946219dbe39bb8938a349ab._comment | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment new file mode 100644 index 0000000000..40f570610a --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="gernot" + ip="213.196.216.21" + subject="comment 2" + date="2011-04-26T18:56:44Z" + content=""" +Thanks for the update, Joey. I think you forgot to change libghc-missingh-dev to libghc6-missingh-dev for the copy & paste instructions though. + +Also, after having checked that I have everything installed I'm still getting this error: + + ... + [15 of 77] Compiling Annex ( Annex.hs, Annex.o ) + + Annex.hs:19:35: + Module `Control.Monad.State' does not export `state' + make[1]: *** [git-annex] Error 1 + make[1]: Leaving directory `/home/gernot/dev/git-annex' + dh_auto_build: make -j1 returned exit code 2 + make: *** [binary] Error 2 + +"""]] From 168f010fdff967e4be8cf36e26f63ca2d712d4e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 19:39:30 -0400 Subject: [PATCH 1612/8313] typo --- doc/install/Debian.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index 6bed4825ba..e9b04dc557 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -11,7 +11,7 @@ First, install build dependencies. Package names have changed since Debian stable, so this must be done by hand, rather than by using `dpkg-checkbuilddeps`: - sudo apt-get install debhelper ghc6 libghc-missingh-dev \ + sudo apt-get install debhelper ghc6 libghc6-missingh-dev \ libghc6-pcre-light-dev libghc6-sha-dev \ libghc6-dataenc-dev libghc6-utf8-string-dev \ libghc6-testpack-dev From 89dc1e5de3b185f5f8871ccf9af9dfb2d4ebe268 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Tue, 26 Apr 2011 23:40:33 +0000 Subject: [PATCH 1613/8313] Added a comment --- .../comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment diff --git a/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment new file mode 100644 index 0000000000..8b41116431 --- /dev/null +++ b/doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 3" + date="2011-04-26T23:40:33Z" + content=""" +Both problems fixed. +"""]] From 33d23a4ef929f3affb86d7cd6c733101f052d624 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 19:42:40 -0400 Subject: [PATCH 1614/8313] Control.Monad.State import fix for debian stable It doesn't export `state` there, so hiding it fails. Just list explicitly what we use. --- Annex.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Annex.hs b/Annex.hs index 03f9130b6a..9915112a59 100644 --- a/Annex.hs +++ b/Annex.hs @@ -16,7 +16,8 @@ module Annex ( gitRepo ) where -import Control.Monad.State hiding (state) +import Control.Monad.State + (liftIO, StateT, runStateT, evalStateT, liftM, get, put) import qualified GitRepo as Git import qualified GitQueue From bb8e3c5b6dda5874ec214d7f6dcfe6f8445d622b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 19:59:01 -0400 Subject: [PATCH 1615/8313] more Control.Monad.State export fix --- Backend.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Backend.hs b/Backend.hs index 0edff2ab5e..aec87ce665 100644 --- a/Backend.hs +++ b/Backend.hs @@ -32,7 +32,7 @@ module Backend ( maybeLookupBackendName ) where -import Control.Monad.State hiding (state) +import Control.Monad.State (liftIO, when) import System.IO.Error (try) import System.FilePath import System.Posix.Files From 948691e89322e88ed2f83bfee4a18dcbdd2f696e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 20:08:36 -0400 Subject: [PATCH 1616/8313] exit nonzero when there were failure, not just errors ya, I need a test suite for my test suite --- test.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.hs b/test.hs index ab4766b428..2cb0415ab9 100644 --- a/test.hs +++ b/test.hs @@ -64,8 +64,8 @@ main = do propigate r propigate :: (Counts, Int) -> IO () -propigate (Counts { errors = e }, _) - | e > 0 = error "failed" +propigate (Counts { errors = e , failures = f }, _) + | e+f > 0 = error "failed" | otherwise = return () quickcheck :: Test From dbdcb67f791502f4f6cc26917c1252b998efa482 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Apr 2011 20:21:24 -0400 Subject: [PATCH 1617/8313] fix test suite when run by root --- test.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test.hs b/test.hs index 2cb0415ab9..4e0eb3fb2a 100644 --- a/test.hs +++ b/test.hs @@ -577,10 +577,14 @@ checkcontent f = do checkunwritable :: FilePath -> Assertion checkunwritable f = do - r <- try $ writeFile f $ content f - case r of - Left _ -> return () -- expected permission error - Right _ -> assertFailure $ "was able to modify annexed file's " ++ f ++ " content" + -- Look at permissions bits rather than trying to write or using + -- fileAccess because if run as root, any file can be modified + -- despite permissions. + s <- getFileStatus f + let mode = fileMode s + if (mode == mode `unionFileModes` ownerWriteMode) + then assertFailure $ "able to modify annexed file's " ++ f ++ " content" + else return () checkwritable :: FilePath -> Assertion checkwritable f = do @@ -640,7 +644,7 @@ runchecks (a:as) f = do annexed_notpresent :: FilePath -> Assertion annexed_notpresent = runchecks - [checklink, checkdangling, checkunwritable, notinlocationlog] + [checklink, checkdangling, notinlocationlog] annexed_present :: FilePath -> Assertion annexed_present = runchecks From 568cb52b6e55b594dd67c466464ae4a9583accb4 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Wed, 27 Apr 2011 14:27:27 +0000 Subject: [PATCH 1618/8313] sftp backend? --- doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn new file mode 100644 index 0000000000..becfe23c79 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn @@ -0,0 +1,5 @@ +i think it would be useful to have a fourth kind of [[special remote]]s that connects to a dumb storage using sftp or rsync. this can be emulated by using sshfs, but that means lots of round-trips through the system and is limited to platforms where sshfs is available. + +typical use cases are backups to storate shared between a group of people where each user only has limited access (sftp or rsync), when using [[bup]] is not an option. + +an alternative to implementing yet another special remote would be to have some kind of plugin system by which external programs can provide an interface to key-value stores (i'd implement the sftp backend myself, but haven't learned haskell yet). From 0d6860bb6fc02d14b3f5c773f0a2c4248463bb48 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 11:07:54 -0400 Subject: [PATCH 1619/8313] drastically simplify instructions for debian stable :) --- doc/install/Debian.mdwn | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/doc/install/Debian.mdwn b/doc/install/Debian.mdwn index e9b04dc557..db1329b45b 100644 --- a/doc/install/Debian.mdwn +++ b/doc/install/Debian.mdwn @@ -1,24 +1,8 @@ If using Debian testing or unstable: - sudo apt-get install git-annex +* `sudo apt-get install git-annex` -### Debian stable +If using Debian 6.0 stable: -git-annex is not yet in a Debian stable release, and nobody has backported it -(yet?), but the source can be built from Debian stable. - -First, install build dependencies. Package names have changed since -Debian stable, so this must be done by hand, rather than by using -`dpkg-checkbuilddeps`: - - sudo apt-get install debhelper ghc6 libghc6-missingh-dev \ - libghc6-pcre-light-dev libghc6-sha-dev \ - libghc6-dataenc-dev libghc6-utf8-string-dev \ - libghc6-testpack-dev - -Then [[download]] git-annex it from git. - -In the `git-annex` directory, run `sudo debian/rules binary` to build -a `git-annex.deb`. - -Note that S3 support will not be available when building on stable. +* Follow the instructions to [enable backports](http://backports.debian.org/Instructions/). +* `sudo apt-get -t squeeze-backports install git-annex` From e68f128a9bf46c8f4ebe51fcb3b6f63955cadd2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 20:06:07 -0400 Subject: [PATCH 1620/8313] rsync special remote Fully tested and working, including resuming and encryption. (Though not resuming when sending *with* encryption; gpg doesn't produce identical output each time.) Uses same layout as the directory special remote and the .git/annex/objects/ directory. --- Content.hs | 12 ++ Remote.hs | 2 + Remote/Git.hs | 2 +- Remote/Rsync.hs | 193 ++++++++++++++++++ Remote/S3real.hs | 20 +- Utility.hs | 5 +- debian/changelog | 2 + ...ist:_special_remote_for_sftp_or_rsync.mdwn | 22 +- doc/special_remotes.mdwn | 1 + doc/special_remotes/directory.mdwn | 2 +- doc/special_remotes/rsync.mdwn | 23 +++ 11 files changed, 265 insertions(+), 19 deletions(-) create mode 100644 Remote/Rsync.hs create mode 100644 doc/special_remotes/rsync.mdwn diff --git a/Content.hs b/Content.hs index dd0ea5ca1f..99770f553d 100644 --- a/Content.hs +++ b/Content.hs @@ -12,6 +12,7 @@ module Content ( logStatusFor, getViaTmp, getViaTmpUnchecked, + withTmp, checkDiskSpace, preventWrite, allowWrite, @@ -127,6 +128,17 @@ getViaTmpUnchecked key action = do -- to resume its transfer return False +{- Creates a temp file, runs an action on it, and cleans up the temp file. -} +withTmp :: Key -> (FilePath -> Annex a) -> Annex a +withTmp key action = do + g <- Annex.gitRepo + let tmp = gitAnnexTmpLocation g key + liftIO $ createDirectoryIfMissing True (parentDir tmp) + res <- action tmp + tmp_exists <- liftIO $ doesFileExist tmp + when tmp_exists $ liftIO $ removeFile tmp + return res + {- Checks that there is disk space available to store a given key, - throwing an error if not. -} checkDiskSpace :: Key -> Annex () diff --git a/Remote.hs b/Remote.hs index 8d2ab0399a..f47bea560b 100644 --- a/Remote.hs +++ b/Remote.hs @@ -48,6 +48,7 @@ import qualified Remote.Git import qualified Remote.S3 import qualified Remote.Bup import qualified Remote.Directory +import qualified Remote.Rsync remoteTypes :: [RemoteType Annex] remoteTypes = @@ -55,6 +56,7 @@ remoteTypes = , Remote.S3.remote , Remote.Bup.remote , Remote.Directory.remote + , Remote.Rsync.remote ] {- Builds a list of all available Remotes. diff --git a/Remote/Git.hs b/Remote/Git.hs index bab452a331..e6df6be46e 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -158,7 +158,7 @@ rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool) rsynchelper r sending key file = do showProgress -- make way for progress bar p <- rsyncParams r sending key file - res <- liftIO $ boolSystem "rsync" p + res <- liftIO $ rsync p if res then return res else do diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs new file mode 100644 index 0000000000..0a62ff92fc --- /dev/null +++ b/Remote/Rsync.hs @@ -0,0 +1,193 @@ +{- A remote that is only accessible by rsync. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Rsync (remote) where + +import qualified Data.ByteString.Lazy.Char8 as L +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad.State (liftIO, when) +import System.FilePath +import System.Directory +import System.Posix.Files +import System.Posix.Process + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import Config +import Content +import Utility +import Remote.Special +import Remote.Encryptable +import Crypto +import Messages +import RsyncFile + +type RsyncUrl = String + +data RsyncOpts = RsyncOpts { + rsyncUrl :: RsyncUrl, + rsyncOptions :: [CommandParam] +} + +remote :: RemoteType Annex +remote = RemoteType { + typename = "rsync", + enumerate = findSpecialRemotes "rsyncurl", + generate = gen, + setup = rsyncSetup +} + +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) +gen r u c = do + url <- getConfig r "rsyncurl" (error "missing rsyncurl") + opts <- getConfig r "rsync-options" "" + let o = RsyncOpts url $ map Param $ words opts + cst <- remoteCost r expensiveRemoteCost + return $ encryptableRemote c + (storeEncrypted o) + (retrieveEncrypted o) + Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store o, + retrieveKeyFile = retrieve o, + removeKey = remove o, + hasKey = checkPresent r o, + hasKeyCheap = True, + config = Nothing + } + +rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +rsyncSetup u c = do + -- verify configuration is sane + let url = case M.lookup "rsyncurl" c of + Nothing -> error "Specify rsyncurl=" + Just d -> d + c' <- encryptionSetup c + + -- The rsyncurl is stored in git config, not only in this remote's + -- persistant state, so it can vary between hosts. + gitConfigSpecialRemote u c' "rsyncurl" url + return c' + +rsyncKey :: RsyncOpts -> Key -> String +rsyncKey o k = rsyncUrl o hashDirMixed k f f + where + f = keyFile k + +store :: RsyncOpts -> Key -> Annex Bool +store o k = do + g <- Annex.gitRepo + rsyncSend o k (gitAnnexLocation g k) + +storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + rsyncSend o enck tmp + +retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool +retrieve o k f = rsyncRemote o + -- use inplace when retrieving to support resuming + [ Param "--inplace" + , Param $ rsyncKey o k + , Param f + ] + +retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do + res <- retrieve o enck tmp + if res + then liftIO $ catchBool $ do + withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f + return True + else return res + +remove :: RsyncOpts -> Key -> Annex Bool +remove o k = withRsyncScratchDir $ \tmp -> do + {- Send an empty directory to rysnc as the parent directory + - of the file to remove. -} + let dummy = tmp keyFile k + liftIO $ createDirectoryIfMissing True dummy + liftIO $ rsync $ rsyncOptions o ++ + [ Params "--delete --recursive" + , partialParams + , Param $ addTrailingPathSeparator dummy + , Param $ parentDir $ rsyncKey o k + ] + +checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool) +checkPresent r o k = do + showNote ("checking " ++ Git.repoDescribe r ++ "...") + -- note: Does not currently differnetiate between rsync failing + -- to connect, and the file not being present. + res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd] + return $ Right res + where + cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null" + testfile = shellEscape $ rsyncKey o k + +{- Rsync params to enable resumes of sending files safely, + - ensure that files are only moved into place once complete + -} +partialParams :: CommandParam +partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial" + +{- Runs an action in an empty scratch directory that can be used to build + - up trees for rsync. -} +withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool +withRsyncScratchDir a = do + g <- Annex.gitRepo + pid <- liftIO $ getProcessID + let tmp = gitAnnexTmpDir g "rsynctmp" show pid + nuke tmp + liftIO $ createDirectoryIfMissing True $ tmp + res <- a tmp + nuke tmp + return res + where + nuke d = liftIO $ do + e <- doesDirectoryExist d + when e $ liftIO $ removeDirectoryRecursive d + +rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool +rsyncRemote o params = do + showProgress -- make way for progress bar + res <- liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params + if res + then return res + else do + showLongNote "rsync failed -- run git annex again to resume file transfer" + return res + where + defaultParams = [Params "--progress"] + +{- To send a single key is slightly tricky; need to build up a temporary + directory structure to pass to rsync so it can create the hash + directories. -} +rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool +rsyncSend o k src = withRsyncScratchDir $ \tmp -> do + let dest = tmp hashDirMixed k f f + liftIO $ createDirectoryIfMissing True $ parentDir $ dest + liftIO $ createLink src dest + res <- rsyncRemote o + [ Param "--recursive" + , partialParams + -- tmp/ to send contents of tmp dir + , Param $ addTrailingPathSeparator tmp + , Param $ rsyncUrl o + ] + return res + where + f = keyFile k diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 07e33368ea..2e198f79d1 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -19,7 +19,6 @@ import Control.Monad (when) import Control.Monad.State (liftIO) import System.Environment import System.Posix.Files -import System.Directory import RemoteClass import Types @@ -33,7 +32,7 @@ import Remote.Special import Remote.Encryptable import Crypto import Key -import Utility +import Content remote :: RemoteType Annex remote = RemoteType { @@ -108,18 +107,15 @@ store r k = s3Action r False $ \(conn, bucket) -> do s3Bool res storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool -storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> do - g <- Annex.gitRepo - let f = gitAnnexLocation g k +storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> -- To get file size of the encrypted content, have to use a temp file. -- (An alternative would be chunking to to a constant size.) - let tmp = gitAnnexTmpLocation g enck - liftIO $ createDirectoryIfMissing True (parentDir tmp) - liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s - res <- liftIO $ storeHelper (conn, bucket) r enck tmp - tmp_exists <- liftIO $ doesFileExist tmp - when tmp_exists $ liftIO $ removeFile tmp - s3Bool res + withTmp enck $ \tmp -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + res <- liftIO $ storeHelper (conn, bucket) r enck tmp + s3Bool res storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> FilePath -> IO (AWSResult ()) storeHelper (conn, bucket) r k file = do diff --git a/Utility.hs b/Utility.hs index 13ebbfccba..51bbc17a32 100644 --- a/Utility.hs +++ b/Utility.hs @@ -95,8 +95,9 @@ boolSystem command params = do restoresignals oldint oldset executeFile command True (toCommand params) Nothing -{- Escapes a filename to be safely able to be exposed to the shell. -} -shellEscape :: FilePath -> String +{- Escapes a filename or other parameter to be safely able to be exposed to + - the shell. -} +shellEscape :: String -> String shellEscape f = "'" ++ escaped ++ "'" where -- replace ' with '"'"' diff --git a/debian/changelog b/debian/changelog index ce5e651b06..78f65e8b97 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ git-annex (0.20110426) UNRELEASED; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on Debian stable. + * Added rsync special remotes. This could be used, for example, to + store annexed content on rsync.net, encrypted naturally. Or anywhere else. -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn index becfe23c79..9807bf91ed 100644 --- a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn @@ -1,5 +1,21 @@ -i think it would be useful to have a fourth kind of [[special remote]]s that connects to a dumb storage using sftp or rsync. this can be emulated by using sshfs, but that means lots of round-trips through the system and is limited to platforms where sshfs is available. +i think it would be useful to have a fourth kind of [[special remote]]s +that connects to a dumb storage using sftp or rsync. this can be emulated +by using sshfs, but that means lots of round-trips through the system and +is limited to platforms where sshfs is available. -typical use cases are backups to storate shared between a group of people where each user only has limited access (sftp or rsync), when using [[bup]] is not an option. +typical use cases are backups to storate shared between a group of people +where each user only has limited access (sftp or rsync), when using [[bup]] +is not an option. -an alternative to implementing yet another special remote would be to have some kind of plugin system by which external programs can provide an interface to key-value stores (i'd implement the sftp backend myself, but haven't learned haskell yet). +an alternative to implementing yet another special remote would be to have +some kind of plugin system by which external programs can provide an +interface to key-value stores (i'd implement the sftp backend myself, but +haven't learned haskell yet). + +> Ask and ye [[shall receive|special_remotes/rsync]]. +> +> Sometimes I almost think that a generic configurable special remote that +> just uses configured shell commands would be useful.. But there's really +> no comparison with sitting down and writing code tuned to work with +> a given transport like rsync, when it comes to reliability and taking +> advantage of its abilities (like resuming). --[[Joey]] diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index a33d3f6127..210f995d2f 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -9,6 +9,7 @@ They cannot be used by other git commands though. * [[Amazon_S3]] * [[bup]] * [[directory]] +* [[rsync]] ## Unused content on special remotes diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index 8006c44fc9..9e4bfa33bd 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -7,4 +7,4 @@ the drive's mountpoint as a directory remote. Setup example: - # git annex initremote usbdrive directory=/media/usbdrive/ encryption=none + # git annex initremote usbdrive type=directory directory=/media/usbdrive/ encryption=none diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn new file mode 100644 index 0000000000..5936442916 --- /dev/null +++ b/doc/special_remotes/rsync.mdwn @@ -0,0 +1,23 @@ +This special remote type rsyncs file contents to somewhere else. + +Setup example: + + # git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net + +## configuration + +These parameters can be passed to `git annex initremote` to configure rsync: + +* `encryption` - Required. Either "none" to disable encryption of content + stored in rsync, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. See [[encryption]]. + +* `rsyncurl` - Required. This is the url or `hostname:/directory` to + pass to rsync to tell it where to store content. + +The `annex-rsync-options` git configuration setting can be used to pass +parameters to rsync. Note that it is **not safe** to put "--delete" +in `annex-rsync-options` when using rsync special remotes. From 39966ba4eeb6046c511d3f3b630a3ee2ced5019a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 20:30:43 -0400 Subject: [PATCH 1621/8313] filter out --delete rsync option rsync does not have a --no-delete, so do it this way instead --- Remote/Rsync.hs | 18 +++++++++++++++--- debian/changelog | 2 +- doc/special_remotes/rsync.mdwn | 3 +-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 0a62ff92fc..21c570a87c 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -48,9 +48,7 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do - url <- getConfig r "rsyncurl" (error "missing rsyncurl") - opts <- getConfig r "rsync-options" "" - let o = RsyncOpts url $ map Param $ words opts + o <- genRsyncOpts r cst <- remoteCost r expensiveRemoteCost return $ encryptableRemote c (storeEncrypted o) @@ -67,6 +65,20 @@ gen r u c = do config = Nothing } +genRsyncOpts :: Git.Repo -> Annex RsyncOpts +genRsyncOpts r = do + url <- getConfig r "rsyncurl" (error "missing rsyncurl") + opts <- getConfig r "rsync-options" "" + return $ RsyncOpts url $ map Param $ filter safe $ words opts + where + safe o + -- Don't allow user to pass --delete to rsync; + -- that could cause it to delete other keys + -- in the same hash bucket as a key it sends. + | o == "--delete" = False + | o == "--delete-excluded" = False + | otherwise = True + rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig rsyncSetup u c = do -- verify configuration is sane diff --git a/debian/changelog b/debian/changelog index 78f65e8b97..991c9e0bb9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,7 +3,7 @@ git-annex (0.20110426) UNRELEASED; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on Debian stable. * Added rsync special remotes. This could be used, for example, to - store annexed content on rsync.net, encrypted naturally. Or anywhere else. + store annexed content on rsync.net (encrypted naturally). Or anywhere else. -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn index 5936442916..b13f9ace76 100644 --- a/doc/special_remotes/rsync.mdwn +++ b/doc/special_remotes/rsync.mdwn @@ -19,5 +19,4 @@ These parameters can be passed to `git annex initremote` to configure rsync: pass to rsync to tell it where to store content. The `annex-rsync-options` git configuration setting can be used to pass -parameters to rsync. Note that it is **not safe** to put "--delete" -in `annex-rsync-options` when using rsync special remotes. +parameters to rsync. From 7a338031933cbba7b021468ee83bb63fb3d6d42a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Apr 2011 22:10:57 -0400 Subject: [PATCH 1622/8313] Avoid pipeline stall when running git annex drop or fsck on a lot of files. When it's stalled, there are 3 processes: git annex git ls-files git check-attr git-annex stalls trying to write to git check-attr, which stalls trying to write to stdout (read by git-annex). git ls-files does not seem to be involved directly; I've seen the stall when it was still streaming out the file list, and after it had exited and zombified. The read and write are supposed to be handled by two different threads, which pipeBoth forks off, thus avoiding deadlock. But it does deadlock. (Certian signals unblock the deadlock for a while, then it stalls again.) So, this is another case of WTF is the ghc IO manager doing today? I avoid the issue by converting the writer to a separate process. Possibly this was caused by some change in ghc 7 -- I'm offline and cannot verify now, but I'm sure I used to be able to run git annex drop w/o it hanging! And the code does not seem to have changed, except for commit c1dc4079419cff94cca72441d5e67a866110ec7e, which I tried reverting without success. In fact, I reverted all the way back to 0.20110316 and still saw the stall. Update: Minimal test case: import System.Cmd.Utils main = do as <- checkAttr "blah" $ map show [1..100000] sequence $ map (putStrLn . show) as checkAttr attr files = do (_, s) <- pipeBoth "git" params $ unlines files return $ lines s where params = ["check-attr", attr, "--stdin"] Bug filed on ghc in debian, #624389 --- GitRepo.hs | 10 +++++++++- debian/changelog | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/GitRepo.hs b/GitRepo.hs index 2bf320eda2..9ecaa8ffcb 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -78,6 +78,7 @@ import Data.Word (Word8) import Codec.Binary.UTF8.String (encode) import Text.Printf import Data.List (isInfixOf, isPrefixOf) +import System.Exit import Utility @@ -482,7 +483,14 @@ checkAttr repo attr files = do -- in its output back to relative. cwd <- getCurrentDirectory let absfiles = map (absPathFrom cwd) files - (_, s) <- pipeBoth "git" (toCommand params) $ join "\0" absfiles + (_, fromh, toh) <- hPipeBoth "git" (toCommand params) + _ <- forkProcess $ do + hClose fromh + hPutStr toh $ join "\0" absfiles + hClose toh + exitSuccess + hClose toh + s <- hGetContents fromh return $ map (topair $ cwd++"/") $ lines s where params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"] diff --git a/debian/changelog b/debian/changelog index 991c9e0bb9..a69f03f5ef 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,8 @@ git-annex (0.20110426) UNRELEASED; urgency=low Debian stable. * Added rsync special remotes. This could be used, for example, to store annexed content on rsync.net (encrypted naturally). Or anywhere else. + * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a + lot of files. Possibly only occured with ghc 7. -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 From 4e6dd7f31988834e513a285eb338e4428e892e3e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 28 Apr 2011 07:47:39 +0000 Subject: [PATCH 1623/8313] Added a comment --- ...comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment new file mode 100644 index 0000000000..c513ed4008 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2011-04-28T07:47:38Z" + content=""" ++1 for a generic user configurable backend that a user can put shell commands in, which has a disclaimer such that if a user hangs themselves with misconfiguration then its their own fault :P + +I would love to be able to quickly plugin an irods/sector set of put/get/delete/stat(get info) commands into git-annex to access my private clouds which aren't s3 compatible. +"""]] From d980a55b47136eb313ceecc6f18f82fb2e3cdbf5 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Thu, 28 Apr 2011 11:27:51 +0000 Subject: [PATCH 1624/8313] thanks & notes on migration --- doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn index 9807bf91ed..47e8b85622 100644 --- a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn @@ -19,3 +19,8 @@ haven't learned haskell yet). > no comparison with sitting down and writing code tuned to work with > a given transport like rsync, when it comes to reliability and taking > advantage of its abilities (like resuming). --[[Joey]] + +>> big thanks, and bonus points for identical formats, so converting from +>> directory to rsync is just a matter of changing ``type`` from ``directory`` +>> to ``rsync`` in ``.git-annex/remote.log`` and replacing the directory info +>> with ``annex-rsyncurl = :`` in ``.git/config``. --[[chrysn]] From 5201c43787e2c712a3917c418228c2fd8feca9ae Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Thu, 28 Apr 2011 13:45:32 +0000 Subject: [PATCH 1625/8313] added example fo rusing rsync over ssh --- doc/special_remotes/rsync.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn index b13f9ace76..7a7f3ab149 100644 --- a/doc/special_remotes/rsync.mdwn +++ b/doc/special_remotes/rsync.mdwn @@ -4,6 +4,10 @@ Setup example: # git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net +Or for using rsync over SSH + + # git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=joey@kitenet.net + ## configuration These parameters can be passed to `git annex initremote` to configure rsync: From 84e1ebfb0eab5490ace84c1e8a66688bd52a1a54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 14:38:01 -0400 Subject: [PATCH 1626/8313] erm, thought I committed this release? --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index a69f03f5ef..cb3913470e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110426) UNRELEASED; urgency=low +git-annex (0.20110427) unstable; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on Debian stable. @@ -7,7 +7,7 @@ git-annex (0.20110426) UNRELEASED; urgency=low * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a lot of files. Possibly only occured with ghc 7. - -- Joey Hess Tue, 26 Apr 2011 11:23:54 -0400 + -- Joey Hess Wed, 27 Apr 2011 22:50:26 -0400 git-annex (0.20110425) unstable; urgency=low From d7b330b33bcfa57164dad05b0fc9990ad4a35275 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 14:39:51 -0400 Subject: [PATCH 1627/8313] Fix hasKeyCheap setting for bup and rsync special remotes. --- Remote/Bup.hs | 2 +- Remote/Rsync.hs | 2 +- debian/changelog | 6 ++++++ doc/news/version_0.20110401.mdwn | 11 ----------- doc/news/version_0.20110427.mdwn | 8 ++++++++ 5 files changed, 16 insertions(+), 13 deletions(-) delete mode 100644 doc/news/version_0.20110401.mdwn create mode 100644 doc/news/version_0.20110427.mdwn diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 6ae002c3b9..0aaff06b25 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -61,7 +61,7 @@ gen r u c = do retrieveKeyFile = retrieve buprepo, removeKey = remove, hasKey = checkPresent r bupr', - hasKeyCheap = True, + hasKeyCheap = bupLocal buprepo, config = c } diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 21c570a87c..682c961748 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -61,7 +61,7 @@ gen r u c = do retrieveKeyFile = retrieve o, removeKey = remove o, hasKey = checkPresent r o, - hasKeyCheap = True, + hasKeyCheap = False, config = Nothing } diff --git a/debian/changelog b/debian/changelog index cb3913470e..870a45820e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110428) UNRELEASED; urgency=low + + * Fix hasKeyCheap setting for bup and rsync special remotes. + + -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 + git-annex (0.20110427) unstable; urgency=low * Switch back to haskell SHA library, so git-annex remains buildable on diff --git a/doc/news/version_0.20110401.mdwn b/doc/news/version_0.20110401.mdwn deleted file mode 100644 index 7c9ca6f5ca..0000000000 --- a/doc/news/version_0.20110401.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 0.20110401 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Amazon S3 is now supported as a special type of remote. - Warning: Encrypting data before sending it to S3 is not yet supported. - * Note that Amazon S3 support is not built in by default on Debian yet, - as hS3 is not packaged. - * fsck: Ensure that files and directories in .git/annex/objects - have proper permissions. - * Added a special type of remote called a directory remote, which - simply stores files in an arbitrary local directory. - * Bugfix: copy --to --fast never really copied, fixed."""]] \ No newline at end of file diff --git a/doc/news/version_0.20110427.mdwn b/doc/news/version_0.20110427.mdwn new file mode 100644 index 0000000000..2764c6de56 --- /dev/null +++ b/doc/news/version_0.20110427.mdwn @@ -0,0 +1,8 @@ +git-annex 0.20110427 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Switch back to haskell SHA library, so git-annex remains buildable on + Debian stable. + * Added rsync special remotes. This could be used, for example, to + store annexed content on rsync.net (encrypted naturally). Or anywhere else. + * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a + lot of files. Possibly only occured with ghc 7."""]] \ No newline at end of file From 07576f2a2c84a54dc3b0e2aa6050e747f29c3a43 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 15:26:21 -0400 Subject: [PATCH 1628/8313] documentation for hook special remotes Releasing before I have quite finished the code. Got a little caught up in Anathem references. Time for a walk and then a tiny bit more coding and possibly testing. --- debian/changelog | 1 + doc/special_remotes.mdwn | 1 + doc/special_remotes/hook.mdwn | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 doc/special_remotes/hook.mdwn diff --git a/debian/changelog b/debian/changelog index 870a45820e..b7f33a7041 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. + * Add hook special remotes. -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn index 210f995d2f..7d55a88612 100644 --- a/doc/special_remotes.mdwn +++ b/doc/special_remotes.mdwn @@ -10,6 +10,7 @@ They cannot be used by other git commands though. * [[bup]] * [[directory]] * [[rsync]] +* [[hook]] ## Unused content on special remotes diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn new file mode 100644 index 0000000000..521b440277 --- /dev/null +++ b/doc/special_remotes/hook.mdwn @@ -0,0 +1,65 @@ +This special remote type runs hooks that you configure to store content. + +It's not recommended to use this remote type when another like [[rsync]] +or [[directory]] will do. If your hooks are not carefully written, data +could be lost. + +## example + +Here's a simple example that stores content on clay tablets. If you +implement this example in the real world, I'd appreciate a tour +next Apert! :) --[[Joey]] + + # git config annex.cuneiform-store-hook 'tocuneiform < $ANNEX_FILE | tablet-writer --implement=stylus --title=$ANNEX_KEY | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' + # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > $ANNEX_FILE' + # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY' + # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY --shout-title' + # git annex initremote special type=hook hooktype=cuneiform encryption=none + +Can you spot the potential data loss bugs in the above simple example? +(Hint: What happens when the `tablet-proofreader` exits nonzero?) + +## configuration + +These parameters can be passed to `git annex initremote`: + +* `encryption` - Required. Either "none" to disable encryption of content, + or a value that can be looked up (using gpg -k) to find a gpg encryption + key that will be given access to the remote. Note that additional gpg + keys can be given access to a remote by rerunning initremote with + the new key id. See [[encryption]]. + +* `hooktype` - Required. This specifies a collection of hooks to use for + this remote. + +## hooks + +Each type of hook remote is specified by a collection of hook commands. +Each hook command is run as a shell command line, and should return nonzero +on failure, and zero on success. + +These environment variables are used to communicate with the hook commands: + +* `ANNEX_KEY` - name of a key to store, retrieve, remove, or check. +* `ANNEX_FILE` - a file containing the key's content +* `ANNEX_HASH_1` - short stable value, based on the key, can be used for hashing +* `ANNEX_HASH_2` - another hash value, can be used for a second level of hashing + +The setting to use in git config for the hook commands are as follows: + +* `annex.$hooktype-store-hook` - Command run to store a key in the special remote. + `ANNEX_FILE` contains the content to be stored. + +* `annex.$hooktype-retrieve-hook` - Command run to retrieve a key from the special remote. + `ANNEX_FILE` is a file that the retrieved content should be written to. + The file may already exist with a partial + copy of the content (or possibly just garbage), to allow for resuming + of partial transfers. + +* `annex.$hooktype-remove-hook` - Command to remove a key from the special remote. + +* `annex.$hooktype-checkpresent-hook` - Command to check if a key is present + in the special remote. Should output the key name to stdout, on its own line, + if and only if the key has been actively verified to be present in the + special remote (caching presence information is a very bad idea); + all other output to stdout will be ignored. From 56eaf4470aee904f3dd6fe245fc59d35618d2b2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 16:08:10 -0400 Subject: [PATCH 1629/8313] bugfix --- doc/special_remotes/hook.mdwn | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 521b440277..7bdd317b98 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -10,11 +10,11 @@ Here's a simple example that stores content on clay tablets. If you implement this example in the real world, I'd appreciate a tour next Apert! :) --[[Joey]] - # git config annex.cuneiform-store-hook 'tocuneiform < $ANNEX_FILE | tablet-writer --implement=stylus --title=$ANNEX_KEY | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' - # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > $ANNEX_FILE' - # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY' - # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title=$ANNEX_KEY --shout-title' - # git annex initremote special type=hook hooktype=cuneiform encryption=none + # git config annex.cuneiform-store-hook 'tocuneiform < "$ANNEX_FILE" | tablet-writer --implement=stylus --title="$ANNEX_KEY" | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' + # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > "$ANNEX_FILE"' + # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY"' + # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" --shout-title' + # git annex initremote library type=hook hooktype=cuneiform encryption=none Can you spot the potential data loss bugs in the above simple example? (Hint: What happens when the `tablet-proofreader` exits nonzero?) From b5072b7b4cab21118f60c55a58497f363f749244 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 16:08:18 -0400 Subject: [PATCH 1630/8313] add boolSystemEnv --- Utility.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Utility.hs b/Utility.hs index 51bbc17a32..0dab371045 100644 --- a/Utility.hs +++ b/Utility.hs @@ -16,6 +16,7 @@ module Utility ( relPathCwdToFile, relPathDirToFile, boolSystem, + boolSystemEnv, shellEscape, shellUnEscape, unsetFileMode, @@ -73,7 +74,10 @@ toCommand l = concat $ map unwrap l - SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} boolSystem :: FilePath -> [CommandParam] -> IO Bool -boolSystem command params = do +boolSystem command params = boolSystemEnv command params Nothing + +boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool +boolSystemEnv command params env = do -- Going low-level because all the high-level system functions -- block SIGINT etc. We need to block SIGCHLD, but allow -- SIGINT to do its default program termination. @@ -93,7 +97,7 @@ boolSystem command params = do setSignalMask oldset childaction oldint oldset = do restoresignals oldint oldset - executeFile command True (toCommand params) Nothing + executeFile command True (toCommand params) env {- Escapes a filename or other parameter to be safely able to be exposed to - the shell. -} From 3ab3f41aea78f6816493d094d2daca7cc0067a91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 17:21:45 -0400 Subject: [PATCH 1631/8313] hook special remote implemented, and tested --- Remote.hs | 2 + Remote/Hook.hs | 157 ++++++++++++++++++++++++++++++++++ doc/special_remotes/hook.mdwn | 3 +- 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 Remote/Hook.hs diff --git a/Remote.hs b/Remote.hs index f47bea560b..bbecdb9995 100644 --- a/Remote.hs +++ b/Remote.hs @@ -49,6 +49,7 @@ import qualified Remote.S3 import qualified Remote.Bup import qualified Remote.Directory import qualified Remote.Rsync +import qualified Remote.Hook remoteTypes :: [RemoteType Annex] remoteTypes = @@ -57,6 +58,7 @@ remoteTypes = , Remote.Bup.remote , Remote.Directory.remote , Remote.Rsync.remote + , Remote.Hook.remote ] {- Builds a list of all available Remotes. diff --git a/Remote/Hook.hs b/Remote/Hook.hs new file mode 100644 index 0000000000..2613fda7a6 --- /dev/null +++ b/Remote/Hook.hs @@ -0,0 +1,157 @@ +{- A remote that provides hooks to run shell commands. + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Remote.Hook (remote) where + +import qualified Data.ByteString.Lazy.Char8 as L +import Control.Exception.Extensible (IOException) +import qualified Data.Map as M +import Control.Monad.State (liftIO) +import System.FilePath +import System.Posix.Process +import System.Posix.IO +import System.IO +import System.IO.Error (try) +import System.Exit + +import RemoteClass +import Types +import qualified GitRepo as Git +import qualified Annex +import UUID +import Locations +import Config +import Content +import Utility +import Remote.Special +import Remote.Encryptable +import Crypto +import Messages + +remote :: RemoteType Annex +remote = RemoteType { + typename = "hook", + enumerate = findSpecialRemotes "hooktype", + generate = gen, + setup = hookSetup +} + +gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) +gen r u c = do + hooktype <- getConfig r "hooktype" (error "missing hooktype") + cst <- remoteCost r expensiveRemoteCost + return $ encryptableRemote c + (storeEncrypted hooktype) + (retrieveEncrypted hooktype) + Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = store hooktype, + retrieveKeyFile = retrieve hooktype, + removeKey = remove hooktype, + hasKey = checkPresent r hooktype, + hasKeyCheap = False, + config = Nothing + } + +hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig +hookSetup u c = do + let hooktype = case M.lookup "hooktype" c of + Nothing -> error "Specify hooktype=" + Just r -> r + c' <- encryptionSetup c + gitConfigSpecialRemote u c' "hooktype" hooktype + return c' + +hookEnv :: Key -> Maybe FilePath -> Maybe [(String, String)] +hookEnv k f = Just $ keyenv : fileenv f + where + env s v = ("ANNEX_" ++ s, v) + keyenv = env "KEY" (show k) + fileenv Nothing = [] + fileenv (Just file) = + [ env "FILE" file + , env "HASH_1" (hashbits !! 0) + , env "HASH_2" (hashbits !! 1) + ] + hashbits = map takeDirectory $ splitPath $ hashDirMixed k + +lookupHook :: String -> String -> Annex (Maybe String) +lookupHook hooktype hook =do + g <- Annex.gitRepo + command <- getConfig g hookname "" + if null command + then do + warning $ "missing configuration for " ++ hookname + return Nothing + else return $ Just command + where + hookname = hooktype ++ "-" ++ hook ++ "-hook" + +runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool +runHook hooktype hook k f a = do + command <- lookupHook hooktype hook + case command of + Nothing -> return False + Just c -> do + showProgress -- make way for hook output + res <- liftIO $ boolSystemEnv + "sh" [Param "-c", Param c] $ hookEnv k f + if res + then a + else do + warning $ hook ++ " hook exited nonzero!" + return res + +store :: String -> Key -> Annex Bool +store h k = do + g <- Annex.gitRepo + runHook h "store" k (Just $ gitAnnexLocation g k) $ return True + +storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool +storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do + g <- Annex.gitRepo + let f = gitAnnexLocation g k + liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s + runHook h "store" enck (Just tmp) $ return True + +retrieve :: String -> Key -> FilePath -> Annex Bool +retrieve h k f = runHook h "retrieve" k (Just f) $ return True + +retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool +retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp -> + runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBool $ do + withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f + return True + +remove :: String -> Key -> Annex Bool +remove h k = runHook h "remove" k Nothing $ do return True + +checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool) +checkPresent r h k = do + showNote ("checking " ++ Git.repoDescribe r ++ "...") + v <- lookupHook h "checkpresent" + liftIO (try (check v) ::IO (Either IOException Bool)) + where + findkey s = (show k) `elem` (lines s) + env = hookEnv k Nothing + check Nothing = error "checkpresent hook misconfigured" + check (Just hook) = do + (frompipe, topipe) <- createPipe + pid <- forkProcess $ do + _ <- dupTo topipe stdOutput + closeFd frompipe + executeFile "sh" True ["-c", hook] env + closeFd topipe + fromh <- fdToHandle frompipe + reply <- hGetContentsStrict fromh + hClose fromh + s <- getProcessStatus True False pid + case s of + Just (Exited (ExitSuccess)) -> return $ findkey reply + _ -> error "checkpresent hook failed" diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 7bdd317b98..74c4029cdd 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -1,4 +1,5 @@ -This special remote type runs hooks that you configure to store content. +This special remote lets you store content in a remote of your own +devising. It's not recommended to use this remote type when another like [[rsync]] or [[directory]] will do. If your hooks are not carefully written, data From 847bbe30b1a00ec5ef419895f96ab5d7cb767a83 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 28 Apr 2011 21:22:04 +0000 Subject: [PATCH 1632/8313] Added a comment --- .../comment_2_84e4414c88ae91c048564a2cdc2d3250._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment new file mode 100644 index 0000000000..6243708f94 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-28T21:22:03Z" + content=""" +Ask and ye shalle receive with an Abbot on top: [[special_remotes/hook]] +"""]] From b0efff86b688e6ea68921221ac56e02a838d2c51 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 17:28:38 -0400 Subject: [PATCH 1633/8313] wording --- doc/special_remotes/hook.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 74c4029cdd..502375fb94 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -1,4 +1,4 @@ -This special remote lets you store content in a remote of your own +This special remote type lets you store content in a remote of your own devising. It's not recommended to use this remote type when another like [[rsync]] From 8b950c5fe8a89bcdefe65ac709e2d3640eb03b1a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Thu, 28 Apr 2011 23:38:36 +0000 Subject: [PATCH 1634/8313] This demonstrates a git-annex failure when hard links are involved --- ...ateSymbolicLink:_already_exists__34__.mdwn | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn diff --git a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn new file mode 100644 index 0000000000..70881fcc84 --- /dev/null +++ b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -0,0 +1,35 @@ +I'm importing a directory where some files are hard links of each other. + +This is confusing git-annex. Here's a small test of that: + +
+paulproteus@pathi:/tmp$ mkdir annex-test
+paulproteus@pathi:/tmp$ cd annex-test
+paulproteus@pathi:/tmp/annex-test$ git init
+Initialized empty Git repository in /tmp/annex-test/.git/
+paulproteus@pathi:/tmp/annex-test$ git annex init testing
+init testing ok
+paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
+paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
+[master dd54b41] Default to sha1
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
+paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
+paulproteus@pathi:/tmp/annex-test$ git annex add file1
+add file1 (checksum...) ok
+(Recording state in git...)
+paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
+[master eb43084] So far, so good
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
+ create mode 120000 file1
+paulproteus@pathi:/tmp/annex-test$ git annex add file2
+add file2 (checksum...) 
+  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
+git-annex: 1 failed
+paulproteus@pathi:/tmp/annex-test$ 
+
+ +I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + +-- Asheesh. From b8e114bce153875db8afd20182eae03e35662737 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Thu, 28 Apr 2011 23:39:28 +0000 Subject: [PATCH 1635/8313] --- ..._a_file___34__createSymbolicLink:_already_exists__34__.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn index 70881fcc84..6bcd2fc4c7 100644 --- a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn +++ b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -30,6 +30,9 @@ git-annex: 1 failed paulproteus@pathi:/tmp/annex-test$ +When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. + I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + -- Asheesh. From eef3f634e9f92e7af486e5ee4afdac9a79b034cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 20:41:40 -0400 Subject: [PATCH 1636/8313] Avoid crashing when an existing key is readded to the annex. --- Content.hs | 37 ++++++++++++--- debian/changelog | 1 + ...ateSymbolicLink:_already_exists__34__.mdwn | 46 +++++++++++++++++++ ...ateSymbolicLink:_already_exists__34__.mdwn | 39 +--------------- 4 files changed, 78 insertions(+), 45 deletions(-) create mode 100644 doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn diff --git a/Content.hs b/Content.hs index 99770f553d..ade936da37 100644 --- a/Content.hs +++ b/Content.hs @@ -182,18 +182,41 @@ allowWrite f = do s <- getFileStatus f setFileMode f $ fileMode s `unionFileModes` ownerWriteMode -{- Moves a file into .git/annex/objects/ -} +{- Moves a file into .git/annex/objects/ + - + - What if the key there already has content? This could happen for + - various reasons; perhaps the same content is being annexed again. + - Perhaps there has been a hash collision generating the keys. + - + - The current strategy is to assume that in this case it's safe to delete + - one of the two copies of the content; and the one already in the annex + - is left there, assuming it's the original, canonical copy. + - + - I considered being more paranoid, and checking that both files had + - the same content. Decided against it because A) users explicitly choose + - a backend based on its hashing properties and so if they're dealing + - with colliding files it's their own fault and B) adding such a check + - would not catch all cases of colliding keys. For example, perhaps + - a remote has a key; if it's then added again with different content then + - the overall system now has two different peices of content for that + - key, and one of them will probably get deleted later. So, adding the + - check here would only raise expectations that git-annex cannot truely + - meet. + -} moveAnnex :: Key -> FilePath -> Annex () moveAnnex key src = do g <- Annex.gitRepo let dest = gitAnnexLocation g key let dir = parentDir dest - liftIO $ do - createDirectoryIfMissing True dir - allowWrite dir -- in case the directory already exists - renameFile src dest - preventWrite dest - preventWrite dir + e <- liftIO $ doesFileExist dest + if e + then liftIO $ removeFile src + else liftIO $ do + createDirectoryIfMissing True dir + allowWrite dir -- in case the directory already exists + renameFile src dest + preventWrite dest + preventWrite dir {- Removes a key's file from .git/annex/objects/ -} removeAnnex :: Key -> Annex () diff --git a/debian/changelog b/debian/changelog index b7f33a7041..92c05a5a69 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. * Add hook special remotes. + * Avoid crashing when an existing key is readded to the annex. -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn new file mode 100644 index 0000000000..e943127740 --- /dev/null +++ b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -0,0 +1,46 @@ +I'm importing a directory where some files are hard links of each other. + +This is confusing git-annex. Here's a small test of that: + +
+paulproteus@pathi:/tmp$ mkdir annex-test
+paulproteus@pathi:/tmp$ cd annex-test
+paulproteus@pathi:/tmp/annex-test$ git init
+Initialized empty Git repository in /tmp/annex-test/.git/
+paulproteus@pathi:/tmp/annex-test$ git annex init testing
+init testing ok
+paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
+paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
+[master dd54b41] Default to sha1
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
+paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
+paulproteus@pathi:/tmp/annex-test$ git annex add file1
+add file1 (checksum...) ok
+(Recording state in git...)
+paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
+[master eb43084] So far, so good
+ 2 files changed, 2 insertions(+), 0 deletions(-)
+ create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
+ create mode 120000 file1
+paulproteus@pathi:/tmp/annex-test$ git annex add file2
+add file2 (checksum...) 
+  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
+git-annex: 1 failed
+paulproteus@pathi:/tmp/annex-test$ 
+
+ +When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. + +I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. + + +-- Asheesh. + +> The same thing happens anytime the key for a file collides with a key +> already in the annex, AFAICS. Including when the files have the same +> content but are not hard links. +> +> I've fixed this bug. The first file in wins. See commit for some +> interesting discussion about why it should not check for hash collisions +> in this situation. [[done]] --[[Joey]] diff --git a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn index 6bcd2fc4c7..38865f49aa 100644 --- a/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn +++ b/doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -1,38 +1 @@ -I'm importing a directory where some files are hard links of each other. - -This is confusing git-annex. Here's a small test of that: - -
-paulproteus@pathi:/tmp$ mkdir annex-test
-paulproteus@pathi:/tmp$ cd annex-test
-paulproteus@pathi:/tmp/annex-test$ git init
-Initialized empty Git repository in /tmp/annex-test/.git/
-paulproteus@pathi:/tmp/annex-test$ git annex init testing
-init testing ok
-paulproteus@pathi:/tmp/annex-test$ echo '* annex.backend=SHA1' >> .gitattributes 
-paulproteus@pathi:/tmp/annex-test$ git commit .gitattributes -m 'Default to sha1'
-[master dd54b41] Default to sha1
- 1 files changed, 1 insertions(+), 0 deletions(-)
-paulproteus@pathi:/tmp/annex-test$ echo "Look at me" > file1
-paulproteus@pathi:/tmp/annex-test$ cp -l file1 file2
-paulproteus@pathi:/tmp/annex-test$ git annex add file1
-add file1 (checksum...) ok
-(Recording state in git...)
-paulproteus@pathi:/tmp/annex-test$ git commit -m 'So far, so good'
-[master eb43084] So far, so good
- 2 files changed, 2 insertions(+), 0 deletions(-)
- create mode 100644 .git-annex/9a3/f1f/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c.log
- create mode 120000 file1
-paulproteus@pathi:/tmp/annex-test$ git annex add file2
-add file2 (checksum...) 
-  git-annex: .git/annex/objects/PM/7p/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c/SHA1-s11--b9c599d64212934582d676c722cf3ec61f60e09c: createSymbolicLink: already exists (File exists)
-git-annex: 1 failed
-paulproteus@pathi:/tmp/annex-test$ 
-
- -When trying to make a small test case for this bug, I noticed that if file1 and file2 have the same contents but are not hard links of each other, they both get annexed just fine. - -I think the right behavior here is to annex file2 just fine, as if they weren't hard links before. - - --- Asheesh. +Moved to [[bugs|bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__]] --[[Joey]] From 8fcac59852e28c36c4512d5ee0a32ed76774aa2d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 20:44:08 -0400 Subject: [PATCH 1637/8313] closing bug my explanation seems to have sufficed --- .../git_annex_migrate_leaves_old_backend_versions_around.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn index e37ee06bb3..7f586b5ff0 100644 --- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn +++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn @@ -16,3 +16,4 @@ would be great if these were purged automatically somehow. > > This way a lot of migrations can be done, and only when you're done you > can do the more expensive cleanup pass if you want to. --[[Joey]] +> [[done]] From 6e95521b85263d9b734fa3f77030a96f850b1e1e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Apr 2011 20:47:36 -0400 Subject: [PATCH 1638/8313] tweak --- ...a_file___34__createSymbolicLink:_already_exists__34__.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn index e943127740..21293af547 100644 --- a/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn +++ b/doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn @@ -38,8 +38,8 @@ I think the right behavior here is to annex file2 just fine, as if they weren't -- Asheesh. > The same thing happens anytime the key for a file collides with a key -> already in the annex, AFAICS. Including when the files have the same -> content but are not hard links. +> already in the annex, AFAICS. (Including when the files have the same +> content but are not hard links... unless you're using WORM backend.) > > I've fixed this bug. The first file in wins. See commit for some > interesting discussion about why it should not check for hash collisions From f581481a75d6f098b622f2079aa183e06c1273fc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:33:27 +0000 Subject: [PATCH 1639/8313] --- ...95__remotes__47__hook_with_tahoe-lafs.mdwn | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn new file mode 100644 index 0000000000..2d04a46946 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -0,0 +1,20 @@ +This is work in progress, since there is now a "hook" for users to plug in whatever they want on the back end, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file + + tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY + tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE + tahoe-remove-hook = tahoe rm tahoe:$ANNEX_KEY + tahoe-checkpresent-hook = tahoe ls tahoe:$ANNEX_KEY 2>&1 || echo FAIL + +The only quirk I've noticed is this... + +
+$ git annex whereis .
+whereis frink.jar (2 copies) 
+  	084603a8-7243-11e0-b1f5-83102bcd7953  -- testtest <-- here
+   	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a
+ok
+
+ +1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup. + +Other than the quirks above, I am able to put and get files from my tahoe-lafs remote. The only thing that I have not figured out is how to "remove a file" on the remote to free up space on the remote. From 66dbf7799f38e0191f317b51fedca6a0f7ee7c3e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:34:02 +0000 Subject: [PATCH 1640/8313] --- .../tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn index 2d04a46946..a1bf942546 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -1,4 +1,4 @@ -This is work in progress, since there is now a "hook" for users to plug in whatever they want on the back end, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file +This is work in progress, since there is now a "hook" for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE From 05fe77c110b0b8f4571380c01bbbd09201a099c0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:36:37 +0000 Subject: [PATCH 1641/8313] --- .../tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn index a1bf942546..50b3078b2d 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -5,6 +5,8 @@ This is work in progress, since there is now a "hook" for users to plug in whate tahoe-remove-hook = tahoe rm tahoe:$ANNEX_KEY tahoe-checkpresent-hook = tahoe ls tahoe:$ANNEX_KEY 2>&1 || echo FAIL +Where `tahoe:` is a tahoe-lafs alias, ideally you should create a new alias (DIR-CAP or whatever the terminolgy is) to store your files, I just used the default `tahoe:` alias for testing. + The only quirk I've noticed is this... 
@@ -15,6 +17,6 @@ whereis frink.jar (2 copies)
 ok
-1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup. +1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a is my [[!google tahoe-lafs]] remote, but there is no label/description on it. The checkpresent-hook was a little confusing when I was setting it up, I'm currently unsure if I am doing the right thing or not with my hook. My get and put commands are a little verbose for now, i might redirect it to /dev/null once I am happier with the overall performance/behaviour my setup. Other than the quirks above, I am able to put and get files from my tahoe-lafs remote. The only thing that I have not figured out is how to "remove a file" on the remote to free up space on the remote. From 75f902844671f363a9f934e760226ff8815a9e12 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:37:17 +0000 Subject: [PATCH 1642/8313] --- .../tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn index 50b3078b2d..4f5f089a89 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn @@ -1,4 +1,4 @@ -This is work in progress, since there is now a "hook" for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file +This is work in progress, since there is now a [[special_remotes/hook]] for users to plug in whatever they want as a remote, here's my recipe for using tahoe-lafs as a remote, this is a copy and paste the relavent section from my .git/config file tahoe-store-hook = tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY tahoe-retrieve-hook = tahoe get tahoe:$ANNEX_KEY $ANNEX_FILE From db2786bf0ec357ebb4d2b5b694a023ac6f495632 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 10:43:31 +0000 Subject: [PATCH 1643/8313] Added a comment --- .../comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment new file mode 100644 index 0000000000..dc21ec4885 --- /dev/null +++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-29T10:43:31Z" + content=""" +Cool!, I just tried adding tahoe-lafs as a remote, and it wasn't too hard. +"""]] From 9b133a81221cdca4f880d200906e75dd1319370d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Fri, 29 Apr 2011 11:48:23 +0000 Subject: [PATCH 1644/8313] Added a comment: Duplication of the filenames is what I am concerned about --- ...3_076cb22057583957d5179d8ba9004605._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment new file mode 100644 index 0000000000..d11119bc3d --- /dev/null +++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" + nickname="Asheesh" + subject="Duplication of the filenames is what I am concerned about" + date="2011-04-29T11:48:22Z" + content=""" +For what it's worth, yes, I want to actually forget I ever had the same file in the filesystem with a duplicated name. I'm not just aiming to clean up the disk's space usage; I'm also aiming to clean things up so that navigating the filesystem is easier. + +I can write my own script to do that based on the symlinks' target (and I wrote something along those lines), but I still think it'd be nicer if git-annex supported this use case. + +Perhaps: + +
git annex drop --by-contents
+ +could let me remove a file from git-annex if the contents are available through a different name. (Right now, \"git annex drop\" requires the name *and* contents match.) + +-- Asheesh. +"""]] From 446585351b0ca0590715a4cab1711425c9767344 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Fri, 29 Apr 2011 13:08:36 +0000 Subject: [PATCH 1645/8313] Added a comment: whereis labels --- ...comment_1_76bb33ce45ce6a91b86454147463193b._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment new file mode 100644 index 0000000000..388641f69e --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="whereis labels" + date="2011-04-29T13:08:35Z" + content=""" +You should be able to fix the missing label by editing .git-annex/uuid.log and adding + + 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a tahoe +"""]] From eca02546286d881bb6ff71d9246a21c79e8679a3 Mon Sep 17 00:00:00 2001 From: "http://christian.amsuess.com/chrysn" Date: Fri, 29 Apr 2011 14:48:08 +0000 Subject: [PATCH 1646/8313] no way the goon would be admitted in the library; he doesn't understand the floor/shelf system anyway --- doc/special_remotes/hook.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 502375fb94..e3b70d2026 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -13,7 +13,7 @@ next Apert! :) --[[Joey]] # git config annex.cuneiform-store-hook 'tocuneiform < "$ANNEX_FILE" | tablet-writer --implement=stylus --title="$ANNEX_KEY" | tablet-proofreader | librarian --shelve --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2' # git config annex.cuneiform-retrieve-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | tablet-reader --implement=coffee --implement=glasses --force-monastic-dedication | fromcuneiform > "$ANNEX_FILE"' - # git config annex.cuneiform-remove-hook 'goon --hit-with-hammer --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY"' + # git config annex.cuneiform-remove-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | goon --hit-with-hammer' # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" --shout-title' # git annex initremote library type=hook hooktype=cuneiform encryption=none From 0ea7f966dc61948d107372fac6f2ec90a1dc28f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 11:16:54 -0400 Subject: [PATCH 1647/8313] add git annex describe to special remote setup examples --- doc/special_remotes/directory.mdwn | 1 + doc/special_remotes/hook.mdwn | 1 + doc/special_remotes/rsync.mdwn | 2 ++ 3 files changed, 4 insertions(+) diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn index 9e4bfa33bd..0a38c763cc 100644 --- a/doc/special_remotes/directory.mdwn +++ b/doc/special_remotes/directory.mdwn @@ -8,3 +8,4 @@ the drive's mountpoint as a directory remote. Setup example: # git annex initremote usbdrive type=directory directory=/media/usbdrive/ encryption=none + # git annex describe usbdrive "usb drive on /media/usbdrive/" diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index e3b70d2026..5b636613c9 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -16,6 +16,7 @@ next Apert! :) --[[Joey]] # git config annex.cuneiform-remove-hook 'librarian --get --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" | goon --hit-with-hammer' # git config annex.cuneiform-checkpresent-hook 'librarian --find --force-distrust-catalog --floor=$ANNEX_HASH_1 --shelf=$ANNEX_HASH_2 --title="$ANNEX_KEY" --shout-title' # git annex initremote library type=hook hooktype=cuneiform encryption=none + # git annex describe library "the reborn Library of Alexandria (upgrade to bronze plates pending)" Can you spot the potential data loss bugs in the above simple example? (Hint: What happens when the `tablet-proofreader` exits nonzero?) diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn index 7a7f3ab149..90d544a1e1 100644 --- a/doc/special_remotes/rsync.mdwn +++ b/doc/special_remotes/rsync.mdwn @@ -3,10 +3,12 @@ This special remote type rsyncs file contents to somewhere else. Setup example: # git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=joey@kitenet.net + # git annex describe myrsync "rsync server" Or for using rsync over SSH # git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=joey@kitenet.net + # git annex describe myrsync "rsync server" ## configuration From a83adb0acddba7accb3c7f30aaa4f084a28b632b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 11:22:55 -0400 Subject: [PATCH 1648/8313] note number of hash buckets --- doc/special_remotes/hook.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn index 5b636613c9..9a7dbf7a19 100644 --- a/doc/special_remotes/hook.mdwn +++ b/doc/special_remotes/hook.mdwn @@ -45,6 +45,7 @@ These environment variables are used to communicate with the hook commands: * `ANNEX_KEY` - name of a key to store, retrieve, remove, or check. * `ANNEX_FILE` - a file containing the key's content * `ANNEX_HASH_1` - short stable value, based on the key, can be used for hashing + into 1024 buckets. * `ANNEX_HASH_2` - another hash value, can be used for a second level of hashing The setting to use in git config for the hook commands are as follows: From 15711872f7e9e9b0b3267f1a58592b48bd94f93b Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 29 Apr 2011 15:24:56 +0000 Subject: [PATCH 1649/8313] Added a comment --- ...comment_2_4d9b9d47d01d606a475678f630797bf9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment new file mode 100644 index 0000000000..e7c3d619dd --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-04-29T15:24:56Z" + content=""" +If `tahoe ls` outputs only the key, on its own line, and exits nonzero if it's not present, then I think you did the right thing. + +To remove a file, use `git annex move file --from tahoe` and then you can drop it locally. +"""]] From 4a13bdc3df8f0a66885157813d1090d6008a0b6f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 15:33:54 +0000 Subject: [PATCH 1650/8313] Added a comment --- ...mment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment new file mode 100644 index 0000000000..16ad9e9886 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 3" + date="2011-04-29T15:33:24Z" + content=""" +@justin, I discovered that \"git annex describe\" did what I wanted + +@joey, yep that is the behaviour of \"tahoe ls\", thanks for the tip on removing the file from the remote. + +It seems to be working okay for now, the only concern is that on the remote everything is dumped into the same directory, but I can live with that, since I want to track biggish blobs and not lots of small little files. +"""]] From 976ed575d5dcfc08ce8a43f43732b0bc9dd0d8f8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 16:17:23 +0000 Subject: [PATCH 1651/8313] Added a comment --- ..._fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment new file mode 100644 index 0000000000..5d271c6f3c --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 4" + date="2011-04-29T16:17:11Z" + content=""" +I've just tried to use the ANNEX_HASH_ variables, example of my configuration + +
+    git config annex.tahoe-store-hook 'tahoe mkdir $ANNEX_HASH_1 && tahoe put $ANNEX_FILE tahoe:$ANNEX_HASH_1/$ANNEX_KEY'
+    git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_KEY $ANNEX_FILE'
+    git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_KEY'
+    git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_KEY 2>&1 || echo FAIL'
+    git annex initremote library type=hook hooktype=tahoe encryption=none
+    git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
+
+ +It's seems to work quite well for me now, I did run across this when I tried to drop a file locally, leaving the file on my remote + +
+jtang@x00:/tmp/annex3 $ git annex drop .
+drop frink.sh (checking library...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+  Try making some of these repositories available:
+  	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a  -- library
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+drop t/frink.jar (checking library...) (unsafe) 
+  Could only verify the existence of 0 out of 1 necessary copies
+  Try making some of these repositories available:
+  	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a  -- library
+  (Use --force to override this check, or adjust annex.numcopies.)
+failed
+git-annex: 2 failed
+1|jtang@x00:/tmp/annex3 $ 
+
+ +I do know that the files exist in my library as I have just inserted them, it seemed to work when I didnt have the hashing, it appears that the checkpresent doesn't seem to pass the ANNEX_HASH_* variables (from the limited debugging I did) +"""]] From 5d8e0d5a1c09b95043d02d7517f1e00604b37244 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 12:20:59 -0400 Subject: [PATCH 1652/8313] remove unused file --- Backend/SHA256.hs | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 Backend/SHA256.hs diff --git a/Backend/SHA256.hs b/Backend/SHA256.hs deleted file mode 100644 index 42d3d48c7d..0000000000 --- a/Backend/SHA256.hs +++ /dev/null @@ -1,14 +0,0 @@ -{- git-annex "SHA256" backend - - - - Copyright 2011 Joey Hess - - - - Licensed under the GNU GPL version 3 or higher. - -} - -module Backend.SHA256 (backend) where - -import Types -import Backend.SHA - -backend :: Backend Annex -backend = genBackend 256 From 49efc6c39928baec03d7dd0d5cb37f346432f1d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 13:12:26 -0400 Subject: [PATCH 1653/8313] add -fspec-constr-count workaround ghc 7 produces these warnings http://hackage.haskell.org/trac/ghc/ticket/4288 The specialization is enabled by -O2, and the default limit of 3 is there to avoid specialization blowing up binary size. Perhaps that default is a little low? I needed 4 to avoid a warning on Unused.hs, and 5 to avoid warnings on test.hs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 69fb518074..01a1a6a54c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr IGNORE=-ignore-package monads-fd -GHCFLAGS=-O2 -Wall $(IGNORE) +GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=5 ifdef PROFILE GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE) endif From 43f0a666f0f6cc152a2b778921831d6d7daedcaf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 13:59:00 -0400 Subject: [PATCH 1654/8313] unused: Now also lists files fsck places in .git/annex/bad/ --- Command/DropUnused.hs | 78 ++++++++++++++++++-------------- Command/Unused.hs | 100 +++++++++++++++++++++++++----------------- Locations.hs | 11 +++-- debian/changelog | 1 + doc/git-annex.mdwn | 2 +- 5 files changed, 114 insertions(+), 78 deletions(-) diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs index 1eec688202..b129235e1d 100644 --- a/Command/DropUnused.hs +++ b/Command/DropUnused.hs @@ -21,54 +21,66 @@ import qualified Annex import qualified Command.Drop import qualified Command.Move import qualified Remote +import qualified GitRepo as Git import Backend import Key +type UnusedMap = M.Map String Key + command :: [Command] command = [repoCommand "dropunused" (paramRepeating paramNumber) seek "drop unused file content"] seek :: [CommandSeek] -seek = [withUnusedMap] +seek = [withUnusedMaps] -{- Read unusedlog once, and pass the map to each start action. -} -withUnusedMap :: CommandSeek -withUnusedMap params = do - m <- readUnusedLog - return $ map (start m) params +{- Read unused logs once, and pass the maps to each start action. -} +withUnusedMaps :: CommandSeek +withUnusedMaps params = do + unused <- readUnusedLog "" + unusedbad <- readUnusedLog "bad" + unusedtmp <- readUnusedLog "tmp" + return $ map (start (unused, unusedbad, unusedtmp)) params -start :: M.Map String Key -> CommandStartString -start m s = notBareRepo $ do - case M.lookup s m of - Nothing -> return Nothing - Just key -> do - showStart "dropunused" s - from <- Annex.getState Annex.fromremote - case from of - Just name -> do - r <- Remote.byName name - return $ Just $ performRemote r key - _ -> return $ Just $ perform key +start :: (UnusedMap, UnusedMap, UnusedMap) -> CommandStartString +start (unused, unusedbad, unusedtmp) s = notBareRepo $ search + [ (unused, perform) + , (unusedbad, performOther gitAnnexBadLocation) + , (unusedtmp, performOther gitAnnexTmpLocation) + ] + where + search [] = return Nothing + search ((m, a):rest) = do + case M.lookup s m of + Nothing -> search rest + Just key -> do + showStart "dropunused" s + return $ Just $ a key -{- drop both content in the backend and any tmp file for the key -} perform :: Key -> CommandPerform perform key = do - g <- Annex.gitRepo - let tmp = gitAnnexTmpLocation g key - tmp_exists <- liftIO $ doesFileExist tmp - when tmp_exists $ liftIO $ removeFile tmp - backend <- keyBackend key - Command.Drop.perform key backend (Just 0) -- force drop + from <- Annex.getState Annex.fromremote + case from of + Just name -> do + r <- Remote.byName name + showNote $ "from " ++ Remote.name r ++ "..." + return $ Just $ Command.Move.fromCleanup r True key + _ -> do + backend <- keyBackend key + Command.Drop.perform key backend (Just 0) -- force drop -performRemote :: Remote.Remote Annex -> Key -> CommandPerform -performRemote r key = do - showNote $ "from " ++ Remote.name r ++ "..." - return $ Just $ Command.Move.fromCleanup r True key - -readUnusedLog :: Annex (M.Map String Key) -readUnusedLog = do +performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform +performOther filespec key = do g <- Annex.gitRepo - let f = gitAnnexUnusedLog g + let f = filespec g key + e <- liftIO $ doesFileExist f + when e $ liftIO $ removeFile f + return $ Just $ return True + +readUnusedLog :: FilePath -> Annex UnusedMap +readUnusedLog prefix = do + g <- Annex.gitRepo + let f = gitAnnexUnusedLog prefix g e <- liftIO $ doesFileExist f if e then do diff --git a/Command/Unused.hs b/Command/Unused.hs index a3fb6fe232..67f10581d6 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -7,7 +7,7 @@ module Command.Unused where -import Control.Monad (filterM, unless, forM_) +import Control.Monad (filterM, unless, forM_, when) import Control.Monad.State (liftIO) import qualified Data.Set as S import Data.Maybe @@ -51,14 +51,17 @@ perform = do checkUnused :: Annex () checkUnused = do - (unused, staletmp) <- unusedKeys - let unusedlist = number 0 unused - let staletmplist = number (length unused) staletmp - let list = unusedlist ++ staletmplist - writeUnusedFile list - unless (null unused) $ showLongNote $ unusedMsg unusedlist - unless (null staletmp) $ showLongNote $ staleTmpMsg staletmplist - unless (null list) $ showLongNote $ "\n" + (unused, stalebad, staletmp) <- unusedKeys + n <- list "" unusedMsg unused 0 + n' <- list "bad" staleBadMsg stalebad n + _ <- list "tmp" staleTmpMsg staletmp n' + return () + where + list file msg l c = do + let unusedlist = number c l + when (not $ null l) $ showLongNote $ msg unusedlist + writeUnusedFile file unusedlist + return $ length l checkRemoteUnused :: Remote.Remote Annex -> Annex () checkRemoteUnused r = do @@ -69,7 +72,7 @@ checkRemoteUnused r = do remotehas <- filterM isthere logged let remoteunused = remotehas `exclude` referenced let list = number 0 remoteunused - writeUnusedFile list + writeUnusedFile "" list unless (null remoteunused) $ do showLongNote $ remoteUnusedMsg r list showLongNote $ "\n" @@ -80,10 +83,10 @@ checkRemoteUnused r = do return $ uuid `elem` us uuid = Remote.uuid r -writeUnusedFile :: [(Int, Key)] -> Annex () -writeUnusedFile l = do +writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex () +writeUnusedFile prefix l = do g <- Annex.gitRepo - liftIO $ safeWriteFile (gitAnnexUnusedLog g) $ + liftIO $ safeWriteFile (gitAnnexUnusedLog prefix g) $ unlines $ map (\(n, k) -> show n ++ " " ++ show k) l table :: [(Int, Key)] -> [String] @@ -100,7 +103,12 @@ staleTmpMsg :: [(Int, Key)] -> String staleTmpMsg t = unlines $ ["Some partially transferred data exists in temporary files:"] ++ table t ++ [dropMsg Nothing] - + +staleBadMsg :: [(Int, Key)] -> String +staleBadMsg t = unlines $ + ["Some corrupted files have been preserved by fsck, just in case:"] + ++ table t ++ [dropMsg Nothing] + unusedMsg :: [(Int, Key)] -> String unusedMsg u = unusedMsg' u ["Some annexed data is no longer used by any files in the repository:"] @@ -127,36 +135,28 @@ dropMsg :: Maybe (Remote.Remote Annex) -> String dropMsg Nothing = dropMsg' "" dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r dropMsg' :: String -> String -dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)" +dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)\n" {- Finds keys whose content is present, but that do not seem to be used - - by any files in the git repo, or that are only present as tmp files. -} -unusedKeys :: Annex ([Key], [Key]) + - by any files in the git repo, or that are only present as bad or tmp + - files. -} +unusedKeys :: Annex ([Key], [Key], [Key]) unusedKeys = do - g <- Annex.gitRepo - fast <- Annex.getState Annex.fast if fast then do - showNote "fast mode enabled; only finding temporary files" - tmps <- tmpKeys - return ([], tmps) + showNote "fast mode enabled; only finding stale files" + tmp <- staleKeys' gitAnnexTmpDir + bad <- staleKeys' gitAnnexBadDir + return ([], bad, tmp) else do showNote "checking for unused data..." present <- getKeysPresent referenced <- getKeysReferenced - tmps <- tmpKeys - let unused = present `exclude` referenced - let staletmp = tmps `exclude` present - let duptmp = tmps `exclude` staletmp - - -- Tmp files that are dups of content already present - -- can simply be removed. - liftIO $ forM_ duptmp $ \t -> removeFile $ - gitAnnexTmpLocation g t - - return (unused, staletmp) + staletmp <- staleKeys gitAnnexTmpDir present + stalebad <- staleKeys gitAnnexBadDir present + return (unused, stalebad, staletmp) {- Finds items in the first, smaller list, that are not - present in the second, larger list. @@ -178,16 +178,34 @@ getKeysReferenced = do keypairs <- mapM Backend.lookupFile files return $ map fst $ catMaybes keypairs -{- List of keys that have temp files in the git repo. -} -tmpKeys :: Annex [Key] -tmpKeys = do +{- Looks in the specified directory for bad/tmp keys, and returns a list + - of those that might still have value, or might be stale and removable. + - + - When a list of presently available keys is provided, stale keys + - that no longer have value are deleted. + -} +staleKeys :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key] +staleKeys dirspec present = do + contents <- staleKeys' dirspec + + let stale = contents `exclude` present + let dup = contents `exclude` stale + g <- Annex.gitRepo - let tmp = gitAnnexTmpDir g - exists <- liftIO $ doesDirectoryExist tmp - if (not exists) + let dir = dirspec g + liftIO $ forM_ dup $ \t -> removeFile $ dir keyFile t + + return stale + +staleKeys' :: (Git.Repo -> FilePath) -> Annex [Key] +staleKeys' dirspec = do + g <- Annex.gitRepo + let dir = dirspec g + exists <- liftIO $ doesDirectoryExist dir + if not exists then return [] else do - contents <- liftIO $ getDirectoryContents tmp + contents <- liftIO $ getDirectoryContents dir files <- liftIO $ filterM doesFileExist $ - map (tmp ) contents + map (dir ) contents return $ catMaybes $ map (fileKey . takeFileName) files diff --git a/Locations.hs b/Locations.hs index f263ea526b..1c4f8296e8 100644 --- a/Locations.hs +++ b/Locations.hs @@ -17,6 +17,7 @@ module Locations ( gitAnnexTmpDir, gitAnnexTmpLocation, gitAnnexBadDir, + gitAnnexBadLocation, gitAnnexUnusedLog, isLinkToAnnex, logFile, @@ -105,9 +106,13 @@ gitAnnexTmpLocation r key = gitAnnexTmpDir r keyFile key gitAnnexBadDir :: Git.Repo -> FilePath gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r "bad" -{- .git/annex/unused is used to number possibly unused keys -} -gitAnnexUnusedLog :: Git.Repo -> FilePath -gitAnnexUnusedLog r = gitAnnexDir r "unused" +{- The bad file to use for a given key. -} +gitAnnexBadLocation :: Git.Repo -> Key -> FilePath +gitAnnexBadLocation r key = gitAnnexBadDir r keyFile key + +{- .git/annex/*unused is used to number possibly unused keys -} +gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath +gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") {- Checks a symlink target to see if it appears to point to annexed content. -} isLinkToAnnex :: FilePath -> Bool diff --git a/debian/changelog b/debian/changelog index 92c05a5a69..813816079a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. * Add hook special remotes. * Avoid crashing when an existing key is readded to the annex. + * unused: Now also lists files fsck places in .git/annex/bad/ -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 3e91e7ad92..450b95a0dd 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -158,7 +158,7 @@ Many git-annex commands will stage changes for later `git commit` by you. Checks the annex for data that does not correspond to any files currently in the respository, and prints a numbered list of the data. - To only show unused temp files, specify --fast + To only show unused temp and bad files, specify --fast To check data on a remote that does not correspond to any files currently in the local repository, specify --from. From cf501d3b9be89931bfede402da85cb3bdc455041 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Apr 2011 14:04:20 -0400 Subject: [PATCH 1655/8313] set ANNEX_HASH_* always --- Remote/Hook.hs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 2613fda7a6..ba38355caf 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -69,16 +69,16 @@ hookSetup u c = do return c' hookEnv :: Key -> Maybe FilePath -> Maybe [(String, String)] -hookEnv k f = Just $ keyenv : fileenv f +hookEnv k f = Just $ fileenv f ++ keyenv where env s v = ("ANNEX_" ++ s, v) - keyenv = env "KEY" (show k) - fileenv Nothing = [] - fileenv (Just file) = - [ env "FILE" file + keyenv = + [ env "KEY" (show k) , env "HASH_1" (hashbits !! 0) , env "HASH_2" (hashbits !! 1) ] + fileenv Nothing = [] + fileenv (Just file) = [env "FILE" file] hashbits = map takeDirectory $ splitPath $ hashDirMixed k lookupHook :: String -> String -> Annex (Maybe String) From a14368aa8746bd12c599b7ae02a11ec9ef419c5a Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Fri, 29 Apr 2011 18:01:04 +0000 Subject: [PATCH 1656/8313] Added a comment --- .../comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment new file mode 100644 index 0000000000..9127cdeeaa --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 5" + date="2011-04-29T18:01:04Z" + content=""" +I've corrected the missing `ANNEX_HASH_*` oversight. (It also affected removal, btw.) +"""]] From 110b1e2b0a4d8355b3de5ebde1710b6b7cd61911 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Apr 2011 20:11:09 +0000 Subject: [PATCH 1657/8313] Added a comment --- ..._2e9da5a919bbbc27b32de3b243867d4f._comment | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment new file mode 100644 index 0000000000..748b4f29d6 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment @@ -0,0 +1,21 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 6" + date="2011-04-29T20:11:08Z" + content=""" +Cool, that seems to make things work as expected, here's an updated recipe + + +
+git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY'
+git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY $ANNEX_FILE'
+git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
+git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY 2>&1 || echo FAIL'
+git annex initremote library type=hook hooktype=tahoe encryption=none
+git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
+
+ + +I just needs some of the output redirected to /dev/null. +"""]] From 1f84c7a9642378e26d2b076def52255361591a04 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 14:05:10 -0400 Subject: [PATCH 1658/8313] S3: When encryption is enabled, the Amazon S3 login credentials are stored, encrypted, in .git-annex/remotes.log, so environment variables need not be set after the remote is initialized. --- Crypto.hs | 3 + Remote/Encryptable.hs | 30 ++++++---- Remote/S3real.hs | 88 ++++++++++++++++++++-------- debian/changelog | 3 + doc/special_remotes/Amazon_S3.mdwn | 6 ++ doc/walkthrough/using_Amazon_S3.mdwn | 4 +- 6 files changed, 96 insertions(+), 38 deletions(-) diff --git a/Crypto.hs b/Crypto.hs index 20b999f65a..2f544f21fd 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -22,6 +22,8 @@ module Crypto ( withDecryptedHandle, withEncryptedContent, withDecryptedContent, + toB64, + fromB64, prop_hmacWithCipher_sane ) where @@ -252,6 +254,7 @@ fromB64 s = case B64.decode s of Nothing -> error "bad base64 encoded data" Just ws -> w82s ws + hmacWithCipher :: Cipher -> String -> String hmacWithCipher c = hmacWithCipher' (cipherHmac c) hmacWithCipher' :: String -> String -> String diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 493ff12143..27c4e7f46c 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -70,23 +70,27 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = Nothing -> a k Just (_, k') -> a k' -{- Gets encryption Cipher, and encrypted version of Key. - - - - The decrypted Cipher is cached in the Annex state. -} -cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) -cipherKey Nothing _ = return Nothing -cipherKey (Just c) k = do +{- Gets encryption Cipher. The decrypted Cipher is cached in the Annex + - state. -} +remoteCipher :: RemoteConfig -> Annex (Maybe Cipher) +remoteCipher c = do cache <- Annex.getState Annex.cipher case cache of - Just cipher -> ret cipher + Just cipher -> return $ Just cipher Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do - showNote "gpg" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) - ret cipher - where - ret cipher = do - k' <- liftIO $ encryptKey cipher k - return $ Just (cipher, k') + return $ Just cipher + +{- Gets encryption Cipher, and encrypted version of Key. -} +cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key)) +cipherKey Nothing _ = return Nothing +cipherKey (Just c) k = do + cipher <- remoteCipher c + case cipher of + Just ciphertext -> do + k' <- liftIO $ encryptKey ciphertext k + return $ Just (ciphertext, k') + Nothing -> return Nothing diff --git a/Remote/S3real.hs b/Remote/S3real.hs index 2e198f79d1..c1ae0fbcc6 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -15,7 +15,7 @@ import Network.AWS.AWSResult import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import Data.Maybe -import Control.Monad (when) +import Control.Monad (when, liftM) import Control.Monad.State (liftIO) import System.Environment import System.Posix.Files @@ -45,9 +45,10 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost - return $ gen' r u c cst + c' <- s3GetCreds c + return $ gen' r u c' cst gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex -gen' r u c cst = +gen' r u c cst = do encryptableRemote c (storeEncrypted this) (retrieveEncrypted this) @@ -69,12 +70,12 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane c' <- encryptionSetup c - let fullconfig = M.union c' defaults + c'' <- liftM fromJust (s3GetCreds $ Just c') + let fullconfig = M.union c'' defaults -- check bucket location to see if the bucket exists, and create it let datacenter = fromJust $ M.lookup "datacenter" fullconfig conn <- s3ConnectionRequired fullconfig - showNote "checking bucket" loc <- liftIO $ getBucketLocation conn bucket case loc of @@ -88,7 +89,7 @@ s3Setup u c = do Left err -> s3Error err gitConfigSpecialRemote u fullconfig "s3" "true" - return fullconfig + s3SetCreds fullconfig where remotename = fromJust (M.lookup "name" c) bucket = remotename ++ "-" ++ u @@ -186,6 +187,19 @@ s3Bool res = do Right _ -> return True Left e -> s3Warning e +s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a +s3Action r noconn action = do + when (config r == Nothing) $ + error $ "Missing configuration for special remote " ++ name r + let bucket = M.lookup "bucket" $ fromJust $ config r + conn <- s3Connection $ fromJust $ config r + case (bucket, conn) of + (Just b, Just c) -> action (c, b) + _ -> return noconn + +bucketKey :: String -> Key -> S3Object +bucketKey bucket k = S3Object bucket (show k) "" [] L.empty + s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection s3ConnectionRequired c = do conn <- s3Connection c @@ -195,30 +209,58 @@ s3ConnectionRequired c = do s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) s3Connection c = do - ak <- getEnvKey "AWS_ACCESS_KEY_ID" - sk <- getEnvKey "AWS_SECRET_ACCESS_KEY" - if (null ak || null sk) - then do - warning "Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3" + case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of + (Just ak, Just sk) -> return $ Just $ AWSConnection host port ak sk + _ -> do + warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey ++ " to use S3" return Nothing - else return $ Just $ AWSConnection host port ak sk where host = fromJust $ (M.lookup "host" c) port = let s = fromJust $ (M.lookup "port" c) in case reads s of [(p, _)] -> p _ -> error $ "bad S3 port value: " ++ s + +{- S3 creds come from the environment if set. + - Otherwise, might be stored encrypted in the remote's config. -} +s3GetCreds :: Maybe RemoteConfig -> Annex (Maybe RemoteConfig) +s3GetCreds Nothing = return Nothing +s3GetCreds (Just c) = do + ak <- getEnvKey s3AccessKey + sk <- getEnvKey s3SecretKey + if (null ak || null sk) + then do + mcipher <- remoteCipher c + case (M.lookup "s3creds" c, mcipher) of + (Just encrypted, Just cipher) -> do + s <- liftIO $ withDecryptedContent cipher + (return $ L.pack $ fromB64 encrypted) + (return . L.unpack) + let line = lines s + creds (line !! 0) (line !! 1) + _ -> return $ Just c + else creds ak sk + where getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") + creds ak sk = return $ Just $ M.insert s3AccessKey ak $ M.insert s3SecretKey sk c -s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a -s3Action r noconn action = do - when (config r == Nothing) $ - error $ "Missing configuration for special remote " ++ name r - let bucket = M.lookup "bucket" $ fromJust $ config r - conn <- s3Connection (fromJust $ config r) - case (bucket, conn) of - (Just b, Just c) -> action (c, b) - _ -> return noconn +{- Stores S3 creds encrypted in the remote's config if possible. -} +s3SetCreds :: RemoteConfig -> Annex RemoteConfig +s3SetCreds c = do + let cleanconfig = M.delete s3AccessKey $ M.delete s3SecretKey c + case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of + (Just ak, Just sk) -> do + mcipher <- remoteCipher c + case mcipher of + Just cipher -> do + s <- liftIO $ withEncryptedContent cipher + (return $ L.pack $ unlines [ak, sk]) + (return . L.unpack) + return $ M.insert "s3creds" (toB64 s) cleanconfig + Nothing -> return cleanconfig + _ -> return cleanconfig -bucketKey :: String -> Key -> S3Object -bucketKey bucket k = S3Object bucket (show k) "" [] L.empty +s3AccessKey :: String +s3AccessKey = "AWS_ACCESS_KEY_ID" +s3SecretKey :: String +s3SecretKey = "AWS_SECRET_ACCESS_KEY" diff --git a/debian/changelog b/debian/changelog index 813816079a..b28728b0ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (0.20110428) UNRELEASED; urgency=low * Add hook special remotes. * Avoid crashing when an existing key is readded to the annex. * unused: Now also lists files fsck places in .git/annex/bad/ + * S3: When encryption is enabled, the Amazon S3 login credentials + are stored, encrypted, in .git-annex/remotes.log, so environment + variables need not be set after the remote is initialized. -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/Amazon_S3.mdwn index 87cde32993..35397dc2a9 100644 --- a/doc/special_remotes/Amazon_S3.mdwn +++ b/doc/special_remotes/Amazon_S3.mdwn @@ -29,3 +29,9 @@ the S3 remote. * `bucket` - S3 requires that buckets have a globally unique name, so by default, a bucket name is chosen based on the remote name and UUID. This can be specified to pick a bucket name. + +The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and +`ANNEX_S3_SECRET_ACCESS_KEY` can be used to supply login credentials +for Amazon. When encryption is enabled, they are stored in encrypted form +by `git annex initremote`, so you do not need to keep the environment +variables set after the initial initalization of the remote. diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index c842583546..63bed5d63a 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -30,8 +30,8 @@ repository use the same S3 remote is easy: Now the remote can be used like any other remote. # git annex copy my_cool_big_file --to cloud - copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok + copy my_cool_big_file (checking cloud...) (to cloud...) ok # git annex move video/hackity_hack_and_kaxxt.mov --to cloud - move video/hackity_hack_and_kaxxt.mov (gpg) (checking cloud...) (to cloud...) ok + move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok See [[special_remotes/Amazon_S3]] for details. From 3095e1631180d87cba112c210dfdfeee9b57ef54 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 14:09:07 -0400 Subject: [PATCH 1659/8313] mention that the cipher can also be used to crypt access keys --- doc/design/encryption.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index b30e01cdda..bcd6a11bc6 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -85,6 +85,15 @@ really have content. If it's later determined to be safe to not encrypt the HMAC cipher, the current design allows changing that, even for existing remotes. +## other use of the symmetric cipher + +The symmetric cipher can be used to encrypt other content than the content +sent to the remote. In particular, it may make sense to encrypt whatever +access keys are used by the special remote with the cipher, and store that +in remotes.log. This way anyone whose gpg key has been given access to +the cipher can get access to whatever other credentials are needed to +use the special remote. + ## risks A risk of this scheme is that, once the symmetric cipher has been obtained, it From 2ddade8132169ea751628f72ae5b03c5921abafc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 14:27:40 -0400 Subject: [PATCH 1660/8313] factor out base64 code --- Base64.hs | 20 ++++++++++++++++++++ Crypto.hs | 14 +------------- Remote/S3real.hs | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 Base64.hs diff --git a/Base64.hs b/Base64.hs new file mode 100644 index 0000000000..cc6346b415 --- /dev/null +++ b/Base64.hs @@ -0,0 +1,20 @@ +{- Simple Base64 access + - + - Copyright 2011 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Base64 (toB64, fromB64) where + +import Codec.Binary.Base64 +import Data.Bits.Utils + +toB64 :: String -> String +toB64 = encode . s2w8 + +fromB64 :: String -> String +fromB64 s = + case decode s of + Nothing -> error "bad base64 encoded data" + Just ws -> w82s ws diff --git a/Crypto.hs b/Crypto.hs index 2f544f21fd..53cd48dd59 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -22,21 +22,17 @@ module Crypto ( withDecryptedHandle, withEncryptedContent, withDecryptedContent, - toB64, - fromB64, prop_hmacWithCipher_sane ) where import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M -import qualified Codec.Binary.Base64 as B64 import Data.ByteString.Lazy.UTF8 (fromString) import Data.Digest.Pure.SHA import System.Cmd.Utils import Data.String.Utils import Data.List -import Data.Bits.Utils import System.IO import System.Posix.IO import System.Posix.Types @@ -50,6 +46,7 @@ import Types import Key import RemoteClass import Utility +import Base64 import CryptoTypes {- The first half of a Cipher is used for HMAC; the remainder @@ -246,15 +243,6 @@ configGet c key = Just v -> v Nothing -> error $ "missing " ++ key ++ " in remote config" -toB64 :: String -> String -toB64 = B64.encode . s2w8 - -fromB64 :: String -> String -fromB64 s = - case B64.decode s of - Nothing -> error "bad base64 encoded data" - Just ws -> w82s ws - hmacWithCipher :: Cipher -> String -> String hmacWithCipher c = hmacWithCipher' (cipherHmac c) hmacWithCipher' :: String -> String -> String diff --git a/Remote/S3real.hs b/Remote/S3real.hs index c1ae0fbcc6..d6bfe54874 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -33,6 +33,7 @@ import Remote.Encryptable import Crypto import Key import Content +import Base64 remote :: RemoteType Annex remote = RemoteType { From 3c319cd844f23edeab800d37ed0256d92a88a818 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 May 2011 15:13:54 -0400 Subject: [PATCH 1661/8313] avoid always decrypting cipher Last change moved cipher decryption to remote setup time. Fixed this with a bit of a hack. --- Remote/Encryptable.hs | 1 + Remote/S3real.hs | 43 +++++++++++++++------------- doc/walkthrough/using_Amazon_S3.mdwn | 6 ++-- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs index 27c4e7f46c..31ef1f37a7 100644 --- a/Remote/Encryptable.hs +++ b/Remote/Encryptable.hs @@ -80,6 +80,7 @@ remoteCipher c = do Nothing -> case extractCipher c of Nothing -> return Nothing Just encipher -> do + showNote "gpg" cipher <- liftIO $ decryptCipher c encipher Annex.changeState (\s -> s { Annex.cipher = Just cipher }) return $ Just cipher diff --git a/Remote/S3real.hs b/Remote/S3real.hs index d6bfe54874..b0371eb5ea 100644 --- a/Remote/S3real.hs +++ b/Remote/S3real.hs @@ -15,10 +15,11 @@ import Network.AWS.AWSResult import qualified Data.ByteString.Lazy.Char8 as L import qualified Data.Map as M import Data.Maybe -import Control.Monad (when, liftM) +import Control.Monad (when) import Control.Monad.State (liftIO) import System.Environment import System.Posix.Files +import System.Posix.Env (setEnv) import RemoteClass import Types @@ -46,8 +47,7 @@ remote = RemoteType { gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex) gen r u c = do cst <- remoteCost r expensiveRemoteCost - c' <- s3GetCreds c - return $ gen' r u c' cst + return $ gen' r u c cst gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex gen' r u c cst = do encryptableRemote c @@ -71,8 +71,7 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig s3Setup u c = do -- verify configuration is sane c' <- encryptionSetup c - c'' <- liftM fromJust (s3GetCreds $ Just c') - let fullconfig = M.union c'' defaults + let fullconfig = M.union c' defaults -- check bucket location to see if the bucket exists, and create it let datacenter = fromJust $ M.lookup "datacenter" fullconfig @@ -210,8 +209,9 @@ s3ConnectionRequired c = do s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection) s3Connection c = do - case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of - (Just ak, Just sk) -> return $ Just $ AWSConnection host port ak sk + creds <- s3GetCreds c + case creds of + Just (ak, sk) -> return $ Just $ AWSConnection host port ak sk _ -> do warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey ++ " to use S3" return Nothing @@ -224,9 +224,8 @@ s3Connection c = do {- S3 creds come from the environment if set. - Otherwise, might be stored encrypted in the remote's config. -} -s3GetCreds :: Maybe RemoteConfig -> Annex (Maybe RemoteConfig) -s3GetCreds Nothing = return Nothing -s3GetCreds (Just c) = do +s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String)) +s3GetCreds c = do ak <- getEnvKey s3AccessKey sk <- getEnvKey s3SecretKey if (null ak || null sk) @@ -238,28 +237,32 @@ s3GetCreds (Just c) = do (return $ L.pack $ fromB64 encrypted) (return . L.unpack) let line = lines s - creds (line !! 0) (line !! 1) - _ -> return $ Just c - else creds ak sk + let ak' = line !! 0 + let sk' = line !! 1 + liftIO $ do + setEnv s3AccessKey ak True + setEnv s3SecretKey sk True + return $ Just (ak', sk') + _ -> return Nothing + else return $ Just (ak, sk) where getEnvKey s = liftIO $ catch (getEnv s) (const $ return "") - creds ak sk = return $ Just $ M.insert s3AccessKey ak $ M.insert s3SecretKey sk c {- Stores S3 creds encrypted in the remote's config if possible. -} s3SetCreds :: RemoteConfig -> Annex RemoteConfig s3SetCreds c = do - let cleanconfig = M.delete s3AccessKey $ M.delete s3SecretKey c - case (M.lookup s3AccessKey c, M.lookup s3SecretKey c) of - (Just ak, Just sk) -> do + creds <- s3GetCreds c + case creds of + Just (ak, sk) -> do mcipher <- remoteCipher c case mcipher of Just cipher -> do s <- liftIO $ withEncryptedContent cipher (return $ L.pack $ unlines [ak, sk]) (return . L.unpack) - return $ M.insert "s3creds" (toB64 s) cleanconfig - Nothing -> return cleanconfig - _ -> return cleanconfig + return $ M.insert "s3creds" (toB64 s) c + Nothing -> return c + _ -> return c s3AccessKey :: String s3AccessKey = "AWS_ACCESS_KEY_ID" diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 63bed5d63a..512ef961f9 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -15,7 +15,7 @@ like "2512E3C7" Next, create the S3 remote, and describe it. # git annex initremote cloud type=S3 encryption=2512E3C7 - initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) ok + initremote cloud (encryption setup with gpg key C910D9222512E3C7) (checking bucket) (creating bucket in US) (gpg) ok # git annex describe cloud "at Amazon's US datacenter" describe cloud ok @@ -25,12 +25,12 @@ repository use the same S3 remote is easy: # cd /media/usb/annex # git pull laptop master # git annex initremote cloud - initremote cloud (checking bucket) ok + initremote cloud (gpg) (checking bucket) ok Now the remote can be used like any other remote. # git annex copy my_cool_big_file --to cloud - copy my_cool_big_file (checking cloud...) (to cloud...) ok + copy my_cool_big_file (gpg) (checking cloud...) (to cloud...) ok # git annex move video/hackity_hack_and_kaxxt.mov --to cloud move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok From 86d3205061dd6f10b126e1578eec135376ae6e99 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 May 2011 21:49:20 -0400 Subject: [PATCH 1662/8313] releasing version 0.20110503 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index b28728b0ff..73f5924a95 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (0.20110428) UNRELEASED; urgency=low +git-annex (0.20110503) unstable; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. * Add hook special remotes. @@ -8,7 +8,7 @@ git-annex (0.20110428) UNRELEASED; urgency=low are stored, encrypted, in .git-annex/remotes.log, so environment variables need not be set after the remote is initialized. - -- Joey Hess Thu, 28 Apr 2011 14:38:16 -0400 + -- Joey Hess Tue, 03 May 2011 20:56:01 -0400 git-annex (0.20110427) unstable; urgency=low From e70fc1340a9cf4e33356e85b4ec4793fe04fa2f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 May 2011 21:50:01 -0400 Subject: [PATCH 1663/8313] add news item for git-annex 0.20110503 --- doc/news/version_0.20110417.mdwn | 21 --------------------- doc/news/version_0.20110503.mdwn | 9 +++++++++ 2 files changed, 9 insertions(+), 21 deletions(-) delete mode 100644 doc/news/version_0.20110417.mdwn create mode 100644 doc/news/version_0.20110503.mdwn diff --git a/doc/news/version_0.20110417.mdwn b/doc/news/version_0.20110417.mdwn deleted file mode 100644 index 7e28ea2138..0000000000 --- a/doc/news/version_0.20110417.mdwn +++ /dev/null @@ -1,21 +0,0 @@ -git-annex 0.20110417 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * bup is now supported as a special type of remote. - * The data sent to special remotes (Amazon S3, bup, etc) can be encrypted - using GPG for privacy. - * Use lowercase hash directories for locationlog files, to avoid - some issues with git on OSX with the mixed-case directories. - No migration is needed; the old mixed case hash directories are still - read; new information is written to the new directories. - * Unused files on remotes, particulary special remotes, can now be - identified and dropped, by using "--from remote" with git annex unused - and git annex dropunused. - * Clear up short option confusion between --from and --force (-f is now - --from, and there is no short option for --force). - * Add build depend on perlmagick so docs are consistently built. - Closes: #[621410](http://bugs.debian.org/621410) - * Add doc-base file. Closes: #[621408](http://bugs.debian.org/621408) - * Periodically flush git command queue, to avoid boating memory usage - too much. - * Support "sha1" and "sha512" commands on FreeBSD, and allow building - if any/all SHA commands are not available. Thanks, Fraser Tweedale"""]] \ No newline at end of file diff --git a/doc/news/version_0.20110503.mdwn b/doc/news/version_0.20110503.mdwn new file mode 100644 index 0000000000..f24f0c48f6 --- /dev/null +++ b/doc/news/version_0.20110503.mdwn @@ -0,0 +1,9 @@ +git-annex 0.20110503 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Fix hasKeyCheap setting for bup and rsync special remotes. + * Add hook special remotes. + * Avoid crashing when an existing key is readded to the annex. + * unused: Now also lists files fsck places in .git/annex/bad/ + * S3: When encryption is enabled, the Amazon S3 login credentials + are stored, encrypted, in .git-annex/remotes.log, so environment + variables need not be set after the remote is initialized."""]] \ No newline at end of file From 405745acdd1300d8a1e1db1ca653d867c556adf5 Mon Sep 17 00:00:00 2001 From: "https://me.yahoo.com/a/wpdhh7Et0MiET3shW4BlKe60GFs_mXI-#16fd9" Date: Thu, 5 May 2011 22:35:44 +0000 Subject: [PATCH 1664/8313] --- ...hlist:_Prevent_repeated_password_prompts_for_one_command.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn new file mode 100644 index 0000000000..6c12b1d155 --- /dev/null +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn @@ -0,0 +1 @@ +Simple, when performing various git annex command over ssh, in particular a multi-file get, and using password authentication, git annex will prompt more than once for a user password. This makes batch updates very inconvenient. From d3dfdb9f9d48cd914a6ce2ac54645502a8932a2f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Thu, 5 May 2011 23:57:40 +0000 Subject: [PATCH 1665/8313] Initial commit --- ...ncorrectly_parses_bare_IPv6_addresses.mdwn | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn new file mode 100644 index 0000000000..e5a2c03377 --- /dev/null +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -0,0 +1,41 @@ +I have a git remote in a git-annex-enabled repository. Here's what it looks like in .git/config: + +
+[remote "renaissance"]
+        url = ssh://[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/home/paulproteus/Music/annex/
+        fetch = +refs/heads/*:refs/remotes/renaissance/*
+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64
+
+ +I wanted to "git annex get" some data. git-annex appears to pass incorrectly-formatted IPv6 addresses to rsync: + +
+get primary/emusiq/Arab Strap/Monday At The Hug And Pint/01-The Shy Retirer.mp3 (copying from renaissance...) 
+ssh: Could not resolve hostname [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]: Name or service not known
+rsync: connection unexpectedly closed (0 bytes received so far) [Receiver]
+rsync error: unexplained error (code 255) at io.c(601) [Receiver=3.0.7]
+
+  rsync failed -- run git annex again to resume file transfer
+  Unable to access these remotes: renaissance
+  Try making some of these repositories available:
+  	2992752e-1a13-11e0-ba68-57d3c800da64
+failed
+
+ +In this case, the square brackets should not be there. + +I tried changing the .git/config syntax slightly, and got a different, also-incorrect behavior: + +
+[remote "renaissance"]
+        url = [2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/
+        fetch = +refs/heads/*:refs/remotes/renaissance/*
+        annex-uuid = 2992752e-1a13-11e0-ba68-57d3c800da64
+
+ +
+paulproteus@pathi:~/Music/annex$ git annex get
+git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulproteus/Music/annex/
+
+ +(Note that both these .git/config entries work fine with "git fetch".) From c7ff9d3dab19c2d504e74d5d92e8447b4719e872 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Fri, 6 May 2011 00:00:17 +0000 Subject: [PATCH 1666/8313] Add sign-off --- doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn index e5a2c03377..fdaf5d5d1c 100644 --- a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -39,3 +39,5 @@ git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulprote (Note that both these .git/config entries work fine with "git fetch".) + +-- Asheesh. From d59666aed614f85e6d4fbdc0e19d7870ebe1961d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 18:28:43 +0000 Subject: [PATCH 1667/8313] Added a comment --- ...ment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment new file mode 100644 index 0000000000..3cc0b08344 --- /dev/null +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-06T18:28:41Z" + content=""" +Unless you are forced to use a password, you should really be using a ssh key. + +ssh-keygen +put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600) +ssh-add +git annex whatever +"""]] From 489156f255c0bbd1195db00fe0158e6f5c0be549 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 18:29:23 +0000 Subject: [PATCH 1668/8313] removed --- ...ment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment deleted file mode 100644 index 3cc0b08344..0000000000 --- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_e4f87beb5e67f00dc83ce3dd40ac3633._comment +++ /dev/null @@ -1,13 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" - nickname="Richard" - subject="comment 1" - date="2011-05-06T18:28:41Z" - content=""" -Unless you are forced to use a password, you should really be using a ssh key. - -ssh-keygen -put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600) -ssh-add -git annex whatever -"""]] From 33041e4afb66984a56e646ff4c8240adc8c5e01d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 18:30:03 +0000 Subject: [PATCH 1669/8313] Added a comment --- ...ent_1_3f9c0d08932c2ede61c802a91261a1f7._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment new file mode 100644 index 0000000000..2801d8e68f --- /dev/null +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-06T18:30:02Z" + content=""" +Unless you are forced to use a password, you should really be using a ssh key. + + ssh-keygen + #put local .ssh/id_?sa.pub into remote .ssh/authorized_keys (which needs to be chmod 600) + ssh-add + git annex whatever + +"""]] From f1fbe33cfeba70123a702b3cdd1139361b75b875 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 May 2011 14:43:40 -0400 Subject: [PATCH 1670/8313] response --- ..._Prevent_repeated_password_prompts_for_one_command.mdwn | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn index 6c12b1d155..6d1552fe4e 100644 --- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn +++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn @@ -1 +1,8 @@ Simple, when performing various git annex command over ssh, in particular a multi-file get, and using password authentication, git annex will prompt more than once for a user password. This makes batch updates very inconvenient. + +> I'd suggest using ssh-agent, or a passwordless ssh key. Possibly in +> combination with [[git-annex-shell]] if you want to lock down a +> particular ssh key to only being able to use git-annex and git-daemon. +> +> Combining multiple operations into a single ssh is on the todo list, but +> very far down it. --[[Joey]] From 078a6fbd76190c48cfa5c588bb9d2174baef5852 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 May 2011 15:21:30 -0400 Subject: [PATCH 1671/8313] Work around a bug in Network.URI's handling of bracketed ipv6 addresses. --- GitRepo.hs | 18 +++++++++++++++--- debian/changelog | 6 ++++++ ...incorrectly_parses_bare_IPv6_addresses.mdwn | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/GitRepo.hs b/GitRepo.hs index 9ecaa8ffcb..49024abe0e 100644 --- a/GitRepo.hs +++ b/GitRepo.hs @@ -280,9 +280,21 @@ urlScheme :: Repo -> String urlScheme Repo { location = Url u } = uriScheme u urlScheme repo = assertUrl repo $ error "internal" +{- Work around a bug in the real uriRegName + - -} +uriRegName' :: URIAuth -> String +uriRegName' a = fixup $ uriRegName a + where + fixup x@('[':rest) + | rest !! len == ']' = take len rest + | otherwise = x + where + len = (length rest) - 1 + fixup x = x + {- Hostname of an URL repo. -} urlHost :: Repo -> String -urlHost = urlAuthPart uriRegName +urlHost = urlAuthPart uriRegName' {- Port of an URL repo, if it has a nonstandard one. -} urlPort :: Repo -> Maybe Integer @@ -294,11 +306,11 @@ urlPort r = {- Hostname of an URL repo, including any username (ie, "user@host") -} urlHostUser :: Repo -> String -urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName r +urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r {- The full authority portion an URL repo. (ie, "user@host:port") -} urlAuthority :: Repo -> String -urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName a ++ uriPort a +urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName' a ++ uriPort a where a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u) urlAuthority repo = assertUrl repo $ error "internal" diff --git a/debian/changelog b/debian/changelog index 73f5924a95..09dde346c0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (0.20110504) UNRELEASED; urgency=low + + * Work around a bug in Network.URI's handling of bracketed ipv6 addresses. + + -- Joey Hess Fri, 06 May 2011 15:20:38 -0400 + git-annex (0.20110503) unstable; urgency=low * Fix hasKeyCheap setting for bup and rsync special remotes. diff --git a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn index fdaf5d5d1c..c94952b490 100644 --- a/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn +++ b/doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn @@ -41,3 +41,19 @@ git-annex: bad url ssh://[2001/~/0:53aa:64c:24ef:5ce4:2ef9:cdda]:/home/paulprote (Note that both these .git/config entries work fine with "git fetch".) -- Asheesh. + +> Technically, this seems to be a bug in the haskell URI library; it honors +> the `[]` in parsing, but does not remove them when the URI is queried for +> the host part. + +
+Prelude Network.URI> let (Just u) = parseURI "http://foo@[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]/bar"
+Prelude Network.URI> let (Just a) = uriAuthority u
+Prelude Network.URI> uriRegName a
+"[2001:0:53aa:64c:24ef:5ce4:2ef9:cdda]"
+Prelude Network.URI> isIPv6address $ uriRegName a
+False
+
+ +> I have filed a [bug upstream](http://trac.haskell.org/network/ticket/40), and put a workaround in git-annex. [[done]] +> --[[Joey]] From 7020d5bc5bada2533cda27ffa696c0a924158a8c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Fri, 6 May 2011 22:35:28 +0000 Subject: [PATCH 1672/8313] --- doc/todo/wishlist:_swift_backend.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/todo/wishlist:_swift_backend.mdwn diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn new file mode 100644 index 0000000000..498f9588a7 --- /dev/null +++ b/doc/todo/wishlist:_swift_backend.mdwn @@ -0,0 +1,5 @@ +[swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode. + +I can provide a test account soonish if need be, else rackspace.com might already be offering their object storage via swift. + +Richard From 579ade130aa10b5907536f9c6f698689d246171f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 May 2011 18:44:49 -0400 Subject: [PATCH 1673/8313] update --- doc/install/OSX.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn index c8c381486b..46a285e131 100644 --- a/doc/install/OSX.mdwn +++ b/doc/install/OSX.mdwn @@ -6,6 +6,7 @@ sudo cabal install utf8-string sudo port install pcre sudo cabal install pcre-light sudo cabal install quickcheck +sudo cabal install SHA # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland) export PATH=$PATH:/opt/local/libexec/gnubin From 50fecbcb410ebb69a0013ab54630fcca59550291 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 9 May 2011 12:23:50 +0000 Subject: [PATCH 1674/8313] --- doc/todo/wishlist:_swift_backend.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn index 498f9588a7..5e2991871e 100644 --- a/doc/todo/wishlist:_swift_backend.mdwn +++ b/doc/todo/wishlist:_swift_backend.mdwn @@ -1,5 +1,5 @@ [swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode. -I can provide a test account soonish if need be, else rackspace.com might already be offering their object storage via swift. +I can provide a test account soonish if need be, else https://auth.api.rackspacecloud.com/v1.0 Richard From 59e960a51cf88ffa8bc664f41a026ce1ca08e4d4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Mon, 9 May 2011 12:24:28 +0000 Subject: [PATCH 1675/8313] --- doc/todo/wishlist:_swift_backend.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/todo/wishlist:_swift_backend.mdwn b/doc/todo/wishlist:_swift_backend.mdwn index 5e2991871e..28bd265faf 100644 --- a/doc/todo/wishlist:_swift_backend.mdwn +++ b/doc/todo/wishlist:_swift_backend.mdwn @@ -1,5 +1,5 @@ [swift](http://swift.openstack.org/) is the object storage of Openstack. Think S3, but fully open source. As it's backed by rackspace.com, NASA, Dell and several other major players, adoption rates will explode. -I can provide a test account soonish if need be, else https://auth.api.rackspacecloud.com/v1.0 +I can provide a test account soonish if need be, else rackspace.com if offering swift storage. Their API gateway lives at https://auth.api.rackspacecloud.com/v1.0 Richard From 62cd26c64a5c66c86e6c834aba51d314457cca9f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 12 May 2011 00:07:31 +0000 Subject: [PATCH 1676/8313] Added a comment --- ...1_6a41bf7e2db83db3a01722b516fb6886._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment b/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment new file mode 100644 index 0000000000..59c30de534 --- /dev/null +++ b/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="comment 1" + date="2011-05-12T00:07:29Z" + content=""" +I followed this to re-inject files which git annex fsck listed as missing. + +For everyone of those files, I get + + git-annex-shell: key is already present in annex + rsync: connection unexpectedly closed (0 bytes received so far) [sender] + rsync error: error in rsync protocol data stream (code 12) at io.c(601) [sender=3.0.8] + +when trying to copy the files to the remote. + +-- Richard +"""]] From f7174d09b60992ba0bf106956f83b35bac038b16 Mon Sep 17 00:00:00 2001 From: "http://joey.kitenet.net/" Date: Thu, 12 May 2011 01:01:34 +0000 Subject: [PATCH 1677/8313] Added a comment --- .../comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment b/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment new file mode 100644 index 0000000000..44aab3baa0 --- /dev/null +++ b/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joey.kitenet.net/" + nickname="joey" + subject="comment 2" + date="2011-05-12T01:01:34Z" + content=""" +Sounds like you probably didn't commit after the fsck, or didn't push so the other repository did not know the first had the content again -- but I'm not 100% sure. +"""]] From 1fdc6b3aadc336a3a4ed2e35db9ad6797fd3b4d5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 14:55:27 -0400 Subject: [PATCH 1678/8313] update --- doc/design/encryption.mdwn | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index bcd6a11bc6..2b9769aa7a 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -1,5 +1,6 @@ -This was the design doc for [[encryption]] and is preserved for -the curious. +This was the design doc for [[/encryption]] and is preserved for +the curious. For an example of using git-annex with an encrypted S3 remote, +see [[using_Amazon_S3]]. [[!toc]] @@ -32,7 +33,7 @@ Allowing multiple ciphers to be used within a single remote would add a lot of complexity, so is not planned to be supported. Instead, if you want a new cipher, create a new S3 bucket, or whatever. There does not seem to be much benefit to using the same cipher for -two different enrypted remotes. +two different encrypted remotes. So, the encrypted cipher could just be stored with the rest of a remote's configuration in `.git-annex/remotes.log` (see [[internals]]). When `git From e9815a2eb64232da7f7f594bdfaeeaf716e5818d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 14:56:25 -0400 Subject: [PATCH 1679/8313] update --- doc/design/encryption.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn index 2b9769aa7a..11056478bd 100644 --- a/doc/design/encryption.mdwn +++ b/doc/design/encryption.mdwn @@ -1,6 +1,6 @@ This was the design doc for [[/encryption]] and is preserved for the curious. For an example of using git-annex with an encrypted S3 remote, -see [[using_Amazon_S3]]. +see [[walkthrough/using_Amazon_S3]]. [[!toc]] From 552aee694c75e331eb796c6ef93220d1b5feabdb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 14:58:48 -0400 Subject: [PATCH 1680/8313] layout --- doc/walkthrough/using_Amazon_S3.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn index 512ef961f9..6b0f496392 100644 --- a/doc/walkthrough/using_Amazon_S3.mdwn +++ b/doc/walkthrough/using_Amazon_S3.mdwn @@ -23,7 +23,7 @@ The configuration for the S3 remote is stored in git. So to make another repository use the same S3 remote is easy: # cd /media/usb/annex - # git pull laptop master + # git pull laptop master # git annex initremote cloud initremote cloud (gpg) (checking bucket) ok From 6cf797e8c3f779f1cc3b17c82fd1685613a8b4d3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 19:15:15 -0400 Subject: [PATCH 1681/8313] add github repo mirror should be populated by Branchable when it sees this commit --- doc/download.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/download.mdwn b/doc/download.mdwn index 748ac0ca34..ff780bd4ca 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,5 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] +(The repo is mirrored to [github](https://github.com/joeyh/git-annex).) Some operating systems include git-annex in easily prepackaged form and others need some manual work. See [[install]] for details. From d0ea28e9c7731a629e6722e69490da81518f0f0a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 19:17:54 -0400 Subject: [PATCH 1682/8313] wording --- doc/download.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/download.mdwn b/doc/download.mdwn index ff780bd4ca..405bc2fc4b 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -1,6 +1,6 @@ The main git repository for git-annex is `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)] -(The repo is mirrored to [github](https://github.com/joeyh/git-annex).) +(The repo is mirrored [at github](https://github.com/joeyh/git-annex).) Some operating systems include git-annex in easily prepackaged form and others need some manual work. See [[install]] for details. From 536036cfa2c1b28932f72408f1244f8e345eba82 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 19:20:54 -0400 Subject: [PATCH 1683/8313] add README --- README | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 README diff --git a/README b/README new file mode 100644 index 0000000000..ce67d68166 --- /dev/null +++ b/README @@ -0,0 +1,6 @@ +git-annex allows managing files with git, without checking the file +contents into git. While that may seem paradoxical, it is useful when +dealing with files larger than git can currently easily handle, whether due +to limitations in memory, checksumming time, or disk space. + +For documentation, see doc/ or From e72c1c33ae60f7fe544acd0a1f1331138b880e22 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 May 2011 23:05:14 -0400 Subject: [PATCH 1684/8313] would be nice.. --- doc/todo/support_S3_multipart_uploads.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/todo/support_S3_multipart_uploads.mdwn diff --git a/doc/todo/support_S3_multipart_uploads.mdwn b/doc/todo/support_S3_multipart_uploads.mdwn new file mode 100644 index 0000000000..711ac41b2a --- /dev/null +++ b/doc/todo/support_S3_multipart_uploads.mdwn @@ -0,0 +1,14 @@ +Did not know of this when I wrote S3 support. Ability to resume large +uploads would be good. + + + +Also allows supporting files > 5 gb, a S3 limit I was not aware of. + +NB: It would work just as well to split the object and upload the N parts +to S3, but not bother with S3's paperwork to rejoin them into one object. +Only reasons not to do that are a) backwards compatability with +the existing S3 remote and b) this would not allow accessing the content +in S3 w/o using git-annex, which could be useful in some scenarios. + +--[[Joey]] From fcdbfbdc9132d3c023da0d567807e5bb29fb695b Mon Sep 17 00:00:00 2001 From: zooko Date: Sat, 14 May 2011 05:07:18 +0000 Subject: [PATCH 1685/8313] Added a comment: request for information, plus some ideas --- ..._d636c868524b2055ee85832527437f90._comment | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment new file mode 100644 index 0000000000..1d75fb9631 --- /dev/null +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment @@ -0,0 +1,20 @@ +[[!comment format=mdwn + username="zooko" + ip="97.118.97.117" + subject="request for information, plus some ideas" + date="2011-05-14T05:07:17Z" + content=""" +Hey Jimmy: how's this working for you now? I would expect it to go slower and slower since Tahoe-LAFS has an O(N) algorithm for reading or updating directories. + +Of course, if it is still fast enough for your uses then that's okay. :-) + +(We're working on optimizations of this for future releases of Tahoe-LAFS.) + +I'd like to understand the desired behavior of store-hook and retrieve-hook better, in order to see if there is a more efficient way to use Tahoe-LAFS for this. + +Off to look for docs. + +Regards, + +Zooko +"""]] From 037fa707aa4f8ab43b5d18055e90c3bd42d818e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 14 May 2011 02:55:12 -0400 Subject: [PATCH 1686/8313] fix bug: put file in hashed directory structure, not tahoe root --- .../comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment index 748b4f29d6..80874db31d 100644 --- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment +++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment @@ -8,7 +8,7 @@ Cool, that seems to make things work as expected, here's an updated recipe 
-git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_KEY'
+git config annex.tahoe-store-hook 'tahoe mkdir tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2 && tahoe put $ANNEX_FILE tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
 git config annex.tahoe-retrieve-hook 'tahoe get tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY $ANNEX_FILE'
 git config annex.tahoe-remove-hook 'tahoe rm tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY'
 git config annex.tahoe-checkpresent-hook 'tahoe ls tahoe:$ANNEX_HASH_1/$ANNEX_HASH_2/$ANNEX_KEY 2>&1 || echo FAIL'
@@ -18,4 +18,6 @@ git annex describe 1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a library
 
 
 I just needs some of the output redirected to /dev/null.
+
+(I updated this comment to fix a bug. --[[Joey]])
 """]]

From 99c6dd556f940a51b862bacd00ce5f67eb1937a2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 09:06:54 +0000
Subject: [PATCH 1687/8313] Added a comment

---
 ...ent_3_b596b5cfd3377e58dbbb5d509d026b90._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment b/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
new file mode 100644
index 0000000000..4744db995c
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 3"
+ date="2011-05-14T09:06:54Z"
+ content="""
+As my comment from work is stuck in moderation:
+
+I ran this twice:
+
+    git pull && git annex add . && git annex copy . --to  --fast --quiet && git commit -a -m \"$HOST $(date +%F--%H-%M-%S-%Z)\" && git push
+
+but nothing changed
+"""]]

From 70d97467491f613bac30790462920c37b221c2d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 14 May 2011 10:02:30 +0000
Subject: [PATCH 1688/8313] Added a comment

---
 ...comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment

diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment
new file mode 100644
index 0000000000..dc97128bd1
--- /dev/null
+++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 8"
+ date="2011-05-14T10:02:26Z"
+ content="""
+@joey thanks for the update in the previous comment, I had forgotten about updating it.
+
+@zooko it's working okay for me right now, since I'm only putting fairly big blogs on stuff on to it and only things that I *really* care about. On the performance side, if it ran faster then it would be nicer :)
+"""]]

From 12ecf6c0c624ecb883140b456e9933fd3cd4a5b4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 14 May 2011 10:04:37 +0000
Subject: [PATCH 1689/8313] Added a comment

---
 .../comment_1_e6efbb35f61ee521b473a92674036788._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment

diff --git a/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment b/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment
new file mode 100644
index 0000000000..98a998c1cf
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-05-14T10:04:36Z"
+ content="""
+I don't suppose this SWIFT api is compatible with the eucalytpus walrus api ?
+"""]]

From d96749588b65ef3f28775d140bb9e8b67cde90cc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 14 May 2011 10:23:44 +0000
Subject: [PATCH 1690/8313] mention that tahoe-lafs has limited testing as a
 special remote (me being the only user isn't very good testing)

---
 doc/special_remotes.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 7d55a88612..7e4b9d377b 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -11,6 +11,7 @@ They cannot be used by other git commands though.
 * [[directory]]
 * [[rsync]]
 * [[hook]]
+  * [tahoe-lafs][] - limited testing
 
 ## Unused content on special remotes
 
@@ -34,3 +35,5 @@ on special remotes, instead use `git annex unused --from`. Example:
 Do be cautious when using this; it cannot detect if content in a remote
 is used by that remote, or is the last copy of data that is used by
 some *other* remote.
+
+[tahoe-lafs]: http://git-annex.branchable.com/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/

From 8c152836ab2acc6142c544330823b79a214bb32c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 15:00:51 +0000
Subject: [PATCH 1691/8313] Added a comment

---
 .../comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment

diff --git a/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment b/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment
new file mode 100644
index 0000000000..97863b095f
--- /dev/null
+++ b/doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-05-14T15:00:51Z"
+ content="""
+It does offer a S3 compability layer, but that is de facto non-functioning as of right now.
+"""]]

From 3623c17a7ef3cc15bfd5ab7f0e13c10887bd4e9f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 11:57:40 -0400
Subject: [PATCH 1692/8313] use wikilink

---
 doc/special_remotes.mdwn | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 7e4b9d377b..13c18122e7 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -11,7 +11,7 @@ They cannot be used by other git commands though.
 * [[directory]]
 * [[rsync]]
 * [[hook]]
-  * [tahoe-lafs][] - limited testing
+* [[tahoe-lafs|forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] - limited testing
 
 ## Unused content on special remotes
 
@@ -35,5 +35,3 @@ on special remotes, instead use `git annex unused --from`. Example:
 Do be cautious when using this; it cannot detect if content in a remote
 is used by that remote, or is the last copy of data that is used by
 some *other* remote.
-
-[tahoe-lafs]: http://git-annex.branchable.com/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/

From 56182e2609aee14c2e743e8b83310899efbbecb0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 15:58:57 +0000
Subject: [PATCH 1693/8313] Comment moderation

---
 ...ment_2_c2b0ce025805b774dc77ce264a222824._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment

diff --git a/doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment b/doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment
new file mode 100644
index 0000000000..21f9d713cf
--- /dev/null
+++ b/doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="format, respect working directory"
+ date="2011-04-26T12:31:02Z"
+ content="""
+we could include the information about the current directory as well, if the command is not issued in the local git root directory. to avoid large numbers of similar lines, that could look like this:
+
+    Estimated annex size: B MiB (of C MiB; [B/C]%)
+    Estimated annex size in $PWD: B' MiB (of C' MiB; [B'/C']%)
+
+with the percentages being replaced with \"complete\" if really all files are present (and not just many enough for the value to be rounded to 100%).
+"""]]

From 81bca602dacfb038a5bb1f66146dbc85ba184223 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 16:13:58 +0000
Subject: [PATCH 1694/8313] Added a comment

---
 ...mment_4_d7112c315fb016a8a399e24e9b6461d8._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment b/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
new file mode 100644
index 0000000000..1fb19ab192
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-05-14T16:13:58Z"
+ content="""
+Hmm. Old versions may have forgotten to git add a .git-annex location log file when recovering content with fsck. That could be another reason things are out of sync.
+
+But I'm not clear on which repo is trying to copy files to which.
+
+(NB: If the files were recovered on a bare git repo, fsck cannot update the location log there, which could also explain this.)
+"""]]

From 60a1aa418ce68b573ed2aebb808c4186add4768a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 12:23:26 -0400
Subject: [PATCH 1695/8313] typo

---
 doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
index c447f75f0c..118f6fbb7c 100644
--- a/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
+++ b/doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn
@@ -16,4 +16,4 @@ make: *** [Touch.hs] Error 1
 
 I dug around the OSX documentation and fcntl.h header file and it seems that UTIME_OMIT, UTIME_NOW, AT_FDCWD and AT_SYMLINK_NOFOLLOW aren't defined (at least on OSX). I suspect the BSD's in general will have problems compiling git-annex.
 
-[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"]
+[[!meta title="annexed symlink mtime matching code is disabled on non-linux systems; needs testing"]]

From 4ff8b30d2c8390bfca8b345b3dd7c300477e95f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 12:26:06 -0400
Subject: [PATCH 1696/8313] layout

---
 doc/git-annex-shell.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 060d23ca18..1fc9647c88 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -42,7 +42,7 @@ Any command not listed below is passed through to git-shell.
 
   This runs rsync in server mode to transfer out the content of a key.
 
-## OPTIONS
+# OPTIONS
 
 Same as git-annex or git-shell, depending on the command being run.
 

From fdc5e5f56c6fca3d4a1f72bd5338b5d2ec9ab10c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 16:28:36 +0000
Subject: [PATCH 1697/8313] Added a comment

---
 ...ent_3_be62be5fe819acc0cb8b878802decd46._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment

diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment
new file mode 100644
index 0000000000..9c56452e53
--- /dev/null
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-14T16:28:36Z"
+ content="""
+To re-inject new content for a file, you really want to get a new key for the file. Otherwise, other repos that have the old file will never get the new content. So:
+
+
+git rm file
+mv ~/newcontent file
+git annex add file
+

+"""]]

From 020e9b013dea9e7429cee49cb19a5b7bc958a19f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 May 2011 12:28:42 -0400
Subject: [PATCH 1698/8313] close

---
 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn
index ced4fc5a0f..8df3bde481 100644
--- a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn
@@ -8,3 +8,5 @@ Then, I copied the annex object to the correct place in .git/annex/objects/...,
 
 
 Long story short, I think there should be a `git annex reinject $file` or similar which will take a file, either one replacing the symlink or with an arbitrary path, and put it into the correct place in the object store. Called normally, it should reject all reinjects where the checksum does not match. With --force, this should be overridden. For reasons of safety, WORM should always require --force.
+
+> [[closing|done]], seems addressed --[[Joey]] 

From a43a2731764e59e743fce4a75a5a416f24697754 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 16:29:35 +0000
Subject: [PATCH 1699/8313] Added a comment

---
 .../comment_4_480a4f72445a636eab1b1c0f816d365c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment

diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
new file mode 100644
index 0000000000..4a22d414b2
--- /dev/null
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-05-14T16:29:35Z"
+ content="""
+Although, if you really do want to shoot yourself in the foot, or know you have the old content, you can use `git-annex setkey`.
+"""]]

From 9a42ab53bef804f7e13ecfbbe3eba6ca3ec5eac1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 19:03:44 +0000
Subject: [PATCH 1700/8313] Added a comment

---
 ...ent_5_4ea29a6f8152eddf806c536de33ef162._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment b/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
new file mode 100644
index 0000000000..0a546bd88c
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 5"
+ date="2011-05-14T19:03:43Z"
+ content="""
+Version: 0.20110503
+
+My local non-bare repo is copying to a remote bare repo.
+
+I have been recovering in a non-bare repo.
+
+If there is anything I can send you to help... If I removed said files and went through http://git-annex.branchable.com/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/ -- would that help?
+"""]]

From 385268da2ea0fb66d6cda5be6d0b643db9e4f706 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 14 May 2011 19:23:45 +0000
Subject: [PATCH 1701/8313] Added a comment

---
 .../comment_6_0d85f114a103bd6532a3b3b24466012e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment b/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
new file mode 100644
index 0000000000..1e3f325319
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-05-14T19:23:45Z"
+ content="""
+Well, focus on a specific file that exhibits the problem. What does `git annex whereis` say about it? Is the content actually present in annex/objects/ on the bare repository? Does that contradict whereis?
+"""]]

From a9ba88dfbd0cef84fd5bdc438d1c89f16f23b89c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 14 May 2011 23:13:15 +0000
Subject: [PATCH 1702/8313] Added a comment

---
 ...mment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment b/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
new file mode 100644
index 0000000000..f7dfad68ca
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 7"
+ date="2011-05-14T23:13:15Z"
+ content="""
+It exists locally, whereis tells me it exists locally and locally, only.
+
+The object is _not_ in the bare repo.
+
+The file _might_ have gone missing before I upgraded my annex backend version to 2. Could this be a factor?
+"""]]

From 1b2e0c183df47456e7225198cc7ebed1b74d5b87 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 May 2011 00:09:34 +0000
Subject: [PATCH 1703/8313] Added a comment

---
 ...comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment b/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
new file mode 100644
index 0000000000..01248914c3
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-05-15T00:09:34Z"
+ content="""
+What you're describing should be impossible; the error message shown can only occur if the object is present in the annex where `git-annex-shell recvkey` is run. So something strange is going on.
+
+Try reproducing it by running on the remote system, `git-annex-shell recvkey /remote/repo.git $key` .. if you can reproduce it, I guess the next thing to do will be to strace the command and see why it's thinking the object is there.
+"""]]

From b400984ddf9aeb24c3d67e87cfeb29470618636c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 01:33:25 -0400
Subject: [PATCH 1704/8313] fixup

---
 test.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 4e0eb3fb2a..73eff662b9 100644
--- a/test.hs
+++ b/test.hs
@@ -466,7 +466,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	where
 		checkunused expectedkeys = do
 			git_annex "unused" ["-q"] @? "unused failed"
-			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog
+			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog ""
 			let unusedkeys = M.elems unusedmap
 			assertEqual "unused keys differ"
 				(sort expectedkeys) (sort unusedkeys)

From 56bc3e95cabb85e5f23e30b453f90438c33efbb8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 02:02:46 -0400
Subject: [PATCH 1705/8313] refactor some boilerplate

---
 Command.hs            | 10 ++++++++++
 Command/Add.hs        |  8 ++++----
 Command/ConfigList.hs |  2 +-
 Command/Describe.hs   |  4 ++--
 Command/Drop.hs       | 12 ++++++------
 Command/DropKey.hs    |  7 +++----
 Command/DropUnused.hs |  8 ++++----
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  6 +++---
 Command/FromKey.hs    |  4 ++--
 Command/Fsck.hs       |  6 +++---
 Command/Get.hs        |  9 ++++-----
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  6 +++---
 Command/InitRemote.hs |  4 ++--
 Command/Lock.hs       |  4 ++--
 Command/Map.hs        |  2 +-
 Command/Migrate.hs    | 11 +++++------
 Command/Move.hs       | 22 +++++++++++-----------
 Command/PreCommit.hs  |  4 ++--
 Command/Semitrust.hs  |  4 ++--
 Command/SetKey.hs     |  4 ++--
 Command/Trust.hs      |  4 ++--
 Command/Unannex.hs    |  8 ++++----
 Command/Uninit.hs     |  4 ++--
 Command/Unlock.hs     |  4 ++--
 Command/Untrust.hs    |  4 ++--
 Command/Unused.hs     |  4 ++--
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  6 +++---
 31 files changed, 93 insertions(+), 86 deletions(-)

diff --git a/Command.hs b/Command.hs
index 9c908b8004..d9e94a2f36 100644
--- a/Command.hs
+++ b/Command.hs
@@ -65,12 +65,22 @@ data Command = Command {
 	cmdusesrepo :: Bool
 }
 
+{- Most commands operate on files in a git repo. -}
 repoCommand :: String -> String -> [CommandSeek] -> String -> Command
 repoCommand n p s d = Command n p s d True
 
+{- Others can run anywhere. -}
 standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
 standaloneCommand n p s d = Command n p s d False
 
+{- For start and perform stages to indicate what step to run next. -}
+next :: a -> Annex (Maybe a)
+next a = return $ Just a
+
+{- Or to indicate nothing needs to be done. -}
+stop :: Annex (Maybe a)
+stop = return Nothing
+
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
diff --git a/Command/Add.hs b/Command/Add.hs
index b532ab045d..29a1518e84 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -34,19 +34,19 @@ start :: CommandStartBackendFile
 start pair@(file, _) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if (isSymbolicLink s) || (not $ isRegularFile s)
-		then return Nothing
+		then stop
 		else do
 			showStart "add" file
-			return $ Just $ perform pair
+			next $ perform pair
 
 perform :: BackendFile -> CommandPerform
 perform (file, backend) = do
 	stored <- Backend.storeFileKey file backend
 	case stored of
-		Nothing -> return Nothing
+		Nothing -> stop
 		Just (key, _) -> do
 			moveAnnex key file
-			return $ Just $ cleanup file key
+			next $ cleanup file key
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 476d73cfbb..d8dbff03af 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -25,4 +25,4 @@ start = do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	liftIO $ putStrLn $ "annex.uuid=" ++ u
-	return Nothing
+	stop
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 9e98a81437..dcabef7fbf 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -29,9 +29,9 @@ start params = notBareRepo $ do
 	
 	showStart "describe" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u description
+	next $ perform u description
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
 	describeUUID u description
-	return $ Just $ Command.Init.cleanup
+	next $ Command.Init.cleanup
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 52b724e62b..05c956fddf 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -29,11 +29,11 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 start :: CommandStartAttrFile
 start (file, attr) = isAnnexed file $ \(key, backend) -> do
 	inbackend <- Backend.hasKey key
-	if not inbackend
-		then return Nothing
-		else do
+	if inbackend
+		then do
 			showStart "drop" file
-			return $ Just $ perform key backend numcopies
+			next $ perform key backend numcopies
+		else stop
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
@@ -41,8 +41,8 @@ perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform
 perform key backend numcopies = do
 	success <- Backend.removeKey backend key numcopies
 	if success
-		then return $ Just $ cleanup key
-		else return Nothing
+		then next $ cleanup key
+		else stop
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 4c6f1ab2e1..780fe0adf0 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -26,20 +26,19 @@ start key = do
 	present <- inAnnex key
 	force <- Annex.getState Annex.force
 	if not present
-		then return Nothing
+		then stop
 		else if not force
 			then error "dropkey is can cause data loss; use --force if you're sure you want to do this"
 			else do
 				showStart "dropkey" (show key)
-				return $ Just $ perform key
+				next $ perform key
 
 perform :: Key -> CommandPerform
 perform key = do
 	removeAnnex key
-	return $ Just $ cleanup key
+	next $ cleanup key
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
 	logStatus key ValueMissing
 	return True
-
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index b129235e1d..861c78c905 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -49,13 +49,13 @@ start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 	, (unusedtmp, performOther gitAnnexTmpLocation)
 	]
 	where
-		search [] = return Nothing
+		search [] = stop
 		search ((m, a):rest) = do
 			case M.lookup s m of
 				Nothing -> search rest
 				Just key -> do
 					showStart "dropunused" s
-					return $ Just $ a key
+					next $ a key
 
 perform :: Key -> CommandPerform
 perform key = do
@@ -64,7 +64,7 @@ perform key = do
 		Just name -> do
 			r <- Remote.byName name
 			showNote $ "from " ++ Remote.name r ++ "..."
-			return $ Just $ Command.Move.fromCleanup r True key
+			next $ Command.Move.fromCleanup r True key
 		_ -> do
 			backend <- keyBackend key
 			Command.Drop.perform key backend (Just 0) -- force drop
@@ -75,7 +75,7 @@ performOther filespec key = do
 	let f = filespec g key
 	e <- liftIO $ doesFileExist f
 	when e $ liftIO $ removeFile f
-	return $ Just $ return True
+	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
diff --git a/Command/Find.hs b/Command/Find.hs
index 6a6ae29787..eecf3cd7da 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -25,4 +25,4 @@ start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
 	exists <- inAnnex key
 	when exists $ liftIO $ putStrLn file
-	return Nothing
+	stop
diff --git a/Command/Fix.hs b/Command/Fix.hs
index d898ce517d..60627e9df0 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -30,17 +30,17 @@ start file = isAnnexed file $ \(key, _) -> do
 	link <- calcGitLink file key
 	l <- liftIO $ readSymbolicLink file
 	if link == l
-		then return Nothing
+		then stop
 		else do
 			showStart "fix" file
-			return $ Just $ perform file link
+			next $ perform file link
 
 perform :: FilePath -> FilePath -> CommandPerform
 perform file link = do
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 	liftIO $ removeFile file
 	liftIO $ createSymbolicLink link file
-	return $ Just $ cleanup file
+	next $ cleanup file
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index eadaa13e1f..ca61094eb4 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -34,7 +34,7 @@ start file = notBareRepo $ do
 	unless inbackend $ error $
 		"key ("++keyName key++") is not present in backend"
 	showStart "fromkey" file
-	return $ Just $ perform file
+	next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
@@ -42,7 +42,7 @@ perform file = do
 	link <- calcGitLink file key
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 	liftIO $ createSymbolicLink link file
-	return $ Just $ cleanup file
+	next $ cleanup file
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 20ef2c8083..adfd702de7 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -31,7 +31,7 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 start :: CommandStartAttrFile
 start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	return $ Just $ perform key file backend numcopies
+	next $ perform key file backend numcopies
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
@@ -42,8 +42,8 @@ perform key file backend numcopies = do
 	locationlogok <- verifyLocationLog key file
 	backendok <- Backend.fsckKey backend key (Just file) numcopies
 	if locationlogok && backendok
-		then return $ Just $ return True
-		else return Nothing
+		then next $ return True
+		else stop
 
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
diff --git a/Command/Get.hs b/Command/Get.hs
index 0463dccb05..90c0540960 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -25,15 +25,14 @@ start :: CommandStartString
 start file = isAnnexed file $ \(key, backend) -> do
 	inannex <- inAnnex key
 	if inannex
-		then return Nothing
+		then stop
 		else do
 			showStart "get" file
-			return $ Just $ perform key backend
+			next $ perform key backend
 
 perform :: Key -> Backend Annex -> CommandPerform
 perform key backend = do
 	ok <- getViaTmp key (Backend.retrieveKeyFile backend key)
 	if ok
-		then return $ Just $ return True -- no cleanup needed
-		else return Nothing
-
+		then next $ return True -- no cleanup needed
+		else stop
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index a7e2ecff60..b5b59ccf7d 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -24,5 +24,5 @@ start :: CommandStartKey
 start key = do
 	present <- inAnnex key
 	if present
-		then return Nothing
+		then stop
 		else liftIO $ exitFailure
diff --git a/Command/Init.hs b/Command/Init.hs
index cca2e8faef..668b5c87d6 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -35,7 +35,7 @@ start description = do
 	when (null description) $
 		error "please specify a description of this repository\n"
 	showStart "init" description
-	return $ Just $ perform description
+	next $ perform description
 
 perform :: String -> CommandPerform
 perform description = do
@@ -48,12 +48,12 @@ perform description = do
 				"This is a bare repository, so its description cannot be committed.\n" ++
 				"To record the description, run this command in a clone of this repository:\n" ++
 				"   git annex describe " ++ show u ++ " " ++ show description ++ "\n\n"
-			return $ Just $ return True
+			next $ return True
 		else do
 			describeUUID u description
 			liftIO $ gitAttributesWrite g
 			gitPreCommitHookWrite g
-			return $ Just cleanup
+			next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 4c2fc3a078..eda50ee5de 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -39,7 +39,7 @@ start params = notBareRepo $ do
 	t <- findType fullconfig
 	
 	showStart "initremote" name
-	return $ Just $ perform t u $ M.union config c
+	next $ perform t u $ M.union config c
 
 	where
 		ws = words params
@@ -49,7 +49,7 @@ start params = notBareRepo $ do
 perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform
 perform t u c = do
 	c' <- RemoteClass.setup t u c
-	return $ Just $ cleanup u c'
+	next $ cleanup u c'
 
 cleanup :: UUID -> RemoteClass.RemoteConfig -> CommandCleanup
 cleanup u c = do
diff --git a/Command/Lock.hs b/Command/Lock.hs
index cdbc560194..1ae4882272 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -26,7 +26,7 @@ seek = [withFilesUnlocked start]
 start :: CommandStartBackendFile
 start (file, _) = do
 	showStart "lock" file
-	return $ Just $ perform file
+	next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
@@ -36,4 +36,4 @@ perform file = do
 	liftIO $ Git.run g "reset" [Params "-q --", File file]
 	-- checkout the symlink
 	liftIO $ Git.run g "checkout" [Param "--", File file]
-	return $ Just $ return True -- no cleanup needed
+	next $ return True -- no cleanup needed
diff --git a/Command/Map.hs b/Command/Map.hs
index 2325c87e14..3c94fc75b5 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -45,7 +45,7 @@ start = do
 	showLongNote $ "running: dot -Tx11 " ++ file
 	showProgress
 	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]
-	return $ Just $ return $ Just $ return r
+	next $ next $ return r
 	where
 		file = "map.dot"
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 0d21fcbdf9..35855d5270 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -35,9 +35,8 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	if (newbackend /= oldbackend || upgradable) && exists
 		then do
 			showStart "migrate" file
-			return $ Just $ perform file key newbackend
-		else
-			return Nothing
+			next $ perform file key newbackend
+		else stop
 	where
 		choosebackend Nothing = do
 			backends <- Backend.list
@@ -55,7 +54,7 @@ perform file oldkey newbackend = do
 	let src = gitAnnexLocation g oldkey
 	stored <- Backend.storeFileKey src $ Just newbackend
 	case stored of
-		Nothing -> return Nothing
+		Nothing -> stop
 		Just (newkey, _) -> do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
@@ -68,5 +67,5 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
-					return $ Just $ Command.Add.cleanup file newkey
-				else return Nothing
+					next $ Command.Add.cleanup file newkey
+				else stop
diff --git a/Command/Move.hs b/Command/Move.hs
index 476bf866a0..623003e47a 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -73,10 +73,10 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 	u <- getUUID g
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
-		then return Nothing -- not here, so nothing to do
+		then stop -- not here, so nothing to do
 		else do
 			showAction move file
-			return $ Just $ toPerform dest move key
+			next $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
 	-- Checking the remote is expensive, so not done in the start step.
@@ -92,14 +92,14 @@ toPerform dest move key = do
 	case isthere of
 		Left err -> do
 			showNote $ show err
-			return Nothing
+			stop
 		Right False -> do
 			showNote $ "to " ++ Remote.name dest ++ "..."
 			ok <- Remote.storeKey dest key
 			if ok
-				then return $ Just $ toCleanup dest move key
-				else return Nothing -- failed
-		Right True -> return $ Just $ toCleanup dest move key
+				then next $ toCleanup dest move key
+				else stop -- failed
+		Right True -> next $ toCleanup dest move key
 toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
 	remoteHasKey dest key True
@@ -119,21 +119,21 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 	u <- getUUID g
 	(remotes, _) <- Remote.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
-		then return Nothing
+		then stop
 		else do
 			showAction move file
-			return $ Just $ fromPerform src move key
+			next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = do
 	ishere <- inAnnex key
 	if ishere
-		then return $ Just $ fromCleanup src move key
+		then next $ fromCleanup src move key
 		else do
 			showNote $ "from " ++ Remote.name src ++ "..."
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			if ok
-				then return $ Just $ fromCleanup src move key
-				else return Nothing -- fail
+				then next $ fromCleanup src move key
+				else stop -- fail
 fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 fromCleanup src True key = do
 	ok <- Remote.removeKey src key
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 1db40f75fa..d7f2487137 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -27,13 +27,13 @@ seek = [withFilesToBeCommitted Command.Fix.start,
 	withFilesUnlockedToBeCommitted start]
 
 start :: CommandStartBackendFile
-start pair = return $ Just $ perform pair
+start pair = next $ perform pair
 
 perform :: BackendFile -> CommandPerform
 perform pair@(file, _) = do
 	ok <- doCommand $ Command.Add.start pair
 	if ok
-		then return $ Just $ cleanup file
+		then next $ cleanup file
 		else error $ "failed to add " ++ file ++ "; canceling commit"
 
 cleanup :: FilePath -> CommandCleanup
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index e64d418f83..fc1bcbbcdc 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -24,9 +24,9 @@ start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "semitrust" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u
+	next $ perform u
 
 perform :: UUID -> CommandPerform
 perform uuid = do
 	trustSet uuid SemiTrusted
-	return $ Just $ return True
+	next $ return True
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 6f6078e4ba..dbad148b25 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -26,7 +26,7 @@ seek = [withTempFile start]
 start :: CommandStartString
 start file = do
 	showStart "setkey" file
-	return $ Just $ perform file
+	next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
@@ -40,7 +40,7 @@ perform file = do
 				boolSystem "mv" [File file, File dest]
 			else return True
 	if ok
-		then return $ Just $ cleanup
+		then next cleanup
 		else error "mv failed!"
 
 cleanup :: CommandCleanup
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 05505cd045..ef03828c23 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -24,9 +24,9 @@ start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "trust" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u
+	next $ perform u
 
 perform :: UUID -> CommandPerform
 perform uuid = do
 	trustSet uuid Trusted
-	return $ Just $ return True
+	next $ return True
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 94db500c68..0a5381d562 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -43,16 +43,16 @@ start file = isAnnexed file $ \(key, backend) -> do
 				Annex.changeState $ \s -> s { Annex.force = True }
 
 			showStart "unannex" file
-			return $ Just $ perform file key backend
-		else return Nothing
+			next $ perform file key backend
+		else stop
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file key backend = do
 	-- force backend to always remove
 	ok <- Backend.removeKey backend key (Just 0)
 	if ok
-		then return $ Just $ cleanup file key
-		else return Nothing
+		then next $ cleanup file key
+		else stop
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ee0cbde6b3..d3d7ac3398 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -30,7 +30,7 @@ seek = [withFilesInGit Command.Unannex.start, withNothing start]
 start :: CommandStartNothing
 start = do
 	showStart "uninit" ""
-	return $ Just $ perform
+	next perform
 
 perform :: CommandPerform
 perform = do
@@ -39,7 +39,7 @@ perform = do
 	gitPreCommitHookUnWrite g
 	liftIO $ gitAttributesUnWrite g
 
-	return $ Just $ return True
+	next $ return True
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index bf593e1e99..d65579ec73 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -34,7 +34,7 @@ seek = [withFilesInGit start]
 start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "unlock" file
-	return $ Just $ perform file key
+	next $ perform file key
 
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
@@ -52,5 +52,5 @@ perform dest key = do
         if ok
                 then do
 			liftIO $ allowWrite dest
-			return $ Just $ return True
+			next $ return True
                 else error "copy failed!"
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 311ec6eeb7..ebe9c31b3b 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -24,9 +24,9 @@ start :: CommandStartString
 start name = notBareRepo $ do
 	showStart "untrust" name
 	u <- Remote.nameToUUID name
-	return $ Just $ perform u
+	next $ perform u
 
 perform :: UUID -> CommandPerform
 perform uuid = do
 	trustSet uuid UnTrusted
-	return $ Just $ return True
+	next $ return True
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 67f10581d6..7570dfe90a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -37,7 +37,7 @@ seek = [withNothing start]
 start :: CommandStartNothing
 start = notBareRepo $ do
 	showStart "unused" ""
-	return $ Just perform
+	next perform
 
 perform :: CommandPerform
 perform = do
@@ -47,7 +47,7 @@ perform = do
 			r <- Remote.byName name
 			checkRemoteUnused r
 		_ -> checkUnused
-	return $ Just $ return True
+	next $ return True
 
 checkUnused :: Annex ()
 checkUnused = do
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 880a5324f7..b3c0468039 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -24,4 +24,4 @@ start = do
 	showStart "upgrade" ""
 	r <- upgrade
 	checkVersion
-	return $ Just $ return $ Just $ return r
+	next $ next $ return r
diff --git a/Command/Version.hs b/Command/Version.hs
index 2b294c80be..755b95acca 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -28,6 +28,6 @@ start = do
 	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
 	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
 	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
-	return Nothing
+	stop
 	where
 		vs l = join " " l
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 599df44676..2e0fa15f6f 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -26,7 +26,7 @@ seek = [withFilesInGit start]
 start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "whereis" file
-	return $ Just $ perform key
+	next $ perform key
 
 perform :: Key -> CommandPerform
 perform key = do
@@ -35,12 +35,12 @@ perform key = do
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
 	if null $ uuids
-		then return Nothing
+		then stop
 		else do
 			pp <- prettyPrintUUIDs uuids
 			showLongNote $ pp
 			showProgress	
-			return $ Just $ return True
+			next $ return True
 	where
 		copiesplural 1 = "copy"
 		copiesplural _ = "copies"

From efa7f544050c0d5be6bc1b0fc0125278e475c213 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 02:12:17 -0400
Subject: [PATCH 1706/8313] cleanup

---
 Command.hs | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/Command.hs b/Command.hs
index d9e94a2f36..0e3958c180 100644
--- a/Command.hs
+++ b/Command.hs
@@ -58,20 +58,20 @@ type CommandSeekNothing = CommandStart -> CommandSeek
 type CommandStartNothing = CommandStart
 
 data Command = Command {
+	cmdusesrepo :: Bool,
 	cmdname :: String,
 	cmdparams :: String,
 	cmdseek :: [CommandSeek],
-	cmddesc :: String,
-	cmdusesrepo :: Bool
+	cmddesc :: String
 }
 
 {- Most commands operate on files in a git repo. -}
 repoCommand :: String -> String -> [CommandSeek] -> String -> Command
-repoCommand n p s d = Command n p s d True
+repoCommand = Command True
 
 {- Others can run anywhere. -}
 standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
-standaloneCommand n p s d = Command n p s d False
+standaloneCommand = Command False
 
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
@@ -102,13 +102,8 @@ doCommand start = do
 					return False
 				Just cleanup -> do
 					c <- cleanup
-					if c
-						then do
-							showEndOk
-							return True
-						else do
-							showEndFail
-							return False
+					if c then showEndOk else showEndFail
+					return c
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = do

From cad0e1c8b7eb21f8dceca8dd9fa3bc1d1aa7eabd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 02:49:43 -0400
Subject: [PATCH 1707/8313] simplified a bunch of Maybe handling

---
 Backend.hs            | 29 ++++++++++-------------------
 Base64.hs             |  6 ++----
 Command.hs            | 26 +++++++-------------------
 Command/DropUnused.hs |  9 ++++-----
 Command/InitRemote.hs | 17 +++++++++--------
 Command/Map.hs        | 16 +++++-----------
 Command/Unused.hs     | 14 ++++++--------
 Content.hs            | 10 ++++------
 Crypto.hs             |  6 ++----
 Dot.hs                |  5 +----
 GitRepo.hs            | 13 ++++---------
 LocationLog.hs        |  4 +---
 Remote/Bup.hs         |  5 ++---
 Remote/Directory.hs   |  5 ++---
 Remote/Encryptable.hs |  9 ++++-----
 Remote/Hook.hs        | 15 ++++++---------
 Remote/Rsync.hs       |  5 ++---
 Remote/S3real.hs      | 16 ++++++----------
 Utility.hs            | 11 ++++-------
 19 files changed, 81 insertions(+), 140 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index aec87ce665..6140664cee 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -76,10 +76,9 @@ list = do
 
 {- Looks up a backend in a list. May fail if unknown. -}
 lookupBackendName :: [Backend Annex] -> String -> Backend Annex
-lookupBackendName bs s =
-	case maybeLookupBackendName bs s of
-		Just b -> b
-		Nothing -> error $ "unknown backend " ++ s
+lookupBackendName bs s = maybe unknown id $ maybeLookupBackendName bs s
+	where
+		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex)
 maybeLookupBackendName bs s =
 	if 1 /= length matches
@@ -91,23 +90,18 @@ maybeLookupBackendName bs s =
 storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
 storeFileKey file trybackend = do
 	bs <- list
-	let bs' = case trybackend of
-		Nothing -> bs
-		Just backend -> backend:bs
+	let bs' = maybe bs (:bs) trybackend
 	storeFileKey' bs' file
 storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
 storeFileKey' [] _ = return Nothing
-storeFileKey' (b:bs) file = do
-	result <- (B.getKey b) file
-	case result of
-		Nothing -> nextbackend
-		Just key -> do
+storeFileKey' (b:bs) file = maybe nextbackend store =<< (B.getKey b) file
+	where
+		nextbackend = storeFileKey' bs file
+		store key = do
 			stored <- (B.storeFileKey b) file key
 			if (not stored)
 				then nextbackend
 				else return $ Just (key, b)
-	where
-		nextbackend = storeFileKey' bs file
 
 {- Attempts to retrieve an key from one of the backends, saving it to
  - a specified location. -}
@@ -148,11 +142,8 @@ lookupFile file = do
 		getsymlink = do
 			l <- readSymbolicLink file
 			return $ takeFileName l
-		makekey bs l =
-			case fileKey l of
-				Just k -> makeret k l bs
-				Nothing -> return Nothing
-		makeret k l bs =
+		makekey bs l = maybe (return Nothing) (makeret bs l) (fileKey l)
+		makeret bs l k =
 			case maybeLookupBackendName bs bname of
 					Just backend -> return $ Just (k, backend)
 					Nothing -> do
diff --git a/Base64.hs b/Base64.hs
index cc6346b415..153049751d 100644
--- a/Base64.hs
+++ b/Base64.hs
@@ -14,7 +14,5 @@ toB64 :: String -> String
 toB64 = encode . s2w8
 
 fromB64 :: String -> String
-fromB64 s =
-	case decode s of
-		Nothing -> error "bad base64 encoded data"
-		Just ws -> w82s ws
+fromB64 s = maybe bad w82s $ decode s
+	where bad = error "bad base64 encoded data"
diff --git a/Command.hs b/Command.hs
index 0e3958c180..c6c1fe5c55 100644
--- a/Command.hs
+++ b/Command.hs
@@ -14,6 +14,7 @@ import Control.Monad (filterM, liftM, when)
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
+import Data.Maybe
 
 import Types
 import qualified Backend
@@ -106,18 +107,10 @@ doCommand start = do
 					return c
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
-notAnnexed file a = do
-	r <- Backend.lookupFile file
-	case r of
-		Just _ -> return Nothing
-		Nothing -> a
+notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
 isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
-isAnnexed file a = do
-	r <- Backend.lookupFile file
-	case r of
-		Just v -> a v
-		Nothing -> return Nothing
+isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
@@ -183,9 +176,7 @@ withFilesUnlocked' typechanged a params = do
 withKeys :: CommandSeekKeys
 withKeys a params = return $ map a $ map parse params
 	where
-		parse p = case readKey p of
-			Just k -> k
-			Nothing -> error "bad key"
+		parse p = maybe (error "bad key") id $ readKey p
 withTempFile :: CommandSeekStrings
 withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
@@ -206,9 +197,7 @@ filterFiles l = do
 		else return $ filter (notExcluded $ wildsRegex exclude) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
-		notExcluded r f = case match r f [] of
-			Nothing -> True
-			Just _ -> False
+		notExcluded r f = isJust $ match r f []
 
 wildsRegex :: [String] -> Regex
 wildsRegex ws = compile regex []
@@ -257,11 +246,10 @@ cmdlineKey  = do
 	case k of
 		Nothing -> nokey
 		Just "" -> nokey
-		Just kstring -> case readKey kstring of
-			Nothing -> error "bad key"
-			Just key -> return key
+		Just kstring -> maybe badkey return $ readKey kstring
 	where
 		nokey = error "please specify the key with --key"
+		badkey = error "bad key"
 
 {- Given an original list of files, and an expanded list derived from it,
  - ensures that the original list's ordering is preserved. 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 861c78c905..965a99ed56 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -58,14 +58,13 @@ start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 					next $ a key
 
 perform :: Key -> CommandPerform
-perform key = do
-	from <- Annex.getState Annex.fromremote
-	case from of
-		Just name -> do
+perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
+	where
+		dropremote name = do
 			r <- Remote.byName name
 			showNote $ "from " ++ Remote.name r ++ "..."
 			next $ Command.Move.fromCleanup r True key
-		_ -> do
+		droplocal = do
 			backend <- keyBackend key
 			Command.Drop.perform key backend (Just 0) -- force drop
 
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index eda50ee5de..261ccdc8b7 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -68,11 +68,11 @@ cleanup u c = do
 findByName :: String -> Annex (UUID, RemoteClass.RemoteConfig)
 findByName name = do
 	m <- Remote.readRemoteLog
-	case findByName' name m of
-		Just i -> return i
-		Nothing -> do
+	maybe generate return $ findByName' name m
+	where
+		generate = do
 			uuid <- liftIO $ genUUID
-			return $ (uuid, M.insert nameKey name M.empty)
+			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID RemoteClass.RemoteConfig -> Maybe (UUID, RemoteClass.RemoteConfig)
 findByName' n m = if null matches then Nothing else Just $ head matches
@@ -86,12 +86,13 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 
 {- find the specified remote type -}
 findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex)
-findType config = 
-	case M.lookup typeKey config of
-		Nothing -> error "Specify the type of remote with type="
-		Just s -> case filter (\i -> RemoteClass.typename i == s) Remote.remoteTypes of
+findType config = maybe unspecified specified $ M.lookup typeKey config
+	where
+		unspecified = error "Specify the type of remote with type="
+		specified s = case filter (findtype s) Remote.remoteTypes of
 			[] -> error $ "Unknown remote type " ++ s
 			(t:_) -> return t
+		findtype s i = RemoteClass.typename i == s
 
 {- The name of a configured remote is stored in its config using this key. -}
 nameKey :: String
diff --git a/Command/Map.hs b/Command/Map.hs
index 3c94fc75b5..7a9121b69b 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -84,10 +84,7 @@ repoName umap r
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
-		fallback =
-			case (Git.repoRemoteName r) of
-				Just n -> n
-				Nothing -> "unknown"
+		fallback = maybe "unknown" id $ Git.repoRemoteName r
 
 {- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
@@ -121,13 +118,10 @@ edge umap fullinfo from to =
 		{- Only name an edge if the name is different than the name
 		 - that will be used for the destination node, and is
 		 - different from its hostname. (This reduces visual clutter.) -}
-		edgename =
-			case (Git.repoRemoteName to) of
-				Nothing -> Nothing
-				Just n ->
-					if (n == repoName umap fullto || n == hostname fullto)
-						then Nothing
-						else Just n
+		edgename = maybe Nothing calcname $ Git.repoRemoteName to
+		calcname n
+			| n == repoName umap fullto || n == hostname fullto = Nothing
+			| otherwise = Just n
 
 unreachable :: String -> String
 unreachable = Dot.fillColor "red"
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7570dfe90a..a2e1c86de1 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -41,12 +41,7 @@ start = notBareRepo $ do
 
 perform :: CommandPerform
 perform = do
-	from <- Annex.getState Annex.fromremote
-	case from of
-		Just name -> do
-			r <- Remote.byName name
-			checkRemoteUnused r
-		_ -> checkUnused
+	maybe checkUnused checkRemoteUnused =<< Annex.getState Annex.fromremote
 	next $ return True
 
 checkUnused :: Annex ()
@@ -63,8 +58,11 @@ checkUnused = do
 			writeUnusedFile file unusedlist
 			return $ length l
 
-checkRemoteUnused :: Remote.Remote Annex -> Annex ()
-checkRemoteUnused r = do
+checkRemoteUnused :: String -> Annex ()
+checkRemoteUnused name = checkRemoteUnused' =<< Remote.byName name
+
+checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
+checkRemoteUnused' r = do
 	g <- Annex.gitRepo
 	showNote $ "checking for unused data on " ++ Remote.name r ++ "..."
 	referenced <- getKeysReferenced
diff --git a/Content.hs b/Content.hs
index ade936da37..9040383be9 100644
--- a/Content.hs
+++ b/Content.hs
@@ -57,11 +57,11 @@ calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
 	g <- Annex.gitRepo
 	cwd <- liftIO $ getCurrentDirectory
-	let absfile = case absNormPath cwd file of
-		Just f -> f
-		Nothing -> error $ "unable to normalize " ++ file
+	let absfile = maybe whoops id $ absNormPath cwd file
 	return $ relPathDirToFile (parentDir absfile) 
 			(Git.workTree g)  ".git"  annexLocation key
+	where
+		whoops = error $ "unable to normalize " ++ file
 
 {- Updates the LocationLog when a key's presence changes in the current
  - repository.
@@ -148,9 +148,7 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
 	r <- getConfig g "diskreserve" ""
-	let reserve = case readSize dataUnits r of
-		Nothing -> megabyte
-		Just v -> v
+	let reserve = maybe megabyte id $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
diff --git a/Crypto.hs b/Crypto.hs
index 53cd48dd59..42f1389507 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -238,10 +238,8 @@ configKeyIds c = do
 		keyIdField s = (split ":" s) !! 4
 
 configGet :: RemoteConfig -> String -> String
-configGet c key =
-	case M.lookup key c of
-		Just v -> v
-		Nothing -> error $ "missing " ++ key ++ " in remote config"
+configGet c key = maybe missing id $ M.lookup key c
+	where missing = error $ "missing " ++ key ++ " in remote config"
 
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
diff --git a/Dot.hs b/Dot.hs
index 592b21f691..deba10201e 100644
--- a/Dot.hs
+++ b/Dot.hs
@@ -20,10 +20,7 @@ graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid desc = indent $
-	case desc of
-		Nothing -> edge
-		Just d -> label d edge
+graphEdge fromid toid desc = indent $ maybe edge (\d -> label d edge) desc
 	where
 		edge = quote fromid ++ " -> " ++ quote toid
 
diff --git a/GitRepo.hs b/GitRepo.hs
index 49024abe0e..b20ff7db3a 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -122,9 +122,8 @@ repoFromUrl url
 	| startswith "file://" url = repoFromAbsPath $ uriPath u
 	| otherwise = return $ newFrom $ Url u
 		where
-			u = case (parseURI url) of
-				Just v -> v
-				Nothing -> error $ "bad url " ++ url
+			u = maybe bad id $ parseURI url
+			bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
 repoFromUnknown :: Repo
@@ -264,9 +263,7 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 		absrepo = case (absNormPath "/" d) of
 			Just f -> addTrailingPathSeparator f
 			Nothing -> error $ "bad repo" ++ repoDescribe repo
-		absfile c = case (secureAbsNormPath c file) of
-			Just f -> f
-			Nothing -> file
+		absfile c = maybe file id $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
 workTreeFile repo _ = assertLocal repo $ error "internal"
 
@@ -352,9 +349,7 @@ reap :: IO ()
 reap = do
 	-- throws an exception when there are no child processes
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
-	case r of
-		Nothing -> return ()
-		Just _ -> reap
+	maybe (return ()) (const reap) r
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/LocationLog.hs b/LocationLog.hs
index e0ccb642b3..6759b47fec 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -71,9 +71,7 @@ instance Read LogLine where
 	-- Such lines have a status of Undefined.
 	readsPrec _ string = 
 		if length w == 3
-			then case pdate of
-				Just v -> good v
-				Nothing -> bad
+			then maybe bad good pdate
 			else bad
 		where
 			w = words string
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 0aaff06b25..d2b771bf77 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -68,9 +68,8 @@ gen r u c = do
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 bupSetup u c = do
 	-- verify configuration is sane
-	let buprepo = case M.lookup "buprepo" c of
-		Nothing -> error "Specify buprepo="
-		Just r -> r
+	let buprepo = maybe (error "Specify buprepo=") id $
+		M.lookup "buprepo" c
 	c' <- encryptionSetup c
 
 	-- bup init will create the repository.
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index c680d61212..0cd3760d63 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -60,9 +60,8 @@ gen r u c = do
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 directorySetup u c = do
 	-- verify configuration is sane
-	let dir = case M.lookup "directory" c of
-		Nothing -> error "Specify directory="
-		Just d -> d
+	let dir = maybe (error "Specify directory=") id $
+		M.lookup "directory" c
 	e <- liftIO $ doesDirectoryExist dir
 	when (not e) $ error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index 31ef1f37a7..f9b388c8ae 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -73,11 +73,10 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 {- Gets encryption Cipher. The decrypted Cipher is cached in the Annex
  - state. -}
 remoteCipher :: RemoteConfig -> Annex (Maybe Cipher)
-remoteCipher c = do
-	cache <- Annex.getState Annex.cipher
-	case cache of
-		Just cipher -> return $ Just cipher
-		Nothing -> case extractCipher c of
+remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
+	where
+		cached cipher = return $ Just cipher
+		expensive = case extractCipher c of
 			Nothing -> return Nothing
 			Just encipher -> do
 				showNote "gpg"
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ba38355caf..7f2d5dbee2 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -61,9 +61,8 @@ gen r u c = do
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 hookSetup u c = do
-	let hooktype = case M.lookup "hooktype" c of
-		Nothing -> error "Specify hooktype="
-		Just r -> r
+	let hooktype = maybe (error "Specify hooktype=") id $
+		M.lookup "hooktype" c
 	c' <- encryptionSetup c
 	gitConfigSpecialRemote u c' "hooktype" hooktype
 	return c'
@@ -94,14 +93,12 @@ lookupHook hooktype hook =do
 		hookname =  hooktype ++ "-" ++ hook ++ "-hook"
 
 runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
-runHook hooktype hook k f a = do
-	command <- lookupHook hooktype hook
-	case command of
-		Nothing -> return False
-		Just c -> do
+runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook
+	where
+		run command = do
 			showProgress -- make way for hook output
 			res <- liftIO $ boolSystemEnv
-				"sh" [Param "-c", Param c] $ hookEnv k f
+				"sh" [Param "-c", Param command] $ hookEnv k f
 			if res
 				then a
 				else do
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 682c961748..c15ab37a75 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -82,9 +82,8 @@ genRsyncOpts r = do
 rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 rsyncSetup u c = do
 	-- verify configuration is sane
-	let url = case M.lookup "rsyncurl" c of
-		Nothing -> error "Specify rsyncurl="
-		Just d -> d
+	let url = maybe (error "Specify rsyncurl=") id $
+		M.lookup "rsyncurl" c
 	c' <- encryptionSetup c
 
 	-- The rsyncurl is stored in git config, not only in this remote's
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b0371eb5ea..eaa6590b19 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -123,11 +123,7 @@ storeHelper (conn, bucket) r k file = do
 	content <- liftIO $ L.readFile file
 	-- size is provided to S3 so the whole content does not need to be
 	-- buffered to calculate it
-	size <- case keySize k of
-		Just s -> return $ fromIntegral s
-		Nothing -> do
-			s <- liftIO $ getFileStatus file
-			return $ fileSize s
+	size <- maybe getsize (return . fromIntegral) $ keySize k
 	let object = setStorageClass storageclass $ 
 		S3Object bucket (show k) ""
 			[("Content-Length",(show size))] content
@@ -137,6 +133,9 @@ storeHelper (conn, bucket) r k file = do
 			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
+		getsize = do
+			s <- liftIO $ getFileStatus file
+			return $ fileSize s
 
 retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
 retrieve r k f = s3Action r False $ \(conn, bucket) -> do
@@ -201,11 +200,8 @@ bucketKey :: String -> Key -> S3Object
 bucketKey bucket k = S3Object bucket (show k) "" [] L.empty
 
 s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection
-s3ConnectionRequired c = do
-	conn <- s3Connection c
-	case conn of
-		Nothing -> error "Cannot connect to S3"
-		Just conn' -> return conn'
+s3ConnectionRequired c =
+	maybe (error "Cannot connect to S3") return =<< s3Connection c
 
 s3Connection :: RemoteConfig -> Annex (Maybe AWSConnection)
 s3Connection c = do
diff --git a/Utility.hs b/Utility.hs
index 0dab371045..44c8cdd650 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -165,9 +165,7 @@ prop_parentDir_basics dir
 dirContains :: FilePath -> FilePath -> Bool
 dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 	where
-		norm p = case (absNormPath p ".") of
-			Just r -> r
-			Nothing -> ""
+		norm p = maybe "" id $ absNormPath p "."
 		a' = norm a
 		b' = norm b
 
@@ -180,10 +178,9 @@ absPath file = do
 {- Converts a filename into a normalized, absolute path
  - from the specified cwd. -}
 absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file =
-	case absNormPath cwd file of
-		Just f -> f
-		Nothing -> error $ "unable to normalize " ++ file
+absPathFrom cwd file = maybe bad id $ absNormPath cwd file
+	where
+		bad = error $ "unable to normalize " ++ file
 
 {- Constructs a relative path from the CWD to a file.
  -

From dfdfbe5f22116b73da8ae439ac4a7ab8b662f8fe Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 09:16:53 +0000
Subject: [PATCH 1708/8313] Added a comment

---
 .../comment_9_2cee4f6bd6db7518fd61453c595162c6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment b/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
new file mode 100644
index 0000000000..2755cf3317
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 9"
+ date="2011-05-15T09:16:49Z"
+ content="""
+Just to make sure: How do I get $key? What I did was look at the path in the object store of the local repo and see if that exact same path & file existed in the remote.
+"""]]

From 3e15a8a791d15c166557fa18f240639891a8754f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 12:25:58 -0400
Subject: [PATCH 1709/8313] Maybe reduction pass 2

---
 Command.hs            | 20 ++++++--------------
 GitRepo.hs            | 29 ++++++++++++-----------------
 Messages.hs           |  4 ++++
 Remote/Encryptable.hs | 31 +++++++++++--------------------
 4 files changed, 33 insertions(+), 51 deletions(-)

diff --git a/Command.hs b/Command.hs
index c6c1fe5c55..4f835a3adc 100644
--- a/Command.hs
+++ b/Command.hs
@@ -91,20 +91,12 @@ prepCommand Command { cmdseek = seek } params = do
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
-doCommand start = do
-	s <- start
-	case s of
-		Nothing -> return True
-		Just perform -> do
-			p <- perform
-			case p of
-				Nothing -> do
-					showEndFail
-					return False
-				Just cleanup -> do
-					c <- cleanup
-					if c then showEndOk else showEndFail
-					return c
+doCommand = start
+	where
+		start   = stage $ maybe (return True) perform
+		perform = stage $ maybe (showEndFail >> return False) cleanup
+		cleanup = stage $ \r -> showEndResult r >> return r
+		stage a b = b >>= a
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
diff --git a/GitRepo.hs b/GitRepo.hs
index b20ff7db3a..3c5a1e129e 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -210,9 +210,9 @@ assertUrl repo action =
 				" not supported"
 
 configBare :: Repo -> Bool
-configBare repo = case Map.lookup "core.bare" $ config repo of
-	Just v -> configTrue v
-	Nothing -> error $ "it is not known if git repo " ++
+configBare repo = maybe unknown configTrue $ Map.lookup "core.bare" $ config repo
+	where
+		unknown = error $ "it is not known if git repo " ++
 			repoDescribe repo ++
 			" is a bare repository; config not read"
 
@@ -260,11 +260,10 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 	where
 		-- normalize both repo and file, so that repo
 		-- will be substring of file
-		absrepo = case (absNormPath "/" d) of
-			Just f -> addTrailingPathSeparator f
-			Nothing -> error $ "bad repo" ++ repoDescribe repo
+		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d
 		absfile c = maybe file id $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
+		bad = error $ "bad repo" ++ repoDescribe repo
 workTreeFile repo _ = assertLocal repo $ error "internal"
 
 {- Path of an URL repo. -}
@@ -627,23 +626,19 @@ expandTilde = expandt True
 
 {- Finds the current git repository, which may be in a parent directory. -}
 repoFromCwd :: IO Repo
-repoFromCwd = do
-	cwd <- getCurrentDirectory
-	top <- seekUp cwd isRepoTop
-	case top of
-		-- repoFromAbsPath is not used to avoid looking for
-		-- "dir.git" directories.
-		(Just dir) -> return $ newFrom $ Dir dir
-		Nothing -> error "Not in a git repository."
+repoFromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
+	where
+		makerepo = return . newFrom . Dir
+		norepo = error "Not in a git repository."
 
-seekUp :: FilePath -> (FilePath -> IO Bool) -> IO (Maybe FilePath)
-seekUp dir want = do
+seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
+seekUp want dir = do
 	ok <- want dir
 	if ok
 		then return (Just dir)
 		else case (parentDir dir) of
 			"" -> return Nothing
-			d -> seekUp d want
+			d -> seekUp want d
 
 isRepoTop :: FilePath -> IO Bool
 isRepoTop dir = do
diff --git a/Messages.hs b/Messages.hs
index 733638ce12..c44e44eea1 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -45,6 +45,10 @@ showEndOk = verbose $ liftIO $ putStrLn "ok"
 showEndFail :: Annex ()
 showEndFail = verbose $ liftIO $ putStrLn "\nfailed"
 
+showEndResult :: Bool -> Annex ()
+showEndResult True = showEndOk
+showEndResult False = showEndFail
+
 showErr :: (Show a) => a -> Annex ()
 showErr e = warning $ "git-annex: " ++ show e
 
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index f9b388c8ae..68ecfd01e6 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -54,21 +54,14 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 		cost = cost r + encryptedRemoteCostAdj
 	}
 	where
-		store k = do
-			v <- cipherKey c k
-			case v of
-				Nothing -> (storeKey r) k
-				Just x -> storeKeyEncrypted x k
-		retrieve k f = do
-			v <- cipherKey c k
-			case v of
-				Nothing -> (retrieveKeyFile r) k f
-				Just x -> retrieveKeyFileEncrypted x f
-		withkey a k = do
-			v <- cipherKey c k
-			case v of
-				Nothing -> a k
-				Just (_, k') -> a k'
+		store k = cip k >>= maybe
+			(storeKey r k)
+			(\x -> storeKeyEncrypted x k)
+		retrieve k f = cip k >>= maybe
+			(retrieveKeyFile r k f)
+			(\x -> retrieveKeyFileEncrypted x f)
+		withkey a k = cip k >>= maybe (a k) (a . snd)
+		cip = cipherKey c
 
 {- Gets encryption Cipher. The decrypted Cipher is cached in the Annex
  - state. -}
@@ -87,10 +80,8 @@ remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
 cipherKey Nothing _ = return Nothing
-cipherKey (Just c) k = do
-	cipher <- remoteCipher c
-	case cipher of	
-		Just ciphertext -> do
+cipherKey (Just c) k = remoteCipher c >>= maybe (return Nothing) encrypt
+	where
+		encrypt ciphertext = do
 			k' <- liftIO $ encryptKey ciphertext k
 			return $ Just (ciphertext, k')
-		Nothing -> return Nothing

From db540b746b0eb93fa76f8bec0bdb63dd9ea5a5e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 12:32:18 -0400
Subject: [PATCH 1710/8313] don't optimise test suite

This avoids needing to adjust -fspec-constr-count, which
continues to need high values (> 8 now) when building the test suite.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 01a1a6a54c..286c3a6e54 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,7 @@ install: all
 	fi
 
 test: $(bins)
-	if ! $(GHCMAKE) test; then \
+	if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
 		./test; \

From 7010b9e5c5c1519ba5d87d4c14bc36a01e298e1f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 May 2011 16:47:53 +0000
Subject: [PATCH 1711/8313] Added a comment

---
 .../comment_10_435f87d54052f264096a8f23e99eae06._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment b/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment
new file mode 100644
index 0000000000..ec24c478d8
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 10"
+ date="2011-05-15T16:47:53Z"
+ content="""
+The key is the basename of the symlink target.
+"""]]

From 9708940cedcbc4e9f0ce49826bdfb710cfb25e7f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 18:53:27 +0000
Subject: [PATCH 1712/8313] Added a comment

---
 ..._9be0aef403a002c1706d17deee45763c._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment b/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment
new file mode 100644
index 0000000000..7bc54573ed
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 11"
+ date="2011-05-15T18:53:26Z"
+ content="""
+It seems the objects are in the remote after all, but the remote is unaware of this fact. No idea where/why the remote lost that info, but.. Anyway, with the SHA backends, wouldn't it make sense to simply return \"OK\" and update the annex logs accordingly, no?
+
+Local:
+
+    % ls -l foo
+    lrwxrwxrwx 1 richih richih 312 Apr  3 01:18 foo -> .git/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    % 
+
+Remote:
+
+    % git-annex-shell recvkey  SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    git-annex-shell: key is already present in annex
+    % strace git-annex-shell recvkey /base/git-annex/fun SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491 2>&1 | grep SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    stat64(\"/base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491\", {st_mode=S_IFREG|0444, st_size=80781, ...}) = 0
+    % ls -l /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    -r--r--r-- 1 richih richih 80781 2011-04-01 12:44 /base/git-annex/fun/annex/objects/gG/VW/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491/SHA512-s80781--cef3966a19c7435acceb8fbfbff1feebe6decab7c81a0c197f00932cf9ef0eac330784cc3f0d211bd4acf56a6d16daaebe9b598aa4dfd5bfec73f4e6df3f0491
+    % 
+"""]]

From 6aab88fa251a14fbf31c7a8d80296c78db0ed048 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 15:27:49 -0400
Subject: [PATCH 1713/8313] more monadic operator use

---
 Annex.hs       |  4 +---
 Content.hs     |  3 +--
 LocationLog.hs |  6 ++----
 Remote.hs      | 12 +++++-------
 UUID.hs        |  6 ++----
 5 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 9915112a59..b2257281fd 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -86,9 +86,7 @@ getState c = liftM c get
  - Example: changeState (\s -> s { quiet = True })
  -}
 changeState :: (AnnexState -> AnnexState) -> Annex ()
-changeState a = do
-	state <- get
-	put (a state)
+changeState a = put . a =<< get
 
 {- Returns the git repository being acted on -}
 gitRepo :: Annex Git.Repo
diff --git a/Content.hs b/Content.hs
index 9040383be9..0758fcdb13 100644
--- a/Content.hs
+++ b/Content.hs
@@ -72,8 +72,7 @@ calcGitLink file key = do
  - updated instead. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	g <- Annex.gitRepo
-	u <- getUUID g
+	u <- getUUID =<< Annex.gitRepo
 	logStatusFor u key status
 
 {- Updates the LocationLog when a key's presence changes in a repository
diff --git a/LocationLog.hs b/LocationLog.hs
index 6759b47fec..b2d423cf99 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -150,16 +150,13 @@ mapLog m l =
 		then Map.insert u l m
 		else m
 	where
-		better = case Map.lookup u m of
-			Just l' -> (date l' <= date l)
-			Nothing -> True
+		better = maybe True (\l' -> date l' <= date l) $ Map.lookup u m
 		u = uuid l
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Git.Repo -> IO [Key]
 loggedKeys repo = do
-	let dir = gitStateDir repo
 	exists <- doesDirectoryExist dir
 	if exists
 		then do
@@ -172,3 +169,4 @@ loggedKeys repo = do
 		else return []
 	where
 		tryDirContents d = catch (dirContents d) (return . const [])
+		dir = gitStateDir repo
diff --git a/Remote.hs b/Remote.hs
index bbecdb9995..211168b153 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -75,10 +75,10 @@ genList = do
 			return rs'
 		else return rs
 	where
-		process m t = do
-			l <- enumerate t
-			l' <- filterM remoteNotIgnored l
-			mapM (gen m t) l'
+		process m t = 
+			enumerate t >>=
+			filterM remoteNotIgnored >>=
+			mapM (gen m t)
 		gen m t r = do
 			u <- getUUID r
 			generate t r u (M.lookup u m)
@@ -97,9 +97,7 @@ byName n = do
 
 {- Looks up a remote by name (or by UUID), and returns its UUID. -}
 nameToUUID :: String -> Annex UUID
-nameToUUID "." = do -- special case for current repo
-	g <- Annex.gitRepo
-	getUUID g
+nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = liftM uuid (byName n)
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
diff --git a/UUID.hs b/UUID.hs
index eb1fb319c6..0d7aee1414 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -79,8 +79,7 @@ getUncachedUUID r = Git.configGet r configkey ""
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = do
-	g <- Annex.gitRepo
-	u <- getUUID g
+	u <- getUUID =<< Annex.gitRepo
 	when ("" == u) $ do
 		uuid <- liftIO $ genUUID
 		setConfig configkey uuid
@@ -88,8 +87,7 @@ prepUUID = do
 {- Pretty-prints a list of UUIDs -}
 prettyPrintUUIDs :: [UUID] -> Annex String
 prettyPrintUUIDs uuids = do
-	g <- Annex.gitRepo
-	here <- getUUID g
+	here <- getUUID =<< Annex.gitRepo
 	m <- uuidMap
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
 	where

From 59ae1df7a0180263c5434c13b01eb4e7e4ed3267 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 May 2011 19:40:47 +0000
Subject: [PATCH 1714/8313] Added a comment

---
 ...mment_12_26d60661196f63fd01ee4fbb6e2340e7._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment b/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
new file mode 100644
index 0000000000..b458a37b69
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-05-15T19:40:47Z"
+ content="""
+So, it appears that you're using git annex copy --fast. As documented that assumes the location log is correct. So it avoids directly checking if the bare repo contains the file, and tries to upload it, and the bare repo is all like \"but I've already got this file!\". The only way to improve that behavior might be to let rsync go ahead and retransfer the file, which, with recovery, should require sending little data etc. But I can't say I like the idea much, as the repo already has the content, so unlocking it and letting rsync mess with it is an unnecessary risk. I think it's ok for --force to blow up
+if its assumptions turn out to be wrong.
+
+If you use git annex copy without --fast in this situation, it will do the right thing.
+"""]]

From 4d140e3901b7e3c8d66a95817d845b9f35ac6b05 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 20:25:25 +0000
Subject: [PATCH 1715/8313] Added a comment

---
 ..._ead55b915d3b92a62549b2957ad211c8._comment | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment b/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
new file mode 100644
index 0000000000..d92ecbba03
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 13"
+ date="2011-05-15T20:25:25Z"
+ content="""
+Yes, makes sense. I am so used to using --fast, I forgot a non-fast mode existed. I still think it would be a good idea to fall back to non-fast mode if --fast runs into an error from the remote, but as that is well without my abilities how about this patch?
+
+
+    From 4855510c7a84eb5d28fdada429580a8a42b7112a Mon Sep 17 00:00:00 2001
+    From: Richard Hartmann 
+    Date: Sun, 15 May 2011 22:20:42 +0200
+    Subject: [PATCH] Make error in RecvKey.hs suggest possible solution
+    
+    ---
+     Command/RecvKey.hs |    2 +-
+     1 files changed, 1 insertions(+), 1 deletions(-)
+    
+    diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
+    index 126608f..b917a1c 100644
+    --- a/Command/RecvKey.hs
+    +++ b/Command/RecvKey.hs
+    @@ -27,7 +27,7 @@ start :: CommandStartKey
+     start key = do
+        present <- inAnnex key
+        when present $
+    -       error \"key is already present in annex\"
+    +       error \"key is already present in annex. If you are running copy, try without '--fast'\"
+        
+        ok <- getViaTmp key (liftIO . rsyncServerReceive)
+        if ok
+    -- 
+    1.7.4.4
+
+"""]]

From 120bc4208d45f848e4ab0890b54a255bef6a90bb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 20:50:27 +0000
Subject: [PATCH 1716/8313] Added a comment

---
 ...omment_14_191de89d3988083d9cf001799818ff4a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment b/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment
new file mode 100644
index 0000000000..f45bd70468
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 14"
+ date="2011-05-15T20:50:26Z"
+ content="""
+Or, even better, wouldn't it make sense to have SHA backends always default to --fast and only use non-fast when any snags are hit, use non-fast mode for that file.
+
+Though if we continue here, we should probably move this to its own page.
+"""]]

From 4c312d8d8fba8840edd86fe8b951b6b20349985f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 15 May 2011 21:38:47 +0000
Subject: [PATCH 1717/8313] Added a comment

---
 .../comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment b/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
new file mode 100644
index 0000000000..b4a00bd7e1
--- /dev/null
+++ b/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 15"
+ date="2011-05-15T21:38:47Z"
+ content="""
+PS: Just to make this clear, I am using a custom alias for all my copying needs and thus didn't even see that I used --fast. :p
+"""]]

From af54d40d5b61df9c2172d6b533b063d19c29438e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 20:33:39 -0400
Subject: [PATCH 1718/8313] move conversation to a real bug report

---
 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn       | 2 ++
 .../comment_1_6a41bf7e2db83db3a01722b516fb6886._comment         | 0
 .../comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment         | 0
 .../comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment         | 0
 .../comment_4_d7112c315fb016a8a399e24e9b6461d8._comment         | 0
 .../comment_5_4ea29a6f8152eddf806c536de33ef162._comment         | 0
 .../comment_6_0d85f114a103bd6532a3b3b24466012e._comment         | 0
 .../comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment         | 0
 .../comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment         | 0
 .../comment_9_2cee4f6bd6db7518fd61453c595162c6._comment         | 0
 10 files changed, 2 insertions(+)
 create mode 100644 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_5_4ea29a6f8152eddf806c536de33ef162._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_6_0d85f114a103bd6532a3b3b24466012e._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment (100%)

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
new file mode 100644
index 0000000000..4ed72f76d6
--- /dev/null
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -0,0 +1,2 @@
+Conversation moved from [[walkthrough/recover_data_from_lost+found]]
+to a proper bug. --[[Joey]]
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment

From c91764b257b93c523b7884d96a1f027472ba652c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 20:35:39 -0400
Subject: [PATCH 1719/8313] move other comments

---
 .../comment_10_435f87d54052f264096a8f23e99eae06._comment          | 0
 .../comment_11_9be0aef403a002c1706d17deee45763c._comment          | 0
 .../comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment          | 0
 .../comment_13_ead55b915d3b92a62549b2957ad211c8._comment          | 0
 .../comment_14_191de89d3988083d9cf001799818ff4a._comment          | 0
 .../comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment          | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_10_435f87d54052f264096a8f23e99eae06._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_11_9be0aef403a002c1706d17deee45763c._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_13_ead55b915d3b92a62549b2957ad211c8._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_14_191de89d3988083d9cf001799818ff4a._comment (100%)
 rename doc/{walkthrough/recover_data_from_lost+found => bugs/copy_fast_confusing_with_broken_locationlog}/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment (100%)

diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_10_435f87d54052f264096a8f23e99eae06._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_11_9be0aef403a002c1706d17deee45763c._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_14_191de89d3988083d9cf001799818ff4a._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment
diff --git a/doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment
rename to doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment

From d67998b3d37acc8c31df8d6200385805d24921ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 May 2011 20:40:03 -0400
Subject: [PATCH 1720/8313] meh

---
 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
index 4ed72f76d6..2767b5aa9c 100644
--- a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -1,2 +1,4 @@
 Conversation moved from [[walkthrough/recover_data_from_lost+found]]
 to a proper bug. --[[Joey]]
+
+(Unfortunatly that scrambled the comment creation times and thus order.)

From 647f7cf47cd659ae34d27a18d3aa068c1a0755eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:07:59 -0400
Subject: [PATCH 1721/8313] added documentation for using the Internet Archive
 as a remote via S3

Renamed Amazon_S3 page to just S3.
---
 doc/cheatsheet.mdwn                           |  1 +
 doc/index.mdwn                                |  2 +-
 doc/special_remotes.mdwn                      |  2 +-
 .../{Amazon_S3.mdwn => S3.mdwn}               |  0
 doc/walkthrough/Internet_Archive_via_S3.mdwn  | 48 +++++++++++++++++++
 doc/walkthrough/using_Amazon_S3.mdwn          |  2 +-
 6 files changed, 52 insertions(+), 3 deletions(-)
 rename doc/special_remotes/{Amazon_S3.mdwn => S3.mdwn} (100%)
 create mode 100644 doc/walkthrough/Internet_Archive_via_S3.mdwn

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 4287756a6c..3f5960972d 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -11,4 +11,5 @@ A suppliment to the [[walkthrough]].
 	walkthrough/untrusted_repositories
 	walkthrough/what_to_do_when_you_lose_a_repository
 	walkthrough/recover_data_from_lost+found
+	walkthrough/Internet_Archive_via_S3
 """]]
diff --git a/doc/index.mdwn b/doc/index.mdwn
index eb6307f327..8975c82de7 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -43,7 +43,7 @@ files with git.
 
 * [[git-annex man page|git-annex]]
 * [[key-value backends|backends]] for data storage
-* [[special_remotes]] (including [[special_remotes/Amazon_S3]] and [[special_remotes/bup]])
+* [[special_remotes]] (including [[special_remotes/S3]] and [[special_remotes/bup]])
 * [[encryption]]
 * [[bare_repositories]]
 * [[internals]]
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 13c18122e7..dcb6b5063e 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -6,7 +6,7 @@ But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
 They cannot be used by other git commands though.
 
-* [[Amazon_S3]]
+* [[S3]] (Amazon S3, and other compatible services)
 * [[bup]]
 * [[directory]]
 * [[rsync]]
diff --git a/doc/special_remotes/Amazon_S3.mdwn b/doc/special_remotes/S3.mdwn
similarity index 100%
rename from doc/special_remotes/Amazon_S3.mdwn
rename to doc/special_remotes/S3.mdwn
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
new file mode 100644
index 0000000000..089102d142
--- /dev/null
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -0,0 +1,48 @@
+[The Internet Archive](http://www.archive.org/) allows members to upload
+collections using an Amazon S3 
+[compatible API](http://www.archive.org/help/abouts3.txt), and this can
+be used with git-annex's [[special_remotes/S3]] support. 
+
+So, if you're an archivist, you can locally archive things with git-annex,
+and define remotes that correspond to "items" at the Internet Archive,
+and use git-annex to upload your files to there.
+Of course, your use of the Internet Archive must comply with their
+[terms of service](http://www.archive.org/about/terms.php).
+
+## step 0
+
+Sign up for an account, and get your access keys here:
+
+	
+	# export AWS_ACCESS_KEY_ID=blahblah
+	# export AWS_SECRET_ACCESS_KEY=xxxxxxx
+
+Now go to  and create the item.
+This allows you to fill in metadata which git-annex cannot provide to the
+Internet Archive. (It also works around a bug with bucket creation.)
+
+(Note that there seems to be a bug in either hS3 or the archive that
+breaks authentication when the item name contains spaces or upper-case
+letters.. use all lowercase and no spaces.)
+
+Specify `host=s3.us.archive.org` when doing initremote to set up
+a remote at the Archive. It does not make sense to use encryption.
+For the bucket name, specify the item name created in step 1.
+
+	# git annex initremote panama type=S3 encryption=none host=s3.us.archive.org bucket=panama-canal-lock-blueprints
+	initremote archive-panama (checking bucket) (creating bucket in US) ok
+	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
+	describe archive-panama ok
+
+Then you can annex files and copy them to the remote as usual:
+
+	# git annex add photo1.jpeg
+	add photo1.jpeg ok
+	# git annex copy photo1.jpeg --to archive-panama
+	copy (checking archive-panama...) (to archive-panama...) ok
+
+Note that it probably makes the most sense to use the WORM backend
+for files, since that exposes the original filename in the key stored
+in the Archive, which allows its special processing for sound files,
+movies, etc to be done. Also, the Internet Archive has restrictions
+on what is allowed in a filename; particularly no spaces are allowed.
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 6b0f496392..7f972afe11 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -34,4 +34,4 @@ Now the remote can be used like any other remote.
 	# git annex move video/hackity_hack_and_kaxxt.mov --to cloud
 	move video/hackity_hack_and_kaxxt.mov (checking cloud...) (to cloud...) ok
 
-See [[special_remotes/Amazon_S3]] for details.
+See [[special_remotes/S3]] for details.

From 8564498b5d87a424effec695825562b56f1391d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:15:00 -0400
Subject: [PATCH 1722/8313] cleanup

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 089102d142..554f4b03b1 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -9,8 +9,6 @@ and use git-annex to upload your files to there.
 Of course, your use of the Internet Archive must comply with their
 [terms of service](http://www.archive.org/about/terms.php).
 
-## step 0
-
 Sign up for an account, and get your access keys here:
 
 	

From 275c72850943d78677929b0bac5899d1e7e5e5b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:18:28 -0400
Subject: [PATCH 1723/8313] cleanup

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 554f4b03b1..3cd83a2e7c 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -25,7 +25,7 @@ letters.. use all lowercase and no spaces.)
 
 Specify `host=s3.us.archive.org` when doing initremote to set up
 a remote at the Archive. It does not make sense to use encryption.
-For the bucket name, specify the item name created in step 1.
+For the bucket name, specify the item name you created earlier.
 
 	# git annex initremote panama type=S3 encryption=none host=s3.us.archive.org bucket=panama-canal-lock-blueprints
 	initremote archive-panama (checking bucket) (creating bucket in US) ok

From e259c86975a6ec1ab604811684dec7166f57b7bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 02:21:40 -0400
Subject: [PATCH 1724/8313] cleanup

---
 doc/special_remotes/S3.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 35397dc2a9..abd61ac797 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -1,6 +1,5 @@
 This special remote type stores file contents in a bucket in Amazon S3
-or a similar service, such as 
-[Archive.org's S3 API](http://www.archive.org/help/abouts3.txt).
+or a similar service.
 
 See [[walkthrough/using_Amazon_S3]] for usage examples.
 

From 79c74bf27dfb9795ad35bc4e4c2061004212621d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 09:42:54 -0400
Subject: [PATCH 1725/8313] refactor

---
 Remote/S3real.hs | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index eaa6590b19..e8c700e2c7 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -73,21 +73,7 @@ s3Setup u c = do
 	c' <- encryptionSetup c
 	let fullconfig = M.union c' defaults
 
-	-- check bucket location to see if the bucket exists, and create it
-	let datacenter = fromJust $ M.lookup "datacenter" fullconfig
-	conn <- s3ConnectionRequired fullconfig
-	showNote "checking bucket"
-	loc <- liftIO $ getBucketLocation conn bucket 
-	case loc of
-		Right _ -> return ()
-		Left err@(NetworkError _) -> s3Error err
-		Left (AWSError _ _) -> do
-			showNote $ "creating bucket in " ++ datacenter
-			res <- liftIO $ createBucketIn conn bucket datacenter
-			case res of
-				Right _ -> return ()
-				Left err -> s3Error err
-
+	genBucket fullconfig
 	gitConfigSpecialRemote u fullconfig "s3" "true"
 	s3SetCreds fullconfig
 	where
@@ -126,7 +112,7 @@ storeHelper (conn, bucket) r k file = do
 	size <- maybe getsize (return . fromIntegral) $ keySize k
 	let object = setStorageClass storageclass $ 
 		S3Object bucket (show k) ""
-			[("Content-Length",(show size))] content
+			[("Content-Length",(show size)), ("x-amz-auto-make-bucket","1")] content
 	sendObject conn object
 	where
 		storageclass =
@@ -199,6 +185,24 @@ s3Action r noconn action = do
 bucketKey :: String -> Key -> S3Object
 bucketKey bucket k = S3Object bucket (show k) "" [] L.empty
 
+genBucket :: RemoteConfig -> Annex ()
+genBucket c = do
+	conn <- s3ConnectionRequired c
+	showNote "checking bucket"
+	loc <- liftIO $ getBucketLocation conn bucket 
+	case loc of
+		Right _ -> return ()
+		Left err@(NetworkError _) -> s3Error err
+		Left (AWSError _ _) -> do
+			showNote $ "creating bucket in " ++ datacenter
+			res <- liftIO $ createBucketIn conn bucket datacenter
+			case res of
+				Right _ -> return ()
+				Left err -> s3Error err
+	where
+		bucket = fromJust $ M.lookup "bucket" c
+		datacenter = fromJust $ M.lookup "datacenter" c
+
 s3ConnectionRequired :: RemoteConfig -> Annex AWSConnection
 s3ConnectionRequired c =
 	maybe (error "Cannot connect to S3") return =<< s3Connection c

From 1d2984441c654f01e88e427f3289f8066cd2e6b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:20:30 -0400
Subject: [PATCH 1726/8313] add a few tweaks to make it easy to use the
 Internet Archive's variant of S3

In particular, munge key filenames to comply with the IA's filename limits,
disable encryption, support their nonstandard way of creating buckets, and
allow x-amz-* headers to be specified in initremote to set item metadata.

Still TODO: initremote does not handle multiword metadata headers right.
---
 Remote/S3real.hs                             | 90 ++++++++++++++++----
 debian/changelog                             |  5 ++
 doc/special_remotes/S3.mdwn                  | 13 +--
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 40 +++++----
 4 files changed, 108 insertions(+), 40 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index e8c700e2c7..7d6b5d5bab 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -15,6 +15,9 @@ import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.Maybe
+import Data.List
+import Data.Char
+import Data.String.Utils
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Environment
@@ -68,24 +71,53 @@ gen' r u c cst = do
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
-s3Setup u c = do
-	-- verify configuration is sane
-	c' <- encryptionSetup c
-	let fullconfig = M.union c' defaults
-
-	genBucket fullconfig
-	gitConfigSpecialRemote u fullconfig "s3" "true"
-	s3SetCreds fullconfig
+s3Setup u c = handlehost $ M.lookup "host" c
 	where
 		remotename = fromJust (M.lookup "name" c)
-		bucket = remotename ++ "-" ++ u
+		defbucket = remotename ++ "-" ++ u
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
 			, ("host", defaultAmazonS3Host)
 			, ("port", show defaultAmazonS3Port)
-			, ("bucket", bucket)
+			, ("bucket", defbucket)
 			]
+		
+		handlehost Nothing = defaulthost
+		handlehost (Just h)
+			| ".archive.org" `isSuffixOf` (map toLower h) = archiveorg
+			| otherwise = defaulthost
+
+		use fullconfig = do
+			genBucket fullconfig
+			gitConfigSpecialRemote u fullconfig "s3" "true"
+			s3SetCreds fullconfig	
+
+		defaulthost = do
+			c' <- encryptionSetup c
+			use $ M.union c' defaults
+
+		archiveorg = do
+			showNote $ "Internet Archive mode"
+			maybe (error "specify bucket=") (const $ return ()) $
+				M.lookup "bucket" archiveconfig
+			use archiveconfig
+			where
+				archiveconfig =
+					-- hS3 does not pass through
+					-- x-archive-* headers
+					M.mapKeys (replace "x-archive-" "x-amz-") $
+					-- encryption does not make sense here
+					M.insert "encryption" "none" $
+					M.union c $
+					-- special constraints on key names
+					M.insert "mungekeys" "ia" $
+					-- buckets created only as files
+					-- are uploaded
+					M.insert "x-amz-auto-make-bucket" "1" $
+					-- no default bucket name; should
+					-- be human-readable
+					M.delete "bucket" defaults
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
@@ -111,8 +143,8 @@ storeHelper (conn, bucket) r k file = do
 	-- buffered to calculate it
 	size <- maybe getsize (return . fromIntegral) $ keySize k
 	let object = setStorageClass storageclass $ 
-		S3Object bucket (show k) ""
-			[("Content-Length",(show size)), ("x-amz-auto-make-bucket","1")] content
+		S3Object bucket (bucketFile r k) ""
+			(("Content-Length", show size) : xheaders) content
 	sendObject conn object
 	where
 		storageclass =
@@ -122,10 +154,13 @@ storeHelper (conn, bucket) r k file = do
 		getsize = do
 			s <- liftIO $ getFileStatus file
 			return $ fileSize s
+		
+		xheaders = filter isxheader $ M.assocs $ fromJust $ config r
+		isxheader (h, _) = "x-amz-" `isPrefixOf` h
 
 retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
 retrieve r k f = s3Action r False $ \(conn, bucket) -> do
-	res <- liftIO $ getObject conn $ bucketKey bucket k
+	res <- liftIO $ getObject conn $ bucketKey r bucket k
 	case res of
 		Right o -> do
 			liftIO $ L.writeFile f $ obj_data o
@@ -134,7 +169,7 @@ retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 
 retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
-	res <- liftIO $ getObject conn $ bucketKey bucket enck
+	res <- liftIO $ getObject conn $ bucketKey r bucket enck
 	case res of
 		Right o -> liftIO $ 
 			withDecryptedContent cipher (return $ obj_data o) $ \content -> do
@@ -144,13 +179,13 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
 
 remove :: Remote Annex -> Key -> Annex Bool
 remove r k = s3Action r False $ \(conn, bucket) -> do
-	res <- liftIO $ deleteObject conn $ bucketKey bucket k
+	res <- liftIO $ deleteObject conn $ bucketKey r bucket k
 	s3Bool res
 
 checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showNote ("checking " ++ name r ++ "...")
-	res <- liftIO $ getObjectInfo conn $ bucketKey bucket k
+	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
 	case res of
 		Right _ -> return $ Right True
 		Left (AWSError _ _) -> return $ Right False
@@ -182,8 +217,27 @@ s3Action r noconn action = do
 		(Just b, Just c) -> action (c, b)
 		_ -> return noconn
 
-bucketKey :: String -> Key -> S3Object
-bucketKey bucket k = S3Object bucket (show k) "" [] L.empty
+bucketFile :: Remote Annex -> Key -> FilePath
+bucketFile r k = (munge $ show k)
+	where
+		munge s = case M.lookup "mungekeys" $ fromJust $ config r of
+			Just "ia" -> iaMunge s
+			_ -> s
+
+bucketKey :: Remote Annex -> String -> Key -> S3Object
+bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty
+
+{- Internet Archive limits filenames to a subset of ascii,
+ - with no whitespace. Other characters are xml entity
+ - encoded. -}
+iaMunge :: String -> String
+iaMunge = concat . (map munge)
+	where
+		munge c
+			| isAsciiUpper c || isAsciiLower c || isNumber c = [c]
+			| c `elem` "_-.\"" = [c]
+			| isSpace c = []
+			| otherwise = "&" ++ show (ord c) ++ ";"
 
 genBucket :: RemoteConfig -> Annex ()
 genBucket c = do
diff --git a/debian/changelog b/debian/changelog
index 09dde346c0..59ebb1deb7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,11 @@
 git-annex (0.20110504) UNRELEASED; urgency=low
 
   * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.
+  * Add a few tweaks to make it easy to use the Internet Archive's variant
+    of S3. In particular, munge key filenames to comply with the IA's filename
+    limits, disable encryption, support their nonstandard way of creating
+    buckets, and allow x-amz-* headers to be specified in initremote to set
+    item metadata.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
 
diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index abd61ac797..d6a7229e33 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -5,6 +5,12 @@ See [[walkthrough/using_Amazon_S3]] for usage examples.
 
 ## configuration
 
+The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and 
+`ANNEX_S3_SECRET_ACCESS_KEY` are used to supply login credentials
+for Amazon. When encryption is enabled, they are stored in encrypted form
+by `git annex initremote`, so you do not need to keep the environment
+variables set after the initial initalization of the remote.
+
 A number of parameters can be passed to `git annex initremote` to configure
 the S3 remote.
 
@@ -29,8 +35,5 @@ the S3 remote.
   so by default, a bucket name is chosen based on the remote name
   and UUID. This can be specified to pick a bucket name.
 
-The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and 
-`ANNEX_S3_SECRET_ACCESS_KEY` can be used to supply login credentials
-for Amazon. When encryption is enabled, they are stored in encrypted form
-by `git annex initremote`, so you do not need to keep the environment
-variables set after the initial initalization of the remote.
+* `x-amz-*` are passed through as http headers when storing keys
+  in S3.
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 3cd83a2e7c..e0f8fafb44 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -15,20 +15,18 @@ Sign up for an account, and get your access keys here:
 	# export AWS_ACCESS_KEY_ID=blahblah
 	# export AWS_SECRET_ACCESS_KEY=xxxxxxx
 
-Now go to  and create the item.
-This allows you to fill in metadata which git-annex cannot provide to the
-Internet Archive. (It also works around a bug with bucket creation.)
+Specify `host=s3.us.archive.org` when doing `initremote` to set up
+a remote at the Archive. This will enable a special Internet Archive mode:
+Encryption is not allowed; you are required to specify a bucket name
+rather than letting git-annex pick a random one; and you can optionally
+specify `x-archive-meta*` headers to add metadata as explained in their
+[documentation](http://www.archive.org/help/abouts3.txt).
 
-(Note that there seems to be a bug in either hS3 or the archive that
-breaks authentication when the item name contains spaces or upper-case
-letters.. use all lowercase and no spaces.)
-
-Specify `host=s3.us.archive.org` when doing initremote to set up
-a remote at the Archive. It does not make sense to use encryption.
-For the bucket name, specify the item name you created earlier.
-
-	# git annex initremote panama type=S3 encryption=none host=s3.us.archive.org bucket=panama-canal-lock-blueprints
-	initremote archive-panama (checking bucket) (creating bucket in US) ok
+	# git annex initremote archive-panama type=S3
+	# host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
+		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
+		x-archive-meta-title="original Panama Canal lock design blueprints"
+	initremote archive-panama (Internet Archive mode) (checking bucket) (creating bucket in US) ok
 	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
 	describe archive-panama ok
 
@@ -36,11 +34,19 @@ Then you can annex files and copy them to the remote as usual:
 
 	# git annex add photo1.jpeg
 	add photo1.jpeg ok
-	# git annex copy photo1.jpeg --to archive-panama
-	copy (checking archive-panama...) (to archive-panama...) ok
+	# git annex copy photo1.jpeg --fast --to archive-panama
+	copy (to archive-panama...) ok
+
+-----
 
 Note that it probably makes the most sense to use the WORM backend
 for files, since that exposes the original filename in the key stored
 in the Archive, which allows its special processing for sound files,
-movies, etc to be done. Also, the Internet Archive has restrictions
-on what is allowed in a filename; particularly no spaces are allowed.
+movies, etc to be done. 
+
+Also, the Internet Archive has restrictions on what is allowed in a
+filename; particularly no spaces are allowed. 
+
+There seems to be a bug in either hS3 or the archive that breaks
+authentication when the bucket name contains spaces or upper-case letters..
+use all lowercase and no spaces when making the bucket with `initremote`.

From 2a8efc7af19aa149dbf0ebc158954bb376f9c3a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:46:34 -0400
Subject: [PATCH 1727/8313] Added filename extension preserving variant
 backends SHA1E, SHA256E, etc.

---
 Backend/SHA.hs                               | 44 ++++++++++++++++----
 debian/changelog                             |  1 +
 doc/backends.mdwn                            |  3 ++
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 18 ++++----
 4 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index d9aeb72aa4..6d721038c3 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -14,6 +14,7 @@ import System.IO
 import System.Directory
 import Data.Maybe
 import System.Posix.Files
+import System.FilePath
 
 import qualified Backend.File
 import BackendClass
@@ -27,11 +28,14 @@ import qualified SysConfig
 import Key
 
 type SHASize = Int
- 
+
+sizes :: [Int]
+sizes = [1, 256, 512, 224, 384]
+
 backends :: [Backend Annex]
 -- order is slightly significant; want sha1 first ,and more general
 -- sizes earlier
-backends = catMaybes $ map genBackend [1, 256, 512, 224, 384]
+backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
 genBackend :: SHASize -> Maybe (Backend Annex)
 genBackend size
@@ -44,6 +48,15 @@ genBackend size
 			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
 			}
 
+genBackendE :: SHASize -> Maybe (Backend Annex)
+genBackendE size =
+	case genBackend size of
+		Nothing -> Nothing
+		Just b -> Just $ b
+			{ name = shaNameE size
+			, getKey = keyValueE size
+			}
+
 shaCommand :: SHASize -> Maybe String
 shaCommand 1 = SysConfig.sha1
 shaCommand 256 = SysConfig.sha256
@@ -55,6 +68,9 @@ shaCommand _ = Nothing
 shaName :: SHASize -> String
 shaName size = "SHA" ++ show size
 
+shaNameE :: SHASize -> String
+shaNameE size = shaName size ++ "E"
+
 shaN :: SHASize -> FilePath -> Annex String
 shaN size file = do
 	showNote "checksum..."
@@ -72,11 +88,25 @@ keyValue :: SHASize -> FilePath -> Annex (Maybe Key)
 keyValue size file = do
 	s <- shaN size file	
 	stat <- liftIO $ getFileStatus file
-	return $ Just $ stubKey {
-		keyName = s,
-		keyBackendName = shaName size,
-		keySize = Just $ fromIntegral $ fileSize stat
-	}
+	return $ Just $ stubKey
+		{ keyName = s
+		, keyBackendName = shaName size
+		, keySize = Just $ fromIntegral $ fileSize stat
+		}
+
+{- Extension preserving keys. -}
+keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
+keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
+	where
+		addE k = return $ Just $ k
+			{ keyName = keyName k ++ extension
+			, keyBackendName = shaNameE size
+			}
+		naiveextension = takeExtension file
+		extension = 
+			if length naiveextension > 6
+				then "" -- probably not really an extension
+				else naiveextension
 
 -- A key's checksum is checked during fsck.
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
diff --git a/debian/changelog b/debian/changelog
index 59ebb1deb7..a7f6039810 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (0.20110504) UNRELEASED; urgency=low
     limits, disable encryption, support their nonstandard way of creating
     buckets, and allow x-amz-* headers to be specified in initremote to set
     item metadata.
+  * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
 
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index b0a2c882aa..4290da33ba 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -23,6 +23,9 @@ these backends.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
   researching checksum collisions and wants to annex their colliding data. ;)
+* `SHA1E`, `SHA512E`, etc -- Variants that preserve filename extension as
+  part of the key. Useful for archival tasks where the filename extension
+  contains metadata that should be preserved.
 
 These backends store file contents in other key/value stores.
 
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index e0f8fafb44..f92e0ee9d7 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -32,20 +32,18 @@ specify `x-archive-meta*` headers to add metadata as explained in their
 
 Then you can annex files and copy them to the remote as usual:
 
-	# git annex add photo1.jpeg
-	add photo1.jpeg ok
+	# git annex add photo1.jpeg --backend=SHA1E
+	add photo1.jpeg (checksum...) ok
 	# git annex copy photo1.jpeg --fast --to archive-panama
 	copy (to archive-panama...) ok
 
------
+Note the use of the SHA1E [[backend|backends]]. It makes most sense 
+to use the WORM or SHA1E backend for files that will be stored in
+the Internet Archive, since the key name will be exposed as the filename
+there, and since the Archive does special processing of files based on
+their extension.
 
-Note that it probably makes the most sense to use the WORM backend
-for files, since that exposes the original filename in the key stored
-in the Archive, which allows its special processing for sound files,
-movies, etc to be done. 
-
-Also, the Internet Archive has restrictions on what is allowed in a
-filename; particularly no spaces are allowed. 
+----
 
 There seems to be a bug in either hS3 or the archive that breaks
 authentication when the bucket name contains spaces or upper-case letters..

From e7b309ce02902474b0073d4bb3577bdca1686b67 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:49:52 -0400
Subject: [PATCH 1728/8313] clarify

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index a7f6039810..666abc94e7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,8 +4,8 @@ git-annex (0.20110504) UNRELEASED; urgency=low
   * Add a few tweaks to make it easy to use the Internet Archive's variant
     of S3. In particular, munge key filenames to comply with the IA's filename
     limits, disable encryption, support their nonstandard way of creating
-    buckets, and allow x-amz-* headers to be specified in initremote to set
-    item metadata.
+    buckets, and allow x-archive-* headers to be specified in initremote to
+    set item metadata.
   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400

From b4301c208f96414899f3ba40c2fa5d3bb43089a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:52:33 -0400
Subject: [PATCH 1729/8313] cleanup

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 25 ++++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index f92e0ee9d7..89e8564705 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -3,11 +3,10 @@ collections using an Amazon S3
 [compatible API](http://www.archive.org/help/abouts3.txt), and this can
 be used with git-annex's [[special_remotes/S3]] support. 
 
-So, if you're an archivist, you can locally archive things with git-annex,
-and define remotes that correspond to "items" at the Internet Archive,
-and use git-annex to upload your files to there.
-Of course, your use of the Internet Archive must comply with their
-[terms of service](http://www.archive.org/about/terms.php).
+So, you can locally archive things with git-annex, define remotes that
+correspond to "items" at the Internet Archive, and use git-annex to upload
+your files to there. Of course, your use of the Internet Archive must
+comply with their [terms of service](http://www.archive.org/about/terms.php).
 
 Sign up for an account, and get your access keys here:
 
@@ -22,8 +21,14 @@ rather than letting git-annex pick a random one; and you can optionally
 specify `x-archive-meta*` headers to add metadata as explained in their
 [documentation](http://www.archive.org/help/abouts3.txt).
 
-	# git annex initremote archive-panama type=S3
-	# host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
+[[!template id=note text="""
+There seems to be a bug in either hS3 or the archive that breaks
+authentication when the bucket name contains spaces or upper-case letters..
+use all lowercase and no spaces when making the bucket with `initremote`.
+"""]]
+
+	# git annex initremote archive-panama type=S3 \
+		host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
 		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
 		x-archive-meta-title="original Panama Canal lock design blueprints"
 	initremote archive-panama (Internet Archive mode) (checking bucket) (creating bucket in US) ok
@@ -42,9 +47,3 @@ to use the WORM or SHA1E backend for files that will be stored in
 the Internet Archive, since the key name will be exposed as the filename
 there, and since the Archive does special processing of files based on
 their extension.
-
-----
-
-There seems to be a bug in either hS3 or the archive that breaks
-authentication when the bucket name contains spaces or upper-case letters..
-use all lowercase and no spaces when making the bucket with `initremote`.

From 27285adc202afee35f6198d0e3aadcff2c662ef8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:55:07 -0400
Subject: [PATCH 1730/8313] link

---
 doc/special_remotes/S3.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index d6a7229e33..047798e238 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -1,7 +1,8 @@
 This special remote type stores file contents in a bucket in Amazon S3
 or a similar service.
 
-See [[walkthrough/using_Amazon_S3]] for usage examples.
+See [[walkthrough/using_Amazon_S3]] and
+[[walkthrough/Internet_Archive_via_S3]] for usage examples.
 
 ## configuration
 

From 267eeb995b8bfb779c017086df75b4700a103485 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 11:55:33 -0400
Subject: [PATCH 1731/8313] tweak

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 89e8564705..5ed71a123e 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -22,7 +22,7 @@ specify `x-archive-meta*` headers to add metadata as explained in their
 [documentation](http://www.archive.org/help/abouts3.txt).
 
 [[!template id=note text="""
-There seems to be a bug in either hS3 or the archive that breaks
+/!\ There seems to be a bug in either hS3 or the archive that breaks
 authentication when the bucket name contains spaces or upper-case letters..
 use all lowercase and no spaces when making the bucket with `initremote`.
 """]]

From 5256a6b011ba7c8ddd4e07232a5a25a5562c1b88 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 12:10:08 -0400
Subject: [PATCH 1732/8313] migrate: Use current filename when generating new
 key, for backends where the filename affects the key name.

---
 Command/Migrate.hs | 12 ++++++++++--
 debian/changelog   |  2 ++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 35855d5270..790d5d365b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,9 +8,10 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (unless)
+import Control.Monad (unless, when)
 import System.Posix.Files
 import System.Directory
+import System.FilePath
 
 import Command
 import qualified Annex
@@ -52,7 +53,10 @@ perform file oldkey newbackend = do
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
 	let src = gitAnnexLocation g oldkey
-	stored <- Backend.storeFileKey src $ Just newbackend
+	let tmpfile = gitAnnexTmpDir g  takeFileName file
+	liftIO $ createLink src tmpfile
+	stored <- Backend.storeFileKey tmpfile $ Just newbackend
+	liftIO $ cleantmp tmpfile
 	case stored of
 		Nothing -> stop
 		Just (newkey, _) -> do
@@ -69,3 +73,7 @@ perform file oldkey newbackend = do
 					liftIO $ removeFile file
 					next $ Command.Add.cleanup file newkey
 				else stop
+	where
+		cleantmp t = do
+			exists <- doesFileExist t
+			when exists $ removeFile t
diff --git a/debian/changelog b/debian/changelog
index 666abc94e7..3827a4632b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (0.20110504) UNRELEASED; urgency=low
     buckets, and allow x-archive-* headers to be specified in initremote to
     set item metadata.
   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
+  * migrate: Use current filename when generating new key, for backends
+    where the filename affects the key name.
 
  -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
 

From 8fa17eaba08b99250d6290915379f048844d83d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 12:12:03 -0400
Subject: [PATCH 1733/8313] tweak

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 5ed71a123e..1d5fc88f53 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -17,7 +17,7 @@ Sign up for an account, and get your access keys here:
 Specify `host=s3.us.archive.org` when doing `initremote` to set up
 a remote at the Archive. This will enable a special Internet Archive mode:
 Encryption is not allowed; you are required to specify a bucket name
-rather than letting git-annex pick a random one; and you can optionally
+rather than having git-annex pick a random one; and you can optionally
 specify `x-archive-meta*` headers to add metadata as explained in their
 [documentation](http://www.archive.org/help/abouts3.txt).
 

From ceff04ff3e7fff4b0ea6e8ad4334cca80d291880 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 12:25:54 -0400
Subject: [PATCH 1734/8313] better multiword parameter handling

This way, individual words as entered on the command line are available
to commands.
---
 Command.hs            | 6 ++++--
 Command/Describe.hs   | 8 ++++----
 Command/Init.hs       | 8 +++++---
 Command/InitRemote.hs | 9 +++++----
 Command/Semitrust.hs  | 7 ++++---
 Command/Trust.hs      | 7 ++++---
 Command/Untrust.hs    | 7 ++++---
 7 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/Command.hs b/Command.hs
index 4f835a3adc..abbe897b5b 100644
--- a/Command.hs
+++ b/Command.hs
@@ -47,6 +47,8 @@ type CommandCleanup = Annex Bool
  - functions. -}
 type CommandSeekStrings = CommandStartString -> CommandSeek
 type CommandStartString = String -> CommandStart
+type CommandSeekWords = CommandStartWords -> CommandSeek
+type CommandStartWords = [String] -> CommandStart
 type CommandSeekKeys = CommandStartKey -> CommandSeek
 type CommandStartKey = Key -> CommandStart
 type BackendFile = (FilePath, Maybe (Backend Annex))
@@ -143,8 +145,8 @@ withFilesNotInGit a params = do
 	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
-withString :: CommandSeekStrings
-withString a params = return [a $ unwords params]
+withWords :: CommandSeekWords
+withWords a params = return [a params]
 withStrings :: CommandSeekStrings
 withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
diff --git a/Command/Describe.hs b/Command/Describe.hs
index dcabef7fbf..57f884e037 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -18,12 +18,12 @@ command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start params = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
 	let (name, description) =
-		case (words params) of
+		case ws of
 			(n:d) -> (n,unwords d)
 			_ -> error "Specify a repository and a description."
 	
diff --git a/Command/Init.hs b/Command/Init.hs
index 668b5c87d6..b7a0787991 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -27,15 +27,17 @@ command = [repoCommand "init" paramDesc seek
 		"initialize git-annex with repository description"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
 {- Stores description for the repository etc. -}
-start :: CommandStartString
-start description = do
+start :: CommandStartWords
+start ws = do
 	when (null description) $
 		error "please specify a description of this repository\n"
 	showStart "init" description
 	next $ perform description
+	where
+		description = unwords ws
 
 perform :: String -> CommandPerform
 perform description = do
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 261ccdc8b7..ae22e3564e 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -28,21 +28,22 @@ command = [repoCommand "initremote"
 	"sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start params = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
 	when (null ws) $ error "Specify a name for the remote"
 
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
 	t <- findType fullconfig
 	
+	liftIO $ putStrLn $ show fullconfig
+
 	showStart "initremote" name
 	next $ perform t u $ M.union config c
 
 	where
-		ws = words params
 		name = head ws
 		config = Remote.keyValToConfig $ tail ws
 
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index fc1bcbbcdc..11742137f1 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -18,10 +18,11 @@ command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start name = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
+	let name = unwords ws
 	showStart "semitrust" name
 	u <- Remote.nameToUUID name
 	next $ perform u
diff --git a/Command/Trust.hs b/Command/Trust.hs
index ef03828c23..d5444affe6 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -18,10 +18,11 @@ command = [repoCommand "trust" (paramRepeating paramRemote) seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start name = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
+	let name = unwords ws
 	showStart "trust" name
 	u <- Remote.nameToUUID name
 	next $ perform u
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index ebe9c31b3b..174c395066 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -18,10 +18,11 @@ command = [repoCommand "untrust" (paramRepeating paramRemote) seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
-seek = [withString start]
+seek = [withWords start]
 
-start :: CommandStartString
-start name = notBareRepo $ do
+start :: CommandStartWords
+start ws = notBareRepo $ do
+	let name = unwords ws
 	showStart "untrust" name
 	u <- Remote.nameToUUID name
 	next $ perform u

From 93c5fb5da7f085cc772e28d8ded08f4ea0b0bf15 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:07:56 -0400
Subject: [PATCH 1735/8313] support remote config values with spaces and other
 characters

---
 Command/InitRemote.hs |  2 --
 Remote.hs             | 38 +++++++++++++++++++++++++++++++++++---
 test.hs               |  1 +
 3 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index ae22e3564e..ad0718e38e 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -37,8 +37,6 @@ start ws = notBareRepo $ do
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
 	t <- findType fullconfig
-	
-	liftIO $ putStrLn $ show fullconfig
 
 	showStart "initremote" name
 	next $ perform t u $ M.union config c
diff --git a/Remote.hs b/Remote.hs
index 211168b153..7bbe910000 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -25,7 +25,10 @@ module Remote (
 	remoteLog,
 	readRemoteLog,
 	configSet,
-	keyValToConfig
+	keyValToConfig,
+	configToKeyVal,
+	
+	prop_idempotent_configEscape
 ) where
 
 import Control.Monad.State (liftIO)
@@ -33,6 +36,7 @@ import Control.Monad (when, liftM, filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
+import Data.Char
 
 import RemoteClass
 import Types
@@ -176,9 +180,37 @@ keyValToConfig ws = M.fromList $ map (/=/) ws
 		(/=/) s = (k, v)
 			where
 				k = takeWhile (/= '=') s
-				v = drop (1 + length k) s
+				v = configUnEscape $ drop (1 + length k) s
 
 configToKeyVal :: M.Map String String -> [String]
 configToKeyVal m = map toword $ sort $ M.toList m
 	where
-		toword (k, v) = k ++ "=" ++ v
+		toword (k, v) = k ++ "=" ++ configEscape v
+
+configEscape :: String -> String
+configEscape = concat . (map escape)
+	where
+		escape c
+			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
+			| otherwise = [c]
+
+configUnEscape :: String -> String
+configUnEscape = unescape
+	where
+		unescape [] = []
+		unescape (c:rest)
+			| c == '&' = entity rest
+			| otherwise = c : unescape rest
+		entity s = if ok
+				then chr (read num) : unescape rest
+				else '&' : unescape s
+			where
+				num = takeWhile isNumber s
+				r = drop (length num) s
+				rest = drop 1 r
+				ok = not (null num) && 
+					not (null r) && r !! 0 == ';'
+
+{- for quickcheck -}
+prop_idempotent_configEscape :: String -> Bool
+prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
diff --git a/test.hs b/test.hs
index 73eff662b9..456c09060c 100644
--- a/test.hs
+++ b/test.hs
@@ -75,6 +75,7 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_idempotent_key_read_show" Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
+	, qctest "prop_idempotent_configEscape" Remote.prop_idempotent_configEscape
 	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
 	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane

From 0a7bcd47aeec9a2d1c9a42ef8d9ea539a6aef0d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:10:26 -0400
Subject: [PATCH 1736/8313] IA: do not create bucket at initremote time

This way, the metadata sent when uploading a file is applied to the bucket
then.
---
 Remote/S3real.hs                             | 7 ++++---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 7d6b5d5bab..1359669036 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -89,13 +89,14 @@ s3Setup u c = handlehost $ M.lookup "host" c
 			| otherwise = defaulthost
 
 		use fullconfig = do
-			genBucket fullconfig
 			gitConfigSpecialRemote u fullconfig "s3" "true"
 			s3SetCreds fullconfig	
 
 		defaulthost = do
 			c' <- encryptionSetup c
-			use $ M.union c' defaults
+			let fullconfig = M.union c' defaults
+			genBucket fullconfig
+			use fullconfig
 
 		archiveorg = do
 			showNote $ "Internet Archive mode"
@@ -112,7 +113,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 					M.union c $
 					-- special constraints on key names
 					M.insert "mungekeys" "ia" $
-					-- buckets created only as files
+					-- bucket created only when files
 					-- are uploaded
 					M.insert "x-amz-auto-make-bucket" "1" $
 					-- no default bucket name; should
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index 1d5fc88f53..b80f0a4b7c 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -31,7 +31,7 @@ use all lowercase and no spaces when making the bucket with `initremote`.
 		host=s3.us.archive.org bucket=panama-canal-lock-blueprints \
 		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
 		x-archive-meta-title="original Panama Canal lock design blueprints"
-	initremote archive-panama (Internet Archive mode) (checking bucket) (creating bucket in US) ok
+	initremote archive-panama (Internet Archive mode) ok
 	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
 	describe archive-panama ok
 

From d006586cd0b706c9cc92b2747b2ba3487f52c04a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:27:19 -0400
Subject: [PATCH 1737/8313] add a message in potenatially confusing copy --fast
 failure situation

---
 Command/Move.hs                                        | 10 ++++++++--
 .../copy_fast_confusing_with_broken_locationlog.mdwn   |  2 ++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 623003e47a..f49fe20e00 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,6 +7,8 @@
 
 module Command.Move where
 
+import Control.Monad (when)
+
 import Command
 import qualified Command.Drop
 import qualified Annex
@@ -84,7 +86,8 @@ toPerform dest move key = do
 	-- and an explicit check is not done, when copying. When moving,
 	-- it has to be done, to avoid inaverdent data loss.
 	fast <- Annex.getState Annex.fast
-	isthere <- if fast && not move && not (Remote.hasKeyCheap dest)
+	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
+	isthere <- if fastcheck
 		then do
 			(remotes, _) <- Remote.keyPossibilities key
 			return $ Right $ dest `elem` remotes
@@ -98,7 +101,10 @@ toPerform dest move key = do
 			ok <- Remote.storeKey dest key
 			if ok
 				then next $ toCleanup dest move key
-				else stop -- failed
+				else do
+					when fastcheck $
+						warning "This could have failed because --fast is enabled."
+					stop
 		Right True -> next $ toCleanup dest move key
 toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
index 2767b5aa9c..47fc77fa4c 100644
--- a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -2,3 +2,5 @@ Conversation moved from [[walkthrough/recover_data_from_lost+found]]
 to a proper bug. --[[Joey]]
 
 (Unfortunatly that scrambled the comment creation times and thus order.)
+
+> Added a message [[done]] --[[Joey]] 

From 57428c356ea81ea13193ac5dfc32d9a824ed4d65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 13:33:33 -0400
Subject: [PATCH 1738/8313] heh

---
 doc/walkthrough/Internet_Archive_via_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/walkthrough/Internet_Archive_via_S3.mdwn
index b80f0a4b7c..8c0f2dde74 100644
--- a/doc/walkthrough/Internet_Archive_via_S3.mdwn
+++ b/doc/walkthrough/Internet_Archive_via_S3.mdwn
@@ -32,7 +32,7 @@ use all lowercase and no spaces when making the bucket with `initremote`.
 		x-archive-meta-mediatype=texts x-archive-meta-language=eng \
 		x-archive-meta-title="original Panama Canal lock design blueprints"
 	initremote archive-panama (Internet Archive mode) ok
-	# git annex describe archive-panama "Internet Archive item for my grandfather's Panama Canal lock design blueprints"
+	# git annex describe archive-panama "a man, a plan, a canal: panama"
 	describe archive-panama ok
 
 Then you can annex files and copy them to the remote as usual:

From 760cde28b67c14e0ad68e8649c0abe0544c44947 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 14:49:28 -0400
Subject: [PATCH 1739/8313] more pointless monadic golfing

---
 GitRepo.hs       | 3 +--
 Locations.hs     | 2 +-
 Remote/S3real.hs | 2 +-
 Utility.hs       | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 3c5a1e129e..87cceece41 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -577,8 +577,7 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
 				e_num c = showoctal $ ord c
 				-- unicode character is decomposed to
 				-- Word8s and each is shown in octal
-				e_utf c = concat $ map showoctal $
-						(encode [c] :: [Word8])
+				e_utf c = showoctal =<< (encode [c] :: [Word8])
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
diff --git a/Locations.hs b/Locations.hs
index 1c4f8296e8..38a320a2b2 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -175,7 +175,7 @@ prop_idempotent_fileKey s = Just k == fileKey (keyFile k)
 hashDirMixed :: Key -> FilePath
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
 	where
-		dir = take 4 $ concat $ map display_32bits_as_dir [a,b,c,d]
+		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]
 		ABCD (a,b,c,d) = md5 $ Str $ show k
 
 {- Generates a hash directory that is all lower case. -}
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 1359669036..baf570593e 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -232,7 +232,7 @@ bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty
  - with no whitespace. Other characters are xml entity
  - encoded. -}
 iaMunge :: String -> String
-iaMunge = concat . (map munge)
+iaMunge = (>>= munge)
 	where
 		munge c
 			| isAsciiUpper c || isAsciiLower c || isNumber c = [c]
diff --git a/Utility.hs b/Utility.hs
index 44c8cdd650..6dd7d329c8 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -59,7 +59,7 @@ data CommandParam = Params String | Param String | File FilePath
 {- Used to pass a list of CommandParams to a function that runs
  - a command and expects Strings. -}
 toCommand :: [CommandParam] -> [String]
-toCommand l = concat $ map unwrap l
+toCommand = (>>= unwrap)
 	where
 		unwrap (Param s) = [s]
 		unwrap (Params s) = filter (not . null) (split " " s)

From 3ab15b9f4f22aaa77a09e198d03dd4296a83bb44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 15:01:05 -0400
Subject: [PATCH 1740/8313] releasing version 0.20110516

---
 Remote.hs        | 2 +-
 debian/changelog | 6 +++---
 debian/control   | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 7bbe910000..7df84a5da6 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -188,7 +188,7 @@ configToKeyVal m = map toword $ sort $ M.toList m
 		toword (k, v) = k ++ "=" ++ configEscape v
 
 configEscape :: String -> String
-configEscape = concat . (map escape)
+configEscape = (>>= escape)
 	where
 		escape c
 			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
diff --git a/debian/changelog b/debian/changelog
index 3827a4632b..5cae0a8b51 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,5 @@
-git-annex (0.20110504) UNRELEASED; urgency=low
+git-annex (0.20110516) unstable; urgency=low
 
-  * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.
   * Add a few tweaks to make it easy to use the Internet Archive's variant
     of S3. In particular, munge key filenames to comply with the IA's filename
     limits, disable encryption, support their nonstandard way of creating
@@ -9,8 +8,9 @@ git-annex (0.20110504) UNRELEASED; urgency=low
   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
   * migrate: Use current filename when generating new key, for backends
     where the filename affects the key name.
+  * Work around a bug in Network.URI's handling of bracketed ipv6 addresses.
 
- -- Joey Hess   Fri, 06 May 2011 15:20:38 -0400
+ -- Joey Hess   Mon, 16 May 2011 14:16:52 -0400
 
 git-annex (0.20110503) unstable; urgency=low
 
diff --git a/debian/control b/debian/control
index b75b35afbc..f2f9cecd39 100644
--- a/debian/control
+++ b/debian/control
@@ -17,7 +17,7 @@ Build-Depends:
 	uuid,
 	rsync,
 Maintainer: Joey Hess 
-Standards-Version: 3.9.1
+Standards-Version: 3.9.2
 Vcs-Git: git://git.kitenet.net/git-annex
 Homepage: http://git-annex.branchable.com/
 

From 8d4d84b80f8d652a28baa12a51bf5e24681aada4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 15:01:42 -0400
Subject: [PATCH 1741/8313] add news item for git-annex 0.20110516

---
 doc/news/version_0.20110419.mdwn |  7 -------
 doc/news/version_0.20110516.mdwn | 11 +++++++++++
 2 files changed, 11 insertions(+), 7 deletions(-)
 delete mode 100644 doc/news/version_0.20110419.mdwn
 create mode 100644 doc/news/version_0.20110516.mdwn

diff --git a/doc/news/version_0.20110419.mdwn b/doc/news/version_0.20110419.mdwn
deleted file mode 100644
index 2506ad0a87..0000000000
--- a/doc/news/version_0.20110419.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.20110419 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Don't run gpg in batch mode, so it can prompt for passphrase when
-     there is no agent.
-   * Add missing build dep on dataenc.
-   * S3: Fix stalls when transferring encrypted data.
-   * bup: Avoid memory leak when transferring encrypted data."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110516.mdwn b/doc/news/version_0.20110516.mdwn
new file mode 100644
index 0000000000..aa350a4848
--- /dev/null
+++ b/doc/news/version_0.20110516.mdwn
@@ -0,0 +1,11 @@
+git-annex 0.20110516 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Add a few tweaks to make it easy to use the Internet Archive's variant
+     of S3. In particular, munge key filenames to comply with the IA's filename
+     limits, disable encryption, support their nonstandard way of creating
+     buckets, and allow x-archive-* headers to be specified in initremote to
+     set item metadata.
+   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
+   * migrate: Use current filename when generating new key, for backends
+     where the filename affects the key name.
+   * Work around a bug in Network.URI's handling of bracketed ipv6 addresses."""]]
\ No newline at end of file

From 786b990d5127ce21e2924336f6f679fb7569358d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 16 May 2011 20:01:29 +0000
Subject: [PATCH 1742/8313] Added a comment

---
 .../comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment
new file mode 100644
index 0000000000..6d3dabb92b
--- /dev/null
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 16"
+ date="2011-05-16T20:01:28Z"
+ content="""
+Thanks.
+"""]]

From a5beb61b90d318d6ab367fd761d182b658a577f4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 17 May 2011 01:15:12 +0000
Subject: [PATCH 1743/8313] Added a comment

---
 ..._d1fd70c67243971c96d59e1ffb7ef6e7._comment | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment

diff --git a/doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment b/doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment
new file mode 100644
index 0000000000..39986144be
--- /dev/null
+++ b/doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-17T01:15:10Z"
+ content="""
+What a good idea!
+
+150 lines of haskell later, I have this:
+
+
+# git annex status
+supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+supported remote types: git S3 bup directory rsync hook
+local annex keys: 32
+local annex size: 58 megabytes
+total annex keys: 38158
+total annex size: 6 terabytes (but 1632 keys have unknown size)
+backend usage: 
+	SHA1: 1789
+	WORM: 36369
+

+"""]]

From a8816efc140108cc62713cc6227db69ef96cd913 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 21:18:34 -0400
Subject: [PATCH 1744/8313] status: New subcommand to show info about an annex,
 including its size.

---
 Command/Status.hs  | 151 +++++++++++++++++++++++++++++++++++++++++++++
 Command/Unused.hs  |  18 +++---
 GitAnnex.hs        |   2 +
 debian/changelog   |   6 ++
 doc/git-annex.mdwn |  10 +++
 5 files changed, 178 insertions(+), 9 deletions(-)
 create mode 100644 Command/Status.hs

diff --git a/Command/Status.hs b/Command/Status.hs
new file mode 100644
index 0000000000..e8fce3bca1
--- /dev/null
+++ b/Command/Status.hs
@@ -0,0 +1,151 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Status where
+
+import Control.Monad.State
+import Data.Maybe
+import System.IO
+import Data.List
+import qualified Data.Map as M
+
+import qualified Annex
+import qualified BackendClass
+import qualified RemoteClass
+import qualified Remote
+import qualified Command.Unused
+import Command
+import Types
+import DataUnits
+import Content
+import Key
+
+-- a named computation that produces a statistic
+type Stat = (String, StatState String)
+
+-- cached info that multiple Stats may need
+type SizeList a = ([a], Int)
+data StatInfo = StatInfo
+	{ keysPresentCache :: (Maybe (SizeList Key))
+	, keysReferencedCache :: (Maybe (SizeList Key))
+	}
+
+-- a state monad for running Stats in
+type StatState = StateT StatInfo Annex
+
+command :: [Command]
+command = [repoCommand "status" (paramNothing) seek
+	"shows status information about the annex"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+{- Order is significant. Less expensive operations, and operations
+ - that share data go together.
+ -}
+faststats :: [Stat]
+faststats = 
+	[ supported_backends
+	, supported_remote_types
+	, local_annex_keys
+	, local_annex_size
+	]
+slowstats :: [Stat]
+slowstats =
+	[ total_annex_keys
+	, total_annex_size
+	, backend_usage
+	]
+
+start :: CommandStartNothing
+start = do
+	fast <- Annex.getState Annex.fast
+	let todo = if fast then faststats else faststats ++ slowstats
+	evalStateT (mapM_ showStat todo) (StatInfo Nothing Nothing)
+	stop
+
+stat :: String -> StatState String -> Stat
+stat desc a = (desc, a)
+
+showStat :: Stat -> StatState ()
+showStat (desc, a) = do
+	liftIO $ putStr $ desc ++ ": "
+	liftIO $ hFlush stdout
+	liftIO . putStrLn =<< a
+
+
+supported_backends :: Stat
+supported_backends = stat "supported backends" $
+	lift (Annex.getState Annex.supportedBackends) >>=
+		return . unwords . (map BackendClass.name)
+
+supported_remote_types :: Stat
+supported_remote_types = stat "supported remote types" $
+	return $ unwords $ map RemoteClass.typename Remote.remoteTypes
+
+local_annex_size :: Stat
+local_annex_size = stat "local annex size" $
+	cachedKeysPresent >>= keySizeSum
+
+total_annex_size :: Stat
+total_annex_size = stat "total annex size" $
+	cachedKeysReferenced >>= keySizeSum
+
+local_annex_keys :: Stat
+local_annex_keys = stat "local annex keys" $ 
+	return . show . snd =<< cachedKeysPresent
+
+total_annex_keys :: Stat
+total_annex_keys = stat "total annex keys" $
+	return . show . snd =<< cachedKeysReferenced
+
+backend_usage :: Stat
+backend_usage = stat "backend usage" $
+	return . usage =<< cachedKeysReferenced
+	where
+		usage (ks, _) = pp "" $ sort $ map tflip $ splits ks
+		splits :: [Key] -> [(String, Integer)]
+		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
+		tcount k = (keyBackendName k, 1)
+		tflip (a, b) = (b, a)
+		pp c [] = c
+		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
+
+cachedKeysPresent :: StatState (SizeList Key)
+cachedKeysPresent = do
+	s <- get
+	case keysPresentCache s of
+		Just v -> return v
+		Nothing -> do
+			keys <- lift $ getKeysPresent
+			let v = (keys, length keys)
+			put s { keysPresentCache = Just v }
+			return v
+
+cachedKeysReferenced :: StatState (SizeList Key)
+cachedKeysReferenced = do
+	s <- get
+	case keysReferencedCache s of
+		Just v -> return v
+		Nothing -> do
+			keys <- lift $ Command.Unused.getKeysReferenced
+			-- A given key may be referenced repeatedly.
+			-- nub does not seem too slow (yet)..
+			let uniques = nub keys
+			let v = (uniques, length uniques)
+			put s { keysReferencedCache = Just v }
+			return v
+
+keySizeSum :: SizeList Key -> StatState String
+keySizeSum (keys, len) = do
+	let knownsize = catMaybes $ map keySize keys
+	let total = roughSize storageUnits False $ foldl (+) 0 knownsize
+	let missing = len - length knownsize
+	return $ total ++
+		if missing > 0
+			then " (but " ++ show missing ++ " keys have unknown size)"
+			else ""
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a2e1c86de1..1482f057e8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -144,16 +144,16 @@ unusedKeys = do
 	if fast
 		then do
 			showNote "fast mode enabled; only finding stale files"
-			tmp <- staleKeys' gitAnnexTmpDir
-			bad <- staleKeys' gitAnnexBadDir
+			tmp <- staleKeys gitAnnexTmpDir
+			bad <- staleKeys gitAnnexBadDir
 			return ([], bad, tmp)
 		else do
 			showNote "checking for unused data..."
 			present <- getKeysPresent
 			referenced <- getKeysReferenced
 			let unused = present `exclude` referenced
-			staletmp <- staleKeys gitAnnexTmpDir present
-			stalebad <- staleKeys gitAnnexBadDir present
+			staletmp <- staleKeysPrune gitAnnexTmpDir present
+			stalebad <- staleKeysPrune gitAnnexBadDir present
 			return (unused, stalebad, staletmp)
 
 {- Finds items in the first, smaller list, that are not
@@ -182,9 +182,9 @@ getKeysReferenced = do
  - When a list of presently available keys is provided, stale keys
  - that no longer have value are deleted.
  -}
-staleKeys :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
-staleKeys dirspec present = do
-	contents <- staleKeys' dirspec
+staleKeysPrune :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
+staleKeysPrune dirspec present = do
+	contents <- staleKeys dirspec
 		
 	let stale = contents `exclude` present
 	let dup = contents `exclude` stale
@@ -195,8 +195,8 @@ staleKeys dirspec present = do
 
 	return stale
 
-staleKeys' :: (Git.Repo -> FilePath) -> Annex [Key]
-staleKeys' dirspec = do
+staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
+staleKeys dirspec = do
 	g <- Annex.gitRepo
 	let dir = dirspec g
 	exists <- liftIO $ doesDirectoryExist dir
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 736b430e60..99aec187a9 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -36,6 +36,7 @@ import qualified Command.Lock
 import qualified Command.PreCommit
 import qualified Command.Find
 import qualified Command.Whereis
+import qualified Command.Status
 import qualified Command.Migrate
 import qualified Command.Uninit
 import qualified Command.Trust
@@ -72,6 +73,7 @@ cmds = concat
 	, Command.DropUnused.command
 	, Command.Find.command
 	, Command.Whereis.command
+	, Command.Status.command
 	, Command.Migrate.command
 	, Command.Map.command
 	, Command.Upgrade.command
diff --git a/debian/changelog b/debian/changelog
index 5cae0a8b51..d759d3672a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110517) UNRELEASED; urgency=low
+
+  * status: New subcommand to show info about an annex, including its size.
+
+ -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
+
 git-annex (0.20110516) unstable; urgency=low
 
   * Add a few tweaks to make it easy to use the Internet Archive's variant
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 450b95a0dd..e2a04d27b9 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -182,6 +182,16 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
+* status
+
+  Displays some statistics and other information, including how much data
+  is in the annex.
+
+  Some of the statistics can take a while to generate, and those
+  come last. You can ctrl-c this command once it's displayed the
+  information you wanted to see. Or, use --fast to only display
+  the first, fast(ish) statistics.
+
 * migrate [path ...]
 
   Changes the specified annexed files to store their content in the

From 1e3da8efb0f303f8e03c39c0c0e521c19708c88d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:01:50 -0400
Subject: [PATCH 1745/8313] add info about any temp files and bad content files

---
 Command/Status.hs | 50 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index e8fce3bca1..a82fc9e1c1 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -18,17 +18,18 @@ import qualified BackendClass
 import qualified RemoteClass
 import qualified Remote
 import qualified Command.Unused
+import qualified GitRepo as Git
 import Command
 import Types
 import DataUnits
 import Content
 import Key
+import Locations
 
 -- a named computation that produces a statistic
-type Stat = (String, StatState String)
+type Stat = StatState (Maybe (String, StatState String))
 
 -- cached info that multiple Stats may need
-type SizeList a = ([a], Int)
 data StatInfo = StatInfo
 	{ keysPresentCache :: (Maybe (SizeList Key))
 	, keysReferencedCache :: (Maybe (SizeList Key))
@@ -37,6 +38,11 @@ data StatInfo = StatInfo
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
+type SizeList a = ([a], Int)
+
+sizeList :: [a] -> SizeList a
+sizeList l = (l, length l)
+
 command :: [Command]
 command = [repoCommand "status" (paramNothing) seek
 	"shows status information about the annex"]
@@ -53,6 +59,8 @@ faststats =
 	, supported_remote_types
 	, local_annex_keys
 	, local_annex_size
+	, tmp_size
+	, bad_data_size
 	]
 slowstats :: [Stat]
 slowstats =
@@ -69,14 +77,19 @@ start = do
 	stop
 
 stat :: String -> StatState String -> Stat
-stat desc a = (desc, a)
+stat desc a = return $ Just (desc, a)
+
+nostat :: Stat
+nostat = return $ Nothing
 
 showStat :: Stat -> StatState ()
-showStat (desc, a) = do
-	liftIO $ putStr $ desc ++ ": "
-	liftIO $ hFlush stdout
-	liftIO . putStrLn =<< a
-
+showStat s = calc =<< s
+	where
+		calc (Just (desc, a)) = do
+			liftIO $ putStr $ desc ++ ": "
+			liftIO $ hFlush stdout
+			liftIO . putStrLn =<< a
+		calc Nothing = return ()
 
 supported_backends :: Stat
 supported_backends = stat "supported backends" $
@@ -103,6 +116,12 @@ total_annex_keys :: Stat
 total_annex_keys = stat "total annex keys" $
 	return . show . snd =<< cachedKeysReferenced
 
+tmp_size :: Stat
+tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
+
+bad_data_size :: Stat
+bad_data_size = staleSize "bad keys size" gitAnnexBadDir
+
 backend_usage :: Stat
 backend_usage = stat "backend usage" $
 	return . usage =<< cachedKeysReferenced
@@ -115,6 +134,7 @@ backend_usage = stat "backend usage" $
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
+
 cachedKeysPresent :: StatState (SizeList Key)
 cachedKeysPresent = do
 	s <- get
@@ -122,7 +142,7 @@ cachedKeysPresent = do
 		Just v -> return v
 		Nothing -> do
 			keys <- lift $ getKeysPresent
-			let v = (keys, length keys)
+			let v = sizeList keys
 			put s { keysPresentCache = Just v }
 			return v
 
@@ -135,8 +155,7 @@ cachedKeysReferenced = do
 			keys <- lift $ Command.Unused.getKeysReferenced
 			-- A given key may be referenced repeatedly.
 			-- nub does not seem too slow (yet)..
-			let uniques = nub keys
-			let v = (uniques, length uniques)
+			let v = sizeList $ nub keys
 			put s { keysReferencedCache = Just v }
 			return v
 
@@ -149,3 +168,12 @@ keySizeSum (keys, len) = do
 		if missing > 0
 			then " (but " ++ show missing ++ " keys have unknown size)"
 			else ""
+
+staleSize :: String -> (Git.Repo -> FilePath) -> Stat
+staleSize label dirspec = do
+	keys <- lift (Command.Unused.staleKeys dirspec)
+	if null keys
+		then nostat
+		else stat label $ do
+			s <- keySizeSum $ sizeList keys
+			return $ s ++ " (clean up with git-annex unused)"

From 5068985020a4fbd699721cd506250316c55f129f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:19:15 -0400
Subject: [PATCH 1746/8313] rejigger what's --fast

---
 Command/Status.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index a82fc9e1c1..85a6a5a4be 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -57,14 +57,14 @@ faststats :: [Stat]
 faststats = 
 	[ supported_backends
 	, supported_remote_types
-	, local_annex_keys
-	, local_annex_size
 	, tmp_size
 	, bad_data_size
 	]
 slowstats :: [Stat]
 slowstats =
-	[ total_annex_keys
+	[ local_annex_keys
+	, local_annex_size
+	, total_annex_keys
 	, total_annex_size
 	, backend_usage
 	]

From 21953a802a0f55399288b52834cbfa970fa40d0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:22:37 -0400
Subject: [PATCH 1747/8313] am I silly to worry about length overflowing int
 max?

---
 Command/Status.hs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 85a6a5a4be..c2f7692c5f 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -38,10 +38,13 @@ data StatInfo = StatInfo
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
-type SizeList a = ([a], Int)
+-- a list with a known length
+-- (Integer is used for the length to avoid
+-- blowing up if someone annexed billions of files..)
+type SizeList a = ([a], Integer)
 
 sizeList :: [a] -> SizeList a
-sizeList l = (l, length l)
+sizeList l = (l, genericLength l)
 
 command :: [Command]
 command = [repoCommand "status" (paramNothing) seek
@@ -163,7 +166,7 @@ keySizeSum :: SizeList Key -> StatState String
 keySizeSum (keys, len) = do
 	let knownsize = catMaybes $ map keySize keys
 	let total = roughSize storageUnits False $ foldl (+) 0 knownsize
-	let missing = len - length knownsize
+	let missing = len - genericLength knownsize
 	return $ total ++
 		if missing > 0
 			then " (but " ++ show missing ++ " keys have unknown size)"

From 51cc71fac176878de2ccb960f62db419bb63d00f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:37:31 -0400
Subject: [PATCH 1748/8313] longterm todo item

---
 doc/todo/cache_key_info.mdwn | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/todo/cache_key_info.mdwn

diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
new file mode 100644
index 0000000000..d26d055129
--- /dev/null
+++ b/doc/todo/cache_key_info.mdwn
@@ -0,0 +1,36 @@
+Most of git-annex is designed to be fast no matter how many other files are
+in the annex. Things like add/get/drop/move/fsck have good locality;
+they will only operate on as many files as you need them to. 
+
+(git commit can get a little slow with a great deal of files,
+but that's out of scope -- and recent git-annex versions use queuing
+to save git add from piling up too much in the index.)
+
+But currently two git-annex commands are quite slow when annexes become large
+in quantity of files. These are unused and stats
+(Both have --fast versions that don't do as much).
+
+Both are slow because both need two peices of information that are not
+quick to look up, and require examining the whole repo, very seekily:
+
+1. The keys present in the annex. Found by looking thru .git/annex/objects.
+2. The keys referenced by files in git. Found by finding every file
+   in git, and looking at its symlink.
+
+Of these, the first is less expensive (typically, an annex does not have every
+key in it). It could be optimized fairly simply, by adding a database
+of keys present in the annex that is optimised to list them all. The
+database would be updated by the few functions that move content in and
+out.
+
+The second is harder to optimise, because the user can delete, revert,
+copy, add, etc files in git at will, and git-annex does not have a good way
+to watch that and maintain a database of what keys are being referenced.
+
+It could use a post-commit hook and examine files changed by commits, etc.
+But then staged files would be left out. It might be sufficient to 
+make --fast trust the database... except unused will suggest *deleting*
+data if nothing references it. Or maybe it could be required to have a
+clean tree with nothing staged before running git-annex unused.
+
+Anyway, this is a semi-longterm item for me. --[[Joey]] 

From 33e6425d102af17f32593e87286aaf8b11a7e1d4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 May 2011 22:49:41 -0400
Subject: [PATCH 1749/8313] tweak

---
 Command/Status.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index c2f7692c5f..39e2300219 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -169,7 +169,7 @@ keySizeSum (keys, len) = do
 	let missing = len - genericLength knownsize
 	return $ total ++
 		if missing > 0
-			then " (but " ++ show missing ++ " keys have unknown size)"
+			then aside $ "but " ++ show missing ++ " keys have unknown size"
 			else ""
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
@@ -179,4 +179,7 @@ staleSize label dirspec = do
 		then nostat
 		else stat label $ do
 			s <- keySizeSum $ sizeList keys
-			return $ s ++ " (clean up with git-annex unused)"
+			return $ s ++ aside "clean up with git-annex unused"
+
+aside :: String -> String
+aside s = "\t(" ++ s ++ ")"

From 75a3f5027f74565d909fb940893636d081d9872a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 01:59:44 -0400
Subject: [PATCH 1750/8313] tweak

---
 Command/Status.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 39e2300219..43a9484433 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -11,6 +11,7 @@ import Control.Monad.State
 import Data.Maybe
 import System.IO
 import Data.List
+import Data.Tuple
 import qualified Data.Map as M
 
 import qualified Annex
@@ -129,11 +130,10 @@ backend_usage :: Stat
 backend_usage = stat "backend usage" $
 	return . usage =<< cachedKeysReferenced
 	where
-		usage (ks, _) = pp "" $ sort $ map tflip $ splits ks
+		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
-		tflip (a, b) = (b, a)
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 

From c91929f6934fc4e94603d0fa004e824d5e2cfb65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 03:10:13 -0400
Subject: [PATCH 1751/8313] add whenM and unlessM

Just more golfing.. I am pretty sure something in a library somewhere can
do this, but I have been unable to find it.
---
 Command/Drop.hs       |  5 +----
 Command/DropUnused.hs |  5 ++---
 Command/Find.hs       |  5 ++---
 Command/Migrate.hs    | 10 +++-------
 Command/RecvKey.hs    |  6 ++----
 Command/SendKey.hs    |  7 +++----
 Command/Uninit.hs     |  7 ++-----
 Command/Unlock.hs     |  6 ++----
 Content.hs            |  6 ++----
 CopyFile.hs           |  4 +---
 GitRepo.hs            |  6 +++---
 Remote/Bup.hs         |  9 ++++-----
 Remote/Directory.hs   |  4 ++--
 Remote/Rsync.hs       |  7 +++----
 Utility.hs            | 32 +++++++++++++++++++++++++++-----
 git-annex-shell.hs    |  4 +---
 16 files changed, 60 insertions(+), 63 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 05c956fddf..07cec1a677 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,8 +7,6 @@
 
 module Command.Drop where
 
-import Control.Monad (when)
-
 import Command
 import qualified Backend
 import LocationLog
@@ -46,7 +44,6 @@ perform key backend numcopies = do
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
-	inannex <- inAnnex key
-	when inannex $ removeAnnex key
+	whenM (inAnnex key) $ removeAnnex key
 	logStatus key ValueMissing
 	return True
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 965a99ed56..1bb3b7f970 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,7 +7,6 @@
 
 module Command.DropUnused where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Directory
@@ -24,6 +23,7 @@ import qualified Remote
 import qualified GitRepo as Git
 import Backend
 import Key
+import Utility
 
 type UnusedMap = M.Map String Key
 
@@ -72,8 +72,7 @@ performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
 	g <- Annex.gitRepo
 	let f = filespec g key
-	e <- liftIO $ doesFileExist f
-	when e $ liftIO $ removeFile f
+	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
diff --git a/Command/Find.hs b/Command/Find.hs
index eecf3cd7da..9d760ff5a8 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,11 @@
 
 module Command.Find where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 
 import Command
 import Content
+import Utility
 
 command :: [Command]
 command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
@@ -23,6 +23,5 @@ seek = [withFilesInGit start]
 {- Output a list of files. -}
 start :: CommandStartString
 start file = isAnnexed file $ \(key, _) -> do
-	exists <- inAnnex key
-	when exists $ liftIO $ putStrLn file
+	whenM (inAnnex key) $ liftIO $ putStrLn file
 	stop
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 790d5d365b..09ff6df7da 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,7 +8,6 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (unless, when)
 import System.Posix.Files
 import System.Directory
 import System.FilePath
@@ -20,6 +19,7 @@ import Locations
 import Types
 import Content
 import Messages
+import Utility
 import qualified Command.Add
 
 command :: [Command]
@@ -63,9 +63,7 @@ perform file oldkey newbackend = do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
 				-- Make a hard link to the old backend's
 				-- cached key, to avoid wasting disk space.
-				liftIO $ do
-					exists <- doesFileExist t
-					unless exists $ createLink src t
+				liftIO $ unlessM (doesFileExist t) $ createLink src t
 				return True
 			if ok
 				then do
@@ -74,6 +72,4 @@ perform file oldkey newbackend = do
 					next $ Command.Add.cleanup file newkey
 				else stop
 	where
-		cleantmp t = do
-			exists <- doesFileExist t
-			when exists $ removeFile t
+		cleantmp t = whenM (doesFileExist t) $ removeFile t
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 126608f614..b49116de45 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,13 +7,13 @@
 
 module Command.RecvKey where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Exit
 
 import Command
 import CmdLine
 import Content
+import Utility
 import RsyncFile
 
 command :: [Command]
@@ -25,9 +25,7 @@ seek = [withKeys start]
 
 start :: CommandStartKey
 start key = do
-	present <- inAnnex key
-	when present $
-		error "key is already present in annex"
+	whenM (inAnnex key) $ error "key is already present in annex"
 	
 	ok <- getViaTmp key (liftIO . rsyncServerReceive)
 	if ok
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 871a530af7..7497ce3bfe 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,7 +7,6 @@
 
 module Command.SendKey where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Exit
 
@@ -15,6 +14,7 @@ import Locations
 import qualified Annex
 import Command
 import Content
+import Utility
 import RsyncFile
 
 command :: [Command]
@@ -26,9 +26,8 @@ seek = [withKeys start]
 
 start :: CommandStartKey
 start key = do
-	present <- inAnnex key
 	g <- Annex.gitRepo
 	let file = gitAnnexLocation g key
-	when present $
-		liftIO $ rsyncServerSend file
+	whenM (inAnnex key) $
+		liftIO $ rsyncServerSend file -- does not return
 	liftIO exitFailure
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index d3d7ac3398..1e96e1e6f7 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -8,7 +8,6 @@
 module Command.Uninit where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when)
 import System.Directory
 
 import Command
@@ -44,8 +43,7 @@ perform = do
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
 	let hook = Command.Init.preCommitHook repo
-	hookexists <- liftIO $ doesFileExist hook
-	when hookexists $ do
+	whenM (liftIO $ doesFileExist hook) $ do
 		c <- liftIO $ readFile hook
 		if c == Command.Init.preCommitScript
 			then liftIO $ removeFile hook
@@ -56,8 +54,7 @@ gitPreCommitHookUnWrite repo = do
 gitAttributesUnWrite :: Git.Repo -> IO ()
 gitAttributesUnWrite repo = do
 	let attributes = Git.attributes repo
-	attrexists <- doesFileExist attributes
-	when attrexists $ do
+	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
 		safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` Command.Init.attrLines) $ lines c
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index d65579ec73..161df2ddf9 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,7 +7,6 @@
 
 module Command.Unlock where
 
-import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory hiding (copyFile)
 
@@ -19,6 +18,7 @@ import Messages
 import Locations
 import Content
 import CopyFile
+import Utility
 
 command :: [Command]
 command =
@@ -38,9 +38,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
-	inbackend <- Backend.hasKey key
-	when (not inbackend) $
-		error "content not present"
+	unlessM (Backend.hasKey key) $ error "content not present"
 	
 	checkDiskSpace key
 
diff --git a/Content.hs b/Content.hs
index 0758fcdb13..ec7a3776bf 100644
--- a/Content.hs
+++ b/Content.hs
@@ -134,8 +134,7 @@ withTmp key action = do
 	let tmp = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	res <- action tmp
-	tmp_exists <- liftIO $ doesFileExist tmp
-	when tmp_exists $ liftIO $ removeFile tmp
+	liftIO $ whenM (doesFileExist tmp) $ liftIO $ removeFile tmp
 	return res
 
 {- Checks that there is disk space available to store a given key,
@@ -160,8 +159,7 @@ checkDiskSpace' adjustment key = do
 		megabyte :: Integer
 		megabyte = 1000000
 		needmorespace n = do
-			force <- Annex.getState Annex.force
-			unless force $ 
+			unlessM (Annex.getState Annex.force) $
 				error $ "not enough free space, need " ++ 
 					roughSize storageUnits True n ++
 					" more (use --force to override this check or adjust annex.diskreserve)"
diff --git a/CopyFile.hs b/CopyFile.hs
index 4575fb08ad..b08ede3c88 100644
--- a/CopyFile.hs
+++ b/CopyFile.hs
@@ -7,7 +7,6 @@
 
 module CopyFile (copyFile) where
 
-import Control.Monad (when)
 import System.Directory (doesFileExist, removeFile)
 
 import Utility
@@ -17,8 +16,7 @@ import qualified SysConfig
  - and because this allows easy access to features like cp --reflink. -}
 copyFile :: FilePath -> FilePath -> IO Bool
 copyFile src dest = do
-	e <- doesFileExist dest
-	when e $
+	whenM (doesFileExist dest) $
 		removeFile dest
 	boolSystem "cp" [params, File src, File dest]
 	where
diff --git a/GitRepo.hs b/GitRepo.hs
index 87cceece41..d070bc89ef 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -329,9 +329,9 @@ gitCommandLine repo _ = assertLocal repo $ error "internal"
 
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: Repo -> String -> [CommandParam] -> IO ()
-run repo subcommand params = assertLocal repo $ do
-	ok <- boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
-	unless ok $ error $ "git " ++ show params ++ " failed"
+run repo subcommand params = assertLocal repo $
+	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+		<|> error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns it output, lazily. 
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index d2b771bf77..51a5d05d17 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -11,7 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import IO
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad (unless, when)
+import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Process
 import System.Exit
@@ -75,8 +75,7 @@ bupSetup u c = do
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
 	showNote "bup init"
-	ok <- bup "init" buprepo []
-	unless ok $ error "bup init failed"
+	bup "init" buprepo [] <|> error "bup init failed"
 
 	storeBupUUID u buprepo
 
@@ -172,9 +171,9 @@ storeBupUUID u buprepo = do
 	if Git.repoIsUrl r
 		then do
 			showNote "storing uuid"
-			ok <- onBupRemote r boolSystem "git"
+			onBupRemote r boolSystem "git"
 				[Params $ "config annex.uuid " ++ u]
-			unless ok $ do error "ssh failed"
+				<|> error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 0cd3760d63..f69aa1256b 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -62,8 +62,8 @@ directorySetup u c = do
 	-- verify configuration is sane
 	let dir = maybe (error "Specify directory=") id $
 		M.lookup "directory" c
-	e <- liftIO $ doesDirectoryExist dir
-	when (not e) $ error $ "Directory does not exist: " ++ dir
+	liftIO $ doesDirectoryExist dir
+		<|> error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
 
 	-- The directory is stored in git config, not in this remote's
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index c15ab37a75..53418a9ef8 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -10,7 +10,7 @@ module Remote.Rsync (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad.State (liftIO, when)
+import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
 import System.Posix.Files
@@ -168,9 +168,8 @@ withRsyncScratchDir a = do
 	nuke tmp
 	return res
 	where
-		nuke d = liftIO $ do
-			e <- doesDirectoryExist d
-			when e $ liftIO $ removeDirectoryRecursive d
+		nuke d = liftIO $ 
+			doesDirectoryExist d <&> removeDirectoryRecursive d
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
diff --git a/Utility.hs b/Utility.hs
index 6dd7d329c8..5aa0afea7d 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -1,4 +1,4 @@
-{- git-annex utility functions
+{- general purpose utility functions
  -
  - Copyright 2010-2011 Joey Hess 
  -
@@ -26,6 +26,10 @@ module Utility (
 	dirContents,
 	myHomeDir,
 	catchBool,
+	whenM,
+	(<&>),
+	unlessM,
+	(<|>),
 	
 	prop_idempotent_shellEscape,
 	prop_idempotent_shellEscape_multiword,
@@ -46,7 +50,8 @@ import System.FilePath
 import System.Directory
 import Foreign (complement)
 import Data.List
-import Control.Monad (liftM2)
+import Data.Maybe
+import Control.Monad (liftM2, when, unless)
 
 {- A type for parameters passed to a shell command. A command can
  - be passed either some Params (multiple parameters can be included,
@@ -110,7 +115,7 @@ shellEscape f = "'" ++ escaped ++ "'"
 {- Unescapes a set of shellEscaped words or filenames. -}
 shellUnEscape :: String -> [String]
 shellUnEscape [] = []
-shellUnEscape s = word:(shellUnEscape rest)
+shellUnEscape s = word : shellUnEscape rest
 	where
 		(word, rest) = findword "" s
 		findword w [] = (w, "")
@@ -165,7 +170,7 @@ prop_parentDir_basics dir
 dirContains :: FilePath -> FilePath -> Bool
 dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 	where
-		norm p = maybe "" id $ absNormPath p "."
+		norm p = fromMaybe "" $ absNormPath p "."
 		a' = norm a
 		b' = norm b
 
@@ -178,7 +183,7 @@ absPath file = do
 {- Converts a filename into a normalized, absolute path
  - from the specified cwd. -}
 absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file = maybe bad id $ absNormPath cwd file
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
 	where
 		bad = error $ "unable to normalize " ++ file
 
@@ -258,3 +263,20 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
+
+{- when with a monadic conditional -}
+whenM :: Monad m => m Bool -> m () -> m ()
+whenM c a = c >>= flip when a
+
+unlessM :: Monad m => m Bool -> m () -> m ()
+unlessM c a = c >>= flip unless a
+
+(<&>) :: Monad m => m Bool -> m () -> m ()
+(<&>) = whenM
+
+(<|>) :: Monad m => m Bool -> m () -> m ()
+(<|>) = unlessM
+
+-- low fixity allows eg, foo bar <|> error $ "failed " ++ meep
+infixr 0 <&>
+infixr 0 <|>
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index e8a744748b..1487a61616 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -6,7 +6,6 @@
  -}
 
 import System.Environment
-import Control.Monad (when)
 import Data.List
 
 import qualified GitRepo as Git
@@ -66,8 +65,7 @@ builtin cmd dir params = do
 
 external :: [String] -> IO ()
 external params = do
-	ret <- boolSystem "git-shell" $ map Param $ ("-c":filterparams params)
-	when (not ret) $
+	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
 		error "git-shell failed"
 
 -- Drop all args after "--".

From e6c95a6a5e514fbe35b051c6bbec6c22793dbc05 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 17 May 2011 07:25:33 +0000
Subject: [PATCH 1752/8313]

---
 doc/todo/cache_key_info.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
index d26d055129..791584836d 100644
--- a/doc/todo/cache_key_info.mdwn
+++ b/doc/todo/cache_key_info.mdwn
@@ -10,7 +10,7 @@ But currently two git-annex commands are quite slow when annexes become large
 in quantity of files. These are unused and stats
 (Both have --fast versions that don't do as much).
 
-Both are slow because both need two peices of information that are not
+Both are slow because both need two pieces of information that are not
 quick to look up, and require examining the whole repo, very seekily:
 
 1. The keys present in the annex. Found by looking thru .git/annex/objects.

From 3154dc036abf36c59df28e78da3a8748a8240b67 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 17 May 2011 07:27:03 +0000
Subject: [PATCH 1753/8313] Added a comment

---
 ...omment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment

diff --git a/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment
new file mode 100644
index 0000000000..086e7f3e84
--- /dev/null
+++ b/doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-05-17T07:27:02Z"
+ content="""
+Sounds like a good idea.
+
+* git annex fsck (or similar) should check/rebuild the caches
+* I would simply require a clean tree with a verbose error. 80/20 rule and defaulting to save actions.
+"""]]

From 21d9c84e7292a8984ea5d46c0134ddc6ff19babc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 11:44:13 -0400
Subject: [PATCH 1754/8313] more standard names for whenM and unlessM operators

These are defined in ifelse, but it's not currently available and I don't
want to pull in a library for 6 lines of code anyhow.

Also, ifelse sets the fixity to 1, which does not allow >>? error $ ...
---
 GitRepo.hs          |  2 +-
 Remote/Bup.hs       |  4 ++--
 Remote/Directory.hs |  2 +-
 Remote/Rsync.hs     |  2 +-
 Utility.hs          | 16 ++++++++--------
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index d070bc89ef..f489dfe35d 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -331,7 +331,7 @@ gitCommandLine repo _ = assertLocal repo $ error "internal"
 run :: Repo -> String -> [CommandParam] -> IO ()
 run repo subcommand params = assertLocal repo $
 	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
-		<|> error $ "git " ++ show params ++ " failed"
+		>>! error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns it output, lazily. 
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 51a5d05d17..c40826e5eb 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -75,7 +75,7 @@ bupSetup u c = do
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
 	showNote "bup init"
-	bup "init" buprepo [] <|> error "bup init failed"
+	bup "init" buprepo [] >>! error "bup init failed"
 
 	storeBupUUID u buprepo
 
@@ -173,7 +173,7 @@ storeBupUUID u buprepo = do
 			showNote "storing uuid"
 			onBupRemote r boolSystem "git"
 				[Params $ "config annex.uuid " ++ u]
-				<|> error "ssh failed"
+					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index f69aa1256b..dedab473f3 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -63,7 +63,7 @@ directorySetup u c = do
 	let dir = maybe (error "Specify directory=") id $
 		M.lookup "directory" c
 	liftIO $ doesDirectoryExist dir
-		<|> error $ "Directory does not exist: " ++ dir
+		>>! error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
 
 	-- The directory is stored in git config, not in this remote's
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 53418a9ef8..9d32ad19b9 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -169,7 +169,7 @@ withRsyncScratchDir a = do
 	return res
 	where
 		nuke d = liftIO $ 
-			doesDirectoryExist d <&> removeDirectoryRecursive d
+			doesDirectoryExist d >>? removeDirectoryRecursive d
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
diff --git a/Utility.hs b/Utility.hs
index 5aa0afea7d..816464373b 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -27,9 +27,9 @@ module Utility (
 	myHomeDir,
 	catchBool,
 	whenM,
-	(<&>),
+	(>>?),
 	unlessM,
-	(<|>),
+	(>>!),
 	
 	prop_idempotent_shellEscape,
 	prop_idempotent_shellEscape_multiword,
@@ -271,12 +271,12 @@ whenM c a = c >>= flip when a
 unlessM :: Monad m => m Bool -> m () -> m ()
 unlessM c a = c >>= flip unless a
 
-(<&>) :: Monad m => m Bool -> m () -> m ()
-(<&>) = whenM
+(>>?) :: Monad m => m Bool -> m () -> m ()
+(>>?) = whenM
 
-(<|>) :: Monad m => m Bool -> m () -> m ()
-(<|>) = unlessM
+(>>!) :: Monad m => m Bool -> m () -> m ()
+(>>!) = unlessM
 
 -- low fixity allows eg, foo bar <|> error $ "failed " ++ meep
-infixr 0 <&>
-infixr 0 <|>
+infixr 0 >>?
+infixr 0 >>!

From ebfa50b72959de4c4e917771c57b0efcea98e55b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 12:14:46 -0400
Subject: [PATCH 1755/8313] add

---
 doc/todo/tahoe_lfs_for_reals.mdwn | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/todo/tahoe_lfs_for_reals.mdwn

diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
new file mode 100644
index 0000000000..053a763d7e
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -0,0 +1,28 @@
+[[forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] is a good
+start, but Zooko points out that using Tahoe's directory translation layer
+incurs O(N^2) overhead as the number of objects grows. Also, making
+hash subdirectories in Tahoe is expensive. Instead it would be good to use
+it as a key/value store directly. The catch is that doing so involves
+sending the content to Tahoe, and getting back a key identifier.
+
+This would be fairly easy to do as a [[backend]], which can assign its
+own key names (although typically done before data is stored in it),
+but a tahoe-lafs special remote would be more flexible.
+
+To support a special remote, a mapping is needed from git-annex keys to
+Tahoe keys.
+
+The best place to store this mapping is perhaps as a new field in the
+location log:
+
+	date present repo-uuid newfields
+
+This way, each remote can store its own key-specfic data in the same place
+as other key-specific data, with minimal overhead.
+
+Unfortunatly, the current location log parser throws out lines that it 
+cannot parse, so making this change would involve something of a flag day
+upgrade. Also unfortunatly, the location log and other .git-annex/ data
+does not have its own version that can be checked to force an upgrade
+across all clones. It might be best to deal with this at the same time
+the ideas in [[branching]] are done. --[[Joey]]

From f664a2538a848d204922de908fac9add7c1db0fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 12:18:50 -0400
Subject: [PATCH 1756/8313] fix

---
 doc/todo/tahoe_lfs_for_reals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
index 053a763d7e..4e0bf2fb16 100644
--- a/doc/todo/tahoe_lfs_for_reals.mdwn
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -5,7 +5,7 @@ hash subdirectories in Tahoe is expensive. Instead it would be good to use
 it as a key/value store directly. The catch is that doing so involves
 sending the content to Tahoe, and getting back a key identifier.
 
-This would be fairly easy to do as a [[backend]], which can assign its
+This would be fairly easy to do as a [[backend|backends]], which can assign its
 own key names (although typically done before data is stored in it),
 but a tahoe-lafs special remote would be more flexible.
 

From f63412a42c3cc3df35e0e1c945d415b81f811caa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 May 2011 12:19:52 -0400
Subject: [PATCH 1757/8313] close

---
 doc/todo/S3.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/S3.mdwn b/doc/todo/S3.mdwn
index dd0174cc55..7e417336f0 100644
--- a/doc/todo/S3.mdwn
+++ b/doc/todo/S3.mdwn
@@ -20,3 +20,5 @@ and you forget to drop it from S3, cruft can build up there.
 This could be fixed by adding a hook to list all keys present in a remote.
 Then unused could scan remotes for keys, and if they were not used locally,
 offer the possibility to drop them from the remote.
+
+[[done]]

From 44499bbdf8317664e5a2e83aa63c904cefbf64c1 Mon Sep 17 00:00:00 2001
From: zooko 
Date: Tue, 17 May 2011 19:20:39 +0000
Subject: [PATCH 1758/8313] Added a comment: performance

---
 ...comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment

diff --git a/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
new file mode 100644
index 0000000000..16ef882a42
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="97.118.97.117"
+ subject="performance"
+ date="2011-05-17T19:20:39Z"
+ content="""
+Hm... O(N^2)? I think it just takes O(N). To read an entry out of a directory you have to download the entire directory (and store it in RAM and parse it). The constants are basically \"too big to be good but not big enough to be prohibitive\", I think. jctang has reported that his special remote hook performs well enough to use, but it would be nice if it were faster.
+
+The Tahoe-LAFS folks are working on speeding up mutable files, by the way, after which we would be able to speed up directories.
+"""]]

From 93b0f21c95c32b15e8fc611f2da3dd910d8045f1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 17 May 2011 19:57:33 +0000
Subject: [PATCH 1759/8313] Added a comment

---
 ...comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment

diff --git a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
new file mode 100644
index 0000000000..a32461615f
--- /dev/null
+++ b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-05-17T19:57:33Z"
+ content="""
+Whoops! You'd only told me O(N) twice before..
+
+So this is not too high priority. I think I would like to get the per-remote storage sorted out anyway, since probably it will be the thing needed to convert the URL backend into a special remote, which would then allow ripping out the otherwise unused pluggable backend infrastructure.
+"""]]

From dd44e53c0ca3e5eec78f03a3a222b4922d6d097c Mon Sep 17 00:00:00 2001
From: zooko 
Date: Wed, 18 May 2011 04:32:17 +0000
Subject: [PATCH 1760/8313] Added a comment: Tahoe-LAFS comes with encryption

---
 ...comment_1_1afca8d7182075d46db41f6ad3dd5911._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment

diff --git a/doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment b/doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment
new file mode 100644
index 0000000000..db93bf63f8
--- /dev/null
+++ b/doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="zooko"
+ ip="75.220.153.232"
+ subject="Tahoe-LAFS comes with encryption"
+ date="2011-05-18T04:32:14Z"
+ content="""
+The Tahoe-LAFS special remote automatically encrypts and adds cryptography integrity checks/digital signatures. For that special remote you should not use the git-annex encryption scheme.
+
+Tahoe-LAFS encryption generates a new independent key for each file. This means that you can share access to one of the files without thereby sharing access to all of them, and it means that individual files can be deduplicated among multiple users.
+"""]]

From cd8354187273ee07c8953bea72df2d60b8b50c5d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 May 2011 19:34:46 -0400
Subject: [PATCH 1761/8313] --backend now overrides any backend configured in
 .gitattributes files.

---
 Annex.hs         |  4 ++--
 Backend.hs       | 15 +++++++++++----
 Options.hs       |  6 +++---
 debian/changelog |  1 +
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index b2257281fd..2148dd6252 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -38,7 +38,7 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
-	, defaultbackend :: Maybe String
+	, forcebackend :: Maybe String
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
@@ -56,7 +56,7 @@ newState gitrepo allbackends = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
-	, defaultbackend = Nothing
+	, forcebackend = Nothing
 	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
diff --git a/Backend.hs b/Backend.hs
index 6140664cee..ab15974c8a 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -33,6 +33,7 @@ module Backend (
 ) where
 
 import Control.Monad.State (liftIO, when)
+import Control.Monad (liftM)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
@@ -56,7 +57,7 @@ list = do
 		then return l
 		else do
 			s <- getstandard
-			d <- Annex.getState Annex.defaultbackend
+			d <- Annex.getState Annex.forcebackend
 			handle d s
 	where
 		parseBackendList l [] = l
@@ -161,9 +162,15 @@ lookupFile file = do
 chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))]
 chooseBackends fs = do
 	g <- Annex.gitRepo
-	bs <- Annex.getState Annex.supportedBackends
-	pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-	return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs
+	forced <- Annex.getState Annex.forcebackend
+	if forced /= Nothing
+		then do
+			l <- list
+			return $ map (\f -> (f, Just $ head l)) fs
+		else do
+			bs <- Annex.getState Annex.supportedBackends
+			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
+			return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs
 
 {- Returns the backend to use for a key. -}
 keyBackend :: Key -> Annex (Backend Annex)
diff --git a/Options.hs b/Options.hs
index 31b73e25a8..ae5707e4a0 100644
--- a/Options.hs
+++ b/Options.hs
@@ -28,11 +28,11 @@ commonOptions =
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setquiet False))
 		"allow verbose output"
-	, Option ['b'] ["backend"] (ReqArg setdefaultbackend paramName)
-		"specify default key-value backend to use"
+	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
+		"specify key-value backend to use"
 	]
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
-		setdefaultbackend v = Annex.changeState $ \s -> s { Annex.defaultbackend = Just v }
+		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
diff --git a/debian/changelog b/debian/changelog
index d759d3672a..f8b5f02cf2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (0.20110517) UNRELEASED; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
+  * --backend now overrides any backend configured in .gitattributes files.
 
  -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
 

From 22bf25b102722b35af79d85a9014801ae18d5f64 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
 
Date: Thu, 19 May 2011 08:19:09 +0000
Subject: [PATCH 1762/8313] Added a comment: gitweb supplies --no-name as of
 1.7.5.1

---
 .../comment_4_10fc013865c7542c2ed9d6c0963bb391._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment

diff --git a/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment b/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
new file mode 100644
index 0000000000..9bb9aa8ae3
--- /dev/null
+++ b/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
+ nickname="Fraser"
+ subject="gitweb supplies --no-name as of 1.7.5.1"
+ date="2011-05-19T08:19:02Z"
+ content="""
+git v1.7.5.1 fixes the gitweb gzip issue.  If the git instance is updated we
+can have stable distributions (and I can finally write a FreeBSD port ^_^)
+"""]]

From fa8ae628f28486ad5114767f1ee3cd3d4a3c5af0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 19 May 2011 16:10:36 +0000
Subject: [PATCH 1763/8313] Added a comment

---
 .../comment_5_c6b1bc40226fc2c8ba3e558150856992._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment

diff --git a/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment b/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
new file mode 100644
index 0000000000..76ba75edc4
--- /dev/null
+++ b/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-05-19T16:10:35Z"
+ content="""
+Hmm, I've upgraded to that version, but I see nothing in its changelog, commit log, code, or runtime behavior to indicate that it's producing stable gzip output.
+"""]]

From 463fc3ea58a1273704d1f500d494a9e85e2c4ea6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 19 May 2011 19:21:52 +0000
Subject: [PATCH 1764/8313] Added a comment: anonymous git push

---
 ...comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment

diff --git a/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment b/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment
new file mode 100644
index 0000000000..830d678ca1
--- /dev/null
+++ b/doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="anonymous git push"
+ date="2011-05-19T19:21:51Z"
+ content="""
+@Jimmy mentioned anonymous git push -- that is now enabled for this wiki. Enjoy!
+
+I may try to spend more time on #vcs-home -- or I can be summoned there from my other lurking places on irc, I guess.
+"""]]

From dbf02220d57948132cd0a7a76714f2b562806a08 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 May 2011 15:44:22 -0400
Subject: [PATCH 1765/8313] update with irc channel

---
 doc/contact.mdwn | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/contact.mdwn b/doc/contact.mdwn
index 1f122fdad9..22f072cb2b 100644
--- a/doc/contact.mdwn
+++ b/doc/contact.mdwn
@@ -1,4 +1,10 @@
-Joey Hess  is the author of git-annex.
+Joey Hess  is the author of git-annex. If you need to
+talk about something privatly, email me.
+
+The [[forum]] is the best place to discuss git-annex.
 
 The [VCS-home mailing list](http://lists.madduck.net/listinfo/vcs-home)
-is a good place to discuss it, or use the [[forum]].
+is a good mailing list for users who want to use git-annex in the context
+of managing their large personal files.
+
+For realtime chat, use the `#vcs-home` channel on irc.oftc.net.

From fbdaf212f0346a3543187c586f20ac2bef73ac20 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 May 2011 15:56:19 -0400
Subject: [PATCH 1766/8313] main git repo switching to branchable one

It allows anonymous git push, and it's simpler to just have it there.
The kite repo will remain, as a mirror.
---
 doc/download.mdwn | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 405bc2fc4b..19070a70aa 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -1,6 +1,11 @@
-The main git repository for git-annex is `git://git.kitenet.net/git-annex`
-[[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
-(The repo is mirrored [at github](https://github.com/joeyh/git-annex).)
+The main git repository for git-annex is `git://git-annex.branchable.com/`
+
+(You can push changes to this wiki from that anonymous git checkout.)
+
+Other mirrors of the git repository:
+
+* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
+* [at github](https://github.com/joeyh/git-annex)
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.

From 4c8ce2d86681fbc7bb7b56c556fce4d58c612d3c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 19 May 2011 20:09:34 +0000
Subject: [PATCH 1767/8313] test commit at 4:09

---
 doc/sandbox.mdwn | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/sandbox.mdwn

diff --git a/doc/sandbox.mdwn b/doc/sandbox.mdwn
new file mode 100644
index 0000000000..ed220722ee
--- /dev/null
+++ b/doc/sandbox.mdwn
@@ -0,0 +1,34 @@
+This is the SandBox, a page anyone can edit to learn how to use the wiki.
+
+Hello!
+
+----
+
+Here's a paragraph.
+
+Here's another one with *emphasised* text.
+
+# Header
+
+## Subheader
+
+> This is a blockquote.
+>
+> This is the first level of quoting.
+>
+> > This is nested blockquote.
+>
+> Back to the first level.
+
+Numbered list 
+
+1. First item.
+1. Another.
+1. And another..
+
+Bulleted list
+
+* *item*
+* item
+
+[[ikiwiki/WikiLink]]

From 813ff7f4b0cef294e020e50d4c6f973d1e7f35ae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 May 2011 16:23:05 -0400
Subject: [PATCH 1768/8313] fixed url

---
 doc/download.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 19070a70aa..3607e403f0 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -4,7 +4,7 @@ The main git repository for git-annex is `git://git-annex.branchable.com/`
 
 Other mirrors of the git repository:
 
-* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex;a=summary)]
+* `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
 Some operating systems include git-annex in easily prepackaged form and

From c7b7595f50ed276d97f18931eb7b2092f22104f4 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Fri, 20 May 2011 20:17:15 +0000
Subject: [PATCH 1769/8313] Added a comment

---
 ...mment_2_adb76f06a7997abe4559d3169a3181c3._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment

diff --git a/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment
new file mode 100644
index 0000000000..6ecce52c42
--- /dev/null
+++ b/doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="comment 2"
+ date="2011-05-20T20:14:15Z"
+ content="""
+I agree with Christian.
+
+One should first make a better use of connections to remotes before exploring parallel possibilities. One should pipeline the requests and answers.
+
+Of course this could be implemented using parallel&concurrency features of Haskell to do this.
+"""]]

From 1876db50f258a1a6a67d874049e93a84d34cac32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 11:07:08 -0400
Subject: [PATCH 1770/8313] found a few places I can use newtype

for presumably some speedups
---
 Backend.hs     | 1 -
 CryptoTypes.hs | 5 +++--
 Touch.hsc      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index ab15974c8a..645bfdfc3f 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -33,7 +33,6 @@ module Backend (
 ) where
 
 import Control.Monad.State (liftIO, when)
-import Control.Monad (liftM)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
diff --git a/CryptoTypes.hs b/CryptoTypes.hs
index 944a9d34e0..ba22c4cbe8 100644
--- a/CryptoTypes.hs
+++ b/CryptoTypes.hs
@@ -9,11 +9,12 @@ module CryptoTypes where
 
 import Data.String.Utils
 
-data Cipher = Cipher String -- XXX ideally, this would be a locked memory region
+-- XXX ideally, this would be a locked memory region
+newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
 
-data KeyIds = KeyIds [String]
+newtype KeyIds = KeyIds [String]
 
 instance Show KeyIds where
 	show (KeyIds ks) = join "," ks
diff --git a/Touch.hsc b/Touch.hsc
index fd3500f869..4f26855d21 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -16,7 +16,7 @@ module Touch (
 import Foreign
 import Foreign.C
 
-data TimeSpec = TimeSpec CTime
+newtype TimeSpec = TimeSpec CTime
 
 {- Changes the access and modification times of an existing file.
    Can follow symlinks, or not. Throws IO error on failure. -}

From 93a4f3d4e6970b05116fc25b8d57f0dd9d9ec675 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 11:52:13 -0400
Subject: [PATCH 1771/8313] Add --debug option. Closes: #627499

This takes advantage of the debug logging done by missingh, and I added
my own debug messages for executeFile calls. There are still some other
low-level ways git-annex runs stuff that are not shown by debugging,
but this gets most of it easily.
---
 Options.hs         |  7 ++++++-
 Remote/Hook.hs     |  2 +-
 RsyncFile.hs       |  1 -
 Utility.hs         | 12 +++++++++++-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  4 ++++
 6 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/Options.hs b/Options.hs
index ae5707e4a0..f8dbfb6bc0 100644
--- a/Options.hs
+++ b/Options.hs
@@ -8,6 +8,8 @@
 module Options where
 
 import System.Console.GetOpt
+import System.Log.Logger
+import Control.Monad.State (liftIO)
 
 import qualified Annex
 import Types
@@ -27,7 +29,9 @@ commonOptions =
 	, Option ['q'] ["quiet"] (NoArg (setquiet True))
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setquiet False))
-		"allow verbose output"
+		"allow verbose output (default)"
+	, Option ['d'] ["debug"] (NoArg (setdebug))
+		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
 		"specify key-value backend to use"
 	]
@@ -36,3 +40,4 @@ commonOptions =
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
+		setdebug = liftIO $ updateGlobalLogger "" $ setLevel DEBUG
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 7f2d5dbee2..dc4d392741 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -12,7 +12,7 @@ import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad.State (liftIO)
 import System.FilePath
-import System.Posix.Process
+import System.Posix.Process hiding (executeFile)
 import System.Posix.IO
 import System.IO
 import System.IO.Error (try)
diff --git a/RsyncFile.hs b/RsyncFile.hs
index afff46c0ce..48d927fcf2 100644
--- a/RsyncFile.hs
+++ b/RsyncFile.hs
@@ -7,7 +7,6 @@
 
 module RsyncFile where
 
-import System.Posix.Process
 import Data.String.Utils
 
 import Utility
diff --git a/Utility.hs b/Utility.hs
index 816464373b..47d10ed759 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -17,6 +17,7 @@ module Utility (
 	relPathDirToFile,
 	boolSystem,
 	boolSystemEnv,
+	executeFile,
 	shellEscape,
 	shellUnEscape,
 	unsetFileMode,
@@ -39,7 +40,8 @@ module Utility (
 
 import System.IO
 import System.Exit
-import System.Posix.Process
+import qualified System.Posix.Process
+import System.Posix.Process hiding (executeFile)
 import System.Posix.Signals
 import System.Posix.Files
 import System.Posix.Types
@@ -52,6 +54,7 @@ import Foreign (complement)
 import Data.List
 import Data.Maybe
 import Control.Monad (liftM2, when, unless)
+import System.Log.Logger
 
 {- A type for parameters passed to a shell command. A command can
  - be passed either some Params (multiple parameters can be included,
@@ -104,6 +107,13 @@ boolSystemEnv command params env = do
 			restoresignals oldint oldset
 			executeFile command True (toCommand params) env
 
+{- executeFile with debug logging -}
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO a
+executeFile c path p e = do
+	debugM "Utility.executeFile" $
+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
+	System.Posix.Process.executeFile c path p e
+
 {- Escapes a filename or other parameter to be safely able to be exposed to
  - the shell. -}
 shellEscape :: String -> String
diff --git a/debian/changelog b/debian/changelog
index f8b5f02cf2..16821c848d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (0.20110517) UNRELEASED; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
   * --backend now overrides any backend configured in .gitattributes files.
+  * Add --debug option. Closes: #627499
 
  -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e2a04d27b9..7f2fce9d23 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -330,6 +330,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Enable verbose logging.
 
+* --debug
+
+  Show debug messages.
+
 * --from=repository
 
   Specifies a repository that content will be retrieved from, or that

From 944b1207dcedac199729842e27d47ce89a1dd90c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 11:58:35 -0400
Subject: [PATCH 1772/8313] releasing version 0.20110521

---
 debian/changelog |  4 ++--
 doc/sandbox.mdwn | 34 ----------------------------------
 2 files changed, 2 insertions(+), 36 deletions(-)
 delete mode 100644 doc/sandbox.mdwn

diff --git a/debian/changelog b/debian/changelog
index 16821c848d..cf1c9baf95 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,10 @@
-git-annex (0.20110517) UNRELEASED; urgency=low
+git-annex (0.20110521) unstable; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
   * --backend now overrides any backend configured in .gitattributes files.
   * Add --debug option. Closes: #627499
 
- -- Joey Hess   Mon, 16 May 2011 20:27:46 -0400
+ -- Joey Hess   Sat, 21 May 2011 11:52:53 -0400
 
 git-annex (0.20110516) unstable; urgency=low
 
diff --git a/doc/sandbox.mdwn b/doc/sandbox.mdwn
deleted file mode 100644
index ed220722ee..0000000000
--- a/doc/sandbox.mdwn
+++ /dev/null
@@ -1,34 +0,0 @@
-This is the SandBox, a page anyone can edit to learn how to use the wiki.
-
-Hello!
-
-----
-
-Here's a paragraph.
-
-Here's another one with *emphasised* text.
-
-# Header
-
-## Subheader
-
-> This is a blockquote.
->
-> This is the first level of quoting.
->
-> > This is nested blockquote.
->
-> Back to the first level.
-
-Numbered list 
-
-1. First item.
-1. Another.
-1. And another..
-
-Bulleted list
-
-* *item*
-* item
-
-[[ikiwiki/WikiLink]]

From abfd8dd6c7126f012008f422725ed5b7505281d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 12:02:41 -0400
Subject: [PATCH 1773/8313] add news item for git-annex 0.20110521

---
 doc/news/version_0.20110420.mdwn | 8 --------
 doc/news/version_0.20110521.mdwn | 5 +++++
 2 files changed, 5 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_0.20110420.mdwn
 create mode 100644 doc/news/version_0.20110521.mdwn

diff --git a/doc/news/version_0.20110420.mdwn b/doc/news/version_0.20110420.mdwn
deleted file mode 100644
index bb7fee219b..0000000000
--- a/doc/news/version_0.20110420.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 0.20110420 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Update Debian build dependencies for ghc 7.
-   * Debian package is now built with S3 support.
-     Thanks Joachim Breitner for making this possible.
-   * Somewhat improved memory usage of S3, still work to do.
-     Thanks Greg Heartsfield for ongoing work to improve the hS3 library
-     for git-annex."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110521.mdwn b/doc/news/version_0.20110521.mdwn
new file mode 100644
index 0000000000..f64392a760
--- /dev/null
+++ b/doc/news/version_0.20110521.mdwn
@@ -0,0 +1,5 @@
+git-annex 0.20110521 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * status: New subcommand to show info about an annex, including its size.
+   * --backend now overrides any backend configured in .gitattributes files.
+   * Add --debug option. Closes: #[627499](http://bugs.debian.org/627499)"""]]
\ No newline at end of file

From 8ed27db18fa38c0ba975805e5d3a32b40825387d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 13:03:13 -0400
Subject: [PATCH 1774/8313] add explict build dep on hslogger

pulled in by missingh, but now used directly by git-annex
---
 debian/control | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/control b/debian/control
index f2f9cecd39..4294a64820 100644
--- a/debian/control
+++ b/debian/control
@@ -5,6 +5,7 @@ Build-Depends:
 	debhelper (>= 7.0.50),
 	ghc,
 	libghc-missingh-dev,
+	ligghc-hslogger-dev,
 	libghc-pcre-light-dev,
 	libghc-sha-dev,
 	libghc-dataenc-dev,

From 078d1dd0cbb22dbdd8d97a96c77a391d7df829c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 May 2011 20:18:35 -0400
Subject: [PATCH 1775/8313] clarify synopsis re options

---
 git-annex-shell.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 1487a61616..940db71c34 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -32,7 +32,7 @@ cmds = map adddirparam $ concat
 		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
 
 header :: String
-header = "Usage: git-annex-shell [-c] command [option ..]"
+header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
 
 main :: IO ()
 main = do

From f81c1f10e6ce452636eb06209c3702d2da05c49f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 May 2011 14:03:06 -0400
Subject: [PATCH 1776/8313] show a warning message when failing to find
 requested key

Otherwise, the user sees only a rsync protocol error message and then
git-annex's less specific failure message.
---
 Command/SendKey.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 7497ce3bfe..c2f793f8fe 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -16,6 +16,7 @@ import Command
 import Content
 import Utility
 import RsyncFile
+import Messages
 
 command :: [Command]
 command = [repoCommand "sendkey" paramKey seek
@@ -30,4 +31,5 @@ start key = do
 	let file = gitAnnexLocation g key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
+	warning "requested key is not present"
 	liftIO exitFailure

From 5b941980aa47ced29f702e2cc84346d517b78391 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 May 2011 14:12:16 -0400
Subject: [PATCH 1777/8313] Closer emulation of git's behavior when told to use
 "foo/.git" as a git repository instead of just "foo". Closes: #627563

---
 GitRepo.hs                               | 20 +++++++++++++++++---
 debian/changelog                         |  7 +++++++
 doc/bugs/weird_local_clone_confuses.mdwn |  2 ++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index f489dfe35d..24bc9b5c2b 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -77,7 +77,7 @@ import Data.Char
 import Data.Word (Word8)
 import Codec.Binary.UTF8.String (encode)
 import Text.Printf
-import Data.List (isInfixOf, isPrefixOf)
+import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
 
 import Utility
@@ -111,10 +111,24 @@ repoFromAbsPath dir
 	| "/" `isPrefixOf` dir = do
  		-- Git always looks for "dir.git" in preference to
 		-- to "dir", even if dir ends in a "/".
-		let dir' = (dropTrailingPathSeparator dir) ++ ".git"
+		let canondir = dropTrailingPathSeparator dir
+		let dir' = canondir ++ ".git"
 		e <- doesDirectoryExist dir'
-		return $ newFrom $ Dir $ if e then dir' else dir
+		if e
+			then ret dir'
+			else if "/.git" `isSuffixOf` canondir
+				then do
+					-- When dir == "foo/.git", git looks
+					-- for "foo/.git/.git", and failing
+					-- that, uses "foo" as the repository.
+					e' <- doesDirectoryExist $ dir  ".git"
+					if e'
+						then ret dir
+						else ret $ takeDirectory canondir
+				else ret dir
 	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
+	where
+		ret = return . newFrom . Dir
 
 {- Remote Repo constructor. Throws exception on invalid url. -}
 repoFromUrl :: String -> IO Repo
diff --git a/debian/changelog b/debian/changelog
index cf1c9baf95..76495c9017 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (0.20110522) UNRELEASED; urgency=low
+
+  * Closer emulation of git's behavior when told to use "foo/.git" as a
+    git repository instead of just "foo". Closes: #627563
+
+ -- Joey Hess   Sun, 22 May 2011 14:03:42 -0400
+
 git-annex (0.20110521) unstable; urgency=low
 
   * status: New subcommand to show info about an annex, including its size.
diff --git a/doc/bugs/weird_local_clone_confuses.mdwn b/doc/bugs/weird_local_clone_confuses.mdwn
index 371f6d9a5e..aa838f1670 100644
--- a/doc/bugs/weird_local_clone_confuses.mdwn
+++ b/doc/bugs/weird_local_clone_confuses.mdwn
@@ -16,3 +16,5 @@ Just tested, and the new support for bare repositories didn't solve this.
 I think this is not something git-annex should go out of its way to
 support. [[done]]
 --[[Joey]] 
+
+Later.. Fixed this after all. --[[Joey]] 

From 1ab9743b5cd0282942c28086b840aaea459acfe9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
 
Date: Sun, 22 May 2011 23:02:40 +0000
Subject: [PATCH 1778/8313] Added a comment

---
 ...omment_6_3a52993d3553deb9a413debec9a5f92d._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment

diff --git a/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment b/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
new file mode 100644
index 0000000000..0dbd88b1e5
--- /dev/null
+++ b/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
+ nickname="Fraser"
+ subject="comment 6"
+ date="2011-05-22T23:02:39Z"
+ content="""
+Whups, the fix landed in git's `maint' branch just after 1.7.5 but 1.7.5.1 was
+tagged on a different branch.
+
+Will look closer in future, and let you know when it's really released.
+"""]]

From 32a946af7d87d4c58c6f1d9dcaf201d906e4298f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 May 2011 19:07:20 -0400
Subject: [PATCH 1779/8313] clarify

---
 Options.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Options.hs b/Options.hs
index f8dbfb6bc0..7f78f44f62 100644
--- a/Options.hs
+++ b/Options.hs
@@ -40,4 +40,5 @@ commonOptions =
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
-		setdebug = liftIO $ updateGlobalLogger "" $ setLevel DEBUG
+		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
+			setLevel DEBUG

From 7a22f9d5161a24f1a98ddcead3c2ea3fd56e159c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 May 2011 00:38:51 -0400
Subject: [PATCH 1780/8313] update

---
 doc/install/OSX.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 46a285e131..a6afd408bd 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,12 +1,12 @@
 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils
+sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
-sudo port install pcre
 sudo cabal install pcre-light
 sudo cabal install quickcheck  
-sudo cabal install SHA  
+sudo cabal install SHA
+sudo cabal install dataenc
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin

From 224977900d1b9961149b753df33b54d8045448ef Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
 
Date: Fri, 27 May 2011 01:27:38 +0000
Subject: [PATCH 1781/8313] Added a comment

---
 .../comment_7_a5eebd214b135f34b18274a682211943._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment

diff --git a/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment b/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
new file mode 100644
index 0000000000..9960e0ea85
--- /dev/null
+++ b/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
+ nickname="Fraser"
+ subject="comment 7"
+ date="2011-05-27T01:27:37Z"
+ content="""
+v1.7.5.3 has it.
+"""]]

From 001edb008aee205a4f1885fe5f3335b2e2a47c33 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:20:20 -0400
Subject: [PATCH 1782/8313] Fix bug in --exclude introduced in 0.20110516.

---
 Command.hs       | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index abbe897b5b..fd49531f44 100644
--- a/Command.hs
+++ b/Command.hs
@@ -191,7 +191,7 @@ filterFiles l = do
 		else return $ filter (notExcluded $ wildsRegex exclude) l'
 	where
 		notState f = not $ stateDir `isPrefixOf` f
-		notExcluded r f = isJust $ match r f []
+		notExcluded r f = isNothing $ match r f []
 
 wildsRegex :: [String] -> Regex
 wildsRegex ws = compile regex []
diff --git a/debian/changelog b/debian/changelog
index 76495c9017..1834bea15e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (0.20110522) UNRELEASED; urgency=low
 
   * Closer emulation of git's behavior when told to use "foo/.git" as a
     git repository instead of just "foo". Closes: #627563
+  * Fix bug in --exclude introduced in 0.20110516.
 
  -- Joey Hess   Sun, 22 May 2011 14:03:42 -0400
 

From 82b88d0676a26e5ce62da91560f6bd179d663601 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:21:13 -0400
Subject: [PATCH 1783/8313] typo

---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 4294a64820..8c533af114 100644
--- a/debian/control
+++ b/debian/control
@@ -5,7 +5,7 @@ Build-Depends:
 	debhelper (>= 7.0.50),
 	ghc,
 	libghc-missingh-dev,
-	ligghc-hslogger-dev,
+	libghc-hslogger-dev,
 	libghc-pcre-light-dev,
 	libghc-sha-dev,
 	libghc-dataenc-dev,

From 7ea54e1c6ec04b644f47deb997076f804c4b558a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:28:01 -0400
Subject: [PATCH 1784/8313] releasing version 0.20110522

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 1834bea15e..991244a885 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,10 @@
-git-annex (0.20110522) UNRELEASED; urgency=low
+git-annex (0.20110522) unstable; urgency=low
 
   * Closer emulation of git's behavior when told to use "foo/.git" as a
     git repository instead of just "foo". Closes: #627563
   * Fix bug in --exclude introduced in 0.20110516.
 
- -- Joey Hess   Sun, 22 May 2011 14:03:42 -0400
+ -- Joey Hess   Fri, 27 May 2011 20:20:41 -0400
 
 git-annex (0.20110521) unstable; urgency=low
 

From de817bafa0ed871c26d59a572a2e6495cdaf55f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 May 2011 20:30:58 -0400
Subject: [PATCH 1785/8313] add news item for git-annex 0.20110522

---
 doc/news/version_0.20110425.mdwn | 7 -------
 doc/news/version_0.20110522.mdwn | 5 +++++
 2 files changed, 5 insertions(+), 7 deletions(-)
 delete mode 100644 doc/news/version_0.20110425.mdwn
 create mode 100644 doc/news/version_0.20110522.mdwn

diff --git a/doc/news/version_0.20110425.mdwn b/doc/news/version_0.20110425.mdwn
deleted file mode 100644
index 8f5c6515a0..0000000000
--- a/doc/news/version_0.20110425.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 0.20110425 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Use haskell Crypto library instead of haskell SHA library.
-   * Remove testpack from build depends for non x86 architectures where it
-     is not available. The test suite will not be run if it cannot be compiled.
-   * Avoid using absolute paths when staging location log, as that can
-     confuse git when a remote's path contains a symlink. Closes: #[621386](http://bugs.debian.org/621386)"""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110522.mdwn b/doc/news/version_0.20110522.mdwn
new file mode 100644
index 0000000000..5dccb993e2
--- /dev/null
+++ b/doc/news/version_0.20110522.mdwn
@@ -0,0 +1,5 @@
+git-annex 0.20110522 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Closer emulation of git's behavior when told to use "foo/.git" as a
+     git repository instead of just "foo". Closes: #[627563](http://bugs.debian.org/627563)
+   * Fix bug in --exclude introduced in 0.20110516."""]]
\ No newline at end of file

From 27847585cbe4792e92f8b718f003605e5a329421 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 28 May 2011 12:18:07 +0000
Subject: [PATCH 1786/8313]

---
 ..._issue_with_latest_release_0.20110522-1-gde817ba.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn

diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
new file mode 100644
index 0000000000..662e35b4b5
--- /dev/null
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -0,0 +1,9 @@
+
+[70 of 81] Compiling Command.DropUnused ( Command/DropUnused.hs, Command/DropUnused.o )
+[71 of 81] Compiling Command.Status   ( Command/Status.hs, Command/Status.o )
+
+Command/Status.hs:133:37: Not in scope: `swap'
+make: *** [git-annex] Error 1
+

+
+it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install.

From f13863281c01f7ce8aaeea3b90c8b6b5180b35a0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 28 May 2011 12:22:14 +0000
Subject: [PATCH 1787/8313]

---
 .../build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
index 662e35b4b5..5743a219b5 100644
--- a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -1,3 +1,5 @@
+A recent checkout of git-annex fails to build for me (I've installed the new dependancies as well)
+
 
 [70 of 81] Compiling Command.DropUnused ( Command/DropUnused.hs, Command/DropUnused.o )
 [71 of 81] Compiling Command.Status   ( Command/Status.hs, Command/Status.o )

From 7db690bdc2d434f9bbe36a2d3bdac7202eb355f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 28 May 2011 12:26:08 +0000
Subject: [PATCH 1788/8313]

---
 .../build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
index 5743a219b5..c5fddd63c6 100644
--- a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -8,4 +8,4 @@ Command/Status.hs:133:37: Not in scope: `swap'
 make: *** [git-annex] Error 1
 

 
-it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install.
+it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install. I ran a bisect and found that commit 75a3f5027f74565d909fb940893636d081d9872a seems to have broken git-annex for me, reverting the commit allows me to build git-annex, I have not run the tests to verify everything is working correctly though.

From c38f8264355ebf99c2e1594279120d4fbdb0efc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 11:56:48 -0400
Subject: [PATCH 1789/8313] Data.Tuple.swap not available with ghc 6.12.3

---
 Command/Status.hs                                              | 2 +-
 .../build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 43a9484433..dd518416cf 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -11,7 +11,6 @@ import Control.Monad.State
 import Data.Maybe
 import System.IO
 import Data.List
-import Data.Tuple
 import qualified Data.Map as M
 
 import qualified Annex
@@ -134,6 +133,7 @@ backend_usage = stat "backend usage" $
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
+		swap (a, b) = (b, a)
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
diff --git a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
index c5fddd63c6..a7bae50b8b 100644
--- a/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
+++ b/doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn
@@ -9,3 +9,6 @@ make: *** [git-annex] Error 1
 

 
 it fails on OSX 10.6.x with ghc 6.12.3 and a corresponding haskell-platform install. I ran a bisect and found that commit 75a3f5027f74565d909fb940893636d081d9872a seems to have broken git-annex for me, reverting the commit allows me to build git-annex, I have not run the tests to verify everything is working correctly though.
+
+> Probably `swap` appeared only in a newer GHC. I've reverted to avoid a
+> versioned build dependency. [[done]] --[[Joey]]

From d147b822bdfe46c1bd38a9490344e108944963b0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 28 May 2011 16:04:51 +0000
Subject: [PATCH 1790/8313] Added a comment

---
 .../comment_8_59a976de6c7d333709b92f7cd5830850._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment

diff --git a/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment b/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment
new file mode 100644
index 0000000000..5aa4f8c94a
--- /dev/null
+++ b/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-05-28T16:04:51Z"
+ content="""
+And that is now installed on kitenet.net and verified to work.
+"""]]

From 0359b9b63802fcf4eca2b72fb454f7e7f5c18e28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:07:10 -0400
Subject: [PATCH 1791/8313] add link to stable tarball download from new gitweb
 version

---
 doc/download.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 3607e403f0..c0d2d261d3 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,5 +7,8 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
+To download a tarball of a particular release, use an url like
+
+
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.

From 1c85e878ac3b6a181b28f72c7ab613776ee1fd88 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:24:59 -0400
Subject: [PATCH 1792/8313] add a redirect to the current version's tarball

---
 doc/download.mdwn  | 3 +--
 doc/latest-version | 1 +
 doc/tarball.mdwn   | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)
 create mode 100644 doc/latest-version
 create mode 100644 doc/tarball.mdwn

diff --git a/doc/download.mdwn b/doc/download.mdwn
index c0d2d261d3..78ab9537cf 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,8 +7,7 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-To download a tarball of a particular release, use an url like
-
+Or download a [[tarball]] of the latest release.
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
diff --git a/doc/latest-version b/doc/latest-version
new file mode 100644
index 0000000000..1c4efa2b79
--- /dev/null
+++ b/doc/latest-version
@@ -0,0 +1 @@
+0.20110522
diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
new file mode 100644
index 0000000000..d544268ad4
--- /dev/null
+++ b/doc/tarball.mdwn
@@ -0,0 +1 @@
+[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes files=latest-version]]"]]

From 8581ba59234a4fd1b60100e929f05ccd166624aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:27:02 -0400
Subject: [PATCH 1793/8313] typo

---
 doc/tarball.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
index d544268ad4..84df11d3c0 100644
--- a/doc/tarball.mdwn
+++ b/doc/tarball.mdwn
@@ -1 +1 @@
-[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes files=latest-version]]"]]
+[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"]]

From d2b199219db7d3b19f8b5e34bffa8ee4b685635a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:30:08 -0400
Subject: [PATCH 1794/8313] debugging my inline

---
 doc/tarball.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
index 84df11d3c0..5512dbd0ec 100644
--- a/doc/tarball.mdwn
+++ b/doc/tarball.mdwn
@@ -1 +1 @@
-[[!meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"]]
+meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"

From f336878765baabf55cba7519fbce5d4ee05a3349 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:33:08 -0400
Subject: [PATCH 1795/8313] lose redir page

meta does not support nested inlines inside redir urls
---
 doc/download.mdwn | 3 ++-
 doc/tarball.mdwn  | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)
 delete mode 100644 doc/tarball.mdwn

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 78ab9537cf..1e65624837 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,7 +7,8 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-Or download a [[tarball]] of the latest release.
+Or download a [tarball](http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]])
+of the latest release.
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
diff --git a/doc/tarball.mdwn b/doc/tarball.mdwn
deleted file mode 100644
index 5512dbd0ec..0000000000
--- a/doc/tarball.mdwn
+++ /dev/null
@@ -1 +0,0 @@
-meta redir="http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]]"

From e8ab4f7292f059ba028275131aa4f12e7e494f79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:34:52 -0400
Subject: [PATCH 1796/8313] remove newline

---
 doc/latest-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/latest-version b/doc/latest-version
index 1c4efa2b79..3015879ce5 100644
--- a/doc/latest-version
+++ b/doc/latest-version
@@ -1 +1 @@
-0.20110522
+0.20110522
\ No newline at end of file

From ed2b1eda909d14b604b487b9ddeb7a39bd2b8ce4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 12:37:12 -0400
Subject: [PATCH 1797/8313] bleagh

---
 doc/download.mdwn  | 4 ++--
 doc/latest-version | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)
 delete mode 100644 doc/latest-version

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 1e65624837..c0d2d261d3 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,8 +7,8 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-Or download a [tarball](http://git.kitenet.net/?p=git-annex.git;a=snapshot;sf=tgz;h=refs/tags/[[!inline raw=yes pages=latest-version]])
-of the latest release.
+To download a tarball of a particular release, use an url like
+
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
diff --git a/doc/latest-version b/doc/latest-version
deleted file mode 100644
index 3015879ce5..0000000000
--- a/doc/latest-version
+++ /dev/null
@@ -1 +0,0 @@
-0.20110522
\ No newline at end of file

From 8a4a3be9f6573c090d8d919ea1453d18264941ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 16:09:11 -0400
Subject: [PATCH 1798/8313] simplify

---
 Command.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index fd49531f44..a07ae6e097 100644
--- a/Command.hs
+++ b/Command.hs
@@ -15,6 +15,7 @@ import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
 import Data.Maybe
+import Data.String.Utils
 
 import Types
 import qualified Backend
@@ -195,11 +196,9 @@ filterFiles l = do
 
 wildsRegex :: [String] -> Regex
 wildsRegex ws = compile regex []
-	where regex = "^(" ++ wildsRegex' ws "" ++ ")"
-wildsRegex' :: [String] -> String -> String
-wildsRegex' [] c = c
-wildsRegex' (w:ws) "" = wildsRegex' ws (wildToRegex w)
-wildsRegex' (w:ws) c = wildsRegex' ws (c ++ "|" ++ wildToRegex w)
+	where
+		regex = "^(" ++ alternatives ++ ")"
+		alternatives = join "|" $ map wildToRegex ws
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool

From a19d81a42e4af0e8fc2ad0c09f1ebd72fb98cdd4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 20:01:45 -0400
Subject: [PATCH 1799/8313] show error message on unexpected parameters to
 commands that take none

Before it would exit nonzero w/o doing anything, which was confusing.
---
 Command.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index a07ae6e097..0da847d24d 100644
--- a/Command.hs
+++ b/Command.hs
@@ -176,7 +176,7 @@ withTempFile :: CommandSeekStrings
 withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
 withNothing a [] = return [a]
-withNothing _ _ = return []
+withNothing _ _ = error "This command takes no parameters."
 
 backendPairs :: CommandSeekBackendFiles
 backendPairs a files = liftM (map a) $ Backend.chooseBackends files

From fdead6b9bb0883dd982e72e955fc58dee844f2ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 21:58:48 -0400
Subject: [PATCH 1800/8313] improve error message when no remote name is
 specified

list available remotes in case user wanted to use or change one of them
---
 Command/InitRemote.hs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index ad0718e38e..460f14de2c 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -10,6 +10,8 @@ module Command.InitRemote where
 import qualified Data.Map as M
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
+import Data.Maybe
+import Data.String.Utils
 
 import Command
 import qualified Annex
@@ -32,7 +34,7 @@ seek = [withWords start]
 
 start :: CommandStartWords
 start ws = notBareRepo $ do
-	when (null ws) $ error "Specify a name for the remote"
+	when (null ws) $ needname
 
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
@@ -44,6 +46,13 @@ start ws = notBareRepo $ do
 	where
 		name = head ws
 		config = Remote.keyValToConfig $ tail ws
+		needname = do
+			let err s = error $ "Specify a name for the remote. " ++ s
+			names <- remoteNames
+			if null names
+				then err ""
+				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names
+			
 
 perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform
 perform t u c = do
@@ -83,6 +92,11 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 				| n' == n -> True
 				| otherwise -> False
 
+remoteNames :: Annex [String]
+remoteNames = do
+	m <- Remote.readRemoteLog
+	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
+
 {- find the specified remote type -}
 findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex)
 findType config = maybe unspecified specified $ M.lookup typeKey config

From 25b13673f0574ad2cc7baa055d08c13da134821a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:20:22 -0400
Subject: [PATCH 1801/8313] improve unused command's output

Display the name of the remote being checked, with "." for the current
remote, echoing the way describe takes that to change its description.
---
 Command/Unused.hs                             | 27 ++++++++++---------
 doc/special_remotes.mdwn                      |  2 +-
 .../fsck:_verifying_your_data.mdwn            |  2 +-
 doc/walkthrough/unused_data.mdwn              |  2 +-
 4 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1482f057e8..7e5549c093 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -36,21 +36,22 @@ seek = [withNothing start]
 {- Finds unused content in the annex. -} 
 start :: CommandStartNothing
 start = notBareRepo $ do
-	showStart "unused" ""
-	next perform
+	from <- Annex.getState Annex.fromremote
+	case from of
+		Nothing -> pass "." checkUnused
+		Just n -> pass n $ checkRemoteUnused n
+	where
+		pass n a = do
+			showStart "unused" n
+			next a
 
-perform :: CommandPerform
-perform = do
-	maybe checkUnused checkRemoteUnused =<< Annex.getState Annex.fromremote
-	next $ return True
-
-checkUnused :: Annex ()
+checkUnused :: CommandPerform
 checkUnused = do
 	(unused, stalebad, staletmp) <- unusedKeys
 	n  <- list "" unusedMsg unused 0
 	n' <- list "bad" staleBadMsg stalebad n
 	_  <- list "tmp" staleTmpMsg staletmp n'
-	return ()
+	next $ return True
 	where
 		list file msg l c = do
 			let unusedlist = number c l
@@ -58,13 +59,15 @@ checkUnused = do
 			writeUnusedFile file unusedlist
 			return $ length l
 
-checkRemoteUnused :: String -> Annex ()
-checkRemoteUnused name = checkRemoteUnused' =<< Remote.byName name
+checkRemoteUnused :: String -> CommandPerform
+checkRemoteUnused name = do
+	checkRemoteUnused' =<< Remote.byName name
+	next $ return True
 
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
+	showNote $ "checking for unused data..."
 	g <- Annex.gitRepo
-	showNote $ "checking for unused data on " ++ Remote.name r ++ "..."
 	referenced <- getKeysReferenced
 	logged <- liftIO $ loggedKeys g
 	remotehas <- filterM isthere logged
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index dcb6b5063e..12e0aedb1b 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -21,7 +21,7 @@ repository is found by running `git annex unused`. To detect unused content
 on special remotes, instead use `git annex unused --from`. Example:
 
 	$ git annex unused --from mys3
-	unused  (checking for unused data on mys3...) 
+	unused mys3 (checking for unused data...) 
 	  Some annexed data on mys3 is not used by any files in this repository.
 	    NUMBER  KEY
 	    1       WORM-s3-m1301674316--foo
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
index cd3a47a8a9..7e05469a12 100644
--- a/doc/walkthrough/fsck:_verifying_your_data.mdwn
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -5,7 +5,7 @@ the checksums of your files are good. Fsck also checks that the annex.numcopies
 setting is satisfied for all files.
 
 	# git annex fsck
-	unused  (checking for unused data...) ok
+	fsck some_file (checksum...) ok
 	fsck my_cool_big_file (checksum...) ok
 	...
 
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index 2f8edcd388..fb84193034 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -9,7 +9,7 @@ preserving it. So from time to time, you may want to check for such data and
 eliminate it to save space.
 
 	# git annex unused
-	unused  (checking for unused data...) 
+	unused . (checking for unused data...) 
 	  Some annexed data is no longer used by any files in the repository.
 	    NUMBER  KEY
 	    1       WORM-s3-m1289672605--file

From 66a99d3740d5996d328424c9492ed83fc36a77e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:24:48 -0400
Subject: [PATCH 1802/8313] tweak

---
 Command/Unused.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7e5549c093..08f0148dcf 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -48,9 +48,9 @@ start = notBareRepo $ do
 checkUnused :: CommandPerform
 checkUnused = do
 	(unused, stalebad, staletmp) <- unusedKeys
-	n  <- list "" unusedMsg unused 0
-	n' <- list "bad" staleBadMsg stalebad n
-	_  <- list "tmp" staleTmpMsg staletmp n'
+	_ <- list "" unusedMsg unused 0 >>=
+		list "bad" staleBadMsg stalebad >>=
+			list "tmp" staleTmpMsg staletmp
 	next $ return True
 	where
 		list file msg l c = do

From 14ffb5d47bdb07e8263a2112b1634b403a299005 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:28:14 -0400
Subject: [PATCH 1803/8313] bugfix: fix unused list numbering

Introduced in 43f0a666f0f6cc152a2b778921831d6d7daedcaf
---
 Command/Unused.hs | 2 +-
 debian/changelog  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 08f0148dcf..96001683df 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -57,7 +57,7 @@ checkUnused = do
 			let unusedlist = number c l
 			when (not $ null l) $ showLongNote $ msg unusedlist
 			writeUnusedFile file unusedlist
-			return $ length l
+			return $ c + length l
 
 checkRemoteUnused :: String -> CommandPerform
 checkRemoteUnused name = do
diff --git a/debian/changelog b/debian/changelog
index 991244a885..13b53280b4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110523) UNRELEASED; urgency=low
+
+  * Minor bugfixes and error message improvements.
+
+ -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
+
 git-annex (0.20110522) unstable; urgency=low
 
   * Closer emulation of git's behavior when told to use "foo/.git" as a

From 86c5bd0327ba1ff0d6aed8ce47031a8497a3e2fb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 May 2011 22:37:17 -0400
Subject: [PATCH 1804/8313] unused --from . checks local repo, for consistency

---
 Command/Unused.hs | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 96001683df..5422dad69f 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -37,13 +37,12 @@ seek = [withNothing start]
 start :: CommandStartNothing
 start = notBareRepo $ do
 	from <- Annex.getState Annex.fromremote
-	case from of
-		Nothing -> pass "." checkUnused
-		Just n -> pass n $ checkRemoteUnused n
-	where
-		pass n a = do
-			showStart "unused" n
-			next a
+	let (name, action) = case from of
+		Nothing -> (".", checkUnused)
+		Just "." -> (".", checkUnused)
+		Just n -> (n, checkRemoteUnused n)
+	showStart "unused" name
+	next action
 
 checkUnused :: CommandPerform
 checkUnused = do

From c4f848d29f0ee79a8a46bd27ff141d4c62257c76 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 31 May 2011 17:47:13 +0000
Subject: [PATCH 1805/8313]

---
 .../__34__git_annex_lock__34___very_slow_for_big_repo.mdwn   | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
new file mode 100644
index 0000000000..8ca3916847
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
@@ -0,0 +1,5 @@
+I found the command "git annex lock" very slow (much slower than the initial "git annex add" with SHA1), for a not so big directory, when run in a big repo.
+It seems that each underlying git command is not fast, so I thought it would be better to run them once with all files as arguments.
+I had to stop the lock command, and ran "git checkout ." (I did not change any file), is this a correct alternative?
+
+Thanks a LOT for this software, one that I missed since a long time (but wasn't able to write)!

From 6e6e77f9c05fe67a047e93b93bc198e6d9f431ce Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 31 May 2011 17:47:34 +0000
Subject: [PATCH 1806/8313]

---
 .../__34__git_annex_lock__34___very_slow_for_big_repo.mdwn      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
index 8ca3916847..9bacf28dc0 100644
--- a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn
@@ -3,3 +3,5 @@ It seems that each underlying git command is not fast, so I thought it would be
 I had to stop the lock command, and ran "git checkout ." (I did not change any file), is this a correct alternative?
 
 Thanks a LOT for this software, one that I missed since a long time (but wasn't able to write)!
+
+Rafaël

From fafe60768f98ed33b8aae6cba147a15a6608eaaf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 May 2011 14:50:41 -0400
Subject: [PATCH 1807/8313] Massively sped up `git annex lock` by avoiding use
 of the uber-slow `git reset`, and only running `git checkout` once, even when
 many files are being locked.

---
 Command/Lock.hs  | 12 +++++-------
 debian/changelog |  3 +++
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/Command/Lock.hs b/Command/Lock.hs
index 1ae4882272..e21792143a 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -12,8 +12,7 @@ import System.Directory
 
 import Command
 import Messages
-import qualified Annex
-import qualified GitRepo as Git
+import qualified AnnexQueue
 import Utility
 	
 command :: [Command]
@@ -31,9 +30,8 @@ start (file, _) = do
 perform :: FilePath -> CommandPerform
 perform file = do
 	liftIO $ removeFile file
-	g <- Annex.gitRepo
-	-- first reset the file to drop any changes checked into the index
-	liftIO $ Git.run g "reset" [Params "-q --", File file]
-	-- checkout the symlink
-	liftIO $ Git.run g "checkout" [Param "--", File file]
+	-- Checkout from HEAD to get rid of any changes that might be 
+	-- staged in the index, and get back to the previous symlink to
+	-- the content.
+	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] file
 	next $ return True -- no cleanup needed
diff --git a/debian/changelog b/debian/changelog
index 13b53280b4..783496614f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (0.20110523) UNRELEASED; urgency=low
 
   * Minor bugfixes and error message improvements.
+  * Massively sped up `git annex lock` by avoiding use of the uber-slow
+    `git reset`, and only running `git checkout` once, even when many files
+    are being locked.
 
  -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
 

From 181920fab9d352d74404d0e64e0728b6654922d9 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 31 May 2011 18:51:13 +0000
Subject: [PATCH 1808/8313] Added a comment: fixed

---
 ...t_1_044f1c5e5f7a939315c28087495a8ba8._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment
new file mode 100644
index 0000000000..0e2773bda3
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="fixed"
+ date="2011-05-31T18:51:13Z"
+ content="""
+Running `git checkout` by hand is fine, of course.
+
+Underlying problem is that git has some O(N) scalability of operations on the index with regards to the number of files in the repo. So a repo with a whole lot of files will have a big index, and any operation that changes the index, like the `git reset` this needs to do, has to read in the entire index, and write out a new, modified version. It seems that git could be much smarter about its index data structures here, but I confess I don't understand the index's data structures at all. I hope someone takes it on, as git's scalability to number of files in the repo is becoming a new pain point, now that scalability to large files is \"solved\". ;)
+
+Still, it is possible to speed this up at git-annex's level. Rather than doing a `git reset` followed by a git checkout, it can just `git checkout HEAD -- file`, and since that's one command, it can then be fed into the queueing machinery in git-annex (that exists mostly to work around this git malfescence), and so only a single git command will need to be run to lock multiple files.
+
+I've just implemented the above. In my music repo, this changed an lock of a CD's worth of files from taking ctrl-c long to 1.75 seconds. Enjoy!
+
+(Hey, this even speeds up the one file case greatly, since `git reset -- file` is slooooow -- it seems to scan the *entire* repository tree. Yipes.)
+"""]]

From fb259033d44cda1e2470d5029940d7b0725b4add Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 May 2011 14:58:06 -0400
Subject: [PATCH 1809/8313] Fix locking of files with staged changes.

Previously, lock would skip files that had staged changes, but that is
counterintuitive, I think.
---
 Command/Lock.hs  | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/Lock.hs b/Command/Lock.hs
index e21792143a..e55cd9e79a 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -19,7 +19,7 @@ command :: [Command]
 command = [repoCommand "lock" paramPath seek "undo unlock command"]
 
 seek :: [CommandSeek]
-seek = [withFilesUnlocked start]
+seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
 {- Undo unlock -}
 start :: CommandStartBackendFile
diff --git a/debian/changelog b/debian/changelog
index 783496614f..393a6161bd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (0.20110523) UNRELEASED; urgency=low
   * Massively sped up `git annex lock` by avoiding use of the uber-slow
     `git reset`, and only running `git checkout` once, even when many files
     are being locked.
+  * Fix locking of files with staged changes.
 
  -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
 

From 038da52bdd94311e4b9512a21952ce0f256643e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 May 2011 16:08:37 -0400
Subject: [PATCH 1810/8313] Somewhat sped up `git commit` of modifications to
 unlocked files.

Avoid git reset here too, so I no longer need to care that it's much more
expensive than seems wise (but I asked the git list about that anyway).

It's not necessary to reset the staged file content from the index, as
the `git add` of the the symlink will replace it anyway.

`git commit` of unlocked files is still slow, since git still has to shove
their entire content into the index, only to have it be thrown away. So it's
still better to use `git annex add`
---
 Command/PreCommit.hs | 12 +-----------
 debian/changelog     |  1 +
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index d7f2487137..6d8f7b9b3f 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -33,15 +33,5 @@ perform :: BackendFile -> CommandPerform
 perform pair@(file, _) = do
 	ok <- doCommand $ Command.Add.start pair
 	if ok
-		then next $ cleanup file
+		then next $ return True
 		else error $ "failed to add " ++ file ++ "; canceling commit"
-
-cleanup :: FilePath -> CommandCleanup
-cleanup file = do
-	-- git commit will have staged the file's content;
-	-- drop that and run command queued by Add.state to
-	-- stage the symlink
-	g <- Annex.gitRepo
-	liftIO $ Git.run g "reset" [Params "-q --", File file]
-	AnnexQueue.flush True
-	return True
diff --git a/debian/changelog b/debian/changelog
index 393a6161bd..ceb218de55 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (0.20110523) UNRELEASED; urgency=low
     `git reset`, and only running `git checkout` once, even when many files
     are being locked.
   * Fix locking of files with staged changes.
+  * Somewhat sped up `git commit` of modifications to unlocked files.
 
  -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
 

From 7760ba9d97dbade1701797714858df3d44587e1f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 31 May 2011 21:43:23 +0000
Subject: [PATCH 1811/8313] Added a comment

---
 ...ment_2_e854b93415d5ab80eda8e3be3b145ec2._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment
new file mode 100644
index 0000000000..9e9e778ce9
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 2"
+ date="2011-05-31T21:43:22Z"
+ content="""
+Nice!
+So if I understand correctly, 'git reset -- file' was there to discard staged (but not commited) changes made to 'file', before checking out, so that it is equivalent to directly 'git checkout HEAD -- file' ?
+I'm curious about the \"queueing machinery in git-annex\": does it end up calling the one git command with multiple files as arguments? does it correspond to the message \"(Recording state in git...)\" ?
+Thanks!
+
+
+"""]]

From b3f6621a5342f1c0ff373339ded491af9fd997ca Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 31 May 2011 21:54:23 +0000
Subject: [PATCH 1812/8313] Added a comment

---
 .../comment_3_95c110500bc54013bc1969c1a9c8f842._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment

diff --git a/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment
new file mode 100644
index 0000000000..87da0c396d
--- /dev/null
+++ b/doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-05-31T21:54:23Z"
+ content="""
+@Rafaël , you're correct on all counts.
+"""]]

From 869cb82f4927adff636051645dd3c693ed22ff31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 11:53:43 -0400
Subject: [PATCH 1813/8313] remove unnecessary imports

---
 Command/PreCommit.hs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 6d8f7b9b3f..3046d35626 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -7,15 +7,9 @@
 
 module Command.PreCommit where
 
-import Control.Monad.State (liftIO)
-
 import Command
-import qualified Annex
-import qualified AnnexQueue
-import qualified GitRepo as Git
 import qualified Command.Add
 import qualified Command.Fix
-import Utility
 
 command :: [Command]
 command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]

From dc92a788c767f14533147758044a019737707a52 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 12:00:25 -0400
Subject: [PATCH 1814/8313] releasing version 0.20110601

---
 debian/changelog | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ceb218de55..5e10b3d268 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110523) UNRELEASED; urgency=low
+git-annex (0.20110601) unstable; urgency=low
 
   * Minor bugfixes and error message improvements.
   * Massively sped up `git annex lock` by avoiding use of the uber-slow
@@ -6,8 +6,9 @@ git-annex (0.20110523) UNRELEASED; urgency=low
     are being locked.
   * Fix locking of files with staged changes.
   * Somewhat sped up `git commit` of modifications to unlocked files.
+  * Build fix for older ghc.
 
- -- Joey Hess   Sat, 28 May 2011 22:29:37 -0400
+ -- Joey Hess   Wed, 01 Jun 2011 11:50:47 -0400
 
 git-annex (0.20110522) unstable; urgency=low
 

From e47de0f6cd3c7309e88b59e554e710779e4a6878 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 12:02:20 -0400
Subject: [PATCH 1815/8313] add news item for git-annex 0.20110601

---
 doc/news/version_0.20110427.mdwn | 8 --------
 doc/news/version_0.20110601.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_0.20110427.mdwn
 create mode 100644 doc/news/version_0.20110601.mdwn

diff --git a/doc/news/version_0.20110427.mdwn b/doc/news/version_0.20110427.mdwn
deleted file mode 100644
index 2764c6de56..0000000000
--- a/doc/news/version_0.20110427.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 0.20110427 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Switch back to haskell SHA library, so git-annex remains buildable on
-     Debian stable.
-   * Added rsync special remotes. This could be used, for example, to
-     store annexed content on rsync.net (encrypted naturally). Or anywhere else.
-   * Bugfix: Avoid pipeline stall when running git annex drop or fsck on a
-     lot of files. Possibly only occured with ghc 7."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110601.mdwn b/doc/news/version_0.20110601.mdwn
new file mode 100644
index 0000000000..59079088da
--- /dev/null
+++ b/doc/news/version_0.20110601.mdwn
@@ -0,0 +1,9 @@
+git-annex 0.20110601 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Minor bugfixes and error message improvements.
+   * Massively sped up `git annex lock` by avoiding use of the uber-slow
+     `git reset`, and only running `git checkout` once, even when many files
+     are being locked.
+   * Fix locking of files with staged changes.
+   * Somewhat sped up `git commit` of modifications to unlocked files.
+   * Build fix for older ghc."""]]
\ No newline at end of file

From 5aec8b8f160834e3b11048c1de54b6f7dabd81ea Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 17:08:15 +0000
Subject: [PATCH 1816/8313]

---
 doc/forum/new_microfeatures.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/forum/new_microfeatures.mdwn

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
new file mode 100644
index 0000000000..cf3ddfd628
--- /dev/null
+++ b/doc/forum/new_microfeatures.mdwn
@@ -0,0 +1,20 @@
+I'm soliciting ideas for new small features that let git-annex do things that currently have to be done manually or whatever.
+
+Here are a few I've been considering:
+
+--
+
+* --numcopies would be a useful command line switch.
+* A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
+
+Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
+
+---
+
+[[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
+
+---
+
+Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
+
+--[[Joey]]

From af7a0692637382bb00d44aedd12124f58de62d2e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 17:08:52 +0000
Subject: [PATCH 1817/8313]

---
 doc/forum/new_microfeatures.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index cf3ddfd628..4b4389e3d1 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -2,7 +2,7 @@ I'm soliciting ideas for new small features that let git-annex do things that cu
 
 Here are a few I've been considering:
 
---
+---
 
 * --numcopies would be a useful command line switch.
 * A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 

From 82e83528f41de789275435076806afe27b53332e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Wed, 1 Jun 2011 17:36:51 +0000
Subject: [PATCH 1818/8313] Added a comment

---
 .../comment_1_058bd517c6fffaf3446b1f5d5be63623._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment

diff --git a/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment b/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment
new file mode 100644
index 0000000000..84fdd325dc
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2011-06-01T17:36:50Z"
+ content="""
+I've been longing for an automated way of removing references to a remote assuming I know the exact uuid that I want to remove. i.e. I have lost a portable HDD due to a destructive process, I now want to delete all references to copies of data that was on that disk. Unless this feature exists, I would love to see it implemented.
+"""]]

From 12e0e95916b81352b6f0dfdf95d3e623d4a12738 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 20:24:33 +0000
Subject: [PATCH 1819/8313] Added a comment

---
 .../comment_2_41ad904c68e89c85e1fc49c9e9106969._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment

diff --git a/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment b/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment
new file mode 100644
index 0000000000..4451e20baf
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-06-01T20:24:33Z"
+ content="""
+@jimmy [[walkthrough/what_to_do_when_you_lose_a_repository]].. I have not seen a convincing argument that removing the location tracking data entirely serves any purpose
+"""]]

From 3d567aa64f263c6ee55c92e8b962087de063ebc8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 16:49:17 -0400
Subject: [PATCH 1820/8313] Add --numcopies option.

---
 Annex.hs                         |  2 ++
 Backend/File.hs                  | 12 ++++++++----
 GitAnnex.hs                      | 12 ++++++++----
 debian/changelog                 |  6 ++++++
 doc/forum/new_microfeatures.mdwn |  3 +++
 doc/git-annex.mdwn               |  5 +++++
 6 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 2148dd6252..c5a098d986 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -39,6 +39,7 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, forcebackend :: Maybe String
+	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
@@ -57,6 +58,7 @@ newState gitrepo allbackends = AnnexState
 	, force = False
 	, fast = False
 	, forcebackend = Nothing
+	, forcenumcopies = Nothing
 	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
diff --git a/Backend/File.hs b/Backend/File.hs
index b86413e400..543f02af76 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -152,12 +152,16 @@ showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
 		(join ", " $ map Remote.name remotes)
 
+{- If a value is specified, it is used; otherwise the default is looked up
+ - in git config. forcenumcopies overrides everything. -}
 getNumCopies :: Maybe Int -> Annex Int
-getNumCopies (Just n) = return n
-getNumCopies Nothing = do
-	g <- Annex.gitRepo
-	return $ read $ Git.configGet g config "1"
+getNumCopies v = 
+	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
 	where
+		use (Just n) = return n
+		use Nothing = do
+			g <- Annex.gitRepo
+			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
 
 {- Ideally, all keys have file size metadata. Old keys may not. -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 99aec187a9..37a73424af 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -13,6 +13,7 @@ import qualified GitRepo as Git
 import CmdLine
 import Command
 import Options
+import Utility
 import qualified Annex
 
 import qualified Command.Add
@@ -82,20 +83,23 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['k'] ["key"] (ReqArg setkey paramKey)
-		"specify a key to use"
-	, Option ['t'] ["to"] (ReqArg setto paramRemote)
+	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
 	, Option ['x'] ["exclude"] (ReqArg addexclude paramGlob)
 		"skip files matching the glob pattern"
+	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
+		"override default number of copies"
+	, Option ['k'] ["key"] (ReqArg setkey paramKey)
+		"specify a key to use"
 	]
 	where
-		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) }
+		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
+		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/debian/changelog b/debian/changelog
index 5e10b3d268..0d531a3206 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (0.20110602) UNRELEASED; urgency=low
+
+  * Add --numcopies option.
+
+ -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
+
 git-annex (0.20110601) unstable; urgency=low
 
   * Minor bugfixes and error message improvements.
diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index 4b4389e3d1..fe91addca8 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -5,6 +5,9 @@ Here are a few I've been considering:
 ---
 
 * --numcopies would be a useful command line switch.
+  > Update: Added. Also allows for things like `git annex drop
+  > --numcopies=2` when in a repo that normally needs 3 copies, if you need
+  > to urgently free up space.
 * A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
 
 Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 7f2fce9d23..e4924d373d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -354,6 +354,11 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   This option can be specified multiple times.
 
+* --numcopies=n
+
+  Overrides the `annex.numcopies` setting, forcing git-annex to ensure the
+  specified number of copies exist.
+
 * --backend=name
 
   Specifies which key-value backend to use. This can be used when

From e280c0a4fca41eee68961db86446c91ed2d6adab Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 1 Jun 2011 21:26:39 +0000
Subject: [PATCH 1821/8313]

---
 doc/forum/new_microfeatures.mdwn | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index fe91addca8..9b1f8231c0 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -16,8 +16,35 @@ Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=r
 
 [[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
 
+  > Contrary to the "basic" solution, I would love to have a git annex distribute which is smart enough to simply distribute all data according to certain rules. My ideal, personal use case during the next holidays where I will have two external disks, several SD cards with 32 GB each and a local disk with 20 GB (yes....) would be:
+
+    cd ~/photos.annex # this repository does not have any objects!
+    git annex inject --bare /path/to/SD/card  # this adds softlinks, but does **not** add anything to the index. it would calculate checksums (if enabled) and have to add a temporary location list, though
+    git annex distribute # this checks the config. it would see that my two external disks have a low cost whereas the two remotes have a higher cost.
+     # check numcopies. it's 3
+     # copy to external disk one (cost x)
+     # copy to external disk two (cost x)
+     # copy to remote one (cost x * 2)
+     # remove file from temporary tracking list
+    git annex fsck # everything ok. yay!
+
+Come to think of it, the inject --bare thing is probably not a microfeature. Should I add a new wishlist item for that? -- RichiH
+
 ---
 
 Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
 
+---
+
+If a remote could send on received files to another remote, I could use my own local bandwith efficiently while still having my git-annex repos replicate data. -- RichiH
+
+---
+
+Really micro:
+
+    % grep annex-push .git/config
+        annex-push = !git pull && git annex add . && git annex copy . --to origin --fast --quiet && git commit -a -m "$HOST $(date +%F--%H-%M-%S-%Z)" && git push
+    %
+
+-- RichiH
 --[[Joey]]

From 7a3d9d8c2e2bd53d0d4290e99186c6e37f18456d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 16:54:28 -0400
Subject: [PATCH 1822/8313] mention --numcopies

---
 doc/copies.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 39a714d3bb..16eba19c81 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -8,7 +8,8 @@ to keep N copies of a file's content available across all repositories.
 
 By default, N is 1; it is configured by annex.numcopies. This default
 can be overridden on a per-file-type basis by the annex.numcopies
-setting in `.gitattributes` files.
+setting in `.gitattributes` files. The --numcopies switch allows
+temporarily using a different value.
 
 `git annex drop` attempts to check with other git remotes, to check that N
 copies of the file exist. If enough repositories cannot be verified to have

From a8fb97d2ce8e75b36b8e1572a83efd341e67d43e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 17:49:37 -0400
Subject: [PATCH 1823/8313] Add --trust, --untrust, and --semitrust options.

---
 Annex.hs                         |  3 +++
 Backend/File.hs                  |  5 ++--
 Command/Move.hs                  |  5 ++--
 GitAnnex.hs                      | 14 ++++++++---
 Remote.hs                        | 28 +--------------------
 RemoteUtils.hs                   | 42 ++++++++++++++++++++++++++++++++
 Trust.hs                         | 30 +++++++++--------------
 TrustLevel.hs                    | 23 +++++++++++++++++
 debian/changelog                 |  1 +
 doc/forum/new_microfeatures.mdwn |  5 ++--
 doc/git-annex.mdwn               |  8 ++++++
 doc/trust.mdwn                   | 11 ++++++---
 12 files changed, 117 insertions(+), 58 deletions(-)
 create mode 100644 RemoteUtils.hs
 create mode 100644 TrustLevel.hs

diff --git a/Annex.hs b/Annex.hs
index c5a098d986..13505de468 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -24,6 +24,7 @@ import qualified GitQueue
 import qualified BackendClass
 import qualified RemoteClass
 import qualified CryptoTypes
+import TrustLevel
 
 -- git-annex's monad
 type Annex = StateT AnnexState IO
@@ -44,6 +45,7 @@ data AnnexState = AnnexState
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, exclude :: [String]
+	, forcetrust :: [(String, TrustLevel)]
 	, cipher :: Maybe CryptoTypes.Cipher
 	}
 
@@ -63,6 +65,7 @@ newState gitrepo allbackends = AnnexState
 	, toremote = Nothing
 	, fromremote = Nothing
 	, exclude = []
+	, forcetrust = []
 	, cipher = Nothing
 	}
 
diff --git a/Backend/File.hs b/Backend/File.hs
index 543f02af76..58506c861b 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -21,6 +21,7 @@ import Data.String.Utils
 import BackendClass
 import LocationLog
 import qualified Remote
+import qualified RemoteUtils
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -53,7 +54,7 @@ dummyStore _ _ = return True
  - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- Remote.keyPossibilities key
+	(remotes, _) <- RemoteUtils.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -96,7 +97,7 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilities key
+			(remotes, trusteduuids) <- RemoteUtils.keyPossibilities key
 			untrusteduuids <- trustGet UnTrusted
 			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
diff --git a/Command/Move.hs b/Command/Move.hs
index f49fe20e00..6a23aee92a 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -16,6 +16,7 @@ import LocationLog
 import Types
 import Content
 import qualified Remote
+import qualified RemoteUtils
 import UUID
 import Messages
 
@@ -89,7 +90,7 @@ toPerform dest move key = do
 	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
 	isthere <- if fastcheck
 		then do
-			(remotes, _) <- Remote.keyPossibilities key
+			(remotes, _) <- RemoteUtils.keyPossibilities key
 			return $ Right $ dest `elem` remotes
 		else Remote.hasKey dest key
 	case isthere of
@@ -123,7 +124,7 @@ fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	(remotes, _) <- Remote.keyPossibilities key
+	(remotes, _) <- RemoteUtils.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then stop
 		else do
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 37a73424af..64b0888b05 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -14,6 +14,7 @@ import CmdLine
 import Command
 import Options
 import Utility
+import TrustLevel
 import qualified Annex
 
 import qualified Command.Add
@@ -83,7 +84,9 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
+	[ Option ['k'] ["key"] (ReqArg setkey paramKey)
+		"specify a key to use"
+	, Option ['t'] ["to"] (ReqArg setto paramRemote)
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
@@ -91,8 +94,12 @@ options = commonOptions ++
 		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
-	, Option ['k'] ["key"] (ReqArg setkey paramKey)
-		"specify a key to use"
+	, Option [] ["trust"] (ReqArg (settrust Trusted) paramRemote)
+		"override trust setting"
+	, Option [] ["semitrust"] (ReqArg (settrust SemiTrusted) paramRemote)
+		"override trust setting back to default value"
+	, Option [] ["untrust"] (ReqArg (settrust UnTrusted) paramRemote)
+		"override trust setting to untrusted"
 	]
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
@@ -100,6 +107,7 @@ options = commonOptions ++
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
+		settrust t v = Annex.changeState $ \s -> s { Annex.forcetrust = (v, t):(Annex.forcetrust s) }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Remote.hs b/Remote.hs
index 7df84a5da6..51da5e4715 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,9 +16,9 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
+	genList,
 	byName,
 	nameToUUID,
-	keyPossibilities,
 	remotesWithUUID,
 	remotesWithoutUUID,
 
@@ -42,8 +42,6 @@ import RemoteClass
 import Types
 import UUID
 import qualified Annex
-import Trust
-import LocationLog
 import Locations
 import Utility
 import Config
@@ -104,30 +102,6 @@ nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = liftM uuid (byName n)
 
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - Also returns a list of UUIDs that are trusted to have the key
- - (some may not have configured remotes).
- -}
-keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilities key = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	trusted <- trustGet Trusted
-
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
-	let validuuids = filter (/= u) uuids
-
-	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
-
-	-- remotes that match uuids that have the key
-	allremotes <- genList
-	let validremotes = remotesWithUUID allremotes validuuids
-
-	return (sort validremotes, validtrusteduuids)
-
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
diff --git a/RemoteUtils.hs b/RemoteUtils.hs
new file mode 100644
index 0000000000..d042780e46
--- /dev/null
+++ b/RemoteUtils.hs
@@ -0,0 +1,42 @@
+{- git-annex remotes overflow (can't go in there due to dependency cycles)
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteUtils where
+
+import Control.Monad.State (liftIO)
+import Data.List
+
+import Annex
+import Trust
+import Remote
+import UUID
+import LocationLog
+import Key
+
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -
+ - Also returns a list of UUIDs that are trusted to have the key
+ - (some may not have configured remotes).
+ -}
+keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	trusted <- trustGet Trusted
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- liftIO $ keyLocations g key
+	let validuuids = filter (/= u) uuids
+
+	-- note that validuuids is assumed to not have dups
+	let validtrusteduuids = intersect validuuids trusted
+
+	-- remotes that match uuids that have the key
+	allremotes <- genList
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return (sort validremotes, validtrusteduuids)
diff --git a/Trust.hs b/Trust.hs
index 7b2cf9ff88..d6d0516abb 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -1,4 +1,4 @@
-{- git-annex trust levels
+{- git-annex trust
  -
  - Copyright 2010 Joey Hess 
  -
@@ -17,26 +17,15 @@ module Trust (
 import Control.Monad.State
 import qualified Data.Map as M
 
+import TrustLevel
 import qualified GitRepo as Git
 import Types
 import UUID
 import Locations
 import qualified Annex
+import qualified Remote
 import Utility
 
-data TrustLevel = SemiTrusted | UnTrusted | Trusted
-	deriving Eq
-
-instance Show TrustLevel where
-        show SemiTrusted = "?"
-        show UnTrusted = "0"
-        show Trusted = "1"
-
-instance Read TrustLevel where
-        readsPrec _ "1" = [(Trusted, "")]
-        readsPrec _ "0" = [(UnTrusted, "")]
-	readsPrec _ _ = [(SemiTrusted, "")]
-
 {- Filename of trust.log. -}
 trustLog :: Annex FilePath
 trustLog = do
@@ -49,18 +38,23 @@ trustGet level = do
 	m <- trustMap
 	return $ M.keys $ M.filter (== level) m
 
-{- Read the trustLog into a map. -}
+{- Read the trustLog into a map, overriding with any
+ - values from forcetrust -}
 trustMap :: Annex (M.Map UUID TrustLevel)
 trustMap = do
 	logfile <- trustLog
+	overrides <- Annex.getState Annex.forcetrust >>= mapM findoverride
 	s <- liftIO $ catch (readFile logfile) ignoreerror
-	return $ trustMapParse s
+	return $ M.fromList $ trustMapParse s ++ overrides
 	where
                 ignoreerror _ = return ""
+		findoverride (name, t) = do
+			uuid <- Remote.nameToUUID name
+			return (uuid, t)
 
 {- Trust map parser. -}
-trustMapParse :: String -> M.Map UUID TrustLevel
-trustMapParse s = M.fromList $ map pair $ filter (not . null) $ lines s
+trustMapParse :: String -> [(UUID, TrustLevel)]
+trustMapParse s = map pair $ filter (not . null) $ lines s
 	where
 		pair l
 			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)
diff --git a/TrustLevel.hs b/TrustLevel.hs
new file mode 100644
index 0000000000..5da142ca30
--- /dev/null
+++ b/TrustLevel.hs
@@ -0,0 +1,23 @@
+{- git-annex trust levels
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module TrustLevel (
+	TrustLevel(..),
+) where
+
+data TrustLevel = SemiTrusted | UnTrusted | Trusted
+	deriving Eq
+
+instance Show TrustLevel where
+        show SemiTrusted = "?"
+        show UnTrusted = "0"
+        show Trusted = "1"
+
+instance Read TrustLevel where
+        readsPrec _ "1" = [(Trusted, "")]
+        readsPrec _ "0" = [(UnTrusted, "")]
+	readsPrec _ _ = [(SemiTrusted, "")]
diff --git a/debian/changelog b/debian/changelog
index 0d531a3206..8cb256099e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (0.20110602) UNRELEASED; urgency=low
 
   * Add --numcopies option.
+  * Add --trust, --untrust, and --semitrust options.
 
  -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
 
diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index 9b1f8231c0..e992bfb4fe 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -5,13 +5,14 @@ Here are a few I've been considering:
 ---
 
 * --numcopies would be a useful command line switch.
-  > Update: Added. Also allows for things like `git annex drop
-  > --numcopies=2` when in a repo that normally needs 3 copies, if you need
+  > Update: Added. Also allows for things like `git annex drop --numcopies=2` when in a repo that normally needs 3 copies, if you need
   > to urgently free up space.
 * A way to make `drop` and other commands temporarily trust a given remote, or possibly all remotes. 
 
 Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=repob` to remove files that have been replicated out to the other 2 repositories, which could be offline. (Slightly unsafe, but in this case the files are podcasts so not really.)
 
+> Update: done --[[Joey]] 
+
 ---
 
 [[wishlist:_git-annex_replicate]] suggests some way for git-annex to have the smarts to copy content around on its own to ensure numcopies is satisfied. I'd be satisfied with a `git annex copy --to foo --if-needed-by-numcopies`
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e4924d373d..b15ce1a296 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -359,6 +359,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Overrides the `annex.numcopies` setting, forcing git-annex to ensure the
   specified number of copies exist.
 
+* --trust=repository
+* --semitrust=repository
+* --untrust=repository
+
+  Overrides trust settings for a repository. May be specified more than once.
+  
+  The repository should be specified using the name of a configured remote.
+
 * --backend=name
 
   Specifies which key-value backend to use. This can be used when
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index 317e4b541f..7505a7af65 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -20,7 +20,9 @@ depended on to retain a copy of the file content; possibly the only
 [[copy|copies]].
 
 (Being semitrusted is the default. The `git annex semitrust` command
-restores a repository to this default, when it has been overridden.)
+restores a repository to this default, when it has been overridden.
+The `--semitrust` option can temporarily restore a repository to this
+default.)
 
 ## untrusted
 
@@ -42,7 +44,8 @@ archival drive, from which you rarely or never remove content. Deciding
 when it makes sense to trust the tracking info is up to you.
 
 One way to handle this is just to use `--force` when a command cannot
-access a remote you trust.
+access a remote you trust. Or to use `--trust` to specify a repisitory to
+trust temporarily.
 
-To configure a repository as fully trusted, use the `git annex trust`
-command.
+To configure a repository as fully and permanently trusted,
+use the `git annex trust` command.

From 80efafe4960e0fb33d1e6783bd34eaf459febea1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 1 Jun 2011 22:15:58 +0000
Subject: [PATCH 1824/8313]

---
 doc/forum/new_microfeatures.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index e992bfb4fe..683cc69b8b 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -31,6 +31,8 @@ Combined, this would allow `git annex drop --numcopies=2 --trust=repoa --trust=r
 
 Come to think of it, the inject --bare thing is probably not a microfeature. Should I add a new wishlist item for that? -- RichiH
 
+> I've thought about such things before; does not seem really micro and I'm unsure how well it would work, but it would be worth a [[todo]]. --[[Joey]]
+
 ---
 
 Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.

From 971ab27e7820a3228f71dd42f3e870c0fc2f4345 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 19:10:38 -0400
Subject: [PATCH 1825/8313] better types allowed breaking module dep loop

---
 Annex.hs        | 27 ++++++++++++++-------------
 Backend/File.hs |  5 ++---
 Command/Move.hs |  5 ++---
 GitAnnex.hs     |  7 +++++--
 Remote.hs       | 28 ++++++++++++++++++++++++++++
 RemoteUtils.hs  | 42 ------------------------------------------
 Trust.hs        |  6 +-----
 UUID.hs         |  3 +--
 UUIDType.hs     | 11 +++++++++++
 9 files changed, 64 insertions(+), 70 deletions(-)
 delete mode 100644 RemoteUtils.hs
 create mode 100644 UUIDType.hs

diff --git a/Annex.hs b/Annex.hs
index 13505de468..92a4911ea1 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -20,11 +20,12 @@ import Control.Monad.State
 	(liftIO, StateT, runStateT, evalStateT, liftM, get, put)
 
 import qualified GitRepo as Git
-import qualified GitQueue
-import qualified BackendClass
-import qualified RemoteClass
-import qualified CryptoTypes
+import GitQueue
+import BackendClass
+import RemoteClass
+import CryptoTypes
 import TrustLevel
+import UUIDType
 
 -- git-annex's monad
 type Annex = StateT AnnexState IO
@@ -32,10 +33,10 @@ type Annex = StateT AnnexState IO
 -- internal state storage
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
-	, backends :: [BackendClass.Backend Annex]
-	, supportedBackends :: [BackendClass.Backend Annex]
-	, remotes :: [RemoteClass.Remote Annex]
-	, repoqueue :: GitQueue.Queue
+	, backends :: [Backend Annex]
+	, supportedBackends :: [Backend Annex]
+	, remotes :: [Remote Annex]
+	, repoqueue :: Queue
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
@@ -45,17 +46,17 @@ data AnnexState = AnnexState
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, exclude :: [String]
-	, forcetrust :: [(String, TrustLevel)]
-	, cipher :: Maybe CryptoTypes.Cipher
+	, forcetrust :: [(UUID, TrustLevel)]
+	, cipher :: Maybe Cipher
 	}
 
-newState :: Git.Repo -> [BackendClass.Backend Annex] -> AnnexState
+newState :: Git.Repo -> [Backend Annex] -> AnnexState
 newState gitrepo allbackends = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
 	, supportedBackends = allbackends
-	, repoqueue = GitQueue.empty
+	, repoqueue = empty
 	, quiet = False
 	, force = False
 	, fast = False
@@ -70,7 +71,7 @@ newState gitrepo allbackends = AnnexState
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
-new :: Git.Repo -> [BackendClass.Backend Annex] -> IO AnnexState
+new :: Git.Repo -> [Backend Annex] -> IO AnnexState
 new gitrepo allbackends = do
 	gitrepo' <- liftIO $ Git.configRead gitrepo
 	return $ newState gitrepo' allbackends
diff --git a/Backend/File.hs b/Backend/File.hs
index 58506c861b..543f02af76 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -21,7 +21,6 @@ import Data.String.Utils
 import BackendClass
 import LocationLog
 import qualified Remote
-import qualified RemoteUtils
 import qualified GitRepo as Git
 import Content
 import qualified Annex
@@ -54,7 +53,7 @@ dummyStore _ _ = return True
  - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- RemoteUtils.keyPossibilities key
+	(remotes, _) <- Remote.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -97,7 +96,7 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- RemoteUtils.keyPossibilities key
+			(remotes, trusteduuids) <- Remote.keyPossibilities key
 			untrusteduuids <- trustGet UnTrusted
 			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
diff --git a/Command/Move.hs b/Command/Move.hs
index 6a23aee92a..f49fe20e00 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -16,7 +16,6 @@ import LocationLog
 import Types
 import Content
 import qualified Remote
-import qualified RemoteUtils
 import UUID
 import Messages
 
@@ -90,7 +89,7 @@ toPerform dest move key = do
 	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
 	isthere <- if fastcheck
 		then do
-			(remotes, _) <- RemoteUtils.keyPossibilities key
+			(remotes, _) <- Remote.keyPossibilities key
 			return $ Right $ dest `elem` remotes
 		else Remote.hasKey dest key
 	case isthere of
@@ -124,7 +123,7 @@ fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	(remotes, _) <- RemoteUtils.keyPossibilities key
+	(remotes, _) <- Remote.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then stop
 		else do
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 64b0888b05..2a9fcbe3e7 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -16,6 +16,7 @@ import Options
 import Utility
 import TrustLevel
 import qualified Annex
+import qualified Remote
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -104,10 +105,12 @@ options = commonOptions ++
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
-		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:(Annex.exclude s) }
+		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
-		settrust t v = Annex.changeState $ \s -> s { Annex.forcetrust = (v, t):(Annex.forcetrust s) }
+		settrust t v = do
+			r <- Remote.nameToUUID v
+			Annex.changeState $ \s -> s { Annex.forcetrust = (r, t):Annex.forcetrust s }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Remote.hs b/Remote.hs
index 51da5e4715..9685b4612f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -14,6 +14,7 @@ module Remote (
 	removeKey,
 	hasKey,
 	hasKeyCheap,
+	keyPossibilities,
 
 	remoteTypes,
 	genList,
@@ -45,6 +46,8 @@ import qualified Annex
 import Locations
 import Utility
 import Config
+import Trust
+import LocationLog
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -110,6 +113,31 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -
+ - Also returns a list of UUIDs that are trusted to have the key
+ - (some may not have configured remotes).
+ -}
+keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	trusted <- trustGet Trusted
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- liftIO $ keyLocations g key
+	let validuuids = filter (/= u) uuids
+
+	-- note that validuuids is assumed to not have dups
+	let validtrusteduuids = intersect validuuids trusted
+
+	-- remotes that match uuids that have the key
+	allremotes <- genList
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return (sort validremotes, validtrusteduuids)
+
+
 {- Filename of remote.log. -}
 remoteLog :: Annex FilePath
 remoteLog = do
diff --git a/RemoteUtils.hs b/RemoteUtils.hs
deleted file mode 100644
index d042780e46..0000000000
--- a/RemoteUtils.hs
+++ /dev/null
@@ -1,42 +0,0 @@
-{- git-annex remotes overflow (can't go in there due to dependency cycles)
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module RemoteUtils where
-
-import Control.Monad.State (liftIO)
-import Data.List
-
-import Annex
-import Trust
-import Remote
-import UUID
-import LocationLog
-import Key
-
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
- -
- - Also returns a list of UUIDs that are trusted to have the key
- - (some may not have configured remotes).
- -}
-keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilities key = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	trusted <- trustGet Trusted
-
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
-	let validuuids = filter (/= u) uuids
-
-	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
-
-	-- remotes that match uuids that have the key
-	allremotes <- genList
-	let validremotes = remotesWithUUID allremotes validuuids
-
-	return (sort validremotes, validtrusteduuids)
diff --git a/Trust.hs b/Trust.hs
index d6d0516abb..aaca3b3706 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -23,7 +23,6 @@ import Types
 import UUID
 import Locations
 import qualified Annex
-import qualified Remote
 import Utility
 
 {- Filename of trust.log. -}
@@ -43,14 +42,11 @@ trustGet level = do
 trustMap :: Annex (M.Map UUID TrustLevel)
 trustMap = do
 	logfile <- trustLog
-	overrides <- Annex.getState Annex.forcetrust >>= mapM findoverride
+	overrides <- Annex.getState Annex.forcetrust
 	s <- liftIO $ catch (readFile logfile) ignoreerror
 	return $ M.fromList $ trustMapParse s ++ overrides
 	where
                 ignoreerror _ = return ""
-		findoverride (name, t) = do
-			uuid <- Remote.nameToUUID name
-			return (uuid, t)
 
 {- Trust map parser. -}
 trustMapParse :: String -> [(UUID, TrustLevel)]
diff --git a/UUID.hs b/UUID.hs
index 0d7aee1414..33835e261b 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -36,8 +36,7 @@ import qualified Annex
 import Utility
 import qualified SysConfig
 import Config
-
-type UUID = String
+import UUIDType
 
 configkey :: String
 configkey = "annex.uuid"
diff --git a/UUIDType.hs b/UUIDType.hs
new file mode 100644
index 0000000000..8e207b444e
--- /dev/null
+++ b/UUIDType.hs
@@ -0,0 +1,11 @@
+{- git-annex UUID type
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module UUIDType where
+
+-- might be nice to have a newtype, but lots of stuff treats uuids as strings
+type UUID = String

From 703c437bd9c6cb9e4675b65ac2b107f76b135d71 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Jun 2011 21:56:04 -0400
Subject: [PATCH 1826/8313] rename modules for data types into Types/ directory

---
 Annex.hs                            |  8 ++++----
 Backend.hs                          |  4 ++--
 Backend/File.hs                     |  4 ++--
 Backend/SHA.hs                      |  4 ++--
 Backend/URL.hs                      |  4 ++--
 Backend/WORM.hs                     |  4 ++--
 Command.hs                          |  2 +-
 Command/DropUnused.hs               |  2 +-
 Command/FromKey.hs                  |  2 +-
 Command/InitRemote.hs               | 16 ++++++++--------
 Command/Status.hs                   | 10 +++++-----
 Content.hs                          |  2 +-
 Crypto.hs                           |  6 +++---
 Locations.hs                        |  2 +-
 Remote.hs                           |  2 +-
 Remote/Bup.hs                       |  2 +-
 Remote/Directory.hs                 |  2 +-
 Remote/Encryptable.hs               |  2 +-
 Remote/Git.hs                       |  2 +-
 Remote/Hook.hs                      |  2 +-
 Remote/Rsync.hs                     |  2 +-
 Remote/S3real.hs                    |  4 ++--
 Remote/Special.hs                   |  2 +-
 Types.hs                            |  4 ++--
 BackendClass.hs => Types/Backend.hs |  4 ++--
 CryptoTypes.hs => Types/Crypto.hs   |  2 +-
 Key.hs => Types/Key.hs              |  4 +++-
 RemoteClass.hs => Types/Remote.hs   |  6 +++---
 UUIDType.hs => Types/UUID.hs        |  2 +-
 UUID.hs                             |  2 +-
 Upgrade/V1.hs                       |  2 +-
 test.hs                             |  4 ++--
 32 files changed, 61 insertions(+), 59 deletions(-)
 rename BackendClass.hs => Types/Backend.hs (96%)
 rename CryptoTypes.hs => Types/Crypto.hs (94%)
 rename Key.hs => Types/Key.hs (95%)
 rename RemoteClass.hs => Types/Remote.hs (96%)
 rename UUIDType.hs => Types/UUID.hs (90%)

diff --git a/Annex.hs b/Annex.hs
index 92a4911ea1..06d642b742 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -21,11 +21,11 @@ import Control.Monad.State
 
 import qualified GitRepo as Git
 import GitQueue
-import BackendClass
-import RemoteClass
-import CryptoTypes
+import Types.Backend
+import Types.Remote
+import Types.Crypto
 import TrustLevel
-import UUIDType
+import Types.UUID
 
 -- git-annex's monad
 type Annex = StateT AnnexState IO
diff --git a/Backend.hs b/Backend.hs
index 645bfdfc3f..78a53d02c7 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -42,8 +42,8 @@ import Locations
 import qualified GitRepo as Git
 import qualified Annex
 import Types
-import Key
-import qualified BackendClass as B
+import Types.Key
+import qualified Types.Backend as B
 import Messages
 import Content
 import DataUnits
diff --git a/Backend/File.hs b/Backend/File.hs
index 543f02af76..bf21224a92 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -18,7 +18,7 @@ import Control.Monad.State (liftIO)
 import Data.List
 import Data.String.Utils
 
-import BackendClass
+import Types.Backend
 import LocationLog
 import qualified Remote
 import qualified GitRepo as Git
@@ -28,7 +28,7 @@ import Types
 import UUID
 import Messages
 import Trust
-import Key
+import Types.Key
 
 backend :: Backend Annex
 backend = Backend {
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 6d721038c3..94ebe093ed 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -17,15 +17,15 @@ import System.Posix.Files
 import System.FilePath
 
 import qualified Backend.File
-import BackendClass
 import Messages
 import qualified Annex
 import Locations
 import Content
 import Types
+import Types.Backend
+import Types.Key
 import Utility
 import qualified SysConfig
-import Key
 
 type SHASize = Int
 
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 3068c30270..e41004dd46 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,10 +10,10 @@ module Backend.URL (backends) where
 import Control.Monad.State (liftIO)
 
 import Types
-import BackendClass
+import Types.Backend
 import Utility
 import Messages
-import Key
+import Types.Key
 
 backends :: [Backend Annex]
 backends = [backend]
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index b33c607632..dc2e48adce 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -12,9 +12,9 @@ import System.FilePath
 import System.Posix.Files
 
 import qualified Backend.File
-import BackendClass
+import Types.Backend
 import Types
-import Key
+import Types.Key
 
 backends :: [Backend Annex]
 backends = [backend]
diff --git a/Command.hs b/Command.hs
index 0da847d24d..228c1f40e9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -24,7 +24,7 @@ import qualified Annex
 import qualified GitRepo as Git
 import Locations
 import Utility
-import Key
+import Types.Key
 
 {- A command runs in four stages.
  -
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 1bb3b7f970..0f99814471 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -22,7 +22,7 @@ import qualified Command.Move
 import qualified Remote
 import qualified GitRepo as Git
 import Backend
-import Key
+import Types.Key
 import Utility
 
 type UnusedMap = M.Map String Key
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index ca61094eb4..34816d6574 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -18,7 +18,7 @@ import Utility
 import qualified Backend
 import Content
 import Messages
-import Key
+import Types.Key
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 460f14de2c..41d3c37c71 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -16,7 +16,7 @@ import Data.String.Utils
 import Command
 import qualified Annex
 import qualified Remote
-import qualified RemoteClass
+import qualified Types.Remote as R
 import qualified GitRepo as Git
 import Utility
 import Types
@@ -54,12 +54,12 @@ start ws = notBareRepo $ do
 				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names
 			
 
-perform :: RemoteClass.RemoteType Annex -> UUID -> RemoteClass.RemoteConfig -> CommandPerform
+perform :: R.RemoteType Annex -> UUID -> R.RemoteConfig -> CommandPerform
 perform t u c = do
-	c' <- RemoteClass.setup t u c
+	c' <- R.setup t u c
 	next $ cleanup u c'
 
-cleanup :: UUID -> RemoteClass.RemoteConfig -> CommandCleanup
+cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
 	Remote.configSet u c
 	g <- Annex.gitRepo
@@ -73,7 +73,7 @@ cleanup u c = do
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
-findByName :: String -> Annex (UUID, RemoteClass.RemoteConfig)
+findByName :: String -> Annex (UUID, R.RemoteConfig)
 findByName name = do
 	m <- Remote.readRemoteLog
 	maybe generate return $ findByName' name m
@@ -82,7 +82,7 @@ findByName name = do
 			uuid <- liftIO $ genUUID
 			return (uuid, M.insert nameKey name M.empty)
 
-findByName' :: String ->  M.Map UUID RemoteClass.RemoteConfig -> Maybe (UUID, RemoteClass.RemoteConfig)
+findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
 findByName' n m = if null matches then Nothing else Just $ head matches
 	where
 		matches = filter (matching . snd) $ M.toList m
@@ -98,14 +98,14 @@ remoteNames = do
 	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
 
 {- find the specified remote type -}
-findType :: RemoteClass.RemoteConfig -> Annex (RemoteClass.RemoteType Annex)
+findType :: R.RemoteConfig -> Annex (R.RemoteType Annex)
 findType config = maybe unspecified specified $ M.lookup typeKey config
 	where
 		unspecified = error "Specify the type of remote with type="
 		specified s = case filter (findtype s) Remote.remoteTypes of
 			[] -> error $ "Unknown remote type " ++ s
 			(t:_) -> return t
-		findtype s i = RemoteClass.typename i == s
+		findtype s i = R.typename i == s
 
 {- The name of a configured remote is stored in its config using this key. -}
 nameKey :: String
diff --git a/Command/Status.hs b/Command/Status.hs
index dd518416cf..1a7f694ba8 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -14,8 +14,8 @@ import Data.List
 import qualified Data.Map as M
 
 import qualified Annex
-import qualified BackendClass
-import qualified RemoteClass
+import qualified Types.Backend as B
+import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
 import qualified GitRepo as Git
@@ -23,7 +23,7 @@ import Command
 import Types
 import DataUnits
 import Content
-import Key
+import Types.Key
 import Locations
 
 -- a named computation that produces a statistic
@@ -97,11 +97,11 @@ showStat s = calc =<< s
 supported_backends :: Stat
 supported_backends = stat "supported backends" $
 	lift (Annex.getState Annex.supportedBackends) >>=
-		return . unwords . (map BackendClass.name)
+		return . unwords . (map B.name)
 
 supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
-	return $ unwords $ map RemoteClass.typename Remote.remoteTypes
+	return $ unwords $ map R.typename Remote.remoteTypes
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
diff --git a/Content.hs b/Content.hs
index ec7a3776bf..57977ce344 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,7 @@ import qualified Annex
 import qualified AnnexQueue
 import Utility
 import StatFS
-import Key
+import Types.Key
 import DataUnits
 import Config
 
diff --git a/Crypto.hs b/Crypto.hs
index 42f1389507..e84e397f2e 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -43,11 +43,11 @@ import System.Exit
 import System.Environment
 
 import Types
-import Key
-import RemoteClass
+import Types.Key
+import Types.Remote
 import Utility
 import Base64
-import CryptoTypes
+import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
  - is used as the GPG symmetric encryption passphrase.
diff --git a/Locations.hs b/Locations.hs
index 38a320a2b2..da781ac83a 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -36,7 +36,7 @@ import Word
 import Data.Hash.MD5
 
 import Types
-import Key
+import Types.Key
 import qualified GitRepo as Git
 
 {- Conventions:
diff --git a/Remote.hs b/Remote.hs
index 9685b4612f..e7ef5f1952 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -39,8 +39,8 @@ import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
 
-import RemoteClass
 import Types
+import Types.Remote
 import UUID
 import qualified Annex
 import Locations
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c40826e5eb..c011c979ca 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -19,8 +19,8 @@ import System.FilePath
 import Data.List.Utils
 import System.Cmd.Utils
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index dedab473f3..7b5917dca8 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -16,8 +16,8 @@ import Control.Monad.State (liftIO)
 import System.Directory hiding (copyFile)
 import System.FilePath
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index 68ecfd01e6..443f5cf83d 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -11,7 +11,7 @@ import qualified Data.Map as M
 import Control.Monad.State (liftIO)
 
 import Types
-import RemoteClass
+import Types.Remote
 import Crypto
 import qualified Annex
 import Messages
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e6df6be46e..67d49df7d2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,8 +12,8 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import qualified AnnexQueue
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index dc4d392741..cc511965f8 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -18,8 +18,8 @@ import System.IO
 import System.IO.Error (try)
 import System.Exit
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9d32ad19b9..bf1bbd8707 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -16,8 +16,8 @@ import System.Directory
 import System.Posix.Files
 import System.Posix.Process
 
-import RemoteClass
 import Types
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index baf570593e..2479dfa023 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -24,8 +24,9 @@ import System.Environment
 import System.Posix.Files
 import System.Posix.Env (setEnv)
 
-import RemoteClass
 import Types
+import Types.Remote
+import Types.Key
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
@@ -35,7 +36,6 @@ import Config
 import Remote.Special
 import Remote.Encryptable
 import Crypto
-import Key
 import Content
 import Base64
 
diff --git a/Remote/Special.hs b/Remote/Special.hs
index 53ac2c6eed..7d2ea1d704 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -13,7 +13,7 @@ import Data.String.Utils
 import Control.Monad.State (liftIO)
 
 import Types
-import RemoteClass
+import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
 import UUID
diff --git a/Types.hs b/Types.hs
index 503e27d312..6353f6da68 100644
--- a/Types.hs
+++ b/Types.hs
@@ -11,6 +11,6 @@ module Types (
 	Key
 ) where
 
-import BackendClass
 import Annex
-import Key
+import Types.Backend
+import Types.Key
diff --git a/BackendClass.hs b/Types/Backend.hs
similarity index 96%
rename from BackendClass.hs
rename to Types/Backend.hs
index b2d8879c2f..8100eaf285 100644
--- a/BackendClass.hs
+++ b/Types/Backend.hs
@@ -7,9 +7,9 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module BackendClass where
+module Types.Backend where
 
-import Key
+import Types.Key
 
 data Backend a = Backend {
 	-- name of this backend
diff --git a/CryptoTypes.hs b/Types/Crypto.hs
similarity index 94%
rename from CryptoTypes.hs
rename to Types/Crypto.hs
index ba22c4cbe8..a39a016b8b 100644
--- a/CryptoTypes.hs
+++ b/Types/Crypto.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module CryptoTypes where
+module Types.Crypto where
 
 import Data.String.Utils
 
diff --git a/Key.hs b/Types/Key.hs
similarity index 95%
rename from Key.hs
rename to Types/Key.hs
index e1d8ee34d0..1d9bf8e11c 100644
--- a/Key.hs
+++ b/Types/Key.hs
@@ -1,11 +1,13 @@
 {- git-annex Key data type
+ - 
+ - Most things should not need this, using Types instead
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Key (
+module Types.Key (
 	Key(..),
 	stubKey,
 	readKey,
diff --git a/RemoteClass.hs b/Types/Remote.hs
similarity index 96%
rename from RemoteClass.hs
rename to Types/Remote.hs
index f954e4ff8f..01ced04ae1 100644
--- a/RemoteClass.hs
+++ b/Types/Remote.hs
@@ -1,4 +1,4 @@
-{- git-annex remotes class
+{- git-annex remotes types
  -
  - Most things should not need this, using Remote instead
  -
@@ -7,13 +7,13 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RemoteClass where
+module Types.Remote where
 
 import Control.Exception
 import Data.Map as M
 
 import qualified GitRepo as Git
-import Key
+import Types.Key
 
 type RemoteConfig = M.Map String String
 
diff --git a/UUIDType.hs b/Types/UUID.hs
similarity index 90%
rename from UUIDType.hs
rename to Types/UUID.hs
index 8e207b444e..eb3497fa94 100644
--- a/UUIDType.hs
+++ b/Types/UUID.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module UUIDType where
+module Types.UUID where
 
 -- might be nice to have a newtype, but lots of stuff treats uuids as strings
 type UUID = String
diff --git a/UUID.hs b/UUID.hs
index 33835e261b..f222f7a9d4 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -31,12 +31,12 @@ import Data.Maybe
 
 import qualified GitRepo as Git
 import Types
+import Types.UUID
 import Locations
 import qualified Annex
 import Utility
 import qualified SysConfig
 import Config
-import UUIDType
 
 configkey :: String
 configkey = "annex.uuid"
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 9278bce603..1e634e00e8 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -18,7 +18,7 @@ import System.Posix.Types
 import Data.Maybe
 import Data.Char
 
-import Key
+import Types.Key
 import Content
 import Types
 import Locations
diff --git a/test.hs b/test.hs
index 456c09060c..221607755a 100644
--- a/test.hs
+++ b/test.hs
@@ -31,7 +31,7 @@ import qualified Backend
 import qualified GitRepo as Git
 import qualified Locations
 import qualified Utility
-import qualified BackendClass
+import qualified Type.Backend
 import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
@@ -40,7 +40,7 @@ import qualified Trust
 import qualified Remote
 import qualified Content
 import qualified Command.DropUnused
-import qualified Key
+import qualified Type.Key
 import qualified Config
 import qualified Crypto
 

From eb4278834bccb040f32d14c1f899b55250c634d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:51:56 -0400
Subject: [PATCH 1827/8313] add fedora install instructions

---
 doc/install.mdwn        |  1 +
 doc/install/Fedora.mdwn | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 doc/install/Fedora.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 3d15eac604..c5324bddae 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -3,6 +3,7 @@
 * [[OSX]]
 * [[Debian]]
 * [[Ubuntu]]
+* [[Fedora]]
 
 ## Generic instructions
 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
new file mode 100644
index 0000000000..0050295e86
--- /dev/null
+++ b/doc/install/Fedora.mdwn
@@ -0,0 +1,21 @@
+Installation recipe for Fedora 14.
+
+
+sudo yum install ghc cabal-install
+sudo cabal update
+sudo cabal install missingh
+sudo cabal install utf8-string
+sudo cabal install pcre-light
+sudo cabal install quickcheck
+sudo cabal install SHA
+sudo cabal install dataenc
+sudo cabal install hS3
+
+git clone git://git-annex.branchable.com/
+
+cd git-annex
+sudo make   # For some reason you need to use sudo here as otherwise the cabal installed packages doesn't seem to be there...
+sudo install git-annex
+

+
+Originally posted by Jon at  --[[Joey]]

From f94a0aed833b5a0f0311a25a3401a214f38edd81 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:52:25 -0400
Subject: [PATCH 1828/8313] adjust to use primary git repo

---
 doc/install/OSX.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index a6afd408bd..e7bb763937 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -11,7 +11,7 @@ sudo cabal install dataenc
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
-git clone  git://git.kitenet.net/git-annex
+git clone  git://git-annex.branchable.com/
 
 cd git-annex
 make

From 3d7d61e679a375abefeb9bdddd789dc84243bcb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:54:25 -0400
Subject: [PATCH 1829/8313] update

---
 .../comment_2_cf0f829536744098d6846500db998b6a._comment      | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
index 134024908e..81d5a2c629 100644
--- a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
+++ b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
@@ -9,4 +9,9 @@ Because I haven't learned Cabal yet.
 But also because I've had bad experiences with both a) tying a particular program to a particular language's pet build system and then having to add ugliness when I later need to do something in the build that has nothing to do with that language and b) as a user, needing to deal with the pet build systems of languages when I just need to make some small change to the build process that is trivial in a Makefile.
 
 With that said, I do have a configure program written in Haskell, so at least it doesn't use autotools. :)
+
+Update: I did try using cabal, but git-annex includes 3 programs, and they
+all link to a lot of git-annex modules, and cabal wanted to build nearly
+every module 3 times, which was too slow for me and I could not find a way
+around.
 """]]

From c3c127a667eba4b02562203329618a1c0e8ca453 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 00:56:22 -0400
Subject: [PATCH 1830/8313] update

---
 doc/install/Ubuntu.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/Ubuntu.mdwn b/doc/install/Ubuntu.mdwn
index ffc763ff31..afcf22b6d4 100644
--- a/doc/install/Ubuntu.mdwn
+++ b/doc/install/Ubuntu.mdwn
@@ -2,4 +2,4 @@ If using Ubuntu natty or newer:
 
 	sudo apt-get install git-annex
 
-Otherwise, see [[Debian]] manual installation instructions.
+Otherwise, see [[manual_installation_instructions|install]].

From ac6510b337026b50e0610f1e334e887e5dac2a6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 01:01:14 -0400
Subject: [PATCH 1831/8313] reformat

---
 doc/install.mdwn | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index c5324bddae..8c71b56a8b 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -9,24 +9,25 @@
 
 To build and use git-annex, you will need:
 
-* `git`: 
-* The Haskell Platform: 
-* MissingH: 
-* pcre-light: 
-* utf8-string: 
-* SHA: 
-* dataenc: 
-* TestPack 
-* QuickCheck 2 
-* hS3  (optional, but recommended)
-* `uuid`: 
+* Haskell stuff
+  * [The Haskell Platform](http://haskell.org/platform/)
+  * [MissingH](http://github.com/jgoerzen/missingh/wiki
+  * [pcre-light](http://hackage.haskell.org/package/pcre-light)
+  * [utf8-string](http://hackage.haskell.org/package/utf8-string)
+  * [SHA]()
+  * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
+  * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
+  * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+* [git](http://git-scm.com/)
+* [uuid](http://www.ossp.org/pkg/lib/uuid/)
   (or `uuidgen` from util-linux)
-* `xargs`: 
-* `rsync`: 
-* `curl` :  (optional, but recommended)
-* `sha1sum`:  (optional, but recommended)
-* `gpg`:  (optional; needed for encryption)
-* [Ikiwiki](http://ikiwiki.info) is needed to build the documentation,
-  but that will be skipped if it is not installed.
+* [xargs](http://savannah.gnu.org/projects/findutils/)
+* [rsync](http://rsync.samba.org/)
+* [curl](http://http://curl.haxx.se/) (optional, but recommended)
+* [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
+  a sha1 command will also do)
+* [gpg](http://gnupg.org/) (optional; needed for encryption)
+* [Ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`

From 28cb279765b42f6a4df2026858911df40c627e77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 01:01:59 -0400
Subject: [PATCH 1832/8313] fix

---
 doc/install.mdwn | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 8c71b56a8b..bb0942b257 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -11,11 +11,11 @@ To build and use git-annex, you will need:
 
 * Haskell stuff
   * [The Haskell Platform](http://haskell.org/platform/)
-  * [MissingH](http://github.com/jgoerzen/missingh/wiki
+  * [MissingH](http://github.com/jgoerzen/missingh/wiki)
   * [pcre-light](http://hackage.haskell.org/package/pcre-light)
   * [utf8-string](http://hackage.haskell.org/package/utf8-string)
-  * [SHA]()
+  * [SHA](http://hackage.haskell.org/package/SHA)
+  * [dataenc](http://hackage.haskell.org/package/dataenc)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)

From 76be8c34acfacfc884b3fdd086bcf60b28570237 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 01:02:57 -0400
Subject: [PATCH 1833/8313] update

---
 doc/install.mdwn | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index bb0942b257..7818aaf152 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -19,15 +19,16 @@ To build and use git-annex, you will need:
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
-* [git](http://git-scm.com/)
-* [uuid](http://www.ossp.org/pkg/lib/uuid/)
-  (or `uuidgen` from util-linux)
-* [xargs](http://savannah.gnu.org/projects/findutils/)
-* [rsync](http://rsync.samba.org/)
-* [curl](http://http://curl.haxx.se/) (optional, but recommended)
-* [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
-  a sha1 command will also do)
-* [gpg](http://gnupg.org/) (optional; needed for encryption)
-* [Ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
+* Shell commands
+  * [git](http://git-scm.com/)
+  * [uuid](http://www.ossp.org/pkg/lib/uuid/)
+    (or `uuidgen` from util-linux)
+  * [xargs](http://savannah.gnu.org/projects/findutils/)
+  * [rsync](http://rsync.samba.org/)
+  * [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
+    a sha1 command will also do)
+  * [gpg](http://gnupg.org/) (optional; needed for encryption)
+  * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`

From f2cc87860ccb4ccb4a51dd9d255717a0e749fe76 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 02:33:31 -0400
Subject: [PATCH 1834/8313] refactor

---
 GitAnnex.hs | 9 +++------
 Remote.hs   | 6 ++++++
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 2a9fcbe3e7..b22313d3c5 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -95,11 +95,11 @@ options = commonOptions ++
 		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
-	, Option [] ["trust"] (ReqArg (settrust Trusted) paramRemote)
+	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
 		"override trust setting"
-	, Option [] ["semitrust"] (ReqArg (settrust SemiTrusted) paramRemote)
+	, Option [] ["semitrust"] (ReqArg (Remote.forceTrust SemiTrusted) paramRemote)
 		"override trust setting back to default value"
-	, Option [] ["untrust"] (ReqArg (settrust UnTrusted) paramRemote)
+	, Option [] ["untrust"] (ReqArg (Remote.forceTrust UnTrusted) paramRemote)
 		"override trust setting to untrusted"
 	]
 	where
@@ -108,9 +108,6 @@ options = commonOptions ++
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
-		settrust t v = do
-			r <- Remote.nameToUUID v
-			Annex.changeState $ \s -> s { Annex.forcetrust = (r, t):Annex.forcetrust s }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Remote.hs b/Remote.hs
index e7ef5f1952..2e956cb81c 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,6 +15,7 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 	keyPossibilities,
+	forceTrust,
 
 	remoteTypes,
 	genList,
@@ -137,6 +138,11 @@ keyPossibilities key = do
 
 	return (sort validremotes, validtrusteduuids)
 
+forceTrust :: TrustLevel -> String -> Annex ()
+forceTrust level remotename = do
+	r <- Remote.nameToUUID remotename
+	Annex.changeState $ \s ->
+		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
 
 {- Filename of remote.log. -}
 remoteLog :: Annex FilePath

From bcb72ce0f2623a77d18f8064738ba26661dad762 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Jun 2011 02:40:43 -0400
Subject: [PATCH 1835/8313] tweak

---
 CmdLine.hs         | 4 ++--
 GitAnnex.hs        | 4 +---
 git-annex-shell.hs | 7 +++----
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 684ebf979a..861a31be97 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -28,8 +28,8 @@ import Messages
 import UUID
 
 {- Runs the passed command line. -}
-dispatch :: Git.Repo -> [String] -> [Command] -> [Option] -> String -> IO ()
-dispatch gitrepo args cmds options header = do
+dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
+dispatch args cmds options header gitrepo = do
 	setupConsole
 	state <- Annex.new gitrepo allBackends
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
diff --git a/GitAnnex.hs b/GitAnnex.hs
index b22313d3c5..103ee262f2 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -113,6 +113,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = do
-	gitrepo <- Git.repoFromCwd
-	dispatch gitrepo args cmds options header
+run args = dispatch args cmds options header =<< Git.repoFromCwd
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 940db71c34..55f34e1027 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -58,10 +58,9 @@ builtins :: [String]
 builtins = map cmdname cmds
 
 builtin :: String -> String -> [String] -> IO ()
-builtin cmd dir params = do
-	dir' <- Git.repoAbsPath dir
-	gitrepo <- Git.repoFromAbsPath dir'
-	dispatch gitrepo (cmd:(filterparams params)) cmds commonOptions header
+builtin cmd dir params =
+	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
+		dispatch (cmd:(filterparams params)) cmds commonOptions header
 
 external :: [String] -> IO ()
 external params = do

From 269e3627a14102ab65fc26283936c56b1ac549f5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 2 Jun 2011 11:34:44 +0000
Subject: [PATCH 1836/8313] Added a comment: git annex unlock --readonly

---
 ...nt_3_a1a9347b5bc517f2a89a8b292c3f8517._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment

diff --git a/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment b/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment
new file mode 100644
index 0000000000..4bb3aa684f
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unlock --readonly"
+ date="2011-06-02T11:34:42Z"
+ content="""
+This was already asked [here](http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=606577), but I have a use case where I need to unlock with the files being hardlinked instead of copied (my fs does not support CoW), even though 'git annex lock' is now much faster ;-) . The idea is that 1) I want the external world see my repo \"as if\" it wasn't annexed (because of its own limitation to deal with soft links), and 2) I know what I do, and am sure that files won't be written to but only read.
+
+My case is: the repo contains a snapshot A1 of a certain remote directory. Later I want to rsync this dir into a new snapshot A2. Of course, I want to transfer only new or changed files, with the --copy-dest=A1 (or --compare-dest) rsync's options. Unfortunately, rsync won't recognize soft-links from git-annex, and will re-transfer everything.
+
+
+Maybe I'm overusing git-annex ;-) but still, I find it is a legitimate use case, and even though there are workarounds (I don't even remember what I had to do), it would be much more straightforward to have 'git annex unlock --readonly' (or '--readonly-unsafe'?), ... or have rsync take soft-links into account, but I did not see the author ask for microfeatures ideas :) (it was discussed, and only some convoluted workarounds were proposed). Thanks.
+
+
+"""]]

From 24f2acd27258416420e5e4a17efffc94c95534e2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 2 Jun 2011 11:55:59 +0000
Subject: [PATCH 1837/8313] Added a comment: git annex unused

---
 ...4_5a6786dc52382fff5cc42fdb05770196._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment

diff --git a/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment b/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment
new file mode 100644
index 0000000000..cc98109e6b
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unused"
+ date="2011-06-02T11:55:58Z"
+ content="""
+Before dropping unsused items, sometimes I want to check the content of the files manually.
+But currently, from e.g. a sha1 key, I don't know how to find the corresponding file, except with
+'find .git/annex/objects -type f -name 'SHA1-s1678--70....', wich is too slow (I'm in the case where \"git log --stat -S'KEY'\"
+won't work, either because it is too slow or it was never commited). By the way,
+is it documented somewhere how to determine the 2 (nested) sub-directories in which a given
+(by name) object is located?
+
+So I would like 'git-annex unused' be able to give me the list of *paths* to the unused items.
+Also, I would really appreciate a command like 'git annex unused --log NUMBER [NUMBER2...]' which would do for me the suggested command
+\"git log --stat -S'KEY'\", where NUMBER is from the 'git annex unused' output.
+Thanks.
+"""]]

From 75a966534992eda522e9a0e9c0f1213c0d4a544e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 2 Jun 2011 19:51:50 +0000
Subject: [PATCH 1838/8313] Added a comment: git annex unused

---
 .../comment_5_3c627d275586ff499d928a8f8136babf._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment

diff --git a/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment b/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment
new file mode 100644
index 0000000000..f7361f5d1c
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex unused"
+ date="2011-06-02T19:51:49Z"
+ content="""
+ps: concerning the command 'find .git/annex/objects -type f -name 'SHA1-s1678--70....' from my previous comment, it is \"significantly\" faster to search for the containing directory which have the same name: 'find .git/annex/objects -maxdepth 2 -mindepth 2 -type d -name 'SHA1-s1678--70....'. I am just curious: what is the need to have each file object in its own directory, itself nested under two more sub-directories?
+"""]]

From 0ee760fcedae58ede4684337d78ae451da23d8a8 Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Sat, 4 Jun 2011 06:18:13 +0000
Subject: [PATCH 1839/8313]

---
 doc/install/OSX.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index e7bb763937..ba10071a16 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,6 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install hS3   # stub S3 class (used if you don't have hS3 installed) has a bug, so you want to install this
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin

From cd60c364573918f37d70856442cf69a1e38654d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnEpj9IoEzNagnLDTTc9ormo5orwHW5aaI"
 
Date: Wed, 8 Jun 2011 00:22:56 +0000
Subject: [PATCH 1840/8313]

---
 doc/forum/incompatible_versions__63__.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/incompatible_versions__63__.mdwn

diff --git a/doc/forum/incompatible_versions__63__.mdwn b/doc/forum/incompatible_versions__63__.mdwn
new file mode 100644
index 0000000000..13eb181491
--- /dev/null
+++ b/doc/forum/incompatible_versions__63__.mdwn
@@ -0,0 +1 @@
+Are versions 0.14 and 0.20110522 incompatible? I can't seem to copy files from a system running 0.14 to one running 20110522.

From c9d8b830c6db43a9c3140e7680a972e39663bfd6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 8 Jun 2011 00:40:54 +0000
Subject: [PATCH 1841/8313] Added a comment

---
 .../comment_1_629f28258746d413e452cbd42a1a43f4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment

diff --git a/doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment b/doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment
new file mode 100644
index 0000000000..3702fde6ea
--- /dev/null
+++ b/doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-06-08T00:40:54Z"
+ content="""
+They are not. See [[upgrades]]
+"""]]

From 90dd245522ccffee0e77eba3b79e32d6029977fc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 9 Jun 2011 18:54:49 -0400
Subject: [PATCH 1842/8313] get --from is the same as copy --from

get not honoring --from has surprised me a few times, so least surprise
suggests it should just behave like copy --from. This leaves the difference
between get and copy being that copy always requires the remote to copy
from, while get will decide whether to get a file from a key/value store or
a remote.
---
 Backend/File.hs                                       |  2 +-
 Command/Get.hs                                        | 11 +++++++++--
 debian/changelog                                      |  1 +
 doc/bugs/fsck_output.mdwn                             |  2 +-
 doc/git-annex.mdwn                                    |  2 +-
 doc/walkthrough/getting_file_content.mdwn             |  4 ++--
 .../transferring_files:_When_things_go_wrong.mdwn     |  2 +-
 doc/walkthrough/using_ssh_remotes.mdwn                |  2 +-
 8 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index bf21224a92..386af02663 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -81,7 +81,7 @@ copyKeyFile key file = do
 						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "copying from " ++ Remote.name r ++ "..."
+			showNote $ "from " ++ Remote.name r ++ "..."
 			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
diff --git a/Command/Get.hs b/Command/Get.hs
index 90c0540960..50dc009feb 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -9,9 +9,12 @@ module Command.Get where
 
 import Command
 import qualified Backend
+import qualified Annex
+import qualified Remote
 import Types
 import Content
 import Messages
+import qualified Command.Move
 
 command :: [Command]
 command = [repoCommand "get" paramPath seek
@@ -20,7 +23,6 @@ command = [repoCommand "get" paramPath seek
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
-{- Gets an annexed file from one of the backends. -}
 start :: CommandStartString
 start file = isAnnexed file $ \(key, backend) -> do
 	inannex <- inAnnex key
@@ -28,7 +30,12 @@ start file = isAnnexed file $ \(key, backend) -> do
 		then stop
 		else do
 			showStart "get" file
-			next $ perform key backend
+			from <- Annex.getState Annex.fromremote
+			case from of
+				Nothing -> next $ perform key backend
+				Just name -> do
+					src <- Remote.byName name
+					next $ Command.Move.fromPerform src False key
 
 perform :: Key -> Backend Annex -> CommandPerform
 perform key backend = do
diff --git a/debian/changelog b/debian/changelog
index 8cb256099e..4fe1febd31 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (0.20110602) UNRELEASED; urgency=low
 
   * Add --numcopies option.
   * Add --trust, --untrust, and --semitrust options.
+  * get --from is the same as copy --from
 
  -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
 
diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 3ded1b409e..90af1600d8 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -30,7 +30,7 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > A related problem occurs if an error message is unexpetedly printed.
 > Dummying up an example:
 > 
-> O get test1 (copying from foo...) E git-annex: failed to run ssh
+> O get test1 (from foo...) E git-annex: failed to run ssh
 > failed
 > 
 > --[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index b15ce1a296..25f053af69 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -41,7 +41,7 @@ content from the key-value store.
 	# sudo mount /media/usb
 	# git remote add usbdrive /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
-	get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok
+	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
 	# git commit -a -m "got a video I want to rewatch on the plane"
 	
 	# git annex add iso
diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index 5c899ee3c5..bf45fd97fd 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -8,8 +8,8 @@ USB drive.
 	# cd /media/usb/annex
 	# git pull laptop master
 	# git annex get .
-	get my_cool_big_file (copying from laptop...) ok
-	get iso/debian.iso (copying from laptop...) ok
+	get my_cool_big_file (from laptop...) ok
+	get iso/debian.iso (from laptop...) ok
 
 Notice that you had to git pull from laptop first, this lets git-annex know
 what has changed in laptop, and so it knows about the files present there and
diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
index d8f0a19bd6..936d088f1f 100644
--- a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -14,5 +14,5 @@ it:
 	failed
 	# sudo mount /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
-	get video/hackity_hack_and_kaxxt.mov (copying from usbdrive...) ok
+	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
 	# git commit -a -m "got a video I want to rewatch on the plane"
diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn
index 4c2f830de8..fbbbbe0701 100644
--- a/doc/walkthrough/using_ssh_remotes.mdwn
+++ b/doc/walkthrough/using_ssh_remotes.mdwn
@@ -12,7 +12,7 @@ to clone the laptop's annex to it:
 Now you can get files and they will be transferred (using `rsync` via `ssh`):
 
 	# git annex get my_cool_big_file
-	get my_cool_big_file (getting UUID for origin...) (copying from origin...)
+	get my_cool_big_file (getting UUID for origin...) (from origin...)
 	WORM-s2159-m1285650548--my_cool_big_file       100% 2159     2.1KB/s   00:00
 	ok
 

From 9a272815ddd33272008a4052ea74e7bbc4c842f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 11:43:28 -0400
Subject: [PATCH 1843/8313] Bugfix: Fix fsck to not think all SHAnE keys are
 bad.

---
 Backend/SHA.hs   | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 94ebe093ed..8ed00b7073 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -119,7 +119,7 @@ checkKeyChecksum size key = do
 		then return True
 		else do
 			s <- shaN size file
-			if s == keyName key
+			if s == dropExtension (keyName key)
 				then return True
 				else do
 					dest <- moveBad key
diff --git a/debian/changelog b/debian/changelog
index 4fe1febd31..25c00963c2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (0.20110602) UNRELEASED; urgency=low
   * Add --numcopies option.
   * Add --trust, --untrust, and --semitrust options.
   * get --from is the same as copy --from
+  * Bugfix: Fix fsck to not think all SHAnE keys are bad.
 
  -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
 

From 38e0100a6929b4538bbcd5ec82827b4bc996915d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 11:58:21 -0400
Subject: [PATCH 1844/8313] releasing version 0.20110610

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 25c00963c2..09655df65c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (0.20110602) UNRELEASED; urgency=low
+git-annex (0.20110610) unstable; urgency=low
 
   * Add --numcopies option.
   * Add --trust, --untrust, and --semitrust options.
   * get --from is the same as copy --from
   * Bugfix: Fix fsck to not think all SHAnE keys are bad.
 
- -- Joey Hess   Wed, 01 Jun 2011 16:26:48 -0400
+ -- Joey Hess   Fri, 10 Jun 2011 11:48:40 -0400
 
 git-annex (0.20110601) unstable; urgency=low
 

From a4735605159e2f447d71f50aa10ca5094de8b8f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 11:58:36 -0400
Subject: [PATCH 1845/8313] add news item for git-annex 0.20110610

---
 doc/news/version_0.20110503.mdwn | 9 ---------
 doc/news/version_0.20110610.mdwn | 6 ++++++
 2 files changed, 6 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_0.20110503.mdwn
 create mode 100644 doc/news/version_0.20110610.mdwn

diff --git a/doc/news/version_0.20110503.mdwn b/doc/news/version_0.20110503.mdwn
deleted file mode 100644
index f24f0c48f6..0000000000
--- a/doc/news/version_0.20110503.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 0.20110503 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix hasKeyCheap setting for bup and rsync special remotes.
-   * Add hook special remotes.
-   * Avoid crashing when an existing key is readded to the annex.
-   * unused: Now also lists files fsck places in .git/annex/bad/
-   * S3: When encryption is enabled, the Amazon S3 login credentials
-     are stored, encrypted, in .git-annex/remotes.log, so environment
-     variables need not be set after the remote is initialized."""]]
\ No newline at end of file
diff --git a/doc/news/version_0.20110610.mdwn b/doc/news/version_0.20110610.mdwn
new file mode 100644
index 0000000000..9ab9e09076
--- /dev/null
+++ b/doc/news/version_0.20110610.mdwn
@@ -0,0 +1,6 @@
+git-annex 0.20110610 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Add --numcopies option.
+   * Add --trust, --untrust, and --semitrust options.
+   * get --from is the same as copy --from
+   * Bugfix: Fix fsck to not think all SHAnE keys are bad."""]]
\ No newline at end of file

From 19428ea2f4cd1ef9f1c91b5670933bdf989a2af3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 12:06:40 -0400
Subject: [PATCH 1846/8313] fix building with S3 stub

---
 Remote/S3stub.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
index 0d6ec47de3..d91a222e86 100644
--- a/Remote/S3stub.hs
+++ b/Remote/S3stub.hs
@@ -1,7 +1,7 @@
 -- stub for when hS3 is not available
 module Remote.S3 (remote) where
 
-import RemoteClass
+import Types.Remote
 import Types
 
 remote :: RemoteType Annex

From 0b85ffd6e7e875ee4ac671ca8d145586ef77ab38 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Jun 2011 12:11:55 -0400
Subject: [PATCH 1847/8313] S3 stub bug got fixed

---
 doc/install/OSX.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index ba10071a16..81ffe1d035 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,7 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
-sudo cabal install hS3   # stub S3 class (used if you don't have hS3 installed) has a bug, so you want to install this
+sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin

From ef6209db43d2dcea8a74805d5425988b0f7bf4ab Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 16:39:45 +0000
Subject: [PATCH 1848/8313] Comment moderation

---
 ..._90a8a15bedd94480945a374f9d706b86._comment | 10 ++++++++++
 ..._7e328b970169fffb8bce373d1522743b._comment | 19 +++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment

diff --git a/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
new file mode 100644
index 0000000000..722cbdd9e7
--- /dev/null
+++ b/doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://ethan.betacantrips.com/"
+ nickname="ethan.glasser.camp"
+ subject="no symlinks"
+ date="2011-06-08T20:59:38Z"
+ content="""
+If you try to clone a git repo that has a symlink over to a VFAT filesystem, you get (in its place) a regular file that contains the name of the symlink target.  So why can't git-annex use that?  I could still do git annex get on this file, git annex would still \"know\" that it's a symlink, and could replace it with a copy of the real file (instead of putting it in .git/annex).
+
+I know if it were that simple, someone would have done it already, so what am I missing?  I guess trying to get the file FROM the repository would fail because it wouldn't find the file in .git/annex?  Couldn't you store a reverse mapping?  You wouldn't be able to move the file around, but you already lose that once you give up symlinks.  It would also be a little harder to tell which symlinks were \"dangling\"; I don't see an easy way to get around that.  It would still be better than a bare repo..
+"""]]
diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment
new file mode 100644
index 0000000000..8f0f5ef180
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="ssqq"
+ ip="208.70.196.4"
+ subject="Still a problem on 0.20110523"
+ date="2011-06-02T20:31:55Z"
+ content="""
+Hi,
+
+(I'm new to git and git annex, so please forgive any mistakes I make...)
+
+My repo is messed up right now.  The fact that I copied the repo with rsync -a back and forth from a case insensitive filesystem to a case sensitive one, probably didn't help.
+
+I believe the annexed files in .git/annex/objects/ are still using a mixed case directory hashing scheme.  That's the problem I'm having.  The symlinks point to the wrong case and are now broken.  I don't think the latest versions of git-annex changed that (it only changed the hashing under .git-annex, right?).
+
+Even if I clean up my repo, I think I'm still going to have a problem because I have one repo on an OS X case insensitive filesystem and my other repos on case sensitive Linux filesystems.  Potentially the directory name under .git/annex/objects will have a different case.  Then the symlink might have a different case than my Linux FS.  Does git-annex track changes in git by the contents of the symlink?  In which case the case difference would show up as a change even though there is no change?
+
+Is it possible to change the directory hashing scheme under .git/annex/objects to use lowercase names?
+
+"""]]

From 68ed12eab4e0f73008a2af152b534ff08d92a57b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 16:41:43 +0000
Subject: [PATCH 1849/8313] Added a comment

---
 .../comment_5_64bbf89de0836673224b83fdefa0407b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment

diff --git a/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment
new file mode 100644
index 0000000000..1063b0f910
--- /dev/null
+++ b/doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-06-10T16:41:43Z"
+ content="""
+@ethan the reason that wouldn't work is because git would then see a file that was checked in and had its one line symlinkish content replaced with a huge binary blob. And git commit would try to commit that etc. The potential for foot-shooting is too high.
+"""]]

From 8da84d8860e9530ad223fc7b81811f7cc97db121 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 16:46:03 +0000
Subject: [PATCH 1850/8313] Added a comment

---
 ...omment_21_98f632652b0db9131b0173d3572f4d62._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment

diff --git a/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment
new file mode 100644
index 0000000000..453a8be11b
--- /dev/null
+++ b/doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 21"
+ date="2011-06-10T16:46:03Z"
+ content="""
+@seqq git-annex always uses the same case when creating and accessing the files pointed to by the symlinks. So it will not matter if it's used on a case-insensative, or case-insensative but preserving system like OSX.
+
+You need to fix up the cases of the files in .git/annex/objects to what it expects. I'm not sure what would be the best way to do that. The method described in [[walkthrough/recover_data_from_lost+found]] might work well.
+"""]]

From 88bdf17e1a26f65cb760d7bf208b07ecc4651639 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmByD9tmR48HuYgS4qWEGDDaoVTTC3m4kc"
 
Date: Fri, 10 Jun 2011 18:08:37 +0000
Subject: [PATCH 1851/8313] Added a comment: Any chance to get git-annex going
 on windows?

---
 ...comment_3_cff163ea3e7cad926f4ed9e78b896598._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment

diff --git a/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment b/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
new file mode 100644
index 0000000000..6b47ed0e3e
--- /dev/null
+++ b/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmByD9tmR48HuYgS4qWEGDDaoVTTC3m4kc"
+ nickname="Jonas"
+ subject="Any chance to get git-annex going on windows?"
+ date="2011-06-10T18:08:36Z"
+ content="""
+Would be great! :-)
+
+Jonas
+"""]]

From 76df8d5f86fa7c978ed6a695a5c2817d835a3307 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 10 Jun 2011 19:55:38 +0000
Subject: [PATCH 1852/8313] Added a comment: short answer: no

---
 ..._82a17eee4a076c6c79fddeda347e0c9a._comment | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment

diff --git a/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment b/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment
new file mode 100644
index 0000000000..678847ecae
--- /dev/null
+++ b/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment
@@ -0,0 +1,69 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="short answer: no"
+ date="2011-06-10T19:55:38Z"
+ content="""
+Long answer, quoting from a mail to someone else:
+
+Well, I can tell you that it assumes a POSIX system, both in available
+utilities and system calls, So you'd need to use cygwin or something
+like that. (Perhaps you already are for git, I think git also assumes a
+POSIX system.) So you need a Haskell that can target that. What this
+page refers to as \"GHC-Cygwin\":
+
+I don't know where to get one. Did find this:
+
+
+(There are probably also still some places where it assumes / as a path
+separator, although I fixed some.)
+
+FWIW, git-annex works fine on OS X and other fine proprietary unixen. ;P
+
+----
+
+Alternatively, windows versions of these functions could be found,
+which are all the ones that need POSIX, I think. A fair amount of this,
+the stuff to do with signals and users, could be empty stubs in windows.
+The file manipulation, particularly symlinks, would probably be the main
+challenge.
+
+
+addSignal
+blockSignals
+changeWorkingDirectory
+createLink
+createSymbolicLink
+emptySignalSet
+executeFile
+fileMode
+fileSize
+forkProcess
+getAnyProcessStatus
+getEffectiveUserID
+getEnvDefault
+getFileStatus
+getProcessID
+getProcessStatus
+getSignalMask
+getSymbolicLinkStatus
+getUserEntryForID
+getUserEntryForName
+groupWriteMode
+homeDirectory
+installHandler
+intersectFileModes
+isRegularFile
+isSymbolicLink
+modificationTime
+otherWriteMode
+ownerWriteMode
+readSymbolicLink
+setEnv
+setFileMode
+setSignalMask
+sigCHLD
+sigINT
+unionFileModes
+

+"""]]

From 57ff2e297a0a9cc610a3fd5e1b8c635fc44270b5 Mon Sep 17 00:00:00 2001
From: "https://lithitux.org/openidserver/users/pavel" 
Date: Mon, 13 Jun 2011 10:13:47 +0000
Subject: [PATCH 1853/8313]

---
 ...selecting_files_based_on_meta-information.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn

diff --git a/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn
new file mode 100644
index 0000000000..1de06f7cda
--- /dev/null
+++ b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn
@@ -0,0 +1,15 @@
+It would be extremely useful to have some additional ways to select files (for git annex copy/move/get and maybe others) based on the meta-information available to git-annex, rather than just by file or directory name.
+
+An example of what I'd like to do is this:
+
+    host1$ git annex copy --to usb-drive --missing-on host2
+
+This would check location tracking information and copy each file from host1's annex which is not present on host2 onto the usb-drive annex -- i.e. it's what I want when I need to do a sneakernet synchronisation of host1 and host2 (for backup purposes, for example). Note that of course I could copy --to host2, assuming network connectivity, but that would take a long time.
+
+There's probably other selectors that we can imagine; an obvious one could be --present-on  -- useful for judiciously dropping only those files that you have easily available in a local annex (as you may want to keep files that are hard to make available even if --numcopies would nominally be satisfied).
+
+Other similar ideas for file content selectors:
+
+ * Files that have less than n, exactly n or more than n copies -- for when you need to satisfy your --numcopies policy over sneakernet.
+ * Files that are present (or not present) on some trusted annex -- for making sure you have trusted copies of everything.
+ * Boolean combinations of these filters -- "git annex drop --present-on lanserver1 --or --present-on lanserver2" or similar syntax, although obviously doing this in full generality may be quite fiddly.

From a85e9b1276cd2b5bdcd9fee0828c3e6432580807 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 13 Jun 2011 16:57:19 +0000
Subject: [PATCH 1854/8313]

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
new file mode 100644
index 0000000000..dc36b709ea
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -0,0 +1,5 @@
+What is says on the tin:
+
+git annex fsck is a no-op in bare repos
+
+See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html

From eb8ecd857d4a5796455b9a608753042241eb6165 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 13 Jun 2011 16:58:53 +0000
Subject: [PATCH 1855/8313] Added a comment

---
 .../comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment
new file mode 100644
index 0000000000..d50938a784
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-06-13T16:58:52Z"
+ content="""
+And, maybe, a way to start a fsck from remote? At least when the other side is a ssh or git annex shell, this would work.
+"""]]

From eb5fbc4cb97b73ad5d5d194600cc52c99edc89a0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 13:03:23 -0400
Subject: [PATCH 1856/8313] response

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index dc36b709ea..b6ff6bba11 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -3,3 +3,10 @@ What is says on the tin:
 git annex fsck is a no-op in bare repos
 
 See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
+
+> Thinking about this some more, it would be difficult to do anything
+> when bad content is found, since it also cannot update the location log.
+> 
+> So this may be another thing blocked by [[todo/branching]], assuming
+> that is fixed in a way that makes `.git-annex` available to bare repos.
+> --[[Joey]] 

From 056a784f3d747809c6f256603ff0f8cfb9f6c042 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Mon, 13 Jun 2011 18:09:58 +0000
Subject: [PATCH 1857/8313]

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index b6ff6bba11..249a175055 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -10,3 +10,6 @@ See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
 > So this may be another thing blocked by [[todo/branching]], assuming
 > that is fixed in a way that makes `.git-annex` available to bare repos.
 > --[[Joey]] 
+
+>> Even if there is nothing it can _do_, knowing that the data is intact,
+>> or not, is valuable in and as of itself. -- RichiH

From 30d7cce7ecb7dbcbabe83ab3f6cd5f32ce685992 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 20:23:47 -0400
Subject: [PATCH 1858/8313] rsync is now used when copying files from repos on
 other filesystems

cp is still used when copying file from repos on the same filesystem, since
--reflink=auto can make it significantly faster on filesystems such as
btrfs.

Directory special remotes still use cp, not rsync. It's not clear what
tmp file should be used when rsyncing to such a remote.
---
 Remote/Git.hs                        | 49 +++++++++++++++++++---------
 debian/changelog                     |  9 +++++
 doc/todo/file_copy_progress_bar.mdwn |  2 ++
 3 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 67d49df7d2..0d8e2425a3 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -11,6 +11,7 @@ import Control.Exception.Extensible
 import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
+import System.Posix.Files
 
 import Types
 import Types.Remote
@@ -130,10 +131,10 @@ dropKey r key =
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
-	| not $ Git.repoIsUrl r = liftIO $ copyFile (gitAnnexLocation r key) file
-	| Git.repoIsSsh r = rsynchelper r True key file
+	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
+	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| otherwise = error "copying from non-ssh repo not supported"
-
+		
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
@@ -145,19 +146,18 @@ copyToRemote r key
 			a <- Annex.new r []
 			Annex.eval a $ do
 				ok <- Content.getViaTmp key $
-					\f -> liftIO $ copyFile keysrc f
+					rsyncOrCopyFile r keysrc
 				AnnexQueue.flush True
 				return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
-		rsynchelper r False key keysrc
+		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
-rsynchelper :: Git.Repo -> Bool -> Key -> FilePath -> Annex (Bool)
-rsynchelper r sending key file = do
+rsyncHelper :: [CommandParam] -> Annex (Bool)
+rsyncHelper p = do
 	showProgress -- make way for progress bar
-	p <- rsyncParams r sending key file
 	res <- liftIO $ rsync p
 	if res
 		then return res
@@ -165,10 +165,22 @@ rsynchelper r sending key file = do
 			showLongNote "rsync failed -- run git annex again to resume file transfer"
 			return res
 
+{- Copys a file with rsync unless both locations are on the same
+ - filesystem. Then cp could be faster. -}
+rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool
+rsyncOrCopyFile r src dest = do
+	ss <- liftIO $ getFileStatus src
+	ds <- liftIO $ getFileStatus dest
+	if deviceID ss == deviceID ds
+		then liftIO $ copyFile src dest
+		else do
+			params <- rsyncParams r
+			rsyncHelper $ params ++ [Param src, Param dest]
+
 {- Generates rsync parameters that ssh to the remote and asks it
  - to either receive or send the key's content. -}
-rsyncParams :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
-rsyncParams r sending key file = do
+rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam]
+rsyncParamsRemote r sending key file = do
 	Just (shellcmd, shellparams) <- git_annex_shell r
 		(if sending then "sendkey" else "recvkey")
 		[ Param $ show key
@@ -179,15 +191,20 @@ rsyncParams r sending key file = do
 		]
 	-- Convert the ssh command into rsync command line.
 	let eparam = rsyncShell (Param shellcmd:shellparams)
-	o <- getConfig r "rsync-options" ""
-	let base = options ++ map Param (words o) ++ eparam
+	o <- rsyncParams r
 	if sending
-		then return $ base ++ [dummy, File file]
-		else return $ base ++ [File file, dummy]
+		then return $ o ++ eparam ++ [dummy, File file]
+		else return $ o ++ eparam ++ [File file, dummy]
 	where
-		-- inplace makes rsync resume partial files
-		options = [Params "-p --progress --inplace"]
 		-- the rsync shell parameter controls where rsync
 		-- goes, so the source/dest parameter can be a dummy value,
 		-- that just enables remote rsync mode.
 		dummy = Param ":"
+
+rsyncParams :: Git.Repo -> Annex [CommandParam]
+rsyncParams r = do
+	o <- getConfig r "rsync-options" ""
+	return $ options ++ map Param (words o)
+	where
+ 		-- --inplace to resume partial files
+		options = [Params "-p --progress --inplace"]
diff --git a/debian/changelog b/debian/changelog
index 09655df65c..0837a9f345 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (0.20110611) UNRELEASED; urgency=low
+
+  * rsync is now used when copying files from repos on other filesystems.
+    cp is still used when copying file from repos on the same filesystem,
+    since --reflink=auto can make it significantly faster on filesystems
+    such as btrfs.
+
+ -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
+
 git-annex (0.20110610) unstable; urgency=low
 
   * Add --numcopies option.
diff --git a/doc/todo/file_copy_progress_bar.mdwn b/doc/todo/file_copy_progress_bar.mdwn
index cd4ea33b78..847c1d1eb6 100644
--- a/doc/todo/file_copy_progress_bar.mdwn
+++ b/doc/todo/file_copy_progress_bar.mdwn
@@ -1,3 +1,5 @@
 Find a way to copy a file with a progress bar, while still preserving
 stat. Easiest way might be to use pv and fix up the permissions etc
 after?
+
+[[done]]

From d0482d4154e90e7e5c3d70113a760eb76e5d4e83 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 21:46:28 -0400
Subject: [PATCH 1859/8313] bigfix: stat parent dirs

---
 Remote/Git.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 0d8e2425a3..5b9d5d3dfe 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -169,8 +169,8 @@ rsyncHelper p = do
  - filesystem. Then cp could be faster. -}
 rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool
 rsyncOrCopyFile r src dest = do
-	ss <- liftIO $ getFileStatus src
-	ds <- liftIO $ getFileStatus dest
+	ss <- liftIO $ getFileStatus $ parentDir src
+	ds <- liftIO $ getFileStatus $ parentDir dest
 	if deviceID ss == deviceID ds
 		then liftIO $ copyFile src dest
 		else do

From 66f5b390fe761946280c8999648d13e917fba45b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 21:51:52 -0400
Subject: [PATCH 1860/8313] fix test suite

---
 test.hs | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/test.hs b/test.hs
index 221607755a..498c7b6806 100644
--- a/test.hs
+++ b/test.hs
@@ -31,7 +31,7 @@ import qualified Backend
 import qualified GitRepo as Git
 import qualified Locations
 import qualified Utility
-import qualified Type.Backend
+import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
@@ -40,20 +40,20 @@ import qualified Trust
 import qualified Remote
 import qualified Content
 import qualified Command.DropUnused
-import qualified Type.Key
+import qualified Types.Key
 import qualified Config
 import qualified Crypto
 
 -- for quickcheck
-instance Arbitrary Key.Key where
+instance Arbitrary Types.Key.Key where
 	arbitrary = do
 		n <- arbitrary
 		b <- elements ['A'..'Z']
-		return $ Key.Key {
-			Key.keyName = n,
-			Key.keyBackendName = [b],
-			Key.keySize = Nothing,
-			Key.keyMtime = Nothing
+		return $ Types.Key.Key {
+			Types.Key.keyName = n,
+			Types.Key.keyBackendName = [b],
+			Types.Key.keySize = Nothing,
+			Types.Key.keyMtime = Nothing
 		}
 
 main :: IO ()
@@ -72,7 +72,7 @@ quickcheck :: Test
 quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
-	, qctest "prop_idempotent_key_read_show" Key.prop_idempotent_key_read_show
+	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
 	, qctest "prop_idempotent_configEscape" Remote.prop_idempotent_configEscape
@@ -139,7 +139,7 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup]
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	writeFile tmp $ content sha1annexedfile
-	r <- annexeval $ BackendClass.getKey backendSHA1 tmp
+	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
 	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"

From f547277b751d81f516a638bc281735fa1946b17d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Jun 2011 22:19:44 -0400
Subject: [PATCH 1861/8313] Allow --trust etc to specify a repository by name,
 for temporarily  trusting repositories that are not configured remotes.

---
 Remote.hs          | 29 ++++++++++++++++++++++-------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  3 ++-
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 2e956cb81c..d975c2404f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -34,7 +34,7 @@ module Remote (
 ) where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when, liftM, filterM)
+import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
@@ -91,20 +91,35 @@ genList = do
 
 {- Looks up a remote by name. (Or by UUID.) -}
 byName :: String -> Annex (Remote Annex)
-byName "" = error "no remote specified"
 byName n = do
+	res <- byName' n
+	case res of
+		Left e -> error e
+		Right r -> return r
+byName' :: String -> Annex (Either String (Remote Annex))
+byName' "" = return $ Left "no remote specified"
+byName' n = do
 	allremotes <- genList
 	let match = filter matching allremotes
-	when (null match) $ error $
-		"there is no git remote named \"" ++ n ++ "\""
-	return $ head match
+	if (null match)
+		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
+		else return $ Right $ head match
 	where
 		matching r = n == name r || n == uuid r
 
-{- Looks up a remote by name (or by UUID), and returns its UUID. -}
+{- Looks up a remote by name (or by UUID, or even by description),
+ - and returns its UUID. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
-nameToUUID n = liftM uuid (byName n)
+nameToUUID n = do
+	res <- byName' n
+	case res of
+		Left e -> return . (maybe (error e) id) =<< byDescription
+		Right r -> return $ uuid r
+	where
+		byDescription = return . M.lookup n . invertMap =<< uuidMap
+		invertMap = M.fromList . map swap . M.toList
+		swap (a, b) = (b, a)
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
diff --git a/debian/changelog b/debian/changelog
index 0837a9f345..de012de5bd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (0.20110611) UNRELEASED; urgency=low
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems
     such as btrfs.
+  * Allow --trust etc to specify a repository by name, for temporarily 
+    trusting repositories that are not configured remotes.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 25f053af69..12756d8020 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -365,7 +365,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Overrides trust settings for a repository. May be specified more than once.
   
-  The repository should be specified using the name of a configured remote.
+  The repository should be specified using the name of a configured remote,
+  or the UUID or description of a repository.
 
 * --backend=name
 

From f4609a5d288525abbb47e361194601d21684df8e Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Tue, 14 Jun 2011 17:01:13 +0000
Subject: [PATCH 1862/8313] minor bug report

---
 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn

diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
new file mode 100644
index 0000000000..6e3a60e3bf
--- /dev/null
+++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
@@ -0,0 +1,10 @@
+a [[!taglink minor]]  issue: `git annex initremote` (in particular, adding
+a key as described in [[encryption]] -- `git annex initremote my_remote
+encryption=my_key`) seems to iterate over the `.git-annex/???/???/*.log` files
+with lstat (tested using strace).
+
+in a 50k key git-annex on a slow disk, this takes quite a while, while not
+seeming necessary (it's just re-encrypting the shared secret, is it?).
+
+could you verify the observed behavior?

From 64d6520704fde80ee1584cacd4a16a54d88970fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Jun 2011 19:05:45 -0400
Subject: [PATCH 1863/8313] git is slow

---
 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
index 6e3a60e3bf..2457057c81 100644
--- a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
+++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
@@ -8,3 +8,12 @@ in a 50k key git-annex on a slow disk, this takes quite a while, while not
 seeming necessary (it's just re-encrypting the shared secret, is it?).
 
 could you verify the observed behavior?
+
+> This is due to `git commit` being called. `git commit` exposes git's 
+> rather innefficient handling of the index; in order to make a commit
+> it has to write a new index file, and it does this by scanning every
+> file in the repository. I think that git generally needs its index
+> file handleing overhauled, particularly to deal with repositories with
+> large numbers of files. git-annex is seems to already be running
+> `git commit` in its most efficient mode, by specifying exactly what file
+> to commit. [[done]] --[[Joey]]

From 601b07196ee122b96c984133637b4e2c3debc4b6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 20 Jun 2011 17:43:05 +0000
Subject: [PATCH 1864/8313] Comment moderation

---
 .../comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment

diff --git a/doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment b/doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment
new file mode 100644
index 0000000000..f006f88a0a
--- /dev/null
+++ b/doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="status of other remotes?"
+ date="2011-06-15T08:39:24Z"
+ content="""
+using the location tracking information, it should be possible to show the status of other remotes as well. what about supporting `--from=...` or `--all`? (thus, among other things, one could determine if a remote has a complete checkout.)
+"""]]

From dd4de9deb464adb6631585abd3c5a7e1401c8c6f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Mon, 20 Jun 2011 21:03:00 +0000
Subject: [PATCH 1865/8313]

---
 doc/forum/git_annex_unlock_is_not_atomic.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/git_annex_unlock_is_not_atomic.mdwn

diff --git a/doc/forum/git_annex_unlock_is_not_atomic.mdwn b/doc/forum/git_annex_unlock_is_not_atomic.mdwn
new file mode 100644
index 0000000000..a7751f6b7d
--- /dev/null
+++ b/doc/forum/git_annex_unlock_is_not_atomic.mdwn
@@ -0,0 +1,5 @@
+Running a command like
+
+git annex unlock myfile
+
+is not atomic, that is if the execution is aborted you may end up with an incomplete version of myfile in the directory. If you don't notice this you may lock it again and then propagate this bad version of the file to your other repositories. A simple workaround is to simply name it something else while unlocking and then rename it to the correct filename once it's completely copied. I don't know Haskel yet so I can not fix this issue otherwise I would sure try. A part from this, I love git annex.

From bc731ba6e0b94c7d5d4be5ba3c5c13d6644183f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Jun 2011 18:27:01 -0400
Subject: [PATCH 1866/8313] pointless golfing

---
 Annex.hs | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 06d642b742..0d6b309cca 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,7 +17,6 @@ module Annex (
 ) where
 
 import Control.Monad.State
-	(liftIO, StateT, runStateT, evalStateT, liftM, get, put)
 
 import qualified GitRepo as Git
 import GitQueue
@@ -50,8 +49,8 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-newState :: Git.Repo -> [Backend Annex] -> AnnexState
-newState gitrepo allbackends = AnnexState
+newState :: [Backend Annex] -> Git.Repo -> AnnexState
+newState allbackends gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
@@ -72,27 +71,26 @@ newState gitrepo allbackends = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> [Backend Annex] -> IO AnnexState
-new gitrepo allbackends = do
-	gitrepo' <- liftIO $ Git.configRead gitrepo
-	return $ newState gitrepo' allbackends
+new gitrepo allbackends =
+	newState allbackends `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run state action = runStateT action state
+run = flip runStateT
 eval :: AnnexState -> Annex a -> IO a
-eval state action = evalStateT action state
+eval = flip evalStateT
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}
 getState :: (AnnexState -> a) -> Annex a
-getState c = liftM c get
+getState = gets
 
 {- Applies a state mutation function to change the internal state. 
  -
- - Example: changeState (\s -> s { quiet = True })
+ - Example: changeState $ \s -> s { quiet = True }
  -}
 changeState :: (AnnexState -> AnnexState) -> Annex ()
-changeState a = put . a =<< get
+changeState = modify
 
 {- Returns the git repository being acted on -}
 gitRepo :: Annex Git.Repo

From e593a81041f3c55e8b2803d4cc07adc9b206b46f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 13:19:08 -0400
Subject: [PATCH 1867/8313] update

---
 doc/todo/branching.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 79f278480e..a035e53e80 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -47,6 +47,8 @@ Let's use one branch per uuid, named git-annex/$UUID.
   branches?
 - A given repo only ever writes to its UUID branch. So no conflicts.
   - **problem**: git annex move needs to update log info for other repos!
+    (possibly solvable by having git-annex-shell update the log info
+    when content is moved using it)
 - (BTW, UUIDs probably don't compress well, and this reduces the bloat of having
   them repeated lots of times in the tree.)
 - Per UUID branches mean that if it wants to find a file's location
@@ -77,7 +79,7 @@ with a indication of the presense/absense of the key is found.
   or transfered.
 - It could get pretty slow when digging deeper.
 - Only 3 places in git-annex will be affected by any slowdown: move --from,
-  get and drop.
+  get and drop. (Update: Now also unused, whereis, fsck) 
 
 ## alternate
 
@@ -109,7 +111,7 @@ which could get expensive.
 ## way outside the box approach
 
 Another approach I have been mulling over is keeping the log file
-branch checked out in .git-annex/logs/ -- this would be a checkout of a git
+branch checked out in .git/annex/logs/ -- this would be a checkout of a git
 repository inside a git repository, using "git fake bare" techniques. This
 would solve the merge problem, since git auto merge could be used. It would
 still mean all the log files are on-disk, which annoys some. It would

From 2fcb8e3b11cdbb9de253050c1fbb6f08ef1071e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 13:48:02 -0400
Subject: [PATCH 1868/8313] update

---
 doc/todo/branching.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index a035e53e80..3a7a2d3ca9 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -118,6 +118,13 @@ still mean all the log files are on-disk, which annoys some. It would
 require some tighter integration with git, so that after a pull, the log
 repo is updated with the data pulled. --[[Joey]] 
 
+> Seems I can't use git fake bare exactly. Instead, the best option
+> seems to be `git clone --shared` to make a clone that uses
+> `.git/annex/logs/.git` to hold its index etc, but (mostly) uses
+> objects from the main repo. There would be some bloat,
+> as commits to the logs made in there would not be shared with the main
+> repo. Using `GIT_OBJECT_DIRECTORY` might be a way to avoid that bloat.
+
 ## notes
 
 Another approach could be to use git-notes. It supports merging branches

From 91e50782ce6d634ffc8c2f809c80b6d4ff94a5ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 15:22:07 -0400
Subject: [PATCH 1869/8313] thought

---
 doc/todo/branching.mdwn | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 3a7a2d3ca9..9c44c03feb 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -135,3 +135,25 @@ Problem: Notes are usually attached to git
 objects, and there are no git objects corresponding to git-annex keys.
 
 Problem: Notes are not normally copied when cloning.
+
+------
+
+## elminating the merge problem
+
+Most of the above options are complicated by the problem of how to merge
+changes from remotes. It should be possible to deal with the merge
+problem generically. Something like this:
+
+* We have a local branch `B`.
+* For remotes, there are also `origin/B`, `otherremote/B`, etc.
+* To merge two branches `B` and `foo/B`, construct a merge commit that
+  makes each file have all lines that were in either version of the file,
+  with duplicates removed (probably). Do this without checking out a tree,
+  or using a temporary directory. (One easy but expensive way is to just
+  check out the branch to a temp dir, union merge into it, and remove the
+  temp dir ... but it should be possible to do it without using a temp dir.)
+* As a `post-merge` hook, merge `*/B` into `B`. This will ensure `B`
+  is always up-to-date after a pull from a remote.
+* When pushing to a remote, nothing need to be done, except ensure
+  `B` is either successfully pushed, or the push fails (and a pull needs to
+  be done to get the remote's changes merged into `B`).

From c835166a7cebfa44d232bbed7c5b5e22bdfeb2bd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 19:44:45 -0400
Subject: [PATCH 1870/8313] add git-union-merge

This is a new git subcommand, that does a generic union merge operation
between two refs, storing the result in a branch. It operates efficiently
without touching the working tree. It does need to write out a temporary
index file, and may need to write out some other temp files as well.

This could be useful for anything that stores data in a branch,
and needs to merge changes into that branch without actually checking the
branch out. Since conflict handling can't be done without a working copy,
the merge type is always a union merge, which is fine for data stored in
log format (as git-annex does), or in non-conflicting files
(as pristine-tar does).

This probably belongs in git proper, but it will live in git-annex for now.

---

Plan is to move .git-annex/ to a git-annex branch, and use git-union-merge
to handle merging changes when pulling from remotes.

Some preliminary benchmarking using real .git-annex/ data indicates
that it's quite fast, except for the "git add" call, which is as slow
as "git add" tends to be with a big index.
---
 .gitignore               |   2 +
 GitRepo.hs               |   3 +-
 Makefile                 |   6 +-
 debian/changelog         |   2 +
 doc/git-union-merge.mdwn |  38 +++++++++++++
 doc/todo/branching.mdwn  |   6 +-
 git-union-merge.hs       | 120 +++++++++++++++++++++++++++++++++++++++
 7 files changed, 170 insertions(+), 7 deletions(-)
 create mode 100644 doc/git-union-merge.mdwn
 create mode 100644 git-union-merge.hs

diff --git a/.gitignore b/.gitignore
index b73167c925..9a4bc80de3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,8 +5,10 @@ configure
 SysConfig.hs
 git-annex
 git-annex-shell
+git-union-merge
 git-annex.1
 git-annex-shell.1
+git-union-merge.1
 doc/.ikiwiki
 html
 *.tix
diff --git a/GitRepo.hs b/GitRepo.hs
index 24bc9b5c2b..0bee2842a5 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -38,6 +38,7 @@ module GitRepo (
 	gitCommandLine,
 	run,
 	pipeRead,
+	pipeNullSplit,
 	attributes,
 	remotes,
 	remotesAdd,
@@ -412,7 +413,7 @@ typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
 		end = [Param "--"] ++ map File l
 
 {- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it into a list of files. -}
+ - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
 pipeNullSplit repo params = do
 	fs0 <- pipeRead repo params
diff --git a/Makefile b/Makefile
index 286c3a6e54..915b0bf0b2 100644
--- a/Makefile
+++ b/Makefile
@@ -6,8 +6,8 @@ GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)
 endif
 GHCMAKE=ghc $(GHCFLAGS) --make
 
-bins=git-annex git-annex-shell
-mans=git-annex.1 git-annex-shell.1
+bins=git-annex git-annex-shell git-union-merge
+mans=git-annex.1 git-annex-shell.1 git-union-merge.1
 
 all: $(bins) $(mans) docs
 
@@ -33,6 +33,8 @@ git-annex.1: doc/git-annex.mdwn
 	./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1
 git-annex-shell.1: doc/git-annex-shell.mdwn
 	./mdwn2man git-annex-shell 1 doc/git-annex-shell.mdwn > git-annex-shell.1
+git-union-merge.1: doc/git-union-merge.mdwn
+	./mdwn2man git-union-merge 1 doc/git-union-merge.mdwn > git-union-merge.1
 
 install: all
 	install -d $(DESTDIR)$(PREFIX)/bin
diff --git a/debian/changelog b/debian/changelog
index de012de5bd..b96b9f43d6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (0.20110611) UNRELEASED; urgency=low
     such as btrfs.
   * Allow --trust etc to specify a repository by name, for temporarily 
     trusting repositories that are not configured remotes.
+  * git-union-merge: New git subcommand, that does a generic union merge
+    operation, and operates efficiently without touching the working tree.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
new file mode 100644
index 0000000000..ac8e8f7a99
--- /dev/null
+++ b/doc/git-union-merge.mdwn
@@ -0,0 +1,38 @@
+# NAME
+
+git-union-merge - Join branches together using a union merge
+
+# SYNOPSIS
+
+git union-merge branch ref ref
+
+# DESCRIPTION
+
+Does a union merge between two refs, storing the result in the
+specified branch.
+
+The union merge will always succeed, but assumes that files can be merged
+simply by concacenating together lines from all the oldrefs, in any order.
+So, this is useful only for branches containing log-type data.
+
+That this does not touch the checked out working copy. It operates
+entirely on git refs and branches.
+
+# EXAMPLE
+
+	git union-merge git-annex git-annex origin/git-annex
+
+Merges the current git-annex branch, and a version from origin,
+storing the result in the git-annex branch.
+
+# BUGS
+
+File modes are not currently merged.
+
+# AUTHOR
+
+Joey Hess 
+
+
+
+Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 9c44c03feb..37e7b6edd2 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -148,10 +148,8 @@ problem generically. Something like this:
 * For remotes, there are also `origin/B`, `otherremote/B`, etc.
 * To merge two branches `B` and `foo/B`, construct a merge commit that
   makes each file have all lines that were in either version of the file,
-  with duplicates removed (probably). Do this without checking out a tree,
-  or using a temporary directory. (One easy but expensive way is to just
-  check out the branch to a temp dir, union merge into it, and remove the
-  temp dir ... but it should be possible to do it without using a temp dir.)
+  with duplicates removed (probably). Do this without checking out a tree.
+  -- now implemented as git-union-merge
 * As a `post-merge` hook, merge `*/B` into `B`. This will ensure `B`
   is always up-to-date after a pull from a remote.
 * When pushing to a remote, nothing need to be done, except ensure
diff --git a/git-union-merge.hs b/git-union-merge.hs
new file mode 100644
index 0000000000..482f66daa0
--- /dev/null
+++ b/git-union-merge.hs
@@ -0,0 +1,120 @@
+{- git-union-merge program
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+import System.Environment
+import System.FilePath
+import System.Directory
+import System.Cmd
+import System.Cmd.Utils
+import System.Posix.Env (setEnv)
+import System.Posix.Directory (changeWorkingDirectory)
+import Control.Monad (when, unless)
+import Data.List
+
+import qualified GitRepo as Git
+import Utility
+
+header :: String
+header = "Usage: git-union-merge branch ref ref"
+
+usage :: IO a
+usage = error $ "bad parameters\n\n" ++ header
+
+main :: IO ()
+main = do
+	[branch, aref, bref] <- parseArgs
+	g <- setup
+	stage g aref bref
+	commit g branch aref bref
+	cleanup g
+
+parseArgs :: IO [String]
+parseArgs = do
+	args <- getArgs
+	if (length args /= 3)
+		then usage
+		else return args
+
+tmpDir :: Git.Repo -> FilePath
+tmpDir g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge"
+
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge.index"
+
+{- Moves to a temporary directory, and configures git to use it as its
+ - working tree, and to use a temporary index file as well. -}
+setup :: IO Git.Repo
+setup = do
+	g <- Git.configRead =<< Git.repoFromCwd
+	cleanup g -- idempotency
+	let tmp = tmpDir g
+	createDirectoryIfMissing True tmp
+	changeWorkingDirectory tmp
+	-- Note that due to these variables being set, Git.run and
+	-- similar helpers cannot be used, as they override the work tree.
+	-- It is only safe to use Git.run etc when doing things that do
+	-- not operate on the work tree.
+	setEnv "GIT_WORK_TREE" tmp True
+	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
+	return g
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = do
+	e <- doesDirectoryExist (tmpDir g)
+	when e $ removeDirectoryRecursive (tmpDir g)
+	e' <- doesFileExist (tmpIndex g)
+	when e' $ removeFile (tmpIndex g)
+
+{- Stages the content of both refs into the index. -}
+stage :: Git.Repo -> String -> String -> IO ()
+stage g aref bref = do
+	-- populate index with the contents of aref, as a starting point
+	_ <- system $ "git ls-tree -r --full-name --full-tree " ++ aref ++ 
+		" | git update-index --index-info"
+	-- identify files that are different in bref, and stage merged files
+	diff <- Git.pipeNullSplit g $ map Param
+		["diff-tree", "--raw", "-z", "--no-renames", "-l0", aref, bref]
+	mapM_ genfile (pairs diff)
+	_ <- system "git add ."
+	return ()
+	where
+		pairs [] = []
+		pairs (_:[]) = error "parse error"
+		pairs (a:b:rest) = (a,b):pairs rest
+
+		nullsha = take 40 $ repeat '0'
+
+		genfile (info, file) = do
+			let [_colonamode, _bmode, asha, bsha, _status] = words info
+			let shas = 
+				if bsha == nullsha
+					then [] -- staged from aref
+					else
+						if asha == nullsha
+							then [bsha]
+							else [asha, bsha]
+			unless (null shas) $ do
+				content <- Git.pipeRead g $ map Param ("show":shas)
+				writeFile file $ unlines $ nub $ lines content
+
+{- Commits the index into the specified branch. -}
+commit :: Git.Repo -> String -> String -> String -> IO ()
+commit g branch aref bref = do
+	tree <- getsha $
+		pipeFrom "git" ["write-tree"]
+	sha <- getsha $ 
+		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
+			"union merge"
+	Git.run g "update-ref" [Param $ "refs/heads/" ++ branch, Param sha]
+	where
+		getsha a = do
+			(_, t) <- a
+			let t' = if last t == '\n'
+				then take (length t - 1) t
+				else t
+			when (null t') $ error "failed to read sha from git"
+			return t'

From d519bc71372fe1962a9e03fc5472f9fe870066f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 21:35:39 -0400
Subject: [PATCH 1871/8313] sped up git-union-merge

Avoided the slow git add, instead inject content directly into git and
populate the index all in one pass. Now this runs on my large real-world
repo in 10 seconds, which is acceptable.

Also lots of code cleanups.
---
 GitRepo.hs         |  16 ++++++-
 git-union-merge.hs | 111 ++++++++++++++++++++++++---------------------
 2 files changed, 74 insertions(+), 53 deletions(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 0bee2842a5..9f4a38a5fb 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -38,6 +38,8 @@ module GitRepo (
 	gitCommandLine,
 	run,
 	pipeRead,
+	pipeWrite,
+	pipeWriteRead,
 	pipeNullSplit,
 	attributes,
 	remotes,
@@ -348,7 +350,7 @@ run repo subcommand params = assertLocal repo $
 	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
 		>>! error $ "git " ++ show params ++ " failed"
 
-{- Runs a git subcommand and returns it output, lazily. 
+{- Runs a git subcommand and returns its output, lazily. 
  -
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
@@ -358,6 +360,18 @@ pipeRead repo params = assertLocal repo $ do
 	(_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params
 	return s
 
+{- Runs a git subcommand, feeding it input.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWrite :: Repo -> [CommandParam] -> String -> IO PipeHandle
+pipeWrite repo params s = assertLocal repo $
+	pipeTo "git" (toCommand $ gitCommandLine repo params) s
+
+{- Runs a git subcommand, feeding it input, and returning its output.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWriteRead :: Repo -> [CommandParam] -> String -> IO (PipeHandle, String)
+pipeWriteRead repo params s = assertLocal repo $
+	pipeBoth "git" (toCommand $ gitCommandLine repo params) s
+
 {- Reaps any zombie git processes. -}
 reap :: IO ()
 reap = do
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 482f66daa0..b0c59a6d36 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -8,12 +8,12 @@
 import System.Environment
 import System.FilePath
 import System.Directory
-import System.Cmd
 import System.Cmd.Utils
 import System.Posix.Env (setEnv)
-import System.Posix.Directory (changeWorkingDirectory)
-import Control.Monad (when, unless)
+import Control.Monad (when)
 import Data.List
+import Data.Maybe
+import Data.String.Utils
 
 import qualified GitRepo as Git
 import Utility
@@ -39,82 +39,89 @@ parseArgs = do
 		then usage
 		else return args
 
-tmpDir :: Git.Repo -> FilePath
-tmpDir g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge"
-
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "tmp"  "git-union-merge.index"
+tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
 
-{- Moves to a temporary directory, and configures git to use it as its
- - working tree, and to use a temporary index file as well. -}
+{- Configures git to use a temporary index file. -}
 setup :: IO Git.Repo
 setup = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	cleanup g -- idempotency
-	let tmp = tmpDir g
-	createDirectoryIfMissing True tmp
-	changeWorkingDirectory tmp
-	-- Note that due to these variables being set, Git.run and
-	-- similar helpers cannot be used, as they override the work tree.
-	-- It is only safe to use Git.run etc when doing things that do
-	-- not operate on the work tree.
-	setEnv "GIT_WORK_TREE" tmp True
 	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
 	return g
 
 cleanup :: Git.Repo -> IO ()
 cleanup g = do
-	e <- doesDirectoryExist (tmpDir g)
-	when e $ removeDirectoryRecursive (tmpDir g)
 	e' <- doesFileExist (tmpIndex g)
 	when e' $ removeFile (tmpIndex g)
 
 {- Stages the content of both refs into the index. -}
 stage :: Git.Repo -> String -> String -> IO ()
 stage g aref bref = do
-	-- populate index with the contents of aref, as a starting point
-	_ <- system $ "git ls-tree -r --full-name --full-tree " ++ aref ++ 
-		" | git update-index --index-info"
-	-- identify files that are different in bref, and stage merged files
-	diff <- Git.pipeNullSplit g $ map Param
-		["diff-tree", "--raw", "-z", "--no-renames", "-l0", aref, bref]
-	mapM_ genfile (pairs diff)
-	_ <- system "git add ."
-	return ()
+	-- Get the contents of aref, as a starting point.
+	ls <- fromgit
+		["ls-tree", "-z", "-r", "--full-tree", aref]
+	-- Identify files that are different between aref and bref, and
+	-- inject merged versions into git.
+	diff <- fromgit
+		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
+	ls' <- mapM mergefile (pairs diff)
+	-- Populate the index file. Later lines override earlier ones.
+	togit ["update-index", "-z", "--index-info"]
+		(join "\0" $ ls++catMaybes ls')
 	where
+		fromgit l = Git.pipeNullSplit g (map Param l)
+		togit l content = Git.pipeWrite g (map Param l) content
+			>>= forceSuccess
+		tofromgit l content = do
+			(h, s) <- Git.pipeWriteRead g (map Param l) content
+			length s `seq` do
+				forceSuccess h
+				Git.reap
+				return ((), s)
+
 		pairs [] = []
 		pairs (_:[]) = error "parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
-
-		nullsha = take 40 $ repeat '0'
-
-		genfile (info, file) = do
+		
+		nullsha = take shaSize $ repeat '0'
+		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+		unionmerge = unlines . nub . lines
+		
+		mergefile (info, file) = do
 			let [_colonamode, _bmode, asha, bsha, _status] = words info
-			let shas = 
-				if bsha == nullsha
-					then [] -- staged from aref
-					else
-						if asha == nullsha
-							then [bsha]
-							else [asha, bsha]
-			unless (null shas) $ do
-				content <- Git.pipeRead g $ map Param ("show":shas)
-				writeFile file $ unlines $ nub $ lines content
+			if bsha == nullsha
+				then return Nothing -- already staged from aref
+				else mergefile' file asha bsha
+		mergefile' file asha bsha = do
+			let shas = filter (/= nullsha) [asha, bsha]
+			content <- Git.pipeRead g $ map Param ("show":shas)
+			sha <- getSha "hash-object" $
+				tofromgit ["hash-object", "-w", "--stdin"] $
+					unionmerge content
+			return $ Just $ ls_tree_line sha file
 
 {- Commits the index into the specified branch. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
 commit g branch aref bref = do
-	tree <- getsha $
+	tree <- getSha "write-tree"  $
 		pipeFrom "git" ["write-tree"]
-	sha <- getsha $ 
+	sha <- getSha "commit-tree" $
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
 	Git.run g "update-ref" [Param $ "refs/heads/" ++ branch, Param sha]
-	where
-		getsha a = do
-			(_, t) <- a
-			let t' = if last t == '\n'
-				then take (length t - 1) t
-				else t
-			when (null t') $ error "failed to read sha from git"
-			return t'
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO (a, String) -> IO String
+getSha subcommand a = do
+	(_, t) <- a
+	let t' = if last t == '\n'
+		then take (length t - 1) t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return t'
+
+shaSize :: Int
+shaSize = 40

From 01e8a0a9e5de13b4c1f5baa5d44e8898692f62e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 21:38:52 -0400
Subject: [PATCH 1872/8313] allow git-union-merge to write to any ref

Not just refs/heads/* branches.
---
 doc/git-union-merge.mdwn | 6 +++---
 git-union-merge.hs       | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index ac8e8f7a99..0322641efa 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -4,12 +4,12 @@ git-union-merge - Join branches together using a union merge
 
 # SYNOPSIS
 
-git union-merge branch ref ref
+git union-merge newref ref ref
 
 # DESCRIPTION
 
 Does a union merge between two refs, storing the result in the
-specified branch.
+specified newref.
 
 The union merge will always succeed, but assumes that files can be merged
 simply by concacenating together lines from all the oldrefs, in any order.
@@ -20,7 +20,7 @@ entirely on git refs and branches.
 
 # EXAMPLE
 
-	git union-merge git-annex git-annex origin/git-annex
+	git union-merge refs/heads/git-annex git-annex origin/git-annex
 
 Merges the current git-annex branch, and a version from origin,
 storing the result in the git-annex branch.
diff --git a/git-union-merge.hs b/git-union-merge.hs
index b0c59a6d36..8e7e0367b7 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -19,7 +19,7 @@ import qualified GitRepo as Git
 import Utility
 
 header :: String
-header = "Usage: git-union-merge branch ref ref"
+header = "Usage: git-union-merge newref ref ref"
 
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
@@ -109,7 +109,7 @@ commit g branch aref bref = do
 	sha <- getSha "commit-tree" $
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
-	Git.run g "update-ref" [Param $ "refs/heads/" ++ branch, Param sha]
+	Git.run g "update-ref" [Param branch, Param sha]
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}

From 8b749d4bfdc81155e0dc17026f1575620bc132e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 21:42:17 -0400
Subject: [PATCH 1873/8313] reorder git-union-merge params

---
 doc/git-union-merge.mdwn | 4 ++--
 git-union-merge.hs       | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index 0322641efa..495612f36d 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -4,7 +4,7 @@ git-union-merge - Join branches together using a union merge
 
 # SYNOPSIS
 
-git union-merge newref ref ref
+git union-merge ref ref newref
 
 # DESCRIPTION
 
@@ -20,7 +20,7 @@ entirely on git refs and branches.
 
 # EXAMPLE
 
-	git union-merge refs/heads/git-annex git-annex origin/git-annex
+	git union-merge git-annex origin/git-annex refs/heads/git-annex 
 
 Merges the current git-annex branch, and a version from origin,
 storing the result in the git-annex branch.
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 8e7e0367b7..2a2570cf4f 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -26,10 +26,10 @@ usage = error $ "bad parameters\n\n" ++ header
 
 main :: IO ()
 main = do
-	[branch, aref, bref] <- parseArgs
+	[aref, bref, newref] <- parseArgs
 	g <- setup
 	stage g aref bref
-	commit g branch aref bref
+	commit g aref bref newref
 	cleanup g
 
 parseArgs :: IO [String]
@@ -103,13 +103,13 @@ stage g aref bref = do
 
 {- Commits the index into the specified branch. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
-commit g branch aref bref = do
+commit g aref bref newref = do
 	tree <- getSha "write-tree"  $
 		pipeFrom "git" ["write-tree"]
 	sha <- getSha "commit-tree" $
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
-	Git.run g "update-ref" [Param branch, Param sha]
+	Git.run g "update-ref" [Param newref, Param sha]
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}

From a9a464914887b953d216fc2af8c691b0ee9b61d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 22:21:02 -0400
Subject: [PATCH 1874/8313] typo

---
 git-union-merge.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-union-merge.hs b/git-union-merge.hs
index 2a2570cf4f..e11f93701c 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -19,7 +19,7 @@ import qualified GitRepo as Git
 import Utility
 
 header :: String
-header = "Usage: git-union-merge newref ref ref"
+header = "Usage: git-union-merge ref ref newref"
 
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header

From 53706ad9bf9bf22359154edab0ca61acea9027e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 22:29:46 -0400
Subject: [PATCH 1875/8313] move bug report

---
 doc/{forum => bugs}/git_annex_unlock_is_not_atomic.mdwn | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/{forum => bugs}/git_annex_unlock_is_not_atomic.mdwn (100%)

diff --git a/doc/forum/git_annex_unlock_is_not_atomic.mdwn b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
similarity index 100%
rename from doc/forum/git_annex_unlock_is_not_atomic.mdwn
rename to doc/bugs/git_annex_unlock_is_not_atomic.mdwn

From 9f9e17aa0f4898063a58c88661bca01465b126a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Jun 2011 22:38:18 -0400
Subject: [PATCH 1876/8313] unlock: Made atomic.

---
 Command/Unlock.hs                            | 10 +++++++---
 debian/changelog                             |  1 +
 doc/bugs/git_annex_unlock_is_not_atomic.mdwn |  2 ++
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 161df2ddf9..ca8b62502a 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -44,11 +44,15 @@ perform dest key = do
 
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g key
-	liftIO $ removeFile dest
+	let tmpdest = gitAnnexTmpLocation g key
+	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showNote "copying..."
-	ok <- liftIO $ copyFile src dest
+	ok <- liftIO $ copyFile src tmpdest
         if ok
                 then do
-			liftIO $ allowWrite dest
+			liftIO $ do
+				removeFile dest
+				renameFile tmpdest dest
+				allowWrite dest
 			next $ return True
                 else error "copy failed!"
diff --git a/debian/changelog b/debian/changelog
index b96b9f43d6..6439eb89fa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (0.20110611) UNRELEASED; urgency=low
     such as btrfs.
   * Allow --trust etc to specify a repository by name, for temporarily 
     trusting repositories that are not configured remotes.
+  * unlock: Made atomic.
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
 
diff --git a/doc/bugs/git_annex_unlock_is_not_atomic.mdwn b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
index a7751f6b7d..6d324ff500 100644
--- a/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
+++ b/doc/bugs/git_annex_unlock_is_not_atomic.mdwn
@@ -3,3 +3,5 @@ Running a command like
 git annex unlock myfile
 
 is not atomic, that is if the execution is aborted you may end up with an incomplete version of myfile in the directory. If you don't notice this you may lock it again and then propagate this bad version of the file to your other repositories. A simple workaround is to simply name it something else while unlocking and then rename it to the correct filename once it's completely copied. I don't know Haskel yet so I can not fix this issue otherwise I would sure try. A part from this, I love git annex.
+
+> [[fixed|done]] --[[Joey]] 

From e735d459b531246798622994718eaccfcf0086ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:09:06 -0400
Subject: [PATCH 1877/8313] moved to library

---
 GitUnionMerge.hs   | 115 +++++++++++++++++++++++++++++++++++++++++++++
 git-union-merge.hs | 107 ++---------------------------------------
 2 files changed, 120 insertions(+), 102 deletions(-)
 create mode 100644 GitUnionMerge.hs

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
new file mode 100644
index 0000000000..dde4b7a043
--- /dev/null
+++ b/GitUnionMerge.hs
@@ -0,0 +1,115 @@
+{- git-union-merge library
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module GitUnionMerge (
+	unionMerge
+) where
+
+import System.FilePath
+import System.Directory
+import System.Cmd.Utils
+import System.Posix.Env (setEnv, unsetEnv)
+import Control.Monad (when)
+import Data.List
+import Data.Maybe
+import Data.String.Utils
+
+import qualified GitRepo as Git
+import Utility
+
+unionMerge :: Git.Repo -> String -> String -> String -> IO ()
+unionMerge g aref bref newref = do
+	setup g
+	stage g aref bref
+	commit g aref bref newref
+	cleanup g
+
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+
+{- Configures git to use a temporary index file. -}
+setup :: Git.Repo -> IO ()
+setup g = do
+	cleanup g -- idempotency
+	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = do
+	unsetEnv "GIT_INDEX_FILE"
+	e' <- doesFileExist (tmpIndex g)
+	when e' $ removeFile (tmpIndex g)
+
+{- Stages the content of both refs into the index. -}
+stage :: Git.Repo -> String -> String -> IO ()
+stage g aref bref = do
+	-- Get the contents of aref, as a starting point.
+	ls <- fromgit
+		["ls-tree", "-z", "-r", "--full-tree", aref]
+	-- Identify files that are different between aref and bref, and
+	-- inject merged versions into git.
+	diff <- fromgit
+		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
+	ls' <- mapM mergefile (pairs diff)
+	-- Populate the index file. Later lines override earlier ones.
+	togit ["update-index", "-z", "--index-info"]
+		(join "\0" $ ls++catMaybes ls')
+	where
+		fromgit l = Git.pipeNullSplit g (map Param l)
+		togit l content = Git.pipeWrite g (map Param l) content
+			>>= forceSuccess
+		tofromgit l content = do
+			(h, s) <- Git.pipeWriteRead g (map Param l) content
+			length s `seq` do
+				forceSuccess h
+				Git.reap
+				return ((), s)
+
+		pairs [] = []
+		pairs (_:[]) = error "parse error"
+		pairs (a:b:rest) = (a,b):pairs rest
+		
+		nullsha = take shaSize $ repeat '0'
+		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+		unionmerge = unlines . nub . lines
+		
+		mergefile (info, file) = do
+			let [_colonamode, _bmode, asha, bsha, _status] = words info
+			if bsha == nullsha
+				then return Nothing -- already staged from aref
+				else mergefile' file asha bsha
+		mergefile' file asha bsha = do
+			let shas = filter (/= nullsha) [asha, bsha]
+			content <- Git.pipeRead g $ map Param ("show":shas)
+			sha <- getSha "hash-object" $
+				tofromgit ["hash-object", "-w", "--stdin"] $
+					unionmerge content
+			return $ Just $ ls_tree_line sha file
+
+{- Commits the index into the specified branch. -}
+commit :: Git.Repo -> String -> String -> String -> IO ()
+commit g aref bref newref = do
+	tree <- getSha "write-tree"  $
+		pipeFrom "git" ["write-tree"]
+	sha <- getSha "commit-tree" $
+		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
+			"union merge"
+	Git.run g "update-ref" [Param newref, Param sha]
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO (a, String) -> IO String
+getSha subcommand a = do
+	(_, t) <- a
+	let t' = if last t == '\n'
+		then take (length t - 1) t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return t'
+
+shaSize :: Int
+shaSize = 40
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e11f93701c..62a79a4c22 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -6,17 +6,9 @@
  -}
 
 import System.Environment
-import System.FilePath
-import System.Directory
-import System.Cmd.Utils
-import System.Posix.Env (setEnv)
-import Control.Monad (when)
-import Data.List
-import Data.Maybe
-import Data.String.Utils
 
+import GitUnionMerge
 import qualified GitRepo as Git
-import Utility
 
 header :: String
 header = "Usage: git-union-merge ref ref newref"
@@ -24,14 +16,6 @@ header = "Usage: git-union-merge ref ref newref"
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
-main :: IO ()
-main = do
-	[aref, bref, newref] <- parseArgs
-	g <- setup
-	stage g aref bref
-	commit g aref bref newref
-	cleanup g
-
 parseArgs :: IO [String]
 parseArgs = do
 	args <- getArgs
@@ -39,89 +23,8 @@ parseArgs = do
 		then usage
 		else return args
 
-tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
-
-{- Configures git to use a temporary index file. -}
-setup :: IO Git.Repo
-setup = do
+main :: IO ()
+main = do
+	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	cleanup g -- idempotency
-	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
-	return g
-
-cleanup :: Git.Repo -> IO ()
-cleanup g = do
-	e' <- doesFileExist (tmpIndex g)
-	when e' $ removeFile (tmpIndex g)
-
-{- Stages the content of both refs into the index. -}
-stage :: Git.Repo -> String -> String -> IO ()
-stage g aref bref = do
-	-- Get the contents of aref, as a starting point.
-	ls <- fromgit
-		["ls-tree", "-z", "-r", "--full-tree", aref]
-	-- Identify files that are different between aref and bref, and
-	-- inject merged versions into git.
-	diff <- fromgit
-		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
-	ls' <- mapM mergefile (pairs diff)
-	-- Populate the index file. Later lines override earlier ones.
-	togit ["update-index", "-z", "--index-info"]
-		(join "\0" $ ls++catMaybes ls')
-	where
-		fromgit l = Git.pipeNullSplit g (map Param l)
-		togit l content = Git.pipeWrite g (map Param l) content
-			>>= forceSuccess
-		tofromgit l content = do
-			(h, s) <- Git.pipeWriteRead g (map Param l) content
-			length s `seq` do
-				forceSuccess h
-				Git.reap
-				return ((), s)
-
-		pairs [] = []
-		pairs (_:[]) = error "parse error"
-		pairs (a:b:rest) = (a,b):pairs rest
-		
-		nullsha = take shaSize $ repeat '0'
-		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
-		unionmerge = unlines . nub . lines
-		
-		mergefile (info, file) = do
-			let [_colonamode, _bmode, asha, bsha, _status] = words info
-			if bsha == nullsha
-				then return Nothing -- already staged from aref
-				else mergefile' file asha bsha
-		mergefile' file asha bsha = do
-			let shas = filter (/= nullsha) [asha, bsha]
-			content <- Git.pipeRead g $ map Param ("show":shas)
-			sha <- getSha "hash-object" $
-				tofromgit ["hash-object", "-w", "--stdin"] $
-					unionmerge content
-			return $ Just $ ls_tree_line sha file
-
-{- Commits the index into the specified branch. -}
-commit :: Git.Repo -> String -> String -> String -> IO ()
-commit g aref bref newref = do
-	tree <- getSha "write-tree"  $
-		pipeFrom "git" ["write-tree"]
-	sha <- getSha "commit-tree" $
-		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
-			"union merge"
-	Git.run g "update-ref" [Param newref, Param sha]
-
-{- Runs an action that causes a git subcommand to emit a sha, and strips
-   any trailing newline, returning the sha. -}
-getSha :: String -> IO (a, String) -> IO String
-getSha subcommand a = do
-	(_, t) <- a
-	let t' = if last t == '\n'
-		then take (length t - 1) t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return t'
-
-shaSize :: Int
-shaSize = 40
+	unionMerge g aref bref newref

From c03af0ed0c6530742206740e8b3fc75f5f959325 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:29:09 -0400
Subject: [PATCH 1878/8313] code to update a git-annex branch

There is no suitable git hook to run code when pulling changes that
might need to be merged into the git-annex branch. The post-merge hook
is only run when changes are merged into HEAD, and it's possible,
and indeed likely that many pulls will only have changes in git-annex,
but not in HEAD, and not trigger it.

So, git-annex will have to take care to update the branch before reading
from it, to make sure it has merged in current info from remotes. Happily,
this can be done quite inexpensively, just a git-show-ref to list
branches, and a minimalized git-log to see if there are unmerged changes
on the branches. To further speed up, it will be done only once per
git-annex run, max.
---
 Annex.hs  |  2 ++
 Branch.hs | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 Branch.hs

diff --git a/Annex.hs b/Annex.hs
index 0d6b309cca..bede0cbfb7 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -39,6 +39,7 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
+	, updated :: Bool
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -59,6 +60,7 @@ newState allbackends gitrepo = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
+	, updated = False
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
new file mode 100644
index 0000000000..db8dc23da8
--- /dev/null
+++ b/Branch.hs
@@ -0,0 +1,50 @@
+{- git-annex branch management
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Branch where
+
+import Control.Monad (unless)
+import Control.Monad.State (liftIO)
+
+import GitUnionMerge
+import GitRepo as Git
+import qualified Annex
+import Utility
+import Types
+import Messages
+
+name :: String
+name = "git-annex"
+
+fullname :: String
+fullname = "refs/heads/" ++ name
+
+{- Ensures that the branch is up-to-date; should be called before
+ - data is read from it. Runs only once per git-annex run. -}
+update :: Annex ()
+update = do
+	updated <- Annex.getState Annex.updated
+	unless updated $ do
+		g <- Annex.gitRepo
+		refs <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+		mapM_ updateRef $ map (last . words) (lines refs)
+		Annex.changeState $ \s -> s { Annex.updated = True }
+
+{- Ensures that a given ref has been merged into the local git-annex branch. -}
+updateRef :: String -> Annex ()
+updateRef ref
+	| ref == fullname = return ()
+	| otherwise = do
+		g <- Annex.gitRepo
+		diffs <- liftIO $ Git.pipeRead g [
+			Param "log",
+			Param (name++".."++ref),
+			Params "--oneline -n1"
+			]
+		unless (null diffs) $ do
+			showSideAction "merging " ++ ref ++ " into " ++ name ++ "..."
+			liftIO $ unionMerge g fullname ref fullname

From 9a1f0fcee2bbd96e2cebbc3df6cac8d64d38b8a5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:34:08 -0400
Subject: [PATCH 1879/8313] start v3

---
 Version.hs        | 4 ++--
 doc/upgrades.mdwn | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Version.hs b/Version.hs
index 947af8cef5..72d06f6639 100644
--- a/Version.hs
+++ b/Version.hs
@@ -20,13 +20,13 @@ import Config
 type Version = String
 
 defaultVersion :: Version
-defaultVersion = "2"
+defaultVersion = "3"
 
 supportedVersions :: [Version]
 supportedVersions = [defaultVersion]
 
 upgradableVersions :: [Version]
-upgradableVersions = ["0", "1"]
+upgradableVersions = ["0", "1", "2"]
 
 versionField :: String
 versionField = "annex.version"
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 0a07ef7aa8..516e5b3bb8 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -45,6 +45,10 @@ Example upgrade process:
 
 ## Upgrade events, so far
 
+### v2 -> v3 (git-annex version 0.20110610 to version 0.20110622)
+
+Involved moving the .git-annex/ directory into a separate git-annex branch.
+
 ### v1 -> v2 (git-annex version 0.23 to version 0.20110316)
 
 Involved adding hashing to .git/annex/ and changing the names of all keys.

From a5e6802b5b6f9354e065936998d9882e8ceecb5b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 14:44:56 -0400
Subject: [PATCH 1880/8313] typos in comments

---
 Locations.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index da781ac83a..8f7b11a5c2 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -94,7 +94,7 @@ gitAnnexObjectDir r
 	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
 	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  objectDir
 
-{- .git-annex/tmp/ is used for temp files -}
+{- .git/annex/tmp/ is used for temp files -}
 gitAnnexTmpDir :: Git.Repo -> FilePath
 gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
@@ -102,7 +102,7 @@ gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath
 gitAnnexTmpLocation r key = gitAnnexTmpDir r  keyFile key
 
-{- .git-annex/bad/ is used for bad files found during fsck -}
+{- .git/annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath
 gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 

From 7e7428f173ba1b72b4de69fd482f44161ee84420 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 16:08:09 -0400
Subject: [PATCH 1881/8313] refactor

---
 Branch.hs          | 14 ++++++++++++--
 GitRepo.hs         | 10 ++++++++++
 GitUnionMerge.hs   | 22 +++-------------------
 git-union-merge.hs | 20 +++++++++++++++++++-
 4 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index db8dc23da8..4b62fd645e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -1,11 +1,14 @@
-{- git-annex branch management
+{- management of the git-annex branch
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Branch where
+module Branch (
+	update,
+	change
+) where
 
 import Control.Monad (unless)
 import Control.Monad.State (liftIO)
@@ -48,3 +51,10 @@ updateRef ref
 		unless (null diffs) $ do
 			showSideAction "merging " ++ ref ++ " into " ++ name ++ "..."
 			liftIO $ unionMerge g fullname ref fullname
+
+{- Stages the content of a file to be committed to the branch. -}
+change :: FilePath -> String -> Annex ()
+change file content = do
+	update
+
+{- Commits staged changes to the branch. -}
diff --git a/GitRepo.hs b/GitRepo.hs
index 9f4a38a5fb..11511f77d6 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -58,6 +58,7 @@ module GitRepo (
 	typeChangedStagedFiles,
 	repoAbsPath,
 	reap,
+	withIndex,
 
 	prop_idempotent_deencode
 ) where
@@ -82,6 +83,7 @@ import Codec.Binary.UTF8.String (encode)
 import Text.Printf
 import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
+import System.Posix.Env (setEnv, unsetEnv)
 
 import Utility
 
@@ -379,6 +381,14 @@ reap = do
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
 	maybe (return ()) (const reap) r
 
+{- Runs an action using a specified index file. -}
+withIndex :: FilePath -> IO a -> IO a
+withIndex index a = do
+	setEnv "GIT_INDEX_FILE" index True
+	r <- a
+	unsetEnv "GIT_INDEX_FILE"
+	return r
+
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index dde4b7a043..8aa04f53a3 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -12,7 +12,6 @@ module GitUnionMerge (
 import System.FilePath
 import System.Directory
 import System.Cmd.Utils
-import System.Posix.Env (setEnv, unsetEnv)
 import Control.Monad (when)
 import Data.List
 import Data.Maybe
@@ -21,27 +20,12 @@ import Data.String.Utils
 import qualified GitRepo as Git
 import Utility
 
+{- Performs a union merge. Should be run with a temporary index file
+ - configured by Git.withIndex. -}
 unionMerge :: Git.Repo -> String -> String -> String -> IO ()
 unionMerge g aref bref newref = do
-	setup g
 	stage g aref bref
 	commit g aref bref newref
-	cleanup g
-
-tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
-
-{- Configures git to use a temporary index file. -}
-setup :: Git.Repo -> IO ()
-setup g = do
-	cleanup g -- idempotency
-	setEnv "GIT_INDEX_FILE" (tmpIndex g) True
-
-cleanup :: Git.Repo -> IO ()
-cleanup g = do
-	unsetEnv "GIT_INDEX_FILE"
-	e' <- doesFileExist (tmpIndex g)
-	when e' $ removeFile (tmpIndex g)
 
 {- Stages the content of both refs into the index. -}
 stage :: Git.Repo -> String -> String -> IO ()
@@ -89,7 +73,7 @@ stage g aref bref = do
 					unionmerge content
 			return $ Just $ ls_tree_line sha file
 
-{- Commits the index into the specified branch. -}
+{- Commits the index into the specified branch, as a merge commit. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
 commit g aref bref newref = do
 	tree <- getSha "write-tree"  $
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 62a79a4c22..12f49adc6f 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -6,6 +6,9 @@
  -}
 
 import System.Environment
+import System.FilePath
+import System.Directory
+import Control.Monad (when)
 
 import GitUnionMerge
 import qualified GitRepo as Git
@@ -16,6 +19,18 @@ header = "Usage: git-union-merge ref ref newref"
 usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
+tmpIndex :: Git.Repo -> FilePath
+tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+
+setup :: Git.Repo -> IO ()
+setup g = do
+	cleanup g -- idempotency
+
+cleanup :: Git.Repo -> IO ()
+cleanup g = do
+	e' <- doesFileExist (tmpIndex g)
+	when e' $ removeFile (tmpIndex g)
+
 parseArgs :: IO [String]
 parseArgs = do
 	args <- getArgs
@@ -27,4 +42,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	unionMerge g aref bref newref
+	Git.withIndex (tmpIndex g) $ do
+		setup g
+		unionMerge g aref bref newref
+		cleanup g

From 40ec8a9726586f24357a5ae2057a092a971c1046 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 17:39:45 -0400
Subject: [PATCH 1882/8313] Branch module complete

Refactored some code that it needs into GitRepo.
---
 Branch.hs          | 76 +++++++++++++++++++++++++++++++++++++++++-----
 GitRepo.hs         | 50 ++++++++++++++++++++++++------
 GitUnionMerge.hs   | 60 +++++++++++++-----------------------
 git-union-merge.hs |  8 ++---
 4 files changed, 135 insertions(+), 59 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 4b62fd645e..9152a03250 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -12,26 +12,63 @@ module Branch (
 
 import Control.Monad (unless)
 import Control.Monad.State (liftIO)
+import System.FilePath
+import System.Directory
+import Data.String.Utils
+import System.Cmd.Utils
 
 import GitUnionMerge
-import GitRepo as Git
+import qualified GitRepo as Git
 import qualified Annex
 import Utility
 import Types
 import Messages
 
+{- Name of the branch that is used to store git-annex's information. -}
 name :: String
 name = "git-annex"
 
+{- Fully qualified name of the branch. -}
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+{- A separate index file for the branch. -}
+index :: Git.Repo -> FilePath
+index g = Git.workTree g  Git.gitDir g  "index." ++ name
+
+{- Populates the branch's index file with the current branch contents.
+ - 
+ - Usually, this is only done when the index doesn't yet exist, and
+ - the index is used to build up changes to be commited to the branch.
+ -}
+genIndex :: FilePath -> Git.Repo -> IO ()
+genIndex f g = do
+	ls <- Git.pipeNullSplit g $
+		map Param ["ls-tree", "-z", "-r", "--full-tree", fullname]
+	forceSuccess =<< Git.pipeWrite g
+		(map Param ["update-index", "-z", "--index-info"])
+		(join "\0" ls)
+
+{- Runs an action using the branch's index file. -}
+withIndex :: Annex a -> Annex a
+withIndex a = do
+	g <- Annex.gitRepo
+	let f = index g
+	liftIO $ Git.useIndex f
+
+	e <- liftIO $ doesFileExist f
+	unless e $ liftIO $ genIndex f g
+
+	r <- a
+	liftIO $ Git.useDefaultIndex
+	return r
+
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
 	updated <- Annex.getState Annex.updated
-	unless updated $ do
+	unless updated $ withIndex $ do
 		g <- Annex.gitRepo
 		refs <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		mapM_ updateRef $ map (last . words) (lines refs)
@@ -49,12 +86,37 @@ updateRef ref
 			Params "--oneline -n1"
 			]
 		unless (null diffs) $ do
-			showSideAction "merging " ++ ref ++ " into " ++ name ++ "..."
-			liftIO $ unionMerge g fullname ref fullname
+			showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
+			liftIO $ unionMerge g fullname ref fullname True
 
-{- Stages the content of a file to be committed to the branch. -}
+{- Stages the content of a file into the branch's index. -}
 change :: FilePath -> String -> Annex ()
-change file content = do
-	update
+change file content = update >> do
+	g <- Annex.gitRepo
+	sha <- liftIO $ Git.hashObject g content
+	withIndex $ liftIO $ Git.run g "update-index"
+		[ Params "--add --cacheinfo 100644 ",
+		  Param sha, File file]
 
 {- Commits staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = withIndex $ do
+	g <- Annex.gitRepo
+	-- It would be expensive to check if anything needs to be
+	-- committed, so --allow-empty is used.
+	liftIO $ Git.run g "commit"
+		[Param "--allow-empty", Param "-m", Param message]
+
+{- Gets the content of a file on the branch, or content staged in the index
+ - if it's newer. Returns an empty string if the file didn't exist yet. -}
+get :: FilePath -> Annex String
+get file = withIndex $ do
+	g <- Annex.gitRepo
+	liftIO $ catch (cat g) (const $ return "")
+	where
+		-- To avoid stderr from cat-file when file does not exist,
+		-- first run it with -e to check that it exists.
+		cat g = do
+			Git.run g "cat-file" [Param "-e", catfile]
+			Git.pipeRead g [Param "cat-file", Param "blob", catfile]
+		catfile = Param $ ':':file
diff --git a/GitRepo.hs b/GitRepo.hs
index 11511f77d6..91ddf6dca9 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -58,12 +58,16 @@ module GitRepo (
 	typeChangedStagedFiles,
 	repoAbsPath,
 	reap,
-	withIndex,
+	useIndex,
+	useDefaultIndex,
+	hashObject,
+	getSha,
+	shaSize,
 
 	prop_idempotent_deencode
 ) where
 
-import Control.Monad (unless)
+import Control.Monad (unless, when)
 import System.Directory
 import System.FilePath
 import System.Posix.Directory
@@ -381,13 +385,41 @@ reap = do
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
 	maybe (return ()) (const reap) r
 
-{- Runs an action using a specified index file. -}
-withIndex :: FilePath -> IO a -> IO a
-withIndex index a = do
-	setEnv "GIT_INDEX_FILE" index True
-	r <- a
-	unsetEnv "GIT_INDEX_FILE"
-	return r
+{- Forces git to use the specified index file. -}
+useIndex :: FilePath -> IO ()
+useIndex index = setEnv "GIT_INDEX_FILE" index True
+
+{- Undoes useIndex -}
+useDefaultIndex :: IO ()
+useDefaultIndex = unsetEnv "GIT_INDEX_FILE"
+
+{- Injects some content into git, returning its hash. -}
+hashObject :: Repo -> String -> IO String
+hashObject repo content = getSha subcmd $ do
+	(h, s) <- pipeWriteRead repo (map Param params) content
+	length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-w", "--stdin"]
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO String -> IO String
+getSha subcommand a = do
+	t <- a
+	let t' = if last t == '\n'
+		then take (length t - 1) t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return t'
+
+{- Size of a git sha. -}
+shaSize :: Int
+shaSize = 40
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 8aa04f53a3..ba9ea79e4f 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -9,10 +9,7 @@ module GitUnionMerge (
 	unionMerge
 ) where
 
-import System.FilePath
-import System.Directory
 import System.Cmd.Utils
-import Control.Monad (when)
 import Data.List
 import Data.Maybe
 import Data.String.Utils
@@ -21,18 +18,24 @@ import qualified GitRepo as Git
 import Utility
 
 {- Performs a union merge. Should be run with a temporary index file
- - configured by Git.withIndex. -}
-unionMerge :: Git.Repo -> String -> String -> String -> IO ()
-unionMerge g aref bref newref = do
-	stage g aref bref
+ - configured by Git.useIndex.
+ -
+ - Use indexpopulated only if the index file already contains exactly the
+ - contents of aref.
+ -}
+unionMerge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
+unionMerge g aref bref newref indexpopulated = do
+	stage g aref bref indexpopulated
 	commit g aref bref newref
 
 {- Stages the content of both refs into the index. -}
-stage :: Git.Repo -> String -> String -> IO ()
-stage g aref bref = do
-	-- Get the contents of aref, as a starting point.
-	ls <- fromgit
-		["ls-tree", "-z", "-r", "--full-tree", aref]
+stage :: Git.Repo -> String -> String -> Bool -> IO ()
+stage g aref bref indexpopulated = do
+	-- Get the contents of aref, as a starting point, unless
+	-- the index is already populated with it.
+	ls <- if indexpopulated
+		then return []
+		else fromgit ["ls-tree", "-z", "-r", "--full-tree", aref]
 	-- Identify files that are different between aref and bref, and
 	-- inject merged versions into git.
 	diff <- fromgit
@@ -45,18 +48,12 @@ stage g aref bref = do
 		fromgit l = Git.pipeNullSplit g (map Param l)
 		togit l content = Git.pipeWrite g (map Param l) content
 			>>= forceSuccess
-		tofromgit l content = do
-			(h, s) <- Git.pipeWriteRead g (map Param l) content
-			length s `seq` do
-				forceSuccess h
-				Git.reap
-				return ((), s)
 
 		pairs [] = []
 		pairs (_:[]) = error "parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
 		
-		nullsha = take shaSize $ repeat '0'
+		nullsha = take Git.shaSize $ repeat '0'
 		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 		unionmerge = unlines . nub . lines
 		
@@ -68,32 +65,17 @@ stage g aref bref = do
 		mergefile' file asha bsha = do
 			let shas = filter (/= nullsha) [asha, bsha]
 			content <- Git.pipeRead g $ map Param ("show":shas)
-			sha <- getSha "hash-object" $
-				tofromgit ["hash-object", "-w", "--stdin"] $
-					unionmerge content
+			sha <- Git.hashObject g $ unionmerge content
 			return $ Just $ ls_tree_line sha file
 
 {- Commits the index into the specified branch, as a merge commit. -}
 commit :: Git.Repo -> String -> String -> String -> IO ()
 commit g aref bref newref = do
-	tree <- getSha "write-tree"  $
+	tree <- Git.getSha "write-tree" $ ignorehandle $
 		pipeFrom "git" ["write-tree"]
-	sha <- getSha "commit-tree" $
+	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
 		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
 			"union merge"
 	Git.run g "update-ref" [Param newref, Param sha]
-
-{- Runs an action that causes a git subcommand to emit a sha, and strips
-   any trailing newline, returning the sha. -}
-getSha :: String -> IO (a, String) -> IO String
-getSha subcommand a = do
-	(_, t) <- a
-	let t' = if last t == '\n'
-		then take (length t - 1) t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return t'
-
-shaSize :: Int
-shaSize = 40
+	where
+		ignorehandle a = return . snd =<< a
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 12f49adc6f..e8ac0a0c54 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -42,7 +42,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	Git.withIndex (tmpIndex g) $ do
-		setup g
-		unionMerge g aref bref newref
-		cleanup g
+	Git.useIndex (tmpIndex g)
+	setup g
+	unionMerge g aref bref newref False
+	cleanup g

From 5d20ac5800f6577fb40ebfa7c8d68df43757a3e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 19:09:20 -0400
Subject: [PATCH 1883/8313] export the commit function and generalize

---
 GitUnionMerge.hs   | 21 ++++++++++++---------
 git-union-merge.hs |  4 ++--
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index ba9ea79e4f..bc12cbe275 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -6,7 +6,9 @@
  -}
 
 module GitUnionMerge (
-	unionMerge
+	merge,
+	stage,
+	commit
 ) where
 
 import System.Cmd.Utils
@@ -23,10 +25,10 @@ import Utility
  - Use indexpopulated only if the index file already contains exactly the
  - contents of aref.
  -}
-unionMerge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
-unionMerge g aref bref newref indexpopulated = do
+merge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
+merge g aref bref newref indexpopulated = do
 	stage g aref bref indexpopulated
-	commit g aref bref newref
+	commit g "union merge" newref [aref, bref]
 
 {- Stages the content of both refs into the index. -}
 stage :: Git.Repo -> String -> String -> Bool -> IO ()
@@ -68,14 +70,15 @@ stage g aref bref indexpopulated = do
 			sha <- Git.hashObject g $ unionmerge content
 			return $ Just $ ls_tree_line sha file
 
-{- Commits the index into the specified branch, as a merge commit. -}
-commit :: Git.Repo -> String -> String -> String -> IO ()
-commit g aref bref newref = do
+{- Commits the index into the specified branch. If refs are specified,
+ - commits a merge. -}
+commit :: Git.Repo -> String -> String -> [String] -> IO ()
+commit g message newref mergedrefs = do
 	tree <- Git.getSha "write-tree" $ ignorehandle $
 		pipeFrom "git" ["write-tree"]
 	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
-		pipeBoth "git" ["commit-tree", tree, "-p", aref, "-p", bref]
-			"union merge"
+		pipeBoth "git" (["commit-tree", tree] ++ ps) message
 	Git.run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = return . snd =<< a
+		ps = concatMap (\r -> ["-p", r]) mergedrefs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e8ac0a0c54..f02db6be3d 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -10,7 +10,7 @@ import System.FilePath
 import System.Directory
 import Control.Monad (when)
 
-import GitUnionMerge
+import qualified GitUnionMerge
 import qualified GitRepo as Git
 
 header :: String
@@ -44,5 +44,5 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	Git.useIndex (tmpIndex g)
 	setup g
-	unionMerge g aref bref newref False
+	GitUnionMerge.merge g aref bref newref False
 	cleanup g

From 5e0adb26375413aaeef7c41dd72e8761e3cc1ada Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 19:11:55 -0400
Subject: [PATCH 1884/8313] fixes

make commit commit to the right branch

when getting content from the branch, update first
---
 Branch.hs | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 9152a03250..82ff5fcad7 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -7,7 +7,9 @@
 
 module Branch (
 	update,
-	change
+	get,
+	change,
+	commit
 ) where
 
 import Control.Monad (unless)
@@ -17,8 +19,8 @@ import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
 
-import GitUnionMerge
 import qualified GitRepo as Git
+import qualified GitUnionMerge
 import qualified Annex
 import Utility
 import Types
@@ -91,7 +93,7 @@ updateRef ref
 
 {- Stages the content of a file into the branch's index. -}
 change :: FilePath -> String -> Annex ()
-change file content = update >> do
+change file content = do
 	g <- Annex.gitRepo
 	sha <- liftIO $ Git.hashObject g content
 	withIndex $ liftIO $ Git.run g "update-index"
@@ -102,17 +104,15 @@ change file content = update >> do
 commit :: String -> Annex ()
 commit message = withIndex $ do
 	g <- Annex.gitRepo
-	-- It would be expensive to check if anything needs to be
-	-- committed, so --allow-empty is used.
-	liftIO $ Git.run g "commit"
-		[Param "--allow-empty", Param "-m", Param message]
+	liftIO $ GitUnionMerge.commit g message branch []
 
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
 get :: FilePath -> Annex String
-get file = withIndex $ do
-	g <- Annex.gitRepo
-	liftIO $ catch (cat g) (const $ return "")
+get file = update >> do
+	withIndex $ do
+		g <- Annex.gitRepo
+		liftIO $ catch (cat g) (const $ return "")
 	where
 		-- To avoid stderr from cat-file when file does not exist,
 		-- first run it with -e to check that it exists.

From 7a693394f4acd05d1dedbaffec0762007a954d9e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 19:52:40 -0400
Subject: [PATCH 1885/8313] allow for union merges between a tree and the
 content in the index

This is needed for robust handling of the git-annex branch. Since changes
are staged to its index as git-annex runs, and committed at the end,
it's possible that git-annex is interrupted, and leaves a dirty index.

When it next runs, it needs to be able to merge the git-annex branch
as necessary, without losing the existing changes in the index.

Note that this assumes that the git-annex branch is only modified by
git-annex. Any changes to it will be lost when git-annex updates the
branch. I don't see a good, inexpensive way to find changes in
the git-annex branch that arn't in the index, and union merging the
git-annex branch into the index every time would likewise be expensive.
---
 Branch.hs          | 31 ++++++++++++------
 GitUnionMerge.hs   | 79 +++++++++++++++++++++++++++-------------------
 git-union-merge.hs |  3 +-
 3 files changed, 70 insertions(+), 43 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 82ff5fcad7..9d7b1b0941 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -12,12 +12,13 @@ module Branch (
 	commit
 ) where
 
-import Control.Monad (unless)
+import Control.Monad (unless, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
+import Data.Maybe
 
 import qualified GitRepo as Git
 import qualified GitUnionMerge
@@ -72,14 +73,18 @@ update = do
 	updated <- Annex.getState Annex.updated
 	unless updated $ withIndex $ do
 		g <- Annex.gitRepo
-		refs <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-		mapM_ updateRef $ map (last . words) (lines refs)
+		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+		let refs = map (last . words) (lines r)
+		updated <- catMaybes `liftM` mapM updateRef refs
+		unless (null updated) $ liftIO $
+			GitUnionMerge.commit g "update" fullname
+				(fullname:updated)
 		Annex.changeState $ \s -> s { Annex.updated = True }
 
-{- Ensures that a given ref has been merged into the local git-annex branch. -}
-updateRef :: String -> Annex ()
+{- Ensures that a given ref has been merged into the index. -}
+updateRef :: String -> Annex (Maybe String)
 updateRef ref
-	| ref == fullname = return ()
+	| ref == fullname = return Nothing
 	| otherwise = do
 		g <- Annex.gitRepo
 		diffs <- liftIO $ Git.pipeRead g [
@@ -87,9 +92,15 @@ updateRef ref
 			Param (name++".."++ref),
 			Params "--oneline -n1"
 			]
-		unless (null diffs) $ do
-			showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
-			liftIO $ unionMerge g fullname ref fullname True
+		if (null diffs)
+			then return Nothing
+			else do
+				showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
+				-- By passing only one ref, it is actually
+				-- merged into the index, preserving any
+				-- changes that may already be staged.
+				liftIO $ GitUnionMerge.merge g [ref]
+				return $ Just ref
 
 {- Stages the content of a file into the branch's index. -}
 change :: FilePath -> String -> Annex ()
@@ -104,7 +115,7 @@ change file content = do
 commit :: String -> Annex ()
 commit message = withIndex $ do
 	g <- Annex.gitRepo
-	liftIO $ GitUnionMerge.commit g message branch []
+	liftIO $ GitUnionMerge.commit g message fullname []
 
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index bc12cbe275..82f01cc0ff 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -7,7 +7,6 @@
 
 module GitUnionMerge (
 	merge,
-	stage,
 	commit
 ) where
 
@@ -19,38 +18,54 @@ import Data.String.Utils
 import qualified GitRepo as Git
 import Utility
 
-{- Performs a union merge. Should be run with a temporary index file
- - configured by Git.useIndex.
+{- Performs a union merge between two branches, staging it in the index.
+ - Any previously staged changes in the index will be lost.
  -
- - Use indexpopulated only if the index file already contains exactly the
- - contents of aref.
+ - When only one branch is specified, it is merged into the index.
+ - In this case, previously staged changes in the index are preserved.
+ -
+ - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Git.Repo -> String -> String -> String -> Bool -> IO ()
-merge g aref bref newref indexpopulated = do
-	stage g aref bref indexpopulated
-	commit g "union merge" newref [aref, bref]
+merge :: Git.Repo -> [String] -> IO ()
+merge g (x:y:[]) = do
+	a <- ls_tree g x
+	b <- merge_trees g x y
+	update_index g (a++b)
+merge g [x] = merge_tree_index g x >>= update_index g
+merge _ _ = error "wrong number of branches to merge"
 
-{- Stages the content of both refs into the index. -}
-stage :: Git.Repo -> String -> String -> Bool -> IO ()
-stage g aref bref indexpopulated = do
-	-- Get the contents of aref, as a starting point, unless
-	-- the index is already populated with it.
-	ls <- if indexpopulated
-		then return []
-		else fromgit ["ls-tree", "-z", "-r", "--full-tree", aref]
-	-- Identify files that are different between aref and bref, and
-	-- inject merged versions into git.
-	diff <- fromgit
-		["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", aref, bref]
-	ls' <- mapM mergefile (pairs diff)
-	-- Populate the index file. Later lines override earlier ones.
-	togit ["update-index", "-z", "--index-info"]
-		(join "\0" $ ls++catMaybes ls')
+{- Feeds a list into update-index. Later items in the list can override
+ - earlier ones, so the list can be generated from any combination of
+ - ls_tree, merge_trees, and merge_tree. -}
+update_index :: Git.Repo -> [String] -> IO ()
+update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		fromgit l = Git.pipeNullSplit g (map Param l)
-		togit l content = Git.pipeWrite g (map Param l) content
+		togit ps content = Git.pipeWrite g (map Param ps) content
 			>>= forceSuccess
 
+{- Gets the contents of a tree in a format suitable for update_index. -}
+ls_tree :: Git.Repo -> String -> IO [String]
+ls_tree g x = Git.pipeNullSplit g $ 
+	map Param ["ls-tree", "-z", "-r", "--full-tree", x]
+
+{- For merging two trees. -}
+merge_trees :: Git.Repo -> String -> String -> IO [String]
+merge_trees g x y = calc_merge g
+	["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", x, y]
+
+{- For merging a single tree into the index. -}
+merge_tree_index :: Git.Repo -> String -> IO [String]
+merge_tree_index g x = calc_merge g
+	["diff-index", "--raw", "-z", "-r", "--no-renames", "-l0", x]
+
+{- Calculates how to perform a merge, using git to get a raw diff,
+ - and returning a list suitable for update_index. -}
+calc_merge :: Git.Repo -> [String] -> IO [String]
+calc_merge g differ = do
+	diff <- Git.pipeNullSplit g $ map Param differ
+	l <- mapM mergefile (pairs diff)
+	return $ catMaybes l
+	where
 		pairs [] = []
 		pairs (_:[]) = error "parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
@@ -62,7 +77,7 @@ stage g aref bref indexpopulated = do
 		mergefile (info, file) = do
 			let [_colonamode, _bmode, asha, bsha, _status] = words info
 			if bsha == nullsha
-				then return Nothing -- already staged from aref
+				then return Nothing -- already staged
 				else mergefile' file asha bsha
 		mergefile' file asha bsha = do
 			let shas = filter (/= nullsha) [asha, bsha]
@@ -70,10 +85,10 @@ stage g aref bref indexpopulated = do
 			sha <- Git.hashObject g $ unionmerge content
 			return $ Just $ ls_tree_line sha file
 
-{- Commits the index into the specified branch. If refs are specified,
- - commits a merge. -}
+{- Commits the index into the specified branch, 
+ - with the specified parent refs. -}
 commit :: Git.Repo -> String -> String -> [String] -> IO ()
-commit g message newref mergedrefs = do
+commit g message newref parentrefs = do
 	tree <- Git.getSha "write-tree" $ ignorehandle $
 		pipeFrom "git" ["write-tree"]
 	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
@@ -81,4 +96,4 @@ commit g message newref mergedrefs = do
 	Git.run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = return . snd =<< a
-		ps = concatMap (\r -> ["-p", r]) mergedrefs
+		ps = concatMap (\r -> ["-p", r]) parentrefs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index f02db6be3d..7c0c1cd843 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -44,5 +44,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	Git.useIndex (tmpIndex g)
 	setup g
-	GitUnionMerge.merge g aref bref newref False
+	GitUnionMerge.merge g [aref, bref]
+	GitUnionMerge.commit g "union merge" newref [aref, bref]
 	cleanup g

From 818ae0c6da1d74e02d74e15b8832a58ca9514fef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Jun 2011 20:21:33 -0400
Subject: [PATCH 1886/8313] docs for v3

---
 debian/NEWS      | 16 ++++++++++++++++
 debian/changelog |  2 ++
 2 files changed, 18 insertions(+)

diff --git a/debian/NEWS b/debian/NEWS
index a8b258bfe4..4e085bb002 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,3 +1,19 @@
+git-annex (0.20110622) unstable; urgency=low
+
+  There has been another change to the git-annex data store.
+  Use `git annex upgrade` to migrate your repositories to the new
+  layout.
+
+  The significant change this time is that the .git-annex/ directory
+  is gone; instead there is a git-annex branch that is automatically
+  maintained by git-annex, and encapsulates all its state nicely out
+  of your way.
+
+  You should make sure you include the git-annex branch when
+  git pushing and pulling.
+
+ -- Joey Hess   Tue, 21 Jun 2011 20:18:00 -0400
+
 git-annex (0.20110316) experimental; urgency=low
 
   This version reorganises the layout of git-annex's files in your repository.
diff --git a/debian/changelog b/debian/changelog
index 6439eb89fa..360121cbf7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,7 @@
 git-annex (0.20110611) UNRELEASED; urgency=low
 
+  * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
+
   * rsync is now used when copying files from repos on other filesystems.
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems

From 1cca8b4edb963b980e64ed0b7de7814b5380e214 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 13:59:42 -0400
Subject: [PATCH 1887/8313] rework core merge code

More likely to be 100% correct now, I think.
---
 GitUnionMerge.hs | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 82f01cc0ff..267376ed57 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -36,7 +36,7 @@ merge _ _ = error "wrong number of branches to merge"
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
- - ls_tree, merge_trees, and merge_tree. -}
+ - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Git.Repo -> [String] -> IO ()
 update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
@@ -63,27 +63,29 @@ merge_tree_index g x = calc_merge g
 calc_merge :: Git.Repo -> [String] -> IO [String]
 calc_merge g differ = do
 	diff <- Git.pipeNullSplit g $ map Param differ
-	l <- mapM mergefile (pairs diff)
+	l <- mapM (mergeFile g) (pairs diff)
 	return $ catMaybes l
 	where
 		pairs [] = []
-		pairs (_:[]) = error "parse error"
+		pairs (_:[]) = error "calc_merge parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
-		
+
+{- Given an info line from a git raw diff, and the filename, generates
+ - a line suitable for update_index that union merges the two sides of the
+ - diff. -}
+mergeFile :: Git.Repo -> (String, FilePath) -> IO (Maybe String)
+mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
+	[] -> return Nothing
+	(sha:[]) -> return $ Just $ ls_tree_line sha
+	shas -> do
+		content <- Git.pipeRead g $ map Param ("show":shas)
+		sha <- Git.hashObject g $ unionmerge content
+		return $ Just $ ls_tree_line sha
+	where
+		[_colonamode, _bmode, asha, bsha, _status] = words info
+		ls_tree_line sha = "100644 blob " ++ sha ++ "\t" ++ file
 		nullsha = take Git.shaSize $ repeat '0'
-		ls_tree_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 		unionmerge = unlines . nub . lines
-		
-		mergefile (info, file) = do
-			let [_colonamode, _bmode, asha, bsha, _status] = words info
-			if bsha == nullsha
-				then return Nothing -- already staged
-				else mergefile' file asha bsha
-		mergefile' file asha bsha = do
-			let shas = filter (/= nullsha) [asha, bsha]
-			content <- Git.pipeRead g $ map Param ("show":shas)
-			sha <- Git.hashObject g $ unionmerge content
-			return $ Just $ ls_tree_line sha file
 
 {- Commits the index into the specified branch, 
  - with the specified parent refs. -}

From 78a325b09315efd593e6b729de18f15871a0d643 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 14:18:49 -0400
Subject: [PATCH 1888/8313] add a small cache of the most recently accessed
 item from the git-annex branch

This will speed up typical cases like git-annex get, which currently
has to read the location log once, then read it a second time in order to
add a line to it. Since these reads now involve more than just reading
in a file, it seemed good to add a cache layer.

Only the most recent thing needs to be cached, because git-annex has
good locality; it operates on one file at a time, and only cares
about one item from the branch per file.
---
 Annex.hs        |  7 +++++--
 Branch.hs       | 21 +++++++++++++++++----
 Types/Branch.hs | 16 ++++++++++++++++
 3 files changed, 38 insertions(+), 6 deletions(-)
 create mode 100644 Types/Branch.hs

diff --git a/Annex.hs b/Annex.hs
index bede0cbfb7..b6834d6dd7 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -23,6 +23,7 @@ import GitQueue
 import Types.Backend
 import Types.Remote
 import Types.Crypto
+import Types.Branch
 import TrustLevel
 import Types.UUID
 
@@ -39,7 +40,8 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
-	, updated :: Bool
+	, branchupdated :: Bool
+	, branchcache :: BranchCache
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -60,7 +62,8 @@ newState allbackends gitrepo = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
-	, updated = False
+	, branchupdated = False
+	, branchcache = emptyBranchCache
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
index 9d7b1b0941..442f47ed53 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -20,6 +20,7 @@ import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
 
+import Types.Branch
 import qualified GitRepo as Git
 import qualified GitUnionMerge
 import qualified Annex
@@ -66,11 +67,19 @@ withIndex a = do
 	liftIO $ Git.useDefaultIndex
 	return r
 
+{- There is a small cache of the most recently accessed item from the
+ - branch. git-annex has good locality, so that is enough. -}
+setCache :: FilePath -> String -> Annex ()
+setCache file content = Annex.changeState $ \s -> s { Annex.branchcache = BranchCache (Just file) content }
+
+invalidateCache :: Annex ()
+invalidateCache = Annex.changeState $ \s -> s { Annex.branchcache = emptyBranchCache }
+
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
-	updated <- Annex.getState Annex.updated
+	updated <- Annex.getState Annex.branchupdated
 	unless updated $ withIndex $ do
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
@@ -79,7 +88,8 @@ update = do
 		unless (null updated) $ liftIO $
 			GitUnionMerge.commit g "update" fullname
 				(fullname:updated)
-		Annex.changeState $ \s -> s { Annex.updated = True }
+		Annex.changeState $ \s -> s { Annex.branchupdated = True }
+		invalidateCache
 
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: String -> Annex (Maybe String)
@@ -108,8 +118,9 @@ change file content = do
 	g <- Annex.gitRepo
 	sha <- liftIO $ Git.hashObject g content
 	withIndex $ liftIO $ Git.run g "update-index"
-		[ Params "--add --cacheinfo 100644 ",
+		[ Param "--add", Param "--cacheinfo", Param "100644",
 		  Param sha, File file]
+	setCache file content
 
 {- Commits staged changes to the branch. -}
 commit :: String -> Annex ()
@@ -123,7 +134,9 @@ get :: FilePath -> Annex String
 get file = update >> do
 	withIndex $ do
 		g <- Annex.gitRepo
-		liftIO $ catch (cat g) (const $ return "")
+		content <- liftIO $ catch (cat g) (const $ return "")
+		setCache file content
+		return content
 	where
 		-- To avoid stderr from cat-file when file does not exist,
 		-- first run it with -e to check that it exists.
diff --git a/Types/Branch.hs b/Types/Branch.hs
new file mode 100644
index 0000000000..c0ccb5ca03
--- /dev/null
+++ b/Types/Branch.hs
@@ -0,0 +1,16 @@
+{- git-annex branch data types
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.Branch where
+
+data BranchCache = BranchCache {
+	cachedFile :: Maybe FilePath,
+	cachedContent :: String
+}
+
+emptyBranchCache :: BranchCache
+emptyBranchCache = BranchCache Nothing ""

From d3f0106f2ed15a4e4abbc09cc3e985a27dfee662 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 14:27:50 -0400
Subject: [PATCH 1889/8313] move LocationLog into Annex monad from IO

It will need to run in Annex so it can use Branch
---
 Backend/File.hs    |  5 ++---
 Command/Fsck.hs    |  2 +-
 Command/Unused.hs  |  4 ++--
 Command/Whereis.hs |  4 +---
 Content.hs         |  2 +-
 LocationLog.hs     | 25 +++++++++++++------------
 Remote.hs          |  2 +-
 Upgrade/V1.hs      |  6 +++---
 test.hs            |  2 +-
 9 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 386af02663..20cb3e95ad 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -14,7 +14,6 @@
 
 module Backend.File (backend, checkKey) where
 
-import Control.Monad.State (liftIO)
 import Data.List
 import Data.String.Utils
 
@@ -132,7 +131,7 @@ showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	uuids <- liftIO $ keyLocations g key
+	uuids <- keyLocations g key
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
@@ -190,7 +189,7 @@ checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
 	g <- Annex.gitRepo
-	locations <- liftIO $ keyLocations g key
+	locations <- keyLocations g key
 	untrusted <- trustGet UnTrusted
 	let untrustedlocations = intersect untrusted locations
 	let safelocations = filter (`notElem` untrusted) locations
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index adfd702de7..7c840d5288 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -60,7 +60,7 @@ verifyLocationLog key file = do
 		preventWrite (parentDir f)
 
 	u <- getUUID g
-        uuids <- liftIO $ keyLocations g key
+        uuids <- keyLocations g key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5422dad69f..4389b2209e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -68,7 +68,7 @@ checkRemoteUnused' r = do
 	showNote $ "checking for unused data..."
 	g <- Annex.gitRepo
 	referenced <- getKeysReferenced
-	logged <- liftIO $ loggedKeys g
+	logged <- loggedKeys g
 	remotehas <- filterM isthere logged
 	let remoteunused = remotehas `exclude` referenced
 	let list = number 0 remoteunused
@@ -79,7 +79,7 @@ checkRemoteUnused' r = do
 	where
 		isthere k = do
 			g <- Annex.gitRepo
-			us <- liftIO $ keyLocations g k
+			us <- keyLocations g k
 			return $ uuid `elem` us
 		uuid = Remote.uuid r
 
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 2e0fa15f6f..bcd4a2e228 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,8 +7,6 @@
 
 module Command.Whereis where
 
-import Control.Monad.State (liftIO)
-
 import qualified Annex
 import LocationLog
 import Command
@@ -31,7 +29,7 @@ start file = isAnnexed file $ \(key, _) -> do
 perform :: Key -> CommandPerform
 perform key = do
 	g <- Annex.gitRepo
-	uuids <- liftIO $ keyLocations g key
+	uuids <- keyLocations g key
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
 	if null $ uuids
diff --git a/Content.hs b/Content.hs
index 57977ce344..ccd51a553e 100644
--- a/Content.hs
+++ b/Content.hs
@@ -81,7 +81,7 @@ logStatusFor :: UUID -> Key -> LogStatus -> Annex ()
 logStatusFor u key status = do
 	g <- Annex.gitRepo
 	unless (Git.repoIsLocalBare g) $ do
-		logfile <- liftIO $ logChange g key u status
+		logfile <- logChange g key u status
 		rellogfile <- liftIO $ Git.workTreeFile g logfile
 		AnnexQueue.add "add" [Param "--"] rellogfile
 
diff --git a/LocationLog.hs b/LocationLog.hs
index b2d423cf99..1b55abfb29 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -36,6 +36,7 @@ import System.FilePath
 import qualified Data.Map as Map
 import Control.Monad (when)
 import Data.Maybe
+import Control.Monad.State (liftIO)
 
 import qualified GitRepo as Git
 import Utility
@@ -86,7 +87,7 @@ instance Read LogLine where
 
 {- Log a change in the presence of a key's value in a repository,
  - and returns the filename of the logfile. -}
-logChange :: Git.Repo -> Key -> UUID -> LogStatus -> IO FilePath
+logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex FilePath
 logChange repo key u s = do
 	when (null u) $
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
@@ -100,8 +101,8 @@ logChange repo key u s = do
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
-readLog :: FilePath -> IO [LogLine]
-readLog file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog :: FilePath -> Annex [LogLine]
+readLog file = liftIO $ catch (return . parseLog =<< readFileStrict file) (const $ return [])
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
@@ -110,18 +111,18 @@ parseLog s = filter parsable $ map read $ lines s
 		parsable l = status l /= Undefined
 
 {- Writes a set of lines to a log file -}
-writeLog :: FilePath -> [LogLine] -> IO ()
-writeLog file ls = safeWriteFile file (unlines $ map show ls)
+writeLog :: FilePath -> [LogLine] -> Annex ()
+writeLog file ls = liftIO $ safeWriteFile file (unlines $ map show ls)
 
 {- Generates a new LogLine with the current date. -}
-logNow :: LogStatus -> UUID -> IO LogLine
+logNow :: LogStatus -> UUID -> Annex LogLine
 logNow s u = do
-	now <- getPOSIXTime
+	now <- liftIO $ getPOSIXTime
 	return $ LogLine now s u
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
-keyLocations :: Git.Repo -> Key -> IO [UUID]
+keyLocations :: Git.Repo -> Key -> Annex [UUID]
 keyLocations thisrepo key = do
 	ls <- readLog $ logFile thisrepo key
 	ls' <- readLog $ logFileOld thisrepo key
@@ -155,18 +156,18 @@ mapLog m l =
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
-loggedKeys :: Git.Repo -> IO [Key]
+loggedKeys :: Git.Repo -> Annex [Key]
 loggedKeys repo = do
-	exists <- doesDirectoryExist dir
+	exists <- liftIO $ doesDirectoryExist dir
 	if exists
 		then do
 			-- 2 levels of hashing
-			levela <- dirContents dir
+			levela <- liftIO $ dirContents dir
 			levelb <- mapM tryDirContents levela
 			files <- mapM tryDirContents (concat levelb)
 			return $ catMaybes $
 				map (logFileKey . takeFileName) (concat files)
 		else return []
 	where
-		tryDirContents d = catch (dirContents d) (return . const [])
+		tryDirContents d = liftIO $ catch (dirContents d) (return . const [])
 		dir = gitStateDir repo
diff --git a/Remote.hs b/Remote.hs
index d975c2404f..2706bf20b2 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -141,7 +141,7 @@ keyPossibilities key = do
 	trusted <- trustGet Trusted
 
 	-- get uuids of all remotes that are recorded to have the key
-	uuids <- liftIO $ keyLocations g key
+	uuids <- keyLocations g key
 	let validuuids = filter (/= u) uuids
 
 	-- note that validuuids is assumed to not have dups
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 1e634e00e8..c09bd74c1c 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -134,9 +134,9 @@ moveLocationLogs = do
 				-- log files that are not checked into git,
 				-- as well as merging with already upgraded
 				-- logs that have been pulled from elsewhere
-				old <- liftIO $ readLog f
-				new <- liftIO $ readLog dest
-				liftIO $ writeLog dest (old++new)
+				old <- readLog f
+				new <- readLog dest
+				writeLog dest (old++new)
 				AnnexQueue.add "add" [Param "--"] dest
 				AnnexQueue.add "add" [Param "--"] f
 				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f
diff --git a/test.hs b/test.hs
index 498c7b6806..1eac942b1c 100644
--- a/test.hs
+++ b/test.hs
@@ -611,7 +611,7 @@ checklocationlog f expected = do
 		Just (k, _) -> do
 			uuids <- annexeval $ do
 				g <- Annex.gitRepo
-				liftIO $ LocationLog.keyLocations g k
+				LocationLog.keyLocations g k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
 				expected (thisuuid `elem` uuids)
 

From 8166facaef8357a6e74b1038c082bd86386c2ecd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 15:58:30 -0400
Subject: [PATCH 1890/8313] Branch handling improvements

Support creating the branch.

Unified branch state into a single data type.

Only commit changes when the index has been changed.
---
 Annex.hs             |   8 ++--
 Branch.hs            | 104 +++++++++++++++++++++++++++++++------------
 Types/Branch.hs      |  16 -------
 Types/BranchState.hs |  18 ++++++++
 4 files changed, 97 insertions(+), 49 deletions(-)
 delete mode 100644 Types/Branch.hs
 create mode 100644 Types/BranchState.hs

diff --git a/Annex.hs b/Annex.hs
index b6834d6dd7..2bd090e906 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -23,7 +23,7 @@ import GitQueue
 import Types.Backend
 import Types.Remote
 import Types.Crypto
-import Types.Branch
+import Types.BranchState
 import TrustLevel
 import Types.UUID
 
@@ -40,8 +40,7 @@ data AnnexState = AnnexState
 	, quiet :: Bool
 	, force :: Bool
 	, fast :: Bool
-	, branchupdated :: Bool
-	, branchcache :: BranchCache
+	, branchstate :: BranchState
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -62,8 +61,7 @@ newState allbackends gitrepo = AnnexState
 	, quiet = False
 	, force = False
 	, fast = False
-	, branchupdated = False
-	, branchcache = emptyBranchCache
+	, branchstate = startBranchState
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
index 442f47ed53..85928765d8 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -6,13 +6,14 @@
  -}
 
 module Branch (
+	create,
 	update,
 	get,
 	change,
 	commit
 ) where
 
-import Control.Monad (unless, liftM)
+import Control.Monad (unless, when, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
@@ -20,7 +21,7 @@ import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
 
-import Types.Branch
+import Types.BranchState
 import qualified GitRepo as Git
 import qualified GitUnionMerge
 import qualified Annex
@@ -45,8 +46,8 @@ index g = Git.workTree g  Git.gitDir g  "index." ++ name
  - Usually, this is only done when the index doesn't yet exist, and
  - the index is used to build up changes to be commited to the branch.
  -}
-genIndex :: FilePath -> Git.Repo -> IO ()
-genIndex f g = do
+genIndex :: Git.Repo -> IO ()
+genIndex g = do
 	ls <- Git.pipeNullSplit g $
 		map Param ["ls-tree", "-z", "-r", "--full-tree", fullname]
 	forceSuccess =<< Git.pipeWrite g
@@ -61,26 +62,71 @@ withIndex a = do
 	liftIO $ Git.useIndex f
 
 	e <- liftIO $ doesFileExist f
-	unless e $ liftIO $ genIndex f g
+	unless e $ liftIO $ genIndex g
 
 	r <- a
 	liftIO $ Git.useDefaultIndex
 	return r
 
-{- There is a small cache of the most recently accessed item from the
- - branch. git-annex has good locality, so that is enough. -}
+withIndexUpdate :: Annex a -> Annex a
+withIndexUpdate a = update >> withIndex a
+
+getState :: Annex BranchState
+getState = Annex.getState Annex.branchstate
+
+setState :: BranchState -> Annex ()
+setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
+
 setCache :: FilePath -> String -> Annex ()
-setCache file content = Annex.changeState $ \s -> s { Annex.branchcache = BranchCache (Just file) content }
+setCache file content = do
+	state <- getState
+	setState state { cachedFile = Just file, cachedContent = content }
+
+setCacheChanged :: FilePath -> String -> Annex ()
+setCacheChanged file content = do
+	state <- getState
+	setState state { cachedFile = Just file, cachedContent = content, branchChanged = True }
 
 invalidateCache :: Annex ()
-invalidateCache = Annex.changeState $ \s -> s { Annex.branchcache = emptyBranchCache }
+invalidateCache = do
+	state <- getState
+	setState state { cachedFile = Nothing, cachedContent = "" }
+
+getCache :: FilePath -> Annex (Maybe String)
+getCache file = getState >>= handle
+	where
+		handle state
+			| cachedFile state == Just file =
+				return $ Just $ cachedContent state
+			| otherwise = return Nothing
+
+{- Creates the branch, if it does not already exist. -}
+create :: Annex ()
+create = do
+	exists <- refexists fullname
+	unless exists $ do
+		g <- Annex.gitRepo
+		inorigin <- refexists origin
+		if inorigin
+			then liftIO $ Git.run g "branch" [Param name, Param origin]
+			else liftIO $ do
+				let f = index g
+				liftIO $ Git.useIndex f
+				GitUnionMerge.commit g "branch created" fullname []
+				liftIO $ Git.useDefaultIndex
+	where
+		origin = "origin/" ++ name
+		refexists ref = do
+			g <- Annex.gitRepo
+			liftIO $ Git.runBool g "show-ref"
+				[Param "--verify", Param "-q", Param ref]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
-	updated <- Annex.getState Annex.branchupdated
-	unless updated $ withIndex $ do
+	state <- Annex.getState Annex.branchstate
+	unless (branchUpdated state) $ withIndex $ do
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		let refs = map (last . words) (lines r)
@@ -88,7 +134,7 @@ update = do
 		unless (null updated) $ liftIO $
 			GitUnionMerge.commit g "update" fullname
 				(fullname:updated)
-		Annex.changeState $ \s -> s { Annex.branchupdated = True }
+		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
 {- Ensures that a given ref has been merged into the index. -}
@@ -120,27 +166,29 @@ change file content = do
 	withIndex $ liftIO $ Git.run g "update-index"
 		[ Param "--add", Param "--cacheinfo", Param "100644",
 		  Param sha, File file]
-	setCache file content
+	setCacheChanged file content
 
-{- Commits staged changes to the branch. -}
+{- Commits any staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = withIndex $ do
-	g <- Annex.gitRepo
-	liftIO $ GitUnionMerge.commit g message fullname []
+commit message = do
+	state <- getState
+	when (branchChanged state) $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $
+			GitUnionMerge.commit g message fullname [fullname]
 
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
 get :: FilePath -> Annex String
-get file = update >> do
-	withIndex $ do
-		g <- Annex.gitRepo
-		content <- liftIO $ catch (cat g) (const $ return "")
-		setCache file content
-		return content
+get file = do
+	cached <- getCache file
+	case cached of
+		Just content -> return content
+		Nothing -> withIndexUpdate $ do
+			g <- Annex.gitRepo
+			content <- liftIO $ catch (cat g) (const $ return "")
+			setCache file content
+			return content
 	where
-		-- To avoid stderr from cat-file when file does not exist,
-		-- first run it with -e to check that it exists.
-		cat g = do
-			Git.run g "cat-file" [Param "-e", catfile]
-			Git.pipeRead g [Param "cat-file", Param "blob", catfile]
+		cat g = Git.pipeRead g [Param "cat-file", Param "blob", catfile]
 		catfile = Param $ ':':file
diff --git a/Types/Branch.hs b/Types/Branch.hs
deleted file mode 100644
index c0ccb5ca03..0000000000
--- a/Types/Branch.hs
+++ /dev/null
@@ -1,16 +0,0 @@
-{- git-annex branch data types
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Types.Branch where
-
-data BranchCache = BranchCache {
-	cachedFile :: Maybe FilePath,
-	cachedContent :: String
-}
-
-emptyBranchCache :: BranchCache
-emptyBranchCache = BranchCache Nothing ""
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
new file mode 100644
index 0000000000..65d0642a14
--- /dev/null
+++ b/Types/BranchState.hs
@@ -0,0 +1,18 @@
+{- git-annex BranchState data type
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.BranchState where
+
+data BranchState = BranchState {
+	branchUpdated :: Bool,
+	branchChanged :: Bool,
+	cachedFile :: Maybe FilePath,
+	cachedContent :: String
+}
+
+startBranchState :: BranchState
+startBranchState = BranchState False False Nothing ""

From 17a09fccad3cf2f958967081326b61c403b995cd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:00:04 -0400
Subject: [PATCH 1891/8313] commit changes to git-annex branch on shutdown

---
 CmdLine.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CmdLine.hs b/CmdLine.hs
index 861a31be97..a5bda695ab 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,6 +19,7 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified GitRepo as Git
+import qualified Branch
 import Types
 import Command
 import BackendList
@@ -102,6 +103,7 @@ startup = do
 {- Cleanup actions. -}
 shutdown :: Annex Bool
 shutdown = do
+	Branch.commit "update"
 	AnnexQueue.flush False
 
 	liftIO $ Git.reap

From 06c58922bdb26da34a4070c84968407350f1e3c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:00:32 -0400
Subject: [PATCH 1892/8313] stop changing gitattributes on update from v1

gitattributes changes are not needed, and will be removed in the v2 upgrade
---
 Upgrade/V1.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c09bd74c1c..b139e2820b 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -30,7 +30,6 @@ import Backend
 import Messages
 import Version
 import Utility
-import qualified Command.Init
 
 -- v2 adds hashing of filenames of content and location log files.
 -- Key information is encoded in filenames differently, so
@@ -72,10 +71,6 @@ upgrade = do
 			AnnexQueue.flush True
 			setVersion
 
-			-- add new line to auto-merge hashed location logs
-			-- this commits, so has to come after the upgrade
-			liftIO $ Command.Init.gitAttributesWrite g
-
 	return True
 
 moveContent :: Annex ()

From 2e5c8ca6bf016058f2aba4eaba5b89955e3e0e95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:01:32 -0400
Subject: [PATCH 1893/8313] use git-annex branch for location log

---
 Content.hs     |  4 +---
 LocationLog.hs | 24 +++++++++---------------
 2 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/Content.hs b/Content.hs
index ccd51a553e..fc8ebeb214 100644
--- a/Content.hs
+++ b/Content.hs
@@ -81,9 +81,7 @@ logStatusFor :: UUID -> Key -> LogStatus -> Annex ()
 logStatusFor u key status = do
 	g <- Annex.gitRepo
 	unless (Git.repoIsLocalBare g) $ do
-		logfile <- logChange g key u status
-		rellogfile <- liftIO $ Git.workTreeFile g logfile
-		AnnexQueue.add "add" [Param "--"] rellogfile
+		logChange g key u status
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 
diff --git a/LocationLog.hs b/LocationLog.hs
index 1b55abfb29..68a1eb7907 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -3,16 +3,12 @@
  - git-annex keeps track of which repositories have the contents of annexed
  - files.
  -
- - Location tracking information is stored in `.git-annex/key.log`.
  - Repositories record their UUID and the date when they --get or --drop
  - a value.
  -
  - A line of the log will look like: "date N UUID"
  - Where N=1 when the repo has the file, and 0 otherwise.
  - 
- - Git is configured to use a union merge for this file,
- - so the lines may be in arbitrary order, but it will never conflict.
- -
  - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
@@ -39,6 +35,7 @@ import Data.Maybe
 import Control.Monad.State (liftIO)
 
 import qualified GitRepo as Git
+import qualified Branch
 import Utility
 import UUID
 import Types
@@ -85,24 +82,21 @@ instance Read LogLine where
 			bad = ret $ LogLine 0 Undefined ""
 			ret v = [(v, "")]
 
-{- Log a change in the presence of a key's value in a repository,
- - and returns the filename of the logfile. -}
-logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex FilePath
+{- Log a change in the presence of a key's value in a repository. -}
+logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
 logChange repo key u s = do
 	when (null u) $
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
 			" (have you run git annex init there?)"
 	line <- logNow s u
 	let f = logFile repo key
-	ls' <- readLog $ logFileOld repo key
 	ls <- readLog f
-	writeLog f (compactLog $ line:ls'++ls)
-	return f
+	writeLog f (compactLog $ line:ls)
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = liftIO $ catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog file = return . parseLog =<< Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
@@ -110,9 +104,9 @@ parseLog s = filter parsable $ map read $ lines s
 		-- some lines may be unparseable, avoid them
 		parsable l = status l /= Undefined
 
-{- Writes a set of lines to a log file -}
+{- Stores a set of lines in a log file -}
 writeLog :: FilePath -> [LogLine] -> Annex ()
-writeLog file ls = liftIO $ safeWriteFile file (unlines $ map show ls)
+writeLog file ls = Branch.change file (unlines $ map show ls)
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> UUID -> Annex LogLine
@@ -125,8 +119,7 @@ logNow s u = do
 keyLocations :: Git.Repo -> Key -> Annex [UUID]
 keyLocations thisrepo key = do
 	ls <- readLog $ logFile thisrepo key
-	ls' <- readLog $ logFileOld thisrepo key
-	return $ map uuid $ filterPresent $ ls'++ls
+	return $ map uuid $ filterPresent ls
 
 {- Filters the list of LogLines to find ones where the value
  - is (or should still be) present. -}
@@ -158,6 +151,7 @@ mapLog m l =
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Git.Repo -> Annex [Key]
 loggedKeys repo = do
+	error "FIXME.. does not look in git-annex branch yet"
 	exists <- liftIO $ doesDirectoryExist dir
 	if exists
 		then do

From ae2be332d463bb2942c5951ffa4acf4d32d63ce2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:02:07 -0400
Subject: [PATCH 1894/8313] add runBool

---
 GitRepo.hs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/GitRepo.hs b/GitRepo.hs
index 91ddf6dca9..d1f122fba5 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -37,6 +37,7 @@ module GitRepo (
 	configTrue,
 	gitCommandLine,
 	run,
+	runBool,
 	pipeRead,
 	pipeWrite,
 	pipeWriteRead,
@@ -350,10 +351,15 @@ gitCommandLine repo@(Repo { location = Dir d} ) params =
 	] ++ params
 gitCommandLine repo _ = assertLocal repo $ error "internal"
 
+{- Runs git in the specified repo. -}
+runBool :: Repo -> String -> [CommandParam] -> IO Bool
+runBool repo subcommand params = assertLocal repo $
+	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: Repo -> String -> [CommandParam] -> IO ()
 run repo subcommand params = assertLocal repo $
-	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+	runBool repo subcommand params
 		>>! error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns its output, lazily. 

From 80274f4c92397a88c62bf82459fe0c1a9bf03bf7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:02:22 -0400
Subject: [PATCH 1895/8313] use git-annex branch for uuid.log

---
 UUID.hs | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/UUID.hs b/UUID.hs
index f222f7a9d4..c02f51480c 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -30,11 +30,10 @@ import qualified Data.Map as M
 import Data.Maybe
 
 import qualified GitRepo as Git
+import qualified Branch
 import Types
 import Types.UUID
-import Locations
 import qualified Annex
-import Utility
 import qualified SysConfig
 import Config
 
@@ -103,26 +102,21 @@ describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
 	m <- uuidMap
 	let m' = M.insert uuid desc m
-	logfile <- uuidLog
-	liftIO $ safeWriteFile logfile (serialize m')
+	Branch.change uuidLog (serialize m')
 	where
 		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
 
 {- Read and parse the uuidLog into a Map -}
 uuidMap :: Annex (M.Map UUID String)
 uuidMap = do
-	logfile <- uuidLog
-	s <- liftIO $ catch (readFile logfile) ignoreerror
+	s <- Branch.get uuidLog
 	return $ M.fromList $ map pair $ lines s
 	where
 		pair l =
 			if 1 < length (words l)
 				then (head $ words l, unwords $ drop 1 $ words l)
 				else ("", "")
-		ignoreerror _ = return ""
 
 {- Filename of uuid.log. -}
-uuidLog :: Annex FilePath
-uuidLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "uuid.log"
+uuidLog :: FilePath
+uuidLog = "uuid.log"

From 5c706d1ec48172f98e1826684ab380a69079b66a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:02:43 -0400
Subject: [PATCH 1896/8313] stop undoing gitattributes on uninit

v2 upgrade will undo them
---
 Command/Uninit.hs | 11 -----------
 Upgrade/V2.hs     | 48 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 11 deletions(-)
 create mode 100644 Upgrade/V2.hs

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 1e96e1e6f7..9698ed8200 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -34,10 +34,7 @@ start = do
 perform :: CommandPerform
 perform = do
 	g <- Annex.gitRepo
-
 	gitPreCommitHookUnWrite g
-	liftIO $ gitAttributesUnWrite g
-
 	next $ return True
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
@@ -50,11 +47,3 @@ gitPreCommitHookUnWrite repo = do
 			else warning $ "pre-commit hook (" ++ hook ++ 
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
-
-gitAttributesUnWrite :: Git.Repo -> IO ()
-gitAttributesUnWrite repo = do
-	let attributes = Git.attributes repo
-	whenM (doesFileExist attributes) $ do
-		c <- readFileStrict attributes
-		safeWriteFile attributes $ unlines $
-			filter (\l -> not $ l `elem` Command.Init.attrLines) $ lines c
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
new file mode 100644
index 0000000000..deb231d529
--- /dev/null
+++ b/Upgrade/V2.hs
@@ -0,0 +1,48 @@
+{- git-annex v2 -> v2 upgrade support
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Upgrade.V1 where
+
+import System.IO.Error (try)
+import System.Directory
+import Control.Monad.State (liftIO)
+import Control.Monad (filterM, forM_, unless)
+import System.Posix.Files
+import System.FilePath
+import Data.String.Utils
+import System.Posix.Types
+import Data.Maybe
+import Data.Char
+
+import Types.Key
+import Content
+import Types
+import Locations
+import LocationLog
+import qualified Annex
+import qualified AnnexQueue
+import qualified GitRepo as Git
+import Backend
+import Messages
+import Version
+import Utility
+import qualified Command.Init
+
+{- Old .gitattributes contents, not needed anymore. -}
+attrLines :: [String]
+attrLines =
+	[ stateDir  "*.log merge=union"
+	, stateDir  "*/*/*.log merge=union"
+	]
+
+gitAttributesUnWrite :: Git.Repo -> IO ()
+gitAttributesUnWrite repo = do
+	let attributes = Git.attributes repo
+	whenM (doesFileExist attributes) $ do
+		c <- readFileStrict attributes
+		safeWriteFile attributes $ unlines $
+			filter (\l -> not $ l `elem` attrLines) $ lines c

From e0bd9d43a21bae8193cb0a56be2246ee8cdafdaa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:03:26 -0400
Subject: [PATCH 1897/8313] update for git-annex branch

stop changing gitattributes on init

create git-annex branch on init

ugly special case for init in a bare repository goes away, yay!

git annex init is also faster, at least in a large existing repo, as
it does not need to run the slow 'git add'
---
 Command/Describe.hs |  3 +--
 Command/Init.hs     | 62 ++++-----------------------------------------
 2 files changed, 6 insertions(+), 59 deletions(-)

diff --git a/Command/Describe.hs b/Command/Describe.hs
index 57f884e037..2ad3e01ee5 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -11,7 +11,6 @@ import Command
 import qualified Remote
 import UUID
 import Messages
-import qualified Command.Init
 
 command :: [Command]
 command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
@@ -34,4 +33,4 @@ start ws = notBareRepo $ do
 perform :: UUID -> String -> CommandPerform
 perform u description = do
 	describeUUID u description
-	next $ Command.Init.cleanup
+	next $ return True
diff --git a/Command/Init.hs b/Command/Init.hs
index b7a0787991..1e1eb527a5 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -15,6 +15,7 @@ import System.FilePath
 import Command
 import qualified Annex
 import qualified GitRepo as Git
+import qualified Branch
 import UUID
 import Version
 import Messages
@@ -29,7 +30,6 @@ command = [repoCommand "init" paramDesc seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-{- Stores description for the repository etc. -}
 start :: CommandStartWords
 start ws = do
 	when (null description) $
@@ -41,65 +41,13 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
+	Branch.create
 	g <- Annex.gitRepo
 	u <- getUUID g
 	setVersion
-	if Git.repoIsLocalBare g
-		then do
-			showLongNote $
-				"This is a bare repository, so its description cannot be committed.\n" ++
-				"To record the description, run this command in a clone of this repository:\n" ++
-				"   git annex describe " ++ show u ++ " " ++ show description ++ "\n\n"
-			next $ return True
-		else do
-			describeUUID u description
-			liftIO $ gitAttributesWrite g
-			gitPreCommitHookWrite g
-			next cleanup
-
-cleanup :: CommandCleanup
-cleanup = do
-	g <- Annex.gitRepo
-	logfile <- uuidLog
-	liftIO $ Git.run g "add" [File logfile]
-	liftIO $ Git.run g "commit" 
-		[ Params "-q -m"
-		, Param "git annex repository description"
-		, File logfile
-		]
-	return True
-
-{- configure git to use union merge driver on state files, if it is not
- - already -}
-gitAttributesWrite :: Git.Repo -> IO ()
-gitAttributesWrite repo = do
-	exists <- doesFileExist attributes
-	if not exists
-		then do
-			safeWriteFile attributes $ unlines attrLines
-			commit
-		else do
-			content <- readFile attributes
-			let present = lines content
-			let missing = filter (\l -> not $ l `elem` present) attrLines
-			unless (null missing) $ do
-				appendFile attributes $ unlines missing
-				commit
-	where
-		attributes = Git.attributes repo
-		commit = do
-			Git.run repo "add" [Param attributes]
-			Git.run repo "commit"
-				[ Params "-q -m"
-				, Param "git-annex setup"
-				, Param attributes
-				]
-
-attrLines :: [String]
-attrLines =
-	[ stateDir  "*.log merge=union"
-	, stateDir  "*/*/*.log merge=union"
-	]
+	describeUUID u description
+	gitPreCommitHookWrite g
+	next $ return True
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()

From 1870186632e3d4f99e9b87f71f0ddea83ad04568 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:13:43 -0400
Subject: [PATCH 1898/8313] fixed logFile

---
 Backend/File.hs    |  5 ++---
 Command/Fsck.hs    |  2 +-
 Command/Init.hs    |  4 +---
 Command/Unused.hs  |  3 +--
 Command/Whereis.hs |  4 +---
 Content.hs         |  1 -
 LocationLog.hs     | 13 ++++++-------
 Locations.hs       | 16 ++--------------
 Remote.hs          |  2 +-
 Upgrade/V1.hs      |  2 +-
 Upgrade/V2.hs      | 10 ++++++++++
 11 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 20cb3e95ad..eab987ef81 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -131,7 +131,7 @@ showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	uuids <- keyLocations g key
+	uuids <- keyLocations key
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
@@ -188,8 +188,7 @@ checkKeyOnly = checkKey (\_ -> return True)
 checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
-	g <- Annex.gitRepo
-	locations <- keyLocations g key
+	locations <- keyLocations key
 	untrusted <- trustGet UnTrusted
 	let untrustedlocations = intersect untrusted locations
 	let safelocations = filter (`notElem` untrusted) locations
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 7c840d5288..0e3df03dd0 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -60,7 +60,7 @@ verifyLocationLog key file = do
 		preventWrite (parentDir f)
 
 	u <- getUUID g
-        uuids <- keyLocations g key
+        uuids <- keyLocations key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
diff --git a/Command/Init.hs b/Command/Init.hs
index 1e1eb527a5..dbf5666cd9 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -8,9 +8,8 @@
 module Command.Init where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
+import Control.Monad (when)
 import System.Directory
-import System.FilePath
 
 import Command
 import qualified Annex
@@ -19,7 +18,6 @@ import qualified Branch
 import UUID
 import Version
 import Messages
-import Locations
 import Types
 import Utility
 	
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 4389b2209e..5d4e433ad8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -78,8 +78,7 @@ checkRemoteUnused' r = do
 		showLongNote $ "\n"
 	where
 		isthere k = do
-			g <- Annex.gitRepo
-			us <- keyLocations g k
+			us <- keyLocations k
 			return $ uuid `elem` us
 		uuid = Remote.uuid r
 
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index bcd4a2e228..3a7213217a 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,7 +7,6 @@
 
 module Command.Whereis where
 
-import qualified Annex
 import LocationLog
 import Command
 import Messages
@@ -28,8 +27,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> CommandPerform
 perform key = do
-	g <- Annex.gitRepo
-	uuids <- keyLocations g key
+	uuids <- keyLocations key
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
 	if null $ uuids
diff --git a/Content.hs b/Content.hs
index fc8ebeb214..5d77cc9795 100644
--- a/Content.hs
+++ b/Content.hs
@@ -38,7 +38,6 @@ import LocationLog
 import UUID
 import qualified GitRepo as Git
 import qualified Annex
-import qualified AnnexQueue
 import Utility
 import StatFS
 import Types.Key
diff --git a/LocationLog.hs b/LocationLog.hs
index 68a1eb7907..8dbeb729ca 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -20,8 +20,7 @@ module LocationLog (
 	readLog,
 	writeLog,
 	keyLocations,
-	loggedKeys,
-	logFile
+	loggedKeys
 ) where
 
 import Data.Time.Clock.POSIX
@@ -89,7 +88,7 @@ logChange repo key u s = do
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
 			" (have you run git annex init there?)"
 	line <- logNow s u
-	let f = logFile repo key
+	let f = logFile key
 	ls <- readLog f
 	writeLog f (compactLog $ line:ls)
 
@@ -116,9 +115,9 @@ logNow s u = do
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
-keyLocations :: Git.Repo -> Key -> Annex [UUID]
-keyLocations thisrepo key = do
-	ls <- readLog $ logFile thisrepo key
+keyLocations :: Key -> Annex [UUID]
+keyLocations key = do
+	ls <- readLog $ logFile key
 	return $ map uuid $ filterPresent ls
 
 {- Filters the list of LogLines to find ones where the value
@@ -151,7 +150,7 @@ mapLog m l =
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Git.Repo -> Annex [Key]
 loggedKeys repo = do
-	error "FIXME.. does not look in git-annex branch yet"
+	_ <- error "FIXME.. does not look in git-annex branch yet"
 	exists <- liftIO $ doesDirectoryExist dir
 	if exists
 		then do
diff --git a/Locations.hs b/Locations.hs
index 8f7b11a5c2..d2241636ef 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,7 +21,6 @@ module Locations (
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
 	logFile,
-	logFileOld,
 	logFileKey,
 	hashDirMixed,
 
@@ -119,19 +118,8 @@ isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
 {- The filename of the log file for a given key. -}
-logFile :: Git.Repo -> Key -> String
-logFile = logFile' hashDirLower
-
-{- The old filename of the log file for a key. These can have mixed
- - case, which turned out to be a bad idea for directories whose contents
- - are checked into git. There was no conversion, so these have to be checked
- - for and merged in at runtime. -}
-logFileOld :: Git.Repo -> Key -> String
-logFileOld = logFile' hashDirMixed
-
-logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
-logFile' hasher repo key =
-	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+logFile :: Key -> String
+logFile key = hashDirLower key ++ keyFile key ++ ".log"
 
 {- Converts a log filename into a key. -}
 logFileKey :: FilePath -> Maybe Key
diff --git a/Remote.hs b/Remote.hs
index 2706bf20b2..804c0ef5af 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -141,7 +141,7 @@ keyPossibilities key = do
 	trusted <- trustGet Trusted
 
 	-- get uuids of all remotes that are recorded to have the key
-	uuids <- keyLocations g key
+	uuids <- keyLocations key
 	let validuuids = filter (/= u) uuids
 
 	-- note that validuuids is assumed to not have dups
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b139e2820b..1f327f77b9 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -121,7 +121,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
-				let dest = logFile g k
+				let dest = logFile k
 				let dir = gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index deb231d529..03ef7ba69d 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -46,3 +46,13 @@ gitAttributesUnWrite repo = do
 		c <- readFileStrict attributes
 		safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
+
+oldlogFile :: Git.Repo -> Key -> String
+oldlogFile = logFile' hashDirLower
+
+oldlogFileOld :: Git.Repo -> Key -> String
+oldlogFileOld = logFile' hashDirMixed
+
+logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
+logFile' hasher repo key =
+	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"

From 235e2e63a13c629dcca1aa1638f5f47a8d3983ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 16:30:34 -0400
Subject: [PATCH 1899/8313] move

---
 Locations.hs | 5 +++++
 UUID.hs      | 6 +-----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index d2241636ef..9175c4f613 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
+	uuidLog,
 	logFile,
 	logFileKey,
 	hashDirMixed,
@@ -117,6 +118,10 @@ gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
+{- Filename of uuid.log. -}
+uuidLog :: FilePath
+uuidLog = "uuid.log"
+
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
 logFile key = hashDirLower key ++ keyFile key ++ ".log"
diff --git a/UUID.hs b/UUID.hs
index c02f51480c..78667f235e 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -19,7 +19,6 @@ module UUID (
 	genUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidLog,
 	uuidMap
 ) where
 
@@ -36,6 +35,7 @@ import Types.UUID
 import qualified Annex
 import qualified SysConfig
 import Config
+import Locations
 
 configkey :: String
 configkey = "annex.uuid"
@@ -116,7 +116,3 @@ uuidMap = do
 			if 1 < length (words l)
 				then (head $ words l, unwords $ drop 1 $ words l)
 				else ("", "")
-
-{- Filename of uuid.log. -}
-uuidLog :: FilePath
-uuidLog = "uuid.log"

From 4c4ebf2d7570030a70fdbd313b8b60e9fa727eee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:08:51 -0400
Subject: [PATCH 1900/8313] store trust.log and remote.log in the git-annex
 branch

.. and I think that's everything that will use the branch
---
 Branch.hs             | 18 +++++++++---------
 CmdLine.hs            |  2 +-
 Command/InitRemote.hs | 11 -----------
 Locations.hs          |  5 -----
 Remote.hs             | 20 +++++---------------
 Trust.hs              | 27 ++++++---------------------
 UUID.hs               |  8 ++++++--
 7 files changed, 27 insertions(+), 64 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 85928765d8..ad9b805d0e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -168,15 +168,6 @@ change file content = do
 		  Param sha, File file]
 	setCacheChanged file content
 
-{- Commits any staged changes to the branch. -}
-commit :: String -> Annex ()
-commit message = do
-	state <- getState
-	when (branchChanged state) $ do
-		g <- Annex.gitRepo
-		withIndex $ liftIO $
-			GitUnionMerge.commit g message fullname [fullname]
-
 {- Gets the content of a file on the branch, or content staged in the index
  - if it's newer. Returns an empty string if the file didn't exist yet. -}
 get :: FilePath -> Annex String
@@ -192,3 +183,12 @@ get file = do
 	where
 		cat g = Git.pipeRead g [Param "cat-file", Param "blob", catfile]
 		catfile = Param $ ':':file
+
+{- Commits any staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = do
+	state <- getState
+	when (branchChanged state) $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $
+			GitUnionMerge.commit g message fullname [fullname]
diff --git a/CmdLine.hs b/CmdLine.hs
index a5bda695ab..d10516bb9c 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -103,8 +103,8 @@ startup = do
 {- Cleanup actions. -}
 shutdown :: Annex Bool
 shutdown = do
-	Branch.commit "update"
 	AnnexQueue.flush False
+	Branch.commit "update"
 
 	liftIO $ Git.reap
 	
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 41d3c37c71..67030689da 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -14,11 +14,8 @@ import Data.Maybe
 import Data.String.Utils
 
 import Command
-import qualified Annex
 import qualified Remote
 import qualified Types.Remote as R
-import qualified GitRepo as Git
-import Utility
 import Types
 import UUID
 import Messages
@@ -62,14 +59,6 @@ perform t u c = do
 cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
 	Remote.configSet u c
-	g <- Annex.gitRepo
-	logfile <- Remote.remoteLog
-	liftIO $ Git.run g "add" [File logfile]
-        liftIO $ Git.run g "commit"
-                [ Params "-q --allow-empty -m"
-                , Param "git annex initremote"
-                , File logfile
-                ]
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
diff --git a/Locations.hs b/Locations.hs
index 9175c4f613..d2241636ef 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,7 +20,6 @@ module Locations (
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
 	isLinkToAnnex,
-	uuidLog,
 	logFile,
 	logFileKey,
 	hashDirMixed,
@@ -118,10 +117,6 @@ gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
-{- Filename of uuid.log. -}
-uuidLog :: FilePath
-uuidLog = "uuid.log"
-
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
 logFile key = hashDirLower key ++ keyFile key ++ ".log"
diff --git a/Remote.hs b/Remote.hs
index 804c0ef5af..5122423421 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -33,19 +33,17 @@ module Remote (
 	prop_idempotent_configEscape
 ) where
 
-import Control.Monad.State (liftIO)
 import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
 
+import qualified Branch
 import Types
 import Types.Remote
 import UUID
 import qualified Annex
-import Locations
-import Utility
 import Config
 import Trust
 import LocationLog
@@ -160,29 +158,21 @@ forceTrust level remotename = do
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
 
 {- Filename of remote.log. -}
-remoteLog :: Annex FilePath
-remoteLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "remote.log"
+remoteLog :: FilePath
+remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
 configSet u c = do
 	m <- readRemoteLog
-	l <- remoteLog
-	liftIO $ safeWriteFile l $ unlines $ sort $
+	Branch.change remoteLog $ unlines $ sort $
 		map toline $ M.toList $ M.insert u c m
 	where
 		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = do
-	l <- remoteLog
-	s <- liftIO $ catch (readFile l) ignoreerror
-	return $ remoteLogParse s
-	where
-		ignoreerror _ = return ""
+readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Trust.hs b/Trust.hs
index aaca3b3706..d328235bfa 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -18,18 +18,14 @@ import Control.Monad.State
 import qualified Data.Map as M
 
 import TrustLevel
-import qualified GitRepo as Git
+import qualified Branch
 import Types
 import UUID
-import Locations
 import qualified Annex
-import Utility
 
 {- Filename of trust.log. -}
-trustLog :: Annex FilePath
-trustLog = do
-	g <- Annex.gitRepo
-	return $ gitStateDir g ++ "trust.log"
+trustLog :: FilePath
+trustLog = "trust.log"
 
 {- Returns a list of UUIDs at the specified trust level. -}
 trustGet :: TrustLevel -> Annex [UUID]
@@ -41,12 +37,9 @@ trustGet level = do
  - values from forcetrust -}
 trustMap :: Annex (M.Map UUID TrustLevel)
 trustMap = do
-	logfile <- trustLog
 	overrides <- Annex.getState Annex.forcetrust
-	s <- liftIO $ catch (readFile logfile) ignoreerror
+	s <- Branch.get trustLog
 	return $ M.fromList $ trustMapParse s ++ overrides
-	where
-                ignoreerror _ = return ""
 
 {- Trust map parser. -}
 trustMapParse :: String -> [(UUID, TrustLevel)]
@@ -60,7 +53,7 @@ trustMapParse s = map pair $ filter (not . null) $ lines s
 			where
 				w = words l
 
-{- Changes the trust level for a uuid in the trustLog, and commits it. -}
+{- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
@@ -68,15 +61,7 @@ trustSet uuid level = do
         m <- trustMap
 	when (M.lookup uuid m /= Just level) $ do
 		let m' = M.insert uuid level m
-	        logfile <- trustLog
-	        liftIO $ safeWriteFile logfile (serialize m')
-		g <- Annex.gitRepo
-		liftIO $ Git.run g "add" [File logfile]
-		liftIO $ Git.run g "commit"
-			[ Params "-q -m"
-			, Param "git annex trust change"
-			, File logfile
-			]
+		Branch.change trustLog (serialize m')
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
diff --git a/UUID.hs b/UUID.hs
index 78667f235e..5d8304f83d 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -19,7 +19,8 @@ module UUID (
 	genUUID,
 	prettyPrintUUIDs,
 	describeUUID,
-	uuidMap
+	uuidMap,
+	uuidLog
 ) where
 
 import Control.Monad.State
@@ -35,11 +36,14 @@ import Types.UUID
 import qualified Annex
 import qualified SysConfig
 import Config
-import Locations
 
 configkey :: String
 configkey = "annex.uuid"
 
+{- Filename of uuid.log. -}
+uuidLog :: FilePath
+uuidLog = "uuid.log"
+
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID

From d70e9a945b4ac44ff42872b08dcf09051759eb9c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:15:33 -0400
Subject: [PATCH 1901/8313] remove some tests that no longer make sense

---
 test.hs | 24 +-----------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/test.hs b/test.hs
index 1eac942b1c..528f78063a 100644
--- a/test.hs
+++ b/test.hs
@@ -105,12 +105,7 @@ blackbox = TestLabel "blackbox" $ TestList
 test_init :: Test
 test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 	git_annex "init" ["-q", reponame] @? "init failed"
-	e <- doesFileExist annexlog
-	e @? (annexlog ++ " not created")
-	c <- readFile annexlog
-	reponame `isInfixOf` c @? annexlog ++ " does not contain repo name"
 	where
-		annexlog = ".git-annex/uuid.log"
 		reponame = "test repo"
 
 test_add :: Test
@@ -609,26 +604,9 @@ checklocationlog f expected = do
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do
-			uuids <- annexeval $ do
-				g <- Annex.gitRepo
-				LocationLog.keyLocations g k
+			uuids <- annexeval $ LocationLog.keyLocations k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
 				expected (thisuuid `elem` uuids)
-
-			-- Location log files should always be checked
-			-- into git, and any modifications staged for
-			-- commit. This is a regression test, as some
-			-- commands forgot to.
-			(fs, ufs) <- annexeval $ do
-				g <- Annex.gitRepo
-				let lf = LocationLog.logFile g k
-				fs <- liftIO $ Git.inRepo g [lf]
-				ufs <- liftIO $ Git.changedUnstagedFiles g [lf]
-				return (fs, ufs)
-			when (null fs) $
-				assertFailure $ f ++ " logfile not added to git repo"
-			when (not $ null ufs) $
-				assertFailure $ f ++ " logfile changes not staged"
 		_ -> assertFailure $ f ++ " failed to look up key"
 
 inlocationlog :: FilePath -> Assertion

From b1acf41036a1eddea29cc69c6b2a595582378465 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:26:34 -0400
Subject: [PATCH 1902/8313] update documentation that mentioned .git-annex/

---
 doc/bare_repositories.mdwn                   |  4 ----
 doc/design/encryption.mdwn                   |  2 +-
 doc/git-annex.mdwn                           | 12 ------------
 doc/internals.mdwn                           | 16 ++++++++++++----
 doc/location_tracking.mdwn                   |  2 +-
 doc/walkthrough/modifying_annexed_files.mdwn |  3 +--
 6 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index a9ccab8d1b..5cbad5c2d8 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -17,10 +17,6 @@ Known to work ok:
 
 There are a few caveats to keep in mind when using bare repositories:
 
-* `git annex init` can be run in a bare repository, but it cannot
-  store the name you gave the repository in .git-annex/uuid.log (because
-  the bare repository has no such file to commit to). Instead, it will
-  tell you a command to run in some non-bare clone of the repository.
 * Some subcommands, like `fsck`, `trust`, `unused` and `fromkey`, 
   cannot be run in a bare repository. Those subcommands will
   refuse to do anything.
diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn
index 11056478bd..e5053134ee 100644
--- a/doc/design/encryption.mdwn
+++ b/doc/design/encryption.mdwn
@@ -36,7 +36,7 @@ There does not seem to be much benefit to using the same cipher for
 two different encrypted remotes.
 
 So, the encrypted cipher could just be stored with the rest of a remote's
-configuration in `.git-annex/remotes.log` (see [[internals]]). When `git
+configuration in `remotes.log` (see [[internals]]). When `git
 annex intiremote` makes a remote, it can generate a random symmetric
 cipher, and encrypt it with the specified gpg key. To allow another gpg
 public key access, update the encrypted cipher to be encrypted to both gpg
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 12756d8020..ced6fc1b3a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -497,18 +497,6 @@ These files are used by git-annex, in your git repository:
 `.git/annex/objects/` contains the annexed file contents that are currently
 available. Annexed files in your git repository symlink to that content.
 
-`.git-annex/uuid.log` is used to map between repository UUID and
-descriptions.
-
-`.git-annex/trust.log` is used to indicate which repositories are trusted
-and untrusted.
-
-`.git-annex/*.log` is where git-annex records its content tracking
-information. These files should be committed to git.
-
-`.gitattributes` is configured to use git's union merge driver
-to avoid conflicts when merging files in the `.git-annex` directory.
-
 # SEE ALSO
 
 Most of git-annex's documentation is available on its web site, 
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index b362e68e1e..419096744d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -17,7 +17,15 @@ This two-level structure is used because it allows the write bit to be removed
 from the subdirectories as well as from the files. That prevents accidentially
 deleting or changing the file contents.
 
-## `.git-annex/uuid.log`
+## The git-annex branch
+
+This branch is managed by git-annex, with the contents listed below.
+
+Note that it assumes only it will modify the branch. If you go in and make
+changes, be sure to remove `.git/index.git-annex` before running git-annex,
+otherwise it will probably revert your changes in its next commit to the branch.
+
+### `uuid.log`
 
 Records the UUIDs of known repositories, and associates them with a
 description of the repository. This allows git-annex to display something
@@ -30,7 +38,7 @@ space and then the description through to the end of the line. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
 	26339d22-446b-11e0-9101-002170d25c55 usb disk
 
-## `git-annex/remotes.log`
+## `remotes.log`
 
 Holds persistent configuration settings for [[special_remotes]] such as
 Amazon S3.
@@ -39,7 +47,7 @@ The file format is one line per remote, starting with the uuid of the
 remote, followed by a space, and then a series of key=value pairs,
 each separated by whitespace.
 
-## `.git-annex/trust.log`
+## `trust.log`
 
 Records the [[trust]] information for repositories. Does not exist unless
 [[trust]] values are configured.
@@ -53,7 +61,7 @@ Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 1
 	26339d22-446b-11e0-9101-002170d25c55 ?
 
-## `.git-annex/aaa/bbb/*.log`
+## `aaa/bbb/*.log`
 
 The remainder of the log files record [[location_tracking]] information
 for file contents. Again these are placed in two levels of subdirectories
diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn
index 301282b6ff..85bb3d1b55 100644
--- a/doc/location_tracking.mdwn
+++ b/doc/location_tracking.mdwn
@@ -1,5 +1,5 @@
 git-annex keeps track of in which repositories it last saw a file's content.
-This location tracking information is stored in `.git-annex/$key.log`.
+This location tracking information is stored in the git-annex branch.
 Repositories record their UUID and the date when they get or drop 
 a file's content. (Git is configured to use a union merge for this file,
 so the lines may be in arbitrary order, but it will never conflict.)
diff --git a/doc/walkthrough/modifying_annexed_files.mdwn b/doc/walkthrough/modifying_annexed_files.mdwn
index f75b73a24c..1f7a7efb77 100644
--- a/doc/walkthrough/modifying_annexed_files.mdwn
+++ b/doc/walkthrough/modifying_annexed_files.mdwn
@@ -26,8 +26,7 @@ and this symlink is what gets committed to git in the end.
 	# git commit my_cool_big_file -m "changed an annexed file"
 	add my_cool_big_file ok
 	[master 64cda67] changed an annexed file
-	 2 files changed, 2 insertions(+), 1 deletions(-)
-	 create mode 100644 .git-annex/WORM-s30-m1289672605--file.log
+	 1 files changed, 1 insertions(+), 1 deletions(-)
 
 There is one problem with using `git commit` like this: Git wants to first
 stage the entire contents of the file in its index. That can be slow for

From 2035b22a01ba45505ce6367cdbeb1f6e86e9bd70 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:47:06 -0400
Subject: [PATCH 1903/8313] better branch display

---
 Branch.hs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index ad9b805d0e..e6896aa849 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -10,7 +10,8 @@ module Branch (
 	update,
 	get,
 	change,
-	commit
+	commit,
+	shortref
 ) where
 
 import Control.Monad (unless, when, liftM)
@@ -20,6 +21,7 @@ import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
+import Data.List
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -37,6 +39,13 @@ name = "git-annex"
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+shortref :: String -> String
+shortref = remove "refs/heads/" . remove "refs/remotes/"
+	where
+		remove prefix s
+			| prefix `isPrefixOf` s = drop (length prefix) s
+			| otherwise = s
+
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
 index g = Git.workTree g  Git.gitDir g  "index." ++ name
@@ -151,7 +160,7 @@ updateRef ref
 		if (null diffs)
 			then return Nothing
 			else do
-				showSideAction $ "merging " ++ ref ++ " into " ++ name ++ "..."
+				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.

From c3d96ee38a07e2cd9b27241155f80c5020a814f1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:47:14 -0400
Subject: [PATCH 1904/8313] adjust walkthrough for git-annex branch

don't just pull master.. pull everything
---
 doc/walkthrough/getting_file_content.mdwn | 2 +-
 doc/walkthrough/using_Amazon_S3.mdwn      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index bf45fd97fd..71f95f79e0 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,7 +6,7 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git pull laptop master
+	# git pull laptop
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/walkthrough/using_Amazon_S3.mdwn
index 7f972afe11..b59ca9b4f8 100644
--- a/doc/walkthrough/using_Amazon_S3.mdwn
+++ b/doc/walkthrough/using_Amazon_S3.mdwn
@@ -23,7 +23,7 @@ The configuration for the S3 remote is stored in git. So to make another
 repository use the same S3 remote is easy:
 
 	# cd /media/usb/annex
-	# git pull laptop master
+	# git pull laptop
 	# git annex initremote cloud
 	initremote cloud (gpg) (checking bucket) ok
 

From 1a182d4d047c24e217663dbccfa39aae907cbbc0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:51:48 -0400
Subject: [PATCH 1905/8313] stub in v2 upgrade

---
 Locations.hs  |  1 +
 Upgrade.hs    |  2 ++
 Upgrade/V2.hs | 24 +++++++-----------------
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index d2241636ef..7f9626a11b 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -23,6 +23,7 @@ module Locations (
 	logFile,
 	logFileKey,
 	hashDirMixed,
+	hashDirLower,
 
 	prop_idempotent_fileKey
 ) where
diff --git a/Upgrade.hs b/Upgrade.hs
index 08481755f3..6cd75cf3e8 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -11,6 +11,7 @@ import Types
 import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
+import qualified Upgrade.V2
 
 {- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
@@ -19,4 +20,5 @@ upgrade = do
 	case version of
 		"0" -> Upgrade.V0.upgrade
 		"1" -> Upgrade.V1.upgrade
+		"2" -> Upgrade.V2.upgrade
 		_ -> return True
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 03ef7ba69d..df1f62b8c8 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -5,32 +5,22 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Upgrade.V1 where
+module Upgrade.V2 where
 
-import System.IO.Error (try)
 import System.Directory
-import Control.Monad.State (liftIO)
-import Control.Monad (filterM, forM_, unless)
-import System.Posix.Files
 import System.FilePath
-import Data.String.Utils
-import System.Posix.Types
-import Data.Maybe
-import Data.Char
 
 import Types.Key
-import Content
 import Types
-import Locations
-import LocationLog
-import qualified Annex
-import qualified AnnexQueue
 import qualified GitRepo as Git
-import Backend
 import Messages
-import Version
 import Utility
-import qualified Command.Init
+import Locations
+
+upgrade :: Annex Bool
+upgrade = do
+	showNote "v2 to v3"
+	error "TODO"
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]

From c7a1690f0247fefedc9b735bee1273660fc94e77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 17:56:07 -0400
Subject: [PATCH 1906/8313] update

---
 doc/bugs/bare_git_repos.mdwn | 2 +-
 doc/todo/branching.mdwn      | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/bare_git_repos.mdwn b/doc/bugs/bare_git_repos.mdwn
index f219840e71..5e9100acfe 100644
--- a/doc/bugs/bare_git_repos.mdwn
+++ b/doc/bugs/bare_git_repos.mdwn
@@ -11,7 +11,7 @@ However, that is not currently supported. Problems include:
   a git repo at all!)
 * `.git-annex/` needs to have state recorded to it and committed, and that
   is not possible with a bare repo. (If [[todo/branching]] were done,
-  that might be fixed.)
+  that might be fixed.) (now fixed)
 
 ----
 
diff --git a/doc/todo/branching.mdwn b/doc/todo/branching.mdwn
index 37e7b6edd2..ad7ece6f10 100644
--- a/doc/todo/branching.mdwn
+++ b/doc/todo/branching.mdwn
@@ -1,3 +1,5 @@
+[[done]] !!!
+
 The use of `.git-annex` to store logs means that if a repo has branches 
 and the user switched between them, git-annex will see different logs in
 the different branches, and so may miss info about what remotes have which

From 944c51ba26efc39416c5f148b6ec36151dc7f42e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:07:45 -0400
Subject: [PATCH 1907/8313] improve version checking for v3

Do not set annex.version whenever any command is run. Just do it in init.
This ensures that, if a repo has annex.version=3, it has a git-annex
branch, so we don't have to run a command every time to check for the
branch.

Remove the old ad-hoc logic for v0 and v1, to simplify version checking.
---
 Command/Init.hs    |  2 +-
 Command/Version.hs |  3 ++-
 Upgrade.hs         |  6 +++---
 Version.hs         | 37 ++++++++++---------------------------
 4 files changed, 16 insertions(+), 32 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index dbf5666cd9..df416eed36 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 	
 command :: [Command]
-command = [repoCommand "init" paramDesc seek
+command = [standaloneCommand "init" paramDesc seek
 		"initialize git-annex with repository description"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index 755b95acca..bb7acd12dd 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -9,6 +9,7 @@ module Command.Version where
 
 import Control.Monad.State (liftIO)
 import Data.String.Utils
+import Data.Maybe
 
 import Command
 import qualified SysConfig
@@ -24,7 +25,7 @@ start :: CommandStartNothing
 start = do
 	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
 	v <- getVersion
-	liftIO $ putStrLn $ "local repository version: " ++ v
+	liftIO $ putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
 	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
 	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
 	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
diff --git a/Upgrade.hs b/Upgrade.hs
index 6cd75cf3e8..a724ecce31 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -18,7 +18,7 @@ upgrade :: Annex Bool
 upgrade = do
 	version <- getVersion
 	case version of
-		"0" -> Upgrade.V0.upgrade
-		"1" -> Upgrade.V1.upgrade
-		"2" -> Upgrade.V2.upgrade
+		Just "0" -> Upgrade.V0.upgrade
+		Just "1" -> Upgrade.V1.upgrade
+		Just "2" -> Upgrade.V2.upgrade
 		_ -> return True
diff --git a/Version.hs b/Version.hs
index 72d06f6639..690e693e23 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,14 +7,11 @@
 
 module Version where
 
-import Control.Monad.State (liftIO)
 import Control.Monad (unless)
-import System.Directory
 
 import Types
 import qualified Annex
 import qualified GitRepo as Git
-import Locations
 import Config
 
 type Version = String
@@ -31,40 +28,26 @@ upgradableVersions = ["0", "1", "2"]
 versionField :: String
 versionField = "annex.version"
 
-getVersion :: Annex Version
+getVersion :: Annex (Maybe Version)
 getVersion = do
 	g <- Annex.gitRepo
 	let v = Git.configGet g versionField ""
 	if not $ null v
-		then return v
-		else do
-			-- version 0 was not recorded in .git/config;
-			-- such a repo should have an gitAnnexDir but no
-			-- gitAnnexObjectDir.
-			--
-			-- version 1 may not be recorded if the user
-			-- forgot to init. Such a repo should have a
-			-- gitAnnexObjectDir already.
-			d <- liftIO $ doesDirectoryExist $ gitAnnexDir g
-			o <- liftIO $ doesDirectoryExist $ gitAnnexObjectDir g
-			case (d, o) of
-				(True, False) -> return "0"
-				(True, True) -> return "1"
-				_ -> do
-					setVersion
-					return defaultVersion
+		then return $ Just v
+		else return Nothing
 
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
 checkVersion :: Annex ()
-checkVersion = do
-	v <- getVersion
-	unless (v `elem` supportedVersions) $ do
-		error $ "Repository version " ++ v ++ 
-			" is not supported. " ++
-			msg v
+checkVersion = getVersion >>= handle
 	where
+		handle Nothing = error "First run: git-annex init"
+		handle (Just v) = do
+			unless (v `elem` supportedVersions) $ do
+			error $ "Repository version " ++ v ++ 
+				" is not supported. " ++
+				msg v
 		msg v
 			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
 			| otherwise = "Upgrade git-annex."

From 80302d0b46c5d45df1cf290796e0e27d9264ece8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:32:41 -0400
Subject: [PATCH 1908/8313] improve bare repo handing

Many more commands can work in bare repos now, thanks to the git-annex
branch.
---
 Command/Describe.hs        |  2 +-
 Command/Fsck.hs            |  8 ++++----
 Command/Init.hs            |  5 +++--
 Command/InitRemote.hs      |  2 +-
 Command/Move.hs            |  6 +++---
 Command/Semitrust.hs       |  2 +-
 Command/Trust.hs           |  2 +-
 Command/Untrust.hs         |  2 +-
 Content.hs                 | 14 +++-----------
 debian/changelog           |  2 ++
 doc/bare_repositories.mdwn |  9 +++------
 11 files changed, 23 insertions(+), 31 deletions(-)

diff --git a/Command/Describe.hs b/Command/Describe.hs
index 2ad3e01ee5..453e4ebafc 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -20,7 +20,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let (name, description) =
 		case ws of
 			(n:d) -> (n,unwords d)
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0e3df03dd0..cb062342d7 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -64,11 +64,11 @@ verifyLocationLog key file = do
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix u ValuePresent
+				fix g u ValuePresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix u ValueMissing
+				fix g u ValueMissing
 				warning $
 					"** Based on the location log, " ++ file
 					++ "\n** was expected to be present, " ++
@@ -77,6 +77,6 @@ verifyLocationLog key file = do
 		_ -> return True
 	
 	where
-		fix u s = do
+		fix g u s = do
 			showNote "fixing location log"
-			logStatusFor u key s
+			logChange g key u s
diff --git a/Command/Init.hs b/Command/Init.hs
index df416eed36..7f5773117a 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -8,7 +8,7 @@
 module Command.Init where
 
 import Control.Monad.State (liftIO)
-import Control.Monad (when)
+import Control.Monad (when, unless)
 import System.Directory
 
 import Command
@@ -44,7 +44,8 @@ perform description = do
 	u <- getUUID g
 	setVersion
 	describeUUID u description
-	gitPreCommitHookWrite g
+	unless (Git.repoIsLocalBare g) $
+		gitPreCommitHookWrite g
 	next $ return True
 
 {- set up a git pre-commit hook, if one is not already present -}
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 67030689da..a3054630c3 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -30,7 +30,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	when (null ws) $ needname
 
 	(u, c) <- findByName name
diff --git a/Command/Move.hs b/Command/Move.hs
index f49fe20e00..7fa195bab5 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -51,12 +51,12 @@ showAction False file = showStart "copy" file
 {- Used to log a change in a remote's having a key. The change is logged
  - in the local repo, not on the remote. The process of transferring the
  - key to the remote, or removing the key from it *may* log the change
- - on the remote, but this cannot be relied on. For example, it's not done
- - for bare repos. -}
+ - on the remote, but this cannot be relied on. -}
 remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	let remoteuuid = Remote.uuid remote
-	logStatusFor remoteuuid key status
+	g <- Annex.gitRepo
+	logChange g key remoteuuid status
 	where
 		status = if present then ValuePresent else ValueMissing
 
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 11742137f1..b467861bf9 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let name = unwords ws
 	showStart "semitrust" name
 	u <- Remote.nameToUUID name
diff --git a/Command/Trust.hs b/Command/Trust.hs
index d5444affe6..41eb17ccdd 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let name = unwords ws
 	showStart "trust" name
 	u <- Remote.nameToUUID name
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 174c395066..ea23208006 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
-start ws = notBareRepo $ do
+start ws = do
 	let name = unwords ws
 	showStart "untrust" name
 	u <- Remote.nameToUUID name
diff --git a/Content.hs b/Content.hs
index 5d77cc9795..45653fc9e7 100644
--- a/Content.hs
+++ b/Content.hs
@@ -9,7 +9,6 @@ module Content (
 	inAnnex,
 	calcGitLink,
 	logStatus,
-	logStatusFor,
 	getViaTmp,
 	getViaTmpUnchecked,
 	withTmp,
@@ -27,7 +26,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, unless, filterM)
+import Control.Monad (when, filterM)
 import System.Posix.Files
 import System.FilePath
 import Data.Maybe
@@ -71,16 +70,9 @@ calcGitLink file key = do
  - updated instead. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	u <- getUUID =<< Annex.gitRepo
-	logStatusFor u key status
-
-{- Updates the LocationLog when a key's presence changes in a repository
- - identified by UUID. -}
-logStatusFor :: UUID -> Key -> LogStatus -> Annex ()
-logStatusFor u key status = do
 	g <- Annex.gitRepo
-	unless (Git.repoIsLocalBare g) $ do
-		logChange g key u status
+	u <- getUUID g
+	logChange g key u status
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 
diff --git a/debian/changelog b/debian/changelog
index 360121cbf7..a76a0534e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (0.20110611) UNRELEASED; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
 
+  * Improved handling of bare git repos with annexes. Many more commands will
+    work in them.
   * rsync is now used when copying files from repos on other filesystems.
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index 5cbad5c2d8..c5663d84cf 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -14,12 +14,9 @@ Known to work ok:
 * `git annex drop` can check that a bare repository has a copy of data
   that is being dropped.
 * `git annex get` can transfer data from a bare repository.
+* Most other stuff (ie, init, describe, trust, etc.)
 
 There are a few caveats to keep in mind when using bare repositories:
 
-* Some subcommands, like `fsck`, `trust`, `unused` and `fromkey`, 
-  cannot be run in a bare repository. Those subcommands will
-  refuse to do anything.
-* `git annex setkey` is a plumbing-level command; using it manually
-  to add content to a bare repository is not recommended, since there
-  will be no record that the content is stored there.
+* A few subcommands, like `unused` cannot be run in a bare repository.
+  Those subcommands will refuse to do anything.

From ad3770e0b203b32a07fa142d6d83c980b23310ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:46:45 -0400
Subject: [PATCH 1909/8313] add merge subcommand

---
 Command/Merge.hs   | 29 +++++++++++++++++++++++++++++
 Command/Status.hs  |  2 +-
 GitAnnex.hs        |  2 ++
 debian/changelog   |  3 ++-
 doc/git-annex.mdwn | 11 ++++++++---
 5 files changed, 42 insertions(+), 5 deletions(-)
 create mode 100644 Command/Merge.hs

diff --git a/Command/Merge.hs b/Command/Merge.hs
new file mode 100644
index 0000000000..04328e8c5a
--- /dev/null
+++ b/Command/Merge.hs
@@ -0,0 +1,29 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Merge where
+
+import Command
+import qualified Branch
+import Messages
+
+command :: [Command]
+command = [repoCommand "merge" paramNothing seek
+		"auto-merges remote changes into the git-annex branch"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStartNothing
+start = do
+	showStart "merge" "."
+	next perform
+
+perform :: CommandPerform
+perform = do
+	Branch.update
+	next $ return True
diff --git a/Command/Status.hs b/Command/Status.hs
index 1a7f694ba8..3b096d9793 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -47,7 +47,7 @@ sizeList :: [a] -> SizeList a
 sizeList l = (l, genericLength l)
 
 command :: [Command]
-command = [repoCommand "status" (paramNothing) seek
+command = [repoCommand "status" paramNothing seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 103ee262f2..727e0c3961 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -39,6 +39,7 @@ import qualified Command.Lock
 import qualified Command.PreCommit
 import qualified Command.Find
 import qualified Command.Whereis
+import qualified Command.Merge
 import qualified Command.Status
 import qualified Command.Migrate
 import qualified Command.Uninit
@@ -76,6 +77,7 @@ cmds = concat
 	, Command.DropUnused.command
 	, Command.Find.command
 	, Command.Whereis.command
+	, Command.Merge.command
 	, Command.Status.command
 	, Command.Migrate.command
 	, Command.Map.command
diff --git a/debian/changelog b/debian/changelog
index a76a0534e2..03ccbe03f0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,8 @@
 git-annex (0.20110611) UNRELEASED; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
-
+  * git-annex now stores its logs in a git-annex branch.
+  * merge: New subcommand. Auto-merges the new git-annex branch.
   * Improved handling of bare git repos with annexes. Many more commands will
     work in them.
   * rsync is now used when copying files from repos on other filesystems.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ced6fc1b3a..1cb079ae9a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -182,6 +182,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
+* merge
+
+  Automatically merges any changes from remotes into the git-annex branch.
+  While git-annex mostly handles keeping the git-annex branch merged
+  automatically, if you find you are unable to push the git-annex branch
+  due non-fast-forward, this will fix it.
+
 * status
 
   Displays some statistics and other information, including how much data
@@ -301,9 +308,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * upgrade
 
-  Upgrades the repository to current layout. Upgrades are done automatically
-  whenever a newer git annex encounters an old repository; this command
-  allows explcitly starting an upgrade.
+  Upgrades the repository to current layout.
 
 * version
 

From 1285763015bb357297f573031cc3793d17fa702c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 18:56:08 -0400
Subject: [PATCH 1910/8313] decruft

---
 .gitattributes | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index b83edcae09..5d425843f2 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,3 +1 @@
 debian/changelog merge=dpkg-mergechangelogs
-.git-annex/*.log merge=union
-.git-annex/*/*/*.log merge=union

From 36109a286e867a6a70b5f0194332f78cd64ca277 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 19:48:04 -0400
Subject: [PATCH 1911/8313] squelched git-cat-file's error message when file
 DNE

This seemed much too hard to do. I just wanted to close stderr
when running it.
---
 Branch.hs | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index e6896aa849..2cd658fe8e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -22,6 +22,9 @@ import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
 import Data.List
+import System.IO
+import System.Posix.IO
+import System.Posix.Process
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -130,6 +133,15 @@ create = do
 			liftIO $ Git.runBool g "show-ref"
 				[Param "--verify", Param "-q", Param ref]
 
+{- Commits any staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = do
+	state <- getState
+	when (branchChanged state) $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $
+			GitUnionMerge.commit g message fullname [fullname]
+
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
@@ -190,14 +202,23 @@ get file = do
 			setCache file content
 			return content
 	where
-		cat g = Git.pipeRead g [Param "cat-file", Param "blob", catfile]
-		catfile = Param $ ':':file
+		cat g = cmdOutput "git" $ toCommand $ Git.gitCommandLine g
+			[Param "cat-file", Param "blob", Param $ ':':file]
 
-{- Commits any staged changes to the branch. -}
-commit :: String -> Annex ()
-commit message = do
-	state <- getState
-	when (branchChanged state) $ do
-		g <- Annex.gitRepo
-		withIndex $ liftIO $
-			GitUnionMerge.commit g message fullname [fullname]
+{- Runs a command, returning its output, ignoring nonzero exit
+ - status, and discarding stderr. -}
+cmdOutput :: FilePath -> [String] -> IO String
+cmdOutput cmd params = do
+	pipepair <- createPipe
+	let callfunc _ = do
+		closeFd (snd pipepair)
+		h <- fdToHandle (fst pipepair)
+		x <- hGetContentsStrict h
+		hClose h
+		return $! x
+	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params
+		(closeFd (fst pipepair) >> closeFd stdError)
+	retval <- callfunc $! pid
+	let rv = seq retval retval
+	_ <- getProcessStatus True False pid
+	return rv

From 4c8770c646c41598b73b3c86280514f1068e0d36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 19:52:13 -0400
Subject: [PATCH 1912/8313] reove 2 tests that no longer make sense (state is
 not autocommitted)

---
 test.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test.hs b/test.hs
index 528f78063a..9729818211 100644
--- a/test.hs
+++ b/test.hs
@@ -19,7 +19,6 @@ import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import Control.Exception (throw)
-import Control.Monad.State (liftIO)
 import Maybe
 import qualified Data.Map as M
 import System.Path (recurseDir)
@@ -152,8 +151,6 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy]
 			annexed_notpresent annexedfile
 		withcopy = "with content" ~: intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"]
-				@? "git commit of state failed"
 			annexed_present annexedfile
 			git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed"
 			unannexed annexedfile
@@ -167,8 +164,6 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m statechanged"]
-				@? "git commit of state failed"
 			Utility.boolSystem "git" [Utility.Params "remote rm origin"]
 				@? "git remote rm origin failed"
 			r <- git_annex "drop" ["-q", annexedfile]

From cfe0894736aaabf320d080ac098ce0a65279fe27 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 20:35:09 -0400
Subject: [PATCH 1913/8313] merge bugfix

Use GitRepo functions to call git, the bug occurred when it was run
in a git repo that was not the same as the repo being acted on.
---
 GitUnionMerge.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 267376ed57..096a153a48 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -91,10 +91,10 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
  - with the specified parent refs. -}
 commit :: Git.Repo -> String -> String -> [String] -> IO ()
 commit g message newref parentrefs = do
-	tree <- Git.getSha "write-tree" $ ignorehandle $
-		pipeFrom "git" ["write-tree"]
-	sha <- Git.getSha "commit-tree" $ ignorehandle $ 
-		pipeBoth "git" (["commit-tree", tree] ++ ps) message
+	tree <- Git.getSha "write-tree" $
+		Git.pipeRead g [Param "write-tree"]
+	sha <- Git.getSha "commit-tree" $ ignorehandle $
+		Git.pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
 	Git.run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = return . snd =<< a

From c0fbd3017f215cd61d79d24e110ce177e4823089 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 20:42:00 -0400
Subject: [PATCH 1914/8313] ssh

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 915b0bf0b2..49a80eb4b4 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,7 @@ install: all
 	fi
 
 test: $(bins)
-	if ! $(GHCMAKE) -O0 test; then \
+	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
 		./test; \

From c4e6730042e64e3b2f92626aee4a6b38a8f9c70c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 21:19:52 -0400
Subject: [PATCH 1915/8313] commit git-annex branch when copying to a remote
 (locally)

Otherwise, the location log changes are only staged in its index,
and this can confuse matters if pulling or cloning from the remote.

The test suite was failing because this wasn't done.
---
 CmdLine.hs    |  7 ++-----
 Content.hs    | 11 ++++++++++-
 Remote/Git.hs |  3 +--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index d10516bb9c..ab7236847f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,7 +19,7 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified GitRepo as Git
-import qualified Branch
+import Content
 import Types
 import Command
 import BackendList
@@ -103,9 +103,6 @@ startup = do
 {- Cleanup actions. -}
 shutdown :: Annex Bool
 shutdown = do
-	AnnexQueue.flush False
-	Branch.commit "update"
-
+	saveState
 	liftIO $ Git.reap
-	
 	return True
diff --git a/Content.hs b/Content.hs
index 45653fc9e7..d733ad8b36 100644
--- a/Content.hs
+++ b/Content.hs
@@ -19,7 +19,8 @@ module Content (
 	removeAnnex,
 	fromAnnex,
 	moveBad,
-	getKeysPresent
+	getKeysPresent,
+	saveState
 ) where
 
 import System.IO.Error (try)
@@ -37,6 +38,8 @@ import LocationLog
 import UUID
 import qualified GitRepo as Git
 import qualified Annex
+import qualified AnnexQueue
+import qualified Branch
 import Utility
 import StatFS
 import Types.Key
@@ -263,3 +266,9 @@ getKeysPresent' dir = do
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
+
+{- Things to do to record changes to content. -}
+saveState :: Annex ()
+saveState = do
+	AnnexQueue.flush False
+	Branch.commit "update"
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 5b9d5d3dfe..c8290c9a79 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -17,7 +17,6 @@ import Types
 import Types.Remote
 import qualified GitRepo as Git
 import qualified Annex
-import qualified AnnexQueue
 import Locations
 import UUID
 import Utility
@@ -147,7 +146,7 @@ copyToRemote r key
 			Annex.eval a $ do
 				ok <- Content.getViaTmp key $
 					rsyncOrCopyFile r keysrc
-				AnnexQueue.flush True
+				Content.saveState
 				return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo

From a4ef0e4da4431755c98fa104204af5254727e7a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 22:56:27 -0400
Subject: [PATCH 1916/8313] bugfix: restore index file env var

This fixes precommit, since in that hook, git sets the env var to write
to the lock file, which avoids git add failing due to the presence of the
lock file. (Took me a good hour and a half of confusion to figure this out.)

Test suite now passes 100%! Only the upgrade code still remains to be
written.
---
 Branch.hs  | 20 ++++++++++----------
 GitRepo.hs | 23 ++++++++++++++---------
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 2cd658fe8e..4f204929ea 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -68,16 +68,19 @@ genIndex g = do
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
-withIndex a = do
+withIndex = withIndex' False
+withIndex' :: Bool -> Annex a -> Annex a
+withIndex' bootstrapping a = do
 	g <- Annex.gitRepo
 	let f = index g
-	liftIO $ Git.useIndex f
+	reset <- liftIO $ Git.useIndex f
 
-	e <- liftIO $ doesFileExist f
-	unless e $ liftIO $ genIndex g
+	unless bootstrapping $ do
+		e <- liftIO $ doesFileExist f
+		unless e $ liftIO $ genIndex g
 
 	r <- a
-	liftIO $ Git.useDefaultIndex
+	liftIO reset
 	return r
 
 withIndexUpdate :: Annex a -> Annex a
@@ -121,11 +124,8 @@ create = do
 		inorigin <- refexists origin
 		if inorigin
 			then liftIO $ Git.run g "branch" [Param name, Param origin]
-			else liftIO $ do
-				let f = index g
-				liftIO $ Git.useIndex f
-				GitUnionMerge.commit g "branch created" fullname []
-				liftIO $ Git.useDefaultIndex
+			else withIndex' True $
+				liftIO $ GitUnionMerge.commit g "branch created" fullname []
 	where
 		origin = "origin/" ++ name
 		refexists ref = do
diff --git a/GitRepo.hs b/GitRepo.hs
index d1f122fba5..4a6c4d7637 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -60,7 +60,6 @@ module GitRepo (
 	repoAbsPath,
 	reap,
 	useIndex,
-	useDefaultIndex,
 	hashObject,
 	getSha,
 	shaSize,
@@ -79,6 +78,7 @@ import System.Cmd.Utils
 import IO (bracket_)
 import Data.String.Utils
 import System.IO
+import IO (try)
 import qualified Data.Map as Map hiding (map, split)
 import Network.URI
 import Data.Maybe
@@ -88,7 +88,7 @@ import Codec.Binary.UTF8.String (encode)
 import Text.Printf
 import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
-import System.Posix.Env (setEnv, unsetEnv)
+import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
 import Utility
 
@@ -391,13 +391,18 @@ reap = do
 	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
 	maybe (return ()) (const reap) r
 
-{- Forces git to use the specified index file. -}
-useIndex :: FilePath -> IO ()
-useIndex index = setEnv "GIT_INDEX_FILE" index True
-
-{- Undoes useIndex -}
-useDefaultIndex :: IO ()
-useDefaultIndex = unsetEnv "GIT_INDEX_FILE"
+{- Forces git to use the specified index file.
+ - Returns an action that will reset back to the default
+ - index file. -}
+useIndex :: FilePath -> IO (IO ())
+useIndex index = do
+	res <- try $ getEnv var
+	setEnv var index True
+	return $ reset res
+	where
+		var = "GIT_INDEX_FILE"
+		reset (Right (Just v)) = setEnv var v True
+		reset _ = unsetEnv var
 
 {- Injects some content into git, returning its hash. -}
 hashObject :: Repo -> String -> IO String

From 68783fd5e0e23ce698f2c2c2e2bd28c54cadf9c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 23:02:58 -0400
Subject: [PATCH 1917/8313] let's have the major version number be
 annex.version

---
 debian/NEWS      | 2 +-
 debian/changelog | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/NEWS b/debian/NEWS
index 4e085bb002..00531d41bc 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,4 +1,4 @@
-git-annex (0.20110622) unstable; urgency=low
+git-annex (3.20110622) unstable; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new
diff --git a/debian/changelog b/debian/changelog
index 03ccbe03f0..4543ef271a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (0.20110611) UNRELEASED; urgency=low
+git-annex (3.20110611) UNRELEASED; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
   * git-annex now stores its logs in a git-annex branch.

From aad73c5721490a5679820ab9c16a8b462fa0e0f2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 23:24:14 -0400
Subject: [PATCH 1918/8313] rewrite loggedkeys to use git-annex branch

That sucking sound is a whole page of code vanishing to be replaced with
  return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files
What can I say, git is my database, and haskell my copilot.
---
 Branch.hs         |  9 ++++++++-
 Command/Unused.hs |  4 +---
 LocationLog.hs    | 21 +++------------------
 3 files changed, 12 insertions(+), 22 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 4f204929ea..a43ee227b0 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -11,7 +11,7 @@ module Branch (
 	get,
 	change,
 	commit,
-	shortref
+	files
 ) where
 
 import Control.Monad (unless, when, liftM)
@@ -222,3 +222,10 @@ cmdOutput cmd params = do
 	let rv = seq retval retval
 	_ <- getProcessStatus True False pid
 	return rv
+
+{- Lists all files on the branch. -}
+files :: Annex [FilePath]
+files = withIndexUpdate $ do
+	g <- Annex.gitRepo
+	liftIO $ Git.pipeNullSplit g
+		[Params "ls-tree --name-only -r -z", Param fullname]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5d4e433ad8..51964cc57c 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -66,10 +66,8 @@ checkRemoteUnused name = do
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
 	showNote $ "checking for unused data..."
-	g <- Annex.gitRepo
 	referenced <- getKeysReferenced
-	logged <- loggedKeys g
-	remotehas <- filterM isthere logged
+	remotehas <- filterM isthere =<< loggedKeys
 	let remoteunused = remotehas `exclude` referenced
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
diff --git a/LocationLog.hs b/LocationLog.hs
index 8dbeb729ca..4e2caca959 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -26,7 +26,6 @@ module LocationLog (
 import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
-import System.Directory
 import System.FilePath
 import qualified Data.Map as Map
 import Control.Monad (when)
@@ -35,7 +34,6 @@ import Control.Monad.State (liftIO)
 
 import qualified GitRepo as Git
 import qualified Branch
-import Utility
 import UUID
 import Types
 import Locations
@@ -148,19 +146,6 @@ mapLog m l =
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
-loggedKeys :: Git.Repo -> Annex [Key]
-loggedKeys repo = do
-	_ <- error "FIXME.. does not look in git-annex branch yet"
-	exists <- liftIO $ doesDirectoryExist dir
-	if exists
-		then do
-			-- 2 levels of hashing
-			levela <- liftIO $ dirContents dir
-			levelb <- mapM tryDirContents levela
-			files <- mapM tryDirContents (concat levelb)
-			return $ catMaybes $
-				map (logFileKey . takeFileName) (concat files)
-		else return []
-	where
-		tryDirContents d = liftIO $ catch (dirContents d) (return . const [])
-		dir = gitStateDir repo
+loggedKeys :: Annex [Key]
+loggedKeys =
+	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files

From 66ceb9270266be677bdb0731a9c95569bad37d28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 22 Jun 2011 23:37:46 -0400
Subject: [PATCH 1919/8313] docs

---
 Upgrade/V2.hs    | 15 +++++++++++++++
 debian/changelog |  4 ++++
 2 files changed, 19 insertions(+)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index df1f62b8c8..c249e340bf 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -17,6 +17,21 @@ import Messages
 import Utility
 import Locations
 
+{- .git-annex/ moved to a git-annex branch.
+ - 
+ - Strategy:
+ - 
+ - * Create the git-annex branch.
+ - * Find each location log file in .git-annex/, and inject its content
+ -   into the git-annex branch, unioning with any content already in
+ -   there. (in passing, this deals with the semi transition that left
+ -   some location logs hashed two different ways; both are found and
+ -   merged).
+ - * Also inject remote.log, trust.log, and uuid.log.
+ - * git rm -rf .git-annex
+ - * Remove stuff that used to be needed in .gitattributes.
+ - * Commit changes.
+ -}
 upgrade :: Annex Bool
 upgrade = do
 	showNote "v2 to v3"
diff --git a/debian/changelog b/debian/changelog
index 4543ef271a..3fc7216e20 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,10 @@ git-annex (3.20110611) UNRELEASED; urgency=low
   * merge: New subcommand. Auto-merges the new git-annex branch.
   * Improved handling of bare git repos with annexes. Many more commands will
     work in them.
+  * Sped up many commands.
+  * git-annex is now more robust; it will never leave state files
+    uncommitted when some other git process comes along and locks the index
+    at an inconvenient time.
   * rsync is now used when copying files from repos on other filesystems.
     cp is still used when copying file from repos on the same filesystem,
     since --reflink=auto can make it significantly faster on filesystems

From af10b2854a199ed9985cde938d46b252f4d5e503 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 02:30:20 -0400
Subject: [PATCH 1920/8313] v3 upgrade code works

but write the index file a lot, so slow
---
 Command/Upgrade.hs |  4 +--
 Upgrade/V1.hs      |  5 +--
 Upgrade/V2.hs      | 54 +++++++++++++++++++++++--------
 doc/upgrades.mdwn  | 81 +++++++++++++++++++++++++++-------------------
 4 files changed, 94 insertions(+), 50 deletions(-)

diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index b3c0468039..b79b13cd3c 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -21,7 +21,7 @@ seek = [withNothing start]
 
 start :: CommandStartNothing
 start = do
-	showStart "upgrade" ""
+	showStart "upgrade" "."
 	r <- upgrade
-	checkVersion
+	setVersion
 	next $ next $ return r
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 1f327f77b9..61a8018590 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -30,6 +30,7 @@ import Backend
 import Messages
 import Version
 import Utility
+import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
 -- Key information is encoded in filenames differently, so
@@ -70,8 +71,8 @@ upgrade = do
 	
 			AnnexQueue.flush True
 			setVersion
-
-	return True
+	
+	Upgrade.V2.upgrade
 
 moveContent :: Annex ()
 moveContent = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index c249e340bf..36ba1a0f21 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -1,4 +1,4 @@
-{- git-annex v2 -> v2 upgrade support
+{- git-annex v2 -> v3 upgrade support
  -
  - Copyright 2011 Joey Hess 
  -
@@ -9,14 +9,22 @@ module Upgrade.V2 where
 
 import System.Directory
 import System.FilePath
+import Control.Monad.State (liftIO)
+import List
+import Data.Maybe
 
 import Types.Key
 import Types
+import qualified Annex
 import qualified GitRepo as Git
+import qualified Branch
 import Messages
 import Utility
 import Locations
 
+olddir :: FilePath
+olddir = ".git-annex"
+
 {- .git-annex/ moved to a git-annex branch.
  - 
  - Strategy:
@@ -35,7 +43,36 @@ import Locations
 upgrade :: Annex Bool
 upgrade = do
 	showNote "v2 to v3"
-	error "TODO"
+	g <- Annex.gitRepo
+	Branch.create
+	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
+	mapM_ (\f -> inject f f) =<< logFiles olddir
+	liftIO $ do
+		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File olddir]
+		gitAttributesUnWrite g
+	return True
+
+locationLogs :: Git.Repo -> Annex [(Key, FilePath)]
+locationLogs repo = liftIO $ do
+	levela <- dirContents dir
+	levelb <- mapM tryDirContents levela
+	files <- mapM tryDirContents (concat levelb)
+	return $ catMaybes $ map islogfile (concat files)
+	where
+		tryDirContents d = catch (dirContents d) (return . const [])
+		dir = gitStateDir repo
+		islogfile f = maybe Nothing (\k -> Just $ (k, f)) $
+				logFileKey $ takeFileName f
+
+inject :: FilePath -> FilePath -> Annex ()
+inject source dest = do
+	new <- liftIO (readFile $ olddir  source)
+	prev <- Branch.get dest
+	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
+
+logFiles :: FilePath -> Annex [FilePath]
+logFiles dir = return . filter (".log" `isSuffixOf`)
+		=<< liftIO (getDirectoryContents dir)
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]
@@ -49,15 +86,6 @@ gitAttributesUnWrite repo = do
 	let attributes = Git.attributes repo
 	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
-		safeWriteFile attributes $ unlines $
+		liftIO $ safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
-
-oldlogFile :: Git.Repo -> Key -> String
-oldlogFile = logFile' hashDirLower
-
-oldlogFileOld :: Git.Repo -> Key -> String
-oldlogFileOld = logFile' hashDirMixed
-
-logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
-logFile' hasher repo key =
-	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+		Git.run repo "add" [File attributes]
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 516e5b3bb8..bf12f7e438 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -7,27 +7,63 @@ There's a committment that git-annex will always support upgrades from all
 past versions. After all, you may have offline drives from an earlier
 git-annex, and might want to use them with a newer git-annex.
 
-## Upgrade process
-
 git-annex will notice if it is run in a repository that
 needs an upgrade, and refuse to do anything. To upgrade,
 use the "git annex upgrade" command. The upgrade can tend
 to take a while, if you have a lot of files.
 
-Each clone of a repository should be individually upgraded.
-Until a repository's remotes have been upgraded, git-annex
-will refuse to communicate with them.
-
-Generally, start by upgrading one repository, and then you can commit
-the changes git-annex staged during upgrade, and push them out to other
-repositories. And then upgrade those other repositories. Doing it this
-way avoids git-annex doing some duplicate work during the upgrade.
-
 The upgrade process is guaranteed to be conflict-free. Unless you
 already have git conflicts in your repository or between repositories.
 Upgrading a repository with conflicts is not recommended; resolve the
 conflicts first before upgrading git-annex.
 
+## Upgrade events, so far
+
+### v2 -> v3 (git-annex version 3.x)
+
+Involved moving the .git-annex/ directory into a separate git-annex branch.
+
+### tips for this upgrade
+
+This upgrade is easier than the previous upgrades. You don't need to
+upgrade every repository at once; it's sufficient to upgrade each
+repository only when you next use it.
+
+This upgrade can be sped up by, before you start, making
+.git/index.git-annex into a symlink to a file on a ramdisk.
+For example: `ln -s /run/shm/index.git-annex.$(git config annex.uuid) .git/index.git-annex`
+but, if you do that, be sure to remove the symlink after the upgrade!
+
+After the upgrade is complete, commit the changes it staged.
+	
+	git commit -m "upgrade v2 to v3"
+
+### v1 -> v2 (git-annex version 0.20110316)
+
+Involved adding hashing to .git/annex/ and changing the names of all keys.
+Symlinks changed.
+
+Also, hashing was added to location log files in .git-annex/.
+And .gitattributes needed to have another line added to it.
+
+Previously, files added to the SHA [[backends]] did not have their file
+size tracked, while files added to the WORM backend did. Files added to
+the SHA backends after the conversion will have their file size tracked,
+and that information will be used by git-annex for disk free space checking.
+To ensure that information is available for all your annexed files, see
+[[upgrades/SHA_size]].
+
+### tips for this upgrade
+
+Each clone of a repository should be individually upgraded.
+Until a repository's remotes have been upgraded, git-annex
+will refuse to communicate with them.
+
+Start by upgrading one repository, and then you can commit
+the changes git-annex staged during upgrade, and push them out to other
+repositories. And then upgrade those other repositories. Doing it this
+way avoids git-annex doing some duplicate work during the upgrade.
+
 Example upgrade process:
 
 	cd localrepo
@@ -43,28 +79,7 @@ Example upgrade process:
 	git annex upgrade
 	...
 
-## Upgrade events, so far
-
-### v2 -> v3 (git-annex version 0.20110610 to version 0.20110622)
-
-Involved moving the .git-annex/ directory into a separate git-annex branch.
-
-### v1 -> v2 (git-annex version 0.23 to version 0.20110316)
-
-Involved adding hashing to .git/annex/ and changing the names of all keys.
-Symlinks changed.
-
-Also, hashing was added to location log files in .git-annex/.
-And .gitattributes needed to have another line added to it.
-
-Previously, files added to the SHA [[backends]] did not have their file
-size tracked, while files added to the WORM backend did. Files added to
-the SHA backends after the conversion will have their file size tracked,
-and that information will be used by git-annex for disk free space checking.
-To ensure that information is available for all your annexed files, see
-[[upgrades/SHA_size]].
-
-### v0 -> v1 (git-annex version 0.03 to version 0.04)
+### v0 -> v1 (git-annex version 0.04)
 
 Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
 

From 9e37898e2186c8f7bb71a5d2bd7a02303410b363 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 08:48:13 -0400
Subject: [PATCH 1921/8313] remove stateDir

---
 Command.hs        | 11 ++++-------
 Locations.hs      |  9 ---------
 Upgrade/V1.hs     |  6 +++---
 Upgrade/V2.hs     |  6 ++++++
 doc/upgrades.mdwn |  1 -
 5 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/Command.hs b/Command.hs
index 228c1f40e9..129233d740 100644
--- a/Command.hs
+++ b/Command.hs
@@ -22,7 +22,6 @@ import qualified Backend
 import Messages
 import qualified Annex
 import qualified GitRepo as Git
-import Locations
 import Utility
 import Types.Key
 
@@ -181,17 +180,15 @@ withNothing _ _ = error "This command takes no parameters."
 backendPairs :: CommandSeekBackendFiles
 backendPairs a files = liftM (map a) $ Backend.chooseBackends files
 
-{- Filter out files from the state directory, and those matching the
- - exclude glob pattern, if it was specified. -}
+{- Filter out files those matching the exclude glob pattern,
+ - if it was specified. -}
 filterFiles :: [FilePath] -> Annex [FilePath]
 filterFiles l = do
-	let l' = filter notState l
 	exclude <- Annex.getState Annex.exclude
 	if null exclude
-		then return l'
-		else return $ filter (notExcluded $ wildsRegex exclude) l'
+		then return l
+		else return $ filter (notExcluded $ wildsRegex exclude) l
 	where
-		notState f = not $ stateDir `isPrefixOf` f
 		notExcluded r f = isNothing $ match r f []
 
 wildsRegex :: [String] -> Regex
diff --git a/Locations.hs b/Locations.hs
index 7f9626a11b..df4957f3e1 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -6,8 +6,6 @@
  -}
 
 module Locations (
-	gitStateDir,
-	stateDir,
 	keyFile,
 	fileKey,
 	gitAnnexLocation,
@@ -52,13 +50,6 @@ import qualified GitRepo as Git
  - Everything else should use relative paths.
  -}
 
-{- Long-term, cross-repo state is stored in files inside the .git-annex
- - directory, in the git repository's working tree. -}
-stateDir :: FilePath
-stateDir = addTrailingPathSeparator $ ".git-annex"
-gitStateDir :: Git.Repo -> FilePath
-gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
-
 {- The directory git annex uses for local state, relative to the .git
  - directory -}
 annexDir :: FilePath
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 61a8018590..b06f00d34d 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -113,7 +113,7 @@ moveLocationLogs = do
 		where
 			oldlocationlogs = do
 				g <- Annex.gitRepo
-				let dir = gitStateDir g
+				let dir = Upgrade.V2.gitStateDir g
 				exists <- liftIO $ doesDirectoryExist dir
 				if exists
 					then do
@@ -123,7 +123,7 @@ moveLocationLogs = do
 			move (l, k) = do
 				g <- Annex.gitRepo
 				let dest = logFile k
-				let dir = gitStateDir g
+				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
 				-- could just git mv, but this way deals with
@@ -186,7 +186,7 @@ fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
 logFile1 :: Git.Repo -> Key -> String
-logFile1 repo key = gitStateDir repo ++ keyFile1 key ++ ".log"
+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 36ba1a0f21..7e4cfb13a7 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -89,3 +89,9 @@ gitAttributesUnWrite repo = do
 		liftIO $ safeWriteFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
 		Git.run repo "add" [File attributes]
+
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
+gitStateDir :: Git.Repo -> FilePath
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
+
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index bf12f7e438..63fbcf75b7 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -69,7 +69,6 @@ Example upgrade process:
 	cd localrepo
 	git pull
 	git annex upgrade
-	(Upgrading object directory layout v1 to v2...)
 	git commit -m "upgrade v1 to v2"
 	git push
 

From 9672496a9357c84a4436ead109ba2dc7bc010e8c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 08:49:27 -0400
Subject: [PATCH 1922/8313] update

---
 doc/bugs/annex_add_in_annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/annex_add_in_annex.mdwn b/doc/bugs/annex_add_in_annex.mdwn
index 80084cec12..e12826f00e 100644
--- a/doc/bugs/annex_add_in_annex.mdwn
+++ b/doc/bugs/annex_add_in_annex.mdwn
@@ -2,3 +2,5 @@ I accidentally annexed some files in the .git-annex directory and it cause git-a
 
 > There is a guard against `git annex add .git-annex/foo`, but it doesn't
 > notice `cd .git-annex; git annex add foo`. --[[Joey]]
+
+> Now fixed, by removing the .git-annex directory. [[done]] --[[Joey]] 

From 23e765b67c38a9f02b3b5152e7e123819bb696de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 09:56:04 -0400
Subject: [PATCH 1923/8313] update re git-annex branch direct modification

---
 Branch.hs          |  8 ++++++++
 Locations.hs       |  5 +++++
 doc/internals.mdwn | 11 ++++++++---
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index a43ee227b0..00f406135e 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -164,6 +164,8 @@ updateRef ref
 	| ref == fullname = return Nothing
 	| otherwise = do
 		g <- Annex.gitRepo
+		-- checking with log to see if there have been changes
+		-- is less expensive than always merging
 		diffs <- liftIO $ Git.pipeRead g [
 			Param "log",
 			Param (name++".."++ref),
@@ -176,6 +178,12 @@ updateRef ref
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.
+				--
+				-- However, any changes in the git-annex
+				-- branch that are *not* reflected in the
+				-- index will be removed. So, documentation
+				-- advises users not to directly modify the
+				-- branch.
 				liftIO $ GitUnionMerge.merge g [ref]
 				return $ Just ref
 
diff --git a/Locations.hs b/Locations.hs
index df4957f3e1..f93b0cc50d 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -105,6 +105,11 @@ gitAnnexBadLocation r key = gitAnnexBadDir r  keyFile key
 gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
 gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 
+{- .git/annex/journal/ is used to journal changes made to the git-annex
+ - branch -}
+gitAnnexJournalDir :: Git.Repo -> FilePath
+gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 419096744d..aaa125599d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -21,9 +21,14 @@ deleting or changing the file contents.
 
 This branch is managed by git-annex, with the contents listed below.
 
-Note that it assumes only it will modify the branch. If you go in and make
-changes, be sure to remove `.git/index.git-annex` before running git-annex,
-otherwise it will probably revert your changes in its next commit to the branch.
+Note that git-annex assumes only it will modify this branch. If you go in
+and make changes directly, it will probably revert your changes in its next
+commit to the branch.
+
+The best way to make changes to the git-annex branch is instead
+to create a branch of it, with a name like "my/git-annex", and then
+use "git annex merge" to automerge your branch into the main git-annex
+branch.
 
 ### `uuid.log`
 

From 5f494154a34bad7cc54915a2a408b830e8ca77be Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 11:37:26 -0400
Subject: [PATCH 1924/8313] add journaling to speed up changes to the git-annex
 branch

git is slow when the index file is large and has to be rewritten each time
a file is changed. To speed this up, added a journal where changes are
recorded before being fed into the index file and committed to the
git-annex branch. The entire journal can be fed into git with just 2
commands, and only one write of the index file.
---
 Branch.hs            | 137 +++++++++++++++++++++++++++++++++++--------
 GitUnionMerge.hs     |  14 +++--
 Locations.hs         |   1 +
 Types/BranchState.hs |   3 +-
 doc/internals.mdwn   |  10 +++-
 doc/upgrades.mdwn    |   5 --
 git-union-merge.hs   |   2 +-
 7 files changed, 132 insertions(+), 40 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 00f406135e..f0d97bfc36 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -33,6 +33,7 @@ import qualified Annex
 import Utility
 import Types
 import Messages
+import Locations
 
 {- Name of the branch that is used to store git-annex's information. -}
 name :: String
@@ -42,6 +43,8 @@ name = "git-annex"
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+{- Converts a fully qualified git ref into a short version for human
+ - consumptiom. -}
 shortref :: String -> String
 shortref = remove "refs/heads/" . remove "refs/remotes/"
 	where
@@ -56,7 +59,8 @@ index g = Git.workTree g  Git.gitDir g  "index." ++ name
 {- Populates the branch's index file with the current branch contents.
  - 
  - Usually, this is only done when the index doesn't yet exist, and
- - the index is used to build up changes to be commited to the branch.
+ - the index is used to build up changes to be commited to the branch,
+ - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
 genIndex g = do
@@ -97,11 +101,6 @@ setCache file content = do
 	state <- getState
 	setState state { cachedFile = Just file, cachedContent = content }
 
-setCacheChanged :: FilePath -> String -> Annex ()
-setCacheChanged file content = do
-	state <- getState
-	setState state { cachedFile = Just file, cachedContent = content, branchChanged = True }
-
 invalidateCache :: Annex ()
 invalidateCache = do
 	state <- getState
@@ -133,11 +132,11 @@ create = do
 			liftIO $ Git.runBool g "show-ref"
 				[Param "--verify", Param "-q", Param ref]
 
-{- Commits any staged changes to the branch. -}
+{- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = do
-	state <- getState
-	when (branchChanged state) $ do
+	staged <- stageJournalFiles
+	when staged $ do
 		g <- Annex.gitRepo
 		withIndex $ liftIO $
 			GitUnionMerge.commit g message fullname [fullname]
@@ -187,28 +186,32 @@ updateRef ref
 				liftIO $ GitUnionMerge.merge g [ref]
 				return $ Just ref
 
-{- Stages the content of a file into the branch's index. -}
+{- Records changed content of a file into the journal. -}
 change :: FilePath -> String -> Annex ()
 change file content = do
-	g <- Annex.gitRepo
-	sha <- liftIO $ Git.hashObject g content
-	withIndex $ liftIO $ Git.run g "update-index"
-		[ Param "--add", Param "--cacheinfo", Param "100644",
-		  Param sha, File file]
-	setCacheChanged file content
+	setJournalFile file content
+	setCache file content
 
-{- Gets the content of a file on the branch, or content staged in the index
- - if it's newer. Returns an empty string if the file didn't exist yet. -}
+{- Gets the content of a file on the branch, or content from the journal, or
+ - staged in the index.
+ -
+ - Returns an empty string if the file doesn't exist yet. -}
 get :: FilePath -> Annex String
 get file = do
 	cached <- getCache file
 	case cached of
 		Just content -> return content
-		Nothing -> withIndexUpdate $ do
-			g <- Annex.gitRepo
-			content <- liftIO $ catch (cat g) (const $ return "")
-			setCache file content
-			return content
+		Nothing -> do
+			j <- getJournalFile file
+			case j of
+				Just content -> do
+					setCache file content
+					return content
+				Nothing -> withIndexUpdate $ do
+					g <- Annex.gitRepo
+					content <- liftIO $ catch (cat g) (const $ return "")
+					setCache file content
+					return content
 	where
 		cat g = cmdOutput "git" $ toCommand $ Git.gitCommandLine g
 			[Param "cat-file", Param "blob", Param $ ':':file]
@@ -231,9 +234,93 @@ cmdOutput cmd params = do
 	_ <- getProcessStatus True False pid
 	return rv
 
-{- Lists all files on the branch. -}
+{- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
 	g <- Annex.gitRepo
-	liftIO $ Git.pipeNullSplit g
+	bfiles <- liftIO $ Git.pipeNullSplit g
 		[Params "ls-tree --name-only -r -z", Param fullname]
+	jfiles <- getJournalFiles
+	return $ jfiles ++ bfiles
+
+{- Records content for a file in the branch to the journal.
+ -
+ - Using the journal, rather than immediatly staging content to the index
+ - avoids git needing to rewrite the index after every change. -}
+setJournalFile :: FilePath -> String -> Annex ()
+setJournalFile file content = do
+	g <- Annex.gitRepo
+	liftIO $ catch (write g) $ const $ do
+		createDirectoryIfMissing True $ gitAnnexJournalDir g
+		createDirectoryIfMissing True $ gitAnnexTmpDir g
+		write g
+	where
+		-- journal file is written atomically
+		write g = do
+			let jfile = journalFile g file
+			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
+			writeFile tmpfile content
+			renameFile tmpfile jfile
+
+{- Gets journalled content for a file in the branch. -}
+getJournalFile :: FilePath -> Annex (Maybe String)
+getJournalFile file = do
+	g <- Annex.gitRepo
+	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
+		(const $ return Nothing)
+
+{- List of journal files. -}
+getJournalFiles :: Annex [FilePath]
+getJournalFiles = getJournalFilesRaw >>= return . map fileJournal
+
+getJournalFilesRaw :: Annex [FilePath]
+getJournalFilesRaw = do
+	g <- Annex.gitRepo
+	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
+		(const $ return [])
+	return $ filter (\f -> f /= "." && f /= "..") fs
+
+{- Stages all journal files into the index, and returns True if the index
+ - was modified. -}
+stageJournalFiles :: Annex Bool
+stageJournalFiles = do
+	l <- getJournalFilesRaw
+	if null l
+		then return False
+		else do
+			g <- Annex.gitRepo
+			withIndex $ liftIO $ stage g l
+			return True
+	where
+		stage g fs = do
+			let dir = gitAnnexJournalDir g
+			let paths = map (dir ) fs
+			-- inject all the journal files directly into git
+			-- in one quick command
+			(h, s) <- Git.pipeWriteRead g [Param "hash-object",
+				Param "-w", Param "--stdin-paths"] $ unlines paths
+			-- update the index, also in just one command
+			GitUnionMerge.update_index g $
+				index_lines (lines s) $ map fileJournal fs
+			forceSuccess h
+			mapM_ removeFile paths
+		index_lines shas fs = map genline $ zip shas fs
+		genline (sha, file) = GitUnionMerge.update_index_line sha file
+
+{- Produces a filename to use in the journal for a file on the branch.
+ -
+ - The journal typically won't have a lot of files in it, so the hashing
+ - used in the branch is not necessary, and all the files are put directly
+ - in the journal directory.
+ -}
+journalFile :: Git.Repo -> FilePath -> FilePath
+journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
+	where
+		mangle '/' = "_"
+		mangle '_' = "__"
+		mangle c = [c]
+
+{- Converts a journal file (relative to the journal dir) back to the
+ - filename on the branch. -}
+fileJournal :: FilePath -> FilePath
+fileJournal = replace "//" "_" . replace "_" "/"
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 096a153a48..fa14a6bc3f 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -7,7 +7,9 @@
 
 module GitUnionMerge (
 	merge,
-	commit
+	commit,
+	update_index,
+	update_index_line
 ) where
 
 import System.Cmd.Utils
@@ -43,6 +45,11 @@ update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
 		togit ps content = Git.pipeWrite g (map Param ps) content
 			>>= forceSuccess
 
+{- Generates a line suitable to be fed into update-index, to add
+ - a given file with a given sha. -}
+update_index_line :: String -> FilePath -> String
+update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+
 {- Gets the contents of a tree in a format suitable for update_index. -}
 ls_tree :: Git.Repo -> String -> IO [String]
 ls_tree g x = Git.pipeNullSplit g $ 
@@ -76,14 +83,13 @@ calc_merge g differ = do
 mergeFile :: Git.Repo -> (String, FilePath) -> IO (Maybe String)
 mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
-	(sha:[]) -> return $ Just $ ls_tree_line sha
+	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
 		content <- Git.pipeRead g $ map Param ("show":shas)
 		sha <- Git.hashObject g $ unionmerge content
-		return $ Just $ ls_tree_line sha
+		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		ls_tree_line sha = "100644 blob " ++ sha ++ "\t" ++ file
 		nullsha = take Git.shaSize $ repeat '0'
 		unionmerge = unlines . nub . lines
 
diff --git a/Locations.hs b/Locations.hs
index f93b0cc50d..bfb0d3af9e 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -17,6 +17,7 @@ module Locations (
 	gitAnnexBadDir,
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
+	gitAnnexJournalDir,
 	isLinkToAnnex,
 	logFile,
 	logFileKey,
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index 65d0642a14..40d7f5c2c7 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -9,10 +9,9 @@ module Types.BranchState where
 
 data BranchState = BranchState {
 	branchUpdated :: Bool,
-	branchChanged :: Bool,
 	cachedFile :: Maybe FilePath,
 	cachedContent :: String
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False False Nothing ""
+startBranchState = BranchState False Nothing ""
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index aaa125599d..27b5bb1f29 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -21,9 +21,13 @@ deleting or changing the file contents.
 
 This branch is managed by git-annex, with the contents listed below.
 
-Note that git-annex assumes only it will modify this branch. If you go in
-and make changes directly, it will probably revert your changes in its next
-commit to the branch.
+The file `.git/index.git-annex` is a separate git index file it uses
+to accumlate changes for the branch. Also, `.git/annex/journal/` is used
+to record changes before they are added to git.
+
+Note that for speed reasons, git-annex assumes only it will modify this
+branch. If you go in and make changes directly, it will probably revert
+your changes in its next commit to the branch.
 
 The best way to make changes to the git-annex branch is instead
 to create a branch of it, with a name like "my/git-annex", and then
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 63fbcf75b7..2e8f201fb0 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -29,11 +29,6 @@ This upgrade is easier than the previous upgrades. You don't need to
 upgrade every repository at once; it's sufficient to upgrade each
 repository only when you next use it.
 
-This upgrade can be sped up by, before you start, making
-.git/index.git-annex into a symlink to a file on a ramdisk.
-For example: `ln -s /run/shm/index.git-annex.$(git config annex.uuid) .git/index.git-annex`
-but, if you do that, be sure to remove the symlink after the upgrade!
-
 After the upgrade is complete, commit the changes it staged.
 	
 	git commit -m "upgrade v2 to v3"
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 7c0c1cd843..57232be67b 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -42,7 +42,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
-	Git.useIndex (tmpIndex g)
+	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	GitUnionMerge.merge g [aref, bref]
 	GitUnionMerge.commit g "union merge" newref [aref, bref]

From 224a8098b5cc27a78b31094026e545137781b27e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 11:46:55 -0400
Subject: [PATCH 1925/8313] v3 upgrade is fast!

The journal sped this up approximatly 100-fold; it runs in just a few
minutes for a large repository with 30 thousand log files.
---
 doc/upgrades.mdwn | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 2e8f201fb0..509e296956 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -9,8 +9,7 @@ git-annex, and might want to use them with a newer git-annex.
 
 git-annex will notice if it is run in a repository that
 needs an upgrade, and refuse to do anything. To upgrade,
-use the "git annex upgrade" command. The upgrade can tend
-to take a while, if you have a lot of files.
+use the "git annex upgrade" command.
 
 The upgrade process is guaranteed to be conflict-free. Unless you
 already have git conflicts in your repository or between repositories.
@@ -25,9 +24,9 @@ Involved moving the .git-annex/ directory into a separate git-annex branch.
 
 ### tips for this upgrade
 
-This upgrade is easier than the previous upgrades. You don't need to
-upgrade every repository at once; it's sufficient to upgrade each
-repository only when you next use it.
+This upgrade is easier (and faster!) than the previous upgrades.
+You don't need to upgrade every repository at once; it's sufficient
+to upgrade each repository only when you next use it.
 
 After the upgrade is complete, commit the changes it staged.
 	
@@ -50,6 +49,8 @@ To ensure that information is available for all your annexed files, see
 
 ### tips for this upgrade
 
+This upgrade can tend to take a while, if you have a lot of files.
+
 Each clone of a repository should be individually upgraded.
 Until a repository's remotes have been upgraded, git-annex
 will refuse to communicate with them.

From d05fd113925bd3528a3b8643c58536c96d7151bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 12:11:03 -0400
Subject: [PATCH 1926/8313] updates

---
 doc/internals.mdwn | 4 ++--
 doc/upgrades.mdwn  | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 27b5bb1f29..1096845e5e 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -21,8 +21,8 @@ deleting or changing the file contents.
 
 This branch is managed by git-annex, with the contents listed below.
 
-The file `.git/index.git-annex` is a separate git index file it uses
-to accumlate changes for the branch. Also, `.git/annex/journal/` is used
+The file `.git/annex/index` is a separate git index file it uses
+to accumlate changes for the git-annex. Also, `.git/annex/journal/` is used
 to record changes before they are added to git.
 
 Note that for speed reasons, git-annex assumes only it will modify this
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 509e296956..1813d079a1 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -32,6 +32,9 @@ After the upgrade is complete, commit the changes it staged.
 	
 	git commit -m "upgrade v2 to v3"
 
+Running `git gc` after this upgrade will likely free up significant disk 
+space. (Tens to hundreds of megabytes.)
+
 ### v1 -> v2 (git-annex version 0.20110316)
 
 Involved adding hashing to .git/annex/ and changing the names of all keys.

From 1b21dd99c58c40ddd5bce9d575a47c66f121a29c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 12:11:09 -0400
Subject: [PATCH 1927/8313] rename git-annex index file

---
 Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Branch.hs b/Branch.hs
index f0d97bfc36..6b2e8c23eb 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -54,7 +54,7 @@ shortref = remove "refs/heads/" . remove "refs/remotes/"
 
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
-index g = Git.workTree g  Git.gitDir g  "index." ++ name
+index g = gitAnnexDir g  "index"
 
 {- Populates the branch's index file with the current branch contents.
  - 

From 89fd7b34ce815fc5816de5c71f07382e30f50bd5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 12:23:25 -0400
Subject: [PATCH 1928/8313] unused command updates for branches

Now that branches are more likely, unused needs to more explicitly warn
that it does not look in them.
---
 Command/Unused.hs  | 18 +++++++++++-------
 doc/git-annex.mdwn |  8 ++++----
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 51964cc57c..5744f84fd4 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -54,7 +54,9 @@ checkUnused = do
 	where
 		list file msg l c = do
 			let unusedlist = number c l
-			when (not $ null l) $ showLongNote $ msg unusedlist
+			when (not $ null l) $ do
+				showLongNote $ msg unusedlist
+				showLongNote $ "\n"
 			writeUnusedFile file unusedlist
 			return $ c + length l
 
@@ -108,16 +110,18 @@ staleBadMsg t = unlines $
 
 unusedMsg :: [(Int, Key)] -> String
 unusedMsg u = unusedMsg' u
-	["Some annexed data is no longer used by any files in the repository:"]
-	[dropMsg Nothing]
+	["Some annexed data is no longer used by any files in the current branch:"]
+	[dropMsg Nothing,
+	"Please be cautious -- are you sure that another branch, or another",
+	"repository does not still use this data?"]
 
 remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
 remoteUnusedMsg r u = unusedMsg' u
 	["Some annexed data on " ++ name ++ 
-	 " is not used by any files in this repository."]
+	 " is not used by any files in the current branch:"]
 	[dropMsg $ Just r,
-	 "Please be cautious -- are you sure that the remote repository",
-	 "does not use this data?"]
+	 "Please be cautious -- Are you sure that the remote repository",
+	 "does not use this data? Or that it's not used by another branch?"]
 	where
 		name = Remote.name r 
 
@@ -132,7 +136,7 @@ dropMsg :: Maybe (Remote.Remote Annex) -> String
 dropMsg Nothing = dropMsg' ""
 dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
 dropMsg' :: String -> String
-dropMsg' s = "(To remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER)\n"
+dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n"
 
 {- Finds keys whose content is present, but that do not seem to be used
  - by any files in the git repo, or that are only present as bad or tmp
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1cb079ae9a..4f02db12f4 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -155,13 +155,13 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * unused
 
-  Checks the annex for data that does not correspond to any files currently
-  in the respository, and prints a numbered list of the data.
+  Checks the annex for data that does not correspond to any files present
+  in the currently checked out branch, and prints a numbered list of the data.
 
   To only show unused temp and bad files, specify --fast
 
-  To check data on a remote that does not correspond to any files currently
-  in the local repository, specify --from.
+  To check data on a remote that does not correspond to any files present
+  on the locally checked out branch, specify --from.
 
 * dropunused [number ...]
 

From aec4709c3f96fa17fef2fde812ed75167ddfbc60 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 13:32:29 -0400
Subject: [PATCH 1929/8313] fix gotcha with closed stderr and --debug

---
 Branch.hs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 6b2e8c23eb..cb2feea6a0 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -25,6 +25,7 @@ import Data.List
 import System.IO
 import System.Posix.IO
 import System.Posix.Process
+import System.Log.Logger
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -227,8 +228,16 @@ cmdOutput cmd params = do
 		x <- hGetContentsStrict h
 		hClose h
 		return $! x
-	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params
-		(closeFd (fst pipepair) >> closeFd stdError)
+	let child = do
+		closeFd (fst pipepair)
+ 		-- disable stderr output by this child,
+		-- and since the logger uses it, also disable it
+		liftIO $ updateGlobalLogger rootLoggerName $ setLevel EMERGENCY
+		closeFd stdError
+
+	debugM "Utility.executeFile" $ cmd ++ " " ++ show params
+	
+	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params child
 	retval <- callfunc $! pid
 	let rv = seq retval retval
 	_ <- getProcessStatus True False pid

From 7ee636f6ddca4b872dde36383077875563b0b369 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 13:39:04 -0400
Subject: [PATCH 1930/8313] avoid unnecessary read of trust.log

---
 Backend/File.hs  |  4 ++--
 Command/Move.hs  |  4 ++--
 Remote.hs        | 22 ++++++++++++++++++++--
 debian/changelog |  1 -
 4 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index eab987ef81..675b483076 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -52,7 +52,7 @@ dummyStore _ _ = return True
  - and copy it to here. -}
 copyKeyFile :: Key -> FilePath -> Annex Bool
 copyKeyFile key file = do
-	(remotes, _) <- Remote.keyPossibilities key
+	remotes <- Remote.keyPossibilities key
 	if null remotes
 		then do
 			showNote "not available"
@@ -95,7 +95,7 @@ checkRemoveKey key numcopiesM = do
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilities key
+			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 			untrusteduuids <- trustGet UnTrusted
 			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 			numcopies <- getNumCopies numcopiesM
diff --git a/Command/Move.hs b/Command/Move.hs
index 7fa195bab5..03b605ce57 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -89,7 +89,7 @@ toPerform dest move key = do
 	let fastcheck = fast && not move && not (Remote.hasKeyCheap dest)
 	isthere <- if fastcheck
 		then do
-			(remotes, _) <- Remote.keyPossibilities key
+			remotes <- Remote.keyPossibilities key
 			return $ Right $ dest `elem` remotes
 		else Remote.hasKey dest key
 	case isthere of
@@ -123,7 +123,7 @@ fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	(remotes, _) <- Remote.keyPossibilities key
+	remotes <- Remote.keyPossibilities key
 	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
 		then stop
 		else do
diff --git a/Remote.hs b/Remote.hs
index 5122423421..6295fc9476 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,6 +15,7 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 	keyPossibilities,
+	keyPossibilitiesTrusted,
 	forceTrust,
 
 	remoteTypes,
@@ -127,13 +128,30 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
+{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+ -}
+keyPossibilities :: Key -> Annex [Remote Annex]
+keyPossibilities key = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+
+	-- get uuids of all remotes that are recorded to have the key
+	uuids <- keyLocations key
+	let validuuids = filter (/= u) uuids
+
+	-- remotes that match uuids that have the key
+	allremotes <- genList
+	let validremotes = remotesWithUUID allremotes validuuids
+
+	return $ sort validremotes
+
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilities :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilities key = do
+keyPossibilitiesTrusted :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilitiesTrusted key = do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	trusted <- trustGet Trusted
diff --git a/debian/changelog b/debian/changelog
index 3fc7216e20..69b28c4551 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,6 @@ git-annex (3.20110611) UNRELEASED; urgency=low
   * merge: New subcommand. Auto-merges the new git-annex branch.
   * Improved handling of bare git repos with annexes. Many more commands will
     work in them.
-  * Sped up many commands.
   * git-annex is now more robust; it will never leave state files
     uncommitted when some other git process comes along and locks the index
     at an inconvenient time.

From 068703c40565cf8312c103b40d8c992b9d20a113 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 14:49:21 -0400
Subject: [PATCH 1931/8313] improve post-upgrade push instructions

---
 Upgrade/V2.hs     |  5 +++++
 debian/NEWS       |  3 ++-
 doc/upgrades.mdwn | 12 +++++++-----
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 7e4cfb13a7..9d32764e76 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -50,6 +50,11 @@ upgrade = do
 	liftIO $ do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File olddir]
 		gitAttributesUnWrite g
+
+	showLongNote $
+		"git-annex branch created\n" ++
+		"Now you should push the new branch: git push origin git-annex"
+
 	return True
 
 locationLogs :: Git.Repo -> Annex [(Key, FilePath)]
diff --git a/debian/NEWS b/debian/NEWS
index 00531d41bc..4218bf5681 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -2,7 +2,8 @@ git-annex (3.20110622) unstable; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new
-  layout.
+  layout. See  or
+  /usr/share/doc/git-annex/html/upgrades.html
 
   The significant change this time is that the .git-annex/ directory
   is gone; instead there is a git-annex branch that is automatically
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 1813d079a1..f83a8c9477 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -27,13 +27,15 @@ Involved moving the .git-annex/ directory into a separate git-annex branch.
 This upgrade is easier (and faster!) than the previous upgrades.
 You don't need to upgrade every repository at once; it's sufficient
 to upgrade each repository only when you next use it.
-
-After the upgrade is complete, commit the changes it staged.
 	
-	git commit -m "upgrade v2 to v3"
+Example upgrade process:
 
-Running `git gc` after this upgrade will likely free up significant disk 
-space. (Tens to hundreds of megabytes.)
+	cd localrepo
+	git pull
+	git annex upgrade
+	git commit -m "upgrade v2 to v3"
+	git push origin git-annex master
+	git gc
 
 ### v1 -> v2 (git-annex version 0.20110316)
 

From c5531046bc6cd62cedfc528b844e0df9cdade177 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 15:30:04 -0400
Subject: [PATCH 1932/8313] refactor

---
 Remote.hs | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 6295fc9476..1accabf6d9 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -131,19 +131,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
-keyPossibilities key = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- keyLocations key
-	let validuuids = filter (/= u) uuids
-
-	-- remotes that match uuids that have the key
-	allremotes <- genList
-	let validremotes = remotesWithUUID allremotes validuuids
-
-	return $ sort validremotes
+keyPossibilities key = return . fst =<< keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
@@ -151,10 +139,13 @@ keyPossibilities key = do
  - (some may not have configured remotes).
  -}
 keyPossibilitiesTrusted :: Key -> Annex ([Remote Annex], [UUID])
-keyPossibilitiesTrusted key = do
+keyPossibilitiesTrusted = keyPossibilities' True
+
+keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities' withtrusted key = do
 	g <- Annex.gitRepo
 	u <- getUUID g
-	trusted <- trustGet Trusted
+	trusted <- if withtrusted then trustGet Trusted else return []
 
 	-- get uuids of all remotes that are recorded to have the key
 	uuids <- keyLocations key

From 780ee5ff6d3fdcbbe69450d40260c1083315ca47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 15:38:52 -0400
Subject: [PATCH 1933/8313] fix bootstrapping, broken by move of
 .git/annex/index

---
 Branch.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index cb2feea6a0..10a7906314 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -80,9 +80,10 @@ withIndex' bootstrapping a = do
 	let f = index g
 	reset <- liftIO $ Git.useIndex f
 
-	unless bootstrapping $ do
-		e <- liftIO $ doesFileExist f
-		unless e $ liftIO $ genIndex g
+	e <- liftIO $ doesFileExist f
+	unless e $ liftIO $ do
+		createDirectoryIfMissing True $ takeDirectory f
+		unless bootstrapping $ genIndex g
 
 	r <- a
 	liftIO reset

From 7981eb4cb512fbe3c49a3dd165c31be14ae4bc49 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 15:51:07 -0400
Subject: [PATCH 1934/8313] fix consistency, and partially close a race during
 merge

Only "partially" because the journal is not locked during the merge, so
there's a small window where a different git-annex process could write info
to the journal that overwrites info taken from the merge.

That could be dealt with by locking, but the lock would really need to be
around the whole git-annex, to only let one run at a time. Otherwise, even
with the journal locked during the merge, another git-annex could already
be running, generate an overwriting change, and only store it in the journal
after the merge was complete. And similarly, two git-annex processes could
fight and overwrite each other's information independant of any merging.

So, a toplevel lock for git-annex may get added; it's something I've
considered before, as these potential, unlikely problems are not new.
(OTOH, fsck will deal with such problems.)
---
 Branch.hs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/Branch.hs b/Branch.hs
index 10a7906314..e79ca3f60a 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -149,6 +149,21 @@ update :: Annex ()
 update = do
 	state <- Annex.getState Annex.branchstate
 	unless (branchUpdated state) $ withIndex $ do
+		{- Since branches get merged into the index, it's important to
+		 - first stage the journal into the index. Otherwise, any
+		 - changes in the journal would later get staged, and might
+		 - overwrite changes made during the merge.
+		 -
+		 - It would be cleaner to handle the merge by updating the
+		 - journal, not the index, with changes from the branches.
+		 -
+		 - XXX Anything written to the journal during the merge,
+		 - by another process could still race the merge. The
+		 - journal should really be blocking locked during the
+		 - merge.
+		 -}
+		_ <- stageJournalFiles
+
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		let refs = map (last . words) (lines r)

From 1686f60f8481e996291a5970b2217bc1d64f635d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 16:44:26 -0400
Subject: [PATCH 1935/8313] commit after merge if any journal files were staged

---
 Branch.hs | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index e79ca3f60a..6e9dae9b9a 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -156,19 +156,14 @@ update = do
 		 -
 		 - It would be cleaner to handle the merge by updating the
 		 - journal, not the index, with changes from the branches.
-		 -
-		 - XXX Anything written to the journal during the merge,
-		 - by another process could still race the merge. The
-		 - journal should really be blocking locked during the
-		 - merge.
 		 -}
-		_ <- stageJournalFiles
+		staged <- stageJournalFiles
 
 		g <- Annex.gitRepo
 		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 		let refs = map (last . words) (lines r)
 		updated <- catMaybes `liftM` mapM updateRef refs
-		unless (null updated) $ liftIO $
+		unless (null updated && not staged) $ liftIO $
 			GitUnionMerge.commit g "update" fullname
 				(fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }

From 22243b87d2a43d6fd82581ef70a0537cb8ae8661 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 17:08:11 -0400
Subject: [PATCH 1936/8313] layout

---
 Upgrade/V2.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 9d32764e76..57a2ba7893 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -53,7 +53,7 @@ upgrade = do
 
 	showLongNote $
 		"git-annex branch created\n" ++
-		"Now you should push the new branch: git push origin git-annex"
+		"Now you should push the new branch: git push origin git-annex\n"
 
 	return True
 

From ab9b971f8f772cf7e89a904d665bc88ddb3afd47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 17:37:23 -0400
Subject: [PATCH 1937/8313] simplified to use existing functions

---
 Branch.hs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 6e9dae9b9a..fad024550f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -64,12 +64,7 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = do
-	ls <- Git.pipeNullSplit g $
-		map Param ["ls-tree", "-z", "-r", "--full-tree", fullname]
-	forceSuccess =<< Git.pipeWrite g
-		(map Param ["update-index", "-z", "--index-info"])
-		(join "\0" ls)
+genIndex g = GitUnionMerge.ls_tree g fullname >>= GitUnionMerge.update_index g
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a

From c4cc6ee42f63bb8196f44608aaf35f7f9f411fe1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 17:38:27 -0400
Subject: [PATCH 1938/8313] fix merge_tree_index

--cached is needed when calling git-diff-index, as it is not diffing
against currently checked out branch.
---
 GitUnionMerge.hs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index fa14a6bc3f..74579ebcc9 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -9,7 +9,8 @@ module GitUnionMerge (
 	merge,
 	commit,
 	update_index,
-	update_index_line
+	update_index_line,
+	ls_tree
 ) where
 
 import System.Cmd.Utils
@@ -40,7 +41,7 @@ merge _ _ = error "wrong number of branches to merge"
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Git.Repo -> [String] -> IO ()
-update_index g l =  togit ["update-index", "-z", "--index-info"] (join "\0" l)
+update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
 		togit ps content = Git.pipeWrite g (map Param ps) content
 			>>= forceSuccess
@@ -57,13 +58,14 @@ ls_tree g x = Git.pipeNullSplit g $
 
 {- For merging two trees. -}
 merge_trees :: Git.Repo -> String -> String -> IO [String]
-merge_trees g x y = calc_merge g
-	["diff-tree", "--raw", "-z", "-r", "--no-renames", "-l0", x, y]
+merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
 merge_tree_index :: Git.Repo -> String -> IO [String]
-merge_tree_index g x = calc_merge g
-	["diff-index", "--raw", "-z", "-r", "--no-renames", "-l0", x]
+merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]
+
+diff_opts :: [String]
+diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}

From e3384eb4764787daef926c307004d001e224a9fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 19:56:24 -0400
Subject: [PATCH 1939/8313] tweak fsck wording so file is at the end of the
 line

---
 Backend/File.hs                                 | 4 ++--
 doc/walkthrough/fsck:_when_things_go_wrong.mdwn | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Backend/File.hs b/Backend/File.hs
index 675b483076..1c25f89db2 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -205,14 +205,14 @@ checkKeyNumCopies key file numcopies = do
 
 missingNote :: String -> Int -> Int -> String -> String
 missingNote file 0 _ [] = 
-		"** No known copies of " ++ file ++ " exist!"
+		"** No known copies exist of " ++ file
 missingNote file 0 _ untrusted =
 		"Only these untrusted locations may have copies of " ++ file ++
 		"\n" ++ untrusted ++
 		"Back it up to trusted locations with git-annex copy."
 missingNote file present needed [] =
 		"Only " ++ show present ++ " of " ++ show needed ++ 
-		" trustworthy copies of " ++ file ++ " exist." ++
+		" trustworthy copies exist of " ++ file ++
 		"\nBack it up with git-annex copy."
 missingNote file present needed untrusted = 
 		missingNote file present needed [] ++
diff --git a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
index 05b9f385c0..85d9f20fe0 100644
--- a/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
+++ b/doc/walkthrough/fsck:_when_things_go_wrong.mdwn
@@ -5,7 +5,7 @@ might say about a badly messed up annex:
 	# git annex fsck
 	fsck my_cool_big_file (checksum...)
 	git-annex: Bad file content; moved to .git/annex/bad/SHA1:7da006579dd64330eb2456001fd01948430572f2
-	git-annex: ** No known copies of the file exist!
+	git-annex: ** No known copies exist of my_cool_big_file
 	failed
 	fsck important_file
 	git-annex: Only 1 of 2 copies exist. Run git annex get somewhere else to back it up.

From a61154baf5e205af74766f44fe99cbbd63411f57 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 20:52:56 -0400
Subject: [PATCH 1940/8313] add

---
 doc/todo/speed_up_fsck.mdwn | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/todo/speed_up_fsck.mdwn

diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
new file mode 100644
index 0000000000..aceb5868cc
--- /dev/null
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -0,0 +1,18 @@
+moving to the git-annex branch has slowed down fsck worse than most
+commands. Actually, some commands have sped up, while others like get
+are slightly slower but are swamped by the normal runtime. 
+
+For fsck though, it has to pull each file's location log info out of git.
+And, it's typically run on the entire tree.
+
+It would be possible to run a single `git cat-file --batch` and pass it
+sha1s of location logs for file that is going to be fsked (gotten via
+`read-tree`). Then just read its output until the next requested sha1 to
+chunk it, and pass this in to fsck in a closure.
+
+The difficulty, besides writing that is that everything that works with
+location logs now reads them out of git, would need to find a way to
+provide the info on a side channel of some sort.
+
+If this is implemented, the same infrastructure could be used for other
+commands like whereis and add. --[[Joey]]

From 69d3c1cec9f6be1dba1ffb391bf69464c52f5936 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 23 Jun 2011 21:25:39 -0400
Subject: [PATCH 1941/8313] cache the trustmap

Doubles the speed of fsck, and speeds up drop as well.
---
 Annex.hs                             |  4 +++-
 GitAnnex.hs                          |  2 +-
 Trust.hs                             | 19 ++++++++++++-------
 TrustLevel.hs => Types/TrustLevel.hs |  9 ++++++++-
 4 files changed, 24 insertions(+), 10 deletions(-)
 rename TrustLevel.hs => Types/TrustLevel.hs (79%)

diff --git a/Annex.hs b/Annex.hs
index 2bd090e906..82908881da 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -24,7 +24,7 @@ import Types.Backend
 import Types.Remote
 import Types.Crypto
 import Types.BranchState
-import TrustLevel
+import Types.TrustLevel
 import Types.UUID
 
 -- git-annex's monad
@@ -48,6 +48,7 @@ data AnnexState = AnnexState
 	, fromremote :: Maybe String
 	, exclude :: [String]
 	, forcetrust :: [(UUID, TrustLevel)]
+	, trustmap :: Maybe TrustMap
 	, cipher :: Maybe Cipher
 	}
 
@@ -69,6 +70,7 @@ newState allbackends gitrepo = AnnexState
 	, fromremote = Nothing
 	, exclude = []
 	, forcetrust = []
+	, trustmap = Nothing
 	, cipher = Nothing
 	}
 
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 727e0c3961..49ba63b8ab 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -14,7 +14,7 @@ import CmdLine
 import Command
 import Options
 import Utility
-import TrustLevel
+import Types.TrustLevel
 import qualified Annex
 import qualified Remote
 
diff --git a/Trust.hs b/Trust.hs
index d328235bfa..365186da22 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -9,15 +9,13 @@ module Trust (
 	TrustLevel(..),
 	trustLog,
 	trustGet,
-	trustMap,
-	trustMapParse,
 	trustSet
 ) where
 
 import Control.Monad.State
 import qualified Data.Map as M
 
-import TrustLevel
+import Types.TrustLevel
 import qualified Branch
 import Types
 import UUID
@@ -35,11 +33,17 @@ trustGet level = do
 
 {- Read the trustLog into a map, overriding with any
  - values from forcetrust -}
-trustMap :: Annex (M.Map UUID TrustLevel)
+trustMap :: Annex TrustMap
 trustMap = do
-	overrides <- Annex.getState Annex.forcetrust
-	s <- Branch.get trustLog
-	return $ M.fromList $ trustMapParse s ++ overrides
+	cached <- Annex.getState Annex.trustmap
+	case cached of
+		Just m -> return m
+		Nothing -> do
+			overrides <- Annex.getState Annex.forcetrust
+			l <- Branch.get trustLog
+			let m = M.fromList $ trustMapParse l ++ overrides
+			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
+			return m
 
 {- Trust map parser. -}
 trustMapParse :: String -> [(UUID, TrustLevel)]
@@ -62,6 +66,7 @@ trustSet uuid level = do
 	when (M.lookup uuid m /= Just level) $ do
 		let m' = M.insert uuid level m
 		Branch.change trustLog (serialize m')
+		Annex.changeState $ \s -> s { Annex.trustmap = Just m' }
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
diff --git a/TrustLevel.hs b/Types/TrustLevel.hs
similarity index 79%
rename from TrustLevel.hs
rename to Types/TrustLevel.hs
index 5da142ca30..058ce4595c 100644
--- a/TrustLevel.hs
+++ b/Types/TrustLevel.hs
@@ -5,10 +5,15 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module TrustLevel (
+module Types.TrustLevel (
 	TrustLevel(..),
+	TrustMap
 ) where
 
+import qualified Data.Map as M
+
+import Types.UUID
+
 data TrustLevel = SemiTrusted | UnTrusted | Trusted
 	deriving Eq
 
@@ -21,3 +26,5 @@ instance Read TrustLevel where
         readsPrec _ "1" = [(Trusted, "")]
         readsPrec _ "0" = [(UnTrusted, "")]
 	readsPrec _ _ = [(SemiTrusted, "")]
+
+type TrustMap = M.Map UUID TrustLevel

From 59b2e4ec1d7de3cfc9c477a68ee418d403bc21d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 01:13:33 -0400
Subject: [PATCH 1942/8313] fixes for upgrading bare repos

---
 Upgrade/V2.hs | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 57a2ba7893..7f0062cf34 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -10,6 +10,7 @@ module Upgrade.V2 where
 import System.Directory
 import System.FilePath
 import Control.Monad.State (liftIO)
+import Control.Monad.State (unless)
 import List
 import Data.Maybe
 
@@ -22,8 +23,10 @@ import Messages
 import Utility
 import Locations
 
-olddir :: FilePath
-olddir = ".git-annex"
+olddir :: Git.Repo -> FilePath
+olddir g
+	| Git.repoIsLocalBare g = ""
+	| otherwise = ".git-annex"
 
 {- .git-annex/ moved to a git-annex branch.
  - 
@@ -46,14 +49,15 @@ upgrade = do
 	g <- Annex.gitRepo
 	Branch.create
 	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
-	mapM_ (\f -> inject f f) =<< logFiles olddir
+	mapM_ (\f -> inject f f) =<< logFiles (olddir g)
 	liftIO $ do
-		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File olddir]
-		gitAttributesUnWrite g
+		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
+		unless (Git.repoIsLocalBare g) $ gitAttributesUnWrite g
 
 	showLongNote $
 		"git-annex branch created\n" ++
 		"Now you should push the new branch: git push origin git-annex\n"
+	showProgress
 
 	return True
 
@@ -71,7 +75,8 @@ locationLogs repo = liftIO $ do
 
 inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
-	new <- liftIO (readFile $ olddir  source)
+	g <- Annex.gitRepo
+	new <- liftIO (readFile $ olddir g  source)
 	prev <- Branch.get dest
 	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
 

From 49d77156ac75dd9459f3e2f2be3baa3ff3b172bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 01:15:12 -0400
Subject: [PATCH 1943/8313] more upgrades fixes for bare repos

---
 Upgrade/V2.hs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 7f0062cf34..af8f15919e 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -47,17 +47,20 @@ upgrade :: Annex Bool
 upgrade = do
 	showNote "v2 to v3"
 	g <- Annex.gitRepo
+	let bare = Git.repoIsLocalBare g
+
 	Branch.create
 	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
 	mapM_ (\f -> inject f f) =<< logFiles (olddir g)
 	liftIO $ do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-		unless (Git.repoIsLocalBare g) $ gitAttributesUnWrite g
+		unless bare $ gitAttributesUnWrite g
 
-	showLongNote $
-		"git-annex branch created\n" ++
-		"Now you should push the new branch: git push origin git-annex\n"
-	showProgress
+	unless bare $ do
+		showLongNote $
+			"git-annex branch created\n" ++
+			"Now you should push the new branch: git push origin git-annex\n"
+		showProgress
 
 	return True
 

From 50354a49163b8841af2024876a62e78b6caaa2b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 02:33:44 -0400
Subject: [PATCH 1944/8313] save state before message

to avoid long delay after
---
 Upgrade/V2.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index af8f15919e..2372edb3db 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -22,6 +22,7 @@ import qualified Branch
 import Messages
 import Utility
 import Locations
+import Content
 
 olddir :: Git.Repo -> FilePath
 olddir g
@@ -56,6 +57,8 @@ upgrade = do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
 		unless bare $ gitAttributesUnWrite g
 
+	saveState
+
 	unless bare $ do
 		showLongNote $
 			"git-annex branch created\n" ++

From ad38c0dfadc0fc68ed6213c75541fe06c7caca2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 11:59:34 -0400
Subject: [PATCH 1945/8313] better setup of git-annex branch pushing on upgrade

---
 Branch.hs         | 34 +++++++++++++++++++++++-----------
 Upgrade/V2.hs     | 41 ++++++++++++++++++++++++++++++++---------
 doc/upgrades.mdwn |  1 -
 3 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index fad024550f..3f5e0ee1ec 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -11,7 +11,10 @@ module Branch (
 	get,
 	change,
 	commit,
-	files
+	files,
+	refExists,
+	hasOrigin,
+	name	
 ) where
 
 import Control.Monad (unless, when, liftM)
@@ -44,6 +47,10 @@ name = "git-annex"
 fullname :: String
 fullname = "refs/heads/" ++ name
 
+{- Branch's name in origin. -}
+originname :: String
+originname = "origin/" ++ name
+
 {- Converts a fully qualified git ref into a short version for human
  - consumptiom. -}
 shortref :: String -> String
@@ -114,20 +121,14 @@ getCache file = getState >>= handle
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = do
-	exists <- refexists fullname
+	exists <- refExists fullname
 	unless exists $ do
 		g <- Annex.gitRepo
-		inorigin <- refexists origin
-		if inorigin
-			then liftIO $ Git.run g "branch" [Param name, Param origin]
+		e <- hasOrigin
+		if e
+			then liftIO $ Git.run g "branch" [Param name, Param originname]
 			else withIndex' True $
 				liftIO $ GitUnionMerge.commit g "branch created" fullname []
-	where
-		origin = "origin/" ++ name
-		refexists ref = do
-			g <- Annex.gitRepo
-			liftIO $ Git.runBool g "show-ref"
-				[Param "--verify", Param "-q", Param ref]
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
@@ -164,6 +165,17 @@ update = do
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = refExists originname
+
+{- Checks if a git ref exists. -}
+refExists :: String -> Annex Bool
+refExists ref = do
+	g <- Annex.gitRepo
+	liftIO $ Git.runBool g "show-ref"
+		[Param "--verify", Param "-q", Param ref]
+
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: String -> Annex (Maybe String)
 updateRef ref
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 2372edb3db..8eab236ff5 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,8 +9,7 @@ module Upgrade.V2 where
 
 import System.Directory
 import System.FilePath
-import Control.Monad.State (liftIO)
-import Control.Monad.State (unless)
+import Control.Monad.State (unless, liftIO)
 import List
 import Data.Maybe
 
@@ -58,12 +57,7 @@ upgrade = do
 		unless bare $ gitAttributesUnWrite g
 
 	saveState
-
-	unless bare $ do
-		showLongNote $
-			"git-annex branch created\n" ++
-			"Now you should push the new branch: git push origin git-annex\n"
-		showProgress
+	unless bare $ push
 
 	return True
 
@@ -90,6 +84,36 @@ logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
 		=<< liftIO (getDirectoryContents dir)
 
+push :: Annex ()
+push = do
+	origin_master <- Branch.refExists "origin/master"
+	origin_gitannex <- Branch.hasOrigin
+	case (origin_master, origin_gitannex) of
+		(_, True) -> do
+			-- Merge in the origin's git-annex branch,
+			-- so that pushing the git-annex branch
+			-- will immediately work. Not pushed here,
+			-- because it's less obnoxious to let the user
+			-- push.
+			Branch.update
+		(True, False) -> do
+			-- push git-annex to origin, so that
+			-- "git push" will from then on
+			-- automatically push it
+			Branch.update -- just in case
+			showNote "pushing new git-annex branch to origin"
+			showProgress
+			g <- Annex.gitRepo
+			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
+		_ -> do
+			-- no origin exists, so just let the user
+			-- know about the new branch
+			Branch.update
+			showLongNote $
+				"git-annex branch created\n" ++
+				"Be sure to push this branch when pushing to remotes.\n"
+			showProgress
+
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]
 attrLines =
@@ -110,4 +134,3 @@ stateDir :: FilePath
 stateDir = addTrailingPathSeparator $ ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
-
diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index f83a8c9477..337c65767a 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -34,7 +34,6 @@ Example upgrade process:
 	git pull
 	git annex upgrade
 	git commit -m "upgrade v2 to v3"
-	git push origin git-annex master
 	git gc
 
 ### v1 -> v2 (git-annex version 0.20110316)

From d7018500fac3f496489888e2f5fd124c66c0b746 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 12:09:04 -0400
Subject: [PATCH 1946/8313] fix upgrade when .git-annex has already been
 entirely converted

---
 Upgrade/V2.hs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 8eab236ff5..8537a50226 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,7 +9,7 @@ module Upgrade.V2 where
 
 import System.Directory
 import System.FilePath
-import Control.Monad.State (unless, liftIO)
+import Control.Monad.State (unless, when, liftIO)
 import List
 import Data.Maybe
 
@@ -50,11 +50,13 @@ upgrade = do
 	let bare = Git.repoIsLocalBare g
 
 	Branch.create
-	mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
-	mapM_ (\f -> inject f f) =<< logFiles (olddir g)
-	liftIO $ do
-		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-		unless bare $ gitAttributesUnWrite g
+	e <- liftIO $ doesDirectoryExist (olddir g)
+	when e $ do
+		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
+		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
+		liftIO $ do
+			Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
+			unless bare $ gitAttributesUnWrite g
 
 	saveState
 	unless bare $ push

From 874fc044c1dec05fc398b44ccd69a2f501a52de2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 14:58:07 -0400
Subject: [PATCH 1947/8313] releasing version 3.20110624

---
 debian/NEWS      | 2 +-
 debian/changelog | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/NEWS b/debian/NEWS
index 4218bf5681..22835ace84 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,4 +1,4 @@
-git-annex (3.20110622) unstable; urgency=low
+git-annex (3.20110624) exerimental; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new
diff --git a/debian/changelog b/debian/changelog
index 69b28c4551..47652241a0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110611) UNRELEASED; urgency=low
+git-annex (3.20110624) experimental; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
   * git-annex now stores its logs in a git-annex branch.
@@ -18,7 +18,7 @@ git-annex (3.20110611) UNRELEASED; urgency=low
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
 
- -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
+ -- Joey Hess   Fri, 24 Jun 2011 14:32:18 -0400
 
 git-annex (0.20110610) unstable; urgency=low
 

From 14e2765ba8c29041575a13f9b06ce6de8ebc26ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Jun 2011 14:58:20 -0400
Subject: [PATCH 1948/8313] add news item for git-annex 3.20110624

---
 doc/news/version_0.20110516.mdwn | 11 -----------
 doc/news/version_3.20110624.mdwn | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 11 deletions(-)
 delete mode 100644 doc/news/version_0.20110516.mdwn
 create mode 100644 doc/news/version_3.20110624.mdwn

diff --git a/doc/news/version_0.20110516.mdwn b/doc/news/version_0.20110516.mdwn
deleted file mode 100644
index aa350a4848..0000000000
--- a/doc/news/version_0.20110516.mdwn
+++ /dev/null
@@ -1,11 +0,0 @@
-git-annex 0.20110516 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Add a few tweaks to make it easy to use the Internet Archive's variant
-     of S3. In particular, munge key filenames to comply with the IA's filename
-     limits, disable encryption, support their nonstandard way of creating
-     buckets, and allow x-archive-* headers to be specified in initremote to
-     set item metadata.
-   * Added filename extension preserving variant backends SHA1E, SHA256E, etc.
-   * migrate: Use current filename when generating new key, for backends
-     where the filename affects the key name.
-   * Work around a bug in Network.URI's handling of bracketed ipv6 addresses."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
new file mode 100644
index 0000000000..5a91fa4295
--- /dev/null
+++ b/doc/news/version_3.20110624.mdwn
@@ -0,0 +1,32 @@
+News for git-annex 3.20110624:
+
+   There has been another change to the git-annex data store.
+   Use `git annex upgrade` to migrate your repositories to the new
+   layout. See <http://git-annex.branchable.com/upgrades/> or
+   /usr/share/doc/git-annex/html/upgrades.html
+   The significant change this time is that the .git-annex/ directory
+   is gone; instead there is a git-annex branch that is automatically
+   maintained by git-annex, and encapsulates all its state nicely out
+   of your way.
+   You should make sure you include the git-annex branch when
+   git pushing and pulling.
+
+git-annex 3.20110624 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
+   * git-annex now stores its logs in a git-annex branch.
+   * merge: New subcommand. Auto-merges the new git-annex branch.
+   * Improved handling of bare git repos with annexes. Many more commands will
+     work in them.
+   * git-annex is now more robust; it will never leave state files
+     uncommitted when some other git process comes along and locks the index
+     at an inconvenient time.
+   * rsync is now used when copying files from repos on other filesystems.
+     cp is still used when copying file from repos on the same filesystem,
+     since --reflink=auto can make it significantly faster on filesystems
+     such as btrfs.
+   * Allow --trust etc to specify a repository by name, for temporarily
+     trusting repositories that are not configured remotes.
+   * unlock: Made atomic.
+   * git-union-merge: New git subcommand, that does a generic union merge
+     operation, and operates efficiently without touching the working tree."""]]
\ No newline at end of file

From 45bce3a3a8706c12b8e1eb98a6fe0d62e378d10b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 26 Jun 2011 19:00:46 -0400
Subject: [PATCH 1949/8313] freebsd port now available

---
 doc/install.mdwn         | 1 +
 doc/install/FreeBSD.mdwn | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 doc/install/FreeBSD.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7818aaf152..1c038d0cf9 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -4,6 +4,7 @@
 * [[Debian]]
 * [[Ubuntu]]
 * [[Fedora]]
+* [[FreeBSD]]
 
 ## Generic instructions
 
diff --git a/doc/install/FreeBSD.mdwn b/doc/install/FreeBSD.mdwn
new file mode 100644
index 0000000000..23152ac204
--- /dev/null
+++ b/doc/install/FreeBSD.mdwn
@@ -0,0 +1,2 @@
+git-annex is in FreeBSD ports in
+[devel/git-annex](http://www.freshports.org/devel/git-annex/)

From c90652f015b370e270da8174b4ac61e454e06ffd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 26 Jun 2011 21:01:19 -0400
Subject: [PATCH 1950/8313] Always ensure git-annex branch exists.

---
 Branch.hs        | 7 ++++---
 debian/changelog | 6 ++++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 3f5e0ee1ec..916261a9c9 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -83,9 +83,10 @@ withIndex' bootstrapping a = do
 	reset <- liftIO $ Git.useIndex f
 
 	e <- liftIO $ doesFileExist f
-	unless e $ liftIO $ do
-		createDirectoryIfMissing True $ takeDirectory f
-		unless bootstrapping $ genIndex g
+	unless e $ do
+		unless bootstrapping $ create
+		liftIO $ createDirectoryIfMissing True $ takeDirectory f
+		liftIO $ unless bootstrapping $ genIndex g
 
 	r <- a
 	liftIO reset
diff --git a/debian/changelog b/debian/changelog
index 47652241a0..854314856d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110625) UNRELEASED; urgency=low
+
+  * Always ensure git-annex branch exists.
+
+ -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
+
 git-annex (3.20110624) experimental; urgency=low
 
   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.

From e8068f2ffbeb25bc094ecb3763da6ace278586cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 28 Jun 2011 14:14:49 -0400
Subject: [PATCH 1951/8313] tweaks

---
 Branch.hs | 42 ++++++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 916261a9c9..26aad44070 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -17,7 +17,7 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, when, liftM)
+import Control.Monad (unless, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
@@ -39,21 +39,23 @@ import Types
 import Messages
 import Locations
 
+type GitRef = String
+
 {- Name of the branch that is used to store git-annex's information. -}
-name :: String
+name :: GitRef
 name = "git-annex"
 
 {- Fully qualified name of the branch. -}
-fullname :: String
+fullname :: GitRef
 fullname = "refs/heads/" ++ name
 
 {- Branch's name in origin. -}
-originname :: String
+originname :: GitRef
 originname = "origin/" ++ name
 
 {- Converts a fully qualified git ref into a short version for human
  - consumptiom. -}
-shortref :: String -> String
+shortref :: GitRef -> String
 shortref = remove "refs/heads/" . remove "refs/remotes/"
 	where
 		remove prefix s
@@ -121,24 +123,20 @@ getCache file = getState >>= handle
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = do
-	exists <- refExists fullname
-	unless exists $ do
-		g <- Annex.gitRepo
-		e <- hasOrigin
-		if e
-			then liftIO $ Git.run g "branch" [Param name, Param originname]
-			else withIndex' True $
-				liftIO $ GitUnionMerge.commit g "branch created" fullname []
+create = unlessM (refExists fullname) $ do
+	g <- Annex.gitRepo
+	e <- hasOrigin
+	if e
+		then liftIO $ Git.run g "branch" [Param name, Param originname]
+		else withIndex' True $
+			liftIO $ GitUnionMerge.commit g "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = do
-	staged <- stageJournalFiles
-	when staged $ do
-		g <- Annex.gitRepo
-		withIndex $ liftIO $
-			GitUnionMerge.commit g message fullname [fullname]
+commit message = whenM stageJournalFiles $ do
+	g <- Annex.gitRepo
+	withIndex $ liftIO $
+		GitUnionMerge.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
@@ -171,14 +169,14 @@ hasOrigin :: Annex Bool
 hasOrigin = refExists originname
 
 {- Checks if a git ref exists. -}
-refExists :: String -> Annex Bool
+refExists :: GitRef -> Annex Bool
 refExists ref = do
 	g <- Annex.gitRepo
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
 {- Ensures that a given ref has been merged into the index. -}
-updateRef :: String -> Annex (Maybe String)
+updateRef :: GitRef -> Annex (Maybe String)
 updateRef ref
 	| ref == fullname = return Nothing
 	| otherwise = do

From 5034d8c2985dafeb141bba383ab70d75729b3cb6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 28 Jun 2011 16:15:50 -0400
Subject: [PATCH 1952/8313] Modify location log parser to allow future
 expansion.

Since the logs have just been moved into the git-annex branch, don't need
to worry about backwards compatability with old versions of git-annex that
would fail to parse location logs with extra fields tacked on.
---
 LocationLog.hs                    | 4 +++-
 debian/changelog                  | 1 +
 doc/todo/tahoe_lfs_for_reals.mdwn | 7 -------
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index 4e2caca959..2a2bc63013 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -8,6 +8,8 @@
  -
  - A line of the log will look like: "date N UUID"
  - Where N=1 when the repo has the file, and 0 otherwise.
+ - (After the UUID can optionally come a white space and other data, 
+ - for future expansion.)
  - 
  - Copyright 2010-2011 Joey Hess 
  -
@@ -65,7 +67,7 @@ instance Read LogLine where
 	-- read without an exception being thrown.
 	-- Such lines have a status of Undefined.
 	readsPrec _ string = 
-		if length w == 3
+		if length w >= 3
 			then maybe bad good pdate
 			else bad
 		where
diff --git a/debian/changelog b/debian/changelog
index 854314856d..688a4adb85 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110625) UNRELEASED; urgency=low
 
   * Always ensure git-annex branch exists.
+  * Modify location log parser to allow future expansion.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
 
diff --git a/doc/todo/tahoe_lfs_for_reals.mdwn b/doc/todo/tahoe_lfs_for_reals.mdwn
index 4e0bf2fb16..9019767eb9 100644
--- a/doc/todo/tahoe_lfs_for_reals.mdwn
+++ b/doc/todo/tahoe_lfs_for_reals.mdwn
@@ -19,10 +19,3 @@ location log:
 
 This way, each remote can store its own key-specfic data in the same place
 as other key-specific data, with minimal overhead.
-
-Unfortunatly, the current location log parser throws out lines that it 
-cannot parse, so making this change would involve something of a flag day
-upgrade. Also unfortunatly, the location log and other .git-annex/ data
-does not have its own version that can be checked to force an upgrade
-across all clones. It might be best to deal with this at the same time
-the ideas in [[branching]] are done. --[[Joey]]

From fb7663ceb86b309dd223ebc113c8b586955a0e03 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Wed, 29 Jun 2011 14:46:41 +0000
Subject: [PATCH 1953/8313]

---
 ..._annex_should_use___39__git_add_-f__39___internally.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn

diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
new file mode 100644
index 0000000000..66b3aa36a3
--- /dev/null
+++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
@@ -0,0 +1,6 @@
+I have this line in the .gitignore file of one of my repos:
+*log
+
+So the command 'git annex init name' fails to add the file ".git-annex/uuid.log", and the same problem happens when git-annex-add'ing files.
+
+Also, when a file is git-ignored, it should be possible to 'git annex add' it with a -f/--force option, the same way git does it.

From b3aaf980e460c2287fc1ef2b262685b1879e6ed0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 11:42:00 -0400
Subject: [PATCH 1954/8313] --force will cause add, etc, to operate on ignored
 files.

---
 Command.hs                                         |  3 ++-
 Command/Add.hs                                     |  6 +++++-
 GitRepo.hs                                         | 14 ++++++++------
 debian/changelog                                   |  1 +
 ...ould_use___39__git_add_-f__39___internally.mdwn |  5 +++++
 doc/git-annex.mdwn                                 |  3 ++-
 6 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/Command.hs b/Command.hs
index 228c1f40e9..ee81bd0f6f 100644
--- a/Command.hs
+++ b/Command.hs
@@ -143,7 +143,8 @@ withFilesMissing a params = do
 withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
-	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo) params
+	force <- Annex.getState Annex.force
+	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo force) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withWords :: CommandSeekWords
diff --git a/Command/Add.hs b/Command/Add.hs
index 29a1518e84..5133ee1fd5 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -11,6 +11,7 @@ import Control.Monad.State (liftIO)
 import System.Posix.Files
 
 import Command
+import qualified Annex
 import qualified AnnexQueue
 import qualified Backend
 import LocationLog
@@ -60,5 +61,8 @@ cleanup file key = do
 	let mtime = modificationTime s
 	liftIO $ touch file (TimeSpec mtime) False
 
-	AnnexQueue.add "add" [Param "--"] file
+	force <- Annex.getState Annex.force
+	if force
+		then AnnexQueue.add "add" [Param "-f", Param "--"] file
+		else AnnexQueue.add "add" [Param "--"] file
 	return True
diff --git a/GitRepo.hs b/GitRepo.hs
index 9f4a38a5fb..844917b880 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -382,13 +382,15 @@ reap = do
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --cached --exclude-standard -z --"] ++ map File l
+	[Params "ls-files --cached -z --"] ++ map File l
 
-{- Scans for files at the specified locations that are not checked into git,
- - and not gitignored. -}
-notInRepo :: Repo -> [FilePath] -> IO [FilePath]
-notInRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --others --exclude-standard -z --"] ++ map File l
+{- Scans for files at the specified locations that are not checked into
+ - git. -}
+notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
+notInRepo repo include_ignored l =
+	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
+	where
+		exclude = if include_ignored then [] else [Param "--exclude-standard"]
 
 {- Returns a list of all files that are staged for commit. -}
 stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/debian/changelog b/debian/changelog
index 6439eb89fa..8848cf8611 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,7 @@ git-annex (0.20110611) UNRELEASED; urgency=low
   * unlock: Made atomic.
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
+  * --force will cause add, etc, to operate on ignored files.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
index 66b3aa36a3..a92f5871b5 100644
--- a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
+++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn
@@ -3,4 +3,9 @@ I have this line in the .gitignore file of one of my repos:
 
 So the command 'git annex init name' fails to add the file ".git-annex/uuid.log", and the same problem happens when git-annex-add'ing files.
 
+> This is avoided on the v3 branch, which does not store these files in the
+> same branch as your repository.
+
 Also, when a file is git-ignored, it should be possible to 'git annex add' it with a -f/--force option, the same way git does it.
+
+> Reasonable, [[done]] --[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 12756d8020..177df09527 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -314,7 +314,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * --force
 
   Force unsafe actions, such as dropping a file's content when no other
-  source of it can be verified to still exist. Use with care.
+  source of it can be verified to still exist, or adding ignored files.
+  Use with care.
 
 * --fast
 

From 06a1f5f74286795708b219de8fb080077ff134a7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 11:55:16 -0400
Subject: [PATCH 1955/8313] factor out file list stuff from GitRepo

GitRepo is getting too large an interface; these all fit nicely into a
submodule.
---
 Command.hs         | 15 +++++-----
 Command/Unannex.hs |  3 +-
 Command/Unused.hs  |  3 +-
 GitRepo.hs         | 56 --------------------------------------
 GitRepo/LsFiles.hs | 68 ++++++++++++++++++++++++++++++++++++++++++++++
 Upgrade/V1.hs      |  3 +-
 6 files changed, 82 insertions(+), 66 deletions(-)
 create mode 100644 GitRepo/LsFiles.hs

diff --git a/Command.hs b/Command.hs
index a8cc6a132a..45a4cc70fd 100644
--- a/Command.hs
+++ b/Command.hs
@@ -22,6 +22,7 @@ import qualified Backend
 import Messages
 import qualified Annex
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import Utility
 import Types.Key
 
@@ -118,17 +119,17 @@ notBareRepo a = do
 withFilesInGit :: CommandSeekStrings
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ filterFiles files
 withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (Git.inRepo repo) params
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
 withFilesMissing :: CommandSeekStrings
@@ -143,7 +144,7 @@ withFilesNotInGit :: CommandSeekBackendFiles
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
 	force <- Annex.getState Annex.force
-	newfiles <- liftIO $ runPreserveOrder (Git.notInRepo repo force) params
+	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
 withWords :: CommandSeekWords
@@ -153,12 +154,12 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: CommandSeekStrings
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserveOrder (Git.stagedFilesNotDeleted repo) params
+	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 	liftM (map a) $ filterFiles tocommit
 withFilesUnlocked :: CommandSeekBackendFiles
-withFilesUnlocked = withFilesUnlocked' Git.typeChangedFiles
+withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles
-withFilesUnlockedToBeCommitted = withFilesUnlocked' Git.typeChangedStagedFiles
+withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 0a5381d562..0de98b1d3f 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -20,6 +20,7 @@ import LocationLog
 import Types
 import Content
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import Messages
 
 command :: [Command]
@@ -37,7 +38,7 @@ start file = isAnnexed file $ \(key, backend) -> do
 			force <- Annex.getState Annex.force
 			unless force $ do
 				g <- Annex.gitRepo
-				staged <- liftIO $ Git.stagedFiles g [Git.workTree g]
+				staged <- liftIO $ LsFiles.staged g [Git.workTree g]
 				unless (null staged) $
 					error "This command cannot be run when there are already files staged for commit."
 				Annex.changeState $ \s -> s { Annex.force = True }
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5744f84fd4..c0a3471796 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -23,6 +23,7 @@ import Utility
 import LocationLog
 import qualified Annex
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import qualified Backend
 import qualified Remote
 
@@ -175,7 +176,7 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 getKeysReferenced :: Annex [Key]
 getKeysReferenced = do
 	g <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo g [Git.workTree g]
+	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
 
diff --git a/GitRepo.hs b/GitRepo.hs
index cfe949d5e6..cc4636868d 100644
--- a/GitRepo.hs
+++ b/GitRepo.hs
@@ -47,16 +47,9 @@ module GitRepo (
 	remotesAdd,
 	repoRemoteName,
 	repoRemoteNameSet,
-	inRepo,
-	notInRepo,
-	stagedFiles,
-	stagedFilesNotDeleted,
-	changedUnstagedFiles,
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
-	typeChangedFiles,
-	typeChangedStagedFiles,
 	repoAbsPath,
 	reap,
 	useIndex,
@@ -432,55 +425,6 @@ getSha subcommand a = do
 shaSize :: Int
 shaSize = 40
 
-{- Scans for files that are checked into git at the specified locations. -}
-inRepo :: Repo -> [FilePath] -> IO [FilePath]
-inRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --cached -z --"] ++ map File l
-
-{- Scans for files at the specified locations that are not checked into
- - git. -}
-notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
-notInRepo repo include_ignored l =
-	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
-	where
-		exclude = if include_ignored then [] else [Param "--exclude-standard"]
-
-{- Returns a list of all files that are staged for commit. -}
-stagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-stagedFiles repo l = stagedFiles' repo l []
-
-{- Returns a list of the files, staged for commit, that are being added,
- - moved, or changed (but not deleted), from the specified locations. -}
-stagedFilesNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
-stagedFilesNotDeleted repo l = stagedFiles' repo l [Param "--diff-filter=ACMRT"]
-
-stagedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-stagedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
-	where
-		start = [Params "diff --cached --name-only -z"]
-		end = [Param "--"] ++ map File l
-
-{- Returns a list of files that have unstaged changes. -}
-changedUnstagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-changedUnstagedFiles repo l = pipeNullSplit repo $
-	[Params "diff --name-only -z --"] ++ map File l
-
-{- Returns a list of the files in the specified locations that are staged
- - for commit, and whose type has changed. -}
-typeChangedStagedFiles :: Repo -> [FilePath] -> IO [FilePath]
-typeChangedStagedFiles repo l = typeChangedFiles' repo l [Param "--cached"]
-
-{- Returns a list of the files in the specified locations whose type has
- - changed.  Files only staged for commit will not be included. -}
-typeChangedFiles :: Repo -> [FilePath] -> IO [FilePath]
-typeChangedFiles repo l = typeChangedFiles' repo l []
-
-typeChangedFiles' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-typeChangedFiles' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
-	where
-		start = [Params "diff --name-only --diff-filter=T -z"]
-		end = [Param "--"] ++ map File l
-
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
diff --git a/GitRepo/LsFiles.hs b/GitRepo/LsFiles.hs
new file mode 100644
index 0000000000..16604c319a
--- /dev/null
+++ b/GitRepo/LsFiles.hs
@@ -0,0 +1,68 @@
+{- git ls-files interface
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module GitRepo.LsFiles (
+	inRepo,
+	notInRepo,
+	staged,
+	stagedNotDeleted,
+	changedUnstaged,
+	typeChanged,
+	typeChangedStaged,
+) where
+
+import GitRepo
+import Utility
+
+{- Scans for files that are checked into git at the specified locations. -}
+inRepo :: Repo -> [FilePath] -> IO [FilePath]
+inRepo repo l = pipeNullSplit repo $
+	[Params "ls-files --cached -z --"] ++ map File l
+
+{- Scans for files at the specified locations that are not checked into
+ - git. -}
+notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
+notInRepo repo include_ignored l =
+	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
+	where
+		exclude = if include_ignored then [] else [Param "--exclude-standard"]
+
+{- Returns a list of all files that are staged for commit. -}
+staged :: Repo -> [FilePath] -> IO [FilePath]
+staged repo l = staged' repo l []
+
+{- Returns a list of the files, staged for commit, that are being added,
+ - moved, or changed (but not deleted), from the specified locations. -}
+stagedNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
+stagedNotDeleted repo l = staged' repo l [Param "--diff-filter=ACMRT"]
+
+staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
+staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+	where
+		start = [Params "diff --cached --name-only -z"]
+		end = [Param "--"] ++ map File l
+
+{- Returns a list of files that have unstaged changes. -}
+changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]
+changedUnstaged repo l = pipeNullSplit repo $
+	[Params "diff --name-only -z --"] ++ map File l
+
+{- Returns a list of the files in the specified locations that are staged
+ - for commit, and whose type has changed. -}
+typeChangedStaged :: Repo -> [FilePath] -> IO [FilePath]
+typeChangedStaged repo l = typeChanged' repo l [Param "--cached"]
+
+{- Returns a list of the files in the specified locations whose type has
+ - changed.  Files only staged for commit will not be included. -}
+typeChanged :: Repo -> [FilePath] -> IO [FilePath]
+typeChanged repo l = typeChanged' repo l []
+
+typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
+typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+	where
+		start = [Params "diff --name-only --diff-filter=T -z"]
+		end = [Param "--"] ++ map File l
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b06f00d34d..cb6928fcee 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -26,6 +26,7 @@ import LocationLog
 import qualified Annex
 import qualified AnnexQueue
 import qualified GitRepo as Git
+import qualified GitRepo.LsFiles as LsFiles
 import Backend
 import Messages
 import Version
@@ -92,7 +93,7 @@ updateSymlinks :: Annex ()
 updateSymlinks = do
 	showNote "updating symlinks..."
 	g <- Annex.gitRepo
-	files <- liftIO $ Git.inRepo g [Git.workTree g]
+	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	forM_ files $ fixlink
 	where
 		fixlink f = do

From 8725fde5c66984d9769558a07612361b112be58f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 20:22:19 -0400
Subject: [PATCH 1956/8313] new plan

---
 doc/todo/speed_up_fsck.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
index aceb5868cc..e22c01766d 100644
--- a/doc/todo/speed_up_fsck.mdwn
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -5,6 +5,8 @@ are slightly slower but are swamped by the normal runtime.
 For fsck though, it has to pull each file's location log info out of git.
 And, it's typically run on the entire tree.
 
+Another slow one in `git annex copy --from`.
+
 It would be possible to run a single `git cat-file --batch` and pass it
 sha1s of location logs for file that is going to be fsked (gotten via
 `read-tree`). Then just read its output until the next requested sha1 to
@@ -16,3 +18,21 @@ provide the info on a side channel of some sort.
 
 If this is implemented, the same infrastructure could be used for other
 commands like whereis and add. --[[Joey]]
+
+> Updated plan:
+> 
+> Run `git ls-file --batch`, and cache its stdin and out handles in Branch
+> state.
+> 
+> To see a git-annex branch file, send it something like
+> "git-annex:uuid.log", and read the content fron stdout handle.
+> 
+> To detect the end of content, send "TOKEN\n", and look for 
+> "TOKEN missing" in its output. A good choice for TOKEN is anything
+> that will never exist in the repo; 40 0's would be a fairly good choice,
+> but even better seems to be something completely invalid and impossible
+> to have as a sha1 or filename or ref: "".
+> 
+> Hmm, except that's actually an error message sent to stderr. Unless
+> stderr is connected to stdout, it might be better to look for a known,
+> empty object. Could just add a git-annex:empty file to that end.

From e1c18ddec455e5d1259ab46ccccbe6a9c7079de6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 21:23:40 -0400
Subject: [PATCH 1957/8313] Sped back up fsck, copy --from etc

All commands that often have to read a lot of information from
the git-annex branch should now be nearly as fast as before
the branch was introduced.

Before fsck was taking approximatly 3 hours, now it's running in 8 minutes.

The code is very nasty. It should be rewritten to read the header line
from git cat-file, and then read the specified number of bytes of content.
---
 Branch.hs                   | 74 ++++++++++++++++++++-----------------
 Types/BranchState.hs        | 11 +++++-
 debian/changelog            |  3 ++
 doc/todo/speed_up_fsck.mdwn |  2 +
 4 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 26aad44070..4f568e36bc 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,9 +26,6 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
-import System.Posix.IO
-import System.Posix.Process
-import System.Log.Logger
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -142,7 +139,7 @@ commit message = whenM stageJournalFiles $ do
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
-	state <- Annex.getState Annex.branchstate
+	state <- getState
 	unless (branchUpdated state) $ withIndex $ do
 		{- Since branches get merged into the index, it's important to
 		 - first stage the journal into the index. Otherwise, any
@@ -226,39 +223,48 @@ get file = do
 					setCache file content
 					return content
 				Nothing -> withIndexUpdate $ do
-					g <- Annex.gitRepo
-					content <- liftIO $ catch (cat g) (const $ return "")
+					content <- catFile file
 					setCache file content
 					return content
+
+{- Uses git cat-file in batch mode to read the content of a file.
+ -
+ - Only one process is run, and it persists and is used for all accesses. -}
+catFile :: FilePath -> Annex String
+catFile file = do
+	state <- getState
+	maybe (startup state) ask (catFileHandles state)
 	where
-		cat g = cmdOutput "git" $ toCommand $ Git.gitCommandLine g
-			[Param "cat-file", Param "blob", Param $ ':':file]
-
-{- Runs a command, returning its output, ignoring nonzero exit
- - status, and discarding stderr. -}
-cmdOutput :: FilePath -> [String] -> IO String
-cmdOutput cmd params = do
-	pipepair <- createPipe
-	let callfunc _ = do
-		closeFd (snd pipepair)
-		h <- fdToHandle (fst pipepair)
-		x <- hGetContentsStrict h
-		hClose h
-		return $! x
-	let child = do
-		closeFd (fst pipepair)
- 		-- disable stderr output by this child,
-		-- and since the logger uses it, also disable it
-		liftIO $ updateGlobalLogger rootLoggerName $ setLevel EMERGENCY
-		closeFd stdError
-
-	debugM "Utility.executeFile" $ cmd ++ " " ++ show params
-	
-	pid <- pOpen3Raw Nothing (Just (snd pipepair)) Nothing cmd params child
-	retval <- callfunc $! pid
-	let rv = seq retval retval
-	_ <- getProcessStatus True False pid
-	return rv
+		startup state = do
+			g <- Annex.gitRepo
+			let cmd = Git.gitCommandLine g
+				[Param "cat-file", Param "--batch"]
+			let gitcmd = join " " $ "git" : toCommand cmd
+			(_, from, to) <- liftIO $ hPipeBoth "sh"
+				-- want stderr on stdin for sentinal, and
+				-- to ignore other error messages
+				["-c", gitcmd ++ " 2>&1"]
+			setState state { catFileHandles = Just (from, to) }
+			ask (from, to)
+		ask (from, to) = do
+			_ <- liftIO $ do
+				hPutStr to $
+					fullname ++ ":" ++ file ++ "\n" ++
+					sentinal ++ "\n"
+				hFlush to
+			return . unlines =<< readContent from []
+		readContent from ls = do
+			l <- liftIO $ hGetLine from
+			if l == sentinal_line
+				-- first line is blob info,
+				-- or maybe an error message
+				then return $ drop 1 $ reverse ls
+				else readContent from (l:ls)
+		-- To find the end of a catted file, ask for a sentinal
+		-- value that is always missing, and look for the error
+		-- message. Utterly nasty, probably will break one day.
+		sentinal = ":"
+		sentinal_line = sentinal ++ " missing"
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index 40d7f5c2c7..bc1d32e693 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -7,11 +7,18 @@
 
 module Types.BranchState where
 
+import System.IO
+
 data BranchState = BranchState {
-	branchUpdated :: Bool,
+	branchUpdated :: Bool, -- has the branch been updated this run?
+
+	-- (from, to) handles used to talk to a git-cat-file process
+	catFileHandles :: Maybe (Handle, Handle),
+
+	-- the content of one file is cached
 	cachedFile :: Maybe FilePath,
 	cachedContent :: String
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False Nothing ""
+startBranchState = BranchState False Nothing Nothing ""
diff --git a/debian/changelog b/debian/changelog
index 5dff0bbe56..a87a89860a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (3.20110625) UNRELEASED; urgency=low
   * Always ensure git-annex branch exists.
   * Modify location log parser to allow future expansion.
   * --force will cause add, etc, to operate on ignored files.
+  * Sped back up fsck, copy --from, and other commands that often
+    have to read a lot of information from the git-annex branch. Should
+    now be nearly as fast as before the branch was introduced.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
 
diff --git a/doc/todo/speed_up_fsck.mdwn b/doc/todo/speed_up_fsck.mdwn
index e22c01766d..5d5e867f80 100644
--- a/doc/todo/speed_up_fsck.mdwn
+++ b/doc/todo/speed_up_fsck.mdwn
@@ -36,3 +36,5 @@ commands like whereis and add. --[[Joey]]
 > Hmm, except that's actually an error message sent to stderr. Unless
 > stderr is connected to stdout, it might be better to look for a known,
 > empty object. Could just add a git-annex:empty file to that end.
+
+[[done]] --[[Joey]] 

From 899ecbfba1c015c2c80f729c7e0d5544d7bcc415 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 22:19:40 -0400
Subject: [PATCH 1958/8313] improve git cat-file code

Now it reads the size specified, rather than using the sentinal hack to
determine EOF.

It still depends on error messages to handle files that are not present.
---
 Branch.hs | 58 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 4f568e36bc..033b7c6d02 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -17,7 +17,7 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, liftM)
+import Control.Monad (when, unless, liftM)
 import Control.Monad.State (liftIO)
 import System.FilePath
 import System.Directory
@@ -26,6 +26,9 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
+import System.IO.Unsafe
+import Foreign
+import Data.Char
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -239,32 +242,39 @@ catFile file = do
 			g <- Annex.gitRepo
 			let cmd = Git.gitCommandLine g
 				[Param "cat-file", Param "--batch"]
-			let gitcmd = join " " $ "git" : toCommand cmd
+			let gitcmd = join " " ("git" : toCommand cmd)
 			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				-- want stderr on stdin for sentinal, and
-				-- to ignore other error messages
-				["-c", gitcmd ++ " 2>&1"]
+				-- want stderr on stdin to handle error messages
+				["-c", "LANG=C exec " ++ gitcmd ++ " 2>&1"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
-		ask (from, to) = do
-			_ <- liftIO $ do
-				hPutStr to $
-					fullname ++ ":" ++ file ++ "\n" ++
-					sentinal ++ "\n"
-				hFlush to
-			return . unlines =<< readContent from []
-		readContent from ls = do
-			l <- liftIO $ hGetLine from
-			if l == sentinal_line
-				-- first line is blob info,
-				-- or maybe an error message
-				then return $ drop 1 $ reverse ls
-				else readContent from (l:ls)
-		-- To find the end of a catted file, ask for a sentinal
-		-- value that is always missing, and look for the error
-		-- message. Utterly nasty, probably will break one day.
-		sentinal = ":"
-		sentinal_line = sentinal ++ " missing"
+		ask (from, to) = liftIO $ do
+			let want = fullname ++ ":" ++ file
+			hPutStrLn to want
+			hFlush to
+			header <- hGetLine from
+			if header == want ++ " missing"
+				then return ""
+				else do
+					let [_sha, _type, size] = words header
+					let bytes = read size
+					fp <- mallocForeignPtrBytes (fromIntegral bytes)
+					len <- withForeignPtr fp $ \buf -> hGetBuf from buf (fromIntegral bytes)
+					when (len /= bytes) $
+						error "short read from git cat-file"
+					content <- lazySlurp fp 0 len
+					c <- hGetChar from
+					when (c /= '\n') $
+						error "missing newline from git cat-file"
+					return content
+
+lazySlurp :: ForeignPtr Word8 -> Int -> Int -> IO String
+lazySlurp fp ix len
+	| ix == len = return []
+	| otherwise = do
+		c <- withForeignPtr fp $ \p -> peekElemOff p ix
+		cs <- unsafeInterleaveIO (lazySlurp fp (ix+1) len)
+		return $ chr (fromIntegral c) : cs
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]

From b089fba7b4e68c4482909319f562d32a3f00f379 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Jun 2011 23:56:47 -0400
Subject: [PATCH 1959/8313] use ByteString for hGet

Avoids the crazy low-level hGetBuf stuff. Also slightly faster.
---
 Branch.hs | 20 +++-----------------
 1 file changed, 3 insertions(+), 17 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 033b7c6d02..9cdb096fd1 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,9 +26,7 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
-import System.IO.Unsafe
-import Foreign
-import Data.Char
+import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
 import qualified GitRepo as Git
@@ -258,23 +256,11 @@ catFile file = do
 				else do
 					let [_sha, _type, size] = words header
 					let bytes = read size
-					fp <- mallocForeignPtrBytes (fromIntegral bytes)
-					len <- withForeignPtr fp $ \buf -> hGetBuf from buf (fromIntegral bytes)
-					when (len /= bytes) $
-						error "short read from git cat-file"
-					content <- lazySlurp fp 0 len
+					content <- B.hGet from bytes
 					c <- hGetChar from
 					when (c /= '\n') $
 						error "missing newline from git cat-file"
-					return content
-
-lazySlurp :: ForeignPtr Word8 -> Int -> Int -> IO String
-lazySlurp fp ix len
-	| ix == len = return []
-	| otherwise = do
-		c <- withForeignPtr fp $ \p -> peekElemOff p ix
-		cs <- unsafeInterleaveIO (lazySlurp fp (ix+1) len)
-		return $ chr (fromIntegral c) : cs
+					return $ B.unpack content
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]

From d72fb5acc28af3ea6380dd09518f7d1382dea8d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 00:35:51 -0400
Subject: [PATCH 1960/8313] Fix encoding of utf-8 etc when storing the
 description of repository and other content.

Write files in raw mode, to avoid mangling the encoding of content
provided.

Note: This was a longstanding problem, it was not introduced in v3.
---
 Branch.hs        | 3 ++-
 debian/changelog | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Branch.hs b/Branch.hs
index 9cdb096fd1..ab24e47525 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,6 +26,7 @@ import System.Cmd.Utils
 import Data.Maybe
 import Data.List
 import System.IO
+import System.IO.Binary
 import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
@@ -287,7 +288,7 @@ setJournalFile file content = do
 		write g = do
 			let jfile = journalFile g file
 			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
-			writeFile tmpfile content
+			writeBinaryFile tmpfile content
 			renameFile tmpfile jfile
 
 {- Gets journalled content for a file in the branch. -}
diff --git a/debian/changelog b/debian/changelog
index a87a89860a..f84f8ab498 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20110625) UNRELEASED; urgency=low
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Should
     now be nearly as fast as before the branch was introduced.
+  * Fix encoding of utf-8 etc when storing the description of repository
+    and other content.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
 

From 2cda9d0a0fcdd1cc2aebc066ef19282fbe36e898 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 00:42:09 -0400
Subject: [PATCH 1961/8313] generalized safeWriteFile to viaTmp

---
 Command/Init.hs   |  2 +-
 Command/Unused.hs |  2 +-
 Upgrade/V2.hs     |  2 +-
 Utility.hs        | 11 ++++++-----
 4 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 7f5773117a..8cde8bb9fb 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -55,7 +55,7 @@ gitPreCommitHookWrite repo = do
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
 		else liftIO $ do
-			safeWriteFile hook preCommitScript
+			viaTmp writeFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
 	where
diff --git a/Command/Unused.hs b/Command/Unused.hs
index c0a3471796..a9d5f90a10 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -86,7 +86,7 @@ checkRemoteUnused' r = do
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
 	g <- Annex.gitRepo
-	liftIO $ safeWriteFile (gitAnnexUnusedLog prefix g) $
+	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
 table :: [(Int, Key)] -> [String]
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 8537a50226..ea68e78c95 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -128,7 +128,7 @@ gitAttributesUnWrite repo = do
 	let attributes = Git.attributes repo
 	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
-		liftIO $ safeWriteFile attributes $ unlines $
+		liftIO $ viaTmp writeFile attributes $ unlines $
 			filter (\l -> not $ l `elem` attrLines) $ lines c
 		Git.run repo "add" [File attributes]
 
diff --git a/Utility.hs b/Utility.hs
index 47d10ed759..7831a4ab4b 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -22,7 +22,7 @@ module Utility (
 	shellUnEscape,
 	unsetFileMode,
 	readMaybe,
-	safeWriteFile,
+	viaTmp,
 	dirContains,
 	dirContents,
 	myHomeDir,
@@ -243,13 +243,14 @@ readMaybe s = case reads s of
 	((x,_):_) -> Just x
 	_ -> Nothing
 
-{- Writes a file using a temp file that is renamed atomically into place. -}
-safeWriteFile :: FilePath -> String -> IO ()
-safeWriteFile file content = do
+{- Runs an action like writeFile, writing to a tmp file first and
+ - then moving it into place. -}
+viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
+viaTmp a file content = do
 	pid <- getProcessID
         let tmpfile = file ++ ".tmp" ++ show pid
 	createDirectoryIfMissing True (parentDir file)
-	writeFile tmpfile content
+	a tmpfile content
 	renameFile tmpfile file
 
 {- Lists the contents of a directory.

From 8562e6096ca9a6819c04b4fd1938202ccd68c701 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 01:16:53 -0400
Subject: [PATCH 1962/8313] v3 is now faster than v2

Rebenchmarked v2 vs v3, and v3 is now actually faster. Yes, storing data
in git, using git as a filesystem is actually faster than just using the
filesystem. If you do it just right. :)
---
 debian/changelog | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index f84f8ab498..9013abd135 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,12 +1,13 @@
 git-annex (3.20110625) UNRELEASED; urgency=low
 
+  * Sped back up fsck, copy --from, and other commands that often
+    have to read a lot of information from the git-annex branch. Such
+    commands are now faster than they were before introduction of the
+    git-annex branch.
   * Always ensure git-annex branch exists.
   * Modify location log parser to allow future expansion.
   * --force will cause add, etc, to operate on ignored files.
-  * Sped back up fsck, copy --from, and other commands that often
-    have to read a lot of information from the git-annex branch. Should
-    now be nearly as fast as before the branch was introduced.
-  * Fix encoding of utf-8 etc when storing the description of repository
+  * Avoid mangling encoding when storing the description of repository
     and other content.
 
  -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400

From 5fe02f280726442496303859e83f9ce1c48be0cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:12:51 -0400
Subject: [PATCH 1963/8313] more robust git cat-file output parser

Only remaining ugliness is the handling of error messages for files that
are not present on the branch.
---
 Branch.hs | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index ab24e47525..5ad89df202 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -243,8 +243,8 @@ catFile file = do
 				[Param "cat-file", Param "--batch"]
 			let gitcmd = join " " ("git" : toCommand cmd)
 			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				-- want stderr on stdin to handle error messages
-				["-c", "LANG=C exec " ++ gitcmd ++ " 2>&1"]
+				-- want stderr on stdin to see error messages
+				["-c", "exec " ++ gitcmd ++ " 2>&1"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
 		ask (from, to) = liftIO $ do
@@ -252,16 +252,22 @@ catFile file = do
 			hPutStrLn to want
 			hFlush to
 			header <- hGetLine from
-			if header == want ++ " missing"
-				then return ""
-				else do
-					let [_sha, _type, size] = words header
-					let bytes = read size
-					content <- B.hGet from bytes
-					c <- hGetChar from
-					when (c /= '\n') $
-						error "missing newline from git cat-file"
-					return $ B.unpack content
+			case words header of
+				[sha, blob, size]
+					| length sha == Git.shaSize &&
+					  blob == "blob" -> handle from size
+					| otherwise -> empty
+				_ -> empty
+		handle from size = case reads size of
+			[(bytes, "")] -> readcontent from bytes
+			_ -> empty
+		readcontent from bytes = do
+			content <- B.hGet from bytes
+			c <- hGetChar from
+			when (c /= '\n') $
+				error "missing newline from git cat-file"
+			return $ B.unpack content
+		empty = return ""
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]

From f6063a094ec02caec314b42dc05f2f0595ae0ce4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:16:57 -0400
Subject: [PATCH 1964/8313] renamed GitRepo to Git

It was always imported qualified as Git anyway
---
 Annex.hs                    | 2 +-
 Backend.hs                  | 2 +-
 Backend/File.hs             | 2 +-
 Branch.hs                   | 2 +-
 CmdLine.hs                  | 2 +-
 Command.hs                  | 4 ++--
 Command/DropUnused.hs       | 2 +-
 Command/Init.hs             | 2 +-
 Command/Map.hs              | 2 +-
 Command/Status.hs           | 2 +-
 Command/Unannex.hs          | 4 ++--
 Command/Uninit.hs           | 2 +-
 Command/Unused.hs           | 4 ++--
 Config.hs                   | 2 +-
 Content.hs                  | 2 +-
 GitRepo.hs => Git.hs        | 4 ++--
 {GitRepo => Git}/LsFiles.hs | 4 ++--
 GitAnnex.hs                 | 2 +-
 GitQueue.hs                 | 2 +-
 GitUnionMerge.hs            | 2 +-
 LocationLog.hs              | 2 +-
 Locations.hs                | 2 +-
 Remote/Bup.hs               | 2 +-
 Remote/Directory.hs         | 2 +-
 Remote/Git.hs               | 2 +-
 Remote/Hook.hs              | 2 +-
 Remote/Rsync.hs             | 2 +-
 Remote/S3real.hs            | 2 +-
 Remote/Special.hs           | 2 +-
 Ssh.hs                      | 2 +-
 Types/Remote.hs             | 2 +-
 UUID.hs                     | 2 +-
 Upgrade/V1.hs               | 4 ++--
 Upgrade/V2.hs               | 2 +-
 Version.hs                  | 2 +-
 git-annex-shell.hs          | 2 +-
 git-annex.hs                | 4 +---
 git-union-merge.hs          | 2 +-
 test.hs                     | 2 +-
 39 files changed, 45 insertions(+), 47 deletions(-)
 rename GitRepo.hs => Git.hs (99%)
 rename {GitRepo => Git}/LsFiles.hs (98%)

diff --git a/Annex.hs b/Annex.hs
index 82908881da..e2f4a10206 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -18,7 +18,7 @@ module Annex (
 
 import Control.Monad.State
 
-import qualified GitRepo as Git
+import qualified Git
 import GitQueue
 import Types.Backend
 import Types.Remote
diff --git a/Backend.hs b/Backend.hs
index 78a53d02c7..b1cd4c8f0e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -39,7 +39,7 @@ import System.Posix.Files
 import System.Directory
 
 import Locations
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import Types
 import Types.Key
diff --git a/Backend/File.hs b/Backend/File.hs
index 1c25f89db2..b8d3bb65ad 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -20,7 +20,7 @@ import Data.String.Utils
 import Types.Backend
 import LocationLog
 import qualified Remote
-import qualified GitRepo as Git
+import qualified Git
 import Content
 import qualified Annex
 import Types
diff --git a/Branch.hs b/Branch.hs
index 5ad89df202..7650fe8619 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -30,7 +30,7 @@ import System.IO.Binary
 import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
-import qualified GitRepo as Git
+import qualified Git
 import qualified GitUnionMerge
 import qualified Annex
 import Utility
diff --git a/CmdLine.hs b/CmdLine.hs
index ab7236847f..85423e5e89 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -18,7 +18,7 @@ import Control.Monad (when)
 
 import qualified Annex
 import qualified AnnexQueue
-import qualified GitRepo as Git
+import qualified Git
 import Content
 import Types
 import Command
diff --git a/Command.hs b/Command.hs
index 45a4cc70fd..d36f675d24 100644
--- a/Command.hs
+++ b/Command.hs
@@ -21,8 +21,8 @@ import Types
 import qualified Backend
 import Messages
 import qualified Annex
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import Utility
 import Types.Key
 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 0f99814471..2125abdc3c 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -20,7 +20,7 @@ import qualified Annex
 import qualified Command.Drop
 import qualified Command.Move
 import qualified Remote
-import qualified GitRepo as Git
+import qualified Git
 import Backend
 import Types.Key
 import Utility
diff --git a/Command/Init.hs b/Command/Init.hs
index 8cde8bb9fb..71e87050d8 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -13,7 +13,7 @@ import System.Directory
 
 import Command
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import UUID
 import Version
diff --git a/Command/Map.hs b/Command/Map.hs
index 7a9121b69b..7bb435ff81 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -15,7 +15,7 @@ import Data.List.Utils
 
 import Command
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import Messages
 import Types
 import Utility
diff --git a/Command/Status.hs b/Command/Status.hs
index 3b096d9793..53589030b3 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -18,7 +18,7 @@ import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
-import qualified GitRepo as Git
+import qualified Git
 import Command
 import Types
 import DataUnits
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 0de98b1d3f..fe413b25b4 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -19,8 +19,8 @@ import qualified Backend
 import LocationLog
 import Types
 import Content
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import Messages
 
 command :: [Command]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 9698ed8200..c47ac0c3af 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -14,7 +14,7 @@ import Command
 import Messages
 import Types
 import Utility
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import qualified Command.Init
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a9d5f90a10..3f51e2c2c2 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -22,8 +22,8 @@ import Locations
 import Utility
 import LocationLog
 import qualified Annex
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import qualified Backend
 import qualified Remote
 
diff --git a/Config.hs b/Config.hs
index a324427d44..1016845e75 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,7 +10,7 @@ module Config where
 import Data.Maybe
 import Control.Monad.State (liftIO)
 
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import Types
 import Utility
diff --git a/Content.hs b/Content.hs
index d733ad8b36..d24fc9ac70 100644
--- a/Content.hs
+++ b/Content.hs
@@ -36,7 +36,7 @@ import Types
 import Locations
 import LocationLog
 import UUID
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
diff --git a/GitRepo.hs b/Git.hs
similarity index 99%
rename from GitRepo.hs
rename to Git.hs
index cc4636868d..2b0cdd06d5 100644
--- a/GitRepo.hs
+++ b/Git.hs
@@ -3,12 +3,12 @@
  - This is written to be completely independant of git-annex and should be
  - suitable for other uses.
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitRepo (
+module Git (
 	Repo,
 	repoFromCwd,
 	repoFromAbsPath,
diff --git a/GitRepo/LsFiles.hs b/Git/LsFiles.hs
similarity index 98%
rename from GitRepo/LsFiles.hs
rename to Git/LsFiles.hs
index 16604c319a..b88c9144e6 100644
--- a/GitRepo/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitRepo.LsFiles (
+module Git.LsFiles (
 	inRepo,
 	notInRepo,
 	staged,
@@ -15,7 +15,7 @@ module GitRepo.LsFiles (
 	typeChangedStaged,
 ) where
 
-import GitRepo
+import Git
 import Utility
 
 {- Scans for files that are checked into git at the specified locations. -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 49ba63b8ab..58b512f718 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -9,7 +9,7 @@ module GitAnnex where
 
 import System.Console.GetOpt
 
-import qualified GitRepo as Git
+import qualified Git
 import CmdLine
 import Command
 import Options
diff --git a/GitQueue.hs b/GitQueue.hs
index be0fcfc4af..5da3ba1d6e 100644
--- a/GitQueue.hs
+++ b/GitQueue.hs
@@ -21,7 +21,7 @@ import Data.String.Utils
 import Control.Monad (unless, forM_)
 import Utility
 
-import qualified GitRepo as Git
+import qualified Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
diff --git a/GitUnionMerge.hs b/GitUnionMerge.hs
index 74579ebcc9..84c7b44eae 100644
--- a/GitUnionMerge.hs
+++ b/GitUnionMerge.hs
@@ -18,7 +18,7 @@ import Data.List
 import Data.Maybe
 import Data.String.Utils
 
-import qualified GitRepo as Git
+import qualified Git
 import Utility
 
 {- Performs a union merge between two branches, staging it in the index.
diff --git a/LocationLog.hs b/LocationLog.hs
index 2a2bc63013..b7deb3ed9a 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -34,7 +34,7 @@ import Control.Monad (when)
 import Data.Maybe
 import Control.Monad.State (liftIO)
 
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import UUID
 import Types
diff --git a/Locations.hs b/Locations.hs
index bfb0d3af9e..6142b2393a 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -36,7 +36,7 @@ import Data.Hash.MD5
 
 import Types
 import Types.Key
-import qualified GitRepo as Git
+import qualified Git
 
 {- Conventions:
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c011c979ca..11c0ec4daf 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -21,7 +21,7 @@ import System.Cmd.Utils
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 7b5917dca8..991ccbe481 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -18,7 +18,7 @@ import System.FilePath
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/Git.hs b/Remote/Git.hs
index c8290c9a79..471417e345 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -15,7 +15,7 @@ import System.Posix.Files
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import Locations
 import UUID
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index cc511965f8..86a7bca560 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -20,7 +20,7 @@ import System.Exit
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index bf1bbd8707..aa2507fff4 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -18,7 +18,7 @@ import System.Posix.Process
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Locations
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 2479dfa023..829c58ad06 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -27,7 +27,7 @@ import System.Posix.Env (setEnv)
 import Types
 import Types.Remote
 import Types.Key
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Messages
diff --git a/Remote/Special.hs b/Remote/Special.hs
index 7d2ea1d704..9a00dbd82e 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -14,7 +14,7 @@ import Control.Monad.State (liftIO)
 
 import Types
 import Types.Remote
-import qualified GitRepo as Git
+import qualified Git
 import qualified Annex
 import UUID
 import Utility
diff --git a/Ssh.hs b/Ssh.hs
index 0cf2919c25..21e72c0839 100644
--- a/Ssh.hs
+++ b/Ssh.hs
@@ -9,7 +9,7 @@ module Ssh where
 
 import Control.Monad.State (liftIO)
 
-import qualified GitRepo as Git
+import qualified Git
 import Utility
 import Types
 import Config
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 01ced04ae1..1d67ad5cd7 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -12,7 +12,7 @@ module Types.Remote where
 import Control.Exception
 import Data.Map as M
 
-import qualified GitRepo as Git
+import qualified Git
 import Types.Key
 
 type RemoteConfig = M.Map String String
diff --git a/UUID.hs b/UUID.hs
index 5d8304f83d..0723abeca4 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -29,7 +29,7 @@ import System.IO
 import qualified Data.Map as M
 import Data.Maybe
 
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import Types
 import Types.UUID
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index cb6928fcee..39b8e47c5f 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -25,8 +25,8 @@ import Locations
 import LocationLog
 import qualified Annex
 import qualified AnnexQueue
-import qualified GitRepo as Git
-import qualified GitRepo.LsFiles as LsFiles
+import qualified Git
+import qualified Git.LsFiles as LsFiles
 import Backend
 import Messages
 import Version
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ea68e78c95..ce0424b302 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -16,7 +16,7 @@ import Data.Maybe
 import Types.Key
 import Types
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import qualified Branch
 import Messages
 import Utility
diff --git a/Version.hs b/Version.hs
index 690e693e23..7e7a4c7ce6 100644
--- a/Version.hs
+++ b/Version.hs
@@ -11,7 +11,7 @@ import Control.Monad (unless)
 
 import Types
 import qualified Annex
-import qualified GitRepo as Git
+import qualified Git
 import Config
 
 type Version = String
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 55f34e1027..c240b1c190 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -8,7 +8,7 @@
 import System.Environment
 import Data.List
 
-import qualified GitRepo as Git
+import qualified Git
 import CmdLine
 import Command
 import Utility
diff --git a/git-annex.hs b/git-annex.hs
index 9d6012f2c1..a53697cdbb 100644
--- a/git-annex.hs
+++ b/git-annex.hs
@@ -10,6 +10,4 @@ import System.Environment
 import GitAnnex
 
 main :: IO ()
-main = do
-	args <- getArgs
-	run args
+main = run =<< getArgs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 57232be67b..4939fb57d1 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -11,7 +11,7 @@ import System.Directory
 import Control.Monad (when)
 
 import qualified GitUnionMerge
-import qualified GitRepo as Git
+import qualified Git
 
 header :: String
 header = "Usage: git-union-merge ref ref newref"
diff --git a/test.hs b/test.hs
index 9729818211..44a792f14c 100644
--- a/test.hs
+++ b/test.hs
@@ -27,7 +27,7 @@ import System.IO.HVFS (SystemFS(..))
 import qualified Annex
 import qualified BackendList
 import qualified Backend
-import qualified GitRepo as Git
+import qualified Git
 import qualified Locations
 import qualified Utility
 import qualified Types.Backend

From 0f2859454cb1d1edd779fccd9b35e2b5d78b0861 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:22:10 -0400
Subject: [PATCH 1965/8313] tweak

---
 git-annex-shell.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index c240b1c190..a64552c721 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -35,9 +35,7 @@ header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
 
 main :: IO ()
-main = do
-	args <- getArgs
-	main' args
+main = main' =<< getArgs
 
 main' :: [String] -> IO ()
 main' [] = failure

From f0497312a77d59f24c8273245ac324b02bb1eb13 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:25:37 -0400
Subject: [PATCH 1966/8313] rename GitQueue to Git.Queue

---
 Annex.hs                    |  2 +-
 AnnexQueue.hs               | 12 ++++++------
 GitQueue.hs => Git/Queue.hs | 10 +++++-----
 3 files changed, 12 insertions(+), 12 deletions(-)
 rename GitQueue.hs => Git/Queue.hs (93%)

diff --git a/Annex.hs b/Annex.hs
index e2f4a10206..c21cfb37ca 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -19,7 +19,7 @@ module Annex (
 import Control.Monad.State
 
 import qualified Git
-import GitQueue
+import Git.Queue
 import Types.Backend
 import Types.Remote
 import Types.Crypto
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 58e77a6e85..4c35adfb87 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -16,7 +16,7 @@ import Control.Monad (when, unless)
 
 import Annex
 import Messages
-import qualified GitQueue
+import qualified Git.Queue
 import Utility
 
 {- Adds a git command to the queue, possibly running previously queued
@@ -24,24 +24,24 @@ import Utility
 add :: String -> [CommandParam] -> FilePath -> Annex ()
 add command params file = do
 	q <- getState repoqueue
-	store $ GitQueue.add q command params file
+	store $ Git.Queue.add q command params file
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
 flushWhenFull = do
 	q <- getState repoqueue
-	when (GitQueue.full q) $ flush False
+	when (Git.Queue.full q) $ flush False
 
 {- Runs (and empties) the queue. -}
 flush :: Bool -> Annex ()
 flush silent = do
 	q <- getState repoqueue
-	unless (0 == GitQueue.size q) $ do
+	unless (0 == Git.Queue.size q) $ do
 		unless silent $
 			showSideAction "Recording state in git..."
 		g <- gitRepo
-		q' <- liftIO $ GitQueue.flush g q
+		q' <- liftIO $ Git.Queue.flush g q
 		store q'
 
-store :: GitQueue.Queue -> Annex ()
+store :: Git.Queue.Queue -> Annex ()
 store q = changeState $ \s -> s { repoqueue = q }
diff --git a/GitQueue.hs b/Git/Queue.hs
similarity index 93%
rename from GitQueue.hs
rename to Git/Queue.hs
index 5da3ba1d6e..e1ec0cd313 100644
--- a/GitQueue.hs
+++ b/Git/Queue.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitQueue (
+module Git.Queue (
 	Queue,
 	empty,
 	add,
@@ -21,7 +21,7 @@ import Data.String.Utils
 import Control.Monad (unless, forM_)
 import Utility
 
-import qualified Git
+import Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
@@ -72,7 +72,7 @@ full :: Queue -> Bool
 full (Queue n _) = n > maxSize
 
 {- Runs a queue on a git repository. -}
-flush :: Git.Repo -> Queue -> IO Queue
+flush :: Repo -> Queue -> IO Queue
 flush repo (Queue _ m) = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
 	return empty
@@ -80,10 +80,10 @@ flush repo (Queue _ m) = do
 {- Runs an Action on a list of files in a git repository.
  -
  - Complicated by commandline length limits. -}
-runAction :: Git.Repo -> Action -> [FilePath] -> IO ()
+runAction :: Repo -> Action -> [FilePath] -> IO ()
 runAction repo action files = unless (null files) runxargs
 	where
 		runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
-		params = toCommand $ Git.gitCommandLine repo
+		params = toCommand $ gitCommandLine repo
 			(Param (getSubcommand action):getParams action)
 		feedxargs h = hPutStr h $ join "\0" files

From 896726cde425f6c74273b35cde30c1909551ff66 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 13:32:47 -0400
Subject: [PATCH 1967/8313] rename GitUnionMerge to Git.UnionMerge

Also, moved commit function into Git proper, it's not union merge specific.
---
 Branch.hs                             | 18 +++++------
 Git.hs                                | 14 +++++++++
 GitUnionMerge.hs => Git/UnionMerge.hs | 44 +++++++++------------------
 git-union-merge.hs                    |  6 ++--
 4 files changed, 40 insertions(+), 42 deletions(-)
 rename GitUnionMerge.hs => Git/UnionMerge.hs (67%)

diff --git a/Branch.hs b/Branch.hs
index 7650fe8619..59f2d3134f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -31,7 +31,7 @@ import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
 import qualified Git
-import qualified GitUnionMerge
+import qualified Git.UnionMerge
 import qualified Annex
 import Utility
 import Types
@@ -72,7 +72,7 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = GitUnionMerge.ls_tree g fullname >>= GitUnionMerge.update_index g
+genIndex g = Git.UnionMerge.ls_tree g fullname >>= Git.UnionMerge.update_index g
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
@@ -128,14 +128,13 @@ create = unlessM (refExists fullname) $ do
 	if e
 		then liftIO $ Git.run g "branch" [Param name, Param originname]
 		else withIndex' True $
-			liftIO $ GitUnionMerge.commit g "branch created" fullname []
+			liftIO $ Git.commit g "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM stageJournalFiles $ do
 	g <- Annex.gitRepo
-	withIndex $ liftIO $
-		GitUnionMerge.commit g message fullname [fullname]
+	withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
@@ -158,8 +157,7 @@ update = do
 		let refs = map (last . words) (lines r)
 		updated <- catMaybes `liftM` mapM updateRef refs
 		unless (null updated && not staged) $ liftIO $
-			GitUnionMerge.commit g "update" fullname
-				(fullname:updated)
+			Git.commit g "update" fullname (fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
@@ -200,7 +198,7 @@ updateRef ref
 				-- index will be removed. So, documentation
 				-- advises users not to directly modify the
 				-- branch.
-				liftIO $ GitUnionMerge.merge g [ref]
+				liftIO $ Git.UnionMerge.merge g [ref]
 				return $ Just ref
 
 {- Records changed content of a file into the journal. -}
@@ -335,12 +333,12 @@ stageJournalFiles = do
 			(h, s) <- Git.pipeWriteRead g [Param "hash-object",
 				Param "-w", Param "--stdin-paths"] $ unlines paths
 			-- update the index, also in just one command
-			GitUnionMerge.update_index g $
+			Git.UnionMerge.update_index g $
 				index_lines (lines s) $ map fileJournal fs
 			forceSuccess h
 			mapM_ removeFile paths
 		index_lines shas fs = map genline $ zip shas fs
-		genline (sha, file) = GitUnionMerge.update_index_line sha file
+		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 
 {- Produces a filename to use in the journal for a file on the branch.
  -
diff --git a/Git.hs b/Git.hs
index 2b0cdd06d5..7005f24f55 100644
--- a/Git.hs
+++ b/Git.hs
@@ -56,6 +56,7 @@ module Git (
 	hashObject,
 	getSha,
 	shaSize,
+	commit,
 
 	prop_idempotent_deencode
 ) where
@@ -425,6 +426,19 @@ getSha subcommand a = do
 shaSize :: Int
 shaSize = 40
 
+{- Commits the index into the specified branch, 
+ - with the specified parent refs. -}
+commit :: Repo -> String -> String -> [String] -> IO ()
+commit g message newref parentrefs = do
+	tree <- getSha "write-tree" $
+		pipeRead g [Param "write-tree"]
+	sha <- getSha "commit-tree" $ ignorehandle $
+		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
+	run g "update-ref" [Param newref, Param sha]
+	where
+		ignorehandle a = return . snd =<< a
+		ps = concatMap (\r -> ["-p", r]) parentrefs
+
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
diff --git a/GitUnionMerge.hs b/Git/UnionMerge.hs
similarity index 67%
rename from GitUnionMerge.hs
rename to Git/UnionMerge.hs
index 84c7b44eae..4e0361b858 100644
--- a/GitUnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -5,9 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module GitUnionMerge (
+module Git.UnionMerge (
 	merge,
-	commit,
 	update_index,
 	update_index_line,
 	ls_tree
@@ -18,7 +17,7 @@ import Data.List
 import Data.Maybe
 import Data.String.Utils
 
-import qualified Git
+import Git
 import Utility
 
 {- Performs a union merge between two branches, staging it in the index.
@@ -29,7 +28,7 @@ import Utility
  -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Git.Repo -> [String] -> IO ()
+merge :: Repo -> [String] -> IO ()
 merge g (x:y:[]) = do
 	a <- ls_tree g x
 	b <- merge_trees g x y
@@ -40,10 +39,10 @@ merge _ _ = error "wrong number of branches to merge"
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
-update_index :: Git.Repo -> [String] -> IO ()
+update_index :: Repo -> [String] -> IO ()
 update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		togit ps content = Git.pipeWrite g (map Param ps) content
+		togit ps content = pipeWrite g (map Param ps) content
 			>>= forceSuccess
 
 {- Generates a line suitable to be fed into update-index, to add
@@ -52,16 +51,16 @@ update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
 {- Gets the contents of a tree in a format suitable for update_index. -}
-ls_tree :: Git.Repo -> String -> IO [String]
-ls_tree g x = Git.pipeNullSplit g $ 
+ls_tree :: Repo -> String -> IO [String]
+ls_tree g x = pipeNullSplit g $ 
 	map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: Git.Repo -> String -> String -> IO [String]
+merge_trees :: Repo -> String -> String -> IO [String]
 merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: Git.Repo -> String -> IO [String]
+merge_tree_index :: Repo -> String -> IO [String]
 merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
@@ -69,9 +68,9 @@ diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: Git.Repo -> [String] -> IO [String]
+calc_merge :: Repo -> [String] -> IO [String]
 calc_merge g differ = do
-	diff <- Git.pipeNullSplit g $ map Param differ
+	diff <- pipeNullSplit g $ map Param differ
 	l <- mapM (mergeFile g) (pairs diff)
 	return $ catMaybes l
 	where
@@ -82,28 +81,15 @@ calc_merge g differ = do
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: Git.Repo -> (String, FilePath) -> IO (Maybe String)
+mergeFile :: Repo -> (String, FilePath) -> IO (Maybe String)
 mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- Git.pipeRead g $ map Param ("show":shas)
-		sha <- Git.hashObject g $ unionmerge content
+		content <- pipeRead g $ map Param ("show":shas)
+		sha <- hashObject g $ unionmerge content
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		nullsha = take Git.shaSize $ repeat '0'
+		nullsha = take shaSize $ repeat '0'
 		unionmerge = unlines . nub . lines
-
-{- Commits the index into the specified branch, 
- - with the specified parent refs. -}
-commit :: Git.Repo -> String -> String -> [String] -> IO ()
-commit g message newref parentrefs = do
-	tree <- Git.getSha "write-tree" $
-		Git.pipeRead g [Param "write-tree"]
-	sha <- Git.getSha "commit-tree" $ ignorehandle $
-		Git.pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
-	Git.run g "update-ref" [Param newref, Param sha]
-	where
-		ignorehandle a = return . snd =<< a
-		ps = concatMap (\r -> ["-p", r]) parentrefs
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 4939fb57d1..38df0df6a2 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -10,7 +10,7 @@ import System.FilePath
 import System.Directory
 import Control.Monad (when)
 
-import qualified GitUnionMerge
+import qualified Git.UnionMerge
 import qualified Git
 
 header :: String
@@ -44,6 +44,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
-	GitUnionMerge.merge g [aref, bref]
-	GitUnionMerge.commit g "union merge" newref [aref, bref]
+	Git.UnionMerge.merge g [aref, bref]
+	Git.commit g "union merge" newref [aref, bref]
 	cleanup g

From 56aeeb4565dd419c315d370f6e648abfe009a7d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 14:55:03 -0400
Subject: [PATCH 1968/8313] cabal can now be used to build git-annex.

This is substantially slower than using make, does not build or install
documentation, does not run the test suite, and is not particularly
recommended, but could be useful to some.
---
 .gitignore                                    |  1 +
 Makefile                                      |  6 ++-
 Setup.hs                                      | 17 +++++++
 configure.hs                                  | 14 +++++-
 debian/changelog                              |  4 ++
 doc/install.mdwn                              |  7 +++
 ..._d9f7b851567445c7aa7ebbb440781819._comment |  8 ---
 ..._cf0f829536744098d6846500db998b6a._comment | 17 -------
 git-annex.cabal                               | 49 +++++++++++++++++++
 9 files changed, 96 insertions(+), 27 deletions(-)
 create mode 100644 Setup.hs
 delete mode 100644 doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
 delete mode 100644 doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
 create mode 100644 git-annex.cabal

diff --git a/.gitignore b/.gitignore
index 9a4bc80de3..d5bf54c813 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ html
 Touch.hs
 StatFS.hs
 Remote/S3.hs
+dist
diff --git a/Makefile b/Makefile
index 915b0bf0b2..2f72cdbf9d 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,8 @@ mans=git-annex.1 git-annex-shell.1 git-union-merge.1
 
 all: $(bins) $(mans) docs
 
+sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs
+
 SysConfig.hs: configure.hs TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
@@ -19,8 +21,10 @@ SysConfig.hs: configure.hs TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-Remote/S3.o:
+Remote/S3.hs:
 	@ln -sf S3real.hs Remote/S3.hs
+
+Remote/S3.o: Remote/S3.hs
 	@if ! $(GHCMAKE) Remote/S3.hs; then \
 		ln -sf S3stub.hs Remote/S3.hs; \
 		echo "** building without S3 support"; \
diff --git a/Setup.hs b/Setup.hs
new file mode 100644
index 0000000000..547d6a156e
--- /dev/null
+++ b/Setup.hs
@@ -0,0 +1,17 @@
+{- cabal setup file -}
+
+import Distribution.Simple
+import System.Cmd
+
+main = defaultMainWithHooks simpleUserHooks {
+	preConf = makeSources,
+	postClean = makeClean
+}
+
+makeSources _ _ = do
+	system "make sources"
+	return (Nothing, [])
+
+makeClean _ _ _ _ = do
+	system "make clean"
+	return ()
diff --git a/configure.hs b/configure.hs
index c81aa17e69..2e39feb166 100644
--- a/configure.hs
+++ b/configure.hs
@@ -56,12 +56,24 @@ unicodeFilePath = do
 {- Pulls package version out of the changelog. -}
 getVersion :: Test
 getVersion = do
-	changelog <- readFile "debian/changelog"
+	changelog <- readFile "CHANGELOG"
 	let verline = head $ lines changelog
 	let version = middle (words verline !! 1)
+
+	-- Replace Version field in cabal file, so I don't have to maintain
+	-- the version there too.
+	cabal <- readFile cabalfile
+	writeFile tmpcabalfile $ unlines $ map (setversion version) $ lines cabal
+	renameFile tmpcabalfile cabalfile
+
 	return $ Config "packageversion" (StringConfig version)
 	where
 		middle s = drop 1 $ take (length s - 1) s
+		cabalfile = "git-annex.cabal"
+		tmpcabalfile = cabalfile++".tmp"
+		setversion version s
+			| "Version:" `isPrefixOf` s = "Version: " ++ version
+			| otherwise = s
 
 setup :: IO ()
 setup = do
diff --git a/debian/changelog b/debian/changelog
index 8848cf8611..277b542590 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,10 @@ git-annex (0.20110611) UNRELEASED; urgency=low
   * git-union-merge: New git subcommand, that does a generic union merge
     operation, and operates efficiently without touching the working tree.
   * --force will cause add, etc, to operate on ignored files.
+  * cabal can now be used to build git-annex. This is substantially
+    slower than using make, does not build or install documentation,
+    does not run the test suite, and is not particularly recommended,
+    but could be useful to some.
 
  -- Joey Hess   Mon, 13 Jun 2011 19:53:24 -0400
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7818aaf152..8a3edcb643 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -32,3 +32,10 @@ To build and use git-annex, you will need:
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`
+
+## Using cabal
+
+As a haskell package, git-annex can be built using cabal. For example:
+
+	cabal configure
+	cabal install --bindir=$HOME/bin
diff --git a/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment b/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
deleted file mode 100644
index 616b3c4dd5..0000000000
--- a/doc/install/comment_1_d9f7b851567445c7aa7ebbb440781819._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="84.189.1.247"
- subject="Why isn't this package built with Cabal?"
- date="2011-03-23T11:31:06Z"
- content="""
-It would be a lot easier to compile this package, if it had a Cabal file to describe the build; especially the build-time dependencies. Why isn't Cabal used?
-"""]]
diff --git a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment b/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
deleted file mode 100644
index 81d5a2c629..0000000000
--- a/doc/install/comment_2_cf0f829536744098d6846500db998b6a._comment
+++ /dev/null
@@ -1,17 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 2"
- date="2011-03-23T15:18:29Z"
- content="""
-Because I haven't learned Cabal yet.
-
-But also because I've had bad experiences with both a) tying a particular program to a particular language's pet build system and then having to add ugliness when I later need to do something in the build that has nothing to do with that language and b) as a user, needing to deal with the pet build systems of languages when I just need to make some small change to the build process that is trivial in a Makefile.
-
-With that said, I do have a configure program written in Haskell, so at least it doesn't use autotools. :)
-
-Update: I did try using cabal, but git-annex includes 3 programs, and they
-all link to a lot of git-annex modules, and cabal wanted to build nearly
-every module 3 times, which was too slow for me and I could not find a way
-around.
-"""]]
diff --git a/git-annex.cabal b/git-annex.cabal
new file mode 100644
index 0000000000..a3b04b60e4
--- /dev/null
+++ b/git-annex.cabal
@@ -0,0 +1,49 @@
+Name: git-annex
+Version: 0.20110611
+Cabal-Version: >= 1.2
+License: GPL
+Maintainer: Joey Hess 
+Author: Joey Hess
+Stability: Stable
+Copyright: 2010-2011 Joey Hess
+License-File: GPL
+Extra-Source-Files: 
+Homepage: http://git-annex.branchable.com/
+Build-type: Custom
+Category: Utility
+Synopsis: manage files with git, without checking their contents into git
+Description:
+ git-annex allows managing files with git, without checking the file
+ contents into git. While that may seem paradoxical, it is useful when
+ dealing with files larger than git can currently easily handle, whether due
+ to limitations in memory, checksumming time, or disk space.
+ .
+ Even without file content tracking, being able to manage files with git,
+ move files around and delete files with versioned directory trees, and use
+ branches and distributed clones, are all very handy reasons to use git. And
+ annexed files can co-exist in the same git repository with regularly
+ versioned files, which is convenient for maintaining documents, Makefiles,
+ etc that are associated with annexed files but that benefit from full
+ revision control.
+
+Executable git-annex
+  Main-Is: git-annex.hs
+  GHC-Options: -O2
+  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+
+Executable git-annex-shell
+  Main-Is: git-annex-shell.hs
+  GHC-Options: -O2
+  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+
+Executable git-union-merge
+  Main-Is: git-union-merge.hs
+  GHC-Options: -O2
+
+source-repository head
+  type: git
+  location: git://git-annex.branchable.com/

From 20565027cc2f78679a9c54c8350fb7b34e8ff94f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 15:07:10 -0400
Subject: [PATCH 1969/8313] cabal tweaks

---
 git-annex.cabal | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index f602143b5b..294a3baced 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,6 +1,6 @@
 Name: git-annex
 Version: 3.20110625
-Cabal-Version: >= 1.2
+Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
 Author: Joey Hess
@@ -28,21 +28,18 @@ Description:
 
 Executable git-annex
   Main-Is: git-annex.hs
-  GHC-Options: -O2
   Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  GHC-Options: -O2
   Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs
-  GHC-Options: -O2
 
 source-repository head
   type: git

From b3ab44f8bbc6a1e253bb81dfe9d6939adcdcba56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 15:37:35 -0400
Subject: [PATCH 1970/8313] add a filelist for cabal sdist

I hate hard-coded 40 kilobyte lone file lists, and just once would like to
see a build system that does not assume it's a good idea to have a file
list, or a hardcoded file list, or a file list that can only be generated
with a crippled form of globs. But not today, thank you cabal.
---
 Makefile        |  2 +-
 configure.hs    | 37 ++++++++++++++++++++++++++-----------
 git-annex.cabal |  2 +-
 3 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/Makefile b/Makefile
index da80228dd3..0bc02dc352 100644
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,7 @@ docs: $(mans)
 clean:
 	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
 		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
-	rm -rf doc/.ikiwiki html
+	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
 .PHONY: $(bins) test install
diff --git a/configure.hs b/configure.hs
index 2e39feb166..5f0ff5a1e3 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,6 +2,8 @@
 
 import System.Directory
 import Data.List
+import Data.String.Utils
+import System.Cmd.Utils
 
 import TestConfig
 
@@ -56,24 +58,36 @@ unicodeFilePath = do
 {- Pulls package version out of the changelog. -}
 getVersion :: Test
 getVersion = do
+	version <- getVersionString
+	return $ Config "packageversion" (StringConfig version)
+	
+getVersionString :: IO String
+getVersionString = do
 	changelog <- readFile "CHANGELOG"
 	let verline = head $ lines changelog
-	let version = middle (words verline !! 1)
-
-	-- Replace Version field in cabal file, so I don't have to maintain
-	-- the version there too.
-	cabal <- readFile cabalfile
-	writeFile tmpcabalfile $ unlines $ map (setversion version) $ lines cabal
-	renameFile tmpcabalfile cabalfile
-
-	return $ Config "packageversion" (StringConfig version)
+	return $ middle (words verline !! 1)
 	where
 		middle s = drop 1 $ take (length s - 1) s
+
+{- Set up cabal file with version. -}
+cabalSetup :: IO ()
+cabalSetup = do
+	version <- getVersionString
+	(_, filelist) <- pipeLinesFrom "find" (words ". -name .git -prune -o -name dist -prune -o -not -name *.hi -not -name *.o -not -name configure -not -name *.tmp -type f -print")
+	cabal <- readFile cabalfile
+	writeFile tmpcabalfile $ unlines $ 
+		map (setfield "Version" version) $
+		map (setfield "Extra-Source-Files" $ join ", " $ sort filelist) $
+		lines cabal
+	renameFile tmpcabalfile cabalfile
+	where
 		cabalfile = "git-annex.cabal"
 		tmpcabalfile = cabalfile++".tmp"
-		setversion version s
-			| "Version:" `isPrefixOf` s = "Version: " ++ version
+		setfield field value s
+			| fullfield `isPrefixOf` s = fullfield ++ value
 			| otherwise = s
+			where
+				fullfield = field ++ ": "
 
 setup :: IO ()
 setup = do
@@ -90,3 +104,4 @@ main = do
 	config <- runTests tests
 	writeSysConfig config
 	cleanup
+	cabalSetup
diff --git a/git-annex.cabal b/git-annex.cabal
index 294a3baced..2560b90a94 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -7,7 +7,6 @@ Author: Joey Hess
 Stability: Stable
 Copyright: 2010-2011 Joey Hess
 License-File: GPL
-Extra-Source-Files: 
 Homepage: http://git-annex.branchable.com/
 Build-type: Custom
 Category: Utility
@@ -25,6 +24,7 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
+Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex-shell.hs, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge.hs, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
 
 Executable git-annex
   Main-Is: git-annex.hs

From e27e20c6d64200ae0804dce62e31eaf55d5581fa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 16:16:52 -0400
Subject: [PATCH 1971/8313] add

---
 doc/upgrades.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index 337c65767a..d62e9dcc78 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -22,6 +22,9 @@ conflicts first before upgrading git-annex.
 
 Involved moving the .git-annex/ directory into a separate git-annex branch.
 
+After this upgrade, you should make sure you include the git-annex branch
+when git pushing and pulling.
+
 ### tips for this upgrade
 
 This upgrade is easier (and faster!) than the previous upgrades.

From b9721fffac0815fcb9c69cca6b9ef2c7982438e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 30 Jun 2011 19:25:21 -0400
Subject: [PATCH 1972/8313] cabal check

---
 git-annex.cabal | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 2560b90a94..5a68e4dea4 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -28,15 +28,17 @@ Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs,
 
 Executable git-annex
   Main-Is: git-annex.hs
-  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  Build-Depends: haskell98, base, MissingH, hslogger, directory, filepath,
+  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   base < 5
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs

From ceb887d82669b3ec694f31a899b59eefe0f5f352 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 01:11:41 -0400
Subject: [PATCH 1973/8313] tweaks

---
 Content.hs | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/Content.hs b/Content.hs
index d24fc9ac70..937ebd5426 100644
--- a/Content.hs
+++ b/Content.hs
@@ -235,10 +235,11 @@ moveBad key = do
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g key
 	let dest = gitAnnexBadDir g  takeFileName src
-	liftIO $ createDirectoryIfMissing True (parentDir dest)
-	liftIO $ allowWrite (parentDir src)
-	liftIO $ renameFile src dest
-	liftIO $ removeDirectory (parentDir src)
+	liftIO $ do
+		createDirectoryIfMissing True (parentDir dest)
+		allowWrite (parentDir src)
+		renameFile src dest
+		removeDirectory (parentDir src)
 	logStatus key ValueMissing
 	return dest
 
@@ -252,12 +253,12 @@ getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
 	if (not exists)
 		then return []
-		else do
+		else liftIO $ do
 			-- 2 levels of hashing
-			levela <- liftIO $ dirContents dir
-			levelb <- liftIO $ mapM dirContents levela
-			contents <- liftIO $ mapM dirContents (concat levelb)
-			files <- liftIO $ filterM present (concat contents)
+			levela <- dirContents dir
+			levelb <- mapM dirContents levela
+			contents <- mapM dirContents (concat levelb)
+			files <- filterM present (concat contents)
 			return $ catMaybes $ map (fileKey . takeFileName) files
 	where
 		present d = do

From cdbcd6f495580ee927a85af0581661b486c8ef77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 15:24:07 -0400
Subject: [PATCH 1974/8313] add web special remote

Generalized LocationLog to PresenceLog, and use a presence log to record
urls for the web special remote.
---
 Backend/File.hs    |   6 +--
 Command/Add.hs     |   2 +-
 Command/Drop.hs    |   2 +-
 Command/DropKey.hs |   2 +-
 Command/Fsck.hs    |   4 +-
 Command/Move.hs    |   2 +-
 Command/SetKey.hs  |   2 +-
 Command/Unannex.hs |   2 +-
 Command/Whereis.hs |   2 +-
 Content.hs         |  11 ++--
 LocationLog.hs     | 103 +------------------------------------
 PresenceLog.hs     | 123 +++++++++++++++++++++++++++++++++++++++++++++
 Remote.hs          |  23 ++++++++-
 Remote/Web.hs      | 107 +++++++++++++++++++++++++++++++++++++++
 UUID.hs            |  16 ------
 debian/changelog   |   2 +
 git-annex.cabal    |   2 +-
 17 files changed, 272 insertions(+), 139 deletions(-)
 create mode 100644 PresenceLog.hs
 create mode 100644 Remote/Web.hs

diff --git a/Backend/File.hs b/Backend/File.hs
index b8d3bb65ad..174da4e6dc 100644
--- a/Backend/File.hs
+++ b/Backend/File.hs
@@ -135,8 +135,8 @@ showLocations key exclude = do
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- prettyPrintUUIDs uuidsskipped
+	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
 		filteruuids list x = filter (`notElem` x) list
@@ -195,7 +195,7 @@ checkKeyNumCopies key file numcopies = do
 	let present = length safelocations
 	if present < needed
 		then do
-			ppuuids <- prettyPrintUUIDs untrustedlocations
+			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
 			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
diff --git a/Command/Add.hs b/Command/Add.hs
index 5133ee1fd5..6a1ffb5da6 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -51,7 +51,7 @@ perform (file, backend) = do
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	logStatus key ValuePresent
+	logStatus key InfoPresent
 
 	link <- calcGitLink file key
 	liftIO $ createSymbolicLink link file
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 07cec1a677..bd47407413 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -45,5 +45,5 @@ perform key backend numcopies = do
 cleanup :: Key -> CommandCleanup
 cleanup key = do
 	whenM (inAnnex key) $ removeAnnex key
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	return True
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 780fe0adf0..16a3e35d64 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -40,5 +40,5 @@ perform key = do
 
 cleanup :: Key -> CommandCleanup
 cleanup key = do
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	return True
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index cb062342d7..988cfd28d1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -64,11 +64,11 @@ verifyLocationLog key file = do
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix g u ValuePresent
+				fix g u InfoPresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix g u ValueMissing
+				fix g u InfoMissing
 				warning $
 					"** Based on the location log, " ++ file
 					++ "\n** was expected to be present, " ++
diff --git a/Command/Move.hs b/Command/Move.hs
index 03b605ce57..6bf6e05827 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -58,7 +58,7 @@ remoteHasKey remote key present	= do
 	g <- Annex.gitRepo
 	logChange g key remoteuuid status
 	where
-		status = if present then ValuePresent else ValueMissing
+		status = if present then InfoPresent else InfoMissing
 
 {- Moves (or copies) the content of an annexed file to a remote.
  -
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index dbad148b25..546bd8e2d0 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -46,5 +46,5 @@ perform file = do
 cleanup :: CommandCleanup
 cleanup = do
 	key <- cmdlineKey
-	logStatus key ValuePresent
+	logStatus key InfoPresent
 	return True
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index fe413b25b4..d30f8d20f7 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -65,7 +65,7 @@ cleanup file key = do
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 
 	fromAnnex key file
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 3a7213217a..0e4858f8b7 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -10,7 +10,7 @@ module Command.Whereis where
 import LocationLog
 import Command
 import Messages
-import UUID
+import Remote
 import Types
 
 command :: [Command]
diff --git a/Content.hs b/Content.hs
index 937ebd5426..a2f38ddc96 100644
--- a/Content.hs
+++ b/Content.hs
@@ -65,12 +65,7 @@ calcGitLink file key = do
 		whoops = error $ "unable to normalize " ++ file
 
 {- Updates the LocationLog when a key's presence changes in the current
- - repository.
- -
- - Note that the LocationLog is not updated in bare repositories.
- - Operations that change a bare repository should be done from
- - a non-bare repository, and the LocationLog in that repository be
- - updated instead. -}
+ - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
 	g <- Annex.gitRepo
@@ -112,7 +107,7 @@ getViaTmpUnchecked key action = do
 	if success
 		then do
 			moveAnnex key tmp
-			logStatus key ValuePresent
+			logStatus key InfoPresent
 			return True
 		else do
 			-- the tmp file is left behind, in case caller wants
@@ -240,7 +235,7 @@ moveBad key = do
 		allowWrite (parentDir src)
 		renameFile src dest
 		removeDirectory (parentDir src)
-	logStatus key ValueMissing
+	logStatus key InfoMissing
 	return dest
 
 {- List of keys whose content exists in .git/annex/objects/ -}
diff --git a/LocationLog.hs b/LocationLog.hs
index b7deb3ed9a..a5db7d1218 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -5,11 +5,6 @@
  -
  - Repositories record their UUID and the date when they --get or --drop
  - a value.
- -
- - A line of the log will look like: "date N UUID"
- - Where N=1 when the repo has the file, and 0 otherwise.
- - (After the UUID can optionally come a white space and other data, 
- - for future expansion.)
  - 
  - Copyright 2010-2011 Joey Hess 
  -
@@ -25,61 +20,16 @@ module LocationLog (
 	loggedKeys
 ) where
 
-import Data.Time.Clock.POSIX
-import Data.Time
-import System.Locale
 import System.FilePath
-import qualified Data.Map as Map
 import Control.Monad (when)
 import Data.Maybe
-import Control.Monad.State (liftIO)
 
 import qualified Git
 import qualified Branch
 import UUID
 import Types
 import Locations
-
-data LogLine = LogLine {
-	date :: POSIXTime,
-	status :: LogStatus,
-	uuid :: UUID
-} deriving (Eq)
-
-data LogStatus = ValuePresent | ValueMissing | Undefined
-	deriving (Eq)
-
-instance Show LogStatus where
-	show ValuePresent = "1"
-	show ValueMissing = "0"
-	show Undefined = "undefined"
-
-instance Read LogStatus where
-	readsPrec _ "1" = [(ValuePresent, "")]
-	readsPrec _ "0" = [(ValueMissing, "")]
-	readsPrec _ _   = [(Undefined, "")]
-
-instance Show LogLine where
-	show (LogLine d s u) = unwords [show d, show s, u]
-
-instance Read LogLine where
-	-- This parser is robust in that even unparsable log lines are
-	-- read without an exception being thrown.
-	-- Such lines have a status of Undefined.
-	readsPrec _ string = 
-		if length w >= 3
-			then maybe bad good pdate
-			else bad
-		where
-			w = words string
-			s = read $ w !! 1
-			u = w !! 2
-			pdate :: Maybe UTCTime
-			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w
-
-			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s u
-			bad = ret $ LogLine 0 Undefined ""
-			ret v = [(v, "")]
+import PresenceLog
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
@@ -92,59 +42,10 @@ logChange repo key u s = do
 	ls <- readLog f
 	writeLog f (compactLog $ line:ls)
 
-{- Reads a log file.
- - Note that the LogLines returned may be in any order. -}
-readLog :: FilePath -> Annex [LogLine]
-readLog file = return . parseLog =<< Branch.get file
-
-parseLog :: String -> [LogLine]
-parseLog s = filter parsable $ map read $ lines s
-	where
-		-- some lines may be unparseable, avoid them
-		parsable l = status l /= Undefined
-
-{- Stores a set of lines in a log file -}
-writeLog :: FilePath -> [LogLine] -> Annex ()
-writeLog file ls = Branch.change file (unlines $ map show ls)
-
-{- Generates a new LogLine with the current date. -}
-logNow :: LogStatus -> UUID -> Annex LogLine
-logNow s u = do
-	now <- liftIO $ getPOSIXTime
-	return $ LogLine now s u
-
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = do
-	ls <- readLog $ logFile key
-	return $ map uuid $ filterPresent ls
-
-{- Filters the list of LogLines to find ones where the value
- - is (or should still be) present. -}
-filterPresent :: [LogLine] -> [LogLine]
-filterPresent ls = filter (\l -> ValuePresent == status l) $ compactLog ls
-
-type LogMap = Map.Map UUID LogLine
-
-{- Compacts a set of logs, returning a subset that contains the current
- - status. -}
-compactLog :: [LogLine] -> [LogLine]
-compactLog ls = compactLog' Map.empty ls
-compactLog' :: LogMap -> [LogLine] -> [LogLine]
-compactLog' m [] = Map.elems m
-compactLog' m (l:ls) = compactLog' (mapLog m l) ls
-
-{- Inserts a log into a map of logs, if the log has better (ie, newer)
- - information about a repo than the other logs in the map -}
-mapLog :: LogMap -> LogLine -> LogMap
-mapLog m l = 
-	if better
-		then Map.insert u l m
-		else m
-	where
-		better = maybe True (\l' -> date l' <= date l) $ Map.lookup u m
-		u = uuid l
+keyLocations key = currentLog $ logFile key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/PresenceLog.hs b/PresenceLog.hs
new file mode 100644
index 0000000000..71d78f1eda
--- /dev/null
+++ b/PresenceLog.hs
@@ -0,0 +1,123 @@
+{- git-annex presence log
+ -
+ - This is used to store presence information in the git-annex branch in
+ - a way that can be union merged.
+ -
+ - A line of the log will look like: "date N INFO"
+ - Where N=1 when the INFO is present, and 0 otherwise.
+ - 
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module PresenceLog (
+	LogStatus(..),
+	readLog,
+	writeLog,
+	logNow,
+	compactLog,
+	currentLog
+) where
+
+import Data.Time.Clock.POSIX
+import Data.Time
+import System.Locale
+import qualified Data.Map as Map
+import Control.Monad.State (liftIO)
+
+import qualified Branch
+import Types
+
+data LogLine = LogLine {
+	date :: POSIXTime,
+	status :: LogStatus,
+	info :: String
+} deriving (Eq)
+
+data LogStatus = InfoPresent | InfoMissing | Undefined
+	deriving (Eq)
+
+instance Show LogStatus where
+	show InfoPresent = "1"
+	show InfoMissing = "0"
+	show Undefined = "undefined"
+
+instance Read LogStatus where
+	readsPrec _ "1" = [(InfoPresent, "")]
+	readsPrec _ "0" = [(InfoMissing, "")]
+	readsPrec _ _   = [(Undefined, "")]
+
+instance Show LogLine where
+	show (LogLine d s i) = unwords [show d, show s, i]
+
+instance Read LogLine where
+	-- This parser is robust in that even unparsable log lines are
+	-- read without an exception being thrown.
+	-- Such lines have a status of Undefined.
+	readsPrec _ string = 
+		if length w >= 3
+			then maybe bad good pdate
+			else bad
+		where
+			w = words string
+			s = read $ w !! 1
+			i = w !! 2
+			pdate :: Maybe UTCTime
+			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w
+
+			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s i
+			bad = ret $ LogLine 0 Undefined ""
+			ret v = [(v, "")]
+
+{- Reads a log file.
+ - Note that the LogLines returned may be in any order. -}
+readLog :: FilePath -> Annex [LogLine]
+readLog file = return . parseLog =<< Branch.get file
+
+parseLog :: String -> [LogLine]
+parseLog s = filter parsable $ map read $ lines s
+	where
+		-- some lines may be unparseable, avoid them
+		parsable l = status l /= Undefined
+
+{- Stores a set of lines in a log file -}
+writeLog :: FilePath -> [LogLine] -> Annex ()
+writeLog file ls = Branch.change file (unlines $ map show ls)
+
+{- Generates a new LogLine with the current date. -}
+logNow :: LogStatus -> String -> Annex LogLine
+logNow s i = do
+	now <- liftIO $ getPOSIXTime
+	return $ LogLine now s i
+
+{- Reads a log and returns only the info that is still in effect. -}
+currentLog :: FilePath -> Annex [String]
+currentLog file = do
+	ls <- readLog file
+	return $ map info $ filterPresent ls
+
+{- Returns the info from LogLines that are in effect. -}
+filterPresent :: [LogLine] -> [LogLine]
+filterPresent ls = filter (\l -> InfoPresent == status l) $ compactLog ls
+
+type LogMap = Map.Map String LogLine
+
+{- Compacts a set of logs, returning a subset that contains the current
+ - status. -}
+compactLog :: [LogLine] -> [LogLine]
+compactLog ls = compactLog' Map.empty ls
+compactLog' :: LogMap -> [LogLine] -> [LogLine]
+compactLog' m [] = Map.elems m
+compactLog' m (l:ls) = compactLog' (mapLog m l) ls
+
+{- Inserts a log into a map of logs, if the log has better (ie, newer)
+ - information than the other logs in the map -}
+mapLog :: LogMap -> LogLine -> LogMap
+mapLog m l = 
+	if better
+		then Map.insert i l m
+		else m
+	where
+		better = maybe True (\l' -> date l' <= date l) $ Map.lookup i m
+		i = info l
diff --git a/Remote.hs b/Remote.hs
index 1accabf6d9..28c2e39cdd 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -24,6 +24,7 @@ module Remote (
 	nameToUUID,
 	remotesWithUUID,
 	remotesWithoutUUID,
+	prettyPrintUUIDs,
 
 	remoteLog,
 	readRemoteLog,
@@ -34,7 +35,7 @@ module Remote (
 	prop_idempotent_configEscape
 ) where
 
-import Control.Monad (filterM)
+import Control.Monad (filterM, liftM2)
 import Data.List
 import qualified Data.Map as M
 import Data.Maybe
@@ -54,6 +55,7 @@ import qualified Remote.S3
 import qualified Remote.Bup
 import qualified Remote.Directory
 import qualified Remote.Rsync
+import qualified Remote.Web
 import qualified Remote.Hook
 
 remoteTypes :: [RemoteType Annex]
@@ -63,6 +65,7 @@ remoteTypes =
 	, Remote.Bup.remote
 	, Remote.Directory.remote
 	, Remote.Rsync.remote
+	, Remote.Web.remote
 	, Remote.Hook.remote
 	]
 
@@ -120,6 +123,24 @@ nameToUUID n = do
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
+{- Pretty-prints a list of UUIDs of remotes. -}
+prettyPrintUUIDs :: [UUID] -> Annex String
+prettyPrintUUIDs uuids = do
+	here <- getUUID =<< Annex.gitRepo
+	-- Show descriptions from the uuid log, falling back to remote names,
+	-- as some remotes may not be in the uuid log.
+	m <- liftM2 M.union uuidMap $
+		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
+	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
+	where
+		prettify m u here = base ++ ishere
+			where
+				base = if not $ null $ findlog m u
+					then u ++ "  -- " ++ findlog m u
+					else u
+				ishere = if here == u then " <-- here" else ""
+		findlog m u = M.findWithDefault "" u m
+
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
diff --git a/Remote/Web.hs b/Remote/Web.hs
new file mode 100644
index 0000000000..201f923cf7
--- /dev/null
+++ b/Remote/Web.hs
@@ -0,0 +1,107 @@
+{- Web remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Web (
+	remote
+) where
+
+import Control.Monad.State (liftIO)
+import Control.Exception
+import System.FilePath
+import Network.Curl.Easy
+import Network.Curl.Opts
+import Network.Curl.Types
+import Network.Curl.Code
+
+import Types
+import Types.Remote
+import qualified Git
+import Messages
+import Utility
+import UUID
+import Config
+import PresenceLog
+
+remote :: RemoteType Annex
+remote = RemoteType {
+	typename = "web",
+	enumerate = list,
+	generate = gen,
+	setup = error "not supported"
+}
+
+-- There is only one web remote, and it always exists.
+-- (If the web should cease to exist, remove this module and redistribute
+-- a new release to the survivors by carrier pigeon.)
+list :: Annex [Git.Repo]
+list = return [Git.repoRemoteNameSet Git.repoFromUnknown "remote.web.dummy"]
+
+-- Dummy uuid for the whole web. Do not alter.
+webUUID :: UUID
+webUUID = "00000000-0000-0000-0000-000000000001"
+
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen r _ _ = 
+	return $ Remote {
+		uuid = webUUID,
+		cost = expensiveRemoteCost,
+		name = Git.repoDescribe r,
+		storeKey = upload,
+		retrieveKeyFile = download,
+		removeKey = remove,
+		hasKey = check,
+		hasKeyCheap = False,
+		config = Nothing
+	}
+
+{- The urls for a key are stored in remote/web/key.log in the git-annex branch. -}
+urlLog :: Key -> FilePath
+urlLog key = "remote/web"  show key ++ ".log"
+
+urls :: Key -> Annex [URLString]
+urls key = currentLog (urlLog key)
+
+download :: Key -> FilePath -> Annex Bool
+download key file = download' file =<< urls key
+download' :: FilePath -> [URLString] -> Annex Bool
+download' _ [] = return False
+download' file (url:us) = do
+	showProgress -- make way for curl progress bar
+	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
+	if ok then return ok else download' file us
+
+upload :: Key -> Annex Bool
+upload _ = do
+	warning "upload to web not supported"
+	return False
+
+remove :: Key -> Annex Bool
+remove _ = do
+	warning "removal from web not supported"
+	return False
+
+check :: Key -> Annex (Either IOException Bool)
+check key = do
+	us <- urls key
+	if null us
+		then return $ Right False
+		else return . Right =<< check' us
+check' :: [URLString] -> Annex Bool
+check' [] = return False
+check' (u:us) = do
+	showNote ("checking " ++ u)
+	e <- liftIO $ urlexists u
+	if e then return e else check' us
+
+urlexists :: URLString -> IO Bool
+urlexists url = do
+	curl <- initialize
+	_ <- setopt curl (CurlURL url)
+	_ <- setopt curl (CurlNoBody True)
+	_ <- setopt curl (CurlFailOnError True)
+	res <- perform curl
+	return $ res == CurlOK
diff --git a/UUID.hs b/UUID.hs
index 0723abeca4..3ccc729069 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -17,7 +17,6 @@ module UUID (
 	getUncachedUUID,
 	prepUUID,
 	genUUID,
-	prettyPrintUUIDs,
 	describeUUID,
 	uuidMap,
 	uuidLog
@@ -86,21 +85,6 @@ prepUUID = do
 		uuid <- liftIO $ genUUID
 		setConfig configkey uuid
 
-{- Pretty-prints a list of UUIDs -}
-prettyPrintUUIDs :: [UUID] -> Annex String
-prettyPrintUUIDs uuids = do
-	here <- getUUID =<< Annex.gitRepo
-	m <- uuidMap
-	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
-	where
-		prettify m u here = base ++ ishere
-			where
-				base = if not $ null $ findlog m u
-					then u ++ "  -- " ++ findlog m u
-					else u
-				ishere = if here == u then " <-- here" else ""
-		findlog m u = M.findWithDefault "" u m
-
 {- Records a description for a uuid in the uuidLog. -}
 describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
diff --git a/debian/changelog b/debian/changelog
index f3cbbef9ce..53fc99e8c3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,7 @@
 git-annex (3.20110625) UNRELEASED; urgency=low
 
+  * Now the web can be used as a special remote. This feature
+    replaces the old URL backend.
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Such
     commands are now faster than they were before introduction of the
diff --git a/git-annex.cabal b/git-annex.cabal
index 5a68e4dea4..aa11ba381f 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -24,7 +24,7 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
-Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex-shell.hs, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge.hs, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
+Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./PresenceLog.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./Remote/Web.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/.ikiwiki/indexdb, ./doc/.ikiwiki/lockfile, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex, ./git-annex-shell, ./git-annex-shell.1, ./git-annex-shell.hs, ./git-annex.1, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge, ./git-union-merge.1, ./git-union-merge.hs, ./html/GPL, ./html/backends.html, ./html/bare_repositories.html, ./html/bugs.html, ./html/bugs/Displayed_copy_speed_is_wrong.html, ./html/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/bugs/Makefile_is_missing_dependancies.html, ./html/bugs/Name_scheme_does_not_follow_git__39__s_rules.html, ./html/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.html, ./html/bugs/No_version_information_from_cli.html, ./html/bugs/Problems_running_make_on_osx.html, ./html/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.html, ./html/bugs/S3_memory_leaks.html, ./html/bugs/Unfortunate_interaction_with_Calibre.html, ./html/bugs/WORM:_Handle_long_filenames_correctly.html, ./html/bugs/add_range_argument_to___34__git_annex_dropunused__34___.html, ./html/bugs/annex_add_in_annex.html, ./html/bugs/backend_version_upgrade_leaves_repo_unusable.html, ./html/bugs/bare_git_repos.html, ./html/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.html, ./html/bugs/building_on_lenny.html, ./html/bugs/check_for_curl_in_configure.hs.html, ./html/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.html, ./html/bugs/conflicting_haskell_packages.html, ./html/bugs/copy_fast_confusing_with_broken_locationlog.html, ./html/bugs/done.html, ./html/bugs/dotdot_problem.html, ./html/bugs/dropping_files_with_a_URL_backend_fails.html, ./html/bugs/encrypted_S3_stalls.html, ./html/bugs/error_propigation.html, ./html/bugs/error_with_file_names_starting_with_dash.html, ./html/bugs/fat_support.html, ./html/bugs/free_space_checking.html, ./html/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.html, ./html/bugs/fsck_output.html, ./html/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.html, ./html/bugs/git-annex_directory_hashing_problems_on_osx.html, ./html/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.html, ./html/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.html, ./html/bugs/git_annex_copy_--fast_does_not_copy_files.html, ./html/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.html, ./html/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.html, ./html/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.html, ./html/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.html, ./html/bugs/git_annex_initremote_walks_.git-annex.html, ./html/bugs/git_annex_migrate_leaves_old_backend_versions_around.html, ./html/bugs/git_annex_should_use___39__git_add_-f__39___internally.html, ./html/bugs/git_annex_unlock_is_not_atomic.html, ./html/bugs/git_annex_unused_failes_on_empty_repository.html, ./html/bugs/git_annex_unused_seems_to_check_for_current_path.html, ./html/bugs/git_rename_detection_on_file_move.html, ./html/bugs/minor_bug:_errors_are_not_verbose_enough.html, ./html/bugs/ordering.html, ./html/bugs/problem_commit_normal_links.html, ./html/bugs/problems_with_utf8_names.html, ./html/bugs/scp_interrupt_to_background.html, ./html/bugs/softlink_mtime.html, ./html/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.html, ./html/bugs/tmp_file_handling.html, ./html/bugs/touch.hsc_has_problems_on_non-linux_based_systems.html, ./html/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.html, ./html/bugs/unannex_vs_unlock_hook_confusion.html, ./html/bugs/unhappy_without_UTF8_locale.html, ./html/bugs/upgrade_left_untracked_.git-annex__47____42___directories.html, ./html/bugs/weird_local_clone_confuses.html, ./html/cheatsheet.html, ./html/comments.html, ./html/contact.html, ./html/copies.html, ./html/design.html, ./html/design/encryption.html, ./html/distributed_version_control.html, ./html/download.html, ./html/encryption.html, ./html/feeds.html, ./html/forum.html, ./html/forum/Behaviour_of_fsck.html, ./html/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.html, ./html/forum/Need_new_build_instructions_for_Debian_stable.html, ./html/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.html, ./html/forum/OSX__39__s_haskell-platform_statically_links_things.html, ./html/forum/Problems_with_large_numbers_of_files.html, ./html/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.html, ./html/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.html, ./html/forum/__34__git_annex_lock__34___very_slow_for_big_repo.html, ./html/forum/bainstorming:_git_annex_push___38___pull.html, ./html/forum/batch_check_on_remote_when_using_copy.html, ./html/forum/can_git-annex_replace_ddm__63__.html, ./html/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.html, ./html/forum/git-annex_communication_channels.html, ./html/forum/git-annex_on_OSX.html, ./html/forum/hashing_objects_directories.html, ./html/forum/incompatible_versions__63__.html, ./html/forum/migrate_existing_git_repository_to_git-annex.html, ./html/forum/migration_to_git-annex_and_rsync.html, ./html/forum/new_microfeatures.html, ./html/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.html, ./html/forum/relying_on_git_for_numcopies.html, ./html/forum/rsync_over_ssh__63__.html, ./html/forum/seems_to_build_fine_on_haskell_platform_2011.html, ./html/forum/sparse_git_checkouts_with_annex.html, ./html/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.html, ./html/forum/unannex_alternatives.html, ./html/forum/wishlist:_command_options_changes.html, ./html/forum/wishlist:_define_remotes_that_must_have_all_files.html, ./html/forum/wishlist:_do_round_robin_downloading_of_data.html, ./html/forum/wishlist:_git-annex_replicate.html, ./html/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.html, ./html/forum/wishlist:_git_annex_status.html, ./html/forum/wishlist:_git_backend_for_git-annex.html, ./html/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.html, ./html/forum/wishlist:_special_remote_for_sftp_or_rsync.html, ./html/forum/wishlist:_support_for_more_ssh_urls_.html, ./html/forum/wishlist:_traffic_accounting_for_git-annex.html, ./html/forum/wishlist:alias_system.html, ./html/forum/working_without_git-annex_commits.html, ./html/future_proofing.html, ./html/git-annex-shell.html, ./html/git-annex.html, ./html/git-union-merge.html, ./html/ikiwiki/ikiwiki.js, ./html/ikiwiki/relativedate.js, ./html/ikiwiki/toggle.js, ./html/index.html, ./html/install.html, ./html/install/Debian.html, ./html/install/Fedora.html, ./html/install/FreeBSD.html, ./html/install/OSX.html, ./html/install/Ubuntu.html, ./html/internals.html, ./html/location_tracking.html, ./html/logo.png, ./html/logo_small.png, ./html/news.html, ./html/not.html, ./html/repomap.png, ./html/special_remotes.html, ./html/special_remotes/S3.html, ./html/special_remotes/bup.html, ./html/special_remotes/directory.html, ./html/special_remotes/hook.html, ./html/special_remotes/rsync.html, ./html/summary.html, ./html/templates/bare.tmpl, ./html/templates/walkthrough.tmpl, ./html/todo.html, ./html/todo/S3.html, ./html/todo/add_--exclude_option_to_git_annex_find.html, ./html/todo/add_a_git_backend.html, ./html/todo/auto_remotes.html, ./html/todo/auto_remotes/discussion.html, ./html/todo/backendSHA1.html, ./html/todo/branching.html, ./html/todo/cache_key_info.html, ./html/todo/checkout.html, ./html/todo/done.html, ./html/todo/file_copy_progress_bar.html, ./html/todo/fsck.html, ./html/todo/git-annex-shell.html, ./html/todo/git-annex_unused_eats_memory.html, ./html/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.html, ./html/todo/gitrm.html, ./html/todo/hidden_files.html, ./html/todo/immutable_annexed_files.html, ./html/todo/network_remotes.html, ./html/todo/object_dir_reorg_v2.html, ./html/todo/parallel_possibilities.html, ./html/todo/pushpull.html, ./html/todo/rsync.html, ./html/todo/smudge.html, ./html/todo/speed_up_fsck.html, ./html/todo/support-non-utf8-locales.html, ./html/todo/support_S3_multipart_uploads.html, ./html/todo/symlink_farming_commit_hook.html, ./html/todo/tahoe_lfs_for_reals.html, ./html/todo/union_mounting.html, ./html/todo/use_cp_reflink.html, ./html/todo/using_url_backend.html, ./html/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.html, ./html/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.html, ./html/todo/wishlist:___34__git_annex_add__34___multiple_processes.html, ./html/todo/wishlist:_support_for_more_ssh_urls_.html, ./html/todo/wishlist:_swift_backend.html, ./html/transferring_data.html, ./html/trust.html, ./html/upgrades.html, ./html/upgrades/SHA_size.html, ./html/use_case/Alice.html, ./html/use_case/Bob.html, ./html/users.html, ./html/users/chrysn.html, ./html/users/fmarier.html, ./html/users/joey.html, ./html/walkthrough.html, ./html/walkthrough/Internet_Archive_via_S3.html, ./html/walkthrough/adding_a_remote.html, ./html/walkthrough/adding_files.html, ./html/walkthrough/backups.html, ./html/walkthrough/creating_a_repository.html, ./html/walkthrough/fsck:_verifying_your_data.html, ./html/walkthrough/fsck:_when_things_go_wrong.html, ./html/walkthrough/getting_file_content.html, ./html/walkthrough/migrating_data_to_a_new_backend.html, ./html/walkthrough/modifying_annexed_files.html, ./html/walkthrough/more.html, ./html/walkthrough/moving_file_content_between_repositories.html, ./html/walkthrough/recover_data_from_lost+found.html, ./html/walkthrough/removing_files.html, ./html/walkthrough/removing_files:_When_things_go_wrong.html, ./html/walkthrough/renaming_files.html, ./html/walkthrough/transferring_files:_When_things_go_wrong.html, ./html/walkthrough/untrusted_repositories.html, ./html/walkthrough/unused_data.html, ./html/walkthrough/using_Amazon_S3.html, ./html/walkthrough/using_bup.html, ./html/walkthrough/using_ssh_remotes.html, ./html/walkthrough/using_the_SHA1_backend.html, ./html/walkthrough/using_the_URL_backend.html, ./html/walkthrough/what_to_do_when_you_lose_a_repository.html, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
 
 Executable git-annex
   Main-Is: git-annex.hs

From 3efba481b50ad8de6efe5869db90ac69f5c022a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 15:31:36 -0400
Subject: [PATCH 1975/8313] remove Extra-Source-Files nonsense

will need to find a better way to make sdist work, this is not livable
---
 configure.hs    | 4 ----
 git-annex.cabal | 1 -
 2 files changed, 5 deletions(-)

diff --git a/configure.hs b/configure.hs
index 5f0ff5a1e3..d47dddf1df 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,8 +2,6 @@
 
 import System.Directory
 import Data.List
-import Data.String.Utils
-import System.Cmd.Utils
 
 import TestConfig
 
@@ -73,11 +71,9 @@ getVersionString = do
 cabalSetup :: IO ()
 cabalSetup = do
 	version <- getVersionString
-	(_, filelist) <- pipeLinesFrom "find" (words ". -name .git -prune -o -name dist -prune -o -not -name *.hi -not -name *.o -not -name configure -not -name *.tmp -type f -print")
 	cabal <- readFile cabalfile
 	writeFile tmpcabalfile $ unlines $ 
 		map (setfield "Version" version) $
-		map (setfield "Extra-Source-Files" $ join ", " $ sort filelist) $
 		lines cabal
 	renameFile tmpcabalfile cabalfile
 	where
diff --git a/git-annex.cabal b/git-annex.cabal
index aa11ba381f..a4b8aef993 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -24,7 +24,6 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
-Extra-Source-Files: ./.gitattributes, ./.gitignore, ./Annex.hs, ./AnnexQueue.hs, ./Backend.hs, ./Backend/File.hs, ./Backend/SHA.hs, ./Backend/URL.hs, ./Backend/WORM.hs, ./BackendList.hs, ./Base64.hs, ./Branch.hs, ./CmdLine.hs, ./Command.hs, ./Command/Add.hs, ./Command/ConfigList.hs, ./Command/Copy.hs, ./Command/Describe.hs, ./Command/Drop.hs, ./Command/DropKey.hs, ./Command/DropUnused.hs, ./Command/Find.hs, ./Command/Fix.hs, ./Command/FromKey.hs, ./Command/Fsck.hs, ./Command/Get.hs, ./Command/InAnnex.hs, ./Command/Init.hs, ./Command/InitRemote.hs, ./Command/Lock.hs, ./Command/Map.hs, ./Command/Merge.hs, ./Command/Migrate.hs, ./Command/Move.hs, ./Command/PreCommit.hs, ./Command/RecvKey.hs, ./Command/Semitrust.hs, ./Command/SendKey.hs, ./Command/SetKey.hs, ./Command/Status.hs, ./Command/Trust.hs, ./Command/Unannex.hs, ./Command/Uninit.hs, ./Command/Unlock.hs, ./Command/Untrust.hs, ./Command/Unused.hs, ./Command/Upgrade.hs, ./Command/Version.hs, ./Command/Whereis.hs, ./Config.hs, ./Content.hs, ./CopyFile.hs, ./Crypto.hs, ./DataUnits.hs, ./Dot.hs, ./Git.hs, ./Git/LsFiles.hs, ./Git/Queue.hs, ./Git/UnionMerge.hs, ./GitAnnex.hs, ./LocationLog.hs, ./Locations.hs, ./Makefile, ./Messages.hs, ./Options.hs, ./PresenceLog.hs, ./README, ./Remote.hs, ./Remote/Bup.hs, ./Remote/Directory.hs, ./Remote/Encryptable.hs, ./Remote/Git.hs, ./Remote/Hook.hs, ./Remote/Rsync.hs, ./Remote/S3real.hs, ./Remote/S3stub.hs, ./Remote/Special.hs, ./Remote/Web.hs, ./RsyncFile.hs, ./Setup.hs, ./Ssh.hs, ./StatFS.hs, ./StatFS.hsc, ./SysConfig.hs, ./TestConfig.hs, ./Touch.hs, ./Touch.hsc, ./Trust.hs, ./Types.hs, ./Types/Backend.hs, ./Types/BranchState.hs, ./Types/Crypto.hs, ./Types/Key.hs, ./Types/Remote.hs, ./Types/TrustLevel.hs, ./Types/UUID.hs, ./UUID.hs, ./Upgrade.hs, ./Upgrade/V0.hs, ./Upgrade/V1.hs, ./Upgrade/V2.hs, ./Utility.hs, ./Version.hs, ./configure.hs, ./debian/NEWS, ./debian/changelog, ./debian/compat, ./debian/control, ./debian/copyright, ./debian/doc-base, ./debian/manpages, ./debian/rules, ./doc/.ikiwiki/indexdb, ./doc/.ikiwiki/lockfile, ./doc/GPL, ./doc/backends.mdwn, ./doc/bare_repositories.mdwn, ./doc/bugs.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong.mdwn, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_1_74de3091e8bfd7acd6795e61f39f07c6._comment, ./doc/bugs/Displayed_copy_speed_is_wrong/comment_2_8b240de1d5ae9229fa2d77d1cc15a552._comment, ./doc/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/bugs/Makefile_is_missing_dependancies.mdwn, ./doc/bugs/Makefile_is_missing_dependancies/comment_1_5a3da5f79c8563c7a450aa29728abe7c._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_2_416f12dbd0c2b841fac8164645b81df5._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_3_c38b6f4abc9b9ad413c3b83ca04386c3._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_4_cc13873175edf191047282700315beee._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_5_0a1c52e2c96d19b9c3eb7e99b8c2434f._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_6_24119fc5d5963ce9dd669f7dcf006859._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_7_96fd4725df4b54e670077a18d3ac4943._comment, ./doc/bugs/Makefile_is_missing_dependancies/comment_8_a3555e3286cdc2bfeb9cde0ff727ba74._comment, ./doc/bugs/Name_scheme_does_not_follow_git__39__s_rules.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.mdwn, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_1_c871605e187f539f3bfe7478433e7fb5._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_2_e6f1e9eee8b8dfb60ca10c8cfd807ac9._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_3_be62be5fe819acc0cb8b878802decd46._comment, ./doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment, ./doc/bugs/No_version_information_from_cli.mdwn, ./doc/bugs/Problems_running_make_on_osx.mdwn, ./doc/bugs/Problems_running_make_on_osx/comment_10_94e4ac430140042a2d0fb5a16d86b4e5._comment, ./doc/bugs/Problems_running_make_on_osx/comment_11_56f1143fa191361d63b441741699e17f._comment, ./doc/bugs/Problems_running_make_on_osx/comment_12_ec5131624d0d2285d3b6880e47033f97._comment, ./doc/bugs/Problems_running_make_on_osx/comment_13_88ed095a448096bf8a69015a04e64df1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_14_89a960b6706ed703b390a81a8bc4e311._comment, ./doc/bugs/Problems_running_make_on_osx/comment_15_6b8867b8e48bf807c955779c9f8f0909._comment, ./doc/bugs/Problems_running_make_on_osx/comment_16_5c2dd6002aadaab30841b77a5f5aed34._comment, ./doc/bugs/Problems_running_make_on_osx/comment_17_62fccb04b0e4b695312f7a3f32fb96ee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_18_64fab50d95de619eb2e8f08f90237de1._comment, ./doc/bugs/Problems_running_make_on_osx/comment_19_4253988ed178054c8b6400beeed68a29._comment, ./doc/bugs/Problems_running_make_on_osx/comment_1_34120e82331ace01a6a4960862d38f2d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_20_7db27d1a22666c831848bc6c06d66a84._comment, ./doc/bugs/Problems_running_make_on_osx/comment_2_cc53d1681d576186dbc868dd9801d551._comment, ./doc/bugs/Problems_running_make_on_osx/comment_3_68f0f8ae953589ae26d57310b40c878d._comment, ./doc/bugs/Problems_running_make_on_osx/comment_4_c52be386f79f14c8570a8f1397c68581._comment, ./doc/bugs/Problems_running_make_on_osx/comment_5_7f1330a1e541b0f3e2192e596d7f7bee._comment, ./doc/bugs/Problems_running_make_on_osx/comment_6_0c46f5165ceb5a7b9ea9689c33b3a4f8._comment, ./doc/bugs/Problems_running_make_on_osx/comment_7_237a137cce58a28abcc736cbf2c420b0._comment, ./doc/bugs/Problems_running_make_on_osx/comment_8_efafa203addf8fa79e33e21a87fb5a2b._comment, ./doc/bugs/Problems_running_make_on_osx/comment_9_cc283b485b3c95ba7eebc8f0c96969b3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.mdwn, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_1_dc5ae7af499203cfd903e866595b8fea._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_2_c62daf5b3bfcd2f684262c96ef6628c1._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_3_e1f39c4af5bdb0daabf000da80858cd9._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_4_bb6b814ab961818d514f6553455d2bf3._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_5_5bb128f6d2ca4b5e4d881fae297fa1f8._comment, ./doc/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing/comment_6_63fb74da342751fc35e1850409c506f6._comment, ./doc/bugs/S3_memory_leaks.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre.mdwn, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_1_7cb5561f11dfc7726a537ddde2477489._comment, ./doc/bugs/Unfortunate_interaction_with_Calibre/comment_2_b8ae4bc589c787dacc08ab2ee5491d6e._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_1_77aa9cafbe20367a41377f3edccc9ddb._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_2_fe735d728878d889ccd34ec12b3a7dea._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_3_2bf0f02d27190578e8f4a32ddb195a0a._comment, ./doc/bugs/WORM:_Handle_long_filenames_correctly/comment_4_8f7ba9372463863dda5aae13205861bf._comment, ./doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn, ./doc/bugs/annex_add_in_annex.mdwn, ./doc/bugs/backend_version_upgrade_leaves_repo_unusable.mdwn, ./doc/bugs/bare_git_repos.mdwn, ./doc/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.mdwn, ./doc/bugs/building_on_lenny.mdwn, ./doc/bugs/check_for_curl_in_configure.hs.mdwn, ./doc/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.mdwn, ./doc/bugs/conflicting_haskell_packages.mdwn, ./doc/bugs/conflicting_haskell_packages/comment_1_e552a6cc6d7d1882e14130edfc2d6b3b._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_10_435f87d54052f264096a8f23e99eae06._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_11_9be0aef403a002c1706d17deee45763c._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_12_26d60661196f63fd01ee4fbb6e2340e7._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_13_ead55b915d3b92a62549b2957ad211c8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_14_191de89d3988083d9cf001799818ff4a._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_15_b3e3b338ccfa0a32510c78ba1b1bb617._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_16_04a9f4468c3246c8eff3dbe21dd90101._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_1_6a41bf7e2db83db3a01722b516fb6886._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_2_9f5f1dbffb2dd24f4fcf8c2027bf0384._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_3_b596b5cfd3377e58dbbb5d509d026b90._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_4_d7112c315fb016a8a399e24e9b6461d8._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_5_4ea29a6f8152eddf806c536de33ef162._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_6_0d85f114a103bd6532a3b3b24466012e._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_7_d38d5bee6d360b0ea852f39e3a7b1bc6._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_8_29c3de4bf5fbd990b230c443c0303cbe._comment, ./doc/bugs/copy_fast_confusing_with_broken_locationlog/comment_9_2cee4f6bd6db7518fd61453c595162c6._comment, ./doc/bugs/done.mdwn, ./doc/bugs/dotdot_problem.mdwn, ./doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn, ./doc/bugs/encrypted_S3_stalls.mdwn, ./doc/bugs/error_propigation.mdwn, ./doc/bugs/error_with_file_names_starting_with_dash.mdwn, ./doc/bugs/fat_support.mdwn, ./doc/bugs/fat_support/comment_1_04bcc4795d431e8cb32293aab29bbfe2._comment, ./doc/bugs/fat_support/comment_2_bb4a97ebadb5c53809fc78431eabd7c8._comment, ./doc/bugs/fat_support/comment_3_df3b943bc1081a8f3f7434ae0c8e061e._comment, ./doc/bugs/fat_support/comment_4_90a8a15bedd94480945a374f9d706b86._comment, ./doc/bugs/fat_support/comment_5_64bbf89de0836673224b83fdefa0407b._comment, ./doc/bugs/free_space_checking.mdwn, ./doc/bugs/free_space_checking/comment_1_a868e805be43c5a7c19c41f1af8e41e6._comment, ./doc/bugs/free_space_checking/comment_2_8a65f6d3dcf5baa3f7f2dbe1346e2615._comment, ./doc/bugs/free_space_checking/comment_3_0fc6ff79a357b1619d13018ccacc7c10._comment, ./doc/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.mdwn, ./doc/bugs/fsck_output.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.mdwn, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_1_1c19e716069911f17bbebd196d9e4b61._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_2_a4d66f29d257044e548313e014ca3dc3._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_3_f5f1081eb18143383b2fb1f57d8640f5._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_4_b1f818b85c3540591c48e7ba8560d070._comment, ./doc/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799/comment_5_67406dd8d9bd4944202353508468c907._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx.mdwn, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_10_f3594de3ba2ab17771a4b116031511bb._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_11_97de7252bf5d2a4f1381f4b2b4e24ef8._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_12_f1c53c3058a587185e7a78d84987539d._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_13_4f56aea35effe5c10ef37d7ad7adb48c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_14_cc2a53c31332fe4b828ef1e72c2a4d49._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_15_37f1d669c1fa53ee371f781c7bb820ae._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_16_8a4ab1af59098f4950726cf53636c2b3._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_17_515d5c5fbf5bd0c188a4f1e936d913e2._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_18_db64c91dd1322a0ab168190686db494f._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_19_ff555c271637af065203ca99c9eeaf89._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_1_9a7b09de132097100c1a68ea7b846727._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_20_7e328b970169fffb8bce373d1522743b._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_21_98f632652b0db9131b0173d3572f4d62._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_2_174952fc3e3be12912e5fcfe78f2dd13._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_3_a18ada7ac74c63be5753fdb2fe68dae5._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_4_039e945617a6c1852c96974a402db29c._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_5_eacd0b18475c05ab9feed8cf7290b79a._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_6_e55117cb628dc532e468519252571474._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_7_0f4f471102e394ebb01da40e4d0fd9f6._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_8_68e2d6ccdb9622b879e4bc7005804623._comment, ./doc/bugs/git-annex_directory_hashing_problems_on_osx/comment_9_45b11ddd200261115b653c7a14d28aa9._comment, ./doc/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.mdwn, ./doc/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.mdwn, ./doc/bugs/git_annex_copy_--fast_does_not_copy_files.mdwn, ./doc/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn, ./doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_1_fc59fbd1cdf8ca97b0a4471d9914aaa1._comment, ./doc/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.mdwn, ./doc/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.mdwn, ./doc/bugs/git_annex_initremote_walks_.git-annex.mdwn, ./doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn, ./doc/bugs/git_annex_should_use___39__git_add_-f__39___internally.mdwn, ./doc/bugs/git_annex_unlock_is_not_atomic.mdwn, ./doc/bugs/git_annex_unused_failes_on_empty_repository.mdwn, ./doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn, ./doc/bugs/git_rename_detection_on_file_move.mdwn, ./doc/bugs/git_rename_detection_on_file_move/comment_1_0531dcfa833b0321a7009526efe3df33._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_2_7101d07400ad5935f880dc00d89bf90e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_3_57010bcaca42089b451ad8659a1e018e._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_4_79d96599f757757f34d7b784e6c0e81c._comment, ./doc/bugs/git_rename_detection_on_file_move/comment_5_d61f5693d947b9736b29fca1dbc7ad76._comment, ./doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn, ./doc/bugs/ordering.mdwn, ./doc/bugs/problem_commit_normal_links.mdwn, ./doc/bugs/problems_with_utf8_names.mdwn, ./doc/bugs/scp_interrupt_to_background.mdwn, ./doc/bugs/softlink_mtime.mdwn, ./doc/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.mdwn, ./doc/bugs/tmp_file_handling.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems.mdwn, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_1_1d38283c9ea87174f3bbef9a58f5cb88._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_2_bf112edd075fbebe4fc959a387946eb9._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_3_a46080fbe82adf0986c5dc045e382501._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_4_760437bf3ba972a775bb190fb4b38202._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_5_060ba5ea88dcab2f4a0c199f13ef4f67._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_6_548303d6ffb21a9370b6904f41ff49c1._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_7_7ca00527ab5db058aadec4fe813e51fd._comment, ./doc/bugs/touch.hsc_has_problems_on_non-linux_based_systems/comment_8_881aecb9ae671689453f6d5d780d844b._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.mdwn, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_1_1931e733f0698af5603a8b92267203d4._comment, ./doc/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken/comment_2_40920b88537b7715395808d8aa94bf03._comment, ./doc/bugs/unannex_vs_unlock_hook_confusion.mdwn, ./doc/bugs/unhappy_without_UTF8_locale.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories.mdwn, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_1_9ca2da52f3c8add0276b72d6099516a6._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_2_e14e84b770305893f2fc6e4938359f47._comment, ./doc/bugs/upgrade_left_untracked_.git-annex__47____42___directories/comment_3_ec04e306c96fd20ab912aea54a8340aa._comment, ./doc/bugs/weird_local_clone_confuses.mdwn, ./doc/cheatsheet.mdwn, ./doc/comments.mdwn, ./doc/contact.mdwn, ./doc/copies.mdwn, ./doc/design.mdwn, ./doc/design/encryption.mdwn, ./doc/design/encryption/comment_1_4715ffafb3c4a9915bc33f2b26aaa9c1._comment, ./doc/design/encryption/comment_2_a610b3d056a059899178859a3a821ea5._comment, ./doc/design/encryption/comment_3_cca186a9536cd3f6e86994631b14231c._comment, ./doc/design/encryption/comment_4_8f3ba3e504b058791fc6e6f9c38154cf._comment, ./doc/distributed_version_control.mdwn, ./doc/download.mdwn, ./doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment, ./doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment, ./doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment, ./doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment, ./doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment, ./doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment, ./doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment, ./doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment, ./doc/encryption.mdwn, ./doc/encryption/comment_1_1afca8d7182075d46db41f6ad3dd5911._comment, ./doc/feeds.mdwn, ./doc/forum.mdwn, ./doc/forum/Behaviour_of_fsck.mdwn, ./doc/forum/Behaviour_of_fsck/comment_1_0e40f158b3f4ccdcaab1408d858b68b8._comment, ./doc/forum/Behaviour_of_fsck/comment_2_ead36a23c3e6efa1c41e4555f93e014e._comment, ./doc/forum/Behaviour_of_fsck/comment_3_97848f9a3db89c0427cfb671ba13300e._comment, ./doc/forum/Behaviour_of_fsck/comment_4_e4911dc6793f98fb81151daacbe49968._comment, ./doc/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.mdwn, ./doc/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__/comment_1_c25900b9d2d62cc0b8c77150bcfebadf._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable.mdwn, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_1_8c1eea6dfec8b7e1c7a371b6e9c26118._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_2_f6ff8306c946219dbe39bb8938a349ab._comment, ./doc/forum/Need_new_build_instructions_for_Debian_stable/comment_3_bcda70cbfc7c1a14fa82da70f9f876e2._comment, ./doc/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.mdwn, ./doc/forum/OSX__39__s_haskell-platform_statically_links_things.mdwn, ./doc/forum/Problems_with_large_numbers_of_files.mdwn, ./doc/forum/Problems_with_large_numbers_of_files/comment_1_08791cb78b982087c2a07316fe3ed46c._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_2_0392a11219463e40c53bae73c8188b69._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_3_537e9884c1488a7a4bcf131ea63b71f7._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_4_7cb65d013e72bd2b7e90452079d42ac9._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_5_86a42ee3173a5d38f803e64b79496ab3._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_6_4551274288383c9cc27cbf85b122d307._comment, ./doc/forum/Problems_with_large_numbers_of_files/comment_7_d18cf944352f8303799c86f2c0354e8e._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.mdwn, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_1_426482e6eb3a27687a48f24f6ef2332f._comment, ./doc/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__/comment_2_af4f8b52526d8bea2904c95406fd2796._comment, ./doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo.mdwn, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_1_044f1c5e5f7a939315c28087495a8ba8._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_2_e854b93415d5ab80eda8e3be3b145ec2._comment, ./doc/forum/__34__git_annex_lock__34___very_slow_for_big_repo/comment_3_95c110500bc54013bc1969c1a9c8f842._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull.mdwn, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_1_3a0bf74b51586354b7a91f8b43472376._comment, ./doc/forum/bainstorming:_git_annex_push___38___pull/comment_2_b02ca09914e788393c01196686f95831._comment, ./doc/forum/batch_check_on_remote_when_using_copy.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__.mdwn, ./doc/forum/can_git-annex_replace_ddm__63__/comment_1_aa05008dfe800474ff76678a400099e1._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_2_008554306dd082d7f543baf283510e92._comment, ./doc/forum/can_git-annex_replace_ddm__63__/comment_3_4c69097fe2ee81359655e59a03a9bb8d._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_1_3deb2c31cad37a49896f00d600253ee3._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_2_627f54d158d3ca4b72e45b4da70ff5cd._comment, ./doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote/comment_3_3f49dab11aae5df0c4eb5e4b8d741379._comment, ./doc/forum/git-annex_communication_channels.mdwn, ./doc/forum/git-annex_communication_channels/comment_1_198325d2e9337c90f026396de89eec0e._comment, ./doc/forum/git-annex_communication_channels/comment_2_c7aeefa6ef9a2e75d8667b479ade1b7f._comment, ./doc/forum/git-annex_communication_channels/comment_3_1ff08a3e0e63fa0e560cbc9602245caa._comment, ./doc/forum/git-annex_communication_channels/comment_4_1ba6ddf54843c17c7d19a9996f2ab712._comment, ./doc/forum/git-annex_communication_channels/comment_5_404b723a681eb93fee015cea8024b6bc._comment, ./doc/forum/git-annex_communication_channels/comment_6_0d87d0e26461494b1d7f8a701a924729._comment, ./doc/forum/git-annex_communication_channels/comment_7_2c87c7a0648fe87c2bf6b4391f1cc468._comment, ./doc/forum/git-annex_on_OSX.mdwn, ./doc/forum/hashing_objects_directories.mdwn, ./doc/forum/hashing_objects_directories/comment_1_c55c56076be4f54251b0b7f79f28a607._comment, ./doc/forum/hashing_objects_directories/comment_2_504c96959c779176f991f4125ea22009._comment, ./doc/forum/hashing_objects_directories/comment_3_9134bde0a13aac0b6a4e5ebabd7f22e8._comment, ./doc/forum/hashing_objects_directories/comment_4_0de9170e429cbfea66f5afa8980d45ac._comment, ./doc/forum/hashing_objects_directories/comment_5_ef6cfd49d24c180c2d0a062e5bd3a0be._comment, ./doc/forum/incompatible_versions__63__.mdwn, ./doc/forum/incompatible_versions__63__/comment_1_629f28258746d413e452cbd42a1a43f4._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex.mdwn, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_1_4181bf34c71e2e8845e6e5fb55d53381._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_2_5f08da5e21c0b3b5a8d1e4408c0d6405._comment, ./doc/forum/migrate_existing_git_repository_to_git-annex/comment_3_f483038c006cf7dcccf1014fa771744f._comment, ./doc/forum/migration_to_git-annex_and_rsync.mdwn, ./doc/forum/new_microfeatures.mdwn, ./doc/forum/new_microfeatures/comment_1_058bd517c6fffaf3446b1f5d5be63623._comment, ./doc/forum/new_microfeatures/comment_2_41ad904c68e89c85e1fc49c9e9106969._comment, ./doc/forum/new_microfeatures/comment_3_a1a9347b5bc517f2a89a8b292c3f8517._comment, ./doc/forum/new_microfeatures/comment_4_5a6786dc52382fff5cc42fdb05770196._comment, ./doc/forum/new_microfeatures/comment_5_3c627d275586ff499d928a8f8136babf._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_1_b3f22f9be02bc4f2d5a121db3d753ff5._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_2_f94abce32ef818176b42a3cc860691ae._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_3_0c8e77fe248e00bd990d568623e5a5c9._comment, ./doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk/comment_4_4b7e8f9521d61900d9ad418e74808ffb._comment, ./doc/forum/relying_on_git_for_numcopies.mdwn, ./doc/forum/relying_on_git_for_numcopies/comment_1_8ad3cccd7f66f6423341d71241ba89fc._comment, ./doc/forum/relying_on_git_for_numcopies/comment_2_be6acbc26008a9cb54e7b8f498f2c2a2._comment, ./doc/forum/relying_on_git_for_numcopies/comment_3_43d8e1513eb9947f8a503f094c03f307._comment, ./doc/forum/rsync_over_ssh__63__.mdwn, ./doc/forum/rsync_over_ssh__63__/comment_1_ee21f32e90303e20339e0a568321bbbe._comment, ./doc/forum/rsync_over_ssh__63__/comment_2_aa690da6ecfb2b30fc5080ad76dc77b1._comment, ./doc/forum/seems_to_build_fine_on_haskell_platform_2011.mdwn, ./doc/forum/sparse_git_checkouts_with_annex.mdwn, ./doc/forum/sparse_git_checkouts_with_annex/comment_1_c7dc199c5740a0e7ba606dfb5e3e579a._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_2_e357db3ccc4079f07a291843975535eb._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_3_fcfafca994194d57dccf5319c7c9e646._comment, ./doc/forum/sparse_git_checkouts_with_annex/comment_4_04dc14880f31eee2b6d767d4d4258c5a._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_1_76bb33ce45ce6a91b86454147463193b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_2_4d9b9d47d01d606a475678f630797bf9._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_3_8a812b11fcc2dc3b6fcf01cdbbb8459d._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_4_fc98c819bc5eb4d7c9e74d87fb4f6f3b._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_5_c459fb479fe7b13eaea2377cfc1923a6._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_6_2e9da5a919bbbc27b32de3b243867d4f._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_7_d636c868524b2055ee85832527437f90._comment, ./doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs/comment_8_39dc449cc60a787c3bfbfaaac6f9be0c._comment, ./doc/forum/unannex_alternatives.mdwn, ./doc/forum/unannex_alternatives/comment_1_dcd4cd41280b41512bbdffafaf307993._comment, ./doc/forum/unannex_alternatives/comment_2_58a72a9fe0f58c7af0b4d7927a2dd21d._comment, ./doc/forum/unannex_alternatives/comment_3_b1687fc8f9e7744327bbeb6f0635d1cd._comment, ./doc/forum/wishlist:_command_options_changes.mdwn, ./doc/forum/wishlist:_command_options_changes/comment_1_bfba72a696789bf21b2435dea15f967a._comment, ./doc/forum/wishlist:_command_options_changes/comment_2_f6a637c78c989382e3c22d41b7fb4cc2._comment, ./doc/forum/wishlist:_command_options_changes/comment_3_bf1114533d2895804e531e76eb6b8095._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files.mdwn, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_1_cceccc1a1730ac688d712b81a44e31c3._comment, ./doc/forum/wishlist:_define_remotes_that_must_have_all_files/comment_2_eec848fcf3979c03cbff2b7407c75a7a._comment, ./doc/forum/wishlist:_do_round_robin_downloading_of_data.mdwn, ./doc/forum/wishlist:_do_round_robin_downloading_of_data/comment_1_460335b0e59ad03871c524f1fe812357._comment, ./doc/forum/wishlist:_git-annex_replicate.mdwn, ./doc/forum/wishlist:_git-annex_replicate/comment_1_9926132ec6052760cdf28518a24e2358._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_2_c43932f4194aba8fb2470b18e0817599._comment, ./doc/forum/wishlist:_git-annex_replicate/comment_3_c13f4f9c3d5884fc6255fd04feadc2b1._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.mdwn, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_1_d5413c8acce308505e4e2bec82fb1261._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_2_0aa227c85d34dfff4e94febca44abea8._comment, ./doc/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults/comment_3_2082f4d708a584a1403cc1d4d005fb56._comment, ./doc/forum/wishlist:_git_annex_status.mdwn, ./doc/forum/wishlist:_git_annex_status/comment_1_994bfd12c5d82e08040d6116915c5090._comment, ./doc/forum/wishlist:_git_annex_status/comment_2_c2b0ce025805b774dc77ce264a222824._comment, ./doc/forum/wishlist:_git_annex_status/comment_3_d1fd70c67243971c96d59e1ffb7ef6e7._comment, ./doc/forum/wishlist:_git_annex_status/comment_4_9aeeb83d202dc8fb33ff364b0705ad94._comment, ./doc/forum/wishlist:_git_backend_for_git-annex.mdwn, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_1_04319051fedc583e6c326bb21fcce5a5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_2_7f529f19a47e10b571f65ab382e97fd5._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_3_a077bbad3e4b07cce019eb55a45330e7._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_4_ecca429e12d734b509c671166a676c9d._comment, ./doc/forum/wishlist:_git_backend_for_git-annex/comment_5_3459f0b41d818c23c8fb33edb89df634._comment, ./doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_1_6f07d9cc92cf8b4927b3a7d1820c9140._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_2_84e4414c88ae91c048564a2cdc2d3250._comment, ./doc/forum/wishlist:_special_remote_for_sftp_or_rsync/comment_3_79de7ac44e3c0f0f5691a56d3fb88897._comment, ./doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/forum/wishlist:_traffic_accounting_for_git-annex.mdwn, ./doc/forum/wishlist:alias_system.mdwn, ./doc/forum/working_without_git-annex_commits.mdwn, ./doc/future_proofing.mdwn, ./doc/git-annex-shell.mdwn, ./doc/git-annex.mdwn, ./doc/git-union-merge.mdwn, ./doc/index.mdwn, ./doc/install.mdwn, ./doc/install/Debian.mdwn, ./doc/install/Fedora.mdwn, ./doc/install/FreeBSD.mdwn, ./doc/install/OSX.mdwn, ./doc/install/Ubuntu.mdwn, ./doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment, ./doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment, ./doc/internals.mdwn, ./doc/location_tracking.mdwn, ./doc/logo.png, ./doc/logo_small.png, ./doc/news.mdwn, ./doc/news/LWN_article.mdwn, ./doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn, ./doc/news/version_0.20110521.mdwn, ./doc/news/version_0.20110522.mdwn, ./doc/news/version_0.20110601.mdwn, ./doc/news/version_0.20110610.mdwn, ./doc/news/version_3.20110624.mdwn, ./doc/not.mdwn, ./doc/repomap.png, ./doc/special_remotes.mdwn, ./doc/special_remotes/S3.mdwn, ./doc/special_remotes/bup.mdwn, ./doc/special_remotes/directory.mdwn, ./doc/special_remotes/hook.mdwn, ./doc/special_remotes/rsync.mdwn, ./doc/summary.mdwn, ./doc/templates/bare.tmpl, ./doc/templates/walkthrough.tmpl, ./doc/todo.mdwn, ./doc/todo/S3.mdwn, ./doc/todo/add_--exclude_option_to_git_annex_find.mdwn, ./doc/todo/add_a_git_backend.mdwn, ./doc/todo/auto_remotes.mdwn, ./doc/todo/auto_remotes/discussion.mdwn, ./doc/todo/backendSHA1.mdwn, ./doc/todo/branching.mdwn, ./doc/todo/cache_key_info.mdwn, ./doc/todo/cache_key_info/comment_1_578df1b3b2cbfdc4aa1805378f35dc48._comment, ./doc/todo/checkout.mdwn, ./doc/todo/done.mdwn, ./doc/todo/file_copy_progress_bar.mdwn, ./doc/todo/fsck.mdwn, ./doc/todo/git-annex-shell.mdwn, ./doc/todo/git-annex_unused_eats_memory.mdwn, ./doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn, ./doc/todo/gitrm.mdwn, ./doc/todo/hidden_files.mdwn, ./doc/todo/immutable_annexed_files.mdwn, ./doc/todo/network_remotes.mdwn, ./doc/todo/object_dir_reorg_v2.mdwn, ./doc/todo/object_dir_reorg_v2/comment_1_ba03333dc76ff49eccaba375e68cb525._comment, ./doc/todo/object_dir_reorg_v2/comment_2_81276ac309959dc741bc90101c213ab7._comment, ./doc/todo/object_dir_reorg_v2/comment_3_79bdf9c51dec9f52372ce95b53233bb2._comment, ./doc/todo/object_dir_reorg_v2/comment_4_93aada9b1680fed56cc6f0f7c3aca5e5._comment, ./doc/todo/object_dir_reorg_v2/comment_5_821c382987f105da72a50e0a5ce61fdc._comment, ./doc/todo/object_dir_reorg_v2/comment_6_8834c3a3f1258c4349d23aff8549bf35._comment, ./doc/todo/object_dir_reorg_v2/comment_7_42501404c82ca07147e2cce0cff59474._comment, ./doc/todo/parallel_possibilities.mdwn, ./doc/todo/parallel_possibilities/comment_1_d8e34fc2bc4e5cf761574608f970d496._comment, ./doc/todo/parallel_possibilities/comment_2_adb76f06a7997abe4559d3169a3181c3._comment, ./doc/todo/pushpull.mdwn, ./doc/todo/rsync.mdwn, ./doc/todo/smudge.mdwn, ./doc/todo/smudge/comment_1_4ea616bcdbc9e9a6fae9f2e2795c31c9._comment, ./doc/todo/smudge/comment_2_e04b32caa0d2b4c577cdaf382a3ff7f6._comment, ./doc/todo/speed_up_fsck.mdwn, ./doc/todo/support-non-utf8-locales.mdwn, ./doc/todo/support_S3_multipart_uploads.mdwn, ./doc/todo/symlink_farming_commit_hook.mdwn, ./doc/todo/tahoe_lfs_for_reals.mdwn, ./doc/todo/tahoe_lfs_for_reals/comment_1_0a4793ce6a867638f6e510e71dd4bb44._comment, ./doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment, ./doc/todo/union_mounting.mdwn, ./doc/todo/use_cp_reflink.mdwn, ./doc/todo/using_url_backend.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn, ./doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command/comment_1_3f9c0d08932c2ede61c802a91261a1f7._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_1_fd213310ee548d8726791d2b02237fde._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_2_4394bde1c6fd44acae649baffe802775._comment, ./doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_3_076cb22057583957d5179d8ba9004605._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_1_85b14478411a33e6186a64bd41f0910d._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_2_82e857f463cfdf73c70f6c0a9f9a31d6._comment, ./doc/todo/wishlist:___34__git_annex_add__34___multiple_processes/comment_3_8af85eba7472d9025c6fae4f03e3ad75._comment, ./doc/todo/wishlist:_support_for_more_ssh_urls_.mdwn, ./doc/todo/wishlist:_swift_backend.mdwn, ./doc/todo/wishlist:_swift_backend/comment_1_e6efbb35f61ee521b473a92674036788._comment, ./doc/todo/wishlist:_swift_backend/comment_2_5d8c83b0485112e98367b7abaab3f4e3._comment, ./doc/transferring_data.mdwn, ./doc/trust.mdwn, ./doc/upgrades.mdwn, ./doc/upgrades/SHA_size.mdwn, ./doc/use_case/Alice.mdwn, ./doc/use_case/Bob.mdwn, ./doc/users.mdwn, ./doc/users/chrysn.mdwn, ./doc/users/fmarier.mdwn, ./doc/users/joey.mdwn, ./doc/walkthrough.mdwn, ./doc/walkthrough/Internet_Archive_via_S3.mdwn, ./doc/walkthrough/adding_a_remote.mdwn, ./doc/walkthrough/adding_a_remote/comment_1_0a59355bd33a796aec97173607e6adc9._comment, ./doc/walkthrough/adding_a_remote/comment_2_f8cd79ef1593a8181a7f1086a87713e8._comment, ./doc/walkthrough/adding_a_remote/comment_3_60691af4400521b5a8c8d75efe3b44cb._comment, ./doc/walkthrough/adding_a_remote/comment_4_6f7cf5c330272c96b3abeb6612075c9d._comment, ./doc/walkthrough/adding_files.mdwn, ./doc/walkthrough/backups.mdwn, ./doc/walkthrough/creating_a_repository.mdwn, ./doc/walkthrough/fsck:_verifying_your_data.mdwn, ./doc/walkthrough/fsck:_when_things_go_wrong.mdwn, ./doc/walkthrough/getting_file_content.mdwn, ./doc/walkthrough/migrating_data_to_a_new_backend.mdwn, ./doc/walkthrough/modifying_annexed_files.mdwn, ./doc/walkthrough/more.mdwn, ./doc/walkthrough/moving_file_content_between_repositories.mdwn, ./doc/walkthrough/moving_file_content_between_repositories/comment_1_4c30ade91fc7113a95960aa3bd1d5427._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_2_7d90e1e150e7524ba31687108fcc38d6._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_3_558d80384434207b9cfc033763863de3._comment, ./doc/walkthrough/moving_file_content_between_repositories/comment_4_a2f343eceed9e9fba1670f21e0fc6af4._comment, ./doc/walkthrough/recover_data_from_lost+found.mdwn, ./doc/walkthrough/removing_files.mdwn, ./doc/walkthrough/removing_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/renaming_files.mdwn, ./doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn, ./doc/walkthrough/untrusted_repositories.mdwn, ./doc/walkthrough/unused_data.mdwn, ./doc/walkthrough/using_Amazon_S3.mdwn, ./doc/walkthrough/using_bup.mdwn, ./doc/walkthrough/using_ssh_remotes.mdwn, ./doc/walkthrough/using_the_SHA1_backend.mdwn, ./doc/walkthrough/using_the_URL_backend.mdwn, ./doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn, ./git-annex, ./git-annex-shell, ./git-annex-shell.1, ./git-annex-shell.hs, ./git-annex.1, ./git-annex.cabal, ./git-annex.hs, ./git-union-merge, ./git-union-merge.1, ./git-union-merge.hs, ./html/GPL, ./html/backends.html, ./html/bare_repositories.html, ./html/bugs.html, ./html/bugs/Displayed_copy_speed_is_wrong.html, ./html/bugs/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/bugs/Makefile_is_missing_dependancies.html, ./html/bugs/Name_scheme_does_not_follow_git__39__s_rules.html, ./html/bugs/No_easy_way_to_re-inject_a_file_into_an_annex.html, ./html/bugs/No_version_information_from_cli.html, ./html/bugs/Problems_running_make_on_osx.html, ./html/bugs/S3_bucket_uses_the_same_key_for_encryption_and_hashing.html, ./html/bugs/S3_memory_leaks.html, ./html/bugs/Unfortunate_interaction_with_Calibre.html, ./html/bugs/WORM:_Handle_long_filenames_correctly.html, ./html/bugs/add_range_argument_to___34__git_annex_dropunused__34___.html, ./html/bugs/annex_add_in_annex.html, ./html/bugs/backend_version_upgrade_leaves_repo_unusable.html, ./html/bugs/bare_git_repos.html, ./html/bugs/build_issue_with_latest_release_0.20110522-1-gde817ba.html, ./html/bugs/building_on_lenny.html, ./html/bugs/check_for_curl_in_configure.hs.html, ./html/bugs/configure_script_should_detect_uuidgen_instead_of_just_uuid.html, ./html/bugs/conflicting_haskell_packages.html, ./html/bugs/copy_fast_confusing_with_broken_locationlog.html, ./html/bugs/done.html, ./html/bugs/dotdot_problem.html, ./html/bugs/dropping_files_with_a_URL_backend_fails.html, ./html/bugs/encrypted_S3_stalls.html, ./html/bugs/error_propigation.html, ./html/bugs/error_with_file_names_starting_with_dash.html, ./html/bugs/fat_support.html, ./html/bugs/free_space_checking.html, ./html/bugs/fsck__47__fix_should_check__47__fix_the_permissions_of_.git__47__annex.html, ./html/bugs/fsck_output.html, ./html/bugs/git-annex-shell:_internal_error:_evacuate__40__static__41__:_strange_closure_type_30799.html, ./html/bugs/git-annex_directory_hashing_problems_on_osx.html, ./html/bugs/git-annex_has_issues_with_git_when_staging__47__commiting_logs.html, ./html/bugs/git-annex_incorrectly_parses_bare_IPv6_addresses.html, ./html/bugs/git_annex_copy_--fast_does_not_copy_files.html, ./html/bugs/git_annex_copy_-f_REMOTE_._doesn__39__t_work_as_expected.html, ./html/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.html, ./html/bugs/git_annex_get_choke_when_remote_is_an_ssh_url_with_a_port.html, ./html/bugs/git_annex_gets_confused_about_remotes_with_dots_in_their_names.html, ./html/bugs/git_annex_initremote_walks_.git-annex.html, ./html/bugs/git_annex_migrate_leaves_old_backend_versions_around.html, ./html/bugs/git_annex_should_use___39__git_add_-f__39___internally.html, ./html/bugs/git_annex_unlock_is_not_atomic.html, ./html/bugs/git_annex_unused_failes_on_empty_repository.html, ./html/bugs/git_annex_unused_seems_to_check_for_current_path.html, ./html/bugs/git_rename_detection_on_file_move.html, ./html/bugs/minor_bug:_errors_are_not_verbose_enough.html, ./html/bugs/ordering.html, ./html/bugs/problem_commit_normal_links.html, ./html/bugs/problems_with_utf8_names.html, ./html/bugs/scp_interrupt_to_background.html, ./html/bugs/softlink_mtime.html, ./html/bugs/tests_fail_when_there_is_no_global_.gitconfig_for_the_user.html, ./html/bugs/tmp_file_handling.html, ./html/bugs/touch.hsc_has_problems_on_non-linux_based_systems.html, ./html/bugs/unannex_and_uninit_do_not_work_when_git_index_is_broken.html, ./html/bugs/unannex_vs_unlock_hook_confusion.html, ./html/bugs/unhappy_without_UTF8_locale.html, ./html/bugs/upgrade_left_untracked_.git-annex__47____42___directories.html, ./html/bugs/weird_local_clone_confuses.html, ./html/cheatsheet.html, ./html/comments.html, ./html/contact.html, ./html/copies.html, ./html/design.html, ./html/design/encryption.html, ./html/distributed_version_control.html, ./html/download.html, ./html/encryption.html, ./html/feeds.html, ./html/forum.html, ./html/forum/Behaviour_of_fsck.html, ./html/forum/Error_while_adding_a_file___34__createSymbolicLink:_already_exists__34__.html, ./html/forum/Is_an_automagic_upgrade_of_the_object_directory_safe__63__.html, ./html/forum/Need_new_build_instructions_for_Debian_stable.html, ./html/forum/OSX__39__s_default_sshd_behaviour_has_limited_paths_set.html, ./html/forum/OSX__39__s_haskell-platform_statically_links_things.html, ./html/forum/Problems_with_large_numbers_of_files.html, ./html/forum/Will_git_annex_work_on_a_FAT32_formatted_key__63__.html, ./html/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information.html, ./html/forum/__34__git_annex_lock__34___very_slow_for_big_repo.html, ./html/forum/bainstorming:_git_annex_push___38___pull.html, ./html/forum/batch_check_on_remote_when_using_copy.html, ./html/forum/can_git-annex_replace_ddm__63__.html, ./html/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.html, ./html/forum/git-annex_communication_channels.html, ./html/forum/git-annex_on_OSX.html, ./html/forum/hashing_objects_directories.html, ./html/forum/incompatible_versions__63__.html, ./html/forum/migrate_existing_git_repository_to_git-annex.html, ./html/forum/migration_to_git-annex_and_rsync.html, ./html/forum/new_microfeatures.html, ./html/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.html, ./html/forum/relying_on_git_for_numcopies.html, ./html/forum/rsync_over_ssh__63__.html, ./html/forum/seems_to_build_fine_on_haskell_platform_2011.html, ./html/forum/sparse_git_checkouts_with_annex.html, ./html/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.html, ./html/forum/unannex_alternatives.html, ./html/forum/wishlist:_command_options_changes.html, ./html/forum/wishlist:_define_remotes_that_must_have_all_files.html, ./html/forum/wishlist:_do_round_robin_downloading_of_data.html, ./html/forum/wishlist:_git-annex_replicate.html, ./html/forum/wishlist:_git_annex_put_--_same_as_get__44___but_for_defaults.html, ./html/forum/wishlist:_git_annex_status.html, ./html/forum/wishlist:_git_backend_for_git-annex.html, ./html/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one.html, ./html/forum/wishlist:_special_remote_for_sftp_or_rsync.html, ./html/forum/wishlist:_support_for_more_ssh_urls_.html, ./html/forum/wishlist:_traffic_accounting_for_git-annex.html, ./html/forum/wishlist:alias_system.html, ./html/forum/working_without_git-annex_commits.html, ./html/future_proofing.html, ./html/git-annex-shell.html, ./html/git-annex.html, ./html/git-union-merge.html, ./html/ikiwiki/ikiwiki.js, ./html/ikiwiki/relativedate.js, ./html/ikiwiki/toggle.js, ./html/index.html, ./html/install.html, ./html/install/Debian.html, ./html/install/Fedora.html, ./html/install/FreeBSD.html, ./html/install/OSX.html, ./html/install/Ubuntu.html, ./html/internals.html, ./html/location_tracking.html, ./html/logo.png, ./html/logo_small.png, ./html/news.html, ./html/not.html, ./html/repomap.png, ./html/special_remotes.html, ./html/special_remotes/S3.html, ./html/special_remotes/bup.html, ./html/special_remotes/directory.html, ./html/special_remotes/hook.html, ./html/special_remotes/rsync.html, ./html/summary.html, ./html/templates/bare.tmpl, ./html/templates/walkthrough.tmpl, ./html/todo.html, ./html/todo/S3.html, ./html/todo/add_--exclude_option_to_git_annex_find.html, ./html/todo/add_a_git_backend.html, ./html/todo/auto_remotes.html, ./html/todo/auto_remotes/discussion.html, ./html/todo/backendSHA1.html, ./html/todo/branching.html, ./html/todo/cache_key_info.html, ./html/todo/checkout.html, ./html/todo/done.html, ./html/todo/file_copy_progress_bar.html, ./html/todo/fsck.html, ./html/todo/git-annex-shell.html, ./html/todo/git-annex_unused_eats_memory.html, ./html/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.html, ./html/todo/gitrm.html, ./html/todo/hidden_files.html, ./html/todo/immutable_annexed_files.html, ./html/todo/network_remotes.html, ./html/todo/object_dir_reorg_v2.html, ./html/todo/parallel_possibilities.html, ./html/todo/pushpull.html, ./html/todo/rsync.html, ./html/todo/smudge.html, ./html/todo/speed_up_fsck.html, ./html/todo/support-non-utf8-locales.html, ./html/todo/support_S3_multipart_uploads.html, ./html/todo/symlink_farming_commit_hook.html, ./html/todo/tahoe_lfs_for_reals.html, ./html/todo/union_mounting.html, ./html/todo/use_cp_reflink.html, ./html/todo/using_url_backend.html, ./html/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.html, ./html/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.html, ./html/todo/wishlist:___34__git_annex_add__34___multiple_processes.html, ./html/todo/wishlist:_support_for_more_ssh_urls_.html, ./html/todo/wishlist:_swift_backend.html, ./html/transferring_data.html, ./html/trust.html, ./html/upgrades.html, ./html/upgrades/SHA_size.html, ./html/use_case/Alice.html, ./html/use_case/Bob.html, ./html/users.html, ./html/users/chrysn.html, ./html/users/fmarier.html, ./html/users/joey.html, ./html/walkthrough.html, ./html/walkthrough/Internet_Archive_via_S3.html, ./html/walkthrough/adding_a_remote.html, ./html/walkthrough/adding_files.html, ./html/walkthrough/backups.html, ./html/walkthrough/creating_a_repository.html, ./html/walkthrough/fsck:_verifying_your_data.html, ./html/walkthrough/fsck:_when_things_go_wrong.html, ./html/walkthrough/getting_file_content.html, ./html/walkthrough/migrating_data_to_a_new_backend.html, ./html/walkthrough/modifying_annexed_files.html, ./html/walkthrough/more.html, ./html/walkthrough/moving_file_content_between_repositories.html, ./html/walkthrough/recover_data_from_lost+found.html, ./html/walkthrough/removing_files.html, ./html/walkthrough/removing_files:_When_things_go_wrong.html, ./html/walkthrough/renaming_files.html, ./html/walkthrough/transferring_files:_When_things_go_wrong.html, ./html/walkthrough/untrusted_repositories.html, ./html/walkthrough/unused_data.html, ./html/walkthrough/using_Amazon_S3.html, ./html/walkthrough/using_bup.html, ./html/walkthrough/using_ssh_remotes.html, ./html/walkthrough/using_the_SHA1_backend.html, ./html/walkthrough/using_the_URL_backend.html, ./html/walkthrough/what_to_do_when_you_lose_a_repository.html, ./mdwn2man, ./test.hs, ./testdata/unicode-test-ö
 
 Executable git-annex
   Main-Is: git-annex.hs

From 6ba866ca738d6a63858916f84979cfd346bcb403 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 15:39:30 -0400
Subject: [PATCH 1976/8313] updates for web remote and removing URL backend

---
 debian/control                              |  1 +
 doc/backends.mdwn                           |  4 ----
 doc/cheatsheet.mdwn                         |  1 -
 doc/git-annex.mdwn                          | 14 ++++--------
 doc/install.mdwn                            |  1 +
 doc/walkthrough/using_the_SHA1_backend.mdwn |  2 +-
 doc/walkthrough/using_the_URL_backend.mdwn  | 24 ---------------------
 git-annex.cabal                             |  4 ++--
 8 files changed, 9 insertions(+), 42 deletions(-)
 delete mode 100644 doc/walkthrough/using_the_URL_backend.mdwn

diff --git a/debian/control b/debian/control
index 8c533af114..07a9e4bbb2 100644
--- a/debian/control
+++ b/debian/control
@@ -10,6 +10,7 @@ Build-Depends:
 	libghc-sha-dev,
 	libghc-dataenc-dev,
 	libghc-utf8-string-dev,
+	libghc-curl-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	ikiwiki,
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 4290da33ba..03502eaa18 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -27,10 +27,6 @@ these backends.
   part of the key. Useful for archival tasks where the filename extension
   contains metadata that should be preserved.
 
-These backends store file contents in other key/value stores.
-
-* `URL` -- This backend downloads the file's content from an external URL.
-
 The `annex.backends` git-config setting can be used to list the backends
 git-annex should use. The first one listed will be used by default when
 new files are added.
diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 3f5960972d..e88efc0ab0 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -5,7 +5,6 @@ A suppliment to the [[walkthrough]].
 [[!inline feeds=no show=0 template=walkthrough pagenames="""
 	walkthrough/using_Amazon_S3
 	walkthrough/using_bup
-	walkthrough/using_the_URL_backend
 	walkthrough/using_the_SHA1_backend
 	walkthrough/migrating_data_to_a_new_backend
 	walkthrough/untrusted_repositories
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a5ddf3172a..92fd824543 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -277,13 +277,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * fromkey file
 
-  This can be used to manually set up a file to link to a specified key
-  in the key-value backend. How you determine an existing key in the backend
-  varies. For the URL backend, the key is based on an URL to the content.
-
-  Example:
-
-	git annex fromkey --key=URL--http://www.archive.org/somefile somefile
+  This plumbing-level command can be used to manually set up a file
+  to link to a specified key in the key-value backend.
 
 * dropkey [key ...]
 
@@ -299,8 +294,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * setkey file
 
-  This plumbing-level command sets the annexed data for a key to the content of
-  the specified file, and then removes the file.
+  This plumbing-level command sets the annexed data for a key to the
+  content of the specified file, and then removes the file.
 
   Example:
 
@@ -402,7 +397,6 @@ Here are all the supported configuration settings.
 
   Space-separated list of names of the key-value backends to use. 
   The first listed is used to store new files by default. 
-  (default: "WORM SHA1 URL")
 
 * `remote..annex-cost`
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 1a83c2fd63..8a5e8d5ecd 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -19,6 +19,7 @@ To build and use git-annex, you will need:
   * [dataenc](http://hackage.haskell.org/package/dataenc)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
+  * [curl](http://hackage.haskell.org/package/curl)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
 * Shell commands
   * [git](http://git-scm.com/)
diff --git a/doc/walkthrough/using_the_SHA1_backend.mdwn b/doc/walkthrough/using_the_SHA1_backend.mdwn
index c04729e2cd..70dc2ef759 100644
--- a/doc/walkthrough/using_the_SHA1_backend.mdwn
+++ b/doc/walkthrough/using_the_SHA1_backend.mdwn
@@ -1,4 +1,4 @@
-Another handy alternative to the default [[backend|backends]] is the
+A handy alternative to the default [[backend|backends]] is the
 SHA1 backend. This backend provides more git-style assurance that your data
 has not been damaged. And the checksum means that when you add the same
 content to the annex twice, only one copy need be stored in the backend.
diff --git a/doc/walkthrough/using_the_URL_backend.mdwn b/doc/walkthrough/using_the_URL_backend.mdwn
deleted file mode 100644
index 50da4dad89..0000000000
--- a/doc/walkthrough/using_the_URL_backend.mdwn
+++ /dev/null
@@ -1,24 +0,0 @@
-git-annex has multiple key-value [[backends]]. So far you have been using
-the default, WORM (Write Once, Read Many) backend.
-
-Another handy backend is the URL backend, which can fetch file's content
-from remote URLs. Here's how to set up some files in your repository
-that use this backend:
-
-	# git annex fromkey --key=URL--http://www.archive.org/somefile somefile
-	fromkey somefile ok
-	# git commit -m "added a file from the Internet Archive"
-
-Now you if you ask git-annex to get that file, it will download it, 
-and cache it locally.
-
-	# git annex get somefile
-	get somefile (downloading)
-	#########################################################################100.0%
-	ok
-
-You can always drop files downloaded by the URL backend. It is assumed
-that the URL is stable; no local backup is kept.
-
-	# git annex drop somefile
-	drop somefile (ok)
diff --git a/git-annex.cabal b/git-annex.cabal
index a4b8aef993..fab2c71b96 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -29,14 +29,14 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
    base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
    base < 5
 
 Executable git-union-merge

From 2cdacfbae6519eceed2d5dcbea052de244a0b8ec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 16:00:44 -0400
Subject: [PATCH 1977/8313] remove URL backend

---
 Backend/URL.hs | 61 --------------------------------------------------
 BackendList.hs |  2 --
 debian/NEWS    |  6 +++++
 3 files changed, 6 insertions(+), 63 deletions(-)
 delete mode 100644 Backend/URL.hs

diff --git a/Backend/URL.hs b/Backend/URL.hs
deleted file mode 100644
index e41004dd46..0000000000
--- a/Backend/URL.hs
+++ /dev/null
@@ -1,61 +0,0 @@
-{- git-annex "URL" backend
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.URL (backends) where
-
-import Control.Monad.State (liftIO)
-
-import Types
-import Types.Backend
-import Utility
-import Messages
-import Types.Key
-
-backends :: [Backend Annex]
-backends = [backend]
-
-backend :: Backend Annex
-backend = Backend {
-	name = "URL",
-	getKey = keyValue,
-	storeFileKey = dummyStore,
-	retrieveKeyFile = downloadUrl,
-	-- allow keys to be removed; presumably they can always be
-	-- downloaded again
-	removeKey = dummyRemove,
-	-- similarly, keys are always assumed to be out there on the web
-	hasKey = dummyOk,
-	-- and nothing needed to fsck
-	fsckKey = dummyFsck,
-	-- and key upgrade not needed
-	upgradableKey = \_ -> return False
-}
-
--- cannot generate url from filename
-keyValue :: FilePath -> Annex (Maybe Key)
-keyValue _ = return Nothing
-
--- cannot change url contents
-dummyStore :: FilePath -> Key -> Annex Bool
-dummyStore _ _ = return False
-
-dummyRemove :: Key -> Maybe a -> Annex Bool
-dummyRemove _ _ = return True
-
-dummyFsck :: Key -> Maybe FilePath -> Maybe a -> Annex Bool
-dummyFsck _ _ _ = return True
-
-dummyOk :: Key -> Annex Bool
-dummyOk _ = return True
-
-downloadUrl :: Key -> FilePath -> Annex Bool
-downloadUrl key file = do
-	showNote $ "downloading"
-	showProgress -- make way for curl progress bar
-	liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
-	where
-		url = keyName key
diff --git a/BackendList.hs b/BackendList.hs
index bc3fd83142..e4e1d76fe2 100644
--- a/BackendList.hs
+++ b/BackendList.hs
@@ -10,12 +10,10 @@ module BackendList (allBackends) where
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
 import qualified Backend.SHA
-import qualified Backend.URL
 import Types
 
 allBackends :: [Backend Annex]
 allBackends = concat 
 	[ Backend.WORM.backends
 	, Backend.SHA.backends
-	, Backend.URL.backends
 	]
diff --git a/debian/NEWS b/debian/NEWS
index 22835ace84..ad4e946e6c 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,3 +1,9 @@
+git-annex (3.20110702) unstable; urgency=low
+
+  The URL backend has been removed. Instead the new web remote can be used.
+
+ -- Joey Hess   Fri, 01 Jul 2011 15:40:51 -0400
+
 git-annex (3.20110624) exerimental; urgency=low
 
   There has been another change to the git-annex data store.

From a140f7148f3ea0bef2d8c060c7847b3d1be4d25e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 16:01:26 -0400
Subject: [PATCH 1978/8313] documentation for using the web

---
 debian/changelog                   |  3 ++-
 doc/cheatsheet.mdwn                |  1 +
 doc/git-annex.mdwn                 |  4 ++++
 doc/special_remotes.mdwn           |  1 +
 doc/special_remotes/web.mdwn       |  7 +++++++
 doc/walkthrough/using_the_web.mdwn | 32 ++++++++++++++++++++++++++++++
 6 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 doc/special_remotes/web.mdwn
 create mode 100644 doc/walkthrough/using_the_web.mdwn

diff --git a/debian/changelog b/debian/changelog
index 53fc99e8c3..b4a77de7dc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,8 @@
-git-annex (3.20110625) UNRELEASED; urgency=low
+git-annex (3.20110702) UNRELEASED; urgency=low
 
   * Now the web can be used as a special remote. This feature
     replaces the old URL backend.
+  * addurl: New command to download an url and store it in the annex.
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Such
     commands are now faster than they were before introduction of the
diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index e88efc0ab0..9f7c146c82 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -5,6 +5,7 @@ A suppliment to the [[walkthrough]].
 [[!inline feeds=no show=0 template=walkthrough pagenames="""
 	walkthrough/using_Amazon_S3
 	walkthrough/using_bup
+	walkthrough/using_the_web
 	walkthrough/using_the_SHA1_backend
 	walkthrough/migrating_data_to_a_new_backend
 	walkthrough/untrusted_repositories
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 92fd824543..c450887488 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -275,6 +275,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Returns a repository to the default semi trusted state.
 
+* addurl [url ...]
+
+  Downloads each url to a file, which is added to the annex.
+
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 12e0aedb1b..afc6a2cf23 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -10,6 +10,7 @@ They cannot be used by other git commands though.
 * [[bup]]
 * [[directory]]
 * [[rsync]]
+* [[web]]
 * [[hook]]
 * [[tahoe-lafs|forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] - limited testing
 
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
new file mode 100644
index 0000000000..68df31ef4d
--- /dev/null
+++ b/doc/special_remotes/web.mdwn
@@ -0,0 +1,7 @@
+git-annex can use the WWW as a special remote, downloading urls to files.
+See [[walkthrough/using_web_web]] for usage examples.
+
+## notes
+
+Currently git-annex only supports downloading content from the web; 
+it cannot upload to it or remove content.
diff --git a/doc/walkthrough/using_the_web.mdwn b/doc/walkthrough/using_the_web.mdwn
new file mode 100644
index 0000000000..9d5525758a
--- /dev/null
+++ b/doc/walkthrough/using_the_web.mdwn
@@ -0,0 +1,32 @@
+The web can be used as a [[special_remote|special_remotes]] too.
+
+	# git annex addurl http://example.com/video.mpeg
+	addurl video.mpeg (downloading http://example.com/video.mpeg)
+	########################################################## 100.0%
+	ok
+
+Now the file is downloaded, and has been added to the annex like any other
+file. So it can be copied to other repositories, and so on.
+
+Note that git-annex assumes that, if the web site does not 404, the file is
+still present on the web, and this counts as one [[copy|copies]] of the
+file. So it will let you remove your last copy, trusting it can be
+downloaded again:
+
+	# git annex drop video.mpeg
+	drop video.mpeg (checking http://example.com/video.mpeg) ok
+
+If you don't [[trust]] the web to this degree, just let git-annex know:
+
+	# git annex untrust web
+	untrust web ok
+
+With the result that it will hang onto files:
+
+	# git annex drop video.mpeg
+	drop video.mpeg (unsafe) 
+	  Could only verify the existence of 0 out of 1 necessary copies
+	  Also these untrusted repositories may contain the file:
+	  	00000000-0000-0000-0000-000000000001  -- web
+	  (Use --force to override this check, or adjust annex.numcopies.)
+	failed

From 6bddebdb79ca8ed168e143d533a6101c7d469628 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:15:46 -0400
Subject: [PATCH 1979/8313] add the addurl command

---
 Command/AddUrl.hs | 74 +++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs       |  2 ++
 LocationLog.hs    |  5 +---
 PresenceLog.hs    |  6 ++++
 Remote/Web.hs     | 69 ++++++++++++++++++++++++++-----------------
 5 files changed, 125 insertions(+), 31 deletions(-)
 create mode 100644 Command/AddUrl.hs

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
new file mode 100644
index 0000000000..713a486a5c
--- /dev/null
+++ b/Command/AddUrl.hs
@@ -0,0 +1,74 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.AddUrl where
+
+import Control.Monad.State (liftIO, when)
+import Network.URI
+import Data.String.Utils
+import System.Directory
+
+import Command
+import qualified Backend
+import qualified Remote.Web
+import qualified Command.Add
+import Messages
+import Content
+import PresenceLog
+
+command :: [Command]
+command = [repoCommand "addurl" paramPath seek "add urls to annex"]
+
+seek :: [CommandSeek]
+seek = [withStrings start]
+
+start :: CommandStartString
+start s = do
+	let u = parseURI s
+	case u of
+		Nothing -> error $ "bad url " ++ s
+		Just url -> do
+			file <- liftIO $ url2file url
+			showStart "addurl" file
+			next $ perform s file
+			
+perform :: String -> FilePath -> CommandPerform
+perform url file = do
+	[(_, backend)] <- Backend.chooseBackends [file]
+	showNote $ "downloading " ++ url
+	ok <- Remote.Web.download file [url]
+	if ok
+		then do
+			stored <- Backend.storeFileKey file backend
+			case stored of
+				Nothing -> stop
+				Just (key, _) -> do
+					moveAnnex key file
+					Remote.Web.setUrl key url InfoPresent
+					next $ Command.Add.cleanup file key
+		else stop
+
+url2file :: URI -> IO FilePath
+url2file url = do
+	let parts = filter safe $ split "/" $ uriPath url
+	if null parts
+		then fallback
+		else do
+			let file = last parts
+			e <- doesFileExist file
+			if e then fallback else return file
+	where
+		fallback = do
+			let file = replace "/" "_" $ show url
+			e <- doesFileExist file
+			when e $ error "already have this url"
+			return file
+		safe s
+			| null s = False
+			| s == "." = False
+			| s == ".." = False
+			| otherwise = True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 58b512f718..85eb2bf26e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -46,6 +46,7 @@ import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
+import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
 import qualified Command.Version
@@ -68,6 +69,7 @@ cmds = concat
 	, Command.Trust.command
 	, Command.Untrust.command
 	, Command.Semitrust.command
+	, Command.AddUrl.command
 	, Command.FromKey.command
 	, Command.DropKey.command
 	, Command.SetKey.command
diff --git a/LocationLog.hs b/LocationLog.hs
index a5db7d1218..19a8eb83a1 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -37,10 +37,7 @@ logChange repo key u s = do
 	when (null u) $
 		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
 			" (have you run git annex init there?)"
-	line <- logNow s u
-	let f = logFile key
-	ls <- readLog f
-	writeLog f (compactLog $ line:ls)
+	addLog (logFile key) =<< logNow s u
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 71d78f1eda..0777db209b 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -13,6 +13,7 @@
 
 module PresenceLog (
 	LogStatus(..),
+	addLog,
 	readLog,
 	writeLog,
 	logNow,
@@ -70,6 +71,11 @@ instance Read LogLine where
 			bad = ret $ LogLine 0 Undefined ""
 			ret v = [(v, "")]
 
+addLog :: FilePath -> LogLine -> Annex ()
+addLog file line = do
+	ls <- readLog file
+	writeLog file (compactLog $ line:ls)
+
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 201f923cf7..342acef918 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -6,7 +6,9 @@
  -}
 
 module Remote.Web (
-	remote
+	remote,
+	setUrl,
+	download
 ) where
 
 import Control.Monad.State (liftIO)
@@ -20,11 +22,13 @@ import Network.Curl.Code
 import Types
 import Types.Remote
 import qualified Git
+import qualified Annex
 import Messages
 import Utility
 import UUID
 import Config
 import PresenceLog
+import LocationLog
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -50,10 +54,10 @@ gen r _ _ =
 		uuid = webUUID,
 		cost = expensiveRemoteCost,
 		name = Git.repoDescribe r,
-		storeKey = upload,
-		retrieveKeyFile = download,
-		removeKey = remove,
-		hasKey = check,
+		storeKey = uploadKey,
+		retrieveKeyFile = downloadKey,
+		removeKey = dropKey,
+		hasKey = checkKey,
 		hasKeyCheap = False,
 		config = Nothing
 	}
@@ -62,40 +66,44 @@ gen r _ _ =
 urlLog :: Key -> FilePath
 urlLog key = "remote/web"  show key ++ ".log"
 
-urls :: Key -> Annex [URLString]
-urls key = currentLog (urlLog key)
+getUrls :: Key -> Annex [URLString]
+getUrls key = currentLog (urlLog key)
 
-download :: Key -> FilePath -> Annex Bool
-download key file = download' file =<< urls key
-download' :: FilePath -> [URLString] -> Annex Bool
-download' _ [] = return False
-download' file (url:us) = do
-	showProgress -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
-	if ok then return ok else download' file us
+{- Records a change in an url for a key. -}
+setUrl :: Key -> URLString -> LogStatus -> Annex ()
+setUrl key url status = do
+	g <- Annex.gitRepo
+	addLog (urlLog key) =<< logNow status url
 
-upload :: Key -> Annex Bool
-upload _ = do
+	-- update location log to indicate that the web has the key, or not
+	us <- getUrls key
+	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
+
+downloadKey :: Key -> FilePath -> Annex Bool
+downloadKey key file = download file =<< getUrls key
+
+uploadKey :: Key -> Annex Bool
+uploadKey _ = do
 	warning "upload to web not supported"
 	return False
 
-remove :: Key -> Annex Bool
-remove _ = do
+dropKey :: Key -> Annex Bool
+dropKey _ = do
 	warning "removal from web not supported"
 	return False
 
-check :: Key -> Annex (Either IOException Bool)
-check key = do
-	us <- urls key
+checkKey :: Key -> Annex (Either IOException Bool)
+checkKey key = do
+	us <- getUrls key
 	if null us
 		then return $ Right False
-		else return . Right =<< check' us
-check' :: [URLString] -> Annex Bool
-check' [] = return False
-check' (u:us) = do
+		else return . Right =<< checkKey' us
+checkKey' :: [URLString] -> Annex Bool
+checkKey' [] = return False
+checkKey' (u:us) = do
 	showNote ("checking " ++ u)
 	e <- liftIO $ urlexists u
-	if e then return e else check' us
+	if e then return e else checkKey' us
 
 urlexists :: URLString -> IO Bool
 urlexists url = do
@@ -105,3 +113,10 @@ urlexists url = do
 	_ <- setopt curl (CurlFailOnError True)
 	res <- perform curl
 	return $ res == CurlOK
+
+download :: FilePath -> [URLString] -> Annex Bool
+download _ [] = return False
+download file (url:us) = do
+	showProgress -- make way for curl progress bar
+	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
+	if ok then return ok else download file us

From fb58d1a560f7c4c94826ec63de16e0276d1f17f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:17:02 -0400
Subject: [PATCH 1980/8313] wording

---
 Backend/SHA.hs    | 2 +-
 doc/backends.mdwn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 8ed00b7073..8930e4b938 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -1,4 +1,4 @@
-{- git-annex SHA abstract backend
+{- git-annex SHA backend
  -
  - Copyright 2011 Joey Hess 
  -
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 03502eaa18..9e698032d8 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -9,7 +9,7 @@ to retrieve the file's content (its value).
 Multiple pluggable backends are supported, and a single repository
 can use different backends for different files.
 
-These backends can transfer file contents in configured git remotes.
+These backends can transfer file contents between configured git remotes.
 It's also possible to use [[special_remotes]], such as Amazon S3 with
 these backends.
 

From 79016c197ca87182dfae9f6dfb994ff5079fc952 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:23:01 -0400
Subject: [PATCH 1981/8313] add hashing to web log files

---
 LocationLog.hs | 17 ++++++++++++++++-
 Locations.hs   | 14 --------------
 Remote/Web.hs  |  6 ++++--
 Upgrade/V2.hs  |  2 +-
 4 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index 19a8eb83a1..eb48b7916a 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -17,7 +17,9 @@ module LocationLog (
 	readLog,
 	writeLog,
 	keyLocations,
-	loggedKeys
+	loggedKeys,
+	logFile,
+	logFileKey	
 ) where
 
 import System.FilePath
@@ -28,6 +30,7 @@ import qualified Git
 import qualified Branch
 import UUID
 import Types
+import Types.Key
 import Locations
 import PresenceLog
 
@@ -49,3 +52,15 @@ keyLocations key = currentLog $ logFile key
 loggedKeys :: Annex [Key]
 loggedKeys =
 	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files
+
+{- The filename of the log file for a given key. -}
+logFile :: Key -> String
+logFile key = hashDirLower key ++ keyFile key ++ ".log"
+
+{- Converts a log filename into a key. -}
+logFileKey :: FilePath -> Maybe Key
+logFileKey file
+	| end == ".log" = readKey beginning
+	| otherwise = Nothing
+	where
+		(beginning, end) = splitAt (length file - 4) file
diff --git a/Locations.hs b/Locations.hs
index 6142b2393a..347b08ce14 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -19,8 +19,6 @@ module Locations (
 	gitAnnexUnusedLog,
 	gitAnnexJournalDir,
 	isLinkToAnnex,
-	logFile,
-	logFileKey,
 	hashDirMixed,
 	hashDirLower,
 
@@ -115,18 +113,6 @@ gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
-{- The filename of the log file for a given key. -}
-logFile :: Key -> String
-logFile key = hashDirLower key ++ keyFile key ++ ".log"
-
-{- Converts a log filename into a key. -}
-logFileKey :: FilePath -> Maybe Key
-logFileKey file
-	| end == ".log" = readKey beginning
-	| otherwise = Nothing
-	where
-		(beginning, end) = splitAt (length file - 4) file
-
 {- Converts a key into a filename fragment.
  -
  - Escape "/" in the key name, to keep a flat tree of files and avoid
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 342acef918..7425dec87e 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -29,6 +29,7 @@ import UUID
 import Config
 import PresenceLog
 import LocationLog
+import Locations
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -62,9 +63,10 @@ gen r _ _ =
 		config = Nothing
 	}
 
-{- The urls for a key are stored in remote/web/key.log in the git-annex branch. -}
+{- The urls for a key are stored in remote/web/hash/key.log 
+ - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web"  show key ++ ".log"
+urlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
 getUrls :: Key -> Annex [URLString]
 getUrls key = currentLog (urlLog key)
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ce0424b302..14e328edb4 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -20,7 +20,7 @@ import qualified Git
 import qualified Branch
 import Messages
 import Utility
-import Locations
+import LocationLog
 import Content
 
 olddir :: Git.Repo -> FilePath

From 457d28c676f48d61c523df74d665040e24132ed7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:24:11 -0400
Subject: [PATCH 1982/8313] wording

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index b4a77de7dc..4be7e674a6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,7 @@
 git-annex (3.20110702) UNRELEASED; urgency=low
 
-  * Now the web can be used as a special remote. This feature
-    replaces the old URL backend.
+  * Now the web can be used as a special remote. 
+    This feature replaces the old URL backend.
   * addurl: New command to download an url and store it in the annex.
   * Sped back up fsck, copy --from, and other commands that often
     have to read a lot of information from the git-annex branch. Such

From 5d154b84365770ff27c09137ad039f930505ebfd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 17:28:31 -0400
Subject: [PATCH 1983/8313] document web special remote log files

---
 doc/internals.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 1096845e5e..a4ec5c417b 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -72,7 +72,7 @@ Example:
 
 ## `aaa/bbb/*.log`
 
-The remainder of the log files record [[location_tracking]] information
+These log files record [[location_tracking]] information
 for file contents. Again these are placed in two levels of subdirectories
 for hashing. The name of the key is the filename, and the content
 consists of a timestamp, either 1 (present) or 0 (not present), and
@@ -85,3 +85,9 @@ Example:
 
 These files are designed to be auto-merged using git's union merge driver.
 The timestamps allow the most recent information to be identified.
+
+## `remote/web/aaa/bbb/*.log`
+
+These log files record urls used by the
+[[web_special_remote|special_remotes/web]]. Their format is similar
+to the location tracking files, but with urls rather than UUIDs.

From ace9de37e8db9c99db4b121392ca63091e48bfac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 18:46:07 -0400
Subject: [PATCH 1984/8313] download urls via tmp file, and support resuming

---
 Command/AddUrl.hs                                | 16 ++++++++++++----
 Remote/Web.hs                                    | 14 ++++++++------
 ...t_2_80b9e848edfdc7be21baab7d0cef0e3a._comment |  3 +++
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 713a486a5c..ebf0810bae 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -16,9 +16,13 @@ import Command
 import qualified Backend
 import qualified Remote.Web
 import qualified Command.Add
+import qualified Annex
 import Messages
 import Content
 import PresenceLog
+import Types.Key
+import Locations
+import Utility
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
@@ -38,16 +42,20 @@ start s = do
 			
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
-	[(_, backend)] <- Backend.chooseBackends [file]
+	g <- Annex.gitRepo
 	showNote $ "downloading " ++ url
-	ok <- Remote.Web.download file [url]
+	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
+	let tmp = gitAnnexTmpLocation g dummykey
+	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	ok <- Remote.Web.download [url] tmp
 	if ok
 		then do
-			stored <- Backend.storeFileKey file backend
+			[(_, backend)] <- Backend.chooseBackends [file]
+			stored <- Backend.storeFileKey tmp backend
 			case stored of
 				Nothing -> stop
 				Just (key, _) -> do
-					moveAnnex key file
+					moveAnnex key tmp
 					Remote.Web.setUrl key url InfoPresent
 					next $ Command.Add.cleanup file key
 		else stop
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 7425dec87e..304f191d31 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -82,7 +82,9 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = download file =<< getUrls key
+downloadKey key file = do
+	us <- getUrls key
+	download us file
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
@@ -116,9 +118,9 @@ urlexists url = do
 	res <- perform curl
 	return $ res == CurlOK
 
-download :: FilePath -> [URLString] -> Annex Bool
-download _ [] = return False
-download file (url:us) = do
+download :: [URLString] -> FilePath -> Annex Bool
+download [] _ = return False
+download (url:us) file = do
 	showProgress -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-# -o", File file, File url]
-	if ok then return ok else download file us
+	ok <- liftIO $ boolSystem "curl" [Params "-C - -# -o", File file, File url]
+	if ok then return ok else download us file
diff --git a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
index a32461615f..6dba86c47c 100644
--- a/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
+++ b/doc/todo/tahoe_lfs_for_reals/comment_2_80b9e848edfdc7be21baab7d0cef0e3a._comment
@@ -7,4 +7,7 @@
 Whoops! You'd only told me O(N) twice before..
 
 So this is not too high priority. I think I would like to get the per-remote storage sorted out anyway, since probably it will be the thing needed to convert the URL backend into a special remote, which would then allow ripping out the otherwise unused pluggable backend infrastructure.
+
+Update: Per-remote storage is now sorted out, so this could be implemented
+if it actually made sense to do so.
 """]]

From e6b9539a655f15c402aad3f5fa3690c8e4fb06aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 1 Jul 2011 21:52:27 -0400
Subject: [PATCH 1985/8313] make curl follow redirs

---
 Remote/Web.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 304f191d31..71591b7aa0 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -115,6 +115,7 @@ urlexists url = do
 	_ <- setopt curl (CurlURL url)
 	_ <- setopt curl (CurlNoBody True)
 	_ <- setopt curl (CurlFailOnError True)
+	_ <- setopt curl (CurlFollowLocation True)
 	res <- perform curl
 	return $ res == CurlOK
 
@@ -122,5 +123,5 @@ download :: [URLString] -> FilePath -> Annex Bool
 download [] _ = return False
 download (url:us) file = do
 	showProgress -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-C - -# -o", File file, File url]
+	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 	if ok then return ok else download us file

From 80459918307730f9abc9c6b681d4888758b6d522 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 14:40:57 -0400
Subject: [PATCH 1986/8313] better cabal command

(which will work once I upload to hackage)
---
 doc/install.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 8a5e8d5ecd..25eb8bedfb 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -39,5 +39,4 @@ Then just [[download]] git-annex and run: `make; make install`
 
 As a haskell package, git-annex can be built using cabal. For example:
 
-	cabal configure
-	cabal install --bindir=$HOME/bin
+	cabal install git-annex --bindir=$HOME/bin

From 648827861024783ef6fa9c03e8ca5210c68fc7cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 14:53:55 -0400
Subject: [PATCH 1987/8313] cabal sdist: plan C

---
 Makefile                |  8 ++++++++
 configure.hs            | 14 --------------
 git-annex.cabal         |  1 +
 testdata/unicode-test-ö |  1 -
 4 files changed, 9 insertions(+), 15 deletions(-)
 delete mode 100644 testdata/unicode-test-ö

diff --git a/Makefile b/Makefile
index 0bc02dc352..afd2de99e0 100644
--- a/Makefile
+++ b/Makefile
@@ -87,4 +87,12 @@ clean:
 	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
+# Workaround for cabal sdist not running Setup hooks, so I cannot
+# generate a file list there.
+sdist: clean
+	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -type f -print)!i" < git-annex.cabal.orig > git-annex.cabal
+	@cabal sdist
+	@mv git-annex.cabal.orig git-annex.cabal
+
 .PHONY: $(bins) test install
diff --git a/configure.hs b/configure.hs
index d47dddf1df..8639af44b1 100644
--- a/configure.hs
+++ b/configure.hs
@@ -17,7 +17,6 @@ tests =
 	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
-	, TestCase "unicode FilePath support" $ unicodeFilePath
 	] ++ shaTestCases [1, 256, 512, 224, 384]
 
 shaTestCases :: [Int] -> [TestCase]
@@ -40,19 +39,6 @@ testCp k option = TestCase cmd $ testCmd k run
 		cmd = "cp " ++ option
 		run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new"
 
-{- Checks if FilePaths contain decoded unicode, or not. The testdata
- - directory contains a "unicode-test-ü" file; try to find the file,
- - and see if the "ü" is encoded correctly.
- -
- - Note that the file is shipped with git-annex, rather than created,
- - to avoid other potential unicode issues.
- -}
-unicodeFilePath :: Test
-unicodeFilePath = do
-	fs <- getDirectoryContents "testdata"
-	let file = head $ filter (isInfixOf "unicode-test") fs
-	return $ Config "unicodefilepath" (BoolConfig $ isInfixOf "ü" file)
-
 {- Pulls package version out of the changelog. -}
 getVersion :: Test
 getVersion = do
diff --git a/git-annex.cabal b/git-annex.cabal
index fab2c71b96..59220f47b3 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -7,6 +7,7 @@ Author: Joey Hess
 Stability: Stable
 Copyright: 2010-2011 Joey Hess
 License-File: GPL
+Extra-Source-Files: use-make-sdist-instead
 Homepage: http://git-annex.branchable.com/
 Build-type: Custom
 Category: Utility
diff --git a/testdata/unicode-test-ö b/testdata/unicode-test-ö
deleted file mode 100644
index 45b983be36..0000000000
--- a/testdata/unicode-test-ö
+++ /dev/null
@@ -1 +0,0 @@
-hi

From 48db40857cfc84454ae0121b8d014e3c5b17e753 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 15:08:05 -0400
Subject: [PATCH 1988/8313] releasing version 3.20110702

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4be7e674a6..266a747ee0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110702) UNRELEASED; urgency=low
+git-annex (3.20110702) unstable; urgency=low
 
   * Now the web can be used as a special remote. 
     This feature replaces the old URL backend.
@@ -17,7 +17,7 @@ git-annex (3.20110702) UNRELEASED; urgency=low
     does not run the test suite, and is not particularly recommended,
     but could be useful to some.
 
- -- Joey Hess   Sun, 26 Jun 2011 21:01:06 -0400
+ -- Joey Hess   Sat, 02 Jul 2011 15:00:18 -0400
 
 git-annex (3.20110624) experimental; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 59220f47b3..54a526160e 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110625
+Version: 3.20110702
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 791dfaac65434cf6b6c9423e4ec3ff3252a71b13 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 15:08:49 -0400
Subject: [PATCH 1989/8313] add news item for git-annex 3.20110702

---
 doc/news/version_0.20110521.mdwn |  5 -----
 doc/news/version_3.20110702.mdwn | 22 ++++++++++++++++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_0.20110521.mdwn
 create mode 100644 doc/news/version_3.20110702.mdwn

diff --git a/doc/news/version_0.20110521.mdwn b/doc/news/version_0.20110521.mdwn
deleted file mode 100644
index f64392a760..0000000000
--- a/doc/news/version_0.20110521.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 0.20110521 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * status: New subcommand to show info about an annex, including its size.
-   * --backend now overrides any backend configured in .gitattributes files.
-   * Add --debug option. Closes: #[627499](http://bugs.debian.org/627499)"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
new file mode 100644
index 0000000000..0bb3adf2d8
--- /dev/null
+++ b/doc/news/version_3.20110702.mdwn
@@ -0,0 +1,22 @@
+News for git-annex 3.20110702:
+
+   The URL backend has been removed. Instead the new web remote can be used.
+
+git-annex 3.20110702 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Now the web can be used as a special remote.
+     This feature replaces the old URL backend.
+   * addurl: New command to download an url and store it in the annex.
+   * Sped back up fsck, copy --from, and other commands that often
+     have to read a lot of information from the git-annex branch. Such
+     commands are now faster than they were before introduction of the
+     git-annex branch.
+   * Always ensure git-annex branch exists.
+   * Modify location log parser to allow future expansion.
+   * --force will cause add, etc, to operate on ignored files.
+   * Avoid mangling encoding when storing the description of repository
+     and other content.
+   * cabal can now be used to build git-annex. This is substantially
+     slower than using make, does not build or install documentation,
+     does not run the test suite, and is not particularly recommended,
+     but could be useful to some."""]]
\ No newline at end of file

From 17a2b13e648c70db768a778d3b29ab933e8a39c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 15:11:20 -0400
Subject: [PATCH 1990/8313] formatting

---
 doc/news/version_3.20110624.mdwn | 23 ++++++++++++-----------
 doc/news/version_3.20110702.mdwn |  4 ++--
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
index 5a91fa4295..6204673bd2 100644
--- a/doc/news/version_3.20110624.mdwn
+++ b/doc/news/version_3.20110624.mdwn
@@ -1,15 +1,16 @@
 News for git-annex 3.20110624:
 
-   There has been another change to the git-annex data store.
-   Use `git annex upgrade` to migrate your repositories to the new
-   layout. See <http://git-annex.branchable.com/upgrades/> or
-   /usr/share/doc/git-annex/html/upgrades.html
-   The significant change this time is that the .git-annex/ directory
-   is gone; instead there is a git-annex branch that is automatically
-   maintained by git-annex, and encapsulates all its state nicely out
-   of your way.
-   You should make sure you include the git-annex branch when
-   git pushing and pulling.
+There has been another change to the git-annex data store.
+Use `git annex upgrade` to migrate your repositories to the new
+layout. See [[upgrades]].
+
+The significant change this time is that the .git-annex/ directory
+is gone; instead there is a git-annex branch that is automatically
+maintained by git-annex, and encapsulates all its state nicely out
+of your way.
+
+You should make sure you include the git-annex branch when
+git pushing and pulling.
 
 git-annex 3.20110624 released with [[!toggle text="these changes"]]
 [[!toggleable text="""
@@ -29,4 +30,4 @@ git-annex 3.20110624 released with [[!toggle text="these changes"]]
      trusting repositories that are not configured remotes.
    * unlock: Made atomic.
    * git-union-merge: New git subcommand, that does a generic union merge
-     operation, and operates efficiently without touching the working tree."""]]
\ No newline at end of file
+     operation, and operates efficiently without touching the working tree."""]]
diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
index 0bb3adf2d8..a5bb925552 100644
--- a/doc/news/version_3.20110702.mdwn
+++ b/doc/news/version_3.20110702.mdwn
@@ -1,6 +1,6 @@
 News for git-annex 3.20110702:
 
-   The URL backend has been removed. Instead the new web remote can be used.
+The URL backend has been removed. Instead the new web remote can be used.
 
 git-annex 3.20110702 released with [[!toggle text="these changes"]]
 [[!toggleable text="""
@@ -19,4 +19,4 @@ git-annex 3.20110702 released with [[!toggle text="these changes"]]
    * cabal can now be used to build git-annex. This is substantially
      slower than using make, does not build or install documentation,
      does not run the test suite, and is not particularly recommended,
-     but could be useful to some."""]]
\ No newline at end of file
+     but could be useful to some."""]]

From 686d08718b84a70d4b36039fdf595cc4c0fa176a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:10:25 -0400
Subject: [PATCH 1991/8313] link to hackage

---
 doc/download.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index c0d2d261d3..ddb2ee82a2 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -10,5 +10,8 @@ Other mirrors of the git repository:
 To download a tarball of a particular release, use an url like
 
 
+From time to time, releases of git-annex are uploaded
+[to hackage](http://hackage.haskell.org/package/git-annex).
+
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.

From 84a9fee6f2afaa260054ef42da3b08ca0dddb130 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:10:43 -0400
Subject: [PATCH 1992/8313] fix sdist to include symlinks, without which it
 doesn't build

oops.. will upload a new minor version to hackage
---
 Makefile        | 2 +-
 git-annex.cabal | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index afd2de99e0..786fd919e4 100644
--- a/Makefile
+++ b/Makefile
@@ -91,7 +91,7 @@ clean:
 # generate a file list there.
 sdist: clean
 	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
-	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -type f -print)!i" < git-annex.cabal.orig > git-annex.cabal
+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print)!i" < git-annex.cabal.orig > git-annex.cabal
 	@cabal sdist
 	@mv git-annex.cabal.orig git-annex.cabal
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 54a526160e..d5a066cdff 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110702
+Version: 3.20110702.2
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From e3ffa4330c0c5855c6e3542aac83a2c54fc8c4de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:14:27 -0400
Subject: [PATCH 1993/8313] update

---
 doc/download.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index ddb2ee82a2..e1257d2618 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -8,7 +8,7 @@ Other mirrors of the git repository:
 * [at github](https://github.com/joeyh/git-annex)
 
 To download a tarball of a particular release, use an url like
-
+
 
 From time to time, releases of git-annex are uploaded
 [to hackage](http://hackage.haskell.org/package/git-annex).

From bd5884d8dd873d6bc7a5e938afb2d835b7bd968f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 17:29:20 -0400
Subject: [PATCH 1994/8313] reorder

---
 doc/install.mdwn | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 25eb8bedfb..23a0da902d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -6,7 +6,13 @@
 * [[Fedora]]
 * [[FreeBSD]]
 
-## Generic instructions
+## Using cabal
+
+As a haskell package, git-annex can be built using cabal. For example:
+
+	cabal install git-annex --bindir=$HOME/bin
+
+## Installation by hand
 
 To build and use git-annex, you will need:
 
@@ -34,9 +40,3 @@ To build and use git-annex, you will need:
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`
-
-## Using cabal
-
-As a haskell package, git-annex can be built using cabal. For example:
-
-	cabal install git-annex --bindir=$HOME/bin

From f626512b07f3675f184101c4c945861381862339 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 19:16:28 -0400
Subject: [PATCH 1995/8313] further improved git cat-file error handling

---
 Branch.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 59f2d3134f..d9cbe61a72 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -241,8 +241,7 @@ catFile file = do
 				[Param "cat-file", Param "--batch"]
 			let gitcmd = join " " ("git" : toCommand cmd)
 			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				-- want stderr on stdin to see error messages
-				["-c", "exec " ++ gitcmd ++ " 2>&1"]
+				["-c", "exec " ++ gitcmd ++ " 2>/dev/null"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
 		ask (from, to) = liftIO $ do
@@ -255,7 +254,9 @@ catFile file = do
 					| length sha == Git.shaSize &&
 					  blob == "blob" -> handle from size
 					| otherwise -> empty
-				_ -> empty
+				_
+					| header == want ++ " missing" -> empty
+					| otherwise -> error $ "unknown response from git cat-file " ++ header
 		handle from size = case reads size of
 			[(bytes, "")] -> readcontent from bytes
 			_ -> empty

From 591e293f43f7e4450b570f0e6e95c1a33cd50f44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 2 Jul 2011 19:22:11 -0400
Subject: [PATCH 1996/8313] simplify git cat-file startup

---
 Branch.hs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index d9cbe61a72..c0c96611bf 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -237,11 +237,9 @@ catFile file = do
 	where
 		startup state = do
 			g <- Annex.gitRepo
-			let cmd = Git.gitCommandLine g
-				[Param "cat-file", Param "--batch"]
-			let gitcmd = join " " ("git" : toCommand cmd)
-			(_, from, to) <- liftIO $ hPipeBoth "sh"
-				["-c", "exec " ++ gitcmd ++ " 2>/dev/null"]
+			(_, from, to) <- liftIO $ hPipeBoth "git" $
+				toCommand $ Git.gitCommandLine g
+					[Param "cat-file", Param "--batch"]
 			setState state { catFileHandles = Just (from, to) }
 			ask (from, to)
 		ask (from, to) = liftIO $ do

From 3904d2e4b90a7a36b3e4a8ed58caebbd7fc0ae22 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 11:32:17 +0000
Subject: [PATCH 1997/8313]

---
 .../annex_unannex__47__uninit_should_handle_copies.mdwn   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
new file mode 100644
index 0000000000..6d1799cd72
--- /dev/null
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
@@ -0,0 +1,8 @@
+Just starting using v3, even more awesome, thanks!
+
+With git-annex, I take the habit to do copies of files without restriction, as they end up into (cheap) symlink copies.
+However, if 2 copies are unannexed, only one is restored, the other becomes a broken symlink, so I kind of loose some information 
+(my use case: I have a repo on which I recently started using annex, but most of the files, which i would want to be annexed, are only in git,
+so my plan is to unninit this repo, delete the .git dir, and then annex everything, as I don't mind the history).
+
+Rafaël

From d97cc2fd52fab7a18d5c06a008c4d532615babef Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 11:56:45 +0000
Subject: [PATCH 1998/8313] Added a comment

---
 .../comment_1_7683bf02cf9e97830fb4690314501568._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment

diff --git a/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment
new file mode 100644
index 0000000000..c556fbd771
--- /dev/null
+++ b/doc/bugs/git_annex_should_use___39__git_add_-f__39___internally/comment_1_7683bf02cf9e97830fb4690314501568._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 1"
+ date="2011-07-03T11:56:45Z"
+ content="""
+And what about emitting a warning, as git does, that some files were not annex-added (when not using --force)?
+"""]]

From e6ca68250e1be5f092de1bc5f2c40f5838785aea Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 14:39:42 +0000
Subject: [PATCH 1999/8313] Added a comment: git annex fetch

---
 .../comment_6_31ea08c008500560c0b96c6601bc6362._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment

diff --git a/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment b/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment
new file mode 100644
index 0000000000..868e9677c8
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_6_31ea08c008500560c0b96c6601bc6362._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex fetch"
+ date="2011-07-03T14:39:41Z"
+ content="""
+I'm not sure it is worth adding a command for such a small feature, but I would certainly use it: having something like \"git annex fetch remote\" do \"git fetch remote && git annex copy --from=remote\", and \"git annex push remote\" do \"git push remote && git annex copy --to=remote\". And maybe the same for a pull operation?
+"""]]

From de408626b72b0bdb98cf1534d7406910318516c2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 3 Jul 2011 17:57:00 +0000
Subject: [PATCH 2000/8313] Added a comment: git annex fetch

---
 ...comment_7_94045b9078b1fff877933b012d1b49e2._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment

diff --git a/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment b/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment
new file mode 100644
index 0000000000..e39e162322
--- /dev/null
+++ b/doc/forum/new_microfeatures/comment_7_94045b9078b1fff877933b012d1b49e2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="git annex fetch"
+ date="2011-07-03T17:57:00Z"
+ content="""
+My last comment is a bit confused. The \"git fetch\" command allows to get all the information from a remote, and it is then possible to merge while being offline (without access to the remote). I would like a \"git annex fetch remote\" command to be able to get all annexed files from remote, so that if I later merge with remote, all annexed files are already here. And \"git annex fetch\" could (optionally) call \"git fetch\" before getting the files.
+
+It seems also that in my last post, I should have written \"git annex get --from=remote\" instead of \"git annex copy --from=remote\", because \"annex copy --from\" copies all files, even if the local repo already have them (is this the case? if yes, when is it useful?)
+"""]]

From bd54dadb0b92945db9fc004d03d1fb32a453225c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 12:27:47 -0400
Subject: [PATCH 2001/8313] response

---
 ...annex_unannex__47__uninit_should_handle_copies.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
index 6d1799cd72..751e1afa9b 100644
--- a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
@@ -6,3 +6,13 @@ However, if 2 copies are unannexed, only one is restored, the other becomes a br
 so my plan is to unninit this repo, delete the .git dir, and then annex everything, as I don't mind the history).
 
 Rafaël
+
+> The only way for git-annex to support this in its current state would be
+> for the unannex command to copy the file content from the annex, rather
+> than moving it out. Then multiple links to the same content could be
+> unannexed.
+> 
+> But, this would be slower, and would depend on a later `unused` and
+> `dropunused` to actually remove the content. While doable, by use case
+> for unannex is more to quickly undo a mistaken add, and it's unlikely there
+> are multiple symlinks to the same content in this situation. --[[Joey]] 

From 109814d8bb4ea3fe61f8f02181e26f9cdad6ac5a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 4 Jul 2011 16:29:40 +0000
Subject: [PATCH 2002/8313] Comment moderation

---
 ...nt_2_273a45e6977d40d39e0d9ab924a83240._comment |  9 +++++++++
 ...nt_1_818f38aa988177d3a9415055e084f0fb._comment | 15 +++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment
 create mode 100644 doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment
new file mode 100644
index 0000000000..b01590a7a7
--- /dev/null
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos/comment_2_273a45e6977d40d39e0d9ab924a83240._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://ertai.myopenid.com/"
+ nickname="npouillard"
+ subject="git annex fsck --from remote"
+ date="2011-06-25T16:20:44Z"
+ content="""
+Currently fsck silently ignores --to/--from.
+It should at least complain if it is not supported.
+"""]]
diff --git a/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment
new file mode 100644
index 0000000000..11b44b8094
--- /dev/null
+++ b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_1_818f38aa988177d3a9415055e084f0fb._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="filtering based on git-commits"
+ date="2011-06-23T13:56:35Z"
+ content="""
+additional filter criteria could come from the git history:
+
+* `git annex get --touched-in HEAD~5..` to fetch what has recently been worked on
+* `git annex get --touched-by chrysn --touched-in version-1.0..HEAD` to fetch what i've been workin on recently (based on regexp or substring match in author; git experts could probably craft much more meaningful expressions)
+
+these options could also apply to `git annex find` -- actually, looking at the normal file system tools for such tasks, that might even be sufficient (think `git annex find --numcopies-gt 3 --present-on lanserver1 --drop` like `find -iname '*foo*' -delete`
+
+(i was about to open a new forum discussion for commit-based getting, but this is close enough to be usefully joint in a discussion)
+"""]]

From 18d82459c62f5a27c5fd8d7279cc760486ae82fa Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Mon, 4 Jul 2011 16:57:26 +0000
Subject: [PATCH 2003/8313] Added a comment

---
 ...comment_1_c896ff6589f62178b60e606771e4f2bf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment
new file mode 100644
index 0000000000..839992477a
--- /dev/null
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_1_c896ff6589f62178b60e606771e4f2bf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 1"
+ date="2011-07-04T16:57:25Z"
+ content="""
+You convince me for unannex, but isn't the goal of uninit to revert all annex operations? In the current state, a clean revert is not possible (because of the broken symlinks after uninit). Instead of copying, using hard links is out of question?
+
+For my needs, is the command \"git annex unlock .\" (from the root of the repo) a correct workaround?
+"""]]

From 5c63b409d4b046f2179ae3c542bfd47d86c8c90c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 15:50:30 -0400
Subject: [PATCH 2004/8313] uninit: Delete the git-annex branch.

---
 Command/Uninit.hs | 17 ++++++++++++-----
 debian/changelog  |  6 ++++++
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index c47ac0c3af..22006f7dcd 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -9,6 +9,7 @@ module Command.Uninit where
 
 import Control.Monad.State (liftIO)
 import System.Directory
+import System.Exit
 
 import Command
 import Messages
@@ -18,6 +19,8 @@ import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import qualified Command.Init
+import qualified Branch
+import Content
 
 command :: [Command]
 command = [repoCommand "uninit" paramPath seek 
@@ -27,15 +30,19 @@ seek :: [CommandSeek]
 seek = [withFilesInGit Command.Unannex.start, withNothing start]
 
 start :: CommandStartNothing
-start = do
-	showStart "uninit" ""
-	next perform
+start = next perform
 
 perform :: CommandPerform
-perform = do
+perform = next cleanup
+
+cleanup :: CommandCleanup
+cleanup = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
-	next $ return True
+	saveState
+	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
+	-- bypass normal shutdown, which writes to the deleted branch
+	liftIO exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/debian/changelog b/debian/changelog
index 266a747ee0..fb5b80ac0e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110703) UNRELEASED; urgency=low
+
+  * uninit: Delete the git-annex branch.
+
+ -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
+
 git-annex (3.20110702) unstable; urgency=low
 
   * Now the web can be used as a special remote. 

From 5beb6bc76fb3edbc28c238eb9596fc828aa49bfc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 15:55:03 -0400
Subject: [PATCH 2005/8313] uninit: delete .git/annex/

---
 Command/Uninit.hs | 9 ++++++---
 debian/changelog  | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 22006f7dcd..5cb66e83a1 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -21,6 +21,7 @@ import qualified Command.Unannex
 import qualified Command.Init
 import qualified Branch
 import Content
+import Locations
 
 command :: [Command]
 command = [repoCommand "uninit" paramPath seek 
@@ -40,9 +41,11 @@ cleanup = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 	saveState
-	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
-	-- bypass normal shutdown, which writes to the deleted branch
-	liftIO exitSuccess
+	liftIO $ do
+		Git.run g "branch" [Param "-D", Param Branch.name]
+		removeDirectoryRecursive (gitAnnexDir g)
+		-- avoid normal shutdown
+		exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/debian/changelog b/debian/changelog
index fb5b80ac0e..3564acd17b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (3.20110703) UNRELEASED; urgency=low
 
-  * uninit: Delete the git-annex branch.
+  * uninit: Delete the git-annex branch and .git/annex/
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 

From 22a4f5b348c72a07fac99786613f6efc2eeb3b17 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 16:06:28 -0400
Subject: [PATCH 2006/8313] unannex: In --fast mode, file content is left in
 the annex, and a hard link made to it.

---
 Command/Unannex.hs | 13 +++++++++++--
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  3 +++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d30f8d20f7..f0c1b27c6c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -10,6 +10,7 @@ module Command.Unannex where
 import Control.Monad.State (liftIO)
 import Control.Monad (unless)
 import System.Directory
+import System.Posix.Files
 
 import Command
 import qualified Annex
@@ -22,6 +23,7 @@ import Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Messages
+import Locations
 
 command :: [Command]
 command = [repoCommand "unannex" paramPath seek "undo accidential add command"]
@@ -64,8 +66,15 @@ cleanup file key = do
 	-- git rm deletes empty directories; put them back
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 
-	fromAnnex key file
-	logStatus key InfoMissing
+	fast <- Annex.getState Annex.fast
+	if fast
+		then liftIO $ do
+			-- fast mode: hard link to content in annex
+			createLink (gitAnnexLocation g key) file
+			allowWrite file
+		else do
+			fromAnnex key file
+			logStatus key InfoMissing
 	
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
diff --git a/debian/changelog b/debian/changelog
index 3564acd17b..844c6c3add 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20110703) UNRELEASED; urgency=low
 
   * uninit: Delete the git-annex branch and .git/annex/
+  * unannex: In --fast mode, file content is left in the annex, and a
+    hard link made to it.
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c450887488..dfb23f5f21 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -239,6 +239,9 @@ Many git-annex commands will stage changes for later `git commit` by you.
   file and don't want its contents any more. In that case you should use
   `git annex drop` instead, and you can also `git rm` the file.
 
+  In --fast mode, this command leaves content in the annex, simply making
+  a hard link to it.
+
 * uninit
 
   Use this to stop using git annex. It will unannex every file in the

From d7ce51af5abd5c8582a31c39c3968597f5cae34e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 16:13:44 -0400
Subject: [PATCH 2007/8313] fix usage for setkey

---
 Command/SetKey.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 546bd8e2d0..b000a4e8bf 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -16,7 +16,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [repoCommand "setkey" (paramRepeating paramKey) seek
+command = [repoCommand "setkey" (paramPath) seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]

From 71c783bf24f2aa4ab911d8279081bcad08951ece Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 16:19:04 -0400
Subject: [PATCH 2008/8313] uninit: Use unannex in --fast mode, to support
 unannexing multiple files that link to the same content.

---
 Command/Uninit.hs                             | 20 +++++++++++++------
 debian/changelog                              |  2 ++
 ...nnex__47__uninit_should_handle_copies.mdwn |  2 ++
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 5cb66e83a1..1497bbfd19 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,7 +28,15 @@ command = [repoCommand "uninit" paramPath seek
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit Command.Unannex.start, withNothing start]
+seek = [withFilesInGit startUnannex, withNothing start]
+
+startUnannex :: CommandStartString
+startUnannex file = do
+	-- Force fast mode before running unannex. This way, if multiple
+	-- files link to a key, it will be left in the annex and hardlinked
+	-- to by each.
+	Annex.changeState $ \s -> s { Annex.fast = True }
+	Command.Unannex.start file
 
 start :: CommandStartNothing
 start = next perform
@@ -40,12 +48,12 @@ cleanup :: CommandCleanup
 cleanup = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
+	mapM_ removeAnnex =<< getKeysPresent
+	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
+	-- avoid normal shutdown
 	saveState
-	liftIO $ do
-		Git.run g "branch" [Param "-D", Param Branch.name]
-		removeDirectoryRecursive (gitAnnexDir g)
-		-- avoid normal shutdown
-		exitSuccess
+	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
+	liftIO $ exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/debian/changelog b/debian/changelog
index 844c6c3add..8f559bb028 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20110703) UNRELEASED; urgency=low
   * uninit: Delete the git-annex branch and .git/annex/
   * unannex: In --fast mode, file content is left in the annex, and a
     hard link made to it.
+  * uninit: Use unannex in --fast mode, to support unannexing multiple
+    files that link to the same content.
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 
diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
index 751e1afa9b..e830f11564 100644
--- a/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies.mdwn
@@ -16,3 +16,5 @@ Rafaël
 > `dropunused` to actually remove the content. While doable, by use case
 > for unannex is more to quickly undo a mistaken add, and it's unlikely there
 > are multiple symlinks to the same content in this situation. --[[Joey]] 
+
+[[!tag done]]

From 02f2c744bd7335155a40f069b4d7bb91413765e9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY"
 
Date: Mon, 4 Jul 2011 20:25:39 +0000
Subject: [PATCH 2009/8313] Added a comment

---
 .../comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment

diff --git a/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment
new file mode 100644
index 0000000000..21c0c449b0
--- /dev/null
+++ b/doc/bugs/annex_unannex__47__uninit_should_handle_copies/comment_2_9249609f83f8e9c7521cd2f007c1a39e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmJfIszzreLNvCqzqzvTayA9_9L6gb9RtY"
+ nickname="Joey"
+ subject="comment 2"
+ date="2011-07-04T20:25:38Z"
+ content="""
+Indeed, uninit needed to be improved. I've done so. Also, unannex --fast can be used to make hard links to content left in the annex.
+"""]]

From 9869ebb260de96a04e464e8348e6de749f2b9a11 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Mon, 4 Jul 2011 22:54:15 +0000
Subject: [PATCH 2010/8313]

---
 doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
new file mode 100644
index 0000000000..10804e45dd
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
@@ -0,0 +1 @@
+On several of my repos, the upgrade to v3 seemed to take forever. A Crl-C followed by another "git annex upgrade" "solved" the problem in some cases. Sometimes, I had to also delete the .git/annex/journal dir to have the upgrade. I didn't notice anything special about the non-working repos to help diagnose the problem.

From 0ef0f277c17cc5ebedd47213494f2b54c5deec53 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 4 Jul 2011 22:58:46 +0000
Subject: [PATCH 2011/8313] Added a comment

---
 .../comment_1_5f60006c9bb095167d817f234a14d20b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment
new file mode 100644
index 0000000000..0cf0ad4618
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_1_5f60006c9bb095167d817f234a14d20b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-04T22:58:46Z"
+ content="""
+Well if it happens again why don't you use `ps` or `strace` to see what it's doing.
+"""]]

From 5c69ac14eb47e284ab4f4dec44ed6ab3581d416f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 19:31:45 -0400
Subject: [PATCH 2012/8313] Drop the dependency on the haskell curl bindings,
 use regular haskell HTTP.

---
 Remote/Web.hs           | 29 ++++++++++++++++++-----------
 debian/changelog        |  1 +
 debian/control          |  2 +-
 doc/install.mdwn        |  2 +-
 doc/install/Fedora.mdwn |  1 +
 doc/install/OSX.mdwn    |  1 +
 git-annex.cabal         |  4 ++--
 7 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 71591b7aa0..d3d140d73b 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -14,10 +14,9 @@ module Remote.Web (
 import Control.Monad.State (liftIO)
 import Control.Exception
 import System.FilePath
-import Network.Curl.Easy
-import Network.Curl.Opts
-import Network.Curl.Types
-import Network.Curl.Code
+import Network.Browser
+import Network.HTTP
+import Network.URI
 
 import Types
 import Types.Remote
@@ -31,6 +30,8 @@ import PresenceLog
 import LocationLog
 import Locations
 
+type URLString = String
+
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "web",
@@ -111,13 +112,19 @@ checkKey' (u:us) = do
 
 urlexists :: URLString -> IO Bool
 urlexists url = do
-	curl <- initialize
-	_ <- setopt curl (CurlURL url)
-	_ <- setopt curl (CurlNoBody True)
-	_ <- setopt curl (CurlFailOnError True)
-	_ <- setopt curl (CurlFollowLocation True)
-	res <- perform curl
-	return $ res == CurlOK
+	case parseURI url of
+		Nothing -> return False
+		Just u -> do
+			(_, r) <- Network.Browser.browse $ do
+				setErrHandler ignore
+				setOutHandler ignore
+				setAllowRedirects True
+				request (mkRequest HEAD u :: Request_String)
+			case rspCode r of
+				(2,_,_) -> return True
+				_ -> return False
+	where
+		ignore = const $ return ()
 
 download :: [URLString] -> FilePath -> Annex Bool
 download [] _ = return False
diff --git a/debian/changelog b/debian/changelog
index 8f559bb028..37d03ceb20 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110703) UNRELEASED; urgency=low
     hard link made to it.
   * uninit: Use unannex in --fast mode, to support unannexing multiple
     files that link to the same content.
+  * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
 
  -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
 
diff --git a/debian/control b/debian/control
index 07a9e4bbb2..4347233841 100644
--- a/debian/control
+++ b/debian/control
@@ -9,8 +9,8 @@ Build-Depends:
 	libghc-pcre-light-dev,
 	libghc-sha-dev,
 	libghc-dataenc-dev,
+	libghc-http-dev,
 	libghc-utf8-string-dev,
-	libghc-curl-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	ikiwiki,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 23a0da902d..38963695b8 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -25,7 +25,7 @@ To build and use git-annex, you will need:
   * [dataenc](http://hackage.haskell.org/package/dataenc)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
-  * [curl](http://hackage.haskell.org/package/curl)
+  * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
 * Shell commands
   * [git](http://git-scm.com/)
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 0050295e86..7814eec940 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -9,6 +9,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install HTTP
 sudo cabal install hS3
 
 git clone git://git-annex.branchable.com/
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 81ffe1d035..ade4fa30e8 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,6 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install HTTP
 sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
diff --git a/git-annex.cabal b/git-annex.cabal
index d5a066cdff..f7a2bde858 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,14 +30,14 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
    base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, curl,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
    base < 5
 
 Executable git-union-merge

From c9a81fa841e3502f0426627de2d0eed439680b6d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 19:36:36 -0400
Subject: [PATCH 2013/8313] update build deps

---
 git-annex.cabal | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index f7a2bde858..3779780ca9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,15 +30,11 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
    base < 5
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
-   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP
-   base < 5
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs

From e5ee6508ddab17487e6b6aa7f4d3ac7665d8f40c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jul 2011 19:54:46 -0400
Subject: [PATCH 2014/8313] use a more specific type for executeFile

Apparently the generic -> IO a type fails with some version of GHC.
Possibly due to System.Posix.Process.executeFile having a more specific
type.
---
 Utility.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility.hs b/Utility.hs
index 7831a4ab4b..63fa6eda35 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -108,7 +108,7 @@ boolSystemEnv command params env = do
 			executeFile command True (toCommand params) env
 
 {- executeFile with debug logging -}
-executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO a
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
 executeFile c path p e = do
 	debugM "Utility.executeFile" $
 		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e

From bddbb66ea4caf22e799e1bd8261e01a789697cc1 Mon Sep 17 00:00:00 2001
From: "https://lithitux.org/openidserver/users/pavel" 
Date: Tue, 5 Jul 2011 15:54:21 +0000
Subject: [PATCH 2015/8313] Added a comment: "Me too"

---
 ..._cd0123392b16d89db41b45464165c247._comment | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment
new file mode 100644
index 0000000000..4bef5f6454
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_2_cd0123392b16d89db41b45464165c247._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="https://lithitux.org/openidserver/users/pavel"
+ nickname="pavel"
+ subject=""Me too""
+ date="2011-07-05T15:54:19Z"
+ content="""
+I've also seen this apparent hang during upgrade to v3. A few more details:
+
+The annex in question has just under 18k files (and hence that many log files), which can slow down directory operations when they're all in the same place (like, for example, .git/annex/journal).
+
+git-annex uses virtually no CPU time and disk IO when it's hanging like this; the first time it happened, 'ps' showed three defunct git processes, with two \"git-annex\" processes and three \"git\" procs:
+
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex cat-file --batch
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex hash-object -w --stdin-paths
+ * git --git-dir=/mnt/annex/.git --work-tree=/mnt/annex update-index -z --index-info
+
+I Ctrl+C'd that and tried again, but it hung again -- this time without the defunct gits.
+
+An strace of the process and its children at the time of hang can be found at http://pastebin.com/4kNh4zEJ . It showed somewhat weird behaviour: When I attached with strace, it would scroll through a whole bunch of syscalls making up the open-fstat-read-close-write loop on .git/annex/journal files, but then would block on a write (sorry, don't have that in my scrollback any more so can't give more details) until I Ctrl+C'd strace; when attaching again, it would again scroll through the syscalls for a second or so and then hang with no output.
+
+Ultimately I detached/reattached with strace about two dozen times and that caused it (?) to finish the upgrade; not really sure how to explain it, but it seems like too much of a timing coincidence.
+
+"""]]

From 44e973dd0948afb25c47f176db83c34f90f49932 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 13:26:59 -0400
Subject: [PATCH 2016/8313] fork a process to feed git hash-object

This is another workaround for bug #624389. I hope it will fix
http://git-annex.branchable.com/bugs/problem_with_upgrade_v2_-__62___v3/
---
 Branch.hs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index c0c96611bf..d091b6edab 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -27,6 +27,8 @@ import Data.Maybe
 import Data.List
 import System.IO
 import System.IO.Binary
+import System.Posix.Process
+import System.Exit
 import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
@@ -329,12 +331,19 @@ stageJournalFiles = do
 			let paths = map (dir ) fs
 			-- inject all the journal files directly into git
 			-- in one quick command
-			(h, s) <- Git.pipeWriteRead g [Param "hash-object",
-				Param "-w", Param "--stdin-paths"] $ unlines paths
+			(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
+				Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
+			_ <- forkProcess $ do
+				hPutStr toh $ unlines paths
+				hClose toh
+				exitSuccess
+			hClose toh
+			s <- hGetContents fromh
 			-- update the index, also in just one command
 			Git.UnionMerge.update_index g $
 				index_lines (lines s) $ map fileJournal fs
-			forceSuccess h
+			hClose fromh
+			forceSuccess pid
 			mapM_ removeFile paths
 		index_lines shas fs = map genline $ zip shas fs
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file

From 5070340ca7477b746be8a2f4f1877c3115e1af2e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 5 Jul 2011 17:31:22 +0000
Subject: [PATCH 2017/8313] Added a comment

---
 ...t_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment
new file mode 100644
index 0000000000..e314e73fa0
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_3_86d9e7244ae492bcbe62720b8c4fc4a9._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-07-05T17:31:22Z"
+ content="""
+I've seen this kind of piping stall that is unblocked by strace before. It can vary with versions of GHC, so it would be good to know what version built git-annex (and on what OS version). I filed a bug report upstream before at .
+
+I really need a full strace -f from the top, or at least a complete `strace -o log` of git-annex from one hang through to another hang. The strace you pastebinned does not seem complete. If I can work out which specific git command is being written to when it hangs I can lift the writing out into a separate thread or process to fix it.
+
+@pavel, you mentioned three defunct git processes, and then showed ps output for 3 git processes. Were there 6 git processes in total? And then when you ran it again you said there were no defunct gits -- where the other 3 git processes running once again?
+
+As best I can make out from the (apparently) running git processes, it seems like the journal files for the upgrade had all been written, and the hang occurred when staging them all into the index in preparation for a commit. I have committed a change that lifts the code that does that write out into a new process, which, if I am guessing right on the limited info I have, will avoid the hang.
+
+However, since I can't reproduce it, even when I put 200 thousand files in the journal and have git-annex process them, I can't be sure.
+"""]]

From 82eb082ab9a33713ed1ec3674be2b95f9b81d861 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 14:36:21 -0400
Subject: [PATCH 2018/8313] my fix is confirmed to have worked

Also audited for other uses of pipeBoth and hPipeBoth and they mostly seem
safe.
---
 doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
index 10804e45dd..7f37668ad1 100644
--- a/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3.mdwn
@@ -1 +1,3 @@
 On several of my repos, the upgrade to v3 seemed to take forever. A Crl-C followed by another "git annex upgrade" "solved" the problem in some cases. Sometimes, I had to also delete the .git/annex/journal dir to have the upgrade. I didn't notice anything special about the non-working repos to help diagnose the problem.
+
+[[!tag done]]

From 502bac1c719eaec7eda38fdcb314b4e6c9ba65f1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 5 Jul 2011 18:37:21 +0000
Subject: [PATCH 2019/8313] Added a comment

---
 .../comment_4_91439d4dbbf1461e281b276eb0003691._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment
new file mode 100644
index 0000000000..7bc32c259f
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_4_91439d4dbbf1461e281b276eb0003691._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-07-05T18:37:21Z"
+ content="""
+I've managed to reproduce this and confirmed my fix works.
+"""]]

From d31b84c777b6ba7158be8947fc2236b2a15e29bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 14:58:33 -0400
Subject: [PATCH 2020/8313] better display of thrown errors

---
 CmdLine.hs     | 4 +++-
 Command.hs     | 9 +++++++--
 Command/Map.hs | 1 +
 Messages.hs    | 6 ++++--
 4 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 85423e5e89..46b980fbcb 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -87,7 +87,9 @@ tryRun' state errnum (a:as) = do
 		a
 	case result of
 		Left err -> do
-			Annex.eval state $ showErr err
+			Annex.eval state $ do
+				showEndFail
+				showErr err
 			tryRun' state (errnum + 1) as
 		Right (True,state') -> tryRun' state' errnum as
 		Right (False,state') -> tryRun' state' (errnum + 1) as
diff --git a/Command.hs b/Command.hs
index d36f675d24..c666ddbd29 100644
--- a/Command.hs
+++ b/Command.hs
@@ -96,10 +96,15 @@ prepCommand Command { cmdseek = seek } params = do
 doCommand :: CommandStart -> CommandCleanup
 doCommand = start
 	where
-		start   = stage $ maybe (return True) perform
-		perform = stage $ maybe (showEndFail >> return False) cleanup
+		start   = stage $ maybe success perform
+		perform = stage $ maybe failure cleanup
 		cleanup = stage $ \r -> showEndResult r >> return r
 		stage a b = b >>= a
+		success = return True
+		failure = do
+			showProgress
+			showEndFail
+			return False
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
diff --git a/Command/Map.hs b/Command/Map.hs
index 7bb435ff81..940db54c89 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -175,6 +175,7 @@ scan r = do
 			showEndOk
 			return r'
 		Nothing -> do
+			showProgress
 			showEndFail
 			return r
 
diff --git a/Messages.hs b/Messages.hs
index c44e44eea1..038e4c0bc7 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -43,14 +43,16 @@ showEndOk :: Annex ()
 showEndOk = verbose $ liftIO $ putStrLn "ok"
 
 showEndFail :: Annex ()
-showEndFail = verbose $ liftIO $ putStrLn "\nfailed"
+showEndFail = verbose $ liftIO $ putStrLn "failed"
 
 showEndResult :: Bool -> Annex ()
 showEndResult True = showEndOk
 showEndResult False = showEndFail
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = warning $ "git-annex: " ++ show e
+showErr e = do
+	liftIO $ hFlush stdout
+	liftIO $ hPutStrLn stderr $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
 warning w = do

From d360ca3ed9ebfc4abc34694138ff5628838fed5f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 15:06:20 -0400
Subject: [PATCH 2021/8313] make upgrade more robust

don't remove .git-annex until state has been succeffully saved
---
 Upgrade/V2.hs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 14e328edb4..4824f4bbaf 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -54,11 +54,13 @@ upgrade = do
 	when e $ do
 		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
 		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
-		liftIO $ do
-			Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-			unless bare $ gitAttributesUnWrite g
 
 	saveState
+
+	when e $ liftIO $ do
+		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
+		unless bare $ gitAttributesUnWrite g
+
 	unless bare $ push
 
 	return True

From fb433a5ba2e69485b4dd2e84919a8da23fc039de Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 5 Jul 2011 19:06:48 +0000
Subject: [PATCH 2022/8313] Added a comment

---
 .../comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment
new file mode 100644
index 0000000000..8649dc77a8
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_5_ca33a9ca0df33f7c1b58353d7ffb943d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-07-05T19:06:48Z"
+ content="""
+By the way, the original bug reporter mentioned deleting .git/annex/journal. This is not recommended, and doing it during an upgrade can result in git-annex losing location tracking information. You should probably run `git annex fsck` or reset to the old git tree (and `git config annex.version 2`) and upgrade again.
+"""]]

From d583e04d23005d16d9f25388b84ce27eadecac69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 15:21:38 -0400
Subject: [PATCH 2023/8313] releasing version 3.20110705

---
 debian/changelog | 5 +++--
 git-annex.cabal  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 37d03ceb20..773c91b7eb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110703) UNRELEASED; urgency=low
+git-annex (3.20110705) unstable; urgency=low
 
   * uninit: Delete the git-annex branch and .git/annex/
   * unannex: In --fast mode, file content is left in the annex, and a
@@ -6,8 +6,9 @@ git-annex (3.20110703) UNRELEASED; urgency=low
   * uninit: Use unannex in --fast mode, to support unannexing multiple
     files that link to the same content.
   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
+  * Fix a pipeline stall when upgrading (caused by #624389).
 
- -- Joey Hess   Mon, 04 Jul 2011 15:50:21 -0400
+ -- Joey Hess   Tue, 05 Jul 2011 14:37:39 -0400
 
 git-annex (3.20110702) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 3779780ca9..3ea7e5fb39 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110702.2
+Version: 3.20110705
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 674768abac3efb2646479c6afba76d9ff27fd802 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 15:22:12 -0400
Subject: [PATCH 2024/8313] add news item for git-annex 3.20110705

---
 doc/news/version_0.20110522.mdwn | 5 -----
 doc/news/version_3.20110705.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_0.20110522.mdwn
 create mode 100644 doc/news/version_3.20110705.mdwn

diff --git a/doc/news/version_0.20110522.mdwn b/doc/news/version_0.20110522.mdwn
deleted file mode 100644
index 5dccb993e2..0000000000
--- a/doc/news/version_0.20110522.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 0.20110522 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Closer emulation of git's behavior when told to use "foo/.git" as a
-     git repository instead of just "foo". Closes: #[627563](http://bugs.debian.org/627563)
-   * Fix bug in --exclude introduced in 0.20110516."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110705.mdwn b/doc/news/version_3.20110705.mdwn
new file mode 100644
index 0000000000..bb4665c047
--- /dev/null
+++ b/doc/news/version_3.20110705.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110705 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * uninit: Delete the git-annex branch and .git/annex/
+   * unannex: In --fast mode, file content is left in the annex, and a
+     hard link made to it.
+   * uninit: Use unannex in --fast mode, to support unannexing multiple
+     files that link to the same content.
+   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
+   * Fix a pipeline stall when upgrading (caused by #624389)."""]]
\ No newline at end of file

From 9f1577f74684d8d627e75d3021eb1ff50ef7492f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 18:31:46 -0400
Subject: [PATCH 2025/8313] remove unused backend machinery

The only remaining vestiage of backends is different types of keys. These
are still called "backends", mostly to avoid needing to change user interface
and configuration. But everything to do with storing keys in different
backends was gone; instead different types of remotes are used.

In the refactoring, lots of code was moved out of odd corners like
Backend.File, to closer to where it's used, like Command.Drop and
Command.Fsck. Quite a lot of dead code was removed. Several data structures
became simpler, which may result in better runtime efficiency. There should
be no user-visible changes.
---
 Annex.hs              |  11 +--
 Backend.hs            | 168 ++++++++++----------------------
 Backend/File.hs       | 220 ------------------------------------------
 Backend/SHA.hs        |   5 +-
 Backend/WORM.hs       |   6 +-
 BackendList.hs        |  19 ----
 CmdLine.hs            |   3 +-
 Command/Add.hs        |   4 +-
 Command/AddUrl.hs     |   4 +-
 Command/Drop.hs       |  60 ++++++++++--
 Command/DropUnused.hs |   5 +-
 Command/FromKey.hs    |   3 +-
 Command/Fsck.hs       |  77 ++++++++++++++-
 Command/Get.hs        |  49 ++++++++--
 Command/Migrate.hs    |  17 ++--
 Command/Status.hs     |   6 +-
 Command/Unannex.hs    |  13 ++-
 Command/Unlock.hs     |   3 +-
 Config.hs             |  13 +++
 LocationLog.hs        |   2 +-
 Remote.hs             |  33 ++++++-
 Remote/Git.hs         |   4 +-
 Types/Backend.hs      |  16 +--
 Upgrade/V1.hs         |   7 +-
 test.hs               |   5 +-
 25 files changed, 308 insertions(+), 445 deletions(-)
 delete mode 100644 Backend/File.hs
 delete mode 100644 BackendList.hs

diff --git a/Annex.hs b/Annex.hs
index c21cfb37ca..f7e3e29f82 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -34,7 +34,6 @@ type Annex = StateT AnnexState IO
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [Backend Annex]
-	, supportedBackends :: [Backend Annex]
 	, remotes :: [Remote Annex]
 	, repoqueue :: Queue
 	, quiet :: Bool
@@ -52,12 +51,11 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-newState :: [Backend Annex] -> Git.Repo -> AnnexState
-newState allbackends gitrepo = AnnexState
+newState :: Git.Repo -> AnnexState
+newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, supportedBackends = allbackends
 	, repoqueue = empty
 	, quiet = False
 	, force = False
@@ -75,9 +73,8 @@ newState allbackends gitrepo = AnnexState
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
-new :: Git.Repo -> [Backend Annex] -> IO AnnexState
-new gitrepo allbackends =
-	newState allbackends `liftM` (liftIO . Git.configRead) gitrepo
+new :: Git.Repo -> IO AnnexState
+new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Backend.hs b/Backend.hs
index b1cd4c8f0e..cf976d2b8a 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -1,16 +1,4 @@
-{- git-annex key-value storage backends
- -
- - git-annex uses a key-value abstraction layer to allow files contents to be
- - stored in different ways. In theory, any key-value storage system could be
- - used to store the file contents, and git-annex would then retrieve them
- - as needed and put them in `.git/annex/`.
- - 
- - When a file is annexed, a key is generated from its content and/or metadata.
- - This key can later be used to retrieve the file's content (its value). This
- - key generation must be stable for a given file content, name, and size.
- - 
- - Multiple pluggable backends are supported, and more than one can be used
- - to store different files' contents in a given repository.
+{- git-annex key/value backends
  -
  - Copyright 2010 Joey Hess 
  -
@@ -19,15 +7,10 @@
 
 module Backend (
 	list,
-	storeFileKey,
-	retrieveKeyFile,
-	removeKey,
-	hasKey,
-	fsckKey,
-	upgradableKey,
+	orderedList,
+	genKey,
 	lookupFile,
 	chooseBackends,
-	keyBackend,
 	lookupBackendName,
 	maybeLookupBackendName
 ) where
@@ -36,7 +19,6 @@ import Control.Monad.State (liftIO, when)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
-import System.Directory
 
 import Locations
 import qualified Git
@@ -45,12 +27,20 @@ import Types
 import Types.Key
 import qualified Types.Backend as B
 import Messages
-import Content
-import DataUnits
+
+-- When adding a new backend, import it here and add it to the list.
+import qualified Backend.WORM
+import qualified Backend.SHA
+
+list :: [Backend Annex]
+list = concat 
+	[ Backend.WORM.backends
+	, Backend.SHA.backends
+	]
 
 {- List of backends in the order to try them when storing a new key. -}
-list :: Annex [Backend Annex]
-list = do
+orderedList :: Annex [Backend Annex]
+orderedList = do
 	l <- Annex.getState Annex.backends -- list is cached here
 	if not $ null l
 		then return l
@@ -59,92 +49,49 @@ list = do
 			d <- Annex.getState Annex.forcebackend
 			handle d s
 	where
-		parseBackendList l [] = l
-		parseBackendList bs s = map (lookupBackendName bs) $ words s
+		parseBackendList [] = list
+		parseBackendList s = map lookupBackendName $ words s
 		handle Nothing s = return s
 		handle (Just "") s = return s
 		handle (Just name) s = do
-			bs <- Annex.getState Annex.supportedBackends
-			let l' = (lookupBackendName bs name):s
+			let l' = (lookupBackendName name):s
 			Annex.changeState $ \state -> state { Annex.backends = l' }
 			return l'
 		getstandard = do
-			bs <- Annex.getState Annex.supportedBackends
 			g <- Annex.gitRepo
-			return $ parseBackendList bs $
+			return $ parseBackendList $
 				Git.configGet g "annex.backends" ""
 
-{- Looks up a backend in a list. May fail if unknown. -}
-lookupBackendName :: [Backend Annex] -> String -> Backend Annex
-lookupBackendName bs s = maybe unknown id $ maybeLookupBackendName bs s
-	where
-		unknown = error $ "unknown backend " ++ s
-maybeLookupBackendName :: [Backend Annex] -> String -> Maybe (Backend Annex)
-maybeLookupBackendName bs s =
-	if 1 /= length matches
-		then Nothing
-		else Just $ head matches
-	where matches = filter (\b -> s == B.name b) bs
-
-{- Attempts to store a file in one of the backends. -}
-storeFileKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
-storeFileKey file trybackend = do
-	bs <- list
+{- Generates a key for a file, trying each backend in turn until one
+ - accepts it. -}
+genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
+genKey file trybackend = do
+	bs <- orderedList
 	let bs' = maybe bs (:bs) trybackend
-	storeFileKey' bs' file
-storeFileKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
-storeFileKey' [] _ = return Nothing
-storeFileKey' (b:bs) file = maybe nextbackend store =<< (B.getKey b) file
-	where
-		nextbackend = storeFileKey' bs file
-		store key = do
-			stored <- (B.storeFileKey b) file key
-			if (not stored)
-				then nextbackend
-				else return $ Just (key, b)
-
-{- Attempts to retrieve an key from one of the backends, saving it to
- - a specified location. -}
-retrieveKeyFile :: Backend Annex -> Key -> FilePath -> Annex Bool
-retrieveKeyFile backend key dest = (B.retrieveKeyFile backend) key dest
-
-{- Removes a key from a backend. -}
-removeKey :: Backend Annex -> Key -> Maybe Int -> Annex Bool
-removeKey backend key numcopies = (B.removeKey backend) key numcopies
-
-{- Checks if a key is present in its backend. -}
-hasKey :: Key -> Annex Bool
-hasKey key = do
-	backend <- keyBackend key
-	(B.hasKey backend) key
-
-{- Checks a key for problems. -}
-fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-fsckKey backend key file numcopies = do
-	size_ok <- checkKeySize key
-	backend_ok <-(B.fsckKey backend) key file numcopies
-	return $ size_ok && backend_ok
-
-{- Checks if a key is upgradable to a newer representation. -}
-upgradableKey :: Backend Annex -> Key -> Annex Bool
-upgradableKey backend key = (B.upgradableKey backend) key
+	genKey' bs' file
+genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
+genKey' [] _ = return Nothing
+genKey' (b:bs) file = do
+	r <- (B.getKey b) file
+	case r of
+		Nothing -> genKey' bs file
+		Just k -> return $ Just (k, b)
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile file = do
-	bs <- Annex.getState Annex.supportedBackends
 	tl <- liftIO $ try getsymlink
 	case tl of
 		Left _ -> return Nothing
-		Right l -> makekey bs l
+		Right l -> makekey l
 	where
 		getsymlink = do
 			l <- readSymbolicLink file
 			return $ takeFileName l
-		makekey bs l = maybe (return Nothing) (makeret bs l) (fileKey l)
-		makeret bs l k =
-			case maybeLookupBackendName bs bname of
+		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
+		makeret l k =
+			case maybeLookupBackendName bname of
 					Just backend -> return $ Just (k, backend)
 					Nothing -> do
 						when (isLinkToAnnex l) $
@@ -164,37 +111,20 @@ chooseBackends fs = do
 	forced <- Annex.getState Annex.forcebackend
 	if forced /= Nothing
 		then do
-			l <- list
+			l <- orderedList
 			return $ map (\f -> (f, Just $ head l)) fs
 		else do
-			bs <- Annex.getState Annex.supportedBackends
 			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-			return $ map (\(f,b) -> (f, maybeLookupBackendName bs b)) pairs
+			return $ map (\(f,b) -> (f, maybeLookupBackendName b)) pairs
 
-{- Returns the backend to use for a key. -}
-keyBackend :: Key -> Annex (Backend Annex)
-keyBackend key = do
-	bs <- Annex.getState Annex.supportedBackends
-	return $ lookupBackendName bs $ keyBackendName key
-
-{- The size of the data for a key is checked against the size encoded in
- - the key's metadata, if available. -}
-checkKeySize :: Key -> Annex Bool
-checkKeySize key = do
-	g <- Annex.gitRepo
-	let file = gitAnnexLocation g key
-	present <- liftIO $ doesFileExist file
-	case (present, keySize key) of
-		(_, Nothing) -> return True
-		(False, _) -> return True
-		(True, Just size) -> do
-			stat <- liftIO $ getFileStatus file
-			let size' = fromIntegral (fileSize stat)
-			if size == size'
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file size (" ++
-						compareSizes storageUnits True size size' ++ 
-						"); moved to " ++ dest
-					return False
+{- Looks up a backend by name. May fail if unknown. -}
+lookupBackendName :: String -> Backend Annex
+lookupBackendName s = maybe unknown id $ maybeLookupBackendName s
+	where
+		unknown = error $ "unknown backend " ++ s
+maybeLookupBackendName :: String -> Maybe (Backend Annex)
+maybeLookupBackendName s =
+	if 1 /= length matches
+		then Nothing
+		else Just $ head matches
+	where matches = filter (\b -> s == B.name b) list
diff --git a/Backend/File.hs b/Backend/File.hs
deleted file mode 100644
index 174da4e6dc..0000000000
--- a/Backend/File.hs
+++ /dev/null
@@ -1,220 +0,0 @@
-{- git-annex pseudo-backend
- -
- - This backend does not really do any independant data storage,
- - it relies on the file contents in .git/annex/ in this repo,
- - and other accessible repos.
- -
- - This is an abstract backend; name, getKey and fsckKey have to be implemented
- - to complete it.
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Backend.File (backend, checkKey) where
-
-import Data.List
-import Data.String.Utils
-
-import Types.Backend
-import LocationLog
-import qualified Remote
-import qualified Git
-import Content
-import qualified Annex
-import Types
-import UUID
-import Messages
-import Trust
-import Types.Key
-
-backend :: Backend Annex
-backend = Backend {
-	name = mustProvide,
-	getKey = mustProvide,
-	storeFileKey = dummyStore,
-	retrieveKeyFile = copyKeyFile,
-	removeKey = checkRemoveKey,
-	hasKey = inAnnex,
-	fsckKey = checkKeyOnly,
-	upgradableKey = checkUpgradableKey
-}
-
-mustProvide :: a
-mustProvide = error "must provide this field"
-
-{- Storing a key is a no-op. -}
-dummyStore :: FilePath -> Key -> Annex Bool
-dummyStore _ _ = return True
-
-{- Try to find a copy of the file in one of the remotes,
- - and copy it to here. -}
-copyKeyFile :: Key -> FilePath -> Annex Bool
-copyKeyFile key file = do
-	remotes <- Remote.keyPossibilities key
-	if null remotes
-		then do
-			showNote "not available"
-			showLocations key []
-			return False
-		else trycopy remotes remotes
-	where
-		trycopy full [] = do
-			showTriedRemotes full
-			showLocations key []
-			return False
-		trycopy full (r:rs) = do
-			probablythere <- probablyPresent r
-			if probablythere
-				then docopy r (trycopy full rs)
-				else trycopy full rs
-		-- This check is to avoid an ugly message if a remote is a
-		-- drive that is not mounted.
-		probablyPresent r =
-			if Remote.hasKeyCheap r
-				then do
-					res <- Remote.hasKey r key
-					case res of
-						Right b -> return b
-						Left _ -> return False
-				else return True
-		docopy r continue = do
-			showNote $ "from " ++ Remote.name r ++ "..."
-			copied <- Remote.retrieveKeyFile r key file
-			if copied
-				then return True
-				else continue
-
-{- Checks remotes to verify that enough copies of a key exist to allow
- - for a key to be safely removed (with no data loss), and fails with an
- - error if not. -}
-checkRemoveKey :: Key -> Maybe Int -> Annex Bool
-checkRemoveKey key numcopiesM = do
-	force <- Annex.getState Annex.force
-	if force || numcopiesM == Just 0
-		then return True
-		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
-			untrusteduuids <- trustGet UnTrusted
-			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
-			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies trusteduuids tocheck []
-	where
-		findcopies need have [] bad
-			| length have >= need = return True
-			| otherwise = notEnoughCopies need have bad
-		findcopies need have (r:rs) bad
-			| length have >= need = return True
-			| otherwise = do
-				let u = Remote.uuid r
-				let dup = u `elem` have
-				haskey <- Remote.hasKey r key
-				case (dup, haskey) of
-					(False, Right True)	-> findcopies need (u:have) rs bad
-					(False, Left _)		-> findcopies need have rs (r:bad)
-					_			-> findcopies need have rs bad
-		notEnoughCopies need have bad = do
-			unsafe
-			showLongNote $
-				"Could only verify the existence of " ++
-				show (length have) ++ " out of " ++ show need ++ 
-				" necessary copies"
-			showTriedRemotes bad
-			showLocations key have
-			hint
-			return False
-		unsafe = showNote "unsafe"
-		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"
-
-showLocations :: Key -> [UUID] -> Annex ()
-showLocations key exclude = do
-	g <- Annex.gitRepo
-	u <- getUUID g
-	uuids <- keyLocations key
-	untrusteduuids <- trustGet UnTrusted
-	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
-	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
-	showLongNote $ message ppuuidswanted ppuuidsskipped
-	where
-		filteruuids list x = filter (`notElem` x) list
-		message [] [] = "No other repository is known to contain the file."
-		message rs [] = "Try making some of these repositories available:\n" ++ rs
-		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
-		message rs us = message rs [] ++ message [] us
-
-showTriedRemotes :: [Remote.Remote Annex] -> Annex ()
-showTriedRemotes [] = return ()	
-showTriedRemotes remotes =
-	showLongNote $ "Unable to access these remotes: " ++
-		(join ", " $ map Remote.name remotes)
-
-{- If a value is specified, it is used; otherwise the default is looked up
- - in git config. forcenumcopies overrides everything. -}
-getNumCopies :: Maybe Int -> Annex Int
-getNumCopies v = 
-	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
-	where
-		use (Just n) = return n
-		use Nothing = do
-			g <- Annex.gitRepo
-			return $ read $ Git.configGet g config "1"
-		config = "annex.numcopies"
-
-{- Ideally, all keys have file size metadata. Old keys may not. -}
-checkUpgradableKey :: Key -> Annex Bool
-checkUpgradableKey key
-	| keySize key == Nothing = return True
-	| otherwise = return False
-
-{- This is used to check that numcopies is satisfied for the key on fsck.
- - This trusts data in the the location log, and so can check all keys, even
- - those with data not present in the current annex.
- -
- - The passed action is first run to allow backends deriving this one
- - to do their own checks.
- -}
-checkKey :: (Key -> Annex Bool) -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-checkKey a key file numcopies = do
-	a_ok <- a key
-	copies_ok <- checkKeyNumCopies key file numcopies
-	return $ a_ok && copies_ok
-
-checkKeyOnly :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-checkKeyOnly = checkKey (\_ -> return True)
-
-checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-checkKeyNumCopies key file numcopies = do
-	needed <- getNumCopies numcopies
-	locations <- keyLocations key
-	untrusted <- trustGet UnTrusted
-	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (`notElem` untrusted) locations
-	let present = length safelocations
-	if present < needed
-		then do
-			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
-			warning $ missingNote (filename file key) present needed ppuuids
-			return False
-		else return True
-	where
-		filename Nothing k = show k
-		filename (Just f) _ = f
-
-missingNote :: String -> Int -> Int -> String -> String
-missingNote file 0 _ [] = 
-		"** No known copies exist of " ++ file
-missingNote file 0 _ untrusted =
-		"Only these untrusted locations may have copies of " ++ file ++
-		"\n" ++ untrusted ++
-		"Back it up to trusted locations with git-annex copy."
-missingNote file present needed [] =
-		"Only " ++ show present ++ " of " ++ show needed ++ 
-		" trustworthy copies exist of " ++ file ++
-		"\nBack it up with git-annex copy."
-missingNote file present needed untrusted = 
-		missingNote file present needed [] ++
-		"\nThe following untrusted locations may also have copies: " ++
-		"\n" ++ untrusted
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 8930e4b938..bd6e411a05 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -16,7 +16,6 @@ import Data.Maybe
 import System.Posix.Files
 import System.FilePath
 
-import qualified Backend.File
 import Messages
 import qualified Annex
 import Locations
@@ -42,10 +41,10 @@ genBackend size
 	| shaCommand size == Nothing = Nothing
 	| otherwise = Just b
 	where
-		b = Backend.File.backend 
+		b = Types.Backend.Backend
 			{ name = shaName size
 			, getKey = keyValue size
-			, fsckKey = Backend.File.checkKey $ checkKeyChecksum size
+			, fsckKey = checkKeyChecksum size
 			}
 
 genBackendE :: SHASize -> Maybe (Backend Annex)
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index dc2e48adce..036d0564cb 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -11,7 +11,6 @@ import Control.Monad.State
 import System.FilePath
 import System.Posix.Files
 
-import qualified Backend.File
 import Types.Backend
 import Types
 import Types.Key
@@ -20,9 +19,10 @@ backends :: [Backend Annex]
 backends = [backend]
 
 backend :: Backend Annex
-backend = Backend.File.backend {
+backend = Types.Backend.Backend {
 	name = "WORM",
-	getKey = keyValue
+	getKey = keyValue,
+	fsckKey = const (return True)
 }
 
 {- The key includes the file size, modification time, and the
diff --git a/BackendList.hs b/BackendList.hs
deleted file mode 100644
index e4e1d76fe2..0000000000
--- a/BackendList.hs
+++ /dev/null
@@ -1,19 +0,0 @@
-{- git-annex backend list
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module BackendList (allBackends) where
-
--- When adding a new backend, import it here and add it to the list.
-import qualified Backend.WORM
-import qualified Backend.SHA
-import Types
-
-allBackends :: [Backend Annex]
-allBackends = concat 
-	[ Backend.WORM.backends
-	, Backend.SHA.backends
-	]
diff --git a/CmdLine.hs b/CmdLine.hs
index 46b980fbcb..b807046df3 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,6 @@ import qualified Git
 import Content
 import Types
 import Command
-import BackendList
 import Version
 import Options
 import Messages
@@ -32,7 +31,7 @@ import UUID
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
 dispatch args cmds options header gitrepo = do
 	setupConsole
-	state <- Annex.new gitrepo allBackends
+	state <- Annex.new gitrepo
 	(actions, state') <- Annex.run state $ parseCmd args header cmds options
 	tryRun state' $ [startup] ++ actions ++ [shutdown]
 
diff --git a/Command/Add.hs b/Command/Add.hs
index 6a1ffb5da6..2831e1b35f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -42,8 +42,8 @@ start pair@(file, _) = notAnnexed file $ do
 
 perform :: BackendFile -> CommandPerform
 perform (file, backend) = do
-	stored <- Backend.storeFileKey file backend
-	case stored of
+	k <- Backend.genKey file backend
+	case k of
 		Nothing -> stop
 		Just (key, _) -> do
 			moveAnnex key file
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index ebf0810bae..e80fe9621b 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -51,8 +51,8 @@ perform url file = do
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
-			stored <- Backend.storeFileKey tmp backend
-			case stored of
+			k <- Backend.genKey tmp backend
+			case k of
 				Nothing -> stop
 				Just (key, _) -> do
 					moveAnnex key tmp
diff --git a/Command/Drop.hs b/Command/Drop.hs
index bd47407413..14f098349e 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -8,12 +8,15 @@
 module Command.Drop where
 
 import Command
-import qualified Backend
+import qualified Remote
+import qualified Annex
 import LocationLog
 import Types
 import Content
 import Messages
 import Utility
+import Trust
+import Config
 
 command :: [Command]
 command = [repoCommand "drop" paramPath seek
@@ -25,19 +28,19 @@ seek = [withAttrFilesInGit "annex.numcopies" start]
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
 start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, backend) -> do
-	inbackend <- Backend.hasKey key
-	if inbackend
+start (file, attr) = isAnnexed file $ \(key, _) -> do
+	present <- inAnnex key
+	if present
 		then do
 			showStart "drop" file
-			next $ perform key backend numcopies
+			next $ perform key numcopies
 		else stop
 	where
 		numcopies = readMaybe attr :: Maybe Int
 
-perform :: Key -> Backend Annex -> Maybe Int -> CommandPerform
-perform key backend numcopies = do
-	success <- Backend.removeKey backend key numcopies
+perform :: Key -> Maybe Int -> CommandPerform
+perform key numcopies = do
+	success <- dropKey key numcopies
 	if success
 		then next $ cleanup key
 		else stop
@@ -47,3 +50,44 @@ cleanup key = do
 	whenM (inAnnex key) $ removeAnnex key
 	logStatus key InfoMissing
 	return True
+
+{- Checks remotes to verify that enough copies of a key exist to allow
+ - for a key to be safely removed (with no data loss), and fails with an
+ - error if not. -}
+dropKey :: Key -> Maybe Int -> Annex Bool
+dropKey key numcopiesM = do
+	force <- Annex.getState Annex.force
+	if force || numcopiesM == Just 0
+		then return True
+		else do
+			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
+			untrusteduuids <- trustGet UnTrusted
+			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
+			numcopies <- getNumCopies numcopiesM
+			findcopies numcopies trusteduuids tocheck []
+	where
+		findcopies need have [] bad
+			| length have >= need = return True
+			| otherwise = notEnoughCopies need have bad
+		findcopies need have (r:rs) bad
+			| length have >= need = return True
+			| otherwise = do
+				let u = Remote.uuid r
+				let dup = u `elem` have
+				haskey <- Remote.hasKey r key
+				case (dup, haskey) of
+					(False, Right True)	-> findcopies need (u:have) rs bad
+					(False, Left _)		-> findcopies need have rs (r:bad)
+					_			-> findcopies need have rs bad
+		notEnoughCopies need have bad = do
+			unsafe
+			showLongNote $
+				"Could only verify the existence of " ++
+				show (length have) ++ " out of " ++ show need ++ 
+				" necessary copies"
+			Remote.showTriedRemotes bad
+			Remote.showLocations key have
+			hint
+			return False
+		unsafe = showNote "unsafe"
+		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 2125abdc3c..55007c1f73 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -21,7 +21,6 @@ import qualified Command.Drop
 import qualified Command.Move
 import qualified Remote
 import qualified Git
-import Backend
 import Types.Key
 import Utility
 
@@ -64,9 +63,7 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 			r <- Remote.byName name
 			showNote $ "from " ++ Remote.name r ++ "..."
 			next $ Command.Move.fromCleanup r True key
-		droplocal = do
-			backend <- keyBackend key
-			Command.Drop.perform key backend (Just 0) -- force drop
+		droplocal = Command.Drop.perform key (Just 0) -- force drop
 
 performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 34816d6574..fb9ab0775a 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -15,7 +15,6 @@ import Control.Monad (unless)
 import Command
 import qualified AnnexQueue
 import Utility
-import qualified Backend
 import Content
 import Messages
 import Types.Key
@@ -30,7 +29,7 @@ seek = [withFilesMissing start]
 start :: CommandStartString
 start file = notBareRepo $ do
 	key <- cmdlineKey
-	inbackend <- Backend.hasKey key
+	inbackend <- inAnnex key
 	unless inbackend $ error $
 		"key ("++keyName key++") is not present in backend"
 	showStart "fromkey" file
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 988cfd28d1..446d25a449 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -9,10 +9,15 @@ module Command.Fsck where
 
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
+import System.Directory
+import Data.List
+import System.Posix.Files
 
 import Command
-import qualified Backend
 import qualified Annex
+import qualified Remote
+import qualified Types.Backend
+import qualified Types.Key
 import UUID
 import Types
 import Messages
@@ -20,6 +25,9 @@ import Utility
 import Content
 import LocationLog
 import Locations
+import Trust
+import DataUnits
+import Config
 
 command :: [Command]
 command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
@@ -40,7 +48,7 @@ perform key file backend numcopies = do
 	-- the location log is checked first, so that if it has bad data
 	-- that gets corrected
 	locationlogok <- verifyLocationLog key file
-	backendok <- Backend.fsckKey backend key (Just file) numcopies
+	backendok <- fsckKey backend key (Just file) numcopies
 	if locationlogok && backendok
 		then next $ return True
 		else stop
@@ -80,3 +88,68 @@ verifyLocationLog key file = do
 		fix g u s = do
 			showNote "fixing location log"
 			logChange g key u s
+
+{- Checks a key for problems. -}
+fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+fsckKey backend key file numcopies = do
+	size_ok <- checkKeySize key
+	copies_ok <- checkKeyNumCopies key file numcopies
+	backend_ok <-(Types.Backend.fsckKey backend) key
+	return $ size_ok && copies_ok && backend_ok
+
+{- The size of the data for a key is checked against the size encoded in
+ - the key's metadata, if available. -}
+checkKeySize :: Key -> Annex Bool
+checkKeySize key = do
+	g <- Annex.gitRepo
+	let file = gitAnnexLocation g key
+	present <- liftIO $ doesFileExist file
+	case (present, Types.Key.keySize key) of
+		(_, Nothing) -> return True
+		(False, _) -> return True
+		(True, Just size) -> do
+			stat <- liftIO $ getFileStatus file
+			let size' = fromIntegral (fileSize stat)
+			if size == size'
+				then return True
+				else do
+					dest <- moveBad key
+					warning $ "Bad file size (" ++
+						compareSizes storageUnits True size size' ++ 
+						"); moved to " ++ dest
+					return False
+
+
+checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyNumCopies key file numcopies = do
+	needed <- getNumCopies numcopies
+	locations <- keyLocations key
+	untrusted <- trustGet UnTrusted
+	let untrustedlocations = intersect untrusted locations
+	let safelocations = filter (`notElem` untrusted) locations
+	let present = length safelocations
+	if present < needed
+		then do
+			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
+			warning $ missingNote (filename file key) present needed ppuuids
+			return False
+		else return True
+	where
+		filename Nothing k = show k
+		filename (Just f) _ = f
+
+missingNote :: String -> Int -> Int -> String -> String
+missingNote file 0 _ [] = 
+		"** No known copies exist of " ++ file
+missingNote file 0 _ untrusted =
+		"Only these untrusted locations may have copies of " ++ file ++
+		"\n" ++ untrusted ++
+		"Back it up to trusted locations with git-annex copy."
+missingNote file present needed [] =
+		"Only " ++ show present ++ " of " ++ show needed ++ 
+		" trustworthy copies exist of " ++ file ++
+		"\nBack it up with git-annex copy."
+missingNote file present needed untrusted = 
+		missingNote file present needed [] ++
+		"\nThe following untrusted locations may also have copies: " ++
+		"\n" ++ untrusted
diff --git a/Command/Get.hs b/Command/Get.hs
index 50dc009feb..cc780cb6a3 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -8,7 +8,6 @@
 module Command.Get where
 
 import Command
-import qualified Backend
 import qualified Annex
 import qualified Remote
 import Types
@@ -24,7 +23,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 start :: CommandStartString
-start file = isAnnexed file $ \(key, backend) -> do
+start file = isAnnexed file $ \(key, _) -> do
 	inannex <- inAnnex key
 	if inannex
 		then stop
@@ -32,14 +31,52 @@ start file = isAnnexed file $ \(key, backend) -> do
 			showStart "get" file
 			from <- Annex.getState Annex.fromremote
 			case from of
-				Nothing -> next $ perform key backend
+				Nothing -> next $ perform key
 				Just name -> do
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
 
-perform :: Key -> Backend Annex -> CommandPerform
-perform key backend = do
-	ok <- getViaTmp key (Backend.retrieveKeyFile backend key)
+perform :: Key -> CommandPerform
+perform key = do
+	ok <- getViaTmp key (getKeyFile key)
 	if ok
 		then next $ return True -- no cleanup needed
 		else stop
+
+{- Try to find a copy of the file in one of the remotes,
+ - and copy it to here. -}
+getKeyFile :: Key -> FilePath -> Annex Bool
+getKeyFile key file = do
+	remotes <- Remote.keyPossibilities key
+	if null remotes
+		then do
+			showNote "not available"
+			Remote.showLocations key []
+			return False
+		else trycopy remotes remotes
+	where
+		trycopy full [] = do
+			Remote.showTriedRemotes full
+			Remote.showLocations key []
+			return False
+		trycopy full (r:rs) = do
+			probablythere <- probablyPresent r
+			if probablythere
+				then docopy r (trycopy full rs)
+				else trycopy full rs
+		-- This check is to avoid an ugly message if a remote is a
+		-- drive that is not mounted.
+		probablyPresent r =
+			if Remote.hasKeyCheap r
+				then do
+					res <- Remote.hasKey r key
+					case res of
+						Right b -> return b
+						Left _ -> return False
+				else return True
+		docopy r continue = do
+			showNote $ "from " ++ Remote.name r ++ "..."
+			copied <- Remote.retrieveKeyFile r key file
+			if copied
+				then return True
+				else continue
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 09ff6df7da..495bf9fb63 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -15,6 +15,7 @@ import System.FilePath
 import Command
 import qualified Annex
 import qualified Backend
+import qualified Types.Key
 import Locations
 import Types
 import Content
@@ -32,18 +33,20 @@ start :: CommandStartBackendFile
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
-	upgradable <- Backend.upgradableKey oldbackend key
-	if (newbackend /= oldbackend || upgradable) && exists
+	if (newbackend /= oldbackend || upgradableKey key) && exists
 		then do
 			showStart "migrate" file
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = do
-			backends <- Backend.list
-			return $ head backends
+		choosebackend Nothing = return . head =<< Backend.orderedList
 		choosebackend (Just backend) = return backend
 
+{- Checks if a key is upgradable to a newer representation. -}
+{- Ideally, all keys have file size metadata. Old keys may not. -}
+upgradableKey :: Key -> Bool
+upgradableKey key = Types.Key.keySize key == Nothing
+
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
 	g <- Annex.gitRepo
@@ -55,9 +58,9 @@ perform file oldkey newbackend = do
 	let src = gitAnnexLocation g oldkey
 	let tmpfile = gitAnnexTmpDir g  takeFileName file
 	liftIO $ createLink src tmpfile
-	stored <- Backend.storeFileKey tmpfile $ Just newbackend
+	k <- Backend.genKey tmpfile $ Just newbackend
 	liftIO $ cleantmp tmpfile
-	case stored of
+	case k of
 		Nothing -> stop
 		Just (newkey, _) -> do
 			ok <- getViaTmpUnchecked newkey $ \t -> do
diff --git a/Command/Status.hs b/Command/Status.hs
index 53589030b3..2448f65a40 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -25,6 +25,7 @@ import DataUnits
 import Content
 import Types.Key
 import Locations
+import Backend
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -95,9 +96,8 @@ showStat s = calc =<< s
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" $
-	lift (Annex.getState Annex.supportedBackends) >>=
-		return . unwords . (map B.name)
+supported_backends = stat "supported backends" $ 
+	return $ unwords $ map B.name Backend.list
 
 supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index f0c1b27c6c..f22503ee06 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -13,10 +13,10 @@ import System.Directory
 import System.Posix.Files
 
 import Command
+import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
 import Utility
-import qualified Backend
 import LocationLog
 import Types
 import Content
@@ -33,7 +33,7 @@ seek = [withFilesInGit start]
 
 {- The unannex subcommand undoes an add. -}
 start :: CommandStartString
-start file = isAnnexed file $ \(key, backend) -> do
+start file = isAnnexed file $ \(key, _) -> do
 	ishere <- inAnnex key
 	if ishere
 		then do
@@ -46,13 +46,12 @@ start file = isAnnexed file $ \(key, backend) -> do
 				Annex.changeState $ \s -> s { Annex.force = True }
 
 			showStart "unannex" file
-			next $ perform file key backend
+			next $ perform file key
 		else stop
 
-perform :: FilePath -> Key -> Backend Annex -> CommandPerform
-perform file key backend = do
-	-- force backend to always remove
-	ok <- Backend.removeKey backend key (Just 0)
+perform :: FilePath -> Key -> CommandPerform
+perform file key = do
+	ok <- Command.Drop.dropKey key (Just 0) -- always remove
 	if ok
 		then next $ cleanup file key
 		else stop
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index ca8b62502a..8a897c3657 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -12,7 +12,6 @@ import System.Directory hiding (copyFile)
 
 import Command
 import qualified Annex
-import qualified Backend
 import Types
 import Messages
 import Locations
@@ -38,7 +37,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
-	unlessM (Backend.hasKey key) $ error "content not present"
+	unlessM (inAnnex key) $ error "content not present"
 	
 	checkDiskSpace key
 
diff --git a/Config.hs b/Config.hs
index 1016845e75..9cbf2d52fd 100644
--- a/Config.hs
+++ b/Config.hs
@@ -86,3 +86,16 @@ remoteNotIgnored r = do
 		match a = do
 			n <- Annex.getState a
 			return $ n == Git.repoRemoteName r
+
+{- If a value is specified, it is used; otherwise the default is looked up
+ - in git config. forcenumcopies overrides everything. -}
+getNumCopies :: Maybe Int -> Annex Int
+getNumCopies v = 
+	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
+	where
+		use (Just n) = return n
+		use Nothing = do
+			g <- Annex.gitRepo
+			return $ read $ Git.configGet g config "1"
+		config = "annex.numcopies"
+
diff --git a/LocationLog.hs b/LocationLog.hs
index eb48b7916a..28b423e2fa 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -19,7 +19,7 @@ module LocationLog (
 	keyLocations,
 	loggedKeys,
 	logFile,
-	logFileKey	
+	logFileKey
 ) where
 
 import System.FilePath
diff --git a/Remote.hs b/Remote.hs
index 28c2e39cdd..623b85733c 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -14,10 +14,10 @@ module Remote (
 	removeKey,
 	hasKey,
 	hasKeyCheap,
+
 	keyPossibilities,
 	keyPossibilitiesTrusted,
 	forceTrust,
-
 	remoteTypes,
 	genList,
 	byName,
@@ -25,6 +25,8 @@ module Remote (
 	remotesWithUUID,
 	remotesWithoutUUID,
 	prettyPrintUUIDs,
+	showTriedRemotes,
+	showLocations,
 
 	remoteLog,
 	readRemoteLog,
@@ -40,6 +42,7 @@ import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
+import Data.String.Utils
 
 import qualified Branch
 import Types
@@ -49,6 +52,7 @@ import qualified Annex
 import Config
 import Trust
 import LocationLog
+import Messages
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -181,9 +185,34 @@ keyPossibilities' withtrusted key = do
 
 	return (sort validremotes, validtrusteduuids)
 
+{- Displays known locations of a key. -}
+showLocations :: Key -> [UUID] -> Annex ()
+showLocations key exclude = do
+	g <- Annex.gitRepo
+	u <- getUUID g
+	uuids <- keyLocations key
+	untrusteduuids <- trustGet UnTrusted
+	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
+	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
+	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
+	showLongNote $ message ppuuidswanted ppuuidsskipped
+	where
+		filteruuids l x = filter (`notElem` x) l
+		message [] [] = "No other repository is known to contain the file."
+		message rs [] = "Try making some of these repositories available:\n" ++ rs
+		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
+		message rs us = message rs [] ++ message [] us
+
+showTriedRemotes :: [Remote Annex] -> Annex ()
+showTriedRemotes [] = return ()	
+showTriedRemotes remotes =
+	showLongNote $ "Unable to access these remotes: " ++
+		(join ", " $ map name remotes)
+
 forceTrust :: TrustLevel -> String -> Annex ()
 forceTrust level remotename = do
-	r <- Remote.nameToUUID remotename
+	r <- nameToUUID remotename
 	Annex.changeState $ \s ->
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 471417e345..b4006d7fd1 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -112,7 +112,7 @@ inAnnex r key = if Git.repoIsUrl r
 		checklocal = do
 			-- run a local check inexpensively,
 			-- by making an Annex monad using the remote
-			a <- Annex.new r []
+			a <- Annex.new r
 			Annex.eval a (Content.inAnnex key)
 		checkremote = do
 			showNote ("checking " ++ Git.repoDescribe r ++ "...")
@@ -142,7 +142,7 @@ copyToRemote r key
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ do
-			a <- Annex.new r []
+			a <- Annex.new r
 			Annex.eval a $ do
 				ok <- Content.getViaTmp key $
 					rsyncOrCopyFile r keysrc
diff --git a/Types/Backend.hs b/Types/Backend.hs
index 8100eaf285..f86d0845cd 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -16,22 +16,8 @@ data Backend a = Backend {
 	name :: String,
 	-- converts a filename to a key
 	getKey :: FilePath -> a (Maybe Key),
-	-- stores a file's contents to a key
-	storeFileKey :: FilePath -> Key -> a Bool,
-	-- retrieves a key's contents to a file
-	retrieveKeyFile :: Key -> FilePath -> a Bool,
-	-- removes a key, optionally checking that enough copies are stored
-	-- elsewhere
-	removeKey :: Key -> Maybe Int -> a Bool,
-	-- checks if a backend is storing the content of a key
-	hasKey :: Key -> a Bool,
 	-- called during fsck to check a key
-	-- (second parameter may be the filename associated with it)
-	-- (third parameter may be the number of copies that there should
-	-- be of the key)
-	fsckKey :: Key -> Maybe FilePath -> Maybe Int -> a Bool,
-	-- Is a newer repesentation possible for a key?
-	upgradableKey :: Key -> a Bool
+	fsckKey :: Key -> a Bool
 }
 
 instance Show (Backend a) where
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 39b8e47c5f..c0bbeebaf4 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -191,17 +191,16 @@ logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
-	bs <- Annex.getState Annex.supportedBackends
 	tl <- liftIO $ try getsymlink
 	case tl of
 		Left _ -> return Nothing
-		Right l -> makekey bs l
+		Right l -> makekey l
 	where
 		getsymlink = do
 			l <- readSymbolicLink file
 			return $ takeFileName l
-		makekey bs l = do
-			case maybeLookupBackendName bs bname of
+		makekey l = do
+			case maybeLookupBackendName bname of
 				Nothing -> do
 					unless (null kname || null bname ||
 					        not (isLinkToAnnex l)) $
diff --git a/test.hs b/test.hs
index 44a792f14c..76ffe40479 100644
--- a/test.hs
+++ b/test.hs
@@ -25,7 +25,6 @@ import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
 import qualified Annex
-import qualified BackendList
 import qualified Backend
 import qualified Git
 import qualified Locations
@@ -483,7 +482,7 @@ annexeval :: Types.Annex a -> IO a
 annexeval a = do
 	g <- Git.repoFromCwd
 	g' <- Git.configRead g
-	s <- Annex.new g' BackendList.allBackends
+	s <- Annex.new g'
 	Annex.eval s a
 
 innewrepo :: Assertion -> Assertion
@@ -684,4 +683,4 @@ backendWORM :: Types.Backend Types.Annex
 backendWORM = backend_ "WORM"
 
 backend_ :: String -> Types.Backend Types.Annex
-backend_ name = Backend.lookupBackendName BackendList.allBackends name
+backend_ name = Backend.lookupBackendName name

From 6040d8aed17de582f5d5c179040e29c599315e31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:16:57 -0400
Subject: [PATCH 2026/8313] factor out RemoteLog

---
 Command/InitRemote.hs |  9 ++--
 Remote.hs             | 85 +------------------------------------
 RemoteLog.hs          | 97 +++++++++++++++++++++++++++++++++++++++++++
 test.hs               |  3 +-
 4 files changed, 106 insertions(+), 88 deletions(-)
 create mode 100644 RemoteLog.hs

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index a3054630c3..15962ad991 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -15,6 +15,7 @@ import Data.String.Utils
 
 import Command
 import qualified Remote
+import qualified RemoteLog
 import qualified Types.Remote as R
 import Types
 import UUID
@@ -42,7 +43,7 @@ start ws = do
 
 	where
 		name = head ws
-		config = Remote.keyValToConfig $ tail ws
+		config = RemoteLog.keyValToConfig $ tail ws
 		needname = do
 			let err s = error $ "Specify a name for the remote. " ++ s
 			names <- remoteNames
@@ -58,13 +59,13 @@ perform t u c = do
 
 cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
-	Remote.configSet u c
+	RemoteLog.configSet u c
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
 findByName :: String -> Annex (UUID, R.RemoteConfig)
 findByName name = do
-	m <- Remote.readRemoteLog
+	m <- RemoteLog.readRemoteLog
 	maybe generate return $ findByName' name m
 	where
 		generate = do
@@ -83,7 +84,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 
 remoteNames :: Annex [String]
 remoteNames = do
-	m <- Remote.readRemoteLog
+	m <- RemoteLog.readRemoteLog
 	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
 
 {- find the specified remote type -}
diff --git a/Remote.hs b/Remote.hs
index 623b85733c..a86e1022cf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -17,7 +17,6 @@ module Remote (
 
 	keyPossibilities,
 	keyPossibilitiesTrusted,
-	forceTrust,
 	remoteTypes,
 	genList,
 	byName,
@@ -27,24 +26,14 @@ module Remote (
 	prettyPrintUUIDs,
 	showTriedRemotes,
 	showLocations,
-
-	remoteLog,
-	readRemoteLog,
-	configSet,
-	keyValToConfig,
-	configToKeyVal,
-	
-	prop_idempotent_configEscape
+	forceTrust
 ) where
 
 import Control.Monad (filterM, liftM2)
 import Data.List
 import qualified Data.Map as M
-import Data.Maybe
-import Data.Char
 import Data.String.Utils
 
-import qualified Branch
 import Types
 import Types.Remote
 import UUID
@@ -53,6 +42,7 @@ import Config
 import Trust
 import LocationLog
 import Messages
+import RemoteLog
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -215,74 +205,3 @@ forceTrust level remotename = do
 	r <- nameToUUID remotename
 	Annex.changeState $ \s ->
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
-
-{- Filename of remote.log. -}
-remoteLog :: FilePath
-remoteLog = "remote.log"
-
-{- Adds or updates a remote's config in the log. -}
-configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = do
-	m <- readRemoteLog
-	Branch.change remoteLog $ unlines $ sort $
-		map toline $ M.toList $ M.insert u c m
-	where
-		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
-
-{- Map of remotes by uuid containing key/value config maps. -}
-readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
-
-remoteLogParse :: String -> M.Map UUID RemoteConfig
-remoteLogParse s =
-	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
-	where
-		parseline l
-			| length w > 2 = Just (u, c)
-			| otherwise = Nothing
-			where
-				w = words l
-				u = w !! 0
-				c = keyValToConfig $ tail w
-
-{- Given Strings like "key=value", generates a RemoteConfig. -}
-keyValToConfig :: [String] -> RemoteConfig
-keyValToConfig ws = M.fromList $ map (/=/) ws
-	where
-		(/=/) s = (k, v)
-			where
-				k = takeWhile (/= '=') s
-				v = configUnEscape $ drop (1 + length k) s
-
-configToKeyVal :: M.Map String String -> [String]
-configToKeyVal m = map toword $ sort $ M.toList m
-	where
-		toword (k, v) = k ++ "=" ++ configEscape v
-
-configEscape :: String -> String
-configEscape = (>>= escape)
-	where
-		escape c
-			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
-			| otherwise = [c]
-
-configUnEscape :: String -> String
-configUnEscape = unescape
-	where
-		unescape [] = []
-		unescape (c:rest)
-			| c == '&' = entity rest
-			| otherwise = c : unescape rest
-		entity s = if ok
-				then chr (read num) : unescape rest
-				else '&' : unescape s
-			where
-				num = takeWhile isNumber s
-				r = drop (length num) s
-				rest = drop 1 r
-				ok = not (null num) && 
-					not (null r) && r !! 0 == ';'
-
-{- for quickcheck -}
-prop_idempotent_configEscape :: String -> Bool
-prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
diff --git a/RemoteLog.hs b/RemoteLog.hs
new file mode 100644
index 0000000000..c2065db9da
--- /dev/null
+++ b/RemoteLog.hs
@@ -0,0 +1,97 @@
+{- git-annex remote log
+ - 
+ - Copyright 2011 Joey Hess 
+ - 
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module RemoteLog (
+	remoteLog,
+	readRemoteLog,
+	configSet,
+	keyValToConfig,
+	configToKeyVal,
+
+	prop_idempotent_configEscape
+) where
+
+import Data.List
+import qualified Data.Map as M
+import Data.Maybe
+import Data.Char
+
+import qualified Branch
+import Types
+import Types.Remote
+import UUID
+
+{- Filename of remote.log. -}
+remoteLog :: FilePath
+remoteLog = "remote.log"
+
+{- Adds or updates a remote's config in the log. -}
+configSet :: UUID -> RemoteConfig -> Annex ()
+configSet u c = do
+	m <- readRemoteLog
+	Branch.change remoteLog $ unlines $ sort $
+		map toline $ M.toList $ M.insert u c m
+	where
+		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
+
+{- Map of remotes by uuid containing key/value config maps. -}
+readRemoteLog :: Annex (M.Map UUID RemoteConfig)
+readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
+
+remoteLogParse :: String -> M.Map UUID RemoteConfig
+remoteLogParse s =
+	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
+	where
+		parseline l
+			| length w > 2 = Just (u, c)
+			| otherwise = Nothing
+			where
+				w = words l
+				u = w !! 0
+				c = keyValToConfig $ tail w
+
+{- Given Strings like "key=value", generates a RemoteConfig. -}
+keyValToConfig :: [String] -> RemoteConfig
+keyValToConfig ws = M.fromList $ map (/=/) ws
+	where
+		(/=/) s = (k, v)
+			where
+				k = takeWhile (/= '=') s
+				v = configUnEscape $ drop (1 + length k) s
+
+configToKeyVal :: M.Map String String -> [String]
+configToKeyVal m = map toword $ sort $ M.toList m
+	where
+		toword (k, v) = k ++ "=" ++ configEscape v
+
+configEscape :: String -> String
+configEscape = (>>= escape)
+	where
+		escape c
+			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"
+			| otherwise = [c]
+
+configUnEscape :: String -> String
+configUnEscape = unescape
+	where
+		unescape [] = []
+		unescape (c:rest)
+			| c == '&' = entity rest
+			| otherwise = c : unescape rest
+		entity s = if ok
+				then chr (read num) : unescape rest
+				else '&' : unescape s
+			where
+				num = takeWhile isNumber s
+				r = drop (length num) s
+				rest = drop 1 r
+				ok = not (null num) && 
+					not (null r) && r !! 0 == ';'
+
+{- for quickcheck -}
+prop_idempotent_configEscape :: String -> Bool
+prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
diff --git a/test.hs b/test.hs
index 76ffe40479..9dad37e0c0 100644
--- a/test.hs
+++ b/test.hs
@@ -36,6 +36,7 @@ import qualified LocationLog
 import qualified UUID
 import qualified Trust
 import qualified Remote
+import qualified RemoteLog
 import qualified Content
 import qualified Command.DropUnused
 import qualified Types.Key
@@ -73,7 +74,7 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
-	, qctest "prop_idempotent_configEscape" Remote.prop_idempotent_configEscape
+	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
 	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
 	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane

From c98b5cf36e785cdf2c971eaf9b0329db06b68ef8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:24:10 -0400
Subject: [PATCH 2027/8313] rename

---
 Command/RecvKey.hs                   | 2 +-
 Command/SendKey.hs                   | 2 +-
 Command/Unlock.hs                    | 2 +-
 Crypto.hs                            | 2 +-
 Remote/Directory.hs                  | 2 +-
 Remote/Git.hs                        | 4 ++--
 Remote/Rsync.hs                      | 2 +-
 Remote/S3real.hs                     | 2 +-
 Base64.hs => Utility/Base64.hs       | 2 +-
 CopyFile.hs => Utility/CopyFile.hs   | 2 +-
 RsyncFile.hs => Utility/RsyncFile.hs | 2 +-
 11 files changed, 12 insertions(+), 12 deletions(-)
 rename Base64.hs => Utility/Base64.hs (88%)
 rename CopyFile.hs => Utility/CopyFile.hs (94%)
 rename RsyncFile.hs => Utility/RsyncFile.hs (97%)

diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index b49116de45..e2f7c74abb 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Command
 import CmdLine
 import Content
 import Utility
-import RsyncFile
+import Utility.RsyncFile
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index c2f793f8fe..02fedb349e 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -15,7 +15,7 @@ import qualified Annex
 import Command
 import Content
 import Utility
-import RsyncFile
+import Utility.RsyncFile
 import Messages
 
 command :: [Command]
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 8a897c3657..d189545f5d 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -16,7 +16,7 @@ import Types
 import Messages
 import Locations
 import Content
-import CopyFile
+import Utility.CopyFile
 import Utility
 
 command :: [Command]
diff --git a/Crypto.hs b/Crypto.hs
index e84e397f2e..485fb6e931 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -46,7 +46,7 @@ import Types
 import Types.Key
 import Types.Remote
 import Utility
-import Base64
+import Utility.Base64
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 991ccbe481..05d42136f1 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -22,7 +22,7 @@ import qualified Git
 import qualified Annex
 import UUID
 import Locations
-import CopyFile
+import Utility.CopyFile
 import Config
 import Content
 import Utility
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b4006d7fd1..4a8f8ee928 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -22,8 +22,8 @@ import UUID
 import Utility
 import qualified Content
 import Messages
-import CopyFile
-import RsyncFile
+import Utility.CopyFile
+import Utility.RsyncFile
 import Ssh
 import Config
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index aa2507fff4..80e194fed1 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -29,7 +29,7 @@ import Remote.Special
 import Remote.Encryptable
 import Crypto
 import Messages
-import RsyncFile
+import Utility.RsyncFile
 
 type RsyncUrl = String
 
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 829c58ad06..52d1ed1be1 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -37,7 +37,7 @@ import Remote.Special
 import Remote.Encryptable
 import Crypto
 import Content
-import Base64
+import Utility.Base64
 
 remote :: RemoteType Annex
 remote = RemoteType {
diff --git a/Base64.hs b/Utility/Base64.hs
similarity index 88%
rename from Base64.hs
rename to Utility/Base64.hs
index 153049751d..dd739fd4fb 100644
--- a/Base64.hs
+++ b/Utility/Base64.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Base64 (toB64, fromB64) where
+module Utility.Base64 (toB64, fromB64) where
 
 import Codec.Binary.Base64
 import Data.Bits.Utils
diff --git a/CopyFile.hs b/Utility/CopyFile.hs
similarity index 94%
rename from CopyFile.hs
rename to Utility/CopyFile.hs
index b08ede3c88..5ee4a91df7 100644
--- a/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module CopyFile (copyFile) where
+module Utility.CopyFile (copyFile) where
 
 import System.Directory (doesFileExist, removeFile)
 
diff --git a/RsyncFile.hs b/Utility/RsyncFile.hs
similarity index 97%
rename from RsyncFile.hs
rename to Utility/RsyncFile.hs
index 48d927fcf2..c68909d2dc 100644
--- a/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RsyncFile where
+module Utility.RsyncFile where
 
 import Data.String.Utils
 

From cab4ac247ca990a03537f7611b299efca8edaffe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:36:43 -0400
Subject: [PATCH 2028/8313] rename

---
 Command/Fsck.hs                      | 2 +-
 Command/Map.hs                       | 4 ++--
 Command/Status.hs                    | 2 +-
 Content.hs                           | 2 +-
 Remote/Bup.hs                        | 2 +-
 Remote/Git.hs                        | 2 +-
 Ssh.hs => Remote/Ssh.hs              | 4 ++--
 DataUnits.hs => Utility/DataUnits.hs | 2 +-
 Dot.hs => Utility/Dot.hs             | 2 +-
 9 files changed, 11 insertions(+), 11 deletions(-)
 rename Ssh.hs => Remote/Ssh.hs (96%)
 rename DataUnits.hs => Utility/DataUnits.hs (99%)
 rename Dot.hs => Utility/Dot.hs (97%)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 446d25a449..ec3f1d8e7d 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -26,7 +26,7 @@ import Content
 import LocationLog
 import Locations
 import Trust
-import DataUnits
+import Utility.DataUnits
 import Config
 
 command :: [Command]
diff --git a/Command/Map.hs b/Command/Map.hs
index 940db54c89..0391ab8e8f 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -21,8 +21,8 @@ import Types
 import Utility
 import UUID
 import Trust
-import Ssh
-import qualified Dot
+import Remote.Ssh
+import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
diff --git a/Command/Status.hs b/Command/Status.hs
index 2448f65a40..1ec4782362 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -21,7 +21,7 @@ import qualified Command.Unused
 import qualified Git
 import Command
 import Types
-import DataUnits
+import Utility.DataUnits
 import Content
 import Types.Key
 import Locations
diff --git a/Content.hs b/Content.hs
index a2f38ddc96..94f8b8c2ac 100644
--- a/Content.hs
+++ b/Content.hs
@@ -43,7 +43,7 @@ import qualified Branch
 import Utility
 import StatFS
 import Types.Key
-import DataUnits
+import Utility.DataUnits
 import Config
 
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 11c0ec4daf..5a44397f0d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -28,7 +28,7 @@ import Locations
 import Config
 import Utility
 import Messages
-import Ssh
+import Remote.Ssh
 import Remote.Special
 import Remote.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 4a8f8ee928..fb85123825 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -24,7 +24,7 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Ssh
+import Remote.Ssh
 import Config
 
 remote :: RemoteType Annex
diff --git a/Ssh.hs b/Remote/Ssh.hs
similarity index 96%
rename from Ssh.hs
rename to Remote/Ssh.hs
index 21e72c0839..0d4842a1ab 100644
--- a/Ssh.hs
+++ b/Remote/Ssh.hs
@@ -1,11 +1,11 @@
-{- git-annex repository access with ssh
+{- git-annex remote access with ssh
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Ssh where
+module Remote.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/DataUnits.hs b/Utility/DataUnits.hs
similarity index 99%
rename from DataUnits.hs
rename to Utility/DataUnits.hs
index c81c6e42e5..7af2eadafb 100644
--- a/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module DataUnits (
+module Utility.DataUnits (
 	dataUnits,
 	storageUnits,
 	memoryUnits,
diff --git a/Dot.hs b/Utility/Dot.hs
similarity index 97%
rename from Dot.hs
rename to Utility/Dot.hs
index deba10201e..8696849963 100644
--- a/Dot.hs
+++ b/Utility/Dot.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Dot where -- import qualified
+module Utility.Dot where -- import qualified
 
 {- generates a graph description from a list of lines -}
 graph :: [String] -> String

From 497b1e60926d822f8acdeb6f3df80f597e81086e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jul 2011 20:53:58 -0400
Subject: [PATCH 2029/8313] Fix sign bug in disk free space checking.

Giulio Eulisse reported that on OSX, bad free space numbers were being
shown. It thought he had negative free space.

While the documentation is not clear, especially across OS's, it seems
likely that statfs uses unsigned long. It doesn't make sense for any
numbers to be negative.
---
 StatFS.hsc       | 2 +-
 debian/changelog | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/StatFS.hsc b/StatFS.hsc
index 0828f1378c..feb82aa9af 100644
--- a/StatFS.hsc
+++ b/StatFS.hsc
@@ -96,7 +96,7 @@ foreign import ccall unsafe "sys/vfs.h statfs64"
   c_statfs :: CString -> Ptr CStatfs -> IO CInt
 #endif
 
-toI :: CLong -> Integer
+toI :: CULong -> Integer
 toI = toInteger
 
 getFileSystemStats :: String -> IO (Maybe FileSystemStats)
diff --git a/debian/changelog b/debian/changelog
index 773c91b7eb..8d1a396cb5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110706) UNRELEASED; urgency=low
+
+  * Fix sign bug in disk free space checking.
+
+ -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
+
 git-annex (3.20110705) unstable; urgency=low
 
   * uninit: Delete the git-annex branch and .git/annex/

From 10e72acb0100820e30dab4398ba90b6232b56fcd Mon Sep 17 00:00:00 2001
From: "https://lithitux.org/openidserver/users/pavel" 
Date: Wed, 6 Jul 2011 08:14:26 +0000
Subject: [PATCH 2030/8313] Added a comment

---
 ...comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment

diff --git a/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment
new file mode 100644
index 0000000000..0852db0795
--- /dev/null
+++ b/doc/bugs/problem_with_upgrade_v2_-__62___v3/comment_6_f360f0006bc9115bc5a3e2eb9fe58abd._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://lithitux.org/openidserver/users/pavel"
+ nickname="pavel"
+ subject="comment 6"
+ date="2011-07-06T08:14:26Z"
+ content="""
+Ah, great, thanks very much for the quick fix!
+
+Yes, when I mentioned three defunct git processes, there were three processes shown as \"git [defunct]\", plus the three git processes I listed, plus two \"git-annex\" processes. Upon cancel/resume, there were no defunct git processes when I checked, but by the time I found the bug report on the forum and commented I'd already successfully upgraded by annex (by repeatedly attaching strace) and couldn't really easily get at either additional 'ps' info or a fuller strace than what I posted (that was just the log from one of the attach/detach cycles), so it's a relief you managed to pinpoint the problem.
+"""]]

From b5733069db61dc9e03bc68a9e3aa2a8946cc3dec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jul 2011 16:06:10 -0400
Subject: [PATCH 2031/8313] tweak

---
 Remote.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index a86e1022cf..de1c7f38db 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,15 +15,15 @@ module Remote (
 	hasKey,
 	hasKeyCheap,
 
-	keyPossibilities,
-	keyPossibilitiesTrusted,
 	remoteTypes,
 	genList,
 	byName,
-	nameToUUID,
+	prettyPrintUUIDs,
 	remotesWithUUID,
 	remotesWithoutUUID,
-	prettyPrintUUIDs,
+	keyPossibilities,
+	keyPossibilitiesTrusted,
+	nameToUUID,
 	showTriedRemotes,
 	showLocations,
 	forceTrust

From a6c2bea91f17b42899b49a05ae6c2ce71c944cbc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Wed, 6 Jul 2011 20:39:11 +0000
Subject: [PATCH 2032/8313]

---
 ...unlock__34___files_without_copying_the_file_data__63__.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
new file mode 100644
index 0000000000..21c80819a4
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
@@ -0,0 +1,3 @@
+I have a DLink Boxee media player and it can not play content from symbolic links, it needs to access regular media files. Unfortunately unlocking/locking is quite slow for such a large amount of data due to the required data copying, but it should not even be needed since I do not need write access to any file to watch the movie or to play the song.
+
+Is it currently possible or would it be possible to add a commands like "unlock" which would not copy the file data but simply move files out from the data store into the tree while still keeping the files read only? A corresponding "lock" command would also be needed to restore the normal symbolic link tree structure.

From c5296dee1fa0dd5252b496b7974808538edd4564 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Wed, 6 Jul 2011 21:29:37 +0000
Subject: [PATCH 2033/8313]

---
 ...unlock__34___files_without_copying_the_file_data__63__.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
index 21c80819a4..1a7930fec4 100644
--- a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__.mdwn
@@ -1,3 +1,6 @@
 I have a DLink Boxee media player and it can not play content from symbolic links, it needs to access regular media files. Unfortunately unlocking/locking is quite slow for such a large amount of data due to the required data copying, but it should not even be needed since I do not need write access to any file to watch the movie or to play the song.
 
 Is it currently possible or would it be possible to add a commands like "unlock" which would not copy the file data but simply move files out from the data store into the tree while still keeping the files read only? A corresponding "lock" command would also be needed to restore the normal symbolic link tree structure.
+
+Update:
+I tried the rsync special remote http://git-annex.branchable.com/special_remotes/rsync/ and it works but the file structure created reflects the data store not the view given by the symbolic links.

From 8da1dd033684b854b595d39f0b2e86fbfef4d265 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Thu, 7 Jul 2011 11:08:35 +0000
Subject: [PATCH 2034/8313]

---
 doc/bugs/making_annex-merge_try_a_fast-forward.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/making_annex-merge_try_a_fast-forward.mdwn

diff --git a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
new file mode 100644
index 0000000000..532e771bd1
--- /dev/null
+++ b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
@@ -0,0 +1,3 @@
+While merging the git-annex branch, annex-merge does not end up in a fast-forward even when it would be possible.
+But as sometimes annex-merge takes time, it would probably be worth it
+(but maybe I miss something with my workflow...).

From f854d5ae7a17bc2ee07f9581ea6e659cd1afc341 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 7 Jul 2011 15:27:28 +0000
Subject: [PATCH 2035/8313] Added a comment

---
 ...comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment
new file mode 100644
index 0000000000..3ab518714d
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_1_1cf4ab29dfa2cff59b86305fc0018251._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T15:27:28Z"
+ content="""
+The rsync or directory special remotes would work if the media player uses metadata in the files, rather than directory locations.
+
+Beyond that there is the [[todo/smudge]] idea, which is hoped to be supported sometime.
+"""]]

From 944cc2fde8c1ad37827adfdbd5e83eb18fe4f9b0 Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Thu, 7 Jul 2011 18:49:49 +0000
Subject: [PATCH 2036/8313]

---
 ...annex_add_eats_files_when_filename_is_too_long.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
new file mode 100644
index 0000000000..6baf5ce815
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
@@ -0,0 +1,10 @@
+Recently I ran into the following situation under Ubuntu with an encrypted home directory (which shortens the length that filenames can be):
+
+$ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+git-annex: 1 failed
+
+The file seems to be completely gone.  It no longer exists in the current directory, or under .git/annex.
+
+I don't mind horribly that git-annex failed due to the name length limit, but it shouldn't have deleted my file in the process (fortunately the file wasn't very important, or hard to recover).

From 1ffe7f777037e08abc4b068a4547e2baa0e714ee Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Thu, 7 Jul 2011 19:44:40 +0000
Subject: [PATCH 2037/8313]

---
 ...acter___40__and_probably_others__41__.mdwn | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn

diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn
new file mode 100644
index 0000000000..28d11d6ee6
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__.mdwn
@@ -0,0 +1,32 @@
+"git annex upgrade" has lost track of some of my files.  Most of them have "&" characters.  The others contain "%" characters (I haven't tried the testcase below with "%" however).
+
+Testcase:
+
+    # (With git annex v2)
+    mkdir ~/testannex1
+    cd ~/testannex1
+    git init
+    git annex init "testannex1"
+    touch '02 - Afternoons & Coffeespoons.mp3'
+    touch 'no ampersand.mp3'
+    git annex add '02 - Afternoons & Coffeespoons.mp3'
+    git annex add 'no ampersand.mp3'
+    git commit -m added
+    git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    git annex whereis 'no ampersand.mp3'
+    # (Upgrade git-annex binary to v3 and then...)
+    git annex upgrade
+    git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    git annex whereis 'no ampersand.mp3'
+
+This produces:
+
+    12:38:40 ~/testannex1 (master)$ git annex whereis '02 - Afternoons & Coffeespoons.mp3'
+    whereis 02 - Afternoons & Coffeespoons.mp3 (0 copies) 
+    failed
+    git-annex: 1 failed
+    12:38:40 ~/testannex1 (master)$ git annex whereis 'no ampersand.mp3'
+    whereis no ampersand.mp3 (1 copy) 
+      	a7b680fc-a8d0-11e0-b0fe-4f94e86d1fb7  -- testannex1 <-- here
+    ok
+

From 33ebaf3f9ff77526b0fae2ee39d049117ffd188a Mon Sep 17 00:00:00 2001
From: ssqq 
Date: Thu, 7 Jul 2011 19:45:31 +0000
Subject: [PATCH 2038/8313]

---
 ...it_annex_add_eats_files_when_filename_is_too_long.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
index 6baf5ce815..af807b65d2 100644
--- a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
@@ -1,9 +1,9 @@
 Recently I ran into the following situation under Ubuntu with an encrypted home directory (which shortens the length that filenames can be):
 
-$ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
-add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
-git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
-git-annex: 1 failed
+    $ git annex add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+    add 687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+    git-annex: /home/lhuhn/annex/.git/annex/tmp/155_518_WORM-s426663-m1310064100--687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+    git-annex: 1 failed
 
 The file seems to be completely gone.  It no longer exists in the current directory, or under .git/annex.
 

From 0c52362359687b47b5c70ca9985e4ee6fc9409dd Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 7 Jul 2011 20:27:33 +0000
Subject: [PATCH 2039/8313] Added a comment

---
 ..._9650284913bec2a00cf551b90ab5d8ff._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment
new file mode 100644
index 0000000000..1df159181d
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_1_9650284913bec2a00cf551b90ab5d8ff._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T20:27:33Z"
+ content="""
+When I reproduce this, the file is not gone, it's been moved under .git/annex/objects. There is no way an add can delete a file, since all it does is rename it. It would be good for it to error unwind and move the file back though.
+
+
+joey@gnu:~/tmp/a>touch 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+joey@gnu:~/tmp/a>git annex add *.gif
+add 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/joey/tmp/a/.git/annex/tmp/8e2_6a4_WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+joey@gnu:~/tmp/a>touch 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+joey@gnu:~/tmp/a>git annex add *.gif
+add 663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif failed
+git-annex: /home/joey/tmp/a/.git/annex/tmp/8e2_6a4_WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif.log: openBinaryFile: invalid argument (File name too long)
+joey@gnu:~/tmp/a>find .git/annex/objects -type f
+.git/annex/objects/Mk/92/WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif/WORM-s0-m1310069979--663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966687474703a2f2f6d656469612e74756d626c722e636f6d2f74756d626c725f6c656673756557324c703171663879656b2e676966.gif
+

+"""]]

From 2fb771f135ad0a5adec0349a6270cadc518e04f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 17:04:21 -0400
Subject: [PATCH 2040/8313] Bugfix: Forgot to de-escape keys when upgrading.

Could result in bad location log data for keys that contain [&:%] in their
names. (A workaround for this problem is to run git annex fsck.)

`git annex unused --from remote` could also run into the broken code.
---
 LocationLog.hs   | 2 +-
 debian/changelog | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index 28b423e2fa..aab817f3fc 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -60,7 +60,7 @@ logFile key = hashDirLower key ++ keyFile key ++ ".log"
 {- Converts a log filename into a key. -}
 logFileKey :: FilePath -> Maybe Key
 logFileKey file
-	| end == ".log" = readKey beginning
+	| end == ".log" = fileKey beginning
 	| otherwise = Nothing
 	where
 		(beginning, end) = splitAt (length file - 4) file
diff --git a/debian/changelog b/debian/changelog
index 8d1a396cb5..ef0ec77191 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (3.20110706) UNRELEASED; urgency=low
 
   * Fix sign bug in disk free space checking.
+  * Bugfix: Forgot to de-escape keys when upgrading. Could result in
+    bad location log data for keys that contain [&:%] in their names.
+    (A workaround for this problem is to run git annex fsck.)
 
  -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
 

From dac158c7b206b2f4d82ae6b993189e8148ad5f09 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 7 Jul 2011 21:04:23 +0000
Subject: [PATCH 2041/8313] Added a comment

---
 ...mment_1_861506e40e0d04d2be98bbfe9188be89._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment

diff --git a/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment
new file mode 100644
index 0000000000..194b36ac10
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_loses_track_of_files_with___34____38____34___character___40__and_probably_others__41__/comment_1_861506e40e0d04d2be98bbfe9188be89._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-07-07T21:04:23Z"
+ content="""
+What an evil little bug. In retrospect, this probably bit my own test upgrades, but I ran `git annex fsck` everywhere and so avoided the location log breakage.
+
+I've fixed the bug, which also involved files with other punctuation in their names [&:%] when using the WORM backend. 
+
+The only way I have to recover repos that have already been upgraded is to run `git annex fsck --fast` in each clone of such a repo, which will let it rebuild the location log information. I think that is the best way to recover; ie I can't think of a way to recover that doesn't need to do everything fsck does anyway.
+"""]]

From 67dcc1f171f0bbe2b57d20fbafce9f6c9b8f781e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:29:36 -0400
Subject: [PATCH 2042/8313] add: Avoid a failure mode that resulted in the file
 seemingly being deleted (content put in the annex but no symlink present).

---
 Command/Add.hs                                                | 4 ++--
 debian/changelog                                              | 2 ++
 .../git_annex_add_eats_files_when_filename_is_too_long.mdwn   | 4 ++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 2831e1b35f..e7d16b6c07 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -51,11 +51,11 @@ perform (file, backend) = do
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	logStatus key InfoPresent
-
 	link <- calcGitLink file key
 	liftIO $ createSymbolicLink link file
 
+	logStatus key InfoPresent
+
 	-- touch the symlink to have the same mtime as the file it points to
 	s <- liftIO $ getFileStatus file
 	let mtime = modificationTime s
diff --git a/debian/changelog b/debian/changelog
index ef0ec77191..eb9037771b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20110706) UNRELEASED; urgency=low
   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
     bad location log data for keys that contain [&:%] in their names.
     (A workaround for this problem is to run git annex fsck.)
+  * add: Avoid a failure mode that resulted in the file seemingly being
+    deleted (content put in the annex but no symlink present).
 
  -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
 
diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
index af807b65d2..d17e569f17 100644
--- a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long.mdwn
@@ -8,3 +8,7 @@ Recently I ran into the following situation under Ubuntu with an encrypted home
 The file seems to be completely gone.  It no longer exists in the current directory, or under .git/annex.
 
 I don't mind horribly that git-annex failed due to the name length limit, but it shouldn't have deleted my file in the process (fortunately the file wasn't very important, or hard to recover).
+
+> [[done]], as noted it did not delete content and now it makes the symlink
+> before trying to write to the location log, avoiding that gotcha.
+> --[[Joey]] 

From 4d4f297c9620348c8c86ddd2155cf49bf38424ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:37:49 -0400
Subject: [PATCH 2043/8313] releasing version 3.20110707

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index eb9037771b..626e388371 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110706) UNRELEASED; urgency=low
+git-annex (3.20110707) unstable; urgency=low
 
   * Fix sign bug in disk free space checking.
   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
@@ -7,7 +7,7 @@ git-annex (3.20110706) UNRELEASED; urgency=low
   * add: Avoid a failure mode that resulted in the file seemingly being
     deleted (content put in the annex but no symlink present).
 
- -- Joey Hess   Tue, 05 Jul 2011 20:52:11 -0400
+ -- Joey Hess   Thu, 07 Jul 2011 19:29:39 -0400
 
 git-annex (3.20110705) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 3ea7e5fb39..eb02ae0cad 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110705
+Version: 3.20110707
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From f7be0d5077ef95fa76332c8ede5aec3cb0acf408 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:38:05 -0400
Subject: [PATCH 2044/8313] add news item for git-annex 3.20110707

---
 doc/news/version_0.20110601.mdwn | 9 ---------
 doc/news/version_3.20110707.mdwn | 8 ++++++++
 2 files changed, 8 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_0.20110601.mdwn
 create mode 100644 doc/news/version_3.20110707.mdwn

diff --git a/doc/news/version_0.20110601.mdwn b/doc/news/version_0.20110601.mdwn
deleted file mode 100644
index 59079088da..0000000000
--- a/doc/news/version_0.20110601.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 0.20110601 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Minor bugfixes and error message improvements.
-   * Massively sped up `git annex lock` by avoiding use of the uber-slow
-     `git reset`, and only running `git checkout` once, even when many files
-     are being locked.
-   * Fix locking of files with staged changes.
-   * Somewhat sped up `git commit` of modifications to unlocked files.
-   * Build fix for older ghc."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110707.mdwn b/doc/news/version_3.20110707.mdwn
new file mode 100644
index 0000000000..3f489ae5d6
--- /dev/null
+++ b/doc/news/version_3.20110707.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20110707 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix sign bug in disk free space checking.
+   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
+     bad location log data for keys that contain [&:%] in their names.
+     (A workaround for this problem is to run git annex fsck.)
+   * add: Avoid a failure mode that resulted in the file seemingly being
+     deleted (content put in the annex but no symlink present)."""]]
\ No newline at end of file

From 8c1fa1ab5f15c096e76fd2727f41b3bc042a2f3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:49:24 -0400
Subject: [PATCH 2045/8313] add a nasty workaround for a nasty cabal limitation

It croaks on long filenames.. probably >= 100 chars
100 characters was a (historial) limit on filenames in tarballs.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 786fd919e4..ccf98f6252 100644
--- a/Makefile
+++ b/Makefile
@@ -91,7 +91,7 @@ clean:
 # generate a file list there.
 sdist: clean
 	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
-	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print)!i" < git-annex.cabal.orig > git-annex.cabal
+	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print | perl -ne 'print unless length >= 100')!i" < git-annex.cabal.orig > git-annex.cabal
 	@cabal sdist
 	@mv git-annex.cabal.orig git-annex.cabal
 

From 9633deab573922c8178e714c72ea9709dfd23104 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Fri, 8 Jul 2011 00:21:32 +0000
Subject: [PATCH 2046/8313] Added a comment: this happens also when the user
 has not the permission to set the file mode

---
 .../comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment
new file mode 100644
index 0000000000..bd53627bbc
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_2_c6c8d2a1f444d85c582bc5396b08e148._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="this happens also when the user has not the permission to set the file mode"
+ date="2011-07-08T00:21:31Z"
+ content="""
+For example if the file is owned by root, I guess git-annex fails when it tries to remove write permissions (I retested with the last version of today (whose \"version\" subcommand still outputs 3.20110702)).By the way, it would be nice to have a log file created containing the list of all failures, to avoid having to scan manually all the output of a long git-annex operation.
+"""]]

From 4857f4e86156c5f119374110aff6c62a884a5d96 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Fri, 8 Jul 2011 00:45:30 +0000
Subject: [PATCH 2047/8313] Added a comment

---
 .../comment_3_5776864d78d56849001dd12e3adb9cbe._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment
new file mode 100644
index 0000000000..f9d1b5d682
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_3_5776864d78d56849001dd12e3adb9cbe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafaël"
+ subject="comment 3"
+ date="2011-07-08T00:45:30Z"
+ content="""
+comment on the output of 'git-annex version' (from my last comment): now I get the right version 3.20110707. But I checked in my console that the three commands \"git checkout 3.20110707\", \"make\" and \"./git-annex version\" gave me before 3.20110702, I don't know why...
+"""]]

From 2a108982ad95f9bba9bb1ae1c6152ef9b10be53c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 19:10:42 -0400
Subject: [PATCH 2048/8313] add monad-control to build depends

Will use this to handle exceptions in the Annex monad, yay.
---
 debian/control          | 1 +
 doc/install.mdwn        | 1 +
 doc/install/Fedora.mdwn | 1 +
 doc/install/OSX.mdwn    | 1 +
 git-annex.cabal         | 2 +-
 5 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 4347233841..d519c42b71 100644
--- a/debian/control
+++ b/debian/control
@@ -13,6 +13,7 @@ Build-Depends:
 	libghc-utf8-string-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
+	libghc-monad-control-dev,
 	ikiwiki,
 	perlmagick,
 	git | git-core,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 38963695b8..49ddd913f0 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -23,6 +23,7 @@ To build and use git-annex, you will need:
   * [utf8-string](http://hackage.haskell.org/package/utf8-string)
   * [SHA](http://hackage.haskell.org/package/SHA)
   * [dataenc](http://hackage.haskell.org/package/dataenc)
+  * [monad-control](http://hackage.haskell.org/package/monad-control)
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 7814eec940..0c1da2e6a5 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -9,6 +9,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install monad-control
 sudo cabal install HTTP
 sudo cabal install hS3
 
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index ade4fa30e8..23cb1b62e2 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,6 +7,7 @@ sudo cabal install pcre-light
 sudo cabal install quickcheck  
 sudo cabal install SHA
 sudo cabal install dataenc
+sudo cabal install monad-control
 sudo cabal install HTTP
 sudo cabal install hS3 # optional
 
diff --git a/git-annex.cabal b/git-annex.cabal
index eb02ae0cad..29ad80a58a 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5
+   base < 5, monad-control
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 2640ee820f4269ccc1b5f4cd184aaf895fcf405d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 20:59:05 -0400
Subject: [PATCH 2049/8313] cleanup

---
 LocationLog.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index aab817f3fc..fe09482b9f 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -30,7 +30,6 @@ import qualified Git
 import qualified Branch
 import UUID
 import Types
-import Types.Key
 import Locations
 import PresenceLog
 

From 40c6ba99f51875db28f3e1e8b309812c66594e32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jul 2011 21:29:31 -0400
Subject: [PATCH 2050/8313] add: Be even more robust to avoid ever leaving the
 file seemingly deleted.

A failure at any point after the file is annexed will result in an undo
that puts the original file back into place and wipes the location log.
---
 Command/Add.hs   | 44 ++++++++++++++++++++++++++++++++++----------
 debian/changelog |  6 ++++++
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index e7d16b6c07..5c7cad044e 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -9,6 +9,10 @@ module Command.Add where
 
 import Control.Monad.State (liftIO)
 import System.Posix.Files
+import System.Directory
+import Control.Exception.Control (handle)
+import Control.Exception.Base (throwIO)
+import Control.Exception.Extensible (IOException)
 
 import Command
 import qualified Annex
@@ -20,6 +24,7 @@ import Content
 import Messages
 import Utility
 import Touch
+import Locations
 
 command :: [Command]
 command = [repoCommand "add" paramPath seek "add files to annex"]
@@ -46,20 +51,39 @@ perform (file, backend) = do
 	case k of
 		Nothing -> stop
 		Just (key, _) -> do
-			moveAnnex key file
+			handle (undo file key) $ moveAnnex key file
 			next $ cleanup file key
 
+{- On error, put the file back so it doesn't seem to have vanished.
+ - This can be called before or after the symlink is in place. -}
+undo :: FilePath -> Key -> IOException -> Annex a
+undo file key e = do
+	unlessM (inAnnex key) $ rethrow -- no cleanup to do
+	liftIO $ whenM (doesFileExist file) $ do removeFile file
+	handle tryharder $ fromAnnex key file
+	logStatus key InfoMissing
+	rethrow
+	where
+		rethrow = liftIO $ throwIO e
+
+		-- fromAnnex could fail if the file ownership is weird
+		tryharder :: IOException -> Annex ()
+		tryharder _ = do
+			g <- Annex.gitRepo
+			liftIO $ renameFile (gitAnnexLocation g key) file
+
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	link <- calcGitLink file key
-	liftIO $ createSymbolicLink link file
-
-	logStatus key InfoPresent
-
-	-- touch the symlink to have the same mtime as the file it points to
-	s <- liftIO $ getFileStatus file
-	let mtime = modificationTime s
-	liftIO $ touch file (TimeSpec mtime) False
+	handle (undo file key) $ do
+		link <- calcGitLink file key
+		liftIO $ createSymbolicLink link file
+		logStatus key InfoPresent
+	
+		-- touch the symlink to have the same mtime as the
+		-- file it points to
+		s <- liftIO $ getFileStatus file
+		let mtime = modificationTime s
+		liftIO $ touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/debian/changelog b/debian/changelog
index 626e388371..80fe84256b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110708) UNRELEASED; urgency=low
+
+  * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+
+ -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
+
 git-annex (3.20110707) unstable; urgency=low
 
   * Fix sign bug in disk free space checking.

From 151b1d85c7f5cc4d52f29ac64499622ce6c3a5c8 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 8 Jul 2011 01:32:30 +0000
Subject: [PATCH 2051/8313] Added a comment

---
 .../comment_4_371ec7b4ae73280ede31edfe90b42a95._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment

diff --git a/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment
new file mode 100644
index 0000000000..1ba57c1992
--- /dev/null
+++ b/doc/bugs/git_annex_add_eats_files_when_filename_is_too_long/comment_4_371ec7b4ae73280ede31edfe90b42a95._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-07-08T01:32:30Z"
+ content="""
+Indeed, I've made it even more robust now, handling the case where the file has weird permissions too, and undoing the failed add so the file is always back at the start state. Had to add a dependency on another haskell module to allow this, so it took some time to figure out how to do it..
+
+"""]]

From a8fe35f645234d91b8259a12106ed7b0bc8b55dd Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 8 Jul 2011 03:39:22 +0000
Subject: [PATCH 2052/8313]

---
 ...e_of_massively_disconnected_operation.mdwn | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 doc/forum/example_of_massively_disconnected_operation.mdwn

diff --git a/doc/forum/example_of_massively_disconnected_operation.mdwn b/doc/forum/example_of_massively_disconnected_operation.mdwn
new file mode 100644
index 0000000000..00a5d8d6c5
--- /dev/null
+++ b/doc/forum/example_of_massively_disconnected_operation.mdwn
@@ -0,0 +1,33 @@
+I found this archival drive that had been offline since October 26th 2010. Since I released git-annex 0.02 on October 27th, this must have been made using the very first release of git-annex, ever.
+
+So, I synced it back up! :) --[[Joey]]
+
+
+commit 4151f4595fe6205d4aed653617ab23eb3335130a
+Author: Joey Hess 
+Date:   Tue Oct 26 02:18:03 2010 -0400
+
+joey> git pull
+remote: Counting objects: 428782, done.
+remote: Compressing objects: 100% (280714/280714), done.
+remote: Total 416692 (delta 150923), reused 389593 (delta 125143)
+Receiving objects: 100% (416692/416692), 44.71 MiB | 495 KiB/s, done.
+Resolving deltas: 100% (150923/150923), completed with 818 local objects.
+ * [new branch]      git-annex  -> origin/git-annex
+   1893f9c..9ebcc0e  master     -> origin/master
+Updating 1893f9c..9ebcc0e
+Checking out files: 100% (76884/76884), done.
+joey> git annex version
+git-annex version: 3.20110611
+local repository version: unknown
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+joey> git config annex.version 0
+joey> git annex upgrade
+upgrade . (v0 to v1...) (v1 to v2) (moving content...) (updating symlinks...)  (moving location logs...) (v2 to v3) (merging origin/git-annex into git-annex...)
+
+  git-annex branch created
+  Be sure to push this branch when pushing to remotes.
+ok
+


From 7da059e5570652de5f882149ff981672d3424355 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 8 Jul 2011 01:26:18 -0400
Subject: [PATCH 2053/8313] update

---
 doc/upgrades.mdwn | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/upgrades.mdwn b/doc/upgrades.mdwn
index d62e9dcc78..c4aac53482 100644
--- a/doc/upgrades.mdwn
+++ b/doc/upgrades.mdwn
@@ -88,5 +88,6 @@ Involved a reorganisation of the layout of .git/annex/. Symlinks changed.
 Handled more or less transparently, although git-annex was just 2 weeks
 old at the time, and had few users other than Joey.
 
-This upgrade is believed to still be supported, but has not been tested
-lately.
+Before doing this upgrade, set annex.version:
+
+	git config annex.version 0

From 085eeaa6529df0b14f7b6b1c9ca576caa2b18c0b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sat, 9 Jul 2011 01:49:09 +0000
Subject: [PATCH 2054/8313]

---
 ..._39__git_add__39___for_parent_relative_path.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn

diff --git a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
new file mode 100644
index 0000000000..1243bbfb20
--- /dev/null
+++ b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
@@ -0,0 +1,13 @@
+The following commands show the failure:
+
+$ mkdir d && touch d/f
+
+$ mkdir g && cd g && git annex add ../d/f 
+
+add ... ok
+
+error: Invalid path '.git/annex/objects/Jx/...
+
+...
+
+Then it seems it is enough to 'git add ../d/f' to complete the operation.

From 7919de73af81d1788867ecbcbcc17e25b348ad1d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jul 2011 13:52:53 -0400
Subject: [PATCH 2055/8313] Bugfix: Make add ../ work.

The complication of check-attr returning absolute paths that have to be
converted back to relative paths..
---
 Git.hs                                              | 13 ++++++++-----
 debian/changelog                                    |  1 +
 ..._39__git_add__39___for_parent_relative_path.mdwn |  2 ++
 test.hs                                             | 10 +++++++++-
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/Git.hs b/Git.hs
index 7005f24f55..7827f80514 100644
--- a/Git.hs
+++ b/Git.hs
@@ -530,6 +530,7 @@ checkAttr repo attr files = do
 	-- directory. Convert to absolute, and then convert the filenames
 	-- in its output back to relative.
 	cwd <- getCurrentDirectory
+	let top = workTree repo
 	let absfiles = map (absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
@@ -539,19 +540,21 @@ checkAttr repo attr files = do
                 exitSuccess
         hClose toh
 	s <- hGetContents fromh
-	return $ map (topair $ cwd++"/") $ lines s
+	return $ map (topair cwd top) $ lines s
 	where
 		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
-		topair cwd l = (relfile, value)
+		topair cwd top l = (relfile, value)
 			where 
-				relfile 
-					| startswith cwd file = drop (length cwd) file
-					| otherwise = file
+				relfile
+					| startswith cwd' file = drop (length cwd') file
+					| otherwise = relPathDirToFile top' file
 				file = decodeGitFile $ join sep $ take end bits
 				value = bits !! end
 				end = length bits - 1
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
+				cwd' = cwd ++ "/"
+				top' = top ++ "/"
 
 {- Some git commands output encoded filenames. Decode that (annoyingly
  - complex) encoding. -}
diff --git a/debian/changelog b/debian/changelog
index 80fe84256b..ec3176d12a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110708) UNRELEASED; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+  * Bugfix: Make add ../ work.
 
  -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
 
diff --git a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
index 1243bbfb20..f129abf623 100644
--- a/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
+++ b/doc/bugs/__39__annex_add__39___fails_to___39__git_add__39___for_parent_relative_path.mdwn
@@ -11,3 +11,5 @@ error: Invalid path '.git/annex/objects/Jx/...
 ...
 
 Then it seems it is enough to 'git add ../d/f' to complete the operation.
+
+> Thanks for reporting, [[fixed|done]] --[[Joey]] 
diff --git a/test.hs b/test.hs
index 9dad37e0c0..51ccc600e6 100644
--- a/test.hs
+++ b/test.hs
@@ -108,7 +108,7 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 		reponame = "test repo"
 
 test_add :: Test
-test_add = "git-annex add" ~: TestList [basic, sha1dup]
+test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 	where
 		-- this test case runs in the main repo, to set up a basic
 		-- annexed file that later tests will use
@@ -129,6 +129,14 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup]
 			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
 			annexed_present sha1annexedfile
+		subdirs = TestCase $ intmpclonerepo $ do
+			createDirectory "dir"
+			writeFile "dir/foo" $ content annexedfile
+			git_annex "add" ["-q", "dir"] @? "add of subdir failed"
+			createDirectory "dir2"
+			writeFile "dir2/foo" $ content annexedfile
+			changeWorkingDirectory "dir"
+			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
 
 test_setkey :: Test
 test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do

From c38dd9adc844688fb7ebdc8f41db16cf52de0939 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jul 2011 15:37:29 -0400
Subject: [PATCH 2056/8313] analysis

---
 ...making_annex-merge_try_a_fast-forward.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
index 532e771bd1..a2bd8c7479 100644
--- a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
+++ b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
@@ -1,3 +1,29 @@
 While merging the git-annex branch, annex-merge does not end up in a fast-forward even when it would be possible.
 But as sometimes annex-merge takes time, it would probably be worth it
 (but maybe I miss something with my workflow...).
+
+> I don't think a fast-forward will make things much faster.
+> 
+> git-annex needs its index file to be updated to reflect the merge.
+> With the union merge it does now, this can be accomplished by using
+> `git-diff-index` to efficiently get a list of files that have changed,
+> and only merge those changes into the index with `git-update-index`.
+> Then the index gets committed, generating the merge.
+> 
+> To fast-forward, it would just reset the git-annex branch to the new
+> head of the remote it's merging to. But then the index needs to be
+> updated to reflect this new head too. To do that needs the same method
+> described above, essentially (with the difference that it can replace
+> files in the index with the version from the git-annex branch, rather
+> than merging in the changes... but only if the index is known to be
+> already committed and have no other changes, which would require both
+> an attempt to commit it first, and
+> locking). 
+> 
+> So will take basically the same amount of time, except
+> it would not need to commit the index at the end of the merge. The
+> most expensive work is the `git-diff-index` and `git-update-index`,
+> which are not avoided.
+> 
+> Although, perhaps fast-forward merge would use slightly
+> less space. --[[Joey]]

From 0fee31a164aaaa951a8b8bd66acce65b1e94ca43 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 13 Jul 2011 10:35:18 +0000
Subject: [PATCH 2057/8313]

---
 ..._normal_files_in_the_git-annex_git_repository__63__.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn

diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
new file mode 100644
index 0000000000..1105cd0271
--- /dev/null
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
@@ -0,0 +1,6 @@
+Is it possible to story ordinary files in the git repository, or is this going to confuse git-annex? In other words, can I safely run
+
+  git add .gitattributes
+  git commit -m 'remember attributes' .gitattributes
+
+..., or do I have to use `git-annex add` all time?

From 709f75f1878961d70e913a0f1020a134a7e16bc2 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 13 Jul 2011 10:35:47 +0000
Subject: [PATCH 2058/8313]

---
 ...re_normal_files_in_the_git-annex_git_repository__63__.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
index 1105cd0271..6d1083dd57 100644
--- a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__.mdwn
@@ -1,6 +1,6 @@
 Is it possible to story ordinary files in the git repository, or is this going to confuse git-annex? In other words, can I safely run
 
-  git add .gitattributes
-  git commit -m 'remember attributes' .gitattributes
+    git add .gitattributes
+    git commit -m 'remember attributes' .gitattributes
 
 ..., or do I have to use `git-annex add` all time?

From a4f2dd2fc60da291f5aa37ba814affeae134ff90 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 13 Jul 2011 16:21:26 +0000
Subject: [PATCH 2059/8313] Added a comment: Solved

---
 .../comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment

diff --git a/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment
new file mode 100644
index 0000000000..8873edcde8
--- /dev/null
+++ b/doc/forum/Can_I_store_normal_files_in_the_git-annex_git_repository__63__/comment_1_c8f9923d8dc76b8bed25dce5ae09b520._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.12.196.1"
+ subject="Solved"
+ date="2011-07-13T16:21:25Z"
+ content="""
+I got my answer on #vcs-home: Yes, git-annex and git get along fine.
+"""]]

From 020787bb974da8eafb5002b9427d44e21b3c802e Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Thu, 14 Jul 2011 12:15:42 +0000
Subject: [PATCH 2060/8313]

---
 ..._command_has_tons_of_redundant_-a_paramters.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn

diff --git a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
new file mode 100644
index 0000000000..80bf562ec7
--- /dev/null
+++ b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
@@ -0,0 +1,13 @@
+This doesn't look right:
+
+    simons   11148  0.0  0.0  15572  1268 pts/1    SN+  04:00   0:00              |               \_ git annex unannex stuff
+    simons   11150  0.5  0.0 130504 11212 pts/1    SN+  04:00   3:40              |               |   \_ git-annex unannex stuff
+    simons   11152  0.0  0.1  39536 23932 pts/1    SN+  04:00   0:00              |               |       \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex ls-files --cached -z -- stuff
+    simons   11288  0.0  0.0      0     0 pts/1    ZN+  04:01   0:00              |               |       \_ [git] 
+    simons   11339  0.0  0.0      0     0 pts/1    ZN+  04:02   0:00              |               |       \_ [git-annex] 
+    simons   11442  0.0  0.0      0     0 pts/1    ZN+  04:06   0:00              |               |       \_ [git] 
+    simons   11443  0.0  0.0      0     0 pts/1    ZN+  04:06   0:05              |               |       \_ [git] 
+    simons   16541  0.0  0.0      0     0 pts/1    ZN+  04:14   0:00              |               |       \_ [git] 
+    simons   16543  0.3  0.0  15644  1744 pts/1    SN+  04:14   2:13              |               |       \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex cat-file --batch
+    simons   14224  0.0  0.0 100744   796 pts/1    SN+  14:10   0:00              |               |       \_ xargs -0 git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex
+    simons   14225  0.4  0.1  32684 18652 pts/1    DN+  14:10   0:00              |               |           \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a

From dddbc09ff0199967602ad5d4112b2b4e04780177 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 16:41:17 -0400
Subject: [PATCH 2061/8313] allow configStore to be run incrementally to
 override configs

---
 Git.hs | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/Git.hs b/Git.hs
index 7827f80514..2d03163bbf 100644
--- a/Git.hs
+++ b/Git.hs
@@ -466,13 +466,24 @@ hConfigRead repo h = do
 	val <- hGetContentsStrict h
 	configStore repo val
 
-{- Parses a git config and returns a version of the repo using it. -}
+{- Stores a git config into a repo, returning the new version of the repo.
+ - The git config may be multiple lines, or a single line. Config settings
+ - can be updated inrementally. -}
 configStore :: Repo -> String -> IO Repo
 configStore repo s = do
-	rs <- configRemotes r
-	return $ r { remotes = rs }
+	let repo' = repo { config = Map.union (configParse s) (config repo) }
+	rs <- configRemotes repo'
+	return $ repo' { remotes = rs }
+
+{- Parses git config --list output into a config map. -}
+configParse :: String -> Map.Map String String
+configParse s = Map.fromList $ map pair $ lines s
 	where
-		r = repo { config = configParse s }
+		pair l = (key l, val l)
+		key l = head $ keyval l
+		val l = join sep $ drop 1 $ keyval l
+		keyval l = split sep l :: [String]
+		sep = "="
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 configRemotes :: Repo -> IO [Repo]
@@ -503,16 +514,6 @@ configRemotes repo = mapM construct remotepairs
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
-{- Parses git config --list output into a config map. -}
-configParse :: String -> Map.Map String String
-configParse s = Map.fromList $ map pair $ lines s
-	where
-		pair l = (key l, val l)
-		key l = head $ keyval l
-		val l = join sep $ drop 1 $ keyval l
-		keyval l = split sep l :: [String]
-		sep = "="
-
 {- Returns a single git config setting, or a default value if not set. -}
 configGet :: Repo -> String -> String -> String
 configGet repo key defaultValue = 

From 0c46cbab09af8cc8761668885e58944d397b856d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 16:44:23 -0400
Subject: [PATCH 2062/8313] Support the standard git -c name=value

This allows eg, `git-annex -c annex.rsync-options=-6 get file`

The overridden git configs are not passed on to git plumbing commands
that are run. Perhaps someone will find a need to do that, but I don't yet
and it would require storing more state to know what config settings
have been overridden and need to be passed on.
---
 GitAnnex.hs        | 11 ++++++++++-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  4 ++++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 85eb2bf26e..6f4e5d4921 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -8,12 +8,14 @@
 module GitAnnex where
 
 import System.Console.GetOpt
+import Control.Monad.State (liftIO)
 
 import qualified Git
 import CmdLine
 import Command
 import Options
 import Utility
+import Types
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
@@ -102,9 +104,11 @@ options = commonOptions ++
 	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
 		"override trust setting"
 	, Option [] ["semitrust"] (ReqArg (Remote.forceTrust SemiTrusted) paramRemote)
-		"override trust setting back to default value"
+		"override trust setting back to default"
 	, Option [] ["untrust"] (ReqArg (Remote.forceTrust UnTrusted) paramRemote)
 		"override trust setting to untrusted"
+	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
+		"override git configuration setting"
 	]
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
@@ -112,6 +116,11 @@ options = commonOptions ++
 		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
+		setgitconfig :: String -> Annex ()
+		setgitconfig v = do
+			g <- Annex.gitRepo
+			g' <- liftIO $ Git.configStore g v
+			Annex.changeState $ \s -> s { Annex.repo = g' }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/debian/changelog b/debian/changelog
index ec3176d12a..673084d2d8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110708) UNRELEASED; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
   * Bugfix: Make add ../ work.
+  * Support the standard git -c name=value
 
  -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dfb23f5f21..11f617f1ba 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -387,6 +387,10 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Specifies a key to operate on.
 
+* -c name=value
+
+  Used to override git configuration settings. May be specified multiple times.
+
 # CONFIGURATION
 
 Like other git commands, git-annex is configured via `.git/config`.

From ded259112449f592bc42207e89c82268f3795f12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 16:56:06 -0400
Subject: [PATCH 2063/8313] unannex: Clean up use of git commit -a.

This was more complex than would be expected. unannex has to use git commit -a
since it's removing files from git; git commit filelist won't do.

Allow commands to be added to the Git queue that have no associated files,
and run such commands once.
---
 AnnexQueue.hs                                   |  6 +++---
 Command/Add.hs                                  |  4 ++--
 Command/Fix.hs                                  |  2 +-
 Command/FromKey.hs                              |  2 +-
 Command/Lock.hs                                 |  2 +-
 Command/Unannex.hs                              |  2 +-
 Git/Queue.hs                                    | 17 ++++++++++-------
 Upgrade/V1.hs                                   |  8 ++++----
 debian/changelog                                |  1 +
 ...mand_has_tons_of_redundant_-a_paramters.mdwn |  2 ++
 10 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 4c35adfb87..b1678df07f 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -21,10 +21,10 @@ import Utility
 
 {- Adds a git command to the queue, possibly running previously queued
  - actions if enough have accumulated. -}
-add :: String -> [CommandParam] -> FilePath -> Annex ()
-add command params file = do
+add :: String -> [CommandParam] -> [FilePath] -> Annex ()
+add command params files = do
 	q <- getState repoqueue
-	store $ Git.Queue.add q command params file
+	store $ Git.Queue.add q command params files
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
diff --git a/Command/Add.hs b/Command/Add.hs
index 5c7cad044e..51b95b9b5b 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -87,6 +87,6 @@ cleanup file key = do
 
 	force <- Annex.getState Annex.force
 	if force
-		then AnnexQueue.add "add" [Param "-f", Param "--"] file
-		else AnnexQueue.add "add" [Param "--"] file
+		then AnnexQueue.add "add" [Param "-f", Param "--"] [file]
+		else AnnexQueue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 60627e9df0..47b0c4c9a0 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -44,5 +44,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] file
+	AnnexQueue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index fb9ab0775a..d59f1de397 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -45,5 +45,5 @@ perform file = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] file
+	AnnexQueue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/Lock.hs b/Command/Lock.hs
index e55cd9e79a..d39df5f335 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -33,5 +33,5 @@ perform file = do
 	-- Checkout from HEAD to get rid of any changes that might be 
 	-- staged in the index, and get back to the previous symlink to
 	-- the content.
-	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] file
+	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] [file]
 	next $ return True -- no cleanup needed
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index f22503ee06..d3623ed997 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -78,6 +78,6 @@ cleanup file key = do
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
-	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] "-a"
+	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] []
 	
 	return True
diff --git a/Git/Queue.hs b/Git/Queue.hs
index e1ec0cd313..e080476b79 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -53,15 +53,15 @@ empty :: Queue
 empty = Queue 0 M.empty
 
 {- Adds an action to a queue. -}
-add :: Queue -> String -> [CommandParam] -> FilePath -> Queue
-add (Queue n m) subcommand params file = Queue (n + 1) m'
+add :: Queue -> String -> [CommandParam] -> [FilePath] -> Queue
+add (Queue n m) subcommand params files = Queue (n + 1) m'
 	where
 		action = Action subcommand params
 		-- There are probably few items in the map, but there
 		-- can be a lot of files per item. So, optimise adding
 		-- files.
-		m' = M.insertWith' const action files m
-		files = file:(M.findWithDefault [] action m)
+		m' = M.insertWith' const action fs m
+		fs = files ++ (M.findWithDefault [] action m)
 
 {- Number of items in a queue. -}
 size :: Queue -> Int
@@ -79,11 +79,14 @@ flush repo (Queue _ m) = do
 
 {- Runs an Action on a list of files in a git repository.
  -
- - Complicated by commandline length limits. -}
+ - Complicated by commandline length limits.
+ -
+ - Intentionally runs the command even if the list of files is empty;
+ - this allows queueing commands that do not need a list of files. -}
 runAction :: Repo -> Action -> [FilePath] -> IO ()
-runAction repo action files = unless (null files) runxargs
+runAction repo action files =
+	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
-		runxargs = pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 		params = toCommand $ gitCommandLine repo
 			(Param (getSubcommand action):getParams action)
 		feedxargs h = hPutStr h $ join "\0" files
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c0bbeebaf4..165a482628 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -104,7 +104,7 @@ updateSymlinks = do
 					link <- calcGitLink f k
 					liftIO $ removeFile f
 					liftIO $ createSymbolicLink link f
-					AnnexQueue.add "add" [Param "--"] f
+					AnnexQueue.add "add" [Param "--"] [f]
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
@@ -134,9 +134,9 @@ moveLocationLogs = do
 				old <- readLog f
 				new <- readLog dest
 				writeLog dest (old++new)
-				AnnexQueue.add "add" [Param "--"] dest
-				AnnexQueue.add "add" [Param "--"] f
-				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] f
+				AnnexQueue.add "add" [Param "--"] [dest]
+				AnnexQueue.add "add" [Param "--"] [f]
+				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
 		
 oldlog2key :: FilePath -> Maybe (FilePath, Key)
 oldlog2key l = 
diff --git a/debian/changelog b/debian/changelog
index 673084d2d8..9a7c3e7b0f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110708) UNRELEASED; urgency=low
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
   * Bugfix: Make add ../ work.
   * Support the standard git -c name=value
+  * unannex: Clean up use of git commit -a.
 
  -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
 
diff --git a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
index 80bf562ec7..181b02b5c1 100644
--- a/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
+++ b/doc/bugs/git_command_line_constructed_by_unannex_command_has_tons_of_redundant_-a_paramters.mdwn
@@ -11,3 +11,5 @@ This doesn't look right:
     simons   16543  0.3  0.0  15644  1744 pts/1    SN+  04:14   2:13              |               |       \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex cat-file --batch
     simons   14224  0.0  0.0 100744   796 pts/1    SN+  14:10   0:00              |               |       \_ xargs -0 git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex
     simons   14225  0.4  0.1  32684 18652 pts/1    DN+  14:10   0:00              |               |           \_ git --git-dir=/home/simons/annex/.git --work-tree=/home/simons/annex commit -a -m content removed from git annex -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a -a
+
+> [[Fixed|done]] --[[Joey]] 

From 9bb797c0eae3c9d2f119a734762a6d5fa7321a80 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jul 2011 17:18:53 -0400
Subject: [PATCH 2064/8313] unannex: only commit, no -a

-a is actually not needed; only commit staged changes
---
 Command/Unannex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d3623ed997..960f99722f 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -78,6 +78,6 @@ cleanup file key = do
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
-	AnnexQueue.add "commit" [Params "-a -m", Param "content removed from git annex"] []
+	AnnexQueue.add "commit" [Param "-m", Param "content removed from git annex"] []
 	
 	return True

From e78475737636a5d1e0d0a36b475c300cc7bb56cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 03:12:05 -0400
Subject: [PATCH 2065/8313] hlint tweaks

Did all sources except Remotes/* and Command/*
---
 Backend.hs         | 10 +++----
 Backend/SHA.hs     |  2 +-
 Backend/WORM.hs    |  2 +-
 Branch.hs          |  6 ++--
 CmdLine.hs         | 22 +++++++-------
 Command.hs         |  6 ++--
 Content.hs         | 71 +++++++++++++++++++++++-----------------------
 Crypto.hs          | 29 ++++++++++---------
 Git.hs             | 51 ++++++++++++++++-----------------
 Git/LsFiles.hs     |  8 +++---
 Git/Queue.hs       |  4 +--
 Git/UnionMerge.hs  |  2 +-
 LocationLog.hs     |  3 +-
 Locations.hs       |  2 +-
 Messages.hs        |  2 +-
 PresenceLog.hs     |  4 +--
 Remote.hs          |  7 +++--
 RemoteLog.hs       | 10 +++----
 TestConfig.hs      | 23 ++++++++-------
 Types/Key.hs       |  4 +--
 Types/Remote.hs    |  3 +-
 UUID.hs            |  4 +--
 Upgrade/V0.hs      |  2 +-
 Upgrade/V1.hs      | 25 +++++++---------
 Upgrade/V2.hs      | 12 ++++----
 Utility.hs         |  4 +--
 Version.hs         |  3 +-
 configure.hs       |  5 ++--
 git-annex-shell.hs |  4 +--
 git-annex.cabal    |  2 +-
 git-union-merge.hs |  5 ++--
 test.hs            | 14 ++++-----
 32 files changed, 172 insertions(+), 179 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index cf976d2b8a..3429e8f42c 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,6 +19,7 @@ import Control.Monad.State (liftIO, when)
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
+import Data.Maybe
 
 import Locations
 import qualified Git
@@ -33,10 +34,7 @@ import qualified Backend.WORM
 import qualified Backend.SHA
 
 list :: [Backend Annex]
-list = concat 
-	[ Backend.WORM.backends
-	, Backend.SHA.backends
-	]
+list = Backend.WORM.backends ++ Backend.SHA.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
@@ -54,7 +52,7 @@ orderedList = do
 		handle Nothing s = return s
 		handle (Just "") s = return s
 		handle (Just name) s = do
-			let l' = (lookupBackendName name):s
+			let l' = lookupBackendName name : s
 			Annex.changeState $ \state -> state { Annex.backends = l' }
 			return l'
 		getstandard = do
@@ -119,7 +117,7 @@ chooseBackends fs = do
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
-lookupBackendName s = maybe unknown id $ maybeLookupBackendName s
+lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index bd6e411a05..dc27b30003 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -114,7 +114,7 @@ checkKeyChecksum size key = do
 	fast <- Annex.getState Annex.fast
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
-	if (not present || fast)
+	if not present || fast
 		then return True
 		else do
 			s <- shaN size file
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 036d0564cb..831c9e8ced 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -35,7 +35,7 @@ backend = Types.Backend.Backend {
 keyValue :: FilePath -> Annex (Maybe Key)
 keyValue file = do
 	stat <- liftIO $ getFileStatus file
-	return $ Just $ Key {
+	return $ Just Key {
 		keyName = takeFileName file,
 		keyBackendName = name backend,
 		keySize = Just $ fromIntegral $ fileSize stat,
diff --git a/Branch.hs b/Branch.hs
index d091b6edab..c8e6bc2bb4 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -87,7 +87,7 @@ withIndex' bootstrapping a = do
 
 	e <- liftIO $ doesFileExist f
 	unless e $ do
-		unless bootstrapping $ create
+		unless bootstrapping create
 		liftIO $ createDirectoryIfMissing True $ takeDirectory f
 		liftIO $ unless bootstrapping $ genIndex g
 
@@ -187,7 +187,7 @@ updateRef ref
 			Param (name++".."++ref),
 			Params "--oneline -n1"
 			]
-		if (null diffs)
+		if null diffs
 			then return Nothing
 			else do
 				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."
@@ -305,7 +305,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = getJournalFilesRaw >>= return . map fileJournal
+getJournalFiles = fmap (map fileJournal) getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do
diff --git a/CmdLine.hs b/CmdLine.hs
index b807046df3..c33c497856 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -39,14 +39,13 @@ dispatch args cmds options header gitrepo = do
  - list of actions to be run in the Annex monad. -}
 parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool]
 parseCmd argv header cmds options = do
-	(flags, params) <- liftIO $ getopt
+	(flags, params) <- liftIO getopt
 	when (null params) $ error $ "missing command" ++ usagemsg
 	case lookupCmd (head params) of
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			when (cmdusesrepo command) $
-				checkVersion
+			when (cmdusesrepo command) checkVersion
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -78,9 +77,9 @@ usage header cmds options =
  - (but explicitly thrown errors terminate the whole command).
  -}
 tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()
-tryRun state actions = tryRun' state 0 actions
-tryRun' :: Annex.AnnexState -> Integer -> [Annex Bool] -> IO ()
-tryRun' state errnum (a:as) = do
+tryRun = tryRun' 0
+tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
+tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
 		AnnexQueue.flushWhenFull
 		a
@@ -89,11 +88,10 @@ tryRun' state errnum (a:as) = do
 			Annex.eval state $ do
 				showEndFail
 				showErr err
-			tryRun' state (errnum + 1) as
-		Right (True,state') -> tryRun' state' errnum as
-		Right (False,state') -> tryRun' state' (errnum + 1) as
-tryRun' _ errnum [] = do
-	when (errnum > 0) $ error $ show errnum ++ " failed"
+			tryRun' (errnum + 1) state as
+		Right (True,state') -> tryRun' errnum state' as
+		Right (False,state') -> tryRun' (errnum + 1) state' as
+tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
@@ -105,5 +103,5 @@ startup = do
 shutdown :: Annex Bool
 shutdown = do
 	saveState
-	liftIO $ Git.reap
+	liftIO Git.reap
 	return True
diff --git a/Command.hs b/Command.hs
index c666ddbd29..729e442fc4 100644
--- a/Command.hs
+++ b/Command.hs
@@ -115,7 +115,7 @@ isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
 	g <- Annex.gitRepo
-	when (Git.repoIsLocalBare g) $ do
+	when (Git.repoIsLocalBare g) $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
@@ -175,9 +175,9 @@ withFilesUnlocked' typechanged a params = do
 	unlockedfiles' <- filterFiles unlockedfiles
 	backendPairs a unlockedfiles'
 withKeys :: CommandSeekKeys
-withKeys a params = return $ map a $ map parse params
+withKeys a params = return $ map (a . parse) params
 	where
-		parse p = maybe (error "bad key") id $ readKey p
+		parse p = fromMaybe (error "bad key") $ readKey p
 withTempFile :: CommandSeekStrings
 withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
diff --git a/Content.hs b/Content.hs
index 94f8b8c2ac..c63042dfb5 100644
--- a/Content.hs
+++ b/Content.hs
@@ -57,8 +57,8 @@ inAnnex key = do
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
 	g <- Annex.gitRepo
-	cwd <- liftIO $ getCurrentDirectory
-	let absfile = maybe whoops id $ absNormPath cwd file
+	cwd <- liftIO getCurrentDirectory
+	let absfile = fromMaybe whoops $ absNormPath cwd file
 	return $ relPathDirToFile (parentDir absfile) 
 			(Git.workTree g)  ".git"  annexLocation key
 	where
@@ -94,15 +94,19 @@ getViaTmp key action = do
 
 	getViaTmpUnchecked key action
 
+prepTmp :: Key -> Annex FilePath
+prepTmp key = do
+	g <- Annex.gitRepo
+	let tmp = gitAnnexTmpLocation g key
+	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	return tmp
+
 {- Like getViaTmp, but does not check that there is enough disk space
  - for the incoming key. For use when the key content is already on disk
  - and not being copied into place. -}
 getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmpUnchecked key action = do
-	g <- Annex.gitRepo
-	let tmp = gitAnnexTmpLocation g key
-
-	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	tmp <- prepTmp key
 	success <- action tmp
 	if success
 		then do
@@ -117,9 +121,7 @@ getViaTmpUnchecked key action = do
 {- Creates a temp file, runs an action on it, and cleans up the temp file. -}
 withTmp :: Key -> (FilePath -> Annex a) -> Annex a
 withTmp key action = do
-	g <- Annex.gitRepo
-	let tmp = gitAnnexTmpLocation g key
-	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	tmp <- prepTmp key
 	res <- action tmp
 	liftIO $ whenM (doesFileExist tmp) $ liftIO $ removeFile tmp
 	return res
@@ -133,23 +135,21 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- Annex.gitRepo
 	r <- getConfig g "diskreserve" ""
-	let reserve = maybe megabyte id $ readSize dataUnits r
+	let reserve = fromMaybe megabyte $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
 		(_, Nothing) -> return ()
 		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
-			if (need + reserve > have + adjustment)
-				then needmorespace (need + reserve - have - adjustment)
-				else return ()
+			when (need + reserve > have + adjustment) $
+				needmorespace (need + reserve - have - adjustment)
 	where
 		megabyte :: Integer
 		megabyte = 1000000
-		needmorespace n = do
-			unlessM (Annex.getState Annex.force) $
-				error $ "not enough free space, need " ++ 
-					roughSize storageUnits True n ++
-					" more (use --force to override this check or adjust annex.diskreserve)"
+		needmorespace n = unlessM (Annex.getState Annex.force) $
+			error $ "not enough free space, need " ++ 
+				roughSize storageUnits True n ++
+				" more (use --force to override this check or adjust annex.diskreserve)"
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
@@ -200,28 +200,27 @@ moveAnnex key src = do
 			preventWrite dest
 			preventWrite dir
 
-{- Removes a key's file from .git/annex/objects/ -}
-removeAnnex :: Key -> Annex ()
-removeAnnex key = do
+withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
+withObjectLoc key a = do
 	g <- Annex.gitRepo
 	let file = gitAnnexLocation g key
 	let dir = parentDir file
-	liftIO $ do
-		allowWrite dir
-		removeFile file
-		removeDirectory dir
+	a (dir, file)
+
+{- Removes a key's file from .git/annex/objects/ -}
+removeAnnex :: Key -> Annex ()
+removeAnnex key = withObjectLoc key $ \(dir, file) -> liftIO $ do
+	allowWrite dir
+	removeFile file
+	removeDirectory dir
 
 {- Moves a key's file out of .git/annex/objects/ -}
 fromAnnex :: Key -> FilePath -> Annex ()
-fromAnnex key dest = do
-	g <- Annex.gitRepo
-	let file = gitAnnexLocation g key
-	let dir = parentDir file
-	liftIO $ do
-		allowWrite dir
-		allowWrite file
-		renameFile file dest
-		removeDirectory dir
+fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
+	allowWrite dir
+	allowWrite file
+	renameFile file dest
+	removeDirectory dir
 
 {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
  - returns the file it was moved to. -}
@@ -246,7 +245,7 @@ getKeysPresent = do
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
-	if (not exists)
+	if not exists
 		then return []
 		else liftIO $ do
 			-- 2 levels of hashing
@@ -254,7 +253,7 @@ getKeysPresent' dir = do
 			levelb <- mapM dirContents levela
 			contents <- mapM dirContents (concat levelb)
 			files <- filterM present (concat contents)
-			return $ catMaybes $ map (fileKey . takeFileName) files
+			return $ mapMaybe (fileKey . takeFileName) files
 	where
 		present d = do
 			result <- try $
diff --git a/Crypto.hs b/Crypto.hs
index 485fb6e931..4fc41ede04 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -33,6 +33,7 @@ import Data.Digest.Pure.SHA
 import System.Cmd.Utils
 import Data.String.Utils
 import Data.List
+import Data.Maybe
 import System.IO
 import System.Posix.IO
 import System.Posix.Types
@@ -125,11 +126,11 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 	return $ EncryptedCipher encipher (KeyIds ks')
 	where
 		encrypt = [ Params "--encrypt" ]
-		recipients l = 
-			-- Force gpg to only encrypt to the specified
-			-- recipients, not configured defaults.
-			[ Params "--no-encrypt-to --no-default-recipient"] ++
-			(concat $ map (\k -> [Param "--recipient", Param k]) l)
+		recipients l = force_recipients :
+			concatMap (\k -> [Param "--recipient", Param k]) l
+		-- Force gpg to only encrypt to the specified
+		-- recipients, not configured defaults.
+		force_recipients = Params "--no-encrypt-to --no-default-recipient"
 
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
@@ -152,24 +153,24 @@ encryptKey c k =
 
 {- Runs an action, passing it a handle from which it can 
  - stream encrypted content. -}
-withEncryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a
+withEncryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
 withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"]
 
 {- Runs an action, passing it a handle from which it can
  - stream decrypted content. -}
-withDecryptedHandle :: Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a
+withDecryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
 withDecryptedHandle = gpgCipherHandle [Param "--decrypt"]
 
 {- Streams encrypted content to an action. -}
-withEncryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a
+withEncryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 withEncryptedContent = pass withEncryptedHandle
 
 {- Streams decrypted content to an action. -}
-withDecryptedContent :: Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a
+withDecryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 withDecryptedContent = pass withDecryptedHandle
 
-pass :: (Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a) 
-      -> Cipher -> (IO L.ByteString) -> (L.ByteString -> IO a) -> IO a
+pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a) 
+      -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 pass to c i a = to c i $ \h -> a =<< L.hGetContents h
 
 gpgParams :: [CommandParam] -> IO [String]
@@ -202,7 +203,7 @@ gpgPipeStrict params input = do
  -
  - Note that to avoid deadlock with the cleanup stage,
  - the action must fully consume gpg's input before returning. -}
-gpgCipherHandle :: [CommandParam] -> Cipher -> (IO L.ByteString) -> (Handle -> IO a) -> IO a
+gpgCipherHandle :: [CommandParam] -> Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
 gpgCipherHandle params c a b = do
 	-- pipe the passphrase into gpg on a fd
 	(frompipe, topipe) <- createPipe
@@ -235,10 +236,10 @@ configKeyIds c = do
 	where
 		parseWithColons s = map keyIdField $ filter pubKey $ lines s
 		pubKey = isPrefixOf "pub:"
-		keyIdField s = (split ":" s) !! 4
+		keyIdField s = split ":" s !! 4
 
 configGet :: RemoteConfig -> String -> String
-configGet c key = maybe missing id $ M.lookup key c
+configGet c key = fromMaybe missing $ M.lookup key c
 	where missing = error $ "missing " ++ key ++ " in remote config"
 
 hmacWithCipher :: Cipher -> String -> String
diff --git a/Git.hs b/Git.hs
index 2d03163bbf..9b7ac7ea91 100644
--- a/Git.hs
+++ b/Git.hs
@@ -69,11 +69,10 @@ import System.Posix.User
 import System.Posix.Process
 import System.Path
 import System.Cmd.Utils
-import IO (bracket_)
+import IO (bracket_, try)
 import Data.String.Utils
 import System.IO
-import IO (try)
-import qualified Data.Map as Map hiding (map, split)
+import qualified Data.Map as M hiding (map, split)
 import Network.URI
 import Data.Maybe
 import Data.Char
@@ -93,7 +92,7 @@ data RepoLocation = Dir FilePath | Url URI | Unknown
 
 data Repo = Repo {
 	location :: RepoLocation,
-	config :: Map.Map String String,
+	config :: M.Map String String,
 	remotes :: [Repo],
 	-- remoteName holds the name used for this repo in remotes
 	remoteName :: Maybe String 
@@ -103,7 +102,7 @@ newFrom :: RepoLocation -> Repo
 newFrom l = 
 	Repo {
 		location = l,
-		config = Map.empty,
+		config = M.empty,
 		remotes = [],
 		remoteName = Nothing
 	}
@@ -140,7 +139,7 @@ repoFromUrl url
 	| startswith "file://" url = repoFromAbsPath $ uriPath u
 	| otherwise = return $ newFrom $ Url u
 		where
-			u = maybe bad id $ parseURI url
+			u = fromMaybe bad $ parseURI url
 			bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
@@ -208,7 +207,7 @@ repoIsSsh Repo { location = Url url }
 repoIsSsh _ = False
 
 configAvail ::Repo -> Bool
-configAvail Repo { config = c } = c /= Map.empty
+configAvail Repo { config = c } = c /= M.empty
 
 repoIsLocalBare :: Repo -> Bool
 repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
@@ -228,7 +227,7 @@ assertUrl repo action =
 				" not supported"
 
 configBare :: Repo -> Bool
-configBare repo = maybe unknown configTrue $ Map.lookup "core.bare" $ config repo
+configBare repo = maybe unknown configTrue $ M.lookup "core.bare" $ config repo
 	where
 		unknown = error $ "it is not known if git repo " ++
 			repoDescribe repo ++
@@ -272,14 +271,14 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 	let file' = absfile cwd
 	unless (inrepo file') $
 		error $ file ++ " is not located inside git repository " ++ absrepo
-	if (inrepo $ addTrailingPathSeparator cwd)
+	if inrepo $ addTrailingPathSeparator cwd
 		then return $ relPathDirToFile cwd file'
 		else return $ drop (length absrepo) file'
 	where
 		-- normalize both repo and file, so that repo
 		-- will be substring of file
 		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d
-		absfile c = maybe file id $ secureAbsNormPath c file
+		absfile c = fromMaybe file $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
 		bad = error $ "bad repo" ++ repoDescribe repo
 workTreeFile repo _ = assertLocal repo $ error "internal"
@@ -303,7 +302,7 @@ uriRegName' a = fixup $ uriRegName a
 			| rest !! len == ']' = take len rest
 			| otherwise = x
 			where
-				len  = (length rest) - 1
+				len  = length rest - 1
 		fixup x = x
 
 {- Hostname of an URL repo. -}
@@ -348,7 +347,7 @@ gitCommandLine repo _ = assertLocal repo $ error "internal"
 {- Runs git in the specified repo. -}
 runBool :: Repo -> String -> [CommandParam] -> IO Bool
 runBool repo subcommand params = assertLocal repo $
-	boolSystem "git" (gitCommandLine repo ((Param subcommand):params))
+	boolSystem "git" $ gitCommandLine repo $ Param subcommand : params
 
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: Repo -> String -> [CommandParam] -> IO ()
@@ -471,13 +470,13 @@ hConfigRead repo h = do
  - can be updated inrementally. -}
 configStore :: Repo -> String -> IO Repo
 configStore repo s = do
-	let repo' = repo { config = Map.union (configParse s) (config repo) }
+	let repo' = repo { config = configParse s `M.union` config repo }
 	rs <- configRemotes repo'
 	return $ repo' { remotes = rs }
 
 {- Parses git config --list output into a config map. -}
-configParse :: String -> Map.Map String String
-configParse s = Map.fromList $ map pair $ lines s
+configParse :: String -> M.Map String String
+configParse s = M.fromList $ map pair $ lines s
 	where
 		pair l = (key l, val l)
 		key l = head $ keyval l
@@ -489,8 +488,8 @@ configParse s = Map.fromList $ map pair $ lines s
 configRemotes :: Repo -> IO [Repo]
 configRemotes repo = mapM construct remotepairs
 	where
-		remotepairs = Map.toList $ filterremotes $ config repo
-		filterremotes = Map.filterWithKey (\k _ -> isremote k)
+		remotepairs = M.toList $ filterremotes $ config repo
+		filterremotes = M.filterWithKey (\k _ -> isremote k)
 		isremote k = startswith "remote." k && endswith ".url" k
 		construct (k,v) = do
 			r <- gen v
@@ -499,15 +498,15 @@ configRemotes repo = mapM construct remotepairs
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
 		-- git remotes can be written scp style -- [user@]host:dir
-		scpstyle v = ":" `isInfixOf` v && (not $ "//" `isInfixOf` v)
+		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
 				bits = split ":" v
-				host = bits !! 0
+				host = head bits
 				dir = join ":" $ drop 1 bits
 				slash d	| d == "" = "/~/" ++ dir
-					| d !! 0 == '/' = dir
-					| d !! 0 == '~' = '/':dir
+					| head d == '/' = dir
+					| head d == '~' = '/':dir
 					| otherwise = "/~/" ++ dir
 
 {- Checks if a string from git config is a true value. -}
@@ -517,11 +516,11 @@ configTrue s = map toLower s == "true"
 {- Returns a single git config setting, or a default value if not set. -}
 configGet :: Repo -> String -> String -> String
 configGet repo key defaultValue = 
-	Map.findWithDefault defaultValue key (config repo)
+	M.findWithDefault defaultValue key (config repo)
 
 {- Access to raw config Map -}
-configMap :: Repo -> Map.Map String String
-configMap repo = config repo
+configMap :: Repo -> M.Map String String
+configMap = config
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
@@ -680,8 +679,8 @@ seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
 seekUp want dir = do
 	ok <- want dir
 	if ok
-		then return (Just dir)
-		else case (parentDir dir) of
+		then return $ Just dir
+		else case parentDir dir of
 			"" -> return Nothing
 			d -> seekUp want d
 
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index b88c9144e6..23b383a09d 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -21,7 +21,7 @@ import Utility
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
-	[Params "ls-files --cached -z --"] ++ map File l
+	Params "ls-files --cached -z --" : map File l
 
 {- Scans for files at the specified locations that are not checked into
  - git. -}
@@ -44,12 +44,12 @@ staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
 staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
 	where
 		start = [Params "diff --cached --name-only -z"]
-		end = [Param "--"] ++ map File l
+		end = Param "--" : map File l
 
 {- Returns a list of files that have unstaged changes. -}
 changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]
 changedUnstaged repo l = pipeNullSplit repo $
-	[Params "diff --name-only -z --"] ++ map File l
+	Params "diff --name-only -z --" : map File l
 
 {- Returns a list of the files in the specified locations that are staged
  - for commit, and whose type has changed. -}
@@ -65,4 +65,4 @@ typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
 typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
 	where
 		start = [Params "diff --name-only --diff-filter=T -z"]
-		end = [Param "--"] ++ map File l
+		end = Param "--" : map File l
diff --git a/Git/Queue.hs b/Git/Queue.hs
index e080476b79..0016be4727 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -18,7 +18,7 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Control.Monad (unless, forM_)
+import Control.Monad (forM_)
 import Utility
 
 import Git
@@ -61,7 +61,7 @@ add (Queue n m) subcommand params files = Queue (n + 1) m'
 		-- can be a lot of files per item. So, optimise adding
 		-- files.
 		m' = M.insertWith' const action fs m
-		fs = files ++ (M.findWithDefault [] action m)
+		fs = files ++ M.findWithDefault [] action m
 
 {- Number of items in a queue. -}
 size :: Queue -> Int
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 4e0361b858..b0da071703 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -91,5 +91,5 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		nullsha = take shaSize $ repeat '0'
+		nullsha = replicate shaSize '0'
 		unionmerge = unlines . nub . lines
diff --git a/LocationLog.hs b/LocationLog.hs
index fe09482b9f..768483fa1b 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -49,8 +49,7 @@ keyLocations key = currentLog $ logFile key
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys =
-	return . catMaybes . map (logFileKey . takeFileName) =<< Branch.files
+loggedKeys = return . mapMaybe (logFileKey . takeFileName) =<< Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/Locations.hs b/Locations.hs
index 347b08ce14..2dbf2f55ea 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -52,7 +52,7 @@ import qualified Git
 {- The directory git annex uses for local state, relative to the .git
  - directory -}
 annexDir :: FilePath
-annexDir = addTrailingPathSeparator $ "annex"
+annexDir = addTrailingPathSeparator "annex"
 
 {- The directory git annex uses for locally available object content,
  - relative to the .git directory -}
diff --git a/Messages.hs b/Messages.hs
index 038e4c0bc7..5f150aafb4 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -37,7 +37,7 @@ showProgress :: Annex ()
 showProgress = verbose $ liftIO $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ "\n" ++ indent s
+showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
 
 showEndOk :: Annex ()
 showEndOk = verbose $ liftIO $ putStrLn "ok"
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 0777db209b..9c516a8db1 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -94,7 +94,7 @@ writeLog file ls = Branch.change file (unlines $ map show ls)
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> String -> Annex LogLine
 logNow s i = do
-	now <- liftIO $ getPOSIXTime
+	now <- liftIO getPOSIXTime
 	return $ LogLine now s i
 
 {- Reads a log and returns only the info that is still in effect. -}
@@ -112,7 +112,7 @@ type LogMap = Map.Map String LogLine
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
-compactLog ls = compactLog' Map.empty ls
+compactLog = compactLog' Map.empty
 compactLog' :: LogMap -> [LogLine] -> [LogLine]
 compactLog' m [] = Map.elems m
 compactLog' m (l:ls) = compactLog' (mapLog m l) ls
diff --git a/Remote.hs b/Remote.hs
index de1c7f38db..1a5006f6fb 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -33,6 +33,7 @@ import Control.Monad (filterM, liftM2)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -97,7 +98,7 @@ byName' "" = return $ Left "no remote specified"
 byName' n = do
 	allremotes <- genList
 	let match = filter matching allremotes
-	if (null match)
+	if null match
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ head match
 	where
@@ -110,7 +111,7 @@ nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
-		Left e -> return . (maybe (error e) id) =<< byDescription
+		Left e -> return . fromMaybe (error e) =<< byDescription
 		Right r -> return $ uuid r
 	where
 		byDescription = return . M.lookup n . invertMap =<< uuidMap
@@ -122,7 +123,7 @@ prettyPrintUUIDs :: [UUID] -> Annex String
 prettyPrintUUIDs uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	-- Show descriptions from the uuid log, falling back to remote names,
-	-- as some remotes may not be in the uuid log.
+	-- as some remotes may not be in the uuid log
 	m <- liftM2 M.union uuidMap $
 		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
diff --git a/RemoteLog.hs b/RemoteLog.hs
index c2065db9da..69a82f4987 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -36,7 +36,7 @@ configSet u c = do
 	Branch.change remoteLog $ unlines $ sort $
 		map toline $ M.toList $ M.insert u c m
 	where
-		toline (u', c') = u' ++ " " ++ (unwords $ configToKeyVal c')
+		toline (u', c') = u' ++ " " ++ unwords (configToKeyVal c')
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
@@ -44,14 +44,14 @@ readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
-	M.fromList $ catMaybes $ map parseline $ filter (not . null) $ lines s
+	M.fromList $ mapMaybe parseline $ filter (not . null) $ lines s
 	where
 		parseline l
 			| length w > 2 = Just (u, c)
 			| otherwise = Nothing
 			where
 				w = words l
-				u = w !! 0
+				u = head w
 				c = keyValToConfig $ tail w
 
 {- Given Strings like "key=value", generates a RemoteConfig. -}
@@ -90,8 +90,8 @@ configUnEscape = unescape
 				r = drop (length num) s
 				rest = drop 1 r
 				ok = not (null num) && 
-					not (null r) && r !! 0 == ';'
+					not (null r) && head r == ';'
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool
-prop_idempotent_configEscape s = s == (configUnEscape $ configEscape s)
+prop_idempotent_configEscape s = s == (configUnEscape . configEscape) s
diff --git a/TestConfig.hs b/TestConfig.hs
index bab297003a..8cfae7f0c7 100644
--- a/TestConfig.hs
+++ b/TestConfig.hs
@@ -45,7 +45,7 @@ writeSysConfig config = writeFile "SysConfig.hs" body
 
 runTests :: [TestCase] -> IO [Config]
 runTests [] = return []
-runTests ((TestCase tname t):ts) = do
+runTests (TestCase tname t : ts) = do
 	testStart tname
 	c <- t
 	testEnd c
@@ -62,7 +62,7 @@ requireCmd k cmdline = do
 		handle r = do
 			testEnd r
 			error $ "** the " ++ c ++ " command is required"
-		c = (words cmdline) !! 0
+		c = head $ words cmdline
 
 {- Checks if a command is available by running a command line. -}
 testCmd :: ConfigKey -> String -> Test
@@ -74,7 +74,7 @@ testCmd k cmdline = do
  - turn. The Config is set to the first one found. -}
 selectCmd :: ConfigKey -> [String] -> String -> Test
 selectCmd k = searchCmd
-		(\match -> return $ Config k $ StringConfig match)
+		(return . Config k . StringConfig)
 		(\cmds -> do
 			testEnd $ Config k $ BoolConfig False
 			error $ "* need one of these commands, but none are available: " ++ show cmds
@@ -82,7 +82,7 @@ selectCmd k = searchCmd
 
 maybeSelectCmd :: ConfigKey -> [String] -> String -> Test
 maybeSelectCmd k = searchCmd
-		(\match -> return $ Config k $ MaybeStringConfig $ Just match)
+		(return . Config k . MaybeStringConfig . Just)
 		(\_ -> return $ Config k $ MaybeStringConfig Nothing)
 
 searchCmd :: (String -> Test) -> ([String] -> Test) -> [String] -> String -> Test
@@ -91,7 +91,7 @@ searchCmd success failure cmds param = search cmds
 		search [] = failure cmds
 		search (c:cs) = do
 			ret <- system $ quiet c ++ " " ++ param
-			if (ret == ExitSuccess)
+			if ret == ExitSuccess
 				then success c
 				else search cs
 
@@ -104,8 +104,11 @@ testStart s = do
 	hFlush stdout
 
 testEnd :: Config -> IO ()
-testEnd (Config _ (BoolConfig True)) = putStrLn $ " yes"
-testEnd (Config _ (BoolConfig False)) = putStrLn $ " no"
-testEnd (Config _ (StringConfig s)) = putStrLn $ " " ++ s
-testEnd (Config _ (MaybeStringConfig (Just s))) = putStrLn $ " " ++ s
-testEnd (Config _ (MaybeStringConfig Nothing)) = putStrLn $ " not available"
+testEnd (Config _ (BoolConfig True)) = status "yes"
+testEnd (Config _ (BoolConfig False)) = status "no"
+testEnd (Config _ (StringConfig s)) = status s
+testEnd (Config _ (MaybeStringConfig (Just s))) = status s
+testEnd (Config _ (MaybeStringConfig Nothing)) = status "not available"
+
+status :: String -> IO ()
+status s = putStrLn $ ' ':s
diff --git a/Types/Key.hs b/Types/Key.hs
index 1d9bf8e11c..b26bb89896 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -48,7 +48,7 @@ instance Show Key where
 			"" +++ y = y
 			x +++ "" = x
 			x +++ y = x ++ fieldSep:y
-			c ?: (Just v) = c:(show v)
+			c ?: (Just v) = c : show v
 			_ ?: _ = ""
 
 readKey :: String -> Maybe Key
@@ -73,4 +73,4 @@ readKey s = if key == Just stubKey then Nothing else key
 		addfield _ _ _ = Nothing
 
 prop_idempotent_key_read_show :: Key -> Bool
-prop_idempotent_key_read_show k = Just k == (readKey $ show k)
+prop_idempotent_key_read_show k = Just k == (readKey . show) k
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 1d67ad5cd7..8d9622c519 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -11,6 +11,7 @@ module Types.Remote where
 
 import Control.Exception
 import Data.Map as M
+import Data.Ord
 
 import qualified Git
 import Types.Key
@@ -62,4 +63,4 @@ instance Eq (Remote a) where
 
 -- order remotes by cost
 instance Ord (Remote a) where
-	compare x y = compare (cost x) (cost y)
+	compare = comparing cost
diff --git a/UUID.hs b/UUID.hs
index 3ccc729069..cfa519baa3 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -49,7 +49,7 @@ genUUID :: IO UUID
 genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 	where
 		command = SysConfig.uuid
-		params = if (command == "uuid")
+		params = if command == "uuid"
 			-- request a random uuid be generated
 			then ["-m"]
 			-- uuidgen generates random uuid by default
@@ -82,7 +82,7 @@ prepUUID :: Annex ()
 prepUUID = do
 	u <- getUUID =<< Annex.gitRepo
 	when ("" == u) $ do
-		uuid <- liftIO $ genUUID
+		uuid <- liftIO genUUID
 		setConfig configkey uuid
 
 {- Records a description for a uuid in the uuidLog. -}
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index eabd030098..071fd12ee1 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -48,7 +48,7 @@ lookupFile0 = Upgrade.V1.lookupFile1
 getKeysPresent0 :: FilePath -> Annex [Key]
 getKeysPresent0 dir = do
 	exists <- liftIO $ doesDirectoryExist dir
-	if (not exists)
+	if not exists
 		then return []
 		else do
 			contents <- liftIO $ getDirectoryContents dir
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 165a482628..8a3d37a642 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -94,7 +94,7 @@ updateSymlinks = do
 	showNote "updating symlinks..."
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
-	forM_ files $ fixlink
+	forM_ files fixlink
 	where
 		fixlink f = do
 			r <- lookupFile1 f
@@ -119,7 +119,7 @@ moveLocationLogs = do
 				if exists
 					then do
 						contents <- liftIO $ getDirectoryContents dir
-						return $ catMaybes $ map oldlog2key contents
+						return $ mapMaybe oldlog2key contents
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
@@ -196,17 +196,14 @@ lookupFile1 file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = do
-			l <- readSymbolicLink file
-			return $ takeFileName l
-		makekey l = do
-			case maybeLookupBackendName bname of
-				Nothing -> do
-					unless (null kname || null bname ||
-					        not (isLinkToAnnex l)) $
-						warning skip
-					return Nothing
-				Just backend -> return $ Just (k, backend)
+		getsymlink = return . takeFileName =<< readSymbolicLink file
+		makekey l = case maybeLookupBackendName bname of
+			Nothing -> do
+				unless (null kname || null bname ||
+				        not (isLinkToAnnex l)) $
+					warning skip
+				return Nothing
+			Just backend -> return $ Just (k, backend)
 			where
 				k = fileKey1 l
 				bname = keyBackendName k
@@ -221,7 +218,7 @@ getKeyFilesPresent1  = do
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
 getKeyFilesPresent1' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
-	if (not exists)
+	if not exists
 		then return []
 		else do
 			dirs <- liftIO $ getDirectoryContents dir
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 4824f4bbaf..99c7806d27 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -10,7 +10,7 @@ module Upgrade.V2 where
 import System.Directory
 import System.FilePath
 import Control.Monad.State (unless, when, liftIO)
-import List
+import Data.List
 import Data.Maybe
 
 import Types.Key
@@ -61,7 +61,7 @@ upgrade = do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
 		unless bare $ gitAttributesUnWrite g
 
-	unless bare $ push
+	unless bare push
 
 	return True
 
@@ -70,11 +70,11 @@ locationLogs repo = liftIO $ do
 	levela <- dirContents dir
 	levelb <- mapM tryDirContents levela
 	files <- mapM tryDirContents (concat levelb)
-	return $ catMaybes $ map islogfile (concat files)
+	return $ mapMaybe islogfile (concat files)
 	where
 		tryDirContents d = catch (dirContents d) (return . const [])
 		dir = gitStateDir repo
-		islogfile f = maybe Nothing (\k -> Just $ (k, f)) $
+		islogfile f = maybe Nothing (\k -> Just (k, f)) $
 				logFileKey $ takeFileName f
 
 inject :: FilePath -> FilePath -> Annex ()
@@ -131,10 +131,10 @@ gitAttributesUnWrite repo = do
 	whenM (doesFileExist attributes) $ do
 		c <- readFileStrict attributes
 		liftIO $ viaTmp writeFile attributes $ unlines $
-			filter (\l -> not $ l `elem` attrLines) $ lines c
+			filter (`notElem` attrLines) $ lines c
 		Git.run repo "add" [File attributes]
 
 stateDir :: FilePath
-stateDir = addTrailingPathSeparator $ ".git-annex"
+stateDir = addTrailingPathSeparator ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/Utility.hs b/Utility.hs
index 63fa6eda35..786c6a1770 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -141,9 +141,9 @@ shellUnEscape s = word : shellUnEscape rest
 
 {- For quickcheck. -}
 prop_idempotent_shellEscape :: String -> Bool
-prop_idempotent_shellEscape s = [s] == (shellUnEscape $ shellEscape s)
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
 prop_idempotent_shellEscape_multiword :: [String] -> Bool
-prop_idempotent_shellEscape_multiword s = s == (shellUnEscape $ unwords $ map shellEscape s)
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
diff --git a/Version.hs b/Version.hs
index 7e7a4c7ce6..7e6910fbe4 100644
--- a/Version.hs
+++ b/Version.hs
@@ -43,8 +43,7 @@ checkVersion :: Annex ()
 checkVersion = getVersion >>= handle
 	where
 		handle Nothing = error "First run: git-annex init"
-		handle (Just v) = do
-			unless (v `elem` supportedVersions) $ do
+		handle (Just v) = unless (v `elem` supportedVersions) $
 			error $ "Repository version " ++ v ++ 
 				" is not supported. " ++
 				msg v
diff --git a/configure.hs b/configure.hs
index 8639af44b1..bfdfa32dd5 100644
--- a/configure.hs
+++ b/configure.hs
@@ -7,7 +7,7 @@ import TestConfig
 
 tests :: [TestCase]
 tests =
-	[ TestCase "version" $ getVersion
+	[ TestCase "version" getVersion
 	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
@@ -77,8 +77,7 @@ setup = do
 	writeFile testFile "test file contents"
 
 cleanup :: IO ()
-cleanup = do
-	removeDirectoryRecursive tmpDir
+cleanup = removeDirectoryRecursive tmpDir
 
 main :: IO ()
 main = do
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index a64552c721..29ac63aea1 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -58,10 +58,10 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params =
 	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
-		dispatch (cmd:(filterparams params)) cmds commonOptions header
+		dispatch (cmd : filterparams params) cmds commonOptions header
 
 external :: [String] -> IO ()
-external params = do
+external params =
 	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
 		error "git-shell failed"
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 29ad80a58a..bbbcbf9fb4 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110707
+Version: 3.20110708
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 38df0df6a2..e763376077 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -23,8 +23,7 @@ tmpIndex :: Git.Repo -> FilePath
 tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
-setup g = do
-	cleanup g -- idempotency
+setup g = cleanup g -- idempotency
 
 cleanup :: Git.Repo -> IO ()
 cleanup g = do
@@ -34,7 +33,7 @@ cleanup g = do
 parseArgs :: IO [String]
 parseArgs = do
 	args <- getArgs
-	if (length args /= 3)
+	if length args /= 3
 		then usage
 		else return args
 
diff --git a/test.hs b/test.hs
index 51ccc600e6..2352df36a6 100644
--- a/test.hs
+++ b/test.hs
@@ -19,7 +19,7 @@ import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import Control.Exception (throw)
-import Maybe
+import Data.Maybe
 import qualified Data.Map as M
 import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
@@ -48,7 +48,7 @@ instance Arbitrary Types.Key.Key where
 	arbitrary = do
 		n <- arbitrary
 		b <- elements ['A'..'Z']
-		return $ Types.Key.Key {
+		return Types.Key.Key {
 			Types.Key.keyName = n,
 			Types.Key.keyBackendName = [b],
 			Types.Key.keySize = Nothing,
@@ -278,7 +278,7 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	-- write different content, to verify that lock
 	-- throws it away
 	changecontent annexedfile
-	writeFile annexedfile $ (content annexedfile) ++ "foo"
+	writeFile annexedfile $ content annexedfile ++ "foo"
 	git_annex "lock" ["-q", annexedfile] @? "lock failed"
 	annexed_present annexedfile
 	git_annex "unlock" ["-q", annexedfile] @? "unlock failed"		
@@ -287,7 +287,7 @@ test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	git_annex "add" ["-q", annexedfile] @? "add of modified file failed"
 	runchecks [checklink, checkunwritable] annexedfile
 	c <- readFile annexedfile
-	assertEqual ("content of modified file") c (changedcontent annexedfile)
+	assertEqual "content of modified file" c (changedcontent annexedfile)
 	r' <- git_annex "drop" ["-q", annexedfile]
 	not r' @? "drop wrongly succeeded with no known copy of modified file"
 
@@ -312,9 +312,9 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 					@? "git commit of edited file failed"
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
-		assertEqual ("content of modified file") c (changedcontent annexedfile)
+		assertEqual "content of modified file" c (changedcontent annexedfile)
 		r <- git_annex "drop" ["-q", annexedfile]
-		(not r) @? "drop wrongly succeeded with no known copy of modified file"
+		not r @? "drop wrongly succeeded with no known copy of modified file"
 
 test_fix :: Test
 test_fix = "git-annex fix" ~: intmpclonerepo $ do
@@ -331,7 +331,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
 	c <- readFile newfile
-	assertEqual ("content of moved file") c (content annexedfile)
+	assertEqual "content of moved file" c (content annexedfile)
 	where
 		subdir = "s"
 		newfile = subdir ++ "/" ++ annexedfile

From 185f0b687081f47d059cc0503f4f6b671868f753 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 15 Jul 2011 15:33:35 +0000
Subject: [PATCH 2066/8313]

---
 ...nannex_command_doesn__39__t_all_files.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/unannex_command_doesn__39__t_all_files.mdwn

diff --git a/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
new file mode 100644
index 0000000000..c3128c0fc2
--- /dev/null
+++ b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
@@ -0,0 +1,21 @@
+    $ git init ; git annex init test ; dd if=/dev/urandom of=file1 count=128 ; cp file1 file2 ; git annex add --backend=SHA1 file? ; git commit -m init ; git annex unannex ; ls -l
+    Initialized empty Git repository in /tmp/annex/.git/
+    init test ok
+    128+0 records in
+    128+0 records out
+    65536 bytes (66 kB) copied, 0.007173 s, 9.1 MB/s
+    add file1 (checksum...) ok
+    add file2 (checksum...) ok
+    (Recording state in git...)
+    [master (root-commit) 2177b10] init
+     2 files changed, 2 insertions(+), 0 deletions(-)
+     create mode 120000 file1
+     create mode 120000 file2
+    unannex file1 ok
+    (Recording state in git...)
+    [master bef78b1] content removed from git annex
+     1 files changed, 0 insertions(+), 1 deletions(-)
+     delete mode 120000 file1
+    total 72
+    -rw-r--r-- 1 simons users 65536 Jul 15 17:29 file1
+    lrwxrwxrwx 1 simons users   132 Jul 15 17:29 file2 -> .git/annex/objects/jp/Fk/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f

From 6c396a256c93464d726c66a95132536941871ee8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:47:14 -0400
Subject: [PATCH 2067/8313] finished hlint pass

---
 Command/Add.hs        |  6 +++---
 Command/DropUnused.hs |  9 ++++-----
 Command/Fsck.hs       |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/InitRemote.hs |  8 ++++----
 Command/Map.hs        | 19 ++++++++++---------
 Command/Move.hs       |  2 +-
 Command/SetKey.hs     |  4 ++--
 Command/Status.hs     | 16 ++++++++--------
 Command/Uninit.hs     |  5 +++--
 Command/Unused.hs     | 16 ++++++++--------
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  4 ++--
 Locations.hs          |  4 ++--
 Remote/Bup.hs         | 20 +++++++++++---------
 Remote/Directory.hs   |  5 +++--
 Remote/Encryptable.hs |  4 ++--
 Remote/Git.hs         |  6 +++---
 Remote/Hook.hs        | 16 +++++++++-------
 Remote/Rsync.hs       | 12 ++++++------
 Remote/S3real.hs      | 24 ++++++++++--------------
 Remote/Special.hs     |  2 +-
 Remote/Ssh.hs         |  2 +-
 Remote/Web.hs         |  4 ++--
 Touch.hsc             |  5 ++---
 Utility/CopyFile.hs   | 12 +++++-------
 Utility/DataUnits.hs  |  6 +++---
 Utility/Dot.hs        | 10 +++++-----
 Utility/RsyncFile.hs  |  2 +-
 29 files changed, 114 insertions(+), 115 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 51b95b9b5b..58c0143dd0 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -39,7 +39,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
 start :: CommandStartBackendFile
 start pair@(file, _) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
-	if (isSymbolicLink s) || (not $ isRegularFile s)
+	if isSymbolicLink s || not (isRegularFile s)
 		then stop
 		else do
 			showStart "add" file
@@ -58,8 +58,8 @@ perform (file, backend) = do
  - This can be called before or after the symlink is in place. -}
 undo :: FilePath -> Key -> IOException -> Annex a
 undo file key e = do
-	unlessM (inAnnex key) $ rethrow -- no cleanup to do
-	liftIO $ whenM (doesFileExist file) $ do removeFile file
+	unlessM (inAnnex key) rethrow -- no cleanup to do
+	liftIO $ whenM (doesFileExist file) $ removeFile file
 	handle tryharder $ fromAnnex key file
 	logStatus key InfoMissing
 	rethrow
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 55007c1f73..a01e08ab51 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -49,7 +49,7 @@ start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 	]
 	where
 		search [] = stop
-		search ((m, a):rest) = do
+		search ((m, a):rest) =
 			case M.lookup s m of
 				Nothing -> search rest
 				Just key -> do
@@ -78,10 +78,9 @@ readUnusedLog prefix = do
 	let f = gitAnnexUnusedLog prefix g
 	e <- liftIO $ doesFileExist f
 	if e
-		then do
-			l <- liftIO $ readFile f
-			return $ M.fromList $ map parse $ lines l
-		else return $ M.empty
+		then return . M.fromList . map parse . lines
+			=<< liftIO (readFile f)
+		else return M.empty
 	where
 		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)
 			where
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index ec3f1d8e7d..0d3ecb58f1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -94,7 +94,7 @@ fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 fsckKey backend key file numcopies = do
 	size_ok <- checkKeySize key
 	copies_ok <- checkKeyNumCopies key file numcopies
-	backend_ok <-(Types.Backend.fsckKey backend) key
+	backend_ok <- (Types.Backend.fsckKey backend) key
 	return $ size_ok && copies_ok && backend_ok
 
 {- The size of the data for a key is checked against the size encoded in
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index b5b59ccf7d..24f7162ace 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -25,4 +25,4 @@ start key = do
 	present <- inAnnex key
 	if present
 		then stop
-		else liftIO $ exitFailure
+		else liftIO exitFailure
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 15962ad991..9859308e56 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -24,7 +24,7 @@ import Messages
 command :: [Command]
 command = [repoCommand "initremote"
 	(paramPair paramName $
-		paramOptional $ paramRepeating $ paramKeyValue) seek
+		paramOptional $ paramRepeating paramKeyValue) seek
 	"sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
@@ -32,7 +32,7 @@ seek = [withWords start]
 
 start :: CommandStartWords
 start ws = do
-	when (null ws) $ needname
+	when (null ws) needname
 
 	(u, c) <- findByName name
 	let fullconfig = M.union config c	
@@ -69,7 +69,7 @@ findByName name = do
 	maybe generate return $ findByName' name m
 	where
 		generate = do
-			uuid <- liftIO $ genUUID
+			uuid <- liftIO genUUID
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
@@ -85,7 +85,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 remoteNames :: Annex [String]
 remoteNames = do
 	m <- RemoteLog.readRemoteLog
-	return $ catMaybes $ map ((M.lookup nameKey) . snd) $ M.toList m
+	return $ mapMaybe (M.lookup nameKey . snd) $ M.toList m
 
 {- find the specified remote type -}
 findType :: R.RemoteConfig -> Annex (R.RemoteType Annex)
diff --git a/Command/Map.hs b/Command/Map.hs
index 0391ab8e8f..557ae27871 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -12,6 +12,7 @@ import Control.Exception.Extensible
 import System.Cmd.Utils
 import qualified Data.Map as M
 import Data.List.Utils
+import Data.Maybe
 
 import Command
 import qualified Annex
@@ -58,7 +59,7 @@ start = do
  - the repositories first, followed by uuids that were not matched
  - to a repository.
  -}
-drawMap :: [Git.Repo] -> (M.Map UUID String) -> [UUID] -> String
+drawMap :: [Git.Repo] -> M.Map UUID String -> [UUID] -> String
 drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 	where
 		repos = map (node umap rs) rs
@@ -78,23 +79,23 @@ basehostname r = head $ split "." $ hostname r
 
 {- A name to display for a repo. Uses the name from uuid.log if available,
  - or the remote name if not. -}
-repoName :: (M.Map UUID String) -> Git.Repo -> String
+repoName :: M.Map UUID String -> Git.Repo -> String
 repoName umap r
 	| null repouuid = fallback
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
-		fallback = maybe "unknown" id $ Git.repoRemoteName r
+		fallback = fromMaybe "unknown" $ Git.repoRemoteName r
 
 {- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
 nodeId r =
-	case (getUncachedUUID r) of
+	case getUncachedUUID r of
 		"" -> Git.repoLocation r
 		u -> u
 
 {- A node representing a repo. -}
-node :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> String
+node :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> String
 node umap fullinfo r = unlines $ n:edges
 	where
 		n = Dot.subGraph (hostname r) (basehostname r) "lightblue" $
@@ -105,14 +106,14 @@ node umap fullinfo r = unlines $ n:edges
 			| otherwise = reachable
 
 {- An edge between two repos. The second repo is a remote of the first. -}
-edge :: (M.Map UUID String) -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
+edge :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> Git.Repo -> String	
 edge umap fullinfo from to =
 	Dot.graphEdge (nodeId from) (nodeId fullto) edgename
 	where
 		-- get the full info for the remote, to get its UUID
 		fullto = findfullinfo to
 		findfullinfo n =
-			case (filter (same n) fullinfo) of
+			case filter (same n) fullinfo of
 				[] -> n
 				(n':_) -> n'
 		{- Only name an edge if the name is different than the name
@@ -120,7 +121,7 @@ edge umap fullinfo from to =
 		 - different from its hostname. (This reduces visual clutter.) -}
 		edgename = maybe Nothing calcname $ Git.repoRemoteName to
 		calcname n
-			| n == repoName umap fullto || n == hostname fullto = Nothing
+			| n `elem` [repoName umap fullto, hostname fullto] = Nothing
 			| otherwise = Just n
 
 unreachable :: String -> String
@@ -188,7 +189,7 @@ tryScan r
 	| otherwise = safely $ Git.configRead r
 	where
 		safely a = do
-			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
 			case result of
 				Left _ -> return Nothing
 				Right r' -> return $ Just r'
diff --git a/Command/Move.hs b/Command/Move.hs
index 6bf6e05827..a98276e7ec 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -124,7 +124,7 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
 	remotes <- Remote.keyPossibilities key
-	if (u == Remote.uuid src) || (null $ filter (== src) remotes)
+	if u == Remote.uuid src || not (any (== src) remotes)
 		then stop
 		else do
 			showAction move file
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index b000a4e8bf..f2a5259bac 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -16,7 +16,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [repoCommand "setkey" (paramPath) seek
+command = [repoCommand "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
@@ -34,7 +34,7 @@ perform file = do
 	-- the file might be on a different filesystem, so mv is used
 	-- rather than simply calling moveToObjectDir; disk space is also
 	-- checked this way.
-	ok <- getViaTmp key $ \dest -> do
+	ok <- getViaTmp key $ \dest ->
 		if dest /= file
 			then liftIO $
 				boolSystem "mv" [File file, File dest]
diff --git a/Command/Status.hs b/Command/Status.hs
index 1ec4782362..aef4df2329 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -32,8 +32,8 @@ type Stat = StatState (Maybe (String, StatState String))
 
 -- cached info that multiple Stats may need
 data StatInfo = StatInfo
-	{ keysPresentCache :: (Maybe (SizeList Key))
-	, keysReferencedCache :: (Maybe (SizeList Key))
+	{ keysPresentCache :: Maybe (SizeList Key)
+	, keysReferencedCache :: Maybe (SizeList Key)
 	}
 
 -- a state monad for running Stats in
@@ -84,7 +84,7 @@ stat :: String -> StatState String -> Stat
 stat desc a = return $ Just (desc, a)
 
 nostat :: Stat
-nostat = return $ Nothing
+nostat = return Nothing
 
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
@@ -144,7 +144,7 @@ cachedKeysPresent = do
 	case keysPresentCache s of
 		Just v -> return v
 		Nothing -> do
-			keys <- lift $ getKeysPresent
+			keys <- lift getKeysPresent
 			let v = sizeList keys
 			put s { keysPresentCache = Just v }
 			return v
@@ -155,7 +155,7 @@ cachedKeysReferenced = do
 	case keysReferencedCache s of
 		Just v -> return v
 		Nothing -> do
-			keys <- lift $ Command.Unused.getKeysReferenced
+			keys <- lift Command.Unused.getKeysReferenced
 			-- A given key may be referenced repeatedly.
 			-- nub does not seem too slow (yet)..
 			let v = sizeList $ nub keys
@@ -164,9 +164,9 @@ cachedKeysReferenced = do
 
 keySizeSum :: SizeList Key -> StatState String
 keySizeSum (keys, len) = do
-	let knownsize = catMaybes $ map keySize keys
-	let total = roughSize storageUnits False $ foldl (+) 0 knownsize
-	let missing = len - genericLength knownsize
+	let knownsizes = mapMaybe keySize keys
+	let total = roughSize storageUnits False $ sum knownsizes
+	let missing = len - genericLength knownsizes
 	return $ total ++
 		if missing > 0
 			then aside $ "but " ++ show missing ++ " keys have unknown size"
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 1497bbfd19..8b8d7e364d 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -52,8 +52,9 @@ cleanup = do
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
 	-- avoid normal shutdown
 	saveState
-	liftIO $ Git.run g "branch" [Param "-D", Param Branch.name]
-	liftIO $ exitSuccess
+	liftIO $ do
+		Git.run g "branch" [Param "-D", Param Branch.name]
+		exitSuccess
 
 gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
 gitPreCommitHookUnWrite repo = do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 3f51e2c2c2..870c993f18 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -7,7 +7,7 @@
 
 module Command.Unused where
 
-import Control.Monad (filterM, unless, forM_, when)
+import Control.Monad (filterM, unless, forM_)
 import Control.Monad.State (liftIO)
 import qualified Data.Set as S
 import Data.Maybe
@@ -55,9 +55,9 @@ checkUnused = do
 	where
 		list file msg l c = do
 			let unusedlist = number c l
-			when (not $ null l) $ do
+			unless (null l) $ do
 				showLongNote $ msg unusedlist
-				showLongNote $ "\n"
+				showLongNote "\n"
 			writeUnusedFile file unusedlist
 			return $ c + length l
 
@@ -68,7 +68,7 @@ checkRemoteUnused name = do
 
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
-	showNote $ "checking for unused data..."
+	showNote "checking for unused data..."
 	referenced <- getKeysReferenced
 	remotehas <- filterM isthere =<< loggedKeys
 	let remoteunused = remotehas `exclude` referenced
@@ -76,7 +76,7 @@ checkRemoteUnused' r = do
 	writeUnusedFile "" list
 	unless (null remoteunused) $ do
 		showLongNote $ remoteUnusedMsg r list
-		showLongNote $ "\n"
+		showLongNote "\n"
 	where
 		isthere k = do
 			us <- keyLocations k
@@ -90,14 +90,14 @@ writeUnusedFile prefix l = do
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
 table :: [(Int, Key)] -> [String]
-table l = ["  NUMBER  KEY"] ++ map cols l
+table l = "  NUMBER  KEY" : map cols l
 	where
 		cols (n,k) = "  " ++ pad 6 (show n) ++ "  " ++ show k
 		pad n s = s ++ replicate (n - length s) ' '
 
 number :: Int -> [a] -> [(Int, a)]
 number _ [] = []
-number n (x:xs) = (n+1, x):(number (n+1) xs)
+number n (x:xs) = (n+1, x) : number (n+1) xs
 
 staleTmpMsg :: [(Int, Key)] -> String
 staleTmpMsg t = unlines $ 
@@ -210,4 +210,4 @@ staleKeys dirspec = do
 			contents <- liftIO $ getDirectoryContents dir
 			files <- liftIO $ filterM doesFileExist $
 				map (dir ) contents
-			return $ catMaybes $ map (fileKey . takeFileName) files
+			return $ mapMaybe (fileKey . takeFileName) files
diff --git a/Command/Version.hs b/Command/Version.hs
index bb7acd12dd..2392c9bf6b 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -31,4 +31,4 @@ start = do
 	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
 	stop
 	where
-		vs l = join " " l
+		vs = join " "
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 0e4858f8b7..05748e8d60 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -30,11 +30,11 @@ perform key = do
 	uuids <- keyLocations key
 	let num = length uuids
 	showNote $ show num ++ " " ++ copiesplural num
-	if null $ uuids
+	if null uuids
 		then stop
 		else do
 			pp <- prettyPrintUUIDs uuids
-			showLongNote $ pp
+			showLongNote pp
 			showProgress	
 			next $ return True
 	where
diff --git a/Locations.hs b/Locations.hs
index 2dbf2f55ea..942b687bbf 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -167,8 +167,8 @@ display_32bits_as_dir w = trim $ swap_pairs cs
 		-- a real word, use letters that appear less frequently.
 		chars = ['0'..'9'] ++ "zqjxkmvwgpfZQJXKMVWGPF"
 		cs = map (\x -> getc $ (shiftR w (6*x)) .&. 31) [0..7]
-		getc n = chars !! (fromIntegral n)
+		getc n = chars !! fromIntegral n
 		swap_pairs (x1:x2:xs) = x2:x1:swap_pairs xs
 		swap_pairs _ = []
 		-- Last 2 will always be 00, so omit.
-		trim s = take 6 s
+		trim = take 6
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 5a44397f0d..4ea455226e 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -8,7 +8,8 @@
 module Remote.Bup (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import IO
+import System.IO
+import System.IO.Error
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad (when)
@@ -16,6 +17,7 @@ import Control.Monad.State (liftIO)
 import System.Process
 import System.Exit
 import System.FilePath
+import Data.Maybe
 import Data.List.Utils
 import System.Cmd.Utils
 
@@ -68,7 +70,7 @@ gen r u c = do
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 bupSetup u c = do
 	-- verify configuration is sane
-	let buprepo = maybe (error "Specify buprepo=") id $
+	let buprepo = fromMaybe (error "Specify buprepo=") $
 		M.lookup "buprepo" c
 	c' <- encryptionSetup c
 
@@ -87,7 +89,7 @@ bupSetup u c = do
 
 bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam]
 bupParams command buprepo params = 
-	(Param command) : [Param "-r", Param buprepo] ++ params
+	Param command : [Param "-r", Param buprepo] ++ params
 
 bup :: String -> BupRepo -> [CommandParam] -> Annex Bool
 bup command buprepo params = do
@@ -123,8 +125,8 @@ storeEncrypted r buprepo (cipher, enck) k = do
 	g <- Annex.gitRepo
 	let src = gitAnnexLocation g k
 	params <- bupSplitParams r buprepo enck (Param "-")
-	liftIO $ catchBool $ do
-		withEncryptedHandle cipher (L.readFile src) $ \h -> do
+	liftIO $ catchBool $
+		withEncryptedHandle cipher (L.readFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
 retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
@@ -184,7 +186,7 @@ onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [
 onBupRemote r a command params = do
 	let dir = shellEscape (Git.workTree r)
 	sshparams <- sshToRepo r [Param $
-			"cd " ++ dir ++ " && " ++ (unwords $ command : toCommand params)]
+			"cd " ++ dir ++ " && " ++ unwords (command : toCommand params)]
 	liftIO $ a "ssh" sshparams
 
 {- Allow for bup repositories on removable media by checking
@@ -215,20 +217,20 @@ bup2GitRemote "" = do
 	Git.repoFromAbsPath $ h  ".bup"
 bup2GitRemote r
 	| bupLocal r = 
-		if r !! 0 == '/'
+		if head r == '/'
 			then Git.repoFromAbsPath r
 			else error "please specify an absolute path"
 	| otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir
 		where
 			bits = split ":" r
-			host = bits !! 0
+			host = head bits
 			dir = join ":" $ drop 1 bits
 			-- "host:~user/dir" is not supported specially by bup;
 			-- "host:dir" is relative to the home directory;
 			-- "host:" goes in ~/.bup
 			slash d
 				| d == "" = "/~/.bup"
-				| d !! 0 == '/' = d
+				| head d == '/' = d
 				| otherwise = "/~/" ++ d
 
 bupLocal :: BupRepo -> Bool
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 05d42136f1..235f613000 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -8,13 +8,14 @@
 module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import IO
+import System.IO.Error
 import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory hiding (copyFile)
 import System.FilePath
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -60,7 +61,7 @@ gen r u c = do
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 directorySetup u c = do
 	-- verify configuration is sane
-	let dir = maybe (error "Specify directory=") id $
+	let dir = fromMaybe (error "Specify directory=") $
 		M.lookup "directory" c
 	liftIO $ doesDirectoryExist dir
 		>>! error $ "Directory does not exist: " ++ dir
diff --git a/Remote/Encryptable.hs b/Remote/Encryptable.hs
index 443f5cf83d..66e1738ac2 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Encryptable.hs
@@ -56,10 +56,10 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 	where
 		store k = cip k >>= maybe
 			(storeKey r k)
-			(\x -> storeKeyEncrypted x k)
+			(`storeKeyEncrypted` k)
 		retrieve k f = cip k >>= maybe
 			(retrieveKeyFile r k f)
-			(\x -> retrieveKeyFileEncrypted x f)
+			(`retrieveKeyFileEncrypted` f)
 		withkey a k = cip k >>= maybe (a k) (a . snd)
 		cip = cipherKey c
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index fb85123825..1f22ad11c6 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -57,7 +57,7 @@ gen r u _ = do
 	let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost
 	cst <- remoteCost r' defcst
 
-	return $ Remote {
+	return Remote {
 		uuid = u',
 		cost = cst,
 		name = Git.repoDescribe r',
@@ -81,7 +81,7 @@ tryGitConfigRead r
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
 		safely a = do
-			result <- liftIO (try (a)::IO (Either SomeException Git.Repo))
+			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
 			case result of
 				Left _ -> return r
 				Right r' -> return r'
@@ -154,7 +154,7 @@ copyToRemote r key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
-rsyncHelper :: [CommandParam] -> Annex (Bool)
+rsyncHelper :: [CommandParam] -> Annex Bool
 rsyncHelper p = do
 	showProgress -- make way for progress bar
 	res <- liftIO $ rsync p
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 86a7bca560..f0e4d5bfbc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -17,6 +17,7 @@ import System.Posix.IO
 import System.IO
 import System.IO.Error (try)
 import System.Exit
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -61,7 +62,7 @@ gen r u c = do
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 hookSetup u c = do
-	let hooktype = maybe (error "Specify hooktype=") id $
+	let hooktype = fromMaybe (error "Specify hooktype=") $
 		M.lookup "hooktype" c
 	c' <- encryptionSetup c
 	gitConfigSpecialRemote u c' "hooktype" hooktype
@@ -73,12 +74,13 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 		env s v = ("ANNEX_" ++ s, v)
 		keyenv =
 			[ env "KEY" (show k)
-			, env "HASH_1" (hashbits !! 0)
-			, env "HASH_2" (hashbits !! 1)
+			, env "HASH_1" hash_1
+			, env "HASH_2" hash_2
 			]
 		fileenv Nothing = []
 		fileenv (Just file) =  [env "FILE" file]
-		hashbits = map takeDirectory $ splitPath $ hashDirMixed k
+		[hash_1, hash_2, _rest] =
+			map takeDirectory $ splitPath $ hashDirMixed k
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
@@ -127,7 +129,7 @@ retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
 		return True
 
 remove :: String -> Key -> Annex Bool
-remove h k = runHook h "remove" k Nothing $ do return True
+remove h k = runHook h "remove" k Nothing $ return True
 
 checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)
 checkPresent r h k = do
@@ -135,7 +137,7 @@ checkPresent r h k = do
 	v <- lookupHook h "checkpresent"
 	liftIO (try (check v) ::IO (Either IOException Bool))
 	where
-		findkey s = (show k) `elem` (lines s)
+		findkey s = show k `elem` lines s
 		env = hookEnv k Nothing
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
@@ -150,5 +152,5 @@ checkPresent r h k = do
 			hClose fromh
 			s <- getProcessStatus True False pid
 			case s of
-				Just (Exited (ExitSuccess)) -> return $ findkey reply
+				Just (Exited ExitSuccess) -> return $ findkey reply
 				_ -> error "checkpresent hook failed"
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 80e194fed1..ca4236276f 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -15,6 +15,7 @@ import System.FilePath
 import System.Directory
 import System.Posix.Files
 import System.Posix.Process
+import Data.Maybe
 
 import Types
 import Types.Remote
@@ -82,7 +83,7 @@ genRsyncOpts r = do
 rsyncSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 rsyncSetup u c = do
 	-- verify configuration is sane
-	let url = maybe (error "Specify rsyncurl=") id $
+	let url = fromMaybe (error "Specify rsyncurl=") $
 		M.lookup "rsyncurl" c
 	c' <- encryptionSetup c
 
@@ -160,10 +161,10 @@ partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"
 withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
 	g <- Annex.gitRepo
-	pid <- liftIO $ getProcessID
+	pid <- liftIO getProcessID
 	let tmp = gitAnnexTmpDir g  "rsynctmp"  show pid
 	nuke tmp
-	liftIO $ createDirectoryIfMissing True $ tmp
+	liftIO $ createDirectoryIfMissing True tmp
 	res <- a tmp
 	nuke tmp
 	return res
@@ -189,15 +190,14 @@ rsyncRemote o params = do
 rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool
 rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
 	let dest = tmp  hashDirMixed k  f  f
-	liftIO $ createDirectoryIfMissing True $ parentDir $ dest
+	liftIO $ createDirectoryIfMissing True $ parentDir dest
 	liftIO $ createLink src dest
-	res <- rsyncRemote o
+	rsyncRemote o
 		[ Param "--recursive"
 		, partialParams
  		  -- tmp/ to send contents of tmp dir
 		, Param $ addTrailingPathSeparator tmp
 		, Param $ rsyncUrl o
 		]
-	return res
 	where
 		f = keyFile k
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 52d1ed1be1..cbd3ef6225 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -52,7 +52,7 @@ gen r u c = do
 	cst <- remoteCost r expensiveRemoteCost
 	return $ gen' r u c cst
 gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex
-gen' r u c cst = do
+gen' r u c cst =
 	encryptableRemote c
 		(storeEncrypted this)
 		(retrieveEncrypted this)
@@ -85,7 +85,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 		
 		handlehost Nothing = defaulthost
 		handlehost (Just h)
-			| ".archive.org" `isSuffixOf` (map toLower h) = archiveorg
+			| ".archive.org" `isSuffixOf` map toLower h = archiveorg
 			| otherwise = defaulthost
 
 		use fullconfig = do
@@ -99,7 +99,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 			use fullconfig
 
 		archiveorg = do
-			showNote $ "Internet Archive mode"
+			showNote "Internet Archive mode"
 			maybe (error "specify bucket=") (const $ return ()) $
 				M.lookup "bucket" archiveconfig
 			use archiveconfig
@@ -203,10 +203,8 @@ s3Error :: ReqError -> a
 s3Error e = error $ prettyReqError e
 
 s3Bool :: AWSResult () -> Annex Bool
-s3Bool res = do
-	case res of
-		Right _ -> return True
-		Left e -> s3Warning e
+s3Bool (Right _) = return True
+s3Bool (Left e) = s3Warning e
 
 s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r noconn action = do
@@ -219,7 +217,7 @@ s3Action r noconn action = do
 		_ -> return noconn
 
 bucketFile :: Remote Annex -> Key -> FilePath
-bucketFile r k = (munge $ show k)
+bucketFile r = munge . show
 	where
 		munge s = case M.lookup "mungekeys" $ fromJust $ config r of
 			Just "ia" -> iaMunge s
@@ -271,8 +269,8 @@ s3Connection c = do
 			warning $ "Set both " ++ s3AccessKey ++ " and " ++ s3SecretKey  ++ " to use S3"
 			return Nothing
 	where
-		host = fromJust $ (M.lookup "host" c)
-		port = let s = fromJust $ (M.lookup "port" c) in
+		host = fromJust $ M.lookup "host" c
+		port = let s = fromJust $ M.lookup "port" c in
 			case reads s of
 			[(p, _)] -> p
 			_ -> error $ "bad S3 port value: " ++ s
@@ -283,7 +281,7 @@ s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String))
 s3GetCreds c = do
 	ak <- getEnvKey s3AccessKey
 	sk <- getEnvKey s3SecretKey
-	if (null ak || null sk)
+	if null ak || null sk
 		then do
 			mcipher <- remoteCipher c
 			case (M.lookup "s3creds" c, mcipher) of
@@ -291,9 +289,7 @@ s3GetCreds c = do
 					s <- liftIO $ withDecryptedContent cipher
 						(return $ L.pack $ fromB64 encrypted)
 						(return . L.unpack)
-					let line = lines s
-					let ak' = line !! 0
-					let sk' = line !! 1
+					let [ak', sk', _rest] = lines s
 					liftIO $ do
 						setEnv s3AccessKey ak True
 						setEnv s3SecretKey sk True
diff --git a/Remote/Special.hs b/Remote/Special.hs
index 9a00dbd82e..d6f362ce30 100644
--- a/Remote/Special.hs
+++ b/Remote/Special.hs
@@ -38,7 +38,7 @@ gitConfigSpecialRemote u c k v = do
 	g <- Annex.gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting $ "annex-uuid"), Param u]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
 	where
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/Ssh.hs b/Remote/Ssh.hs
index 0d4842a1ab..fe4e6dfc1a 100644
--- a/Remote/Ssh.hs
+++ b/Remote/Ssh.hs
@@ -39,7 +39,7 @@ git_annex_shell r command params
 	where
 		dir = Git.workTree r
 		shellcmd = "git-annex-shell"
-		shellopts = (Param command):(File dir):params
+		shellopts = Param command : File dir : params
 		sshcmd = shellcmd ++ " " ++ 
 			unwords (map shellEscape $ toCommand shellopts)
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index d3d140d73b..60f64cfe02 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -52,7 +52,7 @@ webUUID = "00000000-0000-0000-0000-000000000001"
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
-	return $ Remote {
+	return Remote {
 		uuid = webUUID,
 		cost = expensiveRemoteCost,
 		name = Git.repoDescribe r,
@@ -111,7 +111,7 @@ checkKey' (u:us) = do
 	if e then return e else checkKey' us
 
 urlexists :: URLString -> IO Bool
-urlexists url = do
+urlexists url =
 	case parseURI url of
 		Nothing -> return False
 		Just u -> do
diff --git a/Touch.hsc b/Touch.hsc
index 4f26855d21..dd0c38984e 100644
--- a/Touch.hsc
+++ b/Touch.hsc
@@ -15,6 +15,7 @@ module Touch (
 
 import Foreign
 import Foreign.C
+import Control.Monad (when)
 
 newtype TimeSpec = TimeSpec CTime
 
@@ -66,9 +67,7 @@ touchBoth file atime mtime follow =
 	withCString file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- c_utimensat at_fdcwd f ptr flags
-		if (r /= 0)
-			then throwErrno "touchBoth"
-			else return ()
+		when (r /= 0) $ throwErrno "touchBoth"
 	where
 		flags = if follow
 			then 0
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 5ee4a91df7..2e06dd92bb 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -20,10 +20,8 @@ copyFile src dest = do
 		removeFile dest
 	boolSystem "cp" [params, File src, File dest]
 	where
-		params = if SysConfig.cp_reflink_auto
-			then Params "--reflink=auto"
-			else if SysConfig.cp_a
-				then Params "-a"
-				else if SysConfig.cp_p
-					then Params "-p"
-					else Params ""
+		params
+			| SysConfig.cp_reflink_auto = Params "--reflink=auto"
+			| SysConfig.cp_a = Params "-a"
+			| SysConfig.cp_p = Params "-p"
+			| otherwise = Params ""
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index 7af2eadafb..f2bc333ea0 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -106,7 +106,7 @@ oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
 {- approximate display of a particular number of bytes -}
 roughSize :: [Unit] -> Bool -> ByteSize -> String
 roughSize units abbrev i
-	| i < 0 = "-" ++ findUnit units' (negate i)
+	| i < 0 = '-' : findUnit units' (negate i)
 	| otherwise = findUnit units' i
 	where
 		units' = reverse $ sort units -- largest first
@@ -139,10 +139,10 @@ readSize :: [Unit] -> String -> Maybe ByteSize
 readSize units input
 	| null parsednum = Nothing
 	| null parsedunit = Nothing
-	| otherwise = Just $ round $ number * (fromIntegral multiplier)
+	| otherwise = Just $ round $ number * fromIntegral multiplier
 	where
 		(number, rest) = head parsednum
-		multiplier = head $ parsedunit
+		multiplier = head parsedunit
 		unitname = takeWhile isAlpha $ dropWhile isSpace rest
 
 		parsednum = reads input :: [(Double, String)]
diff --git a/Utility/Dot.hs b/Utility/Dot.hs
index 8696849963..83f52a3cc1 100644
--- a/Utility/Dot.hs
+++ b/Utility/Dot.hs
@@ -20,13 +20,13 @@ graphNode nodeid desc = label desc $ quote nodeid
 
 {- an edge between two nodes -}
 graphEdge :: String -> String -> Maybe String -> String
-graphEdge fromid toid desc = indent $ maybe edge (\d -> label d edge) desc
+graphEdge fromid toid desc = indent $ maybe edge (`label` edge) desc
 	where
 		edge = quote fromid ++ " -> " ++ quote toid
 
 {- adds a label to a node or edge -}
 label :: String -> String -> String
-label l s = attr "label" l s
+label = attr "label"
 
 {- adds an attribute to a node or edge
  - (can be called multiple times for multiple attributes) -}
@@ -35,7 +35,7 @@ attr a v s = s ++ " [ " ++ a ++ "=" ++ quote v ++ " ]"
 
 {- fills a node with a color -}
 fillColor :: String -> String -> String
-fillColor color s = attr "fillcolor" color $ attr "style" "filled" $ s
+fillColor color s = attr "fillcolor" color $ attr "style" "filled" s
 
 {- apply to graphNode to put the node in a labeled box -}
 subGraph :: String -> String -> String -> String -> String
@@ -52,10 +52,10 @@ subGraph subid l color s =
 		setlabel = "label=" ++ quote l
 		setfilled = "style=" ++ quote "filled"
 		setcolor = "fillcolor=" ++ quote color
-		ii x = (indent $ indent x) ++ "\n"
+		ii x = indent (indent x) ++ "\n"
 
 indent ::String -> String
-indent s = "\t" ++ s
+indent s = '\t' : s
 
 quote :: String -> String
 quote s = "\"" ++ s' ++ "\""
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index c68909d2dc..6e21ba0632 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -19,7 +19,7 @@ rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand comman
 		{- rsync requires some weird, non-shell like quoting in
                  - here. A doubled single quote inside the single quoted
                  - string is a single quote. -}
-		escape s = "'" ++  (join "''" $ split "'" s) ++ "'"
+		escape s = "'" ++  join "''" (split "'" s) ++ "'"
 
 {- Runs rsync in server mode to send a file, and exits. -}
 rsyncServerSend :: FilePath -> IO ()

From 7fa7601490c1346ab7dfcd645fe9b822eddee68a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:51:06 -0400
Subject: [PATCH 2068/8313] response

---
 doc/bugs/unannex_command_doesn__39__t_all_files.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
index c3128c0fc2..8094ed9182 100644
--- a/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
+++ b/doc/bugs/unannex_command_doesn__39__t_all_files.mdwn
@@ -19,3 +19,8 @@
     total 72
     -rw-r--r-- 1 simons users 65536 Jul 15 17:29 file1
     lrwxrwxrwx 1 simons users   132 Jul 15 17:29 file2 -> .git/annex/objects/jp/Fk/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f/SHA1-s65536--795b58cc4e5190b02e7026fd9e94a10c98c6475f
+
+> This was recently discussed in
+> [[annex_unannex__47__uninit_should_handle_copies]] and `unannex --fast`
+> added to leave contents behind in the annex, which allows handling
+> copies. But needs manual cleanup later with dropunused. --[[Joey]] 

From 39ae33386d0b2b3f55a766839e1eabfded3a28e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:53:58 -0400
Subject: [PATCH 2069/8313] update

---
 doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index 249a175055..3fd1a8082c 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -13,3 +13,7 @@ See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
 
 >> Even if there is nothing it can _do_, knowing that the data is intact,
 >> or not, is valuable in and as of itself. -- RichiH
+
+>>> While storing the data is no longer an issue in bare repos, fsck would
+>>> need a special mode that examines all the location logs, since it
+>>> cannot run thru the checked out files. --[[Joey]] 

From 13187b1ba75f800d28a3667aa80a252c9f1a5e29 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jul 2011 12:55:05 -0400
Subject: [PATCH 2070/8313] close

---
 doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn
index 608704d9d7..3c9374100c 100644
--- a/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn
+++ b/doc/bugs/WORM:_Handle_long_filenames_correctly.mdwn
@@ -1 +1,4 @@
 I have files with very long filenames on an xfs at home. On my laptop the annex should have been checked out on an encfs, but there filenames can't be as long as on the xfs. So perhaps it would be good to limit the keysize to a sane substring of the filename e.g. use only the first 120 characters.
+
+> Since there seems no strong argument for a WORM100, and better options
+> exist, closing. [[done]] --[[Joey]] 

From 6a58f61d2870d6631f8a039a6fdc07f262a7e2af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 16 Jul 2011 15:09:57 -0400
Subject: [PATCH 2071/8313] typo

---
 Utility.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility.hs b/Utility.hs
index 786c6a1770..dab3b4d0ea 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -288,6 +288,6 @@ unlessM c a = c >>= flip unless a
 (>>!) :: Monad m => m Bool -> m () -> m ()
 (>>!) = unlessM
 
--- low fixity allows eg, foo bar <|> error $ "failed " ++ meep
+-- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
 infixr 0 >>?
 infixr 0 >>!

From ec9e9343d9fa99b0786ee93ff142484e2402d3c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jul 2011 19:05:50 -0400
Subject: [PATCH 2072/8313] add closure for new bug that I already fixed

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index 9a7c3e7b0f..3a6c731fe4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110708) UNRELEASED; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+    Closes: #634233
   * Bugfix: Make add ../ work.
   * Support the standard git -c name=value
   * unannex: Clean up use of git commit -a.

From 00153eed48a2328969cc08688ef674a4c19c2014 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jul 2011 14:07:23 -0400
Subject: [PATCH 2073/8313] unify elipsis handling

And add a simple dots-based progress display, currently only used in v2
upgrade.
---
 AnnexQueue.hs         |  2 +-
 Backend/SHA.hs        |  2 +-
 Branch.hs             |  2 +-
 Command.hs            |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Get.hs        |  2 +-
 Command/Map.hs        |  8 ++++----
 Command/Move.hs       | 14 +++++++-------
 Command/Unlock.hs     |  2 +-
 Command/Unused.hs     |  4 ++--
 Command/Whereis.hs    |  2 +-
 Messages.hs           | 39 ++++++++++++++++++++++++---------------
 Remote/Bup.hs         | 10 +++++-----
 Remote/Git.hs         |  4 ++--
 Remote/Hook.hs        |  4 ++--
 Remote/Rsync.hs       |  4 ++--
 Remote/S3real.hs      |  6 +++---
 Remote/Web.hs         |  4 ++--
 Upgrade/V0.hs         |  2 +-
 Upgrade/V1.hs         |  8 ++++----
 Upgrade/V2.hs         | 13 +++++++++----
 22 files changed, 76 insertions(+), 62 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index b1678df07f..79116c48af 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -38,7 +38,7 @@ flush silent = do
 	q <- getState repoqueue
 	unless (0 == Git.Queue.size q) $ do
 		unless silent $
-			showSideAction "Recording state in git..."
+			showSideAction "Recording state in git"
 		g <- gitRepo
 		q' <- liftIO $ Git.Queue.flush g q
 		store q'
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index dc27b30003..c1d7136485 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -72,7 +72,7 @@ shaNameE size = shaName size ++ "E"
 
 shaN :: SHASize -> FilePath -> Annex String
 shaN size file = do
-	showNote "checksum..."
+	showAction "checksum"
 	liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do
 		line <- hGetLine h
 		let bits = split " " line
diff --git a/Branch.hs b/Branch.hs
index c8e6bc2bb4..35e3050936 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -190,7 +190,7 @@ updateRef ref
 		if null diffs
 			then return Nothing
 			else do
-				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name ++ "..."
+				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.
diff --git a/Command.hs b/Command.hs
index 729e442fc4..02bbd29d44 100644
--- a/Command.hs
+++ b/Command.hs
@@ -102,7 +102,7 @@ doCommand = start
 		stage a b = b >>= a
 		success = return True
 		failure = do
-			showProgress
+			showOutput -- avoid clutter around error message
 			showEndFail
 			return False
 
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e80fe9621b..1b12362e9f 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -43,7 +43,7 @@ start s = do
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
 	g <- Annex.gitRepo
-	showNote $ "downloading " ++ url
+	showAction $ "downloading " ++ url ++ " "
 	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index a01e08ab51..41bcd6aa78 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -61,7 +61,7 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 	where
 		dropremote name = do
 			r <- Remote.byName name
-			showNote $ "from " ++ Remote.name r ++ "..."
+			showAction $ "from " ++ Remote.name r
 			next $ Command.Move.fromCleanup r True key
 		droplocal = Command.Drop.perform key (Just 0) -- force drop
 
diff --git a/Command/Get.hs b/Command/Get.hs
index cc780cb6a3..e0436a8680 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -75,7 +75,7 @@ getKeyFile key file = do
 						Left _ -> return False
 				else return True
 		docopy r continue = do
-			showNote $ "from " ++ Remote.name r ++ "..."
+			showAction $ "from " ++ Remote.name r
 			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
diff --git a/Command/Map.hs b/Command/Map.hs
index 557ae27871..07f127f14e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -44,7 +44,7 @@ start = do
 
 	liftIO $ writeFile file (drawMap rs umap trusted)
 	showLongNote $ "running: dot -Tx11 " ++ file
-	showProgress
+	showOutput
 	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]
 	next $ next $ return r
 	where
@@ -176,7 +176,7 @@ scan r = do
 			showEndOk
 			return r'
 		Nothing -> do
-			showProgress
+			showOutput
 			showEndFail
 			return r
 
@@ -224,5 +224,5 @@ tryScan r
 				ok -> return ok
 
 		sshnote = do
-			showNote "sshing..."
-			showProgress
+			showAction "sshing"
+			showOutput
diff --git a/Command/Move.hs b/Command/Move.hs
index a98276e7ec..a081a863f2 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -44,9 +44,9 @@ start move file = do
 			fromStart src move file
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 
-showAction :: Bool -> FilePath -> Annex ()
-showAction True file = showStart "move" file
-showAction False file = showStart "copy" file
+showMoveAction :: Bool -> FilePath -> Annex ()
+showMoveAction True file = showStart "move" file
+showMoveAction False file = showStart "copy" file
 
 {- Used to log a change in a remote's having a key. The change is logged
  - in the local repo, not on the remote. The process of transferring the
@@ -77,7 +77,7 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 	if not ishere || u == Remote.uuid dest
 		then stop -- not here, so nothing to do
 		else do
-			showAction move file
+			showMoveAction move file
 			next $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 toPerform dest move key = do
@@ -97,7 +97,7 @@ toPerform dest move key = do
 			showNote $ show err
 			stop
 		Right False -> do
-			showNote $ "to " ++ Remote.name dest ++ "..."
+			showAction $ "to " ++ Remote.name dest
 			ok <- Remote.storeKey dest key
 			if ok
 				then next $ toCleanup dest move key
@@ -127,7 +127,7 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 	if u == Remote.uuid src || not (any (== src) remotes)
 		then stop
 		else do
-			showAction move file
+			showMoveAction move file
 			next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = do
@@ -135,7 +135,7 @@ fromPerform src move key = do
 	if ishere
 		then next $ fromCleanup src move key
 		else do
-			showNote $ "from " ++ Remote.name src ++ "..."
+			showAction $ "from " ++ Remote.name src
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			if ok
 				then next $ fromCleanup src move key
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index d189545f5d..280eff9de7 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -45,7 +45,7 @@ perform dest key = do
 	let src = gitAnnexLocation g key
 	let tmpdest = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
-	showNote "copying..."
+	showAction "copying"
 	ok <- liftIO $ copyFile src tmpdest
         if ok
                 then do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 870c993f18..e7065b3c36 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -68,7 +68,7 @@ checkRemoteUnused name = do
 
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
-	showNote "checking for unused data..."
+	showAction "checking for unused data"
 	referenced <- getKeysReferenced
 	remotehas <- filterM isthere =<< loggedKeys
 	let remoteunused = remotehas `exclude` referenced
@@ -152,7 +152,7 @@ unusedKeys = do
 			bad <- staleKeys gitAnnexBadDir
 			return ([], bad, tmp)
 		else do
-			showNote "checking for unused data..."
+			showAction "checking for unused data"
 			present <- getKeysPresent
 			referenced <- getKeysReferenced
 			let unused = present `exclude` referenced
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 05748e8d60..314fef7826 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -35,7 +35,7 @@ perform key = do
 		else do
 			pp <- prettyPrintUUIDs uuids
 			showLongNote pp
-			showProgress	
+			showOutput
 			next $ return True
 	where
 		copiesplural 1 = "copy"
diff --git a/Messages.hs b/Messages.hs
index 5f150aafb4..36f0b89c5c 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -20,21 +20,29 @@ verbose a = do
 	q <- Annex.getState Annex.quiet
 	unless q a
 
-showSideAction :: String -> Annex ()
-showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ ")"
-
 showStart :: String -> String -> Annex ()
-showStart command file = verbose $ do
-	liftIO $ putStr $ command ++ " " ++ file ++ " "
-	liftIO $ hFlush stdout
+showStart command file = verbose $ liftIO $ do
+	putStr $ command ++ " " ++ file ++ " "
+	hFlush stdout
 
 showNote :: String -> Annex ()
-showNote s = verbose $ do
-	liftIO $ putStr $ "(" ++ s ++ ") "
-	liftIO $ hFlush stdout
+showNote s = verbose $ liftIO $ do
+	putStr $ "(" ++ s ++ ") "
+	hFlush stdout
+
+showAction :: String -> Annex ()
+showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = verbose $ liftIO $ putStr "\n"
+showProgress = verbose $ liftIO $ do
+	putStr "."
+	hFlush stdout
+
+showSideAction :: String -> Annex ()
+showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ "...)"
+
+showOutput :: Annex ()
+showOutput = verbose $ liftIO $ putStr "\n"
 
 showLongNote :: String -> Annex ()
 showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
@@ -50,15 +58,16 @@ showEndResult True = showEndOk
 showEndResult False = showEndFail
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = do
-	liftIO $ hFlush stdout
-	liftIO $ hPutStrLn stderr $ "git-annex: " ++ show e
+showErr e = liftIO $ do
+	hFlush stdout
+	hPutStrLn stderr $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
 warning w = do
 	verbose $ liftIO $ putStr "\n"
-	liftIO $ hFlush stdout
-	liftIO $ hPutStrLn stderr $ indent w
+	liftIO $ do
+		hFlush stdout
+		hPutStrLn stderr $ indent w
 
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4ea455226e..1023cda186 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -76,7 +76,7 @@ bupSetup u c = do
 
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
-	showNote "bup init"
+	showAction "bup init"
 	bup "init" buprepo [] >>! error "bup init failed"
 
 	storeBupUUID u buprepo
@@ -93,7 +93,7 @@ bupParams command buprepo params =
 
 bup :: String -> BupRepo -> [CommandParam] -> Annex Bool
 bup command buprepo params = do
-	showProgress -- make way for bup output
+	showOutput -- make way for bup output
 	liftIO $ boolSystem "bup" $ bupParams command buprepo params
 
 pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool
@@ -109,7 +109,7 @@ bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandPa
 bupSplitParams r buprepo k src = do
 	o <- getConfig r "bup-split-options" ""
 	let os = map Param $ words o
-	showProgress -- make way for bup output
+	showOutput -- make way for bup output
 	return $ bupParams "split" buprepo 
 		(os ++ [Param "-n", Param (show k), src])
 
@@ -157,7 +157,7 @@ remove _ = do
 checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool)
 checkPresent r bupr k
 	| Git.repoIsUrl bupr = do
-		showNote ("checking " ++ Git.repoDescribe r ++ "...")
+		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
 	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params
@@ -172,7 +172,7 @@ storeBupUUID u buprepo = do
 	r <- liftIO $ bup2GitRemote buprepo
 	if Git.repoIsUrl r
 		then do
-			showNote "storing uuid"
+			showAction "storing uuid"
 			onBupRemote r boolSystem "git"
 				[Params $ "config annex.uuid " ++ u]
 					>>! error "ssh failed"
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 1f22ad11c6..de51c891e2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -115,7 +115,7 @@ inAnnex r key = if Git.repoIsUrl r
 			a <- Annex.new r
 			Annex.eval a (Content.inAnnex key)
 		checkremote = do
-			showNote ("checking " ++ Git.repoDescribe r ++ "...")
+			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
@@ -156,7 +156,7 @@ copyToRemote r key
 
 rsyncHelper :: [CommandParam] -> Annex Bool
 rsyncHelper p = do
-	showProgress -- make way for progress bar
+	showOutput -- make way for progress bar
 	res <- liftIO $ rsync p
 	if res
 		then return res
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index f0e4d5bfbc..87f86ffe4f 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -98,7 +98,7 @@ runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
 runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook
 	where
 		run command = do
-			showProgress -- make way for hook output
+			showOutput -- make way for hook output
 			res <- liftIO $ boolSystemEnv
 				"sh" [Param "-c", Param command] $ hookEnv k f
 			if res
@@ -133,7 +133,7 @@ remove h k = runHook h "remove" k Nothing $ return True
 
 checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)
 checkPresent r h k = do
-	showNote ("checking " ++ Git.repoDescribe r ++ "...")
+	showAction $ "checking " ++ Git.repoDescribe r
 	v <- lookupHook h "checkpresent"
 	liftIO (try (check v) ::IO (Either IOException Bool))
 	where
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index ca4236276f..f073e7bd79 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -141,7 +141,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
 checkPresent r o k = do
-	showNote ("checking " ++ Git.repoDescribe r ++ "...")
+	showAction $ "checking " ++ Git.repoDescribe r
 	-- note: Does not currently differnetiate between rsync failing
 	-- to connect, and the file not being present.
 	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
@@ -174,7 +174,7 @@ withRsyncScratchDir a = do
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
-	showProgress -- make way for progress bar
+	showOutput -- make way for progress bar
 	res <- liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params
 	if res
 		then return res
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index cbd3ef6225..e4dcc2a71d 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -185,7 +185,7 @@ remove r k = s3Action r False $ \(conn, bucket) -> do
 
 checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
-	showNote ("checking " ++ name r ++ "...")
+	showAction $ "checking " ++ name r
 	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
 	case res of
 		Right _ -> return $ Right True
@@ -241,13 +241,13 @@ iaMunge = (>>= munge)
 genBucket :: RemoteConfig -> Annex ()
 genBucket c = do
 	conn <- s3ConnectionRequired c
-	showNote "checking bucket"
+	showAction "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
 		Right _ -> return ()
 		Left err@(NetworkError _) -> s3Error err
 		Left (AWSError _ _) -> do
-			showNote $ "creating bucket in " ++ datacenter
+			showAction $ "creating bucket in " ++ datacenter
 			res <- liftIO $ createBucketIn conn bucket datacenter
 			case res of
 				Right _ -> return ()
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 60f64cfe02..2f8fac23b5 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -106,7 +106,7 @@ checkKey key = do
 checkKey' :: [URLString] -> Annex Bool
 checkKey' [] = return False
 checkKey' (u:us) = do
-	showNote ("checking " ++ u)
+	showAction $ "checking " ++ u
 	e <- liftIO $ urlexists u
 	if e then return e else checkKey' us
 
@@ -129,6 +129,6 @@ urlexists url =
 download :: [URLString] -> FilePath -> Annex Bool
 download [] _ = return False
 download (url:us) file = do
-	showProgress -- make way for curl progress bar
+	showOutput -- make way for curl progress bar
 	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 	if ok then return ok else download us file
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index 071fd12ee1..3aabe07700 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -23,7 +23,7 @@ import qualified Upgrade.V1
 
 upgrade :: Annex Bool
 upgrade = do
-	showNote "v0 to v1..."
+	showAction "v0 to v1"
 	g <- Annex.gitRepo
 
 	-- do the reorganisation of the key files
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 8a3d37a642..c41310880f 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -58,7 +58,7 @@ import qualified Upgrade.V2
 
 upgrade :: Annex Bool
 upgrade = do
-	showNote "v1 to v2"
+	showAction "v1 to v2"
 
 	g <- Annex.gitRepo
 	if Git.repoIsLocalBare g
@@ -77,7 +77,7 @@ upgrade = do
 
 moveContent :: Annex ()
 moveContent = do
-	showNote "moving content..."
+	showAction "moving content"
 	files <- getKeyFilesPresent1
 	forM_ files move
 	where
@@ -91,7 +91,7 @@ moveContent = do
 
 updateSymlinks :: Annex ()
 updateSymlinks = do
-	showNote "updating symlinks..."
+	showAction "updating symlinks"
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	forM_ files fixlink
@@ -108,7 +108,7 @@ updateSymlinks = do
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
-	showNote "moving location logs..."
+	showAction "moving location logs"
 	logkeys <- oldlocationlogs
 	forM_ logkeys move
 		where
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 99c7806d27..0b1d69f8e1 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -45,21 +45,25 @@ olddir g
  -}
 upgrade :: Annex Bool
 upgrade = do
-	showNote "v2 to v3"
+	showAction "v2 to v3"
 	g <- Annex.gitRepo
 	let bare = Git.repoIsLocalBare g
 
 	Branch.create
+	showProgress
+
 	e <- liftIO $ doesDirectoryExist (olddir g)
 	when e $ do
 		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
 		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
 
 	saveState
+	showProgress
 
 	when e $ liftIO $ do
 		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
 		unless bare $ gitAttributesUnWrite g
+	showProgress
 
 	unless bare push
 
@@ -83,6 +87,7 @@ inject source dest = do
 	new <- liftIO (readFile $ olddir g  source)
 	prev <- Branch.get dest
 	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
+	showProgress
 
 logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
@@ -105,8 +110,8 @@ push = do
 			-- "git push" will from then on
 			-- automatically push it
 			Branch.update -- just in case
-			showNote "pushing new git-annex branch to origin"
-			showProgress
+			showAction "pushing new git-annex branch to origin"
+			showOutput
 			g <- Annex.gitRepo
 			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
 		_ -> do
@@ -116,7 +121,7 @@ push = do
 			showLongNote $
 				"git-annex branch created\n" ++
 				"Be sure to push this branch when pushing to remotes.\n"
-			showProgress
+			showOutput
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]

From a8a71b9d915f6c274fb07636e0840bcc26ab6731 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jul 2011 23:52:09 -0400
Subject: [PATCH 2074/8313] releasing version 3.20110719

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 3a6c731fe4..fab4927cdd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110708) UNRELEASED; urgency=low
+git-annex (3.20110719) unstable; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
     Closes: #634233
@@ -6,7 +6,7 @@ git-annex (3.20110708) UNRELEASED; urgency=low
   * Support the standard git -c name=value
   * unannex: Clean up use of git commit -a.
 
- -- Joey Hess   Thu, 07 Jul 2011 21:28:49 -0400
+ -- Joey Hess   Tue, 19 Jul 2011 23:39:53 -0400
 
 git-annex (3.20110707) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index bbbcbf9fb4..807158f3eb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110708
+Version: 3.20110719
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From d8f21b7955c48f121924d187b1be07f53edbff93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jul 2011 23:54:36 -0400
Subject: [PATCH 2075/8313] add news item for git-annex 3.20110719

---
 doc/news/version_0.20110610.mdwn | 6 ------
 doc/news/version_3.20110719.mdwn | 7 +++++++
 2 files changed, 7 insertions(+), 6 deletions(-)
 delete mode 100644 doc/news/version_0.20110610.mdwn
 create mode 100644 doc/news/version_3.20110719.mdwn

diff --git a/doc/news/version_0.20110610.mdwn b/doc/news/version_0.20110610.mdwn
deleted file mode 100644
index 9ab9e09076..0000000000
--- a/doc/news/version_0.20110610.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 0.20110610 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Add --numcopies option.
-   * Add --trust, --untrust, and --semitrust options.
-   * get --from is the same as copy --from
-   * Bugfix: Fix fsck to not think all SHAnE keys are bad."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110719.mdwn b/doc/news/version_3.20110719.mdwn
new file mode 100644
index 0000000000..5beae32270
--- /dev/null
+++ b/doc/news/version_3.20110719.mdwn
@@ -0,0 +1,7 @@
+git-annex 3.20110719 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
+     Closes: #[634233](http://bugs.debian.org/634233)
+   * Bugfix: Make add ../ work.
+   * Support the standard git -c name=value
+   * unannex: Clean up use of git commit -a."""]]
\ No newline at end of file

From 1d88b966e29e0b064a142a712d6a350b3f19e2d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 24 Jul 2011 15:15:43 +0200
Subject: [PATCH 2076/8313] fix broken img

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 8975c82de7..4b7159cd53 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -12,7 +12,7 @@ To get a feel for it, see the [[walkthrough]].
 * [[forum]]
 * [[comments]]
 * [[contact]]
-* Flattr this
+* Flattr this
 
 [[News]]:
 

From 50edbb03eb0582dd568a6fb83fcdb410828c1ea1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkvSZ1AFJdY_1FeutZr_KWeqtzjZta1PNE"
 
Date: Thu, 28 Jul 2011 17:40:53 +0000
Subject: [PATCH 2077/8313]

---
 ...es_which_have_names_containing_spaces.mdwn | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn

diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
new file mode 100644
index 0000000000..464c06b442
--- /dev/null
+++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
@@ -0,0 +1,48 @@
+    ~$ mkdir test annex
+    ~$ cd test
+    ~$ git init
+    Initialized empty Git repository in /home/user/test/.git/
+    ~$ git annex init test
+    init test ok
+    ~$ git annex initremote localrsync encryption=none type=rsync rsyncurl=localhost:annex/
+    initremote localrsync ok
+    ~$ cp /home/user/Music/Charming\ Hostess/Eat/03\ Mi\ Nuera.ogg ./
+    ~$ git annex add 03\ Mi\ Nuera.ogg
+    add 03 Mi Nuera.ogg ok
+    (Recording state in git...)
+    ~$ git commit -m "add ogg"
+    fatal: No HEAD commit to compare with (yet)
+    fatal: No HEAD commit to compare with (yet)
+    [master (root-commit) 12608af] add ogg
+     1 files changed, 1 insertions(+), 0 deletions(-)
+     create mode 120000 03 Mi Nuera.ogg
+    ~$ git annex move 03\ Mi\ Nuera.ogg --to localrsync
+    move 03 Mi Nuera.ogg (checking localrsync...) (to localrsync...) 
+    sending incremental file list
+    1X/
+    1X/39/
+    1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/
+    1X/39/WORM-s6296772-m1311874383--03 Mi Nuera.ogg/WORM-s6296772-m1311874383--03 Mi Nuera.ogg
+         6296772 100%   42.98MB/s    0:00:00 (xfer#1, to-check=0/5)
+    
+    sent 6297754 bytes  received 43 bytes  4198531.33 bytes/sec
+    total size is 6296772  speedup is 1.00
+    ok
+    ~$ git annex get 03\ Mi\ Nuera.ogg
+    get 03 Mi Nuera.ogg (from localrsync...) 
+    rsync: link_stat "/home/user/annex/1X/39/WORM-s6296772-m1311874383--03" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Mi" failed: No such file or directory (2)
+    rsync: change_dir "/home/user/Nuera.ogg" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Mi" failed: No such file or directory (2)
+    rsync: link_stat "/home/user/Nuera.ogg" failed: No such file or directory (2)
+    
+    sent 8 bytes  received 12 bytes  13.33 bytes/sec
+    total size is 0  speedup is 0.00
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1526) [Receiver=3.0.7]
+    
+      rsync failed -- run git annex again to resume file transfer
+      Unable to access these remotes: localrsync
+      Try making some of these repositories available:
+            b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6  -- localrsync
+    failed
+    git-annex: 1 failed

From 45bbf210a1210172c7c7b87879ed74f7c8ccbdba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 29 Jul 2011 15:28:21 +0200
Subject: [PATCH 2078/8313] Fix shell escaping in rsync special remote.

---
 Remote/Rsync.hs                                        | 10 ++++++----
 debian/changelog                                       |  6 ++++++
 ..._96___files_which_have_names_containing_spaces.mdwn |  2 ++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index f073e7bd79..9535376097 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -93,10 +93,13 @@ rsyncSetup u c = do
 	return c'
 
 rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o  hashDirMixed k  f  f
+rsyncKey o k = rsyncUrl o  hashDirMixed k  shellEscape (f  f)
         where
                 f = keyFile k
 
+rsyncKeyDir :: RsyncOpts -> Key -> String
+rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
+
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = do
 	g <- Annex.gitRepo
@@ -136,7 +139,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 		[ Params "--delete --recursive"
 		, partialParams
 		, Param $ addTrailingPathSeparator dummy
-		, Param $ parentDir $ rsyncKey o k
+		, Param $ rsyncKeyDir o k
 		]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
@@ -147,8 +150,7 @@ checkPresent r o k = do
 	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
 	return $ Right res
 	where
-		cmd = "rsync --quiet " ++ testfile ++ " 2>/dev/null"
-		testfile = shellEscape $ rsyncKey o k
+		cmd = "rsync --quiet " ++ shellEscape (rsyncKey o k) ++ " 2>/dev/null"
 
 {- Rsync params to enable resumes of sending files safely,
  - ensure that files are only moved into place once complete
diff --git a/debian/changelog b/debian/changelog
index fab4927cdd..748a945d7b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110720) UNRELEASED; urgency=low
+
+  * Fix shell escaping in rsync special remote.
+
+ -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
+
 git-annex (3.20110719) unstable; urgency=low
 
   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
diff --git a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
index 464c06b442..040d86bb87 100644
--- a/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
+++ b/doc/bugs/rsync_special_remote_fails_to___96__get__96___files_which_have_names_containing_spaces.mdwn
@@ -46,3 +46,5 @@
             b8b1ea7a-b93f-11e0-b712-d7bffb6e61e6  -- localrsync
     failed
     git-annex: 1 failed
+
+> [[fixed|done]] --[[Joey]] 

From 16c55f61f2d4f060f20094335fdfcac827f7d508 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sat, 30 Jul 2011 17:28:42 +0000
Subject: [PATCH 2079/8313]

---
 doc/install/OSX.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 23cb1b62e2..2d376d053f 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,5 +1,7 @@
+Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. The version provided by Macports is too old to work with current versions of git-annex. Then execute
+
 
-sudo port install haskell-platform git-core ossp-uuid md5sha1sum coreutils pcre
+sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string
@@ -21,7 +23,7 @@ make
 sudo make install
 

 
-Originally posted by Jon at  --[[Joey]]
+Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]
 
 See also:
 

From d9cdb5eb457c5069d20c0738158de2423770b3e4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sat, 30 Jul 2011 18:04:25 +0000
Subject: [PATCH 2080/8313]

---
 doc/install/OSX.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 2d376d053f..983ac3a2fd 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -2,6 +2,8 @@ Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. T
 
 
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
+
+sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 sudo cabal update
 sudo cabal install missingh
 sudo cabal install utf8-string

From ad4528cce83709eb075158ef15fb1fd6faeb1171 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 31 Jul 2011 01:20:35 +0000
Subject: [PATCH 2081/8313]

---
 doc/bugs/Prevent_accidental_merges.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Prevent_accidental_merges.mdwn

diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
new file mode 100644
index 0000000000..9b0c8a3de6
--- /dev/null
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -0,0 +1,9 @@
+With the storage layout v3, pulling the git-annex branch into the master branch is... less than ideal.
+
+The fact that the two branches contain totally different data make an accidental merge worse, arguably.
+
+Adding a tiny binary file called .gitnomerge to both branches would solve that without any noticeable overhead.
+
+Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
+
+-- Richard

From 344f2c3ec18baaefe6de14a32fc1209459470bf5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
 
Date: Sun, 31 Jul 2011 15:24:29 +0000
Subject: [PATCH 2082/8313] Added a comment: Solution

---
 ...nt_2_f5ebb7f43dcef861ecc13373fb1e263f._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment

diff --git a/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
new file mode 100644
index 0000000000..9601003798
--- /dev/null
+++ b/doc/forum/Wishlist:_Is_it_possible_to___34__unlock__34___files_without_copying_the_file_data__63__/comment_2_f5ebb7f43dcef861ecc13373fb1e263f._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg"
+ nickname="Kristian"
+ subject="Solution"
+ date="2011-07-31T15:24:25Z"
+ content="""
+Yes, it can read id3-tags and guess titles from movie filenames but it sometimes gets confused by the filename metadata provided by the WORM-backend.
+
+I think I have a good enough solution to this problem. It's not efficient when it comes to renames but handles adding and deletion just fine
+
+    rsync -vaL --delete source dest
+
+The -L flag looks at symbolic links and copies the actual data they are pointing to. Of course \"source\" must have all data locally for this to work.
+
+"""]]

From b48fec3846ac3e32b473499c007c98865236848c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 1 Aug 2011 09:52:22 +0000
Subject: [PATCH 2083/8313]

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing.mdwn

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
new file mode 100644
index 0000000000..03d31ea34f
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -0,0 +1,8 @@
+Just issuing the command `cabal install` results in the following error message.
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?

From a746623a334fa1673179621fe5c9c9d6d407851f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 1 Aug 2011 09:54:51 +0000
Subject: [PATCH 2084/8313]

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index 03d31ea34f..cf4b138128 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -5,4 +5,5 @@ Just issuing the command `cabal install` results in the following error message.
                            (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
           arising from a use of `handle' at Command/Add.hs:54:3-24
 
-Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?
+Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
+-- Thomas

From d3f6f4fe34e812fafbc487c969e6aaa0d49de33f Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Thu, 4 Aug 2011 15:25:21 +0000
Subject: [PATCH 2085/8313] fixed internal link

---
 doc/special_remotes/web.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 68df31ef4d..6991ce4e44 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -1,5 +1,5 @@
 git-annex can use the WWW as a special remote, downloading urls to files.
-See [[walkthrough/using_web_web]] for usage examples.
+See [[walkthrough/using_the_web]] for usage examples.
 
 ## notes
 

From 578e66996446f3c29f431fc6da0ce9a525c15135 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 5 Aug 2011 10:27:22 -0400
Subject: [PATCH 2086/8313] response

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index cf4b138128..b5213b8aa5 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -7,3 +7,8 @@ Just issuing the command `cabal install` results in the following error message.
 
 Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?  
 -- Thomas
+
+> No, it's already satisfied by `monad-control` being listed as a
+> dependency in the cabal file. Your system might be old/new/or broken,
+> perhaps it's time to provide some details about the version of haskell
+> and of `monad-control` you have installed? --[[Joey]] 

From 46ddb4e66aa1ba2a569eb8734c8ca0f40f3930d4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 5 Aug 2011 10:29:21 -0400
Subject: [PATCH 2087/8313] response

---
 doc/bugs/Prevent_accidental_merges.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Prevent_accidental_merges.mdwn b/doc/bugs/Prevent_accidental_merges.mdwn
index 9b0c8a3de6..3e30e02235 100644
--- a/doc/bugs/Prevent_accidental_merges.mdwn
+++ b/doc/bugs/Prevent_accidental_merges.mdwn
@@ -7,3 +7,8 @@ Adding a tiny binary file called .gitnomerge to both branches would solve that w
 Yes, there is an argument to be made that this is too much hand-holding, but I still think it's worth it.
 
 -- Richard
+
+> It should be as easy to undo such an accidential merge
+> as it is to undo any other git commit, right? I quite like that git-annex 
+> no longer adds any clutter to the master branch, and would be reluctant
+> to change that. --[[Joey]]

From d2492f990fdfa13a04429edaa5373a39e75cc9f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawliqfHEW134uawIUPwyKiyOdoF-oI5TxnQ"
 
Date: Fri, 5 Aug 2011 22:35:48 +0000
Subject: [PATCH 2088/8313]

---
 doc/cheatsheet.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 9f7c146c82..9ccb22e3e2 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -1,4 +1,4 @@
-A suppliment to the [[walkthrough]].
+A supplement to the [[walkthrough]].
 
 [[!toc]]
 

From 3ffc0bb4f57e20350f59c0e331656e54877916aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 12:50:20 -0400
Subject: [PATCH 2089/8313] foo

---
 Backend.hs     | 2 +-
 Backend/SHA.hs | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 3429e8f42c..6942692e83 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -65,7 +65,7 @@ orderedList = do
 genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
 genKey file trybackend = do
 	bs <- orderedList
-	let bs' = maybe bs (:bs) trybackend
+	let bs' = maybe bs (: bs) trybackend
 	genKey' bs' file
 genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
 genKey' [] _ = return Nothing
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index c1d7136485..bae19be003 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -32,7 +32,7 @@ sizes :: [Int]
 sizes = [1, 256, 512, 224, 384]
 
 backends :: [Backend Annex]
--- order is slightly significant; want sha1 first ,and more general
+-- order is slightly significant; want sha1 first, and more general
 -- sizes earlier
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
@@ -107,7 +107,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 				then "" -- probably not really an extension
 				else naiveextension
 
--- A key's checksum is checked during fsck.
+{- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	g <- Annex.gitRepo

From dd8e649f49212c46df23f329299c64e13a3c90e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 14:45:58 -0400
Subject: [PATCH 2090/8313] fix file name for web remote log files

The key name was not being sufficiently escaped, although it didn't break
anything due to luck. Switch to properly escaped key names for the log
filename, with a fallback to the buggy old name.
---
 Remote/Web.hs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 2f8fac23b5..cd028a06d9 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -67,10 +67,17 @@ gen r _ _ =
 {- The urls for a key are stored in remote/web/hash/key.log 
  - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
+urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
+oldurlLog :: Key -> FilePath
+{- A bug used to store the urls elsewhere. -}
+oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
 getUrls :: Key -> Annex [URLString]
-getUrls key = currentLog (urlLog key)
+getUrls key = do
+	us <- currentLog (urlLog key)
+	if null us
+		then currentLog (oldurlLog key)
+		else return us
 
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()

From dede05171bc9431778da72e5e1235c69db9fa38e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 6 Aug 2011 14:57:22 -0400
Subject: [PATCH 2091/8313] addurl: --fast can be used to avoid immediately
 downloading the url.

The tricky part about this is that to generate a key, the file must be
present already. Worked around by adding (back) an URL key type, which
is used for addurl --fast.
---
 Backend.hs         |  3 ++-
 Backend/URL.hs     | 28 ++++++++++++++++++++++++++++
 Command/Add.hs     | 21 ++++++++++++---------
 Command/AddUrl.hs  | 27 +++++++++++++++++++--------
 Command/Migrate.hs |  2 +-
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  2 ++
 7 files changed, 65 insertions(+), 19 deletions(-)
 create mode 100644 Backend/URL.hs

diff --git a/Backend.hs b/Backend.hs
index 6942692e83..0bb9f4b575 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -32,9 +32,10 @@ import Messages
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
 import qualified Backend.SHA
+import qualified Backend.URL
 
 list :: [Backend Annex]
-list = Backend.WORM.backends ++ Backend.SHA.backends
+list = Backend.WORM.backends ++ Backend.SHA.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
diff --git a/Backend/URL.hs b/Backend/URL.hs
new file mode 100644
index 0000000000..f20aa1f95e
--- /dev/null
+++ b/Backend/URL.hs
@@ -0,0 +1,28 @@
+{- git-annex "URL" backend -- keys whose content is available from urls.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Backend.URL (
+	backends,
+	fromUrl
+) where
+
+import Types.Backend
+import Types.Key
+import Types
+
+backends :: [Backend Annex]
+backends = [backend]
+
+backend :: Backend Annex
+backend = Types.Backend.Backend {
+	name = "URL",
+	getKey = const (return Nothing),
+	fsckKey = const (return True)
+}
+
+fromUrl :: String -> Key
+fromUrl url = stubKey { keyName = url, keyBackendName = "URL" }
diff --git a/Command/Add.hs b/Command/Add.hs
index 58c0143dd0..d8947fb07c 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -8,6 +8,7 @@
 module Command.Add where
 
 import Control.Monad.State (liftIO)
+import Control.Monad (when)
 import System.Posix.Files
 import System.Directory
 import Control.Exception.Control (handle)
@@ -52,7 +53,7 @@ perform (file, backend) = do
 		Nothing -> stop
 		Just (key, _) -> do
 			handle (undo file key) $ moveAnnex key file
-			next $ cleanup file key
+			next $ cleanup file key True
 
 {- On error, put the file back so it doesn't seem to have vanished.
  - This can be called before or after the symlink is in place. -}
@@ -72,18 +73,20 @@ undo file key e = do
 			g <- Annex.gitRepo
 			liftIO $ renameFile (gitAnnexLocation g key) file
 
-cleanup :: FilePath -> Key -> CommandCleanup
-cleanup file key = do
+cleanup :: FilePath -> Key -> Bool -> CommandCleanup
+cleanup file key hascontent = do
 	handle (undo file key) $ do
 		link <- calcGitLink file key
 		liftIO $ createSymbolicLink link file
-		logStatus key InfoPresent
+
+		when hascontent $ do
+			logStatus key InfoPresent
 	
-		-- touch the symlink to have the same mtime as the
-		-- file it points to
-		s <- liftIO $ getFileStatus file
-		let mtime = modificationTime s
-		liftIO $ touch file (TimeSpec mtime) False
+			-- touch the symlink to have the same mtime as the
+			-- file it points to
+			s <- liftIO $ getFileStatus file
+			let mtime = modificationTime s
+			liftIO $ touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 1b12362e9f..e87de384b3 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -17,10 +17,10 @@ import qualified Backend
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
+import qualified Backend.URL
 import Messages
 import Content
 import PresenceLog
-import Types.Key
 import Locations
 import Utility
 
@@ -42,9 +42,14 @@ start s = do
 			
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
+	fast <- Annex.getState Annex.fast
+	if fast then nodownload url file else download url file
+
+download :: String -> FilePath -> CommandPerform
+download url file = do
 	g <- Annex.gitRepo
 	showAction $ "downloading " ++ url ++ " "
-	let dummykey = stubKey { keyName = url, keyBackendName = "URL" }
+	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	ok <- Remote.Web.download [url] tmp
@@ -57,9 +62,16 @@ perform url file = do
 				Just (key, _) -> do
 					moveAnnex key tmp
 					Remote.Web.setUrl key url InfoPresent
-					next $ Command.Add.cleanup file key
+					next $ Command.Add.cleanup file key True
 		else stop
 
+nodownload :: String -> FilePath -> CommandPerform
+nodownload url file = do
+	let key = Backend.URL.fromUrl url
+	Remote.Web.setUrl key url InfoPresent
+	
+	next $ Command.Add.cleanup file key False
+
 url2file :: URI -> IO FilePath
 url2file url = do
 	let parts = filter safe $ split "/" $ uriPath url
@@ -75,8 +87,7 @@ url2file url = do
 			e <- doesFileExist file
 			when e $ error "already have this url"
 			return file
-		safe s
-			| null s = False
-			| s == "." = False
-			| s == ".." = False
-			| otherwise = True
+		safe "" = False
+		safe "." = False
+		safe ".." = False
+		safe _ = True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 495bf9fb63..5ae8354406 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -72,7 +72,7 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
-					next $ Command.Add.cleanup file newkey
+					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
 		cleantmp t = whenM (doesFileExist t) $ removeFile t
diff --git a/debian/changelog b/debian/changelog
index 748a945d7b..fcc0b58b89 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110720) UNRELEASED; urgency=low
 
   * Fix shell escaping in rsync special remote.
+  * addurl: --fast can be used to avoid immediately downloading the url.
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 11f617f1ba..2865c8af5d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -282,6 +282,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   Downloads each url to a file, which is added to the annex.
 
+  To avoid immediately downloading the url, specify --fast
+
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file

From 614d8f98566ba8acc2bfeb661410b5d5eaec624c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
 
Date: Mon, 8 Aug 2011 09:04:21 +0000
Subject: [PATCH 2092/8313] Added a comment

---
 ..._14be660aa57fadec0d81b32a8b52c66f._comment | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
new file mode 100644
index 0000000000..8e38205f00
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_1_14be660aa57fadec0d81b32a8b52c66f._comment
@@ -0,0 +1,75 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmFgsNxmnGznb5bbmcoWhoQOoxZZ-io61s"
+ nickname="Thomas"
+ subject="comment 1"
+ date="2011-08-08T09:04:20Z"
+ content="""
+I use Debian Squeeze, I have the Debian package cabal-install 0.8.0-1 installed.
+
+    $ git clone git://git-annex.branchable.com/
+    $ cd git-annex.branchable.com
+    $ cabal update
+    $ cabal install cabal-install
+
+This installed: Cabal-1.10.2.0, zlib-0.5.3.1, cabal-install 0.10.2.
+No version of monad-control or monadIO installed.
+
+    $ ~/.cabal/bin/cabal install
+    Registering QuickCheck-2.4.1.1...
+    Registering Crypto-4.2.3...
+    Registering base-unicode-symbols-0.2.2.1...
+    Registering deepseq-1.1.0.2...
+    Registering hxt-charproperties-9.1.0...
+    Registering hxt-regex-xmlschema-9.0.0...
+    Registering hxt-unicode-9.0.1...
+    Registering hxt-9.1.2...
+    Registering stm-2.2.0.1...
+    Registering hS3-0.5.6...
+    Registering transformers-0.2.2.0...
+    Registering monad-control-0.2.0.1...
+    [1 of 1] Compiling Main             ( Setup.hs, dist/setup/Main.o )
+    Linking ./dist/setup/setup ...
+    ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make configure
+    [1 of 2] Compiling TestConfig       ( TestConfig.hs, TestConfig.o )
+    [2 of 2] Compiling Main             ( configure.hs, configure.o )
+    Linking configure ...
+    ./configure
+      checking version... 3.20110720
+      checking cp -a... yes
+      checking cp -p... yes
+      checking cp --reflink=auto... yes
+      checking uuid generator... uuid
+      checking xargs -0... yes
+      checking rsync... yes
+      checking curl... yes
+      checking bup... yes
+      checking gpg... yes
+      checking sha1... sha1sum
+      checking sha256... sha256sum
+      checking sha512... sha512sum
+      checking sha224... sha224sum
+      checking sha384... sha384sum
+
+    ...
+
+    Command/Add.hs:54:3:
+        No instance for (Control.Monad.IO.Control.MonadControlIO
+                           (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+          arising from a use of `handle' at Command/Add.hs:54:3-24
+        Possible fix:
+          add an instance declaration for
+          (Control.Monad.IO.Control.MonadControlIO
+             (Control.Monad.State.Lazy.StateT Annex.AnnexState IO))
+        In the first argument of `($)', namely `handle (undo file key)'
+        In a stmt of a 'do' expression:
+              handle (undo file key) $ moveAnnex key file
+        In the expression:
+            do { handle (undo file key) $ moveAnnex key file;
+                 next $ cleanup file key }
+    cabal: Error: some packages failed to install:
+    git-annex-3.20110719 failed during the building phase. The exception was:
+    ExitFailure 1
+
+After I added a depencency for monadIO to the git-annex.cabal file, it installed correctly.  
+-- Thomas
+"""]]

From 065e1a507b74e8e41017a1a5368fc2b8aaac7f19 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:00:21 +0000
Subject: [PATCH 2093/8313]

---
 .../--git-dir_and_--work-tree_options.mdwn    | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/--git-dir_and_--work-tree_options.mdwn

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
new file mode 100644
index 0000000000..2e5d1cf0c8
--- /dev/null
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -0,0 +1,28 @@
+git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
+
+> mkdir /tmp/test
+> cd /tmp/test
+> git init
+Initialized empty Git repository in /tmp/test/.git/
+> git annex init test
+init test ok
+> touch foo
+> cd
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+git-annex: Not in a git repository.
+
+regular git add works:
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
+> git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+# On branch master
+#
+# Initial commit
+#
+# Changes to be committed:
+#   (use "git rm --cached ..." to unstage)
+#
+#       new file:   foo
+#
+
+git-annex version: 3.20110702
+

From 1e934c29d66f44ff074a9d02004e2bd6abb90462 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:02:32 +0000
Subject: [PATCH 2094/8313]

---
 .../--git-dir_and_--work-tree_options.mdwn    | 43 ++++++++++---------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
index 2e5d1cf0c8..ebae56b049 100644
--- a/doc/bugs/--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -1,28 +1,29 @@
 git-annex does not take into account the --git-dir and --work-tree command line options (while they can be useful when scripting).
 
-> mkdir /tmp/test
-> cd /tmp/test
-> git init
-Initialized empty Git repository in /tmp/test/.git/
-> git annex init test
-init test ok
-> touch foo
-> cd
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
-git-annex: Not in a git repository.
+    > mkdir /tmp/test
+    > cd /tmp/test
+    > git init
+    Initialized empty Git repository in /tmp/test/.git/
+    > git annex init test
+    init test ok
+    > touch foo
+    > cd
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test annex add foo
+    git-annex: Not in a git repository.
 
 regular git add works:
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
-> git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
-# On branch master
-#
-# Initial commit
-#
-# Changes to be committed:
-#   (use "git rm --cached ..." to unstage)
-#
-#       new file:   foo
-#
+
+    > git --git-dir =/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
+    # On branch master
+    #
+    # Initial commit
+    #
+    # Changes to be committed:
+    #   (use "git rm --cached ..." to unstage)
+    #
+    #       new file:   foo
+    #
 
 git-annex version: 3.20110702
 

From 20a8f8c85bbda5d8af90782a596abbb1b0ac1e86 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm4or5sJLWB0evPKp70Q2OND-JmFPnOkLA"
 
Date: Tue, 9 Aug 2011 13:04:34 +0000
Subject: [PATCH 2095/8313]

---
 doc/bugs/--git-dir_and_--work-tree_options.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
index ebae56b049..d76a42bfff 100644
--- a/doc/bugs/--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/--git-dir_and_--work-tree_options.mdwn
@@ -13,7 +13,7 @@ git-annex does not take into account the --git-dir and --work-tree command line
 
 regular git add works:
 
-    > git --git-dir =/tmp/test/.git --work-tree=/tmp/test add foo
+    > git --git-dir=/tmp/test/.git --work-tree=/tmp/test add foo
     > git --git-dir=/tmp/test/.git --work-tree=/tmp/test status 
     # On branch master
     #

From f8282b55cece7561cd04788637627e79d2f29e17 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Wed, 10 Aug 2011 16:56:51 +0000
Subject: [PATCH 2096/8313] problems with version 3 upgrade instructions

---
 doc/forum/version_3_upgrade.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/version_3_upgrade.mdwn

diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
new file mode 100644
index 0000000000..476b8c235f
--- /dev/null
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -0,0 +1,9 @@
+after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i didn't manage to follow this line in the directions:
+
+> After this upgrade, you should make sure you include the git-annex branch when git pushing and pulling.
+
+could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
+
+    [branch "git-annex"]
+    	remote = poseidon
+    	merge = git-annex

From 8b847517a810d384a79178124b9766141b89bc17 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Wed, 10 Aug 2011 17:00:09 +0000
Subject: [PATCH 2097/8313] (i don't have a remote origin, but it's more common
 and doesn't change the example)

---
 doc/forum/version_3_upgrade.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/version_3_upgrade.mdwn b/doc/forum/version_3_upgrade.mdwn
index 476b8c235f..7fdbcbc805 100644
--- a/doc/forum/version_3_upgrade.mdwn
+++ b/doc/forum/version_3_upgrade.mdwn
@@ -5,5 +5,5 @@ after upgrading to git-annex 3, i'm stuck with diverging git-annex branches -- i
 could you explain how to do that in a littel more detail? git pull seems to only merge master, although i have these ``.git/config`` settings:
 
     [branch "git-annex"]
-    	remote = poseidon
+    	remote = origin
     	merge = git-annex

From 18012c7ec8e8bcbcdc8712be29046c12adeaf6d2 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Fri, 12 Aug 2011 13:09:11 +0000
Subject: [PATCH 2098/8313] enhancement suggestion

---
 ...e_commit_messages_in_git-annex_branch.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn

diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
new file mode 100644
index 0000000000..68b24c2fba
--- /dev/null
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -0,0 +1,26 @@
+as of git-annex version 3.20110719, all git-annex commits only contain the word "update" as a commit message. given that the contents of the commit are pretty non-descriptive (SHA1 hashes for file names, uuids for repository names), i suggest to have more descriptive commit messages, as shown here:
+
+    /mnt/usb_disk/photos/2011$ git annex get
+    /mnt/usb_disk/photos/2011$ git show git-annex
+    [...]
+    usb-disk-photos: get 2011
+    
+    * 10 files retrieved from 2 sources (9 from local-harddisk, 1 from my-server)
+    * 120 files were already present
+    * 2 files could not be retrieved
+    /mnt/usb_disk/photos/2011$ cd ~/photos/2011/07
+    ~/photos/2011/07$ git copy --to my-server
+    ~/photos/2011/07$ git show git-annex
+    [...]
+    local-harddisk: copy 2011/07 to my-server
+    
+    * 20 files pushed
+    ~/photos/2011/07$
+
+in my opinion, the messages should at least contain
+
+* what command was used
+* in which repository they were executed
+* which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
+
+--[[chrysn]]

From 829ea31b72b1a9c42d770815363ef350ec2bda31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 15 Aug 2011 13:30:59 -0400
Subject: [PATCH 2099/8313] typo

---
 doc/internals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index a4ec5c417b..e80ecbac0d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -22,7 +22,7 @@ deleting or changing the file contents.
 This branch is managed by git-annex, with the contents listed below.
 
 The file `.git/annex/index` is a separate git index file it uses
-to accumlate changes for the git-annex. Also, `.git/annex/journal/` is used
+to accumulate changes for the git-annex. Also, `.git/annex/journal/` is used
 to record changes before they are added to git.
 
 Note that for speed reasons, git-annex assumes only it will modify this

From 9f719e5674280f49b71c4d58dcc0b0002845f852 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
 
Date: Mon, 15 Aug 2011 18:18:22 +0000
Subject: [PATCH 2100/8313]

---
 ...ith_the_annex_directory_exposed_to_http.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn

diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
new file mode 100644
index 0000000000..c1b790be1e
--- /dev/null
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -0,0 +1,17 @@
+Let's say that http://people.collabora.com/~alsuren/git/fate-suite.git/ is a bare git repo. It has been 'git update-server-info'd so that it can be served on a dumb http server.
+
+The repo is also a git annex remote, created using the following commands:
+
+* git remote add alsuren git+ssh://people.collabora.co.uk/user/alsuren/public_html/fate-suite.git
+* git push alsuren --all
+* git annex copy --to=alsuren
+
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though
+
+I would like to be able to use the following commands to get a clone of the repo:
+
+* git clone http://people.collabora.com/~alsuren/git/fate-suite.git/
+* cd fate-suite
+* git annex get
+
+This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.

From c3f74c6959cf67ad511dc9b6bed93112c754c247 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkHscTHMCNvjJ6nLI1VpsBrJFI5FTwhUT4"
 
Date: Mon, 15 Aug 2011 18:19:47 +0000
Subject: [PATCH 2101/8313]

---
 ...git_repo__44___with_the_annex_directory_exposed_to_http.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
index c1b790be1e..a2e344c115 100644
--- a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -6,7 +6,7 @@ The repo is also a git annex remote, created using the following commands:
 * git push alsuren --all
 * git annex copy --to=alsuren
 
-so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though
+so http://people.collabora.com/~alsuren/git/fate-suite.git/annex is a valid git annex (though listing dirs is forbidden, so you need to know the filenames ahead of time).
 
 I would like to be able to use the following commands to get a clone of the repo:
 

From c04f1f4c30d5334b06b606c89fcb8e693bd88701 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 16:11:55 -0400
Subject: [PATCH 2102/8313] response

---
 ...escriptive_commit_messages_in_git-annex_branch.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
index 68b24c2fba..aad119ffd3 100644
--- a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -24,3 +24,13 @@ in my opinion, the messages should at least contain
 * which files or directories they affected (not necessarily all files, but what was given on command line or implicitly from the working directory)
 
 --[[chrysn]]
+
+> The implementation of the git-annex branch precludes more descriptive
+> commit messages, since a single commit can include changes that were
+> previously staged to the branch's index file, or spooled to its journal
+> by other git-annex commands (either concurrently running or
+> interrupted commands, or even changes needed to automatically merge
+> other git-annex branches).
+> 
+> It would be possible to make it *less* verbose, with an empty commit
+> message. :) --[[Joey]] 

From cfcd7805b441c48e404826903480113a82cff9cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 19:23:56 -0400
Subject: [PATCH 2103/8313] add repoIsHttp

---
 Git.hs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Git.hs b/Git.hs
index 9b7ac7ea91..b226bce0f2 100644
--- a/Git.hs
+++ b/Git.hs
@@ -17,6 +17,7 @@ module Git (
 	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
+	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
 	repoLocation,
@@ -206,6 +207,13 @@ repoIsSsh Repo { location = Url url }
 	| otherwise = False
 repoIsSsh _ = False
 
+repoIsHttp :: Repo -> Bool
+repoIsHttp Repo { location = Url url } 
+	| uriScheme url == "http:" = True
+	| uriScheme url == "https:" = True
+	| otherwise = False
+repoIsHttp _ = False
+
 configAvail ::Repo -> Bool
 configAvail Repo { config = c } = c /= M.empty
 

From 354c5f349bcd8c43b45191983dce4a6c7489e9ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:45:58 -0400
Subject: [PATCH 2104/8313] add withTempFile

This is essentially the same as withSystemTempFile from System.IO.Temp,
but that library is not packaged for Debian, and may not be widely used.
I see various other withTempFile implementations here and there, none canonical.
Sigh.
---
 Utility.hs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/Utility.hs b/Utility.hs
index dab3b4d0ea..511350898a 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -23,6 +23,7 @@ module Utility (
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
+	withTempFile,
 	dirContains,
 	dirContents,
 	myHomeDir,
@@ -38,6 +39,7 @@ module Utility (
 	prop_relPathDirToFile_basics
 ) where
 
+import IO (bracket)
 import System.IO
 import System.Exit
 import qualified System.Posix.Process
@@ -253,6 +255,18 @@ viaTmp a file content = do
 	a tmpfile content
 	renameFile tmpfile file
 
+{- Runs an action with a temp file, then removes the file. -}
+withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile template action = bracket create remove use
+	where
+		create = do
+			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			openTempFile tmpdir template
+		remove (name, handle) = do
+			hClose handle
+			catchBool (removeFile name >> return True)
+		use (name, handle) = action name handle
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]

From 5000aba76e6f066fd310d9635ea4369f07684b86 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:47:48 -0400
Subject: [PATCH 2105/8313] avoid namespace conflict

---
 Command.hs        | 2 --
 Command/SetKey.hs | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/Command.hs b/Command.hs
index 02bbd29d44..d3c1640ee0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -178,8 +178,6 @@ withKeys :: CommandSeekKeys
 withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
-withTempFile :: CommandSeekStrings
-withTempFile a params = return $ map a params
 withNothing :: CommandSeekNothing
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index f2a5259bac..807cbd5b93 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -20,7 +20,7 @@ command = [repoCommand "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
-seek = [withTempFile start]
+seek = [withStrings start]
 
 {- Sets cached content for a key. -}
 start :: CommandStartString

From 07f2e7ee726f3d7f60cd478e928afc69db60c0c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:48:11 -0400
Subject: [PATCH 2106/8313] support reading git config from http remotes

The config file is downloaded to a temp file, and git-config run on that
to parse it.
---
 Remote/Git.hs | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index de51c891e2..c8facb47a7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -12,6 +12,7 @@ import Control.Monad.State (liftIO)
 import qualified Data.Map as M
 import System.Cmd.Utils
 import System.Posix.Files
+import System.IO
 
 import Types
 import Types.Remote
@@ -24,7 +25,8 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Ssh
+import Remote.Helper.Ssh
+import qualified Remote.Helper.Url as Url
 import Config
 
 remote :: RemoteType Annex
@@ -75,6 +77,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
+	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ Git.configRead r
 	where
@@ -85,9 +88,19 @@ tryGitConfigRead r
 			case result of
 				Left _ -> return r
 				Right r' -> return r'
+
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
 				Git.hConfigRead r
+
+		geturlconfig = do
+			s <- Url.get (Git.repoLocation r ++ "/config")
+			withTempFile "git-annex.tmp" $ \tmpfile -> \h -> do
+				hPutStr h s
+				hClose h
+				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
+					Git.hConfigRead r
+
 		store a = do
 			r' <- a
 			g <- Annex.gitRepo
@@ -95,6 +108,7 @@ tryGitConfigRead r
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 			return r'
+
 		exchange [] _ = []
 		exchange (old:ls) new =
 			if Git.repoRemoteName old == Git.repoRemoteName new

From 4545a0e78cf675c6bbbcdd86b5c06bf99bb0c7e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:49:04 -0400
Subject: [PATCH 2107/8313] split out generic url stuff into a helper library
 from Remote.Web

---
 Command/AddUrl.hs    |  3 +-
 Remote/Helper/Url.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++
 Remote/Web.hs        | 42 ++++++----------------------
 3 files changed, 77 insertions(+), 34 deletions(-)
 create mode 100644 Remote/Helper/Url.hs

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e87de384b3..add71c8209 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,6 +14,7 @@ import System.Directory
 
 import Command
 import qualified Backend
+import qualified Remote.Helper.Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
@@ -52,7 +53,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Web.download [url] tmp
+	ok <- Remote.Helper.Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
diff --git a/Remote/Helper/Url.hs b/Remote/Helper/Url.hs
new file mode 100644
index 0000000000..d3aea56220
--- /dev/null
+++ b/Remote/Helper/Url.hs
@@ -0,0 +1,66 @@
+{- Url downloading for remotes.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Helper.Url (
+	exists,
+	download,
+	get
+) where
+
+import Control.Monad (liftM)
+import Control.Monad.State (liftIO)
+import qualified Network.Browser as Browser
+import Network.HTTP
+import Network.URI
+
+import Types
+import Messages
+import Utility
+
+type URLString = String
+
+{- Checks that an url exists and could be successfully downloaded. -}
+exists :: URLString -> IO Bool
+exists url =
+	case parseURI url of
+		Nothing -> return False
+		Just u -> do
+			r <- request u HEAD
+			case rspCode r of
+				(2,_,_) -> return True
+				_ -> return False
+
+{- Used to download large files, such as the contents of keys.
+ - Uses curl program for its progress bar. -}
+download :: URLString -> FilePath -> Annex Bool
+download url file = do
+	showOutput -- make way for curl progress bar
+	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
+
+{- Downloads a small file. -}
+get :: URLString -> IO String
+get url =
+	case parseURI url of
+		Nothing -> error "url parse error"
+		Just u -> do
+			r <- request u GET
+			case rspCode r of
+				(2,_,_) -> return $ rspBody r
+				_ -> error $ rspReason r
+
+{- Makes a http request of an url. For example, HEAD can be used to
+ - check if the url exists, or GET used to get the url content (best for
+ - small urls). -}
+request :: URI -> RequestMethod -> IO (Response String)
+request url requesttype = Browser.browse $ do
+	Browser.setErrHandler ignore
+	Browser.setOutHandler ignore
+	Browser.setAllowRedirects True
+	liftM snd $ Browser.request
+		(mkRequest requesttype url :: Request_String)
+	where
+		ignore = const $ return ()
diff --git a/Remote/Web.hs b/Remote/Web.hs
index cd028a06d9..cc96d5306d 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -7,28 +7,24 @@
 
 module Remote.Web (
 	remote,
-	setUrl,
-	download
+	setUrl
 ) where
 
 import Control.Monad.State (liftIO)
 import Control.Exception
 import System.FilePath
-import Network.Browser
-import Network.HTTP
-import Network.URI
 
 import Types
 import Types.Remote
 import qualified Git
 import qualified Annex
 import Messages
-import Utility
 import UUID
 import Config
 import PresenceLog
 import LocationLog
 import Locations
+import qualified Remote.Helper.Url as Url
 
 type URLString = String
 
@@ -90,9 +86,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = do
-	us <- getUrls key
-	download us file
+downloadKey key file = iter =<< getUrls key
+	where
+		iter [] = return False
+		iter (url:urls) = do
+			ok <- Url.download url file
+			if ok then return ok else iter urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
@@ -114,28 +113,5 @@ checkKey' :: [URLString] -> Annex Bool
 checkKey' [] = return False
 checkKey' (u:us) = do
 	showAction $ "checking " ++ u
-	e <- liftIO $ urlexists u
+	e <- liftIO $ Url.exists u
 	if e then return e else checkKey' us
-
-urlexists :: URLString -> IO Bool
-urlexists url =
-	case parseURI url of
-		Nothing -> return False
-		Just u -> do
-			(_, r) <- Network.Browser.browse $ do
-				setErrHandler ignore
-				setOutHandler ignore
-				setAllowRedirects True
-				request (mkRequest HEAD u :: Request_String)
-			case rspCode r of
-				(2,_,_) -> return True
-				_ -> return False
-	where
-		ignore = const $ return ()
-
-download :: [URLString] -> FilePath -> Annex Bool
-download [] _ = return False
-download (url:us) file = do
-	showOutput -- make way for curl progress bar
-	ok <- liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
-	if ok then return ok else download us file

From a55faff08fd9173edaf22a1de46cf7fafe89ebb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 20:49:54 -0400
Subject: [PATCH 2108/8313] reorg Remote/*

---
 Command/Map.hs                     | 2 +-
 Remote/Bup.hs                      | 6 +++---
 Remote/Directory.hs                | 4 ++--
 Remote/{ => Helper}/Encryptable.hs | 2 +-
 Remote/{ => Helper}/Special.hs     | 2 +-
 Remote/{ => Helper}/Ssh.hs         | 2 +-
 Remote/Hook.hs                     | 4 ++--
 Remote/Rsync.hs                    | 4 ++--
 Remote/S3real.hs                   | 4 ++--
 9 files changed, 15 insertions(+), 15 deletions(-)
 rename Remote/{ => Helper}/Encryptable.hs (98%)
 rename Remote/{ => Helper}/Special.hs (97%)
 rename Remote/{ => Helper}/Ssh.hs (98%)

diff --git a/Command/Map.hs b/Command/Map.hs
index 07f127f14e..75c5b0b550 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 import UUID
 import Trust
-import Remote.Ssh
+import Remote.Helper.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 1023cda186..c82f84745d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -30,9 +30,9 @@ import Locations
 import Config
 import Utility
 import Messages
-import Remote.Ssh
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Ssh
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 type BupRepo = String
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 235f613000..fd227f85d8 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,8 +27,8 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 
 remote :: RemoteType Annex
diff --git a/Remote/Encryptable.hs b/Remote/Helper/Encryptable.hs
similarity index 98%
rename from Remote/Encryptable.hs
rename to Remote/Helper/Encryptable.hs
index 66e1738ac2..04041c6553 100644
--- a/Remote/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Encryptable where
+module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 import Control.Monad.State (liftIO)
diff --git a/Remote/Special.hs b/Remote/Helper/Special.hs
similarity index 97%
rename from Remote/Special.hs
rename to Remote/Helper/Special.hs
index d6f362ce30..c302a0ff5e 100644
--- a/Remote/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Special where
+module Remote.Helper.Special where
 
 import qualified Data.Map as M
 import Data.Maybe
diff --git a/Remote/Ssh.hs b/Remote/Helper/Ssh.hs
similarity index 98%
rename from Remote/Ssh.hs
rename to Remote/Helper/Ssh.hs
index fe4e6dfc1a..478b018812 100644
--- a/Remote/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Ssh where
+module Remote.Helper.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 87f86ffe4f..ef52d0482b 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,8 +28,8 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9535376097..eba67e3fd6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,8 +26,8 @@ import Locations
 import Config
 import Content
 import Utility
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index e4dcc2a71d..456a77f0e4 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -33,8 +33,8 @@ import UUID
 import Messages
 import Locations
 import Config
-import Remote.Special
-import Remote.Encryptable
+import Remote.Helper.Special
+import Remote.Helper.Encryptable
 import Crypto
 import Content
 import Utility.Base64

From 5ccb926b51b0a270c8b1d754dac78d2074e07bdf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:04:23 -0400
Subject: [PATCH 2109/8313] support for getting files from http git remotes

---
 Remote/Git.hs | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index c8facb47a7..1adf8cfeb7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -119,9 +119,10 @@ tryGitConfigRead r
  - If the remote cannot be accessed, returns a Left error.
  -}
 inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
-inAnnex r key = if Git.repoIsUrl r
-		then checkremote
-		else liftIO (try checklocal ::IO (Either IOException Bool))
+inAnnex r key
+	| Git.repoIsHttp r = safely checkhttp
+	| Git.repoIsUrl r = checkremote
+	| otherwise = safely checklocal
 	where
 		checklocal = do
 			-- run a local check inexpensively,
@@ -133,7 +134,12 @@ inAnnex r key = if Git.repoIsUrl r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
 				[Param (show key)]
 			return $ Right inannex
-	
+		checkhttp = Url.exists $ keyUrl r key
+		safely a = liftIO (try a ::IO (Either IOException Bool))
+
+keyUrl :: Git.Repo -> Key -> String
+keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
+
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key = 
 	onRemote r (boolSystem, False) "dropkey"
@@ -146,8 +152,9 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| otherwise = error "copying from non-ssh repo not supported"
-		
+	| Git.repoIsHttp r = Url.download (keyUrl r key) file
+	| otherwise = error "copying from non-ssh, non-http repo not supported"
+
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key

From e6752cc06424df18f3e14406674f9a630c3387ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:12:48 -0400
Subject: [PATCH 2110/8313] Added support for getting content from git remotes
 using http (and https).

---
 debian/changelog                                             | 1 +
 ..._repo__44___with_the_annex_directory_exposed_to_http.mdwn | 3 +++
 doc/special_remotes.mdwn                                     | 5 +++--
 doc/special_remotes/web.mdwn                                 | 4 ++++
 4 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index fcc0b58b89..89cc794e7e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110720) UNRELEASED; urgency=low
 
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
+  * Added support for getting content from git remotes using http (and https).
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
index a2e344c115..ba7dcad300 100644
--- a/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
+++ b/doc/bugs/support_bare_git_repo__44___with_the_annex_directory_exposed_to_http.mdwn
@@ -15,3 +15,6 @@ I would like to be able to use the following commands to get a clone of the repo
 * git annex get
 
 This would allow contributors to quickly get a copy of our upstream repo and start contributing with minimal bandwidth/effort.
+
+> This is now supported.. I look forward to seeing your project using it!
+> --[[Joey]] [[!tag done]]
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index afc6a2cf23..55cd1f1a0f 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -1,6 +1,7 @@
 Most [[backends]] can transfer data to and from configured git remotes.
-Normally those remotes are normal git repositories (bare and non-bare),
-that store the file contents in their own git annex directory.
+Normally those remotes are normal git repositories (bare and non-bare;
+local and remote), that store the file contents in their own git annex
+directory.
 
 But, git-annex also extends git's concept of remotes, with these special
 types of remotes. These can be used just like any normal remote by git-annex.
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index 6991ce4e44..a969fb071d 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -5,3 +5,7 @@ See [[walkthrough/using_the_web]] for usage examples.
 
 Currently git-annex only supports downloading content from the web; 
 it cannot upload to it or remove content.
+
+This special remote uses arbitrary urls on the web as the source for content.
+git-annex can also download content from a normal git remote, accessible by
+http.

From 790b0f38795ce9b62deda416c458376f1e7b7016 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:16:23 -0400
Subject: [PATCH 2111/8313] update

---
 doc/transferring_data.mdwn | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index f6ae9bfcde..57873f6f0e 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -1,8 +1,12 @@
 git-annex can transfer data to or from any of a repository's git remotes.
 Depending on where the remote is, the data transfer is done using rsync
-(over ssh, with automatic resume), or plain cp (with copy-on-write
-optimisations on supported filesystems). Some [[special_remotes]]
-are also supported that are not traditional git remotes.
+(over ssh or locally), or plain cp (with copy-on-write
+optimisations on supported filesystems), or using curl (for repositories
+on the web). Some [[special_remotes]] are also supported that are not
+traditional git remotes.
+
+If a data transfer is interrupted, git-annex retains the partial transfer
+to allow it to be automatically resumed later.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever

From f5449aae16af431ce6474080d123a930209b2cde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 21:20:14 -0400
Subject: [PATCH 2112/8313] error out when dropping from http repo

---
 Remote/Git.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 1adf8cfeb7..d4847d6105 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -141,8 +141,9 @@ keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
 
 dropKey :: Git.Repo -> Key -> Annex Bool
-dropKey r key = 
-	onRemote r (boolSystem, False) "dropkey"
+dropKey r key
+	| Git.repoIsHttp r = error "dropping from http repo not supported"
+	| otherwise = onRemote r (boolSystem, False) "dropkey"
 		[ Params "--quiet --force"
 		, Param $ show key
 		]

From dffbf49d90236aedb559a3db26a96e2ae7904ece Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 01:33:08 +0000
Subject: [PATCH 2113/8313] Added a comment

---
 ...ment_1_05fc9c9cad26c520bebb98c852c71e35._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment

diff --git a/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
new file mode 100644
index 0000000000..18746225e9
--- /dev/null
+++ b/doc/forum/version_3_upgrade/comment_1_05fc9c9cad26c520bebb98c852c71e35._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-17T01:33:08Z"
+ content="""
+It's ok that `git pull` does not merge the git-annex branch. You can merge it with `git annex merge`, or it will be done
+automatically when you use other git-annex commands.
+
+If you use `git pull` and `git push` without any options, the defaults will make git pull and push the git-annex branch automatically.
+
+But if you're in the habit of doing `git push origin master`, that won't cause the git-annex branch to be pushed (use `git push origin git-annex` to manually push it then). Similarly, `git pull origin master` won't pull it. And also, the `remote.origin.fetch` setting in `.git/config` can be modified in ways that make `git pull` not automatically pull the git-annex branch. So those are the things to avoid after upgrade to v3, basically.
+"""]]

From 4a023dd1aafb725ba5ccff5f09adc7055a2643ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 16 Aug 2011 22:22:00 -0400
Subject: [PATCH 2114/8313] Added curl to Debian package dependencies.

---
 debian/changelog | 1 +
 debian/control   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 89cc794e7e..dc03647a72 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110720) UNRELEASED; urgency=low
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
   * Added support for getting content from git remotes using http (and https).
+  * Added curl to Debian package dependencies.
 
  -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
 
diff --git a/debian/control b/debian/control
index d519c42b71..975faf5ea5 100644
--- a/debian/control
+++ b/debian/control
@@ -31,6 +31,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
+	curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git

From cde8221051546902ef24a4fa51f2108177b67a4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 04:56:30 +0000
Subject: [PATCH 2115/8313] Added a comment

---
 .../comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
new file mode 100644
index 0000000000..adf7a34e66
--- /dev/null
+++ b/doc/bugs/Cabal_dependency_monadIO_missing/comment_2_4f4d8e1e00a2a4f7e8a8ab082e16adac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-08-17T04:56:30Z"
+ content="""
+Finally got a chance to try to reproduce this. I followed your recipe exactly in a clean squeeze chroot. monadIO was not installed, but git-annex built ok, using monad-control.
+"""]]

From f0c21307001ec03437cca066d21f35f541b66de9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 01:34:15 -0400
Subject: [PATCH 2116/8313] releasing version 3.20110817

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index dc03647a72..3eab43578a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (3.20110720) UNRELEASED; urgency=low
+git-annex (3.20110817) unstable; urgency=low
 
   * Fix shell escaping in rsync special remote.
   * addurl: --fast can be used to avoid immediately downloading the url.
   * Added support for getting content from git remotes using http (and https).
   * Added curl to Debian package dependencies.
 
- -- Joey Hess   Fri, 29 Jul 2011 15:27:30 +0200
+ -- Joey Hess   Wed, 17 Aug 2011 01:29:02 -0400
 
 git-annex (3.20110719) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 807158f3eb..14188cd350 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110719
+Version: 3.20110817
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 00cc34a4c3a496f4d8fb5dfc5de826e321e6ac56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 01:34:43 -0400
Subject: [PATCH 2117/8313] add news item for git-annex 3.20110817

---
 doc/news/version_3.20110624.mdwn | 33 --------------------------------
 doc/news/version_3.20110817.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 33 deletions(-)
 delete mode 100644 doc/news/version_3.20110624.mdwn
 create mode 100644 doc/news/version_3.20110817.mdwn

diff --git a/doc/news/version_3.20110624.mdwn b/doc/news/version_3.20110624.mdwn
deleted file mode 100644
index 6204673bd2..0000000000
--- a/doc/news/version_3.20110624.mdwn
+++ /dev/null
@@ -1,33 +0,0 @@
-News for git-annex 3.20110624:
-
-There has been another change to the git-annex data store.
-Use `git annex upgrade` to migrate your repositories to the new
-layout. See [[upgrades]].
-
-The significant change this time is that the .git-annex/ directory
-is gone; instead there is a git-annex branch that is automatically
-maintained by git-annex, and encapsulates all its state nicely out
-of your way.
-
-You should make sure you include the git-annex branch when
-git pushing and pulling.
-
-git-annex 3.20110624 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * New repository format, annex.version=3. Use `git annex upgrade` to migrate.
-   * git-annex now stores its logs in a git-annex branch.
-   * merge: New subcommand. Auto-merges the new git-annex branch.
-   * Improved handling of bare git repos with annexes. Many more commands will
-     work in them.
-   * git-annex is now more robust; it will never leave state files
-     uncommitted when some other git process comes along and locks the index
-     at an inconvenient time.
-   * rsync is now used when copying files from repos on other filesystems.
-     cp is still used when copying file from repos on the same filesystem,
-     since --reflink=auto can make it significantly faster on filesystems
-     such as btrfs.
-   * Allow --trust etc to specify a repository by name, for temporarily
-     trusting repositories that are not configured remotes.
-   * unlock: Made atomic.
-   * git-union-merge: New git subcommand, that does a generic union merge
-     operation, and operates efficiently without touching the working tree."""]]
diff --git a/doc/news/version_3.20110817.mdwn b/doc/news/version_3.20110817.mdwn
new file mode 100644
index 0000000000..51388f3c78
--- /dev/null
+++ b/doc/news/version_3.20110817.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20110817 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix shell escaping in rsync special remote.
+   * addurl: --fast can be used to avoid immediately downloading the url.
+   * Added support for getting content from git remotes using http (and https).
+   * Added curl to Debian package dependencies."""]]
\ No newline at end of file

From e950947cc9af07e03d4d2c335154790bc2943016 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
 
Date: Wed, 17 Aug 2011 12:34:48 +0000
Subject: [PATCH 2118/8313] Added a comment: squeeze-backports update?

---
 .../comment_1_029486088d098c2d4f1099f2f0e701a9._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment

diff --git a/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
new file mode 100644
index 0000000000..9a4ed7c31d
--- /dev/null
+++ b/doc/install/Debian/comment_1_029486088d098c2d4f1099f2f0e701a9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawla7u6eLKNYZ09Z7xwBffqLaXquMQC07fU"
+ nickname="Matthias"
+ subject="squeeze-backports update?"
+ date="2011-08-17T12:34:46Z"
+ content="""
+Is there going to be an update of git-annex in debian squeeze-backports to a version that supports repository version 3?
+Thx
+"""]]

From a131fabc0dd73a50e4f9a994e876ed1bca608f19 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 17 Aug 2011 15:34:29 +0000
Subject: [PATCH 2119/8313] Added a comment: Re: squeeze-backports update?

---
 .../comment_2_648e3467e260cdf233acdb0b53313ce0._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment

diff --git a/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
new file mode 100644
index 0000000000..b8b3d68f33
--- /dev/null
+++ b/doc/install/Debian/comment_2_648e3467e260cdf233acdb0b53313ce0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Re: squeeze-backports update?"
+ date="2011-08-17T15:34:29Z"
+ content="""
+Yes, I uploaded it last night.
+"""]]

From 3b5f7221305ba6e768711d8326a01b78fb1e4f79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 13:07:59 -0400
Subject: [PATCH 2120/8313] init no longer creates .gitattributes

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2865c8af5d..9c202e6689 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -122,7 +122,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * init description
 
   Initializes git-annex with a description of the git repository,
-  and sets up `.gitattributes` and the pre-commit hook.
+  and sets up the pre-commit hook.
 
 * describe repository description
 

From 56f6923ccbc6fb1932137b53458a4cece47e69b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:14:43 -0400
Subject: [PATCH 2121/8313] Now "git annex init" only has to be run once

when a git repository is first being created. Clones will automatically
notice that git-annex is in use and automatically perform a basic
initalization. It's still recommended to run "git annex init" in any
clones, to describe them.
---
 Branch.hs         | 32 ++++++++++++++++------
 CmdLine.hs        | 22 +++++++++++----
 Command/Init.hs   | 38 +++-----------------------
 Command/Uninit.hs | 17 ++----------
 Init.hs           | 69 +++++++++++++++++++++++++++++++++++++++++++++++
 Version.hs        |  6 ++---
 debian/changelog  |  9 +++++++
 7 files changed, 127 insertions(+), 66 deletions(-)
 create mode 100644 Init.hs

diff --git a/Branch.hs b/Branch.hs
index 35e3050936..fd4ce4cede 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -14,6 +14,7 @@ module Branch (
 	files,
 	refExists,
 	hasOrigin,
+	hasSomeBranch,
 	name	
 ) where
 
@@ -124,7 +125,7 @@ getCache file = getState >>= handle
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM (refExists fullname) $ do
+create = unlessM hasBranch $ do
 	g <- Annex.gitRepo
 	e <- hasOrigin
 	if e
@@ -154,19 +155,14 @@ update = do
 		 -}
 		staged <- stageJournalFiles
 
-		g <- Annex.gitRepo
-		r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-		let refs = map (last . words) (lines r)
+		refs <- siblingBranches
 		updated <- catMaybes `liftM` mapM updateRef refs
+		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
-{- Does origin/git-annex exist? -}
-hasOrigin :: Annex Bool
-hasOrigin = refExists originname
-
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
 refExists ref = do
@@ -174,6 +170,26 @@ refExists ref = do
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
+{- Does the main git-annex branch exist? -}
+hasBranch :: Annex Bool
+hasBranch = refExists fullname
+
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = refExists originname
+
+{- Does the git-annex branch or a foo/git-annex branch exist? -}
+hasSomeBranch :: Annex Bool
+hasSomeBranch = liftM (not . null) siblingBranches
+
+{- List of all git-annex branches, including the main one and any
+ - from remotes. -}
+siblingBranches :: Annex [String]
+siblingBranches = do
+	g <- Annex.gitRepo
+	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+	return $ map (last . words) (lines r)
+
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: GitRef -> Annex (Maybe String)
 updateRef ref
diff --git a/CmdLine.hs b/CmdLine.hs
index c33c497856..ff1758f0dc 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,13 +19,14 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
+import qualified Branch
 import Content
 import Types
 import Command
 import Version
 import Options
 import Messages
-import UUID
+import Init
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
@@ -45,7 +46,7 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			when (cmdusesrepo command) checkVersion
+			checkCmdEnviron command
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -57,6 +58,19 @@ parseCmd argv header cmds options = do
 		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
 		usagemsg = "\n\n" ++ usage header cmds options
 
+{- Checks that the command can be run in the current environment. -}
+checkCmdEnviron :: Command -> Annex ()
+checkCmdEnviron command = do
+	when (cmdusesrepo command) $ checkVersion $ do
+		{- Automatically initialize if there is already a git-annex
+		   branch from somewhere. Otherwise, require a manual init
+		   to avoid git-annex accidentially being run in git
+		   repos that did not intend to use it. -}
+		annexed <- Branch.hasSomeBranch
+		if annexed
+			then initialize
+			else error "First run: git-annex init"
+
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
 usage header cmds options =
@@ -95,9 +109,7 @@ tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
-startup = do
-	prepUUID
-	return True
+startup = return True
 
 {- Cleanup actions. -}
 shutdown :: Annex Bool
diff --git a/Command/Init.hs b/Command/Init.hs
index 71e87050d8..0191060511 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,19 +7,13 @@
 
 module Command.Init where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
-import System.Directory
+import Control.Monad (when)
 
 import Command
 import qualified Annex
-import qualified Git
-import qualified Branch
 import UUID
-import Version
 import Messages
-import Types
-import Utility
+import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
@@ -39,34 +33,8 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	Branch.create
+	initialize
 	g <- Annex.gitRepo
 	u <- getUUID g
-	setVersion
 	describeUUID u description
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
 	next $ return True
-
-{- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
-	exists <- liftIO $ doesFileExist hook
-	if exists
-		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
-		else liftIO $ do
-			viaTmp writeFile hook preCommitScript
-			p <- getPermissions hook
-			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
-
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
-
-preCommitScript :: String
-preCommitScript = 
-		"#!/bin/sh\n" ++
-		"# automatically configured by git-annex\n" ++ 
-		"git annex pre-commit .\n"
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8b8d7e364d..195246aa84 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,13 +12,11 @@ import System.Directory
 import System.Exit
 
 import Command
-import Messages
-import Types
 import Utility
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
-import qualified Command.Init
+import Init
 import qualified Branch
 import Content
 import Locations
@@ -47,7 +45,7 @@ perform = next cleanup
 cleanup :: CommandCleanup
 cleanup = do
 	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
 	-- avoid normal shutdown
@@ -55,14 +53,3 @@ cleanup = do
 	liftIO $ do
 		Git.run g "branch" [Param "-D", Param Branch.name]
 		exitSuccess
-
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = Command.Init.preCommitHook repo
-	whenM (liftIO $ doesFileExist hook) $ do
-		c <- liftIO $ readFile hook
-		if c == Command.Init.preCommitScript
-			then liftIO $ removeFile hook
-			else warning $ "pre-commit hook (" ++ hook ++ 
-				") contents modified; not deleting." ++
-				" Edit it to remove call to git annex."
diff --git a/Init.hs b/Init.hs
new file mode 100644
index 0000000000..41256a9530
--- /dev/null
+++ b/Init.hs
@@ -0,0 +1,69 @@
+{- git-annex repository initialization
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Init (initialize, uninitialize) where
+
+import Control.Monad.State (liftIO)
+import Control.Monad (unless)
+import System.Directory
+
+import qualified Annex
+import qualified Git
+import qualified Branch
+import Version
+import Messages
+import Types
+import Utility
+import UUID
+
+initialize :: Annex ()
+initialize = do
+	prepUUID
+	Branch.create
+	setVersion
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) $
+		gitPreCommitHookWrite g
+
+uninitialize :: Annex ()
+uninitialize = do
+	g <- Annex.gitRepo
+	gitPreCommitHookUnWrite g
+
+{- set up a git pre-commit hook, if one is not already present -}
+gitPreCommitHookWrite :: Git.Repo -> Annex ()
+gitPreCommitHookWrite repo = do
+	exists <- liftIO $ doesFileExist hook
+	if exists
+		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
+		else liftIO $ do
+			viaTmp writeFile hook preCommitScript
+			p <- getPermissions hook
+			setPermissions hook $ p {executable = True}
+	where
+		hook = preCommitHook repo
+
+gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
+gitPreCommitHookUnWrite repo = do
+	let hook = preCommitHook repo
+	whenM (liftIO $ doesFileExist hook) $ do
+		c <- liftIO $ readFile hook
+		if c == preCommitScript
+			then liftIO $ removeFile hook
+			else warning $ "pre-commit hook (" ++ hook ++ 
+				") contents modified; not deleting." ++
+				" Edit it to remove call to git annex."
+
+preCommitHook :: Git.Repo -> FilePath
+preCommitHook repo = 
+	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
+
+preCommitScript :: String
+preCommitScript = 
+		"#!/bin/sh\n" ++
+		"# automatically configured by git-annex\n" ++ 
+		"git annex pre-commit .\n"
diff --git a/Version.hs b/Version.hs
index 7e6910fbe4..44fd2e9de1 100644
--- a/Version.hs
+++ b/Version.hs
@@ -39,10 +39,10 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex ()
-checkVersion = getVersion >>= handle
+checkVersion :: Annex () -> Annex ()
+checkVersion initaction = getVersion >>= handle
 	where
-		handle Nothing = error "First run: git-annex init"
+		handle Nothing = initaction
 		handle (Just v) = unless (v `elem` supportedVersions) $
 			error $ "Repository version " ++ v ++ 
 				" is not supported. " ++
diff --git a/debian/changelog b/debian/changelog
index 3eab43578a..9230c00218 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (3.20110818) UNRELEASED; urgency=low
+
+  * Now "git annex init" only has to be run once, when a git repository
+    is first being created. Clones will automatically notice that git-annex
+    is in use and automatically perform a basic initalization. It's
+    still recommended to run "git annex init" in any clones, to describe them.
+
+ -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
+
 git-annex (3.20110817) unstable; urgency=low
 
   * Fix shell escaping in rsync special remote.

From cf33eff684de5193379e99745d83c80fd2fb09c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:17:12 -0400
Subject: [PATCH 2122/8313] git-annex-shell configlist should not be standalone

This makes it initialize the repository with a uuid, and list the uuid,
allowing automatic setup of bare repositories when git-annex is used.
---
 Command/ConfigList.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d8dbff03af..1b1bb3c34b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -14,7 +14,7 @@ import Command
 import UUID
 
 command :: [Command]
-command = [standaloneCommand "configlist" paramNothing seek
+command = [repoCommand "configlist" paramNothing seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]

From 32f27cc3e839a3c243641b953fd4bd0f15dda08a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:36:20 -0400
Subject: [PATCH 2123/8313] when reading configs of local repos, first
 initializeSafe

This auto-generates a uuid if the local repo does not already have one.
---
 CmdLine.hs    | 12 +-----------
 Init.hs       | 17 ++++++++++++++++-
 Remote/Git.hs | 30 +++++++++++++++++-------------
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index ff1758f0dc..0590f11124 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -19,7 +19,6 @@ import Control.Monad (when)
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
-import qualified Branch
 import Content
 import Types
 import Command
@@ -60,16 +59,7 @@ parseCmd argv header cmds options = do
 
 {- Checks that the command can be run in the current environment. -}
 checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = do
-	when (cmdusesrepo command) $ checkVersion $ do
-		{- Automatically initialize if there is already a git-annex
-		   branch from somewhere. Otherwise, require a manual init
-		   to avoid git-annex accidentially being run in git
-		   repos that did not intend to use it. -}
-		annexed <- Branch.hasSomeBranch
-		if annexed
-			then initialize
-			else error "First run: git-annex init"
+checkCmdEnviron command = when (cmdusesrepo command) $ checkVersion $ initializeSafe
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
diff --git a/Init.hs b/Init.hs
index 41256a9530..36d3ed0fac 100644
--- a/Init.hs
+++ b/Init.hs
@@ -5,7 +5,11 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Init (initialize, uninitialize) where
+module Init (
+	initialize,
+	initializeSafe,
+	uninitialize
+) where
 
 import Control.Monad.State (liftIO)
 import Control.Monad (unless)
@@ -34,6 +38,17 @@ uninitialize = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 
+{- Call to automatically initialize if there is already a git-annex
+   branch from somewhere. Otherwise, require a manual init
+   to avoid git-annex accidentially being run in git
+   repos that did not intend to use it. -}
+initializeSafe :: Annex ()
+initializeSafe = do
+	annexed <- Branch.hasSomeBranch
+	if annexed
+		then initialize
+		else error "First run: git-annex init"
+
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
 gitPreCommitHookWrite repo = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d4847d6105..c588cc73bb 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -28,6 +28,7 @@ import Utility.RsyncFile
 import Remote.Helper.Ssh
 import qualified Remote.Helper.Url as Url
 import Config
+import Init
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -79,7 +80,9 @@ tryGitConfigRead r
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
-	| otherwise = store $ safely $ Git.configRead r
+	| otherwise = store $ safely $ do
+		onLocal r initializeSafe
+		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
@@ -124,11 +127,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = do
-			-- run a local check inexpensively,
-			-- by making an Annex monad using the remote
-			a <- Annex.new r
-			Annex.eval a (Content.inAnnex key)
+		checklocal = onLocal r (Content.inAnnex key)
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -137,6 +136,13 @@ inAnnex r key
 		checkhttp = Url.exists $ keyUrl r key
 		safely a = liftIO (try a ::IO (Either IOException Bool))
 
+{- Runs an action on a local repository inexpensively, by making an annex
+ - monad using that repository. -}
+onLocal :: Git.Repo -> Annex a -> IO a
+onLocal r a = do
+	annex <- Annex.new r
+	Annex.eval annex a
+
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
 
@@ -163,13 +169,11 @@ copyToRemote r key
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
-		liftIO $ do
-			a <- Annex.new r
-			Annex.eval a $ do
-				ok <- Content.getViaTmp key $
-					rsyncOrCopyFile r keysrc
-				Content.saveState
-				return ok
+		liftIO $ onLocal r $ do
+			ok <- Content.getViaTmp key $
+				rsyncOrCopyFile r keysrc
+			Content.saveState
+			return ok
 	| Git.repoIsSsh r = do
 		g <- Annex.gitRepo
 		let keysrc = gitAnnexLocation g key

From 228a724d1d1e1d2f27763bd5f71473acc4d022d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 14:43:38 -0400
Subject: [PATCH 2124/8313] improve docs for init

---
 doc/git-annex.mdwn | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9c202e6689..9d63ca156d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -121,8 +121,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * init description
 
-  Initializes git-annex with a description of the git repository,
-  and sets up the pre-commit hook.
+  Initializes git-annex with a description of the git repository.
+
+  Until a repository (or one of its remotes) has been initialized,
+  git-annex will refuse to operate on it, to avoid accidentially
+  using it in a repository that was not intended to have an annex.
+
+  It's useful, but not mandatory, to initialize each new clone
+  of a repository with its own description.
 
 * describe repository description
 

From b7a4ff1c3185404d36d34b372b016be052394a95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:38:26 -0400
Subject: [PATCH 2125/8313] optimise initialized check

Avoid running external command if annex.version is set.
---
 CmdLine.hs    |  3 +--
 Init.hs       | 20 ++++++++++++--------
 Remote/Git.hs |  2 +-
 Version.hs    | 19 +++++++------------
 4 files changed, 21 insertions(+), 23 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0590f11124..2652e5b8fc 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,7 +22,6 @@ import qualified Git
 import Content
 import Types
 import Command
-import Version
 import Options
 import Messages
 import Init
@@ -59,7 +58,7 @@ parseCmd argv header cmds options = do
 
 {- Checks that the command can be run in the current environment. -}
 checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = when (cmdusesrepo command) $ checkVersion $ initializeSafe
+checkCmdEnviron command = when (cmdusesrepo command) ensureInitialized
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
diff --git a/Init.hs b/Init.hs
index 36d3ed0fac..ae92998bb9 100644
--- a/Init.hs
+++ b/Init.hs
@@ -6,8 +6,8 @@
  -}
 
 module Init (
+	ensureInitialized,
 	initialize,
-	initializeSafe,
 	uninitialize
 ) where
 
@@ -38,16 +38,20 @@ uninitialize = do
 	g <- Annex.gitRepo
 	gitPreCommitHookUnWrite g
 
-{- Call to automatically initialize if there is already a git-annex
+{- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
    to avoid git-annex accidentially being run in git
    repos that did not intend to use it. -}
-initializeSafe :: Annex ()
-initializeSafe = do
-	annexed <- Branch.hasSomeBranch
-	if annexed
-		then initialize
-		else error "First run: git-annex init"
+ensureInitialized :: Annex ()
+ensureInitialized = do
+	v <- getVersion
+	case v of
+		Just version -> checkVersion version
+		Nothing -> do
+			annexed <- Branch.hasSomeBranch
+			if annexed
+				then initialize
+				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Git.Repo -> Annex ()
diff --git a/Remote/Git.hs b/Remote/Git.hs
index c588cc73bb..d8ecd33c4a 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -81,7 +81,7 @@ tryGitConfigRead r
 	| Git.repoIsHttp r = store $ safely $ geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ do
-		onLocal r initializeSafe
+		onLocal r ensureInitialized
 		Git.configRead r
 	where
 		-- Reading config can fail due to IO error or
diff --git a/Version.hs b/Version.hs
index 44fd2e9de1..fcf6bc4d1d 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,8 +7,6 @@
 
 module Version where
 
-import Control.Monad (unless)
-
 import Types
 import qualified Annex
 import qualified Git
@@ -39,14 +37,11 @@ getVersion = do
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
 
-checkVersion :: Annex () -> Annex ()
-checkVersion initaction = getVersion >>= handle
+checkVersion :: Version -> Annex ()
+checkVersion v
+	| v `elem` supportedVersions = return ()
+	| v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade"
+	| otherwise = err "Upgrade git-annex."
 	where
-		handle Nothing = initaction
-		handle (Just v) = unless (v `elem` supportedVersions) $
-			error $ "Repository version " ++ v ++ 
-				" is not supported. " ++
-				msg v
-		msg v
-			| v `elem` upgradableVersions = "Upgrade this repository: git-annex upgrade"
-			| otherwise = "Upgrade git-annex."
+		err msg = error $ "Repository version " ++ v ++
+			" is not supported. " ++ msg

From 9e763954ae96201ce577cb51f904bf7756597f14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:42:49 -0400
Subject: [PATCH 2126/8313] tweak

---
 Init.hs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/Init.hs b/Init.hs
index ae92998bb9..a11a7c51c9 100644
--- a/Init.hs
+++ b/Init.hs
@@ -1,6 +1,6 @@
 {- git-annex repository initialization
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -43,11 +43,9 @@ uninitialize = do
    to avoid git-annex accidentially being run in git
    repos that did not intend to use it. -}
 ensureInitialized :: Annex ()
-ensureInitialized = do
-	v <- getVersion
-	case v of
-		Just version -> checkVersion version
-		Nothing -> do
+ensureInitialized = getVersion >>= maybe needsinit checkVersion
+	where
+		needsinit = do
 			annexed <- Branch.hasSomeBranch
 			if annexed
 				then initialize

From 0c53ccc675b52d4995b3406d5814116605fe6ec7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 17 Aug 2011 18:52:58 -0400
Subject: [PATCH 2127/8313] tweak

---
 Init.hs | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/Init.hs b/Init.hs
index a11a7c51c9..4708701fa7 100644
--- a/Init.hs
+++ b/Init.hs
@@ -29,14 +29,11 @@ initialize = do
 	prepUUID
 	Branch.create
 	setVersion
-	g <- Annex.gitRepo
-	unless (Git.repoIsLocalBare g) $
-		gitPreCommitHookWrite g
+	gitPreCommitHookWrite
 
 uninitialize :: Annex ()
 uninitialize = do
-	g <- Annex.gitRepo
-	gitPreCommitHookUnWrite g
+	gitPreCommitHookUnWrite
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
@@ -52,8 +49,9 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}
-gitPreCommitHookWrite :: Git.Repo -> Annex ()
-gitPreCommitHookWrite repo = do
+gitPreCommitHookWrite :: Annex ()
+gitPreCommitHookWrite = unlessBare $ do
+	hook <- preCommitHook
 	exists <- liftIO $ doesFileExist hook
 	if exists
 		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
@@ -61,12 +59,10 @@ gitPreCommitHookWrite repo = do
 			viaTmp writeFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
-	where
-		hook = preCommitHook repo
 
-gitPreCommitHookUnWrite :: Git.Repo -> Annex ()
-gitPreCommitHookUnWrite repo = do
-	let hook = preCommitHook repo
+gitPreCommitHookUnWrite :: Annex ()
+gitPreCommitHookUnWrite = unlessBare $ do
+	hook <- preCommitHook
 	whenM (liftIO $ doesFileExist hook) $ do
 		c <- liftIO $ readFile hook
 		if c == preCommitScript
@@ -75,9 +71,15 @@ gitPreCommitHookUnWrite repo = do
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
 
-preCommitHook :: Git.Repo -> FilePath
-preCommitHook repo = 
-	Git.workTree repo ++ "/" ++ Git.gitDir repo ++ "/hooks/pre-commit"
+unlessBare :: Annex () -> Annex ()
+unlessBare a = do
+	g <- Annex.gitRepo
+	unless (Git.repoIsLocalBare g) a
+
+preCommitHook :: Annex FilePath
+preCommitHook = do
+	g <- Annex.gitRepo
+	return $ Git.workTree g ++ "/" ++ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 

From 8a2197adfab9d8425f92000621a802dce37e124c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 18 Aug 2011 12:20:47 -0400
Subject: [PATCH 2128/8313] Added annex-cost-command configuration, which can
 be used to vary the cost of a remote based on the output of a shell command.

Also avoided crashing if the user specified cost value cannot be parsed.
---
 Config.hs          | 21 ++++++++++++++++-----
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  6 ++++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/Config.hs b/Config.hs
index 9cbf2d52fd..6f2014483b 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,6 +9,8 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
+import Control.Monad (liftM)
+import System.Cmd.Utils
 
 import qualified Git
 import qualified Annex
@@ -40,14 +42,23 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
 {- Calculates cost for a remote.
  -
  - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost
+ - repositories; it can also be configured by remote..annex-cost,
+ - or if remote..annex-cost-command is set and prints a number, that
+ - is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
-	c <- getConfig r "cost" ""
-	if not $ null c
-		then return $ read c
-		else return def
+	cmd <- getConfig r "cost-command" ""
+	return . safeparse =<< if not $ null cmd
+			then liftM snd $ liftIO $ pipeFrom "sh" ["-c", cmd]
+			else getConfig r "cost" ""
+	where
+		safeparse v
+			| null ws || null ps = def
+			| otherwise = (fst . head) ps
+			where
+				ws = words v
+				ps = reads $ head ws
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/debian/changelog b/debian/changelog
index 9230c00218..1ace861a46 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     is first being created. Clones will automatically notice that git-annex
     is in use and automatically perform a basic initalization. It's
     still recommended to run "git annex init" in any clones, to describe them.
+  * Added annex-cost-command configuration, which can be used to vary the
+    cost of a remote based on the output of a shell command.
 
  -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9d63ca156d..a262d465fd 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -424,6 +424,12 @@ Here are all the supported configuration settings.
   The default cost is 100 for local repositories, and 200 for remote
   repositories.
 
+* `remote..annex-cost-command`
+
+  If set, the command is run, and the number it outputs is used as the cost.
+  This allows varying the cost based on eg, the current network. The
+  cost-command can be any shell command line.
+
 * `remote..annex-ignore`
 
   If set to `true`, prevents git-annex

From 7cedd28ab00d93b4a8d70810511a7c28cae65fd1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 18 Aug 2011 12:26:28 -0400
Subject: [PATCH 2129/8313] tweak

---
 Config.hs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Config.hs b/Config.hs
index 6f2014483b..568eb71386 100644
--- a/Config.hs
+++ b/Config.hs
@@ -39,12 +39,9 @@ getConfig r key def = do
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
-{- Calculates cost for a remote.
- -
- - The default cost is 100 for local repositories, and 200 for remote
- - repositories; it can also be configured by remote..annex-cost,
- - or if remote..annex-cost-command is set and prints a number, that
- - is used.
+{- Calculates cost for a remote. Either the default, or as configured 
+ - by remote..annex-cost, or if remote..annex-cost-command
+ - is set and prints a number, that is used.
  -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do

From e97fede8cd86d0eb804ced6d2877f617ba15b1a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 12:59:07 -0400
Subject: [PATCH 2130/8313] make gitDir absolute

---
 Git.hs             | 12 ++++++------
 Init.hs            |  2 +-
 git-union-merge.hs |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Git.hs b/Git.hs
index b226bce0f2..168fe4154e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -247,11 +247,11 @@ attributes repo
 	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
-{- Path to a repository's .git directory, relative to its workTree. -}
+{- Path to a repository's .git directory. -}
 gitDir :: Repo -> String
 gitDir repo
-	| configBare repo = ""
-	| otherwise = ".git"
+	| configBare repo = workTree repo
+	| otherwise = workTree repo  ".git"
 
 {- Path to a repository's --work-tree, that is, its top.
  -
@@ -345,10 +345,10 @@ urlAuthPart _ repo = assertUrl repo $ error "internal"
 
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]
-gitCommandLine repo@(Repo { location = Dir d} ) params =
+gitCommandLine repo@(Repo { location = Dir _ } ) params =
 	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ d ++ "/" ++ gitDir repo)
-	, Param ("--work-tree=" ++ d)
+	[ Param ("--git-dir=" ++ gitDir repo)
+	, Param ("--work-tree=" ++ workTree repo)
 	] ++ params
 gitCommandLine repo _ = assertLocal repo $ error "internal"
 
diff --git a/Init.hs b/Init.hs
index 4708701fa7..a469657a11 100644
--- a/Init.hs
+++ b/Init.hs
@@ -79,7 +79,7 @@ unlessBare a = do
 preCommitHook :: Annex FilePath
 preCommitHook = do
 	g <- Annex.gitRepo
-	return $ Git.workTree g ++ "/" ++ Git.gitDir g ++ "/hooks/pre-commit"
+	return $ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e763376077..4e1a932b45 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -20,7 +20,7 @@ usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.workTree g  Git.gitDir g  "index.git-union-merge"
+tmpIndex g = Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
 setup g = cleanup g -- idempotency

From 021e8e1e0e40543f3861ed8fb0fc12cd2be46d46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 14:28:07 -0400
Subject: [PATCH 2131/8313] make Annex an opaque data type

Was a type alias; using newtype has the benefit that type errors will
show "Annex foo" rather than two lines of internal type nonsense. Yay!
There should be no other effects to size or runtime.

I've tried to do this at least twice before (each time I read RWH chapter 10);
finally understood how to this time.. sorta.
---
 Annex.hs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f7e3e29f82..fd7e3b391c 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+
 module Annex (
 	Annex,
 	AnnexState(..),
@@ -17,6 +19,7 @@ module Annex (
 ) where
 
 import Control.Monad.State
+import Control.Monad.IO.Control
 
 import qualified Git
 import Git.Queue
@@ -28,7 +31,14 @@ import Types.TrustLevel
 import Types.UUID
 
 -- git-annex's monad
-type Annex = StateT AnnexState IO
+newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
+	deriving (
+		Functor,
+		Monad,
+		MonadIO,
+		MonadControlIO,
+		MonadState AnnexState
+	)
 
 -- internal state storage
 data AnnexState = AnnexState
@@ -78,9 +88,9 @@ new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run = flip runStateT
+run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
-eval = flip evalStateT
+eval s a = evalStateT (runAnnex a) s
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}

From e3ca08fad89d6f3b5951da74a8af3631c5ef9604 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 14:36:52 -0400
Subject: [PATCH 2132/8313] drop an unnecessart liftIO

the liftM on its own can lift all the way into IO.
---
 Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex.hs b/Annex.hs
index fd7e3b391c..46dfc9df56 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -84,7 +84,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` (liftIO . Git.configRead) gitrepo
+new gitrepo = newState `liftM` Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)

From 01cd775d9271dfe96917c5557efe93ce0e6f52c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:05:08 -0400
Subject: [PATCH 2133/8313] Fix broken upgrade from V1 repository. Closes:
 #638584

Had forgotten to keep several old versions of functions needed during this
upgrade.
---
 PresenceLog.hs   |  4 +++-
 Upgrade/V1.hs    | 33 ++++++++++++++++++++++++++-------
 debian/changelog |  1 +
 3 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/PresenceLog.hs b/PresenceLog.hs
index 9c516a8db1..ccb75ff5be 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -15,10 +15,12 @@ module PresenceLog (
 	LogStatus(..),
 	addLog,
 	readLog,
+	parseLog,
 	writeLog,
 	logNow,
 	compactLog,
-	currentLog
+	currentLog,
+	LogLine
 ) where
 
 import Data.Time.Clock.POSIX
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index c41310880f..160d9309fe 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -22,7 +22,7 @@ import Types.Key
 import Content
 import Types
 import Locations
-import LocationLog
+import PresenceLog
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
@@ -123,7 +123,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				g <- Annex.gitRepo
-				let dest = logFile k
+				let dest = logFile2 g k
 				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
@@ -131,9 +131,9 @@ moveLocationLogs = do
 				-- log files that are not checked into git,
 				-- as well as merging with already upgraded
 				-- logs that have been pulled from elsewhere
-				old <- readLog f
-				new <- readLog dest
-				writeLog dest (old++new)
+				old <- liftIO $ readLog1 f
+				new <- liftIO $ readLog1 dest
+				liftIO $ writeLog1 dest (old++new)
 				AnnexQueue.add "add" [Param "--"] [dest]
 				AnnexQueue.add "add" [Param "--"] [f]
 				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
@@ -186,8 +186,11 @@ fileKey1 :: FilePath -> Key
 fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
-logFile1 :: Git.Repo -> Key -> String
-logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+writeLog1 :: FilePath -> [LogLine] -> IO ()
+writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
+
+readLog1 :: FilePath -> IO [LogLine]
+readLog1 file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -230,3 +233,19 @@ getKeyFilesPresent1' dir = do
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
+
+logFile1 :: Git.Repo -> Key -> String
+logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
+
+logFile2 :: Git.Repo -> Key -> String
+logFile2 = logFile' hashDirLower
+
+logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
+logFile' hasher repo key =
+	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
+
+stateDir :: FilePath
+stateDir = addTrailingPathSeparator $ ".git-annex"
+
+gitStateDir :: Git.Repo -> FilePath
+gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/debian/changelog b/debian/changelog
index 1ace861a46..7d3315103e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     still recommended to run "git annex init" in any clones, to describe them.
   * Added annex-cost-command configuration, which can be used to vary the
     cost of a remote based on the output of a shell command.
+  * Fix broken upgrade from V1 repository. Closes: #638584
 
  -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
 

From 3786f8d3487fe5a460d40045d3bcaaa69f3f84d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:38:36 -0400
Subject: [PATCH 2134/8313] releasing version 3.20110819

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7d3315103e..a8280a156d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110818) UNRELEASED; urgency=low
+git-annex (3.20110819) unstable; urgency=low
 
   * Now "git annex init" only has to be run once, when a git repository
     is first being created. Clones will automatically notice that git-annex
@@ -8,7 +8,7 @@ git-annex (3.20110818) UNRELEASED; urgency=low
     cost of a remote based on the output of a shell command.
   * Fix broken upgrade from V1 repository. Closes: #638584
 
- -- Joey Hess   Wed, 17 Aug 2011 13:44:44 -0400
+ -- Joey Hess   Fri, 19 Aug 2011 20:34:09 -0400
 
 git-annex (3.20110817) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 14188cd350..a37c187c90 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110817
+Version: 3.20110819
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 71215263855e2d5e940bb0086efd7691aa0139d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 19 Aug 2011 20:39:03 -0400
Subject: [PATCH 2135/8313] add news item for git-annex 3.20110819

---
 doc/news/version_3.20110702.mdwn | 22 ----------------------
 doc/news/version_3.20110819.mdwn |  9 +++++++++
 2 files changed, 9 insertions(+), 22 deletions(-)
 delete mode 100644 doc/news/version_3.20110702.mdwn
 create mode 100644 doc/news/version_3.20110819.mdwn

diff --git a/doc/news/version_3.20110702.mdwn b/doc/news/version_3.20110702.mdwn
deleted file mode 100644
index a5bb925552..0000000000
--- a/doc/news/version_3.20110702.mdwn
+++ /dev/null
@@ -1,22 +0,0 @@
-News for git-annex 3.20110702:
-
-The URL backend has been removed. Instead the new web remote can be used.
-
-git-annex 3.20110702 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Now the web can be used as a special remote.
-     This feature replaces the old URL backend.
-   * addurl: New command to download an url and store it in the annex.
-   * Sped back up fsck, copy --from, and other commands that often
-     have to read a lot of information from the git-annex branch. Such
-     commands are now faster than they were before introduction of the
-     git-annex branch.
-   * Always ensure git-annex branch exists.
-   * Modify location log parser to allow future expansion.
-   * --force will cause add, etc, to operate on ignored files.
-   * Avoid mangling encoding when storing the description of repository
-     and other content.
-   * cabal can now be used to build git-annex. This is substantially
-     slower than using make, does not build or install documentation,
-     does not run the test suite, and is not particularly recommended,
-     but could be useful to some."""]]
diff --git a/doc/news/version_3.20110819.mdwn b/doc/news/version_3.20110819.mdwn
new file mode 100644
index 0000000000..bbc6abdbc7
--- /dev/null
+++ b/doc/news/version_3.20110819.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110819 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Now "git annex init" only has to be run once, when a git repository
+     is first being created. Clones will automatically notice that git-annex
+     is in use and automatically perform a basic initalization. It's
+     still recommended to run "git annex init" in any clones, to describe them.
+   * Added annex-cost-command configuration, which can be used to vary the
+     cost of a remote based on the output of a shell command.
+   * Fix broken upgrade from V1 repository. Closes: #[638584](http://bugs.debian.org/638584)"""]]
\ No newline at end of file

From ec746c511f5666fc214eba1a477d1ababfe9d367 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 20 Aug 2011 12:52:29 -0400
Subject: [PATCH 2136/8313] note about why curl -# is used

I'd rather use wget really, but as git-annex uses libcurl elsewhere, it
seems best to stick with curl. And making this configurable seems
overboard.
---
 Remote/Helper/Url.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Remote/Helper/Url.hs b/Remote/Helper/Url.hs
index d3aea56220..af1fee8f04 100644
--- a/Remote/Helper/Url.hs
+++ b/Remote/Helper/Url.hs
@@ -39,6 +39,10 @@ exists url =
 download :: URLString -> FilePath -> Annex Bool
 download url file = do
 	showOutput -- make way for curl progress bar
+	-- Uses the -# progress display, because the normal one is very
+	-- confusing when resuming, showing the remainder to download
+	-- as the whole file, and not indicating how much percent was
+	-- downloaded before the resume.
 	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}

From 737b5d14c91101d46e20999e33461e9059dd9f28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 20 Aug 2011 16:11:42 -0400
Subject: [PATCH 2137/8313] moved files around

---
 .gitignore                           |  2 +-
 Backend/SHA.hs                       |  2 +-
 TestConfig.hs => Build/TestConfig.hs |  8 ++++----
 Command/Add.hs                       |  2 +-
 Command/AddUrl.hs                    |  4 ++--
 Command/Map.hs                       |  2 +-
 Command/Version.hs                   |  2 +-
 Content.hs                           |  2 +-
 Makefile                             | 10 ++++------
 Remote/Bup.hs                        |  2 +-
 Remote/Git.hs                        |  4 ++--
 Remote/Web.hs                        |  2 +-
 UUID.hs                              |  2 +-
 Utility/CopyFile.hs                  |  2 +-
 {Remote/Helper => Utility}/Ssh.hs    |  2 +-
 StatFS.hsc => Utility/StatFS.hsc     |  2 +-
 Touch.hsc => Utility/Touch.hsc       |  2 +-
 {Remote/Helper => Utility}/Url.hs    |  4 ++--
 configure.hs                         |  2 +-
 19 files changed, 28 insertions(+), 30 deletions(-)
 rename TestConfig.hs => Build/TestConfig.hs (94%)
 rename {Remote/Helper => Utility}/Ssh.hs (98%)
 rename StatFS.hsc => Utility/StatFS.hsc (98%)
 rename Touch.hsc => Utility/Touch.hsc (99%)
 rename {Remote/Helper => Utility}/Url.hs (96%)

diff --git a/.gitignore b/.gitignore
index d5bf54c813..1fdad216dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,7 @@
 *.o
 test
 configure
-SysConfig.hs
+Build/SysConfig.hs
 git-annex
 git-annex-shell
 git-union-merge
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index bae19be003..b8a0d254b7 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -24,7 +24,7 @@ import Types
 import Types.Backend
 import Types.Key
 import Utility
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
 
diff --git a/TestConfig.hs b/Build/TestConfig.hs
similarity index 94%
rename from TestConfig.hs
rename to Build/TestConfig.hs
index 8cfae7f0c7..e8a0d13368 100644
--- a/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -1,6 +1,6 @@
-{- Tests the system and generates SysConfig.hs. -}
+{- Tests the system and generates Build.SysConfig.hs. -}
 
-module TestConfig where
+module Build.TestConfig where
 
 import System.IO
 import System.Cmd
@@ -33,12 +33,12 @@ instance Show Config where
 			valuetype (MaybeStringConfig _) = "Maybe String"
 
 writeSysConfig :: [Config] -> IO ()
-writeSysConfig config = writeFile "SysConfig.hs" body
+writeSysConfig config = writeFile "Build/SysConfig.hs" body
 	where
 		body = unlines $ header ++ map show config ++ footer
 		header = [
 			  "{- Automatically generated. -}"
-			, "module SysConfig where"
+			, "module Build.SysConfig where"
 			, ""
 			]
 		footer = []
diff --git a/Command/Add.hs b/Command/Add.hs
index d8947fb07c..407a36093f 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -24,7 +24,7 @@ import Types
 import Content
 import Messages
 import Utility
-import Touch
+import Utility.Touch
 import Locations
 
 command :: [Command]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index add71c8209..72a6b671dd 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -14,7 +14,7 @@ import System.Directory
 
 import Command
 import qualified Backend
-import qualified Remote.Helper.Url
+import qualified Utility.Url as Url
 import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
@@ -53,7 +53,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Remote.Helper.Url.download url tmp
+	ok <- Url.download url tmp
 	if ok
 		then do
 			[(_, backend)] <- Backend.chooseBackends [file]
diff --git a/Command/Map.hs b/Command/Map.hs
index 75c5b0b550..3fd6e42a10 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,7 +22,7 @@ import Types
 import Utility
 import UUID
 import Trust
-import Remote.Helper.Ssh
+import Utility.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Command/Version.hs b/Command/Version.hs
index 2392c9bf6b..1ff829a22a 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -12,7 +12,7 @@ import Data.String.Utils
 import Data.Maybe
 
 import Command
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Version
 
 command :: [Command]
diff --git a/Content.hs b/Content.hs
index c63042dfb5..e89f4c45af 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,7 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
-import StatFS
+import Utility.StatFS
 import Types.Key
 import Utility.DataUnits
 import Config
diff --git a/Makefile b/Makefile
index ccf98f6252..fa087c1ae2 100644
--- a/Makefile
+++ b/Makefile
@@ -8,12 +8,11 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
+sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
 all: $(bins) $(mans) docs
 
-sources: SysConfig.hs StatFS.hs Touch.hs Remote/S3.hs
-
-SysConfig.hs: configure.hs TestConfig.hs
+Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure
 	./configure
 
@@ -30,7 +29,7 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): SysConfig.hs Touch.hs StatFS.hs Remote/S3.o
+$(bins): $(sources)
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
@@ -82,8 +81,7 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure  *.tix .hpc \
-		StatFS.hs Touch.hs SysConfig.hs Remote/S3.hs
+	rm -rf build $(bins) $(mans) test configure  *.tix .hpc $(sources)
 	rm -rf doc/.ikiwiki html dist
 	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index c82f84745d..069209792d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -30,7 +30,7 @@ import Locations
 import Config
 import Utility
 import Messages
-import Remote.Helper.Ssh
+import Utility.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d8ecd33c4a..80ba8a153d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -25,8 +25,8 @@ import qualified Content
 import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
-import Remote.Helper.Ssh
-import qualified Remote.Helper.Url as Url
+import Utility.Ssh
+import qualified Utility.Url as Url
 import Config
 import Init
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index cc96d5306d..5bc6a204b9 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -24,7 +24,7 @@ import Config
 import PresenceLog
 import LocationLog
 import Locations
-import qualified Remote.Helper.Url as Url
+import qualified Utility.Url as Url
 
 type URLString = String
 
diff --git a/UUID.hs b/UUID.hs
index cfa519baa3..fa71bed391 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -33,7 +33,7 @@ import qualified Branch
 import Types
 import Types.UUID
 import qualified Annex
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 import Config
 
 configkey :: String
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 2e06dd92bb..befb00f8f2 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -10,7 +10,7 @@ module Utility.CopyFile (copyFile) where
 import System.Directory (doesFileExist, removeFile)
 
 import Utility
-import qualified SysConfig
+import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
  - and because this allows easy access to features like cp --reflink. -}
diff --git a/Remote/Helper/Ssh.hs b/Utility/Ssh.hs
similarity index 98%
rename from Remote/Helper/Ssh.hs
rename to Utility/Ssh.hs
index 478b018812..6cbc362a03 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Helper.Ssh where
+module Utility.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/StatFS.hsc b/Utility/StatFS.hsc
similarity index 98%
rename from StatFS.hsc
rename to Utility/StatFS.hsc
index feb82aa9af..d3e4a507e5 100644
--- a/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -45,7 +45,7 @@
 {-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
 
 
-module StatFS ( FileSystemStats(..), getFileSystemStats ) where
+module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 
 import Foreign
 import Foreign.C.Types
diff --git a/Touch.hsc b/Utility/Touch.hsc
similarity index 99%
rename from Touch.hsc
rename to Utility/Touch.hsc
index dd0c38984e..f27ac31360 100644
--- a/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -7,7 +7,7 @@
 
 {-# LANGUAGE ForeignFunctionInterface #-}
 
-module Touch (
+module Utility.Touch (
 	TimeSpec(..),
 	touchBoth,
 	touch
diff --git a/Remote/Helper/Url.hs b/Utility/Url.hs
similarity index 96%
rename from Remote/Helper/Url.hs
rename to Utility/Url.hs
index af1fee8f04..5954e0ff7e 100644
--- a/Remote/Helper/Url.hs
+++ b/Utility/Url.hs
@@ -1,11 +1,11 @@
-{- Url downloading for remotes.
+{- Url downloading.
  -
  - Copyright 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Helper.Url (
+module Utility.Url (
 	exists,
 	download,
 	get
diff --git a/configure.hs b/configure.hs
index bfdfa32dd5..9f7179c539 100644
--- a/configure.hs
+++ b/configure.hs
@@ -3,7 +3,7 @@
 import System.Directory
 import Data.List
 
-import TestConfig
+import Build.TestConfig
 
 tests :: [TestCase]
 tests =

From 1c25df3c1b19ad8406cbe2aef9dfd215ef9f5ce8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 12:59:49 -0400
Subject: [PATCH 2138/8313] outdated comment

---
 AnnexQueue.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 79116c48af..56d71f3238 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -19,8 +19,7 @@ import Messages
 import qualified Git.Queue
 import Utility
 
-{- Adds a git command to the queue, possibly running previously queued
- - actions if enough have accumulated. -}
+{- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
 add command params files = do
 	q <- getState repoqueue

From 06ce5741365ae4602ca64204729e6a23705059ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 13:17:12 -0400
Subject: [PATCH 2139/8313] tweak

---
 Config.hs  | 5 ++---
 Utility.hs | 6 +++---
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/Config.hs b/Config.hs
index 568eb71386..493a907009 100644
--- a/Config.hs
+++ b/Config.hs
@@ -51,11 +51,10 @@ remoteCost r def = do
 			else getConfig r "cost" ""
 	where
 		safeparse v
-			| null ws || null ps = def
-			| otherwise = (fst . head) ps
+			| null ws = def
+			| otherwise = fromMaybe def $ readMaybe $ head ws
 			where
 				ws = words v
-				ps = reads $ head ws
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/Utility.hs b/Utility.hs
index 511350898a..8a332601b7 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -75,7 +75,7 @@ toCommand = (>>= unwrap)
 		unwrap (Params s) = filter (not . null) (split " " s)
 		-- Files that start with a dash are modified to avoid
 		-- the command interpreting them as options.
-		unwrap (File ('-':s)) = ["./-" ++ s]
+		unwrap (File s@('-':_)) = ["./" ++ s]
 		unwrap (File s) = [s]
 
 {- Run a system command, and returns True or False
@@ -257,7 +257,7 @@ viaTmp a file content = do
 
 {- Runs an action with a temp file, then removes the file. -}
 withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
-withTempFile template action = bracket create remove use
+withTempFile template a = bracket create remove use
 	where
 		create = do
 			tmpdir <- catch getTemporaryDirectory (const $ return ".")
@@ -265,7 +265,7 @@ withTempFile template action = bracket create remove use
 		remove (name, handle) = do
 			hClose handle
 			catchBool (removeFile name >> return True)
-		use (name, handle) = action name handle
+		use (name, handle) = a name handle
 
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}

From 06f509854af4e019434ddca73309fcd2a5b47463 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 13:19:33 -0400
Subject: [PATCH 2140/8313] file moved

---
 debian/copyright | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/copyright b/debian/copyright
index 112ad54aaa..a8a38913e4 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -8,7 +8,7 @@ License: GPL-3+
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
 
-Files: StatFS.hsc
+Files: Utility/StatFS.hsc
 Copyright: Jose A Ortega Ruiz 
 License: BSD-3-clause
  -- All rights reserved.

From 4c73d77b42e97ad740d5731ad73c40a31c0c84f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 21 Aug 2011 14:59:34 -0400
Subject: [PATCH 2141/8313] avoid the functor

fmap = liftM
---
 Annex.hs  | 1 -
 Branch.hs | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 46dfc9df56..07316bd370 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -33,7 +33,6 @@ import Types.UUID
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 	deriving (
-		Functor,
 		Monad,
 		MonadIO,
 		MonadControlIO,
diff --git a/Branch.hs b/Branch.hs
index fd4ce4cede..6abf6d9c2c 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -321,7 +321,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = fmap (map fileJournal) getJournalFilesRaw
+getJournalFiles = liftM (map fileJournal) getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do

From 203148363f459635b1be40b8c6ed376073230dda Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 22 Aug 2011 16:14:12 -0400
Subject: [PATCH 2142/8313] split groups of related functions out of Utility

---
 .gitignore               |   4 +-
 AnnexQueue.hs            |   2 +-
 Backend/SHA.hs           |   2 +-
 Branch.hs                |   2 +
 Command/Add.hs           |   3 +-
 Command/AddUrl.hs        |   2 +-
 Command/Drop.hs          |   1 +
 Command/DropUnused.hs    |   2 +-
 Command/Find.hs          |   2 +-
 Command/Fix.hs           |   3 +-
 Command/FromKey.hs       |   3 +-
 Command/Fsck.hs          |   1 +
 Command/Lock.hs          |   2 +-
 Command/Map.hs           |   2 +-
 Command/Migrate.hs       |   2 +-
 Command/RecvKey.hs       |   2 +-
 Command/SendKey.hs       |   2 +-
 Command/SetKey.hs        |   2 +-
 Command/Unannex.hs       |   3 +-
 Command/Uninit.hs        |   2 +-
 Command/Unlock.hs        |   3 +-
 Config.hs                |   1 +
 Content.hs               |   2 +
 Crypto.hs                |   1 +
 Git.hs                   |   3 +
 Git/LsFiles.hs           |   2 +-
 Git/Queue.hs             |   8 +-
 Git/UnionMerge.hs        |   2 +-
 Init.hs                  |   1 +
 Remote/Bup.hs            |   2 +
 Remote/Directory.hs      |   2 +
 Remote/Git.hs            |   2 +
 Remote/Helper/Special.hs |   2 +-
 Remote/Hook.hs           |   1 +
 Remote/Rsync.hs          |   3 +
 Upgrade/V1.hs            |   2 +
 Upgrade/V2.hs            |   2 +
 Utility.hs               | 215 +--------------------------------------
 Utility/Conditional.hs   |  26 +++++
 Utility/CopyFile.hs      |   3 +-
 Utility/Path.hs          |  92 +++++++++++++++++
 Utility/RsyncFile.hs     |   2 +-
 Utility/SafeCommand.hs   | 104 +++++++++++++++++++
 Utility/Ssh.hs           |   2 +-
 Utility/Url.hs           |   2 +-
 git-annex-shell.hs       |   3 +-
 test.hs                  |  45 ++++----
 47 files changed, 312 insertions(+), 265 deletions(-)
 create mode 100644 Utility/Conditional.hs
 create mode 100644 Utility/Path.hs
 create mode 100644 Utility/SafeCommand.hs

diff --git a/.gitignore b/.gitignore
index 1fdad216dc..7d2504de6f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,7 +13,7 @@ doc/.ikiwiki
 html
 *.tix
 .hpc
-Touch.hs
-StatFS.hs
+Utility/Touch.hs
+Utility/StatFS.hs
 Remote/S3.hs
 dist
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index 56d71f3238..d155a7b81f 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -17,7 +17,7 @@ import Control.Monad (when, unless)
 import Annex
 import Messages
 import qualified Git.Queue
-import Utility
+import Utility.SafeCommand
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index b8a0d254b7..ed2a47db9b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -23,7 +23,7 @@ import Content
 import Types
 import Types.Backend
 import Types.Key
-import Utility
+import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
diff --git a/Branch.hs b/Branch.hs
index 6abf6d9c2c..d5bfe1b09f 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -37,6 +37,8 @@ import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Types
 import Messages
 import Locations
diff --git a/Command/Add.hs b/Command/Add.hs
index 407a36093f..579c4171b1 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -23,8 +23,9 @@ import LocationLog
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import Utility.Touch
+import Utility.SafeCommand
 import Locations
 
 command :: [Command]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 72a6b671dd..55e51100ce 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -23,7 +23,7 @@ import Messages
 import Content
 import PresenceLog
 import Locations
-import Utility
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 14f098349e..6e688d6632 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -15,6 +15,7 @@ import Types
 import Content
 import Messages
 import Utility
+import Utility.Conditional
 import Trust
 import Config
 
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 41bcd6aa78..4ad2aa85bb 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -22,7 +22,7 @@ import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
-import Utility
+import Utility.Conditional
 
 type UnusedMap = M.Map String Key
 
diff --git a/Command/Find.hs b/Command/Find.hs
index 9d760ff5a8..0716c5297e 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -11,7 +11,7 @@ import Control.Monad.State (liftIO)
 
 import Command
 import Content
-import Utility
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 47b0c4c9a0..b24f8e33c2 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,8 @@ import System.Directory
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.Path
+import Utility.SafeCommand
 import Content
 import Messages
 
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index d59f1de397..89c3f4e912 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,10 +14,11 @@ import Control.Monad (unless)
 
 import Command
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 import Content
 import Messages
 import Types.Key
+import Utility.Path
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0d3ecb58f1..6ccec05fb3 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -27,6 +27,7 @@ import LocationLog
 import Locations
 import Trust
 import Utility.DataUnits
+import Utility.Path
 import Config
 
 command :: [Command]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index d39df5f335..77d1ff94f9 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import System.Directory
 import Command
 import Messages
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
 	
 command :: [Command]
 command = [repoCommand "lock" paramPath seek "undo unlock command"]
diff --git a/Command/Map.hs b/Command/Map.hs
index 3fd6e42a10..ef8e04d909 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -19,7 +19,7 @@ import qualified Annex
 import qualified Git
 import Messages
 import Types
-import Utility
+import Utility.SafeCommand
 import UUID
 import Trust
 import Utility.Ssh
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 5ae8354406..25227ae162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -20,7 +20,7 @@ import Locations
 import Types
 import Content
 import Messages
-import Utility
+import Utility.Conditional
 import qualified Command.Add
 
 command :: [Command]
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index e2f7c74abb..be6163558f 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -13,8 +13,8 @@ import System.Exit
 import Command
 import CmdLine
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 02fedb349e..f676ae947a 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -14,8 +14,8 @@ import Locations
 import qualified Annex
 import Command
 import Content
-import Utility
 import Utility.RsyncFile
+import Utility.Conditional
 import Messages
 
 command :: [Command]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 807cbd5b93..2f6f9ea9ee 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -10,7 +10,7 @@ module Command.SetKey where
 import Control.Monad.State (liftIO)
 
 import Command
-import Utility
+import Utility.SafeCommand
 import LocationLog
 import Content
 import Messages
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 960f99722f..54ef2fc68e 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -16,7 +16,8 @@ import Command
 import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
-import Utility
+import Utility.SafeCommand
+import Utility.Path
 import LocationLog
 import Types
 import Content
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 195246aa84..fadae0e5a9 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,7 +12,7 @@ import System.Directory
 import System.Exit
 
 import Command
-import Utility
+import Utility.SafeCommand
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 280eff9de7..0daf1b3218 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -16,8 +16,9 @@ import Types
 import Messages
 import Locations
 import Content
+import Utility.Conditional
 import Utility.CopyFile
-import Utility
+import Utility.Path
 
 command :: [Command]
 command =
diff --git a/Config.hs b/Config.hs
index 493a907009..12f6480470 100644
--- a/Config.hs
+++ b/Config.hs
@@ -16,6 +16,7 @@ import qualified Git
 import qualified Annex
 import Types
 import Utility
+import Utility.SafeCommand
 
 type ConfigKey = String
 
diff --git a/Content.hs b/Content.hs
index e89f4c45af..ba99f1330d 100644
--- a/Content.hs
+++ b/Content.hs
@@ -41,7 +41,9 @@ import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
 import Utility
+import Utility.Conditional
 import Utility.StatFS
+import Utility.Path
 import Types.Key
 import Utility.DataUnits
 import Config
diff --git a/Crypto.hs b/Crypto.hs
index 4fc41ede04..ed29747aa6 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -48,6 +48,7 @@ import Types.Key
 import Types.Remote
 import Utility
 import Utility.Base64
+import Utility.SafeCommand
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
diff --git a/Git.hs b/Git.hs
index 168fe4154e..7155b26343 100644
--- a/Git.hs
+++ b/Git.hs
@@ -85,6 +85,9 @@ import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
 import Utility
+import Utility.Path
+import Utility.Conditional
+import Utility.SafeCommand
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 23b383a09d..1ecbb029b5 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -16,7 +16,7 @@ module Git.LsFiles (
 ) where
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 0016be4727..25b9ffad08 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -19,15 +19,15 @@ import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
 import Control.Monad (forM_)
-import Utility
+import Utility.SafeCommand
 
 import Git
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
-data Action = Action {
-		getSubcommand :: String,
-		getParams :: [CommandParam]
+data Action = Action
+	{ getSubcommand :: String
+	, getParams :: [CommandParam]
 	} deriving (Show, Eq, Ord)
 
 {- A queue of actions to perform (in any order) on a git repository,
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index b0da071703..a5bcbeac4a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -18,7 +18,7 @@ import Data.Maybe
 import Data.String.Utils
 
 import Git
-import Utility
+import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
diff --git a/Init.hs b/Init.hs
index a469657a11..2067c524cf 100644
--- a/Init.hs
+++ b/Init.hs
@@ -22,6 +22,7 @@ import Version
 import Messages
 import Types
 import Utility
+import Utility.Conditional
 import UUID
 
 initialize :: Annex ()
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 069209792d..ebb4b10a5b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -29,6 +29,8 @@ import UUID
 import Locations
 import Config
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Messages
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index fd227f85d8..7ddb60462f 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -27,6 +27,8 @@ import Utility.CopyFile
 import Config
 import Content
 import Utility
+import Utility.Conditional
+import Utility.Path
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 80ba8a153d..fadd48a036 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -26,6 +26,8 @@ import Messages
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
+import Utility.SafeCommand
+import Utility.Path
 import qualified Utility.Url as Url
 import Config
 import Init
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c302a0ff5e..b842588c0c 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -17,7 +17,7 @@ import Types.Remote
 import qualified Git
 import qualified Annex
 import UUID
-import Utility
+import Utility.SafeCommand
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ef52d0482b..54b9806ffc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -28,6 +28,7 @@ import Locations
 import Config
 import Content
 import Utility
+import Utility.SafeCommand
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index eba67e3fd6..3707966ad6 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -26,11 +26,14 @@ import Locations
 import Config
 import Content
 import Utility
+import Utility.Conditional
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 import Messages
 import Utility.RsyncFile
+import Utility.SafeCommand
+import Utility.Path
 
 type RsyncUrl = String
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 160d9309fe..b4567a0b71 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -31,6 +31,8 @@ import Backend
 import Messages
 import Version
 import Utility
+import Utility.SafeCommand
+import Utility.Path
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 0b1d69f8e1..ffd0f06535 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -20,6 +20,8 @@ import qualified Git
 import qualified Branch
 import Messages
 import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import LocationLog
 import Content
 
diff --git a/Utility.hs b/Utility.hs
index 8a332601b7..788dc41030 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -6,20 +6,8 @@
  -}
 
 module Utility (
-	CommandParam(..),
-	toCommand,
 	hGetContentsStrict,
 	readFileStrict,
-	parentDir,
-	absPath,
-	absPathFrom,
-	relPathCwdToFile,
-	relPathDirToFile,
-	boolSystem,
-	boolSystemEnv,
-	executeFile,
-	shellEscape,
-	shellUnEscape,
 	unsetFileMode,
 	readMaybe,
 	viaTmp,
@@ -27,125 +15,19 @@ module Utility (
 	dirContains,
 	dirContents,
 	myHomeDir,
-	catchBool,
-	whenM,
-	(>>?),
-	unlessM,
-	(>>!),
-	
-	prop_idempotent_shellEscape,
-	prop_idempotent_shellEscape_multiword,
-	prop_parentDir_basics,
-	prop_relPathDirToFile_basics
+	catchBool
 ) where
 
 import IO (bracket)
 import System.IO
-import System.Exit
-import qualified System.Posix.Process
 import System.Posix.Process hiding (executeFile)
-import System.Posix.Signals
 import System.Posix.Files
 import System.Posix.Types
 import System.Posix.User
-import Data.String.Utils
-import System.Path
 import System.FilePath
 import System.Directory
 import Foreign (complement)
-import Data.List
-import Data.Maybe
-import Control.Monad (liftM2, when, unless)
-import System.Log.Logger
-
-{- A type for parameters passed to a shell command. A command can
- - be passed either some Params (multiple parameters can be included,
- - whitespace-separated, or a single Param (for when parameters contain
- - whitespace), or a File.
- -}
-data CommandParam = Params String | Param String | File FilePath
-	deriving (Eq, Show, Ord)
-
-{- Used to pass a list of CommandParams to a function that runs
- - a command and expects Strings. -}
-toCommand :: [CommandParam] -> [String]
-toCommand = (>>= unwrap)
-	where
-		unwrap (Param s) = [s]
-		unwrap (Params s) = filter (not . null) (split " " s)
-		-- Files that start with a dash are modified to avoid
-		-- the command interpreting them as options.
-		unwrap (File s@('-':_)) = ["./" ++ s]
-		unwrap (File s) = [s]
-
-{- Run a system command, and returns True or False
- - if it succeeded or failed.
- -
- - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
- -}
-boolSystem :: FilePath -> [CommandParam] -> IO Bool
-boolSystem command params = boolSystemEnv command params Nothing
-
-boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
-boolSystemEnv command params env = do
-	-- Going low-level because all the high-level system functions
-	-- block SIGINT etc. We need to block SIGCHLD, but allow
-	-- SIGINT to do its default program termination.
-	let sigset = addSignal sigCHLD emptySignalSet
-	oldint <- installHandler sigINT Default Nothing
-	oldset <- getSignalMask
-	blockSignals sigset
-	childpid <- forkProcess $ childaction oldint oldset
-	mps <- getProcessStatus True False childpid
-	restoresignals oldint oldset
-	case mps of
-		Just (Exited ExitSuccess) -> return True
-		_ -> return False
-	where
-		restoresignals oldint oldset = do
-			_ <- installHandler sigINT oldint Nothing
-			setSignalMask oldset
-		childaction oldint oldset = do
-			restoresignals oldint oldset
-			executeFile command True (toCommand params) env
-
-{- executeFile with debug logging -}
-executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
-executeFile c path p e = do
-	debugM "Utility.executeFile" $
-		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
-	System.Posix.Process.executeFile c path p e
-
-{- Escapes a filename or other parameter to be safely able to be exposed to
- - the shell. -}
-shellEscape :: String -> String
-shellEscape f = "'" ++ escaped ++ "'"
-	where
-		-- replace ' with '"'"'
-		escaped = join "'\"'\"'" $ split "'" f
-
-{- Unescapes a set of shellEscaped words or filenames. -}
-shellUnEscape :: String -> [String]
-shellUnEscape [] = []
-shellUnEscape s = word : shellUnEscape rest
-	where
-		(word, rest) = findword "" s
-		findword w [] = (w, "")
-		findword w (c:cs)
-			| c == ' ' = (w, cs)
-			| c == '\'' = inquote c w cs
-			| c == '"' = inquote c w cs
-			| otherwise = findword (w++[c]) cs
-		inquote _ w [] = (w, "")
-		inquote q w (c:cs)
-			| c == q = findword w cs
-			| otherwise = inquote q (w++[c]) cs
-
-{- For quickcheck. -}
-prop_idempotent_shellEscape :: String -> Bool
-prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
-prop_idempotent_shellEscape_multiword :: [String] -> Bool
-prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
+import Utility.Path
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -156,82 +38,6 @@ hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict f = readFile f >>= \s -> length s `seq` return s
 
-{- Returns the parent directory of a path. Parent of / is "" -}
-parentDir :: FilePath -> FilePath
-parentDir dir =
-	if not $ null dirs
-	then slash ++ join s (take (length dirs - 1) dirs)
-	else ""
-		where
-			dirs = filter (not . null) $ split s dir
-			slash = if isAbsolute dir then s else ""
-			s = [pathSeparator]
-
-prop_parentDir_basics :: FilePath -> Bool
-prop_parentDir_basics dir
-	| null dir = True
-	| dir == "/" = parentDir dir == ""
-	| otherwise = p /= dir
-	where
-		p = parentDir dir
-
-{- Checks if the first FilePath is, or could be said to contain the second.
- - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
- - are all equivilant.
- -}
-dirContains :: FilePath -> FilePath -> Bool
-dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
-	where
-		norm p = fromMaybe "" $ absNormPath p "."
-		a' = norm a
-		b' = norm b
-
-{- Converts a filename into a normalized, absolute path. -}
-absPath :: FilePath -> IO FilePath
-absPath file = do
-	cwd <- getCurrentDirectory
-	return $ absPathFrom cwd file
-
-{- Converts a filename into a normalized, absolute path
- - from the specified cwd. -}
-absPathFrom :: FilePath -> FilePath -> FilePath
-absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
-	where
-		bad = error $ "unable to normalize " ++ file
-
-{- Constructs a relative path from the CWD to a file.
- -
- - For example, assuming CWD is /tmp/foo/bar:
- -    relPathCwdToFile "/tmp/foo" == ".."
- -    relPathCwdToFile "/tmp/foo/bar" == "" 
- -}
-relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
-
-{- Constructs a relative path from a directory to a file.
- -
- - Both must be absolute, and normalized (eg with absNormpath).
- -}
-relPathDirToFile :: FilePath -> FilePath -> FilePath
-relPathDirToFile from to = path
-	where
-		s = [pathSeparator]
-		pfrom = split s from
-		pto = split s to
-		common = map fst $ filter same $ zip pfrom pto
-		same (c,d) = c == d
-		uncommon = drop numcommon pto
-		dotdots = replicate (length pfrom - numcommon) ".."
-		numcommon = length common
-		path = join s $ dotdots ++ uncommon
-
-prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
-prop_relPathDirToFile_basics from to
-	| from == to = null r
-	| otherwise = not (null r)
-	where
-		r = relPathDirToFile from to 
-
 {- Removes a FileMode from a file.
  - For example, call with otherWriteMode to chmod o-w -}
 unsetFileMode :: FilePath -> FileMode -> IO ()
@@ -288,20 +94,3 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
-
-{- when with a monadic conditional -}
-whenM :: Monad m => m Bool -> m () -> m ()
-whenM c a = c >>= flip when a
-
-unlessM :: Monad m => m Bool -> m () -> m ()
-unlessM c a = c >>= flip unless a
-
-(>>?) :: Monad m => m Bool -> m () -> m ()
-(>>?) = whenM
-
-(>>!) :: Monad m => m Bool -> m () -> m ()
-(>>!) = unlessM
-
--- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
-infixr 0 >>?
-infixr 0 >>!
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
new file mode 100644
index 0000000000..85e39ec64c
--- /dev/null
+++ b/Utility/Conditional.hs
@@ -0,0 +1,26 @@
+{- monadic conditional operators
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Conditional where
+
+import Control.Monad (when, unless)
+
+whenM :: Monad m => m Bool -> m () -> m ()
+whenM c a = c >>= flip when a
+
+unlessM :: Monad m => m Bool -> m () -> m ()
+unlessM c a = c >>= flip unless a
+
+(>>?) :: Monad m => m Bool -> m () -> m ()
+(>>?) = whenM
+
+(>>!) :: Monad m => m Bool -> m () -> m ()
+(>>!) = unlessM
+
+-- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
+infixr 0 >>?
+infixr 0 >>!
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index befb00f8f2..9019357196 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -9,7 +9,8 @@ module Utility.CopyFile (copyFile) where
 
 import System.Directory (doesFileExist, removeFile)
 
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
diff --git a/Utility/Path.hs b/Utility/Path.hs
new file mode 100644
index 0000000000..517c175bc4
--- /dev/null
+++ b/Utility/Path.hs
@@ -0,0 +1,92 @@
+{- path manipulation
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Path where
+
+import Data.String.Utils
+import System.Path
+import System.FilePath
+import System.Directory
+import Data.List
+import Data.Maybe
+import Control.Monad (liftM2)
+
+{- Returns the parent directory of a path. Parent of / is "" -}
+parentDir :: FilePath -> FilePath
+parentDir dir =
+	if not $ null dirs
+	then slash ++ join s (take (length dirs - 1) dirs)
+	else ""
+		where
+			dirs = filter (not . null) $ split s dir
+			slash = if isAbsolute dir then s else ""
+			s = [pathSeparator]
+
+prop_parentDir_basics :: FilePath -> Bool
+prop_parentDir_basics dir
+	| null dir = True
+	| dir == "/" = parentDir dir == ""
+	| otherwise = p /= dir
+	where
+		p = parentDir dir
+
+{- Checks if the first FilePath is, or could be said to contain the second.
+ - For example, "foo/" contains "foo/bar". Also, "foo", "./foo", "foo/" etc
+ - are all equivilant.
+ -}
+dirContains :: FilePath -> FilePath -> Bool
+dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
+	where
+		norm p = fromMaybe "" $ absNormPath p "."
+		a' = norm a
+		b' = norm b
+
+{- Converts a filename into a normalized, absolute path. -}
+absPath :: FilePath -> IO FilePath
+absPath file = do
+	cwd <- getCurrentDirectory
+	return $ absPathFrom cwd file
+
+{- Converts a filename into a normalized, absolute path
+ - from the specified cwd. -}
+absPathFrom :: FilePath -> FilePath -> FilePath
+absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
+	where
+		bad = error $ "unable to normalize " ++ file
+
+{- Constructs a relative path from the CWD to a file.
+ -
+ - For example, assuming CWD is /tmp/foo/bar:
+ -    relPathCwdToFile "/tmp/foo" == ".."
+ -    relPathCwdToFile "/tmp/foo/bar" == "" 
+ -}
+relPathCwdToFile :: FilePath -> IO FilePath
+relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
+
+{- Constructs a relative path from a directory to a file.
+ -
+ - Both must be absolute, and normalized (eg with absNormpath).
+ -}
+relPathDirToFile :: FilePath -> FilePath -> FilePath
+relPathDirToFile from to = path
+	where
+		s = [pathSeparator]
+		pfrom = split s from
+		pto = split s to
+		common = map fst $ filter same $ zip pfrom pto
+		same (c,d) = c == d
+		uncommon = drop numcommon pto
+		dotdots = replicate (length pfrom - numcommon) ".."
+		numcommon = length common
+		path = join s $ dotdots ++ uncommon
+
+prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
+prop_relPathDirToFile_basics from to
+	| from == to = null r
+	| otherwise = not (null r)
+	where
+		r = relPathDirToFile from to 
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index 6e21ba0632..b6c2267e87 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -9,7 +9,7 @@ module Utility.RsyncFile where
 
 import Data.String.Utils
 
-import Utility
+import Utility.SafeCommand
 
 {- Generates parameters to make rsync use a specified command as its remote
  - shell. -}
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
new file mode 100644
index 0000000000..ba9362603e
--- /dev/null
+++ b/Utility/SafeCommand.hs
@@ -0,0 +1,104 @@
+{- safely running shell commands
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.SafeCommand where
+
+import System.Exit
+import qualified System.Posix.Process
+import System.Posix.Process hiding (executeFile)
+import System.Posix.Signals
+import Data.String.Utils
+import System.Log.Logger
+
+{- A type for parameters passed to a shell command. A command can
+ - be passed either some Params (multiple parameters can be included,
+ - whitespace-separated, or a single Param (for when parameters contain
+ - whitespace), or a File.
+ -}
+data CommandParam = Params String | Param String | File FilePath
+	deriving (Eq, Show, Ord)
+
+{- Used to pass a list of CommandParams to a function that runs
+ - a command and expects Strings. -}
+toCommand :: [CommandParam] -> [String]
+toCommand = (>>= unwrap)
+	where
+		unwrap (Param s) = [s]
+		unwrap (Params s) = filter (not . null) (split " " s)
+		-- Files that start with a dash are modified to avoid
+		-- the command interpreting them as options.
+		unwrap (File s@('-':_)) = ["./" ++ s]
+		unwrap (File s) = [s]
+
+{- Run a system command, and returns True or False
+ - if it succeeded or failed.
+ -
+ - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
+ -}
+boolSystem :: FilePath -> [CommandParam] -> IO Bool
+boolSystem command params = boolSystemEnv command params Nothing
+
+boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
+boolSystemEnv command params env = do
+	-- Going low-level because all the high-level system functions
+	-- block SIGINT etc. We need to block SIGCHLD, but allow
+	-- SIGINT to do its default program termination.
+	let sigset = addSignal sigCHLD emptySignalSet
+	oldint <- installHandler sigINT Default Nothing
+	oldset <- getSignalMask
+	blockSignals sigset
+	childpid <- forkProcess $ childaction oldint oldset
+	mps <- getProcessStatus True False childpid
+	restoresignals oldint oldset
+	case mps of
+		Just (Exited ExitSuccess) -> return True
+		_ -> return False
+	where
+		restoresignals oldint oldset = do
+			_ <- installHandler sigINT oldint Nothing
+			setSignalMask oldset
+		childaction oldint oldset = do
+			restoresignals oldint oldset
+			executeFile command True (toCommand params) env
+
+{- executeFile with debug logging -}
+executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO ()
+executeFile c path p e = do
+	debugM "Utility.SafeCommand.executeFile" $
+		"Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e
+	System.Posix.Process.executeFile c path p e
+
+{- Escapes a filename or other parameter to be safely able to be exposed to
+ - the shell. -}
+shellEscape :: String -> String
+shellEscape f = "'" ++ escaped ++ "'"
+	where
+		-- replace ' with '"'"'
+		escaped = join "'\"'\"'" $ split "'" f
+
+{- Unescapes a set of shellEscaped words or filenames. -}
+shellUnEscape :: String -> [String]
+shellUnEscape [] = []
+shellUnEscape s = word : shellUnEscape rest
+	where
+		(word, rest) = findword "" s
+		findword w [] = (w, "")
+		findword w (c:cs)
+			| c == ' ' = (w, cs)
+			| c == '\'' = inquote c w cs
+			| c == '"' = inquote c w cs
+			| otherwise = findword (w++[c]) cs
+		inquote _ w [] = (w, "")
+		inquote q w (c:cs)
+			| c == q = findword w cs
+			| otherwise = inquote q (w++[c]) cs
+
+{- For quickcheck. -}
+prop_idempotent_shellEscape :: String -> Bool
+prop_idempotent_shellEscape s = [s] == (shellUnEscape . shellEscape) s
+prop_idempotent_shellEscape_multiword :: [String] -> Bool
+prop_idempotent_shellEscape_multiword s = s == (shellUnEscape . unwords . map shellEscape) s
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 6cbc362a03..05269552c7 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -10,7 +10,7 @@ module Utility.Ssh where
 import Control.Monad.State (liftIO)
 
 import qualified Git
-import Utility
+import Utility.SafeCommand
 import Types
 import Config
 
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 5954e0ff7e..69b53c34c0 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -19,7 +19,7 @@ import Network.URI
 
 import Types
 import Messages
-import Utility
+import Utility.SafeCommand
 
 type URLString = String
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 29ac63aea1..1fb928f9da 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -11,7 +11,8 @@ import Data.List
 import qualified Git
 import CmdLine
 import Command
-import Utility
+import Utility.Conditional
+import Utility.SafeCommand
 import Options
 
 import qualified Command.ConfigList
diff --git a/test.hs b/test.hs
index 2352df36a6..e4e1fb131d 100644
--- a/test.hs
+++ b/test.hs
@@ -24,11 +24,12 @@ import qualified Data.Map as M
 import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
+import Utility.SafeCommand
+
 import qualified Annex
 import qualified Backend
 import qualified Git
 import qualified Locations
-import qualified Utility
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
@@ -42,6 +43,7 @@ import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
 import qualified Crypto
+import qualified Utility.Path
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -72,11 +74,12 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
-	, qctest "prop_idempotent_shellEscape" Utility.prop_idempotent_shellEscape
-	, qctest "prop_idempotent_shellEscape_multiword" Utility.prop_idempotent_shellEscape_multiword
+	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape
+	, qctest "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword
 	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
-	, qctest "prop_parentDir_basics" Utility.prop_parentDir_basics
-	, qctest "prop_relPathDirToFile_basics" Utility.prop_relPathDirToFile_basics
+	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
+
+	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
 	]
@@ -117,8 +120,8 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			git_annex "add" ["-q", annexedfile] @? "add failed"
 			annexed_present annexedfile
 			writeFile ingitfile $ content ingitfile
-			Utility.boolSystem "git" [Utility.Param "add", Utility.File ingitfile] @? "git add failed"
-			Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
+			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
@@ -145,7 +148,7 @@ test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
 	let key = show $ fromJust r
 	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
 	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"
-	Utility.boolSystem "git" [Utility.Params "commit -q -a -m commit"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 	annexed_present sha1annexedfile
 	where
 		tmp = "tmpfile"
@@ -172,7 +175,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
 			git_annex "get" ["-q", annexedfile] @? "get failed"
-			Utility.boolSystem "git" [Utility.Params "remote rm origin"]
+			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
 			r <- git_annex "drop" ["-q", annexedfile]
 			not r @? "drop wrongly succeeded with no known copy of file"
@@ -303,12 +306,12 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 			then do
 				-- pre-commit depends on the file being
 				-- staged, normally git commit does this
-				Utility.boolSystem "git" [Utility.Param "add", Utility.File annexedfile]
+				boolSystem "git" [Param "add", File annexedfile]
 					@? "git add of edited file failed"
 				git_annex "pre-commit" ["-q"]
 					@? "pre-commit failed"
 			else do
-				Utility.boolSystem "git" [Utility.Params "commit -q -a -m contentchanged"]
+				boolSystem "git" [Params "commit -q -a -m contentchanged"]
 					@? "git commit of edited file failed"
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
@@ -326,7 +329,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	git_annex "fix" ["-q", annexedfile] @? "fix of present file failed"
 	annexed_present annexedfile
 	createDirectory subdir
-	Utility.boolSystem "git" [Utility.Param "mv", Utility.File annexedfile, Utility.File subdir]
+	boolSystem "git" [Param "mv", File annexedfile, File subdir]
 		@? "git mv failed"
 	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
@@ -364,9 +367,9 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" ["-q"] @? "fsck failed"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			fsck_should_fail "numcopies unsatisfied"
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 1"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
@@ -377,7 +380,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
 			git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo"
 		withremoteuntrusted = TestCase $ intmpclonerepo $ do
-			Utility.boolSystem "git" [Utility.Params "config annex.numcopies 2"] @? "git config failed"
+			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			git_annex "get" ["-q", annexedfile] @? "get failed"
 			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
 			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
@@ -448,9 +451,9 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	git_annex "get" ["-q", annexedfile] @? "get of file failed"
 	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
 	checkunused []
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey]
-	Utility.boolSystem "git" [Utility.Params "rm -q", Utility.File sha1annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also
@@ -526,10 +529,10 @@ setuprepo :: FilePath -> IO FilePath
 setuprepo dir = do
 	cleanup dir
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "init -q", Utility.File dir] @? "git init failed"
+	boolSystem "git" [Params "init -q", File dir] @? "git init failed"
 	indir dir $ do
-		Utility.boolSystem "git" [Utility.Params "config user.name", Utility.Param "Test User"] @? "git config failed"
-		Utility.boolSystem "git" [Utility.Params "config user.email test@example.com"] @? "git config failed"
+		boolSystem "git" [Params "config user.name", Param "Test User"] @? "git config failed"
+		boolSystem "git" [Params "config user.email test@example.com"] @? "git config failed"
 	return dir
 
 -- clones are always done as local clones; we cannot test ssh clones
@@ -537,7 +540,7 @@ clonerepo :: FilePath -> FilePath -> IO FilePath
 clonerepo old new = do
 	cleanup new
 	ensuretmpdir
-	Utility.boolSystem "git" [Utility.Params "clone -q", Utility.File old, Utility.File new] @? "git clone failed"
+	boolSystem "git" [Params "clone -q", File old, File new] @? "git clone failed"
 	indir new $ git_annex "init" ["-q", new] @? "git annex init failed"
 	return new
 	

From 20259c2955e408a72e0960207fc8be4cbeec2e21 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 23 Aug 2011 13:41:32 -0400
Subject: [PATCH 2143/8313] Set EMAIL when running test suite so that git does
 not need to be configured first. Closes: #638998

---
 debian/changelog | 7 +++++++
 test.hs          | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index a8280a156d..bb06e1fba9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20110820) UNRELEASED; urgency=low
+
+  * Set EMAIL when running test suite so that git does not need to be
+    configured first. Closes: #638998
+
+ -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
+
 git-annex (3.20110819) unstable; urgency=low
 
   * Now "git annex init" only has to be run once, when a git repository
diff --git a/test.hs b/test.hs
index e4e1fb131d..4d751a707b 100644
--- a/test.hs
+++ b/test.hs
@@ -646,6 +646,9 @@ prepare = do
 	p <- getEnvDefault  "PATH" ""
 	setEnv "PATH" (cwd ++ ":" ++ p) True
 	setEnv "TOPDIR" cwd True
+	-- Avoid git complaining if it cannot determine the user's email
+	-- address.
+	setEnv "EMAIL" "git-annex test " True
 
 changeToTmpDir :: FilePath -> IO ()
 changeToTmpDir t = do

From 678726c10c13481c082743808a5188d28567e2b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 25 Aug 2011 00:28:55 -0400
Subject: [PATCH 2144/8313] code simplification thanks to applicative functors

---
 Annex.hs           |  7 +++++--
 Branch.hs          |  7 ++++---
 Command.hs         |  7 ++++---
 Command/Migrate.hs |  3 ++-
 Command/Status.hs  | 10 ++++------
 Config.hs          |  6 +++---
 Crypto.hs          |  3 ++-
 Git.hs             |  3 ++-
 LocationLog.hs     |  3 ++-
 PresenceLog.hs     |  3 ++-
 Remote.hs          | 13 +++++++------
 RemoteLog.hs       |  3 ++-
 Upgrade/V1.hs      |  5 +++--
 Utility/Path.hs    |  4 ++--
 Utility/Url.hs     |  5 ++---
 15 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 07316bd370..287aed875e 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -20,6 +20,7 @@ module Annex (
 
 import Control.Monad.State
 import Control.Monad.IO.Control
+import Control.Applicative hiding (empty)
 
 import qualified Git
 import Git.Queue
@@ -36,7 +37,9 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 		Monad,
 		MonadIO,
 		MonadControlIO,
-		MonadState AnnexState
+		MonadState AnnexState,
+		Functor,
+		Applicative
 	)
 
 -- internal state storage
@@ -83,7 +86,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState `liftM` Git.configRead gitrepo
+new gitrepo = newState <$> Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Branch.hs b/Branch.hs
index d5bfe1b09f..5008b2e200 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -20,6 +20,7 @@ module Branch (
 
 import Control.Monad (when, unless, liftM)
 import Control.Monad.State (liftIO)
+import Control.Applicative ((<$>))
 import System.FilePath
 import System.Directory
 import Data.String.Utils
@@ -158,7 +159,7 @@ update = do
 		staged <- stageJournalFiles
 
 		refs <- siblingBranches
-		updated <- catMaybes `liftM` mapM updateRef refs
+		updated <- catMaybes <$> mapM updateRef refs
 		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
@@ -182,7 +183,7 @@ hasOrigin = refExists originname
 
 {- Does the git-annex branch or a foo/git-annex branch exist? -}
 hasSomeBranch :: Annex Bool
-hasSomeBranch = liftM (not . null) siblingBranches
+hasSomeBranch = not . null <$> siblingBranches
 
 {- List of all git-annex branches, including the main one and any
  - from remotes. -}
@@ -323,7 +324,7 @@ getJournalFile file = do
 
 {- List of journal files. -}
 getJournalFiles :: Annex [FilePath]
-getJournalFiles = liftM (map fileJournal) getJournalFilesRaw
+getJournalFiles = map fileJournal <$> getJournalFilesRaw
 
 getJournalFilesRaw :: Annex [FilePath]
 getJournalFilesRaw = do
diff --git a/Command.hs b/Command.hs
index d3c1640ee0..21c50f9c02 100644
--- a/Command.hs
+++ b/Command.hs
@@ -11,6 +11,7 @@ import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
 import Control.Monad (filterM, liftM, when)
+import Control.Applicative
 import System.Path.WildMatch
 import Text.Regex.PCRE.Light.Char8
 import Data.List
@@ -183,7 +184,7 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 backendPairs :: CommandSeekBackendFiles
-backendPairs a files = liftM (map a) $ Backend.chooseBackends files
+backendPairs a files = map a <$> Backend.chooseBackends files
 
 {- Filter out files those matching the exclude glob pattern,
  - if it was specified. -}
@@ -204,7 +205,7 @@ wildsRegex ws = compile regex []
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
-notSymlink f = liftM (not . isSymbolicLink) $ liftIO $ getSymbolicLinkStatus f
+notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
 {- Descriptions of params used in usage messages. -}
 paramRepeating :: String -> String
@@ -271,4 +272,4 @@ preserveOrder orig new = collect orig new
  - of git file list commands, that assumption tends to hold.
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserveOrder a files = liftM (preserveOrder files) (a files)
+runPreserveOrder a files = preserveOrder files <$> a files
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 25227ae162..6ad7e239c9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -8,6 +8,7 @@
 module Command.Migrate where
 
 import Control.Monad.State (liftIO)
+import Control.Applicative
 import System.Posix.Files
 import System.Directory
 import System.FilePath
@@ -39,7 +40,7 @@ start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = return . head =<< Backend.orderedList
+		choosebackend Nothing = head <$> Backend.orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
diff --git a/Command/Status.hs b/Command/Status.hs
index aef4df2329..5c82744b10 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -8,6 +8,7 @@
 module Command.Status where
 
 import Control.Monad.State
+import Control.Applicative
 import Data.Maybe
 import System.IO
 import Data.List
@@ -112,12 +113,10 @@ total_annex_size = stat "total annex size" $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $ 
-	return . show . snd =<< cachedKeysPresent
+local_annex_keys = stat "local annex keys" $ show . snd <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $
-	return . show . snd =<< cachedKeysReferenced
+total_annex_keys = stat "total annex keys" $ show . snd <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -126,8 +125,7 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $
-	return . usage =<< cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
 		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
diff --git a/Config.hs b/Config.hs
index 12f6480470..b4f4c0b922 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,7 +9,7 @@ module Config where
 
 import Data.Maybe
 import Control.Monad.State (liftIO)
-import Control.Monad (liftM)
+import Control.Applicative
 import System.Cmd.Utils
 
 import qualified Git
@@ -47,8 +47,8 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
-	return . safeparse =<< if not $ null cmd
-			then liftM snd $ liftIO $ pipeFrom "sh" ["-c", cmd]
+	safeparse <$> if not $ null cmd
+			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
 			else getConfig r "cost" ""
 	where
 		safeparse v
diff --git a/Crypto.hs b/Crypto.hs
index ed29747aa6..d789b44556 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -38,6 +38,7 @@ import System.IO
 import System.Posix.IO
 import System.Posix.Types
 import System.Posix.Process
+import Control.Applicative
 import Control.Concurrent
 import Control.Exception (finally)
 import System.Exit
@@ -136,7 +137,7 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
 decryptCipher _ (EncryptedCipher encipher _) = 
-	return . Cipher =<< gpgPipeStrict decrypt encipher
+	Cipher <$> gpgPipeStrict decrypt encipher
 	where
 		decrypt = [ Param "--decrypt" ]
 
diff --git a/Git.hs b/Git.hs
index 7155b26343..ab43504e1a 100644
--- a/Git.hs
+++ b/Git.hs
@@ -63,6 +63,7 @@ module Git (
 ) where
 
 import Control.Monad (unless, when)
+import Control.Applicative
 import System.Directory
 import System.FilePath
 import System.Posix.Directory
@@ -446,7 +447,7 @@ commit g message newref parentrefs = do
 		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
 	run g "update-ref" [Param newref, Param sha]
 	where
-		ignorehandle a = return . snd =<< a
+		ignorehandle a = snd <$> a
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
 {- Reads null terminated output of a git command (as enabled by the -z 
diff --git a/LocationLog.hs b/LocationLog.hs
index 768483fa1b..fa660c8b67 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -24,6 +24,7 @@ module LocationLog (
 
 import System.FilePath
 import Control.Monad (when)
+import Control.Applicative
 import Data.Maybe
 
 import qualified Git
@@ -49,7 +50,7 @@ keyLocations key = currentLog $ logFile key
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys = return . mapMaybe (logFileKey . takeFileName) =<< Branch.files
+loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/PresenceLog.hs b/PresenceLog.hs
index ccb75ff5be..e0c8729979 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -28,6 +28,7 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as Map
 import Control.Monad.State (liftIO)
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -81,7 +82,7 @@ addLog file line = do
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = return . parseLog =<< Branch.get file
+readLog file = parseLog <$> Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog s = filter parsable $ map read $ lines s
diff --git a/Remote.hs b/Remote.hs
index 1a5006f6fb..2c883f1a8f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -29,11 +29,12 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad (filterM, liftM2)
+import Control.Monad (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
+import Control.Applicative
 
 import Types
 import Types.Remote
@@ -111,10 +112,10 @@ nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
-		Left e -> return . fromMaybe (error e) =<< byDescription
+		Left e -> fromMaybe (error e) <$> byDescription
 		Right r -> return $ uuid r
 	where
-		byDescription = return . M.lookup n . invertMap =<< uuidMap
+		byDescription = M.lookup n . invertMap <$> uuidMap
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
@@ -124,10 +125,10 @@ prettyPrintUUIDs uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	-- Show descriptions from the uuid log, falling back to remote names,
 	-- as some remotes may not be in the uuid log
-	m <- liftM2 M.union uuidMap $
-		return . M.fromList . map (\r -> (uuid r, name r)) =<< genList
+	m <- M.union <$> uuidMap <*> availMap
 	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
 	where
+		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		prettify m u here = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
@@ -147,7 +148,7 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
-keyPossibilities key = return . fst =<< keyPossibilities' False key
+keyPossibilities key = fst <$> keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the LocationLog indicate may have a key.
  -
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 69a82f4987..620c0d8757 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -19,6 +19,7 @@ import Data.List
 import qualified Data.Map as M
 import Data.Maybe
 import Data.Char
+import Control.Applicative
 
 import qualified Branch
 import Types
@@ -40,7 +41,7 @@ configSet u c = do
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = return . remoteLogParse =<< Branch.get remoteLog
+readRemoteLog = remoteLogParse <$> Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index b4567a0b71..9c3fd99595 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,6 +11,7 @@ import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import Control.Monad (filterM, forM_, unless)
+import Control.Applicative
 import System.Posix.Files
 import System.FilePath
 import Data.String.Utils
@@ -192,7 +193,7 @@ writeLog1 :: FilePath -> [LogLine] -> IO ()
 writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
 
 readLog1 :: FilePath -> IO [LogLine]
-readLog1 file = catch (return . parseLog =<< readFileStrict file) (const $ return [])
+readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
@@ -201,7 +202,7 @@ lookupFile1 file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = return . takeFileName =<< readSymbolicLink file
+		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = case maybeLookupBackendName bname of
 			Nothing -> do
 				unless (null kname || null bname ||
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 517c175bc4..9b8041dad0 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -13,7 +13,7 @@ import System.FilePath
 import System.Directory
 import Data.List
 import Data.Maybe
-import Control.Monad (liftM2)
+import Control.Applicative
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
@@ -65,7 +65,7 @@ absPathFrom cwd file = fromMaybe bad $ absNormPath cwd file
  -    relPathCwdToFile "/tmp/foo/bar" == "" 
  -}
 relPathCwdToFile :: FilePath -> IO FilePath
-relPathCwdToFile f = liftM2 relPathDirToFile getCurrentDirectory (absPath f)
+relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
 
 {- Constructs a relative path from a directory to a file.
  -
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 69b53c34c0..f678720ed8 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -11,7 +11,7 @@ module Utility.Url (
 	get
 ) where
 
-import Control.Monad (liftM)
+import Control.Applicative
 import Control.Monad.State (liftIO)
 import qualified Network.Browser as Browser
 import Network.HTTP
@@ -64,7 +64,6 @@ request url requesttype = Browser.browse $ do
 	Browser.setErrHandler ignore
 	Browser.setOutHandler ignore
 	Browser.setAllowRedirects True
-	liftM snd $ Browser.request
-		(mkRequest requesttype url :: Request_String)
+	snd <$> Browser.request (mkRequest requesttype url :: Request_String)
 	where
 		ignore = const $ return ()

From 9170e1b87d1eac000e9ea4d3944a73209ba9c718 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 26 Aug 2011 10:55:43 +0000
Subject: [PATCH 2145/8313]

---
 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn

diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
new file mode 100644
index 0000000000..7a579f8fe6
--- /dev/null
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -0,0 +1 @@
+When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.

From f82da1d9dca0712cdd87e3fc0ed8a2c2e2440228 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 07:08:15 -0400
Subject: [PATCH 2146/8313] show a message if asked to get something from the
 web that is not there

---
 Remote/Web.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 5bc6a204b9..0e4b757673 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -86,8 +86,12 @@ setUrl key url status = do
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
 
 downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = iter =<< getUrls key
+downloadKey key file = get =<< getUrls key
 	where
+		get [] = do
+			warning "no known url"
+			return False
+		get a = iter a
 		iter [] = return False
 		iter (url:urls) = do
 			ok <- Url.download url file

From 6e750764b7d30d9cb0684cdaadd79ec091a4fda6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 12:31:50 -0400
Subject: [PATCH 2147/8313] The wget command will now be used in preference to
 curl, if available.

Got tired of curl's various ugly progress bars.
---
 Utility.hs       | 12 +++++++++++-
 Utility/Url.hs   | 24 +++++++++++++++++-------
 debian/changelog |  1 +
 debian/control   |  2 +-
 doc/install.mdwn |  2 +-
 5 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/Utility.hs b/Utility.hs
index 788dc41030..451b1b44fc 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -15,7 +15,8 @@ module Utility (
 	dirContains,
 	dirContents,
 	myHomeDir,
-	catchBool
+	catchBool,
+	inPath
 ) where
 
 import IO (bracket)
@@ -94,3 +95,12 @@ myHomeDir = do
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
+
+{- Checks if a command is available in PATH. -}
+inPath :: String -> IO Bool
+inPath command = search =<< getSearchPath
+	where
+		search [] = return False
+		search (d:ds) = do
+			e <- doesFileExist $ d  command
+			if e then return True else search ds
diff --git a/Utility/Url.hs b/Utility/Url.hs
index f678720ed8..6ddeecc14f 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -20,6 +20,7 @@ import Network.URI
 import Types
 import Messages
 import Utility.SafeCommand
+import Utility
 
 type URLString = String
 
@@ -35,15 +36,24 @@ exists url =
 				_ -> return False
 
 {- Used to download large files, such as the contents of keys.
- - Uses curl program for its progress bar. -}
+ - Uses wget or curl program for its progress bar. (Wget has a better one,
+ - so is preferred.) -}
 download :: URLString -> FilePath -> Annex Bool
 download url file = do
-	showOutput -- make way for curl progress bar
-	-- Uses the -# progress display, because the normal one is very
-	-- confusing when resuming, showing the remainder to download
-	-- as the whole file, and not indicating how much percent was
-	-- downloaded before the resume.
-	liftIO $ boolSystem "curl" [Params "-L -C - -# -o", File file, File url]
+	showOutput -- make way for program's progress bar
+	e <- liftIO $ inPath "wget"
+	if e
+		then
+			liftIO $ boolSystem "wget"
+				[Params "-c -O", File file, File url]
+		else
+			-- Uses the -# progress display, because the normal
+			-- one is very confusing when resuming, showing
+			-- the remainder to download as the whole file,
+			-- and not indicating how much percent was
+			-- downloaded before the resume.
+			liftIO $ boolSystem "curl" 
+				[Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}
 get :: URLString -> IO String
diff --git a/debian/changelog b/debian/changelog
index bb06e1fba9..54fdbd835c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
+  * The wget command will now be used in preference to curl, if available.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/debian/control b/debian/control
index 975faf5ea5..63488dc68d 100644
--- a/debian/control
+++ b/debian/control
@@ -31,7 +31,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	git | git-core,
 	uuid,
 	rsync,
-	curl,
+	wget | curl,
 	openssh-client
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 49ddd913f0..ac521da187 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -34,7 +34,7 @@ To build and use git-annex, you will need:
     (or `uuidgen` from util-linux)
   * [xargs](http://savannah.gnu.org/projects/findutils/)
   * [rsync](http://rsync.samba.org/)
-  * [curl](http://http://curl.haxx.se/) (optional, but recommended)
+  * [wget](http://www.gnu.org/software/wget/) or [curl](http://http://curl.haxx.se/) (optional, but recommended)
   * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
     a sha1 command will also do)
   * [gpg](http://gnupg.org/) (optional; needed for encryption)

From b26ee162f33858aa4ef82fbe3d56eecc00590755 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 27 Aug 2011 12:34:02 -0400
Subject: [PATCH 2148/8313] guess not everyone configures their shell to show
 nonzero exit codes..

---
 Makefile                                              | 4 +++-
 doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index fa087c1ae2..20d1347140 100644
--- a/Makefile
+++ b/Makefile
@@ -53,7 +53,9 @@ test: $(bins)
 	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
-		./test; \
+		if ! ./test; then \
+			echo "** test suite failed!" >&2; \
+		fi; \
 	fi
 
 testcoverage: $(bins)
diff --git a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
index 7a579f8fe6..2f486ad652 100644
--- a/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
+++ b/doc/bugs/test_suite_shouldn__39__t_fail_silently.mdwn
@@ -1 +1,3 @@
 When the test suite cannot be compiled, the build just fails silenty. This means that in automated builds there is no easy way to ensure that the generated binaries have passed the test suite, because it may not even have been run! IMHO, "make test" should fail (i.e. return a non-zero exit code) when it can't succeeed.
+
+> Ok, fixed. --[[Joey]] [[done]]

From 999d5df90b013a7cc8a390c940785118400faf8a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 28 Aug 2011 15:46:49 -0400
Subject: [PATCH 2149/8313] factor out firstM and anyM

Control.Monad.Loops has these, but has no Debian package yet.
---
 Remote/Web.hs |  7 ++-----
 Utility.hs    | 28 ++++++++++++++++++++++------
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/Remote/Web.hs b/Remote/Web.hs
index 0e4b757673..3695bb1ab3 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -24,6 +24,7 @@ import Config
 import PresenceLog
 import LocationLog
 import Locations
+import Utility
 import qualified Utility.Url as Url
 
 type URLString = String
@@ -91,11 +92,7 @@ downloadKey key file = get =<< getUrls key
 		get [] = do
 			warning "no known url"
 			return False
-		get a = iter a
-		iter [] = return False
-		iter (url:urls) = do
-			ok <- Url.download url file
-			if ok then return ok else iter urls
+		get urls = anyM (`Url.download` file) urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
diff --git a/Utility.hs b/Utility.hs
index 451b1b44fc..ce17363488 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -16,7 +16,9 @@ module Utility (
 	dirContents,
 	myHomeDir,
 	catchBool,
-	inPath
+	inPath,
+	firstM,
+	anyM
 ) where
 
 import IO (bracket)
@@ -29,6 +31,8 @@ import System.FilePath
 import System.Directory
 import Foreign (complement)
 import Utility.Path
+import Data.Maybe
+import Control.Monad (liftM)
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -96,11 +100,23 @@ myHomeDir = do
 catchBool :: IO Bool -> IO Bool
 catchBool = flip catch (const $ return False)
 
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = do
+	q <- p x
+	if q
+		then return (Just x)
+		else firstM p xs
+
+{- Returns true if any value in the list satisfies the preducate,
+ - stopping once one is found. -}
+anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
+
 {- Checks if a command is available in PATH. -}
 inPath :: String -> IO Bool
-inPath command = search =<< getSearchPath
+inPath command = getSearchPath >>= anyM indir
 	where
-		search [] = return False
-		search (d:ds) = do
-			e <- doesFileExist $ d  command
-			if e then return True else search ds
+		indir d = doesFileExist $ d  command

From bbba6c19bd03f2b4a4ce8a38a2423c794826b1c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 28 Aug 2011 16:28:38 -0400
Subject: [PATCH 2150/8313] update documentation for new, neutered key-value
 backends

Backends are now only used to generate keys (and check them); they
are not arbitrary key-value stores for data, because it turned out such
a store is better modeled as a special remote. Updated docs to not
imply backends do more than they do now.

Sometimes I'm tempted to rename "backend" to "keytype" or something,
which would really be more clear. But it would be an annoying transition
for users, with annex.backends etc.
---
 doc/backends.mdwn                             | 22 +++++-----------
 doc/copies.mdwn                               |  6 ++---
 doc/git-annex.mdwn                            | 26 +++++++++----------
 .../fsck:_verifying_your_data.mdwn            | 10 +++----
 doc/walkthrough/unused_data.mdwn              |  2 +-
 5 files changed, 29 insertions(+), 37 deletions(-)

diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index 9e698032d8..f2f1c5580c 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -1,24 +1,16 @@
-git-annex uses a key-value abstraction layer to allow file contents to be
-stored in different ways. In theory, any key-value storage system could be
-used to store file contents.
-
 When a file is annexed, a key is generated from its content and/or metadata.
 The file checked into git symlinks to the key. This key can later be used
 to retrieve the file's content (its value).
 
-Multiple pluggable backends are supported, and a single repository
-can use different backends for different files.
+Multiple pluggable key-value backends are supported, and a single repository
+can use different ones for different files.
 
-These backends can transfer file contents between configured git remotes.
-It's also possible to use [[special_remotes]], such as Amazon S3 with
-these backends.
-
-* `WORM` ("Write Once, Read Many") This backend assumes that any file with
-  the same basename, size, and modification time has the same content. So with
-  this backend, files can be moved around, but should never be added to
+* `WORM` ("Write Once, Read Many") This assumes that any file with
+  the same basename, size, and modification time has the same content. So
+  files can be moved around, but should never be added to
   or changed. This is the default, and the least expensive backend.
-* `SHA1` -- This backend uses a key based on a sha1 checksum. This backend
-  allows modifications of files to be tracked. Its need to generate checksums
+* `SHA1` -- This uses a key based on a sha1 checksum. This allows
+  modifications of files to be tracked. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
diff --git a/doc/copies.mdwn b/doc/copies.mdwn
index 16eba19c81..93cbd8ea80 100644
--- a/doc/copies.mdwn
+++ b/doc/copies.mdwn
@@ -1,8 +1,8 @@
-The WORM and SHA1 key-value [[backends]] store data inside
-your git repository's `.git` directory, not in some external data store.
+Annexed data is stored inside  your git repository's `.git/annex` directory.
+Some [[special_remotes]] can store annexed data elsewhere.
 
 It's important that data not get lost by an ill-considered `git annex drop`
-command.  So, then using those backends, git-annex can be configured to try
+command.  So, git-annex can be configured to try
 to keep N copies of a file's content available across all repositories. 
 (Although [[untrusted_repositories|trust]] don't count toward this total.)
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a262d465fd..52599611ed 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -72,15 +72,15 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * get [path ...]
 
-  Makes the content of annexed files available in this repository. Depending
-  on the backend used, this will involve copying them from another repository,
-  or downloading them, or transferring them from some kind of key-value store.
+  Makes the content of annexed files available in this repository. This
+  will involve copying them from another repository, or downloading them,
+  or transferring them from some kind of key-value store.
 
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
 
-  git-annex may refuse to drop content if the backend does not think
+  git-annex may refuse to drop content if it does not think
   it is safe to do so, typically because of the setting of annex.numcopies.
 
 * move [path ...]
@@ -207,14 +207,14 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * migrate [path ...]
 
-  Changes the specified annexed files to store their content in the
-  default backend (or the one specified with --backend). Only files whose
-  content is currently available are migrated.
+  Changes the specified annexed files to use the default key-value backend
+  (or the one specified with --backend). Only files whose content
+  is currently available are migrated.
 
-  Note that the content is not removed from the backend it was previously in.
-  Use `git annex unused` to find and remove such content.
+  Note that the content is also still available using the old key after
+  migration. Use `git annex unused` to find and remove the old key.
 
-  Normally, nothing will be done to files already in the backend.
+  Normally, nothing will be done to files already using the new backend.
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
@@ -293,7 +293,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
-  to link to a specified key in the key-value backend.
+  in the git repository to link to a specified key.
 
 * dropkey [key ...]
 
@@ -500,8 +500,8 @@ Here are all the supported configuration settings.
 
 # CONFIGURATION VIA .gitattributes
 
-The backend used when adding a new file to the annex can be configured
-on a per-file-type basis via `.gitattributes` files. In the file,
+The key-value backend used when adding a new file to the annex can be
+configured on a per-file-type basis via `.gitattributes` files. In the file,
 the `annex.backend` attribute can be set to the name of the backend to
 use. For example, this here's how to use the WORM backend by default,
 but the SHA1 backend for ogg files:
diff --git a/doc/walkthrough/fsck:_verifying_your_data.mdwn b/doc/walkthrough/fsck:_verifying_your_data.mdwn
index 7e05469a12..d036332fb3 100644
--- a/doc/walkthrough/fsck:_verifying_your_data.mdwn
+++ b/doc/walkthrough/fsck:_verifying_your_data.mdwn
@@ -1,8 +1,8 @@
-You can use the fsck subcommand to check for problems in your data.
-What can be checked depends on the [[backend|backends]] you've used to store
-the data. For example, when you use the SHA1 backend, fsck will verify that
-the checksums of your files are good. Fsck also checks that the annex.numcopies
-setting is satisfied for all files.
+You can use the fsck subcommand to check for problems in your data. What
+can be checked depends on the key-value [[backend|backends]] you've used
+for the data. For example, when you use the SHA1 backend, fsck will verify
+that the checksums of your files are good. Fsck also checks that the
+annex.numcopies setting is satisfied for all files.
 
 	# git annex fsck
 	fsck some_file (checksum...) ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index fb84193034..e142b576c0 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -2,7 +2,7 @@ It's possible for data to accumulate in the annex that no files point to
 anymore. One way it can happen is if you `git rm` a file without 
 first calling `git annex drop`. And, when you modify an annexed file, the old
 content of the file remains in the annex. Another way is when migrating
-between backends.
+between key-value [[backends|backend]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and

From b6758746f6433a46ac14bad58f9f6e0220a42e3e Mon Sep 17 00:00:00 2001
From: cwg 
Date: Mon, 29 Aug 2011 12:35:19 +0000
Subject: [PATCH 2151/8313]

---
 doc/forum/advantages_of_SHA__42___over_WORM.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/advantages_of_SHA__42___over_WORM.mdwn

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
new file mode 100644
index 0000000000..b959db550c
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -0,0 +1,6 @@
+Thanks for creating git-annex.
+
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.
+
+Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, is there actually any advantage in using SHA*?

From eb4607aafc193073468da09a3cac5d54f6c16735 Mon Sep 17 00:00:00 2001
From: cwg 
Date: Mon, 29 Aug 2011 12:37:23 +0000
Subject: [PATCH 2152/8313]

---
 doc/forum/advantages_of_SHA__42___over_WORM.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
index b959db550c..5b544593f5 100644
--- a/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
+++ b/doc/forum/advantages_of_SHA__42___over_WORM.mdwn
@@ -1,6 +1,5 @@
 Thanks for creating git-annex.
 
-I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.
+I am confused about the advantages of the SHA* backends over WORM.  The "backends" page in this wiki says that with WORM, files "can be moved around, but should never be added to or changed".  But I don't see any difference to SHA* files as long as the premise of WORM that "any file with the same basename, size, and modification time has the same content" is true.  Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
 
-Using "git annex unlock", WORM files can be modified in the same way as SHA* files.
-If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, is there actually any advantage in using SHA*?
+If the storage I use is dependable (i.e. I don't need SHA checksums for detection of corruption), and I don't need to optimize for the case that the modification date of a file is changed but the contents stay the same, and if it is unlikely that several files will be identical, is there actually any advantage in using SHA*?

From 2a76ad41673968793a479f79ed33ba83a0b10773 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 29 Aug 2011 16:10:38 +0000
Subject: [PATCH 2153/8313] Added a comment

---
 .../comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment

diff --git a/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
new file mode 100644
index 0000000000..218027ca53
--- /dev/null
+++ b/doc/forum/advantages_of_SHA__42___over_WORM/comment_1_96c354cac4b5ce5cf6664943bc84db1d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-08-29T16:10:38Z"
+ content="""
+You're right -- as long as nothing changes a file without letting the modification time update, editing WORM files is safe.
+"""]]

From 025e66e3d35d89fc3c3dd2c81e2f618941130a26 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 12:08:54 -0400
Subject: [PATCH 2154/8313] update to not overstate the danger or WORM

---
 doc/backends.mdwn | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index f2f1c5580c..ebcdedc2a7 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -6,11 +6,11 @@ Multiple pluggable key-value backends are supported, and a single repository
 can use different ones for different files.
 
 * `WORM` ("Write Once, Read Many") This assumes that any file with
-  the same basename, size, and modification time has the same content. So
-  files can be moved around, but should never be added to
-  or changed. This is the default, and the least expensive backend.
+  the same basename, size, and modification time has the same content.
+  This is the default, and the least expensive backend.
 * `SHA1` -- This uses a key based on a sha1 checksum. This allows
-  modifications of files to be tracked. Its need to generate checksums
+  verifying that the file content is right, and can avoid duplicates of
+  files with the same content. Its need to generate checksums
   can make it slower for large files.
 * `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
   checksums. Mostly useful for the very paranoid, or anyone who is
@@ -28,7 +28,7 @@ files, the `.gitattributes` file can be used. The `annex.backend`
 attribute can be set to the name of the backend to use for matching files.
 
 For example, to use the SHA1 backend for sound files, which tend to be
-smallish and might be modified over time, you could set in
+smallish and might be modified or copied over time, you could set in
 `.gitattributes`:
 
 	*.mp3 annex.backend=SHA1

From 676c467801051e71ea7668615985301f26ab0f3e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 12:49:38 -0400
Subject: [PATCH 2155/8313] close

---
 ...ude_repo_description_and__47__or_UUID_in_commit_message.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
index 9ca61bff55..be7e2dacc8 100644
--- a/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
+++ b/doc/todo/git_annex_init_:_include_repo_description_and__47__or_UUID_in_commit_message.mdwn
@@ -9,3 +9,5 @@ I'm not sure that the above suggestion is going down a path that really
 makes sense. If you want a list of repository UUIDs and descriptions,
 it's there in machine-usable form in `.git-annex/uuid.log`, there is no
 need to try to pull this info out of git commit messages. --[[Joey]]
+
+[[done]]

From b2c5639dcc53f6e734643878f9696405fa6cae64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 13:29:39 -0400
Subject: [PATCH 2156/8313] update

---
 doc/todo/smudge.mdwn | 46 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index c51662b282..f78b215ac4 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -19,6 +19,26 @@ add` files, and just being able to use `git add` or `git commit -a`,
 and have it use git-annex when .gitattributes says to. Also, annexed
 files can be directly modified without having to `git annex unlock`.
 
+### design
+
+In .gitattributes, the user would put something like "* filter=git-annex".
+This way they could control which files are annexed vs added normally.
+
+(git-annex could have further controls to allow eg, passing small files
+through to regular processing. At least .gitattributes is a special case,
+it should never be annexed...)
+
+For files not configured this way, git-annex could continue to use
+its symlink method -- this would preserve backwards compatability,
+and even allow mixing the two methods in a repo as desired.
+
+To find files in the repository that are annexed, git-annex would do
+`ls-files` as now, but would check if found files have the appropriate
+filter, rather than the current symlink checks. To determine the key
+of a file, rather than reading its symlink, git-annex would need to
+look up the git blob associated with the file -- this can be done
+efficiently using the existing code in `Branch.catFile`.
+
 ### efficiency
 
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
@@ -30,12 +50,16 @@ This avoids it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
 
+(The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.)
+
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
 
 SHA1 has a harder job. Would not want to re-sha1 the file every time,
-probably. So it'd need a cache of file stat info, mapped to known objects.
+probably. So it'd need a local cache of file stat info, mapped to known
+objects.
 
 ### dealing with partial content availability
 
@@ -59,9 +83,10 @@ huge-smudge:
 
 #!/bin/sh
 read sha1
+file="$1"
 echo "smudging $sha1" >&2
 if [ -e ~/$sha1 ]; then
-	cat ~/$sha1
+	cat ~/$sha1 # possibly expensive copy here
 else
 	echo "$sha1 not available"
 fi
@@ -71,16 +96,15 @@ huge-clean:
 
 
 #!/bin/sh
-cat >temp
-if grep -q 'not available' temp; then
-	awk '{print $1}' temp # provide what we would if the content were avail!
-	rm temp
+temp="$1"
+if grep -q 'not available' "$temp"; then
+	awk '{print $1}' "$temp" # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum temp | cut -d' ' -f1`
+sha1=`sha1sum "$temp" | cut -d' ' -f1`
 echo "cleaning $sha1" >&2
-ls -l temp >&2
-mv temp ~/$sha1
+ls -l "$temp" >&2
+ln -f "$temp" ~/$sha1 # can't delete temp file
 echo $sha1
 

 
@@ -94,6 +118,6 @@ in .git/config:
 
 
 [filter "huge"]
-        clean = huge-clean
-        smudge = huge-smudge
+        clean = huge-clean %f
+        smudge = huge-smudge %f
 

From d1154d0837f4dac18771a6a5c3ab96de4d5fcd90 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 14:13:38 -0400
Subject: [PATCH 2157/8313] init: Make description an optional parameter.

---
 Command/Init.hs    | 6 ++----
 debian/changelog   | 1 +
 doc/git-annex.mdwn | 4 +---
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 0191060511..2abe4c6615 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import Control.Monad (when)
+import Control.Monad
 
 import Command
 import qualified Annex
@@ -17,15 +17,13 @@ import Init
 	
 command :: [Command]
 command = [standaloneCommand "init" paramDesc seek
-		"initialize git-annex with repository description"]
+		"initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: CommandStartWords
 start ws = do
-	when (null description) $
-		error "please specify a description of this repository\n"
 	showStart "init" description
 	next $ perform description
 	where
diff --git a/debian/changelog b/debian/changelog
index 54fdbd835c..7803c9a62a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
   * The wget command will now be used in preference to curl, if available.
+  * init: Make description an optional parameter.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 52599611ed..e7ac9adf7a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -119,9 +119,7 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
-* init description
-
-  Initializes git-annex with a description of the git repository.
+* init [description]
 
   Until a repository (or one of its remotes) has been initialized,
   git-annex will refuse to operate on it, to avoid accidentially

From cd199e442f31743ee7ddc7c545f390533ed30a9d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 16:31:47 -0400
Subject: [PATCH 2158/8313] update; showstopper issue with current git

developed a patch for git, we'll see if they like it..
---
 doc/todo/smudge.mdwn | 49 +++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index f78b215ac4..2f5d21d7e1 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -41,18 +41,17 @@ efficiently using the existing code in `Branch.catFile`.
 
 ### efficiency
 
+#### clean
+
 The trick is doing it efficiently. Since git a2b665d, v1.7.4.1,
 something like this works to provide a filename to the clean script:
 
 	git config --global filter.huge.clean huge-clean %f
 
-This avoids it needing to read all the current file content from stdin
+This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
 
-(The smudge script can also be provided a filename with %f, but it
-cannot directly write to the file or git gets unhappy.)
-
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
 do nothing, file content is already cached.
@@ -61,6 +60,21 @@ SHA1 has a harder job. Would not want to re-sha1 the file every time,
 probably. So it'd need a local cache of file stat info, mapped to known
 objects.
 
+But: Even with %f, git actually passes the full file content to the clean
+filter, and if it fails to consume it all, it will crash (may only happen
+if the file is larger than some chunk size; tried with 500 mb file and 
+saw a SIGPIPE.) This means unnecessary works needs to be done, 
+and it slows down *everything*, from `git status` to `git commit`.
+**showstopper** I have sent a patch to the git mailing list to address
+this.
+
+#### smudge
+
+The smudge script can also be provided a filename with %f, but it
+cannot directly write to the file or git gets unhappy.
+
+
+
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -82,13 +96,13 @@ huge-smudge:
 
 
 #!/bin/sh
-read sha1
+read f
 file="$1"
-echo "smudging $sha1" >&2
-if [ -e ~/$sha1 ]; then
-	cat ~/$sha1 # possibly expensive copy here
+echo "smudging $f" >&2
+if [ -e ~/$f ]; then
+	cat ~/$f # possibly expensive copy here
 else
-	echo "$sha1 not available"
+	echo "$f not available"
 fi
 

 
@@ -96,16 +110,17 @@ huge-clean:
 
 
 #!/bin/sh
-temp="$1"
-if grep -q 'not available' "$temp"; then
-	awk '{print $1}' "$temp" # provide what we would if the content were avail!
+file="$1"
+# in real life, this should be done more efficiently, not trying to read
+# the whole file content!
+if grep -q 'not available' "$file"; then
+	awk '{print $1}' "$file" # provide what we would if the content were avail!
 	exit 0
 fi
-sha1=`sha1sum "$temp" | cut -d' ' -f1`
-echo "cleaning $sha1" >&2
-ls -l "$temp" >&2
-ln -f "$temp" ~/$sha1 # can't delete temp file
-echo $sha1
+echo "cleaning $file" >&2
+ls -l "$file" >&2
+ln -f "$file" ~/$file # can't delete temp file
+echo $file
 

 
 .gitattributes:

From 5ef11350aafe2aec48f66799743423a9c6e42556 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 29 Aug 2011 16:41:47 -0400
Subject: [PATCH 2159/8313] link to patch

---
 doc/todo/smudge.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 2f5d21d7e1..7d8f9a136d 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -66,7 +66,7 @@ if the file is larger than some chunk size; tried with 500 mb file and
 saw a SIGPIPE.) This means unnecessary works needs to be done, 
 and it slows down *everything*, from `git status` to `git commit`.
 **showstopper** I have sent a patch to the git mailing list to address
-this.
+this. 
 
 #### smudge
 

From 9e135a6ee2b6b7ac31bbf76d0d590a0415f9f512 Mon Sep 17 00:00:00 2001
From: "http://www.schleptet.net/~cfm/" 
Date: Tue, 30 Aug 2011 14:31:38 +0000
Subject: [PATCH 2160/8313] Added a comment

---
 ...ment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment

diff --git a/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
new file mode 100644
index 0000000000..1148a87cab
--- /dev/null
+++ b/doc/install/OSX/comment_1_0a1760bf0db1f1ba89bdb4c62032f631._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://www.schleptet.net/~cfm/"
+ ip="64.30.148.100"
+ subject="comment 1"
+ date="2011-08-30T14:31:36Z"
+ content="""
+You can also use Homebrew instead of MacPorts.  Homebrew's `haskell-platform` is up-to-date, too:
+
+    brew install haskell-platform git ossp-uuid md5sha1sum coreutils pcre
+    ln -s /usr/local/include/pcre.h /usr/include/pcre.h
+
+As of this writing, however, Homebrew's `md5sha1sum` has a broken mirror.  I wound up getting that from MacPorts anyway.
+"""]]

From a64c16bf7d6cf3ba5295a23496e10ed70c1656dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 13:23:21 -0400
Subject: [PATCH 2161/8313] tweak

---
 Backend.hs           | 7 +++----
 Utility/DataUnits.hs | 3 +--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 0bb9f4b575..75327de80b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -122,8 +122,7 @@ lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
-maybeLookupBackendName s =
-	if 1 /= length matches
-		then Nothing
-		else Just $ head matches
+maybeLookupBackendName s
+	| length matches == 1 = Just $ head matches
+	| otherwise = Nothing
 	where matches = filter (\b -> s == B.name b) list
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index f2bc333ea0..0baa5dd896 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -137,8 +137,7 @@ compareSizes units abbrev old new
 {- Parses strings like "10 kilobytes" or "0.5tb". -}
 readSize :: [Unit] -> String -> Maybe ByteSize
 readSize units input
-	| null parsednum = Nothing
-	| null parsedunit = Nothing
+	| null parsednum || null parsedunit = Nothing
 	| otherwise = Just $ round $ number * fromIntegral multiplier
 	where
 		(number, rest) = head parsednum

From b96443364e80720172638f0fcd9caa5b9948555d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 13:29:07 -0400
Subject: [PATCH 2162/8313] smudge update: Not practical.

---
 doc/todo/smudge.mdwn | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 7d8f9a136d..55de5f5789 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -1,5 +1,18 @@
 git-annex should use smudge/clean filters.
 
+----
+
+Update: Currently, this does not look likely to work. In particular,
+the clean filter needs to consume all stdin from git, which consists of the
+entire content of the file. It cannot optimise by directly accessing
+the file in the repository, because git may be cleaning a different
+version of the file during a merge. 
+
+So every `git status` would need to read the entire content of all
+available files, and checksum them, which is too expensive.
+
+----
+
 The clean filter is run when files are staged for commit. So a user could copy
 any file into the annex, git add it, and git-annex's clean filter causes
 the file's key to be staged, while its value is added to the annex.
@@ -8,7 +21,7 @@ The smudge filter is run when files are checked out. Since git annex
 repos have partial content, this would not git annex get the file content.
 Instead, if the content is not currently available, it would need to do
 something like return empty file content. (Sadly, it cannot create a
-symlink, as git still wants to write the file afterwards.
+symlink, as git still wants to write the file afterwards.)
 
 So the nice current behavior of unavailable files being clearly missing due
 to dangling symlinks, would be lost when using smudge/clean filters.
@@ -39,7 +52,13 @@ of a file, rather than reading its symlink, git-annex would need to
 look up the git blob associated with the file -- this can be done
 efficiently using the existing code in `Branch.catFile`.
 
-### efficiency
+The clean filter would inject the file's content into the annex, and hard
+link from the annex to the file. Avoiding duplication of data.
+
+The smudge filter can't do that, so to avoid duplication of data, it
+might always create an empty file. To get the content, `git annex get`
+could be used (which would hard link it). A `post-checkout` hook might
+be used to set up hard links for all currently available content.
 
 #### clean
 
@@ -51,6 +70,8 @@ something like this works to provide a filename to the clean script:
 This could avoid it needing to read all the current file content from stdin
 when doing eg, a git status or git commit. Instead it is passed the
 filename that git is operating on, in the working directory.
+(Update: No, doesn't work; git may be cleaning a different file content
+than is currently on disk, and git requires all stdin be consumed too.)
 
 So, WORM could just look at that file and easily tell if it is one
 it already knows (same mtime and size). If so, it can short-circuit and
@@ -66,15 +87,14 @@ if the file is larger than some chunk size; tried with 500 mb file and
 saw a SIGPIPE.) This means unnecessary works needs to be done, 
 and it slows down *everything*, from `git status` to `git commit`.
 **showstopper** I have sent a patch to the git mailing list to address
-this. 
+this.  (Update: apparently
+can't be fixed.)
 
 #### smudge
 
 The smudge script can also be provided a filename with %f, but it
 cannot directly write to the file or git gets unhappy.
 
-
-
 ### dealing with partial content availability
 
 The smudge filter cannot be allowed to fail, that leaves the tree and
@@ -111,15 +131,15 @@ huge-clean:
 
 #!/bin/sh
 file="$1"
+cat >/tmp/file
 # in real life, this should be done more efficiently, not trying to read
 # the whole file content!
-if grep -q 'not available' "$file"; then
-	awk '{print $1}' "$file" # provide what we would if the content were avail!
+if grep -q 'not available' /tmp/file; then
+	awk '{print $1}' /tmp/file # provide what we would if the content were avail!
 	exit 0
 fi
 echo "cleaning $file" >&2
-ls -l "$file" >&2
-ln -f "$file" ~/$file # can't delete temp file
+# XXX store file content here
 echo $file
 

 

From ea7b1828d48a5dce42393d252f4bbda5cba10d24 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 30 Aug 2011 15:16:34 -0400
Subject: [PATCH 2163/8313] unused, status: Sped up by avoiding unnecessary
 stats of annexed files.

Statting files returned by dirContents to see if they exist and are regular
files seems pretty useless. This code was originally part of fsck, and
perhaps the idea then was to avoid things returned by dirContents that were
not files. But it's certianly not needed in the current use cases for
getKeysPresent.
---
 Content.hs       | 9 +--------
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/Content.hs b/Content.hs
index ba99f1330d..1c2475240b 100644
--- a/Content.hs
+++ b/Content.hs
@@ -254,15 +254,8 @@ getKeysPresent' dir = do
 			levela <- dirContents dir
 			levelb <- mapM dirContents levela
 			contents <- mapM dirContents (concat levelb)
-			files <- filterM present (concat contents)
+			let files = concat contents
 			return $ mapMaybe (fileKey . takeFileName) files
-	where
-		present d = do
-			result <- try $
-				getFileStatus $ d  takeFileName d
-			case result of
-				Right s -> return $ isRegularFile s
-				Left _ -> return False
 
 {- Things to do to record changes to content. -}
 saveState :: Annex ()
diff --git a/debian/changelog b/debian/changelog
index 7803c9a62a..c8c60579e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
     configured first. Closes: #638998
   * The wget command will now be used in preference to curl, if available.
   * init: Make description an optional parameter.
+  * unused, status: Sped up by avoiding unnecessary stats of annexed files.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 

From baa43cb9c0b38135e136a8ff807c4caeb909b538 Mon Sep 17 00:00:00 2001
From: "http://jefferai.org/" 
Date: Tue, 30 Aug 2011 20:56:44 +0000
Subject: [PATCH 2164/8313]

---
 doc/bugs/add_script-friendly_output_options.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/add_script-friendly_output_options.mdwn

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
new file mode 100644
index 0000000000..efcd54553f
--- /dev/null
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -0,0 +1,3 @@
+I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
+
+This could perhaps be triggered by a --json or --xml flag to the relevant commands.

From f600444ab6d00826cc16c4a5cc67aca84cc1f9e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 31 Aug 2011 19:13:02 -0400
Subject: [PATCH 2165/8313] unused --remote: Reduced memory use to 1/4th what
 was used before.

Using a single strictness annotation, in just the right place.
Tried several others, none of which helped and some of which potentially
hurt. This is only the second time I've really had to deal with this in
a year of using haskell, which is, I suppose not that bad.
---
 Command/Unused.hs                          | 5 ++++-
 debian/changelog                           | 1 +
 doc/todo/git-annex_unused_eats_memory.mdwn | 6 ------
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index e7065b3c36..27f5af1b45 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Unused where
 
 import Control.Monad (filterM, unless, forM_)
@@ -80,7 +82,8 @@ checkRemoteUnused' r = do
 	where
 		isthere k = do
 			us <- keyLocations k
-			return $ uuid `elem` us
+			let !there = uuid `elem` us
+			return there
 		uuid = Remote.uuid r
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
diff --git a/debian/changelog b/debian/changelog
index c8c60579e2..4ee0b80f2e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * The wget command will now be used in preference to curl, if available.
   * init: Make description an optional parameter.
   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+  * unused --remote: Reduced memory use to 1/4th what was used before.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 6ce7140045..fcb09a1af7 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -17,9 +17,3 @@ currently present in the repository (possibly using a bloom filter again),
 and that would yield a shortlist of keys that are probably not used. 
 Then scan thru all files in the repo to make sure that none point to keys
 on the shortlist.
-
-----
-
-`git annex unused --from remote` is much worse, using hundreds of mb of
-memory. It has not been profiled at all yet, and can probably be improved
-somewhat by fixing whatever memory leak it (probably) has.

From 55783d886d3300555d4b68ff7323e2365928e059 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 31 Aug 2011 19:50:08 -0400
Subject: [PATCH 2166/8313] add explanation for why strictness is needed here

---
 Command/Unused.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 27f5af1b45..6a62cde5f8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -80,6 +80,8 @@ checkRemoteUnused' r = do
 		showLongNote $ remoteUnusedMsg r list
 		showLongNote "\n"
 	where
+		{- This should run strictly to avoid the filterM
+		 - building many thunks containing keyLocations data. -}
 		isthere k = do
 			us <- keyLocations k
 			let !there = uuid `elem` us

From 57dd34c6be5dbc01286108fd943ff9e02956e8aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 13:35:07 -0400
Subject: [PATCH 2167/8313] generalize quiet flag to output type

This will allow adding other styles of output.
---
 Annex.hs        | 7 +++++--
 Command/Init.hs | 2 --
 Content.hs      | 3 +--
 Messages.hs     | 7 ++++---
 Options.hs      | 6 +++---
 5 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 287aed875e..fac5d27e49 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -10,6 +10,7 @@
 module Annex (
 	Annex,
 	AnnexState(..),
+	OutputType(..),
 	new,
 	run,
 	eval,
@@ -48,7 +49,7 @@ data AnnexState = AnnexState
 	, backends :: [Backend Annex]
 	, remotes :: [Remote Annex]
 	, repoqueue :: Queue
-	, quiet :: Bool
+	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
 	, branchstate :: BranchState
@@ -63,13 +64,15 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
+data OutputType = NormalOutput | QuietOutput
+
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
 	, repoqueue = empty
-	, quiet = False
+	, output = NormalOutput
 	, force = False
 	, fast = False
 	, branchstate = startBranchState
diff --git a/Command/Init.hs b/Command/Init.hs
index 2abe4c6615..6ba7df6829 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,8 +7,6 @@
 
 module Command.Init where
 
-import Control.Monad
-
 import Command
 import qualified Annex
 import UUID
diff --git a/Content.hs b/Content.hs
index 1c2475240b..e4bbee5282 100644
--- a/Content.hs
+++ b/Content.hs
@@ -23,11 +23,10 @@ module Content (
 	saveState
 ) where
 
-import System.IO.Error (try)
 import System.Directory
 import Control.Monad.State (liftIO)
 import System.Path
-import Control.Monad (when, filterM)
+import Control.Monad
 import System.Posix.Files
 import System.FilePath
 import Data.Maybe
diff --git a/Messages.hs b/Messages.hs
index 36f0b89c5c..b2c871ede1 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -9,7 +9,6 @@ module Messages where
 
 import Control.Monad.State (liftIO)
 import System.IO
-import Control.Monad (unless)
 import Data.String.Utils
 
 import Types
@@ -17,8 +16,10 @@ import qualified Annex
 
 verbose :: Annex () -> Annex ()
 verbose a = do
-	q <- Annex.getState Annex.quiet
-	unless q a
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> a
+		_ -> return ()
 
 showStart :: String -> String -> Annex ()
 showStart command file = verbose $ liftIO $ do
diff --git a/Options.hs b/Options.hs
index 7f78f44f62..768a1c289d 100644
--- a/Options.hs
+++ b/Options.hs
@@ -26,9 +26,9 @@ commonOptions =
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
 		"avoid slow operations"
-	, Option ['q'] ["quiet"] (NoArg (setquiet True))
+	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
 		"avoid verbose output"
-	, Option ['v'] ["verbose"] (NoArg (setquiet False))
+	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
@@ -38,7 +38,7 @@ commonOptions =
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
-		setquiet v = Annex.changeState $ \s -> s { Annex.quiet = v }
+		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG

From 2f4d4d1c4552a93a5f26a8a0a713e3916612329e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 15:16:31 -0400
Subject: [PATCH 2168/8313] basic json support

This includes a generic JSONStream library built on top of Text.JSON
(somewhat hackishly).

It would be possible to stream out a single json document describing
all actions, but it's probably better for consumers if they can expect
one json document per line, so I did it that way instead.

Output from external programs used for transferring files is not
currently hidden when outputting json, which probably makes it not very
useful there. This may be dealt with if there is demand for json
output for --get or --move to be parsable.

The version, status, and find subcommands have hand-crafted output and
don't do json. The whereis subcommand needs to be modified to produce
useful json.
---
 Annex.hs              |  2 +-
 Messages.hs           | 61 +++++++++++++++++++++++++++++--------------
 Messages/JSON.hs      | 23 ++++++++++++++++
 Options.hs            |  2 ++
 Utility/JSONStream.hs | 44 +++++++++++++++++++++++++++++++
 debian/changelog      |  1 +
 debian/control        |  1 +
 doc/git-annex.mdwn    | 10 +++++--
 doc/install.mdwn      |  1 +
 9 files changed, 123 insertions(+), 22 deletions(-)
 create mode 100644 Messages/JSON.hs
 create mode 100644 Utility/JSONStream.hs

diff --git a/Annex.hs b/Annex.hs
index fac5d27e49..f5c3e4de45 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -64,7 +64,7 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-data OutputType = NormalOutput | QuietOutput
+data OutputType = NormalOutput | QuietOutput | JSONOutput
 
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
diff --git a/Messages.hs b/Messages.hs
index b2c871ede1..87d414f172 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -5,7 +5,22 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Messages where
+module Messages (
+	showStart,
+	showNote,
+	showAction,
+	showProgress,
+	showSideAction,
+	showOutput,
+	showLongNote,
+	showEndOk,
+	showEndFail,
+	showEndResult,
+	showErr,
+	warning,
+	indent,
+	setupConsole
+) where
 
 import Control.Monad.State (liftIO)
 import System.IO
@@ -13,21 +28,15 @@ import Data.String.Utils
 
 import Types
 import qualified Annex
-
-verbose :: Annex () -> Annex ()
-verbose a = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> a
-		_ -> return ()
+import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = verbose $ liftIO $ do
+showStart command file = handle (JSON.start command file) $ do
 	putStr $ command ++ " " ++ file ++ " "
 	hFlush stdout
 
 showNote :: String -> Annex ()
-showNote s = verbose $ liftIO $ do
+showNote s = handle (JSON.note s) $ do
 	putStr $ "(" ++ s ++ ") "
 	hFlush stdout
 
@@ -35,28 +44,31 @@ showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = verbose $ liftIO $ do
+showProgress = handle q $ do
 	putStr "."
 	hFlush stdout
 
 showSideAction :: String -> Annex ()
-showSideAction s = verbose $ liftIO $ putStrLn $ "(" ++ s ++ "...)"
+showSideAction s = handle q $ putStrLn $ "(" ++ s ++ "...)"
 
 showOutput :: Annex ()
-showOutput = verbose $ liftIO $ putStr "\n"
+showOutput = handle q $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = verbose $ liftIO $ putStr $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $ putStr $ '\n' : indent s
 
 showEndOk :: Annex ()
-showEndOk = verbose $ liftIO $ putStrLn "ok"
+showEndOk = showEndResult True
 
 showEndFail :: Annex ()
-showEndFail = verbose $ liftIO $ putStrLn "failed"
+showEndFail = showEndResult False
 
 showEndResult :: Bool -> Annex ()
-showEndResult True = showEndOk
-showEndResult False = showEndFail
+showEndResult b = handle (JSON.end b) $ putStrLn msg
+	where
+		msg
+			| b = "ok"
+			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
 showErr e = liftIO $ do
@@ -65,7 +77,7 @@ showErr e = liftIO $ do
 
 warning :: String -> Annex ()
 warning w = do
-	verbose $ liftIO $ putStr "\n"
+	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
 		hPutStrLn stderr $ indent w
@@ -85,3 +97,14 @@ setupConsole :: IO ()
 setupConsole = do
 	hSetBinaryMode stdout True
 	hSetBinaryMode stderr True
+
+handle :: IO () -> IO () -> Annex ()
+handle json normal = do
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> liftIO normal
+		Annex.QuietOutput -> q
+		Annex.JSONOutput -> liftIO json
+
+q :: Monad m => m ()
+q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
new file mode 100644
index 0000000000..ee6ea34a32
--- /dev/null
+++ b/Messages/JSON.hs
@@ -0,0 +1,23 @@
+{- git-annex JSON output
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Messages.JSON (
+	start,
+	end,
+	note
+) where
+
+import qualified Utility.JSONStream as Stream
+
+start :: String -> String -> IO ()
+start command file = putStr $ Stream.start [("command", command), ("file", file)]
+
+end :: Bool -> IO ()
+end b = putStr $ Stream.add [("success", b)] ++ Stream.end
+
+note :: String -> IO ()
+note s = putStr $ Stream.add [("note", s)]
diff --git a/Options.hs b/Options.hs
index 768a1c289d..e0ca48c01b 100644
--- a/Options.hs
+++ b/Options.hs
@@ -30,6 +30,8 @@ commonOptions =
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
 		"allow verbose output (default)"
+	, Option ['j'] ["json"] (NoArg (setoutput Annex.JSONOutput))
+		"enable JSON output"
 	, Option ['d'] ["debug"] (NoArg (setdebug))
 		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs
new file mode 100644
index 0000000000..af3766948f
--- /dev/null
+++ b/Utility/JSONStream.hs
@@ -0,0 +1,44 @@
+{- Streaming JSON output.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.JSONStream (
+	start,
+	add,
+	end
+) where
+
+import Text.JSON
+
+{- Text.JSON does not support building up a larger JSON document piece by
+   piece as a stream. To support streaming, a hack. The JSObject is converted
+   to a string with its final "}" is left off, allowing it to be added to
+   later. -}
+start :: JSON a => [(String, a)] -> String
+start l
+	| last s == endchar = take (length s - 1)  s
+	| otherwise = bad s
+	where
+		s = encodeStrict $ toJSObject l
+
+add :: JSON a => [(String, a)] -> String
+add l
+	| head s == startchar = ',' : drop 1 s
+	| otherwise = bad s
+	where
+		s = start l
+
+end :: String
+end = [endchar, '\n']
+
+startchar :: Char
+startchar = '{'
+
+endchar :: Char
+endchar = '}'
+
+bad :: String -> a
+bad s = error $ "Text.JSON returned unexpected string: " ++ s
diff --git a/debian/changelog b/debian/changelog
index 4ee0b80f2e..ca23ab4735 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * init: Make description an optional parameter.
   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
   * unused --remote: Reduced memory use to 1/4th what was used before.
+  * Add --json switch, to produce machine-consumable output.
 
  -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
 
diff --git a/debian/control b/debian/control
index 63488dc68d..cb5a8212ab 100644
--- a/debian/control
+++ b/debian/control
@@ -14,6 +14,7 @@ Build-Depends:
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
 	libghc-monad-control-dev,
+	libghc-json-dev,
 	ikiwiki,
 	perlmagick,
 	git | git-core,
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e7ac9adf7a..0a484a3842 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -337,12 +337,18 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * --quiet
 
-  Avoid the default verbose logging of what is done; only show errors
+  Avoid the default verbose display of what is done; only show errors
   and progress displays.
 
 * --verbose
 
-  Enable verbose logging.
+  Enable verbose display.
+
+* --json
+
+  Rather than the normal output, generate JSON. This is intended to be
+  parsed by programs that use git-annex. Each line of output is a JSON
+  object.
 
 * --debug
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index ac521da187..cd51b96d23 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -28,6 +28,7 @@ To build and use git-annex, you will need:
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+  * [json](http://hackage.haskell.org/package/json)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)

From 5bc32c7f3438a329878f4da7ad0b514c12a54332 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:02:01 -0400
Subject: [PATCH 2169/8313] add json formatted list of remotes

Wherever a list of remotes is shown, --json now enables a json formatted
list.
---
 Command/Fsck.hs    |  2 +-
 Command/Whereis.hs |  2 +-
 Remote.hs          | 35 +++++++++++++++++++++++++----------
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 6ccec05fb3..bad60d30dd 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -131,7 +131,7 @@ checkKeyNumCopies key file numcopies = do
 	let present = length safelocations
 	if present < needed
 		then do
-			ppuuids <- Remote.prettyPrintUUIDs untrustedlocations
+			ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations
 			warning $ missingNote (filename file key) present needed ppuuids
 			return False
 		else return True
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 314fef7826..f80c823b70 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -33,7 +33,7 @@ perform key = do
 	if null uuids
 		then stop
 		else do
-			pp <- prettyPrintUUIDs uuids
+			pp <- prettyPrintUUIDs "whereis" uuids
 			showLongNote pp
 			showOutput
 			next $ return True
diff --git a/Remote.hs b/Remote.hs
index 2c883f1a8f..ad1768da64 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -30,11 +30,14 @@ module Remote (
 ) where
 
 import Control.Monad (filterM)
+import Control.Monad.State (liftIO)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
 import Data.Maybe
 import Control.Applicative
+import Text.JSON
+import Text.JSON.Generic
 
 import Types
 import Types.Remote
@@ -44,6 +47,7 @@ import Config
 import Trust
 import LocationLog
 import Messages
+import qualified Utility.JSONStream
 import RemoteLog
 
 import qualified Remote.Git
@@ -119,23 +123,34 @@ nameToUUID n = do
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
-{- Pretty-prints a list of UUIDs of remotes. -}
-prettyPrintUUIDs :: [UUID] -> Annex String
-prettyPrintUUIDs uuids = do
+{- Pretty-prints a list of UUIDs of remotes, for human display.
+ -
+ - Shows descriptions from the uuid log, falling back to remote names,
+ - as some remotes may not be in the uuid log.
+ -
+ - When JSON is enabled, also generates a machine-readable description
+ - of the UUIDs. -}
+prettyPrintUUIDs :: String -> [UUID] -> Annex String
+prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
-	-- Show descriptions from the uuid log, falling back to remote names,
-	-- as some remotes may not be in the uuid log
 	m <- M.union <$> uuidMap <*> availMap
-	return $ unwords $ map (\u -> "\t" ++ prettify m u here ++ "\n") uuids
+	liftIO . putStr $ Utility.JSONStream.add
+		[(desc, map (jsonify m here) uuids)]
+	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
 		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
-		prettify m u here = base ++ ishere
+		findlog m u = M.findWithDefault "" u m
+		prettify m here u = base ++ ishere
 			where
 				base = if not $ null $ findlog m u
 					then u ++ "  -- " ++ findlog m u
 					else u
 				ishere = if here == u then " <-- here" else ""
-		findlog m u = M.findWithDefault "" u m
+		jsonify m here u = toJSObject
+			[ ("uuid", toJSON u)
+			, ("description", toJSON $ findlog m u)
+			, ("here", toJSON $ here == u)
+			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
@@ -186,8 +201,8 @@ showLocations key exclude = do
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
 	let uuidsskipped = filteruuids uuids (u:exclude++uuidswanted)
-	ppuuidswanted <- Remote.prettyPrintUUIDs uuidswanted
-	ppuuidsskipped <- Remote.prettyPrintUUIDs uuidsskipped
+	ppuuidswanted <- Remote.prettyPrintUUIDs "wanted" uuidswanted
+	ppuuidsskipped <- Remote.prettyPrintUUIDs "skipped" uuidsskipped
 	showLongNote $ message ppuuidswanted ppuuidsskipped
 	where
 		filteruuids l x = filter (`notElem` x) l

From e4a74c0dc533922ce1e7cd56dfa7ac08efc064cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:11:21 -0400
Subject: [PATCH 2170/8313] close bug with some caveats

---
 doc/bugs/add_script-friendly_output_options.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
index efcd54553f..08f7e6c7b8 100644
--- a/doc/bugs/add_script-friendly_output_options.mdwn
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -1,3 +1,18 @@
 I have a need to use git-annex from a larger program. It'd be great if the information output by some of the commands that is descriptive (for example, whereis) could be sent to stdout in a machine-readable format like (preferably) JSON, or XML. That way I can simply read in the output of the command and use the data directly instead of having to parse it via regexes or other such string manipulation.
 
 This could perhaps be triggered by a --json or --xml flag to the relevant commands.
+
+> This is [[done]], --json is supported by all commands, more or less.
+> 
+> Caveats: 
+> 
+>  * the version, status, and find commands produce custom output and so
+>    no json. This could change for version and status; find needs to just
+>    be a simple list of files, I think 
+>  * The "note" fields may repeat multiple times per object with different
+>    notes and are of course not machine readable, and subject to change.
+>  * Output of helper commands like rsync is not diverted away, and
+>    could clutter up the json output badly. Should only affect commands
+>    that transfer data.
+> 
+> --[[Joey]] 

From cb5dacfd403a5ca4e3bdeb3eb3d0009136966f91 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Sep 2011 16:15:57 -0400
Subject: [PATCH 2171/8313] rsync and wget use stderr for progress, so no
 problem

---
 doc/bugs/add_script-friendly_output_options.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/add_script-friendly_output_options.mdwn b/doc/bugs/add_script-friendly_output_options.mdwn
index 08f7e6c7b8..7d7bdfc51a 100644
--- a/doc/bugs/add_script-friendly_output_options.mdwn
+++ b/doc/bugs/add_script-friendly_output_options.mdwn
@@ -13,6 +13,7 @@ This could perhaps be triggered by a --json or --xml flag to the relevant comman
 >    notes and are of course not machine readable, and subject to change.
 >  * Output of helper commands like rsync is not diverted away, and
 >    could clutter up the json output badly. Should only affect commands
->    that transfer data.
+>    that transfer data. And AFAICS, wget and rsync both output their
+>    progress displays to stderr, so shouldn't be a problem.
 > 
 > --[[Joey]] 

From d1dd40c49bb95416adec5b33e4dc1e7672a2f7f7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 16:44:04 -0400
Subject: [PATCH 2172/8313] avoid showing json lists of remotes when not in
 json mode

---
 Messages.hs      |  6 ++++++
 Messages/JSON.hs | 10 ++++++++--
 Remote.hs        |  5 +----
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/Messages.hs b/Messages.hs
index 87d414f172..faa4dbcdec 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -19,12 +19,14 @@ module Messages (
 	showErr,
 	warning,
 	indent,
+	maybeShowJSON,
 	setupConsole
 ) where
 
 import Control.Monad.State (liftIO)
 import System.IO
 import Data.String.Utils
+import Text.JSON
 
 import Types
 import qualified Annex
@@ -106,5 +108,9 @@ handle json normal = do
 		Annex.QuietOutput -> q
 		Annex.JSONOutput -> liftIO json
 
+{- Shows a JSON value only when in json mode. -}
+maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
+maybeShowJSON v = handle (JSON.add v) q
+
 q :: Monad m => m ()
 q = return ()
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index ee6ea34a32..fb95f550e8 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -8,9 +8,12 @@
 module Messages.JSON (
 	start,
 	end,
-	note
+	note,
+	add
 ) where
 
+import Text.JSON
+
 import qualified Utility.JSONStream as Stream
 
 start :: String -> String -> IO ()
@@ -20,4 +23,7 @@ end :: Bool -> IO ()
 end b = putStr $ Stream.add [("success", b)] ++ Stream.end
 
 note :: String -> IO ()
-note s = putStr $ Stream.add [("note", s)]
+note s = add [("note", s)]
+
+add :: JSON a => [(String, a)] -> IO ()
+add v = putStr $ Stream.add v
diff --git a/Remote.hs b/Remote.hs
index ad1768da64..e54d2e2334 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -30,7 +30,6 @@ module Remote (
 ) where
 
 import Control.Monad (filterM)
-import Control.Monad.State (liftIO)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
@@ -47,7 +46,6 @@ import Config
 import Trust
 import LocationLog
 import Messages
-import qualified Utility.JSONStream
 import RemoteLog
 
 import qualified Remote.Git
@@ -134,8 +132,7 @@ prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
 	m <- M.union <$> uuidMap <*> availMap
-	liftIO . putStr $ Utility.JSONStream.add
-		[(desc, map (jsonify m here) uuids)]
+	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
 		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList

From d238bbd9d9f5e96334b9214c7aeca7875054e99e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 21:32:05 -0400
Subject: [PATCH 2173/8313] releasing version 3.20110902

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ca23ab4735..35144f678d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110820) UNRELEASED; urgency=low
+git-annex (3.20110902) unstable; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
     configured first. Closes: #638998
@@ -8,7 +8,7 @@ git-annex (3.20110820) UNRELEASED; urgency=low
   * unused --remote: Reduced memory use to 1/4th what was used before.
   * Add --json switch, to produce machine-consumable output.
 
- -- Joey Hess   Tue, 23 Aug 2011 13:41:01 -0400
+ -- Joey Hess   Fri, 02 Sep 2011 21:20:37 -0400
 
 git-annex (3.20110819) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index a37c187c90..7850f88a77 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110819
+Version: 3.20110902
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From ea621d0c2d418c6dfe247b0aa3f78976c002b2b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Sep 2011 21:32:21 -0400
Subject: [PATCH 2174/8313] add news item for git-annex 3.20110902

---
 doc/news/version_3.20110705.mdwn | 9 ---------
 doc/news/version_3.20110902.mdwn | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110705.mdwn
 create mode 100644 doc/news/version_3.20110902.mdwn

diff --git a/doc/news/version_3.20110705.mdwn b/doc/news/version_3.20110705.mdwn
deleted file mode 100644
index bb4665c047..0000000000
--- a/doc/news/version_3.20110705.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110705 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * uninit: Delete the git-annex branch and .git/annex/
-   * unannex: In --fast mode, file content is left in the annex, and a
-     hard link made to it.
-   * uninit: Use unannex in --fast mode, to support unannexing multiple
-     files that link to the same content.
-   * Drop the dependency on the haskell curl bindings, use regular haskell HTTP.
-   * Fix a pipeline stall when upgrading (caused by #624389)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110902.mdwn b/doc/news/version_3.20110902.mdwn
new file mode 100644
index 0000000000..e354874ea3
--- /dev/null
+++ b/doc/news/version_3.20110902.mdwn
@@ -0,0 +1,9 @@
+git-annex 3.20110902 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Set EMAIL when running test suite so that git does not need to be
+     configured first. Closes: #[638998](http://bugs.debian.org/638998)
+   * The wget command will now be used in preference to curl, if available.
+   * init: Make description an optional parameter.
+   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
+   * unused --remote: Reduced memory use to 1/4th what was used before.
+   * Add --json switch, to produce machine-consumable output."""]]
\ No newline at end of file

From c5c525d930480ccbf230420efb3b07cfd3ea78d7 Mon Sep 17 00:00:00 2001
From: DavidEdmondson 
Date: Mon, 5 Sep 2011 15:43:26 +0000
Subject: [PATCH 2175/8313] Added a comment: Is it necessary to commit after
 the 'drop'?

---
 .../comment_1_cb65e7c510b75be1c51f655b058667c6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment

diff --git a/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
new file mode 100644
index 0000000000..1c8719cecd
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_1_cb65e7c510b75be1c51f655b058667c6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="DavidEdmondson"
+ subject="Is it necessary to commit after the 'drop'?"
+ date="2011-09-05T15:43:25Z"
+ content="""
+In fact is it possible? Nothing changed as far as git is concerned.
+
+"""]]

From f0777d9b5a90326c495d557f79c8420e0c7d284a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 5 Sep 2011 15:59:27 +0000
Subject: [PATCH 2176/8313] Added a comment

---
 .../comment_2_64709ea4558915edd5c8ca4486965b07._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment

diff --git a/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
new file mode 100644
index 0000000000..f5fb8dc7f5
--- /dev/null
+++ b/doc/walkthrough/removing_files/comment_2_64709ea4558915edd5c8ca4486965b07._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-05T15:59:27Z"
+ content="""
+Good catch. It used to be necessary before there was a git-annex branch, but not now.
+"""]]

From dbef6a045c3f80510af3dd5ce80d94371a21a229 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Sep 2011 11:56:30 -0400
Subject: [PATCH 2177/8313] remove now-unnecessary commit after drop

---
 doc/walkthrough/removing_files.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/walkthrough/removing_files.mdwn b/doc/walkthrough/removing_files.mdwn
index 85a7d50a6b..0df6e82a1f 100644
--- a/doc/walkthrough/removing_files.mdwn
+++ b/doc/walkthrough/removing_files.mdwn
@@ -3,4 +3,3 @@ has the file before removing it.
 
 	# git annex drop iso/debian.iso
 	drop iso/Debian_5.0.iso ok
-	# git commit -a -m "freed up space"

From 14f75ced75aa73d4f8dbc45cdc697bac1535fb37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Sep 2011 11:57:58 -0400
Subject: [PATCH 2178/8313] remove now unnecessary commit after get

---
 doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
index 936d088f1f..cfb70aaf9a 100644
--- a/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
+++ b/doc/walkthrough/transferring_files:_When_things_go_wrong.mdwn
@@ -15,4 +15,3 @@ it:
 	# sudo mount /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
 	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
-	# git commit -a -m "got a video I want to rewatch on the plane"

From 07125dca53ece4896a83fad00b7f43101855ce9c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 13:46:08 -0400
Subject: [PATCH 2179/8313] Improve display of newlines around error and
 warning messages.

---
 CmdLine.hs                |  2 +-
 Command.hs                |  1 -
 Messages.hs               | 11 ++++++-----
 debian/changelog          |  6 ++++++
 doc/bugs/fsck_output.mdwn | 10 ++++++++++
 5 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 2652e5b8fc..4ccd2c2c2f 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -89,8 +89,8 @@ tryRun' errnum state (a:as) = do
 	case result of
 		Left err -> do
 			Annex.eval state $ do
-				showEndFail
 				showErr err
+				showEndFail
 			tryRun' (errnum + 1) state as
 		Right (True,state') -> tryRun' errnum state' as
 		Right (False,state') -> tryRun' (errnum + 1) state' as
diff --git a/Command.hs b/Command.hs
index 21c50f9c02..78f9823fb3 100644
--- a/Command.hs
+++ b/Command.hs
@@ -103,7 +103,6 @@ doCommand = start
 		stage a b = b >>= a
 		success = return True
 		failure = do
-			showOutput -- avoid clutter around error message
 			showEndFail
 			return False
 
diff --git a/Messages.hs b/Messages.hs
index faa4dbcdec..4922519819 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -73,16 +73,17 @@ showEndResult b = handle (JSON.end b) $ putStrLn msg
 			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
-showErr e = liftIO $ do
-	hFlush stdout
-	hPutStrLn stderr $ "git-annex: " ++ show e
+showErr e = warning' $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning w = do
+warning w = warning' (indent w)
+
+warning' :: String -> Annex ()
+warning' w = do
 	handle q $ putStr "\n"
 	liftIO $ do
 		hFlush stdout
-		hPutStrLn stderr $ indent w
+		hPutStrLn stderr w
 
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
diff --git a/debian/changelog b/debian/changelog
index 35144f678d..9ab3b4fd3d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110903) UNRELEASED; urgency=low
+
+  * Improve display of newlines around error and warning messages.
+
+ -- Joey Hess   Tue, 06 Sep 2011 13:45:16 -0400
+
 git-annex (3.20110902) unstable; urgency=low
 
   * Set EMAIL when running test suite so that git does not need to be
diff --git a/doc/bugs/fsck_output.mdwn b/doc/bugs/fsck_output.mdwn
index 90af1600d8..1b00dd7b37 100644
--- a/doc/bugs/fsck_output.mdwn
+++ b/doc/bugs/fsck_output.mdwn
@@ -34,3 +34,13 @@ The newline is in the wrong place and confuses the user. It should be printed _a
 > failed
 > 
 > --[[Joey]] 
+
+>> Well, I fixed this in all cases except a thrown non-IO error (last
+>> example aboce), which output is printed by haskell's runtime. I'd
+>> have to add a second error handler to handle those, and it's not
+>> clear what it would do. Often an error will occur before anything
+>> else is printed, and then the current behavior is right; if something
+>> has been printed it would be nice to have a newline before the error,
+>> but by the time the error is caught we'd be out of the annex monad
+>> and not really have any way to know if something has been printed.
+>> I think my fix is good enough [[done]] --[[Joey]] 

From fcfd2776cb010f35557a6e617d30327f4274e0ff Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl0-EtQjVUNysjom6sTlQxRUwkwD6uPx88"
 
Date: Tue, 6 Sep 2011 18:27:08 +0000
Subject: [PATCH 2180/8313]

---
 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn

diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
new file mode 100644
index 0000000000..61c87b56a0
--- /dev/null
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -0,0 +1,9 @@
+While following the instructions given at the OSX build page , I get this error:
+
+$ make
+ghc -O2 -Wall -ignore-package monads-fd -fspec-constr-count=5 --make git-annex
+
+Utility/JSONStream.hs:14:8:
+    Could not find module `Text.JSON':
+      Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1

From ca4eb842a4c7aa491c266e79ee1e1b340615ba01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:36:21 -0400
Subject: [PATCH 2181/8313] add json to build scripts

---
 doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn | 2 ++
 doc/install/Fedora.mdwn                   | 1 +
 doc/install/OSX.mdwn                      | 1 +
 3 files changed, 4 insertions(+)

diff --git a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
index 61c87b56a0..43fb0323c4 100644
--- a/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
+++ b/doc/bugs/Build_error_on_Mac_OSX_10.6.mdwn
@@ -7,3 +7,5 @@ Utility/JSONStream.hs:14:8:
     Could not find module `Text.JSON':
       Use -v to see a list of the files searched for.
 make: *** [git-annex] Error 1
+
+> Updated the instructions. [[done]] --[[Joey]] 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 0c1da2e6a5..17b8635218 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -11,6 +11,7 @@ sudo cabal install SHA
 sudo cabal install dataenc
 sudo cabal install monad-control
 sudo cabal install HTTP
+sudo cabal install json
 sudo cabal install hS3
 
 git clone git://git-annex.branchable.com/
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 983ac3a2fd..12502e0900 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -13,6 +13,7 @@ sudo cabal install SHA
 sudo cabal install dataenc
 sudo cabal install monad-control
 sudo cabal install HTTP
+sudo cabal install json
 sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)

From 7722147e1625e0a16cfcf13fd967a4a961decaf7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:36:51 -0400
Subject: [PATCH 2182/8313] add josn build dep

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7850f88a77..48ac48bb2d 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5, monad-control
+   base < 5, monad-control, json
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 3778e8897dcd8b2a5a54d076cc5d3a6d0a0bb689 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 14:48:15 -0400
Subject: [PATCH 2183/8313] switch to using cabal-install git-annex

---
 doc/install/Fedora.mdwn | 19 +------------------
 doc/install/OSX.mdwn    | 18 ++----------------
 2 files changed, 3 insertions(+), 34 deletions(-)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 17b8635218..068d5c111c 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -3,22 +3,5 @@ Installation recipe for Fedora 14.
 
 sudo yum install ghc cabal-install
 sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install monad-control
-sudo cabal install HTTP
-sudo cabal install json
-sudo cabal install hS3
-
-git clone git://git-annex.branchable.com/
-
-cd git-annex
-sudo make   # For some reason you need to use sudo here as otherwise the cabal installed packages doesn't seem to be there...
-sudo install git-annex
+sudo cabal install git-annex
 

-
-Originally posted by Jon at  --[[Joey]]
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 12502e0900..680c331ee6 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -4,26 +4,12 @@ Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. T
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
-sudo cabal update
-sudo cabal install missingh
-sudo cabal install utf8-string
-sudo cabal install pcre-light
-sudo cabal install quickcheck  
-sudo cabal install SHA
-sudo cabal install dataenc
-sudo cabal install monad-control
-sudo cabal install HTTP
-sudo cabal install json
-sudo cabal install hS3 # optional
 
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
-git clone  git://git-annex.branchable.com/
-
-cd git-annex
-make
-sudo make install
+sudo cabal update
+sudo cabal install git-annex
 

 
 Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]

From ebb92221fd2d42e360bf24a72b53971129908f01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:35:13 -0400
Subject: [PATCH 2184/8313] Fix Makefile to work with cabal again.

---
 Makefile         | 4 +++-
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 20d1347140..3f36928afb 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,9 @@ Remote/S3.o: Remote/S3.hs
 		echo "** building without S3 support"; \
 	fi
 
-$(bins): $(sources)
+sources: $(sources)
+
+$(bins): sources
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
diff --git a/debian/changelog b/debian/changelog
index 9ab3b4fd3d..973623e200 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110903) UNRELEASED; urgency=low
 
   * Improve display of newlines around error and warning messages.
+  * Fix Makefile to work with cabal again.
 
  -- Joey Hess   Tue, 06 Sep 2011 13:45:16 -0400
 

From 6fd0df7c2ff20710491923abfbb7a1bc4d3750bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:54:21 -0400
Subject: [PATCH 2185/8313] releasing version 3.20110906

---
 debian/changelog | 2 +-
 git-annex.cabal  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 973623e200..0fca41ee5c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110903) UNRELEASED; urgency=low
+git-annex (3.20110906) unstable; urgency=low
 
   * Improve display of newlines around error and warning messages.
   * Fix Makefile to work with cabal again.
diff --git a/git-annex.cabal b/git-annex.cabal
index 48ac48bb2d..34c8e76222 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110902
+Version: 3.20110906
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From b7bcd942c57631f50d686cb8d45a83eaf377c33a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 15:54:43 -0400
Subject: [PATCH 2186/8313] add news item for git-annex 3.20110906

---
 doc/news/version_3.20110707.mdwn | 8 --------
 doc/news/version_3.20110906.mdwn | 4 ++++
 2 files changed, 4 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_3.20110707.mdwn
 create mode 100644 doc/news/version_3.20110906.mdwn

diff --git a/doc/news/version_3.20110707.mdwn b/doc/news/version_3.20110707.mdwn
deleted file mode 100644
index 3f489ae5d6..0000000000
--- a/doc/news/version_3.20110707.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20110707 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix sign bug in disk free space checking.
-   * Bugfix: Forgot to de-escape keys when upgrading. Could result in
-     bad location log data for keys that contain [&:%] in their names.
-     (A workaround for this problem is to run git annex fsck.)
-   * add: Avoid a failure mode that resulted in the file seemingly being
-     deleted (content put in the annex but no symlink present)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110906.mdwn b/doc/news/version_3.20110906.mdwn
new file mode 100644
index 0000000000..d91776ccaf
--- /dev/null
+++ b/doc/news/version_3.20110906.mdwn
@@ -0,0 +1,4 @@
+git-annex 3.20110906 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Improve display of newlines around error and warning messages.
+   * Fix Makefile to work with cabal again."""]]
\ No newline at end of file

From 6f98fd53914b2490e866a2613e86b93f689034bf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 16:59:53 -0400
Subject: [PATCH 2187/8313] whereis: Show untrusted locations separately and do
 not include in location count.

---
 Command/Whereis.hs | 27 ++++++++++++++++++---------
 debian/changelog   |  7 +++++++
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index f80c823b70..d5c090e493 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,11 +7,15 @@
 
 module Command.Whereis where
 
+import Control.Monad
+import Data.List
+
 import LocationLog
 import Command
 import Messages
 import Remote
 import Types
+import Trust
 
 command :: [Command]
 command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek
@@ -27,16 +31,21 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> CommandPerform
 perform key = do
-	uuids <- keyLocations key
-	let num = length uuids
+	locations <- keyLocations key
+	untrusted <- trustGet UnTrusted
+	let untrustedlocations = intersect untrusted locations
+	let safelocations = filter (`notElem` untrusted) locations
+	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
-	if null uuids
-		then stop
-		else do
-			pp <- prettyPrintUUIDs "whereis" uuids
-			showLongNote pp
-			showOutput
-			next $ return True
+	pp <- prettyPrintUUIDs "whereis" safelocations
+	unless (null safelocations) $
+		showLongNote pp
+	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
+	unless (null untrustedlocations) $
+		showLongNote $ untrustedheader ++ pp'
+	unless (null locations) showOutput
+	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
 		copiesplural _ = "copies"
+		untrustedheader = "The following untrusted locations may also have copies:\n"
diff --git a/debian/changelog b/debian/changelog
index 0fca41ee5c..d900ab1840 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20110907) UNRELEASED; urgency=low
+
+  * whereis: Show untrusted locations separately and do not include in
+    location count.
+
+ -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
+
 git-annex (3.20110906) unstable; urgency=low
 
   * Improve display of newlines around error and warning messages.

From 3623d831d193d029a35aac81571d67768b176534 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Sep 2011 17:19:29 -0400
Subject: [PATCH 2188/8313] refactor

---
 Command/Fsck.hs    |  6 +-----
 Command/Whereis.hs |  5 +----
 Remote.hs          |  2 +-
 Trust.hs           | 12 ++++++++++--
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index bad60d30dd..529a5015a9 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -10,7 +10,6 @@ module Command.Fsck where
 import Control.Monad (when)
 import Control.Monad.State (liftIO)
 import System.Directory
-import Data.List
 import System.Posix.Files
 
 import Command
@@ -124,10 +123,7 @@ checkKeySize key = do
 checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
-	locations <- keyLocations key
-	untrusted <- trustGet UnTrusted
-	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (`notElem` untrusted) locations
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
 	let present = length safelocations
 	if present < needed
 		then do
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index d5c090e493..850975048a 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -8,7 +8,6 @@
 module Command.Whereis where
 
 import Control.Monad
-import Data.List
 
 import LocationLog
 import Command
@@ -32,9 +31,7 @@ start file = isAnnexed file $ \(key, _) -> do
 perform :: Key -> CommandPerform
 perform key = do
 	locations <- keyLocations key
-	untrusted <- trustGet UnTrusted
-	let untrustedlocations = intersect untrusted locations
-	let safelocations = filter (`notElem` untrusted) locations
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations
diff --git a/Remote.hs b/Remote.hs
index e54d2e2334..429f9058b6 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -181,7 +181,7 @@ keyPossibilities' withtrusted key = do
 	let validuuids = filter (/= u) uuids
 
 	-- note that validuuids is assumed to not have dups
-	let validtrusteduuids = intersect validuuids trusted
+	let validtrusteduuids = validuuids `intersect` trusted
 
 	-- remotes that match uuids that have the key
 	allremotes <- genList
diff --git a/Trust.hs b/Trust.hs
index 365186da22..232eea6a5d 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -9,11 +9,13 @@ module Trust (
 	TrustLevel(..),
 	trustLog,
 	trustGet,
-	trustSet
+	trustSet,
+	trustPartition
 ) where
 
 import Control.Monad.State
 import qualified Data.Map as M
+import Data.List
 
 import Types.TrustLevel
 import qualified Branch
@@ -32,7 +34,7 @@ trustGet level = do
 	return $ M.keys $ M.filter (== level) m
 
 {- Read the trustLog into a map, overriding with any
- - values from forcetrust -}
+ - values from forcetrust. The map is cached for speed. -}
 trustMap :: Annex TrustMap
 trustMap = do
 	cached <- Annex.getState Annex.trustmap
@@ -70,3 +72,9 @@ trustSet uuid level = do
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
+
+{- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
+trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
+trustPartition level ls = do
+	candidates <- trustGet level
+	return $ partition (`elem` candidates) ls

From 72b54d617006fd5ddce92ee577c52f2bff279310 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Sep 2011 10:21:19 -0400
Subject: [PATCH 2189/8313] Fix build without S3.

---
 Makefile         | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3f36928afb..ff58362766 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ Remote/S3.o: Remote/S3.hs
 
 sources: $(sources)
 
-$(bins): sources
+$(bins): sources Remote/S3.o
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
diff --git a/debian/changelog b/debian/changelog
index d900ab1840..e79685a9ab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110907) UNRELEASED; urgency=low
 
   * whereis: Show untrusted locations separately and do not include in
     location count.
+  * Fix build without S3.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 

From 7c768c09841d7346444d65721b132d144835fc99 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Sep 2011 18:57:38 -0400
Subject: [PATCH 2190/8313] simplify

---
 Git.hs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/Git.hs b/Git.hs
index ab43504e1a..cd6cdfbfd0 100644
--- a/Git.hs
+++ b/Git.hs
@@ -335,10 +335,8 @@ urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
 
 {- The full authority portion an URL repo. (ie, "user@host:port") -}
 urlAuthority :: Repo -> String
-urlAuthority Repo { location = Url u } = uriUserInfo a ++ uriRegName' a ++ uriPort a
-	where
-		a = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
-urlAuthority repo = assertUrl repo $ error "internal"
+urlAuthority r = flip urlAuthPart r $ \a ->
+	uriUserInfo a ++ uriRegName' a ++ uriPort a
 
 {- Applies a function to extract part of the uriAuthority of an URL repo. -}
 urlAuthPart :: (URIAuth -> a) -> Repo -> a

From 03d6209e1ccee4a8df7d1b0336c1d5587a2b3ff6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Sep 2011 19:04:51 -0400
Subject: [PATCH 2191/8313] addurl: Always use whole url as destination
 filename, rather than only its file component.

First, this ensures that git annex addurl, when run repeatedly with the
same url, doesn't create duplicate files, which it did before when it
fell back to the longer filename.

Secondly, the file part of an url is frequently not very descriptive on its
own.

The uri scheme, auth, and port is intentionally left out, as clutter.
---
 Command/AddUrl.hs                  | 26 +++++++++-----------------
 debian/changelog                   |  2 ++
 doc/walkthrough/using_the_web.mdwn | 12 ++++++------
 3 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 55e51100ce..9fc68ca037 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -7,9 +7,10 @@
 
 module Command.AddUrl where
 
-import Control.Monad.State (liftIO, when)
+import Control.Monad.State
 import Network.URI
 import Data.String.Utils
+import Data.Maybe
 import System.Directory
 
 import Command
@@ -24,6 +25,7 @@ import Content
 import PresenceLog
 import Locations
 import Utility.Path
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "addurl" paramPath seek "add urls to annex"]
@@ -75,20 +77,10 @@ nodownload url file = do
 
 url2file :: URI -> IO FilePath
 url2file url = do
-	let parts = filter safe $ split "/" $ uriPath url
-	if null parts
-		then fallback
-		else do
-			let file = last parts
-			e <- doesFileExist file
-			if e then fallback else return file
+	whenM (doesFileExist file) $
+		error $ "already have this url in " ++ file
+	return file
 	where
-		fallback = do
-			let file = replace "/" "_" $ show url
-			e <- doesFileExist file
-			when e $ error "already have this url"
-			return file
-		safe "" = False
-		safe "." = False
-		safe ".." = False
-		safe _ = True
+		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
+		escape = replace "/?" $ repeat '_'
+		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url
diff --git a/debian/changelog b/debian/changelog
index e79685a9ab..9ff745566f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * whereis: Show untrusted locations separately and do not include in
     location count.
   * Fix build without S3.
+  * addurl: Always use whole url as destination filename, rather than
+    only its file component.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/walkthrough/using_the_web.mdwn b/doc/walkthrough/using_the_web.mdwn
index 9d5525758a..8009927a49 100644
--- a/doc/walkthrough/using_the_web.mdwn
+++ b/doc/walkthrough/using_the_web.mdwn
@@ -1,20 +1,20 @@
 The web can be used as a [[special_remote|special_remotes]] too.
 
 	# git annex addurl http://example.com/video.mpeg
-	addurl video.mpeg (downloading http://example.com/video.mpeg)
+	addurl example.com_video.mpeg (downloading http://example.com/video.mpeg)
 	########################################################## 100.0%
 	ok
 
 Now the file is downloaded, and has been added to the annex like any other
-file. So it can be copied to other repositories, and so on.
+file. So it can be renamed, copied to other repositories, and so on.
 
 Note that git-annex assumes that, if the web site does not 404, the file is
 still present on the web, and this counts as one [[copy|copies]] of the
 file. So it will let you remove your last copy, trusting it can be
 downloaded again:
 
-	# git annex drop video.mpeg
-	drop video.mpeg (checking http://example.com/video.mpeg) ok
+	# git annex drop example.com_video.mpeg
+	drop example.com_video.mpeg (checking http://example.com/video.mpeg) ok
 
 If you don't [[trust]] the web to this degree, just let git-annex know:
 
@@ -23,8 +23,8 @@ If you don't [[trust]] the web to this degree, just let git-annex know:
 
 With the result that it will hang onto files:
 
-	# git annex drop video.mpeg
-	drop video.mpeg (unsafe) 
+	# git annex drop example.com_video.mpeg
+	drop example.com_video.mpeg (unsafe) 
 	  Could only verify the existence of 0 out of 1 necessary copies
 	  Also these untrusted repositories may contain the file:
 	  	00000000-0000-0000-0000-000000000001  -- web

From e4ba0934c2992f00275bd24e8c4d25d6b9ea10ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Sep 2011 00:11:32 -0400
Subject: [PATCH 2192/8313] fix / escape

---
 Command/AddUrl.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 9fc68ca037..b4e3ad9bdf 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -79,8 +79,9 @@ url2file :: URI -> IO FilePath
 url2file url = do
 	whenM (doesFileExist file) $
 		error $ "already have this url in " ++ file
+	liftIO $ print file
 	return file
 	where
 		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
-		escape = replace "/?" $ repeat '_'
+		escape = replace "/" "_" . replace "?" "_"
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url

From e296da4bfef21c55fc23376300becb17a08eea83 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Sep 2011 01:45:41 -0400
Subject: [PATCH 2193/8313] more newline fixes

Adds a missing newline when a longnote is followed by a endresult.

Multiple longnotes in a row will now be separated by a blank line, which
could be a bug or a feature depending on taste.

Removed several places where newlines were explicitly displayed after
longnotes.
---
 Command/Unused.hs  | 8 ++------
 Command/Whereis.hs | 1 -
 Messages.hs        | 2 +-
 Upgrade/V2.hs      | 1 -
 4 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 6a62cde5f8..960b2e1dfb 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -57,9 +57,7 @@ checkUnused = do
 	where
 		list file msg l c = do
 			let unusedlist = number c l
-			unless (null l) $ do
-				showLongNote $ msg unusedlist
-				showLongNote "\n"
+			unless (null l) $ showLongNote $ msg unusedlist
 			writeUnusedFile file unusedlist
 			return $ c + length l
 
@@ -76,9 +74,7 @@ checkRemoteUnused' r = do
 	let remoteunused = remotehas `exclude` referenced
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
-	unless (null remoteunused) $ do
-		showLongNote $ remoteUnusedMsg r list
-		showLongNote "\n"
+	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
 	where
 		{- This should run strictly to avoid the filterM
 		 - building many thunks containing keyLocations data. -}
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 850975048a..74660f2834 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -40,7 +40,6 @@ perform key = do
 	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
 	unless (null untrustedlocations) $
 		showLongNote $ untrustedheader ++ pp'
-	unless (null locations) showOutput
 	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
diff --git a/Messages.hs b/Messages.hs
index 4922519819..c663c17c20 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -57,7 +57,7 @@ showOutput :: Annex ()
 showOutput = handle q $ putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = handle (JSON.note s) $ putStr $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $ putStrLn $ '\n' : indent s
 
 showEndOk :: Annex ()
 showEndOk = showEndResult True
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ffd0f06535..e99a7cf817 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -123,7 +123,6 @@ push = do
 			showLongNote $
 				"git-annex branch created\n" ++
 				"Be sure to push this branch when pushing to remotes.\n"
-			showOutput
 
 {- Old .gitattributes contents, not needed anymore. -}
 attrLines :: [String]

From 1ac6217c74b63b9b154d5ee14ed72df8b5aa9268 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 13:32:46 -0400
Subject: [PATCH 2194/8313] shorten synopsis

---
 Command/Merge.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Merge.hs b/Command/Merge.hs
index 04328e8c5a..faaef906b5 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -13,7 +13,7 @@ import Messages
 
 command :: [Command]
 command = [repoCommand "merge" paramNothing seek
-		"auto-merges remote changes into the git-annex branch"]
+		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]

From 949b3f69d0f2b2a5c32a00d05d09a0b312fad35a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 13:47:22 -0400
Subject: [PATCH 2195/8313] optimize: A new subcommand that either gets or
 drops file content as needed to work toward meeting the configured numcopies
 setting.

This is currently rather simplistic, though still useful.
In the future, it could become smarter about what content is stored where,
etc.
---
 Command.hs                                   |  2 ++
 Command/Fsck.hs                              |  2 +-
 Command/Optimize.hs                          | 35 ++++++++++++++++++++
 GitAnnex.hs                                  |  2 ++
 debian/changelog                             |  2 ++
 doc/git-annex.mdwn                           |  5 +++
 doc/walkthrough.mdwn                         |  1 +
 doc/walkthrough/optimizing_repositories.mdwn | 13 ++++++++
 test.hs                                      | 12 +++++++
 9 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 Command/Optimize.hs
 create mode 100644 doc/walkthrough/optimizing_repositories.mdwn

diff --git a/Command.hs b/Command.hs
index 78f9823fb3..75c3b4412a 100644
--- a/Command.hs
+++ b/Command.hs
@@ -131,6 +131,8 @@ withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
+withNumCopies :: CommandSeekAttrFiles
+withNumCopies = withAttrFilesInGit "annex.numcopies"
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 529a5015a9..cdc68581ee 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -34,7 +34,7 @@ command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
 	"check for problems"]
 
 seek :: [CommandSeek]
-seek = [withAttrFilesInGit "annex.numcopies" start]
+seek = [withNumCopies start]
 
 start :: CommandStartAttrFile
 start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
diff --git a/Command/Optimize.hs b/Command/Optimize.hs
new file mode 100644
index 0000000000..40625fc2f0
--- /dev/null
+++ b/Command/Optimize.hs
@@ -0,0 +1,35 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Optimize where
+
+import Command
+import Utility
+import LocationLog
+import Trust
+import Config
+import qualified Command.Get
+import qualified Command.Drop
+
+command :: [Command]
+command = [repoCommand "optimize" (paramOptional $ paramRepeating paramPath) seek
+	"get or drop content to best use available space"]
+
+seek :: [CommandSeek]
+seek = [withNumCopies start]
+
+start :: CommandStartAttrFile
+start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
+	needed <- getNumCopies numcopies
+	(_, safelocations) <- trustPartition UnTrusted =<< keyLocations key
+	dispatch needed (length safelocations)
+	where
+		dispatch needed present
+			| present < needed = Command.Get.start file
+			| present > needed = Command.Drop.start p
+			| otherwise = stop
+		numcopies = readMaybe attr :: Maybe Int
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 6f4e5d4921..8b9e557500 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -34,6 +34,7 @@ import qualified Command.Init
 import qualified Command.Describe
 import qualified Command.InitRemote
 import qualified Command.Fsck
+import qualified Command.Optimize
 import qualified Command.Unused
 import qualified Command.DropUnused
 import qualified Command.Unlock
@@ -77,6 +78,7 @@ cmds = concat
 	, Command.SetKey.command
 	, Command.Fix.command
 	, Command.Fsck.command
+	, Command.Optimize.command
 	, Command.Unused.command
 	, Command.DropUnused.command
 	, Command.Find.command
diff --git a/debian/changelog b/debian/changelog
index 9ff745566f..b02f6a15b2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * Fix build without S3.
   * addurl: Always use whole url as destination filename, rather than
     only its file component.
+  * optimize: A new subcommand that either gets or drops file content
+    as needed to work toward meeting the configured numcopies setting.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0a484a3842..8264c31b3a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -157,6 +157,11 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   To avoid expensive checksum calculations, specify --fast
 
+* optimize [path ...]
+  
+  Either gets or drops file content, as needed, to work toward meeting the
+  configured numcopies setting.
+
 * unused
 
   Checks the annex for data that does not correspond to any files present
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index eaae6b455c..b0eb258153 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -18,5 +18,6 @@ A walkthrough of the basic features of git-annex.
 	fsck:_verifying_your_data
 	fsck:_when_things_go_wrong
 	backups
+	optimizing_repositories
 	more
 """]]
diff --git a/doc/walkthrough/optimizing_repositories.mdwn b/doc/walkthrough/optimizing_repositories.mdwn
new file mode 100644
index 0000000000..0f17f1deae
--- /dev/null
+++ b/doc/walkthrough/optimizing_repositories.mdwn
@@ -0,0 +1,13 @@
+Once you have multiple repositories, and have perhaps configured numcopies,
+any given file can have many more copies than is needed, or perhaps fewer
+than you would like. Fsck can detect the latter problem, but there's another
+command that can help deal with both problems.
+
+The optimize subcommand either gets or drops file content, as needed,
+to work toward meeting the configured numcopies setting.
+
+	# git annex optimize
+	get my_cool_big_file (from laptop...) ok
+	drop other_file ok
+	# git annex optimize --numcopies=2
+	get other_file ok
diff --git a/test.hs b/test.hs
index 4d751a707b..bd2e1e46c5 100644
--- a/test.hs
+++ b/test.hs
@@ -93,6 +93,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_unannex
 	, test_drop
 	, test_get
+	, test_optimize
 	, test_move
 	, test_copy
 	, test_lock
@@ -216,6 +217,17 @@ test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
 
+test_optimize :: Test
+test_optimize = "git-annex optimize" ~: TestCase $ intmpclonerepo $ do
+	inmainrepo $ annexed_present annexedfile
+	annexed_notpresent annexedfile
+	git_annex "optimize" ["-q", annexedfile, "--numcopies=2"] @? "optimize of file failed"
+	inmainrepo $ annexed_present annexedfile
+	annexed_present annexedfile
+	git_annex "optimize" ["-q", annexedfile] @? "optimize of file failed"
+	inmainrepo $ annexed_present annexedfile
+	annexed_notpresent annexedfile
+
 test_move :: Test
 test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile

From 5a1f10325f218bb2289e263ce1bdfd4a504467e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 13:48:39 -0400
Subject: [PATCH 2196/8313] refactor

---
 Command/Drop.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 6e688d6632..8afffd3cb2 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -24,7 +24,7 @@ command = [repoCommand "drop" paramPath seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
-seek = [withAttrFilesInGit "annex.numcopies" start]
+seek = [withNumCopies start]
 
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}

From 11994ebb3dc50a74dea9601907493950ac3540a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 15:33:21 -0400
Subject: [PATCH 2197/8313] tweak

---
 Command/Whereis.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 74660f2834..ef4303b3ae 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -30,8 +30,7 @@ start file = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> CommandPerform
 perform key = do
-	locations <- keyLocations key
-	(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations

From 59fe0b29a6f15a61276702e1594b9476e1363fd9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Sep 2011 16:01:40 -0400
Subject: [PATCH 2198/8313] simplify

---
 Command/Drop.hs     | 4 +---
 Command/Fsck.hs     | 4 +---
 Command/Optimize.hs | 3 +--
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 8afffd3cb2..a18729940f 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -34,10 +34,8 @@ start (file, attr) = isAnnexed file $ \(key, _) -> do
 	if present
 		then do
 			showStart "drop" file
-			next $ perform key numcopies
+			next $ perform key $ readMaybe attr
 		else stop
-	where
-		numcopies = readMaybe attr :: Maybe Int
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index cdc68581ee..08bf2b97f4 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -39,9 +39,7 @@ seek = [withNumCopies start]
 start :: CommandStartAttrFile
 start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	next $ perform key file backend numcopies
-	where
-		numcopies = readMaybe attr :: Maybe Int
+	next $ perform key file backend $ readMaybe attr
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
 perform key file backend numcopies = do
diff --git a/Command/Optimize.hs b/Command/Optimize.hs
index 40625fc2f0..1a2b2237ff 100644
--- a/Command/Optimize.hs
+++ b/Command/Optimize.hs
@@ -24,7 +24,7 @@ seek = [withNumCopies start]
 
 start :: CommandStartAttrFile
 start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
-	needed <- getNumCopies numcopies
+	needed <- getNumCopies $ readMaybe attr
 	(_, safelocations) <- trustPartition UnTrusted =<< keyLocations key
 	dispatch needed (length safelocations)
 	where
@@ -32,4 +32,3 @@ start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
 			| present < needed = Command.Get.start file
 			| present > needed = Command.Drop.start p
 			| otherwise = stop
-		numcopies = readMaybe attr :: Maybe Int

From 5c96411314837db235b5cd1d412243806b83aa8a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 12:36:27 -0400
Subject: [PATCH 2199/8313] fix synopsis

---
 Command.hs        | 2 ++
 Command/AddUrl.hs | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index 75c3b4412a..710f36ea3e 100644
--- a/Command.hs
+++ b/Command.hs
@@ -221,6 +221,8 @@ paramKey :: String
 paramKey = "KEY"
 paramDesc :: String
 paramDesc = "DESC"
+paramUrl :: String
+paramUrl = "URL"
 paramNumber :: String
 paramNumber = "NUMBER"
 paramRemote :: String
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index b4e3ad9bdf..c78569ffc8 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -28,7 +28,8 @@ import Utility.Path
 import Utility.Conditional
 
 command :: [Command]
-command = [repoCommand "addurl" paramPath seek "add urls to annex"]
+command = [repoCommand "addurl" (paramRepeating $ paramUrl) seek
+	"add urls to annex"]
 
 seek :: [CommandSeek]
 seek = [withStrings start]

From 81984e60acb920fffc969818e63604060636aa09 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 12:37:27 -0400
Subject: [PATCH 2200/8313] better var name

---
 Command/Drop.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index a18729940f..8f886f283c 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -29,12 +29,12 @@ seek = [withNumCopies start]
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
 start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) -> do
+start (file, numcopies) = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
 	if present
 		then do
 			showStart "drop" file
-			next $ perform key $ readMaybe attr
+			next $ perform key $ readMaybe numcopies
 		else stop
 
 perform :: Key -> Maybe Int -> CommandPerform

From 984c9fc0523bcd3bfcd7de83f4f7974daa6872bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 13:30:04 -0400
Subject: [PATCH 2201/8313] remove optimize subcommand; use --auto instead

get, drop: Added --auto option, which decides whether to get/drop content
as needed to work toward the configured numcopies.

The problem with bundling it up in optimize was that I then found I wanted
to run an optmize that did not drop files, only got them. Considered adding
a --only-get switch to it, but that seemed wrong. Instead, let's make
existing subcommands optionally smarter.

Note that the only actual difference between drop and drop --auto is that
the latter does not even try to drop a file if it knows of not enough
copies, and does not print any error messages about files it was unable to
drop.

It might be nice to make get avoid asking git for attributes when not in
auto mode. For now it always asks for attributes.
---
 Annex.hs                                      |  2 +
 Command.hs                                    | 19 ++++++++++
 Command/Drop.hs                               |  8 ++--
 Command/Get.hs                                | 11 ++++--
 Command/Optimize.hs                           | 34 -----------------
 GitAnnex.hs                                   |  2 -
 Options.hs                                    |  3 ++
 debian/changelog                              |  4 +-
 doc/git-annex.mdwn                            | 22 +++++++----
 doc/walkthrough.mdwn                          |  2 +-
 .../automatically_managing_content.mdwn       | 38 +++++++++++++++++++
 doc/walkthrough/optimizing_repositories.mdwn  | 13 -------
 test.hs                                       | 12 ------
 13 files changed, 92 insertions(+), 78 deletions(-)
 delete mode 100644 Command/Optimize.hs
 create mode 100644 doc/walkthrough/automatically_managing_content.mdwn
 delete mode 100644 doc/walkthrough/optimizing_repositories.mdwn

diff --git a/Annex.hs b/Annex.hs
index f5c3e4de45..2c8ea1d61d 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -52,6 +52,7 @@ data AnnexState = AnnexState
 	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
+	, auto :: Bool
 	, branchstate :: BranchState
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
@@ -75,6 +76,7 @@ newState gitrepo = AnnexState
 	, output = NormalOutput
 	, force = False
 	, fast = False
+	, auto = False
 	, branchstate = startBranchState
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
diff --git a/Command.hs b/Command.hs
index 710f36ea3e..9d85fbe9d7 100644
--- a/Command.hs
+++ b/Command.hs
@@ -26,6 +26,9 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Utility
 import Types.Key
+import Trust
+import LocationLog
+import Config
 
 {- A command runs in four stages.
  -
@@ -276,3 +279,19 @@ preserveOrder orig new = collect orig new
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
 runPreserveOrder a files = preserveOrder files <$> a files
+
+{- Used for commands that have an auto mode that checks the number of known
+ - copies of a key.
+ -
+ - In auto mode, first checks that the number of known
+ - copies of the key is > or < than the numcopies setting, before running
+ - the action. -}
+autoCopies :: Key -> (Int -> Int -> Bool) -> Maybe Int -> CommandStart -> CommandStart
+autoCopies key vs numcopiesattr a = do
+	auto <- Annex.getState Annex.auto
+	if auto
+		then do
+			needed <- getNumCopies numcopiesattr
+			(_, have) <- trustPartition UnTrusted =<< keyLocations key
+			if length have `vs` needed then a else stop
+		else a
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 8f886f283c..32cb81e808 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -29,13 +29,15 @@ seek = [withNumCopies start]
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
 start :: CommandStartAttrFile
-start (file, numcopies) = isAnnexed file $ \(key, _) -> do
+start (file, attr) = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
 	if present
-		then do
+		then autoCopies key (>) numcopies $ do
 			showStart "drop" file
-			next $ perform key $ readMaybe numcopies
+			next $ perform key numcopies
 		else stop
+	where
+		numcopies = readMaybe attr
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
diff --git a/Command/Get.hs b/Command/Get.hs
index e0436a8680..1a50158e38 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -13,6 +13,7 @@ import qualified Remote
 import Types
 import Content
 import Messages
+import Utility
 import qualified Command.Move
 
 command :: [Command]
@@ -20,14 +21,14 @@ command = [repoCommand "get" paramPath seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withNumCopies start]
 
-start :: CommandStartString
-start file = isAnnexed file $ \(key, _) -> do
+start :: CommandStartAttrFile
+start (file, attr) = isAnnexed file $ \(key, _) -> do
 	inannex <- inAnnex key
 	if inannex
 		then stop
-		else do
+		else autoCopies key (<) numcopies $ do
 			showStart "get" file
 			from <- Annex.getState Annex.fromremote
 			case from of
@@ -35,6 +36,8 @@ start file = isAnnexed file $ \(key, _) -> do
 				Just name -> do
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
+	where
+		numcopies = readMaybe attr
 
 perform :: Key -> CommandPerform
 perform key = do
diff --git a/Command/Optimize.hs b/Command/Optimize.hs
deleted file mode 100644
index 1a2b2237ff..0000000000
--- a/Command/Optimize.hs
+++ /dev/null
@@ -1,34 +0,0 @@
-{- git-annex command
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Command.Optimize where
-
-import Command
-import Utility
-import LocationLog
-import Trust
-import Config
-import qualified Command.Get
-import qualified Command.Drop
-
-command :: [Command]
-command = [repoCommand "optimize" (paramOptional $ paramRepeating paramPath) seek
-	"get or drop content to best use available space"]
-
-seek :: [CommandSeek]
-seek = [withNumCopies start]
-
-start :: CommandStartAttrFile
-start p@(file, attr) = notBareRepo $ isAnnexed file $ \(key, _) -> do
-	needed <- getNumCopies $ readMaybe attr
-	(_, safelocations) <- trustPartition UnTrusted =<< keyLocations key
-	dispatch needed (length safelocations)
-	where
-		dispatch needed present
-			| present < needed = Command.Get.start file
-			| present > needed = Command.Drop.start p
-			| otherwise = stop
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 8b9e557500..6f4e5d4921 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -34,7 +34,6 @@ import qualified Command.Init
 import qualified Command.Describe
 import qualified Command.InitRemote
 import qualified Command.Fsck
-import qualified Command.Optimize
 import qualified Command.Unused
 import qualified Command.DropUnused
 import qualified Command.Unlock
@@ -78,7 +77,6 @@ cmds = concat
 	, Command.SetKey.command
 	, Command.Fix.command
 	, Command.Fsck.command
-	, Command.Optimize.command
 	, Command.Unused.command
 	, Command.DropUnused.command
 	, Command.Find.command
diff --git a/Options.hs b/Options.hs
index e0ca48c01b..ee3ce66203 100644
--- a/Options.hs
+++ b/Options.hs
@@ -26,6 +26,8 @@ commonOptions =
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
 		"avoid slow operations"
+	, Option ['a'] ["auto"] (NoArg (setauto True))
+		"automatic mode"
 	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
 		"avoid verbose output"
 	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
@@ -40,6 +42,7 @@ commonOptions =
 	where
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
+		setauto v = Annex.changeState $ \s -> s { Annex.auto = v }
 		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
diff --git a/debian/changelog b/debian/changelog
index b02f6a15b2..a3ae839abf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,8 +5,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * Fix build without S3.
   * addurl: Always use whole url as destination filename, rather than
     only its file component.
-  * optimize: A new subcommand that either gets or drops file content
-    as needed to work toward meeting the configured numcopies setting.
+  * get, drop: Added --auto option, which decides whether to get/drop
+    content as needed to work toward the configured numcopies.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8264c31b3a..83bfc7e405 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -76,12 +76,19 @@ Many git-annex commands will stage changes for later `git commit` by you.
   will involve copying them from another repository, or downloading them,
   or transferring them from some kind of key-value store.
 
+  When the --auto switch is used, only gets content of files if needed
+  to satisfy the setting of annex.numcopies
+
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
 
-  git-annex may refuse to drop content if it does not think
-  it is safe to do so, typically because of the setting of annex.numcopies.
+  git-annex will refuse to drop content if it cannot verify it is
+  safe to do so. At least one copy of content needs to exist in another
+  remote. This can be overridden with the --force switch.
+
+  When the --auto switch is used, only tries to drop content if
+  more than annex.numcopies copies exist.
 
 * move [path ...]
   
@@ -157,11 +164,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
   To avoid expensive checksum calculations, specify --fast
 
-* optimize [path ...]
-  
-  Either gets or drops file content, as needed, to work toward meeting the
-  configured numcopies setting.
-
 * unused
 
   Checks the annex for data that does not correspond to any files present
@@ -340,6 +342,12 @@ Many git-annex commands will stage changes for later `git commit` by you.
   Enables less expensive, but also less thorough versions of some commands.
   What is avoided depends on the command.
 
+* --auto
+
+  Enable automatic mode, in which git-annex decides whether to perform
+  actions on files. See descriptions of individual commands to see what
+  they do in automatic mode.
+
 * --quiet
 
   Avoid the default verbose display of what is done; only show errors
diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index b0eb258153..68f94a6f27 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -18,6 +18,6 @@ A walkthrough of the basic features of git-annex.
 	fsck:_verifying_your_data
 	fsck:_when_things_go_wrong
 	backups
-	optimizing_repositories
+	automatically_managing_content
 	more
 """]]
diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
new file mode 100644
index 0000000000..74a4c1b5b7
--- /dev/null
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -0,0 +1,38 @@
+Once you have multiple repositories, and have perhaps configured numcopies,
+any given file can have many more copies than is needed, or perhaps fewer
+than you would like. How to manage this?
+
+The whereis subcommand can be used to see how many copies of a file are known,
+but then you have to decide what to get or drop. In this example, there
+are rather too many copies of `other_file` and perhaps not enough of the
+other file.
+
+	# cd /media/usbdrive
+	# git annex whereis
+	whereis my_cool_big_file (1 copy)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+	whereis other_file (3 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+What would be handy is some automated versions of get and drop, that only
+get a file if there are not yet enough copies of it, or only drop a file
+if there are too many copies. Well, these exist, just use the --auto
+option.
+
+	# git annex get --auto --numcopies=2
+	get my_cool_big_file (from laptop...) ok
+	# git annex drop --auto --numcopies=2
+	drop other_file ok
+
+With two quick commands, git-annex was able to decide for you how to
+work toward having two copies of your files.
+
+	# git annex whereis
+	whereis my_cool_big_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+	whereis other_file (2 copies)
+		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
+		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
diff --git a/doc/walkthrough/optimizing_repositories.mdwn b/doc/walkthrough/optimizing_repositories.mdwn
deleted file mode 100644
index 0f17f1deae..0000000000
--- a/doc/walkthrough/optimizing_repositories.mdwn
+++ /dev/null
@@ -1,13 +0,0 @@
-Once you have multiple repositories, and have perhaps configured numcopies,
-any given file can have many more copies than is needed, or perhaps fewer
-than you would like. Fsck can detect the latter problem, but there's another
-command that can help deal with both problems.
-
-The optimize subcommand either gets or drops file content, as needed,
-to work toward meeting the configured numcopies setting.
-
-	# git annex optimize
-	get my_cool_big_file (from laptop...) ok
-	drop other_file ok
-	# git annex optimize --numcopies=2
-	get other_file ok
diff --git a/test.hs b/test.hs
index bd2e1e46c5..4d751a707b 100644
--- a/test.hs
+++ b/test.hs
@@ -93,7 +93,6 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_unannex
 	, test_drop
 	, test_get
-	, test_optimize
 	, test_move
 	, test_copy
 	, test_lock
@@ -217,17 +216,6 @@ test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
 
-test_optimize :: Test
-test_optimize = "git-annex optimize" ~: TestCase $ intmpclonerepo $ do
-	inmainrepo $ annexed_present annexedfile
-	annexed_notpresent annexedfile
-	git_annex "optimize" ["-q", annexedfile, "--numcopies=2"] @? "optimize of file failed"
-	inmainrepo $ annexed_present annexedfile
-	annexed_present annexedfile
-	git_annex "optimize" ["-q", annexedfile] @? "optimize of file failed"
-	inmainrepo $ annexed_present annexedfile
-	annexed_notpresent annexedfile
-
 test_move :: Test
 test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile

From 9fe3c6d21134638a29c6ad4ced6a11ed9b0242ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 14:33:37 -0400
Subject: [PATCH 2202/8313] clean up params in usage display

---
 Command.hs           | 14 ++++++++------
 Command/Add.hs       |  2 +-
 Command/Copy.hs      |  2 +-
 Command/Drop.hs      |  2 +-
 Command/Find.hs      |  3 +--
 Command/Fix.hs       |  2 +-
 Command/Fsck.hs      |  3 +--
 Command/Get.hs       |  2 +-
 Command/Lock.hs      |  2 +-
 Command/Migrate.hs   |  3 ++-
 Command/Move.hs      |  2 +-
 Command/PreCommit.hs |  2 +-
 Command/Unannex.hs   |  2 +-
 Command/Uninit.hs    |  2 +-
 Command/Unlock.hs    |  4 ++--
 Command/Whereis.hs   |  2 +-
 16 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/Command.hs b/Command.hs
index 9d85fbe9d7..c1069f0d92 100644
--- a/Command.hs
+++ b/Command.hs
@@ -212,12 +212,8 @@ notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
 {- Descriptions of params used in usage messages. -}
-paramRepeating :: String -> String
-paramRepeating s = s ++ " ..."
-paramOptional :: String -> String
-paramOptional s = "[" ++ s ++ "]"
-paramPair :: String -> String -> String
-paramPair a b = a ++ " " ++ b
+paramPaths :: String
+paramPaths = paramOptional $ paramRepeating paramPath -- most often used
 paramPath :: String
 paramPath = "PATH"
 paramKey :: String
@@ -240,6 +236,12 @@ paramKeyValue :: String
 paramKeyValue = "K=V"
 paramNothing :: String
 paramNothing = ""
+paramRepeating :: String -> String
+paramRepeating s = s ++ " ..."
+paramOptional :: String -> String
+paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b
 
 {- The Key specified by the --key parameter. -}
 cmdlineKey :: Annex Key
diff --git a/Command/Add.hs b/Command/Add.hs
index 579c4171b1..f6dfd2da43 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -29,7 +29,7 @@ import Utility.SafeCommand
 import Locations
 
 command :: [Command]
-command = [repoCommand "add" paramPath seek "add files to annex"]
+command = [repoCommand "add" paramPaths seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 46d49dd058..7d40f438dc 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -11,7 +11,7 @@ import Command
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "copy" paramPath seek
+command = [repoCommand "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 32cb81e808..1325e3dcbd 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -20,7 +20,7 @@ import Trust
 import Config
 
 command :: [Command]
-command = [repoCommand "drop" paramPath seek
+command = [repoCommand "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 0716c5297e..0ff68b7ed1 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -14,8 +14,7 @@ import Content
 import Utility.Conditional
 
 command :: [Command]
-command = [repoCommand "find" (paramOptional $ paramRepeating paramPath) seek
-	"lists available files"]
+command = [repoCommand "find" paramPaths seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index b24f8e33c2..0044052e9c 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -19,7 +19,7 @@ import Content
 import Messages
 
 command :: [Command]
-command = [repoCommand "fix" paramPath seek
+command = [repoCommand "fix" paramPaths seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 08bf2b97f4..4e8eeb0433 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -30,8 +30,7 @@ import Utility.Path
 import Config
 
 command :: [Command]
-command = [repoCommand "fsck" (paramOptional $ paramRepeating paramPath) seek
-	"check for problems"]
+command = [repoCommand "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index 1a50158e38..358105cac1 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -17,7 +17,7 @@ import Utility
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "get" paramPath seek
+command = [repoCommand "get" paramPaths seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 77d1ff94f9..07721e9370 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -16,7 +16,7 @@ import qualified AnnexQueue
 import Utility.SafeCommand
 	
 command :: [Command]
-command = [repoCommand "lock" paramPath seek "undo unlock command"]
+command = [repoCommand "lock" paramPaths seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 6ad7e239c9..ec570acb7b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -25,7 +25,8 @@ import Utility.Conditional
 import qualified Command.Add
 
 command :: [Command]
-command = [repoCommand "migrate" paramPath seek "switch data to different backend"]
+command = [repoCommand "migrate" paramPaths seek
+	"switch data to different backend"]
 
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
diff --git a/Command/Move.hs b/Command/Move.hs
index a081a863f2..88da92f0a4 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -20,7 +20,7 @@ import UUID
 import Messages
 
 command :: [Command]
-command = [repoCommand "move" paramPath seek
+command = [repoCommand "move" paramPaths seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 3046d35626..6884a4787c 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -12,7 +12,7 @@ import qualified Command.Add
 import qualified Command.Fix
 
 command :: [Command]
-command = [repoCommand "pre-commit" paramPath seek "run by git pre-commit hook"]
+command = [repoCommand "pre-commit" paramPaths seek "run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 54ef2fc68e..748da4066c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -27,7 +27,7 @@ import Messages
 import Locations
 
 command :: [Command]
-command = [repoCommand "unannex" paramPath seek "undo accidential add command"]
+command = [repoCommand "unannex" paramPaths seek "undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index fadae0e5a9..4c70ec80c4 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -22,7 +22,7 @@ import Content
 import Locations
 
 command :: [Command]
-command = [repoCommand "uninit" paramPath seek 
+command = [repoCommand "uninit" paramPaths seek 
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 0daf1b3218..ba6d023874 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -22,8 +22,8 @@ import Utility.Path
 
 command :: [Command]
 command =
-	[ repoCommand "unlock" paramPath seek "unlock files for modification"
-	, repoCommand "edit" paramPath seek "same as unlock"
+	[ repoCommand "unlock" paramPaths seek "unlock files for modification"
+	, repoCommand "edit" paramPaths seek "same as unlock"
 	]
 
 seek :: [CommandSeek]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index ef4303b3ae..7d0ab188c0 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -17,7 +17,7 @@ import Types
 import Trust
 
 command :: [Command]
-command = [repoCommand "whereis" (paramOptional $ paramRepeating paramPath) seek
+command = [repoCommand "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]

From fe5e4bdc647f779e65f935f42e6fe798ab036165 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 14:34:15 -0400
Subject: [PATCH 2203/8313] comment

---
 Command/Get.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Get.hs b/Command/Get.hs
index 358105cac1..bc617c9fa1 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -34,6 +34,7 @@ start (file, attr) = isAnnexed file $ \(key, _) -> do
 			case from of
 				Nothing -> next $ perform key
 				Just name -> do
+					-- get --from = copy --from
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
 	where

From 7b90cb72fc7d86a8f99d839378fba1ef1c86ff4a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:15:47 -0400
Subject: [PATCH 2204/8313] document --auto all in one place

---
 doc/git-annex.mdwn | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 83bfc7e405..0d3aebe2be 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -76,9 +76,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
   will involve copying them from another repository, or downloading them,
   or transferring them from some kind of key-value store.
 
-  When the --auto switch is used, only gets content of files if needed
-  to satisfy the setting of annex.numcopies
-
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 
@@ -87,9 +84,6 @@ Many git-annex commands will stage changes for later `git commit` by you.
   safe to do so. At least one copy of content needs to exist in another
   remote. This can be overridden with the --force switch.
 
-  When the --auto switch is used, only tries to drop content if
-  more than annex.numcopies copies exist.
-
 * move [path ...]
   
   When used with the --from option, moves the content of annexed files
@@ -344,9 +338,8 @@ Many git-annex commands will stage changes for later `git commit` by you.
 
 * --auto
 
-  Enable automatic mode, in which git-annex decides whether to perform
-  actions on files. See descriptions of individual commands to see what
-  they do in automatic mode.
+  Enables automatic mode. Commands that get, drop, or move file contents
+  will only do so when needed to help satisfy the setting of annex.numcopies.
 
 * --quiet
 

From a0d3a343b52fba63df523d49849b43217ce744ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:27:45 -0400
Subject: [PATCH 2205/8313] copy --auto

Only does copy when numcopies is not yet satisfied.
---
 Command/Copy.hs                                     | 13 +++++++++++--
 debian/changelog                                    |  4 ++--
 doc/walkthrough/automatically_managing_content.mdwn |  3 +++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/Command/Copy.hs b/Command/Copy.hs
index 7d40f438dc..125e0bb9f8 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -9,11 +9,20 @@ module Command.Copy where
 
 import Command
 import qualified Command.Move
+import Utility
 
 command :: [Command]
 command = [repoCommand "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
--- A copy is just a move that does not delete the source file.
 seek :: [CommandSeek]
-seek = [withFilesInGit $ Command.Move.start False]
+seek = [withNumCopies start]
+
+-- A copy is just a move that does not delete the source file.
+-- However, --auto mode avoids unnecessary copies.
+start :: CommandStartAttrFile
+start (file, attr) = isAnnexed file $ \(key, _) ->
+	autoCopies key (<) numcopies $
+		Command.Move.start False file
+	where
+		numcopies = readMaybe attr
diff --git a/debian/changelog b/debian/changelog
index a3ae839abf..d9e606f829 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,8 +5,8 @@ git-annex (3.20110907) UNRELEASED; urgency=low
   * Fix build without S3.
   * addurl: Always use whole url as destination filename, rather than
     only its file component.
-  * get, drop: Added --auto option, which decides whether to get/drop
-    content as needed to work toward the configured numcopies.
+  * get, drop, copy: Added --auto option, which decides whether
+    to get/drop content as needed to work toward the configured numcopies.
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 
diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
index 74a4c1b5b7..5107d2f6f8 100644
--- a/doc/walkthrough/automatically_managing_content.mdwn
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -36,3 +36,6 @@ work toward having two copies of your files.
 	whereis other_file (2 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
+
+The --auto option can also be used with the copy command,
+again this lets git-annex decide whether to actually copy content.

From e47d1fd43e2433966b7baa6fc179ec6b70774214 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:33:20 -0400
Subject: [PATCH 2206/8313] add error for move --auto

It probably does not make sense to enable auto mode for move. I cannot
think of a situation where it would make sense to try to use it.

A hypothetical auto mode for move would only differ from a normal
move in one case -- when both repositories have a file, move deletes it
from one, and this reduces the number of copies. So an auto mode would
either only let move work in that situation, or avoid removing the file
in that situation, depending on the number of copies. This would be
complex to implement, and is perhaps not a very obvious behavior.

The error is a good thing to have, so users don't expect it to do something
it does not.
---
 Command/Move.hs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Command/Move.hs b/Command/Move.hs
index 88da92f0a4..f4310a2b88 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -18,6 +18,7 @@ import Content
 import qualified Remote
 import UUID
 import Messages
+import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
@@ -32,6 +33,7 @@ seek = [withFilesInGit $ start True]
  - moving data in the key-value backend. -}
 start :: Bool -> CommandStartString
 start move file = do
+	noAuto
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
 	case (from, to) of
@@ -43,6 +45,9 @@ start move file = do
 			src <- Remote.byName name
 			fromStart src move file
 		(_ ,  _) -> error "only one of --from or --to can be specified"
+	where
+		noAuto = when move $ whenM (Annex.getState Annex.auto) $ error
+			"--auto is not supported for move"
 
 showMoveAction :: Bool -> FilePath -> Annex ()
 showMoveAction True file = showStart "move" file

From aedf84f7d638eb222811c13da1c642268ca06600 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:39:48 -0400
Subject: [PATCH 2207/8313] wording

---
 doc/walkthrough/automatically_managing_content.mdwn | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
index 5107d2f6f8..ba0cad6092 100644
--- a/doc/walkthrough/automatically_managing_content.mdwn
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -4,8 +4,8 @@ than you would like. How to manage this?
 
 The whereis subcommand can be used to see how many copies of a file are known,
 but then you have to decide what to get or drop. In this example, there
-are rather too many copies of `other_file` and perhaps not enough of the
-other file.
+are perhaps not enough copies of the first file, and too many of the second
+file.
 
 	# cd /media/usbdrive
 	# git annex whereis
@@ -17,9 +17,8 @@ other file.
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
 
 What would be handy is some automated versions of get and drop, that only
-get a file if there are not yet enough copies of it, or only drop a file
-if there are too many copies. Well, these exist, just use the --auto
-option.
+gets a file if there are not yet enough copies of it, or only drops a file
+if there are too many copies. Well, these exist, just use the --auto option.
 
 	# git annex get --auto --numcopies=2
 	get my_cool_big_file (from laptop...) ok

From d036cd590f5c3c4edcd025effcf57c3d16886559 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 15:44:32 -0400
Subject: [PATCH 2208/8313] bugfix: drop and fsck did not honor --exclude

---
 Command.hs       | 3 ++-
 debian/changelog | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command.hs b/Command.hs
index c1069f0d92..bcc05c03c9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -133,7 +133,8 @@ withAttrFilesInGit :: String -> CommandSeekAttrFiles
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	liftM (map a) $ liftIO $ Git.checkAttr repo attr files
+	files' <- filterFiles files
+	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
 withNumCopies :: CommandSeekAttrFiles
 withNumCopies = withAttrFilesInGit "annex.numcopies"
 withBackendFilesInGit :: CommandSeekBackendFiles
diff --git a/debian/changelog b/debian/changelog
index d9e606f829..e940989da7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,7 @@ git-annex (3.20110907) UNRELEASED; urgency=low
     only its file component.
   * get, drop, copy: Added --auto option, which decides whether
     to get/drop content as needed to work toward the configured numcopies.
+  * bugfix: drop and fsck did not honor --exclude
 
  -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
 

From 456b45b9b3982d9440a43ec014635dee15066f0e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 16:24:47 -0400
Subject: [PATCH 2209/8313] move annex.numcopies parsing into withNumCopies

---
 Command.hs      | 13 +++++++------
 Command/Copy.hs |  7 ++-----
 Command/Drop.hs |  7 ++-----
 Command/Fsck.hs |  7 +++----
 Command/Get.hs  |  7 ++-----
 5 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/Command.hs b/Command.hs
index bcc05c03c9..05b215ec22 100644
--- a/Command.hs
+++ b/Command.hs
@@ -59,9 +59,6 @@ type CommandStartKey = Key -> CommandStart
 type BackendFile = (FilePath, Maybe (Backend Annex))
 type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek
 type CommandStartBackendFile = BackendFile -> CommandStart
-type AttrFile = (FilePath, String)
-type CommandSeekAttrFiles = CommandStartAttrFile -> CommandSeek
-type CommandStartAttrFile = AttrFile -> CommandStart
 type CommandSeekNothing = CommandStart -> CommandSeek
 type CommandStartNothing = CommandStart
 
@@ -129,14 +126,18 @@ withFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	liftM (map a) $ filterFiles files
-withAttrFilesInGit :: String -> CommandSeekAttrFiles
+withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	files' <- filterFiles files
 	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
-withNumCopies :: CommandSeekAttrFiles
-withNumCopies = withAttrFilesInGit "annex.numcopies"
+withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
+withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
+	where
+		go (file, v) = do
+			let numcopies = readMaybe v
+			a file numcopies
 withBackendFilesInGit :: CommandSeekBackendFiles
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 125e0bb9f8..d7625ccdb0 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -9,7 +9,6 @@ module Command.Copy where
 
 import Command
 import qualified Command.Move
-import Utility
 
 command :: [Command]
 command = [repoCommand "copy" paramPaths seek
@@ -20,9 +19,7 @@ seek = [withNumCopies start]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) ->
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = isAnnexed file $ \(key, _) ->
 	autoCopies key (<) numcopies $
 		Command.Move.start False file
-	where
-		numcopies = readMaybe attr
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 1325e3dcbd..4a7596921f 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -14,7 +14,6 @@ import LocationLog
 import Types
 import Content
 import Messages
-import Utility
 import Utility.Conditional
 import Trust
 import Config
@@ -28,16 +27,14 @@ seek = [withNumCopies start]
 
 {- Indicates a file's content is not wanted anymore, and should be removed
  - if it's safe to do so. -}
-start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
 	if present
 		then autoCopies key (>) numcopies $ do
 			showStart "drop" file
 			next $ perform key numcopies
 		else stop
-	where
-		numcopies = readMaybe attr
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 4e8eeb0433..142f755a7f 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,7 +20,6 @@ import qualified Types.Key
 import UUID
 import Types
 import Messages
-import Utility
 import Content
 import LocationLog
 import Locations
@@ -35,10 +34,10 @@ command = [repoCommand "fsck" paramPaths seek "check for problems"]
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
-start :: CommandStartAttrFile
-start (file, attr) = notBareRepo $ isAnnexed file $ \(key, backend) -> do
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
-	next $ perform key file backend $ readMaybe attr
+	next $ perform key file backend numcopies
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
 perform key file backend numcopies = do
diff --git a/Command/Get.hs b/Command/Get.hs
index bc617c9fa1..4fd654f63e 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -13,7 +13,6 @@ import qualified Remote
 import Types
 import Content
 import Messages
-import Utility
 import qualified Command.Move
 
 command :: [Command]
@@ -23,8 +22,8 @@ command = [repoCommand "get" paramPaths seek
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
-start :: CommandStartAttrFile
-start (file, attr) = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> Maybe Int -> CommandStart
+start file numcopies = isAnnexed file $ \(key, _) -> do
 	inannex <- inAnnex key
 	if inannex
 		then stop
@@ -37,8 +36,6 @@ start (file, attr) = isAnnexed file $ \(key, _) -> do
 					-- get --from = copy --from
 					src <- Remote.byName name
 					next $ Command.Move.fromPerform src False key
-	where
-		numcopies = readMaybe attr
 
 perform :: Key -> CommandPerform
 perform key = do

From 35145202d2e463569989b710ab5b87f6d9a8fdc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 16:50:49 -0400
Subject: [PATCH 2210/8313] remove command type definitions

These were a mistake, they make the type signatures harder to read and
less flexible. The CommandSeek, CommandStart, CommandPerform, and
CommandCleanup types were a good idea, but composing them with the
parameters expected is going too far.
---
 Command.hs            | 39 ++++++++++++++-------------------------
 Command/Add.hs        |  6 +++---
 Command/AddUrl.hs     |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Describe.hs   |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/InAnnex.hs    |  3 ++-
 Command/Init.hs       |  2 +-
 Command/InitRemote.hs |  2 +-
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  2 +-
 Command/Merge.hs      |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Move.hs       |  6 +++---
 Command/PreCommit.hs  |  4 ++--
 Command/RecvKey.hs    |  3 ++-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  3 ++-
 Command/SetKey.hs     |  2 +-
 Command/Status.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  2 +-
 Command/Uninit.hs     |  4 ++--
 Command/Unlock.hs     |  2 +-
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  2 +-
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 33 files changed, 55 insertions(+), 63 deletions(-)

diff --git a/Command.hs b/Command.hs
index 05b215ec22..6bd451a7e9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -48,19 +48,8 @@ type CommandPerform = Annex (Maybe CommandCleanup)
 {- 3. The cleanup stage is run only if the perform stage succeeds, and it
  -    returns the overall success/fail of the command. -}
 type CommandCleanup = Annex Bool
-{- Some helper functions are used to build up CommandSeek and CommandStart
- - functions. -}
-type CommandSeekStrings = CommandStartString -> CommandSeek
-type CommandStartString = String -> CommandStart
-type CommandSeekWords = CommandStartWords -> CommandSeek
-type CommandStartWords = [String] -> CommandStart
-type CommandSeekKeys = CommandStartKey -> CommandSeek
-type CommandStartKey = Key -> CommandStart
+
 type BackendFile = (FilePath, Maybe (Backend Annex))
-type CommandSeekBackendFiles = CommandStartBackendFile -> CommandSeek
-type CommandStartBackendFile = BackendFile -> CommandStart
-type CommandSeekNothing = CommandStart -> CommandSeek
-type CommandStartNothing = CommandStart
 
 data Command = Command {
 	cmdusesrepo :: Bool,
@@ -121,7 +110,7 @@ notBareRepo a = do
 
 {- These functions find appropriate files or other things based on a
    user's parameters, and run a specified action on them. -}
-withFilesInGit :: CommandSeekStrings
+withFilesInGit :: (String -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
@@ -138,13 +127,13 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 		go (file, v) = do
 			let numcopies = readMaybe v
 			a file numcopies
-withBackendFilesInGit :: CommandSeekBackendFiles
+withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	files' <- filterFiles files
 	backendPairs a files'
-withFilesMissing :: CommandSeekStrings
+withFilesMissing :: (String -> CommandStart) -> CommandSeek
 withFilesMissing a params = do
 	files <- liftIO $ filterM missing params
 	liftM (map a) $ filterFiles files
@@ -152,27 +141,27 @@ withFilesMissing a params = do
 		missing f = do
 			e <- doesFileExist f
 			return $ not e
-withFilesNotInGit :: CommandSeekBackendFiles
+withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
 	force <- Annex.getState Annex.force
 	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
 	newfiles' <- filterFiles newfiles
 	backendPairs a newfiles'
-withWords :: CommandSeekWords
+withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
-withStrings :: CommandSeekStrings
+withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
-withFilesToBeCommitted :: CommandSeekStrings
+withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
 	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 	liftM (map a) $ filterFiles tocommit
-withFilesUnlocked :: CommandSeekBackendFiles
+withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
-withFilesUnlockedToBeCommitted :: CommandSeekBackendFiles
+withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
-withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> CommandSeekBackendFiles
+withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	repo <- Annex.gitRepo
@@ -181,15 +170,15 @@ withFilesUnlocked' typechanged a params = do
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
 	unlockedfiles' <- filterFiles unlockedfiles
 	backendPairs a unlockedfiles'
-withKeys :: CommandSeekKeys
+withKeys :: (Key -> CommandStart) -> CommandSeek
 withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
-withNothing :: CommandSeekNothing
+withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
-backendPairs :: CommandSeekBackendFiles
+backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a files = map a <$> Backend.chooseBackends files
 
 {- Filter out files those matching the exclude glob pattern,
diff --git a/Command/Add.hs b/Command/Add.hs
index f6dfd2da43..c6ab4d0ad7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -38,14 +38,14 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
 {- The add subcommand annexes a file, storing it in a backend, and then
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
-start :: CommandStartBackendFile
-start pair@(file, _) = notAnnexed file $ do
+start :: BackendFile -> CommandStart
+start p@(file, _) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if isSymbolicLink s || not (isRegularFile s)
 		then stop
 		else do
 			showStart "add" file
-			next $ perform pair
+			next $ perform p
 
 perform :: BackendFile -> CommandPerform
 perform (file, backend) = do
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index c78569ffc8..1fae358b2d 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -34,7 +34,7 @@ command = [repoCommand "addurl" (paramRepeating $ paramUrl) seek
 seek :: [CommandSeek]
 seek = [withStrings start]
 
-start :: CommandStartString
+start :: String -> CommandStart
 start s = do
 	let u = parseURI s
 	case u of
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 1b1bb3c34b..3de26c8925 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -20,7 +20,7 @@ command = [repoCommand "configlist" paramNothing seek
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	g <- Annex.gitRepo
 	u <- getUUID g
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 453e4ebafc..8d2f9071b0 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -19,7 +19,7 @@ command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let (name, description) =
 		case ws of
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 16a3e35d64..b9938585e9 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -21,7 +21,7 @@ command = [repoCommand "dropkey" (paramRepeating paramKey) seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	present <- inAnnex key
 	force <- Annex.getState Annex.force
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 4ad2aa85bb..90fea050e8 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -41,7 +41,7 @@ withUnusedMaps params = do
 	unusedtmp <- readUnusedLog "tmp"
 	return $ map (start (unused, unusedbad, unusedtmp)) params
 
-start :: (UnusedMap, UnusedMap, UnusedMap) -> CommandStartString
+start :: (UnusedMap, UnusedMap, UnusedMap) -> FilePath -> CommandStart
 start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
 	[ (unused, perform)
 	, (unusedbad, performOther gitAnnexBadLocation)
diff --git a/Command/Find.hs b/Command/Find.hs
index 0ff68b7ed1..51849f6b85 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -20,7 +20,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 {- Output a list of files. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	whenM (inAnnex key) $ liftIO $ putStrLn file
 	stop
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 0044052e9c..481da52f2b 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -26,7 +26,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 {- Fixes the symlink to an annexed file. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	link <- calcGitLink file key
 	l <- liftIO $ readSymbolicLink file
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 89c3f4e912..9ff126a45e 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -27,7 +27,7 @@ command = [repoCommand "fromkey" paramPath seek
 seek :: [CommandSeek]
 seek = [withFilesMissing start]
 
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = notBareRepo $ do
 	key <- cmdlineKey
 	inbackend <- inAnnex key
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 24f7162ace..713492c2fe 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,6 +12,7 @@ import System.Exit
 
 import Command
 import Content
+import Types
 
 command :: [Command]
 command = [repoCommand "inannex" (paramRepeating paramKey) seek
@@ -20,7 +21,7 @@ command = [repoCommand "inannex" (paramRepeating paramKey) seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	present <- inAnnex key
 	if present
diff --git a/Command/Init.hs b/Command/Init.hs
index 6ba7df6829..2351763a95 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -20,7 +20,7 @@ command = [standaloneCommand "init" paramDesc seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	showStart "init" description
 	next $ perform description
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 9859308e56..671f945d22 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -30,7 +30,7 @@ command = [repoCommand "initremote"
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	when (null ws) needname
 
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 07721e9370..1c9a747f43 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -22,7 +22,7 @@ seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
 {- Undo unlock -}
-start :: CommandStartBackendFile
+start :: BackendFile -> CommandStart
 start (file, _) = do
 	showStart "lock" file
 	next $ perform file
diff --git a/Command/Map.hs b/Command/Map.hs
index ef8e04d909..7e23da774d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -34,7 +34,7 @@ command = [repoCommand "map" paramNothing seek "generate map of repositories"]
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	g <- Annex.gitRepo
 	rs <- spider g
diff --git a/Command/Merge.hs b/Command/Merge.hs
index faaef906b5..832cde5127 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -18,7 +18,7 @@ command = [repoCommand "merge" paramNothing seek
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	showStart "merge" "."
 	next perform
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index ec570acb7b..69fe61e1d0 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,7 +31,7 @@ command = [repoCommand "migrate" paramPaths seek
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
 
-start :: CommandStartBackendFile
+start :: BackendFile -> CommandStart
 start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
diff --git a/Command/Move.hs b/Command/Move.hs
index f4310a2b88..15dae39385 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -31,7 +31,7 @@ seek = [withFilesInGit $ start True]
  -
  - This only operates on the cached file content; it does not involve
  - moving data in the key-value backend. -}
-start :: Bool -> CommandStartString
+start :: Bool -> FilePath -> CommandStart
 start move file = do
 	noAuto
 	to <- Annex.getState Annex.toremote
@@ -74,7 +74,7 @@ remoteHasKey remote key present	= do
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Remote.Remote Annex -> Bool -> CommandStartString
+toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 toStart dest move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
@@ -124,7 +124,7 @@ toCleanup dest move key = do
  - If the current repository already has the content, it is still removed
  - from the remote.
  -}
-fromStart :: Remote.Remote Annex -> Bool -> CommandStartString
+fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 fromStart src move file = isAnnexed file $ \(key, _) -> do
 	g <- Annex.gitRepo
 	u <- getUUID g
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 6884a4787c..bcc1c943ee 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -20,8 +20,8 @@ seek :: [CommandSeek]
 seek = [withFilesToBeCommitted Command.Fix.start,
 	withFilesUnlockedToBeCommitted start]
 
-start :: CommandStartBackendFile
-start pair = next $ perform pair
+start :: BackendFile -> CommandStart
+start p = next $ perform p
 
 perform :: BackendFile -> CommandPerform
 perform pair@(file, _) = do
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index be6163558f..33792e5b6e 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -15,6 +15,7 @@ import CmdLine
 import Content
 import Utility.RsyncFile
 import Utility.Conditional
+import Types
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
@@ -23,7 +24,7 @@ command = [repoCommand "recvkey" paramKey seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	whenM (inAnnex key) $ error "key is already present in annex"
 	
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index b467861bf9..3b12bb747d 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -20,7 +20,7 @@ command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let name = unwords ws
 	showStart "semitrust" name
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index f676ae947a..98d2573380 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -17,6 +17,7 @@ import Content
 import Utility.RsyncFile
 import Utility.Conditional
 import Messages
+import Types
 
 command :: [Command]
 command = [repoCommand "sendkey" paramKey seek
@@ -25,7 +26,7 @@ command = [repoCommand "sendkey" paramKey seek
 seek :: [CommandSeek]
 seek = [withKeys start]
 
-start :: CommandStartKey
+start :: Key -> CommandStart
 start key = do
 	g <- Annex.gitRepo
 	let file = gitAnnexLocation g key
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 2f6f9ea9ee..c03c5d0445 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -23,7 +23,7 @@ seek :: [CommandSeek]
 seek = [withStrings start]
 
 {- Sets cached content for a key. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = do
 	showStart "setkey" file
 	next $ perform file
diff --git a/Command/Status.hs b/Command/Status.hs
index 5c82744b10..76659a75e1 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -74,7 +74,7 @@ slowstats =
 	, backend_usage
 	]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	fast <- Annex.getState Annex.fast
 	let todo = if fast then faststats else faststats ++ slowstats
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 41eb17ccdd..5e25b519bc 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -20,7 +20,7 @@ command = [repoCommand "trust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let name = unwords ws
 	showStart "trust" name
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 748da4066c..3dedd007ef 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -33,7 +33,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
 {- The unannex subcommand undoes an add. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	ishere <- inAnnex key
 	if ishere
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 4c70ec80c4..ce12665424 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,7 +28,7 @@ command = [repoCommand "uninit" paramPaths seek
 seek :: [CommandSeek]
 seek = [withFilesInGit startUnannex, withNothing start]
 
-startUnannex :: CommandStartString
+startUnannex :: FilePath -> CommandStart
 startUnannex file = do
 	-- Force fast mode before running unannex. This way, if multiple
 	-- files link to a key, it will be left in the annex and hardlinked
@@ -36,7 +36,7 @@ startUnannex file = do
 	Annex.changeState $ \s -> s { Annex.fast = True }
 	Command.Unannex.start file
 
-start :: CommandStartNothing
+start :: CommandStart
 start = next perform
 
 perform :: CommandPerform
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index ba6d023874..5817e8f221 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -31,7 +31,7 @@ seek = [withFilesInGit start]
 
 {- The unlock subcommand replaces the symlink with a copy of the file's
  - content. -}
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "unlock" file
 	next $ perform file key
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index ea23208006..9f7e521987 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -20,7 +20,7 @@ command = [repoCommand "untrust" (paramRepeating paramRemote) seek
 seek :: [CommandSeek]
 seek = [withWords start]
 
-start :: CommandStartWords
+start :: [String] -> CommandStart
 start ws = do
 	let name = unwords ws
 	showStart "untrust" name
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 960b2e1dfb..535b9b33e8 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -37,7 +37,7 @@ seek :: [CommandSeek]
 seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
-start :: CommandStartNothing
+start :: CommandStart
 start = notBareRepo $ do
 	from <- Annex.getState Annex.fromremote
 	let (name, action) = case from of
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index b79b13cd3c..5d9ed92fae 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -19,7 +19,7 @@ command = [standaloneCommand "upgrade" paramNothing seek
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	showStart "upgrade" "."
 	r <- upgrade
diff --git a/Command/Version.hs b/Command/Version.hs
index 1ff829a22a..af547949c6 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -21,7 +21,7 @@ command = [standaloneCommand "version" paramNothing seek "show version info"]
 seek :: [CommandSeek]
 seek = [withNothing start]
 
-start :: CommandStartNothing
+start :: CommandStart
 start = do
 	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
 	v <- getVersion
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 7d0ab188c0..a414428f72 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -23,7 +23,7 @@ command = [repoCommand "whereis" paramPaths seek
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
-start :: CommandStartString
+start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
 	showStart "whereis" file
 	next $ perform key

From 5ff04bf2afcc62d1762adbc1c2a0374952328c0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 16:57:02 -0400
Subject: [PATCH 2211/8313] tweak

---
 Backend.hs           | 9 ++++++---
 Command.hs           | 5 ++---
 Command/Add.hs       | 5 +++--
 Command/AddUrl.hs    | 2 +-
 Command/Lock.hs      | 3 ++-
 Command/Migrate.hs   | 3 ++-
 Command/PreCommit.hs | 3 ++-
 7 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 75327de80b..0c9ea8d0b7 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -6,6 +6,7 @@
  -}
 
 module Backend (
+	BackendFile,
 	list,
 	orderedList,
 	genKey,
@@ -101,20 +102,22 @@ lookupFile file = do
 				skip = "skipping " ++ file ++ 
 					" (unknown backend " ++ bname ++ ")"
 
+type BackendFile = (Maybe (Backend Annex), FilePath)
+
 {- Looks up the backends that should be used for each file in a list.
  - That can be configured on a per-file basis in the gitattributes file.
  -}
-chooseBackends :: [FilePath] -> Annex [(FilePath, Maybe (Backend Annex))]
+chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = do
 	g <- Annex.gitRepo
 	forced <- Annex.getState Annex.forcebackend
 	if forced /= Nothing
 		then do
 			l <- orderedList
-			return $ map (\f -> (f, Just $ head l)) fs
+			return $ map (\f -> (Just $ head l, f)) fs
 		else do
 			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-			return $ map (\(f,b) -> (f, maybeLookupBackendName b)) pairs
+			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
diff --git a/Command.hs b/Command.hs
index 6bd451a7e9..08087a5a5d 100644
--- a/Command.hs
+++ b/Command.hs
@@ -1,4 +1,4 @@
-{- git-annex commands
+{- git-annex command infrastructure
  -
  - Copyright 2010 Joey Hess 
  -
@@ -29,6 +29,7 @@ import Types.Key
 import Trust
 import LocationLog
 import Config
+import Backend
 
 {- A command runs in four stages.
  -
@@ -49,8 +50,6 @@ type CommandPerform = Annex (Maybe CommandCleanup)
  -    returns the overall success/fail of the command. -}
 type CommandCleanup = Annex Bool
 
-type BackendFile = (FilePath, Maybe (Backend Annex))
-
 data Command = Command {
 	cmdusesrepo :: Bool,
 	cmdname :: String,
diff --git a/Command/Add.hs b/Command/Add.hs
index c6ab4d0ad7..4b2ef24cd9 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -27,6 +27,7 @@ import Utility.Conditional
 import Utility.Touch
 import Utility.SafeCommand
 import Locations
+import Backend
 
 command :: [Command]
 command = [repoCommand "add" paramPaths seek "add files to annex"]
@@ -39,7 +40,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
 start :: BackendFile -> CommandStart
-start p@(file, _) = notAnnexed file $ do
+start p@(_, file) = notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if isSymbolicLink s || not (isRegularFile s)
 		then stop
@@ -48,7 +49,7 @@ start p@(file, _) = notAnnexed file $ do
 			next $ perform p
 
 perform :: BackendFile -> CommandPerform
-perform (file, backend) = do
+perform (backend, file) = do
 	k <- Backend.genKey file backend
 	case k of
 		Nothing -> stop
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 1fae358b2d..d9fcc17e2b 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -59,7 +59,7 @@ download url file = do
 	ok <- Url.download url tmp
 	if ok
 		then do
-			[(_, backend)] <- Backend.chooseBackends [file]
+			[(backend, _)] <- Backend.chooseBackends [file]
 			k <- Backend.genKey tmp backend
 			case k of
 				Nothing -> stop
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 1c9a747f43..04d1bb94d6 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -14,6 +14,7 @@ import Command
 import Messages
 import qualified AnnexQueue
 import Utility.SafeCommand
+import Backend
 	
 command :: [Command]
 command = [repoCommand "lock" paramPaths seek "undo unlock command"]
@@ -23,7 +24,7 @@ seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
 {- Undo unlock -}
 start :: BackendFile -> CommandStart
-start (file, _) = do
+start (_, file) = do
 	showStart "lock" file
 	next $ perform file
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 69fe61e1d0..2be9108512 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -23,6 +23,7 @@ import Content
 import Messages
 import Utility.Conditional
 import qualified Command.Add
+import Backend
 
 command :: [Command]
 command = [repoCommand "migrate" paramPaths seek
@@ -32,7 +33,7 @@ seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
 
 start :: BackendFile -> CommandStart
-start (file, b) = isAnnexed file $ \(key, oldbackend) -> do
+start (b, file) = isAnnexed file $ \(key, oldbackend) -> do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
 	if (newbackend /= oldbackend || upgradableKey key) && exists
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index bcc1c943ee..b6323e2b79 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -10,6 +10,7 @@ module Command.PreCommit where
 import Command
 import qualified Command.Add
 import qualified Command.Fix
+import Backend
 
 command :: [Command]
 command = [repoCommand "pre-commit" paramPaths seek "run by git pre-commit hook"]
@@ -24,7 +25,7 @@ start :: BackendFile -> CommandStart
 start p = next $ perform p
 
 perform :: BackendFile -> CommandPerform
-perform pair@(file, _) = do
+perform pair@(_, file) = do
 	ok <- doCommand $ Command.Add.start pair
 	if ok
 		then next $ return True

From cbd9ade0757788506d48578466802d037f4791fb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 22:09:19 -0400
Subject: [PATCH 2212/8313] remove now unnecessary git commits

---
 doc/git-annex.mdwn | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0d3aebe2be..a5163a4b33 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -42,15 +42,12 @@ content from the key-value store.
 	# git remote add usbdrive /media/usb
 	# git annex get video/hackity_hack_and_kaxxt.mov
 	get video/hackity_hack_and_kaxxt.mov (from usbdrive...) ok
-	# git commit -a -m "got a video I want to rewatch on the plane"
 	
 	# git annex add iso
 	add iso/Debian_5.0.iso ok
-	# git commit -a -m "saving Debian CD for later"
 	
 	# git annex drop iso/Debian_4.0.iso
 	drop iso/Debian_4.0.iso ok
-	# git commit -a -m "freed up space"
 	
 	# git annex move iso --to=usbdrive
 	move iso/Debian_5.0.iso (moving to usbdrive...) ok
@@ -63,8 +60,6 @@ files in the directory. If no path is specified, most git-annex commands
 default to acting on all relevant files in the current directory (and
 subdirectories).
 
-Many git-annex commands will stage changes for later `git commit` by you.
-
 * add [path ...]
 
   Adds files in the path to the annex. Files that are already checked into

From 73769190b6f2df7e71e943af860ec904a1336367 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Sep 2011 22:22:43 -0400
Subject: [PATCH 2213/8313] grouped commands into related sections

---
 doc/git-annex.mdwn | 123 ++++++++++++++++++++++++---------------------
 1 file changed, 67 insertions(+), 56 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a5163a4b33..d587f763ca 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -52,11 +52,11 @@ content from the key-value store.
 	# git annex move iso --to=usbdrive
 	move iso/Debian_5.0.iso (moving to usbdrive...) ok
 
-# COMMANDS
+# COMMONLY USED COMMANDS
 
 Like many git commands, git-annex can be passed a path that 
 is either a file or a directory. In the latter case it acts on all relevant
-files in the directory. If no path is specified, most git-annex commands
+files in the directory. When no path is specified, most git-annex commands
 default to acting on all relevant files in the current directory (and
 subdirectories).
 
@@ -115,6 +115,14 @@ subdirectories).
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
+* addurl [url ...]
+
+  Downloads each url to a file, which is added to the annex.
+
+  To avoid immediately downloading the url, specify --fast
+
+# REPOSITORY SETUP COMMANDS
+
 * init [description]
 
   Until a repository (or one of its remotes) has been initialized,
@@ -144,6 +152,24 @@ subdirectories).
 
 	initremote mys3 type=S3 encryption=none datacenter=EU
 
+* trust [repository ...]
+
+  Records that a repository is trusted to not unexpectedly lose
+  content. Use with care.
+
+  To trust the current repository, use "."
+
+* untrust [repository ...]
+
+  Records that a repository is not trusted and could lose content
+  at any time.
+
+* semitrust [repository ...]
+
+  Returns a repository to the default semi trusted state.
+
+# REPOSITORY MAINTENANCE COMMANDS
+
 * fsck [path ...]
 
   With no parameters, this command checks the whole annex for consistency,
@@ -170,6 +196,29 @@ subdirectories).
 
   To drop the data from a remote, specify --from.
 
+* merge
+
+  Automatically merges any changes from remotes into the git-annex branch.
+  While git-annex mostly handles keeping the git-annex branch merged
+  automatically, if you find you are unable to push the git-annex branch
+  due non-fast-forward, this will fix it.
+
+* fix [path ...]
+
+  Fixes up symlinks that have become broken to again point to annexed content.
+  This is useful to run if you have been moving the symlinks around,
+  but is done automatically when committing a change with git too.
+
+* upgrade
+
+  Upgrades the repository to current layout.
+
+# QUERY COMMANDS
+
+* version
+
+  Shows the version of git-annex, as well as repository version information.
+
 * find [path ...]
 
   Outputs a list of annexed files whose content is currently present.
@@ -182,13 +231,6 @@ subdirectories).
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
-* merge
-
-  Automatically merges any changes from remotes into the git-annex branch.
-  While git-annex mostly handles keeping the git-annex branch merged
-  automatically, if you find you are unable to push the git-annex branch
-  due non-fast-forward, this will fix it.
-
 * status
 
   Displays some statistics and other information, including how much data
@@ -199,19 +241,6 @@ subdirectories).
   information you wanted to see. Or, use --fast to only display
   the first, fast(ish) statistics.
 
-* migrate [path ...]
-
-  Changes the specified annexed files to use the default key-value backend
-  (or the one specified with --backend). Only files whose content
-  is currently available are migrated.
-
-  Note that the content is also still available using the old key after
-  migration. Use `git annex unused` to find and remove the old key.
-
-  Normally, nothing will be done to files already using the new backend.
-  However, if a backend changes the information it uses to construct a key,
-  this can also be used to migrate files to use the new key format.
-
 * map
 
   Helps you keep track of your repositories, and the connections between them,
@@ -229,6 +258,21 @@ subdirectories).
   Note that this subcommand can be used to graph any git repository; it
   is not limited to git-annex repositories.
 
+# UTILITY COMMANDS
+
+* migrate [path ...]
+
+  Changes the specified annexed files to use the default key-value backend
+  (or the one specified with --backend). Only files whose content
+  is currently available are migrated.
+
+  Note that the content is also still available using the old key after
+  migration. Use `git annex unused` to find and remove the old key.
+
+  Normally, nothing will be done to files already using the new backend.
+  However, if a backend changes the information it uses to construct a key,
+  this can also be used to migrate files to use the new key format.
+
 * unannex [path ...]
 
   Use this to undo an accidental `git annex add` command. You can use
@@ -248,10 +292,7 @@ subdirectories).
   repository, and remove all of git-annex's other data, leaving you with a
   git repository plus the previously annexed files.
 
-* fix [path ...]
-
-  Fixes up symlinks that have become broken to again point to annexed content.
-  This is useful to run if you have been moving the symlinks around.
+# PLUMBING COMMANDS
 
 * pre-commit [path ...]
 
@@ -262,28 +303,6 @@ subdirectories).
   This is meant to be called from git's pre-commit hook. `git annex init`
   automatically creates a pre-commit hook using this.
 
-* trust [repository ...]
-
-  Records that a repository is trusted to not unexpectedly lose
-  content. Use with care.
-
-  To trust the current repository, use "."
-
-* untrust [repository ...]
-
-  Records that a repository is not trusted and could lose content
-  at any time.
-
-* semitrust [repository ...]
-
-  Returns a repository to the default semi trusted state.
-
-* addurl [url ...]
-
-  Downloads each url to a file, which is added to the annex.
-
-  To avoid immediately downloading the url, specify --fast
-
 * fromkey file
 
   This plumbing-level command can be used to manually set up a file
@@ -310,14 +329,6 @@ subdirectories).
 
 	git annex setkey --key=WORM-s3-m1287765018--file /tmp/file
 
-* upgrade
-
-  Upgrades the repository to current layout.
-
-* version
-
-  Shows the version of git-annex, as well as repository version information.
-
 # OPTIONS
 
 * --force

From 3e73de4054338955ce0f923923a526e8225225aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Sep 2011 09:21:09 -0400
Subject: [PATCH 2214/8313] releasing version 3.20110915

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e940989da7..dff945c287 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110907) UNRELEASED; urgency=low
+git-annex (3.20110915) unstable; urgency=low
 
   * whereis: Show untrusted locations separately and do not include in
     location count.
@@ -9,7 +9,7 @@ git-annex (3.20110907) UNRELEASED; urgency=low
     to get/drop content as needed to work toward the configured numcopies.
   * bugfix: drop and fsck did not honor --exclude
 
- -- Joey Hess   Tue, 06 Sep 2011 16:59:15 -0400
+ -- Joey Hess   Thu, 15 Sep 2011 22:25:46 -0400
 
 git-annex (3.20110906) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 34c8e76222..a518ca8cb9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110906
+Version: 3.20110915
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From b73d8225d5c95241eedb5969846f859a7f69db65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Sep 2011 09:21:21 -0400
Subject: [PATCH 2215/8313] add news item for git-annex 3.20110915

---
 doc/news/version_3.20110719.mdwn |  7 -------
 doc/news/version_3.20110915.mdwn | 10 ++++++++++
 2 files changed, 10 insertions(+), 7 deletions(-)
 delete mode 100644 doc/news/version_3.20110719.mdwn
 create mode 100644 doc/news/version_3.20110915.mdwn

diff --git a/doc/news/version_3.20110719.mdwn b/doc/news/version_3.20110719.mdwn
deleted file mode 100644
index 5beae32270..0000000000
--- a/doc/news/version_3.20110719.mdwn
+++ /dev/null
@@ -1,7 +0,0 @@
-git-annex 3.20110719 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * add: Be even more robust to avoid ever leaving the file seemingly deleted.
-     Closes: #[634233](http://bugs.debian.org/634233)
-   * Bugfix: Make add ../ work.
-   * Support the standard git -c name=value
-   * unannex: Clean up use of git commit -a."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110915.mdwn b/doc/news/version_3.20110915.mdwn
new file mode 100644
index 0000000000..77a97242d9
--- /dev/null
+++ b/doc/news/version_3.20110915.mdwn
@@ -0,0 +1,10 @@
+git-annex 3.20110915 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * whereis: Show untrusted locations separately and do not include in
+     location count.
+   * Fix build without S3.
+   * addurl: Always use whole url as destination filename, rather than
+     only its file component.
+   * get, drop, copy: Added --auto option, which decides whether
+     to get/drop content as needed to work toward the configured numcopies.
+   * bugfix: drop and fsck did not honor --exclude"""]]
\ No newline at end of file

From 756338fb2e920be86f1d25fbe8be7e9bc88d2491 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Sep 2011 18:37:38 -0400
Subject: [PATCH 2216/8313] idea

---
 doc/todo/exclude_files_on_a_given_remote.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/todo/exclude_files_on_a_given_remote.mdwn

diff --git a/doc/todo/exclude_files_on_a_given_remote.mdwn b/doc/todo/exclude_files_on_a_given_remote.mdwn
new file mode 100644
index 0000000000..fbb3f90bef
--- /dev/null
+++ b/doc/todo/exclude_files_on_a_given_remote.mdwn
@@ -0,0 +1,16 @@
+Say I have some files on remote A. But I'm away from it, and transferring
+files from B to C. I'd like to avoid transferring any files I already have
+on A. 
+
+Something like:
+
+	git annex copy --to C --exclude-on A
+
+This would not contact A, just use its cached location log info.
+
+I suppose I might also sometime want to only act on files that are
+thought/known to be on A.
+
+	git annex drop --only-on A
+
+--[[Joey]] 

From 6ebaa85bdb8d13a752d7994e0b35620a10744eaa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 16:02:12 -0400
Subject: [PATCH 2217/8313] a generic matcher

---
 Matcher.hs | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 Matcher.hs

diff --git a/Matcher.hs b/Matcher.hs
new file mode 100644
index 0000000000..32da9f95df
--- /dev/null
+++ b/Matcher.hs
@@ -0,0 +1,88 @@
+{- A generic matcher.
+ -
+ - Can be used to check if a user-supplied condition,
+ - like "foo and ( bar or not baz )" matches. The condition must already
+ - be tokenized, and can contain arbitrary operations.
+ -
+ - If operations are not separated by and/or, they are defaulted to being
+ - anded together, so "foo bar baz" all must match.
+ -
+ - Is forgiving about misplaced closing parens, so "foo and (bar or baz"
+ - will be handled, as will "foo and ( bar or baz ) )"
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Matcher (
+	Operation(..),
+	Token(..),
+	toMatcher,
+	match,
+	runMatch	
+) where
+
+import Control.Monad
+
+{- An Operation is a command and some parameters. -}
+data Operation = Operation String [String]
+	deriving (Show, Eq)
+
+{- A Token can either be a single word, or an Operation. -}
+data Token = Token String | TokenOp Operation
+	deriving (Show, Eq)
+
+data Matcher = Any
+	| And Matcher Matcher
+	| Or Matcher Matcher
+	| Not Matcher
+	| Op Operation
+	deriving (Show, Eq)
+
+{- Converts a list of Tokens into a Matcher. -}
+toMatcher :: [Token] -> Matcher
+toMatcher ts = toMatcher' Any ts
+toMatcher' :: Matcher -> [Token] -> Matcher
+toMatcher' m [] = m
+toMatcher' m ts = toMatcher' m' rest
+	where
+		(m', rest) = consume m ts
+
+{- Consumes one or more tokens, constructs a new Matcher,
+ - and returns unconsumed tokens. -}
+consume :: Matcher -> [Token] -> (Matcher, [Token])
+consume m [] = (m, [])
+consume m ((TokenOp o):ts) = (m `And` Op o, ts)
+consume m ((Token t):ts)
+	| t == "and" = cont $ m `And` next
+	| t == "or" = cont $ m `Or` next
+	| t == "not" = cont $ m `And` (Not next)
+	| t == "(" = let (n, r) = consume next rest in (m `And` n, r)
+	| t == ")" = (m, ts)
+	| otherwise = (m, ts) -- ignore unknown token
+	where
+		(next, rest) = consume Any ts
+		cont v = (v, rest)
+
+{- Checks if a Matcher matches, using a supplied function to check
+ - the value of Operations. -}
+match :: (Operation -> Bool) -> Matcher -> Bool
+match a = go
+	where
+		go Any = True
+		go (And m1 m2) = go m1 && go m2
+		go (Or m1 m2) = go m1 || go m2
+		go (Not m1) = not (go m1)
+		go (Op v) = a v
+
+{- Runs a Matcher in an arbitrary monadic contex, using a supplied
+ - action to evaluate Operations. -}
+runMatch :: Monad m => (Operation -> m Bool) -> Matcher -> m Bool
+runMatch a = go
+	where
+		go Any = return True
+		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
+		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
+		go (Not m1) = liftM not (go m1)
+		go (Op v) = a v

From 0e499e67be1ba32d30eb0a65537d2c6a0f4d6e05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 16:32:39 -0400
Subject: [PATCH 2218/8313] convert to parameterized types, so the Operation
 can be any type the caller needs

Especially handy for running a match monadically.
---
 Matcher.hs | 52 +++++++++++++++++++++++-----------------------------
 1 file changed, 23 insertions(+), 29 deletions(-)

diff --git a/Matcher.hs b/Matcher.hs
index 32da9f95df..91e3bf6f33 100644
--- a/Matcher.hs
+++ b/Matcher.hs
@@ -16,44 +16,39 @@
  -}
 
 module Matcher (
-	Operation(..),
 	Token(..),
-	toMatcher,
+	generate,
 	match,
-	runMatch	
+	run
 ) where
 
 import Control.Monad
 
-{- An Operation is a command and some parameters. -}
-data Operation = Operation String [String]
+{- A Token can either be a single word, or an Operation of an arbitrary type. -}
+data Token op = Token String | Operation op
 	deriving (Show, Eq)
 
-{- A Token can either be a single word, or an Operation. -}
-data Token = Token String | TokenOp Operation
-	deriving (Show, Eq)
-
-data Matcher = Any
-	| And Matcher Matcher
-	| Or Matcher Matcher
-	| Not Matcher
-	| Op Operation
+data Matcher op = Any
+	| And (Matcher op) (Matcher op)
+	| Or (Matcher op) (Matcher op)
+	| Not (Matcher op)
+	| Op op
 	deriving (Show, Eq)
 
 {- Converts a list of Tokens into a Matcher. -}
-toMatcher :: [Token] -> Matcher
-toMatcher ts = toMatcher' Any ts
-toMatcher' :: Matcher -> [Token] -> Matcher
-toMatcher' m [] = m
-toMatcher' m ts = toMatcher' m' rest
+generate :: [Token op] -> Matcher op
+generate ts = generate' Any ts
+generate' :: Matcher op -> [Token op] -> Matcher op
+generate' m [] = m
+generate' m ts = generate' m' rest
 	where
 		(m', rest) = consume m ts
 
-{- Consumes one or more tokens, constructs a new Matcher,
- - and returns unconsumed tokens. -}
-consume :: Matcher -> [Token] -> (Matcher, [Token])
+{- Consumes one or more Tokens, constructs a new Matcher,
+ - and returns unconsumed Tokens. -}
+consume :: Matcher op -> [Token op] -> (Matcher op, [Token op])
 consume m [] = (m, [])
-consume m ((TokenOp o):ts) = (m `And` Op o, ts)
+consume m ((Operation o):ts) = (m `And` Op o, ts)
 consume m ((Token t):ts)
 	| t == "and" = cont $ m `And` next
 	| t == "or" = cont $ m `Or` next
@@ -67,7 +62,7 @@ consume m ((Token t):ts)
 
 {- Checks if a Matcher matches, using a supplied function to check
  - the value of Operations. -}
-match :: (Operation -> Bool) -> Matcher -> Bool
+match :: (op -> Bool) -> Matcher op -> Bool
 match a = go
 	where
 		go Any = True
@@ -76,13 +71,12 @@ match a = go
 		go (Not m1) = not (go m1)
 		go (Op v) = a v
 
-{- Runs a Matcher in an arbitrary monadic contex, using a supplied
- - action to evaluate Operations. -}
-runMatch :: Monad m => (Operation -> m Bool) -> Matcher -> m Bool
-runMatch a = go
+{- Runs a monadic Matcher, where Operations are actions in the monad. -}
+run :: Monad m => Matcher (m Bool) -> m Bool
+run = go
 	where
 		go Any = return True
 		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
 		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
 		go (Not m1) = liftM not (go m1)
-		go (Op v) = a v
+		go (Op o) = o -- run o

From 3e15187ac1082bb086c79109c8b35dd8c20017ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 16:36:30 -0400
Subject: [PATCH 2219/8313] move to Utility

---
 Matcher.hs => Utility/Matcher.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename Matcher.hs => Utility/Matcher.hs (98%)

diff --git a/Matcher.hs b/Utility/Matcher.hs
similarity index 98%
rename from Matcher.hs
rename to Utility/Matcher.hs
index 91e3bf6f33..0f589ec2c0 100644
--- a/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -15,7 +15,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Matcher (
+module Utility.Matcher (
 	Token(..),
 	generate,
 	match,

From 38c0f3eaf86b67d584d4ff30ab15ec2c725a7fad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 17:47:24 -0400
Subject: [PATCH 2220/8313] add a value to match against to match and matchM

---
 Utility/Matcher.hs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 0f589ec2c0..08534fc304 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -17,9 +17,10 @@
 
 module Utility.Matcher (
 	Token(..),
+	Matcher,
 	generate,
 	match,
-	run
+	matchM
 ) where
 
 import Control.Monad
@@ -62,21 +63,21 @@ consume m ((Token t):ts)
 
 {- Checks if a Matcher matches, using a supplied function to check
  - the value of Operations. -}
-match :: (op -> Bool) -> Matcher op -> Bool
-match a = go
+match :: (op -> v -> Bool) -> Matcher op -> v -> Bool
+match a m v = go m
 	where
 		go Any = True
 		go (And m1 m2) = go m1 && go m2
 		go (Or m1 m2) = go m1 || go m2
 		go (Not m1) = not (go m1)
-		go (Op v) = a v
+		go (Op o) = a o v
 
 {- Runs a monadic Matcher, where Operations are actions in the monad. -}
-run :: Monad m => Matcher (m Bool) -> m Bool
-run = go
+matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool
+matchM m v = go m
 	where
 		go Any = return True
 		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
 		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
 		go (Not m1) = liftM not (go m1)
-		go (Op o) = o -- run o
+		go (Op o) = o v

From 8a5a92480b9dcf691af1e8c4849cb71c4158b845 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 17:47:49 -0400
Subject: [PATCH 2221/8313] refactor --exclude to use Utility.Matcher

This should change no behavior, but opens the poissibility to use the
matcher for other sorts of limits on which files git-annex processes.
---
 Annex.hs    |  5 +++--
 Command.hs  | 21 +------------------
 GitAnnex.hs |  4 ++--
 Limit.hs    | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 24 deletions(-)
 create mode 100644 Limit.hs

diff --git a/Annex.hs b/Annex.hs
index 2c8ea1d61d..ad65e05ddf 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -31,6 +31,7 @@ import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
+import qualified Utility.Matcher
 
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
@@ -59,7 +60,7 @@ data AnnexState = AnnexState
 	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
-	, exclude :: [String]
+	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
 	, forcetrust :: [(UUID, TrustLevel)]
 	, trustmap :: Maybe TrustMap
 	, cipher :: Maybe Cipher
@@ -83,7 +84,7 @@ newState gitrepo = AnnexState
 	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
-	, exclude = []
+	, limit = Left []
 	, forcetrust = []
 	, trustmap = Nothing
 	, cipher = Nothing
diff --git a/Command.hs b/Command.hs
index 08087a5a5d..fd58ca801a 100644
--- a/Command.hs
+++ b/Command.hs
@@ -12,11 +12,8 @@ import System.Directory
 import System.Posix.Files
 import Control.Monad (filterM, liftM, when)
 import Control.Applicative
-import System.Path.WildMatch
-import Text.Regex.PCRE.Light.Char8
 import Data.List
 import Data.Maybe
-import Data.String.Utils
 
 import Types
 import qualified Backend
@@ -30,6 +27,7 @@ import Trust
 import LocationLog
 import Config
 import Backend
+import Limit
 
 {- A command runs in four stages.
  -
@@ -180,23 +178,6 @@ withNothing _ _ = error "This command takes no parameters."
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a files = map a <$> Backend.chooseBackends files
 
-{- Filter out files those matching the exclude glob pattern,
- - if it was specified. -}
-filterFiles :: [FilePath] -> Annex [FilePath]
-filterFiles l = do
-	exclude <- Annex.getState Annex.exclude
-	if null exclude
-		then return l
-		else return $ filter (notExcluded $ wildsRegex exclude) l
-	where
-		notExcluded r f = isNothing $ match r f []
-
-wildsRegex :: [String] -> Regex
-wildsRegex ws = compile regex []
-	where
-		regex = "^(" ++ alternatives ++ ")"
-		alternatives = join "|" $ map wildToRegex ws
-
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 6f4e5d4921..bb0f85119c 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -19,6 +19,7 @@ import Types
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
+import qualified Limit
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -97,7 +98,7 @@ options = commonOptions ++
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
-	, Option ['x'] ["exclude"] (ReqArg addexclude paramGlob)
+	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
@@ -113,7 +114,6 @@ options = commonOptions ++
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
-		addexclude v = Annex.changeState $ \s -> s { Annex.exclude = v:Annex.exclude s }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setgitconfig :: String -> Annex ()
diff --git a/Limit.hs b/Limit.hs
new file mode 100644
index 0000000000..324baee2e3
--- /dev/null
+++ b/Limit.hs
@@ -0,0 +1,59 @@
+{- user-specified limits on files to act on
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Limit where
+
+import Text.Regex.PCRE.Light.Char8
+import System.Path.WildMatch
+import Control.Monad (filterM)
+import Data.Maybe
+
+import Annex
+import qualified Utility.Matcher
+
+type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
+
+{- Filter out files not matching user-specified limits. -}
+filterFiles :: [FilePath] -> Annex [FilePath]
+filterFiles l = do
+	matcher <- getMatcher
+	filterM (Utility.Matcher.matchM matcher) l
+
+{- Gets a matcher for the user-specified limits. The matcher is cached for
+ - speed; once it's obtained the user-specified limits can't change. -}
+getMatcher :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+getMatcher = do
+	m <- Annex.getState Annex.limit
+	case m of
+		Right r -> return r
+		Left l -> do
+			let matcher = Utility.Matcher.generate (reverse l)
+			Annex.changeState $ \s -> s { Annex.limit = Right matcher }
+			return matcher
+
+{- Adds something to the limit list. -}
+add :: Limit -> Annex ()
+add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
+	where
+		append (Left ls) = Left $ l:ls
+		append _ = error "internal"
+
+{- Adds a new limit. -}
+addl :: (FilePath -> Annex Bool) -> Annex ()
+addl = add . Utility.Matcher.Operation
+
+{- Adds a new token. -}
+addt :: String -> Annex ()
+addt = add . Utility.Matcher.Token
+
+{- Add a limit to skip files that do not match the glob. -}
+exclude :: String -> Annex ()
+exclude glob = addl $ return . notExcluded
+	where
+		notExcluded f = isNothing $ match cregex f []
+		cregex = compile regex []
+		regex = '^':wildToRegex glob

From b9aa944b09e60badb99c65a87f5689e0ab9010e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 18:21:42 -0400
Subject: [PATCH 2222/8313] add --and --or --not -( and -)

I dislike -( and -), but without using a different option parser, can't
easily use bare parens.

--and and --or will become more useful once there are more interesting
limits than --exclude
---
 GitAnnex.hs |  6 +++---
 Limit.hs    | 12 ++++++------
 Options.hs  | 13 +++++++++++++
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index bb0f85119c..68c7a1805f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -98,8 +98,6 @@ options = commonOptions ++
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
-	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
-		"skip files matching the glob pattern"
 	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
 	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
@@ -110,7 +108,9 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	]
+	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
+		"skip files matching the glob pattern"
+	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 324baee2e3..91ea3453cf 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -35,7 +35,7 @@ getMatcher = do
 			Annex.changeState $ \s -> s { Annex.limit = Right matcher }
 			return matcher
 
-{- Adds something to the limit list. -}
+{- Adds something to the limit list, which is built up reversed. -}
 add :: Limit -> Annex ()
 add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
 	where
@@ -43,16 +43,16 @@ add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
 		append _ = error "internal"
 
 {- Adds a new limit. -}
-addl :: (FilePath -> Annex Bool) -> Annex ()
-addl = add . Utility.Matcher.Operation
+addlimit :: (FilePath -> Annex Bool) -> Annex ()
+addlimit = add . Utility.Matcher.Operation
 
 {- Adds a new token. -}
-addt :: String -> Annex ()
-addt = add . Utility.Matcher.Token
+token :: String -> Annex ()
+token = add . Utility.Matcher.Token
 
 {- Add a limit to skip files that do not match the glob. -}
 exclude :: String -> Annex ()
-exclude glob = addl $ return . notExcluded
+exclude glob = addlimit $ return . notExcluded
 	where
 		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
diff --git a/Options.hs b/Options.hs
index ee3ce66203..44d5f3674c 100644
--- a/Options.hs
+++ b/Options.hs
@@ -14,6 +14,7 @@ import Control.Monad.State (liftIO)
 import qualified Annex
 import Types
 import Command
+import Limit
 
 {- Each dashed command-line option results in generation of an action
  - in the Annex monad that performs the necessary setting.
@@ -47,3 +48,15 @@ commonOptions =
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG
+	
+matcherOptions :: [Option]
+matcherOptions =
+	[ longopt "not" "negate next option"
+	, longopt "and" "both previous and next option must match"
+	, longopt "or" "either previous or next option must match"
+	, shortopt "(" "open group of options"
+	, shortopt ")" "close group of options"
+	]
+	where
+		longopt o d = Option [] [o] (NoArg (token o)) d
+		shortopt o d = Option o [] (NoArg (token o)) d

From d78b9f7d546bd4f13349e01777d5dd45fc01b0af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 18:24:10 -0400
Subject: [PATCH 2223/8313] update man page for file matching options

(--in is not yet implemented)
---
 doc/git-annex.mdwn | 55 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 48 insertions(+), 7 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d587f763ca..03277c6a0e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -379,13 +379,6 @@ subdirectories).
 
   It should be specified using the name of a configured remote.
 
-* --exclude=glob
-
-  Skips files matching the glob pattern. The glob is matched relative to
-  the current directory.
-
-  This option can be specified multiple times.
-
 * --numcopies=n
 
   Overrides the `annex.numcopies` setting, forcing git-annex to ensure the
@@ -415,6 +408,54 @@ subdirectories).
 
   Used to override git configuration settings. May be specified multiple times.
 
+# FILE MATCHING OPTIONS
+
+These options can all be specified multiple times, and can be combined to
+limit which files git-annex acts on.
+
+Arbitrarily complicated expressions can be built using these options.
+For example:
+
+	--exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+
+The above example prevents git-annex from working on mp3 files whose
+file contents are present at either of two repositories.
+
+* --exclude=glob
+
+  Skips files matching the glob pattern. The glob is matched relative to
+  the current directory. For example: --exclude='*.mp3' --exclude='subdir/*'
+
+* --in=repository
+
+  Matches only files that git-annex believes have their contents present
+  in a repository.
+
+  The repository should be specified using the name of a configured remote,
+  or the UUID or description of a repository.
+
+* --not
+
+  Inverts the next file matching option. For example, to only act on
+  mp3s, use: --not --exclude='*.mp3'
+
+* --and
+
+  Requires that both the previous and the next file matching option matches.
+  The default.
+
+* --or
+
+  Requires that either the previous, or the next file matching option matches.
+
+* -(
+
+  Opens a group of file matching options.
+
+* -)
+
+  Closes a group of file matching options.
+
 # CONFIGURATION
 
 Like other git commands, git-annex is configured via `.git/config`.

From dd463a3100f21d72c35ca1af5b0f63f6296cf322 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:11:39 -0400
Subject: [PATCH 2224/8313] rework annex-ignore handling

Only one place need to filter the list of remotes for ignored remotes:
keyPossibilities. Make the full list available to everything else.

This allows getting rid of the special case handing for --from and --to
to make ignored remotes not be ignored with those options.
---
 Annex.hs            |  4 ++--
 Config.hs           | 14 +++-----------
 Remote.hs           |  6 ++----
 Remote/Bup.hs       |  3 ++-
 Remote/Directory.hs |  3 ++-
 Remote/Git.hs       |  3 ++-
 Remote/Hook.hs      |  3 ++-
 Remote/Rsync.hs     |  3 ++-
 Remote/S3real.hs    |  3 ++-
 Remote/Web.hs       |  3 ++-
 Types/Remote.hs     |  4 +++-
 11 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index ad65e05ddf..1517a34708 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -26,7 +26,7 @@ import Control.Applicative hiding (empty)
 import qualified Git
 import Git.Queue
 import Types.Backend
-import Types.Remote
+import qualified Types.Remote
 import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
@@ -48,7 +48,7 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [Backend Annex]
-	, remotes :: [Remote Annex]
+	, remotes :: [Types.Remote.Remote Annex]
 	, repoqueue :: Queue
 	, output :: OutputType
 	, force :: Bool
diff --git a/Config.hs b/Config.hs
index b4f4c0b922..fe847fce1c 100644
--- a/Config.hs
+++ b/Config.hs
@@ -82,18 +82,10 @@ prop_cost_sane = False `notElem`
 {- Checks if a repo should be ignored, based either on annex-ignore
  - setting, or on command-line options. Allows command-line to override
  - annex-ignore. -}
-remoteNotIgnored :: Git.Repo -> Annex Bool
-remoteNotIgnored r = do
+repoNotIgnored :: Git.Repo -> Annex Bool
+repoNotIgnored r = do
 	ignored <- getConfig r "ignore" "false"
-	to <- match Annex.toremote
-	from <- match Annex.fromremote
-	if to || from
-		then return True
-		else return $ not $ Git.configTrue ignored
-	where
-		match a = do
-			n <- Annex.getState a
-			return $ n == Git.repoRemoteName r
+	return $ not $ Git.configTrue ignored
 
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}
diff --git a/Remote.hs b/Remote.hs
index 429f9058b6..0ce01872ac 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,7 +16,6 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
-	genList,
 	byName,
 	prettyPrintUUIDs,
 	remotesWithUUID,
@@ -29,7 +28,7 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad (filterM)
+import Control.Monad.State (filterM)
 import Data.List
 import qualified Data.Map as M
 import Data.String.Utils
@@ -83,7 +82,6 @@ genList = do
 	where
 		process m t = 
 			enumerate t >>=
-			filterM remoteNotIgnored >>=
 			mapM (gen m t)
 		gen m t r = do
 			u <- getUUID r
@@ -184,7 +182,7 @@ keyPossibilities' withtrusted key = do
 	let validtrusteduuids = validuuids `intersect` trusted
 
 	-- remotes that match uuids that have the key
-	allremotes <- genList
+	allremotes <- filterM (repoNotIgnored . repo) =<< genList
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index ebb4b10a5b..29c7a0419f 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -66,7 +66,8 @@ gen r u c = do
 			removeKey = remove,
 			hasKey = checkPresent r bupr',
 			hasKeyCheap = bupLocal buprepo,
-			config = c
+			config = c,
+			repo = r
 		}
 
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 7ddb60462f..b183042ef6 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -57,7 +57,8 @@ gen r u c = do
 			removeKey = remove dir,
 			hasKey = checkPresent dir,
 			hasKeyCheap = True,
-			config = Nothing
+			config = Nothing,
+			repo = r
 		}
 
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Git.hs b/Remote/Git.hs
index fadd48a036..9789a06252 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -71,7 +71,8 @@ gen r u _ = do
 		removeKey = dropKey r',
 		hasKey = inAnnex r',
 		hasKeyCheap = cheap,
-		config = Nothing
+		config = Nothing,
+		repo = r'
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 54b9806ffc..aaeb702c7b 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -58,7 +58,8 @@ gen r u c = do
 			removeKey = remove hooktype,
 			hasKey = checkPresent r hooktype,
 			hasKeyCheap = False,
-			config = Nothing
+			config = Nothing,
+			repo = r
 		}
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 3707966ad6..9d2d7ddcf7 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -66,7 +66,8 @@ gen r u c = do
 			removeKey = remove o,
 			hasKey = checkPresent r o,
 			hasKeyCheap = False,
-			config = Nothing
+			config = Nothing,
+			repo = r
 		}
 
 genRsyncOpts :: Git.Repo -> Annex RsyncOpts
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 456a77f0e4..77b6b6ca40 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -67,7 +67,8 @@ gen' r u c cst =
 			removeKey = remove this,
 			hasKey = checkPresent this,
 			hasKeyCheap = False,
-			config = c
+			config = c,
+			repo = r
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 3695bb1ab3..8fb29ec403 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -58,7 +58,8 @@ gen r _ _ =
 		removeKey = dropKey,
 		hasKey = checkKey,
 		hasKeyCheap = False,
-		config = Nothing
+		config = Nothing,
+		repo = r
 	}
 
 {- The urls for a key are stored in remote/web/hash/key.log 
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 8d9622c519..49f16bfdd7 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -51,7 +51,9 @@ data Remote a = Remote {
 	-- operation.
 	hasKeyCheap :: Bool,
 	-- a Remote can have a persistent configuration store
-	config :: Maybe RemoteConfig
+	config :: Maybe RemoteConfig,
+	-- git configuration for the remote
+	repo :: Git.Repo
 }
 
 instance Show (Remote a) where

From 1fc3ee24232059ae05ab18ed10bf151f86847ac1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:14:18 -0400
Subject: [PATCH 2225/8313] add --in limit

---
 GitAnnex.hs      |  4 +++-
 Limit.hs         | 20 ++++++++++++++++++--
 debian/changelog | 10 ++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 68c7a1805f..bcb30ff41f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -108,8 +108,10 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option ['x'] ["exclude"] (ReqArg (Limit.exclude) paramGlob)
+	, Option ['x'] ["exclude"] (ReqArg (Limit.addExclude) paramGlob)
 		"skip files matching the glob pattern"
+	, Option ['i'] ["in"] (ReqArg (Limit.addIn) paramRemote)
+		"skip files not present in a remote"
 	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 91ea3453cf..13df5f6c27 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -14,6 +14,9 @@ import Data.Maybe
 
 import Annex
 import qualified Utility.Matcher
+import qualified Remote
+import qualified Backend
+import LocationLog
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
@@ -51,9 +54,22 @@ token :: String -> Annex ()
 token = add . Utility.Matcher.Token
 
 {- Add a limit to skip files that do not match the glob. -}
-exclude :: String -> Annex ()
-exclude glob = addlimit $ return . notExcluded
+addExclude :: String -> Annex ()
+addExclude glob = addlimit $ return . notExcluded
 	where
 		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
 		regex = '^':wildToRegex glob
+
+{- Adds a limit to skip files not believed to be present
+ - on a specfied remote. -}
+addIn :: String -> Annex ()
+addIn name = do
+	u <- Remote.nameToUUID name
+	addlimit $ check u
+	where
+		check u f = Backend.lookupFile f >>= handle u
+		handle _ Nothing = return False
+		handle u (Just (key, _)) = do
+			us <- keyLocations key
+			return $ u `elem` us
diff --git a/debian/changelog b/debian/changelog
index dff945c287..0b698bc66f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+git-annex (3.20110916) UNRELEASED; urgency=low
+
+  * --in can be used to make git-annex only operate on files
+    believed to be present in a given repository.
+  * Arbitrarily complex expressions can be built to limit the files git-annex
+    operates on, by combining the options --not --and --or -( and -)
+    Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+
+ -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
+
 git-annex (3.20110915) unstable; urgency=low
 
   * whereis: Show untrusted locations separately and do not include in

From 9da23dff78d80158ba01a271ac2a32830fd9bccc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:23:08 -0400
Subject: [PATCH 2226/8313] --copies=N can be used to make git-annex only
 operate on files with the specified number of copies.

(And --not --copies=N for the inverse.)
---
 GitAnnex.hs        |  2 ++
 Limit.hs           | 15 +++++++++++++++
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  5 +++++
 4 files changed, 24 insertions(+)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index bcb30ff41f..a284daad5c 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -112,6 +112,8 @@ options = commonOptions ++
 		"skip files matching the glob pattern"
 	, Option ['i'] ["in"] (ReqArg (Limit.addIn) paramRemote)
 		"skip files not present in a remote"
+	, Option ['C'] ["copies"] (ReqArg (Limit.addCopies) paramNumber)
+		"skip files with fewer copies"
 	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 13df5f6c27..4aeb8bafb4 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -17,6 +17,7 @@ import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
 import LocationLog
+import Utility
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
@@ -73,3 +74,17 @@ addIn name = do
 		handle u (Just (key, _)) = do
 			us <- keyLocations key
 			return $ u `elem` us
+
+{- Adds a limit to skip files not believed to have the specified number
+ - of copies. -}
+addCopies :: String -> Annex ()
+addCopies num = do
+	case readMaybe num :: Maybe Int of
+		Nothing -> error "bad number for --copies"
+		Just n -> addlimit $ check n
+	where
+		check n f = Backend.lookupFile f >>= handle n
+		handle _ Nothing = return False
+		handle n (Just (key, _)) = do
+			us <- keyLocations key
+			return $ length us >= n
diff --git a/debian/changelog b/debian/changelog
index 0b698bc66f..638661f735 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20110916) UNRELEASED; urgency=low
   * Arbitrarily complex expressions can be built to limit the files git-annex
     operates on, by combining the options --not --and --or -( and -)
     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+  * --copies=N can be used to make git-annex only operate on files with
+    the specified number of copies. (And --not --copies=N for the inverse.)
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 03277c6a0e..836e782cde 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -434,6 +434,11 @@ file contents are present at either of two repositories.
   The repository should be specified using the name of a configured remote,
   or the UUID or description of a repository.
 
+* --copies=number
+
+  Matches only files that git-annex believes to have the specified number
+  of copies.
+
 * --not
 
   Inverts the next file matching option. For example, to only act on

From 33cd1ffbfe200188a4fc61fd5715e7aa29556d7f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:41:51 -0400
Subject: [PATCH 2227/8313] make find show files meeting limits, even when not
 present

find: Rather than only showing files whose contents are present, when used
with --exclude --copies or --in, displays all files that match the
specified conditions.

Note that this is a behavior change for find --exclude! Old behavior
can be gotten with find --in . --exclude=...
---
 Command/Find.hs    |  9 ++++++---
 Limit.hs           |  5 +++++
 Utility/Matcher.hs | 10 +++++++++-
 debian/changelog   |  5 +++++
 doc/git-annex.mdwn |  5 ++++-
 5 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 51849f6b85..effb331840 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,12 @@
 
 module Command.Find where
 
-import Control.Monad.State (liftIO)
+import Control.Monad.State
 
 import Command
 import Content
 import Utility.Conditional
+import Limit
 
 command :: [Command]
 command = [repoCommand "find" paramPaths seek "lists available files"]
@@ -19,8 +20,10 @@ command = [repoCommand "find" paramPaths seek "lists available files"]
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
 
-{- Output a list of files. -}
 start :: FilePath -> CommandStart
 start file = isAnnexed file $ \(key, _) -> do
-	whenM (inAnnex key) $ liftIO $ putStrLn file
+	-- only files inAnnex are shown, unless the user has requested
+	-- others via a limit
+	whenM (liftM2 (||) (inAnnex key) limited) $
+		liftIO $ putStrLn file
 	stop
diff --git a/Limit.hs b/Limit.hs
index 4aeb8bafb4..51f3fc9505 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -10,6 +10,7 @@ module Limit where
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
 import Control.Monad (filterM)
+import Control.Applicative
 import Data.Maybe
 
 import Annex
@@ -27,6 +28,10 @@ filterFiles l = do
 	matcher <- getMatcher
 	filterM (Utility.Matcher.matchM matcher) l
 
+{- Checks if there are user-specified limits. -}
+limited :: Annex Bool
+limited = (not . Utility.Matcher.matchesAny) <$> getMatcher
+
 {- Gets a matcher for the user-specified limits. The matcher is cached for
  - speed; once it's obtained the user-specified limits can't change. -}
 getMatcher :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 08534fc304..5cf000b1b6 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -20,7 +20,8 @@ module Utility.Matcher (
 	Matcher,
 	generate,
 	match,
-	matchM
+	matchM,
+	matchesAny
 ) where
 
 import Control.Monad
@@ -81,3 +82,10 @@ matchM m v = go m
 		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
 		go (Not m1) = liftM not (go m1)
 		go (Op o) = o v
+
+{- Checks is a matcher contains no limits, and so (presumably) matches
+ - anything. Note that this only checks the trivial case; it is possible
+ - to construct matchers that match anything but are more complicated. -}
+matchesAny :: Matcher a -> Bool
+matchesAny Any = True
+matchesAny _ = False
diff --git a/debian/changelog b/debian/changelog
index 638661f735..1f7decc1d8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,11 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
   * --copies=N can be used to make git-annex only operate on files with
     the specified number of copies. (And --not --copies=N for the inverse.)
+  * find: Rather than only showing files whose contents are present,
+    when used with --exclude --copies or --in, displays all files that
+    match the specified conditions.
+  * Note that this is a behavior change for find --exclude! Old behavior
+    can be gotten with: find --in . --exclude=...
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 836e782cde..4a8e28a6ee 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -222,6 +222,8 @@ subdirectories).
 * find [path ...]
 
   Outputs a list of annexed files whose content is currently present.
+  Or, if a file matching option is specified, outputs a list of all
+  matching files, whether or not their content is currently present.
 
   With no parameters, defaults to finding all files in the current directory
   and its subdirectories.
@@ -432,7 +434,8 @@ file contents are present at either of two repositories.
   in a repository.
 
   The repository should be specified using the name of a configured remote,
-  or the UUID or description of a repository.
+  or the UUID or description of a repository. For the current repository,
+  use "--in=."
 
 * --copies=number
 

From 8ea48c3e3976b16ee4588a24fe1524769ef42e56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:56:37 -0400
Subject: [PATCH 2228/8313] update docs

---
 doc/cheatsheet.mdwn                           |  1 +
 doc/todo/exclude_files_on_a_given_remote.mdwn |  2 ++
 doc/walkthrough/powerful_file_matching.mdwn   | 36 +++++++++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 doc/walkthrough/powerful_file_matching.mdwn

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
index 9ccb22e3e2..65cbad187c 100644
--- a/doc/cheatsheet.mdwn
+++ b/doc/cheatsheet.mdwn
@@ -12,4 +12,5 @@ A supplement to the [[walkthrough]].
 	walkthrough/what_to_do_when_you_lose_a_repository
 	walkthrough/recover_data_from_lost+found
 	walkthrough/Internet_Archive_via_S3
+	walkthrough/powerful_file_matching
 """]]
diff --git a/doc/todo/exclude_files_on_a_given_remote.mdwn b/doc/todo/exclude_files_on_a_given_remote.mdwn
index fbb3f90bef..e8bb357d31 100644
--- a/doc/todo/exclude_files_on_a_given_remote.mdwn
+++ b/doc/todo/exclude_files_on_a_given_remote.mdwn
@@ -14,3 +14,5 @@ thought/known to be on A.
 	git annex drop --only-on A
 
 --[[Joey]] 
+
+[[done]]
diff --git a/doc/walkthrough/powerful_file_matching.mdwn b/doc/walkthrough/powerful_file_matching.mdwn
new file mode 100644
index 0000000000..c0af4615c4
--- /dev/null
+++ b/doc/walkthrough/powerful_file_matching.mdwn
@@ -0,0 +1,36 @@
+git-annex has a powerful syntax for making it act on only certian files.
+
+The simplest thing is to exclude some files, using wild cards:
+
+	git annex get --exclude '*.mp3' --exclude '*.ogg'
+
+But you can also exclude files that git-annex's [[location_tracking]]
+information indicates are present in a given repository. For example,
+if you want to populate newarchive with files, but not those already
+on oldarchive, you could do it like this:
+
+	git annex copy --not -in oldarchive --to newarchive
+
+Without the --not, --in makes it act on files that *are* in the specified
+repository. So, to remove files that are on oldarchive:
+
+	git annex drop --in oldarchive
+
+Or maybe you're curious which files have a lot of copies, and then
+also want to know which files have only one copy:
+
+	git annex find --copies 7
+	git annex find --not --copies 2
+
+The above are the simple examples of specifying what files git-annex
+should act on. But you can specify anything you can dream up by combining
+the things above, with --and --or -( and -). Those last two strange-looking
+options are parentheses, for grouping other options. You will probably
+have to escape them from your shell.
+
+Here are the mp3 files that are in either of two repositories, but have
+less than 3 copies:
+
+	git annex find --not --exclude '*.mp3' --and \
+		-\( --in usbdrive --or --in archive -\) --and \
+		--not --copies 3

From b516cecff2c315e495c7dc6ec51af6e5aefaf57c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 20:58:34 -0400
Subject: [PATCH 2229/8313] probably better to error on unknown token

---
 Utility/Matcher.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 5cf000b1b6..451c3f139e 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -57,7 +57,7 @@ consume m ((Token t):ts)
 	| t == "not" = cont $ m `And` (Not next)
 	| t == "(" = let (n, r) = consume next rest in (m `And` n, r)
 	| t == ")" = (m, ts)
-	| otherwise = (m, ts) -- ignore unknown token
+	| otherwise = error $ "unknown token " ++ t
 	where
 		(next, rest) = consume Any ts
 		cont v = (v, rest)

From 8d1e8c0760e0d9935223523f35f5b8ea954730ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 21:02:40 -0400
Subject: [PATCH 2230/8313] golfing with curry

---
 Utility/Matcher.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 451c3f139e..342775d68c 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -42,9 +42,7 @@ generate :: [Token op] -> Matcher op
 generate ts = generate' Any ts
 generate' :: Matcher op -> [Token op] -> Matcher op
 generate' m [] = m
-generate' m ts = generate' m' rest
-	where
-		(m', rest) = consume m ts
+generate' m ts = uncurry generate' $ consume m ts
 
 {- Consumes one or more Tokens, constructs a new Matcher,
  - and returns unconsumed Tokens. -}

From 4f1fea1a856a2c82ed200e805bb18e9f9aaaa67b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 22:40:31 -0400
Subject: [PATCH 2231/8313] fix memory leak

filterM is not a good idea if you were streaming in a large list of files.

Fixing this memory leak that I introduced earlier today was a PITA because
to avoid the filterM, it's necessary to do the filtering only after
building up the data structures like BackendFile, and that means each
separate data structure needs it own function to apply the filter,
at least in this naive implementation.

There is also a minor performance regression, when using copy/drop/get/fsck
with a filter, git is now asked to look up attributes for all files,
since that now comes before the filter is applied. This is only a very
minor thing, since getting the attributes is very fast and --exclude was
probably not typically used to speed it up.
---
 Command.hs | 53 +++++++++++++++++++++++++++++++++--------------------
 Limit.hs   | 15 +++++++--------
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/Command.hs b/Command.hs
index fd58ca801a..3cfff268c8 100644
--- a/Command.hs
+++ b/Command.hs
@@ -107,17 +107,22 @@ notBareRepo a = do
 
 {- These functions find appropriate files or other things based on a
    user's parameters, and run a specified action on them. -}
-withFilesInGit :: (String -> CommandStart) -> CommandSeek
+withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	liftM (map a) $ filterFiles files
+	runFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	files' <- filterFiles files
-	liftM (map a) $ liftIO $ Git.checkAttr repo attr files'
+	run $ liftIO $ Git.checkAttr repo attr files
+	where
+		run fs = do
+			matcher <- Limit.getMatcher
+			liftM (map $ proc matcher) fs
+		proc matcher p@(f, _) = do
+			ok <- matcher f
+			if ok then a p else stop
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
@@ -128,23 +133,17 @@ withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	files' <- filterFiles files
-	backendPairs a files'
+	backendPairs a files
 withFilesMissing :: (String -> CommandStart) -> CommandSeek
-withFilesMissing a params = do
-	files <- liftIO $ filterM missing params
-	liftM (map a) $ filterFiles files
+withFilesMissing a params = runFiltered a $ liftIO $ filterM missing params
 	where
-		missing f = do
-			e <- doesFileExist f
-			return $ not e
+		missing = liftM not . doesFileExist
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	repo <- Annex.gitRepo
 	force <- Annex.getState Annex.force
 	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
-	newfiles' <- filterFiles newfiles
-	backendPairs a newfiles'
+	backendPairs a newfiles
 withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
 withStrings :: (String -> CommandStart) -> CommandSeek
@@ -152,8 +151,8 @@ withStrings a params = return $ map a params
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = do
 	repo <- Annex.gitRepo
-	tocommit <- liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
-	liftM (map a) $ filterFiles tocommit
+	runFiltered a $
+		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
@@ -165,8 +164,7 @@ withFilesUnlocked' typechanged a params = do
 	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
-	unlockedfiles' <- filterFiles unlockedfiles
-	backendPairs a unlockedfiles'
+	backendPairs a unlockedfiles
 withKeys :: (Key -> CommandStart) -> CommandSeek
 withKeys a params = return $ map (a . parse) params
 	where
@@ -175,8 +173,23 @@ withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
+runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
+runFiltered a fs = do
+	matcher <- Limit.getMatcher
+	liftM (map $ proc matcher) fs
+	where
+		proc matcher f = do
+			ok <- matcher f
+			if ok then a f else stop
+
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-backendPairs a files = map a <$> Backend.chooseBackends files
+backendPairs a fs = do
+	matcher <- Limit.getMatcher
+	liftM (map $ proc matcher) (Backend.chooseBackends fs)
+	where
+		proc matcher p@(_, f) = do
+			ok <- matcher f
+			if ok then a p else stop
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool
diff --git a/Limit.hs b/Limit.hs
index 51f3fc9505..602da70013 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -22,20 +22,19 @@ import Utility
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
-{- Filter out files not matching user-specified limits. -}
-filterFiles :: [FilePath] -> Annex [FilePath]
-filterFiles l = do
-	matcher <- getMatcher
-	filterM (Utility.Matcher.matchM matcher) l
-
 {- Checks if there are user-specified limits. -}
 limited :: Annex Bool
-limited = (not . Utility.Matcher.matchesAny) <$> getMatcher
+limited = (not . Utility.Matcher.matchesAny) <$> getMatcher'
 
 {- Gets a matcher for the user-specified limits. The matcher is cached for
  - speed; once it's obtained the user-specified limits can't change. -}
-getMatcher :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+getMatcher :: Annex (FilePath -> Annex Bool)
 getMatcher = do
+	m <- getMatcher'
+	return $ Utility.Matcher.matchM m
+
+getMatcher' :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+getMatcher' = do
 	m <- Annex.getState Annex.limit
 	case m of
 		Right r -> return r

From c31a6a9e100d9d9a822fd5e93f2a59d1562db579 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 18 Sep 2011 23:09:40 -0400
Subject: [PATCH 2232/8313] refactor

---
 Command.hs | 35 ++++++++++++-----------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/Command.hs b/Command.hs
index 3cfff268c8..a568da33b6 100644
--- a/Command.hs
+++ b/Command.hs
@@ -115,20 +115,11 @@ withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandS
 withAttrFilesInGit attr a params = do
 	repo <- Annex.gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	run $ liftIO $ Git.checkAttr repo attr files
-	where
-		run fs = do
-			matcher <- Limit.getMatcher
-			liftM (map $ proc matcher) fs
-		proc matcher p@(f, _) = do
-			ok <- matcher f
-			if ok then a p else stop
+	runFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
-		go (file, v) = do
-			let numcopies = readMaybe v
-			a file numcopies
+		go (file, v) = a file (readMaybe v)
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
 	repo <- Annex.gitRepo
@@ -174,22 +165,20 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
-runFiltered a fs = do
+runFiltered a fs = runFilteredGen a id fs
+
+backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
+backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
+
+runFilteredGen :: (a1 -> Annex (Maybe a)) -> (a1 -> FilePath) -> Annex [a1] -> Annex [Annex (Maybe a)]
+runFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
 	liftM (map $ proc matcher) fs
 	where
-		proc matcher f = do
+		proc matcher v = do
+			let f = d v
 			ok <- matcher f
-			if ok then a f else stop
-
-backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-backendPairs a fs = do
-	matcher <- Limit.getMatcher
-	liftM (map $ proc matcher) (Backend.chooseBackends fs)
-	where
-		proc matcher p@(_, f) = do
-			ok <- matcher f
-			if ok then a p else stop
+			if ok then a v else stop
 
 {- filter out symlinks -}	
 notSymlink :: FilePath -> IO Bool

From 6e80f195148ca689d85c6c8ed7f1a4f9720397a7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:03:16 -0400
Subject: [PATCH 2233/8313] golf

---
 Limit.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index 602da70013..2482c8a656 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -9,7 +9,6 @@ module Limit where
 
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
-import Control.Monad (filterM)
 import Control.Applicative
 import Data.Maybe
 
@@ -29,9 +28,7 @@ limited = (not . Utility.Matcher.matchesAny) <$> getMatcher'
 {- Gets a matcher for the user-specified limits. The matcher is cached for
  - speed; once it's obtained the user-specified limits can't change. -}
 getMatcher :: Annex (FilePath -> Annex Bool)
-getMatcher = do
-	m <- getMatcher'
-	return $ Utility.Matcher.matchM m
+getMatcher = Utility.Matcher.matchM <$> getMatcher'
 
 getMatcher' :: Annex (Utility.Matcher.Matcher (FilePath -> Annex Bool))
 getMatcher' = do

From dcded89129c4647bc71b474aac6d3e334b4321c1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:37:04 -0400
Subject: [PATCH 2234/8313] reorg

---
 Command.hs      | 52 +++++++++++--------------------------------------
 Utility/Path.hs | 27 +++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/Command.hs b/Command.hs
index a568da33b6..cc9bcbf0c9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -1,6 +1,6 @@
 {- git-annex command infrastructure
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -10,9 +10,8 @@ module Command where
 import Control.Monad.State (liftIO)
 import System.Directory
 import System.Posix.Files
-import Control.Monad (filterM, liftM, when)
+import Control.Monad (filterM, liftM)
 import Control.Applicative
-import Data.List
 import Data.Maybe
 
 import Types
@@ -22,6 +21,8 @@ import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Utility
+import Utility.Conditional
+import Utility.Path
 import Types.Key
 import Trust
 import LocationLog
@@ -75,9 +76,8 @@ stop = return Nothing
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
-prepCommand Command { cmdseek = seek } params = do
-	lists <- mapM (\s -> s params) seek
-	return $ map doCommand $ concat lists
+prepCommand Command { cmdseek = seek } params =
+	return . map doCommand . concat =<< mapM (\s -> s params) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
@@ -86,11 +86,9 @@ doCommand = start
 		start   = stage $ maybe success perform
 		perform = stage $ maybe failure cleanup
 		cleanup = stage $ \r -> showEndResult r >> return r
-		stage a b = b >>= a
+		stage = (=<<)
 		success = return True
-		failure = do
-			showEndFail
-			return False
+		failure = showEndFail >> return False
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
@@ -100,13 +98,12 @@ isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
-	g <- Annex.gitRepo
-	when (Git.repoIsLocalBare g) $
+	whenM (Git.repoIsLocalBare <$> Annex.gitRepo) $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
 {- These functions find appropriate files or other things based on a
-   user's parameters, and run a specified action on them. -}
+   user's parameters, and prepare actions operating on them. -}
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
 	repo <- Annex.gitRepo
@@ -170,7 +167,7 @@ runFiltered a fs = runFilteredGen a id fs
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
 
-runFilteredGen :: (a1 -> Annex (Maybe a)) -> (a1 -> FilePath) -> Annex [a1] -> Annex [Annex (Maybe a)]
+runFilteredGen :: (b -> Annex (Maybe a)) -> (b -> FilePath) -> Annex [b] -> Annex [Annex (Maybe a)]
 runFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
 	liftM (map $ proc matcher) fs
@@ -228,33 +225,6 @@ cmdlineKey  = do
 		nokey = error "please specify the key with --key"
 		badkey = error "bad key"
 
-{- Given an original list of files, and an expanded list derived from it,
- - ensures that the original list's ordering is preserved. 
- -
- - The input list may contain a directory, like "dir" or "dir/". Any
- - items in the expanded list that are contained in that directory will
- - appear at the same position as it did in the input list.
- -}
-preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
--- optimisation, only one item in original list, so no reordering needed
-preserveOrder [_] new = new
-preserveOrder orig new = collect orig new
-	where
-		collect [] n = n
-		collect [_] n = n -- optimisation
-		collect (l:ls) n = found ++ collect ls rest
-			where (found, rest)=partition (l `dirContains`) n
-
-{- Runs an action that takes a list of FilePaths, and ensures that 
- - its return list preserves order.
- -
- - This assumes that it's cheaper to call preserveOrder on the result,
- - than it would be to run the action separately with each param. In the case
- - of git file list commands, that assumption tends to hold.
- -}
-runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
-runPreserveOrder a files = preserveOrder files <$> a files
-
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
  -
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 9b8041dad0..fe474ee825 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -90,3 +90,30 @@ prop_relPathDirToFile_basics from to
 	| otherwise = not (null r)
 	where
 		r = relPathDirToFile from to 
+
+{- Given an original list of files, and an expanded list derived from it,
+ - ensures that the original list's ordering is preserved. 
+ -
+ - The input list may contain a directory, like "dir" or "dir/". Any
+ - items in the expanded list that are contained in that directory will
+ - appear at the same position as it did in the input list.
+ -}
+preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
+-- optimisation, only one item in original list, so no reordering needed
+preserveOrder [_] new = new
+preserveOrder orig new = collect orig new
+	where
+		collect [] n = n
+		collect [_] n = n -- optimisation
+		collect (l:ls) n = found ++ collect ls rest
+			where (found, rest)=partition (l `dirContains`) n
+
+{- Runs an action that takes a list of FilePaths, and ensures that 
+ - its return list preserves order.
+ -
+ - This assumes that it's cheaper to call preserveOrder on the result,
+ - than it would be to run the action separately with each param. In the case
+ - of git file list commands, that assumption tends to hold.
+ -}
+runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
+runPreserveOrder a files = preserveOrder files <$> a files

From 94ee28556fb0c42b774e0af9f69fbf9cb5f04d01 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:52:17 -0400
Subject: [PATCH 2235/8313] special case for --in .

Do not need to check the location log in this case, can just check inAnnex.
This is both an optimisation and perhaps a correctness measure
(fsck --in . should fsck files even if the location log is damaged.)
---
 Limit.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index 2482c8a656..d8ec4484c8 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -18,6 +18,7 @@ import qualified Remote
 import qualified Backend
 import LocationLog
 import Utility
+import Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
@@ -64,17 +65,19 @@ addExclude glob = addlimit $ return . notExcluded
 		regex = '^':wildToRegex glob
 
 {- Adds a limit to skip files not believed to be present
- - on a specfied remote. -}
+ - on a specfied repository. -}
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name
-	addlimit $ check u
+	addlimit $ if name == "." then check local else check (remote u)
 	where
-		check u f = Backend.lookupFile f >>= handle u
+		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
-		handle u (Just (key, _)) = do
+		handle a (Just (key, _)) = a key
+		remote u key = do
 			us <- keyLocations key
 			return $ u `elem` us
+		local key = inAnnex key
 
 {- Adds a limit to skip files not believed to have the specified number
  - of copies. -}

From a4aef6f11508078b6d593ec9daea2c4225e630c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:54:20 -0400
Subject: [PATCH 2236/8313] clarify wording

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 1f7decc1d8..fc57fb361c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,8 +10,8 @@ git-annex (3.20110916) UNRELEASED; urgency=low
   * find: Rather than only showing files whose contents are present,
     when used with --exclude --copies or --in, displays all files that
     match the specified conditions.
-  * Note that this is a behavior change for find --exclude! Old behavior
-    can be gotten with: find --in . --exclude=...
+  * Note that this is a behavior change for git-annex find! Old behavior
+    can be gotten by using: git-annex find --in .
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 

From 3b2e4620184e2f7cc2dfd72b80e1e9eb8bd1dfef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 01:57:12 -0400
Subject: [PATCH 2237/8313] tweak

---
 Limit.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index d8ec4484c8..f57895aeef 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -43,10 +43,10 @@ getMatcher' = do
 
 {- Adds something to the limit list, which is built up reversed. -}
 add :: Limit -> Annex ()
-add l = Annex.changeState $ \s -> s { Annex.limit = append $ Annex.limit s }
+add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s }
 	where
-		append (Left ls) = Left $ l:ls
-		append _ = error "internal"
+		prepend (Left ls) = Left $ l:ls
+		prepend _ = error "internal"
 
 {- Adds a new limit. -}
 addlimit :: (FilePath -> Annex Bool) -> Annex ()
@@ -65,7 +65,7 @@ addExclude glob = addlimit $ return . notExcluded
 		regex = '^':wildToRegex glob
 
 {- Adds a limit to skip files not believed to be present
- - on a specfied repository. -}
+ - in a specfied repository. -}
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name

From 73f3a00c1cad9e77ba9517bdc25b8086abc9bc59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 12:21:53 -0400
Subject: [PATCH 2238/8313] typo

---
 doc/walkthrough/powerful_file_matching.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/powerful_file_matching.mdwn b/doc/walkthrough/powerful_file_matching.mdwn
index c0af4615c4..d5d29377c4 100644
--- a/doc/walkthrough/powerful_file_matching.mdwn
+++ b/doc/walkthrough/powerful_file_matching.mdwn
@@ -9,7 +9,7 @@ information indicates are present in a given repository. For example,
 if you want to populate newarchive with files, but not those already
 on oldarchive, you could do it like this:
 
-	git annex copy --not -in oldarchive --to newarchive
+	git annex copy --not --in oldarchive --to newarchive
 
 Without the --not, --in makes it act on files that *are* in the specified
 repository. So, to remove files that are on oldarchive:

From 5c20ebcbf3df9e8641f5b08f6a2307a7cfb3eda3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Sep 2011 18:46:35 +0000
Subject: [PATCH 2239/8313] Added a comment

---
 .../comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment

diff --git a/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment
new file mode 100644
index 0000000000..787cf8f5d7
--- /dev/null
+++ b/doc/forum/Wishlist:_Ways_of_selecting_files_based_on_meta-information/comment_2_97e2ed48bd552d02918c4f98f963e6e1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-09-19T18:46:35Z"
+ content="""
+This is now almost completely implemented. See [[walkthrough/powerful_file_matching]].
+
+"""]]

From 1ddc207b585cf0dded363b5890010afff58aa478 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Sep 2011 14:50:16 -0400
Subject: [PATCH 2240/8313] some of these are now done

---
 doc/forum/new_microfeatures.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/forum/new_microfeatures.mdwn b/doc/forum/new_microfeatures.mdwn
index 683cc69b8b..bfe44272a7 100644
--- a/doc/forum/new_microfeatures.mdwn
+++ b/doc/forum/new_microfeatures.mdwn
@@ -33,10 +33,16 @@ Come to think of it, the inject --bare thing is probably not a microfeature. Sho
 
 > I've thought about such things before; does not seem really micro and I'm unsure how well it would work, but it would be worth a [[todo]]. --[[Joey]]
 
+>> Update: Done as --auto. --[[Joey]] 
+
 ---
 
 Along similar lines, it might be nice to have a mode where git-annex tries to fill up a disk up to the `annex.diskreserve` with files, preferring files that have relatively few copies. Then as storage prices continue to fall, new large drives could just be plopped in and git-annex used to fill it up in a way that improves the overall redundancy without needing to manually pick and choose.
 
+> Update: git annex get --auto basically does this; you can tune
+> --numcopies on the fly to make it get more files than needed by the
+> current numcopies setting. --[[Joey]] 
+
 ---
 
 If a remote could send on received files to another remote, I could use my own local bandwith efficiently while still having my git-annex repos replicate data. -- RichiH

From 10db73426aba8554c62298b59391cab42c677af8 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Sep 2011 18:54:46 +0000
Subject: [PATCH 2241/8313] Added a comment

---
 .../comment_4_63f24abf086d644dced8b01e1a9948c9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment

diff --git a/doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment b/doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment
new file mode 100644
index 0000000000..3805464a69
--- /dev/null
+++ b/doc/forum/wishlist:_git-annex_replicate/comment_4_63f24abf086d644dced8b01e1a9948c9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-09-19T18:54:46Z"
+ content="""
+git annex get/copy/drop all now support a --auto flag, which makes them only act on files that have not enough or too many copies. This allows for some crude replication; it doesn't take into account which repositories should be filled up more (beyond honoring annex.diskreserve), nor does it try to optimally use bandwidth (beyond honoring configured annex-cost). You have to run it in every repository that you want to participate in the replication, too. But it's probably a Good Enough solution. See [[walkthrough/automatically_managing_content]].
+"""]]

From a1578e33dce4a9e7dcbd700077fc73237c35d19a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Sep 2011 18:57:52 +0000
Subject: [PATCH 2242/8313] Added a comment

---
 .../comment_1_3480b0ec629ef29a151408d869186bf8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment

diff --git a/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment
new file mode 100644
index 0000000000..5d0edce2ea
--- /dev/null
+++ b/doc/forum/wishlist:_push_to_cia.vc_from_the_website__39__s_repo__44___not_your_personal_one/comment_1_3480b0ec629ef29a151408d869186bf8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-09-19T18:57:52Z"
+ content="""
+JFTR, pushing now happens automatically from branchable.
+"""]]

From 52533799535ebe1c133e2687ff1c5612d5bdb51d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 00:49:40 -0400
Subject: [PATCH 2243/8313] convert Token to have separate constructors for
 each peice of syntax

---
 Limit.hs           | 18 ++++++-------
 Options.hs         |  4 +--
 Utility/Matcher.hs | 67 +++++++++++++++++++++++++++-------------------
 3 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/Limit.hs b/Limit.hs
index f57895aeef..b3b041396f 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -48,17 +48,17 @@ add l = Annex.changeState $ \s -> s { Annex.limit = prepend $ Annex.limit s }
 		prepend (Left ls) = Left $ l:ls
 		prepend _ = error "internal"
 
-{- Adds a new limit. -}
-addlimit :: (FilePath -> Annex Bool) -> Annex ()
-addlimit = add . Utility.Matcher.Operation
-
 {- Adds a new token. -}
-token :: String -> Annex ()
-token = add . Utility.Matcher.Token
+addToken :: String -> Annex ()
+addToken = add . Utility.Matcher.token
+
+{- Adds a new limit. -}
+addLimit :: (FilePath -> Annex Bool) -> Annex ()
+addLimit = add . Utility.Matcher.Operation
 
 {- Add a limit to skip files that do not match the glob. -}
 addExclude :: String -> Annex ()
-addExclude glob = addlimit $ return . notExcluded
+addExclude glob = addLimit $ return . notExcluded
 	where
 		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
@@ -69,7 +69,7 @@ addExclude glob = addlimit $ return . notExcluded
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name
-	addlimit $ if name == "." then check local else check (remote u)
+	addLimit $ if name == "." then check local else check (remote u)
 	where
 		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
@@ -85,7 +85,7 @@ addCopies :: String -> Annex ()
 addCopies num = do
 	case readMaybe num :: Maybe Int of
 		Nothing -> error "bad number for --copies"
-		Just n -> addlimit $ check n
+		Just n -> addLimit $ check n
 	where
 		check n f = Backend.lookupFile f >>= handle n
 		handle _ Nothing = return False
diff --git a/Options.hs b/Options.hs
index 44d5f3674c..eeb3639b4d 100644
--- a/Options.hs
+++ b/Options.hs
@@ -58,5 +58,5 @@ matcherOptions =
 	, shortopt ")" "close group of options"
 	]
 	where
-		longopt o d = Option [] [o] (NoArg (token o)) d
-		shortopt o d = Option o [] (NoArg (token o)) d
+		longopt o d = Option [] [o] (NoArg (addToken o)) d
+		shortopt o d = Option o [] (NoArg (addToken o)) d
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 342775d68c..323a84bfde 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -18,6 +18,7 @@
 module Utility.Matcher (
 	Token(..),
 	Matcher,
+	token,
 	generate,
 	match,
 	matchM,
@@ -26,20 +27,30 @@ module Utility.Matcher (
 
 import Control.Monad
 
-{- A Token can either be a single word, or an Operation of an arbitrary type. -}
-data Token op = Token String | Operation op
+{- A Token can be an Operation of an arbitrary type, or one of a few
+ - predefined peices of syntax. -}
+data Token op = Operation op | And | Or | Not | Open | Close
 	deriving (Show, Eq)
 
-data Matcher op = Any
-	| And (Matcher op) (Matcher op)
-	| Or (Matcher op) (Matcher op)
-	| Not (Matcher op)
-	| Op op
+data Matcher op = MAny
+	| MAnd (Matcher op) (Matcher op)
+	| MOr (Matcher op) (Matcher op)
+	| MNot (Matcher op)
+	| MOp op
 	deriving (Show, Eq)
 
+{- Converts a word of syntax into a token. Doesn't handle operations. -}
+token :: String -> Token op
+token "and" = And
+token "or" = Or
+token "not" = Not
+token "(" = Open
+token ")" = Close
+token t = error $ "unknown token " ++ t
+
 {- Converts a list of Tokens into a Matcher. -}
 generate :: [Token op] -> Matcher op
-generate ts = generate' Any ts
+generate ts = generate' MAny ts
 generate' :: Matcher op -> [Token op] -> Matcher op
 generate' m [] = m
 generate' m ts = uncurry generate' $ consume m ts
@@ -48,16 +59,16 @@ generate' m ts = uncurry generate' $ consume m ts
  - and returns unconsumed Tokens. -}
 consume :: Matcher op -> [Token op] -> (Matcher op, [Token op])
 consume m [] = (m, [])
-consume m ((Operation o):ts) = (m `And` Op o, ts)
-consume m ((Token t):ts)
-	| t == "and" = cont $ m `And` next
-	| t == "or" = cont $ m `Or` next
-	| t == "not" = cont $ m `And` (Not next)
-	| t == "(" = let (n, r) = consume next rest in (m `And` n, r)
-	| t == ")" = (m, ts)
-	| otherwise = error $ "unknown token " ++ t
+consume m (t:ts) = go t
 	where
-		(next, rest) = consume Any ts
+		go And = cont $ m `MAnd` next
+		go Or = cont $ m `MOr` next
+		go Not = cont $ m `MAnd` (MNot next)
+		go Open = let (n, r) = consume next rest in (m `MAnd` n, r)
+		go Close = (m, ts)
+		go (Operation o) = (m `MAnd` MOp o, ts)
+
+		(next, rest) = consume MAny ts
 		cont v = (v, rest)
 
 {- Checks if a Matcher matches, using a supplied function to check
@@ -65,25 +76,25 @@ consume m ((Token t):ts)
 match :: (op -> v -> Bool) -> Matcher op -> v -> Bool
 match a m v = go m
 	where
-		go Any = True
-		go (And m1 m2) = go m1 && go m2
-		go (Or m1 m2) = go m1 || go m2
-		go (Not m1) = not (go m1)
-		go (Op o) = a o v
+		go MAny = True
+		go (MAnd m1 m2) = go m1 && go m2
+		go (MOr m1 m2) = go m1 || go m2
+		go (MNot m1) = not (go m1)
+		go (MOp o) = a o v
 
 {- Runs a monadic Matcher, where Operations are actions in the monad. -}
 matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool
 matchM m v = go m
 	where
-		go Any = return True
-		go (And m1 m2) = liftM2 (&&) (go m1) (go m2)
-		go (Or m1 m2) =  liftM2 (||) (go m1) (go m2)
-		go (Not m1) = liftM not (go m1)
-		go (Op o) = o v
+		go MAny = return True
+		go (MAnd m1 m2) = liftM2 (&&) (go m1) (go m2)
+		go (MOr m1 m2) =  liftM2 (||) (go m1) (go m2)
+		go (MNot m1) = liftM not (go m1)
+		go (MOp o) = o v
 
 {- Checks is a matcher contains no limits, and so (presumably) matches
  - anything. Note that this only checks the trivial case; it is possible
  - to construct matchers that match anything but are more complicated. -}
 matchesAny :: Matcher a -> Bool
-matchesAny Any = True
+matchesAny MAny = True
 matchesAny _ = False

From b62123c378e7d134914f1479fbfa3409d94aa209 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 00:58:18 -0400
Subject: [PATCH 2244/8313] simplify

---
 Utility/Matcher.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 323a84bfde..69b78be4a0 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -50,10 +50,10 @@ token t = error $ "unknown token " ++ t
 
 {- Converts a list of Tokens into a Matcher. -}
 generate :: [Token op] -> Matcher op
-generate ts = generate' MAny ts
-generate' :: Matcher op -> [Token op] -> Matcher op
-generate' m [] = m
-generate' m ts = uncurry generate' $ consume m ts
+generate = go MAny
+	where
+		go m [] = m
+		go m ts = uncurry go $ consume m ts
 
 {- Consumes one or more Tokens, constructs a new Matcher,
  - and returns unconsumed Tokens. -}

From cabbefd9d2d16b52b28f69a8410a9eb84e506666 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 18:13:08 -0400
Subject: [PATCH 2245/8313] status: In --fast mode, all status info is
 displayed now; but some of it is only approximate, and is marked as such.

---
 Command/Status.hs  | 58 +++++++++++++++++++++++++---------------------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  4 ++--
 3 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 76659a75e1..067128f620 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -23,13 +23,15 @@ import qualified Git
 import Command
 import Types
 import Utility.DataUnits
+import Utility.Conditional
 import Content
 import Types.Key
 import Locations
 import Backend
+import Messages
 
 -- a named computation that produces a statistic
-type Stat = StatState (Maybe (String, StatState String))
+type Stat = StatState (Maybe (String, Bool, StatState String))
 
 -- cached info that multiple Stats may need
 data StatInfo = StatInfo
@@ -58,16 +60,13 @@ seek = [withNothing start]
 {- Order is significant. Less expensive operations, and operations
  - that share data go together.
  -}
-faststats :: [Stat]
-faststats = 
+stats :: [Stat]
+stats = 
 	[ supported_backends
 	, supported_remote_types
 	, tmp_size
 	, bad_data_size
-	]
-slowstats :: [Stat]
-slowstats =
-	[ local_annex_keys
+	, local_annex_keys
 	, local_annex_size
 	, total_annex_keys
 	, total_annex_size
@@ -76,13 +75,16 @@ slowstats =
 
 start :: CommandStart
 start = do
-	fast <- Annex.getState Annex.fast
-	let todo = if fast then faststats else faststats ++ slowstats
-	evalStateT (mapM_ showStat todo) (StatInfo Nothing Nothing)
+	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
+	fastmode_note
 	stop
 
-stat :: String -> StatState String -> Stat
-stat desc a = return $ Just (desc, a)
+fastmode_note :: Annex ()
+fastmode_note = whenM (Annex.getState Annex.fast) $
+	showLongNote "(*) approximate due to fast mode"
+
+stat :: String -> Bool -> StatState String -> Stat
+stat desc approx a = return $ Just (desc, approx, a)
 
 nostat :: Stat
 nostat = return Nothing
@@ -90,33 +92,37 @@ nostat = return Nothing
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
-		calc (Just (desc, a)) = do
-			liftIO $ putStr $ desc ++ ": "
+		calc (Just (desc, approx, a)) = do
+			fast <- lift $ Annex.getState Annex.fast
+			let star = if fast && approx then "(*)" else ""
+			liftIO $ putStr $ desc ++ star ++ ": "
 			liftIO $ hFlush stdout
 			liftIO . putStrLn =<< a
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" $ 
+supported_backends = stat "supported backends" False $ 
 	return $ unwords $ map B.name Backend.list
 
 supported_remote_types :: Stat
-supported_remote_types = stat "supported remote types" $
+supported_remote_types = stat "supported remote types" False $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
 local_annex_size :: Stat
-local_annex_size = stat "local annex size" $
+local_annex_size = stat "local annex size" False $
 	cachedKeysPresent >>= keySizeSum
 
 total_annex_size :: Stat
-total_annex_size = stat "total annex size" $
+total_annex_size = stat "total annex size" True $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $ show . snd <$> cachedKeysPresent
+local_annex_keys = stat "local annex keys" False $
+	show . snd <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $ show . snd <$> cachedKeysReferenced
+total_annex_keys = stat "total annex keys" True $
+	show . snd <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -125,7 +131,7 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
+backend_usage = stat "backend usage" True $ usage <$> cachedKeysReferenced
 	where
 		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
 		splits :: [Key] -> [(String, Integer)]
@@ -135,7 +141,6 @@ backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
-
 cachedKeysPresent :: StatState (SizeList Key)
 cachedKeysPresent = do
 	s <- get
@@ -153,10 +158,11 @@ cachedKeysReferenced = do
 	case keysReferencedCache s of
 		Just v -> return v
 		Nothing -> do
+			-- A given key may be referenced repeatedly,
+			-- so nub is needed for accuracy, but is slow.
 			keys <- lift Command.Unused.getKeysReferenced
-			-- A given key may be referenced repeatedly.
-			-- nub does not seem too slow (yet)..
-			let v = sizeList $ nub keys
+			fast <- lift $ Annex.getState Annex.fast
+			let v = sizeList $ if fast then keys else nub keys
 			put s { keysReferencedCache = Just v }
 			return v
 
@@ -175,7 +181,7 @@ staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else stat label $ do
+		else stat label False $ do
 			s <- keySizeSum $ sizeList keys
 			return $ s ++ aside "clean up with git-annex unused"
 
diff --git a/debian/changelog b/debian/changelog
index fc57fb361c..938e2feaf3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,8 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     match the specified conditions.
   * Note that this is a behavior change for git-annex find! Old behavior
     can be gotten by using: git-annex find --in .
+  * status: In --fast mode, all status info is displayed now; but some
+    of it is only approximate, and is marked as such.
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 4a8e28a6ee..26cfb2c613 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -240,8 +240,8 @@ subdirectories).
 
   Some of the statistics can take a while to generate, and those
   come last. You can ctrl-c this command once it's displayed the
-  information you wanted to see. Or, use --fast to only display
-  the first, fast(ish) statistics.
+  information you wanted to see. Or, use --fast to produce statistics
+  more quickly, but possibly less accurately.
 
 * map
 

From 9f5c7a246b786e350671551cafae0f9678d83648 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 18:57:05 -0400
Subject: [PATCH 2246/8313] status: Massively sped up; remove --fast mode.

Using Sets is the right thing; they have constant size lookup like my
SizeList, and logn insertation, which beats nub to death.

Runs faster than --fast mode did before, and gives accurate counts.

13 seconds total runtime with a warm cache in a repository with 40 thousand
keys.
---
 Command/Status.hs  | 96 ++++++++++++++++++----------------------------
 debian/changelog   |  3 +-
 doc/git-annex.mdwn |  5 ---
 3 files changed, 39 insertions(+), 65 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 067128f620..d06865b6aa 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -13,8 +13,9 @@ import Data.Maybe
 import System.IO
 import Data.List
 import qualified Data.Map as M
+import qualified Data.Set as S
+import Data.Set (Set)
 
-import qualified Annex
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
@@ -23,33 +24,23 @@ import qualified Git
 import Command
 import Types
 import Utility.DataUnits
-import Utility.Conditional
 import Content
 import Types.Key
 import Locations
 import Backend
-import Messages
 
 -- a named computation that produces a statistic
-type Stat = StatState (Maybe (String, Bool, StatState String))
+type Stat = StatState (Maybe (String, StatState String))
 
 -- cached info that multiple Stats may need
 data StatInfo = StatInfo
-	{ keysPresentCache :: Maybe (SizeList Key)
-	, keysReferencedCache :: Maybe (SizeList Key)
+	{ keysPresentCache :: Maybe (Set Key)
+	, keysReferencedCache :: Maybe (Set Key)
 	}
 
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
--- a list with a known length
--- (Integer is used for the length to avoid
--- blowing up if someone annexed billions of files..)
-type SizeList a = ([a], Integer)
-
-sizeList :: [a] -> SizeList a
-sizeList l = (l, genericLength l)
-
 command :: [Command]
 command = [repoCommand "status" paramNothing seek
 	"shows status information about the annex"]
@@ -76,15 +67,10 @@ stats =
 start :: CommandStart
 start = do
 	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
-	fastmode_note
 	stop
 
-fastmode_note :: Annex ()
-fastmode_note = whenM (Annex.getState Annex.fast) $
-	showLongNote "(*) approximate due to fast mode"
-
-stat :: String -> Bool -> StatState String -> Stat
-stat desc approx a = return $ Just (desc, approx, a)
+stat :: String -> StatState String -> Stat
+stat desc a = return $ Just (desc, a)
 
 nostat :: Stat
 nostat = return Nothing
@@ -92,37 +78,35 @@ nostat = return Nothing
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
-		calc (Just (desc, approx, a)) = do
-			fast <- lift $ Annex.getState Annex.fast
-			let star = if fast && approx then "(*)" else ""
-			liftIO $ putStr $ desc ++ star ++ ": "
+		calc (Just (desc, a)) = do
+			liftIO $ putStr $ desc ++ ": "
 			liftIO $ hFlush stdout
 			liftIO . putStrLn =<< a
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" False $ 
+supported_backends = stat "supported backends" $ 
 	return $ unwords $ map B.name Backend.list
 
 supported_remote_types :: Stat
-supported_remote_types = stat "supported remote types" False $
+supported_remote_types = stat "supported remote types" $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
 local_annex_size :: Stat
-local_annex_size = stat "local annex size" False $
+local_annex_size = stat "local annex size" $
 	cachedKeysPresent >>= keySizeSum
 
 total_annex_size :: Stat
-total_annex_size = stat "total annex size" True $
+total_annex_size = stat "total annex size" $
 	cachedKeysReferenced >>= keySizeSum
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" False $
-	show . snd <$> cachedKeysPresent
+local_annex_keys = stat "local annex keys" $
+	show . S.size <$> cachedKeysPresent
 
 total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" True $
-	show . snd <$> cachedKeysReferenced
+total_annex_keys = stat "total annex keys" $
+	show . S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -131,9 +115,9 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" True $ usage <$> cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
-		usage (ks, _) = pp "" $ sort $ map swap $ splits ks
+		usage ks = pp "" $ sort $ map swap $ splits $ S.toList ks
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
@@ -141,48 +125,44 @@ backend_usage = stat "backend usage" True $ usage <$> cachedKeysReferenced
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
 
-cachedKeysPresent :: StatState (SizeList Key)
+cachedKeysPresent :: StatState (Set Key)
 cachedKeysPresent = do
 	s <- get
 	case keysPresentCache s of
 		Just v -> return v
 		Nothing -> do
-			keys <- lift getKeysPresent
-			let v = sizeList keys
-			put s { keysPresentCache = Just v }
-			return v
+			keys <- S.fromList <$> lift getKeysPresent
+			put s { keysPresentCache = Just keys }
+			return keys
 
-cachedKeysReferenced :: StatState (SizeList Key)
+cachedKeysReferenced :: StatState (Set Key)
 cachedKeysReferenced = do
 	s <- get
 	case keysReferencedCache s of
 		Just v -> return v
 		Nothing -> do
-			-- A given key may be referenced repeatedly,
-			-- so nub is needed for accuracy, but is slow.
-			keys <- lift Command.Unused.getKeysReferenced
-			fast <- lift $ Annex.getState Annex.fast
-			let v = sizeList $ if fast then keys else nub keys
-			put s { keysReferencedCache = Just v }
-			return v
+			keys <- S.fromList <$> lift Command.Unused.getKeysReferenced
+			put s { keysReferencedCache = Just keys }
+			return keys
 
-keySizeSum :: SizeList Key -> StatState String
-keySizeSum (keys, len) = do
-	let knownsizes = mapMaybe keySize keys
-	let total = roughSize storageUnits False $ sum knownsizes
-	let missing = len - genericLength knownsizes
+keySizeSum :: Set Key -> StatState String
+keySizeSum s = do
+	let (sizes, unknownsizes) = S.partition isJust $ S.map keySize s
+	let total = roughSize storageUnits False $
+		fromJust $ S.fold (liftM2 (+)) (Just 0) sizes
+	let num = S.size unknownsizes
 	return $ total ++
-		if missing > 0
-			then aside $ "but " ++ show missing ++ " keys have unknown size"
-			else ""
+		if num == 0
+			then ""
+			else aside $ "but " ++ show num ++ " keys have unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else stat label False $ do
-			s <- keySizeSum $ sizeList keys
+		else stat label $ do
+			s <- keySizeSum $ S.fromList keys
 			return $ s ++ aside "clean up with git-annex unused"
 
 aside :: String -> String
diff --git a/debian/changelog b/debian/changelog
index 938e2feaf3..4807956ad9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,8 +12,7 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     match the specified conditions.
   * Note that this is a behavior change for git-annex find! Old behavior
     can be gotten by using: git-annex find --in .
-  * status: In --fast mode, all status info is displayed now; but some
-    of it is only approximate, and is marked as such.
+  * status: Massively sped up; remove --fast mode.
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 26cfb2c613..59da9dd8cf 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -238,11 +238,6 @@ subdirectories).
   Displays some statistics and other information, including how much data
   is in the annex.
 
-  Some of the statistics can take a while to generate, and those
-  come last. You can ctrl-c this command once it's displayed the
-  information you wanted to see. Or, use --fast to produce statistics
-  more quickly, but possibly less accurately.
-
 * map
 
   Helps you keep track of your repositories, and the connections between them,

From 98fbeba0df112753e0c55b0e2f456cd89ed0ede7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 20:14:17 -0400
Subject: [PATCH 2247/8313] bugfix

Different keys can have the same size, so can't make a Set of the sizes.

This version actually runs faster yet, too..
---
 Command/Status.hs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index d06865b6aa..26122f2694 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -147,14 +147,13 @@ cachedKeysReferenced = do
 
 keySizeSum :: Set Key -> StatState String
 keySizeSum s = do
-	let (sizes, unknownsizes) = S.partition isJust $ S.map keySize s
-	let total = roughSize storageUnits False $
-		fromJust $ S.fold (liftM2 (+)) (Just 0) sizes
-	let num = S.size unknownsizes
+	let knownsizes = mapMaybe keySize $ S.toList s
+	let total = roughSize storageUnits False $ sum knownsizes
+	let missing = S.size s - genericLength knownsizes
 	return $ total ++
-		if num == 0
+		if missing == 0
 			then ""
-			else aside $ "but " ++ show num ++ " keys have unknown size"
+			else aside $ "but " ++ show missing ++ " keys have unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do

From 9d26192350b9c7b0402720f6f29c99c24748f364 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 20:18:43 -0400
Subject: [PATCH 2248/8313] pull out pure code

---
 Command/Status.hs | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 26122f2694..fc306bbe5e 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -94,11 +94,11 @@ supported_remote_types = stat "supported remote types" $
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
-	cachedKeysPresent >>= keySizeSum
+	cachedKeysPresent >>= return . keySizeSum
 
 total_annex_size :: Stat
 total_annex_size = stat "total annex size" $
-	cachedKeysReferenced >>= keySizeSum
+	cachedKeysReferenced >>= return . keySizeSum
 
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $
@@ -145,15 +145,17 @@ cachedKeysReferenced = do
 			put s { keysReferencedCache = Just keys }
 			return keys
 
-keySizeSum :: Set Key -> StatState String
-keySizeSum s = do
-	let knownsizes = mapMaybe keySize $ S.toList s
-	let total = roughSize storageUnits False $ sum knownsizes
-	let missing = S.size s - genericLength knownsizes
-	return $ total ++
-		if missing == 0
-			then ""
-			else aside $ "but " ++ show missing ++ " keys have unknown size"
+keySizeSum :: Set Key -> String
+keySizeSum s = total ++ missingnote
+	where
+		knownsizes = mapMaybe keySize $ S.toList s
+		total = roughSize storageUnits False $ sum knownsizes
+		missing = S.size s - genericLength knownsizes
+		missingnote
+			| missing == 0 = ""
+			| otherwise = aside $
+				"but " ++ show missing ++
+				" keys have unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do
@@ -161,7 +163,7 @@ staleSize label dirspec = do
 	if null keys
 		then nostat
 		else stat label $ do
-			s <- keySizeSum $ S.fromList keys
+			let s = keySizeSum $ S.fromList keys
 			return $ s ++ aside "clean up with git-annex unused"
 
 aside :: String -> String

From 9f6b7935ddb3d5dcbe0b4b784dc8acd7288ddba6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 23:24:48 -0400
Subject: [PATCH 2249/8313] go go gadget hlint

---
 Backend.hs            |  2 +-
 Backend/SHA.hs        |  2 +-
 Command.hs            |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/InitRemote.hs |  2 +-
 Command/Migrate.hs    |  3 ++-
 Command/Status.hs     |  4 ++--
 Git.hs                | 13 ++++++-------
 GitAnnex.hs           |  6 +++---
 Init.hs               |  3 +--
 Limit.hs              |  5 ++---
 Options.hs            |  4 ++--
 Remote/Git.hs         |  4 ++--
 Remote/S3real.hs      |  4 ++--
 Upgrade/V1.hs         |  4 ++--
 Utility/JSONStream.hs |  2 +-
 Utility/Matcher.hs    |  2 +-
 Utility/Path.hs       |  2 +-
 Utility/Touch.hsc     |  2 +-
 configure.hs          |  2 +-
 git-union-merge.hs    |  2 +-
 21 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 0c9ea8d0b7..d129139850 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -111,7 +111,7 @@ chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = do
 	g <- Annex.gitRepo
 	forced <- Annex.getState Annex.forcebackend
-	if forced /= Nothing
+	if isJust forced
 		then do
 			l <- orderedList
 			return $ map (\f -> (Just $ head l, f)) fs
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index ed2a47db9b..15d3fa20d1 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -38,7 +38,7 @@ backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
 genBackend :: SHASize -> Maybe (Backend Annex)
 genBackend size
-	| shaCommand size == Nothing = Nothing
+	| isNothing (shaCommand size) = Nothing
 	| otherwise = Just b
 	where
 		b = Types.Backend.Backend
diff --git a/Command.hs b/Command.hs
index cc9bcbf0c9..c061c7c464 100644
--- a/Command.hs
+++ b/Command.hs
@@ -162,7 +162,7 @@ withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
 
 runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
-runFiltered a fs = runFilteredGen a id fs
+runFiltered a = runFilteredGen a id
 
 backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
 backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index d9fcc17e2b..2e9e04fd3e 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -28,7 +28,7 @@ import Utility.Path
 import Utility.Conditional
 
 command :: [Command]
-command = [repoCommand "addurl" (paramRepeating $ paramUrl) seek
+command = [repoCommand "addurl" (paramRepeating paramUrl) seek
 	"add urls to annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 671f945d22..c6d9f52003 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -35,7 +35,7 @@ start ws = do
 	when (null ws) needname
 
 	(u, c) <- findByName name
-	let fullconfig = M.union config c	
+	let fullconfig = config `M.union` c	
 	t <- findType fullconfig
 
 	showStart "initremote" name
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 2be9108512..054db6e27b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -12,6 +12,7 @@ import Control.Applicative
 import System.Posix.Files
 import System.Directory
 import System.FilePath
+import Data.Maybe
 
 import Command
 import qualified Annex
@@ -48,7 +49,7 @@ start (b, file) = isAnnexed file $ \(key, oldbackend) -> do
 {- Checks if a key is upgradable to a newer representation. -}
 {- Ideally, all keys have file size metadata. Old keys may not. -}
 upgradableKey :: Key -> Bool
-upgradableKey key = Types.Key.keySize key == Nothing
+upgradableKey key = isNothing $ Types.Key.keySize key
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
diff --git a/Command/Status.hs b/Command/Status.hs
index fc306bbe5e..6da8064f85 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -94,11 +94,11 @@ supported_remote_types = stat "supported remote types" $
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
-	cachedKeysPresent >>= return . keySizeSum
+	keySizeSum <$> cachedKeysPresent
 
 total_annex_size :: Stat
 total_annex_size = stat "total annex size" $
-	cachedKeysReferenced >>= return . keySizeSum
+	keySizeSum <$> cachedKeysReferenced
 
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $
diff --git a/Git.hs b/Git.hs
index cd6cdfbfd0..86a8c7695c 100644
--- a/Git.hs
+++ b/Git.hs
@@ -62,7 +62,7 @@ module Git (
 	prop_idempotent_deencode
 ) where
 
-import Control.Monad (unless, when)
+import Control.Monad (unless, when, liftM2)
 import Control.Applicative
 import System.Directory
 import System.FilePath
@@ -425,7 +425,7 @@ getSha :: String -> IO String -> IO String
 getSha subcommand a = do
 	t <- a
 	let t' = if last t == '\n'
-		then take (length t - 1) t
+		then init t
 		else t
 	when (length t' /= shaSize) $
 		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
@@ -576,7 +576,7 @@ decodeGitFile f@(c:s)
 	| otherwise = f
 	where
 		e = '\\'
-		middle = take (length s - 1) s
+		middle = init s
 		unescape (b, []) = b
 		-- look for escapes starting with '\'
 		unescape (b, v) = b ++ beginning ++ unescape (decode rest)
@@ -702,7 +702,6 @@ isRepoTop dir = do
 	where
 		isRepo = gitSignature ".git" ".git/config"
 		isBareRepo = gitSignature "objects" "config"
-		gitSignature subdir file = do
-			s <- (doesDirectoryExist (dir ++ "/" ++ subdir))
-			f <- (doesFileExist (dir ++ "/" ++ file))
-			return (s && f)
+		gitSignature subdir file = liftM2 (&&)
+			(doesDirectoryExist (dir ++ "/" ++ subdir))
+			(doesFileExist (dir ++ "/" ++ file))
diff --git a/GitAnnex.hs b/GitAnnex.hs
index a284daad5c..a9d469b44e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -108,11 +108,11 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option ['x'] ["exclude"] (ReqArg (Limit.addExclude) paramGlob)
+	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
-	, Option ['i'] ["in"] (ReqArg (Limit.addIn) paramRemote)
+	, Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote)
 		"skip files not present in a remote"
-	, Option ['C'] ["copies"] (ReqArg (Limit.addCopies) paramNumber)
+	, Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber)
 		"skip files with fewer copies"
 	] ++ matcherOptions
 	where
diff --git a/Init.hs b/Init.hs
index 2067c524cf..4df1599333 100644
--- a/Init.hs
+++ b/Init.hs
@@ -33,8 +33,7 @@ initialize = do
 	gitPreCommitHookWrite
 
 uninitialize :: Annex ()
-uninitialize = do
-	gitPreCommitHookUnWrite
+uninitialize = gitPreCommitHookUnWrite
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
diff --git a/Limit.hs b/Limit.hs
index b3b041396f..10fc0ea6ca 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -69,7 +69,7 @@ addExclude glob = addLimit $ return . notExcluded
 addIn :: String -> Annex ()
 addIn name = do
 	u <- Remote.nameToUUID name
-	addLimit $ if name == "." then check local else check (remote u)
+	addLimit $ if name == "." then check inAnnex else check (remote u)
 	where
 		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
@@ -77,12 +77,11 @@ addIn name = do
 		remote u key = do
 			us <- keyLocations key
 			return $ u `elem` us
-		local key = inAnnex key
 
 {- Adds a limit to skip files not believed to have the specified number
  - of copies. -}
 addCopies :: String -> Annex ()
-addCopies num = do
+addCopies num =
 	case readMaybe num :: Maybe Int of
 		Nothing -> error "bad number for --copies"
 		Just n -> addLimit $ check n
diff --git a/Options.hs b/Options.hs
index eeb3639b4d..b5eaf98cd8 100644
--- a/Options.hs
+++ b/Options.hs
@@ -58,5 +58,5 @@ matcherOptions =
 	, shortopt ")" "close group of options"
 	]
 	where
-		longopt o d = Option [] [o] (NoArg (addToken o)) d
-		shortopt o d = Option o [] (NoArg (addToken o)) d
+		longopt o = Option [] [o] $ NoArg $ addToken o
+		shortopt o = Option o [] $ NoArg $ addToken o
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9789a06252..d50899c674 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -81,7 +81,7 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.configMap r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsHttp r = store $ safely $ geturlconfig
+	| Git.repoIsHttp r = store $ safely geturlconfig
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ do
 		onLocal r ensureInitialized
@@ -101,7 +101,7 @@ tryGitConfigRead r
 
 		geturlconfig = do
 			s <- Url.get (Git.repoLocation r ++ "/config")
-			withTempFile "git-annex.tmp" $ \tmpfile -> \h -> do
+			withTempFile "git-annex.tmp" $ \tmpfile h -> do
 				hPutStr h s
 				hClose h
 				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 77b6b6ca40..cafa4f15a8 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -95,7 +95,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 		defaulthost = do
 			c' <- encryptionSetup c
-			let fullconfig = M.union c' defaults
+			let fullconfig = c' `M.union` defaults
 			genBucket fullconfig
 			use fullconfig
 
@@ -209,7 +209,7 @@ s3Bool (Left e) = s3Warning e
 
 s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r noconn action = do
-	when (config r == Nothing) $
+	when (isNothing $ config r) $
 		error $ "Missing configuration for special remote " ++ name r
 	let bucket = M.lookup "bucket" $ fromJust $ config r
 	conn <- s3Connection $ fromJust $ config r
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 9c3fd99595..78f7d3adbc 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -173,7 +173,7 @@ readKey1 v =
 			then Just (read (bits !! 2) :: Integer)
 			else Nothing
 		wormy = head bits == "WORM"
-		mixup = wormy && (isUpper $ head $ bits !! 1)
+		mixup = wormy && isUpper (head $ bits !! 1)
 
 showKey1 :: Key -> String
 showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =
@@ -248,7 +248,7 @@ logFile' hasher repo key =
 	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
 
 stateDir :: FilePath
-stateDir = addTrailingPathSeparator $ ".git-annex"
+stateDir = addTrailingPathSeparator ".git-annex"
 
 gitStateDir :: Git.Repo -> FilePath
 gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
diff --git a/Utility/JSONStream.hs b/Utility/JSONStream.hs
index af3766948f..7910c11941 100644
--- a/Utility/JSONStream.hs
+++ b/Utility/JSONStream.hs
@@ -19,7 +19,7 @@ import Text.JSON
    later. -}
 start :: JSON a => [(String, a)] -> String
 start l
-	| last s == endchar = take (length s - 1)  s
+	| last s == endchar = init s
 	| otherwise = bad s
 	where
 		s = encodeStrict $ toJSObject l
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 69b78be4a0..01500a2111 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -63,7 +63,7 @@ consume m (t:ts) = go t
 	where
 		go And = cont $ m `MAnd` next
 		go Or = cont $ m `MOr` next
-		go Not = cont $ m `MAnd` (MNot next)
+		go Not = cont $ m `MAnd` MNot next
 		go Open = let (n, r) = consume next rest in (m `MAnd` n, r)
 		go Close = (m, ts)
 		go (Operation o) = (m `MAnd` MOp o, ts)
diff --git a/Utility/Path.hs b/Utility/Path.hs
index fe474ee825..ce54fb3695 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -19,7 +19,7 @@ import Control.Applicative
 parentDir :: FilePath -> FilePath
 parentDir dir =
 	if not $ null dirs
-	then slash ++ join s (take (length dirs - 1) dirs)
+	then slash ++ join s (init dirs)
 	else ""
 		where
 			dirs = filter (not . null) $ split s dir
diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index f27ac31360..fd3320cd1d 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -24,7 +24,7 @@ newtype TimeSpec = TimeSpec CTime
 touchBoth :: FilePath -> TimeSpec -> TimeSpec -> Bool -> IO ()
 
 touch :: FilePath -> TimeSpec -> Bool -> IO ()
-touch file mtime follow = touchBoth file mtime mtime follow
+touch file mtime = touchBoth file mtime mtime
 
 #include 
 #include 
diff --git a/configure.hs b/configure.hs
index 9f7179c539..b68fa12dbc 100644
--- a/configure.hs
+++ b/configure.hs
@@ -51,7 +51,7 @@ getVersionString = do
 	let verline = head $ lines changelog
 	return $ middle (words verline !! 1)
 	where
-		middle s = drop 1 $ take (length s - 1) s
+		middle = drop 1 . init
 
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 4e1a932b45..8b70e678c9 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -23,7 +23,7 @@ tmpIndex :: Git.Repo -> FilePath
 tmpIndex g = Git.gitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
-setup g = cleanup g -- idempotency
+setup = cleanup -- idempotency
 
 cleanup :: Git.Repo -> IO ()
 cleanup g = do

From 6dc23b889e01d56a7aff241f9b7e347dd55cfc47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Sep 2011 23:26:35 -0400
Subject: [PATCH 2250/8313] one more hlint

---
 Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Branch.hs b/Branch.hs
index 5008b2e200..15681e6993 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -331,7 +331,7 @@ getJournalFilesRaw = do
 	g <- Annex.gitRepo
 	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
 		(const $ return [])
-	return $ filter (\f -> f /= "." && f /= "..") fs
+	return $ filter (`notElem` [".", ".."]) fs
 
 {- Stages all journal files into the index, and returns True if the index
  - was modified. -}

From d75da353b9905bb5757df08520e63607fbfd2073 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Sep 2011 18:04:38 -0400
Subject: [PATCH 2251/8313] documentation/warning message update for future
 feature

---
 Command/Unused.hs        | 24 +++++++++---------------
 debian/changelog         |  4 ++++
 doc/git-annex.mdwn       |  5 ++---
 doc/special_remotes.mdwn |  6 ------
 4 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 535b9b33e8..803debef82 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -112,21 +112,8 @@ staleBadMsg t = unlines $
 
 unusedMsg :: [(Int, Key)] -> String
 unusedMsg u = unusedMsg' u
-	["Some annexed data is no longer used by any files in the current branch:"]
-	[dropMsg Nothing,
-	"Please be cautious -- are you sure that another branch, or another",
-	"repository does not still use this data?"]
-
-remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
-remoteUnusedMsg r u = unusedMsg' u
-	["Some annexed data on " ++ name ++ 
-	 " is not used by any files in the current branch:"]
-	[dropMsg $ Just r,
-	 "Please be cautious -- Are you sure that the remote repository",
-	 "does not use this data? Or that it's not used by another branch?"]
-	where
-		name = Remote.name r 
-
+	["Some annexed data is no longer used by any files:"]
+	[dropMsg Nothing]
 unusedMsg' :: [(Int, Key)] -> [String] -> [String] -> String
 unusedMsg' u header trailer = unlines $
 	header ++
@@ -134,6 +121,13 @@ unusedMsg' u header trailer = unlines $
 	["(To see where data was previously used, try: git log --stat -S'KEY')"] ++
 	trailer
 
+remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
+remoteUnusedMsg r u = unusedMsg' u
+	["Some annexed data on " ++ name ++ " is not used by any files:"]
+	[dropMsg $ Just r]
+	where
+		name = Remote.name r 
+
 dropMsg :: Maybe (Remote.Remote Annex) -> String
 dropMsg Nothing = dropMsg' ""
 dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
diff --git a/debian/changelog b/debian/changelog
index 4807956ad9..4bc442d511 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,10 @@ git-annex (3.20110916) UNRELEASED; urgency=low
   * Note that this is a behavior change for git-annex find! Old behavior
     can be gotten by using: git-annex find --in .
   * status: Massively sped up; remove --fast mode.
+  * unused: File contents used by branches and tags are no longer
+    considered unused, even when not used by the current branch. This is
+    the final piece of the puzzle needed for git-annex to to play nicely
+    with branches.
 
  -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 59da9dd8cf..714f23befa 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -182,12 +182,11 @@ subdirectories).
 * unused
 
   Checks the annex for data that does not correspond to any files present
-  in the currently checked out branch, and prints a numbered list of the data.
+  in any tar or branch, and prints a numbered list of the data.
 
   To only show unused temp and bad files, specify --fast
 
-  To check data on a remote that does not correspond to any files present
-  on the locally checked out branch, specify --from.
+  To check for annexed data on a remote, specify --from.
 
 * dropunused [number ...]
 
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 55cd1f1a0f..ddb2fd125d 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -29,11 +29,5 @@ on special remotes, instead use `git annex unused --from`. Example:
 	    1       WORM-s3-m1301674316--foo
 	  (To see where data was previously used, try: git log --stat -S'KEY')
 	  (To remove unwanted data: git-annex dropunused --from mys3 NUMBER)
-	  Please be cautious -- are you sure that the remote repository
-	  does not use this data?
 	$ git annex dropunused --from mys3 1
 	dropunused 12948 (from mys3...) ok
-
-Do be cautious when using this; it cannot detect if content in a remote
-is used by that remote, or is the last copy of data that is used by
-some *other* remote.

From 4bf1a5ef59026a095abf751ea60b586c299aa0b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Sep 2011 18:13:24 -0400
Subject: [PATCH 2252/8313] refactor

---
 Command/Fsck.hs     |  1 +
 Command/Unannex.hs  |  1 +
 Command/Unlock.hs   |  1 +
 Command/Unused.hs   |  2 +-
 Content.hs          | 16 +---------------
 Remote/Directory.hs |  2 +-
 Upgrade/V1.hs       |  1 +
 Utility.hs          | 11 -----------
 Utility/FileMode.hs | 32 ++++++++++++++++++++++++++++++++
 9 files changed, 39 insertions(+), 28 deletions(-)
 create mode 100644 Utility/FileMode.hs

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 142f755a7f..0c58add6a3 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -26,6 +26,7 @@ import Locations
 import Trust
 import Utility.DataUnits
 import Utility.Path
+import Utility.FileMode
 import Config
 
 command :: [Command]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 3dedd007ef..4d4281eb07 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -18,6 +18,7 @@ import qualified Annex
 import qualified AnnexQueue
 import Utility.SafeCommand
 import Utility.Path
+import Utility.FileMode
 import LocationLog
 import Types
 import Content
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 5817e8f221..44b92545c8 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -19,6 +19,7 @@ import Content
 import Utility.Conditional
 import Utility.CopyFile
 import Utility.Path
+import Utility.FileMode
 
 command :: [Command]
 command =
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 803debef82..f62e68c30f 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -184,7 +184,7 @@ getKeysReferenced = do
 staleKeysPrune :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
 staleKeysPrune dirspec present = do
 	contents <- staleKeys dirspec
-		
+	
 	let stale = contents `exclude` present
 	let dup = contents `exclude` stale
 
diff --git a/Content.hs b/Content.hs
index e4bbee5282..f963c48b47 100644
--- a/Content.hs
+++ b/Content.hs
@@ -13,8 +13,6 @@ module Content (
 	getViaTmpUnchecked,
 	withTmp,
 	checkDiskSpace,
-	preventWrite,
-	allowWrite,
 	moveAnnex,
 	removeAnnex,
 	fromAnnex,
@@ -43,6 +41,7 @@ import Utility
 import Utility.Conditional
 import Utility.StatFS
 import Utility.Path
+import Utility.FileMode
 import Types.Key
 import Utility.DataUnits
 import Config
@@ -152,19 +151,6 @@ checkDiskSpace' adjustment key = do
 				roughSize storageUnits True n ++
 				" more (use --force to override this check or adjust annex.diskreserve)"
 
-{- Removes the write bits from a file. -}
-preventWrite :: FilePath -> IO ()
-preventWrite f = unsetFileMode f writebits
-	where
-		writebits = foldl unionFileModes ownerWriteMode
-					[groupWriteMode, otherWriteMode]
-
-{- Turns a file's write bit back on. -}
-allowWrite :: FilePath -> IO ()
-allowWrite f = do
-	s <- getFileStatus f
-	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode
-
 {- Moves a file into .git/annex/objects/
  -
  - What if the key there already has content? This could happen for
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index b183042ef6..18835c5de0 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -25,10 +25,10 @@ import UUID
 import Locations
 import Utility.CopyFile
 import Config
-import Content
 import Utility
 import Utility.Conditional
 import Utility.Path
+import Utility.FileMode
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 78f7d3adbc..329f90ed6c 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -32,6 +32,7 @@ import Backend
 import Messages
 import Version
 import Utility
+import Utility.FileMode
 import Utility.SafeCommand
 import Utility.Path
 import qualified Upgrade.V2
diff --git a/Utility.hs b/Utility.hs
index ce17363488..a3d461d286 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -8,7 +8,6 @@
 module Utility (
 	hGetContentsStrict,
 	readFileStrict,
-	unsetFileMode,
 	readMaybe,
 	viaTmp,
 	withTempFile,
@@ -24,12 +23,9 @@ module Utility (
 import IO (bracket)
 import System.IO
 import System.Posix.Process hiding (executeFile)
-import System.Posix.Files
-import System.Posix.Types
 import System.Posix.User
 import System.FilePath
 import System.Directory
-import Foreign (complement)
 import Utility.Path
 import Data.Maybe
 import Control.Monad (liftM)
@@ -43,13 +39,6 @@ hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict f = readFile f >>= \s -> length s `seq` return s
 
-{- Removes a FileMode from a file.
- - For example, call with otherWriteMode to chmod o-w -}
-unsetFileMode :: FilePath -> FileMode -> IO ()
-unsetFileMode f m = do
-	s <- getFileStatus f
-	setFileMode f $ fileMode s `intersectFileModes` complement m
-
 {- Attempts to read a value from a String. -}
 readMaybe :: (Read a) => String -> Maybe a
 readMaybe s = case reads s of
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
new file mode 100644
index 0000000000..f5b018c84a
--- /dev/null
+++ b/Utility/FileMode.hs
@@ -0,0 +1,32 @@
+{- File mode utilities.
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.FileMode where
+
+import System.Posix.Files
+import System.Posix.Types
+import Foreign (complement)
+
+{- Removes a FileMode from a file.
+ - For example, call with otherWriteMode to chmod o-w -}
+unsetFileMode :: FilePath -> FileMode -> IO ()
+unsetFileMode f m = do
+	s <- getFileStatus f
+	setFileMode f $ fileMode s `intersectFileModes` complement m
+
+{- Removes the write bits from a file. -}
+preventWrite :: FilePath -> IO ()
+preventWrite f = unsetFileMode f writebits
+	where
+		writebits = foldl unionFileModes ownerWriteMode
+					[groupWriteMode, otherWriteMode]
+
+{- Turns a file's write bit back on. -}
+allowWrite :: FilePath -> IO ()
+allowWrite f = do
+	s <- getFileStatus f
+	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode

From b203a68cb75c585b7e8b0b70f46278b41fef2fad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 25 Sep 2011 14:26:05 -0400
Subject: [PATCH 2253/8313] include --bindir in all cabal install examples

---
 doc/install.mdwn        | 2 +-
 doc/install/Fedora.mdwn | 2 +-
 doc/install/OSX.mdwn    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index cd51b96d23..6f892e37a2 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -8,7 +8,7 @@
 
 ## Using cabal
 
-As a haskell package, git-annex can be built using cabal. For example:
+As a haskell package, git-annex can be installed using cabal. For example:
 
 	cabal install git-annex --bindir=$HOME/bin
 
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 068d5c111c..7e983597b2 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -3,5 +3,5 @@ Installation recipe for Fedora 14.
 
 sudo yum install ghc cabal-install
 sudo cabal update
-sudo cabal install git-annex
+cabal install git-annex --bindir=$HOME/bin
 

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 680c331ee6..f65e0bb4fa 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -9,7 +9,7 @@ sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that al
 export PATH=$PATH:/opt/local/libexec/gnubin
 
 sudo cabal update
-sudo cabal install git-annex
+cabal install git-annex --bindir=$HOME/bin
 

 
 Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]

From b57a4566d3468f713ad369fc5f41778dfd133f0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 25 Sep 2011 14:34:07 -0400
Subject: [PATCH 2254/8313] mention that add --force adds ignored files

---
 doc/git-annex.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 59da9dd8cf..9ea7dce970 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -64,6 +64,7 @@ subdirectories).
 
   Adds files in the path to the annex. Files that are already checked into
   git, or that git has been configured to ignore will be silently skipped.
+  (Use --force to add ignored files.)
 
 * get [path ...]
 

From 7724f895a810d5922fb0581403ff17169344f514 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 25 Sep 2011 14:37:13 -0400
Subject: [PATCH 2255/8313] tweak

---
 Git/LsFiles.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 1ecbb029b5..c778e5d69c 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -23,13 +23,16 @@ inRepo :: Repo -> [FilePath] -> IO [FilePath]
 inRepo repo l = pipeNullSplit repo $
 	Params "ls-files --cached -z --" : map File l
 
-{- Scans for files at the specified locations that are not checked into
- - git. -}
+{- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
 notInRepo repo include_ignored l =
-	pipeNullSplit repo $ [Params "ls-files --others"]++exclude++[Params "-z --"] ++ map File l
+	pipeNullSplit repo $
+		[Params "ls-files --others"] ++ exclude ++
+		[Params "-z --"] ++ map File l
 	where
-		exclude = if include_ignored then [] else [Param "--exclude-standard"]
+		exclude
+			| include_ignored = []
+			| otherwise = [Param "--exclude-standard"]
 
 {- Returns a list of all files that are staged for commit. -}
 staged :: Repo -> [FilePath] -> IO [FilePath]

From 1c9c9a0cee33b63e9147863b95e570a8cc349304 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 02:35:23 -0400
Subject: [PATCH 2256/8313] golfing

---
 LocationLog.hs | 13 ++++++-------
 PresenceLog.hs | 20 ++++++++------------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/LocationLog.hs b/LocationLog.hs
index fa660c8b67..7e5e81d7af 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -23,7 +23,6 @@ module LocationLog (
 ) where
 
 import System.FilePath
-import Control.Monad (when)
 import Control.Applicative
 import Data.Maybe
 
@@ -36,16 +35,16 @@ import PresenceLog
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
-logChange repo key u s = do
-	when (null u) $
-		error $ "unknown UUID for " ++ Git.repoDescribe repo ++ 
-			" (have you run git annex init there?)"
-	addLog (logFile key) =<< logNow s u
+logChange repo key u s
+	| null u = error $
+		"unknown UUID for " ++ Git.repoDescribe repo ++ 
+		" (have you run git annex init there?)"
+	| otherwise = addLog (logFile key) =<< logNow s u
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = currentLog $ logFile key
+keyLocations = currentLog . logFile
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/PresenceLog.hs b/PresenceLog.hs
index e0c8729979..5828f76af5 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -85,7 +85,7 @@ readLog :: FilePath -> Annex [LogLine]
 readLog file = parseLog <$> Branch.get file
 
 parseLog :: String -> [LogLine]
-parseLog s = filter parsable $ map read $ lines s
+parseLog = filter parsable . map read . lines
 	where
 		-- some lines may be unparseable, avoid them
 		parsable l = status l /= Undefined
@@ -102,23 +102,18 @@ logNow s i = do
 
 {- Reads a log and returns only the info that is still in effect. -}
 currentLog :: FilePath -> Annex [String]
-currentLog file = do
-	ls <- readLog file
-	return $ map info $ filterPresent ls
+currentLog file = map info . filterPresent <$> readLog file
 
 {- Returns the info from LogLines that are in effect. -}
 filterPresent :: [LogLine] -> [LogLine]
-filterPresent ls = filter (\l -> InfoPresent == status l) $ compactLog ls
-
-type LogMap = Map.Map String LogLine
+filterPresent = filter (\l -> InfoPresent == status l) . compactLog
 
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
-compactLog = compactLog' Map.empty
-compactLog' :: LogMap -> [LogLine] -> [LogLine]
-compactLog' m [] = Map.elems m
-compactLog' m (l:ls) = compactLog' (mapLog m l) ls
+compactLog = Map.elems . foldl mapLog Map.empty
+
+type LogMap = Map.Map String LogLine
 
 {- Inserts a log into a map of logs, if the log has better (ie, newer)
  - information than the other logs in the map -}
@@ -128,5 +123,6 @@ mapLog m l =
 		then Map.insert i l m
 		else m
 	where
-		better = maybe True (\l' -> date l' <= date l) $ Map.lookup i m
+		better = maybe True newer $ Map.lookup i m
+		newer l' = date l' <= date l
 		i = info l

From d7d9e9aca08b752cc2e04809edc654be39413665 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 02:46:54 -0400
Subject: [PATCH 2257/8313] use a foldr

Should be faster here.
---
 PresenceLog.hs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/PresenceLog.hs b/PresenceLog.hs
index 5828f76af5..7742651b84 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -26,7 +26,7 @@ module PresenceLog (
 import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
-import qualified Data.Map as Map
+import qualified Data.Map as M
 import Control.Monad.State (liftIO)
 import Control.Applicative
 
@@ -111,18 +111,18 @@ filterPresent = filter (\l -> InfoPresent == status l) . compactLog
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
-compactLog = Map.elems . foldl mapLog Map.empty
+compactLog = M.elems . foldr mapLog M.empty
 
-type LogMap = Map.Map String LogLine
+type LogMap = M.Map String LogLine
 
 {- Inserts a log into a map of logs, if the log has better (ie, newer)
  - information than the other logs in the map -}
-mapLog :: LogMap -> LogLine -> LogMap
-mapLog m l = 
+mapLog :: LogLine -> LogMap -> LogMap
+mapLog l m = 
 	if better
-		then Map.insert i l m
+		then M.insert i l m
 		else m
 	where
-		better = maybe True newer $ Map.lookup i m
+		better = maybe True newer $ M.lookup i m
 		newer l' = date l' <= date l
 		i = info l

From 93807564d00b3f64ad2353731fffb5c45ef8c01a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 14:03:59 -0400
Subject: [PATCH 2258/8313] add ls-tree interface

This parser should be fast. I hope.
---
 Git/LsTree.hs | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 Git/LsTree.hs

diff --git a/Git/LsTree.hs b/Git/LsTree.hs
new file mode 100644
index 0000000000..8b530d2ad3
--- /dev/null
+++ b/Git/LsTree.hs
@@ -0,0 +1,48 @@
+{- git ls-tree interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.LsTree (
+	lsTree
+) where
+
+import Numeric
+import Control.Applicative
+import Data.Char
+
+import Git
+import Utility.SafeCommand
+
+type Treeish = String
+
+data TreeItem = TreeItem
+	{ mode :: Int
+	, objtype :: String
+	, sha :: String
+	, file :: FilePath
+	} deriving Show
+
+{- Lists the contents of a Treeish -}
+lsTree :: Repo -> Treeish -> IO [TreeItem]
+lsTree repo t = map parseLsTree <$>
+	pipeNullSplit repo [Params "ls-tree --full-tree -z -r --", File t]
+
+{- Parses a line of ls-tree output.
+ - (The --long format is not currently supported.) -}
+parseLsTree :: String -> TreeItem
+parseLsTree l = TreeItem m o s f
+	where
+		-- l =  SP  SP  TAB 
+		-- Since everything until the file is fixed-width,
+		-- do not need to split on words.
+		(m, past_m) = head $ readOct l
+		(o, past_o) = splitAt 4 $ space past_m
+		(s, past_s) = splitAt shaSize $ space past_o
+		f = decodeGitFile $ space past_s
+		space s@(sp:rest)
+			| isSpace sp = rest
+			| otherwise = error $
+				"ls-tree parse error at '" ++ s ++ "' in " ++ l

From a3cb5c47e5f4167c711ab57f4b06d6c9d56536c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 14:14:52 -0400
Subject: [PATCH 2259/8313] use FileMode

---
 Git/LsTree.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 8b530d2ad3..2220cfc505 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -12,6 +12,7 @@ module Git.LsTree (
 import Numeric
 import Control.Applicative
 import Data.Char
+import System.Posix.Types
 
 import Git
 import Utility.SafeCommand
@@ -19,7 +20,7 @@ import Utility.SafeCommand
 type Treeish = String
 
 data TreeItem = TreeItem
-	{ mode :: Int
+	{ mode :: FileMode
 	, objtype :: String
 	, sha :: String
 	, file :: FilePath

From 4f4eaf387ab801157cb8986a9ca3542a977e9e03 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 14:47:56 -0400
Subject: [PATCH 2260/8313] golf

---
 Git.hs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Git.hs b/Git.hs
index 86a8c7695c..b5464859e5 100644
--- a/Git.hs
+++ b/Git.hs
@@ -451,11 +451,8 @@ commit g message newref parentrefs = do
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it into a list of files/lines/whatever. -}
 pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
-pipeNullSplit repo params = do
-	fs0 <- pipeRead repo params
-	return $ split0 fs0
-	where
-		split0 s = filter (not . null) $ split "\0" s
+pipeNullSplit repo params = filter (not . null) . split "\0" <$>
+	pipeRead repo params
 
 {- Runs git config and populates a repo with its config. -}
 configRead :: Repo -> IO Repo

From ad245a6375b32a17a9aa18088ee006cad6b4c1ff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 15:15:42 -0400
Subject: [PATCH 2261/8313] refactor catfile code

split into generic IO code, and a thin Annex wrapper
---
 Annex.hs             |  3 +++
 Branch.hs            | 45 +++----------------------------
 CatFile.hs           | 26 ++++++++++++++++++
 Git/CatFile.hs       | 63 ++++++++++++++++++++++++++++++++++++++++++++
 Types/BranchState.hs |  7 +----
 5 files changed, 96 insertions(+), 48 deletions(-)
 create mode 100644 CatFile.hs
 create mode 100644 Git/CatFile.hs

diff --git a/Annex.hs b/Annex.hs
index 1517a34708..8a386a044b 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -24,6 +24,7 @@ import Control.Monad.IO.Control
 import Control.Applicative hiding (empty)
 
 import qualified Git
+import Git.CatFile
 import Git.Queue
 import Types.Backend
 import qualified Types.Remote
@@ -55,6 +56,7 @@ data AnnexState = AnnexState
 	, fast :: Bool
 	, auto :: Bool
 	, branchstate :: BranchState
+	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, defaultkey :: Maybe String
@@ -79,6 +81,7 @@ newState gitrepo = AnnexState
 	, fast = False
 	, auto = False
 	, branchstate = startBranchState
+	, catfilehandle = Nothing
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, defaultkey = Nothing
diff --git a/Branch.hs b/Branch.hs
index 15681e6993..af3851635d 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -18,7 +18,7 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (when, unless, liftM)
+import Control.Monad (unless, liftM)
 import Control.Monad.State (liftIO)
 import Control.Applicative ((<$>))
 import System.FilePath
@@ -31,7 +31,6 @@ import System.IO
 import System.IO.Binary
 import System.Posix.Process
 import System.Exit
-import qualified Data.ByteString.Char8 as B
 
 import Types.BranchState
 import qualified Git
@@ -43,6 +42,7 @@ import Utility.SafeCommand
 import Types
 import Messages
 import Locations
+import CatFile
 
 type GitRef = String
 
@@ -244,49 +244,10 @@ get file = do
 					setCache file content
 					return content
 				Nothing -> withIndexUpdate $ do
-					content <- catFile file
+					content <- catFile fullname file
 					setCache file content
 					return content
 
-{- Uses git cat-file in batch mode to read the content of a file.
- -
- - Only one process is run, and it persists and is used for all accesses. -}
-catFile :: FilePath -> Annex String
-catFile file = do
-	state <- getState
-	maybe (startup state) ask (catFileHandles state)
-	where
-		startup state = do
-			g <- Annex.gitRepo
-			(_, from, to) <- liftIO $ hPipeBoth "git" $
-				toCommand $ Git.gitCommandLine g
-					[Param "cat-file", Param "--batch"]
-			setState state { catFileHandles = Just (from, to) }
-			ask (from, to)
-		ask (from, to) = liftIO $ do
-			let want = fullname ++ ":" ++ file
-			hPutStrLn to want
-			hFlush to
-			header <- hGetLine from
-			case words header of
-				[sha, blob, size]
-					| length sha == Git.shaSize &&
-					  blob == "blob" -> handle from size
-					| otherwise -> empty
-				_
-					| header == want ++ " missing" -> empty
-					| otherwise -> error $ "unknown response from git cat-file " ++ header
-		handle from size = case reads size of
-			[(bytes, "")] -> readcontent from bytes
-			_ -> empty
-		readcontent from bytes = do
-			content <- B.hGet from bytes
-			c <- hGetChar from
-			when (c /= '\n') $
-				error "missing newline from git cat-file"
-			return $ B.unpack content
-		empty = return ""
-
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
diff --git a/CatFile.hs b/CatFile.hs
new file mode 100644
index 0000000000..0eb1e74f6b
--- /dev/null
+++ b/CatFile.hs
@@ -0,0 +1,26 @@
+{- git cat-file interface, with handle automatically stored in the Annex monad
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module CatFile (
+	catFile
+) where
+
+import Control.Monad.State
+
+import qualified Git.CatFile
+import Types
+import qualified Annex
+
+catFile :: String -> FilePath -> Annex String
+catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
+	where
+		startup = do
+			g <- Annex.gitRepo
+			h <- liftIO $ Git.CatFile.catFileStart g
+			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
+			go h
+		go h = liftIO $ Git.CatFile.catFile h branch file
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
new file mode 100644
index 0000000000..64857c66a9
--- /dev/null
+++ b/Git/CatFile.hs
@@ -0,0 +1,63 @@
+{- git cat-file interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.CatFile (
+	CatFileHandle,
+	catFileStart,
+	catFileStop,
+	catFile
+) where
+
+import Control.Monad.State
+import System.Cmd.Utils
+import System.IO
+import qualified Data.ByteString.Char8 as B
+
+import Git
+import Utility.SafeCommand
+
+type CatFileHandle = (PipeHandle, Handle, Handle)
+
+{- Starts git cat-file running in batch mode in a repo and returns a handle. -}
+catFileStart :: Repo -> IO CatFileHandle
+catFileStart repo = hPipeBoth "git" $ toCommand $
+	Git.gitCommandLine repo [Param "cat-file", Param "--batch"]
+
+{- Stops git cat-file. -}
+catFileStop :: CatFileHandle -> IO ()
+catFileStop (pid, from, to) = do
+	hClose to
+	hClose from
+	forceSuccess pid
+
+{- Uses a running git cat-file read the content of a file from a branch.
+ - Files that do not exist on the branch will have "" returned. -}
+catFile :: CatFileHandle -> String -> FilePath -> IO String
+catFile (_, from, to) branch file = do
+	hPutStrLn to want
+	hFlush to
+	header <- hGetLine from
+	case words header of
+		[sha, blob, size]
+			| length sha == Git.shaSize &&
+			  blob == "blob" -> handle size
+			| otherwise -> empty
+		_
+			| header == want ++ " missing" -> empty
+			| otherwise -> error $ "unknown response from git cat-file " ++ header
+	where
+		want = branch ++ ":" ++ file
+		handle size = case reads size of
+			[(bytes, "")] -> readcontent bytes
+			_ -> empty
+		readcontent bytes = do
+			content <- B.hGet from bytes
+			c <- hGetChar from
+			when (c /= '\n') $
+				error "missing newline from git cat-file"
+			return $ B.unpack content
+		empty = return ""
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index bc1d32e693..777edb32cb 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -7,18 +7,13 @@
 
 module Types.BranchState where
 
-import System.IO
-
 data BranchState = BranchState {
 	branchUpdated :: Bool, -- has the branch been updated this run?
 
-	-- (from, to) handles used to talk to a git-cat-file process
-	catFileHandles :: Maybe (Handle, Handle),
-
 	-- the content of one file is cached
 	cachedFile :: Maybe FilePath,
 	cachedContent :: String
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False Nothing Nothing ""
+startBranchState = BranchState False Nothing ""

From 5ae270001c978b306419e6799e67e5f14a1765a2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 15:17:45 -0400
Subject: [PATCH 2262/8313] fix

---
 test.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test.hs b/test.hs
index 4d751a707b..32278f0fcd 100644
--- a/test.hs
+++ b/test.hs
@@ -44,6 +44,7 @@ import qualified Types.Key
 import qualified Config
 import qualified Crypto
 import qualified Utility.Path
+import qualified Utility.FileMode
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -389,7 +390,7 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 
 		corrupt f = do
 			git_annex "get" ["-q", f] @? "get of file failed"
-			Content.allowWrite f
+			Utility.FileMode.allowWrite f
 			writeFile f (changedcontent f)
 			r <- git_annex "fsck" ["-q"]
 			not r @? "fsck failed to fail with corrupted file content"
@@ -558,7 +559,7 @@ cleanup dir = do
 		-- removed via directory permissions; undo
 		recurseDir SystemFS dir >>=
 			filterM doesDirectoryExist >>=
-			mapM_ Content.allowWrite
+			mapM_ Utility.FileMode.allowWrite
 		removeDirectoryRecursive dir
 	
 checklink :: FilePath -> Assertion

From 297bc648b9a3c1b950e65f23a0e974b7934dc4dd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 16:43:10 -0400
Subject: [PATCH 2263/8313] make unused check branches and tags too

needs time and space optimisation
---
 Backend.hs          |  5 ++---
 Command/Unused.hs   | 31 ++++++++++++++++++++++++++++++-
 Git/LsTree.hs       |  8 +++++---
 Utility/FileMode.hs |  4 ++++
 4 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index d129139850..ca822de5c1 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -17,6 +17,7 @@ module Backend (
 ) where
 
 import Control.Monad.State (liftIO, when)
+import Control.Applicative
 import System.IO.Error (try)
 import System.FilePath
 import System.Posix.Files
@@ -86,9 +87,7 @@ lookupFile file = do
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = do
-			l <- readSymbolicLink file
-			return $ takeFileName l
+		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
 		makeret l k =
 			case maybeLookupBackendName bname of
diff --git a/Command/Unused.hs b/Command/Unused.hs
index f62e68c30f..e629f9fb9e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -15,6 +15,8 @@ import qualified Data.Set as S
 import Data.Maybe
 import System.FilePath
 import System.Directory
+import Data.List
+import Control.Applicative
 
 import Command
 import Types
@@ -22,12 +24,17 @@ import Content
 import Messages
 import Locations
 import Utility
+import Utility.FileMode
+import Utility.SafeCommand
 import LocationLog
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
+import qualified Git.LsTree as LsTree
 import qualified Backend
 import qualified Remote
+import qualified Branch
+import CatFile
 
 command :: [Command]
 command = [repoCommand "unused" paramNothing seek
@@ -173,7 +180,29 @@ getKeysReferenced = do
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
-	return $ map fst $ catMaybes keypairs
+	ingit <- getKeysReferencedInGit
+	return $ concat [ingit, map fst $ catMaybes keypairs]
+
+{- List of keys referenced by symlinks in all git branches and tags. -}
+getKeysReferencedInGit :: Annex [Key]
+getKeysReferencedInGit = do
+	g <- Annex.gitRepo
+	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
+	-- Skip the git-annex branches, and get all other unique refs.
+	let refs = nub $ map head $ filter ourbranches $ map words $ lines c
+	concat <$> mapM (\r -> findkeys r [] =<< liftIO (LsTree.lsTree g r)) refs
+	where
+		ourbranchend = "/" ++ Branch.name
+		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
+		findkeys _ c [] = return c
+		findkeys ref c (l:ls) = do
+			if isSymLink (LsTree.mode l)
+				then do
+					content <- catFile ref $ LsTree.file l
+					case fileKey (takeFileName content) of
+						Nothing -> findkeys ref c ls
+						Just k -> findkeys ref (k:c) ls
+				else findkeys ref c ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 2220cfc505..4a6c509f94 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -6,6 +6,7 @@
  -}
 
 module Git.LsTree (
+	TreeItem(..),
 	lsTree
 ) where
 
@@ -43,7 +44,8 @@ parseLsTree l = TreeItem m o s f
 		(o, past_o) = splitAt 4 $ space past_m
 		(s, past_s) = splitAt shaSize $ space past_o
 		f = decodeGitFile $ space past_s
-		space s@(sp:rest)
+		space (sp:rest)
 			| isSpace sp = rest
-			| otherwise = error $
-				"ls-tree parse error at '" ++ s ++ "' in " ++ l
+			| otherwise = parseerr
+		space [] = parseerr
+		parseerr = "ls-tree parse error: " ++ l
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index f5b018c84a..6c1c06e82a 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -30,3 +30,7 @@ allowWrite :: FilePath -> IO ()
 allowWrite f = do
 	s <- getFileStatus f
 	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode
+
+{- Checks if a file mode indicates it's a symlink. -}
+isSymLink :: FileMode -> Bool
+isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode

From b4d5c10fb71a0aa938c7dde0b9aaf57d9e793874 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:35:47 -0400
Subject: [PATCH 2264/8313] refine new unused code

Fixed the laziness space leak, so it runs in 60 mb or so again. Slightly
faster due to using Data.Set.difference now, although this also makes it
use slightly more memory.

Also added display of the refs being checked, and made unused --from
also check all refs for things in the remote.
---
 Branch.hs         | 12 +--------
 Command/Unused.hs | 63 +++++++++++++++++++++++++++++++----------------
 Git.hs            |  9 +++++++
 3 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index af3851635d..92b1fe29e8 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -26,7 +26,6 @@ import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
 import Data.Maybe
-import Data.List
 import System.IO
 import System.IO.Binary
 import System.Posix.Process
@@ -58,15 +57,6 @@ fullname = "refs/heads/" ++ name
 originname :: GitRef
 originname = "origin/" ++ name
 
-{- Converts a fully qualified git ref into a short version for human
- - consumptiom. -}
-shortref :: GitRef -> String
-shortref = remove "refs/heads/" . remove "refs/remotes/"
-	where
-		remove prefix s
-			| prefix `isPrefixOf` s = drop (length prefix) s
-			| otherwise = s
-
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
 index g = gitAnnexDir g  "index"
@@ -209,7 +199,7 @@ updateRef ref
 		if null diffs
 			then return Nothing
 			else do
-				showSideAction $ "merging " ++ shortref ref ++ " into " ++ name
+				showSideAction $ "merging " ++ Git.refDescribe ref ++ " into " ++ name
 				-- By passing only one ref, it is actually
 				-- merged into the index, preserving any
 				-- changes that may already be staged.
diff --git a/Command/Unused.hs b/Command/Unused.hs
index e629f9fb9e..b15aa001ad 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,7 +16,6 @@ import Data.Maybe
 import System.FilePath
 import System.Directory
 import Data.List
-import Control.Applicative
 
 import Command
 import Types
@@ -76,9 +75,8 @@ checkRemoteUnused name = do
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
 	showAction "checking for unused data"
-	referenced <- getKeysReferenced
 	remotehas <- filterM isthere =<< loggedKeys
-	let remoteunused = remotehas `exclude` referenced
+	remoteunused <- excludeReferenced remotehas
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
 	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
@@ -156,12 +154,40 @@ unusedKeys = do
 		else do
 			showAction "checking for unused data"
 			present <- getKeysPresent
-			referenced <- getKeysReferenced
-			let unused = present `exclude` referenced
+			unused <- excludeReferenced present
 			staletmp <- staleKeysPrune gitAnnexTmpDir present
 			stalebad <- staleKeysPrune gitAnnexBadDir present
 			return (unused, stalebad, staletmp)
 
+{- Finds keys in the list that are not referenced in the git repository. -}
+excludeReferenced :: [Key] -> Annex [Key]
+-- excludeReferenced [] = return [] -- optimisation
+excludeReferenced l = do
+	g <- Annex.gitRepo
+	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
+	excludeReferenced'
+		(getKeysReferenced : (map getKeysReferencedInGit $ refs c))
+		(S.fromList l)
+	where
+		-- Skip the git-annex branches, and get all other unique refs.
+		refs = map last .
+			nubBy cmpheads .
+			filter ourbranches .
+			map words . lines
+		cmpheads a b = head a == head b
+		ourbranchend = "/" ++ Branch.name
+		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
+excludeReferenced' :: ([Annex [Key]]) -> S.Set Key -> Annex [Key]
+excludeReferenced' [] s = return $ S.toList s
+excludeReferenced' (a:as) s
+	-- | s == S.empty = return [] -- optimisation
+	| otherwise = do
+		referenced <- a
+		let !s' = remove referenced
+		excludeReferenced' as s'
+	where
+		remove l = s `S.difference` S.fromList l
+
 {- Finds items in the first, smaller list, that are not
  - present in the second, larger list.
  - 
@@ -180,29 +206,24 @@ getKeysReferenced = do
 	g <- Annex.gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
-	ingit <- getKeysReferencedInGit
-	return $ concat [ingit, map fst $ catMaybes keypairs]
+	return $ map fst $ catMaybes keypairs
 
-{- List of keys referenced by symlinks in all git branches and tags. -}
-getKeysReferencedInGit :: Annex [Key]
-getKeysReferencedInGit = do
+{- List of keys referenced by symlinks in a git ref. -}
+getKeysReferencedInGit :: String -> Annex [Key]
+getKeysReferencedInGit ref = do
+	showAction $ "checking " ++ Git.refDescribe ref
 	g <- Annex.gitRepo
-	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
-	-- Skip the git-annex branches, and get all other unique refs.
-	let refs = nub $ map head $ filter ourbranches $ map words $ lines c
-	concat <$> mapM (\r -> findkeys r [] =<< liftIO (LsTree.lsTree g r)) refs
+	findkeys [] =<< liftIO (LsTree.lsTree g ref)
 	where
-		ourbranchend = "/" ++ Branch.name
-		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
-		findkeys _ c [] = return c
-		findkeys ref c (l:ls) = do
+		findkeys c [] = return c
+		findkeys c (l:ls) = do
 			if isSymLink (LsTree.mode l)
 				then do
 					content <- catFile ref $ LsTree.file l
 					case fileKey (takeFileName content) of
-						Nothing -> findkeys ref c ls
-						Just k -> findkeys ref (k:c) ls
-				else findkeys ref c ls
+						Nothing -> findkeys c ls
+						Just k -> findkeys (k:c) ls
+				else findkeys c ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 
diff --git a/Git.hs b/Git.hs
index b5464859e5..fe2afdcfe3 100644
--- a/Git.hs
+++ b/Git.hs
@@ -20,6 +20,7 @@ module Git (
 	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
+	refDescribe,
 	repoLocation,
 	workTree,
 	workTreeFile,
@@ -171,6 +172,14 @@ repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
+{- Converts a fully qualified git ref into a user-visible version -}
+refDescribe :: String -> String
+refDescribe = remove "refs/heads/" . remove "refs/remotes/"
+	where
+		remove prefix s
+			| prefix `isPrefixOf` s = drop (length prefix) s
+			| otherwise = s
+
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url

From 8e4bd621b9e8ecafc65e029f015a4c460ec95abc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:38:41 -0400
Subject: [PATCH 2265/8313] enable short-circuiting optimisatons

---
 Command/Unused.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index b15aa001ad..e65f0790ba 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -161,7 +161,7 @@ unusedKeys = do
 
 {- Finds keys in the list that are not referenced in the git repository. -}
 excludeReferenced :: [Key] -> Annex [Key]
--- excludeReferenced [] = return [] -- optimisation
+excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
 	g <- Annex.gitRepo
 	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
@@ -180,7 +180,7 @@ excludeReferenced l = do
 excludeReferenced' :: ([Annex [Key]]) -> S.Set Key -> Annex [Key]
 excludeReferenced' [] s = return $ S.toList s
 excludeReferenced' (a:as) s
-	-- | s == S.empty = return [] -- optimisation
+	| s == S.empty = return [] -- optimisation
 	| otherwise = do
 		referenced <- a
 		let !s' = remove referenced

From 26bb45d12a26e51d2e41af0ed66e5300cbe52fee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:48:11 -0400
Subject: [PATCH 2266/8313] update test suite for smarter unused

---
 test.hs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 32278f0fcd..21e09e98e7 100644
--- a/test.hs
+++ b/test.hs
@@ -38,7 +38,6 @@ import qualified UUID
 import qualified Trust
 import qualified Remote
 import qualified RemoteLog
-import qualified Content
 import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
@@ -453,8 +452,14 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
 	checkunused []
 	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
+	checkunused []
+	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
+	checkunused []
+	-- unused checks origin/master; once it's gone it is really unused
+	boolSystem "git" [Params "remote rm origin"] @? "git remote rm origin failed"
 	checkunused [annexedfilekey]
 	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
+	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also

From 7d0adfc5e8bcb318cfedcfc43cd2dca35d037631 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 17:48:45 -0400
Subject: [PATCH 2267/8313] typo

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3ba58f25af..f64f84e088 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -183,7 +183,7 @@ subdirectories).
 * unused
 
   Checks the annex for data that does not correspond to any files present
-  in any tar or branch, and prints a numbered list of the data.
+  in any tag or branch, and prints a numbered list of the data.
 
   To only show unused temp and bad files, specify --fast
 

From ed00bdb9954028e123bb25bfe1e1c1d06a5aaba7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 18:11:53 -0400
Subject: [PATCH 2268/8313] foo

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index f64f84e088..db19e9f7a2 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -185,7 +185,7 @@ subdirectories).
   Checks the annex for data that does not correspond to any files present
   in any tag or branch, and prints a numbered list of the data.
 
-  To only show unused temp and bad files, specify --fast
+  To only show unused temp and bad files, specify --fast.
 
   To check for annexed data on a remote, specify --from.
 

From 7dddb803a07f4eb77f271a9e954d1e07f74de6df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 19:17:12 -0400
Subject: [PATCH 2269/8313] releasing version 3.20110928

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4bc442d511..f337882714 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20110916) UNRELEASED; urgency=low
+git-annex (3.20110928) unstable; urgency=low
 
   * --in can be used to make git-annex only operate on files
     believed to be present in a given repository.
@@ -18,7 +18,7 @@ git-annex (3.20110916) UNRELEASED; urgency=low
     the final piece of the puzzle needed for git-annex to to play nicely
     with branches.
 
- -- Joey Hess   Sun, 18 Sep 2011 18:25:51 -0400
+ -- Joey Hess   Wed, 28 Sep 2011 18:14:02 -0400
 
 git-annex (3.20110915) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index a518ca8cb9..3f31ee4dcc 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110915
+Version: 3.20110928
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 24a8b7f141af4874f7a0c71738614b53f671898d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 19:17:46 -0400
Subject: [PATCH 2270/8313] add news item for git-annex 3.20110928

---
 doc/news/version_3.20110817.mdwn |  6 ------
 doc/news/version_3.20110928.mdwn | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 6 deletions(-)
 delete mode 100644 doc/news/version_3.20110817.mdwn
 create mode 100644 doc/news/version_3.20110928.mdwn

diff --git a/doc/news/version_3.20110817.mdwn b/doc/news/version_3.20110817.mdwn
deleted file mode 100644
index 51388f3c78..0000000000
--- a/doc/news/version_3.20110817.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 3.20110817 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix shell escaping in rsync special remote.
-   * addurl: --fast can be used to avoid immediately downloading the url.
-   * Added support for getting content from git remotes using http (and https).
-   * Added curl to Debian package dependencies."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20110928.mdwn b/doc/news/version_3.20110928.mdwn
new file mode 100644
index 0000000000..3b11e18f5c
--- /dev/null
+++ b/doc/news/version_3.20110928.mdwn
@@ -0,0 +1,19 @@
+git-annex 3.20110928 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * --in can be used to make git-annex only operate on files
+     believed to be present in a given repository.
+   * Arbitrarily complex expressions can be built to limit the files git-annex
+     operates on, by combining the options --not --and --or -( and -)
+     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+   * --copies=N can be used to make git-annex only operate on files with
+     the specified number of copies. (And --not --copies=N for the inverse.)
+   * find: Rather than only showing files whose contents are present,
+     when used with --exclude --copies or --in, displays all files that
+     match the specified conditions.
+   * Note that this is a behavior change for git-annex find! Old behavior
+     can be gotten by using: git-annex find --in .
+   * status: Massively sped up; remove --fast mode.
+   * unused: File contents used by branches and tags are no longer
+     considered unused, even when not used by the current branch. This is
+     the final piece of the puzzle needed for git-annex to to play nicely
+     with branches."""]]
\ No newline at end of file

From 7c2c17f706d990f2f61b6aa702795a31c040c88f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 28 Sep 2011 20:12:11 -0400
Subject: [PATCH 2271/8313] golfing

---
 Command/Unused.hs | 39 +++++++++++++++++----------------------
 1 file changed, 17 insertions(+), 22 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index e65f0790ba..0c1ffe6039 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -165,8 +165,7 @@ excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
 	g <- Annex.gitRepo
 	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
-	excludeReferenced'
-		(getKeysReferenced : (map getKeysReferencedInGit $ refs c))
+	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
@@ -175,18 +174,15 @@ excludeReferenced l = do
 			filter ourbranches .
 			map words . lines
 		cmpheads a b = head a == head b
-		ourbranchend = "/" ++ Branch.name
+		ourbranchend = '/' : Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
-excludeReferenced' :: ([Annex [Key]]) -> S.Set Key -> Annex [Key]
-excludeReferenced' [] s = return $ S.toList s
-excludeReferenced' (a:as) s
-	| s == S.empty = return [] -- optimisation
-	| otherwise = do
-		referenced <- a
-		let !s' = remove referenced
-		excludeReferenced' as s'
-	where
-		remove l = s `S.difference` S.fromList l
+		removewith [] s = return $ S.toList s
+		removewith (a:as) s
+			| s == S.empty = return [] -- optimisation
+			| otherwise = do
+				referenced <- a
+				let !s' = s `S.difference` S.fromList referenced
+				removewith as s'
 
 {- Finds items in the first, smaller list, that are not
  - present in the second, larger list.
@@ -216,14 +212,13 @@ getKeysReferencedInGit ref = do
 	findkeys [] =<< liftIO (LsTree.lsTree g ref)
 	where
 		findkeys c [] = return c
-		findkeys c (l:ls) = do
-			if isSymLink (LsTree.mode l)
-				then do
-					content <- catFile ref $ LsTree.file l
-					case fileKey (takeFileName content) of
-						Nothing -> findkeys c ls
-						Just k -> findkeys (k:c) ls
-				else findkeys c ls
+		findkeys c (l:ls)
+			| isSymLink (LsTree.mode l) = do
+				content <- catFile ref $ LsTree.file l
+				case fileKey (takeFileName content) of
+					Nothing -> findkeys c ls
+					Just k -> findkeys (k:c) ls
+			| otherwise = findkeys c ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 

From 244ffef43f859152907f5202e85161f3e73cfe64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 16:43:00 -0400
Subject: [PATCH 2272/8313] add

---
 ...ex_processes_can_lead_to_locking_issues.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn

diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
new file mode 100644
index 0000000000..ef0a520a06
--- /dev/null
+++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
@@ -0,0 +1,17 @@
+When two git-annex processes are running and both modifying the git-annex
+branch, it's possible one will fail due to git's locking. When this
+happens, git-annex has already recorded its state in the journal (so no
+data is lost), but git-annex does crash, which can be surprising.
+
+I feel that, in general, multiple git-annex processes should be able to run
+concurrently. A big lock around all commands, or even all
+repository-modifying commands is a bad idea. Also, it's probably best to
+only worry about locking conflicts editing the git-annex branch. While `git
+annex add` and a few other commands make changes to the main git repo,
+and can have similar locking issues, so can any git commands that stage
+changes (I think.. check).
+
+Probably should KISS. Just add a lock file that is taken before changes to
+the git-annex branch, and if it's locked, wait. Changes to the git-annex
+branch tend to happen quickly (unless it's committing an enormous set of
+changes, and even that is relatively fast), so waiting seems ok. --[[Joey]] 

From a91c8a15d523791ea729976cd5c76bac1e7ec135 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 19:04:24 -0400
Subject: [PATCH 2273/8313] Sped up unused.

Added Git.ByteString which replaces Git IO methods with ones using lazy
ByteStrings. This can be more efficient when large quantities of data are
being read from git.

In Git.LsTree, parse git ls-tree output more efficiently, thanks
to ByteString. This benchmarks 25% faster, in a benchmark that includes
(probably predominately) the run time for git ls-tree itself.

In real world numbers, this makes git annex unused 2 seconds faster for
each branch it needs to check, in my usual large repo.
---
 Git.hs            |  5 ++--
 Git/ByteString.hs | 62 +++++++++++++++++++++++++++++++++++++++++++++++
 Git/LsTree.hs     | 34 +++++++++++++-------------
 debian/changelog  |  6 +++++
 4 files changed, 88 insertions(+), 19 deletions(-)
 create mode 100644 Git/ByteString.hs

diff --git a/Git.hs b/Git.hs
index fe2afdcfe3..f49cc21b54 100644
--- a/Git.hs
+++ b/Git.hs
@@ -59,6 +59,7 @@ module Git (
 	getSha,
 	shaSize,
 	commit,
+	assertLocal,
 
 	prop_idempotent_deencode
 ) where
@@ -458,8 +459,8 @@ commit g message newref parentrefs = do
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
 {- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it into a list of files/lines/whatever. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [FilePath]
+ - parameter), and splits it. -}
+pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
 pipeNullSplit repo params = filter (not . null) . split "\0" <$>
 	pipeRead repo params
 
diff --git a/Git/ByteString.hs b/Git/ByteString.hs
new file mode 100644
index 0000000000..4eb6a4876b
--- /dev/null
+++ b/Git/ByteString.hs
@@ -0,0 +1,62 @@
+{- module using Data.ByteString.Lazy.Char8 for git IO
+ -
+ - This can be imported instead of Git when more efficient ByteString IO
+ - is needed.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.ByteString (
+	module Git,
+	pipeRead,
+	pipeWrite,
+	pipeWriteRead,
+	pipeNullSplit
+) where
+
+import Control.Applicative
+import System.Cmd.Utils
+import System.IO
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Git hiding (pipeRead, pipeWrite, pipeWriteRead, pipeNullSplit)
+import Utility.SafeCommand
+
+{- Runs a git subcommand and returns its output, lazily. 
+ -
+ - Note that this leaves the git process running, and so zombies will
+ - result unless reap is called.
+ -}
+pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
+pipeRead repo params = assertLocal repo $ do
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
+	hSetBinaryMode h True
+	L.hGetContents h
+
+{- Runs a git subcommand, feeding it input.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
+pipeWrite repo params s = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
+	L.hPut h s
+	hClose h
+	return p
+
+{- Runs a git subcommand, feeding it input, and returning its output.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
+pipeWriteRead repo params s = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
+	hSetBinaryMode from True
+	L.hPut to s
+	hClose to
+	c <- L.hGetContents from
+	return (p, c)
+
+{- Reads null terminated output of a git command (as enabled by the -z 
+ - parameter), and splits it. -}
+pipeNullSplit :: Repo -> [CommandParam] -> IO [L.ByteString]
+pipeNullSplit repo params = filter (not . L.null) . L.split '\0' <$>
+	pipeRead repo params
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 4a6c509f94..9d2fe7a373 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -7,22 +7,23 @@
 
 module Git.LsTree (
 	TreeItem(..),
-	lsTree
+	lsTree,
+	parseLsTree
 ) where
 
 import Numeric
 import Control.Applicative
-import Data.Char
 import System.Posix.Types
+import qualified Data.ByteString.Lazy.Char8 as L
 
-import Git
+import Git.ByteString
 import Utility.SafeCommand
 
 type Treeish = String
 
 data TreeItem = TreeItem
 	{ mode :: FileMode
-	, objtype :: String
+	, typeobj :: String
 	, sha :: String
 	, file :: FilePath
 	} deriving Show
@@ -34,18 +35,17 @@ lsTree repo t = map parseLsTree <$>
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
-parseLsTree :: String -> TreeItem
-parseLsTree l = TreeItem m o s f
+parseLsTree :: L.ByteString -> TreeItem
+parseLsTree l = TreeItem
+		(fst $ head $ readOct $ L.unpack m)
+		(L.unpack t)
+		(L.unpack s)
+		(decodeGitFile $ L.unpack f)
 	where
 		-- l =  SP  SP  TAB 
-		-- Since everything until the file is fixed-width,
-		-- do not need to split on words.
-		(m, past_m) = head $ readOct l
-		(o, past_o) = splitAt 4 $ space past_m
-		(s, past_s) = splitAt shaSize $ space past_o
-		f = decodeGitFile $ space past_s
-		space (sp:rest)
-			| isSpace sp = rest
-			| otherwise = parseerr
-		space [] = parseerr
-		parseerr = "ls-tree parse error: " ++ l
+		-- All fields are fixed, so we can pull them out of
+		-- specific positions in the line.
+		(m, past_m) = L.splitAt 7 l
+		(t, past_t) = L.splitAt 4 past_m
+		(s, past_s) = L.splitAt 40 $ L.tail past_t
+		f = L.tail past_s
diff --git a/debian/changelog b/debian/changelog
index f337882714..b793101e10 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20110929) UNRELEASED; urgency=low
+
+  * Sped up unused.
+
+ -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
+
 git-annex (3.20110928) unstable; urgency=low
 
   * --in can be used to make git-annex only operate on files

From 67f2b7cb3eba19eb1ee55585f497e35172971e1a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 19:19:28 -0400
Subject: [PATCH 2274/8313] use ByteStrings when reading content of files

didn't bother to benchmark this
---
 Git.hs            | 13 -------------
 Git/UnionMerge.hs | 18 ++++++++++++++++--
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/Git.hs b/Git.hs
index f49cc21b54..4c4f00e8df 100644
--- a/Git.hs
+++ b/Git.hs
@@ -55,7 +55,6 @@ module Git (
 	repoAbsPath,
 	reap,
 	useIndex,
-	hashObject,
 	getSha,
 	shaSize,
 	commit,
@@ -417,18 +416,6 @@ useIndex index = do
 		reset (Right (Just v)) = setEnv var v True
 		reset _ = unsetEnv var
 
-{- Injects some content into git, returning its hash. -}
-hashObject :: Repo -> String -> IO String
-hashObject repo content = getSha subcmd $ do
-	(h, s) <- pipeWriteRead repo (map Param params) content
-	length s `seq` do
-		forceSuccess h
-		reap -- XXX unsure why this is needed
-		return s
-	where
-		subcmd = "hash-object"
-		params = [subcmd, "-w", "--stdin"]
-
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}
 getSha :: String -> IO String -> IO String
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a5bcbeac4a..a2b85dbdc5 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -16,8 +16,10 @@ import System.Cmd.Utils
 import Data.List
 import Data.Maybe
 import Data.String.Utils
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import qualified Git.ByteString as GitB
 import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
@@ -78,6 +80,18 @@ calc_merge g differ = do
 		pairs (_:[]) = error "calc_merge parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
 
+{- Injects some content into git, returning its hash. -}
+hashObject :: Repo -> L.ByteString -> IO String
+hashObject repo content = getSha subcmd $ do
+	(h, s) <- GitB.pipeWriteRead repo (map Param params) content
+	L.length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return $ L.unpack s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-w", "--stdin"]
+
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
@@ -86,10 +100,10 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- pipeRead g $ map Param ("show":shas)
+		content <- GitB.pipeRead g $ map Param ("show":shas)
 		sha <- hashObject g $ unionmerge content
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
 		nullsha = replicate shaSize '0'
-		unionmerge = unlines . nub . lines
+		unionmerge = L.unlines . nub . L.lines

From 4050c24d7bc49345b914bc062c0a5aedb93cfc8d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 21:11:48 -0400
Subject: [PATCH 2275/8313] ssh

---
 test.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test.hs b/test.hs
index 21e09e98e7..f8701db669 100644
--- a/test.hs
+++ b/test.hs
@@ -453,13 +453,13 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	checkunused []
 	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused []
-	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -m foo"] @? "git commit failed"
 	checkunused []
 	-- unused checks origin/master; once it's gone it is really unused
 	boolSystem "git" [Params "remote rm origin"] @? "git remote rm origin failed"
 	checkunused [annexedfilekey]
 	boolSystem "git" [Params "rm -q", File sha1annexedfile] @? "git rm failed"
-	boolSystem "git" [Params "commit -m foo"] @? "git commit failed"
+	boolSystem "git" [Params "commit -q -m foo"] @? "git commit failed"
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also

From 949ef94d5e5583e55d6ba9797cf71177b252173d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 22:31:20 -0400
Subject: [PATCH 2276/8313] layout

---
 Git/LsTree.hs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 9d2fe7a373..e0aa5a4431 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -36,11 +36,12 @@ lsTree repo t = map parseLsTree <$>
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
 parseLsTree :: L.ByteString -> TreeItem
-parseLsTree l = TreeItem
-		(fst $ head $ readOct $ L.unpack m)
-		(L.unpack t)
-		(L.unpack s)
-		(decodeGitFile $ L.unpack f)
+parseLsTree l = TreeItem 
+	{ mode = fst $ head $ readOct $ L.unpack m
+	, typeobj = L.unpack t
+	, sha = L.unpack s
+	, file = decodeGitFile $ L.unpack f
+	}
 	where
 		-- l =  SP  SP  TAB 
 		-- All fields are fixed, so we can pull them out of

From 7ff89ccfee13dcfe89cbdef83454e880dabd7186 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 29 Sep 2011 23:43:42 -0400
Subject: [PATCH 2277/8313] convert all git read/write functions to use
 ByteStrings

This yields a second or so speedup in unused, find, etc. Seems that even
when the ByteString is immediately split and then converted to Strings,
it's faster.

I may try to push ByteStrings out into more of git-annex gradually,
although I suspect most of the time-critical parts are already covered
now, and many of the rest rely on libraries that only support Strings.
---
 Branch.hs         |  5 ++--
 Command/Unused.hs |  3 ++-
 Git.hs            | 54 +++++++++++++++++++++++++++--------------
 Git/ByteString.hs | 62 -----------------------------------------------
 Git/LsFiles.hs    |  6 ++---
 Git/LsTree.hs     |  4 +--
 Git/UnionMerge.hs |  7 +++---
 debian/changelog  |  2 +-
 8 files changed, 49 insertions(+), 94 deletions(-)
 delete mode 100644 Git/ByteString.hs

diff --git a/Branch.hs b/Branch.hs
index 92b1fe29e8..e4caeece77 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -30,6 +30,7 @@ import System.IO
 import System.IO.Binary
 import System.Posix.Process
 import System.Exit
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Types.BranchState
 import qualified Git
@@ -181,7 +182,7 @@ siblingBranches :: Annex [String]
 siblingBranches = do
 	g <- Annex.gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-	return $ map (last . words) (lines r)
+	return $ map (last . words . L.unpack) (L.lines r)
 
 {- Ensures that a given ref has been merged into the index. -}
 updateRef :: GitRef -> Annex (Maybe String)
@@ -196,7 +197,7 @@ updateRef ref
 			Param (name++".."++ref),
 			Params "--oneline -n1"
 			]
-		if null diffs
+		if L.null diffs
 			then return Nothing
 			else do
 				showSideAction $ "merging " ++ Git.refDescribe ref ++ " into " ++ name
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 0c1ffe6039..987f367201 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,6 +16,7 @@ import Data.Maybe
 import System.FilePath
 import System.Directory
 import Data.List
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Command
 import Types
@@ -172,7 +173,7 @@ excludeReferenced l = do
 		refs = map last .
 			nubBy cmpheads .
 			filter ourbranches .
-			map words . lines
+			map words . lines . L.unpack
 		cmpheads a b = head a == head b
 		ourbranchend = '/' : Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
diff --git a/Git.hs b/Git.hs
index 4c4f00e8df..d32aaaa562 100644
--- a/Git.hs
+++ b/Git.hs
@@ -44,6 +44,7 @@ module Git (
 	pipeWrite,
 	pipeWriteRead,
 	pipeNullSplit,
+	pipeNullSplitB,
 	attributes,
 	remotes,
 	remotesAdd,
@@ -85,6 +86,7 @@ import Text.Printf
 import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Utility
 import Utility.Path
@@ -379,22 +381,41 @@ run repo subcommand params = assertLocal repo $
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
  -}
-pipeRead :: Repo -> [CommandParam] -> IO String
+pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
 pipeRead repo params = assertLocal repo $ do
-	(_, s) <- pipeFrom "git" $ toCommand $ gitCommandLine repo params
-	return s
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
+	hSetBinaryMode h True
+	L.hGetContents h
 
 {- Runs a git subcommand, feeding it input.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: Repo -> [CommandParam] -> String -> IO PipeHandle
-pipeWrite repo params s = assertLocal repo $
-	pipeTo "git" (toCommand $ gitCommandLine repo params) s
+pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
+pipeWrite repo params s = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
+	L.hPut h s
+	hClose h
+	return p
 
 {- Runs a git subcommand, feeding it input, and returning its output.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: Repo -> [CommandParam] -> String -> IO (PipeHandle, String)
-pipeWriteRead repo params s = assertLocal repo $
-	pipeBoth "git" (toCommand $ gitCommandLine repo params) s
+pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
+pipeWriteRead repo params s = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
+	hSetBinaryMode from True
+	L.hPut to s
+	hClose to
+	c <- L.hGetContents from
+	return (p, c)
+
+{- Reads null terminated output of a git command (as enabled by the -z 
+ - parameter), and splits it. -}
+pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
+pipeNullSplit repo params = map L.unpack <$> pipeNullSplitB repo params
+
+{- For when Strings are not needed. -}
+pipeNullSplitB :: Repo -> [CommandParam] -> IO [L.ByteString]
+pipeNullSplitB repo params = filter (not . L.null) . L.split '\0' <$>
+	pipeRead repo params
 
 {- Reaps any zombie git processes. -}
 reap :: IO ()
@@ -436,21 +457,18 @@ shaSize = 40
  - with the specified parent refs. -}
 commit :: Repo -> String -> String -> [String] -> IO ()
 commit g message newref parentrefs = do
-	tree <- getSha "write-tree" $
+	tree <- getSha "write-tree" $ asString $
 		pipeRead g [Param "write-tree"]
-	sha <- getSha "commit-tree" $ ignorehandle $
-		pipeWriteRead g (map Param $ ["commit-tree", tree] ++ ps) message
+	sha <- getSha "commit-tree" $ asString $
+		ignorehandle $ pipeWriteRead g
+			(map Param $ ["commit-tree", tree] ++ ps)
+			(L.pack message)
 	run g "update-ref" [Param newref, Param sha]
 	where
 		ignorehandle a = snd <$> a
+		asString a = L.unpack <$> a
 		ps = concatMap (\r -> ["-p", r]) parentrefs
 
-{- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
-pipeNullSplit repo params = filter (not . null) . split "\0" <$>
-	pipeRead repo params
-
 {- Runs git config and populates a repo with its config. -}
 configRead :: Repo -> IO Repo
 configRead repo@(Repo { location = Dir d }) = do
diff --git a/Git/ByteString.hs b/Git/ByteString.hs
deleted file mode 100644
index 4eb6a4876b..0000000000
--- a/Git/ByteString.hs
+++ /dev/null
@@ -1,62 +0,0 @@
-{- module using Data.ByteString.Lazy.Char8 for git IO
- -
- - This can be imported instead of Git when more efficient ByteString IO
- - is needed.
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Git.ByteString (
-	module Git,
-	pipeRead,
-	pipeWrite,
-	pipeWriteRead,
-	pipeNullSplit
-) where
-
-import Control.Applicative
-import System.Cmd.Utils
-import System.IO
-import qualified Data.ByteString.Lazy.Char8 as L
-
-import Git hiding (pipeRead, pipeWrite, pipeWriteRead, pipeNullSplit)
-import Utility.SafeCommand
-
-{- Runs a git subcommand and returns its output, lazily. 
- -
- - Note that this leaves the git process running, and so zombies will
- - result unless reap is called.
- -}
-pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
-pipeRead repo params = assertLocal repo $ do
-	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
-	hSetBinaryMode h True
-	L.hGetContents h
-
-{- Runs a git subcommand, feeding it input.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
-pipeWrite repo params s = assertLocal repo $ do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
-	L.hPut h s
-	hClose h
-	return p
-
-{- Runs a git subcommand, feeding it input, and returning its output.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
-pipeWriteRead repo params s = assertLocal repo $ do
-	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
-	hSetBinaryMode from True
-	L.hPut to s
-	hClose to
-	c <- L.hGetContents from
-	return (p, c)
-
-{- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [L.ByteString]
-pipeNullSplit repo params = filter (not . L.null) . L.split '\0' <$>
-	pipeRead repo params
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index c778e5d69c..28e007a4dc 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -20,13 +20,11 @@ import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: Repo -> [FilePath] -> IO [FilePath]
-inRepo repo l = pipeNullSplit repo $
-	Params "ls-files --cached -z --" : map File l
+inRepo repo l = pipeNullSplit repo $ Params "ls-files --cached -z --" : map File l
 
 {- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
-notInRepo repo include_ignored l =
-	pipeNullSplit repo $
+notInRepo repo include_ignored l = pipeNullSplit repo $
 		[Params "ls-files --others"] ++ exclude ++
 		[Params "-z --"] ++ map File l
 	where
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index e0aa5a4431..c072ef5be9 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -16,7 +16,7 @@ import Control.Applicative
 import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Git.ByteString
+import Git
 import Utility.SafeCommand
 
 type Treeish = String
@@ -31,7 +31,7 @@ data TreeItem = TreeItem
 {- Lists the contents of a Treeish -}
 lsTree :: Repo -> Treeish -> IO [TreeItem]
 lsTree repo t = map parseLsTree <$>
-	pipeNullSplit repo [Params "ls-tree --full-tree -z -r --", File t]
+	pipeNullSplitB repo [Params "ls-tree --full-tree -z -r --", File t]
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a2b85dbdc5..ac002b374e 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -19,7 +19,6 @@ import Data.String.Utils
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
-import qualified Git.ByteString as GitB
 import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
@@ -44,7 +43,7 @@ merge _ _ = error "wrong number of branches to merge"
 update_index :: Repo -> [String] -> IO ()
 update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		togit ps content = pipeWrite g (map Param ps) content
+		togit ps content = pipeWrite g (map Param ps) (L.pack content)
 			>>= forceSuccess
 
 {- Generates a line suitable to be fed into update-index, to add
@@ -83,7 +82,7 @@ calc_merge g differ = do
 {- Injects some content into git, returning its hash. -}
 hashObject :: Repo -> L.ByteString -> IO String
 hashObject repo content = getSha subcmd $ do
-	(h, s) <- GitB.pipeWriteRead repo (map Param params) content
+	(h, s) <- pipeWriteRead repo (map Param params) content
 	L.length s `seq` do
 		forceSuccess h
 		reap -- XXX unsure why this is needed
@@ -100,7 +99,7 @@ mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- GitB.pipeRead g $ map Param ("show":shas)
+		content <- pipeRead g $ map Param ("show":shas)
 		sha <- hashObject g $ unionmerge content
 		return $ Just $ update_index_line sha file
 	where
diff --git a/debian/changelog b/debian/changelog
index b793101e10..62b65333d1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
-  * Sped up unused.
+  * Various speed improvements gained by using ByteStrings.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From c86a2f686aa447d1cf30cecaa2feedaf0e2c6ba2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Fri, 30 Sep 2011 04:32:24 +0000
Subject: [PATCH 2278/8313]

---
 doc/forum/location_tracking_cleanup.mdwn | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup.mdwn

diff --git a/doc/forum/location_tracking_cleanup.mdwn b/doc/forum/location_tracking_cleanup.mdwn
new file mode 100644
index 0000000000..7e2e230af7
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup.mdwn
@@ -0,0 +1,24 @@
+I recently started experimenting with git annex, adding files that I've had
+floating across several computers to repositories. During the testing I had
+a few occasions where I wrecked a repository somehow, and decided to wipe it
+and start anew (at this point there was no important files in them so I thought
+this is the easiest way). Well, as it turns out this interacts badly with location
+tracking, since now `git annex whereis` shows files residing in all those destroyed
+repositories, all having same names as some existing repositories. This makes it hard
+to follow whether a repo actually has a file, or was the file only seen in some dead
+repo with the same name.
+
+I planned on cleaning this up by looking up the UUIDs of the now stable, existing
+repos and untrusting all the dead copies (they should effectively disappear from
+git annex´s output then, right?), but I didn't find an easy way to look up the UUID
+of the current repository (maybe this could be included in `git annex status`?)
+I also noticed that untrust cannot remove the trust based on the UUID -- if I try
+it I simply get "there is no git remote named "11908472-...", so I guess untrust
+works with git remote names, which I find a bit confusing, since trust.log logs the
+trust levels based on the UUID. I could just write into trust.log manually, but I'm
+unsure how the changes would get propagated.
+
+What should I do? As a related wishlist item I would ask for some additional mechanisms
+for purging known-dead repositories from the location tracking database. And the ability
+to look up the UUID of the current repo, and to use the UUID to specify repositories when
+applicable (untrust and describe maybe).

From a7e7dda55a82cb5007c5eaa2f7752f5cefdcb1d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 02:23:24 -0400
Subject: [PATCH 2279/8313] Fix referring to remotes by uuid.

I think that I broke this in some fairly recent refactoring.
---
 Remote.hs        | 12 +++++++++---
 debian/changelog |  1 +
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 0ce01872ac..05f70a5d7f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -87,7 +87,8 @@ genList = do
 			u <- getUUID r
 			generate t r u (M.lookup u m)
 
-{- Looks up a remote by name. (Or by UUID.) -}
+{- Looks up a remote by name. (Or by UUID.) Only finds currently configured
+ - git remotes. -}
 byName :: String -> Annex (Remote Annex)
 byName n = do
 	res <- byName' n
@@ -106,7 +107,8 @@ byName' n = do
 		matching r = n == name r || n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
- - and returns its UUID. -}
+ - and returns its UUID. Finds even remotes that are not configured in
+ - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
 nameToUUID n = do
@@ -115,7 +117,11 @@ nameToUUID n = do
 		Left e -> fromMaybe (error e) <$> byDescription
 		Right r -> return $ uuid r
 	where
-		byDescription = M.lookup n . invertMap <$> uuidMap
+		byDescription = do
+			m <- uuidMap
+			case M.lookup n $ invertMap m of
+				Just u -> return $ Just u
+				Nothing -> return $ M.lookup n m
 		invertMap = M.fromList . map swap . M.toList
 		swap (a, b) = (b, a)
 
diff --git a/debian/changelog b/debian/changelog
index 62b65333d1..5354e710db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
   * Various speed improvements gained by using ByteStrings.
+  * Fix referring to remotes by uuid.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 17b29176b8350adf9f6d547d59b97c965f1aad7f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 02:50:34 -0400
Subject: [PATCH 2280/8313] fix handling of uuids with empty descriptions

---
 UUID.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/UUID.hs b/UUID.hs
index fa71bed391..a150dc333c 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -100,7 +100,4 @@ uuidMap = do
 	s <- Branch.get uuidLog
 	return $ M.fromList $ map pair $ lines s
 	where
-		pair l =
-			if 1 < length (words l)
-				then (head $ words l, unwords $ drop 1 $ words l)
-				else ("", "")
+		pair l = (head $ words l, unwords $ drop 1 $ words l)

From 03e54680ff1432915e2e02f87c95a9126af5a420 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 02:51:05 -0400
Subject: [PATCH 2281/8313] really fix referring to remotes by uuid

---
 Remote.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 05f70a5d7f..87fd8aab62 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -119,11 +119,12 @@ nameToUUID n = do
 	where
 		byDescription = do
 			m <- uuidMap
-			case M.lookup n $ invertMap m of
+			case M.lookup n $ transform swap m of
 				Just u -> return $ Just u
-				Nothing -> return $ M.lookup n m
-		invertMap = M.fromList . map swap . M.toList
+				Nothing -> return $ M.lookup n $ transform double m
+		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
+		double (a, _) = (a, a)
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
  -

From c68fc49184d9e59fe3d9cda1addca64b82d513d6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 30 Sep 2011 06:55:34 +0000
Subject: [PATCH 2282/8313] Added a comment

---
 ...comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment

diff --git a/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment b/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment
new file mode 100644
index 0000000000..8915ea3518
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_1_7d6319e8c94dfe998af9cfcbf170efb2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-09-30T06:55:34Z"
+ content="""
+Specifying the UUID was supposed to work, I think I broke it a while ago. Fixed now in git.
+
+I'm not sure why you need to look up the UUID of the current repository. You can always refer to the current repository as \".\". Anyway, the UUID of the current repository is in `.git/config`, or use `git config annex.uuid`.
+"""]]

From a7102ca4d53f8a4ab7148ab3717765683b136cfc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 03:03:59 -0400
Subject: [PATCH 2283/8313] list backends with more keys first, not last

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 6da8064f85..329d3f70bf 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -117,7 +117,7 @@ bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 backend_usage :: Stat
 backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
 	where
-		usage ks = pp "" $ sort $ map swap $ splits $ S.toList ks
+		usage ks = pp "" $ reverse . sort $ map swap $ splits $ S.toList ks
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)

From 15eccdf124d13d8502ee5a23e73968e5208690f2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 03:05:10 -0400
Subject: [PATCH 2284/8313] better output layout

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 329d3f70bf..ad490cae82 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -167,4 +167,4 @@ staleSize label dirspec = do
 			return $ s ++ aside "clean up with git-annex unused"
 
 aside :: String -> String
-aside s = "\t(" ++ s ++ ")"
+aside s = " (" ++ s ++ ")"

From 828f3f1b0c4cf8791c063c6a393797047084eee8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 03:20:24 -0400
Subject: [PATCH 2285/8313] status: List all known repositories.

---
 Command/Status.hs | 12 ++++++++++--
 debian/changelog  |  1 +
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index ad490cae82..07c0958bbe 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -28,6 +28,8 @@ import Content
 import Types.Key
 import Locations
 import Backend
+import UUID
+import Remote
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -55,6 +57,7 @@ stats :: [Stat]
 stats = 
 	[ supported_backends
 	, supported_remote_types
+	, remote_list
 	, tmp_size
 	, bad_data_size
 	, local_annex_keys
@@ -92,6 +95,11 @@ supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
+remote_list :: Stat
+remote_list = stat "known repositories" $ lift $ do
+	s <- prettyPrintUUIDs "repos" =<< M.keys <$> uuidMap
+	return $ '\n':init s
+
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
 	keySizeSum <$> cachedKeysPresent
@@ -154,8 +162,8 @@ keySizeSum s = total ++ missingnote
 		missingnote
 			| missing == 0 = ""
 			| otherwise = aside $
-				"but " ++ show missing ++
-				" keys have unknown size"
+				"+ " ++ show missing ++
+				" keys of unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
 staleSize label dirspec = do
diff --git a/debian/changelog b/debian/changelog
index 5354e710db..3bdd044510 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
 
   * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
+  * status: List all known repositories.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 84801918f60481be20553da58ac5cd9bb9feae29 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Fri, 30 Sep 2011 11:55:36 +0000
Subject: [PATCH 2286/8313] Added a comment

---
 ...nt_2_e7395cb6e01f42da72adf71ea3ebcde4._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment

diff --git a/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment b/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment
new file mode 100644
index 0000000000..f612e53a4d
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_2_e7395cb6e01f42da72adf71ea3ebcde4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
+ nickname="Perttu"
+ subject="comment 2"
+ date="2011-09-30T11:55:35Z"
+ content="""
+Thanks for the quick reply :)
+
+I wanted to look up the UUID of the current repo so that I can find out which repo is alive from the collection of repos with the same name.
+I could have looked for it in .git/config though, since it's pretty obvious. I just looked into the git-annex branch and didn't find it there.
+Thanks for the tip about using \".\". By the way, could there be some kind of warning about using non-unique names for repos? That would make this
+scenario less likely. Or maybe that is a bad idea given the decentralized nature of git.
+
+By the way, do the trust settings propagate to other repos? If I mark some UUID as untrusted on one computer does it become globally untrusted?
+"""]]

From d48ae1b8fd2d73746986bd40d22de450d647d223 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 30 Sep 2011 16:47:27 +0000
Subject: [PATCH 2287/8313] Added a comment

---
 ...comment_3_c15428cec90e969284a5e690fb4b2fde._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment

diff --git a/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment b/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment
new file mode 100644
index 0000000000..c676b9b615
--- /dev/null
+++ b/doc/forum/location_tracking_cleanup/comment_3_c15428cec90e969284a5e690fb4b2fde._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-09-30T16:47:27Z"
+ content="""
+`git annex status` now includes a list of all known repositories.
+
+Yes, trust setting propigate on git push/pull like any other git-annex information.
+"""]]

From f88738223ee5156baac8f0fa0bde6d701f1fdd07 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 14:59:35 -0400
Subject: [PATCH 2288/8313] don't crash on malformed lines in uuid.log

---
 UUID.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/UUID.hs b/UUID.hs
index a150dc333c..b1ccbb250c 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -100,4 +100,8 @@ uuidMap = do
 	s <- Branch.get uuidLog
 	return $ M.fromList $ map pair $ lines s
 	where
-		pair l = (head $ words l, unwords $ drop 1 $ words l)
+		pair l
+			| null ws = ("", "")
+			| otherwise = (head ws, unwords $ drop 1 ws)
+			where
+				ws = words l

From 29032cb70e66d18f25b9b942eb01dceeeb8aa300 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 15:02:08 -0400
Subject: [PATCH 2289/8313] When displaying a list of repositories, show git
 remote names in addition to their descriptions.

---
 Remote.hs                                     | 21 ++++++++++++-------
 debian/changelog                              |  2 ++
 ...95__remotes__47__hook_with_tahoe-lafs.mdwn |  2 +-
 .../automatically_managing_content.mdwn       |  4 ++--
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 87fd8aab62..83a593dd50 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -136,18 +136,25 @@ nameToUUID n = do
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
 	here <- getUUID =<< Annex.gitRepo
-	m <- M.union <$> uuidMap <*> availMap
+	m <- M.unionWith addname <$> uuidMap <*> remoteMap
 	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
 	where
-		availMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
+		addname d n
+			| d == n = d
+			| otherwise = n ++ " (" ++ d ++ ")"
+		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
-		prettify m here u = base ++ ishere
+		prettify m here u
+			| not (null d) = u ++ "  -- " ++ d
+			| otherwise = u
 			where
-				base = if not $ null $ findlog m u
-					then u ++ "  -- " ++ findlog m u
-					else u
-				ishere = if here == u then " <-- here" else ""
+				ishere = here == u
+				n = findlog m u
+				d
+					| null n && ishere = "here"
+					| ishere = addname n "here"
+					| otherwise = n
 		jsonify m here u = toJSObject
 			[ ("uuid", toJSON u)
 			, ("description", toJSON $ findlog m u)
diff --git a/debian/changelog b/debian/changelog
index 3bdd044510..bf41f553d5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
   * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn
index 4f5f089a89..8981200d88 100644
--- a/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn
+++ b/doc/forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs.mdwn
@@ -12,7 +12,7 @@ The only quirk I've noticed is this...
 
 $ git annex whereis .
 whereis frink.jar (2 copies) 
-  	084603a8-7243-11e0-b1f5-83102bcd7953  -- testtest <-- here
+  	084603a8-7243-11e0-b1f5-83102bcd7953  -- here (testtest)
    	1d1bc312-7243-11e0-a9ce-5f10c0ce9b0a
 ok
 

diff --git a/doc/walkthrough/automatically_managing_content.mdwn b/doc/walkthrough/automatically_managing_content.mdwn
index ba0cad6092..ef883efef3 100644
--- a/doc/walkthrough/automatically_managing_content.mdwn
+++ b/doc/walkthrough/automatically_managing_content.mdwn
@@ -13,7 +13,7 @@ file.
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
 	whereis other_file (3 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
-		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- here (usb drive)
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive
 
 What would be handy is some automated versions of get and drop, that only
@@ -31,7 +31,7 @@ work toward having two copies of your files.
 	# git annex whereis
 	whereis my_cool_big_file (2 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
-		62b39bbe-4149-11e0-af01-bb89245a1e61  -- usb drive <-- here
+		62b39bbe-4149-11e0-af01-bb89245a1e61  -- here (usb drive)
 	whereis other_file (2 copies)
 		0c443de8-e644-11df-acbf-f7cd7ca6210d  -- laptop
 		7570b02e-15e9-11e0-adf0-9f3f94cb2eaa  -- backup drive

From 1e6b7e901d47142a99ab4e6608f6307c01bc085e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Sep 2011 16:21:28 -0400
Subject: [PATCH 2290/8313] less space

---
 Remote.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote.hs b/Remote.hs
index 83a593dd50..efa7a5cc8e 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -146,7 +146,7 @@ prettyPrintUUIDs desc uuids = do
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m here u
-			| not (null d) = u ++ "  -- " ++ d
+			| not (null d) = u ++ " -- " ++ d
 			| otherwise = u
 			where
 				ishere = here == u

From 49f21dd9ba4819aedf04ebbc0c12fafdb3dc8ab5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 2 Oct 2011 11:16:34 -0400
Subject: [PATCH 2291/8313] Contain the zombie hordes.a

Specifically, when using gpg, a zombie is forked for each file, so waiting
until shutdown to reap won't do.
---
 CmdLine.hs       | 5 +++--
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 4ccd2c2c2f..0ee0c6adbb 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -85,7 +85,9 @@ tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
 		AnnexQueue.flushWhenFull
-		a
+		r <- a
+		liftIO Git.reap
+		return r
 	case result of
 		Left err -> do
 			Annex.eval state $ do
@@ -104,5 +106,4 @@ startup = return True
 shutdown :: Annex Bool
 shutdown = do
 	saveState
-	liftIO Git.reap
 	return True
diff --git a/debian/changelog b/debian/changelog
index bf41f553d5..834d2a5726 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * status: List all known repositories.
   * When displaying a list of repositories, show git remote names
     in addition to their descriptions.
+  * Contain the zombie hordes.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 61fbea992da0f2a1ed49e2e862ef937b25d8430b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 2 Oct 2011 11:42:34 -0400
Subject: [PATCH 2292/8313] when all you have is a zombie, everything looks
 like a shotgun

Actually, let's do a targeted fix of the actual forkProcess that was not
waited on. The global reap is moved back to the end, after the long-running
git processes actually exit.
---
 CmdLine.hs | 5 ++---
 Crypto.hs  | 3 ++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0ee0c6adbb..38d4754da9 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -85,9 +85,7 @@ tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
 		AnnexQueue.flushWhenFull
-		r <- a
-		liftIO Git.reap
-		return r
+		a
 	case result of
 		Left err -> do
 			Annex.eval state $ do
@@ -106,4 +104,5 @@ startup = return True
 shutdown :: Annex Bool
 shutdown = do
 	saveState
+	liftIO Git.reap -- zombies from long-running git processes
 	return True
diff --git a/Crypto.hs b/Crypto.hs
index d789b44556..9b2d73f28d 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -218,7 +218,7 @@ gpgCipherHandle params c a b = do
 
 	params' <- gpgParams $ passphrase ++ params
 	(pid, fromh, toh) <- hPipeBoth "gpg" params'
-	_ <- forkProcess $ do
+	pid2 <- forkProcess $ do
 		L.hPut toh =<< a
 		hClose toh
 		exitSuccess
@@ -227,6 +227,7 @@ gpgCipherHandle params c a b = do
 
 	-- cleanup
 	forceSuccess pid
+	_ <- getProcessStatus True False pid2
 	closeFd frompipe
 	return ret
 

From 6dfb94b2d783ef848c651ab20818b05c8a0504a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 14:48:04 -0400
Subject: [PATCH 2293/8313] update

---
 ..._processes_can_lead_to_locking_issues.mdwn | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
index ef0a520a06..87880df417 100644
--- a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
+++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
@@ -15,3 +15,37 @@ Probably should KISS. Just add a lock file that is taken before changes to
 the git-annex branch, and if it's locked, wait. Changes to the git-annex
 branch tend to happen quickly (unless it's committing an enormous set of
 changes, and even that is relatively fast), so waiting seems ok. --[[Joey]] 
+
+----
+
+Commit 7981eb4cb512fbe3c49a3dd165c31be14ae4bc49 is more pessimistic, 
+describes some other potential issues. 
+
+* The journal needs to be emptied (done) and kept locked (not done) during
+  a merge, since a merge operates at a level below the journal, and any
+  changes that are journaled during a merge can overwrite changes merged
+  in from another branch.
+
+* Two git-annex processes can be doing conflicting things and inconsistent
+  information be written to the journal.
+
+  - One example would be concurrent get and drop of the same key.
+    But could this really race? If the key was already present, the get
+    would do nothing, so record no changes. If the key was not yet present,
+    the drop would do nothing, and record no changes.
+
+  - Instead, consider two copys of a key to different locations. If the
+    slower copy starts first and ends last, it could cache the location 
+    info, add the  new location, and lose the other location it was copied to.
+    Tested it and the location is not cached during the whole copy (logChange
+    reads the current log immediatly before writing), so this
+    race's window is very small -- but does exist.
+
+---- 
+
+## Updated plan
+
+Make Branch.change transactional, so it takes a lock, reads a file,
+applies a function to it, and writes the changed file.
+
+Make Branch.update hold the same lock.

From f77979b8b5ef1dc59b45c03ba6febfacdf904491 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 15:41:25 -0400
Subject: [PATCH 2294/8313] improved git-annex branch changing

All changes to files in the branch are now made via pure functions that
transform the old file into the new. This will allow adding locking
to prevent read/write races. It also makes the code nicer, and purer.

I noticed a behavior change, really a sort of bug fix. Before,
'git annex untrust foo --trust bar' would change both trust levels
permanantly, now the --trust doesn't get stored.
---
 Branch.hs      | 16 +++++++++++++---
 LocationLog.hs |  1 -
 PresenceLog.hs | 12 +++++-------
 RemoteLog.hs   |  7 +++----
 Trust.hs       |  8 +++-----
 UUID.hs        | 16 ++++++++--------
 Upgrade/V2.hs  |  4 ++--
 7 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index e4caeece77..9340259c7d 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -213,9 +213,19 @@ updateRef ref
 				liftIO $ Git.UnionMerge.merge g [ref]
 				return $ Just ref
 
-{- Records changed content of a file into the journal. -}
-change :: FilePath -> String -> Annex ()
-change file content = do
+{- Applies a function to modifiy the content of a file. -}
+change :: FilePath -> (String -> String) -> Annex ()
+change file a = do
+	lock
+	get file >>= return . a >>= set file
+	unlock
+	where
+		lock = return ()
+		unlock = return ()
+
+{- Records new content of a file into the journal. -}
+set :: FilePath -> String -> Annex ()
+set file content = do
 	setJournalFile file content
 	setCache file content
 
diff --git a/LocationLog.hs b/LocationLog.hs
index 7e5e81d7af..0cdf88bc6b 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -15,7 +15,6 @@ module LocationLog (
 	LogStatus(..),
 	logChange,
 	readLog,
-	writeLog,
 	keyLocations,
 	loggedKeys,
 	logFile,
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 7742651b84..2db1ee59b4 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -16,7 +16,6 @@ module PresenceLog (
 	addLog,
 	readLog,
 	parseLog,
-	writeLog,
 	logNow,
 	compactLog,
 	currentLog,
@@ -75,9 +74,8 @@ instance Read LogLine where
 			ret v = [(v, "")]
 
 addLog :: FilePath -> LogLine -> Annex ()
-addLog file line = do
-	ls <- readLog file
-	writeLog file (compactLog $ line:ls)
+addLog file line = Branch.change file $ \s -> 
+	showLog $ compactLog (line : parseLog s)
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
@@ -90,9 +88,9 @@ parseLog = filter parsable . map read . lines
 		-- some lines may be unparseable, avoid them
 		parsable l = status l /= Undefined
 
-{- Stores a set of lines in a log file -}
-writeLog :: FilePath -> [LogLine] -> Annex ()
-writeLog file ls = Branch.change file (unlines $ map show ls)
+{- Generates a log file. -}
+showLog :: [LogLine] -> String
+showLog = unlines . map show
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> String -> Annex LogLine
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 620c0d8757..f9c7997e41 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -32,11 +32,10 @@ remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = do
-	m <- readRemoteLog
-	Branch.change remoteLog $ unlines $ sort $
-		map toline $ M.toList $ M.insert u c m
+configSet u c = Branch.change remoteLog $
+	serialize . M.insert u c . remoteLogParse
 	where
+		serialize = unlines . sort . map toline . M.toList
 		toline (u', c') = u' ++ " " ++ unwords (configToKeyVal c')
 
 {- Map of remotes by uuid containing key/value config maps. -}
diff --git a/Trust.hs b/Trust.hs
index 232eea6a5d..0c8836c85a 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -64,11 +64,9 @@ trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
 		error "unknown UUID; cannot modify trust level"
-        m <- trustMap
-	when (M.lookup uuid m /= Just level) $ do
-		let m' = M.insert uuid level m
-		Branch.change trustLog (serialize m')
-		Annex.changeState $ \s -> s { Annex.trustmap = Just m' }
+	Branch.change trustLog $
+		serialize . M.insert uuid level . M.fromList . trustMapParse
+	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
         where
                 serialize m = unlines $ map showpair $ M.toList m
 		showpair (u, t) = u ++ " " ++ show t
diff --git a/UUID.hs b/UUID.hs
index b1ccbb250c..eab6bd4dff 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -23,6 +23,7 @@ module UUID (
 ) where
 
 import Control.Monad.State
+import Control.Applicative
 import System.Cmd.Utils
 import System.IO
 import qualified Data.Map as M
@@ -87,18 +88,17 @@ prepUUID = do
 
 {- Records a description for a uuid in the uuidLog. -}
 describeUUID :: UUID -> String -> Annex ()
-describeUUID uuid desc = do
-	m <- uuidMap
-	let m' = M.insert uuid desc m
-	Branch.change uuidLog (serialize m')
+describeUUID uuid desc = Branch.change uuidLog $
+	serialize . M.insert uuid desc . parse
 	where
 		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
 
-{- Read and parse the uuidLog into a Map -}
+{- Read the uuidLog into a Map -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = do
-	s <- Branch.get uuidLog
-	return $ M.fromList $ map pair $ lines s
+uuidMap = parse <$> Branch.get uuidLog
+
+parse :: String -> M.Map UUID String
+parse = M.fromList . map pair . lines
 	where
 		pair l
 			| null ws = ("", "")
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index e99a7cf817..4e686288fb 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -87,8 +87,8 @@ inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
 	g <- Annex.gitRepo
 	new <- liftIO (readFile $ olddir g  source)
-	prev <- Branch.get dest
-	Branch.change dest $ unlines $ nub $ lines prev ++ lines new
+	Branch.change dest $ \prev -> 
+		unlines $ nub $ lines prev ++ lines new
 	showProgress
 
 logFiles :: FilePath -> Annex [FilePath]

From d357556141b716a8c9d622cbfb44c38484065183 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 16:32:36 -0400
Subject: [PATCH 2295/8313] Add locking to avoid races when changing the
 git-annex branch.

---
 Branch.hs                                     | 33 ++++++++++++-------
 Locations.hs                                  |  5 +++
 debian/changelog                              |  1 +
 ..._processes_can_lead_to_locking_issues.mdwn |  2 ++
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 9340259c7d..34486243e3 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -29,6 +29,8 @@ import Data.Maybe
 import System.IO
 import System.IO.Binary
 import System.Posix.Process
+import System.Posix.IO
+import System.Posix.Files
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -129,16 +131,17 @@ create = unlessM hasBranch $ do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = whenM stageJournalFiles $ do
-	g <- Annex.gitRepo
-	withIndex $ liftIO $ Git.commit g message fullname [fullname]
+commit message = lockJournal $
+	whenM stageJournalFiles $ do
+		g <- Annex.gitRepo
+		withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
 update = do
 	state <- getState
-	unless (branchUpdated state) $ withIndex $ do
+	unless (branchUpdated state) $ withIndex $ lockJournal $ do
 		{- Since branches get merged into the index, it's important to
 		 - first stage the journal into the index. Otherwise, any
 		 - changes in the journal would later get staged, and might
@@ -154,6 +157,7 @@ update = do
 		g <- Annex.gitRepo
 		unless (null updated && not staged) $ liftIO $
 			Git.commit g "update" fullname (fullname:updated)
+
 		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 		invalidateCache
 
@@ -215,13 +219,7 @@ updateRef ref
 
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()
-change file a = do
-	lock
-	get file >>= return . a >>= set file
-	unlock
-	where
-		lock = return ()
-		unlock = return ()
+change file a = lockJournal $ get file >>= return . a >>= set file
 
 {- Records new content of a file into the journal. -}
 set :: FilePath -> String -> Annex ()
@@ -277,7 +275,7 @@ setJournalFile file content = do
 			writeBinaryFile tmpfile content
 			renameFile tmpfile jfile
 
-{- Gets journalled content for a file in the branch. -}
+{- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
 getJournalFile file = do
 	g <- Annex.gitRepo
@@ -346,3 +344,14 @@ journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
  - filename on the branch. -}
 fileJournal :: FilePath -> FilePath
 fileJournal = replace "//" "_" . replace "_" "/"
+
+{- Runs an action that modifies the journal, using locking to avoid
+ - contention with other git-annex processes. -}
+lockJournal :: Annex a -> Annex a
+lockJournal a = do
+	g <- Annex.gitRepo
+	h <- liftIO $ createFile (gitAnnexJournalLock g) stdFileMode
+	liftIO $ waitToSetLock h (WriteLock, AbsoluteSeek, 0, 0)
+	r <- a
+	liftIO $ closeFd h
+	return r
diff --git a/Locations.hs b/Locations.hs
index 942b687bbf..b18444e72b 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -18,6 +18,7 @@ module Locations (
 	gitAnnexBadLocation,
 	gitAnnexUnusedLog,
 	gitAnnexJournalDir,
+	gitAnnexJournalLock,
 	isLinkToAnnex,
 	hashDirMixed,
 	hashDirLower,
@@ -109,6 +110,10 @@ gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 gitAnnexJournalDir :: Git.Repo -> FilePath
 gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
 
+{- Lock file for the journal. -}
+gitAnnexJournalLock :: Git.Repo -> FilePath
+gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/debian/changelog b/debian/changelog
index 834d2a5726..7554cd5028 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * When displaying a list of repositories, show git remote names
     in addition to their descriptions.
   * Contain the zombie hordes.
+  * Add locking to avoid races when changing the git-annex branch.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
index 87880df417..2485e7b19e 100644
--- a/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
+++ b/doc/bugs/concurrent_git-annex_processes_can_lead_to_locking_issues.mdwn
@@ -49,3 +49,5 @@ Make Branch.change transactional, so it takes a lock, reads a file,
 applies a function to it, and writes the changed file.
 
 Make Branch.update hold the same lock.
+
+> [[Done]].

From 2636ea79c342f23f28a050bf8ad7f344a05210aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 17:27:48 -0400
Subject: [PATCH 2296/8313] avoid taking journal lock unnecessarily

---
 Branch.hs | 173 ++++++++++++++++++++++++++----------------------------
 1 file changed, 84 insertions(+), 89 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 34486243e3..82ae7029f2 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -18,14 +18,13 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, liftM)
+import Control.Monad (unless, when, liftM, filterM)
 import Control.Monad.State (liftIO)
 import Control.Applicative ((<$>))
 import System.FilePath
 import System.Directory
 import Data.String.Utils
 import System.Cmd.Utils
-import Data.Maybe
 import System.IO
 import System.IO.Binary
 import System.Posix.Process
@@ -131,8 +130,10 @@ create = unlessM hasBranch $ do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = lockJournal $
-	whenM stageJournalFiles $ do
+commit message = do
+	fs <- getJournalFiles
+	when (not $ null fs) $ lockJournal $ do
+		stageJournalFiles fs
 		g <- Annex.gitRepo
 		withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
@@ -141,25 +142,54 @@ commit message = lockJournal $
 update :: Annex ()
 update = do
 	state <- getState
-	unless (branchUpdated state) $ withIndex $ lockJournal $ do
-		{- Since branches get merged into the index, it's important to
-		 - first stage the journal into the index. Otherwise, any
-		 - changes in the journal would later get staged, and might
-		 - overwrite changes made during the merge.
-		 -
-		 - It would be cleaner to handle the merge by updating the
-		 - journal, not the index, with changes from the branches.
-		 -}
-		staged <- stageJournalFiles
-
-		refs <- siblingBranches
-		updated <- catMaybes <$> mapM updateRef refs
-		g <- Annex.gitRepo
-		unless (null updated && not staged) $ liftIO $
-			Git.commit g "update" fullname (fullname:updated)
-
-		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
-		invalidateCache
+	unless (branchUpdated state) $ do
+		-- check what needs updating before taking the lock
+		fs <- getJournalFiles
+		refs <- filterM checkref =<< siblingBranches
+		unless (null fs && null refs) $ withIndex $ lockJournal $ do
+			{- Before refs are merged into the index, it's
+			 - important to first stage the journal into the
+			 - index. Otherwise, any changes in the journal
+			 - would later get staged, and might overwrite
+			 - changes made during the merge.
+			 -
+			 - It would be cleaner to handle the merge by
+			 - updating the journal, not the index, with changes
+			 - from the branches.
+			 -}
+			unless (null fs) $ stageJournalFiles fs
+			mapM_ mergeref refs
+			g <- Annex.gitRepo
+			liftIO $ Git.commit g "update" fullname (fullname:refs)
+			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
+			invalidateCache
+	where
+		checkref ref = do
+			g <- Annex.gitRepo
+			-- checking with log to see if there have been changes
+			-- is less expensive than always merging
+			diffs <- liftIO $ Git.pipeRead g [
+				Param "log",
+				Param (name++".."++ref),
+				Params "--oneline -n1"
+				]
+			return $ not $ L.null diffs
+		mergeref ref = do
+			showSideAction $ "merging " ++
+				Git.refDescribe ref ++ " into " ++ name
+			{- By passing only one ref, it is actually
+			 - merged into the index, preserving any
+			 - changes that may already be staged.
+			 -
+			 - However, any changes in the git-annex
+			 - branch that are *not* reflected in the
+			 - index will be removed. So, documentation
+			 - advises users not to directly modify the
+			 - branch.
+			 -}
+			g <- Annex.gitRepo
+			liftIO $ Git.UnionMerge.merge g [ref]
+			return $ Just ref
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
@@ -188,35 +218,6 @@ siblingBranches = do
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 	return $ map (last . words . L.unpack) (L.lines r)
 
-{- Ensures that a given ref has been merged into the index. -}
-updateRef :: GitRef -> Annex (Maybe String)
-updateRef ref
-	| ref == fullname = return Nothing
-	| otherwise = do
-		g <- Annex.gitRepo
-		-- checking with log to see if there have been changes
-		-- is less expensive than always merging
-		diffs <- liftIO $ Git.pipeRead g [
-			Param "log",
-			Param (name++".."++ref),
-			Params "--oneline -n1"
-			]
-		if L.null diffs
-			then return Nothing
-			else do
-				showSideAction $ "merging " ++ Git.refDescribe ref ++ " into " ++ name
-				-- By passing only one ref, it is actually
-				-- merged into the index, preserving any
-				-- changes that may already be staged.
-				--
-				-- However, any changes in the git-annex
-				-- branch that are *not* reflected in the
-				-- index will be removed. So, documentation
-				-- advises users not to directly modify the
-				-- branch.
-				liftIO $ Git.UnionMerge.merge g [ref]
-				return $ Just ref
-
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()
 change file a = lockJournal $ get file >>= return . a >>= set file
@@ -253,7 +254,7 @@ files = withIndexUpdate $ do
 	g <- Annex.gitRepo
 	bfiles <- liftIO $ Git.pipeNullSplit g
 		[Params "ls-tree --name-only -r -z", Param fullname]
-	jfiles <- getJournalFiles
+	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
 {- Records content for a file in the branch to the journal.
@@ -282,49 +283,43 @@ getJournalFile file = do
 	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
 		(const $ return Nothing)
 
-{- List of journal files. -}
-getJournalFiles :: Annex [FilePath]
-getJournalFiles = map fileJournal <$> getJournalFilesRaw
+{- List of files that have updated content in the journal. -}
+getJournalledFiles :: Annex [FilePath]
+getJournalledFiles = map fileJournal <$> getJournalFiles
 
-getJournalFilesRaw :: Annex [FilePath]
-getJournalFilesRaw = do
+{- List of existing journal files. -}
+getJournalFiles :: Annex [FilePath]
+getJournalFiles = do
 	g <- Annex.gitRepo
 	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
 		(const $ return [])
 	return $ filter (`notElem` [".", ".."]) fs
 
-{- Stages all journal files into the index, and returns True if the index
- - was modified. -}
-stageJournalFiles :: Annex Bool
-stageJournalFiles = do
-	l <- getJournalFilesRaw
-	if null l
-		then return False
-		else do
-			g <- Annex.gitRepo
-			withIndex $ liftIO $ stage g l
-			return True
-	where
-		stage g fs = do
-			let dir = gitAnnexJournalDir g
-			let paths = map (dir ) fs
-			-- inject all the journal files directly into git
-			-- in one quick command
-			(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
-				Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
-			_ <- forkProcess $ do
-				hPutStr toh $ unlines paths
-				hClose toh
-				exitSuccess
+{- Stages the specified journalfiles. -}
+stageJournalFiles :: [FilePath] -> Annex ()
+stageJournalFiles fs = do
+	g <- Annex.gitRepo
+	withIndex $ liftIO $ do
+		let dir = gitAnnexJournalDir g
+		let paths = map (dir ) fs
+		-- inject all the journal files directly into git
+		-- in one quick command
+		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
+			Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
+		_ <- forkProcess $ do
+			hPutStr toh $ unlines paths
 			hClose toh
-			s <- hGetContents fromh
-			-- update the index, also in just one command
-			Git.UnionMerge.update_index g $
-				index_lines (lines s) $ map fileJournal fs
-			hClose fromh
-			forceSuccess pid
-			mapM_ removeFile paths
-		index_lines shas fs = map genline $ zip shas fs
+			exitSuccess
+		hClose toh
+		s <- hGetContents fromh
+		-- update the index, also in just one command
+		Git.UnionMerge.update_index g $
+			index_lines (lines s) $ map fileJournal fs
+		hClose fromh
+		forceSuccess pid
+		mapM_ removeFile paths
+	where
+		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 
 {- Produces a filename to use in the journal for a file on the branch.

From 003a604a6e48a8a0ffd1564e3399b54e8c673e92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 18:20:29 -0400
Subject: [PATCH 2297/8313] drop the lock on error

---
 Branch.hs | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 82ae7029f2..f1ba97c941 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -32,6 +32,8 @@ import System.Posix.IO
 import System.Posix.Files
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
+import Control.Monad.IO.Control (liftIOOp)
+import qualified Control.Exception.Base
 
 import Types.BranchState
 import qualified Git
@@ -345,8 +347,12 @@ fileJournal = replace "//" "_" . replace "_" "/"
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	g <- Annex.gitRepo
-	h <- liftIO $ createFile (gitAnnexJournalLock g) stdFileMode
-	liftIO $ waitToSetLock h (WriteLock, AbsoluteSeek, 0, 0)
-	r <- a
-	liftIO $ closeFd h
-	return r
+	let file = gitAnnexJournalLock g
+	liftIOOp (Control.Exception.Base.bracket (lock file) unlock) run
+	where
+		lock file = do
+			l <- createFile file stdFileMode
+			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
+			return l
+		unlock = closeFd
+		run _ = a

From 8ef2095fa00408ce6729596a42bc0abdc7778098 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 3 Oct 2011 22:24:57 -0400
Subject: [PATCH 2298/8313] factor out common imports

no code changes
---
 Annex.hs                     |  6 ++---
 AnnexCommon.hs               | 13 ++++++++++
 AnnexQueue.hs                |  6 +----
 Backend.hs                   | 12 +++------
 Backend/SHA.hs               | 16 ++----------
 Backend/URL.hs               |  2 +-
 Backend/WORM.hs              |  6 +----
 Branch.hs                    | 50 ++++++++++++------------------------
 CatFile.hs                   |  6 ++---
 CmdLine.hs                   |  5 +---
 Command.hs                   | 27 ++++++-------------
 Command/Add.hs               | 13 ++--------
 Command/AddUrl.hs            | 11 ++------
 Command/ConfigList.hs        |  6 ++---
 Command/Describe.hs          |  2 +-
 Command/Drop.hs              |  8 +++---
 Command/DropKey.hs           |  3 +--
 Command/DropUnused.hs        | 12 +++------
 Command/Find.hs              |  4 +--
 Command/Fix.hs               |  8 +-----
 Command/FromKey.hs           |  9 +------
 Command/Fsck.hs              | 15 +++--------
 Command/Get.hs               |  3 +--
 Command/InAnnex.hs           |  5 +---
 Command/Init.hs              |  5 ++--
 Command/InitRemote.hs        |  7 +----
 Command/Lock.hs              |  6 +----
 Command/Map.hs               | 11 ++------
 Command/Merge.hs             |  2 +-
 Command/Migrate.hs           | 15 ++---------
 Command/Move.hs              | 12 +++------
 Command/RecvKey.hs           |  6 +----
 Command/Semitrust.hs         |  2 +-
 Command/SendKey.hs           | 11 ++------
 Command/SetKey.hs            |  5 +---
 Command/Status.hs            |  7 +----
 Command/Trust.hs             |  2 +-
 Command/Unannex.hs           | 15 +++--------
 Command/Uninit.hs            |  9 ++-----
 Command/Unlock.hs            | 14 +++-------
 Command/Untrust.hs           |  2 +-
 Command/Unused.hs            | 28 +++++++-------------
 Command/Upgrade.hs           |  2 +-
 Command/Version.hs           |  5 +---
 Command/Whereis.hs           |  5 +---
 Common.hs                    | 46 +++++++++++++++++++++++++++++++++
 Config.hs                    | 15 +++--------
 Content.hs                   | 34 ++++++++----------------
 Crypto.hs                    | 15 +++--------
 Git.hs                       | 16 +-----------
 GitAnnex.hs                  |  6 ++---
 Init.hs                      | 14 +++-------
 Limit.hs                     |  6 ++---
 LocationLog.hs               |  7 +----
 Locations.hs                 |  4 +--
 Messages.hs                  |  4 +--
 Options.hs                   |  3 +--
 PresenceLog.hs               |  4 +--
 Remote.hs                    | 16 ++++--------
 Remote/Bup.hs                | 21 +++------------
 Remote/Directory.hs          | 23 +++++------------
 Remote/Git.hs                | 31 ++++++++--------------
 Remote/Helper/Encryptable.hs |  4 +--
 Remote/Helper/Special.hs     | 11 +++-----
 Remote/Hook.hs               | 20 +++------------
 Remote/Rsync.hs              | 22 +++-------------
 Remote/S3real.hs             | 16 +++---------
 Remote/Web.hs                | 12 ++-------
 RemoteLog.hs                 |  5 +---
 Trust.hs                     |  4 +--
 Types/Key.hs                 |  3 ++-
 UUID.hs                      | 12 +++------
 Upgrade.hs                   |  2 +-
 Upgrade/V0.hs                | 12 ++-------
 Upgrade/V1.hs                | 26 +++++--------------
 Upgrade/V2.hs                | 20 +++------------
 Utility.hs                   |  1 -
 Utility/CopyFile.hs          |  6 ++---
 Version.hs                   |  5 ++--
 git-annex-shell.hs           |  4 +--
 git-annex.cabal              |  2 +-
 git-union-merge.hs           |  4 +--
 test.hs                      |  8 ++----
 83 files changed, 264 insertions(+), 619 deletions(-)
 create mode 100644 AnnexCommon.hs
 create mode 100644 Common.hs

diff --git a/Annex.hs b/Annex.hs
index 8a386a044b..6877537740 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -19,10 +19,10 @@ module Annex (
 	gitRepo
 ) where
 
-import Control.Monad.State
 import Control.Monad.IO.Control
-import Control.Applicative hiding (empty)
+import Control.Monad.State
 
+import Common
 import qualified Git
 import Git.CatFile
 import Git.Queue
@@ -75,7 +75,7 @@ newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, repoqueue = empty
+	, repoqueue = Git.Queue.empty
 	, output = NormalOutput
 	, force = False
 	, fast = False
diff --git a/AnnexCommon.hs b/AnnexCommon.hs
new file mode 100644
index 0000000000..bcdc5e264e
--- /dev/null
+++ b/AnnexCommon.hs
@@ -0,0 +1,13 @@
+module AnnexCommon (
+	module Common,
+	module Types,
+	module Annex,
+	module Locations,
+	module Messages,
+) where
+
+import Common
+import Types
+import Annex (gitRepo)
+import Locations
+import Messages
diff --git a/AnnexQueue.hs b/AnnexQueue.hs
index d155a7b81f..66843a75e3 100644
--- a/AnnexQueue.hs
+++ b/AnnexQueue.hs
@@ -11,13 +11,9 @@ module AnnexQueue (
 	flushWhenFull
 ) where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when, unless)
-
+import AnnexCommon
 import Annex
-import Messages
 import qualified Git.Queue
-import Utility.SafeCommand
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
diff --git a/Backend.hs b/Backend.hs
index ca822de5c1..94fe29607e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -16,20 +16,14 @@ module Backend (
 	maybeLookupBackendName
 ) where
 
-import Control.Monad.State (liftIO, when)
-import Control.Applicative
 import System.IO.Error (try)
-import System.FilePath
 import System.Posix.Files
-import Data.Maybe
 
-import Locations
+import AnnexCommon
 import qualified Git
 import qualified Annex
-import Types
 import Types.Key
 import qualified Types.Backend as B
-import Messages
 
 -- When adding a new backend, import it here and add it to the list.
 import qualified Backend.WORM
@@ -59,7 +53,7 @@ orderedList = do
 			Annex.changeState $ \state -> state { Annex.backends = l' }
 			return l'
 		getstandard = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			return $ parseBackendList $
 				Git.configGet g "annex.backends" ""
 
@@ -108,7 +102,7 @@ type BackendFile = (Maybe (Backend Annex), FilePath)
  -}
 chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	forced <- Annex.getState Annex.forcebackend
 	if isJust forced
 		then do
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 15d3fa20d1..0c36ef0dc7 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -7,23 +7,11 @@
 
 module Backend.SHA (backends) where
 
-import Control.Monad.State
-import Data.String.Utils
-import System.Cmd.Utils
-import System.IO
-import System.Directory
-import Data.Maybe
-import System.Posix.Files
-import System.FilePath
-
-import Messages
+import AnnexCommon
 import qualified Annex
-import Locations
 import Content
-import Types
 import Types.Backend
 import Types.Key
-import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
@@ -110,7 +98,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 {- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	fast <- Annex.getState Annex.fast
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
diff --git a/Backend/URL.hs b/Backend/URL.hs
index f20aa1f95e..0745de455d 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,9 +10,9 @@ module Backend.URL (
 	fromUrl
 ) where
 
+import AnnexCommon
 import Types.Backend
 import Types.Key
-import Types
 
 backends :: [Backend Annex]
 backends = [backend]
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 831c9e8ced..80c652558e 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -7,12 +7,8 @@
 
 module Backend.WORM (backends) where
 
-import Control.Monad.State
-import System.FilePath
-import System.Posix.Files
-
+import AnnexCommon
 import Types.Backend
-import Types
 import Types.Key
 
 backends :: [Backend Annex]
diff --git a/Branch.hs b/Branch.hs
index f1ba97c941..3f1153c098 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -18,33 +18,17 @@ module Branch (
 	name	
 ) where
 
-import Control.Monad (unless, when, liftM, filterM)
-import Control.Monad.State (liftIO)
-import Control.Applicative ((<$>))
-import System.FilePath
-import System.Directory
-import Data.String.Utils
-import System.Cmd.Utils
-import System.IO
 import System.IO.Binary
-import System.Posix.Process
-import System.Posix.IO
-import System.Posix.Files
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 import Control.Monad.IO.Control (liftIOOp)
-import qualified Control.Exception.Base
+import qualified Control.Exception
 
+import AnnexCommon
 import Types.BranchState
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
-import Utility
-import Utility.Conditional
-import Utility.SafeCommand
-import Types
-import Messages
-import Locations
 import CatFile
 
 type GitRef = String
@@ -79,7 +63,7 @@ withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = index g
 	reset <- liftIO $ Git.useIndex f
 
@@ -123,7 +107,7 @@ getCache file = getState >>= handle
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = unlessM hasBranch $ do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	e <- hasOrigin
 	if e
 		then liftIO $ Git.run g "branch" [Param name, Param originname]
@@ -136,7 +120,7 @@ commit message = do
 	fs <- getJournalFiles
 	when (not $ null fs) $ lockJournal $ do
 		stageJournalFiles fs
-		g <- Annex.gitRepo
+		g <- gitRepo
 		withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
@@ -161,13 +145,13 @@ update = do
 			 -}
 			unless (null fs) $ stageJournalFiles fs
 			mapM_ mergeref refs
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ Git.commit g "update" fullname (fullname:refs)
 			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
 	where
 		checkref ref = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			-- checking with log to see if there have been changes
 			-- is less expensive than always merging
 			diffs <- liftIO $ Git.pipeRead g [
@@ -189,14 +173,14 @@ update = do
 			 - advises users not to directly modify the
 			 - branch.
 			 -}
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ Git.UnionMerge.merge g [ref]
 			return $ Just ref
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
 refExists ref = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ Git.runBool g "show-ref"
 		[Param "--verify", Param "-q", Param ref]
 
@@ -216,7 +200,7 @@ hasSomeBranch = not . null <$> siblingBranches
  - from remotes. -}
 siblingBranches :: Annex [String]
 siblingBranches = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
 	return $ map (last . words . L.unpack) (L.lines r)
 
@@ -253,7 +237,7 @@ get file = do
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	bfiles <- liftIO $ Git.pipeNullSplit g
 		[Params "ls-tree --name-only -r -z", Param fullname]
 	jfiles <- getJournalledFiles
@@ -265,7 +249,7 @@ files = withIndexUpdate $ do
  - avoids git needing to rewrite the index after every change. -}
 setJournalFile :: FilePath -> String -> Annex ()
 setJournalFile file content = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ catch (write g) $ const $ do
 		createDirectoryIfMissing True $ gitAnnexJournalDir g
 		createDirectoryIfMissing True $ gitAnnexTmpDir g
@@ -281,7 +265,7 @@ setJournalFile file content = do
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
 getJournalFile file = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
 		(const $ return Nothing)
 
@@ -292,7 +276,7 @@ getJournalledFiles = map fileJournal <$> getJournalFiles
 {- List of existing journal files. -}
 getJournalFiles :: Annex [FilePath]
 getJournalFiles = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
 		(const $ return [])
 	return $ filter (`notElem` [".", ".."]) fs
@@ -300,7 +284,7 @@ getJournalFiles = do
 {- Stages the specified journalfiles. -}
 stageJournalFiles :: [FilePath] -> Annex ()
 stageJournalFiles fs = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
 		let paths = map (dir ) fs
@@ -346,9 +330,9 @@ fileJournal = replace "//" "_" . replace "_" "/"
  - contention with other git-annex processes. -}
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexJournalLock g
-	liftIOOp (Control.Exception.Base.bracket (lock file) unlock) run
+	liftIOOp (Control.Exception.bracket (lock file) unlock) run
 	where
 		lock file = do
 			l <- createFile file stdFileMode
diff --git a/CatFile.hs b/CatFile.hs
index 0eb1e74f6b..8762109e78 100644
--- a/CatFile.hs
+++ b/CatFile.hs
@@ -9,17 +9,15 @@ module CatFile (
 	catFile
 ) where
 
-import Control.Monad.State
-
+import AnnexCommon
 import qualified Git.CatFile
-import Types
 import qualified Annex
 
 catFile :: String -> FilePath -> Annex String
 catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			h <- liftIO $ Git.CatFile.catFileStart g
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
 			go h
diff --git a/CmdLine.hs b/CmdLine.hs
index 38d4754da9..34adb25569 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -13,17 +13,14 @@ module CmdLine (
 
 import System.IO.Error (try)
 import System.Console.GetOpt
-import Control.Monad.State (liftIO)
-import Control.Monad (when)
 
+import AnnexCommon
 import qualified Annex
 import qualified AnnexQueue
 import qualified Git
 import Content
-import Types
 import Command
 import Options
-import Messages
 import Init
 
 {- Runs the passed command line. -}
diff --git a/Command.hs b/Command.hs
index c061c7c464..20f3d79b62 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,22 +7,11 @@
 
 module Command where
 
-import Control.Monad.State (liftIO)
-import System.Directory
-import System.Posix.Files
-import Control.Monad (filterM, liftM)
-import Control.Applicative
-import Data.Maybe
-
-import Types
+import AnnexCommon
 import qualified Backend
-import Messages
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
-import Utility
-import Utility.Conditional
-import Utility.Path
 import Types.Key
 import Trust
 import LocationLog
@@ -98,7 +87,7 @@ isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
-	whenM (Git.repoIsLocalBare <$> Annex.gitRepo) $
+	whenM (Git.repoIsLocalBare <$> gitRepo) $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
@@ -106,11 +95,11 @@ notBareRepo a = do
    user's parameters, and prepare actions operating on them. -}
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	runFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	runFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
@@ -119,7 +108,7 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 		go (file, v) = a file (readMaybe v)
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
 	backendPairs a files
 withFilesMissing :: (String -> CommandStart) -> CommandSeek
@@ -128,7 +117,7 @@ withFilesMissing a params = runFiltered a $ liftIO $ filterM missing params
 		missing = liftM not . doesFileExist
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	force <- Annex.getState Annex.force
 	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
 	backendPairs a newfiles
@@ -138,7 +127,7 @@ withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = do
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	runFiltered a $
 		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
 withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
@@ -148,7 +137,7 @@ withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	repo <- Annex.gitRepo
+	repo <- gitRepo
 	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
diff --git a/Command/Add.hs b/Command/Add.hs
index 4b2ef24cd9..c66c381311 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,26 +7,17 @@
 
 module Command.Add where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (when)
-import System.Posix.Files
-import System.Directory
 import Control.Exception.Control (handle)
 import Control.Exception.Base (throwIO)
-import Control.Exception.Extensible (IOException)
 
+import AnnexCommon
 import Command
 import qualified Annex
 import qualified AnnexQueue
 import qualified Backend
 import LocationLog
-import Types
 import Content
-import Messages
-import Utility.Conditional
 import Utility.Touch
-import Utility.SafeCommand
-import Locations
 import Backend
 
 command :: [Command]
@@ -72,7 +63,7 @@ undo file key e = do
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ renameFile (gitAnnexLocation g key) file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 2e9e04fd3e..ce6e70699c 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -7,12 +7,9 @@
 
 module Command.AddUrl where
 
-import Control.Monad.State
 import Network.URI
-import Data.String.Utils
-import Data.Maybe
-import System.Directory
 
+import AnnexCommon
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
@@ -20,12 +17,8 @@ import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
-import Messages
 import Content
 import PresenceLog
-import Locations
-import Utility.Path
-import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "addurl" (paramRepeating paramUrl) seek
@@ -51,7 +44,7 @@ perform url file = do
 
 download :: String -> FilePath -> CommandPerform
 download url file = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	showAction $ "downloading " ++ url ++ " "
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 3de26c8925..c38539fa0b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -7,9 +7,7 @@
 
 module Command.ConfigList where
 
-import Control.Monad.State (liftIO)
-
-import Annex
+import AnnexCommon
 import Command
 import UUID
 
@@ -22,7 +20,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	liftIO $ putStrLn $ "annex.uuid=" ++ u
 	stop
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 8d2f9071b0..b1c144872b 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,10 +7,10 @@
 
 module Command.Describe where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import UUID
-import Messages
 
 command :: [Command]
 command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 4a7596921f..7210184f88 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,14 +7,12 @@
 
 module Command.Drop where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import qualified Annex
 import LocationLog
-import Types
 import Content
-import Messages
-import Utility.Conditional
 import Trust
 import Config
 
@@ -71,9 +69,9 @@ dropKey key numcopiesM = do
 			| length have >= need = return True
 			| otherwise = do
 				let u = Remote.uuid r
-				let dup = u `elem` have
+				let duplicate = u `elem` have
 				haskey <- Remote.hasKey r key
-				case (dup, haskey) of
+				case (duplicate, haskey) of
 					(False, Right True)	-> findcopies need (u:have) rs bad
 					(False, Left _)		-> findcopies need have rs (r:bad)
 					_			-> findcopies need have rs bad
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index b9938585e9..7ead1c4bca 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -7,12 +7,11 @@
 
 module Command.DropKey where
 
+import AnnexCommon
 import Command
 import qualified Annex
 import LocationLog
-import Types
 import Content
-import Messages
 
 command :: [Command]
 command = [repoCommand "dropkey" (paramRepeating paramKey) seek
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 90fea050e8..ed4f71e7e4 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,22 +7,16 @@
 
 module Command.DropUnused where
 
-import Control.Monad.State (liftIO)
 import qualified Data.Map as M
-import System.Directory
-import Data.Maybe
 
+import AnnexCommon
 import Command
-import Types
-import Messages
-import Locations
 import qualified Annex
 import qualified Command.Drop
 import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
-import Utility.Conditional
 
 type UnusedMap = M.Map String Key
 
@@ -67,14 +61,14 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 
 performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = filespec g key
 	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = gitAnnexUnusedLog prefix g
 	e <- liftIO $ doesFileExist f
 	if e
diff --git a/Command/Find.hs b/Command/Find.hs
index effb331840..8d80659d0d 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,9 @@
 
 module Command.Find where
 
-import Control.Monad.State
-
+import AnnexCommon
 import Command
 import Content
-import Utility.Conditional
 import Limit
 
 command :: [Command]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 481da52f2b..a66a1c44a7 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -7,16 +7,10 @@
 
 module Command.Fix where
 
-import Control.Monad.State (liftIO)
-import System.Posix.Files
-import System.Directory
-
+import AnnexCommon
 import Command
 import qualified AnnexQueue
-import Utility.Path
-import Utility.SafeCommand
 import Content
-import Messages
 
 command :: [Command]
 command = [repoCommand "fix" paramPaths seek
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 9ff126a45e..e60025bf7e 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -7,18 +7,11 @@
 
 module Command.FromKey where
 
-import Control.Monad.State (liftIO)
-import System.Posix.Files
-import System.Directory
-import Control.Monad (unless)
-
+import AnnexCommon
 import Command
 import qualified AnnexQueue
-import Utility.SafeCommand
 import Content
-import Messages
 import Types.Key
-import Utility.Path
 
 command :: [Command]
 command = [repoCommand "fromkey" paramPath seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0c58add6a3..33a8405a69 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,25 +7,16 @@
 
 module Command.Fsck where
 
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
-import System.Directory
-import System.Posix.Files
-
+import AnnexCommon
 import Command
-import qualified Annex
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
 import UUID
-import Types
-import Messages
 import Content
 import LocationLog
-import Locations
 import Trust
 import Utility.DataUnits
-import Utility.Path
 import Utility.FileMode
 import Config
 
@@ -54,7 +45,7 @@ perform key file backend numcopies = do
    in this repository only. -}
 verifyLocationLog :: Key -> FilePath -> Annex Bool
 verifyLocationLog key file = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	present <- inAnnex key
 	
 	-- Since we're checking that a key's file is present, throw
@@ -98,7 +89,7 @@ fsckKey backend key file numcopies = do
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexLocation g key
 	present <- liftIO $ doesFileExist file
 	case (present, Types.Key.keySize key) of
diff --git a/Command/Get.hs b/Command/Get.hs
index 4fd654f63e..34f56aa2d5 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -7,12 +7,11 @@
 
 module Command.Get where
 
+import AnnexCommon
 import Command
 import qualified Annex
 import qualified Remote
-import Types
 import Content
-import Messages
 import qualified Command.Move
 
 command :: [Command]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 713492c2fe..36b6d40e6e 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -7,12 +7,9 @@
 
 module Command.InAnnex where
 
-import Control.Monad.State (liftIO)
-import System.Exit
-
+import AnnexCommon
 import Command
 import Content
-import Types
 
 command :: [Command]
 command = [repoCommand "inannex" (paramRepeating paramKey) seek
diff --git a/Command/Init.hs b/Command/Init.hs
index 2351763a95..f3d8834ba1 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,10 +7,9 @@
 
 module Command.Init where
 
+import AnnexCommon
 import Command
-import qualified Annex
 import UUID
-import Messages
 import Init
 	
 command :: [Command]
@@ -30,7 +29,7 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	describeUUID u description
 	next $ return True
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index c6d9f52003..2ce86e9c6b 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -8,18 +8,13 @@
 module Command.InitRemote where
 
 import qualified Data.Map as M
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
-import Data.Maybe
-import Data.String.Utils
 
+import AnnexCommon
 import Command
 import qualified Remote
 import qualified RemoteLog
 import qualified Types.Remote as R
-import Types
 import UUID
-import Messages
 
 command :: [Command]
 command = [repoCommand "initremote"
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 04d1bb94d6..af7b92ad68 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -7,13 +7,9 @@
 
 module Command.Lock where
 
-import Control.Monad.State (liftIO)
-import System.Directory
-
+import AnnexCommon
 import Command
-import Messages
 import qualified AnnexQueue
-import Utility.SafeCommand
 import Backend
 	
 command :: [Command]
diff --git a/Command/Map.hs b/Command/Map.hs
index 7e23da774d..8e63f6dd6a 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -7,19 +7,12 @@
 
 module Command.Map where
 
-import Control.Monad.State (liftIO)
 import Control.Exception.Extensible
-import System.Cmd.Utils
 import qualified Data.Map as M
-import Data.List.Utils
-import Data.Maybe
 
+import AnnexCommon
 import Command
-import qualified Annex
 import qualified Git
-import Messages
-import Types
-import Utility.SafeCommand
 import UUID
 import Trust
 import Utility.Ssh
@@ -36,7 +29,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	rs <- spider g
 
 	umap <- uuidMap
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 832cde5127..b365e0e0cd 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -7,9 +7,9 @@
 
 module Command.Merge where
 
+import AnnexCommon
 import Command
 import qualified Branch
-import Messages
 
 command :: [Command]
 command = [repoCommand "merge" paramNothing seek
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 054db6e27b..24f23baf5c 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -7,22 +7,11 @@
 
 module Command.Migrate where
 
-import Control.Monad.State (liftIO)
-import Control.Applicative
-import System.Posix.Files
-import System.Directory
-import System.FilePath
-import Data.Maybe
-
+import AnnexCommon
 import Command
-import qualified Annex
 import qualified Backend
 import qualified Types.Key
-import Locations
-import Types
 import Content
-import Messages
-import Utility.Conditional
 import qualified Command.Add
 import Backend
 
@@ -53,7 +42,7 @@ upgradableKey key = isNothing $ Types.Key.keySize key
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	-- Store the old backend's cached key in the new backend
 	-- (the file can't be stored as usual, because it's already a symlink).
diff --git a/Command/Move.hs b/Command/Move.hs
index 15dae39385..d2870b1e42 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,18 +7,14 @@
 
 module Command.Move where
 
-import Control.Monad (when)
-
+import AnnexCommon
 import Command
 import qualified Command.Drop
 import qualified Annex
 import LocationLog
-import Types
 import Content
 import qualified Remote
 import UUID
-import Messages
-import Utility.Conditional
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
@@ -60,7 +56,7 @@ showMoveAction False file = showStart "copy" file
 remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
 remoteHasKey remote key present	= do
 	let remoteuuid = Remote.uuid remote
-	g <- Annex.gitRepo
+	g <- gitRepo
 	logChange g key remoteuuid status
 	where
 		status = if present then InfoPresent else InfoMissing
@@ -76,7 +72,7 @@ remoteHasKey remote key present	= do
  -}
 toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 toStart dest move file = isAnnexed file $ \(key, _) -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
@@ -126,7 +122,7 @@ toCleanup dest move key = do
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 fromStart src move file = isAnnexed file $ \(key, _) -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	remotes <- Remote.keyPossibilities key
 	if u == Remote.uuid src || not (any (== src) remotes)
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 33792e5b6e..400e81102b 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,15 +7,11 @@
 
 module Command.RecvKey where
 
-import Control.Monad.State (liftIO)
-import System.Exit
-
+import AnnexCommon
 import Command
 import CmdLine
 import Content
 import Utility.RsyncFile
-import Utility.Conditional
-import Types
 
 command :: [Command]
 command = [repoCommand "recvkey" paramKey seek
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 3b12bb747d..236ba28794 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -7,11 +7,11 @@
 
 module Command.Semitrust where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import UUID
 import Trust
-import Messages
 
 command :: [Command]
 command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 98d2573380..f397d9ae6a 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,17 +7,10 @@
 
 module Command.SendKey where
 
-import Control.Monad.State (liftIO)
-import System.Exit
-
-import Locations
-import qualified Annex
+import AnnexCommon
 import Command
 import Content
 import Utility.RsyncFile
-import Utility.Conditional
-import Messages
-import Types
 
 command :: [Command]
 command = [repoCommand "sendkey" paramKey seek
@@ -28,7 +21,7 @@ seek = [withKeys start]
 
 start :: Key -> CommandStart
 start key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexLocation g key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index c03c5d0445..12ef5b74a5 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -7,13 +7,10 @@
 
 module Command.SetKey where
 
-import Control.Monad.State (liftIO)
-
+import AnnexCommon
 import Command
-import Utility.SafeCommand
 import LocationLog
 import Content
-import Messages
 
 command :: [Command]
 command = [repoCommand "setkey" paramPath seek
diff --git a/Command/Status.hs b/Command/Status.hs
index 07c0958bbe..de49f84d5a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -8,25 +8,20 @@
 module Command.Status where
 
 import Control.Monad.State
-import Control.Applicative
-import Data.Maybe
-import System.IO
-import Data.List
 import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
 
+import AnnexCommon
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
 import qualified Git
 import Command
-import Types
 import Utility.DataUnits
 import Content
 import Types.Key
-import Locations
 import Backend
 import UUID
 import Remote
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 5e25b519bc..04c68a5d32 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,11 +7,11 @@
 
 module Command.Trust where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import Trust
 import UUID
-import Messages
 
 command :: [Command]
 command = [repoCommand "trust" (paramRepeating paramRemote) seek
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 4d4281eb07..c5c5e90a67 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -7,25 +7,16 @@
 
 module Command.Unannex where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (unless)
-import System.Directory
-import System.Posix.Files
-
+import AnnexCommon
 import Command
 import qualified Command.Drop
 import qualified Annex
 import qualified AnnexQueue
-import Utility.SafeCommand
-import Utility.Path
 import Utility.FileMode
 import LocationLog
-import Types
 import Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
-import Messages
-import Locations
 
 command :: [Command]
 command = [repoCommand "unannex" paramPaths seek "undo accidential add command"]
@@ -41,7 +32,7 @@ start file = isAnnexed file $ \(key, _) -> do
 		then do
 			force <- Annex.getState Annex.force
 			unless force $ do
-				g <- Annex.gitRepo
+				g <- gitRepo
 				staged <- liftIO $ LsFiles.staged g [Git.workTree g]
 				unless (null staged) $
 					error "This command cannot be run when there are already files staged for commit."
@@ -60,7 +51,7 @@ perform file key = do
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	liftIO $ removeFile file
 	liftIO $ Git.run g "rm" [Params "--quiet --", File file]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ce12665424..3ba7a7cf33 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,19 +7,14 @@
 
 module Command.Uninit where
 
-import Control.Monad.State (liftIO)
-import System.Directory
-import System.Exit
-
+import AnnexCommon
 import Command
-import Utility.SafeCommand
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import Init
 import qualified Branch
 import Content
-import Locations
 
 command :: [Command]
 command = [repoCommand "uninit" paramPaths seek 
@@ -44,7 +39,7 @@ perform = next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 44b92545c8..220d57829b 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,18 +7,10 @@
 
 module Command.Unlock where
 
-import Control.Monad.State (liftIO)
-import System.Directory hiding (copyFile)
-
+import AnnexCommon
 import Command
-import qualified Annex
-import Types
-import Messages
-import Locations
 import Content
-import Utility.Conditional
 import Utility.CopyFile
-import Utility.Path
 import Utility.FileMode
 
 command :: [Command]
@@ -43,12 +35,12 @@ perform dest key = do
 	
 	checkDiskSpace key
 
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g key
 	let tmpdest = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
-	ok <- liftIO $ copyFile src tmpdest
+	ok <- liftIO $ copyFileExternal src tmpdest
         if ok
                 then do
 			liftIO $ do
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 9f7e521987..30ade85ce0 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,11 +7,11 @@
 
 module Command.Untrust where
 
+import AnnexCommon
 import Command
 import qualified Remote
 import UUID
 import Trust
-import Messages
 
 command :: [Command]
 command = [repoCommand "untrust" (paramRepeating paramRemote) seek
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 987f367201..1ba4f53019 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -9,23 +9,13 @@
 
 module Command.Unused where
 
-import Control.Monad (filterM, unless, forM_)
-import Control.Monad.State (liftIO)
 import qualified Data.Set as S
-import Data.Maybe
-import System.FilePath
-import System.Directory
-import Data.List
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import AnnexCommon
 import Command
-import Types
 import Content
-import Messages
-import Locations
-import Utility
 import Utility.FileMode
-import Utility.SafeCommand
 import LocationLog
 import qualified Annex
 import qualified Git
@@ -92,7 +82,7 @@ checkRemoteUnused' r = do
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
@@ -164,7 +154,7 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
 	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
@@ -200,7 +190,7 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]
 getKeysReferenced = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
@@ -209,7 +199,7 @@ getKeysReferenced = do
 getKeysReferencedInGit :: String -> Annex [Key]
 getKeysReferencedInGit ref = do
 	showAction $ "checking " ++ Git.refDescribe ref
-	g <- Annex.gitRepo
+	g <- gitRepo
 	findkeys [] =<< liftIO (LsTree.lsTree g ref)
 	where
 		findkeys c [] = return c
@@ -232,17 +222,17 @@ staleKeysPrune dirspec present = do
 	contents <- staleKeys dirspec
 	
 	let stale = contents `exclude` present
-	let dup = contents `exclude` stale
+	let dups = contents `exclude` stale
 
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let dir = dirspec g
-	liftIO $ forM_ dup $ \t -> removeFile $ dir  keyFile t
+	liftIO $ forM_ dups $ \t -> removeFile $ dir  keyFile t
 
 	return stale
 
 staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
 staleKeys dirspec = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let dir = dirspec g
 	exists <- liftIO $ doesDirectoryExist dir
 	if not exists
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 5d9ed92fae..d79f895d8d 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -7,10 +7,10 @@
 
 module Command.Upgrade where
 
+import AnnexCommon
 import Command
 import Upgrade
 import Version
-import Messages
 
 command :: [Command]
 command = [standaloneCommand "upgrade" paramNothing seek
diff --git a/Command/Version.hs b/Command/Version.hs
index af547949c6..1e44fbb0b1 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -7,10 +7,7 @@
 
 module Command.Version where
 
-import Control.Monad.State (liftIO)
-import Data.String.Utils
-import Data.Maybe
-
+import AnnexCommon
 import Command
 import qualified Build.SysConfig as SysConfig
 import Version
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index a414428f72..3fb636c04c 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,13 +7,10 @@
 
 module Command.Whereis where
 
-import Control.Monad
-
+import AnnexCommon
 import LocationLog
 import Command
-import Messages
 import Remote
-import Types
 import Trust
 
 command :: [Command]
diff --git a/Common.hs b/Common.hs
new file mode 100644
index 0000000000..e88342ae4a
--- /dev/null
+++ b/Common.hs
@@ -0,0 +1,46 @@
+module Common (
+	module Control.Monad,
+	module Control.Applicative,
+	module Control.Monad.State,
+	module Control.Exception.Extensible,
+	module Data.Maybe,
+	module Data.List,
+	module Data.String.Utils,
+	module System.Path,
+	module System.FilePath,
+	module System.Directory,
+	module System.Cmd.Utils,
+	module System.IO,
+	module System.Posix.Files,
+	module System.Posix.IO,
+	module System.Posix.Process,
+	module System.Exit,
+	module Utility,
+	module Utility.Conditional,
+	module Utility.SafeCommand,
+	module Utility.Path,
+) where
+
+import Control.Monad hiding (join)
+import Control.Applicative
+import Control.Monad.State (liftIO)
+import Control.Exception.Extensible (IOException)
+
+import Data.Maybe
+import Data.List
+import Data.String.Utils
+
+import System.Path
+import System.FilePath
+import System.Directory
+import System.Cmd.Utils
+import System.IO hiding (FilePath)
+import System.Posix.Files
+import System.Posix.IO
+import System.Posix.Process hiding (executeFile)
+import System.Exit
+
+import Utility
+import Utility.Conditional
+import Utility.SafeCommand
+import Utility.Path
diff --git a/Config.hs b/Config.hs
index fe847fce1c..c0328794e1 100644
--- a/Config.hs
+++ b/Config.hs
@@ -7,23 +7,16 @@
 
 module Config where
 
-import Data.Maybe
-import Control.Monad.State (liftIO)
-import Control.Applicative
-import System.Cmd.Utils
-
+import AnnexCommon
 import qualified Git
 import qualified Annex
-import Types
-import Utility
-import Utility.SafeCommand
 
 type ConfigKey = String
 
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ Git.run g "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
 	g' <- liftIO $ Git.configRead g
@@ -33,7 +26,7 @@ setConfig k value = do
  - Failing that, tries looking for a global config option. -}
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let def' = Git.configGet g ("annex." ++ key) def
 	return $ Git.configGet g (remoteConfig r key) def'
 
@@ -95,7 +88,7 @@ getNumCopies v =
 	where
 		use (Just n) = return n
 		use Nothing = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
 
diff --git a/Content.hs b/Content.hs
index f963c48b47..567e4caa5d 100644
--- a/Content.hs
+++ b/Content.hs
@@ -21,26 +21,14 @@ module Content (
 	saveState
 ) where
 
-import System.Directory
-import Control.Monad.State (liftIO)
-import System.Path
-import Control.Monad
-import System.Posix.Files
-import System.FilePath
-import Data.Maybe
-
-import Types
-import Locations
+import AnnexCommon
 import LocationLog
 import UUID
 import qualified Git
 import qualified Annex
 import qualified AnnexQueue
 import qualified Branch
-import Utility
-import Utility.Conditional
 import Utility.StatFS
-import Utility.Path
 import Utility.FileMode
 import Types.Key
 import Utility.DataUnits
@@ -49,14 +37,14 @@ import Config
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
 inAnnex key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"
 	liftIO $ doesFileExist $ gitAnnexLocation g key
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	cwd <- liftIO getCurrentDirectory
 	let absfile = fromMaybe whoops $ absNormPath cwd file
 	return $ relPathDirToFile (parentDir absfile) 
@@ -68,7 +56,7 @@ calcGitLink file key = do
  - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	logChange g key u status
 
@@ -77,7 +65,7 @@ logStatus key status = do
  - the annex as a key's content. -}
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let tmp = gitAnnexTmpLocation g key
 
 	-- Check that there is enough free disk space.
@@ -96,7 +84,7 @@ getViaTmp key action = do
 
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let tmp = gitAnnexTmpLocation g key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	return tmp
@@ -133,7 +121,7 @@ checkDiskSpace = checkDiskSpace' 0
 
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	r <- getConfig g "diskreserve" ""
 	let reserve = fromMaybe megabyte $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
@@ -174,7 +162,7 @@ checkDiskSpace' adjustment key = do
  -}
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let dest = gitAnnexLocation g key
 	let dir = parentDir dest
 	e <- liftIO $ doesFileExist dest
@@ -189,7 +177,7 @@ moveAnnex key src = do
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let file = gitAnnexLocation g key
 	let dir = parentDir file
 	a (dir, file)
@@ -213,7 +201,7 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
  - returns the file it was moved to. -}
 moveBad :: Key -> Annex FilePath
 moveBad key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g key
 	let dest = gitAnnexBadDir g  takeFileName src
 	liftIO $ do
@@ -227,7 +215,7 @@ moveBad key = do
 {- List of keys whose content exists in .git/annex/objects/ -}
 getKeysPresent :: Annex [Key]
 getKeysPresent = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	getKeysPresent' $ gitAnnexObjectDir g
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
diff --git a/Crypto.hs b/Crypto.hs
index 9b2d73f28d..4cc16b4242 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -30,26 +30,17 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.ByteString.Lazy.UTF8 (fromString)
 import Data.Digest.Pure.SHA
-import System.Cmd.Utils
-import Data.String.Utils
-import Data.List
-import Data.Maybe
-import System.IO
-import System.Posix.IO
 import System.Posix.Types
-import System.Posix.Process
 import Control.Applicative
 import Control.Concurrent
 import Control.Exception (finally)
 import System.Exit
 import System.Environment
 
-import Types
+import AnnexCommon
 import Types.Key
 import Types.Remote
-import Utility
 import Utility.Base64
-import Utility.SafeCommand
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
@@ -97,9 +88,9 @@ updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher
 updateCipher c encipher@(EncryptedCipher _ ks) = do
 	ks' <- configKeyIds c
 	cipher <- decryptCipher c encipher
-	encryptCipher cipher (combine ks ks')
+	encryptCipher cipher (merge ks ks')
 	where
-		combine (KeyIds a) (KeyIds b) = KeyIds $ a ++ b
+		merge (KeyIds a) (KeyIds b) = KeyIds $ a ++ b
 
 describeCipher :: EncryptedCipher -> String
 describeCipher (EncryptedCipher _ (KeyIds ks)) =
diff --git a/Git.hs b/Git.hs
index d32aaaa562..7a3d55deaa 100644
--- a/Git.hs
+++ b/Git.hs
@@ -64,34 +64,20 @@ module Git (
 	prop_idempotent_deencode
 ) where
 
-import Control.Monad (unless, when, liftM2)
-import Control.Applicative
-import System.Directory
-import System.FilePath
 import System.Posix.Directory
 import System.Posix.User
-import System.Posix.Process
-import System.Path
-import System.Cmd.Utils
 import IO (bracket_, try)
-import Data.String.Utils
-import System.IO
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
-import Data.Maybe
 import Data.Char
 import Data.Word (Word8)
 import Codec.Binary.UTF8.String (encode)
 import Text.Printf
-import Data.List (isInfixOf, isPrefixOf, isSuffixOf)
 import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Utility
-import Utility.Path
-import Utility.Conditional
-import Utility.SafeCommand
+import Common
 
 {- There are two types of repositories; those on local disk and those
  - accessed via an URL. -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index a9d469b44e..fcfbf44b4b 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -8,14 +8,12 @@
 module GitAnnex where
 
 import System.Console.GetOpt
-import Control.Monad.State (liftIO)
 
+import AnnexCommon
 import qualified Git
 import CmdLine
 import Command
 import Options
-import Utility
-import Types
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
@@ -122,7 +120,7 @@ options = commonOptions ++
 		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
-			g <- Annex.gitRepo
+			g <- gitRepo
 			g' <- liftIO $ Git.configStore g v
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 
diff --git a/Init.hs b/Init.hs
index 4df1599333..57eaf39d20 100644
--- a/Init.hs
+++ b/Init.hs
@@ -11,18 +11,10 @@ module Init (
 	uninitialize
 ) where
 
-import Control.Monad.State (liftIO)
-import Control.Monad (unless)
-import System.Directory
-
-import qualified Annex
+import AnnexCommon
 import qualified Git
 import qualified Branch
 import Version
-import Messages
-import Types
-import Utility
-import Utility.Conditional
 import UUID
 
 initialize :: Annex ()
@@ -73,12 +65,12 @@ gitPreCommitHookUnWrite = unlessBare $ do
 
 unlessBare :: Annex () -> Annex ()
 unlessBare a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	unless (Git.repoIsLocalBare g) a
 
 preCommitHook :: Annex FilePath
 preCommitHook = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	return $ Git.gitDir g ++ "/hooks/pre-commit"
 
 preCommitScript :: String
diff --git a/Limit.hs b/Limit.hs
index 10fc0ea6ca..334ae325d6 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -9,15 +9,13 @@ module Limit where
 
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
-import Control.Applicative
-import Data.Maybe
 
-import Annex
+import AnnexCommon
+import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
 import LocationLog
-import Utility
 import Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
diff --git a/LocationLog.hs b/LocationLog.hs
index 0cdf88bc6b..759bee830e 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -21,15 +21,10 @@ module LocationLog (
 	logFileKey
 ) where
 
-import System.FilePath
-import Control.Applicative
-import Data.Maybe
-
+import AnnexCommon
 import qualified Git
 import qualified Branch
 import UUID
-import Types
-import Locations
 import PresenceLog
 
 {- Log a change in the presence of a key's value in a repository. -}
diff --git a/Locations.hs b/Locations.hs
index b18444e72b..4579fe05b7 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -26,13 +26,11 @@ module Locations (
 	prop_idempotent_fileKey
 ) where
 
-import System.FilePath
-import Data.String.Utils
-import Data.List
 import Bits
 import Word
 import Data.Hash.MD5
 
+import Common
 import Types
 import Types.Key
 import qualified Git
diff --git a/Messages.hs b/Messages.hs
index c663c17c20..e029c50722 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -23,11 +23,9 @@ module Messages (
 	setupConsole
 ) where
 
-import Control.Monad.State (liftIO)
-import System.IO
-import Data.String.Utils
 import Text.JSON
 
+import Common
 import Types
 import qualified Annex
 import qualified Messages.JSON as JSON
diff --git a/Options.hs b/Options.hs
index b5eaf98cd8..9d60292002 100644
--- a/Options.hs
+++ b/Options.hs
@@ -9,10 +9,9 @@ module Options where
 
 import System.Console.GetOpt
 import System.Log.Logger
-import Control.Monad.State (liftIO)
 
+import AnnexCommon
 import qualified Annex
-import Types
 import Command
 import Limit
 
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 2db1ee59b4..23c2882571 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -26,11 +26,9 @@ import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
-import Control.Applicative
 
+import AnnexCommon
 import qualified Branch
-import Types
 
 data LogLine = LogLine {
 	date :: POSIXTime,
diff --git a/Remote.hs b/Remote.hs
index efa7a5cc8e..2371b7bf2d 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -28,23 +28,17 @@ module Remote (
 	forceTrust
 ) where
 
-import Control.Monad.State (filterM)
-import Data.List
 import qualified Data.Map as M
-import Data.String.Utils
-import Data.Maybe
-import Control.Applicative
 import Text.JSON
 import Text.JSON.Generic
 
-import Types
+import AnnexCommon
 import Types.Remote
 import UUID
 import qualified Annex
 import Config
 import Trust
 import LocationLog
-import Messages
 import RemoteLog
 
 import qualified Remote.Git
@@ -110,7 +104,7 @@ byName' n = do
  - and returns its UUID. Finds even remotes that are not configured in
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
-nameToUUID "." = getUUID =<< Annex.gitRepo -- special case for current repo
+nameToUUID "." = getUUID =<< gitRepo -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
@@ -135,7 +129,7 @@ nameToUUID n = do
  - of the UUIDs. -}
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
-	here <- getUUID =<< Annex.gitRepo
+	here <- getUUID =<< gitRepo
 	m <- M.unionWith addname <$> uuidMap <*> remoteMap
 	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
@@ -184,7 +178,7 @@ keyPossibilitiesTrusted = keyPossibilities' True
 
 keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
 keyPossibilities' withtrusted key = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	trusted <- if withtrusted then trustGet Trusted else return []
 
@@ -204,7 +198,7 @@ keyPossibilities' withtrusted key = do
 {- Displays known locations of a key. -}
 showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	u <- getUUID g
 	uuids <- keyLocations key
 	untrusteduuids <- trustGet UnTrusted
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 29c7a0419f..9588310194 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -8,30 +8,15 @@
 module Remote.Bup (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import System.IO
 import System.IO.Error
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
 import System.Process
-import System.Exit
-import System.FilePath
-import Data.Maybe
-import Data.List.Utils
-import System.Cmd.Utils
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
 import Config
-import Utility
-import Utility.Conditional
-import Utility.SafeCommand
-import Messages
 import Utility.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
@@ -118,14 +103,14 @@ bupSplitParams r buprepo k src = do
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	params <- bupSplitParams r buprepo k (File src)
 	liftIO $ boolSystem "bup" params
 
 storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	params <- bupSplitParams r buprepo enck (Param "-")
 	liftIO $ catchBool $
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 18835c5de0..664f8ca5fe 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -9,25 +9,14 @@ module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
-import System.Directory hiding (copyFile)
-import System.FilePath
-import Data.Maybe
 
-import Types
+import AnnexCommon
+import Utility.CopyFile
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
-import Utility.CopyFile
 import Config
-import Utility
-import Utility.Conditional
-import Utility.Path
 import Utility.FileMode
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
@@ -82,14 +71,14 @@ dirKey d k = d  hashDirMixed k  f  f
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	let dest = dirKey d k
-	liftIO $ catchBool $ storeHelper dest $ copyFile src dest
+	liftIO $ catchBool $ storeHelper dest $ copyFileExternal src dest
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let src = gitAnnexLocation g k
 	let dest = dirKey d enck
 	liftIO $ catchBool $ storeHelper dest $ encrypt src dest
@@ -110,7 +99,7 @@ storeHelper dest a = do
 	return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
-retrieve d k f = liftIO $ copyFile (dirKey d k) f
+retrieve d k f = liftIO $ copyFileExternal (dirKey d k) f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d50899c674..a457c5905d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -8,26 +8,17 @@
 module Remote.Git (remote) where
 
 import Control.Exception.Extensible
-import Control.Monad.State (liftIO)
 import qualified Data.Map as M
-import System.Cmd.Utils
-import System.Posix.Files
-import System.IO
 
-import Types
-import Types.Remote
-import qualified Git
-import qualified Annex
-import Locations
-import UUID
-import Utility
-import qualified Content
-import Messages
+import AnnexCommon
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
-import Utility.SafeCommand
-import Utility.Path
+import Types.Remote
+import qualified Git
+import qualified Annex
+import UUID
+import qualified Content
 import qualified Utility.Url as Url
 import Config
 import Init
@@ -42,7 +33,7 @@ remote = RemoteType {
 
 list :: Annex [Git.Repo]
 list = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	return $ Git.remotes g
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
@@ -109,7 +100,7 @@ tryGitConfigRead r
 
 		store a = do
 			r' <- a
-			g <- Annex.gitRepo
+			g <- gitRepo
 			let l = Git.remotes g
 			let g' = Git.remotesAdd g $ exchange l r'
 			Annex.changeState $ \s -> s { Annex.repo = g' }
@@ -169,7 +160,7 @@ copyFromRemote r key file
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
-		g <- Annex.gitRepo
+		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
@@ -178,7 +169,7 @@ copyToRemote r key
 			Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
-		g <- Annex.gitRepo
+		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
@@ -200,7 +191,7 @@ rsyncOrCopyFile r src dest = do
 	ss <- liftIO $ getFileStatus $ parentDir src
 	ds <- liftIO $ getFileStatus $ parentDir dest
 	if deviceID ss == deviceID ds
-		then liftIO $ copyFile src dest
+		then liftIO $ copyFileExternal src dest
 		else do
 			params <- rsyncParams r
 			rsyncHelper $ params ++ [Param src, Param dest]
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 04041c6553..42503e4d41 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -8,13 +8,11 @@
 module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
 
-import Types
+import AnnexCommon
 import Types.Remote
 import Crypto
 import qualified Annex
-import Messages
 import Config
 
 {- Encryption setup for a remote. The user must specify whether to use
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index b842588c0c..905db04c55 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -8,16 +8,11 @@
 module Remote.Helper.Special where
 
 import qualified Data.Map as M
-import Data.Maybe
-import Data.String.Utils
-import Control.Monad.State (liftIO)
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Utility.SafeCommand
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
@@ -25,7 +20,7 @@ import Utility.SafeCommand
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	return $ map construct $ remotepairs g
 	where
 		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
@@ -35,7 +30,7 @@ findSpecialRemotes s = do
 {- Sets up configuration for a special remote in .git/config. -}
 gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex ()
 gitConfigSpecialRemote u c k v = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
 		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index aaeb702c7b..3bbda19240 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -8,31 +8,19 @@
 module Remote.Hook (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
-import System.FilePath
-import System.Posix.Process hiding (executeFile)
-import System.Posix.IO
-import System.IO
 import System.IO.Error (try)
 import System.Exit
-import Data.Maybe
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
 import Config
 import Content
-import Utility
-import Utility.SafeCommand
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
-import Messages
 
 remote :: RemoteType Annex
 remote = RemoteType {
@@ -86,7 +74,7 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	command <- getConfig g hookname ""
 	if null command
 		then do
@@ -111,12 +99,12 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 
 store :: String -> Key -> Annex Bool
 store h k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	runHook h "store" k (Just $ gitAnnexLocation g k) $ return True
 
 storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = gitAnnexLocation g k
 	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 	runHook h "store" enck (Just tmp) $ return True
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9d2d7ddcf7..6a1c297c5d 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -8,32 +8,18 @@
 module Remote.Rsync (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import Control.Exception.Extensible (IOException)
 import qualified Data.Map as M
-import Control.Monad.State (liftIO)
-import System.FilePath
-import System.Directory
-import System.Posix.Files
-import System.Posix.Process
-import Data.Maybe
 
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
 import UUID
-import Locations
 import Config
 import Content
-import Utility
-import Utility.Conditional
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
-import Messages
 import Utility.RsyncFile
-import Utility.SafeCommand
-import Utility.Path
 
 type RsyncUrl = String
 
@@ -106,12 +92,12 @@ rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	rsyncSend o k (gitAnnexLocation g k)
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let f = gitAnnexLocation g k
 	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 	rsyncSend o enck tmp
@@ -166,7 +152,7 @@ partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"
  - up trees for rsync. -}
 withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	pid <- liftIO getProcessID
 	let tmp = gitAnnexTmpDir g  "rsynctmp"  show pid
 	nuke tmp
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index cafa4f15a8..b2ea4b0c86 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -7,31 +7,21 @@
 
 module Remote.S3 (remote) where
 
-import Control.Exception.Extensible (IOException)
 import Network.AWS.AWSConnection
 import Network.AWS.S3Object
 import Network.AWS.S3Bucket hiding (size)
 import Network.AWS.AWSResult
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
-import Data.Maybe
-import Data.List
 import Data.Char
-import Data.String.Utils
-import Control.Monad (when)
-import Control.Monad.State (liftIO)
 import System.Environment
-import System.Posix.Files
 import System.Posix.Env (setEnv)
 
-import Types
+import AnnexCommon
 import Types.Remote
 import Types.Key
 import qualified Git
-import qualified Annex
 import UUID
-import Messages
-import Locations
 import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
@@ -123,7 +113,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k
 	s3Bool res
 
@@ -132,7 +122,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
-		g <- Annex.gitRepo
+		g <- gitRepo
 		let f = gitAnnexLocation g k
 		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 8fb29ec403..732f4d46c0 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,21 +10,13 @@ module Remote.Web (
 	setUrl
 ) where
 
-import Control.Monad.State (liftIO)
-import Control.Exception
-import System.FilePath
-
-import Types
+import AnnexCommon
 import Types.Remote
 import qualified Git
-import qualified Annex
-import Messages
 import UUID
 import Config
 import PresenceLog
 import LocationLog
-import Locations
-import Utility
 import qualified Utility.Url as Url
 
 type URLString = String
@@ -80,7 +72,7 @@ getUrls key = do
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
 setUrl key url status = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	addLog (urlLog key) =<< logNow status url
 
 	-- update location log to indicate that the web has the key, or not
diff --git a/RemoteLog.hs b/RemoteLog.hs
index f9c7997e41..2e43265f5c 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -15,14 +15,11 @@ module RemoteLog (
 	prop_idempotent_configEscape
 ) where
 
-import Data.List
 import qualified Data.Map as M
-import Data.Maybe
 import Data.Char
-import Control.Applicative
 
+import AnnexCommon
 import qualified Branch
-import Types
 import Types.Remote
 import UUID
 
diff --git a/Trust.hs b/Trust.hs
index 0c8836c85a..13f0354bd3 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -13,13 +13,11 @@ module Trust (
 	trustPartition
 ) where
 
-import Control.Monad.State
 import qualified Data.Map as M
-import Data.List
 
+import AnnexCommon
 import Types.TrustLevel
 import qualified Branch
-import Types
 import UUID
 import qualified Annex
 
diff --git a/Types/Key.hs b/Types/Key.hs
index b26bb89896..165f814d4b 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -15,9 +15,10 @@ module Types.Key (
 	prop_idempotent_key_read_show
 ) where
 
-import Utility
 import System.Posix.Types
 
+import Common
+
 {- A Key has a unique name, is associated with a key/value backend,
  - and may contain other optional metadata. -}
 data Key = Key {
diff --git a/UUID.hs b/UUID.hs
index eab6bd4dff..633938be48 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -22,18 +22,12 @@ module UUID (
 	uuidLog
 ) where
 
-import Control.Monad.State
-import Control.Applicative
-import System.Cmd.Utils
-import System.IO
 import qualified Data.Map as M
-import Data.Maybe
 
+import AnnexCommon
 import qualified Git
 import qualified Branch
-import Types
 import Types.UUID
-import qualified Annex
 import qualified Build.SysConfig as SysConfig
 import Config
 
@@ -60,7 +54,7 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
  -}
 getUUID :: Git.Repo -> Annex UUID
 getUUID r = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	let c = cached g
 	let u = getUncachedUUID r
@@ -81,7 +75,7 @@ getUncachedUUID r = Git.configGet r configkey ""
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = do
-	u <- getUUID =<< Annex.gitRepo
+	u <- getUUID =<< gitRepo
 	when ("" == u) $ do
 		uuid <- liftIO genUUID
 		setConfig configkey uuid
diff --git a/Upgrade.hs b/Upgrade.hs
index a724ecce31..666f8d08a7 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -7,7 +7,7 @@
 
 module Upgrade where
 
-import Types
+import AnnexCommon
 import Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index 3aabe07700..f8e6cda56f 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -8,23 +8,15 @@
 module Upgrade.V0 where
 
 import System.IO.Error (try)
-import System.Directory
-import Control.Monad.State (liftIO)
-import Control.Monad (filterM, forM_)
-import System.Posix.Files
-import System.FilePath
 
+import AnnexCommon
 import Content
-import Types
-import Locations
-import qualified Annex
-import Messages
 import qualified Upgrade.V1
 
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v0 to v1"
-	g <- Annex.gitRepo
+	g <- gitRepo
 
 	-- do the reorganisation of the key files
 	let olddir = gitAnnexDir g
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 329f90ed6c..bc50b857c6 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -8,33 +8,19 @@
 module Upgrade.V1 where
 
 import System.IO.Error (try)
-import System.Directory
-import Control.Monad.State (liftIO)
-import Control.Monad (filterM, forM_, unless)
-import Control.Applicative
-import System.Posix.Files
-import System.FilePath
-import Data.String.Utils
 import System.Posix.Types
-import Data.Maybe
 import Data.Char
 
+import AnnexCommon
 import Types.Key
 import Content
-import Types
-import Locations
 import PresenceLog
-import qualified Annex
 import qualified AnnexQueue
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Backend
-import Messages
 import Version
-import Utility
 import Utility.FileMode
-import Utility.SafeCommand
-import Utility.Path
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
@@ -64,7 +50,7 @@ upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
 
-	g <- Annex.gitRepo
+	g <- gitRepo
 	if Git.repoIsLocalBare g
 		then do
 			moveContent
@@ -96,7 +82,7 @@ moveContent = do
 updateSymlinks :: Annex ()
 updateSymlinks = do
 	showAction "updating symlinks"
-	g <- Annex.gitRepo
+	g <- gitRepo
 	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
 	forM_ files fixlink
 	where
@@ -117,7 +103,7 @@ moveLocationLogs = do
 	forM_ logkeys move
 		where
 			oldlocationlogs = do
-				g <- Annex.gitRepo
+				g <- gitRepo
 				let dir = Upgrade.V2.gitStateDir g
 				exists <- liftIO $ doesDirectoryExist dir
 				if exists
@@ -126,7 +112,7 @@ moveLocationLogs = do
 						return $ mapMaybe oldlog2key contents
 					else return []
 			move (l, k) = do
-				g <- Annex.gitRepo
+				g <- gitRepo
 				let dest = logFile2 g k
 				let dir = Upgrade.V2.gitStateDir g
 				let f = dir  l
@@ -220,7 +206,7 @@ lookupFile1 file = do
 
 getKeyFilesPresent1 :: Annex [FilePath]
 getKeyFilesPresent1  = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	getKeyFilesPresent1' $ gitAnnexObjectDir g
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
 getKeyFilesPresent1' dir = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 4e686288fb..922dfff28f 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -7,21 +7,9 @@
 
 module Upgrade.V2 where
 
-import System.Directory
-import System.FilePath
-import Control.Monad.State (unless, when, liftIO)
-import Data.List
-import Data.Maybe
-
-import Types.Key
-import Types
-import qualified Annex
+import AnnexCommon
 import qualified Git
 import qualified Branch
-import Messages
-import Utility
-import Utility.Conditional
-import Utility.SafeCommand
 import LocationLog
 import Content
 
@@ -48,7 +36,7 @@ olddir g
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v2 to v3"
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let bare = Git.repoIsLocalBare g
 
 	Branch.create
@@ -85,7 +73,7 @@ locationLogs repo = liftIO $ do
 
 inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	new <- liftIO (readFile $ olddir g  source)
 	Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
@@ -114,7 +102,7 @@ push = do
 			Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			g <- Annex.gitRepo
+			g <- gitRepo
 			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
diff --git a/Utility.hs b/Utility.hs
index a3d461d286..4e82e63c9d 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -11,7 +11,6 @@ module Utility (
 	readMaybe,
 	viaTmp,
 	withTempFile,
-	dirContains,
 	dirContents,
 	myHomeDir,
 	catchBool,
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 9019357196..5d6855bf01 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.CopyFile (copyFile) where
+module Utility.CopyFile (copyFileExternal) where
 
 import System.Directory (doesFileExist, removeFile)
 
@@ -15,8 +15,8 @@ import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,
  - and because this allows easy access to features like cp --reflink. -}
-copyFile :: FilePath -> FilePath -> IO Bool
-copyFile src dest = do
+copyFileExternal :: FilePath -> FilePath -> IO Bool
+copyFileExternal src dest = do
 	whenM (doesFileExist dest) $
 		removeFile dest
 	boolSystem "cp" [params, File src, File dest]
diff --git a/Version.hs b/Version.hs
index fcf6bc4d1d..304e9f0e18 100644
--- a/Version.hs
+++ b/Version.hs
@@ -7,8 +7,7 @@
 
 module Version where
 
-import Types
-import qualified Annex
+import AnnexCommon
 import qualified Git
 import Config
 
@@ -28,7 +27,7 @@ versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
 getVersion = do
-	g <- Annex.gitRepo
+	g <- gitRepo
 	let v = Git.configGet g versionField ""
 	if not $ null v
 		then return $ Just v
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 1fb928f9da..6147545ab6 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -6,13 +6,11 @@
  -}
 
 import System.Environment
-import Data.List
 
+import AnnexCommon
 import qualified Git
 import CmdLine
 import Command
-import Utility.Conditional
-import Utility.SafeCommand
 import Options
 
 import qualified Command.ConfigList
diff --git a/git-annex.cabal b/git-annex.cabal
index 3f31ee4dcc..5645eb043a 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110928
+Version: 3.20110929
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 8b70e678c9..34f22d06fe 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -6,10 +6,8 @@
  -}
 
 import System.Environment
-import System.FilePath
-import System.Directory
-import Control.Monad (when)
 
+import Common
 import qualified Git.UnionMerge
 import qualified Git
 
diff --git a/test.hs b/test.hs
index f8701db669..654af5713f 100644
--- a/test.hs
+++ b/test.hs
@@ -9,23 +9,19 @@ import Test.HUnit
 import Test.HUnit.Tools
 import Test.QuickCheck
 
-import System.Directory
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
 import IO (bracket_, bracket)
-import Control.Monad (unless, when, filterM)
-import Data.List
 import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import Control.Exception (throw)
-import Data.Maybe
 import qualified Data.Map as M
-import System.Path (recurseDir)
 import System.IO.HVFS (SystemFS(..))
 
-import Utility.SafeCommand
+import Common
 
+import qualified Utility.SafeCommand
 import qualified Annex
 import qualified Backend
 import qualified Git

From 1a96d4ab35ed5c2af95a1598620cbbd13bc295b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 4 Oct 2011 00:19:42 -0400
Subject: [PATCH 2299/8313] use bracket to reset environment

In case the exception is caught higher up, don't leave the environment
dirty.
---
 Branch.hs | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/Branch.hs b/Branch.hs
index 3f1153c098..554f168481 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -65,17 +65,13 @@ withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
 	g <- gitRepo
 	let f = index g
-	reset <- liftIO $ Git.useIndex f
 
-	e <- liftIO $ doesFileExist f
-	unless e $ do
-		unless bootstrapping create
-		liftIO $ createDirectoryIfMissing True $ takeDirectory f
-		liftIO $ unless bootstrapping $ genIndex g
-
-	r <- a
-	liftIO reset
-	return r
+	bracket (Git.useIndex f) id $ do
+		unlessM (liftIO $ doesFileExist f) $ do
+			unless bootstrapping create
+			liftIO $ createDirectoryIfMissing True $ takeDirectory f
+			unless bootstrapping $ liftIO $ genIndex g
+		a
 
 withIndexUpdate :: Annex a -> Annex a
 withIndexUpdate a = update >> withIndex a
@@ -332,11 +328,14 @@ lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	g <- gitRepo
 	let file = gitAnnexJournalLock g
-	liftIOOp (Control.Exception.bracket (lock file) unlock) run
+	bracket (lock file) unlock a
 	where
 		lock file = do
 			l <- createFile file stdFileMode
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
-		run _ = a
+
+bracket :: IO c -> (c -> IO b) -> Annex a -> Annex a
+bracket start cleanup go =
+	liftIOOp (Control.Exception.bracket start cleanup) (const go)

From ff21fd4a652cc6516d0e06ab885adf1c93eddced Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 4 Oct 2011 00:34:04 -0400
Subject: [PATCH 2300/8313] factor out Annex exception handling module

---
 Annex/Exception.hs | 27 +++++++++++++++++++++++++++
 Branch.hs          | 15 +++++----------
 Command/Add.hs     |  6 ++----
 3 files changed, 34 insertions(+), 14 deletions(-)
 create mode 100644 Annex/Exception.hs

diff --git a/Annex/Exception.hs b/Annex/Exception.hs
new file mode 100644
index 0000000000..549ef4fd50
--- /dev/null
+++ b/Annex/Exception.hs
@@ -0,0 +1,27 @@
+{- exception handling in the git-annex monad
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.Exception (
+	bracketIO,
+	handle,
+	throw,
+) where
+
+import Control.Exception.Control (handle)
+import Control.Monad.IO.Control (liftIOOp)
+import Control.Exception hiding (handle, throw)
+
+import AnnexCommon
+
+{- Runs an Annex action, with setup and cleanup both in the IO monad. -}
+bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
+bracketIO setup cleanup go =
+	liftIOOp (Control.Exception.bracket setup cleanup) (const go)
+
+{- Throws an exception in the Annex monad. -}
+throw :: Control.Exception.Exception e => e -> Annex a
+throw = liftIO . throwIO
diff --git a/Branch.hs b/Branch.hs
index 554f168481..a2ddc70ac7 100644
--- a/Branch.hs
+++ b/Branch.hs
@@ -21,10 +21,9 @@ module Branch (
 import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
-import Control.Monad.IO.Control (liftIOOp)
-import qualified Control.Exception
 
 import AnnexCommon
+import Annex.Exception
 import Types.BranchState
 import qualified Git
 import qualified Git.UnionMerge
@@ -66,7 +65,7 @@ withIndex' bootstrapping a = do
 	g <- gitRepo
 	let f = index g
 
-	bracket (Git.useIndex f) id $ do
+	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
@@ -93,9 +92,9 @@ invalidateCache = do
 	setState state { cachedFile = Nothing, cachedContent = "" }
 
 getCache :: FilePath -> Annex (Maybe String)
-getCache file = getState >>= handle
+getCache file = getState >>= go
 	where
-		handle state
+		go state
 			| cachedFile state == Just file =
 				return $ Just $ cachedContent state
 			| otherwise = return Nothing
@@ -328,14 +327,10 @@ lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	g <- gitRepo
 	let file = gitAnnexJournalLock g
-	bracket (lock file) unlock a
+	bracketIO (lock file) unlock a
 	where
 		lock file = do
 			l <- createFile file stdFileMode
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
-
-bracket :: IO c -> (c -> IO b) -> Annex a -> Annex a
-bracket start cleanup go =
-	liftIOOp (Control.Exception.bracket start cleanup) (const go)
diff --git a/Command/Add.hs b/Command/Add.hs
index c66c381311..299b5f36e0 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,10 +7,8 @@
 
 module Command.Add where
 
-import Control.Exception.Control (handle)
-import Control.Exception.Base (throwIO)
-
 import AnnexCommon
+import Annex.Exception
 import Command
 import qualified Annex
 import qualified AnnexQueue
@@ -58,7 +56,7 @@ undo file key e = do
 	logStatus key InfoMissing
 	rethrow
 	where
-		rethrow = liftIO $ throwIO e
+		rethrow = throw e
 
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()

From cfe21e85e7fba61ac588e210f2a9b75f8d081f42 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 4 Oct 2011 00:40:47 -0400
Subject: [PATCH 2301/8313] rename

---
 Branch.hs => Annex/Branch.hs      |  6 +++---
 CatFile.hs => Annex/CatFile.hs    |  4 ++--
 AnnexCommon.hs => Annex/Common.hs |  2 +-
 Content.hs => Annex/Content.hs    | 12 ++++++------
 Annex/Exception.hs                |  2 +-
 AnnexQueue.hs => Annex/Queue.hs   |  4 ++--
 Version.hs => Annex/Version.hs    |  4 ++--
 Backend.hs                        |  2 +-
 Backend/SHA.hs                    |  4 ++--
 Backend/URL.hs                    |  2 +-
 Backend/WORM.hs                   |  2 +-
 CmdLine.hs                        |  8 ++++----
 Command.hs                        |  2 +-
 Command/Add.hs                    | 10 +++++-----
 Command/AddUrl.hs                 |  4 ++--
 Command/ConfigList.hs             |  2 +-
 Command/Describe.hs               |  2 +-
 Command/Drop.hs                   |  4 ++--
 Command/DropKey.hs                |  4 ++--
 Command/DropUnused.hs             |  2 +-
 Command/Find.hs                   |  4 ++--
 Command/Fix.hs                    |  8 ++++----
 Command/FromKey.hs                |  8 ++++----
 Command/Fsck.hs                   |  4 ++--
 Command/Get.hs                    |  4 ++--
 Command/InAnnex.hs                |  4 ++--
 Command/Init.hs                   |  2 +-
 Command/InitRemote.hs             |  2 +-
 Command/Lock.hs                   |  6 +++---
 Command/Map.hs                    |  2 +-
 Command/Merge.hs                  |  6 +++---
 Command/Migrate.hs                |  4 ++--
 Command/Move.hs                   |  4 ++--
 Command/RecvKey.hs                |  4 ++--
 Command/Semitrust.hs              |  2 +-
 Command/SendKey.hs                |  4 ++--
 Command/SetKey.hs                 |  4 ++--
 Command/Status.hs                 |  4 ++--
 Command/Trust.hs                  |  2 +-
 Command/Unannex.hs                |  8 ++++----
 Command/Uninit.hs                 |  8 ++++----
 Command/Unlock.hs                 |  4 ++--
 Command/Untrust.hs                |  2 +-
 Command/Unused.hs                 | 10 +++++-----
 Command/Upgrade.hs                |  4 ++--
 Command/Version.hs                |  4 ++--
 Command/Whereis.hs                |  2 +-
 Config.hs                         |  2 +-
 Crypto.hs                         |  2 +-
 GitAnnex.hs                       |  2 +-
 Init.hs                           | 10 +++++-----
 Limit.hs                          |  4 ++--
 LocationLog.hs                    |  6 +++---
 Options.hs                        |  2 +-
 PresenceLog.hs                    |  8 ++++----
 Remote.hs                         |  2 +-
 Remote/Bup.hs                     |  2 +-
 Remote/Directory.hs               |  2 +-
 Remote/Git.hs                     | 10 +++++-----
 Remote/Helper/Encryptable.hs      |  2 +-
 Remote/Helper/Special.hs          |  2 +-
 Remote/Hook.hs                    |  4 ++--
 Remote/Rsync.hs                   |  4 ++--
 Remote/S3real.hs                  |  4 ++--
 Remote/Web.hs                     |  2 +-
 RemoteLog.hs                      |  8 ++++----
 Trust.hs                          |  8 ++++----
 UUID.hs                           |  8 ++++----
 Upgrade.hs                        |  4 ++--
 Upgrade/V0.hs                     |  4 ++--
 Upgrade/V1.hs                     | 18 +++++++++---------
 Upgrade/V2.hs                     | 22 +++++++++++-----------
 git-annex-shell.hs                |  2 +-
 73 files changed, 173 insertions(+), 173 deletions(-)
 rename Branch.hs => Annex/Branch.hs (99%)
 rename CatFile.hs => Annex/CatFile.hs (93%)
 rename AnnexCommon.hs => Annex/Common.hs (88%)
 rename Content.hs => Annex/Content.hs (97%)
 rename AnnexQueue.hs => Annex/Queue.hs (95%)
 rename Version.hs => Annex/Version.hs (95%)

diff --git a/Branch.hs b/Annex/Branch.hs
similarity index 99%
rename from Branch.hs
rename to Annex/Branch.hs
index a2ddc70ac7..c6db9decaf 100644
--- a/Branch.hs
+++ b/Annex/Branch.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Branch (
+module Annex.Branch (
 	create,
 	update,
 	get,
@@ -22,13 +22,13 @@ import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import AnnexCommon
+import Annex.Common
 import Annex.Exception
 import Types.BranchState
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
-import CatFile
+import Annex.CatFile
 
 type GitRef = String
 
diff --git a/CatFile.hs b/Annex/CatFile.hs
similarity index 93%
rename from CatFile.hs
rename to Annex/CatFile.hs
index 8762109e78..4f98815f8f 100644
--- a/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -5,11 +5,11 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module CatFile (
+module Annex.CatFile (
 	catFile
 ) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git.CatFile
 import qualified Annex
 
diff --git a/AnnexCommon.hs b/Annex/Common.hs
similarity index 88%
rename from AnnexCommon.hs
rename to Annex/Common.hs
index bcdc5e264e..ca7b1bff7e 100644
--- a/AnnexCommon.hs
+++ b/Annex/Common.hs
@@ -1,4 +1,4 @@
-module AnnexCommon (
+module Annex.Common (
 	module Common,
 	module Types,
 	module Annex,
diff --git a/Content.hs b/Annex/Content.hs
similarity index 97%
rename from Content.hs
rename to Annex/Content.hs
index 567e4caa5d..a3fa79da85 100644
--- a/Content.hs
+++ b/Annex/Content.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Content (
+module Annex.Content (
 	inAnnex,
 	calcGitLink,
 	logStatus,
@@ -21,13 +21,13 @@ module Content (
 	saveState
 ) where
 
-import AnnexCommon
+import Annex.Common
 import LocationLog
 import UUID
 import qualified Git
 import qualified Annex
-import qualified AnnexQueue
-import qualified Branch
+import qualified Annex.Queue
+import qualified Annex.Branch
 import Utility.StatFS
 import Utility.FileMode
 import Types.Key
@@ -233,5 +233,5 @@ getKeysPresent' dir = do
 {- Things to do to record changes to content. -}
 saveState :: Annex ()
 saveState = do
-	AnnexQueue.flush False
-	Branch.commit "update"
+	Annex.Queue.flush False
+	Annex.Branch.commit "update"
diff --git a/Annex/Exception.hs b/Annex/Exception.hs
index 549ef4fd50..7ea8fb89a2 100644
--- a/Annex/Exception.hs
+++ b/Annex/Exception.hs
@@ -15,7 +15,7 @@ import Control.Exception.Control (handle)
 import Control.Monad.IO.Control (liftIOOp)
 import Control.Exception hiding (handle, throw)
 
-import AnnexCommon
+import Annex.Common
 
 {- Runs an Annex action, with setup and cleanup both in the IO monad. -}
 bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
diff --git a/AnnexQueue.hs b/Annex/Queue.hs
similarity index 95%
rename from AnnexQueue.hs
rename to Annex/Queue.hs
index 66843a75e3..8d0a32bec9 100644
--- a/AnnexQueue.hs
+++ b/Annex/Queue.hs
@@ -5,13 +5,13 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module AnnexQueue (
+module Annex.Queue (
 	add,
 	flush,
 	flushWhenFull
 ) where
 
-import AnnexCommon
+import Annex.Common
 import Annex
 import qualified Git.Queue
 
diff --git a/Version.hs b/Annex/Version.hs
similarity index 95%
rename from Version.hs
rename to Annex/Version.hs
index 304e9f0e18..e501dbf2ec 100644
--- a/Version.hs
+++ b/Annex/Version.hs
@@ -5,9 +5,9 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Version where
+module Annex.Version where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import Config
 
diff --git a/Backend.hs b/Backend.hs
index 94fe29607e..9a7df692cd 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,7 +19,7 @@ module Backend (
 import System.IO.Error (try)
 import System.Posix.Files
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import qualified Annex
 import Types.Key
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 0c36ef0dc7..2be02c9f62 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -7,9 +7,9 @@
 
 module Backend.SHA (backends) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
-import Content
+import Annex.Content
 import Types.Backend
 import Types.Key
 import qualified Build.SysConfig as SysConfig
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 0745de455d..555e0617cf 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,7 +10,7 @@ module Backend.URL (
 	fromUrl
 ) where
 
-import AnnexCommon
+import Annex.Common
 import Types.Backend
 import Types.Key
 
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 80c652558e..b45ec7b0c9 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -7,7 +7,7 @@
 
 module Backend.WORM (backends) where
 
-import AnnexCommon
+import Annex.Common
 import Types.Backend
 import Types.Key
 
diff --git a/CmdLine.hs b/CmdLine.hs
index 34adb25569..faf5222a2a 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -14,11 +14,11 @@ module CmdLine (
 import System.IO.Error (try)
 import System.Console.GetOpt
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
-import qualified AnnexQueue
+import qualified Annex.Queue
 import qualified Git
-import Content
+import Annex.Content
 import Command
 import Options
 import Init
@@ -81,7 +81,7 @@ tryRun = tryRun' 0
 tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
 tryRun' errnum state (a:as) = do
 	result <- try $ Annex.run state $ do
-		AnnexQueue.flushWhenFull
+		Annex.Queue.flushWhenFull
 		a
 	case result of
 		Left err -> do
diff --git a/Command.hs b/Command.hs
index 20f3d79b62..1f418b870b 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,7 +7,7 @@
 
 module Command where
 
-import AnnexCommon
+import Annex.Common
 import qualified Backend
 import qualified Annex
 import qualified Git
diff --git a/Command/Add.hs b/Command/Add.hs
index 299b5f36e0..70b38e809a 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,14 +7,14 @@
 
 module Command.Add where
 
-import AnnexCommon
+import Annex.Common
 import Annex.Exception
 import Command
 import qualified Annex
-import qualified AnnexQueue
+import qualified Annex.Queue
 import qualified Backend
 import LocationLog
-import Content
+import Annex.Content
 import Utility.Touch
 import Backend
 
@@ -81,6 +81,6 @@ cleanup file key hascontent = do
 
 	force <- Annex.getState Annex.force
 	if force
-		then AnnexQueue.add "add" [Param "-f", Param "--"] [file]
-		else AnnexQueue.add "add" [Param "--"] [file]
+		then Annex.Queue.add "add" [Param "-f", Param "--"] [file]
+		else Annex.Queue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index ce6e70699c..35f85ca26f 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -9,7 +9,7 @@ module Command.AddUrl where
 
 import Network.URI
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
@@ -17,7 +17,7 @@ import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
-import Content
+import Annex.Content
 import PresenceLog
 
 command :: [Command]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index c38539fa0b..443656f174 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -7,7 +7,7 @@
 
 module Command.ConfigList where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import UUID
 
diff --git a/Command/Describe.hs b/Command/Describe.hs
index b1c144872b..48f74dcd87 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,7 +7,7 @@
 
 module Command.Describe where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 7210184f88..45feab2f32 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,12 +7,12 @@
 
 module Command.Drop where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import qualified Annex
 import LocationLog
-import Content
+import Annex.Content
 import Trust
 import Config
 
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 7ead1c4bca..185041ad09 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -7,11 +7,11 @@
 
 module Command.DropKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Annex
 import LocationLog
-import Content
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "dropkey" (paramRepeating paramKey) seek
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index ed4f71e7e4..c4d9b765e3 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -9,7 +9,7 @@ module Command.DropUnused where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Annex
 import qualified Command.Drop
diff --git a/Command/Find.hs b/Command/Find.hs
index 8d80659d0d..b8c9eeec27 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,9 +7,9 @@
 
 module Command.Find where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Limit
 
 command :: [Command]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index a66a1c44a7..44e36009b6 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -7,10 +7,10 @@
 
 module Command.Fix where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified AnnexQueue
-import Content
+import qualified Annex.Queue
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "fix" paramPaths seek
@@ -39,5 +39,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] [file]
+	Annex.Queue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index e60025bf7e..f4dceb3315 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -7,10 +7,10 @@
 
 module Command.FromKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified AnnexQueue
-import Content
+import qualified Annex.Queue
+import Annex.Content
 import Types.Key
 
 command :: [Command]
@@ -39,5 +39,5 @@ perform file = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	AnnexQueue.add "add" [Param "--"] [file]
+	Annex.Queue.add "add" [Param "--"] [file]
 	return True
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 33a8405a69..0a75003207 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,13 +7,13 @@
 
 module Command.Fsck where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
 import UUID
-import Content
+import Annex.Content
 import LocationLog
 import Trust
 import Utility.DataUnits
diff --git a/Command/Get.hs b/Command/Get.hs
index 34f56aa2d5..c9fdf56530 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -7,11 +7,11 @@
 
 module Command.Get where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Annex
 import qualified Remote
-import Content
+import Annex.Content
 import qualified Command.Move
 
 command :: [Command]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 36b6d40e6e..05544366ba 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -7,9 +7,9 @@
 
 module Command.InAnnex where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "inannex" (paramRepeating paramKey) seek
diff --git a/Command/Init.hs b/Command/Init.hs
index f3d8834ba1..1a306ae963 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import UUID
 import Init
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 2ce86e9c6b..5080b7b2b8 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -9,7 +9,7 @@ module Command.InitRemote where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import qualified RemoteLog
diff --git a/Command/Lock.hs b/Command/Lock.hs
index af7b92ad68..9acc5fe4aa 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -7,9 +7,9 @@
 
 module Command.Lock where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified AnnexQueue
+import qualified Annex.Queue
 import Backend
 	
 command :: [Command]
@@ -30,5 +30,5 @@ perform file = do
 	-- Checkout from HEAD to get rid of any changes that might be 
 	-- staged in the index, and get back to the previous symlink to
 	-- the content.
-	AnnexQueue.add "checkout" [Param "HEAD", Param "--"] [file]
+	Annex.Queue.add "checkout" [Param "HEAD", Param "--"] [file]
 	next $ return True -- no cleanup needed
diff --git a/Command/Map.hs b/Command/Map.hs
index 8e63f6dd6a..39737289ca 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -10,7 +10,7 @@ module Command.Map where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Git
 import UUID
diff --git a/Command/Merge.hs b/Command/Merge.hs
index b365e0e0cd..1de1eb6eeb 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -7,9 +7,9 @@
 
 module Command.Merge where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import qualified Branch
+import qualified Annex.Branch
 
 command :: [Command]
 command = [repoCommand "merge" paramNothing seek
@@ -25,5 +25,5 @@ start = do
 
 perform :: CommandPerform
 perform = do
-	Branch.update
+	Annex.Branch.update
 	next $ return True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 24f23baf5c..a9591a81a9 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -7,11 +7,11 @@
 
 module Command.Migrate where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Backend
 import qualified Types.Key
-import Content
+import Annex.Content
 import qualified Command.Add
 import Backend
 
diff --git a/Command/Move.hs b/Command/Move.hs
index d2870b1e42..06d58d602e 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,12 +7,12 @@
 
 module Command.Move where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Command.Drop
 import qualified Annex
 import LocationLog
-import Content
+import Annex.Content
 import qualified Remote
 import UUID
 
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 400e81102b..babe04cd01 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,10 +7,10 @@
 
 module Command.RecvKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import CmdLine
-import Content
+import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 236ba28794..31b44bc59d 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Semitrust where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index f397d9ae6a..f3db996637 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,9 +7,9 @@
 
 module Command.SendKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 12ef5b74a5..dde1ec12ad 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -7,10 +7,10 @@
 
 module Command.SetKey where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import LocationLog
-import Content
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "setkey" paramPath seek
diff --git a/Command/Status.hs b/Command/Status.hs
index de49f84d5a..edb74166d0 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
 
-import AnnexCommon
+import Annex.Common
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
@@ -20,7 +20,7 @@ import qualified Command.Unused
 import qualified Git
 import Command
 import Utility.DataUnits
-import Content
+import Annex.Content
 import Types.Key
 import Backend
 import UUID
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 04c68a5d32..3c3473e219 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,7 +7,7 @@
 
 module Command.Trust where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import Trust
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index c5c5e90a67..9413cb88c2 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -7,14 +7,14 @@
 
 module Command.Unannex where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Command.Drop
 import qualified Annex
-import qualified AnnexQueue
+import qualified Annex.Queue
 import Utility.FileMode
 import LocationLog
-import Content
+import Annex.Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 
@@ -71,6 +71,6 @@ cleanup file key = do
 	-- Commit staged changes at end to avoid confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
-	AnnexQueue.add "commit" [Param "-m", Param "content removed from git annex"] []
+	Annex.Queue.add "commit" [Param "-m", Param "content removed from git annex"] []
 	
 	return True
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 3ba7a7cf33..c5afe30f23 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,14 +7,14 @@
 
 module Command.Uninit where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Git
 import qualified Annex
 import qualified Command.Unannex
 import Init
-import qualified Branch
-import Content
+import qualified Annex.Branch
+import Annex.Content
 
 command :: [Command]
 command = [repoCommand "uninit" paramPaths seek 
@@ -46,5 +46,5 @@ cleanup = do
 	-- avoid normal shutdown
 	saveState
 	liftIO $ do
-		Git.run g "branch" [Param "-D", Param Branch.name]
+		Git.run g "branch" [Param "-D", Param Annex.Branch.name]
 		exitSuccess
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 220d57829b..b2e4aa09a5 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,9 +7,9 @@
 
 module Command.Unlock where
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Utility.CopyFile
 import Utility.FileMode
 
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 30ade85ce0..b8a107d901 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Untrust where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1ba4f53019..6681fca3c7 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -12,9 +12,9 @@ module Command.Unused where
 import qualified Data.Set as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import AnnexCommon
+import Annex.Common
 import Command
-import Content
+import Annex.Content
 import Utility.FileMode
 import LocationLog
 import qualified Annex
@@ -23,8 +23,8 @@ import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
 import qualified Backend
 import qualified Remote
-import qualified Branch
-import CatFile
+import qualified Annex.Branch
+import Annex.CatFile
 
 command :: [Command]
 command = [repoCommand "unused" paramNothing seek
@@ -165,7 +165,7 @@ excludeReferenced l = do
 			filter ourbranches .
 			map words . lines . L.unpack
 		cmpheads a b = head a == head b
-		ourbranchend = '/' : Branch.name
+		ourbranchend = '/' : Annex.Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index d79f895d8d..7b6b127a5e 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -7,10 +7,10 @@
 
 module Command.Upgrade where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import Upgrade
-import Version
+import Annex.Version
 
 command :: [Command]
 command = [standaloneCommand "upgrade" paramNothing seek
diff --git a/Command/Version.hs b/Command/Version.hs
index 1e44fbb0b1..bc895b1945 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -7,10 +7,10 @@
 
 module Command.Version where
 
-import AnnexCommon
+import Annex.Common
 import Command
 import qualified Build.SysConfig as SysConfig
-import Version
+import Annex.Version
 
 command :: [Command]
 command = [standaloneCommand "version" paramNothing seek "show version info"]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 3fb636c04c..fec41f410c 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,7 +7,7 @@
 
 module Command.Whereis where
 
-import AnnexCommon
+import Annex.Common
 import LocationLog
 import Command
 import Remote
diff --git a/Config.hs b/Config.hs
index c0328794e1..80637c3933 100644
--- a/Config.hs
+++ b/Config.hs
@@ -7,7 +7,7 @@
 
 module Config where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import qualified Annex
 
diff --git a/Crypto.hs b/Crypto.hs
index 4cc16b4242..af0a216d7d 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -37,7 +37,7 @@ import Control.Exception (finally)
 import System.Exit
 import System.Environment
 
-import AnnexCommon
+import Annex.Common
 import Types.Key
 import Types.Remote
 import Utility.Base64
diff --git a/GitAnnex.hs b/GitAnnex.hs
index fcfbf44b4b..9814880792 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -9,7 +9,7 @@ module GitAnnex where
 
 import System.Console.GetOpt
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import CmdLine
 import Command
diff --git a/Init.hs b/Init.hs
index 57eaf39d20..fa4598966b 100644
--- a/Init.hs
+++ b/Init.hs
@@ -11,16 +11,16 @@ module Init (
 	uninitialize
 ) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
-import Version
+import qualified Annex.Branch
+import Annex.Version
 import UUID
 
 initialize :: Annex ()
 initialize = do
 	prepUUID
-	Branch.create
+	Annex.Branch.create
 	setVersion
 	gitPreCommitHookWrite
 
@@ -35,7 +35,7 @@ ensureInitialized :: Annex ()
 ensureInitialized = getVersion >>= maybe needsinit checkVersion
 	where
 		needsinit = do
-			annexed <- Branch.hasSomeBranch
+			annexed <- Annex.Branch.hasSomeBranch
 			if annexed
 				then initialize
 				else error "First run: git-annex init"
diff --git a/Limit.hs b/Limit.hs
index 334ae325d6..91857c687d 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -10,13 +10,13 @@ module Limit where
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
 import LocationLog
-import Content
+import Annex.Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
 
diff --git a/LocationLog.hs b/LocationLog.hs
index 759bee830e..a633f26374 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -21,9 +21,9 @@ module LocationLog (
 	logFileKey
 ) where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
+import qualified Annex.Branch
 import UUID
 import PresenceLog
 
@@ -43,7 +43,7 @@ keyLocations = currentLog . logFile
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
 loggedKeys :: Annex [Key]
-loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Branch.files
+loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Annex.Branch.files
 
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
diff --git a/Options.hs b/Options.hs
index 9d60292002..9f573ef5dc 100644
--- a/Options.hs
+++ b/Options.hs
@@ -10,7 +10,7 @@ module Options where
 import System.Console.GetOpt
 import System.Log.Logger
 
-import AnnexCommon
+import Annex.Common
 import qualified Annex
 import Command
 import Limit
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 23c2882571..01c1ad0943 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -27,8 +27,8 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as M
 
-import AnnexCommon
-import qualified Branch
+import Annex.Common
+import qualified Annex.Branch
 
 data LogLine = LogLine {
 	date :: POSIXTime,
@@ -72,13 +72,13 @@ instance Read LogLine where
 			ret v = [(v, "")]
 
 addLog :: FilePath -> LogLine -> Annex ()
-addLog file line = Branch.change file $ \s -> 
+addLog file line = Annex.Branch.change file $ \s -> 
 	showLog $ compactLog (line : parseLog s)
 
 {- Reads a log file.
  - Note that the LogLines returned may be in any order. -}
 readLog :: FilePath -> Annex [LogLine]
-readLog file = parseLog <$> Branch.get file
+readLog file = parseLog <$> Annex.Branch.get file
 
 parseLog :: String -> [LogLine]
 parseLog = filter parsable . map read . lines
diff --git a/Remote.hs b/Remote.hs
index 2371b7bf2d..86fda270e1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -32,7 +32,7 @@ import qualified Data.Map as M
 import Text.JSON
 import Text.JSON.Generic
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import UUID
 import qualified Annex
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 9588310194..2d0ba47423 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -12,7 +12,7 @@ import System.IO.Error
 import qualified Data.Map as M
 import System.Process
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 664f8ca5fe..bfea850e52 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -11,7 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
diff --git a/Remote/Git.hs b/Remote/Git.hs
index a457c5905d..15e8991f54 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -10,7 +10,7 @@ module Remote.Git (remote) where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
@@ -18,7 +18,7 @@ import Types.Remote
 import qualified Git
 import qualified Annex
 import UUID
-import qualified Content
+import qualified Annex.Content
 import qualified Utility.Url as Url
 import Config
 import Init
@@ -121,7 +121,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = onLocal r (Content.inAnnex key)
+		checklocal = onLocal r (Annex.Content.inAnnex key)
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -164,9 +164,9 @@ copyToRemote r key
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
-			ok <- Content.getViaTmp key $
+			ok <- Annex.Content.getViaTmp key $
 				rsyncOrCopyFile r keysrc
-			Content.saveState
+			Annex.Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
 		g <- gitRepo
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 42503e4d41..6f43b4367c 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import Crypto
 import qualified Annex
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 905db04c55..c4e0b7f07f 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Special where
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 3bbda19240..a18bc51e66 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -12,12 +12,12 @@ import qualified Data.Map as M
 import System.IO.Error (try)
 import System.Exit
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
 import Config
-import Content
+import Annex.Content
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 6a1c297c5d..9b870f2b10 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -10,12 +10,12 @@ module Remote.Rsync (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
 import Config
-import Content
+import Annex.Content
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b2ea4b0c86..c846390b8e 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -17,7 +17,7 @@ import Data.Char
 import System.Environment
 import System.Posix.Env (setEnv)
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import Types.Key
 import qualified Git
@@ -26,7 +26,7 @@ import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
-import Content
+import Annex.Content
 import Utility.Base64
 
 remote :: RemoteType Annex
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 732f4d46c0..9132967c7e 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,7 +10,7 @@ module Remote.Web (
 	setUrl
 ) where
 
-import AnnexCommon
+import Annex.Common
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 2e43265f5c..4ffe712050 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -18,8 +18,8 @@ module RemoteLog (
 import qualified Data.Map as M
 import Data.Char
 
-import AnnexCommon
-import qualified Branch
+import Annex.Common
+import qualified Annex.Branch
 import Types.Remote
 import UUID
 
@@ -29,7 +29,7 @@ remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = Branch.change remoteLog $
+configSet u c = Annex.Branch.change remoteLog $
 	serialize . M.insert u c . remoteLogParse
 	where
 		serialize = unlines . sort . map toline . M.toList
@@ -37,7 +37,7 @@ configSet u c = Branch.change remoteLog $
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = remoteLogParse <$> Branch.get remoteLog
+readRemoteLog = remoteLogParse <$> Annex.Branch.get remoteLog
 
 remoteLogParse :: String -> M.Map UUID RemoteConfig
 remoteLogParse s =
diff --git a/Trust.hs b/Trust.hs
index 13f0354bd3..5f76003813 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -15,9 +15,9 @@ module Trust (
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import Types.TrustLevel
-import qualified Branch
+import qualified Annex.Branch
 import UUID
 import qualified Annex
 
@@ -40,7 +40,7 @@ trustMap = do
 		Just m -> return m
 		Nothing -> do
 			overrides <- Annex.getState Annex.forcetrust
-			l <- Branch.get trustLog
+			l <- Annex.Branch.get trustLog
 			let m = M.fromList $ trustMapParse l ++ overrides
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
@@ -62,7 +62,7 @@ trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
 		error "unknown UUID; cannot modify trust level"
-	Branch.change trustLog $
+	Annex.Branch.change trustLog $
 		serialize . M.insert uuid level . M.fromList . trustMapParse
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
         where
diff --git a/UUID.hs b/UUID.hs
index 633938be48..208866ad72 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -24,9 +24,9 @@ module UUID (
 
 import qualified Data.Map as M
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
+import qualified Annex.Branch
 import Types.UUID
 import qualified Build.SysConfig as SysConfig
 import Config
@@ -82,14 +82,14 @@ prepUUID = do
 
 {- Records a description for a uuid in the uuidLog. -}
 describeUUID :: UUID -> String -> Annex ()
-describeUUID uuid desc = Branch.change uuidLog $
+describeUUID uuid desc = Annex.Branch.change uuidLog $
 	serialize . M.insert uuid desc . parse
 	where
 		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
 
 {- Read the uuidLog into a Map -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = parse <$> Branch.get uuidLog
+uuidMap = parse <$> Annex.Branch.get uuidLog
 
 parse :: String -> M.Map UUID String
 parse = M.fromList . map pair . lines
diff --git a/Upgrade.hs b/Upgrade.hs
index 666f8d08a7..6f1da773dc 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -7,8 +7,8 @@
 
 module Upgrade where
 
-import AnnexCommon
-import Version
+import Annex.Common
+import Annex.Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
 import qualified Upgrade.V2
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index f8e6cda56f..af91741a0c 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -9,8 +9,8 @@ module Upgrade.V0 where
 
 import System.IO.Error (try)
 
-import AnnexCommon
-import Content
+import Annex.Common
+import Annex.Content
 import qualified Upgrade.V1
 
 upgrade :: Annex Bool
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index bc50b857c6..f4e44acdcb 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,15 +11,15 @@ import System.IO.Error (try)
 import System.Posix.Types
 import Data.Char
 
-import AnnexCommon
+import Annex.Common
 import Types.Key
-import Content
+import Annex.Content
 import PresenceLog
-import qualified AnnexQueue
+import qualified Annex.Queue
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Backend
-import Version
+import Annex.Version
 import Utility.FileMode
 import qualified Upgrade.V2
 
@@ -60,7 +60,7 @@ upgrade = do
 			updateSymlinks
 			moveLocationLogs
 	
-			AnnexQueue.flush True
+			Annex.Queue.flush True
 			setVersion
 	
 	Upgrade.V2.upgrade
@@ -94,7 +94,7 @@ updateSymlinks = do
 					link <- calcGitLink f k
 					liftIO $ removeFile f
 					liftIO $ createSymbolicLink link f
-					AnnexQueue.add "add" [Param "--"] [f]
+					Annex.Queue.add "add" [Param "--"] [f]
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
@@ -124,9 +124,9 @@ moveLocationLogs = do
 				old <- liftIO $ readLog1 f
 				new <- liftIO $ readLog1 dest
 				liftIO $ writeLog1 dest (old++new)
-				AnnexQueue.add "add" [Param "--"] [dest]
-				AnnexQueue.add "add" [Param "--"] [f]
-				AnnexQueue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
+				Annex.Queue.add "add" [Param "--"] [dest]
+				Annex.Queue.add "add" [Param "--"] [f]
+				Annex.Queue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
 		
 oldlog2key :: FilePath -> Maybe (FilePath, Key)
 oldlog2key l = 
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 922dfff28f..8ac26dc525 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -7,11 +7,11 @@
 
 module Upgrade.V2 where
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
-import qualified Branch
+import qualified Annex.Branch
 import LocationLog
-import Content
+import Annex.Content
 
 olddir :: Git.Repo -> FilePath
 olddir g
@@ -39,7 +39,7 @@ upgrade = do
 	g <- gitRepo
 	let bare = Git.repoIsLocalBare g
 
-	Branch.create
+	Annex.Branch.create
 	showProgress
 
 	e <- liftIO $ doesDirectoryExist (olddir g)
@@ -75,7 +75,7 @@ inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
 	g <- gitRepo
 	new <- liftIO (readFile $ olddir g  source)
-	Branch.change dest $ \prev -> 
+	Annex.Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
 	showProgress
 
@@ -85,8 +85,8 @@ logFiles dir = return . filter (".log" `isSuffixOf`)
 
 push :: Annex ()
 push = do
-	origin_master <- Branch.refExists "origin/master"
-	origin_gitannex <- Branch.hasOrigin
+	origin_master <- Annex.Branch.refExists "origin/master"
+	origin_gitannex <- Annex.Branch.hasOrigin
 	case (origin_master, origin_gitannex) of
 		(_, True) -> do
 			-- Merge in the origin's git-annex branch,
@@ -94,20 +94,20 @@ push = do
 			-- will immediately work. Not pushed here,
 			-- because it's less obnoxious to let the user
 			-- push.
-			Branch.update
+			Annex.Branch.update
 		(True, False) -> do
 			-- push git-annex to origin, so that
 			-- "git push" will from then on
 			-- automatically push it
-			Branch.update -- just in case
+			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
 			g <- gitRepo
-			liftIO $ Git.run g "push" [Param "origin", Param Branch.name]
+			liftIO $ Git.run g "push" [Param "origin", Param Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
-			Branch.update
+			Annex.Branch.update
 			showLongNote $
 				"git-annex branch created\n" ++
 				"Be sure to push this branch when pushing to remotes.\n"
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 6147545ab6..d03b0910ea 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -7,7 +7,7 @@
 
 import System.Environment
 
-import AnnexCommon
+import Annex.Common
 import qualified Git
 import CmdLine
 import Command

From c199a01dd13e3c25bebcf6e67b9f870dc9be5ce8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 5 Oct 2011 15:59:49 -0400
Subject: [PATCH 2302/8313] bug

---
 ...rust.log_and_remote.log_merge_wackiness.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn

diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
new file mode 100644
index 0000000000..af140403ff
--- /dev/null
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -0,0 +1,17 @@
+Since uuid.log, trust.log and remote.log are union merged, it's possible
+for any given item in them to have multiple values after a merge.
+This would happen, for example, if the value was changed in different ways
+in two repos which were then merged. git-annex will use an arbitrary
+one of the multiple values.
+
+A workaround if this should happen to you is to use `git annex describe`
+or other commands to re-set the value you want. The process of setting
+the value will remove the multiple lines.
+
+To fix this the file format needs to be changed to include a timestamp
+as is done with the other log files, then git-annex can consistently
+pick the newest value -- which is as close to the "right" value as can be
+determined in this situation.
+
+(For backwards compatability, git-annex should
+treat lines with no timestamp as being timestamped with 0.)

From 6a6ea06cee8ce69f391f7ce78b98b8f26a599e66 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 5 Oct 2011 16:02:51 -0400
Subject: [PATCH 2303/8313] rename

---
 Annex/Branch.hs                    | 2 +-
 Annex/CatFile.hs                   | 2 +-
 Annex/Content.hs                   | 2 +-
 Annex/Exception.hs                 | 2 +-
 Annex/Queue.hs                     | 2 +-
 Annex/Version.hs                   | 2 +-
 Backend.hs                         | 2 +-
 Backend/SHA.hs                     | 2 +-
 Backend/URL.hs                     | 2 +-
 Backend/WORM.hs                    | 2 +-
 CmdLine.hs                         | 2 +-
 Command.hs                         | 2 +-
 Command/Add.hs                     | 2 +-
 Command/AddUrl.hs                  | 2 +-
 Command/ConfigList.hs              | 2 +-
 Command/Describe.hs                | 2 +-
 Command/Drop.hs                    | 2 +-
 Command/DropKey.hs                 | 2 +-
 Command/DropUnused.hs              | 2 +-
 Command/Find.hs                    | 2 +-
 Command/Fix.hs                     | 2 +-
 Command/FromKey.hs                 | 2 +-
 Command/Fsck.hs                    | 2 +-
 Command/Get.hs                     | 2 +-
 Command/InAnnex.hs                 | 2 +-
 Command/Init.hs                    | 2 +-
 Command/InitRemote.hs              | 2 +-
 Command/Lock.hs                    | 2 +-
 Command/Map.hs                     | 2 +-
 Command/Merge.hs                   | 2 +-
 Command/Migrate.hs                 | 2 +-
 Command/Move.hs                    | 2 +-
 Command/RecvKey.hs                 | 2 +-
 Command/Semitrust.hs               | 2 +-
 Command/SendKey.hs                 | 2 +-
 Command/SetKey.hs                  | 2 +-
 Command/Status.hs                  | 2 +-
 Command/Trust.hs                   | 2 +-
 Command/Unannex.hs                 | 2 +-
 Command/Uninit.hs                  | 2 +-
 Command/Unlock.hs                  | 2 +-
 Command/Untrust.hs                 | 2 +-
 Command/Unused.hs                  | 2 +-
 Command/Upgrade.hs                 | 2 +-
 Command/Version.hs                 | 2 +-
 Command/Whereis.hs                 | 2 +-
 Annex/Common.hs => Common/Annex.hs | 2 +-
 Config.hs                          | 2 +-
 Crypto.hs                          | 2 +-
 GitAnnex.hs                        | 2 +-
 Init.hs                            | 2 +-
 Limit.hs                           | 2 +-
 LocationLog.hs                     | 2 +-
 Options.hs                         | 2 +-
 PresenceLog.hs                     | 2 +-
 Remote.hs                          | 2 +-
 Remote/Bup.hs                      | 2 +-
 Remote/Directory.hs                | 2 +-
 Remote/Git.hs                      | 2 +-
 Remote/Helper/Encryptable.hs       | 2 +-
 Remote/Helper/Special.hs           | 2 +-
 Remote/Hook.hs                     | 2 +-
 Remote/Rsync.hs                    | 2 +-
 Remote/S3real.hs                   | 2 +-
 Remote/Web.hs                      | 2 +-
 RemoteLog.hs                       | 2 +-
 Trust.hs                           | 2 +-
 UUID.hs                            | 2 +-
 Upgrade.hs                         | 2 +-
 Upgrade/V0.hs                      | 2 +-
 Upgrade/V1.hs                      | 2 +-
 Upgrade/V2.hs                      | 2 +-
 git-annex-shell.hs                 | 2 +-
 73 files changed, 73 insertions(+), 73 deletions(-)
 rename Annex/Common.hs => Common/Annex.hs (88%)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c6db9decaf..0b4bea051d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -22,7 +22,7 @@ import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Annex.Common
+import Common.Annex
 import Annex.Exception
 import Types.BranchState
 import qualified Git
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 4f98815f8f..2707ed3ea3 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -9,7 +9,7 @@ module Annex.CatFile (
 	catFile
 ) where
 
-import Annex.Common
+import Common.Annex
 import qualified Git.CatFile
 import qualified Annex
 
diff --git a/Annex/Content.hs b/Annex/Content.hs
index a3fa79da85..21403954ae 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -21,7 +21,7 @@ module Annex.Content (
 	saveState
 ) where
 
-import Annex.Common
+import Common.Annex
 import LocationLog
 import UUID
 import qualified Git
diff --git a/Annex/Exception.hs b/Annex/Exception.hs
index 7ea8fb89a2..c147439a1c 100644
--- a/Annex/Exception.hs
+++ b/Annex/Exception.hs
@@ -15,7 +15,7 @@ import Control.Exception.Control (handle)
 import Control.Monad.IO.Control (liftIOOp)
 import Control.Exception hiding (handle, throw)
 
-import Annex.Common
+import Common.Annex
 
 {- Runs an Annex action, with setup and cleanup both in the IO monad. -}
 bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 8d0a32bec9..4c1182750a 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -11,7 +11,7 @@ module Annex.Queue (
 	flushWhenFull
 ) where
 
-import Annex.Common
+import Common.Annex
 import Annex
 import qualified Git.Queue
 
diff --git a/Annex/Version.hs b/Annex/Version.hs
index e501dbf2ec..935f777abf 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -7,7 +7,7 @@
 
 module Annex.Version where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import Config
 
diff --git a/Backend.hs b/Backend.hs
index 9a7df692cd..d1ff114059 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,7 +19,7 @@ module Backend (
 import System.IO.Error (try)
 import System.Posix.Files
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex
 import Types.Key
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 2be02c9f62..4b5b14cc36 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -7,7 +7,7 @@
 
 module Backend.SHA (backends) where
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import Annex.Content
 import Types.Backend
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 555e0617cf..32a72335a5 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,7 +10,7 @@ module Backend.URL (
 	fromUrl
 ) where
 
-import Annex.Common
+import Common.Annex
 import Types.Backend
 import Types.Key
 
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index b45ec7b0c9..5a3e2d694c 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -7,7 +7,7 @@
 
 module Backend.WORM (backends) where
 
-import Annex.Common
+import Common.Annex
 import Types.Backend
 import Types.Key
 
diff --git a/CmdLine.hs b/CmdLine.hs
index faf5222a2a..b1c9c17285 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -14,7 +14,7 @@ module CmdLine (
 import System.IO.Error (try)
 import System.Console.GetOpt
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import qualified Annex.Queue
 import qualified Git
diff --git a/Command.hs b/Command.hs
index 1f418b870b..6f8684e4af 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,7 +7,7 @@
 
 module Command where
 
-import Annex.Common
+import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
diff --git a/Command/Add.hs b/Command/Add.hs
index 70b38e809a..e6e7a7c770 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -7,7 +7,7 @@
 
 module Command.Add where
 
-import Annex.Common
+import Common.Annex
 import Annex.Exception
 import Command
 import qualified Annex
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 35f85ca26f..c5417bf5bc 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -9,7 +9,7 @@ module Command.AddUrl where
 
 import Network.URI
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 443656f174..d52c33f3b9 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -7,7 +7,7 @@
 
 module Command.ConfigList where
 
-import Annex.Common
+import Common.Annex
 import Command
 import UUID
 
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 48f74dcd87..3368046393 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -7,7 +7,7 @@
 
 module Command.Describe where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 45feab2f32..fabacbb724 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -7,7 +7,7 @@
 
 module Command.Drop where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import qualified Annex
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 185041ad09..35ebfc219b 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -7,7 +7,7 @@
 
 module Command.DropKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex
 import LocationLog
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index c4d9b765e3..0050685565 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -9,7 +9,7 @@ module Command.DropUnused where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex
 import qualified Command.Drop
diff --git a/Command/Find.hs b/Command/Find.hs
index b8c9eeec27..98501078e8 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,7 +7,7 @@
 
 module Command.Find where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Limit
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 44e36009b6..5b6f1f7a47 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -7,7 +7,7 @@
 
 module Command.Fix where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Queue
 import Annex.Content
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index f4dceb3315..1b05d71fb8 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -7,7 +7,7 @@
 
 module Command.FromKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Queue
 import Annex.Content
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0a75003207..689271dd7e 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -7,7 +7,7 @@
 
 module Command.Fsck where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import qualified Types.Backend
diff --git a/Command/Get.hs b/Command/Get.hs
index c9fdf56530..acf7e07228 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -7,7 +7,7 @@
 
 module Command.Get where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex
 import qualified Remote
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 05544366ba..773693b65f 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -7,7 +7,7 @@
 
 module Command.InAnnex where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 
diff --git a/Command/Init.hs b/Command/Init.hs
index 1a306ae963..ace06c2c34 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -7,7 +7,7 @@
 
 module Command.Init where
 
-import Annex.Common
+import Common.Annex
 import Command
 import UUID
 import Init
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 5080b7b2b8..918604f852 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -9,7 +9,7 @@ module Command.InitRemote where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import qualified RemoteLog
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 9acc5fe4aa..c6c66a1585 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -7,7 +7,7 @@
 
 module Command.Lock where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Queue
 import Backend
diff --git a/Command/Map.hs b/Command/Map.hs
index 39737289ca..1155c4a6ef 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -10,7 +10,7 @@ module Command.Map where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Git
 import UUID
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 1de1eb6eeb..eef2f3857a 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -7,7 +7,7 @@
 
 module Command.Merge where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Annex.Branch
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a9591a81a9..23ed6fd162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -7,7 +7,7 @@
 
 module Command.Migrate where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Backend
 import qualified Types.Key
diff --git a/Command/Move.hs b/Command/Move.hs
index 06d58d602e..d650c52510 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -7,7 +7,7 @@
 
 module Command.Move where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index babe04cd01..d3b77d8ac1 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -7,7 +7,7 @@
 
 module Command.RecvKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import CmdLine
 import Annex.Content
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 31b44bc59d..53b29c98ac 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Semitrust where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index f3db996637..ad47cd009f 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -7,7 +7,7 @@
 
 module Command.SendKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Utility.RsyncFile
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index dde1ec12ad..28bc9a48d6 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -7,7 +7,7 @@
 
 module Command.SetKey where
 
-import Annex.Common
+import Common.Annex
 import Command
 import LocationLog
 import Annex.Content
diff --git a/Command/Status.hs b/Command/Status.hs
index edb74166d0..37e13f0d8e 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
 
-import Annex.Common
+import Common.Annex
 import qualified Types.Backend as B
 import qualified Types.Remote as R
 import qualified Remote
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 3c3473e219..bc655e3f61 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -7,7 +7,7 @@
 
 module Command.Trust where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import Trust
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 9413cb88c2..d0cef76781 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -7,7 +7,7 @@
 
 module Command.Unannex where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index c5afe30f23..7b3b4fe329 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,7 +7,7 @@
 
 module Command.Uninit where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Git
 import qualified Annex
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index b2e4aa09a5..9b568b5a6b 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -7,7 +7,7 @@
 
 module Command.Unlock where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Utility.CopyFile
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index b8a107d901..bcde0e0a06 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -7,7 +7,7 @@
 
 module Command.Untrust where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Remote
 import UUID
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 6681fca3c7..abf5a5361a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -12,7 +12,7 @@ module Command.Unused where
 import qualified Data.Set as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
-import Annex.Common
+import Common.Annex
 import Command
 import Annex.Content
 import Utility.FileMode
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 7b6b127a5e..90d3a4e95b 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -7,7 +7,7 @@
 
 module Command.Upgrade where
 
-import Annex.Common
+import Common.Annex
 import Command
 import Upgrade
 import Annex.Version
diff --git a/Command/Version.hs b/Command/Version.hs
index bc895b1945..5ac87099bb 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -7,7 +7,7 @@
 
 module Command.Version where
 
-import Annex.Common
+import Common.Annex
 import Command
 import qualified Build.SysConfig as SysConfig
 import Annex.Version
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index fec41f410c..0eeb174142 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,7 +7,7 @@
 
 module Command.Whereis where
 
-import Annex.Common
+import Common.Annex
 import LocationLog
 import Command
 import Remote
diff --git a/Annex/Common.hs b/Common/Annex.hs
similarity index 88%
rename from Annex/Common.hs
rename to Common/Annex.hs
index ca7b1bff7e..43f1ea0af3 100644
--- a/Annex/Common.hs
+++ b/Common/Annex.hs
@@ -1,4 +1,4 @@
-module Annex.Common (
+module Common.Annex (
 	module Common,
 	module Types,
 	module Annex,
diff --git a/Config.hs b/Config.hs
index 80637c3933..f4c3843af8 100644
--- a/Config.hs
+++ b/Config.hs
@@ -7,7 +7,7 @@
 
 module Config where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex
 
diff --git a/Crypto.hs b/Crypto.hs
index af0a216d7d..21e4d75605 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -37,7 +37,7 @@ import Control.Exception (finally)
 import System.Exit
 import System.Environment
 
-import Annex.Common
+import Common.Annex
 import Types.Key
 import Types.Remote
 import Utility.Base64
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 9814880792..3cd7207d3e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -9,7 +9,7 @@ module GitAnnex where
 
 import System.Console.GetOpt
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import CmdLine
 import Command
diff --git a/Init.hs b/Init.hs
index fa4598966b..145413e8da 100644
--- a/Init.hs
+++ b/Init.hs
@@ -11,7 +11,7 @@ module Init (
 	uninitialize
 ) where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
diff --git a/Limit.hs b/Limit.hs
index 91857c687d..3812ceea4b 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -10,7 +10,7 @@ module Limit where
 import Text.Regex.PCRE.Light.Char8
 import System.Path.WildMatch
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
diff --git a/LocationLog.hs b/LocationLog.hs
index a633f26374..5cbdbb28a1 100644
--- a/LocationLog.hs
+++ b/LocationLog.hs
@@ -21,7 +21,7 @@ module LocationLog (
 	logFileKey
 ) where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import UUID
diff --git a/Options.hs b/Options.hs
index 9f573ef5dc..0c7b4d5f41 100644
--- a/Options.hs
+++ b/Options.hs
@@ -10,7 +10,7 @@ module Options where
 import System.Console.GetOpt
 import System.Log.Logger
 
-import Annex.Common
+import Common.Annex
 import qualified Annex
 import Command
 import Limit
diff --git a/PresenceLog.hs b/PresenceLog.hs
index 01c1ad0943..4e4960f0a6 100644
--- a/PresenceLog.hs
+++ b/PresenceLog.hs
@@ -27,7 +27,7 @@ import Data.Time
 import System.Locale
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import qualified Annex.Branch
 
 data LogLine = LogLine {
diff --git a/Remote.hs b/Remote.hs
index 86fda270e1..27ebd724ac 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -32,7 +32,7 @@ import qualified Data.Map as M
 import Text.JSON
 import Text.JSON.Generic
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import UUID
 import qualified Annex
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 2d0ba47423..986be7fc40 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -12,7 +12,7 @@ import System.IO.Error
 import qualified Data.Map as M
 import System.Process
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index bfea850e52..03b17456a4 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -11,7 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 15e8991f54..704bdc04d3 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -10,7 +10,7 @@ module Remote.Git (remote) where
 import Control.Exception.Extensible
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Utility.CopyFile
 import Utility.RsyncFile
 import Utility.Ssh
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 6f43b4367c..004b70408b 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Encryptable where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import Crypto
 import qualified Annex
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c4e0b7f07f..1fbd7b19ed 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -9,7 +9,7 @@ module Remote.Helper.Special where
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index a18bc51e66..cc8ed69ab5 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import System.IO.Error (try)
 import System.Exit
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 9b870f2b10..3474f8b251 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -10,7 +10,7 @@ module Remote.Rsync (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index c846390b8e..a754731281 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -17,7 +17,7 @@ import Data.Char
 import System.Environment
 import System.Posix.Env (setEnv)
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import Types.Key
 import qualified Git
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 9132967c7e..30a1ff0086 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,7 +10,7 @@ module Remote.Web (
 	setUrl
 ) where
 
-import Annex.Common
+import Common.Annex
 import Types.Remote
 import qualified Git
 import UUID
diff --git a/RemoteLog.hs b/RemoteLog.hs
index 4ffe712050..149104fbac 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -18,7 +18,7 @@ module RemoteLog (
 import qualified Data.Map as M
 import Data.Char
 
-import Annex.Common
+import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
 import UUID
diff --git a/Trust.hs b/Trust.hs
index 5f76003813..1920bc6170 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -15,7 +15,7 @@ module Trust (
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import Types.TrustLevel
 import qualified Annex.Branch
 import UUID
diff --git a/UUID.hs b/UUID.hs
index 208866ad72..bd7b6b1d01 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -24,7 +24,7 @@ module UUID (
 
 import qualified Data.Map as M
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Types.UUID
diff --git a/Upgrade.hs b/Upgrade.hs
index 6f1da773dc..8b2e939dde 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -7,7 +7,7 @@
 
 module Upgrade where
 
-import Annex.Common
+import Common.Annex
 import Annex.Version
 import qualified Upgrade.V0
 import qualified Upgrade.V1
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index af91741a0c..b1443fa460 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -9,7 +9,7 @@ module Upgrade.V0 where
 
 import System.IO.Error (try)
 
-import Annex.Common
+import Common.Annex
 import Annex.Content
 import qualified Upgrade.V1
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index f4e44acdcb..132e694ebc 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -11,7 +11,7 @@ import System.IO.Error (try)
 import System.Posix.Types
 import Data.Char
 
-import Annex.Common
+import Common.Annex
 import Types.Key
 import Annex.Content
 import PresenceLog
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 8ac26dc525..39c7b15611 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -7,7 +7,7 @@
 
 module Upgrade.V2 where
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import LocationLog
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index d03b0910ea..2e0d309794 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -7,7 +7,7 @@
 
 import System.Environment
 
-import Annex.Common
+import Common.Annex
 import qualified Git
 import CmdLine
 import Command

From e139a99aa0528af28de8544564e99562d2333d64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 5 Oct 2011 16:35:01 -0400
Subject: [PATCH 2304/8313] investigated file formats; appending timestamp
 should preserve back-compat

---
 ...ust.log_and_remote.log_merge_wackiness.mdwn | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index af140403ff..2724611fbf 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -13,5 +13,19 @@ as is done with the other log files, then git-annex can consistently
 pick the newest value -- which is as close to the "right" value as can be
 determined in this situation.
 
-(For backwards compatability, git-annex should
-treat lines with no timestamp as being timestamped with 0.)
+----
+
+File format backwards-compatability is the issue. Ideally, old git-annex
+would keep working, ignoring the timestamp.
+
+- uuid.log: "uuid description timestamp" would work; old git-annex 
+  would just treat the timestamp as part of the description which would be
+  ok
+- trust.log: "uuid trustlevel timestamp" would work; old git-annex
+  ignores trailing words
+- remote.log: "uuid key=value ... timestamp" is on the edge but does work
+  (old git-annex will include the timestamp in the key/value map it builds,
+  but that should not break anything really)
+
+Appending "timestamp=xxxxx" would be good for clarity, and make
+it easier to parse the timestamp out from lines that have it.

From dab5bddc64ab4ad479a1104748c15d194e138847 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 11:12:03 -0400
Subject: [PATCH 2305/8313] propigate test suite failure

(but not test suite build failure)
---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index ff58362766..73e4260008 100644
--- a/Makefile
+++ b/Makefile
@@ -57,6 +57,7 @@ test: $(bins)
 	else \
 		if ! ./test; then \
 			echo "** test suite failed!" >&2; \
+			exit 1; \
 		fi; \
 	fi
 

From 52fa4096480ba74c355dffcddbda766994f4d5b7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 15:23:26 -0400
Subject: [PATCH 2306/8313] add UUIDLog, a generic module for mergable
 uuid-based logs

---
 UUIDLog.hs | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 UUIDLog.hs

diff --git a/UUIDLog.hs b/UUIDLog.hs
new file mode 100644
index 0000000000..d6eb8fbbbe
--- /dev/null
+++ b/UUIDLog.hs
@@ -0,0 +1,110 @@
+{- git-annex uuid-based logs
+ -
+ - This is used to store information about a UUID in a way that can
+ - be union merged.
+ -
+ - A line of the log will look like: "UUID[ INFO[ timestamp=foo]]"
+ - The timestamp is last for backwards compatability reasons,
+ - and may not be present on old log lines.
+ - 
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module UUIDLog (
+	Log,
+	LogEntry(..),
+	parseLog,
+	showLog,
+	changeLog,
+	addLog,
+	simpleMap,
+
+	prop_TimeStamp_sane,
+	prop_addLog_sane,
+) where
+
+import qualified Data.Map as M
+import Data.Time.Clock.POSIX
+import Data.Time
+import System.Locale
+
+import Common
+import Types.UUID
+
+data TimeStamp = Unknown | Date POSIXTime
+	deriving (Eq, Ord, Show)
+
+data LogEntry a = LogEntry
+	{ changed :: TimeStamp
+	, value :: a
+	} deriving (Eq, Show)
+
+type Log a = M.Map UUID (LogEntry a)
+
+tskey :: String
+tskey = "timestamp="
+
+showLog :: (a -> String) -> Log a -> String
+showLog shower = unlines . map showpair . M.toList
+	where
+		showpair (k, LogEntry (Date p) v) =
+			unwords [k, shower v, tskey ++ show p]
+		showpair (k, LogEntry Unknown v) =
+			unwords [k, shower v]
+
+parseLog :: (String -> Maybe a) -> String -> Log a
+parseLog parser = M.fromListWith best . catMaybes . map pair . lines
+	where
+		pair line
+			| null ws = Nothing
+			| otherwise = case parser $ unwords info of
+				Nothing -> Nothing
+				Just v -> Just (u, LogEntry c v)
+			where
+				ws = words line
+				u = head ws
+				end = last ws
+				c
+					| tskey `isPrefixOf` end =
+						pdate $ tail $ dropWhile (/= '=') end
+					| otherwise = Unknown
+				info
+					| c == Unknown = drop 1 ws
+					| otherwise = drop 1 $ init ws
+				pdate s = case parseTime defaultTimeLocale "%s%Qs" s of
+					Nothing -> Unknown
+					Just d -> Date $ utcTimeToPOSIXSeconds d
+
+changeLog :: POSIXTime -> UUID -> a -> Log a -> Log a
+changeLog t u v = M.insert u $ LogEntry (Date t) v
+
+{- Only add an LogEntry if it's newer (or at least as new as) than any
+ - existing LogEntry for a UUID. -}
+addLog :: UUID -> LogEntry a -> Log a -> Log a
+addLog = M.insertWith best
+
+{- Converts a Log into a simple Map without the timestamp information.
+ - This is a one-way trip, but useful for code that never needs to change
+ - the log. -}
+simpleMap :: Log a -> M.Map UUID a
+simpleMap = M.map value
+
+best :: LogEntry a -> LogEntry a -> LogEntry a
+best new old
+	| changed old > changed new = old
+	| otherwise = new
+
+-- Unknown is oldest.
+prop_TimeStamp_sane :: Bool
+prop_TimeStamp_sane = Unknown < Date 1
+
+prop_addLog_sane :: Bool
+prop_addLog_sane = newWins && newestWins
+	where
+		newWins = addLog "foo" (LogEntry (Date 1) "new") l == l2
+		newestWins = addLog "foo" (LogEntry (Date 1) "newest") l2 /= l2
+
+		l = M.fromList [("foo", LogEntry (Date 0) "old")]
+		l2 = M.fromList [("foo", LogEntry (Date 1) "new")]

From 3e0d2a080333f3566312da0e1982739873603457 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 15:31:25 -0400
Subject: [PATCH 2307/8313] add timestamp to uuid.log

* New or changed repository descriptions in uuid.log now have a timestamp,
  which is used to ensure the newest description is used when the uuid.log
  has been merged.
* Note that older versions of git-annex will display the timestamp as part
  of the repository description, which is ugly but otherwise harmless.
---
 UUID.hs                                       | 41 ++++++++-----------
 debian/changelog                              |  5 +++
 ...st.log_and_remote.log_merge_wackiness.mdwn |  1 +
 doc/internals.mdwn                            |  6 +--
 test.hs                                       |  3 ++
 5 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/UUID.hs b/UUID.hs
index bd7b6b1d01..c4b870f39f 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -6,7 +6,9 @@
  - UUIDs of remotes are cached in git config, using keys named
  - remote..annex-uuid
  -
- - Copyright 2010 Joey Hess 
+ - uuid.log stores a list of known uuids, and their descriptions.
+ -
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -18,11 +20,11 @@ module UUID (
 	prepUUID,
 	genUUID,
 	describeUUID,
-	uuidMap,
-	uuidLog
+	uuidMap
 ) where
 
 import qualified Data.Map as M
+import Data.Time.Clock.POSIX
 
 import Common.Annex
 import qualified Git
@@ -30,13 +32,14 @@ import qualified Annex.Branch
 import Types.UUID
 import qualified Build.SysConfig as SysConfig
 import Config
+import UUIDLog
 
 configkey :: String
 configkey = "annex.uuid"
 
 {- Filename of uuid.log. -}
-uuidLog :: FilePath
-uuidLog = "uuid.log"
+logfile :: FilePath
+logfile = "uuid.log"
 
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
@@ -50,8 +53,7 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 			-- uuidgen generates random uuid by default
 			else []
 
-{- Looks up a repo's UUID. May return "" if none is known.
- -}
+{- Looks up a repo's UUID. May return "" if none is known. -}
 getUUID :: Git.Repo -> Annex UUID
 getUUID r = do
 	g <- gitRepo
@@ -76,26 +78,17 @@ getUncachedUUID r = Git.configGet r configkey ""
 prepUUID :: Annex ()
 prepUUID = do
 	u <- getUUID =<< gitRepo
-	when ("" == u) $ do
+	when (null u) $ do
 		uuid <- liftIO genUUID
 		setConfig configkey uuid
 
-{- Records a description for a uuid in the uuidLog. -}
+{- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
-describeUUID uuid desc = Annex.Branch.change uuidLog $
-	serialize . M.insert uuid desc . parse
-	where
-		serialize m = unlines $ map (\(u, d) -> u++" "++d) $ M.toList m
+describeUUID uuid desc = do
+	ts <- liftIO $ getPOSIXTime
+	Annex.Branch.change logfile $
+		showLog id . changeLog ts uuid desc . parseLog Just
 
-{- Read the uuidLog into a Map -}
+{- Read the uuidLog into a simple Map -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = parse <$> Annex.Branch.get uuidLog
-
-parse :: String -> M.Map UUID String
-parse = M.fromList . map pair . lines
-	where
-		pair l
-			| null ws = ("", "")
-			| otherwise = (head ws, unwords $ drop 1 ws)
-			where
-				ws = words l
+uuidMap = (simpleMap . parseLog Just) <$> Annex.Branch.get logfile
diff --git a/debian/changelog b/debian/changelog
index 7554cd5028..5eb7870ef5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,11 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     in addition to their descriptions.
   * Contain the zombie hordes.
   * Add locking to avoid races when changing the git-annex branch.
+  * New or changed repository descriptions in uuid.log now have a timestamp,
+    which is used to ensure the newest description is used when the uuid.log
+    has been merged.
+  * Note that older versions of git-annex will display the timestamp as part
+    of the repository description, which is ugly but otherwise harmless.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index 2724611fbf..5a703b2cbd 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -21,6 +21,7 @@ would keep working, ignoring the timestamp.
 - uuid.log: "uuid description timestamp" would work; old git-annex 
   would just treat the timestamp as part of the description which would be
   ok
+  > update: converted! --[[Joey]] 
 - trust.log: "uuid trustlevel timestamp" would work; old git-annex
   ignores trailing words
 - remote.log: "uuid key=value ... timestamp" is on the edge but does work
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index e80ecbac0d..4881588ca6 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -42,10 +42,10 @@ more useful than a UUID when it refers to a repository that does not have
 a configured git remote pointing at it.
 
 The file format is simply one line per repository, with the uuid followed by a
-space and then the description through to the end of the line. Example:
+space and then the description, followed by a timestamp. Example:
 
-	e605dca6-446a-11e0-8b2a-002170d25c55 laptop
-	26339d22-446b-11e0-9101-002170d25c55 usb disk
+	e605dca6-446a-11e0-8b2a-002170d25c55 laptop timestamp=1317929189.157237s
+	26339d22-446b-11e0-9101-002170d25c55 usb disk timestamp=1317929330.769997s
 
 ## `remotes.log`
 
diff --git a/test.hs b/test.hs
index 654af5713f..16f2a2bdf6 100644
--- a/test.hs
+++ b/test.hs
@@ -31,6 +31,7 @@ import qualified Types
 import qualified GitAnnex
 import qualified LocationLog
 import qualified UUID
+import qualified UUIDLog
 import qualified Trust
 import qualified Remote
 import qualified RemoteLog
@@ -78,6 +79,8 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
+	, qctest "prop_TimeStamp_sane" UUIDLog.prop_TimeStamp_sane
+	, qctest "prop_addLog_sane" UUIDLog.prop_addLog_sane
 	]
 
 blackbox :: Test

From f929d0229c05ebf0fe2c26d443fe6f843f270983 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 15:55:50 -0400
Subject: [PATCH 2308/8313] Add timestamps to trust.log.

---
 Trust.hs                                      | 38 ++++++++-----------
 debian/changelog                              |  1 +
 ...st.log_and_remote.log_merge_wackiness.mdwn |  1 +
 3 files changed, 18 insertions(+), 22 deletions(-)

diff --git a/Trust.hs b/Trust.hs
index 1920bc6170..2971256a45 100644
--- a/Trust.hs
+++ b/Trust.hs
@@ -7,29 +7,29 @@
 
 module Trust (
 	TrustLevel(..),
-	trustLog,
 	trustGet,
 	trustSet,
 	trustPartition
 ) where
 
 import qualified Data.Map as M
+import Data.Time.Clock.POSIX
 
 import Common.Annex
 import Types.TrustLevel
 import qualified Annex.Branch
-import UUID
 import qualified Annex
 
+import UUID
+import UUIDLog
+
 {- Filename of trust.log. -}
 trustLog :: FilePath
 trustLog = "trust.log"
 
 {- Returns a list of UUIDs at the specified trust level. -}
 trustGet :: TrustLevel -> Annex [UUID]
-trustGet level = do
-	m <- trustMap
-	return $ M.keys $ M.filter (== level) m
+trustGet level = M.keys . M.filter (== level) <$> trustMap
 
 {- Read the trustLog into a map, overriding with any
  - values from forcetrust. The map is cached for speed. -}
@@ -39,35 +39,29 @@ trustMap = do
 	case cached of
 		Just m -> return m
 		Nothing -> do
-			overrides <- Annex.getState Annex.forcetrust
-			l <- Annex.Branch.get trustLog
-			let m = M.fromList $ trustMapParse l ++ overrides
+			overrides <- M.fromList <$> Annex.getState Annex.forcetrust
+			m <- (M.union overrides . simpleMap . parseLog parseTrust) <$>
+				Annex.Branch.get trustLog
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
 
-{- Trust map parser. -}
-trustMapParse :: String -> [(UUID, TrustLevel)]
-trustMapParse s = map pair $ filter (not . null) $ lines s
+parseTrust :: String -> Maybe TrustLevel
+parseTrust s
+	| length w > 0 = readMaybe $ head w
+	-- back-compat; the trust.log used to only list trusted repos
+	| otherwise = Just Trusted
 	where
-		pair l
-			| length w > 1 = (w !! 0, read (w !! 1) :: TrustLevel)
-			-- for back-compat; the trust log used to only
-			-- list trusted uuids
-			| otherwise = (w !! 0, Trusted)
-			where
-				w = words l
+		w = words s
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid level = do
 	when (null uuid) $
 		error "unknown UUID; cannot modify trust level"
+	ts <- liftIO $ getPOSIXTime
 	Annex.Branch.change trustLog $
-		serialize . M.insert uuid level . M.fromList . trustMapParse
+		showLog show . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
-        where
-                serialize m = unlines $ map showpair $ M.toList m
-		showpair (u, t) = u ++ " " ++ show t
 
 {- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
 trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
diff --git a/debian/changelog b/debian/changelog
index 5eb7870ef5..e5f74bec5b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     has been merged.
   * Note that older versions of git-annex will display the timestamp as part
     of the repository description, which is ugly but otherwise harmless.
+  * Add timestamps to trust.log.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index 5a703b2cbd..f300d8708c 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -24,6 +24,7 @@ would keep working, ignoring the timestamp.
   > update: converted! --[[Joey]] 
 - trust.log: "uuid trustlevel timestamp" would work; old git-annex
   ignores trailing words
+  > update: converted! --[[Joey]] 
 - remote.log: "uuid key=value ... timestamp" is on the edge but does work
   (old git-annex will include the timestamp in the key/value map it builds,
   but that should not break anything really)

From f011033869bbeeb7941c1c6e16a2a138b11c92e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 16:07:51 -0400
Subject: [PATCH 2309/8313] add timestamps to remote.log

---
 RemoteLog.hs                                  | 30 ++++++++-----------
 debian/changelog                              |  2 +-
 ...st.log_and_remote.log_merge_wackiness.mdwn |  3 ++
 doc/internals.mdwn                            | 12 ++++----
 4 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/RemoteLog.hs b/RemoteLog.hs
index 149104fbac..d49635b930 100644
--- a/RemoteLog.hs
+++ b/RemoteLog.hs
@@ -6,7 +6,6 @@
  -}
 
 module RemoteLog (
-	remoteLog,
 	readRemoteLog,
 	configSet,
 	keyValToConfig,
@@ -16,12 +15,14 @@ module RemoteLog (
 ) where
 
 import qualified Data.Map as M
+import Data.Time.Clock.POSIX
 import Data.Char
 
 import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
 import UUID
+import UUIDLog
 
 {- Filename of remote.log. -}
 remoteLog :: FilePath
@@ -29,27 +30,20 @@ remoteLog = "remote.log"
 
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
-configSet u c = Annex.Branch.change remoteLog $
-	serialize . M.insert u c . remoteLogParse
-	where
-		serialize = unlines . sort . map toline . M.toList
-		toline (u', c') = u' ++ " " ++ unwords (configToKeyVal c')
+configSet u c = do
+	ts <- liftIO $ getPOSIXTime
+	Annex.Branch.change remoteLog $
+		showLog showConfig . changeLog ts u c . parseLog parseConfig
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = remoteLogParse <$> Annex.Branch.get remoteLog
+readRemoteLog = (simpleMap . parseLog parseConfig) <$> Annex.Branch.get remoteLog
 
-remoteLogParse :: String -> M.Map UUID RemoteConfig
-remoteLogParse s =
-	M.fromList $ mapMaybe parseline $ filter (not . null) $ lines s
-	where
-		parseline l
-			| length w > 2 = Just (u, c)
-			| otherwise = Nothing
-			where
-				w = words l
-				u = head w
-				c = keyValToConfig $ tail w
+parseConfig :: String -> Maybe RemoteConfig
+parseConfig = Just . keyValToConfig . words
+
+showConfig :: RemoteConfig -> String
+showConfig = unwords . configToKeyVal
 
 {- Given Strings like "key=value", generates a RemoteConfig. -}
 keyValToConfig :: [String] -> RemoteConfig
diff --git a/debian/changelog b/debian/changelog
index e5f74bec5b..31455c9f0c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,7 +12,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     has been merged.
   * Note that older versions of git-annex will display the timestamp as part
     of the repository description, which is ugly but otherwise harmless.
-  * Add timestamps to trust.log.
+  * Add timestamps to trust.log and remote.log too.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
index f300d8708c..a84d8cb568 100644
--- a/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
+++ b/doc/bugs/uuid.log_trust.log_and_remote.log_merge_wackiness.mdwn
@@ -28,6 +28,9 @@ would keep working, ignoring the timestamp.
 - remote.log: "uuid key=value ... timestamp" is on the edge but does work
   (old git-annex will include the timestamp in the key/value map it builds,
   but that should not break anything really)
+  > update: converted! --[[Joey]] 
 
 Appending "timestamp=xxxxx" would be good for clarity, and make
 it easier to parse the timestamp out from lines that have it.
+
+> [[done]] --[[Joey]] 
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 4881588ca6..5559d122b9 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -54,7 +54,7 @@ Amazon S3.
 
 The file format is one line per remote, starting with the uuid of the
 remote, followed by a space, and then a series of key=value pairs,
-each separated by whitespace.
+each separated by whitespace, and finally a timestamp.
 
 ## `trust.log`
 
@@ -62,13 +62,15 @@ Records the [[trust]] information for repositories. Does not exist unless
 [[trust]] values are configured.
 
 The file format is one line per repository, with the uuid followed by a
-space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted).
-Repositories not listed are semi-trusted.
+space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted),
+and finally a timestamp.
 
 Example:
 
-	e605dca6-446a-11e0-8b2a-002170d25c55 1
-	26339d22-446b-11e0-9101-002170d25c55 ?
+	e605dca6-446a-11e0-8b2a-002170d25c55 1 timestamp=1317929189.157237s
+	26339d22-446b-11e0-9101-002170d25c55 ? timestamp=1317929330.769997s
+
+Repositories not listed are semi-trusted.
 
 ## `aaa/bbb/*.log`
 

From 5414bbce58041aa92f2a50a8e721507879000f77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 6 Oct 2011 19:11:30 -0400
Subject: [PATCH 2310/8313] git-annex-shell uuid verification

* git-annex now asks git-annex-shell to verify that it's operating in
  the expected repository.
* Note that this git-annex will not interoperate with remotes using
  older versions of git-annex-shell.

The reason for this check is to avoid git-annex getting confused about
what remote repository actually contains a value. It's a prerequisite for
supporting git insteadOf aliases.
---
 Command.hs         |  2 ++
 Utility/Ssh.hs     | 12 +++++++++---
 debian/changelog   |  5 +++++
 git-annex-shell.hs | 16 ++++++++++++++--
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index 6f8684e4af..54dc9603fa 100644
--- a/Command.hs
+++ b/Command.hs
@@ -189,6 +189,8 @@ paramGlob :: String
 paramGlob = "GLOB"
 paramName :: String
 paramName = "NAME"
+paramUUID :: String
+paramUUID = "UUID"
 paramType :: String
 paramType = "TYPE"
 paramKeyValue :: String
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 05269552c7..4d17a47ba8 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -13,6 +13,7 @@ import qualified Git
 import Utility.SafeCommand
 import Types
 import Config
+import UUID
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
@@ -33,15 +34,20 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat
 git_annex_shell r command params
 	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
 	| Git.repoIsSsh r = do
-		sshparams <- sshToRepo r [Param sshcmd]
+		uuid <- getUUID r
+		sshparams <- sshToRepo r [Param $ sshcmd uuid ]
 		return $ Just ("ssh", sshparams)
 	| otherwise = return Nothing
 	where
 		dir = Git.workTree r
 		shellcmd = "git-annex-shell"
 		shellopts = Param command : File dir : params
-		sshcmd = shellcmd ++ " " ++ 
-			unwords (map shellEscape $ toCommand shellopts)
+		sshcmd uuid = unwords $
+			shellcmd : (map shellEscape $ toCommand shellopts) ++
+			uuidcheck uuid
+		uuidcheck uuid
+			| null uuid = []
+			| otherwise = ["--uuid", uuid]
 
 {- Uses a supplied function (such as boolSystem) to run a git-annex-shell
  - command on a remote.
diff --git a/debian/changelog b/debian/changelog
index 31455c9f0c..6a08d62e89 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,11 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Note that older versions of git-annex will display the timestamp as part
     of the repository description, which is ugly but otherwise harmless.
   * Add timestamps to trust.log and remote.log too.
+  * git-annex-shell: Added the --uuid option.
+  * git-annex now asks git-annex-shell to verify that it's operating in 
+    the expected repository.
+  * Note that this git-annex will not interoperate with remotes using 
+    older versions of git-annex-shell.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 2e0d309794..79b5da69a1 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -6,12 +6,14 @@
  -}
 
 import System.Environment
+import System.Console.GetOpt
 
 import Common.Annex
 import qualified Git
 import CmdLine
 import Command
 import Options
+import UUID
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
@@ -30,6 +32,16 @@ cmds = map adddirparam $ concat
 	where
 		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
 
+options :: [OptDescr (Annex ())]
+options = uuid : commonOptions
+	where
+		uuid = Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
+		check expected = do
+			u <- getUUID =<< gitRepo
+			when (u /= expected) $ error $
+				"expected repository UUID " ++ expected
+					++ " but found UUID " ++ u
+
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
 
@@ -57,7 +69,7 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params =
 	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
-		dispatch (cmd : filterparams params) cmds commonOptions header
+		dispatch (cmd : filterparams params) cmds options header
 
 external :: [String] -> IO ()
 external params =
@@ -72,4 +84,4 @@ filterparams ("--":_) = []
 filterparams (a:as) = a:filterparams as
 
 failure :: IO ()
-failure = error $ "bad parameters\n\n" ++ usage header cmds commonOptions
+failure = error $ "bad parameters\n\n" ++ usage header cmds options

From 3acdba3995941907028905a7c18362309af924b5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:17:54 -0400
Subject: [PATCH 2311/8313] faster union merge of multiple branches into index

only write index once
---
 Annex/Branch.hs    | 28 +++++++++++-----------------
 Git/UnionMerge.hs  | 17 +++++++++--------
 git-union-merge.hs |  2 +-
 3 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0b4bea051d..db4379243f 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -139,8 +139,18 @@ update = do
 			 - from the branches.
 			 -}
 			unless (null fs) $ stageJournalFiles fs
-			mapM_ mergeref refs
 			g <- gitRepo
+			unless (null refs) $ do
+				showSideAction $ "merging " ++
+					(unwords $ map Git.refDescribe refs) ++
+					" into " ++ name
+				{- Note: This merges the branches into the index.
+				 - Any unstaged changes in the git-annex branch
+				 - (if it's checked out) will be removed. So,
+				 - documentation advises users not to directly
+				 - modify the branch.
+				 -}
+				liftIO $ Git.UnionMerge.merge_index g refs
 			liftIO $ Git.commit g "update" fullname (fullname:refs)
 			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
@@ -155,22 +165,6 @@ update = do
 				Params "--oneline -n1"
 				]
 			return $ not $ L.null diffs
-		mergeref ref = do
-			showSideAction $ "merging " ++
-				Git.refDescribe ref ++ " into " ++ name
-			{- By passing only one ref, it is actually
-			 - merged into the index, preserving any
-			 - changes that may already be staged.
-			 -
-			 - However, any changes in the git-annex
-			 - branch that are *not* reflected in the
-			 - index will be removed. So, documentation
-			 - advises users not to directly modify the
-			 - branch.
-			 -}
-			g <- gitRepo
-			liftIO $ Git.UnionMerge.merge g [ref]
-			return $ Just ref
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index ac002b374e..859a66ca07 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -7,6 +7,7 @@
 
 module Git.UnionMerge (
 	merge,
+	merge_index,
 	update_index,
 	update_index_line,
 	ls_tree
@@ -18,24 +19,24 @@ import Data.Maybe
 import Data.String.Utils
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import Common
 import Git
-import Utility.SafeCommand
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
  -
- - When only one branch is specified, it is merged into the index.
- - In this case, previously staged changes in the index are preserved.
- -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Repo -> [String] -> IO ()
-merge g (x:y:[]) = do
+merge :: Repo -> String -> String -> IO ()
+merge g x y = do
 	a <- ls_tree g x
 	b <- merge_trees g x y
 	update_index g (a++b)
-merge g [x] = merge_tree_index g x >>= update_index g
-merge _ _ = error "wrong number of branches to merge"
+
+{- Merges a list of branches into the index. Previously staged changed in
+ - the index are preserved (and participate in the merge). -}
+merge_index :: Repo -> [String] -> IO ()
+merge_index g bs = update_index g =<< concat <$> mapM (merge_tree_index g) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 34f22d06fe..0d1d0819d2 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -41,6 +41,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
-	Git.UnionMerge.merge g [aref, bref]
+	Git.UnionMerge.merge g aref bref
 	Git.commit g "union merge" newref [aref, bref]
 	cleanup g

From 44fc358885f2d6ae20afb21d9526a14b2966901c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:37:01 -0400
Subject: [PATCH 2312/8313] avoid merging multiple branches that point to the
 same tree

avoids git warning "error: duplicate parent xxx ignored"
---
 Annex/Branch.hs | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index db4379243f..ea8ac2cec0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -126,7 +126,8 @@ update = do
 	unless (branchUpdated state) $ do
 		-- check what needs updating before taking the lock
 		fs <- getJournalFiles
-		refs <- filterM checkref =<< siblingBranches
+		c <- filterM changedbranch =<< siblingBranches
+		let (refs, branches) = unzip c
 		unless (null fs && null refs) $ withIndex $ lockJournal $ do
 			{- Before refs are merged into the index, it's
 			 - important to first stage the journal into the
@@ -140,9 +141,9 @@ update = do
 			 -}
 			unless (null fs) $ stageJournalFiles fs
 			g <- gitRepo
-			unless (null refs) $ do
+			unless (null branches) $ do
 				showSideAction $ "merging " ++
-					(unwords $ map Git.refDescribe refs) ++
+					(unwords $ map Git.refDescribe branches) ++
 					" into " ++ name
 				{- Note: This merges the branches into the index.
 				 - Any unstaged changes in the git-annex branch
@@ -150,18 +151,18 @@ update = do
 				 - documentation advises users not to directly
 				 - modify the branch.
 				 -}
-				liftIO $ Git.UnionMerge.merge_index g refs
-			liftIO $ Git.commit g "update" fullname (fullname:refs)
+				liftIO $ Git.UnionMerge.merge_index g branches
+			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
 			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
 	where
-		checkref ref = do
+		changedbranch (_, branch) = do
 			g <- gitRepo
 			-- checking with log to see if there have been changes
 			-- is less expensive than always merging
 			diffs <- liftIO $ Git.pipeRead g [
 				Param "log",
-				Param (name++".."++ref),
+				Param (name ++ ".." ++ branch),
 				Params "--oneline -n1"
 				]
 			return $ not $ L.null diffs
@@ -185,13 +186,15 @@ hasOrigin = refExists originname
 hasSomeBranch :: Annex Bool
 hasSomeBranch = not . null <$> siblingBranches
 
-{- List of all git-annex branches, including the main one and any
+{- List of all git-annex (refs, branches), including the main one and any
  - from remotes. -}
-siblingBranches :: Annex [String]
+siblingBranches :: Annex [(String, String)]
 siblingBranches = do
 	g <- gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-	return $ map (last . words . L.unpack) (L.lines r)
+	return $ map (pair . words . L.unpack) (L.lines r)
+	where
+		pair l = (head l, last l)
 
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()

From 82e655efd060ce3c674681b0c51f2efc1efa069d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:38:56 -0400
Subject: [PATCH 2313/8313] performance fix

It was checking if it needed to merge on every branch access, fix it to
only check once.
---
 Annex/Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ea8ac2cec0..29e3a3956f 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -153,8 +153,8 @@ update = do
 				 -}
 				liftIO $ Git.UnionMerge.merge_index g branches
 			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
-			Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 			invalidateCache
+		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
 	where
 		changedbranch (_, branch) = do
 			g <- gitRepo

From dfee6e1ed65e6df3fe66df6e8351d41a7572903f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 7 Oct 2011 13:59:34 -0400
Subject: [PATCH 2314/8313] better layout

And a theoretical fix to branchstate cache invalidation, but not a bug
that could actually happen.
---
 Annex/Branch.hs | 71 ++++++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 33 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 29e3a3956f..a548798d5d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -121,40 +121,37 @@ commit message = do
 {- Ensures that the branch is up-to-date; should be called before
  - data is read from it. Runs only once per git-annex run. -}
 update :: Annex ()
-update = do
-	state <- getState
-	unless (branchUpdated state) $ do
-		-- check what needs updating before taking the lock
-		fs <- getJournalFiles
-		c <- filterM changedbranch =<< siblingBranches
-		let (refs, branches) = unzip c
-		unless (null fs && null refs) $ withIndex $ lockJournal $ do
-			{- Before refs are merged into the index, it's
-			 - important to first stage the journal into the
-			 - index. Otherwise, any changes in the journal
-			 - would later get staged, and might overwrite
-			 - changes made during the merge.
-			 -
-			 - It would be cleaner to handle the merge by
-			 - updating the journal, not the index, with changes
-			 - from the branches.
+update = onceonly $ do
+	-- check what needs updating before taking the lock
+	fs <- getJournalFiles
+	c <- filterM changedbranch =<< siblingBranches
+	let (refs, branches) = unzip c
+	unless (null fs && null refs) $ withIndex $ lockJournal $ do
+		{- Before refs are merged into the index, it's
+		 - important to first stage the journal into the
+		 - index. Otherwise, any changes in the journal
+		 - would later get staged, and might overwrite
+		 - changes made during the merge.
+		 -
+		 - It would be cleaner to handle the merge by
+		 - updating the journal, not the index, with changes
+		 - from the branches.
+		 -}
+		unless (null fs) $ stageJournalFiles fs
+		g <- gitRepo
+		unless (null branches) $ do
+			showSideAction $ "merging " ++
+				(unwords $ map Git.refDescribe branches) ++
+				" into " ++ name
+			{- Note: This merges the branches into the index.
+			 - Any unstaged changes in the git-annex branch
+			 - (if it's checked out) will be removed. So,
+			 - documentation advises users not to directly
+			 - modify the branch.
 			 -}
-			unless (null fs) $ stageJournalFiles fs
-			g <- gitRepo
-			unless (null branches) $ do
-				showSideAction $ "merging " ++
-					(unwords $ map Git.refDescribe branches) ++
-					" into " ++ name
-				{- Note: This merges the branches into the index.
-				 - Any unstaged changes in the git-annex branch
-				 - (if it's checked out) will be removed. So,
-				 - documentation advises users not to directly
-				 - modify the branch.
-				 -}
-				liftIO $ Git.UnionMerge.merge_index g branches
-			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
-			invalidateCache
-		Annex.changeState $ \s -> s { Annex.branchstate = state { branchUpdated = True } }
+			liftIO $ Git.UnionMerge.merge_index g branches
+		liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
+		invalidateCache
 	where
 		changedbranch (_, branch) = do
 			g <- gitRepo
@@ -166,6 +163,14 @@ update = do
 				Params "--oneline -n1"
 				]
 			return $ not $ L.null diffs
+		onceonly a = unlessM (branchUpdated <$> getState) $ do
+			r <- a
+			Annex.changeState setupdated
+			return r
+		setupdated s = s { Annex.branchstate = new }
+			where
+				new = old { branchUpdated = True }
+				old = Annex.branchstate s
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool

From 81ed7b203d1c1af7a9c7a52c939eb0797fa16ab7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 9 Oct 2011 14:58:32 -0400
Subject: [PATCH 2315/8313] Now supports git's insteadOf configuration, to
 modify the url used to access a remote. Note that pushInsteadOf is not used;
 that and pushurl are reserved for actual git pushes. Closes: #644278

---
 Git.hs           | 22 ++++++++++++++++++----
 debian/changelog |  3 +++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/Git.hs b/Git.hs
index 7a3d55deaa..b87f2a2659 100644
--- a/Git.hs
+++ b/Git.hs
@@ -496,15 +496,29 @@ configParse s = M.fromList $ map pair $ lines s
 configRemotes :: Repo -> IO [Repo]
 configRemotes repo = mapM construct remotepairs
 	where
-		remotepairs = M.toList $ filterremotes $ config repo
-		filterremotes = M.filterWithKey (\k _ -> isremote k)
+		filterconfig f = filter f $ M.toList $ config repo
+		filterkeys f = filterconfig (\(k,_) -> f k)
+		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
 		construct (k,v) = do
-			r <- gen v
+			r <- gen $ calcloc v
 			return $ repoRemoteNameSet r k
-		gen v	| scpstyle v = repoFromUrl $ scptourl v
+		gen v
+			| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
+		-- insteadof config can rewrite remote location
+		calcloc l
+			| null insteadofs = l
+			| otherwise = replacement ++ drop (length replacement) l
+			where
+				replacement = take (length bestkey - length prefix) bestkey
+				bestkey = fst $ maximumBy longestvalue insteadofs
+				longestvalue (_, a) (_, b) = compare b a
+				insteadofs = filterconfig $ \(k, v) -> 
+					endswith prefix k &&
+					startswith v l
+				prefix = ".insteadof"
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
diff --git a/debian/changelog b/debian/changelog
index 6a08d62e89..a72a85a8a7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -18,6 +18,9 @@ git-annex (3.20110929) UNRELEASED; urgency=low
     the expected repository.
   * Note that this git-annex will not interoperate with remotes using 
     older versions of git-annex-shell.
+  * Now supports git's insteadOf configuration, to modify the url
+    used to access a remote. Note that pushInsteadOf is not used;
+    that and pushurl are reserved for actual git pushes. Closes: #644278
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From f0153f9fd7c753b12d612d1e57833abb2a7aa059 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 9 Oct 2011 16:19:09 -0400
Subject: [PATCH 2316/8313] fix a race

Another process may stage journalled files before the lock is
taken, so need to get the list of journalled files afterwards.

It's unfortunate this means getting the directory contents twice,
but it seems better to do that than sometimes take the lock
unnecessarily.
---
 Annex/Branch.hs | 48 ++++++++++++++++++++++++++----------------------
 1 file changed, 26 insertions(+), 22 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a548798d5d..0002befecd 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -111,33 +111,32 @@ create = unlessM hasBranch $ do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = do
-	fs <- getJournalFiles
-	when (not $ null fs) $ lockJournal $ do
-		stageJournalFiles fs
-		g <- gitRepo
-		withIndex $ liftIO $ Git.commit g message fullname [fullname]
+commit message = whenM journalDirty $ lockJournal $ do
+	stageJournalFiles
+	g <- gitRepo
+	withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before
- - data is read from it. Runs only once per git-annex run. -}
+ - data is read from it. Runs only once per git-annex run.
+ -
+ - Before refs are merged into the index, it's
+ - important to first stage the journal into the
+ - index. Otherwise, any changes in the journal
+ - would later get staged, and might overwrite
+ - changes made during the merge.
+ -
+ - It would be cleaner to handle the merge by
+ - updating the journal, not the index, with changes
+ - from the branches.
+ -}
 update :: Annex ()
 update = onceonly $ do
 	-- check what needs updating before taking the lock
-	fs <- getJournalFiles
+	dirty <- journalDirty
 	c <- filterM changedbranch =<< siblingBranches
 	let (refs, branches) = unzip c
-	unless (null fs && null refs) $ withIndex $ lockJournal $ do
-		{- Before refs are merged into the index, it's
-		 - important to first stage the journal into the
-		 - index. Otherwise, any changes in the journal
-		 - would later get staged, and might overwrite
-		 - changes made during the merge.
-		 -
-		 - It would be cleaner to handle the merge by
-		 - updating the journal, not the index, with changes
-		 - from the branches.
-		 -}
-		unless (null fs) $ stageJournalFiles fs
+	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
+		when dirty $ stageJournalFiles
 		g <- gitRepo
 		unless (null branches) $ do
 			showSideAction $ "merging " ++
@@ -279,8 +278,9 @@ getJournalFiles = do
 	return $ filter (`notElem` [".", ".."]) fs
 
 {- Stages the specified journalfiles. -}
-stageJournalFiles :: [FilePath] -> Annex ()
-stageJournalFiles fs = do
+stageJournalFiles :: Annex ()
+stageJournalFiles = do
+	fs <- getJournalFiles
 	g <- gitRepo
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
@@ -305,6 +305,10 @@ stageJournalFiles fs = do
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 
+{- Checks if there are changes in the journal. -}
+journalDirty :: Annex Bool
+journalDirty = not . null <$> getJournalFiles
+
 {- Produces a filename to use in the journal for a file on the branch.
  -
  - The journal typically won't have a lot of files in it, so the hashing

From 10edaf6dc9c5a8f091487e8ba82ac8f54697bf0c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 10 Oct 2011 16:03:32 -0400
Subject: [PATCH 2317/8313] reorder

---
 debian/changelog | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index a72a85a8a7..9bcaa40466 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,12 +1,6 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
-  * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
-  * status: List all known repositories.
-  * When displaying a list of repositories, show git remote names
-    in addition to their descriptions.
-  * Contain the zombie hordes.
-  * Add locking to avoid races when changing the git-annex branch.
   * New or changed repository descriptions in uuid.log now have a timestamp,
     which is used to ensure the newest description is used when the uuid.log
     has been merged.
@@ -21,6 +15,12 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Now supports git's insteadOf configuration, to modify the url
     used to access a remote. Note that pushInsteadOf is not used;
     that and pushurl are reserved for actual git pushes. Closes: #644278
+  * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
+  * Add locking to avoid races when changing the git-annex branch.
+  * Various speed improvements gained by using ByteStrings.
+  * Contain the zombie hordes.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 025ded4a2dfb58a6ec0cb47b9d625d593a4e1977 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 10 Oct 2011 17:37:44 -0400
Subject: [PATCH 2318/8313] tweaks

---
 Annex/Branch.hs  | 30 ++++++++++++++----------------
 Backend.hs       | 42 +++++++++++++++++++-----------------------
 Types/Backend.hs |  2 +-
 3 files changed, 34 insertions(+), 40 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0002befecd..aea0d2bffe 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -215,20 +215,16 @@ set file content = do
  -
  - Returns an empty string if the file doesn't exist yet. -}
 get :: FilePath -> Annex String
-get file = do
-	cached <- getCache file
-	case cached of
-		Just content -> return content
-		Nothing -> do
-			j <- getJournalFile file
-			case j of
-				Just content -> do
-					setCache file content
-					return content
-				Nothing -> withIndexUpdate $ do
-					content <- catFile fullname file
-					setCache file content
-					return content
+get file = fromcache =<< getCache file
+	where
+		fromcache (Just content) = return content
+		fromcache Nothing = fromjournal =<< getJournalFile file
+		fromjournal (Just content) = cache content
+		fromjournal Nothing = withIndexUpdate $
+			cache =<< catFile fullname file
+		cache content = do
+			setCache file content
+			return content
 
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
@@ -287,8 +283,7 @@ stageJournalFiles = do
 		let paths = map (dir ) fs
 		-- inject all the journal files directly into git
 		-- in one quick command
-		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $
-			Git.gitCommandLine g [Param "hash-object", Param "-w", Param "--stdin-paths"]
+		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object g
 		_ <- forkProcess $ do
 			hPutStr toh $ unlines paths
 			hClose toh
@@ -304,6 +299,9 @@ stageJournalFiles = do
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
+		git_hash_object g = Git.gitCommandLine g
+			[Param "hash-object", Param "-w", Param "--stdin-paths"]
+
 
 {- Checks if there are changes in the journal. -}
 journalDirty :: Annex Bool
diff --git a/Backend.hs b/Backend.hs
index d1ff114059..a09fc0e990 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -39,23 +39,20 @@ orderedList = do
 	l <- Annex.getState Annex.backends -- list is cached here
 	if not $ null l
 		then return l
-		else do
-			s <- getstandard
-			d <- Annex.getState Annex.forcebackend
-			handle d s
+		else handle =<< Annex.getState Annex.forcebackend
 	where
-		parseBackendList [] = list
-		parseBackendList s = map lookupBackendName $ words s
-		handle Nothing s = return s
-		handle (Just "") s = return s
-		handle (Just name) s = do
-			let l' = lookupBackendName name : s
-			Annex.changeState $ \state -> state { Annex.backends = l' }
+		handle Nothing = standard
+		handle (Just "") = standard
+		handle (Just name) = do
+			l' <- (lookupBackendName name :) <$> standard
+			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		getstandard = do
+		standard = do
 			g <- gitRepo
 			return $ parseBackendList $
 				Git.configGet g "annex.backends" ""
+		parseBackendList [] = list
+		parseBackendList s = map lookupBackendName $ words s
 
 {- Generates a key for a file, trying each backend in turn until one
  - accepts it. -}
@@ -83,17 +80,15 @@ lookupFile file = do
 	where
 		getsymlink = takeFileName <$> readSymbolicLink file
 		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
-		makeret l k =
+		makeret l k = let bname = keyBackendName k in
 			case maybeLookupBackendName bname of
-					Just backend -> return $ Just (k, backend)
-					Nothing -> do
-						when (isLinkToAnnex l) $
-							warning skip
-						return Nothing
-			where
-				bname = keyBackendName k
-				skip = "skipping " ++ file ++ 
-					" (unknown backend " ++ bname ++ ")"
+				Just backend -> return $ Just (k, backend)
+				Nothing -> do
+					when (isLinkToAnnex l) $ warning $
+						"skipping " ++ file ++
+						" (unknown backend " ++
+						bname ++ ")"
+					return Nothing
 
 type BackendFile = (Maybe (Backend Annex), FilePath)
 
@@ -121,4 +116,5 @@ maybeLookupBackendName :: String -> Maybe (Backend Annex)
 maybeLookupBackendName s
 	| length matches == 1 = Just $ head matches
 	| otherwise = Nothing
-	where matches = filter (\b -> s == B.name b) list
+	where
+		matches = filter (\b -> s == B.name b) list
diff --git a/Types/Backend.hs b/Types/Backend.hs
index f86d0845cd..4f82267045 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -1,6 +1,6 @@
 {- git-annex key/value backend data type
  -
- - Most things should not need this, using Types instead
+ - Most things should not need this, using Remotes instead
  -
  - Copyright 2010 Joey Hess 
  -

From b505ba83e8b62a9ed0ec2fb96448c5fc801184d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 14:43:45 -0400
Subject: [PATCH 2319/8313] minor syntax changes

---
 Annex/Content.hs             |  2 +-
 Backend/SHA.hs               | 16 ++++++++--------
 Command/ConfigList.hs        |  3 +--
 Command/Fsck.hs              |  2 +-
 Command/Init.hs              |  3 +--
 Command/Move.hs              |  6 ++----
 Crypto.hs                    | 25 ++++++++++++-------------
 Init.hs                      |  6 +++---
 Limit.hs                     |  7 +++----
 Messages.hs                  | 27 +++++++++++++++------------
 Remote.hs                    | 12 +++++-------
 Remote/Git.hs                |  2 +-
 Remote/Helper/Encryptable.hs |  6 ++----
 UUID.hs                      | 17 +++++++++--------
 Utility.hs                   | 10 +++-------
 Utility/Path.hs              | 21 ++++++++-------------
 Utility/Ssh.hs               |  2 +-
 git-annex-shell.hs           |  2 +-
 test.hs                      |  4 +---
 19 files changed, 78 insertions(+), 95 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 21403954ae..3827154a67 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -57,7 +57,7 @@ calcGitLink file key = do
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
 	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	logChange g key u status
 
 {- Runs an action, passing it a temporary filename to download,
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 4b5b14cc36..3a54a8871b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -104,11 +104,11 @@ checkKeyChecksum size key = do
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
-		else do
-			s <- shaN size file
-			if s == dropExtension (keyName key)
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file content; moved to " ++ dest
-					return False
+		else check =<< shaN size file
+	where
+		check s
+			| s == dropExtension (keyName key) = return True
+			| otherwise = do
+				dest <- moveBad key
+				warning $ "Bad file content; moved to " ++ dest
+				return False
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d52c33f3b9..60f71eee6d 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -20,7 +20,6 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	liftIO $ putStrLn $ "annex.uuid=" ++ u
 	stop
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 689271dd7e..16e834fbf1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -55,7 +55,7 @@ verifyLocationLog key file = do
 		preventWrite f
 		preventWrite (parentDir f)
 
-	u <- getUUID g
+	u <- getUUID
         uuids <- keyLocations key
 
 	case (present, u `elem` uuids) of
diff --git a/Command/Init.hs b/Command/Init.hs
index ace06c2c34..b9dffb5cd9 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -29,7 +29,6 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	describeUUID u description
 	next $ return True
diff --git a/Command/Move.hs b/Command/Move.hs
index d650c52510..52eb49da14 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -72,8 +72,7 @@ remoteHasKey remote key present	= do
  -}
 toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 toStart dest move file = isAnnexed file $ \(key, _) -> do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
 		then stop -- not here, so nothing to do
@@ -122,8 +121,7 @@ toCleanup dest move key = do
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
 fromStart src move file = isAnnexed file $ \(key, _) -> do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	remotes <- Remote.keyPossibilities key
 	if u == Remote.uuid src || not (any (== src) remotes)
 		then stop
diff --git a/Crypto.hs b/Crypto.hs
index 21e4d75605..ced7c144c4 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -135,13 +135,12 @@ decryptCipher _ (EncryptedCipher encipher _) =
 {- Generates an encrypted form of a Key. The encryption does not need to be
  - reversable, nor does it need to be the same type of encryption used
  - on content. It does need to be repeatable. -}
-encryptKey :: Cipher -> Key -> IO Key
-encryptKey c k =
-	return Key {
-		keyName = hmacWithCipher c (show k),
-		keyBackendName = "GPGHMACSHA1",
-		keySize = Nothing, -- size and mtime omitted
-		keyMtime = Nothing -- to avoid leaking data
+encryptKey :: Cipher -> Key -> Key
+encryptKey c k = Key
+	{ keyName = hmacWithCipher c (show k)
+	, keyBackendName = "GPGHMACSHA1"
+	, keySize = Nothing -- size and mtime omitted
+	, keyMtime = Nothing -- to avoid leaking data
 	}
 
 {- Runs an action, passing it a handle from which it can 
@@ -223,18 +222,18 @@ gpgCipherHandle params c a b = do
 	return ret
 
 configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = do
-	let k = configGet c "encryption"
-	s <- gpgRead [Params "--with-colons --list-public-keys", Param k]
-	return $ KeyIds $ parseWithColons s
+configKeyIds c = parse <$> gpgRead params
 	where
-		parseWithColons s = map keyIdField $ filter pubKey $ lines s
+		params = [Params "--with-colons --list-public-keys",
+			Param $ configGet c "encryption"]
+		parse = KeyIds . map keyIdField . filter pubKey . lines
 		pubKey = isPrefixOf "pub:"
 		keyIdField s = split ":" s !! 4
 
 configGet :: RemoteConfig -> String -> String
 configGet c key = fromMaybe missing $ M.lookup key c
-	where missing = error $ "missing " ++ key ++ " in remote config"
+	where
+		missing = error $ "missing " ++ key ++ " in remote config"
 
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
diff --git a/Init.hs b/Init.hs
index 145413e8da..509cbca15d 100644
--- a/Init.hs
+++ b/Init.hs
@@ -75,6 +75,6 @@ preCommitHook = do
 
 preCommitScript :: String
 preCommitScript = 
-		"#!/bin/sh\n" ++
-		"# automatically configured by git-annex\n" ++ 
-		"git annex pre-commit .\n"
+	"#!/bin/sh\n" ++
+	"# automatically configured by git-annex\n" ++ 
+	"git annex pre-commit .\n"
diff --git a/Limit.hs b/Limit.hs
index 3812ceea4b..8dd88e72b8 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -65,14 +65,13 @@ addExclude glob = addLimit $ return . notExcluded
 {- Adds a limit to skip files not believed to be present
  - in a specfied repository. -}
 addIn :: String -> Annex ()
-addIn name = do
-	u <- Remote.nameToUUID name
-	addLimit $ if name == "." then check inAnnex else check (remote u)
+addIn name = addLimit $ check $ if name == "." then inAnnex else inremote
 	where
 		check a f = Backend.lookupFile f >>= handle a
 		handle _ Nothing = return False
 		handle a (Just (key, _)) = a key
-		remote u key = do
+		inremote key = do
+			u <- Remote.nameToUUID name
 			us <- keyLocations key
 			return $ u `elem` us
 
diff --git a/Messages.hs b/Messages.hs
index e029c50722..6f4880e2d5 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -31,31 +31,31 @@ import qualified Annex
 import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = handle (JSON.start command file) $ do
-	putStr $ command ++ " " ++ file ++ " "
-	hFlush stdout
+showStart command file = handle (JSON.start command file) $
+	flushed $ putStr $ command ++ " " ++ file ++ " "
 
 showNote :: String -> Annex ()
-showNote s = handle (JSON.note s) $ do
-	putStr $ "(" ++ s ++ ") "
-	hFlush stdout
+showNote s = handle (JSON.note s) $
+	flushed $ putStr $ "(" ++ s ++ ") "
 
 showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
 showProgress :: Annex ()
-showProgress = handle q $ do
-	putStr "."
-	hFlush stdout
+showProgress = handle q $
+	flushed $ putStr "."
 
 showSideAction :: String -> Annex ()
-showSideAction s = handle q $ putStrLn $ "(" ++ s ++ "...)"
+showSideAction s = handle q $
+	putStrLn $ "(" ++ s ++ "...)"
 
 showOutput :: Annex ()
-showOutput = handle q $ putStr "\n"
+showOutput = handle q $
+	putStr "\n"
 
 showLongNote :: String -> Annex ()
-showLongNote s = handle (JSON.note s) $ putStrLn $ '\n' : indent s
+showLongNote s = handle (JSON.note s) $
+	putStrLn $ '\n' : indent s
 
 showEndOk :: Annex ()
 showEndOk = showEndResult True
@@ -113,3 +113,6 @@ maybeShowJSON v = handle (JSON.add v) q
 
 q :: Monad m => m ()
 q = return ()
+
+flushed :: IO () -> IO ()
+flushed a = a >> hFlush stdout
diff --git a/Remote.hs b/Remote.hs
index 27ebd724ac..b1305b9e0f 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -78,7 +78,7 @@ genList = do
 			enumerate t >>=
 			mapM (gen m t)
 		gen m t r = do
-			u <- getUUID r
+			u <- getRepoUUID r
 			generate t r u (M.lookup u m)
 
 {- Looks up a remote by name. (Or by UUID.) Only finds currently configured
@@ -104,7 +104,7 @@ byName' n = do
  - and returns its UUID. Finds even remotes that are not configured in
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
-nameToUUID "." = getUUID =<< gitRepo -- special case for current repo
+nameToUUID "." = getUUID -- special case for current repo
 nameToUUID n = do
 	res <- byName' n
 	case res of
@@ -129,7 +129,7 @@ nameToUUID n = do
  - of the UUIDs. -}
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
-	here <- getUUID =<< gitRepo
+	here <- getUUID
 	m <- M.unionWith addname <$> uuidMap <*> remoteMap
 	maybeShowJSON [(desc, map (jsonify m here) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
@@ -178,8 +178,7 @@ keyPossibilitiesTrusted = keyPossibilities' True
 
 keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
 keyPossibilities' withtrusted key = do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	trusted <- if withtrusted then trustGet Trusted else return []
 
 	-- get uuids of all remotes that are recorded to have the key
@@ -198,8 +197,7 @@ keyPossibilities' withtrusted key = do
 {- Displays known locations of a key. -}
 showLocations :: Key -> [UUID] -> Annex ()
 showLocations key exclude = do
-	g <- gitRepo
-	u <- getUUID g
+	u <- getUUID
 	uuids <- keyLocations key
 	untrusteduuids <- trustGet UnTrusted
 	let uuidswanted = filteruuids uuids (u:exclude++untrusteduuids) 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 704bdc04d3..183fcd8548 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -48,7 +48,7 @@ gen r u _ = do
 		(False, "") -> tryGitConfigRead r
 		_ -> return r
 
-	u' <- getUUID r'
+	u' <- getRepoUUID r'
 
 	let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost
 	cst <- remoteCost r' defcst
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 004b70408b..85d269a213 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -78,8 +78,6 @@ remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
 cipherKey Nothing _ = return Nothing
-cipherKey (Just c) k = remoteCipher c >>= maybe (return Nothing) encrypt
+cipherKey (Just c) k = maybe Nothing encrypt <$> remoteCipher c
 	where
-		encrypt ciphertext = do
-			k' <- liftIO $ encryptKey ciphertext k
-			return $ Just (ciphertext, k')
+		encrypt ciphertext = Just (ciphertext, encryptKey ciphertext k)
diff --git a/UUID.hs b/UUID.hs
index c4b870f39f..63ce87f03e 100644
--- a/UUID.hs
+++ b/UUID.hs
@@ -16,6 +16,7 @@
 module UUID (
 	UUID,
 	getUUID,
+	getRepoUUID,
 	getUncachedUUID,
 	prepUUID,
 	genUUID,
@@ -44,7 +45,7 @@ logfile = "uuid.log"
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID
-genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
+genUUID = pOpen ReadFromPipe command params hGetLine
 	where
 		command = SysConfig.uuid
 		params = if command == "uuid"
@@ -53,9 +54,12 @@ genUUID = liftIO $ pOpen ReadFromPipe command params $ \h -> hGetLine h
 			-- uuidgen generates random uuid by default
 			else []
 
+getUUID :: Annex UUID
+getUUID = getRepoUUID =<< gitRepo
+
 {- Looks up a repo's UUID. May return "" if none is known. -}
-getUUID :: Git.Repo -> Annex UUID
-getUUID r = do
+getRepoUUID :: Git.Repo -> Annex UUID
+getRepoUUID r = do
 	g <- gitRepo
 
 	let c = cached g
@@ -76,11 +80,8 @@ getUncachedUUID r = Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
-prepUUID = do
-	u <- getUUID =<< gitRepo
-	when (null u) $ do
-		uuid <- liftIO genUUID
-		setConfig configkey uuid
+prepUUID = whenM (null <$> getUUID) $
+	setConfig configkey =<< liftIO genUUID
 
 {- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
diff --git a/Utility.hs b/Utility.hs
index 4e82e63c9d..8ef60a0814 100644
--- a/Utility.hs
+++ b/Utility.hs
@@ -19,6 +19,7 @@ module Utility (
 	anyM
 ) where
 
+import Control.Applicative
 import IO (bracket)
 import System.IO
 import System.Posix.Process hiding (executeFile)
@@ -69,9 +70,7 @@ withTempFile template a = bracket create remove use
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]
-dirContents d = do
-	c <- getDirectoryContents d
-	return $ map (d ) $ filter notcruft c
+dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
 	where
 		notcruft "." = False
 		notcruft ".." = False
@@ -79,10 +78,7 @@ dirContents d = do
 
 {- Current user's home directory. -}
 myHomeDir :: IO FilePath
-myHomeDir = do
-	uid <- getEffectiveUserID
-	u <- getUserEntryForID uid
-	return $ homeDirectory u
+myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
 
 {- Catches IO errors and returns a Bool -}
 catchBool :: IO Bool -> IO Bool
diff --git a/Utility/Path.hs b/Utility/Path.hs
index ce54fb3695..1c68b87bbc 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -17,10 +17,9 @@ import Control.Applicative
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
-parentDir dir =
-	if not $ null dirs
-	then slash ++ join s (init dirs)
-	else ""
+parentDir dir
+	| not $ null dirs = slash ++ join s (init dirs)
+	| otherwise = ""
 		where
 			dirs = filter (not . null) $ split s dir
 			slash = if isAbsolute dir then s else ""
@@ -72,7 +71,7 @@ relPathCwdToFile f = relPathDirToFile <$> getCurrentDirectory <*> absPath f
  - Both must be absolute, and normalized (eg with absNormpath).
  -}
 relPathDirToFile :: FilePath -> FilePath -> FilePath
-relPathDirToFile from to = path
+relPathDirToFile from to = join s $ dotdots ++ uncommon
 	where
 		s = [pathSeparator]
 		pfrom = split s from
@@ -82,7 +81,6 @@ relPathDirToFile from to = path
 		uncommon = drop numcommon pto
 		dotdots = replicate (length pfrom - numcommon) ".."
 		numcommon = length common
-		path = join s $ dotdots ++ uncommon
 
 prop_relPathDirToFile_basics :: FilePath -> FilePath -> Bool
 prop_relPathDirToFile_basics from to
@@ -99,14 +97,11 @@ prop_relPathDirToFile_basics from to
  - appear at the same position as it did in the input list.
  -}
 preserveOrder :: [FilePath] -> [FilePath] -> [FilePath]
--- optimisation, only one item in original list, so no reordering needed
-preserveOrder [_] new = new
-preserveOrder orig new = collect orig new
+preserveOrder [] new = new
+preserveOrder [_] new = new -- optimisation
+preserveOrder (l:ls) new = found ++ preserveOrder ls rest
 	where
-		collect [] n = n
-		collect [_] n = n -- optimisation
-		collect (l:ls) n = found ++ collect ls rest
-			where (found, rest)=partition (l `dirContains`) n
+		(found, rest)=partition (l `dirContains`) new
 
 {- Runs an action that takes a list of FilePaths, and ensures that 
  - its return list preserves order.
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 4d17a47ba8..a0e52507d6 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -34,7 +34,7 @@ git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePat
 git_annex_shell r command params
 	| not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts)
 	| Git.repoIsSsh r = do
-		uuid <- getUUID r
+		uuid <- getRepoUUID r
 		sshparams <- sshToRepo r [Param $ sshcmd uuid ]
 		return $ Just ("ssh", sshparams)
 	| otherwise = return Nothing
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 79b5da69a1..a4d8dd65b6 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -37,7 +37,7 @@ options = uuid : commonOptions
 	where
 		uuid = Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
 		check expected = do
-			u <- getUUID =<< gitRepo
+			u <- getUUID
 			when (u /= expected) $ error $
 				"expected repository UUID " ++ expected
 					++ " but found UUID " ++ u
diff --git a/test.hs b/test.hs
index 16f2a2bdf6..1c511b4d4b 100644
--- a/test.hs
+++ b/test.hs
@@ -609,9 +609,7 @@ checkdangling f = do
 
 checklocationlog :: FilePath -> Bool -> Assertion
 checklocationlog f expected = do
-	thisuuid <- annexeval $ do
-		g <- Annex.gitRepo
-		UUID.getUUID g
+	thisuuid <- annexeval UUID.getUUID
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do

From 9c04d1e523dc513c64adf251046e13c4e7f7b5dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 22:52:55 -0400
Subject: [PATCH 2320/8313] fix git 1.7.7 breakage

* This version of git-annex only works with git 1.7.7 and newer.
  The breakage with old versions is subtle, and affects
  annex.numcopies .gitattributes settings, so be sure to upgrade git
  to 1.7.7. (Debian package now depends on that version.)
* Don't pass absolute paths to git show-attr, as it started following
  symlinks when that's done in 1.7.7. Instead, use relative paths,
  which show-attr only handles 100% correctly in 1.7.7. Closes: #645046

Unfortunatly I can find no way to work with the old and new gits, as
the old had bugs that require absolute paths, while the new doesn't like
them at all. And the behavior of git show-attr in 1.7.7. is the same as
eg, git add of an absolute path to a symlink, so seems entirely
intentional and not likely to change.
---
 Git.hs           | 19 +++----------------
 debian/changelog | 31 +++++++++++++++++++++++++++++++
 debian/control   |  2 +-
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/Git.hs b/Git.hs
index b87f2a2659..173deba6e8 100644
--- a/Git.hs
+++ b/Git.hs
@@ -547,36 +547,23 @@ configMap = config
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
 checkAttr repo attr files = do
-	-- git check-attr wants files that are absolute (or relative to the
-	-- top of the repo). But we're passed files relative to the current
-	-- directory. Convert to absolute, and then convert the filenames
-	-- in its output back to relative.
-	cwd <- getCurrentDirectory
-	let top = workTree repo
-	let absfiles = map (absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
 		hClose fromh
-                hPutStr toh $ join "\0" absfiles
+                hPutStr toh $ join "\0" files
                 hClose toh
                 exitSuccess
         hClose toh
-	s <- hGetContents fromh
-	return $ map (topair cwd top) $ lines s
+	(map topair . lines) <$> hGetContents fromh
 	where
 		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
-		topair cwd top l = (relfile, value)
+		topair l = (file, value)
 			where 
-				relfile
-					| startswith cwd' file = drop (length cwd') file
-					| otherwise = relPathDirToFile top' file
 				file = decodeGitFile $ join sep $ take end bits
 				value = bits !! end
 				end = length bits - 1
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
-				cwd' = cwd ++ "/"
-				top' = top ++ "/"
 
 {- Some git commands output encoded filenames. Decode that (annoyingly
  - complex) encoding. -}
diff --git a/debian/changelog b/debian/changelog
index 9bcaa40466..85506655fd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,6 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
+<<<<<<<
   * Fix referring to remotes by uuid.
   * New or changed repository descriptions in uuid.log now have a timestamp,
     which is used to ensure the newest description is used when the uuid.log
@@ -21,6 +22,36 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Add locking to avoid races when changing the git-annex branch.
   * Various speed improvements gained by using ByteStrings.
   * Contain the zombie hordes.
+=======
+  * This version of git-annex only works with git 1.7.7 and newer.
+    The breakage with old versions is subtle, and affects
+    annex.numcopies .gitattributes settings, so be sure to upgrade git
+    to 1.7.7. (Debian package now depends on that version.)
+  * Don't pass absolute paths to git show-attr, as it started following
+    symlinks when that's done in 1.7.7. Instead, use relative paths,
+    which show-attr only handles 100% correctly in 1.7.7. Closes: #645046
+  * Various speed improvements gained by using ByteStrings.
+  * Fix referring to remotes by uuid.
+  * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
+  * Contain the zombie hordes.
+  * Add locking to avoid races when changing the git-annex branch.
+  * New or changed repository descriptions in uuid.log now have a timestamp,
+    which is used to ensure the newest description is used when the uuid.log
+    has been merged.
+  * Note that older versions of git-annex will display the timestamp as part
+    of the repository description, which is ugly but otherwise harmless.
+  * Add timestamps to trust.log and remote.log too.
+  * git-annex-shell: Added the --uuid option.
+  * git-annex now asks git-annex-shell to verify that it's operating in 
+    the expected repository.
+  * Note that this git-annex will not interoperate with remotes using 
+    older versions of git-annex-shell.
+  * Now supports git's insteadOf configuration, to modify the url
+    used to access a remote. Note that pushInsteadOf is not used;
+    that and pushurl are reserved for actual git pushes. Closes: #644278
+>>>>>>>
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 
diff --git a/debian/control b/debian/control
index cb5a8212ab..49f564fdb9 100644
--- a/debian/control
+++ b/debian/control
@@ -29,7 +29,7 @@ Package: git-annex
 Architecture: any
 Section: utils
 Depends: ${misc:Depends}, ${shlibs:Depends},
-	git | git-core,
+	git (>= 1:1.7.7),
 	uuid,
 	rsync,
 	wget | curl,

From 402d9c7c5fa79d78fe09e5ed0b5a3b60e16a7f0b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 22:54:38 -0400
Subject: [PATCH 2321/8313] oops

---
 debian/changelog | 36 ++++++------------------------------
 1 file changed, 6 insertions(+), 30 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 85506655fd..e985166dcd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,28 +1,5 @@
 git-annex (3.20110929) UNRELEASED; urgency=low
 
-<<<<<<<
-  * Fix referring to remotes by uuid.
-  * New or changed repository descriptions in uuid.log now have a timestamp,
-    which is used to ensure the newest description is used when the uuid.log
-    has been merged.
-  * Note that older versions of git-annex will display the timestamp as part
-    of the repository description, which is ugly but otherwise harmless.
-  * Add timestamps to trust.log and remote.log too.
-  * git-annex-shell: Added the --uuid option.
-  * git-annex now asks git-annex-shell to verify that it's operating in 
-    the expected repository.
-  * Note that this git-annex will not interoperate with remotes using 
-    older versions of git-annex-shell.
-  * Now supports git's insteadOf configuration, to modify the url
-    used to access a remote. Note that pushInsteadOf is not used;
-    that and pushurl are reserved for actual git pushes. Closes: #644278
-  * status: List all known repositories.
-  * When displaying a list of repositories, show git remote names
-    in addition to their descriptions.
-  * Add locking to avoid races when changing the git-annex branch.
-  * Various speed improvements gained by using ByteStrings.
-  * Contain the zombie hordes.
-=======
   * This version of git-annex only works with git 1.7.7 and newer.
     The breakage with old versions is subtle, and affects
     annex.numcopies .gitattributes settings, so be sure to upgrade git
@@ -30,13 +7,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Don't pass absolute paths to git show-attr, as it started following
     symlinks when that's done in 1.7.7. Instead, use relative paths,
     which show-attr only handles 100% correctly in 1.7.7. Closes: #645046
-  * Various speed improvements gained by using ByteStrings.
   * Fix referring to remotes by uuid.
-  * status: List all known repositories.
-  * When displaying a list of repositories, show git remote names
-    in addition to their descriptions.
-  * Contain the zombie hordes.
-  * Add locking to avoid races when changing the git-annex branch.
   * New or changed repository descriptions in uuid.log now have a timestamp,
     which is used to ensure the newest description is used when the uuid.log
     has been merged.
@@ -51,7 +22,12 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Now supports git's insteadOf configuration, to modify the url
     used to access a remote. Note that pushInsteadOf is not used;
     that and pushurl are reserved for actual git pushes. Closes: #644278
->>>>>>>
+  * status: List all known repositories.
+  * When displaying a list of repositories, show git remote names
+    in addition to their descriptions.
+  * Add locking to avoid races when changing the git-annex branch.
+  * Various speed improvements gained by using ByteStrings.
+  * Contain the zombie hordes.
 
  -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
 

From 11b154e811a9dcea42ceaaf9740f811ac310a615 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:03:19 -0400
Subject: [PATCH 2322/8313] prep release

---
 debian/changelog | 8 ++++----
 debian/control   | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e985166dcd..c4e53e1795 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,8 @@
-git-annex (3.20110929) UNRELEASED; urgency=low
+git-annex (3.20111011) unstable; urgency=low
 
   * This version of git-annex only works with git 1.7.7 and newer.
-    The breakage with old versions is subtle, and affects
-    annex.numcopies .gitattributes settings, so be sure to upgrade git
+    The breakage with old versions is subtle, and affects the
+    annex.numcopies settings in .gitattributes, so be sure to upgrade git
     to 1.7.7. (Debian package now depends on that version.)
   * Don't pass absolute paths to git show-attr, as it started following
     symlinks when that's done in 1.7.7. Instead, use relative paths,
@@ -29,7 +29,7 @@ git-annex (3.20110929) UNRELEASED; urgency=low
   * Various speed improvements gained by using ByteStrings.
   * Contain the zombie hordes.
 
- -- Joey Hess   Thu, 29 Sep 2011 18:58:53 -0400
+ -- Joey Hess   Tue, 11 Oct 2011 23:00:02 -0400
 
 git-annex (3.20110928) unstable; urgency=low
 
diff --git a/debian/control b/debian/control
index 49f564fdb9..6f59ada5b8 100644
--- a/debian/control
+++ b/debian/control
@@ -17,7 +17,7 @@ Build-Depends:
 	libghc-json-dev,
 	ikiwiki,
 	perlmagick,
-	git | git-core,
+	git,
 	uuid,
 	rsync,
 Maintainer: Joey Hess 

From 5a9e3f73772daff3ffadd2359d99914f0567b78c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:39:29 -0400
Subject: [PATCH 2323/8313] force files relative

Other code is currently depending on checkAttr forcing absolute filenames
to relative, so keep it doing so.

This is a quick fix, and should sometime be moved elsewhere.
---
 Git.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Git.hs b/Git.hs
index 173deba6e8..fb998b1b0e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -547,10 +547,12 @@ configMap = config
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
 checkAttr repo attr files = do
+	cwd <- getCurrentDirectory
+	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
 		hClose fromh
-                hPutStr toh $ join "\0" files
+                hPutStr toh $ join "\0" relfiles
                 hClose toh
                 exitSuccess
         hClose toh

From bcec418495f1330ee502b59ba485a69a9b5e3088 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:49:19 -0400
Subject: [PATCH 2324/8313] releasing version 3.20111011

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 5645eb043a..450adea75b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110929
+Version: 3.20111011
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 2b26b95226566773f8019826671d4955600dbffe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:49:25 -0400
Subject: [PATCH 2325/8313] add news item for git-annex 3.20111011

---
 doc/news/version_3.20110819.mdwn |  9 ---------
 doc/news/version_3.20111011.mdwn | 30 ++++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110819.mdwn
 create mode 100644 doc/news/version_3.20111011.mdwn

diff --git a/doc/news/version_3.20110819.mdwn b/doc/news/version_3.20110819.mdwn
deleted file mode 100644
index bbc6abdbc7..0000000000
--- a/doc/news/version_3.20110819.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110819 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Now "git annex init" only has to be run once, when a git repository
-     is first being created. Clones will automatically notice that git-annex
-     is in use and automatically perform a basic initalization. It's
-     still recommended to run "git annex init" in any clones, to describe them.
-   * Added annex-cost-command configuration, which can be used to vary the
-     cost of a remote based on the output of a shell command.
-   * Fix broken upgrade from V1 repository. Closes: #[638584](http://bugs.debian.org/638584)"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111011.mdwn b/doc/news/version_3.20111011.mdwn
new file mode 100644
index 0000000000..0346cbca82
--- /dev/null
+++ b/doc/news/version_3.20111011.mdwn
@@ -0,0 +1,30 @@
+git-annex 3.20111011 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * This version of git-annex only works with git 1.7.7 and newer.
+     The breakage with old versions is subtle, and affects the
+     annex.numcopies settings in .gitattributes, so be sure to upgrade git
+     to 1.7.7. (Debian package now depends on that version.)
+   * Don't pass absolute paths to git show-attr, as it started following
+     symlinks when that's done in 1.7.7. Instead, use relative paths,
+     which show-attr only handles 100% correctly in 1.7.7. Closes: #[645046](http://bugs.debian.org/645046)
+   * Fix referring to remotes by uuid.
+   * New or changed repository descriptions in uuid.log now have a timestamp,
+     which is used to ensure the newest description is used when the uuid.log
+     has been merged.
+   * Note that older versions of git-annex will display the timestamp as part
+     of the repository description, which is ugly but otherwise harmless.
+   * Add timestamps to trust.log and remote.log too.
+   * git-annex-shell: Added the --uuid option.
+   * git-annex now asks git-annex-shell to verify that it's operating in
+     the expected repository.
+   * Note that this git-annex will not interoperate with remotes using
+     older versions of git-annex-shell.
+   * Now supports git's insteadOf configuration, to modify the url
+     used to access a remote. Note that pushInsteadOf is not used;
+     that and pushurl are reserved for actual git pushes. Closes: #[644278](http://bugs.debian.org/644278)
+   * status: List all known repositories.
+   * When displaying a list of repositories, show git remote names
+     in addition to their descriptions.
+   * Add locking to avoid races when changing the git-annex branch.
+   * Various speed improvements gained by using ByteStrings.
+   * Contain the zombie hordes."""]]
\ No newline at end of file

From fa5c016585b61e84c9ef7e7f6e6814693599a66a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 11 Oct 2011 23:49:58 -0400
Subject: [PATCH 2326/8313] add comment about relative/absolute filenames

---
 Git.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Git.hs b/Git.hs
index fb998b1b0e..044fc08aea 100644
--- a/Git.hs
+++ b/Git.hs
@@ -547,6 +547,9 @@ configMap = config
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
 checkAttr repo attr files = do
+	-- git check-attr needs relative filenames input; it will choke
+	-- on some absolute filenames. This also means it will output
+	-- all relative filenames.
 	cwd <- getCurrentDirectory
 	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)

From 82d127de99767a8a6387cb923eb964fa51474aa6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 12 Oct 2011 00:25:05 -0400
Subject: [PATCH 2327/8313] add git version check to configure

Rather ugly dotted version comparison method, but it does work.
---
 configure.hs    | 24 ++++++++++++++++++++++++
 git-annex.cabal |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/configure.hs b/configure.hs
index b68fa12dbc..1d1c023356 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,12 +2,16 @@
 
 import System.Directory
 import Data.List
+import Data.String.Utils
+import System.Cmd.Utils
 
 import Build.TestConfig
 
 tests :: [TestCase]
 tests =
 	[ TestCase "version" getVersion
+	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
+	, TestCase "git version" checkGitVersion
 	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
@@ -53,6 +57,26 @@ getVersionString = do
 	where
 		middle = drop 1 . init
 
+{- Checks for a new enough version of git. -}
+checkGitVersion :: Test
+checkGitVersion = do
+	(_, s) <- pipeFrom "git" ["--version"]
+	let version = last $ words $ head $ lines s
+	if dotted version < dotted need
+		then error $ "git version " ++ version ++ " too old; need " ++ need
+		else return $ Config "gitversion" (StringConfig version)
+	where
+		-- for git-check-attr behavior change
+		need = "1.7.7"
+		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
+		extend n l = l ++ take (n - length l) (repeat 0)
+		mult _ [] = []
+		mult n (x:xs) = (n*x) : (mult (n*100) xs)
+		readi :: String -> Integer
+		readi s = case reads s of
+			((x,_):_) -> x
+			_ -> 0
+
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
 cabalSetup = do
diff --git a/git-annex.cabal b/git-annex.cabal
index 5645eb043a..450adea75b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20110929
+Version: 3.20111011
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 205a5b2aaa76fac31d23f9b5a47180400a3008bf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 12 Oct 2011 00:29:49 -0400
Subject: [PATCH 2328/8313] typo

---
 debian/NEWS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/NEWS b/debian/NEWS
index ad4e946e6c..f807d05255 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -4,7 +4,7 @@ git-annex (3.20110702) unstable; urgency=low
 
  -- Joey Hess   Fri, 01 Jul 2011 15:40:51 -0400
 
-git-annex (3.20110624) exerimental; urgency=low
+git-annex (3.20110624) experimental; urgency=low
 
   There has been another change to the git-annex data store.
   Use `git annex upgrade` to migrate your repositories to the new

From 88e9bba97ef5e321407519f39bd81d992e19e708 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Thu, 13 Oct 2011 14:52:43 +0000
Subject: [PATCH 2329/8313]

---
 doc/forum/git_tag_missing_for_3.20111011.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/git_tag_missing_for_3.20111011.mdwn

diff --git a/doc/forum/git_tag_missing_for_3.20111011.mdwn b/doc/forum/git_tag_missing_for_3.20111011.mdwn
new file mode 100644
index 0000000000..781d0c91a0
--- /dev/null
+++ b/doc/forum/git_tag_missing_for_3.20111011.mdwn
@@ -0,0 +1 @@
+Well, the subject pretty much says it all :)

From 3e07780bf881988eed491e1451b95cab86b7f552 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 13 Oct 2011 15:36:59 +0000
Subject: [PATCH 2330/8313] Added a comment: fixed that

---
 .../comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment

diff --git a/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment b/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment
new file mode 100644
index 0000000000..87cda998bf
--- /dev/null
+++ b/doc/forum/git_tag_missing_for_3.20111011/comment_1_7a53bf273f3078ab3351369ef2b5f2a6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="fixed that"
+ date="2011-10-13T15:36:59Z"
+ content="""
+:)
+"""]]

From 9fa92141064a7682e1559bfa91a360c1ad5cb3dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 14 Oct 2011 18:17:46 -0400
Subject: [PATCH 2331/8313] A remote can have a annexUrl configured, that is
 used by git-annex instead of its usual url. (Similar to pushUrl.)

---
 Git.hs                   | 22 +++++++++++++++++-----
 Remote/Git.hs            | 13 ++++++++++++-
 Remote/Helper/Special.hs |  2 +-
 Remote/Web.hs            |  2 +-
 debian/changelog         |  7 +++++++
 doc/git-annex.mdwn       |  6 ++++++
 6 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/Git.hs b/Git.hs
index 044fc08aea..836e063c2a 100644
--- a/Git.hs
+++ b/Git.hs
@@ -48,8 +48,10 @@ module Git (
 	attributes,
 	remotes,
 	remotesAdd,
+	genRemote,
 	repoRemoteName,
 	repoRemoteNameSet,
+	repoRemoteNameFromKey,
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
@@ -185,10 +187,14 @@ repoRemoteName :: Repo -> Maybe String
 repoRemoteName Repo { remoteName = Just name } = Just name
 repoRemoteName _ = Nothing
 
+{- Sets the name of a remote. -}
+repoRemoteNameSet :: Repo -> String -> Repo
+repoRemoteNameSet r n = r { remoteName = Just n }
+
 {- Sets the name of a remote based on the git config key, such as
    "remote.foo.url". -}
-repoRemoteNameSet :: Repo -> String -> Repo
-repoRemoteNameSet r k = r { remoteName = Just basename }
+repoRemoteNameFromKey :: Repo -> String -> Repo
+repoRemoteNameFromKey r k = repoRemoteNameSet r basename
 	where
 		basename = join "." $ reverse $ drop 1 $
 				reverse $ drop 1 $ split "." k
@@ -501,9 +507,15 @@ configRemotes repo = mapM construct remotepairs
 		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
 		construct (k,v) = do
-			r <- gen $ calcloc v
-			return $ repoRemoteNameSet r k
-		gen v
+			r <- genRemote repo v
+			return $ repoRemoteNameFromKey r k
+
+{- Generates one of a repo's remotes using a given location (ie, an url). -}
+genRemote :: Repo -> String -> IO Repo
+genRemote repo = gen . calcloc
+	where
+		filterconfig f = filter f $ M.toList $ config repo
+		gen v	
 			| scpstyle v = repoFromUrl $ scptourl v
 			| isURI v = repoFromUrl v
 			| otherwise = repoFromRemotePath v repo
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 183fcd8548..e9919e636c 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -34,7 +34,18 @@ remote = RemoteType {
 list :: Annex [Git.Repo]
 list = do
 	g <- gitRepo
-	return $ Git.remotes g
+	let c = Git.configMap g
+	mapM (tweakurl c) $ Git.remotes g
+	where
+		annexurl n = "remote." ++ n ++ ".annexurl"
+		tweakurl c r = do
+			let n = fromJust $ Git.repoRemoteName r
+			case M.lookup (annexurl n) c of
+				Nothing -> return r
+				Just url -> do
+					g <- gitRepo
+					r' <- liftIO $ Git.genRemote g url
+					return $ Git.repoRemoteNameSet r' n
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 1fbd7b19ed..5f66e8cd9f 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -24,7 +24,7 @@ findSpecialRemotes s = do
 	return $ map construct $ remotepairs g
 	where
 		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
-		construct (k,_) = Git.repoRemoteNameSet Git.repoFromUnknown k
+		construct (k,_) = Git.repoRemoteNameFromKey Git.repoFromUnknown k
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 30a1ff0086..ed9c94909e 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -33,7 +33,7 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet Git.repoFromUnknown "remote.web.dummy"]
+list = return [Git.repoRemoteNameSet Git.repoFromUnknown "web"]
 
 -- Dummy uuid for the whole web. Do not alter.
 webUUID :: UUID
diff --git a/debian/changelog b/debian/changelog
index c4e53e1795..6e3450a926 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111012) UNRELEASED; urgency=low
+
+  * A remote can have a annexUrl configured, that is used by git-annex
+    instead of its usual url. (Similar to pushUrl.)
+
+ -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
+
 git-annex (3.20111011) unstable; urgency=low
 
   * This version of git-annex only works with git 1.7.7 and newer.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index db19e9f7a2..eefedbfbf9 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -501,6 +501,12 @@ Here are all the supported configuration settings.
   Or, it could be used if the network connection between two
   repositories is too slow to be used normally.
 
+* `remote..annexUrl`
+
+  Can be used to specify a different url than the regular `remote..url`
+  for git-annex to use when talking with the remote. Similar to the `pushUrl`
+  used by git-push.
+
 * `remote..annex-uuid`
 
   git-annex caches UUIDs of remote repositories here.

From ae2b1308a62981e14a03275e5d6ea65a2bc7d098 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 14 Oct 2011 18:23:17 -0400
Subject: [PATCH 2332/8313] reorg

---
 doc/git-annex.mdwn | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index eefedbfbf9..1b8464b314 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -477,6 +477,20 @@ Here are all the supported configuration settings.
   Space-separated list of names of the key-value backends to use. 
   The first listed is used to store new files by default. 
 
+* `annex.diskreserve`
+
+  Amount of disk space to reserve. Disk space is checked when transferring
+  content to avoid running out, and additional free space can be reserved
+  via this option, to make space for more important content (such as git
+  commit logs). Can be specified with any commonly used units, for example,
+  "0.5 gb" or "100 KiloBytes"
+
+  The default reserve is 1 megabyte.
+
+* `annex.version`
+
+  Automatically maintained, and used to automate upgrades between versions.
+
 * `remote..annex-cost`
 
   When determining which repository to
@@ -532,20 +546,6 @@ Here are all the supported configuration settings.
   Default ssh, rsync, and bup options to use if a remote does not have
   specific options.
 
-* `annex.diskreserve`
-
-  Amount of disk space to reserve. Disk space is checked when transferring
-  content to avoid running out, and additional free space can be reserved
-  via this option, to make space for more important content (such as git
-  commit logs). Can be specified with any commonly used units, for example,
-  "0.5 gb" or "100 KiloBytes"
-
-  The default reserve is 1 megabyte.
-
-* `annex.version`
-
-  Automatically maintained, and used to automate upgrades between versions.
-
 * `remote..buprepo`
 
   Used by bup special remotes, this configures

From 872057303e14dd6623772de2d8d85b511e21214c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 01:37:55 -0400
Subject: [PATCH 2333/8313] tweak

---
 Git.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Git.hs b/Git.hs
index 836e063c2a..b426d5c5fe 100644
--- a/Git.hs
+++ b/Git.hs
@@ -338,8 +338,9 @@ urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
 
 {- The full authority portion an URL repo. (ie, "user@host:port") -}
 urlAuthority :: Repo -> String
-urlAuthority r = flip urlAuthPart r $ \a ->
-	uriUserInfo a ++ uriRegName' a ++ uriPort a
+urlAuthority = urlAuthPart combine
+	where
+		combine a = uriUserInfo a ++ uriRegName' a ++ uriPort a
 
 {- Applies a function to extract part of the uriAuthority of an URL repo. -}
 urlAuthPart :: (URIAuth -> a) -> Repo -> a

From c867ae842ae15e03eb7d36704a33ce62f2a8fe16 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 13:13:57 -0400
Subject: [PATCH 2334/8313] add

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/todo/gitolite_and_gitosis_support.mdwn

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
new file mode 100644
index 0000000000..0131cdc076
--- /dev/null
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -0,0 +1,14 @@
+gitosis and gitolite should support git-annex being used to send/receive
+files from the repositories they manage. Users with read-only access
+could only get files, while users with write access could also put and drop
+files.
+
+Doing this right requires modifying both programs, to add [[git-annex-shell]]
+to the list of things they can run, and only allow through appropriate
+git-annex-shell subcommands to read-only users.
+
+I have posted an RFC for modifying gitolite to the
+[gitolite mailing list](http://groups.google.com/group/gitolite?lnk=srg).
+
+As I don't write python, someone else is needed to work on gitosis.
+--[[Joey]] 

From bae3008d17d492b174db552f64c0cdfaf1804bdd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 13:17:00 -0400
Subject: [PATCH 2335/8313] add a copy of my mailing list post

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
index 0131cdc076..daa54854d3 100644
--- a/doc/todo/gitolite_and_gitosis_support.mdwn
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -10,5 +10,25 @@ git-annex-shell subcommands to read-only users.
 I have posted an RFC for modifying gitolite to the
 [gitolite mailing list](http://groups.google.com/group/gitolite?lnk=srg).
 
+> I have not developed a patch yet, but all that git-annex needs is a way
+> to ssh to the server and run the git-annex-shell command there.
+> git-annex-shell is very similar to git-shell. So, one way to enable
+> it is simply to set GL_ADC_PATH to a directory containing git-annex-shell.
+> 
+> But, that's not optimal, since git-annex-shell will send off receive-pack
+> commands to git, which would bypass gitolite's permissions checking.
+> Also, it makes sense to limit readonly users to only download, not
+> upload/delete files from git-annex. Instead, I suggest adding something
+> like this to gitolite's config:
+ 
+	# If set, users with W access can write file contents into the git-annex,
+	# and users with R access can read file contents from the git-annex.
+	$GL_GIT_ANNEX = 0;
+
+> If this makes sense, I'm sure I can put a patch together for your
+> review. It would involve modifying gl-auth-command so it knows how
+> to run git-annex-shell, and how to parse out the "verb" from a
+> git-annex-shell command line, and modifying R_COMMANDS and W_COMMANDS.
+
 As I don't write python, someone else is needed to work on gitosis.
 --[[Joey]] 

From 279150ccd5ad937a44cbff798ab7bb118ad1dbee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 13:30:04 -0400
Subject: [PATCH 2336/8313] update

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
index daa54854d3..12e26243e4 100644
--- a/doc/todo/gitolite_and_gitosis_support.mdwn
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -32,3 +32,19 @@ I have posted an RFC for modifying gitolite to the
 
 As I don't write python, someone else is needed to work on gitosis.
 --[[Joey]] 
+
+## readonly commands
+
+* git-annex-shell configlist $directory
+* git-annex-shell inannex $directory [$key ...]
+* git-annex-shell sendkey $directory $key
+
+## read-write commands
+
+* git-annex-shell dropkey $directory [$key ...]
+* git-annex-shell recvkey $directory $key
+
+## other git-annex-shell parameters
+
+All parameters like --uuid=foo and --force are safe and need to be allowed
+through.

From 1a29b5b52eec641a5456d7c8dc24356c90107bc0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 16:21:08 -0400
Subject: [PATCH 2337/8313] reorganize log modules

no code changes
---
 Annex/Content.hs                   |  6 +++---
 Command.hs                         |  4 ++--
 Command/Add.hs                     |  2 +-
 Command/AddUrl.hs                  |  2 +-
 Command/ConfigList.hs              |  2 +-
 Command/Describe.hs                |  2 +-
 Command/Drop.hs                    |  4 ++--
 Command/DropKey.hs                 |  2 +-
 Command/Fsck.hs                    |  6 +++---
 Command/Init.hs                    |  2 +-
 Command/InitRemote.hs              | 12 +++++------
 Command/Map.hs                     |  4 ++--
 Command/Move.hs                    |  4 ++--
 Command/Semitrust.hs               |  4 ++--
 Command/SetKey.hs                  |  2 +-
 Command/Status.hs                  |  2 +-
 Command/Trust.hs                   |  4 ++--
 Command/Unannex.hs                 |  2 +-
 Command/Untrust.hs                 |  4 ++--
 Command/Unused.hs                  |  2 +-
 Command/Whereis.hs                 |  4 ++--
 Git.hs                             |  4 ++--
 Init.hs                            |  2 +-
 Limit.hs                           |  2 +-
 LocationLog.hs => Logs/Location.hs |  6 +++---
 PresenceLog.hs => Logs/Presence.hs |  2 +-
 RemoteLog.hs => Logs/Remote.hs     |  6 +++---
 Trust.hs => Logs/Trust.hs          |  6 +++---
 UUID.hs => Logs/UUID.hs            |  4 ++--
 UUIDLog.hs => Logs/UUIDBased.hs    |  2 +-
 Remote.hs                          | 12 +++++------
 Remote/Bup.hs                      |  2 +-
 Remote/Directory.hs                |  2 +-
 Remote/Git.hs                      |  2 +-
 Remote/Helper/Special.hs           |  2 +-
 Remote/Hook.hs                     |  2 +-
 Remote/Rsync.hs                    |  2 +-
 Remote/Web.hs                      |  6 +++---
 Upgrade/V1.hs                      |  2 +-
 Upgrade/V2.hs                      |  2 +-
 Utility/Ssh.hs                     |  2 +-
 git-annex-shell.hs                 |  2 +-
 git-annex.cabal                    |  2 +-
 test.hs                            | 34 +++++++++++++++---------------
 44 files changed, 92 insertions(+), 92 deletions(-)
 rename LocationLog.hs => Logs/Location.hs (96%)
 rename PresenceLog.hs => Logs/Presence.hs (99%)
 rename RemoteLog.hs => Logs/Remote.hs (97%)
 rename Trust.hs => Logs/Trust.hs (96%)
 rename UUID.hs => Logs/UUID.hs (98%)
 rename UUIDLog.hs => Logs/UUIDBased.hs (99%)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 3827154a67..9cf7ea8f21 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -22,8 +22,8 @@ module Annex.Content (
 ) where
 
 import Common.Annex
-import LocationLog
-import UUID
+import Logs.Location
+import Logs.UUID
 import qualified Git
 import qualified Annex
 import qualified Annex.Queue
@@ -52,7 +52,7 @@ calcGitLink file key = do
 	where
 		whoops = error $ "unable to normalize " ++ file
 
-{- Updates the LocationLog when a key's presence changes in the current
+{- Updates the Logs.Location when a key's presence changes in the current
  - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
diff --git a/Command.hs b/Command.hs
index 54dc9603fa..f282791fb3 100644
--- a/Command.hs
+++ b/Command.hs
@@ -13,8 +13,8 @@ import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 import Types.Key
-import Trust
-import LocationLog
+import Logs.Trust
+import Logs.Location
 import Config
 import Backend
 import Limit
diff --git a/Command/Add.hs b/Command/Add.hs
index e6e7a7c770..bfddd72ee7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -13,7 +13,7 @@ import Command
 import qualified Annex
 import qualified Annex.Queue
 import qualified Backend
-import LocationLog
+import Logs.Location
 import Annex.Content
 import Utility.Touch
 import Backend
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index c5417bf5bc..4447dee812 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -18,7 +18,7 @@ import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
 import Annex.Content
-import PresenceLog
+import Logs.Presence
 
 command :: [Command]
 command = [repoCommand "addurl" (paramRepeating paramUrl) seek
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 60f71eee6d..b50c759eee 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -9,7 +9,7 @@ module Command.ConfigList where
 
 import Common.Annex
 import Command
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "configlist" paramNothing seek
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 3368046393..65cd8d0bf5 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -10,7 +10,7 @@ module Command.Describe where
 import Common.Annex
 import Command
 import qualified Remote
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
diff --git a/Command/Drop.hs b/Command/Drop.hs
index fabacbb724..dc858fb29b 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -11,9 +11,9 @@ import Common.Annex
 import Command
 import qualified Remote
 import qualified Annex
-import LocationLog
+import Logs.Location
+import Logs.Trust
 import Annex.Content
-import Trust
 import Config
 
 command :: [Command]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 35ebfc219b..fde6ce02ea 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -10,7 +10,7 @@ module Command.DropKey where
 import Common.Annex
 import Command
 import qualified Annex
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 command :: [Command]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 16e834fbf1..632570b110 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -12,10 +12,10 @@ import Command
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
-import UUID
 import Annex.Content
-import LocationLog
-import Trust
+import Logs.Location
+import Logs.Trust
+import Logs.UUID
 import Utility.DataUnits
 import Utility.FileMode
 import Config
diff --git a/Command/Init.hs b/Command/Init.hs
index b9dffb5cd9..dcc6bfe6b2 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -9,7 +9,7 @@ module Command.Init where
 
 import Common.Annex
 import Command
-import UUID
+import Logs.UUID
 import Init
 	
 command :: [Command]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 918604f852..240528b879 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -12,9 +12,9 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Remote
-import qualified RemoteLog
+import qualified Logs.Remote
 import qualified Types.Remote as R
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "initremote"
@@ -38,7 +38,7 @@ start ws = do
 
 	where
 		name = head ws
-		config = RemoteLog.keyValToConfig $ tail ws
+		config = Logs.Remote.keyValToConfig $ tail ws
 		needname = do
 			let err s = error $ "Specify a name for the remote. " ++ s
 			names <- remoteNames
@@ -54,13 +54,13 @@ perform t u c = do
 
 cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
 cleanup u c = do
-	RemoteLog.configSet u c
+	Logs.Remote.configSet u c
         return True
 
 {- Look up existing remote's UUID and config by name, or generate a new one -}
 findByName :: String -> Annex (UUID, R.RemoteConfig)
 findByName name = do
-	m <- RemoteLog.readRemoteLog
+	m <- Logs.Remote.readRemoteLog
 	maybe generate return $ findByName' name m
 	where
 		generate = do
@@ -79,7 +79,7 @@ findByName' n m = if null matches then Nothing else Just $ head matches
 
 remoteNames :: Annex [String]
 remoteNames = do
-	m <- RemoteLog.readRemoteLog
+	m <- Logs.Remote.readRemoteLog
 	return $ mapMaybe (M.lookup nameKey . snd) $ M.toList m
 
 {- find the specified remote type -}
diff --git a/Command/Map.hs b/Command/Map.hs
index 1155c4a6ef..5cbf51b27f 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,8 +13,8 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
-import UUID
-import Trust
+import Logs.UUID
+import Logs.Trust
 import Utility.Ssh
 import qualified Utility.Dot as Dot
 
diff --git a/Command/Move.hs b/Command/Move.hs
index 52eb49da14..62f38224ca 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -11,10 +11,10 @@ import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
-import LocationLog
+import Logs.Location
 import Annex.Content
 import qualified Remote
-import UUID
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 53b29c98ac..e13785a38f 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -10,8 +10,8 @@ module Command.Semitrust where
 import Common.Annex
 import Command
 import qualified Remote
-import UUID
-import Trust
+import Logs.UUID
+import Logs.Trust
 
 command :: [Command]
 command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 28bc9a48d6..b707e0b918 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -9,7 +9,7 @@ module Command.SetKey where
 
 import Common.Annex
 import Command
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 command :: [Command]
diff --git a/Command/Status.hs b/Command/Status.hs
index 37e13f0d8e..70282b79ee 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -23,7 +23,7 @@ import Utility.DataUnits
 import Annex.Content
 import Types.Key
 import Backend
-import UUID
+import Logs.UUID
 import Remote
 
 -- a named computation that produces a statistic
diff --git a/Command/Trust.hs b/Command/Trust.hs
index bc655e3f61..fb7f47ec0e 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -10,8 +10,8 @@ module Command.Trust where
 import Common.Annex
 import Command
 import qualified Remote
-import Trust
-import UUID
+import Logs.Trust
+import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "trust" (paramRepeating paramRemote) seek
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d0cef76781..083984d0c5 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -13,7 +13,7 @@ import qualified Command.Drop
 import qualified Annex
 import qualified Annex.Queue
 import Utility.FileMode
-import LocationLog
+import Logs.Location
 import Annex.Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index bcde0e0a06..6f2b602038 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -10,8 +10,8 @@ module Command.Untrust where
 import Common.Annex
 import Command
 import qualified Remote
-import UUID
-import Trust
+import Logs.UUID
+import Logs.Trust
 
 command :: [Command]
 command = [repoCommand "untrust" (paramRepeating paramRemote) seek
diff --git a/Command/Unused.hs b/Command/Unused.hs
index abf5a5361a..874b0ca061 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,7 +16,7 @@ import Common.Annex
 import Command
 import Annex.Content
 import Utility.FileMode
-import LocationLog
+import Logs.Location
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 0eeb174142..b1646ae69a 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -8,10 +8,10 @@
 module Command.Whereis where
 
 import Common.Annex
-import LocationLog
+import Logs.Location
 import Command
 import Remote
-import Trust
+import Logs.Trust
 
 command :: [Command]
 command = [repoCommand "whereis" paramPaths seek
diff --git a/Git.hs b/Git.hs
index b426d5c5fe..b05c3b2d5e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -338,9 +338,9 @@ urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
 
 {- The full authority portion an URL repo. (ie, "user@host:port") -}
 urlAuthority :: Repo -> String
-urlAuthority = urlAuthPart combine
+urlAuthority = urlAuthPart assemble
 	where
-		combine a = uriUserInfo a ++ uriRegName' a ++ uriPort a
+		assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a
 
 {- Applies a function to extract part of the uriAuthority of an URL repo. -}
 urlAuthPart :: (URIAuth -> a) -> Repo -> a
diff --git a/Init.hs b/Init.hs
index 509cbca15d..00a3d12fab 100644
--- a/Init.hs
+++ b/Init.hs
@@ -15,7 +15,7 @@ import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
-import UUID
+import Logs.UUID
 
 initialize :: Annex ()
 initialize = do
diff --git a/Limit.hs b/Limit.hs
index 8dd88e72b8..490577e80f 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -15,7 +15,7 @@ import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
diff --git a/LocationLog.hs b/Logs/Location.hs
similarity index 96%
rename from LocationLog.hs
rename to Logs/Location.hs
index 5cbdbb28a1..4e8b2b5357 100644
--- a/LocationLog.hs
+++ b/Logs/Location.hs
@@ -11,7 +11,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module LocationLog (
+module Logs.Location (
 	LogStatus(..),
 	logChange,
 	readLog,
@@ -24,8 +24,8 @@ module LocationLog (
 import Common.Annex
 import qualified Git
 import qualified Annex.Branch
-import UUID
-import PresenceLog
+import Logs.UUID
+import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
diff --git a/PresenceLog.hs b/Logs/Presence.hs
similarity index 99%
rename from PresenceLog.hs
rename to Logs/Presence.hs
index 4e4960f0a6..7211eba039 100644
--- a/PresenceLog.hs
+++ b/Logs/Presence.hs
@@ -11,7 +11,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module PresenceLog (
+module Logs.Presence (
 	LogStatus(..),
 	addLog,
 	readLog,
diff --git a/RemoteLog.hs b/Logs/Remote.hs
similarity index 97%
rename from RemoteLog.hs
rename to Logs/Remote.hs
index d49635b930..47c2d7472b 100644
--- a/RemoteLog.hs
+++ b/Logs/Remote.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module RemoteLog (
+module Logs.Remote (
 	readRemoteLog,
 	configSet,
 	keyValToConfig,
@@ -21,8 +21,8 @@ import Data.Char
 import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
-import UUID
-import UUIDLog
+import Logs.UUID
+import Logs.UUIDBased
 
 {- Filename of remote.log. -}
 remoteLog :: FilePath
diff --git a/Trust.hs b/Logs/Trust.hs
similarity index 96%
rename from Trust.hs
rename to Logs/Trust.hs
index 2971256a45..6966ffdd6e 100644
--- a/Trust.hs
+++ b/Logs/Trust.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Trust (
+module Logs.Trust (
 	TrustLevel(..),
 	trustGet,
 	trustSet,
@@ -20,8 +20,8 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 
-import UUID
-import UUIDLog
+import Logs.UUID
+import Logs.UUIDBased
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
diff --git a/UUID.hs b/Logs/UUID.hs
similarity index 98%
rename from UUID.hs
rename to Logs/UUID.hs
index 63ce87f03e..baf6650015 100644
--- a/UUID.hs
+++ b/Logs/UUID.hs
@@ -13,7 +13,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module UUID (
+module Logs.UUID (
 	UUID,
 	getUUID,
 	getRepoUUID,
@@ -33,7 +33,7 @@ import qualified Annex.Branch
 import Types.UUID
 import qualified Build.SysConfig as SysConfig
 import Config
-import UUIDLog
+import Logs.UUIDBased
 
 configkey :: String
 configkey = "annex.uuid"
diff --git a/UUIDLog.hs b/Logs/UUIDBased.hs
similarity index 99%
rename from UUIDLog.hs
rename to Logs/UUIDBased.hs
index d6eb8fbbbe..46fa80be0d 100644
--- a/UUIDLog.hs
+++ b/Logs/UUIDBased.hs
@@ -12,7 +12,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module UUIDLog (
+module Logs.UUIDBased (
 	Log,
 	LogEntry(..),
 	parseLog,
diff --git a/Remote.hs b/Remote.hs
index b1305b9e0f..87c4e23b26 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -34,12 +34,12 @@ import Text.JSON.Generic
 
 import Common.Annex
 import Types.Remote
-import UUID
 import qualified Annex
 import Config
-import Trust
-import LocationLog
-import RemoteLog
+import Logs.UUID
+import Logs.Trust
+import Logs.Location
+import Logs.Remote
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -163,12 +163,12 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+{- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote Annex]
 keyPossibilities key = fst <$> keyPossibilities' False key
 
-{- Cost ordered lists of remotes that the LocationLog indicate may have a key.
+{- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 986be7fc40..dfc9116882 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,7 +15,7 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 03b17456a4..270c78f838 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -15,7 +15,7 @@ import Common.Annex
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Utility.FileMode
 import Remote.Helper.Special
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e9919e636c..42d1b5858d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -17,7 +17,7 @@ import Utility.Ssh
 import Types.Remote
 import qualified Git
 import qualified Annex
-import UUID
+import Logs.UUID
 import qualified Annex.Content
 import qualified Utility.Url as Url
 import Config
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 5f66e8cd9f..5603d13aa2 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,7 +12,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index cc8ed69ab5..2c6b50c7da 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -15,7 +15,7 @@ import System.Exit
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 3474f8b251..3216567479 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -13,7 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/Web.hs b/Remote/Web.hs
index ed9c94909e..51373a49c3 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -13,10 +13,10 @@ module Remote.Web (
 import Common.Annex
 import Types.Remote
 import qualified Git
-import UUID
 import Config
-import PresenceLog
-import LocationLog
+import Logs.Presence
+import Logs.Location
+import Logs.UUID
 import qualified Utility.Url as Url
 
 type URLString = String
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 132e694ebc..6c6531acea 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -14,7 +14,7 @@ import Data.Char
 import Common.Annex
 import Types.Key
 import Annex.Content
-import PresenceLog
+import Logs.Presence
 import qualified Annex.Queue
 import qualified Git
 import qualified Git.LsFiles as LsFiles
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 39c7b15611..d6334ed654 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -10,7 +10,7 @@ module Upgrade.V2 where
 import Common.Annex
 import qualified Git
 import qualified Annex.Branch
-import LocationLog
+import Logs.Location
 import Annex.Content
 
 olddir :: Git.Repo -> FilePath
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index a0e52507d6..1847ff2440 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -13,7 +13,7 @@ import qualified Git
 import Utility.SafeCommand
 import Types
 import Config
-import UUID
+import Logs.UUID
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index a4d8dd65b6..f19abe6c3f 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -13,7 +13,7 @@ import qualified Git
 import CmdLine
 import Command
 import Options
-import UUID
+import Logs.UUID
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
diff --git a/git-annex.cabal b/git-annex.cabal
index 450adea75b..e9ec089677 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111011
+Version: 3.20111012
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/test.hs b/test.hs
index 1c511b4d4b..6a6ad441e5 100644
--- a/test.hs
+++ b/test.hs
@@ -29,12 +29,12 @@ import qualified Locations
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
-import qualified LocationLog
-import qualified UUID
-import qualified UUIDLog
-import qualified Trust
+import qualified Logs.Location
+import qualified Logs.UUID
+import qualified Logs.UUIDBased
+import qualified Logs.Trust
+import qualified Logs.Remote
 import qualified Remote
-import qualified RemoteLog
 import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Config
@@ -73,14 +73,14 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape
 	, qctest "prop_idempotent_shellEscape_multiword" Utility.SafeCommand.prop_idempotent_shellEscape_multiword
-	, qctest "prop_idempotent_configEscape" RemoteLog.prop_idempotent_configEscape
+	, qctest "prop_idempotent_configEscape" Logs.Remote.prop_idempotent_configEscape
 	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
 
 	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
-	, qctest "prop_TimeStamp_sane" UUIDLog.prop_TimeStamp_sane
-	, qctest "prop_addLog_sane" UUIDLog.prop_addLog_sane
+	, qctest "prop_TimeStamp_sane" Logs.UUIDBased.prop_TimeStamp_sane
+	, qctest "prop_addLog_sane" Logs.UUIDBased.prop_addLog_sane
 	]
 
 blackbox :: Test
@@ -341,22 +341,22 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 test_trust :: Test
 test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 	git_annex "trust" ["-q", repo] @? "trust failed"
-	trustcheck Trust.Trusted "trusted 1"
+	trustcheck Logs.Trust.Trusted "trusted 1"
 	git_annex "trust" ["-q", repo] @? "trust of trusted failed"
-	trustcheck Trust.Trusted "trusted 2"
+	trustcheck Logs.Trust.Trusted "trusted 2"
 	git_annex "untrust" ["-q", repo] @? "untrust failed"
-	trustcheck Trust.UnTrusted "untrusted 1"
+	trustcheck Logs.Trust.UnTrusted "untrusted 1"
 	git_annex "untrust" ["-q", repo] @? "untrust of untrusted failed"
-	trustcheck Trust.UnTrusted "untrusted 2"
+	trustcheck Logs.Trust.UnTrusted "untrusted 2"
 	git_annex "semitrust" ["-q", repo] @? "semitrust failed"
-	trustcheck Trust.SemiTrusted "semitrusted 1"
+	trustcheck Logs.Trust.SemiTrusted "semitrusted 1"
 	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
-	trustcheck Trust.SemiTrusted "semitrusted 2"
+	trustcheck Logs.Trust.SemiTrusted "semitrusted 2"
 	where
 		repo = "origin"
 		trustcheck expected msg = do
 			present <- annexeval $ do
-				l <- Trust.trustGet expected
+				l <- Logs.Trust.trustGet expected
 				u <- Remote.nameToUUID repo
 				return $ u `elem` l
 			assertBool msg present
@@ -609,11 +609,11 @@ checkdangling f = do
 
 checklocationlog :: FilePath -> Bool -> Assertion
 checklocationlog f expected = do
-	thisuuid <- annexeval UUID.getUUID
+	thisuuid <- annexeval Logs.UUID.getUUID
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do
-			uuids <- annexeval $ LocationLog.keyLocations k
+			uuids <- annexeval $ Logs.Location.keyLocations k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"

From b4015064e1492c6591f69499d8d745989999ba53 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 16:25:51 -0400
Subject: [PATCH 2338/8313] break web log handling into a separate module

---
 Logs/Web.hs   | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 Remote/Web.hs | 34 +---------------------------------
 2 files changed, 50 insertions(+), 33 deletions(-)
 create mode 100644 Logs/Web.hs

diff --git a/Logs/Web.hs b/Logs/Web.hs
new file mode 100644
index 0000000000..ff8fbdb6ba
--- /dev/null
+++ b/Logs/Web.hs
@@ -0,0 +1,49 @@
+{- Web url logs.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Logs.Web (
+	URLString,
+	webUUID,
+	setUrl,
+	getUrls
+) where
+
+import Common.Annex
+import Logs.Presence
+import Logs.Location
+import Logs.UUID
+
+type URLString = String
+
+-- Dummy uuid for the whole web. Do not alter.
+webUUID :: UUID
+webUUID = "00000000-0000-0000-0000-000000000001"
+
+{- The urls for a key are stored in remote/web/hash/key.log 
+ - in the git-annex branch. -}
+urlLog :: Key -> FilePath
+urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
+oldurlLog :: Key -> FilePath
+{- A bug used to store the urls elsewhere. -}
+oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
+
+getUrls :: Key -> Annex [URLString]
+getUrls key = do
+	us <- currentLog (urlLog key)
+	if null us
+		then currentLog (oldurlLog key)
+		else return us
+
+{- Records a change in an url for a key. -}
+setUrl :: Key -> URLString -> LogStatus -> Annex ()
+setUrl key url status = do
+	g <- gitRepo
+	addLog (urlLog key) =<< logNow status url
+
+	-- update location log to indicate that the web has the key, or not
+	us <- getUrls key
+	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 51373a49c3..e46937ba5f 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -14,13 +14,10 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import Config
-import Logs.Presence
-import Logs.Location
 import Logs.UUID
+import Logs.Web
 import qualified Utility.Url as Url
 
-type URLString = String
-
 remote :: RemoteType Annex
 remote = RemoteType {
 	typename = "web",
@@ -35,10 +32,6 @@ remote = RemoteType {
 list :: Annex [Git.Repo]
 list = return [Git.repoRemoteNameSet Git.repoFromUnknown "web"]
 
--- Dummy uuid for the whole web. Do not alter.
-webUUID :: UUID
-webUUID = "00000000-0000-0000-0000-000000000001"
-
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
 	return Remote {
@@ -54,31 +47,6 @@ gen r _ _ =
 		repo = r
 	}
 
-{- The urls for a key are stored in remote/web/hash/key.log 
- - in the git-annex branch. -}
-urlLog :: Key -> FilePath
-urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
-oldurlLog :: Key -> FilePath
-{- A bug used to store the urls elsewhere. -}
-oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
-
-getUrls :: Key -> Annex [URLString]
-getUrls key = do
-	us <- currentLog (urlLog key)
-	if null us
-		then currentLog (oldurlLog key)
-		else return us
-
-{- Records a change in an url for a key. -}
-setUrl :: Key -> URLString -> LogStatus -> Annex ()
-setUrl key url status = do
-	g <- gitRepo
-	addLog (urlLog key) =<< logNow status url
-
-	-- update location log to indicate that the web has the key, or not
-	us <- getUrls key
-	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
-
 downloadKey :: Key -> FilePath -> Annex Bool
 downloadKey key file = get =<< getUrls key
 	where

From ec169f84b1cc140b6d4c316fbd0e8407297d038a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 16:36:56 -0400
Subject: [PATCH 2339/8313] migrate: Copy url logs for keys when migrating.

---
 Command/AddUrl.hs  | 7 +++----
 Command/Migrate.hs | 9 +++++++++
 Logs/Web.hs        | 5 +++++
 Remote/Web.hs      | 5 +----
 debian/changelog   | 1 +
 5 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 4447dee812..2756af8807 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -13,12 +13,11 @@ import Common.Annex
 import Command
 import qualified Backend
 import qualified Utility.Url as Url
-import qualified Remote.Web
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
 import Annex.Content
-import Logs.Presence
+import Logs.Web
 
 command :: [Command]
 command = [repoCommand "addurl" (paramRepeating paramUrl) seek
@@ -58,14 +57,14 @@ download url file = do
 				Nothing -> stop
 				Just (key, _) -> do
 					moveAnnex key tmp
-					Remote.Web.setUrl key url InfoPresent
+					setUrlPresent key url
 					next $ Command.Add.cleanup file key True
 		else stop
 
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
 	let key = Backend.URL.fromUrl url
-	Remote.Web.setUrl key url InfoPresent
+	setUrlPresent key url
 	
 	next $ Command.Add.cleanup file key False
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 23ed6fd162..8167ac96eb 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -14,6 +14,7 @@ import qualified Types.Key
 import Annex.Content
 import qualified Command.Add
 import Backend
+import Logs.Web
 
 command :: [Command]
 command = [repoCommand "migrate" paramPaths seek
@@ -65,6 +66,14 @@ perform file oldkey newbackend = do
 				then do
 					-- Update symlink to use the new key.
 					liftIO $ removeFile file
+
+					-- If the old key had some
+					-- associated urls, record them for
+					-- the new key as well.
+					urls <- getUrls oldkey
+					when (not $ null urls) $
+						mapM_ (setUrlPresent newkey) urls
+
 					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
diff --git a/Logs/Web.hs b/Logs/Web.hs
index ff8fbdb6ba..4c8ef7fc00 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -9,6 +9,7 @@ module Logs.Web (
 	URLString,
 	webUUID,
 	setUrl,
+	setUrlPresent,
 	getUrls
 ) where
 
@@ -31,6 +32,7 @@ oldurlLog :: Key -> FilePath
 {- A bug used to store the urls elsewhere. -}
 oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
 
+{- Gets all urls that a key might be available from. -}
 getUrls :: Key -> Annex [URLString]
 getUrls key = do
 	us <- currentLog (urlLog key)
@@ -47,3 +49,6 @@ setUrl key url status = do
 	-- update location log to indicate that the web has the key, or not
 	us <- getUrls key
 	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
+
+setUrlPresent :: Key -> URLString -> Annex ()
+setUrlPresent key url = setUrl key url InfoPresent
diff --git a/Remote/Web.hs b/Remote/Web.hs
index e46937ba5f..21b9818465 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -5,10 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Web (
-	remote,
-	setUrl
-) where
+module Remote.Web (remote) where
 
 import Common.Annex
 import Types.Remote
diff --git a/debian/changelog b/debian/changelog
index 6e3450a926..ce1489e9d0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111012) UNRELEASED; urgency=low
 
   * A remote can have a annexUrl configured, that is used by git-annex
     instead of its usual url. (Similar to pushUrl.)
+  * migrate: Copy url logs for keys when migrating.
 
  -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
 

From ee9af605bc6c59365e6080e34d7c615978ba21d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 17:47:03 -0400
Subject: [PATCH 2340/8313] break out non-log stuff to separate module

---
 Annex/Content.hs         |  2 +-
 Annex/UUID.hs            | 69 ++++++++++++++++++++++++++++++++++++++++
 Command/ConfigList.hs    |  2 +-
 Command/Fsck.hs          |  2 +-
 Command/Init.hs          |  1 +
 Command/InitRemote.hs    |  2 +-
 Command/Map.hs           |  1 +
 Command/Move.hs          |  2 +-
 Command/Semitrust.hs     |  1 -
 Command/Trust.hs         |  1 -
 Command/Untrust.hs       |  1 -
 Init.hs                  |  2 +-
 Logs/Location.hs         |  1 -
 Logs/Remote.hs           |  1 -
 Logs/Trust.hs            |  2 --
 Logs/UUID.hs             | 54 -------------------------------
 Logs/Web.hs              |  1 -
 Remote.hs                |  1 +
 Remote/Bup.hs            |  1 -
 Remote/Directory.hs      |  1 -
 Remote/Git.hs            |  2 +-
 Remote/Helper/Special.hs |  1 -
 Remote/Hook.hs           |  1 -
 Remote/Web.hs            |  1 -
 Types.hs                 |  4 ++-
 Utility/Ssh.hs           |  2 +-
 git-annex-shell.hs       |  2 +-
 test.hs                  |  3 +-
 28 files changed, 86 insertions(+), 78 deletions(-)
 create mode 100644 Annex/UUID.hs

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 9cf7ea8f21..aafdf6f2e6 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,7 +23,7 @@ module Annex.Content (
 
 import Common.Annex
 import Logs.Location
-import Logs.UUID
+import Annex.UUID
 import qualified Git
 import qualified Annex
 import qualified Annex.Queue
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
new file mode 100644
index 0000000000..39e296e5b7
--- /dev/null
+++ b/Annex/UUID.hs
@@ -0,0 +1,69 @@
+{- git-annex uuids
+ -
+ - Each git repository used by git-annex has an annex.uuid setting that
+ - uniquely identifies that repository.
+ -
+ - UUIDs of remotes are cached in git config, using keys named
+ - remote..annex-uuid
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.UUID (
+	getUUID,
+	getRepoUUID,
+	getUncachedUUID,
+	prepUUID,
+	genUUID
+) where
+
+import Common.Annex
+import qualified Git
+import qualified Build.SysConfig as SysConfig
+import Config
+
+configkey :: String
+configkey = "annex.uuid"
+
+{- Generates a UUID. There is a library for this, but it's not packaged,
+ - so use the command line tool. -}
+genUUID :: IO UUID
+genUUID = pOpen ReadFromPipe command params hGetLine
+	where
+		command = SysConfig.uuid
+		params = if command == "uuid"
+			-- request a random uuid be generated
+			then ["-m"]
+			-- uuidgen generates random uuid by default
+			else []
+
+getUUID :: Annex UUID
+getUUID = getRepoUUID =<< gitRepo
+
+{- Looks up a repo's UUID. May return "" if none is known. -}
+getRepoUUID :: Git.Repo -> Annex UUID
+getRepoUUID r = do
+	g <- gitRepo
+
+	let c = cached g
+	let u = getUncachedUUID r
+	
+	if c /= u && u /= ""
+		then do
+			updatecache g u
+			return u
+		else return c
+	where
+		cached g = Git.configGet g cachekey ""
+		updatecache g u = when (g /= r) $ setConfig cachekey u
+		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
+
+getUncachedUUID :: Git.Repo -> UUID
+getUncachedUUID r = Git.configGet r configkey ""
+
+{- Make sure that the repo has an annex.uuid setting. -}
+prepUUID :: Annex ()
+prepUUID = whenM (null <$> getUUID) $
+	setConfig configkey =<< liftIO genUUID
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index b50c759eee..43315f67ce 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -9,7 +9,7 @@ module Command.ConfigList where
 
 import Common.Annex
 import Command
-import Logs.UUID
+import Annex.UUID
 
 command :: [Command]
 command = [repoCommand "configlist" paramNothing seek
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 632570b110..1c1687a002 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -15,7 +15,7 @@ import qualified Types.Key
 import Annex.Content
 import Logs.Location
 import Logs.Trust
-import Logs.UUID
+import Annex.UUID
 import Utility.DataUnits
 import Utility.FileMode
 import Config
diff --git a/Command/Init.hs b/Command/Init.hs
index dcc6bfe6b2..3dd4493295 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -9,6 +9,7 @@ module Command.Init where
 
 import Common.Annex
 import Command
+import Annex.UUID
 import Logs.UUID
 import Init
 	
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 240528b879..073ba72f90 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -14,7 +14,7 @@ import Command
 import qualified Remote
 import qualified Logs.Remote
 import qualified Types.Remote as R
-import Logs.UUID
+import Annex.UUID
 
 command :: [Command]
 command = [repoCommand "initremote"
diff --git a/Command/Map.hs b/Command/Map.hs
index 5cbf51b27f..18cb915e3b 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,6 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
+import Annex.UUID
 import Logs.UUID
 import Logs.Trust
 import Utility.Ssh
diff --git a/Command/Move.hs b/Command/Move.hs
index 62f38224ca..a816aacde5 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -14,7 +14,7 @@ import qualified Annex
 import Logs.Location
 import Annex.Content
 import qualified Remote
-import Logs.UUID
+import Annex.UUID
 
 command :: [Command]
 command = [repoCommand "move" paramPaths seek
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index e13785a38f..5d60977eb4 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -10,7 +10,6 @@ module Command.Semitrust where
 import Common.Annex
 import Command
 import qualified Remote
-import Logs.UUID
 import Logs.Trust
 
 command :: [Command]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index fb7f47ec0e..eeeadc9afe 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import qualified Remote
 import Logs.Trust
-import Logs.UUID
 
 command :: [Command]
 command = [repoCommand "trust" (paramRepeating paramRemote) seek
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 6f2b602038..f8bf498f24 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -10,7 +10,6 @@ module Command.Untrust where
 import Common.Annex
 import Command
 import qualified Remote
-import Logs.UUID
 import Logs.Trust
 
 command :: [Command]
diff --git a/Init.hs b/Init.hs
index 00a3d12fab..43840a1081 100644
--- a/Init.hs
+++ b/Init.hs
@@ -15,7 +15,7 @@ import Common.Annex
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
-import Logs.UUID
+import Annex.UUID
 
 initialize :: Annex ()
 initialize = do
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 4e8b2b5357..8868912dbb 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -24,7 +24,6 @@ module Logs.Location (
 import Common.Annex
 import qualified Git
 import qualified Annex.Branch
-import Logs.UUID
 import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index 47c2d7472b..e2b04bf471 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -21,7 +21,6 @@ import Data.Char
 import Common.Annex
 import qualified Annex.Branch
 import Types.Remote
-import Logs.UUID
 import Logs.UUIDBased
 
 {- Filename of remote.log. -}
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 6966ffdd6e..372d8b3609 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -19,8 +19,6 @@ import Common.Annex
 import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
-
-import Logs.UUID
 import Logs.UUIDBased
 
 {- Filename of trust.log. -}
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index baf6650015..8a93b43fef 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -14,12 +14,6 @@
  -}
 
 module Logs.UUID (
-	UUID,
-	getUUID,
-	getRepoUUID,
-	getUncachedUUID,
-	prepUUID,
-	genUUID,
 	describeUUID,
 	uuidMap
 ) where
@@ -28,61 +22,13 @@ import qualified Data.Map as M
 import Data.Time.Clock.POSIX
 
 import Common.Annex
-import qualified Git
 import qualified Annex.Branch
-import Types.UUID
-import qualified Build.SysConfig as SysConfig
-import Config
 import Logs.UUIDBased
 
-configkey :: String
-configkey = "annex.uuid"
-
 {- Filename of uuid.log. -}
 logfile :: FilePath
 logfile = "uuid.log"
 
-{- Generates a UUID. There is a library for this, but it's not packaged,
- - so use the command line tool. -}
-genUUID :: IO UUID
-genUUID = pOpen ReadFromPipe command params hGetLine
-	where
-		command = SysConfig.uuid
-		params = if command == "uuid"
-			-- request a random uuid be generated
-			then ["-m"]
-			-- uuidgen generates random uuid by default
-			else []
-
-getUUID :: Annex UUID
-getUUID = getRepoUUID =<< gitRepo
-
-{- Looks up a repo's UUID. May return "" if none is known. -}
-getRepoUUID :: Git.Repo -> Annex UUID
-getRepoUUID r = do
-	g <- gitRepo
-
-	let c = cached g
-	let u = getUncachedUUID r
-	
-	if c /= u && u /= ""
-		then do
-			updatecache g u
-			return u
-		else return c
-	where
-		cached g = Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ setConfig cachekey u
-		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
-
-getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = Git.configGet r configkey ""
-
-{- Make sure that the repo has an annex.uuid setting. -}
-prepUUID :: Annex ()
-prepUUID = whenM (null <$> getUUID) $
-	setConfig configkey =<< liftIO genUUID
-
 {- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
diff --git a/Logs/Web.hs b/Logs/Web.hs
index 4c8ef7fc00..605797079f 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -16,7 +16,6 @@ module Logs.Web (
 import Common.Annex
 import Logs.Presence
 import Logs.Location
-import Logs.UUID
 
 type URLString = String
 
diff --git a/Remote.hs b/Remote.hs
index 87c4e23b26..d1714f7751 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -36,6 +36,7 @@ import Common.Annex
 import Types.Remote
 import qualified Annex
 import Config
+import Annex.UUID
 import Logs.UUID
 import Logs.Trust
 import Logs.Location
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index dfc9116882..8d36245a94 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,7 +15,6 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Utility.Ssh
 import Remote.Helper.Special
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 270c78f838..e8cf05a0e5 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -15,7 +15,6 @@ import Common.Annex
 import Utility.CopyFile
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Utility.FileMode
 import Remote.Helper.Special
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 42d1b5858d..10183522fe 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -17,7 +17,7 @@ import Utility.Ssh
 import Types.Remote
 import qualified Git
 import qualified Annex
-import Logs.UUID
+import Annex.UUID
 import qualified Annex.Content
 import qualified Utility.Url as Url
 import Config
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 5603d13aa2..52f2dbf954 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,7 +12,6 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 2c6b50c7da..8b6a6cecfc 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -15,7 +15,6 @@ import System.Exit
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 21b9818465..63963c5309 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import Config
-import Logs.UUID
 import Logs.Web
 import qualified Utility.Url as Url
 
diff --git a/Types.hs b/Types.hs
index 6353f6da68..703edb5c86 100644
--- a/Types.hs
+++ b/Types.hs
@@ -8,9 +8,11 @@
 module Types (
 	Annex,
 	Backend,
-	Key
+	Key,
+	UUID
 ) where
 
 import Annex
 import Types.Backend
 import Types.Key
+import Types.UUID
diff --git a/Utility/Ssh.hs b/Utility/Ssh.hs
index 1847ff2440..34e4390f6b 100644
--- a/Utility/Ssh.hs
+++ b/Utility/Ssh.hs
@@ -13,7 +13,7 @@ import qualified Git
 import Utility.SafeCommand
 import Types
 import Config
-import Logs.UUID
+import Annex.UUID
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index f19abe6c3f..72e130ff08 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -13,7 +13,7 @@ import qualified Git
 import CmdLine
 import Command
 import Options
-import Logs.UUID
+import Annex.UUID
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
diff --git a/test.hs b/test.hs
index 6a6ad441e5..7d4c4293e1 100644
--- a/test.hs
+++ b/test.hs
@@ -23,6 +23,7 @@ import Common
 
 import qualified Utility.SafeCommand
 import qualified Annex
+import qualified Annex.UUID
 import qualified Backend
 import qualified Git
 import qualified Locations
@@ -609,7 +610,7 @@ checkdangling f = do
 
 checklocationlog :: FilePath -> Bool -> Assertion
 checklocationlog f expected = do
-	thisuuid <- annexeval Logs.UUID.getUUID
+	thisuuid <- annexeval Annex.UUID.getUUID
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do

From 1480d71adb1d6acf2cc8863064902244a31f099b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 18:45:32 -0400
Subject: [PATCH 2341/8313] fix

---
 Remote/S3real.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index a754731281..40d7d905d5 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -21,7 +21,7 @@ import Common.Annex
 import Types.Remote
 import Types.Key
 import qualified Git
-import UUID
+import Logs.UUID
 import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable

From 52c8244219dd90102818282b8b09186f2ce93a0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 15 Oct 2011 19:06:35 -0400
Subject: [PATCH 2342/8313] git-annex-shell: GIT_ANNEX_SHELL_READONLY and
 GIT_ANNEX_SHELL_LIMITED environment variables can be set to limit what
 commands can be run.

This could be used by eg, gitolite.
---
 debian/changelog         |  3 +++
 doc/git-annex-shell.mdwn | 24 +++++++++++++++++++----
 git-annex-shell.hs       | 42 ++++++++++++++++++++++++++++++++--------
 3 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index ce1489e9d0..4e0a1e21e4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (3.20111012) UNRELEASED; urgency=low
   * A remote can have a annexUrl configured, that is used by git-annex
     instead of its usual url. (Similar to pushUrl.)
   * migrate: Copy url logs for keys when migrating.
+  * git-annex-shell: GIT_ANNEX_SHELL_READONLY and GIT_ANNEX_SHELL_LIMITED
+    environment variables can be set to limit what commands can be run.
+    This could be used by eg, gitolite.
 
  -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
 
diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 1fc9647c88..fc5bc6c2d6 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -19,6 +19,10 @@ user's restricted login shell.
 
 Any command not listed below is passed through to git-shell.
 
+Note that the directory parameter should be an absolute path, otherwise
+it is assumed to be relative to the user's home directory. Also the
+first "/~/" or "/~user/" is expanded to the specified home directory.
+
 * configlist directory
 
   This outputs a subset of the git configuration, in the same form as
@@ -44,11 +48,23 @@ Any command not listed below is passed through to git-shell.
 
 # OPTIONS
 
-Same as git-annex or git-shell, depending on the command being run.
+Most options are the same as in git-annex. The ones specific
+to git-annex-shell are:
 
-Note that the directory parameter should be an absolute path, otherwise
-it is assumed to be relative to the user's home directory. Also the
-first "/~/" or "/~user/" is expanded to the specified home directory.
+* --uuid=UUID
+
+  git-annex uses this to specify the UUID of the repository it was expecting
+  git-annex-shell to access, as a sanity check.
+
+# ENVIRONMENT
+
+* GIT_ANNEX_SHELL_READONLY
+
+  If set, disallows any command that could modify the repository.
+
+* GIT_ANNEX_SHELL_LIMITED
+
+  If set, disallows running git-shell to handle unknown commands.
 
 # SEE ALSO
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 72e130ff08..41cb72d7e2 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -21,21 +21,29 @@ import qualified Command.DropKey
 import qualified Command.RecvKey
 import qualified Command.SendKey
 
-cmds :: [Command]
-cmds = map adddirparam $ concat
+cmds_readonly :: [Command]
+cmds_readonly = concat
 	[ Command.ConfigList.command
 	, Command.InAnnex.command
-	, Command.DropKey.command
-	, Command.RecvKey.command
 	, Command.SendKey.command
 	]
+
+cmds_notreadonly :: [Command]
+cmds_notreadonly = concat
+	[ Command.RecvKey.command
+	, Command.DropKey.command
+	]
+
+cmds :: [Command]
+cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
 	where
 		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
 
 options :: [OptDescr (Annex ())]
-options = uuid : commonOptions
+options = commonOptions ++
+	[ Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
+	]
 	where
-		uuid = Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
 		check expected = do
 			u <- getUUID
 			when (u /= expected) $ error $
@@ -67,12 +75,14 @@ builtins :: [String]
 builtins = map cmdname cmds
 
 builtin :: String -> String -> [String] -> IO ()
-builtin cmd dir params =
+builtin cmd dir params = do
+	checkNotReadOnly cmd
 	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
 		dispatch (cmd : filterparams params) cmds options header
 
 external :: [String] -> IO ()
-external params =
+external params = do
+	checkNotLimited
 	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
 		error "git-shell failed"
 
@@ -85,3 +95,19 @@ filterparams (a:as) = a:filterparams as
 
 failure :: IO ()
 failure = error $ "bad parameters\n\n" ++ usage header cmds options
+
+checkNotLimited :: IO ()
+checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED"
+
+checkNotReadOnly :: String -> IO ()
+checkNotReadOnly cmd
+	| cmd `elem` map cmdname cmds_readonly = return ()
+	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
+
+checkEnv :: String -> IO ()
+checkEnv var = catch check (const $ return ())
+	where
+		check = do
+			val <- getEnv var
+			when (not $ null val) $
+				error $ "Action blocked by " ++ var

From 91366c896d9c9cb4519b451a64ed4d1e0ff52cb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 16 Oct 2011 00:04:26 -0400
Subject: [PATCH 2343/8313] clean Annex stuff out of Utility/

---
 {Utility => Annex}/Ssh.hs |  2 +-
 Command/AddUrl.hs         |  2 +-
 Command/Map.hs            |  2 +-
 Locations.hs              |  2 +-
 Remote/Bup.hs             |  2 +-
 Remote/Git.hs             |  4 ++--
 Remote/Web.hs             |  4 +++-
 Utility/Url.hs            | 12 ++++--------
 8 files changed, 14 insertions(+), 16 deletions(-)
 rename {Utility => Annex}/Ssh.hs (98%)

diff --git a/Utility/Ssh.hs b/Annex/Ssh.hs
similarity index 98%
rename from Utility/Ssh.hs
rename to Annex/Ssh.hs
index 34e4390f6b..851c7c06b6 100644
--- a/Utility/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Ssh where
+module Annex.Ssh where
 
 import Control.Monad.State (liftIO)
 
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 2756af8807..f32b5b86a9 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -48,7 +48,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	let tmp = gitAnnexTmpLocation g dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- Url.download url tmp
+	ok <- liftIO $ Url.download url tmp
 	if ok
 		then do
 			[(backend, _)] <- Backend.chooseBackends [file]
diff --git a/Command/Map.hs b/Command/Map.hs
index 18cb915e3b..48cba63f9e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,7 +16,7 @@ import qualified Git
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
-import Utility.Ssh
+import Annex.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Locations.hs b/Locations.hs
index 4579fe05b7..ceb6246b98 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -127,7 +127,7 @@ isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
  -     is one to one.
  - ":" is escaped to "&c", because despite it being 2011, people still care
  -     about FAT.
- - -}
+ -}
 keyFile :: Key -> FilePath
 keyFile key = replace "/" "%" $ replace ":" "&c" $
 	replace "%" "&s" $ replace "&" "&a"  $ show key
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 8d36245a94..48014f1dad 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -16,7 +16,7 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import Config
-import Utility.Ssh
+import Annex.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 10183522fe..8857d821d2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -13,7 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Utility.CopyFile
 import Utility.RsyncFile
-import Utility.Ssh
+import Annex.Ssh
 import Types.Remote
 import qualified Git
 import qualified Annex
@@ -164,7 +164,7 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| Git.repoIsHttp r = Url.download (keyUrl r key) file
+	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
 
 {- Tries to copy a key's content to a remote's annex. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 63963c5309..3fea94531a 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -49,7 +49,9 @@ downloadKey key file = get =<< getUrls key
 		get [] = do
 			warning "no known url"
 			return False
-		get urls = anyM (`Url.download` file) urls
+		get urls = do
+			showOutput -- make way for download progress bar
+			liftIO $ anyM (`Url.download` file) urls
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 6ddeecc14f..b5f5b78c00 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -12,13 +12,10 @@ module Utility.Url (
 ) where
 
 import Control.Applicative
-import Control.Monad.State (liftIO)
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
 
-import Types
-import Messages
 import Utility.SafeCommand
 import Utility
 
@@ -38,13 +35,12 @@ exists url =
 {- Used to download large files, such as the contents of keys.
  - Uses wget or curl program for its progress bar. (Wget has a better one,
  - so is preferred.) -}
-download :: URLString -> FilePath -> Annex Bool
+download :: URLString -> FilePath -> IO Bool
 download url file = do
-	showOutput -- make way for program's progress bar
-	e <- liftIO $ inPath "wget"
+	e <- inPath "wget"
 	if e
 		then
-			liftIO $ boolSystem "wget"
+			boolSystem "wget"
 				[Params "-c -O", File file, File url]
 		else
 			-- Uses the -# progress display, because the normal
@@ -52,7 +48,7 @@ download url file = do
 			-- the remainder to download as the whole file,
 			-- and not indicating how much percent was
 			-- downloaded before the resume.
-			liftIO $ boolSystem "curl" 
+			boolSystem "curl" 
 				[Params "-L -C - -# -o", File file, File url]
 
 {- Downloads a small file. -}

From 23f2a12816e250f6780f80443ef6ec31c13fca9e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 16 Oct 2011 00:31:25 -0400
Subject: [PATCH 2344/8313] broke up Utility

---
 Command/Unused.hs    |   1 +
 Common.hs            |   4 +-
 Init.hs              |   1 +
 Remote/Git.hs        |   1 +
 Remote/Rsync.hs      |   1 -
 Remote/S3real.hs     |   1 -
 Remote/Web.hs        |   1 +
 Upgrade/V1.hs        |   1 +
 Upgrade/V2.hs        |   1 +
 Utility.hs           | 106 -------------------------------------------
 Utility/Misc.hs      |  29 ++++++++++++
 Utility/Monad.hs     |  26 +++++++++++
 Utility/Path.hs      |  22 +++++++++
 Utility/RsyncFile.hs |   2 +-
 Utility/TempFile.hs  |  39 ++++++++++++++++
 Utility/Url.hs       |   2 +-
 16 files changed, 126 insertions(+), 112 deletions(-)
 delete mode 100644 Utility.hs
 create mode 100644 Utility/Misc.hs
 create mode 100644 Utility/Monad.hs
 create mode 100644 Utility/TempFile.hs

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 874b0ca061..a901747521 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -16,6 +16,7 @@ import Common.Annex
 import Command
 import Annex.Content
 import Utility.FileMode
+import Utility.TempFile
 import Logs.Location
 import qualified Annex
 import qualified Git
diff --git a/Common.hs b/Common.hs
index e88342ae4a..2e1e4d996c 100644
--- a/Common.hs
+++ b/Common.hs
@@ -15,7 +15,7 @@ module Common (
 	module System.Posix.IO,
 	module System.Posix.Process,
 	module System.Exit,
-	module Utility,
+	module Utility.Misc,
 	module Utility.Conditional,
 	module Utility.SafeCommand,
 	module Utility.Path,
@@ -40,7 +40,7 @@ import System.Posix.IO
 import System.Posix.Process hiding (executeFile)
 import System.Exit
 
-import Utility
+import Utility.Misc
 import Utility.Conditional
 import Utility.SafeCommand
 import Utility.Path
diff --git a/Init.hs b/Init.hs
index 43840a1081..6e024e9fc5 100644
--- a/Init.hs
+++ b/Init.hs
@@ -12,6 +12,7 @@ module Init (
 ) where
 
 import Common.Annex
+import Utility.TempFile
 import qualified Git
 import qualified Annex.Branch
 import Annex.Version
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 8857d821d2..5d31770a22 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -20,6 +20,7 @@ import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
 import qualified Utility.Url as Url
+import Utility.TempFile
 import Config
 import Init
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 3216567479..e79762a382 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -13,7 +13,6 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
-import Logs.UUID
 import Config
 import Annex.Content
 import Remote.Helper.Special
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 40d7d905d5..89b0326379 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -21,7 +21,6 @@ import Common.Annex
 import Types.Remote
 import Types.Key
 import qualified Git
-import Logs.UUID
 import Config
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 3fea94531a..393932d478 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -13,6 +13,7 @@ import qualified Git
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
+import Utility.Monad
 
 remote :: RemoteType Annex
 remote = RemoteType {
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 6c6531acea..331328e816 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -21,6 +21,7 @@ import qualified Git.LsFiles as LsFiles
 import Backend
 import Annex.Version
 import Utility.FileMode
+import Utility.TempFile
 import qualified Upgrade.V2
 
 -- v2 adds hashing of filenames of content and location log files.
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index d6334ed654..1ad41266a7 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -12,6 +12,7 @@ import qualified Git
 import qualified Annex.Branch
 import Logs.Location
 import Annex.Content
+import Utility.TempFile
 
 olddir :: Git.Repo -> FilePath
 olddir g
diff --git a/Utility.hs b/Utility.hs
deleted file mode 100644
index 8ef60a0814..0000000000
--- a/Utility.hs
+++ /dev/null
@@ -1,106 +0,0 @@
-{- general purpose utility functions
- -
- - Copyright 2010-2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Utility (
-	hGetContentsStrict,
-	readFileStrict,
-	readMaybe,
-	viaTmp,
-	withTempFile,
-	dirContents,
-	myHomeDir,
-	catchBool,
-	inPath,
-	firstM,
-	anyM
-) where
-
-import Control.Applicative
-import IO (bracket)
-import System.IO
-import System.Posix.Process hiding (executeFile)
-import System.Posix.User
-import System.FilePath
-import System.Directory
-import Utility.Path
-import Data.Maybe
-import Control.Monad (liftM)
-
-{- A version of hgetContents that is not lazy. Ensures file is 
- - all read before it gets closed. -}
-hGetContentsStrict :: Handle -> IO String
-hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
-
-{- A version of readFile that is not lazy. -}
-readFileStrict :: FilePath -> IO String
-readFileStrict f = readFile f >>= \s -> length s `seq` return s
-
-{- Attempts to read a value from a String. -}
-readMaybe :: (Read a) => String -> Maybe a
-readMaybe s = case reads s of
-	((x,_):_) -> Just x
-	_ -> Nothing
-
-{- Runs an action like writeFile, writing to a tmp file first and
- - then moving it into place. -}
-viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
-viaTmp a file content = do
-	pid <- getProcessID
-        let tmpfile = file ++ ".tmp" ++ show pid
-	createDirectoryIfMissing True (parentDir file)
-	a tmpfile content
-	renameFile tmpfile file
-
-{- Runs an action with a temp file, then removes the file. -}
-withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
-withTempFile template a = bracket create remove use
-	where
-		create = do
-			tmpdir <- catch getTemporaryDirectory (const $ return ".")
-			openTempFile tmpdir template
-		remove (name, handle) = do
-			hClose handle
-			catchBool (removeFile name >> return True)
-		use (name, handle) = a name handle
-
-{- Lists the contents of a directory.
- - Unlike getDirectoryContents, paths are not relative to the directory. -}
-dirContents :: FilePath -> IO [FilePath]
-dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
-	where
-		notcruft "." = False
-		notcruft ".." = False
-		notcruft _ = True
-
-{- Current user's home directory. -}
-myHomeDir :: IO FilePath
-myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
-
-{- Catches IO errors and returns a Bool -}
-catchBool :: IO Bool -> IO Bool
-catchBool = flip catch (const $ return False)
-
-{- Return the first value from a list, if any, satisfying the given
- - predicate -}
-firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
-firstM _ [] = return Nothing
-firstM p (x:xs) = do
-	q <- p x
-	if q
-		then return (Just x)
-		else firstM p xs
-
-{- Returns true if any value in the list satisfies the preducate,
- - stopping once one is found. -}
-anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
-anyM p = liftM isJust . firstM p
-
-{- Checks if a command is available in PATH. -}
-inPath :: String -> IO Bool
-inPath command = getSearchPath >>= anyM indir
-	where
-		indir d = doesFileExist $ d  command
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
new file mode 100644
index 0000000000..bc18347746
--- /dev/null
+++ b/Utility/Misc.hs
@@ -0,0 +1,29 @@
+{- misc utility functions
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Misc where
+
+import System.IO
+
+{- A version of hgetContents that is not lazy. Ensures file is 
+ - all read before it gets closed. -}
+hGetContentsStrict :: Handle -> IO String
+hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
+
+{- A version of readFile that is not lazy. -}
+readFileStrict :: FilePath -> IO String
+readFileStrict f = readFile f >>= \s -> length s `seq` return s
+
+{- Attempts to read a value from a String. -}
+readMaybe :: (Read a) => String -> Maybe a
+readMaybe s = case reads s of
+	((x,_):_) -> Just x
+	_ -> Nothing
+
+{- Catches IO errors and returns a Bool -}
+catchBool :: IO Bool -> IO Bool
+catchBool = flip catch (const $ return False)
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
new file mode 100644
index 0000000000..9523e17165
--- /dev/null
+++ b/Utility/Monad.hs
@@ -0,0 +1,26 @@
+{- monadic stuff
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Monad where
+
+import Data.Maybe
+import Control.Monad (liftM)
+
+{- Return the first value from a list, if any, satisfying the given
+ - predicate -}
+firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM _ [] = return Nothing
+firstM p (x:xs) = do
+	q <- p x
+	if q
+		then return (Just x)
+		else firstM p xs
+
+{- Returns true if any value in the list satisfies the preducate,
+ - stopping once one is found. -}
+anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM p = liftM isJust . firstM p
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 1c68b87bbc..38e7bd05ca 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -14,6 +14,9 @@ import System.Directory
 import Data.List
 import Data.Maybe
 import Control.Applicative
+import System.Posix.User
+
+import Utility.Monad
 
 {- Returns the parent directory of a path. Parent of / is "" -}
 parentDir :: FilePath -> FilePath
@@ -112,3 +115,22 @@ preserveOrder (l:ls) new = found ++ preserveOrder ls rest
  -}
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
 runPreserveOrder a files = preserveOrder files <$> a files
+
+{- Lists the contents of a directory.
+ - Unlike getDirectoryContents, paths are not relative to the directory. -}
+dirContents :: FilePath -> IO [FilePath]
+dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
+	where
+		notcruft "." = False
+		notcruft ".." = False
+		notcruft _ = True
+
+{- Current user's home directory. -}
+myHomeDir :: IO FilePath
+myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
+
+{- Checks if a command is available in PATH. -}
+inPath :: String -> IO Bool
+inPath command = getSearchPath >>= anyM indir
+	where
+		indir d = doesFileExist $ d  command
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index b6c2267e87..056bd8d114 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -1,4 +1,4 @@
-{- git-annex file copying with rsync
+{- file copying with rsync
  -
  - Copyright 2010 Joey Hess 
  -
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
new file mode 100644
index 0000000000..1e823c10ef
--- /dev/null
+++ b/Utility/TempFile.hs
@@ -0,0 +1,39 @@
+{- temp file functions
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.TempFile where
+
+import IO (bracket)
+import System.IO
+import System.Posix.Process hiding (executeFile)
+import System.Directory
+
+import Utility.Misc
+import Utility.Path
+
+{- Runs an action like writeFile, writing to a temp file first and
+ - then moving it into place. The temp file is stored in the same
+ - directory as the final file to avoid cross-device renames. -}
+viaTmp :: (FilePath -> String -> IO ()) -> FilePath -> String -> IO ()
+viaTmp a file content = do
+	pid <- getProcessID
+        let tmpfile = file ++ ".tmp" ++ show pid
+	createDirectoryIfMissing True (parentDir file)
+	a tmpfile content
+	renameFile tmpfile file
+
+{- Runs an action with a temp file, then removes the file. -}
+withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile template a = bracket create remove use
+	where
+		create = do
+			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			openTempFile tmpdir template
+		remove (name, handle) = do
+			hClose handle
+			catchBool (removeFile name >> return True)
+		use (name, handle) = a name handle
diff --git a/Utility/Url.hs b/Utility/Url.hs
index b5f5b78c00..617fe3f4d0 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -17,7 +17,7 @@ import Network.HTTP
 import Network.URI
 
 import Utility.SafeCommand
-import Utility
+import Utility.Path
 
 type URLString = String
 

From 66fa4c947c30ca9848121912229f3e84a855a74f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 16 Oct 2011 01:03:38 -0400
Subject: [PATCH 2345/8313] correct spelling of "gibibyte"

---
 Utility/DataUnits.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index 0baa5dd896..e7552f52fb 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -84,7 +84,7 @@ memoryUnits =
 	, Unit (p 6) "EiB" "exbibyte"
 	, Unit (p 5) "PiB" "pebibyte"
 	, Unit (p 4) "TiB" "tebibyte"
-	, Unit (p 3) "GiB" "gigabyte"
+	, Unit (p 3) "GiB" "gibibyte"
 	, Unit (p 2) "MiB" "mebibyte"
 	, Unit (p 1) "KiB" "kibibyte"
 	, Unit (p 0) "B" "byte"

From 617bdc740f76e0b5cb8d73a8b122cd2b3e6fe961 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 13:56:36 -0400
Subject: [PATCH 2346/8313] reorg

---
 doc/cheatsheet.mdwn                              | 16 ----------------
 doc/tips.mdwn                                    |  4 ++++
 .../Internet_Archive_via_S3.mdwn                 |  0
 .../migrating_data_to_a_new_backend.mdwn         |  0
 .../powerful_file_matching.mdwn                  |  0
 .../recover_data_from_lost+found.mdwn            |  0
 .../untrusted_repositories.mdwn                  |  0
 doc/{walkthrough => tips}/using_Amazon_S3.mdwn   |  0
 .../using_the_SHA1_backend.mdwn                  |  0
 doc/{walkthrough => tips}/using_the_web.mdwn     |  0
 .../what_to_do_when_you_lose_a_repository.mdwn   |  0
 doc/walkthrough/more.mdwn                        |  3 +--
 12 files changed, 5 insertions(+), 18 deletions(-)
 delete mode 100644 doc/cheatsheet.mdwn
 create mode 100644 doc/tips.mdwn
 rename doc/{walkthrough => tips}/Internet_Archive_via_S3.mdwn (100%)
 rename doc/{walkthrough => tips}/migrating_data_to_a_new_backend.mdwn (100%)
 rename doc/{walkthrough => tips}/powerful_file_matching.mdwn (100%)
 rename doc/{walkthrough => tips}/recover_data_from_lost+found.mdwn (100%)
 rename doc/{walkthrough => tips}/untrusted_repositories.mdwn (100%)
 rename doc/{walkthrough => tips}/using_Amazon_S3.mdwn (100%)
 rename doc/{walkthrough => tips}/using_the_SHA1_backend.mdwn (100%)
 rename doc/{walkthrough => tips}/using_the_web.mdwn (100%)
 rename doc/{walkthrough => tips}/what_to_do_when_you_lose_a_repository.mdwn (100%)

diff --git a/doc/cheatsheet.mdwn b/doc/cheatsheet.mdwn
deleted file mode 100644
index 65cbad187c..0000000000
--- a/doc/cheatsheet.mdwn
+++ /dev/null
@@ -1,16 +0,0 @@
-A supplement to the [[walkthrough]].
-
-[[!toc]]
-
-[[!inline feeds=no show=0 template=walkthrough pagenames="""
-	walkthrough/using_Amazon_S3
-	walkthrough/using_bup
-	walkthrough/using_the_web
-	walkthrough/using_the_SHA1_backend
-	walkthrough/migrating_data_to_a_new_backend
-	walkthrough/untrusted_repositories
-	walkthrough/what_to_do_when_you_lose_a_repository
-	walkthrough/recover_data_from_lost+found
-	walkthrough/Internet_Archive_via_S3
-	walkthrough/powerful_file_matching
-"""]]
diff --git a/doc/tips.mdwn b/doc/tips.mdwn
new file mode 100644
index 0000000000..eda84c8672
--- /dev/null
+++ b/doc/tips.mdwn
@@ -0,0 +1,4 @@
+This page is a place to document tips and techniques for using git-annex.
+
+[[!inline pages="tips/* and !tips/*/*" archive="yes"
+rootpage="tips" postformtext="Add a new tip about:" show=0]]
diff --git a/doc/walkthrough/Internet_Archive_via_S3.mdwn b/doc/tips/Internet_Archive_via_S3.mdwn
similarity index 100%
rename from doc/walkthrough/Internet_Archive_via_S3.mdwn
rename to doc/tips/Internet_Archive_via_S3.mdwn
diff --git a/doc/walkthrough/migrating_data_to_a_new_backend.mdwn b/doc/tips/migrating_data_to_a_new_backend.mdwn
similarity index 100%
rename from doc/walkthrough/migrating_data_to_a_new_backend.mdwn
rename to doc/tips/migrating_data_to_a_new_backend.mdwn
diff --git a/doc/walkthrough/powerful_file_matching.mdwn b/doc/tips/powerful_file_matching.mdwn
similarity index 100%
rename from doc/walkthrough/powerful_file_matching.mdwn
rename to doc/tips/powerful_file_matching.mdwn
diff --git a/doc/walkthrough/recover_data_from_lost+found.mdwn b/doc/tips/recover_data_from_lost+found.mdwn
similarity index 100%
rename from doc/walkthrough/recover_data_from_lost+found.mdwn
rename to doc/tips/recover_data_from_lost+found.mdwn
diff --git a/doc/walkthrough/untrusted_repositories.mdwn b/doc/tips/untrusted_repositories.mdwn
similarity index 100%
rename from doc/walkthrough/untrusted_repositories.mdwn
rename to doc/tips/untrusted_repositories.mdwn
diff --git a/doc/walkthrough/using_Amazon_S3.mdwn b/doc/tips/using_Amazon_S3.mdwn
similarity index 100%
rename from doc/walkthrough/using_Amazon_S3.mdwn
rename to doc/tips/using_Amazon_S3.mdwn
diff --git a/doc/walkthrough/using_the_SHA1_backend.mdwn b/doc/tips/using_the_SHA1_backend.mdwn
similarity index 100%
rename from doc/walkthrough/using_the_SHA1_backend.mdwn
rename to doc/tips/using_the_SHA1_backend.mdwn
diff --git a/doc/walkthrough/using_the_web.mdwn b/doc/tips/using_the_web.mdwn
similarity index 100%
rename from doc/walkthrough/using_the_web.mdwn
rename to doc/tips/using_the_web.mdwn
diff --git a/doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
similarity index 100%
rename from doc/walkthrough/what_to_do_when_you_lose_a_repository.mdwn
rename to doc/tips/what_to_do_when_you_lose_a_repository.mdwn
diff --git a/doc/walkthrough/more.mdwn b/doc/walkthrough/more.mdwn
index 1eaf9009f6..0a4a5b94e8 100644
--- a/doc/walkthrough/more.mdwn
+++ b/doc/walkthrough/more.mdwn
@@ -1,4 +1,3 @@
 So ends the walkthrough. By now you should be able to use git-annex.
 
-Want more? See the [[cheatsheet]] for info about all of git-annex's hidden
-features.
+Want more? See [[tips]] for lots more features and advice.

From 9f30134300b2b119895389f73d413320dc897359 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:16:05 -0400
Subject: [PATCH 2347/8313] new tip for gitolite and git-annex

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 71 +++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex.mdwn

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
new file mode 100644
index 0000000000..c4df42cae6
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -0,0 +1,71 @@
+[Gitolite](https://github.com/sitaramc/gitolite) is a git repository
+manager. Here's how to add git-annex support to gitolite, so you can
+`git annex copy` files to a gitolite repository, and `git annex get`
+files from it.
+
+A nice feature of using gitolite with git-annex is that users can be given
+read-only access to a repository, and this allows them to `git annex get`
+file contents, but not change anything.
+
+First, you need new enough versions:
+
+* gitolite 2.2 is needed -- this version contains a git-annex-shell ADC
+  and supports "ua" ADCs.
+* git-annex 3.20111016 or newer needs to be installed on the gitolite
+  server. Don't install an older version, it wouldn't be secure!
+
+And here's how to set it up. The examples are for gitolite as installed
+on Debian with apt-get, but the changes described can be made to any
+gitolite installation, just with different paths.
+
+1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
+
+	echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+
+2. Make the ADC directory, and a "ua" subdirectory.
+
+	mkdir -p /usr/local/lib/gitolite/adc/ua
+
+3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
+   executable.
+
+	cd /usr/local/lib/gitolite/adc/ua/
+	wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
+	chmod +x git-annex-shell
+
+4. Now all gitolite repositories can be used with git-annex just as any
+   ssh remote normally would be used. For example:
+
+
+# git clone gitolite@localhost:testing
+Cloning into testing...
+Receiving objects: 100% (18/18), done.
+# cd testing
+# >git annex init
+init  ok
+# cp /etc/passwd my-cool-big-file
+# git annex add my-cool-big-file
+add my-cool-big-file ok
+(Recording state in git...)
+# git commit -m added
+[master d36c8b4] added
+ 1 files changed, 1 insertions(+), 0 deletions(-)
+ create mode 120000 my-cool-big-file
+# git push --all
+Counting objects: 17, done.
+Delta compression using up to 2 threads.
+Compressing objects: 100% (12/12), done.
+Writing objects: 100% (14/14), 1.39 KiB, done.
+Total 14 (delta 0), reused 1 (delta 0)
+To gitolite@localhost:testing
+   c552a38..db4653e  git-annex -> git-annex
+   29cd204..d36c8b4  master -> master
+# git annex copy --to origin
+copy my-cool-big-file (checking origin...) (to origin...) 
+WORM-s2502-m1318875140--my-cool-big-file
+        2502 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)
+
+sent 2606 bytes  received 31 bytes  1758.00 bytes/sec
+total size is 2502  speedup is 0.95
+ok
+


From 81fe6f775c02630421aba6d62e84e9849a9869c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:17:18 -0400
Subject: [PATCH 2348/8313] close

---
 doc/todo/gitolite_and_gitosis_support.mdwn | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/doc/todo/gitolite_and_gitosis_support.mdwn b/doc/todo/gitolite_and_gitosis_support.mdwn
index 12e26243e4..2fca839863 100644
--- a/doc/todo/gitolite_and_gitosis_support.mdwn
+++ b/doc/todo/gitolite_and_gitosis_support.mdwn
@@ -33,18 +33,7 @@ I have posted an RFC for modifying gitolite to the
 As I don't write python, someone else is needed to work on gitosis.
 --[[Joey]] 
 
-## readonly commands
-
-* git-annex-shell configlist $directory
-* git-annex-shell inannex $directory [$key ...]
-* git-annex-shell sendkey $directory $key
-
-## read-write commands
-
-* git-annex-shell dropkey $directory [$key ...]
-* git-annex-shell recvkey $directory $key
-
-## other git-annex-shell parameters
-
-All parameters like --uuid=foo and --force are safe and need to be allowed
-through.
+> [[done]]; support for gitolite is in its `pu` branch, and some changes
+> made to git-annefor gitolite is in its `pu` branch, and some changes
+> made to git-annex. Word is gitosis is not being maintained so I won't
+> worry about try to support it. --[[Joey]] 

From a6633f857bdcbdd3924a5b75337c1af0cf68d6cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:19:58 -0400
Subject: [PATCH 2349/8313] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index c4df42cae6..4dc6d8ae50 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -19,23 +19,23 @@ on Debian with apt-get, but the changes described can be made to any
 gitolite installation, just with different paths.
 
 1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
-
+   
 	echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
 
 2. Make the ADC directory, and a "ua" subdirectory.
-
+   
 	mkdir -p /usr/local/lib/gitolite/adc/ua
 
 3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
    executable.
-
+   
 	cd /usr/local/lib/gitolite/adc/ua/
 	wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
 	chmod +x git-annex-shell
 
 4. Now all gitolite repositories can be used with git-annex just as any
    ssh remote normally would be used. For example:
-
+   
 
 # git clone gitolite@localhost:testing
 Cloning into testing...

From 9e3783f8fedfc54ee4fbf96c1c5e267650993b4d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:20:36 -0400
Subject: [PATCH 2350/8313] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 4dc6d8ae50..bf3d61434b 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -20,18 +20,24 @@ gitolite installation, just with different paths.
 
 1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
    
-	echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+
+echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+

 
 2. Make the ADC directory, and a "ua" subdirectory.
-   
-	mkdir -p /usr/local/lib/gitolite/adc/ua
+
+
   
+mkdir -p /usr/local/lib/gitolite/adc/ua
+

 
 3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
    executable.
-   
-	cd /usr/local/lib/gitolite/adc/ua/
-	wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
-	chmod +x git-annex-shell
+
+
   
+cd /usr/local/lib/gitolite/adc/ua/
+wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
+chmod +x git-annex-shell
+

 
 4. Now all gitolite repositories can be used with git-annex just as any
    ssh remote normally would be used. For example:

From 1b76324034ce5770a6c5ccf2de5c08e81673032f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:21:44 -0400
Subject: [PATCH 2351/8313] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index bf3d61434b..745bdf450c 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -18,20 +18,20 @@ And here's how to set it up. The examples are for gitolite as installed
 on Debian with apt-get, but the changes described can be made to any
 gitolite installation, just with different paths.
 
-1. Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
+Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
    
 
 echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
 

 
-2. Make the ADC directory, and a "ua" subdirectory.
+Make the ADC directory, and a "ua" subdirectory.
 
 
   
 mkdir -p /usr/local/lib/gitolite/adc/ua
 

 
-3. Install the git-annex-shell ADC into the "ua" subdirectory and make it
-   executable.
+Install the git-annex-shell ADC into the "ua" subdirectory and make it
+executable.
 
 
   
 cd /usr/local/lib/gitolite/adc/ua/
@@ -39,8 +39,8 @@ wget https://raw.github.com/sitaramc/gitolite/pu/contrib/adc/git-annex-shell
 chmod +x git-annex-shell
 

 
-4. Now all gitolite repositories can be used with git-annex just as any
-   ssh remote normally would be used. For example:
+Now all gitolite repositories can be used with git-annex just as any
+ssh remote normally would be used. For example:
    
 
 # git clone gitolite@localhost:testing

From 9c0930a2b2d7b7471a7b154ac4c892ef05687521 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:22:25 -0400
Subject: [PATCH 2352/8313] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 745bdf450c..90bc0195f7 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -19,7 +19,7 @@ on Debian with apt-get, but the changes described can be made to any
 gitolite installation, just with different paths.
 
 Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
-   
+
 
 echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
 

@@ -41,7 +41,7 @@ chmod +x git-annex-shell
 
 Now all gitolite repositories can be used with git-annex just as any
 ssh remote normally would be used. For example:
-   
+
 
 # git clone gitolite@localhost:testing
 Cloning into testing...

From 5df717de437aa89da7be7a1b3030e1a39c69f87b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:23:27 -0400
Subject: [PATCH 2353/8313] layout

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 90bc0195f7..a51d6539b6 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -47,7 +47,7 @@ ssh remote normally would be used. For example:
 Cloning into testing...
 Receiving objects: 100% (18/18), done.
 # cd testing
-# >git annex init
+# git annex init
 init  ok
 # cp /etc/passwd my-cool-big-file
 # git annex add my-cool-big-file

From fa3f68ee63ec2700e76332d0439c3a5859791706 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 17 Oct 2011 14:28:30 -0400
Subject: [PATCH 2354/8313] add link

---
 doc/index.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 4b7159cd53..5bd42074f5 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -7,6 +7,7 @@ To get a feel for it, see the [[walkthrough]].
 
 * **[[download]]**
 * [[install]]
+* [[tips]]
 * [[bugs]]
 * [[todo]]
 * [[forum]]

From 89e0e0e474fede529819dd614b5680918fe47333 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Mon, 17 Oct 2011 18:51:40 +0000
Subject: [PATCH 2355/8313]

---
 .../confusion_with_remotes__44___map.mdwn     | 113 ++++++++++++++++++
 1 file changed, 113 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map.mdwn

diff --git a/doc/forum/confusion_with_remotes__44___map.mdwn b/doc/forum/confusion_with_remotes__44___map.mdwn
new file mode 100644
index 0000000000..0ae75d4e99
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map.mdwn
@@ -0,0 +1,113 @@
+I'm starting out with git-annex and running into some confusion with setting up the remotes.
+
+I have three systems I'm trying to set up (domains edited):
+
+* psychosis: ssh://psychosis.foo.com/vid
+* bacon: ssh://bucket.foo.com/vid
+* bucket: ssh://bucket.bar.org/vid
+
+And one bare repository so that I can have a single place to push/pull:
+
+* origin: https://git.foo.com/jim/vid.git
+
+On psychosis:
+
+    psychosis$ git config --list | grep ^remote | sort
+    remote.bacon.annex-uuid=8f1f0898-f8c1-11e0-9bf2-b387af26ee63
+    remote.bacon.fetch=+refs/heads/*:refs/remotes/bacon/*
+    remote.bacon.url=ssh://bucket.foo.com/vid
+    remote.bucket.annex-uuid=82814942-f8e0-11e0-b053-e70a61e98e19
+    remote.bucket.fetch=+refs/heads/*:refs/remotes/bucket/*
+    remote.bucket.url=ssh://bucket.bar.org/vid
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    
+    psychosis$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- here (psychosis)
+            82814942-f8e0-11e0-b053-e70a61e98e19 -- bucket
+    local annex keys: 2256
+    local annex size: 449 gigabytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+**First point of confusion**: Why doesn't "bacon" show up in "git annex status"?  I can "git annex copy --to bacon filename" and it will copy it there.  Is there some step of setting it up that I missed?  I basically just did "git remote add bacon ssh://bucket.foo.com/vid".
+
+Now I've started setting up the remotes on each host:
+
+On bacon:
+   
+    bacon$ git config --list | grep ^remote | sort
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    remote.psychosis.annex-uuid=09c0b436-f8de-11e0-842f-b7644539d57f
+    remote.psychosis.fetch=+refs/heads/*:refs/remotes/psychosis/*
+    remote.psychosis.url=ssh://psychosis.foo.com/vid
+    
+    bacon$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis
+            8f1f0898-f8c1-11e0-9bf2-b387af26ee63 -- here (bacon)
+    temporary directory size: 366 megabytes (clean up with git-annex unused)
+    local annex keys: 1
+    local annex size: 308 bytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+On bucket:
+
+    bucket$ git config --list | grep ^remote | sort
+    remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
+    remote.origin.url=https://git.foo.com/jim/vid.git
+    remote.psychosis.annex-uuid=09c0b436-f8de-11e0-842f-b7644539d57f
+    remote.psychosis.fetch=+refs/heads/*:refs/remotes/psychosis/*
+    remote.psychosis.url=ssh://psychosis.foo.com/vid
+
+    bucket$ git annex status
+    supported backends: WORM SHA1 SHA256 SHA512 SHA224 SHA384 SHA1E SHA256E SHA512E SHA224E SHA384E URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            09c0b436-f8de-11e0-842f-b7644539d57f -- psychosis
+            82814942-f8e0-11e0-b053-e70a61e98e19 -- here (bucket)
+    temporary directory size: 183 megabytes (clean up with git-annex unused)
+    local annex keys: 3
+    local annex size: 550 megabytes
+    total annex keys: 2256
+    total annex size: 449 gigabytes
+    backend usage: 
+            WORM: 2256
+
+But I'm getting weird results if I try to show the map from psychosis:
+
+    psychosis$ git annex map
+    $ git annex map
+    map /vid/tv ok
+    map bacon (sshing...) 
+    ok
+    map bucket (sshing...) 
+    ok
+    map origin 
+    failed
+    map psychosis (sshing...) 
+    jim@psychosis.foo.com's password: 
+    ok
+    map psychosis (sshing...) 
+    jim@psychosis.foo.com's password: 
+    ok
+    
+      running: dot -Tx11 map.dot
+
+**Second confusion**: it's as if psychosis was considered a new remote each time?
+The generated map has psychosis listed with several redundant links:
+
+![Map](http://jim.sh/~jim/tmp/map.png)
+
+Is this some bug or do I just need to be hit with the clue bat?

From 4004b6860637d59dcab6786f20994269f192ba4a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 17 Oct 2011 19:01:21 +0000
Subject: [PATCH 2356/8313] Added a comment

---
 ...comment_1_a38ded23b7f288292a843abcb1a56f38._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment b/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment
new file mode 100644
index 0000000000..97de93d9ec
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_1_a38ded23b7f288292a843abcb1a56f38._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-17T19:01:21Z"
+ content="""
+My guess is that psychosis has not pulled the git-annex branch since bacon was set up (or that bacon's git-annex branch has not been pushed to origin). git-annex status only shows remotes present in git-annex:uuid.log This may be a bug.
+
+The duplicate links in the map I don't quite understand. I only see duplicate links in my maps when I have the same repository configured as two different git remotes (for example, because the same repository can be accessed two different ways). You don't seem to have that in your config.
+"""]]

From 126bf2b5c10ab802df65c2f07d6a1edc8a9936e6 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 17 Oct 2011 19:02:50 +0000
Subject: [PATCH 2357/8313] Added a comment

---
 .../comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment b/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment
new file mode 100644
index 0000000000..a61b126c0c
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_2_cd1c98b1276444e859a22c3dbd6f2a79._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-10-17T19:02:50Z"
+ content="""
+Actually, there is a hint that, while you ran the git annex map on psychosis, it decided to ssh to itself two times. That seems to be where the duplicate links came from, I guess you must have some git remotes you did not show.
+"""]]

From fcbc6f901d3005bf3c64fe539391ba1c2bd2d9fc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Mon, 17 Oct 2011 19:50:07 +0000
Subject: [PATCH 2358/8313] Added a comment

---
 ..._18531754089c991b6caefc57a5c17fe9._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment b/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment
new file mode 100644
index 0000000000..4c77222619
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_3_18531754089c991b6caefc57a5c17fe9._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 3"
+ date="2011-10-17T19:50:06Z"
+ content="""
+No extra remotes (that I'm aware of); that output was only edited to change hostnames.
+
+On all three hosts, \"git push origin\" and \"git pull origin\" say everything is up to date.
+
+I'm using git-annex 3.20111011 on all hosts (although some were running 3.20110928 when I created the repositories).
+
+Regarding the multiple links, I've put a copy of the dot file [here](http://jim.sh/~jim/tmp/map.dot).
+It shows psychosis in three separate subgraphs, that are just getting rendered together as one,
+if that helps clarify anything.
+
+Wait, I just realized you said \"the git-annex branch\".  My origin only has \"master\".
+Do you mean the one specifically named \"git-annex\"?  I thought that was something that
+gets managed automatically, or is it something I need to manually check out and deal with?
+
+Any other info I could provide?
+
+
+"""]]

From 89536e97fbe3f5024c3f194119be9ab7416a8c13 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Mon, 17 Oct 2011 20:36:57 +0000
Subject: [PATCH 2359/8313] Added a comment

---
 ...omment_4_3b89b6d1518267fcbc050c9de038b9ca._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment b/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment
new file mode 100644
index 0000000000..f6e5993c8e
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_4_3b89b6d1518267fcbc050c9de038b9ca._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 4"
+ date="2011-10-17T20:36:51Z"
+ content="""
+Ok, after pushing the \"git-annex\" branch to origin, then \"git annex status\" knows all repositories on all hosts, so that part makes sense now.  Thanks for the tip.  But the \"git annex map\" output hasn't changed.
+
+
+
+"""]]

From bacb2e1881c9d4066ffe0be9a73c0add71f312d9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 18 Oct 2011 04:59:15 +0000
Subject: [PATCH 2360/8313] Added a comment

---
 ...t_5_27801584325d259fa490f67273f2ff71._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment b/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment
new file mode 100644
index 0000000000..77a2c4adbe
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_5_27801584325d259fa490f67273f2ff71._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 5"
+ date="2011-10-18T04:59:13Z"
+ content="""
+I think:
+
+* The first extra edge is because bucket had \"ssh://psychosis.foo.com/vid/\", while
+bacon had \"ssh://psychosis.foo.com/vid\" with no trailing slash.  That got lost in the hostname/path editing I did, sorry.
+Maybe those should be considered matching?
+* The second extra edge is because, when running \"git annex map\" from psychosis, it doesn't recognize the remote's
+remote URL as pointing back to itself.
+
+For the second case, after the \"spurious\" SSH, it could still recognize that the repositories are the same by the duplicated annex uuid, which currently shows up in `map.dot` twice.  I wonder what it would take to avoid the spurious SSH -- maybe some config that lists \"alternate\" URLs that should be considered the same as the current repository?  Or actually list URLs in uuid.log?  Fortunately, I think this only affects the map, so it's not a big problem.
+"""]]

From aa7da919a7588e733ee568b397f2b16b7e665a2f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 22 Oct 2011 01:18:27 +0000
Subject: [PATCH 2361/8313] Added a comment

---
 .../comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment b/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment
new file mode 100644
index 0000000000..412937f3fc
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_6_496b0d9b86869bbac3a1356d53a3dda4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-10-22T01:18:27Z"
+ content="""
+Hmm, I don't see the spurious ssh edge in the dot file -- that is, I don't see any ssh:// uris in it?
+"""]]

From 3c5a0e78c032f11097cebda5079e09666c54f2fc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Sat, 22 Oct 2011 05:25:48 +0000
Subject: [PATCH 2362/8313] Added a comment

---
 ..._9a456f61f956a3d5e81e723d5a90794c._comment | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment

diff --git a/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment b/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment
new file mode 100644
index 0000000000..85ede3a89c
--- /dev/null
+++ b/doc/forum/confusion_with_remotes__44___map/comment_7_9a456f61f956a3d5e81e723d5a90794c._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 7"
+ date="2011-10-22T05:25:47Z"
+ content="""
+I think that's because the SSH was successful (I entered the password and let it connect), so it got the UUID and put that in the .dot instead.  The same UUID (for psychosis) then ended up in two different \"subgraph\" stanzas, and Graphviz just plotted them together as one node.
+
+
Maybe this will clarify:
+
+
On psychosis, run \"git annex map\" and press ^C at the ssh password prompt: [map-nossh.dot](http://jim.sh/~jim/tmp/map-nossh.dot)
+![Map](http://jim.sh/~jim/tmp/map-nossh.png)
+
+
On psychosis, run \"git annex map\" and type the correct password: [map-goodssh.dot](http://jim.sh/~jim/tmp/map-goodssh.dot)
+![Map](http://jim.sh/~jim/tmp/map-goodssh.png)
+
+As I see it:
+
+* psychosis (\"localhost\") connects to each of its remotes
+* some of them point back to ssh://psychosis
+* psychosis doesn't know that ssh://psychosis is itself, so it tries to connect
+* if successful:
+  * psychosis gets put twice in the .dot as if it was two different hosts, one \"local\" and one \"ssh://psychosis\"
+  * graphviz recognizes it as the same node because the UUID is the same, but graphviz still draws the extra connecting lines
+* if unsuccessful:
+  * ssh://psychosis is shown as an additional host that can't be reached
+"""]]

From 721f236e3a96a23d79dcf86ea7ef3d85d7270305 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 23 Oct 2011 14:37:33 +0000
Subject: [PATCH 2363/8313]

---
 ...ot_run_when_branch_git-annex_is_checked_out.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn

diff --git a/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
new file mode 100644
index 0000000000..d140219364
--- /dev/null
+++ b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
@@ -0,0 +1,13 @@
+Running `git annex uninit` in a repo which has branch git-annex checked out will result in:
+
+    error: Cannot delete the branch 'git-annex' which you are currently on.
+    git-annex: git [Param "-D",Param "git-annex"] failed
+
+and trying to checkout branch master afterwards results in:
+
+    error: The following untracked working tree files would be overwritten by checkout:
+
+Both of which is logical. The best thing would be if git-annex refused to run uninit while in branch git-annex.
+
+
+Richard

From 962fed915ff2b4c652a879c4b7a01086c680c6e0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sun, 23 Oct 2011 15:00:49 +0000
Subject: [PATCH 2364/8313] Added a comment

---
 .../comment_1_4c46a193915eab8f308a04175cb2e40a._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment

diff --git a/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment b/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment
new file mode 100644
index 0000000000..3e28a28cb0
--- /dev/null
+++ b/doc/bugs/Prevent_accidental_merges/comment_1_4c46a193915eab8f308a04175cb2e40a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-23T15:00:48Z"
+ content="""
+Having run into the same issue again, I still think git-annex should ensure no merges take place. The clutter introduced by a .gitnomerge is neglible, imo.
+"""]]

From e2853b3fec1c34bd420b9e1753be4aa1afeb675e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 25 Oct 2011 11:39:15 -0700
Subject: [PATCH 2365/8313] update

---
 debian/changelog     | 2 +-
 doc/todo/smudge.mdwn | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 4e0a1e21e4..0ebda22dc5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111012) UNRELEASED; urgency=low
   * migrate: Copy url logs for keys when migrating.
   * git-annex-shell: GIT_ANNEX_SHELL_READONLY and GIT_ANNEX_SHELL_LIMITED
     environment variables can be set to limit what commands can be run.
-    This could be used by eg, gitolite.
+    This is used by gitolite's new git-annex support!
 
  -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
 
diff --git a/doc/todo/smudge.mdwn b/doc/todo/smudge.mdwn
index 55de5f5789..6103ffa61e 100644
--- a/doc/todo/smudge.mdwn
+++ b/doc/todo/smudge.mdwn
@@ -11,6 +11,10 @@ version of the file during a merge.
 So every `git status` would need to read the entire content of all
 available files, and checksum them, which is too expensive.
 
+> Update from GitTogether: Peff thinks a new interface could be added to
+> git to handle this sort of case in an efficient way.. just needs someone
+> to do the work. --[[Joey]] 
+
 ----
 
 The clean filter is run when files are staged for commit. So a user could copy

From 270c1af087449573c17eb4911ca2dc3db3fb1fc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 25 Oct 2011 13:46:01 -0700
Subject: [PATCH 2366/8313] releasing version 3.20111025

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 0ebda22dc5..81dec6ad06 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111012) UNRELEASED; urgency=low
+git-annex (3.20111025) unstable; urgency=low
 
   * A remote can have a annexUrl configured, that is used by git-annex
     instead of its usual url. (Similar to pushUrl.)
@@ -7,7 +7,7 @@ git-annex (3.20111012) UNRELEASED; urgency=low
     environment variables can be set to limit what commands can be run.
     This is used by gitolite's new git-annex support!
 
- -- Joey Hess   Fri, 14 Oct 2011 18:15:20 -0400
+ -- Joey Hess   Tue, 25 Oct 2011 13:03:08 -0700
 
 git-annex (3.20111011) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index e9ec089677..c3706808c0 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111012
+Version: 3.20111025
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 570aae9bbcdd6a7d5972288d322da3872d0a17be Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 25 Oct 2011 13:46:22 -0700
Subject: [PATCH 2367/8313] add news item for git-annex 3.20111025

---
 doc/news/version_3.20110902.mdwn | 9 ---------
 doc/news/version_3.20111025.mdwn | 8 ++++++++
 2 files changed, 8 insertions(+), 9 deletions(-)
 delete mode 100644 doc/news/version_3.20110902.mdwn
 create mode 100644 doc/news/version_3.20111025.mdwn

diff --git a/doc/news/version_3.20110902.mdwn b/doc/news/version_3.20110902.mdwn
deleted file mode 100644
index e354874ea3..0000000000
--- a/doc/news/version_3.20110902.mdwn
+++ /dev/null
@@ -1,9 +0,0 @@
-git-annex 3.20110902 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Set EMAIL when running test suite so that git does not need to be
-     configured first. Closes: #[638998](http://bugs.debian.org/638998)
-   * The wget command will now be used in preference to curl, if available.
-   * init: Make description an optional parameter.
-   * unused, status: Sped up by avoiding unnecessary stats of annexed files.
-   * unused --remote: Reduced memory use to 1/4th what was used before.
-   * Add --json switch, to produce machine-consumable output."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111025.mdwn b/doc/news/version_3.20111025.mdwn
new file mode 100644
index 0000000000..90e4227bae
--- /dev/null
+++ b/doc/news/version_3.20111025.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20111025 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * A remote can have a annexUrl configured, that is used by git-annex
+     instead of its usual url. (Similar to pushUrl.)
+   * migrate: Copy url logs for keys when migrating.
+   * git-annex-shell: GIT\_ANNEX\_SHELL\_READONLY and GIT\_ANNEX\_SHELL\_LIMITED
+     environment variables can be set to limit what commands can be run.
+     This is used by gitolite's new git-annex support!"""]]
\ No newline at end of file

From 12a0a45fb1ddb464d7863cdd707dd4b328ae5df5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Tue, 25 Oct 2011 21:40:30 +0000
Subject: [PATCH 2368/8313]

---
 doc/todo/support_fsck_in_bare_repos.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/support_fsck_in_bare_repos.mdwn

diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
new file mode 100644
index 0000000000..b126a8170b
--- /dev/null
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -0,0 +1,10 @@
+What is says on the tin:
+
+
+    22:56:54 < RichiH> joeyh_: by the way, i have been thinking about fsck on bare repos
+    22:57:37 < RichiH> joeyh_: the best i could come with is to have a bare and a non-bare access the same repo store
+    22:58:00 < RichiH> joeyh_: alternatively, with the SHA* backend, you have all the information to verify that the local data is correct
+    22:58:41 < RichiH> and verifying that would already be a plus. if there  really _is_ a problem, having the SHA is enough to track issues down
+    23:09:50 < joeyh_> oh, I think I have code that fsck could use on bare repos already.. just a matter of wiring it up
+    23:10:42 < joeyh_> feel free to reopen a bug or whatever so I remember.. the unused command's branch content enumeration could be used in a bare repo
+    23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw

From c8fdffa9f844f6884b064233d2cb7d4b44cc62e4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 15:46:43 +0000
Subject: [PATCH 2369/8313]

---
 doc/bugs/fails_to_handle_lot_of_files.mdwn | 441 +++++++++++++++++++++
 1 file changed, 441 insertions(+)
 create mode 100644 doc/bugs/fails_to_handle_lot_of_files.mdwn

diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn
new file mode 100644
index 0000000000..5c5a6c8fd1
--- /dev/null
+++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn
@@ -0,0 +1,441 @@
+git-annex version: 3.20111011
+local repository version: 3
+default repository version: 3
+supported repository versions: 3
+upgrade supported from repository versions: 0 1 2
+
+I just created a new remote on a USB drive and wanted to copy my files over. git-annex wasn't too happy about that ;)
+I included a few OK transfers as there was an error before git-annex ran into a wall. As I could easily access that temp file after it aborted, I suspect something went wrong internally before git-annex started to throw those errors.
+
+Please note the "_n TIMES_" comments. It's how often I got the same error message...
+
+
+
+    git annex copy . --to USB --fast
+    
+    copy redacted.JPG (to USB...) 
+    redacted
+         4035668 100%   77.91MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4036374 bytes  received 31 bytes  8072810.00 bytes/sec
+    total size is 4035668  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18002094 100%   74.19MB/s    0:00:00 (xfer#1, to-check=0/1)
+    WARNING: redacted failed verification -- update retained (will try again).
+    redacted
+        18002094 100%   19.60MB/s    0:00:00 (xfer#2, to-check=0/1)
+    rsync: open "copy_target/.git/annex/tmp/redacted_E13" failed: Permission denied (13)
+    
+    sent 36008841 bytes  received 52 bytes  24005928.67 bytes/sec
+    total size is 18002094  speedup is 0.50
+    rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1070) [sender=3.0.8]
+    
+      rsync failed -- run git annex again to resume file transfer
+    failed
+    copy redacted.JPG (to USB...) 
+    redacted
+         3687111 100%   39.16MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 3687773 bytes  received 31 bytes  2458536.00 bytes/sec
+    total size is 3687111  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17877177 100%   79.15MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17879573 bytes  received 31 bytes  11919736.00 bytes/sec
+    total size is 17877177  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         3694921 100%   40.14MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 3695583 bytes  received 31 bytes  2463742.67 bytes/sec
+    total size is 3694921  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17875448 100%   71.20MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17877844 bytes  received 31 bytes  11918583.33 bytes/sec
+    total size is 17875448  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         3833377 100%   62.49MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 3834055 bytes  received 31 bytes  2556057.33 bytes/sec
+    total size is 3833377  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17938200 100%   65.43MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17940604 bytes  received 31 bytes  11960423.33 bytes/sec
+    total size is 17938200  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4512557 100%   83.77MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4513319 bytes  received 31 bytes  3008900.00 bytes/sec
+    total size is 4512557  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18001641 100%   76.16MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18004053 bytes  received 31 bytes  12002722.67 bytes/sec
+    total size is 18001641  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4394272 100%   50.11MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4395022 bytes  received 31 bytes  8790106.00 bytes/sec
+    total size is 4394272  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18095781 100%   73.30MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18098205 bytes  received 31 bytes  12065490.67 bytes/sec
+    total size is 18095781  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4683795 100%   65.23MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4684577 bytes  received 31 bytes  9369216.00 bytes/sec
+    total size is 4683795  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18172801 100%   74.25MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18175233 bytes  received 31 bytes  36350528.00 bytes/sec
+    total size is 18172801  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4486231 100%   77.22MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4486989 bytes  received 31 bytes  8974040.00 bytes/sec
+    total size is 4486231  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17860427 100%   68.56MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17862823 bytes  received 31 bytes  35725708.00 bytes/sec
+    total size is 17860427  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4499768 100%   36.41MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4500530 bytes  received 31 bytes  9001122.00 bytes/sec
+    total size is 4499768  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17840132 100%   74.48MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17842524 bytes  received 31 bytes  11895036.67 bytes/sec
+    total size is 17840132  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4358032 100%   75.00MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4358774 bytes  received 31 bytes  8717610.00 bytes/sec
+    total size is 4358032  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18084753 100%   61.48MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18087173 bytes  received 31 bytes  12058136.00 bytes/sec
+    total size is 18084753  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4270213 100%   68.49MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4270947 bytes  received 31 bytes  2847318.67 bytes/sec
+    total size is 4270213  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17661246 100%   68.34MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17663614 bytes  received 31 bytes  11775763.33 bytes/sec
+    total size is 17661246  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4538305 100%   63.19MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4539071 bytes  received 31 bytes  9078204.00 bytes/sec
+    total size is 4538305  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18672466 100%   68.90MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18674958 bytes  received 31 bytes  12449992.67 bytes/sec
+    total size is 18672466  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4453445 100%   73.96MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4454199 bytes  received 31 bytes  8908460.00 bytes/sec
+    total size is 4453445  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18495494 100%   59.28MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18497966 bytes  received 31 bytes  12331998.00 bytes/sec
+    total size is 18495494  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4255858 100%   70.66MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4256588 bytes  received 31 bytes  1702647.60 bytes/sec
+    total size is 4255858  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18376531 100%   69.15MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18378987 bytes  received 31 bytes  36758036.00 bytes/sec
+    total size is 18376531  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4013365 100%   48.67MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4014067 bytes  received 31 bytes  8028196.00 bytes/sec
+    total size is 4013365  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17606341 100%   51.73MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17608705 bytes  received 31 bytes  11739157.33 bytes/sec
+    total size is 17606341  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4179869 100%   74.62MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4180591 bytes  received 31 bytes  8361244.00 bytes/sec
+    total size is 4179869  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18382569 100%   67.05MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18385025 bytes  received 31 bytes  12256704.00 bytes/sec
+    total size is 18382569  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4318363 100%   44.91MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4319101 bytes  received 31 bytes  8638264.00 bytes/sec
+    total size is 4318363  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17715958 100%   72.69MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17718334 bytes  received 31 bytes  11812243.33 bytes/sec
+    total size is 17715958  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4241893 100%   65.81MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4242623 bytes  received 31 bytes  8485308.00 bytes/sec
+    total size is 4241893  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17717287 100%   71.77MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17719663 bytes  received 31 bytes  11813129.33 bytes/sec
+    total size is 17717287  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4488380 100%   49.99MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4489138 bytes  received 31 bytes  2992779.33 bytes/sec
+    total size is 4488380  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        17770208 100%   38.80MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 17772592 bytes  received 31 bytes  11848415.33 bytes/sec
+    total size is 17770208  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4603958 100%   76.48MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4604732 bytes  received 31 bytes  9209526.00 bytes/sec
+    total size is 4603958  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18744380 100%   74.66MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18746884 bytes  received 31 bytes  12497943.33 bytes/sec
+    total size is 18744380  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4592098 100%   79.06MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4592872 bytes  received 31 bytes  3061935.33 bytes/sec
+    total size is 4592098  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18746205 100%   43.00MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18748709 bytes  received 31 bytes  12499160.00 bytes/sec
+    total size is 18746205  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         7493353 100%   80.85MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 7494479 bytes  received 31 bytes  14989020.00 bytes/sec
+    total size is 7493353  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19496768 100%   81.77MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19499360 bytes  received 31 bytes  12999594.00 bytes/sec
+    total size is 19496768  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         5462482 100%   82.19MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 5463360 bytes  received 31 bytes  10926782.00 bytes/sec
+    total size is 5462482  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19669815 100%   80.37MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19672431 bytes  received 31 bytes  13114974.67 bytes/sec
+    total size is 19669815  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         5449487 100%   57.40MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 5450365 bytes  received 31 bytes  3633597.33 bytes/sec
+    total size is 5449487  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19633259 100%   74.18MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19635871 bytes  received 31 bytes  13090601.33 bytes/sec
+    total size is 19633259  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         5392184 100%   62.33MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 5393054 bytes  received 31 bytes  3595390.00 bytes/sec
+    total size is 5392184  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        18912104 100%   65.00MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 18914628 bytes  received 31 bytes  12609772.67 bytes/sec
+    total size is 18912104  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4869300 100%   80.92MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4870106 bytes  received 31 bytes  9740274.00 bytes/sec
+    total size is 4869300  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        20178932 100%   68.13MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 20181608 bytes  received 31 bytes  13454426.00 bytes/sec
+    total size is 20178932  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         4995425 100%   86.05MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 4996247 bytes  received 31 bytes  9992556.00 bytes/sec
+    total size is 4995425  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        19970679 100%   76.36MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 19973331 bytes  received 31 bytes  13315574.67 bytes/sec
+    total size is 19970679  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         7905795 100%   66.45MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 7906973 bytes  received 31 bytes  15814008.00 bytes/sec
+    total size is 7905795  speedup is 1.00
+    ok
+    copy redacted.NEF (to USB...) 
+    redacted
+        21234069 100%   78.07MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 21236877 bytes  received 31 bytes  8494763.20 bytes/sec
+    total size is 21234069  speedup is 1.00
+    ok
+    copy redacted.JPG (to USB...) 
+    redacted
+         7963979 100%   62.51MB/s    0:00:00 (xfer#1, to-check=0/1)
+    
+    sent 7965165 bytes  received 31 bytes  5310130.67 bytes/sec
+    total size is 7963979  speedup is 1.00
+    git ["--git-dir=copy_target/.git","--work-tree=copy_target","update-index","-z","--index-info"]: Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable)
+    
+    git-annex: user error (git ["--git-dir=copy_target/.git","--work-tree=copy_target","update-index","-z","--index-info"]: Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))
+    failed
+    _506 TIMES_ (user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))) failed
+    _11 TIMES_ copy foo (createPipe: resource exhausted (Too many open files)) failed
+    _2 TIMES_ (user error (Error in fork: forkProcess: resource exhausted (Resource temporarily unavailable))) failed
+    _8574 TIMES_: copy foo (createPipe: resource exhausted (Too many open files)) failed
+    git-annex: createPipe: resource exhausted (Too many open files)
+    failed
+    git-annex: 9101 failed
+    
+    % ls copy_target/.git/annex/tmp/redacted_E13 copy_target/.git/annex/tmp/SHA512E-redacted_E13 # works
+    % find source -type l | wc -l
+    13554
+    % find copy_target -type l | wc -l
+    13554
+    % find copy_target/.git/annex/objects -type f | wc -l
+    4455
+    % find source -type f | wc -l
+    13554

From c37a70fbbc114b4deb89998571e03009240b8931 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 15:49:10 +0000
Subject: [PATCH 2370/8313]

---
 doc/bugs/fails_to_handle_lot_of_files.mdwn | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn
index 5c5a6c8fd1..bc5d92aeca 100644
--- a/doc/bugs/fails_to_handle_lot_of_files.mdwn
+++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn
@@ -1,8 +1,8 @@
-git-annex version: 3.20111011
-local repository version: 3
-default repository version: 3
-supported repository versions: 3
-upgrade supported from repository versions: 0 1 2
+    git-annex version: 3.20111011
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3
+    upgrade supported from repository versions: 0 1 2
 
 I just created a new remote on a USB drive and wanted to copy my files over. git-annex wasn't too happy about that ;)
 I included a few OK transfers as there was an error before git-annex ran into a wall. As I could easily access that temp file after it aborted, I suspect something went wrong internally before git-annex started to throw those errors.

From c65293977edd5fdd8f25ae4cdafa3d21003f4b0f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 17:16:53 +0000
Subject: [PATCH 2371/8313] Added a comment

---
 .../comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment

diff --git a/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment b/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment
new file mode 100644
index 0000000000..587b1fd97c
--- /dev/null
+++ b/doc/bugs/fails_to_handle_lot_of_files/comment_1_09d8e4e66d8273fab611bd29e82dc7fc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-26T17:16:52Z"
+ content="""
+After another run, i am at 8909 files in the remote, now.
+"""]]

From c5c682c131ece6e4c05e86206f01838c6abe2f0d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Wed, 26 Oct 2011 18:22:37 +0000
Subject: [PATCH 2372/8313] Added a comment

---
 .../comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment

diff --git a/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment b/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment
new file mode 100644
index 0000000000..8e83fc19f4
--- /dev/null
+++ b/doc/bugs/fails_to_handle_lot_of_files/comment_2_fd2ec05f4b5a7a6ae6bd9f5dbc3156de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-10-26T18:22:34Z"
+ content="""
+In case this matters, I just realized that this disk has been formatted with NTFS instead of a sane FS.
+"""]]

From c3df1c82b9b5bbc67a845667895adf0ca94ac6dd Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 27 Oct 2011 16:31:15 +0000
Subject: [PATCH 2373/8313]

---
 ...ishlist:_support_drop__44___find_on_special_remotes.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn

diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
new file mode 100644
index 0000000000..2dc735e99f
--- /dev/null
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
@@ -0,0 +1,6 @@
+Currently there is no way to drop files, or list what files are available, on a special remote.
+It would be good if "git annex drop" and "git annex find" supported the --from argument.
+
+For commands that don't support the --from argument, it would also be nice to print an error.
+Currently running "git annex drop --from usbdrive" doesn't behave as hoped and instead drops
+all content from the local annex.

From 5bc132b95cebbc5133082e08ba1c45227c679b7b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 27 Oct 2011 16:33:47 +0000
Subject: [PATCH 2374/8313]

---
 ...ishlist:_query_things_like_description__44___trust_level.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn

diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
new file mode 100644
index 0000000000..6404393a89
--- /dev/null
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
@@ -0,0 +1 @@
+It would be helpful to have a way to query things like a repository's description and trust level, without having to poke in the git-annex branch.  For example, "git annex describe ." currently clears the description but could print the current one instead.

From 2aa1c779ef218b5f0b253976d93adb333dfc26a0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 27 Oct 2011 17:09:33 +0000
Subject: [PATCH 2375/8313] Added a comment

---
 ...comment_1_14311384788312b96e550749ab7de9ea._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment

diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment
new file mode 100644
index 0000000000..3ac4ba2678
--- /dev/null
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_1_14311384788312b96e550749ab7de9ea._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-27T17:09:33Z"
+ content="""
+`git annex describe` only sets the description to avoid complication. Imagine using it in a script for example.
+
+`git annex status` shows the description. It does not show the trust level because I have not thought of a visually pleasing and compact way to show it in the repository list there.. suggestions appreciated, since the same list is used by `whereis`, and showing trust levels there would be particularly useful.
+"""]]

From 48d39dd35492952ac7f20da08a4168f43f782c95 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 27 Oct 2011 17:13:44 +0000
Subject: [PATCH 2376/8313] Added a comment

---
 .../comment_1_f11ed642a83d965076778a162f701e84._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment

diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment
new file mode 100644
index 0000000000..6028933b40
--- /dev/null
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes/comment_1_f11ed642a83d965076778a162f701e84._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-10-27T17:13:43Z"
+ content="""
+Well, I don't think you mean \"special remotes\", but just any old remote (special or not).
+"""]]

From b3e660504fce7ed0eab48b4e7deeeb9503318986 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 12:58:01 -0400
Subject: [PATCH 2377/8313] avoid showing parens for empty descriptions

---
 Remote.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote.hs b/Remote.hs
index d1714f7751..49fa63cf92 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -137,6 +137,7 @@ prettyPrintUUIDs desc uuids = do
 	where
 		addname d n
 			| d == n = d
+			| null d = n
 			| otherwise = n ++ " (" ++ d ++ ")"
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m

From 2eefc580703199d3ecca58d453f07747b565d476 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 12:59:55 -0400
Subject: [PATCH 2378/8313] status too

---
 doc/todo/support_fsck_in_bare_repos.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index b126a8170b..e6980fa281 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -8,3 +8,5 @@ What is says on the tin:
     23:09:50 < joeyh_> oh, I think I have code that fsck could use on bare repos already.. just a matter of wiring it up
     23:10:42 < joeyh_> feel free to reopen a bug or whatever so I remember.. the unused command's branch content enumeration could be used in a bare repo
     23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw
+
+> Also `status`'s total annex keys/size could be handled for bare repos. --[[Joey]] 

From 373cad993d9f12a9dfb7473fdba78e92af6eff99 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 14:04:18 -0400
Subject: [PATCH 2379/8313] Sped up some operations on remotes that are on the
 same host.

Specifically, disabled trying to update the git-annex branch on the remote,
since that data is never used by operations that act on such remotes.

Also, when copying content to such a remote, skip committing the presence
information changes to its git-annex branch. Leaving it in the journal there
is ok: Any command run on the remote that needs the info will flush the
journal.

This may partially solve this bug:
http://git-annex.branchable.com/bugs/fails_to_handle_lot_of_files/
Although I still see unreaped git processes piling up when doing a copy --to.
---
 Annex/Branch.hs  | 12 ++++++++++--
 Remote/Git.hs    | 16 +++++++++-------
 debian/changelog |  6 ++++++
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index aea0d2bffe..24da8120c4 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -8,6 +8,7 @@
 module Annex.Branch (
 	create,
 	update,
+	disableUpdate,
 	get,
 	change,
 	commit,
@@ -136,7 +137,7 @@ update = onceonly $ do
 	c <- filterM changedbranch =<< siblingBranches
 	let (refs, branches) = unzip c
 	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
-		when dirty $ stageJournalFiles
+		when dirty stageJournalFiles
 		g <- gitRepo
 		unless (null branches) $ do
 			showSideAction $ "merging " ++
@@ -164,8 +165,15 @@ update = onceonly $ do
 			return $ not $ L.null diffs
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
-			Annex.changeState setupdated
+			disableUpdate
 			return r
+
+{- Avoids updating the branch. A useful optimisation when the branc
+ - is known to have not changed, or git-annex won't be relying on info
+ - from it. -}
+disableUpdate :: Annex ()
+disableUpdate = Annex.changeState setupdated
+	where
 		setupdated s = s { Annex.branchstate = new }
 			where
 				new = old { branchUpdated = True }
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 5d31770a22..e77b005725 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -19,6 +19,7 @@ import qualified Git
 import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
+import qualified Annex.Branch
 import qualified Utility.Url as Url
 import Utility.TempFile
 import Config
@@ -133,7 +134,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = safely checklocal
 	where
-		checklocal = onLocal r (Annex.Content.inAnnex key)
+		checklocal = onLocal r $ Annex.Content.inAnnex key
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			inannex <- onRemote r (boolSystem, False) "inannex" 
@@ -147,7 +148,11 @@ inAnnex r key
 onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
 	annex <- Annex.new r
-	Annex.eval annex a
+	Annex.eval annex $ do
+		-- No need to update the branch; its data is not used
+		-- for anything onLocal is used to do.
+		Annex.Branch.disableUpdate
+		a
 
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
@@ -175,11 +180,8 @@ copyToRemote r key
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
 		-- run copy from perspective of remote
-		liftIO $ onLocal r $ do
-			ok <- Annex.Content.getViaTmp key $
-				rsyncOrCopyFile r keysrc
-			Annex.Content.saveState
-			return ok
+		liftIO $ onLocal r $ Annex.Content.getViaTmp key $
+			rsyncOrCopyFile r keysrc
 	| Git.repoIsSsh r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
diff --git a/debian/changelog b/debian/changelog
index 81dec6ad06..b17de57ff5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20111026) UNRELEASED; urgency=low
+
+  * Sped up some operations on remotes that are on the same host.
+
+ -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
+
 git-annex (3.20111025) unstable; urgency=low
 
   * A remote can have a annexUrl configured, that is used by git-annex

From c30366e95aa8967112cec0103393ea2b39ddf1c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 14:38:59 -0400
Subject: [PATCH 2380/8313] improve config reading when operating on remote on
 same host

Before the config was read each time onLocal was called, and entirely
redundantly since it's read for same-host remotes on startup.

Also a minor bug fix: When rsyncing to a same-host remote, use the
rsync-options from the repository that the user ran git-annex in, not those of
the receiving repository.
---
 Annex.hs      |  1 +
 Remote/Git.hs | 26 ++++++++++++++++----------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 6877537740..b9e71c9314 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -12,6 +12,7 @@ module Annex (
 	AnnexState(..),
 	OutputType(..),
 	new,
+	newState,
 	run,
 	eval,
 	getState,
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e77b005725..4285be653d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -57,7 +57,8 @@ gen r u _ = do
 	 - cached UUID value. -}
 	let cheap = not $ Git.repoIsUrl r
 	r' <- case (cheap, u) of
-		(True, _) -> tryGitConfigRead r
+		(True, _) -> do
+			tryGitConfigRead r
 		(False, "") -> tryGitConfigRead r
 		_ -> return r
 
@@ -147,8 +148,12 @@ inAnnex r key
  - monad using that repository. -}
 onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
-	annex <- Annex.new r
-	Annex.eval annex $ do
+	-- Avoid re-reading the repository's configuration if it was
+	-- already read.
+	state <- if (M.null $ Git.configMap r)
+		then Annex.new r
+		else return $ Annex.newState r
+	Annex.eval state $ do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.Branch.disableUpdate
@@ -168,7 +173,9 @@ dropKey r key
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
-	| not $ Git.repoIsUrl r = rsyncOrCopyFile r (gitAnnexLocation r key) file
+	| not $ Git.repoIsUrl r = do
+		params <- rsyncParams r
+		rsyncOrCopyFile params (gitAnnexLocation r key) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
@@ -179,9 +186,10 @@ copyToRemote r key
 	| not $ Git.repoIsUrl r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
+		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ Annex.Content.getViaTmp key $
-			rsyncOrCopyFile r keysrc
+			rsyncOrCopyFile params keysrc
 	| Git.repoIsSsh r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key
@@ -200,15 +208,13 @@ rsyncHelper p = do
 
 {- Copys a file with rsync unless both locations are on the same
  - filesystem. Then cp could be faster. -}
-rsyncOrCopyFile :: Git.Repo -> FilePath -> FilePath -> Annex Bool
-rsyncOrCopyFile r src dest = do
+rsyncOrCopyFile :: [CommandParam] -> FilePath -> FilePath -> Annex Bool
+rsyncOrCopyFile rsyncparams src dest = do
 	ss <- liftIO $ getFileStatus $ parentDir src
 	ds <- liftIO $ getFileStatus $ parentDir dest
 	if deviceID ss == deviceID ds
 		then liftIO $ copyFileExternal src dest
-		else do
-			params <- rsyncParams r
-			rsyncHelper $ params ++ [Param src, Param dest]
+		else rsyncHelper $ rsyncparams ++ [Param src, Param dest]
 
 {- Generates rsync parameters that ssh to the remote and asks it
  - to either receive or send the key's content. -}

From f84d66fa15bc746517ba61f2c05beade59c846e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 14:48:53 -0400
Subject: [PATCH 2381/8313] reap in onLocal

Each onLocal call involves a new Annex state, so needs to clean up after it.
---
 Remote/Git.hs                              | 4 +++-
 debian/changelog                           | 2 ++
 doc/bugs/fails_to_handle_lot_of_files.mdwn | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 4285be653d..fd3a612436 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -157,7 +157,9 @@ onLocal r a = do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.Branch.disableUpdate
-		a
+		ret <- a
+		liftIO $ Git.reap
+		return ret
 
 keyUrl :: Git.Repo -> Key -> String
 keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
diff --git a/debian/changelog b/debian/changelog
index b17de57ff5..c32ad39503 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20111026) UNRELEASED; urgency=low
 
   * Sped up some operations on remotes that are on the same host.
+  * copy --to: Fixed leak when ran on many files and a remote on the same
+    host.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/fails_to_handle_lot_of_files.mdwn b/doc/bugs/fails_to_handle_lot_of_files.mdwn
index bc5d92aeca..470a5180f0 100644
--- a/doc/bugs/fails_to_handle_lot_of_files.mdwn
+++ b/doc/bugs/fails_to_handle_lot_of_files.mdwn
@@ -439,3 +439,7 @@ Please note the "_n TIMES_" comments. It's how often I got the same error messag
     4455
     % find source -type f | wc -l
     13554
+
+> Fixed unreaped process leak.
+> (This has nothing to do with NTFS). Ran test with 10k files
+> [[done]] --[[Joey]] 

From 83d11c03c44779c000759040eca8797e70e4765c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 15:24:58 -0400
Subject: [PATCH 2382/8313] wording

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index c32ad39503..e177298da8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,7 @@
 git-annex (3.20111026) UNRELEASED; urgency=low
 
   * Sped up some operations on remotes that are on the same host.
-  * copy --to: Fixed leak when ran on many files and a remote on the same
+  * copy --to: Fixed leak when copying many files to a remote on the same
     host.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400

From 66194684acaf8dc5c72e5a163465b42050cf9212 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 15:47:11 -0400
Subject: [PATCH 2383/8313] uninit: Add guard against being run with the
 git-annex branch checked out.

---
 Command/Uninit.hs                               | 17 ++++++++++++++++-
 debian/changelog                                |  1 +
 ...un_when_branch_git-annex_is_checked_out.mdwn |  2 ++
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 7b3b4fe329..8214c4208e 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,6 +7,8 @@
 
 module Command.Uninit where
 
+import qualified Data.ByteString.Lazy.Char8 as B
+
 import Common.Annex
 import Command
 import qualified Git
@@ -21,7 +23,20 @@ command = [repoCommand "uninit" paramPaths seek
         "de-initialize git-annex and clean out repository"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit startUnannex, withNothing start]
+seek = [withNothing startCheck, withFilesInGit startUnannex, withNothing start]
+
+startCheck :: CommandStart
+startCheck = do
+	b <- current_branch	
+	when (b == Annex.Branch.name) $ error $
+		"cannot uninit when the " ++ b ++ " branch is checked out"
+	stop
+	where
+		current_branch = do
+			g <- gitRepo
+			b <- liftIO $
+				Git.pipeRead g [Params "rev-parse --abbrev-ref HEAD"]
+			return $ head $ lines $ B.unpack b
 
 startUnannex :: FilePath -> CommandStart
 startUnannex file = do
diff --git a/debian/changelog b/debian/changelog
index e177298da8..dd72fc296d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * Sped up some operations on remotes that are on the same host.
   * copy --to: Fixed leak when copying many files to a remote on the same
     host.
+  * uninit: Add guard against being run with the git-annex branch checked out.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
index d140219364..e4e407ec87 100644
--- a/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
+++ b/doc/bugs/uninit_should_not_run_when_branch_git-annex_is_checked_out.mdwn
@@ -11,3 +11,5 @@ Both of which is logical. The best thing would be if git-annex refused to run un
 
 
 Richard
+
+> [[done]] --[[Joey]] 

From 5b74b130a39d8c45e7d24520d838d6c1635582c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 16:31:35 -0400
Subject: [PATCH 2384/8313] refactored and generalized pre-command sanity
 checking

---
 CmdLine.hs            |  7 +------
 Command.hs            | 36 ++++++++++++++++++++++--------------
 Command/Add.hs        |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Copy.hs       |  2 +-
 Command/Describe.hs   |  2 +-
 Command/Drop.hs       |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  2 +-
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/Fsck.hs       |  2 +-
 Command/Get.hs        |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  3 +--
 Command/InitRemote.hs |  7 +++----
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  3 ++-
 Command/Merge.hs      |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Move.hs       |  2 +-
 Command/PreCommit.hs  |  3 ++-
 Command/RecvKey.hs    |  2 +-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/SetKey.hs     |  2 +-
 Command/Status.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  3 ++-
 Command/Uninit.hs     | 14 +++++++-------
 Command/Unlock.hs     |  6 ++++--
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  2 +-
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 38 files changed, 73 insertions(+), 67 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index b1c9c17285..1037401e00 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -21,7 +21,6 @@ import qualified Git
 import Annex.Content
 import Command
 import Options
-import Init
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
@@ -41,7 +40,7 @@ parseCmd argv header cmds options = do
 		[] -> error $ "unknown command" ++ usagemsg
 		[command] -> do
 			_ <- sequence flags
-			checkCmdEnviron command
+			checkCommand command
 			prepCommand command (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
@@ -53,10 +52,6 @@ parseCmd argv header cmds options = do
 		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
 		usagemsg = "\n\n" ++ usage header cmds options
 
-{- Checks that the command can be run in the current environment. -}
-checkCmdEnviron :: Command -> Annex ()
-checkCmdEnviron command = when (cmdusesrepo command) ensureInitialized
-
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
 usage header cmds options =
diff --git a/Command.hs b/Command.hs
index f282791fb3..d19dad2601 100644
--- a/Command.hs
+++ b/Command.hs
@@ -18,42 +18,38 @@ import Logs.Location
 import Config
 import Backend
 import Limit
+import Init
 
-{- A command runs in four stages.
+{- A command runs in these stages.
  -
- - 0. The seek stage takes the parameters passed to the command,
+ - a. The check stage is run once and should error out if anything
+ -    prevents the command from running. -}
+type CommandCheck = Annex ()
+{- b. The seek stage takes the parameters passed to the command,
  -    looks through the repo to find the ones that are relevant
  -    to that command (ie, new files to add), and generates
  -    a list of start stage actions. -}
 type CommandSeek = [String] -> Annex [CommandStart]
-{- 1. The start stage is run before anything is printed about the
+{- c. The start stage is run before anything is printed about the
   -   command, is passed some input, and can early abort it
   -   if the input does not make sense. It should run quickly and
   -   should not modify Annex state. -}
 type CommandStart = Annex (Maybe CommandPerform)
-{- 2. The perform stage is run after a message is printed about the command
+{- d. The perform stage is run after a message is printed about the command
  -    being run, and it should be where the bulk of the work happens. -}
 type CommandPerform = Annex (Maybe CommandCleanup)
-{- 3. The cleanup stage is run only if the perform stage succeeds, and it
+{- e. The cleanup stage is run only if the perform stage succeeds, and it
  -    returns the overall success/fail of the command. -}
 type CommandCleanup = Annex Bool
 
 data Command = Command {
-	cmdusesrepo :: Bool,
 	cmdname :: String,
 	cmdparams :: String,
+	cmdcheck :: CommandCheck,
 	cmdseek :: [CommandSeek],
 	cmddesc :: String
 }
 
-{- Most commands operate on files in a git repo. -}
-repoCommand :: String -> String -> [CommandSeek] -> String -> Command
-repoCommand = Command True
-
-{- Others can run anywhere. -}
-standaloneCommand :: String -> String -> [CommandSeek] -> String -> Command
-standaloneCommand = Command False
-
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
 next a = return $ Just a
@@ -62,6 +58,18 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
+needsNothing :: CommandCheck
+needsNothing = return ()
+
+{- Most commands will check this, as they need to be run in an initialized
+ - repo. -}
+needsRepo :: CommandCheck
+needsRepo = ensureInitialized
+
+{- Checks that the command can be run in the current environment. -}
+checkCommand :: Command -> Annex ()
+checkCommand Command { cmdcheck = check } = check
+
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
diff --git a/Command/Add.hs b/Command/Add.hs
index bfddd72ee7..255e787b74 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -19,7 +19,7 @@ import Utility.Touch
 import Backend
 
 command :: [Command]
-command = [repoCommand "add" paramPaths seek "add files to annex"]
+command = [Command "add" paramPaths needsRepo seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index f32b5b86a9..8deb79541f 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -20,7 +20,7 @@ import Annex.Content
 import Logs.Web
 
 command :: [Command]
-command = [repoCommand "addurl" (paramRepeating paramUrl) seek
+command = [Command "addurl" (paramRepeating paramUrl) needsRepo seek
 	"add urls to annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 43315f67ce..35a939b38f 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.UUID
 
 command :: [Command]
-command = [repoCommand "configlist" paramNothing seek
+command = [Command "configlist" paramNothing needsRepo seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index d7625ccdb0..2f10d981c0 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -11,7 +11,7 @@ import Command
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "copy" paramPaths seek
+command = [Command "copy" paramPaths needsRepo seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 65cd8d0bf5..9184ede9cf 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.UUID
 
 command :: [Command]
-command = [repoCommand "describe" (paramPair paramRemote paramDesc) seek
+command = [Command "describe" (paramPair paramRemote paramDesc) needsRepo seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index dc858fb29b..7309c2acdb 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -17,7 +17,7 @@ import Annex.Content
 import Config
 
 command :: [Command]
-command = [repoCommand "drop" paramPaths seek
+command = [Command "drop" paramPaths needsRepo seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index fde6ce02ea..9e35548569 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -14,7 +14,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "dropkey" (paramRepeating paramKey) seek
+command = [Command "dropkey" (paramRepeating paramKey) needsRepo seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 0050685565..019fab0769 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -21,7 +21,7 @@ import Types.Key
 type UnusedMap = M.Map String Key
 
 command :: [Command]
-command = [repoCommand "dropunused" (paramRepeating paramNumber) seek
+command = [Command "dropunused" (paramRepeating paramNumber) needsRepo seek
 	"drop unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 98501078e8..5b13c742ad 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Limit
 
 command :: [Command]
-command = [repoCommand "find" paramPaths seek "lists available files"]
+command = [Command "find" paramPaths needsRepo seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 5b6f1f7a47..5e58f07332 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "fix" paramPaths seek
+command = [Command "fix" paramPaths needsRepo seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 1b05d71fb8..30243964e5 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Types.Key
 
 command :: [Command]
-command = [repoCommand "fromkey" paramPath seek
+command = [Command "fromkey" paramPath needsRepo seek
 	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 1c1687a002..0098a822db 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -21,7 +21,7 @@ import Utility.FileMode
 import Config
 
 command :: [Command]
-command = [repoCommand "fsck" paramPaths seek "check for problems"]
+command = [Command "fsck" paramPaths needsRepo seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index acf7e07228..d9596c3fe8 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -15,7 +15,7 @@ import Annex.Content
 import qualified Command.Move
 
 command :: [Command]
-command = [repoCommand "get" paramPaths seek
+command = [Command "get" paramPaths needsRepo seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 773693b65f..b4db849c9b 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "inannex" (paramRepeating paramKey) seek
+command = [Command "inannex" (paramRepeating paramKey) needsRepo seek
 		"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Init.hs b/Command/Init.hs
index 3dd4493295..06bdf4ad56 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -14,8 +14,7 @@ import Logs.UUID
 import Init
 	
 command :: [Command]
-command = [standaloneCommand "init" paramDesc seek
-		"initialize git-annex"]
+command = [Command "init" paramDesc needsNothing seek "initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 073ba72f90..8f97199b77 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -17,10 +17,9 @@ import qualified Types.Remote as R
 import Annex.UUID
 
 command :: [Command]
-command = [repoCommand "initremote"
-	(paramPair paramName $
-		paramOptional $ paramRepeating paramKeyValue) seek
-	"sets up a special (non-git) remote"]
+command = [Command "initremote"
+	(paramPair paramName $ 	paramOptional $ paramRepeating paramKeyValue)
+	needsRepo seek "sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index c6c66a1585..bf3b125592 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Backend
 	
 command :: [Command]
-command = [repoCommand "lock" paramPaths seek "undo unlock command"]
+command = [Command "lock" paramPaths needsRepo seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 48cba63f9e..6fbc6930ba 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -23,7 +23,8 @@ import qualified Utility.Dot as Dot
 data Link = Link Git.Repo Git.Repo
 
 command :: [Command]
-command = [repoCommand "map" paramNothing seek "generate map of repositories"]
+command = [Command "map" paramNothing needsNothing seek
+	"generate map of repositories"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Merge.hs b/Command/Merge.hs
index eef2f3857a..2b7162946a 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -12,7 +12,7 @@ import Command
 import qualified Annex.Branch
 
 command :: [Command]
-command = [repoCommand "merge" paramNothing seek
+command = [Command "merge" paramNothing needsRepo seek
 		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 8167ac96eb..e3956c5aa7 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -17,7 +17,7 @@ import Backend
 import Logs.Web
 
 command :: [Command]
-command = [repoCommand "migrate" paramPaths seek
+command = [Command "migrate" paramPaths needsRepo seek
 	"switch data to different backend"]
 
 seek :: [CommandSeek]
diff --git a/Command/Move.hs b/Command/Move.hs
index a816aacde5..ae5e0e1d45 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -17,7 +17,7 @@ import qualified Remote
 import Annex.UUID
 
 command :: [Command]
-command = [repoCommand "move" paramPaths seek
+command = [Command "move" paramPaths needsRepo seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index b6323e2b79..50bc2662e1 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -13,7 +13,8 @@ import qualified Command.Fix
 import Backend
 
 command :: [Command]
-command = [repoCommand "pre-commit" paramPaths seek "run by git pre-commit hook"]
+command = [Command "pre-commit" paramPaths needsRepo seek
+	"run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index d3b77d8ac1..9dc436a681 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [repoCommand "recvkey" paramKey seek
+command = [Command "recvkey" paramKey needsRepo seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 5d60977eb4..f6a2f639c7 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "semitrust" (paramRepeating paramRemote) seek
+command = [Command "semitrust" (paramRepeating paramRemote) needsRepo seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index ad47cd009f..e8ba3ae79c 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [repoCommand "sendkey" paramKey seek
+command = [Command "sendkey" paramKey needsRepo seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index b707e0b918..51f344f20f 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -13,7 +13,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "setkey" paramPath seek
+command = [Command "setkey" paramPath needsRepo seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Status.hs b/Command/Status.hs
index 70282b79ee..155e53ee2c 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -39,7 +39,7 @@ data StatInfo = StatInfo
 type StatState = StateT StatInfo Annex
 
 command :: [Command]
-command = [repoCommand "status" paramNothing seek
+command = [Command "status" paramNothing needsRepo seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index eeeadc9afe..1af458630f 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "trust" (paramRepeating paramRemote) seek
+command = [Command "trust" (paramRepeating paramRemote) needsRepo seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 083984d0c5..cdaa790c0f 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -19,7 +19,8 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 
 command :: [Command]
-command = [repoCommand "unannex" paramPaths seek "undo accidential add command"]
+command = [Command "unannex" paramPaths needsRepo seek
+	"undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8214c4208e..60e86cc039 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -19,18 +19,15 @@ import qualified Annex.Branch
 import Annex.Content
 
 command :: [Command]
-command = [repoCommand "uninit" paramPaths seek 
+command = [Command "uninit" paramPaths check seek 
         "de-initialize git-annex and clean out repository"]
 
-seek :: [CommandSeek]
-seek = [withNothing startCheck, withFilesInGit startUnannex, withNothing start]
-
-startCheck :: CommandStart
-startCheck = do
+check :: Annex ()
+check = do
+	needsRepo
 	b <- current_branch	
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ b ++ " branch is checked out"
-	stop
 	where
 		current_branch = do
 			g <- gitRepo
@@ -38,6 +35,9 @@ startCheck = do
 				Git.pipeRead g [Params "rev-parse --abbrev-ref HEAD"]
 			return $ head $ lines $ B.unpack b
 
+seek :: [CommandSeek]
+seek = [withFilesInGit startUnannex, withNothing start]
+
 startUnannex :: FilePath -> CommandStart
 startUnannex file = do
 	-- Force fast mode before running unannex. This way, if multiple
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 9b568b5a6b..c89b61de70 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -15,9 +15,11 @@ import Utility.FileMode
 
 command :: [Command]
 command =
-	[ repoCommand "unlock" paramPaths seek "unlock files for modification"
-	, repoCommand "edit" paramPaths seek "same as unlock"
+	[ c "unlock" "unlock files for modification"
+	, c "edit" "same as unlock"
 	]
+	where
+		c n = Command n paramPaths needsRepo seek
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index f8bf498f24..7d65c1af95 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "untrust" (paramRepeating paramRemote) seek
+command = [Command "untrust" (paramRepeating paramRemote) needsRepo seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a901747521..5cef829d6e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -28,7 +28,7 @@ import qualified Annex.Branch
 import Annex.CatFile
 
 command :: [Command]
-command = [repoCommand "unused" paramNothing seek
+command = [Command "unused" paramNothing needsRepo seek
 	"look for unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 90d3a4e95b..77d15c9306 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -13,7 +13,7 @@ import Upgrade
 import Annex.Version
 
 command :: [Command]
-command = [standaloneCommand "upgrade" paramNothing seek
+command = [Command "upgrade" paramNothing needsNothing seek
 	"upgrade repository layout"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index 5ac87099bb..dae9a31d3b 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,7 @@ import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
 command :: [Command]
-command = [standaloneCommand "version" paramNothing seek "show version info"]
+command = [Command "version" paramNothing needsNothing seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index b1646ae69a..71b3ad96b1 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -14,7 +14,7 @@ import Remote
 import Logs.Trust
 
 command :: [Command]
-command = [repoCommand "whereis" paramPaths seek
+command = [Command "whereis" paramPaths needsRepo seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]

From 1826b3bd67b8d3e146fd833d4cb500b03d1bf096 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:01:46 -0400
Subject: [PATCH 2385/8313] cleanup

---
 Annex/Branch.hs | 2 +-
 test.hs         | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 24da8120c4..645a2de762 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -168,7 +168,7 @@ update = onceonly $ do
 			disableUpdate
 			return r
 
-{- Avoids updating the branch. A useful optimisation when the branc
+{- Avoids updating the branch. A useful optimisation when the branch
  - is known to have not changed, or git-annex won't be relying on info
  - from it. -}
 disableUpdate :: Annex ()
diff --git a/test.hs b/test.hs
index 7d4c4293e1..0ef7449162 100644
--- a/test.hs
+++ b/test.hs
@@ -31,7 +31,6 @@ import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
 import qualified Logs.Location
-import qualified Logs.UUID
 import qualified Logs.UUIDBased
 import qualified Logs.Trust
 import qualified Logs.Remote

From c879eb873ec2fda1ec7d76c0de27d9ace57ba6e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:03:36 -0400
Subject: [PATCH 2386/8313] do commit location changes to remote in copy --to

test suite pointed out that if a file was copied from B to A, and then
A cloned, the clone ought to immediatly know it can get the file from A.
---
 Remote/Git.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index fd3a612436..b0138901d7 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -190,8 +190,11 @@ copyToRemote r key
 		let keysrc = gitAnnexLocation g key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
-		liftIO $ onLocal r $ Annex.Content.getViaTmp key $
-			rsyncOrCopyFile params keysrc
+		liftIO $ onLocal r $ do
+			ok <- Annex.Content.getViaTmp key $
+				rsyncOrCopyFile params keysrc
+			Annex.Content.saveState
+			return ok
 	| Git.repoIsSsh r = do
 		g <- gitRepo
 		let keysrc = gitAnnexLocation g key

From b955238ec7464b12c793d543fc51308c8b213e08 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:56:54 -0400
Subject: [PATCH 2387/8313] Fail if --from or --to is passed to commands that
 do not support them.

---
 Command.hs            | 32 ++++++++++++++++++++++----------
 Command/Add.hs        |  2 +-
 Command/AddUrl.hs     |  2 +-
 Command/ConfigList.hs |  2 +-
 Command/Describe.hs   |  2 +-
 Command/Drop.hs       |  2 +-
 Command/DropKey.hs    |  2 +-
 Command/DropUnused.hs |  4 ++--
 Command/Find.hs       |  2 +-
 Command/Fix.hs        |  2 +-
 Command/FromKey.hs    |  2 +-
 Command/Fsck.hs       |  2 +-
 Command/Get.hs        |  2 +-
 Command/InAnnex.hs    |  2 +-
 Command/Init.hs       |  2 +-
 Command/InitRemote.hs |  2 +-
 Command/Lock.hs       |  2 +-
 Command/Map.hs        |  2 +-
 Command/Merge.hs      |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/PreCommit.hs  |  2 +-
 Command/RecvKey.hs    |  2 +-
 Command/Semitrust.hs  |  2 +-
 Command/SendKey.hs    |  2 +-
 Command/SetKey.hs     |  2 +-
 Command/Status.hs     |  2 +-
 Command/Trust.hs      |  2 +-
 Command/Unannex.hs    |  2 +-
 Command/Uninit.hs     |  3 +--
 Command/Unlock.hs     |  2 +-
 Command/Untrust.hs    |  2 +-
 Command/Unused.hs     |  2 +-
 Command/Upgrade.hs    |  2 +-
 Command/Version.hs    |  2 +-
 Command/Whereis.hs    |  2 +-
 debian/changelog      |  1 +
 36 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/Command.hs b/Command.hs
index d19dad2601..5690118c40 100644
--- a/Command.hs
+++ b/Command.hs
@@ -22,8 +22,8 @@ import Init
 
 {- A command runs in these stages.
  -
- - a. The check stage is run once and should error out if anything
- -    prevents the command from running. -}
+ - a. The check stage runs checks, that error out if
+ -    anything prevents the command from running. -}
 type CommandCheck = Annex ()
 {- b. The seek stage takes the parameters passed to the command,
  -    looks through the repo to find the ones that are relevant
@@ -58,14 +58,6 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-needsNothing :: CommandCheck
-needsNothing = return ()
-
-{- Most commands will check this, as they need to be run in an initialized
- - repo. -}
-needsRepo :: CommandCheck
-needsRepo = ensureInitialized
-
 {- Checks that the command can be run in the current environment. -}
 checkCommand :: Command -> Annex ()
 checkCommand Command { cmdcheck = check } = check
@@ -239,3 +231,23 @@ autoCopies key vs numcopiesattr a = do
 			(_, have) <- trustPartition UnTrusted =<< keyLocations key
 			if length have `vs` needed then a else stop
 		else a
+
+{- Checks -}
+defaultChecks :: CommandCheck
+defaultChecks = noFrom >> noTo >> needsRepo
+
+noChecks :: CommandCheck
+noChecks = return ()
+
+needsRepo :: CommandCheck
+needsRepo = ensureInitialized
+
+noFrom :: CommandCheck
+noFrom = do
+	v <- Annex.getState Annex.fromremote
+	unless (v == Nothing) $ error "cannot use --from with this command"
+
+noTo :: CommandCheck
+noTo = do
+	v <- Annex.getState Annex.toremote
+	unless (v == Nothing) $ error "cannot use --to with this command"
diff --git a/Command/Add.hs b/Command/Add.hs
index 255e787b74..33b636bed7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -19,7 +19,7 @@ import Utility.Touch
 import Backend
 
 command :: [Command]
-command = [Command "add" paramPaths needsRepo seek "add files to annex"]
+command = [Command "add" paramPaths defaultChecks seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 8deb79541f..72e29ff60e 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -20,7 +20,7 @@ import Annex.Content
 import Logs.Web
 
 command :: [Command]
-command = [Command "addurl" (paramRepeating paramUrl) needsRepo seek
+command = [Command "addurl" (paramRepeating paramUrl) defaultChecks seek
 	"add urls to annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 35a939b38f..645d1523ca 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.UUID
 
 command :: [Command]
-command = [Command "configlist" paramNothing needsRepo seek
+command = [Command "configlist" paramNothing defaultChecks seek
 		"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 9184ede9cf..cd5da302e9 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.UUID
 
 command :: [Command]
-command = [Command "describe" (paramPair paramRemote paramDesc) needsRepo seek
+command = [Command "describe" (paramPair paramRemote paramDesc) defaultChecks seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 7309c2acdb..02f44915ce 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -17,7 +17,7 @@ import Annex.Content
 import Config
 
 command :: [Command]
-command = [Command "drop" paramPaths needsRepo seek
+command = [Command "drop" paramPaths defaultChecks seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 9e35548569..3e666b9aba 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -14,7 +14,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [Command "dropkey" (paramRepeating paramKey) needsRepo seek
+command = [Command "dropkey" (paramRepeating paramKey) defaultChecks seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 019fab0769..1236fb8230 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -21,8 +21,8 @@ import Types.Key
 type UnusedMap = M.Map String Key
 
 command :: [Command]
-command = [Command "dropunused" (paramRepeating paramNumber) needsRepo seek
-	"drop unused file content"]
+command = [Command "dropunused" (paramRepeating paramNumber) (noTo >> needsRepo)
+	seek "drop unused file content"]
 
 seek :: [CommandSeek]
 seek = [withUnusedMaps]
diff --git a/Command/Find.hs b/Command/Find.hs
index 5b13c742ad..291904ec03 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Limit
 
 command :: [Command]
-command = [Command "find" paramPaths needsRepo seek "lists available files"]
+command = [Command "find" paramPaths defaultChecks seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 5e58f07332..090558d521 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Annex.Content
 
 command :: [Command]
-command = [Command "fix" paramPaths needsRepo seek
+command = [Command "fix" paramPaths defaultChecks seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 30243964e5..a3e96b20da 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Types.Key
 
 command :: [Command]
-command = [Command "fromkey" paramPath needsRepo seek
+command = [Command "fromkey" paramPath defaultChecks seek
 	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 0098a822db..d025095b1b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -21,7 +21,7 @@ import Utility.FileMode
 import Config
 
 command :: [Command]
-command = [Command "fsck" paramPaths needsRepo seek "check for problems"]
+command = [Command "fsck" paramPaths defaultChecks seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index d9596c3fe8..21ff73bc4e 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -15,7 +15,7 @@ import Annex.Content
 import qualified Command.Move
 
 command :: [Command]
-command = [Command "get" paramPaths needsRepo seek
+command = [Command "get" paramPaths (noTo >> needsRepo) seek
 		"make content of annexed files available"]
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index b4db849c9b..7a5735b742 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.Content
 
 command :: [Command]
-command = [Command "inannex" (paramRepeating paramKey) needsRepo seek
+command = [Command "inannex" (paramRepeating paramKey) defaultChecks seek
 		"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Init.hs b/Command/Init.hs
index 06bdf4ad56..6646432002 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -14,7 +14,7 @@ import Logs.UUID
 import Init
 	
 command :: [Command]
-command = [Command "init" paramDesc needsNothing seek "initialize git-annex"]
+command = [Command "init" paramDesc noChecks seek "initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 8f97199b77..cea1acc8d8 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -19,7 +19,7 @@ import Annex.UUID
 command :: [Command]
 command = [Command "initremote"
 	(paramPair paramName $ 	paramOptional $ paramRepeating paramKeyValue)
-	needsRepo seek "sets up a special (non-git) remote"]
+	defaultChecks seek "sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index bf3b125592..8f0bd78ebe 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -13,7 +13,7 @@ import qualified Annex.Queue
 import Backend
 	
 command :: [Command]
-command = [Command "lock" paramPaths needsRepo seek "undo unlock command"]
+command = [Command "lock" paramPaths defaultChecks seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 6fbc6930ba..05cc9d794c 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -23,7 +23,7 @@ import qualified Utility.Dot as Dot
 data Link = Link Git.Repo Git.Repo
 
 command :: [Command]
-command = [Command "map" paramNothing needsNothing seek
+command = [Command "map" paramNothing noChecks seek
 	"generate map of repositories"]
 
 seek :: [CommandSeek]
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 2b7162946a..33d4c8ffc5 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -12,7 +12,7 @@ import Command
 import qualified Annex.Branch
 
 command :: [Command]
-command = [Command "merge" paramNothing needsRepo seek
+command = [Command "merge" paramNothing defaultChecks seek
 		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index e3956c5aa7..ac8f042ba2 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -17,7 +17,7 @@ import Backend
 import Logs.Web
 
 command :: [Command]
-command = [Command "migrate" paramPaths needsRepo seek
+command = [Command "migrate" paramPaths defaultChecks seek
 	"switch data to different backend"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 50bc2662e1..5dac4f5339 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -13,7 +13,7 @@ import qualified Command.Fix
 import Backend
 
 command :: [Command]
-command = [Command "pre-commit" paramPaths needsRepo seek
+command = [Command "pre-commit" paramPaths defaultChecks seek
 	"run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 9dc436a681..3415de526e 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [Command "recvkey" paramKey needsRepo seek
+command = [Command "recvkey" paramKey defaultChecks seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index f6a2f639c7..4f61531ff0 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "semitrust" (paramRepeating paramRemote) needsRepo seek
+command = [Command "semitrust" (paramRepeating paramRemote) defaultChecks seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index e8ba3ae79c..5118a009b4 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 command :: [Command]
-command = [Command "sendkey" paramKey needsRepo seek
+command = [Command "sendkey" paramKey defaultChecks seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 51f344f20f..a60f539973 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -13,7 +13,7 @@ import Logs.Location
 import Annex.Content
 
 command :: [Command]
-command = [Command "setkey" paramPath needsRepo seek
+command = [Command "setkey" paramPath defaultChecks seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Status.hs b/Command/Status.hs
index 155e53ee2c..df79d4a7fe 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -39,7 +39,7 @@ data StatInfo = StatInfo
 type StatState = StateT StatInfo Annex
 
 command :: [Command]
-command = [Command "status" paramNothing needsRepo seek
+command = [Command "status" paramNothing defaultChecks seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 1af458630f..17b689c345 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "trust" (paramRepeating paramRemote) needsRepo seek
+command = [Command "trust" (paramRepeating paramRemote) defaultChecks seek
 	"trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index cdaa790c0f..76100c8c3c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -19,7 +19,7 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 
 command :: [Command]
-command = [Command "unannex" paramPaths needsRepo seek
+command = [Command "unannex" paramPaths defaultChecks seek
 	"undo accidential add command"]
 
 seek :: [CommandSeek]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 60e86cc039..b2046ec414 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -19,12 +19,11 @@ import qualified Annex.Branch
 import Annex.Content
 
 command :: [Command]
-command = [Command "uninit" paramPaths check seek 
+command = [Command "uninit" paramPaths (check >> defaultChecks) seek 
         "de-initialize git-annex and clean out repository"]
 
 check :: Annex ()
 check = do
-	needsRepo
 	b <- current_branch	
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ b ++ " branch is checked out"
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index c89b61de70..be1d052984 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -19,7 +19,7 @@ command =
 	, c "edit" "same as unlock"
 	]
 	where
-		c n = Command n paramPaths needsRepo seek
+		c n = Command n paramPaths defaultChecks seek
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 7d65c1af95..5a2505a10f 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -13,7 +13,7 @@ import qualified Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "untrust" (paramRepeating paramRemote) needsRepo seek
+command = [Command "untrust" (paramRepeating paramRemote) defaultChecks seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5cef829d6e..d2b45ed6b7 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -28,7 +28,7 @@ import qualified Annex.Branch
 import Annex.CatFile
 
 command :: [Command]
-command = [Command "unused" paramNothing needsRepo seek
+command = [Command "unused" paramNothing (noTo >> needsRepo) seek
 	"look for unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 77d15c9306..9ca3c8d2be 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -13,7 +13,7 @@ import Upgrade
 import Annex.Version
 
 command :: [Command]
-command = [Command "upgrade" paramNothing needsNothing seek
+command = [Command "upgrade" paramNothing noChecks seek
 	"upgrade repository layout"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index dae9a31d3b..905a48a51e 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,7 @@ import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
 command :: [Command]
-command = [Command "version" paramNothing needsNothing seek "show version info"]
+command = [Command "version" paramNothing noChecks seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 71b3ad96b1..06a894fd3f 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -14,7 +14,7 @@ import Remote
 import Logs.Trust
 
 command :: [Command]
-command = [Command "whereis" paramPaths needsRepo seek
+command = [Command "whereis" paramPaths defaultChecks seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
diff --git a/debian/changelog b/debian/changelog
index dd72fc296d..d100bd4e85 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * copy --to: Fixed leak when copying many files to a remote on the same
     host.
   * uninit: Add guard against being run with the git-annex branch checked out.
+  * Fail if --from or --to is passed to commands that do not support them.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From 2888562724acd06ed2acd5c5a318a88d9e33d484 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 18:59:25 -0400
Subject: [PATCH 2388/8313] update

---
 ...t:_support_drop__44___find_on_special_remotes.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
index 2dc735e99f..982f398ae9 100644
--- a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
@@ -1,6 +1,17 @@
 Currently there is no way to drop files, or list what files are available, on a special remote.
 It would be good if "git annex drop" and "git annex find" supported the --from argument.
 
+> I agree, drop should support --from.
+> 
+> To find files *believed* to be present in a given remote, use
+> `git annex find --in remote`
+> Note that it might show out of date info, since it does not actually go
+> check the current contents of the remote. The only reason to support
+> `find --from` would be to always check, but I don't think that's needed.
+> --[[Joey]] 
+
 For commands that don't support the --from argument, it would also be nice to print an error.
 Currently running "git annex drop --from usbdrive" doesn't behave as hoped and instead drops
 all content from the local annex.
+
+> This is done now. --[[Joey]] 

From f66f97c90e5692ab34cb95e6facdec194a72456b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 19:04:12 -0400
Subject: [PATCH 2389/8313] document the little-known get --from

---
 doc/git-annex.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1b8464b314..3dcc117056 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -72,6 +72,9 @@ subdirectories).
   will involve copying them from another repository, or downloading them,
   or transferring them from some kind of key-value store.
 
+  Normally git-annex will choose which repository to copy the content from,
+  but you can override this using the --from option.
+
 * drop [path ...]
 
   Drops the content of annexed files from this repository. 

From 33e18d3d02865ac0677fc1f22de2352b92f184a8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 27 Oct 2011 19:10:10 -0400
Subject: [PATCH 2390/8313] cleanup

---
 Command/Drop.hs    | 11 ++++-------
 Command/Unannex.hs |  7 +------
 2 files changed, 5 insertions(+), 13 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index 02f44915ce..a84d2efcc3 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -23,8 +23,6 @@ command = [Command "drop" paramPaths defaultChecks seek
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
-{- Indicates a file's content is not wanted anymore, and should be removed
- - if it's safe to do so. -}
 start :: FilePath -> Maybe Int -> CommandStart
 start file numcopies = isAnnexed file $ \(key, _) -> do
 	present <- inAnnex key
@@ -36,7 +34,7 @@ start file numcopies = isAnnexed file $ \(key, _) -> do
 
 perform :: Key -> Maybe Int -> CommandPerform
 perform key numcopies = do
-	success <- dropKey key numcopies
+	success <- canDropKey key numcopies
 	if success
 		then next $ cleanup key
 		else stop
@@ -48,10 +46,9 @@ cleanup key = do
 	return True
 
 {- Checks remotes to verify that enough copies of a key exist to allow
- - for a key to be safely removed (with no data loss), and fails with an
- - error if not. -}
-dropKey :: Key -> Maybe Int -> Annex Bool
-dropKey key numcopiesM = do
+ - for a key to be safely removed (with no data loss). -}
+canDropKey :: Key -> Maybe Int -> Annex Bool
+canDropKey key numcopiesM = do
 	force <- Annex.getState Annex.force
 	if force || numcopiesM == Just 0
 		then return True
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 76100c8c3c..b39dc0a5f0 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -9,7 +9,6 @@ module Command.Unannex where
 
 import Common.Annex
 import Command
-import qualified Command.Drop
 import qualified Annex
 import qualified Annex.Queue
 import Utility.FileMode
@@ -44,11 +43,7 @@ start file = isAnnexed file $ \(key, _) -> do
 		else stop
 
 perform :: FilePath -> Key -> CommandPerform
-perform file key = do
-	ok <- Command.Drop.dropKey key (Just 0) -- always remove
-	if ok
-		then next $ cleanup file key
-		else stop
+perform file key = next $ cleanup file key
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do

From 6c31e3a8c3c5ab92ca2e84b4c166f32d02a50f4f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Oct 2011 17:26:38 -0400
Subject: [PATCH 2391/8313] drop --from is now supported to remove file content
 from a remote.

---
 Command/Drop.hs                               | 125 +++++++++++++-----
 Command/DropUnused.hs                         |   5 +-
 Command/Move.hs                               |  25 +---
 Remote.hs                                     |  15 ++-
 debian/changelog                              |   1 +
 ...rt_drop__44___find_on_special_remotes.mdwn |   1 +
 doc/git-annex.mdwn                            |   5 +-
 7 files changed, 114 insertions(+), 63 deletions(-)

diff --git a/Command/Drop.hs b/Command/Drop.hs
index a84d2efcc3..27049fc67d 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -11,76 +11,131 @@ import Common.Annex
 import Command
 import qualified Remote
 import qualified Annex
+import Annex.UUID
 import Logs.Location
 import Logs.Trust
 import Annex.Content
 import Config
 
 command :: [Command]
-command = [Command "drop" paramPaths defaultChecks seek
+command = [Command "drop" paramPaths (noTo >> needsRepo) seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
 
 start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) -> do
+start file numcopies = isAnnexed file $ \(key, _) ->
+	autoCopies key (>) numcopies $ do
+		from <- Annex.getState Annex.fromremote
+		case from of
+			Nothing -> startLocal file numcopies key
+			Just name -> do
+				remote <- Remote.byName name
+				u <- getUUID
+				if Remote.uuid remote == u
+					then startLocal file numcopies key
+					else startRemote file numcopies key remote
+
+startLocal :: FilePath -> Maybe Int -> Key -> CommandStart
+startLocal file numcopies key = do
 	present <- inAnnex key
 	if present
-		then autoCopies key (>) numcopies $ do
+		then do
 			showStart "drop" file
-			next $ perform key numcopies
+			next $ performLocal key numcopies
 		else stop
 
-perform :: Key -> Maybe Int -> CommandPerform
-perform key numcopies = do
-	success <- canDropKey key numcopies
+startRemote :: FilePath -> Maybe Int -> Key -> Remote.Remote Annex -> CommandStart
+startRemote file numcopies key remote = do
+	showStart "drop" file
+	next $ performRemote key numcopies remote
+
+performLocal :: Key -> Maybe Int -> CommandPerform
+performLocal key numcopies = do
+	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
+	untrusteduuids <- trustGet UnTrusted
+	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
+	success <- canDropKey key numcopies trusteduuids tocheck []
 	if success
-		then next $ cleanup key
+		then next $ cleanupLocal key
 		else stop
 
-cleanup :: Key -> CommandCleanup
-cleanup key = do
+performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
+performRemote key numcopies remote = do
+	-- Filter the remote it's being dropped from out of the lists of
+	-- places assumed to have the key, and places to check.
+	-- When the local repo has the key, that's one additional copy.
+	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
+	present <- inAnnex key
+	u <- getUUID
+	let have = filter (/= uuid) $
+		if present then u:trusteduuids else trusteduuids
+	untrusteduuids <- trustGet UnTrusted
+	let tocheck = filter (/= remote) $
+		Remote.remotesWithoutUUID remotes (have++untrusteduuids)
+	success <- canDropKey key numcopies have tocheck [uuid]
+	if success
+		then next $ cleanupRemote key remote
+		else stop
+	where
+		uuid = Remote.uuid remote
+
+cleanupLocal :: Key -> CommandCleanup
+cleanupLocal key = do
 	whenM (inAnnex key) $ removeAnnex key
 	logStatus key InfoMissing
 	return True
 
-{- Checks remotes to verify that enough copies of a key exist to allow
- - for a key to be safely removed (with no data loss). -}
-canDropKey :: Key -> Maybe Int -> Annex Bool
-canDropKey key numcopiesM = do
+cleanupRemote :: Key -> Remote.Remote Annex -> CommandCleanup
+cleanupRemote key remote = do
+	ok <- Remote.removeKey remote key
+	-- better safe than sorry: assume the remote dropped the key
+	-- even if it seemed to fail; the failure could have occurred
+	-- after it really dropped it
+	Remote.remoteHasKey remote key False
+	return ok
+
+{- Checks specified remotes to verify that enough copies of a key exist to
+ - allow it to be safely removed (with no data loss). Can be provided with
+ - some locations where the key is known/assumed to be present. -}
+canDropKey :: Key -> Maybe Int -> [UUID] -> [Remote.Remote Annex] -> [UUID] -> Annex Bool
+canDropKey key numcopiesM have check skip = do
 	force <- Annex.getState Annex.force
 	if force || numcopiesM == Just 0
 		then return True
 		else do
-			(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
-			untrusteduuids <- trustGet UnTrusted
-			let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
-			numcopies <- getNumCopies numcopiesM
-			findcopies numcopies trusteduuids tocheck []
+			need <- getNumCopies numcopiesM
+			findCopies key need skip have check
+
+findCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote.Remote Annex] -> Annex Bool
+findCopies key need skip = helper []
 	where
-		findcopies need have [] bad
+		helper bad have []
 			| length have >= need = return True
-			| otherwise = notEnoughCopies need have bad
-		findcopies need have (r:rs) bad
+			| otherwise = notEnoughCopies key need have skip bad
+		helper bad have (r:rs)
 			| length have >= need = return True
 			| otherwise = do
 				let u = Remote.uuid r
 				let duplicate = u `elem` have
 				haskey <- Remote.hasKey r key
 				case (duplicate, haskey) of
-					(False, Right True)	-> findcopies need (u:have) rs bad
-					(False, Left _)		-> findcopies need have rs (r:bad)
-					_			-> findcopies need have rs bad
-		notEnoughCopies need have bad = do
-			unsafe
-			showLongNote $
-				"Could only verify the existence of " ++
-				show (length have) ++ " out of " ++ show need ++ 
-				" necessary copies"
-			Remote.showTriedRemotes bad
-			Remote.showLocations key have
-			hint
-			return False
+					(False, Right True) -> helper bad (u:have) rs
+					(False, Left _)     -> helper (r:bad) have rs
+					_                   -> helper bad have rs
+
+notEnoughCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote.Remote Annex] -> Annex Bool
+notEnoughCopies key need have skip bad = do
+	unsafe
+	showLongNote $
+		"Could only verify the existence of " ++
+		show (length have) ++ " out of " ++ show need ++ 
+		" necessary copies"
+	Remote.showTriedRemotes bad
+	Remote.showLocations key (have++skip)
+	hint
+	return False
+	where
 		unsafe = showNote "unsafe"
 		hint = showLongNote "(Use --force to override this check, or adjust annex.numcopies.)"
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 1236fb8230..46f2dc9f7f 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -13,7 +13,6 @@ import Common.Annex
 import Command
 import qualified Annex
 import qualified Command.Drop
-import qualified Command.Move
 import qualified Remote
 import qualified Git
 import Types.Key
@@ -56,8 +55,8 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 		dropremote name = do
 			r <- Remote.byName name
 			showAction $ "from " ++ Remote.name r
-			next $ Command.Move.fromCleanup r True key
-		droplocal = Command.Drop.perform key (Just 0) -- force drop
+			next $ Command.Drop.cleanupRemote key r
+		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
 
 performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
diff --git a/Command/Move.hs b/Command/Move.hs
index ae5e0e1d45..2a7402a0d6 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import qualified Command.Drop
 import qualified Annex
-import Logs.Location
 import Annex.Content
 import qualified Remote
 import Annex.UUID
@@ -49,18 +48,6 @@ showMoveAction :: Bool -> FilePath -> Annex ()
 showMoveAction True file = showStart "move" file
 showMoveAction False file = showStart "copy" file
 
-{- Used to log a change in a remote's having a key. The change is logged
- - in the local repo, not on the remote. The process of transferring the
- - key to the remote, or removing the key from it *may* log the change
- - on the remote, but this cannot be relied on. -}
-remoteHasKey :: Remote.Remote Annex -> Key -> Bool -> Annex ()
-remoteHasKey remote key present	= do
-	let remoteuuid = Remote.uuid remote
-	g <- gitRepo
-	logChange g key remoteuuid status
-	where
-		status = if present then InfoPresent else InfoMissing
-
 {- Moves (or copies) the content of an annexed file to a remote.
  -
  - If the remote already has the content, it is still removed from
@@ -108,9 +95,9 @@ toPerform dest move key = do
 		Right True -> next $ toCleanup dest move key
 toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
 toCleanup dest move key = do
-	remoteHasKey dest key True
+	Remote.remoteHasKey dest key True
 	if move
-		then Command.Drop.cleanup key
+		then Command.Drop.cleanupLocal key
 		else return True
 
 {- Moves (or copies) the content of an annexed file from a remote
@@ -140,11 +127,5 @@ fromPerform src move key = do
 				then next $ fromCleanup src move key
 				else stop -- fail
 fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
-fromCleanup src True key = do
-	ok <- Remote.removeKey src key
-	-- better safe than sorry: assume the src dropped the key
-	-- even if it seemed to fail; the failure could have occurred
-	-- after it really dropped it
-	remoteHasKey src key False
-	return ok
+fromCleanup src True key = Command.Drop.cleanupRemote key src
 fromCleanup _ False _ = return True
diff --git a/Remote.hs b/Remote.hs
index 49fa63cf92..6ce4fe0186 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -25,7 +25,8 @@ module Remote (
 	nameToUUID,
 	showTriedRemotes,
 	showLocations,
-	forceTrust
+	forceTrust,
+	remoteHasKey
 ) where
 
 import qualified Data.Map as M
@@ -225,3 +226,15 @@ forceTrust level remotename = do
 	r <- nameToUUID remotename
 	Annex.changeState $ \s ->
 		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
+
+{- Used to log a change in a remote's having a key. The change is logged
+ - in the local repo, not on the remote. The process of transferring the
+ - key to the remote, or removing the key from it *may* log the change
+ - on the remote, but this cannot always be relied on. -}
+remoteHasKey :: Remote Annex -> Key -> Bool -> Annex ()
+remoteHasKey remote key present	= do
+	let remoteuuid = uuid remote
+	g <- gitRepo
+	logChange g key remoteuuid status
+	where
+		status = if present then InfoPresent else InfoMissing
diff --git a/debian/changelog b/debian/changelog
index d100bd4e85..237abb83ff 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     host.
   * uninit: Add guard against being run with the git-annex branch checked out.
   * Fail if --from or --to is passed to commands that do not support them.
+  * drop --from is now supported to remove file content from a remote.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
index 982f398ae9..24cacbf71c 100644
--- a/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
+++ b/doc/bugs/wishlist:_support_drop__44___find_on_special_remotes.mdwn
@@ -2,6 +2,7 @@ Currently there is no way to drop files, or list what files are available, on a
 It would be good if "git annex drop" and "git annex find" supported the --from argument.
 
 > I agree, drop should support --from.
+>> [[done]] --[[Joey]] 
 > 
 > To find files *believed* to be present in a given remote, use
 > `git annex find --in remote`
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3dcc117056..dc0b49ab2c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -80,8 +80,9 @@ subdirectories).
   Drops the content of annexed files from this repository. 
 
   git-annex will refuse to drop content if it cannot verify it is
-  safe to do so. At least one copy of content needs to exist in another
-  remote. This can be overridden with the --force switch.
+  safe to do so. This can be overridden with the --force switch.
+
+  To drop content from a remote, specify --from.
 
 * move [path ...]
   

From ab738a403a0b5f34b9c679c6c9a92b279babdf42 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Oct 2011 19:49:01 -0400
Subject: [PATCH 2392/8313] status: Now always shows the current repository,
 even when it does not appear in uuid.log.

---
 Logs/UUID.hs     | 13 +++++++++++--
 debian/changelog |  2 ++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 8a93b43fef..c05c4e3481 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -24,6 +24,7 @@ import Data.Time.Clock.POSIX
 import Common.Annex
 import qualified Annex.Branch
 import Logs.UUIDBased
+import qualified Annex.UUID
 
 {- Filename of uuid.log. -}
 logfile :: FilePath
@@ -36,6 +37,14 @@ describeUUID uuid desc = do
 	Annex.Branch.change logfile $
 		showLog id . changeLog ts uuid desc . parseLog Just
 
-{- Read the uuidLog into a simple Map -}
+{- Read the uuidLog into a simple Map.
+ -
+ - The UUID of the current repository is included explicitly, since
+ - it may not have been described and so otherwise would not appear. -}
 uuidMap :: Annex (M.Map UUID String)
-uuidMap = (simpleMap . parseLog Just) <$> Annex.Branch.get logfile
+uuidMap = do
+	m <- (simpleMap . parseLog Just) <$> Annex.Branch.get logfile
+	u <- Annex.UUID.getUUID
+	return $ M.insertWith' preferold u "" m
+	where
+		preferold = flip const
diff --git a/debian/changelog b/debian/changelog
index 237abb83ff..4a873af945 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * uninit: Add guard against being run with the git-annex branch checked out.
   * Fail if --from or --to is passed to commands that do not support them.
   * drop --from is now supported to remove file content from a remote.
+  * status: Now always shows the current repository, even when it does not
+    appear in uuid.log.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From aeb4e285eb2b3b880ab6b91c5b1f7fa29d7182cb Mon Sep 17 00:00:00 2001
From: "http://nicolas-schodet.myopenid.com/"
 
Date: Sat, 29 Oct 2011 01:01:38 +0000
Subject: [PATCH 2393/8313] as far as I know, --bwlimit does not accept unit,
 it always takes kilobytes

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dc0b49ab2c..27b4cead8b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -542,7 +542,7 @@ Here are all the supported configuration settings.
 * `remote..annex-bup-split-options`
 
   Options to pass to bup split when storing content in this remote.
-  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k"
+  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100"
   (There is no corresponding option for bup join.)
 
 * `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`

From f65100b408c9b2efe35af340f9438b3e84d69028 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 28 Oct 2011 21:24:24 -0400
Subject: [PATCH 2394/8313] Revert "as far as I know, --bwlimit does not accept
 unit, it always takes kilobytes"

This reverts commit aeb4e285eb2b3b880ab6b91c5b1f7fa29d7182cb.

bup's --bwlimit does take a unit, unlike rsync's.
---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 27b4cead8b..dc0b49ab2c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -542,7 +542,7 @@ Here are all the supported configuration settings.
 * `remote..annex-bup-split-options`
 
   Options to pass to bup split when storing content in this remote.
-  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100"
+  For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k"
   (There is no corresponding option for bup join.)
 
 * `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`

From 7588b042daaecec5de167971b3123d08db849942 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 15:27:06 +0000
Subject: [PATCH 2395/8313]

---
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/bugs/uninit_does_not_work_in_old_repos.mdwn

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
new file mode 100644
index 0000000000..a6f7858639
--- /dev/null
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -0,0 +1,6 @@
+As uninit does not need to actually write out any data, just remove it, it should be possible to uninit in old stores.
+
+    % git annex uninit
+    git-annex: Repository version 2 is not supported. Upgrade this repository: git-annex upgrade
+
+If the repo happens to be broken, this essentially locks in data.

From ce4029c973ac6cab7f30f7452723a0750b0b7ea5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 15:30:10 +0000
Subject: [PATCH 2396/8313] Added a comment

---
 .../comment_1_bc0619c6e17139df74639448aa6a0f72._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment b/doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment
new file mode 100644
index 0000000000..7a1ea582b0
--- /dev/null
+++ b/doc/bugs/uninit_does_not_work_in_old_repos/comment_1_bc0619c6e17139df74639448aa6a0f72._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-29T15:30:09Z"
+ content="""
+After upgrading the repo, I still have to commit the changes, else git-annex won't let me uninit. Arguably a Good Thing, but I wanted to document it here.
+"""]]

From 0dcbe51ed2e966376f506b1f780c37632bd81860 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 15:36:43 +0000
Subject: [PATCH 2397/8313]

---
 ...t_annex_upgrade_output_is_inconsistent_and_spammy.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn

diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
new file mode 100644
index 0000000000..89701144a4
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
@@ -0,0 +1,8 @@
+Upgrading from v1 to v3:
+
+    upgrade . (v1 to v2...) (moving content...) (updating symlinks...) (moving location logs...) (v2 to v3...) ..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
+      git-annex branch created
+      Be sure to push this branch when pushing to remotes.
+    ok
+
+A whirly would be preferable, imo.

From 4d7802bff71d62ceb674eee6c957c74a1e85a2eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 12:45:47 -0400
Subject: [PATCH 2398/8313] responsen

---
 ..._annex_upgrade_output_is_inconsistent_and_spammy.mdwn | 7 +++++++
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn          | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
index 89701144a4..3d75483dc3 100644
--- a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
@@ -6,3 +6,10 @@ Upgrading from v1 to v3:
     ok
 
 A whirly would be preferable, imo.
+
+> Erm, I'm pretty sure you were the one who asked for there to be some
+> progress dots, Richard.
+> 
+> I'm not particularly interested in implementing a whirley that would only
+> be used in this one place, in code that very few users are going to run
+> again. I could remove the dots.. --[[Joey]]
diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
index a6f7858639..1e1b1fec73 100644
--- a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -4,3 +4,12 @@ As uninit does not need to actually write out any data, just remove it, it shoul
     git-annex: Repository version 2 is not supported. Upgrade this repository: git-annex upgrade
 
 If the repo happens to be broken, this essentially locks in data.
+
+> No, because you can always check out the version of git-annex you need
+> for that repository.
+> 
+> uninit, as implemented, runs unannex on every file and then does some
+> cleanup. The cleanup does not need to write state, but the unannex does.
+> And it depends on the object directory layout, which has changed between
+> versions. So supporting old versions in this code would complicate it
+> quite a lot. I don't want to go there. --[[Joey]]

From 39c304be4376eab387851eb94414654ce61a14ee Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 16:57:10 +0000
Subject: [PATCH 2399/8313]

---
 ...led_checksum_when_less_copies_than_required_are_found.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
new file mode 100644
index 0000000000..860a61d2cb
--- /dev/null
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -0,0 +1,4 @@
+    (checksum...) failed
+    fsck foo (fixing location log) 
+      Only 1 of 2 trustworthy copies exist of foo
+      Back it up with git-annex copy.

From 36355e815e0a9c95a4aaea5b091e5c1f06ad6999 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:03:27 +0000
Subject: [PATCH 2400/8313] Added a comment

---
 ...comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment

diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment
new file mode 100644
index 0000000000..4f9565517d
--- /dev/null
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy/comment_1_3a01c81efba321b0e46d1bc0426ad8d1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2011-10-29T17:03:26Z"
+ content="""
+I could dig it out, but I am sure I said dots are fine and a whirly better.
+
+Still, WONTFIX is fine.
+"""]]

From 3868a65663231b929a1c09b02185e85dc70e59e5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:07:44 +0000
Subject: [PATCH 2401/8313]

---
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
index 1e1b1fec73..0e5175abe3 100644
--- a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -13,3 +13,5 @@ If the repo happens to be broken, this essentially locks in data.
 > And it depends on the object directory layout, which has changed between
 > versions. So supporting old versions in this code would complicate it
 > quite a lot. I don't want to go there. --[[Joey]]
+
+>>Requiring a version upgrade for unannex is fine. Yet, I see a problem when a git repo is broken; you are stuck without being able to uninit. In this case an uninit that does nothing but undo the symlinking would be useful. -- Richard

From 978ab987d50211ae43864409c4ad9fb77d7f6974 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 13:16:10 -0400
Subject: [PATCH 2402/8313] pebak

---
 ...led_checksum_when_less_copies_than_required_are_found.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 860a61d2cb..101fc6e36b 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -2,3 +2,7 @@
     fsck foo (fixing location log) 
       Only 1 of 2 trustworthy copies exist of foo
       Back it up with git-annex copy.
+
+> You've given me severely partial output, and no test case, but until
+> it says "fsck foo", the output is pertaining to some other file than foo.
+> As far as I can see, there is no bug here. [[done]] --[[Joey]]

From ad3b46221463b9cb5fa3b0abaef9cbfcf3facdae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 13:17:37 -0400
Subject: [PATCH 2403/8313] sheesh. seriously?

---
 Upgrade/V2.hs                                                   | 1 -
 .../git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn    | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 1ad41266a7..67f0205c14 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -78,7 +78,6 @@ inject source dest = do
 	new <- liftIO (readFile $ olddir g  source)
 	Annex.Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
-	showProgress
 
 logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
diff --git a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
index 3d75483dc3..ec8a10915e 100644
--- a/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
+++ b/doc/bugs/git_annex_upgrade_output_is_inconsistent_and_spammy.mdwn
@@ -12,4 +12,4 @@ A whirly would be preferable, imo.
 > 
 > I'm not particularly interested in implementing a whirley that would only
 > be used in this one place, in code that very few users are going to run
-> again. I could remove the dots.. --[[Joey]]
+> again. I could remove the dots.. [[done]] --[[Joey]]

From 0dfe750c0b2f5b11fba82a077055580864cc1c27 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 13:21:28 -0400
Subject: [PATCH 2404/8313] close

---
 doc/bugs/uninit_does_not_work_in_old_repos.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
index 0e5175abe3..d3df061487 100644
--- a/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
+++ b/doc/bugs/uninit_does_not_work_in_old_repos.mdwn
@@ -15,3 +15,6 @@ If the repo happens to be broken, this essentially locks in data.
 > quite a lot. I don't want to go there. --[[Joey]]
 
 >>Requiring a version upgrade for unannex is fine. Yet, I see a problem when a git repo is broken; you are stuck without being able to uninit. In this case an uninit that does nothing but undo the symlinking would be useful. -- Richard
+
+>>> As I said, version 2 of git-annex is still there for people who need
+>>> it for whatever reason. [[done]] --[[Joey]]

From e9d142e9e8ae7095b0ab19ef70d0b5edac33947e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:27:48 +0000
Subject: [PATCH 2405/8313]

---
 ..._to_provide_UUID_when_running___96__git_annex_init__96__.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn

diff --git a/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
new file mode 100644
index 0000000000..a1a8f2827b
--- /dev/null
+++ b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
@@ -0,0 +1 @@
+As there's no way to permanently hide remotes and I have to recreate two repos now, I would love to be able to re-use the old UUIDs to remove clutter.

From 75e99b16f9ef4ee3bc35791f76e57525375e3364 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 17:45:07 +0000
Subject: [PATCH 2406/8313]

---
 ...ed_checksum_when_less_copies_than_required_are_found.mdwn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 101fc6e36b..a5bde3c1ac 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -5,4 +5,7 @@
 
 > You've given me severely partial output, and no test case, but until
 > it says "fsck foo", the output is pertaining to some other file than foo.
-> As far as I can see, there is no bug here. [[done]] --[[Joey]]
+> As far as I can see, there is no bug here. --[[Joey]]
+
+>> Sorry, I thought it would be obvious, but that's no excuse for not providing additional explanation.
+>> The problem is that fsck tells me a file's fsck has failed without printing extra details. In this case, the checksum is OK while I don't have enough copies to satisfy the fsck. The fact that I don't have enough copies is obviously relevant, but I would still like to know if the checksums are OK. -- Richard

From 0d92aca1aabbbeb2d50d91807312ff6039971751 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 14:17:02 -0400
Subject: [PATCH 2407/8313] responsen

---
 ...ksum_when_less_copies_than_required_are_found.mdwn | 11 +++++++++--
 ..._UUID_when_running___96__git_annex_init__96__.mdwn |  4 ++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index a5bde3c1ac..3f1d28d3da 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -7,5 +7,12 @@
 > it says "fsck foo", the output is pertaining to some other file than foo.
 > As far as I can see, there is no bug here. --[[Joey]]
 
->> Sorry, I thought it would be obvious, but that's no excuse for not providing additional explanation.
->> The problem is that fsck tells me a file's fsck has failed without printing extra details. In this case, the checksum is OK while I don't have enough copies to satisfy the fsck. The fact that I don't have enough copies is obviously relevant, but I would still like to know if the checksums are OK. -- Richard
+>> Sorry, I thought it would be obvious, but that's no excuse for not
+>> providing additional explanation. The problem is that fsck tells me a
+>> file's fsck has failed without printing extra details. In this case, the
+>> checksum is OK while I don't have enough copies to satisfy the fsck. The
+>> fact that I don't have enough copies is obviously relevant, but I would
+>> still like to know if the checksums are OK. -- Richard
+
+>>> I think you're misreading the truncated output you posted. The actual,
+>>> full output would make much more sense. --[[Joey]]
diff --git a/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
index a1a8f2827b..0dc9ec08a2 100644
--- a/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
+++ b/doc/bugs/wishlist:_allow_users_to_provide_UUID_when_running___96__git_annex_init__96__.mdwn
@@ -1 +1,5 @@
 As there's no way to permanently hide remotes and I have to recreate two repos now, I would love to be able to re-use the old UUIDs to remove clutter.
+
+> git-annex already provides a way to do this: Copy `.git/config` from the
+> original repo (or use `git-config` to set `annex.uuid`) *before* running
+> `git annex init`. [[done]] --[[Joey]]

From d46b8c053e0fbea824a9f9faec4dc58d3f777a07 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 18:28:14 +0000
Subject: [PATCH 2408/8313] Added a comment

---
 ..._342d1ac07573c7ef4e27f003a692e261._comment | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment

diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment
new file mode 100644
index 0000000000..3bb92919f9
--- /dev/null
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level/comment_2_342d1ac07573c7ef4e27f003a692e261._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 2"
+ date="2011-10-29T18:28:13Z"
+ content="""
+Possible solutions:
+
+This:
+
+    trusted repositories:
+            UUID -- foo
+    semi-trusted repositories:
+            UUID -- bar
+    untrusted repositories:
+            UUID -- baz
+
+or this:
+
+            UUID -- trusted      -- foo
+            UUID -- semi-trusted -- bar
+            UUID -- untrusted    -- baz
+
+or this:
+
+    known repositories (!/*/X):
+            UUID -- ! foo
+            UUID -- * bar
+            UUID -- X baz
+
+If you want to reformat this output, putting 'here', 'origin', etc into fixed formatting might make sense, as well. -- Richard
+"""]]

From b7d2fd8186a8bcd4615d17b9bf3a96b010b55f68 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 19:09:19 +0000
Subject: [PATCH 2409/8313]

---
 ...iled_checksum_when_less_copies_than_required_are_found.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 3f1d28d3da..57c80d7ca3 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -16,3 +16,6 @@
 
 >>> I think you're misreading the truncated output you posted. The actual,
 >>> full output would make much more sense. --[[Joey]]
+
+>>>> No. I have a total of 14908 annex keys, 3333 of which are on a remote. The only message other than 'checksum OK' and the above is 'git-annex: 11577 failed'.
+>>>> I checked several files manually, their checksums are OK so `git annex status` is reporting those files as completely failed when they "only" miss copies. -- Richard

From f97c783283847c6cc4516673fe638b4d551e671d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 15:19:05 -0400
Subject: [PATCH 2410/8313] clean up check selection code

This new approach allows filtering out checks from the default set that are
not appropriate for a command, rather than having to list every check
that is appropriate. It also reduces some boilerplate.

Haskell does not define Eq for functions, so I had to go a long way around
with each check having a unique id. Meh.
---
 CmdLine.hs            |  6 ++--
 Command.hs            | 47 +++++++++++++++++++------------
 Command/Add.hs        |  4 +--
 Command/AddUrl.hs     |  5 ++--
 Command/ConfigList.hs |  6 ++--
 Command/Copy.hs       |  5 ++--
 Command/Describe.hs   |  4 +--
 Command/Drop.hs       |  4 +--
 Command/DropKey.hs    |  4 +--
 Command/DropUnused.hs |  4 +--
 Command/Find.hs       |  4 +--
 Command/Fix.hs        |  4 +--
 Command/FromKey.hs    |  5 ++--
 Command/Fsck.hs       |  4 +--
 Command/Get.hs        |  6 ++--
 Command/InAnnex.hs    |  6 ++--
 Command/Init.hs       |  5 ++--
 Command/InitRemote.hs |  8 +++---
 Command/Lock.hs       |  4 +--
 Command/Map.hs        |  6 ++--
 Command/Merge.hs      |  4 +--
 Command/Migrate.hs    |  5 ++--
 Command/Move.hs       |  5 ++--
 Command/PreCommit.hs  |  5 ++--
 Command/RecvKey.hs    |  4 +--
 Command/Semitrust.hs  |  4 +--
 Command/SendKey.hs    |  4 +--
 Command/SetKey.hs     |  4 +--
 Command/Status.hs     |  4 +--
 Command/Trust.hs      |  5 ++--
 Command/Unannex.hs    |  5 ++--
 Command/Uninit.hs     |  4 +--
 Command/Unlock.hs     |  6 ++--
 Command/Untrust.hs    |  4 +--
 Command/Unused.hs     |  4 +--
 Command/Upgrade.hs    |  6 ++--
 Command/Version.hs    |  5 ++--
 Command/Whereis.hs    |  4 +--
 GitAnnex.hs           | 64 +++++++++++++++++++++----------------------
 git-annex-shell.hs    | 10 +++----
 40 files changed, 154 insertions(+), 143 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 1037401e00..9f1ded498c 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -38,10 +38,10 @@ parseCmd argv header cmds options = do
 	when (null params) $ error $ "missing command" ++ usagemsg
 	case lookupCmd (head params) of
 		[] -> error $ "unknown command" ++ usagemsg
-		[command] -> do
+		[cmd] -> do
 			_ <- sequence flags
-			checkCommand command
-			prepCommand command (drop 1 params)
+			checkCommand cmd
+			prepCommand cmd (drop 1 params)
 		_ -> error "internal error: multiple matching commands"
 	where
 		getopt = case getOpt Permute options argv of
diff --git a/Command.hs b/Command.hs
index 5690118c40..b039403ca0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -24,7 +24,9 @@ import Init
  -
  - a. The check stage runs checks, that error out if
  -    anything prevents the command from running. -}
-type CommandCheck = Annex ()
+data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () }
+instance Eq CommandCheck where
+	a == b = idCheck a == idCheck b
 {- b. The seek stage takes the parameters passed to the command,
  -    looks through the repo to find the ones that are relevant
  -    to that command (ie, new files to add), and generates
@@ -43,9 +45,9 @@ type CommandPerform = Annex (Maybe CommandCleanup)
 type CommandCleanup = Annex Bool
 
 data Command = Command {
+	cmdcheck :: [CommandCheck],
 	cmdname :: String,
 	cmdparams :: String,
-	cmdcheck :: CommandCheck,
 	cmdseek :: [CommandSeek],
 	cmddesc :: String
 }
@@ -58,9 +60,9 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-{- Checks that the command can be run in the current environment. -}
-checkCommand :: Command -> Annex ()
-checkCommand Command { cmdcheck = check } = check
+{- Generates a command with the common checks. -}
+command :: String -> String -> [CommandSeek] -> String -> Command
+command = Command commonChecks
 
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
@@ -232,22 +234,33 @@ autoCopies key vs numcopiesattr a = do
 			if length have `vs` needed then a else stop
 		else a
 
-{- Checks -}
-defaultChecks :: CommandCheck
-defaultChecks = noFrom >> noTo >> needsRepo
+{- Common checks for commands, and an interface to selectively remove them,
+ - or add others. -}
+commonChecks :: [CommandCheck]
+commonChecks = [fromOpt, toOpt, repoExists]
 
-noChecks :: CommandCheck
-noChecks = return ()
+repoExists :: CommandCheck
+repoExists = CommandCheck 0 ensureInitialized
 
-needsRepo :: CommandCheck
-needsRepo = ensureInitialized
-
-noFrom :: CommandCheck
-noFrom = do
+fromOpt :: CommandCheck
+fromOpt = CommandCheck 1 $ do
 	v <- Annex.getState Annex.fromremote
 	unless (v == Nothing) $ error "cannot use --from with this command"
 
-noTo :: CommandCheck
-noTo = do
+toOpt :: CommandCheck
+toOpt = CommandCheck 2 $ do
 	v <- Annex.getState Annex.toremote
 	unless (v == Nothing) $ error "cannot use --to with this command"
+
+checkCommand :: Command -> Annex ()
+checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
+
+dontCheck :: CommandCheck -> Command -> Command
+dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
+
+addCheck :: Annex () -> Command -> Command
+addCheck check cmd = mutateCheck cmd $
+	\c -> CommandCheck (length c + 100) check : c
+
+mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
+mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }
diff --git a/Command/Add.hs b/Command/Add.hs
index 33b636bed7..82287be0bf 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -18,8 +18,8 @@ import Annex.Content
 import Utility.Touch
 import Backend
 
-command :: [Command]
-command = [Command "add" paramPaths defaultChecks seek "add files to annex"]
+def :: [Command]
+def = [command "add" paramPaths seek "add files to annex"]
 
 {- Add acts on both files not checked into git yet, and unlocked files. -}
 seek :: [CommandSeek]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 72e29ff60e..e974d06a14 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -19,9 +19,8 @@ import qualified Backend.URL
 import Annex.Content
 import Logs.Web
 
-command :: [Command]
-command = [Command "addurl" (paramRepeating paramUrl) defaultChecks seek
-	"add urls to annex"]
+def :: [Command]
+def = [command "addurl" (paramRepeating paramUrl) seek "add urls to annex"]
 
 seek :: [CommandSeek]
 seek = [withStrings start]
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index 645d1523ca..cbc6e801b0 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -11,9 +11,9 @@ import Common.Annex
 import Command
 import Annex.UUID
 
-command :: [Command]
-command = [Command "configlist" paramNothing defaultChecks seek
-		"outputs relevant git configuration"]
+def :: [Command]
+def = [command "configlist" paramNothing seek
+	"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 2f10d981c0..8316b7cabf 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -10,8 +10,9 @@ module Command.Copy where
 import Command
 import qualified Command.Move
 
-command :: [Command]
-command = [Command "copy" paramPaths needsRepo seek
+def :: [Command]
+def = [dontCheck toOpt $ dontCheck fromOpt $ 
+	command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Describe.hs b/Command/Describe.hs
index cd5da302e9..882a0e1bb9 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Remote
 import Logs.UUID
 
-command :: [Command]
-command = [Command "describe" (paramPair paramRemote paramDesc) defaultChecks seek
+def :: [Command]
+def = [command "describe" (paramPair paramRemote paramDesc) seek
 	"change description of a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 27049fc67d..2267bd9416 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -17,8 +17,8 @@ import Logs.Trust
 import Annex.Content
 import Config
 
-command :: [Command]
-command = [Command "drop" paramPaths (noTo >> needsRepo) seek
+def :: [Command]
+def = [dontCheck fromOpt $ command "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index 3e666b9aba..d00bb6c83e 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -13,8 +13,8 @@ import qualified Annex
 import Logs.Location
 import Annex.Content
 
-command :: [Command]
-command = [Command "dropkey" (paramRepeating paramKey) defaultChecks seek
+def :: [Command]
+def = [command "dropkey" (paramRepeating paramKey) seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 46f2dc9f7f..d2eb3df710 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -19,8 +19,8 @@ import Types.Key
 
 type UnusedMap = M.Map String Key
 
-command :: [Command]
-command = [Command "dropunused" (paramRepeating paramNumber) (noTo >> needsRepo)
+def :: [Command]
+def = [dontCheck fromOpt $ command "dropunused" (paramRepeating paramNumber)
 	seek "drop unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Find.hs b/Command/Find.hs
index 291904ec03..46364c9876 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -12,8 +12,8 @@ import Command
 import Annex.Content
 import Limit
 
-command :: [Command]
-command = [Command "find" paramPaths defaultChecks seek "lists available files"]
+def :: [Command]
+def = [command "find" paramPaths seek "lists available files"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 090558d521..c46ddc7ee0 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Annex.Queue
 import Annex.Content
 
-command :: [Command]
-command = [Command "fix" paramPaths defaultChecks seek
+def :: [Command]
+def = [command "fix" paramPaths seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index a3e96b20da..fe9b5c96a0 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -13,9 +13,8 @@ import qualified Annex.Queue
 import Annex.Content
 import Types.Key
 
-command :: [Command]
-command = [Command "fromkey" paramPath defaultChecks seek
-	"adds a file using a specific key"]
+def :: [Command]
+def = [command "fromkey" paramPath seek "adds a file using a specific key"]
 
 seek :: [CommandSeek]
 seek = [withFilesMissing start]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index d025095b1b..5d2e2ee50b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,8 +20,8 @@ import Utility.DataUnits
 import Utility.FileMode
 import Config
 
-command :: [Command]
-command = [Command "fsck" paramPaths defaultChecks seek "check for problems"]
+def :: [Command]
+def = [command "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/Get.hs b/Command/Get.hs
index 21ff73bc4e..4a0908bdc8 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -14,9 +14,9 @@ import qualified Remote
 import Annex.Content
 import qualified Command.Move
 
-command :: [Command]
-command = [Command "get" paramPaths (noTo >> needsRepo) seek
-		"make content of annexed files available"]
+def :: [Command]
+def = [dontCheck fromOpt $ command "get" paramPaths seek
+	"make content of annexed files available"]
 
 seek :: [CommandSeek]
 seek = [withNumCopies start]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 7a5735b742..9c169d0d78 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -11,9 +11,9 @@ import Common.Annex
 import Command
 import Annex.Content
 
-command :: [Command]
-command = [Command "inannex" (paramRepeating paramKey) defaultChecks seek
-		"checks if keys are present in the annex"]
+def :: [Command]
+def = [command "inannex" (paramRepeating paramKey) seek
+	"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
 seek = [withKeys start]
diff --git a/Command/Init.hs b/Command/Init.hs
index 6646432002..e2a6eb7b03 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -13,8 +13,9 @@ import Annex.UUID
 import Logs.UUID
 import Init
 	
-command :: [Command]
-command = [Command "init" paramDesc noChecks seek "initialize git-annex"]
+def :: [Command]
+def = [dontCheck repoExists $
+	command "init" paramDesc seek "initialize git-annex"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index cea1acc8d8..4ba5b07873 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -16,10 +16,10 @@ import qualified Logs.Remote
 import qualified Types.Remote as R
 import Annex.UUID
 
-command :: [Command]
-command = [Command "initremote"
-	(paramPair paramName $ 	paramOptional $ paramRepeating paramKeyValue)
-	defaultChecks seek "sets up a special (non-git) remote"]
+def :: [Command]
+def = [command "initremote"
+	(paramPair paramName $ paramOptional $ paramRepeating paramKeyValue)
+	seek "sets up a special (non-git) remote"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 8f0bd78ebe..329fd3eff7 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Annex.Queue
 import Backend
 	
-command :: [Command]
-command = [Command "lock" paramPaths defaultChecks seek "undo unlock command"]
+def :: [Command]
+def = [command "lock" paramPaths seek "undo unlock command"]
 
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
diff --git a/Command/Map.hs b/Command/Map.hs
index 05cc9d794c..7e61d2e9e8 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -22,9 +22,9 @@ import qualified Utility.Dot as Dot
 -- a link from the first repository to the second (its remote)
 data Link = Link Git.Repo Git.Repo
 
-command :: [Command]
-command = [Command "map" paramNothing noChecks seek
-	"generate map of repositories"]
+def :: [Command]
+def = [dontCheck repoExists $
+	command "map" paramNothing seek "generate map of repositories"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Merge.hs b/Command/Merge.hs
index 33d4c8ffc5..c1f7e899af 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -11,8 +11,8 @@ import Common.Annex
 import Command
 import qualified Annex.Branch
 
-command :: [Command]
-command = [Command "merge" paramNothing defaultChecks seek
+def :: [Command]
+def = [command "merge" paramNothing seek
 		"auto-merge remote changes into git-annex branch"]
 
 seek :: [CommandSeek]
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index ac8f042ba2..a68582996b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -16,9 +16,8 @@ import qualified Command.Add
 import Backend
 import Logs.Web
 
-command :: [Command]
-command = [Command "migrate" paramPaths defaultChecks seek
-	"switch data to different backend"]
+def :: [Command]
+def = [command "migrate" paramPaths seek "switch data to different backend"]
 
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit start]
diff --git a/Command/Move.hs b/Command/Move.hs
index 2a7402a0d6..5a3ea7172a 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -15,8 +15,9 @@ import Annex.Content
 import qualified Remote
 import Annex.UUID
 
-command :: [Command]
-command = [Command "move" paramPaths needsRepo seek
+def :: [Command]
+def = [dontCheck toOpt $ dontCheck fromOpt $
+	command "move" paramPaths seek
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 5dac4f5339..1949de113f 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -12,9 +12,8 @@ import qualified Command.Add
 import qualified Command.Fix
 import Backend
 
-command :: [Command]
-command = [Command "pre-commit" paramPaths defaultChecks seek
-	"run by git pre-commit hook"]
+def :: [Command]
+def = [command "pre-commit" paramPaths seek "run by git pre-commit hook"]
 
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 3415de526e..5243fa9d4b 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -13,8 +13,8 @@ import CmdLine
 import Annex.Content
 import Utility.RsyncFile
 
-command :: [Command]
-command = [Command "recvkey" paramKey defaultChecks seek
+def :: [Command]
+def = [command "recvkey" paramKey seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Semitrust.hs b/Command/Semitrust.hs
index 4f61531ff0..f8c3062131 100644
--- a/Command/Semitrust.hs
+++ b/Command/Semitrust.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "semitrust" (paramRepeating paramRemote) defaultChecks seek
+def :: [Command]
+def = [command "semitrust" (paramRepeating paramRemote) seek
 	"return repository to default trust level"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 5118a009b4..318ea56d03 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -12,8 +12,8 @@ import Command
 import Annex.Content
 import Utility.RsyncFile
 
-command :: [Command]
-command = [Command "sendkey" paramKey defaultChecks seek
+def :: [Command]
+def = [command "sendkey" paramKey seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index a60f539973..9c31abb083 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -12,8 +12,8 @@ import Command
 import Logs.Location
 import Annex.Content
 
-command :: [Command]
-command = [Command "setkey" paramPath defaultChecks seek
+def :: [Command]
+def = [command "setkey" paramPath seek
 	"sets annexed content for a key using a temp file"]
 
 seek :: [CommandSeek]
diff --git a/Command/Status.hs b/Command/Status.hs
index df79d4a7fe..b5f4956dbf 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -38,8 +38,8 @@ data StatInfo = StatInfo
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
 
-command :: [Command]
-command = [Command "status" paramNothing defaultChecks seek
+def :: [Command]
+def = [command "status" paramNothing seek
 	"shows status information about the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/Trust.hs b/Command/Trust.hs
index 17b689c345..d976b86a8f 100644
--- a/Command/Trust.hs
+++ b/Command/Trust.hs
@@ -12,9 +12,8 @@ import Command
 import qualified Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "trust" (paramRepeating paramRemote) defaultChecks seek
-	"trust a repository"]
+def :: [Command]
+def = [command "trust" (paramRepeating paramRemote) seek "trust a repository"]
 
 seek :: [CommandSeek]
 seek = [withWords start]
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index b39dc0a5f0..825f819390 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -17,9 +17,8 @@ import Annex.Content
 import qualified Git
 import qualified Git.LsFiles as LsFiles
 
-command :: [Command]
-command = [Command "unannex" paramPaths defaultChecks seek
-	"undo accidential add command"]
+def :: [Command]
+def = [command "unannex" paramPaths seek "undo accidential add command"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index b2046ec414..5a6ee0be25 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -18,8 +18,8 @@ import Init
 import qualified Annex.Branch
 import Annex.Content
 
-command :: [Command]
-command = [Command "uninit" paramPaths (check >> defaultChecks) seek 
+def :: [Command]
+def = [addCheck check $ command "uninit" paramPaths seek 
         "de-initialize git-annex and clean out repository"]
 
 check :: Annex ()
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index be1d052984..7ecaf0b7fa 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -13,13 +13,13 @@ import Annex.Content
 import Utility.CopyFile
 import Utility.FileMode
 
-command :: [Command]
-command =
+def :: [Command]
+def =
 	[ c "unlock" "unlock files for modification"
 	, c "edit" "same as unlock"
 	]
 	where
-		c n = Command n paramPaths defaultChecks seek
+		c n = command n paramPaths seek
 
 seek :: [CommandSeek]
 seek = [withFilesInGit start]
diff --git a/Command/Untrust.hs b/Command/Untrust.hs
index 5a2505a10f..e16040e6bb 100644
--- a/Command/Untrust.hs
+++ b/Command/Untrust.hs
@@ -12,8 +12,8 @@ import Command
 import qualified Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "untrust" (paramRepeating paramRemote) defaultChecks seek
+def :: [Command]
+def = [command "untrust" (paramRepeating paramRemote) seek
 	"do not trust a repository"]
 
 seek :: [CommandSeek]
diff --git a/Command/Unused.hs b/Command/Unused.hs
index d2b45ed6b7..11c3f429ec 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -27,8 +27,8 @@ import qualified Remote
 import qualified Annex.Branch
 import Annex.CatFile
 
-command :: [Command]
-command = [Command "unused" paramNothing (noTo >> needsRepo) seek
+def :: [Command]
+def = [dontCheck fromOpt $ command "unused" paramNothing seek
 	"look for unused file content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index 9ca3c8d2be..b39fcd99c2 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -12,9 +12,9 @@ import Command
 import Upgrade
 import Annex.Version
 
-command :: [Command]
-command = [Command "upgrade" paramNothing noChecks seek
-	"upgrade repository layout"]
+def :: [Command]
+def = [dontCheck repoExists $ -- because an old version may not seem to exist
+	command "upgrade" paramNothing seek "upgrade repository layout"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Version.hs b/Command/Version.hs
index 905a48a51e..5a45fd77f2 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -12,8 +12,9 @@ import Command
 import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
-command :: [Command]
-command = [Command "version" paramNothing noChecks seek "show version info"]
+def :: [Command]
+def = [dontCheck repoExists $
+	command "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
 seek = [withNothing start]
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 06a894fd3f..7799af08c6 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -13,8 +13,8 @@ import Command
 import Remote
 import Logs.Trust
 
-command :: [Command]
-command = [Command "whereis" paramPaths defaultChecks seek
+def :: [Command]
+def = [command "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 3cd7207d3e..c07e727fa0 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -54,38 +54,38 @@ import qualified Command.Version
 
 cmds :: [Command]
 cmds = concat
-	[ Command.Add.command
-	, Command.Get.command
-	, Command.Drop.command
-	, Command.Move.command
-	, Command.Copy.command
-	, Command.Unlock.command
-	, Command.Lock.command
-	, Command.Init.command
-	, Command.Describe.command
-	, Command.InitRemote.command
-	, Command.Unannex.command
-	, Command.Uninit.command
-	, Command.PreCommit.command
-	, Command.Trust.command
-	, Command.Untrust.command
-	, Command.Semitrust.command
-	, Command.AddUrl.command
-	, Command.FromKey.command
-	, Command.DropKey.command
-	, Command.SetKey.command
-	, Command.Fix.command
-	, Command.Fsck.command
-	, Command.Unused.command
-	, Command.DropUnused.command
-	, Command.Find.command
-	, Command.Whereis.command
-	, Command.Merge.command
-	, Command.Status.command
-	, Command.Migrate.command
-	, Command.Map.command
-	, Command.Upgrade.command
-	, Command.Version.command
+	[ Command.Add.def
+	, Command.Get.def
+	, Command.Drop.def
+	, Command.Move.def
+	, Command.Copy.def
+	, Command.Unlock.def
+	, Command.Lock.def
+	, Command.Init.def
+	, Command.Describe.def
+	, Command.InitRemote.def
+	, Command.Unannex.def
+	, Command.Uninit.def
+	, Command.PreCommit.def
+	, Command.Trust.def
+	, Command.Untrust.def
+	, Command.Semitrust.def
+	, Command.AddUrl.def
+	, Command.FromKey.def
+	, Command.DropKey.def
+	, Command.SetKey.def
+	, Command.Fix.def
+	, Command.Fsck.def
+	, Command.Unused.def
+	, Command.DropUnused.def
+	, Command.Find.def
+	, Command.Whereis.def
+	, Command.Merge.def
+	, Command.Status.def
+	, Command.Migrate.def
+	, Command.Map.def
+	, Command.Upgrade.def
+	, Command.Version.def
 	]
 
 options :: [Option]
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 41cb72d7e2..328d7b1006 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -23,15 +23,15 @@ import qualified Command.SendKey
 
 cmds_readonly :: [Command]
 cmds_readonly = concat
-	[ Command.ConfigList.command
-	, Command.InAnnex.command
-	, Command.SendKey.command
+	[ Command.ConfigList.def
+	, Command.InAnnex.def
+	, Command.SendKey.def
 	]
 
 cmds_notreadonly :: [Command]
 cmds_notreadonly = concat
-	[ Command.RecvKey.command
-	, Command.DropKey.command
+	[ Command.RecvKey.def
+	, Command.DropKey.def
 	]
 
 cmds :: [Command]

From 6c3b87f0de14d4545dc02484e8258fd863dcaf53 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 15:40:32 -0400
Subject: [PATCH 2411/8313] add a tip

---
 ..._to_do_when_a_repository_is_corrupted.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn

diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn
new file mode 100644
index 0000000000..80cb046d90
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted.mdwn
@@ -0,0 +1,22 @@
+A git-annex repository on a removable USB drive is great, until the cable
+falls out at the wrong time and git's repository gets trashed. The way
+git checksums everything and the poor quality of USB media makes this
+perhaps more likely than you would expect. If this happens to you,
+here's a way to recover that makes the most of whatever data is left
+on the drive.
+
+* First, run `git fsck`. If it does not report any problems, your data
+  is fine, and you don't need to proceed further.
+* So `git fsck` says the git repository is corrupted. But probably the data
+  git-annex stored is fine. Your first step is to clone another copy
+  of the git repository from somewhere else. Let's call this clone
+  "$good", and the corrupted repository "$bad".
+* Preserve your git configuration changes, and the `annex.uuid` setting:
+  `mv $bad/.git/config $good/.git/config`
+* Move annexed data into the new repository: `mkdir $good/.git/annex; mv
+  $bad/.git/annex/objects $good/.git/annex/objects`
+* Reinitalize git-annex: `cd $good; git annex init`
+* Check for any problems with the annexed data: `cd $good; git annex fsck`
+* Now you can remove the corrupted repository, the new one is ready to use.
+
+--[[Joey]]

From a93aa2e51ebd88db2e6a88e8dc76e9a34d01b362 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 15:46:16 -0400
Subject: [PATCH 2412/8313] responsen

---
 ..._checksum_when_less_copies_than_required_are_found.mdwn | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 57c80d7ca3..faf67c243e 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -18,4 +18,9 @@
 >>> full output would make much more sense. --[[Joey]]
 
 >>>> No. I have a total of 14908 annex keys, 3333 of which are on a remote. The only message other than 'checksum OK' and the above is 'git-annex: 11577 failed'.
->>>> I checked several files manually, their checksums are OK so `git annex status` is reporting those files as completely failed when they "only" miss copies. -- Richard
+>>>> I checked several files manually, their checksums are OK so `git annex
+>>>> fsck` is reporting those files as completely failed when they "only" miss copies. -- Richard
+
+>>>>> fsck considers not enough copies to be a failure condition; it prints
+>>>>> error messages about it etc. That has nothing to do with checksums.
+>>>>> --[[Joey]]

From 3f322161788b3fc648083f9970cb31cdff03eda7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 20:02:21 +0000
Subject: [PATCH 2413/8313]

---
 ...sum_when_less_copies_than_required_are_found.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index faf67c243e..95848456d1 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -24,3 +24,15 @@
 >>>>> fsck considers not enough copies to be a failure condition; it prints
 >>>>> error messages about it etc. That has nothing to do with checksums.
 >>>>> --[[Joey]]
+
+>>>>>> I get that. Still, I think it would be _extremely_ useful to know what failures occurred, exactly. Not having enough copies is Not Good, yet not having enough copies and a locally correct file is _lot_ better than having not enough copies and a broken file. I.e. I would prefer:
+
+    (checksum...) OK
+    Not enough copies: Only 1 of 2 trustworthy copies exist of foo
+
+>>>>>> or similar and at the end
+
+    git-annex: 0 wrong checksums
+    git-annex: 11577 with too few copies
+
+>>>>>> In the end, it comes down to the distinction of different failure classes. -- Richard

From 36f63ab19e54f51561dd9f2946c68037a8e99791 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 16:21:34 -0400
Subject: [PATCH 2414/8313] getting tired of repeating myself

---
 ...d_checksum_when_less_copies_than_required_are_found.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 95848456d1..1f5667a11c 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -36,3 +36,9 @@
     git-annex: 11577 with too few copies
 
 >>>>>> In the end, it comes down to the distinction of different failure classes. -- Richard
+
+>>>>>>> For the third, and final time: 
+>>>>>>> # You are misreading the truncated output you posted
+>>>>>>> The "checksum" line is regarding **different** file than the 
+>>>>>>> not enough copies message. fsck does not attempt to checksum a file
+>>>>>>> that is not present. [[done]] --[[Joey]]

From fef2cf739872b905bbdf493f9f3ba7124400c633 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 16:45:06 -0400
Subject: [PATCH 2415/8313] refactor

---
 Command/Fsck.hs | 36 ++++++++++++++----------------------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 5d2e2ee50b..9f3ae02636 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -32,14 +32,17 @@ start file numcopies = notBareRepo $ isAnnexed file $ \(key, backend) -> do
 	next $ perform key file backend numcopies
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
-perform key file backend numcopies = do
-	-- the location log is checked first, so that if it has bad data
-	-- that gets corrected
-	locationlogok <- verifyLocationLog key file
-	backendok <- fsckKey backend key (Just file) numcopies
-	if locationlogok && backendok
-		then next $ return True
-		else stop
+perform key file backend numcopies = check =<< sequence
+	-- order matters
+	[ verifyLocationLog key file
+	, checkKeySize key
+	, checkKeyNumCopies key file numcopies
+	, (Types.Backend.fsckKey backend) key
+	]
+	where
+		check vs
+			| all (== True) vs = next $ return True
+			| otherwise = stop
 
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
@@ -77,14 +80,6 @@ verifyLocationLog key file = do
 			showNote "fixing location log"
 			logChange g key u s
 
-{- Checks a key for problems. -}
-fsckKey :: Backend Annex -> Key -> Maybe FilePath -> Maybe Int -> Annex Bool
-fsckKey backend key file numcopies = do
-	size_ok <- checkKeySize key
-	copies_ok <- checkKeyNumCopies key file numcopies
-	backend_ok <- (Types.Backend.fsckKey backend) key
-	return $ size_ok && copies_ok && backend_ok
-
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
@@ -108,7 +103,7 @@ checkKeySize key = do
 					return False
 
 
-checkKeyNumCopies :: Key -> Maybe FilePath -> Maybe Int -> Annex Bool
+checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
 	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
@@ -116,12 +111,9 @@ checkKeyNumCopies key file numcopies = do
 	if present < needed
 		then do
 			ppuuids <- Remote.prettyPrintUUIDs "untrusted" untrustedlocations
-			warning $ missingNote (filename file key) present needed ppuuids
+			warning $ missingNote file present needed ppuuids
 			return False
 		else return True
-	where
-		filename Nothing k = show k
-		filename (Just f) _ = f
 
 missingNote :: String -> Int -> Int -> String -> String
 missingNote file 0 _ [] = 

From a183487cd5477add59f470a42b18ec0c233bfb06 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Sat, 29 Oct 2011 21:36:06 +0000
Subject: [PATCH 2416/8313]

---
 ...um_when_less_copies_than_required_are_found.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
index 1f5667a11c..fe6536b6a7 100644
--- a/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
+++ b/doc/bugs/fsck_claims_failed_checksum_when_less_copies_than_required_are_found.mdwn
@@ -42,3 +42,16 @@
 >>>>>>> The "checksum" line is regarding **different** file than the 
 >>>>>>> not enough copies message. fsck does not attempt to checksum a file
 >>>>>>> that is not present. [[done]] --[[Joey]]
+
+
+>>>>>>>> I realized early on that I pasted the wrong cross-passage, but as there is a ton of the same output, I didn't think it would matter. I wasn't aware that it does not try to checksum when there aren't enough copies. To be fair, you only just mentioned that.
+>>>>>>>> Personally, I think that's a bug as it makes ensuring local correctness before copying a file to remotes impossible.
+>>>>>>>> Either way, I really didn't know it actually _skipped_ checksumming; that part was missing.
+>>>>>>>> For the benefit of anyone else who might read this, this is the correct order:
+
+    fsck foo (fixing location log) 
+      Only 1 of 2 trustworthy copies exist of foo
+      Back it up with git-annex copy.
+    (checksum...) failed
+
+>>>>>>>> If you would like to keep things this way, fine. I think it's less than ideal, but I don't want to argue, either. -- Richard

From 2566eb85fe2bbd0f9d1798d50ca0d88970a4490c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 17:49:37 -0400
Subject: [PATCH 2417/8313] fsck: Now works in bare repositories.

Checks location log information, and file contents.

Does not check that numcopies is satisfied, as .gitattributes information
about numcopies is not available in a bare repository. In practice, that
should not be a problem, since fsck is also run in a checkout and will
check numcopies there.
---
 Command.hs                                    | 25 +++++-----
 Command/Fsck.hs                               | 49 ++++++++++++++++---
 Command/Unused.hs                             | 10 +---
 Logs/Location.hs                              | 13 +++++
 debian/changelog                              |  4 ++
 ...t_annex_fsck_is_a_no-op_in_bare_repos.mdwn |  2 +
 doc/todo/support_fsck_in_bare_repos.mdwn      |  5 +-
 7 files changed, 77 insertions(+), 31 deletions(-)

diff --git a/Command.hs b/Command.hs
index b039403ca0..4e312e66d4 100644
--- a/Command.hs
+++ b/Command.hs
@@ -81,18 +81,6 @@ doCommand = start
 		success = return True
 		failure = showEndFail >> return False
 
-notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
-notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
-
-isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
-isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
-
-notBareRepo :: Annex a -> Annex a
-notBareRepo a = do
-	whenM (Git.repoIsLocalBare <$> gitRepo) $
-		error "You cannot run this subcommand in a bare repository."
-	a
-
 {- These functions find appropriate files or other things based on a
    user's parameters, and prepare actions operating on them. -}
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
@@ -168,7 +156,18 @@ runFilteredGen a d fs = do
 			ok <- matcher f
 			if ok then a v else stop
 
-{- filter out symlinks -}	
+notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
+notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
+
+isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
+isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
+
+notBareRepo :: Annex a -> Annex a
+notBareRepo a = do
+	whenM (Git.repoIsLocalBare <$> gitRepo) $
+		error "You cannot run this subcommand in a bare repository."
+	a
+
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 9f3ae02636..6f184a760c 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -12,6 +12,8 @@ import Command
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
+import qualified Backend
+import qualified Git
 import Annex.Content
 import Logs.Location
 import Logs.Trust
@@ -24,30 +26,61 @@ def :: [Command]
 def = [command "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies start, withBarePresentKeys startBare]
 
 start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = notBareRepo $ isAnnexed file $ \(key, backend) -> do
+start file numcopies = isAnnexed file $ \(key, backend) -> do
 	showStart "fsck" file
 	next $ perform key file backend numcopies
 
 perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
-perform key file backend numcopies = check =<< sequence
+perform key file backend numcopies = check
 	-- order matters
 	[ verifyLocationLog key file
 	, checkKeySize key
 	, checkKeyNumCopies key file numcopies
 	, (Types.Backend.fsckKey backend) key
 	]
+
+{- To fsck a bare repository, fsck each key in the location log. -}
+withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
+withBarePresentKeys a params = do
+	bare <- Git.repoIsLocalBare <$> gitRepo
+	if bare
+		then do
+			unless (null params) $ do
+				error "fsck should be run without parameters in a bare repository"
+			liftM (map a) loggedKeys
+		else return []
+
+startBare :: Key -> CommandStart
+startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of
+	Nothing -> stop
+	Just backend -> do
+		showStart "fsck" (show key)
+		next $ performBare key backend
+
+{- Note that numcopies cannot be checked in a bare repository, because
+ - getting the numcopies value requires a working copy with .gitattributes
+ - files. -}
+performBare :: Key -> Backend Annex -> CommandPerform
+performBare key backend = check
+	[ verifyLocationLog key (show key)
+	, checkKeySize key
+	, (Types.Backend.fsckKey backend) key
+	]
+
+check :: [Annex Bool] -> CommandPerform	
+check s = sequence s >>= dispatch
 	where
-		check vs
+		dispatch vs
 			| all (== True) vs = next $ return True
 			| otherwise = stop
 
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
-verifyLocationLog :: Key -> FilePath -> Annex Bool
-verifyLocationLog key file = do
+verifyLocationLog :: Key -> String -> Annex Bool
+verifyLocationLog key desc = do
 	g <- gitRepo
 	present <- inAnnex key
 	
@@ -69,7 +102,7 @@ verifyLocationLog key file = do
 		(False, True) -> do
 				fix g u InfoMissing
 				warning $
-					"** Based on the location log, " ++ file
+					"** Based on the location log, " ++ desc
 					++ "\n** was expected to be present, " ++
 					"but its content is missing."
 				return False
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 11c3f429ec..a6cced27ff 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -67,19 +67,11 @@ checkRemoteUnused name = do
 checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
 checkRemoteUnused' r = do
 	showAction "checking for unused data"
-	remotehas <- filterM isthere =<< loggedKeys
+	remotehas <- loggedKeysFor (Remote.uuid r)
 	remoteunused <- excludeReferenced remotehas
 	let list = number 0 remoteunused
 	writeUnusedFile "" list
 	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
-	where
-		{- This should run strictly to avoid the filterM
-		 - building many thunks containing keyLocations data. -}
-		isthere k = do
-			us <- keyLocations k
-			let !there = uuid `elem` us
-			return there
-		uuid = Remote.uuid r
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 8868912dbb..8855cf63b5 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -17,6 +17,7 @@ module Logs.Location (
 	readLog,
 	keyLocations,
 	loggedKeys,
+	loggedKeysFor,
 	logFile,
 	logFileKey
 ) where
@@ -44,6 +45,18 @@ keyLocations = currentLog . logFile
 loggedKeys :: Annex [Key]
 loggedKeys = mapMaybe (logFileKey . takeFileName) <$> Annex.Branch.files
 
+{- Finds all keys that have location log information indicating
+ - they are present for the specified repository. -}
+loggedKeysFor :: UUID -> Annex [Key]
+loggedKeysFor u = filterM isthere =<< loggedKeys
+	where
+		{- This should run strictly to avoid the filterM
+		 - building many thunks containing keyLocations data. -}
+		isthere k = do
+			us <- keyLocations k
+			let !there = u `elem` us
+			return there
+
 {- The filename of the log file for a given key. -}
 logFile :: Key -> String
 logFile key = hashDirLower key ++ keyFile key ++ ".log"
diff --git a/debian/changelog b/debian/changelog
index 4a873af945..42454fe8f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,10 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * drop --from is now supported to remove file content from a remote.
   * status: Now always shows the current repository, even when it does not
     appear in uuid.log.
+  * fsck: Now works in bare repositories. Checks location log information,
+    and file contents. Does not check that numcopies is satisfied, as
+    .gitattributes information about numcopies is not available in a bare
+    repository.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
index 3fd1a8082c..9a044860ae 100644
--- a/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
+++ b/doc/bugs/git_annex_fsck_is_a_no-op_in_bare_repos.mdwn
@@ -17,3 +17,5 @@ See http://lists.madduck.net/pipermail/vcs-home/2011-June/000433.html
 >>> While storing the data is no longer an issue in bare repos, fsck would
 >>> need a special mode that examines all the location logs, since it
 >>> cannot run thru the checked out files. --[[Joey]] 
+
+>>>> [[done]]! --[[Joey]] 
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index e6980fa281..53331a4f51 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -1,6 +1,5 @@
 What is says on the tin:
 
-
     22:56:54 < RichiH> joeyh_: by the way, i have been thinking about fsck on bare repos
     22:57:37 < RichiH> joeyh_: the best i could come with is to have a bare and a non-bare access the same repo store
     22:58:00 < RichiH> joeyh_: alternatively, with the SHA* backend, you have all the information to verify that the local data is correct
@@ -10,3 +9,7 @@ What is says on the tin:
     23:14:51 < joeyh_> unused/dropunused could work in bare repos too btw
 
 > Also `status`'s total annex keys/size could be handled for bare repos. --[[Joey]] 
+
+>> Fsck is done. Rest not done yet. --[[Joey]]
+
+[[!meta title="support status, unused, dropunused in bare repos"]]

From 61000904d74ffd4745dd6808bcfa88289affc169 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 18:47:53 -0400
Subject: [PATCH 2418/8313] refactor

---
 Command.hs      | 10 ++++++++--
 Command/Fsck.hs | 12 +++++-------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/Command.hs b/Command.hs
index 4e312e66d4..32f6743f36 100644
--- a/Command.hs
+++ b/Command.hs
@@ -149,22 +149,28 @@ backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
 runFilteredGen :: (b -> Annex (Maybe a)) -> (b -> FilePath) -> Annex [b] -> Annex [Annex (Maybe a)]
 runFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
-	liftM (map $ proc matcher) fs
+	runActions (proc matcher) fs
 	where
 		proc matcher v = do
 			let f = d v
 			ok <- matcher f
 			if ok then a v else stop
 
+runActions :: (b -> Annex (Maybe a)) -> Annex [b] -> Annex [Annex (Maybe a)]
+runActions a fs = liftM (map a) fs
+
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
 isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
 isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
+isBareRepo :: Annex Bool
+isBareRepo = Git.repoIsLocalBare <$> gitRepo
+
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
-	whenM (Git.repoIsLocalBare <$> gitRepo) $
+	whenM isBareRepo $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 6f184a760c..1f30d2eb63 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -13,7 +13,6 @@ import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
 import qualified Backend
-import qualified Git
 import Annex.Content
 import Logs.Location
 import Logs.Trust
@@ -44,14 +43,13 @@ perform key file backend numcopies = check
 
 {- To fsck a bare repository, fsck each key in the location log. -}
 withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
-withBarePresentKeys a params = do
-	bare <- Git.repoIsLocalBare <$> gitRepo
-	if bare
-		then do
+withBarePresentKeys a params = isBareRepo >>= go
+	where
+		go False = return []
+		go True = do
 			unless (null params) $ do
 				error "fsck should be run without parameters in a bare repository"
-			liftM (map a) loggedKeys
-		else return []
+			runActions a loggedKeys
 
 startBare :: Key -> CommandStart
 startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of

From c102e63595d502c2424552b29e338ab71cb4a098 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 19:03:43 -0400
Subject: [PATCH 2419/8313] status: clean up for bare repositories

The backend usage graph shows present keys as well as keys found in the
repository tree, so it will also be populated for bare repositories.

Changed wording to "visible annex keys", which explains why it's 0 in
a bare repository (no keys visible as no tree), and also why it varies
depending on which branch is checked out. This seemed better than doing
something expensive to look up keys from the git-annex branch.
---
 Command/Status.hs                        | 20 ++++++++++----------
 doc/todo/support_fsck_in_bare_repos.mdwn |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index b5f4956dbf..53d64d0428 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -57,8 +57,8 @@ stats =
 	, bad_data_size
 	, local_annex_keys
 	, local_annex_size
-	, total_annex_keys
-	, total_annex_size
+	, visible_annex_keys
+	, visible_annex_size
 	, backend_usage
 	]
 
@@ -99,16 +99,16 @@ local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
 	keySizeSum <$> cachedKeysPresent
 
-total_annex_size :: Stat
-total_annex_size = stat "total annex size" $
-	keySizeSum <$> cachedKeysReferenced
-
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $
 	show . S.size <$> cachedKeysPresent
 
-total_annex_keys :: Stat
-total_annex_keys = stat "total annex keys" $
+visible_annex_size :: Stat
+visible_annex_size = stat "visible annex size" $
+	keySizeSum <$> cachedKeysReferenced
+
+visible_annex_keys :: Stat
+visible_annex_keys = stat "visible annex keys" $
 	show . S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat
@@ -118,9 +118,9 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced
+backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced <*> cachedKeysPresent
 	where
-		usage ks = pp "" $ reverse . sort $ map swap $ splits $ S.toList ks
+		usage a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index 53331a4f51..31481a4a70 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -12,4 +12,4 @@ What is says on the tin:
 
 >> Fsck is done. Rest not done yet. --[[Joey]]
 
-[[!meta title="support status, unused, dropunused in bare repos"]]
+[[!meta title="support unused, dropunused in bare repos"]]

From 22e9f445ab74041321d731fd6290f2922384753b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 19:16:45 -0400
Subject: [PATCH 2420/8313] unused, dropunused: Now work in bare repositories.

Turned out I had already done all the work needed to support this when
unused started checking all branches.
---
 Command/DropUnused.hs                    |  2 +-
 Command/Unused.hs                        |  2 +-
 debian/changelog                         |  1 +
 doc/bare_repositories.mdwn               | 20 ++++++++------------
 doc/todo/support_fsck_in_bare_repos.mdwn |  2 ++
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index d2eb3df710..07ae1b48c3 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -35,7 +35,7 @@ withUnusedMaps params = do
 	return $ map (start (unused, unusedbad, unusedtmp)) params
 
 start :: (UnusedMap, UnusedMap, UnusedMap) -> FilePath -> CommandStart
-start (unused, unusedbad, unusedtmp) s = notBareRepo $ search
+start (unused, unusedbad, unusedtmp) s = search
 	[ (unused, perform)
 	, (unusedbad, performOther gitAnnexBadLocation)
 	, (unusedtmp, performOther gitAnnexTmpLocation)
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a6cced27ff..7e9ffa01f5 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -36,7 +36,7 @@ seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
 start :: CommandStart
-start = notBareRepo $ do
+start = do
 	from <- Annex.getState Annex.fromremote
 	let (name, action) = case from of
 		Nothing -> (".", checkUnused)
diff --git a/debian/changelog b/debian/changelog
index 42454fe8f7..e553877def 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     and file contents. Does not check that numcopies is satisfied, as
     .gitattributes information about numcopies is not available in a bare
     repository.
+  * unused, dropunused: Now work in bare repositories.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index c5663d84cf..f2a3ea4e94 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -7,16 +7,12 @@ Of course, for that to work, the bare repository has to be on a system with
 [[git-annex-shell]] installed. If "origin" is on GitWeb, you still can't
 use git-annex to store stuff there.
 
-Known to work ok:
+It took a while, but bare repositories are now supported exactly as well
+as non-bare repositories. Except for these caveats:
 
-* `git annex move --to` and `--from`, when pointed at a bare repository.
-* `git annex copy` ditto.
-* `git annex drop` can check that a bare repository has a copy of data
-  that is being dropped.
-* `git annex get` can transfer data from a bare repository.
-* Most other stuff (ie, init, describe, trust, etc.)
-
-There are a few caveats to keep in mind when using bare repositories:
-
-* A few subcommands, like `unused` cannot be run in a bare repository.
-  Those subcommands will refuse to do anything.
+* `git annex fsck` works in a bare repository, but does not display
+  warnings about insufficient
+  [[copies]]. To get those warnings, just run it in one of the non-bare
+  checkouts.
+* `git annex unused` in a bare repository only knows about keys used in
+  branches that have been pushed to the bare repository. So use it with care..
diff --git a/doc/todo/support_fsck_in_bare_repos.mdwn b/doc/todo/support_fsck_in_bare_repos.mdwn
index 31481a4a70..32ced467e0 100644
--- a/doc/todo/support_fsck_in_bare_repos.mdwn
+++ b/doc/todo/support_fsck_in_bare_repos.mdwn
@@ -12,4 +12,6 @@ What is says on the tin:
 
 >> Fsck is done. Rest not done yet. --[[Joey]]
 
+>>> all [[done]]! --[[Joey]] 
+
 [[!meta title="support unused, dropunused in bare repos"]]

From ef5330120c0b522ff159a5b3caba7a926236947b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 19:30:48 -0400
Subject: [PATCH 2421/8313] bare cleanup

---
 Command/Add.hs             | 2 +-
 Command/AddUrl.hs          | 2 +-
 doc/bare_repositories.mdwn | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 82287be0bf..cd18f6c725 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -29,7 +29,7 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
 start :: BackendFile -> CommandStart
-start p@(_, file) = notAnnexed file $ do
+start p@(_, file) = notBareRepo $ notAnnexed file $ do
 	s <- liftIO $ getSymbolicLinkStatus file
 	if isSymbolicLink s || not (isRegularFile s)
 		then stop
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e974d06a14..4382a9c07a 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -26,7 +26,7 @@ seek :: [CommandSeek]
 seek = [withStrings start]
 
 start :: String -> CommandStart
-start s = do
+start s = notBareRepo $ do
 	let u = parseURI s
 	case u of
 		Nothing -> error $ "bad url " ++ s
diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index f2a3ea4e94..3bc0a22cbd 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -16,3 +16,5 @@ as non-bare repositories. Except for these caveats:
   checkouts.
 * `git annex unused` in a bare repository only knows about keys used in
   branches that have been pushed to the bare repository. So use it with care..
+* Commands that need a work tree, like `git annex add` won't work in a bare
+  repository, of course.

From 4e9be0d1f86893a469b33b763b55edfe75bdb3aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 29 Oct 2011 23:48:46 -0400
Subject: [PATCH 2422/8313] refactoring and cleanup

No code changes.
---
 Checks.hs          |  45 +++++++++
 CmdLine.hs         |   1 -
 Command.hs         | 233 ++++++---------------------------------------
 Command/FromKey.hs |   1 +
 Command/Fsck.hs    |   2 +-
 Command/SetKey.hs  |   1 +
 Config.hs          |  12 +++
 GitAnnex.hs        |   1 -
 Options.hs         |  39 +++++++-
 Seek.hs            | 117 +++++++++++++++++++++++
 Types/Command.hs   |  45 +++++++++
 git-annex-shell.hs |   1 -
 12 files changed, 288 insertions(+), 210 deletions(-)
 create mode 100644 Checks.hs
 create mode 100644 Seek.hs
 create mode 100644 Types/Command.hs

diff --git a/Checks.hs b/Checks.hs
new file mode 100644
index 0000000000..cd172c6091
--- /dev/null
+++ b/Checks.hs
@@ -0,0 +1,45 @@
+{- git-annex command checks
+ -
+ - Common sanity checks for commands, and an interface to selectively
+ - remove them, or add others.
+ - 
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Checks where
+
+import Common.Annex
+import Types.Command
+import Init
+import qualified Annex
+
+commonChecks :: [CommandCheck]
+commonChecks = [fromOpt, toOpt, repoExists]
+
+repoExists :: CommandCheck
+repoExists = CommandCheck 0 ensureInitialized
+
+fromOpt :: CommandCheck
+fromOpt = CommandCheck 1 $ do
+	v <- Annex.getState Annex.fromremote
+	unless (v == Nothing) $ error "cannot use --from with this command"
+
+toOpt :: CommandCheck
+toOpt = CommandCheck 2 $ do
+	v <- Annex.getState Annex.toremote
+	unless (v == Nothing) $ error "cannot use --to with this command"
+
+checkCommand :: Command -> Annex ()
+checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
+
+dontCheck :: CommandCheck -> Command -> Command
+dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
+
+addCheck :: Annex () -> Command -> Command
+addCheck check cmd = mutateCheck cmd $
+	\c -> CommandCheck (length c + 100) check : c
+
+mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
+mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }
diff --git a/CmdLine.hs b/CmdLine.hs
index 9f1ded498c..fffd343f0d 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -20,7 +20,6 @@ import qualified Annex.Queue
 import qualified Git
 import Annex.Content
 import Command
-import Options
 
 {- Runs the passed command line. -}
 dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
diff --git a/Command.hs b/Command.hs
index 32f6743f36..74b1ff21c6 100644
--- a/Command.hs
+++ b/Command.hs
@@ -5,52 +5,38 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Command where
+module Command (
+	module Types.Command,
+	module Seek,
+	module Checks,
+	module Options,
+	command,
+	next,
+	stop,
+	prepCommand,
+	doCommand,
+	notAnnexed,
+	isAnnexed,
+	notBareRepo,
+	isBareRepo,
+	autoCopies
+) where
 
 import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
-import qualified Git.LsFiles as LsFiles
-import Types.Key
+import Types.Command
 import Logs.Trust
 import Logs.Location
 import Config
-import Backend
-import Limit
-import Init
+import Seek
+import Checks
+import Options
 
-{- A command runs in these stages.
- -
- - a. The check stage runs checks, that error out if
- -    anything prevents the command from running. -}
-data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () }
-instance Eq CommandCheck where
-	a == b = idCheck a == idCheck b
-{- b. The seek stage takes the parameters passed to the command,
- -    looks through the repo to find the ones that are relevant
- -    to that command (ie, new files to add), and generates
- -    a list of start stage actions. -}
-type CommandSeek = [String] -> Annex [CommandStart]
-{- c. The start stage is run before anything is printed about the
-  -   command, is passed some input, and can early abort it
-  -   if the input does not make sense. It should run quickly and
-  -   should not modify Annex state. -}
-type CommandStart = Annex (Maybe CommandPerform)
-{- d. The perform stage is run after a message is printed about the command
- -    being run, and it should be where the bulk of the work happens. -}
-type CommandPerform = Annex (Maybe CommandCleanup)
-{- e. The cleanup stage is run only if the perform stage succeeds, and it
- -    returns the overall success/fail of the command. -}
-type CommandCleanup = Annex Bool
-
-data Command = Command {
-	cmdcheck :: [CommandCheck],
-	cmdname :: String,
-	cmdparams :: String,
-	cmdseek :: [CommandSeek],
-	cmddesc :: String
-}
+{- Generates a command with the common checks. -}
+command :: String -> String -> [CommandSeek] -> String -> Command
+command = Command commonChecks
 
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
@@ -60,15 +46,14 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-{- Generates a command with the common checks. -}
-command :: String -> String -> [CommandSeek] -> String -> Command
-command = Command commonChecks
-
 {- Prepares a list of actions to run to perform a command, based on
  - the parameters passed to it. -}
 prepCommand :: Command -> [String] -> Annex [Annex Bool]
-prepCommand Command { cmdseek = seek } params =
-	return . map doCommand . concat =<< mapM (\s -> s params) seek
+prepCommand cmd ps = return . map doCommand =<< seekCommand cmd ps
+
+{- Runs a command through the seek stage. -}
+seekCommand :: Command -> [String] -> Annex [CommandStart]
+seekCommand Command { cmdseek = seek } ps = concat <$> mapM (\s -> s ps) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
@@ -81,147 +66,20 @@ doCommand = start
 		success = return True
 		failure = showEndFail >> return False
 
-{- These functions find appropriate files or other things based on a
-   user's parameters, and prepare actions operating on them. -}
-withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
-withFilesInGit a params = do
-	repo <- gitRepo
-	runFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
-withAttrFilesInGit attr a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	runFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
-withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
-withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
-	where
-		go (file, v) = a file (readMaybe v)
-withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
-withBackendFilesInGit a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	backendPairs a files
-withFilesMissing :: (String -> CommandStart) -> CommandSeek
-withFilesMissing a params = runFiltered a $ liftIO $ filterM missing params
-	where
-		missing = liftM not . doesFileExist
-withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
-withFilesNotInGit a params = do
-	repo <- gitRepo
-	force <- Annex.getState Annex.force
-	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
-	backendPairs a newfiles
-withWords :: ([String] -> CommandStart) -> CommandSeek
-withWords a params = return [a params]
-withStrings :: (String -> CommandStart) -> CommandSeek
-withStrings a params = return $ map a params
-withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
-withFilesToBeCommitted a params = do
-	repo <- gitRepo
-	runFiltered a $
-		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
-withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
-withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
-withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
-withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
-withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
-withFilesUnlocked' typechanged a params = do
-	-- unlocked files have changed type from a symlink to a regular file
-	repo <- gitRepo
-	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
-	unlockedfiles <- liftIO $ filterM notSymlink $
-		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
-	backendPairs a unlockedfiles
-withKeys :: (Key -> CommandStart) -> CommandSeek
-withKeys a params = return $ map (a . parse) params
-	where
-		parse p = fromMaybe (error "bad key") $ readKey p
-withNothing :: CommandStart -> CommandSeek
-withNothing a [] = return [a]
-withNothing _ _ = error "This command takes no parameters."
-
-runFiltered :: (FilePath -> Annex (Maybe a)) -> Annex [FilePath] -> Annex [Annex (Maybe a)]
-runFiltered a = runFilteredGen a id
-
-backendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-backendPairs a fs = runFilteredGen a snd (Backend.chooseBackends fs)
-
-runFilteredGen :: (b -> Annex (Maybe a)) -> (b -> FilePath) -> Annex [b] -> Annex [Annex (Maybe a)]
-runFilteredGen a d fs = do
-	matcher <- Limit.getMatcher
-	runActions (proc matcher) fs
-	where
-		proc matcher v = do
-			let f = d v
-			ok <- matcher f
-			if ok then a v else stop
-
-runActions :: (b -> Annex (Maybe a)) -> Annex [b] -> Annex [Annex (Maybe a)]
-runActions a fs = liftM (map a) fs
-
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
 isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
 isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
 
-isBareRepo :: Annex Bool
-isBareRepo = Git.repoIsLocalBare <$> gitRepo
-
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
 	whenM isBareRepo $
 		error "You cannot run this subcommand in a bare repository."
 	a
 
-notSymlink :: FilePath -> IO Bool
-notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
-
-{- Descriptions of params used in usage messages. -}
-paramPaths :: String
-paramPaths = paramOptional $ paramRepeating paramPath -- most often used
-paramPath :: String
-paramPath = "PATH"
-paramKey :: String
-paramKey = "KEY"
-paramDesc :: String
-paramDesc = "DESC"
-paramUrl :: String
-paramUrl = "URL"
-paramNumber :: String
-paramNumber = "NUMBER"
-paramRemote :: String
-paramRemote = "REMOTE"
-paramGlob :: String
-paramGlob = "GLOB"
-paramName :: String
-paramName = "NAME"
-paramUUID :: String
-paramUUID = "UUID"
-paramType :: String
-paramType = "TYPE"
-paramKeyValue :: String
-paramKeyValue = "K=V"
-paramNothing :: String
-paramNothing = ""
-paramRepeating :: String -> String
-paramRepeating s = s ++ " ..."
-paramOptional :: String -> String
-paramOptional s = "[" ++ s ++ "]"
-paramPair :: String -> String -> String
-paramPair a b = a ++ " " ++ b
-
-{- The Key specified by the --key parameter. -}
-cmdlineKey :: Annex Key
-cmdlineKey  = do
-	k <- Annex.getState Annex.defaultkey
-	case k of
-		Nothing -> nokey
-		Just "" -> nokey
-		Just kstring -> maybe badkey return $ readKey kstring
-	where
-		nokey = error "please specify the key with --key"
-		badkey = error "bad key"
+isBareRepo :: Annex Bool
+isBareRepo = Git.repoIsLocalBare <$> gitRepo
 
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
@@ -238,34 +96,3 @@ autoCopies key vs numcopiesattr a = do
 			(_, have) <- trustPartition UnTrusted =<< keyLocations key
 			if length have `vs` needed then a else stop
 		else a
-
-{- Common checks for commands, and an interface to selectively remove them,
- - or add others. -}
-commonChecks :: [CommandCheck]
-commonChecks = [fromOpt, toOpt, repoExists]
-
-repoExists :: CommandCheck
-repoExists = CommandCheck 0 ensureInitialized
-
-fromOpt :: CommandCheck
-fromOpt = CommandCheck 1 $ do
-	v <- Annex.getState Annex.fromremote
-	unless (v == Nothing) $ error "cannot use --from with this command"
-
-toOpt :: CommandCheck
-toOpt = CommandCheck 2 $ do
-	v <- Annex.getState Annex.toremote
-	unless (v == Nothing) $ error "cannot use --to with this command"
-
-checkCommand :: Command -> Annex ()
-checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
-
-dontCheck :: CommandCheck -> Command -> Command
-dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
-
-addCheck :: Annex () -> Command -> Command
-addCheck check cmd = mutateCheck cmd $
-	\c -> CommandCheck (length c + 100) check : c
-
-mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
-mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index fe9b5c96a0..4e4644708f 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -12,6 +12,7 @@ import Command
 import qualified Annex.Queue
 import Annex.Content
 import Types.Key
+import Config
 
 def :: [Command]
 def = [command "fromkey" paramPath seek "adds a file using a specific key"]
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 1f30d2eb63..073652d2ce 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -49,7 +49,7 @@ withBarePresentKeys a params = isBareRepo >>= go
 		go True = do
 			unless (null params) $ do
 				error "fsck should be run without parameters in a bare repository"
-			runActions a loggedKeys
+			prepStart a loggedKeys
 
 startBare :: Key -> CommandStart
 startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 9c31abb083..0c70d12b09 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import Logs.Location
 import Annex.Content
+import Config
 
 def :: [Command]
 def = [command "setkey" paramPath seek
diff --git a/Config.hs b/Config.hs
index f4c3843af8..f994002b93 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,6 +10,7 @@ module Config where
 import Common.Annex
 import qualified Git
 import qualified Annex
+import Types.Key (readKey)
 
 type ConfigKey = String
 
@@ -92,3 +93,14 @@ getNumCopies v =
 			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
 
+{- The Key specified by the --key parameter. -}
+cmdlineKey :: Annex Key
+cmdlineKey  = do
+	k <- Annex.getState Annex.defaultkey
+	case k of
+		Nothing -> nokey
+		Just "" -> nokey
+		Just kstring -> maybe badkey return $ readKey kstring
+	where
+		nokey = error "please specify the key with --key"
+		badkey = error "bad key"
diff --git a/GitAnnex.hs b/GitAnnex.hs
index c07e727fa0..89fb4e5919 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -13,7 +13,6 @@ import Common.Annex
 import qualified Git
 import CmdLine
 import Command
-import Options
 import Types.TrustLevel
 import qualified Annex
 import qualified Remote
diff --git a/Options.hs b/Options.hs
index 0c7b4d5f41..a8c165a816 100644
--- a/Options.hs
+++ b/Options.hs
@@ -1,6 +1,6 @@
-{- git-annex dashed options
+{- git-annex command-line options
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -12,7 +12,6 @@ import System.Log.Logger
 
 import Common.Annex
 import qualified Annex
-import Command
 import Limit
 
 {- Each dashed command-line option results in generation of an action
@@ -59,3 +58,37 @@ matcherOptions =
 	where
 		longopt o = Option [] [o] $ NoArg $ addToken o
 		shortopt o = Option o [] $ NoArg $ addToken o
+
+{- Descriptions of params used in usage messages. -}
+paramPaths :: String
+paramPaths = paramOptional $ paramRepeating paramPath -- most often used
+paramPath :: String
+paramPath = "PATH"
+paramKey :: String
+paramKey = "KEY"
+paramDesc :: String
+paramDesc = "DESC"
+paramUrl :: String
+paramUrl = "URL"
+paramNumber :: String
+paramNumber = "NUMBER"
+paramRemote :: String
+paramRemote = "REMOTE"
+paramGlob :: String
+paramGlob = "GLOB"
+paramName :: String
+paramName = "NAME"
+paramUUID :: String
+paramUUID = "UUID"
+paramType :: String
+paramType = "TYPE"
+paramKeyValue :: String
+paramKeyValue = "K=V"
+paramNothing :: String
+paramNothing = ""
+paramRepeating :: String -> String
+paramRepeating s = s ++ " ..."
+paramOptional :: String -> String
+paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b
diff --git a/Seek.hs b/Seek.hs
new file mode 100644
index 0000000000..4ae943157b
--- /dev/null
+++ b/Seek.hs
@@ -0,0 +1,117 @@
+{- git-annex command seeking
+ - 
+ - These functions find appropriate files or other things based on
+ - the values a user passes to a command, and prepare actions operating
+ - on them.
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Seek where
+
+import Common.Annex
+import Types.Command
+import Types.Key
+import Backend
+import qualified Annex
+import qualified Git
+import qualified Git.LsFiles as LsFiles
+import qualified Limit
+
+withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
+withFilesInGit a params = do
+	repo <- gitRepo
+	prepFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+
+withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
+withAttrFilesInGit attr a params = do
+	repo <- gitRepo
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+	prepFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
+
+withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
+withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
+	where
+		go (file, v) = a file (readMaybe v)
+
+withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
+withBackendFilesInGit a params = do
+	repo <- gitRepo
+	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+	prepBackendPairs a files
+
+withFilesMissing :: (String -> CommandStart) -> CommandSeek
+withFilesMissing a params = prepFiltered a $ liftIO $ filterM missing params
+	where
+		missing = liftM not . doesFileExist
+
+withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesNotInGit a params = do
+	repo <- gitRepo
+	force <- Annex.getState Annex.force
+	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
+	prepBackendPairs a newfiles
+
+withWords :: ([String] -> CommandStart) -> CommandSeek
+withWords a params = return [a params]
+
+withStrings :: (String -> CommandStart) -> CommandSeek
+withStrings a params = return $ map a params
+
+withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
+withFilesToBeCommitted a params = do
+	repo <- gitRepo
+	prepFiltered a $
+		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
+
+withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
+
+withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
+
+withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked' typechanged a params = do
+	-- unlocked files have changed type from a symlink to a regular file
+	repo <- gitRepo
+	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
+	unlockedfiles <- liftIO $ filterM notSymlink $
+		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
+	prepBackendPairs a unlockedfiles
+
+withKeys :: (Key -> CommandStart) -> CommandSeek
+withKeys a params = return $ map (a . parse) params
+	where
+		parse p = fromMaybe (error "bad key") $ readKey p
+
+withNothing :: CommandStart -> CommandSeek
+withNothing a [] = return [a]
+withNothing _ _ = error "This command takes no parameters."
+
+
+prepFiltered :: (FilePath -> CommandStart) -> Annex [FilePath] -> Annex [CommandStart]
+prepFiltered a = prepFilteredGen a id
+
+prepBackendPairs :: (BackendFile -> CommandStart) -> CommandSeek
+prepBackendPairs a fs = prepFilteredGen a snd (chooseBackends fs)
+
+prepFilteredGen :: (b -> CommandStart) -> (b -> FilePath) -> Annex [b] -> Annex [CommandStart]
+prepFilteredGen a d fs = do
+	matcher <- Limit.getMatcher
+	prepStart (proc matcher) fs
+	where
+		proc matcher v = do
+			let f = d v
+			ok <- matcher f
+			if ok then a v else return Nothing
+
+{- Generates a list of CommandStart actions that will be run to perform a
+ - command, using a list (ie of files) coming from an action. The list
+ - will be produced and consumed lazily. -}
+prepStart :: (b -> CommandStart) -> Annex [b] -> Annex [CommandStart]
+prepStart a fs = liftM (map a) fs
+
+notSymlink :: FilePath -> IO Bool
+notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/Types/Command.hs b/Types/Command.hs
new file mode 100644
index 0000000000..d39876a7ae
--- /dev/null
+++ b/Types/Command.hs
@@ -0,0 +1,45 @@
+{- git-annex command data types
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.Command where
+
+import Types
+
+{- A command runs in these stages.
+ -
+ - a. The check stage runs checks, that error out if
+ -    anything prevents the command from running. -}
+data CommandCheck = CommandCheck { idCheck :: Int, runCheck :: Annex () }
+{- b. The seek stage takes the parameters passed to the command,
+ -    looks through the repo to find the ones that are relevant
+ -    to that command (ie, new files to add), and generates
+ -    a list of start stage actions. -}
+type CommandSeek = [String] -> Annex [CommandStart]
+{- c. The start stage is run before anything is printed about the
+  -   command, is passed some input, and can early abort it
+  -   if the input does not make sense. It should run quickly and
+  -   should not modify Annex state. -}
+type CommandStart = Annex (Maybe CommandPerform)
+{- d. The perform stage is run after a message is printed about the command
+ -    being run, and it should be where the bulk of the work happens. -}
+type CommandPerform = Annex (Maybe CommandCleanup)
+{- e. The cleanup stage is run only if the perform stage succeeds, and it
+ -    returns the overall success/fail of the command. -}
+type CommandCleanup = Annex Bool
+
+{- A command is defined by specifying these things. -}
+data Command = Command {
+	cmdcheck :: [CommandCheck],
+	cmdname :: String,
+	cmdparams :: String,
+	cmdseek :: [CommandSeek],
+	cmddesc :: String
+}
+
+{- CommandCheck functions can be compared using their unique id. -}
+instance Eq CommandCheck where
+	a == b = idCheck a == idCheck b
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 328d7b1006..10eeb454af 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -12,7 +12,6 @@ import Common.Annex
 import qualified Git
 import CmdLine
 import Command
-import Options
 import Annex.UUID
 
 import qualified Command.ConfigList

From ee715647540a8b5d81254ed60cdf7709f63f42af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:38:48 -0400
Subject: [PATCH 2423/8313] add command name to some output

---
 CmdLine.hs                                    | 34 +++++++++++--------
 ...or_bug:_errors_are_not_verbose_enough.mdwn |  2 ++
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index fffd343f0d..658b38ab1c 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -26,22 +26,25 @@ dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
 dispatch args cmds options header gitrepo = do
 	setupConsole
 	state <- Annex.new gitrepo
-	(actions, state') <- Annex.run state $ parseCmd args header cmds options
-	tryRun state' $ [startup] ++ actions ++ [shutdown]
+	((cmd, actions), state') <- Annex.run state $ parseCmd args header cmds options
+	tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
 
 {- Parses command line, stores configure flags, and returns a 
- - list of actions to be run in the Annex monad. -}
-parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex [Annex Bool]
+ - list of actions to be run in the Annex monad and the Command
+ - being run. -}
+parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex (Command, [Annex Bool])
 parseCmd argv header cmds options = do
 	(flags, params) <- liftIO getopt
 	when (null params) $ error $ "missing command" ++ usagemsg
-	case lookupCmd (head params) of
-		[] -> error $ "unknown command" ++ usagemsg
+	let (c:rest) = params
+	case lookupCmd c of
+		[] -> error $ "unknown command " ++ c ++ " " ++ usagemsg
 		[cmd] -> do
 			_ <- sequence flags
 			checkCommand cmd
-			prepCommand cmd (drop 1 params)
-		_ -> error "internal error: multiple matching commands"
+			as <- prepCommand cmd rest
+			return (cmd, as)
+		_ -> error $ "internal error: multiple matching commands for " ++ c
 	where
 		getopt = case getOpt Permute options argv of
 			(flags, params, []) ->
@@ -70,10 +73,10 @@ usage header cmds options =
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
  -}
-tryRun :: Annex.AnnexState -> [Annex Bool] -> IO ()
+tryRun :: Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
 tryRun = tryRun' 0
-tryRun' :: Integer -> Annex.AnnexState -> [Annex Bool] -> IO ()
-tryRun' errnum state (a:as) = do
+tryRun' :: Integer -> Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
+tryRun' errnum state cmd (a:as) = do
 	result <- try $ Annex.run state $ do
 		Annex.Queue.flushWhenFull
 		a
@@ -82,10 +85,11 @@ tryRun' errnum state (a:as) = do
 			Annex.eval state $ do
 				showErr err
 				showEndFail
-			tryRun' (errnum + 1) state as
-		Right (True,state') -> tryRun' errnum state' as
-		Right (False,state') -> tryRun' (errnum + 1) state' as
-tryRun' errnum _ [] = when (errnum > 0) $ error $ show errnum ++ " failed"
+			tryRun' (errnum + 1) state cmd as
+		Right (True,state') -> tryRun' errnum state' cmd as
+		Right (False,state') -> tryRun' (errnum + 1) state' cmd as
+tryRun' errnum _ cmd [] = when (errnum > 0) $
+		error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
diff --git a/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn
index 8def2e8c3f..a6620f4255 100644
--- a/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn
+++ b/doc/bugs/minor_bug:_errors_are_not_verbose_enough.mdwn
@@ -22,3 +22,5 @@ Better:
 
 
 etc pp.
+
+> [[done]] --[[Joey]] 

From 56080a0febac6e75e61a6e358f09e0b1e77b77bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:44:09 -0400
Subject: [PATCH 2424/8313] closures

---
 doc/bugs/Cabal_dependency_monadIO_missing.mdwn                 | 3 +++
 ...:_more_descriptive_commit_messages_in_git-annex_branch.mdwn | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
index b5213b8aa5..13980dd292 100644
--- a/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
+++ b/doc/bugs/Cabal_dependency_monadIO_missing.mdwn
@@ -12,3 +12,6 @@ Adding the dependency for `monadIO` to `git-annex.cabal` should fix this?
 > dependency in the cabal file. Your system might be old/new/or broken,
 > perhaps it's time to provide some details about the version of haskell
 > and of `monad-control` you have installed? --[[Joey]] 
+
+>> Closing as apparently user error or a broken system.
+>> If you see this problem please do say. [[done]] --[[Joey]] 
diff --git a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
index aad119ffd3..3a891fc9b2 100644
--- a/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
+++ b/doc/bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch.mdwn
@@ -34,3 +34,6 @@ in my opinion, the messages should at least contain
 > 
 > It would be possible to make it *less* verbose, with an empty commit
 > message. :) --[[Joey]] 
+
+>> Closing as this is literally impossible to do without making
+>> git-annex worse. [[done]] --[[Joey]] 

From 3cf811ead0f1dcdcb4c8c6e7ea0c97f99220a38f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:49:49 -0400
Subject: [PATCH 2425/8313] update; status is no longer slow

---
 doc/todo/cache_key_info.mdwn | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/todo/cache_key_info.mdwn b/doc/todo/cache_key_info.mdwn
index 791584836d..d4352ccf7f 100644
--- a/doc/todo/cache_key_info.mdwn
+++ b/doc/todo/cache_key_info.mdwn
@@ -7,13 +7,14 @@ but that's out of scope -- and recent git-annex versions use queuing
 to save git add from piling up too much in the index.)
 
 But currently two git-annex commands are quite slow when annexes become large
-in quantity of files. These are unused and stats
+in quantity of files. These are unused and status.
 (Both have --fast versions that don't do as much).
+> (Update: status has become acceptably fast; most of its slowdown was due to using a bad data structure; scanning the tree is not particularly slow and it no longer looks at the git-annex branch.)
 
-Both are slow because both need two pieces of information that are not
+unused is slow because it needs two pieces of information that are not
 quick to look up, and require examining the whole repo, very seekily:
 
-1. The keys present in the annex. Found by looking thru .git/annex/objects.
+1. The keys present in the annex. Found by looking thru .git/annex/objects
 2. The keys referenced by files in git. Found by finding every file
    in git, and looking at its symlink.
 

From 1530eac31294347a83c2a7973aa2c27ede9184f3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 16:57:20 -0400
Subject: [PATCH 2426/8313] closures

---
 doc/todo/auto_remotes.mdwn                                  | 6 ++++++
 ...hlist:___34__git_annex_add__34___multiple_processes.mdwn | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/doc/todo/auto_remotes.mdwn b/doc/todo/auto_remotes.mdwn
index c12878950a..715dea7207 100644
--- a/doc/todo/auto_remotes.mdwn
+++ b/doc/todo/auto_remotes.mdwn
@@ -21,3 +21,9 @@ Question: When should git-annex update the remote.log?
 (If not just on init.) Whenever it reads in a repo's remotes?
 
 > This sounds useful and the log should be updated every time any remote is being accessed. A counter or timestamp (yes, distributed times may be wrong/different) could be used to auto-prune old entries via a global and per-remote config setting. -- RichiH
+
+---
+
+I no longer think I'd use this myself, I find that my repositories quickly
+grow the paths I actually use, somewhat organically. Unofficial paths
+across university quads come to mind. [[done]] --[[Joey]] 
diff --git a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
index f3d3837015..a04af05b42 100644
--- a/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
+++ b/doc/todo/wishlist:___34__git_annex_add__34___multiple_processes.mdwn
@@ -5,3 +5,6 @@ i'm in the process of managing my music collection with git-annex. The initial "
 Anyway, thanks for this wonderful piece of software.
 
 Jean-Baptiste
+
+> closing as dup of [[parallel possibilities]] (also see comments below)
+> [[done]] --[[Joey]] 

From e09dd6f306b3f69718c77a03364ee9e51a51bb3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 30 Oct 2011 20:04:15 -0400
Subject: [PATCH 2427/8313] cleanup

---
 Checks.hs  |  3 --
 CmdLine.hs | 86 ++++++++++++++++++++++++++----------------------------
 Command.hs | 21 +++++++------
 3 files changed, 51 insertions(+), 59 deletions(-)

diff --git a/Checks.hs b/Checks.hs
index cd172c6091..6a70fc52db 100644
--- a/Checks.hs
+++ b/Checks.hs
@@ -31,9 +31,6 @@ toOpt = CommandCheck 2 $ do
 	v <- Annex.getState Annex.toremote
 	unless (v == Nothing) $ error "cannot use --to with this command"
 
-checkCommand :: Command -> Annex ()
-checkCommand Command { cmdcheck = c } = sequence_ $ map runCheck c
-
 dontCheck :: CommandCheck -> Command -> Command
 dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
 
diff --git a/CmdLine.hs b/CmdLine.hs
index 658b38ab1c..af53abc628 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -21,45 +21,41 @@ import qualified Git
 import Annex.Content
 import Command
 
+type Params = [String]
+type Flags = [Annex ()]
+
 {- Runs the passed command line. -}
-dispatch :: [String] -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
+dispatch :: Params -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
 dispatch args cmds options header gitrepo = do
 	setupConsole
 	state <- Annex.new gitrepo
-	((cmd, actions), state') <- Annex.run state $ parseCmd args header cmds options
+	(actions, state') <- Annex.run state $ do
+		sequence_ flags
+		prepCommand cmd params
 	tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
-
-{- Parses command line, stores configure flags, and returns a 
- - list of actions to be run in the Annex monad and the Command
- - being run. -}
-parseCmd :: [String] -> String -> [Command] -> [Option] -> Annex (Command, [Annex Bool])
-parseCmd argv header cmds options = do
-	(flags, params) <- liftIO getopt
-	when (null params) $ error $ "missing command" ++ usagemsg
-	let (c:rest) = params
-	case lookupCmd c of
-		[] -> error $ "unknown command " ++ c ++ " " ++ usagemsg
-		[cmd] -> do
-			_ <- sequence flags
-			checkCommand cmd
-			as <- prepCommand cmd rest
-			return (cmd, as)
-		_ -> error $ "internal error: multiple matching commands for " ++ c
 	where
-		getopt = case getOpt Permute options argv of
-			(flags, params, []) ->
-				return (flags, params)
-			(_, _, errs) ->
-				ioError (userError (concat errs ++ usagemsg))
-		lookupCmd cmd = filter (\c -> cmd  == cmdname c) cmds
-		usagemsg = "\n\n" ++ usage header cmds options
+		(flags, cmd, params) = parseCmd args cmds options header
+
+{- Parses command line, and returns actions to run to configure flags,
+ - the Command being run, and the remaining parameters for the command. -} 
+parseCmd :: Params -> [Command] -> [Option] -> String -> (Flags, Command, Params)
+parseCmd argv cmds options header = check $ getOpt Permute options argv
+	where
+		check (_, [], []) = err "missing command"
+		check (flags, name:rest, [])
+			| null matches = err $ "unknown command " ++ name
+			| otherwise = (flags, head matches, rest)
+			where
+				matches = filter (\c -> name == cmdname c) cmds
+		check (_, _, errs) = err $ concat errs
+		err msg = error $ msg ++ "\n\n" ++ usage header cmds options
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String
-usage header cmds options =
-	usageInfo (header ++ "\n\nOptions:") options ++
-		"\nCommands:\n" ++ cmddescs
+usage header cmds options = usageInfo top options ++ commands
 	where
+		top = header ++ "\n\nOptions:"
+		commands = "\nCommands:\n" ++ cmddescs
 		cmddescs = unlines $ map (indent . showcmd) cmds
 		showcmd c =
 			cmdname c ++
@@ -73,23 +69,23 @@ usage header cmds options =
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
  -}
-tryRun :: Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
+tryRun :: Annex.AnnexState -> Command -> [CommandCleanup] -> IO ()
 tryRun = tryRun' 0
-tryRun' :: Integer -> Annex.AnnexState -> Command -> [Annex Bool] -> IO ()
-tryRun' errnum state cmd (a:as) = do
-	result <- try $ Annex.run state $ do
-		Annex.Queue.flushWhenFull
-		a
-	case result of
-		Left err -> do
-			Annex.eval state $ do
-				showErr err
-				showEndFail
-			tryRun' (errnum + 1) state cmd as
-		Right (True,state') -> tryRun' errnum state' cmd as
-		Right (False,state') -> tryRun' (errnum + 1) state' cmd as
-tryRun' errnum _ cmd [] = when (errnum > 0) $
-		error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
+tryRun' :: Integer -> Annex.AnnexState -> Command -> [CommandCleanup] -> IO ()
+tryRun' errnum _ cmd []
+	| errnum > 0 = error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
+	| otherwise = return ()
+tryRun' errnum state cmd (a:as) = run >>= handle
+	where
+		run = try $ Annex.run state $ do
+			Annex.Queue.flushWhenFull
+			a
+		handle (Left err) = showerr err >> cont False state
+		handle (Right (success, state')) = cont success state'
+		cont success s = tryRun' (if success then errnum else errnum + 1) s cmd as
+		showerr err = Annex.eval state $ do
+			showErr err
+			showEndFail
 
 {- Actions to perform each time ran. -}
 startup :: Annex Bool
diff --git a/Command.hs b/Command.hs
index 74b1ff21c6..c11b906103 100644
--- a/Command.hs
+++ b/Command.hs
@@ -46,25 +46,24 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
-{- Prepares a list of actions to run to perform a command, based on
- - the parameters passed to it. -}
-prepCommand :: Command -> [String] -> Annex [Annex Bool]
-prepCommand cmd ps = return . map doCommand =<< seekCommand cmd ps
-
-{- Runs a command through the seek stage. -}
-seekCommand :: Command -> [String] -> Annex [CommandStart]
-seekCommand Command { cmdseek = seek } ps = concat <$> mapM (\s -> s ps) seek
+{- Prepares to run a command via the check and seek stages, returning a
+ - list of actions to perform to run the command. -}
+prepCommand :: Command -> [String] -> Annex [CommandCleanup]
+prepCommand Command { cmdseek = seek, cmdcheck = c } params = do
+	sequence_ $ map runCheck c
+	map doCommand . concat <$> mapM (\s -> s params) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
 doCommand :: CommandStart -> CommandCleanup
 doCommand = start
 	where
-		start   = stage $ maybe success perform
+		start   = stage $ maybe skip perform
 		perform = stage $ maybe failure cleanup
-		cleanup = stage $ \r -> showEndResult r >> return r
+		cleanup = stage $ status
 		stage = (=<<)
-		success = return True
+		skip = return True
 		failure = showEndFail >> return False
+		status r = showEndResult r >> return r
 
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file

From cc1ea8f84463c7e333bfa17a815f250d8d088841 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 12:33:41 -0400
Subject: [PATCH 2428/8313] Removed the setkey command, and added a setcontent
 command with a more useful interface.

---
 Command/Fsck.hs       |  7 ++++--
 Command/SetContent.hs | 56 +++++++++++++++++++++++++++++++++++++++++++
 Command/SetKey.hs     | 48 -------------------------------------
 GitAnnex.hs           |  9 +++----
 debian/changelog      |  2 ++
 doc/git-annex.mdwn    | 23 +++++++++++-------
 6 files changed, 80 insertions(+), 65 deletions(-)
 create mode 100644 Command/SetContent.hs
 delete mode 100644 Command/SetKey.hs

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 073652d2ce..8d6f03d76b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -38,7 +38,7 @@ perform key file backend numcopies = check
 	[ verifyLocationLog key file
 	, checkKeySize key
 	, checkKeyNumCopies key file numcopies
-	, (Types.Backend.fsckKey backend) key
+	, checkBackend backend key
 	]
 
 {- To fsck a bare repository, fsck each key in the location log. -}
@@ -65,7 +65,7 @@ performBare :: Key -> Backend Annex -> CommandPerform
 performBare key backend = check
 	[ verifyLocationLog key (show key)
 	, checkKeySize key
-	, (Types.Backend.fsckKey backend) key
+	, checkBackend backend key
 	]
 
 check :: [Annex Bool] -> CommandPerform	
@@ -134,6 +134,9 @@ checkKeySize key = do
 					return False
 
 
+checkBackend :: Backend Annex -> Key -> Annex Bool
+checkBackend backend key =  (Types.Backend.fsckKey backend) key
+
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
diff --git a/Command/SetContent.hs b/Command/SetContent.hs
new file mode 100644
index 0000000000..d62faa445f
--- /dev/null
+++ b/Command/SetContent.hs
@@ -0,0 +1,56 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.SetContent where
+
+import Common.Annex
+import Command
+import Logs.Location
+import Annex.Content
+import qualified Backend
+import qualified Command.Fsck
+
+def :: [Command]
+def = [command "setcontent" (paramPair paramPath paramPath) seek
+	"sets content of annexed file"]
+
+seek :: [CommandSeek]
+seek = [withWords start]
+
+start :: [FilePath] -> CommandStart
+start (src:dest:[]) = do
+	showStart "setkey" dest
+	next $ perform src dest
+start _ = error "specify a src file and a dest file"
+
+perform :: FilePath -> FilePath -> CommandPerform
+perform src dest = go =<< Backend.lookupFile dest
+	where
+		go Nothing = error "dest file is not in annex"
+		go (Just (key, backend)) = do
+			-- the file might be on a different filesystem,
+			-- so mv is used rather than simply calling
+			-- moveToObjectDir; disk space is also
+			-- checked this way.
+			ok <- getViaTmp key $ \tmp ->
+				if dest /= src
+					then liftIO $
+						boolSystem "mv" [File src, File tmp]
+					else return True
+			if ok
+				then next $ cleanup key backend
+				else error "mv failed!"
+
+cleanup :: Key -> Backend Annex -> CommandCleanup
+cleanup key backend = do
+	logStatus key InfoPresent
+
+	-- fsck the new content
+	size_ok <- Command.Fsck.checkKeySize key
+	backend_ok <- Command.Fsck.checkBackend backend key
+
+	return $ size_ok && backend_ok
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
deleted file mode 100644
index 0c70d12b09..0000000000
--- a/Command/SetKey.hs
+++ /dev/null
@@ -1,48 +0,0 @@
-{- git-annex command
- -
- - Copyright 2010 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Command.SetKey where
-
-import Common.Annex
-import Command
-import Logs.Location
-import Annex.Content
-import Config
-
-def :: [Command]
-def = [command "setkey" paramPath seek
-	"sets annexed content for a key using a temp file"]
-
-seek :: [CommandSeek]
-seek = [withStrings start]
-
-{- Sets cached content for a key. -}
-start :: FilePath -> CommandStart
-start file = do
-	showStart "setkey" file
-	next $ perform file
-
-perform :: FilePath -> CommandPerform
-perform file = do
-	key <- cmdlineKey
-	-- the file might be on a different filesystem, so mv is used
-	-- rather than simply calling moveToObjectDir; disk space is also
-	-- checked this way.
-	ok <- getViaTmp key $ \dest ->
-		if dest /= file
-			then liftIO $
-				boolSystem "mv" [File file, File dest]
-			else return True
-	if ok
-		then next cleanup
-		else error "mv failed!"
-
-cleanup :: CommandCleanup
-cleanup = do
-	key <- cmdlineKey
-	logStatus key InfoPresent
-	return True
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 89fb4e5919..09f0a118c0 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -26,7 +26,7 @@ import qualified Command.Copy
 import qualified Command.Get
 import qualified Command.FromKey
 import qualified Command.DropKey
-import qualified Command.SetKey
+import qualified Command.SetContent
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Describe
@@ -63,6 +63,7 @@ cmds = concat
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
+	, Command.SetContent.def
 	, Command.Unannex.def
 	, Command.Uninit.def
 	, Command.PreCommit.def
@@ -72,7 +73,6 @@ cmds = concat
 	, Command.AddUrl.def
 	, Command.FromKey.def
 	, Command.DropKey.def
-	, Command.SetKey.def
 	, Command.Fix.def
 	, Command.Fsck.def
 	, Command.Unused.def
@@ -89,9 +89,7 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['k'] ["key"] (ReqArg setkey paramKey)
-		"specify a key to use"
-	, Option ['t'] ["to"] (ReqArg setto paramRemote)
+	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
 		"specify to where to transfer content"
 	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
 		"specify from where to transfer content"
@@ -116,7 +114,6 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
-		setkey v = Annex.changeState $ \s -> s { Annex.defaultkey = Just v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			g <- gitRepo
diff --git a/debian/changelog b/debian/changelog
index e553877def..91e4a18af1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     .gitattributes information about numcopies is not available in a bare
     repository.
   * unused, dropunused: Now work in bare repositories.
+  * Removed the setkey command, and added a setcontent command with a more
+    useful interface.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dc0b49ab2c..abddaa0dea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -274,6 +274,20 @@ subdirectories).
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
+* setcontent src dest
+
+  Makes the dest file, which must already be tracked by git-annex have the
+  content of the src file. The src file is removed. This can be useful if you
+  have obtained the content of a file from elsewhere and want to put it in
+  the local annex.
+
+  Automatically runs fsck on dest to check that the expected content was
+  provided.
+
+  Example:
+
+	git annex setcontent /tmp/foo.iso foo.iso
+
 * unannex [path ...]
 
   Use this to undo an accidental `git annex add` command. You can use
@@ -321,15 +335,6 @@ subdirectories).
 
 	git annex dropkey SHA1-s10-7da006579dd64330eb2456001fd01948430572f2
 
-* setkey file
-
-  This plumbing-level command sets the annexed data for a key to the
-  content of the specified file, and then removes the file.
-
-  Example:
-
-	git annex setkey --key=WORM-s3-m1287765018--file /tmp/file
-
 # OPTIONS
 
 * --force

From 380839299ea8ffa2c98ce6219e62b979ede0c93b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 12:47:13 -0400
Subject: [PATCH 2429/8313] The fromkey command now takes the key as its first
 parameter. The --key option is no longer used.

---
 Annex.hs                                      |  2 --
 Command/FromKey.hs                            | 22 +++++++++----------
 Command/SetContent.hs                         |  2 +-
 Config.hs                                     | 13 -----------
 debian/changelog                              |  2 ++
 ..._480a4f72445a636eab1b1c0f816d365c._comment |  3 ++-
 doc/git-annex.mdwn                            |  6 +----
 test.hs                                       | 20 ++++++++---------
 8 files changed, 27 insertions(+), 43 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index b9e71c9314..2c6402ac35 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -60,7 +60,6 @@ data AnnexState = AnnexState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
-	, defaultkey :: Maybe String
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
 	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
@@ -85,7 +84,6 @@ newState gitrepo = AnnexState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
-	, defaultkey = Nothing
 	, toremote = Nothing
 	, fromremote = Nothing
 	, limit = Left []
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index 4e4644708f..b910dd1f09 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -12,26 +12,26 @@ import Command
 import qualified Annex.Queue
 import Annex.Content
 import Types.Key
-import Config
 
 def :: [Command]
-def = [command "fromkey" paramPath seek "adds a file using a specific key"]
+def = [command "fromkey" (paramPair paramKey paramPath) seek
+	"adds a file using a specific key"]
 
 seek :: [CommandSeek]
-seek = [withFilesMissing start]
+seek = [withWords start]
 
-start :: FilePath -> CommandStart
-start file = notBareRepo $ do
-	key <- cmdlineKey
+start :: [String] -> CommandStart
+start (keyname:file:[]) = notBareRepo $ do
+	let key = maybe (error "bad key") id $ readKey keyname
 	inbackend <- inAnnex key
 	unless inbackend $ error $
-		"key ("++keyName key++") is not present in backend"
+		"key ("++ keyname ++") is not present in backend"
 	showStart "fromkey" file
-	next $ perform file
+	next $ perform key file
+start _ = error "specify a key and a dest file"
 
-perform :: FilePath -> CommandPerform
-perform file = do
-	key <- cmdlineKey
+perform :: Key -> FilePath -> CommandPerform
+perform key file = do
 	link <- calcGitLink file key
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 	liftIO $ createSymbolicLink link file
diff --git a/Command/SetContent.hs b/Command/SetContent.hs
index d62faa445f..0ecfa34bd4 100644
--- a/Command/SetContent.hs
+++ b/Command/SetContent.hs
@@ -23,7 +23,7 @@ seek = [withWords start]
 
 start :: [FilePath] -> CommandStart
 start (src:dest:[]) = do
-	showStart "setkey" dest
+	showStart "setcontent" dest
 	next $ perform src dest
 start _ = error "specify a src file and a dest file"
 
diff --git a/Config.hs b/Config.hs
index f994002b93..bf929219df 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,7 +10,6 @@ module Config where
 import Common.Annex
 import qualified Git
 import qualified Annex
-import Types.Key (readKey)
 
 type ConfigKey = String
 
@@ -92,15 +91,3 @@ getNumCopies v =
 			g <- gitRepo
 			return $ read $ Git.configGet g config "1"
 		config = "annex.numcopies"
-
-{- The Key specified by the --key parameter. -}
-cmdlineKey :: Annex Key
-cmdlineKey  = do
-	k <- Annex.getState Annex.defaultkey
-	case k of
-		Nothing -> nokey
-		Just "" -> nokey
-		Just kstring -> maybe badkey return $ readKey kstring
-	where
-		nokey = error "please specify the key with --key"
-		badkey = error "bad key"
diff --git a/debian/changelog b/debian/changelog
index 91e4a18af1..25f18f7950 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -15,6 +15,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * unused, dropunused: Now work in bare repositories.
   * Removed the setkey command, and added a setcontent command with a more
     useful interface.
+  * The fromkey command now takes the key as its first parameter. The --key
+    option is no longer used.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 
diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
index 4a22d414b2..787173b0a6 100644
--- a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
@@ -4,5 +4,6 @@
  subject="comment 4"
  date="2011-05-14T16:29:35Z"
  content="""
-Although, if you really do want to shoot yourself in the foot, or know you have the old content, you can use `git-annex setkey`.
+Although, if you really do want to shoot yourself in the foot, or know you
+have the old content, you can use `git-annex setcontent`.
 """]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index abddaa0dea..42997e5342 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -318,7 +318,7 @@ subdirectories).
   This is meant to be called from git's pre-commit hook. `git annex init`
   automatically creates a pre-commit hook using this.
 
-* fromkey file
+* fromkey key file
 
   This plumbing-level command can be used to manually set up a file
   in the git repository to link to a specified key.
@@ -406,10 +406,6 @@ subdirectories).
   are in the annex, their backend is known and this option is not
   necessary.
 
-* --key=name
-
-  Specifies a key to operate on.
-
 * -c name=value
 
   Used to override git configuration settings. May be specified multiple times.
diff --git a/test.hs b/test.hs
index 0ef7449162..fa7657b581 100644
--- a/test.hs
+++ b/test.hs
@@ -88,7 +88,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	-- test order matters, later tests may rely on state from earlier
 	[ test_init
 	, test_add
-	, test_setkey
+	, test_setcontent
 	, test_unannex
 	, test_drop
 	, test_get
@@ -118,15 +118,15 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			writeFile annexedfile $ content annexedfile
 			git_annex "add" ["-q", annexedfile] @? "add failed"
 			annexed_present annexedfile
+			writeFile sha1annexedfile $ content sha1annexedfile
+			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
+			annexed_present sha1annexedfile
 			writeFile ingitfile $ content ingitfile
 			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
 			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
 			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
-			writeFile sha1annexedfile $ content sha1annexedfile
-			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
-			annexed_present sha1annexedfile
 			writeFile sha1annexedfiledup $ content sha1annexedfiledup
 			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
@@ -140,15 +140,15 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			changeWorkingDirectory "dir"
 			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
 
-test_setkey :: Test
-test_setkey = "git-annex setkey/fromkey" ~: TestCase $ inmainrepo $ do
+test_setcontent :: Test
+test_setcontent = "git-annex setcontent/fromkey" ~: TestCase $ intmpclonerepo $ do
+	git_annex "drop" ["-q", "--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
 	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
-	git_annex "setkey" ["-q", "--key", key, tmp] @? "setkey failed"
-	git_annex "fromkey" ["-q", "--key", key, sha1annexedfile] @? "fromkey failed"
-	boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
-	annexed_present sha1annexedfile
+	git_annex "setcontent" ["-q", tmp, sha1annexedfile] @? "setcontent failed"
+	git_annex "fromkey" ["-q", key, sha1annexedfiledup] @? "fromkey failed"
+	annexed_present sha1annexedfiledup
 	where
 		tmp = "tmpfile"
 

From 09861cf4f75e2bb9e5597f8b9ea39ab4a33bf4d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 15:12:02 -0400
Subject: [PATCH 2430/8313] cleanup

---
 Command/SetContent.hs | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/Command/SetContent.hs b/Command/SetContent.hs
index 0ecfa34bd4..b63cc9119a 100644
--- a/Command/SetContent.hs
+++ b/Command/SetContent.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import Logs.Location
 import Annex.Content
-import qualified Backend
 import qualified Command.Fsck
 
 def :: [Command]
@@ -28,22 +27,18 @@ start (src:dest:[]) = do
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> CommandPerform
-perform src dest = go =<< Backend.lookupFile dest
+perform src dest = isAnnexed dest $ \(key, backend) -> do
+	unlessM (move key) $ error "mv failed!"
+	next $ cleanup key backend
 	where
-		go Nothing = error "dest file is not in annex"
-		go (Just (key, backend)) = do
-			-- the file might be on a different filesystem,
-			-- so mv is used rather than simply calling
-			-- moveToObjectDir; disk space is also
-			-- checked this way.
-			ok <- getViaTmp key $ \tmp ->
-				if dest /= src
-					then liftIO $
-						boolSystem "mv" [File src, File tmp]
-					else return True
-			if ok
-				then next $ cleanup key backend
-				else error "mv failed!"
+		-- the file might be on a different filesystem,
+		-- so mv is used rather than simply calling
+		-- moveToObjectDir; disk space is also
+		-- checked this way.
+		move key = getViaTmp key $ \tmp ->
+			if dest /= src
+			then liftIO $ boolSystem "mv" [File src, File tmp]
+			else return True
 
 cleanup :: Key -> Backend Annex -> CommandCleanup
 cleanup key backend = do

From 3d3e1c4c25f4bbefd0f5e3f445444f3293875a93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 15:18:41 -0400
Subject: [PATCH 2431/8313] better command name

---
 Command/{SetContent.hs => Reinject.hs}                |  6 +++---
 GitAnnex.hs                                           |  4 ++--
 debian/changelog                                      |  2 +-
 ...omment_4_480a4f72445a636eab1b1c0f816d365c._comment |  3 +--
 doc/git-annex.mdwn                                    | 11 +++++------
 test.hs                                               |  8 ++++----
 6 files changed, 16 insertions(+), 18 deletions(-)
 rename Command/{SetContent.hs => Reinject.hs} (90%)

diff --git a/Command/SetContent.hs b/Command/Reinject.hs
similarity index 90%
rename from Command/SetContent.hs
rename to Command/Reinject.hs
index b63cc9119a..63309aa522 100644
--- a/Command/SetContent.hs
+++ b/Command/Reinject.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Command.SetContent where
+module Command.Reinject where
 
 import Common.Annex
 import Command
@@ -14,7 +14,7 @@ import Annex.Content
 import qualified Command.Fsck
 
 def :: [Command]
-def = [command "setcontent" (paramPair paramPath paramPath) seek
+def = [command "reinject" (paramPair paramPath paramPath) seek
 	"sets content of annexed file"]
 
 seek :: [CommandSeek]
@@ -22,7 +22,7 @@ seek = [withWords start]
 
 start :: [FilePath] -> CommandStart
 start (src:dest:[]) = do
-	showStart "setcontent" dest
+	showStart "reinject" dest
 	next $ perform src dest
 start _ = error "specify a src file and a dest file"
 
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 09f0a118c0..399b26ef7f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -26,7 +26,7 @@ import qualified Command.Copy
 import qualified Command.Get
 import qualified Command.FromKey
 import qualified Command.DropKey
-import qualified Command.SetContent
+import qualified Command.Reinject
 import qualified Command.Fix
 import qualified Command.Init
 import qualified Command.Describe
@@ -63,7 +63,7 @@ cmds = concat
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
-	, Command.SetContent.def
+	, Command.Reinject.def
 	, Command.Unannex.def
 	, Command.Uninit.def
 	, Command.PreCommit.def
diff --git a/debian/changelog b/debian/changelog
index 25f18f7950..dc6e3c38aa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,7 +13,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     .gitattributes information about numcopies is not available in a bare
     repository.
   * unused, dropunused: Now work in bare repositories.
-  * Removed the setkey command, and added a setcontent command with a more
+  * Removed the setkey command, and added a reinject command with a more
     useful interface.
   * The fromkey command now takes the key as its first parameter. The --key
     option is no longer used.
diff --git a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
index 787173b0a6..fcca0561d4 100644
--- a/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
+++ b/doc/bugs/No_easy_way_to_re-inject_a_file_into_an_annex/comment_4_480a4f72445a636eab1b1c0f816d365c._comment
@@ -4,6 +4,5 @@
  subject="comment 4"
  date="2011-05-14T16:29:35Z"
  content="""
-Although, if you really do want to shoot yourself in the foot, or know you
-have the old content, you can use `git-annex setcontent`.
+Now available as `git-annex reinject`.
 """]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 42997e5342..fdd8dd1c19 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -274,19 +274,18 @@ subdirectories).
   However, if a backend changes the information it uses to construct a key,
   this can also be used to migrate files to use the new key format.
 
-* setcontent src dest
+* reinject src dest
 
-  Makes the dest file, which must already be tracked by git-annex have the
-  content of the src file. The src file is removed. This can be useful if you
-  have obtained the content of a file from elsewhere and want to put it in
-  the local annex.
+  Moves the src file into the annex as the content of the dest file.
+  This can be useful if you have obtained the content of a file from
+  elsewhere and want to put it in the local annex.
 
   Automatically runs fsck on dest to check that the expected content was
   provided.
 
   Example:
 
-	git annex setcontent /tmp/foo.iso foo.iso
+	git annex reinject /tmp/foo.iso foo.iso
 
 * unannex [path ...]
 
diff --git a/test.hs b/test.hs
index fa7657b581..d466d3ad32 100644
--- a/test.hs
+++ b/test.hs
@@ -88,7 +88,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	-- test order matters, later tests may rely on state from earlier
 	[ test_init
 	, test_add
-	, test_setcontent
+	, test_reinject
 	, test_unannex
 	, test_drop
 	, test_get
@@ -140,13 +140,13 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			changeWorkingDirectory "dir"
 			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
 
-test_setcontent :: Test
-test_setcontent = "git-annex setcontent/fromkey" ~: TestCase $ intmpclonerepo $ do
+test_reinject :: Test
+test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do
 	git_annex "drop" ["-q", "--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
 	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
-	git_annex "setcontent" ["-q", tmp, sha1annexedfile] @? "setcontent failed"
+	git_annex "reinject" ["-q", tmp, sha1annexedfile] @? "reinject failed"
 	git_annex "fromkey" ["-q", key, sha1annexedfiledup] @? "fromkey failed"
 	annexed_present sha1annexedfiledup
 	where

From 00988bcf369671bdc3b78e95e3c2ae43f4835b1c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 15:40:57 -0400
Subject: [PATCH 2432/8313] fixed my build environment

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index dc6e3c38aa..b17586bc68 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     useful interface.
   * The fromkey command now takes the key as its first parameter. The --key
     option is no longer used.
+  * Built without any filename containing .git being excluded. Closes: #647215
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From 3d2a9f84051e9dc705ba4bb4828af691e479ae0e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 16:46:51 -0400
Subject: [PATCH 2433/8313] cleanup

---
 Command/Add.hs        | 15 +++++++--------
 Command/AddUrl.hs     | 12 +++++-------
 Command/Describe.hs   | 10 +++-------
 Command/DropKey.hs    | 14 ++++++++------
 Command/DropUnused.hs |  3 +--
 Command/InitRemote.hs |  4 +++-
 Command/Migrate.hs    | 11 ++++++-----
 Command/Reinject.hs   | 12 ++++++------
 Command/Version.hs    | 11 ++++++-----
 Command/Whereis.hs    |  6 ++----
 Init.hs               |  4 +---
 11 files changed, 48 insertions(+), 54 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index cd18f6c725..a633db7b36 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -38,11 +38,10 @@ start p@(_, file) = notBareRepo $ notAnnexed file $ do
 			next $ perform p
 
 perform :: BackendFile -> CommandPerform
-perform (backend, file) = do
-	k <- Backend.genKey file backend
-	case k of
-		Nothing -> stop
-		Just (key, _) -> do
+perform (backend, file) = Backend.genKey file backend >>= go
+	where
+		go Nothing = stop
+		go (Just (key, _)) = do
 			handle (undo file key) $ moveAnnex key file
 			next $ cleanup file key True
 
@@ -75,9 +74,9 @@ cleanup file key hascontent = do
 	
 			-- touch the symlink to have the same mtime as the
 			-- file it points to
-			s <- liftIO $ getFileStatus file
-			let mtime = modificationTime s
-			liftIO $ touch file (TimeSpec mtime) False
+			liftIO $ do
+				mtime <- modificationTime <$> getFileStatus file
+				touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 4382a9c07a..b717e271d1 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -26,15 +26,14 @@ seek :: [CommandSeek]
 seek = [withStrings start]
 
 start :: String -> CommandStart
-start s = notBareRepo $ do
-	let u = parseURI s
-	case u of
-		Nothing -> error $ "bad url " ++ s
-		Just url -> do
+start s = notBareRepo $ go $ parseURI s
+	where
+		go Nothing = error $ "bad url " ++ s
+		go (Just url) = do
 			file <- liftIO $ url2file url
 			showStart "addurl" file
 			next $ perform s file
-			
+
 perform :: String -> FilePath -> CommandPerform
 perform url file = do
 	fast <- Annex.getState Annex.fast
@@ -64,7 +63,6 @@ nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
 	let key = Backend.URL.fromUrl url
 	setUrlPresent key url
-	
 	next $ Command.Add.cleanup file key False
 
 url2file :: URI -> IO FilePath
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 882a0e1bb9..1cc81bcbd5 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -20,15 +20,11 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: [String] -> CommandStart
-start ws = do
-	let (name, description) =
-		case ws of
-			(n:d) -> (n,unwords d)
-			_ -> error "Specify a repository and a description."
-	
+start (name:description) = do
 	showStart "describe" name
 	u <- Remote.nameToUUID name
-	next $ perform u description
+	next $ perform u $ unwords description
+start _ = do error "Specify a repository and a description."	
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index d00bb6c83e..cac955b4ac 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -23,14 +23,16 @@ seek = [withKeys start]
 start :: Key -> CommandStart
 start key = do
 	present <- inAnnex key
-	force <- Annex.getState Annex.force
 	if not present
 		then stop
-		else if not force
-			then error "dropkey is can cause data loss; use --force if you're sure you want to do this"
-			else do
-				showStart "dropkey" (show key)
-				next $ perform key
+		else do
+			checkforced
+			showStart "dropkey" (show key)
+			next $ perform key
+	where
+		checkforced = 
+			unlessM (Annex.getState Annex.force) $
+				error "dropkey can cause data loss; use --force if you're sure you want to do this"
 
 perform :: Key -> CommandPerform
 perform key = do
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 07ae1b48c3..70dcc4cc72 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -71,8 +71,7 @@ readUnusedLog prefix = do
 	let f = gitAnnexUnusedLog prefix g
 	e <- liftIO $ doesFileExist f
 	if e
-		then return . M.fromList . map parse . lines
-			=<< liftIO (readFile f)
+		then M.fromList . map parse . lines <$> liftIO (readFile f)
 		else return M.empty
 	where
 		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 4ba5b07873..600e17eb84 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -67,7 +67,9 @@ findByName name = do
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
-findByName' n m = if null matches then Nothing else Just $ head matches
+findByName' n m
+	| null matches = Nothing
+	| otherwise = Just $ head matches
 	where
 		matches = filter (matching . snd) $ M.toList m
 		matching c = case M.lookup nameKey c of
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a68582996b..2d4d24a224 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -56,11 +56,7 @@ perform file oldkey newbackend = do
 	case k of
 		Nothing -> stop
 		Just (newkey, _) -> do
-			ok <- getViaTmpUnchecked newkey $ \t -> do
-				-- Make a hard link to the old backend's
-				-- cached key, to avoid wasting disk space.
-				liftIO $ unlessM (doesFileExist t) $ createLink src t
-				return True
+			ok <- link src newkey
 			if ok
 				then do
 					-- Update symlink to use the new key.
@@ -77,3 +73,8 @@ perform file oldkey newbackend = do
 				else stop
 	where
 		cleantmp t = whenM (doesFileExist t) $ removeFile t
+		link src newkey = getViaTmpUnchecked newkey $ \t -> do
+			-- Make a hard link to the old backend's
+			-- cached key, to avoid wasting disk space.
+			liftIO $ unlessM (doesFileExist t) $ createLink src t
+			return True	
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 63309aa522..3de24f3fc3 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -21,9 +21,11 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: [FilePath] -> CommandStart
-start (src:dest:[]) = do
-	showStart "reinject" dest
-	next $ perform src dest
+start (src:dest:[])
+	| src == dest = stop
+	| otherwise = do
+		showStart "reinject" dest
+		next $ perform src dest
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> CommandPerform
@@ -36,9 +38,7 @@ perform src dest = isAnnexed dest $ \(key, backend) -> do
 		-- moveToObjectDir; disk space is also
 		-- checked this way.
 		move key = getViaTmp key $ \tmp ->
-			if dest /= src
-			then liftIO $ boolSystem "mv" [File src, File tmp]
-			else return True
+			liftIO $ boolSystem "mv" [File src, File tmp]
 
 cleanup :: Key -> Backend Annex -> CommandCleanup
 cleanup key backend = do
diff --git a/Command/Version.hs b/Command/Version.hs
index 5a45fd77f2..a584264828 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -21,12 +21,13 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	liftIO $ putStrLn $ "git-annex version: " ++ SysConfig.packageversion
 	v <- getVersion
-	liftIO $ putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
-	liftIO $ putStrLn $ "default repository version: " ++ defaultVersion
-	liftIO $ putStrLn $ "supported repository versions: " ++ vs supportedVersions
-	liftIO $ putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
+	liftIO $ do
+		putStrLn $ "git-annex version: " ++ SysConfig.packageversion
+		putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
+		putStrLn $ "default repository version: " ++ defaultVersion
+		putStrLn $ "supported repository versions: " ++ vs supportedVersions
+		putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
 	stop
 	where
 		vs = join " "
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 7799af08c6..0681bfba1e 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -31,11 +31,9 @@ perform key = do
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations
-	unless (null safelocations) $
-		showLongNote pp
+	unless (null safelocations) $ showLongNote pp
 	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
-	unless (null untrustedlocations) $
-		showLongNote $ untrustedheader ++ pp'
+	unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp'
 	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
diff --git a/Init.hs b/Init.hs
index 6e024e9fc5..1fac80820a 100644
--- a/Init.hs
+++ b/Init.hs
@@ -65,9 +65,7 @@ gitPreCommitHookUnWrite = unlessBare $ do
 				" Edit it to remove call to git annex."
 
 unlessBare :: Annex () -> Annex ()
-unlessBare a = do
-	g <- gitRepo
-	unless (Git.repoIsLocalBare g) a
+unlessBare = unlessM $ Git.repoIsLocalBare <$> gitRepo
 
 preCommitHook :: Annex FilePath
 preCommitHook = do

From c643136e32b99a650a6fdea0731ea5af275f6866 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 31 Oct 2011 23:39:55 -0400
Subject: [PATCH 2434/8313] playing with >=>

Apparently in haskell if you teach a man to fish, he'll write
more pointfree code.
---
 Backend/SHA.hs  | 2 +-
 Command/Fsck.hs | 2 +-
 Limit.hs        | 4 ++--
 Utility/Misc.hs | 5 +++--
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3a54a8871b..3f210df1f1 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -83,7 +83,7 @@ keyValue size file = do
 
 {- Extension preserving keys. -}
 keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
-keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
+keyValueE size file = keyValue >>= maybe (return Nothing) addE
 	where
 		addE k = return $ Just $ k
 			{ keyName = keyName k ++ extension
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 8d6f03d76b..d1abb29e37 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -69,7 +69,7 @@ performBare key backend = check
 	]
 
 check :: [Annex Bool] -> CommandPerform	
-check s = sequence s >>= dispatch
+check = sequence >=> dispatch
 	where
 		dispatch vs
 			| all (== True) vs = next $ return True
diff --git a/Limit.hs b/Limit.hs
index 490577e80f..3ae949bfb9 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -67,7 +67,7 @@ addExclude glob = addLimit $ return . notExcluded
 addIn :: String -> Annex ()
 addIn name = addLimit $ check $ if name == "." then inAnnex else inremote
 	where
-		check a f = Backend.lookupFile f >>= handle a
+		check a = Backend.lookupFile >=> handle a
 		handle _ Nothing = return False
 		handle a (Just (key, _)) = a key
 		inremote key = do
@@ -83,7 +83,7 @@ addCopies num =
 		Nothing -> error "bad number for --copies"
 		Just n -> addLimit $ check n
 	where
-		check n f = Backend.lookupFile f >>= handle n
+		check n = Backend.lookupFile >=> handle n
 		handle _ Nothing = return False
 		handle n (Just (key, _)) = do
 			us <- keyLocations key
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index bc18347746..b3bf226120 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -8,15 +8,16 @@
 module Utility.Misc where
 
 import System.IO
+import Control.Monad
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
 hGetContentsStrict :: Handle -> IO String
-hGetContentsStrict h  = hGetContents h >>= \s -> length s `seq` return s
+hGetContentsStrict  = hGetContents >=> \s -> length s `seq` return s
 
 {- A version of readFile that is not lazy. -}
 readFileStrict :: FilePath -> IO String
-readFileStrict f = readFile f >>= \s -> length s `seq` return s
+readFileStrict = readFile >=> \s -> length s `seq` return s
 
 {- Attempts to read a value from a String. -}
 readMaybe :: (Read a) => String -> Maybe a

From e9286e7be7a8b68d9a0e02e909bf58c7936b12af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 12:02:04 -0400
Subject: [PATCH 2435/8313] point to new extension now in mercurial

---
 doc/not.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/not.mdwn b/doc/not.mdwn
index 2827dd12d0..ad278da0dd 100644
--- a/doc/not.mdwn
+++ b/doc/not.mdwn
@@ -43,7 +43,7 @@
   same repository, which git-annex does, and is an important feature for
   large-scale archiving.
 
-* git-annex is not the [Mercurial bfiles extension](http://mercurial.selenic.com/wiki/BfilesExtension).
+* git-annex is not the [Mercurial largefiles extension](http://mercurial.selenic.com/wiki/LargefilesExtension).
   Although mercurial and git have some of the same problems around large
   files, and both try to solve them in similar ways (standin files using
   mostly hashes of the real content).

From 8e249ea0bdbf0b0890d6e3d3ed1e30c1df36e136 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 13:32:19 -0400
Subject: [PATCH 2436/8313] add tip for mode of operation somewhat like
 mercurial bigfiles extension

---
 .../automatically_getting_files_on_checkout.mdwn  | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/tips/automatically_getting_files_on_checkout.mdwn

diff --git a/doc/tips/automatically_getting_files_on_checkout.mdwn b/doc/tips/automatically_getting_files_on_checkout.mdwn
new file mode 100644
index 0000000000..bbb3b302eb
--- /dev/null
+++ b/doc/tips/automatically_getting_files_on_checkout.mdwn
@@ -0,0 +1,15 @@
+Normally git-annex does not retrieve file contents when checking out a
+tree. In some use cases, it makes sense to always have the contents of
+files available after a `git checkout` or `git update`. This can be
+accomplished by installing the following as `.git/hooks/post-checkout`
+
+	#!/bin/sh
+	# Uses git-annex to get all files in the specified directories
+	# (relative to the top of the repository) on checkout.
+	dirs=.
+	top="$(git rev-parse --show-toplevel)"
+	for dir in "$dirs"; do git annex get $top/$dir"; done
+
+By default, all files in the whole repository will be made available. The
+`dirs` setting can be configured if you only want to get files in certian
+directories. 

From eec137f33a2b6982403d7dd9187500ad5a14b868 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 14:18:21 -0400
Subject: [PATCH 2437/8313] Record uuid when auto-initializing a remote so it
 shows in status.

---
 Backend/SHA.hs   |  2 +-
 Init.hs          |  2 ++
 Logs/UUID.hs     | 10 ++++++++++
 debian/changelog |  1 +
 4 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3f210df1f1..3a54a8871b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -83,7 +83,7 @@ keyValue size file = do
 
 {- Extension preserving keys. -}
 keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
-keyValueE size file = keyValue >>= maybe (return Nothing) addE
+keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 	where
 		addE k = return $ Just $ k
 			{ keyName = keyName k ++ extension
diff --git a/Init.hs b/Init.hs
index 1fac80820a..8c79002bcb 100644
--- a/Init.hs
+++ b/Init.hs
@@ -15,6 +15,7 @@ import Common.Annex
 import Utility.TempFile
 import qualified Git
 import qualified Annex.Branch
+import Logs.UUID
 import Annex.Version
 import Annex.UUID
 
@@ -24,6 +25,7 @@ initialize = do
 	Annex.Branch.create
 	setVersion
 	gitPreCommitHookWrite
+	getUUID >>= recordUUID
 
 uninitialize :: Annex ()
 uninitialize = gitPreCommitHookUnWrite
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index c05c4e3481..77cfb5ce0f 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -15,6 +15,7 @@
 
 module Logs.UUID (
 	describeUUID,
+	recordUUID,
 	uuidMap
 ) where
 
@@ -37,6 +38,15 @@ describeUUID uuid desc = do
 	Annex.Branch.change logfile $
 		showLog id . changeLog ts uuid desc . parseLog Just
 
+{- Records the uuid in the log, if it's not already there. -}
+recordUUID :: UUID -> Annex ()
+recordUUID u = go . M.lookup u =<< uuidMap 
+	where
+		go (Just "") = set
+		go Nothing = set
+		go _ = return ()
+		set = describeUUID u ""
+
 {- Read the uuidLog into a simple Map.
  -
  - The UUID of the current repository is included explicitly, since
diff --git a/debian/changelog b/debian/changelog
index b17586bc68..5beb32ee1f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -18,6 +18,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * The fromkey command now takes the key as its first parameter. The --key
     option is no longer used.
   * Built without any filename containing .git being excluded. Closes: #647215
+  * Record uuid when auto-initializing a remote so it shows in status.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From c33313c50b1a9424dd20e6e9995c46d60a48e540 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 14:24:44 -0400
Subject: [PATCH 2438/8313] tweak

---
 Command/Init.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index e2a6eb7b03..69cdc8e8a4 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -30,6 +30,5 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	u <- getUUID
-	describeUUID u description
+	getUUID >>= describeUUID description
 	next $ return True

From 5f3dd3d246c757ee92aa8e68654071519364c0e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 Nov 2011 15:09:19 -0400
Subject: [PATCH 2439/8313] ensure directory exists when locking journal

Fixes git annex init in a bare repository that already has a git-annex
branch.
---
 Annex/Branch.hs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 645a2de762..4c3192f531 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -250,10 +250,9 @@ files = withIndexUpdate $ do
 setJournalFile :: FilePath -> String -> Annex ()
 setJournalFile file content = do
 	g <- gitRepo
-	liftIO $ catch (write g) $ const $ do
+	liftIO $ doRedo (write g) $ do
 		createDirectoryIfMissing True $ gitAnnexJournalDir g
 		createDirectoryIfMissing True $ gitAnnexTmpDir g
-		write g
 	where
 		-- journal file is written atomically
 		write g = do
@@ -342,7 +341,13 @@ lockJournal a = do
 	bracketIO (lock file) unlock a
 	where
 		lock file = do
-			l <- createFile file stdFileMode
+			l <- doRedo (createFile file stdFileMode) $
+				createDirectoryIfMissing True $ takeDirectory file
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
+
+{- Runs an action, catching failure and running something to fix it up, and
+ - retrying if necessary. -}
+doRedo :: IO a -> IO b -> IO a
+doRedo a b = catch a $ const $ b >> a

From 7df8052a4b5289ed48f0bcb9aa95b319caed76d5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Thu, 3 Nov 2011 23:53:51 +0000
Subject: [PATCH 2440/8313]

---
 doc/forum/Recommended_number_of_repositories.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/forum/Recommended_number_of_repositories.mdwn

diff --git a/doc/forum/Recommended_number_of_repositories.mdwn b/doc/forum/Recommended_number_of_repositories.mdwn
new file mode 100644
index 0000000000..9e9f2838d6
--- /dev/null
+++ b/doc/forum/Recommended_number_of_repositories.mdwn
@@ -0,0 +1,4 @@
+With git it is easy to create one repository per project, and it almost always makes sense to do so.  When using git-annex, what is the recommended setup?
+
+Should one have a single annex containing all files, or is it recommended to create different repositories for things like 'photos', 'music', 'isos' ?
+

From 532ff19aa5a18821fdb36df60837ba9690d0c963 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 3 Nov 2011 13:14:38 -0400
Subject: [PATCH 2441/8313] fix link

---
 doc/design/encryption.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn
index e5053134ee..647683bd9f 100644
--- a/doc/design/encryption.mdwn
+++ b/doc/design/encryption.mdwn
@@ -1,6 +1,6 @@
 This was the design doc for [[/encryption]] and is preserved for
 the curious. For an example of using git-annex with an encrypted S3 remote,
-see [[walkthrough/using_Amazon_S3]].
+see [[tips/using_Amazon_S3]].
 
 [[!toc]]
 

From 1089e85d48a0d3c455fc2f4139b82484b94b5bbe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Nov 2011 15:01:43 -0400
Subject: [PATCH 2442/8313] add changelog for bugfix

---
 debian/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 5beb32ee1f..e59b4f4048 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,8 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     option is no longer used.
   * Built without any filename containing .git being excluded. Closes: #647215
   * Record uuid when auto-initializing a remote so it shows in status.
+  * Bugfix: Fixed git-annex init crash in a bare repository when there was
+    already an existing git-annex branch.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From ef3457196ace3669ddfa93039f2d3c15baf54713 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Nov 2011 15:21:45 -0400
Subject: [PATCH 2443/8313] use SHA256 by default

To get old behavior, add a .gitattributes containing: * annex.backend=WORM

I feel that SHA256 is a better default for most people, as long as their
systems are fast enough that checksumming their files isn't a problem.
git-annex should default to preserving the integrity of data as well as git
does. Checksum backends also work better with editing files via
unlock/lock.

I considered just using SHA1, but since that hash is believed to be somewhat
near to being broken, and git-annex deals with large files which would be a
perfect exploit medium, I decided to go to a SHA-2 hash.

SHA512 is annoyingly long when displayed, and git-annex displays it in a
few places (and notably it is shown in ls -l), so I picked the shorter
hash. Considered SHA224 as it's even shorter, but feel it's a bit weird.

I expect git-annex will use SHA-3 at some point in the future, but
probably not soon!

Note that systems without a sha256sum (or sha256) program will fall back to
defaulting to SHA1.
---
 Backend.hs                                    |  4 +--
 Backend/SHA.hs                                |  6 ++--
 debian/changelog                              |  3 ++
 doc/backends.mdwn                             | 32 +++++++++++--------
 doc/walkthrough/adding_files.mdwn             |  4 +--
 ...ing_file_content_between_repositories.mdwn |  2 +-
 doc/walkthrough/unused_data.mdwn              | 14 ++++----
 doc/walkthrough/using_ssh_remotes.mdwn        |  2 +-
 8 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index a09fc0e990..9a40e54598 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -26,12 +26,12 @@ import Types.Key
 import qualified Types.Backend as B
 
 -- When adding a new backend, import it here and add it to the list.
-import qualified Backend.WORM
 import qualified Backend.SHA
+import qualified Backend.WORM
 import qualified Backend.URL
 
 list :: [Backend Annex]
-list = Backend.WORM.backends ++ Backend.SHA.backends ++ Backend.URL.backends
+list = Backend.SHA.backends ++ Backend.WORM.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
 orderedList :: Annex [Backend Annex]
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3a54a8871b..d449821172 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -16,12 +16,12 @@ import qualified Build.SysConfig as SysConfig
 
 type SHASize = Int
 
+-- order is slightly significant; want SHA256 first, and more general
+-- sizes earlier
 sizes :: [Int]
-sizes = [1, 256, 512, 224, 384]
+sizes = [256, 1, 512, 224, 384]
 
 backends :: [Backend Annex]
--- order is slightly significant; want sha1 first, and more general
--- sizes earlier
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
 genBackend :: SHASize -> Maybe (Backend Annex)
diff --git a/debian/changelog b/debian/changelog
index e59b4f4048..e74a190ba5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,8 @@
 git-annex (3.20111026) UNRELEASED; urgency=low
 
+  * The default backend used when adding files to the annex is changed
+    from WORM to SHA256.
+    To get old behavior, add a .gitattributes containing: * annex.backend=WORM
   * Sped up some operations on remotes that are on the same host.
   * copy --to: Fixed leak when copying many files to a remote on the same
     host.
diff --git a/doc/backends.mdwn b/doc/backends.mdwn
index ebcdedc2a7..2030d107a3 100644
--- a/doc/backends.mdwn
+++ b/doc/backends.mdwn
@@ -5,17 +5,19 @@ to retrieve the file's content (its value).
 Multiple pluggable key-value backends are supported, and a single repository
 can use different ones for different files.
 
-* `WORM` ("Write Once, Read Many") This assumes that any file with
-  the same basename, size, and modification time has the same content.
-  This is the default, and the least expensive backend.
-* `SHA1` -- This uses a key based on a sha1 checksum. This allows
+* `SHA256` -- The default backend for new files. This allows
   verifying that the file content is right, and can avoid duplicates of
   files with the same content. Its need to generate checksums
-  can make it slower for large files.
-* `SHA512`, `SHA384`, `SHA256`, `SHA224` -- Like SHA1, but larger
-  checksums. Mostly useful for the very paranoid, or anyone who is
-  researching checksum collisions and wants to annex their colliding data. ;)
-* `SHA1E`, `SHA512E`, etc -- Variants that preserve filename extension as
+  can make it slower for large files. 
+* `WORM` ("Write Once, Read Many") This assumes that any file with
+  the same basename, size, and modification time has the same content.
+  This is the the least expensive backend, recommended for really large
+  files or slow systems.
+* `SHA512` -- Best currently available hash, for the very paranoid.
+* `SHA1` -- Smaller hash than `SHA256` for those who want a checksum
+   but are not concerned about security.
+* `SHA384`, `SHA224` -- Hashes for people who like unusual sizes.
+* `SHA256E`, `SHA1E`, etc -- Variants that preserve filename extension as
   part of the key. Useful for archival tasks where the filename extension
   contains metadata that should be preserved.
 
@@ -27,9 +29,11 @@ For finer control of what backend is used when adding different types of
 files, the `.gitattributes` file can be used. The `annex.backend`
 attribute can be set to the name of the backend to use for matching files.
 
-For example, to use the SHA1 backend for sound files, which tend to be
-smallish and might be modified or copied over time, you could set in
-`.gitattributes`:
+For example, to use the SHA256 backend for sound files, which tend to be
+smallish and might be modified or copied over time,
+while using the WORM backend for everything else, you could set
+in `.gitattributes`:
 
-	*.mp3 annex.backend=SHA1
-	*.ogg annex.backend=SHA1
+	* annex.backend=WORM
+	*.mp3 annex.backend=SHA256
+	*.ogg annex.backend=SHA256
diff --git a/doc/walkthrough/adding_files.mdwn b/doc/walkthrough/adding_files.mdwn
index 77a7fbc154..d1b5a04f77 100644
--- a/doc/walkthrough/adding_files.mdwn
+++ b/doc/walkthrough/adding_files.mdwn
@@ -2,8 +2,8 @@
 	# cp /tmp/big_file .
 	# cp /tmp/debian.iso .
 	# git annex add .
-	add big_file ok
-	add debian.iso ok
+	add big_file (checksum...) ok
+	add debian.iso (checksum...) ok
 	# git commit -a -m added
 
 When you add a file to the annex and commit it, only a symlink to
diff --git a/doc/walkthrough/moving_file_content_between_repositories.mdwn b/doc/walkthrough/moving_file_content_between_repositories.mdwn
index 27dffe9138..3ffcc11750 100644
--- a/doc/walkthrough/moving_file_content_between_repositories.mdwn
+++ b/doc/walkthrough/moving_file_content_between_repositories.mdwn
@@ -9,5 +9,5 @@ makes it very easy.
 	move my_cool_big_file (to usbdrive...) ok
 	# git annex move video/hackity_hack_and_kaxxt.mov --from fileserver
 	move video/hackity_hack_and_kaxxt.mov (from fileserver...)
-	WORM-s86050597-m1274316523--hackity_hack_and_kax 100%   82MB 199.1KB/s   07:02
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e   100%   82MB 199.1KB/s   07:02
 	ok
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index e142b576c0..bd6c398710 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -1,8 +1,8 @@
-It's possible for data to accumulate in the annex that no files point to
-anymore. One way it can happen is if you `git rm` a file without 
-first calling `git annex drop`. And, when you modify an annexed file, the old
-content of the file remains in the annex. Another way is when migrating
-between key-value [[backends|backend]].
+It's possible for data to accumulate in the annex that no files in any
+branch point to anymore. One way it can happen is if you `git rm` a file
+without first calling `git annex drop`. And, when you modify an annexed
+file, the old content of the file remains in the annex. Another way is when
+migrating between key-value [[backends|backend]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and
@@ -12,8 +12,8 @@ eliminate it to save space.
 	unused . (checking for unused data...) 
 	  Some annexed data is no longer used by any files in the repository.
 	    NUMBER  KEY
-	    1       WORM-s3-m1289672605--file
-	    2       WORM-s14-m1289672605--file
+	    1       SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e
+	    2       SHA1-s14--f1358ec1873d57350e3dc62054dc232bc93c2bd1
 	  (To see where data was previously used, try: git log --stat -S'KEY')
 	  (To remove unwanted data: git-annex dropunused NUMBER)
 	ok
diff --git a/doc/walkthrough/using_ssh_remotes.mdwn b/doc/walkthrough/using_ssh_remotes.mdwn
index fbbbbe0701..60011a200b 100644
--- a/doc/walkthrough/using_ssh_remotes.mdwn
+++ b/doc/walkthrough/using_ssh_remotes.mdwn
@@ -13,7 +13,7 @@ Now you can get files and they will be transferred (using `rsync` via `ssh`):
 
 	# git annex get my_cool_big_file
 	get my_cool_big_file (getting UUID for origin...) (from origin...)
-	WORM-s2159-m1285650548--my_cool_big_file       100% 2159     2.1KB/s   00:00
+	SHA256-s86050597--6ae2688bc533437766a48aa19f2c06be14d1bab9c70b468af445d4f07b65f41e  100% 2159     2.1KB/s   00:00
 	ok
 
 When you drop files, git-annex will ssh over to the remote and make

From dabb6a9f265d8babd8885806bf5475a901bdbd90 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 4 Nov 2011 19:59:24 +0000
Subject: [PATCH 2444/8313] Added a comment: depends ...

---
 ...nt_1_3ef256230756be8a9679b107cdbfd018._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment

diff --git a/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment b/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment
new file mode 100644
index 0000000000..46ce0e8d53
--- /dev/null
+++ b/doc/forum/Recommended_number_of_repositories/comment_1_3ef256230756be8a9679b107cdbfd018._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="depends ..."
+ date="2011-11-04T19:59:24Z"
+ content="""
+It makes sense to have separate repositories when you have well-defined uses for them. 
+
+I have a separate repository just for music and podcasts, which I can put various places where I have no need of the overhead of a tree of other files.
+
+If you're using it for whatever arbitrary large files you accumulate, I find it's useful to have them in one repository. This way I can rearrange things as makes sense. It might make sense to have \"photos\" and \"isos\" as categories today, but next year you might prefer to move those under 2011/{photos,isos}. It would certainly make sense to have different repositories for home, work, etc.
+
+How to split repositories up for a home directory is a general problem that the [vcs-home](http://vcs-home.branchable.com)
+project has surely considered at one time or another.
+"""]]

From 502f86604fd90d6f52236b38612a39ec6e713be7 Mon Sep 17 00:00:00 2001
From: Valentin_Haenel 
Date: Fri, 4 Nov 2011 23:19:13 +0000
Subject: [PATCH 2445/8313] a recipe for setting up a bare remote

---
 doc/bare_repositories.mdwn | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index 3bc0a22cbd..bf56d81446 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -18,3 +18,27 @@ as non-bare repositories. Except for these caveats:
   branches that have been pushed to the bare repository. So use it with care..
 * Commands that need a work tree, like `git annex add` won't work in a bare
   repository, of course.
+
+***
+
+Here is a quick example of how to set this up, using `origin` as the remote name, and assuming `~/annex` contains an annex:
+
+On the server:
+
+    mkdir bare-annex
+    git init --bare
+    git annex init origin
+
+Now configure the remote and do the initial push:
+
+    cd ~/annex
+    git remote add origin example.com:bare-annex
+    git push origin master git-annex
+
+Now `git annex status` should show the configured bare remote. If it does not, you may have to pull from the remote first (older versions of `git-annex`)
+
+If you wish to configure git such that you can push/pull without arguments, set the upstream branch:
+
+    git branch master --set-upstream origin/master
+
+   

From 0bb798e3516d5b4e8cdcf059572c54641f7a643a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 Nov 2011 19:38:17 -0400
Subject: [PATCH 2446/8313] Pass -t to rsync to preserve timestamps.

---
 Utility/RsyncFile.hs | 2 ++
 debian/changelog     | 1 +
 2 files changed, 3 insertions(+)

diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index 056bd8d114..c5006a30fa 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -35,6 +35,8 @@ rsyncServerParams =
 	[ Param "--server"
 	-- preserve permissions
 	, Param "-p"
+	-- preserve timestamps
+	, Param "-t"
 	-- allow resuming of transfers of big files
 	, Param "--inplace"
 	-- other options rsync normally uses in server mode
diff --git a/debian/changelog b/debian/changelog
index e74a190ba5..07bfca7a78 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,6 +24,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
   * Record uuid when auto-initializing a remote so it shows in status.
   * Bugfix: Fixed git-annex init crash in a bare repository when there was
     already an existing git-annex branch.
+  * Pass -t to rsync to preserve timestamps.
 
  -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
 

From 526c20d06831706ced573ee0ea5f066976d8ed03 Mon Sep 17 00:00:00 2001
From: Valentin_Haenel 
Date: Sat, 5 Nov 2011 16:31:18 +0000
Subject: [PATCH 2447/8313] add bug report

---
 ...problems_with_urls_containing___126__.mdwn | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn

diff --git a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
new file mode 100644
index 0000000000..c6eb7bc73a
--- /dev/null
+++ b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
@@ -0,0 +1,44 @@
+I discovered a problem with `git annex map` and relative urls containing `~`.
+In this case i have a remote `noam` configured with the the following urls:
+
+    zsh» git remote show noam | head -3
+    * remote noam
+      Fetch URL: noam:bare-annex
+      Push  URL: noam:bare-annex
+
+If i try to run `git annex map` i get the following error:
+
+    zsh» git annex map
+    map /home/esc/annex ok
+    map noam (sshing...)
+    bash: line 0: cd: /~/bare-annex/: No such file or directory
+    Command ssh ["noam","cd '/~/bare-annex/' && git config --list"] failed; exit code 1
+    (sshing...)
+    ok
+
+      running: dot -Tx11 map.dot
+
+    ok
+
+If i run the failing command manually, i get:
+
+    zsh» ssh noam "cd ~/bare-annex && git config --list"
+    core.repositoryformatversion=0
+    core.filemode=true
+    core.bare=true
+    annex.uuid=f267f55c-0732-11e1-a93b-93119f9aaf54
+    annex.version=3
+
+Also i can change the remote url to an absolute one, in which case `git annex
+map` works too:
+
+    zsh» git remote set-url noam noam:/home/esc/bare-annex
+    zsh» git annex map
+    map /home/esc/annex ok
+    map noam (sshing...)
+    ok
+
+      running: dot -Tx11 map.dot
+
+    ok
+

From 0556dc812edf5b809da9836a4f31397af26e138a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 5 Nov 2011 15:55:19 -0400
Subject: [PATCH 2448/8313] releasing version 3.20111105

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 07bfca7a78..fdd3cb7f62 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111026) UNRELEASED; urgency=low
+git-annex (3.20111105) unstable; urgency=low
 
   * The default backend used when adding files to the annex is changed
     from WORM to SHA256.
@@ -26,7 +26,7 @@ git-annex (3.20111026) UNRELEASED; urgency=low
     already an existing git-annex branch.
   * Pass -t to rsync to preserve timestamps.
 
- -- Joey Hess   Thu, 27 Oct 2011 13:58:53 -0400
+ -- Joey Hess   Sat, 05 Nov 2011 15:47:52 -0400
 
 git-annex (3.20111025) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index c3706808c0..f464c46d20 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111025
+Version: 3.20111105
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From cd267dea159677c554aa65e04bd20bc14f5b12eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 5 Nov 2011 15:55:30 -0400
Subject: [PATCH 2449/8313] add news item for git-annex 3.20111105

---
 doc/news/version_3.20110906.mdwn |  4 ----
 doc/news/version_3.20111105.mdwn | 27 +++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)
 delete mode 100644 doc/news/version_3.20110906.mdwn
 create mode 100644 doc/news/version_3.20111105.mdwn

diff --git a/doc/news/version_3.20110906.mdwn b/doc/news/version_3.20110906.mdwn
deleted file mode 100644
index d91776ccaf..0000000000
--- a/doc/news/version_3.20110906.mdwn
+++ /dev/null
@@ -1,4 +0,0 @@
-git-annex 3.20110906 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Improve display of newlines around error and warning messages.
-   * Fix Makefile to work with cabal again."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111105.mdwn b/doc/news/version_3.20111105.mdwn
new file mode 100644
index 0000000000..663c2ef1fc
--- /dev/null
+++ b/doc/news/version_3.20111105.mdwn
@@ -0,0 +1,27 @@
+git-annex 3.20111105 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * The default backend used when adding files to the annex is changed
+     from WORM to SHA256.
+     To get old behavior, add a .gitattributes containing: * annex.backend=WORM
+   * Sped up some operations on remotes that are on the same host.
+   * copy --to: Fixed leak when copying many files to a remote on the same
+     host.
+   * uninit: Add guard against being run with the git-annex branch checked out.
+   * Fail if --from or --to is passed to commands that do not support them.
+   * drop --from is now supported to remove file content from a remote.
+   * status: Now always shows the current repository, even when it does not
+     appear in uuid.log.
+   * fsck: Now works in bare repositories. Checks location log information,
+     and file contents. Does not check that numcopies is satisfied, as
+     .gitattributes information about numcopies is not available in a bare
+     repository.
+   * unused, dropunused: Now work in bare repositories.
+   * Removed the setkey command, and added a reinject command with a more
+     useful interface.
+   * The fromkey command now takes the key as its first parameter. The --key
+     option is no longer used.
+   * Built without any filename containing .git being excluded. Closes: #[647215](http://bugs.debian.org/647215)
+   * Record uuid when auto-initializing a remote so it shows in status.
+   * Bugfix: Fixed git-annex init crash in a bare repository when there was
+     already an existing git-annex branch.
+   * Pass -t to rsync to preserve timestamps."""]]
\ No newline at end of file

From bf07e2c921f66965ca0be816975fc608a81c4276 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Nov 2011 13:53:11 -0400
Subject: [PATCH 2450/8313] typo

---
 doc/git-union-merge.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index 495612f36d..ed1778910b 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -15,7 +15,7 @@ The union merge will always succeed, but assumes that files can be merged
 simply by concacenating together lines from all the oldrefs, in any order.
 So, this is useful only for branches containing log-type data.
 
-That this does not touch the checked out working copy. It operates
+Note that this does not touch the checked out working copy. It operates
 entirely on git refs and branches.
 
 # EXAMPLE

From c99fb589097d96b4e10cd8b137f72881cdb93118 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Nov 2011 15:18:45 -0400
Subject: [PATCH 2451/8313] merge: Use fast-forward merges when possible.

Thanks Valentin Haenel for a test case showing how non-fast-forward merges
could result in an ongoing pull/merge/push cycle.

While the git-annex branch is fast-forwarded, git-annex's index file is still
updated using the union merge strategy as before. There's no other way to
update the index that would be any faster.

It is possible that a union merge and a fast-forward result in different file
contents: Files should have the same lines, but a union merge may change
their order. If this happens, the next commit made to the git-annex branch
will have some unnecessary changes to line orders, but the consistency
of data should be preserved.

Note that when the journal contains changes, a fast-forward is never attempted,
which is fine, because committing those changes would be vanishingly unlikely
to leave the git-annex branch at a commit that already exists in one of
the remotes.

The real difficulty is handling the case where multiple remotes have all
changed. git-annex does find the best (ie, newest) one and fast forwards
to it. If the remotes are diverged, no fast-forward is done at all. It would
be possible to pick one, fast forward to it, and make a merge commit to
the rest, I see no benefit to adding that complexity.

Determining the best of N changed remotes requires N*2+1 calls to git-log, but
these are fast git-log calls, and N is typically small. Also, typically
some or all of the remote refs will be the same, and git-log is not called to
compare those. In the real world I expect this will almost always add only
1 git-log call to the merge process. (Which already makes N anyway.)
---
 Annex/Branch.hs                               | 88 ++++++++++++++-----
 debian/changelog                              |  8 ++
 ...making_annex-merge_try_a_fast-forward.mdwn |  6 ++
 3 files changed, 79 insertions(+), 23 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 4c3192f531..0095b586b4 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -117,28 +117,30 @@ commit message = whenM journalDirty $ lockJournal $ do
 	g <- gitRepo
 	withIndex $ liftIO $ Git.commit g message fullname [fullname]
 
-{- Ensures that the branch is up-to-date; should be called before
- - data is read from it. Runs only once per git-annex run.
+{- Ensures that the branch is up-to-date; should be called before data is
+ - read from it. Runs only once per git-annex run.
  -
- - Before refs are merged into the index, it's
- - important to first stage the journal into the
- - index. Otherwise, any changes in the journal
- - would later get staged, and might overwrite
- - changes made during the merge.
+ - Before refs are merged into the index, it's important to first stage the
+ - journal into the index. Otherwise, any changes in the journal would
+ - later get staged, and might overwrite changes made during the merge.
  -
- - It would be cleaner to handle the merge by
- - updating the journal, not the index, with changes
- - from the branches.
+ - It would be cleaner to handle the merge by updating the journal, not the
+ - index, with changes from the branches.
+ -
+ - The index is always updated using a union merge, as that's the most
+ - efficient way to update it. However, if the branch can be
+ - fast-forwarded, that is then done, rather than adding an unnecessary
+ - commit to it.
  -}
 update :: Annex ()
 update = onceonly $ do
+	g <- gitRepo
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
-	c <- filterM changedbranch =<< siblingBranches
+	c <- filterM (changedBranch name . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
 	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
 		when dirty stageJournalFiles
-		g <- gitRepo
 		unless (null branches) $ do
 			showSideAction $ "merging " ++
 				(unwords $ map Git.refDescribe branches) ++
@@ -150,24 +152,64 @@ update = onceonly $ do
 			 - modify the branch.
 			 -}
 			liftIO $ Git.UnionMerge.merge_index g branches
-		liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
+		ff <- if dirty then return False else tryFastForwardTo refs
+		unless ff $
+			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
 		invalidateCache
 	where
-		changedbranch (_, branch) = do
-			g <- gitRepo
-			-- checking with log to see if there have been changes
-			-- is less expensive than always merging
-			diffs <- liftIO $ Git.pipeRead g [
-				Param "log",
-				Param (name ++ ".." ++ branch),
-				Params "--oneline -n1"
-				]
-			return $ not $ L.null diffs
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
 			disableUpdate
 			return r
 
+{- Checks if the second branch has any commits not present on the first
+ - branch. -}
+changedBranch :: String -> String -> Annex Bool
+changedBranch origbranch newbranch = do
+	g <- gitRepo
+	diffs <- liftIO $ Git.pipeRead g [
+		Param "log",
+		Param (origbranch ++ ".." ++ newbranch),
+		Params "--oneline -n1"
+		]
+	return $ not $ L.null diffs
+
+{- Given a set of refs that are all known to have commits not
+ - on the git-annex branch, tries to update the branch by a
+ - fast-forward.
+ -
+ - In order for that to be possible, one of the refs must contain
+ - every commit present in all the other refs, as well as in the
+ - git-annex branch.
+ -}
+tryFastForwardTo :: [String] -> Annex Bool
+tryFastForwardTo [] = return True
+tryFastForwardTo (first:rest) = do
+	-- First, check that the git-annex branch does not contain any
+	-- new commits that are in the first other branch. If it does,
+	-- cannot fast-forward.
+	diverged <- changedBranch first fullname
+	if diverged
+		then no_ff
+		else maybe no_ff do_ff =<< findbest first rest
+	where
+		no_ff = return False
+		do_ff branch = do
+			g <- gitRepo
+			liftIO $ Git.run g "update-ref" [Param fullname, Param branch]
+			return True
+		findbest c [] = return $ Just c
+		findbest c (r:rs)
+			| c == r = findbest c rs
+			| otherwise = do
+			better <- changedBranch c r
+			worse <- changedBranch r c
+			case (better, worse) of
+				(True, True) -> return Nothing -- divergent fail
+				(True, False) -> findbest r rs -- better
+				(False, True) -> findbest c rs -- worse
+				(False, False) -> findbest c rs -- same
+			
 {- Avoids updating the branch. A useful optimisation when the branch
  - is known to have not changed, or git-annex won't be relying on info
  - from it. -}
diff --git a/debian/changelog b/debian/changelog
index fdd3cb7f62..57be53c859 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+git-annex (3.20111106) UNRELEASED; urgency=low
+
+  * merge: Use fast-forward merges when possible.
+    Thanks Valentin Haenel for a test case showing how non-fast-forward
+    merges could result in an ongoing pull/merge/push cycle.
+
+ -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
+
 git-annex (3.20111105) unstable; urgency=low
 
   * The default backend used when adding files to the annex is changed
diff --git a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
index a2bd8c7479..41a5a2a581 100644
--- a/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
+++ b/doc/bugs/making_annex-merge_try_a_fast-forward.mdwn
@@ -27,3 +27,9 @@ But as sometimes annex-merge takes time, it would probably be worth it
 > 
 > Although, perhaps fast-forward merge would use slightly
 > less space. --[[Joey]]
+
+>> To avoid the ladder-merge between two repositories described at
+>> , seems a fast-forward should be detected and
+>> written to git, even if the index is still updated the current way.
+>> [[done]]
+>> --[[Joey]]

From f2299117152955d622d0ae0fcb396e56b1b7866d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 6 Nov 2011 15:33:15 -0400
Subject: [PATCH 2452/8313] optimization

The last commit added some git-log calls to a merge. This removes some,
by only merging branches that have unique refs.
---
 Annex/Branch.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0095b586b4..163c9ec60f 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -240,15 +240,16 @@ hasOrigin = refExists originname
 hasSomeBranch :: Annex Bool
 hasSomeBranch = not . null <$> siblingBranches
 
-{- List of all git-annex (refs, branches), including the main one and any
- - from remotes. -}
+{- List of git-annex (refs, branches), including the main one and any
+ - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(String, String)]
 siblingBranches = do
 	g <- gitRepo
 	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
-	return $ map (pair . words . L.unpack) (L.lines r)
+	return $ nubBy uref $ map (pair . words . L.unpack) (L.lines r)
 	where
 		pair l = (head l, last l)
+		uref (a, _) (b, _) = a == b
 
 {- Applies a function to modifiy the content of a file. -}
 change :: FilePath -> (String -> String) -> Annex ()

From aae0417d94c4ae81d28f2301bc1fac61d4f499a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 11:50:30 -0400
Subject: [PATCH 2453/8313] Don't try to read config from repos with
 annex-ignore set.

---
 Remote/Git.hs    | 9 +++++----
 debian/changelog | 1 +
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index b0138901d7..0cd64c9215 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -56,10 +56,11 @@ gen r u _ = do
 	 - the config of an URL remote is only read when there is no
 	 - cached UUID value. -}
 	let cheap = not $ Git.repoIsUrl r
-	r' <- case (cheap, u) of
-		(True, _) -> do
-			tryGitConfigRead r
-		(False, "") -> tryGitConfigRead r
+	notignored <- repoNotIgnored r
+	r' <- case (cheap, notignored, u) of
+		(_, False, _) -> return r
+		(True, _, _) -> tryGitConfigRead r
+		(False, _, "") -> tryGitConfigRead r
 		_ -> return r
 
 	u' <- getRepoUUID r'
diff --git a/debian/changelog b/debian/changelog
index 57be53c859..c09ca6578a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20111106) UNRELEASED; urgency=low
   * merge: Use fast-forward merges when possible.
     Thanks Valentin Haenel for a test case showing how non-fast-forward
     merges could result in an ongoing pull/merge/push cycle.
+  * Don't try to read config from repos with annex-ignore set.
 
  -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
 

From 41eecb4601896f7ece2151ff79775dd591e91b37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 12:47:41 -0400
Subject: [PATCH 2454/8313] Bugfix: In the past two releases, git-annex init
 has written the uuid.log in the wrong format, with the UUID and description
 flipped.

This is my own damn fault for not making UUID a real type, and then relying
on the type checker to ensure my refactoring was correct -- which it wasn't!

I should probably add code to clean up bogus entries in the uuid.log, but
right now I want to get the fix out there to prevent people experiencing
this bug.

I should also make UUID a real data type.
---
 Command/Init.hs  | 3 ++-
 debian/changelog | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index 69cdc8e8a4..e2a6eb7b03 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -30,5 +30,6 @@ start ws = do
 perform :: String -> CommandPerform
 perform description = do
 	initialize
-	getUUID >>= describeUUID description
+	u <- getUUID
+	describeUUID u description
 	next $ return True
diff --git a/debian/changelog b/debian/changelog
index c09ca6578a..57be7dd67b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20111106) UNRELEASED; urgency=low
     Thanks Valentin Haenel for a test case showing how non-fast-forward
     merges could result in an ongoing pull/merge/push cycle.
   * Don't try to read config from repos with annex-ignore set.
+  * Bugfix: In the past two releases, git-annex init has written the uuid.log
+    in the wrong format, with the UUID and description flipped.
 
  -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
 

From f8911cc69dc069b47da7c156270620ef6aa758b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:06:58 -0400
Subject: [PATCH 2455/8313] releasing version 3.20111107

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 57be7dd67b..0b4fe3486e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111106) UNRELEASED; urgency=low
+git-annex (3.20111107) unstable; urgency=low
 
   * merge: Use fast-forward merges when possible.
     Thanks Valentin Haenel for a test case showing how non-fast-forward
@@ -7,7 +7,7 @@ git-annex (3.20111106) UNRELEASED; urgency=low
   * Bugfix: In the past two releases, git-annex init has written the uuid.log
     in the wrong format, with the UUID and description flipped.
 
- -- Joey Hess   Sun, 06 Nov 2011 14:57:57 -0400
+ -- Joey Hess   Mon, 07 Nov 2011 12:47:44 -0400
 
 git-annex (3.20111105) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index f464c46d20..4b3a708a4e 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111105
+Version: 3.20111107
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 80787703c34512dec98a794424c63e6977737767 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:07:43 -0400
Subject: [PATCH 2456/8313] add news item for git-annex 3.20111107

---
 doc/news/version_3.20110915.mdwn | 10 ----------
 doc/news/version_3.20111107.mdwn |  8 ++++++++
 2 files changed, 8 insertions(+), 10 deletions(-)
 delete mode 100644 doc/news/version_3.20110915.mdwn
 create mode 100644 doc/news/version_3.20111107.mdwn

diff --git a/doc/news/version_3.20110915.mdwn b/doc/news/version_3.20110915.mdwn
deleted file mode 100644
index 77a97242d9..0000000000
--- a/doc/news/version_3.20110915.mdwn
+++ /dev/null
@@ -1,10 +0,0 @@
-git-annex 3.20110915 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * whereis: Show untrusted locations separately and do not include in
-     location count.
-   * Fix build without S3.
-   * addurl: Always use whole url as destination filename, rather than
-     only its file component.
-   * get, drop, copy: Added --auto option, which decides whether
-     to get/drop content as needed to work toward the configured numcopies.
-   * bugfix: drop and fsck did not honor --exclude"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111107.mdwn b/doc/news/version_3.20111107.mdwn
new file mode 100644
index 0000000000..17431bf219
--- /dev/null
+++ b/doc/news/version_3.20111107.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20111107 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * merge: Use fast-forward merges when possible.
+     Thanks Valentin Haenel for a test case showing how non-fast-forward
+     merges could result in an ongoing pull/merge/push cycle.
+   * Don't try to read config from repos with annex-ignore set.
+   * Bugfix: In the past two releases, git-annex init has written the uuid.log
+     in the wrong format, with the UUID and description flipped."""]]
\ No newline at end of file

From b7cd088433c171d878a69ef116b7a2efe1248b32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:08:47 -0400
Subject: [PATCH 2457/8313] add

---
 .../centralized_git_repository_tutorial.mdwn  | 129 ++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 doc/tips/centralized_git_repository_tutorial.mdwn

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
new file mode 100644
index 0000000000..920ae5d8f7
--- /dev/null
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -0,0 +1,129 @@
+The [[walkthrough]] builds up a decentralized git repository setup, but
+git-annex can also be used with a centralized bare repository, just like
+git can.
+
+## set up the repository, and make a checkout
+
+In this tutorial, I'll set up a centralized repository hosted on
+GitHub. I've created a repository for technical talk videos, which you can
+fork, or make your own repository on GitHub (or elsewhere) now.
+
+On your laptop, [[install]] git-annex, and clone the repository:
+
+	# git clone git@github.com:joeyh/techtalks.git
+	# cd techtalks
+
+Let's tell git-annex that GitHub doesn't support running git-annex-shell there.
+This means you can't store annexed file *contents* on GitHub; it would
+really be better to host the bare repository on your own server, which
+would not have this limitation. (If you want to do that, check out
+[[using_gitolite_with_git-annex]].)
+
+	# git config remote.origin.annex-ignore true
+
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+	# git annex init 'my laptop'
+	init my laptop ok
+
+## add files to the repository
+
+Add some files, obtained however.
+
+	# youtube-dl -t 'http://www.youtube.com/watch?v=b9FagOVqxmI'
+	# git annex add *.mp4
+	add Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 (checksum) ok
+	(Recording state in git...)
+	# git commit -m "added a video. I have not watched it yet but it sounds interesting"
+
+This file is available directly from the web; so git-annex can download it:
+
+	# git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
+	(downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...)
+	(checksum...) ok
+	(Recording state in git...)
+	# git commit -a -m 'added a screencast I made'
+
+Now push your changes back to the central repository. This first time,
+remember to push the git-annex branch, which is used to track the file
+contents.
+
+	# git push origin master git-annex
+	To git@github.com:joeyh/techtalks.git
+	 * [new branch]      master -> master
+	 * [new branch]      git-annex -> git-annex
+
+That push went fast, because it didn't upload large videos to GitHub.
+To check this, you can ask git-annex where the contents of the videos are:
+
+	# git annex whereis
+	whereis Haskell_Amuse_Bouche.mp4 (1 copy) 
+	  	767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
+	ok
+	whereis git-annex_coding_in_haskell.ogg (2 copies) 
+	  	00000000-0000-0000-0000-000000000001 -- web
+	   	767e8558-0955-11e1-be83-cbbeaab7fff8 -- here
+	ok
+
+## make more checkouts
+
+So far you have a central repository, and a checkout on a laptop.
+Let's make another checkout that's used as a backup. You can put it anywhere
+you like, just make it be somewhere your laptop can access. A few options:
+
+* Put it on a USB drive that you can plug into the laptop.
+* Put it on a desktop.
+* Put it on some server in the local network.
+* Put it on a remote VPS.
+* All of the above!
+
+I'll use the VPS option, but these instructions should work for
+any of the above.
+
+	# ssh server
+	server# sudo apt-get install git-annex
+
+Clone the central repository as before. (If the clone fails, you need
+to add your server's ssh public key to github -- see
+[this page](http://help.github.com/ssh-issues/).)
+
+	server# git clone git@github.com:joeyh/techtalks.git
+	server# cd techtalks
+	server# git config remote.origin.annex-ignore true
+	server# git annex init 'backup'
+	init backup (merging origin/git-annex into git-annex...) ok
+
+Notice that the server does not have the contents of any of the files yet.
+If you run `ls`, you'll see broken symlinks. We want to populate this
+backup with the file contents, by copying them from your laptop.
+
+Back on your laptop, you need to configure a git remote for the backup.
+Adjust the url as needed to point to wherever the backup is. (If it
+was on a local USB drive, you'd use the path to the repository.)
+
+	# git remote add backup ssh://server/~/techtalks
+
+Now git-annex on your laptop knows how to reach the backup repository,
+and can do things like copy files to it:
+
+	# git annex copy --to backup git-annex_coding_in_haskell.ogg
+	copy git-annex_coding_in_haskell.ogg (checking backup...)
+	12877824   2%  255.11kB/s    00:00
+	ok
+
+You can also `git annex move` files to it, to free up space on your laptop.
+And then you can `git annex get` files back to your laptop later on, as
+desired.
+
+## take it farther
+
+Of course you can create as many checkouts as you desire. If you have a
+desktop matchine too, you can make a checkout there, and use `git remote
+add` to also let your desktop access the backup repository. 
+
+You can add remotes for each direct connection between machines you find you
+need -- so make the laptop have the desktop as a remote, and the desktop
+have the laptop as a remote, and then on either machine git-annex can
+access files stored on the other.

From 146995c4e18223542992e7c575a670b59b9e2efd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:18:16 -0400
Subject: [PATCH 2458/8313] update

---
 doc/tips/centralized_git_repository_tutorial.mdwn | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 920ae5d8f7..4cc7519d8b 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -1,12 +1,13 @@
 The [[walkthrough]] builds up a decentralized git repository setup, but
 git-annex can also be used with a centralized bare repository, just like
-git can.
+git can. This tutorial shows how to set up a centralized repository hosted on
+GitHub.
 
 ## set up the repository, and make a checkout
 
-In this tutorial, I'll set up a centralized repository hosted on
-GitHub. I've created a repository for technical talk videos, which you can
-fork, or make your own repository on GitHub (or elsewhere) now.
+I've created a repository for technical talk videos, which you can
+[fork on Github](https://github.com/joeyh/techtalks).
+Or make your own repository on GitHub (or elsewhere) now.
 
 On your laptop, [[install]] git-annex, and clone the repository:
 
@@ -115,7 +116,10 @@ and can do things like copy files to it:
 
 You can also `git annex move` files to it, to free up space on your laptop.
 And then you can `git annex get` files back to your laptop later on, as
-desired.
+desired. After you using git-annex to move files around, remember to push,
+which will broadcast its updated location information.
+
+	# git push
 
 ## take it farther
 

From 95b9d726f87dee07c9eab4042e6d7caba612c765 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:26:37 -0400
Subject: [PATCH 2459/8313] update

---
 .../centralized_git_repository_tutorial.mdwn  | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 4cc7519d8b..03c4c9afd5 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -14,6 +14,12 @@ On your laptop, [[install]] git-annex, and clone the repository:
 	# git clone git@github.com:joeyh/techtalks.git
 	# cd techtalks
 
+Tell git-annex to use the repository, and describe where this clone is
+located:
+
+	# git annex init 'my laptop'
+	init my laptop ok
+
 Let's tell git-annex that GitHub doesn't support running git-annex-shell there.
 This means you can't store annexed file *contents* on GitHub; it would
 really be better to host the bare repository on your own server, which
@@ -22,12 +28,6 @@ would not have this limitation. (If you want to do that, check out
 
 	# git config remote.origin.annex-ignore true
 
-Tell git-annex to use the repository, and describe where this clone is
-located:
-
-	# git annex init 'my laptop'
-	init my laptop ok
-
 ## add files to the repository
 
 Add some files, obtained however.
@@ -78,7 +78,6 @@ you like, just make it be somewhere your laptop can access. A few options:
 * Put it on a desktop.
 * Put it on some server in the local network.
 * Put it on a remote VPS.
-* All of the above!
 
 I'll use the VPS option, but these instructions should work for
 any of the above.
@@ -101,8 +100,8 @@ If you run `ls`, you'll see broken symlinks. We want to populate this
 backup with the file contents, by copying them from your laptop.
 
 Back on your laptop, you need to configure a git remote for the backup.
-Adjust the url as needed to point to wherever the backup is. (If it
-was on a local USB drive, you'd use the path to the repository.)
+Adjust the ssh url as needed to point to wherever the backup is. (If it
+was on a local USB drive, you'd use the path to the repository instead.)
 
 	# git remote add backup ssh://server/~/techtalks
 
@@ -116,7 +115,9 @@ and can do things like copy files to it:
 
 You can also `git annex move` files to it, to free up space on your laptop.
 And then you can `git annex get` files back to your laptop later on, as
-desired. After you using git-annex to move files around, remember to push,
+desired.
+
+After you use git-annex to move files around, remember to push, 
 which will broadcast its updated location information.
 
 	# git push

From 97ba3a118de0e37b3f7dac8fd299a727e6859319 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 13:29:00 -0400
Subject: [PATCH 2460/8313] update

---
 doc/tips/centralized_git_repository_tutorial.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 03c4c9afd5..977a583cca 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -47,6 +47,12 @@ This file is available directly from the web; so git-annex can download it:
 	(Recording state in git...)
 	# git commit -a -m 'added a screencast I made'
 
+Feel free the rename the files, etc, using normal git commands:
+
+	# git mv Haskell_Amuse_Bouche-b9FagOVqxmI.mp4 Haskell_Amuse_Bouche.mp4
+	# git mv kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg git-annex_coding_in_haskell.ogg
+	# git commit -m 'better filenames'
+
 Now push your changes back to the central repository. This first time,
 remember to push the git-annex branch, which is used to track the file
 contents.

From b08f7c428b4bc9eabd95596d08594ddd1057a0bf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 14:00:23 -0400
Subject: [PATCH 2461/8313] better usage

---
 Command/Reinject.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 3de24f3fc3..1277edf906 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import qualified Command.Fsck
 
 def :: [Command]
-def = [command "reinject" (paramPair paramPath paramPath) seek
+def = [command "reinject" (paramPair "SRC" "DEST") seek
 	"sets content of annexed file"]
 
 seek :: [CommandSeek]

From 63a292324d20832b68c92f784828e55e644481cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 14:46:01 -0400
Subject: [PATCH 2462/8313] add a UUID type

Should have done this a long time ago.
---
 Annex/Ssh.hs             |  5 ++---
 Annex/UUID.hs            | 17 ++++++++++-------
 Command/ConfigList.hs    |  2 +-
 Command/Map.hs           |  8 ++++----
 Logs/Location.hs         | 11 +++++------
 Logs/Trust.hs            |  5 ++---
 Logs/UUIDBased.hs        | 14 +++++++-------
 Logs/Web.hs              |  2 +-
 Remote.hs                | 15 ++++++++-------
 Remote/Bup.hs            |  8 ++++----
 Remote/Git.hs            |  2 +-
 Remote/Helper/Special.hs |  2 +-
 Remote/S3real.hs         |  2 +-
 Types.hs                 |  2 +-
 Types/Remote.hs          |  7 ++++---
 Types/UUID.hs            | 14 ++++++++++++--
 git-annex-shell.hs       |  4 ++--
 test.hs                  |  2 +-
 18 files changed, 67 insertions(+), 55 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 851c7c06b6..f8cd5d9bc8 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -45,9 +45,8 @@ git_annex_shell r command params
 		sshcmd uuid = unwords $
 			shellcmd : (map shellEscape $ toCommand shellopts) ++
 			uuidcheck uuid
-		uuidcheck uuid
-			| null uuid = []
-			| otherwise = ["--uuid", uuid]
+		uuidcheck NoUUID = []
+		uuidcheck (UUID u) = ["--uuid", u]
 
 {- Uses a supplied function (such as boolSystem) to run a git-annex-shell
  - command on a remote.
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 39e296e5b7..90189bc475 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -30,7 +30,7 @@ configkey = "annex.uuid"
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID
-genUUID = pOpen ReadFromPipe command params hGetLine
+genUUID = pOpen ReadFromPipe command params $ liftM read . hGetLine
 	where
 		command = SysConfig.uuid
 		params = if command == "uuid"
@@ -50,20 +50,23 @@ getRepoUUID r = do
 	let c = cached g
 	let u = getUncachedUUID r
 	
-	if c /= u && u /= ""
+	if c /= u && u /= NoUUID
 		then do
 			updatecache g u
 			return u
 		else return c
 	where
-		cached g = Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ setConfig cachekey u
+		cached g = read $ Git.configGet g cachekey ""
+		updatecache g u = when (g /= r) $ storeUUID cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = Git.configGet r configkey ""
+getUncachedUUID r = read $ Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
-prepUUID = whenM (null <$> getUUID) $
-	setConfig configkey =<< liftIO genUUID
+prepUUID = whenM ((==) NoUUID <$> getUUID) $
+	storeUUID configkey =<< liftIO genUUID
+
+storeUUID :: String -> UUID -> Annex ()
+storeUUID configfield uuid = setConfig configfield (show uuid)
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index cbc6e801b0..fadcbb8435 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -21,5 +21,5 @@ seek = [withNothing start]
 start :: CommandStart
 start = do
 	u <- getUUID
-	liftIO $ putStrLn $ "annex.uuid=" ++ u
+	liftIO $ putStrLn $ "annex.uuid=" ++ show u
 	stop
diff --git a/Command/Map.hs b/Command/Map.hs
index 7e61d2e9e8..803324e999 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -62,7 +62,7 @@ drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 		others = map (unreachable . uuidnode) $
 			filter (`notElem` ruuids) (M.keys umap)
 		trusted = map (trustworthy . uuidnode) ts
-		uuidnode u = Dot.graphNode u $ M.findWithDefault "" u umap
+		uuidnode u = Dot.graphNode (show u) $ M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
@@ -76,7 +76,7 @@ basehostname r = head $ split "." $ hostname r
  - or the remote name if not. -}
 repoName :: M.Map UUID String -> Git.Repo -> String
 repoName umap r
-	| null repouuid = fallback
+	| repouuid == NoUUID = fallback
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
@@ -86,8 +86,8 @@ repoName umap r
 nodeId :: Git.Repo -> String
 nodeId r =
 	case getUncachedUUID r of
-		"" -> Git.repoLocation r
-		u -> u
+		NoUUID -> Git.repoLocation r
+		UUID u -> u
 
 {- A node representing a repo. -}
 node :: M.Map UUID String -> [Git.Repo] -> Git.Repo -> String
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 8855cf63b5..602c46f310 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -29,16 +29,15 @@ import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
-logChange repo key u s
-	| null u = error $
-		"unknown UUID for " ++ Git.repoDescribe repo ++ 
-		" (have you run git annex init there?)"
-	| otherwise = addLog (logFile key) =<< logNow s u
+logChange _ key (UUID u) s = addLog (logFile key) =<< logNow s u
+logChange repo _ NoUUID _ = error $
+	"unknown UUID for " ++ Git.repoDescribe repo ++ 
+	" (have you run git annex init there?)"
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations = currentLog . logFile
+keyLocations key = map read <$> (currentLog . logFile) key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 372d8b3609..53a1bca2c5 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -53,13 +53,12 @@ parseTrust s
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
-trustSet uuid level = do
-	when (null uuid) $
-		error "unknown UUID; cannot modify trust level"
+trustSet uuid@(UUID _) level = do
 	ts <- liftIO $ getPOSIXTime
 	Annex.Branch.change trustLog $
 		showLog show . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
+trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
 
 {- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
 trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 46fa80be0d..7184709fe2 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -50,9 +50,9 @@ showLog :: (a -> String) -> Log a -> String
 showLog shower = unlines . map showpair . M.toList
 	where
 		showpair (k, LogEntry (Date p) v) =
-			unwords [k, shower v, tskey ++ show p]
+			unwords [show k, shower v, tskey ++ show p]
 		showpair (k, LogEntry Unknown v) =
-			unwords [k, shower v]
+			unwords [show k, shower v]
 
 parseLog :: (String -> Maybe a) -> String -> Log a
 parseLog parser = M.fromListWith best . catMaybes . map pair . lines
@@ -61,7 +61,7 @@ parseLog parser = M.fromListWith best . catMaybes . map pair . lines
 			| null ws = Nothing
 			| otherwise = case parser $ unwords info of
 				Nothing -> Nothing
-				Just v -> Just (u, LogEntry c v)
+				Just v -> Just (read u, LogEntry c v)
 			where
 				ws = words line
 				u = head ws
@@ -103,8 +103,8 @@ prop_TimeStamp_sane = Unknown < Date 1
 prop_addLog_sane :: Bool
 prop_addLog_sane = newWins && newestWins
 	where
-		newWins = addLog "foo" (LogEntry (Date 1) "new") l == l2
-		newestWins = addLog "foo" (LogEntry (Date 1) "newest") l2 /= l2
+		newWins = addLog (UUID "foo") (LogEntry (Date 1) "new") l == l2
+		newestWins = addLog (UUID "foo") (LogEntry (Date 1) "newest") l2 /= l2
 
-		l = M.fromList [("foo", LogEntry (Date 0) "old")]
-		l2 = M.fromList [("foo", LogEntry (Date 1) "new")]
+		l = M.fromList [(UUID "foo", LogEntry (Date 0) "old")]
+		l2 = M.fromList [(UUID "foo", LogEntry (Date 1) "new")]
diff --git a/Logs/Web.hs b/Logs/Web.hs
index 605797079f..b52e347e5f 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -21,7 +21,7 @@ type URLString = String
 
 -- Dummy uuid for the whole web. Do not alter.
 webUUID :: UUID
-webUUID = "00000000-0000-0000-0000-000000000001"
+webUUID = UUID "00000000-0000-0000-0000-000000000001"
 
 {- The urls for a key are stored in remote/web/hash/key.log 
  - in the git-annex branch. -}
diff --git a/Remote.hs b/Remote.hs
index 6ce4fe0186..d4fbf36cf0 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -100,7 +100,7 @@ byName' n = do
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ head match
 	where
-		matching r = n == name r || n == uuid r
+		matching r = n == name r || read n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
  - and returns its UUID. Finds even remotes that are not configured in
@@ -115,12 +115,13 @@ nameToUUID n = do
 	where
 		byDescription = do
 			m <- uuidMap
-			case M.lookup n $ transform swap m of
+			case M.lookup wantuuid $ transform swap m of
 				Just u -> return $ Just u
-				Nothing -> return $ M.lookup n $ transform double m
+				Nothing -> return $ M.lookup wantuuid $ transform double m
 		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
-		double (a, _) = (a, a)
+		double (a, _) = (show a, a)
+		wantuuid = read n
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
  -
@@ -143,8 +144,8 @@ prettyPrintUUIDs desc uuids = do
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m here u
-			| not (null d) = u ++ " -- " ++ d
-			| otherwise = u
+			| not (null d) = show u ++ " -- " ++ d
+			| otherwise = show u
 			where
 				ishere = here == u
 				n = findlog m u
@@ -153,7 +154,7 @@ prettyPrintUUIDs desc uuids = do
 					| ishere = addname n "here"
 					| otherwise = n
 		jsonify m here u = toJSObject
-			[ ("uuid", toJSON u)
+			[ ("uuid", toJSON $ show u)
 			, ("description", toJSON $ findlog m u)
 			, ("here", toJSON $ here == u)
 			]
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 48014f1dad..b613225b8f 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -161,13 +161,13 @@ storeBupUUID u buprepo = do
 		then do
 			showAction "storing uuid"
 			onBupRemote r boolSystem "git"
-				[Params $ "config annex.uuid " ++ u]
+				[Params $ "config annex.uuid " ++ show u]
 					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
 			when (olduuid == "") $
-				Git.run r' "config" [Param "annex.uuid", Param u]
+				Git.run r' "config" [Param "annex.uuid", Param $ show u]
 
 onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a
 onBupRemote r a command params = do
@@ -192,8 +192,8 @@ getBupUUID r u
 	| otherwise = liftIO $ do
 		ret <- try $ Git.configRead r
 		case ret of
-			Right r' -> return (Git.configGet r' "annex.uuid" "", r')
-			Left _ -> return ("", r)
+			Right r' -> return (read $ Git.configGet r' "annex.uuid" "", r')
+			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
  - differences in path representation between git and bup. -}
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 0cd64c9215..4c76e8ce6d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -60,7 +60,7 @@ gen r u _ = do
 	r' <- case (cheap, notignored, u) of
 		(_, False, _) -> return r
 		(True, _, _) -> tryGitConfigRead r
-		(False, _, "") -> tryGitConfigRead r
+		(False, _, NoUUID) -> tryGitConfigRead r
 		_ -> return r
 
 	u' <- getRepoUUID r'
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 52f2dbf954..8a9a01a228 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -32,7 +32,7 @@ gitConfigSpecialRemote u c k v = do
 	g <- gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param u]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ show u]
 	where
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 89b0326379..1f5b2bd594 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -64,7 +64,7 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
 s3Setup u c = handlehost $ M.lookup "host" c
 	where
 		remotename = fromJust (M.lookup "name" c)
-		defbucket = remotename ++ "-" ++ u
+		defbucket = remotename ++ "-" ++ show u
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
diff --git a/Types.hs b/Types.hs
index 703edb5c86..fd77bfe575 100644
--- a/Types.hs
+++ b/Types.hs
@@ -9,7 +9,7 @@ module Types (
 	Annex,
 	Backend,
 	Key,
-	UUID
+	UUID(..)
 ) where
 
 import Annex
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 49f16bfdd7..0a4a0fa883 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -15,6 +15,7 @@ import Data.Ord
 
 import qualified Git
 import Types.Key
+import Types.UUID
 
 type RemoteConfig = M.Map String String
 
@@ -25,15 +26,15 @@ data RemoteType a = RemoteType {
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> String -> Maybe RemoteConfig -> a (Remote a),
+	generate :: Git.Repo -> UUID -> Maybe RemoteConfig -> a (Remote a),
 	-- initializes or changes a remote
-	setup :: String -> RemoteConfig -> a RemoteConfig
+	setup :: UUID -> RemoteConfig -> a RemoteConfig
 }
 
 {- An individual remote. -}
 data Remote a = Remote {
 	-- each Remote has a unique uuid
-	uuid :: String,
+	uuid :: UUID,
 	-- each Remote has a human visible name
 	name :: String,
 	-- Remotes have a use cost; higher is more expensive
diff --git a/Types/UUID.hs b/Types/UUID.hs
index eb3497fa94..f7232d0b97 100644
--- a/Types/UUID.hs
+++ b/Types/UUID.hs
@@ -7,5 +7,15 @@
 
 module Types.UUID where
 
--- might be nice to have a newtype, but lots of stuff treats uuids as strings
-type UUID = String
+-- A UUID is either an arbitrary opaque string, or UUID info may be missing.
+data UUID = NoUUID | UUID String
+	deriving (Eq, Ord)
+
+instance Show UUID where
+	show (UUID u) = u
+	show NoUUID = ""
+
+instance Read UUID where
+	readsPrec _ s
+		| null s = [(NoUUID, "")]
+		| otherwise = [(UUID s, "")]
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 10eeb454af..de3160953c 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -45,9 +45,9 @@ options = commonOptions ++
 	where
 		check expected = do
 			u <- getUUID
-			when (u /= expected) $ error $
+			when (u /= read expected) $ error $
 				"expected repository UUID " ++ expected
-					++ " but found UUID " ++ u
+					++ " but found UUID " ++ show u
 
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
diff --git a/test.hs b/test.hs
index d466d3ad32..d4c1366d09 100644
--- a/test.hs
+++ b/test.hs
@@ -614,7 +614,7 @@ checklocationlog f expected = do
 	case r of
 		Just (k, _) -> do
 			uuids <- annexeval $ Logs.Location.keyLocations k
-			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ thisuuid)
+			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ show thisuuid)
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"
 

From 64bc4e4751c5738d3e6c44db9452c46b26245910 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 16:13:06 -0400
Subject: [PATCH 2463/8313] refactor

---
 Command/Init.hs | 6 +-----
 Init.hs         | 9 +++++----
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index e2a6eb7b03..a6d72e4226 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -9,8 +9,6 @@ module Command.Init where
 
 import Common.Annex
 import Command
-import Annex.UUID
-import Logs.UUID
 import Init
 	
 def :: [Command]
@@ -29,7 +27,5 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	initialize
-	u <- getUUID
-	describeUUID u description
+	initialize (Just description)
 	next $ return True
diff --git a/Init.hs b/Init.hs
index 8c79002bcb..8cec98c5fc 100644
--- a/Init.hs
+++ b/Init.hs
@@ -19,13 +19,14 @@ import Logs.UUID
 import Annex.Version
 import Annex.UUID
 
-initialize :: Annex ()
-initialize = do
+initialize :: Maybe String -> Annex ()
+initialize mdescription = do
 	prepUUID
 	Annex.Branch.create
 	setVersion
 	gitPreCommitHookWrite
-	getUUID >>= recordUUID
+	u <- getUUID
+	maybe (recordUUID u) (describeUUID u) mdescription
 
 uninitialize :: Annex ()
 uninitialize = gitPreCommitHookUnWrite
@@ -40,7 +41,7 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 		needsinit = do
 			annexed <- Annex.Branch.hasSomeBranch
 			if annexed
-				then initialize
+				then initialize Nothing
 				else error "First run: git-annex init"
 
 {- set up a git pre-commit hook, if one is not already present -}

From 3c263cc9ea7fe5dc5b07db4a66281cda752fc6b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 16:34:12 -0400
Subject: [PATCH 2464/8313] fix

---
 Remote.hs | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index d4fbf36cf0..1591512ef1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -107,21 +107,19 @@ byName' n = do
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID -- special case for current repo
-nameToUUID n = do
-	res <- byName' n
-	case res of
-		Left e -> fromMaybe (error e) <$> byDescription
-		Right r -> return $ uuid r
+nameToUUID n = byName' n >>= go
 	where
-		byDescription = do
+		go (Right r) = return $ uuid r
+		go (Left e) = fromMaybe (error e) <$> bydescription
+		bydescription = do
 			m <- uuidMap
-			case M.lookup wantuuid $ transform swap m of
+			case M.lookup n $ transform swap m of
 				Just u -> return $ Just u
-				Nothing -> return $ M.lookup wantuuid $ transform double m
+				Nothing -> return $ byuuid m
+		byuuid m = M.lookup (read n) $ transform double m
 		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
-		double (a, _) = (show a, a)
-		wantuuid = read n
+		double (a, _) = (a, a)
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
  -

From 26d3c3b4977405127b52da0cddefe845b855f08f Mon Sep 17 00:00:00 2001
From: gernot 
Date: Mon, 7 Nov 2011 20:55:54 +0000
Subject: [PATCH 2465/8313]

---
 ...nnexed_file_to_a_.gitignored_location.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn

diff --git a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
new file mode 100644
index 0000000000..ca2cf75854
--- /dev/null
+++ b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
@@ -0,0 +1,19 @@
+I just noticed that if you move a git-annex symlink to a location ignored by git, it simply works.  Upon committing that change, however, part of git-annex's `fix` function apparently tries to `git-add` the symlink.  This fails because the new, ignored location requires a `git-add --force`.
+
+Considering that git proper doesn't fail or warn, I think git-annex shouldn't either.
+
+This is the error message:
+
+	$ git mv annexed-file ignored-dir/
+	$ git commit
+	fix ignored-dir/annexed-file ok
+	(Recording state in git...)
+	The following paths are ignored by one of your .gitignore files:
+	ignored-dir
+	Use -f if you really want to add them.
+	fatal: no files added
+	Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123
+
+	git-annex: user error (Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123)
+	failed
+	git-annex: 1 failed

From faa4935047b7083e1970d13f51fbaa6fe7d0fe3d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 18:10:31 -0400
Subject: [PATCH 2466/8313] Handle a case where an annexed file is moved into a
 gitignored directory, by having fix --force add its change.

---
 Command/Fix.hs                                             | 2 +-
 debian/changelog                                           | 7 +++++++
 ...when_moving_annexed_file_to_a_.gitignored_location.mdwn | 2 ++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/Command/Fix.hs b/Command/Fix.hs
index c46ddc7ee0..b46d6e8ecd 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -39,5 +39,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	Annex.Queue.add "add" [Param "--"] [file]
+	Annex.Queue.add "add" [Param "--force", Param "--"] [file]
 	return True
diff --git a/debian/changelog b/debian/changelog
index 0b4fe3486e..265a010c45 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111108) UNRELEASED; urgency=low
+
+  * Handle a case where an annexed file is moved into a gitignored directory,
+    by having fix --force add its change.
+
+ -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
+
 git-annex (3.20111107) unstable; urgency=low
 
   * merge: Use fast-forward merges when possible.
diff --git a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
index ca2cf75854..34d05c0b19 100644
--- a/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
+++ b/doc/bugs/Error_when_moving_annexed_file_to_a_.gitignored_location.mdwn
@@ -17,3 +17,5 @@ This is the error message:
 	git-annex: user error (Command xargs ["-0","git","--git-dir=/home/[...]/repo/.git","--work-tree=/home/[...]/repo","add","--"] failed; exit code 123)
 	failed
 	git-annex: 1 failed
+
+> Weird edge case.. ok, fixed. [[done]] --[[Joey]] 

From fdf988be6d2b3bb931a9eb3dcf3fbb83b1fb8c17 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 21:27:43 -0400
Subject: [PATCH 2467/8313] indent

---
 Utility/Misc.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index b3bf226120..4c4aa4c935 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -13,7 +13,7 @@ import Control.Monad
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
 hGetContentsStrict :: Handle -> IO String
-hGetContentsStrict  = hGetContents >=> \s -> length s `seq` return s
+hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s
 
 {- A version of readFile that is not lazy. -}
 readFileStrict :: FilePath -> IO String

From b11a63a860e8446cf3a4b35a5d8ef76329d5135c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 Nov 2011 23:21:22 -0400
Subject: [PATCH 2468/8313] clean up read/show abuse

Avoid ever using read to parse a non-haskell formatted input string.

show :: Key is arguably still show abuse, but displaying Keys as filenames
is just too useful to give up.
---
 Annex/UUID.hs            |  8 +++---
 Command/ConfigList.hs    |  2 +-
 Command/Map.hs           |  3 ++-
 Common/Annex.hs          |  2 ++
 Crypto.hs                |  8 ++++--
 Logs/Location.hs         |  2 +-
 Logs/Presence.hs         | 54 +++++++++++++---------------------------
 Logs/Trust.hs            | 14 ++++++++---
 Logs/UUIDBased.hs        | 17 ++++++-------
 Remote.hs                | 10 ++++----
 Remote/Bup.hs            | 10 +++++---
 Remote/Helper/Special.hs |  2 +-
 Remote/S3real.hs         |  2 +-
 Types/Crypto.hs          |  8 ------
 Types/TrustLevel.hs      | 10 --------
 Types/UUID.hs            | 15 ++++++-----
 Upgrade/V1.hs            |  2 +-
 git-annex-shell.hs       |  4 +--
 18 files changed, 75 insertions(+), 98 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 90189bc475..d3d674dcc6 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -30,7 +30,7 @@ configkey = "annex.uuid"
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
 genUUID :: IO UUID
-genUUID = pOpen ReadFromPipe command params $ liftM read . hGetLine
+genUUID = pOpen ReadFromPipe command params $ liftM toUUID . hGetLine
 	where
 		command = SysConfig.uuid
 		params = if command == "uuid"
@@ -56,12 +56,12 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached g = read $ Git.configGet g cachekey ""
+		cached g = toUUID $ Git.configGet g cachekey ""
 		updatecache g u = when (g /= r) $ storeUUID cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = read $ Git.configGet r configkey ""
+getUncachedUUID r = toUUID $ Git.configGet r configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
@@ -69,4 +69,4 @@ prepUUID = whenM ((==) NoUUID <$> getUUID) $
 	storeUUID configkey =<< liftIO genUUID
 
 storeUUID :: String -> UUID -> Annex ()
-storeUUID configfield uuid = setConfig configfield (show uuid)
+storeUUID configfield = setConfig configfield . fromUUID
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index fadcbb8435..dcf4d15093 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -21,5 +21,5 @@ seek = [withNothing start]
 start :: CommandStart
 start = do
 	u <- getUUID
-	liftIO $ putStrLn $ "annex.uuid=" ++ show u
+	liftIO $ putStrLn $ "annex.uuid=" ++ fromUUID u
 	stop
diff --git a/Command/Map.hs b/Command/Map.hs
index 803324e999..11808ed636 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -62,7 +62,8 @@ drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 		others = map (unreachable . uuidnode) $
 			filter (`notElem` ruuids) (M.keys umap)
 		trusted = map (trustworthy . uuidnode) ts
-		uuidnode u = Dot.graphNode (show u) $ M.findWithDefault "" u umap
+		uuidnode u = Dot.graphNode (fromUUID u) $
+			M.findWithDefault "" u umap
 
 hostname :: Git.Repo -> String
 hostname r
diff --git a/Common/Annex.hs b/Common/Annex.hs
index 43f1ea0af3..f802ec2533 100644
--- a/Common/Annex.hs
+++ b/Common/Annex.hs
@@ -1,6 +1,7 @@
 module Common.Annex (
 	module Common,
 	module Types,
+	module Types.UUID,
 	module Annex,
 	module Locations,
 	module Messages,
@@ -8,6 +9,7 @@ module Common.Annex (
 
 import Common
 import Types
+import Types.UUID (toUUID, fromUUID)
 import Annex (gitRepo)
 import Locations
 import Messages
diff --git a/Crypto.hs b/Crypto.hs
index ced7c144c4..b3acb30a6e 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -102,14 +102,18 @@ describeCipher (EncryptedCipher _ (KeyIds ks)) =
 {- Stores an EncryptedCipher in a remote's configuration. -}
 storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig
 storeCipher c (EncryptedCipher t ks) = 
-	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (show ks) c
+	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c
+	where
+		showkeys (KeyIds l) = join "," l
 
 {- Extracts an EncryptedCipher from a remote's configuration. -}
 extractCipher :: RemoteConfig -> Maybe EncryptedCipher
 extractCipher c = 
 	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of
-		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (read ks)
+		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks)
 		_ -> Nothing
+	where
+		readkeys = KeyIds . split ","
 
 {- Encrypts a Cipher to the specified KeyIds. -}
 encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 602c46f310..ff874a5964 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -37,7 +37,7 @@ logChange repo _ NoUUID _ = error $
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = map read <$> (currentLog . logFile) key
+keyLocations key = map toUUID <$> (currentLog . logFile) key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/Logs/Presence.hs b/Logs/Presence.hs
index 7211eba039..f5e4f1ea94 100644
--- a/Logs/Presence.hs
+++ b/Logs/Presence.hs
@@ -16,6 +16,7 @@ module Logs.Presence (
 	addLog,
 	readLog,
 	parseLog,
+	showLog,
 	logNow,
 	compactLog,
 	currentLog,
@@ -36,41 +37,9 @@ data LogLine = LogLine {
 	info :: String
 } deriving (Eq)
 
-data LogStatus = InfoPresent | InfoMissing | Undefined
+data LogStatus = InfoPresent | InfoMissing
 	deriving (Eq)
 
-instance Show LogStatus where
-	show InfoPresent = "1"
-	show InfoMissing = "0"
-	show Undefined = "undefined"
-
-instance Read LogStatus where
-	readsPrec _ "1" = [(InfoPresent, "")]
-	readsPrec _ "0" = [(InfoMissing, "")]
-	readsPrec _ _   = [(Undefined, "")]
-
-instance Show LogLine where
-	show (LogLine d s i) = unwords [show d, show s, i]
-
-instance Read LogLine where
-	-- This parser is robust in that even unparsable log lines are
-	-- read without an exception being thrown.
-	-- Such lines have a status of Undefined.
-	readsPrec _ string = 
-		if length w >= 3
-			then maybe bad good pdate
-			else bad
-		where
-			w = words string
-			s = read $ w !! 1
-			i = w !! 2
-			pdate :: Maybe UTCTime
-			pdate = parseTime defaultTimeLocale "%s%Qs" $ head w
-
-			good v = ret $ LogLine (utcTimeToPOSIXSeconds v) s i
-			bad = ret $ LogLine 0 Undefined ""
-			ret v = [(v, "")]
-
 addLog :: FilePath -> LogLine -> Annex ()
 addLog file line = Annex.Branch.change file $ \s -> 
 	showLog $ compactLog (line : parseLog s)
@@ -80,15 +49,26 @@ addLog file line = Annex.Branch.change file $ \s ->
 readLog :: FilePath -> Annex [LogLine]
 readLog file = parseLog <$> Annex.Branch.get file
 
+{- Parses a log file. Unparseable lines are ignored. -}
 parseLog :: String -> [LogLine]
-parseLog = filter parsable . map read . lines
+parseLog = mapMaybe (parseline . words) . lines
 	where
-		-- some lines may be unparseable, avoid them
-		parsable l = status l /= Undefined
+		parseline (a:b:c:_) = do
+			d <- parseTime defaultTimeLocale "%s%Qs" a
+			s <- parsestatus b
+			Just $ LogLine (utcTimeToPOSIXSeconds d) s c
+		parseline _ = Nothing
+		parsestatus "1" = Just InfoPresent
+		parsestatus "0" = Just InfoMissing
+		parsestatus _ = Nothing
 
 {- Generates a log file. -}
 showLog :: [LogLine] -> String
-showLog = unlines . map show
+showLog = unlines . map genline
+	where
+		genline (LogLine d s i) = unwords [show d, genstatus s, i]
+		genstatus InfoPresent = "1"
+		genstatus InfoMissing = "0"
 
 {- Generates a new LogLine with the current date. -}
 logNow :: LogStatus -> String -> Annex LogLine
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 53a1bca2c5..8c4507dcb8 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -45,18 +45,26 @@ trustMap = do
 
 parseTrust :: String -> Maybe TrustLevel
 parseTrust s
-	| length w > 0 = readMaybe $ head w
+	| length w > 0 = Just $ parse $ head w
 	-- back-compat; the trust.log used to only list trusted repos
-	| otherwise = Just Trusted
+	| otherwise = Just $ Trusted
 	where
 		w = words s
+		parse "1" = Trusted
+		parse "0" = UnTrusted
+		parse _ = SemiTrusted
+
+showTrust :: TrustLevel -> String
+showTrust SemiTrusted = "?"
+showTrust UnTrusted = "0"
+showTrust Trusted = "1"
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid@(UUID _) level = do
 	ts <- liftIO $ getPOSIXTime
 	Annex.Branch.change trustLog $
-		showLog show . changeLog ts uuid level . parseLog parseTrust
+		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
 trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
 
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 7184709fe2..9609d73213 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -50,28 +50,27 @@ showLog :: (a -> String) -> Log a -> String
 showLog shower = unlines . map showpair . M.toList
 	where
 		showpair (k, LogEntry (Date p) v) =
-			unwords [show k, shower v, tskey ++ show p]
+			unwords [fromUUID k, shower v, tskey ++ show p]
 		showpair (k, LogEntry Unknown v) =
-			unwords [show k, shower v]
+			unwords [fromUUID k, shower v]
 
 parseLog :: (String -> Maybe a) -> String -> Log a
-parseLog parser = M.fromListWith best . catMaybes . map pair . lines
+parseLog parser = M.fromListWith best . catMaybes . map parse . lines
 	where
-		pair line
+		parse line
 			| null ws = Nothing
-			| otherwise = case parser $ unwords info of
-				Nothing -> Nothing
-				Just v -> Just (read u, LogEntry c v)
+			| otherwise = parser (unwords info) >>= makepair
 			where
+				makepair v = Just (toUUID u, LogEntry ts v)
 				ws = words line
 				u = head ws
 				end = last ws
-				c
+				ts
 					| tskey `isPrefixOf` end =
 						pdate $ tail $ dropWhile (/= '=') end
 					| otherwise = Unknown
 				info
-					| c == Unknown = drop 1 ws
+					| ts == Unknown = drop 1 ws
 					| otherwise = drop 1 $ init ws
 				pdate s = case parseTime defaultTimeLocale "%s%Qs" s of
 					Nothing -> Unknown
diff --git a/Remote.hs b/Remote.hs
index 1591512ef1..6d55cee243 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -100,7 +100,7 @@ byName' n = do
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ head match
 	where
-		matching r = n == name r || read n == uuid r
+		matching r = n == name r || toUUID n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
  - and returns its UUID. Finds even remotes that are not configured in
@@ -116,7 +116,7 @@ nameToUUID n = byName' n >>= go
 			case M.lookup n $ transform swap m of
 				Just u -> return $ Just u
 				Nothing -> return $ byuuid m
-		byuuid m = M.lookup (read n) $ transform double m
+		byuuid m = M.lookup (toUUID n) $ transform double m
 		transform a = M.fromList . map a . M.toList
 		swap (a, b) = (b, a)
 		double (a, _) = (a, a)
@@ -142,8 +142,8 @@ prettyPrintUUIDs desc uuids = do
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m here u
-			| not (null d) = show u ++ " -- " ++ d
-			| otherwise = show u
+			| not (null d) = fromUUID u ++ " -- " ++ d
+			| otherwise = fromUUID u
 			where
 				ishere = here == u
 				n = findlog m u
@@ -152,7 +152,7 @@ prettyPrintUUIDs desc uuids = do
 					| ishere = addname n "here"
 					| otherwise = n
 		jsonify m here u = toJSObject
-			[ ("uuid", toJSON $ show u)
+			[ ("uuid", toJSON $ fromUUID u)
 			, ("description", toJSON $ findlog m u)
 			, ("here", toJSON $ here == u)
 			]
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index b613225b8f..3e621ce569 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -161,13 +161,15 @@ storeBupUUID u buprepo = do
 		then do
 			showAction "storing uuid"
 			onBupRemote r boolSystem "git"
-				[Params $ "config annex.uuid " ++ show u]
+				[Params $ "config annex.uuid " ++ v]
 					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
 			let olduuid = Git.configGet r' "annex.uuid" ""
-			when (olduuid == "") $
-				Git.run r' "config" [Param "annex.uuid", Param $ show u]
+			when (olduuid == "") $ Git.run r' "config"
+				[Param "annex.uuid", Param v]
+	where
+		v = fromUUID u
 
 onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a
 onBupRemote r a command params = do
@@ -192,7 +194,7 @@ getBupUUID r u
 	| otherwise = liftIO $ do
 		ret <- try $ Git.configRead r
 		case ret of
-			Right r' -> return (read $ Git.configGet r' "annex.uuid" "", r')
+			Right r' -> return (toUUID $ Git.configGet r' "annex.uuid" "", r')
 			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 8a9a01a228..38f24eb373 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -32,7 +32,7 @@ gitConfigSpecialRemote u c k v = do
 	g <- gitRepo
 	liftIO $ do
 		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ show u]
+		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ fromUUID u]
 	where
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 1f5b2bd594..1281c27861 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -64,7 +64,7 @@ s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
 s3Setup u c = handlehost $ M.lookup "host" c
 	where
 		remotename = fromJust (M.lookup "name" c)
-		defbucket = remotename ++ "-" ++ show u
+		defbucket = remotename ++ "-" ++ fromUUID u
 		defaults = M.fromList
 			[ ("datacenter", "US")
 			, ("storageclass", "STANDARD")
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index a39a016b8b..a9d3dddc59 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -7,17 +7,9 @@
 
 module Types.Crypto where
 
-import Data.String.Utils
-
 -- XXX ideally, this would be a locked memory region
 newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
 
 newtype KeyIds = KeyIds [String]
-
-instance Show KeyIds where
-	show (KeyIds ks) = join "," ks
-
-instance Read KeyIds where
-	readsPrec _ s = [(KeyIds (split "," s), "")]
diff --git a/Types/TrustLevel.hs b/Types/TrustLevel.hs
index 058ce4595c..ddb8e45e49 100644
--- a/Types/TrustLevel.hs
+++ b/Types/TrustLevel.hs
@@ -17,14 +17,4 @@ import Types.UUID
 data TrustLevel = SemiTrusted | UnTrusted | Trusted
 	deriving Eq
 
-instance Show TrustLevel where
-        show SemiTrusted = "?"
-        show UnTrusted = "0"
-        show Trusted = "1"
-
-instance Read TrustLevel where
-        readsPrec _ "1" = [(Trusted, "")]
-        readsPrec _ "0" = [(UnTrusted, "")]
-	readsPrec _ _ = [(SemiTrusted, "")]
-
 type TrustMap = M.Map UUID TrustLevel
diff --git a/Types/UUID.hs b/Types/UUID.hs
index f7232d0b97..767cd0dfe8 100644
--- a/Types/UUID.hs
+++ b/Types/UUID.hs
@@ -9,13 +9,12 @@ module Types.UUID where
 
 -- A UUID is either an arbitrary opaque string, or UUID info may be missing.
 data UUID = NoUUID | UUID String
-	deriving (Eq, Ord)
+	deriving (Eq, Ord, Show)
 
-instance Show UUID where
-	show (UUID u) = u
-	show NoUUID = ""
+fromUUID :: UUID -> String
+fromUUID (UUID u) = u
+fromUUID NoUUID = ""
 
-instance Read UUID where
-	readsPrec _ s
-		| null s = [(NoUUID, "")]
-		| otherwise = [(UUID s, "")]
+toUUID :: String -> UUID
+toUUID [] = NoUUID
+toUUID s = UUID s
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 331328e816..be9a977ad7 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -178,7 +178,7 @@ fileKey1 file = readKey1 $
 	replace "&a" "&" $ replace "&s" "%" $ replace "%" "/" file
 
 writeLog1 :: FilePath -> [LogLine] -> IO ()
-writeLog1 file ls = viaTmp writeFile file (unlines $ map show ls)
+writeLog1 file ls = viaTmp writeFile file (showLog ls)
 
 readLog1 :: FilePath -> IO [LogLine]
 readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index de3160953c..12cc65e4dd 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -45,9 +45,9 @@ options = commonOptions ++
 	where
 		check expected = do
 			u <- getUUID
-			when (u /= read expected) $ error $
+			when (u /= toUUID expected) $ error $
 				"expected repository UUID " ++ expected
-					++ " but found UUID " ++ show u
+					++ " but found UUID " ++ fromUUID u
 
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"

From 05b7608113a6b9abf92064884361f3e035ef3255 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 01:27:06 -0400
Subject: [PATCH 2469/8313] update

---
 doc/todo/git-annex_unused_eats_memory.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index fcb09a1af7..3e9942e98e 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -2,12 +2,14 @@
 (all keys with content present in the repository,
 with all keys used by files in the repository), and so
 uses more memory than git-annex typically needs; around
-60-80 mb when run in a repository with 80 thousand files.
+50 mb when run in a repository with 80 thousand files.
+
+(Used to be 80 mb, but implementation improved.)
 
 I would like to reduce this. One idea is to use a bloom filter. 
 For example, construct a bloom filter of all keys used by files in
 the repository. Then for each key with content present, check if it's
-in the bloom filter. Since there can be false negatives, this might
+in the bloom filter. Since there can be false positives, this might
 miss finding some unused keys. The probability/size of filter
 could be tunable.
 

From d35cd6ff262bc7013e5bb8fe63e8251c0e4ba8b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 12:16:02 -0400
Subject: [PATCH 2470/8313] wiki updates

---
 doc/distributed_version_control.mdwn                      | 8 ++++++++
 doc/meta.mdwn                                             | 5 +++++
 doc/special_remotes/web.mdwn                              | 2 +-
 ...he_web.mdwn => using_the_web_as_a_special_remote.mdwn} | 0
 4 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 doc/meta.mdwn
 rename doc/tips/{using_the_web.mdwn => using_the_web_as_a_special_remote.mdwn} (100%)

diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
index f9cdb7e99e..738b5adc2e 100644
--- a/doc/distributed_version_control.mdwn
+++ b/doc/distributed_version_control.mdwn
@@ -11,3 +11,11 @@ Each git-annex repository is responsible for storing some of the content,
 and can copy it to or from other repositories. [[Location_tracking]]
 information is committed to git, to let repositories inform other
 repositories what file contents they have available.
+
+--
+
+The [[tutorial]] walks you through creating a distributed set of git-annex
+repositories with no central repository.
+
+Prefer a central repository like GitHub? See the
+[[tips/centralized_git_repository_tutorial]].
diff --git a/doc/meta.mdwn b/doc/meta.mdwn
new file mode 100644
index 0000000000..f78eaf9813
--- /dev/null
+++ b/doc/meta.mdwn
@@ -0,0 +1,5 @@
+This wiki contains [[!pagecount pages="pages(*)"]]
+
+Broken links:
+
+[[!brokenlinks ]]
diff --git a/doc/special_remotes/web.mdwn b/doc/special_remotes/web.mdwn
index a969fb071d..cd20a93bb1 100644
--- a/doc/special_remotes/web.mdwn
+++ b/doc/special_remotes/web.mdwn
@@ -1,5 +1,5 @@
 git-annex can use the WWW as a special remote, downloading urls to files.
-See [[walkthrough/using_the_web]] for usage examples.
+See [[tips/using_the_web_as_a_special_remote]] for usage examples.
 
 ## notes
 
diff --git a/doc/tips/using_the_web.mdwn b/doc/tips/using_the_web_as_a_special_remote.mdwn
similarity index 100%
rename from doc/tips/using_the_web.mdwn
rename to doc/tips/using_the_web_as_a_special_remote.mdwn

From 67c9f84a1f80a0b8335a1e6edce2cd143d71b4c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 12:23:03 -0400
Subject: [PATCH 2471/8313] fix broken links

---
 doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn   | 2 +-
 doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn       | 2 +-
 doc/bugs/fat_support.mdwn                                   | 2 --
 doc/bugs/git_annex_initremote_walks_.git-annex.mdwn         | 2 +-
 doc/distributed_version_control.mdwn                        | 2 +-
 .../getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn   | 4 +++-
 doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn   | 6 +++---
 doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn         | 6 ------
 doc/future_proofing.mdwn                                    | 2 +-
 doc/meta.mdwn                                               | 2 +-
 doc/special_remotes/S3.mdwn                                 | 4 ++--
 doc/upgrades/SHA_size.mdwn                                  | 2 +-
 doc/walkthrough/unused_data.mdwn                            | 2 +-
 13 files changed, 16 insertions(+), 22 deletions(-)
 delete mode 100644 doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn

diff --git a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
index 47fc77fa4c..69fbc816f0 100644
--- a/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
+++ b/doc/bugs/copy_fast_confusing_with_broken_locationlog.mdwn
@@ -1,4 +1,4 @@
-Conversation moved from [[walkthrough/recover_data_from_lost+found]]
+Conversation moved from [[tips/recover_data_from_lost+found]]
 to a proper bug. --[[Joey]]
 
 (Unfortunatly that scrambled the comment creation times and thus order.)
diff --git a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn
index e88bf07f4d..c6ef13f844 100644
--- a/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn
+++ b/doc/bugs/dropping_files_with_a_URL_backend_fails.mdwn
@@ -1,4 +1,4 @@
-I was trying out the example with the walkthrough [[walkthrough/using_the_URL_backend]]. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with
+I was trying out the example with the walkthrough using_the_URL_backend. I tried dropping files that I had after doing an "git annex get ." which have the URL backend associated with the files it fails with
 
 
 
diff --git a/doc/bugs/fat_support.mdwn b/doc/bugs/fat_support.mdwn
index 60633c29bf..70ee3b369c 100644
--- a/doc/bugs/fat_support.mdwn
+++ b/doc/bugs/fat_support.mdwn
@@ -8,8 +8,6 @@ be VFAT formatted:
 - Use of ":" in filenames of object files, also not supported.
   Could easily be fixed by reorganizing the object directory.
 
-[[!tag wishlist]]
-
 [[Done]]; in annex.version 2 repos, colons are entirely avoided in
 filenames. So a bare git clone can be put on VFAT, and git-annex
 used to move stuff --to and --from it, for sneakernet.
diff --git a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
index 2457057c81..acd369bded 100644
--- a/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
+++ b/doc/bugs/git_annex_initremote_walks_.git-annex.mdwn
@@ -1,4 +1,4 @@
-a [[!taglink minor]]  issue: `git annex initremote` (in particular, adding
 a key as described in [[encryption]] -- `git annex initremote my_remote
 encryption=my_key`) seems to iterate over the `.git-annex/???/???/*.log` files
diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
index 738b5adc2e..e7c858c696 100644
--- a/doc/distributed_version_control.mdwn
+++ b/doc/distributed_version_control.mdwn
@@ -14,7 +14,7 @@ repositories what file contents they have available.
 
 --
 
-The [[tutorial]] walks you through creating a distributed set of git-annex
+The [[walkthrough]] shows how to create a distributed set of git-annex
 repositories with no central repository.
 
 Prefer a central repository like GitHub? See the
diff --git a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
index f5d5a66bfc..cc9091ae5b 100644
--- a/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
+++ b/doc/forum/getting_git_annex_to_do_a_force_copy_to_a_remote.mdwn
@@ -2,7 +2,9 @@ I'm not sure if this is my stupidity or if it's a bug, but
 
     git annex copy --force --to REMOTE . 
 
-just zip's through really quickly and doesn't actually force a copy to a remote location. This is just following up on the [[git-annex directory hashing problems on osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a 
+just zip's through really quickly and doesn't actually force a copy to a
+remote location. This is just following up on the
+[[bugs/git-annex_directory_hashing_problems_on_osx]]. I want to just do a force copy of all my data to my portable disk to really make sure that the data is really there. I would similarly would want to make sure I can force a 
 
     git annex copy --force --from REMOTE .
 
diff --git a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn
index 47e8b85622..7fd31efbcf 100644
--- a/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn
+++ b/doc/forum/wishlist:_special_remote_for_sftp_or_rsync.mdwn
@@ -1,11 +1,11 @@
-i think it would be useful to have a fourth kind of [[special remote]]s
+i think it would be useful to have a fourth kind of [[special_remotes]]
 that connects to a dumb storage using sftp or rsync. this can be emulated
 by using sshfs, but that means lots of round-trips through the system and
 is limited to platforms where sshfs is available.
 
 typical use cases are backups to storate shared between a group of people
-where each user only has limited access (sftp or rsync), when using [[bup]]
-is not an option.
+where each user only has limited access (sftp or rsync), when using
+[[special_remotes/bup]] is not an option.
 
 an alternative to implementing yet another special remote would be to have
 some kind of plugin system by which external programs can provide an
diff --git a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn b/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn
deleted file mode 100644
index 6616758730..0000000000
--- a/doc/forum/wishlist:_support_for_more_ssh_urls_.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex does not seem to support all kinds of urls that git does.
-
-> Please use [[bugs]] for bug reports. Moved [[there|bugs/wishlist:_support_for_more_ssh_urls_]].
-> --[[Joey]] 
-
->> And now all fixed! --[[Joey]] 
diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn
index 3937e92656..a7bcce37c9 100644
--- a/doc/future_proofing.mdwn
+++ b/doc/future_proofing.mdwn
@@ -34,4 +34,4 @@ problem:
   metadata. So if a filesystem is badly corrupted and all your annexed
   files end up in `lost+found`, they can easily be lifted back out into
   another clone of the repository. Even if the filenames are lost,
-  it's possible to [[walkthrough/recover_data_from_lost+found]].
+  it's possible to [[tips/recover_data_from_lost+found]].
diff --git a/doc/meta.mdwn b/doc/meta.mdwn
index f78eaf9813..30de003d20 100644
--- a/doc/meta.mdwn
+++ b/doc/meta.mdwn
@@ -1,4 +1,4 @@
-This wiki contains [[!pagecount pages="pages(*)"]]
+This wiki contains [[!pagecount pages="*"]]
 
 Broken links:
 
diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 047798e238..d4d3d02388 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -1,8 +1,8 @@
 This special remote type stores file contents in a bucket in Amazon S3
 or a similar service.
 
-See [[walkthrough/using_Amazon_S3]] and
-[[walkthrough/Internet_Archive_via_S3]] for usage examples.
+See [[tips/using_Amazon_S3]] and
+[[tips/Internet_Archive_via_S3]] for usage examples.
 
 ## configuration
 
diff --git a/doc/upgrades/SHA_size.mdwn b/doc/upgrades/SHA_size.mdwn
index 319b91108a..97603ba913 100644
--- a/doc/upgrades/SHA_size.mdwn
+++ b/doc/upgrades/SHA_size.mdwn
@@ -15,6 +15,6 @@ to repository version 2, in each repository run:
 	git annex migrate
 	git commit -m 'migrated keys for v2'
 
-The usual caveats about [[walkthrough/migrating_data_to_a_new_backend]]
+The usual caveats about [[tips/migrating_data_to_a_new_backend]]
 apply; you will end up with unused keys that you can later clean up with
 `git annex unused`.
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index bd6c398710..518550ac02 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -2,7 +2,7 @@ It's possible for data to accumulate in the annex that no files in any
 branch point to anymore. One way it can happen is if you `git rm` a file
 without first calling `git annex drop`. And, when you modify an annexed
 file, the old content of the file remains in the annex. Another way is when
-migrating between key-value [[backends|backend]].
+migrating between key-value [[backends]].
 
 This might be historical data you want to preserve, so git-annex defaults to
 preserving it. So from time to time, you may want to check for such data and

From 2ff8915365099501382183af9855e739fc234861 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 12:24:56 -0400
Subject: [PATCH 2472/8313] fix

---
 .../git_annex_migrate_leaves_old_backend_versions_around.mdwn   | 2 +-
 doc/meta.mdwn                                                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn
index 7f586b5ff0..263338d64f 100644
--- a/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn
+++ b/doc/bugs/git_annex_migrate_leaves_old_backend_versions_around.mdwn
@@ -2,7 +2,7 @@
 would be great if these were purged automatically somehow. 
 
 > Yes, this is an issue mentioned in the
-> [[walkthrough|walkthrough/migrating_data_to_a_new_backend]].
+> [[tips/migrating_data_to_a_new_backend]].
 > 
 > Since multiple files can point to the same content, it could be that
 > only one file has been migrated, and the content is still used. So
diff --git a/doc/meta.mdwn b/doc/meta.mdwn
index 30de003d20..5ee36f8c0b 100644
--- a/doc/meta.mdwn
+++ b/doc/meta.mdwn
@@ -1,4 +1,4 @@
-This wiki contains [[!pagecount pages="*"]]
+This wiki contains [[!pagecount pages="*"]] pages.
 
 Broken links:
 

From bf460a0a98d7e4c7f4eac525fcf300629db582b6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 8 Nov 2011 15:34:10 -0400
Subject: [PATCH 2473/8313] reorder repo parameters last

Many functions took the repo as their first parameter. Changing it
consistently to be the last parameter allows doing some useful things with
currying, that reduce boilerplate.

In particular, g <- gitRepo is almost never needed now, instead
use inRepo to run an IO action in the repo, and fromRepo to get
a value from the repo.

This also provides more opportunities to use monadic and applicative
combinators.
---
 Annex.hs                 |  16 ++++++-
 Annex/Branch.hs          |  60 ++++++++++-------------
 Annex/CatFile.hs         |   3 +-
 Annex/Content.hs         |  32 +++++--------
 Annex/Queue.hs           |   3 +-
 Annex/UUID.hs            |  14 +++---
 Annex/Version.hs         |  10 ++--
 Backend.hs               |  19 +++-----
 Backend/SHA.hs           |   3 +-
 Command.hs               |   2 +-
 Command/Add.hs           |   4 +-
 Command/AddUrl.hs        |   3 +-
 Command/DropUnused.hs    |   8 ++--
 Command/Fsck.hs          |  20 ++++----
 Command/Map.hs           |   3 +-
 Command/Migrate.hs       |   7 ++-
 Command/SendKey.hs       |   3 +-
 Command/Unannex.hs       |  16 +++----
 Command/Uninit.hs        |  17 +++----
 Command/Unlock.hs        |   5 +-
 Command/Unused.hs        |  20 ++++----
 Common/Annex.hs          |   2 +-
 Config.hs                |  16 +++----
 Git.hs                   | 100 ++++++++++++++++++++-------------------
 Git/CatFile.hs           |   2 +-
 Git/LsFiles.hs           |  43 +++++++++--------
 Git/LsTree.hs            |   6 +--
 Git/Queue.hs             |   8 ++--
 Git/UnionMerge.hs        |  54 +++++++++++----------
 GitAnnex.hs              |   5 +-
 Init.hs                  |   6 +--
 Locations.hs             |  12 ++---
 Remote.hs                |  14 +++---
 Remote/Bup.hs            |  17 ++++---
 Remote/Directory.hs      |   6 +--
 Remote/Git.hs            |  19 +++-----
 Remote/Helper/Special.hs |  10 ++--
 Remote/Hook.hs           |   9 ++--
 Remote/Rsync.hs          |  13 ++---
 Remote/S3real.hs         |   7 ++-
 Remote/Web.hs            |   2 +-
 Seek.hs                  |  34 +++++++------
 Upgrade/V0.hs            |   3 +-
 Upgrade/V1.hs            |  28 +++++------
 Upgrade/V2.hs            |  40 ++++++++--------
 git-union-merge.hs       |   4 +-
 46 files changed, 338 insertions(+), 390 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 2c6402ac35..501e54c662 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,7 +17,9 @@ module Annex (
 	eval,
 	getState,
 	changeState,
-	gitRepo
+	gitRepo,
+	inRepo,
+	fromRepo,
 ) where
 
 import Control.Monad.IO.Control
@@ -114,6 +116,16 @@ getState = gets
 changeState :: (AnnexState -> AnnexState) -> Annex ()
 changeState = modify
 
-{- Returns the git repository being acted on -}
+{- Returns the annex's git repository. -}
 gitRepo :: Annex Git.Repo
 gitRepo = getState repo
+
+{- Runs an IO action in the annex's git repository. -}
+inRepo :: (Git.Repo -> IO a) -> Annex a
+inRepo a = do
+	g <- gitRepo
+	liftIO $ a g
+
+{- Extracts a value from the annex's git repisitory. -}
+fromRepo :: (Git.Repo -> a) -> Annex a
+fromRepo a = a <$> gitRepo
diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 163c9ec60f..189289ad39 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -56,21 +56,19 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.ls_tree g fullname >>= Git.UnionMerge.update_index g
+genIndex g = Git.UnionMerge.ls_tree fullname g >>= Git.UnionMerge.update_index g
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	g <- gitRepo
-	let f = index g
-
+	f <- fromRepo $ index
 	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
-			unless bootstrapping $ liftIO $ genIndex g
+			unless bootstrapping $ inRepo genIndex
 		a
 
 withIndexUpdate :: Annex a -> Annex a
@@ -103,19 +101,17 @@ getCache file = getState >>= go
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = unlessM hasBranch $ do
-	g <- gitRepo
 	e <- hasOrigin
 	if e
-		then liftIO $ Git.run g "branch" [Param name, Param originname]
+		then inRepo $ Git.run "branch" [Param name, Param originname]
 		else withIndex' True $
-			liftIO $ Git.commit g "branch created" fullname []
+			inRepo $ Git.commit "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
 	stageJournalFiles
-	g <- gitRepo
-	withIndex $ liftIO $ Git.commit g message fullname [fullname]
+	withIndex $ inRepo $ Git.commit message fullname [fullname]
 
 {- Ensures that the branch is up-to-date; should be called before data is
  - read from it. Runs only once per git-annex run.
@@ -134,7 +130,6 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
-	g <- gitRepo
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
@@ -151,10 +146,10 @@ update = onceonly $ do
 			 - documentation advises users not to directly
 			 - modify the branch.
 			 -}
-			liftIO $ Git.UnionMerge.merge_index g branches
+			inRepo $ \g -> Git.UnionMerge.merge_index g branches
 		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $
-			liftIO $ Git.commit g "update" fullname (nub $ fullname:refs)
+		unless ff $ inRepo $
+			Git.commit "update" fullname (nub $ fullname:refs)
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
@@ -165,14 +160,13 @@ update = onceonly $ do
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
 changedBranch :: String -> String -> Annex Bool
-changedBranch origbranch newbranch = do
-	g <- gitRepo
-	diffs <- liftIO $ Git.pipeRead g [
-		Param "log",
-		Param (origbranch ++ ".." ++ newbranch),
-		Params "--oneline -n1"
-		]
-	return $ not $ L.null diffs
+changedBranch origbranch newbranch = not . L.null <$> diffs
+	where
+		diffs = inRepo $ Git.pipeRead
+			[ Param "log"
+			, Param (origbranch ++ ".." ++ newbranch)
+			, Params "--oneline -n1"
+			]
 
 {- Given a set of refs that are all known to have commits not
  - on the git-annex branch, tries to update the branch by a
@@ -195,8 +189,7 @@ tryFastForwardTo (first:rest) = do
 	where
 		no_ff = return False
 		do_ff branch = do
-			g <- gitRepo
-			liftIO $ Git.run g "update-ref" [Param fullname, Param branch]
+			inRepo $ Git.run "update-ref" [Param fullname, Param branch]
 			return True
 		findbest c [] = return $ Just c
 		findbest c (r:rs)
@@ -223,10 +216,8 @@ disableUpdate = Annex.changeState setupdated
 
 {- Checks if a git ref exists. -}
 refExists :: GitRef -> Annex Bool
-refExists ref = do
-	g <- gitRepo
-	liftIO $ Git.runBool g "show-ref"
-		[Param "--verify", Param "-q", Param ref]
+refExists ref = inRepo $ Git.runBool "show-ref"
+	[Param "--verify", Param "-q", Param ref]
 
 {- Does the main git-annex branch exist? -}
 hasBranch :: Annex Bool
@@ -244,8 +235,7 @@ hasSomeBranch = not . null <$> siblingBranches
  - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(String, String)]
 siblingBranches = do
-	g <- gitRepo
-	r <- liftIO $ Git.pipeRead g [Param "show-ref", Param name]
+	r <- inRepo $ Git.pipeRead [Param "show-ref", Param name]
 	return $ nubBy uref $ map (pair . words . L.unpack) (L.lines r)
 	where
 		pair l = (head l, last l)
@@ -280,8 +270,7 @@ get file = fromcache =<< getCache file
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
-	g <- gitRepo
-	bfiles <- liftIO $ Git.pipeNullSplit g
+	bfiles <- inRepo $ Git.pipeNullSplit
 		[Params "ls-tree --name-only -r -z", Param fullname]
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
@@ -349,8 +338,8 @@ stageJournalFiles = do
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
-		git_hash_object g = Git.gitCommandLine g
-			[Param "hash-object", Param "-w", Param "--stdin-paths"]
+		git_hash_object g = Git.gitCommandLine
+			[Param "hash-object", Param "-w", Param "--stdin-paths"] g
 
 
 {- Checks if there are changes in the journal. -}
@@ -379,8 +368,7 @@ fileJournal = replace "//" "_" . replace "_" "/"
  - contention with other git-annex processes. -}
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
-	g <- gitRepo
-	let file = gitAnnexJournalLock g
+	file <- fromRepo $ gitAnnexJournalLock
 	bracketIO (lock file) unlock a
 	where
 		lock file = do
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 2707ed3ea3..a043e1ae3d 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -17,8 +17,7 @@ catFile :: String -> FilePath -> Annex String
 catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
-			g <- gitRepo
-			h <- liftIO $ Git.CatFile.catFileStart g
+			h <- inRepo $ Git.CatFile.catFileStart
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
 			go h
 		go h = liftIO $ Git.CatFile.catFile h branch file
diff --git a/Annex/Content.hs b/Annex/Content.hs
index aafdf6f2e6..fc2c2d0921 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -37,18 +37,18 @@ import Config
 {- Checks if a given key is currently present in the gitAnnexLocation. -}
 inAnnex :: Key -> Annex Bool
 inAnnex key = do
-	g <- gitRepo
-	when (Git.repoIsUrl g) $ error "inAnnex cannot check remote repo"
-	liftIO $ doesFileExist $ gitAnnexLocation g key
+	whenM (fromRepo Git.repoIsUrl) $
+		error "inAnnex cannot check remote repo"
+	inRepo $ doesFileExist . gitAnnexLocation key
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
-	g <- gitRepo
 	cwd <- liftIO getCurrentDirectory
 	let absfile = fromMaybe whoops $ absNormPath cwd file
+	top <- fromRepo Git.workTree
 	return $ relPathDirToFile (parentDir absfile) 
-			(Git.workTree g)  ".git"  annexLocation key
+			top  ".git"  annexLocation key
 	where
 		whoops = error $ "unable to normalize " ++ file
 
@@ -65,8 +65,7 @@ logStatus key status = do
  - the annex as a key's content. -}
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmp key action = do
-	g <- gitRepo
-	let tmp = gitAnnexTmpLocation g key
+	tmp <- fromRepo $ gitAnnexTmpLocation key
 
 	-- Check that there is enough free disk space.
 	-- When the temp file already exists, count the space
@@ -84,8 +83,7 @@ getViaTmp key action = do
 
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
-	g <- gitRepo
-	let tmp = gitAnnexTmpLocation g key
+	tmp <- fromRepo $ gitAnnexTmpLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	return tmp
 
@@ -162,8 +160,7 @@ checkDiskSpace' adjustment key = do
  -}
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
-	g <- gitRepo
-	let dest = gitAnnexLocation g key
+	dest <- fromRepo $ gitAnnexLocation key
 	let dir = parentDir dest
 	e <- liftIO $ doesFileExist dest
 	if e
@@ -177,8 +174,7 @@ moveAnnex key src = do
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
-	g <- gitRepo
-	let file = gitAnnexLocation g key
+	file <- fromRepo $gitAnnexLocation key
 	let dir = parentDir file
 	a (dir, file)
 
@@ -201,9 +197,9 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
  - returns the file it was moved to. -}
 moveBad :: Key -> Annex FilePath
 moveBad key = do
-	g <- gitRepo
-	let src = gitAnnexLocation g key
-	let dest = gitAnnexBadDir g  takeFileName src
+	src <- fromRepo $ gitAnnexLocation key
+	bad <- fromRepo $ gitAnnexBadDir
+	let dest = bad  takeFileName src
 	liftIO $ do
 		createDirectoryIfMissing True (parentDir dest)
 		allowWrite (parentDir src)
@@ -214,9 +210,7 @@ moveBad key = do
 
 {- List of keys whose content exists in .git/annex/objects/ -}
 getKeysPresent :: Annex [Key]
-getKeysPresent = do
-	g <- gitRepo
-	getKeysPresent' $ gitAnnexObjectDir g
+getKeysPresent = getKeysPresent' =<< fromRepo gitAnnexObjectDir
 getKeysPresent' :: FilePath -> Annex [Key]
 getKeysPresent' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 4c1182750a..f611cf02eb 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -34,8 +34,7 @@ flush silent = do
 	unless (0 == Git.Queue.size q) $ do
 		unless silent $
 			showSideAction "Recording state in git"
-		g <- gitRepo
-		q' <- liftIO $ Git.Queue.flush g q
+		q' <- inRepo $ Git.Queue.flush q
 		store q'
 
 store :: Git.Queue.Queue -> Annex ()
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index d3d674dcc6..6fc04c0f09 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -45,23 +45,23 @@ getUUID = getRepoUUID =<< gitRepo
 {- Looks up a repo's UUID. May return "" if none is known. -}
 getRepoUUID :: Git.Repo -> Annex UUID
 getRepoUUID r = do
-	g <- gitRepo
-
-	let c = cached g
+	c <- fromRepo cached
 	let u = getUncachedUUID r
 	
 	if c /= u && u /= NoUUID
 		then do
-			updatecache g u
+			updatecache u
 			return u
 		else return c
 	where
-		cached g = toUUID $ Git.configGet g cachekey ""
-		updatecache g u = when (g /= r) $ storeUUID cachekey u
+		cached g = toUUID $ Git.configGet cachekey "" g
+		updatecache u = do
+			g <- gitRepo
+			when (g /= r) $ storeUUID cachekey u
 		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID r = toUUID $ Git.configGet r configkey ""
+getUncachedUUID = toUUID . Git.configGet configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
diff --git a/Annex/Version.hs b/Annex/Version.hs
index 935f777abf..9e694faf14 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -26,12 +26,10 @@ versionField :: String
 versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
-getVersion = do
-	g <- gitRepo
-	let v = Git.configGet g versionField ""
-	if not $ null v
-		then return $ Just v
-		else return Nothing
+getVersion = handle <$> fromRepo (Git.configGet versionField "")
+	where
+		handle [] = Nothing
+		handle v = Just v
 
 setVersion :: Annex ()
 setVersion = setConfig versionField defaultVersion
diff --git a/Backend.hs b/Backend.hs
index 9a40e54598..f7990c22c1 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -47,10 +47,7 @@ orderedList = do
 			l' <- (lookupBackendName name :) <$> standard
 			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		standard = do
-			g <- gitRepo
-			return $ parseBackendList $
-				Git.configGet g "annex.backends" ""
+		standard = fromRepo $ parseBackendList . Git.configGet "annex.backends" ""
 		parseBackendList [] = list
 		parseBackendList s = map lookupBackendName $ words s
 
@@ -96,16 +93,14 @@ type BackendFile = (Maybe (Backend Annex), FilePath)
  - That can be configured on a per-file basis in the gitattributes file.
  -}
 chooseBackends :: [FilePath] -> Annex [BackendFile]
-chooseBackends fs = do
-	g <- gitRepo
-	forced <- Annex.getState Annex.forcebackend
-	if isJust forced
-		then do
+chooseBackends fs = Annex.getState Annex.forcebackend >>= go
+	where
+		go Nothing = do
+			pairs <- inRepo $ Git.checkAttr "annex.backend" fs
+			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
+		go (Just _) = do
 			l <- orderedList
 			return $ map (\f -> (Just $ head l, f)) fs
-		else do
-			pairs <- liftIO $ Git.checkAttr g "annex.backend" fs
-			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index d449821172..a3846a410b 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -98,9 +98,8 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 {- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
-	g <- gitRepo
 	fast <- Annex.getState Annex.fast
-	let file = gitAnnexLocation g key
+	file <- fromRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
diff --git a/Command.hs b/Command.hs
index c11b906103..c436c5b628 100644
--- a/Command.hs
+++ b/Command.hs
@@ -78,7 +78,7 @@ notBareRepo a = do
 	a
 
 isBareRepo :: Annex Bool
-isBareRepo = Git.repoIsLocalBare <$> gitRepo
+isBareRepo = fromRepo Git.repoIsLocalBare
 
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
diff --git a/Command/Add.hs b/Command/Add.hs
index a633db7b36..ab104b53cc 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -60,8 +60,8 @@ undo file key e = do
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
-			g <- gitRepo
-			liftIO $ renameFile (gitAnnexLocation g key) file
+			src <- fromRepo $ gitAnnexLocation key
+			liftIO $ renameFile src file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
 cleanup file key hascontent = do
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index b717e271d1..945848e9f7 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -41,10 +41,9 @@ perform url file = do
 
 download :: String -> FilePath -> CommandPerform
 download url file = do
-	g <- gitRepo
 	showAction $ "downloading " ++ url ++ " "
 	let dummykey = Backend.URL.fromUrl url
-	let tmp = gitAnnexTmpLocation g dummykey
+	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	ok <- liftIO $ Url.download url tmp
 	if ok
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 70dcc4cc72..55c21f83bf 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -58,17 +58,15 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 			next $ Command.Drop.cleanupRemote key r
 		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
 
-performOther :: (Git.Repo -> Key -> FilePath) -> Key -> CommandPerform
+performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
-	g <- gitRepo
-	let f = filespec g key
+	f <- fromRepo $ filespec key
 	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
 
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
-	g <- gitRepo
-	let f = gitAnnexUnusedLog prefix g
+	f <- fromRepo $ gitAnnexUnusedLog prefix
 	e <- liftIO $ doesFileExist f
 	if e
 		then M.fromList . map parse . lines <$> liftIO (readFile f)
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index d1abb29e37..3feabeb9ee 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -79,26 +79,26 @@ check = sequence >=> dispatch
    in this repository only. -}
 verifyLocationLog :: Key -> String -> Annex Bool
 verifyLocationLog key desc = do
-	g <- gitRepo
 	present <- inAnnex key
 	
 	-- Since we're checking that a key's file is present, throw
 	-- in a permission fixup here too.
-	when present $ liftIO $ do
-		let f = gitAnnexLocation g key
-		preventWrite f
-		preventWrite (parentDir f)
+	when present $ do
+		f <- fromRepo $ gitAnnexLocation key
+		liftIO $ do
+			preventWrite f
+			preventWrite (parentDir f)
 
 	u <- getUUID
         uuids <- keyLocations key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix g u InfoPresent
+				fix u InfoPresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix g u InfoMissing
+				fix u InfoMissing
 				warning $
 					"** Based on the location log, " ++ desc
 					++ "\n** was expected to be present, " ++
@@ -107,16 +107,16 @@ verifyLocationLog key desc = do
 		_ -> return True
 	
 	where
-		fix g u s = do
+		fix u s = do
 			showNote "fixing location log"
+			g <- gitRepo
 			logChange g key u s
 
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
-	g <- gitRepo
-	let file = gitAnnexLocation g key
+	file <- fromRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	case (present, Types.Key.keySize key) of
 		(_, Nothing) -> return True
diff --git a/Command/Map.hs b/Command/Map.hs
index 11808ed636..f72cb107ad 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -31,8 +31,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = do
-	g <- gitRepo
-	rs <- spider g
+	rs <- spider =<< gitRepo
 
 	umap <- uuidMap
 	trusted <- trustGet Trusted
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 2d4d24a224..a823466dc7 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -42,14 +42,13 @@ upgradableKey key = isNothing $ Types.Key.keySize key
 
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	g <- gitRepo
-
 	-- Store the old backend's cached key in the new backend
 	-- (the file can't be stored as usual, because it's already a symlink).
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
-	let src = gitAnnexLocation g oldkey
-	let tmpfile = gitAnnexTmpDir g  takeFileName file
+	src <- fromRepo $ gitAnnexLocation oldkey
+	tmp <- fromRepo $ gitAnnexTmpDir
+	let tmpfile = tmp  takeFileName file
 	liftIO $ createLink src tmpfile
 	k <- Backend.genKey tmpfile $ Just newbackend
 	liftIO $ cleantmp tmpfile
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 318ea56d03..5737478671 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -21,8 +21,7 @@ seek = [withKeys start]
 
 start :: Key -> CommandStart
 start key = do
-	g <- gitRepo
-	let file = gitAnnexLocation g key
+	file <- fromRepo $ gitAnnexLocation key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
 	warning "requested key is not present"
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 825f819390..d24f921a93 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -31,8 +31,8 @@ start file = isAnnexed file $ \(key, _) -> do
 		then do
 			force <- Annex.getState Annex.force
 			unless force $ do
-				g <- gitRepo
-				staged <- liftIO $ LsFiles.staged g [Git.workTree g]
+				top <- fromRepo Git.workTree
+				staged <- inRepo $ LsFiles.staged [top]
 				unless (null staged) $
 					error "This command cannot be run when there are already files staged for commit."
 				Annex.changeState $ \s -> s { Annex.force = True }
@@ -46,19 +46,19 @@ perform file key = next $ cleanup file key
 
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
-	g <- gitRepo
-
 	liftIO $ removeFile file
-	liftIO $ Git.run g "rm" [Params "--quiet --", File file]
+	inRepo $ Git.run "rm" [Params "--quiet --", File file]
 	-- git rm deletes empty directories; put them back
 	liftIO $ createDirectoryIfMissing True (parentDir file)
 
 	fast <- Annex.getState Annex.fast
 	if fast
-		then liftIO $ do
+		then do
 			-- fast mode: hard link to content in annex
-			createLink (gitAnnexLocation g key) file
-			allowWrite file
+			src <- fromRepo $ gitAnnexLocation key
+			liftIO $ do
+				createLink src file
+				allowWrite file
 		else do
 			fromAnnex key file
 			logStatus key InfoMissing
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 5a6ee0be25..f317b7620d 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,11 +28,9 @@ check = do
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ b ++ " branch is checked out"
 	where
-		current_branch = do
-			g <- gitRepo
-			b <- liftIO $
-				Git.pipeRead g [Params "rev-parse --abbrev-ref HEAD"]
-			return $ head $ lines $ B.unpack b
+		current_branch = head . lines . B.unpack <$> revhead
+		revhead = inRepo $ Git.pipeRead 
+			[Params "rev-parse --abbrev-ref HEAD"]
 
 seek :: [CommandSeek]
 seek = [withFilesInGit startUnannex, withNothing start]
@@ -53,12 +51,11 @@ perform = next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
-	g <- gitRepo
+	annexdir <- fromRepo $ gitAnnexDir
 	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
-	liftIO $ removeDirectoryRecursive (gitAnnexDir g)
+	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
-	liftIO $ do
-		Git.run g "branch" [Param "-D", Param Annex.Branch.name]
-		exitSuccess
+	inRepo $ Git.run "branch" [Param "-D", Param Annex.Branch.name]
+	liftIO $ exitSuccess
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 7ecaf0b7fa..590b753111 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -37,9 +37,8 @@ perform dest key = do
 	
 	checkDiskSpace key
 
-	g <- gitRepo
-	let src = gitAnnexLocation g key
-	let tmpdest = gitAnnexTmpLocation g key
+	src <- fromRepo $ gitAnnexLocation key
+	tmpdest <- fromRepo $ gitAnnexTmpLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
 	ok <- liftIO $ copyFileExternal src tmpdest
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7e9ffa01f5..9d56d1ff11 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -75,8 +75,8 @@ checkRemoteUnused' r = do
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
-	g <- gitRepo
-	liftIO $ viaTmp writeFile (gitAnnexUnusedLog prefix g) $
+	logfile <- fromRepo $ gitAnnexUnusedLog prefix
+	liftIO $ viaTmp writeFile logfile $
 		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
 
 table :: [(Int, Key)] -> [String]
@@ -147,8 +147,7 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	g <- gitRepo
-	c <- liftIO $ Git.pipeRead g [Param "show-ref"]
+	c <- inRepo $ Git.pipeRead [Param "show-ref"]
 	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
 	where
@@ -183,8 +182,8 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 {- List of keys referenced by symlinks in the git repo. -}
 getKeysReferenced :: Annex [Key]
 getKeysReferenced = do
-	g <- gitRepo
-	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
+	top <- fromRepo Git.workTree
+	files <- inRepo $ LsFiles.inRepo [top]
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
 
@@ -192,8 +191,7 @@ getKeysReferenced = do
 getKeysReferencedInGit :: String -> Annex [Key]
 getKeysReferencedInGit ref = do
 	showAction $ "checking " ++ Git.refDescribe ref
-	g <- gitRepo
-	findkeys [] =<< liftIO (LsTree.lsTree g ref)
+	findkeys [] =<< inRepo (LsTree.lsTree ref)
 	where
 		findkeys c [] = return c
 		findkeys c (l:ls)
@@ -217,16 +215,14 @@ staleKeysPrune dirspec present = do
 	let stale = contents `exclude` present
 	let dups = contents `exclude` stale
 
-	g <- gitRepo
-	let dir = dirspec g
+	dir <- fromRepo dirspec
 	liftIO $ forM_ dups $ \t -> removeFile $ dir  keyFile t
 
 	return stale
 
 staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
 staleKeys dirspec = do
-	g <- gitRepo
-	let dir = dirspec g
+	dir <- fromRepo dirspec
 	exists <- liftIO $ doesDirectoryExist dir
 	if not exists
 		then return []
diff --git a/Common/Annex.hs b/Common/Annex.hs
index f802ec2533..6b5bc31de2 100644
--- a/Common/Annex.hs
+++ b/Common/Annex.hs
@@ -10,6 +10,6 @@ module Common.Annex (
 import Common
 import Types
 import Types.UUID (toUUID, fromUUID)
-import Annex (gitRepo)
+import Annex (gitRepo, inRepo, fromRepo)
 import Locations
 import Messages
diff --git a/Config.hs b/Config.hs
index bf929219df..c24ab9d04d 100644
--- a/Config.hs
+++ b/Config.hs
@@ -16,19 +16,17 @@ type ConfigKey = String
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
-	g <- gitRepo
-	liftIO $ Git.run g "config" [Param k, Param value]
+	inRepo $ Git.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
-	g' <- liftIO $ Git.configRead g
-	Annex.changeState $ \s -> s { Annex.repo = g' }
+	newg <- inRepo $ Git.configRead
+	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
-	g <- gitRepo
-	let def' = Git.configGet g ("annex." ++ key) def
-	return $ Git.configGet g (remoteConfig r key) def'
+	def' <- fromRepo $ Git.configGet ("annex." ++ key) def
+	fromRepo $ Git.configGet (remoteConfig r key) def'
 
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
@@ -87,7 +85,5 @@ getNumCopies v =
 	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
 	where
 		use (Just n) = return n
-		use Nothing = do
-			g <- gitRepo
-			return $ read $ Git.configGet g config "1"
+		use Nothing = read <$> fromRepo (Git.configGet config "1")
 		config = "annex.numcopies"
diff --git a/Git.hs b/Git.hs
index b05c3b2d5e..6fb6e8361c 100644
--- a/Git.hs
+++ b/Git.hs
@@ -188,13 +188,13 @@ repoRemoteName Repo { remoteName = Just name } = Just name
 repoRemoteName _ = Nothing
 
 {- Sets the name of a remote. -}
-repoRemoteNameSet :: Repo -> String -> Repo
-repoRemoteNameSet r n = r { remoteName = Just n }
+repoRemoteNameSet :: String -> Repo -> Repo
+repoRemoteNameSet n r = r { remoteName = Just n }
 
 {- Sets the name of a remote based on the git config key, such as
    "remote.foo.url". -}
-repoRemoteNameFromKey :: Repo -> String -> Repo
-repoRemoteNameFromKey r k = repoRemoteNameSet r basename
+repoRemoteNameFromKey :: String -> Repo -> Repo
+repoRemoteNameFromKey k = repoRemoteNameSet basename
 	where
 		basename = join "." $ reverse $ drop 1 $
 				reverse $ drop 1 $ split "." k
@@ -280,8 +280,8 @@ workTree Repo { location = Unknown } = undefined
  - is itself a symlink). But, if the cwd is "/tmp/repo/subdir",
  - it's best to refer to "../foo".
  -}
-workTreeFile :: Repo -> FilePath -> IO FilePath
-workTreeFile repo@(Repo { location = Dir d }) file = do
+workTreeFile :: FilePath -> Repo -> IO FilePath
+workTreeFile file repo@(Repo { location = Dir d }) = do
 	cwd <- getCurrentDirectory
 	let file' = absfile cwd
 	unless (inrepo file') $
@@ -296,7 +296,7 @@ workTreeFile repo@(Repo { location = Dir d }) file = do
 		absfile c = fromMaybe file $ secureAbsNormPath c file
 		inrepo f = absrepo `isPrefixOf` f
 		bad = error $ "bad repo" ++ repoDescribe repo
-workTreeFile repo _ = assertLocal repo $ error "internal"
+workTreeFile _ repo = assertLocal repo $ error "internal"
 
 {- Path of an URL repo. -}
 urlPath :: Repo -> String
@@ -350,23 +350,23 @@ urlAuthPart a Repo { location = Url u } = a auth
 urlAuthPart _ repo = assertUrl repo $ error "internal"
 
 {- Constructs a git command line operating on the specified repo. -}
-gitCommandLine :: Repo -> [CommandParam] -> [CommandParam]
-gitCommandLine repo@(Repo { location = Dir _ } ) params =
+gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
+gitCommandLine params repo@(Repo { location = Dir _ } ) =
 	-- force use of specified repo via --git-dir and --work-tree
 	[ Param ("--git-dir=" ++ gitDir repo)
 	, Param ("--work-tree=" ++ workTree repo)
 	] ++ params
-gitCommandLine repo _ = assertLocal repo $ error "internal"
+gitCommandLine _ repo = assertLocal repo $ error "internal"
 
 {- Runs git in the specified repo. -}
-runBool :: Repo -> String -> [CommandParam] -> IO Bool
-runBool repo subcommand params = assertLocal repo $
-	boolSystem "git" $ gitCommandLine repo $ Param subcommand : params
+runBool :: String -> [CommandParam] -> Repo -> IO Bool
+runBool subcommand params repo = assertLocal repo $
+	boolSystem "git" $ gitCommandLine (Param subcommand : params) repo
 
 {- Runs git in the specified repo, throwing an error if it fails. -}
-run :: Repo -> String -> [CommandParam] -> IO ()
-run repo subcommand params = assertLocal repo $
-	runBool repo subcommand params
+run :: String -> [CommandParam] -> Repo -> IO ()
+run subcommand params repo = assertLocal repo $
+	runBool subcommand params repo
 		>>! error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns its output, lazily. 
@@ -374,26 +374,26 @@ run repo subcommand params = assertLocal repo $
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
  -}
-pipeRead :: Repo -> [CommandParam] -> IO L.ByteString
-pipeRead repo params = assertLocal repo $ do
-	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine repo params
+pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
+pipeRead params repo = assertLocal repo $ do
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
 	hSetBinaryMode h True
 	L.hGetContents h
 
 {- Runs a git subcommand, feeding it input.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: Repo -> [CommandParam] -> L.ByteString -> IO PipeHandle
-pipeWrite repo params s = assertLocal repo $ do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine repo params)
+pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
+pipeWrite params s repo = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
 	L.hPut h s
 	hClose h
 	return p
 
 {- Runs a git subcommand, feeding it input, and returning its output.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: Repo -> [CommandParam] -> L.ByteString -> IO (PipeHandle, L.ByteString)
-pipeWriteRead repo params s = assertLocal repo $ do
-	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine repo params)
+pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
+pipeWriteRead params s repo = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
 	hSetBinaryMode from True
 	L.hPut to s
 	hClose to
@@ -402,13 +402,13 @@ pipeWriteRead repo params s = assertLocal repo $ do
 
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it. -}
-pipeNullSplit :: Repo -> [CommandParam] -> IO [String]
-pipeNullSplit repo params = map L.unpack <$> pipeNullSplitB repo params
+pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
+pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
 
 {- For when Strings are not needed. -}
-pipeNullSplitB :: Repo -> [CommandParam] -> IO [L.ByteString]
-pipeNullSplitB repo params = filter (not . L.null) . L.split '\0' <$>
-	pipeRead repo params
+pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
+pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
+	pipeRead params repo
 
 {- Reaps any zombie git processes. -}
 reap :: IO ()
@@ -448,15 +448,15 @@ shaSize = 40
 
 {- Commits the index into the specified branch, 
  - with the specified parent refs. -}
-commit :: Repo -> String -> String -> [String] -> IO ()
-commit g message newref parentrefs = do
+commit :: String -> String -> [String] -> Repo -> IO ()
+commit message newref parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
-		pipeRead g [Param "write-tree"]
+		pipeRead [Param "write-tree"] repo
 	sha <- getSha "commit-tree" $ asString $
-		ignorehandle $ pipeWriteRead g
+		ignorehandle $ pipeWriteRead
 			(map Param $ ["commit-tree", tree] ++ ps)
-			(L.pack message)
-	run g "update-ref" [Param newref, Param sha]
+			(L.pack message) repo
+	run "update-ref" [Param newref, Param sha] repo
 	where
 		ignorehandle a = snd <$> a
 		asString a = L.unpack <$> a
@@ -478,13 +478,13 @@ configRead r = assertLocal r $ error "internal"
 hConfigRead :: Repo -> Handle -> IO Repo
 hConfigRead repo h = do
 	val <- hGetContentsStrict h
-	configStore repo val
+	configStore val repo
 
 {- Stores a git config into a repo, returning the new version of the repo.
  - The git config may be multiple lines, or a single line. Config settings
  - can be updated inrementally. -}
-configStore :: Repo -> String -> IO Repo
-configStore repo s = do
+configStore :: String -> Repo -> IO Repo
+configStore s repo = do
 	let repo' = repo { config = configParse s `M.union` config repo }
 	rs <- configRemotes repo'
 	return $ repo' { remotes = rs }
@@ -507,13 +507,11 @@ configRemotes repo = mapM construct remotepairs
 		filterkeys f = filterconfig (\(k,_) -> f k)
 		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
-		construct (k,v) = do
-			r <- genRemote repo v
-			return $ repoRemoteNameFromKey r k
+		construct (k,v) = repoRemoteNameFromKey k <$> genRemote v repo
 
 {- Generates one of a repo's remotes using a given location (ie, an url). -}
-genRemote :: Repo -> String -> IO Repo
-genRemote repo = gen . calcloc
+genRemote :: String -> Repo -> IO Repo
+genRemote s repo = gen $ calcloc s
 	where
 		filterconfig f = filter f $ M.toList $ config repo
 		gen v	
@@ -549,8 +547,8 @@ configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
 {- Returns a single git config setting, or a default value if not set. -}
-configGet :: Repo -> String -> String -> String
-configGet repo key defaultValue = 
+configGet :: String -> String -> Repo -> String
+configGet key defaultValue repo = 
 	M.findWithDefault defaultValue key (config repo)
 
 {- Access to raw config Map -}
@@ -558,8 +556,8 @@ configMap :: Repo -> M.Map String String
 configMap = config
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
-checkAttr :: Repo -> String -> [FilePath] -> IO [(FilePath, String)]
-checkAttr repo attr files = do
+checkAttr :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
+checkAttr attr files repo = do
 	-- git check-attr needs relative filenames input; it will choke
 	-- on some absolute filenames. This also means it will output
 	-- all relative filenames.
@@ -574,7 +572,11 @@ checkAttr repo attr files = do
         hClose toh
 	(map topair . lines) <$> hGetContents fromh
 	where
-		params = gitCommandLine repo [Param "check-attr", Param attr, Params "-z --stdin"]
+		params = gitCommandLine 
+				[ Param "check-attr"
+				, Param attr
+				, Params "-z --stdin"
+				] repo
 		topair l = (file, value)
 			where 
 				file = decodeGitFile $ join sep $ take end bits
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 64857c66a9..51fa585a82 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -25,7 +25,7 @@ type CatFileHandle = (PipeHandle, Handle, Handle)
 {- Starts git cat-file running in batch mode in a repo and returns a handle. -}
 catFileStart :: Repo -> IO CatFileHandle
 catFileStart repo = hPipeBoth "git" $ toCommand $
-	Git.gitCommandLine repo [Param "cat-file", Param "--batch"]
+	Git.gitCommandLine [Param "cat-file", Param "--batch"] repo
 
 {- Stops git cat-file. -}
 catFileStop :: CatFileHandle -> IO ()
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 28e007a4dc..bceee26fcd 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -19,51 +19,52 @@ import Git
 import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
-inRepo :: Repo -> [FilePath] -> IO [FilePath]
-inRepo repo l = pipeNullSplit repo $ Params "ls-files --cached -z --" : map File l
+inRepo :: [FilePath] -> Repo -> IO [FilePath]
+inRepo l repo = pipeNullSplit (Params "ls-files --cached -z --" : map File l) repo
 
 {- Scans for files at the specified locations that are not checked into git. -}
-notInRepo :: Repo -> Bool -> [FilePath] -> IO [FilePath]
-notInRepo repo include_ignored l = pipeNullSplit repo $
-		[Params "ls-files --others"] ++ exclude ++
-		[Params "-z --"] ++ map File l
+notInRepo :: Bool -> [FilePath] -> Repo -> IO [FilePath]
+notInRepo include_ignored l repo = pipeNullSplit params repo
 	where
+		params = [Params "ls-files --others"] ++ exclude ++
+			[Params "-z --"] ++ map File l
 		exclude
 			| include_ignored = []
 			| otherwise = [Param "--exclude-standard"]
 
 {- Returns a list of all files that are staged for commit. -}
-staged :: Repo -> [FilePath] -> IO [FilePath]
-staged repo l = staged' repo l []
+staged :: [FilePath] -> Repo -> IO [FilePath]
+staged = staged' []
 
 {- Returns a list of the files, staged for commit, that are being added,
  - moved, or changed (but not deleted), from the specified locations. -}
-stagedNotDeleted :: Repo -> [FilePath] -> IO [FilePath]
-stagedNotDeleted repo l = staged' repo l [Param "--diff-filter=ACMRT"]
+stagedNotDeleted :: [FilePath] -> Repo -> IO [FilePath]
+stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"]
 
-staged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-staged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+staged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
+staged' middle l = pipeNullSplit $ start ++ middle ++ end
 	where
 		start = [Params "diff --cached --name-only -z"]
 		end = Param "--" : map File l
 
 {- Returns a list of files that have unstaged changes. -}
-changedUnstaged :: Repo -> [FilePath] -> IO [FilePath]
-changedUnstaged repo l = pipeNullSplit repo $
-	Params "diff --name-only -z --" : map File l
+changedUnstaged :: [FilePath] -> Repo -> IO [FilePath]
+changedUnstaged l = pipeNullSplit params
+	where
+		params = Params "diff --name-only -z --" : map File l
 
 {- Returns a list of the files in the specified locations that are staged
  - for commit, and whose type has changed. -}
-typeChangedStaged :: Repo -> [FilePath] -> IO [FilePath]
-typeChangedStaged repo l = typeChanged' repo l [Param "--cached"]
+typeChangedStaged :: [FilePath] -> Repo -> IO [FilePath]
+typeChangedStaged = typeChanged' [Param "--cached"]
 
 {- Returns a list of the files in the specified locations whose type has
  - changed.  Files only staged for commit will not be included. -}
-typeChanged :: Repo -> [FilePath] -> IO [FilePath]
-typeChanged repo l = typeChanged' repo l []
+typeChanged :: [FilePath] -> Repo -> IO [FilePath]
+typeChanged = typeChanged' []
 
-typeChanged' :: Repo -> [FilePath] -> [CommandParam] -> IO [FilePath]
-typeChanged' repo l middle = pipeNullSplit repo $ start ++ middle ++ end
+typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
+typeChanged' middle l = pipeNullSplit $ start ++ middle ++ end
 	where
 		start = [Params "diff --name-only --diff-filter=T -z"]
 		end = Param "--" : map File l
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index c072ef5be9..1fcdf13ed5 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -29,9 +29,9 @@ data TreeItem = TreeItem
 	} deriving Show
 
 {- Lists the contents of a Treeish -}
-lsTree :: Repo -> Treeish -> IO [TreeItem]
-lsTree repo t = map parseLsTree <$>
-	pipeNullSplitB repo [Params "ls-tree --full-tree -z -r --", File t]
+lsTree :: Treeish -> Repo -> IO [TreeItem]
+lsTree t repo = map parseLsTree <$>
+	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File t] repo
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 25b9ffad08..70c766d044 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -72,8 +72,8 @@ full :: Queue -> Bool
 full (Queue n _) = n > maxSize
 
 {- Runs a queue on a git repository. -}
-flush :: Repo -> Queue -> IO Queue
-flush repo (Queue _ m) = do
+flush :: Queue -> Repo -> IO Queue
+flush (Queue _ m) repo = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
 	return empty
 
@@ -87,6 +87,6 @@ runAction :: Repo -> Action -> [FilePath] -> IO ()
 runAction repo action files =
 	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
-		params = toCommand $ gitCommandLine repo
-			(Param (getSubcommand action):getParams action)
+		params = toCommand $ gitCommandLine
+			(Param (getSubcommand action):getParams action) repo
 		feedxargs h = hPutStr h $ join "\0" files
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 859a66ca07..32966c846d 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -27,24 +27,25 @@ import Git
  -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: Repo -> String -> String -> IO ()
-merge g x y = do
-	a <- ls_tree g x
-	b <- merge_trees g x y
-	update_index g (a++b)
+merge :: String -> String -> Repo -> IO ()
+merge x y repo = do
+	a <- ls_tree x repo
+	b <- merge_trees x y repo
+	update_index repo (a++b)
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
 merge_index :: Repo -> [String] -> IO ()
-merge_index g bs = update_index g =<< concat <$> mapM (merge_tree_index g) bs
+merge_index repo bs =
+	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b repo) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index g l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
+update_index repo l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
 	where
-		togit ps content = pipeWrite g (map Param ps) (L.pack content)
+		togit ps content = pipeWrite (map Param ps) (L.pack content) repo
 			>>= forceSuccess
 
 {- Generates a line suitable to be fed into update-index, to add
@@ -53,27 +54,28 @@ update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
 {- Gets the contents of a tree in a format suitable for update_index. -}
-ls_tree :: Repo -> String -> IO [String]
-ls_tree g x = pipeNullSplit g $ 
-	map Param ["ls-tree", "-z", "-r", "--full-tree", x]
+ls_tree :: String -> Repo -> IO [String]
+ls_tree x = pipeNullSplit params
+	where
+		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: Repo -> String -> String -> IO [String]
-merge_trees g x y = calc_merge g $ "diff-tree":diff_opts ++ [x, y]
+merge_trees :: String -> String -> Repo -> IO [String]
+merge_trees x y = calc_merge $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: Repo -> String -> IO [String]
-merge_tree_index g x = calc_merge g $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index :: String -> Repo -> IO [String]
+merge_tree_index x = calc_merge $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: Repo -> [String] -> IO [String]
-calc_merge g differ = do
-	diff <- pipeNullSplit g $ map Param differ
-	l <- mapM (mergeFile g) (pairs diff)
+calc_merge :: [String] -> Repo -> IO [String]
+calc_merge differ repo = do
+	diff <- pipeNullSplit (map Param differ) repo
+	l <- mapM (\p -> mergeFile p repo) (pairs diff)
 	return $ catMaybes l
 	where
 		pairs [] = []
@@ -81,9 +83,9 @@ calc_merge g differ = do
 		pairs (a:b:rest) = (a,b):pairs rest
 
 {- Injects some content into git, returning its hash. -}
-hashObject :: Repo -> L.ByteString -> IO String
-hashObject repo content = getSha subcmd $ do
-	(h, s) <- pipeWriteRead repo (map Param params) content
+hashObject :: L.ByteString -> Repo -> IO String
+hashObject content repo = getSha subcmd $ do
+	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
 		forceSuccess h
 		reap -- XXX unsure why this is needed
@@ -95,13 +97,13 @@ hashObject repo content = getSha subcmd $ do
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: Repo -> (String, FilePath) -> IO (Maybe String)
-mergeFile g (info, file) = case filter (/= nullsha) [asha, bsha] of
+mergeFile :: (String, FilePath) -> Repo -> IO (Maybe String)
+mergeFile (info, file) repo = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- pipeRead g $ map Param ("show":shas)
-		sha <- hashObject g $ unionmerge content
+		content <- pipeRead (map Param ("show":shas)) repo
+		sha <- hashObject (unionmerge content) repo
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 399b26ef7f..f416b7bead 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -116,9 +116,8 @@ options = commonOptions ++
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
-			g <- gitRepo
-			g' <- liftIO $ Git.configStore g v
-			Annex.changeState $ \s -> s { Annex.repo = g' }
+			newg <- inRepo $ Git.configStore v
+			Annex.changeState $ \s -> s { Annex.repo = newg }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
diff --git a/Init.hs b/Init.hs
index 8cec98c5fc..d03e10031f 100644
--- a/Init.hs
+++ b/Init.hs
@@ -68,12 +68,10 @@ gitPreCommitHookUnWrite = unlessBare $ do
 				" Edit it to remove call to git annex."
 
 unlessBare :: Annex () -> Annex ()
-unlessBare = unlessM $ Git.repoIsLocalBare <$> gitRepo
+unlessBare = unlessM $ fromRepo $ Git.repoIsLocalBare
 
 preCommitHook :: Annex FilePath
-preCommitHook = do
-	g <- gitRepo
-	return $ Git.gitDir g ++ "/hooks/pre-commit"
+preCommitHook = () <$> fromRepo Git.gitDir <*> pure "hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 
diff --git a/Locations.hs b/Locations.hs
index ceb6246b98..83897f488f 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -65,8 +65,8 @@ annexLocation key = objectDir  hashDirMixed key  f  f
 		f = keyFile key
 
 {- Annexed file's absolute location in a repository. -}
-gitAnnexLocation :: Git.Repo -> Key -> FilePath
-gitAnnexLocation r key
+gitAnnexLocation :: Key -> Git.Repo -> FilePath
+gitAnnexLocation key r
 	| Git.repoIsLocalBare r = Git.workTree r  annexLocation key
 	| otherwise = Git.workTree r  ".git"  annexLocation key
 
@@ -88,16 +88,16 @@ gitAnnexTmpDir :: Git.Repo -> FilePath
 gitAnnexTmpDir r = addTrailingPathSeparator $ gitAnnexDir r  "tmp"
 
 {- The temp file to use for a given key. -}
-gitAnnexTmpLocation :: Git.Repo -> Key -> FilePath
-gitAnnexTmpLocation r key = gitAnnexTmpDir r  keyFile key
+gitAnnexTmpLocation :: Key -> Git.Repo -> FilePath
+gitAnnexTmpLocation key r = gitAnnexTmpDir r  keyFile key
 
 {- .git/annex/bad/ is used for bad files found during fsck -}
 gitAnnexBadDir :: Git.Repo -> FilePath
 gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 
 {- The bad file to use for a given key. -}
-gitAnnexBadLocation :: Git.Repo -> Key -> FilePath
-gitAnnexBadLocation r key = gitAnnexBadDir r  keyFile key
+gitAnnexBadLocation :: Key -> Git.Repo -> FilePath
+gitAnnexBadLocation key r = gitAnnexBadDir r  keyFile key
 
 {- .git/annex/*unused is used to number possibly unused keys -}
 gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
diff --git a/Remote.hs b/Remote.hs
index 6d55cee243..82af21bde4 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -130,10 +130,10 @@ nameToUUID n = byName' n >>= go
  - of the UUIDs. -}
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
-	here <- getUUID
+	hereu <- getUUID
 	m <- M.unionWith addname <$> uuidMap <*> remoteMap
-	maybeShowJSON [(desc, map (jsonify m here) uuids)]
-	return $ unwords $ map (\u -> "\t" ++ prettify m here u ++ "\n") uuids
+	maybeShowJSON [(desc, map (jsonify m hereu) uuids)]
+	return $ unwords $ map (\u -> "\t" ++ prettify m hereu u ++ "\n") uuids
 	where
 		addname d n
 			| d == n = d
@@ -141,20 +141,20 @@ prettyPrintUUIDs desc uuids = do
 			| otherwise = n ++ " (" ++ d ++ ")"
 		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
-		prettify m here u
+		prettify m hereu u
 			| not (null d) = fromUUID u ++ " -- " ++ d
 			| otherwise = fromUUID u
 			where
-				ishere = here == u
+				ishere = hereu == u
 				n = findlog m u
 				d
 					| null n && ishere = "here"
 					| ishere = addname n "here"
 					| otherwise = n
-		jsonify m here u = toJSObject
+		jsonify m hereu u = toJSObject
 			[ ("uuid", toJSON $ fromUUID u)
 			, ("description", toJSON $ findlog m u)
-			, ("here", toJSON $ here == u)
+			, ("here", toJSON $ hereu == u)
 			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 3e621ce569..b8d7cd3172 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -102,15 +102,13 @@ bupSplitParams r buprepo k src = do
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo k (File src)
 	liftIO $ boolSystem "bup" params
 
 storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck (Param "-")
 	liftIO $ catchBool $
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
@@ -147,7 +145,7 @@ checkPresent r bupr k
 		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
-	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine bupr params
+	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine params bupr
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
@@ -165,9 +163,10 @@ storeBupUUID u buprepo = do
 					>>! error "ssh failed"
 		else liftIO $ do
 			r' <- Git.configRead r
-			let olduuid = Git.configGet r' "annex.uuid" ""
-			when (olduuid == "") $ Git.run r' "config"
-				[Param "annex.uuid", Param v]
+			let olduuid = Git.configGet "annex.uuid" "" r'
+			when (olduuid == "") $
+				Git.run "config"
+					[Param "annex.uuid", Param v] r'
 	where
 		v = fromUUID u
 
@@ -194,7 +193,7 @@ getBupUUID r u
 	| otherwise = liftIO $ do
 		ret <- try $ Git.configRead r
 		case ret of
-			Right r' -> return (toUUID $ Git.configGet r' "annex.uuid" "", r')
+			Right r' -> return (toUUID $ Git.configGet "annex.uuid" "" r', r')
 			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index e8cf05a0e5..8e306e228b 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -70,15 +70,13 @@ dirKey d k = d  hashDirMixed k  f  f
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d k
 	liftIO $ catchBool $ storeHelper dest $ copyFileExternal src dest
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
-	g <- gitRepo
-	let src = gitAnnexLocation g k
+	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d enck
 	liftIO $ catchBool $ storeHelper dest $ encrypt src dest
 	where
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 4c76e8ce6d..75f0ac7576 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -35,19 +35,16 @@ remote = RemoteType {
 
 list :: Annex [Git.Repo]
 list = do
-	g <- gitRepo
-	let c = Git.configMap g
-	mapM (tweakurl c) $ Git.remotes g
+	c <- fromRepo Git.configMap
+	mapM (tweakurl c) =<< fromRepo Git.remotes
 	where
 		annexurl n = "remote." ++ n ++ ".annexurl"
 		tweakurl c r = do
 			let n = fromJust $ Git.repoRemoteName r
 			case M.lookup (annexurl n) c of
 				Nothing -> return r
-				Just url -> do
-					g <- gitRepo
-					r' <- liftIO $ Git.genRemote g url
-					return $ Git.repoRemoteNameSet r' n
+				Just url -> Git.repoRemoteNameSet n <$>
+					inRepo (Git.genRemote url)
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
@@ -178,7 +175,7 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
-		rsyncOrCopyFile params (gitAnnexLocation r key) file
+		rsyncOrCopyFile params (gitAnnexLocation key r) file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
@@ -187,8 +184,7 @@ copyFromRemote r key file
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
-		g <- gitRepo
-		let keysrc = gitAnnexLocation g key
+		keysrc <- fromRepo $ gitAnnexLocation key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
@@ -197,8 +193,7 @@ copyToRemote r key
 			Annex.Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
-		g <- gitRepo
-		let keysrc = gitAnnexLocation g key
+		keysrc <- fromRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 38f24eb373..6cea170347 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -23,16 +23,16 @@ findSpecialRemotes s = do
 	return $ map construct $ remotepairs g
 	where
 		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
-		construct (k,_) = Git.repoRemoteNameFromKey Git.repoFromUnknown k
+		construct (k,_) = Git.repoRemoteNameFromKey k Git.repoFromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
 gitConfigSpecialRemote :: UUID -> RemoteConfig -> String -> String -> Annex ()
 gitConfigSpecialRemote u c k v = do
-	g <- gitRepo
-	liftIO $ do
-		Git.run g "config" [Param (configsetting $ "annex-"++k), Param v]
-		Git.run g "config" [Param (configsetting "annex-uuid"), Param $ fromUUID u]
+	set ("annex-"++k) v
+	set ("annex-uuid") (fromUUID u)
 	where
+		set a b = inRepo $ Git.run "config"
+			[Param (configsetting a), Param b]
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 8b6a6cecfc..06568a3cb5 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -98,14 +98,13 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 
 store :: String -> Key -> Annex Bool
 store h k = do
-	g <- gitRepo
-	runHook h "store" k (Just $ gitAnnexLocation g k) $ return True
+	src <- fromRepo $ gitAnnexLocation k
+	runHook h "store" k (Just src) $ return True
 
 storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- gitRepo
-	let f = gitAnnexLocation g k
-	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
+	src <- fromRepo $ gitAnnexLocation k
+	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
 retrieve :: String -> Key -> FilePath -> Annex Bool
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index e79762a382..0dfad72935 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -90,15 +90,12 @@ rsyncKeyDir :: RsyncOpts -> Key -> String
 rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
-store o k = do
-	g <- gitRepo
-	rsyncSend o k (gitAnnexLocation g k)
+store o k = rsyncSend o k =<< fromRepo (gitAnnexLocation k)
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
-	g <- gitRepo
-	let f = gitAnnexLocation g k
-	liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
+	src <- fromRepo $ gitAnnexLocation k
+	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
@@ -151,9 +148,9 @@ partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial"
  - up trees for rsync. -}
 withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
-	g <- gitRepo
 	pid <- liftIO getProcessID
-	let tmp = gitAnnexTmpDir g  "rsynctmp"  show pid
+	t <- fromRepo gitAnnexTmpDir
+	let tmp = t  "rsynctmp"  show pid
 	nuke tmp
 	liftIO $ createDirectoryIfMissing True tmp
 	res <- a tmp
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 1281c27861..b201b5aadb 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -112,8 +112,8 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
-	g <- gitRepo
-	res <- liftIO $ storeHelper (conn, bucket) r k $ gitAnnexLocation g k
+	dest <- fromRepo $ gitAnnexLocation k
+	res <- liftIO $ storeHelper (conn, bucket) r k dest
 	s3Bool res
 
 storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool
@@ -121,8 +121,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
-		g <- gitRepo
-		let f = gitAnnexLocation g k
+		f <- fromRepo $ gitAnnexLocation k
 		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
 		s3Bool res
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 393932d478..da7f384727 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -27,7 +27,7 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet Git.repoFromUnknown "web"]
+list = return [Git.repoRemoteNameSet "web" Git.repoFromUnknown]
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
diff --git a/Seek.hs b/Seek.hs
index 4ae943157b..3c83ebc35d 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -20,16 +20,18 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 import qualified Limit
 
+seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
+seekHelper a params = do
+	g <- gitRepo
+	liftIO $ runPreserveOrder (\p -> a p g) params
+
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
-withFilesInGit a params = do
-	repo <- gitRepo
-	prepFiltered a $ liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
 
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
-	prepFilteredGen a fst $ liftIO $ Git.checkAttr repo attr files
+	files <- seekHelper LsFiles.inRepo params
+	prepFilteredGen a fst $ inRepo $ Git.checkAttr attr files
 
 withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
@@ -38,8 +40,7 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do
-	repo <- gitRepo
-	files <- liftIO $ runPreserveOrder (LsFiles.inRepo repo) params
+	files <- seekHelper LsFiles.inRepo params
 	prepBackendPairs a files
 
 withFilesMissing :: (String -> CommandStart) -> CommandSeek
@@ -49,9 +50,8 @@ withFilesMissing a params = prepFiltered a $ liftIO $ filterM missing params
 
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
-	repo <- gitRepo
 	force <- Annex.getState Annex.force
-	newfiles <- liftIO $ runPreserveOrder (LsFiles.notInRepo repo force) params
+	newfiles <- seekHelper (LsFiles.notInRepo force) params
 	prepBackendPairs a newfiles
 
 withWords :: ([String] -> CommandStart) -> CommandSeek
@@ -61,10 +61,8 @@ withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
 
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
-withFilesToBeCommitted a params = do
-	repo <- gitRepo
-	prepFiltered a $
-		liftIO $ runPreserveOrder (LsFiles.stagedNotDeleted repo) params
+withFilesToBeCommitted a params = prepFiltered a $
+	seekHelper LsFiles.stagedNotDeleted params
 
 withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
@@ -72,13 +70,13 @@ withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 
-withFilesUnlocked' :: (Git.Repo -> [FilePath] -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	repo <- gitRepo
-	typechangedfiles <- liftIO $ runPreserveOrder (typechanged repo) params
+	top <- fromRepo $ Git.workTree
+	typechangedfiles <- seekHelper typechanged params
 	unlockedfiles <- liftIO $ filterM notSymlink $
-		map (\f -> Git.workTree repo ++ "/" ++ f) typechangedfiles
+		map (\f -> top ++ "/" ++ f) typechangedfiles
 	prepBackendPairs a unlockedfiles
 
 withKeys :: (Key -> CommandStart) -> CommandSeek
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index b1443fa460..eae5c87ce4 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -16,10 +16,9 @@ import qualified Upgrade.V1
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v0 to v1"
-	g <- gitRepo
 
 	-- do the reorganisation of the key files
-	let olddir = gitAnnexDir g
+	olddir <- fromRepo gitAnnexDir
 	keys <- getKeysPresent0 olddir
 	forM_ keys $ \k -> moveAnnex k $ olddir  keyFile0 k
 
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index be9a977ad7..fe59ad3da8 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -50,9 +50,9 @@ import qualified Upgrade.V2
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
-
-	g <- gitRepo
-	if Git.repoIsLocalBare g
+	
+	bare <- fromRepo $ Git.repoIsLocalBare
+	if bare
 		then do
 			moveContent
 			setVersion
@@ -83,8 +83,8 @@ moveContent = do
 updateSymlinks :: Annex ()
 updateSymlinks = do
 	showAction "updating symlinks"
-	g <- gitRepo
-	files <- liftIO $ LsFiles.inRepo g [Git.workTree g]
+	top <- fromRepo Git.workTree
+	files <- inRepo $ LsFiles.inRepo [top]
 	forM_ files fixlink
 	where
 		fixlink f = do
@@ -104,8 +104,7 @@ moveLocationLogs = do
 	forM_ logkeys move
 		where
 			oldlocationlogs = do
-				g <- gitRepo
-				let dir = Upgrade.V2.gitStateDir g
+				dir <- fromRepo Upgrade.V2.gitStateDir
 				exists <- liftIO $ doesDirectoryExist dir
 				if exists
 					then do
@@ -113,9 +112,8 @@ moveLocationLogs = do
 						return $ mapMaybe oldlog2key contents
 					else return []
 			move (l, k) = do
-				g <- gitRepo
-				let dest = logFile2 g k
-				let dir = Upgrade.V2.gitStateDir g
+				dest <- fromRepo $ logFile2 k
+				dir <- fromRepo $ Upgrade.V2.gitStateDir
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
 				-- could just git mv, but this way deals with
@@ -206,9 +204,7 @@ lookupFile1 file = do
 					" (unknown backend " ++ bname ++ ")"
 
 getKeyFilesPresent1 :: Annex [FilePath]
-getKeyFilesPresent1  = do
-	g <- gitRepo
-	getKeyFilesPresent1' $ gitAnnexObjectDir g
+getKeyFilesPresent1  = getKeyFilesPresent1' =<< fromRepo gitAnnexObjectDir
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
 getKeyFilesPresent1' dir = do
 	exists <- liftIO $ doesDirectoryExist dir
@@ -228,11 +224,11 @@ getKeyFilesPresent1' dir = do
 logFile1 :: Git.Repo -> Key -> String
 logFile1 repo key = Upgrade.V2.gitStateDir repo ++ keyFile1 key ++ ".log"
 
-logFile2 :: Git.Repo -> Key -> String
+logFile2 :: Key -> Git.Repo -> String
 logFile2 = logFile' hashDirLower
 
-logFile' :: (Key -> FilePath) -> Git.Repo -> Key -> String
-logFile' hasher repo key =
+logFile' :: (Key -> FilePath) -> Key -> Git.Repo -> String
+logFile' hasher key repo =
 	gitStateDir repo ++ hasher key ++ keyFile key ++ ".log"
 
 stateDir :: FilePath
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 67f0205c14..7ef2a4d18e 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -37,45 +37,46 @@ olddir g
 upgrade :: Annex Bool
 upgrade = do
 	showAction "v2 to v3"
-	g <- gitRepo
-	let bare = Git.repoIsLocalBare g
+	bare <- fromRepo Git.repoIsLocalBare
+	old <- fromRepo olddir
 
 	Annex.Branch.create
 	showProgress
 
-	e <- liftIO $ doesDirectoryExist (olddir g)
+	e <- liftIO $ doesDirectoryExist old
 	when e $ do
-		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs g
-		mapM_ (\f -> inject f f) =<< logFiles (olddir g)
+		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs
+		mapM_ (\f -> inject f f) =<< logFiles old
 
 	saveState
 	showProgress
 
-	when e $ liftIO $ do
-		Git.run g "rm" [Param "-r", Param "-f", Param "-q", File (olddir g)]
-		unless bare $ gitAttributesUnWrite g
+	when e $ do
+		inRepo $ Git.run "rm" [Param "-r", Param "-f", Param "-q", File old]
+		unless bare $ inRepo $ gitAttributesUnWrite
 	showProgress
 
 	unless bare push
 
 	return True
 
-locationLogs :: Git.Repo -> Annex [(Key, FilePath)]
-locationLogs repo = liftIO $ do
-	levela <- dirContents dir
-	levelb <- mapM tryDirContents levela
-	files <- mapM tryDirContents (concat levelb)
-	return $ mapMaybe islogfile (concat files)
+locationLogs :: Annex [(Key, FilePath)]
+locationLogs = do
+	dir <- fromRepo gitStateDir
+	liftIO $ do
+		levela <- dirContents dir
+		levelb <- mapM tryDirContents levela
+		files <- mapM tryDirContents (concat levelb)
+		return $ mapMaybe islogfile (concat files)
 	where
 		tryDirContents d = catch (dirContents d) (return . const [])
-		dir = gitStateDir repo
 		islogfile f = maybe Nothing (\k -> Just (k, f)) $
 				logFileKey $ takeFileName f
 
 inject :: FilePath -> FilePath -> Annex ()
 inject source dest = do
-	g <- gitRepo
-	new <- liftIO (readFile $ olddir g  source)
+	old <- fromRepo olddir
+	new <- liftIO (readFile $ old  source)
 	Annex.Branch.change dest $ \prev -> 
 		unlines $ nub $ lines prev ++ lines new
 
@@ -102,8 +103,7 @@ push = do
 			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			g <- gitRepo
-			liftIO $ Git.run g "push" [Param "origin", Param Annex.Branch.name]
+			inRepo $ Git.run "push" [Param "origin", Param Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
@@ -126,7 +126,7 @@ gitAttributesUnWrite repo = do
 		c <- readFileStrict attributes
 		liftIO $ viaTmp writeFile attributes $ unlines $
 			filter (`notElem` attrLines) $ lines c
-		Git.run repo "add" [File attributes]
+		Git.run "add" [File attributes] repo
 
 stateDir :: FilePath
 stateDir = addTrailingPathSeparator ".git-annex"
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 0d1d0819d2..10ae842177 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -41,6 +41,6 @@ main = do
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
-	Git.UnionMerge.merge g aref bref
-	Git.commit g "union merge" newref [aref, bref]
+	Git.UnionMerge.merge aref bref g
+	Git.commit "union merge" newref [aref, bref] g
 	cleanup g

From 56b8194470f0ddc1b3e1c51da8ca55a5c9c2eb05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 01:15:51 -0400
Subject: [PATCH 2474/8313] cleanup

---
 Annex.hs                 | 6 +++---
 Annex/Content.hs         | 3 +--
 Command/Fsck.hs          | 3 +--
 Logs/Location.hs         | 9 +++------
 Logs/Web.hs              | 3 +--
 Remote.hs                | 5 +----
 Remote/Helper/Special.hs | 6 +++---
 7 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 501e54c662..1c1091799f 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -1,6 +1,6 @@
 {- git-annex monad
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -48,6 +48,8 @@ newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
 		Applicative
 	)
 
+data OutputType = NormalOutput | QuietOutput | JSONOutput
+
 -- internal state storage
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
@@ -70,8 +72,6 @@ data AnnexState = AnnexState
 	, cipher :: Maybe Cipher
 	}
 
-data OutputType = NormalOutput | QuietOutput | JSONOutput
-
 newState :: Git.Repo -> AnnexState
 newState gitrepo = AnnexState
 	{ repo = gitrepo
diff --git a/Annex/Content.hs b/Annex/Content.hs
index fc2c2d0921..ecfec66aab 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -56,9 +56,8 @@ calcGitLink file key = do
  - repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
 logStatus key status = do
-	g <- gitRepo
 	u <- getUUID
-	logChange g key u status
+	logChange key u status
 
 {- Runs an action, passing it a temporary filename to download,
  - and if the action succeeds, moves the temp file into 
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 3feabeb9ee..89485b7788 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -109,8 +109,7 @@ verifyLocationLog key desc = do
 	where
 		fix u s = do
 			showNote "fixing location log"
-			g <- gitRepo
-			logChange g key u s
+			logChange key u s
 
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
diff --git a/Logs/Location.hs b/Logs/Location.hs
index ff874a5964..ab29ffcadd 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -23,16 +23,13 @@ module Logs.Location (
 ) where
 
 import Common.Annex
-import qualified Git
 import qualified Annex.Branch
 import Logs.Presence
 
 {- Log a change in the presence of a key's value in a repository. -}
-logChange :: Git.Repo -> Key -> UUID -> LogStatus -> Annex ()
-logChange _ key (UUID u) s = addLog (logFile key) =<< logNow s u
-logChange repo _ NoUUID _ = error $
-	"unknown UUID for " ++ Git.repoDescribe repo ++ 
-	" (have you run git annex init there?)"
+logChange :: Key -> UUID -> LogStatus -> Annex ()
+logChange key (UUID u) s = addLog (logFile key) =<< logNow s u
+logChange _ NoUUID _ = return ()
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
diff --git a/Logs/Web.hs b/Logs/Web.hs
index b52e347e5f..62656b7ed8 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -42,12 +42,11 @@ getUrls key = do
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
 setUrl key url status = do
-	g <- gitRepo
 	addLog (urlLog key) =<< logNow status url
 
 	-- update location log to indicate that the web has the key, or not
 	us <- getUrls key
-	logChange g key webUUID (if null us then InfoMissing else InfoPresent)
+	logChange key webUUID (if null us then InfoMissing else InfoPresent)
 
 setUrlPresent :: Key -> URLString -> Annex ()
 setUrlPresent key url = setUrl key url InfoPresent
diff --git a/Remote.hs b/Remote.hs
index 82af21bde4..f820b62a18 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -231,9 +231,6 @@ forceTrust level remotename = do
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
 remoteHasKey :: Remote Annex -> Key -> Bool -> Annex ()
-remoteHasKey remote key present	= do
-	let remoteuuid = uuid remote
-	g <- gitRepo
-	logChange g key remoteuuid status
+remoteHasKey remote key present	= logChange key (uuid remote) status
 	where
 		status = if present then InfoPresent else InfoMissing
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 6cea170347..5bbf4169d7 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -19,10 +19,10 @@ import qualified Git
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	g <- gitRepo
-	return $ map construct $ remotepairs g
+	m <- fromRepo $ Git.configMap
+	return $ map construct $ remotepairs m
 	where
-		remotepairs r = M.toList $ M.filterWithKey match $ Git.configMap r
+		remotepairs = M.toList . M.filterWithKey match
 		construct (k,_) = Git.repoRemoteNameFromKey k Git.repoFromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 

From 393b6b1bde7e049845ff125f8a32e566d6bd7adb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 13:34:17 -0400
Subject: [PATCH 2475/8313] problem that came to me at 2 am

---
 doc/bugs/cyclic_drop.mdwn | 53 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 doc/bugs/cyclic_drop.mdwn

diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
new file mode 100644
index 0000000000..b43762e78f
--- /dev/null
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -0,0 +1,53 @@
+drop's verification that a remote still has content can fail
+if the remote is also dropping the content at the same time. Each
+repository checks that the other still has the content, and then both
+drop it. Could also happen with larger cycles of repositories.
+
+---
+
+Fixing this requires locking. (Well, there are other ways, like moving the
+content to a holding area when checking if it's safe to drop, but they
+seem complicated, and would be hard to implement for move --from.)
+
+Add per-content lock files. An exclusive lock is held on content when
+it's in the process of being dropped, or moved. The lock is taken
+nonblocking; if it cannot be obtained, something else is acting on the
+content and git-annex should refuse to do anything.
+
+Then when checking inannex, try to take a shared lock. Note that to avoid
+deadlock, this must be a nonblocking lock. If it fails, the status of
+the content is unknown, so inannex should fail. Note that this needs to be
+distinguishable from "not in annex".
+
+---
+
+move --to can also be included in the cycle, since it can drop data. 
+
+Consider move to a remote that already has the content and 
+is at the same time doing a drop (or a move). The remote
+verifies the content is present on the movee, and removes its copy.
+The movee removes its copy.
+
+So move --to needs to take the content lock on start. Then the inannex
+will fail.
+
+-- 
+
+move --from is similar. Consider a case where both the local and the remote
+are doing a move --from. Both have the content, and confirm the other does,
+via inannex checks. Then both run git-annex-shell dropkey, removing both
+copies.
+
+So move --from needs to take the content lock on start, so the inannex will
+fail.
+
+---
+
+Another cycle might be running move --to and move --from on the same file,
+locally. The exclusivity of the content lock solves this, as only one can
+run at a time.
+
+---
+
+Another cycle might involve move --from and drop, both run on the same
+file, locally. Again, the exclusive lock solves this.

From a243d6e6e9dcc657b5620244c3157da5c86406ec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 14:32:31 -0400
Subject: [PATCH 2476/8313] directly lock content?

---
 doc/bugs/cyclic_drop.mdwn | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index b43762e78f..cc2943b7ed 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -19,6 +19,23 @@ deadlock, this must be a nonblocking lock. If it fails, the status of
 the content is unknown, so inannex should fail. Note that this needs to be
 distinguishable from "not in annex".
 
+> Thinking about these lock files, this would be a lot more files, 
+> and would possibly break some assumptions that everything in
+> `.git/annex/objects` is a key's content. (Or would need lots more
+> directories to put the lock files elsewhere.) There would be more
+> overhead to manage these and have them on disk.
+> 
+> What if it just locked the actual content file? The obvious limitation
+> is only content that was already inannex could be locked, but that
+> happens to be exactly what's needed here; if content is not present,
+> it's not going to get dropped or moved.
+>
+> Of course, if some consumer of a file locked it, then it could prevent it
+> from being dropped or moved. This could be considered a bug, or a feature. :)
+> 
+> However, this would mean that such a hypothetical consumer could also
+> make inannex checks fail.
+
 ---
 
 move --to can also be included in the cycle, since it can drop data. 
@@ -39,7 +56,7 @@ via inannex checks. Then both run git-annex-shell dropkey, removing both
 copies.
 
 So move --from needs to take the content lock on start, so the inannex will
-fail.
+fail.  NB: If the content is not locally present, don't take the lock.
 
 ---
 

From 58563c5b1aa995ea3ce72cddaa1f02d2ea792c2d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 14:37:14 -0400
Subject: [PATCH 2477/8313] warning about version of git-annex shipped in
 recent Ubuntu release

---
 doc/install/Ubuntu.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/install/Ubuntu.mdwn b/doc/install/Ubuntu.mdwn
index afcf22b6d4..8d5341e19a 100644
--- a/doc/install/Ubuntu.mdwn
+++ b/doc/install/Ubuntu.mdwn
@@ -1,5 +1,11 @@
-If using Ubuntu natty or newer:
+If using Ubuntu Oneiric or newer:
 
 	sudo apt-get install git-annex
 
 Otherwise, see [[manual_installation_instructions|install]].
+
+---
+
+Warning: The version of git-annex shipped in Ubuntu Oneiric
+has [a bug that prevents upgrades from v1 git-annex repositories](https://bugs.launchpad.net/ubuntu/+source/git-annex/+bug/875958).
+If you need to upgrade such a repository, get a newer version of git-annex.

From 8ce7e73f74e95276472d18816b7c6a60bab25abb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 16:54:18 -0400
Subject: [PATCH 2478/8313] reorg to allow taking content lock

The lock will only persist during the perform stage, so the content must
be removed from the annex then, rather than in the cleanup stage.

(No lock is actually taken yet.)
---
 Annex/Content.hs          | 12 +++++++++++
 Command/Drop.hs           | 18 ++++++++--------
 Command/DropUnused.hs     |  3 ++-
 Command/Move.hs           | 43 +++++++++++++++++++++++----------------
 doc/bugs/cyclic_drop.mdwn |  4 ++++
 5 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index ecfec66aab..dc714276df 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -7,6 +7,8 @@
 
 module Annex.Content (
 	inAnnex,
+	lockExclusive,
+	lockShared,
 	calcGitLink,
 	logStatus,
 	getViaTmp,
@@ -41,6 +43,16 @@ inAnnex key = do
 		error "inAnnex cannot check remote repo"
 	inRepo $ doesFileExist . gitAnnexLocation key
 
+{- Content is exclusively locked to indicate that it's in the process of
+ - being removed. -}
+lockExclusive :: Key -> Annex a -> Annex a
+lockExclusive key a = a -- TODO
+
+{- Things that rely on content being present can take a shared lock to
+ - avoid it vanishing from under them. -}
+lockShared :: Key -> Annex a -> Annex a
+lockShared key a = a -- TODO
+
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 2267bd9416..e81bd9d7d6 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -52,17 +52,19 @@ startRemote file numcopies key remote = do
 	next $ performRemote key numcopies remote
 
 performLocal :: Key -> Maybe Int -> CommandPerform
-performLocal key numcopies = do
+performLocal key numcopies = lockExclusive key $ do
 	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
 	success <- canDropKey key numcopies trusteduuids tocheck []
 	if success
-		then next $ cleanupLocal key
+		then do
+			whenM (inAnnex key) $ removeAnnex key
+			next $ cleanupLocal key
 		else stop
 
 performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
-performRemote key numcopies remote = do
+performRemote key numcopies remote = lockExclusive key $ do
 	-- Filter the remote it's being dropped from out of the lists of
 	-- places assumed to have the key, and places to check.
 	-- When the local repo has the key, that's one additional copy.
@@ -76,20 +78,20 @@ performRemote key numcopies remote = do
 		Remote.remotesWithoutUUID remotes (have++untrusteduuids)
 	success <- canDropKey key numcopies have tocheck [uuid]
 	if success
-		then next $ cleanupRemote key remote
+		then do
+			ok <- Remote.removeKey remote key
+			next $ cleanupRemote key remote ok
 		else stop
 	where
 		uuid = Remote.uuid remote
 
 cleanupLocal :: Key -> CommandCleanup
 cleanupLocal key = do
-	whenM (inAnnex key) $ removeAnnex key
 	logStatus key InfoMissing
 	return True
 
-cleanupRemote :: Key -> Remote.Remote Annex -> CommandCleanup
-cleanupRemote key remote = do
-	ok <- Remote.removeKey remote key
+cleanupRemote :: Key -> Remote.Remote Annex -> Bool -> CommandCleanup
+cleanupRemote key remote ok = do
 	-- better safe than sorry: assume the remote dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 55c21f83bf..2c3bb296ad 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -55,7 +55,8 @@ perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
 		dropremote name = do
 			r <- Remote.byName name
 			showAction $ "from " ++ Remote.name r
-			next $ Command.Drop.cleanupRemote key r
+			ok <- Remote.removeKey r key
+			next $ Command.Drop.cleanupRemote key r ok
 		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
 
 performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform
diff --git a/Command/Move.hs b/Command/Move.hs
index 5a3ea7172a..e955de827b 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -68,7 +68,7 @@ toStart dest move file = isAnnexed file $ \(key, _) -> do
 			showMoveAction move file
 			next $ toPerform dest move key
 toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
-toPerform dest move key = do
+toPerform dest move key = moveLock move key $ do
 	-- Checking the remote is expensive, so not done in the start step.
 	-- In fast mode, location tracking is assumed to be correct,
 	-- and an explicit check is not done, when copying. When moving,
@@ -88,18 +88,20 @@ toPerform dest move key = do
 			showAction $ "to " ++ Remote.name dest
 			ok <- Remote.storeKey dest key
 			if ok
-				then next $ toCleanup dest move key
+				then finish
 				else do
 					when fastcheck $
 						warning "This could have failed because --fast is enabled."
 					stop
-		Right True -> next $ toCleanup dest move key
-toCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
-toCleanup dest move key = do
-	Remote.remoteHasKey dest key True
-	if move
-		then Command.Drop.cleanupLocal key
-		else return True
+		Right True -> finish
+	where
+		finish = do
+			Remote.remoteHasKey dest key True
+			if move
+				then do
+					whenM (inAnnex key) $ removeAnnex key
+					next $ Command.Drop.cleanupLocal key
+				else next $ return True
 
 {- Moves (or copies) the content of an annexed file from a remote
  - to the current repository.
@@ -117,16 +119,23 @@ fromStart src move file = isAnnexed file $ \(key, _) -> do
 			showMoveAction move file
 			next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
-fromPerform src move key = do
+fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key
 	if ishere
-		then next $ fromCleanup src move key
+		then handle move True
 		else do
 			showAction $ "from " ++ Remote.name src
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
-			if ok
-				then next $ fromCleanup src move key
-				else stop -- fail
-fromCleanup :: Remote.Remote Annex -> Bool -> Key -> CommandCleanup
-fromCleanup src True key = Command.Drop.cleanupRemote key src
-fromCleanup _ False _ = return True
+			handle move ok
+	where
+		handle _ False = stop -- failed
+		handle False True = next $ return True -- copy complete
+		handle True True = do -- finish moving
+			ok <- Remote.removeKey src key
+			next $ Command.Drop.cleanupRemote key src ok
+
+{- Locks a key in order for it to be moved.
+ - No lock is needed when a key is being copied. -}
+moveLock :: Bool -> Key -> Annex a -> Annex a
+moveLock True key a = lockExclusive key a
+moveLock False _ a = a
diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index cc2943b7ed..d3264c7caa 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -38,6 +38,10 @@ distinguishable from "not in annex".
 
 ---
 
+drop --from could also cycle. Locking should fix.
+
+---
+
 move --to can also be included in the cycle, since it can drop data. 
 
 Consider move to a remote that already has the content and 

From 2934a65ac5bbab5ac127c495c8c2492e729c2b67 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 17:28:35 -0400
Subject: [PATCH 2479/8313] add safeSystem

This is more safe than System.Cmd.Utils.safeSystem, since it does not throw
an error on nonzero exit status.
---
 Common.hs              |  2 +-
 Utility/SafeCommand.hs | 20 +++++++++++++++-----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/Common.hs b/Common.hs
index 2e1e4d996c..ef068ac105 100644
--- a/Common.hs
+++ b/Common.hs
@@ -33,7 +33,7 @@ import Data.String.Utils
 import System.Path
 import System.FilePath
 import System.Directory
-import System.Cmd.Utils
+import System.Cmd.Utils hiding (safeSystem)
 import System.IO hiding (FilePath)
 import System.Posix.Files
 import System.Posix.IO
diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs
index ba9362603e..aedf271373 100644
--- a/Utility/SafeCommand.hs
+++ b/Utility/SafeCommand.hs
@@ -13,6 +13,7 @@ import System.Posix.Process hiding (executeFile)
 import System.Posix.Signals
 import Data.String.Utils
 import System.Log.Logger
+import Control.Applicative
 
 {- A type for parameters passed to a shell command. A command can
  - be passed either some Params (multiple parameters can be included,
@@ -36,14 +37,23 @@ toCommand = (>>= unwrap)
 
 {- Run a system command, and returns True or False
  - if it succeeded or failed.
- -
- - SIGINT(ctrl-c) is allowed to propigate and will terminate the program.
  -}
 boolSystem :: FilePath -> [CommandParam] -> IO Bool
 boolSystem command params = boolSystemEnv command params Nothing
 
 boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool
-boolSystemEnv command params env = do
+boolSystemEnv command params env = dispatch <$> safeSystemEnv command params env
+	where
+		dispatch ExitSuccess = True
+		dispatch _ = False
+
+{- Runs a system command, returning the exit status. -}
+safeSystem :: FilePath -> [CommandParam] -> IO ExitCode
+safeSystem command params = safeSystemEnv command params Nothing
+
+{- SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -}
+safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode
+safeSystemEnv command params env = do
 	-- Going low-level because all the high-level system functions
 	-- block SIGINT etc. We need to block SIGCHLD, but allow
 	-- SIGINT to do its default program termination.
@@ -55,8 +65,8 @@ boolSystemEnv command params env = do
 	mps <- getProcessStatus True False childpid
 	restoresignals oldint oldset
 	case mps of
-		Just (Exited ExitSuccess) -> return True
-		_ -> return False
+		Just (Exited code) -> return code
+		_ -> error $ "unknown error running " ++ command
 	where
 		restoresignals oldint oldset = do
 			_ <- installHandler sigINT oldint Nothing

From d3e1a3619ff6939367f43cbd46131b7f60ef6bd0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 18:33:15 -0400
Subject: [PATCH 2480/8313] safer inannex checking

git-annex-shell inannex now returns always 0, 1, or 100 (the last when
it's unclear if content is currently in the index due to it currently being
moved or dropped).

(Actual locking code still not yet written.)
---
 Annex/Content.hs          | 36 ++++++++++++++++++++++++------------
 Command/Drop.hs           |  6 +++---
 Command/InAnnex.hs        | 11 ++++++-----
 Command/Move.hs           |  6 +++---
 Remote.hs                 |  6 +++---
 Remote/Bup.hs             |  8 ++++++--
 Remote/Directory.hs       |  8 ++++++--
 Remote/Git.hs             | 36 ++++++++++++++++++++++++++----------
 Remote/Hook.hs            |  6 ++++--
 Remote/Rsync.hs           |  2 +-
 Remote/S3real.hs          |  2 +-
 Remote/Web.hs             |  2 +-
 Types/Remote.hs           |  5 ++---
 doc/bugs/cyclic_drop.mdwn |  9 +++++++--
 14 files changed, 93 insertions(+), 50 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index dc714276df..efe12bb5d7 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -7,8 +7,8 @@
 
 module Annex.Content (
 	inAnnex,
-	lockExclusive,
-	lockShared,
+	inAnnexSafe,
+	lockContent,
 	calcGitLink,
 	logStatus,
 	getViaTmp,
@@ -36,22 +36,34 @@ import Types.Key
 import Utility.DataUnits
 import Config
 
-{- Checks if a given key is currently present in the gitAnnexLocation. -}
+{- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
-inAnnex key = do
+inAnnex = inAnnex' doesFileExist
+inAnnex' :: (FilePath -> IO a) -> Key -> Annex a
+inAnnex' a key = do
 	whenM (fromRepo Git.repoIsUrl) $
 		error "inAnnex cannot check remote repo"
-	inRepo $ doesFileExist . gitAnnexLocation key
+	inRepo $ a . gitAnnexLocation key
+
+{- A safer check; the key's content must not only be present, but
+ - is not in the process of being removed. -}
+inAnnexSafe :: Key -> Annex (Maybe Bool)
+inAnnexSafe = inAnnex' $ \f -> do
+	e <- doesFileExist f
+	if e
+		then do
+			locked <- testlock f
+			if locked
+				then return Nothing
+				else return $ Just True
+		else return $ Just False
+	where
+		testlock f = return False -- TODO
 
 {- Content is exclusively locked to indicate that it's in the process of
  - being removed. -}
-lockExclusive :: Key -> Annex a -> Annex a
-lockExclusive key a = a -- TODO
-
-{- Things that rely on content being present can take a shared lock to
- - avoid it vanishing from under them. -}
-lockShared :: Key -> Annex a -> Annex a
-lockShared key a = a -- TODO
+lockContent :: Key -> Annex a -> Annex a
+lockContent key a = a -- TODO
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/Command/Drop.hs b/Command/Drop.hs
index e81bd9d7d6..44685ffcd6 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -52,7 +52,7 @@ startRemote file numcopies key remote = do
 	next $ performRemote key numcopies remote
 
 performLocal :: Key -> Maybe Int -> CommandPerform
-performLocal key numcopies = lockExclusive key $ do
+performLocal key numcopies = lockContent key $ do
 	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
@@ -64,7 +64,7 @@ performLocal key numcopies = lockExclusive key $ do
 		else stop
 
 performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
-performRemote key numcopies remote = lockExclusive key $ do
+performRemote key numcopies remote = lockContent key $ do
 	-- Filter the remote it's being dropped from out of the lists of
 	-- places assumed to have the key, and places to check.
 	-- When the local repo has the key, that's one additional copy.
@@ -95,7 +95,7 @@ cleanupRemote key remote ok = do
 	-- better safe than sorry: assume the remote dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
-	Remote.remoteHasKey remote key False
+	Remote.logStatus remote key False
 	return ok
 
 {- Checks specified remotes to verify that enough copies of a key exist to
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index 9c169d0d78..c41f9a92c1 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -19,8 +19,9 @@ seek :: [CommandSeek]
 seek = [withKeys start]
 
 start :: Key -> CommandStart
-start key = do
-	present <- inAnnex key
-	if present
-		then stop
-		else liftIO exitFailure
+start key = inAnnexSafe key >>= dispatch
+	where
+		dispatch (Just True) = stop
+		dispatch (Just False) = exit 1
+		dispatch Nothing = exit 100
+		exit n = liftIO $ exitWith $ ExitFailure n
diff --git a/Command/Move.hs b/Command/Move.hs
index e955de827b..f02f325580 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -82,7 +82,7 @@ toPerform dest move key = moveLock move key $ do
 		else Remote.hasKey dest key
 	case isthere of
 		Left err -> do
-			showNote $ show err
+			showNote $ err
 			stop
 		Right False -> do
 			showAction $ "to " ++ Remote.name dest
@@ -96,7 +96,7 @@ toPerform dest move key = moveLock move key $ do
 		Right True -> finish
 	where
 		finish = do
-			Remote.remoteHasKey dest key True
+			Remote.logStatus dest key True
 			if move
 				then do
 					whenM (inAnnex key) $ removeAnnex key
@@ -137,5 +137,5 @@ fromPerform src move key = moveLock move key $ do
 {- Locks a key in order for it to be moved.
  - No lock is needed when a key is being copied. -}
 moveLock :: Bool -> Key -> Annex a -> Annex a
-moveLock True key a = lockExclusive key a
+moveLock True key a = lockContent key a
 moveLock False _ a = a
diff --git a/Remote.hs b/Remote.hs
index f820b62a18..7c0362d2a7 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -26,7 +26,7 @@ module Remote (
 	showTriedRemotes,
 	showLocations,
 	forceTrust,
-	remoteHasKey
+	logStatus
 ) where
 
 import qualified Data.Map as M
@@ -230,7 +230,7 @@ forceTrust level remotename = do
  - in the local repo, not on the remote. The process of transferring the
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
-remoteHasKey :: Remote Annex -> Key -> Bool -> Annex ()
-remoteHasKey remote key present	= logChange key (uuid remote) status
+logStatus :: Remote Annex -> Key -> Bool -> Annex ()
+logStatus remote key present = logChange key (uuid remote) status
 	where
 		status = if present then InfoPresent else InfoMissing
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index b8d7cd3172..866d4b42de 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -139,17 +139,21 @@ remove _ = do
  - in a bup repository. One way it to check if the git repository has
  - a branch matching the name (as created by bup split -n).
  -}
-checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either IOException Bool)
+checkPresent :: Git.Repo -> Git.Repo -> Key -> Annex (Either String Bool)
 checkPresent r bupr k
 	| Git.repoIsUrl bupr = do
 		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
-	| otherwise = liftIO $ try $ boolSystem "git" $ Git.gitCommandLine params bupr
+	| otherwise = dispatch <$> localcheck
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
 			, Param $ "refs/heads/" ++ show k]
+		localcheck = liftIO $ try $
+			boolSystem "git" $ Git.gitCommandLine params bupr
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
 
 {- Store UUID in the annex.uuid setting of the bup repository. -}
 storeBupUUID :: UUID -> BupRepo -> Annex ()
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 8e306e228b..6d3a5da7d0 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -114,5 +114,9 @@ remove d k = liftIO $ catchBool $ do
 		file = dirKey d k
 		dir = parentDir file
 
-checkPresent :: FilePath -> Key -> Annex (Either IOException Bool)
-checkPresent d k = liftIO $ try $ doesFileExist (dirKey d k)
+checkPresent :: FilePath -> Key -> Annex (Either String Bool)
+checkPresent d k = dispatch <$> check
+	where
+		check = liftIO $ try $ doesFileExist (dirKey d k)
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 75f0ac7576..b63a8f124d 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -125,22 +125,38 @@ tryGitConfigRead r
 				else old : exchange ls new
 
 {- Checks if a given remote has the content for a key inAnnex.
- - If the remote cannot be accessed, returns a Left error.
+ - If the remote cannot be accessed, or if it cannot determine
+ - whether it has the content, returns a Left error message.
  -}
-inAnnex :: Git.Repo -> Key -> Annex (Either IOException Bool)
+inAnnex :: Git.Repo -> Key -> Annex (Either String Bool)
 inAnnex r key
-	| Git.repoIsHttp r = safely checkhttp
+	| Git.repoIsHttp r = checkhttp
 	| Git.repoIsUrl r = checkremote
-	| otherwise = safely checklocal
+	| otherwise = checklocal
 	where
-		checklocal = onLocal r $ Annex.Content.inAnnex key
+		checkhttp = dispatch <$> check
+			where
+				check = safely $ Url.exists $ keyUrl r key
+				dispatch (Left e) = Left $ show e
+				dispatch (Right v) = Right v
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
-			inannex <- onRemote r (boolSystem, False) "inannex" 
-				[Param (show key)]
-			return $ Right inannex
-		checkhttp = Url.exists $ keyUrl r key
-		safely a = liftIO (try a ::IO (Either IOException Bool))
+			onRemote r (check, unknown) "inannex" [Param (show key)]
+			where
+				check c p = dispatch <$> safeSystem c p
+				dispatch ExitSuccess = Right True
+				dispatch (ExitFailure 1) = Right False
+				dispatch _ = unknown
+		checklocal = dispatch <$> check
+			where
+				check = safely $ onLocal r $
+					Annex.Content.inAnnexSafe key
+				dispatch (Left e) = Left $ show e
+				dispatch (Right (Just b)) = Right b
+				dispatch (Right Nothing) = unknown
+		safely :: IO a -> Annex (Either IOException a)
+		safely a = liftIO $ try a
+		unknown = Left $ "unable to check " ++ Git.repoDescribe r
 
 {- Runs an action on a local repository inexpensively, by making an annex
  - monad using that repository. -}
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 06568a3cb5..9f9250e41d 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -119,14 +119,16 @@ retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
 remove :: String -> Key -> Annex Bool
 remove h k = runHook h "remove" k Nothing $ return True
 
-checkPresent :: Git.Repo -> String -> Key -> Annex (Either IOException Bool)
+checkPresent :: Git.Repo -> String -> Key -> Annex (Either String Bool)
 checkPresent r h k = do
 	showAction $ "checking " ++ Git.repoDescribe r
 	v <- lookupHook h "checkpresent"
-	liftIO (try (check v) ::IO (Either IOException Bool))
+	dispatch <$> liftIO (try (check v) ::IO (Either IOException Bool))
 	where
 		findkey s = show k `elem` lines s
 		env = hookEnv k Nothing
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
 			(frompipe, topipe) <- createPipe
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 0dfad72935..54834be135 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -128,7 +128,7 @@ remove o k = withRsyncScratchDir $ \tmp -> do
 		, Param $ rsyncKeyDir o k
 		]
 
-checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either IOException Bool)
+checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
 checkPresent r o k = do
 	showAction $ "checking " ++ Git.repoDescribe r
 	-- note: Does not currently differnetiate between rsync failing
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b201b5aadb..29117b3a41 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -172,7 +172,7 @@ remove r k = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey r bucket k
 	s3Bool res
 
-checkPresent :: Remote Annex -> Key -> Annex (Either IOException Bool)
+checkPresent :: Remote Annex -> Key -> Annex (Either String Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showAction $ "checking " ++ name r
 	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
diff --git a/Remote/Web.hs b/Remote/Web.hs
index da7f384727..64fcd51aaa 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -64,7 +64,7 @@ dropKey _ = do
 	warning "removal from web not supported"
 	return False
 
-checkKey :: Key -> Annex (Either IOException Bool)
+checkKey :: Key -> Annex (Either String Bool)
 checkKey key = do
 	us <- getUrls key
 	if null us
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 0a4a0fa883..ec9b7a7a70 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -9,7 +9,6 @@
 
 module Types.Remote where
 
-import Control.Exception
 import Data.Map as M
 import Data.Ord
 
@@ -46,8 +45,8 @@ data Remote a = Remote {
 	-- removes a key's contents
 	removeKey :: Key -> a Bool,
 	-- Checks if a key is present in the remote; if the remote
-	-- cannot be accessed returns a Left error.
-	hasKey :: Key -> a (Either IOException Bool),
+	-- cannot be accessed returns a Left error message.
+	hasKey :: Key -> a (Either String Bool),
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool,
diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index d3264c7caa..7804380ae0 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -16,8 +16,8 @@ content and git-annex should refuse to do anything.
 
 Then when checking inannex, try to take a shared lock. Note that to avoid
 deadlock, this must be a nonblocking lock. If it fails, the status of
-the content is unknown, so inannex should fail. Note that this needs to be
-distinguishable from "not in annex".
+the content is unknown, so inannex should fail. Note that this failure 
+needs to be distinguishable from "not in annex".
 
 > Thinking about these lock files, this would be a lot more files, 
 > and would possibly break some assumptions that everything in
@@ -52,6 +52,11 @@ The movee removes its copy.
 So move --to needs to take the content lock on start. Then the inannex
 will fail.
 
+This is why it's important for inannex to fail in a way that is
+distinguishable from "not in annex". Otherwise, move --to
+would see the cycle as the remote not having content, and try to
+redundantly send it, drop it locally, and still race.
+
 -- 
 
 move --from is similar. Consider a case where both the local and the remote

From 992bf133823b6884e9fb1b320fd33d3fb337ee30 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 19:47:04 -0400
Subject: [PATCH 2481/8313] lockContent in dropkey

This is needed for drop --from and move --from to check the lock,
as they do not use git-annex-shell inannex.
---
 Command/DropKey.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index cac955b4ac..b63d481bf3 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -35,7 +35,7 @@ start key = do
 				error "dropkey can cause data loss; use --force if you're sure you want to do this"
 
 perform :: Key -> CommandPerform
-perform key = do
+perform key = lockContent key $ do
 	removeAnnex key
 	next $ cleanup key
 

From 737f043c55b13bf8dbd6887d3e78d32d13a8682a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 21:36:11 -0400
Subject: [PATCH 2482/8313] fast build mode for vim

---
 Makefile | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 73e4260008..f0f5360b42 100644
--- a/Makefile
+++ b/Makefile
@@ -1,16 +1,26 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
 GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=5
+
 ifdef PROFILE
 GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)
 endif
+
 GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
 sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
-all: $(bins) $(mans) docs
+all=$(bins) $(mans) docs
+
+# Am I typing :make in vim? Do a fast build without optimisation.
+ifdef VIM
+GHCFLAGS=-Wall $(IGNORE)
+all=$(bins)
+endif
+
+all: $(all)
 
 Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure

From cf0174c922e4a4f473a846ec0488ea4011ab500c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 21:45:03 -0400
Subject: [PATCH 2483/8313] content locking

I've tested that this solves the cyclic drop problem.
Have not looked at cyclic move, etc.
---
 Annex/Content.hs | 52 ++++++++++++++++++++++++++++++++++++------------
 debian/changelog |  1 +
 2 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index efe12bb5d7..65dbe43f6d 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,6 +23,9 @@ module Annex.Content (
 	saveState
 ) where
 
+import Control.Exception (bracket_)
+import System.Posix.Types
+
 import Common.Annex
 import Logs.Location
 import Annex.UUID
@@ -35,6 +38,7 @@ import Utility.FileMode
 import Types.Key
 import Utility.DataUnits
 import Config
+import Annex.Exception
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
@@ -48,22 +52,44 @@ inAnnex' a key = do
 {- A safer check; the key's content must not only be present, but
  - is not in the process of being removed. -}
 inAnnexSafe :: Key -> Annex (Maybe Bool)
-inAnnexSafe = inAnnex' $ \f -> do
-	e <- doesFileExist f
-	if e
-		then do
-			locked <- testlock f
-			if locked
-				then return Nothing
-				else return $ Just True
-		else return $ Just False
+inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
 	where
-		testlock f = return False -- TODO
+		check Nothing = return is_missing
+		check (Just h) = do
+			v <- getLock h (ReadLock, AbsoluteSeek, 0, 0)
+			closeFd h
+			return $ case v of
+				Just _ -> is_locked
+				Nothing -> is_unlocked
+		is_locked = Nothing
+		is_unlocked = Just True
+		is_missing = Just False
 
-{- Content is exclusively locked to indicate that it's in the process of
- - being removed. -}
+{- Content is exclusively locked to indicate that it's in the process
+ - of being removed. (If the content is not present, no locking is done.) -}
 lockContent :: Key -> Annex a -> Annex a
-lockContent key a = a -- TODO
+lockContent key a = do
+	file <- fromRepo $ gitAnnexLocation key
+	bracketIO (openForLock file True >>= lock) unlock a
+	where
+		lock Nothing = return Nothing
+		lock (Just l) = do
+			setLock l (WriteLock, AbsoluteSeek, 0, 0)
+			return $ Just l
+		unlock Nothing = return ()
+		unlock (Just l) = closeFd l
+
+openForLock :: FilePath -> Bool -> IO (Maybe Fd)
+openForLock file writelock = bracket_ prep cleanup $
+	catch (Just <$> openFd file mode Nothing defaultFileFlags)
+		(const $ return Nothing)
+	where
+		mode = if writelock then ReadWrite else ReadOnly
+		-- Since files are stored with the write bit disabled,
+		-- have to fiddle with permissions to open for an
+		-- exclusive lock.
+		prep = when writelock $ allowWrite file
+		cleanup = when writelock $ preventWrite file
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/debian/changelog b/debian/changelog
index 265a010c45..9fa96e06ab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111108) UNRELEASED; urgency=low
 
   * Handle a case where an annexed file is moved into a gitignored directory,
     by having fix --force add its change.
+  * Avoid cyclic drop problems.
 
  -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
 

From a218ce41cfef19f306ca462fb5d57c6647a680e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 9 Nov 2011 22:15:33 -0400
Subject: [PATCH 2484/8313] exclusive locks, ugh

---
 Annex/Content.hs          | 17 ++++++++++-------
 doc/bugs/cyclic_drop.mdwn |  9 +++++++++
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 65dbe43f6d..b111bfabc6 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -65,8 +65,8 @@ inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
 		is_unlocked = Just True
 		is_missing = Just False
 
-{- Content is exclusively locked to indicate that it's in the process
- - of being removed. (If the content is not present, no locking is done.) -}
+{- Content is exclusively locked while running an action that might remove
+ - it. (If the content is not present, no locking is done.) -}
 lockContent :: Key -> Annex a -> Annex a
 lockContent key a = do
 	file <- fromRepo $ gitAnnexLocation key
@@ -85,11 +85,14 @@ openForLock file writelock = bracket_ prep cleanup $
 		(const $ return Nothing)
 	where
 		mode = if writelock then ReadWrite else ReadOnly
-		-- Since files are stored with the write bit disabled,
-		-- have to fiddle with permissions to open for an
-		-- exclusive lock.
-		prep = when writelock $ allowWrite file
-		cleanup = when writelock $ preventWrite file
+		{- Since files are stored with the write bit disabled,
+		 - have to fiddle with permissions to open for an
+		 - exclusive lock. flock locking would avoid this,
+		 - but -}
+		prep = forwritelock $ allowWrite file
+		cleanup = forwritelock $ preventWrite file
+		forwritelock a = 
+			when writelock $ whenM (doesFileExist file) $ a
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index 7804380ae0..beaaa6e35b 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -73,6 +73,15 @@ Another cycle might be running move --to and move --from on the same file,
 locally. The exclusivity of the content lock solves this, as only one can
 run at a time.
 
+Would it work with a shared lock? The --to would run git-annex-shell
+inannex. The --from would also be running, and would run git-annex-shell
+dropkey. So inannex and dropkey would end up running on the remote
+at the same time. Dropkey takes the content lock, and inannex checks it,
+but what if inannex runs first? Then it returns true, and then the content
+is removed, so both the --to and --from see success and the --to proceeds
+to remove the local content that the --from already caused to be removed
+from the remote. So, no, the nasty exclusive lock is needed.
+
 ---
 
 Another cycle might involve move --from and drop, both run on the same

From 95704212510b525bf4fb3c76fc37043da26cb825 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 02:59:13 -0400
Subject: [PATCH 2485/8313] better message when content is locked

---
 Annex/Content.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index b111bfabc6..f50616af93 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,6 +23,7 @@ module Annex.Content (
 	saveState
 ) where
 
+import System.IO.Error (try)
 import Control.Exception (bracket_)
 import System.Posix.Types
 
@@ -74,8 +75,10 @@ lockContent key a = do
 	where
 		lock Nothing = return Nothing
 		lock (Just l) = do
-			setLock l (WriteLock, AbsoluteSeek, 0, 0)
-			return $ Just l
+			v <- try $ setLock l (WriteLock, AbsoluteSeek, 0, 0)
+			case v of
+				Left _ -> error "content is locked"
+				Right _ -> return $ Just l
 		unlock Nothing = return ()
 		unlock (Just l) = closeFd l
 

From e4105df78a9a12eaee7d1c00758854186c818931 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 02:59:41 -0400
Subject: [PATCH 2486/8313] tested all known types of cycles, all are fixed

My testing involved widening the race by adding sleeps, and making sure
something sane happens in each case.
---
 doc/bugs/cyclic_drop.mdwn | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/doc/bugs/cyclic_drop.mdwn b/doc/bugs/cyclic_drop.mdwn
index beaaa6e35b..296d61aac5 100644
--- a/doc/bugs/cyclic_drop.mdwn
+++ b/doc/bugs/cyclic_drop.mdwn
@@ -3,6 +3,8 @@ if the remote is also dropping the content at the same time. Each
 repository checks that the other still has the content, and then both
 drop it. Could also happen with larger cycles of repositories.
 
+> Confirmed fixed now. All cases tested. [[done]]
+
 ---
 
 Fixing this requires locking. (Well, there are other ways, like moving the
@@ -15,9 +17,10 @@ nonblocking; if it cannot be obtained, something else is acting on the
 content and git-annex should refuse to do anything.
 
 Then when checking inannex, try to take a shared lock. Note that to avoid
-deadlock, this must be a nonblocking lock. If it fails, the status of
-the content is unknown, so inannex should fail. Note that this failure 
-needs to be distinguishable from "not in annex".
+deadlock, this must be a nonblocking lock. (Actually, with fcntl locking,
+can just check if there is a lock, without needing to take one.) 
+If it fails, the status of the content is unknown, so inannex should fail.
+Note that this failure needs to be distinguishable from "not in annex".
 
 > Thinking about these lock files, this would be a lot more files, 
 > and would possibly break some assumptions that everything in
@@ -35,11 +38,18 @@ needs to be distinguishable from "not in annex".
 > 
 > However, this would mean that such a hypothetical consumer could also
 > make inannex checks fail.
+>
+> The other downside is that, for fcntl exclusive locking, the file has to
+> be opened for write. Normally the modes of content files are locked down
+> to prevent modifcation. Dealt with, but oh so nasty. Almost makes flock
+> locking seem worth using.
 
 ---
 
 drop --from could also cycle. Locking should fix.
 
+> Confirmed fixed now. 
+
 ---
 
 move --to can also be included in the cycle, since it can drop data. 
@@ -57,6 +67,8 @@ distinguishable from "not in annex". Otherwise, move --to
 would see the cycle as the remote not having content, and try to
 redundantly send it, drop it locally, and still race.
 
+> Confirmed fixed now.
+
 -- 
 
 move --from is similar. Consider a case where both the local and the remote
@@ -67,6 +79,8 @@ copies.
 So move --from needs to take the content lock on start, so the inannex will
 fail.  NB: If the content is not locally present, don't take the lock.
 
+> Confirmed fixed now.
+
 ---
 
 Another cycle might be running move --to and move --from on the same file,
@@ -82,6 +96,8 @@ is removed, so both the --to and --from see success and the --to proceeds
 to remove the local content that the --from already caused to be removed
 from the remote. So, no, the nasty exclusive lock is needed.
 
+> Confirmed fixed now.
+
 ---
 
 Another cycle might involve move --from and drop, both run on the same

From a71c03bc5162916853ff520d5c7c89e849c6a047 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 03:10:17 -0400
Subject: [PATCH 2487/8313] add make fast target

---
 Makefile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index f0f5360b42..cef79be88c 100644
--- a/Makefile
+++ b/Makefile
@@ -14,14 +14,17 @@ sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
 
 all=$(bins) $(mans) docs
 
-# Am I typing :make in vim? Do a fast build without optimisation.
+# Am I typing :make in vim? Do a fast build.
 ifdef VIM
-GHCFLAGS=-Wall $(IGNORE)
-all=$(bins)
+all=fast
 endif
 
 all: $(all)
 
+# Disables optimisation. Not for production use.
+fast: GHCFLAGS=-Wall $(IGNORE)
+fast: $(bins)
+
 Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	$(GHCMAKE) configure
 	./configure

From 49d2177d51b95b4a01c05ee07e166e93751b4c51 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 20:24:24 -0400
Subject: [PATCH 2488/8313] factored out some useful error catching methods

---
 Annex/Branch.hs     | 10 ++++------
 Annex/Content.hs    | 12 +++++-------
 Crypto.hs           |  2 +-
 Git.hs              |  2 +-
 Remote/Bup.hs       | 13 +++++--------
 Remote/Directory.hs | 15 +++++----------
 Remote/Git.hs       | 12 +++---------
 Remote/Hook.hs      |  7 ++-----
 Remote/Rsync.hs     |  2 +-
 Remote/S3real.hs    |  2 +-
 Upgrade/V1.hs       |  2 +-
 Upgrade/V2.hs       |  2 +-
 Utility/Misc.hs     | 21 +++++++++++++++++++--
 Utility/TempFile.hs |  4 ++--
 git-annex-shell.hs  |  9 +++------
 15 files changed, 54 insertions(+), 61 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 189289ad39..6c28a0c84a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -295,10 +295,8 @@ setJournalFile file content = do
 
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
-getJournalFile file = do
-	g <- gitRepo
-	liftIO $ catch (liftM Just . readFileStrict $ journalFile g file)
-		(const $ return Nothing)
+getJournalFile file = inRepo $ \g -> catchMaybeIO $
+	readFileStrict $ journalFile g file
 
 {- List of files that have updated content in the journal. -}
 getJournalledFiles :: Annex [FilePath]
@@ -308,8 +306,8 @@ getJournalledFiles = map fileJournal <$> getJournalFiles
 getJournalFiles :: Annex [FilePath]
 getJournalFiles = do
 	g <- gitRepo
-	fs <- liftIO $ catch (getDirectoryContents $ gitAnnexJournalDir g)
-		(const $ return [])
+	fs <- liftIO $
+		catchDefaultIO (getDirectoryContents $ gitAnnexJournalDir g) []
 	return $ filter (`notElem` [".", ".."]) fs
 
 {- Stages the specified journalfiles. -}
diff --git a/Annex/Content.hs b/Annex/Content.hs
index f50616af93..7586bb96f6 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -83,19 +83,17 @@ lockContent key a = do
 		unlock (Just l) = closeFd l
 
 openForLock :: FilePath -> Bool -> IO (Maybe Fd)
-openForLock file writelock = bracket_ prep cleanup $
-	catch (Just <$> openFd file mode Nothing defaultFileFlags)
-		(const $ return Nothing)
+openForLock file writelock = bracket_ prep cleanup go
 	where
+		go = catchMaybeIO $ openFd file mode Nothing defaultFileFlags
 		mode = if writelock then ReadWrite else ReadOnly
 		{- Since files are stored with the write bit disabled,
 		 - have to fiddle with permissions to open for an
-		 - exclusive lock. flock locking would avoid this,
-		 - but -}
-		prep = forwritelock $ allowWrite file
-		cleanup = forwritelock $ preventWrite file
+		 - exclusive lock. -}
 		forwritelock a = 
 			when writelock $ whenM (doesFileExist file) $ a
+		prep = forwritelock $ allowWrite file
+		cleanup = forwritelock $ preventWrite file
 
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
diff --git a/Crypto.hs b/Crypto.hs
index b3acb30a6e..24bb79ba0d 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -173,7 +173,7 @@ gpgParams :: [CommandParam] -> IO [String]
 gpgParams params = do
 	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
 	-- gpg output about password prompts.
-	e <- catch (getEnv "GPG_AGENT_INFO") (const $ return "")
+	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
 	let batch = if null e then [] else ["--batch"]
 	return $ batch ++ defaults ++ toCommand params
 	where
diff --git a/Git.hs b/Git.hs
index 6fb6e8361c..5ceaa67f76 100644
--- a/Git.hs
+++ b/Git.hs
@@ -414,7 +414,7 @@ pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
 reap :: IO ()
 reap = do
 	-- throws an exception when there are no child processes
-	r <- catch (getAnyProcessStatus False True) (\_ -> return Nothing)
+	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
 	maybe (return ()) (const reap) r
 
 {- Forces git to use the specified index file.
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 866d4b42de..4c826498da 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -110,21 +110,21 @@ storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
 	src <- fromRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck (Param "-")
-	liftIO $ catchBool $
+	liftIO $ catchBoolIO $
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
 retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
 retrieve buprepo k f = do
 	let params = bupParams "join" buprepo [Param $ show k]
-	liftIO $ catchBool $ do
+	liftIO $ catchBoolIO $ do
 		tofile <- openFile f WriteMode
 		pipeBup params Nothing (Just tofile)
 
 retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted buprepo (cipher, enck) f = do
 	let params = bupParams "join" buprepo [Param $ show enck]
-	liftIO $ catchBool $ do
+	liftIO $ catchBoolIO $ do
 		(pid, h) <- hPipeFrom "bup" $ toCommand params
 		withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f
 		forceSuccess pid
@@ -145,15 +145,12 @@ checkPresent r bupr k
 		showAction $ "checking " ++ Git.repoDescribe r
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
-	| otherwise = dispatch <$> localcheck
+	| otherwise = liftIO $ catchMsgIO $
+		boolSystem "git" $ Git.gitCommandLine params bupr
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
 			, Param $ "refs/heads/" ++ show k]
-		localcheck = liftIO $ try $
-			boolSystem "git" $ Git.gitCommandLine params bupr
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
 
 {- Store UUID in the annex.uuid setting of the bup repository. -}
 storeBupUUID :: UUID -> BupRepo -> Annex ()
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 6d3a5da7d0..b592f41ff0 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -8,7 +8,6 @@
 module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import System.IO.Error
 import qualified Data.Map as M
 
 import Common.Annex
@@ -72,13 +71,13 @@ store :: FilePath -> Key -> Annex Bool
 store d k = do
 	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d k
-	liftIO $ catchBool $ storeHelper dest $ copyFileExternal src dest
+	liftIO $ catchBoolIO $ storeHelper dest $ copyFileExternal src dest
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
 	src <- fromRepo $ gitAnnexLocation k
 	let dest = dirKey d enck
-	liftIO $ catchBool $ storeHelper dest $ encrypt src dest
+	liftIO $ catchBoolIO $ storeHelper dest $ encrypt src dest
 	where
 		encrypt src dest = do
 			withEncryptedContent cipher (L.readFile src) $ L.writeFile dest
@@ -100,12 +99,12 @@ retrieve d k f = liftIO $ copyFileExternal (dirKey d k) f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
-	liftIO $ catchBool $ do
+	liftIO $ catchBoolIO $ do
 		withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f
 		return True
 
 remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ catchBool $ do
+remove d k = liftIO $ catchBoolIO $ do
 	allowWrite dir
 	removeFile file
 	removeDirectory dir
@@ -115,8 +114,4 @@ remove d k = liftIO $ catchBool $ do
 		dir = parentDir file
 
 checkPresent :: FilePath -> Key -> Annex (Either String Bool)
-checkPresent d k = dispatch <$> check
-	where
-		check = liftIO $ try $ doesFileExist (dirKey d k)
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
+checkPresent d k = liftIO $ catchMsgIO $ doesFileExist (dirKey d k)
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b63a8f124d..30d992e8ca 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -134,11 +134,7 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = checklocal
 	where
-		checkhttp = dispatch <$> check
-			where
-				check = safely $ Url.exists $ keyUrl r key
-				dispatch (Left e) = Left $ show e
-				dispatch (Right v) = Right v
+		checkhttp = liftIO $ catchMsgIO $ Url.exists $ keyUrl r key
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			onRemote r (check, unknown) "inannex" [Param (show key)]
@@ -149,13 +145,11 @@ inAnnex r key
 				dispatch _ = unknown
 		checklocal = dispatch <$> check
 			where
-				check = safely $ onLocal r $
+				check = liftIO $ catchMsgIO $ onLocal r $
 					Annex.Content.inAnnexSafe key
-				dispatch (Left e) = Left $ show e
+				dispatch (Left e) = Left e
 				dispatch (Right (Just b)) = Right b
 				dispatch (Right Nothing) = unknown
-		safely :: IO a -> Annex (Either IOException a)
-		safely a = liftIO $ try a
 		unknown = Left $ "unable to check " ++ Git.repoDescribe r
 
 {- Runs an action on a local repository inexpensively, by making an annex
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 9f9250e41d..03976fc709 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -9,7 +9,6 @@ module Remote.Hook (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
-import System.IO.Error (try)
 import System.Exit
 
 import Common.Annex
@@ -112,7 +111,7 @@ retrieve h k f = runHook h "retrieve" k (Just f) $ return True
 
 retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
-	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBool $ do
+	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBoolIO $ do
 		withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
 		return True
 
@@ -123,12 +122,10 @@ checkPresent :: Git.Repo -> String -> Key -> Annex (Either String Bool)
 checkPresent r h k = do
 	showAction $ "checking " ++ Git.repoDescribe r
 	v <- lookupHook h "checkpresent"
-	dispatch <$> liftIO (try (check v) ::IO (Either IOException Bool))
+	liftIO $ catchMsgIO $ check v
 	where
 		findkey s = show k `elem` lines s
 		env = hookEnv k Nothing
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
 			(frompipe, topipe) <- createPipe
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 54834be135..86ff2ea5b1 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -110,7 +110,7 @@ retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 	res <- retrieve o enck tmp
 	if res
-		then liftIO $ catchBool $ do
+		then liftIO $ catchBoolIO $ do
 			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
 			return True
 		else return res
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 29117b3a41..97ac648218 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -286,7 +286,7 @@ s3GetCreds c = do
 				_ -> return Nothing
 		else return $ Just (ak, sk)
 	where
-		getEnvKey s = liftIO $ catch (getEnv s) (const $ return "")
+		getEnvKey s = liftIO $ catchDefaultIO (getEnv s) ""
 
 {- Stores S3 creds encrypted in the remote's config if possible. -}
 s3SetCreds :: RemoteConfig -> Annex RemoteConfig
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index fe59ad3da8..377e4b21b3 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -179,7 +179,7 @@ writeLog1 :: FilePath -> [LogLine] -> IO ()
 writeLog1 file ls = viaTmp writeFile file (showLog ls)
 
 readLog1 :: FilePath -> IO [LogLine]
-readLog1 file = catch (parseLog <$> readFileStrict file) (const $ return [])
+readLog1 file = catchDefaultIO (parseLog <$> readFileStrict file) []
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
 lookupFile1 file = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 7ef2a4d18e..6a46ad8a16 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -69,7 +69,7 @@ locationLogs = do
 		files <- mapM tryDirContents (concat levelb)
 		return $ mapMaybe islogfile (concat files)
 	where
-		tryDirContents d = catch (dirContents d) (return . const [])
+		tryDirContents d = catchDefaultIO (dirContents d) []
 		islogfile f = maybe Nothing (\k -> Just (k, f)) $
 				logFileKey $ takeFileName f
 
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 4c4aa4c935..7285987231 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -8,7 +8,9 @@
 module Utility.Misc where
 
 import System.IO
+import System.IO.Error (try)
 import Control.Monad
+import Control.Applicative
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -26,5 +28,20 @@ readMaybe s = case reads s of
 	_ -> Nothing
 
 {- Catches IO errors and returns a Bool -}
-catchBool :: IO Bool -> IO Bool
-catchBool = flip catch (const $ return False)
+catchBoolIO :: IO Bool -> IO Bool
+catchBoolIO a = catchDefaultIO a False
+
+{- Catches IO errors and returns a Maybe -}
+catchMaybeIO :: IO a -> IO (Maybe a)
+catchMaybeIO a = catchDefaultIO (Just <$> a) Nothing
+
+{- Catches IO errors and returns a default value. -}
+catchDefaultIO :: IO a -> a -> IO a
+catchDefaultIO a def = catch a (const $ return def)
+
+{- Catches IO errors and returns the error message. -}
+catchMsgIO :: IO a -> IO (Either String a)
+catchMsgIO a = dispatch <$> try a
+	where
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
index 1e823c10ef..8d50dd8b2c 100644
--- a/Utility/TempFile.hs
+++ b/Utility/TempFile.hs
@@ -31,9 +31,9 @@ withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
 withTempFile template a = bracket create remove use
 	where
 		create = do
-			tmpdir <- catch getTemporaryDirectory (const $ return ".")
+			tmpdir <- catchDefaultIO getTemporaryDirectory "."
 			openTempFile tmpdir template
 		remove (name, handle) = do
 			hClose handle
-			catchBool (removeFile name >> return True)
+			catchBoolIO (removeFile name >> return True)
 		use (name, handle) = a name handle
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 12cc65e4dd..57f6b29162 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -104,9 +104,6 @@ checkNotReadOnly cmd
 	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
 
 checkEnv :: String -> IO ()
-checkEnv var = catch check (const $ return ())
-	where
-		check = do
-			val <- getEnv var
-			when (not $ null val) $
-				error $ "Action blocked by " ++ var
+checkEnv var =
+	whenM (not . null <$> catchDefaultIO (getEnv var) "") $
+		error $ "Action blocked by " ++ var

From 2de1e2c2cee4ea75b971000fe480c48b6cbaab29 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 21:32:42 -0400
Subject: [PATCH 2489/8313] Optimized copy --from and get --from to avoid
 checking the location log for files that are already present.

This can be a significant speedup when running in large trees that are
only missing a few files; it makes copy --from just as fast as get.
---
 Command/Move.hs  | 22 ++++++++++++++--------
 debian/changelog |  2 ++
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index f02f325580..9a6b0190e6 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -110,14 +110,20 @@ toPerform dest move key = moveLock move key $ do
  - from the remote.
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
-fromStart src move file = isAnnexed file $ \(key, _) -> do
-	u <- getUUID
-	remotes <- Remote.keyPossibilities key
-	if u == Remote.uuid src || not (any (== src) remotes)
-		then stop
-		else do
-			showMoveAction move file
-			next $ fromPerform src move key
+fromStart src move file
+	| move == True = isAnnexed file $ \(key, _) -> go key
+	| otherwise = isAnnexed file $ \(key, _) -> do
+		ishere <- inAnnex key
+		if ishere then stop else go key
+	where
+		go key = do
+			u <- getUUID
+			remotes <- Remote.keyPossibilities key
+			if u == Remote.uuid src || not (any (== src) remotes)
+				then stop
+				else do
+					showMoveAction move file
+					next $ fromPerform src move key
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key
diff --git a/debian/changelog b/debian/changelog
index 9fa96e06ab..72110785bb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20111108) UNRELEASED; urgency=low
   * Handle a case where an annexed file is moved into a gitignored directory,
     by having fix --force add its change.
   * Avoid cyclic drop problems.
+  * Optimized copy --from and get --from to avoid checking the location log
+    for files that are already present.
 
  -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
 

From 4389782628a1cc683ef238e848b6311fc4bd82c3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 22:37:52 -0400
Subject: [PATCH 2490/8313] tweak

---
 Command/Move.hs | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 9a6b0190e6..ffa246ab6f 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -28,7 +28,7 @@ seek = [withFilesInGit $ start True]
  - This only operates on the cached file content; it does not involve
  - moving data in the key-value backend. -}
 start :: Bool -> FilePath -> CommandStart
-start move file = do
+start move file = isAnnexed file $ \(key, _) -> do
 	noAuto
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
@@ -36,10 +36,10 @@ start move file = do
 		(Nothing, Nothing) -> error "specify either --from or --to"
 		(Nothing, Just name) -> do
 			dest <- Remote.byName name
-			toStart dest move file
+			toStart dest move file key
 		(Just name, Nothing) -> do
 			src <- Remote.byName name
-			fromStart src move file
+			fromStart src move file key
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 	where
 		noAuto = when move $ whenM (Annex.getState Annex.auto) $ error
@@ -58,8 +58,8 @@ showMoveAction False file = showStart "copy" file
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
-toStart dest move file = isAnnexed file $ \(key, _) -> do
+toStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
+toStart dest move file key = do
 	u <- getUUID
 	ishere <- inAnnex key
 	if not ishere || u == Remote.uuid dest
@@ -109,14 +109,14 @@ toPerform dest move key = moveLock move key $ do
  - If the current repository already has the content, it is still removed
  - from the remote.
  -}
-fromStart :: Remote.Remote Annex -> Bool -> FilePath -> CommandStart
-fromStart src move file
-	| move == True = isAnnexed file $ \(key, _) -> go key
-	| otherwise = isAnnexed file $ \(key, _) -> do
+fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
+fromStart src move file key
+	| move == True = go
+	| otherwise = do
 		ishere <- inAnnex key
-		if ishere then stop else go key
+		if ishere then stop else go
 	where
-		go key = do
+		go = do
 			u <- getUUID
 			remotes <- Remote.keyPossibilities key
 			if u == Remote.uuid src || not (any (== src) remotes)

From b327227ba596d4fc5012138d03390c3eb861b808 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 Nov 2011 23:35:08 -0400
Subject: [PATCH 2491/8313] better limiting of start actions to only run
 whenAnnexed

Mostly only refactoring, but this does remove one redundant stat of the
symlink by copy.
---
 Command.hs           | 10 ++++++----
 Command/Copy.hs      | 10 +++++-----
 Command/Drop.hs      | 25 ++++++++++++-------------
 Command/Find.hs      |  6 +++---
 Command/Fix.hs       |  6 +++---
 Command/Fsck.hs      |  9 ++++++---
 Command/Get.hs       |  6 +++---
 Command/Migrate.hs   |  7 +++----
 Command/Move.hs      |  6 +++---
 Command/PreCommit.hs |  5 +++--
 Command/Reinject.hs  | 10 +++++-----
 Command/Unannex.hs   |  6 +++---
 Command/Uninit.hs    |  8 ++++----
 Command/Unlock.hs    |  6 +++---
 Command/Whereis.hs   |  6 +++---
 Seek.hs              |  4 ++--
 16 files changed, 67 insertions(+), 63 deletions(-)

diff --git a/Command.hs b/Command.hs
index c436c5b628..083be37f22 100644
--- a/Command.hs
+++ b/Command.hs
@@ -15,8 +15,8 @@ module Command (
 	stop,
 	prepCommand,
 	doCommand,
+	whenAnnexed,
 	notAnnexed,
-	isAnnexed,
 	notBareRepo,
 	isBareRepo,
 	autoCopies
@@ -65,12 +65,14 @@ doCommand = start
 		failure = showEndFail >> return False
 		status r = showEndResult r >> return r
 
+{- Modifies an action to only act on files that are already annexed,
+ - and passes the key and backend on to it. -}
+whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
+whenAnnexed a file = maybe (return Nothing) (a file) =<< Backend.lookupFile file
+
 notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
 notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
 
-isAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex (Maybe a)) -> Annex (Maybe a)
-isAnnexed file a = maybe (return Nothing) a =<< Backend.lookupFile file
-
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
 	whenM isBareRepo $
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 8316b7cabf..16de423acb 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -7,6 +7,7 @@
 
 module Command.Copy where
 
+import Common.Annex
 import Command
 import qualified Command.Move
 
@@ -16,11 +17,10 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) ->
-	autoCopies key (<) numcopies $
-		Command.Move.start False file
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, backend) = autoCopies key (<) numcopies $
+	Command.Move.start False file (key, backend)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 44685ffcd6..ee3583869b 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -22,20 +22,19 @@ def = [dontCheck fromOpt $ command "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) ->
-	autoCopies key (>) numcopies $ do
-		from <- Annex.getState Annex.fromremote
-		case from of
-			Nothing -> startLocal file numcopies key
-			Just name -> do
-				remote <- Remote.byName name
-				u <- getUUID
-				if Remote.uuid remote == u
-					then startLocal file numcopies key
-					else startRemote file numcopies key remote
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, _) = autoCopies key (>) numcopies $ do
+	from <- Annex.getState Annex.fromremote
+	case from of
+		Nothing -> startLocal file numcopies key
+		Just name -> do
+			remote <- Remote.byName name
+			u <- getUUID
+			if Remote.uuid remote == u
+				then startLocal file numcopies key
+				else startRemote file numcopies key remote
 
 startLocal :: FilePath -> Maybe Int -> Key -> CommandStart
 startLocal file numcopies key = do
diff --git a/Command/Find.hs b/Command/Find.hs
index 46364c9876..c816ff0712 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -16,10 +16,10 @@ def :: [Command]
 def = [command "find" paramPaths seek "lists available files"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
 	whenM (liftM2 (||) (inAnnex key) limited) $
diff --git a/Command/Fix.hs b/Command/Fix.hs
index b46d6e8ecd..27c4b167e5 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -17,11 +17,11 @@ def = [command "fix" paramPaths seek
 	"fix up symlinks to point to annexed content"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
 {- Fixes the symlink to an annexed file. -}
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	link <- calcGitLink file key
 	l <- liftIO $ readSymbolicLink file
 	if link == l
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 89485b7788..bdc5099410 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -25,10 +25,13 @@ def :: [Command]
 def = [command "fsck" paramPaths seek "check for problems"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start, withBarePresentKeys startBare]
+seek =
+	[ withNumCopies $ \n -> whenAnnexed $ start n
+	, withBarePresentKeys startBare
+	]
 
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, backend) -> do
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, backend) = do
 	showStart "fsck" file
 	next $ perform key file backend numcopies
 
diff --git a/Command/Get.hs b/Command/Get.hs
index 4a0908bdc8..f7d953bb65 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -19,10 +19,10 @@ def = [dontCheck fromOpt $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies start]
+seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
-start :: FilePath -> Maybe Int -> CommandStart
-start file numcopies = isAnnexed file $ \(key, _) -> do
+start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start numcopies file (key, _) = do
 	inannex <- inAnnex key
 	if inannex
 		then stop
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index a823466dc7..3c87f41363 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -13,17 +13,16 @@ import qualified Backend
 import qualified Types.Key
 import Annex.Content
 import qualified Command.Add
-import Backend
 import Logs.Web
 
 def :: [Command]
 def = [command "migrate" paramPaths seek "switch data to different backend"]
 
 seek :: [CommandSeek]
-seek = [withBackendFilesInGit start]
+seek = [withBackendFilesInGit $ \(b, f) -> whenAnnexed (start b) f]
 
-start :: BackendFile -> CommandStart
-start (b, file) = isAnnexed file $ \(key, oldbackend) -> do
+start :: Maybe (Backend Annex) -> FilePath -> (Key, Backend Annex) -> CommandStart
+start b file (key, oldbackend) = do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
 	if (newbackend /= oldbackend || upgradableKey key) && exists
diff --git a/Command/Move.hs b/Command/Move.hs
index ffa246ab6f..9553d1639b 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -21,14 +21,14 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 	"move content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit $ start True]
+seek = [withFilesInGit $ whenAnnexed $ start True]
 
 {- Move (or copy) a file either --to or --from a repository.
  -
  - This only operates on the cached file content; it does not involve
  - moving data in the key-value backend. -}
-start :: Bool -> FilePath -> CommandStart
-start move file = isAnnexed file $ \(key, _) -> do
+start :: Bool -> FilePath -> (Key, Backend Annex) -> CommandStart
+start move file (key, _) = do
 	noAuto
 	to <- Annex.getState Annex.toremote
 	from <- Annex.getState Annex.fromremote
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 1949de113f..57bc7ac138 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -18,8 +18,9 @@ def = [command "pre-commit" paramPaths seek "run by git pre-commit hook"]
 {- The pre-commit hook needs to fix symlinks to all files being committed.
  - And, it needs to inject unlocked files into the annex. -}
 seek :: [CommandSeek]
-seek = [withFilesToBeCommitted Command.Fix.start,
-	withFilesUnlockedToBeCommitted start]
+seek =
+	[ withFilesToBeCommitted $ whenAnnexed Command.Fix.start
+	, withFilesUnlockedToBeCommitted start]
 
 start :: BackendFile -> CommandStart
 start p = next $ perform p
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 1277edf906..cfa0655ef1 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -25,19 +25,19 @@ start (src:dest:[])
 	| src == dest = stop
 	| otherwise = do
 		showStart "reinject" dest
-		next $ perform src dest
+		next $ whenAnnexed (perform src) dest
 start _ = error "specify a src file and a dest file"
 
-perform :: FilePath -> FilePath -> CommandPerform
-perform src dest = isAnnexed dest $ \(key, backend) -> do
-	unlessM (move key) $ error "mv failed!"
+perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform
+perform src _dest (key, backend) = do
+	unlessM move $ error "mv failed!"
 	next $ cleanup key backend
 	where
 		-- the file might be on a different filesystem,
 		-- so mv is used rather than simply calling
 		-- moveToObjectDir; disk space is also
 		-- checked this way.
-		move key = getViaTmp key $ \tmp ->
+		move = getViaTmp key $ \tmp ->
 			liftIO $ boolSystem "mv" [File src, File tmp]
 
 cleanup :: Key -> Backend Annex -> CommandCleanup
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index d24f921a93..b9190ce044 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -21,11 +21,11 @@ def :: [Command]
 def = [command "unannex" paramPaths seek "undo accidential add command"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unannex subcommand undoes an add. -}
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	ishere <- inAnnex key
 	if ishere
 		then do
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index f317b7620d..8987240bee 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -33,15 +33,15 @@ check = do
 			[Params "rev-parse --abbrev-ref HEAD"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit startUnannex, withNothing start]
+seek = [withFilesInGit $ whenAnnexed startUnannex, withNothing start]
 
-startUnannex :: FilePath -> CommandStart
-startUnannex file = do
+startUnannex :: FilePath -> (Key, Backend Annex) -> CommandStart
+startUnannex file info = do
 	-- Force fast mode before running unannex. This way, if multiple
 	-- files link to a key, it will be left in the annex and hardlinked
 	-- to by each.
 	Annex.changeState $ \s -> s { Annex.fast = True }
-	Command.Unannex.start file
+	Command.Unannex.start file info
 
 start :: CommandStart
 start = next perform
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 590b753111..22f9ce7108 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -22,12 +22,12 @@ def =
 		c n = command n paramPaths seek
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unlock subcommand replaces the symlink with a copy of the file's
  - content. -}
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	showStart "unlock" file
 	next $ perform file key
 
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 0681bfba1e..eb2ae3d4e7 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -18,10 +18,10 @@ def = [command "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit start]
+seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> CommandStart
-start file = isAnnexed file $ \(key, _) -> do
+start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start file (key, _) = do
 	showStart "whereis" file
 	next $ perform key
 
diff --git a/Seek.hs b/Seek.hs
index 3c83ebc35d..9863b33c40 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -33,10 +33,10 @@ withAttrFilesInGit attr a params = do
 	files <- seekHelper LsFiles.inRepo params
 	prepFilteredGen a fst $ inRepo $ Git.checkAttr attr files
 
-withNumCopies :: (FilePath -> Maybe Int -> CommandStart) -> CommandSeek
+withNumCopies :: (Maybe Int -> FilePath -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
-		go (file, v) = a file (readMaybe v)
+		go (file, v) = a (readMaybe v) file
 
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params = do

From 637b5feb45013f69f3aacbafeb796de666d3faa3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 01:52:58 -0400
Subject: [PATCH 2492/8313] lint

---
 Annex/Branch.hs          |  8 ++---
 Annex/CatFile.hs         |  2 +-
 Annex/Content.hs         |  4 +--
 Checks.hs                |  4 +--
 Command.hs               | 17 +++++------
 Command/Describe.hs      |  2 +-
 Command/FromKey.hs       |  2 +-
 Command/Fsck.hs          |  4 +--
 Command/Migrate.hs       |  4 +--
 Command/Move.hs          |  4 +--
 Command/Uninit.hs        |  4 +--
 Common.hs                | 63 ++++++++++++++--------------------------
 Common/Annex.hs          | 21 +++++---------
 Config.hs                |  2 +-
 Git/LsFiles.hs           |  2 +-
 Git/UnionMerge.hs        |  2 +-
 Logs/Remote.hs           |  2 +-
 Logs/Trust.hs            |  4 +--
 Logs/UUID.hs             |  2 +-
 Logs/UUIDBased.hs        |  2 +-
 Remote/Git.hs            |  2 +-
 Remote/Helper/Special.hs |  2 +-
 Seek.hs                  |  6 ++--
 Upgrade/V1.hs            |  4 +--
 configure.hs             |  2 +-
 git-annex.cabal          |  2 +-
 26 files changed, 71 insertions(+), 102 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 6c28a0c84a..05c89ed97e 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -63,7 +63,7 @@ withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	f <- fromRepo $ index
+	f <- fromRepo index
 	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
@@ -336,8 +336,8 @@ stageJournalFiles = do
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
-		git_hash_object g = Git.gitCommandLine
-			[Param "hash-object", Param "-w", Param "--stdin-paths"] g
+		git_hash_object = Git.gitCommandLine
+			[Param "hash-object", Param "-w", Param "--stdin-paths"]
 
 
 {- Checks if there are changes in the journal. -}
@@ -366,7 +366,7 @@ fileJournal = replace "//" "_" . replace "_" "/"
  - contention with other git-annex processes. -}
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
-	file <- fromRepo $ gitAnnexJournalLock
+	file <- fromRepo gitAnnexJournalLock
 	bracketIO (lock file) unlock a
 	where
 		lock file = do
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index a043e1ae3d..99cc519f51 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -17,7 +17,7 @@ catFile :: String -> FilePath -> Annex String
 catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
-			h <- inRepo $ Git.CatFile.catFileStart
+			h <- inRepo Git.CatFile.catFileStart
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
 			go h
 		go h = liftIO $ Git.CatFile.catFile h branch file
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 7586bb96f6..83839ea135 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -91,7 +91,7 @@ openForLock file writelock = bracket_ prep cleanup go
 		 - have to fiddle with permissions to open for an
 		 - exclusive lock. -}
 		forwritelock a = 
-			when writelock $ whenM (doesFileExist file) $ a
+			when writelock $ whenM (doesFileExist file) a
 		prep = forwritelock $ allowWrite file
 		cleanup = forwritelock $ preventWrite file
 
@@ -251,7 +251,7 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
 moveBad :: Key -> Annex FilePath
 moveBad key = do
 	src <- fromRepo $ gitAnnexLocation key
-	bad <- fromRepo $ gitAnnexBadDir
+	bad <- fromRepo gitAnnexBadDir
 	let dest = bad  takeFileName src
 	liftIO $ do
 		createDirectoryIfMissing True (parentDir dest)
diff --git a/Checks.hs b/Checks.hs
index 6a70fc52db..e443811cdc 100644
--- a/Checks.hs
+++ b/Checks.hs
@@ -24,12 +24,12 @@ repoExists = CommandCheck 0 ensureInitialized
 fromOpt :: CommandCheck
 fromOpt = CommandCheck 1 $ do
 	v <- Annex.getState Annex.fromremote
-	unless (v == Nothing) $ error "cannot use --from with this command"
+	unless (isNothing v) $ error "cannot use --from with this command"
 
 toOpt :: CommandCheck
 toOpt = CommandCheck 2 $ do
 	v <- Annex.getState Annex.toremote
-	unless (v == Nothing) $ error "cannot use --to with this command"
+	unless (isNothing v) $ error "cannot use --to with this command"
 
 dontCheck :: CommandCheck -> Command -> Command
 dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
diff --git a/Command.hs b/Command.hs
index 083be37f22..d22c2d12f2 100644
--- a/Command.hs
+++ b/Command.hs
@@ -6,10 +6,6 @@
  -}
 
 module Command (
-	module Types.Command,
-	module Seek,
-	module Checks,
-	module Options,
 	command,
 	next,
 	stop,
@@ -19,20 +15,21 @@ module Command (
 	notAnnexed,
 	notBareRepo,
 	isBareRepo,
-	autoCopies
+	autoCopies,
+	module ReExported
 ) where
 
 import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
-import Types.Command
+import Types.Command as ReExported
+import Seek as ReExported
+import Checks as ReExported
+import Options as ReExported
 import Logs.Trust
 import Logs.Location
 import Config
-import Seek
-import Checks
-import Options
 
 {- Generates a command with the common checks. -}
 command :: String -> String -> [CommandSeek] -> String -> Command
@@ -50,7 +47,7 @@ stop = return Nothing
  - list of actions to perform to run the command. -}
 prepCommand :: Command -> [String] -> Annex [CommandCleanup]
 prepCommand Command { cmdseek = seek, cmdcheck = c } params = do
-	sequence_ $ map runCheck c
+	mapM_ runCheck c
 	map doCommand . concat <$> mapM (\s -> s params) seek
 
 {- Runs a command through the start, perform and cleanup stages -}
diff --git a/Command/Describe.hs b/Command/Describe.hs
index 1cc81bcbd5..61297e77c7 100644
--- a/Command/Describe.hs
+++ b/Command/Describe.hs
@@ -24,7 +24,7 @@ start (name:description) = do
 	showStart "describe" name
 	u <- Remote.nameToUUID name
 	next $ perform u $ unwords description
-start _ = do error "Specify a repository and a description."	
+start _ = error "Specify a repository and a description."	
 
 perform :: UUID -> String -> CommandPerform
 perform u description = do
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index b910dd1f09..ec194e06e8 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -22,7 +22,7 @@ seek = [withWords start]
 
 start :: [String] -> CommandStart
 start (keyname:file:[]) = notBareRepo $ do
-	let key = maybe (error "bad key") id $ readKey keyname
+	let key = fromMaybe (error "bad key") $ readKey keyname
 	inbackend <- inAnnex key
 	unless inbackend $ error $
 		"key ("++ keyname ++") is not present in backend"
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index bdc5099410..99dda99e5f 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -50,7 +50,7 @@ withBarePresentKeys a params = isBareRepo >>= go
 	where
 		go False = return []
 		go True = do
-			unless (null params) $ do
+			unless (null params) $
 				error "fsck should be run without parameters in a bare repository"
 			prepStart a loggedKeys
 
@@ -137,7 +137,7 @@ checkKeySize key = do
 
 
 checkBackend :: Backend Annex -> Key -> Annex Bool
-checkBackend backend key =  (Types.Backend.fsckKey backend) key
+checkBackend = Types.Backend.fsckKey
 
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 3c87f41363..860c4bd470 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -46,7 +46,7 @@ perform file oldkey newbackend = do
 	-- The old backend's key is not dropped from it, because there may
 	-- be other files still pointing at that key.
 	src <- fromRepo $ gitAnnexLocation oldkey
-	tmp <- fromRepo $ gitAnnexTmpDir
+	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file
 	liftIO $ createLink src tmpfile
 	k <- Backend.genKey tmpfile $ Just newbackend
@@ -64,7 +64,7 @@ perform file oldkey newbackend = do
 					-- associated urls, record them for
 					-- the new key as well.
 					urls <- getUrls oldkey
-					when (not $ null urls) $
+					unless (null urls) $
 						mapM_ (setUrlPresent newkey) urls
 
 					next $ Command.Add.cleanup file newkey True
diff --git a/Command/Move.hs b/Command/Move.hs
index 9553d1639b..155f4d605f 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -82,7 +82,7 @@ toPerform dest move key = moveLock move key $ do
 		else Remote.hasKey dest key
 	case isthere of
 		Left err -> do
-			showNote $ err
+			showNote err
 			stop
 		Right False -> do
 			showAction $ "to " ++ Remote.name dest
@@ -111,7 +111,7 @@ toPerform dest move key = moveLock move key $ do
  -}
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
-	| move == True = go
+	| move = go
 	| otherwise = do
 		ishere <- inAnnex key
 		if ishere then stop else go
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 8987240bee..ca18c478c5 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -51,11 +51,11 @@ perform = next cleanup
 
 cleanup :: CommandCleanup
 cleanup = do
-	annexdir <- fromRepo $ gitAnnexDir
+	annexdir <- fromRepo gitAnnexDir
 	uninitialize
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
 	inRepo $ Git.run "branch" [Param "-D", Param Annex.Branch.name]
-	liftIO $ exitSuccess
+	liftIO exitSuccess
diff --git a/Common.hs b/Common.hs
index ef068ac105..e0132d9e96 100644
--- a/Common.hs
+++ b/Common.hs
@@ -1,46 +1,25 @@
-module Common (
-	module Control.Monad,
-	module Control.Applicative,
-	module Control.Monad.State,
-	module Control.Exception.Extensible,
-	module Data.Maybe,
-	module Data.List,
-	module Data.String.Utils,
-	module System.Path,
-	module System.FilePath,
-	module System.Directory,
-	module System.Cmd.Utils,
-	module System.IO,
-	module System.Posix.Files,
-	module System.Posix.IO,
-	module System.Posix.Process,
-	module System.Exit,
-	module Utility.Misc,
-	module Utility.Conditional,
-	module Utility.SafeCommand,
-	module Utility.Path,
-) where
+module Common (module X) where
 
-import Control.Monad hiding (join)
-import Control.Applicative
-import Control.Monad.State (liftIO)
-import Control.Exception.Extensible (IOException)
+import Control.Monad as X hiding (join)
+import Control.Applicative as X
+import Control.Monad.State as X (liftIO)
+import Control.Exception.Extensible as X (IOException)
 
-import Data.Maybe
-import Data.List
-import Data.String.Utils
+import Data.Maybe as X
+import Data.List as X
+import Data.String.Utils as X
 
-import System.Path
-import System.FilePath
-import System.Directory
-import System.Cmd.Utils hiding (safeSystem)
-import System.IO hiding (FilePath)
-import System.Posix.Files
-import System.Posix.IO
-import System.Posix.Process hiding (executeFile)
-import System.Exit
+import System.Path as X
+import System.FilePath as X
+import System.Directory as X
+import System.Cmd.Utils as X hiding (safeSystem)
+import System.IO as X hiding (FilePath)
+import System.Posix.Files as X
+import System.Posix.IO as X
+import System.Posix.Process as X hiding (executeFile)
+import System.Exit as X
 
-import Utility.Misc
-import Utility.Conditional
-import Utility.SafeCommand
-import Utility.Path
+import Utility.Misc as X
+import Utility.Conditional as X
+import Utility.SafeCommand as X
+import Utility.Path as X
diff --git a/Common/Annex.hs b/Common/Annex.hs
index 6b5bc31de2..e90825f0e9 100644
--- a/Common/Annex.hs
+++ b/Common/Annex.hs
@@ -1,15 +1,8 @@
-module Common.Annex (
-	module Common,
-	module Types,
-	module Types.UUID,
-	module Annex,
-	module Locations,
-	module Messages,
-) where
+module Common.Annex (module X) where
 
-import Common
-import Types
-import Types.UUID (toUUID, fromUUID)
-import Annex (gitRepo, inRepo, fromRepo)
-import Locations
-import Messages
+import Common as X
+import Types as X
+import Types.UUID as X (toUUID, fromUUID)
+import Annex as X (gitRepo, inRepo, fromRepo)
+import Locations as X
+import Messages as X
diff --git a/Config.hs b/Config.hs
index c24ab9d04d..dbd13ad3fd 100644
--- a/Config.hs
+++ b/Config.hs
@@ -18,7 +18,7 @@ setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
 	inRepo $ Git.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
-	newg <- inRepo $ Git.configRead
+	newg <- inRepo Git.configRead
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Looks up a per-remote config setting in git config.
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index bceee26fcd..85215fe040 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -20,7 +20,7 @@ import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: [FilePath] -> Repo -> IO [FilePath]
-inRepo l repo = pipeNullSplit (Params "ls-files --cached -z --" : map File l) repo
+inRepo l = pipeNullSplit $ Params "ls-files --cached -z --" : map File l
 
 {- Scans for files at the specified locations that are not checked into git. -}
 notInRepo :: Bool -> [FilePath] -> Repo -> IO [FilePath]
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 32966c846d..30778d0344 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -37,7 +37,7 @@ merge x y repo = do
  - the index are preserved (and participate in the merge). -}
 merge_index :: Repo -> [String] -> IO ()
 merge_index repo bs =
-	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b repo) bs
+	update_index repo =<< concat <$> mapM (`merge_tree_index` repo) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index e2b04bf471..8d15f3151e 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -30,7 +30,7 @@ remoteLog = "remote.log"
 {- Adds or updates a remote's config in the log. -}
 configSet :: UUID -> RemoteConfig -> Annex ()
 configSet u c = do
-	ts <- liftIO $ getPOSIXTime
+	ts <- liftIO getPOSIXTime
 	Annex.Branch.change remoteLog $
 		showLog showConfig . changeLog ts u c . parseLog parseConfig
 
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 8c4507dcb8..cb91861fd9 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -47,7 +47,7 @@ parseTrust :: String -> Maybe TrustLevel
 parseTrust s
 	| length w > 0 = Just $ parse $ head w
 	-- back-compat; the trust.log used to only list trusted repos
-	| otherwise = Just $ Trusted
+	| otherwise = Just Trusted
 	where
 		w = words s
 		parse "1" = Trusted
@@ -62,7 +62,7 @@ showTrust Trusted = "1"
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid@(UUID _) level = do
-	ts <- liftIO $ getPOSIXTime
+	ts <- liftIO getPOSIXTime
 	Annex.Branch.change trustLog $
 		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 77cfb5ce0f..da611d7bf5 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -34,7 +34,7 @@ logfile = "uuid.log"
 {- Records a description for a uuid in the log. -}
 describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
-	ts <- liftIO $ getPOSIXTime
+	ts <- liftIO getPOSIXTime
 	Annex.Branch.change logfile $
 		showLog id . changeLog ts uuid desc . parseLog Just
 
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 9609d73213..42908ab1d3 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -55,7 +55,7 @@ showLog shower = unlines . map showpair . M.toList
 			unwords [fromUUID k, shower v]
 
 parseLog :: (String -> Maybe a) -> String -> Log a
-parseLog parser = M.fromListWith best . catMaybes . map parse . lines
+parseLog parser = M.fromListWith best . mapMaybe parse . lines
 	where
 		parse line
 			| null ws = Nothing
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 30d992e8ca..541d8e5f65 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -166,7 +166,7 @@ onLocal r a = do
 		-- for anything onLocal is used to do.
 		Annex.Branch.disableUpdate
 		ret <- a
-		liftIO $ Git.reap
+		liftIO Git.reap
 		return ret
 
 keyUrl :: Git.Repo -> Key -> String
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 5bbf4169d7..77478eb1d5 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -19,7 +19,7 @@ import qualified Git
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	m <- fromRepo $ Git.configMap
+	m <- fromRepo Git.configMap
 	return $ map construct $ remotepairs m
 	where
 		remotepairs = M.toList . M.filterWithKey match
diff --git a/Seek.hs b/Seek.hs
index 9863b33c40..1430ebabd1 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -23,7 +23,7 @@ import qualified Limit
 seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
 seekHelper a params = do
 	g <- gitRepo
-	liftIO $ runPreserveOrder (\p -> a p g) params
+	liftIO $ runPreserveOrder (`a` g) params
 
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
@@ -73,7 +73,7 @@ withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	top <- fromRepo $ Git.workTree
+	top <- fromRepo Git.workTree
 	typechangedfiles <- seekHelper typechanged params
 	unlockedfiles <- liftIO $ filterM notSymlink $
 		map (\f -> top ++ "/" ++ f) typechangedfiles
@@ -109,7 +109,7 @@ prepFilteredGen a d fs = do
  - command, using a list (ie of files) coming from an action. The list
  - will be produced and consumed lazily. -}
 prepStart :: (b -> CommandStart) -> Annex [b] -> Annex [CommandStart]
-prepStart a fs = liftM (map a) fs
+prepStart a = liftM (map a)
 
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 377e4b21b3..567cf8e5bf 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -51,7 +51,7 @@ upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
 	
-	bare <- fromRepo $ Git.repoIsLocalBare
+	bare <- fromRepo Git.repoIsLocalBare
 	if bare
 		then do
 			moveContent
@@ -113,7 +113,7 @@ moveLocationLogs = do
 					else return []
 			move (l, k) = do
 				dest <- fromRepo $ logFile2 k
-				dir <- fromRepo $ Upgrade.V2.gitStateDir
+				dir <- fromRepo Upgrade.V2.gitStateDir
 				let f = dir  l
 				liftIO $ createDirectoryIfMissing True (parentDir dest)
 				-- could just git mv, but this way deals with
diff --git a/configure.hs b/configure.hs
index 1d1c023356..cb73af2a99 100644
--- a/configure.hs
+++ b/configure.hs
@@ -69,7 +69,7 @@ checkGitVersion = do
 		-- for git-check-attr behavior change
 		need = "1.7.7"
 		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
-		extend n l = l ++ take (n - length l) (repeat 0)
+		extend n l = l ++ replicate (n - length l) 0
 		mult _ [] = []
 		mult n (x:xs) = (n*x) : (mult (n*100) xs)
 		readi :: String -> Integer
diff --git a/git-annex.cabal b/git-annex.cabal
index 4b3a708a4e..7b9459b46b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111107
+Version: 3.20111108
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 826d5887b2c31c9dca1415997d7704d9442077b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 13:42:31 -0400
Subject: [PATCH 2493/8313] Automatically fix up badly formatted uuid.log
 entries produced by 3.20111105, whenever the uuid.log is changed (ie, by init
 or describe).

---
 Logs/UUID.hs      | 31 ++++++++++++++++++++++++++++++-
 Logs/UUIDBased.hs |  1 +
 debian/changelog  |  2 ++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index da611d7bf5..20f43d15ca 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -36,7 +36,36 @@ describeUUID :: UUID -> String -> Annex ()
 describeUUID uuid desc = do
 	ts <- liftIO getPOSIXTime
 	Annex.Branch.change logfile $
-		showLog id . changeLog ts uuid desc . parseLog Just
+		showLog id . changeLog ts uuid desc . fixBadUUID . parseLog Just
+
+{- Temporarily here to fix badly formatted uuid logs generated by
+ - versions 3.20111105 and 3.20111025. 
+ -
+ - Those logs contain entries with the UUID and description flipped.
+ - Due to parsing, if the description is multiword, only the first
+ - will be taken to be the UUID. So, if the UUID of an entry does
+ - not look like a UUID, and the last word of the description does,
+ - flip them back.
+ -}
+fixBadUUID :: Log String -> Log String
+fixBadUUID = M.fromList . map fixup . M.toList
+	where
+		fixup (k, v)
+			| isbad = (fixeduuid, LogEntry (Date $ newertime v) fixedvalue)
+			| otherwise = (k, v)
+			where
+				kuuid = fromUUID k
+				isbad = (not $ isuuid kuuid) && isuuid lastword
+				ws = words $ value v
+				lastword = last ws
+				fixeduuid = toUUID lastword
+				fixedvalue = unwords $ kuuid:(take (length ws - 1) ws)
+		-- For the fixed line to take precidence, it should be
+		-- slightly newer, but only slightly.
+		newertime (LogEntry (Date d) _) = d + minimumPOSIXTimeSlice
+		newertime (LogEntry (Unknown) _) = minimumPOSIXTimeSlice
+		minimumPOSIXTimeSlice = 0.000001
+		isuuid s = length s == 36 && length (split "-" s) == 5
 
 {- Records the uuid in the log, if it's not already there. -}
 recordUUID :: UUID -> Annex ()
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 42908ab1d3..04b12887d7 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -15,6 +15,7 @@
 module Logs.UUIDBased (
 	Log,
 	LogEntry(..),
+	TimeStamp(..),
 	parseLog,
 	showLog,
 	changeLog,
diff --git a/debian/changelog b/debian/changelog
index 72110785bb..e1d08ecdb8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20111108) UNRELEASED; urgency=low
   * Avoid cyclic drop problems.
   * Optimized copy --from and get --from to avoid checking the location log
     for files that are already present.
+  * Automatically fix up badly formatted uuid.log entries produced by
+    3.20111105, whenever the uuid.log is changed (ie, by init or describe).
 
  -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
 

From 71b216d1fb2d2308c75ba0da62e6e6a85c5adb19 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 16:18:41 -0400
Subject: [PATCH 2494/8313] map: Support remotes with /~/ and /~user/

More accurately, it was supported already when map uses git-annex-shell,
but not when it does not.

Note that the user name cannot be shell escaped using git-annex's current
approach for shell escaping. I tried and some shells like dash cannot
cd ~'joey'. Rest of directory is still shell escaped, not for security but
in case a directory has a space or other weird character.
---
 Command/Map.hs                                       | 12 +++++++++---
 debian/changelog                                     |  5 +++--
 ...ap_has_problems_with_urls_containing___126__.mdwn |  2 ++
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index f72cb107ad..8755bc7c2d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -195,11 +195,17 @@ tryScan r
 		configlist =
 			onRemote r (pipedconfig, Nothing) "configlist" []
 		manualconfiglist = do
-			let sshcmd =
-				"cd " ++ shellEscape(Git.workTree r) ++ " && " ++
-				"git config --list"
 			sshparams <- sshToRepo r [Param sshcmd]
 			liftIO $ pipedconfig "ssh" sshparams
+			where
+				sshcmd = cddir ++ " && " ++
+					"git config --list"
+				dir = Git.workTree r
+				cddir
+					| take 2 dir == "/~" =
+						let (userhome, reldir) = span (/= '/') (drop 1 dir)
+						in "cd " ++ userhome ++ " && cd " ++ shellEscape (drop 1 reldir)
+					| otherwise = "cd " ++ shellEscape dir
 
 		-- First, try sshing and running git config manually,
 		-- only fall back to git-annex-shell configlist if that
diff --git a/debian/changelog b/debian/changelog
index e1d08ecdb8..9c2de7b083 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111108) UNRELEASED; urgency=low
+git-annex (3.20111111) unstable; urgency=low
 
   * Handle a case where an annexed file is moved into a gitignored directory,
     by having fix --force add its change.
@@ -7,8 +7,9 @@ git-annex (3.20111108) UNRELEASED; urgency=low
     for files that are already present.
   * Automatically fix up badly formatted uuid.log entries produced by
     3.20111105, whenever the uuid.log is changed (ie, by init or describe).
+  * map: Support remotes with /~/ and /~user/
 
- -- Joey Hess   Mon, 07 Nov 2011 18:08:42 -0400
+ -- Joey Hess   Fri, 11 Nov 2011 13:44:18 -0400
 
 git-annex (3.20111107) unstable; urgency=low
 
diff --git a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
index c6eb7bc73a..24f04eeb54 100644
--- a/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
+++ b/doc/bugs/git_annex_map_has_problems_with_urls_containing___126__.mdwn
@@ -42,3 +42,5 @@ map` works too:
 
     ok
 
+
+> [[fixed|done]] --[[Joey]] 

From baa07971ff8e52cddc4ad31e75d7b3a6c4c6ba07 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 16:29:19 -0400
Subject: [PATCH 2495/8313] releasing version 3.20111111

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7b9459b46b..b2638c3ac7 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111108
+Version: 3.20111111
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 15b92ad6a19a7c486fb30fb3552be4a4ef9f9f3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 Nov 2011 16:29:57 -0400
Subject: [PATCH 2496/8313] add news item for git-annex 3.20111111

---
 doc/news/version_3.20110928.mdwn | 19 -------------------
 doc/news/version_3.20111111.mdwn | 10 ++++++++++
 2 files changed, 10 insertions(+), 19 deletions(-)
 delete mode 100644 doc/news/version_3.20110928.mdwn
 create mode 100644 doc/news/version_3.20111111.mdwn

diff --git a/doc/news/version_3.20110928.mdwn b/doc/news/version_3.20110928.mdwn
deleted file mode 100644
index 3b11e18f5c..0000000000
--- a/doc/news/version_3.20110928.mdwn
+++ /dev/null
@@ -1,19 +0,0 @@
-git-annex 3.20110928 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * --in can be used to make git-annex only operate on files
-     believed to be present in a given repository.
-   * Arbitrarily complex expressions can be built to limit the files git-annex
-     operates on, by combining the options --not --and --or -( and -)
-     Example: git annex get --exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
-   * --copies=N can be used to make git-annex only operate on files with
-     the specified number of copies. (And --not --copies=N for the inverse.)
-   * find: Rather than only showing files whose contents are present,
-     when used with --exclude --copies or --in, displays all files that
-     match the specified conditions.
-   * Note that this is a behavior change for git-annex find! Old behavior
-     can be gotten by using: git-annex find --in .
-   * status: Massively sped up; remove --fast mode.
-   * unused: File contents used by branches and tags are no longer
-     considered unused, even when not used by the current branch. This is
-     the final piece of the puzzle needed for git-annex to to play nicely
-     with branches."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111111.mdwn b/doc/news/version_3.20111111.mdwn
new file mode 100644
index 0000000000..2173400154
--- /dev/null
+++ b/doc/news/version_3.20111111.mdwn
@@ -0,0 +1,10 @@
+git-annex 3.20111111 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Handle a case where an annexed file is moved into a gitignored directory,
+     by having fix --force add its change.
+   * Avoid cyclic drop problems.
+   * Optimized copy --from and get --from to avoid checking the location log
+     for files that are already present.
+   * Automatically fix up badly formatted uuid.log entries produced by
+     3.20111105, whenever the uuid.log is changed (ie, by init or describe).
+   * map: Support remotes with /~/ and /~user/"""]]
\ No newline at end of file

From 30f1aeb13c6627e8264f222a799b2702c5f13714 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:24:07 -0400
Subject: [PATCH 2497/8313] golf

---
 Annex.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 1c1091799f..24cc78a647 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -122,9 +122,7 @@ gitRepo = getState repo
 
 {- Runs an IO action in the annex's git repository. -}
 inRepo :: (Git.Repo -> IO a) -> Annex a
-inRepo a = do
-	g <- gitRepo
-	liftIO $ a g
+inRepo a = liftIO . a =<< gitRepo
 
 {- Extracts a value from the annex's git repisitory. -}
 fromRepo :: (Git.Repo -> a) -> Annex a

From 6e946b9a39f8e5ba55651accbc1307e8cce5c4e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:24:14 -0400
Subject: [PATCH 2498/8313] add

---
 doc/todo/optimise_git-annex_merge.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/todo/optimise_git-annex_merge.mdwn

diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
new file mode 100644
index 0000000000..a2cdfb15fe
--- /dev/null
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -0,0 +1,13 @@
+Typically `git-annex merge` is fast, but it could still be sped up.
+
+`git-annex merge` runs `git-hash-object` once per file that needs to be
+merged. Elsewhere in git-annex, `git-hash-object` is used in a faster mode,
+reading files from disk via `--stdin-paths`. But here, the data is not
+in raw files on disk, and I doubt writing them is the best approach.
+Instead, I'd like a way to stream multiple objects into git using stdin.
+Sometime, should look at either extending git-hash-object to support that,
+or possibly look at using git-fast-import instead.
+
+`git-annex merge` also runs `git show` once per file that needs to be
+merged. This could be reduced to a single call to `git-cat-file --batch`,
+There is already a Git.CatFile library that can do this easily. --[[Joey]]

From fe4ad93e4a82eabc2300705a8bfc59caaf4814b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:46:32 -0400
Subject: [PATCH 2499/8313] add

---
 doc/todo/avoid_unnecessary_union_merges.mdwn | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/todo/avoid_unnecessary_union_merges.mdwn

diff --git a/doc/todo/avoid_unnecessary_union_merges.mdwn b/doc/todo/avoid_unnecessary_union_merges.mdwn
new file mode 100644
index 0000000000..67aa285799
--- /dev/null
+++ b/doc/todo/avoid_unnecessary_union_merges.mdwn
@@ -0,0 +1,18 @@
+Some commands cause a union merge unnecessarily. For example, `git annex add`
+modifies the location log, which first requires reading the current log (if
+any), which triggers a merge.
+
+Would be good to avoid these unnecessary union merges. First because it's
+faster and second because it avoids a possible delay when a user might
+ctrl-c and leave the repo in an inconsistent state. In the case of an add,
+the file will be in the annex, but no location log will exist for it (fsck
+fixes that).
+
+It may be that all that's needed is to modify Annex.Branch.change
+to read the current value, without merging. Then commands like `get`, that
+query the branch, will still cause merges, and commands like `add` that
+only modify it, will not. Note that for a command like `get`, the merge
+occurs before it has done anything, so ctrl-c should not be a problem
+there.
+
+This is a delicate change, I need to take care.. --[[Joey]]

From 897bf938f687c1bd9fcfa0097b65708cf0891041 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 14:51:19 -0400
Subject: [PATCH 2500/8313] merge: Improve commit messages to mention what was
 merged.

---
 Annex/Branch.hs  | 15 +++++++++------
 debian/changelog |  6 ++++++
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 05c89ed97e..e86a3d7d18 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -120,8 +120,8 @@ commit message = whenM journalDirty $ lockJournal $ do
  - journal into the index. Otherwise, any changes in the journal would
  - later get staged, and might overwrite changes made during the merge.
  -
- - It would be cleaner to handle the merge by updating the journal, not the
- - index, with changes from the branches.
+ - (It would be cleaner to handle the merge by updating the journal, not the
+ - index, with changes from the branches.)
  -
  - The index is always updated using a union merge, as that's the most
  - efficient way to update it. However, if the branch can be
@@ -136,10 +136,13 @@ update = onceonly $ do
 	let (refs, branches) = unzip c
 	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
 		when dirty stageJournalFiles
-		unless (null branches) $ do
-			showSideAction $ "merging " ++
-				(unwords $ map Git.refDescribe branches) ++
+		let merge_desc = if null branches
+			then "update" 
+			else "merging " ++
+				(unwords $ map Git.refDescribe branches) ++ 
 				" into " ++ name
+		unless (null branches) $ do
+			showSideAction merge_desc
 			{- Note: This merges the branches into the index.
 			 - Any unstaged changes in the git-annex branch
 			 - (if it's checked out) will be removed. So,
@@ -149,7 +152,7 @@ update = onceonly $ do
 			inRepo $ \g -> Git.UnionMerge.merge_index g branches
 		ff <- if dirty then return False else tryFastForwardTo refs
 		unless ff $ inRepo $
-			Git.commit "update" fullname (nub $ fullname:refs)
+			Git.commit merge_desc fullname (nub $ fullname:refs)
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
diff --git a/debian/changelog b/debian/changelog
index 9c2de7b083..ff735a79d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20111112) UNRELEASED; urgency=low
+
+  * merge: Improve commit messages to mention what was merged.
+
+ -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
+
 git-annex (3.20111111) unstable; urgency=low
 
   * Handle a case where an annexed file is moved into a gitignored directory,

From e9bfa8eaed3ff59a4c0bc8d4d677bc493177807c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 15:15:57 -0400
Subject: [PATCH 2501/8313] avoid unnecessary auto-merge when only changing a
 file in the branch.

Avoids doing auto-merging in commands that don't need fully current
information from the git-annex branch. In particular, git annex add no
longer needs to auto-merge. Affected commands: Anything that doesn't
look up data from the branch, but does write a change to it.

It might seem counterintuitive that we can change a value without first
making sure we have the current value. This optimisation works because
these two sequences are equivilant:

1. pull from remote
2. union merge
3. read file from branch
4. modify file and write to branch

vs.

1. read file from branch
2. modify file and write to branch
3. pull from remote
4. union merge

After either sequence, the git-annex branch contains the same logical content
for the modified file. (Possibly with lines in a different order or
additional old lines of course).
---
 Annex/Branch.hs                              | 25 ++++++++++++++++----
 debian/changelog                             |  3 +++
 doc/todo/avoid_unnecessary_union_merges.mdwn |  2 ++
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index e86a3d7d18..fad818fb0a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -244,9 +244,13 @@ siblingBranches = do
 		pair l = (head l, last l)
 		uref (a, _) (b, _) = a == b
 
-{- Applies a function to modifiy the content of a file. -}
+{- Applies a function to modifiy the content of a file.
+ -
+ - Note that this does not cause the branch to be merged, it only
+ - modifes the current content of the file on the branch.
+ -}
 change :: FilePath -> (String -> String) -> Annex ()
-change file a = lockJournal $ get file >>= return . a >>= set file
+change file a = lockJournal $ getStale file >>= return . a >>= set file
 
 {- Records new content of a file into the journal. -}
 set :: FilePath -> String -> Annex ()
@@ -259,13 +263,24 @@ set file content = do
  -
  - Returns an empty string if the file doesn't exist yet. -}
 get :: FilePath -> Annex String
-get file = fromcache =<< getCache file
+get = get' False
+
+{- Like get, but does not merge the branch, so the info returned may not
+ - reflect changes in remotes. (Changing the value this returns, and then
+ - merging is always the same as using get, and then changing its value.) -}
+getStale :: FilePath -> Annex String
+getStale = get' True
+
+get' :: Bool -> FilePath -> Annex String
+get' staleok file = fromcache =<< getCache file
 	where
 		fromcache (Just content) = return content
 		fromcache Nothing = fromjournal =<< getJournalFile file
 		fromjournal (Just content) = cache content
-		fromjournal Nothing = withIndexUpdate $
-			cache =<< catFile fullname file
+		fromjournal Nothing
+			| staleok = withIndex frombranch
+			| otherwise = withIndexUpdate $ frombranch >>= cache
+		frombranch = catFile fullname file
 		cache content = do
 			setCache file content
 			return content
diff --git a/debian/changelog b/debian/changelog
index ff735a79d3..76a29131db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (3.20111112) UNRELEASED; urgency=low
 
   * merge: Improve commit messages to mention what was merged.
+  * Avoid doing auto-merging in commands that don't need fully current
+    information from the git-annex branch. In particular, git annex add
+    no longer needs to auto-merge.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/todo/avoid_unnecessary_union_merges.mdwn b/doc/todo/avoid_unnecessary_union_merges.mdwn
index 67aa285799..5cd4b64373 100644
--- a/doc/todo/avoid_unnecessary_union_merges.mdwn
+++ b/doc/todo/avoid_unnecessary_union_merges.mdwn
@@ -16,3 +16,5 @@ occurs before it has done anything, so ctrl-c should not be a problem
 there.
 
 This is a delicate change, I need to take care.. --[[Joey]]
+
+> [[done]] (assuming I didn't miss any cases where this is not safe!) --[[Joey]] 

From cea65b9e5bf6bcc9a9350703dbbb0951c6e00c82 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 15:42:52 -0400
Subject: [PATCH 2502/8313] init: When run in an already initalized repository,
 and without a description specified, don't delete the old description.

---
 Command/Init.hs  | 2 +-
 debian/changelog | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Init.hs b/Command/Init.hs
index a6d72e4226..bbabdc4c25 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -27,5 +27,5 @@ start ws = do
 
 perform :: String -> CommandPerform
 perform description = do
-	initialize (Just description)
+	initialize $ if null description then Nothing else Just description
 	next $ return True
diff --git a/debian/changelog b/debian/changelog
index 76a29131db..a8ce33435c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * Avoid doing auto-merging in commands that don't need fully current
     information from the git-annex branch. In particular, git annex add
     no longer needs to auto-merge.
+  * init: When run in an already initalized repository, and without
+    a description specified, don't delete the old description. 
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From 04edae6791b4eddaa77dda2407264dc4434d74b7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 12 Nov 2011 17:45:12 -0400
Subject: [PATCH 2503/8313] Optimised union merging; now only runs git cat-file
 once.

---
 Annex/Branch.hs                        |  5 ++--
 Annex/CatFile.hs                       | 17 ++++++++----
 Command/Unused.hs                      |  2 +-
 Git/CatFile.hs                         | 38 ++++++++++++++++----------
 Git/UnionMerge.hs                      | 31 +++++++++++----------
 debian/changelog                       |  1 +
 doc/todo/optimise_git-annex_merge.mdwn |  4 +++
 7 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index fad818fb0a..20134003d3 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -149,7 +149,8 @@ update = onceonly $ do
 			 - documentation advises users not to directly
 			 - modify the branch.
 			 -}
-			inRepo $ \g -> Git.UnionMerge.merge_index g branches
+			h <- catFileHandle
+			inRepo $ \g -> Git.UnionMerge.merge_index h g branches
 		ff <- if dirty then return False else tryFastForwardTo refs
 		unless ff $ inRepo $
 			Git.commit merge_desc fullname (nub $ fullname:refs)
@@ -280,7 +281,7 @@ get' staleok file = fromcache =<< getCache file
 		fromjournal Nothing
 			| staleok = withIndex frombranch
 			| otherwise = withIndexUpdate $ frombranch >>= cache
-		frombranch = catFile fullname file
+		frombranch = L.unpack <$> catFile fullname file
 		cache content = do
 			setCache file content
 			return content
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 99cc519f51..0541f7269d 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -6,18 +6,25 @@
  -}
 
 module Annex.CatFile (
-	catFile
+	catFile,
+	catFileHandle
 ) where
 
+import qualified Data.ByteString.Lazy.Char8 as L
+
 import Common.Annex
 import qualified Git.CatFile
 import qualified Annex
 
-catFile :: String -> FilePath -> Annex String
-catFile branch file = maybe startup go =<< Annex.getState Annex.catfilehandle
+catFile :: String -> FilePath -> Annex L.ByteString
+catFile branch file = do
+	h <- catFileHandle
+	liftIO $ Git.CatFile.catFile h branch file
+
+catFileHandle :: Annex Git.CatFile.CatFileHandle
+catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle
 	where
 		startup = do
 			h <- inRepo Git.CatFile.catFileStart
 			Annex.changeState $ \s -> s { Annex.catfilehandle = Just h }
-			go h
-		go h = liftIO $ Git.CatFile.catFile h branch file
+			return h
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 9d56d1ff11..34d8ac232a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -197,7 +197,7 @@ getKeysReferencedInGit ref = do
 		findkeys c (l:ls)
 			| isSymLink (LsTree.mode l) = do
 				content <- catFile ref $ LsTree.file l
-				case fileKey (takeFileName content) of
+				case fileKey (takeFileName $ L.unpack content) of
 					Nothing -> findkeys c ls
 					Just k -> findkeys (k:c) ls
 			| otherwise = findkeys c ls
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 51fa585a82..83c1235086 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -9,13 +9,15 @@ module Git.CatFile (
 	CatFileHandle,
 	catFileStart,
 	catFileStop,
-	catFile
+	catFile,
+	catObject
 ) where
 
 import Control.Monad.State
 import System.Cmd.Utils
 import System.IO
-import qualified Data.ByteString.Char8 as B
+import qualified Data.ByteString.Char8 as S
+import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
 import Utility.SafeCommand
@@ -34,30 +36,38 @@ catFileStop (pid, from, to) = do
 	hClose from
 	forceSuccess pid
 
-{- Uses a running git cat-file read the content of a file from a branch.
- - Files that do not exist on the branch will have "" returned. -}
-catFile :: CatFileHandle -> String -> FilePath -> IO String
-catFile (_, from, to) branch file = do
-	hPutStrLn to want
+{- Reads a file from a specified branch. -}
+catFile :: CatFileHandle -> String -> FilePath -> IO L.ByteString
+catFile h branch file = catObject h (branch ++ ":" ++ file)
+
+{- Uses a running git cat-file read the content of an object.
+ - Objects that do not exist will have "" returned. -}
+catObject :: CatFileHandle -> String -> IO L.ByteString
+catObject (_, from, to) object = do
+	hPutStrLn to object
 	hFlush to
 	header <- hGetLine from
 	case words header of
-		[sha, blob, size]
+		[sha, objtype, size]
 			| length sha == Git.shaSize &&
-			  blob == "blob" -> handle size
+			  validobjtype objtype -> handle size
 			| otherwise -> empty
 		_
-			| header == want ++ " missing" -> empty
+			| header == object ++ " missing" -> empty
 			| otherwise -> error $ "unknown response from git cat-file " ++ header
 	where
-		want = branch ++ ":" ++ file
 		handle size = case reads size of
 			[(bytes, "")] -> readcontent bytes
 			_ -> empty
 		readcontent bytes = do
-			content <- B.hGet from bytes
+			content <- S.hGet from bytes
 			c <- hGetChar from
 			when (c /= '\n') $
 				error "missing newline from git cat-file"
-			return $ B.unpack content
-		empty = return ""
+			return $ L.fromChunks [content]
+		empty = return L.empty
+		validobjtype t
+			| t == "blob" = True
+			| t == "commit" = True
+			| t == "tree" = True
+			| otherwise = False
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 30778d0344..67e6fd951a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -21,6 +21,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
 import Git
+import Git.CatFile
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
@@ -30,14 +31,16 @@ import Git
 merge :: String -> String -> Repo -> IO ()
 merge x y repo = do
 	a <- ls_tree x repo
-	b <- merge_trees x y repo
+	h <- catFileStart repo
+	b <- merge_trees x y h repo
+	catFileStop h
 	update_index repo (a++b)
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
-merge_index :: Repo -> [String] -> IO ()
-merge_index repo bs =
-	update_index repo =<< concat <$> mapM (`merge_tree_index` repo) bs
+merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
+merge_index h repo bs =
+	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b h repo) bs
 
 {- Feeds a list into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
@@ -60,22 +63,22 @@ ls_tree x = pipeNullSplit params
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> Repo -> IO [String]
-merge_trees x y = calc_merge $ "diff-tree":diff_opts ++ [x, y]
+merge_trees :: String -> String -> CatFileHandle -> Repo -> IO [String]
+merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> Repo -> IO [String]
-merge_tree_index x = calc_merge $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index :: String -> CatFileHandle -> Repo -> IO [String]
+merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: [String] -> Repo -> IO [String]
-calc_merge differ repo = do
+calc_merge :: CatFileHandle -> [String] -> Repo -> IO [String]
+calc_merge h differ repo = do
 	diff <- pipeNullSplit (map Param differ) repo
-	l <- mapM (\p -> mergeFile p repo) (pairs diff)
+	l <- mapM (\p -> mergeFile p h repo) (pairs diff)
 	return $ catMaybes l
 	where
 		pairs [] = []
@@ -97,12 +100,12 @@ hashObject content repo = getSha subcmd $ do
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: (String, FilePath) -> Repo -> IO (Maybe String)
-mergeFile (info, file) repo = case filter (/= nullsha) [asha, bsha] of
+mergeFile :: (String, FilePath) -> CatFileHandle -> Repo -> IO (Maybe String)
+mergeFile (info, file) h repo = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
-		content <- pipeRead (map Param ("show":shas)) repo
+		content <- L.concat <$> mapM (catObject h) shas
 		sha <- hashObject (unionmerge content) repo
 		return $ Just $ update_index_line sha file
 	where
diff --git a/debian/changelog b/debian/changelog
index a8ce33435c..7ff819a14b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     no longer needs to auto-merge.
   * init: When run in an already initalized repository, and without
     a description specified, don't delete the old description. 
+  * Optimised union merging; now only runs git cat-file once.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
index a2cdfb15fe..2df196cfd4 100644
--- a/doc/todo/optimise_git-annex_merge.mdwn
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -8,6 +8,10 @@ Instead, I'd like a way to stream multiple objects into git using stdin.
 Sometime, should look at either extending git-hash-object to support that,
 or possibly look at using git-fast-import instead.
 
+--- 
+
 `git-annex merge` also runs `git show` once per file that needs to be
 merged. This could be reduced to a single call to `git-cat-file --batch`,
 There is already a Git.CatFile library that can do this easily. --[[Joey]]
+
+> This is now done, part above remains todo. --[[Joey]] 

From aa4fbbdd33c4d584b734476a341a0c38980281a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 16:14:17 -0400
Subject: [PATCH 2504/8313] status: Now displays trusted, untrusted, and
 semitrusted repositories separately.

---
 Command/Status.hs                                | 16 +++++++++++-----
 debian/changelog                                 |  2 ++
 ...hings_like_description__44___trust_level.mdwn |  3 +++
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 53d64d0428..e66f59858d 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -24,6 +24,7 @@ import Annex.Content
 import Types.Key
 import Backend
 import Logs.UUID
+import Logs.Trust
 import Remote
 
 -- a named computation that produces a statistic
@@ -52,7 +53,9 @@ stats :: [Stat]
 stats = 
 	[ supported_backends
 	, supported_remote_types
-	, remote_list
+	, remote_list Trusted "trusted"
+	, remote_list UnTrusted "untrusted"
+	, remote_list SemiTrusted "semitrusted"
 	, tmp_size
 	, bad_data_size
 	, local_annex_keys
@@ -90,10 +93,13 @@ supported_remote_types :: Stat
 supported_remote_types = stat "supported remote types" $
 	return $ unwords $ map R.typename Remote.remoteTypes
 
-remote_list :: Stat
-remote_list = stat "known repositories" $ lift $ do
-	s <- prettyPrintUUIDs "repos" =<< M.keys <$> uuidMap
-	return $ '\n':init s
+remote_list :: TrustLevel -> String -> Stat
+remote_list level desc = stat n $ lift $ do
+	us <- M.keys <$> uuidMap
+	s <- prettyPrintUUIDs n =<< fst <$> trustPartition level us
+	return $ if null s then "none" else '\n':init s
+	where
+		n = desc ++ " repositories"
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $
diff --git a/debian/changelog b/debian/changelog
index 7ff819a14b..b49967fe9f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * init: When run in an already initalized repository, and without
     a description specified, don't delete the old description. 
   * Optimised union merging; now only runs git cat-file once.
+  * status: Now displays trusted, untrusted, and semitrusted repositories
+    separately.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
index 6404393a89..d158850cd6 100644
--- a/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
+++ b/doc/bugs/wishlist:_query_things_like_description__44___trust_level.mdwn
@@ -1 +1,4 @@
 It would be helpful to have a way to query things like a repository's description and trust level, without having to poke in the git-annex branch.  For example, "git annex describe ." currently clears the description but could print the current one instead.
+
+> `git annex status` now breaks down the repository list by type. [[done]]
+> --[[Joey]] 

From 364981ad924e7f4fdb92121dba52318bce3644d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 16:15:48 -0400
Subject: [PATCH 2505/8313] probably makes sense to list semitrusted before
 untrusted

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index e66f59858d..3fddb599d5 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -54,8 +54,8 @@ stats =
 	[ supported_backends
 	, supported_remote_types
 	, remote_list Trusted "trusted"
-	, remote_list UnTrusted "untrusted"
 	, remote_list SemiTrusted "semitrusted"
+	, remote_list UnTrusted "untrusted"
 	, tmp_size
 	, bad_data_size
 	, local_annex_keys

From 522df3da5800f821f1d7e7ddb6ddf70befa8f8ac Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Mon, 14 Nov 2011 22:30:02 +0000
Subject: [PATCH 2506/8313]

---
 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn

diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn
new file mode 100644
index 0000000000..f1aa5cc06d
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis.mdwn
@@ -0,0 +1 @@
+I just started experimenting with git annex, and I found that I would like to have a way to figure out metadata (well, size. Maybe modification date) of a non-local file. I first checked if there is "git annex ls" (which could list known files in an ls-like way) and found "git annex whereis" as somewhat a replacement, but it does not give metadata information.

From 4d72c1b69c4423bd13c8d407cbedca3b985c3585 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 14 Nov 2011 22:46:35 +0000
Subject: [PATCH 2507/8313] Added a comment

---
 .../comment_1_7fba10b85f4d9289c7782eccef46949e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment

diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment
new file mode 100644
index 0000000000..379b9f976b
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_1_7fba10b85f4d9289c7782eccef46949e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-14T22:46:35Z"
+ content="""
+When  I want that, I ls -l the file and look at the symlink to the key. Ie, in SHA1-s10481423--efc7eec0d711212842cd6bb8f957e1628146d6ed the size is 10481423 bytes. 
+"""]]

From 02f1d5467ad05315905b9cb90307049ababb4a63 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 14 Nov 2011 22:48:03 +0000
Subject: [PATCH 2508/8313] Added a comment

---
 .../comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment

diff --git a/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment
new file mode 100644
index 0000000000..3dd14bf010
--- /dev/null
+++ b/doc/forum/git_annex_ls___47___metadata_in_git_annex_whereis/comment_2_7dcec124ea7d0291ed40d80e2ffd5c7e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-14T22:48:03Z"
+ content="""
+It might make sense to put this functionality in git annex find. Perhaps a format string with a %s for example. 
+"""]]

From bfe38f8ff16dc6ccc32d545e6fed87817caa26cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 19:27:00 -0400
Subject: [PATCH 2509/8313] status --json --fast for esc

* status: Fix --json mode (only the repository lists are currently
  displayed)
* status: --fast is back
---
 Command/Status.hs  | 22 ++++++++++++++++------
 Messages.hs        | 14 +++++++++++++-
 Messages/JSON.hs   |  8 ++++++--
 debian/changelog   |  3 +++
 doc/git-annex.mdwn |  4 +++-
 5 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 3fddb599d5..39d9400fbd 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -18,6 +18,7 @@ import qualified Types.Remote as R
 import qualified Remote
 import qualified Command.Unused
 import qualified Git
+import qualified Annex
 import Command
 import Utility.DataUnits
 import Annex.Content
@@ -49,14 +50,17 @@ seek = [withNothing start]
 {- Order is significant. Less expensive operations, and operations
  - that share data go together.
  -}
-stats :: [Stat]
-stats = 
+fast_stats :: [Stat]
+fast_stats = 
 	[ supported_backends
 	, supported_remote_types
 	, remote_list Trusted "trusted"
 	, remote_list SemiTrusted "semitrusted"
 	, remote_list UnTrusted "untrusted"
-	, tmp_size
+	]
+slow_stats :: [Stat]
+slow_stats = 
+	[ tmp_size
 	, bad_data_size
 	, local_annex_keys
 	, local_annex_size
@@ -67,6 +71,10 @@ stats =
 
 start :: CommandStart
 start = do
+	showStart "status" "."
+	showWith $ liftIO $ putStrLn ""
+	fast <- Annex.getState Annex.fast
+	let stats = if fast then fast_stats else fast_stats ++ slow_stats
 	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
 	stop
 
@@ -80,9 +88,11 @@ showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
 		calc (Just (desc, a)) = do
-			liftIO $ putStr $ desc ++ ": "
-			liftIO $ hFlush stdout
-			liftIO . putStrLn =<< a
+			r <- a -- run first, it may produce JSON
+			lift . showWith $ do
+				liftIO $ putStr $ desc ++ ": "
+				liftIO $ hFlush stdout
+				liftIO $ putStrLn r
 		calc Nothing = return ()
 
 supported_backends :: Stat
diff --git a/Messages.hs b/Messages.hs
index 6f4880e2d5..d7eabccbb2 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -20,6 +20,8 @@ module Messages (
 	warning,
 	indent,
 	maybeShowJSON,
+	showWith,
+	
 	setupConsole
 ) where
 
@@ -31,7 +33,7 @@ import qualified Annex
 import qualified Messages.JSON as JSON
 
 showStart :: String -> String -> Annex ()
-showStart command file = handle (JSON.start command file) $
+showStart command file = handle (JSON.start command $ Just file) $
 	flushed $ putStr $ command ++ " " ++ file ++ " "
 
 showNote :: String -> Annex ()
@@ -111,6 +113,16 @@ handle json normal = do
 maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
 maybeShowJSON v = handle (JSON.add v) q
 
+{- Performs an a action (such as displaying something) only when
+ - not in json mode, and not quiet. -}
+showWith :: Annex () -> Annex ()
+showWith a = do
+	output <- Annex.getState Annex.output
+	case output of
+		Annex.NormalOutput -> a
+		Annex.QuietOutput -> q
+		Annex.JSONOutput -> q
+
 q :: Monad m => m ()
 q = return ()
 
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index fb95f550e8..a325ef130b 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -16,8 +16,12 @@ import Text.JSON
 
 import qualified Utility.JSONStream as Stream
 
-start :: String -> String -> IO ()
-start command file = putStr $ Stream.start [("command", command), ("file", file)]
+start :: String -> Maybe String -> IO ()
+start command file =
+	putStr $ Stream.start $ ("command", command) : filepart file
+	where
+		filepart Nothing = []
+		filepart (Just f) = [("file", f)]
 
 end :: Bool -> IO ()
 end b = putStr $ Stream.add [("success", b)] ++ Stream.end
diff --git a/debian/changelog b/debian/changelog
index b49967fe9f..aa52730bca 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,9 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * Optimised union merging; now only runs git cat-file once.
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
+  * status: Fix --json mode (only the repository lists are currently
+    displayed)
+  * status: --fast is back
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index fdd8dd1c19..35ba25115d 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -240,7 +240,9 @@ subdirectories).
 * status
 
   Displays some statistics and other information, including how much data
-  is in the annex.
+  is in the annex and a list of all known repositories.
+
+  To only show the data that can be gathered quickly, use --fast.
 
 * map
 

From 2412b7e6896b967bb3f8ec8cf5e6ccf3094a3ad1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 14 Nov 2011 19:29:35 -0400
Subject: [PATCH 2510/8313] fix exit status so json gets terminated properly

---
 Command/Status.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 39d9400fbd..2d59965072 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -76,7 +76,7 @@ start = do
 	fast <- Annex.getState Annex.fast
 	let stats = if fast then fast_stats else fast_stats ++ slow_stats
 	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
-	stop
+	next $ next $ return True
 
 stat :: String -> StatState String -> Stat
 stat desc a = return $ Just (desc, a)

From c093839a40034a6e3fa967a18459a52ea326b867 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 00:41:08 +0000
Subject: [PATCH 2511/8313]

---
 .../Trouble_initializing_git_annex_on_NFS.mdwn     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
new file mode 100644
index 0000000000..23977af34e
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
@@ -0,0 +1,14 @@
+The following occurs in a directory that is shared on an NFS server:
+
+    /media/mybook/movies $ git init
+    Initialized empty Git repository in /media/mybook/movies/.git/
+    /media/mybook/movies $ git annex init mybook-movies
+    init mybook-movies 
+    git-annex: waitToSetLock: resource exhausted (No locks available)
+    failed
+    git-annex: init: 1 failed
+    /media/mybook/movies $
+
+This happens reliably.  Is there any way around it?  I have shell
+access on the NFS server, but it is a NAS, so I don't think it is
+capable of running git-annex.

From 6368c79fe41abc195e809340d10d2b1714188bd4 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 00:47:57 +0000
Subject: [PATCH 2512/8313] Fix typo

---
 doc/tips/centralized_git_repository_tutorial.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/centralized_git_repository_tutorial.mdwn b/doc/tips/centralized_git_repository_tutorial.mdwn
index 977a583cca..00283829fd 100644
--- a/doc/tips/centralized_git_repository_tutorial.mdwn
+++ b/doc/tips/centralized_git_repository_tutorial.mdwn
@@ -131,7 +131,7 @@ which will broadcast its updated location information.
 ## take it farther
 
 Of course you can create as many checkouts as you desire. If you have a
-desktop matchine too, you can make a checkout there, and use `git remote
+desktop machine too, you can make a checkout there, and use `git remote
 add` to also let your desktop access the backup repository. 
 
 You can add remotes for each direct connection between machines you find you

From 019373f827309e4f4a1cf694a50270142e26aa6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 00:30:00 -0400
Subject: [PATCH 2513/8313] better status output

---
 Command/Status.hs | 15 ++++++---------
 Messages.hs       | 42 ++++++++++++++++++++++++++----------------
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 2d59965072..4f7529732d 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -71,12 +71,12 @@ slow_stats =
 
 start :: CommandStart
 start = do
-	showStart "status" "."
-	showWith $ liftIO $ putStrLn ""
 	fast <- Annex.getState Annex.fast
 	let stats = if fast then fast_stats else fast_stats ++ slow_stats
-	evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
-	next $ next $ return True
+	showCustom "status" $ do
+		evalStateT (mapM_ showStat stats) (StatInfo Nothing Nothing)
+		return True
+	stop
 
 stat :: String -> StatState String -> Stat
 stat desc a = return $ Just (desc, a)
@@ -88,11 +88,8 @@ showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
 		calc (Just (desc, a)) = do
-			r <- a -- run first, it may produce JSON
-			lift . showWith $ do
-				liftIO $ putStr $ desc ++ ": "
-				liftIO $ hFlush stdout
-				liftIO $ putStrLn r
+			(lift . showHeader) desc
+			lift . showRaw =<< a
 		calc Nothing = return ()
 
 supported_backends :: Stat
diff --git a/Messages.hs b/Messages.hs
index d7eabccbb2..57b7068041 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -1,6 +1,6 @@
 {- git-annex output messages
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -20,7 +20,9 @@ module Messages (
 	warning,
 	indent,
 	maybeShowJSON,
-	showWith,
+	showCustom,
+	showHeader,
+	showRaw,
 	
 	setupConsole
 ) where
@@ -88,6 +90,28 @@ warning' w = do
 indent :: String -> String
 indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
 
+{- Shows a JSON value only when in json mode. -}
+maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
+maybeShowJSON v = handle (JSON.add v) q
+
+{- Performs an action that outputs nonstandard/customized output, and
+ - in JSON mode wraps its output in JSON.start and JSON.end, so it's
+ - a complete JSON document.
+ - This is only needed when showStart and showEndOk is not used. -}
+showCustom :: String -> Annex Bool -> Annex ()
+showCustom command a = do
+	handle (JSON.start command Nothing) q
+	r <- a
+	handle (JSON.end r) q
+
+showHeader :: String -> Annex ()
+showHeader h = handle q $ do
+	putStr $ h ++ ": "
+	hFlush stdout
+
+showRaw :: String -> Annex ()
+showRaw s = handle q $ putStrLn s
+
 {- By default, haskell honors the user's locale in its output to stdout
  - and stderr. While that's great for proper unicode support, for git-annex
  - all that's really needed is the ability to display simple messages
@@ -109,20 +133,6 @@ handle json normal = do
 		Annex.QuietOutput -> q
 		Annex.JSONOutput -> liftIO json
 
-{- Shows a JSON value only when in json mode. -}
-maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
-maybeShowJSON v = handle (JSON.add v) q
-
-{- Performs an a action (such as displaying something) only when
- - not in json mode, and not quiet. -}
-showWith :: Annex () -> Annex ()
-showWith a = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> a
-		Annex.QuietOutput -> q
-		Annex.JSONOutput -> q
-
 q :: Monad m => m ()
 q = return ()
 

From def0788698f9ed29856aa5b608356ae7b298e3f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 00:33:54 -0400
Subject: [PATCH 2514/8313] show number of repos

---
 Command/Status.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 4f7529732d..2c0afa62c4 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -103,8 +103,9 @@ supported_remote_types = stat "supported remote types" $
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ lift $ do
 	us <- M.keys <$> uuidMap
-	s <- prettyPrintUUIDs n =<< fst <$> trustPartition level us
-	return $ if null s then "none" else '\n':init s
+	rs <- fst <$> trustPartition level us
+	s <- prettyPrintUUIDs n rs
+	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
 	where
 		n = desc ++ " repositories"
 

From a6091dc27180f0db9df647fa08460a2c0f03d3e5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 15 Nov 2011 04:40:35 +0000
Subject: [PATCH 2515/8313] Added a comment

---
 ...comment_1_e26952373150d63b8a5d3643a2762de1._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment
new file mode 100644
index 0000000000..8e951ab7c9
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_1_e26952373150d63b8a5d3643a2762de1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-15T04:40:35Z"
+ content="""
+git-annex uses locking to avoid problems if multiple processes are run at the same time.
+
+I just tested on NFS, with Linux on the server and client, and it works ok. It seems your NFS client (or server) must not support fncl locking. What OS is your NAS running?
+"""]]

From cfc518190c08f88ea7cc4480d6d5f1cf9fab351b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 15 Nov 2011 04:46:13 +0000
Subject: [PATCH 2516/8313] Added a comment

---
 ...comment_2_f80b10ed395738e50e345fc22c708ae5._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment
new file mode 100644
index 0000000000..bd302e6bef
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_2_f80b10ed395738e50e345fc22c708ae5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-15T04:46:13Z"
+ content="""
+You might try mounting your NAS with the mount option `local_lock=all`
+
+This will keep the lock files on your (I assume linux) client. If you do this make sure you don't have another client using git-annex in the same NFS directory.
+"""]]

From eb214f719cae6c322f1fc57ad3a4d8a710c94820 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 05:14:05 +0000
Subject: [PATCH 2517/8313] Added a comment

---
 .../comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment
new file mode 100644
index 0000000000..b95c795eab
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_3_f99e0f05950fc2fc80fdecd35e17012c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 3"
+ date="2011-11-15T05:14:03Z"
+ content="""
+I did a bit of research and my NAS had ancient NFS software on it.  I upgraded that and things are now working as expected. Sorry for the noise.
+"""]]

From 8d9d94f90c26117e65f949cc8d087d1045b7d5b6 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 05:15:47 +0000
Subject: [PATCH 2518/8313] Added a comment

---
 .../comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
new file mode 100644
index 0000000000..5a741b8c95
--- /dev/null
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 4"
+ date="2011-11-15T05:15:47Z"
+ content="""
+I did a bit of research and my NAS had ancient NFS software on it.  I upgraded that and things are now working as expected. Sorry for the noise.
+"""]]

From d99fa3ec4ec647a62177821f666ed5398fe95b0d Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Tue, 15 Nov 2011 05:22:09 +0000
Subject: [PATCH 2519/8313] removed

---
 .../comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment   | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment b/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
deleted file mode 100644
index 5a741b8c95..0000000000
--- a/doc/bugs/Trouble_initializing_git_annex_on_NFS/comment_4_4031fcd0b5c443e8db0304eecb8b4154._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://cgray.myopenid.com/"
- nickname="cgray"
- subject="comment 4"
- date="2011-11-15T05:15:47Z"
- content="""
-I did a bit of research and my NAS had ancient NFS software on it.  I upgraded that and things are now working as expected. Sorry for the noise.
-"""]]

From 3c4537111509ea61138592a254a3a55a185cee23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 01:53:28 -0400
Subject: [PATCH 2520/8313] close as resolved

---
 doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
index 23977af34e..8eb20baf97 100644
--- a/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
+++ b/doc/bugs/Trouble_initializing_git_annex_on_NFS.mdwn
@@ -12,3 +12,5 @@ The following occurs in a directory that is shared on an NFS server:
 This happens reliably.  Is there any way around it?  I have shell
 access on the NFS server, but it is a NAS, so I don't think it is
 capable of running git-annex.
+
+[[done]]

From 7d05ca1d6d01f7e9f8b2acdebac2b656e178a32c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 14:06:38 -0400
Subject: [PATCH 2521/8313] Fix support for insteadOf url remapping. Closes:
 #644278

---
 Git.hs           | 7 ++++---
 debian/changelog | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Git.hs b/Git.hs
index 5ceaa67f76..3ff42d28f5 100644
--- a/Git.hs
+++ b/Git.hs
@@ -523,13 +523,14 @@ genRemote s repo = gen $ calcloc s
 			| null insteadofs = l
 			| otherwise = replacement ++ drop (length replacement) l
 			where
-				replacement = take (length bestkey - length prefix) bestkey
+				replacement = drop (length "url.") $
+					take (length bestkey - length suffix) bestkey
 				bestkey = fst $ maximumBy longestvalue insteadofs
 				longestvalue (_, a) (_, b) = compare b a
 				insteadofs = filterconfig $ \(k, v) -> 
-					endswith prefix k &&
+					endswith suffix k &&
 					startswith v l
-				prefix = ".insteadof"
+				suffix = ".insteadof"
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
diff --git a/debian/changelog b/debian/changelog
index aa52730bca..c145b14f9a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * status: Fix --json mode (only the repository lists are currently
     displayed)
   * status: --fast is back
+  * Fix support for insteadOf url remapping. Closes: #644278
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From 80a0be5116ada2c718fa6db0a90100d24660beec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 19:03:25 -0400
Subject: [PATCH 2522/8313] further insteadOf fix

---
 Git.hs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Git.hs b/Git.hs
index 3ff42d28f5..3f3f746324 100644
--- a/Git.hs
+++ b/Git.hs
@@ -521,16 +521,17 @@ genRemote s repo = gen $ calcloc s
 		-- insteadof config can rewrite remote location
 		calcloc l
 			| null insteadofs = l
-			| otherwise = replacement ++ drop (length replacement) l
+			| otherwise = replacement ++ drop (length bestvalue) l
 			where
-				replacement = drop (length "url.") $
+				replacement = drop (length prefix) $
 					take (length bestkey - length suffix) bestkey
-				bestkey = fst $ maximumBy longestvalue insteadofs
+				(bestkey, bestvalue) = maximumBy longestvalue insteadofs
 				longestvalue (_, a) (_, b) = compare b a
 				insteadofs = filterconfig $ \(k, v) -> 
+					startswith prefix k &&
 					endswith suffix k &&
 					startswith v l
-				suffix = ".insteadof"
+				(prefix, suffix) = ("url." , ".insteadof")
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir

From 5c5ff2cb78230ca6311d49dc2573579af49d8c37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 19:17:07 -0400
Subject: [PATCH 2523/8313] better error message

---
 git-annex-shell.hs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 57f6b29162..658eddd77f 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -40,14 +40,17 @@ cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
 
 options :: [OptDescr (Annex ())]
 options = commonOptions ++
-	[ Option [] ["uuid"] (ReqArg check paramUUID) "repository uuid"
+	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid"
 	]
 	where
-		check expected = do
-			u <- getUUID
-			when (u /= toUUID expected) $ error $
-				"expected repository UUID " ++ expected
-					++ " but found UUID " ++ fromUUID u
+		checkuuid expected = getUUID >>= check
+			where
+				check u | u == toUUID expected = return ()
+				check NoUUID = unexpected "uninitialized repository"
+				check u = unexpected $ "UUID " ++ fromUUID u
+				unexpected s = error $
+					"expected repository UUID " ++
+					expected ++ " but found " ++ s
 
 header :: String
 header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"

From b76dc2d210889a77160a834e3849e3040ef69b12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 22:01:32 -0400
Subject: [PATCH 2524/8313] avoid space leak writing merge

This reduces the memory use of a merge by 1/3rd. The space leak was
apparently because the whole update-index input was generated strictly, not
lazily.

I wondered if the change to ByteStrings contributed to this, due to the
need to convert with L.pack here. But going back to the old code, I still
see a much similar leak, and worse performance besides due to it not using
ByteStrings.

The fix is to just hPutStr the lines repeatedly. (Note the \0 is written
separately, to avoid allocation overheads in adding it to the string.)
The Git.pipeWrite interface is probably just wrong for any large inputs to
git. This was the only place using it for input of any size.

There is still at least one other space leak in the merge code.
---
 Git/UnionMerge.hs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 67e6fd951a..460c767168 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -46,10 +46,13 @@ merge_index h repo bs =
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index repo l = togit ["update-index", "-z", "--index-info"] (join "\0" l)
+update_index repo l = do
+	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
+	mapM_ (\s -> hPutStr h s >> hPutStr h "\0") l
+	hClose h
+	forceSuccess p
 	where
-		togit ps content = pipeWrite (map Param ps) (L.pack content) repo
-			>>= forceSuccess
+		params = map Param ["update-index", "-z", "--index-info"]
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}

From 922e9af5281b01709e5fa631ebe048c7da7c4d71 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 22:40:40 -0400
Subject: [PATCH 2525/8313] cleanup

---
 Git/UnionMerge.hs | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 460c767168..493411a069 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -16,7 +16,6 @@ module Git.UnionMerge (
 import System.Cmd.Utils
 import Data.List
 import Data.Maybe
-import Data.String.Utils
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
@@ -88,18 +87,6 @@ calc_merge h differ repo = do
 		pairs (_:[]) = error "calc_merge parse error"
 		pairs (a:b:rest) = (a,b):pairs rest
 
-{- Injects some content into git, returning its hash. -}
-hashObject :: L.ByteString -> Repo -> IO String
-hashObject content repo = getSha subcmd $ do
-	(h, s) <- pipeWriteRead (map Param params) content repo
-	L.length s `seq` do
-		forceSuccess h
-		reap -- XXX unsure why this is needed
-		return $ L.unpack s
-	where
-		subcmd = "hash-object"
-		params = [subcmd, "-w", "--stdin"]
-
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
@@ -115,3 +102,15 @@ mergeFile (info, file) h repo = case filter (/= nullsha) [asha, bsha] of
 		[_colonamode, _bmode, asha, bsha, _status] = words info
 		nullsha = replicate shaSize '0'
 		unionmerge = L.unlines . nub . L.lines
+
+{- Injects some content into git, returning its hash. -}
+hashObject :: L.ByteString -> Repo -> IO String
+hashObject content repo = getSha subcmd $ do
+	(h, s) <- pipeWriteRead (map Param params) content repo
+	L.length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return $ L.unpack s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-w", "--stdin"]

From eb0c8c955c57d4161b86dd9ed48c93036f9b83a0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Wed, 16 Nov 2011 03:22:29 +0000
Subject: [PATCH 2526/8313]

---
 .../git_annex_version_should_without_being_in_a_repo_.mdwn   | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn

diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
new file mode 100644
index 0000000000..0bae8bdb0e
--- /dev/null
+++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
@@ -0,0 +1,5 @@
+was checking the version of git-annex on a machine before cloning a repo...
+
+    $ git annex version
+    git-annex: Not in a git repository.
+

From e92534e5b5514a1cf4ce536974aeac807d849e7d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Wed, 16 Nov 2011 03:24:31 +0000
Subject: [PATCH 2527/8313] Added a comment

---
 .../comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment

diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment b/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment
new file mode 100644
index 0000000000..ab30d8a452
--- /dev/null
+++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_/comment_1_e7b26eeb1a765fd83280ef907c0deef2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2011-11-16T03:24:30Z"
+ content="""
+oh, and that probably goes for 'help' and other subcommands as well.
+"""]]

From 21a925dcf1ebe088b5c64da0ce159ffb6d535f04 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 23:28:01 -0400
Subject: [PATCH 2528/8313] merge: Now runs in constant space.

Before, a merge was first calculated, by running various actions that
called git and built up a list of lines, which were at the end sent
to git update-index. This necessarily used space proportional to the size
of the diff between the trees being merged.

Now, lines are streamed into git update-index from each of the actions in
turn.

Runtime size of git-annex merge when merging 50000 location log files
drops from around 100 mb to a constant 4 mb.

Presumably it runs quite a lot faster, too.
---
 Annex/Branch.hs   |  3 ++-
 Git/UnionMerge.hs | 55 ++++++++++++++++++++++++++---------------------
 debian/changelog  |  1 +
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 20134003d3..ae33f66cf0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -56,7 +56,8 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.ls_tree fullname g >>= Git.UnionMerge.update_index g
+genIndex g = Git.UnionMerge.update_index_via g
+	[Git.UnionMerge.ls_tree fullname g]
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 493411a069..c86d699795 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -9,13 +9,13 @@ module Git.UnionMerge (
 	merge,
 	merge_index,
 	update_index,
+	update_index_via,
 	update_index_line,
 	ls_tree
 ) where
 
 import System.Cmd.Utils
 import Data.List
-import Data.Maybe
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
@@ -29,47 +29,56 @@ import Git.CatFile
  -}
 merge :: String -> String -> Repo -> IO ()
 merge x y repo = do
-	a <- ls_tree x repo
 	h <- catFileStart repo
-	b <- merge_trees x y h repo
+	update_index_via repo
+		[ ls_tree x repo
+		, merge_trees x y h repo
+		]
 	catFileStop h
-	update_index repo (a++b)
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
 merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
 merge_index h repo bs =
-	update_index repo =<< concat <$> mapM (\b -> merge_tree_index b h repo) bs
+	update_index_via repo $ map (\b -> merge_tree_index b h repo) bs
 
-{- Feeds a list into update-index. Later items in the list can override
+update_index :: Repo -> [String] -> IO ()
+update_index repo ls = update_index_via repo [\h -> mapM_ (sendContent h) ls]
+
+{- Feeds content into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
-update_index :: Repo -> [String] -> IO ()
-update_index repo l = do
+update_index_via :: Repo -> [Handle -> IO ()] -> IO ()
+update_index_via repo ls = do
 	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
-	mapM_ (\s -> hPutStr h s >> hPutStr h "\0") l
+	forM_ ls $ \l -> l h
 	hClose h
 	forceSuccess p
 	where
 		params = map Param ["update-index", "-z", "--index-info"]
 
+sendContent :: Handle -> String -> IO ()
+sendContent h s = do
+	hPutStr h s
+	hPutStr h "\0"
+
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
 update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
-{- Gets the contents of a tree in a format suitable for update_index. -}
-ls_tree :: String -> Repo -> IO [String]
-ls_tree x = pipeNullSplit params
+{- Gets the contents of a tree. -}
+ls_tree :: String -> Repo -> Handle -> IO ()
+ls_tree x repo h = mapM_ (sendContent h) =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> CatFileHandle -> Repo -> IO [String]
+merge_trees :: String -> String -> CatFileHandle -> Repo -> Handle -> IO ()
 merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> CatFileHandle -> Repo -> IO [String]
+merge_tree_index :: String -> CatFileHandle -> Repo -> Handle -> IO ()
 merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
@@ -77,21 +86,19 @@ diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: CatFileHandle -> [String] -> Repo -> IO [String]
-calc_merge h differ repo = do
-	diff <- pipeNullSplit (map Param differ) repo
-	l <- mapM (\p -> mergeFile p h repo) (pairs diff)
-	return $ catMaybes l
+calc_merge :: CatFileHandle -> [String] -> Repo -> Handle -> IO ()
+calc_merge ch differ repo ih = pipeNullSplit (map Param differ) repo >>= go
 	where
-		pairs [] = []
-		pairs (_:[]) = error "calc_merge parse error"
-		pairs (a:b:rest) = (a,b):pairs rest
+		go [] = return ()
+		go (info:file:rest) = mergeFile info file ch repo >>=
+			maybe (go rest) (\l -> sendContent ih l >> go rest)
+		go (_:[]) = error "calc_merge parse error"
 
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
-mergeFile :: (String, FilePath) -> CatFileHandle -> Repo -> IO (Maybe String)
-mergeFile (info, file) h repo = case filter (/= nullsha) [asha, bsha] of
+mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
+mergeFile info file h repo = case filter (/= nullsha) [asha, bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
diff --git a/debian/changelog b/debian/changelog
index c145b14f9a..37578f5975 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     displayed)
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
+  * merge: Now runs in constant space.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From e83b966eb5197171a6af46595049d661c4229c58 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 Nov 2011 23:48:50 -0400
Subject: [PATCH 2529/8313] cleanup

---
 Git/UnionMerge.hs                      | 39 ++++++++++++++------------
 doc/todo/optimise_git-annex_merge.mdwn |  6 ++++
 2 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index c86d699795..a2344e59db 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -42,25 +42,27 @@ merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
 merge_index h repo bs =
 	update_index_via repo $ map (\b -> merge_tree_index b h repo) bs
 
-update_index :: Repo -> [String] -> IO ()
-update_index repo ls = update_index_via repo [\h -> mapM_ (sendContent h) ls]
-
 {- Feeds content into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
-update_index_via :: Repo -> [Handle -> IO ()] -> IO ()
-update_index_via repo ls = do
+update_index :: Repo -> [String] -> IO ()
+update_index repo ls = update_index_via repo [\s -> mapM_ s ls]
+
+type Streamer = (String -> IO ()) -> IO ()
+
+{- Streams content into update-index. -}
+update_index_via :: Repo -> [Streamer] -> IO ()
+update_index_via repo as = do
 	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
-	forM_ ls $ \l -> l h
+	forM_ as (stream h)
 	hClose h
 	forceSuccess p
 	where
 		params = map Param ["update-index", "-z", "--index-info"]
-
-sendContent :: Handle -> String -> IO ()
-sendContent h s = do
-	hPutStr h s
-	hPutStr h "\0"
+		stream h a = a (streamer h)
+		streamer h s = do
+			hPutStr h s
+			hPutStr h "\0"
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
@@ -68,17 +70,17 @@ update_index_line :: String -> FilePath -> String
 update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
 
 {- Gets the contents of a tree. -}
-ls_tree :: String -> Repo -> Handle -> IO ()
-ls_tree x repo h = mapM_ (sendContent h) =<< pipeNullSplit params repo
+ls_tree :: String -> Repo -> Streamer
+ls_tree x repo streamer = mapM_ streamer =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> CatFileHandle -> Repo -> Handle -> IO ()
+merge_trees :: String -> String -> CatFileHandle -> Repo -> Streamer
 merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> CatFileHandle -> Repo -> Handle -> IO ()
+merge_tree_index :: String -> CatFileHandle -> Repo -> Streamer
 merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
@@ -86,12 +88,13 @@ diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
  - and returning a list suitable for update_index. -}
-calc_merge :: CatFileHandle -> [String] -> Repo -> Handle -> IO ()
-calc_merge ch differ repo ih = pipeNullSplit (map Param differ) repo >>= go
+calc_merge :: CatFileHandle -> [String] -> Repo -> Streamer
+calc_merge ch differ repo streamer = gendiff >>= go
 	where
+		gendiff = pipeNullSplit (map Param differ) repo
 		go [] = return ()
 		go (info:file:rest) = mergeFile info file ch repo >>=
-			maybe (go rest) (\l -> sendContent ih l >> go rest)
+			maybe (go rest) (\l -> streamer l >> go rest)
 		go (_:[]) = error "calc_merge parse error"
 
 {- Given an info line from a git raw diff, and the filename, generates
diff --git a/doc/todo/optimise_git-annex_merge.mdwn b/doc/todo/optimise_git-annex_merge.mdwn
index 2df196cfd4..91d18ebd77 100644
--- a/doc/todo/optimise_git-annex_merge.mdwn
+++ b/doc/todo/optimise_git-annex_merge.mdwn
@@ -15,3 +15,9 @@ merged. This could be reduced to a single call to `git-cat-file --batch`,
 There is already a Git.CatFile library that can do this easily. --[[Joey]]
 
 > This is now done, part above remains todo. --[[Joey]] 
+
+---
+
+Merging used to use memory proportional to the size of the diff. It now
+streams data, running in constant space. This probably sped it up a lot,
+as there's much less allocation and GC action. --[[Joey]] 

From 9b71b5f26c158973ca2d60dccde38290a1c9e6ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 00:01:07 -0400
Subject: [PATCH 2530/8313] fix display of semitrusted repos in status

semitrusted uuids rarely are listed in trust.log, so a special case
is needed to get a list of them. Take the difference of all known uuids
with non-semitrusted uuids.
---
 Command/Status.hs | 2 +-
 Logs/Trust.hs     | 5 +++++
 Logs/UUID.hs      | 6 +++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 2c0afa62c4..47b500b90a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -102,7 +102,7 @@ supported_remote_types = stat "supported remote types" $
 
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ lift $ do
-	us <- M.keys <$> uuidMap
+	us <- uuidList
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
 	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index cb91861fd9..6305d281fc 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -20,6 +20,7 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 import Logs.UUIDBased
+import Logs.UUID
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
@@ -27,6 +28,10 @@ trustLog = "trust.log"
 
 {- Returns a list of UUIDs at the specified trust level. -}
 trustGet :: TrustLevel -> Annex [UUID]
+trustGet SemiTrusted = do -- special case; trustMap does not contain all these
+	others <- M.keys . M.filter (/= SemiTrusted) <$> trustMap
+	all <- uuidList
+	return $ all \\ others
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
 {- Read the trustLog into a map, overriding with any
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 20f43d15ca..17b0330c11 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -16,7 +16,8 @@
 module Logs.UUID (
 	describeUUID,
 	recordUUID,
-	uuidMap
+	uuidMap,
+	uuidList
 ) where
 
 import qualified Data.Map as M
@@ -87,3 +88,6 @@ uuidMap = do
 	return $ M.insertWith' preferold u "" m
 	where
 		preferold = flip const
+
+uuidList :: Annex [UUID]
+uuidList = M.keys <$> uuidMap

From 84784e2ca1ababf21342cba36f7e65b4c3cd303b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 00:07:06 -0400
Subject: [PATCH 2531/8313] cleanup

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 37578f5975..46518b2a80 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,14 +6,14 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     no longer needs to auto-merge.
   * init: When run in an already initalized repository, and without
     a description specified, don't delete the old description. 
-  * Optimised union merging; now only runs git cat-file once.
+  * Optimised union merging; now only runs git cat-file once, and runs
+    in constant space.
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
   * status: Fix --json mode (only the repository lists are currently
     displayed)
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
-  * merge: Now runs in constant space.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 

From 2bb6b02948da8a33b2edcd911fcf3c2597b0ee58 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 00:49:09 -0400
Subject: [PATCH 2532/8313] When not run in a git repository, git-annex can
 still display a usage message, and "git annex version" even works.

Things that sound simple, but are made hard by the Annex monad being built
with the assumption that there will always be a git repo.
---
 CmdLine.hs                                    | 24 ++++++++++++-------
 Command.hs                                    | 10 ++++++--
 Command/Version.hs                            |  7 ++++--
 GitAnnex.hs                                   |  2 +-
 Logs/Trust.hs                                 |  4 ++--
 Types/Command.hs                              |  1 +
 debian/changelog                              |  2 ++
 ...rsion_should_without_being_in_a_repo_.mdwn |  2 ++
 git-annex-shell.hs                            |  4 ++--
 9 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index af53abc628..78f46a2e39 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -11,7 +11,9 @@ module CmdLine (
 	shutdown
 ) where
 
-import System.IO.Error (try)
+import qualified System.IO.Error as IO
+import qualified Control.Exception as E
+import Control.Exception (throw)
 import System.Console.GetOpt
 
 import Common.Annex
@@ -25,14 +27,18 @@ type Params = [String]
 type Flags = [Annex ()]
 
 {- Runs the passed command line. -}
-dispatch :: Params -> [Command] -> [Option] -> String -> Git.Repo -> IO ()
-dispatch args cmds options header gitrepo = do
+dispatch :: Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
+dispatch args cmds options header getgitrepo = do
 	setupConsole
-	state <- Annex.new gitrepo
-	(actions, state') <- Annex.run state $ do
-		sequence_ flags
-		prepCommand cmd params
-	tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
+	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
+	case r of
+		Left e -> maybe (throw e) id (cmdnorepo cmd)
+		Right g -> do
+			state <- Annex.new g
+			(actions, state') <- Annex.run state $ do
+				sequence_ flags
+				prepCommand cmd params
+			tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
 	where
 		(flags, cmd, params) = parseCmd args cmds options header
 
@@ -77,7 +83,7 @@ tryRun' errnum _ cmd []
 	| otherwise = return ()
 tryRun' errnum state cmd (a:as) = run >>= handle
 	where
-		run = try $ Annex.run state $ do
+		run = IO.try $ Annex.run state $ do
 			Annex.Queue.flushWhenFull
 			a
 		handle (Left err) = showerr err >> cont False state
diff --git a/Command.hs b/Command.hs
index d22c2d12f2..b662171926 100644
--- a/Command.hs
+++ b/Command.hs
@@ -7,6 +7,7 @@
 
 module Command (
 	command,
+	noRepo,
 	next,
 	stop,
 	prepCommand,
@@ -31,9 +32,14 @@ import Logs.Trust
 import Logs.Location
 import Config
 
-{- Generates a command with the common checks. -}
+{- Generates a normal command -}
 command :: String -> String -> [CommandSeek] -> String -> Command
-command = Command commonChecks
+command = Command Nothing commonChecks
+
+{- Adds a fallback action to a command, that will be run if it's used
+ - outside a git repository. -}
+noRepo :: IO () -> Command -> Command
+noRepo a c = c { cmdnorepo = Just a }
 
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
diff --git a/Command/Version.hs b/Command/Version.hs
index a584264828..9fb7fe5bdb 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -13,7 +13,7 @@ import qualified Build.SysConfig as SysConfig
 import Annex.Version
 
 def :: [Command]
-def = [dontCheck repoExists $
+def = [noRepo showPackageVersion $ dontCheck repoExists $
 	command "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
@@ -23,7 +23,7 @@ start :: CommandStart
 start = do
 	v <- getVersion
 	liftIO $ do
-		putStrLn $ "git-annex version: " ++ SysConfig.packageversion
+		showPackageVersion
 		putStrLn $ "local repository version: " ++ fromMaybe "unknown" v
 		putStrLn $ "default repository version: " ++ defaultVersion
 		putStrLn $ "supported repository versions: " ++ vs supportedVersions
@@ -31,3 +31,6 @@ start = do
 	stop
 	where
 		vs = join " "
+
+showPackageVersion :: IO ()
+showPackageVersion = putStrLn $ "git-annex version: " ++ SysConfig.packageversion
diff --git a/GitAnnex.hs b/GitAnnex.hs
index f416b7bead..7b51602be6 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -123,4 +123,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header =<< Git.repoFromCwd
+run args = dispatch args cmds options header Git.repoFromCwd
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 6305d281fc..072ea41d6d 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -30,8 +30,8 @@ trustLog = "trust.log"
 trustGet :: TrustLevel -> Annex [UUID]
 trustGet SemiTrusted = do -- special case; trustMap does not contain all these
 	others <- M.keys . M.filter (/= SemiTrusted) <$> trustMap
-	all <- uuidList
-	return $ all \\ others
+	alluuids <- uuidList
+	return $ alluuids \\ others
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
 {- Read the trustLog into a map, overriding with any
diff --git a/Types/Command.hs b/Types/Command.hs
index d39876a7ae..5341a40545 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -33,6 +33,7 @@ type CommandCleanup = Annex Bool
 
 {- A command is defined by specifying these things. -}
 data Command = Command {
+	cmdnorepo :: Maybe (IO ()),
 	cmdcheck :: [CommandCheck],
 	cmdname :: String,
 	cmdparams :: String,
diff --git a/debian/changelog b/debian/changelog
index 46518b2a80..fd989546a3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -14,6 +14,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     displayed)
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
+  * When not run in a git repository, git-annex can still display a usage
+    message, and "git annex version" even works.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
index 0bae8bdb0e..5c995852b1 100644
--- a/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
+++ b/doc/bugs/git_annex_version_should_without_being_in_a_repo_.mdwn
@@ -3,3 +3,5 @@ was checking the version of git-annex on a machine before cloning a repo...
     $ git annex version
     git-annex: Not in a git repository.
 
+> made difficult by the Annex monad, but I made it work! --[[Joey]]
+> [[done]]
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 658eddd77f..9a9d2f0925 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -79,8 +79,8 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
-	Git.repoAbsPath dir >>= Git.repoFromAbsPath >>=
-		dispatch (cmd : filterparams params) cmds options header
+	dispatch (cmd : filterparams params) cmds options header $
+		Git.repoAbsPath dir >>= Git.repoFromAbsPath
 
 external :: [String] -> IO ()
 external params = do

From 272a67921cdc6d8d40641a9cb71b744ef1f76128 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 01:46:46 -0400
Subject: [PATCH 2533/8313] better name

---
 Annex/Branch.hs   |  2 +-
 Git/UnionMerge.hs | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ae33f66cf0..a75773e192 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -56,7 +56,7 @@ index g = gitAnnexDir g  "index"
  - and merge in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.update_index_via g
+genIndex g = Git.UnionMerge.stream_update_index g
 	[Git.UnionMerge.ls_tree fullname g]
 
 {- Runs an action using the branch's index file. -}
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a2344e59db..60ccd6dcdc 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -9,7 +9,7 @@ module Git.UnionMerge (
 	merge,
 	merge_index,
 	update_index,
-	update_index_via,
+	stream_update_index,
 	update_index_line,
 	ls_tree
 ) where
@@ -30,7 +30,7 @@ import Git.CatFile
 merge :: String -> String -> Repo -> IO ()
 merge x y repo = do
 	h <- catFileStart repo
-	update_index_via repo
+	stream_update_index repo
 		[ ls_tree x repo
 		, merge_trees x y h repo
 		]
@@ -40,19 +40,19 @@ merge x y repo = do
  - the index are preserved (and participate in the merge). -}
 merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
 merge_index h repo bs =
-	update_index_via repo $ map (\b -> merge_tree_index b h repo) bs
+	stream_update_index repo $ map (\b -> merge_tree_index b h repo) bs
 
 {- Feeds content into update-index. Later items in the list can override
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index repo ls = update_index_via repo [\s -> mapM_ s ls]
+update_index repo ls = stream_update_index repo [\s -> mapM_ s ls]
 
 type Streamer = (String -> IO ()) -> IO ()
 
 {- Streams content into update-index. -}
-update_index_via :: Repo -> [Streamer] -> IO ()
-update_index_via repo as = do
+stream_update_index :: Repo -> [Streamer] -> IO ()
+stream_update_index repo as = do
 	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
 	forM_ as (stream h)
 	hClose h

From 9290095fc21953cd1fe0a71f7c8a454934194e3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 02:23:34 -0400
Subject: [PATCH 2534/8313] improve type signatures with a Ref newtype

In git, a Ref can be a Sha, or a Branch, or a Tag. I added type aliases for
those. Note that this does not prevent mixing up of eg, refs and branches
at the type level. Since git really doesn't care, except rare cases like
git update-ref, or git tag -d, that seems ok for now.

There's also a tree-ish, but let's just use Ref for it. A given Sha or Ref
may or may not be a tree-ish, depending on the object type, so there seems
no point in trying to represent it at the type level.
---
 Annex/Branch.hs    | 47 +++++++++++++++++++++++-----------------------
 Annex/CatFile.hs   |  3 ++-
 Command/Uninit.hs  |  6 +++---
 Command/Unused.hs  |  7 ++++---
 Git.hs             | 36 +++++++++++++++++++++++++----------
 Git/CatFile.hs     | 10 +++++-----
 Git/LsTree.hs      |  8 +++-----
 Git/UnionMerge.hs  | 34 ++++++++++++++++-----------------
 Upgrade/V2.hs      |  4 ++--
 git-union-merge.hs |  2 +-
 10 files changed, 86 insertions(+), 71 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a75773e192..a62a1384c9 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -31,19 +31,17 @@ import qualified Git.UnionMerge
 import qualified Annex
 import Annex.CatFile
 
-type GitRef = String
-
 {- Name of the branch that is used to store git-annex's information. -}
-name :: GitRef
-name = "git-annex"
+name :: Git.Ref
+name = Git.Ref "git-annex"
 
 {- Fully qualified name of the branch. -}
-fullname :: GitRef
-fullname = "refs/heads/" ++ name
+fullname :: Git.Ref
+fullname = Git.Ref $ "refs/heads/" ++ show name
 
 {- Branch's name in origin. -}
-originname :: GitRef
-originname = "origin/" ++ name
+originname :: Git.Ref
+originname = Git.Ref $ "origin/" ++ show name
 
 {- A separate index file for the branch. -}
 index :: Git.Repo -> FilePath
@@ -104,7 +102,8 @@ create :: Annex ()
 create = unlessM hasBranch $ do
 	e <- hasOrigin
 	if e
-		then inRepo $ Git.run "branch" [Param name, Param originname]
+		then inRepo $ Git.run "branch"
+			[Param $ show name, Param $ show originname]
 		else withIndex' True $
 			inRepo $ Git.commit "branch created" fullname []
 
@@ -140,8 +139,8 @@ update = onceonly $ do
 		let merge_desc = if null branches
 			then "update" 
 			else "merging " ++
-				(unwords $ map Git.refDescribe branches) ++ 
-				" into " ++ name
+				(unwords $ map (show . Git.refDescribe) branches) ++ 
+				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc
 			{- Note: This merges the branches into the index.
@@ -164,12 +163,12 @@ update = onceonly $ do
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
-changedBranch :: String -> String -> Annex Bool
+changedBranch :: Git.Branch -> Git.Branch -> Annex Bool
 changedBranch origbranch newbranch = not . L.null <$> diffs
 	where
 		diffs = inRepo $ Git.pipeRead
 			[ Param "log"
-			, Param (origbranch ++ ".." ++ newbranch)
+			, Param (show origbranch ++ ".." ++ show newbranch)
 			, Params "--oneline -n1"
 			]
 
@@ -181,7 +180,7 @@ changedBranch origbranch newbranch = not . L.null <$> diffs
  - every commit present in all the other refs, as well as in the
  - git-annex branch.
  -}
-tryFastForwardTo :: [String] -> Annex Bool
+tryFastForwardTo :: [Git.Ref] -> Annex Bool
 tryFastForwardTo [] = return True
 tryFastForwardTo (first:rest) = do
 	-- First, check that the git-annex branch does not contain any
@@ -194,7 +193,7 @@ tryFastForwardTo (first:rest) = do
 	where
 		no_ff = return False
 		do_ff branch = do
-			inRepo $ Git.run "update-ref" [Param fullname, Param branch]
+			inRepo $ Git.run "update-ref" [Param $ show fullname, Param $ show branch]
 			return True
 		findbest c [] = return $ Just c
 		findbest c (r:rs)
@@ -220,9 +219,9 @@ disableUpdate = Annex.changeState setupdated
 				old = Annex.branchstate s
 
 {- Checks if a git ref exists. -}
-refExists :: GitRef -> Annex Bool
+refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
-	[Param "--verify", Param "-q", Param ref]
+	[Param "--verify", Param "-q", Param $ show ref]
 
 {- Does the main git-annex branch exist? -}
 hasBranch :: Annex Bool
@@ -238,12 +237,12 @@ hasSomeBranch = not . null <$> siblingBranches
 
 {- List of git-annex (refs, branches), including the main one and any
  - from remotes. Duplicate refs are filtered out. -}
-siblingBranches :: Annex [(String, String)]
+siblingBranches :: Annex [(Git.Ref, Git.Branch)]
 siblingBranches = do
-	r <- inRepo $ Git.pipeRead [Param "show-ref", Param name]
-	return $ nubBy uref $ map (pair . words . L.unpack) (L.lines r)
+	r <- inRepo $ Git.pipeRead [Param "show-ref", Param $ show name]
+	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
 	where
-		pair l = (head l, last l)
+		gen l = (Git.Ref $ head l, Git.Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
 {- Applies a function to modifiy the content of a file.
@@ -291,7 +290,7 @@ get' staleok file = fromcache =<< getCache file
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
 	bfiles <- inRepo $ Git.pipeNullSplit
-		[Params "ls-tree --name-only -r -z", Param fullname]
+		[Params "ls-tree --name-only -r -z", Param $ show fullname]
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
@@ -346,10 +345,10 @@ stageJournalFiles = do
 			hClose toh
 			exitSuccess
 		hClose toh
-		s <- hGetContents fromh
+		shas <- map Git.Ref . lines <$> hGetContents fromh
 		-- update the index, also in just one command
 		Git.UnionMerge.update_index g $
-			index_lines (lines s) $ map fileJournal fs
+			index_lines shas (map fileJournal fs)
 		hClose fromh
 		forceSuccess pid
 		mapM_ removeFile paths
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 0541f7269d..1d996edfd5 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -13,10 +13,11 @@ module Annex.CatFile (
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
+import qualified Git
 import qualified Git.CatFile
 import qualified Annex
 
-catFile :: String -> FilePath -> Annex L.ByteString
+catFile :: Git.Branch -> FilePath -> Annex L.ByteString
 catFile branch file = do
 	h <- catFileHandle
 	liftIO $ Git.CatFile.catFile h branch file
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ca18c478c5..48f5b1ac14 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -26,9 +26,9 @@ check :: Annex ()
 check = do
 	b <- current_branch	
 	when (b == Annex.Branch.name) $ error $
-		"cannot uninit when the " ++ b ++ " branch is checked out"
+		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
-		current_branch = head . lines . B.unpack <$> revhead
+		current_branch = Git.Ref . head . lines . B.unpack <$> revhead
 		revhead = inRepo $ Git.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
@@ -57,5 +57,5 @@ cleanup = do
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
-	inRepo $ Git.run "branch" [Param "-D", Param Annex.Branch.name]
+	inRepo $ Git.run "branch" [Param "-D", Param $ show Annex.Branch.name]
 	liftIO exitSuccess
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 34d8ac232a..ccb9c48d81 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -152,12 +152,13 @@ excludeReferenced l = do
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map last .
+		refs = map Git.Ref . 
+			last .
 			nubBy cmpheads .
 			filter ourbranches .
 			map words . lines . L.unpack
 		cmpheads a b = head a == head b
-		ourbranchend = '/' : Annex.Branch.name
+		ourbranchend = '/' : show (Annex.Branch.name)
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
@@ -188,7 +189,7 @@ getKeysReferenced = do
 	return $ map fst $ catMaybes keypairs
 
 {- List of keys referenced by symlinks in a git ref. -}
-getKeysReferencedInGit :: String -> Annex [Key]
+getKeysReferencedInGit :: Git.Ref -> Annex [Key]
 getKeysReferencedInGit ref = do
 	showAction $ "checking " ++ Git.refDescribe ref
 	findkeys [] =<< inRepo (LsTree.lsTree ref)
diff --git a/Git.hs b/Git.hs
index 3f3f746324..ba5d831fea 100644
--- a/Git.hs
+++ b/Git.hs
@@ -10,6 +10,10 @@
 
 module Git (
 	Repo,
+	Ref(..),
+	Branch,
+	Sha,
+	Tag,
 	repoFromCwd,
 	repoFromAbsPath,
 	repoFromUnknown,
@@ -94,6 +98,18 @@ data Repo = Repo {
 	remoteName :: Maybe String 
 } deriving (Show, Eq)
 
+{- A git ref. Can be a sha1, or a branch or tag name. -}
+newtype Ref = Ref String
+	deriving (Eq)
+
+instance Show Ref where
+	show (Ref v) = v
+
+{- Aliases for Ref. -}
+type Branch = Ref
+type Sha = Ref
+type Tag = Ref
+
 newFrom :: RepoLocation -> Repo
 newFrom l = 
 	Repo {
@@ -162,9 +178,9 @@ repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
-{- Converts a fully qualified git ref into a user-visible version -}
-refDescribe :: String -> String
-refDescribe = remove "refs/heads/" . remove "refs/remotes/"
+{- Converts a fully qualified git ref into a user-visible version. -}
+refDescribe :: Ref -> String
+refDescribe = remove "refs/heads/" . remove "refs/remotes/" . show
 	where
 		remove prefix s
 			| prefix `isPrefixOf` s = drop (length prefix) s
@@ -432,7 +448,7 @@ useIndex index = do
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
    any trailing newline, returning the sha. -}
-getSha :: String -> IO String -> IO String
+getSha :: String -> IO String -> IO Sha
 getSha subcommand a = do
 	t <- a
 	let t' = if last t == '\n'
@@ -440,27 +456,27 @@ getSha subcommand a = do
 		else t
 	when (length t' /= shaSize) $
 		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return t'
+	return $ Ref t'
 
 {- Size of a git sha. -}
 shaSize :: Int
 shaSize = 40
 
-{- Commits the index into the specified branch, 
+{- Commits the index into the specified branch (or other ref), 
  - with the specified parent refs. -}
-commit :: String -> String -> [String] -> Repo -> IO ()
+commit :: String -> Ref -> [Ref] -> Repo -> IO ()
 commit message newref parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
 		pipeRead [Param "write-tree"] repo
 	sha <- getSha "commit-tree" $ asString $
 		ignorehandle $ pipeWriteRead
-			(map Param $ ["commit-tree", tree] ++ ps)
+			(map Param $ ["commit-tree", show tree] ++ ps)
 			(L.pack message) repo
-	run "update-ref" [Param newref, Param sha] repo
+	run "update-ref" [Param $ show newref, Param $ show sha] repo
 	where
 		ignorehandle a = snd <$> a
 		asString a = L.unpack <$> a
-		ps = concatMap (\r -> ["-p", r]) parentrefs
+		ps = concatMap (\r -> ["-p", show r]) parentrefs
 
 {- Runs git config and populates a repo with its config. -}
 configRead :: Repo -> IO Repo
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 83c1235086..c1cafb8ba8 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -37,14 +37,14 @@ catFileStop (pid, from, to) = do
 	forceSuccess pid
 
 {- Reads a file from a specified branch. -}
-catFile :: CatFileHandle -> String -> FilePath -> IO L.ByteString
-catFile h branch file = catObject h (branch ++ ":" ++ file)
+catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString
+catFile h branch file = catObject h $ Ref $ show branch ++ ":" ++ file
 
 {- Uses a running git cat-file read the content of an object.
  - Objects that do not exist will have "" returned. -}
-catObject :: CatFileHandle -> String -> IO L.ByteString
+catObject :: CatFileHandle -> Ref -> IO L.ByteString
 catObject (_, from, to) object = do
-	hPutStrLn to object
+	hPutStrLn to $ show object
 	hFlush to
 	header <- hGetLine from
 	case words header of
@@ -53,7 +53,7 @@ catObject (_, from, to) object = do
 			  validobjtype objtype -> handle size
 			| otherwise -> empty
 		_
-			| header == object ++ " missing" -> empty
+			| header == show object ++ " missing" -> empty
 			| otherwise -> error $ "unknown response from git cat-file " ++ header
 	where
 		handle size = case reads size of
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 1fcdf13ed5..8aa16a308b 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -19,8 +19,6 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import Git
 import Utility.SafeCommand
 
-type Treeish = String
-
 data TreeItem = TreeItem
 	{ mode :: FileMode
 	, typeobj :: String
@@ -28,10 +26,10 @@ data TreeItem = TreeItem
 	, file :: FilePath
 	} deriving Show
 
-{- Lists the contents of a Treeish -}
-lsTree :: Treeish -> Repo -> IO [TreeItem]
+{- Lists the contents of a Ref -}
+lsTree :: Ref -> Repo -> IO [TreeItem]
 lsTree t repo = map parseLsTree <$>
-	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File t] repo
+	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File $ show t] repo
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 60ccd6dcdc..edc8cb20ba 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -22,12 +22,14 @@ import Common
 import Git
 import Git.CatFile
 
+type Streamer = (String -> IO ()) -> IO ()
+
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
  -
  - Should be run with a temporary index file configured by Git.useIndex.
  -}
-merge :: String -> String -> Repo -> IO ()
+merge :: Ref -> Ref -> Repo -> IO ()
 merge x y repo = do
 	h <- catFileStart repo
 	stream_update_index repo
@@ -38,7 +40,7 @@ merge x y repo = do
 
 {- Merges a list of branches into the index. Previously staged changed in
  - the index are preserved (and participate in the merge). -}
-merge_index :: CatFileHandle -> Repo -> [String] -> IO ()
+merge_index :: CatFileHandle -> Repo -> [Ref] -> IO ()
 merge_index h repo bs =
 	stream_update_index repo $ map (\b -> merge_tree_index b h repo) bs
 
@@ -48,8 +50,6 @@ merge_index h repo bs =
 update_index :: Repo -> [String] -> IO ()
 update_index repo ls = stream_update_index repo [\s -> mapM_ s ls]
 
-type Streamer = (String -> IO ()) -> IO ()
-
 {- Streams content into update-index. -}
 stream_update_index :: Repo -> [Streamer] -> IO ()
 stream_update_index repo as = do
@@ -66,22 +66,22 @@ stream_update_index repo as = do
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
-update_index_line :: String -> FilePath -> String
-update_index_line sha file = "100644 blob " ++ sha ++ "\t" ++ file
+update_index_line :: Sha -> FilePath -> String
+update_index_line sha file = "100644 blob " ++ show sha ++ "\t" ++ file
 
-{- Gets the contents of a tree. -}
-ls_tree :: String -> Repo -> Streamer
-ls_tree x repo streamer = mapM_ streamer =<< pipeNullSplit params repo
+{- Gets the current tree for a ref. -}
+ls_tree :: Ref -> Repo -> Streamer
+ls_tree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- For merging two trees. -}
-merge_trees :: String -> String -> CatFileHandle -> Repo -> Streamer
-merge_trees x y h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
+merge_trees :: Ref -> Ref -> CatFileHandle -> Repo -> Streamer
+merge_trees (Ref x) (Ref y) h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: String -> CatFileHandle -> Repo -> Streamer
-merge_tree_index x h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index :: Ref -> CatFileHandle -> Repo -> Streamer
+merge_tree_index (Ref x) h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
@@ -101,7 +101,7 @@ calc_merge ch differ repo streamer = gendiff >>= go
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
 mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
-mergeFile info file h repo = case filter (/= nullsha) [asha, bsha] of
+mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	shas -> do
@@ -110,11 +110,11 @@ mergeFile info file h repo = case filter (/= nullsha) [asha, bsha] of
 		return $ Just $ update_index_line sha file
 	where
 		[_colonamode, _bmode, asha, bsha, _status] = words info
-		nullsha = replicate shaSize '0'
+		nullsha = Ref $ replicate shaSize '0'
 		unionmerge = L.unlines . nub . L.lines
 
-{- Injects some content into git, returning its hash. -}
-hashObject :: L.ByteString -> Repo -> IO String
+{- Injects some content into git, returning its Sha. -}
+hashObject :: L.ByteString -> Repo -> IO Sha
 hashObject content repo = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 6a46ad8a16..e76d99b3ec 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -86,7 +86,7 @@ logFiles dir = return . filter (".log" `isSuffixOf`)
 
 push :: Annex ()
 push = do
-	origin_master <- Annex.Branch.refExists "origin/master"
+	origin_master <- Annex.Branch.refExists $ Git.Ref "origin/master"
 	origin_gitannex <- Annex.Branch.hasOrigin
 	case (origin_master, origin_gitannex) of
 		(_, True) -> do
@@ -103,7 +103,7 @@ push = do
 			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			inRepo $ Git.run "push" [Param "origin", Param Annex.Branch.name]
+			inRepo $ Git.run "push" [Param "origin", Param $ show Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 10ae842177..1cec4a0f85 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -37,7 +37,7 @@ parseArgs = do
 
 main :: IO ()
 main = do
-	[aref, bref, newref] <- parseArgs
+	[aref, bref, newref] <- map Git.Ref <$> parseArgs
 	g <- Git.configRead =<< Git.repoFromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g

From 6d9f525f647fb24f4955e4a51156741f216d3931 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 11:59:17 -0400
Subject: [PATCH 2535/8313] clarify

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 35ba25115d..e91e5a0e33 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -441,7 +441,7 @@ file contents are present at either of two repositories.
 * --copies=number
 
   Matches only files that git-annex believes to have the specified number
-  of copies.
+  of copies, or more.
 
 * --not
 

From 1ffd54ef781b0c240d7de6c80cfcb01328efc25f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 Nov 2011 18:56:06 -0400
Subject: [PATCH 2536/8313] ensure branch exists before trying to update it

The branch may not exist, if .git/annex has been copied over from another
repo (or a corrupted repo). I suppose it could also have gotten deleted
somehow. Without this, there is a confusing failure.
---
 Annex/Branch.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a62a1384c9..ccc6145552 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -130,6 +130,8 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
+	-- ensure branch exists
+	create
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches

From abd4e1192f675de8d8afeec7af467069443d6e40 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 17 Nov 2011 21:51:55 +0000
Subject: [PATCH 2537/8313]

---
 ...nterrupting_migration_causes_problems.mdwn | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 doc/bugs/interrupting_migration_causes_problems.mdwn

diff --git a/doc/bugs/interrupting_migration_causes_problems.mdwn b/doc/bugs/interrupting_migration_causes_problems.mdwn
new file mode 100644
index 0000000000..3298a1fa65
--- /dev/null
+++ b/doc/bugs/interrupting_migration_causes_problems.mdwn
@@ -0,0 +1,50 @@
+Killing a migration from WORM to SHA256 with ^C breaks things; future attempts to do the migration fail:
+
+    #!/bin/bash
+    
+    BASE=/tmp/migrate-bug
+    
+    set -x
+    
+    chmod -R +w $BASE
+    rm -rf $BASE
+    mkdir -p $BASE
+    cd $BASE
+    
+    # create annex
+    git init .
+    git annex init
+    
+    # make a big (sparse) file and add it
+    dd if=/dev/zero of=bigfile bs=1 count=0 seek=1G
+    git annex add --backend WORM bigfile
+    git commit -m 'added bigfile'
+    
+    # look at status
+    git annex status
+    
+    # now migrate it, but kill migration during checksum
+    # Simulate ^C by making a new process group and sending SIGINT
+    setsid git annex migrate --backend SHA256 bigfile &
+    PID=$!
+    sleep 1
+    kill -INT -$PID
+    wait
+    
+    # look at status
+    git annex status
+    
+    # this migration fails
+    git annex migrate --backend SHA256 bigfile
+    
+    # but fsck says everything's OK
+    git annex fsck
+
+The error:
+
+    migrate bigfile 
+    git-annex: /tmp/migrate-bug/.git/annex/objects/K9/V1/WORM-s1073741824-m1321566308--bigfile/WORM-s1073741824-m1321566308--bigfile: createLink: already exists (File exists)
+    failed
+    git-annex: migrate: 1 failed
+
+    

From b3bbc1cbb61d18d049ba334dc4fd85d0ca922afa Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 17 Nov 2011 21:56:55 +0000
Subject: [PATCH 2538/8313]

---
 ...ata_isn__39__t_unused_after_migration.mdwn | 47 +++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
new file mode 100644
index 0000000000..322c354a21
--- /dev/null
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -0,0 +1,47 @@
+Old data isn't listed as unused after migrating backends:
+
+    #!/bin/bash
+    
+    BASE=/tmp/migrate-bug-2
+    set -x
+    chmod -R +w $BASE
+    rm -rf $BASE
+    mkdir -p $BASE
+    cd $BASE
+    
+    # create annex
+    git init .
+    git annex init
+    
+    # make a big (sparse) file and add it
+    dd if=/dev/zero of=bigfile bs=1 count=0 seek=1G
+    git annex add --backend WORM bigfile
+    git commit -m 'added bigfile'
+    
+    # migrate it
+    git annex migrate --backend SHA256 bigfile
+    
+    # status shows 2 keys taking up 2G
+    git annex status
+    
+    # but nothing is unused
+    git annex unused
+
+Output:
+
+    ++ git annex status
+    supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+    supported remote types: git S3 bup directory rsync web hook
+    known repositories: 
+            ede95a82-1166-11e1-a475-475d55eb0f8f -- here
+    local annex keys: 2
+    local annex size: 2 gigabytes
+    visible annex keys: 1
+    visible annex size: 1 gigabyte
+    backend usage: 
+            WORM: 1
+            SHA256: 1
+    ++ git annex unused
+    unused . (checking for unused data...) (checking master...) ok
+
+The two files are hardlinked, so it's not taking up extra space, but it would be nice to be able to remove the old keys.

From d66fac1ec82ea350e5d668df0cc5d02a345ca75e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Nov 2011 18:17:34 -0400
Subject: [PATCH 2539/8313] fix typo introduced with the Ref type

---
 Command/Unused.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index ccb9c48d81..7f9edfef25 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -153,7 +153,7 @@ excludeReferenced l = do
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
 		refs = map Git.Ref . 
-			last .
+			map last .
 			nubBy cmpheads .
 			filter ourbranches .
 			map words . lines . L.unpack

From 8b892901a98421b9b70c131bcc7af3a5e2ce62d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Nov 2011 18:20:06 -0400
Subject: [PATCH 2540/8313] analysis; not a bug but a feature

---
 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
index 322c354a21..9d1bbc4e07 100644
--- a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -45,3 +45,11 @@ Output:
     unused . (checking for unused data...) (checking master...) ok
 
 The two files are hardlinked, so it's not taking up extra space, but it would be nice to be able to remove the old keys.
+
+> `git annex unused` checks the content of all branches, and assumes that,
+> when a branch contains a file that points to a key, that key is still 
+> used. In this case, the migration has staged a change to the file,
+> but it is not yet committed, so when it checks the master branch, it
+> still finds a file referring to the old key. 
+> 
+> So, slightly surprising, but not a bug. --[[Joey]] [[done]]

From c70b78d40a1f27176cc01c8ae02355b50d5cb607 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 17 Nov 2011 18:29:28 -0400
Subject: [PATCH 2541/8313] migrate: Don't fall over a stale temp file.

---
 Command/Migrate.hs                                   | 5 +++--
 debian/changelog                                     | 1 +
 doc/bugs/interrupting_migration_causes_problems.mdwn | 4 +++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 860c4bd470..045b8f9b1d 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -48,9 +48,10 @@ perform file oldkey newbackend = do
 	src <- fromRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file
+	cleantmp tmpfile
 	liftIO $ createLink src tmpfile
 	k <- Backend.genKey tmpfile $ Just newbackend
-	liftIO $ cleantmp tmpfile
+	cleantmp tmpfile
 	case k of
 		Nothing -> stop
 		Just (newkey, _) -> do
@@ -70,7 +71,7 @@ perform file oldkey newbackend = do
 					next $ Command.Add.cleanup file newkey True
 				else stop
 	where
-		cleantmp t = whenM (doesFileExist t) $ removeFile t
+		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
 		link src newkey = getViaTmpUnchecked newkey $ \t -> do
 			-- Make a hard link to the old backend's
 			-- cached key, to avoid wasting disk space.
diff --git a/debian/changelog b/debian/changelog
index fd989546a3..6976fd13a8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * Fix support for insteadOf url remapping. Closes: #644278
   * When not run in a git repository, git-annex can still display a usage
     message, and "git annex version" even works.
+  * migrate: Don't fall over a stale temp file.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/interrupting_migration_causes_problems.mdwn b/doc/bugs/interrupting_migration_causes_problems.mdwn
index 3298a1fa65..68426e54af 100644
--- a/doc/bugs/interrupting_migration_causes_problems.mdwn
+++ b/doc/bugs/interrupting_migration_causes_problems.mdwn
@@ -47,4 +47,6 @@ The error:
     failed
     git-annex: migrate: 1 failed
 
-    
+> Fixed it to delete the stale temp file. [[done]]
+> 
+> Thanks for making such clear test cases, Jim! --[[Joey]] 

From 15164588851fe65415a814630e0d00fdb28d5b92 Mon Sep 17 00:00:00 2001
From: "http://ertai.myopenid.com/" 
Date: Fri, 18 Nov 2011 14:11:08 +0000
Subject: [PATCH 2542/8313]

---
 doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn

diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
new file mode 100644
index 0000000000..5e2fac99d4
--- /dev/null
+++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
@@ -0,0 +1,4 @@
+When using `git annex get foo` where foo is available in a rsync remote with encryption I got an error saying that rsync cannot
+find the required file but extra ' are here.
+
+I attached a patch for this.

From ed55a750d5255ac07bcd04adf4365a3255b3f605 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 11:58:55 -0400
Subject: [PATCH 2543/8313] response

---
 .../extraneous_shell_escaping_for_rsync_remotes.mdwn     | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
index 5e2fac99d4..0b292c5c92 100644
--- a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
+++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
@@ -2,3 +2,12 @@ When using `git annex get foo` where foo is available in a rsync remote with enc
 find the required file but extra ' are here.
 
 I attached a patch for this.
+
+> But you didn't, sadly. :(
+>
+> I don't seem to see the problem, set up a rsync over ssh with encryption
+> and sent over a file "foo", and then got it back from rsync, without
+> trouble.
+> 
+> Ah, you're not using rsync over ssh, but just to a local directory,
+> right? --[[Joey]]

From 1326bb863516b5e7efbfd9fba2754a3fe7289315 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 12:53:48 -0400
Subject: [PATCH 2544/8313] Avoid excessive escaping for rsync special remotes
 that are not accessed over ssh.

This is actually tricky, 45bbf210a1210172c7c7b87879ed74f7c8ccbdba added
the escaping because it's needed for rsync that does go over ssh.
So I had to detect whether the remote's rsync url will use ssh or not,
and vary the escaping.
---
 Remote/Rsync.hs                                  |  9 +++++++--
 Utility/RsyncFile.hs                             | 16 ++++++++++++++++
 debian/changelog                                 |  2 ++
 ...raneous_shell_escaping_for_rsync_remotes.mdwn |  2 ++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 86ff2ea5b1..5cd27a6099 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -81,13 +81,18 @@ rsyncSetup u c = do
 	gitConfigSpecialRemote u c' "rsyncurl" url
 	return c'
 
+rsyncEscape :: RsyncOpts -> String -> String
+rsyncEscape o s
+	| rsyncUrlIsShell (rsyncUrl o) = shellEscape s
+	| otherwise = s
+
 rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o  hashDirMixed k  shellEscape (f  f)
+rsyncKey o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (f  f)
         where
                 f = keyFile k
 
 rsyncKeyDir :: RsyncOpts -> Key -> String
-rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  shellEscape (keyFile k)
+rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< fromRepo (gitAnnexLocation k)
diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index c5006a30fa..a691d0a0e6 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -8,6 +8,7 @@
 module Utility.RsyncFile where
 
 import Data.String.Utils
+import Data.List
 
 import Utility.SafeCommand
 
@@ -48,3 +49,18 @@ rsync = boolSystem "rsync"
 
 rsyncExec :: [CommandParam] -> IO ()
 rsyncExec params = executeFile "rsync" True (toCommand params) Nothing
+
+{- Checks if an rsync url involves the remote shell (ssh or rsh).
+ - Use of such urls with rsync or rsyncExec requires additional shell
+ - escaping. -}
+rsyncUrlIsShell :: String -> Bool
+rsyncUrlIsShell s
+	| "rsync://" `isPrefixOf` s = False
+	| otherwise = go s
+	where
+		-- host:dir is rsync protocol, while host:dir is ssh/rsh
+		go [] = False
+		go (c:cs)
+			| c == '/' = False -- got to directory with no colon
+			| c == ':' = not $ ":" `isPrefixOf` cs
+			| otherwise = go cs
diff --git a/debian/changelog b/debian/changelog
index 6976fd13a8..52d08f3037 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,8 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * When not run in a git repository, git-annex can still display a usage
     message, and "git annex version" even works.
   * migrate: Don't fall over a stale temp file.
+  * Avoid excessive escaping for rsync special remotes that are not accessed
+    over ssh.
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
index 0b292c5c92..c4ee8d5bda 100644
--- a/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
+++ b/doc/bugs/extraneous_shell_escaping_for_rsync_remotes.mdwn
@@ -11,3 +11,5 @@ I attached a patch for this.
 > 
 > Ah, you're not using rsync over ssh, but just to a local directory,
 > right? --[[Joey]]
+
+>> [[fixed|done]] --[[Joey]] 

From c50a5fbeb4f90041f16cc84bf6b39577655d42b6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 13:22:48 -0400
Subject: [PATCH 2545/8313] status: Include all special remotes in the list of
 repositories.

Special remotes do not always have a description listed in uuid.log,
and such ones were not listed before.
---
 Command/Status.hs |  2 +-
 Logs/Trust.hs     | 28 ++++++++++++++++------------
 Logs/UUID.hs      |  6 +-----
 Remote.hs         |  6 +++++-
 debian/changelog  |  1 +
 5 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 47b500b90a..7448615cdd 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -102,7 +102,7 @@ supported_remote_types = stat "supported remote types" $
 
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ lift $ do
-	us <- uuidList
+	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
 	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 072ea41d6d..e447fbebc8 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -20,20 +20,30 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 import Logs.UUIDBased
-import Logs.UUID
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
 trustLog = "trust.log"
 
-{- Returns a list of UUIDs at the specified trust level. -}
+{- Returns a list of UUIDs that the trustLog indicates have the
+ - specified trust level.
+ - Note that the list can be incomplete for SemiTrusted, since that's
+ - the default. -}
 trustGet :: TrustLevel -> Annex [UUID]
-trustGet SemiTrusted = do -- special case; trustMap does not contain all these
-	others <- M.keys . M.filter (/= SemiTrusted) <$> trustMap
-	alluuids <- uuidList
-	return $ alluuids \\ others
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
+{- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
+trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
+trustPartition level ls
+	| level == SemiTrusted = do
+		t <- trustGet Trusted
+		u <- trustGet UnTrusted
+		let uncandidates = t ++ u
+		return $ partition (`notElem` uncandidates) ls
+	| otherwise = do
+		candidates <- trustGet level
+		return $ partition (`elem` candidates) ls
+
 {- Read the trustLog into a map, overriding with any
  - values from forcetrust. The map is cached for speed. -}
 trustMap :: Annex TrustMap
@@ -72,9 +82,3 @@ trustSet uuid@(UUID _) level = do
 		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
 trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
-
-{- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
-trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
-trustPartition level ls = do
-	candidates <- trustGet level
-	return $ partition (`elem` candidates) ls
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 17b0330c11..20f43d15ca 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -16,8 +16,7 @@
 module Logs.UUID (
 	describeUUID,
 	recordUUID,
-	uuidMap,
-	uuidList
+	uuidMap
 ) where
 
 import qualified Data.Map as M
@@ -88,6 +87,3 @@ uuidMap = do
 	return $ M.insertWith' preferold u "" m
 	where
 		preferold = flip const
-
-uuidList :: Annex [UUID]
-uuidList = M.keys <$> uuidMap
diff --git a/Remote.hs b/Remote.hs
index 7c0362d2a7..b1be60ec4a 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,6 +16,7 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
+	remoteMap,
 	byName,
 	prettyPrintUUIDs,
 	remotesWithUUID,
@@ -83,6 +84,10 @@ genList = do
 			u <- getRepoUUID r
 			generate t r u (M.lookup u m)
 
+{- Map of UUIDs of Remotes and their names. -}
+remoteMap :: Annex (M.Map UUID String)
+remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
+
 {- Looks up a remote by name. (Or by UUID.) Only finds currently configured
  - git remotes. -}
 byName :: String -> Annex (Remote Annex)
@@ -139,7 +144,6 @@ prettyPrintUUIDs desc uuids = do
 			| d == n = d
 			| null d = n
 			| otherwise = n ++ " (" ++ d ++ ")"
-		remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
 		findlog m u = M.findWithDefault "" u m
 		prettify m hereu u
 			| not (null d) = fromUUID u ++ " -- " ++ d
diff --git a/debian/changelog b/debian/changelog
index 52d08f3037..12bbba11d4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     in constant space.
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
+  * status: Include all special remotes in the list of repositories.
   * status: Fix --json mode (only the repository lists are currently
     displayed)
   * status: --fast is back

From 1de00df63602bc67a19a72ec2a07821edea6619d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Fri, 18 Nov 2011 21:12:30 +0000
Subject: [PATCH 2546/8313]

---
 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
index 9d1bbc4e07..46febe72e0 100644
--- a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -53,3 +53,9 @@ The two files are hardlinked, so it's not taking up extra space, but it would be
 > still finds a file referring to the old key. 
 > 
 > So, slightly surprising, but not a bug. --[[Joey]] [[done]]
+
+>> Thanks for the explanation.  In my real repository, it was a bit trickier:
+>> the migration was commited to `master`, but other *remote* branches still
+>> referenced those keys.  I was just doing a `git pull` from a central repo, but
+>> needed a `git remote update` to remove those references from `remotes/foo/master` too.
+>> --Jim

From 1b90918cec5a649d490a56d6fc8417ca623cbb6a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 Nov 2011 17:09:10 -0400
Subject: [PATCH 2547/8313] avoid error message when doing get --from on file
 not present on remote

---
 Command/Get.hs  | 12 +++++++++---
 Command/Move.hs | 14 +++++++++-----
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/Command/Get.hs b/Command/Get.hs
index f7d953bb65..093cd2cc59 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -27,14 +27,20 @@ start numcopies file (key, _) = do
 	if inannex
 		then stop
 		else autoCopies key (<) numcopies $ do
-			showStart "get" file
 			from <- Annex.getState Annex.fromremote
 			case from of
-				Nothing -> next $ perform key
+				Nothing -> go $ perform key
 				Just name -> do
 					-- get --from = copy --from
 					src <- Remote.byName name
-					next $ Command.Move.fromPerform src False key
+					ok <- Command.Move.fromOk src key
+					if ok
+						then go $ Command.Move.fromPerform src False key
+						else stop
+	where
+		go a = do
+			showStart "get" file
+			next a	
 
 perform :: Key -> CommandPerform
 perform key = do
diff --git a/Command/Move.hs b/Command/Move.hs
index 155f4d605f..4c4534c49d 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -117,13 +117,17 @@ fromStart src move file key
 		if ishere then stop else go
 	where
 		go = do
-			u <- getUUID
-			remotes <- Remote.keyPossibilities key
-			if u == Remote.uuid src || not (any (== src) remotes)
-				then stop
-				else do
+			ok <- fromOk src key
+			if ok
+				then do
 					showMoveAction move file
 					next $ fromPerform src move key
+				else stop
+fromOk :: Remote.Remote Annex -> Key -> Annex Bool
+fromOk src key = do
+	u <- getUUID
+	remotes <- Remote.keyPossibilities key
+	return $ u /= Remote.uuid src && any (== src) remotes
 fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key

From 3905053a18011eb92f00754cb1f7e5331370e2ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:16:38 -0400
Subject: [PATCH 2548/8313] update comment to explain non-obvious temp file

---
 Command/Migrate.hs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 045b8f9b1d..7a329080f7 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -39,12 +39,16 @@ start b file (key, oldbackend) = do
 upgradableKey :: Key -> Bool
 upgradableKey key = isNothing $ Types.Key.keySize key
 
+{- Store the old backend's key in the new backend
+ - The old backend's key is not dropped from it, because there may
+ - be other files still pointing at that key.
+ -
+ - Use the same filename as the file for the temp file name, to support
+ - backends that allow the filename to influence the keys they
+ - generate.
+ -}
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	-- Store the old backend's cached key in the new backend
-	-- (the file can't be stored as usual, because it's already a symlink).
-	-- The old backend's key is not dropped from it, because there may
-	-- be other files still pointing at that key.
 	src <- fromRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file

From 32d9813b1dc50a3300f24f3ddbf534c6cc529d6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:27:10 -0400
Subject: [PATCH 2549/8313] tweak

---
 Command.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command.hs b/Command.hs
index b662171926..4d5bbeb363 100644
--- a/Command.hs
+++ b/Command.hs
@@ -92,11 +92,10 @@ isBareRepo = fromRepo Git.repoIsLocalBare
  - copies of the key is > or < than the numcopies setting, before running
  - the action. -}
 autoCopies :: Key -> (Int -> Int -> Bool) -> Maybe Int -> CommandStart -> CommandStart
-autoCopies key vs numcopiesattr a = do
-	auto <- Annex.getState Annex.auto
-	if auto
-		then do
+autoCopies key vs numcopiesattr a = Annex.getState Annex.auto >>= auto
+	where
+		auto False = a
+		auto True = do
 			needed <- getNumCopies numcopiesattr
 			(_, have) <- trustPartition UnTrusted =<< keyLocations key
 			if length have `vs` needed then a else stop
-		else a

From 0fa1d136dc74828eab820d7c74c811d707cc746a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:40:40 -0400
Subject: [PATCH 2550/8313] tweak

---
 Annex/UUID.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 6fc04c0f09..3b64bf3d82 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -39,10 +39,11 @@ genUUID = pOpen ReadFromPipe command params $ liftM toUUID . hGetLine
 			-- uuidgen generates random uuid by default
 			else []
 
+{- Get current repository's UUID. -}
 getUUID :: Annex UUID
 getUUID = getRepoUUID =<< gitRepo
 
-{- Looks up a repo's UUID. May return "" if none is known. -}
+{- Looks up a repo's UUID, caching it in .git/config if it's not already. -}
 getRepoUUID :: Git.Repo -> Annex UUID
 getRepoUUID r = do
 	c <- fromRepo cached
@@ -54,7 +55,7 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached g = toUUID $ Git.configGet cachekey "" g
+		cached = toUUID . Git.configGet cachekey ""
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u

From 128b4bd01509bcdcdd6120a29d24527cff82d3ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 Nov 2011 15:57:08 -0400
Subject: [PATCH 2551/8313] tweaks

---
 Annex/UUID.hs |  2 +-
 Config.hs     | 10 ++++------
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 3b64bf3d82..e510a7ccd3 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -59,7 +59,7 @@ getRepoUUID r = do
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u
-		cachekey = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-uuid"
+		cachekey = remoteConfig r "uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
 getUncachedUUID = toUUID . Git.configGet configkey ""
diff --git a/Config.hs b/Config.hs
index dbd13ad3fd..cc0c929531 100644
--- a/Config.hs
+++ b/Config.hs
@@ -28,13 +28,13 @@ getConfig r key def = do
 	def' <- fromRepo $ Git.configGet ("annex." ++ key) def
 	fromRepo $ Git.configGet (remoteConfig r key) def'
 
+{- Looks up a per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
 
 {- Calculates cost for a remote. Either the default, or as configured 
  - by remote..annex-cost, or if remote..annex-cost-command
- - is set and prints a number, that is used.
- -}
+ - is set and prints a number, that is used. -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
@@ -55,7 +55,7 @@ semiCheapRemoteCost = 110
 expensiveRemoteCost :: Int
 expensiveRemoteCost = 200
 
-{- Adjust's a remote's cost to reflect it being encrypted. -}
+{- Adjusts a remote's cost to reflect it being encrypted. -}
 encryptedRemoteCostAdj :: Int
 encryptedRemoteCostAdj = 50
 
@@ -74,9 +74,7 @@ prop_cost_sane = False `notElem`
  - setting, or on command-line options. Allows command-line to override
  - annex-ignore. -}
 repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = do
-	ignored <- getConfig r "ignore" "false"
-	return $ not $ Git.configTrue ignored
+repoNotIgnored r = not . Git.configTrue <$> getConfig r "ignore" "false"
 
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}

From d675f1c82e7a3c7aa3f0b3f67284433cce111781 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Nov 2011 14:12:48 -0400
Subject: [PATCH 2552/8313] status --json now shows most things

Left out the backend usage graph for now, and bad/temp directory sizes
are only displayed when present. Also, disk usage is returned as a string
with units, which I can see changing later.
---
 Command/Status.hs | 45 ++++++++++++++++++++++++++++-----------------
 Messages.hs       | 24 +++++++++++-------------
 debian/changelog  |  3 +--
 3 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 7448615cdd..a47f21b91a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -11,6 +11,7 @@ import Control.Monad.State
 import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
+import Text.JSON
 
 import Common.Annex
 import qualified Types.Backend as B
@@ -78,12 +79,21 @@ start = do
 		return True
 	stop
 
-stat :: String -> StatState String -> Stat
-stat desc a = return $ Just (desc, a)
+stat :: String -> (String -> StatState String) -> Stat
+stat desc a = return $ Just (desc, a desc)
 
 nostat :: Stat
 nostat = return Nothing
 
+json :: JSON j => (j -> String) -> StatState j -> String -> StatState String
+json serialize a desc = do
+	j <- a
+	lift $ maybeShowJSON [(desc, j)]
+	return $ serialize j
+
+nojson :: StatState String -> String -> StatState String
+nojson a _ = a
+
 showStat :: Stat -> StatState ()
 showStat s = calc =<< s
 	where
@@ -93,15 +103,15 @@ showStat s = calc =<< s
 		calc Nothing = return ()
 
 supported_backends :: Stat
-supported_backends = stat "supported backends" $ 
-	return $ unwords $ map B.name Backend.list
+supported_backends = stat "supported backends" $ json unwords $
+	return $ map B.name Backend.list
 
 supported_remote_types :: Stat
-supported_remote_types = stat "supported remote types" $
-	return $ unwords $ map R.typename Remote.remoteTypes
+supported_remote_types = stat "supported remote types" $ json unwords $
+	return $ map R.typename Remote.remoteTypes
 
 remote_list :: TrustLevel -> String -> Stat
-remote_list level desc = stat n $ lift $ do
+remote_list level desc = stat n $ nojson $ lift $ do
 	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
@@ -110,20 +120,20 @@ remote_list level desc = stat n $ lift $ do
 		n = desc ++ " repositories"
 
 local_annex_size :: Stat
-local_annex_size = stat "local annex size" $
+local_annex_size = stat "local annex size" $ json id $
 	keySizeSum <$> cachedKeysPresent
 
 local_annex_keys :: Stat
-local_annex_keys = stat "local annex keys" $
-	show . S.size <$> cachedKeysPresent
+local_annex_keys = stat "local annex keys" $ json show $
+	S.size <$> cachedKeysPresent
 
 visible_annex_size :: Stat
-visible_annex_size = stat "visible annex size" $
+visible_annex_size = stat "visible annex size" $ json id $
 	keySizeSum <$> cachedKeysReferenced
 
 visible_annex_keys :: Stat
-visible_annex_keys = stat "visible annex keys" $
-	show . S.size <$> cachedKeysReferenced
+visible_annex_keys = stat "visible annex keys" $ json show $
+	S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -132,7 +142,8 @@ bad_data_size :: Stat
 bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
-backend_usage = stat "backend usage" $ usage <$> cachedKeysReferenced <*> cachedKeysPresent
+backend_usage = stat "backend usage" $ nojson $
+	usage <$> cachedKeysReferenced <*> cachedKeysPresent
 	where
 		usage a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
 		splits :: [Key] -> [(String, Integer)]
@@ -179,9 +190,9 @@ staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else stat label $ do
-			let s = keySizeSum $ S.fromList keys
-			return $ s ++ aside "clean up with git-annex unused"
+		else do
+			stat label $ json (++ aside "clean up with git-annex unused") $
+				return $ keySizeSum $ S.fromList keys
 
 aside :: String -> String
 aside s = " (" ++ s ++ ")"
diff --git a/Messages.hs b/Messages.hs
index 57b7068041..6ea347ca47 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -68,17 +68,17 @@ showEndFail :: Annex ()
 showEndFail = showEndResult False
 
 showEndResult :: Bool -> Annex ()
-showEndResult b = handle (JSON.end b) $ putStrLn msg
+showEndResult ok = handle (JSON.end ok) $ putStrLn msg
 	where
 		msg
-			| b = "ok"
+			| ok = "ok"
 			| otherwise = "failed"
 
 showErr :: (Show a) => a -> Annex ()
 showErr e = warning' $ "git-annex: " ++ show e
 
 warning :: String -> Annex ()
-warning w = warning' (indent w)
+warning = warning' . indent
 
 warning' :: String -> Annex ()
 warning' w = do
@@ -88,7 +88,7 @@ warning' w = do
 		hPutStrLn stderr w
 
 indent :: String -> String
-indent s = join "\n" $ map (\l -> "  " ++ l) $ lines s
+indent = join "\n" . map (\l -> "  " ++ l) . lines
 
 {- Shows a JSON value only when in json mode. -}
 maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
@@ -105,9 +105,8 @@ showCustom command a = do
 	handle (JSON.end r) q
 
 showHeader :: String -> Annex ()
-showHeader h = handle q $ do
-	putStr $ h ++ ": "
-	hFlush stdout
+showHeader h = handle q $
+	flushed $ putStr $ h ++ ": "
 
 showRaw :: String -> Annex ()
 showRaw s = handle q $ putStrLn s
@@ -126,12 +125,11 @@ setupConsole = do
 	hSetBinaryMode stderr True
 
 handle :: IO () -> IO () -> Annex ()
-handle json normal = do
-	output <- Annex.getState Annex.output
-	case output of
-		Annex.NormalOutput -> liftIO normal
-		Annex.QuietOutput -> q
-		Annex.JSONOutput -> liftIO json
+handle json normal = Annex.getState Annex.output >>= go
+	where
+		go Annex.NormalOutput = liftIO normal
+		go Annex.QuietOutput = q
+		go Annex.JSONOutput = liftIO json
 
 q :: Monad m => m ()
 q = return ()
diff --git a/debian/changelog b/debian/changelog
index 12bbba11d4..9f96a4fffe 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,8 +11,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * status: Now displays trusted, untrusted, and semitrusted repositories
     separately.
   * status: Include all special remotes in the list of repositories.
-  * status: Fix --json mode (only the repository lists are currently
-    displayed)
+  * status: Fix --json mode.
   * status: --fast is back
   * Fix support for insteadOf url remapping. Closes: #644278
   * When not run in a git repository, git-annex can still display a usage

From 2768be71362c97f9c8fed15e2cd9528cceb7a4dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Nov 2011 14:19:03 -0400
Subject: [PATCH 2553/8313] response

---
 doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
index 46febe72e0..9d468bdc7d 100644
--- a/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
+++ b/doc/bugs/old_data_isn__39__t_unused_after_migration.mdwn
@@ -59,3 +59,8 @@ The two files are hardlinked, so it's not taking up extra space, but it would be
 >> referenced those keys.  I was just doing a `git pull` from a central repo, but
 >> needed a `git remote update` to remove those references from `remotes/foo/master` too.
 >> --Jim
+
+>>> I have considered making unused ignore remote tracking branches. 
+>>> On the one hand, it can be a little bit confusing, and those branches
+>>> can be out of date. On the other hand, it can be useful to know you're
+>>> not dropping anything that some remote might still refer to. --[[Joey]] 

From 0f0169fa990167aa21d236c5b247ec49998a7e43 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 Nov 2011 22:49:53 -0400
Subject: [PATCH 2554/8313] comment update

---
 Command/Move.hs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 4c4534c49d..fd1ed90198 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -23,10 +23,6 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed $ start True]
 
-{- Move (or copy) a file either --to or --from a repository.
- -
- - This only operates on the cached file content; it does not involve
- - moving data in the key-value backend. -}
 start :: Bool -> FilePath -> (Key, Backend Annex) -> CommandStart
 start move file (key, _) = do
 	noAuto

From 6c0448d94c2be5e3c0658334ff882452bf4e8000 Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Mon, 21 Nov 2011 22:24:04 +0000
Subject: [PATCH 2555/8313]

---
 doc/forum/--print0_option_as_in___34__find__34__.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/--print0_option_as_in___34__find__34__.mdwn

diff --git a/doc/forum/--print0_option_as_in___34__find__34__.mdwn b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
new file mode 100644
index 0000000000..cd537f8adb
--- /dev/null
+++ b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
@@ -0,0 +1,3 @@
+It would be nice if git annex find supported a --print0 option as GNU
+find does.  That way, file names that are printed could be piped to
+xargs even if they have spaces. 

From 35b47069582ff45f7e5bfdd1b43cb22134b0d646 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 22 Nov 2011 07:09:24 +0000
Subject: [PATCH 2556/8313]

---
 ...tory_remote_and_case_sensitivity_on_FAT.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
new file mode 100644
index 0000000000..d1b21a9e7b
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
@@ -0,0 +1,17 @@
+I was copying files to a directory remote with `git annex copy`.  Out of 114 files, 9 of them failed with no message, just:
+
+    copy data/foo.dat (to usbdrive...) failed
+    copy data/bar.dat (to usbdrive...) failed
+
+According to strace:
+
+    31338 mkdir("/media/annex/Zp/9v/SHA256-s1362999320--d650297c8cf8c2dc0575110a52d0c5cc0ff266f294a0599f85796a6b44b23492", 0777) = -1 ENOENT (No such file or directory)
+    31338 mkdir("/media/annex/Zp/9v", 0777) = -1 ENOENT (No such file or directory)
+    31338 mkdir("/media/annex/Zp", 0777)    = -1 EEXIST (File exists)
+    31338 stat("/media/annex/Zp", 0x7f8449f170d0) = -1 ENOENT (No such file or directory)
+
+The filesystem is FAT32 and has weird case semantics.  This was mounted by udisks with its default options:
+
+    /dev/sdb1 on /media/annex type vfat (rw,nosuid,nodev,uhelper=udisks,uid=1000,gid=1000,shortname=mixed,dmask=0077,utf8=1,showexec)
+
+I wonder if the directory remote should use hashDirLower instead of hashDirMixed?

From fc2f0e8b1a4bd016ac29606381dfb7034c88e9f5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 12:37:51 -0400
Subject: [PATCH 2557/8313] response; cannot reproduce

---
 .../directory_remote_and_case_sensitivity_on_FAT.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
index d1b21a9e7b..b686304e59 100644
--- a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
@@ -15,3 +15,14 @@ The filesystem is FAT32 and has weird case semantics.  This was mounted by udisk
     /dev/sdb1 on /media/annex type vfat (rw,nosuid,nodev,uhelper=udisks,uid=1000,gid=1000,shortname=mixed,dmask=0077,utf8=1,showexec)
 
 I wonder if the directory remote should use hashDirLower instead of hashDirMixed?
+
+> git-annex intentionally uses the same layout for directory and rsync
+> special remotes as it does for the .git/annex directory. As far
+> as I know it works ok on case-insensative filesystems.
+> 
+> Based on your strace, if you `ls /media/annex/Zp`, you will see
+> "No such file or directory", but if you `mkdir /media/annex/Zp` it will
+> fail with "File exists". Doesn't make much sense to me.
+> 
+> I cannot reproduce this problem using a vfat filesystem 
+> mounted using the same options you show (linux 3.0.0). --[[Joey]]

From 7f7ae7a3b1cdfbc61879189dfe04a637690015aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 14:06:31 -0400
Subject: [PATCH 2558/8313] find: Support --print0

It would be nice if command-specific options were supported. The first
difficulty is that which command is being called is not known until after
getopt; but that could be worked around by finding the first non-dashed
parameter. Storing the settings without putting them in the annex monad is
the next difficulty; it could perhaps be handled by making the seek stage
pass applicable settings into the start stage (and from there on to perform
as needed). But that still leaves a problem, what data type to use to
represent the options between getopt and seek?
---
 Annex.hs                                              | 2 ++
 Command/Find.hs                                       | 8 ++++++--
 GitAnnex.hs                                           | 3 +++
 debian/changelog                                      | 1 +
 doc/forum/--print0_option_as_in___34__find__34__.mdwn | 2 ++
 doc/git-annex.mdwn                                    | 3 +++
 6 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 24cc78a647..6d245a92d1 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -60,6 +60,7 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
+	, print0 :: Bool
 	, branchstate :: BranchState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
@@ -82,6 +83,7 @@ newState gitrepo = AnnexState
 	, force = False
 	, fast = False
 	, auto = False
+	, print0 = False
 	, branchstate = startBranchState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
diff --git a/Command/Find.hs b/Command/Find.hs
index c816ff0712..47058fa257 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import Annex.Content
 import Limit
+import qualified Annex
 
 def :: [Command]
 def = [command "find" paramPaths seek "lists available files"]
@@ -22,6 +23,9 @@ start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) (inAnnex key) limited) $
-		liftIO $ putStrLn file
+	whenM (liftM2 (||) (inAnnex key) limited) $ do
+		print0 <- Annex.getState Annex.print0
+		if print0
+			then liftIO $ putStr (file ++ "\0")
+			else liftIO $ putStrLn file
 	stop
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7b51602be6..f563c08cbf 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -103,6 +103,8 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
+	, Option [] ["print0"] (NoArg (setprint0 True))
+		"terminate filename with null"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote)
@@ -114,6 +116,7 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
+		setprint0 v = Annex.changeState $ \s -> s { Annex.print0 = v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			newg <- inRepo $ Git.configStore v
diff --git a/debian/changelog b/debian/changelog
index 9f96a4fffe..3c4fa05aac 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,6 +19,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
   * migrate: Don't fall over a stale temp file.
   * Avoid excessive escaping for rsync special remotes that are not accessed
     over ssh.
+  * find: Support --print0
 
  -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
 
diff --git a/doc/forum/--print0_option_as_in___34__find__34__.mdwn b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
index cd537f8adb..7d9a2284dd 100644
--- a/doc/forum/--print0_option_as_in___34__find__34__.mdwn
+++ b/doc/forum/--print0_option_as_in___34__find__34__.mdwn
@@ -1,3 +1,5 @@
 It would be nice if git annex find supported a --print0 option as GNU
 find does.  That way, file names that are printed could be piped to
 xargs even if they have spaces. 
+
+> Done. --[[Joey]] 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e91e5a0e33..c225529de3 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -232,6 +232,9 @@ subdirectories).
   With no parameters, defaults to finding all files in the current directory
   and its subdirectories.
 
+  To output filenames terminated with nulls, for use with xargs -0,
+  specify --print0.
+
 * whereis [path ...]
 
   Displays a list of repositories known to contain the content of the

From 322d9b1cc044c960339779b9d88aa39e21f41d9e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 14:40:11 -0400
Subject: [PATCH 2559/8313] releasing version 3.20111122

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 3c4fa05aac..185ffecebf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111112) UNRELEASED; urgency=low
+git-annex (3.20111122) unstable; urgency=low
 
   * merge: Improve commit messages to mention what was merged.
   * Avoid doing auto-merging in commands that don't need fully current
@@ -21,7 +21,7 @@ git-annex (3.20111112) UNRELEASED; urgency=low
     over ssh.
   * find: Support --print0
 
- -- Joey Hess   Sat, 12 Nov 2011 14:50:21 -0400
+ -- Joey Hess   Tue, 22 Nov 2011 14:31:45 -0400
 
 git-annex (3.20111111) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index b2638c3ac7..f31c7a86e8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111111
+Version: 3.20111122
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 2d9099531b7c8d237e4fee1327f1a6955412e70b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 14:41:52 -0400
Subject: [PATCH 2560/8313] add news item for git-annex 3.20111122

---
 doc/news/version_3.20111011.mdwn | 30 ------------------------------
 doc/news/version_3.20111122.mdwn | 22 ++++++++++++++++++++++
 2 files changed, 22 insertions(+), 30 deletions(-)
 delete mode 100644 doc/news/version_3.20111011.mdwn
 create mode 100644 doc/news/version_3.20111122.mdwn

diff --git a/doc/news/version_3.20111011.mdwn b/doc/news/version_3.20111011.mdwn
deleted file mode 100644
index 0346cbca82..0000000000
--- a/doc/news/version_3.20111011.mdwn
+++ /dev/null
@@ -1,30 +0,0 @@
-git-annex 3.20111011 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * This version of git-annex only works with git 1.7.7 and newer.
-     The breakage with old versions is subtle, and affects the
-     annex.numcopies settings in .gitattributes, so be sure to upgrade git
-     to 1.7.7. (Debian package now depends on that version.)
-   * Don't pass absolute paths to git show-attr, as it started following
-     symlinks when that's done in 1.7.7. Instead, use relative paths,
-     which show-attr only handles 100% correctly in 1.7.7. Closes: #[645046](http://bugs.debian.org/645046)
-   * Fix referring to remotes by uuid.
-   * New or changed repository descriptions in uuid.log now have a timestamp,
-     which is used to ensure the newest description is used when the uuid.log
-     has been merged.
-   * Note that older versions of git-annex will display the timestamp as part
-     of the repository description, which is ugly but otherwise harmless.
-   * Add timestamps to trust.log and remote.log too.
-   * git-annex-shell: Added the --uuid option.
-   * git-annex now asks git-annex-shell to verify that it's operating in
-     the expected repository.
-   * Note that this git-annex will not interoperate with remotes using
-     older versions of git-annex-shell.
-   * Now supports git's insteadOf configuration, to modify the url
-     used to access a remote. Note that pushInsteadOf is not used;
-     that and pushurl are reserved for actual git pushes. Closes: #[644278](http://bugs.debian.org/644278)
-   * status: List all known repositories.
-   * When displaying a list of repositories, show git remote names
-     in addition to their descriptions.
-   * Add locking to avoid races when changing the git-annex branch.
-   * Various speed improvements gained by using ByteStrings.
-   * Contain the zombie hordes."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111122.mdwn b/doc/news/version_3.20111122.mdwn
new file mode 100644
index 0000000000..193394de98
--- /dev/null
+++ b/doc/news/version_3.20111122.mdwn
@@ -0,0 +1,22 @@
+git-annex 3.20111122 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * merge: Improve commit messages to mention what was merged.
+   * Avoid doing auto-merging in commands that don't need fully current
+     information from the git-annex branch. In particular, git annex add
+     no longer needs to auto-merge.
+   * init: When run in an already initalized repository, and without
+     a description specified, don't delete the old description.
+   * Optimised union merging; now only runs git cat-file once, and runs
+     in constant space.
+   * status: Now displays trusted, untrusted, and semitrusted repositories
+     separately.
+   * status: Include all special remotes in the list of repositories.
+   * status: Fix --json mode.
+   * status: --fast is back
+   * Fix support for insteadOf url remapping. Closes: #[644278](http://bugs.debian.org/644278)
+   * When not run in a git repository, git-annex can still display a usage
+     message, and "git annex version" even works.
+   * migrate: Don't fall over a stale temp file.
+   * Avoid excessive escaping for rsync special remotes that are not accessed
+     over ssh.
+   * find: Support --print0"""]]
\ No newline at end of file

From b1c601ac8cfd2d30ee8299a508d36f4ea5c1cc1a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 22 Nov 2011 18:51:04 +0000
Subject: [PATCH 2561/8313] Added a comment: Case sensitivity

---
 ..._bcac9fd7b3f4a2ac28bee59bae674fa0._comment | 79 +++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment
new file mode 100644
index 0000000000..be8b8b0a72
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_1_bcac9fd7b3f4a2ac28bee59bae674fa0._comment
@@ -0,0 +1,79 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Case sensitivity"
+ date="2011-11-22T18:51:03Z"
+ content="""
+I agree, it's weird, but that's what I'm seeing:
+
+    #!/bin/sh
+    
+    if [ $UID != 0 ] ; then echo \"need root\" ; exit 1 ; fi
+    
+    set -x
+    
+    # make image
+    cd /tmp
+    dd if=/dev/zero of=diskimage bs=1M count=40
+    DEV=$(losetup --find --show diskimage)
+    
+    # make FAT32 fs
+    mkfs.vfat -F 32 $DEV
+    
+    # mount it
+    mkdir annex
+    mount -o shortname=mixed,utf8=1 $DEV annex
+    
+    # show bug
+    ( 
+        cd annex
+        mkdir zP
+        mkdir Zp
+        ls Zp
+        ls
+        touch zP
+        touch Zp
+    )
+    
+    # cleanup
+    umount annex
+    rm -r annex
+    losetup -d $DEV
+    rm diskimage
+    
+    # info
+    uname -a
+
+Output:
+
+    + cd /tmp
+    + dd if=/dev/zero of=diskimage bs=1M count=40
+    40+0 records in
+    40+0 records out
+    41943040 bytes (42 MB) copied, 0.0847729 s, 495 MB/s
+    ++ losetup --find --show diskimage
+    + DEV=/dev/loop0
+    + mkfs.vfat -F 32 /dev/loop0
+    mkfs.vfat 3.0.9 (31 Jan 2010)
+    Loop device does not match a floppy size, using default hd params
+    + mkdir annex
+    + mount -o shortname=mixed,utf8=1 /dev/loop0 annex
+    + cd annex
+    + mkdir zP
+    + mkdir Zp
+    mkdir: cannot create directory `Zp': File exists
+    + ls Zp
+    ls: cannot access Zp: No such file or directory
+    + ls
+    zP
+    + touch zP
+    + touch Zp
+    touch: cannot touch `Zp': File exists
+    + umount annex
+    + rm -r annex
+    + losetup -d /dev/loop0
+    + rm diskimage
+    + uname -a
+    Linux pilot 3.0.3+ #1 SMP Mon Aug 29 15:21:18 EDT 2011 x86_64 GNU/Linux
+
+"""]]

From eeda534e9a0ff81e7498a69dd79577865deb76e9 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Nov 2011 19:56:55 +0000
Subject: [PATCH 2562/8313] Added a comment

---
 ...2_c9088060fb9133b66951f1a3075981e8._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment
new file mode 100644
index 0000000000..5040b3120f
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_2_c9088060fb9133b66951f1a3075981e8._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-11-22T19:56:55Z"
+ content="""
+All right, I see the same thing with linux 3.1.0. It seems this behavior has changed since linux 3.0.0. Mounting with shortname=lower avoids the problem. 
+
+I feel a good case could be made that this new behavior is a linux bug. Your example with touch particularly shows how weird it is.
+
+
+$ touch Foo
+$ echo hi > foo
+sh: cannot create foo: File exists
+$ rm foo
+rm: cannot remove `foo': No such file or directory
+

+"""]]

From 6d7e990dcaf4f327fd957196ff71d1ee966ae490 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Tue, 22 Nov 2011 20:35:03 +0000
Subject: [PATCH 2563/8313] Added a comment

---
 ..._5bf34466187cfc9b34bd3ca8c89a07c6._comment | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment
new file mode 100644
index 0000000000..54d6ff50ab
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_3_5bf34466187cfc9b34bd3ca8c89a07c6._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="comment 3"
+ date="2011-11-22T20:35:01Z"
+ content="""
+I see the same results (\"`touch: cannot touch 'Zp': File exists`\") on these Debian systems:
+
+    Linux pilot 3.0.3+ #1 SMP Mon Aug 29 15:21:18 EDT 2011 x86_64 GNU/Linux
+    Linux neurosis 3.0.0-1-amd64 #1 SMP Sun Jul 24 02:24:44 UTC 2011 x86_64 GNU/Linux
+    Linux bucket 2.6.39-2-amd64 #1 SMP Tue Jul 5 02:51:22 UTC 2011 x86_64 GNU/Linux
+    Linux psychosis 2.6.37-trunk-amd64 #1 SMP Thu Jan 6 14:13:28 UTC 2011 x86_64 GNU/Linux
+    Linux bacon 2.6.32-5-amd64 #1 SMP Thu Aug 12 13:01:50 UTC 2010 x86_64 GNU/Linux
+
+It does NOT happen on this Ubuntu system:
+
+    Linux esensor 3.0.0-12-generic #20-Ubuntu SMP Fri Oct 7 14:56:25 UTC 2011 x86_64 x86_64 x86_64 GNU/Linux
+
+So really it seems like only the Ubuntu kernel is the outlier here?  Maybe it has something to do with charsets or something; I think FAT is a mess in that regard and even long versus short filenames can behave differently.
+"""]]

From 112399bf1a924e91473aa4e6d0cf84061a3bd482 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Nov 2011 20:59:55 +0000
Subject: [PATCH 2564/8313] Added a comment

---
 .../comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment
new file mode 100644
index 0000000000..406a6b18ee
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_4_d6201f2d86d5b44051a7fd7a8c9de583._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-11-22T20:59:55Z"
+ content="""
+Your ubuntu system has 3.0.0 which as noted does not have the problem.
+"""]]

From fd81b5047b314ecc96b34e0c9f017f8f5a62f072 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 22 Nov 2011 21:01:14 +0000
Subject: [PATCH 2565/8313] Added a comment

---
 .../comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment

diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment
new file mode 100644
index 0000000000..1656ff2075
--- /dev/null
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT/comment_5_61c5f0889f30a68ac3b57c4ea564ee0e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-11-22T21:01:14Z"
+ content="""
+I am surprised if it happens on 2.6.x though. Debian 3.0.0 seemed to not have the problem but perhaps my test was bad.
+"""]]

From 75a590bdd893e579c5e375e5ad797022f5847496 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 18:20:55 -0400
Subject: [PATCH 2566/8313] Put a workaround in the directory special remote
 for strange behavior with VFAT filesystems on Linux (mounted with
 shortname=mixed)

---
 Remote/Directory.hs                           | 73 +++++++++++++------
 debian/changelog                              |  7 ++
 ...ry_remote_and_case_sensitivity_on_FAT.mdwn | 23 +++++-
 3 files changed, 76 insertions(+), 27 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index b592f41ff0..cadd5e7597 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -62,56 +62,81 @@ directorySetup u c = do
 	gitConfigSpecialRemote u c' "directory" dir
 	return $ M.delete "directory" c'
 
-dirKey :: FilePath -> Key -> FilePath
-dirKey d k = d  hashDirMixed k  f  f
+{- Where to store a given Key in the Directory.
+ -
+ - There are two possible locations to try; this had to be done because
+ - on Linux, vfat filesystem mounted with shortname=mixed have a
+ - variant of case insensativity that causes miserable failure when
+ - hashDirMixed produces eg, "xx" and "XX". The first directory to be
+ - created wins the namespace, and the second one cannot then be created.
+ - But unlike behavior with shortname=lower, "XX/foo" won't look in
+ - "xx/foo".
+ -}
+locations :: FilePath -> Key -> [FilePath]
+locations d k = [using hashDirMixed, using hashDirLower]
 	where
+		using h = d  h k  f  f
 		f = keyFile k
 
+withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
+withCheckedFile _ [] _ _ = return False
+withCheckedFile check d k a = go $ locations d k
+	where
+		go [] = return False
+		go (f:fs) = do
+			use <- check f
+			if use
+				then a f
+				else go fs
+
+withStoredFile :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
+withStoredFile = withCheckedFile doesFileExist
+
 store :: FilePath -> Key -> Annex Bool
 store d k = do
 	src <- fromRepo $ gitAnnexLocation k
-	let dest = dirKey d k
-	liftIO $ catchBoolIO $ storeHelper dest $ copyFileExternal src dest
+	liftIO $ catchBoolIO $ storeHelper d k $ copyFileExternal src
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
 	src <- fromRepo $ gitAnnexLocation k
-	let dest = dirKey d enck
-	liftIO $ catchBoolIO $ storeHelper dest $ encrypt src dest
+	liftIO $ catchBoolIO $ storeHelper d enck $ encrypt src
 	where
 		encrypt src dest = do
 			withEncryptedContent cipher (L.readFile src) $ L.writeFile dest
 			return True
 
-storeHelper :: FilePath -> IO Bool -> IO Bool
-storeHelper dest a = do
-	let dir = parentDir dest
-	createDirectoryIfMissing True dir
-	allowWrite dir	
-	ok <- a
-	when ok $ do
-		preventWrite dest
-		preventWrite dir
-	return ok
+storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
+storeHelper d key a = withCheckedFile check d key go
+	where
+		check dest = isJust <$> mkdir (parentDir dest)
+		mkdir = catchMaybeIO . createDirectoryIfMissing True
+		go dest = do
+			let dir = parentDir dest
+			allowWrite dir
+			ok <- a dest
+			when ok $ do
+				preventWrite dest
+				preventWrite dir
+			return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
-retrieve d k f = liftIO $ copyFileExternal (dirKey d k) f
+retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
-	liftIO $ catchBoolIO $ do
-		withDecryptedContent cipher (L.readFile (dirKey d enck)) $ L.writeFile f
+	liftIO $ withStoredFile d enck $ \file -> catchBoolIO $ do
+		withDecryptedContent cipher (L.readFile file) $ L.writeFile f
 		return True
 
 remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ catchBoolIO $ do
+remove d k = liftIO $ withStoredFile d k $ \file -> catchBoolIO $ do
+	let dir = parentDir file
 	allowWrite dir
 	removeFile file
 	removeDirectory dir
 	return True
-	where
-		file = dirKey d k
-		dir = parentDir file
 
 checkPresent :: FilePath -> Key -> Annex (Either String Bool)
-checkPresent d k = liftIO $ catchMsgIO $ doesFileExist (dirKey d k)
+checkPresent d k = liftIO $ catchMsgIO $ withStoredFile d k $
+	const $ return True -- withStoredFile checked that it exists
diff --git a/debian/changelog b/debian/changelog
index 185ffecebf..e043f6c9fa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111123) UNRELEASED; urgency=low
+
+  * Put a workaround in the directory special remote for strange behavior
+    with VFAT filesystems on Linux (mounted with shortname=mixed)
+
+ -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
+
 git-annex (3.20111122) unstable; urgency=low
 
   * merge: Improve commit messages to mention what was merged.
diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
index b686304e59..ae653d6197 100644
--- a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
@@ -18,11 +18,28 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 
 > git-annex intentionally uses the same layout for directory and rsync
 > special remotes as it does for the .git/annex directory. As far
-> as I know it works ok on case-insensative filesystems.
+> as I know it works ok on (truely) case-insensative filesystems.
 > 
 > Based on your strace, if you `ls /media/annex/Zp`, you will see
 > "No such file or directory", but if you `mkdir /media/annex/Zp` it will
 > fail with "File exists". Doesn't make much sense to me.
 > 
-> I cannot reproduce this problem using a vfat filesystem 
-> mounted using the same options you show (linux 3.0.0). --[[Joey]]
+> The (default) VFAT mount option shortname=mixed causes this behavior.
+> With shortname=lower it works ok. --[[Joey]]
+> 
+>> So, the options for fixing this bug seem to be to fix Linux (which would
+>> be a good idea IMHO but I don't really want to go there), or generally
+>> convert git-annex to using lowercase for its hashing (which would be a
+>> large amount of pain to rewrite all the symlinks in every git repo),
+>> or some special hack around this problem. 
+>> 
+>> I've put in a workaround for the problem in the directory special
+>> remote; it will use mixed case but fall-back to lowercase as necessary.
+>> 
+>> That does leave the case of a bare git repository with annexed content
+>> stored on VFAT. More special casing could fix it, but that is, I
+>> think, an unusual configuration. Leaving the bug open for that case,
+>> and for the even more unlikely configuration of a rsync special remote
+>> stored on VFAT. --[[Joey]] 
+
+[[!meta title="bare git repository not supported on VFAT"]]

From 3dd66fd2f03fa1bffa2368496ef6b8a9793f881e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 Nov 2011 22:13:11 -0400
Subject: [PATCH 2567/8313] update url

---
 doc/install/FreeBSD.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/FreeBSD.mdwn b/doc/install/FreeBSD.mdwn
index 23152ac204..72b402c380 100644
--- a/doc/install/FreeBSD.mdwn
+++ b/doc/install/FreeBSD.mdwn
@@ -1,2 +1,2 @@
 git-annex is in FreeBSD ports in
-[devel/git-annex](http://www.freshports.org/devel/git-annex/)
+[devel/git-annex](http://www.freshports.org/devel/hs-git-annex/)

From 709acf3f240171fa1e3782c8e0362ff4f3239bfb Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Thu, 24 Nov 2011 01:59:45 +0000
Subject: [PATCH 2568/8313]

---
 doc/forum/Podcast_syncing_use-case.mdwn | 34 +++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/forum/Podcast_syncing_use-case.mdwn

diff --git a/doc/forum/Podcast_syncing_use-case.mdwn b/doc/forum/Podcast_syncing_use-case.mdwn
new file mode 100644
index 0000000000..6b3c81cabc
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case.mdwn
@@ -0,0 +1,34 @@
+I've been trying to use git-annex with the following strategy.
+
+* Download podcasts into the annex `gpodder-downloads`
+* Check the podcasts into the annex using `git annex add`.
+* Copy the podcasts over to my mp3 player in the annex `usb-ariaz`.
+  This is a FAT-formatted mp3 player, so I have been using a bare
+  repository.  
+* Move the podcasts to a different annex called `gpodder-on-usbdisk`
+  to indicate that they have been successfully put on the mp3 player.
+* `chmod` the files on the mp3 player to `0600` so that I can delete
+  them from the player when I am done listening to them.
+  
+Then I go for a run or something and listen to a bunch of podcasts,
+deleting them after I have listened to them.  When I get back, I would
+like to find the files that I have listened to and remove them from
+the annexes that are not on the mp3 player.  What I have been hoping
+is that something like
+
+    ~/gpodder-on-usbdisk $ git annex find --not --in usb-ariaz --print0 | xargs -0 git rm
+    ~/gpodder-on-usbdisk $ git annex unused
+    ~/gpodder-on-usbdisk $ git annex dropunused `seq X`
+	
+would work.  However, it appears that `git-annex find` does not
+actually check to see that the file contents are present, but only
+looks at the `git-annex` branch of the `usb-ariaz` repository.  Since
+I have not changed that with my sneaky deletions, it has no way of
+knowing that the files have been deleted.
+
+Is there any way to do this properly?  (And by properly, I don't mean
+"don't delete the files".  That is really the only way I have of
+marking that I have listened to podcasts on this particular mp3 player.)
+
+I tried setting the `usb-ariaz` repository to be untrusted, but that
+did not change the behavior of `git annex find`.

From ba1ec2c60f26bfb39ea69e4e62a09bb85e19cb4a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Fri, 25 Nov 2011 05:56:59 +0000
Subject: [PATCH 2569/8313]

---
 ...9__t_handle_double_spaces_in_filename.mdwn | 83 +++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn

diff --git a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
new file mode 100644
index 0000000000..930e23ea9d
--- /dev/null
+++ b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
@@ -0,0 +1,83 @@
+Unused files with double spaces in their name are not removed by `dropunused`:
+
+Script:
+
+    #!/bin/bash
+    
+    BASE=/tmp/unused-bug
+    
+    # setup
+    set -x
+    chmod -R +w $BASE
+    rm -rf $BASE
+    mkdir -p $BASE
+    cd $BASE
+    
+    # create annex
+    git init .
+    git annex init
+    
+    # make a file with two spaces
+    echo hello > 'foo  bar'
+    
+    # add it
+    git annex add --backend WORM 'foo  bar'
+    git commit -m 'add'
+    
+    # remove it
+    git rm 'foo  bar'
+    git commit -m 'remove'
+    
+    # unused
+    git annex unused
+    git annex dropunused 1
+    git annex unused
+
+Output:
+
+    + chmod -R +w /tmp/unused-bug
+    + rm -rf /tmp/unused-bug
+    + mkdir -p /tmp/unused-bug
+    + cd /tmp/unused-bug
+    + git init .
+    Initialized empty Git repository in /tmp/unused-bug/.git/
+    + git annex init
+    init  ok
+    + echo hello
+    + git annex add --backend WORM 'foo  bar'
+    add foo  bar ok
+    (Recording state in git...)
+    + git commit -m add
+    [master (root-commit) 926f7f5] add
+     1 files changed, 1 insertions(+), 0 deletions(-)
+     create mode 120000 foo  bar
+    + git rm 'foo  bar'
+    rm 'foo  bar'
+    + git commit -m remove
+    [master d025e3f] remove
+     1 files changed, 0 insertions(+), 1 deletions(-)
+     delete mode 120000 foo  bar
+    + git annex unused
+    unused . (checking for unused data...) (checking master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       WORM-s6-m1322200438--foo  bar
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    ok
+    + git annex dropunused 1
+    dropunused 1 ok
+    + git annex unused
+    unused . (checking for unused data...) (checking master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       WORM-s6-m1322200438--foo  bar
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    ok
+
+Strange that `dropunused` still said "ok" when it didn't succeed at removing the file.

From 12243d2279cae65ed163bdec1edb268817514265 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 Nov 2011 11:46:07 -0400
Subject: [PATCH 2570/8313] Flush json output, avoiding a buffering problem
 that could result in doubled output.

The bug was that with --json, output lines were sometimes doubled. For
example, git annex init --json would output two lines, despite only running
one thing. Adding to the weirdness, this only occurred when the output
was redirected to a pipe or a file.

Strace showed two processes outputting the same buffered output.
The second process was this writer process (only needed to work around
bug #624389):

                _ <- forkProcess $ do
                        hPutStr toh $ unlines paths
                        hClose toh
                        exitSuccess

The doubled output occurs when this process exits, and ghc flushes the
inherited stdout buffer. Why only when piping? I don't know, but ghc may
be behaving differently when stdout is not a terminal.

While this is quite possibly a ghc bug, there is a nice fix in git-annex.
Explicitly flushing after each chunk of json is output works around the
problem, and as a side effect, json is streamed rather than being output
all at the end when performing an expensive operaition.

However, note that this means all uses of putStr in git-annex must be
explicitly flushed. The others were, already.
---
 Messages.hs      | 2 +-
 debian/changelog | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Messages.hs b/Messages.hs
index 6ea347ca47..a7f14f4853 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -129,7 +129,7 @@ handle json normal = Annex.getState Annex.output >>= go
 	where
 		go Annex.NormalOutput = liftIO normal
 		go Annex.QuietOutput = q
-		go Annex.JSONOutput = liftIO json
+		go Annex.JSONOutput = liftIO $ flushed $ json
 
 q :: Monad m => m ()
 q = return ()
diff --git a/debian/changelog b/debian/changelog
index e043f6c9fa..cbf69b7410 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
 
   * Put a workaround in the directory special remote for strange behavior
     with VFAT filesystems on Linux (mounted with shortname=mixed)
+  * Flush json output, avoiding a buffering problem that could result in
+    doubled output.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 

From afe9e78401f7842a41bc353b27036baa2e3046c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Nov 2011 12:03:01 -0400
Subject: [PATCH 2571/8313] error handling cleanup

Use Control.Exception bracket_; want to catch all errors.

System.Posix.Env.getEnv doesn't fail, no need to try it.
---
 Git.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/Git.hs b/Git.hs
index ba5d831fea..5bdd4afd4e 100644
--- a/Git.hs
+++ b/Git.hs
@@ -72,7 +72,7 @@ module Git (
 
 import System.Posix.Directory
 import System.Posix.User
-import IO (bracket_, try)
+import Control.Exception (bracket_)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
 import Data.Char
@@ -438,12 +438,12 @@ reap = do
  - index file. -}
 useIndex :: FilePath -> IO (IO ())
 useIndex index = do
-	res <- try $ getEnv var
+	res <- getEnv var
 	setEnv var index True
 	return $ reset res
 	where
 		var = "GIT_INDEX_FILE"
-		reset (Right (Just v)) = setEnv var v True
+		reset (Just v) = setEnv var v True
 		reset _ = unsetEnv var
 
 {- Runs an action that causes a git subcommand to emit a sha, and strips
@@ -484,10 +484,8 @@ configRead repo@(Repo { location = Dir d }) = do
 	{- Cannot use pipeRead because it relies on the config having
 	   been already read. Instead, chdir to the repo. -}
 	cwd <- getCurrentDirectory
-	bracket_ (changeWorkingDirectory d)
-		(\_ -> changeWorkingDirectory cwd) $
-			pOpen ReadFromPipe "git" ["config", "--list"] $
-				hConfigRead repo
+	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
+		pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo
 configRead r = assertLocal r $ error "internal"
 
 {- Reads git config from a handle and populates a repo with it. -}

From 041d32412535ecaba103806f95a7d74234034416 Mon Sep 17 00:00:00 2001
From: Mark Wright 
Date: Sat, 26 Nov 2011 23:39:47 +1100
Subject: [PATCH 2572/8313] Remove haskell98 to build with ghc 7.2.2, also
 built with ghc 7.0.4

Signed-off-by: Joey Hess 
---
 Locations.hs        | 4 ++--
 Logs/Location.hs    | 2 ++
 Utility/TempFile.hs | 2 +-
 git-annex.cabal     | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 83897f488f..3cb632aae1 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -26,8 +26,8 @@ module Locations (
 	prop_idempotent_fileKey
 ) where
 
-import Bits
-import Word
+import Data.Bits
+import Data.Word
 import Data.Hash.MD5
 
 import Common
diff --git a/Logs/Location.hs b/Logs/Location.hs
index ab29ffcadd..cb21a2d1c0 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -1,3 +1,5 @@
+{-# LANGUAGE BangPatterns #-}
+
 {- git-annex location log
  -
  - git-annex keeps track of which repositories have the contents of annexed
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
index 8d50dd8b2c..3887b422b6 100644
--- a/Utility/TempFile.hs
+++ b/Utility/TempFile.hs
@@ -7,7 +7,7 @@
 
 module Utility.TempFile where
 
-import IO (bracket)
+import Control.Exception (bracket)
 import System.IO
 import System.Posix.Process hiding (executeFile)
 import System.Directory
diff --git a/git-annex.cabal b/git-annex.cabal
index f31c7a86e8..c257eaff63 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -28,7 +28,7 @@ Description:
 
 Executable git-annex
   Main-Is: git-annex.hs
-  Build-Depends: haskell98, MissingH, hslogger, directory, filepath,
+  Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
    base < 5, monad-control, json

From a72f0ecc273a28bb4b12d8454e2e0f99278012ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Nov 2011 12:06:03 -0400
Subject: [PATCH 2573/8313] changelog

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index cbf69b7410..35f3a7c8d7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
     with VFAT filesystems on Linux (mounted with shortname=mixed)
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
+  * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 

From 9a67f9cb8d0ce6da314b080228ac2cea55ee612a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 Nov 2011 12:08:54 -0400
Subject: [PATCH 2574/8313] use Control.Exception's brackets

---
 test.hs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test.hs b/test.hs
index d4c1366d09..e625fbd756 100644
--- a/test.hs
+++ b/test.hs
@@ -11,7 +11,7 @@ import Test.QuickCheck
 
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
-import IO (bracket_, bracket)
+import Control.Exception (bracket_, bracket)
 import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
@@ -523,8 +523,7 @@ indir dir a = do
 	-- Assertion failures throw non-IO errors; catch
 	-- any type of error and change back to cwd before
 	-- rethrowing.
-	r <- bracket_ (changeToTmpDir dir)
-		(\_ -> changeWorkingDirectory cwd)
+	r <- bracket_ (changeToTmpDir dir) (changeWorkingDirectory cwd)
 		(E.try (a)::IO (Either E.SomeException ()))
 	case r of
 		Right () -> return ()

From 2bf3addf4997023584e32812a9d8cbc46833d672 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Nov 2011 13:50:05 -0400
Subject: [PATCH 2575/8313] Bugfix: dropunused did not drop keys with two
 spaces in their name.

---
 Command/DropUnused.hs                                         | 4 ++--
 debian/changelog                                              | 1 +
 ...punused_doesn__39__t_handle_double_spaces_in_filename.mdwn | 4 ++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 2c3bb296ad..3df9ab6c2b 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -73,6 +73,6 @@ readUnusedLog prefix = do
 		then M.fromList . map parse . lines <$> liftIO (readFile f)
 		else return M.empty
 	where
-		parse line = (head ws, fromJust $ readKey $ unwords $ tail ws)
+		parse line = (num, fromJust $ readKey $ tail rest)
 			where
-				ws = words line
+				(num, rest) = break (== ' ') line
diff --git a/debian/changelog b/debian/changelog
index 35f3a7c8d7..943d1e01cd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
+  * Bugfix: dropunused did not drop keys with two spaces in their name.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
index 930e23ea9d..a6b44cd2a3 100644
--- a/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
+++ b/doc/bugs/dropunused_doesn__39__t_handle_double_spaces_in_filename.mdwn
@@ -81,3 +81,7 @@ Output:
     ok
 
 Strange that `dropunused` still said "ok" when it didn't succeed at removing the file.
+
+> It was misparsing the unused file, so it thought you'd asked it to drop a
+> key that didn't exist (which means already dropped) so no error. I've
+> fixed the bug. [[done]] --[[Joey]] 

From a3f4ea1a6916ccaab8be26591bc9fc424649f692 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 27 Nov 2011 17:56:31 +0000
Subject: [PATCH 2576/8313] Added a comment

---
 ...comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment

diff --git a/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment b/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment
new file mode 100644
index 0000000000..fe396c39f3
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case/comment_1_ace6f9d3a950348a3ac0ff592b62e786._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-11-27T17:56:31Z"
+ content="""
+Right, --in goes by git-annex's [[location_tracking]] information; actually checking if a remote still has the files would make --in too expensive in many cases.
+
+So you need to give `gpodder-on-usbdisk` current information. You can do that by going to `usb-ariaz` and doing a `git annex fsck`. That will find the deleted files and update the location information. Then, back on `gpodder-on-usbdisk`, `git pull usb-ariaz`, and then you can proceed with the commands you showed.
+"""]]

From faf55ac2b702de1e4f2f308f87bb22284a2bc128 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Nov 2011 13:57:32 -0400
Subject: [PATCH 2577/8313] update

---
 doc/location_tracking.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/location_tracking.mdwn b/doc/location_tracking.mdwn
index 85bb3d1b55..d40a7206fd 100644
--- a/doc/location_tracking.mdwn
+++ b/doc/location_tracking.mdwn
@@ -1,8 +1,7 @@
 git-annex keeps track of in which repositories it last saw a file's content.
 This location tracking information is stored in the git-annex branch.
 Repositories record their UUID and the date when they get or drop 
-a file's content. (Git is configured to use a union merge for this file,
-so the lines may be in arbitrary order, but it will never conflict.)
+a file's content.
 
 This location tracking information is useful if you have multiple
 repositories, and not all are always accessible. For example, perhaps one
@@ -10,7 +9,7 @@ is on a home file server, and you are away from home. Then git-annex can
 tell you what git remote it needs access to in order to get a file:
 
 	# git annex get myfile 
-	get myfile(not available)
+	get myfile (not available)
 	  I was unable to access these remotes: home
 
 Another way the location tracking comes in handy is if you put repositories

From 178a49af75166999c690fd92a02d74d2fbe47270 Mon Sep 17 00:00:00 2001
From: "http://hcs.furuvik.net/" 
Date: Sun, 27 Nov 2011 20:52:15 +0000
Subject: [PATCH 2578/8313]

---
 ...le_to_have_annex_on_a_separate_filesystem.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn

diff --git a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
new file mode 100644
index 0000000000..3e05b60931
--- /dev/null
+++ b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
@@ -0,0 +1,15 @@
+I belive I have found a regression.
+
+Inspired by 
+
+I tried to only have .git/annex/objects (also tested moving .git/annex) on NFS while having the rest on local SSD disk.
+
+But when trying to add files i get:
+
+     > git annex add testfile
+    add testfile (checksum...) 
+    git-annex: testfile: rename: unsupported operation (Invalid cross-device link)
+    failed
+    git-annex: add: 1 failed
+
+I have tried both using bind-mount and with a sym-link.

From ff2d9c828379ce29e5feb6ac770996be04ac072f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 Nov 2011 17:09:29 -0400
Subject: [PATCH 2579/8313] response

---
 ...ossible_to_have_annex_on_a_separate_filesystem.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
index 3e05b60931..e9a3ee95a4 100644
--- a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
+++ b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
@@ -13,3 +13,13 @@ But when trying to add files i get:
     git-annex: add: 1 failed
 
 I have tried both using bind-mount and with a sym-link.
+
+> I don't think this was a reversion; the forum post doesn't really
+> indicate it ever worked.
+> 
+> Grepping for `renameFile` and `createLink` will find all the places
+> in git-annex that assume one filesystem. These would have to be changed
+> to catch errors and fall back to expensive copying.
+> 
+> Putting a separate repository on the file server could work better
+> depending on what you're trying to do. --[[Joey]] 

From d9e770a836ffefa7749ddfba06e54301f34cec2e Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Sun, 27 Nov 2011 22:10:45 +0000
Subject: [PATCH 2580/8313] Added a comment

---
 .../comment_2_930a6620b4d516e69ed952f9da5371bb._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment

diff --git a/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment b/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment
new file mode 100644
index 0000000000..97eb3c681c
--- /dev/null
+++ b/doc/forum/Podcast_syncing_use-case/comment_2_930a6620b4d516e69ed952f9da5371bb._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 2"
+ date="2011-11-27T22:10:44Z"
+ content="""
+Thanks, that works perfectly!
+"""]]

From 6869e6023e21698038da7e4a858cbaf6f7b7bbed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 15:26:27 -0400
Subject: [PATCH 2581/8313] support .git/annex on a different disk than the
 rest of the repo

The only fully supported thing is to have the main repository on one disk,
and .git/annex on another. Only commands that move data in/out of the annex
will need to copy it across devices.

There is only partial support for putting arbitrary subdirectories of
.git/annex on different devices. For one thing, but this can require more
copies to be done. For example, when .git/annex/tmp is on one device, and
.git/annex/journal on another, every journal write involves a call to
mv(1). Also, there are a few places that make hard links between various
subdirectories of .git/annex with createLink, that are not handled.

In the common case without cross-device, the new moveFile is actually
faster than renameFile, avoiding an unncessary stat to check that a file
(not a directory) is being moved. Of course if a cross-device move is
needed, it is as slow as mv(1) of the data.
---
 Annex/Branch.hs                               |  2 +-
 Annex/Content.hs                              |  8 +--
 Command/Add.hs                                |  2 +-
 Command/Unlock.hs                             |  2 +-
 Common.hs                                     |  1 +
 Utility/Directory.hs                          | 51 +++++++++++++++++++
 debian/changelog                              |  2 +
 ...o_have_annex_on_a_separate_filesystem.mdwn | 13 +++--
 ...it_on_ssd__44___annex_on_spindle_disk.mdwn | 11 +++-
 9 files changed, 81 insertions(+), 11 deletions(-)
 create mode 100644 Utility/Directory.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ccc6145552..a92f05b2cc 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -312,7 +312,7 @@ setJournalFile file content = do
 			let jfile = journalFile g file
 			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
 			writeBinaryFile tmpfile content
-			renameFile tmpfile jfile
+			moveFile tmpfile jfile
 
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 83839ea135..f5571b54af 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -113,7 +113,7 @@ logStatus key status = do
 	u <- getUUID
 	logChange key u status
 
-{- Runs an action, passing it a temporary filename to download,
+{- Runs an action, passing it a temporary filename to get,
  - and if the action succeeds, moves the temp file into 
  - the annex as a key's content. -}
 getViaTmp :: Key -> (FilePath -> Annex Bool) -> Annex Bool
@@ -221,7 +221,7 @@ moveAnnex key src = do
 		else liftIO $ do
 			createDirectoryIfMissing True dir
 			allowWrite dir -- in case the directory already exists
-			renameFile src dest
+			moveFile src dest
 			preventWrite dest
 			preventWrite dir
 
@@ -243,7 +243,7 @@ fromAnnex :: Key -> FilePath -> Annex ()
 fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
 	allowWrite dir
 	allowWrite file
-	renameFile file dest
+	moveFile file dest
 	removeDirectory dir
 
 {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
@@ -256,7 +256,7 @@ moveBad key = do
 	liftIO $ do
 		createDirectoryIfMissing True (parentDir dest)
 		allowWrite (parentDir src)
-		renameFile src dest
+		moveFile src dest
 		removeDirectory (parentDir src)
 	logStatus key InfoMissing
 	return dest
diff --git a/Command/Add.hs b/Command/Add.hs
index ab104b53cc..130f5e3110 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -61,7 +61,7 @@ undo file key e = do
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
 			src <- fromRepo $ gitAnnexLocation key
-			liftIO $ renameFile src file
+			liftIO $ moveFile src file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
 cleanup file key hascontent = do
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 22f9ce7108..b6f39488da 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -46,7 +46,7 @@ perform dest key = do
                 then do
 			liftIO $ do
 				removeFile dest
-				renameFile tmpdest dest
+				moveFile tmpdest dest
 				allowWrite dest
 			next $ return True
                 else error "copy failed!"
diff --git a/Common.hs b/Common.hs
index e0132d9e96..a3802da5f2 100644
--- a/Common.hs
+++ b/Common.hs
@@ -23,3 +23,4 @@ import Utility.Misc as X
 import Utility.Conditional as X
 import Utility.SafeCommand as X
 import Utility.Path as X
+import Utility.Directory as X
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
new file mode 100644
index 0000000000..7f8822fca5
--- /dev/null
+++ b/Utility/Directory.hs
@@ -0,0 +1,51 @@
+{- directory manipulation
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Directory where
+
+import System.IO.Error
+import System.Posix.Files
+import System.Directory
+import Control.Exception (throw)
+
+import Utility.SafeCommand
+import Utility.Conditional
+import Utility.TempFile
+
+{- Moves one filename to another.
+ - First tries a rename, but falls back to moving across devices if needed. -}
+moveFile :: FilePath -> FilePath -> IO ()
+moveFile src dest = try (rename src dest) >>= onrename
+	where
+		onrename (Right _) = return ()
+		onrename (Left e)
+			| isPermissionError e = rethrow
+			| isDoesNotExistError e = rethrow
+			| otherwise = do
+				-- copyFile is likely not as optimised as
+				-- the mv command, so we'll use the latter.
+				-- But, mv will move into a directory if
+				-- dest is one, which is not desired.
+				whenM (isdir dest) rethrow
+				viaTmp mv dest undefined
+			where
+				rethrow = throw e
+				mv tmp _ = do
+					ok <- boolSystem "mv" [Param "-f",
+						Param src, Param tmp]
+					if ok
+						then return ()
+						else do
+							-- delete any partial
+							_ <- try $
+								removeFile tmp
+							rethrow
+		isdir f = do
+			r <- try (getFileStatus f)
+			case r of
+				(Left _) -> return False
+				(Right s) -> return $ isDirectory s
diff --git a/debian/changelog b/debian/changelog
index 943d1e01cd..265ba71846 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
   * Bugfix: dropunused did not drop keys with two spaces in their name.
+  * Support for storing .git/annex on a different device than the rest of the
+    git repository.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
index e9a3ee95a4..7daf03284b 100644
--- a/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
+++ b/doc/bugs/not_possible_to_have_annex_on_a_separate_filesystem.mdwn
@@ -14,12 +14,19 @@ But when trying to add files i get:
 
 I have tried both using bind-mount and with a sym-link.
 
-> I don't think this was a reversion; the forum post doesn't really
-> indicate it ever worked.
-> 
 > Grepping for `renameFile` and `createLink` will find all the places
 > in git-annex that assume one filesystem. These would have to be changed
 > to catch errors and fall back to expensive copying.
 > 
 > Putting a separate repository on the file server could work better
 > depending on what you're trying to do. --[[Joey]] 
+
+>> I've added support for putting `.git/annex` on a separate filesystem
+>> from the rest of the git repository.
+>> 
+>> Putting individual subdirectories like `.git/annex/objects` on separate
+>> filesystems from other subdirectories is not fully supported; it may
+>> work but it may be slow and a few things (like `git annex migrate`) are
+>> known to fail due to using hard links. I don't think this is worth
+>> supporting. [[done]]
+>> --[[Joey]]
diff --git a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn
index a04c8b040b..f70c127025 100644
--- a/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn
+++ b/doc/forum/performance_improvement:_git_on_ssd__44___annex_on_spindle_disk.mdwn
@@ -1,3 +1,12 @@
 This works with bind-mount, I might try with softlinks as well.
 
-Going through git's data on push/pull can take ages on a spindle disk even if the repo is rather small in size. This is especially true if you are used to ssd speeds, but ssd storage is expensive. Storing the annex objects on a cheap spindle disk and everything else on a ssd makes things a _lot_ faster.
+Going through git's data on push/pull can take ages on a spindle disk even
+if the repo is rather small in size. This is especially true if you are
+used to ssd speeds, but ssd storage is expensive. Storing the annex objects
+on a cheap spindle disk and everything else on a ssd makes things a _lot_
+faster.
+
+> Update: git-annex supports `.git/annex/` being moved to a different disk
+> than the rest of the repisitory, but does *not* support individual 
+> subdirectories, like `.git/annex/objects/` being on a different disk
+> than the main `.git/annex/` directory. --[[Joey]]

From e32ab766b08b9a64bf12fe5ab6bf33f2d82b0fb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 17:37:15 -0400
Subject: [PATCH 2582/8313] --inbackend can be used to make git-annex only
 operate on files whose content is stored using a specified key-value backend.

---
 GitAnnex.hs        | 2 ++
 Limit.hs           | 7 +++++++
 debian/changelog   | 2 ++
 doc/git-annex.mdwn | 5 +++++
 4 files changed, 16 insertions(+)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index f563c08cbf..42a6b7fd78 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -111,6 +111,8 @@ options = commonOptions ++
 		"skip files not present in a remote"
 	, Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber)
 		"skip files with fewer copies"
+	, Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName)
+		"skip files not using a key-value backend"
 	] ++ matcherOptions
 	where
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
diff --git a/Limit.hs b/Limit.hs
index 3ae949bfb9..c68c3bdd8b 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -88,3 +88,10 @@ addCopies num =
 		handle n (Just (key, _)) = do
 			us <- keyLocations key
 			return $ length us >= n
+
+{- Adds a limit to skip files not using a specified key-value backend. -}
+addInBackend :: String -> Annex ()
+addInBackend name = addLimit $ Backend.lookupFile >=> check
+	where
+		wanted = Backend.lookupBackendName name
+		check = return . maybe False ((==) wanted . snd)
diff --git a/debian/changelog b/debian/changelog
index 265ba71846..9cd915885c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * Bugfix: dropunused did not drop keys with two spaces in their name.
   * Support for storing .git/annex on a different device than the rest of the
     git repository.
+  * --inbackend can be used to make git-annex only operate on files
+    whose content is stored using a specified key-value backend.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c225529de3..cedb2e782c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -446,6 +446,11 @@ file contents are present at either of two repositories.
   Matches only files that git-annex believes to have the specified number
   of copies, or more.
 
+* --inbackend=name
+
+  Matches only files whose content is stored using the specified key-value
+  backend.
+
 * --not
 
   Inverts the next file matching option. For example, to only act on

From 2b3c120506f1f25b4c3d0e19342b9826bde0b3b5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 17:49:03 -0400
Subject: [PATCH 2583/8313] clarify extent of limit checks

---
 doc/git-annex.mdwn | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index cedb2e782c..e98c89fe39 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -435,7 +435,8 @@ file contents are present at either of two repositories.
 * --in=repository
 
   Matches only files that git-annex believes have their contents present
-  in a repository.
+  in a repository. Note that it does not check the repository to verify
+  that it still has the content.
 
   The repository should be specified using the name of a configured remote,
   or the UUID or description of a repository. For the current repository,
@@ -444,7 +445,8 @@ file contents are present at either of two repositories.
 * --copies=number
 
   Matches only files that git-annex believes to have the specified number
-  of copies, or more.
+  of copies, or more. Note that it does not check remotes to verify that
+  the copies still exist.
 
 * --inbackend=name
 

From da9cd315beb03570b96f83063a39e799fe01b166 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 22:43:51 -0400
Subject: [PATCH 2584/8313] add support for using hashDirLower in addition to
 hashDirMixed

Supporting multiple directory hash types will allow converting to a
different one, without a flag day.

gitAnnexLocation now checks which of the possible locations have a file.
This means more statting of files. Several places currently use
gitAnnexLocation and immediately check if the returned file exists;
those need to be optimised.
---
 Annex/Content.hs    | 17 ++++++++---------
 Backend/SHA.hs      |  2 +-
 Command/Add.hs      |  2 +-
 Command/Fsck.hs     |  4 ++--
 Command/Migrate.hs  |  2 +-
 Command/SendKey.hs  |  2 +-
 Command/Unannex.hs  |  2 +-
 Command/Unlock.hs   |  2 +-
 Locations.hs        | 35 +++++++++++++++++++++++++----------
 Remote/Bup.hs       |  4 ++--
 Remote/Directory.hs |  4 ++--
 Remote/Git.hs       | 29 ++++++++++++++++++++++-------
 Remote/Hook.hs      |  4 ++--
 Remote/Rsync.hs     |  4 ++--
 Remote/S3real.hs    |  4 ++--
 15 files changed, 73 insertions(+), 44 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index f5571b54af..90bde2975a 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -43,12 +43,12 @@ import Annex.Exception
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
-inAnnex = inAnnex' doesFileExist
+inAnnex = inAnnex' $ doesFileExist
 inAnnex' :: (FilePath -> IO a) -> Key -> Annex a
 inAnnex' a key = do
 	whenM (fromRepo Git.repoIsUrl) $
 		error "inAnnex cannot check remote repo"
-	inRepo $ a . gitAnnexLocation key
+	inRepo $ \g -> gitAnnexLocation key g >>= a
 
 {- A safer check; the key's content must not only be present, but
  - is not in the process of being removed. -}
@@ -70,7 +70,7 @@ inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
  - it. (If the content is not present, no locking is done.) -}
 lockContent :: Key -> Annex a -> Annex a
 lockContent key a = do
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	bracketIO (openForLock file True >>= lock) unlock a
 	where
 		lock Nothing = return Nothing
@@ -100,9 +100,8 @@ calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
 	cwd <- liftIO getCurrentDirectory
 	let absfile = fromMaybe whoops $ absNormPath cwd file
-	top <- fromRepo Git.workTree
-	return $ relPathDirToFile (parentDir absfile) 
-			top  ".git"  annexLocation key
+	loc <- inRepo $ gitAnnexLocation key
+	return $ relPathDirToFile (parentDir absfile) loc
 	where
 		whoops = error $ "unable to normalize " ++ file
 
@@ -213,7 +212,7 @@ checkDiskSpace' adjustment key = do
  -}
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
-	dest <- fromRepo $ gitAnnexLocation key
+	dest <- inRepo $ gitAnnexLocation key
 	let dir = parentDir dest
 	e <- liftIO $ doesFileExist dest
 	if e
@@ -227,7 +226,7 @@ moveAnnex key src = do
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
-	file <- fromRepo $gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	let dir = parentDir file
 	a (dir, file)
 
@@ -250,7 +249,7 @@ fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
  - returns the file it was moved to. -}
 moveBad :: Key -> Annex FilePath
 moveBad key = do
-	src <- fromRepo $ gitAnnexLocation key
+	src <- inRepo $ gitAnnexLocation key
 	bad <- fromRepo gitAnnexBadDir
 	let dest = bad  takeFileName src
 	liftIO $ do
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index a3846a410b..2ae0cfcf42 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -99,7 +99,7 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
 checkKeyChecksum size key = do
 	fast <- Annex.getState Annex.fast
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
diff --git a/Command/Add.hs b/Command/Add.hs
index 130f5e3110..9fdbdcaa69 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -60,7 +60,7 @@ undo file key e = do
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do
-			src <- fromRepo $ gitAnnexLocation key
+			src <- inRepo $ gitAnnexLocation key
 			liftIO $ moveFile src file
 
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 99dda99e5f..a803207e20 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -87,7 +87,7 @@ verifyLocationLog key desc = do
 	-- Since we're checking that a key's file is present, throw
 	-- in a permission fixup here too.
 	when present $ do
-		f <- fromRepo $ gitAnnexLocation key
+		f <- inRepo $ gitAnnexLocation key
 		liftIO $ do
 			preventWrite f
 			preventWrite (parentDir f)
@@ -118,7 +118,7 @@ verifyLocationLog key desc = do
  - the key's metadata, if available. -}
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	case (present, Types.Key.keySize key) of
 		(_, Nothing) -> return True
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 7a329080f7..c85d7c2ac3 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -49,7 +49,7 @@ upgradableKey key = isNothing $ Types.Key.keySize key
  -}
 perform :: FilePath -> Key -> Backend Annex -> CommandPerform
 perform file oldkey newbackend = do
-	src <- fromRepo $ gitAnnexLocation oldkey
+	src <- inRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
 	let tmpfile = tmp  takeFileName file
 	cleantmp tmpfile
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 5737478671..7b1cd3ecae 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -21,7 +21,7 @@ seek = [withKeys start]
 
 start :: Key -> CommandStart
 start key = do
-	file <- fromRepo $ gitAnnexLocation key
+	file <- inRepo $ gitAnnexLocation key
 	whenM (inAnnex key) $
 		liftIO $ rsyncServerSend file -- does not return
 	warning "requested key is not present"
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index b9190ce044..e97b6d05d8 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -55,7 +55,7 @@ cleanup file key = do
 	if fast
 		then do
 			-- fast mode: hard link to content in annex
-			src <- fromRepo $ gitAnnexLocation key
+			src <- inRepo $ gitAnnexLocation key
 			liftIO $ do
 				createLink src file
 				allowWrite file
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index b6f39488da..673a7038a0 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -37,7 +37,7 @@ perform dest key = do
 	
 	checkDiskSpace key
 
-	src <- fromRepo $ gitAnnexLocation key
+	src <- inRepo $ gitAnnexLocation key
 	tmpdest <- fromRepo $ gitAnnexTmpLocation key
 	liftIO $ createDirectoryIfMissing True (parentDir tmpdest)
 	showAction "copying"
diff --git a/Locations.hs b/Locations.hs
index 3cb632aae1..425e4fdcf9 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -9,7 +9,7 @@ module Locations (
 	keyFile,
 	fileKey,
 	gitAnnexLocation,
-	annexLocation,
+	annexLocations,
 	gitAnnexDir,
 	gitAnnexObjectDir,
 	gitAnnexTmpDir,
@@ -58,17 +58,33 @@ annexDir = addTrailingPathSeparator "annex"
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
-{- Annexed file's location relative to the .git directory. -}
-annexLocation :: Key -> FilePath
-annexLocation key = objectDir  hashDirMixed key  f  f
+{- Annexed file's possible locations relative to the .git directory.
+ - There are two different possibilities, using different hashes;
+ - the first is the default for new content. -}
+annexLocations :: Key -> [FilePath]
+annexLocations key = [using hashDirMixed, using hashDirLower]
 	where
+		using h = objectDir  h key  f  f
 		f = keyFile key
 
-{- Annexed file's absolute location in a repository. -}
-gitAnnexLocation :: Key -> Git.Repo -> FilePath
+{- Annexed file's absolute location in a repository.
+ - Out of the possible annexLocations, returns the one where the file
+ - is actually present. When the file is not present, returns the
+ - one where the file should be put.
+ -}
+gitAnnexLocation :: Key -> Git.Repo -> IO FilePath
 gitAnnexLocation key r
-	| Git.repoIsLocalBare r = Git.workTree r  annexLocation key
-	| otherwise = Git.workTree r  ".git"  annexLocation key
+	| Git.repoIsLocalBare r =
+		go (Git.workTree r) $ annexLocations key
+	| otherwise =
+		go (Git.workTree r  ".git") $ annexLocations key
+	where
+		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
+		check _ [] = return Nothing
+		check dir (l:ls) = do
+			let f = dir  l
+			e <- doesFileExist f
+			if e then return (Just f) else check dir ls
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
@@ -76,8 +92,7 @@ gitAnnexDir r
 	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  annexDir
 	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  annexDir
 
-{- The part of the annex directory where file contents are stored.
- -}
+{- The part of the annex directory where file contents are stored. -}
 gitAnnexObjectDir :: Git.Repo -> FilePath
 gitAnnexObjectDir r
 	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4c826498da..589dea91d9 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -102,13 +102,13 @@ bupSplitParams r buprepo k src = do
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo k (File src)
 	liftIO $ boolSystem "bup" params
 
 storeEncrypted :: Git.Repo -> BupRepo -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r buprepo (cipher, enck) k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	params <- bupSplitParams r buprepo enck (Param "-")
 	liftIO $ catchBoolIO $
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index cadd5e7597..891a19ef6b 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -94,12 +94,12 @@ withStoredFile = withCheckedFile doesFileExist
 
 store :: FilePath -> Key -> Annex Bool
 store d k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ catchBoolIO $ storeHelper d k $ copyFileExternal src
 
 storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d (cipher, enck) k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ catchBoolIO $ storeHelper d enck $ encrypt src
 	where
 		encrypt src dest = do
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 541d8e5f65..07afc02742 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -134,7 +134,14 @@ inAnnex r key
 	| Git.repoIsUrl r = checkremote
 	| otherwise = checklocal
 	where
-		checkhttp = liftIO $ catchMsgIO $ Url.exists $ keyUrl r key
+		checkhttp = liftIO $ go undefined $ keyUrls r key
+			where
+				go e [] = return $ Left e
+				go _ (u:us) = do
+					res <- catchMsgIO $ Url.exists u
+					case res of
+						Left e -> go e us
+						v -> return v
 		checkremote = do
 			showAction $ "checking " ++ Git.repoDescribe r
 			onRemote r (check, unknown) "inannex" [Param (show key)]
@@ -169,8 +176,10 @@ onLocal r a = do
 		liftIO Git.reap
 		return ret
 
-keyUrl :: Git.Repo -> Key -> String
-keyUrl r key = Git.repoLocation r ++ "/" ++ annexLocation key
+keyUrls :: Git.Repo -> Key -> [String]
+keyUrls r key = map tourl (annexLocations key)
+	where
+		tourl l = Git.repoLocation r ++ "/" ++ l
 
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key
@@ -185,16 +194,22 @@ copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
 copyFromRemote r key file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
-		rsyncOrCopyFile params (gitAnnexLocation key r) file
+		loc <- liftIO $ gitAnnexLocation key r
+		rsyncOrCopyFile params loc file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| Git.repoIsHttp r = liftIO $ Url.download (keyUrl r key) file
+	| Git.repoIsHttp r = liftIO $ downloadurls $ keyUrls r key
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
+	where
+		downloadurls [] = return False
+		downloadurls (u:us) = do
+			ok <- Url.download u file
+			if ok then return ok else downloadurls us
 
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
 	| not $ Git.repoIsUrl r = do
-		keysrc <- fromRepo $ gitAnnexLocation key
+		keysrc <- inRepo $ gitAnnexLocation key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
@@ -203,7 +218,7 @@ copyToRemote r key
 			Annex.Content.saveState
 			return ok
 	| Git.repoIsSsh r = do
-		keysrc <- fromRepo $ gitAnnexLocation key
+		keysrc <- inRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 03976fc709..ab84533b28 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -97,12 +97,12 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 
 store :: String -> Key -> Annex Bool
 store h k = do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	runHook h "store" k (Just src) $ return True
 
 storeEncrypted :: String -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 5cd27a6099..836b93b314 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -95,11 +95,11 @@ rsyncKeyDir :: RsyncOpts -> Key -> String
 rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (keyFile k)
 
 store :: RsyncOpts -> Key -> Annex Bool
-store o k = rsyncSend o k =<< fromRepo (gitAnnexLocation k)
+store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
-	src <- fromRepo $ gitAnnexLocation k
+	src <- inRepo $ gitAnnexLocation k
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	rsyncSend o enck tmp
 
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 97ac648218..b79939b902 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -112,7 +112,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 store :: Remote Annex -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
-	dest <- fromRepo $ gitAnnexLocation k
+	dest <- inRepo $ gitAnnexLocation k
 	res <- liftIO $ storeHelper (conn, bucket) r k dest
 	s3Bool res
 
@@ -121,7 +121,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
 	withTmp enck $ \tmp -> do
-		f <- fromRepo $ gitAnnexLocation k
+		f <- inRepo $ gitAnnexLocation k
 		liftIO $ withEncryptedContent cipher (L.readFile f) $ \s -> L.writeFile tmp s
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
 		s3Bool res

From f4bf444ae0933f80b4ec1849ea1c44da43008499 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 22:47:11 -0400
Subject: [PATCH 2585/8313] store content in hashDirLower directories in bare
 repositories

When storing content in bare repositories, use the hashDirLower
directories. Bare repositories can be on USB drives, which might
use the FAT filesystem, and fall afoul of recent bugs in linux's handling
of mixed case on FAT. Using hashDirLower avoids that.
---
 Locations.hs                                              | 5 ++++-
 debian/changelog                                          | 8 ++++++--
 ...nsitivity_on_FAT.mdwn => case_sensitivity_on_FAT.mdwn} | 3 ++-
 3 files changed, 12 insertions(+), 4 deletions(-)
 rename doc/bugs/{directory_remote_and_case_sensitivity_on_FAT.mdwn => case_sensitivity_on_FAT.mdwn} (95%)

diff --git a/Locations.hs b/Locations.hs
index 425e4fdcf9..fb5ee517ee 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -75,7 +75,10 @@ annexLocations key = [using hashDirMixed, using hashDirLower]
 gitAnnexLocation :: Key -> Git.Repo -> IO FilePath
 gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
-		go (Git.workTree r) $ annexLocations key
+		-- bare repositories default to hashDirLower for new
+		-- content, as it's more portable, so check locations
+		-- in reverse order
+		go (Git.workTree r) $ reverse $ annexLocations key
 	| otherwise =
 		go (Git.workTree r  ".git") $ annexLocations key
 	where
diff --git a/debian/changelog b/debian/changelog
index 9cd915885c..8da74af2f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,11 @@
 git-annex (3.20111123) UNRELEASED; urgency=low
 
-  * Put a workaround in the directory special remote for strange behavior
-    with VFAT filesystems on Linux (mounted with shortname=mixed)
+  * The VFAT filesystem on recent versions of Linux, when mounted with
+    shortname=mixed, does not get along well with git-annex's mixed case
+    .git/annex/objects hash directories. To better support it, bare
+    repositories (and the directory special remote) now store content
+    in all-lowercase hash directories. Mixed case hash directories are
+    still used for non-bare directories, which cannot be put in VFAT.
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
diff --git a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn
similarity index 95%
rename from doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
rename to doc/bugs/case_sensitivity_on_FAT.mdwn
index ae653d6197..cb3424e34d 100644
--- a/doc/bugs/directory_remote_and_case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/case_sensitivity_on_FAT.mdwn
@@ -42,4 +42,5 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 >> and for the even more unlikely configuration of a rsync special remote
 >> stored on VFAT. --[[Joey]] 
 
-[[!meta title="bare git repository not supported on VFAT"]]
+>>> Bare repositories now use lowercase. rsync is the only remaining
+>>> unsupported possibility. --[[Joey]]

From e6ef66cea39b8c97a1e5115251e5ed0163c66fb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 23:08:11 -0400
Subject: [PATCH 2586/8313] optimize gitAnnexLocation

For non-bare it's back to doing no work.
---
 Locations.hs | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index fb5ee517ee..a7a1f0cd1f 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -59,28 +59,35 @@ objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
 {- Annexed file's possible locations relative to the .git directory.
- - There are two different possibilities, using different hashes;
- - the first is the default for new content. -}
+ - There are two different possibilities, using different hashes. -}
 annexLocations :: Key -> [FilePath]
-annexLocations key = [using hashDirMixed, using hashDirLower]
+annexLocations key = map (annexLocation key) [hashDirMixed, hashDirLower]
+annexLocation :: Key -> (Key -> FilePath) -> FilePath
+annexLocation key hasher = objectDir  hasher key  f  f
 	where
-		using h = objectDir  h key  f  f
 		f = keyFile key
 
 {- Annexed file's absolute location in a repository.
- - Out of the possible annexLocations, returns the one where the file
- - is actually present. When the file is not present, returns the
- - one where the file should be put.
+ -
+ - When there are multiple possible locations, returns the one where the
+ - file is actually present.
+ -
+ - When the file is not present, returns the location where the file should
+ - be stored.
  -}
 gitAnnexLocation :: Key -> Git.Repo -> IO FilePath
 gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
-		-- bare repositories default to hashDirLower for new
-		-- content, as it's more portable, so check locations
-		-- in reverse order
-		go (Git.workTree r) $ reverse $ annexLocations key
+		{- Bare repositories default to hashDirLower for new
+		 - content, as it's more portable. -}
+		go (Git.workTree r) $
+			map (annexLocation key) [hashDirLower, hashDirMixed]
 	| otherwise =
-		go (Git.workTree r  ".git") $ annexLocations key
+		{- Non-bare repositories only use hashDirMixed, so
+		 - don't need to do any work to check if the file is
+		 - present. -}
+		return $ Git.workTree r  ".git" 
+				annexLocation key hashDirMixed
 	where
 		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
 		check _ [] = return Nothing

From bff6ca2634e5f796d83c1632a66ac00adab0be43 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 23:20:31 -0400
Subject: [PATCH 2587/8313] refactor

---
 Locations.hs        | 11 ++++++++++-
 Remote/Directory.hs | 14 ++------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index a7a1f0cd1f..53a80043ac 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	isLinkToAnnex,
+	annexHashes,
 	hashDirMixed,
 	hashDirLower,
 
@@ -58,10 +59,18 @@ annexDir = addTrailingPathSeparator "annex"
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
+{- Two different directory hashes may be used. The mixed case hash
+ - came first, and is fine, except for the problem of case-strict
+ - filesystems such as Linux VFAT (mounted with shortname=mixed),
+ - which do not allow using a directory "XX" when "xx" already exists.
+ - To support that, some repositories will use a lower case hash. -}
+annexHashes :: [Key -> FilePath]
+annexHashes = [hashDirMixed, hashDirLower]
+
 {- Annexed file's possible locations relative to the .git directory.
  - There are two different possibilities, using different hashes. -}
 annexLocations :: Key -> [FilePath]
-annexLocations key = map (annexLocation key) [hashDirMixed, hashDirLower]
+annexLocations key = map (annexLocation key) annexHashes
 annexLocation :: Key -> (Key -> FilePath) -> FilePath
 annexLocation key hasher = objectDir  hasher key  f  f
 	where
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 891a19ef6b..83302b65a5 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -62,20 +62,10 @@ directorySetup u c = do
 	gitConfigSpecialRemote u c' "directory" dir
 	return $ M.delete "directory" c'
 
-{- Where to store a given Key in the Directory.
- -
- - There are two possible locations to try; this had to be done because
- - on Linux, vfat filesystem mounted with shortname=mixed have a
- - variant of case insensativity that causes miserable failure when
- - hashDirMixed produces eg, "xx" and "XX". The first directory to be
- - created wins the namespace, and the second one cannot then be created.
- - But unlike behavior with shortname=lower, "XX/foo" won't look in
- - "xx/foo".
- -}
+{- Locations to try to access a given Key in the Directory. -}
 locations :: FilePath -> Key -> [FilePath]
-locations d k = [using hashDirMixed, using hashDirLower]
+locations d k = map (\h -> d  h k  f  f) annexHashes
 	where
-		using h = d  h k  f  f
 		f = keyFile k
 
 withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool

From 998d8f796857132688c6d8a539aaf18135439205 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 Nov 2011 23:23:14 -0400
Subject: [PATCH 2588/8313] clarify

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 8da74af2f7..816a571e38 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
     .git/annex/objects hash directories. To better support it, bare
-    repositories (and the directory special remote) now store content
+    repositories (and the directory special remote) now store new content
     in all-lowercase hash directories. Mixed case hash directories are
     still used for non-bare directories, which cannot be put in VFAT.
   * Flush json output, avoiding a buffering problem that could result in

From 598eb2e2daac5d2338b9788814e45bb5b2c16e61 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 Nov 2011 12:01:15 -0400
Subject: [PATCH 2589/8313] cleanup

---
 Annex/Branch.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a92f05b2cc..a890668810 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -186,7 +186,7 @@ tryFastForwardTo :: [Git.Ref] -> Annex Bool
 tryFastForwardTo [] = return True
 tryFastForwardTo (first:rest) = do
 	-- First, check that the git-annex branch does not contain any
-	-- new commits that are in the first other branch. If it does,
+	-- new commits that are not in the first other branch. If it does,
 	-- cannot fast-forward.
 	diverged <- changedBranch first fullname
 	if diverged
@@ -195,7 +195,8 @@ tryFastForwardTo (first:rest) = do
 	where
 		no_ff = return False
 		do_ff branch = do
-			inRepo $ Git.run "update-ref" [Param $ show fullname, Param $ show branch]
+			inRepo $ Git.run "update-ref"
+				[Param $ show fullname, Param $ show branch]
 			return True
 		findbest c [] = return $ Just c
 		findbest c (r:rs)

From 3953f7ab81f4edd540ab66fe25ebec56dc9c7ebc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Dec 2011 17:40:26 -0400
Subject: [PATCH 2590/8313] idea

---
 doc/todo/add_-all_option.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/todo/add_-all_option.mdwn

diff --git a/doc/todo/add_-all_option.mdwn b/doc/todo/add_-all_option.mdwn
new file mode 100644
index 0000000000..cb4722fa78
--- /dev/null
+++ b/doc/todo/add_-all_option.mdwn
@@ -0,0 +1,16 @@
+`--all` would make git-annex operate on either every key with content
+present (or in some cases like `get` and `copy --from` on 
+every keys with content not present).
+
+This would be useful when a repository has a history with deleted files
+whose content you want to keep (so you're not using `dropunused`).
+Or when you have a lot of branches and just want to be able to fsck
+every file referenced in any branch.
+
+A problem with the idea is that `.gitattributes` values for keys not
+currently in the tree would not be available (without horrific anounts of
+grubbing thru history to find where/when the key used to exist). So
+`numcopies` set via `.gitattributes` would not work. This would be a
+particular problem for `drop` and for `--auto`.
+
+--[[Joey]]

From 97f809c0069c7e7e107f10dab614e3f765255abe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 14:18:55 -0400
Subject: [PATCH 2591/8313] wording

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 816a571e38..45088db522 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,10 +2,10 @@ git-annex (3.20111123) UNRELEASED; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
-    .git/annex/objects hash directories. To better support it, bare
+    .git/annex/objects hash directories. To avoid this problem, bare
     repositories (and the directory special remote) now store new content
     in all-lowercase hash directories. Mixed case hash directories are
-    still used for non-bare directories, which cannot be put in VFAT.
+    still used for non-bare repositories, which cannot be put on FAT.
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.

From 0815cc2fc1ffccd89bb942a9129a2c29e291b038 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 14:39:47 -0400
Subject: [PATCH 2592/8313] refactor

---
 Locations.hs        | 52 ++++++++++++++++++++++++++++-----------------
 Remote/Directory.hs |  4 +---
 2 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 53a80043ac..1b5f8108df 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -1,6 +1,6 @@
 {- git-annex file locations
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -8,6 +8,7 @@
 module Locations (
 	keyFile,
 	fileKey,
+	keyPaths,
 	gitAnnexLocation,
 	annexLocations,
 	gitAnnexDir,
@@ -59,22 +60,12 @@ annexDir = addTrailingPathSeparator "annex"
 objectDir :: FilePath
 objectDir = addTrailingPathSeparator $ annexDir  "objects"
 
-{- Two different directory hashes may be used. The mixed case hash
- - came first, and is fine, except for the problem of case-strict
- - filesystems such as Linux VFAT (mounted with shortname=mixed),
- - which do not allow using a directory "XX" when "xx" already exists.
- - To support that, some repositories will use a lower case hash. -}
-annexHashes :: [Key -> FilePath]
-annexHashes = [hashDirMixed, hashDirLower]
-
 {- Annexed file's possible locations relative to the .git directory.
  - There are two different possibilities, using different hashes. -}
 annexLocations :: Key -> [FilePath]
 annexLocations key = map (annexLocation key) annexHashes
-annexLocation :: Key -> (Key -> FilePath) -> FilePath
-annexLocation key hasher = objectDir  hasher key  f  f
-	where
-		f = keyFile key
+annexLocation :: Key -> Hasher -> FilePath
+annexLocation key hasher = objectDir  keyPath key hasher
 
 {- Annexed file's absolute location in a repository.
  -
@@ -150,7 +141,7 @@ gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
 
-{- Converts a key into a filename fragment.
+{- Converts a key into a filename fragment without any directory.
  -
  - Escape "/" in the key name, to keep a flat tree of files and avoid
  - issues with keys containing "/../" or ending with "/" etc. 
@@ -166,6 +157,22 @@ keyFile :: Key -> FilePath
 keyFile key = replace "/" "%" $ replace ":" "&c" $
 	replace "%" "&s" $ replace "&" "&a"  $ show key
 
+{- A location to store a key on the filesystem. A directory hash is used,
+ - to protect against filesystems that dislike having many items in a
+ - single directory.
+ -
+ - The file is put in a directory with the same name, this allows
+ - write-protecting the directory to avoid accidental deletion of the file.
+ -}
+keyPath :: Key -> Hasher -> FilePath
+keyPath key hasher = hasher key  f  f
+	where
+		f = keyFile key
+
+{- All possibile locations to store a key using different directory hashes. -}
+keyPaths :: Key -> [FilePath]
+keyPaths key = map (keyPath key) annexHashes
+
 {- Reverses keyFile, converting a filename fragment (ie, the basename of
  - the symlink target) into a key. -}
 fileKey :: FilePath -> Maybe Key
@@ -178,17 +185,22 @@ prop_idempotent_fileKey :: String -> Bool
 prop_idempotent_fileKey s = Just k == fileKey (keyFile k)
 	where k = stubKey { keyName = s, keyBackendName = "test" }
 
-{- Given a key, generates a short directory name to put it in,
- - to do hashing to protect against filesystems that dislike having
- - many items in a single directory. -}
-hashDirMixed :: Key -> FilePath
+{- Two different directory hashes may be used. The mixed case hash
+ - came first, and is fine, except for the problem of case-strict
+ - filesystems such as Linux VFAT (mounted with shortname=mixed),
+ - which do not allow using a directory "XX" when "xx" already exists.
+ - To support that, some repositories will use a lower case hash. -}
+type Hasher = Key -> FilePath
+annexHashes :: [Hasher]
+annexHashes = [hashDirMixed, hashDirLower]
+
+hashDirMixed :: Hasher
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
 	where
 		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]
 		ABCD (a,b,c,d) = md5 $ Str $ show k
 
-{- Generates a hash directory that is all lower case. -}
-hashDirLower :: Key -> FilePath
+hashDirLower :: Hasher
 hashDirLower k = addTrailingPathSeparator $ take 3 dir  drop 3 dir
 	where
 		dir = take 6 $ md5s $ Str $ show k
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 83302b65a5..5f294f0bef 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -64,9 +64,7 @@ directorySetup u c = do
 
 {- Locations to try to access a given Key in the Directory. -}
 locations :: FilePath -> Key -> [FilePath]
-locations d k = map (\h -> d  h k  f  f) annexHashes
-	where
-		f = keyFile k
+locations d k = map (d ) (keyLocations k)
 
 withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 withCheckedFile _ [] _ _ = return False

From db5b479f3f9c68c05bd172b90fe5cab0336f378d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 14:56:48 -0400
Subject: [PATCH 2593/8313] use lowercase hash by default; non-bare repos are a
 special case

Directory special remotes will now always store keys in the lowercase name,
which avoids the complication of catching failures to create the mixed case
name.

Git remotes using http will now try the lowercase name first.
---
 Locations.hs        |  8 +++-----
 Remote/Directory.hs | 24 +++++++++++-------------
 2 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 1b5f8108df..2f4a9200db 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,7 +21,6 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	isLinkToAnnex,
-	annexHashes,
 	hashDirMixed,
 	hashDirLower,
 
@@ -80,8 +79,7 @@ gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
 		{- Bare repositories default to hashDirLower for new
 		 - content, as it's more portable. -}
-		go (Git.workTree r) $
-			map (annexLocation key) [hashDirLower, hashDirMixed]
+		go (Git.workTree r) (annexLocations key)
 	| otherwise =
 		{- Non-bare repositories only use hashDirMixed, so
 		 - don't need to do any work to check if the file is
@@ -189,10 +187,10 @@ prop_idempotent_fileKey s = Just k == fileKey (keyFile k)
  - came first, and is fine, except for the problem of case-strict
  - filesystems such as Linux VFAT (mounted with shortname=mixed),
  - which do not allow using a directory "XX" when "xx" already exists.
- - To support that, some repositories will use a lower case hash. -}
+ - To support that, most repositories use the lower case hash for new data. -}
 type Hasher = Key -> FilePath
 annexHashes :: [Hasher]
-annexHashes = [hashDirMixed, hashDirLower]
+annexHashes = [hashDirLower, hashDirMixed]
 
 hashDirMixed :: Hasher
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 5f294f0bef..a6077d813c 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -64,7 +64,7 @@ directorySetup u c = do
 
 {- Locations to try to access a given Key in the Directory. -}
 locations :: FilePath -> Key -> [FilePath]
-locations d k = map (d ) (keyLocations k)
+locations d k = map (d ) (keyPaths k)
 
 withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 withCheckedFile _ [] _ _ = return False
@@ -95,18 +95,16 @@ storeEncrypted d (cipher, enck) k = do
 			return True
 
 storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
-storeHelper d key a = withCheckedFile check d key go
-	where
-		check dest = isJust <$> mkdir (parentDir dest)
-		mkdir = catchMaybeIO . createDirectoryIfMissing True
-		go dest = do
-			let dir = parentDir dest
-			allowWrite dir
-			ok <- a dest
-			when ok $ do
-				preventWrite dest
-				preventWrite dir
-			return ok
+storeHelper d key a = do
+	let dest = head $ locations d key
+	let dir = parentDir dest
+	createDirectoryIfMissing True dir
+	allowWrite dir
+	ok <- a dest
+	when ok $ do
+		preventWrite dest
+		preventWrite dir
+	return ok
 
 retrieve :: FilePath -> Key -> FilePath -> Annex Bool
 retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f

From fb68a7881f725a7b097f8b0f1b347f24dfea5d59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 15:50:27 -0400
Subject: [PATCH 2594/8313] convert rsync special backend to using both hash
 directory types

---
 Locations.hs     |  1 +
 Remote/Rsync.hs  | 64 +++++++++++++++++++++++++++++-------------------
 debian/changelog |  9 ++++---
 3 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 2f4a9200db..1179886ade 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,6 +21,7 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	isLinkToAnnex,
+	annexHashes,
 	hashDirMixed,
 	hashDirLower,
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 836b93b314..651ed4de87 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -86,13 +86,26 @@ rsyncEscape o s
 	| rsyncUrlIsShell (rsyncUrl o) = shellEscape s
 	| otherwise = s
 
-rsyncKey :: RsyncOpts -> Key -> String
-rsyncKey o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (f  f)
-        where
+rsyncUrls :: RsyncOpts -> Key -> [String]
+rsyncUrls o k = map use annexHashes
+	where
+		use h = rsyncUrl o  h k  rsyncEscape o (f  f)
                 f = keyFile k
 
-rsyncKeyDir :: RsyncOpts -> Key -> String
-rsyncKeyDir o k = rsyncUrl o  hashDirMixed k  rsyncEscape o (keyFile k)
+rsyncUrlDirs :: RsyncOpts -> Key -> [String]
+rsyncUrlDirs o k = map use annexHashes
+	where
+		use h = rsyncUrl o  h k  rsyncEscape o (keyFile k)
+
+withRsyncUrl :: RsyncOpts -> Key -> (FilePath -> Annex Bool) -> Annex Bool
+withRsyncUrl o k a = go $ rsyncUrls o k
+	where
+		go [] = return False
+		go (u:us) = do
+			ok <- a u
+			if ok
+				then return ok
+				else go us
 
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
@@ -104,10 +117,10 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieve o k f = rsyncRemote o
+retrieve o k f = withRsyncUrl o k $ \u -> rsyncRemote o
 	-- use inplace when retrieving to support resuming
 	[ Param "--inplace"
-	, Param $ rsyncKey o k
+	, Param u
 	, Param f
 	]
 
@@ -121,27 +134,30 @@ retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 		else return res
 
 remove :: RsyncOpts -> Key -> Annex Bool
-remove o k = withRsyncScratchDir $ \tmp -> do
-	{- Send an empty directory to rysnc as the parent directory
-         - of the file to remove. -}
-	let dummy = tmp  keyFile k
-	liftIO $ createDirectoryIfMissing True dummy
-	liftIO $ rsync $ rsyncOptions o ++
-		[ Params "--delete --recursive"
-		, partialParams
-		, Param $ addTrailingPathSeparator dummy
-		, Param $ rsyncKeyDir o k
-		]
+remove o k = any (== True) <$> sequence (map go (rsyncUrlDirs o k))
+	where
+		go d = withRsyncScratchDir $ \tmp -> liftIO $ do
+			{- Send an empty directory to rysnc as the
+			 - parent directory of the file to remove. -}
+			let dummy = tmp  keyFile k
+			createDirectoryIfMissing True dummy
+			rsync $ rsyncOptions o ++
+				[ Params "--quiet --delete --recursive"
+				, partialParams
+				, Param $ addTrailingPathSeparator dummy
+				, Param d
+				]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
 checkPresent r o k = do
 	showAction $ "checking " ++ Git.repoDescribe r
-	-- note: Does not currently differnetiate between rsync failing
+	-- note: Does not currently differentiate between rsync failing
 	-- to connect, and the file not being present.
-	res <- liftIO $ boolSystem "sh" [Param "-c", Param cmd]
-	return $ Right res
+	Right <$> check
 	where
-		cmd = "rsync --quiet " ++ shellEscape (rsyncKey o k) ++ " 2>/dev/null"
+ 		check = withRsyncUrl o k $ \u ->
+			liftIO $ boolSystem "sh" [Param "-c", Param (cmd u)]
+		cmd u = "rsync --quiet " ++ shellEscape u ++ " 2>/dev/null"
 
 {- Rsync params to enable resumes of sending files safely,
  - ensure that files are only moved into place once complete
@@ -182,7 +198,7 @@ rsyncRemote o params = do
    directories. -}
 rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool
 rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
-	let dest = tmp  hashDirMixed k  f  f
+	let dest = tmp  head (keyPaths k)
 	liftIO $ createDirectoryIfMissing True $ parentDir dest
 	liftIO $ createLink src dest
 	rsyncRemote o
@@ -192,5 +208,3 @@ rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
 		, Param $ addTrailingPathSeparator tmp
 		, Param $ rsyncUrl o
 		]
-	where
-		f = keyFile k
diff --git a/debian/changelog b/debian/changelog
index 45088db522..d038c7849c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,10 +2,11 @@ git-annex (3.20111123) UNRELEASED; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
-    .git/annex/objects hash directories. To avoid this problem, bare
-    repositories (and the directory special remote) now store new content
-    in all-lowercase hash directories. Mixed case hash directories are
-    still used for non-bare repositories, which cannot be put on FAT.
+    .git/annex/objects hash directories. To avoid this problem, new content
+    is now stored in all-lowercase hash directories. Except for non-bare
+    repositories which would be a pain to transition and cannot be put on FAT.
+    (Old mixed-case hash directories are still tried for backwards
+    compatibility.)
   * Flush json output, avoiding a buffering problem that could result in
     doubled output.
   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.

From e19dc8554723a148e6b809da4989a747f3aa925e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:10:52 -0400
Subject: [PATCH 2595/8313] factor out untilTrue

---
 Remote/Git.hs          |  5 +----
 Remote/Rsync.hs        | 50 +++++++++++++++++-------------------------
 Remote/Web.hs          |  6 ++---
 Utility/Conditional.hs |  6 +++++
 4 files changed, 29 insertions(+), 38 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 07afc02742..99ca9fe8e2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -200,10 +200,7 @@ copyFromRemote r key file
 	| Git.repoIsHttp r = liftIO $ downloadurls $ keyUrls r key
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
 	where
-		downloadurls [] = return False
-		downloadurls (u:us) = do
-			ok <- Url.download u file
-			if ok then return ok else downloadurls us
+		downloadurls us = untilTrue us $ \u -> Url.download u file
 
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 651ed4de87..81107cb561 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -97,16 +97,6 @@ rsyncUrlDirs o k = map use annexHashes
 	where
 		use h = rsyncUrl o  h k  rsyncEscape o (keyFile k)
 
-withRsyncUrl :: RsyncOpts -> Key -> (FilePath -> Annex Bool) -> Annex Bool
-withRsyncUrl o k a = go $ rsyncUrls o k
-	where
-		go [] = return False
-		go (u:us) = do
-			ok <- a u
-			if ok
-				then return ok
-				else go us
-
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
 
@@ -117,12 +107,13 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieve o k f = withRsyncUrl o k $ \u -> rsyncRemote o
-	-- use inplace when retrieving to support resuming
-	[ Param "--inplace"
-	, Param u
-	, Param f
-	]
+retrieve o k f = untilTrue (rsyncUrls o k) $ \u ->
+	rsyncRemote o
+		-- use inplace when retrieving to support resuming
+		[ Param "--inplace"
+		, Param u
+		, Param f
+		]
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
@@ -134,19 +125,18 @@ retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 		else return res
 
 remove :: RsyncOpts -> Key -> Annex Bool
-remove o k = any (== True) <$> sequence (map go (rsyncUrlDirs o k))
-	where
-		go d = withRsyncScratchDir $ \tmp -> liftIO $ do
-			{- Send an empty directory to rysnc as the
-			 - parent directory of the file to remove. -}
-			let dummy = tmp  keyFile k
-			createDirectoryIfMissing True dummy
-			rsync $ rsyncOptions o ++
-				[ Params "--quiet --delete --recursive"
-				, partialParams
-				, Param $ addTrailingPathSeparator dummy
-				, Param d
-				]
+remove o k = untilTrue (rsyncUrlDirs o k) $ \d ->
+	withRsyncScratchDir $ \tmp -> liftIO $ do
+		{- Send an empty directory to rysnc as the
+		 - parent directory of the file to remove. -}
+		let dummy = tmp  keyFile k
+		createDirectoryIfMissing True dummy
+		rsync $ rsyncOptions o ++
+			[ Params "--quiet --delete --recursive"
+			, partialParams
+			, Param $ addTrailingPathSeparator dummy
+			, Param d
+			]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
 checkPresent r o k = do
@@ -155,7 +145,7 @@ checkPresent r o k = do
 	-- to connect, and the file not being present.
 	Right <$> check
 	where
- 		check = withRsyncUrl o k $ \u ->
+ 		check = untilTrue (rsyncUrls o k) $ \u ->
 			liftIO $ boolSystem "sh" [Param "-c", Param (cmd u)]
 		cmd u = "rsync --quiet " ++ shellEscape u ++ " 2>/dev/null"
 
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 64fcd51aaa..5871ae8dae 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -71,8 +71,6 @@ checkKey key = do
 		then return $ Right False
 		else return . Right =<< checkKey' us
 checkKey' :: [URLString] -> Annex Bool
-checkKey' [] = return False
-checkKey' (u:us) = do
+checkKey' us = untilTrue us $ \u -> do
 	showAction $ "checking " ++ u
-	e <- liftIO $ Url.exists u
-	if e then return e else checkKey' us
+	liftIO $ Url.exists u
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
index 85e39ec64c..7a0df4b482 100644
--- a/Utility/Conditional.hs
+++ b/Utility/Conditional.hs
@@ -9,6 +9,12 @@ module Utility.Conditional where
 
 import Control.Monad (when, unless)
 
+untilTrue :: Monad m => [v] -> (v -> m Bool) -> m Bool
+untilTrue [] _ = return False
+untilTrue (v:vs) a = do
+	ok <- a v
+	if ok then return ok else untilTrue vs a
+
 whenM :: Monad m => m Bool -> m () -> m ()
 whenM c a = c >>= flip when a
 

From 6f29f9db725131d37af86e2746ff83a6765b4239 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:15:18 -0400
Subject: [PATCH 2596/8313] rename

---
 ...tree_options.mdwn => on--git-dir_and_--work-tree_options.mdwn} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/bugs/{--git-dir_and_--work-tree_options.mdwn => on--git-dir_and_--work-tree_options.mdwn} (100%)

diff --git a/doc/bugs/--git-dir_and_--work-tree_options.mdwn b/doc/bugs/on--git-dir_and_--work-tree_options.mdwn
similarity index 100%
rename from doc/bugs/--git-dir_and_--work-tree_options.mdwn
rename to doc/bugs/on--git-dir_and_--work-tree_options.mdwn

From 6d8d25262c40d7f422743ddbfa13e362635721e8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:26:56 -0400
Subject: [PATCH 2597/8313] close

---
 doc/bugs/case_sensitivity_on_FAT.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn
index cb3424e34d..682acc71d7 100644
--- a/doc/bugs/case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/case_sensitivity_on_FAT.mdwn
@@ -44,3 +44,6 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 
 >>> Bare repositories now use lowercase. rsync is the only remaining
 >>> unsupported possibility. --[[Joey]]
+>>>> Everything now uses lowercase, with the exception of non-bare
+>>>> repos, which cannot be on FAT anyway due to using symlinks. [[done]]
+>>>> --[[Joey]] 

From 7b08584c5553c22b322a2d9c268fda855666f4ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:59:09 -0400
Subject: [PATCH 2598/8313] close

---
 doc/bugs/case_sensitivity_on_FAT.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/case_sensitivity_on_FAT.mdwn b/doc/bugs/case_sensitivity_on_FAT.mdwn
index cb3424e34d..682acc71d7 100644
--- a/doc/bugs/case_sensitivity_on_FAT.mdwn
+++ b/doc/bugs/case_sensitivity_on_FAT.mdwn
@@ -44,3 +44,6 @@ I wonder if the directory remote should use hashDirLower instead of hashDirMixed
 
 >>> Bare repositories now use lowercase. rsync is the only remaining
 >>> unsupported possibility. --[[Joey]]
+>>>> Everything now uses lowercase, with the exception of non-bare
+>>>> repos, which cannot be on FAT anyway due to using symlinks. [[done]]
+>>>> --[[Joey]] 

From 251c01d51e22dc295359ba1f85144afc4c178e7a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 16:59:55 -0400
Subject: [PATCH 2599/8313] dead: A command which says that a repository is
 gone for good and you don't want git-annex to mention it again.

---
 GitAnnex.hs                                    |  2 ++
 Logs/Location.hs                               | 10 ++++++++--
 Logs/Trust.hs                                  |  6 ++++--
 Types/TrustLevel.hs                            |  2 +-
 debian/changelog                               |  2 ++
 doc/git-annex.mdwn                             |  5 +++++
 .../what_to_do_when_you_lose_a_repository.mdwn | 18 +++++++++---------
 doc/trust.mdwn                                 | 10 +++++++++-
 8 files changed, 40 insertions(+), 15 deletions(-)

diff --git a/GitAnnex.hs b/GitAnnex.hs
index 42a6b7fd78..d768499ddb 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -46,6 +46,7 @@ import qualified Command.Uninit
 import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
+import qualified Command.Dead
 import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
@@ -70,6 +71,7 @@ cmds = concat
 	, Command.Trust.def
 	, Command.Untrust.def
 	, Command.Semitrust.def
+	, Command.Dead.def
 	, Command.AddUrl.def
 	, Command.FromKey.def
 	, Command.DropKey.def
diff --git a/Logs/Location.hs b/Logs/Location.hs
index cb21a2d1c0..27b4d709e7 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -27,6 +27,7 @@ module Logs.Location (
 import Common.Annex
 import qualified Annex.Branch
 import Logs.Presence
+import Logs.Trust
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Key -> UUID -> LogStatus -> Annex ()
@@ -34,9 +35,14 @@ logChange key (UUID u) s = addLog (logFile key) =<< logNow s u
 logChange _ NoUUID _ = return ()
 
 {- Returns a list of repository UUIDs that, according to the log, have
- - the value of a key. -}
+ - the value of a key.
+ -
+ - Dead repositories are skipped.
+ -}
 keyLocations :: Key -> Annex [UUID]
-keyLocations key = map toUUID <$> (currentLog . logFile) key
+keyLocations key = do
+	l <- map toUUID <$> (currentLog . logFile) key
+	snd <$> trustPartition DeadTrusted l
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index e447fbebc8..f18f425110 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -67,12 +67,14 @@ parseTrust s
 		w = words s
 		parse "1" = Trusted
 		parse "0" = UnTrusted
+		parse "X" = DeadTrusted
 		parse _ = SemiTrusted
 
 showTrust :: TrustLevel -> String
-showTrust SemiTrusted = "?"
-showTrust UnTrusted = "0"
 showTrust Trusted = "1"
+showTrust UnTrusted = "0"
+showTrust DeadTrusted = "X"
+showTrust SemiTrusted = "?"
 
 {- Changes the trust level for a uuid in the trustLog. -}
 trustSet :: UUID -> TrustLevel -> Annex ()
diff --git a/Types/TrustLevel.hs b/Types/TrustLevel.hs
index ddb8e45e49..99d7497303 100644
--- a/Types/TrustLevel.hs
+++ b/Types/TrustLevel.hs
@@ -14,7 +14,7 @@ import qualified Data.Map as M
 
 import Types.UUID
 
-data TrustLevel = SemiTrusted | UnTrusted | Trusted
+data TrustLevel = Trusted | SemiTrusted | UnTrusted | DeadTrusted
 	deriving Eq
 
 type TrustMap = M.Map UUID TrustLevel
diff --git a/debian/changelog b/debian/changelog
index d038c7849c..4a2a3dd06b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -15,6 +15,8 @@ git-annex (3.20111123) UNRELEASED; urgency=low
     git repository.
   * --inbackend can be used to make git-annex only operate on files
     whose content is stored using a specified key-value backend.
+  * dead: A command which says that a repository is gone for good
+    and you don't want git-annex to mention it again.
 
  -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index e98c89fe39..9df5c3c6d1 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -173,6 +173,11 @@ subdirectories).
 
   Returns a repository to the default semi trusted state.
 
+* dead [repository ...]
+
+  Indicates that the repository has been irretrevably lost.
+  (To undo, use semitrust.)
+
 # REPOSITORY MAINTENANCE COMMANDS
 
 * fsck [path ...]
diff --git a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
index 16a55b37b3..3be13b8abd 100644
--- a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
+++ b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
@@ -4,16 +4,16 @@ drive died or some other misfortune has befallen your data.
 Unless you configured backups, git-annex can't get your data back. But it
 can help you deal with the loss.
 
-First, go somewhere that knows about the lost repository, and mark it as
-untrusted.
+Go somewhere that knows about the lost repository, and mark it as
+dead:
 
-	git annex untrust usbdrive
+	git annex dead usbdrive
 
-To remind yourself later what happened, you can change its description, too:
+This retains the [[location_tracking]] information for the repository,
+but avoids trying to access it, or list it as a location where files
+are present.
 
-	git annex describe usbdrive "USB drive lost in Timbuktu. Probably gone forever."
+If you later found the drive, you could let git-annex know it's found
+like so:
 
-This retains the [[location_tracking]] information for the repository.
-Maybe you'll find the drive later. Maybe that's impossible. Either way,
-this lets git-annex tell you why a file is no longer accessible, and
-it avoids it relying on that drive to hold any content.
+	git annex semitrusted usbdrive
diff --git a/doc/trust.mdwn b/doc/trust.mdwn
index 7505a7af65..1fd47fd1d3 100644
--- a/doc/trust.mdwn
+++ b/doc/trust.mdwn
@@ -1,8 +1,9 @@
-Git-annex supports three levels of trust of a repository:
+Git-annex supports several levels of trust of a repository:
 
 * semitrusted (default)
 * untrusted
 * trusted
+* dead
 
 ## semitrusted
 
@@ -49,3 +50,10 @@ trust temporarily.
 
 To configure a repository as fully and permanently trusted,
 use the `git annex trust` command.
+
+## dead
+
+This is used to indicate that you have no trust that the repository
+exists at all. It's appropriate to use when a drive has been lost,
+or a directory irretrevably deleted. It will make git-annex avoid
+even showing the repository as a place where data might still reside.

From f0cc42685e42a493c83eb85de02e61c54f69e4f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 19:21:56 -0400
Subject: [PATCH 2600/8313] fix display of dead repositories in status

---
 Command/Status.hs | 1 +
 Logs/Trust.hs     | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index a47f21b91a..0fefda1f6b 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -58,6 +58,7 @@ fast_stats =
 	, remote_list Trusted "trusted"
 	, remote_list SemiTrusted "semitrusted"
 	, remote_list UnTrusted "untrusted"
+	, remote_list DeadTrusted "dead"
 	]
 slow_stats :: [Stat]
 slow_stats = 
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index f18f425110..196666a84e 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -38,7 +38,8 @@ trustPartition level ls
 	| level == SemiTrusted = do
 		t <- trustGet Trusted
 		u <- trustGet UnTrusted
-		let uncandidates = t ++ u
+		d <- trustGet DeadTrusted
+		let uncandidates = t ++ u ++ d
 		return $ partition (`notElem` uncandidates) ls
 	| otherwise = do
 		candidates <- trustGet level

From b5930f6d076d266b337b415447f448fbb14d9ea3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 2 Dec 2011 19:22:43 -0400
Subject: [PATCH 2601/8313] add

---
 Command/Dead.hs | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 Command/Dead.hs

diff --git a/Command/Dead.hs b/Command/Dead.hs
new file mode 100644
index 0000000000..192551e207
--- /dev/null
+++ b/Command/Dead.hs
@@ -0,0 +1,32 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Dead where
+
+import Common.Annex
+import Command
+import qualified Remote
+import Logs.Trust
+
+def :: [Command]
+def = [command "dead" (paramRepeating paramRemote) seek
+	"hide a lost repository"]
+
+seek :: [CommandSeek]
+seek = [withWords start]
+
+start :: [String] -> CommandStart
+start ws = do
+	let name = unwords ws
+	showStart "dead " name
+	u <- Remote.nameToUUID name
+	next $ perform u
+
+perform :: UUID -> CommandPerform
+perform uuid = do
+	trustSet uuid DeadTrusted
+	next $ return True

From 64672c6262048e9498be667f8dc7460c0a9189e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 09:10:23 -0400
Subject: [PATCH 2602/8313] refactor

---
 Common.hs              | 1 +
 Locations.hs           | 6 +-----
 Remote/Web.hs          | 1 -
 Utility/Conditional.hs | 6 ------
 Utility/Monad.hs       | 4 ++++
 5 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/Common.hs b/Common.hs
index a3802da5f2..7e8dd9a2a5 100644
--- a/Common.hs
+++ b/Common.hs
@@ -24,3 +24,4 @@ import Utility.Conditional as X
 import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
+import Utility.Monad as X
diff --git a/Locations.hs b/Locations.hs
index 1179886ade..3843495f9c 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -89,11 +89,7 @@ gitAnnexLocation key r
 				annexLocation key hashDirMixed
 	where
 		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
-		check _ [] = return Nothing
-		check dir (l:ls) = do
-			let f = dir  l
-			e <- doesFileExist f
-			if e then return (Just f) else check dir ls
+		check dir = firstM $ \f -> doesFileExist $ dir  f
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 5871ae8dae..d5acd7d862 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -13,7 +13,6 @@ import qualified Git
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
-import Utility.Monad
 
 remote :: RemoteType Annex
 remote = RemoteType {
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
index 7a0df4b482..85e39ec64c 100644
--- a/Utility/Conditional.hs
+++ b/Utility/Conditional.hs
@@ -9,12 +9,6 @@ module Utility.Conditional where
 
 import Control.Monad (when, unless)
 
-untilTrue :: Monad m => [v] -> (v -> m Bool) -> m Bool
-untilTrue [] _ = return False
-untilTrue (v:vs) a = do
-	ok <- a v
-	if ok then return ok else untilTrue vs a
-
 whenM :: Monad m => m Bool -> m () -> m ()
 whenM c a = c >>= flip when a
 
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 9523e17165..0d1675fa49 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -24,3 +24,7 @@ firstM p (x:xs) = do
  - stopping once one is found. -}
 anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
 anyM p = liftM isJust . firstM p
+
+{- Runs an action on values from a list until it succeeds. -}
+untilTrue :: (Monad m) => [a] -> (a -> m Bool) -> m Bool
+untilTrue = flip anyM

From 2f8d75638ac70e968b48a9675be32d2eead6dbc5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 21:01:22 -0400
Subject: [PATCH 2603/8313] update for dead repos

---
 doc/internals.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 5559d122b9..d84b3c4898 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -62,8 +62,8 @@ Records the [[trust]] information for repositories. Does not exist unless
 [[trust]] values are configured.
 
 The file format is one line per repository, with the uuid followed by a
-space, and then either 1 (trusted), 0 (untrusted), or ? (semi-trusted),
-and finally a timestamp.
+space, and then either `1` (trusted), `0` (untrusted), `?` (semi-trusted),
+`X` (dead) and finally a timestamp.
 
 Example:
 

From ff5df842ea85378288519b4e964faf28d67b6dbb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 21:13:21 -0400
Subject: [PATCH 2604/8313] releasing version 3.20111203

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 4a2a3dd06b..41c251b849 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111123) UNRELEASED; urgency=low
+git-annex (3.20111203) unstable; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with
     shortname=mixed, does not get along well with git-annex's mixed case
@@ -18,7 +18,7 @@ git-annex (3.20111123) UNRELEASED; urgency=low
   * dead: A command which says that a repository is gone for good
     and you don't want git-annex to mention it again.
 
- -- Joey Hess   Tue, 22 Nov 2011 17:53:42 -0400
+ -- Joey Hess   Sat, 03 Dec 2011 21:01:45 -0400
 
 git-annex (3.20111122) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index c257eaff63..7be78053f8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111122
+Version: 3.20111203
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 58567045b737d1ffbcd143934a85e22e62da8044 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Dec 2011 21:13:29 -0400
Subject: [PATCH 2605/8313] add news item for git-annex 3.20111203

---
 doc/news/version_3.20111025.mdwn |  8 --------
 doc/news/version_3.20111203.mdwn | 19 +++++++++++++++++++
 2 files changed, 19 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_3.20111025.mdwn
 create mode 100644 doc/news/version_3.20111203.mdwn

diff --git a/doc/news/version_3.20111025.mdwn b/doc/news/version_3.20111025.mdwn
deleted file mode 100644
index 90e4227bae..0000000000
--- a/doc/news/version_3.20111025.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20111025 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * A remote can have a annexUrl configured, that is used by git-annex
-     instead of its usual url. (Similar to pushUrl.)
-   * migrate: Copy url logs for keys when migrating.
-   * git-annex-shell: GIT\_ANNEX\_SHELL\_READONLY and GIT\_ANNEX\_SHELL\_LIMITED
-     environment variables can be set to limit what commands can be run.
-     This is used by gitolite's new git-annex support!"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111203.mdwn b/doc/news/version_3.20111203.mdwn
new file mode 100644
index 0000000000..5be6e21424
--- /dev/null
+++ b/doc/news/version_3.20111203.mdwn
@@ -0,0 +1,19 @@
+git-annex 3.20111203 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * The VFAT filesystem on recent versions of Linux, when mounted with
+     shortname=mixed, does not get along well with git-annex's mixed case
+     .git/annex/objects hash directories. To avoid this problem, new content
+     is now stored in all-lowercase hash directories. Except for non-bare
+     repositories which would be a pain to transition and cannot be put on FAT.
+     (Old mixed-case hash directories are still tried for backwards
+     compatibility.)
+   * Flush json output, avoiding a buffering problem that could result in
+     doubled output.
+   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
+   * Bugfix: dropunused did not drop keys with two spaces in their name.
+   * Support for storing .git/annex on a different device than the rest of the
+     git repository.
+   * --inbackend can be used to make git-annex only operate on files
+     whose content is stored using a specified key-value backend.
+   * dead: A command which says that a repository is gone for good
+     and you don't want git-annex to mention it again."""]]
\ No newline at end of file

From b6c8a0119ad68f466e24ec2bab3a2bacf6ea726e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Dec 2011 12:23:10 -0400
Subject: [PATCH 2606/8313] map: Fix a failure to detect a loop when both
 repositories are local and refer to each other with relative paths.

---
 Command/Map.hs   | 9 +++++----
 debian/changelog | 7 +++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index 8755bc7c2d..6b1e8d5bbc 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -138,15 +138,16 @@ spider' (r:rs) known
 
 		-- The remotes will be relative to r', and need to be
 		-- made absolute for later use.
-		let remotes = map (absRepo r') (Git.remotes r')
+		remotes <- mapM (absRepo r') (Git.remotes r')
 		let r'' = Git.remotesAdd r' remotes
 
 		spider' (rs ++ remotes) (r'':known)
 
-absRepo :: Git.Repo -> Git.Repo -> Git.Repo
+{- Converts repos to a common absolute form. -}
+absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo
 absRepo reference r
-	| Git.repoIsUrl reference = Git.localToUrl reference r
-	| otherwise = r
+	| Git.repoIsUrl reference = return $ Git.localToUrl reference r
+	| otherwise = liftIO $ Git.repoFromAbsPath =<< absPath (Git.workTree r)
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
diff --git a/debian/changelog b/debian/changelog
index 41c251b849..7feb8f8ca9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20111204) UNRELEASED; urgency=low
+
+  * map: Fix a failure to detect a loop when both repositories are local
+    and refer to each other with relative paths.
+
+ -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
+
 git-annex (3.20111203) unstable; urgency=low
 
   * The VFAT filesystem on recent versions of Linux, when mounted with

From 04e5bf3644b5999d9b5251dc5fa0ee43bac6b435 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Mon, 5 Dec 2011 19:09:14 +0000
Subject: [PATCH 2607/8313]

---
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn

diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
new file mode 100644
index 0000000000..b2124ae793
--- /dev/null
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -0,0 +1 @@
+Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version?

From 622c2b9b8b780a7df73fa5f02fb8b68116584d9d Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Mon, 5 Dec 2011 19:13:20 +0000
Subject: [PATCH 2608/8313]

---
 doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn

diff --git a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
new file mode 100644
index 0000000000..474d9c9d8f
--- /dev/null
+++ b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
@@ -0,0 +1 @@
+A failure during "make test" should be signalled to the caller by means of a non-zero exit code. Without that signal, it's very hard to run the regression test suite in an automated fashion.

From fd5c9791212c049594f4dfc4daf5522c4655cb46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 15:38:04 -0400
Subject: [PATCH 2609/8313] fixed a long time ago

---
 ...Please_abort_build_if___34__make_test__34___fails.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
index 474d9c9d8f..592b5e0773 100644
--- a/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
+++ b/doc/todo/Please_abort_build_if___34__make_test__34___fails.mdwn
@@ -1 +1,7 @@
-A failure during "make test" should be signalled to the caller by means of a non-zero exit code. Without that signal, it's very hard to run the regression test suite in an automated fashion.
+A failure during "make test" should be signalled to the caller by means of
+a non-zero exit code. Without that signal, it's very hard to run the
+regression test suite in an automated fashion.
+
+> git-annex used to have a Makefile that ignored make test exit status,
+> but that was fixed in commit dab5bddc64ab4ad479a1104748c15d194e138847,
+> in October 6th. [[done]] --[[Joey]] 

From 2a1e3bceb30a51f38bebc93e9321ed12d9567ec0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 15:42:02 -0400
Subject: [PATCH 2610/8313] respond

---
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
index b2124ae793..8b88f103ec 100644
--- a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -1 +1,4 @@
 Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version?
+
+> I hope not. I have been waiting for it to land in Debian before trying to 
+> deal with its changes. --[[Joey]]

From f3a2f60abc7c7c5a8e29ce96675da46c1861c50e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 22:51:37 -0400
Subject: [PATCH 2611/8313] adjust to build with monad-control-0.3

I had to, I hope temporarily, lose my nice Annex newtype, and use a type
synonym. This because I cannot find a way to derive a MonadBaseControl
instance of the Annex newtype. I've emailed Bas van Dijk in hope he can
help get the newtype back.

Otherwise appears to build & work.
---
 Annex.hs           | 16 ++++------------
 Annex/Exception.hs |  6 +++---
 2 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 6d245a92d1..d21f0a06c6 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -22,7 +22,7 @@ module Annex (
 	fromRepo,
 ) where
 
-import Control.Monad.IO.Control
+import Control.Monad.Trans.Control
 import Control.Monad.State
 
 import Common
@@ -38,15 +38,7 @@ import Types.UUID
 import qualified Utility.Matcher
 
 -- git-annex's monad
-newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
-	deriving (
-		Monad,
-		MonadIO,
-		MonadControlIO,
-		MonadState AnnexState,
-		Functor,
-		Applicative
-	)
+type Annex = StateT AnnexState IO
 
 data OutputType = NormalOutput | QuietOutput | JSONOutput
 
@@ -102,9 +94,9 @@ new gitrepo = newState <$> Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run s a = runStateT (runAnnex a) s
+run s a = runStateT a s
 eval :: AnnexState -> Annex a -> IO a
-eval s a = evalStateT (runAnnex a) s
+eval s a = evalStateT a s
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}
diff --git a/Annex/Exception.hs b/Annex/Exception.hs
index c147439a1c..cb36d1bdbc 100644
--- a/Annex/Exception.hs
+++ b/Annex/Exception.hs
@@ -11,8 +11,8 @@ module Annex.Exception (
 	throw,
 ) where
 
-import Control.Exception.Control (handle)
-import Control.Monad.IO.Control (liftIOOp)
+import Control.Exception.Lifted (handle)
+import Control.Monad.Trans.Control (liftBaseOp)
 import Control.Exception hiding (handle, throw)
 
 import Common.Annex
@@ -20,7 +20,7 @@ import Common.Annex
 {- Runs an Annex action, with setup and cleanup both in the IO monad. -}
 bracketIO :: IO c -> (c -> IO b) -> Annex a -> Annex a
 bracketIO setup cleanup go =
-	liftIOOp (Control.Exception.bracket setup cleanup) (const go)
+	liftBaseOp (Control.Exception.bracket setup cleanup) (const go)
 
 {- Throws an exception in the Annex monad. -}
 throw :: Control.Exception.Exception e => e -> Annex a

From 20d729514adaac04a33f0fbf99016a1276782f81 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Dec 2011 22:54:30 -0400
Subject: [PATCH 2612/8313] added a branch for the new monad-control 0.3

---
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
index 8b88f103ec..ca68c2c913 100644
--- a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -1,4 +1,7 @@
 Git-annex doesn't compile with the latest version of monad-control. Would it be hard to support that new version?
 
-> I hope not. I have been waiting for it to land in Debian before trying to 
-> deal with its changes. --[[Joey]]
+> I have been waiting for it to land in Debian before trying to 
+> deal with its changes.
+> 
+> There is now a branch in git called `new-monad-control` that will build
+> with the new monad-control. --[[Joey]]

From 27ce7ba5b49cf2ec4c8a778f90320d714fd17155 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 12:43:56 +0000
Subject: [PATCH 2613/8313]

---
 ...nex_add_crash_and_subsequent_recovery.mdwn | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn b/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn
new file mode 100644
index 0000000000..3f3b943a0b
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery.mdwn
@@ -0,0 +1,25 @@
+Perhaps stupidly I added some very large bare git repos into a git-annex.
+
+This took a very long time, used lot's of memory, and then crashed.  I didn't catch the error (which is annoying) - sorry about that.  IIRC it is the same error if one Ctrl-c's the addition.
+
+I ran `git annex add .` a second time and eventually killed it (I perhaps should have waited - I now think it was working).
+
+A `git annex unannex` fixed up some files but somehow I managed to end up with tonnes of files all sym-linked into the git annex object directory but not somehow recognised as annexed files.  I'm assuming that they somehow didn't make it into git annex's meta-data layer (or equivalent).
+
+Commands such as `git annex {fsck,whereis,unannex} weirdfile` immediately returned without error.
+
+I've now spent a lot of manual time copying the files back. Doing the following, not the cleverest but I was a little panicky about my data...
+
+    find . -type l -exec mv \{} \{}.link \; #Move link names out of the way
+    find . -type l -exec cp \{} \{}.cp \; #Copy follows links so we can copy target back to link location
+    find . -type f -name "*.link.cp" | xargs -n 1 rename 's/\.link\.cp//' #Change to original name
+    find . -type l -exec rm \{} \; #Ditch the links
+    git annex unused
+    git annex dropunused `seq 9228`
+
+9228 files were found to be unused, this gives an idea of the scale of the number of "lost" files for want of a better term.
+
+A pretty poor bug report as these things go.  Anyone any idea what might have happened (it didn't seem space or memory related)?  Or how I might have fixed it a little more cleverly?
+
+For reference I am using stable Debian, git annex version 3.20111011.
+

From d62b060f0f8f765b8b7f79c453b5f1c363403f19 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 12:50:28 +0000
Subject: [PATCH 2614/8313] Added a comment

---
 ...1_062d0153a379c1ba1df8585b90220d3d._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment
new file mode 100644
index 0000000000..e879441ff8
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_1_062d0153a379c1ba1df8585b90220d3d._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 1"
+ date="2011-12-06T12:50:27Z"
+ content="""
+Ah HA!  Looks like I found the cause of this.
+
+    [matt@rss01:~/files/matt_ford]0> git annex add mhs
+    add mhs/Accessing_Web_Manager_V10.pdf ok
+    ....
+    add mhs/MAHSC Costing Request Form Dual
+    Organisations - FINAL v20 Oct 2010.xls git-annex: unknown response from git cat-file refs/heads/git-annex:8d5/ed4/WORM-s568832-m1323164214--MAHSC Costing Request Form Dual missing
+
+Spot the file name with a newline character in it!  This causes the error message above.  It seems that the files proceeding this badly named file are sym-linked but not registered.
+
+Perhaps a bug?
+"""]]

From 01f7c74d1fb581ed79c46aa16841a36f5e23e945 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 13:05:11 +0000
Subject: [PATCH 2615/8313]

---
 .../bad_behaviour_with_file_names_with_newline_in_them.mdwn    | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn

diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
new file mode 100644
index 0000000000..5673d8060c
--- /dev/null
+++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
@@ -0,0 +1,3 @@
+Found this out the hard way.  See the comment in the below post for what happens.
+
+[[/forum/git_annex_add_crash_and_subsequent_recovery/]]

From c3d3bf332952811b59a2404ee4c568799245472f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 13:36:35 +0000
Subject: [PATCH 2616/8313]

---
 doc/forum/git_pull_remote_git-annex.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex.mdwn

diff --git a/doc/forum/git_pull_remote_git-annex.mdwn b/doc/forum/git_pull_remote_git-annex.mdwn
new file mode 100644
index 0000000000..349610693b
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex.mdwn
@@ -0,0 +1,11 @@
+I thought I'd followed the walk through when initially setting up my repos.
+
+However I find that I have to do the following to sync my annex's.
+
+    git pull remote master
+    git checkout git-annex
+    git pull remote git-annex
+    git checkout master
+    git annex get .
+
+Has something gone wrong? I see no mention of syncing git-annex repos in the walk-through...

From 5e7e873853f9ffdf1fc191c5477829c3627da0c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Dec 2011 11:37:58 -0400
Subject: [PATCH 2617/8313] the Annex newtype is back

Thanks to Bas van Dijk for providing the instance declarations I needed.
Grody stuff. Bas is talking about perhaps providing utility functions that
contain the ugly parts, so this code may be able to be removed using a
future version of monad-control.
---
 Annex.hs | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index d21f0a06c6..d60e08e2d9 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving, TypeFamilies, MultiParamTypeClasses #-}
 
 module Annex (
 	Annex,
@@ -22,8 +22,9 @@ module Annex (
 	fromRepo,
 ) where
 
-import Control.Monad.Trans.Control
 import Control.Monad.State
+import Control.Monad.Trans.Control (StM, MonadBaseControl, liftBaseWith, restoreM)
+import Control.Monad.Base (liftBase, MonadBase)
 
 import Common
 import qualified Git
@@ -38,7 +39,25 @@ import Types.UUID
 import qualified Utility.Matcher
 
 -- git-annex's monad
-type Annex = StateT AnnexState IO
+newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
+	deriving (
+		Monad,
+		MonadIO,
+		MonadState AnnexState,
+		Functor,
+		Applicative
+	)
+
+instance MonadBase IO Annex where
+	liftBase = Annex . liftBase
+
+instance MonadBaseControl IO Annex where
+	newtype StM Annex a = StAnnex (StM (StateT AnnexState IO) a)
+	liftBaseWith f = Annex $ liftBaseWith $ \runInIO ->
+		f $ liftM StAnnex . runInIO . runAnnex
+	restoreM = Annex . restoreM . unStAnnex
+		where
+			unStAnnex (StAnnex st) = st
 
 data OutputType = NormalOutput | QuietOutput | JSONOutput
 
@@ -94,9 +113,9 @@ new gitrepo = newState <$> Git.configRead gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
-run s a = runStateT a s
+run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
-eval s a = evalStateT a s
+eval s a = evalStateT (runAnnex a) s
 
 {- Gets a value from the internal state, selected by the passed value
  - constructor. -}

From 340d206efee926f0b39fedddb6afe0dfec43682f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Dec 2011 16:43:29 +0000
Subject: [PATCH 2618/8313] Added a comment

---
 ..._9c245db3518d8b889ecdf5115ad9e053._comment | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment b/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment
new file mode 100644
index 0000000000..989ab9bcd8
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_1_9c245db3518d8b889ecdf5115ad9e053._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-06T16:43:29Z"
+ content="""
+You're taking a very long and strange way to a place that you can reach as follows:
+
+
+git pull remote
+git annex get .
+

+
+Which is just as shown in [[walkthrough/getting_file_content]]. 
+
+In particular, \"git pull remote\" first fetches all branches from the remote, including the git-annex branch.
+When you say \"git pull remote master\", you're preventing it from fetching the git-annex branch. 
+If for some reason you want the slightly longer way around, it is:
+
+
+git pull remote master
+git fetch remote git-annex
+git annex get .
+

+
+Or, eqivilantly but with less network connections:
+
+
+git fetch remote
+git merge remote/master
+git annex get .
+

+
+BTW, notice that this is all bog-standard git branch pulling stuff, not specific to git-annex in the least.
+Consult your extensive and friendly git documentation for details. :)
+"""]]

From cf5353acb4746cd0c2e736eecd066bd505555af3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Dec 2011 16:49:32 +0000
Subject: [PATCH 2619/8313] Added a comment

---
 ...comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment

diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment
new file mode 100644
index 0000000000..7ff8f8e3d9
--- /dev/null
+++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them/comment_1_92dfe6e9089c79eb64e2177fb135ef55._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-06T16:49:32Z"
+ content="""
+This only happens with the WORM backend (or possibly with SHA1E if the file's extension has a newline).
+
+The problem is not the newline in the file, but the newline in the key generated for the file. It's probably best to just disallow such keys being created.
+"""]]

From 480495beb4a3422f006ee529df807a20cc944727 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Dec 2011 13:02:50 -0400
Subject: [PATCH 2620/8313] Prevent key names from containing newlines.

There are several places where it's assumed a key can be written on one
line. One is in the format of the .git/annex/unused files. The difficult
one is that filenames derived from keys are fed into git cat-file --batch,
which has a line based input. (And no -z option.)

So, for now it's best to block such keys being created.
---
 Backend.hs                                             |  8 +++++++-
 Backend/SHA.hs                                         | 10 ++++++----
 debian/changelog                                       |  1 +
 ...behaviour_with_file_names_with_newline_in_them.mdwn |  2 ++
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index f7990c22c1..136c2eb7a6 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -64,7 +64,13 @@ genKey' (b:bs) file = do
 	r <- (B.getKey b) file
 	case r of
 		Nothing -> genKey' bs file
-		Just k -> return $ Just (k, b)
+		Just k -> return $ Just (makesane k, b)
+	where
+		-- keyNames should not contain newline characters.
+		makesane k = k { keyName = map fixbadchar (keyName k) }
+		fixbadchar c
+			| c == '\n' = '_'
+			| otherwise = c
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 2ae0cfcf42..7935b6d262 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -90,10 +90,12 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 			, keyBackendName = shaNameE size
 			}
 		naiveextension = takeExtension file
-		extension = 
-			if length naiveextension > 6
-				then "" -- probably not really an extension
-				else naiveextension
+		extension
+			-- long or newline containing extensions are 
+			-- probably not really an extension
+			| length naiveextension > 6 ||
+			  '\n' `elem` naiveextension = ""
+			| otherwise = naiveextension
 
 {- A key's checksum is checked during fsck. -}
 checkKeyChecksum :: SHASize -> Key -> Annex Bool
diff --git a/debian/changelog b/debian/changelog
index 7feb8f8ca9..a4d3a6492a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111204) UNRELEASED; urgency=low
 
   * map: Fix a failure to detect a loop when both repositories are local
     and refer to each other with relative paths.
+  * Prevent key names from containing newlines.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
index 5673d8060c..530a8da5d5 100644
--- a/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
+++ b/doc/bugs/bad_behaviour_with_file_names_with_newline_in_them.mdwn
@@ -1,3 +1,5 @@
 Found this out the hard way.  See the comment in the below post for what happens.
 
 [[/forum/git_annex_add_crash_and_subsequent_recovery/]]
+
+> [[fixed|done]] --[[Joey]]

From ba74e2069cdf3c69a5c729c53abd73ce11330828 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Dec 2011 17:08:37 +0000
Subject: [PATCH 2621/8313] Added a comment

---
 ..._6fc6be43c488c468a4811cd0a1360225._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment
new file mode 100644
index 0000000000..38f2434f49
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_2_6fc6be43c488c468a4811cd0a1360225._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-06T17:08:37Z"
+ content="""
+The bug with newlines is now fixed.
+
+Thought I'd mention how to clean up from interrupting `git annex add`.
+When you do that, it doesn't get a chance to `git add` the files it's
+added (this is normally done at the end, or sometimes at points in the middle when you're adding a *lot* of files).
+Which is also why fsck, whereis, and unannex wouldn't operate on them, since they only deal with files in git.
+
+So the first step is to manually use `git add` on any symlinks.
+
+Then, `git commit` as usual.
+
+At that point, `git annex unannex` would get you back to your starting state.
+"""]]

From 730041688d616bff4df745c6605bbaff52733513 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Dec 2011 13:13:42 -0400
Subject: [PATCH 2622/8313] add modules needed for using the new monad-control

---
 git-annex.cabal | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7be78053f8..d701cd6999 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,8 +30,8 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5, monad-control, json
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
+   base < 5, monad-control, transformers-base, lifted-base
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From adb1dc65bc9342d76d641fa06ec187f3e2c7b783 Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Tue, 6 Dec 2011 20:25:06 +0000
Subject: [PATCH 2623/8313]

---
 ...rsync_remotes_with_encryption_enabled.mdwn | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
new file mode 100644
index 0000000000..105e24aace
--- /dev/null
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -0,0 +1,40 @@
+Somehow git-annex has again lost a complete rsync remote with encryption enabled...
+
+Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
+All commands are executed on the git repository on my laptop.
+Target of origin is a gitolite repository without annex support (thus the two rsync remotes).
+
+Is there a way in git-annex to verify that all files fullfill the numcopies, in my case
+numcopies=2, and can be read from the remotes their are on?
+I thought that *copy* would verify that, but seems not.
+    
+    % g a copy --to remoteserver tools
+    copy tools/md5_sha1_utility.exe (gpg) (checking remoteserver...) ok
+    copy tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking remoteserver...) ok
+    
+    % g a copy --to localserver tools
+    copy tools/md5_sha1_utility.exe (gpg) (checking localserver...) ok
+    copy tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking localserver...) ok
+    
+    % g a drop tools
+    drop tools/md5_sha1_utility.exe (gpg) (checking localserver...) (checking remoteserver...) (unsafe)
+      Could only verify the existence of 1 out of 2 necessary copies
+      
+      Try making some of these repositories available:
+            718a9b5c-1b4a-11e1-8211-6f094f20e050 -- remoteserver (remote backupserver)
+      
+      (Use --force to override this check, or adjust annex.numcopies.)
+    failed
+    drop tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checking localserver...) (checking remoteserver...) (unsafe)
+      Could only verify the existence of 1 out of 2 necessary copies
+      
+      Try making some of these repositories available:
+            718a9b5c-1b4a-11e1-8211-6f094f20e050 -- remoteserver (remote backupserver)
+      
+      (Use --force to override this check, or adjust annex.numcopies.)
+    failed
+    git-annex: drop: 2 failed
+    
+    % g a fsck tools
+    fsck tools/md5_sha1_utility.exe (checksum...) ok
+    fsck tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checksum...) ok

From 6f221f1fc3647de5b28179574125f6caa211fe0c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Dec 2011 17:06:08 -0400
Subject: [PATCH 2624/8313] response

---
 ...ing_rsync_remotes_with_encryption_enabled.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 105e24aace..914b45d184 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -1,5 +1,9 @@
 Somehow git-annex has again lost a complete rsync remote with encryption enabled...
 
+> "once again" ? When did it do it before?
+
+> "lost" ? How is the remote lost?
+
 Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
 All commands are executed on the git repository on my laptop.
 Target of origin is a gitolite repository without annex support (thus the two rsync remotes).
@@ -38,3 +42,14 @@ I thought that *copy* would verify that, but seems not.
     % g a fsck tools
     fsck tools/md5_sha1_utility.exe (checksum...) ok
     fsck tools/win32diskimager-RELEASE-0.2-r23-win32.zip (checksum...) ok
+
+> Copy does do an explicit check that the content is present on remoteserver,
+> and based on the above, the content was found to be already there,
+> which is why it did not copy it again.
+> 
+> Drop does an indentical check that the content is present, and 
+> since it failed to find it, I am left thinking something must have
+> happened to the remove in between the copy and the drop to cause the
+> content to go away.
+> 
+> What happens if you copy the data to remoteserver again? --[[Joey]]

From 42c81e5dc00988964c1ea264c2a13ca2ce95c78a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Tue, 6 Dec 2011 23:23:31 +0000
Subject: [PATCH 2625/8313] Added a comment

---
 ...ent_2_0f7f4a311b0ec1d89613e80847e69b42._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment b/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment
new file mode 100644
index 0000000000..198f95cee8
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_2_0f7f4a311b0ec1d89613e80847e69b42._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 2"
+ date="2011-12-06T23:23:29Z"
+ content="""
+Doh! Total brain melt on my part. Thanks for the additional info.  Not taking my time and reading things properly - kept assuming that the full remote pull failed due to the warning:
+
+    You asked to pull from the remote 'rss', but did not specify
+    a branch. Because this is not the default configured remote
+    for your current branch, you must specify a branch on the command line.
+
+Rookie mistake indeed.
+"""]]

From ee26ebe6eb63bc58156bfc418d7692a912390a6a Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Wed, 7 Dec 2011 03:14:24 +0000
Subject: [PATCH 2626/8313]

---
 doc/users/gebi.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/users/gebi.mdwn

diff --git a/doc/users/gebi.mdwn b/doc/users/gebi.mdwn
new file mode 100644
index 0000000000..121bedbdd7
--- /dev/null
+++ b/doc/users/gebi.mdwn
@@ -0,0 +1 @@
+Michael Gebetsroither 

From 919d58667a65df2ae702ba438dd53c8f202dee8b Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Wed, 7 Dec 2011 03:20:43 +0000
Subject: [PATCH 2627/8313]

---
 ...nex_losing_rsync_remotes_with_encryption_enabled.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 914b45d184..968f296dd0 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -2,8 +2,15 @@ Somehow git-annex has again lost a complete rsync remote with encryption enabled
 
 > "once again" ? When did it do it before?
 
+It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]
+
 > "lost" ? How is the remote lost?
 
+git-annex is not able to find any files on the encrypted rsync remote anymore.
+Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state.
+I've also had the state where the content was already on the remote side but git-annex copy would copy it again,
+ignoring all the data on the remote side. --[[gebi]]
+
 Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
 All commands are executed on the git repository on my laptop.
 Target of origin is a gitolite repository without annex support (thus the two rsync remotes).
@@ -53,3 +60,5 @@ I thought that *copy* would verify that, but seems not.
 > content to go away.
 > 
 > What happens if you copy the data to remoteserver again? --[[Joey]]
+
+The commands above are executed within a few seconds and completely repeatable. --[[gebi]]

From 51b7b828700438523d56f44d08e07679ef3e849f Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Wed, 7 Dec 2011 05:48:47 +0000
Subject: [PATCH 2628/8313]

---
 .../git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 968f296dd0..d727d69174 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -1,5 +1,7 @@
 Somehow git-annex has again lost a complete rsync remote with encryption enabled...
 
+git-annex version was 3.20111111
+
 > "once again" ? When did it do it before?
 
 It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]

From c929f9df0f45db80445f9247f644bb87e8ca0a35 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Wed, 7 Dec 2011 07:39:16 +0000
Subject: [PATCH 2629/8313] Added a comment

---
 ...comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment
new file mode 100644
index 0000000000..b58f81c5b7
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_3_45efaaf27d9b580c4c75cbcdc4f65b64._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 3"
+ date="2011-12-07T07:39:15Z"
+ content="""
+Ah - very good to know that recovery is easier than the method I used.  
+
+I wonder if it could be made a feature to automatically and safely recover/resume from an interrupted `git add`?
+"""]]

From 5926be6f307c03f5ed1a02dffff5dce0d987e689 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Dec 2011 12:36:15 -0400
Subject: [PATCH 2630/8313] response

---
 ...rsync_remotes_with_encryption_enabled.mdwn | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index d727d69174..93588b49c1 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -4,14 +4,14 @@ git-annex version was 3.20111111
 
 > "once again" ? When did it do it before?
 
-It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]
+>> It's the second time i uploaded all the files to an encrypted rsync remote and git-annex is not able to find it anymore. --[[gebi]]
 
 > "lost" ? How is the remote lost?
 
-git-annex is not able to find any files on the encrypted rsync remote anymore.
-Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state.
-I've also had the state where the content was already on the remote side but git-annex copy would copy it again,
-ignoring all the data on the remote side. --[[gebi]]
+>> git-annex is not able to find any files on the encrypted rsync remote anymore.
+>> Copy does not copy the content again but drop doesn't find it, thus it's somehow "lost" and in an strange state.
+>> I've also had the state where the content was already on the remote side but git-annex copy would copy it again,
+>> ignoring all the data on the remote side. --[[gebi]]
 
 Both *remoteserver* and *localserver* are rsync remotes with enabled encryption.
 All commands are executed on the git repository on my laptop.
@@ -64,3 +64,15 @@ I thought that *copy* would verify that, but seems not.
 > What happens if you copy the data to remoteserver again? --[[Joey]]
 
 The commands above are executed within a few seconds and completely repeatable. --[[gebi]]
+
+> In that case, why don't you run the commands with `-d` to see the actual
+> rsync command it's running to check if the content is present. 
+> Then you can try repeatedly running the command by hand and see why it
+> sometimes succeeds and sometimes fail.
+> 
+> The command will be something like this:
+> `rsync --quiet hostname:/dir/file 2>/dev/null`
+> 
+> The exit status is what's used to see if content is present -- and
+> currently any failure even a failure to connect is taken to mean it's not
+> present. --[[Joey]]

From d2ff311a3496fc498ad540b194767853ffdc1fc0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Dec 2011 13:17:00 -0400
Subject: [PATCH 2631/8313] change footer

---
 doc/git-annex-shell.mdwn | 2 +-
 doc/git-annex.mdwn       | 2 +-
 doc/git-union-merge.mdwn | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index fc5bc6c2d6..7a65f10775 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -78,4 +78,4 @@ Joey Hess 
 
 
 
-Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
+Warning: Automatically converted into a man page by mdwn2man. Edit with care
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9df5c3c6d1..08def0d62b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -623,4 +623,4 @@ Joey Hess 
 
 
 
-Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
+Warning: Automatically converted into a man page by mdwn2man. Edit with care
diff --git a/doc/git-union-merge.mdwn b/doc/git-union-merge.mdwn
index ed1778910b..8e3c34f8f1 100644
--- a/doc/git-union-merge.mdwn
+++ b/doc/git-union-merge.mdwn
@@ -35,4 +35,4 @@ Joey Hess 
 
 
 
-Warning: this page is automatically made into a man page via [mdwn2man](http://git.ikiwiki.info/?p=ikiwiki;a=blob;f=mdwn2man;hb=HEAD).  Edit with care
+Warning: Automatically converted into a man page by mdwn2man. Edit with care

From 8047bba5b92a6f77ef305c1a74e59b5dacbcc9a2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 7 Dec 2011 16:53:53 -0400
Subject: [PATCH 2632/8313] add: If interrupted, add can leave files converted
 to symlinks but not yet added to git. Running the add again will now clean up
 this situtation.

---
 Command.hs       |  8 ++++----
 Command/Add.hs   | 20 ++++++++++++++------
 debian/changelog |  2 ++
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/Command.hs b/Command.hs
index 4d5bbeb363..0cd0bf4910 100644
--- a/Command.hs
+++ b/Command.hs
@@ -13,7 +13,7 @@ module Command (
 	prepCommand,
 	doCommand,
 	whenAnnexed,
-	notAnnexed,
+	ifAnnexed,
 	notBareRepo,
 	isBareRepo,
 	autoCopies,
@@ -71,10 +71,10 @@ doCommand = start
 {- Modifies an action to only act on files that are already annexed,
  - and passes the key and backend on to it. -}
 whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
-whenAnnexed a file = maybe (return Nothing) (a file) =<< Backend.lookupFile file
+whenAnnexed a file = ifAnnexed file (a file) (return Nothing)
 
-notAnnexed :: FilePath -> Annex (Maybe a) -> Annex (Maybe a)
-notAnnexed file a = maybe a (const $ return Nothing) =<< Backend.lookupFile file
+ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> (Annex a) -> Annex a
+ifAnnexed file yes no = maybe no yes =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
 notBareRepo a = do
diff --git a/Command/Add.hs b/Command/Add.hs
index 9fdbdcaa69..9410601b8b 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -29,13 +29,21 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
 start :: BackendFile -> CommandStart
-start p@(_, file) = notBareRepo $ notAnnexed file $ do
-	s <- liftIO $ getSymbolicLinkStatus file
-	if isSymbolicLink s || not (isRegularFile s)
-		then stop
-		else do
+start p@(_, file) = notBareRepo $ ifAnnexed file fixup add
+	where
+		add = do
+			s <- liftIO $ getSymbolicLinkStatus file
+			if isSymbolicLink s || not (isRegularFile s)
+				then stop
+				else do
+					showStart "add" file
+					next $ perform p
+		fixup (key, _) = do
+			-- fixup from an interrupted add; the symlink
+			-- is present but not yet added to git
 			showStart "add" file
-			next $ perform p
+			liftIO $ removeFile file
+			next $ next $ cleanup file key =<< inAnnex key
 
 perform :: BackendFile -> CommandPerform
 perform (backend, file) = Backend.genKey file backend >>= go
diff --git a/debian/changelog b/debian/changelog
index a4d3a6492a..588ea6d5be 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * map: Fix a failure to detect a loop when both repositories are local
     and refer to each other with relative paths.
   * Prevent key names from containing newlines.
+  * add: If interrupted, add can leave files converted to symlinks but not
+    yet added to git. Running the add again will now clean up this situtation.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 

From 2515bb65209cd42c0291be949eaced4365babf8f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 7 Dec 2011 20:54:51 +0000
Subject: [PATCH 2633/8313] Added a comment

---
 .../comment_4_c560eae40867512b0af2cbef161fc8ac._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment

diff --git a/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment
new file mode 100644
index 0000000000..8fca16cada
--- /dev/null
+++ b/doc/forum/git_annex_add_crash_and_subsequent_recovery/comment_4_c560eae40867512b0af2cbef161fc8ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-12-07T20:54:51Z"
+ content="""
+Good idea! I've made `git annex add` recover when ran a second time.
+"""]]

From 2568beee07928d1bd462d631372881f146b190ee Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Thu, 8 Dec 2011 14:24:08 +0000
Subject: [PATCH 2634/8313]

---
 ..._rsync_remotes_with_encryption_enabled.mdwn | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 93588b49c1..af1fed2eed 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -69,10 +69,26 @@ The commands above are executed within a few seconds and completely repeatable.
 > rsync command it's running to check if the content is present. 
 > Then you can try repeatedly running the command by hand and see why it
 > sometimes succeeds and sometimes fail.
-> 
+
+The commands fail and succeed consistently, not either or.
+git annex copy succeeds consistently with not copying the content to remote because it checks and it's already there.
+
+git annex drop fails consistently with error because content is missing on the exact same remote git annex copy checks
+and thinks the content is there. --[[gebi]]
+
 > The command will be something like this:
 > `rsync --quiet hostname:/dir/file 2>/dev/null`
 > 
 > The exit status is what's used to see if content is present -- and
 > currently any failure even a failure to connect is taken to mean it's not
 > present. --[[Joey]]
+
+hm... thats interesting, git annex drop and git annex copy check for different hashes on the same file at the same remote...
+
+git annex drop -d tools/md5_sha1_utility.exe
+> Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/JF/z7/'\"'\"'GPGHMACSHA1--7ffb3840f0e37aee964352e98808403655e8473a/GPGHMACSHA1--7ffb3840f0e37aee964352e98808403655e8473a'\"'\"'' 2>/dev/null"]
+
+git annex copy --to remoteserver -d tools/md5_sha1_utility.exe
+> Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/1F/PQ/'\"'\"'GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35/GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35'\"'\"'' 2>/dev/null"]
+
+And yes, only the hash *annex copy* is checking for exists on the remote side. --[[gebi]]

From e3f1568e0ff7dc872f3782115c74b9e7d8c291b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 8 Dec 2011 16:01:46 -0400
Subject: [PATCH 2635/8313] Fix caching of decrypted ciphers, which failed when
 drop had to check multiple different encrypted special remotes.

---
 Annex.hs                                      |  5 ++--
 Remote/Helper/Encryptable.hs                  | 23 +++++++++++--------
 Types/Crypto.hs                               |  2 ++
 debian/changelog                              |  2 ++
 ...rsync_remotes_with_encryption_enabled.mdwn |  7 ++++++
 5 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 6d245a92d1..d8d9c63b40 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -36,6 +36,7 @@ import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
 import qualified Utility.Matcher
+import qualified Data.Map as M
 
 -- git-annex's monad
 newtype Annex a = Annex { runAnnex :: StateT AnnexState IO a }
@@ -70,7 +71,7 @@ data AnnexState = AnnexState
 	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
 	, forcetrust :: [(UUID, TrustLevel)]
 	, trustmap :: Maybe TrustMap
-	, cipher :: Maybe Cipher
+	, ciphers :: M.Map EncryptedCipher Cipher
 	}
 
 newState :: Git.Repo -> AnnexState
@@ -93,7 +94,7 @@ newState gitrepo = AnnexState
 	, limit = Left []
 	, forcetrust = []
 	, trustmap = Nothing
-	, cipher = Nothing
+	, ciphers = M.empty
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 85d269a213..99f48fe7b0 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -61,19 +61,22 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 		withkey a k = cip k >>= maybe (a k) (a . snd)
 		cip = cipherKey c
 
-{- Gets encryption Cipher. The decrypted Cipher is cached in the Annex
+{- Gets encryption Cipher. The decrypted Ciphers are cached in the Annex
  - state. -}
 remoteCipher :: RemoteConfig -> Annex (Maybe Cipher)
-remoteCipher c = maybe expensive cached =<< Annex.getState Annex.cipher
+remoteCipher c = go $ extractCipher c
 	where
-		cached cipher = return $ Just cipher
-		expensive = case extractCipher c of
-			Nothing -> return Nothing
-			Just encipher -> do
-				showNote "gpg"
-				cipher <- liftIO $ decryptCipher c encipher
-				Annex.changeState (\s -> s { Annex.cipher = Just cipher })
-				return $ Just cipher
+		go Nothing = return Nothing
+		go (Just encipher) = do
+			cache <- Annex.getState Annex.ciphers
+			case M.lookup encipher cache of
+				Just cipher -> return $ Just cipher
+				Nothing -> decrypt encipher cache
+		decrypt encipher cache = do
+			showNote "gpg"
+			cipher <- liftIO $ decryptCipher c encipher
+			Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
+			return $ Just cipher
 
 {- Gets encryption Cipher, and encrypted version of Key. -}
 cipherKey :: Maybe RemoteConfig -> Key -> Annex (Maybe (Cipher, Key))
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index a9d3dddc59..29a4cd099c 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -11,5 +11,7 @@ module Types.Crypto where
 newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
+	deriving (Ord, Eq)
 
 newtype KeyIds = KeyIds [String]
+	deriving (Ord, Eq)
diff --git a/debian/changelog b/debian/changelog
index 588ea6d5be..04e9116720 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * Prevent key names from containing newlines.
   * add: If interrupted, add can leave files converted to symlinks but not
     yet added to git. Running the add again will now clean up this situtation.
+  * Fix caching of decrypted ciphers, which failed when drop had to check
+    multiple different encrypted special remotes.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index af1fed2eed..0dad8856e8 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -92,3 +92,10 @@ git annex copy --to remoteserver -d tools/md5_sha1_utility.exe
 > Running: sh ["-c","rsync --quiet 'REMOVED_HOST:annex/work/1F/PQ/'\"'\"'GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35/GPGHMACSHA1--ff075e57f649300c5698e346be74fb6e22d70e35'\"'\"'' 2>/dev/null"]
 
 And yes, only the hash *annex copy* is checking for exists on the remote side. --[[gebi]]
+
+> Ok, this is due to too aggressive caching of the decrypted cipher
+> for a remote. When dopping, it decrypts localserver's cipher, 
+> caches it, and then when checking remoteserver it says hey, 
+> here's an already decrypted cipher -- it must be the right one!
+> 
+> Problem reproduced here, and fixed. [[done]] --[[Joey]]

From e0e40964ab2d0a305fe2503215f306e1228aac24 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Thu, 8 Dec 2011 22:42:29 +0000
Subject: [PATCH 2636/8313]

---
 ..._no_fixed_hostname_and_optimising_ssh.mdwn | 72 +++++++++++++++++++
 1 file changed, 72 insertions(+)
 create mode 100644 doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
new file mode 100644
index 0000000000..db108f48b9
--- /dev/null
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -0,0 +1,72 @@
+## Intro
+
+This tip is based on my experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
+
+I'm not using a bare repository that allows pushing (an alternate way to sort the issue) nor did I fancy allowing `git push` to run against my desktop checked out repository.
+
+None of this stuff is really `git annex` specific but is useful to know...
+
+## Dealing with no fixed hostname
+
+Essentially set up two repos as per the [[walkthrough]].
+
+Desktop as follows:
+
+    cd ~/annex
+    git init
+    git annex init "desktop"
+
+And the laptop like this
+
+    git clone ssh://desktop/annex
+    git init
+    git annex init "laptop"
+
+Now we want to add the the repos as remotes of each other.  
+
+For the laptop it is easy:
+
+    git remote add desktop ssh://desktop/~/annex 
+
+However for the desktop to add an ever changing laptops hostname it's a little tricky.  We make use of remote SSH tunnels to do this.  Essentially we have the laptop which always knows it's own name and address and that of the desktop create a tunnel starting on an arbitrary port on the desktop and heading back to itself on it's own SSH server port (22).
+
+To do this make part of your laptops SSH config look like this:
+
+    Host desktop
+    User matt
+    HostName desktop.example.org
+    RemoteForward 2222 localhost:22
+
+Now on the desktop to connect over the tunnel to the laptops SSH port you need this:
+
+    Host laptop
+    User matt
+    HostName localhost
+    port 2222
+
+So to add the desktops remote:
+
+a) From the laptop ensure the tunnel is up
+
+    ssh desktop
+
+b) From the desktop add the remote
+
+    git remote add laptop ssh://laptop/~/annex
+
+So now you can work on the train, pop on the wifi at work upon arrival and sync up with a `git pull && git annex get`.
+
+An alternate solution maybe to use direct tunnels over Openvpn.
+
+## Optimising SSH
+
+Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this is a feature request?)
+
+Lot's of new small files in an _annex_ cause lot's of connections to be made quickly: this is an relatively expensive overhead and is enough for connection limiting to start in my case.  The process can be made much faster by using SSH's connection sharing capabilities.  An SSH config like this should do it:
+
+    # Global Settings
+    ControlMaster auto
+    ControlPersist 30
+    ControlPath ~/.ssh/master-%r@%h:%p
+
+This will create a master connection for sharing if one isn't present, maintain it for 30 seconds after closing down the connection (just-in-cases') and automatically use the master connection for subsequent connections.  Wins all round!

From b3ac4af6b0722db35b8295d1131b41c827cdd708 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Thu, 8 Dec 2011 22:50:45 +0000
Subject: [PATCH 2637/8313]

---
 ...h_no_fixed_hostname_and_optimising_ssh.mdwn | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
index db108f48b9..3f02ea47d3 100644
--- a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -2,9 +2,9 @@
 
 This tip is based on my experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
 
-I'm not using a bare repository that allows pushing (an alternate way to sort the issue) nor did I fancy allowing `git push` to run against my desktop checked out repository.
+I'm not using a bare repository that allows pushing (an alternative solution) nor do I fancy allowing `git push` to run against my desktop checked out repository (perhaps I worry over nothing?)
 
-None of this stuff is really `git annex` specific but is useful to know...
+None of this is really `git annex` specific but I think it is useful to know...
 
 ## Dealing with no fixed hostname
 
@@ -28,23 +28,23 @@ For the laptop it is easy:
 
     git remote add desktop ssh://desktop/~/annex 
 
-However for the desktop to add an ever changing laptops hostname it's a little tricky.  We make use of remote SSH tunnels to do this.  Essentially we have the laptop which always knows it's own name and address and that of the desktop create a tunnel starting on an arbitrary port on the desktop and heading back to itself on it's own SSH server port (22).
+However for the desktop to add an ever changing laptops hostname it's a little tricky.  We make use of remote SSH tunnels to do this.  Essentially we have the laptop (which always knows it's own name and address and knows the address of the desktop) create a tunnel starting on an arbitrary port at the desktop and heads back to the laptop on it's own SSH server port (22).
 
-To do this make part of your laptops SSH config look like this:
+To do this make part of your laptop's SSH config look like this:
 
     Host desktop
     User matt
     HostName desktop.example.org
     RemoteForward 2222 localhost:22
 
-Now on the desktop to connect over the tunnel to the laptops SSH port you need this:
+Now on the desktop to connect over the tunnel to the laptop's SSH port you need this:
 
     Host laptop
     User matt
     HostName localhost
     port 2222
 
-So to add the desktops remote:
+So to add the desktop's remote:
 
 a) From the laptop ensure the tunnel is up
 
@@ -54,13 +54,13 @@ b) From the desktop add the remote
 
     git remote add laptop ssh://laptop/~/annex
 
-So now you can work on the train, pop on the wifi at work upon arrival and sync up with a `git pull && git annex get`.
+So now you can work on the train, pop on the wifi at work upon arrival, and sync up with a `git pull && git annex get`.
 
-An alternate solution maybe to use direct tunnels over Openvpn.
+An alternative solution may be to use direct tunnels over Openvpn.
 
 ## Optimising SSH
 
-Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this is a feature request?)
+Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this should be a feature request?)
 
 Lot's of new small files in an _annex_ cause lot's of connections to be made quickly: this is an relatively expensive overhead and is enough for connection limiting to start in my case.  The process can be made much faster by using SSH's connection sharing capabilities.  An SSH config like this should do it:
 

From d64132a43ae176e8a1353d5463c5387a93da9ad7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 01:57:13 -0400
Subject: [PATCH 2638/8313] hslint

---
 Annex/Branch.hs      |  2 +-
 Annex/Content.hs     |  2 +-
 Annex/Ssh.hs         |  2 +-
 CmdLine.hs           |  2 +-
 Command.hs           |  2 +-
 Command/Map.hs       |  2 +-
 Command/Status.hs    |  5 ++---
 Command/Unused.hs    |  5 ++---
 Git/UnionMerge.hs    |  2 +-
 Logs/UUID.hs         |  6 +++---
 Remote/Git.hs        |  2 +-
 Upgrade/V2.hs        |  2 +-
 Utility/DataUnits.hs |  2 +-
 Utility/Directory.hs | 13 ++++++-------
 configure.hs         |  2 +-
 test.hs              |  3 +--
 16 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a890668810..8f0a09fd6b 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -141,7 +141,7 @@ update = onceonly $ do
 		let merge_desc = if null branches
 			then "update" 
 			else "merging " ++
-				(unwords $ map (show . Git.refDescribe) branches) ++ 
+				unwords (map (show . Git.refDescribe) branches) ++ 
 				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 90bde2975a..3f1db37b53 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -43,7 +43,7 @@ import Annex.Exception
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
-inAnnex = inAnnex' $ doesFileExist
+inAnnex = inAnnex' doesFileExist
 inAnnex' :: (FilePath -> IO a) -> Key -> Annex a
 inAnnex' a key = do
 	whenM (fromRepo Git.repoIsUrl) $
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index f8cd5d9bc8..6893f94ef4 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -43,7 +43,7 @@ git_annex_shell r command params
 		shellcmd = "git-annex-shell"
 		shellopts = Param command : File dir : params
 		sshcmd uuid = unwords $
-			shellcmd : (map shellEscape $ toCommand shellopts) ++
+			shellcmd : map shellEscape (toCommand shellopts) ++
 			uuidcheck uuid
 		uuidcheck NoUUID = []
 		uuidcheck (UUID u) = ["--uuid", u]
diff --git a/CmdLine.hs b/CmdLine.hs
index 78f46a2e39..672969c30a 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -32,7 +32,7 @@ dispatch args cmds options header getgitrepo = do
 	setupConsole
 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
 	case r of
-		Left e -> maybe (throw e) id (cmdnorepo cmd)
+		Left e -> fromMaybe (throw e) (cmdnorepo cmd)
 		Right g -> do
 			state <- Annex.new g
 			(actions, state') <- Annex.run state $ do
diff --git a/Command.hs b/Command.hs
index 0cd0bf4910..86a83e30c0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -73,7 +73,7 @@ doCommand = start
 whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
 whenAnnexed a file = ifAnnexed file (a file) (return Nothing)
 
-ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> (Annex a) -> Annex a
+ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> Annex a -> Annex a
 ifAnnexed file yes no = maybe no yes =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
diff --git a/Command/Map.hs b/Command/Map.hs
index 6b1e8d5bbc..57b48d5030 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -203,7 +203,7 @@ tryScan r
 					"git config --list"
 				dir = Git.workTree r
 				cddir
-					| take 2 dir == "/~" =
+					| "/~" `isPrefixOf` dir =
 						let (userhome, reldir) = span (/= '/') (drop 1 dir)
 						in "cd " ++ userhome ++ " && cd " ++ shellEscape (drop 1 reldir)
 					| otherwise = "cd " ++ shellEscape dir
diff --git a/Command/Status.hs b/Command/Status.hs
index 0fefda1f6b..09da41987b 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -191,9 +191,8 @@ staleSize label dirspec = do
 	keys <- lift (Command.Unused.staleKeys dirspec)
 	if null keys
 		then nostat
-		else do
-			stat label $ json (++ aside "clean up with git-annex unused") $
-				return $ keySizeSum $ S.fromList keys
+		else stat label $ json (++ aside "clean up with git-annex unused") $
+			return $ keySizeSum $ S.fromList keys
 
 aside :: String -> String
 aside s = " (" ++ s ++ ")"
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 7f9edfef25..be0107752e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -152,13 +152,12 @@ excludeReferenced l = do
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map Git.Ref . 
-			map last .
+		refs = map (Git.Ref .  last) .
 			nubBy cmpheads .
 			filter ourbranches .
 			map words . lines . L.unpack
 		cmpheads a b = head a == head b
-		ourbranchend = '/' : show (Annex.Branch.name)
+		ourbranchend = '/' : show Annex.Branch.name
 		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index edc8cb20ba..ddbff6a82f 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -48,7 +48,7 @@ merge_index h repo bs =
  - earlier ones, so the list can be generated from any combination of
  - ls_tree, merge_trees, and merge_tree_index. -}
 update_index :: Repo -> [String] -> IO ()
-update_index repo ls = stream_update_index repo [\s -> mapM_ s ls]
+update_index repo ls = stream_update_index repo [(`mapM_` ls)]
 
 {- Streams content into update-index. -}
 stream_update_index :: Repo -> [Streamer] -> IO ()
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 20f43d15ca..b325c78b6f 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -55,15 +55,15 @@ fixBadUUID = M.fromList . map fixup . M.toList
 			| otherwise = (k, v)
 			where
 				kuuid = fromUUID k
-				isbad = (not $ isuuid kuuid) && isuuid lastword
+				isbad = not (isuuid kuuid) && isuuid lastword
 				ws = words $ value v
 				lastword = last ws
 				fixeduuid = toUUID lastword
-				fixedvalue = unwords $ kuuid:(take (length ws - 1) ws)
+				fixedvalue = unwords $ kuuid: init ws
 		-- For the fixed line to take precidence, it should be
 		-- slightly newer, but only slightly.
 		newertime (LogEntry (Date d) _) = d + minimumPOSIXTimeSlice
-		newertime (LogEntry (Unknown) _) = minimumPOSIXTimeSlice
+		newertime (LogEntry Unknown _) = minimumPOSIXTimeSlice
 		minimumPOSIXTimeSlice = 0.000001
 		isuuid s = length s == 36 && length (split "-" s) == 5
 
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 99ca9fe8e2..d172ec7c04 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -165,7 +165,7 @@ onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
 	-- Avoid re-reading the repository's configuration if it was
 	-- already read.
-	state <- if (M.null $ Git.configMap r)
+	state <- if M.null $ Git.configMap r
 		then Annex.new r
 		else return $ Annex.newState r
 	Annex.eval state $ do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index e76d99b3ec..08bf83e837 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -53,7 +53,7 @@ upgrade = do
 
 	when e $ do
 		inRepo $ Git.run "rm" [Param "-r", Param "-f", Param "-q", File old]
-		unless bare $ inRepo $ gitAttributesUnWrite
+		unless bare $ inRepo gitAttributesUnWrite
 	showProgress
 
 	unless bare push
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index e7552f52fb..5d80a04b9c 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -99,7 +99,7 @@ bandwidthUnits = error "stop trying to rip people off"
 
 {- Do you yearn for the days when men were men and megabytes were megabytes? -}
 oldSchoolUnits :: [Unit]
-oldSchoolUnits = map mingle $ zip storageUnits memoryUnits
+oldSchoolUnits = zipWith (curry mingle) storageUnits memoryUnits
 	where
 		mingle (Unit _ a n, Unit s' _ _) = Unit s' a n
 
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 7f8822fca5..249ed69356 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -11,6 +11,7 @@ import System.IO.Error
 import System.Posix.Files
 import System.Directory
 import Control.Exception (throw)
+import Control.Monad
 
 import Utility.SafeCommand
 import Utility.Conditional
@@ -37,13 +38,11 @@ moveFile src dest = try (rename src dest) >>= onrename
 				mv tmp _ = do
 					ok <- boolSystem "mv" [Param "-f",
 						Param src, Param tmp]
-					if ok
-						then return ()
-						else do
-							-- delete any partial
-							_ <- try $
-								removeFile tmp
-							rethrow
+					unless ok $ do
+						-- delete any partial
+						_ <- try $
+							removeFile tmp
+						rethrow
 		isdir f = do
 			r <- try (getFileStatus f)
 			case r of
diff --git a/configure.hs b/configure.hs
index cb73af2a99..0d96b39554 100644
--- a/configure.hs
+++ b/configure.hs
@@ -71,7 +71,7 @@ checkGitVersion = do
 		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
 		extend n l = l ++ replicate (n - length l) 0
 		mult _ [] = []
-		mult n (x:xs) = (n*x) : (mult (n*100) xs)
+		mult n (x:xs) = (n*x) : mult (n*100) xs
 		readi :: String -> Integer
 		readi s = case reads s of
 			((x,_):_) -> x
diff --git a/test.hs b/test.hs
index e625fbd756..91c11873da 100644
--- a/test.hs
+++ b/test.hs
@@ -11,11 +11,10 @@ import Test.QuickCheck
 
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
-import Control.Exception (bracket_, bracket)
+import Control.Exception (bracket_, bracket, throw)
 import System.IO.Error
 import System.Posix.Env
 import qualified Control.Exception.Extensible as E
-import Control.Exception (throw)
 import qualified Data.Map as M
 import System.IO.HVFS (SystemFS(..))
 

From e664af5d8cf310ac3dcb7e27a5a6846360917fbf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Fri, 9 Dec 2011 10:03:20 +0000
Subject: [PATCH 2639/8313]

---
 ...ing_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
index 3f02ea47d3..8fb2bf9db1 100644
--- a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -1,6 +1,6 @@
 ## Intro
 
-This tip is based on my experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
+This tip is based on my (Matt Ford) experience of using `git annex` with my out-and-about netbook which hits many different wifi networks and has no fixed home or address.
 
 I'm not using a bare repository that allows pushing (an alternative solution) nor do I fancy allowing `git push` to run against my desktop checked out repository (perhaps I worry over nothing?)
 

From 3f5f28b48754bc91620a6354ca70afe4c61c9894 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 12:23:45 -0400
Subject: [PATCH 2640/8313] factor out a stopUnless

code melt for lunch
---
 Command.hs         |  7 +++++++
 Command/AddUrl.hs  | 21 +++++++++------------
 Command/Drop.hs    | 28 +++++++++-------------------
 Command/DropKey.hs | 17 +++++------------
 Command/Fix.hs     |  9 +++------
 Command/Get.hs     | 32 ++++++++++++--------------------
 Command/Migrate.hs | 24 ++++++++++--------------
 Command/Move.hs    | 14 ++++----------
 Command/Unannex.hs | 24 ++++++++++--------------
 9 files changed, 69 insertions(+), 107 deletions(-)

diff --git a/Command.hs b/Command.hs
index 86a83e30c0..813a239cb0 100644
--- a/Command.hs
+++ b/Command.hs
@@ -10,6 +10,7 @@ module Command (
 	noRepo,
 	next,
 	stop,
+	stopUnless,
 	prepCommand,
 	doCommand,
 	whenAnnexed,
@@ -49,6 +50,12 @@ next a = return $ Just a
 stop :: Annex (Maybe a)
 stop = return Nothing
 
+{- Stops unless a condition is met. -}
+stopUnless :: Annex Bool -> Annex (Maybe a) -> Annex (Maybe a)
+stopUnless c a = do
+	ok <- c
+	if ok then a else stop
+
 {- Prepares to run a command via the check and seek stages, returning a
  - list of actions to perform to run the command. -}
 prepCommand :: Command -> [String] -> Annex [CommandCleanup]
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 945848e9f7..75ca740314 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -45,18 +45,15 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	ok <- liftIO $ Url.download url tmp
-	if ok
-		then do
-			[(backend, _)] <- Backend.chooseBackends [file]
-			k <- Backend.genKey tmp backend
-			case k of
-				Nothing -> stop
-				Just (key, _) -> do
-					moveAnnex key tmp
-					setUrlPresent key url
-					next $ Command.Add.cleanup file key True
-		else stop
+	stopUnless (liftIO $ Url.download url tmp) $ do
+		[(backend, _)] <- Backend.chooseBackends [file]
+		k <- Backend.genKey tmp backend
+		case k of
+			Nothing -> stop
+			Just (key, _) -> do
+				moveAnnex key tmp
+				setUrlPresent key url
+				next $ Command.Add.cleanup file key True
 
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
diff --git a/Command/Drop.hs b/Command/Drop.hs
index ee3583869b..0a4c9dfd6f 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -37,13 +37,9 @@ start numcopies file (key, _) = autoCopies key (>) numcopies $ do
 				else startRemote file numcopies key remote
 
 startLocal :: FilePath -> Maybe Int -> Key -> CommandStart
-startLocal file numcopies key = do
-	present <- inAnnex key
-	if present
-		then do
-			showStart "drop" file
-			next $ performLocal key numcopies
-		else stop
+startLocal file numcopies key = stopUnless (inAnnex key) $ do
+	showStart "drop" file
+	next $ performLocal key numcopies
 
 startRemote :: FilePath -> Maybe Int -> Key -> Remote.Remote Annex -> CommandStart
 startRemote file numcopies key remote = do
@@ -55,12 +51,9 @@ performLocal key numcopies = lockContent key $ do
 	(remotes, trusteduuids) <- Remote.keyPossibilitiesTrusted key
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = Remote.remotesWithoutUUID remotes (trusteduuids++untrusteduuids)
-	success <- canDropKey key numcopies trusteduuids tocheck []
-	if success
-		then do
-			whenM (inAnnex key) $ removeAnnex key
-			next $ cleanupLocal key
-		else stop
+	stopUnless (canDropKey key numcopies trusteduuids tocheck []) $ do
+		whenM (inAnnex key) $ removeAnnex key
+		next $ cleanupLocal key
 
 performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
 performRemote key numcopies remote = lockContent key $ do
@@ -75,12 +68,9 @@ performRemote key numcopies remote = lockContent key $ do
 	untrusteduuids <- trustGet UnTrusted
 	let tocheck = filter (/= remote) $
 		Remote.remotesWithoutUUID remotes (have++untrusteduuids)
-	success <- canDropKey key numcopies have tocheck [uuid]
-	if success
-		then do
-			ok <- Remote.removeKey remote key
-			next $ cleanupRemote key remote ok
-		else stop
+	stopUnless (canDropKey key numcopies have tocheck [uuid]) $ do
+		ok <- Remote.removeKey remote key
+		next $ cleanupRemote key remote ok
 	where
 		uuid = Remote.uuid remote
 
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index b63d481bf3..ae2ad8b6a8 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -21,18 +21,11 @@ seek :: [CommandSeek]
 seek = [withKeys start]
 
 start :: Key -> CommandStart
-start key = do
-	present <- inAnnex key
-	if not present
-		then stop
-		else do
-			checkforced
-			showStart "dropkey" (show key)
-			next $ perform key
-	where
-		checkforced = 
-			unlessM (Annex.getState Annex.force) $
-				error "dropkey can cause data loss; use --force if you're sure you want to do this"
+start key = stopUnless (not <$> inAnnex key) $ do
+	unlessM (Annex.getState Annex.force) $
+		error "dropkey can cause data loss; use --force if you're sure you want to do this"
+	showStart "dropkey" (show key)
+	next $ perform key
 
 perform :: Key -> CommandPerform
 perform key = lockContent key $ do
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 27c4b167e5..f264106c3f 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -23,12 +23,9 @@ seek = [withFilesInGit $ whenAnnexed start]
 start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	link <- calcGitLink file key
-	l <- liftIO $ readSymbolicLink file
-	if link == l
-		then stop
-		else do
-			showStart "fix" file
-			next $ perform file link
+	stopUnless ((/=) link <$> liftIO (readSymbolicLink file)) $ do
+		showStart "fix" file
+		next $ perform file link
 
 perform :: FilePath -> FilePath -> CommandPerform
 perform file link = do
diff --git a/Command/Get.hs b/Command/Get.hs
index 093cd2cc59..b7023e2de8 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -22,32 +22,24 @@ seek :: [CommandSeek]
 seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
 start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
-start numcopies file (key, _) = do
-	inannex <- inAnnex key
-	if inannex
-		then stop
-		else autoCopies key (<) numcopies $ do
-			from <- Annex.getState Annex.fromremote
-			case from of
-				Nothing -> go $ perform key
-				Just name -> do
-					-- get --from = copy --from
-					src <- Remote.byName name
-					ok <- Command.Move.fromOk src key
-					if ok
-						then go $ Command.Move.fromPerform src False key
-						else stop
+start numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
+	autoCopies key (<) numcopies $ do
+		from <- Annex.getState Annex.fromremote
+		case from of
+			Nothing -> go $ perform key
+			Just name -> do
+				-- get --from = copy --from
+				src <- Remote.byName name
+				stopUnless (Command.Move.fromOk src key) $
+					go $ Command.Move.fromPerform src False key
 	where
 		go a = do
 			showStart "get" file
 			next a	
 
 perform :: Key -> CommandPerform
-perform key = do
-	ok <- getViaTmp key (getKeyFile key)
-	if ok
-		then next $ return True -- no cleanup needed
-		else stop
+perform key = stopUnless (getViaTmp key $ getKeyFile key) $ do
+	next $ return True -- no cleanup needed
 
 {- Try to find a copy of the file in one of the remotes,
  - and copy it to here. -}
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index c85d7c2ac3..30288fc162 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -58,22 +58,18 @@ perform file oldkey newbackend = do
 	cleantmp tmpfile
 	case k of
 		Nothing -> stop
-		Just (newkey, _) -> do
-			ok <- link src newkey
-			if ok
-				then do
-					-- Update symlink to use the new key.
-					liftIO $ removeFile file
+		Just (newkey, _) -> stopUnless (link src newkey) $ do
+			-- Update symlink to use the new key.
+			liftIO $ removeFile file
 
-					-- If the old key had some
-					-- associated urls, record them for
-					-- the new key as well.
-					urls <- getUrls oldkey
-					unless (null urls) $
-						mapM_ (setUrlPresent newkey) urls
+			-- If the old key had some
+			-- associated urls, record them for
+			-- the new key as well.
+			urls <- getUrls oldkey
+			unless (null urls) $
+				mapM_ (setUrlPresent newkey) urls
 
-					next $ Command.Add.cleanup file newkey True
-				else stop
+			next $ Command.Add.cleanup file newkey True
 	where
 		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
 		link src newkey = getViaTmpUnchecked newkey $ \t -> do
diff --git a/Command/Move.hs b/Command/Move.hs
index fd1ed90198..cc26eecda1 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -108,17 +108,11 @@ toPerform dest move key = moveLock move key $ do
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
 	| move = go
-	| otherwise = do
-		ishere <- inAnnex key
-		if ishere then stop else go
+	| otherwise = stopUnless (inAnnex key) go
 	where
-		go = do
-			ok <- fromOk src key
-			if ok
-				then do
-					showMoveAction move file
-					next $ fromPerform src move key
-				else stop
+		go = stopUnless (fromOk src key) $ do
+			showMoveAction move file
+			next $ fromPerform src move key
 fromOk :: Remote.Remote Annex -> Key -> Annex Bool
 fromOk src key = do
 	u <- getUUID
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index e97b6d05d8..263ff88b40 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -25,21 +25,17 @@ seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unannex subcommand undoes an add. -}
 start :: FilePath -> (Key, Backend Annex) -> CommandStart
-start file (key, _) = do
-	ishere <- inAnnex key
-	if ishere
-		then do
-			force <- Annex.getState Annex.force
-			unless force $ do
-				top <- fromRepo Git.workTree
-				staged <- inRepo $ LsFiles.staged [top]
-				unless (null staged) $
-					error "This command cannot be run when there are already files staged for commit."
-				Annex.changeState $ \s -> s { Annex.force = True }
+start file (key, _) = stopUnless (inAnnex key) $ do
+	force <- Annex.getState Annex.force
+	unless force $ do
+		top <- fromRepo Git.workTree
+		staged <- inRepo $ LsFiles.staged [top]
+		unless (null staged) $
+			error "This command cannot be run when there are already files staged for commit."
+		Annex.changeState $ \s -> s { Annex.force = True }
 
-			showStart "unannex" file
-			next $ perform file key
-		else stop
+	showStart "unannex" file
+	next $ perform file key
 
 perform :: FilePath -> Key -> CommandPerform
 perform file key = next $ cleanup file key

From 14e9b87d44831fe0b6510422ffb4266f253496d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 13:07:31 -0400
Subject: [PATCH 2641/8313] unannex improvements

Added files don't have to be committed before they can be unannexed.

unannex no longer commits existing staged changes

unannex of the last file in a directory now works, before it failed because
git rm deleted the directory out from under it,
---
 Command/Unannex.hs | 30 ++++++++++++------------------
 debian/changelog   |  2 ++
 2 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 263ff88b40..aafc189d00 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -23,17 +23,8 @@ def = [command "unannex" paramPaths seek "undo accidential add command"]
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
 
-{- The unannex subcommand undoes an add. -}
 start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = stopUnless (inAnnex key) $ do
-	force <- Annex.getState Annex.force
-	unless force $ do
-		top <- fromRepo Git.workTree
-		staged <- inRepo $ LsFiles.staged [top]
-		unless (null staged) $
-			error "This command cannot be run when there are already files staged for commit."
-		Annex.changeState $ \s -> s { Annex.force = True }
-
 	showStart "unannex" file
 	next $ perform file key
 
@@ -43,9 +34,17 @@ perform file key = next $ cleanup file key
 cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
 	liftIO $ removeFile file
-	inRepo $ Git.run "rm" [Params "--quiet --", File file]
-	-- git rm deletes empty directories; put them back
-	liftIO $ createDirectoryIfMissing True (parentDir file)
+	-- git rm deletes empty directory without --cached
+	inRepo $ Git.run "rm" [Params "--cached --quiet --", File file]
+	
+	-- If the file was already committed, it is now staged for removal.
+	-- Commit that removal now, to avoid later confusing the
+	-- pre-commit hook if this file is later added back to
+	-- git as a normal, non-annexed file.
+	whenM (not . null <$> inRepo (LsFiles.staged [file])) $ do
+		inRepo $ Git.run "commit" [
+			Param "-m", Param "content removed from git annex",
+			Param "--", File file]
 
 	fast <- Annex.getState Annex.fast
 	if fast
@@ -58,10 +57,5 @@ cleanup file key = do
 		else do
 			fromAnnex key file
 			logStatus key InfoMissing
-	
-	-- Commit staged changes at end to avoid confusing the
-	-- pre-commit hook if this file is later added back to
-	-- git as a normal, non-annexed file.
-	Annex.Queue.add "commit" [Param "-m", Param "content removed from git annex"] []
-	
+
 	return True
diff --git a/debian/changelog b/debian/changelog
index 04e9116720..1c27ad5666 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     yet added to git. Running the add again will now clean up this situtation.
   * Fix caching of decrypted ciphers, which failed when drop had to check
     multiple different encrypted special remotes.
+  * unannex: Can be run on files that have been added to the annex, but not
+    yet committed.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 

From 252b2e92b0ddd5c97565d37c84771b471b6b5082 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 13:31:51 -0400
Subject: [PATCH 2642/8313] cleanup

---
 Command/Unannex.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index aafc189d00..bed857b060 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -10,7 +10,6 @@ module Command.Unannex where
 import Common.Annex
 import Command
 import qualified Annex
-import qualified Annex.Queue
 import Utility.FileMode
 import Logs.Location
 import Annex.Content

From 95e748cbd4bb858a3b87621e60f5b43d53b50480 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 13:32:09 -0400
Subject: [PATCH 2643/8313] inverted logic

---
 Command/DropKey.hs | 2 +-
 Command/Move.hs    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index ae2ad8b6a8..aaaa224661 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -21,7 +21,7 @@ seek :: [CommandSeek]
 seek = [withKeys start]
 
 start :: Key -> CommandStart
-start key = stopUnless (not <$> inAnnex key) $ do
+start key = stopUnless (inAnnex key) $ do
 	unlessM (Annex.getState Annex.force) $
 		error "dropkey can cause data loss; use --force if you're sure you want to do this"
 	showStart "dropkey" (show key)
diff --git a/Command/Move.hs b/Command/Move.hs
index cc26eecda1..85fdff7398 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -108,7 +108,7 @@ toPerform dest move key = moveLock move key $ do
 fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
 	| move = go
-	| otherwise = stopUnless (inAnnex key) go
+	| otherwise = stopUnless (not <$> inAnnex key) go
 	where
 		go = stopUnless (fromOk src key) $ do
 			showMoveAction move file

From d69cf79e202b7d60465ce2068f736243aab14b3b Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 9 Dec 2011 20:47:00 +0000
Subject: [PATCH 2644/8313]

---
 doc/forum/pure_git-annex_only_workflow.mdwn | 46 +++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow.mdwn

diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
new file mode 100644
index 0000000000..e0cbcd5384
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -0,0 +1,46 @@
+I’m using git annex to manage my movie collection on various devices – my laptop, a NSLU tucked away somewhere with lots of space, some external hard drives. For this use case, I do not need the full power of git as a version control system, so having to run "git commit" and coming up with commit messages is annoying. Also, this makes sense for a version control system, but not for my media collection:
+
+	$ git annex add Hot\ Fuzz\ -\ English.mkv 
+	add Hot Fuzz - English.mkv (checksum...) ok
+	(Recording state in git...)
+	$ git commit -m 'another movie added'
+	[master 851dc8a] another movie added
+	 1 files changed, 1 insertions(+), 0 deletions(-)
+	 create mode 120000 00 Noch nicht gesehen/Hot Fuzz - English.mkv
+	$ git push jeff
+	Counting objects: 38, done.
+	Delta compression using up to 2 threads.
+	Compressing objects: 100% (20/20), done.
+	Writing objects: 100% (26/26), 2.00 KiB, done.
+	Total 26 (delta 11), reused 0 (delta 0)
+	remote: error: refusing to update checked out branch: refs/heads/master
+	remote: error: By default, updating the current branch in a non-bare repository
+	remote: error: is denied, because it will make the index and work tree inconsistent
+	remote: error: with what you pushed, and will require 'git reset --hard' to match
+	remote: error: the work tree to HEAD.
+	remote: error: 
+	remote: error: You can set 'receive.denyCurrentBranch' configuration variable to
+	remote: error: 'ignore' or 'warn' in the remote repository to allow pushing into
+	remote: error: its current branch; however, this is not recommended unless you
+	remote: error: arranged to update its work tree to match what you pushed in some
+	remote: error: other way.
+	remote: error: 
+	remote: error: To squelch this message and still keep the default behaviour, set
+	remote: error: 'receive.denyCurrentBranch' configuration variable to 'refuse'.
+	To jeff:/mnt/media/Movies
+	 ! [rejected]        git-annex -> git-annex (non-fast-forward)
+	 ! [remote rejected] master -> master (branch is currently checked out)
+	error: failed to push some refs to 'jeff:/mnt/media/Movies'
+	To prevent you from losing history, non-fast-forward updates were rejected
+	Merge the remote changes (e.g. 'git pull') before pushing again.  See the
+	'Note about fast-forwards' section of 'git push --help' for details.
+
+It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my currenct machine.	
+
+What I would like to have is that
+
+* git annex add does not require a commit afterwards.
+* Changes to the files are automatically picked up with the next git-annex call (similar to how etckeeper works).
+* Commands "git annex push" and "git annex pull" that will syn the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
+
+Summay: git-annex is great. git is not always. Please make it possible to use git annex without having to use git.

From 07c32dd49ce389cd0ba1888b42429ae5fe54a972 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 9 Dec 2011 20:57:00 +0000
Subject: [PATCH 2645/8313] typo

---
 doc/forum/pure_git-annex_only_workflow.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
index e0cbcd5384..48226671c1 100644
--- a/doc/forum/pure_git-annex_only_workflow.mdwn
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -35,7 +35,7 @@ I’m using git annex to manage my movie collection on various devices – my la
 	Merge the remote changes (e.g. 'git pull') before pushing again.  See the
 	'Note about fast-forwards' section of 'git push --help' for details.
 
-It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my currenct machine.	
+It seems that to successfully make the new files known to the other side, I have to log into jeff and pull _from_ my current machine.	
 
 What I would like to have is that
 

From 2cd22c1a13d6b4e1c6d2407d249f2f8d92643fbd Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 9 Dec 2011 21:25:37 +0000
Subject: [PATCH 2646/8313] brr, so many typos today.

---
 doc/forum/pure_git-annex_only_workflow.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/pure_git-annex_only_workflow.mdwn b/doc/forum/pure_git-annex_only_workflow.mdwn
index 48226671c1..36648a9058 100644
--- a/doc/forum/pure_git-annex_only_workflow.mdwn
+++ b/doc/forum/pure_git-annex_only_workflow.mdwn
@@ -41,6 +41,6 @@ What I would like to have is that
 
 * git annex add does not require a commit afterwards.
 * Changes to the files are automatically picked up with the next git-annex call (similar to how etckeeper works).
-* Commands "git annex push" and "git annex pull" that will syn the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
+* Commands "git annex push" and "git annex pull" that will sync the metadata (i.e. the list of files) in both directions without further manual intervention, at least not until the two repositories have diverged in a way that is not possible to merge sensible.
 
 Summay: git-annex is great. git is not always. Please make it possible to use git annex without having to use git.

From 28699c95a7de284f07a5c0e34fb1755868301f3c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 18:10:41 -0400
Subject: [PATCH 2647/8313] some work on avoiding partial functions

There are still hundreds of places that use partial functions head, tail,
init, and last.
---
 Command/DropUnused.hs |  4 ++--
 Git.hs                | 18 ++++++------------
 Utility/BadPrelude.hs | 24 ++++++++++++++++++++++++
 Utility/Misc.hs       | 13 +++++++++++++
 4 files changed, 45 insertions(+), 14 deletions(-)
 create mode 100644 Utility/BadPrelude.hs

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 3df9ab6c2b..244f378d97 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -73,6 +73,6 @@ readUnusedLog prefix = do
 		then M.fromList . map parse . lines <$> liftIO (readFile f)
 		else return M.empty
 	where
-		parse line = (num, fromJust $ readKey $ tail rest)
+		parse line = (num, fromJust $ readKey rest)
 			where
-				(num, rest) = break (== ' ') line
+				(num, rest) = separate (== ' ') line
diff --git a/Git.hs b/Git.hs
index 5bdd4afd4e..84153be5d8 100644
--- a/Git.hs
+++ b/Git.hs
@@ -507,11 +507,7 @@ configStore s repo = do
 configParse :: String -> M.Map String String
 configParse s = M.fromList $ map pair $ lines s
 	where
-		pair l = (key l, val l)
-		key l = head $ keyval l
-		val l = join sep $ drop 1 $ keyval l
-		keyval l = split sep l :: [String]
-		sep = "="
+		pair = separate (== '=')
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 configRemotes :: Repo -> IO [Repo]
@@ -550,13 +546,11 @@ genRemote s repo = gen $ calcloc s
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
 			where
-				bits = split ":" v
-				host = head bits
-				dir = join ":" $ drop 1 bits
-				slash d	| d == "" = "/~/" ++ dir
-					| head d == '/' = dir
-					| head d == '~' = '/':dir
-					| otherwise = "/~/" ++ dir
+				(host, dir) = separate (== ':') v
+				slash d	| d == "" = "/~/" ++ d
+					| "/" `isPrefixOf` d = d
+					| "~" `isPrefixOf` d = '/':d
+					| otherwise = "/~/" ++ d
 
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
new file mode 100644
index 0000000000..1bb70adfb3
--- /dev/null
+++ b/Utility/BadPrelude.hs
@@ -0,0 +1,24 @@
+{- Some stuff from Prelude should not be used, as it tends to be a source
+ - of bugs.
+ -
+ - This exports functions that conflict with the prelude, which avoids
+ - them being accidentially used.
+ -}
+
+module Utility.BadPrelude where
+
+{- head is a partial function; head [] is an error -}
+head :: [a] -> a
+head = Prelude.head
+
+{- tail is also partial -}
+tail :: [a] -> a
+tail = Prelude.tail
+
+{- init too -}
+init :: [a] -> a
+init = Prelude.init
+
+{- last too -}
+last :: [a] -> a
+last = Prelude.last
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 7285987231..541e150b77 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -27,6 +27,19 @@ readMaybe s = case reads s of
 	((x,_):_) -> Just x
 	_ -> Nothing
 
+{- Like break, but the character matching the condition is not included
+ - in the second result list.
+ -
+ - separate (== ':') "foo:bar" = ("foo", "bar")
+ - separate (== ':') "foobar" = ("foo, "")
+ -}
+separate :: (a -> Bool) -> [a] -> ([a], [a])
+separate c l = unbreak $ break c l
+	where
+		unbreak r@(a, b)
+			| null b = r
+			| otherwise = (a, tail b)
+
 {- Catches IO errors and returns a Bool -}
 catchBoolIO :: IO Bool -> IO Bool
 catchBoolIO a = catchDefaultIO a False

From be67294a1ebf1a49eddaf9698c662888d80d6a7a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 9 Dec 2011 22:56:11 +0000
Subject: [PATCH 2648/8313] Added a comment

---
 ..._a32f7efd18d174845099a4ed59e6feae._comment | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment b/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment
new file mode 100644
index 0000000000..def1794a3e
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_1_a32f7efd18d174845099a4ed59e6feae._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-09T22:56:11Z"
+ content="""
+First, you need a bare git repository that you can push to, and pull from. This simplifies most git workflow.
+
+Secondly, I use [mr](http://kitenet.net/~joey/code/mr/), with this in `.mrconfig`:
+
+
+[DEFAULT]
+lib =
+        annexupdate() {
+                git commit -a -m update || true
+                git pull \"$@\"
+                git annex merge
+                git push || true
+        }
+
+[lib/sound]
+update = annexupdate
+[lib/big]
+update = annexupdate
+

+
+Which makes \"mr update\" in repositories where I rarely care about git details take care of syncing my changes.
+
+I also make \"mr update\" do a \"git annex get\" of some files in some repositories that I want to always populate. git-annex and mr go well together. :)
+
+Perhaps my annexupdate above should be available as \"git annex sync\"?
+"""]]

From 4b3c4c0c2b88aa17a9b7822470e3e5dd2a3e7c2b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 18:57:09 -0400
Subject: [PATCH 2649/8313] avoid some read

---
 Config.hs             | 7 ++++---
 Git.hs                | 2 +-
 Utility/BadPrelude.hs | 4 ++++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/Config.hs b/Config.hs
index cc0c929531..c6107fc8eb 100644
--- a/Config.hs
+++ b/Config.hs
@@ -79,9 +79,10 @@ repoNotIgnored r = not . Git.configTrue <$> getConfig r "ignore" "false"
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}
 getNumCopies :: Maybe Int -> Annex Int
-getNumCopies v = 
-	Annex.getState Annex.forcenumcopies >>= maybe (use v) (return . id)
+getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
-		use Nothing = read <$> fromRepo (Git.configGet config "1")
+		use Nothing = perhaps (return 1) =<< 
+			readMaybe <$> fromRepo (Git.configGet config "1")
+		perhaps fallback = maybe fallback (return . id)
 		config = "annex.numcopies"
diff --git a/Git.hs b/Git.hs
index 84153be5d8..8bc32b7cc7 100644
--- a/Git.hs
+++ b/Git.hs
@@ -345,7 +345,7 @@ urlPort :: Repo -> Maybe Integer
 urlPort r = 
 	case urlAuthPart uriPort r of
 		":" -> Nothing
-		(':':p) -> Just (read p)
+		(':':p) -> readMaybe p
 		_ -> Nothing
 
 {- Hostname of an URL repo, including any username (ie, "user@host") -}
diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 1bb70adfb3..7921a7e9b7 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -22,3 +22,7 @@ init = Prelude.init
 {- last too -}
 last :: [a] -> a
 last = Prelude.last
+
+{- read should be avoided, as it throws an error -}
+read :: Read a => String -> a
+read = Prelude.read

From fb8231f3a1eb67e61a7df0381dc82be2f0ca0417 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Dec 2011 20:27:22 -0400
Subject: [PATCH 2650/8313] sync: New command that synchronises the local
 repository and default remote, by running git commit, pull, and push for you.

---
 Command/Sync.hs    | 64 ++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  4 ++-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn | 11 ++++++++
 4 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100644 Command/Sync.hs

diff --git a/Command/Sync.hs b/Command/Sync.hs
new file mode 100644
index 0000000000..c0d7f37ad0
--- /dev/null
+++ b/Command/Sync.hs
@@ -0,0 +1,64 @@
+{- git-annex command
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Sync where
+
+import Common.Annex
+import Command
+import qualified Annex.Branch
+import qualified Git
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+def :: [Command]
+def = [command "sync" paramPaths seek "synchronize local repository with remote"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStart
+start = do
+        showStart "sync" "."
+	showOutput
+        next perform
+
+perform :: CommandPerform
+perform = do
+	remote <- defaultRemote =<< currentBranch
+	checkRemote remote
+	commit
+	inRepo $ Git.run "pull" [Param remote]
+	Annex.Branch.update
+	inRepo $ Git.run "push" [Param remote, matchingbranches]
+	next $ return True
+	where
+		-- git push may be configured to not push matching
+		-- branches; this should ensure it always does.
+		matchingbranches = Param ":"
+
+commit :: Annex ()
+commit = do
+	-- Commit will fail when the tree is clean (or when in a confliced
+	-- merge, etc). Ignore failure.
+	_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
+	return ()
+
+-- the remote defaults to origin when not configured
+defaultRemote :: String -> Annex String
+defaultRemote branch =
+	fromRepo $ Git.configGet ("branch." ++ branch ++ ".remote") "origin"
+
+currentBranch :: Annex String
+currentBranch = last . split "/" . L.unpack . head . L.lines <$>
+	inRepo (Git.pipeRead [Param "symbolic-ref", Param "HEAD"])
+
+checkRemote :: String -> Annex ()
+checkRemote remote = do
+	remoteurl <- fromRepo $
+		Git.configGet ("remote." ++ remote ++ ".url") ""
+	when (null remoteurl) $ do
+		error $ "No url is configured for the remote: " ++ remote
diff --git a/GitAnnex.hs b/GitAnnex.hs
index d768499ddb..7871638e44 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -47,6 +47,7 @@ import qualified Command.Trust
 import qualified Command.Untrust
 import qualified Command.Semitrust
 import qualified Command.Dead
+import qualified Command.Sync
 import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
@@ -61,6 +62,8 @@ cmds = concat
 	, Command.Copy.def
 	, Command.Unlock.def
 	, Command.Lock.def
+	, Command.Sync.def
+	, Command.AddUrl.def
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
@@ -72,7 +75,6 @@ cmds = concat
 	, Command.Untrust.def
 	, Command.Semitrust.def
 	, Command.Dead.def
-	, Command.AddUrl.def
 	, Command.FromKey.def
 	, Command.DropKey.def
 	, Command.Fix.def
diff --git a/debian/changelog b/debian/changelog
index 1c27ad5666..b481d99997 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     multiple different encrypted special remotes.
   * unannex: Can be run on files that have been added to the annex, but not
     yet committed.
+  * sync: New command that synchronises the local repository and default
+    remote, by running git commit, pull, and push for you.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 08def0d62b..d7a51663fd 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -120,6 +120,17 @@ subdirectories).
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
+* sync
+
+  Use this command when you want to synchronize the local repository
+  with its default remote (typically "origin"). The sync process involves
+  first committing all local changes, then pulling and merging any changes
+  from the remote, and finally pushing the repository's state to the remote.
+  You can use standard git commands to do each of those steps by hand,
+  or if you don't want to worry about the details, you can use sync.
+
+  Note that sync does not transfer any file contents from or to the remote.
+
 * addurl [url ...]
 
   Downloads each url to a file, which is added to the annex.

From 4200b8038a79aa788ce0b62579bd5820b7d75e3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 12:21:22 -0400
Subject: [PATCH 2651/8313] separate operations

---
 Command/Sync.hs | 54 +++++++++++++++++++++++++++----------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index c0d7f37ad0..7dc5f4d24c 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -17,39 +17,45 @@ import qualified Data.ByteString.Lazy.Char8 as L
 def :: [Command]
 def = [command "sync" paramPaths seek "synchronize local repository with remote"]
 
+-- syncing involves several operations, any of which can independantly fail
 seek :: [CommandSeek]
-seek = [withNothing start]
+seek = map withNothing [commit, pull, push]
 
-start :: CommandStart
-start = do
-        showStart "sync" "."
-	showOutput
-        next perform
+commit :: CommandStart
+commit = do
+	showStart "commit" ""
+	next $ next $ do
+		showOutput
+		-- Commit will fail when the tree is clean, so ignore failure.
+		_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
+		return True
 
-perform :: CommandPerform
-perform = do
-	remote <- defaultRemote =<< currentBranch
-	checkRemote remote
-	commit
-	inRepo $ Git.run "pull" [Param remote]
-	Annex.Branch.update
-	inRepo $ Git.run "push" [Param remote, matchingbranches]
-	next $ return True
+pull :: CommandStart
+pull = do
+	remote <- defaultRemote
+	showStart "pull" remote
+	next $ next $ do
+		showOutput
+		checkRemote remote
+		inRepo $ Git.runBool "pull" [Param remote]
+
+push :: CommandStart
+push = do
+	remote <- defaultRemote
+	showStart "push" remote
+	next $ next $ do
+		Annex.Branch.update
+		showOutput
+		inRepo $ Git.runBool "push" [Param remote, matchingbranches]
 	where
 		-- git push may be configured to not push matching
 		-- branches; this should ensure it always does.
 		matchingbranches = Param ":"
 
-commit :: Annex ()
-commit = do
-	-- Commit will fail when the tree is clean (or when in a confliced
-	-- merge, etc). Ignore failure.
-	_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
-	return ()
-
 -- the remote defaults to origin when not configured
-defaultRemote :: String -> Annex String
-defaultRemote branch =
+defaultRemote :: Annex String
+defaultRemote = do
+	branch <- currentBranch
 	fromRepo $ Git.configGet ("branch." ++ branch ++ ".remote") "origin"
 
 currentBranch :: Annex String

From e0e9d1cabd0aca52aa410be37de7251533b5e237 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sat, 10 Dec 2011 16:28:29 +0000
Subject: [PATCH 2652/8313] Added a comment

---
 ...nt_2_66dc9b65523a9912411db03c039ba848._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment b/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment
new file mode 100644
index 0000000000..473a0287d0
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_2_66dc9b65523a9912411db03c039ba848._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2011-12-10T16:28:29Z"
+ content="""
+Thanks for the tips so far. I guess a bare-only repo helps, but as well is something that I don’t _need_ (for my use case), any only have to do because git works like this.
+
+Also, if I have a mobile device that I want to push to, then I’d have to have two repositories on the device, as I might not be able to reach my main bare repository when traveling, but I cannot push to the „real“ repo on the mobile device from my computer. I guess I am spoiled by darcs, which will happily push to a checked out 
+remote repository, updating the checkout if possible without conflict.
+
+If I introduce a central bare repository to push to and from; I’d still have to have the other non-bare repos as remotes, so that git-annex will know about them and their files, right?
+
+I’d appreciate a \"git annex sync\" that does what you described (commit all, pull, merge, push). Especially if it comes in a \"git annex sync --all\" variant that syncs all reachable repositories.
+"""]]

From eecaf424856e42290d653e0b4b4043665da18b00 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 12:30:31 -0400
Subject: [PATCH 2653/8313] no need to show, it's a string

---
 Annex/Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 8f0a09fd6b..37237ec667 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -141,7 +141,7 @@ update = onceonly $ do
 		let merge_desc = if null branches
 			then "update" 
 			else "merging " ++
-				unwords (map (show . Git.refDescribe) branches) ++ 
+				unwords (map Git.refDescribe branches) ++ 
 				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc

From c5267802f38eb020edee8143d5a097296232bb35 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 12:49:44 -0400
Subject: [PATCH 2654/8313] version dependency on old monad-control

This should let cabal build it with the right version.
---
 debian/changelog | 1 +
 git-annex.cabal  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index b481d99997..8dbcb07dc6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,6 +11,7 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     yet committed.
   * sync: New command that synchronises the local repository and default
     remote, by running git commit, pull, and push for you.
+  * Version monad-control dependency in cabal file.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 7be78053f8..8f5d6ebb86 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, HTTP,
-   base < 5, monad-control, json
+   base < 5, monad-control < 0.3, json
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 6cf28585b66264e174eee45496e01d23667e0d20 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 10 Dec 2011 19:43:04 +0000
Subject: [PATCH 2655/8313] Added a comment

---
 ...mment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment b/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment
new file mode 100644
index 0000000000..9b6e6d7c4d
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_3_9b7d89da52f7ebb7801f9ec8545c3aba._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2011-12-10T19:43:04Z"
+ content="""
+Git can actually push into a non-bare repository, so long as the branch you change there is not a checked out one. Pushing into `remotes/$foo/master` and `remotes/$foo/git-annex` would work, however determining the value that the repository expects for `$foo` is something git cannot do on its own. And of course you'd still have to `git merge remotes/$foo/master` to get the changes. 
+
+Yes, you still keep the non-bare repos as remotes when adding a bare repository, so git-annex knows how to get to them.
+
+I've made `git annex sync` run the simple script above. Perhaps it can later be improved to sync all repositories.
+"""]]

From bfdc9f28fc6ef19f7e3ec2ea724a540af104a386 Mon Sep 17 00:00:00 2001
From: "http://schnouki.net/" 
Date: Sat, 10 Dec 2011 21:40:01 +0000
Subject: [PATCH 2656/8313]

---
 ...__git-annex_get__34___with_3.20111203.mdwn | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn

diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
new file mode 100644
index 0000000000..2e77713862
--- /dev/null
+++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
@@ -0,0 +1,21 @@
+Hi there,
+
+After updating to 3.20111203 (on Arch Linux) I noticed I was not able to use `git annex get` from a SSH remote (server running Arch Linux, same version of git-annex): "requested key is not present". Same behavior with current master (commit 6cf28585). I had no issue with the previous version (3.20111122).
+
+On this server, I was able to track down the issue using `git-annex-shell inannex` and `strace`:
+
+    $ strace -f -o log git-annex-shell inannex ~/photos-annex.git WORM-s369360-m1321602916--2011-11-17.jpg   
+    $ echo $?
+    1
+    $ tail -n20 log
+    [...]
+    25623 chdir("/home/schnouki/git-annex") = 0
+    25623 stat("/home/schnouki/photos-annex.git/annex/objects/082/676/WORM-s369360-m1321602916--2011-11-17.jpg/WORM-s369360-m1321602916--2011-11-17.jpg", {st_mode=S_IFREG|0400, st_size=369360, ...}) = 0
+    25623 open("annex/objects/082/676/WORM-s369360-m1321602916--2011-11-17.jpg/WORM-s369360-m1321602916--2011-11-17.jpg", O_RDONLY) = -1 ENOENT (No such file or directory)
+    [...]
+
+Note there is a call to `stat()` with the full path to the requested file, and *then* a call to `open()` with a relative path -- which calls this call to fail, and git-annex-shell to return 1. With 3.20111122, there was no call to `stat()`, just a successful call to `open()` with a full absolute path.
+
+Using `git bisect` I was able to determine that this bug appeared in commit 64672c62 ("refactor"). Reverting it makes `git-annex-shell` work as expected, but I'm sure there are better ways to fix this. However I don't know enough Haskell to do it myself.
+
+Could you please try to fix this in a future version?

From 10e8028a42fe65e8586b7424557b792cf36e215c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 18:45:55 -0400
Subject: [PATCH 2657/8313] Fix bug in last version in getting contents from
 bare repositories.

---
 Locations.hs                                             | 9 ++++-----
 debian/changelog                                         | 1 +
 ..._39__t___34__git-annex_get__34___with_3.20111203.mdwn | 5 +++++
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 3843495f9c..080766841e 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -80,16 +80,15 @@ gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
 		{- Bare repositories default to hashDirLower for new
 		 - content, as it's more portable. -}
-		go (Git.workTree r) (annexLocations key)
+		check (map inrepo $ annexLocations key)
 	| otherwise =
 		{- Non-bare repositories only use hashDirMixed, so
 		 - don't need to do any work to check if the file is
 		 - present. -}
-		return $ Git.workTree r  ".git" 
-				annexLocation key hashDirMixed
+		return $ inrepo ".git"  annexLocation key hashDirMixed
 	where
-		go dir locs = fromMaybe (dir  head locs) <$> check dir locs
-		check dir = firstM $ \f -> doesFileExist $ dir  f
+		inrepo = () (Git.workTree r)
+		check locs = fromMaybe (head locs) <$> firstM doesFileExist locs
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
diff --git a/debian/changelog b/debian/changelog
index 8dbcb07dc6..5c68fde2db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -12,6 +12,7 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * sync: New command that synchronises the local repository and default
     remote, by running git commit, pull, and push for you.
   * Version monad-control dependency in cabal file.
+  * Fix bug in last version in getting contents from bare repositories.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
index 2e77713862..00ab9f3f07 100644
--- a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
+++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
@@ -19,3 +19,8 @@ Note there is a call to `stat()` with the full path to the requested file, and *
 Using `git bisect` I was able to determine that this bug appeared in commit 64672c62 ("refactor"). Reverting it makes `git-annex-shell` work as expected, but I'm sure there are better ways to fix this. However I don't know enough Haskell to do it myself.
 
 Could you please try to fix this in a future version?
+
+> Thanks for a very good bug report. 
+> 
+> I've fixed this stupid mistake introduced in the code refactoring.
+> --[[Joey]]

From 9ba99a544b79f9304839a5f4e756d8025e9f18f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 18:51:01 -0400
Subject: [PATCH 2658/8313] update

---
 Remote/Bup.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 589dea91d9..e705bbb344 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -182,7 +182,7 @@ onBupRemote r a command params = do
  - local bup repositories to see if they are available, and getting their
  - uuid (which may be different from the stored uuid for the bup remote).
  -
- - If a bup repository is not available, returns a dummy uuid of "".
+ - If a bup repository is not available, returns NoUUID.
  - This will cause checkPresent to indicate nothing from the bup remote
  - is known to be present.
  -

From f44f715f51aad6cf164501b0d53b7163f3a5e758 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 19:38:27 -0400
Subject: [PATCH 2659/8313] ensure local remote is initialized when copying to
 it

Needed due to this scenario: Bare repo origin is made, foo is cloned from it;
foo is initalized; a file is added to foo's annex; git annex move --to origin

Since the git-annex branch has not yet been pushed to origin, it doesn't
auto-initialize. When the content is sent to it, it's stored, but
the remote has NoUUID, and so nothing is logged in the location log.
Then the content is removed from the local repo, and git-annex has lost
track of it.

git annex fsck in origin will find the lost content, but let's not let this
happen. Content should only be sent to initalized remotes.

This cannot happen for non-local remotes, since git-annex-shell always
checks that the repo is initialized.
---
 Remote/Git.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index d172ec7c04..05743a28d5 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -210,6 +210,7 @@ copyToRemote r key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
+			ensureInitialized
 			ok <- Annex.Content.getViaTmp key $
 				rsyncOrCopyFile params keysrc
 			Annex.Content.saveState

From 583ba809923948eb3ce9b3f91892c7492e5db14f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Dec 2011 20:53:42 -0400
Subject: [PATCH 2660/8313] better syntax

---
 Locations.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Locations.hs b/Locations.hs
index 080766841e..f7784a56d5 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -87,7 +87,7 @@ gitAnnexLocation key r
 		 - present. -}
 		return $ inrepo ".git"  annexLocation key hashDirMixed
 	where
-		inrepo = () (Git.workTree r)
+		inrepo d = Git.workTree r  d
 		check locs = fromMaybe (head locs) <$> firstM doesFileExist locs
 
 {- The annex directory of a repository. -}

From bf6a3b757adebe92b330c4216fb32fbf1a6d64d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 01:15:27 -0400
Subject: [PATCH 2661/8313] close

---
 .../Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
index 00ab9f3f07..ea56c37320 100644
--- a/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
+++ b/doc/bugs/Can__39__t___34__git-annex_get__34___with_3.20111203.mdwn
@@ -23,4 +23,5 @@ Could you please try to fix this in a future version?
 > Thanks for a very good bug report. 
 > 
 > I've fixed this stupid mistake introduced in the code refactoring.
+> [[done]]
 > --[[Joey]]

From 0ba4b1de18acf9e2318da6c8e7d80ce4bb216850 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 14:14:28 -0400
Subject: [PATCH 2662/8313] move a file location to Locations.hs

---
 Annex/Branch.hs | 6 +-----
 Locations.hs    | 5 +++++
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 37237ec667..44ee69e627 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -43,10 +43,6 @@ fullname = Git.Ref $ "refs/heads/" ++ show name
 originname :: Git.Ref
 originname = Git.Ref $ "origin/" ++ show name
 
-{- A separate index file for the branch. -}
-index :: Git.Repo -> FilePath
-index g = gitAnnexDir g  "index"
-
 {- Populates the branch's index file with the current branch contents.
  - 
  - Usually, this is only done when the index doesn't yet exist, and
@@ -62,7 +58,7 @@ withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
-	f <- fromRepo index
+	f <- fromRepo gitAnnexIndex
 	bracketIO (Git.useIndex f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
diff --git a/Locations.hs b/Locations.hs
index f7784a56d5..fdb8232844 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -20,6 +20,7 @@ module Locations (
 	gitAnnexUnusedLog,
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
+	gitAnnexIndex,
 	isLinkToAnnex,
 	annexHashes,
 	hashDirMixed,
@@ -131,6 +132,10 @@ gitAnnexJournalDir r = addTrailingPathSeparator $ gitAnnexDir r  "journal"
 gitAnnexJournalLock :: Git.Repo -> FilePath
 gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
 
+{- .git/annex/index is used to stage changes to the git-annex branch -}
+gitAnnexIndex :: Git.Repo -> FilePath
+gitAnnexIndex r = gitAnnexDir r  "index"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s

From 59971c923029a6f10c47a526e7878130637c139e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 14:47:44 -0400
Subject: [PATCH 2663/8313] new bug

---
 doc/bugs/git-annex_branch_corruption.mdwn | 89 +++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 doc/bugs/git-annex_branch_corruption.mdwn

diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn
new file mode 100644
index 0000000000..63633fb080
--- /dev/null
+++ b/doc/bugs/git-annex_branch_corruption.mdwn
@@ -0,0 +1,89 @@
+Below is a test case which shows a way that the git-annex branch
+can become corrupted and lose data, including location log records and
+uuid.log lines.
+
+At the end, a commit on the git-annex branch removes one of the 2 lines
+from the uuid.log; which should never happen.
+
+The actual problem occurs earlier, at the "push point". Here a repo is
+cloned from the main one, initialized (adding the last uuid.log line),
+and then pushed back to the main one. That push is a fast-forward, so is
+allowed to directly update the git-annex branch in the main repo:
+
+	b884fe5..c497739  git-annex -> git-annex
+
+Now the git-annex branch has a change that is not reflected in
+`.git/annex/index`, so the next time a change is made, it's committed
+using the out of date index, which causes a reversion of the changes
+that were pushed to the branch.
+
+---
+
+## Thoughts
+
+This is essentially the same reason why git blocks pushes to the checked-out
+branch of a non-bare repository. 
+
+This problem only affects workflows that involve pushing. Pulling workflows
+do not directly update the local git-annex branch, so avoid the problem.
+
+And while bare repos are pushed to, they rarely have changes made directly
+to their git-annex branches, so while I think the same problem could
+happen with pushing to a bare repo, it's unlikely.
+
+None of which is to say this is not a bad bug that needs to be comprehensively
+fixed. 
+
+Probably git-annex needs to record which ref of the git-annex branch
+corresponds to its index, and if the branch is at a different ref,
+merge it into the index. I am still considering how to do that atomically;
+what if a push comes in while git-annex is updating its index?
+
+---
+
+## Workaround
+
+Users who want to prevent this bug from occuring when pushing to their
+non-bare repositories can install this script as `.git/hooks/update`
+
+
+#!/bin/sh
+if [ "$1" = refs/heads/git-annex ]; then
+	exit 1
+fi
+

+
+--[[Joey]] 
+
+---
+
+## Test Case
+
+#!/bin/sh
+mkdir annextest
+cd annextest
+
+git init dir1
+cd dir1
+git annex init
+touch foo 
+echo hi > bar
+git annex add
+git commit -m add
+
+cd ..
+git clone dir1 dir2
+cd dir2
+git annex init otherdir
+git annex get
+# push point
+git push
+
+cd ..
+cd dir1
+echo "before"
+git show git-annex:uuid.log
+git annex drop foo --force
+echo "after"
+git show git-annex:uuid.log
+


From 8680c415dedcc1fdbbcc0b9cdd7e37e860b22130 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 14:51:20 -0400
Subject: [PATCH 2664/8313] slow, stupid, and safe index updating

Always merge the git-annex branch into .git/annex/index before making a
commit from the index.

This ensures that, when the branch has been changed in any way
(by a push being received, or changes pulled directly into it, or
even by the user checking it out, and committing a change), the index
reflects those changes.

This is much too slow; it needs to be optimised to only update the
index when the branch has really changed, not every time.

Also, there is an unhandled race, when a change is made to the branch
right after the index gets updated. I left it in for now because it's
unlikely and I didn't want to complicate things with additional locking
yet.
---
 Annex/Branch.hs                           | 37 ++++++++++++++---------
 debian/changelog                          |  5 +++
 doc/bugs/git-annex_branch_corruption.mdwn |  3 ++
 doc/internals.mdwn                        | 14 ++-------
 4 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 44ee69e627..be19a9be0c 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -45,14 +45,26 @@ originname = Git.Ref $ "origin/" ++ show name
 
 {- Populates the branch's index file with the current branch contents.
  - 
- - Usually, this is only done when the index doesn't yet exist, and
- - the index is used to build up changes to be commited to the branch,
- - and merge in changes from other branches.
+ - This is only done when the index doesn't yet exist, and the index 
+ - is used to build up changes to be commited to the branch, and merge
+ - in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
 genIndex g = Git.UnionMerge.stream_update_index g
 	[Git.UnionMerge.ls_tree fullname g]
 
+{- Merges the specified branches into the index.
+ - Any changes staged in the index will be preserved. -}
+mergeIndex :: [Git.Ref] -> Annex ()
+mergeIndex branches = do
+	h <- catFileHandle
+	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+
+{- Updates the branch's index to reflect the current contents of the branch.
+ - Any changes staged in the index will be preserved. -}
+updateIndex :: Annex ()
+updateIndex = withIndex $ mergeIndex [fullname]
+
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
@@ -66,6 +78,8 @@ withIndex' bootstrapping a = do
 			unless bootstrapping $ inRepo genIndex
 		a
 
+{- Runs an action using the branch's index file, first making sure that
+ - the branch and index are up-to-date. -}
 withIndexUpdate :: Annex a -> Annex a
 withIndexUpdate a = update >> withIndex a
 
@@ -106,11 +120,12 @@ create = unlessM hasBranch $ do
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
+	updateIndex
 	stageJournalFiles
 	withIndex $ inRepo $ Git.commit message fullname [fullname]
 
-{- Ensures that the branch is up-to-date; should be called before data is
- - read from it. Runs only once per git-annex run.
+{- Ensures that the branch and index are is up-to-date; should be
+ - called before data is read from it. Runs only once per git-annex run.
  -
  - Before refs are merged into the index, it's important to first stage the
  - journal into the index. Otherwise, any changes in the journal would
@@ -126,8 +141,9 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
-	-- ensure branch exists
+	-- ensure branch exists, and index is up-to-date
 	create
+	updateIndex
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
@@ -141,14 +157,7 @@ update = onceonly $ do
 				" into " ++ show name
 		unless (null branches) $ do
 			showSideAction merge_desc
-			{- Note: This merges the branches into the index.
-			 - Any unstaged changes in the git-annex branch
-			 - (if it's checked out) will be removed. So,
-			 - documentation advises users not to directly
-			 - modify the branch.
-			 -}
-			h <- catFileHandle
-			inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+			mergeIndex branches
 		ff <- if dirty then return False else tryFastForwardTo refs
 		unless ff $ inRepo $
 			Git.commit merge_desc fullname (nub $ fullname:refs)
diff --git a/debian/changelog b/debian/changelog
index 5c68fde2db..405e98b749 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,11 @@ git-annex (3.20111204) UNRELEASED; urgency=low
     remote, by running git commit, pull, and push for you.
   * Version monad-control dependency in cabal file.
   * Fix bug in last version in getting contents from bare repositories.
+  * Ensure that git-annex branch changes are merged into git-annex's index,
+    which fixes a bug that could cause changes that were pushed to the
+    git-annex branch to get reverted. As a side effect, it's now safe
+    for users to check out and commit changes directly to the git-annex
+    branch.
 
  -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
 
diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn
index 63633fb080..5249e63d82 100644
--- a/doc/bugs/git-annex_branch_corruption.mdwn
+++ b/doc/bugs/git-annex_branch_corruption.mdwn
@@ -39,6 +39,9 @@ corresponds to its index, and if the branch is at a different ref,
 merge it into the index. I am still considering how to do that atomically;
 what if a push comes in while git-annex is updating its index?
 
+> Now git-annex always updates the index with the git-annex branch, which
+> is a slow, but safe way to avoid this problem. [[done]] --[[Joey]]
+
 ---
 
 ## Workaround
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index d84b3c4898..68cc7c3cd9 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -22,17 +22,9 @@ deleting or changing the file contents.
 This branch is managed by git-annex, with the contents listed below.
 
 The file `.git/annex/index` is a separate git index file it uses
-to accumulate changes for the git-annex. Also, `.git/annex/journal/` is used
-to record changes before they are added to git.
-
-Note that for speed reasons, git-annex assumes only it will modify this
-branch. If you go in and make changes directly, it will probably revert
-your changes in its next commit to the branch.
-
-The best way to make changes to the git-annex branch is instead
-to create a branch of it, with a name like "my/git-annex", and then
-use "git annex merge" to automerge your branch into the main git-annex
-branch.
+to accumulate changes for the git-annex branch.
+Also, `.git/annex/journal/` is used to record changes before they
+are added to git.
 
 ### `uuid.log`
 

From cfbbda99f4dd3e510e52dbb499d132300ad203e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 16:11:13 -0400
Subject: [PATCH 2665/8313] optimize index updating

The last branch ref that the index was updated to is stored in
.git/annex/index.lck, and the index only updated when the current
branch ref differs.

(The .lck file should later be used for locking too.)

Some more optimization is still needed, since there is some redundancy in
calls to git show-ref.
---
 Annex/Branch.hs    | 54 ++++++++++++++++++++++++++++++++++++----------
 Git.hs             |  5 +++--
 Locations.hs       |  5 +++++
 git-union-merge.hs |  2 +-
 4 files changed, 52 insertions(+), 14 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index be19a9be0c..52e82e25cc 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -61,9 +61,26 @@ mergeIndex branches = do
 	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
 
 {- Updates the branch's index to reflect the current contents of the branch.
- - Any changes staged in the index will be preserved. -}
+ - Any changes staged in the index will be preserved.
+ -
+ - Compares the ref stored in the lock file with the current
+ - ref of the branch to see if an update is needed.
+ -}
 updateIndex :: Annex ()
-updateIndex = withIndex $ mergeIndex [fullname]
+updateIndex = do
+        lock <- fromRepo gitAnnexIndexLock
+	lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
+	branchref <- getRef fullname
+	when (lockref /= branchref) $ do
+		withIndex $ mergeIndex [fullname]
+		setIndexRef branchref
+
+{- Record that the branch's index has been updated to correspond to a
+ - given ref of the branch. -}
+setIndexRef :: Git.Ref -> Annex ()
+setIndexRef ref = do
+        lock <- fromRepo gitAnnexIndexLock
+	liftIO $ writeFile lock $ show ref ++ "\n"
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
@@ -109,12 +126,13 @@ getCache file = getState >>= go
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM hasBranch $ do
-	e <- hasOrigin
-	if e
-		then inRepo $ Git.run "branch"
-			[Param $ show name, Param $ show originname]
-		else withIndex' True $
+create = unlessM hasBranch $ hasOrigin >>= go >>= setIndexRef
+	where
+		go True = do
+			inRepo $ Git.run "branch"
+				[Param $ show name, Param $ show originname]
+			getRef fullname
+		go False = withIndex' True $
 			inRepo $ Git.commit "branch created" fullname []
 
 {- Stages the journal, and commits staged changes to the branch. -}
@@ -122,7 +140,8 @@ commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
 	updateIndex
 	stageJournalFiles
-	withIndex $ inRepo $ Git.commit message fullname [fullname]
+	withIndex $ 
+		setIndexRef =<< inRepo (Git.commit message fullname [fullname])
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
@@ -159,8 +178,9 @@ update = onceonly $ do
 			showSideAction merge_desc
 			mergeIndex branches
 		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $ inRepo $
-			Git.commit merge_desc fullname (nub $ fullname:refs)
+		unless ff $ 
+			setIndexRef =<< 
+				inRepo (Git.commit merge_desc fullname (nub $ fullname:refs))
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
@@ -253,6 +273,18 @@ siblingBranches = do
 		gen l = (Git.Ref $ head l, Git.Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
+{- Get the ref of a branch. -}
+getRef :: Git.Ref -> Annex Git.Ref
+getRef branch = firstRef . L.unpack <$> showref
+	where
+		showref = inRepo $ Git.pipeRead [Param "show-ref",
+			Param "--hash", -- get the hash
+			Param "--verify", -- only exact match
+			Param $ show branch]
+
+firstRef :: String-> Git.Ref
+firstRef = Git.Ref . takeWhile (/= '\n')
+
 {- Applies a function to modifiy the content of a file.
  -
  - Note that this does not cause the branch to be merged, it only
diff --git a/Git.hs b/Git.hs
index 8bc32b7cc7..1da5997c18 100644
--- a/Git.hs
+++ b/Git.hs
@@ -463,8 +463,8 @@ shaSize :: Int
 shaSize = 40
 
 {- Commits the index into the specified branch (or other ref), 
- - with the specified parent refs. -}
-commit :: String -> Ref -> [Ref] -> Repo -> IO ()
+ - with the specified parent refs, and returns the new ref -}
+commit :: String -> Ref -> [Ref] -> Repo -> IO Ref
 commit message newref parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
 		pipeRead [Param "write-tree"] repo
@@ -473,6 +473,7 @@ commit message newref parentrefs repo = do
 			(map Param $ ["commit-tree", show tree] ++ ps)
 			(L.pack message) repo
 	run "update-ref" [Param $ show newref, Param $ show sha] repo
+	return sha
 	where
 		ignorehandle a = snd <$> a
 		asString a = L.unpack <$> a
diff --git a/Locations.hs b/Locations.hs
index fdb8232844..85fcb98887 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -21,6 +21,7 @@ module Locations (
 	gitAnnexJournalDir,
 	gitAnnexJournalLock,
 	gitAnnexIndex,
+	gitAnnexIndexLock,
 	isLinkToAnnex,
 	annexHashes,
 	hashDirMixed,
@@ -136,6 +137,10 @@ gitAnnexJournalLock r = gitAnnexDir r  "journal.lck"
 gitAnnexIndex :: Git.Repo -> FilePath
 gitAnnexIndex r = gitAnnexDir r  "index"
 
+{- Lock file for .git/annex/index. -}
+gitAnnexIndexLock :: Git.Repo -> FilePath
+gitAnnexIndexLock r = gitAnnexDir r  "index.lck"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 1cec4a0f85..edd9330c80 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -42,5 +42,5 @@ main = do
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
-	Git.commit "union merge" newref [aref, bref] g
+	_ <- Git.commit "union merge" newref [aref, bref] g
 	cleanup g

From 0236bb020b7b055676b06d1cdf582746864f3d78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 16:19:54 -0400
Subject: [PATCH 2666/8313] update

---
 doc/bugs/git-annex_branch_corruption.mdwn | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/git-annex_branch_corruption.mdwn b/doc/bugs/git-annex_branch_corruption.mdwn
index 5249e63d82..9c864d85f0 100644
--- a/doc/bugs/git-annex_branch_corruption.mdwn
+++ b/doc/bugs/git-annex_branch_corruption.mdwn
@@ -36,11 +36,14 @@ fixed.
 
 Probably git-annex needs to record which ref of the git-annex branch
 corresponds to its index, and if the branch is at a different ref,
-merge it into the index. I am still considering how to do that atomically;
-what if a push comes in while git-annex is updating its index?
+merge it into the index.
 
-> Now git-annex always updates the index with the git-annex branch, which
-> is a slow, but safe way to avoid this problem. [[done]] --[[Joey]]
+> And now that's [[done]]. I managed to do it with very little slowdown.
+> 
+> A side benefit is that users can now safely check out the git-annex
+> branch and commit changes to it, and git-annex will notice them.
+> Before, it was documented to ignore such changes.
+> --[[Joey]] 
 
 ---
 

From 81f311103d99ec5bfd31ae5a76d6add05ff40121 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 16:41:56 -0400
Subject: [PATCH 2667/8313] a new bug report to track a race

---
 doc/bugs/git-annex_branch_push_race.mdwn | 43 ++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 doc/bugs/git-annex_branch_push_race.mdwn

diff --git a/doc/bugs/git-annex_branch_push_race.mdwn b/doc/bugs/git-annex_branch_push_race.mdwn
new file mode 100644
index 0000000000..257c477bff
--- /dev/null
+++ b/doc/bugs/git-annex_branch_push_race.mdwn
@@ -0,0 +1,43 @@
+The fix for the [[git-annex_branch_corruption]] bug is subject to a race.
+With that fix, git-annex does this when committing a change to the branch:
+
+1. lock the journal file (this avoids git-annex racing itself, FWIW)
+2. check what the head of the branch points to, to see if a newer branch
+   has appeared
+3. if so, updates the index file from the branch
+4. stages changes in the index
+5. commits to the branch using the index file
+
+If a push to the branch comes in during 2-5, then
+[[git-annex_branch_corruption]] could still occur.
+
+---
+
+## approach 1, using locking
+
+Add an update hook and a post-update hook. The update hook
+will use locking to ensure that no git-annex is currently running
+a commit, and block any git-annex's from starting one. It
+will background itself, and remain running during the push.
+The post-update hook will signal it to exit.
+
+I don't like this approach much, since it involves a daemon, two hooks,
+and lots of things to go wrong. And it blocks using git-annex during a
+push. This approach should be a last resort.
+
+## approach 2, lockless method
+
+After a commit is made to the branch, check to see if the parent of
+the commit is the same ref that the index file was last updated to. If it's
+not, then the race occurred.
+
+How to recover from the race? Well, just union merging the parent of the
+commit into the index file and re-committing should work, I think. When
+the race occurs, the commit reverts its parent's changes, and this will
+redo them. 
+
+(Of course, this re-commit will also be subject to the race, and
+will need the same check for the race as the other commits. It won't loop
+forever, I hope.)
+
+--[[Joey]]

From c4c965d602687a6277d43ae003c12dc4c132ba28 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 18:39:53 -0400
Subject: [PATCH 2668/8313] detect and recover from branch push/commit race

Dealing with a race without using locking is exceedingly difficult and tricky.
Fully tested, I hope.

There are three places left where the branch can be updated, that are not
covered by the race recovery code. Let's prove they're all immune to the
race:

1. tryFastForwardTo checks to see if a fast-forward can be done,
   and then does git-update-ref on the branch to fast-forward it.

   If a push comes in before the check, then either no fast-forward
   will be done (ok), or the push set the branch to a ref that can
   still be fast-forwarded (also ok)

   If a push comes in after the check, the git-update-ref will
   undo the ref change made by the push. It's as if the push did not come
   in, and the next git-push will see this, and try to re-do it.
   (acceptable)

2. When creating the branch for the very first time, an empty index
   is created, and a commit of it made to the branch. The commit's ref
   is recorded as the current state of the index. If a push came in
   during that, it will be noticed the next time a commit is made to the
   branch, since the branch will have changed. (ok)

3. Creating the branch from an existing remote branch involves making
   the branch, and then getting its ref, and recording that the index
   reflects that ref.

   If a push creates the branch first, git-branch will fail (ok).

   If the branch is created and a racing push is then able to change it
   (highly unlikely!) we're still ok, because it first records the ref into
   the index.lck, and then updating the index. The race can cause the
   index.lck to have the old branch ref, while the index has the newly pushed
   branch merged into it, but that only results in an unnecessary update of
   the index file later on.
---
 Annex/Branch.hs                          | 122 ++++++++++++++++-------
 Annex/CatFile.hs                         |   6 ++
 doc/bugs/git-annex_branch_push_race.mdwn |   2 +
 3 files changed, 94 insertions(+), 36 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 52e82e25cc..0ac4199944 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -60,28 +60,6 @@ mergeIndex branches = do
 	h <- catFileHandle
 	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
 
-{- Updates the branch's index to reflect the current contents of the branch.
- - Any changes staged in the index will be preserved.
- -
- - Compares the ref stored in the lock file with the current
- - ref of the branch to see if an update is needed.
- -}
-updateIndex :: Annex ()
-updateIndex = do
-        lock <- fromRepo gitAnnexIndexLock
-	lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
-	branchref <- getRef fullname
-	when (lockref /= branchref) $ do
-		withIndex $ mergeIndex [fullname]
-		setIndexRef branchref
-
-{- Record that the branch's index has been updated to correspond to a
- - given ref of the branch. -}
-setIndexRef :: Git.Ref -> Annex ()
-setIndexRef ref = do
-        lock <- fromRepo gitAnnexIndexLock
-	liftIO $ writeFile lock $ show ref ++ "\n"
-
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
 withIndex = withIndex' False
@@ -95,6 +73,80 @@ withIndex' bootstrapping a = do
 			unless bootstrapping $ inRepo genIndex
 		a
 
+{- Updates the branch's index to reflect the current contents of the branch.
+ - Any changes staged in the index will be preserved.
+ -
+ - Compares the ref stored in the lock file with the current
+ - ref of the branch to see if an update is needed.
+ -}
+updateIndex :: Annex (Maybe Git.Ref)
+updateIndex = do
+	branchref <- getRef fullname
+	go branchref
+	return branchref
+	where
+		go Nothing = return ()
+		go (Just branchref) = do
+			lock <- fromRepo gitAnnexIndexLock
+			lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
+			when (lockref /= branchref) $ do
+				withIndex $ mergeIndex [fullname]
+				setIndexRef branchref
+
+{- Record that the branch's index has been updated to correspond to a
+ - given ref of the branch. -}
+setIndexRef :: Git.Ref -> Annex ()
+setIndexRef ref = do
+        lock <- fromRepo gitAnnexIndexLock
+	liftIO $ writeFile lock $ show ref ++ "\n"
+
+{- Commits the staged changes in the index to the branch.
+ - 
+ - Ensures that the branch's index file is first updated to include the
+ - current state of the branch, before running the commit action. This
+ - is needed because the branch may have had changes pushed to it, that
+ - are not yet reflected in the index.
+ -
+ - Also safely handles a race that can occur if a change is being pushed
+ - into the branch at the same time. When the race happens, the commit will
+ - be made on top of the newly pushed change, but without the index file
+ - being updated to include it. The result is that the newly pushed
+ - change is reverted. This race is detected and another commit made
+ - to fix it.
+ -}
+commitBranch :: String -> [Git.Ref] -> Annex ()
+commitBranch message parents = do
+	expected <- updateIndex
+	committedref <- inRepo $ Git.commit message fullname parents
+	setIndexRef committedref
+	parentrefs <- commitparents <$> catObject committedref
+	when (racedetected expected parentrefs) $
+		fixrace committedref parentrefs
+	where
+		-- look for "parent ref" lines and return the refs
+		commitparents = map (Git.Ref . snd) . filter isparent .
+			map (toassoc . L.unpack) . L.lines
+		toassoc = separate (== ' ')
+		isparent (k,_) = k == "parent"
+		
+ 		{- The race can be detected by checking the commit's
+		 - parent, which will be the newly pushed branch,
+		 - instead of the expected ref that the index was updated to. -}
+		racedetected Nothing parentrefs
+			| null parentrefs = False -- first commit, no parents
+			| otherwise = True -- race on first commit
+		racedetected (Just expectedref) parentrefs
+			| expectedref `elem` parentrefs = False -- good parent
+			| otherwise = True -- race!
+		
+		{- To recover from the race, union merge the lost refs
+		 - into the index, and recommit on top of the bad commit. -}
+		fixrace committedref lostrefs = do
+			mergeIndex lostrefs
+			commitBranch racemessage [committedref]
+		
+		racemessage = message ++ " (recovery from race)"
+
 {- Runs an action using the branch's index file, first making sure that
  - the branch and index are up-to-date. -}
 withIndexUpdate :: Annex a -> Annex a
@@ -126,22 +178,20 @@ getCache file = getState >>= go
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM hasBranch $ hasOrigin >>= go >>= setIndexRef
+create = unlessM hasBranch $ hasOrigin >>= go
 	where
 		go True = do
 			inRepo $ Git.run "branch"
 				[Param $ show name, Param $ show originname]
-			getRef fullname
-		go False = withIndex' True $
-			inRepo $ Git.commit "branch created" fullname []
+			maybe (return ()) setIndexRef =<< getRef fullname
+		go False = withIndex' True $ 
+			setIndexRef =<< (inRepo $ Git.commit "branch created" fullname [])
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
-	updateIndex
 	stageJournalFiles
-	withIndex $ 
-		setIndexRef =<< inRepo (Git.commit message fullname [fullname])
+	withIndex $ commitBranch message [fullname]
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
@@ -162,7 +212,7 @@ update :: Annex ()
 update = onceonly $ do
 	-- ensure branch exists, and index is up-to-date
 	create
-	updateIndex
+	_ <- updateIndex
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
@@ -178,9 +228,7 @@ update = onceonly $ do
 			showSideAction merge_desc
 			mergeIndex branches
 		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $ 
-			setIndexRef =<< 
-				inRepo (Git.commit merge_desc fullname (nub $ fullname:refs))
+		unless ff $ commitBranch merge_desc (nub $ fullname:refs)
 		invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
@@ -274,13 +322,15 @@ siblingBranches = do
 		uref (a, _) (b, _) = a == b
 
 {- Get the ref of a branch. -}
-getRef :: Git.Ref -> Annex Git.Ref
-getRef branch = firstRef . L.unpack <$> showref
+getRef :: Git.Ref -> Annex (Maybe Git.Ref)
+getRef branch = process . L.unpack <$> showref
 	where
 		showref = inRepo $ Git.pipeRead [Param "show-ref",
 			Param "--hash", -- get the hash
-			Param "--verify", -- only exact match
+			Params "--verify", -- only exact match
 			Param $ show branch]
+		process [] = Nothing
+		process s = Just $ firstRef s
 
 firstRef :: String-> Git.Ref
 firstRef = Git.Ref . takeWhile (/= '\n')
diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index 1d996edfd5..bcf44551e2 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -7,6 +7,7 @@
 
 module Annex.CatFile (
 	catFile,
+	catObject,
 	catFileHandle
 ) where
 
@@ -22,6 +23,11 @@ catFile branch file = do
 	h <- catFileHandle
 	liftIO $ Git.CatFile.catFile h branch file
 
+catObject :: Git.Ref -> Annex L.ByteString
+catObject ref = do
+	h <- catFileHandle
+	liftIO $ Git.CatFile.catObject h ref
+
 catFileHandle :: Annex Git.CatFile.CatFileHandle
 catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle
 	where
diff --git a/doc/bugs/git-annex_branch_push_race.mdwn b/doc/bugs/git-annex_branch_push_race.mdwn
index 257c477bff..013ff70dd5 100644
--- a/doc/bugs/git-annex_branch_push_race.mdwn
+++ b/doc/bugs/git-annex_branch_push_race.mdwn
@@ -40,4 +40,6 @@ redo them.
 will need the same check for the race as the other commits. It won't loop
 forever, I hope.)
 
+> [[done]] and tested.
+
 --[[Joey]]

From 29b88ad657b4d84e84bdc366617d765726e29bb0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 21:40:36 -0400
Subject: [PATCH 2669/8313] avoid redundant call to updateIndex

commitBranch calls updateIndex
---
 Annex/Branch.hs | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 0ac4199944..3da8bb1986 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -203,38 +203,42 @@ commit message = whenM journalDirty $ lockJournal $ do
  - (It would be cleaner to handle the merge by updating the journal, not the
  - index, with changes from the branches.)
  -
- - The index is always updated using a union merge, as that's the most
- - efficient way to update it. However, if the branch can be
- - fast-forwarded, that is then done, rather than adding an unnecessary
- - commit to it.
+ - The branch is fast-forwarded if possible, otherwise a merge commit is
+ - made.
  -}
 update :: Annex ()
 update = onceonly $ do
-	-- ensure branch exists, and index is up-to-date
+	-- ensure branch exists
 	create
-	_ <- updateIndex
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch name . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
-	unless (not dirty && null refs) $ withIndex $ lockJournal $ do
-		when dirty stageJournalFiles
-		let merge_desc = if null branches
-			then "update" 
-			else "merging " ++
-				unwords (map Git.refDescribe branches) ++ 
-				" into " ++ show name
-		unless (null branches) $ do
-			showSideAction merge_desc
-			mergeIndex branches
-		ff <- if dirty then return False else tryFastForwardTo refs
-		unless ff $ commitBranch merge_desc (nub $ fullname:refs)
-		invalidateCache
+	if (not dirty && null refs)
+		then simpleupdate
+		else withIndex $ lockJournal $ do
+			when dirty stageJournalFiles
+			let merge_desc = if null branches
+				then "update" 
+				else "merging " ++
+					unwords (map Git.refDescribe branches) ++ 
+					" into " ++ show name
+			unless (null branches) $ do
+				showSideAction merge_desc
+				mergeIndex branches
+			ff <- if dirty then return False else tryFastForwardTo refs
+			if ff
+				then simpleupdate
+				else commitBranch merge_desc (nub $ fullname:refs)
+			invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
 			disableUpdate
 			return r
+		simpleupdate = do
+			_ <- updateIndex
+			return ()
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}

From acb2d5a5a6878b43ba5fa52b6f4790c765f9f54d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 21:55:51 -0400
Subject: [PATCH 2670/8313] releasing version 3.20111211

---
 debian/changelog | 16 ++++++++--------
 git-annex.cabal  |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 405e98b749..b676c3c3d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,11 @@
-git-annex (3.20111204) UNRELEASED; urgency=low
+git-annex (3.20111211) unstable; urgency=medium
 
+  * Fix bug in last version in getting contents from bare repositories.
+  * Ensure that git-annex branch changes are merged into git-annex's index,
+    which fixes a bug that could cause changes that were pushed to the
+    git-annex branch to get reverted. As a side effect, it's now safe
+    for users to check out and commit changes directly to the git-annex
+    branch.
   * map: Fix a failure to detect a loop when both repositories are local
     and refer to each other with relative paths.
   * Prevent key names from containing newlines.
@@ -12,14 +18,8 @@ git-annex (3.20111204) UNRELEASED; urgency=low
   * sync: New command that synchronises the local repository and default
     remote, by running git commit, pull, and push for you.
   * Version monad-control dependency in cabal file.
-  * Fix bug in last version in getting contents from bare repositories.
-  * Ensure that git-annex branch changes are merged into git-annex's index,
-    which fixes a bug that could cause changes that were pushed to the
-    git-annex branch to get reverted. As a side effect, it's now safe
-    for users to check out and commit changes directly to the git-annex
-    branch.
 
- -- Joey Hess   Sun, 04 Dec 2011 12:22:37 -0400
+ -- Joey Hess   Sun, 11 Dec 2011 21:24:39 -0400
 
 git-annex (3.20111203) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 8f5d6ebb86..35b3e690f9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111203
+Version: 3.20111211
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From d3d9c8a9a68fe74208d08ca4e59670934891d489 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 21:56:17 -0400
Subject: [PATCH 2671/8313] add news item for git-annex 3.20111211

---
 doc/news/version_3.20111105.mdwn | 27 ---------------------------
 doc/news/version_3.20111211.mdwn | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 27 deletions(-)
 delete mode 100644 doc/news/version_3.20111105.mdwn
 create mode 100644 doc/news/version_3.20111211.mdwn

diff --git a/doc/news/version_3.20111105.mdwn b/doc/news/version_3.20111105.mdwn
deleted file mode 100644
index 663c2ef1fc..0000000000
--- a/doc/news/version_3.20111105.mdwn
+++ /dev/null
@@ -1,27 +0,0 @@
-git-annex 3.20111105 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * The default backend used when adding files to the annex is changed
-     from WORM to SHA256.
-     To get old behavior, add a .gitattributes containing: * annex.backend=WORM
-   * Sped up some operations on remotes that are on the same host.
-   * copy --to: Fixed leak when copying many files to a remote on the same
-     host.
-   * uninit: Add guard against being run with the git-annex branch checked out.
-   * Fail if --from or --to is passed to commands that do not support them.
-   * drop --from is now supported to remove file content from a remote.
-   * status: Now always shows the current repository, even when it does not
-     appear in uuid.log.
-   * fsck: Now works in bare repositories. Checks location log information,
-     and file contents. Does not check that numcopies is satisfied, as
-     .gitattributes information about numcopies is not available in a bare
-     repository.
-   * unused, dropunused: Now work in bare repositories.
-   * Removed the setkey command, and added a reinject command with a more
-     useful interface.
-   * The fromkey command now takes the key as its first parameter. The --key
-     option is no longer used.
-   * Built without any filename containing .git being excluded. Closes: #[647215](http://bugs.debian.org/647215)
-   * Record uuid when auto-initializing a remote so it shows in status.
-   * Bugfix: Fixed git-annex init crash in a bare repository when there was
-     already an existing git-annex branch.
-   * Pass -t to rsync to preserve timestamps."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111211.mdwn b/doc/news/version_3.20111211.mdwn
new file mode 100644
index 0000000000..5d2c57e455
--- /dev/null
+++ b/doc/news/version_3.20111211.mdwn
@@ -0,0 +1,20 @@
+git-annex 3.20111211 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix bug in last version in getting contents from bare repositories.
+   * Ensure that git-annex branch changes are merged into git-annex's index,
+     which fixes a bug that could cause changes that were pushed to the
+     git-annex branch to get reverted. As a side effect, it's now safe
+     for users to check out and commit changes directly to the git-annex
+     branch.
+   * map: Fix a failure to detect a loop when both repositories are local
+     and refer to each other with relative paths.
+   * Prevent key names from containing newlines.
+   * add: If interrupted, add can leave files converted to symlinks but not
+     yet added to git. Running the add again will now clean up this situtation.
+   * Fix caching of decrypted ciphers, which failed when drop had to check
+     multiple different encrypted special remotes.
+   * unannex: Can be run on files that have been added to the annex, but not
+     yet committed.
+   * sync: New command that synchronises the local repository and default
+     remote, by running git commit, pull, and push for you.
+   * Version monad-control dependency in cabal file."""]]
\ No newline at end of file

From b9ac5854549636493449fea6830364a01159fbf6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Dec 2011 23:02:25 -0400
Subject: [PATCH 2672/8313] more efficient union merges

Tries to avoid generating a new object when the merged content has the same
lines that were in the old object.

I've noticed some merge commits that only move lines around, like this:

- 1323478057.181191s 1 be23c3ac-0ee5-11e0-b185-3b0f9b5b00c5
  1323204972.062151s 1 87e06c7a-7388-11e0-ba07-03cdf300bd87
++1323478057.181191s 1 be23c3ac-0ee5-11e0-b185-3b0f9b5b00c5

Unsure if this will really save anything in practice, since it only looks
at one of the two old objects, and maybe I didn't pick the best one.
---
 Git/UnionMerge.hs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index ddbff6a82f..27113c85ac 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -104,14 +104,17 @@ mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
-	shas -> do
-		content <- L.concat <$> mapM (catObject h) shas
-		sha <- hashObject (unionmerge content) repo
-		return $ Just $ update_index_line sha file
+	(sha:shas) -> do
+		origcontent <- L.lines <$> catObject h sha
+		content <- map L.lines <$> mapM (catObject h) shas
+		let newcontent = nub $ concat $ origcontent:content
+		newsha <- if (newcontent == origcontent)
+			then return sha
+			else hashObject (L.unlines $ newcontent) repo
+		return $ Just $ update_index_line newsha file
 	where
-		[_colonamode, _bmode, asha, bsha, _status] = words info
+		[_colonmode, _bmode, asha, bsha, _status] = words info
 		nullsha = Ref $ replicate shaSize '0'
-		unionmerge = L.unlines . nub . L.lines
 
 {- Injects some content into git, returning its Sha. -}
 hashObject :: L.ByteString -> Repo -> IO Sha

From 0cbab5de657e025057dd10b087a874d6b3a7b13e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 00:48:25 -0400
Subject: [PATCH 2673/8313] refactor

---
 Git/UnionMerge.hs | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 27113c85ac..89fcf83e02 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -105,20 +105,24 @@ mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> return $ Just $ update_index_line sha file
 	(sha:shas) -> do
-		origcontent <- L.lines <$> catObject h sha
-		content <- map L.lines <$> mapM (catObject h) shas
-		let newcontent = nub $ concat $ origcontent:content
-		newsha <- if (newcontent == origcontent)
-			then return sha
-			else hashObject (L.unlines $ newcontent) repo
+		newsha <- maybe (return sha) (hashObject repo . L.unlines) =<<
+			unionmerge
+				<$> (L.lines <$> catObject h sha)
+				<*> (map L.lines <$> mapM (catObject h) shas)
 		return $ Just $ update_index_line newsha file
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		nullsha = Ref $ replicate shaSize '0'
 
+		unionmerge origcontent content
+			| newcontent == origcontent = Nothing
+			| otherwise = Just newcontent
+			where
+				newcontent = nub $ concat $ origcontent:content
+
 {- Injects some content into git, returning its Sha. -}
-hashObject :: L.ByteString -> Repo -> IO Sha
-hashObject content repo = getSha subcmd $ do
+hashObject :: Repo -> L.ByteString -> IO Sha
+hashObject repo content = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
 		forceSuccess h

From acd7a52dfd2cad24fd946ffcf8c4b1d07eb474ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 01:33:02 -0400
Subject: [PATCH 2674/8313] always find optimal merge

Testing b9ac5854549636493449fea6830364a01159fbf6, it didn't find the
optimal union merge, the second sha was the one to use, at least in
the case I tried. Let's just try all shas to see if any can be reused.

I stopped using the expensive nub, so despite the use of sets to
sort/uniq file contents, this is probably as fast or faster than it
was before.
---
 Git/UnionMerge.hs | 34 ++++++++++++++++++++--------------
 debian/changelog  |  6 ++++++
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 89fcf83e02..0345af3994 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -15,8 +15,8 @@ module Git.UnionMerge (
 ) where
 
 import System.Cmd.Utils
-import Data.List
 import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Set as S
 
 import Common
 import Git
@@ -103,22 +103,14 @@ calc_merge ch differ repo streamer = gendiff >>= go
 mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
-	(sha:[]) -> return $ Just $ update_index_line sha file
-	(sha:shas) -> do
-		newsha <- maybe (return sha) (hashObject repo . L.unlines) =<<
-			unionmerge
-				<$> (L.lines <$> catObject h sha)
-				<*> (map L.lines <$> mapM (catObject h) shas)
-		return $ Just $ update_index_line newsha file
+	(sha:[]) -> use sha
+	shas -> use =<< either return (hashObject repo . L.unlines) =<<
+		calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		nullsha = Ref $ replicate shaSize '0'
-
-		unionmerge origcontent content
-			| newcontent == origcontent = Nothing
-			| otherwise = Just newcontent
-			where
-				newcontent = nub $ concat $ origcontent:content
+		getcontents s = L.lines <$> catObject h s
+		use sha = return $ Just $ update_index_line sha file
 
 {- Injects some content into git, returning its Sha. -}
 hashObject :: Repo -> L.ByteString -> IO Sha
@@ -131,3 +123,17 @@ hashObject repo content = getSha subcmd $ do
 	where
 		subcmd = "hash-object"
 		params = [subcmd, "-w", "--stdin"]
+
+{- Calculates a union merge between a list of refs, with contents.
+ -
+ - When possible, reuses the content of an existing ref, rather than
+ - generating new content.
+ -}
+calcMerge :: [(Ref, [L.ByteString])] -> Either Ref [L.ByteString]
+calcMerge shacontents
+	| null reuseable = Right $ new
+	| otherwise = Left $ fst $ head reuseable
+	where
+		reuseable = filter (\c -> sorteduniq (snd c) == new) shacontents
+		new = sorteduniq $ concat $ map snd shacontents
+		sorteduniq = S.toList . S.fromList
diff --git a/debian/changelog b/debian/changelog
index b676c3c3d3..db23decbb9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20111212) UNRELEASED; urgency=low
+
+  * Union merge now finds the least expensive way to represent the merge.
+
+ -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
+
 git-annex (3.20111211) unstable; urgency=medium
 
   * Fix bug in last version in getting contents from bare repositories.

From 2332afb4bc207ab04b4a1d7adfb3a2df7f5d7874 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 02:04:48 -0400
Subject: [PATCH 2675/8313] cleanup

---
 Annex/Branch.hs | 28 +++++++++++++---------------
 Utility/Misc.hs |  4 ++++
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 3da8bb1986..b108281ce7 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -88,7 +88,8 @@ updateIndex = do
 		go Nothing = return ()
 		go (Just branchref) = do
 			lock <- fromRepo gitAnnexIndexLock
-			lockref <- firstRef <$> liftIO (catchDefaultIO (readFileStrict lock) "")
+			lockref <- Git.Ref . firstLine <$>
+				liftIO (catchDefaultIO (readFileStrict lock) "")
 			when (lockref /= branchref) $ do
 				withIndex $ mergeIndex [fullname]
 				setIndexRef branchref
@@ -303,6 +304,17 @@ refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
 	[Param "--verify", Param "-q", Param $ show ref]
 
+{- Get the ref of a branch. -}
+getRef :: Git.Branch -> Annex (Maybe Git.Ref)
+getRef branch = process . L.unpack <$> showref
+	where
+		showref = inRepo $ Git.pipeRead [Param "show-ref",
+			Param "--hash", -- get the hash
+			Params "--verify", -- only exact match
+			Param $ show branch]
+		process [] = Nothing
+		process s = Just $ Git.Ref $ firstLine s
+
 {- Does the main git-annex branch exist? -}
 hasBranch :: Annex Bool
 hasBranch = refExists fullname
@@ -325,20 +337,6 @@ siblingBranches = do
 		gen l = (Git.Ref $ head l, Git.Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
-{- Get the ref of a branch. -}
-getRef :: Git.Ref -> Annex (Maybe Git.Ref)
-getRef branch = process . L.unpack <$> showref
-	where
-		showref = inRepo $ Git.pipeRead [Param "show-ref",
-			Param "--hash", -- get the hash
-			Params "--verify", -- only exact match
-			Param $ show branch]
-		process [] = Nothing
-		process s = Just $ firstRef s
-
-firstRef :: String-> Git.Ref
-firstRef = Git.Ref . takeWhile (/= '\n')
-
 {- Applies a function to modifiy the content of a file.
  -
  - Note that this does not cause the branch to be merged, it only
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 541e150b77..e95ac4319d 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -40,6 +40,10 @@ separate c l = unbreak $ break c l
 			| null b = r
 			| otherwise = (a, tail b)
 
+{- Breaks out the first line. -}
+firstLine :: String-> String
+firstLine = takeWhile (/= '\n')
+
 {- Catches IO errors and returns a Bool -}
 catchBoolIO :: IO Bool -> IO Bool
 catchBoolIO a = catchDefaultIO a False

From c7e65bbb12db3db314d809e0186db0905c677d2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 02:24:37 -0400
Subject: [PATCH 2676/8313] optimiation

avoids reading the config of a local remote twice in a row
---
 Remote/Git.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 05743a28d5..0251da5583 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -86,9 +86,9 @@ tryGitConfigRead r
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely geturlconfig
 	| Git.repoIsUrl r = return r
-	| otherwise = store $ safely $ do
-		onLocal r ensureInitialized
-		Git.configRead r
+	| otherwise = store $ safely $ onLocal r $ do 
+		ensureInitialized
+		Annex.getState Annex.repo
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.

From f9cd3f6ad18ca04a1da7082c6c10215dc5435154 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 02:31:07 -0400
Subject: [PATCH 2677/8313] optimisation

avoids a useless diff from git-annex..refs/heads/git-annex
---
 Annex/Branch.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index b108281ce7..e6c92fbe54 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -213,7 +213,7 @@ update = onceonly $ do
 	create
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
-	c <- filterM (changedBranch name . snd) =<< siblingBranches
+	c <- filterM (changedBranch fullname . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
 	if (not dirty && null refs)
 		then simpleupdate
@@ -244,7 +244,9 @@ update = onceonly $ do
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
 changedBranch :: Git.Branch -> Git.Branch -> Annex Bool
-changedBranch origbranch newbranch = not . L.null <$> diffs
+changedBranch origbranch newbranch
+	| origbranch == newbranch = return False
+	| otherwise = not . L.null <$> diffs
 	where
 		diffs = inRepo $ Git.pipeRead
 			[ Param "log"

From 79345ad5fc10fc96dcd3599f2e092b3967291549 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 03:30:47 -0400
Subject: [PATCH 2678/8313] optimisation

avoids a redundant call to git show-ref
---
 Annex/Branch.hs | 79 +++++++++++++++++++++++--------------------------
 1 file changed, 37 insertions(+), 42 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index e6c92fbe54..c657525b1d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -79,20 +79,14 @@ withIndex' bootstrapping a = do
  - Compares the ref stored in the lock file with the current
  - ref of the branch to see if an update is needed.
  -}
-updateIndex :: Annex (Maybe Git.Ref)
-updateIndex = do
-	branchref <- getRef fullname
-	go branchref
-	return branchref
-	where
-		go Nothing = return ()
-		go (Just branchref) = do
-			lock <- fromRepo gitAnnexIndexLock
-			lockref <- Git.Ref . firstLine <$>
-				liftIO (catchDefaultIO (readFileStrict lock) "")
-			when (lockref /= branchref) $ do
-				withIndex $ mergeIndex [fullname]
-				setIndexRef branchref
+updateIndex :: Git.Ref -> Annex ()
+updateIndex branchref = do
+	lock <- fromRepo gitAnnexIndexLock
+	lockref <- Git.Ref . firstLine <$>
+		liftIO (catchDefaultIO (readFileStrict lock) "")
+	when (lockref /= branchref) $ do
+		withIndex $ mergeIndex [fullname]
+		setIndexRef branchref
 
 {- Record that the branch's index has been updated to correspond to a
  - given ref of the branch. -}
@@ -115,13 +109,13 @@ setIndexRef ref = do
  - change is reverted. This race is detected and another commit made
  - to fix it.
  -}
-commitBranch :: String -> [Git.Ref] -> Annex ()
-commitBranch message parents = do
-	expected <- updateIndex
+commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
+commitBranch branchref message parents = do
+	updateIndex branchref
 	committedref <- inRepo $ Git.commit message fullname parents
 	setIndexRef committedref
 	parentrefs <- commitparents <$> catObject committedref
-	when (racedetected expected parentrefs) $
+	when (racedetected branchref parentrefs) $
 		fixrace committedref parentrefs
 	where
 		-- look for "parent ref" lines and return the refs
@@ -133,10 +127,7 @@ commitBranch message parents = do
  		{- The race can be detected by checking the commit's
 		 - parent, which will be the newly pushed branch,
 		 - instead of the expected ref that the index was updated to. -}
-		racedetected Nothing parentrefs
-			| null parentrefs = False -- first commit, no parents
-			| otherwise = True -- race on first commit
-		racedetected (Just expectedref) parentrefs
+		racedetected expectedref parentrefs
 			| expectedref `elem` parentrefs = False -- good parent
 			| otherwise = True -- race!
 		
@@ -144,7 +135,7 @@ commitBranch message parents = do
 		 - into the index, and recommit on top of the bad commit. -}
 		fixrace committedref lostrefs = do
 			mergeIndex lostrefs
-			commitBranch racemessage [committedref]
+			commitBranch committedref racemessage [committedref]
 		
 		racemessage = message ++ " (recovery from race)"
 
@@ -179,20 +170,31 @@ getCache file = getState >>= go
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = unlessM hasBranch $ hasOrigin >>= go
+create = do
+	_ <- getBranch
+	return ()
+
+{- Returns the ref of the branch, creating it first if necessary. -}
+getBranch :: Annex (Git.Ref)
+getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< getRef fullname
 	where
 		go True = do
 			inRepo $ Git.run "branch"
 				[Param $ show name, Param $ show originname]
-			maybe (return ()) setIndexRef =<< getRef fullname
-		go False = withIndex' True $ 
-			setIndexRef =<< (inRepo $ Git.commit "branch created" fullname [])
+			fromMaybe (error $ "failed to create " ++ show name)
+				<$> getRef fullname
+		go False = withIndex' True $ do
+			inRepo $ Git.commit "branch created" fullname []
+		use ref = do
+			setIndexRef ref
+			return ref
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
 	stageJournalFiles
-	withIndex $ commitBranch message [fullname]
+	ref <- getBranch
+	withIndex $ commitBranch ref message [fullname]
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
@@ -209,14 +211,14 @@ commit message = whenM journalDirty $ lockJournal $ do
  -}
 update :: Annex ()
 update = onceonly $ do
-	-- ensure branch exists
-	create
+	-- ensure branch exists, and get its current ref
+	branchref <- getBranch
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	c <- filterM (changedBranch fullname . snd) =<< siblingBranches
 	let (refs, branches) = unzip c
 	if (not dirty && null refs)
-		then simpleupdate
+		then updateIndex branchref
 		else withIndex $ lockJournal $ do
 			when dirty stageJournalFiles
 			let merge_desc = if null branches
@@ -229,17 +231,15 @@ update = onceonly $ do
 				mergeIndex branches
 			ff <- if dirty then return False else tryFastForwardTo refs
 			if ff
-				then simpleupdate
-				else commitBranch merge_desc (nub $ fullname:refs)
+				then updateIndex branchref
+				else commitBranch branchref merge_desc
+					(nub $ fullname:refs)
 			invalidateCache
 	where
 		onceonly a = unlessM (branchUpdated <$> getState) $ do
 			r <- a
 			disableUpdate
 			return r
-		simpleupdate = do
-			_ <- updateIndex
-			return ()
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
@@ -306,21 +306,16 @@ refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
 	[Param "--verify", Param "-q", Param $ show ref]
 
-{- Get the ref of a branch. -}
+{- Get the ref of a branch. (Must be a fully qualified branch name) -}
 getRef :: Git.Branch -> Annex (Maybe Git.Ref)
 getRef branch = process . L.unpack <$> showref
 	where
 		showref = inRepo $ Git.pipeRead [Param "show-ref",
 			Param "--hash", -- get the hash
-			Params "--verify", -- only exact match
 			Param $ show branch]
 		process [] = Nothing
 		process s = Just $ Git.Ref $ firstLine s
 
-{- Does the main git-annex branch exist? -}
-hasBranch :: Annex Bool
-hasBranch = refExists fullname
-
 {- Does origin/git-annex exist? -}
 hasOrigin :: Annex Bool
 hasOrigin = refExists originname

From 6edaabd0407287becda24904ed19413a5371979c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 13:43:52 -0400
Subject: [PATCH 2679/8313] reinject: Add a sanity check for using an annexed
 file as the source file.

---
 Command/Reinject.hs | 5 +++--
 debian/changelog    | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index cfa0655ef1..906f7c5176 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -24,8 +24,9 @@ start :: [FilePath] -> CommandStart
 start (src:dest:[])
 	| src == dest = stop
 	| otherwise = do
-		showStart "reinject" dest
-		next $ whenAnnexed (perform src) dest
+		ifAnnexed src
+			(error $ "cannot used annexed file as src: " ++ src)
+			(next $ whenAnnexed (perform src) dest)
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform
diff --git a/debian/changelog b/debian/changelog
index db23decbb9..5fc01f9ead 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20111212) UNRELEASED; urgency=low
 
   * Union merge now finds the least expensive way to represent the merge.
+  * reinject: Add a sanity check for using an annexed file as the source file.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From b2f934e07ad32cfe44ea1fab2ca154379f30efe8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 17:24:12 -0400
Subject: [PATCH 2680/8313] update comment

---
 Annex/Branch.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c657525b1d..b79a8975d0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -97,8 +97,8 @@ setIndexRef ref = do
 
 {- Commits the staged changes in the index to the branch.
  - 
- - Ensures that the branch's index file is first updated to include the
- - current state of the branch, before running the commit action. This
+ - Ensures that the branch's index file is first updated to the state
+ - of the brannch at branchref, before running the commit action. This
  - is needed because the branch may have had changes pushed to it, that
  - are not yet reflected in the index.
  -
@@ -108,6 +108,10 @@ setIndexRef ref = do
  - being updated to include it. The result is that the newly pushed
  - change is reverted. This race is detected and another commit made
  - to fix it.
+ - 
+ - The branchref value can have been obtained using getBranch at any
+ - previous point, though getting it a long time ago makes the race
+ - more likely to occur.
  -}
 commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
 commitBranch branchref message parents = do

From 98dfc0c9b0024c156d0fea99bf8d2355e06244a7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 17:38:46 -0400
Subject: [PATCH 2681/8313] split out Annex/BranchState.hs

---
 Annex/Branch.hs      | 45 ++---------------------------------
 Annex/BranchState.hs | 56 ++++++++++++++++++++++++++++++++++++++++++++
 Remote/Git.hs        |  4 ++--
 3 files changed, 60 insertions(+), 45 deletions(-)
 create mode 100644 Annex/BranchState.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index b79a8975d0..699bc0323c 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -8,7 +8,6 @@
 module Annex.Branch (
 	create,
 	update,
-	disableUpdate,
 	get,
 	change,
 	commit,
@@ -25,7 +24,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
 import Annex.Exception
-import Types.BranchState
+import Annex.BranchState
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Annex
@@ -148,30 +147,6 @@ commitBranch branchref message parents = do
 withIndexUpdate :: Annex a -> Annex a
 withIndexUpdate a = update >> withIndex a
 
-getState :: Annex BranchState
-getState = Annex.getState Annex.branchstate
-
-setState :: BranchState -> Annex ()
-setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
-
-setCache :: FilePath -> String -> Annex ()
-setCache file content = do
-	state <- getState
-	setState state { cachedFile = Just file, cachedContent = content }
-
-invalidateCache :: Annex ()
-invalidateCache = do
-	state <- getState
-	setState state { cachedFile = Nothing, cachedContent = "" }
-
-getCache :: FilePath -> Annex (Maybe String)
-getCache file = getState >>= go
-	where
-		go state
-			| cachedFile state == Just file =
-				return $ Just $ cachedContent state
-			| otherwise = return Nothing
-
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
 create = do
@@ -214,7 +189,7 @@ commit message = whenM journalDirty $ lockJournal $ do
  - made.
  -}
 update :: Annex ()
-update = onceonly $ do
+update = runUpdateOnce $ do
 	-- ensure branch exists, and get its current ref
 	branchref <- getBranch
 	-- check what needs updating before taking the lock
@@ -239,11 +214,6 @@ update = onceonly $ do
 				else commitBranch branchref merge_desc
 					(nub $ fullname:refs)
 			invalidateCache
-	where
-		onceonly a = unlessM (branchUpdated <$> getState) $ do
-			r <- a
-			disableUpdate
-			return r
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
@@ -294,17 +264,6 @@ tryFastForwardTo (first:rest) = do
 				(False, True) -> findbest c rs -- worse
 				(False, False) -> findbest c rs -- same
 			
-{- Avoids updating the branch. A useful optimisation when the branch
- - is known to have not changed, or git-annex won't be relying on info
- - from it. -}
-disableUpdate :: Annex ()
-disableUpdate = Annex.changeState setupdated
-	where
-		setupdated s = s { Annex.branchstate = new }
-			where
-				new = old { branchUpdated = True }
-				old = Annex.branchstate s
-
 {- Checks if a git ref exists. -}
 refExists :: Git.Ref -> Annex Bool
 refExists ref = inRepo $ Git.runBool "show-ref"
diff --git a/Annex/BranchState.hs b/Annex/BranchState.hs
new file mode 100644
index 0000000000..0950e9a967
--- /dev/null
+++ b/Annex/BranchState.hs
@@ -0,0 +1,56 @@
+{- git-annex branch state management
+ -
+ - Runtime state about the git-annex branch, including a small read cache.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.BranchState where
+
+import Common.Annex
+import Types.BranchState
+import qualified Annex
+
+getState :: Annex BranchState
+getState = Annex.getState Annex.branchstate
+
+setState :: BranchState -> Annex ()
+setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
+
+setCache :: FilePath -> String -> Annex ()
+setCache file content = do
+	state <- getState
+	setState state { cachedFile = Just file, cachedContent = content }
+
+getCache :: FilePath -> Annex (Maybe String)
+getCache file = getState >>= go
+	where
+		go state
+			| cachedFile state == Just file =
+				return $ Just $ cachedContent state
+			| otherwise = return Nothing
+
+invalidateCache :: Annex ()
+invalidateCache = do
+	state <- getState
+	setState state { cachedFile = Nothing, cachedContent = "" }
+
+{- Runs an action to update the branch, if it's not been updated before
+ - in this run of git-annex. -}
+runUpdateOnce :: Annex () -> Annex ()
+runUpdateOnce a = unlessM (branchUpdated <$> getState) $ do
+	a
+	disableUpdate
+
+{- Avoids updating the branch. A useful optimisation when the branch
+ - is known to have not changed, or git-annex won't be relying on info
+ - from it. -}
+disableUpdate :: Annex ()
+disableUpdate = Annex.changeState setupdated
+	where
+		setupdated s = s { Annex.branchstate = new }
+			where
+				new = old { branchUpdated = True }
+				old = Annex.branchstate s
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 0251da5583..2f9288e1b4 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -19,7 +19,7 @@ import qualified Git
 import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
-import qualified Annex.Branch
+import qualified Annex.BranchState
 import qualified Utility.Url as Url
 import Utility.TempFile
 import Config
@@ -171,7 +171,7 @@ onLocal r a = do
 	Annex.eval state $ do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
-		Annex.Branch.disableUpdate
+		Annex.BranchState.disableUpdate
 		ret <- a
 		liftIO Git.reap
 		return ret

From da95cbadca5b7ef3058b91a384d5f3a48cc39039 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 18:03:28 -0400
Subject: [PATCH 2682/8313] split out Annex/Journal.hs

---
 Annex/Branch.hs  | 92 ++++-------------------------------------------
 Annex/Journal.hs | 94 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+), 85 deletions(-)
 create mode 100644 Annex/Journal.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 699bc0323c..1dac8ef793 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -18,16 +18,15 @@ module Annex.Branch (
 	name	
 ) where
 
-import System.IO.Binary
 import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
 import Annex.Exception
 import Annex.BranchState
+import Annex.Journal
 import qualified Git
 import qualified Git.UnionMerge
-import qualified Annex
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -171,7 +170,7 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< getRef fullname
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
 commit message = whenM journalDirty $ lockJournal $ do
-	stageJournalFiles
+	stageJournal
 	ref <- getBranch
 	withIndex $ commitBranch ref message [fullname]
 
@@ -199,7 +198,7 @@ update = runUpdateOnce $ do
 	if (not dirty && null refs)
 		then updateIndex branchref
 		else withIndex $ lockJournal $ do
-			when dirty stageJournalFiles
+			when dirty stageJournal
 			let merge_desc = if null branches
 				then "update" 
 				else "merging " ++
@@ -305,7 +304,7 @@ siblingBranches = do
 change :: FilePath -> (String -> String) -> Annex ()
 change file a = lockJournal $ getStale file >>= return . a >>= set file
 
-{- Records new content of a file into the journal. -}
+{- Records new content of a file into the journal and cache. -}
 set :: FilePath -> String -> Annex ()
 set file content = do
 	setJournalFile file content
@@ -346,44 +345,9 @@ files = withIndexUpdate $ do
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
-{- Records content for a file in the branch to the journal.
- -
- - Using the journal, rather than immediatly staging content to the index
- - avoids git needing to rewrite the index after every change. -}
-setJournalFile :: FilePath -> String -> Annex ()
-setJournalFile file content = do
-	g <- gitRepo
-	liftIO $ doRedo (write g) $ do
-		createDirectoryIfMissing True $ gitAnnexJournalDir g
-		createDirectoryIfMissing True $ gitAnnexTmpDir g
-	where
-		-- journal file is written atomically
-		write g = do
-			let jfile = journalFile g file
-			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
-			writeBinaryFile tmpfile content
-			moveFile tmpfile jfile
-
-{- Gets any journalled content for a file in the branch. -}
-getJournalFile :: FilePath -> Annex (Maybe String)
-getJournalFile file = inRepo $ \g -> catchMaybeIO $
-	readFileStrict $ journalFile g file
-
-{- List of files that have updated content in the journal. -}
-getJournalledFiles :: Annex [FilePath]
-getJournalledFiles = map fileJournal <$> getJournalFiles
-
-{- List of existing journal files. -}
-getJournalFiles :: Annex [FilePath]
-getJournalFiles = do
-	g <- gitRepo
-	fs <- liftIO $
-		catchDefaultIO (getDirectoryContents $ gitAnnexJournalDir g) []
-	return $ filter (`notElem` [".", ".."]) fs
-
-{- Stages the specified journalfiles. -}
-stageJournalFiles :: Annex ()
-stageJournalFiles = do
+{- Stages the journal into the index. -}
+stageJournal :: Annex ()
+stageJournal = do
 	fs <- getJournalFiles
 	g <- gitRepo
 	withIndex $ liftIO $ do
@@ -409,45 +373,3 @@ stageJournalFiles = do
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
 		git_hash_object = Git.gitCommandLine
 			[Param "hash-object", Param "-w", Param "--stdin-paths"]
-
-
-{- Checks if there are changes in the journal. -}
-journalDirty :: Annex Bool
-journalDirty = not . null <$> getJournalFiles
-
-{- Produces a filename to use in the journal for a file on the branch.
- -
- - The journal typically won't have a lot of files in it, so the hashing
- - used in the branch is not necessary, and all the files are put directly
- - in the journal directory.
- -}
-journalFile :: Git.Repo -> FilePath -> FilePath
-journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
-	where
-		mangle '/' = "_"
-		mangle '_' = "__"
-		mangle c = [c]
-
-{- Converts a journal file (relative to the journal dir) back to the
- - filename on the branch. -}
-fileJournal :: FilePath -> FilePath
-fileJournal = replace "//" "_" . replace "_" "/"
-
-{- Runs an action that modifies the journal, using locking to avoid
- - contention with other git-annex processes. -}
-lockJournal :: Annex a -> Annex a
-lockJournal a = do
-	file <- fromRepo gitAnnexJournalLock
-	bracketIO (lock file) unlock a
-	where
-		lock file = do
-			l <- doRedo (createFile file stdFileMode) $
-				createDirectoryIfMissing True $ takeDirectory file
-			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
-			return l
-		unlock = closeFd
-
-{- Runs an action, catching failure and running something to fix it up, and
- - retrying if necessary. -}
-doRedo :: IO a -> IO b -> IO a
-doRedo a b = catch a $ const $ b >> a
diff --git a/Annex/Journal.hs b/Annex/Journal.hs
new file mode 100644
index 0000000000..9c5be89b19
--- /dev/null
+++ b/Annex/Journal.hs
@@ -0,0 +1,94 @@
+{- management of the git-annex journal and cache
+ -
+ - The journal is used to queue up changes before they are committed to the
+ - git-annex branch. Amoung other things, it ensures that if git-annex is
+ - interrupted, its recorded data is not lost.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.Journal where
+
+import System.IO.Binary
+
+import Common.Annex
+import Annex.Exception
+import qualified Git
+
+{- Records content for a file in the branch to the journal.
+ -
+ - Using the journal, rather than immediatly staging content to the index
+ - avoids git needing to rewrite the index after every change. -}
+setJournalFile :: FilePath -> String -> Annex ()
+setJournalFile file content = do
+	g <- gitRepo
+	liftIO $ doRedo (write g) $ do
+		createDirectoryIfMissing True $ gitAnnexJournalDir g
+		createDirectoryIfMissing True $ gitAnnexTmpDir g
+	where
+		-- journal file is written atomically
+		write g = do
+			let jfile = journalFile g file
+			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
+			writeBinaryFile tmpfile content
+			moveFile tmpfile jfile
+
+{- Gets any journalled content for a file in the branch. -}
+getJournalFile :: FilePath -> Annex (Maybe String)
+getJournalFile file = inRepo $ \g -> catchMaybeIO $
+	readFileStrict $ journalFile g file
+
+{- List of files that have updated content in the journal. -}
+getJournalledFiles :: Annex [FilePath]
+getJournalledFiles = map fileJournal <$> getJournalFiles
+
+{- List of existing journal files. -}
+getJournalFiles :: Annex [FilePath]
+getJournalFiles = do
+	g <- gitRepo
+	fs <- liftIO $
+		catchDefaultIO (getDirectoryContents $ gitAnnexJournalDir g) []
+	return $ filter (`notElem` [".", ".."]) fs
+
+{- Checks if there are changes in the journal. -}
+journalDirty :: Annex Bool
+journalDirty = not . null <$> getJournalFiles
+
+{- Produces a filename to use in the journal for a file on the branch.
+ -
+ - The journal typically won't have a lot of files in it, so the hashing
+ - used in the branch is not necessary, and all the files are put directly
+ - in the journal directory.
+ -}
+journalFile :: Git.Repo -> FilePath -> FilePath
+journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
+	where
+		mangle '/' = "_"
+		mangle '_' = "__"
+		mangle c = [c]
+
+{- Converts a journal file (relative to the journal dir) back to the
+ - filename on the branch. -}
+fileJournal :: FilePath -> FilePath
+fileJournal = replace "//" "_" . replace "_" "/"
+
+{- Runs an action that modifies the journal, using locking to avoid
+ - contention with other git-annex processes. -}
+lockJournal :: Annex a -> Annex a
+lockJournal a = do
+	file <- fromRepo gitAnnexJournalLock
+	bracketIO (lock file) unlock a
+	where
+		lock file = do
+			l <- doRedo (createFile file stdFileMode) $
+				createDirectoryIfMissing True $ takeDirectory file
+			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
+			return l
+		unlock = closeFd
+
+{- Runs an action, catching failure and running something to fix it up, and
+ - retrying if necessary. -}
+doRedo :: IO a -> IO b -> IO a
+doRedo a b = catch a $ const $ b >> a

From 543d0d250104c1f5908e1b7b258d36d95488a029 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 18:23:24 -0400
Subject: [PATCH 2683/8313] split out Git/Ref.hs

---
 Annex/Branch.hs   | 57 ++++++++++++++++-------------------------------
 Command/Unused.hs |  3 ++-
 Git.hs            | 17 ++++----------
 Git/Ref.hs        | 47 ++++++++++++++++++++++++++++++++++++++
 Init.hs           |  2 +-
 Upgrade/V2.hs     |  3 ++-
 6 files changed, 75 insertions(+), 54 deletions(-)
 create mode 100644 Git/Ref.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 1dac8ef793..c8a538acd0 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -12,10 +12,9 @@ module Annex.Branch (
 	change,
 	commit,
 	files,
-	refExists,
+	name,
 	hasOrigin,
-	hasSomeBranch,
-	name	
+	hasSibling,
 ) where
 
 import System.Exit
@@ -27,6 +26,7 @@ import Annex.BranchState
 import Annex.Journal
 import qualified Git
 import qualified Git.UnionMerge
+import qualified Git.Ref
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -84,12 +84,12 @@ updateIndex branchref = do
 		liftIO (catchDefaultIO (readFileStrict lock) "")
 	when (lockref /= branchref) $ do
 		withIndex $ mergeIndex [fullname]
-		setIndexRef branchref
+		setIndexSha branchref
 
 {- Record that the branch's index has been updated to correspond to a
  - given ref of the branch. -}
-setIndexRef :: Git.Ref -> Annex ()
-setIndexRef ref = do
+setIndexSha :: Git.Ref -> Annex ()
+setIndexSha ref = do
         lock <- fromRepo gitAnnexIndexLock
 	liftIO $ writeFile lock $ show ref ++ "\n"
 
@@ -115,7 +115,7 @@ commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
 commitBranch branchref message parents = do
 	updateIndex branchref
 	committedref <- inRepo $ Git.commit message fullname parents
-	setIndexRef committedref
+	setIndexSha committedref
 	parentrefs <- commitparents <$> catObject committedref
 	when (racedetected branchref parentrefs) $
 		fixrace committedref parentrefs
@@ -154,18 +154,19 @@ create = do
 
 {- Returns the ref of the branch, creating it first if necessary. -}
 getBranch :: Annex (Git.Ref)
-getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< getRef fullname
+getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 	where
 		go True = do
 			inRepo $ Git.run "branch"
 				[Param $ show name, Param $ show originname]
 			fromMaybe (error $ "failed to create " ++ show name)
-				<$> getRef fullname
+				<$> branchsha
 		go False = withIndex' True $ do
 			inRepo $ Git.commit "branch created" fullname []
-		use ref = do
-			setIndexRef ref
-			return ref
+		use sha = do
+			setIndexSha sha
+			return sha
+		branchsha = inRepo $ Git.Ref.sha fullname
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
@@ -202,7 +203,7 @@ update = runUpdateOnce $ do
 			let merge_desc = if null branches
 				then "update" 
 				else "merging " ++
-					unwords (map Git.refDescribe branches) ++ 
+					unwords (map Git.Ref.describe branches) ++ 
 					" into " ++ show name
 			unless (null branches) $ do
 				showSideAction merge_desc
@@ -263,38 +264,18 @@ tryFastForwardTo (first:rest) = do
 				(False, True) -> findbest c rs -- worse
 				(False, False) -> findbest c rs -- same
 			
-{- Checks if a git ref exists. -}
-refExists :: Git.Ref -> Annex Bool
-refExists ref = inRepo $ Git.runBool "show-ref"
-	[Param "--verify", Param "-q", Param $ show ref]
-
-{- Get the ref of a branch. (Must be a fully qualified branch name) -}
-getRef :: Git.Branch -> Annex (Maybe Git.Ref)
-getRef branch = process . L.unpack <$> showref
-	where
-		showref = inRepo $ Git.pipeRead [Param "show-ref",
-			Param "--hash", -- get the hash
-			Param $ show branch]
-		process [] = Nothing
-		process s = Just $ Git.Ref $ firstLine s
-
 {- Does origin/git-annex exist? -}
 hasOrigin :: Annex Bool
-hasOrigin = refExists originname
+hasOrigin = inRepo $ Git.Ref.exists originname
 
-{- Does the git-annex branch or a foo/git-annex branch exist? -}
-hasSomeBranch :: Annex Bool
-hasSomeBranch = not . null <$> siblingBranches
+{- Does the git-annex branch or a sibling foo/git-annex branch exist? -}
+hasSibling :: Annex Bool
+hasSibling = not . null <$> siblingBranches
 
 {- List of git-annex (refs, branches), including the main one and any
  - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(Git.Ref, Git.Branch)]
-siblingBranches = do
-	r <- inRepo $ Git.pipeRead [Param "show-ref", Param $ show name]
-	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
-	where
-		gen l = (Git.Ref $ head l, Git.Ref $ last l)
-		uref (a, _) (b, _) = a == b
+siblingBranches = inRepo $ Git.Ref.matching name
 
 {- Applies a function to modifiy the content of a file.
  -
diff --git a/Command/Unused.hs b/Command/Unused.hs
index be0107752e..cd1cd16024 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -20,6 +20,7 @@ import Utility.TempFile
 import Logs.Location
 import qualified Annex
 import qualified Git
+import qualified Git.Ref
 import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
 import qualified Backend
@@ -190,7 +191,7 @@ getKeysReferenced = do
 {- List of keys referenced by symlinks in a git ref. -}
 getKeysReferencedInGit :: Git.Ref -> Annex [Key]
 getKeysReferencedInGit ref = do
-	showAction $ "checking " ++ Git.refDescribe ref
+	showAction $ "checking " ++ Git.Ref.describe ref
 	findkeys [] =<< inRepo (LsTree.lsTree ref)
 	where
 		findkeys c [] = return c
diff --git a/Git.hs b/Git.hs
index 1da5997c18..9af68a1942 100644
--- a/Git.hs
+++ b/Git.hs
@@ -24,7 +24,6 @@ module Git (
 	repoIsHttp,
 	repoIsLocalBare,
 	repoDescribe,
-	refDescribe,
 	repoLocation,
 	workTree,
 	workTreeFile,
@@ -178,14 +177,6 @@ repoDescribe Repo { location = Url url } = show url
 repoDescribe Repo { location = Dir dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
-{- Converts a fully qualified git ref into a user-visible version. -}
-refDescribe :: Ref -> String
-refDescribe = remove "refs/heads/" . remove "refs/remotes/" . show
-	where
-		remove prefix s
-			| prefix `isPrefixOf` s = drop (length prefix) s
-			| otherwise = s
-
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url
@@ -463,16 +454,16 @@ shaSize :: Int
 shaSize = 40
 
 {- Commits the index into the specified branch (or other ref), 
- - with the specified parent refs, and returns the new ref -}
-commit :: String -> Ref -> [Ref] -> Repo -> IO Ref
-commit message newref parentrefs repo = do
+ - with the specified parent refs, and returns the committed sha -}
+commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
+commit message branch parentrefs repo = do
 	tree <- getSha "write-tree" $ asString $
 		pipeRead [Param "write-tree"] repo
 	sha <- getSha "commit-tree" $ asString $
 		ignorehandle $ pipeWriteRead
 			(map Param $ ["commit-tree", show tree] ++ ps)
 			(L.pack message) repo
-	run "update-ref" [Param $ show newref, Param $ show sha] repo
+	run "update-ref" [Param $ show branch, Param $ show sha] repo
 	return sha
 	where
 		ignorehandle a = snd <$> a
diff --git a/Git/Ref.hs b/Git/Ref.hs
new file mode 100644
index 0000000000..723bea6817
--- /dev/null
+++ b/Git/Ref.hs
@@ -0,0 +1,47 @@
+{- git ref stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Ref where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Common
+import Git
+
+{- Converts a fully qualified git ref into a user-visible version. -}
+describe :: Ref -> String
+describe = remove "refs/heads/" . remove "refs/remotes/" . show
+	where
+		remove prefix s
+			| prefix `isPrefixOf` s = drop (length prefix) s
+			| otherwise = s
+
+{- Checks if a ref exists. -}
+exists :: Ref -> Repo -> IO Bool
+exists ref = runBool "show-ref" 
+	[Param "--verify", Param "-q", Param $ show ref]
+
+{- Get the sha of a fully qualified git ref, if it exists. -}
+sha :: Branch -> Repo -> IO (Maybe Sha)
+sha branch repo = process . L.unpack <$> showref repo
+	where
+		showref = pipeRead [Param "show-ref",
+			Param "--hash", -- get the hash
+			Param $ show branch]
+		process [] = Nothing
+		process s = Just $ Ref $ firstLine s
+
+{- List of (refs, branches) matching a given ref spec.
+ - Duplicate refs are filtered out. -}
+matching :: Ref -> Repo -> IO [(Git.Ref, Git.Branch)]
+matching ref repo = do
+	r <- Git.pipeRead [Param "show-ref", Param $ show ref] repo
+	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
+	where
+		gen l = (Git.Ref $ head l, Git.Ref $ last l)
+		uref (a, _) (b, _) = a == b
+
diff --git a/Init.hs b/Init.hs
index d03e10031f..c8deadf3b4 100644
--- a/Init.hs
+++ b/Init.hs
@@ -39,7 +39,7 @@ ensureInitialized :: Annex ()
 ensureInitialized = getVersion >>= maybe needsinit checkVersion
 	where
 		needsinit = do
-			annexed <- Annex.Branch.hasSomeBranch
+			annexed <- Annex.Branch.hasSibling
 			if annexed
 				then initialize Nothing
 				else error "First run: git-annex init"
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 08bf83e837..3440d504ba 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,6 +9,7 @@ module Upgrade.V2 where
 
 import Common.Annex
 import qualified Git
+import qualified Git.Ref
 import qualified Annex.Branch
 import Logs.Location
 import Annex.Content
@@ -86,7 +87,7 @@ logFiles dir = return . filter (".log" `isSuffixOf`)
 
 push :: Annex ()
 push = do
-	origin_master <- Annex.Branch.refExists $ Git.Ref "origin/master"
+	origin_master <- inRepo $ Git.Ref.exists $ Git.Ref "origin/master"
 	origin_gitannex <- Annex.Branch.hasOrigin
 	case (origin_master, origin_gitannex) of
 		(_, True) -> do

From 31a0c07ee91af9e3bf434f416a4d711d841aa223 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 21:12:51 -0400
Subject: [PATCH 2684/8313] broke out Git/Branch.hs and reorganized

---
 Annex/Branch.hs | 407 ++++++++++++++++++++++--------------------------
 Git/Branch.hs   |  60 +++++++
 2 files changed, 242 insertions(+), 225 deletions(-)
 create mode 100644 Git/Branch.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c8a538acd0..42940f4ff5 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -6,15 +6,15 @@
  -}
 
 module Annex.Branch (
+	name,
+	hasOrigin,
+	hasSibling,
 	create,
 	update,
 	get,
 	change,
 	commit,
 	files,
-	name,
-	hasOrigin,
-	hasSibling,
 ) where
 
 import System.Exit
@@ -27,6 +27,7 @@ import Annex.Journal
 import qualified Git
 import qualified Git.UnionMerge
 import qualified Git.Ref
+import qualified Git.Branch
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -41,57 +42,132 @@ fullname = Git.Ref $ "refs/heads/" ++ show name
 originname :: Git.Ref
 originname = Git.Ref $ "origin/" ++ show name
 
-{- Populates the branch's index file with the current branch contents.
- - 
- - This is only done when the index doesn't yet exist, and the index 
- - is used to build up changes to be commited to the branch, and merge
- - in changes from other branches.
- -}
-genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.stream_update_index g
-	[Git.UnionMerge.ls_tree fullname g]
+{- Does origin/git-annex exist? -}
+hasOrigin :: Annex Bool
+hasOrigin = inRepo $ Git.Ref.exists originname
 
-{- Merges the specified branches into the index.
- - Any changes staged in the index will be preserved. -}
-mergeIndex :: [Git.Ref] -> Annex ()
-mergeIndex branches = do
-	h <- catFileHandle
-	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+{- Does the git-annex branch or a sibling foo/git-annex branch exist? -}
+hasSibling :: Annex Bool
+hasSibling = not . null <$> siblingBranches
 
-{- Runs an action using the branch's index file. -}
-withIndex :: Annex a -> Annex a
-withIndex = withIndex' False
-withIndex' :: Bool -> Annex a -> Annex a
-withIndex' bootstrapping a = do
-	f <- fromRepo gitAnnexIndex
-	bracketIO (Git.useIndex f) id $ do
-		unlessM (liftIO $ doesFileExist f) $ do
-			unless bootstrapping create
-			liftIO $ createDirectoryIfMissing True $ takeDirectory f
-			unless bootstrapping $ inRepo genIndex
-		a
+{- List of git-annex (refs, branches), including the main one and any
+ - from remotes. Duplicate refs are filtered out. -}
+siblingBranches :: Annex [(Git.Ref, Git.Branch)]
+siblingBranches = inRepo $ Git.Ref.matching name
 
-{- Updates the branch's index to reflect the current contents of the branch.
- - Any changes staged in the index will be preserved.
+{- Creates the branch, if it does not already exist. -}
+create :: Annex ()
+create = do
+	_ <- getBranch
+	return ()
+
+{- Returns the ref of the branch, creating it first if necessary. -}
+getBranch :: Annex (Git.Ref)
+getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
+	where
+		go True = do
+			inRepo $ Git.run "branch"
+				[Param $ show name, Param $ show originname]
+			fromMaybe (error $ "failed to create " ++ show name)
+				<$> branchsha
+		go False = withIndex' True $ do
+			inRepo $ Git.commit "branch created" fullname []
+		use sha = do
+			setIndexSha sha
+			return sha
+		branchsha = inRepo $ Git.Ref.sha fullname
+
+{- Ensures that the branch and index are is up-to-date; should be
+ - called before data is read from it. Runs only once per git-annex run.
  -
- - Compares the ref stored in the lock file with the current
- - ref of the branch to see if an update is needed.
+ - Before refs are merged into the index, it's important to first stage the
+ - journal into the index. Otherwise, any changes in the journal would
+ - later get staged, and might overwrite changes made during the merge.
+ -
+ - (It would be cleaner to handle the merge by updating the journal, not the
+ - index, with changes from the branches.)
+ -
+ - The branch is fast-forwarded if possible, otherwise a merge commit is
+ - made.
  -}
-updateIndex :: Git.Ref -> Annex ()
-updateIndex branchref = do
-	lock <- fromRepo gitAnnexIndexLock
-	lockref <- Git.Ref . firstLine <$>
-		liftIO (catchDefaultIO (readFileStrict lock) "")
-	when (lockref /= branchref) $ do
-		withIndex $ mergeIndex [fullname]
-		setIndexSha branchref
+update :: Annex ()
+update = runUpdateOnce $ do
+	-- ensure branch exists, and get its current ref
+	branchref <- getBranch
+	-- check what needs updating before taking the lock
+	dirty <- journalDirty
+	(refs, branches) <- unzip <$> newerSiblings
+	if (not dirty && null refs)
+		then updateIndex branchref
+		else withIndex $ lockJournal $ do
+			when dirty stageJournal
+			let merge_desc = if null branches
+				then "update" 
+				else "merging " ++
+					unwords (map Git.Ref.describe branches) ++ 
+					" into " ++ show name
+			unless (null branches) $ do
+				showSideAction merge_desc
+				mergeIndex branches
+			ff <- if dirty
+				then return False
+				else inRepo $ Git.Branch.fastForward fullname refs
+			if ff
+				then updateIndex branchref
+				else commitBranch branchref merge_desc
+					(nub $ fullname:refs)
+			invalidateCache
+	where
+		newerSiblings = filterM isnewer =<< siblingBranches
+		isnewer (_, b) = inRepo $ Git.Branch.changed fullname b
 
-{- Record that the branch's index has been updated to correspond to a
- - given ref of the branch. -}
-setIndexSha :: Git.Ref -> Annex ()
-setIndexSha ref = do
-        lock <- fromRepo gitAnnexIndexLock
-	liftIO $ writeFile lock $ show ref ++ "\n"
+{- Gets the content of a file on the branch, or content from the journal, or
+ - staged in the index.
+ -
+ - Returns an empty string if the file doesn't exist yet. -}
+get :: FilePath -> Annex String
+get = get' False
+
+{- Like get, but does not merge the branch, so the info returned may not
+ - reflect changes in remotes. (Changing the value this returns, and then
+ - merging is always the same as using get, and then changing its value.) -}
+getStale :: FilePath -> Annex String
+getStale = get' True
+
+get' :: Bool -> FilePath -> Annex String
+get' staleok file = fromcache =<< getCache file
+	where
+		fromcache (Just content) = return content
+		fromcache Nothing = fromjournal =<< getJournalFile file
+		fromjournal (Just content) = cache content
+		fromjournal Nothing
+			| staleok = withIndex frombranch
+			| otherwise = withIndexUpdate $ frombranch >>= cache
+		frombranch = L.unpack <$> catFile fullname file
+		cache content = do
+			setCache file content
+			return content
+
+{- Applies a function to modifiy the content of a file.
+ -
+ - Note that this does not cause the branch to be merged, it only
+ - modifes the current content of the file on the branch.
+ -}
+change :: FilePath -> (String -> String) -> Annex ()
+change file a = lockJournal $ getStale file >>= return . a >>= set file
+
+{- Records new content of a file into the journal and cache. -}
+set :: FilePath -> String -> Annex ()
+set file content = do
+	setJournalFile file content
+	setCache file content
+
+{- Stages the journal, and commits staged changes to the branch. -}
+commit :: String -> Annex ()
+commit message = whenM journalDirty $ lockJournal $ do
+	stageJournal
+	ref <- getBranch
+	withIndex $ commitBranch ref message [fullname]
 
 {- Commits the staged changes in the index to the branch.
  - 
@@ -141,183 +217,6 @@ commitBranch branchref message parents = do
 		
 		racemessage = message ++ " (recovery from race)"
 
-{- Runs an action using the branch's index file, first making sure that
- - the branch and index are up-to-date. -}
-withIndexUpdate :: Annex a -> Annex a
-withIndexUpdate a = update >> withIndex a
-
-{- Creates the branch, if it does not already exist. -}
-create :: Annex ()
-create = do
-	_ <- getBranch
-	return ()
-
-{- Returns the ref of the branch, creating it first if necessary. -}
-getBranch :: Annex (Git.Ref)
-getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
-	where
-		go True = do
-			inRepo $ Git.run "branch"
-				[Param $ show name, Param $ show originname]
-			fromMaybe (error $ "failed to create " ++ show name)
-				<$> branchsha
-		go False = withIndex' True $ do
-			inRepo $ Git.commit "branch created" fullname []
-		use sha = do
-			setIndexSha sha
-			return sha
-		branchsha = inRepo $ Git.Ref.sha fullname
-
-{- Stages the journal, and commits staged changes to the branch. -}
-commit :: String -> Annex ()
-commit message = whenM journalDirty $ lockJournal $ do
-	stageJournal
-	ref <- getBranch
-	withIndex $ commitBranch ref message [fullname]
-
-{- Ensures that the branch and index are is up-to-date; should be
- - called before data is read from it. Runs only once per git-annex run.
- -
- - Before refs are merged into the index, it's important to first stage the
- - journal into the index. Otherwise, any changes in the journal would
- - later get staged, and might overwrite changes made during the merge.
- -
- - (It would be cleaner to handle the merge by updating the journal, not the
- - index, with changes from the branches.)
- -
- - The branch is fast-forwarded if possible, otherwise a merge commit is
- - made.
- -}
-update :: Annex ()
-update = runUpdateOnce $ do
-	-- ensure branch exists, and get its current ref
-	branchref <- getBranch
-	-- check what needs updating before taking the lock
-	dirty <- journalDirty
-	c <- filterM (changedBranch fullname . snd) =<< siblingBranches
-	let (refs, branches) = unzip c
-	if (not dirty && null refs)
-		then updateIndex branchref
-		else withIndex $ lockJournal $ do
-			when dirty stageJournal
-			let merge_desc = if null branches
-				then "update" 
-				else "merging " ++
-					unwords (map Git.Ref.describe branches) ++ 
-					" into " ++ show name
-			unless (null branches) $ do
-				showSideAction merge_desc
-				mergeIndex branches
-			ff <- if dirty then return False else tryFastForwardTo refs
-			if ff
-				then updateIndex branchref
-				else commitBranch branchref merge_desc
-					(nub $ fullname:refs)
-			invalidateCache
-
-{- Checks if the second branch has any commits not present on the first
- - branch. -}
-changedBranch :: Git.Branch -> Git.Branch -> Annex Bool
-changedBranch origbranch newbranch
-	| origbranch == newbranch = return False
-	| otherwise = not . L.null <$> diffs
-	where
-		diffs = inRepo $ Git.pipeRead
-			[ Param "log"
-			, Param (show origbranch ++ ".." ++ show newbranch)
-			, Params "--oneline -n1"
-			]
-
-{- Given a set of refs that are all known to have commits not
- - on the git-annex branch, tries to update the branch by a
- - fast-forward.
- -
- - In order for that to be possible, one of the refs must contain
- - every commit present in all the other refs, as well as in the
- - git-annex branch.
- -}
-tryFastForwardTo :: [Git.Ref] -> Annex Bool
-tryFastForwardTo [] = return True
-tryFastForwardTo (first:rest) = do
-	-- First, check that the git-annex branch does not contain any
-	-- new commits that are not in the first other branch. If it does,
-	-- cannot fast-forward.
-	diverged <- changedBranch first fullname
-	if diverged
-		then no_ff
-		else maybe no_ff do_ff =<< findbest first rest
-	where
-		no_ff = return False
-		do_ff branch = do
-			inRepo $ Git.run "update-ref"
-				[Param $ show fullname, Param $ show branch]
-			return True
-		findbest c [] = return $ Just c
-		findbest c (r:rs)
-			| c == r = findbest c rs
-			| otherwise = do
-			better <- changedBranch c r
-			worse <- changedBranch r c
-			case (better, worse) of
-				(True, True) -> return Nothing -- divergent fail
-				(True, False) -> findbest r rs -- better
-				(False, True) -> findbest c rs -- worse
-				(False, False) -> findbest c rs -- same
-			
-{- Does origin/git-annex exist? -}
-hasOrigin :: Annex Bool
-hasOrigin = inRepo $ Git.Ref.exists originname
-
-{- Does the git-annex branch or a sibling foo/git-annex branch exist? -}
-hasSibling :: Annex Bool
-hasSibling = not . null <$> siblingBranches
-
-{- List of git-annex (refs, branches), including the main one and any
- - from remotes. Duplicate refs are filtered out. -}
-siblingBranches :: Annex [(Git.Ref, Git.Branch)]
-siblingBranches = inRepo $ Git.Ref.matching name
-
-{- Applies a function to modifiy the content of a file.
- -
- - Note that this does not cause the branch to be merged, it only
- - modifes the current content of the file on the branch.
- -}
-change :: FilePath -> (String -> String) -> Annex ()
-change file a = lockJournal $ getStale file >>= return . a >>= set file
-
-{- Records new content of a file into the journal and cache. -}
-set :: FilePath -> String -> Annex ()
-set file content = do
-	setJournalFile file content
-	setCache file content
-
-{- Gets the content of a file on the branch, or content from the journal, or
- - staged in the index.
- -
- - Returns an empty string if the file doesn't exist yet. -}
-get :: FilePath -> Annex String
-get = get' False
-
-{- Like get, but does not merge the branch, so the info returned may not
- - reflect changes in remotes. (Changing the value this returns, and then
- - merging is always the same as using get, and then changing its value.) -}
-getStale :: FilePath -> Annex String
-getStale = get' True
-
-get' :: Bool -> FilePath -> Annex String
-get' staleok file = fromcache =<< getCache file
-	where
-		fromcache (Just content) = return content
-		fromcache Nothing = fromjournal =<< getJournalFile file
-		fromjournal (Just content) = cache content
-		fromjournal Nothing
-			| staleok = withIndex frombranch
-			| otherwise = withIndexUpdate $ frombranch >>= cache
-		frombranch = L.unpack <$> catFile fullname file
-		cache content = do
-			setCache file content
-			return content
-
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
@@ -326,6 +225,64 @@ files = withIndexUpdate $ do
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
 
+
+{- Populates the branch's index file with the current branch contents.
+ - 
+ - This is only done when the index doesn't yet exist, and the index 
+ - is used to build up changes to be commited to the branch, and merge
+ - in changes from other branches.
+ -}
+genIndex :: Git.Repo -> IO ()
+genIndex g = Git.UnionMerge.stream_update_index g
+	[Git.UnionMerge.ls_tree fullname g]
+
+{- Merges the specified branches into the index.
+ - Any changes staged in the index will be preserved. -}
+mergeIndex :: [Git.Ref] -> Annex ()
+mergeIndex branches = do
+	h <- catFileHandle
+	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+
+{- Runs an action using the branch's index file. -}
+withIndex :: Annex a -> Annex a
+withIndex = withIndex' False
+withIndex' :: Bool -> Annex a -> Annex a
+withIndex' bootstrapping a = do
+	f <- fromRepo gitAnnexIndex
+	bracketIO (Git.useIndex f) id $ do
+		unlessM (liftIO $ doesFileExist f) $ do
+			unless bootstrapping create
+			liftIO $ createDirectoryIfMissing True $ takeDirectory f
+			unless bootstrapping $ inRepo genIndex
+		a
+
+{- Runs an action using the branch's index file, first making sure that
+ - the branch and index are up-to-date. -}
+withIndexUpdate :: Annex a -> Annex a
+withIndexUpdate a = update >> withIndex a
+
+{- Updates the branch's index to reflect the current contents of the branch.
+ - Any changes staged in the index will be preserved.
+ -
+ - Compares the ref stored in the lock file with the current
+ - ref of the branch to see if an update is needed.
+ -}
+updateIndex :: Git.Ref -> Annex ()
+updateIndex branchref = do
+	lock <- fromRepo gitAnnexIndexLock
+	lockref <- Git.Ref . firstLine <$>
+		liftIO (catchDefaultIO (readFileStrict lock) "")
+	when (lockref /= branchref) $ do
+		withIndex $ mergeIndex [fullname]
+		setIndexSha branchref
+
+{- Record that the branch's index has been updated to correspond to a
+ - given ref of the branch. -}
+setIndexSha :: Git.Ref -> Annex ()
+setIndexSha ref = do
+        lock <- fromRepo gitAnnexIndexLock
+	liftIO $ writeFile lock $ show ref ++ "\n"
+
 {- Stages the journal into the index. -}
 stageJournal :: Annex ()
 stageJournal = do
diff --git a/Git/Branch.hs b/Git/Branch.hs
new file mode 100644
index 0000000000..e69e96f288
--- /dev/null
+++ b/Git/Branch.hs
@@ -0,0 +1,60 @@
+{- git branch stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Branch where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Common
+import Git
+
+{- Checks if the second branch has any commits not present on the first
+ - branch. -}
+changed :: Branch -> Branch -> Repo -> IO Bool
+changed origbranch newbranch repo
+	| origbranch == newbranch = return False
+	| otherwise = not . L.null <$> diffs
+	where
+		diffs = Git.pipeRead
+			[ Param "log"
+			, Param (show origbranch ++ ".." ++ show newbranch)
+			, Params "--oneline -n1"
+			] repo
+
+{- Given a set of refs that are all known to have commits not
+ - on the branch, tries to update the branch by a fast-forward.
+ -
+ - In order for that to be possible, one of the refs must contain
+ - every commit present in all the other refs.
+ -}
+fastForward :: Branch -> [Ref] -> Repo -> IO Bool
+fastForward _ [] _ = return True
+fastForward branch (first:rest) repo = do
+	-- First, check that the branch does not contain any
+	-- new commits that are not in the first ref. If it does,
+	-- cannot fast-forward.
+	diverged <- changed first branch repo
+	if diverged
+		then no_ff
+		else maybe no_ff do_ff =<< findbest first rest
+	where
+		no_ff = return False
+		do_ff to = do
+			Git.run "update-ref"
+				[Param $ show branch, Param $ show to] repo
+			return True
+		findbest c [] = return $ Just c
+		findbest c (r:rs)
+			| c == r = findbest c rs
+			| otherwise = do
+			better <- changed c r repo
+			worse <- changed r c repo
+			case (better, worse) of
+				(True, True) -> return Nothing -- divergent fail
+				(True, False) -> findbest r rs -- better
+				(False, True) -> findbest c rs -- worse
+				(False, False) -> findbest c rs -- same

From 0e45b762a07d12dbc099936a8481bda9c02d0318 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 21:24:55 -0400
Subject: [PATCH 2685/8313] broke out Git/HashObject.hs

---
 Annex/Branch.hs   | 19 +++----------------
 Git/HashObject.hs | 29 +++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 Git/HashObject.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 42940f4ff5..556df976fe 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -17,7 +17,6 @@ module Annex.Branch (
 	files,
 ) where
 
-import System.Exit
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common.Annex
@@ -25,9 +24,10 @@ import Annex.Exception
 import Annex.BranchState
 import Annex.Journal
 import qualified Git
-import qualified Git.UnionMerge
 import qualified Git.Ref
 import qualified Git.Branch
+import qualified Git.UnionMerge
+import qualified Git.HashObject
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -291,23 +291,10 @@ stageJournal = do
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
 		let paths = map (dir ) fs
-		-- inject all the journal files directly into git
-		-- in one quick command
-		(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object g
-		_ <- forkProcess $ do
-			hPutStr toh $ unlines paths
-			hClose toh
-			exitSuccess
-		hClose toh
-		shas <- map Git.Ref . lines <$> hGetContents fromh
-		-- update the index, also in just one command
+		shas <- Git.HashObject.hashFiles paths g
 		Git.UnionMerge.update_index g $
 			index_lines shas (map fileJournal fs)
-		hClose fromh
-		forceSuccess pid
 		mapM_ removeFile paths
 	where
 		index_lines shas = map genline . zip shas
 		genline (sha, file) = Git.UnionMerge.update_index_line sha file
-		git_hash_object = Git.gitCommandLine
-			[Param "hash-object", Param "-w", Param "--stdin-paths"]
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
new file mode 100644
index 0000000000..f28d865b13
--- /dev/null
+++ b/Git/HashObject.hs
@@ -0,0 +1,29 @@
+{- git hash-object interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.HashObject where
+
+import Common
+import Git
+
+{- Injects a set of files into git, returning the shas of the objects. -}
+hashFiles :: [FilePath] -> Repo -> IO [Sha]
+hashFiles paths repo = do
+	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
+	_ <- forkProcess (feeder toh)
+	hClose toh
+	shas <- map Git.Ref . lines <$> hGetContents fromh
+	hClose fromh
+	forceSuccess pid
+	return shas
+	where
+		git_hash_object = Git.gitCommandLine
+			[Param "hash-object", Param "-w", Param "--stdin-paths"]
+		feeder toh = do
+			hPutStr toh $ unlines paths
+			hClose toh
+			exitSuccess

From 46588674b081cd4ea5820680d8fc15c81ed175ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Dec 2011 21:41:37 -0400
Subject: [PATCH 2686/8313] avoid closing pipe before all the shas are read
 from it

Could have just used hGetContentsStrict here, but that would require
storing all the shas in memory. Since this is called at the end of a
git-annex run, it may have created a *lot* of shas, so I avoid that memory
use and stream them out like before.
---
 Annex/Branch.hs   |  3 ++-
 Git/HashObject.hs | 14 ++++++++------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 556df976fe..f9fa6cbb31 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -291,9 +291,10 @@ stageJournal = do
 	withIndex $ liftIO $ do
 		let dir = gitAnnexJournalDir g
 		let paths = map (dir ) fs
-		shas <- Git.HashObject.hashFiles paths g
+		(shas, cleanup) <- Git.HashObject.hashFiles paths g
 		Git.UnionMerge.update_index g $
 			index_lines shas (map fileJournal fs)
+		cleanup
 		mapM_ removeFile paths
 	where
 		index_lines shas = map genline . zip shas
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index f28d865b13..99b96afcb9 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -10,16 +10,15 @@ module Git.HashObject where
 import Common
 import Git
 
-{- Injects a set of files into git, returning the shas of the objects. -}
-hashFiles :: [FilePath] -> Repo -> IO [Sha]
+{- Injects a set of files into git, returning the shas of the objects
+ - and an IO action to call ones the the shas have been used. -}
+hashFiles :: [FilePath] -> Repo -> IO ([Sha], IO ())
 hashFiles paths repo = do
 	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
 	_ <- forkProcess (feeder toh)
 	hClose toh
-	shas <- map Git.Ref . lines <$> hGetContents fromh
-	hClose fromh
-	forceSuccess pid
-	return shas
+	shas <- map Git.Ref . lines <$> hGetContentsStrict fromh
+	return (shas, ender fromh pid)
 	where
 		git_hash_object = Git.gitCommandLine
 			[Param "hash-object", Param "-w", Param "--stdin-paths"]
@@ -27,3 +26,6 @@ hashFiles paths repo = do
 			hPutStr toh $ unlines paths
 			hClose toh
 			exitSuccess
+		ender fromh pid = do
+			hClose fromh
+			forceSuccess pid

From 51b95fbc0792b6fc9ba546692896274987b48822 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Tue, 13 Dec 2011 18:16:09 +0000
Subject: [PATCH 2687/8313] Added a comment

---
 ..._dc8a3f75533906ad3756fcc47f7e96bb._comment | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment b/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment
new file mode 100644
index 0000000000..1ac9e798a8
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_4_dc8a3f75533906ad3756fcc47f7e96bb._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 4"
+ date="2011-12-13T18:16:08Z"
+ content="""
+I thought about this some more, and I think I have a pretty decent solution that avoids a central bare repository. Instead of pushing to master (which git does not like) or trying to guess the remote branch name on the other side, there is a well-known branch name, say git-annex-master. Then a sync command would do something like this (untested):
+
+    git commit -a -m 'git annex sync' # ideally with a description derived from the diff
+    git merge git-annex-master
+    git pull someremote git-annex-master # for all reachable remotes. Or better to use fetch and then merge everything in one command?
+    git branch -f git-annex-master # (or checkout git-annex-master, merge master, checkout master, but since we merged before this should have the same effect
+    git annex merge
+    git push someremote git-annex-master # for all reachable remotes
+
+The nice things are: One can push to any remote repository, and thus avoid the issue of pushing to a portable device; the merging happens on the master branch, so if it fails to merge automatically, regular git foo can resolve it, and all changes eventually reach every repository.
+
+What do you think?
+    
+"""]]

From 1a06455f5cedb52324c6da6b13b7eedd72abe430 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Tue, 13 Dec 2011 18:47:18 +0000
Subject: [PATCH 2688/8313] Added a comment

---
 ..._afe5035a6b35ed2c7e193fb69cc182e2._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment b/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment
new file mode 100644
index 0000000000..0847daae9d
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_5_afe5035a6b35ed2c7e193fb69cc182e2._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 5"
+ date="2011-12-13T18:47:18Z"
+ content="""
+After some experimentation, this seems to work better:
+
+    	git commit -a -m 'git annex sync'
+	git merge git-annex-master
+	for remote in $(git remote)
+	do
+		git fetch $remote
+		git merge $remote git-annex-master
+	done
+	git branch -f git-annex-master
+	git annex merge
+	for remote in $(git remote)
+	do
+		git push $remote git-annex git-annex-master
+	done
+
+Maybe this approach can be enhance to skip stuff gracefully if there is no git-annex-master branch and then be added to what \"git annex sync\" does, this way those who want to use the feature can do so by running \"git branch git-annex-master\" once. Or, if you like this and want to make it default, just make git-annex-init create the git-annex-master branch :-)
+"""]]

From 13fff71f2019ae098c3f8532ac2734cb1ab11498 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:05:07 -0400
Subject: [PATCH 2689/8313] split out three modules from Git

Constructors and configuration make sense in separate modules.
A separate Git.Types is needed to avoid cycles.
---
 Annex.hs                 |   3 +-
 Annex/UUID.hs            |   5 +-
 Annex/Version.hs         |   4 +-
 Backend.hs               |   3 +-
 Command/Map.hs           |  10 +-
 Command/Sync.hs          |   5 +-
 Config.hs                |   9 +-
 Git.hs                   | 250 +--------------------------------------
 Git/Config.hs            |  58 +++++++++
 Git/Construct.hs         | 198 +++++++++++++++++++++++++++++++
 Git/Types.hs             |  36 ++++++
 GitAnnex.hs              |   7 +-
 Remote/Bup.hs            |  16 +--
 Remote/Git.hs            |   8 +-
 Remote/Helper/Special.hs |   3 +-
 Remote/Web.hs            |   3 +-
 git-annex-shell.hs       |   4 +-
 git-annex.cabal          |   2 +-
 git-union-merge.hs       |   4 +-
 test.hs                  |   6 +-
 20 files changed, 349 insertions(+), 285 deletions(-)
 create mode 100644 Git/Config.hs
 create mode 100644 Git/Construct.hs
 create mode 100644 Git/Types.hs

diff --git a/Annex.hs b/Annex.hs
index d8d9c63b40..acc38a5db9 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -27,6 +27,7 @@ import Control.Monad.State
 
 import Common
 import qualified Git
+import qualified Git.Config
 import Git.CatFile
 import Git.Queue
 import Types.Backend
@@ -99,7 +100,7 @@ newState gitrepo = AnnexState
 
 {- Create and returns an Annex state object for the specified git repo. -}
 new :: Git.Repo -> IO AnnexState
-new gitrepo = newState <$> Git.configRead gitrepo
+new gitrepo = newState <$> Git.Config.read gitrepo
 
 {- performs an action in the Annex monad -}
 run :: AnnexState -> Annex a -> IO (a, AnnexState)
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index e510a7ccd3..48bf71f104 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -21,6 +21,7 @@ module Annex.UUID (
 
 import Common.Annex
 import qualified Git
+import qualified Git.Config
 import qualified Build.SysConfig as SysConfig
 import Config
 
@@ -55,14 +56,14 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached = toUUID . Git.configGet cachekey ""
+		cached = toUUID . Git.Config.get cachekey ""
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u
 		cachekey = remoteConfig r "uuid"
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID = toUUID . Git.configGet configkey ""
+getUncachedUUID = toUUID . Git.Config.get configkey ""
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
diff --git a/Annex/Version.hs b/Annex/Version.hs
index 9e694faf14..917859eae4 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -8,7 +8,7 @@
 module Annex.Version where
 
 import Common.Annex
-import qualified Git
+import qualified Git.Config
 import Config
 
 type Version = String
@@ -26,7 +26,7 @@ versionField :: String
 versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
-getVersion = handle <$> fromRepo (Git.configGet versionField "")
+getVersion = handle <$> fromRepo (Git.Config.get versionField "")
 	where
 		handle [] = Nothing
 		handle v = Just v
diff --git a/Backend.hs b/Backend.hs
index 136c2eb7a6..c7cb57440b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -21,6 +21,7 @@ import System.Posix.Files
 
 import Common.Annex
 import qualified Git
+import qualified Git.Config
 import qualified Annex
 import Types.Key
 import qualified Types.Backend as B
@@ -47,7 +48,7 @@ orderedList = do
 			l' <- (lookupBackendName name :) <$> standard
 			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		standard = fromRepo $ parseBackendList . Git.configGet "annex.backends" ""
+		standard = fromRepo $ parseBackendList . Git.Config.get "annex.backends" ""
 		parseBackendList [] = list
 		parseBackendList s = map lookupBackendName $ words s
 
diff --git a/Command/Map.hs b/Command/Map.hs
index 57b48d5030..815b142e72 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,6 +13,8 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
@@ -146,8 +148,8 @@ spider' (r:rs) known
 {- Converts repos to a common absolute form. -}
 absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo
 absRepo reference r
-	| Git.repoIsUrl reference = return $ Git.localToUrl reference r
-	| otherwise = liftIO $ Git.repoFromAbsPath =<< absPath (Git.workTree r)
+	| Git.repoIsUrl reference = return $ Git.Construct.localToUrl reference r
+	| otherwise = liftIO $ Git.Construct.fromAbsPath =<< absPath (Git.workTree r)
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
@@ -182,7 +184,7 @@ tryScan :: Git.Repo -> Annex (Maybe Git.Repo)
 tryScan r
 	| Git.repoIsSsh r = sshscan
 	| Git.repoIsUrl r = return Nothing
-	| otherwise = safely $ Git.configRead r
+	| otherwise = safely $ Git.Config.read r
 	where
 		safely a = do
 			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
@@ -191,7 +193,7 @@ tryScan r
 				Right r' -> return $ Just r'
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
-				Git.hConfigRead r
+				Git.Config.hRead r
 
 		configlist =
 			onRemote r (pipedconfig, Nothing) "configlist" []
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 7dc5f4d24c..987eb6138f 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import qualified Annex.Branch
 import qualified Git
+import qualified Git.Config
 
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -56,7 +57,7 @@ push = do
 defaultRemote :: Annex String
 defaultRemote = do
 	branch <- currentBranch
-	fromRepo $ Git.configGet ("branch." ++ branch ++ ".remote") "origin"
+	fromRepo $ Git.Config.get ("branch." ++ branch ++ ".remote") "origin"
 
 currentBranch :: Annex String
 currentBranch = last . split "/" . L.unpack . head . L.lines <$>
@@ -65,6 +66,6 @@ currentBranch = last . split "/" . L.unpack . head . L.lines <$>
 checkRemote :: String -> Annex ()
 checkRemote remote = do
 	remoteurl <- fromRepo $
-		Git.configGet ("remote." ++ remote ++ ".url") ""
+		Git.Config.get ("remote." ++ remote ++ ".url") ""
 	when (null remoteurl) $ do
 		error $ "No url is configured for the remote: " ++ remote
diff --git a/Config.hs b/Config.hs
index c6107fc8eb..322dc8af82 100644
--- a/Config.hs
+++ b/Config.hs
@@ -9,6 +9,7 @@ module Config where
 
 import Common.Annex
 import qualified Git
+import qualified Git.Config
 import qualified Annex
 
 type ConfigKey = String
@@ -18,15 +19,15 @@ setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
 	inRepo $ Git.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
-	newg <- inRepo Git.configRead
+	newg <- inRepo Git.Config.read
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
 getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
 getConfig r key def = do
-	def' <- fromRepo $ Git.configGet ("annex." ++ key) def
-	fromRepo $ Git.configGet (remoteConfig r key) def'
+	def' <- fromRepo $ Git.Config.get ("annex." ++ key) def
+	fromRepo $ Git.Config.get (remoteConfig r key) def'
 
 {- Looks up a per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
@@ -83,6 +84,6 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
 		use Nothing = perhaps (return 1) =<< 
-			readMaybe <$> fromRepo (Git.configGet config "1")
+			readMaybe <$> fromRepo (Git.Config.get config "1")
 		perhaps fallback = maybe fallback (return . id)
 		config = "annex.numcopies"
diff --git a/Git.hs b/Git.hs
index 9af68a1942..cb7cc19c25 100644
--- a/Git.hs
+++ b/Git.hs
@@ -3,7 +3,7 @@
  - This is written to be completely independant of git-annex and should be
  - suitable for other uses.
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010, 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -14,11 +14,6 @@ module Git (
 	Branch,
 	Sha,
 	Tag,
-	repoFromCwd,
-	repoFromAbsPath,
-	repoFromUnknown,
-	repoFromUrl,
-	localToUrl,
 	repoIsUrl,
 	repoIsSsh,
 	repoIsHttp,
@@ -34,11 +29,7 @@ module Git (
 	urlHostUser,
 	urlAuthority,
 	urlScheme,
-	configGet,
 	configMap,
-	configRead,
-	hConfigRead,
-	configStore,
 	configTrue,
 	gitCommandLine,
 	run,
@@ -51,14 +42,12 @@ module Git (
 	attributes,
 	remotes,
 	remotesAdd,
-	genRemote,
 	repoRemoteName,
 	repoRemoteNameSet,
 	repoRemoteNameFromKey,
 	checkAttr,
 	decodeGitFile,
 	encodeGitFile,
-	repoAbsPath,
 	reap,
 	useIndex,
 	getSha,
@@ -69,9 +58,6 @@ module Git (
 	prop_idempotent_deencode
 ) where
 
-import System.Posix.Directory
-import System.Posix.User
-import Control.Exception (bracket_)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
 import Data.Char
@@ -83,92 +69,7 @@ import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
-
-{- There are two types of repositories; those on local disk and those
- - accessed via an URL. -}
-data RepoLocation = Dir FilePath | Url URI | Unknown
-	deriving (Show, Eq)
-
-data Repo = Repo {
-	location :: RepoLocation,
-	config :: M.Map String String,
-	remotes :: [Repo],
-	-- remoteName holds the name used for this repo in remotes
-	remoteName :: Maybe String 
-} deriving (Show, Eq)
-
-{- A git ref. Can be a sha1, or a branch or tag name. -}
-newtype Ref = Ref String
-	deriving (Eq)
-
-instance Show Ref where
-	show (Ref v) = v
-
-{- Aliases for Ref. -}
-type Branch = Ref
-type Sha = Ref
-type Tag = Ref
-
-newFrom :: RepoLocation -> Repo
-newFrom l = 
-	Repo {
-		location = l,
-		config = M.empty,
-		remotes = [],
-		remoteName = Nothing
-	}
-
-{- Local Repo constructor, requires an absolute path to the repo be
- - specified. -}
-repoFromAbsPath :: FilePath -> IO Repo
-repoFromAbsPath dir
-	| "/" `isPrefixOf` dir = do
- 		-- Git always looks for "dir.git" in preference to
-		-- to "dir", even if dir ends in a "/".
-		let canondir = dropTrailingPathSeparator dir
-		let dir' = canondir ++ ".git"
-		e <- doesDirectoryExist dir'
-		if e
-			then ret dir'
-			else if "/.git" `isSuffixOf` canondir
-				then do
-					-- When dir == "foo/.git", git looks
-					-- for "foo/.git/.git", and failing
-					-- that, uses "foo" as the repository.
-					e' <- doesDirectoryExist $ dir  ".git"
-					if e'
-						then ret dir
-						else ret $ takeDirectory canondir
-				else ret dir
-	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
-	where
-		ret = return . newFrom . Dir
-
-{- Remote Repo constructor. Throws exception on invalid url. -}
-repoFromUrl :: String -> IO Repo
-repoFromUrl url
-	| startswith "file://" url = repoFromAbsPath $ uriPath u
-	| otherwise = return $ newFrom $ Url u
-		where
-			u = fromMaybe bad $ parseURI url
-			bad = error $ "bad url " ++ url
-
-{- Creates a repo that has an unknown location. -}
-repoFromUnknown :: Repo
-repoFromUnknown = newFrom Unknown
-
-{- Converts a Local Repo into a remote repo, using the reference repo
- - which is assumed to be on the same host. -}
-localToUrl :: Repo -> Repo -> Repo
-localToUrl reference r
-	| not $ repoIsUrl reference = error "internal error; reference repo not url"
-	| repoIsUrl r = r
-	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
-	where
-		absurl =
-			urlScheme reference ++ "//" ++
-			urlAuthority reference ++
-			workTree r
+import Git.Types
 
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
@@ -470,89 +371,10 @@ commit message branch parentrefs repo = do
 		asString a = L.unpack <$> a
 		ps = concatMap (\r -> ["-p", show r]) parentrefs
 
-{- Runs git config and populates a repo with its config. -}
-configRead :: Repo -> IO Repo
-configRead repo@(Repo { location = Dir d }) = do
-	{- Cannot use pipeRead because it relies on the config having
-	   been already read. Instead, chdir to the repo. -}
-	cwd <- getCurrentDirectory
-	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
-		pOpen ReadFromPipe "git" ["config", "--list"] $ hConfigRead repo
-configRead r = assertLocal r $ error "internal"
-
-{- Reads git config from a handle and populates a repo with it. -}
-hConfigRead :: Repo -> Handle -> IO Repo
-hConfigRead repo h = do
-	val <- hGetContentsStrict h
-	configStore val repo
-
-{- Stores a git config into a repo, returning the new version of the repo.
- - The git config may be multiple lines, or a single line. Config settings
- - can be updated inrementally. -}
-configStore :: String -> Repo -> IO Repo
-configStore s repo = do
-	let repo' = repo { config = configParse s `M.union` config repo }
-	rs <- configRemotes repo'
-	return $ repo' { remotes = rs }
-
-{- Parses git config --list output into a config map. -}
-configParse :: String -> M.Map String String
-configParse s = M.fromList $ map pair $ lines s
-	where
-		pair = separate (== '=')
-
-{- Calculates a list of a repo's configured remotes, by parsing its config. -}
-configRemotes :: Repo -> IO [Repo]
-configRemotes repo = mapM construct remotepairs
-	where
-		filterconfig f = filter f $ M.toList $ config repo
-		filterkeys f = filterconfig (\(k,_) -> f k)
-		remotepairs = filterkeys isremote
-		isremote k = startswith "remote." k && endswith ".url" k
-		construct (k,v) = repoRemoteNameFromKey k <$> genRemote v repo
-
-{- Generates one of a repo's remotes using a given location (ie, an url). -}
-genRemote :: String -> Repo -> IO Repo
-genRemote s repo = gen $ calcloc s
-	where
-		filterconfig f = filter f $ M.toList $ config repo
-		gen v	
-			| scpstyle v = repoFromUrl $ scptourl v
-			| isURI v = repoFromUrl v
-			| otherwise = repoFromRemotePath v repo
-		-- insteadof config can rewrite remote location
-		calcloc l
-			| null insteadofs = l
-			| otherwise = replacement ++ drop (length bestvalue) l
-			where
-				replacement = drop (length prefix) $
-					take (length bestkey - length suffix) bestkey
-				(bestkey, bestvalue) = maximumBy longestvalue insteadofs
-				longestvalue (_, a) (_, b) = compare b a
-				insteadofs = filterconfig $ \(k, v) -> 
-					startswith prefix k &&
-					endswith suffix k &&
-					startswith v l
-				(prefix, suffix) = ("url." , ".insteadof")
-		-- git remotes can be written scp style -- [user@]host:dir
-		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
-		scptourl v = "ssh://" ++ host ++ slash dir
-			where
-				(host, dir) = separate (== ':') v
-				slash d	| d == "" = "/~/" ++ d
-					| "/" `isPrefixOf` d = d
-					| "~" `isPrefixOf` d = '/':d
-					| otherwise = "/~/" ++ d
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
 
-{- Returns a single git config setting, or a default value if not set. -}
-configGet :: String -> String -> Repo -> String
-configGet key defaultValue repo = 
-	M.findWithDefault defaultValue key (config repo)
-
 {- Access to raw config Map -}
 configMap :: Repo -> M.Map String String
 configMap = config
@@ -658,71 +480,3 @@ encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
 prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
-
-{- Constructs a Repo from the path specified in the git remotes of
- - another Repo. -}
-repoFromRemotePath :: FilePath -> Repo -> IO Repo
-repoFromRemotePath dir repo = do
-	dir' <- expandTilde dir
-	repoFromAbsPath $ workTree repo  dir'
-
-{- Git remotes can have a directory that is specified relative
- - to the user's home directory, or that contains tilde expansions.
- - This converts such a directory to an absolute path.
- - Note that it has to run on the system where the remote is.
- -}
-repoAbsPath :: FilePath -> IO FilePath
-repoAbsPath d = do
-	d' <- expandTilde d
-	h <- myHomeDir
-	return $ h  d'
-
-expandTilde :: FilePath -> IO FilePath
-expandTilde = expandt True
-	where
-		expandt _ [] = return ""
-		expandt _ ('/':cs) = do
-			v <- expandt True cs
-			return ('/':v)
-		expandt True ('~':'/':cs) = do
-			h <- myHomeDir
-			return $ h  cs
-		expandt True ('~':cs) = do
-			let (name, rest) = findname "" cs
-			u <- getUserEntryForName name
-			return $ homeDirectory u  rest
-		expandt _ (c:cs) = do
-			v <- expandt False cs
-			return (c:v)
-		findname n [] = (n, "")
-		findname n (c:cs)
-			| c == '/' = (n, cs)
-			| otherwise = findname (n++[c]) cs
-
-{- Finds the current git repository, which may be in a parent directory. -}
-repoFromCwd :: IO Repo
-repoFromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
-	where
-		makerepo = return . newFrom . Dir
-		norepo = error "Not in a git repository."
-
-seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
-seekUp want dir = do
-	ok <- want dir
-	if ok
-		then return $ Just dir
-		else case parentDir dir of
-			"" -> return Nothing
-			d -> seekUp want d
-
-isRepoTop :: FilePath -> IO Bool
-isRepoTop dir = do
-	r <- isRepo
-	b <- isBareRepo
-	return (r || b)
-	where
-		isRepo = gitSignature ".git" ".git/config"
-		isBareRepo = gitSignature "objects" "config"
-		gitSignature subdir file = liftM2 (&&)
-			(doesDirectoryExist (dir ++ "/" ++ subdir))
-			(doesFileExist (dir ++ "/" ++ file))
diff --git a/Git/Config.hs b/Git/Config.hs
new file mode 100644
index 0000000000..5f0e3fdc21
--- /dev/null
+++ b/Git/Config.hs
@@ -0,0 +1,58 @@
+{- git repository configuration handling
+ -
+ - Copyright 2010,2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Config (
+	get,
+	read,
+	hRead,
+	store
+) where
+
+import Prelude hiding (read)
+import System.Posix.Directory
+import Control.Exception (bracket_)
+import qualified Data.Map as M
+
+import Common
+import Git
+import Git.Types
+import qualified Git.Construct
+
+{- Returns a single git config setting, or a default value if not set. -}
+get :: String -> String -> Repo -> String
+get key defaultValue repo = M.findWithDefault defaultValue key (config repo)
+
+{- Runs git config and populates a repo with its config. -}
+read :: Repo -> IO Repo
+read repo@(Repo { location = Dir d }) = do
+	{- Cannot use pipeRead because it relies on the config having
+	   been already read. Instead, chdir to the repo. -}
+	cwd <- getCurrentDirectory
+	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
+		pOpen ReadFromPipe "git" ["config", "--list"] $ hRead repo
+read r = assertLocal r $ error "internal"
+
+{- Reads git config from a handle and populates a repo with it. -}
+hRead :: Repo -> Handle -> IO Repo
+hRead repo h = do
+	val <- hGetContentsStrict h
+	store val repo
+
+{- Stores a git config into a repo, returning the new version of the repo.
+ - The git config may be multiple lines, or a single line. Config settings
+ - can be updated inrementally. -}
+store :: String -> Repo -> IO Repo
+store s repo = do
+	let repo' = repo { config = parse s `M.union` config repo }
+	rs <- Git.Construct.fromRemotes repo'
+	return $ repo' { remotes = rs }
+
+{- Parses git config --list output into a config map. -}
+parse :: String -> M.Map String String
+parse s = M.fromList $ map pair $ lines s
+	where
+		pair = separate (== '=')
diff --git a/Git/Construct.hs b/Git/Construct.hs
new file mode 100644
index 0000000000..9149ab9ec0
--- /dev/null
+++ b/Git/Construct.hs
@@ -0,0 +1,198 @@
+{- Construction of Git Repo objects
+ -
+ - Copyright 2010,2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Construct (
+	fromCwd,
+	fromAbsPath,
+	fromUrl,
+	fromUnknown,
+	localToUrl,
+	fromRemotes,
+	fromRemoteLocation,
+	repoAbsPath,
+) where
+
+import System.Posix.User
+import qualified Data.Map as M hiding (map, split)
+import Network.URI
+
+import Common
+import Git.Types
+import Git
+
+{- Finds the current git repository, which may be in a parent directory. -}
+fromCwd :: IO Repo
+fromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
+	where
+		makerepo = return . newFrom . Dir
+		norepo = error "Not in a git repository."
+
+{- Local Repo constructor, requires an absolute path to the repo be
+ - specified. -}
+fromAbsPath :: FilePath -> IO Repo
+fromAbsPath dir
+	| "/" `isPrefixOf` dir = do
+ 		-- Git always looks for "dir.git" in preference to
+		-- to "dir", even if dir ends in a "/".
+		let canondir = dropTrailingPathSeparator dir
+		let dir' = canondir ++ ".git"
+		e <- doesDirectoryExist dir'
+		if e
+			then ret dir'
+			else if "/.git" `isSuffixOf` canondir
+				then do
+					-- When dir == "foo/.git", git looks
+					-- for "foo/.git/.git", and failing
+					-- that, uses "foo" as the repository.
+					e' <- doesDirectoryExist $ dir  ".git"
+					if e'
+						then ret dir
+						else ret $ takeDirectory canondir
+				else ret dir
+	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
+	where
+		ret = return . newFrom . Dir
+
+{- Remote Repo constructor. Throws exception on invalid url. -}
+fromUrl :: String -> IO Repo
+fromUrl url
+	| startswith "file://" url = fromAbsPath $ uriPath u
+	| otherwise = return $ newFrom $ Url u
+		where
+			u = fromMaybe bad $ parseURI url
+			bad = error $ "bad url " ++ url
+
+{- Creates a repo that has an unknown location. -}
+fromUnknown :: Repo
+fromUnknown = newFrom Unknown
+
+{- Converts a local Repo into a remote repo, using the reference repo
+ - which is assumed to be on the same host. -}
+localToUrl :: Repo -> Repo -> Repo
+localToUrl reference r
+	| not $ repoIsUrl reference = error "internal error; reference repo not url"
+	| repoIsUrl r = r
+	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
+	where
+		absurl =
+			urlScheme reference ++ "//" ++
+			urlAuthority reference ++
+			workTree r
+
+{- Calculates a list of a repo's configured remotes, by parsing its config. -}
+fromRemotes :: Repo -> IO [Repo]
+fromRemotes repo = mapM construct remotepairs
+	where
+		filterconfig f = filter f $ M.toList $ config repo
+		filterkeys f = filterconfig (\(k,_) -> f k)
+		remotepairs = filterkeys isremote
+		isremote k = startswith "remote." k && endswith ".url" k
+		construct (k,v) = repoRemoteNameFromKey k <$> fromRemoteLocation v repo
+
+{- Constructs a new Repo for one of a Repo's remotes using a given
+ - location (ie, an url). -}
+fromRemoteLocation :: String -> Repo -> IO Repo
+fromRemoteLocation s repo = gen $ calcloc s
+	where
+		filterconfig f = filter f $ M.toList $ config repo
+		gen v	
+			| scpstyle v = fromUrl $ scptourl v
+			| isURI v = fromUrl v
+			| otherwise = fromRemotePath v repo
+		-- insteadof config can rewrite remote location
+		calcloc l
+			| null insteadofs = l
+			| otherwise = replacement ++ drop (length bestvalue) l
+			where
+				replacement = drop (length prefix) $
+					take (length bestkey - length suffix) bestkey
+				(bestkey, bestvalue) = maximumBy longestvalue insteadofs
+				longestvalue (_, a) (_, b) = compare b a
+				insteadofs = filterconfig $ \(k, v) -> 
+					startswith prefix k &&
+					endswith suffix k &&
+					startswith v l
+				(prefix, suffix) = ("url." , ".insteadof")
+		-- git remotes can be written scp style -- [user@]host:dir
+		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
+		scptourl v = "ssh://" ++ host ++ slash dir
+			where
+				(host, dir) = separate (== ':') v
+				slash d	| d == "" = "/~/" ++ d
+					| "/" `isPrefixOf` d = d
+					| "~" `isPrefixOf` d = '/':d
+					| otherwise = "/~/" ++ d
+
+{- Constructs a Repo from the path specified in the git remotes of
+ - another Repo. -}
+fromRemotePath :: FilePath -> Repo -> IO Repo
+fromRemotePath dir repo = do
+	dir' <- expandTilde dir
+	fromAbsPath $ workTree repo  dir'
+
+{- Git remotes can have a directory that is specified relative
+ - to the user's home directory, or that contains tilde expansions.
+ - This converts such a directory to an absolute path.
+ - Note that it has to run on the system where the remote is.
+ -}
+repoAbsPath :: FilePath -> IO FilePath
+repoAbsPath d = do
+	d' <- expandTilde d
+	h <- myHomeDir
+	return $ h  d'
+
+expandTilde :: FilePath -> IO FilePath
+expandTilde = expandt True
+	where
+		expandt _ [] = return ""
+		expandt _ ('/':cs) = do
+			v <- expandt True cs
+			return ('/':v)
+		expandt True ('~':'/':cs) = do
+			h <- myHomeDir
+			return $ h  cs
+		expandt True ('~':cs) = do
+			let (name, rest) = findname "" cs
+			u <- getUserEntryForName name
+			return $ homeDirectory u  rest
+		expandt _ (c:cs) = do
+			v <- expandt False cs
+			return (c:v)
+		findname n [] = (n, "")
+		findname n (c:cs)
+			| c == '/' = (n, cs)
+			| otherwise = findname (n++[c]) cs
+
+seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
+seekUp want dir = do
+	ok <- want dir
+	if ok
+		then return $ Just dir
+		else case parentDir dir of
+			"" -> return Nothing
+			d -> seekUp want d
+
+isRepoTop :: FilePath -> IO Bool
+isRepoTop dir = do
+	r <- isRepo
+	b <- isBareRepo
+	return (r || b)
+	where
+		isRepo = gitSignature ".git" ".git/config"
+		isBareRepo = gitSignature "objects" "config"
+		gitSignature subdir file = liftM2 (&&)
+			(doesDirectoryExist (dir ++ "/" ++ subdir))
+			(doesFileExist (dir ++ "/" ++ file))
+
+newFrom :: RepoLocation -> Repo
+newFrom l = 
+	Repo {
+		location = l,
+		config = M.empty,
+		remotes = [],
+		remoteName = Nothing
+	}
diff --git a/Git/Types.hs b/Git/Types.hs
new file mode 100644
index 0000000000..250da5f5e5
--- /dev/null
+++ b/Git/Types.hs
@@ -0,0 +1,36 @@
+{- git data types
+ -
+ - Copyright 2010,2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Types where
+
+import Network.URI
+import qualified Data.Map as M
+
+{- There are two types of repositories; those on local disk and those
+ - accessed via an URL. -}
+data RepoLocation = Dir FilePath | Url URI | Unknown
+	deriving (Show, Eq)
+
+data Repo = Repo {
+	location :: RepoLocation,
+	config :: M.Map String String,
+	remotes :: [Repo],
+	-- remoteName holds the name used for this repo in remotes
+	remoteName :: Maybe String 
+} deriving (Show, Eq)
+
+{- A git ref. Can be a sha1, or a branch or tag name. -}
+newtype Ref = Ref String
+	deriving (Eq)
+
+instance Show Ref where
+	show (Ref v) = v
+
+{- Aliases for Ref. -}
+type Branch = Ref
+type Sha = Ref
+type Tag = Ref
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7871638e44..a5b9609b6d 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -10,7 +10,8 @@ module GitAnnex where
 import System.Console.GetOpt
 
 import Common.Annex
-import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import CmdLine
 import Command
 import Types.TrustLevel
@@ -125,11 +126,11 @@ options = commonOptions ++
 		setprint0 v = Annex.changeState $ \s -> s { Annex.print0 = v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
-			newg <- inRepo $ Git.configStore v
+			newg <- inRepo $ Git.Config.store v
 			Annex.changeState $ \s -> s { Annex.repo = newg }
 
 header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header Git.repoFromCwd
+run args = dispatch args cmds options header Git.Construct.fromCwd
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index e705bbb344..4d63d88e1d 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,6 +15,8 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import Config
 import Annex.Ssh
 import Remote.Helper.Special
@@ -163,8 +165,8 @@ storeBupUUID u buprepo = do
 				[Params $ "config annex.uuid " ++ v]
 					>>! error "ssh failed"
 		else liftIO $ do
-			r' <- Git.configRead r
-			let olduuid = Git.configGet "annex.uuid" "" r'
+			r' <- Git.Config.read r
+			let olduuid = Git.Config.get "annex.uuid" "" r'
 			when (olduuid == "") $
 				Git.run "config"
 					[Param "annex.uuid", Param v] r'
@@ -192,9 +194,9 @@ getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo)
 getBupUUID r u
 	| Git.repoIsUrl r = return (u, r)
 	| otherwise = liftIO $ do
-		ret <- try $ Git.configRead r
+		ret <- try $ Git.Config.read r
 		case ret of
-			Right r' -> return (toUUID $ Git.configGet "annex.uuid" "" r', r')
+			Right r' -> return (toUUID $ Git.Config.get "annex.uuid" "" r', r')
 			Left _ -> return (NoUUID, r)
 
 {- Converts a bup remote path spec into a Git.Repo. There are some
@@ -203,13 +205,13 @@ bup2GitRemote :: BupRepo -> IO Git.Repo
 bup2GitRemote "" = do
 	-- bup -r "" operates on ~/.bup
 	h <- myHomeDir
-	Git.repoFromAbsPath $ h  ".bup"
+	Git.Construct.fromAbsPath $ h  ".bup"
 bup2GitRemote r
 	| bupLocal r = 
 		if head r == '/'
-			then Git.repoFromAbsPath r
+			then Git.Construct.fromAbsPath r
 			else error "please specify an absolute path"
-	| otherwise = Git.repoFromUrl $ "ssh://" ++ host ++ slash dir
+	| otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir
 		where
 			bits = split ":" r
 			host = head bits
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2f9288e1b4..9d80f4c1cb 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -16,6 +16,8 @@ import Utility.RsyncFile
 import Annex.Ssh
 import Types.Remote
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import qualified Annex
 import Annex.UUID
 import qualified Annex.Content
@@ -44,7 +46,7 @@ list = do
 			case M.lookup (annexurl n) c of
 				Nothing -> return r
 				Just url -> Git.repoRemoteNameSet n <$>
-					inRepo (Git.genRemote url)
+					inRepo (Git.Construct.fromRemoteLocation url)
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
@@ -100,7 +102,7 @@ tryGitConfigRead r
 
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
-				Git.hConfigRead r
+				Git.Config.hRead r
 
 		geturlconfig = do
 			s <- Url.get (Git.repoLocation r ++ "/config")
@@ -108,7 +110,7 @@ tryGitConfigRead r
 				hPutStr h s
 				hClose h
 				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
-					Git.hConfigRead r
+					Git.Config.hRead r
 
 		store a = do
 			r' <- a
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 77478eb1d5..72c4842d88 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,6 +12,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Construct
 
 {- Special remotes don't have a configured url, so Git.Repo does not
  - automatically generate remotes for them. This looks for a different
@@ -23,7 +24,7 @@ findSpecialRemotes s = do
 	return $ map construct $ remotepairs m
 	where
 		remotepairs = M.toList . M.filterWithKey match
-		construct (k,_) = Git.repoRemoteNameFromKey k Git.repoFromUnknown
+		construct (k,_) = Git.repoRemoteNameFromKey k Git.Construct.fromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index d5acd7d862..c4e9f8bd69 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -10,6 +10,7 @@ module Remote.Web (remote) where
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Construct
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
@@ -26,7 +27,7 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet "web" Git.repoFromUnknown]
+list = return [Git.repoRemoteNameSet "web" Git.Construct.fromUnknown]
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 9a9d2f0925..872dabc58b 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -9,7 +9,7 @@ import System.Environment
 import System.Console.GetOpt
 
 import Common.Annex
-import qualified Git
+import qualified Git.Construct
 import CmdLine
 import Command
 import Annex.UUID
@@ -80,7 +80,7 @@ builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
 	dispatch (cmd : filterparams params) cmds options header $
-		Git.repoAbsPath dir >>= Git.repoFromAbsPath
+		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
 
 external :: [String] -> IO ()
 external params = do
diff --git a/git-annex.cabal b/git-annex.cabal
index 35b3e690f9..ae6a129b3b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111211
+Version: 3.20111212
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index edd9330c80..eeb6944018 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -9,6 +9,8 @@ import System.Environment
 
 import Common
 import qualified Git.UnionMerge
+import qualified Git.Config
+import qualified Git.Construct
 import qualified Git
 
 header :: String
@@ -38,7 +40,7 @@ parseArgs = do
 main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
-	g <- Git.configRead =<< Git.repoFromCwd
+	g <- Git.Config.read =<< Git.Construct.fromCwd
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
diff --git a/test.hs b/test.hs
index 91c11873da..14994865f8 100644
--- a/test.hs
+++ b/test.hs
@@ -25,6 +25,8 @@ import qualified Annex
 import qualified Annex.UUID
 import qualified Backend
 import qualified Git
+import qualified Git.Config
+import qualified Git.Construct
 import qualified Locations
 import qualified Types.Backend
 import qualified Types
@@ -496,8 +498,8 @@ git_annex command params = do
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a
 annexeval a = do
-	g <- Git.repoFromCwd
-	g' <- Git.configRead g
+	g <- Git.Construct.fromCwd
+	g' <- Git.Config.read g
 	s <- Annex.new g'
 	Annex.eval s a
 

From 25b2cc4148e4cc8f7435cdbcf4b124cc317c1305 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:08:44 -0400
Subject: [PATCH 2690/8313] move commit to Git.Branch

---
 Annex/Branch.hs |  4 ++--
 Git.hs          | 18 ------------------
 Git/Branch.hs   | 17 +++++++++++++++++
 3 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index f9fa6cbb31..a2ecd50a70 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -71,7 +71,7 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 			fromMaybe (error $ "failed to create " ++ show name)
 				<$> branchsha
 		go False = withIndex' True $ do
-			inRepo $ Git.commit "branch created" fullname []
+			inRepo $ Git.Branch.commit "branch created" fullname []
 		use sha = do
 			setIndexSha sha
 			return sha
@@ -190,7 +190,7 @@ commit message = whenM journalDirty $ lockJournal $ do
 commitBranch :: Git.Ref -> String -> [Git.Ref] -> Annex ()
 commitBranch branchref message parents = do
 	updateIndex branchref
-	committedref <- inRepo $ Git.commit message fullname parents
+	committedref <- inRepo $ Git.Branch.commit message fullname parents
 	setIndexSha committedref
 	parentrefs <- commitparents <$> catObject committedref
 	when (racedetected branchref parentrefs) $
diff --git a/Git.hs b/Git.hs
index cb7cc19c25..36b83c65b5 100644
--- a/Git.hs
+++ b/Git.hs
@@ -52,7 +52,6 @@ module Git (
 	useIndex,
 	getSha,
 	shaSize,
-	commit,
 	assertLocal,
 
 	prop_idempotent_deencode
@@ -354,23 +353,6 @@ getSha subcommand a = do
 shaSize :: Int
 shaSize = 40
 
-{- Commits the index into the specified branch (or other ref), 
- - with the specified parent refs, and returns the committed sha -}
-commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
-commit message branch parentrefs repo = do
-	tree <- getSha "write-tree" $ asString $
-		pipeRead [Param "write-tree"] repo
-	sha <- getSha "commit-tree" $ asString $
-		ignorehandle $ pipeWriteRead
-			(map Param $ ["commit-tree", show tree] ++ ps)
-			(L.pack message) repo
-	run "update-ref" [Param $ show branch, Param $ show sha] repo
-	return sha
-	where
-		ignorehandle a = snd <$> a
-		asString a = L.unpack <$> a
-		ps = concatMap (\r -> ["-p", show r]) parentrefs
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
diff --git a/Git/Branch.hs b/Git/Branch.hs
index e69e96f288..8b0d1e5afc 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -58,3 +58,20 @@ fastForward branch (first:rest) repo = do
 				(True, False) -> findbest r rs -- better
 				(False, True) -> findbest c rs -- worse
 				(False, False) -> findbest c rs -- same
+
+{- Commits the index into the specified branch (or other ref), 
+ - with the specified parent refs, and returns the committed sha -}
+commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
+commit message branch parentrefs repo = do
+	tree <- getSha "write-tree" $ asString $
+		pipeRead [Param "write-tree"] repo
+	sha <- getSha "commit-tree" $ asString $
+		ignorehandle $ pipeWriteRead
+			(map Param $ ["commit-tree", show tree] ++ ps)
+			(L.pack message) repo
+	run "update-ref" [Param $ show branch, Param $ show sha] repo
+	return sha
+	where
+		ignorehandle a = snd <$> a
+		asString a = L.unpack <$> a
+		ps = concatMap (\r -> ["-p", show r]) parentrefs

From 9db8ec210f8491de78cd7e83b94c50ead8049e72 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:22:43 -0400
Subject: [PATCH 2691/8313] split out two more Git modules

---
 Backend.hs         |   3 +-
 Git.hs             | 113 +--------------------------------------------
 Git/CheckAttr.hs   |  44 ++++++++++++++++++
 Git/Filename.hs    |  84 +++++++++++++++++++++++++++++++++
 Git/LsTree.hs      |   3 +-
 Seek.hs            |   3 +-
 git-union-merge.hs |   3 +-
 test.hs            |   3 +-
 8 files changed, 139 insertions(+), 117 deletions(-)
 create mode 100644 Git/CheckAttr.hs
 create mode 100644 Git/Filename.hs

diff --git a/Backend.hs b/Backend.hs
index c7cb57440b..38f0629f48 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -22,6 +22,7 @@ import System.Posix.Files
 import Common.Annex
 import qualified Git
 import qualified Git.Config
+import qualified Git.CheckAttr
 import qualified Annex
 import Types.Key
 import qualified Types.Backend as B
@@ -103,7 +104,7 @@ chooseBackends :: [FilePath] -> Annex [BackendFile]
 chooseBackends fs = Annex.getState Annex.forcebackend >>= go
 	where
 		go Nothing = do
-			pairs <- inRepo $ Git.checkAttr "annex.backend" fs
+			pairs <- inRepo $ Git.CheckAttr.lookup "annex.backend" fs
 			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 		go (Just _) = do
 			l <- orderedList
diff --git a/Git.hs b/Git.hs
index 36b83c65b5..0280acedcd 100644
--- a/Git.hs
+++ b/Git.hs
@@ -45,25 +45,16 @@ module Git (
 	repoRemoteName,
 	repoRemoteNameSet,
 	repoRemoteNameFromKey,
-	checkAttr,
-	decodeGitFile,
-	encodeGitFile,
 	reap,
 	useIndex,
 	getSha,
 	shaSize,
 	assertLocal,
-
-	prop_idempotent_deencode
 ) where
 
-import qualified Data.Map as M hiding (map, split)
+import qualified Data.Map as M
 import Network.URI
 import Data.Char
-import Data.Word (Word8)
-import Codec.Binary.UTF8.String (encode)
-import Text.Printf
-import System.Exit
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -360,105 +351,3 @@ configTrue s = map toLower s == "true"
 {- Access to raw config Map -}
 configMap :: Repo -> M.Map String String
 configMap = config
-
-{- Efficiently looks up a gitattributes value for each file in a list. -}
-checkAttr :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
-checkAttr attr files repo = do
-	-- git check-attr needs relative filenames input; it will choke
-	-- on some absolute filenames. This also means it will output
-	-- all relative filenames.
-	cwd <- getCurrentDirectory
-	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
-	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
-        _ <- forkProcess $ do
-		hClose fromh
-                hPutStr toh $ join "\0" relfiles
-                hClose toh
-                exitSuccess
-        hClose toh
-	(map topair . lines) <$> hGetContents fromh
-	where
-		params = gitCommandLine 
-				[ Param "check-attr"
-				, Param attr
-				, Params "-z --stdin"
-				] repo
-		topair l = (file, value)
-			where 
-				file = decodeGitFile $ join sep $ take end bits
-				value = bits !! end
-				end = length bits - 1
-				bits = split sep l
-				sep = ": " ++ attr ++ ": "
-
-{- Some git commands output encoded filenames. Decode that (annoyingly
- - complex) encoding. -}
-decodeGitFile :: String -> FilePath
-decodeGitFile [] = []
-decodeGitFile f@(c:s)
-	-- encoded strings will be inside double quotes
-	| c == '"' = unescape ("", middle)
-	| otherwise = f
-	where
-		e = '\\'
-		middle = init s
-		unescape (b, []) = b
-		-- look for escapes starting with '\'
-		unescape (b, v) = b ++ beginning ++ unescape (decode rest)
-			where
-				pair = span (/= e) v
-				beginning = fst pair
-				rest = snd pair
-		isescape x = x == e
-		-- \NNN is an octal encoded character
-		decode (x:n1:n2:n3:rest)
-			| isescape x && alloctal = (fromoctal, rest)
-				where
-					alloctal = isOctDigit n1 &&
-						isOctDigit n2 &&
-						isOctDigit n3
-					fromoctal = [chr $ readoctal [n1, n2, n3]]
-					readoctal o = read $ "0o" ++ o :: Int
-		-- \C is used for a few special characters
-		decode (x:nc:rest)
-			| isescape x = ([echar nc], rest)
-			where
-				echar 'a' = '\a'
-				echar 'b' = '\b'
-				echar 'f' = '\f'
-				echar 'n' = '\n'
-				echar 'r' = '\r'
-				echar 't' = '\t'
-				echar 'v' = '\v'
-				echar a = a
-		decode n = ("", n)
-
-{- Should not need to use this, except for testing decodeGitFile. -}
-encodeGitFile :: FilePath -> String
-encodeGitFile s = foldl (++) "\"" (map echar s) ++ "\""
-	where
-		e c = '\\' : [c]
-		echar '\a' = e 'a'
-		echar '\b' = e 'b'
-		echar '\f' = e 'f'
-		echar '\n' = e 'n'
-		echar '\r' = e 'r'
-		echar '\t' = e 't'
-		echar '\v' = e 'v'
-		echar '\\' = e '\\'
-		echar '"'  = e '"'
-		echar x
-			| ord x < 0x20 = e_num x -- low ascii
-			| ord x >= 256 = e_utf x
-			| ord x > 0x7E = e_num x -- high ascii
-			| otherwise = [x]        -- printable ascii
-			where 
-				showoctal i = '\\' : printf "%03o" i
-				e_num c = showoctal $ ord c
-				-- unicode character is decomposed to
-				-- Word8s and each is shown in octal
-				e_utf c = showoctal =<< (encode [c] :: [Word8])
-
-{- for quickcheck -}
-prop_idempotent_deencode :: String -> Bool
-prop_idempotent_deencode s = s == decodeGitFile (encodeGitFile s)
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
new file mode 100644
index 0000000000..e9269b1edb
--- /dev/null
+++ b/Git/CheckAttr.hs
@@ -0,0 +1,44 @@
+{- git check-attr interface
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.CheckAttr where
+
+import System.Exit
+
+import Common
+import Git
+import qualified Git.Filename
+
+{- Efficiently looks up a gitattributes value for each file in a list. -}
+lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
+lookup attr files repo = do
+	-- git check-attr needs relative filenames input; it will choke
+	-- on some absolute filenames. This also means it will output
+	-- all relative filenames.
+	cwd <- getCurrentDirectory
+	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
+	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
+        _ <- forkProcess $ do
+		hClose fromh
+                hPutStr toh $ join "\0" relfiles
+                hClose toh
+                exitSuccess
+        hClose toh
+	(map topair . lines) <$> hGetContents fromh
+	where
+		params = gitCommandLine 
+				[ Param "check-attr"
+				, Param attr
+				, Params "-z --stdin"
+				] repo
+		topair l = (file, value)
+			where 
+				file = Git.Filename.decode $ join sep $ take end bits
+				value = bits !! end
+				end = length bits - 1
+				bits = split sep l
+				sep = ": " ++ attr ++ ": "
diff --git a/Git/Filename.hs b/Git/Filename.hs
new file mode 100644
index 0000000000..69f36d0861
--- /dev/null
+++ b/Git/Filename.hs
@@ -0,0 +1,84 @@
+{- Some git commands output encoded filenames, in a rather annoyingly complex
+ - C-style encoding.
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Filename where
+
+import qualified Codec.Binary.UTF8.String
+import Data.Char
+import Data.Word (Word8)
+import Text.Printf
+
+decode :: String -> FilePath
+decode [] = []
+decode f@(c:s)
+	-- encoded strings will be inside double quotes
+	| c == '"' = unescape ("", middle)
+	| otherwise = f
+	where
+		e = '\\'
+		middle = init s
+		unescape (b, []) = b
+		-- look for escapes starting with '\'
+		unescape (b, v) = b ++ beginning ++ unescape (handle rest)
+			where
+				pair = span (/= e) v
+				beginning = fst pair
+				rest = snd pair
+		isescape x = x == e
+		-- \NNN is an octal encoded character
+		handle (x:n1:n2:n3:rest)
+			| isescape x && alloctal = (fromoctal, rest)
+				where
+					alloctal = isOctDigit n1 &&
+						isOctDigit n2 &&
+						isOctDigit n3
+					fromoctal = [chr $ readoctal [n1, n2, n3]]
+					readoctal o = read $ "0o" ++ o :: Int
+		-- \C is used for a few special characters
+		handle (x:nc:rest)
+			| isescape x = ([echar nc], rest)
+			where
+				echar 'a' = '\a'
+				echar 'b' = '\b'
+				echar 'f' = '\f'
+				echar 'n' = '\n'
+				echar 'r' = '\r'
+				echar 't' = '\t'
+				echar 'v' = '\v'
+				echar a = a
+		handle n = ("", n)
+
+{- Should not need to use this, except for testing decode. -}
+encode :: FilePath -> String
+encode s = foldl (++) "\"" (map echar s) ++ "\""
+	where
+		e c = '\\' : [c]
+		echar '\a' = e 'a'
+		echar '\b' = e 'b'
+		echar '\f' = e 'f'
+		echar '\n' = e 'n'
+		echar '\r' = e 'r'
+		echar '\t' = e 't'
+		echar '\v' = e 'v'
+		echar '\\' = e '\\'
+		echar '"'  = e '"'
+		echar x
+			| ord x < 0x20 = e_num x -- low ascii
+			| ord x >= 256 = e_utf x
+			| ord x > 0x7E = e_num x -- high ascii
+			| otherwise = [x]        -- printable ascii
+			where 
+				showoctal i = '\\' : printf "%03o" i
+				e_num c = showoctal $ ord c
+				-- unicode character is decomposed to
+				-- Word8s and each is shown in octal
+				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+
+{- for quickcheck -}
+prop_idempotent_deencode :: String -> Bool
+prop_idempotent_deencode s = s == decode (encode s)
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 8aa16a308b..342a125eb8 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -17,6 +17,7 @@ import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import qualified Git.Filename
 import Utility.SafeCommand
 
 data TreeItem = TreeItem
@@ -38,7 +39,7 @@ parseLsTree l = TreeItem
 	{ mode = fst $ head $ readOct $ L.unpack m
 	, typeobj = L.unpack t
 	, sha = L.unpack s
-	, file = decodeGitFile $ L.unpack f
+	, file = Git.Filename.decode $ L.unpack f
 	}
 	where
 		-- l =  SP  SP  TAB 
diff --git a/Seek.hs b/Seek.hs
index 1430ebabd1..28c6ffc00c 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -18,6 +18,7 @@ import Backend
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
+import qualified Git.CheckAttr
 import qualified Limit
 
 seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
@@ -31,7 +32,7 @@ withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
 withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
 withAttrFilesInGit attr a params = do
 	files <- seekHelper LsFiles.inRepo params
-	prepFilteredGen a fst $ inRepo $ Git.checkAttr attr files
+	prepFilteredGen a fst $ inRepo $ Git.CheckAttr.lookup attr files
 
 withNumCopies :: (Maybe Int -> FilePath -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
diff --git a/git-union-merge.hs b/git-union-merge.hs
index eeb6944018..f67414bdd3 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -11,6 +11,7 @@ import Common
 import qualified Git.UnionMerge
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Git.Branch
 import qualified Git
 
 header :: String
@@ -44,5 +45,5 @@ main = do
 	_ <- Git.useIndex (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
-	_ <- Git.commit "union merge" newref [aref, bref] g
+	_ <- Git.Branch.commit "union merge" newref [aref, bref] g
 	cleanup g
diff --git a/test.hs b/test.hs
index 14994865f8..1ce9d103d1 100644
--- a/test.hs
+++ b/test.hs
@@ -27,6 +27,7 @@ import qualified Backend
 import qualified Git
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Git.Filename
 import qualified Locations
 import qualified Types.Backend
 import qualified Types
@@ -69,7 +70,7 @@ propigate (Counts { errors = e , failures = f }, _)
 
 quickcheck :: Test
 quickcheck = TestLabel "quickcheck" $ TestList
-	[ qctest "prop_idempotent_deencode" Git.prop_idempotent_deencode
+	[ qctest "prop_idempotent_deencode" Git.Filename.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape

From 3c851368750a5faf19ff3437854a60ae4c679c69 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 13 Dec 2011 15:27:38 -0400
Subject: [PATCH 2692/8313] remove dead code

---
 Git.hs | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/Git.hs b/Git.hs
index 0280acedcd..b4cbd91aa0 100644
--- a/Git.hs
+++ b/Git.hs
@@ -21,7 +21,6 @@ module Git (
 	repoDescribe,
 	repoLocation,
 	workTree,
-	workTreeFile,
 	gitDir,
 	urlPath,
 	urlHost,
@@ -166,36 +165,6 @@ workTree r@(Repo { location = Url _ }) = urlPath r
 workTree (Repo { location = Dir d }) = d
 workTree Repo { location = Unknown } = undefined
 
-{- Given a relative or absolute filename inside a git repository's
- - workTree, calculates the name to use to refer to that file to git.
- -
- - This is complicated because the best choice can vary depending on
- - whether the cwd is in a subdirectory of the git repository, or not.
- -
- - For example, when adding a file "/tmp/repo/foo", it's best to refer
- - to it as "foo" if the cwd is outside the repository entirely
- - (this avoids a gotcha with using the full path name when /tmp/repo
- - is itself a symlink). But, if the cwd is "/tmp/repo/subdir",
- - it's best to refer to "../foo".
- -}
-workTreeFile :: FilePath -> Repo -> IO FilePath
-workTreeFile file repo@(Repo { location = Dir d }) = do
-	cwd <- getCurrentDirectory
-	let file' = absfile cwd
-	unless (inrepo file') $
-		error $ file ++ " is not located inside git repository " ++ absrepo
-	if inrepo $ addTrailingPathSeparator cwd
-		then return $ relPathDirToFile cwd file'
-		else return $ drop (length absrepo) file'
-	where
-		-- normalize both repo and file, so that repo
-		-- will be substring of file
-		absrepo = maybe bad addTrailingPathSeparator $ absNormPath "/" d
-		absfile c = fromMaybe file $ secureAbsNormPath c file
-		inrepo f = absrepo `isPrefixOf` f
-		bad = error $ "bad repo" ++ repoDescribe repo
-workTreeFile _ repo = assertLocal repo $ error "internal"
-
 {- Path of an URL repo. -}
 urlPath :: Repo -> String
 urlPath Repo { location = Url u } = uriPath u

From 490810ff9fba8e4711fef8025c655b2fa66b8801 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 13 Dec 2011 20:53:23 +0000
Subject: [PATCH 2693/8313] Added a comment

---
 ...mment_6_3660d45c5656f68924acbd23790024ee._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment b/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment
new file mode 100644
index 0000000000..fc66fbb8e1
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_6_3660d45c5656f68924acbd23790024ee._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-12-13T20:53:23Z"
+ content="""
+It would be clearer to call \"git-annex-master\" \"synced/master\" (or really \"synced/$current_branch\"). That does highlight that this method of syncing is not particularly specific to git-annex.
+
+I think this would be annoying to those who do use a central bare repository, because of the unnecessary pushing and pulling to other repos, which could be expensive to do, especially if you have a lot of interconnected repos. So having a way to enable/disable it seems best.
+
+Maybe you should work up a patch to Command/Sync.hs, since I know you know haskell :)
+"""]]

From c92d407efdcff347376a4186abe84fa8f2240996 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Tue, 13 Dec 2011 22:01:13 +0000
Subject: [PATCH 2694/8313] add a link to git-union-merge

---
 doc/internals.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 68cc7c3cd9..b2fd1e5545 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -77,7 +77,7 @@ Example:
 	1287290776.765152s 1 e605dca6-446a-11e0-8b2a-002170d25c55
 	1287290767.478634s 0 26339d22-446b-11e0-9101-002170d25c55
 
-These files are designed to be auto-merged using git's union merge driver.
+These files are designed to be auto-merged using git's [[union merge driver|git-union-merge]].
 The timestamps allow the most recent information to be identified.
 
 ## `remote/web/aaa/bbb/*.log`

From 020c845058b926c608148759b2782e2539841fc3 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Tue, 13 Dec 2011 23:26:58 +0000
Subject: [PATCH 2695/8313]

---
 .../syncing_non-git_trees_with_git-annex.mdwn | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex.mdwn

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex.mdwn b/doc/forum/syncing_non-git_trees_with_git-annex.mdwn
new file mode 100644
index 0000000000..9973782610
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex.mdwn
@@ -0,0 +1,46 @@
+I have a bunch of directory trees with large data files scattered over various computers and disk drives - they contain photos, videos, music, and so on.  In many cases I initially copied one of these trees from one machine to another just as a cheap and dirty backup, and then made small modifications to both trees in ways I no longer remember. For example, I returned from a trip with a bunch of new photos, and then might have rotated some of them 90 degrees on one machine, and edited or renamed them on another.
+
+What I want to do now is use git-annex as a way of initially synchronising the trees, and then fully managing them on an ongoing basis.  Note that the trees are *not* yet git repositories.  In order to be able to detect straight-forward file renames, I believe that [[the SHA1 backend|tips/using_the_SHA1_backend]] probably makes the most sense.
+
+I've been playing around and arrived at the following setup procedure.  For the sake of discussion, I assume that we have two trees `a` and `b` which live in the same directory referred to by `$td`, and that all large files end with the `.avi` suffix.
+
+     # Setup git in 'a'.
+     cd $td/a
+     git init
+
+     # Setup git-annex in 'a'.
+     echo '* annex.backend=SHA1' > .gitattributes
+     git add .gitattributes
+     git commit -m'use SHA1 backend'
+     git annex init
+
+     # Annex all large files.
+     find -name \*.avi | xargs git annex add
+     git add .
+     git commit -m'Initial import'
+
+     # Setup git in 'b'.
+     cd $td/b
+     git clone -n $td/a new
+     mv new/.git .
+     rmdir new
+     git reset # reset git index to b's wd - hangover from cloning from 'a'
+
+     # Setup git-annex in 'b'.
+     # This merges a's (origin's) git-annex branch into the local git-annex branch.
+     git annex init
+
+     # Annex all large files - because we're using SHA1 backend, some
+     # should hash to the same keys as in 'a'.
+     find -name \*.avi | xargs git annex add
+     git add .
+     git commit -m'Changes in b tree'
+
+     git remote add a $td/a
+
+     # Now pull changes in 'b' back to 'a'.
+     cd $td/a
+     git remote add b $td/b
+     git pull b master
+
+This seems to work, but have I missed anything?

From 25a5f6664ed98a20bc59bc0ca3107dd8bf7bba0b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 14 Dec 2011 17:31:31 +0000
Subject: [PATCH 2696/8313] Added a comment

---
 ..._7f9593bdfd95e4a8814e6cc5c44619e6._comment | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
new file mode 100644
index 0000000000..c1b9cfa370
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-14T17:31:31Z"
+ content="""
+This is an entirely reasonable way to go about it.
+
+However, doing it this way causes files in B to always \"win\" -- If the same filename is in both repositories, with differing content, the version added in B will superscede the version from A. If A has a file that is not in B, a git commit -a in B will commit a deletion of that file.
+
+I might do it your way and look at the changes in B before (or even after) committing them to see if files from A were deleted or changed.
+
+Or, I might just instead keep B in a separate subdirectory in the repository, set up like so:
+
+
+mv b old_b
+git clone a b
+cd b
+mv ../old_b .
+git annex add old_b --exclude --not '*.avi'
+

+
+Or, a third way would be to commit A to a branch like branchA and B to a separate branchB, and not merge the branches at all.
+"""]]

From 2b24e16a633575703a43e1fb991f34b290a1d7e4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Dec 2011 13:32:13 -0400
Subject: [PATCH 2697/8313] typo

---
 .../comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
index c1b9cfa370..bdec508792 100644
--- a/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_1_7f9593bdfd95e4a8814e6cc5c44619e6._comment
@@ -17,7 +17,7 @@ mv b old_b
 git clone a b
 cd b
 mv ../old_b .
-git annex add old_b --exclude --not '*.avi'
+git annex add old_b --not --exclude '*.avi'
 

 
 Or, a third way would be to commit A to a branch like branchA and B to a separate branchB, and not merge the branches at all.

From 02f1bd2bf47d3ff49a222e9428ec27708ef55f64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Dec 2011 15:30:14 -0400
Subject: [PATCH 2698/8313] split more stuff out of Git.hs

---
 Annex/Branch.hs          |   3 +-
 Annex/Ssh.hs             |   5 +-
 Backend.hs               |   1 -
 Command/Map.hs           |  13 ++--
 Config.hs                |   2 +-
 Git.hs                   | 148 +++------------------------------------
 Git/Branch.hs            |   5 +-
 Git/CatFile.hs           |   5 +-
 Git/Construct.hs         |  27 +++++--
 Git/HashObject.hs        |   4 +-
 Git/Index.hs             |  24 +++++++
 Git/Ref.hs               |   6 +-
 Git/Sha.hs               |  27 +++++++
 Git/UnionMerge.hs        |   5 +-
 Git/Url.hs               |  70 ++++++++++++++++++
 Remote/Git.hs            |  17 ++---
 Remote/Helper/Special.hs |   6 +-
 Remote/Web.hs            |   4 +-
 git-union-merge.hs       |   3 +-
 test.hs                  |   1 -
 20 files changed, 197 insertions(+), 179 deletions(-)
 create mode 100644 Git/Index.hs
 create mode 100644 Git/Sha.hs
 create mode 100644 Git/Url.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a2ecd50a70..a22a4adcfe 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -28,6 +28,7 @@ import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.UnionMerge
 import qualified Git.HashObject
+import qualified Git.Index
 import Annex.CatFile
 
 {- Name of the branch that is used to store git-annex's information. -}
@@ -249,7 +250,7 @@ withIndex = withIndex' False
 withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
 	f <- fromRepo gitAnnexIndex
-	bracketIO (Git.useIndex f) id $ do
+	bracketIO (Git.Index.override f) id $ do
 		unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 6893f94ef4..fe83aad001 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -10,6 +10,7 @@ module Annex.Ssh where
 import Control.Monad.State (liftIO)
 
 import qualified Git
+import qualified Git.Url
 import Utility.SafeCommand
 import Types
 import Config
@@ -22,10 +23,10 @@ sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
 	s <- getConfig repo "ssh-options" ""
 	let sshoptions = map Param (words s)
-	let sshport = case Git.urlPort repo of
+	let sshport = case Git.Url.port repo of
 		Nothing -> []
 		Just p -> [Param "-p", Param (show p)]
-	let sshhost = Param $ Git.urlHostUser repo
+	let sshhost = Param $ Git.Url.hostuser repo
 	return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd
 
 {- Generates parameters to run a git-annex-shell command on a remote
diff --git a/Backend.hs b/Backend.hs
index 38f0629f48..4743bb202e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -20,7 +20,6 @@ import System.IO.Error (try)
 import System.Posix.Files
 
 import Common.Annex
-import qualified Git
 import qualified Git.Config
 import qualified Git.CheckAttr
 import qualified Annex
diff --git a/Command/Map.hs b/Command/Map.hs
index 815b142e72..ae8a694043 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -13,6 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Command
 import qualified Git
+import qualified Git.Url
 import qualified Git.Config
 import qualified Git.Construct
 import Annex.UUID
@@ -68,7 +69,7 @@ drawMap rs umap ts = Dot.graph $ repos ++ trusted ++ others
 
 hostname :: Git.Repo -> String
 hostname r
-	| Git.repoIsUrl r = Git.urlHost r
+	| Git.repoIsUrl r = Git.Url.host r
 	| otherwise = "localhost"
 
 basehostname :: Git.Repo -> String
@@ -82,7 +83,7 @@ repoName umap r
 	| otherwise = M.findWithDefault fallback repouuid umap
 	where
 		repouuid = getUncachedUUID r
-		fallback = fromMaybe "unknown" $ Git.repoRemoteName r
+		fallback = fromMaybe "unknown" $ Git.remoteName r
 
 {- A unique id for the node for a repo. Uses the annex.uuid if available. -}
 nodeId :: Git.Repo -> String
@@ -99,7 +100,7 @@ node umap fullinfo r = unlines $ n:edges
 			decorate $ Dot.graphNode (nodeId r) (repoName umap r)
 		edges = map (edge umap fullinfo r) (Git.remotes r)
 		decorate
-			| Git.configMap r == M.empty = unreachable
+			| Git.config r == M.empty = unreachable
 			| otherwise = reachable
 
 {- An edge between two repos. The second repo is a remote of the first. -}
@@ -116,7 +117,7 @@ edge umap fullinfo from to =
 		{- Only name an edge if the name is different than the name
 		 - that will be used for the destination node, and is
 		 - different from its hostname. (This reduces visual clutter.) -}
-		edgename = maybe Nothing calcname $ Git.repoRemoteName to
+		edgename = maybe Nothing calcname $ Git.remoteName to
 		calcname n
 			| n `elem` [repoName umap fullto, hostname fullto] = Nothing
 			| otherwise = Just n
@@ -141,7 +142,7 @@ spider' (r:rs) known
 		-- The remotes will be relative to r', and need to be
 		-- made absolute for later use.
 		remotes <- mapM (absRepo r') (Git.remotes r')
-		let r'' = Git.remotesAdd r' remotes
+		let r'' = r' { Git.remotes = remotes }
 
 		spider' (rs ++ remotes) (r'':known)
 
@@ -154,7 +155,7 @@ absRepo reference r
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
 same a b
-	| both Git.repoIsSsh = matching Git.urlAuthority && matching Git.workTree
+	| both Git.repoIsSsh = matching Git.Url.authority && matching Git.workTree
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
 	| neither Git.repoIsSsh = matching Git.workTree
 	| otherwise = False
diff --git a/Config.hs b/Config.hs
index 322dc8af82..07c9eedad7 100644
--- a/Config.hs
+++ b/Config.hs
@@ -31,7 +31,7 @@ getConfig r key def = do
 
 {- Looks up a per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
-remoteConfig r key = "remote." ++ fromMaybe "" (Git.repoRemoteName r) ++ ".annex-" ++ key
+remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key
 
 {- Calculates cost for a remote. Either the default, or as configured 
  - by remote..annex-cost, or if remote..annex-cost-command
diff --git a/Git.hs b/Git.hs
index b4cbd91aa0..a3f2ad74cb 100644
--- a/Git.hs
+++ b/Git.hs
@@ -9,7 +9,7 @@
  -}
 
 module Git (
-	Repo,
+	Repo(..),
 	Ref(..),
 	Branch,
 	Sha,
@@ -22,13 +22,6 @@ module Git (
 	repoLocation,
 	workTree,
 	gitDir,
-	urlPath,
-	urlHost,
-	urlPort,
-	urlHostUser,
-	urlAuthority,
-	urlScheme,
-	configMap,
 	configTrue,
 	gitCommandLine,
 	run,
@@ -39,23 +32,14 @@ module Git (
 	pipeNullSplit,
 	pipeNullSplitB,
 	attributes,
-	remotes,
-	remotesAdd,
-	repoRemoteName,
-	repoRemoteNameSet,
-	repoRemoteNameFromKey,
 	reap,
-	useIndex,
-	getSha,
-	shaSize,
 	assertLocal,
 ) where
 
 import qualified Data.Map as M
-import Network.URI
 import Data.Char
-import System.Posix.Env (setEnv, unsetEnv, getEnv)
 import qualified Data.ByteString.Lazy.Char8 as L
+import Network.URI (uriPath, uriScheme)
 
 import Common
 import Git.Types
@@ -73,29 +57,6 @@ repoLocation Repo { location = Url url } = show url
 repoLocation Repo { location = Dir dir } = dir
 repoLocation Repo { location = Unknown } = undefined
 
-{- Constructs and returns an updated version of a repo with
- - different remotes list. -}
-remotesAdd :: Repo -> [Repo] -> Repo
-remotesAdd repo rs = repo { remotes = rs }
-
-{- Returns the name of the remote that corresponds to the repo, if
- - it is a remote. -}
-repoRemoteName :: Repo -> Maybe String
-repoRemoteName Repo { remoteName = Just name } = Just name
-repoRemoteName _ = Nothing
-
-{- Sets the name of a remote. -}
-repoRemoteNameSet :: String -> Repo -> Repo
-repoRemoteNameSet n r = r { remoteName = Just n }
-
-{- Sets the name of a remote based on the git config key, such as
-   "remote.foo.url". -}
-repoRemoteNameFromKey :: String -> Repo -> Repo
-repoRemoteNameFromKey k = repoRemoteNameSet basename
-	where
-		basename = join "." $ reverse $ drop 1 $
-				reverse $ drop 1 $ split "." k
-
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
 repoIsUrl :: Repo -> Bool
@@ -104,11 +65,13 @@ repoIsUrl _ = False
 
 repoIsSsh :: Repo -> Bool
 repoIsSsh Repo { location = Url url } 
-	| uriScheme url == "ssh:" = True
+	| scheme == "ssh:" = True
 	-- git treats these the same as ssh
-	| uriScheme url == "git+ssh:" = True
-	| uriScheme url == "ssh+git:" = True
+	| scheme == "git+ssh:" = True
+	| scheme == "ssh+git:" = True
 	| otherwise = False
+	where
+		scheme = uriScheme url
 repoIsSsh _ = False
 
 repoIsHttp :: Repo -> Bool
@@ -129,15 +92,8 @@ assertLocal :: Repo -> a -> a
 assertLocal repo action = 
 	if not $ repoIsUrl repo
 		then action
-		else error $ "acting on URL git repo " ++  repoDescribe repo ++ 
+		else error $ "acting on non-local git repo " ++  repoDescribe repo ++ 
 				" not supported"
-assertUrl :: Repo -> a -> a
-assertUrl repo action = 
-	if repoIsUrl repo
-		then action
-		else error $ "acting on local git repo " ++  repoDescribe repo ++ 
-				" not supported"
-
 configBare :: Repo -> Bool
 configBare repo = maybe unknown configTrue $ M.lookup "core.bare" $ config repo
 	where
@@ -161,61 +117,10 @@ gitDir repo
  -
  - Note that for URL repositories, this is the path on the remote host. -}
 workTree :: Repo -> FilePath
-workTree r@(Repo { location = Url _ }) = urlPath r
-workTree (Repo { location = Dir d }) = d
+workTree Repo { location = Url u } = uriPath u
+workTree Repo { location = Dir d } = d
 workTree Repo { location = Unknown } = undefined
 
-{- Path of an URL repo. -}
-urlPath :: Repo -> String
-urlPath Repo { location = Url u } = uriPath u
-urlPath repo = assertUrl repo $ error "internal"
-
-{- Scheme of an URL repo. -}
-urlScheme :: Repo -> String
-urlScheme Repo { location = Url u } = uriScheme u
-urlScheme repo = assertUrl repo $ error "internal"
-
-{- Work around a bug in the real uriRegName
- -  -}
-uriRegName' :: URIAuth -> String
-uriRegName' a = fixup $ uriRegName a
-	where
-		fixup x@('[':rest)
-			| rest !! len == ']' = take len rest
-			| otherwise = x
-			where
-				len  = length rest - 1
-		fixup x = x
-
-{- Hostname of an URL repo. -}
-urlHost :: Repo -> String
-urlHost = urlAuthPart uriRegName'
-
-{- Port of an URL repo, if it has a nonstandard one. -}
-urlPort :: Repo -> Maybe Integer
-urlPort r = 
-	case urlAuthPart uriPort r of
-		":" -> Nothing
-		(':':p) -> readMaybe p
-		_ -> Nothing
-
-{- Hostname of an URL repo, including any username (ie, "user@host") -}
-urlHostUser :: Repo -> String
-urlHostUser r = urlAuthPart uriUserInfo r ++ urlAuthPart uriRegName' r
-
-{- The full authority portion an URL repo. (ie, "user@host:port") -}
-urlAuthority :: Repo -> String
-urlAuthority = urlAuthPart assemble
-	where
-		assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a
-
-{- Applies a function to extract part of the uriAuthority of an URL repo. -}
-urlAuthPart :: (URIAuth -> a) -> Repo -> a
-urlAuthPart a Repo { location = Url u } = a auth
-	where
-		auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
-urlAuthPart _ repo = assertUrl repo $ error "internal"
-
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
 gitCommandLine params repo@(Repo { location = Dir _ } ) =
@@ -284,39 +189,6 @@ reap = do
 	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
 	maybe (return ()) (const reap) r
 
-{- Forces git to use the specified index file.
- - Returns an action that will reset back to the default
- - index file. -}
-useIndex :: FilePath -> IO (IO ())
-useIndex index = do
-	res <- getEnv var
-	setEnv var index True
-	return $ reset res
-	where
-		var = "GIT_INDEX_FILE"
-		reset (Just v) = setEnv var v True
-		reset _ = unsetEnv var
-
-{- Runs an action that causes a git subcommand to emit a sha, and strips
-   any trailing newline, returning the sha. -}
-getSha :: String -> IO String -> IO Sha
-getSha subcommand a = do
-	t <- a
-	let t' = if last t == '\n'
-		then init t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return $ Ref t'
-
-{- Size of a git sha. -}
-shaSize :: Int
-shaSize = 40
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
-
-{- Access to raw config Map -}
-configMap :: Repo -> M.Map String String
-configMap = config
diff --git a/Git/Branch.hs b/Git/Branch.hs
index 8b0d1e5afc..3e08e19c28 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -11,6 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
 import Git
+import Git.Sha
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
@@ -19,7 +20,7 @@ changed origbranch newbranch repo
 	| origbranch == newbranch = return False
 	| otherwise = not . L.null <$> diffs
 	where
-		diffs = Git.pipeRead
+		diffs = pipeRead
 			[ Param "log"
 			, Param (show origbranch ++ ".." ++ show newbranch)
 			, Params "--oneline -n1"
@@ -44,7 +45,7 @@ fastForward branch (first:rest) repo = do
 	where
 		no_ff = return False
 		do_ff to = do
-			Git.run "update-ref"
+			run "update-ref"
 				[Param $ show branch, Param $ show to] repo
 			return True
 		findbest c [] = return $ Just c
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index c1cafb8ba8..44c2a9f5ef 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -20,6 +20,7 @@ import qualified Data.ByteString.Char8 as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import Git.Sha
 import Utility.SafeCommand
 
 type CatFileHandle = (PipeHandle, Handle, Handle)
@@ -27,7 +28,7 @@ type CatFileHandle = (PipeHandle, Handle, Handle)
 {- Starts git cat-file running in batch mode in a repo and returns a handle. -}
 catFileStart :: Repo -> IO CatFileHandle
 catFileStart repo = hPipeBoth "git" $ toCommand $
-	Git.gitCommandLine [Param "cat-file", Param "--batch"] repo
+	gitCommandLine [Param "cat-file", Param "--batch"] repo
 
 {- Stops git cat-file. -}
 catFileStop :: CatFileHandle -> IO ()
@@ -49,7 +50,7 @@ catObject (_, from, to) object = do
 	header <- hGetLine from
 	case words header of
 		[sha, objtype, size]
-			| length sha == Git.shaSize &&
+			| length sha == shaSize &&
 			  validobjtype objtype -> handle size
 			| otherwise -> empty
 		_
diff --git a/Git/Construct.hs b/Git/Construct.hs
index 9149ab9ec0..a35a87cc77 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -11,6 +11,8 @@ module Git.Construct (
 	fromUrl,
 	fromUnknown,
 	localToUrl,
+	remoteNamed,
+	remoteNamedFromKey,
 	fromRemotes,
 	fromRemoteLocation,
 	repoAbsPath,
@@ -23,6 +25,7 @@ import Network.URI
 import Common
 import Git.Types
 import Git
+import qualified Git.Url as Url
 
 {- Finds the current git repository, which may be in a parent directory. -}
 fromCwd :: IO Repo
@@ -67,8 +70,8 @@ fromUrl url
 			bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
-fromUnknown :: Repo
-fromUnknown = newFrom Unknown
+fromUnknown :: IO Repo
+fromUnknown = return $ newFrom Unknown
 
 {- Converts a local Repo into a remote repo, using the reference repo
  - which is assumed to be on the same host. -}
@@ -79,8 +82,8 @@ localToUrl reference r
 	| otherwise = r { location = Url $ fromJust $ parseURI absurl }
 	where
 		absurl =
-			urlScheme reference ++ "//" ++
-			urlAuthority reference ++
+			Url.scheme reference ++ "//" ++
+			Url.authority reference ++
 			workTree r
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
@@ -91,7 +94,21 @@ fromRemotes repo = mapM construct remotepairs
 		filterkeys f = filterconfig (\(k,_) -> f k)
 		remotepairs = filterkeys isremote
 		isremote k = startswith "remote." k && endswith ".url" k
-		construct (k,v) = repoRemoteNameFromKey k <$> fromRemoteLocation v repo
+		construct (k,v) = remoteNamedFromKey k $ fromRemoteLocation v repo
+
+{- Sets the name of a remote when constructing the Repo to represent it. -}
+remoteNamed :: String -> IO Repo -> IO Repo
+remoteNamed n constructor = do
+	r <- constructor
+	return $ r { remoteName = Just n }
+
+{- Sets the name of a remote based on the git config key, such as
+   "remote.foo.url". -}
+remoteNamedFromKey :: String -> IO Repo -> IO Repo
+remoteNamedFromKey k = remoteNamed basename
+	where
+		basename = join "." $ reverse $ drop 1 $
+				reverse $ drop 1 $ split "." k
 
 {- Constructs a new Repo for one of a Repo's remotes using a given
  - location (ie, an url). -}
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 99b96afcb9..60822f3f07 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -17,10 +17,10 @@ hashFiles paths repo = do
 	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
 	_ <- forkProcess (feeder toh)
 	hClose toh
-	shas <- map Git.Ref . lines <$> hGetContentsStrict fromh
+	shas <- map Ref . lines <$> hGetContentsStrict fromh
 	return (shas, ender fromh pid)
 	where
-		git_hash_object = Git.gitCommandLine
+		git_hash_object = gitCommandLine
 			[Param "hash-object", Param "-w", Param "--stdin-paths"]
 		feeder toh = do
 			hPutStr toh $ unlines paths
diff --git a/Git/Index.hs b/Git/Index.hs
new file mode 100644
index 0000000000..aaf54e032e
--- /dev/null
+++ b/Git/Index.hs
@@ -0,0 +1,24 @@
+{- git index file stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Index where
+
+import System.Posix.Env (setEnv, unsetEnv, getEnv)
+
+{- Forces git to use the specified index file.
+ -
+ - Returns an action that will reset back to the default
+ - index file. -}
+override :: FilePath -> IO (IO ())
+override index = do
+	res <- getEnv var
+	setEnv var index True
+	return $ reset res
+	where
+		var = "GIT_INDEX_FILE"
+		reset (Just v) = setEnv var v True
+		reset _ = unsetEnv var
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 723bea6817..3b550cf5b9 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -37,11 +37,11 @@ sha branch repo = process . L.unpack <$> showref repo
 
 {- List of (refs, branches) matching a given ref spec.
  - Duplicate refs are filtered out. -}
-matching :: Ref -> Repo -> IO [(Git.Ref, Git.Branch)]
+matching :: Ref -> Repo -> IO [(Ref, Branch)]
 matching ref repo = do
-	r <- Git.pipeRead [Param "show-ref", Param $ show ref] repo
+	r <- pipeRead [Param "show-ref", Param $ show ref] repo
 	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
 	where
-		gen l = (Git.Ref $ head l, Git.Ref $ last l)
+		gen l = (Ref $ head l, Ref $ last l)
 		uref (a, _) (b, _) = a == b
 
diff --git a/Git/Sha.hs b/Git/Sha.hs
new file mode 100644
index 0000000000..475c2ba5f3
--- /dev/null
+++ b/Git/Sha.hs
@@ -0,0 +1,27 @@
+{- git SHA stuff
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Sha where
+
+import Common
+import Git.Types
+
+{- Runs an action that causes a git subcommand to emit a sha, and strips
+   any trailing newline, returning the sha. -}
+getSha :: String -> IO String -> IO Sha
+getSha subcommand a = do
+	t <- a
+	let t' = if last t == '\n'
+		then init t
+		else t
+	when (length t' /= shaSize) $
+		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
+	return $ Ref t'
+
+{- Size of a git sha. -}
+shaSize :: Int
+shaSize = 40
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 0345af3994..a623e1cebb 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -20,6 +20,7 @@ import qualified Data.Set as S
 
 import Common
 import Git
+import Git.Sha
 import Git.CatFile
 
 type Streamer = (String -> IO ()) -> IO ()
@@ -27,7 +28,7 @@ type Streamer = (String -> IO ()) -> IO ()
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
  -
- - Should be run with a temporary index file configured by Git.useIndex.
+ - Should be run with a temporary index file configured by useIndex.
  -}
 merge :: Ref -> Ref -> Repo -> IO ()
 merge x y repo = do
@@ -53,7 +54,7 @@ update_index repo ls = stream_update_index repo [(`mapM_` ls)]
 {- Streams content into update-index. -}
 stream_update_index :: Repo -> [Streamer] -> IO ()
 stream_update_index repo as = do
-	(p, h) <- hPipeTo "git" (toCommand $ Git.gitCommandLine params repo)
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
 	forM_ as (stream h)
 	hClose h
 	forceSuccess p
diff --git a/Git/Url.hs b/Git/Url.hs
new file mode 100644
index 0000000000..6a893d92fe
--- /dev/null
+++ b/Git/Url.hs
@@ -0,0 +1,70 @@
+{- git repository urls
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Url (
+	scheme,
+	host,
+	port,
+	hostuser,
+	authority,
+) where
+
+import Network.URI hiding (scheme, authority)
+
+import Common
+import Git.Types
+import Git
+
+{- Scheme of an URL repo. -}
+scheme :: Repo -> String
+scheme Repo { location = Url u } = uriScheme u
+scheme repo = notUrl repo
+
+{- Work around a bug in the real uriRegName
+ -  -}
+uriRegName' :: URIAuth -> String
+uriRegName' a = fixup $ uriRegName a
+	where
+		fixup x@('[':rest)
+			| rest !! len == ']' = take len rest
+			| otherwise = x
+			where
+				len  = length rest - 1
+		fixup x = x
+
+{- Hostname of an URL repo. -}
+host :: Repo -> String
+host = authpart uriRegName'
+
+{- Port of an URL repo, if it has a nonstandard one. -}
+port :: Repo -> Maybe Integer
+port r = 
+	case authpart uriPort r of
+		":" -> Nothing
+		(':':p) -> readMaybe p
+		_ -> Nothing
+
+{- Hostname of an URL repo, including any username (ie, "user@host") -}
+hostuser :: Repo -> String
+hostuser r = authpart uriUserInfo r ++ authpart uriRegName' r
+
+{- The full authority portion an URL repo. (ie, "user@host:port") -}
+authority :: Repo -> String
+authority = authpart assemble
+	where
+		assemble a = uriUserInfo a ++ uriRegName' a ++ uriPort a
+
+{- Applies a function to extract part of the uriAuthority of an URL repo. -}
+authpart :: (URIAuth -> a) -> Repo -> a
+authpart a Repo { location = Url u } = a auth
+	where
+		auth = fromMaybe (error $ "bad url " ++ show u) (uriAuthority u)
+authpart _ repo = notUrl repo
+
+notUrl :: Repo -> a
+notUrl repo = error $
+	"acting on local git repo " ++ repoDescribe repo ++ " not supported"
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 9d80f4c1cb..d848a21b33 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -37,16 +37,17 @@ remote = RemoteType {
 
 list :: Annex [Git.Repo]
 list = do
-	c <- fromRepo Git.configMap
+	c <- fromRepo Git.config
 	mapM (tweakurl c) =<< fromRepo Git.remotes
 	where
 		annexurl n = "remote." ++ n ++ ".annexurl"
 		tweakurl c r = do
-			let n = fromJust $ Git.repoRemoteName r
+			let n = fromJust $ Git.remoteName r
 			case M.lookup (annexurl n) c of
 				Nothing -> return r
-				Just url -> Git.repoRemoteNameSet n <$>
-					inRepo (Git.Construct.fromRemoteLocation url)
+				Just url -> inRepo $ \g ->
+					Git.Construct.remoteNamed n $
+						Git.Construct.fromRemoteLocation url g
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r u _ = do
@@ -84,7 +85,7 @@ gen r u _ = do
  - returns the updated repo. -}
 tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
-	| not $ M.null $ Git.configMap r = return r -- already read
+	| not $ M.null $ Git.config r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
 	| Git.repoIsHttp r = store $ safely geturlconfig
 	| Git.repoIsUrl r = return r
@@ -116,13 +117,13 @@ tryGitConfigRead r
 			r' <- a
 			g <- gitRepo
 			let l = Git.remotes g
-			let g' = Git.remotesAdd g $ exchange l r'
+			let g' = g { Git.remotes = exchange l r' }
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 			return r'
 
 		exchange [] _ = []
 		exchange (old:ls) new =
-			if Git.repoRemoteName old == Git.repoRemoteName new
+			if Git.remoteName old == Git.remoteName new
 				then new : exchange ls new
 				else old : exchange ls new
 
@@ -167,7 +168,7 @@ onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
 	-- Avoid re-reading the repository's configuration if it was
 	-- already read.
-	state <- if M.null $ Git.configMap r
+	state <- if M.null $ Git.config r
 		then Annex.new r
 		else return $ Annex.newState r
 	Annex.eval state $ do
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index 72c4842d88..c374a16aaf 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -20,11 +20,11 @@ import qualified Git.Construct
  -}
 findSpecialRemotes :: String -> Annex [Git.Repo]
 findSpecialRemotes s = do
-	m <- fromRepo Git.configMap
-	return $ map construct $ remotepairs m
+	m <- fromRepo Git.config
+	liftIO $ mapM construct $ remotepairs m
 	where
 		remotepairs = M.toList . M.filterWithKey match
-		construct (k,_) = Git.repoRemoteNameFromKey k Git.Construct.fromUnknown
+		construct (k,_) = Git.Construct.remoteNamedFromKey k Git.Construct.fromUnknown
 		match k _ = startswith "remote." k && endswith (".annex-"++s) k
 
 {- Sets up configuration for a special remote in .git/config. -}
diff --git a/Remote/Web.hs b/Remote/Web.hs
index c4e9f8bd69..e31539f885 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -27,7 +27,9 @@ remote = RemoteType {
 -- (If the web should cease to exist, remove this module and redistribute
 -- a new release to the survivors by carrier pigeon.)
 list :: Annex [Git.Repo]
-list = return [Git.repoRemoteNameSet "web" Git.Construct.fromUnknown]
+list = do
+	r <- liftIO $ Git.Construct.remoteNamed "web" Git.Construct.fromUnknown
+	return [r]
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
 gen r _ _ = 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index f67414bdd3..6fd19c8dae 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -12,6 +12,7 @@ import qualified Git.UnionMerge
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Git.Branch
+import qualified Git.Index
 import qualified Git
 
 header :: String
@@ -42,7 +43,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
 	g <- Git.Config.read =<< Git.Construct.fromCwd
-	_ <- Git.useIndex (tmpIndex g)
+	_ <- Git.Index.override (tmpIndex g)
 	setup g
 	Git.UnionMerge.merge aref bref g
 	_ <- Git.Branch.commit "union merge" newref [aref, bref] g
diff --git a/test.hs b/test.hs
index 1ce9d103d1..daa2661b6e 100644
--- a/test.hs
+++ b/test.hs
@@ -24,7 +24,6 @@ import qualified Utility.SafeCommand
 import qualified Annex
 import qualified Annex.UUID
 import qualified Backend
-import qualified Git
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Git.Filename

From ef28b3fef7e236d8c27ce35308c0e37ece58d20c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Dec 2011 15:56:11 -0400
Subject: [PATCH 2699/8313] split out Git/Command.hs

---
 Annex/Branch.hs          |  5 ++-
 CmdLine.hs               |  3 +-
 Command/Sync.hs          | 11 +++---
 Command/Unannex.hs       |  6 +--
 Command/Uninit.hs        |  6 ++-
 Command/Unused.hs        |  3 +-
 Config.hs                |  3 +-
 Git.hs                   | 78 --------------------------------------
 Git/Branch.hs            |  1 +
 Git/CatFile.hs           |  1 +
 Git/CheckAttr.hs         |  1 +
 Git/Command.hs           | 82 ++++++++++++++++++++++++++++++++++++++++
 Git/HashObject.hs        |  1 +
 Git/LsFiles.hs           |  1 +
 Git/LsTree.hs            |  1 +
 Git/Queue.hs             |  1 +
 Git/Ref.hs               |  1 +
 Git/UnionMerge.hs        |  1 +
 Remote/Bup.hs            |  5 ++-
 Remote/Git.hs            |  3 +-
 Remote/Helper/Special.hs |  3 +-
 Upgrade/V2.hs            |  8 ++--
 22 files changed, 125 insertions(+), 100 deletions(-)
 create mode 100644 Git/Command.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a22a4adcfe..af1878479a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -24,6 +24,7 @@ import Annex.Exception
 import Annex.BranchState
 import Annex.Journal
 import qualified Git
+import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.UnionMerge
@@ -67,7 +68,7 @@ getBranch :: Annex (Git.Ref)
 getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 	where
 		go True = do
-			inRepo $ Git.run "branch"
+			inRepo $ Git.Command.run "branch"
 				[Param $ show name, Param $ show originname]
 			fromMaybe (error $ "failed to create " ++ show name)
 				<$> branchsha
@@ -221,7 +222,7 @@ commitBranch branchref message parents = do
 {- Lists all files on the branch. There may be duplicates in the list. -}
 files :: Annex [FilePath]
 files = withIndexUpdate $ do
-	bfiles <- inRepo $ Git.pipeNullSplit
+	bfiles <- inRepo $ Git.Command.pipeNullSplit
 		[Params "ls-tree --name-only -r -z", Param $ show fullname]
 	jfiles <- getJournalledFiles
 	return $ jfiles ++ bfiles
diff --git a/CmdLine.hs b/CmdLine.hs
index 672969c30a..ebcca25aaf 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -20,6 +20,7 @@ import Common.Annex
 import qualified Annex
 import qualified Annex.Queue
 import qualified Git
+import qualified Git.Command
 import Annex.Content
 import Command
 
@@ -101,5 +102,5 @@ startup = return True
 shutdown :: Annex Bool
 shutdown = do
 	saveState
-	liftIO Git.reap -- zombies from long-running git processes
+	liftIO Git.Command.reap -- zombies from long-running git processes
 	return True
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 987eb6138f..a25bcad8c1 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -10,7 +10,7 @@ module Command.Sync where
 import Common.Annex
 import Command
 import qualified Annex.Branch
-import qualified Git
+import qualified Git.Command
 import qualified Git.Config
 
 import qualified Data.ByteString.Lazy.Char8 as L
@@ -28,7 +28,8 @@ commit = do
 	next $ next $ do
 		showOutput
 		-- Commit will fail when the tree is clean, so ignore failure.
-		_ <- inRepo $ Git.runBool "commit" [Param "-a", Param "-m", Param "sync"]
+		_ <- inRepo $ Git.Command.runBool "commit"
+			[Param "-a", Param "-m", Param "sync"]
 		return True
 
 pull :: CommandStart
@@ -38,7 +39,7 @@ pull = do
 	next $ next $ do
 		showOutput
 		checkRemote remote
-		inRepo $ Git.runBool "pull" [Param remote]
+		inRepo $ Git.Command.runBool "pull" [Param remote]
 
 push :: CommandStart
 push = do
@@ -47,7 +48,7 @@ push = do
 	next $ next $ do
 		Annex.Branch.update
 		showOutput
-		inRepo $ Git.runBool "push" [Param remote, matchingbranches]
+		inRepo $ Git.Command.runBool "push" [Param remote, matchingbranches]
 	where
 		-- git push may be configured to not push matching
 		-- branches; this should ensure it always does.
@@ -61,7 +62,7 @@ defaultRemote = do
 
 currentBranch :: Annex String
 currentBranch = last . split "/" . L.unpack . head . L.lines <$>
-	inRepo (Git.pipeRead [Param "symbolic-ref", Param "HEAD"])
+	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
 
 checkRemote :: String -> Annex ()
 checkRemote remote = do
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index bed857b060..8a511bf4da 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -13,7 +13,7 @@ import qualified Annex
 import Utility.FileMode
 import Logs.Location
 import Annex.Content
-import qualified Git
+import qualified Git.Command
 import qualified Git.LsFiles as LsFiles
 
 def :: [Command]
@@ -34,14 +34,14 @@ cleanup :: FilePath -> Key -> CommandCleanup
 cleanup file key = do
 	liftIO $ removeFile file
 	-- git rm deletes empty directory without --cached
-	inRepo $ Git.run "rm" [Params "--cached --quiet --", File file]
+	inRepo $ Git.Command.run "rm" [Params "--cached --quiet --", File file]
 	
 	-- If the file was already committed, it is now staged for removal.
 	-- Commit that removal now, to avoid later confusing the
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
 	whenM (not . null <$> inRepo (LsFiles.staged [file])) $ do
-		inRepo $ Git.run "commit" [
+		inRepo $ Git.Command.run "commit" [
 			Param "-m", Param "content removed from git annex",
 			Param "--", File file]
 
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 48f5b1ac14..fc6f0cc275 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -12,6 +12,7 @@ import qualified Data.ByteString.Lazy.Char8 as B
 import Common.Annex
 import Command
 import qualified Git
+import qualified Git.Command
 import qualified Annex
 import qualified Command.Unannex
 import Init
@@ -29,7 +30,7 @@ check = do
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
 		current_branch = Git.Ref . head . lines . B.unpack <$> revhead
-		revhead = inRepo $ Git.pipeRead 
+		revhead = inRepo $ Git.Command.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
 seek :: [CommandSeek]
@@ -57,5 +58,6 @@ cleanup = do
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
 	saveState
-	inRepo $ Git.run "branch" [Param "-D", Param $ show Annex.Branch.name]
+	inRepo $ Git.Command.run "branch"
+		[Param "-D", Param $ show Annex.Branch.name]
 	liftIO exitSuccess
diff --git a/Command/Unused.hs b/Command/Unused.hs
index cd1cd16024..8a70ff3356 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -20,6 +20,7 @@ import Utility.TempFile
 import Logs.Location
 import qualified Annex
 import qualified Git
+import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
@@ -148,7 +149,7 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	c <- inRepo $ Git.pipeRead [Param "show-ref"]
+	c <- inRepo $ Git.Command.pipeRead [Param "show-ref"]
 	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
 		(S.fromList l)
 	where
diff --git a/Config.hs b/Config.hs
index 07c9eedad7..4cc4c18668 100644
--- a/Config.hs
+++ b/Config.hs
@@ -10,6 +10,7 @@ module Config where
 import Common.Annex
 import qualified Git
 import qualified Git.Config
+import qualified Git.Command
 import qualified Annex
 
 type ConfigKey = String
@@ -17,7 +18,7 @@ type ConfigKey = String
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig k value = do
-	inRepo $ Git.run "config" [Param k, Param value]
+	inRepo $ Git.Command.run "config" [Param k, Param value]
 	-- re-read git config and update the repo's state
 	newg <- inRepo Git.Config.read
 	Annex.changeState $ \s -> s { Annex.repo = newg }
diff --git a/Git.hs b/Git.hs
index a3f2ad74cb..9420810a67 100644
--- a/Git.hs
+++ b/Git.hs
@@ -23,22 +23,12 @@ module Git (
 	workTree,
 	gitDir,
 	configTrue,
-	gitCommandLine,
-	run,
-	runBool,
-	pipeRead,
-	pipeWrite,
-	pipeWriteRead,
-	pipeNullSplit,
-	pipeNullSplitB,
 	attributes,
-	reap,
 	assertLocal,
 ) where
 
 import qualified Data.Map as M
 import Data.Char
-import qualified Data.ByteString.Lazy.Char8 as L
 import Network.URI (uriPath, uriScheme)
 
 import Common
@@ -121,74 +111,6 @@ workTree Repo { location = Url u } = uriPath u
 workTree Repo { location = Dir d } = d
 workTree Repo { location = Unknown } = undefined
 
-{- Constructs a git command line operating on the specified repo. -}
-gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
-gitCommandLine params repo@(Repo { location = Dir _ } ) =
-	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ gitDir repo)
-	, Param ("--work-tree=" ++ workTree repo)
-	] ++ params
-gitCommandLine _ repo = assertLocal repo $ error "internal"
-
-{- Runs git in the specified repo. -}
-runBool :: String -> [CommandParam] -> Repo -> IO Bool
-runBool subcommand params repo = assertLocal repo $
-	boolSystem "git" $ gitCommandLine (Param subcommand : params) repo
-
-{- Runs git in the specified repo, throwing an error if it fails. -}
-run :: String -> [CommandParam] -> Repo -> IO ()
-run subcommand params repo = assertLocal repo $
-	runBool subcommand params repo
-		>>! error $ "git " ++ show params ++ " failed"
-
-{- Runs a git subcommand and returns its output, lazily. 
- -
- - Note that this leaves the git process running, and so zombies will
- - result unless reap is called.
- -}
-pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
-pipeRead params repo = assertLocal repo $ do
-	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
-	hSetBinaryMode h True
-	L.hGetContents h
-
-{- Runs a git subcommand, feeding it input.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
-pipeWrite params s repo = assertLocal repo $ do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
-	L.hPut h s
-	hClose h
-	return p
-
-{- Runs a git subcommand, feeding it input, and returning its output.
- - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
-pipeWriteRead params s repo = assertLocal repo $ do
-	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
-	hSetBinaryMode from True
-	L.hPut to s
-	hClose to
-	c <- L.hGetContents from
-	return (p, c)
-
-{- Reads null terminated output of a git command (as enabled by the -z 
- - parameter), and splits it. -}
-pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
-pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
-
-{- For when Strings are not needed. -}
-pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
-pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
-	pipeRead params repo
-
-{- Reaps any zombie git processes. -}
-reap :: IO ()
-reap = do
-	-- throws an exception when there are no child processes
-	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
-	maybe (return ()) (const reap) r
-
 {- Checks if a string from git config is a true value. -}
 configTrue :: String -> Bool
 configTrue s = map toLower s == "true"
diff --git a/Git/Branch.hs b/Git/Branch.hs
index 3e08e19c28..cce56dcfa4 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -12,6 +12,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import Common
 import Git
 import Git.Sha
+import Git.Command
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 44c2a9f5ef..2cef9d5b3d 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -21,6 +21,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
 import Git.Sha
+import Git.Command
 import Utility.SafeCommand
 
 type CatFileHandle = (PipeHandle, Handle, Handle)
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index e9269b1edb..1ea38beeac 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -11,6 +11,7 @@ import System.Exit
 
 import Common
 import Git
+import Git.Command
 import qualified Git.Filename
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
diff --git a/Git/Command.hs b/Git/Command.hs
new file mode 100644
index 0000000000..2350bb0ca3
--- /dev/null
+++ b/Git/Command.hs
@@ -0,0 +1,82 @@
+{- running git commands
+ -
+ - Copyright 2010, 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Command where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+
+import Common
+import Git
+import Git.Types
+
+{- Constructs a git command line operating on the specified repo. -}
+gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
+gitCommandLine params repo@(Repo { location = Dir _ } ) =
+	-- force use of specified repo via --git-dir and --work-tree
+	[ Param ("--git-dir=" ++ gitDir repo)
+	, Param ("--work-tree=" ++ workTree repo)
+	] ++ params
+gitCommandLine _ repo = assertLocal repo $ error "internal"
+
+{- Runs git in the specified repo. -}
+runBool :: String -> [CommandParam] -> Repo -> IO Bool
+runBool subcommand params repo = assertLocal repo $
+	boolSystem "git" $ gitCommandLine (Param subcommand : params) repo
+
+{- Runs git in the specified repo, throwing an error if it fails. -}
+run :: String -> [CommandParam] -> Repo -> IO ()
+run subcommand params repo = assertLocal repo $
+	runBool subcommand params repo
+		>>! error $ "git " ++ show params ++ " failed"
+
+{- Runs a git subcommand and returns its output, lazily. 
+ -
+ - Note that this leaves the git process running, and so zombies will
+ - result unless reap is called.
+ -}
+pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
+pipeRead params repo = assertLocal repo $ do
+	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
+	hSetBinaryMode h True
+	L.hGetContents h
+
+{- Runs a git subcommand, feeding it input.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
+pipeWrite params s repo = assertLocal repo $ do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
+	L.hPut h s
+	hClose h
+	return p
+
+{- Runs a git subcommand, feeding it input, and returning its output.
+ - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
+pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
+pipeWriteRead params s repo = assertLocal repo $ do
+	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
+	hSetBinaryMode from True
+	L.hPut to s
+	hClose to
+	c <- L.hGetContents from
+	return (p, c)
+
+{- Reads null terminated output of a git command (as enabled by the -z 
+ - parameter), and splits it. -}
+pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
+pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
+
+{- For when Strings are not needed. -}
+pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
+pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
+	pipeRead params repo
+
+{- Reaps any zombie git processes. -}
+reap :: IO ()
+reap = do
+	-- throws an exception when there are no child processes
+	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
+	maybe (return ()) (const reap) r
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 60822f3f07..f5e6d50cdf 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -9,6 +9,7 @@ module Git.HashObject where
 
 import Common
 import Git
+import Git.Command
 
 {- Injects a set of files into git, returning the shas of the objects
  - and an IO action to call ones the the shas have been used. -}
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 85215fe040..0c71ed884b 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -16,6 +16,7 @@ module Git.LsFiles (
 ) where
 
 import Git
+import Git.Command
 import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 342a125eb8..919e9af83a 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -17,6 +17,7 @@ import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
 import Git
+import Git.Command
 import qualified Git.Filename
 import Utility.SafeCommand
 
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 70c766d044..73470b1f0e 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -22,6 +22,7 @@ import Control.Monad (forM_)
 import Utility.SafeCommand
 
 import Git
+import Git.Command
 
 {- An action to perform in a git repository. The file to act on
  - is not included, and must be able to be appended after the params. -}
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 3b550cf5b9..117ead8f22 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -11,6 +11,7 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 import Common
 import Git
+import Git.Command
 
 {- Converts a fully qualified git ref into a user-visible version. -}
 describe :: Ref -> String
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a623e1cebb..a9a51007f8 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -22,6 +22,7 @@ import Common
 import Git
 import Git.Sha
 import Git.CatFile
+import Git.Command
 
 type Streamer = (String -> IO ()) -> IO ()
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4d63d88e1d..8bd484b7dd 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -15,6 +15,7 @@ import System.Process
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
 import Config
@@ -148,7 +149,7 @@ checkPresent r bupr k
 		ok <- onBupRemote bupr boolSystem "git" params
 		return $ Right ok
 	| otherwise = liftIO $ catchMsgIO $
-		boolSystem "git" $ Git.gitCommandLine params bupr
+		boolSystem "git" $ Git.Command.gitCommandLine params bupr
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
@@ -168,7 +169,7 @@ storeBupUUID u buprepo = do
 			r' <- Git.Config.read r
 			let olduuid = Git.Config.get "annex.uuid" "" r'
 			when (olduuid == "") $
-				Git.run "config"
+				Git.Command.run "config"
 					[Param "annex.uuid", Param v] r'
 	where
 		v = fromUUID u
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d848a21b33..f27d170840 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -16,6 +16,7 @@ import Utility.RsyncFile
 import Annex.Ssh
 import Types.Remote
 import qualified Git
+import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Annex
@@ -176,7 +177,7 @@ onLocal r a = do
 		-- for anything onLocal is used to do.
 		Annex.BranchState.disableUpdate
 		ret <- a
-		liftIO Git.reap
+		liftIO Git.Command.reap
 		return ret
 
 keyUrls :: Git.Repo -> Key -> [String]
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index c374a16aaf..3f6c9c155f 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -12,6 +12,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Types.Remote
 import qualified Git
+import qualified Git.Command
 import qualified Git.Construct
 
 {- Special remotes don't have a configured url, so Git.Repo does not
@@ -33,7 +34,7 @@ gitConfigSpecialRemote u c k v = do
 	set ("annex-"++k) v
 	set ("annex-uuid") (fromUUID u)
 	where
-		set a b = inRepo $ Git.run "config"
+		set a b = inRepo $ Git.Command.run "config"
 			[Param (configsetting a), Param b]
 		remotename = fromJust (M.lookup "name" c)
 		configsetting s = "remote." ++ remotename ++ "." ++ s
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 3440d504ba..ffc2f60022 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -9,6 +9,7 @@ module Upgrade.V2 where
 
 import Common.Annex
 import qualified Git
+import qualified Git.Command
 import qualified Git.Ref
 import qualified Annex.Branch
 import Logs.Location
@@ -53,7 +54,7 @@ upgrade = do
 	showProgress
 
 	when e $ do
-		inRepo $ Git.run "rm" [Param "-r", Param "-f", Param "-q", File old]
+		inRepo $ Git.Command.run "rm" [Param "-r", Param "-f", Param "-q", File old]
 		unless bare $ inRepo gitAttributesUnWrite
 	showProgress
 
@@ -104,7 +105,8 @@ push = do
 			Annex.Branch.update -- just in case
 			showAction "pushing new git-annex branch to origin"
 			showOutput
-			inRepo $ Git.run "push" [Param "origin", Param $ show Annex.Branch.name]
+			inRepo $ Git.Command.run "push"
+				[Param "origin", Param $ show Annex.Branch.name]
 		_ -> do
 			-- no origin exists, so just let the user
 			-- know about the new branch
@@ -127,7 +129,7 @@ gitAttributesUnWrite repo = do
 		c <- readFileStrict attributes
 		liftIO $ viaTmp writeFile attributes $ unlines $
 			filter (`notElem` attrLines) $ lines c
-		Git.run "add" [File attributes] repo
+		Git.Command.run "add" [File attributes] repo
 
 stateDir :: FilePath
 stateDir = addTrailingPathSeparator ".git-annex"

From 09cd04277577381464eddceb42558ad300c49378 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 12:46:04 -0400
Subject: [PATCH 2700/8313] Properly handle multiline git config values.

A crash on parsing was fixed a while ago. This adds support for fully
correctly parsing multiline git config values, using git config --null.

Since git-annex-shell configlist uses normal git config output, I left in
support for that too; the two forms of config output can be easily
identified by the parser. Since configlist only prints the annex.uuid
config, there's no risk of multiline values there, so no need to change it.
---
 Command/Map.hs   |  2 +-
 Git/Config.hs    | 14 +++++++++++---
 Remote/Git.hs    |  2 +-
 debian/changelog |  1 +
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index ae8a694043..15ca5e1496 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -203,7 +203,7 @@ tryScan r
 			liftIO $ pipedconfig "ssh" sshparams
 			where
 				sshcmd = cddir ++ " && " ++
-					"git config --list"
+					"git config --null --list"
 				dir = Git.workTree r
 				cddir
 					| "/~" `isPrefixOf` dir =
diff --git a/Git/Config.hs b/Git/Config.hs
index 5f0e3fdc21..1fe9484992 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -33,7 +33,8 @@ read repo@(Repo { location = Dir d }) = do
 	   been already read. Instead, chdir to the repo. -}
 	cwd <- getCurrentDirectory
 	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
-		pOpen ReadFromPipe "git" ["config", "--list"] $ hRead repo
+		pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
+			hRead repo
 read r = assertLocal r $ error "internal"
 
 {- Reads git config from a handle and populates a repo with it. -}
@@ -51,8 +52,15 @@ store s repo = do
 	rs <- Git.Construct.fromRemotes repo'
 	return $ repo' { remotes = rs }
 
-{- Parses git config --list output into a config map. -}
+{- Parses git config --list or git config --null --list output into a
+ - config map. -}
 parse :: String -> M.Map String String
-parse s = M.fromList $ map pair $ lines s
+parse [] = M.empty
+parse s
+	-- --list output will have an = in the first line
+	| '=' `elem` head ls = M.fromList $ map (separate (== '=')) ls
+	-- --null --list output separates keys from values with newlines
+	| otherwise = M.fromList $ map (separate (== '\n')) $ split "\0" s
 	where
 		pair = separate (== '=')
+		ls = lines s
diff --git a/Remote/Git.hs b/Remote/Git.hs
index f27d170840..e527fa4fee 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -111,7 +111,7 @@ tryGitConfigRead r
 			withTempFile "git-annex.tmp" $ \tmpfile h -> do
 				hPutStr h s
 				hClose h
-				pOpen ReadFromPipe "git" ["config", "--list", "--file", tmpfile] $
+				pOpen ReadFromPipe "git" ["config", "--null", "--list", "--file", tmpfile] $
 					Git.Config.hRead r
 
 		store a = do
diff --git a/debian/changelog b/debian/changelog
index 5fc01f9ead..3c977e817d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
 
   * Union merge now finds the least expensive way to represent the merge.
   * reinject: Add a sanity check for using an annexed file as the source file.
+  * Properly handle multiline git config values.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From a8643ca44c49cc945f0ecb0e02c2946595415054 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 13:05:47 -0400
Subject: [PATCH 2701/8313] refactor

---
 Git/Config.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 1fe9484992..8d79639944 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -58,9 +58,9 @@ parse :: String -> M.Map String String
 parse [] = M.empty
 parse s
 	-- --list output will have an = in the first line
-	| '=' `elem` head ls = M.fromList $ map (separate (== '=')) ls
+	| '=' `elem` head ls = sep '=' ls
 	-- --null --list output separates keys from values with newlines
-	| otherwise = M.fromList $ map (separate (== '\n')) $ split "\0" s
+	| otherwise = sep '\n' $ split "\0" s
 	where
-		pair = separate (== '=')
 		ls = lines s
+		sep c = M.fromList . map (separate (== c))

From 52fe8a17f381a45288c6db7d2d92d3b32c47a472 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 13:12:17 -0400
Subject: [PATCH 2702/8313] remove leftover debug print

---
 Command/AddUrl.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 75ca740314..027c508bcb 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -65,7 +65,6 @@ url2file :: URI -> IO FilePath
 url2file url = do
 	whenM (doesFileExist file) $
 		error $ "already have this url in " ++ file
-	liftIO $ print file
 	return file
 	where
 		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url

From 38b02de1a84728ff087bc18f08c597c384eacc78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 15:21:03 -0400
Subject: [PATCH 2703/8313] update

---
 doc/todo/add_-all_option.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/todo/add_-all_option.mdwn b/doc/todo/add_-all_option.mdwn
index cb4722fa78..e6fa0b339d 100644
--- a/doc/todo/add_-all_option.mdwn
+++ b/doc/todo/add_-all_option.mdwn
@@ -5,7 +5,8 @@ every keys with content not present).
 This would be useful when a repository has a history with deleted files
 whose content you want to keep (so you're not using `dropunused`).
 Or when you have a lot of branches and just want to be able to fsck
-every file referenced in any branch.
+every file referenced in any branch. It could also be useful (or even a
+good default) in a bare repository.
 
 A problem with the idea is that `.gitattributes` values for keys not
 currently in the tree would not be available (without horrific anounts of

From e7a555bf211c5b7a3c998dcd3d4f0e3f0ce65974 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 15:39:33 -0400
Subject: [PATCH 2704/8313] fix types

---
 Utility/BadPrelude.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 7921a7e9b7..8e4105ceea 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -12,11 +12,11 @@ head :: [a] -> a
 head = Prelude.head
 
 {- tail is also partial -}
-tail :: [a] -> a
+tail :: [a] -> [a]
 tail = Prelude.tail
 
 {- init too -}
-init :: [a] -> a
+init :: [a] -> [a]
 init = Prelude.init
 
 {- last too -}

From 111b6937ec1110feed024deee6fa95cdb78b9c95 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 15:57:47 -0400
Subject: [PATCH 2705/8313] avoid partial functions, and added check for
 correct sha content

---
 Git/Sha.hs | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/Git/Sha.hs b/Git/Sha.hs
index 475c2ba5f3..9b3a346505 100644
--- a/Git/Sha.hs
+++ b/Git/Sha.hs
@@ -10,17 +10,26 @@ module Git.Sha where
 import Common
 import Git.Types
 
-{- Runs an action that causes a git subcommand to emit a sha, and strips
+{- Runs an action that causes a git subcommand to emit a Sha, and strips
    any trailing newline, returning the sha. -}
 getSha :: String -> IO String -> IO Sha
-getSha subcommand a = do
-	t <- a
-	let t' = if last t == '\n'
-		then init t
-		else t
-	when (length t' /= shaSize) $
-		error $ "failed to read sha from git " ++ subcommand ++ " (" ++ t' ++ ")"
-	return $ Ref t'
+getSha subcommand a = maybe bad return =<< extractSha <$> a
+	where
+		bad = error $ "failed to read sha from git " ++ subcommand
+
+{- Extracts the Sha from a string. There can be a trailing newline after
+ - it, but nothing else. -}
+extractSha :: String -> Maybe Sha
+extractSha s
+	| len == shaSize = val s
+	| len == shaSize + 1 && length s' == shaSize = val s'
+	| otherwise = Nothing
+	where
+		len = length s
+		s' = firstLine s
+		val v
+			| all (`elem` "1234567890ABCDEFabcdef") v = Just $ Ref v
+			| otherwise = Nothing
 
 {- Size of a git sha. -}
 shaSize :: Int

From eb132a854ec9098acbc2a3caac5fd09947879268 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:04:08 -0400
Subject: [PATCH 2706/8313] avoid partial head function

(although it was used safely)
---
 Git/Config.hs | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 8d79639944..7b72eba5a3 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -5,14 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Git.Config (
-	get,
-	read,
-	hRead,
-	store
-) where
+module Git.Config where
 
-import Prelude hiding (read)
 import System.Posix.Directory
 import Control.Exception (bracket_)
 import qualified Data.Map as M
@@ -58,7 +52,7 @@ parse :: String -> M.Map String String
 parse [] = M.empty
 parse s
 	-- --list output will have an = in the first line
-	| '=' `elem` head ls = sep '=' ls
+	| all ('=' `elem`) (take 1 ls) = sep '=' ls
 	-- --null --list output separates keys from values with newlines
 	| otherwise = sep '\n' $ split "\0" s
 	where

From fbc3d32f7dca870bb68f16dcec0b601840313eb1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:58:04 -0400
Subject: [PATCH 2707/8313] avoid partial function, and parse git-ref output
 better

It's possible that a ref name might contain a space, this properly
preserves the space.
---
 Git/Ref.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Git/Ref.hs b/Git/Ref.hs
index 117ead8f22..0197ae7893 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -41,8 +41,8 @@ sha branch repo = process . L.unpack <$> showref repo
 matching :: Ref -> Repo -> IO [(Ref, Branch)]
 matching ref repo = do
 	r <- pipeRead [Param "show-ref", Param $ show ref] repo
-	return $ nubBy uref $ map (gen . words . L.unpack) (L.lines r)
+	return $ nubBy uniqref $ map (gen . L.unpack) (L.lines r)
 	where
-		gen l = (Ref $ head l, Ref $ last l)
-		uref (a, _) (b, _) = a == b
-
+		uniqref (a, _) (b, _) = a == b
+		gen l = let (r, b) = separate (== ' ') l in
+			(Ref r, Ref b)

From 0f9859ae51324ceb21471a09b423715d9fec7f23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:58:58 -0400
Subject: [PATCH 2708/8313] avoid partial function

---
 Locations.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Locations.hs b/Locations.hs
index 85fcb98887..73a2473b56 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -90,7 +90,8 @@ gitAnnexLocation key r
 		return $ inrepo ".git"  annexLocation key hashDirMixed
 	where
 		inrepo d = Git.workTree r  d
-		check locs = fromMaybe (head locs) <$> firstM doesFileExist locs
+		check locs@(l:_) = fromMaybe l <$> firstM doesFileExist locs
+		check [] = error "internal"
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath

From 817b1936e6dac3831721e8f07400181d87f365ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:59:18 -0400
Subject: [PATCH 2709/8313] add beginning, end

Safe versions of init and last
---
 Utility/BadPrelude.hs | 54 ++++++++++++++++++++++++++++++-------------
 1 file changed, 38 insertions(+), 16 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 8e4105ceea..49837b9273 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -7,22 +7,44 @@
 
 module Utility.BadPrelude where
 
-{- head is a partial function; head [] is an error -}
-head :: [a] -> a
-head = Prelude.head
-
-{- tail is also partial -}
-tail :: [a] -> [a]
-tail = Prelude.tail
-
-{- init too -}
-init :: [a] -> [a]
-init = Prelude.init
-
-{- last too -}
-last :: [a] -> a
-last = Prelude.last
-
 {- read should be avoided, as it throws an error -}
 read :: Read a => String -> a
 read = Prelude.read
+
+{- head is a partial function; head [] is an error
+ - Instead, use: take 1 -}
+head :: [a] -> a
+head = Prelude.head
+
+{- tail is also partial
+ - Instead, use: drop 1 -}
+tail :: [a] -> [a]
+tail = Prelude.tail
+
+{- init too
+ - Instead, use: beginning -}
+init :: [a] -> [a]
+init = Prelude.init
+
+{- last too
+ - Instead, use: end -}
+last :: [a] -> a
+last = Prelude.last
+
+{- All but the last element of a list.
+ - (Like init, but no error on an empty list.) -}
+beginning :: [a] -> [a]
+beginning [] = []
+beginning (x:xs) =  beginning' x xs
+	where
+		beginning' _ [] = []
+		beginning' y (z:zs) = y : beginning' z zs
+
+{- Like last, but no error on an empty list. -}
+end :: [a] -> [a]
+end [] = []
+end (x:xs) = end' x xs
+	where
+		end' y [] = [y]
+		end' _ (y:ys) = end' y ys
+

From b7e0d39abbc9a09c21c6f0103ad6c9f4547f81fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 16:59:48 -0400
Subject: [PATCH 2710/8313] remove some partial functions

A few were too hard to get rid of, and safe since the code
does check for an empty line.
---
 Logs/UUIDBased.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index 04b12887d7..b09d93f903 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -64,15 +64,15 @@ parseLog parser = M.fromListWith best . mapMaybe parse . lines
 			where
 				makepair v = Just (toUUID u, LogEntry ts v)
 				ws = words line
-				u = head ws
-				end = last ws
+				u = Prelude.head ws
+				t = Prelude.last ws
 				ts
-					| tskey `isPrefixOf` end =
-						pdate $ tail $ dropWhile (/= '=') end
+					| tskey `isPrefixOf` t =
+						pdate $ drop 1 $ dropWhile (/= '=') t
 					| otherwise = Unknown
 				info
 					| ts == Unknown = drop 1 ws
-					| otherwise = drop 1 $ init ws
+					| otherwise = drop 1 $ beginning ws
 				pdate s = case parseTime defaultTimeLocale "%s%Qs" s of
 					Nothing -> Unknown
 					Just d -> Date $ utcTimeToPOSIXSeconds d

From 95d2391f58ae240e7100f0d5488dd7246f71f3bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 18:11:42 -0400
Subject: [PATCH 2711/8313] more partial function removal

Left a few Prelude.head's in where it was checked not null and too hard to
remove, etc.
---
 Backend.hs            |  6 ++----
 Backend/SHA.hs        |  7 +++----
 CmdLine.hs            |  2 +-
 Command/InitRemote.hs | 25 +++++++++----------------
 Command/Map.hs        |  2 +-
 Command/Migrate.hs    |  2 +-
 Command/Status.hs     |  2 +-
 Command/Sync.hs       |  4 +++-
 Command/Uninit.hs     |  2 +-
 Command/Unused.hs     | 10 +++++-----
 Common.hs             |  4 +++-
 Config.hs             |  9 ++-------
 Git/CheckAttr.hs      |  7 +++----
 Git/UnionMerge.hs     |  2 +-
 Logs/Location.hs      |  4 ++--
 Logs/Remote.hs        |  5 ++---
 Logs/Trust.hs         | 14 ++++++--------
 Logs/UUID.hs          |  4 ++--
 Remote.hs             |  2 +-
 Remote/Bup.hs         |  8 ++++----
 Remote/Directory.hs   |  2 +-
 Remote/Rsync.hs       |  2 +-
 Upgrade/V1.hs         | 12 ++++++------
 Utility/BadPrelude.hs | 14 ++++++++++++--
 24 files changed, 73 insertions(+), 78 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 4743bb202e..2f788fcd00 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -107,7 +107,7 @@ chooseBackends fs = Annex.getState Annex.forcebackend >>= go
 			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
 		go (Just _) = do
 			l <- orderedList
-			return $ map (\f -> (Just $ head l, f)) fs
+			return $ map (\f -> (Just $ Prelude.head l, f)) fs
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend Annex
@@ -115,8 +115,6 @@ lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
 maybeLookupBackendName :: String -> Maybe (Backend Annex)
-maybeLookupBackendName s
-	| length matches == 1 = Just $ head matches
-	| otherwise = Nothing
+maybeLookupBackendName s = headMaybe matches
 	where
 		matches = filter (\b -> s == B.name b) list
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 7935b6d262..eca312944e 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -62,11 +62,10 @@ shaN :: SHASize -> FilePath -> Annex String
 shaN size file = do
 	showAction "checksum"
 	liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do
-		line <- hGetLine h
-		let bits = split " " line
-		if null bits
+		sha <- fst . separate (== ' ') <$> hGetLine h
+		if null sha
 			then error $ command ++ " parse error"
-			else return $ head bits
+			else return sha
 	where
 		command = fromJust $ shaCommand size
 
diff --git a/CmdLine.hs b/CmdLine.hs
index ebcca25aaf..7f708f15a1 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -51,7 +51,7 @@ parseCmd argv cmds options header = check $ getOpt Permute options argv
 		check (_, [], []) = err "missing command"
 		check (flags, name:rest, [])
 			| null matches = err $ "unknown command " ++ name
-			| otherwise = (flags, head matches, rest)
+			| otherwise = (flags, Prelude.head matches, rest)
 			where
 				matches = filter (\c -> name == cmdname c) cmds
 		check (_, _, errs) = err $ concat errs
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 600e17eb84..1e6bc2ef17 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -25,9 +25,13 @@ seek :: [CommandSeek]
 seek = [withWords start]
 
 start :: [String] -> CommandStart
-start ws = do
-	when (null ws) needname
-
+start [] = do
+	names <- remoteNames
+	error $ "Specify a name for the remote. " ++
+		if null names
+			then ""
+			else "Either a new name, or one of these existing special remotes: " ++ join " " names
+start (name:ws) = do
 	(u, c) <- findByName name
 	let fullconfig = config `M.union` c	
 	t <- findType fullconfig
@@ -36,15 +40,7 @@ start ws = do
 	next $ perform t u $ M.union config c
 
 	where
-		name = head ws
-		config = Logs.Remote.keyValToConfig $ tail ws
-		needname = do
-			let err s = error $ "Specify a name for the remote. " ++ s
-			names <- remoteNames
-			if null names
-				then err ""
-				else err $ "Either a new name, or one of these existing special remotes: " ++ join " " names
-			
+		config = Logs.Remote.keyValToConfig ws
 
 perform :: R.RemoteType Annex -> UUID -> R.RemoteConfig -> CommandPerform
 perform t u c = do
@@ -67,11 +63,8 @@ findByName name = do
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
-findByName' n m
-	| null matches = Nothing
-	| otherwise = Just $ head matches
+findByName' n = headMaybe . filter (matching . snd) . M.toList
 	where
-		matches = filter (matching . snd) $ M.toList m
 		matching c = case M.lookup nameKey c of
 			Nothing -> False
 			Just n'
diff --git a/Command/Map.hs b/Command/Map.hs
index 15ca5e1496..da129c8f65 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -73,7 +73,7 @@ hostname r
 	| otherwise = "localhost"
 
 basehostname :: Git.Repo -> String
-basehostname r = head $ split "." $ hostname r
+basehostname r = Prelude.head $ split "." $ hostname r
 
 {- A name to display for a repo. Uses the name from uuid.log if available,
  - or the remote name if not. -}
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 30288fc162..8778743ff5 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -31,7 +31,7 @@ start b file (key, oldbackend) = do
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = head <$> Backend.orderedList
+		choosebackend Nothing = Prelude.head <$> Backend.orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
diff --git a/Command/Status.hs b/Command/Status.hs
index 09da41987b..736d897ef3 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -116,7 +116,7 @@ remote_list level desc = stat n $ nojson $ lift $ do
 	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
-	return $ if null s then "0" else show (length rs) ++ "\n" ++ init s
+	return $ if null s then "0" else show (length rs) ++ "\n" ++ beginning s
 	where
 		n = desc ++ " repositories"
 
diff --git a/Command/Sync.hs b/Command/Sync.hs
index a25bcad8c1..36c4eeef06 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -12,6 +12,8 @@ import Command
 import qualified Annex.Branch
 import qualified Git.Command
 import qualified Git.Config
+import qualified Git.Ref
+import qualified Git
 
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -61,7 +63,7 @@ defaultRemote = do
 	fromRepo $ Git.Config.get ("branch." ++ branch ++ ".remote") "origin"
 
 currentBranch :: Annex String
-currentBranch = last . split "/" . L.unpack . head . L.lines <$>
+currentBranch = Git.Ref.describe . Git.Ref . firstLine . L.unpack <$>
 	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
 
 checkRemote :: String -> Annex ()
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index fc6f0cc275..21ad4c7df5 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -29,7 +29,7 @@ check = do
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
-		current_branch = Git.Ref . head . lines . B.unpack <$> revhead
+		current_branch = Git.Ref . Prelude.head . lines . B.unpack <$> revhead
 		revhead = inRepo $ Git.Command.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 8a70ff3356..ef398b01e1 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -154,13 +154,13 @@ excludeReferenced l = do
 		(S.fromList l)
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map (Git.Ref .  last) .
-			nubBy cmpheads .
+		refs = map (Git.Ref .  snd) .
+			nubBy uniqref .
 			filter ourbranches .
-			map words . lines . L.unpack
-		cmpheads a b = head a == head b
+			map (separate (== ' ')) . lines . L.unpack
+		uniqref (a, _) (b, _) = a == b
 		ourbranchend = '/' : show Annex.Branch.name
-		ourbranches ws = not $ ourbranchend `isSuffixOf` last ws
+		ourbranches (_, b) = not $ ourbranchend `isSuffixOf` b
 		removewith [] s = return $ S.toList s
 		removewith (a:as) s
 			| s == S.empty = return [] -- optimisation
diff --git a/Common.hs b/Common.hs
index 7e8dd9a2a5..f3dd701b14 100644
--- a/Common.hs
+++ b/Common.hs
@@ -6,7 +6,7 @@ import Control.Monad.State as X (liftIO)
 import Control.Exception.Extensible as X (IOException)
 
 import Data.Maybe as X
-import Data.List as X
+import Data.List as X hiding (head, tail, init, last)
 import Data.String.Utils as X
 
 import System.Path as X
@@ -25,3 +25,5 @@ import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
 import Utility.Monad as X
+
+import Utility.BadPrelude as X
diff --git a/Config.hs b/Config.hs
index 4cc4c18668..aa88858738 100644
--- a/Config.hs
+++ b/Config.hs
@@ -40,15 +40,10 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" +
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
-	safeparse <$> if not $ null cmd
+	(fromMaybe def . readMaybe) <$>
+		if not $ null cmd
 			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
 			else getConfig r "cost" ""
-	where
-		safeparse v
-			| null ws = def
-			| otherwise = fromMaybe def $ readMaybe $ head ws
-			where
-				ws = words v
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 1ea38beeac..0d3e798a1e 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -36,10 +36,9 @@ lookup attr files repo = do
 				, Param attr
 				, Params "-z --stdin"
 				] repo
-		topair l = (file, value)
+		topair l = (Git.Filename.decode file, value)
 			where 
-				file = Git.Filename.decode $ join sep $ take end bits
-				value = bits !! end
-				end = length bits - 1
+				file = join sep $ beginning bits
+				value = end bits !! 0
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index a9a51007f8..d5323af1d1 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -134,7 +134,7 @@ hashObject repo content = getSha subcmd $ do
 calcMerge :: [(Ref, [L.ByteString])] -> Either Ref [L.ByteString]
 calcMerge shacontents
 	| null reuseable = Right $ new
-	| otherwise = Left $ fst $ head reuseable
+	| otherwise = Left $ fst $ Prelude.head reuseable
 	where
 		reuseable = filter (\c -> sorteduniq (snd c) == new) shacontents
 		new = sorteduniq $ concat $ map snd shacontents
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 27b4d709e7..588962bc57 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -68,7 +68,7 @@ logFile key = hashDirLower key ++ keyFile key ++ ".log"
 {- Converts a log filename into a key. -}
 logFileKey :: FilePath -> Maybe Key
 logFileKey file
-	| end == ".log" = fileKey beginning
+	| ext == ".log" = fileKey base
 	| otherwise = Nothing
 	where
-		(beginning, end) = splitAt (length file - 4) file
+		(base, ext) = splitAt (length file - 4) file
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index 8d15f3151e..d9b41d8c47 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -73,14 +73,13 @@ configUnEscape = unescape
 			| c == '&' = entity rest
 			| otherwise = c : unescape rest
 		entity s = if ok
-				then chr (read num) : unescape rest
+				then chr (Prelude.read num) : unescape rest
 				else '&' : unescape s
 			where
 				num = takeWhile isNumber s
 				r = drop (length num) s
 				rest = drop 1 r
-				ok = not (null num) && 
-					not (null r) && head r == ';'
+				ok = not (null num) && take 1 r == ";"
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 196666a84e..5d769bd247 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -54,18 +54,16 @@ trustMap = do
 		Just m -> return m
 		Nothing -> do
 			overrides <- M.fromList <$> Annex.getState Annex.forcetrust
-			m <- (M.union overrides . simpleMap . parseLog parseTrust) <$>
+			m <- (M.union overrides . simpleMap . parseLog (Just . parseTrust)) <$>
 				Annex.Branch.get trustLog
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
 
-parseTrust :: String -> Maybe TrustLevel
-parseTrust s
-	| length w > 0 = Just $ parse $ head w
-	-- back-compat; the trust.log used to only list trusted repos
-	| otherwise = Just Trusted
+{- The trust.log used to only list trusted repos, without a field for the
+ - trust status, which is why this defaults to Trusted. -}
+parseTrust :: String -> TrustLevel
+parseTrust s = maybe Trusted parse $ headMaybe $ words s
 	where
-		w = words s
 		parse "1" = Trusted
 		parse "0" = UnTrusted
 		parse "X" = DeadTrusted
@@ -82,6 +80,6 @@ trustSet :: UUID -> TrustLevel -> Annex ()
 trustSet uuid@(UUID _) level = do
 	ts <- liftIO getPOSIXTime
 	Annex.Branch.change trustLog $
-		showLog showTrust . changeLog ts uuid level . parseLog parseTrust
+		showLog showTrust . changeLog ts uuid level . parseLog (Just . parseTrust)
 	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
 trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index b325c78b6f..18cbee61e4 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -57,9 +57,9 @@ fixBadUUID = M.fromList . map fixup . M.toList
 				kuuid = fromUUID k
 				isbad = not (isuuid kuuid) && isuuid lastword
 				ws = words $ value v
-				lastword = last ws
+				lastword = Prelude.last ws
 				fixeduuid = toUUID lastword
-				fixedvalue = unwords $ kuuid: init ws
+				fixedvalue = unwords $ kuuid: Prelude.init ws
 		-- For the fixed line to take precidence, it should be
 		-- slightly newer, but only slightly.
 		newertime (LogEntry (Date d) _) = d + minimumPOSIXTimeSlice
diff --git a/Remote.hs b/Remote.hs
index b1be60ec4a..aa86934143 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -103,7 +103,7 @@ byName' n = do
 	let match = filter matching allremotes
 	if null match
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
-		else return $ Right $ head match
+		else return $ Right $ Prelude.head match
 	where
 		matching r = n == name r || toUUID n == uuid r
 
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 8bd484b7dd..cbd5d584ac 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -209,20 +209,20 @@ bup2GitRemote "" = do
 	Git.Construct.fromAbsPath $ h  ".bup"
 bup2GitRemote r
 	| bupLocal r = 
-		if head r == '/'
+		if "/" `isPrefixOf` r
 			then Git.Construct.fromAbsPath r
 			else error "please specify an absolute path"
 	| otherwise = Git.Construct.fromUrl $ "ssh://" ++ host ++ slash dir
 		where
 			bits = split ":" r
-			host = head bits
+			host = Prelude.head bits
 			dir = join ":" $ drop 1 bits
 			-- "host:~user/dir" is not supported specially by bup;
 			-- "host:dir" is relative to the home directory;
 			-- "host:" goes in ~/.bup
 			slash d
-				| d == "" = "/~/.bup"
-				| head d == '/' = d
+				| null d = "/~/.bup"
+				| "/" `isPrefixOf` d = d
 				| otherwise = "/~/" ++ d
 
 bupLocal :: BupRepo -> Bool
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index a6077d813c..7f78b2f493 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -96,7 +96,7 @@ storeEncrypted d (cipher, enck) k = do
 
 storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 storeHelper d key a = do
-	let dest = head $ locations d key
+	let dest = Prelude.head $ locations d key
 	let dir = parentDir dest
 	createDirectoryIfMissing True dir
 	allowWrite dir
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 81107cb561..c281420773 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -188,7 +188,7 @@ rsyncRemote o params = do
    directories. -}
 rsyncSend :: RsyncOpts -> Key -> FilePath -> Annex Bool
 rsyncSend o k src = withRsyncScratchDir $ \tmp -> do
-	let dest = tmp  head (keyPaths k)
+	let dest = tmp  Prelude.head (keyPaths k)
 	liftIO $ createDirectoryIfMissing True $ parentDir dest
 	liftIO $ createLink src dest
 	rsyncRemote o
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 567cf8e5bf..80554dc3bc 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -146,20 +146,20 @@ oldlog2key l =
 readKey1 :: String -> Key
 readKey1 v = 
 	if mixup
-		then fromJust $ readKey $ join ":" $ tail bits
+		then fromJust $ readKey $ join ":" $ Prelude.tail bits
 		else Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
 	where
 		bits = split ":" v
-		b = head bits
+		b = Prelude.head bits
 		n = join ":" $ drop (if wormy then 3 else 1) bits
 		t = if wormy
-			then Just (read (bits !! 1) :: EpochTime)
+			then Just (Prelude.read (bits !! 1) :: EpochTime)
 			else Nothing
 		s = if wormy
-			then Just (read (bits !! 2) :: Integer)
+			then Just (Prelude.read (bits !! 2) :: Integer)
 			else Nothing
-		wormy = head bits == "WORM"
-		mixup = wormy && isUpper (head $ bits !! 1)
+		wormy = Prelude.head bits == "WORM"
+		mixup = wormy && isUpper (Prelude.head $ bits !! 1)
 
 showKey1 :: Key -> String
 showKey1 Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t } =
diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 49837b9273..47d38ae7ba 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -12,7 +12,7 @@ read :: Read a => String -> a
 read = Prelude.read
 
 {- head is a partial function; head [] is an error
- - Instead, use: take 1 -}
+ - Instead, use: take 1 or headMaybe -}
 head :: [a] -> a
 head = Prelude.head
 
@@ -27,10 +27,20 @@ init :: [a] -> [a]
 init = Prelude.init
 
 {- last too
- - Instead, use: end -}
+ - Instead, use: end or lastMaybe -}
 last :: [a] -> a
 last = Prelude.last
 
+{- Like head but Nothing on empty list. -}
+headMaybe :: [a] -> Maybe a
+headMaybe [] = Nothing
+headMaybe v = Just $ Prelude.head v
+
+{- Like last but Nothing on empty list. -}
+lastMaybe :: [a] -> Maybe a
+lastMaybe [] = Nothing
+lastMaybe v = Just $ Prelude.last v
+
 {- All but the last element of a list.
  - (Like init, but no error on an empty list.) -}
 beginning :: [a] -> [a]

From 9901fc04a0dd9972e8910b5039a9e51e43d2c732 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 18:23:07 -0400
Subject: [PATCH 2712/8313] move

---
 Utility/BadPrelude.hs | 13 ++++++++++++-
 Utility/Misc.hs       |  6 ------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 47d38ae7ba..825adfa025 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -7,7 +7,8 @@
 
 module Utility.BadPrelude where
 
-{- read should be avoided, as it throws an error -}
+{- read should be avoided, as it throws an error
+ - Instead, use: readMaybe -}
 read :: Read a => String -> a
 read = Prelude.read
 
@@ -31,6 +32,16 @@ init = Prelude.init
 last :: [a] -> a
 last = Prelude.last
 
+{- Attempts to read a value from a String.
+ -
+ - Ignores leading/trailing whitespace, and throws away any trailing
+ - text after the part that can be read.
+ -}
+readMaybe :: (Read a) => String -> Maybe a
+readMaybe s = case reads s of
+	((x,_):_) -> Just x
+	_ -> Nothing
+
 {- Like head but Nothing on empty list. -}
 headMaybe :: [a] -> Maybe a
 headMaybe [] = Nothing
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index e95ac4319d..1d3c0e6763 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -21,12 +21,6 @@ hGetContentsStrict = hGetContents >=> \s -> length s `seq` return s
 readFileStrict :: FilePath -> IO String
 readFileStrict = readFile >=> \s -> length s `seq` return s
 
-{- Attempts to read a value from a String. -}
-readMaybe :: (Read a) => String -> Maybe a
-readMaybe s = case reads s of
-	((x,_):_) -> Just x
-	_ -> Nothing
-
 {- Like break, but the character matching the condition is not included
  - in the second result list.
  -

From ec11908799201024f052203a08e95b75efa3c098 Mon Sep 17 00:00:00 2001
From: "http://gebi.myopenid.com/" 
Date: Fri, 16 Dec 2011 00:13:54 +0000
Subject: [PATCH 2713/8313]

---
 .../git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
index 0dad8856e8..8df3608dbe 100644
--- a/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
+++ b/doc/bugs/git-annex_losing_rsync_remotes_with_encryption_enabled.mdwn
@@ -99,3 +99,5 @@ And yes, only the hash *annex copy* is checking for exists on the remote side. -
 > here's an already decrypted cipher -- it must be the right one!
 > 
 > Problem reproduced here, and fixed. [[done]] --[[Joey]]
+
+THX Joey! -- [[gebi]]

From 718a278f973e8005ce4ba76717218abcabe5d0b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Dec 2011 22:19:05 -0400
Subject: [PATCH 2714/8313] simplify

---
 Utility/BadPrelude.hs | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/Utility/BadPrelude.hs b/Utility/BadPrelude.hs
index 825adfa025..04c9d9b0b1 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/BadPrelude.hs
@@ -56,16 +56,9 @@ lastMaybe v = Just $ Prelude.last v
  - (Like init, but no error on an empty list.) -}
 beginning :: [a] -> [a]
 beginning [] = []
-beginning (x:xs) =  beginning' x xs
-	where
-		beginning' _ [] = []
-		beginning' y (z:zs) = y : beginning' z zs
+beginning l = Prelude.init l
 
 {- Like last, but no error on an empty list. -}
 end :: [a] -> [a]
 end [] = []
-end (x:xs) = end' x xs
-	where
-		end' y [] = [y]
-		end' _ (y:ys) = end' y ys
-
+end l = [Prelude.last l]

From a3e4bf8c168c1b67960735bc244fc4d96c9406b5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm_-2XlXNyd6cCLI4n_jaBNqVUOWwJquko"
 
Date: Fri, 16 Dec 2011 09:23:21 +0000
Subject: [PATCH 2715/8313]

---
 ...sed_repository:_starting_from_nothing.mdwn | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 doc/tips/centralised_repository:_starting_from_nothing.mdwn

diff --git a/doc/tips/centralised_repository:_starting_from_nothing.mdwn b/doc/tips/centralised_repository:_starting_from_nothing.mdwn
new file mode 100644
index 0000000000..b12246d368
--- /dev/null
+++ b/doc/tips/centralised_repository:_starting_from_nothing.mdwn
@@ -0,0 +1,75 @@
+If you are starting from nothing (no existing `git` or `git-annex` repository) and want to use a server as a centralised repository, try the following steps.
+
+On the server where you'll hold the "master" repository:
+
+	server$ cd /one/git
+	server$ mkdir m
+	server$ cd m
+	server$ git init --bare
+	Initialized empty Git repository in /one/git/m/
+	server$ git annex init origin
+	init origin ok
+	server$ 
+
+Clone that to the laptop:
+
+	laptop$ cd /other
+	laptop$ git clone ssh://server//one/git/m
+	Cloning into 'm'...
+	Warning: No xauth data; using fake authentication data for X11 forwarding.
+	remote: Counting objects: 5, done.        
+	remote: Compressing objects: 100% (3/3), done.        
+	remote: Total 5 (delta 0), reused 0 (delta 0)        
+	Receiving objects: 100% (5/5), done.
+	warning: remote HEAD refers to nonexistent ref, unable to checkout.
+
+	laptop$ cd m
+	laptop$ git annex init laptop
+	init laptop ok
+        laptop$ 
+
+Merge the `git-annex` repository (this is the bit that is often
+overlooked!):
+
+	laptop$ git annex merge	
+	merge . (merging "origin/git-annex" into git-annex...)
+	ok
+	laptop$ 
+
+Add some content:
+
+	laptop$ git annex addurl http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	"kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg"
+	addurl kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg (downloading http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg ...) --2011-12-15 08:13:10--  http://kitenet.net/~joey/screencasts/git-annex_coding_in_haskell.ogg
+	Resolving kitenet.net (kitenet.net)... 2001:41c8:125:49::10, 80.68.85.49
+	Connecting to kitenet.net (kitenet.net)|2001:41c8:125:49::10|:80... connected.
+	HTTP request sent, awaiting response... 200 OK
+	Length: 39362757 (38M) [audio/ogg]
+	Saving to: `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg'
+
+	100%[======================================>] 39,362,757  2.31M/s   in 17s     
+
+	2011-12-15 08:13:27 (2.21 MB/s) - `/other/m/.git/annex/tmp/URL--http&c%%kitenet.net%~joey%screencasts%git-annex_coding_in_haskell.ogg' saved [39362757/39362757]
+
+	(checksum...) ok
+	(Recording state in git...)
+	laptop$ git commit -m 'See Joey play.'
+	[master (root-commit) 106e923] See Joey play.
+	 1 files changed, 1 insertions(+), 0 deletions(-)
+	  create mode 120000 kitenet.net_~joey_screencasts_git-annex_coding_in_haskell.ogg
+	laptop$ 
+
+All fine, now push it back to the centralised master:
+
+	laptop$ git push
+	Counting objects: 20, done.
+	Delta compression using up to 4 threads.
+	Compressing objects: 100% (11/11), done.
+	Writing objects: 100% (18/18), 1.50 KiB, done.
+	Total 18 (delta 1), reused 1 (delta 0)
+	To ssh://server//one/git/m
+	   3ba1386..ad3bc9e  git-annex -> git-annex
+	laptop$ 
+
+You can add more "client" repositories by following the `laptop`
+sequence of operations.

From 87c1c103eacf153aeaa79be9fd896df147a76c10 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Dec 2011 16:56:31 -0400
Subject: [PATCH 2716/8313] add back message

---
 Command/Reinject.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 906f7c5176..0648e90fca 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -26,7 +26,11 @@ start (src:dest:[])
 	| otherwise = do
 		ifAnnexed src
 			(error $ "cannot used annexed file as src: " ++ src)
-			(next $ whenAnnexed (perform src) dest)
+			go
+	where
+		go = do
+			showStart "reinject" dest
+			next $ whenAnnexed (perform src) dest
 start _ = error "specify a src file and a dest file"
 
 perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform

From 9698eecf0f7410e03300db772c528a465ef58dc1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
 
Date: Sun, 18 Dec 2011 09:35:40 +0000
Subject: [PATCH 2717/8313]

---
 ..._synchronize_2_directories___40__like_unison__41__.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn

diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn
new file mode 100644
index 0000000000..86e317da87
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__.mdwn
@@ -0,0 +1,7 @@
+I would like to use git-annex to synchronize 2 directories in the same manner as unison.
+
+I'm starting with 2 directories. There is an overlap of the same set of files in each directory, but each directory also has additional files as well.
+
+I create a git annex in each directory but when I do a git pull it merges and produces conflicts on those files that are the same.
+
+What is the correct workflow for this type of scenario?

From 3f01795a26f80a42e0e6b7397e7b521d9dbbd7b3 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sun, 18 Dec 2011 11:59:11 +0000
Subject: [PATCH 2718/8313]

---
 doc/bugs/Remote_repo_and_set_operation_with_find.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/bugs/Remote_repo_and_set_operation_with_find.mdwn

diff --git a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
new file mode 100644
index 0000000000..2153cca4b2
--- /dev/null
+++ b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
@@ -0,0 +1,3 @@
+Currently, git annex find lists files that are present in the current repository, possibly restricted to a subdirectory. But it does not easily seem possible to get this information about a remote repository.
+
+I would find it useful if this command understood flags that makes it tell me what is present somewhere else (maybe "--on remote") and combinations of the flags ("--on remote1 --and --not-on remote2" or "--on disk1 --or --on disk2").

From a16f1accbcaae32372ecb9ffd2a9cf0f4ec5dec7 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sun, 18 Dec 2011 12:08:52 +0000
Subject: [PATCH 2719/8313] Added a comment

---
 ...ent_7_33db51096f568c65b22b4be0b5538c0d._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
new file mode 100644
index 0000000000..ce9f4266ff
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 7"
+ date="2011-12-18T12:08:51Z"
+ content="""
+I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
+
+Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
+* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+* \"unsynced\", which makes git annex sync skip the repo.
+
+Maybe central is enough.
+"""]]

From fa9d83f144d8c35d58f584fe2690ed575ce9e096 Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Sun, 18 Dec 2011 13:11:41 +0100
Subject: [PATCH 2720/8313] fix syntax

---
 .../comment_7_33db51096f568c65b22b4be0b5538c0d._comment       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
index ce9f4266ff..02d68d245c 100644
--- a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -7,8 +7,8 @@
 I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
 
 Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
-* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
-* \"unsynced\", which makes git annex sync skip the repo.
+ * \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+ * \"unsynced\", which makes git annex sync skip the repo.
 
 Maybe central is enough.
 """]]

From 0bba09cd5b553118a0e956859cb5ffc9bc79a92c Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Sun, 18 Dec 2011 13:12:28 +0100
Subject: [PATCH 2721/8313] fix syntax

---
 .../comment_7_33db51096f568c65b22b4be0b5538c0d._comment      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
index 02d68d245c..753a2af169 100644
--- a/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
+++ b/doc/forum/pure_git-annex_only_workflow/comment_7_33db51096f568c65b22b4be0b5538c0d._comment
@@ -7,8 +7,9 @@
 I agree on the naming suggestions, and that it does not suit everybody. Maybe I’ll think some more about it. The point is: I’m trying to make live easy for those who do not want to manually create some complicated setup, so if it needs configuration, it is already off that track. But turning the current behavior into something people have to configure is also not well received by the users.
 
 Given that \"git annex sync\" is a new command, maybe it is fine to have this as a default behavior, and offer an easy way out. The easy way out could be one of two flags that can be set for a repo (or a remote):
- * \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
- * \"unsynced\", which makes git annex sync skip the repo.
+
+* \"central\", which makes git annex sync only push and pull to and that repo (unless a different remote is given on the command line)
+* \"unsynced\", which makes git annex sync skip the repo.
 
 Maybe central is enough.
 """]]

From 6a9b3c2f22eb348307c53fa16b39f0807196873c Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sun, 18 Dec 2011 13:57:34 +0000
Subject: [PATCH 2722/8313] Added a comment

---
 ...comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment

diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment
new file mode 100644
index 0000000000..4682ea64f7
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_1_5c3ee8a8aaa6d0918c0cc9683ce177ae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 1"
+ date="2011-12-18T13:57:33Z"
+ content="""
+Are the files identical or different? I today did something like that with similar, but not identical directories containing media files, and git happily merged them. but there, same files had same content.
+
+Also, make sure you use the same backend. In my case, one of the machines runs Debian stable, so I use the WORM backend, not the SHA backend.
+"""]]

From e341cdf0da8ae9398f0f96370b0a64950c1ffc30 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Mon, 19 Dec 2011 12:45:19 +0000
Subject: [PATCH 2723/8313] Added a comment: extra level of indirection

---
 ..._f63de6fe2f7189c8c2908cc41c4bc963._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment
new file mode 100644
index 0000000000..7398ac5614
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_6_f63de6fe2f7189c8c2908cc41c4bc963._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="extra level of indirection"
+ date="2011-12-19T12:45:18Z"
+ content="""
+Surely this could be handled with an extra layer of indirection?
+
+git-annex would ensure that every directory containing annexed data contains a new symlink `.git-annex` which points to `$git_root/.git/annex`.  Then every symlink to an annexed object uses a relative symlink via this: `.git_annex/objects/xx/yy/ZZZZZZZZZZ`.  Even though this symlink is relative, moving it to a different directory would not break anything: if the move destination directory already contained other annexed data, it would also already contain `.git-annex` so git-annex wouldn't need to do anything.  And if it didn't, git-annex would simply create a new `.git-annex` symlink there.
+
+These `.git-annex` symlinks could either be added to `.gitignore`, or manually/automatically checked in to the current branch - I'm not sure which would be best.  There's also the option of using multiple levels of indirection:
+
+    foo/bar/baz/.git-annex -> ../.git-annex
+    foo/bar/.git-annex -> ../.git-annex
+    foo/.git-annex -> ../.git-annex
+    .git-annex -> .git/annex
+
+I'm not sure whether this would bring any advantages.  It might bring a performance hit due to the kernel having to traverse more symlinks, but without benchmarking it's difficult to say how much.  I'd expect it only to be an issue with a large number of deep directory trees.
+"""]]

From 9b32d02a241a547feb827465ee7693c1f832626b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Dec 2011 18:22:25 +0000
Subject: [PATCH 2724/8313] Added a comment

---
 ...comment_7_7f20d0b2f6ed1c34021a135438037306._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment
new file mode 100644
index 0000000000..0a045feb63
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_7_7f20d0b2f6ed1c34021a135438037306._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-12-19T18:22:25Z"
+ content="""
+That seems an excellent idea, also eliminating the need for git annex fix after moving. 
+
+However, I think CVS and svn have taught us the pain associated with a version control system putting something in every subdirectory. Would this pain be worth avoiding the minor pain of needing git annex fix and sometimes being unable to follow renames?
+"""]]

From cddd521e1feb64a1264a3f2c222076a0a994abff Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Dec 2011 18:24:59 +0000
Subject: [PATCH 2725/8313] Added a comment

---
 ...comment_2_648946353c6d90c57351cce4010f1301._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment

diff --git a/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment
new file mode 100644
index 0000000000..bdd4b25e43
--- /dev/null
+++ b/doc/forum/using_git_annex_to_merge_and_synchronize_2_directories___40__like_unison__41__/comment_2_648946353c6d90c57351cce4010f1301._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-19T18:24:59Z"
+ content="""
+I'd recommend using the SHA backend for this, the WORM backend would produce conflicts if the files' modification times changed. 
+
+[[syncing_non-git_trees_with_git-annex]] describes one way to do it.
+"""]]

From 0b38397c6dd49fb0abd399ecebf1298f148e87c3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Dec 2011 18:29:01 +0000
Subject: [PATCH 2726/8313] Added a comment

---
 ...mment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment b/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment
new file mode 100644
index 0000000000..d33a296ca1
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_8_6e5b42fdb7801daadc0b3046cbc3d51e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2011-12-19T18:29:01Z"
+ content="""
+I don't mind changing the behavior of git-annex sync, certianly..
+
+Looking thru git's documentation, I found some existing configuration that could be reused following your idea.
+There is a remote.name.skipDefaultUpdate and a remote.name.skipFetchAll. Though both have to do with fetches, not pushes.
+Another approach might be to use git's remote group stuff.
+"""]]

From f0f84dbe48764b043c3373dc0e6a014b3a400b49 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 19 Dec 2011 14:31:33 -0400
Subject: [PATCH 2727/8313] close

---
 doc/bugs/Remote_repo_and_set_operation_with_find.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
index 2153cca4b2..3e1acd4a81 100644
--- a/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
+++ b/doc/bugs/Remote_repo_and_set_operation_with_find.mdwn
@@ -1,3 +1,6 @@
 Currently, git annex find lists files that are present in the current repository, possibly restricted to a subdirectory. But it does not easily seem possible to get this information about a remote repository.
 
 I would find it useful if this command understood flags that makes it tell me what is present somewhere else (maybe "--on remote") and combinations of the flags ("--on remote1 --and --not-on remote2" or "--on disk1 --or --on disk2").
+
+> Almost. You're looking for `--in remote`, which was added 2 months ago.
+> [[done]] --[[Joey]] 

From 89fd3b2e4bf302dffbf37b885c490fa3c716ee1b Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Mon, 19 Dec 2011 22:56:26 +0000
Subject: [PATCH 2728/8313] Added a comment

---
 ...ent_9_ace319652f9c7546883b5152ddc82591._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment b/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment
new file mode 100644
index 0000000000..de656d6629
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_9_ace319652f9c7546883b5152ddc82591._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 9"
+ date="2011-12-19T22:56:26Z"
+ content="""
+Another option that would please the naive user without hindering the more advanced user: \"git annex init\", by default, creates a synced/master branch. \"git annex sync\" will pull from every /sync/master branch it finds, and also push to any /sync/master branch it finds, but will not create any. So by default (at least for new users), this provides simple one-step syncing.
+
+Advanced users can disable this per-repo by just deleting the synced/master branch. Presumably the logic will be: Every repo that should not be pushed to, because it has access to some central repo, should not have a synced/master branch. Every other repo, including the (or one of the few) central repos, will have the branch.
+
+This is not the most expressive solution, as it does not allow configuring syncing between arbitrary pairs of repos, but it feels like a good compromise between that and simplicity and transparency.
+
+I think it's about time that I provide less talk and more code. I’ll see when I find the time :-)
+"""]]

From 0de4341c97735e90ea8d90fa7f32579c268d0f84 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Tue, 20 Dec 2011 12:00:18 +0000
Subject: [PATCH 2729/8313] Added a comment

---
 ..._6a00500b24ba53248c78e1ffc8d1a591._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment
new file mode 100644
index 0000000000..d53022302d
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_8_6a00500b24ba53248c78e1ffc8d1a591._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 8"
+ date="2011-12-20T12:00:11Z"
+ content="""
+Personally I'd rather have working rename detection but I agree it's not 100% ideal to be littering multiple directories like this, so perhaps you could make it optional, e.g. based on a git config setting?
+
+Here are a few more considerations, some in defence of the approach, some against it:
+
+* `.git-annex` is hidden; `CVS/` is not.
+* Unlike `CVS/` and `.svn/`, it's only a symlink, not a directory containing other files.
+* It doesn't contain any data specific to that directory and could easily be regenerated if deleted accidentally or otherwise.
+* If a whole directory containing `.git-annex` was moved within the repository:
+    * git-annex would need to fix up these symlinks if and only if it's moved to a different depth within the tree.
+    * However, if the multi-level indirection approach is used, `.git-annex` in any subdirectory is *always* a symlink to `../.git-annex` so instead you would need to check that all of the new ancestors contain this symlink too, and optionally remove any no longer needed symlinks.
+    * In either case, git-annex already goes to the trouble of fixing symlinks, and if anything, I *think* this approach would reduce the number of symlinks which need checking (right?)
+* find `$git_root/foo -follow`, `diff -r` etc. would traverse into `$git_root/.git/annex`
+
+This last point is the only downside to this approach I can think of which gives me any noticeable cause for concern.  However, people are already use to working around this from CVS and svn days, e.g. `diff -r -x .svn` so I don't think it's anywhere near bad enough to rule it out.
+"""]]

From 92de7c8dcd598e6319e81778261ce900c2efe534 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 20 Dec 2011 14:56:12 +0000
Subject: [PATCH 2730/8313] Added a comment

---
 .../comment_9_75e0973f6d573df615e01005ebcea87d._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment
new file mode 100644
index 0000000000..919455bdcc
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_9_75e0973f6d573df615e01005ebcea87d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2011-12-20T14:56:12Z"
+ content="""
+Git can follow the rename fine if the file is committed before `git annex fix` (you can git commit -n to see this), so 
+making git-annex pre-commit generate a fixup commit before the staged commit would be one way. Or the other two ways I originally mentioned when writing down this minor issue. I like all those approaches better than .git-annex clutter.
+"""]]

From dc5ed8d3b677f3c45118f5d5a89ad528ede7b171 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 11:01:50 -0400
Subject: [PATCH 2731/8313] amusing name

This is both a partial Prelude that conflicts with the real one, and a
way to guard against the Prelude's partial functions.
---
 Common.hs                                    | 2 +-
 Utility/{BadPrelude.hs => PartialPrelude.hs} | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename Utility/{BadPrelude.hs => PartialPrelude.hs} (92%)

diff --git a/Common.hs b/Common.hs
index f3dd701b14..90895f08e8 100644
--- a/Common.hs
+++ b/Common.hs
@@ -26,4 +26,4 @@ import Utility.Path as X
 import Utility.Directory as X
 import Utility.Monad as X
 
-import Utility.BadPrelude as X
+import Utility.PartialPrelude as X
diff --git a/Utility/BadPrelude.hs b/Utility/PartialPrelude.hs
similarity index 92%
rename from Utility/BadPrelude.hs
rename to Utility/PartialPrelude.hs
index 04c9d9b0b1..ad857196d6 100644
--- a/Utility/BadPrelude.hs
+++ b/Utility/PartialPrelude.hs
@@ -1,11 +1,11 @@
-{- Some stuff from Prelude should not be used, as it tends to be a source
- - of bugs.
+{- Parts of the Prelude are partial functions, which are a common source of
+ - bugs.
  -
  - This exports functions that conflict with the prelude, which avoids
  - them being accidentially used.
  -}
 
-module Utility.BadPrelude where
+module Utility.PartialPrelude where
 
 {- read should be avoided, as it throws an error
  - Instead, use: readMaybe -}

From da0bdc1a57d44ecfc7040affed725db4528d359a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 12:23:49 -0400
Subject: [PATCH 2732/8313] Fix the hook special remote, which bitrotted a
 while ago.

---
 Remote/Hook.hs   | 7 +++----
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index ab84533b28..5c761f43b0 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -62,13 +62,12 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 		env s v = ("ANNEX_" ++ s, v)
 		keyenv =
 			[ env "KEY" (show k)
-			, env "HASH_1" hash_1
-			, env "HASH_2" hash_2
+			, env "HASH_1" (hashbits !! 0)
+			, env "HASH_2" (hashbits !! 1)
 			]
 		fileenv Nothing = []
 		fileenv (Just file) =  [env "FILE" file]
-		[hash_1, hash_2, _rest] =
-			map takeDirectory $ splitPath $ hashDirMixed k
+		hashbits = map takeDirectory $ splitPath $ hashDirMixed k
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
diff --git a/debian/changelog b/debian/changelog
index 3c977e817d..e187d8f6fa 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Union merge now finds the least expensive way to represent the merge.
   * reinject: Add a sanity check for using an annexed file as the source file.
   * Properly handle multiline git config values.
+  * Fix the hook special remote, which bitrotted a while ago.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From ee3b5b2a4279292d55af43c772cdfd0c56420798 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 14:37:53 -0400
Subject: [PATCH 2733/8313] use Common in a few more modules

---
 Annex.hs        |  6 +++---
 Annex/Ssh.hs    |  4 +---
 Git/CatFile.hs  | 10 +++++-----
 Git/Filename.hs | 11 +++++------
 Git/Index.hs    |  2 ++
 Git/LsFiles.hs  | 14 +++++++-------
 Git/LsTree.hs   |  4 ++--
 Git/Queue.hs    | 10 +++++-----
 8 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index acc38a5db9..3021732f05 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -29,7 +29,7 @@ import Common
 import qualified Git
 import qualified Git.Config
 import Git.CatFile
-import Git.Queue
+import qualified Git.Queue
 import Types.Backend
 import qualified Types.Remote
 import Types.Crypto
@@ -57,7 +57,7 @@ data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [Backend Annex]
 	, remotes :: [Types.Remote.Remote Annex]
-	, repoqueue :: Queue
+	, repoqueue :: Git.Queue.Queue
 	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
@@ -80,7 +80,7 @@ newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, repoqueue = Git.Queue.empty
+	, repoqueue = Git.Queue.new
 	, output = NormalOutput
 	, force = False
 	, fast = False
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index fe83aad001..81e488b41d 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -7,11 +7,9 @@
 
 module Annex.Ssh where
 
-import Control.Monad.State (liftIO)
-
+import Common
 import qualified Git
 import qualified Git.Url
-import Utility.SafeCommand
 import Types
 import Config
 import Annex.UUID
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 2cef9d5b3d..16f0b11b95 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -19,10 +19,10 @@ import System.IO
 import qualified Data.ByteString.Char8 as S
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import Common
 import Git
 import Git.Sha
 import Git.Command
-import Utility.SafeCommand
 
 type CatFileHandle = (PipeHandle, Handle, Handle)
 
@@ -53,21 +53,21 @@ catObject (_, from, to) object = do
 		[sha, objtype, size]
 			| length sha == shaSize &&
 			  validobjtype objtype -> handle size
-			| otherwise -> empty
+			| otherwise -> dne
 		_
-			| header == show object ++ " missing" -> empty
+			| header == show object ++ " missing" -> dne
 			| otherwise -> error $ "unknown response from git cat-file " ++ header
 	where
 		handle size = case reads size of
 			[(bytes, "")] -> readcontent bytes
-			_ -> empty
+			_ -> dne
 		readcontent bytes = do
 			content <- S.hGet from bytes
 			c <- hGetChar from
 			when (c /= '\n') $
 				error "missing newline from git cat-file"
 			return $ L.fromChunks [content]
-		empty = return L.empty
+		dne = return L.empty
 		validobjtype t
 			| t == "blob" = True
 			| t == "commit" = True
diff --git a/Git/Filename.hs b/Git/Filename.hs
index 69f36d0861..35b5532507 100644
--- a/Git/Filename.hs
+++ b/Git/Filename.hs
@@ -13,22 +13,21 @@ import Data.Char
 import Data.Word (Word8)
 import Text.Printf
 
+import Common
+
 decode :: String -> FilePath
 decode [] = []
 decode f@(c:s)
 	-- encoded strings will be inside double quotes
-	| c == '"' = unescape ("", middle)
+	| c == '"' && end s == ['"'] = unescape ("", beginning s)
 	| otherwise = f
 	where
 		e = '\\'
-		middle = init s
 		unescape (b, []) = b
 		-- look for escapes starting with '\'
-		unescape (b, v) = b ++ beginning ++ unescape (handle rest)
+		unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
 			where
 				pair = span (/= e) v
-				beginning = fst pair
-				rest = snd pair
 		isescape x = x == e
 		-- \NNN is an octal encoded character
 		handle (x:n1:n2:n3:rest)
@@ -38,7 +37,7 @@ decode f@(c:s)
 						isOctDigit n2 &&
 						isOctDigit n3
 					fromoctal = [chr $ readoctal [n1, n2, n3]]
-					readoctal o = read $ "0o" ++ o :: Int
+					readoctal o = Prelude.read $ "0o" ++ o :: Int
 		-- \C is used for a few special characters
 		handle (x:nc:rest)
 			| isescape x = ([echar nc], rest)
diff --git a/Git/Index.hs b/Git/Index.hs
index aaf54e032e..e58f1edd67 100644
--- a/Git/Index.hs
+++ b/Git/Index.hs
@@ -9,6 +9,8 @@ module Git.Index where
 
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
+import Common
+
 {- Forces git to use the specified index file.
  -
  - Returns an action that will reset back to the default
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 0c71ed884b..0de86383d3 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -15,9 +15,9 @@ module Git.LsFiles (
 	typeChangedStaged,
 ) where
 
+import Common
 import Git
 import Git.Command
-import Utility.SafeCommand
 
 {- Scans for files that are checked into git at the specified locations. -}
 inRepo :: [FilePath] -> Repo -> IO [FilePath]
@@ -43,10 +43,10 @@ stagedNotDeleted :: [FilePath] -> Repo -> IO [FilePath]
 stagedNotDeleted = staged' [Param "--diff-filter=ACMRT"]
 
 staged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
-staged' middle l = pipeNullSplit $ start ++ middle ++ end
+staged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix
 	where
-		start = [Params "diff --cached --name-only -z"]
-		end = Param "--" : map File l
+		prefix = [Params "diff --cached --name-only -z"]
+		suffix = Param "--" : map File l
 
 {- Returns a list of files that have unstaged changes. -}
 changedUnstaged :: [FilePath] -> Repo -> IO [FilePath]
@@ -65,7 +65,7 @@ typeChanged :: [FilePath] -> Repo -> IO [FilePath]
 typeChanged = typeChanged' []
 
 typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
-typeChanged' middle l = pipeNullSplit $ start ++ middle ++ end
+typeChanged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix
 	where
-		start = [Params "diff --name-only --diff-filter=T -z"]
-		end = Param "--" : map File l
+		prefix = [Params "diff --name-only --diff-filter=T -z"]
+		suffix = Param "--" : map File l
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 919e9af83a..aae7f1263b 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -16,10 +16,10 @@ import Control.Applicative
 import System.Posix.Types
 import qualified Data.ByteString.Lazy.Char8 as L
 
+import Common
 import Git
 import Git.Command
 import qualified Git.Filename
-import Utility.SafeCommand
 
 data TreeItem = TreeItem
 	{ mode :: FileMode
@@ -37,7 +37,7 @@ lsTree t repo = map parseLsTree <$>
  - (The --long format is not currently supported.) -}
 parseLsTree :: L.ByteString -> TreeItem
 parseLsTree l = TreeItem 
-	{ mode = fst $ head $ readOct $ L.unpack m
+	{ mode = fst $ Prelude.head $ readOct $ L.unpack m
 	, typeobj = L.unpack t
 	, sha = L.unpack s
 	, file = Git.Filename.decode $ L.unpack f
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 73470b1f0e..25c5b073c7 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -7,7 +7,7 @@
 
 module Git.Queue (
 	Queue,
-	empty,
+	new,
 	add,
 	size,
 	full,
@@ -18,9 +18,9 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Control.Monad (forM_)
 import Utility.SafeCommand
 
+import Common
 import Git
 import Git.Command
 
@@ -50,8 +50,8 @@ maxSize :: Int
 maxSize = 10240
 
 {- Constructor for empty queue. -}
-empty :: Queue
-empty = Queue 0 M.empty
+new :: Queue
+new = Queue 0 M.empty
 
 {- Adds an action to a queue. -}
 add :: Queue -> String -> [CommandParam] -> [FilePath] -> Queue
@@ -76,7 +76,7 @@ full (Queue n _) = n > maxSize
 flush :: Queue -> Repo -> IO Queue
 flush (Queue _ m) repo = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
-	return empty
+	return new
 
 {- Runs an Action on a list of files in a git repository.
  -

From 4eec36c816fd0160f9963515d17d63031b2cab44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 15:13:36 -0400
Subject: [PATCH 2734/8313] added a test of the hook special remote

---
 test.hs | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/test.hs b/test.hs
index daa2661b6e..86423ba0f2 100644
--- a/test.hs
+++ b/test.hs
@@ -102,6 +102,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_fsck
 	, test_migrate
 	, test_unused
+	, test_hook_remote
 	]
 
 test_init :: Test
@@ -482,6 +483,32 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 			r <- Backend.lookupFile f
 			return $ fst $ fromJust r
 
+test_hook_remote :: Test
+test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
+	git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed"
+	createDirectory dir
+	git_config "annex.foo-store-hook" $
+		"cp $ANNEX_FILE " ++ loc
+	git_config "annex.foo-retrieve-hook" $
+		"cp " ++ loc ++ " $ANNEX_FILE"
+	git_config "annex.foo-remove-hook" $
+		"rm -f " ++ loc
+	git_config "annex.foo-checkpresent-hook" $
+		"if [ -e " ++ loc ++ " ]; then echo $ANNEX_KEY; fi"
+	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to hook remote failed"
+	annexed_present annexedfile
+	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from hook remote failed"
+	annexed_present annexedfile
+	where
+		dir = "dir"
+		loc = dir ++ "/$ANNEX_KEY"
+		git_config k v = boolSystem "git" [Param "config", Param k, Param v]
+			@? "git config failed"
+
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
 git_annex :: String -> [String] -> IO Bool

From b6664d5c94a4cd63da10cadde487cb2ced265a23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 15:17:56 -0400
Subject: [PATCH 2735/8313] add tests of directory and rsync special remotes

---
 test.hs | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/test.hs b/test.hs
index 86423ba0f2..1474c975ef 100644
--- a/test.hs
+++ b/test.hs
@@ -103,6 +103,8 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_migrate
 	, test_unused
 	, test_hook_remote
+	, test_directory_remote
+	, test_rsync_remote
 	]
 
 test_init :: Test
@@ -503,12 +505,47 @@ test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from hook remote failed"
 	annexed_present annexedfile
+	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	not r @? "drop failed to fail"
+	annexed_present annexedfile
 	where
 		dir = "dir"
 		loc = dir ++ "/$ANNEX_KEY"
 		git_config k v = boolSystem "git" [Param "config", Param k, Param v]
 			@? "git config failed"
 
+test_directory_remote :: Test
+test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do
+	createDirectory "dir"
+	git_annex "initremote" (words $ "foo type=directory encryption=none directory=dir") @? "initremote failed"
+	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to directory remote failed"
+	annexed_present annexedfile
+	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from directory remote failed"
+	annexed_present annexedfile
+	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	not r @? "drop failed to fail"
+	annexed_present annexedfile
+
+test_rsync_remote :: Test
+test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
+	createDirectory "dir"
+	git_annex "initremote" (words $ "foo type=rsync encryption=none rsyncurl=dir") @? "initremote failed"
+	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to rsync remote failed"
+	annexed_present annexedfile
+	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
+	annexed_present annexedfile
+	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	not r @? "drop failed to fail"
+	annexed_present annexedfile
+
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
 git_annex :: String -> [String] -> IO Bool

From c4c8c80f49878aecf7b8fc8200fefb1962e23574 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 15:48:25 -0400
Subject: [PATCH 2736/8313] factor out -q

---
 test.hs | 214 ++++++++++++++++++++++++++++----------------------------
 1 file changed, 107 insertions(+), 107 deletions(-)

diff --git a/test.hs b/test.hs
index 1474c975ef..86eb006a38 100644
--- a/test.hs
+++ b/test.hs
@@ -109,7 +109,7 @@ blackbox = TestLabel "blackbox" $ TestList
 
 test_init :: Test
 test_init = "git-annex init" ~: TestCase $ innewrepo $ do
-	git_annex "init" ["-q", reponame] @? "init failed"
+	git_annex "init" [reponame] @? "init failed"
 	where
 		reponame = "test repo"
 
@@ -120,38 +120,38 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 		-- annexed file that later tests will use
 		basic = TestCase $ inmainrepo $ do
 			writeFile annexedfile $ content annexedfile
-			git_annex "add" ["-q", annexedfile] @? "add failed"
+			git_annex "add" [annexedfile] @? "add failed"
 			annexed_present annexedfile
 			writeFile sha1annexedfile $ content sha1annexedfile
-			git_annex "add" ["-q", sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
+			git_annex "add" [sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
 			annexed_present sha1annexedfile
 			writeFile ingitfile $ content ingitfile
 			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
 			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
-			git_annex "add" ["-q", ingitfile] @? "add ingitfile should be no-op"
+			git_annex "add" [ingitfile] @? "add ingitfile should be no-op"
 			unannexed ingitfile
 		sha1dup = TestCase $ intmpclonerepo $ do
 			writeFile sha1annexedfiledup $ content sha1annexedfiledup
-			git_annex "add" ["-q", sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
+			git_annex "add" [sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
 			annexed_present sha1annexedfile
 		subdirs = TestCase $ intmpclonerepo $ do
 			createDirectory "dir"
 			writeFile "dir/foo" $ content annexedfile
-			git_annex "add" ["-q", "dir"] @? "add of subdir failed"
+			git_annex "add" ["dir"] @? "add of subdir failed"
 			createDirectory "dir2"
 			writeFile "dir2/foo" $ content annexedfile
 			changeWorkingDirectory "dir"
-			git_annex "add" ["-q", "../dir2"] @? "add of ../subdir failed"
+			git_annex "add" ["../dir2"] @? "add of ../subdir failed"
 
 test_reinject :: Test
 test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do
-	git_annex "drop" ["-q", "--force", sha1annexedfile] @? "drop failed"
+	git_annex "drop" ["--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
 	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
 	let key = show $ fromJust r
-	git_annex "reinject" ["-q", tmp, sha1annexedfile] @? "reinject failed"
-	git_annex "fromkey" ["-q", key, sha1annexedfiledup] @? "fromkey failed"
+	git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed"
+	git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed"
 	annexed_present sha1annexedfiledup
 	where
 		tmp = "tmpfile"
@@ -161,44 +161,44 @@ test_unannex = "git-annex unannex" ~: TestList [nocopy, withcopy]
 	where
 		nocopy = "no content" ~: intmpclonerepo $ do
 			annexed_notpresent annexedfile
-			git_annex "unannex" ["-q", annexedfile] @? "unannex failed with no copy"
+			git_annex "unannex" [annexedfile] @? "unannex failed with no copy"
 			annexed_notpresent annexedfile
 		withcopy = "with content" ~: intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			git_annex "unannex" ["-q", annexedfile, sha1annexedfile] @? "unannex failed"
+			git_annex "unannex" [annexedfile, sha1annexedfile] @? "unannex failed"
 			unannexed annexedfile
-			git_annex "unannex" ["-q", annexedfile] @? "unannex failed on non-annexed file"
+			git_annex "unannex" [annexedfile] @? "unannex failed on non-annexed file"
 			unannexed annexedfile
-			git_annex "unannex" ["-q", ingitfile] @? "unannex ingitfile should be no-op"
+			git_annex "unannex" [ingitfile] @? "unannex ingitfile should be no-op"
 			unannexed ingitfile
 
 test_drop :: Test
 test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 	where
 		noremote = "no remotes" ~: TestCase $ intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
-			r <- git_annex "drop" ["-q", annexedfile]
+			r <- git_annex "drop" [annexedfile]
 			not r @? "drop wrongly succeeded with no known copy of file"
 			annexed_present annexedfile
-			git_annex "drop" ["-q", "--force", annexedfile] @? "drop --force failed"
+			git_annex "drop" ["--force", annexedfile] @? "drop --force failed"
 			annexed_notpresent annexedfile
-			git_annex "drop" ["-q", annexedfile] @? "drop of dropped file failed"
-			git_annex "drop" ["-q", ingitfile] @? "drop ingitfile should be no-op"
+			git_annex "drop" [annexedfile] @? "drop of dropped file failed"
+			git_annex "drop" [ingitfile] @? "drop ingitfile should be no-op"
 			unannexed ingitfile
 		withremote = "with remote" ~: TestCase $ intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			git_annex "drop" ["-q", annexedfile] @? "drop failed though origin has copy"
+			git_annex "drop" [annexedfile] @? "drop failed though origin has copy"
 			annexed_notpresent annexedfile
 			inmainrepo $ annexed_present annexedfile
 		untrustedremote = "untrusted remote" ~: TestCase $ intmpclonerepo $ do
-			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
-			git_annex "get" ["-q", annexedfile] @? "get failed"
+			git_annex "untrust" ["origin"] @? "untrust of origin failed"
+			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			r <- git_annex "drop" ["-q", annexedfile]
+			r <- git_annex "drop" [annexedfile]
 			not r @? "drop wrongly suceeded with only an untrusted copy of the file"
 			annexed_present annexedfile
 			inmainrepo $ annexed_present annexedfile
@@ -207,15 +207,15 @@ test_get :: Test
 test_get = "git-annex get" ~: TestCase $ intmpclonerepo $ do
 	inmainrepo $ annexed_present annexedfile
 	annexed_notpresent annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_present annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file already here failed"
+	git_annex "get" [annexedfile] @? "get of file already here failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_present annexedfile
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
-	git_annex "get" ["-q", ingitfile] @? "get ingitfile should be no-op"
+	git_annex "get" [ingitfile] @? "get ingitfile should be no-op"
 	inmainrepo $ unannexed ingitfile
 	unannexed ingitfile
 
@@ -223,24 +223,24 @@ test_move :: Test
 test_move = "git-annex move" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file failed"
+	git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_notpresent annexedfile
-	git_annex "move" ["-q", "--from", "origin", annexedfile] @? "move --from of file already here failed"
+	git_annex "move" ["--from", "origin", annexedfile] @? "move --from of file already here failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_notpresent annexedfile
-	git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file failed"
+	git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed"
+	git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed"
 	inmainrepo $ annexed_present annexedfile
 	annexed_notpresent annexedfile
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "move" ["-q", "--to", "origin", ingitfile] @? "move of ingitfile should be no-op"
+	git_annex "move" ["--to", "origin", ingitfile] @? "move of ingitfile should be no-op"
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "move" ["-q", "--from", "origin", ingitfile] @? "move of ingitfile should be no-op"
+	git_annex "move" ["--from", "origin", ingitfile] @? "move of ingitfile should be no-op"
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
 
@@ -248,24 +248,24 @@ test_copy :: Test
 test_copy = "git-annex copy" ~: TestCase $ intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file failed"
+	git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "copy" ["-q", "--from", "origin", annexedfile] @? "copy --from of file already here failed"
+	git_annex "copy" ["--from", "origin", annexedfile] @? "copy --from of file already here failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "copy" ["-q", "--to", "origin", annexedfile] @? "copy --to of file already there failed"
+	git_annex "copy" ["--to", "origin", annexedfile] @? "copy --to of file already there failed"
 	annexed_present annexedfile
 	inmainrepo $ annexed_present annexedfile
-	git_annex "move" ["-q", "--to", "origin", annexedfile] @? "move --to of file already there failed"
+	git_annex "move" ["--to", "origin", annexedfile] @? "move --to of file already there failed"
 	annexed_notpresent annexedfile
 	inmainrepo $ annexed_present annexedfile
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "copy" ["-q", "--to", "origin", ingitfile] @? "copy of ingitfile should be no-op"
+	git_annex "copy" ["--to", "origin", ingitfile] @? "copy of ingitfile should be no-op"
 	unannexed ingitfile
 	inmainrepo $ unannexed ingitfile
-	git_annex "copy" ["-q", "--from", "origin", ingitfile] @? "copy of ingitfile should be no-op"
+	git_annex "copy" ["--from", "origin", ingitfile] @? "copy of ingitfile should be no-op"
 	checkregularfile ingitfile
 	checkcontent ingitfile
 
@@ -273,36 +273,36 @@ test_lock :: Test
 test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	-- regression test: unlock of not present file should skip it
 	annexed_notpresent annexedfile
-	r <- git_annex "unlock" ["-q", annexedfile]
+	r <- git_annex "unlock" [annexedfile]
 	not r @? "unlock failed to fail with not present file"
 	annexed_notpresent annexedfile
 
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "unlock" ["-q", annexedfile] @? "unlock failed"		
+	git_annex "unlock" [annexedfile] @? "unlock failed"		
 	unannexed annexedfile
 	-- write different content, to verify that lock
 	-- throws it away
 	changecontent annexedfile
 	writeFile annexedfile $ content annexedfile ++ "foo"
-	git_annex "lock" ["-q", annexedfile] @? "lock failed"
+	git_annex "lock" [annexedfile] @? "lock failed"
 	annexed_present annexedfile
-	git_annex "unlock" ["-q", annexedfile] @? "unlock failed"		
+	git_annex "unlock" [annexedfile] @? "unlock failed"		
 	unannexed annexedfile
 	changecontent annexedfile
-	git_annex "add" ["-q", annexedfile] @? "add of modified file failed"
+	git_annex "add" [annexedfile] @? "add of modified file failed"
 	runchecks [checklink, checkunwritable] annexedfile
 	c <- readFile annexedfile
 	assertEqual "content of modified file" c (changedcontent annexedfile)
-	r' <- git_annex "drop" ["-q", annexedfile]
+	r' <- git_annex "drop" [annexedfile]
 	not r' @? "drop wrongly succeeded with no known copy of modified file"
 
 test_edit :: Test
 test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 	where t precommit = TestCase $ intmpclonerepo $ do
-		git_annex "get" ["-q", annexedfile] @? "get of file failed"
+		git_annex "get" [annexedfile] @? "get of file failed"
 		annexed_present annexedfile
-		git_annex "edit" ["-q", annexedfile] @? "edit failed"
+		git_annex "edit" [annexedfile] @? "edit failed"
 		unannexed annexedfile
 		changecontent annexedfile
 		if precommit
@@ -311,7 +311,7 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 				-- staged, normally git commit does this
 				boolSystem "git" [Param "add", File annexedfile]
 					@? "git add of edited file failed"
-				git_annex "pre-commit" ["-q"]
+				git_annex "pre-commit" []
 					@? "pre-commit failed"
 			else do
 				boolSystem "git" [Params "commit -q -a -m contentchanged"]
@@ -319,22 +319,22 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
 		assertEqual "content of modified file" c (changedcontent annexedfile)
-		r <- git_annex "drop" ["-q", annexedfile]
+		r <- git_annex "drop" [annexedfile]
 		not r @? "drop wrongly succeeded with no known copy of modified file"
 
 test_fix :: Test
 test_fix = "git-annex fix" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
-	git_annex "fix" ["-q", annexedfile] @? "fix of not present failed"
+	git_annex "fix" [annexedfile] @? "fix of not present failed"
 	annexed_notpresent annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "fix" ["-q", annexedfile] @? "fix of present file failed"
+	git_annex "fix" [annexedfile] @? "fix of present file failed"
 	annexed_present annexedfile
 	createDirectory subdir
 	boolSystem "git" [Param "mv", File annexedfile, File subdir]
 		@? "git mv failed"
-	git_annex "fix" ["-q", newfile] @? "fix of moved file failed"
+	git_annex "fix" [newfile] @? "fix of moved file failed"
 	runchecks [checklink, checkunwritable] newfile
 	c <- readFile newfile
 	assertEqual "content of moved file" c (content annexedfile)
@@ -344,17 +344,17 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 
 test_trust :: Test
 test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
-	git_annex "trust" ["-q", repo] @? "trust failed"
+	git_annex "trust" [repo] @? "trust failed"
 	trustcheck Logs.Trust.Trusted "trusted 1"
-	git_annex "trust" ["-q", repo] @? "trust of trusted failed"
+	git_annex "trust" [repo] @? "trust of trusted failed"
 	trustcheck Logs.Trust.Trusted "trusted 2"
-	git_annex "untrust" ["-q", repo] @? "untrust failed"
+	git_annex "untrust" [repo] @? "untrust failed"
 	trustcheck Logs.Trust.UnTrusted "untrusted 1"
-	git_annex "untrust" ["-q", repo] @? "untrust of untrusted failed"
+	git_annex "untrust" [repo] @? "untrust of untrusted failed"
 	trustcheck Logs.Trust.UnTrusted "untrusted 2"
-	git_annex "semitrust" ["-q", repo] @? "semitrust failed"
+	git_annex "semitrust" [repo] @? "semitrust failed"
 	trustcheck Logs.Trust.SemiTrusted "semitrusted 1"
-	git_annex "semitrust" ["-q", repo] @? "semitrust of semitrusted failed"
+	git_annex "semitrust" [repo] @? "semitrust of semitrusted failed"
 	trustcheck Logs.Trust.SemiTrusted "semitrusted 2"
 	where
 		repo = "origin"
@@ -369,36 +369,36 @@ test_fsck :: Test
 test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
-			git_annex "fsck" ["-q"] @? "fsck failed"
+			git_annex "fsck" [] @? "fsck failed"
 			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
 			fsck_should_fail "numcopies unsatisfied"
 			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
-			git_annex "get" ["-q", annexedfile] @? "get failed"
-			git_annex "untrust" ["-q", "origin"] @? "untrust of origin repo failed"
-			git_annex "untrust" ["-q", "."] @? "untrust of current repo failed"
+			git_annex "get" [annexedfile] @? "get failed"
+			git_annex "untrust" ["origin"] @? "untrust of origin repo failed"
+			git_annex "untrust" ["."] @? "untrust of current repo failed"
 			fsck_should_fail "content only available in untrusted (current) repository"
-			git_annex "trust" ["-q", "."] @? "trust of current repo failed"
-			git_annex "fsck" ["-q", annexedfile] @? "fsck failed on file present in trusted repo"
+			git_annex "trust" ["."] @? "trust of current repo failed"
+			git_annex "fsck" [annexedfile] @? "fsck failed on file present in trusted repo"
 		withremoteuntrusted = TestCase $ intmpclonerepo $ do
 			boolSystem "git" [Params "config annex.numcopies 2"] @? "git config failed"
-			git_annex "get" ["-q", annexedfile] @? "get failed"
-			git_annex "get" ["-q", sha1annexedfile] @? "get failed"
-			git_annex "fsck" ["-q"] @? "fsck failed with numcopies=2 and 2 copies"
-			git_annex "untrust" ["-q", "origin"] @? "untrust of origin failed"
+			git_annex "get" [annexedfile] @? "get failed"
+			git_annex "get" [sha1annexedfile] @? "get failed"
+			git_annex "fsck" [] @? "fsck failed with numcopies=2 and 2 copies"
+			git_annex "untrust" ["origin"] @? "untrust of origin failed"
 			fsck_should_fail "content not replicated to enough non-untrusted repositories"
 
 		corrupt f = do
-			git_annex "get" ["-q", f] @? "get of file failed"
+			git_annex "get" [f] @? "get of file failed"
 			Utility.FileMode.allowWrite f
 			writeFile f (changedcontent f)
-			r <- git_annex "fsck" ["-q"]
+			r <- git_annex "fsck" []
 			not r @? "fsck failed to fail with corrupted file content"
-			git_annex "fsck" ["-q"] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
+			git_annex "fsck" [] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
 		fsck_should_fail m = do
-			r <- git_annex "fsck" ["-q"]
+			r <- git_annex "fsck" []
 			not r @? "fsck failed to fail with " ++ m
 
 test_migrate :: Test
@@ -406,23 +406,23 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 	where t usegitattributes = TestCase $ intmpclonerepo $ do
 		annexed_notpresent annexedfile
 		annexed_notpresent sha1annexedfile
-		git_annex "migrate" ["-q", annexedfile] @? "migrate of not present failed"
-		git_annex "migrate" ["-q", sha1annexedfile] @? "migrate of not present failed"
-		git_annex "get" ["-q", annexedfile] @? "get of file failed"
-		git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
+		git_annex "migrate" [annexedfile] @? "migrate of not present failed"
+		git_annex "migrate" [sha1annexedfile] @? "migrate of not present failed"
+		git_annex "get" [annexedfile] @? "get of file failed"
+		git_annex "get" [sha1annexedfile] @? "get of file failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
 		if usegitattributes
 			then do
 				writeFile ".gitattributes" $ "* annex.backend=SHA1"
-				git_annex "migrate" ["-q", sha1annexedfile]
+				git_annex "migrate" [sha1annexedfile]
 					@? "migrate sha1annexedfile failed"
-				git_annex "migrate" ["-q", annexedfile]
+				git_annex "migrate" [annexedfile]
 					@? "migrate annexedfile failed"
 			else do
-				git_annex "migrate" ["-q", sha1annexedfile, "--backend", "SHA1"]
+				git_annex "migrate" [sha1annexedfile, "--backend", "SHA1"]
 					@? "migrate sha1annexedfile failed"
-				git_annex "migrate" ["-q", annexedfile, "--backend", "SHA1"]
+				git_annex "migrate" [annexedfile, "--backend", "SHA1"]
 					@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
@@ -431,9 +431,9 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 
 		-- check that reversing a migration works
 		writeFile ".gitattributes" $ "* annex.backend=WORM"
-		git_annex "migrate" ["-q", sha1annexedfile]
+		git_annex "migrate" [sha1annexedfile]
 			@? "migrate sha1annexedfile failed"
-		git_annex "migrate" ["-q", annexedfile]
+		git_annex "migrate" [annexedfile]
 			@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
@@ -451,8 +451,8 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	-- keys have to be looked up before files are removed
 	annexedfilekey <- annexeval $ findkey annexedfile
 	sha1annexedfilekey <- annexeval $ findkey sha1annexedfile
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
-	git_annex "get" ["-q", sha1annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
+	git_annex "get" [sha1annexedfile] @? "get of file failed"
 	checkunused []
 	boolSystem "git" [Params "rm -q", File annexedfile] @? "git rm failed"
 	checkunused []
@@ -466,17 +466,17 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	checkunused [annexedfilekey, sha1annexedfilekey]
 
 	-- good opportunity to test dropkey also
-	git_annex "dropkey" ["-q", "--force", show annexedfilekey]
+	git_annex "dropkey" ["--force", show annexedfilekey]
 		@? "dropkey failed"
 	checkunused [sha1annexedfilekey]
 
-	git_annex "dropunused" ["-q", "1", "2"] @? "dropunused failed"
+	git_annex "dropunused" ["1", "2"] @? "dropunused failed"
 	checkunused []
-	git_annex "dropunused" ["-q", "10", "501"] @? "dropunused failed on bogus numbers"
+	git_annex "dropunused" ["10", "501"] @? "dropunused failed on bogus numbers"
 
 	where
 		checkunused expectedkeys = do
-			git_annex "unused" ["-q"] @? "unused failed"
+			git_annex "unused" [] @? "unused failed"
 			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog ""
 			let unusedkeys = M.elems unusedmap
 			assertEqual "unused keys differ"
@@ -497,15 +497,15 @@ test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 		"rm -f " ++ loc
 	git_config "annex.foo-checkpresent-hook" $
 		"if [ -e " ++ loc ++ " ]; then echo $ANNEX_KEY; fi"
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to hook remote failed"
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to hook remote failed"
 	annexed_present annexedfile
-	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from hook remote failed"
+	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from hook remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
 	not r @? "drop failed to fail"
 	annexed_present annexedfile
 	where
@@ -518,15 +518,15 @@ test_directory_remote :: Test
 test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do
 	createDirectory "dir"
 	git_annex "initremote" (words $ "foo type=directory encryption=none directory=dir") @? "initremote failed"
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to directory remote failed"
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to directory remote failed"
 	annexed_present annexedfile
-	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from directory remote failed"
+	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from directory remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
 	not r @? "drop failed to fail"
 	annexed_present annexedfile
 
@@ -534,15 +534,15 @@ test_rsync_remote :: Test
 test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	createDirectory "dir"
 	git_annex "initremote" (words $ "foo type=rsync encryption=none rsyncurl=dir") @? "initremote failed"
-	git_annex "get" ["-q", annexedfile] @? "get of file failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
 	annexed_present annexedfile
-	git_annex "copy" ["-q", annexedfile, "--to", "foo"] @? "copy --to rsync remote failed"
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to rsync remote failed"
 	annexed_present annexedfile
-	git_annex "drop" ["-q", annexedfile, "--numcopies=2"] @? "drop failed"
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
 	annexed_notpresent annexedfile
-	git_annex "move" ["-q", annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
+	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" ["-q", annexedfile, "--numcopies=2"]
+	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
 	not r @? "drop failed to fail"
 	annexed_present annexedfile
 
@@ -556,7 +556,7 @@ git_annex command params = do
 		Right _ -> return True
 		Left _ -> return False
 	where
-		run = GitAnnex.run (command:params)
+		run = GitAnnex.run (command:"-q":params)
 
 -- Runs an action in the current annex. Note that shutdown actions
 -- are not run; this should only be used for actions that query state.

From f30d545256a56c9c39f2893aff2f75aad9dba924 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:02:59 -0400
Subject: [PATCH 2737/8313] remove seemingly unneeded dependencies

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index cef79be88c..6d2a2f8193 100644
--- a/Makefile
+++ b/Makefile
@@ -64,7 +64,7 @@ install: all
 		rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \
 	fi
 
-test: $(bins)
+test:
 	@if ! $(GHCMAKE) -O0 test; then \
 		echo "** not running test suite" >&2; \
 	else \
@@ -74,7 +74,7 @@ test: $(bins)
 		fi; \
 	fi
 
-testcoverage: $(bins)
+testcoverage:
 	rm -f test.tix test
 	ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test
 	./test

From 3d188d8db95d2a7e92e9adc4d672fa119ac6ebc8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:03:09 -0400
Subject: [PATCH 2738/8313] fix comment

---
 Annex.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex.hs b/Annex.hs
index 3021732f05..e5792fbcb2 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -115,7 +115,7 @@ getState = gets
 
 {- Applies a state mutation function to change the internal state. 
  -
- - Example: changeState $ \s -> s { quiet = True }
+ - Example: changeState $ \s -> s { output = QuietOutput }
  -}
 changeState :: (AnnexState -> AnnexState) -> Annex ()
 changeState = modify

From 9f65410fd823c5e98037bcb4be282892feedf53c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:03:14 -0400
Subject: [PATCH 2739/8313] run annex monad in quiet mode when testing

This cleans up the merging messages.
---
 test.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 86eb006a38..5c43c3d4d9 100644
--- a/test.hs
+++ b/test.hs
@@ -565,7 +565,7 @@ annexeval a = do
 	g <- Git.Construct.fromCwd
 	g' <- Git.Config.read g
 	s <- Annex.new g'
-	Annex.eval s a
+	Annex.eval s { Annex.output = Annex.QuietOutput } a
 
 innewrepo :: Assertion -> Assertion
 innewrepo a = withgitrepo $ \r -> indir r a

From 7dff46eeec44e34075f9506ce076e6ec0298c178 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:06:15 -0400
Subject: [PATCH 2740/8313] cleanup backend used when reversing migration

WORM is no longer the starting backend, so let's really reverse, getting
back to SHA256.
---
 test.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test.hs b/test.hs
index 5c43c3d4d9..d3d7609d8a 100644
--- a/test.hs
+++ b/test.hs
@@ -430,15 +430,15 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 		checkbackend sha1annexedfile backendSHA1
 
 		-- check that reversing a migration works
-		writeFile ".gitattributes" $ "* annex.backend=WORM"
+		writeFile ".gitattributes" $ "* annex.backend=SHA256"
 		git_annex "migrate" [sha1annexedfile]
 			@? "migrate sha1annexedfile failed"
 		git_annex "migrate" [annexedfile]
 			@? "migrate annexedfile failed"
 		annexed_present annexedfile
 		annexed_present sha1annexedfile
-		checkbackend annexedfile backendWORM
-		checkbackend sha1annexedfile backendWORM
+		checkbackend annexedfile backendSHA256
+		checkbackend sha1annexedfile backendSHA256
 		
 		where
 			checkbackend file expected = do
@@ -761,8 +761,8 @@ changedcontent f = (content f) ++ " (modified)"
 backendSHA1 :: Types.Backend Types.Annex
 backendSHA1 = backend_ "SHA1"
 
-backendWORM :: Types.Backend Types.Annex
-backendWORM = backend_ "WORM"
+backendSHA256 :: Types.Backend Types.Annex
+backendSHA256 = backend_ "SHA256"
 
 backend_ :: String -> Types.Backend Types.Annex
 backend_ name = Backend.lookupBackendName name

From 9cfa7a969c7449ef48fe97daadf8ff89e8a103b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:07:34 -0400
Subject: [PATCH 2741/8313] note

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 6d2a2f8193..93586762a9 100644
--- a/Makefile
+++ b/Makefile
@@ -81,6 +81,7 @@ testcoverage:
 	@echo ""
 	@hpc report test --exclude=Main --exclude=QC
 	@hpc markup test --exclude=Main --exclude=QC --destdir=.hpc >/dev/null
+	@echo "(See .hpc/ for test coverage details.)"
 
 # If ikiwiki is available, build static html docs suitable for being
 # shipped in the software package.

From 9308a60bb2457568bf600901f6dad5c96d6b53ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:26:39 -0400
Subject: [PATCH 2742/8313] add basic tests for some of the newer commands

For most this just makes sure they run successfully, and improves test
coverage statistics; it's hard to test the output of eg, find and status.
---
 test.hs | 43 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index d3d7609d8a..a3935cf66e 100644
--- a/test.hs
+++ b/test.hs
@@ -102,6 +102,12 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_fsck
 	, test_migrate
 	, test_unused
+	, test_addurl
+	, test_describe
+	, test_find
+	, test_merge
+	, test_status
+	, test_sync
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
@@ -343,7 +349,7 @@ test_fix = "git-annex fix" ~: intmpclonerepo $ do
 		newfile = subdir ++ "/" ++ annexedfile
 
 test_trust :: Test
-test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
+test_trust = "git-annex trust/untrust/semitrust/dead" ~: intmpclonerepo $ do
 	git_annex "trust" [repo] @? "trust failed"
 	trustcheck Logs.Trust.Trusted "trusted 1"
 	git_annex "trust" [repo] @? "trust of trusted failed"
@@ -352,6 +358,10 @@ test_trust = "git-annex trust/untrust/semitrust" ~: intmpclonerepo $ do
 	trustcheck Logs.Trust.UnTrusted "untrusted 1"
 	git_annex "untrust" [repo] @? "untrust of untrusted failed"
 	trustcheck Logs.Trust.UnTrusted "untrusted 2"
+	git_annex "dead" [repo] @? "dead failed"
+	trustcheck Logs.Trust.DeadTrusted "deadtrusted 1"
+	git_annex "dead" [repo] @? "dead of dead failed"
+	trustcheck Logs.Trust.DeadTrusted "deadtrusted 2"
 	git_annex "semitrust" [repo] @? "semitrust failed"
 	trustcheck Logs.Trust.SemiTrusted "semitrusted 1"
 	git_annex "semitrust" [repo] @? "semitrust of semitrusted failed"
@@ -485,6 +495,37 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 			r <- Backend.lookupFile f
 			return $ fst $ fromJust r
 
+test_addurl :: Test
+test_addurl = "git-annex addurl" ~: intmpclonerepo $ do
+	annexed_notpresent annexedfile
+	-- can't check download; test suite should not access network,
+	-- and starting up a web server seems excessive
+	git_annex "addurl" ["--fast", "http://example.com/nosuchfile"] @? "addurl failed"
+
+test_describe :: Test
+test_describe = "git-annex describe" ~: intmpclonerepo $ do
+	git_annex "describe" [".", "this repo"] @? "describe 1 failed"
+	git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed"
+
+test_find :: Test
+test_find = "git-annex find" ~: intmpclonerepo $ do
+	annexed_notpresent annexedfile
+	git_annex "find" [] @? "find failed"
+	git_annex "get" [] @? "get failed"
+	git_annex "find" [] @? "find failed"
+
+test_merge :: Test
+test_merge = "git-annex merge" ~: intmpclonerepo $ do
+	git_annex "merge" [] @? "merge failed"
+
+test_status :: Test
+test_status = "git-annex status" ~: intmpclonerepo $ do
+	git_annex "status" [] @? "status failed"
+
+test_sync :: Test
+test_sync = "git-annex sync" ~: intmpclonerepo $ do
+	git_annex "sync" [] @? "sync failed"
+
 test_hook_remote :: Test
 test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed"

From 1c28237e0c7a920823354d4a10381b300aeb95e0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 16:31:59 -0400
Subject: [PATCH 2743/8313] map: --fast disables use of dot to display map

Generally useful, and allows the test suite to test it.
---
 Command/Map.hs     | 13 +++++++++----
 debian/changelog   |  1 +
 doc/git-annex.mdwn |  5 +++--
 test.hs            | 12 ++++++++++++
 4 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/Command/Map.hs b/Command/Map.hs
index da129c8f65..0f32e1130d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -16,6 +16,7 @@ import qualified Git
 import qualified Git.Url
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Annex
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
@@ -40,10 +41,14 @@ start = do
 	trusted <- trustGet Trusted
 
 	liftIO $ writeFile file (drawMap rs umap trusted)
-	showLongNote $ "running: dot -Tx11 " ++ file
-	showOutput
-	r <- liftIO $ boolSystem "dot" [Param "-Tx11", File file]
-	next $ next $ return r
+	next $ next $ do
+		fast <- Annex.getState Annex.fast
+		if fast
+			then return True
+			else do
+				showLongNote $ "running: dot -Tx11 " ++ file
+				showOutput
+				liftIO $ boolSystem "dot" [Param "-Tx11", File file]
 	where
 		file = "map.dot"
 
diff --git a/debian/changelog b/debian/changelog
index e187d8f6fa..600687d94b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * reinject: Add a sanity check for using an annexed file as the source file.
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
+  * map: --fast disables use of dot to display map
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d7a51663fd..b3d671bb89 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -268,9 +268,10 @@ subdirectories).
   Helps you keep track of your repositories, and the connections between them,
   by going out and looking at all the ones it can get to, and generating a
   Graphviz file displaying it all. If the `dot` command is available, it is
-  used to display the file to your screen (using x11 backend).
+  used to display the file to your screen (using x11 backend). (To disable
+  this display, specify --fast)
 
-  Note that this only connects to hosts that the host it's run on can
+  This command only connects to hosts that the host it's run on can
   directly connect to. It does not try to tunnel through intermediate hosts.
   So it might not show all connections between the repositories in the network.
 
diff --git a/test.hs b/test.hs
index a3935cf66e..aa2cd8be75 100644
--- a/test.hs
+++ b/test.hs
@@ -108,6 +108,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_merge
 	, test_status
 	, test_sync
+	, test_map
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
@@ -526,6 +527,17 @@ test_sync :: Test
 test_sync = "git-annex sync" ~: intmpclonerepo $ do
 	git_annex "sync" [] @? "sync failed"
 
+test_map :: Test
+test_map = "git-annex map" ~: intmpclonerepo $ do
+	-- set descriptions, that will be looked for in the map
+	git_annex "describe" [".", "this repo"] @? "describe 1 failed"
+	git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed"
+	-- --fast avoids it running graphviz, not a build dependency
+	git_annex "map" ["--fast"] @? "map failed"
+	doesFileExist "map.dot" @? "map.dot not generated"
+	c <- readFile "map.dot"
+	not ("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? "map.dot bad content"
+
 test_hook_remote :: Test
 test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	git_annex "initremote" (words "foo type=hook encryption=none hooktype=foo") @? "initremote failed"

From cc88abd0ad84c0a196edbc7a67cec493f6f532b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 17:31:25 -0400
Subject: [PATCH 2744/8313] Test suite improvements. Current top-level test
 coverage: 68%

Been higher before, but a lot of new code has been added.
---
 debian/changelog |  1 +
 test.hs          | 62 +++++++++++++++++++++++++++++++++---------------
 2 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 600687d94b..e3e8079dcf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
+  * Test suite improvements. Current top-level test coverage: 68%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index aa2cd8be75..9ba0e5dd91 100644
--- a/test.hs
+++ b/test.hs
@@ -107,8 +107,12 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_find
 	, test_merge
 	, test_status
+	, test_version
 	, test_sync
 	, test_map
+	, test_uninit
+	, test_upgrade
+	, test_whereis
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
@@ -187,8 +191,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 			git_annex "get" [annexedfile] @? "get failed"
 			boolSystem "git" [Params "remote rm origin"]
 				@? "git remote rm origin failed"
-			r <- git_annex "drop" [annexedfile]
-			not r @? "drop wrongly succeeded with no known copy of file"
+			not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of file"
 			annexed_present annexedfile
 			git_annex "drop" ["--force", annexedfile] @? "drop --force failed"
 			annexed_notpresent annexedfile
@@ -205,8 +208,7 @@ test_drop = "git-annex drop" ~: TestList [noremote, withremote, untrustedremote]
 			git_annex "untrust" ["origin"] @? "untrust of origin failed"
 			git_annex "get" [annexedfile] @? "get failed"
 			annexed_present annexedfile
-			r <- git_annex "drop" [annexedfile]
-			not r @? "drop wrongly suceeded with only an untrusted copy of the file"
+			not <$> git_annex "drop" [annexedfile] @? "drop wrongly suceeded with only an untrusted copy of the file"
 			annexed_present annexedfile
 			inmainrepo $ annexed_present annexedfile
 
@@ -280,8 +282,7 @@ test_lock :: Test
 test_lock = "git-annex unlock/lock" ~: intmpclonerepo $ do
 	-- regression test: unlock of not present file should skip it
 	annexed_notpresent annexedfile
-	r <- git_annex "unlock" [annexedfile]
-	not r @? "unlock failed to fail with not present file"
+	not <$> git_annex "unlock" [annexedfile] @? "unlock failed to fail with not present file"
 	annexed_notpresent annexedfile
 
 	git_annex "get" [annexedfile] @? "get of file failed"
@@ -326,8 +327,7 @@ test_edit = "git-annex edit/commit" ~: TestList [t False, t True]
 		runchecks [checklink, checkunwritable] annexedfile
 		c <- readFile annexedfile
 		assertEqual "content of modified file" c (changedcontent annexedfile)
-		r <- git_annex "drop" [annexedfile]
-		not r @? "drop wrongly succeeded with no known copy of modified file"
+		not <$> git_annex "drop" [annexedfile] @? "drop wrongly succeeded with no known copy of modified file"
 
 test_fix :: Test
 test_fix = "git-annex fix" ~: intmpclonerepo $ do
@@ -405,12 +405,10 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			git_annex "get" [f] @? "get of file failed"
 			Utility.FileMode.allowWrite f
 			writeFile f (changedcontent f)
-			r <- git_annex "fsck" []
-			not r @? "fsck failed to fail with corrupted file content"
+			not <$> git_annex "fsck" [] @? "fsck failed to fail with corrupted file content"
 			git_annex "fsck" [] @? "fsck unexpectedly failed again; previous one did not fix problem with " ++ f
 		fsck_should_fail m = do
-			r <- git_annex "fsck" []
-			not r @? "fsck failed to fail with " ++ m
+			not <$> git_annex "fsck" [] @? "fsck failed to fail with " ++ m
 
 test_migrate :: Test
 test_migrate = "git-annex migrate" ~: TestList [t False, t True]
@@ -523,6 +521,10 @@ test_status :: Test
 test_status = "git-annex status" ~: intmpclonerepo $ do
 	git_annex "status" [] @? "status failed"
 
+test_version :: Test
+test_version = "git-annex version" ~: intmpclonerepo $ do
+	git_annex "version" [] @? "version failed"
+
 test_sync :: Test
 test_sync = "git-annex sync" ~: intmpclonerepo $ do
 	git_annex "sync" [] @? "sync failed"
@@ -536,7 +538,32 @@ test_map = "git-annex map" ~: intmpclonerepo $ do
 	git_annex "map" ["--fast"] @? "map failed"
 	doesFileExist "map.dot" @? "map.dot not generated"
 	c <- readFile "map.dot"
-	not ("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? "map.dot bad content"
+	("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? ("map.dot bad content: " ++ c)
+
+test_uninit :: Test
+test_uninit = "git-annex uninit" ~: intmpclonerepo $ do
+	git_annex "get" [] @? "get failed"
+	annexed_present annexedfile
+	boolSystem "git" [Params "checkout git-annex"] @? "git checkout git-annex"
+	not <$> git_annex "uninit" [] @? "uninit failed to fail when git-annex branch was checked out"
+	boolSystem "git" [Params "checkout master"] @? "git checkout master"
+	git_annex "unannex" [] @? "unannex failed"
+	checkregularfile annexedfile
+	doesDirectoryExist ".git" @? ".git vanished in uninit"
+
+test_upgrade :: Test
+test_upgrade = "git-annex upgrade" ~: intmpclonerepo $ do
+	git_annex "upgrade" [] @? "upgrade from same version failed"
+
+test_whereis :: Test
+test_whereis = "git-annex whereis" ~: intmpclonerepo $ do
+	annexed_notpresent annexedfile
+	git_annex "whereis" [annexedfile] @? "whereis on non-present file failed"
+	git_annex "untrust" ["origin"] @? "untrust failed"
+	not <$> git_annex "whereis" [annexedfile] @? "whereis on non-present file only present in untrusted repo failed to fail"
+	git_annex "get" [annexedfile] @? "get failed"
+	annexed_present annexedfile
+	git_annex "whereis" [annexedfile] @? "whereis on present file failed"
 
 test_hook_remote :: Test
 test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
@@ -558,8 +585,7 @@ test_hook_remote = "git-annex hook remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from hook remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
-	not r @? "drop failed to fail"
+	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 	where
 		dir = "dir"
@@ -579,8 +605,7 @@ test_directory_remote = "git-annex directory remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from directory remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
-	not r @? "drop failed to fail"
+	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
 test_rsync_remote :: Test
@@ -595,8 +620,7 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
 	git_annex "move" [annexedfile, "--from", "foo"] @? "move --from rsync remote failed"
 	annexed_present annexedfile
-	r <- git_annex "drop" [annexedfile, "--numcopies=2"]
-	not r @? "drop failed to fail"
+	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
 -- This is equivilant to running git-annex, but it's all run in-process

From 58a89c243cd7beeaff8fb8b45bf20d934d6882ed Mon Sep 17 00:00:00 2001
From: Ian 
Date: Tue, 20 Dec 2011 22:36:03 +0000
Subject: [PATCH 2745/8313]

---
 doc/forum/A_really_stupid_question.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/A_really_stupid_question.mdwn

diff --git a/doc/forum/A_really_stupid_question.mdwn b/doc/forum/A_really_stupid_question.mdwn
new file mode 100644
index 0000000000..38c7bcb56a
--- /dev/null
+++ b/doc/forum/A_really_stupid_question.mdwn
@@ -0,0 +1,3 @@
+Sorry, but all this wiki and the manpage seem to gloss over the most obvious question:
+
+What happens when you commit conflicting edits in different repositories?

From a2e482b59037e783729cf0d96a4d58553db0dfb4 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 20 Dec 2011 23:07:25 +0000
Subject: [PATCH 2746/8313] Added a comment: Good question!

---
 ..._40e02556de0b00b94f245a0196b5a89f._comment | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment

diff --git a/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment b/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment
new file mode 100644
index 0000000000..2a400db3b0
--- /dev/null
+++ b/doc/forum/A_really_stupid_question/comment_1_40e02556de0b00b94f245a0196b5a89f._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="Good question!"
+ date="2011-12-20T23:07:25Z"
+ content="""
+You get a regular git merge conflict, which can be resolved in any of the regular ways, except that conflicting files are just symlinks.
+
+Example:
+
+
+$ git pull
+...
+Auto-merging myfile
+CONFLICT (add/add): Merge conflict in myfile
+Automatic merge failed; fix conflicts and then commit the result.
+$ git status
+# On branch master
+# Your branch and 'origin/master' have diverged,
+# and have 1 and 1 different commit(s) each, respectively.
+#
+# Unmerged paths:
+#   (use \"git add/rm ...\" as appropriate to mark resolution)
+#
+#	both added:         myfile
+#
+no changes added to commit (use \"git add\" and/or \"git commit -a\")
+$ git add myfile
+$ git commit -m \"took local version of the conflicting file\"
+

+"""]]

From 815d1318d7a54836f65ba4b72703808959223bb2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 18:00:09 -0400
Subject: [PATCH 2747/8313] comment

---
 Utility/Url.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Utility/Url.hs b/Utility/Url.hs
index 617fe3f4d0..7ce43d9bd1 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -33,8 +33,12 @@ exists url =
 				_ -> return False
 
 {- Used to download large files, such as the contents of keys.
+ -
  - Uses wget or curl program for its progress bar. (Wget has a better one,
- - so is preferred.) -}
+ - so is preferred.) Which program to use is determined at run time; it
+ - would not be appropriate to test at configure time and build support
+ - for only one in.
+ -}
 download :: URLString -> FilePath -> IO Bool
 download url file = do
 	e <- inPath "wget"

From 8e2f74f7ab188b72b1053140f28f1f4a6a792675 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 18:14:20 -0400
Subject: [PATCH 2748/8313] update

---
 Utility/Url.hs | 4 ++++
 configure.hs   | 1 +
 2 files changed, 5 insertions(+)

diff --git a/Utility/Url.hs b/Utility/Url.hs
index 7ce43d9bd1..f215a1ebdb 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -7,6 +7,7 @@
 
 module Utility.Url (
 	exists,
+	canDownload,
 	download,
 	get
 ) where
@@ -32,6 +33,9 @@ exists url =
 				(2,_,_) -> return True
 				_ -> return False
 
+canDownload :: IO Bool
+canDownload = (||) <$> inPath "wget" <*> inPath "curl"
+
 {- Used to download large files, such as the contents of keys.
  -
  - Uses wget or curl program for its progress bar. (Wget has a better one,
diff --git a/configure.hs b/configure.hs
index 0d96b39554..bf25de5079 100644
--- a/configure.hs
+++ b/configure.hs
@@ -19,6 +19,7 @@ tests =
 	, TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
 	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"
+	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
 	] ++ shaTestCases [1, 256, 512, 224, 384]

From bb84f6e4bd57b15b9e83e2baf1b678d66d5009be Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 21:30:00 -0400
Subject: [PATCH 2749/8313] make gpg code more generic

---
 Crypto.hs | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/Crypto.hs b/Crypto.hs
index 24bb79ba0d..70ee6183be 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -71,7 +71,7 @@ genCipher c = do
 	random <- genrandom
 	encryptCipher (Cipher random) ks
 	where
-		genrandom = gpgRead
+		genrandom = gpgReadStrict
 			-- Armor the random data, to avoid newlines,
 			-- since gpg only reads ciphers up to the first
 			-- newline.
@@ -150,12 +150,12 @@ encryptKey c k = Key
 {- Runs an action, passing it a handle from which it can 
  - stream encrypted content. -}
 withEncryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withEncryptedHandle = gpgCipherHandle [Params "--symmetric --force-mdc"]
+withEncryptedHandle = gpgPassphraseHandle [Params "--symmetric --force-mdc"] . cipherPassphrase
 
 {- Runs an action, passing it a handle from which it can
  - stream decrypted content. -}
 withDecryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withDecryptedHandle = gpgCipherHandle [Param "--decrypt"]
+withDecryptedHandle = gpgPassphraseHandle [Param "--decrypt"] . cipherPassphrase
 
 {- Streams encrypted content to an action. -}
 withEncryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
@@ -180,11 +180,14 @@ gpgParams params = do
 		-- be quiet, even about checking the trustdb
 		defaults = ["--quiet", "--trust-model", "always"]
 
-gpgRead :: [CommandParam] -> IO String
-gpgRead params = do
+{- Runs gpg with some params and returns its stdout, strictly. -}
+gpgReadStrict :: [CommandParam] -> IO String
+gpgReadStrict params = do
 	params' <- gpgParams params
 	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
 
+{- Runs gpg, piping an input value to it, and returninging its stdout,
+ - strictly. -}
 gpgPipeStrict :: [CommandParam] -> String -> IO String
 gpgPipeStrict params input = do
 	params' <- gpgParams params
@@ -194,23 +197,24 @@ gpgPipeStrict params input = do
 	forceSuccess pid
 	return output
 
-{- Runs gpg with a cipher and some parameters, feeding it an input,
- - and passing a handle to its output to an action.
+{- Runs gpg with some parameters, first feeding it a passphrase via
+ - --passphrase-fd, then feeding it an input, and passing a handle
+ - to its output to an action.
  -
  - Note that to avoid deadlock with the cleanup stage,
  - the action must fully consume gpg's input before returning. -}
-gpgCipherHandle :: [CommandParam] -> Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-gpgCipherHandle params c a b = do
+gpgPassphraseHandle :: [CommandParam] -> String -> IO L.ByteString -> (Handle -> IO a) -> IO a
+gpgPassphraseHandle params passphrase a b = do
 	-- pipe the passphrase into gpg on a fd
 	(frompipe, topipe) <- createPipe
 	_ <- forkIO $ do
 		toh <- fdToHandle topipe
-		hPutStrLn toh $ cipherPassphrase c
+		hPutStrLn toh passphrase
 		hClose toh
-	let Fd passphrasefd = frompipe
-	let passphrase = [Param "--passphrase-fd", Param $ show passphrasefd]
+	let Fd pfd = frompipe
+	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
 
-	params' <- gpgParams $ passphrase ++ params
+	params' <- gpgParams $ passphrasefd ++ params
 	(pid, fromh, toh) <- hPipeBoth "gpg" params'
 	pid2 <- forkProcess $ do
 		L.hPut toh =<< a
@@ -226,7 +230,7 @@ gpgCipherHandle params c a b = do
 	return ret
 
 configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = parse <$> gpgRead params
+configKeyIds c = parse <$> gpgReadStrict params
 	where
 		params = [Params "--with-colons --list-public-keys",
 			Param $ configGet c "encryption"]

From c11cfea35555ae3bab429c283d8c7571d285d4b1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 21:47:56 -0400
Subject: [PATCH 2750/8313] split out Utility.Gpg with the generic gpg
 interface, from Crypto

---
 Crypto.hs       | 84 ++++-----------------------------------------
 Types/Crypto.hs | 11 +++---
 Utility/Gpg.hs  | 91 +++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 105 insertions(+), 81 deletions(-)
 create mode 100644 Utility/Gpg.hs

diff --git a/Crypto.hs b/Crypto.hs
index 70ee6183be..cb1ca40d14 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -30,14 +30,10 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import Data.ByteString.Lazy.UTF8 (fromString)
 import Data.Digest.Pure.SHA
-import System.Posix.Types
 import Control.Applicative
-import Control.Concurrent
-import Control.Exception (finally)
-import System.Exit
-import System.Environment
 
 import Common.Annex
+import qualified Utility.Gpg as Gpg
 import Types.Key
 import Types.Remote
 import Utility.Base64
@@ -71,7 +67,7 @@ genCipher c = do
 	random <- genrandom
 	encryptCipher (Cipher random) ks
 	where
-		genrandom = gpgReadStrict
+		genrandom = Gpg.readStrict
 			-- Armor the random data, to avoid newlines,
 			-- since gpg only reads ciphers up to the first
 			-- newline.
@@ -119,7 +115,7 @@ extractCipher c =
 encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher
 encryptCipher (Cipher c) (KeyIds ks) = do
 	let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids
-	encipher <- gpgPipeStrict (encrypt++recipients ks') c
+	encipher <- Gpg.pipeStrict (encrypt++recipients ks') c
 	return $ EncryptedCipher encipher (KeyIds ks')
 	where
 		encrypt = [ Params "--encrypt" ]
@@ -132,7 +128,7 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
 decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
 decryptCipher _ (EncryptedCipher encipher _) = 
-	Cipher <$> gpgPipeStrict decrypt encipher
+	Cipher <$> Gpg.pipeStrict decrypt encipher
 	where
 		decrypt = [ Param "--decrypt" ]
 
@@ -150,12 +146,12 @@ encryptKey c k = Key
 {- Runs an action, passing it a handle from which it can 
  - stream encrypted content. -}
 withEncryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withEncryptedHandle = gpgPassphraseHandle [Params "--symmetric --force-mdc"] . cipherPassphrase
+withEncryptedHandle = Gpg.passphraseHandle [Params "--symmetric --force-mdc"] . cipherPassphrase
 
 {- Runs an action, passing it a handle from which it can
  - stream decrypted content. -}
 withDecryptedHandle :: Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a
-withDecryptedHandle = gpgPassphraseHandle [Param "--decrypt"] . cipherPassphrase
+withDecryptedHandle = Gpg.passphraseHandle [Param "--decrypt"] . cipherPassphrase
 
 {- Streams encrypted content to an action. -}
 withEncryptedContent :: Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
@@ -169,74 +165,8 @@ pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a)
       -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 pass to c i a = to c i $ \h -> a =<< L.hGetContents h
 
-gpgParams :: [CommandParam] -> IO [String]
-gpgParams params = do
-	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
-	-- gpg output about password prompts.
-	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
-	let batch = if null e then [] else ["--batch"]
-	return $ batch ++ defaults ++ toCommand params
-	where
-		-- be quiet, even about checking the trustdb
-		defaults = ["--quiet", "--trust-model", "always"]
-
-{- Runs gpg with some params and returns its stdout, strictly. -}
-gpgReadStrict :: [CommandParam] -> IO String
-gpgReadStrict params = do
-	params' <- gpgParams params
-	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
-
-{- Runs gpg, piping an input value to it, and returninging its stdout,
- - strictly. -}
-gpgPipeStrict :: [CommandParam] -> String -> IO String
-gpgPipeStrict params input = do
-	params' <- gpgParams params
-	(pid, fromh, toh) <- hPipeBoth "gpg" params'
-	_ <- forkIO $ finally (hPutStr toh input) (hClose toh)
-	output <- hGetContentsStrict fromh
-	forceSuccess pid
-	return output
-
-{- Runs gpg with some parameters, first feeding it a passphrase via
- - --passphrase-fd, then feeding it an input, and passing a handle
- - to its output to an action.
- -
- - Note that to avoid deadlock with the cleanup stage,
- - the action must fully consume gpg's input before returning. -}
-gpgPassphraseHandle :: [CommandParam] -> String -> IO L.ByteString -> (Handle -> IO a) -> IO a
-gpgPassphraseHandle params passphrase a b = do
-	-- pipe the passphrase into gpg on a fd
-	(frompipe, topipe) <- createPipe
-	_ <- forkIO $ do
-		toh <- fdToHandle topipe
-		hPutStrLn toh passphrase
-		hClose toh
-	let Fd pfd = frompipe
-	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
-
-	params' <- gpgParams $ passphrasefd ++ params
-	(pid, fromh, toh) <- hPipeBoth "gpg" params'
-	pid2 <- forkProcess $ do
-		L.hPut toh =<< a
-		hClose toh
-		exitSuccess
-	hClose toh
-	ret <- b fromh
-
-	-- cleanup
-	forceSuccess pid
-	_ <- getProcessStatus True False pid2
-	closeFd frompipe
-	return ret
-
 configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = parse <$> gpgReadStrict params
-	where
-		params = [Params "--with-colons --list-public-keys",
-			Param $ configGet c "encryption"]
-		parse = KeyIds . map keyIdField . filter pubKey . lines
-		pubKey = isPrefixOf "pub:"
-		keyIdField s = split ":" s !! 4
+configKeyIds c = Gpg.findPubKeys $ configGet c "encryption"
 
 configGet :: RemoteConfig -> String -> String
 configGet c key = fromMaybe missing $ M.lookup key c
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index 29a4cd099c..686bf5c1a6 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -5,13 +5,16 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Types.Crypto where
+module Types.Crypto (
+	Cipher(..),
+	EncryptedCipher(..),
+	KeyIds(..),
+) where
+
+import Utility.Gpg (KeyIds(..))
 
 -- XXX ideally, this would be a locked memory region
 newtype Cipher = Cipher String
 
 data EncryptedCipher = EncryptedCipher String KeyIds
 	deriving (Ord, Eq)
-
-newtype KeyIds = KeyIds [String]
-	deriving (Ord, Eq)
diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
new file mode 100644
index 0000000000..c74c2bfd04
--- /dev/null
+++ b/Utility/Gpg.hs
@@ -0,0 +1,91 @@
+{- gpg interface
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Gpg where
+
+import qualified Data.ByteString.Lazy.Char8 as L
+import System.Posix.Types
+import Control.Applicative
+import Control.Concurrent
+import Control.Exception (finally)
+import System.Exit
+import System.Environment
+
+import Common
+
+newtype KeyIds = KeyIds [String]
+        deriving (Ord, Eq)
+
+stdParams :: [CommandParam] -> IO [String]
+stdParams params = do
+	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
+	-- gpg output about password prompts.
+	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
+	let batch = if null e then [] else ["--batch"]
+	return $ batch ++ defaults ++ toCommand params
+	where
+		-- be quiet, even about checking the trustdb
+		defaults = ["--quiet", "--trust-model", "always"]
+
+{- Runs gpg with some params and returns its stdout, strictly. -}
+readStrict :: [CommandParam] -> IO String
+readStrict params = do
+	params' <- stdParams params
+	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
+
+{- Runs gpg, piping an input value to it, and returninging its stdout,
+ - strictly. -}
+pipeStrict :: [CommandParam] -> String -> IO String
+pipeStrict params input = do
+	params' <- stdParams params
+	(pid, fromh, toh) <- hPipeBoth "gpg" params'
+	_ <- forkIO $ finally (hPutStr toh input) (hClose toh)
+	output <- hGetContentsStrict fromh
+	forceSuccess pid
+	return output
+
+{- Runs gpg with some parameters, first feeding it a passphrase via
+ - --passphrase-fd, then feeding it an input, and passing a handle
+ - to its output to an action.
+ -
+ - Note that to avoid deadlock with the cleanup stage,
+ - the action must fully consume gpg's input before returning. -}
+passphraseHandle :: [CommandParam] -> String -> IO L.ByteString -> (Handle -> IO a) -> IO a
+passphraseHandle params passphrase a b = do
+	-- pipe the passphrase into gpg on a fd
+	(frompipe, topipe) <- createPipe
+	_ <- forkIO $ do
+		toh <- fdToHandle topipe
+		hPutStrLn toh passphrase
+		hClose toh
+	let Fd pfd = frompipe
+	let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
+
+	params' <- stdParams $ passphrasefd ++ params
+	(pid, fromh, toh) <- hPipeBoth "gpg" params'
+	pid2 <- forkProcess $ do
+		L.hPut toh =<< a
+		hClose toh
+		exitSuccess
+	hClose toh
+	ret <- b fromh
+
+	-- cleanup
+	forceSuccess pid
+	_ <- getProcessStatus True False pid2
+	closeFd frompipe
+	return ret
+
+{- Finds gpg public keys matching some string. (Could be an email address,
+ - a key id, or a name. -}
+findPubKeys :: String -> IO KeyIds
+findPubKeys for = KeyIds . parse <$> readStrict params
+	where
+		params = [Params "--with-colons --list-public-keys", Param for]
+		parse = map keyIdField . filter pubKey . lines
+		pubKey = isPrefixOf "pub:"
+		keyIdField s = split ":" s !! 4

From 82a145df91ca93a55020172076297e79ff6c52e5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 20 Dec 2011 23:20:36 -0400
Subject: [PATCH 2751/8313] test encrypted special remote

This involved adding a test harness to run gpg with a dummy key, and lots
of fun.
---
 Utility/Gpg.hs   | 113 ++++++++++++++++++++++++++++++++++++++++++++---
 debian/changelog |   2 +-
 test.hs          |  29 ++++++++++++
 3 files changed, 138 insertions(+), 6 deletions(-)

diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index c74c2bfd04..f3a1ac0bb5 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -11,9 +11,9 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.Posix.Types
 import Control.Applicative
 import Control.Concurrent
-import Control.Exception (finally)
+import Control.Exception (finally, bracket)
 import System.Exit
-import System.Environment
+import System.Posix.Env (setEnv, unsetEnv, getEnv)	
 
 import Common
 
@@ -24,8 +24,8 @@ stdParams :: [CommandParam] -> IO [String]
 stdParams params = do
 	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
 	-- gpg output about password prompts.
-	e <- catchDefaultIO (getEnv "GPG_AGENT_INFO") ""
-	let batch = if null e then [] else ["--batch"]
+	e <- getEnv "GPG_AGENT_INFO"
+	let batch = if isNothing e then [] else ["--batch"]
 	return $ batch ++ defaults ++ toCommand params
 	where
 		-- be quiet, even about checking the trustdb
@@ -37,7 +37,7 @@ readStrict params = do
 	params' <- stdParams params
 	pOpen ReadFromPipe "gpg" params' hGetContentsStrict
 
-{- Runs gpg, piping an input value to it, and returninging its stdout,
+{- Runs gpg, piping an input value to it, and returning its stdout,
  - strictly. -}
 pipeStrict :: [CommandParam] -> String -> IO String
 pipeStrict params input = do
@@ -89,3 +89,106 @@ findPubKeys for = KeyIds . parse <$> readStrict params
 		parse = map keyIdField . filter pubKey . lines
 		pubKey = isPrefixOf "pub:"
 		keyIdField s = split ":" s !! 4
+
+
+
+{- A test key. This is provided pre-generated since generating a new gpg
+ - key is too much work (requires too much entropy) for a test suite to
+ - do.
+ -
+ - This key was generated with no exipiration date, and a small keysize. 
+ - It has an empty passphrase. -}
+testKeyId :: String
+testKeyId = "129D6E0AC537B9C7"
+testKey :: String
+testKey = keyBlock True
+	[ "mI0ETvFAZgEEAKnqwWgZqznMhi1RQExem2H8t3OyKDxaNN3rBN8T6LWGGqAYV4wT"
+	, "r8In5tfsnz64bKpE1Qi68JURFwYmthgUL9N48tbODU8t3xzijdjLOSaTyqkH1ik6"
+	, "EyulfKN63xLne9i4F9XqNwpiZzukXYbNfHkDA2yb0M6g4UFKLY/fNzGXABEBAAG0"
+	, "W2luc2VjdXJlIHRlc3Qga2V5ICh0aGlzIGlzIGEgdGVzdCBrZXksIGRvIG5vdCB1"
+	, "c2UgZm9yIGFjdHVhbCBlbmNyeXB0aW9uKSA8dGVzdEBleGFtcGxlLmNvbT6IuAQT"
+	, "AQgAIgUCTvFAZgIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQEp1uCsU3"
+	, "uceQ9wP/YMd1f0+/eLLcwGXNBvGqyVhUOfAKknO1bMzGbqTsq9g60qegy/cldqee"
+	, "xVxNfy0VN//JeMfgdcb8+RgJYLoaMrTy9CcsUcFPxtwN9tcLmsM0V2/fNmmFBO9t"
+	, "v75iH+zeFbNg0/FbPkHiN6Mjw7P2gXYKQXgTvQZBWaphk8oQlBm4jQRO8UBmAQQA"
+	, "vdi50M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1Q"
+	, "Y+gLOH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ"
+	, "8B5zeKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAYifBBgBCAAJBQJO"
+	, "8UBmAhsMAAoJEBKdbgrFN7nHclAEAKBShuP/toH03atDUQTbGE34CA4yEC9BVghi"
+	, "7kviOZlOz2s8xAfp/8AYsrECx1kgbXcA7JD902eNyp7NzXsdJX0zJwHqiuZW0XlD"
+	, "T8ZJu4qrYRYgl/790WPESZ+ValvHD/fqkR38RF4tfxvyoMhhp0roGmJY33GASIG/"
+	, "+gQkDF9/"
+	, "=1k11"
+	]
+testSecretKey :: String
+testSecretKey = keyBlock False
+	[ "lQHYBE7xQGYBBACp6sFoGas5zIYtUUBMXpth/Ldzsig8WjTd6wTfE+i1hhqgGFeM"
+	, "E6/CJ+bX7J8+uGyqRNUIuvCVERcGJrYYFC/TePLWzg1PLd8c4o3Yyzkmk8qpB9Yp"
+	, "OhMrpXyjet8S53vYuBfV6jcKYmc7pF2GzXx5AwNsm9DOoOFBSi2P3zcxlwARAQAB"
+	, "AAP+PlRboxy7Z0XjuG70N6+CrzSddQbW5KCwgPFrxYsPk7sAPFcBkmRMVlv9vZpS"
+	, "phbP4bvDK+MrSntM51g+9uE802yhPhSWdmEbImiWfV2ucEhlLjD8gw7JDex9XZ0a"
+	, "EbTOV56wOsILuedX/jF/6i6IQzy5YmuMeo+ip1XQIsIN+80CAMyXepOBJgHw/gBD"
+	, "VdXh/l//vUkQQlhInQYwgkKbr0POCTdr8DM1qdKLcUD9Q1khgNRp0vZGGz+5xsrc"
+	, "KaODUlMCANSczLJcYWa8yPqB3S14yTe7qmtDiOS362+SeVUwQA7eQ06PcHLPsN+p"
+	, "NtWoHRfYazxrs+g0JvmoQOYdj4xSQy0CAMq7H/l6aeG1n8tpyMxqE7OvBOsvzdu5"
+	, "XS7I1AnwllVFgvTadVvqgf7b+hdYd91doeHDUGqSYO78UG1GgaBHJdylqrRbaW5z"
+	, "ZWN1cmUgdGVzdCBrZXkgKHRoaXMgaXMgYSB0ZXN0IGtleSwgZG8gbm90IHVzZSBm"
+	, "b3IgYWN0dWFsIGVuY3J5cHRpb24pIDx0ZXN0QGV4YW1wbGUuY29tPoi4BBMBCAAi"
+	, "BQJO8UBmAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRASnW4KxTe5x5D3"
+	, "A/9gx3V/T794stzAZc0G8arJWFQ58AqSc7VszMZupOyr2DrSp6DL9yV2p57FXE1/"
+	, "LRU3/8l4x+B1xvz5GAlguhoytPL0JyxRwU/G3A321wuawzRXb982aYUE722/vmIf"
+	, "7N4Vs2DT8Vs+QeI3oyPDs/aBdgpBeBO9BkFZqmGTyhCUGZ0B2ARO8UBmAQQAvdi5"
+	, "0M/WRCkOLt2RsUve8V8brMWYTJBJTTWoHUeRr82v4NCdX7OE1BsoVK8cy/1QY+gL"
+	, "OH9PqinuGGNWRmPV2Ju/RYn5H7sdewXA8E80xWhc4phHRMJ8Jjhg/GVPamkJ8B5z"
+	, "eKF0jcLFl7cuVdOyQakhoeDWJd0CyfW837nmPtMAEQEAAQAD/RaVtFFTkF1udun7"
+	, "YOwzJvQXCO9OWHZvSdEeG4BUNdAwy4YWu0oZzKkBDBS6+lWILqqb/c28U4leUJ1l"
+	, "H+viz5svN9BWWyj/UpI00uwUo9JaIqalemwfLx6vsh69b54L1B4exLZHYGLvy/B3"
+	, "5T6bT0gpOE+53BRtKcJaOh/McQeJAgDTOCBU5weWOf6Bhqnw3Vr/gRfxntAz2okN"
+	, "gqz/h79mWbCc/lHKoYQSsrCdMiwziHSjXwvehUrdWE/AcomtW0vbAgDmGJqJ2fNr"
+	, "HvdsGx4Ld/BxyiZbCURJLUQ5CwzfHGIvBu9PMT8zM26NOSncaXRjxDna2Ggh8Uum"
+	, "ANEwbnhxFwZpAf9L9RLYIMTtAqwBjfXJg/lHcc2R+VP0hL5c8zFz+S+w7bRqINwL"
+	, "ff1JstKuHT2nJnu0ustK66by8YI3T0hDFFahnNCInwQYAQgACQUCTvFAZgIbDAAK"
+	, "CRASnW4KxTe5x3JQBACgUobj/7aB9N2rQ1EE2xhN+AgOMhAvQVYIYu5L4jmZTs9r"
+	, "PMQH6f/AGLKxAsdZIG13AOyQ/dNnjcqezc17HSV9MycB6ormVtF5Q0/GSbuKq2EW"
+	, "IJf+/dFjxEmflWpbxw/36pEd/EReLX8b8qDIYadK6BpiWN9xgEiBv/oEJAxffw=="
+	, "=LDsg"
+	]
+keyBlock :: Bool -> [String] -> String
+keyBlock public ls = unlines
+	[ "-----BEGIN PGP "++t++" KEY BLOCK-----"
+	, "Version: GnuPG v1.4.11 (GNU/Linux)"
+	, ""
+	, unlines ls
+	, "-----END PGP "++t++" KEY BLOCK-----"
+	]
+	where
+		t
+			| public = "PUBLIC"
+			| otherwise = "PRIVATE"
+
+{- Runs an action using gpg in a test harness, in which gpg does
+ - not use ~/.gpg/, but a directory with the test key set up to be used. -}
+testHarness :: IO a -> IO a
+testHarness a = do
+	orig <- getEnv var
+	bracket setup (cleanup orig) (const a)
+	where
+		var = "GNUPGHOME"		
+
+		setup = do
+			base <- getTemporaryDirectory
+			dir <- mktmpdir $ base  "gpgtmpXXXXXX"
+			setEnv var dir True
+			_ <- pipeStrict [Params "--import -q"] $ unlines
+				[testSecretKey, testKey]
+			return dir
+		
+		cleanup orig tmpdir = removeDirectoryRecursive tmpdir >> reset orig
+                reset (Just v) = setEnv var v True
+                reset _ = unsetEnv var
+
+{- Tests the test harness. -}
+testTestHarness :: IO Bool
+testTestHarness = do
+	keys <- testHarness $ findPubKeys testKeyId
+	return $ KeyIds [testKeyId] == keys
diff --git a/debian/changelog b/debian/changelog
index e3e8079dcf..e6fa9c16f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 68%
+  * Test suite improvements. Current top-level test coverage: 70%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index 9ba0e5dd91..7c47443880 100644
--- a/test.hs
+++ b/test.hs
@@ -42,6 +42,8 @@ import qualified Config
 import qualified Crypto
 import qualified Utility.Path
 import qualified Utility.FileMode
+import qualified Utility.Gpg
+import qualified Build.SysConfig
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -116,6 +118,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
+	, test_crypto
 	]
 
 test_init :: Test
@@ -623,6 +626,32 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
+test_crypto :: Test
+test_crypto = "git-annex crypto" ~: intmpclonerepo $
+	-- gpg is not a build dependency, so only test when it's available
+	when Build.SysConfig.gpg $ do
+		Utility.Gpg.testTestHarness @? "test harness self-test failed"
+		Utility.Gpg.testHarness $ do
+			createDirectory "dir"
+			let initremote = git_annex "initremote"
+				[ "foo"
+				, "type=directory"
+				, "encryption=" ++ Utility.Gpg.testKeyId
+				, "directory=dir"
+				]
+			initremote @? "initremote failed"
+			initremote @? "initremote failed when run twice in a row"
+			git_annex "get" [annexedfile] @? "get of file failed"
+			annexed_present annexedfile
+			git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"
+			annexed_present annexedfile
+			git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
+			annexed_notpresent annexedfile
+			git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed"
+			annexed_present annexedfile
+			not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
+			annexed_present annexedfile	
+
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
 git_annex :: String -> [String] -> IO Bool

From a99af6338e9487d2212d4e82aa1afa4873cbfca8 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
 
Date: Wed, 21 Dec 2011 16:06:26 +0000
Subject: [PATCH 2752/8313] Added a comment

---
 ...ent_3_1aa89725b5196e40a16edeeb5ccfa371._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment b/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment
new file mode 100644
index 0000000000..0ead32dad2
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_3_1aa89725b5196e40a16edeeb5ccfa371._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnXybLxkPMYpP3yw4b_I6IdC3cKTD-xEdU"
+ nickname="Matt"
+ subject="comment 3"
+ date="2011-12-21T16:06:25Z"
+ content="""
+hmmmm - I'm still not sure I get this.
+
+If I'm using a whole bunch of distributed annexs with no central repo, then I can not do a `git pull remote` without either specifying the branch to use or changing default tracked remote via `git branch --set-upstream`.  The former like you note doesn't pull the git-annex branch down the latter only works one-at-a-time.
+
+The docs read to me as though I ought to be able to do a `git pull remote ; git annex get .` using anyone of my distributed annexs.
+
+Am I doing something wrong?  Or is the above correct?
+"""]]

From c61f3d7b7b61583a61a97b2a7f3adbf4233cd93e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 02:32:40 -0400
Subject: [PATCH 2753/8313] test coverage improvements

---
 debian/changelog |  2 +-
 test.hs          | 47 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index e6fa9c16f7..33a9cd575f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 70%
+  * Test suite improvements. Current top-level test coverage: 72%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index 7c47443880..eaa3fb8b64 100644
--- a/test.hs
+++ b/test.hs
@@ -17,6 +17,7 @@ import System.Posix.Env
 import qualified Control.Exception.Extensible as E
 import qualified Data.Map as M
 import System.IO.HVFS (SystemFS(..))
+import Text.JSON
 
 import Common
 
@@ -512,9 +513,16 @@ test_describe = "git-annex describe" ~: intmpclonerepo $ do
 test_find :: Test
 test_find = "git-annex find" ~: intmpclonerepo $ do
 	annexed_notpresent annexedfile
-	git_annex "find" [] @? "find failed"
-	git_annex "get" [] @? "get failed"
-	git_annex "find" [] @? "find failed"
+	git_annex_expectoutput "find" [] []
+	git_annex "get" [annexedfile] @? "get failed"
+	annexed_present annexedfile
+	annexed_notpresent sha1annexedfile
+	git_annex_expectoutput "find" [] [annexedfile]
+	git_annex_expectoutput "find" ["--exclude", annexedfile, "--and", "--exclude", sha1annexedfile] []
+	git_annex_expectoutput "find" ["--not", "--in", "origin"] []
+	git_annex_expectoutput "find" ["--copies", "1", "--and", "--not", "--copies", "2"] [sha1annexedfile]
+	git_annex_expectoutput "find" ["--inbackend", "SHA1"] [sha1annexedfile]
+	git_annex_expectoutput "find" ["--inbackend", "WORM"] []
 
 test_merge :: Test
 test_merge = "git-annex merge" ~: intmpclonerepo $ do
@@ -522,7 +530,10 @@ test_merge = "git-annex merge" ~: intmpclonerepo $ do
 
 test_status :: Test
 test_status = "git-annex status" ~: intmpclonerepo $ do
-	git_annex "status" [] @? "status failed"
+	json <- git_annex_output "status" ["--json"]
+	case Text.JSON.decodeStrict json :: Text.JSON.Result (JSObject JSValue) of
+		Ok _ -> return ()
+		Error e -> assertFailure e
 
 test_version :: Test
 test_version = "git-annex version" ~: intmpclonerepo $ do
@@ -550,9 +561,10 @@ test_uninit = "git-annex uninit" ~: intmpclonerepo $ do
 	boolSystem "git" [Params "checkout git-annex"] @? "git checkout git-annex"
 	not <$> git_annex "uninit" [] @? "uninit failed to fail when git-annex branch was checked out"
 	boolSystem "git" [Params "checkout master"] @? "git checkout master"
-	git_annex "unannex" [] @? "unannex failed"
+	_ <- git_annex "uninit" [] -- exit status not checked; does abnormal exit
 	checkregularfile annexedfile
 	doesDirectoryExist ".git" @? ".git vanished in uninit"
+	not <$> doesDirectoryExist ".git/annex" @? ".git/annex still present after uninit"
 
 test_upgrade :: Test
 test_upgrade = "git-annex upgrade" ~: intmpclonerepo $ do
@@ -664,6 +676,31 @@ git_annex command params = do
 	where
 		run = GitAnnex.run (command:"-q":params)
 
+{- Runs git-annex and returns its output. -}
+git_annex_output :: String -> [String] -> IO String
+git_annex_output command params = do
+	(frompipe, topipe) <- createPipe
+	pid <- forkProcess $ do
+		_ <- dupTo topipe stdOutput
+		closeFd frompipe
+		_ <- git_annex command params
+		exitSuccess
+	-- XXX since the above is a separate process, code coverage stats are
+	-- not gathered for things run in it.
+	closeFd topipe
+	fromh <- fdToHandle frompipe
+	got <- hGetContentsStrict fromh
+	hClose fromh
+	_ <- getProcessStatus True False pid
+	-- XXX hack Run same command again, to get code coverage.
+	_ <- git_annex command params
+	return got
+
+git_annex_expectoutput :: String -> [String] -> [String] -> IO ()
+git_annex_expectoutput command params expected = do
+	got <- lines <$> git_annex_output command params
+	got == expected @? ("unexpected value running " ++ command ++ " " ++ show params ++ " -- got: " ++ show got ++ " expected: " ++ show expected)
+
 -- Runs an action in the current annex. Note that shutdown actions
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a

From 8cdcd78b210756f184188a583c263569d87f08b4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 13:50:33 -0400
Subject: [PATCH 2754/8313] test bup special remote (74% coverage)

---
 debian/changelog |  2 +-
 test.hs          | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 33a9cd575f..59c6392fe7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 72%
+  * Test suite improvements. Current top-level test coverage: 74%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index eaa3fb8b64..0cd820a2b9 100644
--- a/test.hs
+++ b/test.hs
@@ -119,6 +119,7 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_hook_remote
 	, test_directory_remote
 	, test_rsync_remote
+	, test_bup_remote
 	, test_crypto
 	]
 
@@ -638,6 +639,24 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
 	annexed_present annexedfile
 
+test_bup_remote :: Test
+test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ checkbup $ do
+	dir <- absPath "dir" -- bup special remote needs an absolute path
+	createDirectory dir
+	git_annex "initremote" (words $ "foo type=bup encryption=none buprepo="++dir) @? "initremote failed"
+	git_annex "get" [annexedfile] @? "get of file failed"
+	annexed_present annexedfile
+	git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to bup remote failed"
+	annexed_present annexedfile
+	git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
+	annexed_notpresent annexedfile
+	git_annex "copy" [annexedfile, "--from", "foo"] @? "copy --from bup remote failed"
+	annexed_present annexedfile
+	not <$> git_annex "move" [annexedfile, "--from", "foo"] @? "move --from bup remote failed to fail"
+	annexed_present annexedfile
+	where
+		checkbup = whenM (inPath "bup")
+
 test_crypto :: Test
 test_crypto = "git-annex crypto" ~: intmpclonerepo $
 	-- gpg is not a build dependency, so only test when it's available

From 57235315c4dcab9d2271adcca94aba610b5ed0c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 14:10:36 -0400
Subject: [PATCH 2755/8313] improve

---
 test.hs | 74 +++++++++++++++++++++++++++++++++------------------------
 1 file changed, 43 insertions(+), 31 deletions(-)

diff --git a/test.hs b/test.hs
index 0cd820a2b9..1f595eaf23 100644
--- a/test.hs
+++ b/test.hs
@@ -141,6 +141,12 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			writeFile sha1annexedfile $ content sha1annexedfile
 			git_annex "add" [sha1annexedfile, "--backend=SHA1"] @? "add with SHA1 failed"
 			annexed_present sha1annexedfile
+			checkbackend sha1annexedfile backendSHA1
+			writeFile wormannexedfile $ content wormannexedfile
+			git_annex "add" [wormannexedfile, "--backend=WORM"] @? "add with WORM failed"
+			annexed_present wormannexedfile
+			checkbackend wormannexedfile backendWORM
+			boolSystem "git" [Params "rm --force -q", File wormannexedfile] @? "git rm failed"
 			writeFile ingitfile $ content ingitfile
 			boolSystem "git" [Param "add", File ingitfile] @? "git add failed"
 			boolSystem "git" [Params "commit -q -a -m commit"] @? "git commit failed"
@@ -455,10 +461,6 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 		checkbackend sha1annexedfile backendSHA256
 		
 		where
-			checkbackend file expected = do
-				r <- annexeval $ Backend.lookupFile file
-				let b = snd $ fromJust r
-				assertEqual ("backend for " ++ file) expected b
 
 test_unused :: Test
 test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
@@ -640,7 +642,7 @@ test_rsync_remote = "git-annex rsync remote" ~: intmpclonerepo $ do
 	annexed_present annexedfile
 
 test_bup_remote :: Test
-test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ checkbup $ do
+test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ when Build.SysConfig.bup $ do
 	dir <- absPath "dir" -- bup special remote needs an absolute path
 	createDirectory dir
 	git_annex "initremote" (words $ "foo type=bup encryption=none buprepo="++dir) @? "initremote failed"
@@ -654,34 +656,31 @@ test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ checkbup $ do
 	annexed_present annexedfile
 	not <$> git_annex "move" [annexedfile, "--from", "foo"] @? "move --from bup remote failed to fail"
 	annexed_present annexedfile
-	where
-		checkbup = whenM (inPath "bup")
 
+-- gpg is not a build dependency, so only test when it's available
 test_crypto :: Test
-test_crypto = "git-annex crypto" ~: intmpclonerepo $
-	-- gpg is not a build dependency, so only test when it's available
-	when Build.SysConfig.gpg $ do
-		Utility.Gpg.testTestHarness @? "test harness self-test failed"
-		Utility.Gpg.testHarness $ do
-			createDirectory "dir"
-			let initremote = git_annex "initremote"
-				[ "foo"
-				, "type=directory"
-				, "encryption=" ++ Utility.Gpg.testKeyId
-				, "directory=dir"
-				]
-			initremote @? "initremote failed"
-			initremote @? "initremote failed when run twice in a row"
-			git_annex "get" [annexedfile] @? "get of file failed"
-			annexed_present annexedfile
-			git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"
-			annexed_present annexedfile
-			git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
-			annexed_notpresent annexedfile
-			git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed"
-			annexed_present annexedfile
-			not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
-			annexed_present annexedfile	
+test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $ do
+	Utility.Gpg.testTestHarness @? "test harness self-test failed"
+	Utility.Gpg.testHarness $ do
+		createDirectory "dir"
+		let initremote = git_annex "initremote"
+			[ "foo"
+			, "type=directory"
+			, "encryption=" ++ Utility.Gpg.testKeyId
+			, "directory=dir"
+			]
+		initremote @? "initremote failed"
+		initremote @? "initremote failed when run twice in a row"
+		git_annex "get" [annexedfile] @? "get of file failed"
+		annexed_present annexedfile
+		git_annex "copy" [annexedfile, "--to", "foo"] @? "copy --to encrypted remote failed"
+		annexed_present annexedfile
+		git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed"
+		annexed_notpresent annexedfile
+		git_annex "move" [annexedfile, "--from", "foo"] @? "move --from encrypted remote failed"
+		annexed_present annexedfile
+		not <$> git_annex "drop" [annexedfile, "--numcopies=2"] @? "drop failed to fail"
+		annexed_present annexedfile	
 
 -- This is equivilant to running git-annex, but it's all run in-process
 -- so test coverage collection works.
@@ -844,6 +843,12 @@ checklocationlog f expected = do
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"
 
+checkbackend :: FilePath -> Types.Backend Types.Annex -> Assertion			
+checkbackend file expected = do
+	r <- annexeval $ Backend.lookupFile file
+	let b = snd $ fromJust r
+	assertEqual ("backend for " ++ file) expected b
+
 inlocationlog :: FilePath -> Assertion
 inlocationlog f = checklocationlog f True
 
@@ -897,6 +902,9 @@ tmprepodir = tmpdir ++ "/tmprepo"
 annexedfile :: String
 annexedfile = "foo"
 
+wormannexedfile :: String
+wormannexedfile = "apple"
+
 sha1annexedfile :: String
 sha1annexedfile = "sha1foo"
 
@@ -912,6 +920,7 @@ content f
 	| f == ingitfile = "normal file content"
 	| f == sha1annexedfile ="sha1 annexed file content"
 	| f == sha1annexedfiledup = content sha1annexedfile
+	| f == wormannexedfile = "worm annexed file content"
 	| otherwise = "unknown file " ++ f
 
 changecontent :: FilePath -> IO ()
@@ -926,5 +935,8 @@ backendSHA1 = backend_ "SHA1"
 backendSHA256 :: Types.Backend Types.Annex
 backendSHA256 = backend_ "SHA256"
 
+backendWORM :: Types.Backend Types.Annex
+backendWORM = backend_ "WORM"
+
 backend_ :: String -> Types.Backend Types.Annex
 backend_ name = Backend.lookupBackendName name

From a76b13b848612595a254025f3eb415c2395a92de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 14:20:41 -0400
Subject: [PATCH 2756/8313] test fsck in bare repos (75%)

---
 debian/changelog |  2 +-
 test.hs          | 20 +++++++++++++-------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 59c6392fe7..62c7d40f23 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,7 +5,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Properly handle multiline git config values.
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
-  * Test suite improvements. Current top-level test coverage: 74%
+  * Test suite improvements. Current top-level test coverage: 75%
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/test.hs b/test.hs
index 1f595eaf23..645da588e8 100644
--- a/test.hs
+++ b/test.hs
@@ -388,7 +388,7 @@ test_trust = "git-annex trust/untrust/semitrust/dead" ~: intmpclonerepo $ do
 			assertBool msg present
 
 test_fsck :: Test
-test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withremoteuntrusted]
+test_fsck = "git-annex fsck" ~: TestList [basicfsck, barefsck, withlocaluntrusted, withremoteuntrusted]
 	where
 		basicfsck = TestCase $ intmpclonerepo $ do
 			git_annex "fsck" [] @? "fsck failed"
@@ -397,6 +397,8 @@ test_fsck = "git-annex fsck" ~: TestList [basicfsck, withlocaluntrusted, withrem
 			boolSystem "git" [Params "config annex.numcopies 1"] @? "git config failed"
 			corrupt annexedfile
 			corrupt sha1annexedfile
+		barefsck = TestCase $ intmpbareclonerepo $ do
+			git_annex "fsck" [] @? "fsck failed"
 		withlocaluntrusted = TestCase $ intmpclonerepo $ do
 			git_annex "get" [annexedfile] @? "get failed"
 			git_annex "untrust" ["origin"] @? "untrust of origin repo failed"
@@ -735,10 +737,13 @@ inmainrepo :: Assertion -> Assertion
 inmainrepo a = indir mainrepodir a
 
 intmpclonerepo :: Assertion -> Assertion
-intmpclonerepo a = withtmpclonerepo $ \r -> indir r a
+intmpclonerepo a = withtmpclonerepo False $ \r -> indir r a
 
-withtmpclonerepo :: (FilePath -> Assertion) -> Assertion
-withtmpclonerepo = bracket (clonerepo mainrepodir tmprepodir) cleanup
+intmpbareclonerepo :: Assertion -> Assertion
+intmpbareclonerepo a = withtmpclonerepo True $ \r -> indir r a
+
+withtmpclonerepo :: Bool -> (FilePath -> Assertion) -> Assertion
+withtmpclonerepo bare = bracket (clonerepo mainrepodir tmprepodir bare) cleanup
 
 withgitrepo :: (FilePath -> Assertion) -> Assertion
 withgitrepo = bracket (setuprepo mainrepodir) return
@@ -766,11 +771,12 @@ setuprepo dir = do
 	return dir
 
 -- clones are always done as local clones; we cannot test ssh clones
-clonerepo :: FilePath -> FilePath -> IO FilePath
-clonerepo old new = do
+clonerepo :: FilePath -> FilePath -> Bool -> IO FilePath
+clonerepo old new bare = do
 	cleanup new
 	ensuretmpdir
-	boolSystem "git" [Params "clone -q", File old, File new] @? "git clone failed"
+	let b = if bare then " --bare" else ""
+	boolSystem "git" [Params ("clone -q" ++ b), File old, File new] @? "git clone failed"
 	indir new $ git_annex "init" ["-q", new] @? "git annex init failed"
 	return new
 	

From 20482712d0d909f60707db7cde634e460e17058b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Dec 2011 16:56:48 -0400
Subject: [PATCH 2757/8313] Improve deletion of files from rsync special
 remotes. Closes: #652849

Rsync is only run once, with include / exclude rules used to specify
exactly what to delete. This is faster, and avoids ugly error messages
from rsync, and doesn't fail if the content already got deleted somehow.
---
 Remote/Rsync.hs  | 39 +++++++++++++++++++++++----------------
 debian/changelog |  1 +
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index c281420773..68566c52a5 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -92,11 +92,6 @@ rsyncUrls o k = map use annexHashes
 		use h = rsyncUrl o  h k  rsyncEscape o (f  f)
                 f = keyFile k
 
-rsyncUrlDirs :: RsyncOpts -> Key -> [String]
-rsyncUrlDirs o k = map use annexHashes
-	where
-		use h = rsyncUrl o  h k  rsyncEscape o (keyFile k)
-
 store :: RsyncOpts -> Key -> Annex Bool
 store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
 
@@ -125,17 +120,29 @@ retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
 		else return res
 
 remove :: RsyncOpts -> Key -> Annex Bool
-remove o k = untilTrue (rsyncUrlDirs o k) $ \d ->
-	withRsyncScratchDir $ \tmp -> liftIO $ do
-		{- Send an empty directory to rysnc as the
-		 - parent directory of the file to remove. -}
-		let dummy = tmp  keyFile k
-		createDirectoryIfMissing True dummy
-		rsync $ rsyncOptions o ++
-			[ Params "--quiet --delete --recursive"
-			, partialParams
-			, Param $ addTrailingPathSeparator dummy
-			, Param d
+remove o k = withRsyncScratchDir $ \tmp -> liftIO $ do
+	{- Send an empty directory to rysnc to make it delete. -}
+	let dummy = tmp  keyFile k
+	createDirectoryIfMissing True dummy
+	rsync $ rsyncOptions o ++
+		map (\s -> Param $ "--include=" ++ s) includes ++
+		[ Param "--exclude=*" -- exclude everything else
+		, Params "--quiet --delete --recursive"
+		, partialParams
+		, Param $ addTrailingPathSeparator dummy
+		, Param $ rsyncUrl o
+		]
+	where
+		{- Specify include rules to match the directories where the
+		 - content could be. Note that the parent directories have
+		 - to also be explicitly included, due to how rsync
+		 - traverses directories. -}
+		includes = concatMap use annexHashes
+		use h = let dir = h k in
+			[ parentDir dir
+			, dir
+			-- match content directory and anything in it
+			, dir  keyFile k  "***"
 			]
 
 checkPresent :: Git.Repo -> RsyncOpts -> Key -> Annex (Either String Bool)
diff --git a/debian/changelog b/debian/changelog
index 62c7d40f23..564e144038 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Fix the hook special remote, which bitrotted a while ago.
   * map: --fast disables use of dot to display map
   * Test suite improvements. Current top-level test coverage: 75%
+  * Improve deletion of files from rsync special remotes. Closes: #652849
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From 97bef4af733ac8c42adfe6809ba6ae7269530473 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 12:31:36 +0000
Subject: [PATCH 2758/8313] Added a comment: List the duplicate filenames, then
 let the user decide what to do

---
 ..._f120d1e83c1a447f2ecce302fc69cf74._comment | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment
new file mode 100644
index 0000000000..a218ee3d51
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_4_f120d1e83c1a447f2ecce302fc69cf74._comment
@@ -0,0 +1,35 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="List the duplicate filenames, then let the user decide what to do"
+ date="2011-12-22T12:31:29Z"
+ content="""
+I have the same use case as Asheesh but I want to be able to see which filenames point to the same objects and then decide which of the duplicates to drop myself.  I think
+
+    git annex drop --by-contents
+
+would be the wrong approach because how does git-annex know which ones to drop?  There's too much potential for error.
+
+Instead it would be great to have something like
+
+    git annex finddups
+
+While it's easy enough to knock up a bit of shell or Perl to achieve this, that relies on knowledge of the annex symlink structure, so I think really it belongs inside git-annex.
+
+If this command gave output similar to the excellent `fastdup` utility:
+
+    Scanning for files... 672 files in 10.439 seconds
+    Comparing 2 sets of files...
+
+    2 files (70.71 MB/ea)
+            /home/adam/media/flat/tour/flat-tour.3gp
+            /home/adam/videos/tour.3gp
+
+    Found 1 duplicate of 1 file (70.71 MB wasted)
+    Scanned 672 files (1.96 GB) in 11.415 seconds
+
+then you could do stuff like
+
+    git annex finddups | grep /home/adam/media/flat | xargs rm
+
+"""]]

From 6808b08c1ab096eb8c7c986379ec7a24d70434e3 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 15:43:52 +0000
Subject: [PATCH 2759/8313] Added a comment: Here's a Perl version

---
 ...comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment
new file mode 100644
index 0000000000..e48a4a9b38
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_5_5c30294b3c59fdebb1eef0ae5da4cd4f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Here's a Perl version"
+ date="2011-12-22T15:43:51Z"
+ content="""
+https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups
+
+but it would be better in git-annex itself ...
+"""]]

From 30cf6ce81ca8ff99f5284c5b991e546eb1da72ae Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 22 Dec 2011 16:39:24 +0000
Subject: [PATCH 2760/8313] Added a comment

---
 ...ent_6_f24541ada1c86d755acba7e9fa7cff24._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
new file mode 100644
index 0000000000..93d3d41f43
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2011-12-22T16:39:24Z"
+ content="""
+My main concern with putting this in git-annex is that finding duplicates necessarily involves storing a list of every key and file in the repository, and git-annex is very carefully built to avoid things that require non-constant memory use, so that it can scale to very big repositories. (The only exception is the `unused` command, and reducing its memory usage is a continuing goal.)
+
+So I would rather come at this from a different angle.. like providing a way to output a list of files and their associated keys, which the user can then use in their own shell pipelines to find duplicate keys:
+
+      git annex find --include '*' --format=\"%f %k\n\" | sort foo --key 2 | uniq --all-repeated --skip-fields=1
+
+(Making that properly handle filenames with spaces is left as an exercise for the reader..)
+"""]]

From 6bffe509d7f1ec60168522585925a43dbfffbd36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 13:53:06 -0400
Subject: [PATCH 2761/8313] Add --include, which is the same as --not
 --exclude.

---
 Git/Index.hs       |  2 --
 GitAnnex.hs        |  2 ++
 Limit.hs           | 10 ++++++++--
 debian/changelog   |  1 +
 doc/git-annex.mdwn | 22 ++++++++++++++++------
 test.hs            |  1 +
 6 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/Git/Index.hs b/Git/Index.hs
index e58f1edd67..aaf54e032e 100644
--- a/Git/Index.hs
+++ b/Git/Index.hs
@@ -9,8 +9,6 @@ module Git.Index where
 
 import System.Posix.Env (setEnv, unsetEnv, getEnv)
 
-import Common
-
 {- Forces git to use the specified index file.
  -
  - Returns an action that will reset back to the default
diff --git a/GitAnnex.hs b/GitAnnex.hs
index a5b9609b6d..40ebed0d69 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -112,6 +112,8 @@ options = commonOptions ++
 		"terminate filename with null"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
+	, Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob)
+		"don't skip files matching the glob pattern"
 	, Option ['i'] ["in"] (ReqArg Limit.addIn paramRemote)
 		"skip files not present in a remote"
 	, Option ['C'] ["copies"] (ReqArg Limit.addCopies paramNumber)
diff --git a/Limit.hs b/Limit.hs
index c68c3bdd8b..26e5d689c9 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -55,10 +55,16 @@ addLimit :: (FilePath -> Annex Bool) -> Annex ()
 addLimit = add . Utility.Matcher.Operation
 
 {- Add a limit to skip files that do not match the glob. -}
+addInclude :: String -> Annex ()
+addInclude glob = addLimit $ return . matchglob glob
+
+{- Add a limit to skip files that match the glob. -}
 addExclude :: String -> Annex ()
-addExclude glob = addLimit $ return . notExcluded
+addExclude glob = addLimit $ return . not . matchglob glob
+
+matchglob :: String -> FilePath -> Bool
+matchglob glob f = isJust $ match cregex f []
 	where
-		notExcluded f = isNothing $ match cregex f []
 		cregex = compile regex []
 		regex = '^':wildToRegex glob
 
diff --git a/debian/changelog b/debian/changelog
index 564e144038..13bca33260 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,7 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * map: --fast disables use of dot to display map
   * Test suite improvements. Current top-level test coverage: 75%
   * Improve deletion of files from rsync special remotes. Closes: #652849
+  * Add --include, which is the same as --not --exclude.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index b3d671bb89..f4eef5c4c7 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -241,12 +241,13 @@ subdirectories).
 
 * find [path ...]
 
-  Outputs a list of annexed files whose content is currently present.
-  Or, if a file matching option is specified, outputs a list of all
-  matching files, whether or not their content is currently present.
+  Outputs a list of annexed files in the specified path. With no path,
+  finds files in the current directory and its subdirectories.
 
-  With no parameters, defaults to finding all files in the current directory
-  and its subdirectories.
+  By default, only lists annexed files whose content is currently present.
+  This can be changed by specifying file matching options. To list all
+  annexed files, present or not, specify --include "*". To list all
+  annexed files whose content is not present, specify --not --in "."
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0.
@@ -447,7 +448,16 @@ file contents are present at either of two repositories.
 * --exclude=glob
 
   Skips files matching the glob pattern. The glob is matched relative to
-  the current directory. For example: --exclude='*.mp3' --exclude='subdir/*'
+  the current directory. For example:
+
+	--exclude='*.mp3' --exclude='subdir/*'
+
+* --include=glob
+
+  Skips files not matching the glob pattern.  (Same as --not --exclude.)
+  For example, to include only mp3 and ogg files:
+
+	--include='*.mp3' --or --include='*.ogg'
 
 * --in=repository
 
diff --git a/test.hs b/test.hs
index 645da588e8..b4823fda8d 100644
--- a/test.hs
+++ b/test.hs
@@ -524,6 +524,7 @@ test_find = "git-annex find" ~: intmpclonerepo $ do
 	annexed_notpresent sha1annexedfile
 	git_annex_expectoutput "find" [] [annexedfile]
 	git_annex_expectoutput "find" ["--exclude", annexedfile, "--and", "--exclude", sha1annexedfile] []
+	git_annex_expectoutput "find" ["--include", annexedfile] [annexedfile]
 	git_annex_expectoutput "find" ["--not", "--in", "origin"] []
 	git_annex_expectoutput "find" ["--copies", "1", "--and", "--not", "--copies", "2"] [sha1annexedfile]
 	git_annex_expectoutput "find" ["--inbackend", "SHA1"] [sha1annexedfile]

From 7892397020fe345886ef9aa84a6c1580ebee5348 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 14:50:20 -0400
Subject: [PATCH 2762/8313] improve output

---
 Command/Unannex.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 8a511bf4da..66611cbd74 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -41,8 +41,10 @@ cleanup file key = do
 	-- pre-commit hook if this file is later added back to
 	-- git as a normal, non-annexed file.
 	whenM (not . null <$> inRepo (LsFiles.staged [file])) $ do
+		showOutput
 		inRepo $ Git.Command.run "commit" [
-			Param "-m", Param "content removed from git annex",
+			Param "-q",
+			Params "-m", Param "content removed from git annex",
 			Param "--", File file]
 
 	fast <- Annex.getState Annex.fast

From 5a275a3f5d79977170a70b3e3ee16e38ac32916f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 14:59:25 -0400
Subject: [PATCH 2763/8313] Can now be built with older git versions (before
 1.7.7); the resulting binary should only be used with old git.

Remove git old version check from configure, and use the git version
it was built against in the git check-attr code.
---
 Git/CheckAttr.hs | 34 ++++++++++++++++++++++++++++------
 Git/Version.hs   | 38 ++++++++++++++++++++++++++++++++++++++
 configure.hs     | 23 ++++-------------------
 debian/changelog |  2 ++
 4 files changed, 72 insertions(+), 25 deletions(-)
 create mode 100644 Git/Version.hs

diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 0d3e798a1e..eedaf66420 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -13,32 +13,54 @@ import Common
 import Git
 import Git.Command
 import qualified Git.Filename
+import qualified Git.Version
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
 lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
 lookup attr files repo = do
-	-- git check-attr needs relative filenames input; it will choke
-	-- on some absolute filenames. This also means it will output
-	-- all relative filenames.
 	cwd <- getCurrentDirectory
-	let relfiles = map (relPathDirToFile cwd . absPathFrom cwd) files
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
         _ <- forkProcess $ do
 		hClose fromh
-                hPutStr toh $ join "\0" relfiles
+                hPutStr toh $ join "\0" $ input cwd
                 hClose toh
                 exitSuccess
         hClose toh
-	(map topair . lines) <$> hGetContents fromh
+	output cwd . lines <$> hGetContents fromh
 	where
 		params = gitCommandLine 
 				[ Param "check-attr"
 				, Param attr
 				, Params "-z --stdin"
 				] repo
+
+		{- Before git 1.7.7, git check-attr worked best with
+		 - absolute filenames; using them worked around some bugs
+		 - with relative filenames.
+		 - 
+		 - With newer git, git check-attr chokes on some absolute
+		 - filenames, and the bugs that necessitated them were fixed,
+		 - so use relative filenames. -}
+		oldgit = Git.Version.older "1.7.7"
+		input cwd
+			| oldgit = map (absPathFrom cwd) files
+			| otherwise = map (relPathDirToFile cwd . absPathFrom cwd) files
+		output cwd
+			| oldgit = map (torel cwd . topair)
+			| otherwise = map topair
+
 		topair l = (Git.Filename.decode file, value)
 			where 
 				file = join sep $ beginning bits
 				value = end bits !! 0
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
+
+		torel cwd (file, value) = (relfile, value)
+			where
+				relfile
+					| startswith cwd' file = drop (length cwd') file
+					| otherwise = relPathDirToFile top' file
+				top = workTree repo
+				cwd' = cwd ++ "/"
+				top' = top ++ "/"
diff --git a/Git/Version.hs b/Git/Version.hs
new file mode 100644
index 0000000000..c8bc121d66
--- /dev/null
+++ b/Git/Version.hs
@@ -0,0 +1,38 @@
+{- git version checking
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.Version where
+
+import Common
+import qualified Build.SysConfig
+
+{- Using the version it was configured for avoids running git to check its
+ - version, at the cost that upgrading git won't be noticed.
+ - This is only acceptable because it's rare that git's version influences
+ - code's behavior. -}
+version :: String
+version = Build.SysConfig.gitversion
+
+older :: String -> Bool
+older v = normalize version < normalize v
+
+{- To compare dotted versions like 1.7.7 and 1.8, they are normalized to
+ - a somewhat arbitrary integer representation. -}
+normalize :: String -> Integer
+normalize = sum . mult 1 . reverse .
+		extend precision . take precision .
+		map readi . split "."
+	where
+		extend n l = l ++ replicate (n - length l) 0
+		mult _ [] = []
+		mult n (x:xs) = (n*x) : mult (n*10^width) xs
+		readi :: String -> Integer
+		readi s = case reads s of
+			((x,_):_) -> x
+			_ -> 0
+		precision = 10 -- number of segments of the version to compare
+		width = length "yyyymmddhhmmss" -- maximum width of a segment
diff --git a/configure.hs b/configure.hs
index bf25de5079..3b3626dd22 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,7 +2,6 @@
 
 import System.Directory
 import Data.List
-import Data.String.Utils
 import System.Cmd.Utils
 
 import Build.TestConfig
@@ -11,7 +10,7 @@ tests :: [TestCase]
 tests =
 	[ TestCase "version" getVersion
 	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
-	, TestCase "git version" checkGitVersion
+	, TestCase "git version" getGitVersion
 	, testCp "cp_a" "-a"
 	, testCp "cp_p" "-p"
 	, testCp "cp_reflink_auto" "--reflink=auto"
@@ -58,25 +57,11 @@ getVersionString = do
 	where
 		middle = drop 1 . init
 
-{- Checks for a new enough version of git. -}
-checkGitVersion :: Test
-checkGitVersion = do
+getGitVersion :: Test
+getGitVersion = do
 	(_, s) <- pipeFrom "git" ["--version"]
 	let version = last $ words $ head $ lines s
-	if dotted version < dotted need
-		then error $ "git version " ++ version ++ " too old; need " ++ need
-		else return $ Config "gitversion" (StringConfig version)
-	where
-		-- for git-check-attr behavior change
-		need = "1.7.7"
-		dotted = sum . mult 1 . reverse . extend 10 . map readi . split "." 
-		extend n l = l ++ replicate (n - length l) 0
-		mult _ [] = []
-		mult n (x:xs) = (n*x) : mult (n*100) xs
-		readi :: String -> Integer
-		readi s = case reads s of
-			((x,_):_) -> x
-			_ -> 0
+	return $ Config "gitversion" (StringConfig version)
 
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
diff --git a/debian/changelog b/debian/changelog
index 13bca33260..6c5b6effb9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Test suite improvements. Current top-level test coverage: 75%
   * Improve deletion of files from rsync special remotes. Closes: #652849
   * Add --include, which is the same as --not --exclude.
+  * Can now be built with older git versions (before 1.7.7); the resulting
+    binary should only be used with old git.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From 1071a3cf02a2be9b8fd7ec2222e80f6090c4b053 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlrnOT_XLcNNtmIwVdAJCJYu1BwAAOYtBI"
 
Date: Thu, 22 Dec 2011 19:21:50 +0000
Subject: [PATCH 2764/8313]

---
 .../link_file_to_remote_repo_feature.mdwn     | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 doc/todo/link_file_to_remote_repo_feature.mdwn

diff --git a/doc/todo/link_file_to_remote_repo_feature.mdwn b/doc/todo/link_file_to_remote_repo_feature.mdwn
new file mode 100644
index 0000000000..4bbd2b1a8f
--- /dev/null
+++ b/doc/todo/link_file_to_remote_repo_feature.mdwn
@@ -0,0 +1,48 @@
+I have two repos, using SHA1 backend and both using git.
+The first one is a laptop, the second one is a usb drive.
+
+When I drop a file on the laptop repo, the file is not available on that repo until I run *git annex get*
+But when the usb drive is plugged in the file is actually available.
+
+How about adding a feature to link some/all files to the remote repo?
+
+e.g. 
+We have *railscasts/196-nested-model-form-part-1.mp4* file added to git, and only available on the usb drive:
+
+      $ git annex whereis 196-nested-model-form-part-1.mp4 
+         whereis 196-nested-model-form-part-1.mp4 (1 copy) 
+  	   a7b7d7a4-2a8a-11e1-aebc-d3c589296e81 -- origin (Portable usb drive)
+
+I can see the link with:
+
+    $ cd railscasts
+    $ ls -ls 196*
+    8 lrwxr-xr-x  1 framallo  staff  193 Dec 20 05:49 196-nested-model-form-part-1.mp4 -> ../.git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e
+
+I save this in a variable just to make the example more clear:
+
+    ID=".git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e"
+
+The file doesn't exist on the local repo:
+
+    $ ls ../$ID
+    ls: ../$ID: No such file or directory
+
+however I can create a link to access that file on the remote repo.
+First I create a needed dir:
+
+    $ mkdir ../.git/annex/objects/Wz/6P/SHA256-s16898930--43679c67cd968243f58f8f7fb30690b5f3f067574e318d609a01613a2a14351e/
+
+Then I link to the remote file:
+
+    $ ln -s /mnt/usb_drive/repo_folder/$ID ../$ID
+
+now I can open the file in the laptop repo.
+
+
+I think it could be easy to implement. Maybe It's a naive approach, but looks apealing.
+Checking if it's a real file or a link shouldn't impact on performance.
+The limitation is that it would work only with remote repos on local dirs
+
+Also allows you to have one directory structure like AFS or other distributed FS. If the file is not local I go to the remote server. 
+Which is great for apps like Picasa, Itunes, and friends that depends on the file location.

From 2a7246864eb2a43f041eb35ffe58166ca2c59229 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlrnOT_XLcNNtmIwVdAJCJYu1BwAAOYtBI"
 
Date: Thu, 22 Dec 2011 19:33:11 +0000
Subject: [PATCH 2765/8313]

---
 doc/forum/vlc_and_git-annex.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/vlc_and_git-annex.mdwn

diff --git a/doc/forum/vlc_and_git-annex.mdwn b/doc/forum/vlc_and_git-annex.mdwn
new file mode 100644
index 0000000000..cb07f8183c
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex.mdwn
@@ -0,0 +1,11 @@
+I used to save movies with the srt subtitle files next to them. 
+
+Usually vlc finds it because it's on the same directory than the movie file, however with git annex the link is located on another folder.
+So after adding movies to git, the subtitles doesn't load anymore.
+
+couldn't find a quick fix. I'm thinking a bash script, but wanted to discuss it here with all annex users.
+
+I know It's out of annex scope, but I think a movie archive is a great scenario for git-annex.
+most of my HD is filled up with movies from the camcorder, screencast, etc... 
+And we usually don't modify those files
+

From 58956c2a9071193ac1faeaab6551a4dba12e22fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 15:38:15 -0400
Subject: [PATCH 2766/8313] cleanup

---
 test.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test.hs b/test.hs
index b4823fda8d..75d169105b 100644
--- a/test.hs
+++ b/test.hs
@@ -461,8 +461,6 @@ test_migrate = "git-annex migrate" ~: TestList [t False, t True]
 		annexed_present sha1annexedfile
 		checkbackend annexedfile backendSHA256
 		checkbackend sha1annexedfile backendSHA256
-		
-		where
 
 test_unused :: Test
 test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do

From 38ad1065c9b26d98ba63185ba53407410e058455 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 20:04:21 +0000
Subject: [PATCH 2767/8313] Added a comment

---
 ..._c39f1bb7c61a89b238c61bee1c049767._comment | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment
new file mode 100644
index 0000000000..a337002804
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_7_c39f1bb7c61a89b238c61bee1c049767._comment
@@ -0,0 +1,54 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 7"
+ date="2011-12-22T20:04:14Z"
+ content="""
+> My main concern with putting this in git-annex is that finding
+> duplicates necessarily involves storing a list of every key and file
+> in the repository
+
+Only if you want to search the *whole* repository for duplicates, and if
+you do, then you're necessarily going to have to chew up memory in
+some process anyway, so what difference whether it's git-annex or
+(say) a Perl wrapper?
+
+> and git-annex is very carefully built to avoid things that require
+> non-constant memory use, so that it can scale to very big
+> repositories.
+
+That's a worthy goal, but if everything could be implemented with an
+O(1) memory footprint then we'd be in much more pleasant world :-)
+Even O(n) isn't that bad ...
+
+That aside, I like your `--format=\"%f %k\n\"` idea a lot.  That opens
+up the \"black box\" of `.git/annex/objects` and makes nice things
+possible, as your pipeline already demonstrates.  However, I'm not
+sure why you think `git annex find | sort | uniq` would be more
+efficient.  Not only does the sort require the very thing you were
+trying to avoid (i.e. the whole list in memory), but it's also 
+O(n log n) which is significantly slower than my O(n) Perl script 
+linked above.
+
+More considerations about this pipeline:
+
+* Doesn't it only include locally available files?  Ideally it should
+  spot duplicates even when the backing blob is not available locally.
+* What's the point of `--include '*'` ?  Doesn't `git annex find` 
+  with no arguments already include all files, modulo the requirement
+  above that they're locally available?
+* Any user using this `git annex find | ...` approach is likely to
+  run up against its limitations sooner rather than later, because
+  they're already used to the plethora of options `find(1)` provides.
+  Rather than reinventing the wheel, is there some way `git annex find`
+  could harness the power of `find(1)` ?
+
+Those considerations aside, a combined approach would be to implement
+
+    git annex find --format=...
+
+and then alter my Perl wrapper to `popen(2)` from that rather than using
+`File::Find`.  But I doubt you would want to ship Perl wrappers in the
+distribution, so if you don't provide a Haskell equivalent then users
+who can't code are left high and dry.
+"""]]

From c02605919b3d3cd30cb2f8e14ba5545e0ffaf205 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 22 Dec 2011 20:15:23 +0000
Subject: [PATCH 2768/8313] Added a comment: How much memory would it actually
 use anyway?

---
 .../comment_8_221ed2e53420278072a6d879c6f251d1._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment
new file mode 100644
index 0000000000..5ac292afeb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_8_221ed2e53420278072a6d879c6f251d1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="How much memory would it actually use anyway?"
+ date="2011-12-22T20:15:22Z"
+ content="""
+Another thought - an SHA1 digest is 20 bytes.  That means you can fit over 50 million keys into 1GB of RAM.  Granted you also need memory to store the values (pathnames) which in many cases will be longer, and some users may also choose more expensive backends than SHA1 ... but even so, it seems to me that you are at risk of throwing the baby out with the bath water.  
+"""]]

From cf496f09ab3777cc4ffe601e876b432dc320bd12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 17:59:14 -0400
Subject: [PATCH 2769/8313] add a text formatter

This is built for speed; a format string is parsed once, generating a
Format, that can be applied repeatedly to different sets of variables
to generate output.
---
 Utility/Format.hs | 94 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 Utility/Format.hs

diff --git a/Utility/Format.hs b/Utility/Format.hs
new file mode 100644
index 0000000000..a49d95ff87
--- /dev/null
+++ b/Utility/Format.hs
@@ -0,0 +1,94 @@
+{- Formatted string handling.
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Format (gen, format) where
+
+import Text.Printf (printf)
+import Data.String.Utils (replace)
+import Data.Char (isAlphaNum)
+import qualified Data.Map as M
+import Data.Maybe
+
+import Utility.PartialPrelude
+
+type FormatString = String
+
+{- A format consists of a list of fragments, with other text suffixed to
+ - the end. -}
+data Format = Format { spans :: [Frag], suffix :: String }
+	deriving (Show)
+
+{- A fragment is a variable (which may be padded), prefixed by some text. -}
+data Frag = Frag { prefix :: String, varname :: String, pad :: Int }
+	deriving (Show)
+
+newFormat :: Format
+newFormat = Format [] ""
+
+{- Expands a Format using some variables, generating a formatted string.
+ - This can be repeatedly called, efficiently. -}
+format :: Format -> M.Map String String -> String
+format f vars = concat $ concat $ reverse $ [suffix f] : go (spans f) []
+	where
+		go [] c = c
+		go (s:rest) c = go rest $ [prefix s, val s]:c
+		val (Frag { varname = var, pad = p }) =
+			justify p $ fromMaybe "" $ M.lookup var vars
+		justify p v
+			| p > 0 = take (p - length v) spaces ++ v
+			| p < 0 = v ++ take (-1 * (length v + p)) spaces
+			| otherwise = v
+		spaces = repeat ' '
+
+{- Generates a Format that can be used to expand variables in a
+ - format string, such as "${foo} ${bar}\n"
+ -
+ - To handle \n etc, printf is used, first escaping %, to
+ - avoid it needing any printf arguments.
+ -
+ - Left padding is enabled by "${var;width}"
+ - Right padding is enabled by "${var;-width}"
+ -
+ - (This is the same type of format string used by dpkg-query.)
+ -}
+gen :: FormatString -> Format
+gen = scan newFormat . printf . escapeprintf
+	where
+		escapeprintf = replace "%" "%%"
+		-- The Format is built up with fields reversed, for
+		-- efficiency.
+		finalize f v = f
+			{ suffix = (reverse $ suffix f) ++ v
+			, spans = (reverse $ spans f)
+			}
+		scan f (a:b:cs)
+			| a == '$' && b == '{' = invar f [] cs
+			| otherwise = scan f { suffix = a:suffix f } (b:cs)
+		scan f v = finalize f v
+		invar f var [] = finalize f $ novar var
+		invar f var (c:cs)
+			| c == '}' = foundvar f var 0 cs
+			| isAlphaNum c = invar f (c:var) cs
+			| c == ';' = inpad "" f var cs
+			| otherwise = scan f { suffix = (reverse $ novar $ c:var) ++ suffix f } cs
+		inpad p f var (c:cs)
+			| c == '}' = foundvar f var (readpad $ reverse p) cs
+			| otherwise = inpad (c:p) f var cs
+		inpad p f var [] = finalize f $ novar $ p++";"++var
+		readpad = fromMaybe 0 . readMaybe
+		novar v = "${" ++ reverse v
+		foundvar f v p cs = scan f' cs
+			where
+				f' = f
+					{ suffix = ""
+					, spans = newspan:spans f
+					}
+				newspan = Frag
+					{ prefix = reverse $ suffix f
+					, varname = reverse v
+					, pad = p
+					}

From 06bafae9e016b4d65c023546516e51bd32b08649 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 18:31:44 -0400
Subject: [PATCH 2770/8313] Format strings can be specified using the new
 --find option, to control what is output by git annex find.

---
 Annex.hs           |  5 +++--
 Command/Find.hs    | 23 +++++++++++++++++++----
 GitAnnex.hs        | 10 +++++++---
 Options.hs         |  2 ++
 Utility/Format.hs  |  2 +-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn | 14 +++++++++++++-
 7 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index e5792fbcb2..e82ffc5d1f 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -37,6 +37,7 @@ import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
 import qualified Utility.Matcher
+import qualified Utility.Format
 import qualified Data.Map as M
 
 -- git-annex's monad
@@ -62,7 +63,7 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
-	, print0 :: Bool
+	, format :: Maybe Utility.Format.Format
 	, branchstate :: BranchState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
@@ -85,7 +86,7 @@ newState gitrepo = AnnexState
 	, force = False
 	, fast = False
 	, auto = False
-	, print0 = False
+	, format = Nothing
 	, branchstate = startBranchState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
diff --git a/Command/Find.hs b/Command/Find.hs
index 47058fa257..6050ff7bbb 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -7,11 +7,16 @@
 
 module Command.Find where
 
+import qualified Data.Map as M
+
 import Common.Annex
 import Command
 import Annex.Content
 import Limit
 import qualified Annex
+import qualified Utility.Format
+import Utility.DataUnits
+import Types.Key
 
 def :: [Command]
 def = [command "find" paramPaths seek "lists available files"]
@@ -24,8 +29,18 @@ start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
 	whenM (liftM2 (||) (inAnnex key) limited) $ do
-		print0 <- Annex.getState Annex.print0
-		if print0
-			then liftIO $ putStr (file ++ "\0")
-			else liftIO $ putStrLn file
+		f <- Annex.getState Annex.format
+		case f of
+			Nothing -> liftIO $ putStrLn file
+			Just formatter -> liftIO $ putStr $
+				Utility.Format.format formatter vars
 	stop
+	where
+		vars = M.fromList
+			[ ("file", file)
+			, ("key", show key)
+			, ("backend", keyBackendName key)
+			, ("bytesize", size show)
+			, ("humansize", size $ roughSize storageUnits True)
+			]
+		size c = maybe "unknown" c $ keySize key
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 40ebed0d69..7243d69cb0 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -18,6 +18,7 @@ import Types.TrustLevel
 import qualified Annex
 import qualified Remote
 import qualified Limit
+import qualified Utility.Format
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -108,8 +109,10 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option [] ["print0"] (NoArg (setprint0 True))
-		"terminate filename with null"
+	, Option [] ["print0"] (NoArg setprint0)
+		"terminate output with null"
+	, Option [] ["format"] (ReqArg setformat paramFormat)
+		"control format of output"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob)
@@ -125,7 +128,8 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
-		setprint0 v = Annex.changeState $ \s -> s { Annex.print0 = v }
+		setformat v = Annex.changeState $ \s -> s { Annex.format = Just $ Utility.Format.gen v }
+		setprint0 = setformat "${file}\0"
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			newg <- inRepo $ Git.Config.store v
diff --git a/Options.hs b/Options.hs
index a8c165a816..cce750316e 100644
--- a/Options.hs
+++ b/Options.hs
@@ -82,6 +82,8 @@ paramUUID :: String
 paramUUID = "UUID"
 paramType :: String
 paramType = "TYPE"
+paramFormat :: String
+paramFormat = "FORMAT"
 paramKeyValue :: String
 paramKeyValue = "K=V"
 paramNothing :: String
diff --git a/Utility/Format.hs b/Utility/Format.hs
index a49d95ff87..cde63f57cd 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Format (gen, format) where
+module Utility.Format (Format, gen, format) where
 
 import Text.Printf (printf)
 import Data.String.Utils (replace)
diff --git a/debian/changelog b/debian/changelog
index 6c5b6effb9..a27611f553 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Add --include, which is the same as --not --exclude.
   * Can now be built with older git versions (before 1.7.7); the resulting
     binary should only be used with old git.
+  * Format strings can be specified using the new --find option, to control
+    what is output by git annex find.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index f4eef5c4c7..70e54a91f4 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -250,7 +250,11 @@ subdirectories).
   annexed files whose content is not present, specify --not --in "."
 
   To output filenames terminated with nulls, for use with xargs -0,
-  specify --print0.
+  specify --print0. Or, a custom output formatting can be specified using
+  --format. The default output format is the same as --format='${file}\n'
+
+  These variables are available for use in formats: file, key, backend,
+  bytesize, humansize
 
 * whereis [path ...]
 
@@ -428,6 +432,14 @@ subdirectories).
   are in the annex, their backend is known and this option is not
   necessary.
 
+* --format=value
+
+  Specifies a custom output format. The value is a format string, 
+  in which '${var}' is expanded to the value of a variable. To right-align
+  a variable with whitespace, use '${var;width}' ; to left-align
+  a variable, use '${var;-width}'. Also, '\n' is a newline, '\0' is a NULL,
+  etc.
+
 * -c name=value
 
   Used to override git configuration settings. May be specified multiple times.

From a0872a8ec34894689489dd0cf47887d8609b9f47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 19:56:31 -0400
Subject: [PATCH 2771/8313] better data type

---
 Utility/Format.hs  | 86 ++++++++++++++++++++--------------------------
 doc/git-annex.mdwn |  4 +--
 2 files changed, 40 insertions(+), 50 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index cde63f57cd..5a74da96b2 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -17,78 +17,68 @@ import Utility.PartialPrelude
 
 type FormatString = String
 
-{- A format consists of a list of fragments, with other text suffixed to
- - the end. -}
-data Format = Format { spans :: [Frag], suffix :: String }
+{- A format consists of a list of fragments. -}
+type Format = [Frag]
+
+{- A fragment is either a constant string, or a variable, with a padding. -}
+data Frag = Const String | Var String Padding
 	deriving (Show)
 
-{- A fragment is a variable (which may be padded), prefixed by some text. -}
-data Frag = Frag { prefix :: String, varname :: String, pad :: Int }
-	deriving (Show)
+{- Positive padding is right justification; negative padding is left
+ - justification. -}
+type Padding = Int
 
-newFormat :: Format
-newFormat = Format [] ""
+empty :: Frag -> Bool
+empty (Const "") = True
+empty _ = False
 
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
 format :: Format -> M.Map String String -> String
-format f vars = concat $ concat $ reverse $ [suffix f] : go (spans f) []
+format f vars = concatMap expand f
 	where
-		go [] c = c
-		go (s:rest) c = go rest $ [prefix s, val s]:c
-		val (Frag { varname = var, pad = p }) =
-			justify p $ fromMaybe "" $ M.lookup var vars
-		justify p v
-			| p > 0 = take (p - length v) spaces ++ v
-			| p < 0 = v ++ take (-1 * (length v + p)) spaces
-			| otherwise = v
+		expand (Const s) = s
+		expand (Var name padding) = justify padding $
+			fromMaybe "" $ M.lookup name vars
+		justify p s
+			| p > 0 = take (p - length s) spaces ++ s
+			| p < 0 = s ++ take (-1 * (length s + p)) spaces
+			| otherwise = s
 		spaces = repeat ' '
 
 {- Generates a Format that can be used to expand variables in a
- - format string, such as "${foo} ${bar}\n"
- -
- - To handle \n etc, printf is used, first escaping %, to
- - avoid it needing any printf arguments.
- -
- - Left padding is enabled by "${var;width}"
- - Right padding is enabled by "${var;-width}"
+ - format string, such as "${foo} ${bar;10} ${baz;-10}\n"
  -
  - (This is the same type of format string used by dpkg-query.)
  -}
 gen :: FormatString -> Format
-gen = scan newFormat . printf . escapeprintf
+gen = finalize . scan []
 	where
-		escapeprintf = replace "%" "%%"
-		-- The Format is built up with fields reversed, for
-		-- efficiency.
-		finalize f v = f
-			{ suffix = (reverse $ suffix f) ++ v
-			, spans = (reverse $ spans f)
-			}
+		-- The Format is built up in reverse, for efficiency,
+		-- To finalize it, fix the reversing and do some
+		-- optimisations, including fusing adjacent Consts.
+		finalize = filter (not . empty) . fuse []
+		fuse f [] = f
+		fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
+		fuse f (v:vs) = fuse (v:f) vs
+
 		scan f (a:b:cs)
 			| a == '$' && b == '{' = invar f [] cs
-			| otherwise = scan f { suffix = a:suffix f } (b:cs)
-		scan f v = finalize f v
-		invar f var [] = finalize f $ novar var
+			| otherwise = scan (Const [a] : f ) (b:cs)
+		scan f v = Const v : f
+
+		invar f var [] = Const (novar var) : f
 		invar f var (c:cs)
 			| c == '}' = foundvar f var 0 cs
 			| isAlphaNum c = invar f (c:var) cs
 			| c == ';' = inpad "" f var cs
-			| otherwise = scan f { suffix = (reverse $ novar $ c:var) ++ suffix f } cs
+			| otherwise = scan ((Const $ reverse $ novar $ c:var):f) cs
+
 		inpad p f var (c:cs)
 			| c == '}' = foundvar f var (readpad $ reverse p) cs
 			| otherwise = inpad (c:p) f var cs
-		inpad p f var [] = finalize f $ novar $ p++";"++var
+		inpad p f var [] = Const (novar $ p++";"++var) : f
 		readpad = fromMaybe 0 . readMaybe
+
 		novar v = "${" ++ reverse v
-		foundvar f v p cs = scan f' cs
-			where
-				f' = f
-					{ suffix = ""
-					, spans = newspan:spans f
-					}
-				newspan = Frag
-					{ prefix = reverse $ suffix f
-					, varname = reverse v
-					, pad = p
-					}
+		foundvar f v p cs = scan (Var (reverse v) p : f) cs
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 70e54a91f4..050a56980f 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -435,8 +435,8 @@ subdirectories).
 * --format=value
 
   Specifies a custom output format. The value is a format string, 
-  in which '${var}' is expanded to the value of a variable. To right-align
-  a variable with whitespace, use '${var;width}' ; to left-align
+  in which '${var}' is expanded to the value of a variable. To right-justify
+  a variable with whitespace, use '${var;width}' ; to left-justify
   a variable, use '${var;-width}'. Also, '\n' is a newline, '\0' is a NULL,
   etc.
 

From cba3ce08dfaa3318aa80b414e4d6d4f40d843d15 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:14:35 -0400
Subject: [PATCH 2772/8313] handle C-style escapes in Format

I was happily able to repurpose some code from Git.Filename to handle this.

I remember writing that code... a whole afternoon at a coffee shop, after
which I felt I'd struggled with Haskell and git, and sorta lost, in needing
to write this nasty peice of code. But was also pleased at the use of a
pair of functions and quickcheck that allowed me to get it 100% right.
So, turns out I not only got it right, but the code wasn't as special-purpose
as I'd feared. Yay!
---
 Git/Filename.hs   | 61 ++-----------------------------
 Utility/Format.hs | 93 ++++++++++++++++++++++++++++++++++++++++++-----
 test.hs           |  4 +-
 3 files changed, 90 insertions(+), 68 deletions(-)

diff --git a/Git/Filename.hs b/Git/Filename.hs
index 35b5532507..5e076d3b5a 100644
--- a/Git/Filename.hs
+++ b/Git/Filename.hs
@@ -8,10 +8,7 @@
 
 module Git.Filename where
 
-import qualified Codec.Binary.UTF8.String
-import Data.Char
-import Data.Word (Word8)
-import Text.Printf
+import Utility.Format (decode_c, encode_c)
 
 import Common
 
@@ -19,64 +16,12 @@ decode :: String -> FilePath
 decode [] = []
 decode f@(c:s)
 	-- encoded strings will be inside double quotes
-	| c == '"' && end s == ['"'] = unescape ("", beginning s)
+	| c == '"' && end s == ['"'] = decode_c $ beginning s
 	| otherwise = f
-	where
-		e = '\\'
-		unescape (b, []) = b
-		-- look for escapes starting with '\'
-		unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
-			where
-				pair = span (/= e) v
-		isescape x = x == e
-		-- \NNN is an octal encoded character
-		handle (x:n1:n2:n3:rest)
-			| isescape x && alloctal = (fromoctal, rest)
-				where
-					alloctal = isOctDigit n1 &&
-						isOctDigit n2 &&
-						isOctDigit n3
-					fromoctal = [chr $ readoctal [n1, n2, n3]]
-					readoctal o = Prelude.read $ "0o" ++ o :: Int
-		-- \C is used for a few special characters
-		handle (x:nc:rest)
-			| isescape x = ([echar nc], rest)
-			where
-				echar 'a' = '\a'
-				echar 'b' = '\b'
-				echar 'f' = '\f'
-				echar 'n' = '\n'
-				echar 'r' = '\r'
-				echar 't' = '\t'
-				echar 'v' = '\v'
-				echar a = a
-		handle n = ("", n)
 
 {- Should not need to use this, except for testing decode. -}
 encode :: FilePath -> String
-encode s = foldl (++) "\"" (map echar s) ++ "\""
-	where
-		e c = '\\' : [c]
-		echar '\a' = e 'a'
-		echar '\b' = e 'b'
-		echar '\f' = e 'f'
-		echar '\n' = e 'n'
-		echar '\r' = e 'r'
-		echar '\t' = e 't'
-		echar '\v' = e 'v'
-		echar '\\' = e '\\'
-		echar '"'  = e '"'
-		echar x
-			| ord x < 0x20 = e_num x -- low ascii
-			| ord x >= 256 = e_utf x
-			| ord x > 0x7E = e_num x -- high ascii
-			| otherwise = [x]        -- printable ascii
-			where 
-				showoctal i = '\\' : printf "%03o" i
-				e_num c = showoctal $ ord c
-				-- unicode character is decomposed to
-				-- Word8s and each is shown in octal
-				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+encode s = "\"" ++ encode_c s ++ "\""
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
diff --git a/Utility/Format.hs b/Utility/Format.hs
index 5a74da96b2..804dbff4c8 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -1,17 +1,25 @@
 {- Formatted string handling.
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2010, 2011 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Format (Format, gen, format) where
+module Utility.Format (
+	Format,
+	gen,
+	format,
+	decode_c,
+	encode_c,
+	prop_idempotent_deencode
+) where
 
 import Text.Printf (printf)
-import Data.String.Utils (replace)
-import Data.Char (isAlphaNum)
+import Data.Char (isAlphaNum, isOctDigit, chr, ord)
+import Data.Maybe (fromMaybe)
+import Data.Word (Word8)
+import qualified Codec.Binary.UTF8.String
 import qualified Data.Map as M
-import Data.Maybe
 
 import Utility.PartialPrelude
 
@@ -52,12 +60,11 @@ format f vars = concatMap expand f
  - (This is the same type of format string used by dpkg-query.)
  -}
 gen :: FormatString -> Format
-gen = finalize . scan []
+gen = filter (not . empty) . fuse [] . scan [] . decode_c
 	where
 		-- The Format is built up in reverse, for efficiency,
-		-- To finalize it, fix the reversing and do some
-		-- optimisations, including fusing adjacent Consts.
-		finalize = filter (not . empty) . fuse []
+		-- and can have adjacent Consts. Fusing it fixes both
+		-- problems.
 		fuse f [] = f
 		fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
 		fuse f (v:vs) = fuse (v:f) vs
@@ -82,3 +89,71 @@ gen = finalize . scan []
 
 		novar v = "${" ++ reverse v
 		foundvar f v p cs = scan (Var (reverse v) p : f) cs
+
+
+{- Decodes a C-style encoding, where \n is a newline, \NNN is an octal
+ - encoded character, etc.
+ -}
+decode_c :: FormatString -> FormatString
+decode_c [] = []
+decode_c s = unescape ("", s)
+	where
+		e = '\\'
+		unescape (b, []) = b
+		-- look for escapes starting with '\'
+		unescape (b, v) = b ++ fst pair ++ unescape (handle $ snd pair)
+			where
+				pair = span (/= e) v
+		isescape x = x == e
+		-- \NNN is an octal encoded character
+		handle (x:n1:n2:n3:rest)
+			| isescape x && alloctal = (fromoctal, rest)
+				where
+					alloctal = isOctDigit n1 &&
+						isOctDigit n2 &&
+						isOctDigit n3
+					fromoctal = [chr $ readoctal [n1, n2, n3]]
+					readoctal o = Prelude.read $ "0o" ++ o :: Int
+		-- \C is used for a few special characters
+		handle (x:nc:rest)
+			| isescape x = ([echar nc], rest)
+			where
+				echar 'a' = '\a'
+				echar 'b' = '\b'
+				echar 'f' = '\f'
+				echar 'n' = '\n'
+				echar 'r' = '\r'
+				echar 't' = '\t'
+				echar 'v' = '\v'
+				echar a = a
+		handle n = ("", n)
+
+{- Should not need to use this, except for testing decode_c. -}
+encode_c :: FormatString -> FormatString
+encode_c s = concatMap echar s
+	where
+		e c = '\\' : [c]
+		echar '\a' = e 'a'
+		echar '\b' = e 'b'
+		echar '\f' = e 'f'
+		echar '\n' = e 'n'
+		echar '\r' = e 'r'
+		echar '\t' = e 't'
+		echar '\v' = e 'v'
+		echar '\\' = e '\\'
+		echar '"'  = e '"'
+		echar x
+			| ord x < 0x20 = e_num x -- low ascii
+			| ord x >= 256 = e_utf x
+			| ord x > 0x7E = e_num x -- high ascii
+			| otherwise = [x]        -- printable ascii
+			where 
+				showoctal i = '\\' : printf "%03o" i
+				e_num c = showoctal $ ord c
+				-- unicode character is decomposed to
+				-- Word8s and each is shown in octal
+				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+
+{- for quickcheck -}
+prop_idempotent_deencode :: String -> Bool
+prop_idempotent_deencode s = s == decode_c (encode_c s)
diff --git a/test.hs b/test.hs
index 75d169105b..a2fa98e4df 100644
--- a/test.hs
+++ b/test.hs
@@ -45,6 +45,7 @@ import qualified Utility.Path
 import qualified Utility.FileMode
 import qualified Utility.Gpg
 import qualified Build.SysConfig
+import qualified Utility.Format
 
 -- for quickcheck
 instance Arbitrary Types.Key.Key where
@@ -72,7 +73,8 @@ propigate (Counts { errors = e , failures = f }, _)
 
 quickcheck :: Test
 quickcheck = TestLabel "quickcheck" $ TestList
-	[ qctest "prop_idempotent_deencode" Git.Filename.prop_idempotent_deencode
+	[ qctest "prop_idempotent_deencode_git" Git.Filename.prop_idempotent_deencode
+	, qctest "prop_idempotent_deencode" Utility.Format.prop_idempotent_deencode
 	, qctest "prop_idempotent_fileKey" Locations.prop_idempotent_fileKey
 	, qctest "prop_idempotent_key_read_show" Types.Key.prop_idempotent_key_read_show
 	, qctest "prop_idempotent_shellEscape" Utility.SafeCommand.prop_idempotent_shellEscape

From eb7ef6264ee57f69520960ae818ad0b94e5873c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:18:33 -0400
Subject: [PATCH 2773/8313] fix

---
 doc/git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 050a56980f..7ad3fac69e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -251,7 +251,7 @@ subdirectories).
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0. Or, a custom output formatting can be specified using
-  --format. The default output format is the same as --format='${file}\n'
+  --format. The default output format is the same as --format='${file}\\n'
 
   These variables are available for use in formats: file, key, backend,
   bytesize, humansize
@@ -437,7 +437,7 @@ subdirectories).
   Specifies a custom output format. The value is a format string, 
   in which '${var}' is expanded to the value of a variable. To right-justify
   a variable with whitespace, use '${var;width}' ; to left-justify
-  a variable, use '${var;-width}'. Also, '\n' is a newline, '\0' is a NULL,
+  a variable, use '${var;-width}'. Also, '\\n' is a newline, '\\000' is a NULL,
   etc.
 
 * -c name=value

From db964e358f16d19e9a2e6124d6be8a9c87fdd88b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:21:42 -0400
Subject: [PATCH 2774/8313] reorg

---
 Utility/Format.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index 804dbff4c8..d167087011 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -36,10 +36,6 @@ data Frag = Const String | Var String Padding
  - justification. -}
 type Padding = Int
 
-empty :: Frag -> Bool
-empty (Const "") = True
-empty _ = False
-
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
 format :: Format -> M.Map String String -> String
@@ -63,7 +59,7 @@ gen :: FormatString -> Format
 gen = filter (not . empty) . fuse [] . scan [] . decode_c
 	where
 		-- The Format is built up in reverse, for efficiency,
-		-- and can have adjacent Consts. Fusing it fixes both
+		-- and can have many adjacent Consts. Fusing it fixes both
 		-- problems.
 		fuse f [] = f
 		fuse f (Const c1:Const c2:vs) = fuse f $ Const (c2++c1) : vs
@@ -90,6 +86,9 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 		novar v = "${" ++ reverse v
 		foundvar f v p cs = scan (Var (reverse v) p : f) cs
 
+empty :: Frag -> Bool
+empty (Const "") = True
+empty _ = False
 
 {- Decodes a C-style encoding, where \n is a newline, \NNN is an octal
  - encoded character, etc.

From 13a0c292b3bc72917cb8ce89e96f805602e81904 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 20:29:24 -0400
Subject: [PATCH 2775/8313] update example to actually work with new --format
 option

---
 ...__git_annex__34___command_that_will_skip_duplicates.mdwn | 2 ++
 .../comment_6_f24541ada1c86d755acba7e9fa7cff24._comment     | 6 ++++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index eda17aea66..ca18afc578 100644
--- a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -24,3 +24,5 @@ I want this because I have copies of various of mine (photos, in particular) sca
 (Another way to do this would be to "git annex add" them all, and then use a "git annex remove-duplicates" that could prompt me about which files are duplicates of each other, and then I could pipe that command's output into xargs git rm.)
 
 (As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)
+
+> 
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
index 93d3d41f43..5d8ac8e61b 100644
--- a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_6_f24541ada1c86d755acba7e9fa7cff24._comment
@@ -8,7 +8,9 @@ My main concern with putting this in git-annex is that finding duplicates necess
 
 So I would rather come at this from a different angle.. like providing a way to output a list of files and their associated keys, which the user can then use in their own shell pipelines to find duplicate keys:
 
-      git annex find --include '*' --format=\"%f %k\n\" | sort foo --key 2 | uniq --all-repeated --skip-fields=1
+      git annex find --include '*' --format='${file} ${key}\n' | sort --key 2 | uniq --all-repeated --skip-fields=1
 
-(Making that properly handle filenames with spaces is left as an exercise for the reader..)
+Which is implemented now!
+
+(Making that pipeline properly handle filenames with spaces is left as an exercise for the reader..)
 """]]

From 7227dd8f21f24c2ccadd38e1a3dec7b888a23e92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 21:23:11 -0400
Subject: [PATCH 2776/8313] add escape_var hack

Makes it easy to find files with duplicate contents, anyway.. :)
---
 Utility/Format.hs                             | 48 ++++++++++++-------
 doc/git-annex.mdwn                            |  6 ++-
 doc/tips/finding_duplicate_files.mdwn         | 21 ++++++++
 ...4___command_that_will_skip_duplicates.mdwn |  2 +-
 4 files changed, 58 insertions(+), 19 deletions(-)
 create mode 100644 doc/tips/finding_duplicate_files.mdwn

diff --git a/Utility/Format.hs b/Utility/Format.hs
index d167087011..c0ba465806 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -18,6 +18,8 @@ import Text.Printf (printf)
 import Data.Char (isAlphaNum, isOctDigit, chr, ord)
 import Data.Maybe (fromMaybe)
 import Data.Word (Word8)
+import Data.List (isPrefixOf)
+import Data.String.Utils (replace)
 import qualified Codec.Binary.UTF8.String
 import qualified Data.Map as M
 
@@ -42,8 +44,16 @@ format :: Format -> M.Map String String -> String
 format f vars = concatMap expand f
 	where
 		expand (Const s) = s
-		expand (Var name padding) = justify padding $
-			fromMaybe "" $ M.lookup name vars
+		expand (Var name padding) = justify padding $ getvar name
+		getvar name
+			| "escaped_" `isPrefixOf` name = 
+				-- escape whitespace too
+				replace " " (e_asc ' ') $
+				replace "\t" (e_asc '\t') $
+				encode_c $
+				getvar' $ drop (length "escaped_") name
+			| otherwise = getvar' name
+		getvar' name = fromMaybe "" $ M.lookup name vars
 		justify p s
 			| p > 0 = take (p - length s) spaces ++ s
 			| p < 0 = s ++ take (-1 * (length s + p)) spaces
@@ -73,9 +83,9 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 		invar f var [] = Const (novar var) : f
 		invar f var (c:cs)
 			| c == '}' = foundvar f var 0 cs
-			| isAlphaNum c = invar f (c:var) cs
+			| isAlphaNum c || c == '_' = invar f (c:var) cs
 			| c == ';' = inpad "" f var cs
-			| otherwise = scan ((Const $ reverse $ novar $ c:var):f) cs
+			| otherwise = scan ((Const $ novar $ c:var):f) cs
 
 		inpad p f var (c:cs)
 			| c == '}' = foundvar f var (readpad $ reverse p) cs
@@ -127,7 +137,7 @@ decode_c s = unescape ("", s)
 				echar a = a
 		handle n = ("", n)
 
-{- Should not need to use this, except for testing decode_c. -}
+{- Inverse of decode_c. -}
 encode_c :: FormatString -> FormatString
 encode_c s = concatMap echar s
 	where
@@ -141,17 +151,23 @@ encode_c s = concatMap echar s
 		echar '\v' = e 'v'
 		echar '\\' = e '\\'
 		echar '"'  = e '"'
-		echar x
-			| ord x < 0x20 = e_num x -- low ascii
-			| ord x >= 256 = e_utf x
-			| ord x > 0x7E = e_num x -- high ascii
-			| otherwise = [x]        -- printable ascii
-			where 
-				showoctal i = '\\' : printf "%03o" i
-				e_num c = showoctal $ ord c
-				-- unicode character is decomposed to
-				-- Word8s and each is shown in octal
-				e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+		echar c
+			| ord c < 0x20 = e_asc c -- low ascii
+			| ord c >= 256 = e_utf c
+			| ord c > 0x7E = e_asc c -- high ascii
+			| otherwise = [c]        -- printable ascii
+
+-- unicode character is decomposed to individual Word8s,
+-- and each is shown in octal
+e_utf :: Char -> String
+e_utf c = showoctal . toInteger =<<
+	(Codec.Binary.UTF8.String.encode [c] :: [Word8])
+		
+e_asc :: Char -> String
+e_asc c = showoctal $ toInteger $ ord c
+
+showoctal :: Integer -> String				
+showoctal i = '\\' : printf "%03o" i
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 7ad3fac69e..2d0d2597ea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -437,8 +437,10 @@ subdirectories).
   Specifies a custom output format. The value is a format string, 
   in which '${var}' is expanded to the value of a variable. To right-justify
   a variable with whitespace, use '${var;width}' ; to left-justify
-  a variable, use '${var;-width}'. Also, '\\n' is a newline, '\\000' is a NULL,
-  etc.
+  a variable, use '${var;-width}'; to escape unusual characters in a variable,
+  use '${escaped_var}'
+
+  Also, '\\n' is a newline, '\\000' is a NULL, etc.
 
 * -c name=value
 
diff --git a/doc/tips/finding_duplicate_files.mdwn b/doc/tips/finding_duplicate_files.mdwn
new file mode 100644
index 0000000000..94fc85400e
--- /dev/null
+++ b/doc/tips/finding_duplicate_files.mdwn
@@ -0,0 +1,21 @@
+Maybe you had a lot of files scattered around on different drives, and you
+added them all into a single git-annex repository. Some of the files are
+surely duplicates of others.
+
+While git-annex stores the file contents efficiently, it would still
+help in cleaning up this mess if you could find, and perhaps remove
+the duplicate files.
+
+Here's a command line that will show duplicate sets of files grouped together:
+
+	git annex find --include '*' --format='${file} ${escaped_key}\n' | \
+		sort -k2 | uniq --all-repeated=separate -f1 | \
+		sed 's/ [^ ]*$//'
+
+Here's a command line that will remove one of each duplicate set of files:
+
+	git annex find --include '*' --format='${file} ${escaped_key}\n' | \
+		sort -k2 | uniq --repeated -f1 | sed 's/ [^ ]*$//' | \
+		xargs -d '\n' git rm
+
+--[[Joey]] 
diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
index ca18afc578..9336535788 100644
--- a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates.mdwn
@@ -25,4 +25,4 @@ I want this because I have copies of various of mine (photos, in particular) sca
 
 (As I write this, I realize it's possible to parse the destination of the symlink in a way that does this..)
 
-> 
+> [[done]]; see [[tips/finding_duplicate_files]] --[[Joey]]

From fdf02986cf2c7db74cc8ca8ebd2786922652ff9b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Dec 2011 22:03:18 -0400
Subject: [PATCH 2777/8313] find --json

---
 Command/Find.hs  | 16 +++++++++-------
 Messages.hs      | 10 +++++++++-
 Messages/JSON.hs |  9 ++++++++-
 debian/changelog |  3 ++-
 4 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 6050ff7bbb..91386bbd08 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -28,15 +28,17 @@ start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) (inAnnex key) limited) $ do
-		f <- Annex.getState Annex.format
-		case f of
-			Nothing -> liftIO $ putStrLn file
-			Just formatter -> liftIO $ putStr $
-				Utility.Format.format formatter vars
+	whenM (liftM2 (||) (inAnnex key) limited) $
+		unlessM (showFullJSON vars) $ do
+			f <- Annex.getState Annex.format
+			case f of
+				Nothing -> liftIO $ putStrLn file
+				Just formatter -> liftIO $ putStr $
+					Utility.Format.format formatter $
+						M.fromList vars
 	stop
 	where
-		vars = M.fromList
+		vars =
 			[ ("file", file)
 			, ("key", show key)
 			, ("backend", keyBackendName key)
diff --git a/Messages.hs b/Messages.hs
index a7f14f4853..1294e44f69 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -20,6 +20,7 @@ module Messages (
 	warning,
 	indent,
 	maybeShowJSON,
+	showFullJSON,
 	showCustom,
 	showHeader,
 	showRaw,
@@ -90,10 +91,17 @@ warning' w = do
 indent :: String -> String
 indent = join "\n" . map (\l -> "  " ++ l) . lines
 
-{- Shows a JSON value only when in json mode. -}
+{- Shows a JSON fragment only when in json mode. -}
 maybeShowJSON :: JSON a => [(String, a)] -> Annex ()
 maybeShowJSON v = handle (JSON.add v) q
 
+{- Shows a complete JSON value, only when in json mode. -}
+showFullJSON :: JSON a => [(String, a)] -> Annex Bool
+showFullJSON v = Annex.getState Annex.output >>= liftIO . go
+	where
+		go Annex.JSONOutput = JSON.complete v >> return True
+		go _ = return False
+
 {- Performs an action that outputs nonstandard/customized output, and
  - in JSON mode wraps its output in JSON.start and JSON.end, so it's
  - a complete JSON document.
diff --git a/Messages/JSON.hs b/Messages/JSON.hs
index a325ef130b..f7a031e381 100644
--- a/Messages/JSON.hs
+++ b/Messages/JSON.hs
@@ -9,7 +9,8 @@ module Messages.JSON (
 	start,
 	end,
 	note,
-	add
+	add,
+	complete
 ) where
 
 import Text.JSON
@@ -31,3 +32,9 @@ note s = add [("note", s)]
 
 add :: JSON a => [(String, a)] -> IO ()
 add v = putStr $ Stream.add v
+
+complete :: JSON a => [(String, a)] -> IO ()
+complete v = putStr $ concat
+	[ Stream.start v
+	, Stream.end
+	]
diff --git a/debian/changelog b/debian/changelog
index a27611f553..33d196fa32 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,8 +10,9 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Add --include, which is the same as --not --exclude.
   * Can now be built with older git versions (before 1.7.7); the resulting
     binary should only be used with old git.
-  * Format strings can be specified using the new --find option, to control
+  * Format strings can be specified using the new --format option, to control
     what is output by git annex find.
+  * Support git annex find --json
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From f015ef5fde9184b6756ee74c2be1bb39ae5f54ca Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 00:36:25 -0400
Subject: [PATCH 2778/8313] cleanup

---
 Utility/Format.hs | 79 +++++++++++++++++++++++------------------------
 1 file changed, 39 insertions(+), 40 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index c0ba465806..2c2042cc22 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -15,11 +15,10 @@ module Utility.Format (
 ) where
 
 import Text.Printf (printf)
-import Data.Char (isAlphaNum, isOctDigit, chr, ord)
+import Data.Char (isAlphaNum, isOctDigit, isSpace, chr, ord)
 import Data.Maybe (fromMaybe)
 import Data.Word (Word8)
 import Data.List (isPrefixOf)
-import Data.String.Utils (replace)
 import qualified Codec.Binary.UTF8.String
 import qualified Data.Map as M
 
@@ -30,13 +29,13 @@ type FormatString = String
 {- A format consists of a list of fragments. -}
 type Format = [Frag]
 
-{- A fragment is either a constant string, or a variable, with a padding. -}
-data Frag = Const String | Var String Padding
+{- A fragment is either a constant string,
+ - or a variable, with a justification. -}
+data Frag = Const String | Var String Justify
 	deriving (Show)
 
-{- Positive padding is right justification; negative padding is left
- - justification. -}
-type Padding = Int
+data Justify = LeftJustified Int | RightJustified Int | UnJustified
+	deriving (Show)
 
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
@@ -44,20 +43,16 @@ format :: Format -> M.Map String String -> String
 format f vars = concatMap expand f
 	where
 		expand (Const s) = s
-		expand (Var name padding) = justify padding $ getvar name
-		getvar name
-			| "escaped_" `isPrefixOf` name = 
-				-- escape whitespace too
-				replace " " (e_asc ' ') $
-				replace "\t" (e_asc '\t') $
-				encode_c $
-				getvar' $ drop (length "escaped_") name
-			| otherwise = getvar' name
-		getvar' name = fromMaybe "" $ M.lookup name vars
-		justify p s
-			| p > 0 = take (p - length s) spaces ++ s
-			| p < 0 = s ++ take (-1 * (length s + p)) spaces
-			| otherwise = s
+		expand (Var name j)
+			| "escaped_" `isPrefixOf` name =
+				justify j $ encode_c_strict $
+					getvar $ drop (length "escaped_") name
+			| otherwise = justify j $ getvar name
+		getvar name = fromMaybe "" $ M.lookup name vars
+		justify UnJustified s        = s
+		justify (LeftJustified i) s  = s ++ pad i s
+		justify (RightJustified i) s = pad i s ++ s
+		pad i s = take (i - length s) spaces
 		spaces = repeat ' '
 
 {- Generates a Format that can be used to expand variables in a
@@ -82,17 +77,20 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 
 		invar f var [] = Const (novar var) : f
 		invar f var (c:cs)
-			| c == '}' = foundvar f var 0 cs
+			| c == '}' = foundvar f var UnJustified cs
 			| isAlphaNum c || c == '_' = invar f (c:var) cs
 			| c == ';' = inpad "" f var cs
 			| otherwise = scan ((Const $ novar $ c:var):f) cs
 
 		inpad p f var (c:cs)
-			| c == '}' = foundvar f var (readpad $ reverse p) cs
+			| c == '}' = foundvar f var (readjustify $ reverse p) cs
 			| otherwise = inpad (c:p) f var cs
 		inpad p f var [] = Const (novar $ p++";"++var) : f
-		readpad = fromMaybe 0 . readMaybe
-
+		readjustify = getjustify . fromMaybe 0 . readMaybe
+		getjustify i
+			| i == 0 = UnJustified
+			| i < 0 = LeftJustified (-1 * i)
+			| otherwise = RightJustified i
 		novar v = "${" ++ reverse v
 		foundvar f v p cs = scan (Var (reverse v) p : f) cs
 
@@ -139,7 +137,14 @@ decode_c s = unescape ("", s)
 
 {- Inverse of decode_c. -}
 encode_c :: FormatString -> FormatString
-encode_c s = concatMap echar s
+encode_c = encode_c' (const False)
+
+{- Encodes more strictly, including whitespace. -}
+encode_c_strict :: FormatString -> FormatString
+encode_c_strict = encode_c' isSpace
+
+encode_c' :: (Char -> Bool) -> FormatString -> FormatString
+encode_c' p = concatMap echar
 	where
 		e c = '\\' : [c]
 		echar '\a' = e 'a'
@@ -153,21 +158,15 @@ encode_c s = concatMap echar s
 		echar '"'  = e '"'
 		echar c
 			| ord c < 0x20 = e_asc c -- low ascii
-			| ord c >= 256 = e_utf c
+			| ord c >= 256 = e_utf c -- unicode
 			| ord c > 0x7E = e_asc c -- high ascii
-			| otherwise = [c]        -- printable ascii
-
--- unicode character is decomposed to individual Word8s,
--- and each is shown in octal
-e_utf :: Char -> String
-e_utf c = showoctal . toInteger =<<
-	(Codec.Binary.UTF8.String.encode [c] :: [Word8])
-		
-e_asc :: Char -> String
-e_asc c = showoctal $ toInteger $ ord c
-
-showoctal :: Integer -> String				
-showoctal i = '\\' : printf "%03o" i
+			| p c          = e_asc c -- unprintable ascii
+			| otherwise    = [c]     -- printable ascii
+		-- unicode character is decomposed to individual Word8s,
+		-- and each is shown in octal
+		e_utf c = showoctal =<< (Codec.Binary.UTF8.String.encode [c] :: [Word8])
+		e_asc c = showoctal $ ord c
+		showoctal i = '\\' : printf "%03o" i
 
 {- for quickcheck -}
 prop_idempotent_deencode :: String -> Bool

From 77ffd070adda29cafb4a20076eeda3bba4805e07 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 13:31:34 +0000
Subject: [PATCH 2779/8313] Added a comment

---
 ...comment_1_c0b7682a2b6f3078457b85683c825baf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment

diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment
new file mode 100644
index 0000000000..e627ead47c
--- /dev/null
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh/comment_1_c0b7682a2b6f3078457b85683c825baf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 1"
+ date="2011-12-23T13:31:33Z"
+ content="""
+ControlPersist is awesome - thanks!
+
+Here's [an alternative, git-specific approach](http://thread.gmane.org/gmane.comp.version-control.home-dir/502).
+"""]]

From abba5d3e827b5d31766b95b2e2003aa821f289fc Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 14:04:46 +0000
Subject: [PATCH 2780/8313] Added a comment: I think Matt is right.

---
 ..._646f2077edcabc000a7d9cb75a93cf55._comment | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment b/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment
new file mode 100644
index 0000000000..6ba1796939
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_4_646f2077edcabc000a7d9cb75a93cf55._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="I think Matt is right."
+ date="2011-12-23T14:04:44Z"
+ content="""
+I got bitten by this too.  It seems that the user is expected to fetch
+remote git-annex branches themselves, but this is not documented
+anywhere.
+
+The man page says of \"git annex merge\":
+
+    Automatically merges any changes from remotes into the git-annex
+    branch.
+
+I am not a git newbie, but even so I had incorrectly assumed that git
+annex merge would take care of pulling the git-annex branch from the
+remote prior to merging, thereby ensuring all versions of the
+git-annex branch would be merged, and that the location tracking data
+would be synced across all peer repositories.
+
+My master branches do not track any specific upstream branch, because
+I am operating in a decentralized fashion.  Therefore the error
+message caused by `git pull $remote` succeeded in encouraging me to
+instead use `git pull $remote master`, and this excludes the git-annex
+branch from the fetch.  Even worse, a git newbie might realise this
+and be tempted to do `git pull $remote git-annex`.
+
+Therefore I think it needs to be explicitly documented that
+
+    git fetch $remote
+    git merge $remote/master
+
+is required when the local branch doesn't track an upstream branch.
+Or maybe a `--fetch` option could be added to `git annex merge` to
+perform the fetch from all remotes before running the merge(s).
+"""]]

From 538665f4779825427f7f42d6245e83032459951b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:07:39 +0000
Subject: [PATCH 2781/8313] Added a comment

---
 ...ent_9_aecfa896c97b9448f235bce18a40621d._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment
new file mode 100644
index 0000000000..82c6921ebb
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_9_aecfa896c97b9448f235bce18a40621d._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2011-12-23T16:07:39Z"
+ content="""
+Adam, to answer a lot of points breifly..
+
+* --include='*' makes find list files whether their contents are present or not
+* Your perl script is not O(n). Inserting into perl hash tables has overhead of minimum O(n log n). Not counting the overhead of resizing hash tables, the grevious slowdown if the bucket size is overcome by data (it probably falls back to a linked list or something then), and the overhead of traversing the hash tables to get data out.
+* I think that git-annex's set of file matching options is coming along nicely, and new ones can easily be added, so see no need to pull in unix find(1).
+* Your memory size calculations ignore the overhead of a hash table or other data structure to store the data in, which will tend to be more than the actual data size it's storing.  I estimate your 50 million number is off by at least one order of magnitude, and more likely two; in any case I don't want git-annex to use 1 gb of ram.
+* Little known fact: sort(1) will use a temp file as a buffer if too much memory is needed to hold the data to sort. It's also written in the most efficient language possible and has been ruthlessly optimised for 30 years, so I would be very surprised if it was not the best choice.
+"""]]

From e2b2c67f05945db9bf65bc6cc490244ffa5cf0e8 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:16:19 +0000
Subject: [PATCH 2782/8313] Added a comment

---
 ...comment_1_9c9ab8ce463cf74418aa2f385955f165._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment

diff --git a/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment b/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment
new file mode 100644
index 0000000000..700b3808de
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex/comment_1_9c9ab8ce463cf74418aa2f385955f165._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-23T16:16:19Z"
+ content="""
+From what you say, it seems that vlc is following the symlink to the movie content, and then looking for subtitles next to the file the symlink points to. It would have to explicitly realpath the symlink to have this behavior, and this sounds like a misfeature.. perhaps you could point out to the vlc people the mistake in doing so?
+
+There's a simple use-case where this behavior is obviously wrong, without involving git-annex. Suppose I have a movie, and one version of subtitles for it, in directory `foo`. I want to modify the subtitles, so I make a new directory `bar`, symlink the large movie file from `foo` to save space, and copy over and edit the subtitles from `foo`. Now I run vlc in `bar` to test my new subtitles. If it ignores the locally present subtitles and goes off looking for the ones in `bar`, I say this is broken behavior.
+"""]]

From 0df51acc529c7dfc5beae7e6031d00a6064c64a0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:20:43 +0000
Subject: [PATCH 2783/8313]

---
 doc/todo/link_file_to_remote_repo_feature.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/todo/link_file_to_remote_repo_feature.mdwn b/doc/todo/link_file_to_remote_repo_feature.mdwn
index 4bbd2b1a8f..d6b41e8059 100644
--- a/doc/todo/link_file_to_remote_repo_feature.mdwn
+++ b/doc/todo/link_file_to_remote_repo_feature.mdwn
@@ -46,3 +46,7 @@ The limitation is that it would work only with remote repos on local dirs
 
 Also allows you to have one directory structure like AFS or other distributed FS. If the file is not local I go to the remote server. 
 Which is great for apps like Picasa, Itunes, and friends that depends on the file location.
+
+> This is a duplicate of [[union_mounting]]. So closing it: [[done]]. 
+> 
+> It's a good idea, but making sure git-annex correctly handles these links in all cases is a subtle problem that has not yet been tackled. --[[Joey]]

From 9e6ff9e73425e53a8aac884fd97e3ea8f56bc74c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:48:59 -0400
Subject: [PATCH 2784/8313] improve wording to not imply a pull is done by
 merge

---
 doc/git-annex.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 2d0d2597ea..39fe5d99ea 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -218,10 +218,10 @@ subdirectories).
 
 * merge
 
-  Automatically merges any changes from remotes into the git-annex branch.
-  While git-annex mostly handles keeping the git-annex branch merged
-  automatically, if you find you are unable to push the git-annex branch
-  due non-fast-forward, this will fix it.
+  Automatically merges remote tracking branches */git-annex into
+  the git-annex branch. While git-annex mostly handles keeping the
+  git-annex branch merged automatically, if you find you are unable
+  to push the git-annex branch due non-fast-forward, this will fix it.
 
 * fix [path ...]
 

From 304930f1fc952b8f2dbe019049d5e02d8eb582dc Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 16:50:26 +0000
Subject: [PATCH 2785/8313] Added a comment

---
 ...comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment b/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment
new file mode 100644
index 0000000000..c01f241202
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_5_4f2a05ef6551806dd0ec65372f183ca4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2011-12-23T16:50:26Z"
+ content="""
+My goal for `git-annex merge` is that users should not need to know about it, so it should not be doing expensive pulls.
+
+I hope that `git annex sync` will grow some useful features to support fully distributed git usage, as being discussed in [[pure_git-annex_only_workflow]]. I still use centralized git to avoid these problems myself.
+"""]]

From a0227e81f9c82afc12ac1bd1cecd63cc0894d751 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:55:11 -0400
Subject: [PATCH 2786/8313] put in explicit fetch ; merge in walkthrough for
 now

and link to centralized repository tutorial
---
 doc/walkthrough/getting_file_content.mdwn | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index 71f95f79e0..5ad97bc325 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,14 +6,18 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git pull laptop
+	# git fetch laptop
+	# git merge laptop/master
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok
 
-Notice that you had to git pull from laptop first, this lets git-annex know
-what has changed in laptop, and so it knows about the files present there and
-can get them. The alternate approach is to set up a central bare repository,
-and always push changes to it after committing them, then in the above,
-you can just pull from the central repository to get synced up to all
-repositories.
+Notice that you had to git fetch and merge from laptop first, this lets
+git-annex know what has changed in laptop, and so it knows about the files
+present there and can get them.
+
+The alternate approach is to set up a
+[[central bare repository|tips/centralized_git_repository_tutorial]], and
+always push changes to it after committing them, then in the above,
+you can just pull from the central repository to get synced up to
+all repositories.

From 6fcf76bcd162a31b7099bc6ae624b11015ea52b4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:57:52 -0400
Subject: [PATCH 2787/8313] slight simplification

---
 doc/walkthrough/getting_file_content.mdwn | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index 5ad97bc325..fac6aa2a78 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,8 +6,7 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git fetch laptop
-	# git merge laptop/master
+	# git fetch laptop; git merge laptop
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok

From 3b1c80d7950216ea90f706d975ff663cb92e52c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 12:58:39 -0400
Subject: [PATCH 2788/8313] fix

---
 doc/walkthrough/getting_file_content.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index fac6aa2a78..cdb4a72e0a 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -6,7 +6,7 @@ We can use this to copy everything in the laptop's annex to the
 USB drive.
 
 	# cd /media/usb/annex
-	# git fetch laptop; git merge laptop
+	# git fetch laptop; git merge laptop/master
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok

From fe65981de87cea18c18240b541ac54b36534390e Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 17:14:03 +0000
Subject: [PATCH 2789/8313] Added a comment

---
 .../comment_6_3925d1aa56bce9380f712e238d63080f._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment b/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment
new file mode 100644
index 0000000000..f4b5ebec20
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_6_3925d1aa56bce9380f712e238d63080f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 6"
+ date="2011-12-23T17:14:03Z"
+ content="""
+Extending `git annex sync` would be nice, although auto-commit does not suit every use case, so it would be better not to couple one to the other.
+"""]]

From b05c08b5c1ad0deacec8943128e1eec8362471a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 13:19:28 -0400
Subject: [PATCH 2790/8313] reorder less expensive terminal first

Out of general principles, it did not seem to actually speed it
up appreciably. (I suspect ghc is being smart.)
---
 Command/Find.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 91386bbd08..1961e6b748 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -28,7 +28,7 @@ start :: FilePath -> (Key, Backend Annex) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) (inAnnex key) limited) $
+	whenM (liftM2 (||) limited (inAnnex key)) $
 		unlessM (showFullJSON vars) $ do
 			f <- Annex.getState Annex.format
 			case f of

From d3e80eabe8bc58fa232c49e10f442c94c0107d79 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 17:22:12 +0000
Subject: [PATCH 2791/8313] Added a comment

---
 ..._d78d79fb2f3713aa69f45d2691cf8dfe._comment | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment
new file mode 100644
index 0000000000..5dbb66cf66
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_10_d78d79fb2f3713aa69f45d2691cf8dfe._comment
@@ -0,0 +1,68 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 10"
+ date="2011-12-23T17:22:11Z"
+ content="""
+> Your perl script is not O(n). Inserting into perl hash tables has
+> overhead of minimum O(n log n).
+
+What's your source for this assertion?  I would expect an amortized
+average of `O(1)` per insertion, i.e. `O(n)` for full population.
+
+> Not counting the overhead of resizing hash tables,
+> the grevious slowdown if the bucket size is overcome by data (it
+> probably falls back to a linked list or something then), and the
+> overhead of traversing the hash tables to get data out.
+
+None of which necessarily change the algorithmic complexity.  However
+real benchmarks are far more useful here than complexity analysis, and
+[the dangers of premature optimization](http://c2.com/cgi/wiki?PrematureOptimization) 
+should not be forgotten.
+
+> Your memory size calculations ignore the overhead of a hash table or
+> other data structure to store the data in, which will tend to be
+> more than the actual data size it's storing. I estimate your 50
+> million number is off by at least one order of magnitude, and more
+> likely two;
+
+Sure, I was aware of that, but my point still stands.  Even 500k keys
+per 1GB of RAM does not sound expensive to me.
+
+> in any case I don't want git-annex to use 1 gb of ram.
+
+Why not?  What's the maximum it should use?  512MB?  256MB?
+32MB?  I don't see the sense in the author of a program 
+dictating thresholds which are entirely dependent on the context
+in which the program is *run*, not the context in which it's *written*.
+That's why systems have files such as `/etc/security/limits.conf`.
+
+You said you want git-annex to scale to enormous repositories.  If you
+impose an arbitrary memory restriction such as the above, that means
+avoiding implementing *any* kind of functionality which requires `O(n)`
+memory or worse.  Isn't it reasonable to assume that many users use
+git-annex on repositories which are *not* enormous?  Even when they do
+work with enormous repositories, just like with any other program,
+they would naturally expect certain operations to take longer or
+become impractical without sufficient RAM.  That's why I say that this
+restriction amounts to throwing out the baby with the bathwater.
+It just means that those who need the functionality would have to
+reimplement it themselves, assuming they are able, which is likely
+to result in more wheel reinventions.  I've already shared
+[my implementation](https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups)
+but how many people are likely to find it, let alone get it working?
+
+> Little known fact: sort(1) will use a temp file as a buffer if too
+> much memory is needed to hold the data to sort.
+
+Interesting.  Presumably you are referring to some undocumented
+behaviour, rather than `--batch-size` which only applies when merging
+multiple files, and not when only sorting STDIN.
+
+> It's also written in the most efficient language possible and has
+> been ruthlessly optimised for 30 years, so I would be very surprised
+> if it was not the best choice.
+
+It's the best choice for sorting.  But sorting purely to detect
+duplicates is a dismally bad choice.
+"""]]

From 74e86b6da20ae7d7465a30627e29eef81b3244ce Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 17:24:59 +0000
Subject: [PATCH 2792/8313] Added a comment

---
 .../comment_7_24c45ee981b18bc78325c768242e635d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment b/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment
new file mode 100644
index 0000000000..dad2c0af21
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_7_24c45ee981b18bc78325c768242e635d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="comment 7"
+ date="2011-12-23T17:24:58Z"
+ content="""
+P.S. I see you already [fixed the docs](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=a0227e81f9c82afc12ac1bd1cecd63cc0894d751) - thanks! :)
+"""]]

From d5889f827105d31131a69d0feb2e2574d8be8b9e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 17:52:21 +0000
Subject: [PATCH 2793/8313] Added a comment

---
 .../comment_11_4316d9d74312112dc4c823077af7febe._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment
new file mode 100644
index 0000000000..286487eee5
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_11_4316d9d74312112dc4c823077af7febe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 11"
+ date="2011-12-23T17:52:21Z"
+ content="""
+I don't think that [[tips/finding_duplicate_files]] is hard to find, and the multiple different ways it shows to deal with the duplicate files shows the flexability of the unix pipeline approach.
+"""]]

From cbaf13e587e340debf22aa1de505f4ebb15f583c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 18:02:24 +0000
Subject: [PATCH 2794/8313] Added a comment

---
 .../comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment

diff --git a/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment
new file mode 100644
index 0000000000..909beed837
--- /dev/null
+++ b/doc/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/comment_12_ed6d07f16a11c6eee7e3d5005e8e6fa3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-12-23T18:02:24Z"
+ content="""
+BTW, sort -S '90%' benchmarks consistently 2x as fast as perl's hashes all the way up to 1 million files. Of course the pipeline approach allows you to swap in perl or whatever else is best for you at scale.
+"""]]

From 1940e52793ff20cf1c48f78fef95985434ed7dee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 14:08:04 -0400
Subject: [PATCH 2795/8313] skip repos without a description set when --in=""
 is specified

Picking one of probably several remotes with no description set was not
useful behavior.
---
 Remote.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Remote.hs b/Remote.hs
index aa86934143..10bf9d7694 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -112,6 +112,7 @@ byName' n = do
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID -- special case for current repo
+nameToUUID "" = error "no remote specified"
 nameToUUID n = byName' n >>= go
 	where
 		go (Right r) = return $ uuid r

From 131d560ba66405badf85e2450529d222f15e1e21 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 18:43:05 +0000
Subject: [PATCH 2796/8313] Added a comment

---
 .../comment_2_037f94c1deeac873dbdb36cd4c927e45._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment

diff --git a/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment b/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment
new file mode 100644
index 0000000000..3c69f5fe46
--- /dev/null
+++ b/doc/forum/vlc_and_git-annex/comment_2_037f94c1deeac873dbdb36cd4c927e45._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-23T18:43:05Z"
+ content="""
+Since subtitle files are typically pretty small, a workaround is to simply check them into git directly, and only use git-annex for the movies. (Or `git annex unannex` the ones you've already annexed.)
+"""]]

From bfa5509dbdb3615584023cc6106fbc42937fa7ac Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Fri, 23 Dec 2011 19:16:51 +0000
Subject: [PATCH 2797/8313] Added a comment: Cool

---
 .../comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment

diff --git a/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment b/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment
new file mode 100644
index 0000000000..d1bd4475e5
--- /dev/null
+++ b/doc/tips/finding_duplicate_files/comment_1_ddb477ca242ffeb21e0df394d8fdf5d2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Cool"
+ date="2011-12-23T19:16:50Z"
+ content="""
+Very nice :)  Just for reference, here's [my Perl implementation](https://github.com/aspiers/git-config/blob/master/bin/git-annex-finddups).  As per [this discussion](http://git-annex.branchable.com/todo/wishlist:_Provide_a___34__git_annex__34___command_that_will_skip_duplicates/#comment-fb15d5829a52cd05bcbd5dc53edaffb2) it would be interesting to benchmark these two approaches and see if one is substantially more efficient than the other w.r.t. CPU and memory usage.
+"""]]

From 56e4acfa2450ed94194a3ab5284a81e432e24c1d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 23 Dec 2011 19:19:53 +0000
Subject: [PATCH 2798/8313] Added a comment

---
 .../comment_1_b0d22822017646775869ce1292e676f4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment

diff --git a/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment b/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment
new file mode 100644
index 0000000000..22857af3e8
--- /dev/null
+++ b/doc/tips/centralised_repository:_starting_from_nothing/comment_1_b0d22822017646775869ce1292e676f4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2011-12-23T19:19:53Z"
+ content="""
+See also: [[centralized_git_repository_tutorial]]
+"""]]

From 4fb31142287b05291ba66258477d28670c0d3562 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
 
Date: Fri, 23 Dec 2011 22:04:09 +0000
Subject: [PATCH 2799/8313] Added a comment

---
 ...t_2_49f15478781a0ad5e46e75319070335c._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment
new file mode 100644
index 0000000000..94b5c2ec11
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_2_49f15478781a0ad5e46e75319070335c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmKPMUX0YHBjE93eBsEnacwZsddSDue3PY"
+ nickname="Oliver"
+ subject="comment 2"
+ date="2011-12-23T22:04:08Z"
+ content="""
+As joey points out the problem is B overwrites A, so that any files in A that aren't in B will be removed. But the suggestion to keep B in a separate subdirectory in the repository means I'll end up with duplicates of files in both A and B. What I want is to have the merged superset of all files from both A and B with only one copy of identical files. 
+
+The problem is that unique symlinks in A/master are deleted when B/master is merged in. To add back the deleted files after the merge you can do this:
+
+    git checkout master~1 deleted_file_name                                                              #checkout a single deleted file called deleted_file_name
+    git diff master~1 master --numstat --name-only --diff-filter=D                                       #get the names of all files deleted between master and master~1
+    git diff master~1 master --numstat --name-only --diff-filter=D | xargs git checkout master~1         #checkout all deleted files between master and master~1
+
+Once the first merge has been done after set up, you can continue to make changes to A and B and future merges won't require accounting for deleted files in this way.
+"""]]

From 010780dc5534461afdba6e5c29b7f8bbc8fdf04a Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 00:13:43 +0000
Subject: [PATCH 2800/8313] add a page for openSUSE install instructions

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 6f892e37a2..cc26ee91d5 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -5,6 +5,7 @@
 * [[Ubuntu]]
 * [[Fedora]]
 * [[FreeBSD]]
+* [[openSUSE]]
 
 ## Using cabal
 

From 1202e605d7b036693112e24148f89188d4f61aec Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 00:45:43 +0000
Subject: [PATCH 2801/8313]

---
 doc/install/openSUSE.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/install/openSUSE.mdwn

diff --git a/doc/install/openSUSE.mdwn b/doc/install/openSUSE.mdwn
new file mode 100644
index 0000000000..762db1f2d2
--- /dev/null
+++ b/doc/install/openSUSE.mdwn
@@ -0,0 +1,4 @@
+Unfortunately there is currently no git-annex rpm available for openSUSE; however it is possible to build it via cabal or from source as described on the [[install]] page.  Fulfilling the dependencies listed on that page should not be a problem, except for obtaining a suitable version of the Haskell library.
+
+The last [official release of Haskell for openSUSE](https://build.opensuse.org/project/show?project=devel:languages:haskell) is quite old, and may not satisfy the dependencies needed by git-annex.  Fortunately [searching the openSUSE build service](http://software.opensuse.org/search?q=cabal&baseproject=openSUSE%3A11.4&lang=en&include_home=true&exclude_debug=true) reveals that Peter Trommler has built a [newer Haskell suite](https://build.opensuse.org/project/show?project=home%3Aptrommler%3Adevel%3Alanguages%3Ahaskell) based on ghc 7.2.
+To install this, simply visit the previous URL and click on the relevant "1-Click Install" link.

From f78b0cb4e77be51344b2eb6d1f29b62192748e7b Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 00:48:37 +0000
Subject: [PATCH 2802/8313]

---
 doc/install/openSUSE.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/openSUSE.mdwn b/doc/install/openSUSE.mdwn
index 762db1f2d2..0383cbbf2a 100644
--- a/doc/install/openSUSE.mdwn
+++ b/doc/install/openSUSE.mdwn
@@ -1,4 +1,4 @@
 Unfortunately there is currently no git-annex rpm available for openSUSE; however it is possible to build it via cabal or from source as described on the [[install]] page.  Fulfilling the dependencies listed on that page should not be a problem, except for obtaining a suitable version of the Haskell library.
 
 The last [official release of Haskell for openSUSE](https://build.opensuse.org/project/show?project=devel:languages:haskell) is quite old, and may not satisfy the dependencies needed by git-annex.  Fortunately [searching the openSUSE build service](http://software.opensuse.org/search?q=cabal&baseproject=openSUSE%3A11.4&lang=en&include_home=true&exclude_debug=true) reveals that Peter Trommler has built a [newer Haskell suite](https://build.opensuse.org/project/show?project=home%3Aptrommler%3Adevel%3Alanguages%3Ahaskell) based on ghc 7.2.
-To install this, simply visit the previous URL and click on the relevant "1-Click Install" link.
+To install this, simply click on the relevant "1-Click Install" link in the openSUSE build service search results.

From 4eafd43cce00942625b24adb50917e9074ae7e4b Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 24 Dec 2011 01:05:08 +0000
Subject: [PATCH 2803/8313] Added a comment: Any update on this?

---
 ..._1_3c7e3f021c2c94277eecf9c8af6cec5f._comment | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment b/doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment
new file mode 100644
index 0000000000..692b5d5378
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_1_3c7e3f021c2c94277eecf9c8af6cec5f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="Any update on this?"
+ date="2011-12-24T01:05:07Z"
+ content="""
+I just noticed this issue, and was wondering what the current status is.
+
+    % ls -l 04\ -\ Orixás.mp3
+    -rw-r--r-- 1 adam users 8377816 Jul 12  2007 04 - Orixás.mp3
+    % echo 04\ -\ Orixás.mp3 | od -c
+    0000000   0   4       -       O   r   i   x 303 241   s   .   m   p   3
+    0000020  \n
+    0000021
+    % git annex add 04\ -\ Orixás.mp3
+    git-annex: /home/adam/music/RotC/transcribe/04 - Orixás.mp3: getSymbolicLinkStatus: does not exist (No such file or directory)
+"""]]

From ccddaf2f5c024be02efd44494ed11d09b2102872 Mon Sep 17 00:00:00 2001
From: "http://www.openid.albertlash.com/openid/"
 
Date: Sat, 24 Dec 2011 06:08:46 +0000
Subject: [PATCH 2804/8313] Added a comment

---
 ...nt_1_9a2a2a8eac9af97e0c984ad105763a73._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment b/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment
new file mode 100644
index 0000000000..807180660b
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_1_9a2a2a8eac9af97e0c984ad105763a73._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://www.openid.albertlash.com/openid/"
+ ip="71.178.29.218"
+ subject="comment 1"
+ date="2011-12-24T06:08:45Z"
+ content="""
+Looks like you are missing a closing double quote on the line:
+
+
+echo '$GL_ADC_PATH = \"/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+
+right after /;
+
+I got this working by the way - great stuff.
+"""]]

From ffe9973ed5163e88d93b61ecc3496e11e6a0084e Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sat, 24 Dec 2011 12:49:41 +0000
Subject: [PATCH 2805/8313] Added a comment

---
 ...comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment b/doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment
new file mode 100644
index 0000000000..a45706e4a3
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_2_bad4c4c5f54358d1bc0ab2adc713782a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2011-12-24T12:49:40Z"
+ content="""
+This (rather longish) thread discusses the current situation, the planned changes for 7.2 and the various issues: http://haskell.org/pipermail/glasgow-haskell-users/2011-November/021115.html
+
+The summary seems to be: From 7.2 on, getDirectoryContents _will_ return proper Strings, i.e. where a Char represents a Unicode code point, and not a Word8, which will fix the problem of outputting them.
+"""]]

From ab1435310c36ba3309c075b544bc8b3bf8e7ffa8 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Sat, 24 Dec 2011 12:51:43 +0000
Subject: [PATCH 2806/8313] Added a comment

---
 .../comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment b/doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment
new file mode 100644
index 0000000000..9fef2eb1f2
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_3_4f936a5d3f9c7df64c8a87e62b7fbfdc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 3"
+ date="2011-12-24T12:51:43Z"
+ content="""
+An alternative that is available from ghc 7.4 on is a pure ByteString based unix API: http://thread.gmane.org/gmane.comp.lang.haskell.libraries/16556
+"""]]

From c87a81404656da4f5d3607687c3cb5da32295af3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 24 Dec 2011 16:49:13 +0000
Subject: [PATCH 2807/8313] Added a comment

---
 ...comment_4_93bee35f5fa7744834994bc7a253a6f9._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment b/doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment
new file mode 100644
index 0000000000..5e11af6abe
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_4_93bee35f5fa7744834994bc7a253a6f9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-12-24T16:49:13Z"
+ content="""
+Adam, this bug was fixed a long time ago, first using option #2 above, but later switching to option #3 -- git-annex treats filenames as opaque binary blobs and never decodes them in any encoding; haskell's normal encoding support for stdio is disabled. 
+
+And it never resulted in a failure like you show. I cannot reproduce your problem, but it is a different bug, please open a new bug report.
+"""]]

From 0c107c98de1c3ffa9dea3d9ccc3029984a3d2ffa Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 24 Dec 2011 16:54:31 +0000
Subject: [PATCH 2808/8313] Added a comment

---
 .../comment_2_d8efea4ab9576555fadbb47666ecefa9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment b/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment
new file mode 100644
index 0000000000..007a009ea1
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_2_d8efea4ab9576555fadbb47666ecefa9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2011-12-24T16:54:31Z"
+ content="""
+I've fixed the typo (anyone can edit pages in this wiki FWIW.)
+"""]]

From 82cc5d3d3120a00cd7f823794f2242fcd3b42817 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Dec 2011 14:14:00 -0400
Subject: [PATCH 2809/8313] slightly improve typography

---
 doc/git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 39fe5d99ea..8096005ce2 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -247,7 +247,7 @@ subdirectories).
   By default, only lists annexed files whose content is currently present.
   This can be changed by specifying file matching options. To list all
   annexed files, present or not, specify --include "*". To list all
-  annexed files whose content is not present, specify --not --in "."
+  annexed files whose content is not present, specify --not --in="."
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0. Or, a custom output formatting can be specified using
@@ -454,7 +454,7 @@ limit which files git-annex acts on.
 Arbitrarily complicated expressions can be built using these options.
 For example:
 
-	--exclude '*.mp3' --and --not -( --in usbdrive --or --in archive -)
+	--exclude '*.mp3' --and --not -( --in=usbdrive --or --in=archive -)
 
 The above example prevents git-annex from working on mp3 files whose
 file contents are present at either of two repositories.

From ffefe742d8cc2655e29fa356c82326e867e27aa9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Dec 2011 12:51:03 -0400
Subject: [PATCH 2810/8313] fix typo

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index a51d6539b6..ac0a1f6b6c 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -21,7 +21,7 @@ gitolite installation, just with different paths.
 Set `$GL_ADC_PATH` in `.gitolite.rc`, if you have not already done so.
 
 
-echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;' >>~gitolite/.gitolite.rc
+echo '$GL_ADC_PATH = "/usr/local/lib/gitolite/adc/;"' >>~gitolite/.gitolite.rc
 

 
 Make the ADC directory, and a "ua" subdirectory.

From 899b20f89c1ba7d6ac1a09ea83c7f37052c43166 Mon Sep 17 00:00:00 2001
From: "http://www.openid.albertlash.com/openid/"
 
Date: Sat, 24 Dec 2011 17:05:38 +0000
Subject: [PATCH 2811/8313]

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index ac0a1f6b6c..92048c4429 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -75,3 +75,15 @@ sent 2606 bytes  received 31 bytes  1758.00 bytes/sec
 total size is 2502  speedup is 0.95
 ok
 

+
+
+### Troubleshooting
+
+I got an error like this when setting up gitolite *after* setting up a local git repo and git annex:
+
+
+git-annex-shell: First run: git-annex init
+Command ssh ["git@git.example.com","git-annex-shell 'configlist' '/~/myrepo.git'"] failed; exit code 1
+

+
+because I forgot to "git pull --all" after adding the new gitolite remote.

From 2101ef20d152891176401447f76758935313d7de Mon Sep 17 00:00:00 2001
From: "http://www.openid.albertlash.com/openid/"
 
Date: Sun, 25 Dec 2011 01:55:45 +0000
Subject: [PATCH 2812/8313]

---
 doc/tips/using_gitolite_with_git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_gitolite_with_git-annex.mdwn b/doc/tips/using_gitolite_with_git-annex.mdwn
index 92048c4429..0d89a255be 100644
--- a/doc/tips/using_gitolite_with_git-annex.mdwn
+++ b/doc/tips/using_gitolite_with_git-annex.mdwn
@@ -86,4 +86,4 @@ git-annex-shell: First run: git-annex init
 Command ssh ["git@git.example.com","git-annex-shell 'configlist' '/~/myrepo.git'"] failed; exit code 1
 

 
-because I forgot to "git pull --all" after adding the new gitolite remote.
+because I forgot to "git push --all" after adding the new gitolite remote.

From 85f1f3a63a025204e931959ec9c5ae2a7812ddff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Dec 2011 23:05:23 -0400
Subject: [PATCH 2813/8313] Updated to build with monad-control 0.3.

---
 debian/changelog                                         | 5 +++--
 debian/control                                           | 2 +-
 doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn | 2 ++
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 33d196fa32..61b10c80f3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,11 +8,12 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Test suite improvements. Current top-level test coverage: 75%
   * Improve deletion of files from rsync special remotes. Closes: #652849
   * Add --include, which is the same as --not --exclude.
-  * Can now be built with older git versions (before 1.7.7); the resulting
-    binary should only be used with old git.
   * Format strings can be specified using the new --format option, to control
     what is output by git annex find.
   * Support git annex find --json
+  * Can now be built with older git versions (before 1.7.7); the resulting
+    binary should only be used with old git.
+  * Updated to build with monad-control 0.3.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/debian/control b/debian/control
index 6f59ada5b8..a3035e8807 100644
--- a/debian/control
+++ b/debian/control
@@ -13,7 +13,7 @@ Build-Depends:
 	libghc-utf8-string-dev,
 	libghc-hs3-dev (>= 0.5.6),
 	libghc-testpack-dev [any-i386 any-amd64],
-	libghc-monad-control-dev,
+	libghc-monad-control-dev (>= 0.3),
 	libghc-json-dev,
 	ikiwiki,
 	perlmagick,
diff --git a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
index ca68c2c913..f822249916 100644
--- a/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
+++ b/doc/todo/Please_add_support_for_monad-control_0.3.x.mdwn
@@ -5,3 +5,5 @@ Git-annex doesn't compile with the latest version of monad-control. Would it be
 > 
 > There is now a branch in git called `new-monad-control` that will build
 > with the new monad-control. --[[Joey]]
+
+>> Now merged to master. [[done]] --[[Joey]]

From 6068cef0388a3a16d2cfe0f7b19882adfde0b9c5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 26 Dec 2011 18:50:36 +0000
Subject: [PATCH 2814/8313] Added a comment: git tweak-fetch

---
 ...mment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment

diff --git a/doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment b/doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment
new file mode 100644
index 0000000000..5943d93127
--- /dev/null
+++ b/doc/forum/git_pull_remote_git-annex/comment_8_7e76ee9b6520cbffaf484c9299a63ad3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="git tweak-fetch"
+ date="2011-12-26T18:50:35Z"
+ content="""
+The git tweak-fetch hook that I have been developing, and hope will be accepted into git soon, provides some abilities that could be used to make \"git pull remote\" always merge remote/master. Normall, git can only be configured to do that merge automatically for one remote (ie, origin). But the tweak-fetch hook can flag arbitrary branches as needing merge. 
+
+So, it could always flag tracking branches of the currently checked out branch for merge. This would be enabled by some setting, probably, since it's not necessarily the case that everyone wants to auto-merge when they pull like this. (Which is why git doesn't do it by default after all.)
+
+(The tweak-fetch hook will also entirely eliminate the need to run git annex merge manually, since it can always take care of merging the git-annex branch.)
+"""]]

From 559bbdb4242d6eadf0e3eb24ca6fc81c08865383 Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Tue, 27 Dec 2011 19:23:41 +0100
Subject: [PATCH 2815/8313] Outline extended git annex sync semantics in the
 documentation

---
 doc/git-annex.mdwn | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 39fe5d99ea..2c0e23e9ce 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -122,13 +122,18 @@ subdirectories).
 
 * sync
 
-  Use this command when you want to synchronize the local repository
-  with its default remote (typically "origin"). The sync process involves
-  first committing all local changes, then pulling and merging any changes
-  from the remote, and finally pushing the repository's state to the remote.
+  Use this command when you want to synchronize the local repository with
+  one or more other repositories. The sync process involves first committing
+  all local changes, then fetching and merging the `synced/master` and the
+  `git-annex` branch from the remote repositories and finally pushing the
+  changes back to these remote branches.
   You can use standard git commands to do each of those steps by hand,
   or if you don't want to worry about the details, you can use sync.
 
+  By default, `git annex sync` will sync all remote repositories that have a
+  `synced/master` branch. If you want to include/exclude a repository from
+  this list, just create or delete this branch.
+
   Note that sync does not transfer any file contents from or to the remote.
 
 * addurl [url ...]

From 0ee1141f30b188e5ef52125c163ff2cf80c661ee Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Thu, 29 Dec 2011 18:37:30 +0100
Subject: [PATCH 2816/8313] Implement branch-syncing in Command.Sync

as described in the previous commit to the documentation. The loggin UI
is not great yet.
---
 Command/Sync.hs | 113 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 81 insertions(+), 32 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 36c4eeef06..a9089463d4 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -9,20 +9,41 @@ module Command.Sync where
 
 import Common.Annex
 import Command
+import qualified Remote
 import qualified Annex.Branch
 import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Ref
 import qualified Git
+import qualified Command.Merge
 
 import qualified Data.ByteString.Lazy.Char8 as L
 
 def :: [Command]
-def = [command "sync" paramPaths seek "synchronize local repository with remote"]
+def = [command "sync" (paramOptional (paramRepeating paramRemote))
+               [seek] "synchronize local repository with remote repositories"]
 
 -- syncing involves several operations, any of which can independantly fail
-seek :: [CommandSeek]
-seek = map withNothing [commit, pull, push]
+seek :: CommandSeek
+seek args = do
+    remotes <- if null args
+               then defaultSyncRemotes
+               else mapM Remote.byName args
+    branch <- currentBranch
+    showStart "syncing" $ "branch " ++ Git.Ref.describe branch ++ " with remote repositories " ++ intercalate "," (map Remote.name remotes)
+    showOutput
+    return $
+        [ commit
+        , mergeLocal branch
+        ] ++
+        [ fetch remote | remote <- remotes ] ++
+        [ mergeRemote remote branch | remote <- remotes ] ++
+        [ Command.Merge.start ] ++
+        [ pushLocal branch ] ++
+        [ pushRemote remote branch | remote <- remotes ]
+
+defaultSyncRemotes :: Annex [Remote.Remote Annex]
+defaultSyncRemotes = undefined
 
 commit :: CommandStart
 commit = do
@@ -31,44 +52,72 @@ commit = do
 		showOutput
 		-- Commit will fail when the tree is clean, so ignore failure.
 		_ <- inRepo $ Git.Command.runBool "commit"
-			[Param "-a", Param "-m", Param "sync"]
+			[Param "-a", Param "-m", Param "git-annex automatic sync"]
 		return True
 
-pull :: CommandStart
-pull = do
-	remote <- defaultRemote
-	showStart "pull" remote
+mergeLocal :: Git.Ref -> CommandStart
+mergeLocal branch =
+    mergeFromIfExists $ Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+
+pushLocal :: Git.Ref -> CommandStart
+pushLocal branch = do
+    let syncBranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+    ex <- inRepo $ Git.Ref.exists syncBranch
+    if ex then do
+        showStart "updateing" $
+            Git.Ref.describe syncBranch ++
+            " to the state of " ++ Git.Ref.describe branch ++ "..."
+        next $ next $
+            inRepo $ Git.Command.runBool "branch" [Param "-f", Param (Git.Ref.describe syncBranch)]
+      else
+        return Nothing
+
+mergeFromIfExists :: Git.Ref -> CommandStart
+mergeFromIfExists fromBranch = do
+    ex <- inRepo $ Git.Ref.exists fromBranch
+    if ex then do
+        showStart "merging" $ Git.Ref.describe fromBranch ++ "..."
+        next $ next $
+            inRepo $ Git.Command.runBool "merge" [Param (show fromBranch)]
+      else do
+        showNote $ Git.Ref.describe fromBranch ++ " does not exist, not merging."
+        showOutput
+        return Nothing
+
+
+fetch :: Remote.Remote Annex -> CommandStart
+fetch remote = do
+	showStart "fetching from" (Remote.name remote)
 	next $ next $ do
 		showOutput
 		checkRemote remote
-		inRepo $ Git.Command.runBool "pull" [Param remote]
+		inRepo $ Git.Command.runBool "fetch" [Param (Remote.name remote)]
 
-push :: CommandStart
-push = do
-	remote <- defaultRemote
-	showStart "push" remote
-	next $ next $ do
-		Annex.Branch.update
-		showOutput
-		inRepo $ Git.Command.runBool "push" [Param remote, matchingbranches]
-	where
-		-- git push may be configured to not push matching
-		-- branches; this should ensure it always does.
-		matchingbranches = Param ":"
+mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+mergeRemote remote branch =
+    mergeFromIfExists $ Git.Ref $ "refs/remotes/" ++ Remote.name remote ++ "/synced/" ++ Git.Ref.describe branch
 
--- the remote defaults to origin when not configured
-defaultRemote :: Annex String
-defaultRemote = do
-	branch <- currentBranch
-	fromRepo $ Git.Config.get ("branch." ++ branch ++ ".remote") "origin"
+pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+pushRemote remote branch = do
+    showStart "pushing to" (Remote.name remote)
+    let syncbranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+    let syncbranchRemote = Git.Ref $ "refs/remotes/" ++ Remote.name remote ++ "/" ++ Git.Ref.describe syncbranch
+    let refspec = Git.Ref.describe branch ++ ":" ++ Git.Ref.describe syncbranch
+    ex <- inRepo $ Git.Ref.exists syncbranchRemote
+    next $ next $ do
+            showOutput
+            inRepo $ Git.Command.runBool "push" $
+                [ Param (Remote.name remote)
+                , Param (Git.Ref.describe Annex.Branch.name) ] ++ 
+                [ Param refspec | ex ]
 
-currentBranch :: Annex String
-currentBranch = Git.Ref.describe . Git.Ref . firstLine . L.unpack <$>
+currentBranch :: Annex Git.Ref
+currentBranch = Git.Ref . firstLine . L.unpack <$>
 	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
 
-checkRemote :: String -> Annex ()
+checkRemote :: Remote.Remote Annex -> Annex ()
 checkRemote remote = do
 	remoteurl <- fromRepo $
-		Git.Config.get ("remote." ++ remote ++ ".url") ""
-	when (null remoteurl) $ do
-		error $ "No url is configured for the remote: " ++ remote
+		Git.Config.get ("remote." ++ Remote.name remote ++ ".url") ""
+	when (null remoteurl) $
+		error $ "No url is configured for the remote: " ++ Remote.name remote

From b6e7b40be4721595bafd3b1ea9c439cead07b7ff Mon Sep 17 00:00:00 2001
From: Joachim Breitner 
Date: Thu, 29 Dec 2011 20:50:57 +0100
Subject: [PATCH 2817/8313] By default, sync with all remotes having the
 synced/ branch

---
 Command/Sync.hs | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index a9089463d4..cc88188892 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -1,3 +1,4 @@
+{-# LANGUAGE BangPatterns #-}
 {- git-annex command
  -
  - Copyright 2011 Joey Hess 
@@ -26,10 +27,10 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote))
 -- syncing involves several operations, any of which can independantly fail
 seek :: CommandSeek
 seek args = do
+    !branch <- currentBranch
     remotes <- if null args
-               then defaultSyncRemotes
+               then defaultSyncRemotes branch
                else mapM Remote.byName args
-    branch <- currentBranch
     showStart "syncing" $ "branch " ++ Git.Ref.describe branch ++ " with remote repositories " ++ intercalate "," (map Remote.name remotes)
     showOutput
     return $
@@ -42,8 +43,16 @@ seek args = do
         [ pushLocal branch ] ++
         [ pushRemote remote branch | remote <- remotes ]
 
-defaultSyncRemotes :: Annex [Remote.Remote Annex]
-defaultSyncRemotes = undefined
+defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
+defaultSyncRemotes branch = mapM Remote.byName =<< process . L.unpack <$> inRepo showref
+    where 
+        syncbranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+        showref = Git.Command.pipeRead
+            [Param "show-ref", Param (Git.Ref.describe syncbranch)]
+        process = map getRemoteName . filter isRemote . map getBranchName . lines
+        isRemote r = "refs/remotes/" `isPrefixOf` r
+        getBranchName = snd . separate (== ' ')
+        getRemoteName = fst . separate (== '/') . snd . separate (== '/') . snd . separate (== '/')
 
 commit :: CommandStart
 commit = do

From 3dffcf9ccb4eebba48e904b9fb5bfe5d8446b761 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Thu, 29 Dec 2011 19:58:31 +0000
Subject: [PATCH 2818/8313] Added a comment: Finally some code

---
 .../comment_10_683768c9826b0bf0f267e8734b9eb872._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment b/doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment
new file mode 100644
index 0000000000..dc499cbc90
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_10_683768c9826b0bf0f267e8734b9eb872._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="Finally some code"
+ date="2011-12-29T19:58:31Z"
+ content="""
+The repository at http://git.nomeata.de/?p=git-annex.git;a=summary contains changes to Commands/Sync.hs (and to the manpage) that implements this behavior. The functionality should be fine; the progress output is not very nice yet, but I’m not sure if I really understood the various Command types. It also should be more easily discoverable how to activate the behavior (by running \"git branch synced/master\") by providing a helpful message, at least unless git annex init creates the branch by default.
+"""]]

From 5287d1dc3f293b6eb7f6759fe9f25be1ad85fbae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 14:07:17 -0400
Subject: [PATCH 2819/8313] fixed behavior when multiple insteadOf configs are
 provided for the same url base

Consider this git config --list case:

url.git+ssh://git@example.com/.insteadOf=gl
url.git+ssh://git@example.com/.insteadOf=shared

Since config is stored in a Map, only the last of the values for this key
was stored and available for use by the insteadOf code. But that
is wrong; git allows either "gl" or "shared" to be used in an url and
the insteadOf value to be substituted in.

To support this, it seems best to keep the existing config map as-is,
and add a second map that accumulates a list of multiple values for
config keys. This new fullconfig map can be used in the rare places where
multiple values for a key make sense, without needing to complicate
everything else.

Haskell's laziness and data sharing keep the overhead of adding
this second map low.
---
 Git/Config.hs    | 11 ++++++++---
 Git/Construct.hs |  6 +++++-
 Git/Types.hs     |  2 ++
 debian/changelog |  2 ++
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 7b72eba5a3..b2587aa44c 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -42,13 +42,17 @@ hRead repo h = do
  - can be updated inrementally. -}
 store :: String -> Repo -> IO Repo
 store s repo = do
-	let repo' = repo { config = parse s `M.union` config repo }
+	let c = parse s
+	let repo' = repo
+		{ config = (M.map Prelude.head c) `M.union` config repo
+		, fullconfig = M.unionWith (++) c (fullconfig repo)
+		}
 	rs <- Git.Construct.fromRemotes repo'
 	return $ repo' { remotes = rs }
 
 {- Parses git config --list or git config --null --list output into a
  - config map. -}
-parse :: String -> M.Map String String
+parse :: String -> M.Map String [String]
 parse [] = M.empty
 parse s
 	-- --list output will have an = in the first line
@@ -57,4 +61,5 @@ parse s
 	| otherwise = sep '\n' $ split "\0" s
 	where
 		ls = lines s
-		sep c = M.fromList . map (separate (== c))
+		sep c = M.fromListWith (++) . map (\(k,v) -> (k, [v])) .
+			map (separate (== c))
diff --git a/Git/Construct.hs b/Git/Construct.hs
index a35a87cc77..2cc965b4ee 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -115,7 +115,6 @@ remoteNamedFromKey k = remoteNamed basename
 fromRemoteLocation :: String -> Repo -> IO Repo
 fromRemoteLocation s repo = gen $ calcloc s
 	where
-		filterconfig f = filter f $ M.toList $ config repo
 		gen v	
 			| scpstyle v = fromUrl $ scptourl v
 			| isURI v = fromUrl v
@@ -133,6 +132,10 @@ fromRemoteLocation s repo = gen $ calcloc s
 					startswith prefix k &&
 					endswith suffix k &&
 					startswith v l
+				filterconfig f = filter f $
+					concatMap splitconfigs $
+						M.toList $ fullconfig repo
+				splitconfigs (k, vs) = map (\v -> (k, v)) vs
 				(prefix, suffix) = ("url." , ".insteadof")
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
@@ -210,6 +213,7 @@ newFrom l =
 	Repo {
 		location = l,
 		config = M.empty,
+		fullconfig = M.empty,
 		remotes = [],
 		remoteName = Nothing
 	}
diff --git a/Git/Types.hs b/Git/Types.hs
index 250da5f5e5..6063ad213f 100644
--- a/Git/Types.hs
+++ b/Git/Types.hs
@@ -18,6 +18,8 @@ data RepoLocation = Dir FilePath | Url URI | Unknown
 data Repo = Repo {
 	location :: RepoLocation,
 	config :: M.Map String String,
+	-- a given git config key can actually have multiple values
+	fullconfig :: M.Map String [String],
 	remotes :: [Repo],
 	-- remoteName holds the name used for this repo in remotes
 	remoteName :: Maybe String 
diff --git a/debian/changelog b/debian/changelog
index 33d196fa32..bbed54765a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -13,6 +13,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Format strings can be specified using the new --format option, to control
     what is output by git annex find.
   * Support git annex find --json
+  * Fixed behavior when multiple insteadOf configs are provided for the
+    same url base.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From 925b6390aa27f6b22935c6e2b214adf35889f0b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 15:57:28 -0400
Subject: [PATCH 2820/8313] add forceUpdate

This code is picked from my tweak-fetch branch, which already did the needed
refactoring.
---
 Annex/Branch.hs | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index af1878479a..8c9d1582ae 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -9,8 +9,11 @@ module Annex.Branch (
 	name,
 	hasOrigin,
 	hasSibling,
+	siblingBranches,
 	create,
 	update,
+	forceUpdate,
+	updateTo,
 	get,
 	change,
 	commit,
@@ -81,10 +84,23 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
 
 {- Ensures that the branch and index are is up-to-date; should be
  - called before data is read from it. Runs only once per git-annex run.
+ -}
+update :: Annex ()
+update = runUpdateOnce $ updateTo =<< siblingBranches
+
+{- Forces an update even if one has already been run. -}
+forceUpdate :: Annex ()
+forceUpdate = updateTo =<< siblingBranches
+
+{- Merges the specified Refs into the index, if they have any changes not
+ - already in it. The Branch names are only used in the commit message;
+ - it's even possible that the provided Branches have not been updated to
+ - point to the Refs yet.
  -
  - Before refs are merged into the index, it's important to first stage the
  - journal into the index. Otherwise, any changes in the journal would
  - later get staged, and might overwrite changes made during the merge.
+ - If no Refs are provided, the journal is still staged and committed.
  -
  - (It would be cleaner to handle the merge by updating the journal, not the
  - index, with changes from the branches.)
@@ -92,13 +108,13 @@ getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
  - The branch is fast-forwarded if possible, otherwise a merge commit is
  - made.
  -}
-update :: Annex ()
-update = runUpdateOnce $ do
+updateTo :: [(Git.Ref, Git.Branch)] -> Annex ()
+updateTo pairs = do
 	-- ensure branch exists, and get its current ref
 	branchref <- getBranch
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
-	(refs, branches) <- unzip <$> newerSiblings
+	(refs, branches) <- unzip <$> filterM isnewer pairs
 	if (not dirty && null refs)
 		then updateIndex branchref
 		else withIndex $ lockJournal $ do
@@ -110,7 +126,7 @@ update = runUpdateOnce $ do
 					" into " ++ show name
 			unless (null branches) $ do
 				showSideAction merge_desc
-				mergeIndex branches
+				mergeIndex refs
 			ff <- if dirty
 				then return False
 				else inRepo $ Git.Branch.fastForward fullname refs
@@ -120,8 +136,7 @@ update = runUpdateOnce $ do
 					(nub $ fullname:refs)
 			invalidateCache
 	where
-		newerSiblings = filterM isnewer =<< siblingBranches
-		isnewer (_, b) = inRepo $ Git.Branch.changed fullname b
+		isnewer (r, _) = inRepo $ Git.Branch.changed fullname r
 
 {- Gets the content of a file on the branch, or content from the journal, or
  - staged in the index.
@@ -238,7 +253,7 @@ genIndex :: Git.Repo -> IO ()
 genIndex g = Git.UnionMerge.stream_update_index g
 	[Git.UnionMerge.ls_tree fullname g]
 
-{- Merges the specified branches into the index.
+{- Merges the specified refs into the index.
  - Any changes staged in the index will be preserved. -}
 mergeIndex :: [Git.Ref] -> Annex ()
 mergeIndex branches = do

From 5728bb58e0c48b892998715ffb6061a1d0c8a354 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 16:03:41 -0400
Subject: [PATCH 2821/8313] force git-annex branch update after fetching
 remotes

git-annex normally only runs the branch update once per run, for speed, but
since this fetches new remote git-annex tracking branches, they need to be
merged in after that fetch. An earlier call to Remote.byName was causing
the update to run before the fetch sometimes, but it could have been
anything. Just force the update to happen in the right place.
---
 Command/Sync.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index cc88188892..c3d5a26364 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -16,7 +16,6 @@ import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Ref
 import qualified Git
-import qualified Command.Merge
 
 import qualified Data.ByteString.Lazy.Char8 as L
 
@@ -39,7 +38,7 @@ seek args = do
         ] ++
         [ fetch remote | remote <- remotes ] ++
         [ mergeRemote remote branch | remote <- remotes ] ++
-        [ Command.Merge.start ] ++
+        [ mergeAnnex ] ++
         [ pushLocal branch ] ++
         [ pushRemote remote branch | remote <- remotes ]
 
@@ -130,3 +129,8 @@ checkRemote remote = do
 		Git.Config.get ("remote." ++ Remote.name remote ++ ".url") ""
 	when (null remoteurl) $
 		error $ "No url is configured for the remote: " ++ Remote.name remote
+
+mergeAnnex :: CommandStart
+mergeAnnex = next $ next $ do
+	Annex.Branch.forceUpdate
+	return True

From 5d17da5eb321d08e801e31c67f1bd9d748cc593d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 16:24:30 -0400
Subject: [PATCH 2822/8313] update to my indentation style

---
 Command/Sync.hs | 132 ++++++++++++++++++++++++++----------------------
 1 file changed, 71 insertions(+), 61 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index c3d5a26364..d519943432 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -1,11 +1,13 @@
-{-# LANGUAGE BangPatterns #-}
 {- git-annex command
  -
  - Copyright 2011 Joey Hess 
+ - Copyright 2011 Joachim Breitner 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Sync where
 
 import Common.Annex
@@ -21,37 +23,42 @@ import qualified Data.ByteString.Lazy.Char8 as L
 
 def :: [Command]
 def = [command "sync" (paramOptional (paramRepeating paramRemote))
-               [seek] "synchronize local repository with remote repositories"]
+	[seek] "synchronize local repository with remote repositories"]
 
 -- syncing involves several operations, any of which can independantly fail
 seek :: CommandSeek
 seek args = do
-    !branch <- currentBranch
-    remotes <- if null args
-               then defaultSyncRemotes branch
-               else mapM Remote.byName args
-    showStart "syncing" $ "branch " ++ Git.Ref.describe branch ++ " with remote repositories " ++ intercalate "," (map Remote.name remotes)
-    showOutput
-    return $
-        [ commit
-        , mergeLocal branch
-        ] ++
-        [ fetch remote | remote <- remotes ] ++
-        [ mergeRemote remote branch | remote <- remotes ] ++
-        [ mergeAnnex ] ++
-        [ pushLocal branch ] ++
-        [ pushRemote remote branch | remote <- remotes ]
+	!branch <- currentBranch
+	remotes <- syncRemotes branch args
+	showStart "syncing" $ "branch " ++ Git.Ref.describe branch ++ " with remote repositories " ++ intercalate "," (map Remote.name remotes)
+	showOutput
+	return $
+		[ commit
+		, mergeLocal branch
+		] ++
+		[ fetch remote | remote <- remotes ] ++
+		[ mergeRemote remote branch | remote <- remotes ] ++
+		[ mergeAnnex ] ++
+		[ pushLocal branch ] ++
+		[ pushRemote remote branch | remote <- remotes ]
+
+syncRemotes :: Git.Ref -> [String] -> Annex [Remote.Remote Annex]
+syncRemotes branch [] = defaultSyncRemotes branch
+syncRemotes _ rs = mapM Remote.byName rs
 
 defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
-defaultSyncRemotes branch = mapM Remote.byName =<< process . L.unpack <$> inRepo showref
-    where 
-        syncbranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
-        showref = Git.Command.pipeRead
-            [Param "show-ref", Param (Git.Ref.describe syncbranch)]
-        process = map getRemoteName . filter isRemote . map getBranchName . lines
-        isRemote r = "refs/remotes/" `isPrefixOf` r
-        getBranchName = snd . separate (== ' ')
-        getRemoteName = fst . separate (== '/') . snd . separate (== '/') . snd . separate (== '/')
+defaultSyncRemotes branch = 
+	mapM Remote.byName
+		=<< process . L.unpack <$> inRepo showref
+	where 
+		syncbranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+		showref = Git.Command.pipeRead
+			[Param "show-ref", Param (Git.Ref.describe syncbranch)]
+		process = map getRemoteName . filter isRemote .
+			map getBranchName . lines
+	        isRemote r = "refs/remotes/" `isPrefixOf` r
+		getBranchName = snd . separate (== ' ')
+		getRemoteName = fst . separate (== '/') . snd . separate (== '/') . snd . separate (== '/')
 
 commit :: CommandStart
 commit = do
@@ -64,34 +71,34 @@ commit = do
 		return True
 
 mergeLocal :: Git.Ref -> CommandStart
-mergeLocal branch =
-    mergeFromIfExists $ Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+mergeLocal branch = mergeFromIfExists $ Git.Ref $
+	"refs/heads/synced/" ++ Git.Ref.describe branch
 
 pushLocal :: Git.Ref -> CommandStart
 pushLocal branch = do
-    let syncBranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
-    ex <- inRepo $ Git.Ref.exists syncBranch
-    if ex then do
-        showStart "updateing" $
-            Git.Ref.describe syncBranch ++
-            " to the state of " ++ Git.Ref.describe branch ++ "..."
-        next $ next $
-            inRepo $ Git.Command.runBool "branch" [Param "-f", Param (Git.Ref.describe syncBranch)]
-      else
-        return Nothing
+	let syncBranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+	ex <- inRepo $ Git.Ref.exists syncBranch
+	if ex then do
+		showStart "updateing" $
+			Git.Ref.describe syncBranch ++
+			" to the state of " ++ Git.Ref.describe branch ++ "..."
+		next $ next $
+			inRepo $ Git.Command.runBool "branch"
+				[Param "-f", Param (Git.Ref.describe syncBranch)]
+	else
+		return Nothing
 
 mergeFromIfExists :: Git.Ref -> CommandStart
 mergeFromIfExists fromBranch = do
-    ex <- inRepo $ Git.Ref.exists fromBranch
-    if ex then do
-        showStart "merging" $ Git.Ref.describe fromBranch ++ "..."
-        next $ next $
-            inRepo $ Git.Command.runBool "merge" [Param (show fromBranch)]
-      else do
-        showNote $ Git.Ref.describe fromBranch ++ " does not exist, not merging."
-        showOutput
-        return Nothing
-
+	ex <- inRepo $ Git.Ref.exists fromBranch
+	if ex then do
+		showStart "merging" $ Git.Ref.describe fromBranch ++ "..."
+		next $ next $
+			inRepo $ Git.Command.runBool "merge" [Param (show fromBranch)]
+	else do
+		showNote $ Git.Ref.describe fromBranch ++ " does not exist, not merging."
+		showOutput
+		return Nothing
 
 fetch :: Remote.Remote Annex -> CommandStart
 fetch remote = do
@@ -102,22 +109,25 @@ fetch remote = do
 		inRepo $ Git.Command.runBool "fetch" [Param (Remote.name remote)]
 
 mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
-mergeRemote remote branch =
-    mergeFromIfExists $ Git.Ref $ "refs/remotes/" ++ Remote.name remote ++ "/synced/" ++ Git.Ref.describe branch
+mergeRemote remote branch = mergeFromIfExists $ Git.Ref $
+	"refs/remotes/" ++ Remote.name remote ++ 
+	"/synced/" ++ Git.Ref.describe branch
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pushRemote remote branch = do
-    showStart "pushing to" (Remote.name remote)
-    let syncbranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
-    let syncbranchRemote = Git.Ref $ "refs/remotes/" ++ Remote.name remote ++ "/" ++ Git.Ref.describe syncbranch
-    let refspec = Git.Ref.describe branch ++ ":" ++ Git.Ref.describe syncbranch
-    ex <- inRepo $ Git.Ref.exists syncbranchRemote
-    next $ next $ do
-            showOutput
-            inRepo $ Git.Command.runBool "push" $
-                [ Param (Remote.name remote)
-                , Param (Git.Ref.describe Annex.Branch.name) ] ++ 
-                [ Param refspec | ex ]
+	showStart "pushing to" (Remote.name remote)
+	let syncbranch = Git.Ref $ "refs/heads/synced/" ++
+		Git.Ref.describe branch
+	let syncbranchRemote = Git.Ref $ "refs/remotes/" ++
+		Remote.name remote ++ "/" ++ Git.Ref.describe syncbranch
+	let refspec = Git.Ref.describe branch ++ ":" ++ Git.Ref.describe syncbranch
+	ex <- inRepo $ Git.Ref.exists syncbranchRemote
+	next $ next $ do
+		showOutput
+		inRepo $ Git.Command.runBool "push" $
+			[ Param (Remote.name remote)
+			, Param (Git.Ref.describe Annex.Branch.name) ] ++ 
+			[ Param refspec | ex ]
 
 currentBranch :: Annex Git.Ref
 currentBranch = Git.Ref . firstLine . L.unpack <$>

From 26040d64195a9fe39a3e7fc04d06d3180ab2497a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 16:48:26 -0400
Subject: [PATCH 2823/8313] add base, under

The describe function was only intended to generate a human-visible
description of a branch, but taking the base of a branch is a useful
operation to be able to do no matter the human-visible representation.

Converting a branch like refs/heads/master to refs/heads/origin/master
is also a useful operation, and under can do that.
---
 Git/Ref.hs | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/Git/Ref.hs b/Git/Ref.hs
index 0197ae7893..3341cf648c 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -13,14 +13,26 @@ import Common
 import Git
 import Git.Command
 
-{- Converts a fully qualified git ref into a user-visible version. -}
+{- Converts a fully qualified git ref into a user-visible string. -}
 describe :: Ref -> String
-describe = remove "refs/heads/" . remove "refs/remotes/" . show
+describe = show . base
+
+{- Often git refs are fully qualified (eg: refs/heads/master).
+ - Converts such a fully qualified ref into a base ref (eg: master). -}
+base :: Ref -> Ref
+base = Ref . remove "refs/heads/" . remove "refs/remotes/" . show
 	where
 		remove prefix s
 			| prefix `isPrefixOf` s = drop (length prefix) s
 			| otherwise = s
 
+
+{- Given a directory such as "refs/remotes/origin", and a ref such as
+ - refs/heads/master, yields a version of that ref under the directory,
+ - such as refs/remotes/origin/master. -}
+under :: String -> Ref -> Ref
+under dir r = Ref $ dir  show (base r)
+
 {- Checks if a ref exists. -}
 exists :: Ref -> Repo -> IO Bool
 exists ref = runBool "show-ref" 

From 556618a3ec132a8270a48670243674bec563cb6b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 16:50:05 -0400
Subject: [PATCH 2824/8313] avoid using Git.Ref.describe except for when
 generating user messages

The other uses of it can all be simplified using Git.Ref.base,
Git.Ref.under, and show.

In some cases, describe was being used to shorten the branch name
unnecessarily, and I instead pass the fully qualified name to git.
---
 Command/Sync.hs | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index d519943432..38805811a9 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -47,13 +47,12 @@ syncRemotes branch [] = defaultSyncRemotes branch
 syncRemotes _ rs = mapM Remote.byName rs
 
 defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
-defaultSyncRemotes branch = 
-	mapM Remote.byName
+defaultSyncRemotes branch = mapM Remote.byName
 		=<< process . L.unpack <$> inRepo showref
-	where 
-		syncbranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+	where
+		syncbranch = Git.Ref.under "refs/heads/synced/" branch
 		showref = Git.Command.pipeRead
-			[Param "show-ref", Param (Git.Ref.describe syncbranch)]
+			[Param "show-ref", Param $ show $ Git.Ref.base syncbranch]
 		process = map getRemoteName . filter isRemote .
 			map getBranchName . lines
 	        isRemote r = "refs/remotes/" `isPrefixOf` r
@@ -71,20 +70,21 @@ commit = do
 		return True
 
 mergeLocal :: Git.Ref -> CommandStart
-mergeLocal branch = mergeFromIfExists $ Git.Ref $
-	"refs/heads/synced/" ++ Git.Ref.describe branch
+mergeLocal = mergeFromIfExists . Git.Ref.under "refs/heads/synced"
 
 pushLocal :: Git.Ref -> CommandStart
 pushLocal branch = do
-	let syncBranch = Git.Ref $ "refs/heads/synced/" ++ Git.Ref.describe branch
+	let syncBranch = Git.Ref.under "refs/heads/synced" branch
 	ex <- inRepo $ Git.Ref.exists syncBranch
 	if ex then do
-		showStart "updateing" $
+		showStart "updating" $
 			Git.Ref.describe syncBranch ++
 			" to the state of " ++ Git.Ref.describe branch ++ "..."
 		next $ next $
 			inRepo $ Git.Command.runBool "branch"
-				[Param "-f", Param (Git.Ref.describe syncBranch)]
+				[ Param "-f"
+				, Param $ show $ Git.Ref.base syncBranch
+				]
 	else
 		return Nothing
 
@@ -109,24 +109,22 @@ fetch remote = do
 		inRepo $ Git.Command.runBool "fetch" [Param (Remote.name remote)]
 
 mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
-mergeRemote remote branch = mergeFromIfExists $ Git.Ref $
-	"refs/remotes/" ++ Remote.name remote ++ 
-	"/synced/" ++ Git.Ref.describe branch
+mergeRemote remote = mergeFromIfExists .
+	Git.Ref.under ("refs/remotes/" ++ Remote.name remote ++ "/synced")
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pushRemote remote branch = do
 	showStart "pushing to" (Remote.name remote)
-	let syncbranch = Git.Ref $ "refs/heads/synced/" ++
-		Git.Ref.describe branch
-	let syncbranchRemote = Git.Ref $ "refs/remotes/" ++
-		Remote.name remote ++ "/" ++ Git.Ref.describe syncbranch
-	let refspec = Git.Ref.describe branch ++ ":" ++ Git.Ref.describe syncbranch
+	let syncbranch = Git.Ref.under "refs/heads/synced" branch
+	let syncbranchRemote = Git.Ref.under
+		("refs/remotes/" ++ Remote.name remote) syncbranch
+	let refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
 	ex <- inRepo $ Git.Ref.exists syncbranchRemote
 	next $ next $ do
 		showOutput
 		inRepo $ Git.Command.runBool "push" $
 			[ Param (Remote.name remote)
-			, Param (Git.Ref.describe Annex.Branch.name) ] ++ 
+			, Param (show $ Annex.Branch.name) ] ++ 
 			[ Param refspec | ex ]
 
 currentBranch :: Annex Git.Ref

From 4400f65967c2b610f03725c1c5e2864c93a0978f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 17:38:38 -0400
Subject: [PATCH 2825/8313] message cleanup

---
 Command/Sync.hs | 99 +++++++++++++++++++++++++------------------------
 1 file changed, 50 insertions(+), 49 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 38805811a9..458a114df2 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -29,30 +29,29 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote))
 seek :: CommandSeek
 seek args = do
 	!branch <- currentBranch
-	remotes <- syncRemotes branch args
-	showStart "syncing" $ "branch " ++ Git.Ref.describe branch ++ " with remote repositories " ++ intercalate "," (map Remote.name remotes)
-	showOutput
+	let syncbranch = Git.Ref.under "refs/heads/synced/" branch
+	remotes <- syncRemotes syncbranch args
 	return $
 		[ commit
 		, mergeLocal branch
 		] ++
-		[ fetch remote | remote <- remotes ] ++
-		[ mergeRemote remote branch | remote <- remotes ] ++
+		[ update remote branch | remote <- remotes ] ++
 		[ mergeAnnex ] ++
-		[ pushLocal branch ] ++
-		[ pushRemote remote branch | remote <- remotes ]
+		[ pushLocal syncbranch ] ++
+		[ pushRemote remote branch syncbranch | remote <- remotes ]
 
 syncRemotes :: Git.Ref -> [String] -> Annex [Remote.Remote Annex]
 syncRemotes branch [] = defaultSyncRemotes branch
 syncRemotes _ rs = mapM Remote.byName rs
 
 defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
-defaultSyncRemotes branch = mapM Remote.byName
+defaultSyncRemotes syncbranch = mapM Remote.byName
 		=<< process . L.unpack <$> inRepo showref
 	where
-		syncbranch = Git.Ref.under "refs/heads/synced/" branch
 		showref = Git.Command.pipeRead
-			[Param "show-ref", Param $ show $ Git.Ref.base syncbranch]
+			[ Param "show-ref"
+			, Param $ show $ Git.Ref.base syncbranch
+			]
 		process = map getRemoteName . filter isRemote .
 			map getBranchName . lines
 	        isRemote r = "refs/remotes/" `isPrefixOf` r
@@ -70,52 +69,54 @@ commit = do
 		return True
 
 mergeLocal :: Git.Ref -> CommandStart
-mergeLocal = mergeFromIfExists . Git.Ref.under "refs/heads/synced"
+mergeLocal branch = do
+	let mergebranch = Git.Ref.under "refs/heads/synced" branch
+	showStart "merge" $ Git.Ref.describe mergebranch
+	next $ next $ mergeFromIfExists mergebranch
 
 pushLocal :: Git.Ref -> CommandStart
-pushLocal branch = do
-	let syncBranch = Git.Ref.under "refs/heads/synced" branch
-	ex <- inRepo $ Git.Ref.exists syncBranch
-	if ex then do
-		showStart "updating" $
-			Git.Ref.describe syncBranch ++
-			" to the state of " ++ Git.Ref.describe branch ++ "..."
-		next $ next $
-			inRepo $ Git.Command.runBool "branch"
-				[ Param "-f"
-				, Param $ show $ Git.Ref.base syncBranch
-				]
-	else
-		return Nothing
+pushLocal syncbranch = go =<< inRepo (Git.Ref.exists syncbranch)
+	where
+		go False = stop
+		go True = do
+			unlessM (updatebranch) $
+				error $ "failed to update " ++ show syncbranch
+			stop
+		updatebranch = inRepo $ Git.Command.runBool "branch"
+			[ Param "-f"
+			, Param $ show $ Git.Ref.base syncbranch
+			]
 
-mergeFromIfExists :: Git.Ref -> CommandStart
-mergeFromIfExists fromBranch = do
-	ex <- inRepo $ Git.Ref.exists fromBranch
-	if ex then do
-		showStart "merging" $ Git.Ref.describe fromBranch ++ "..."
-		next $ next $
-			inRepo $ Git.Command.runBool "merge" [Param (show fromBranch)]
-	else do
-		showNote $ Git.Ref.describe fromBranch ++ " does not exist, not merging."
-		showOutput
-		return Nothing
+mergeFromIfExists :: Git.Ref -> CommandCleanup
+mergeFromIfExists branch = go =<< inRepo (Git.Ref.exists branch)
+	where
+		go True = do
+			showOutput
+			inRepo $ Git.Command.runBool "merge"
+				[Param (show branch)]
+		go False = do
+			showNote $ Git.Ref.describe branch ++
+				" does not exist, not merging"
+			return False
 
-fetch :: Remote.Remote Annex -> CommandStart
-fetch remote = do
-	showStart "fetching from" (Remote.name remote)
-	next $ next $ do
-		showOutput
+update :: Remote.Remote Annex -> Git.Ref -> CommandStart
+update remote branch = do
+	showStart "update" (Remote.name remote)
+	next $ do
 		checkRemote remote
-		inRepo $ Git.Command.runBool "fetch" [Param (Remote.name remote)]
+		showOutput
+		fetched <- inRepo $ Git.Command.runBool "fetch" [Param (Remote.name remote)]
+		if fetched
+			then next $ mergeRemote remote branch
+			else stop
 
-mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
 mergeRemote remote = mergeFromIfExists .
 	Git.Ref.under ("refs/remotes/" ++ Remote.name remote ++ "/synced")
 
-pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
-pushRemote remote branch = do
-	showStart "pushing to" (Remote.name remote)
-	let syncbranch = Git.Ref.under "refs/heads/synced" branch
+pushRemote :: Remote.Remote Annex -> Git.Ref -> Git.Ref -> CommandStart
+pushRemote remote branch syncbranch = do
+	showStart "push" (Remote.name remote)
 	let syncbranchRemote = Git.Ref.under
 		("refs/remotes/" ++ Remote.name remote) syncbranch
 	let refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
@@ -139,6 +140,6 @@ checkRemote remote = do
 		error $ "No url is configured for the remote: " ++ Remote.name remote
 
 mergeAnnex :: CommandStart
-mergeAnnex = next $ next $ do
+mergeAnnex = do
 	Annex.Branch.forceUpdate
-	return True
+	stop

From 4610f28f17d7a07dc70e42abba33ec5961e2795e Mon Sep 17 00:00:00 2001
From: bremner 
Date: Fri, 30 Dec 2011 21:41:15 +0000
Subject: [PATCH 2826/8313] Added a comment: repo name conventions?

---
 .../comment_3_807035f38509ccb9f93f1929ecd37417._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment b/doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment
new file mode 100644
index 0000000000..243764054d
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_3_807035f38509ccb9f93f1929ecd37417._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="156.34.79.193"
+ subject="repo name conventions?"
+ date="2011-12-30T21:41:13Z"
+ content="""
+I'm confused by the fact that the git-annex-shell adc rejects any repo names that don't start with /~/ since none of my repos start that way. It seems work ok if I just delete /\~ from the front of the regex, but I feel like I must be missing something. 
+"""]]

From 2260eca688320100e28f4ad3c7dac04c00f51ebf Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 30 Dec 2011 21:49:07 +0000
Subject: [PATCH 2827/8313] Added a comment

---
 ..._6b541ed834ef45606f3b98779a25a148._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment b/doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment
new file mode 100644
index 0000000000..30932e301f
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_11_6b541ed834ef45606f3b98779a25a148._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 11"
+ date="2011-12-30T21:49:06Z"
+ content="""
+OMG, my first sizable haskell patch!
+
+So trying this out..
+
+In each repo I want to sync, I first `git branch synced/master`
+
+Then in each repo, I found I had to pull from each of its remotes, to get the tracking branches that `defaultSyncRemotes` looks for to know those remotes are syncable. This was the surprising thing for me, I had expected sync to somehow work out which remotes were syncable without my explicit pull. And it was not very obvious that sync was not doing its thing before I did that, since it still does a lot of \"stuff\".
+
+Once set up properly, `git annex sync` fetches from each remote, merges, and then pushes to each remote that has a synced branch. Changes propigate around even when some links are one-directional. Cool!
+
+So it works fine, but I think more needs to be done to make setting up syncing easier. Ideally, all a user would need to do is run \"git annex sync\" and it syncs from all remotes, without needing to manually set up the synced/master branch.
+
+While this would lose the ability to control which remotes are synced, I think that being able to `git annex sync origin` and only sync from/to origin is sufficient, for the centralized use case.
+
+---
+
+Code review:
+
+Why did you make `branch` strict?
+
+There is a bit of a bug in your use of Command.Merge.start. The git-annex branch merge code only runs once per git-annex run, and often this comes before sync fetches from the remotes, leading to a push conflict. I've fixed this in my \"sync\" branch, along with a few other minor things.
+
+`mergeRemote` merges from `refs/remotes/foo/synced/master`. But that will only be up-to-date if `git annex sync` has recently been run there. Is there any reason it couldn't merge from `refs/remotes/foo/master`?
+"""]]

From 9d85baa31425e04f0eb79e43c09c8d4e4a414c6b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 17:54:09 -0400
Subject: [PATCH 2828/8313] improve wording

---
 Command/Sync.hs    | 16 ++++++++--------
 doc/git-annex.mdwn |  6 ++++--
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 458a114df2..790bf53b59 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -31,14 +31,14 @@ seek args = do
 	!branch <- currentBranch
 	let syncbranch = Git.Ref.under "refs/heads/synced/" branch
 	remotes <- syncRemotes syncbranch args
-	return $
-		[ commit
-		, mergeLocal branch
-		] ++
-		[ update remote branch | remote <- remotes ] ++
-		[ mergeAnnex ] ++
-		[ pushLocal syncbranch ] ++
-		[ pushRemote remote branch syncbranch | remote <- remotes ]
+	return $ concat $
+		[ [ commit ]
+		, [ mergeLocal branch ]
+		, [ update remote branch | remote <- remotes ]
+		, [ mergeAnnex ]
+		, [ pushLocal syncbranch ]
+		, [ pushRemote remote branch syncbranch | remote <- remotes ]
+		]
 
 syncRemotes :: Git.Ref -> [String] -> Annex [Remote.Remote Annex]
 syncRemotes branch [] = defaultSyncRemotes branch
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 320453f185..3ca2ce1773 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -126,7 +126,8 @@ subdirectories).
   one or more other repositories. The sync process involves first committing
   all local changes, then fetching and merging the `synced/master` and the
   `git-annex` branch from the remote repositories and finally pushing the
-  changes back to these remote branches.
+  changes back to these remote repositories.
+
   You can use standard git commands to do each of those steps by hand,
   or if you don't want to worry about the details, you can use sync.
 
@@ -134,7 +135,8 @@ subdirectories).
   `synced/master` branch. If you want to include/exclude a repository from
   this list, just create or delete this branch.
 
-  Note that sync does not transfer any file contents from or to the remote.
+  Note that sync does not transfer any file contents from or to the remote
+  repositories.
 
 * addurl [url ...]
 

From f2fa29bf3bb3d54b9c4d789b36d04ba693dee53a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 18:04:01 -0400
Subject: [PATCH 2829/8313] check if branches are up-to-date before merging,
 pushing

This optimises away the need to run anything in some common cases.
It's particularly useful on push; no need to push if the tracking branch
we just pulled is the same as the branch we're going to push.
---
 Command/Sync.hs | 48 ++++++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 790bf53b59..423a772f38 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -15,6 +15,7 @@ import Command
 import qualified Remote
 import qualified Annex.Branch
 import qualified Git.Command
+import qualified Git.Branch
 import qualified Git.Config
 import qualified Git.Ref
 import qualified Git
@@ -34,7 +35,7 @@ seek args = do
 	return $ concat $
 		[ [ commit ]
 		, [ mergeLocal branch ]
-		, [ update remote branch | remote <- remotes ]
+		, [ pullRemote remote branch | remote <- remotes ]
 		, [ mergeAnnex ]
 		, [ pushLocal syncbranch ]
 		, [ pushRemote remote branch syncbranch | remote <- remotes ]
@@ -69,10 +70,13 @@ commit = do
 		return True
 
 mergeLocal :: Git.Ref -> CommandStart
-mergeLocal branch = do
-	let mergebranch = Git.Ref.under "refs/heads/synced" branch
-	showStart "merge" $ Git.Ref.describe mergebranch
-	next $ next $ mergeFromIfExists mergebranch
+mergeLocal branch = go =<< inRepo (Git.Branch.changed branch mergebranch)
+	where
+		mergebranch = Git.Ref.under "refs/heads/synced" branch
+		go False = stop
+		go True = do
+			showStart "merge" $ Git.Ref.describe mergebranch
+			next $ next $ mergeFromIfExists mergebranch
 
 pushLocal :: Git.Ref -> CommandStart
 pushLocal syncbranch = go =<< inRepo (Git.Ref.exists syncbranch)
@@ -99,9 +103,9 @@ mergeFromIfExists branch = go =<< inRepo (Git.Ref.exists branch)
 				" does not exist, not merging"
 			return False
 
-update :: Remote.Remote Annex -> Git.Ref -> CommandStart
-update remote branch = do
-	showStart "update" (Remote.name remote)
+pullRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+pullRemote remote branch = do
+	showStart "pull" (Remote.name remote)
 	next $ do
 		checkRemote remote
 		showOutput
@@ -115,18 +119,22 @@ mergeRemote remote = mergeFromIfExists .
 	Git.Ref.under ("refs/remotes/" ++ Remote.name remote ++ "/synced")
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> Git.Ref -> CommandStart
-pushRemote remote branch syncbranch = do
-	showStart "push" (Remote.name remote)
-	let syncbranchRemote = Git.Ref.under
-		("refs/remotes/" ++ Remote.name remote) syncbranch
-	let refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
-	ex <- inRepo $ Git.Ref.exists syncbranchRemote
-	next $ next $ do
-		showOutput
-		inRepo $ Git.Command.runBool "push" $
-			[ Param (Remote.name remote)
-			, Param (show $ Annex.Branch.name) ] ++ 
-			[ Param refspec | ex ]
+pushRemote remote branch syncbranch = go =<< newer
+	where
+		newer = inRepo $ Git.Branch.changed syncbranchRemote syncbranch
+		go False = stop
+		go True = do		
+			showStart "push" (Remote.name remote)
+			ex <- inRepo $ Git.Ref.exists syncbranchRemote
+			next $ next $ do
+				showOutput
+				inRepo $ Git.Command.runBool "push" $
+					[ Param (Remote.name remote)
+					, Param (show $ Annex.Branch.name) ] ++ 
+					[ Param refspec | ex ]
+		refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
+		syncbranchRemote = Git.Ref.under 
+			("refs/remotes/" ++ Remote.name remote) syncbranch
 
 currentBranch :: Annex Git.Ref
 currentBranch = Git.Ref . firstLine . L.unpack <$>

From 56488e807bcc7dcc6e9f5b3e359ddf4f4fb5d8d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 18:19:45 -0400
Subject: [PATCH 2830/8313] check that synced/master exists before trying to
 use it

and a nice error message if syncing is not set up yet
---
 Command/Sync.hs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 423a772f38..1c20c409c6 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -70,9 +70,13 @@ commit = do
 		return True
 
 mergeLocal :: Git.Ref -> CommandStart
-mergeLocal branch = go =<< inRepo (Git.Branch.changed branch mergebranch)
+mergeLocal branch = go =<< needmerge
 	where
 		mergebranch = Git.Ref.under "refs/heads/synced" branch
+		needmerge = do
+			unlessM (inRepo $ Git.Ref.exists mergebranch) $
+				error $ Git.Ref.describe mergebranch ++ " does not exist; create it to enable sync"
+			inRepo $ Git.Branch.changed branch mergebranch
 		go False = stop
 		go True = do
 			showStart "merge" $ Git.Ref.describe mergebranch
@@ -109,7 +113,8 @@ pullRemote remote branch = do
 	next $ do
 		checkRemote remote
 		showOutput
-		fetched <- inRepo $ Git.Command.runBool "fetch" [Param (Remote.name remote)]
+		fetched <- inRepo $ Git.Command.runBool "fetch"
+			[Param $ Remote.name remote]
 		if fetched
 			then next $ mergeRemote remote branch
 			else stop
@@ -123,7 +128,7 @@ pushRemote remote branch syncbranch = go =<< newer
 	where
 		newer = inRepo $ Git.Branch.changed syncbranchRemote syncbranch
 		go False = stop
-		go True = do		
+		go True = do
 			showStart "push" (Remote.name remote)
 			ex <- inRepo $ Git.Ref.exists syncbranchRemote
 			next $ next $ do

From 52104dae6f5175f01395a657fd40423d1196604a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 18:36:40 -0400
Subject: [PATCH 2831/8313] refactor

---
 Annex/Branch.hs |  2 +-
 Command/Sync.hs | 14 ++++----------
 Git/Ref.hs      | 13 +++++++++----
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 8c9d1582ae..d3a81d8e50 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -58,7 +58,7 @@ hasSibling = not . null <$> siblingBranches
 {- List of git-annex (refs, branches), including the main one and any
  - from remotes. Duplicate refs are filtered out. -}
 siblingBranches :: Annex [(Git.Ref, Git.Branch)]
-siblingBranches = inRepo $ Git.Ref.matching name
+siblingBranches = inRepo $ Git.Ref.matchingUniq name
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 1c20c409c6..e5f23e63ef 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -46,18 +46,12 @@ syncRemotes branch [] = defaultSyncRemotes branch
 syncRemotes _ rs = mapM Remote.byName rs
 
 defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
-defaultSyncRemotes syncbranch = mapM Remote.byName
-		=<< process . L.unpack <$> inRepo showref
+defaultSyncRemotes syncbranch = mapM Remote.byName =<<
+	map getRemoteName . filter isRemote . map (show . snd) <$>
+		inRepo (Git.Ref.matching $ Git.Ref.base syncbranch)
 	where
-		showref = Git.Command.pipeRead
-			[ Param "show-ref"
-			, Param $ show $ Git.Ref.base syncbranch
-			]
-		process = map getRemoteName . filter isRemote .
-			map getBranchName . lines
-	        isRemote r = "refs/remotes/" `isPrefixOf` r
-		getBranchName = snd . separate (== ' ')
 		getRemoteName = fst . separate (== '/') . snd . separate (== '/') . snd . separate (== '/')
+	        isRemote r = "refs/remotes/" `isPrefixOf` r
 
 commit :: CommandStart
 commit = do
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 3341cf648c..557d24a372 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -48,13 +48,18 @@ sha branch repo = process . L.unpack <$> showref repo
 		process [] = Nothing
 		process s = Just $ Ref $ firstLine s
 
-{- List of (refs, branches) matching a given ref spec.
- - Duplicate refs are filtered out. -}
+{- List of (refs, branches) matching a given ref spec. -}
 matching :: Ref -> Repo -> IO [(Ref, Branch)]
 matching ref repo = do
 	r <- pipeRead [Param "show-ref", Param $ show ref] repo
-	return $ nubBy uniqref $ map (gen . L.unpack) (L.lines r)
+	return $ map (gen . L.unpack) (L.lines r)
 	where
-		uniqref (a, _) (b, _) = a == b
 		gen l = let (r, b) = separate (== ' ') l in
 			(Ref r, Ref b)
+
+{- List of (refs, branches) matching a given ref spec.
+ - Duplicate refs are filtered out. -}
+matchingUniq :: Ref -> Repo -> IO [(Ref, Branch)]
+matchingUniq ref repo = nubBy uniqref <$> matching ref repo
+	where
+		uniqref (a, _) (b, _) = a == b

From 14d16b77b325c6e1c6ee0aea3dbf7b21f288165c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 18:37:55 -0400
Subject: [PATCH 2832/8313] refactor

---
 Command/Sync.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index e5f23e63ef..0979738485 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -47,9 +47,9 @@ syncRemotes _ rs = mapM Remote.byName rs
 
 defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
 defaultSyncRemotes syncbranch = mapM Remote.byName =<<
-	map getRemoteName . filter isRemote . map (show . snd) <$>
-		inRepo (Git.Ref.matching $ Git.Ref.base syncbranch)
+	map getRemoteName . filter isRemote . map (show . snd) <$> siblings
 	where
+		siblings = inRepo (Git.Ref.matching $ Git.Ref.base syncbranch)
 		getRemoteName = fst . separate (== '/') . snd . separate (== '/') . snd . separate (== '/')
 	        isRemote r = "refs/remotes/" `isPrefixOf` r
 

From f6f7ee71310df1dd409819bef15ce020e290f4a8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 18:52:24 -0400
Subject: [PATCH 2833/8313] automatically create the syncbranch

---
 Command/Sync.hs | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 0979738485..85fffdda87 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -34,7 +34,7 @@ seek args = do
 	remotes <- syncRemotes syncbranch args
 	return $ concat $
 		[ [ commit ]
-		, [ mergeLocal branch ]
+		, [ mergeLocal branch syncbranch ]
 		, [ pullRemote remote branch | remote <- remotes ]
 		, [ mergeAnnex ]
 		, [ pushLocal syncbranch ]
@@ -63,28 +63,31 @@ commit = do
 			[Param "-a", Param "-m", Param "git-annex automatic sync"]
 		return True
 
-mergeLocal :: Git.Ref -> CommandStart
-mergeLocal branch = go =<< needmerge
+mergeLocal :: Git.Ref -> Git.Ref -> CommandStart
+mergeLocal branch syncbranch = go =<< needmerge
 	where
-		mergebranch = Git.Ref.under "refs/heads/synced" branch
 		needmerge = do
-			unlessM (inRepo $ Git.Ref.exists mergebranch) $
-				error $ Git.Ref.describe mergebranch ++ " does not exist; create it to enable sync"
-			inRepo $ Git.Branch.changed branch mergebranch
+			unlessM (inRepo $ Git.Ref.exists syncbranch) $
+				updateBranch syncbranch
+			inRepo $ Git.Branch.changed branch syncbranch
 		go False = stop
 		go True = do
-			showStart "merge" $ Git.Ref.describe mergebranch
-			next $ next $ mergeFromIfExists mergebranch
+			showStart "merge" $ Git.Ref.describe syncbranch
+			next $ next $ mergeFromIfExists syncbranch
 
 pushLocal :: Git.Ref -> CommandStart
 pushLocal syncbranch = go =<< inRepo (Git.Ref.exists syncbranch)
 	where
 		go False = stop
 		go True = do
-			unlessM (updatebranch) $
-				error $ "failed to update " ++ show syncbranch
+			updateBranch syncbranch
 			stop
-		updatebranch = inRepo $ Git.Command.runBool "branch"
+
+updateBranch :: Git.Ref -> Annex ()
+updateBranch syncbranch = 
+	unlessM go $ error $ "failed to update " ++ show syncbranch
+	where
+		go = inRepo $ Git.Command.runBool "branch"
 			[ Param "-f"
 			, Param $ show $ Git.Ref.base syncbranch
 			]

From dba8fc8a1c55a091cb7992b6adfc6332eb2da585 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 19:10:54 -0400
Subject: [PATCH 2834/8313] export remoteList

---
 Remote.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 10bf9d7694..643ba2d0d1 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -16,6 +16,7 @@ module Remote (
 	hasKeyCheap,
 
 	remoteTypes,
+	remoteList,
 	remoteMap,
 	byName,
 	prettyPrintUUIDs,
@@ -65,8 +66,8 @@ remoteTypes =
 
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached. -}
-genList :: Annex [Remote Annex]
-genList = do
+remoteList :: Annex [Remote Annex]
+remoteList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
 		then do
@@ -86,7 +87,7 @@ genList = do
 
 {- Map of UUIDs of Remotes and their names. -}
 remoteMap :: Annex (M.Map UUID String)
-remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> genList
+remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> remoteList
 
 {- Looks up a remote by name. (Or by UUID.) Only finds currently configured
  - git remotes. -}
@@ -99,8 +100,7 @@ byName n = do
 byName' :: String -> Annex (Either String (Remote Annex))
 byName' "" = return $ Left "no remote specified"
 byName' n = do
-	allremotes <- genList
-	let match = filter matching allremotes
+	match <- filter matching <$> remoteList
 	if null match
 		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
 		else return $ Right $ Prelude.head match
@@ -196,7 +196,7 @@ keyPossibilities' withtrusted key = do
 	let validtrusteduuids = validuuids `intersect` trusted
 
 	-- remotes that match uuids that have the key
-	allremotes <- filterM (repoNotIgnored . repo) =<< genList
+	allremotes <- filterM (repoNotIgnored . repo) =<< remoteList
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)

From 79872e360edcac8106ddb366018519ab41d70695 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 19:11:22 -0400
Subject: [PATCH 2835/8313] automated syncing

Some changes to make automated syncing nicer. Merge from both the remote's
$branch and its synced/$branch; either could have new changes. Create
synced/$branch on the remote when pushing.
---
 Command/Sync.hs | 74 ++++++++++++++++++++++++++-----------------------
 1 file changed, 40 insertions(+), 34 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 85fffdda87..4b11e3ac11 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -30,28 +30,25 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote))
 seek :: CommandSeek
 seek args = do
 	!branch <- currentBranch
-	let syncbranch = Git.Ref.under "refs/heads/synced/" branch
-	remotes <- syncRemotes syncbranch args
+	remotes <- syncRemotes args
 	return $ concat $
 		[ [ commit ]
-		, [ mergeLocal branch syncbranch ]
+		, [ mergeLocal branch ]
 		, [ pullRemote remote branch | remote <- remotes ]
 		, [ mergeAnnex ]
-		, [ pushLocal syncbranch ]
-		, [ pushRemote remote branch syncbranch | remote <- remotes ]
+		, [ pushLocal branch ]
+		, [ pushRemote remote branch | remote <- remotes ]
 		]
 
-syncRemotes :: Git.Ref -> [String] -> Annex [Remote.Remote Annex]
-syncRemotes branch [] = defaultSyncRemotes branch
-syncRemotes _ rs = mapM Remote.byName rs
+syncBranch :: Git.Ref -> Git.Ref
+syncBranch = Git.Ref.under "refs/heads/synced/"
 
-defaultSyncRemotes :: Git.Ref -> Annex [Remote.Remote Annex]
-defaultSyncRemotes syncbranch = mapM Remote.byName =<<
-	map getRemoteName . filter isRemote . map (show . snd) <$> siblings
+syncRemotes :: [String] -> Annex [Remote.Remote Annex]
+syncRemotes [] = filterM hasurl =<< Remote.remoteList
 	where
-		siblings = inRepo (Git.Ref.matching $ Git.Ref.base syncbranch)
-		getRemoteName = fst . separate (== '/') . snd . separate (== '/') . snd . separate (== '/')
-	        isRemote r = "refs/remotes/" `isPrefixOf` r
+		hasurl r = not . null <$> geturl r
+		geturl r = fromRepo $ Git.Config.get ("remote." ++ Remote.name r ++ ".url") ""
+syncRemotes rs = mapM Remote.byName rs
 
 commit :: CommandStart
 commit = do
@@ -63,9 +60,10 @@ commit = do
 			[Param "-a", Param "-m", Param "git-annex automatic sync"]
 		return True
 
-mergeLocal :: Git.Ref -> Git.Ref -> CommandStart
-mergeLocal branch syncbranch = go =<< needmerge
+mergeLocal :: Git.Ref -> CommandStart
+mergeLocal branch = go =<< needmerge
 	where
+		syncbranch = syncBranch branch
 		needmerge = do
 			unlessM (inRepo $ Git.Ref.exists syncbranch) $
 				updateBranch syncbranch
@@ -76,8 +74,9 @@ mergeLocal branch syncbranch = go =<< needmerge
 			next $ next $ mergeFromIfExists syncbranch
 
 pushLocal :: Git.Ref -> CommandStart
-pushLocal syncbranch = go =<< inRepo (Git.Ref.exists syncbranch)
+pushLocal branch = go =<< inRepo (Git.Ref.exists syncbranch)
 	where
+		syncbranch = syncBranch branch
 		go False = stop
 		go True = do
 			updateBranch syncbranch
@@ -108,7 +107,6 @@ pullRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pullRemote remote branch = do
 	showStart "pull" (Remote.name remote)
 	next $ do
-		checkRemote remote
 		showOutput
 		fetched <- inRepo $ Git.Command.runBool "fetch"
 			[Param $ Remote.name remote]
@@ -116,25 +114,40 @@ pullRemote remote branch = do
 			then next $ mergeRemote remote branch
 			else stop
 
+{- The remote probably has both a master and a synced/master branch.
+ - Which to merge from? Well, the master has whatever latest changes
+ - were committed, while the synced/master may have changes that some
+ - other remote synced to this remote. So, merge them both. -}
 mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
-mergeRemote remote = mergeFromIfExists .
-	Git.Ref.under ("refs/remotes/" ++ Remote.name remote ++ "/synced")
-
-pushRemote :: Remote.Remote Annex -> Git.Ref -> Git.Ref -> CommandStart
-pushRemote remote branch syncbranch = go =<< newer
+mergeRemote remote branch = all (== True) <$> mapM go [branch, syncBranch branch]
 	where
-		newer = inRepo $ Git.Branch.changed syncbranchRemote syncbranch
+		go b = do
+			e <- inRepo $ Git.Branch.changed branch b
+			if e
+				then mergeFromIfExists $ remotebranch b
+				else return True
+		remotebranch = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
+
+pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+pushRemote remote branch = go =<< newer
+	where
+		newer = do
+			e <- inRepo (Git.Ref.exists syncbranchRemote)
+			if e
+				then inRepo $ Git.Branch.changed syncbranchRemote syncbranch
+				else return True
 		go False = stop
 		go True = do
 			showStart "push" (Remote.name remote)
-			ex <- inRepo $ Git.Ref.exists syncbranchRemote
 			next $ next $ do
 				showOutput
 				inRepo $ Git.Command.runBool "push" $
 					[ Param (Remote.name remote)
-					, Param (show $ Annex.Branch.name) ] ++ 
-					[ Param refspec | ex ]
+					, Param (show $ Annex.Branch.name)
+					, Param refspec
+					]
 		refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
+		syncbranch = syncBranch branch
 		syncbranchRemote = Git.Ref.under 
 			("refs/remotes/" ++ Remote.name remote) syncbranch
 
@@ -142,13 +155,6 @@ currentBranch :: Annex Git.Ref
 currentBranch = Git.Ref . firstLine . L.unpack <$>
 	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
 
-checkRemote :: Remote.Remote Annex -> Annex ()
-checkRemote remote = do
-	remoteurl <- fromRepo $
-		Git.Config.get ("remote." ++ Remote.name remote ++ ".url") ""
-	when (null remoteurl) $
-		error $ "No url is configured for the remote: " ++ Remote.name remote
-
 mergeAnnex :: CommandStart
 mergeAnnex = do
 	Annex.Branch.forceUpdate

From a31b7d93c85afde1e801748b6a7cf5372492bee8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 19:38:46 -0400
Subject: [PATCH 2836/8313] push when git-annex branch changed

I was too heavy-handed in optimising away pushes
---
 Command/Sync.hs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 4b11e3ac11..a3450278ce 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -129,12 +129,16 @@ mergeRemote remote branch = all (== True) <$> mapM go [branch, syncBranch branch
 		remotebranch = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
-pushRemote remote branch = go =<< newer
+pushRemote remote branch = go =<< needpush
 	where
-		newer = do
-			e <- inRepo (Git.Ref.exists syncbranchRemote)
+		needpush = (||)
+			<$> newer syncbranch
+			<*> newer Annex.Branch.name
+		newer b = do
+			let r = remotebranch b
+			e <- inRepo (Git.Ref.exists r)
 			if e
-				then inRepo $ Git.Branch.changed syncbranchRemote syncbranch
+				then inRepo $ Git.Branch.changed r b
 				else return True
 		go False = stop
 		go True = do
@@ -148,8 +152,7 @@ pushRemote remote branch = go =<< newer
 					]
 		refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
 		syncbranch = syncBranch branch
-		syncbranchRemote = Git.Ref.under 
-			("refs/remotes/" ++ Remote.name remote) syncbranch
+		remotebranch = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
 
 currentBranch :: Annex Git.Ref
 currentBranch = Git.Ref . firstLine . L.unpack <$>

From 34c89dc1468a62ba4e53e935db366ad18bd0a3a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 19:45:23 -0400
Subject: [PATCH 2837/8313] improve description of sync

---
 doc/git-annex.mdwn | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3ca2ce1773..acdb1313da 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -120,20 +120,23 @@ subdirectories).
   Use this to undo an unlock command if you don't want to modify
   the files, or have made modifications you want to discard.
 
-* sync
+* sync [remote ...]
 
   Use this command when you want to synchronize the local repository with
-  one or more other repositories. The sync process involves first committing
-  all local changes, then fetching and merging the `synced/master` and the
-  `git-annex` branch from the remote repositories and finally pushing the
-  changes back to these remote repositories.
+  one or more of its remotes. You can specifiy the remotes to sync with;
+  the default is to sync with all remotes.
 
-  You can use standard git commands to do each of those steps by hand,
-  or if you don't want to worry about the details, you can use sync.
+  The sync process involves first committing all local changes, then
+  fetching and merging the `synced/master` and the `git-annex` branch
+  from the remote repositories and finally pushing the changes back to
+  those branches on the remote repositories. You can use standard git
+  commands to do each of those steps by hand, or if you don't want to
+  worry about the details, you can use sync.
 
-  By default, `git annex sync` will sync all remote repositories that have a
-  `synced/master` branch. If you want to include/exclude a repository from
-  this list, just create or delete this branch.
+  Note that syncing with a remote will not update the remote's working
+  tree with changes made to the local repository. However, those changes
+  are pushed to the remote, so can be merged into its working tree
+  by running "git annex sync" on the remote.
 
   Note that sync does not transfer any file contents from or to the remote
   repositories.

From 5a2a515fffbd7c19397f3825d3b2adcf8cf82f76 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 30 Dec 2011 23:45:58 +0000
Subject: [PATCH 2838/8313] Added a comment

---
 ...omment_12_ca8ca35d6cd4a9f94568536736c12adc._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment b/doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment
new file mode 100644
index 0000000000..5b0259b97a
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_12_ca8ca35d6cd4a9f94568536736c12adc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 12"
+ date="2011-12-30T23:45:57Z"
+ content="""
+I have made a new `autosync` branch, where all that the user needs to do is run `git annex sync` and it automatically sets up the synced/master branch. I find this very easy to use, what do you think?
+
+Note that `autosync` is also pretty smart about not running commands like \"git merge\" and \"git push\" when they would not do anything. So you may find `git annex sync` not showing all the steps you'd expect. The only step a sync always performs now is pulling from the remotes.
+"""]]

From b538f45fafdf161c24f07562ac1bd73f1476ace9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 20:02:55 -0400
Subject: [PATCH 2839/8313] add a sync page documenting sync in detail

---
 doc/index.mdwn |  1 +
 doc/sync.mdwn  | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 doc/sync.mdwn

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 5bd42074f5..e0791bf71a 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -45,6 +45,7 @@ files with git.
 * [[git-annex man page|git-annex]]
 * [[key-value backends|backends]] for data storage
 * [[special_remotes]] (including [[special_remotes/S3]] and [[special_remotes/bup]])
+* [[sync]]
 * [[encryption]]
 * [[bare_repositories]]
 * [[internals]]
diff --git a/doc/sync.mdwn b/doc/sync.mdwn
new file mode 100644
index 0000000000..765c1e43fd
--- /dev/null
+++ b/doc/sync.mdwn
@@ -0,0 +1,37 @@
+The `git annex sync` command provides an easy way to keep several
+repositories in sync. 
+
+Often git is used in a centralized fashion with a central bare repositry
+which changes are pulled and pushed to using normal git commands.
+That works fine, if you don't mind having a central repository.
+
+But it can be harder to use git in a fully decentralized fashion, with no
+central repository and still keep repositories in sync with one another.
+You have to remember to pull from each remote, and merge the appopriate
+branch after pulling. It's difficult to *push* to a remote, since git does
+not allow pushes into the currently checked out branch.
+
+`git annex sync` makes it easier using a scheme devised by Joachim
+Breitner. The idea is to have a branch `synced/master` (actually,
+`synced/$currentbranch`), that is never directly checked out, and serves
+as a drop-point for other repositories to use to push changes.
+
+When you run `git annex sync`, it merges the `synced/master` branch
+into `master`, receiving anything that's been pushed to it. Then it
+fetches from each remote, and merges in any changes that have been made
+to the remotes too. Finally, it updates `synced/master` to reflect the new
+state of `master`, and pushes it out to each of the remotes.
+
+This way, changes propigate around between repositories as `git annex sync`
+is run on each of them. Every repository does not need to be able to talk
+to every other repository; as long as the graph of repositories is
+connected, and `git annex sync` is run from time to time on each, a given
+change, made anywhere, will eventually reach every other repository.
+
+The workflow for using `git annex sync` is simple:
+
+* Make some changes to files in the repository, using `git-annex`,
+  or anything else.
+* Run `git annex sync` to save the changes.
+* Next time you're working on a different clone of that repository,
+  run `git annex sync` to update it.

From 435a349ef52dd6027f7071db2fb78c449242dc6f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 20:20:25 -0400
Subject: [PATCH 2840/8313] increate spec-constr-count

Was getting SpecConstr warnings on Command.Find
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 93586762a9..cda8fed6b3 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
-GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=5
+GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=8
 
 ifdef PROFILE
 GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)

From 133170fb2314f9eb09343460263fed9bebb5b383 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 31 Dec 2011 00:29:45 +0000
Subject: [PATCH 2841/8313] Added a comment

---
 ..._eb81f824aadc97f098379c5f7e4fba4c._comment | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment b/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment
new file mode 100644
index 0000000000..a94eec638f
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment
@@ -0,0 +1,33 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2011-12-31T00:29:45Z"
+ content="""
+Well a repo url like `gitolite@localhost:testing` puts it in the gitolite user's /~/testing
+
+This worked when I added the gitolite stuff, anyway.. Let's see if it still does:
+
+

+joey@gnu:~/tmp>mkdir g
+joey@gnu:~/tmp>cd g
+joey@gnu:~/tmp/g>git init
+Initialized empty Git repository in /home/joey/tmp/g/.git/
+joey@gnu:~/tmp/g>git annex init
+init  ok
+joey@gnu:~/tmp/g>git remote add test 'gitolite@localhost:testing'
+joey@gnu:~/tmp/g>touch foo
+joey@gnu:~/tmp/g>git annex add foo
+add foo (checksum...) ok
+(Recording state in git...)
+joey@gnu:~/tmp/g>git annex copy foo --to test --debug
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"foo\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"check-attr\",\"annex.numcopies\",\"-z\",\"--stdin\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"show-ref\",\"git-annex\"]
+git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"cat-file\",\"--batch\"]
+Running: ssh [\"-4\",\"gitolite@localhost\",\"git-annex-shell 'configlist' '/~/testing'\"]
+
+
+Still seems right, the ADC's regexp will match this the git-annex shell command.
+"""]]

From 230bc8334bb7a947a61a0e213c51fca18decde1c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 20:30:36 -0400
Subject: [PATCH 2842/8313] typo

---
 .../comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment b/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment
index a94eec638f..c53ce01d94 100644
--- a/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment
+++ b/doc/tips/using_gitolite_with_git-annex/comment_4_eb81f824aadc97f098379c5f7e4fba4c._comment
@@ -8,7 +8,7 @@ Well a repo url like `gitolite@localhost:testing` puts it in the gitolite user's
 
 This worked when I added the gitolite stuff, anyway.. Let's see if it still does:
 
-

+
 joey@gnu:~/tmp>mkdir g
 joey@gnu:~/tmp>cd g
 joey@gnu:~/tmp/g>git init
@@ -27,7 +27,7 @@ git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"show
 git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"show-ref\",\"git-annex\"]
 git [\"--git-dir=/home/joey/tmp/g/.git\",\"--work-tree=/home/joey/tmp/g\",\"cat-file\",\"--batch\"]
 Running: ssh [\"-4\",\"gitolite@localhost\",\"git-annex-shell 'configlist' '/~/testing'\"]
-
+

 
 Still seems right, the ADC's regexp will match this the git-annex shell command.
 """]]

From dd8451f0f8092450049472793c325bcaa35f0fb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 20:40:59 -0400
Subject: [PATCH 2843/8313] update

---
 debian/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index f3e830c013..7b8e347767 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Can now be built with older git versions (before 1.7.7); the resulting
     binary should only be used with old git.
   * Updated to build with monad-control 0.3.
+  * sync: Improved to work well without a central bare repository. 
+    Thanks to Joachim Breitner.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 

From e7d3e546c298add0a39ec1a979d9b1574b9b9b76 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 30 Dec 2011 21:17:36 -0400
Subject: [PATCH 2844/8313] sync --fast: Selects some of the remotes with the
 lowest annex.cost and syncs those, in addition to any specified at the
 command line.

---
 Command/Sync.hs    | 23 +++++++++++++++++++----
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  3 ++-
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index a3450278ce..81b77e5cc1 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -13,14 +13,17 @@ module Command.Sync where
 import Common.Annex
 import Command
 import qualified Remote
+import qualified Annex
 import qualified Annex.Branch
 import qualified Git.Command
 import qualified Git.Branch
 import qualified Git.Config
 import qualified Git.Ref
 import qualified Git
+import qualified Types.Remote
 
 import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Map as M
 
 def :: [Command]
 def = [command "sync" (paramOptional (paramRepeating paramRemote))
@@ -28,9 +31,9 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote))
 
 -- syncing involves several operations, any of which can independantly fail
 seek :: CommandSeek
-seek args = do
+seek rs = do
 	!branch <- currentBranch
-	remotes <- syncRemotes args
+	remotes <- syncRemotes rs
 	return $ concat $
 		[ [ commit ]
 		, [ mergeLocal branch ]
@@ -44,11 +47,23 @@ syncBranch :: Git.Ref -> Git.Ref
 syncBranch = Git.Ref.under "refs/heads/synced/"
 
 syncRemotes :: [String] -> Annex [Remote.Remote Annex]
-syncRemotes [] = filterM hasurl =<< Remote.remoteList
+syncRemotes rs = do
+	fast <- Annex.getState Annex.fast
+	if fast
+		then nub <$> pickfast
+		else wanted
 	where
+		wanted
+			| null rs = filterM hasurl =<< Remote.remoteList
+			| otherwise = listed
+		listed = mapM Remote.byName rs
 		hasurl r = not . null <$> geturl r
 		geturl r = fromRepo $ Git.Config.get ("remote." ++ Remote.name r ++ ".url") ""
-syncRemotes rs = mapM Remote.byName rs
+		pickfast = (++) <$> listed <*> (fastest <$> Remote.remoteList)
+		fastest = fromMaybe [] . headMaybe .
+			map snd . sort . M.toList . costmap
+		costmap = M.fromListWith (++) . map costpair
+		costpair r = (Types.Remote.cost r, [r])
 
 commit :: CommandStart
 commit = do
diff --git a/debian/changelog b/debian/changelog
index 7b8e347767..db447fea4a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -18,6 +18,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Updated to build with monad-control 0.3.
   * sync: Improved to work well without a central bare repository. 
     Thanks to Joachim Breitner.
+  * sync --fast: Selects some of the remotes with the lowest annex.cost
+    and syncs those, in addition to any specified at the command line.
 
  -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index acdb1313da..a0dd3d3f1b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -124,7 +124,8 @@ subdirectories).
 
   Use this command when you want to synchronize the local repository with
   one or more of its remotes. You can specifiy the remotes to sync with;
-  the default is to sync with all remotes.
+  the default is to sync with all remotes. Or specify --fast to sync with
+  the remotes with the lowest annex-cost value.
 
   The sync process involves first committing all local changes, then
   fetching and merging the `synced/master` and the `git-annex` branch

From db55fa1c07f34785399d9d48db42261c18544f5e Mon Sep 17 00:00:00 2001
From: bremner 
Date: Sat, 31 Dec 2011 01:50:50 +0000
Subject: [PATCH 2845/8313] Added a comment: gitolite gets different paths for
 different urls

---
 ..._f688309532d2993630e9e72e87fb9c46._comment | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment b/doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment
new file mode 100644
index 0000000000..052fc90d6b
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_5_f688309532d2993630e9e72e87fb9c46._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="156.34.79.193"
+ subject="gitolite gets different paths for different urls"
+ date="2011-12-31T01:50:49Z"
+ content="""
+I guess there is some path rewriting going in in gitolite proper because if try a url of the form 
+ssh://git@localhost/testing, then it still works with gitolite, but fails with the ADC because 
+the repo is passed as /testing:
+
+Running: ssh [\"git@host\",\"git-annex-shell 'configlist' '/recommend'\"]
+Running: ssh [\"git@host\",\"git-annex-shell 'configlist' '/recommend'\"]
+

+
+What I have to ask Sitaram and or find in the docs is if this is a bug or a feature in gitolite.  I can see how the leading slash would get swallowed up by this line 
+
+$repo = \"'$REPO_BASE/$repo.git'\"
+

+in gl-auth-command, but I guess that isn't the whole story.
+"""]]

From e073add830609850ecd70a3db81b71d7b84cfbc4 Mon Sep 17 00:00:00 2001
From: bremner 
Date: Sat, 31 Dec 2011 03:34:19 +0000
Subject: [PATCH 2846/8313] Added a comment: ssh://gitolite-host/repo-name is
 supposed to work

---
 ..._3e203e010a4df5bf03899f867718adc5._comment | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment b/doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment
new file mode 100644
index 0000000000..ce888cb135
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_6_3e203e010a4df5bf03899f867718adc5._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="156.34.79.193"
+ subject="ssh://gitolite-host/repo-name is supposed to work"
+ date="2011-12-31T03:34:17Z"
+ content="""
+I confirmed with Sitaram that this is intentional, if probably under-documented.
+Since the ADC strips the leading /~/ in assigning $start anyway, I guess something like the following will  work 
+
+
+diff --git a/contrib/adc/git-annex-shell b/contrib/adc/git-annex-shell
+index 7f9f5b8..523dfed 100755
+--- a/contrib/adc/git-annex-shell
++++ b/contrib/adc/git-annex-shell
+@@ -28,7 +28,7 @@ my $cmd=$ENV{SSH_ORIGINAL_COMMAND};
+ # the second parameter.
+ # Further parameters are not validated here (see below).
+ die \"bad git-annex-shell command: $cmd\"
+-    unless $cmd =~ m#^(git-annex-shell '\w+' ')/\~/([0-9a-zA-Z][0-9a-zA-Z._\@/+-
++    unless $cmd =~ m#^(git-annex-shell '\w+' ')/(?:\~\/)?([0-9a-zA-Z][0-9a-zA-Z.
+ my $start = $1;
+ my $repo = $2;
+ my $end = $3;
+

+"""]]

From 015a497914d0614afff705d5f9acd79d956740ec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 01:42:42 -0400
Subject: [PATCH 2847/8313] avoid syncing remotes configured annex-ignore,
 unless explicitly specified

---
 Command/Sync.hs | 4 ++--
 Remote.hs       | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 81b77e5cc1..d8eb0fc391 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -54,12 +54,12 @@ syncRemotes rs = do
 		else wanted
 	where
 		wanted
-			| null rs = filterM hasurl =<< Remote.remoteList
+			| null rs = filterM hasurl =<< Remote.enabledRemoteList
 			| otherwise = listed
 		listed = mapM Remote.byName rs
 		hasurl r = not . null <$> geturl r
 		geturl r = fromRepo $ Git.Config.get ("remote." ++ Remote.name r ++ ".url") ""
-		pickfast = (++) <$> listed <*> (fastest <$> Remote.remoteList)
+		pickfast = (++) <$> listed <*> (fastest <$> Remote.enabledRemoteList)
 		fastest = fromMaybe [] . headMaybe .
 			map snd . sort . M.toList . costmap
 		costmap = M.fromListWith (++) . map costpair
diff --git a/Remote.hs b/Remote.hs
index 643ba2d0d1..94c301192d 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -17,6 +17,7 @@ module Remote (
 
 	remoteTypes,
 	remoteList,
+	enabledRemoteList,
 	remoteMap,
 	byName,
 	prettyPrintUUIDs,
@@ -85,6 +86,10 @@ remoteList = do
 			u <- getRepoUUID r
 			generate t r u (M.lookup u m)
 
+{- All remotes that are not ignored. -}
+enabledRemoteList :: Annex [Remote Annex]
+enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
+
 {- Map of UUIDs of Remotes and their names. -}
 remoteMap :: Annex (M.Map UUID String)
 remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> remoteList
@@ -196,7 +201,7 @@ keyPossibilities' withtrusted key = do
 	let validtrusteduuids = validuuids `intersect` trusted
 
 	-- remotes that match uuids that have the key
-	allremotes <- filterM (repoNotIgnored . repo) =<< remoteList
+	allremotes <- enabledRemoteList
 	let validremotes = remotesWithUUID allremotes validuuids
 
 	return (sort validremotes, validtrusteduuids)

From 79231bcff07fce6c08f2266b1e34e8090b12233e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 01:51:39 -0400
Subject: [PATCH 2848/8313] minor cleanups

mergeFrom is never called on branches that don't exist anymore
---
 Command/Sync.hs | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index d8eb0fc391..052f54c444 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -86,7 +86,7 @@ mergeLocal branch = go =<< needmerge
 		go False = stop
 		go True = do
 			showStart "merge" $ Git.Ref.describe syncbranch
-			next $ next $ mergeFromIfExists syncbranch
+			next $ next $ mergeFrom syncbranch
 
 pushLocal :: Git.Ref -> CommandStart
 pushLocal branch = go =<< inRepo (Git.Ref.exists syncbranch)
@@ -106,17 +106,10 @@ updateBranch syncbranch =
 			, Param $ show $ Git.Ref.base syncbranch
 			]
 
-mergeFromIfExists :: Git.Ref -> CommandCleanup
-mergeFromIfExists branch = go =<< inRepo (Git.Ref.exists branch)
-	where
-		go True = do
-			showOutput
-			inRepo $ Git.Command.runBool "merge"
-				[Param (show branch)]
-		go False = do
-			showNote $ Git.Ref.describe branch ++
-				" does not exist, not merging"
-			return False
+mergeFrom :: Git.Ref -> CommandCleanup
+mergeFrom branch = do
+	showOutput
+	inRepo $ Git.Command.runBool "merge" [Param $ show branch]
 
 pullRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pullRemote remote branch = do
@@ -134,14 +127,15 @@ pullRemote remote branch = do
  - were committed, while the synced/master may have changes that some
  - other remote synced to this remote. So, merge them both. -}
 mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
-mergeRemote remote branch = all (== True) <$> mapM go [branch, syncBranch branch]
+mergeRemote remote branch = all id <$> mapM go [branch, syncBranch branch]
 	where
 		go b = do
 			e <- inRepo $ Git.Branch.changed branch b
 			if e
-				then mergeFromIfExists $ remotebranch b
+				then mergeFrom $ remotebranch b
 				else return True
-		remotebranch = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
+		remotebranch = Git.Ref.under $
+			"refs/remotes/" ++ Remote.name remote
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pushRemote remote branch = go =<< needpush

From f2b584ad740a95fb6ef53ae48c5a0e5fbc5c2532 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 02:03:39 -0400
Subject: [PATCH 2849/8313] fix check that remote branch needs merged

---
 Command/Sync.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 052f54c444..4fe5e5bc4c 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -130,8 +130,8 @@ mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
 mergeRemote remote branch = all id <$> mapM go [branch, syncBranch branch]
 	where
 		go b = do
-			e <- inRepo $ Git.Branch.changed branch b
-			if e
+			c <- inRepo $ Git.Branch.changed b (remotebranch b)
+			if c
 				then mergeFrom $ remotebranch b
 				else return True
 		remotebranch = Git.Ref.under $

From 0396f9c795749bc1ef14b26bdc1e1165b9c842b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 02:15:13 -0400
Subject: [PATCH 2850/8313] tweak

---
 Command/Sync.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 4fe5e5bc4c..e29f2893fc 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -140,9 +140,7 @@ mergeRemote remote branch = all id <$> mapM go [branch, syncBranch branch]
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pushRemote remote branch = go =<< needpush
 	where
-		needpush = (||)
-			<$> newer syncbranch
-			<*> newer Annex.Branch.name
+		needpush = anyM newer [syncbranch, Annex.Branch.name]
 		newer b = do
 			let r = remotebranch b
 			e <- inRepo (Git.Ref.exists r)

From 9a7a77488e3a02c5c834b3880e48a48fe3ea160c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 02:18:16 -0400
Subject: [PATCH 2851/8313] tweak

---
 Command/Sync.hs | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index e29f2893fc..29211be92e 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -27,9 +27,9 @@ import qualified Data.Map as M
 
 def :: [Command]
 def = [command "sync" (paramOptional (paramRepeating paramRemote))
-	[seek] "synchronize local repository with remote repositories"]
+	[seek] "synchronize local repository with remotes"]
 
--- syncing involves several operations, any of which can independantly fail
+-- syncing involves several operations, any of which can independently fail
 seek :: CommandSeek
 seek rs = do
 	!branch <- currentBranch
@@ -106,11 +106,6 @@ updateBranch syncbranch =
 			, Param $ show $ Git.Ref.base syncbranch
 			]
 
-mergeFrom :: Git.Ref -> CommandCleanup
-mergeFrom branch = do
-	showOutput
-	inRepo $ Git.Command.runBool "merge" [Param $ show branch]
-
 pullRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pullRemote remote branch = do
 	showStart "pull" (Remote.name remote)
@@ -161,11 +156,16 @@ pushRemote remote branch = go =<< needpush
 		syncbranch = syncBranch branch
 		remotebranch = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
 
-currentBranch :: Annex Git.Ref
-currentBranch = Git.Ref . firstLine . L.unpack <$>
-	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
-
 mergeAnnex :: CommandStart
 mergeAnnex = do
 	Annex.Branch.forceUpdate
 	stop
+
+currentBranch :: Annex Git.Ref
+currentBranch = Git.Ref . firstLine . L.unpack <$>
+	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
+
+mergeFrom :: Git.Ref -> CommandCleanup
+mergeFrom branch = do
+	showOutput
+	inRepo $ Git.Command.runBool "merge" [Param $ show branch]

From 2998340abb6bd4f9c79ef0079f914955f73f5177 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 02:45:12 -0400
Subject: [PATCH 2852/8313] really fix check that remote needs merged

---
 Command/Sync.hs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 29211be92e..2a836ce98d 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -122,15 +122,17 @@ pullRemote remote branch = do
  - were committed, while the synced/master may have changes that some
  - other remote synced to this remote. So, merge them both. -}
 mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
-mergeRemote remote branch = all id <$> mapM go [branch, syncBranch branch]
+mergeRemote remote branch = all id <$> (mapM merge =<< tomerge)
 	where
-		go b = do
-			c <- inRepo $ Git.Branch.changed b (remotebranch b)
-			if c
-				then mergeFrom $ remotebranch b
-				else return True
 		remotebranch = Git.Ref.under $
 			"refs/remotes/" ++ Remote.name remote
+		merge = mergeFrom . remotebranch
+		tomerge = filterM changed [branch, syncBranch branch]
+		changed b = do
+			e <- inRepo $ Git.Ref.exists $ remotebranch b
+			if e
+				then inRepo $ Git.Branch.changed b $ remotebranch b
+				else return False
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pushRemote remote branch = go =<< needpush

From aa64b8ceaf9c1566990ce3bab93b913deee64741 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 03:01:18 -0400
Subject: [PATCH 2853/8313] refactor

---
 Command/Sync.hs | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 2a836ce98d..01c8f22dee 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -46,6 +46,9 @@ seek rs = do
 syncBranch :: Git.Ref -> Git.Ref
 syncBranch = Git.Ref.under "refs/heads/synced/"
 
+remoteBranch :: Remote.Remote Annex -> Git.Ref -> Git.Ref
+remoteBranch remote = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
+
 syncRemotes :: [String] -> Annex [Remote.Remote Annex]
 syncRemotes rs = do
 	fast <- Annex.getState Annex.fast
@@ -124,26 +127,13 @@ pullRemote remote branch = do
 mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
 mergeRemote remote branch = all id <$> (mapM merge =<< tomerge)
 	where
-		remotebranch = Git.Ref.under $
-			"refs/remotes/" ++ Remote.name remote
-		merge = mergeFrom . remotebranch
-		tomerge = filterM changed [branch, syncBranch branch]
-		changed b = do
-			e <- inRepo $ Git.Ref.exists $ remotebranch b
-			if e
-				then inRepo $ Git.Branch.changed b $ remotebranch b
-				else return False
+		merge = mergeFrom . remoteBranch remote
+		tomerge = filterM (changed remote) [branch, syncBranch branch]
 
 pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
 pushRemote remote branch = go =<< needpush
 	where
-		needpush = anyM newer [syncbranch, Annex.Branch.name]
-		newer b = do
-			let r = remotebranch b
-			e <- inRepo (Git.Ref.exists r)
-			if e
-				then inRepo $ Git.Branch.changed r b
-				else return True
+		needpush = anyM (newer remote) [syncbranch, Annex.Branch.name]
 		go False = stop
 		go True = do
 			showStart "push" (Remote.name remote)
@@ -156,7 +146,6 @@ pushRemote remote branch = go =<< needpush
 					]
 		refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
 		syncbranch = syncBranch branch
-		remotebranch = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
 
 mergeAnnex :: CommandStart
 mergeAnnex = do
@@ -171,3 +160,19 @@ mergeFrom :: Git.Ref -> CommandCleanup
 mergeFrom branch = do
 	showOutput
 	inRepo $ Git.Command.runBool "merge" [Param $ show branch]
+
+changed :: Remote.Remote Annex -> Git.Ref -> Annex Bool
+changed remote b = do
+	let r = remoteBranch remote b
+	e <- inRepo $ Git.Ref.exists r
+	if e
+		then inRepo $ Git.Branch.changed b r
+		else return False
+
+newer :: Remote.Remote Annex -> Git.Ref -> Annex Bool
+newer remote b = do
+	let r = remoteBranch remote b
+	e <- inRepo $ Git.Ref.exists r
+	if e
+		then inRepo $ Git.Branch.changed r b
+		else return True

From c61642ef0c0203e2e4b698fc7efd6078521a28d0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 03:08:41 -0400
Subject: [PATCH 2854/8313] remove unnecessary check

mergeLocal always creates the local sync branch, so no need to check that
it exists later.
---
 Command/Sync.hs | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 01c8f22dee..cd8ac389dd 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -92,13 +92,9 @@ mergeLocal branch = go =<< needmerge
 			next $ next $ mergeFrom syncbranch
 
 pushLocal :: Git.Ref -> CommandStart
-pushLocal branch = go =<< inRepo (Git.Ref.exists syncbranch)
-	where
-		syncbranch = syncBranch branch
-		go False = stop
-		go True = do
-			updateBranch syncbranch
-			stop
+pushLocal branch = do
+	updateBranch $ syncBranch branch
+	stop
 
 updateBranch :: Git.Ref -> Annex ()
 updateBranch syncbranch = 

From 6cd4c7efcdea9a8897aa6b9e2b30e7e3426574bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 03:14:05 -0400
Subject: [PATCH 2855/8313] never pick special remotes in --fast

even if they have the lowest cost, we cannot use them
---
 Command/Sync.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index cd8ac389dd..f3a83517c1 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -56,13 +56,14 @@ syncRemotes rs = do
 		then nub <$> pickfast
 		else wanted
 	where
+		pickfast = (++) <$> listed <*> (fastest <$> available)
 		wanted
-			| null rs = filterM hasurl =<< Remote.enabledRemoteList
+			| null rs = available
 			| otherwise = listed
 		listed = mapM Remote.byName rs
+		available = filterM hasurl =<< Remote.enabledRemoteList
 		hasurl r = not . null <$> geturl r
 		geturl r = fromRepo $ Git.Config.get ("remote." ++ Remote.name r ++ ".url") ""
-		pickfast = (++) <$> listed <*> (fastest <$> Remote.enabledRemoteList)
 		fastest = fromMaybe [] . headMaybe .
 			map snd . sort . M.toList . costmap
 		costmap = M.fromListWith (++) . map costpair

From 8a33573caff38b557fdf60c9547a78a5cc8c4ddc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 03:27:37 -0400
Subject: [PATCH 2856/8313] better filtering out of special remotes

---
 Command/Sync.hs     | 7 +++----
 Remote/Bup.hs       | 3 ++-
 Remote/Directory.hs | 3 ++-
 Remote/Git.hs       | 3 ++-
 Remote/Hook.hs      | 3 ++-
 Remote/Rsync.hs     | 3 ++-
 Remote/S3real.hs    | 3 ++-
 Remote/Web.hs       | 3 ++-
 Types/Remote.hs     | 7 ++++++-
 9 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index f3a83517c1..6e78543ef0 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -17,10 +17,10 @@ import qualified Annex
 import qualified Annex.Branch
 import qualified Git.Command
 import qualified Git.Branch
-import qualified Git.Config
 import qualified Git.Ref
 import qualified Git
 import qualified Types.Remote
+import qualified Remote.Git
 
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
@@ -61,9 +61,8 @@ syncRemotes rs = do
 			| null rs = available
 			| otherwise = listed
 		listed = mapM Remote.byName rs
-		available = filterM hasurl =<< Remote.enabledRemoteList
-		hasurl r = not . null <$> geturl r
-		geturl r = fromRepo $ Git.Config.get ("remote." ++ Remote.name r ++ ".url") ""
+		available = filter nonspecial <$> Remote.enabledRemoteList
+		nonspecial r = Types.Remote.remotetype r == Remote.Git.remote
 		fastest = fromMaybe [] . headMaybe .
 			map snd . sort . M.toList . costmap
 		costmap = M.fromListWith (++) . map costpair
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index cbd5d584ac..9311d18e5f 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -54,7 +54,8 @@ gen r u c = do
 			hasKey = checkPresent r bupr',
 			hasKeyCheap = bupLocal buprepo,
 			config = c,
-			repo = r
+			repo = r,
+			remotetype = remote
 		}
 
 bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 7f78b2f493..e1b17c2411 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -45,7 +45,8 @@ gen r u c = do
 			hasKey = checkPresent dir,
 			hasKeyCheap = True,
 			config = Nothing,
-			repo = r
+			repo = r,
+			remotetype = remote
 		}
 
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e527fa4fee..2f2be5bee1 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -79,7 +79,8 @@ gen r u _ = do
 		hasKey = inAnnex r',
 		hasKeyCheap = cheap,
 		config = Nothing,
-		repo = r'
+		repo = r',
+		remotetype = remote
 	}
 
 {- Tries to read the config for a specified remote, updates state, and
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 5c761f43b0..08be1a0ee2 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -45,7 +45,8 @@ gen r u c = do
 			hasKey = checkPresent r hooktype,
 			hasKeyCheap = False,
 			config = Nothing,
-			repo = r
+			repo = r,
+			remotetype = remote
 		}
 
 hookSetup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 68566c52a5..91a72e88e0 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -52,7 +52,8 @@ gen r u c = do
 			hasKey = checkPresent r o,
 			hasKeyCheap = False,
 			config = Nothing,
-			repo = r
+			repo = r,
+			remotetype = remote
 		}
 
 genRsyncOpts :: Git.Repo -> Annex RsyncOpts
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index b79939b902..23a5897265 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -57,7 +57,8 @@ gen' r u c cst =
 			hasKey = checkPresent this,
 			hasKeyCheap = False,
 			config = c,
-			repo = r
+			repo = r,
+			remotetype = remote
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/Web.hs b/Remote/Web.hs
index e31539f885..c0d54132a1 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -43,7 +43,8 @@ gen r _ _ =
 		hasKey = checkKey,
 		hasKeyCheap = False,
 		config = Nothing,
-		repo = r
+		repo = r,
+		remotetype = remote
 	}
 
 downloadKey :: Key -> FilePath -> Annex Bool
diff --git a/Types/Remote.hs b/Types/Remote.hs
index ec9b7a7a70..3a8a23f314 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -30,6 +30,9 @@ data RemoteType a = RemoteType {
 	setup :: UUID -> RemoteConfig -> a RemoteConfig
 }
 
+instance Eq (RemoteType a) where
+	x == y = typename x == typename y
+
 {- An individual remote. -}
 data Remote a = Remote {
 	-- each Remote has a unique uuid
@@ -53,7 +56,9 @@ data Remote a = Remote {
 	-- a Remote can have a persistent configuration store
 	config :: Maybe RemoteConfig,
 	-- git configuration for the remote
-	repo :: Git.Repo
+	repo :: Git.Repo,
+	-- the type of the remote
+	remotetype :: RemoteType a
 }
 
 instance Show (Remote a) where

From a2ec2d3760f5ae17836ade3b0238dde7f9de5bd2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 03:38:58 -0400
Subject: [PATCH 2857/8313] refactor and check for a detached HEAD

---
 Command/Sync.hs | 9 +++------
 Git/Branch.hs   | 8 ++++++++
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 6e78543ef0..9426b1c001 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -22,7 +22,6 @@ import qualified Git
 import qualified Types.Remote
 import qualified Remote.Git
 
-import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 
 def :: [Command]
@@ -32,7 +31,7 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote))
 -- syncing involves several operations, any of which can independently fail
 seek :: CommandSeek
 seek rs = do
-	!branch <- currentBranch
+	!branch <- fromMaybe nobranch <$> inRepo (Git.Branch.current)
 	remotes <- syncRemotes rs
 	return $ concat $
 		[ [ commit ]
@@ -42,6 +41,8 @@ seek rs = do
 		, [ pushLocal branch ]
 		, [ pushRemote remote branch | remote <- remotes ]
 		]
+	where
+		nobranch = error "no branch is checked out"
 
 syncBranch :: Git.Ref -> Git.Ref
 syncBranch = Git.Ref.under "refs/heads/synced/"
@@ -148,10 +149,6 @@ mergeAnnex = do
 	Annex.Branch.forceUpdate
 	stop
 
-currentBranch :: Annex Git.Ref
-currentBranch = Git.Ref . firstLine . L.unpack <$>
-	inRepo (Git.Command.pipeRead [Param "symbolic-ref", Param "HEAD"])
-
 mergeFrom :: Git.Ref -> CommandCleanup
 mergeFrom branch = do
 	showOutput
diff --git a/Git/Branch.hs b/Git/Branch.hs
index cce56dcfa4..98811a9876 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -14,6 +14,14 @@ import Git
 import Git.Sha
 import Git.Command
 
+{- The currently checked out branch. -}
+current :: Repo -> IO (Maybe Git.Ref)
+current r = parse <$> pipeRead [Param "symbolic-ref", Param "HEAD"] r
+	where
+		parse v
+			| L.null v = Nothing
+			| otherwise = Just $ Git.Ref $ firstLine $ L.unpack v
+
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
 changed :: Branch -> Branch -> Repo -> IO Bool

From 4a02c2ea629e1825c824bcc09449806b12408699 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 04:11:39 -0400
Subject: [PATCH 2858/8313] type alias cleanup

---
 Annex.hs                     |  4 ++--
 Backend.hs                   | 16 ++++++++--------
 Backend/SHA.hs               |  8 ++++----
 Backend/URL.hs               |  6 +++---
 Backend/WORM.hs              |  6 +++---
 Command.hs                   |  4 ++--
 Command/Copy.hs              |  2 +-
 Command/Drop.hs              | 14 +++++++-------
 Command/Find.hs              |  2 +-
 Command/Fix.hs               |  2 +-
 Command/Fsck.hs              |  8 ++++----
 Command/Get.hs               |  2 +-
 Command/InitRemote.hs        |  4 ++--
 Command/Migrate.hs           |  4 ++--
 Command/Move.hs              | 12 ++++++------
 Command/Reinject.hs          |  4 ++--
 Command/Sync.hs              | 14 +++++++-------
 Command/Unannex.hs           |  2 +-
 Command/Uninit.hs            |  2 +-
 Command/Unlock.hs            |  2 +-
 Command/Unused.hs            |  6 +++---
 Command/Whereis.hs           |  2 +-
 Remote.hs                    | 24 ++++++++++++------------
 Remote/Bup.hs                |  4 ++--
 Remote/Directory.hs          |  4 ++--
 Remote/Git.hs                |  4 ++--
 Remote/Helper/Encryptable.hs |  4 ++--
 Remote/Hook.hs               |  4 ++--
 Remote/Rsync.hs              |  4 ++--
 Remote/S3real.hs             | 26 +++++++++++++-------------
 Remote/S3stub.hs             |  2 +-
 Remote/Web.hs                |  4 ++--
 Types.hs                     |  9 ++++++++-
 Types/Backend.hs             |  6 +++---
 Types/Remote.hs              | 16 ++++++++--------
 Upgrade/V0.hs                |  2 +-
 Upgrade/V1.hs                |  2 +-
 test.hs                      | 10 +++++-----
 38 files changed, 129 insertions(+), 122 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 8f8936937e..22fd1b8748 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -67,8 +67,8 @@ data OutputType = NormalOutput | QuietOutput | JSONOutput
 -- internal state storage
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
-	, backends :: [Backend Annex]
-	, remotes :: [Types.Remote.Remote Annex]
+	, backends :: [BackendA Annex]
+	, remotes :: [Types.Remote.RemoteA Annex]
 	, repoqueue :: Git.Queue.Queue
 	, output :: OutputType
 	, force :: Bool
diff --git a/Backend.hs b/Backend.hs
index 2f788fcd00..003d62bfcd 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -31,11 +31,11 @@ import qualified Backend.SHA
 import qualified Backend.WORM
 import qualified Backend.URL
 
-list :: [Backend Annex]
+list :: [Backend]
 list = Backend.SHA.backends ++ Backend.WORM.backends ++ Backend.URL.backends
 
 {- List of backends in the order to try them when storing a new key. -}
-orderedList :: Annex [Backend Annex]
+orderedList :: Annex [Backend]
 orderedList = do
 	l <- Annex.getState Annex.backends -- list is cached here
 	if not $ null l
@@ -54,12 +54,12 @@ orderedList = do
 
 {- Generates a key for a file, trying each backend in turn until one
  - accepts it. -}
-genKey :: FilePath -> Maybe (Backend Annex) -> Annex (Maybe (Key, Backend Annex))
+genKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend))
 genKey file trybackend = do
 	bs <- orderedList
 	let bs' = maybe bs (: bs) trybackend
 	genKey' bs' file
-genKey' :: [Backend Annex] -> FilePath -> Annex (Maybe (Key, Backend Annex))
+genKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend))
 genKey' [] _ = return Nothing
 genKey' (b:bs) file = do
 	r <- (B.getKey b) file
@@ -75,7 +75,7 @@ genKey' (b:bs) file = do
 
 {- Looks up the key and backend corresponding to an annexed file,
  - by examining what the file symlinks to. -}
-lookupFile :: FilePath -> Annex (Maybe (Key, Backend Annex))
+lookupFile :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile file = do
 	tl <- liftIO $ try getsymlink
 	case tl of
@@ -94,7 +94,7 @@ lookupFile file = do
 						bname ++ ")"
 					return Nothing
 
-type BackendFile = (Maybe (Backend Annex), FilePath)
+type BackendFile = (Maybe Backend, FilePath)
 
 {- Looks up the backends that should be used for each file in a list.
  - That can be configured on a per-file basis in the gitattributes file.
@@ -110,11 +110,11 @@ chooseBackends fs = Annex.getState Annex.forcebackend >>= go
 			return $ map (\f -> (Just $ Prelude.head l, f)) fs
 
 {- Looks up a backend by name. May fail if unknown. -}
-lookupBackendName :: String -> Backend Annex
+lookupBackendName :: String -> Backend
 lookupBackendName s = fromMaybe unknown $ maybeLookupBackendName s
 	where
 		unknown = error $ "unknown backend " ++ s
-maybeLookupBackendName :: String -> Maybe (Backend Annex)
+maybeLookupBackendName :: String -> Maybe Backend
 maybeLookupBackendName s = headMaybe matches
 	where
 		matches = filter (\b -> s == B.name b) list
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index eca312944e..a1124dfe2e 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -21,21 +21,21 @@ type SHASize = Int
 sizes :: [Int]
 sizes = [256, 1, 512, 224, 384]
 
-backends :: [Backend Annex]
+backends :: [Backend]
 backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes
 
-genBackend :: SHASize -> Maybe (Backend Annex)
+genBackend :: SHASize -> Maybe Backend
 genBackend size
 	| isNothing (shaCommand size) = Nothing
 	| otherwise = Just b
 	where
-		b = Types.Backend.Backend
+		b = Backend
 			{ name = shaName size
 			, getKey = keyValue size
 			, fsckKey = checkKeyChecksum size
 			}
 
-genBackendE :: SHASize -> Maybe (Backend Annex)
+genBackendE :: SHASize -> Maybe Backend
 genBackendE size =
 	case genBackend size of
 		Nothing -> Nothing
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 32a72335a5..7f621b00f2 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -14,11 +14,11 @@ import Common.Annex
 import Types.Backend
 import Types.Key
 
-backends :: [Backend Annex]
+backends :: [Backend]
 backends = [backend]
 
-backend :: Backend Annex
-backend = Types.Backend.Backend {
+backend :: Backend
+backend = Backend {
 	name = "URL",
 	getKey = const (return Nothing),
 	fsckKey = const (return True)
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index 5a3e2d694c..ae9833e30c 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -11,11 +11,11 @@ import Common.Annex
 import Types.Backend
 import Types.Key
 
-backends :: [Backend Annex]
+backends :: [Backend]
 backends = [backend]
 
-backend :: Backend Annex
-backend = Types.Backend.Backend {
+backend :: Backend
+backend = Backend {
 	name = "WORM",
 	getKey = keyValue,
 	fsckKey = const (return True)
diff --git a/Command.hs b/Command.hs
index 813a239cb0..dea6a97a3e 100644
--- a/Command.hs
+++ b/Command.hs
@@ -77,10 +77,10 @@ doCommand = start
 
 {- Modifies an action to only act on files that are already annexed,
  - and passes the key and backend on to it. -}
-whenAnnexed :: (FilePath -> (Key, Backend Annex) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
+whenAnnexed :: (FilePath -> (Key, Backend) -> Annex (Maybe a)) -> FilePath -> Annex (Maybe a)
 whenAnnexed a file = ifAnnexed file (a file) (return Nothing)
 
-ifAnnexed :: FilePath -> ((Key, Backend Annex) -> Annex a) -> Annex a -> Annex a
+ifAnnexed :: FilePath -> ((Key, Backend) -> Annex a) -> Annex a -> Annex a
 ifAnnexed file yes no = maybe no yes =<< Backend.lookupFile file
 
 notBareRepo :: Annex a -> Annex a
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 16de423acb..77beb4b4f4 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -21,6 +21,6 @@ seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start numcopies file (key, backend) = autoCopies key (<) numcopies $
 	Command.Move.start False file (key, backend)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 0a4c9dfd6f..89e7c8e42a 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -24,7 +24,7 @@ def = [dontCheck fromOpt $ command "drop" paramPaths seek
 seek :: [CommandSeek]
 seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
-start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start numcopies file (key, _) = autoCopies key (>) numcopies $ do
 	from <- Annex.getState Annex.fromremote
 	case from of
@@ -41,7 +41,7 @@ startLocal file numcopies key = stopUnless (inAnnex key) $ do
 	showStart "drop" file
 	next $ performLocal key numcopies
 
-startRemote :: FilePath -> Maybe Int -> Key -> Remote.Remote Annex -> CommandStart
+startRemote :: FilePath -> Maybe Int -> Key -> Remote -> CommandStart
 startRemote file numcopies key remote = do
 	showStart "drop" file
 	next $ performRemote key numcopies remote
@@ -55,7 +55,7 @@ performLocal key numcopies = lockContent key $ do
 		whenM (inAnnex key) $ removeAnnex key
 		next $ cleanupLocal key
 
-performRemote :: Key -> Maybe Int -> Remote.Remote Annex -> CommandPerform
+performRemote :: Key -> Maybe Int -> Remote -> CommandPerform
 performRemote key numcopies remote = lockContent key $ do
 	-- Filter the remote it's being dropped from out of the lists of
 	-- places assumed to have the key, and places to check.
@@ -79,7 +79,7 @@ cleanupLocal key = do
 	logStatus key InfoMissing
 	return True
 
-cleanupRemote :: Key -> Remote.Remote Annex -> Bool -> CommandCleanup
+cleanupRemote :: Key -> Remote -> Bool -> CommandCleanup
 cleanupRemote key remote ok = do
 	-- better safe than sorry: assume the remote dropped the key
 	-- even if it seemed to fail; the failure could have occurred
@@ -90,7 +90,7 @@ cleanupRemote key remote ok = do
 {- Checks specified remotes to verify that enough copies of a key exist to
  - allow it to be safely removed (with no data loss). Can be provided with
  - some locations where the key is known/assumed to be present. -}
-canDropKey :: Key -> Maybe Int -> [UUID] -> [Remote.Remote Annex] -> [UUID] -> Annex Bool
+canDropKey :: Key -> Maybe Int -> [UUID] -> [Remote] -> [UUID] -> Annex Bool
 canDropKey key numcopiesM have check skip = do
 	force <- Annex.getState Annex.force
 	if force || numcopiesM == Just 0
@@ -99,7 +99,7 @@ canDropKey key numcopiesM have check skip = do
 			need <- getNumCopies numcopiesM
 			findCopies key need skip have check
 
-findCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote.Remote Annex] -> Annex Bool
+findCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote] -> Annex Bool
 findCopies key need skip = helper []
 	where
 		helper bad have []
@@ -116,7 +116,7 @@ findCopies key need skip = helper []
 					(False, Left _)     -> helper (r:bad) have rs
 					_                   -> helper bad have rs
 
-notEnoughCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote.Remote Annex] -> Annex Bool
+notEnoughCopies :: Key -> Int -> [UUID] -> [UUID] -> [Remote] -> Annex Bool
 notEnoughCopies key need have skip bad = do
 	unsafe
 	showLongNote $
diff --git a/Command/Find.hs b/Command/Find.hs
index 1961e6b748..0c96369ee9 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -24,7 +24,7 @@ def = [command "find" paramPaths seek "lists available files"]
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
diff --git a/Command/Fix.hs b/Command/Fix.hs
index f264106c3f..c4f9813811 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -20,7 +20,7 @@ seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
 
 {- Fixes the symlink to an annexed file. -}
-start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, _) = do
 	link <- calcGitLink file key
 	stopUnless ((/=) link <$> liftIO (readSymbolicLink file)) $ do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index a803207e20..4e83455e1b 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -30,12 +30,12 @@ seek =
 	, withBarePresentKeys startBare
 	]
 
-start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start numcopies file (key, backend) = do
 	showStart "fsck" file
 	next $ perform key file backend numcopies
 
-perform :: Key -> FilePath -> Backend Annex -> Maybe Int -> CommandPerform
+perform :: Key -> FilePath -> Backend -> Maybe Int -> CommandPerform
 perform key file backend numcopies = check
 	-- order matters
 	[ verifyLocationLog key file
@@ -64,7 +64,7 @@ startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName ke
 {- Note that numcopies cannot be checked in a bare repository, because
  - getting the numcopies value requires a working copy with .gitattributes
  - files. -}
-performBare :: Key -> Backend Annex -> CommandPerform
+performBare :: Key -> Backend -> CommandPerform
 performBare key backend = check
 	[ verifyLocationLog key (show key)
 	, checkKeySize key
@@ -136,7 +136,7 @@ checkKeySize key = do
 					return False
 
 
-checkBackend :: Backend Annex -> Key -> Annex Bool
+checkBackend :: Backend -> Key -> Annex Bool
 checkBackend = Types.Backend.fsckKey
 
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
diff --git a/Command/Get.hs b/Command/Get.hs
index b7023e2de8..f2b70baebd 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -21,7 +21,7 @@ def = [dontCheck fromOpt $ command "get" paramPaths seek
 seek :: [CommandSeek]
 seek = [withNumCopies $ \n -> whenAnnexed $ start n]
 
-start :: Maybe Int -> FilePath -> (Key, Backend Annex) -> CommandStart
+start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
 	autoCopies key (<) numcopies $ do
 		from <- Annex.getState Annex.fromremote
diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 1e6bc2ef17..698d604552 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -42,7 +42,7 @@ start (name:ws) = do
 	where
 		config = Logs.Remote.keyValToConfig ws
 
-perform :: R.RemoteType Annex -> UUID -> R.RemoteConfig -> CommandPerform
+perform :: RemoteType -> UUID -> R.RemoteConfig -> CommandPerform
 perform t u c = do
 	c' <- R.setup t u c
 	next $ cleanup u c'
@@ -77,7 +77,7 @@ remoteNames = do
 	return $ mapMaybe (M.lookup nameKey . snd) $ M.toList m
 
 {- find the specified remote type -}
-findType :: R.RemoteConfig -> Annex (R.RemoteType Annex)
+findType :: R.RemoteConfig -> Annex RemoteType
 findType config = maybe unspecified specified $ M.lookup typeKey config
 	where
 		unspecified = error "Specify the type of remote with type="
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 8778743ff5..f6467463d0 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -21,7 +21,7 @@ def = [command "migrate" paramPaths seek "switch data to different backend"]
 seek :: [CommandSeek]
 seek = [withBackendFilesInGit $ \(b, f) -> whenAnnexed (start b) f]
 
-start :: Maybe (Backend Annex) -> FilePath -> (Key, Backend Annex) -> CommandStart
+start :: Maybe Backend -> FilePath -> (Key, Backend) -> CommandStart
 start b file (key, oldbackend) = do
 	exists <- inAnnex key
 	newbackend <- choosebackend b
@@ -47,7 +47,7 @@ upgradableKey key = isNothing $ Types.Key.keySize key
  - backends that allow the filename to influence the keys they
  - generate.
  -}
-perform :: FilePath -> Key -> Backend Annex -> CommandPerform
+perform :: FilePath -> Key -> Backend -> CommandPerform
 perform file oldkey newbackend = do
 	src <- inRepo $ gitAnnexLocation oldkey
 	tmp <- fromRepo gitAnnexTmpDir
diff --git a/Command/Move.hs b/Command/Move.hs
index 85fdff7398..bd1490b0cd 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -23,7 +23,7 @@ def = [dontCheck toOpt $ dontCheck fromOpt $
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed $ start True]
 
-start :: Bool -> FilePath -> (Key, Backend Annex) -> CommandStart
+start :: Bool -> FilePath -> (Key, Backend) -> CommandStart
 start move file (key, _) = do
 	noAuto
 	to <- Annex.getState Annex.toremote
@@ -54,7 +54,7 @@ showMoveAction False file = showStart "copy" file
  - A file's content can be moved even if there are insufficient copies to
  - allow it to be dropped.
  -}
-toStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
+toStart :: Remote -> Bool -> FilePath -> Key -> CommandStart
 toStart dest move file key = do
 	u <- getUUID
 	ishere <- inAnnex key
@@ -63,7 +63,7 @@ toStart dest move file key = do
 		else do
 			showMoveAction move file
 			next $ toPerform dest move key
-toPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
+toPerform :: Remote -> Bool -> Key -> CommandPerform
 toPerform dest move key = moveLock move key $ do
 	-- Checking the remote is expensive, so not done in the start step.
 	-- In fast mode, location tracking is assumed to be correct,
@@ -105,7 +105,7 @@ toPerform dest move key = moveLock move key $ do
  - If the current repository already has the content, it is still removed
  - from the remote.
  -}
-fromStart :: Remote.Remote Annex -> Bool -> FilePath -> Key -> CommandStart
+fromStart :: Remote -> Bool -> FilePath -> Key -> CommandStart
 fromStart src move file key
 	| move = go
 	| otherwise = stopUnless (not <$> inAnnex key) go
@@ -113,12 +113,12 @@ fromStart src move file key
 		go = stopUnless (fromOk src key) $ do
 			showMoveAction move file
 			next $ fromPerform src move key
-fromOk :: Remote.Remote Annex -> Key -> Annex Bool
+fromOk :: Remote -> Key -> Annex Bool
 fromOk src key = do
 	u <- getUUID
 	remotes <- Remote.keyPossibilities key
 	return $ u /= Remote.uuid src && any (== src) remotes
-fromPerform :: Remote.Remote Annex -> Bool -> Key -> CommandPerform
+fromPerform :: Remote -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key
 	if ishere
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 0648e90fca..480806e11c 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -33,7 +33,7 @@ start (src:dest:[])
 			next $ whenAnnexed (perform src) dest
 start _ = error "specify a src file and a dest file"
 
-perform :: FilePath -> FilePath -> (Key, Backend Annex) -> CommandPerform
+perform :: FilePath -> FilePath -> (Key, Backend) -> CommandPerform
 perform src _dest (key, backend) = do
 	unlessM move $ error "mv failed!"
 	next $ cleanup key backend
@@ -45,7 +45,7 @@ perform src _dest (key, backend) = do
 		move = getViaTmp key $ \tmp ->
 			liftIO $ boolSystem "mv" [File src, File tmp]
 
-cleanup :: Key -> Backend Annex -> CommandCleanup
+cleanup :: Key -> Backend -> CommandCleanup
 cleanup key backend = do
 	logStatus key InfoPresent
 
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 9426b1c001..759df36ea3 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -47,10 +47,10 @@ seek rs = do
 syncBranch :: Git.Ref -> Git.Ref
 syncBranch = Git.Ref.under "refs/heads/synced/"
 
-remoteBranch :: Remote.Remote Annex -> Git.Ref -> Git.Ref
+remoteBranch :: Remote -> Git.Ref -> Git.Ref
 remoteBranch remote = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
 
-syncRemotes :: [String] -> Annex [Remote.Remote Annex]
+syncRemotes :: [String] -> Annex [Remote]
 syncRemotes rs = do
 	fast <- Annex.getState Annex.fast
 	if fast
@@ -106,7 +106,7 @@ updateBranch syncbranch =
 			, Param $ show $ Git.Ref.base syncbranch
 			]
 
-pullRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+pullRemote :: Remote -> Git.Ref -> CommandStart
 pullRemote remote branch = do
 	showStart "pull" (Remote.name remote)
 	next $ do
@@ -121,13 +121,13 @@ pullRemote remote branch = do
  - Which to merge from? Well, the master has whatever latest changes
  - were committed, while the synced/master may have changes that some
  - other remote synced to this remote. So, merge them both. -}
-mergeRemote :: Remote.Remote Annex -> Git.Ref -> CommandCleanup
+mergeRemote :: Remote -> Git.Ref -> CommandCleanup
 mergeRemote remote branch = all id <$> (mapM merge =<< tomerge)
 	where
 		merge = mergeFrom . remoteBranch remote
 		tomerge = filterM (changed remote) [branch, syncBranch branch]
 
-pushRemote :: Remote.Remote Annex -> Git.Ref -> CommandStart
+pushRemote :: Remote -> Git.Ref -> CommandStart
 pushRemote remote branch = go =<< needpush
 	where
 		needpush = anyM (newer remote) [syncbranch, Annex.Branch.name]
@@ -154,7 +154,7 @@ mergeFrom branch = do
 	showOutput
 	inRepo $ Git.Command.runBool "merge" [Param $ show branch]
 
-changed :: Remote.Remote Annex -> Git.Ref -> Annex Bool
+changed :: Remote -> Git.Ref -> Annex Bool
 changed remote b = do
 	let r = remoteBranch remote b
 	e <- inRepo $ Git.Ref.exists r
@@ -162,7 +162,7 @@ changed remote b = do
 		then inRepo $ Git.Branch.changed b r
 		else return False
 
-newer :: Remote.Remote Annex -> Git.Ref -> Annex Bool
+newer :: Remote -> Git.Ref -> Annex Bool
 newer remote b = do
 	let r = remoteBranch remote b
 	e <- inRepo $ Git.Ref.exists r
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 66611cbd74..fee67429df 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -22,7 +22,7 @@ def = [command "unannex" paramPaths seek "undo accidential add command"]
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, _) = stopUnless (inAnnex key) $ do
 	showStart "unannex" file
 	next $ perform file key
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 21ad4c7df5..cef89a5cf3 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -36,7 +36,7 @@ check = do
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed startUnannex, withNothing start]
 
-startUnannex :: FilePath -> (Key, Backend Annex) -> CommandStart
+startUnannex :: FilePath -> (Key, Backend) -> CommandStart
 startUnannex file info = do
 	-- Force fast mode before running unannex. This way, if multiple
 	-- files link to a key, it will be left in the annex and hardlinked
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 673a7038a0..afee101459 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -26,7 +26,7 @@ seek = [withFilesInGit $ whenAnnexed start]
 
 {- The unlock subcommand replaces the symlink with a copy of the file's
  - content. -}
-start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, _) = do
 	showStart "unlock" file
 	next $ perform file key
diff --git a/Command/Unused.hs b/Command/Unused.hs
index ef398b01e1..8d45c51cbf 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -66,7 +66,7 @@ checkRemoteUnused name = do
 	checkRemoteUnused' =<< Remote.byName name
 	next $ return True
 
-checkRemoteUnused' :: Remote.Remote Annex -> Annex ()
+checkRemoteUnused' :: Remote -> Annex ()
 checkRemoteUnused' r = do
 	showAction "checking for unused data"
 	remotehas <- loggedKeysFor (Remote.uuid r)
@@ -112,14 +112,14 @@ unusedMsg' u header trailer = unlines $
 	["(To see where data was previously used, try: git log --stat -S'KEY')"] ++
 	trailer
 
-remoteUnusedMsg :: Remote.Remote Annex -> [(Int, Key)] -> String
+remoteUnusedMsg :: Remote -> [(Int, Key)] -> String
 remoteUnusedMsg r u = unusedMsg' u
 	["Some annexed data on " ++ name ++ " is not used by any files:"]
 	[dropMsg $ Just r]
 	where
 		name = Remote.name r 
 
-dropMsg :: Maybe (Remote.Remote Annex) -> String
+dropMsg :: Maybe Remote -> String
 dropMsg Nothing = dropMsg' ""
 dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
 dropMsg' :: String -> String
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index eb2ae3d4e7..9e57f361b8 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -20,7 +20,7 @@ def = [command "whereis" paramPaths seek
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
 
-start :: FilePath -> (Key, Backend Annex) -> CommandStart
+start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, _) = do
 	showStart "whereis" file
 	next $ perform key
diff --git a/Remote.hs b/Remote.hs
index 94c301192d..8046175d27 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -54,7 +54,7 @@ import qualified Remote.Rsync
 import qualified Remote.Web
 import qualified Remote.Hook
 
-remoteTypes :: [RemoteType Annex]
+remoteTypes :: [RemoteType]
 remoteTypes =
 	[ Remote.Git.remote
 	, Remote.S3.remote
@@ -67,7 +67,7 @@ remoteTypes =
 
 {- Builds a list of all available Remotes.
  - Since doing so can be expensive, the list is cached. -}
-remoteList :: Annex [Remote Annex]
+remoteList :: Annex [Remote]
 remoteList = do
 	rs <- Annex.getState Annex.remotes
 	if null rs
@@ -87,7 +87,7 @@ remoteList = do
 			generate t r u (M.lookup u m)
 
 {- All remotes that are not ignored. -}
-enabledRemoteList :: Annex [Remote Annex]
+enabledRemoteList :: Annex [Remote]
 enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
 
 {- Map of UUIDs of Remotes and their names. -}
@@ -96,13 +96,13 @@ remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> remoteList
 
 {- Looks up a remote by name. (Or by UUID.) Only finds currently configured
  - git remotes. -}
-byName :: String -> Annex (Remote Annex)
+byName :: String -> Annex (Remote)
 byName n = do
 	res <- byName' n
 	case res of
 		Left e -> error e
 		Right r -> return r
-byName' :: String -> Annex (Either String (Remote Annex))
+byName' :: String -> Annex (Either String Remote)
 byName' "" = return $ Left "no remote specified"
 byName' n = do
 	match <- filter matching <$> remoteList
@@ -168,16 +168,16 @@ prettyPrintUUIDs desc uuids = do
 			]
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
-remotesWithUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
+remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 
 {- Filters a list of remotes to ones that do not have the listed uuids. -}
-remotesWithoutUUID :: [Remote Annex] -> [UUID] -> [Remote Annex]
+remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
 {- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
  -}
-keyPossibilities :: Key -> Annex [Remote Annex]
+keyPossibilities :: Key -> Annex [Remote]
 keyPossibilities key = fst <$> keyPossibilities' False key
 
 {- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
@@ -185,10 +185,10 @@ keyPossibilities key = fst <$> keyPossibilities' False key
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
-keyPossibilitiesTrusted :: Key -> Annex ([Remote Annex], [UUID])
+keyPossibilitiesTrusted :: Key -> Annex ([Remote], [UUID])
 keyPossibilitiesTrusted = keyPossibilities' True
 
-keyPossibilities' :: Bool -> Key -> Annex ([Remote Annex], [UUID])
+keyPossibilities' :: Bool -> Key -> Annex ([Remote], [UUID])
 keyPossibilities' withtrusted key = do
 	u <- getUUID
 	trusted <- if withtrusted then trustGet Trusted else return []
@@ -224,7 +224,7 @@ showLocations key exclude = do
 		message [] us = "Also these untrusted repositories may contain the file:\n" ++ us
 		message rs us = message rs [] ++ message [] us
 
-showTriedRemotes :: [Remote Annex] -> Annex ()
+showTriedRemotes :: [Remote] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
@@ -240,7 +240,7 @@ forceTrust level remotename = do
  - in the local repo, not on the remote. The process of transferring the
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
-logStatus :: Remote Annex -> Key -> Bool -> Annex ()
+logStatus :: Remote -> Key -> Bool -> Annex ()
 logStatus remote key present = logChange key (uuid remote) status
 	where
 		status = if present then InfoPresent else InfoMissing
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 9311d18e5f..04cd490265 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -26,7 +26,7 @@ import Crypto
 
 type BupRepo = String
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "bup",
 	enumerate = findSpecialRemotes "buprepo",
@@ -34,7 +34,7 @@ remote = RemoteType {
 	setup = bupSetup
 }
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
 	buprepo <- getConfig r "buprepo" (error "missing buprepo")
 	cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index e1b17c2411..8ca2a28750 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -20,7 +20,7 @@ import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "directory",
 	enumerate = findSpecialRemotes "directory",
@@ -28,7 +28,7 @@ remote = RemoteType {
 	setup = directorySetup
 }
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
 	dir <- getConfig r "directory" (error "missing directory")
 	cst <- remoteCost r cheapRemoteCost
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2f2be5bee1..e790d01a75 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -28,7 +28,7 @@ import Utility.TempFile
 import Config
 import Init
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "git",
 	enumerate = list,
@@ -50,7 +50,7 @@ list = do
 					Git.Construct.remoteNamed n $
 						Git.Construct.fromRemoteLocation url g
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u _ = do
  	{- It's assumed to be cheap to read the config of non-URL remotes,
 	 - so this is done each time git-annex is run. Conversely,
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 99f48fe7b0..3abea7bc6a 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -41,8 +41,8 @@ encryptableRemote
 	:: Maybe RemoteConfig
 	-> ((Cipher, Key) -> Key -> Annex Bool)
 	-> ((Cipher, Key) -> FilePath -> Annex Bool)
-	-> Remote Annex 
-	-> Remote Annex
+	-> Remote
+	-> Remote
 encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = 
 	r {
 		storeKey = store,
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 08be1a0ee2..6c4a044ac9 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -20,7 +20,7 @@ import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "hook",
 	enumerate = findSpecialRemotes "hooktype",
@@ -28,7 +28,7 @@ remote = RemoteType {
 	setup = hookSetup
 }
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
 	hooktype <- getConfig r "hooktype" (error "missing hooktype")
 	cst <- remoteCost r expensiveRemoteCost
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 91a72e88e0..2fe302ba52 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -27,7 +27,7 @@ data RsyncOpts = RsyncOpts {
 	rsyncOptions :: [CommandParam]
 }
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "rsync",
 	enumerate = findSpecialRemotes "rsyncurl",
@@ -35,7 +35,7 @@ remote = RemoteType {
 	setup = rsyncSetup
 }
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
 	o <- genRsyncOpts r
 	cst <- remoteCost r expensiveRemoteCost
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index 23a5897265..bef89b5539 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -28,7 +28,7 @@ import Crypto
 import Annex.Content
 import Utility.Base64
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "S3",
 	enumerate = findSpecialRemotes "s3",
@@ -36,11 +36,11 @@ remote = RemoteType {
 	setup = s3Setup
 }
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
 	cst <- remoteCost r expensiveRemoteCost
 	return $ gen' r u c cst
-gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote Annex
+gen' :: Git.Repo -> UUID -> Maybe RemoteConfig -> Int -> Remote
 gen' r u c cst =
 	encryptableRemote c
 		(storeEncrypted this)
@@ -111,13 +111,13 @@ s3Setup u c = handlehost $ M.lookup "host" c
 					-- be human-readable
 					M.delete "bucket" defaults
 
-store :: Remote Annex -> Key -> Annex Bool
+store :: Remote -> Key -> Annex Bool
 store r k = s3Action r False $ \(conn, bucket) -> do
 	dest <- inRepo $ gitAnnexLocation k
 	res <- liftIO $ storeHelper (conn, bucket) r k dest
 	s3Bool res
 
-storeEncrypted :: Remote Annex -> (Cipher, Key) -> Key -> Annex Bool
+storeEncrypted :: Remote -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) -> 
 	-- To get file size of the encrypted content, have to use a temp file.
 	-- (An alternative would be chunking to to a constant size.)
@@ -127,7 +127,7 @@ storeEncrypted r (cipher, enck) k = s3Action r False $ \(conn, bucket) ->
 		res <- liftIO $ storeHelper (conn, bucket) r enck tmp
 		s3Bool res
 
-storeHelper :: (AWSConnection, String) -> Remote Annex -> Key -> FilePath -> IO (AWSResult ())
+storeHelper :: (AWSConnection, String) -> Remote -> Key -> FilePath -> IO (AWSResult ())
 storeHelper (conn, bucket) r k file = do
 	content <- liftIO $ L.readFile file
 	-- size is provided to S3 so the whole content does not need to be
@@ -149,7 +149,7 @@ storeHelper (conn, bucket) r k file = do
 		xheaders = filter isxheader $ M.assocs $ fromJust $ config r
 		isxheader (h, _) = "x-amz-" `isPrefixOf` h
 
-retrieve :: Remote Annex -> Key -> FilePath -> Annex Bool
+retrieve :: Remote -> Key -> FilePath -> Annex Bool
 retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket k
 	case res of
@@ -158,7 +158,7 @@ retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 			return True
 		Left e -> s3Warning e
 
-retrieveEncrypted :: Remote Annex -> (Cipher, Key) -> FilePath -> Annex Bool
+retrieveEncrypted :: Remote -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket enck
 	case res of
@@ -168,12 +168,12 @@ retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
 				return True
 		Left e -> s3Warning e
 
-remove :: Remote Annex -> Key -> Annex Bool
+remove :: Remote -> Key -> Annex Bool
 remove r k = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ deleteObject conn $ bucketKey r bucket k
 	s3Bool res
 
-checkPresent :: Remote Annex -> Key -> Annex (Either String Bool)
+checkPresent :: Remote -> Key -> Annex (Either String Bool)
 checkPresent r k = s3Action r noconn $ \(conn, bucket) -> do
 	showAction $ "checking " ++ name r
 	res <- liftIO $ getObjectInfo conn $ bucketKey r bucket k
@@ -196,7 +196,7 @@ s3Bool :: AWSResult () -> Annex Bool
 s3Bool (Right _) = return True
 s3Bool (Left e) = s3Warning e
 
-s3Action :: Remote Annex -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
+s3Action :: Remote -> a -> ((AWSConnection, String) -> Annex a) -> Annex a
 s3Action r noconn action = do
 	when (isNothing $ config r) $
 		error $ "Missing configuration for special remote " ++ name r
@@ -206,14 +206,14 @@ s3Action r noconn action = do
 		(Just b, Just c) -> action (c, b)
 		_ -> return noconn
 
-bucketFile :: Remote Annex -> Key -> FilePath
+bucketFile :: Remote -> Key -> FilePath
 bucketFile r = munge . show
 	where
 		munge s = case M.lookup "mungekeys" $ fromJust $ config r of
 			Just "ia" -> iaMunge s
 			_ -> s
 
-bucketKey :: Remote Annex -> String -> Key -> S3Object
+bucketKey :: Remote -> String -> Key -> S3Object
 bucketKey r bucket k = S3Object bucket (bucketFile r k) "" [] L.empty
 
 {- Internet Archive limits filenames to a subset of ascii,
diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
index d91a222e86..31e8a339ef 100644
--- a/Remote/S3stub.hs
+++ b/Remote/S3stub.hs
@@ -4,7 +4,7 @@ module Remote.S3 (remote) where
 import Types.Remote
 import Types
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "S3",
 	enumerate = return [],
diff --git a/Remote/Web.hs b/Remote/Web.hs
index c0d54132a1..93e5770f07 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -15,7 +15,7 @@ import Config
 import Logs.Web
 import qualified Utility.Url as Url
 
-remote :: RemoteType Annex
+remote :: RemoteType
 remote = RemoteType {
 	typename = "web",
 	enumerate = list,
@@ -31,7 +31,7 @@ list = do
 	r <- liftIO $ Git.Construct.remoteNamed "web" Git.Construct.fromUnknown
 	return [r]
 
-gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex (Remote Annex)
+gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r _ _ = 
 	return Remote {
 		uuid = webUUID,
diff --git a/Types.hs b/Types.hs
index fd77bfe575..c8839b7ebb 100644
--- a/Types.hs
+++ b/Types.hs
@@ -9,10 +9,17 @@ module Types (
 	Annex,
 	Backend,
 	Key,
-	UUID(..)
+	UUID(..),
+	Remote,
+	RemoteType
 ) where
 
 import Annex
 import Types.Backend
 import Types.Key
 import Types.UUID
+import Types.Remote
+
+type Backend = BackendA Annex
+type Remote = RemoteA Annex
+type RemoteType = RemoteTypeA Annex
diff --git a/Types/Backend.hs b/Types/Backend.hs
index 4f82267045..c9daa46716 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -11,7 +11,7 @@ module Types.Backend where
 
 import Types.Key
 
-data Backend a = Backend {
+data BackendA a = Backend {
 	-- name of this backend
 	name :: String,
 	-- converts a filename to a key
@@ -20,8 +20,8 @@ data Backend a = Backend {
 	fsckKey :: Key -> a Bool
 }
 
-instance Show (Backend a) where
+instance Show (BackendA a) where
 	show backend = "Backend { name =\"" ++ name backend ++ "\" }"
 
-instance Eq (Backend a) where
+instance Eq (BackendA a) where
 	a == b = name a == name b
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 3a8a23f314..e44e2a9de6 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -19,22 +19,22 @@ import Types.UUID
 type RemoteConfig = M.Map String String
 
 {- There are different types of remotes. -}
-data RemoteType a = RemoteType {
+data RemoteTypeA a = RemoteType {
 	-- human visible type name
 	typename :: String,
 	-- enumerates remotes of this type
 	enumerate :: a [Git.Repo],
 	-- generates a remote of this type
-	generate :: Git.Repo -> UUID -> Maybe RemoteConfig -> a (Remote a),
+	generate :: Git.Repo -> UUID -> Maybe RemoteConfig -> a (RemoteA a),
 	-- initializes or changes a remote
 	setup :: UUID -> RemoteConfig -> a RemoteConfig
 }
 
-instance Eq (RemoteType a) where
+instance Eq (RemoteTypeA a) where
 	x == y = typename x == typename y
 
 {- An individual remote. -}
-data Remote a = Remote {
+data RemoteA a = Remote {
 	-- each Remote has a unique uuid
 	uuid :: UUID,
 	-- each Remote has a human visible name
@@ -58,16 +58,16 @@ data Remote a = Remote {
 	-- git configuration for the remote
 	repo :: Git.Repo,
 	-- the type of the remote
-	remotetype :: RemoteType a
+	remotetype :: RemoteTypeA a
 }
 
-instance Show (Remote a) where
+instance Show (RemoteA a) where
 	show remote = "Remote { name =\"" ++ name remote ++ "\" }"
 
 -- two remotes are the same if they have the same uuid
-instance Eq (Remote a) where
+instance Eq (RemoteA a) where
 	x == y = uuid x == uuid y
 
 -- order remotes by cost
-instance Ord (Remote a) where
+instance Ord (RemoteA a) where
 	compare = comparing cost
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index eae5c87ce4..c5310c641a 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -33,7 +33,7 @@ keyFile0 :: Key -> FilePath
 keyFile0 = Upgrade.V1.keyFile1
 fileKey0 :: FilePath -> Key
 fileKey0 = Upgrade.V1.fileKey1
-lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend Annex))
+lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile0 = Upgrade.V1.lookupFile1
 
 getKeysPresent0 :: FilePath -> Annex [Key]
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 80554dc3bc..add50fcf3a 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -181,7 +181,7 @@ writeLog1 file ls = viaTmp writeFile file (showLog ls)
 readLog1 :: FilePath -> IO [LogLine]
 readLog1 file = catchDefaultIO (parseLog <$> readFileStrict file) []
 
-lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend Annex))
+lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile1 file = do
 	tl <- liftIO $ try getsymlink
 	case tl of
diff --git a/test.hs b/test.hs
index a2fa98e4df..7350a07697 100644
--- a/test.hs
+++ b/test.hs
@@ -850,7 +850,7 @@ checklocationlog f expected = do
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"
 
-checkbackend :: FilePath -> Types.Backend Types.Annex -> Assertion			
+checkbackend :: FilePath -> Types.Backend -> Assertion
 checkbackend file expected = do
 	r <- annexeval $ Backend.lookupFile file
 	let b = snd $ fromJust r
@@ -936,14 +936,14 @@ changecontent f = writeFile f $ changedcontent f
 changedcontent :: FilePath -> String
 changedcontent f = (content f) ++ " (modified)"
 
-backendSHA1 :: Types.Backend Types.Annex
+backendSHA1 :: Types.Backend
 backendSHA1 = backend_ "SHA1"
 
-backendSHA256 :: Types.Backend Types.Annex
+backendSHA256 :: Types.Backend
 backendSHA256 = backend_ "SHA256"
 
-backendWORM :: Types.Backend Types.Annex
+backendWORM :: Types.Backend
 backendWORM = backend_ "WORM"
 
-backend_ :: String -> Types.Backend Types.Annex
+backend_ :: String -> Types.Backend
 backend_ name = Backend.lookupBackendName name

From ae6fcb853d59681057a7f23424c321c97c4c0776 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 04:14:33 -0400
Subject: [PATCH 2859/8313] fix comment

---
 Types/Backend.hs | 2 +-
 Types/Remote.hs  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Types/Backend.hs b/Types/Backend.hs
index c9daa46716..025293a906 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -1,6 +1,6 @@
 {- git-annex key/value backend data type
  -
- - Most things should not need this, using Remotes instead
+ - Most things should not need this, using Types instead
  -
  - Copyright 2010 Joey Hess 
  -
diff --git a/Types/Remote.hs b/Types/Remote.hs
index e44e2a9de6..216b34857d 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -1,6 +1,6 @@
 {- git-annex remotes types
  -
- - Most things should not need this, using Remote instead
+ - Most things should not need this, using Types instead
  -
  - Copyright 2011 Joey Hess 
  -

From 25e4b116c7ab436492a86236f796310d405517a1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 04:19:10 -0400
Subject: [PATCH 2860/8313] type alias

---
 Annex.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Annex.hs b/Annex.hs
index 22fd1b8748..91d374aec3 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -64,6 +64,8 @@ instance MonadBaseControl IO Annex where
 
 data OutputType = NormalOutput | QuietOutput | JSONOutput
 
+type Matcher a = Either [Utility.Matcher.Token a] (Utility.Matcher.Matcher a)
+
 -- internal state storage
 data AnnexState = AnnexState
 	{ repo :: Git.Repo
@@ -81,7 +83,7 @@ data AnnexState = AnnexState
 	, forcenumcopies :: Maybe Int
 	, toremote :: Maybe String
 	, fromremote :: Maybe String
-	, limit :: Either [Utility.Matcher.Token (FilePath -> Annex Bool)] (Utility.Matcher.Matcher (FilePath -> Annex Bool))
+	, limit :: Matcher (FilePath -> Annex Bool)
 	, forcetrust :: [(UUID, TrustLevel)]
 	, trustmap :: Maybe TrustMap
 	, ciphers :: M.Map EncryptedCipher Cipher

From f0957426c586610d16ad9694e002b73324baa29a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 04:50:39 -0400
Subject: [PATCH 2861/8313] skip local remotes that are not available (ie, not
 mounted)

With --fast, unavailable local remotes are filtered out of the fast set.
This way, if there are local remotes, --fast always acts only on them,
and if none are mounted, acts on nothing. This consistency is better
than --fast acting on different remotes depending on what's mounted.
---
 Command/Sync.hs | 5 +++--
 Git/Config.hs   | 3 ++-
 Remote/Git.hs   | 9 ++++++++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 759df36ea3..445a371370 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -57,12 +57,13 @@ syncRemotes rs = do
 		then nub <$> pickfast
 		else wanted
 	where
-		pickfast = (++) <$> listed <*> (fastest <$> available)
+		pickfast = (++) <$> listed <*> (good =<< fastest <$> available)
 		wanted
-			| null rs = available
+			| null rs = good =<< available
 			| otherwise = listed
 		listed = mapM Remote.byName rs
 		available = filter nonspecial <$> Remote.enabledRemoteList
+		good = filterM $ Remote.Git.repoAvail . Types.Remote.repo
 		nonspecial r = Types.Remote.remotetype r == Remote.Git.remote
 		fastest = fromMaybe [] . headMaybe .
 			map snd . sort . M.toList . costmap
diff --git a/Git/Config.hs b/Git/Config.hs
index b2587aa44c..d9109548b8 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -29,7 +29,8 @@ read repo@(Repo { location = Dir d }) = do
 	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
 		pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
 			hRead repo
-read r = assertLocal r $ error "internal"
+read r = assertLocal r $
+	error $ "internal error; trying to read config of " ++ show r
 
 {- Reads git config from a handle and populates a repo with it. -}
 hRead :: Repo -> Handle -> IO Repo
diff --git a/Remote/Git.hs b/Remote/Git.hs
index e790d01a75..b9d9966a46 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Remote.Git (remote) where
+module Remote.Git (remote, repoAvail) where
 
 import Control.Exception.Extensible
 import qualified Data.Map as M
@@ -164,6 +164,13 @@ inAnnex r key
 				dispatch (Right Nothing) = unknown
 		unknown = Left $ "unable to check " ++ Git.repoDescribe r
 
+{- Checks inexpensively if a repository is available for use. -}
+repoAvail :: Git.Repo -> Annex Bool
+repoAvail r 
+	| Git.repoIsHttp r = return True
+	| Git.repoIsUrl r = return True
+	| otherwise = liftIO $ catchBoolIO $ onLocal r $ return True
+
 {- Runs an action on a local repository inexpensively, by making an annex
  - monad using that repository. -}
 onLocal :: Git.Repo -> Annex a -> IO a

From 38195a6363e54874ce072eb2d3ced448e0b68e02 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 31 Dec 2011 18:32:28 +0000
Subject: [PATCH 2862/8313] Added a comment

---
 ...comment_7_f8fd08b6ab47378ad88c87348057220d._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment b/doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment
new file mode 100644
index 0000000000..bdbecd4d99
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_7_f8fd08b6ab47378ad88c87348057220d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 7"
+ date="2011-12-31T18:32:28Z"
+ content="""
+That patch seems ok, it doesn't seem to allow through any repo locations that were blocked before.
+
+So, it has my blessing.. but the ADC is in gitolite and will need to be patched there.
+"""]]

From 68489c5cf293a8ff09ff33757cbec448caed4501 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 31 Dec 2011 18:34:31 +0000
Subject: [PATCH 2863/8313] Added a comment

---
 .../comment_13_00c82d320c7b4bb51078beba17e14dc8._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment b/doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment
new file mode 100644
index 0000000000..84515ec6c5
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_13_00c82d320c7b4bb51078beba17e14dc8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 13"
+ date="2011-12-31T18:34:31Z"
+ content="""
+I have merged my autosync branch, the improved sync command will be in this year's last git-annex release!
+"""]]

From f165e4aa997ecb34e2f225d405781a1789806d75 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 14:50:40 -0400
Subject: [PATCH 2864/8313] add section on syncing to the walkthrough

---
 doc/walkthrough.mdwn                      |  1 +
 doc/walkthrough/getting_file_content.mdwn | 10 ---------
 doc/walkthrough/syncing.mdwn              | 25 +++++++++++++++++++++++
 3 files changed, 26 insertions(+), 10 deletions(-)
 create mode 100644 doc/walkthrough/syncing.mdwn

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index 68f94a6f27..f93e28393e 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -8,6 +8,7 @@ A walkthrough of the basic features of git-annex.
 	adding_files
 	renaming_files
 	getting_file_content
+	syncing
 	transferring_files:_When_things_go_wrong
 	removing_files
 	removing_files:_When_things_go_wrong
diff --git a/doc/walkthrough/getting_file_content.mdwn b/doc/walkthrough/getting_file_content.mdwn
index cdb4a72e0a..f41e17770a 100644
--- a/doc/walkthrough/getting_file_content.mdwn
+++ b/doc/walkthrough/getting_file_content.mdwn
@@ -10,13 +10,3 @@ USB drive.
 	# git annex get .
 	get my_cool_big_file (from laptop...) ok
 	get iso/debian.iso (from laptop...) ok
-
-Notice that you had to git fetch and merge from laptop first, this lets
-git-annex know what has changed in laptop, and so it knows about the files
-present there and can get them.
-
-The alternate approach is to set up a
-[[central bare repository|tips/centralized_git_repository_tutorial]], and
-always push changes to it after committing them, then in the above,
-you can just pull from the central repository to get synced up to
-all repositories.
diff --git a/doc/walkthrough/syncing.mdwn b/doc/walkthrough/syncing.mdwn
new file mode 100644
index 0000000000..38f0f8acc8
--- /dev/null
+++ b/doc/walkthrough/syncing.mdwn
@@ -0,0 +1,25 @@
+Notice that in the [[previous example|getting_file_content]], you had to
+git fetch and merge from laptop first. This lets git-annex know what has
+changed in laptop, and so it knows about the files present there and can
+get them.
+
+If you have a lot of repositories to keep in sync, manually fetching and 
+merging from them can become tedious. To automate it there is a handy
+sync command, which also even commits your changes for you.
+
+	# cd /media/usb/annex
+	# git annex sync
+	commit
+	nothing to commit (working directory clean)
+	ok
+	pull laptop
+	ok
+	push laptop
+	ok
+
+After you run sync, the repository will be updated with all changes made to
+its remotes, and any changes in the repository will be pushed out to its
+remotes, where a sync will get them. This is especially useful when using
+git in a distributed fashion, without a 
+[[central bare repository|tips/centralized_git_repository_tutorial]]. See
+[[sync]] for details.

From 9b12701b9ec28f342d8bc25fa51d941b8cf02320 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 15:07:45 -0400
Subject: [PATCH 2865/8313] releasing version 3.20111231

---
 debian/changelog | 14 ++++++++------
 git-annex.cabal  |  2 +-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index db447fea4a..2732e87a6d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,11 @@
-git-annex (3.20111212) UNRELEASED; urgency=low
+git-annex (3.20111231) unstable; urgency=low
 
+  * sync: Improved to work well without a central bare repository.
+    Thanks to Joachim Breitner.
+  * Rather than manually committing, pushing, pulling, merging, and git annex
+    merging, we encourage you to give "git annex sync" a try.
+  * sync --fast: Selects some of the remotes with the lowest annex.cost
+    and syncs those, in addition to any specified at the command line.
   * Union merge now finds the least expensive way to represent the merge.
   * reinject: Add a sanity check for using an annexed file as the source file.
   * Properly handle multiline git config values.
@@ -16,12 +22,8 @@ git-annex (3.20111212) UNRELEASED; urgency=low
   * Can now be built with older git versions (before 1.7.7); the resulting
     binary should only be used with old git.
   * Updated to build with monad-control 0.3.
-  * sync: Improved to work well without a central bare repository. 
-    Thanks to Joachim Breitner.
-  * sync --fast: Selects some of the remotes with the lowest annex.cost
-    and syncs those, in addition to any specified at the command line.
 
- -- Joey Hess   Mon, 12 Dec 2011 01:57:49 -0400
+ -- Joey Hess   Sat, 31 Dec 2011 14:55:29 -0400
 
 git-annex (3.20111211) unstable; urgency=medium
 
diff --git a/git-annex.cabal b/git-annex.cabal
index a0980b1f0e..902acd0deb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111212
+Version: 3.20111231
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 4a575426887bdbd0c186237aac0fa92a2acf6948 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 15:12:16 -0400
Subject: [PATCH 2866/8313] update

---
 doc/news/version_3.20111107.mdwn |  8 --------
 doc/news/version_3.20111231.mdwn | 24 ++++++++++++++++++++++++
 2 files changed, 24 insertions(+), 8 deletions(-)
 delete mode 100644 doc/news/version_3.20111107.mdwn
 create mode 100644 doc/news/version_3.20111231.mdwn

diff --git a/doc/news/version_3.20111107.mdwn b/doc/news/version_3.20111107.mdwn
deleted file mode 100644
index 17431bf219..0000000000
--- a/doc/news/version_3.20111107.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20111107 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * merge: Use fast-forward merges when possible.
-     Thanks Valentin Haenel for a test case showing how non-fast-forward
-     merges could result in an ongoing pull/merge/push cycle.
-   * Don't try to read config from repos with annex-ignore set.
-   * Bugfix: In the past two releases, git-annex init has written the uuid.log
-     in the wrong format, with the UUID and description flipped."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111231.mdwn b/doc/news/version_3.20111231.mdwn
new file mode 100644
index 0000000000..35c38fb525
--- /dev/null
+++ b/doc/news/version_3.20111231.mdwn
@@ -0,0 +1,24 @@
+git-annex 3.20111231 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * sync: Improved to work well without a central bare repository.
+     Thanks to Joachim Breitner.
+   * Rather than manually committing, pushing, pulling, merging, and git annex
+     merging, we encourage you to give "git annex sync" a try.
+   * sync --fast: Selects some of the remotes with the lowest annex.cost
+     and syncs those, in addition to any specified at the command line.
+   * Union merge now finds the least expensive way to represent the merge.
+   * reinject: Add a sanity check for using an annexed file as the source file.
+   * Properly handle multiline git config values.
+   * Fix the hook special remote, which bitrotted a while ago.
+   * map: --fast disables use of dot to display map
+   * Test suite improvements. Current top-level test coverage: 75%
+   * Improve deletion of files from rsync special remotes. Closes: #[652849](http://bugs.debian.org/652849)
+   * Add --include, which is the same as --not --exclude.
+   * Format strings can be specified using the new --format option, to control
+     what is output by git annex find.
+   * Support git annex find --json
+   * Fixed behavior when multiple insteadOf configs are provided for the
+     same url base.
+   * Can now be built with older git versions (before 1.7.7); the resulting
+     binary should only be used with old git.
+   * Updated to build with monad-control 0.3."""]]
\ No newline at end of file

From 97d578915720554bbc1349a3f9deea66c6d413d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 31 Dec 2011 15:18:29 -0400
Subject: [PATCH 2867/8313] formatting

---
 doc/distributed_version_control.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/distributed_version_control.mdwn b/doc/distributed_version_control.mdwn
index e7c858c696..ccf82b91c7 100644
--- a/doc/distributed_version_control.mdwn
+++ b/doc/distributed_version_control.mdwn
@@ -12,7 +12,7 @@ and can copy it to or from other repositories. [[Location_tracking]]
 information is committed to git, to let repositories inform other
 repositories what file contents they have available.
 
---
+---
 
 The [[walkthrough]] shows how to create a distributed set of git-annex
 repositories with no central repository.

From 6744c41024951019af5bfbbf832836f7bfaa884b Mon Sep 17 00:00:00 2001
From: bremner 
Date: Sat, 31 Dec 2011 22:29:40 +0000
Subject: [PATCH 2868/8313] Added a comment: afaict git annex normalizes urls
 on the client side.

---
 ..._8249772c142117f88e37975d058aa936._comment | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment b/doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment
new file mode 100644
index 0000000000..0717bab1c2
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_8_8249772c142117f88e37975d058aa936._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="bremner"
+ ip="156.34.79.193"
+ subject="afaict git annex normalizes urls on the client side."
+ date="2011-12-31T22:29:38Z"
+ content="""
+After some debugging printing, here is my current understanding.
+
+- urls of the form git@host:~repo or ssh://git@host
+
+  - git sends commands like  \"git-receive-pack '~/repo'
+  - gitolite converts these to $REPO_BASE/~/repo which fails.  ~/repo would also fail fwiw.
+  - git-annex sends seems /~/repo, which works
+
+- urls of the form git@host:/repo or ssh://git@host/repo
+
+  - git sends \"git-receive-pack '/db/cs3383'\"
+  - gitolite converts this to $REPO_BASE/repo which works
+  - git annex sends \"git-annex-shell 'inannex' '/repo' ...\" which works, but only with the patch above.
+
+- urls of the form git@host:repo
+
+  - git sends \"git-receive-pack 'repo'
+  - gitolite converts this to $REPO_BASE/repo, which works
+  - git-annex sends \"git-annex-shell 'inannex' '/~/db/cs3383'...\", which also works for git-annex-shell.
+
+So the weird case is the last one where git and git-annex are sending different things over the wire.
+I don't know if you have other motivations for doing the url normalization on the client side, but it isn't needed for gitolite, and in some sense complicates things a little.  On the other hand, now that I see what is going on, it isn't a big deal to just strip the leading /~ off in the adc. It does lead to the odd situation of some URLs working for git-annex but not git.
+"""]]

From 3aed5ff88ab8b347787f1a86bdc437fb69152004 Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Mon, 2 Jan 2012 14:02:05 +0000
Subject: [PATCH 2869/8313] Added a comment

---
 ..._b63568b327215ef8f646a39d760fdfc0._comment | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment b/doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment
new file mode 100644
index 0000000000..194415dfb6
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_14_b63568b327215ef8f646a39d760fdfc0._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 14"
+ date="2012-01-02T14:02:04Z"
+ content="""
+Sorry for not replying earlier, but my non-mailinglist-communications-workflows are suboptimal :-)
+
+> Then in each repo, I found I had to pull from each of its remotes, to get the tracking branches that defaultSyncRemotes looks for to know those remotes are syncable.  This was the surprising thing for me, I had expected sync to somehow work out which remotes were syncable without my explicit pull. And it was not very obvious that sync was not doing its thing before I did that, since it still does a lot of \"stuff\".
+
+Right. But \"git fetch\" ought to be enough.
+
+Personally, I’d just pull and push everywhere, but you pointed out that it ought to be manageable. The existence of the synced/master branch is the flag that indicates this, so you need to propagate this once. Note that if the branch were already created by \"git annex init\", then this would not be a problem.
+
+It is not required to use \"git fetch\" once, you can also call \"git annex sync \" once with the remote explicitly mentioned; this would involve a fetch.
+
+> While this would lose the ability to control which remotes are synced, I think that being able to git annex sync origin and only sync from/to origin is sufficient, for the centralized use case.
+
+I’d leave this decision to you. But I see that you took the decision already, as your code now creates the synced/master branch when it does not exist (e290f4a8).
+
+> Why did you make branch strict?
+
+Because it did not work otherwise :-). It uses pipeRead, which is lazy, and for some reason git and/or your utility functions did not like that the output of the command was not consumed before the next git command was called. I did not investigate further. For better code, I’d suggest to add a function like pipeRead that completely reads the git output before returning, thus avoiding any issues with lazyIO.
+
+> mergeRemote merges from refs/remotes/foo/synced/master. But that will only be up-to-date if git annex sync has recently been run there. Is there any reason it couldn't merge from refs/remotes/foo/master?
+
+Hmm, good question. It is probably save to merge from both, and push only to synced/master. But which one first? synced/master can be ahead if the repo was synced to from somewhere else, master can be ahead if there are local changes. Maybe git merge should be called on all remote heads simultaniously, thus generating only one commit for the merge. I don’t know how well that works in practice.
+
+Thanks for including my code,
+Joachim
+
+"""]]

From 442202dd6d2ede66b35843f8e4708624041d25b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 11:01:08 -0400
Subject: [PATCH 2870/8313] add a new useful thing

---
 Utility/Monad.hs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 0d1675fa49..9e2a16e8c9 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -28,3 +28,11 @@ anyM p = liftM isJust . firstM p
 {- Runs an action on values from a list until it succeeds. -}
 untilTrue :: (Monad m) => [a] -> (a -> m Bool) -> m Bool
 untilTrue = flip anyM
+
+{- Runs a monadic action, passing its value to an observer
+ - before returning it. -}
+observe :: (Monad m) => (a -> m b) -> m a -> m a
+observe observer a = do
+	r <- a
+	_ <- observer r
+	return r

From 508b427c7b0885f6ed6e3f92800976d02c6341f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 11:57:02 -0400
Subject: [PATCH 2871/8313] tweak

---
 Command/Sync.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index 445a371370..e5884cc4a7 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -31,7 +31,7 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote))
 -- syncing involves several operations, any of which can independently fail
 seek :: CommandSeek
 seek rs = do
-	!branch <- fromMaybe nobranch <$> inRepo (Git.Branch.current)
+	!branch <- fromMaybe nobranch <$> inRepo Git.Branch.current
 	remotes <- syncRemotes rs
 	return $ concat $
 		[ [ commit ]

From 0e9444e4741c0b51c12bc3f00cf41f946f5f8702 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 2 Jan 2012 16:01:49 +0000
Subject: [PATCH 2872/8313] Added a comment

---
 ...ment_15_cb7c856d8141b2de3cc95874753f1ee5._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment

diff --git a/doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment b/doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment
new file mode 100644
index 0000000000..a526d5db8a
--- /dev/null
+++ b/doc/forum/pure_git-annex_only_workflow/comment_15_cb7c856d8141b2de3cc95874753f1ee5._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 15"
+ date="2012-01-02T16:01:49Z"
+ content="""
+With a lazy branch, I get \"git-annex: no branch is checked out\". Weird.. my best guess is that it's because this is running at the seek stage, which is unusual, and the value is not used until a later stage and so perhaps the git command gets reaped by some cleanup code before its output is read.
+
+(pipeRead is lazy because often it's used to read large quantities of data from git that are processed progressively.)
+
+I did make it merge both branches, separately. It would be possible to do one single merge, but it's probably harder for the user to recover if there are conflicts in an octopus merge. The order of the merges does not seem to me to matter much, barring conflicts it will work either way. Dealing with conflicts during sync is probably a weakness of all this; after the first conflict the rest of the sync will continue failing.
+"""]]

From 1bdd04210f2ef4a0cb3b87a915a6db29dfe51636 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 2 Jan 2012 16:27:55 +0000
Subject: [PATCH 2873/8313] Added a comment

---
 .../comment_9_28418635a6ed7231b89e02211cd3c236._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment

diff --git a/doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment b/doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment
new file mode 100644
index 0000000000..fc297ff175
--- /dev/null
+++ b/doc/tips/using_gitolite_with_git-annex/comment_9_28418635a6ed7231b89e02211cd3c236._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 9"
+ date="2012-01-02T16:27:55Z"
+ content="""
+Ah right. git-annex normalizes all git ssh style user@host:dir to valid uris, which is where the `/~/` comes from. I don't anticipate this changing on the git-annex side.
+"""]]

From b7f0f9cd0ce12ad9320c5f7e9766838a4d92442e Mon Sep 17 00:00:00 2001
From: Nicolas Pouillard 
Date: Mon, 2 Jan 2012 18:35:38 +0100
Subject: [PATCH 2874/8313] Add specific instructions for ArchLinux

---
 doc/install.mdwn           | 1 +
 doc/install/ArchLinux.mdwn | 9 +++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 doc/install/ArchLinux.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index ceaa3544fb..3e89c67758 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -6,6 +6,7 @@
 * [[Fedora]]
 * [[FreeBSD]]
 * [[openSUSE]]
+* [[ArchLinux]]
 
 ## Using cabal
 
diff --git a/doc/install/ArchLinux.mdwn b/doc/install/ArchLinux.mdwn
new file mode 100644
index 0000000000..e531fc968e
--- /dev/null
+++ b/doc/install/ArchLinux.mdwn
@@ -0,0 +1,9 @@
+There is a non-official source package for git-annex in
+[AUR](https://aur.archlinux.org/packages.php?ID=44272).
+
+You can then build it yourself or use a wrapper for AUR
+such as yaourt:
+
+
+$ yaourt -Sy git-annex
+


From 815fba3b39646775f1b3947c06da0acbfdd88500 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 13:44:12 -0400
Subject: [PATCH 2875/8313] add arch link

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index ceaa3544fb..3e89c67758 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -6,6 +6,7 @@
 * [[Fedora]]
 * [[FreeBSD]]
 * [[openSUSE]]
+* [[ArchLinux]]
 
 ## Using cabal
 

From aa0882691bb2aa64fb13f0df85be0469fd33d98d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 14:20:20 -0400
Subject: [PATCH 2876/8313] Added remote.name.annex-web-options configuration
 setting, which can be used to provide parameters to whichever of wget or curl
 git-annex uses (depends on which is available, but most of their important
 options suitable for use here are the same).

---
 Annex/Content.hs   | 11 ++++++++++-
 Command/AddUrl.hs  |  3 +--
 Remote/Git.hs      |  4 +---
 Remote/Web.hs      |  3 ++-
 Utility/Url.hs     | 13 +++++++------
 debian/changelog   |  9 +++++++++
 doc/git-annex.mdwn |  6 ++++++
 7 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 3f1db37b53..1713b5e12d 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -20,7 +20,8 @@ module Annex.Content (
 	fromAnnex,
 	moveBad,
 	getKeysPresent,
-	saveState
+	saveState,
+	downloadUrl,
 ) where
 
 import System.IO.Error (try)
@@ -36,6 +37,7 @@ import qualified Annex.Queue
 import qualified Annex.Branch
 import Utility.StatFS
 import Utility.FileMode
+import qualified Utility.Url as Url
 import Types.Key
 import Utility.DataUnits
 import Config
@@ -281,3 +283,10 @@ saveState :: Annex ()
 saveState = do
 	Annex.Queue.flush False
 	Annex.Branch.commit "update"
+
+{- Downloads content from any of a list of urls. -}
+downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
+downloadUrl urls file = do
+	g <- gitRepo
+	o <- map Param . words <$> getConfig g "web-options" ""
+	liftIO $ anyM (\u -> Url.download u o file) urls
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 027c508bcb..46584f0d81 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -12,7 +12,6 @@ import Network.URI
 import Common.Annex
 import Command
 import qualified Backend
-import qualified Utility.Url as Url
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
@@ -45,7 +44,7 @@ download url file = do
 	let dummykey = Backend.URL.fromUrl url
 	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
-	stopUnless (liftIO $ Url.download url tmp) $ do
+	stopUnless (downloadUrl [url] tmp) $ do
 		[(backend, _)] <- Backend.chooseBackends [file]
 		k <- Backend.genKey tmp backend
 		case k of
diff --git a/Remote/Git.hs b/Remote/Git.hs
index b9d9966a46..da81241eb1 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -209,10 +209,8 @@ copyFromRemote r key file
 		loc <- liftIO $ gitAnnexLocation key r
 		rsyncOrCopyFile params loc file
 	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
-	| Git.repoIsHttp r = liftIO $ downloadurls $ keyUrls r key
+	| Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
-	where
-		downloadurls us = untilTrue us $ \u -> Url.download u file
 
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 93e5770f07..4d6348e596 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Types.Remote
 import qualified Git
 import qualified Git.Construct
+import Annex.Content
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
@@ -55,7 +56,7 @@ downloadKey key file = get =<< getUrls key
 			return False
 		get urls = do
 			showOutput -- make way for download progress bar
-			liftIO $ anyM (`Url.download` file) urls
+			downloadUrl urls file
 
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
diff --git a/Utility/Url.hs b/Utility/Url.hs
index f215a1ebdb..e10b8a92a4 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -6,6 +6,7 @@
  -}
 
 module Utility.Url (
+	URLString,
 	exists,
 	canDownload,
 	download,
@@ -43,21 +44,21 @@ canDownload = (||) <$> inPath "wget" <*> inPath "curl"
  - would not be appropriate to test at configure time and build support
  - for only one in.
  -}
-download :: URLString -> FilePath -> IO Bool
-download url file = do
+download :: URLString -> [CommandParam] -> FilePath -> IO Bool
+download url options file = do
 	e <- inPath "wget"
 	if e
 		then
-			boolSystem "wget"
-				[Params "-c -O", File file, File url]
+			go "wget" [Params "-c -O", File file, File url]
 		else
 			-- Uses the -# progress display, because the normal
 			-- one is very confusing when resuming, showing
 			-- the remainder to download as the whole file,
 			-- and not indicating how much percent was
 			-- downloaded before the resume.
-			boolSystem "curl" 
-				[Params "-L -C - -# -o", File file, File url]
+			go "curl" [Params "-L -C - -# -o", File file, File url]
+	where
+		go cmd opts = boolSystem cmd (options++opts)
 
 {- Downloads a small file. -}
 get :: URLString -> IO String
diff --git a/debian/changelog b/debian/changelog
index 2732e87a6d..80a39964d0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (3.20111232) UNRELEASED; urgency=low
+
+  * Added remote.name.annex-web-options configuration setting, which can be
+    used to provide parameters to whichever of wget or curl git-annex uses
+    (depends on which is available, but most of their important options
+    suitable for use here are the same).
+
+ -- Joey Hess   Mon, 02 Jan 2012 14:19:19 -0400
+
 git-annex (3.20111231) unstable; urgency=low
 
   * sync: Improved to work well without a central bare repository.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a0dd3d3f1b..ee7137e134 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -603,6 +603,12 @@ Here are all the supported configuration settings.
   to or from this remote. For example, to force ipv6, and limit
   the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100"
 
+* `remote..annex-web-options`
+
+  Options to use when using wget or curl to download a file from the web.
+  (wget is always used in preference to curl if available).
+  For example, to force ipv4 only, set it to "-4"
+
 * `remote..annex-bup-split-options`
 
   Options to pass to bup split when storing content in this remote.

From ae99b64b538f13e2100429afd8b4de7960db8cff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 14:20:51 -0400
Subject: [PATCH 2877/8313] mention this commits -a

---
 doc/git-annex.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index ee7137e134..896ccc311f 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -127,8 +127,8 @@ subdirectories).
   the default is to sync with all remotes. Or specify --fast to sync with
   the remotes with the lowest annex-cost value.
 
-  The sync process involves first committing all local changes, then
-  fetching and merging the `synced/master` and the `git-annex` branch
+  The sync process involves first committing all local changes (git commit -a),
+  then fetching and merging the `synced/master` and the `git-annex` branch
   from the remote repositories and finally pushing the changes back to
   those branches on the remote repositories. You can use standard git
   commands to do each of those steps by hand, or if you don't want to

From f0c4a1c770b7baf6e122bd990fcdd33b90eb2d53 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 14:22:50 -0400
Subject: [PATCH 2878/8313] annex.web-options also works

---
 debian/changelog   | 2 +-
 doc/git-annex.mdwn | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 80a39964d0..33cced98e5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,6 @@
 git-annex (3.20111232) UNRELEASED; urgency=low
 
-  * Added remote.name.annex-web-options configuration setting, which can be
+  * Added annex-web-options configuration settings, which can be
     used to provide parameters to whichever of wget or curl git-annex uses
     (depends on which is available, but most of their important options
     suitable for use here are the same).
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 896ccc311f..f94a3d8089 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -615,10 +615,10 @@ Here are all the supported configuration settings.
   For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k"
   (There is no corresponding option for bup join.)
 
-* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`
+* `annex.ssh-options`, `annex.rsync-options`, `annex.web-options, `annex.bup-split-options`
 
-  Default ssh, rsync, and bup options to use if a remote does not have
-  specific options.
+  Default ssh, rsync, wget/curl, and bup options to use if a remote does not
+  have specific options.
 
 * `remote..buprepo`
 

From 3e678b114047906c96977bb26f70feeb1749c430 Mon Sep 17 00:00:00 2001
From: "http://a-or-b.myopenid.com/" 
Date: Tue, 3 Jan 2012 00:10:45 +0000
Subject: [PATCH 2879/8313]

---
 doc/bugs/conq:_invalid_command_syntax.mdwn | 29 ++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/bugs/conq:_invalid_command_syntax.mdwn

diff --git a/doc/bugs/conq:_invalid_command_syntax.mdwn b/doc/bugs/conq:_invalid_command_syntax.mdwn
new file mode 100644
index 0000000000..ac7271fc9c
--- /dev/null
+++ b/doc/bugs/conq:_invalid_command_syntax.mdwn
@@ -0,0 +1,29 @@
+I've been getting an occasional error from git-annex.
+
+The error is:   'conq: invalid command syntax.'
+
+For example, the last two commands I ran are:
+
+    $ git annex unused
+    unused . (checking for unused data...) (checking master...) (checking origin/master...) 
+      Some annexed data is no longer used by any files:
+        NUMBER  KEY
+        1       SHA256-s.....
+      (To see where data was previously used, try: git log --stat -S'KEY')
+      
+      To remove unwanted data: git-annex dropunused NUMBER
+      
+    ok
+    
+    $ git annex dropunused 1
+    dropunused 1 conq: invalid command syntax.
+    ok
+
+
+
+*OS:*  OSX + port installs of the GNU tools
+
+*git-annex-version:*  3.20111211
+
+*git-version:*  1.7.7.4
+

From eec59e73623ab232a180ef7ff5edc6c2255ca156 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 00:35:00 +0000
Subject: [PATCH 2880/8313] Added a comment

---
 ...comment_1_f33b83025ce974e496f83f248275a66a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment

diff --git a/doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment b/doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment
new file mode 100644
index 0000000000..62881bfc7a
--- /dev/null
+++ b/doc/bugs/conq:_invalid_command_syntax/comment_1_f33b83025ce974e496f83f248275a66a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-03T00:34:59Z"
+ content="""
+AFAICs, you probably just have a \"conq\" program that is running in the background and emitted this error.
+
+The error message is not part of git-annex; it does not run any \"conq\" thing itself. Although you could try passing the --debug parameter to check the commands it does run to see if one of them somehow causes this conq thing.
+"""]]

From ddc4b3de331060975deefbf3e3e7e6224ee59e07 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 00:41:08 +0000
Subject: [PATCH 2881/8313] Added a comment

---
 .../comment_2_195106ca8dedad5f4d755f625e38e8af._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment

diff --git a/doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment b/doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment
new file mode 100644
index 0000000000..39ece1eca2
--- /dev/null
+++ b/doc/bugs/conq:_invalid_command_syntax/comment_2_195106ca8dedad5f4d755f625e38e8af._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-01-03T00:41:08Z"
+ content="""
+A google search 
+finds other examples of this error message related to ssh, mercurial, and bitbucket. What that has to do with git is anyone's guess, but I'm pretty sure git-annex is not related to it at all.
+"""]]

From 9f7ca3fe0da2e19d16a8abf08804b43498417ed3 Mon Sep 17 00:00:00 2001
From: "http://a-or-b.myopenid.com/" 
Date: Tue, 3 Jan 2012 00:43:54 +0000
Subject: [PATCH 2882/8313]

---
 ...ial_remote_when_content_changes__63__.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn

diff --git a/doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn b/doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn
new file mode 100644
index 0000000000..5b7be71575
--- /dev/null
+++ b/doc/forum/Handling_web_special_remote_when_content_changes__63__.mdwn
@@ -0,0 +1,19 @@
+I am in the process of organising all of my documents (pdf/ps/etc.) and putting them into an annex.  It seems like the easiest (and smartest?) way for me to manage my 15,000+ document library...  (Yeah, I've not read them all!)
+
+However, one of the issues I have run into is when I want to include something like a software manual...
+
+I'm not even sure if git-annex is the correct way to handle these sort of document, but...
+
+What would you do if the URL stays the same, but the file content changes over time?
+
+For example, the administration manual for R gets updated and re-released for each release of R, but the URL stays the same.
+
+    http://cran.r-project.org/doc/manuals/R-admin.pdf
+
+I'm not particularly worried about whether my annex version is the most recent, just that if I want to 'annex get' it, it will pull *a* version from somewhere.
+
+Any thoughts?
+
+I'd bet that the SHA-hash would change between releases(!) so would a WORM backend be the best approach?  It would mess up my one-file-in-multiple-directories (i.e. multiple simlinks to the one SHA-...-.....) approach.
+
+ 

From 17735698f68b5197aca56171d3fbfdb11c892aa9 Mon Sep 17 00:00:00 2001
From: "http://a-or-b.myopenid.com/" 
Date: Tue, 3 Jan 2012 00:49:43 +0000
Subject: [PATCH 2883/8313] Added a comment

---
 ...nt_3_55af43e2f43a4c373f7a0a33678d0b1c._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment

diff --git a/doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment b/doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment
new file mode 100644
index 0000000000..75132c1d6b
--- /dev/null
+++ b/doc/bugs/conq:_invalid_command_syntax/comment_3_55af43e2f43a4c373f7a0a33678d0b1c._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 3"
+ date="2012-01-03T00:49:41Z"
+ content="""
+Yeah,  I saw those google links myself, but couldn't see why the bitbucket/ssh would be relevant.
+
+The strange thing is that I *only* get this message when running git-annex.
+
+I also don't have a conq in my path so I don't know where it is running from.
+ 
+
+Oh well, if I ever sort it out I'll post back here.
+"""]]

From 2f241f7ddeec79916f0e44fd1175b2ff1a6fca65 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 00:57:55 +0000
Subject: [PATCH 2884/8313] Added a comment

---
 ...mment_1_05ee6a1b1943ef3c90634e52233bde1c._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment

diff --git a/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment
new file mode 100644
index 0000000000..0f0bf2f327
--- /dev/null
+++ b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_1_05ee6a1b1943ef3c90634e52233bde1c._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-03T00:57:55Z"
+ content="""
+The web special remote will happily download files when you `git annex get` even if they don't have the same content that they did before.
+
+`git annex fsck` will detect such mismatched content to the best ability of the backend (so checking the SHA1, or verifying the file size at least matches if you use WORM), and complain and move such mismatched content aside. `git annex addurl --fast` deserves a special mention; it uses a backend that only records the URL, and so if it's used, fsck cannot later detect such changes. Which might be what you want..
+
+For most users, this is one of the reasons `git annex untrust web` is a recommended configuration. Once you untrust the web, any content you download from the web will be kept around in one of your own git-annex repositories, rather than the untrustworthy web being the old copy.
+"""]]

From 5cd44282a92dd6fa7eacc62858c8b8e553590195 Mon Sep 17 00:00:00 2001
From: "http://a-or-b.myopenid.com/" 
Date: Tue, 3 Jan 2012 02:49:24 +0000
Subject: [PATCH 2885/8313] Added a comment

---
 ...comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment

diff --git a/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment
new file mode 100644
index 0000000000..93c43137d3
--- /dev/null
+++ b/doc/forum/Handling_web_special_remote_when_content_changes__63__/comment_2_48d82e391812d8ec0d4e6562d0607fe7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://a-or-b.myopenid.com/"
+ ip="203.45.2.230"
+ subject="comment 2"
+ date="2012-01-03T02:49:18Z"
+ content="""
+Thanks!  `git annex addurl --fast` does exactly what I want it to do.
+
+Wow.  Yet another special backend for me to play with.  :-)
+"""]]

From fc80b8d96bf287a0b83c1402e3a7c5ebfc7bc4a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 14:54:23 -0400
Subject: [PATCH 2886/8313] factor observe_

---
 Remote/Git.hs    | 15 +++++----------
 Remote/Rsync.hs  |  4 +---
 Utility/Monad.hs |  4 ++++
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index da81241eb1..2e32ca2394 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -115,13 +115,11 @@ tryGitConfigRead r
 				pOpen ReadFromPipe "git" ["config", "--null", "--list", "--file", tmpfile] $
 					Git.Config.hRead r
 
-		store a = do
-			r' <- a
+		store = observe $ \r' -> do
 			g <- gitRepo
 			let l = Git.remotes g
 			let g' = g { Git.remotes = exchange l r' }
 			Annex.changeState $ \s -> s { Annex.repo = g' }
-			return r'
 
 		exchange [] _ = []
 		exchange (old:ls) new =
@@ -184,9 +182,7 @@ onLocal r a = do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.BranchState.disableUpdate
-		ret <- a
-		liftIO Git.Command.reap
-		return ret
+		observe_ (liftIO Git.Command.reap) a
 
 keyUrls :: Git.Repo -> Key -> [String]
 keyUrls r key = map tourl (annexLocations key)
@@ -221,10 +217,9 @@ copyToRemote r key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
 			ensureInitialized
-			ok <- Annex.Content.getViaTmp key $
-				rsyncOrCopyFile params keysrc
-			Annex.Content.saveState
-			return ok
+			observe_ Annex.Content.saveState $
+				Annex.Content.getViaTmp key $
+					rsyncOrCopyFile params keysrc
 	| Git.repoIsSsh r = do
 		keysrc <- inRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 2fe302ba52..1461b96cde 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -172,9 +172,7 @@ withRsyncScratchDir a = do
 	let tmp = t  "rsynctmp"  show pid
 	nuke tmp
 	liftIO $ createDirectoryIfMissing True tmp
-	res <- a tmp
-	nuke tmp
-	return res
+	observe_ (nuke tmp) (a tmp)
 	where
 		nuke d = liftIO $ 
 			doesDirectoryExist d >>? removeDirectoryRecursive d
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 9e2a16e8c9..7f9c7b1bca 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -36,3 +36,7 @@ observe observer a = do
 	r <- a
 	_ <- observer r
 	return r
+
+{- Like observe, but the observer is not passed the value. -}
+observe_ :: (Monad m) => m b -> m a -> m a
+observe_ observer = observe (const observer)

From ab8fa0d2056e55af829bdf843e162e97ac903f4d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 2 Jan 2012 23:30:41 -0400
Subject: [PATCH 2887/8313] dead code

---
 Seek.hs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Seek.hs b/Seek.hs
index 28c6ffc00c..251db50b82 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -44,11 +44,6 @@ withBackendFilesInGit a params = do
 	files <- seekHelper LsFiles.inRepo params
 	prepBackendPairs a files
 
-withFilesMissing :: (String -> CommandStart) -> CommandSeek
-withFilesMissing a params = prepFiltered a $ liftIO $ filterM missing params
-	where
-		missing = liftM not . doesFileExist
-
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	force <- Annex.getState Annex.force

From 34abd7bca80a8cc012f92d64116014449b1b2392 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 00:09:09 -0400
Subject: [PATCH 2888/8313] no implicit dotfiles in add

Dotfiles, and files inside dotdirs are not added by "git annex add" unless
the dotfile or directory is explicitly listed. So "git annex add ." will
add all untracked files in the current directory except for those in
dotdirs.

One reason for this is that it will make git-annex more usable with vcsh,
where you don't want "vcsh big annex add" to check in all the dotfiles
that are already versioned in other repositories.

(If you're using vcsh for repos that contain non-dotfiles, this won't help,
and you'll need to .gitignore such things, but this will cover the common
case.)

A more general reason why this seems like a good idea is the same reason ls
ignores dotfiles, just the unix convention that they are cruft that is kept
out of the way most of the time.

All the other git-annex commands still do deal with any dotfiles that do
get into the annex. This seemed right because if I've gone to the trouble
to add a dotfile, I will want "git annex get ." to get it along with
everything else.
---
 Seek.hs            | 13 ++++++++++---
 Utility/Path.hs    | 11 +++++++++++
 debian/changelog   |  4 ++++
 doc/git-annex.mdwn |  3 ++-
 4 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/Seek.hs b/Seek.hs
index 251db50b82..80f31dd969 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -46,9 +46,16 @@ withBackendFilesInGit a params = do
 
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
-	force <- Annex.getState Annex.force
-	newfiles <- seekHelper (LsFiles.notInRepo force) params
-	prepBackendPairs a newfiles
+	{- dotfiles are not acted on unless explicitly listed -}
+	files <- filter (not . dotfile) <$> seek ps
+	dotfiles <- if null dotps then return [] else seek dotps
+	prepBackendPairs a $ preserveOrder params (files++dotfiles)
+	where
+		(dotps, ps) = partition dotfile params
+		seek l = do
+			force <- Annex.getState Annex.force
+			g <- gitRepo
+			liftIO $ (\p -> LsFiles.notInRepo force p g) l
 
 withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 38e7bd05ca..9f4fe29277 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -134,3 +134,14 @@ inPath :: String -> IO Bool
 inPath command = getSearchPath >>= anyM indir
 	where
 		indir d = doesFileExist $ d  command
+
+{- Checks if a filename is a unix dotfile. All files inside dotdirs
+ - count as dotfiles. -}
+dotfile :: FilePath -> Bool
+dotfile file
+	| f == "." = False
+	| f == ".." = False
+	| f == "" = False
+	| otherwise = "." `isPrefixOf` f || dotfile (takeDirectory file)
+	where
+		f = takeFileName file
diff --git a/debian/changelog b/debian/changelog
index 33cced98e5..c9619114ee 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,10 @@ git-annex (3.20111232) UNRELEASED; urgency=low
     used to provide parameters to whichever of wget or curl git-annex uses
     (depends on which is available, but most of their important options
     suitable for use here are the same).
+  * Dotfiles, and files inside dotdirs are not added by "git annex add"
+    unless the dotfile or directory is explicitly listed. So "git annex add ."
+    will add all untracked files in the current directory except for those in
+    dotdirs.
 
  -- Joey Hess   Mon, 02 Jan 2012 14:19:19 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index f94a3d8089..9751560a96 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -64,7 +64,8 @@ subdirectories).
 
   Adds files in the path to the annex. Files that are already checked into
   git, or that git has been configured to ignore will be silently skipped.
-  (Use --force to add ignored files.)
+  (Use --force to add ignored files.) Dotfiles are skipped unless explicitly
+  listed.
 
 * get [path ...]
 

From ee554542c1f3ee63efe5533267cf9cd1e5c83d52 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 00:29:27 -0400
Subject: [PATCH 2889/8313] after is a better name for observe_

---
 Remote/Git.hs    | 8 ++++----
 Remote/Rsync.hs  | 2 +-
 Utility/Monad.hs | 9 ++++-----
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2e32ca2394..7d034d2425 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -182,7 +182,7 @@ onLocal r a = do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.BranchState.disableUpdate
-		observe_ (liftIO Git.Command.reap) a
+		liftIO Git.Command.reap `after` a
 
 keyUrls :: Git.Repo -> Key -> [String]
 keyUrls r key = map tourl (annexLocations key)
@@ -217,9 +217,9 @@ copyToRemote r key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
 			ensureInitialized
-			observe_ Annex.Content.saveState $
-				Annex.Content.getViaTmp key $
-					rsyncOrCopyFile params keysrc
+			Annex.Content.saveState `after`
+				Annex.Content.getViaTmp key
+					(rsyncOrCopyFile params keysrc)
 	| Git.repoIsSsh r = do
 		keysrc <- inRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 1461b96cde..c7b60467c4 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -172,7 +172,7 @@ withRsyncScratchDir a = do
 	let tmp = t  "rsynctmp"  show pid
 	nuke tmp
 	liftIO $ createDirectoryIfMissing True tmp
-	observe_ (nuke tmp) (a tmp)
+	nuke tmp `after` a tmp
 	where
 		nuke d = liftIO $ 
 			doesDirectoryExist d >>? removeDirectoryRecursive d
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 7f9c7b1bca..be48072cbf 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -29,14 +29,13 @@ anyM p = liftM isJust . firstM p
 untilTrue :: (Monad m) => [a] -> (a -> m Bool) -> m Bool
 untilTrue = flip anyM
 
-{- Runs a monadic action, passing its value to an observer
- - before returning it. -}
+{- Runs an action, passing its value to an observer before returning it. -}
 observe :: (Monad m) => (a -> m b) -> m a -> m a
 observe observer a = do
 	r <- a
 	_ <- observer r
 	return r
 
-{- Like observe, but the observer is not passed the value. -}
-observe_ :: (Monad m) => m b -> m a -> m a
-observe_ observer = observe (const observer)
+{- b `after` a runs first a, then b, and returns the value of a -}
+after :: (Monad m) => m b -> m a -> m a
+after = observe . const

From b6e7b44a12bf56a3260e872b5cf07f69d6db20b9 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 3 Jan 2012 16:32:35 +0000
Subject: [PATCH 2890/8313]

---
 doc/bugs/__34__make_test__34___fails_silently.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/__34__make_test__34___fails_silently.mdwn

diff --git a/doc/bugs/__34__make_test__34___fails_silently.mdwn b/doc/bugs/__34__make_test__34___fails_silently.mdwn
new file mode 100644
index 0000000000..3bd4bd8691
--- /dev/null
+++ b/doc/bugs/__34__make_test__34___fails_silently.mdwn
@@ -0,0 +1 @@
+`make test` fails silently when the test program cannot be built. This happens, for example, when attempting to compile git-annex with `QuickCheck-2.4.2`.

From 7cdeebdcc8cbb8c7b43501ccc98f0b6b0e153f4a Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 3 Jan 2012 16:38:47 +0000
Subject: [PATCH 2891/8313]

---
 doc/forum/git-subtree_support__63__.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__.mdwn

diff --git a/doc/forum/git-subtree_support__63__.mdwn b/doc/forum/git-subtree_support__63__.mdwn
new file mode 100644
index 0000000000..a95277a235
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+I am a happy user of [git-subtree](http://github.com/apenwarr/git-subtree), and I wonder whether it integrates nicely with git-annex?
+
+My use-case looks like this: I have two annex repositories -- one at home and one at work. The annex at work is a strict subset of the one at home, i.e. all files that I have at work ought to be part of the annex at home, but not the other way round. Now, I realize that I could have one annex and selectively copy files to the checked out copy at work, but I don't want to do that because I don't want to have (broken) symlinks for all kinds of stuff visible on my machine at work that is not supposed to be there (such as MP3 files, etc.). Instead, I would like to use git-subtree to import the work annex into a sub-directory of the one at home, so that both annex are logically separate, but still the one at home always contains everything that the one at work contains.
+
+Is that possible?
+
+And if not, is there maybe another way to accomplish this kind of thing?

From 69875de8ea7aefdabec1ff81c567c8c5158ed5c1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 16:54:19 +0000
Subject: [PATCH 2892/8313] Added a comment

---
 ...t_1_f868e34f41d828d4571968d1ab07820a._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment

diff --git a/doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment b/doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment
new file mode 100644
index 0000000000..00f67ad114
--- /dev/null
+++ b/doc/bugs/__34__make_test__34___fails_silently/comment_1_f868e34f41d828d4571968d1ab07820a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-03T16:54:19Z"
+ content="""
+The code is:
+
+
+@if ! $(GHCMAKE) -O0 test; then \
+                echo \"** not running test suite\" >&2; \
+else \
+

+
+The error message from the compiler, followed by the above error message does not seem \"silent\". It does exit 0 without running the test suite if it cannot be built.
+"""]]

From 79f9a2d18d5211f81d27a2ef0f57f5e14b5ef99e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 17:00:53 +0000
Subject: [PATCH 2893/8313] Added a comment

---
 ...comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment

diff --git a/doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment b/doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment
new file mode 100644
index 0000000000..e4ded4c577
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_1_4f333cb71ed1ff259bbfd86704806aa6._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-03T17:00:53Z"
+ content="""
+I have no experience using git-subtree, but as long as the home repository has the work one as a git remote, it will automatically merge work's git-annex branch with its own git-annex branch, and so will know what files are present at work, and will be able to get them.
+
+Probably you won't want to make work have home as a remote, so work's git-annex will not know which files home has, nor will it be able to copy files to home (but home will be able to copy files to work).
+"""]]

From 390982414bc7282337bacd8920fc1c8b99f9f694 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 3 Jan 2012 18:09:42 +0000
Subject: [PATCH 2894/8313] Added a comment

---
 .../comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment

diff --git a/doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment b/doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment
new file mode 100644
index 0000000000..3b76e3f146
--- /dev/null
+++ b/doc/bugs/__34__make_test__34___fails_silently/comment_2_fb9e8e2716b0dea15b0d4807ae7cd114._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 2"
+ date="2012-01-03T18:09:38Z"
+ content="""
+When \"make test\" fails to run any tests at all, it should not return exit code 0. This behavior is quite misleading, and it means that automated build systems are not going to detect the fact that the test suit could not be run.
+"""]]

From 917a36f9f14b11224d788fb7e9ceacfe374436ab Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 3 Jan 2012 18:11:40 +0000
Subject: [PATCH 2895/8313] Added a comment

---
 .../comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment

diff --git a/doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment b/doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment
new file mode 100644
index 0000000000..9bac6e997b
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_2_73d2a015b1ac79ec99e071a8b1e29034._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 2"
+ date="2012-01-03T18:11:37Z"
+ content="""
+The point of git-subtree is that I can import another repository into sub-directory, i.e. I can have a directory called \"work\" that contains all files from the annex I have at work. If I make the other annex a remote and merge its contents, then all contents is going to be merged at the top-level, which is somewhat undesirable in my particular case.
+"""]]

From 6b7ee8486d00d4e067257d57c574b371f8e4e3d0 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 18:42:08 +0000
Subject: [PATCH 2896/8313] Added a comment

---
 .../comment_3_c533400e22c306c033fcd56e64761b0b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment

diff --git a/doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment b/doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment
new file mode 100644
index 0000000000..83edaf743d
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_3_c533400e22c306c033fcd56e64761b0b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-01-03T18:42:08Z"
+ content="""
+You can make it a remote without merging its contents. Git will not merge its contents by default unless it's named \"origin\". git-annex will be prefectly happy with that.
+"""]]

From 7e6a54f98422bf0bab1d1aad5980df3e60220ff8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 14:52:20 -0400
Subject: [PATCH 2897/8313] Added quickcheck to build dependencies, and fail if
 test suite cannot be built.

---
 Makefile                                           | 11 +++++------
 debian/changelog                                   |  2 ++
 debian/control                                     |  1 +
 doc/bugs/__34__make_test__34___fails_silently.mdwn |  3 +++
 git-annex.cabal                                    |  2 +-
 5 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index cda8fed6b3..6805f93d2a 100644
--- a/Makefile
+++ b/Makefile
@@ -66,12 +66,11 @@ install: all
 
 test:
 	@if ! $(GHCMAKE) -O0 test; then \
-		echo "** not running test suite" >&2; \
-	else \
-		if ! ./test; then \
-			echo "** test suite failed!" >&2; \
-			exit 1; \
-		fi; \
+		echo "** failed to build the test suite" >&2; \
+		exit 1; \
+	elif ! ./test; then \
+		echo "** test suite failed!" >&2; \
+		exit 1; \
 	fi
 
 testcoverage:
diff --git a/debian/changelog b/debian/changelog
index c9619114ee..6075053879 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,8 @@ git-annex (3.20111232) UNRELEASED; urgency=low
     unless the dotfile or directory is explicitly listed. So "git annex add ."
     will add all untracked files in the current directory except for those in
     dotdirs.
+  * Added quickcheck to build dependencies, and fail if test suite cannot be
+    built.
 
  -- Joey Hess   Mon, 02 Jan 2012 14:19:19 -0400
 
diff --git a/debian/control b/debian/control
index 3f88c71eaf..6a312c6b10 100644
--- a/debian/control
+++ b/debian/control
@@ -16,6 +16,7 @@ Build-Depends:
 	libghc-monad-control-dev (>= 0.3),
 	libghc-lifted-base-dev,
 	libghc-json-dev,
+	libghc-quickcheck2-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/bugs/__34__make_test__34___fails_silently.mdwn b/doc/bugs/__34__make_test__34___fails_silently.mdwn
index 3bd4bd8691..8632f03f51 100644
--- a/doc/bugs/__34__make_test__34___fails_silently.mdwn
+++ b/doc/bugs/__34__make_test__34___fails_silently.mdwn
@@ -1 +1,4 @@
 `make test` fails silently when the test program cannot be built. This happens, for example, when attempting to compile git-annex with `QuickCheck-2.4.2`.
+
+> I've made "make test" exit nonzero if the test suite cannot be built.
+> [[done]] --[[Joey]]
diff --git a/git-annex.cabal b/git-annex.cabal
index 902acd0deb..93238f49ff 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
-   base < 5, monad-control, transformers-base, lifted-base
+   base < 5, monad-control, transformers-base, lifted-base, quickcheck >= 2.1
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From 299781265a6e21436b442bc962d50d737aaffcb9 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 3 Jan 2012 19:17:36 +0000
Subject: [PATCH 2898/8313] Added a comment

---
 .../comment_4_75b0e072e668aa46ff0a8d62a6620306._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment

diff --git a/doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment b/doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment
new file mode 100644
index 0000000000..642f952e4d
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_4_75b0e072e668aa46ff0a8d62a6620306._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 4"
+ date="2012-01-03T19:17:36Z"
+ content="""
+Okay, I see, but is `git annex get --auto .` going to import all those files from the work remote into my home if the `master` branch of that remote isn't merged?
+"""]]

From 4fee95dfc63e24c2313a7481832ea0d0ebd184f2 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 3 Jan 2012 19:31:45 +0000
Subject: [PATCH 2899/8313] Added a comment

---
 .../comment_5_f5ec9649d9f1dc122e715de5533bc674._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment

diff --git a/doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment b/doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment
new file mode 100644
index 0000000000..5b2f3dad55
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_5_f5ec9649d9f1dc122e715de5533bc674._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2012-01-03T19:31:45Z"
+ content="""
+Yes; git-annex uses the git-annex branch independently of the branch you have checked out. You may find [[internals]] interesting reading, but the short answer is it will work.
+"""]]

From dda78c61b87496aa87bdb93f7e1c2203cb69610b Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 3 Jan 2012 19:47:12 +0000
Subject: [PATCH 2900/8313] Added a comment

---
 .../comment_6_85df530f7b6d76b74ac8017c6034f95e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment

diff --git a/doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment b/doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment
new file mode 100644
index 0000000000..77af85258b
--- /dev/null
+++ b/doc/forum/git-subtree_support__63__/comment_6_85df530f7b6d76b74ac8017c6034f95e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.188.44.113"
+ subject="comment 6"
+ date="2012-01-03T19:47:11Z"
+ content="""
+Very cool! Thank you for the explanation.
+"""]]

From 1e929c022d3004461c238d4b8230a4ce9a5b106f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 18:36:31 -0400
Subject: [PATCH 2901/8313] typo

---
 Utility/Monad.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index be48072cbf..95964361e6 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -20,7 +20,7 @@ firstM p (x:xs) = do
 		then return (Just x)
 		else firstM p xs
 
-{- Returns true if any value in the list satisfies the preducate,
+{- Returns true if any value in the list satisfies the predicate,
  - stopping once one is found. -}
 anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
 anyM p = liftM isJust . firstM p

From a1aea174d7bc0392cf1cdd71ecc9b863923f1940 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 18:39:39 -0400
Subject: [PATCH 2902/8313] fsck: Do backend-specific check before checking
 numcopies is satisfied.

This way, when a checksum check fails and the content is moved aside,
the numcopies check also warns if there are not enough copies.
---
 Command/Fsck.hs  | 2 +-
 debian/changelog | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 4e83455e1b..723a2e7408 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -40,8 +40,8 @@ perform key file backend numcopies = check
 	-- order matters
 	[ verifyLocationLog key file
 	, checkKeySize key
-	, checkKeyNumCopies key file numcopies
 	, checkBackend backend key
+	, checkKeyNumCopies key file numcopies
 	]
 
 {- To fsck a bare repository, fsck each key in the location log. -}
diff --git a/debian/changelog b/debian/changelog
index 6075053879..d6dc7d2b5a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,7 @@ git-annex (3.20111232) UNRELEASED; urgency=low
     dotdirs.
   * Added quickcheck to build dependencies, and fail if test suite cannot be
     built.
+  * fsck: Do backend-specific check before checking numcopies is satisfied.
 
  -- Joey Hess   Mon, 02 Jan 2012 14:19:19 -0400
 

From 2c121d23ceee9abaf7d2de0484e333047de97c49 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 20:56:44 -0400
Subject: [PATCH 2903/8313] add how it works page

---
 doc/how_it_works.mdwn | 41 +++++++++++++++++++++++++++++++++++++++++
 doc/index.mdwn        |  2 +-
 2 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 doc/how_it_works.mdwn

diff --git a/doc/how_it_works.mdwn b/doc/how_it_works.mdwn
new file mode 100644
index 0000000000..0acf205652
--- /dev/null
+++ b/doc/how_it_works.mdwn
@@ -0,0 +1,41 @@
+This page gives a high-level view of git-annex. For a detailed
+low-level view, see [[the_man_page|git-annex]] and [[internals]].
+
+You do not need to read this page to get started with using git-annex. The
+[[walkthrough]] provides step-by-step instructions.
+
+Still reading? Ok. Git's man page calls it "a stupid content
+tracker". With git-annex, git is instead "a stupid filename and metadata"
+tracker. The contents of large files are not stored in git, only the
+names of the files and some other metadata remain there.
+
+The contents of the files are kept by git-annex in a distributed key/value
+store consisting of every clone of a given git repository. That's a fancy
+way to say that git-annex stores the actual file content somewhere under
+`.git/annex/`. (See [[internals]] for details.)
+
+That was the values; what about the keys? Well, a key is calculated for a
+given file when it's first added into git-annex. Normally this uses a hash
+of its contents, but various [[backends]] can produce different sorts of
+keys. The file that gets checked into git is just a symlink to the key
+under `.git/annex/`. If the content of a file is modified, that produces
+a different key (and the symlink is changed).
+
+A file's content can be [[transferred|transferring_data]] from one
+repository to another by git-annex. Which repositories contain a given
+value is tracked by git-annex (see [[location_tracking]]). It stores this
+tracking information in a separate branch, named "git-annex". All you ever
+do with the "git-annex" branch is push/pull it around between repositories,
+to [[sync]] up git-annex's view of the world.
+
+That's really all there is to it. Oh, there are [[special_remotes]] that
+let values be stored other places than git repositories (anything from
+Amazon S3 to a USB key), and there's a pile of commands listed in
+[[the_man_page|git-annex]] to handle moving the values around and managing
+them. But if you grok the description above, you can see through all that.
+It's really just symlinks, keys, values, and a git-annex branch to store
+additional metadata.
+
+---
+
+Next: [[install]] or [[walkthrough]]
diff --git a/doc/index.mdwn b/doc/index.mdwn
index e0791bf71a..50d48a5f78 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -1,6 +1,6 @@
 [[!inline raw=yes pages="summary"]]
 
-To get a feel for it, see the [[walkthrough]].
+To get a feel for it, see the [[walkthrough]] or read about [[how_it_works]].
 
 [[!sidebar content="""
 [[!img logo_small.png link=no]]

From 629b794eb5dd339e6278a0791d424877f40a6ff3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Jan 2012 21:12:01 -0400
Subject: [PATCH 2904/8313] add sitemap

---
 doc/index.mdwn   | 4 +++-
 doc/sitemap.mdwn | 3 +++
 2 files changed, 6 insertions(+), 1 deletion(-)
 create mode 100644 doc/sitemap.mdwn

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 50d48a5f78..e0dbb69438 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -51,11 +51,13 @@ files with git.
 * [[internals]]
 * [[design]]
 * [[what git annex is not|not]]
-* git-annex is Free Software, licensed under the [[GPL]].
+* [[sitemap]]
 
 

 
 ----
 
+git-annex is Free Software, licensed under the [[GPL]].
+
 git-annex's wiki is powered by [Ikiwiki](http://ikiwiki.info/) and
 hosted by [Branchable](http://branchable.com/).
diff --git a/doc/sitemap.mdwn b/doc/sitemap.mdwn
new file mode 100644
index 0000000000..2e6dddb3fa
--- /dev/null
+++ b/doc/sitemap.mdwn
@@ -0,0 +1,3 @@
+[[!map pages="page(*) and !*/discussion and !recentchanges
+and !bugs/* and !examples/*/* and !news/* and !tips/*
+and !forum/* and !todo/* and !users/* and !ikiwiki/*"]]

From de6a2db36cb97165fed283e6f5063594b443d95a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Wed, 4 Jan 2012 03:44:52 +0000
Subject: [PATCH 2905/8313]

---
 doc/forum/Auto_archiving.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Auto_archiving.mdwn

diff --git a/doc/forum/Auto_archiving.mdwn b/doc/forum/Auto_archiving.mdwn
new file mode 100644
index 0000000000..9a7a19cb7b
--- /dev/null
+++ b/doc/forum/Auto_archiving.mdwn
@@ -0,0 +1,15 @@
+I've been toying with the idea of auto archiving files (that is - removing them from .) based on a set of rules.
+
+Git already provides attribute management for files and I put together a simple script that tries to achieve the following:
+
+* Look for files with X copies (including a local one)
+* Verify that the file is not recent (defined as not having been dropped or getted within X days )
+* Verify that the file has an annex.archive attribute
+* Archive if the above is met
+
+Here is the script:
+http://pastebin.com/53iLqyPd
+
+You just add the annex.archive attribute to files via .gitattributes to use
+
+Open to thoughts / suggestions

From 9298ea0f3a296d77ecc12314b6a7e161ebeb483b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Wed, 4 Jan 2012 03:51:29 +0000
Subject: [PATCH 2906/8313]

---
 doc/forum/Auto_archiving.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/Auto_archiving.mdwn b/doc/forum/Auto_archiving.mdwn
index 9a7a19cb7b..f32a648c23 100644
--- a/doc/forum/Auto_archiving.mdwn
+++ b/doc/forum/Auto_archiving.mdwn
@@ -12,4 +12,6 @@ http://pastebin.com/53iLqyPd
 
 You just add the annex.archive attribute to files via .gitattributes to use
 
+The script runs in preview mode... exec with autoArchive.sh commit to drop the files.
+
 Open to thoughts / suggestions

From 156de9fc40633ff43d8d26bef88cafb29952a37a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 00:45:21 -0400
Subject: [PATCH 2907/8313] add twitter feed

---
 doc/feeds.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/feeds.mdwn b/doc/feeds.mdwn
index 25a3190497..df09e76688 100644
--- a/doc/feeds.mdwn
+++ b/doc/feeds.mdwn
@@ -1,3 +1,5 @@
 Aggregating git-annex mentions from elsewhere on the net..
 
 * [[!aggregate expirecount=25 name="identica" feedurl="http://identi.ca/api/statusnet/tags/timeline/gitannex.rss" url="http://identi.ca/tag/gitannex"]]
+* [[!aggregate expirecount=25 name="twitter" feedurl="http://search.twitter.com/search.atom?q=git-annex" url="http://twitter.com/"]]
+

From 769edd6b08b12c61fd32c6c37aa8352a4100a39a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 13:44:09 -0400
Subject: [PATCH 2908/8313] Run gpg with --no-tty. Closes: #654721

---
 Utility/Gpg.hs   | 5 +++--
 debian/changelog | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index f3a1ac0bb5..b50e775d03 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -28,8 +28,9 @@ stdParams params = do
 	let batch = if isNothing e then [] else ["--batch"]
 	return $ batch ++ defaults ++ toCommand params
 	where
-		-- be quiet, even about checking the trustdb
-		defaults = ["--quiet", "--trust-model", "always"]
+		-- be quiet, even about checking the trustdb,
+		-- and avoid using a tty
+		defaults = ["--quiet", "--trust-model", "always", "--no-tty"]
 
 {- Runs gpg with some params and returns its stdout, strictly. -}
 readStrict :: [CommandParam] -> IO String
diff --git a/debian/changelog b/debian/changelog
index d6dc7d2b5a..18e241e8da 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20111232) UNRELEASED; urgency=low
+git-annex (3.20120105) UNRELEASED; urgency=low
 
   * Added annex-web-options configuration settings, which can be
     used to provide parameters to whichever of wget or curl git-annex uses
@@ -11,6 +11,7 @@ git-annex (3.20111232) UNRELEASED; urgency=low
   * Added quickcheck to build dependencies, and fail if test suite cannot be
     built.
   * fsck: Do backend-specific check before checking numcopies is satisfied.
+  * Run gpg with --no-tty. Closes: #654721
 
  -- Joey Hess   Mon, 02 Jan 2012 14:19:19 -0400
 

From 338d472ca2507f2eee5bfe45f0e2fc19501ed91e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 13:51:13 -0400
Subject: [PATCH 2909/8313] releasing version 3.20120105

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 18e241e8da..721dde450a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120105) UNRELEASED; urgency=low
+git-annex (3.20120105) unstable; urgency=low
 
   * Added annex-web-options configuration settings, which can be
     used to provide parameters to whichever of wget or curl git-annex uses
@@ -13,7 +13,7 @@ git-annex (3.20120105) UNRELEASED; urgency=low
   * fsck: Do backend-specific check before checking numcopies is satisfied.
   * Run gpg with --no-tty. Closes: #654721
 
- -- Joey Hess   Mon, 02 Jan 2012 14:19:19 -0400
+ -- Joey Hess   Thu, 05 Jan 2012 13:44:12 -0400
 
 git-annex (3.20111231) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 93238f49ff..bc87b467c8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20111231
+Version: 3.20120105
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 3e3ed62bdfd3860945e11f648ef7d90c7bacfe98 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 13:51:32 -0400
Subject: [PATCH 2910/8313] add news item for git-annex 3.20120105

---
 doc/news/version_3.20111111.mdwn | 10 ----------
 doc/news/version_3.20120105.mdwn | 14 ++++++++++++++
 2 files changed, 14 insertions(+), 10 deletions(-)
 delete mode 100644 doc/news/version_3.20111111.mdwn
 create mode 100644 doc/news/version_3.20120105.mdwn

diff --git a/doc/news/version_3.20111111.mdwn b/doc/news/version_3.20111111.mdwn
deleted file mode 100644
index 2173400154..0000000000
--- a/doc/news/version_3.20111111.mdwn
+++ /dev/null
@@ -1,10 +0,0 @@
-git-annex 3.20111111 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Handle a case where an annexed file is moved into a gitignored directory,
-     by having fix --force add its change.
-   * Avoid cyclic drop problems.
-   * Optimized copy --from and get --from to avoid checking the location log
-     for files that are already present.
-   * Automatically fix up badly formatted uuid.log entries produced by
-     3.20111105, whenever the uuid.log is changed (ie, by init or describe).
-   * map: Support remotes with /~/ and /~user/"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120105.mdwn b/doc/news/version_3.20120105.mdwn
new file mode 100644
index 0000000000..2bec87cdab
--- /dev/null
+++ b/doc/news/version_3.20120105.mdwn
@@ -0,0 +1,14 @@
+git-annex 3.20120105 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Added annex-web-options configuration settings, which can be
+     used to provide parameters to whichever of wget or curl git-annex uses
+     (depends on which is available, but most of their important options
+     suitable for use here are the same).
+   * Dotfiles, and files inside dotdirs are not added by "git annex add"
+     unless the dotfile or directory is explicitly listed. So "git annex add ."
+     will add all untracked files in the current directory except for those in
+     dotdirs.
+   * Added quickcheck to build dependencies, and fail if test suite cannot be
+     built.
+   * fsck: Do backend-specific check before checking numcopies is satisfied.
+   * Run gpg with --no-tty. Closes: #[654721](http://bugs.debian.org/654721)"""]]
\ No newline at end of file

From 0b27e6baa0bd6ed61e2369bd5452d61f3e63e545 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 14:32:20 -0400
Subject: [PATCH 2911/8313] Support unescaped repository urls, like git does.

Turns out that git will accept a .git/config containing an url with eg,
spaces in its name. Handle this by escaping the url if it's not valid.

This also fixes support for urls containing escaped characters like %20
for space. Before, the path from the url was not unescaped properly.
---
 Git.hs           |  4 ++--
 Git/Construct.hs | 20 +++++++++++++++-----
 debian/changelog |  6 ++++++
 3 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/Git.hs b/Git.hs
index 9420810a67..d8db471e90 100644
--- a/Git.hs
+++ b/Git.hs
@@ -29,7 +29,7 @@ module Git (
 
 import qualified Data.Map as M
 import Data.Char
-import Network.URI (uriPath, uriScheme)
+import Network.URI (uriPath, uriScheme, unEscapeString)
 
 import Common
 import Git.Types
@@ -107,7 +107,7 @@ gitDir repo
  -
  - Note that for URL repositories, this is the path on the remote host. -}
 workTree :: Repo -> FilePath
-workTree Repo { location = Url u } = uriPath u
+workTree Repo { location = Url u } = unEscapeString $ uriPath u
 workTree Repo { location = Dir d } = d
 workTree Repo { location = Unknown } = undefined
 
diff --git a/Git/Construct.hs b/Git/Construct.hs
index 2cc965b4ee..51fa656bc3 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -60,14 +60,23 @@ fromAbsPath dir
 	where
 		ret = return . newFrom . Dir
 
-{- Remote Repo constructor. Throws exception on invalid url. -}
+{- Remote Repo constructor. Throws exception on invalid url.
+ -
+ - Git is somewhat forgiving about urls to repositories, allowing
+ - eg spaces that are not normally allowed unescaped in urls.
+ -}
 fromUrl :: String -> IO Repo
 fromUrl url
+	| not (isURI url) = fromUrlStrict $ escapeURIString isUnescapedInURI url
+	| otherwise = fromUrlStrict url
+
+fromUrlStrict :: String -> IO Repo
+fromUrlStrict url
 	| startswith "file://" url = fromAbsPath $ uriPath u
 	| otherwise = return $ newFrom $ Url u
-		where
-			u = fromMaybe bad $ parseURI url
-			bad = error $ "bad url " ++ url
+	where
+		u = fromMaybe bad $ parseURI url
+		bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
 fromUnknown :: IO Repo
@@ -117,7 +126,7 @@ fromRemoteLocation s repo = gen $ calcloc s
 	where
 		gen v	
 			| scpstyle v = fromUrl $ scptourl v
-			| isURI v = fromUrl v
+			| urlstyle v = fromUrl v
 			| otherwise = fromRemotePath v repo
 		-- insteadof config can rewrite remote location
 		calcloc l
@@ -137,6 +146,7 @@ fromRemoteLocation s repo = gen $ calcloc s
 						M.toList $ fullconfig repo
 				splitconfigs (k, vs) = map (\v -> (k, v)) vs
 				(prefix, suffix) = ("url." , ".insteadof")
+		urlstyle v = isURI v || ":" `isInfixOf` v && "//" `isInfixOf` v
 		-- git remotes can be written scp style -- [user@]host:dir
 		scpstyle v = ":" `isInfixOf` v && not ("//" `isInfixOf` v)
 		scptourl v = "ssh://" ++ host ++ slash dir
diff --git a/debian/changelog b/debian/changelog
index 721dde450a..e5687aac14 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120106) UNRELEASED; urgency=low
+
+  * Support unescaped repository urls, like git does.
+
+ -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400
+
 git-annex (3.20120105) unstable; urgency=low
 
   * Added annex-web-options configuration settings, which can be

From 88d3443aeaa8a74b1b3a31612270ab1671177199 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Thu, 5 Jan 2012 20:58:05 +0000
Subject: [PATCH 2912/8313]

---
 doc/transferring_data.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/transferring_data.mdwn b/doc/transferring_data.mdwn
index 57873f6f0e..d1ec5963f5 100644
--- a/doc/transferring_data.mdwn
+++ b/doc/transferring_data.mdwn
@@ -10,7 +10,7 @@ to allow it to be automatically resumed later.
 
 It's equally easy to transfer a single file to or from a repository,
 or to launch a retrievel of a massive pile of files from whatever
-repositories they are scattered amoung.
+repositories they are scattered amongst.
 
 git-annex automatically uses whatever remotes are currently accessible,
 preferring ones that are less expensive to talk to.

From ba0c3eb904706fe2afb0b0425390e638883c3c67 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:25:08 +0000
Subject: [PATCH 2913/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 doc/bugs/Lost_S3_Remote.mdwn

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
new file mode 100644
index 0000000000..80e90d5933
--- /dev/null
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -0,0 +1,36 @@
+Somehow I've lost my S3 remote... git-annex knows it's there, but its not associating it with the git remote in .git/config
+
+$ git-annex whereis pebuilder.iso 
+whereis pebuilder.iso (3 copies) 
+  	3b6fc6f6-3025-11e1-b496-33bffbc0f3ed -- housebackup (external seagate drive on /mnt/back/RemoteStore)
+   	6b1326d8-2abb-11e1-8f43-979159a7f900 -- synology
+   	9b297772-2ab2-11e1-a86f-2fd669cb2417 -- Amazon S3
+ok
+
+Amazon S3 is the description from the remote.  My .git/config file contains this block:
+[remote "cloud"]
+        annex-s3 = true
+        annex-uuid = 9b297772-2ab2-11e1-a86f-2fd669cb2417
+        annex-cost = 70
+
+The UUID matches... But I cannot access it... see below:
+
+[39532:39531 - 0:626] 08:20:38 [vivitron@tronlap:o +3] ~/annex/ISO 
+$ git-annex get pebuilder.iso --from=cloud
+git-annex: there is no git remote named "cloud"
+
+[39532:39531 - 0:627] 08:20:56 [vivitron@tronlap:o +3] ~/annex/ISO 
+$ git-annex get pebuilder.iso --from="Amazon S3"
+git-annex: there is no git remote named "Amazon S3"
+
+[39532:39531 - 0:628] 08:21:01 [vivitron@tronlap:o +3] ~/annex/ISO 
+$ git-annex get pebuilder.iso --from=9b297772-2ab2-11e1-a86f-2fd669cb2417
+git-annex: there is no git remote named "9b297772-2ab2-11e1-a86f-2fd669cb2417"
+
+[39532:39531 - 0:629] 08:21:08 [vivitron@tronlap:o +3] ~/annex/ISO 
+$ 
+
+
+I appreciate any help....
+
+

From 47d98bb4466b32f9e5d540b1610fda82bc40b78a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:27:26 +0000
Subject: [PATCH 2914/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 45 ++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index 80e90d5933..97ad8f961f 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -1,36 +1,37 @@
 Somehow I've lost my S3 remote... git-annex knows it's there, but its not associating it with the git remote in .git/config
 
-$ git-annex whereis pebuilder.iso 
-whereis pebuilder.iso (3 copies) 
-  	3b6fc6f6-3025-11e1-b496-33bffbc0f3ed -- housebackup (external seagate drive on /mnt/back/RemoteStore)
+    $ git-annex whereis pebuilder.iso 
+    whereis pebuilder.iso (3 copies) 
+      	3b6fc6f6-3025-11e1-b496-33bffbc0f3ed -- housebackup (external seagate drive on /mnt/back/RemoteStore)
    	6b1326d8-2abb-11e1-8f43-979159a7f900 -- synology
    	9b297772-2ab2-11e1-a86f-2fd669cb2417 -- Amazon S3
-ok
+    ok
 
 Amazon S3 is the description from the remote.  My .git/config file contains this block:
-[remote "cloud"]
-        annex-s3 = true
-        annex-uuid = 9b297772-2ab2-11e1-a86f-2fd669cb2417
-        annex-cost = 70
+
+    [remote "cloud"]
+      annex-s3 = true
+      annex-uuid = 9b297772-2ab2-11e1-a86f-2fd669cb2417
+      annex-cost = 70
 
 The UUID matches... But I cannot access it... see below:
 
-[39532:39531 - 0:626] 08:20:38 [vivitron@tronlap:o +3] ~/annex/ISO 
-$ git-annex get pebuilder.iso --from=cloud
-git-annex: there is no git remote named "cloud"
+    [39532:39531 - 0:626] 08:20:38 [vivitron@tronlap:o +3] ~/annex/ISO 
+    $ git-annex get pebuilder.iso --from=cloud
+    git-annex: there is no git remote named "cloud"
+    
+    [39532:39531 - 0:627] 08:20:56 [vivitron@tronlap:o +3] ~/annex/ISO 
+    $ git-annex get pebuilder.iso --from="Amazon S3"
+    git-annex: there is no git remote named "Amazon S3"
+    
+    [39532:39531 - 0:628] 08:21:01 [vivitron@tronlap:o +3] ~/annex/ISO 
+    $ git-annex get pebuilder.iso --from=9b297772-2ab2-11e1-a86f-2fd669cb2417
+    git-annex: there is no git remote named "9b297772-2ab2-11e1-a86f-2fd669cb2417"
 
-[39532:39531 - 0:627] 08:20:56 [vivitron@tronlap:o +3] ~/annex/ISO 
-$ git-annex get pebuilder.iso --from="Amazon S3"
-git-annex: there is no git remote named "Amazon S3"
-
-[39532:39531 - 0:628] 08:21:01 [vivitron@tronlap:o +3] ~/annex/ISO 
-$ git-annex get pebuilder.iso --from=9b297772-2ab2-11e1-a86f-2fd669cb2417
-git-annex: there is no git remote named "9b297772-2ab2-11e1-a86f-2fd669cb2417"
-
-[39532:39531 - 0:629] 08:21:08 [vivitron@tronlap:o +3] ~/annex/ISO 
-$ 
+    [39532:39531 - 0:629] 08:21:08 [vivitron@tronlap:o +3] ~/annex/ISO 
+    $ 
 
 
-I appreciate any help....
+I appreciate any help....  I've tested versions 3.20111211 and 3.20111231
 
 

From 17d86461646db6262f835793fc56c92e897eb2d9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:27:53 +0000
Subject: [PATCH 2915/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index 97ad8f961f..95d7f509b6 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -34,4 +34,7 @@ The UUID matches... But I cannot access it... see below:
 
 I appreciate any help....  I've tested versions 3.20111211 and 3.20111231
 
+    $ git --version
+    git version 1.7.8.1
+
 

From aedd4c19843d547269bf32e25ba2d8af18507778 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:31:40 +0000
Subject: [PATCH 2916/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index 95d7f509b6..c7cc7b4c6c 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -32,7 +32,7 @@ The UUID matches... But I cannot access it... see below:
     $ 
 
 
-I appreciate any help....  I've tested versions 3.20111211 and 3.20111231
+I appreciate any help....  I've tested versions 3.20111211, 3.20111231, and 3.20120105
 
     $ git --version
     git version 1.7.8.1

From 9cb3303a2780bc87567a43ded4cede87b2cfa946 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:40:04 +0000
Subject: [PATCH 2917/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index c7cc7b4c6c..0041fa0322 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -31,6 +31,15 @@ The UUID matches... But I cannot access it... see below:
     [39532:39531 - 0:629] 08:21:08 [vivitron@tronlap:o +3] ~/annex/ISO 
     $ 
 
+git remote lists "cloud" as a remote:
+
+  $ git remote
+  all
+  cloud
+  cs
+  es3
+
+
 
 I appreciate any help....  I've tested versions 3.20111211, 3.20111231, and 3.20120105
 

From 79b95aae090c31f5181e5392140d636a036e7d36 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:40:45 +0000
Subject: [PATCH 2918/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index 0041fa0322..7b1a750957 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -39,6 +39,12 @@ git remote lists "cloud" as a remote:
   cs
   es3
 
+git-annex status lists S3 support:
+
+    $ git-annex status
+    supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+    supported remote types: git S3 bup directory rsync web hook
+
 
 
 I appreciate any help....  I've tested versions 3.20111211, 3.20111231, and 3.20120105

From 2f2e0f7e0e159ace3aab70d53d899a93a7614d1b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 01:41:27 +0000
Subject: [PATCH 2919/8313]

---
 doc/bugs/Lost_S3_Remote.mdwn | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index 7b1a750957..7f8b6b5305 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -33,11 +33,12 @@ The UUID matches... But I cannot access it... see below:
 
 git remote lists "cloud" as a remote:
 
-  $ git remote
-  all
-  cloud
-  cs
-  es3
+    $ git remote
+    all
+    cloud
+    cs
+    es3
+    origin
 
 git-annex status lists S3 support:
 

From 8858c24210e460c9697bdde69e301d807680e999 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 6 Jan 2012 03:04:35 +0000
Subject: [PATCH 2920/8313] Added a comment

---
 ...comment_1_6e80e6db6671581d471fc9a54181c04c._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment

diff --git a/doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment b/doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment
new file mode 100644
index 0000000000..b4f7bdc3cf
--- /dev/null
+++ b/doc/bugs/Lost_S3_Remote/comment_1_6e80e6db6671581d471fc9a54181c04c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-06T03:04:35Z"
+ content="""
+Despite `status` listing S3 support, your git-annex is actually built with S3stub, probably because it failed to find the necessary S3 module at build time. Rebuild git-annex and watch closely, you'll see \"** building without S3 support\". Look above that for the error and fix it.
+
+It was certianly a bug that it showed S3 as supported when built without it. I've fixed that.
+"""]]

From cdcf0aac71bf6135d45b7f5d13342ed3d040596a Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 6 Jan 2012 03:08:28 +0000
Subject: [PATCH 2921/8313] Added a comment

---
 .../comment_2_c99c65882a3924f4890e500f9492b442._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment

diff --git a/doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment b/doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment
new file mode 100644
index 0000000000..6112bc0890
--- /dev/null
+++ b/doc/bugs/Lost_S3_Remote/comment_2_c99c65882a3924f4890e500f9492b442._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-01-06T03:08:28Z"
+ content="""
+BTW, you'll want to \"make clean\", since the S3stub hack symlinks a file into place and it will continue building with S3stub even if you fix the problem until you clean.
+"""]]

From 47be4383b714320c9e3f49cc23315101fad5735b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 20:40:31 -0400
Subject: [PATCH 2922/8313] Command data structure tweaking

---
 CmdLine.hs         |  4 ++--
 Types/Command.hs   | 10 +++++-----
 git-annex-shell.hs |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 7f708f15a1..fb2792cf4b 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -67,8 +67,8 @@ usage header cmds options = usageInfo top options ++ commands
 		showcmd c =
 			cmdname c ++
 			pad (longest cmdname + 1) (cmdname c) ++
-			cmdparams c ++
-			pad (longest cmdparams + 2) (cmdparams c) ++
+			cmdparamdesc c ++
+			pad (longest cmdparamdesc + 2) (cmdparamdesc c) ++
 			cmddesc c
 		pad n s = replicate (n - length s) ' '
 		longest f = foldl max 0 $ map (length . f) cmds
diff --git a/Types/Command.hs b/Types/Command.hs
index 5341a40545..3cabf7318a 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -33,12 +33,12 @@ type CommandCleanup = Annex Bool
 
 {- A command is defined by specifying these things. -}
 data Command = Command {
-	cmdnorepo :: Maybe (IO ()),
-	cmdcheck :: [CommandCheck],
+	cmdnorepo :: Maybe (IO ()), -- an action to run when not in a repo
+	cmdcheck :: [CommandCheck], -- check stage
 	cmdname :: String,
-	cmdparams :: String,
-	cmdseek :: [CommandSeek],
-	cmddesc :: String
+	cmdparamdesc :: String,     -- description of params for usage
+	cmdseek :: [CommandSeek],   -- seek stage
+	cmddesc :: String           -- description of command for usage
 }
 
 {- CommandCheck functions can be compared using their unique id. -}
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 872dabc58b..1ff0bba447 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -36,7 +36,9 @@ cmds_notreadonly = concat
 cmds :: [Command]
 cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
 	where
-		adddirparam c = c { cmdparams = "DIRECTORY " ++ cmdparams c }
+		adddirparam c = c
+			{ cmdparamdesc = "DIRECTORY " ++ cmdparamdesc c
+			}
 
 options :: [OptDescr (Annex ())]
 options = commonOptions ++

From ad43f0362688a601ba43f462e80f5a91bf398c02 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 22:48:59 -0400
Subject: [PATCH 2923/8313] per-command options

Finally commands can define their own options.

Moved --format and --print0 to be options only of find.
---
 CmdLine.hs        | 41 +++++++++--------------
 Command.hs        |  9 ++++-
 Command/Find.hs   |  9 +++--
 Command/Status.hs |  4 +--
 GitAnnex.hs       |  7 ----
 Options.hs        | 60 +++++++++++----------------------
 Types/Command.hs  | 18 +++++-----
 Types/Option.hs   | 17 ++++++++++
 Usage.hs          | 84 +++++++++++++++++++++++++++++++++++++++++++++++
 9 files changed, 162 insertions(+), 87 deletions(-)
 create mode 100644 Types/Option.hs
 create mode 100644 Usage.hs

diff --git a/CmdLine.hs b/CmdLine.hs
index fb2792cf4b..6ac0b423fd 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -29,7 +29,7 @@ type Flags = [Annex ()]
 
 {- Runs the passed command line. -}
 dispatch :: Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
-dispatch args cmds options header getgitrepo = do
+dispatch args cmds commonoptions header getgitrepo = do
 	setupConsole
 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
 	case r of
@@ -41,37 +41,26 @@ dispatch args cmds options header getgitrepo = do
 				prepCommand cmd params
 			tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
 	where
-		(flags, cmd, params) = parseCmd args cmds options header
+		(flags, cmd, params) = parseCmd args cmds commonoptions header
 
 {- Parses command line, and returns actions to run to configure flags,
  - the Command being run, and the remaining parameters for the command. -} 
 parseCmd :: Params -> [Command] -> [Option] -> String -> (Flags, Command, Params)
-parseCmd argv cmds options header = check $ getOpt Permute options argv
+parseCmd argv cmds commonoptions header
+	| name == Nothing = err "missing command"
+	| null matches = err $ "unknown command " ++ fromJust name
+	| otherwise = check $ getOpt Permute (commonoptions ++ cmdoptions cmd) args
 	where
-		check (_, [], []) = err "missing command"
-		check (flags, name:rest, [])
-			| null matches = err $ "unknown command " ++ name
-			| otherwise = (flags, Prelude.head matches, rest)
-			where
-				matches = filter (\c -> name == cmdname c) cmds
+		(name, args) = findname argv []
+		findname [] c = (Nothing, reverse c)
+		findname (a:as) c
+			| "-" `isPrefixOf` a = findname as (a:c)
+			| otherwise = (Just a, reverse c ++ as)
+		matches = filter (\c -> name == Just (cmdname c)) cmds
+		cmd = Prelude.head matches
+		check (flags, rest, []) = (flags, cmd, rest)
 		check (_, _, errs) = err $ concat errs
-		err msg = error $ msg ++ "\n\n" ++ usage header cmds options
-
-{- Usage message with lists of commands and options. -}
-usage :: String -> [Command] -> [Option] -> String
-usage header cmds options = usageInfo top options ++ commands
-	where
-		top = header ++ "\n\nOptions:"
-		commands = "\nCommands:\n" ++ cmddescs
-		cmddescs = unlines $ map (indent . showcmd) cmds
-		showcmd c =
-			cmdname c ++
-			pad (longest cmdname + 1) (cmdname c) ++
-			cmdparamdesc c ++
-			pad (longest cmdparamdesc + 2) (cmdparamdesc c) ++
-			cmddesc c
-		pad n s = replicate (n - length s) ' '
-		longest f = foldl max 0 $ map (length . f) cmds
+		err msg = error $ msg ++ "\n\n" ++ usage header cmds commonoptions
 
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
diff --git a/Command.hs b/Command.hs
index dea6a97a3e..b287629ae4 100644
--- a/Command.hs
+++ b/Command.hs
@@ -8,6 +8,7 @@
 module Command (
 	command,
 	noRepo,
+	withOptions,
 	next,
 	stop,
 	stopUnless,
@@ -26,22 +27,28 @@ import qualified Backend
 import qualified Annex
 import qualified Git
 import Types.Command as ReExported
+import Types.Option as ReExported
 import Seek as ReExported
 import Checks as ReExported
 import Options as ReExported
+import Usage as ReExported
 import Logs.Trust
 import Logs.Location
 import Config
 
 {- Generates a normal command -}
 command :: String -> String -> [CommandSeek] -> String -> Command
-command = Command Nothing commonChecks
+command = Command [] Nothing commonChecks
 
 {- Adds a fallback action to a command, that will be run if it's used
  - outside a git repository. -}
 noRepo :: IO () -> Command -> Command
 noRepo a c = c { cmdnorepo = Just a }
 
+{- Adds options to a command. -}
+withOptions :: [Option] -> Command -> Command
+withOptions o c = c { cmdoptions = o }
+
 {- For start and perform stages to indicate what step to run next. -}
 next :: a -> Annex (Maybe a)
 next a = return $ Just a
diff --git a/Command/Find.hs b/Command/Find.hs
index 0c96369ee9..c86db5fa65 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -19,7 +19,12 @@ import Utility.DataUnits
 import Types.Key
 
 def :: [Command]
-def = [command "find" paramPaths seek "lists available files"]
+def = [withOptions [formatOption, print0Option] $
+	command "find" paramPaths seek "lists available files"]
+
+print0Option :: Option
+print0Option = Option [] ["print0"] (NoArg $ setFormat "${file}\0")
+	"terminate output with null"
 
 seek :: [CommandSeek]
 seek = [withFilesInGit $ whenAnnexed start]
diff --git a/Command/Status.hs b/Command/Status.hs
index 736d897ef3..d2d8d4c077 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -144,9 +144,9 @@ bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
 backend_usage = stat "backend usage" $ nojson $
-	usage <$> cachedKeysReferenced <*> cachedKeysPresent
+	calc <$> cachedKeysReferenced <*> cachedKeysPresent
 	where
-		usage a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
+		calc a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
 		splits :: [Key] -> [(String, Integer)]
 		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
 		tcount k = (keyBackendName k, 1)
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7243d69cb0..3ce4518108 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -18,7 +18,6 @@ import Types.TrustLevel
 import qualified Annex
 import qualified Remote
 import qualified Limit
-import qualified Utility.Format
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -109,10 +108,6 @@ options = commonOptions ++
 		"override trust setting to untrusted"
 	, Option ['c'] ["config"] (ReqArg setgitconfig "NAME=VALUE")
 		"override git configuration setting"
-	, Option [] ["print0"] (NoArg setprint0)
-		"terminate output with null"
-	, Option [] ["format"] (ReqArg setformat paramFormat)
-		"control format of output"
 	, Option ['x'] ["exclude"] (ReqArg Limit.addExclude paramGlob)
 		"skip files matching the glob pattern"
 	, Option ['I'] ["include"] (ReqArg Limit.addInclude paramGlob)
@@ -128,8 +123,6 @@ options = commonOptions ++
 		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
 		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
-		setformat v = Annex.changeState $ \s -> s { Annex.format = Just $ Utility.Format.gen v }
-		setprint0 = setformat "${file}\0"
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			newg <- inRepo $ Git.Config.store v
diff --git a/Options.hs b/Options.hs
index cce750316e..fa008d0644 100644
--- a/Options.hs
+++ b/Options.hs
@@ -5,7 +5,15 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Options where
+module Options (
+	commonOptions,
+	matcherOptions,
+	formatOption,
+	setFormat,
+	ArgDescr(..),
+	Option,
+	OptDescr(..),
+) where
 
 import System.Console.GetOpt
 import System.Log.Logger
@@ -13,11 +21,9 @@ import System.Log.Logger
 import Common.Annex
 import qualified Annex
 import Limit
-
-{- Each dashed command-line option results in generation of an action
- - in the Annex monad that performs the necessary setting.
- -}
-type Option = OptDescr (Annex ())
+import Types.Option
+import Usage
+import qualified Utility.Format
 
 commonOptions :: [Option]
 commonOptions =
@@ -59,38 +65,10 @@ matcherOptions =
 		longopt o = Option [] [o] $ NoArg $ addToken o
 		shortopt o = Option o [] $ NoArg $ addToken o
 
-{- Descriptions of params used in usage messages. -}
-paramPaths :: String
-paramPaths = paramOptional $ paramRepeating paramPath -- most often used
-paramPath :: String
-paramPath = "PATH"
-paramKey :: String
-paramKey = "KEY"
-paramDesc :: String
-paramDesc = "DESC"
-paramUrl :: String
-paramUrl = "URL"
-paramNumber :: String
-paramNumber = "NUMBER"
-paramRemote :: String
-paramRemote = "REMOTE"
-paramGlob :: String
-paramGlob = "GLOB"
-paramName :: String
-paramName = "NAME"
-paramUUID :: String
-paramUUID = "UUID"
-paramType :: String
-paramType = "TYPE"
-paramFormat :: String
-paramFormat = "FORMAT"
-paramKeyValue :: String
-paramKeyValue = "K=V"
-paramNothing :: String
-paramNothing = ""
-paramRepeating :: String -> String
-paramRepeating s = s ++ " ..."
-paramOptional :: String -> String
-paramOptional s = "[" ++ s ++ "]"
-paramPair :: String -> String -> String
-paramPair a b = a ++ " " ++ b
+formatOption :: Option
+formatOption = Option [] ["format"] (ReqArg setFormat paramFormat)
+		"control format of output"
+
+setFormat :: String -> Annex ()
+setFormat v = Annex.changeState $ \s ->
+	s { Annex.format = Just $ Utility.Format.gen v }
diff --git a/Types/Command.hs b/Types/Command.hs
index 3cabf7318a..b173b61c9d 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -8,6 +8,7 @@
 module Types.Command where
 
 import Types
+import Types.Option
 
 {- A command runs in these stages.
  -
@@ -32,14 +33,15 @@ type CommandPerform = Annex (Maybe CommandCleanup)
 type CommandCleanup = Annex Bool
 
 {- A command is defined by specifying these things. -}
-data Command = Command {
-	cmdnorepo :: Maybe (IO ()), -- an action to run when not in a repo
-	cmdcheck :: [CommandCheck], -- check stage
-	cmdname :: String,
-	cmdparamdesc :: String,     -- description of params for usage
-	cmdseek :: [CommandSeek],   -- seek stage
-	cmddesc :: String           -- description of command for usage
-}
+data Command = Command
+	{ cmdoptions :: [Option]     -- command-specific options
+	, cmdnorepo :: Maybe (IO ()) -- an action to run when not in a repo
+	, cmdcheck :: [CommandCheck] -- check stage
+	, cmdname :: String
+	, cmdparamdesc :: String     -- description of params for usage
+	, cmdseek :: [CommandSeek]   -- seek stage
+	, cmddesc :: String          -- description of command for usage
+	}
 
 {- CommandCheck functions can be compared using their unique id. -}
 instance Eq CommandCheck where
diff --git a/Types/Option.hs b/Types/Option.hs
new file mode 100644
index 0000000000..0362578388
--- /dev/null
+++ b/Types/Option.hs
@@ -0,0 +1,17 @@
+{- git-annex command options
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.Option where
+
+import System.Console.GetOpt
+
+import Annex
+
+{- Each dashed command-line option results in generation of an action
+ - in the Annex monad that performs the necessary setting.
+ -}
+type Option = OptDescr (Annex ())
diff --git a/Usage.hs b/Usage.hs
new file mode 100644
index 0000000000..428a53fded
--- /dev/null
+++ b/Usage.hs
@@ -0,0 +1,84 @@
+{- git-annex usage messages
+ -
+ - Copyright 2010-2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Usage where
+
+import System.Console.GetOpt
+
+import Types.Command
+import Types.Option
+
+{- Usage message with lists of commands and options. -}
+usage :: String -> [Command] -> [Option] -> String
+usage header cmds commonoptions = unlines $ 
+	[ header
+	, ""
+	, "Options:"
+	] ++ optlines ++
+	[ ""
+	, "Commands:"
+	, ""
+	] ++ cmdlines
+	where
+		-- To get consistent indentation of options, generate the
+		-- usage for all options at once. A command's options will
+		-- be displayed after the command.
+		alloptlines = filter (not . null) $
+			lines $ usageInfo "" $
+				concatMap cmdoptions cmds ++ commonoptions
+		(cmdlines, optlines) = go cmds alloptlines []
+		go [] os ls = (ls, os)
+		go (c:cs) os ls = go cs os' (ls++(l:o))
+			where
+				(o, os') = splitAt (length $ cmdoptions c) os
+				l = concat 
+					[ cmdname c
+					, namepad (cmdname c)
+					, cmdparamdesc c
+					, descpad (cmdparamdesc c)
+					, cmddesc c
+					]
+		pad n s = replicate (n - length s) ' '
+		namepad = pad $ longest cmdname + 1
+		descpad = pad $ longest cmdparamdesc + 2
+		longest f = foldl max 0 $ map (length . f) cmds
+
+{- Descriptions of params used in usage messages. -}
+paramPaths :: String
+paramPaths = paramOptional $ paramRepeating paramPath -- most often used
+paramPath :: String
+paramPath = "PATH"
+paramKey :: String
+paramKey = "KEY"
+paramDesc :: String
+paramDesc = "DESC"
+paramUrl :: String
+paramUrl = "URL"
+paramNumber :: String
+paramNumber = "NUMBER"
+paramRemote :: String
+paramRemote = "REMOTE"
+paramGlob :: String
+paramGlob = "GLOB"
+paramName :: String
+paramName = "NAME"
+paramUUID :: String
+paramUUID = "UUID"
+paramType :: String
+paramType = "TYPE"
+paramFormat :: String
+paramFormat = "FORMAT"
+paramKeyValue :: String
+paramKeyValue = "K=V"
+paramNothing :: String
+paramNothing = ""
+paramRepeating :: String -> String
+paramRepeating s = s ++ " ..."
+paramOptional :: String -> String
+paramOptional s = "[" ++ s ++ "]"
+paramPair :: String -> String -> String
+paramPair a b = a ++ " " ++ b

From c371c40a889c73b79f7f8918b2918e2fbb75f212 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 23:10:19 -0400
Subject: [PATCH 2924/8313] Don't list S3 as a remote type when built without
 S3 support.

---
 Remote.hs        | 14 +++++++-------
 Remote/S3real.hs |  6 +++---
 Remote/S3stub.hs | 10 ++--------
 debian/changelog |  1 +
 4 files changed, 13 insertions(+), 18 deletions(-)

diff --git a/Remote.hs b/Remote.hs
index 8046175d27..8dd80d4191 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -55,14 +55,14 @@ import qualified Remote.Web
 import qualified Remote.Hook
 
 remoteTypes :: [RemoteType]
-remoteTypes =
-	[ Remote.Git.remote
+remoteTypes = catMaybes
+	[ Just Remote.Git.remote
 	, Remote.S3.remote
-	, Remote.Bup.remote
-	, Remote.Directory.remote
-	, Remote.Rsync.remote
-	, Remote.Web.remote
-	, Remote.Hook.remote
+	, Just Remote.Bup.remote
+	, Just Remote.Directory.remote
+	, Just Remote.Rsync.remote
+	, Just Remote.Web.remote
+	, Just Remote.Hook.remote
 	]
 
 {- Builds a list of all available Remotes.
diff --git a/Remote/S3real.hs b/Remote/S3real.hs
index bef89b5539..96a831e340 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3real.hs
@@ -28,8 +28,8 @@ import Crypto
 import Annex.Content
 import Utility.Base64
 
-remote :: RemoteType
-remote = RemoteType {
+remote :: Maybe RemoteType
+remote = Just $ RemoteType {
 	typename = "S3",
 	enumerate = findSpecialRemotes "s3",
 	generate = gen,
@@ -58,7 +58,7 @@ gen' r u c cst =
 			hasKeyCheap = False,
 			config = c,
 			repo = r,
-			remotetype = remote
+			remotetype = fromJust remote
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
index 31e8a339ef..5bd2b1c795 100644
--- a/Remote/S3stub.hs
+++ b/Remote/S3stub.hs
@@ -1,13 +1,7 @@
 -- stub for when hS3 is not available
 module Remote.S3 (remote) where
 
-import Types.Remote
 import Types
 
-remote :: RemoteType
-remote = RemoteType {
-	typename = "S3",
-	enumerate = return [],
-	generate = error "S3 not enabled",
-	setup = error "S3 not enabled"
-}
+remote :: Maybe RemoteType
+remote = Nothing
diff --git a/debian/changelog b/debian/changelog
index e5687aac14..ac6e9b80c1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20120106) UNRELEASED; urgency=low
 
   * Support unescaped repository urls, like git does.
+  * Don't list S3 as a remote type when built without S3 support.
 
  -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400
 

From f534fcc7b1a01f30e4ee41a0a364fe2cbf25d5a8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 23:14:10 -0400
Subject: [PATCH 2925/8313] remove S3stub stuff

Let's keep that in a no-s3 branch, which can be merged into eg,
debian-stable.
---
 Makefile                    | 15 ++-------------
 Remote.hs                   | 14 +++++++-------
 Remote/{S3real.hs => S3.hs} |  6 +++---
 Remote/S3stub.hs            |  7 -------
 debian/changelog            |  1 -
 doc/install.mdwn            |  2 +-
 6 files changed, 13 insertions(+), 32 deletions(-)
 rename Remote/{S3real.hs => S3.hs} (99%)
 delete mode 100644 Remote/S3stub.hs

diff --git a/Makefile b/Makefile
index 6805f93d2a..93d4567ec5 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex git-annex-shell git-union-merge
 mans=git-annex.1 git-annex-shell.1 git-union-merge.1
-sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs Remote/S3.hs
+sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs
 
 all=$(bins) $(mans) docs
 
@@ -33,18 +33,7 @@ Build/SysConfig.hs: configure.hs Build/TestConfig.hs
 	hsc2hs $<
 	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
-Remote/S3.hs:
-	@ln -sf S3real.hs Remote/S3.hs
-
-Remote/S3.o: Remote/S3.hs
-	@if ! $(GHCMAKE) Remote/S3.hs; then \
-		ln -sf S3stub.hs Remote/S3.hs; \
-		echo "** building without S3 support"; \
-	fi
-
-sources: $(sources)
-
-$(bins): sources Remote/S3.o
+$(bins): $(sources)
 	$(GHCMAKE) $@
 
 git-annex.1: doc/git-annex.mdwn
diff --git a/Remote.hs b/Remote.hs
index 8dd80d4191..8046175d27 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -55,14 +55,14 @@ import qualified Remote.Web
 import qualified Remote.Hook
 
 remoteTypes :: [RemoteType]
-remoteTypes = catMaybes
-	[ Just Remote.Git.remote
+remoteTypes =
+	[ Remote.Git.remote
 	, Remote.S3.remote
-	, Just Remote.Bup.remote
-	, Just Remote.Directory.remote
-	, Just Remote.Rsync.remote
-	, Just Remote.Web.remote
-	, Just Remote.Hook.remote
+	, Remote.Bup.remote
+	, Remote.Directory.remote
+	, Remote.Rsync.remote
+	, Remote.Web.remote
+	, Remote.Hook.remote
 	]
 
 {- Builds a list of all available Remotes.
diff --git a/Remote/S3real.hs b/Remote/S3.hs
similarity index 99%
rename from Remote/S3real.hs
rename to Remote/S3.hs
index 96a831e340..bef89b5539 100644
--- a/Remote/S3real.hs
+++ b/Remote/S3.hs
@@ -28,8 +28,8 @@ import Crypto
 import Annex.Content
 import Utility.Base64
 
-remote :: Maybe RemoteType
-remote = Just $ RemoteType {
+remote :: RemoteType
+remote = RemoteType {
 	typename = "S3",
 	enumerate = findSpecialRemotes "s3",
 	generate = gen,
@@ -58,7 +58,7 @@ gen' r u c cst =
 			hasKeyCheap = False,
 			config = c,
 			repo = r,
-			remotetype = fromJust remote
+			remotetype = remote
 		}
 
 s3Setup :: UUID -> RemoteConfig -> Annex RemoteConfig
diff --git a/Remote/S3stub.hs b/Remote/S3stub.hs
deleted file mode 100644
index 5bd2b1c795..0000000000
--- a/Remote/S3stub.hs
+++ /dev/null
@@ -1,7 +0,0 @@
--- stub for when hS3 is not available
-module Remote.S3 (remote) where
-
-import Types
-
-remote :: Maybe RemoteType
-remote = Nothing
diff --git a/debian/changelog b/debian/changelog
index ac6e9b80c1..e5687aac14 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,6 @@
 git-annex (3.20120106) UNRELEASED; urgency=low
 
   * Support unescaped repository urls, like git does.
-  * Don't list S3 as a remote type when built without S3 support.
 
  -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 3e89c67758..26af5a091d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -30,7 +30,7 @@ To build and use git-annex, you will need:
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
-  * [hS3](http://hackage.haskell.org/package/hS3) (optional, but recommended)
+  * [hS3](http://hackage.haskell.org/package/hS3)
   * [json](http://hackage.haskell.org/package/json)
 * Shell commands
   * [git](http://git-scm.com/)

From d926e29b9f14568d9c248fa94ce4dc0042de7ed4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 23:21:03 -0400
Subject: [PATCH 2926/8313] update ignores

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 7d2504de6f..250d0381c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,5 +15,4 @@ html
 .hpc
 Utility/Touch.hs
 Utility/StatFS.hs
-Remote/S3.hs
 dist

From 3b450cb6bc62acfb004f47cb8052ba219fff02f6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
 
Date: Fri, 6 Jan 2012 03:38:33 +0000
Subject: [PATCH 2927/8313] Added a comment: Thank you!

---
 .../comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment

diff --git a/doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment b/doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment
new file mode 100644
index 0000000000..69063966c6
--- /dev/null
+++ b/doc/bugs/Lost_S3_Remote/comment_3_1e434d5a20a692cd9dc7f6f8f20f30dd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkey8WuXUh_x5JC2c9_it1CYRnVTgdGu1M"
+ nickname="Dustin"
+ subject="Thank you!"
+ date="2012-01-06T03:38:27Z"
+ content="""
+make clean and rebuild worked...  Thank you
+"""]]

From 2051d804625be0c19e277764ab9820b428a5d2b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Jan 2012 23:42:06 -0400
Subject: [PATCH 2928/8313] close

---
 doc/bugs/Lost_S3_Remote.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/Lost_S3_Remote.mdwn b/doc/bugs/Lost_S3_Remote.mdwn
index 7f8b6b5305..c359b52103 100644
--- a/doc/bugs/Lost_S3_Remote.mdwn
+++ b/doc/bugs/Lost_S3_Remote.mdwn
@@ -54,3 +54,6 @@ I appreciate any help....  I've tested versions 3.20111211, 3.20111231, and 3.20
     git version 1.7.8.1
 
 
+
+> [[done]]; I've fixed the build system so this confusing thing cannot
+> happen anymore. --[[Joey]]

From 0a36f92a31196451c2d838fd0ae15527e8bbce18 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 03:06:25 -0400
Subject: [PATCH 2929/8313] more command-specific options

Made --from and --to command-specific options.

Added generic storage for values of command-specific options,
which allows removing some of the special case fields in AnnexState.

(Also added generic storage for command-specific flags, although there are
not yet any.)

Note that this storage uses a Map, so repeatedly looking up the same value
is slightly more expensive than looking up an AnnexState field. But, the
value can be looked up once in the seek stage, transformed as necessary,
and passed in a closure to the start stage, and this avoids that overhead.

Still, I'm hesitant to use this for things like force or fast flags.
It's probably best to reserve it for flags that are only used by a few
commands, or options like --from and --to that it's important only be
allowed to be used with commands that implement them, to avoid user
confusion.
---
 Annex.hs              | 33 ++++++++++++++++++++++++++-------
 Checks.hs             | 13 +------------
 Command/Copy.hs       | 12 ++++++------
 Command/Drop.hs       | 15 +++++++++------
 Command/DropUnused.hs |  7 ++++---
 Command/Find.hs       | 21 +++++++++++++--------
 Command/Get.hs        | 11 +++++------
 Command/Move.hs       | 23 +++++++++++++++--------
 Command/Unused.hs     |  9 ++++++---
 GitAnnex.hs           |  8 +-------
 Options.hs            | 19 ++++++++++---------
 Seek.hs               |  9 +++++++++
 Usage.hs              |  2 ++
 13 files changed, 107 insertions(+), 75 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 91d374aec3..f1e46126a9 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -17,6 +17,10 @@ module Annex (
 	eval,
 	getState,
 	changeState,
+	setFlag,
+	setField,
+	getFlag,
+	getField,
 	gitRepo,
 	inRepo,
 	fromRepo,
@@ -38,7 +42,6 @@ import Types.BranchState
 import Types.TrustLevel
 import Types.UUID
 import qualified Utility.Matcher
-import qualified Utility.Format
 import qualified Data.Map as M
 
 -- git-annex's monad
@@ -76,17 +79,16 @@ data AnnexState = AnnexState
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
-	, format :: Maybe Utility.Format.Format
 	, branchstate :: BranchState
 	, catfilehandle :: Maybe CatFileHandle
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
-	, toremote :: Maybe String
-	, fromremote :: Maybe String
 	, limit :: Matcher (FilePath -> Annex Bool)
 	, forcetrust :: [(UUID, TrustLevel)]
 	, trustmap :: Maybe TrustMap
 	, ciphers :: M.Map EncryptedCipher Cipher
+	, flags :: M.Map String Bool
+	, fields :: M.Map String String
 	}
 
 newState :: Git.Repo -> AnnexState
@@ -99,17 +101,16 @@ newState gitrepo = AnnexState
 	, force = False
 	, fast = False
 	, auto = False
-	, format = Nothing
 	, branchstate = startBranchState
 	, catfilehandle = Nothing
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
-	, toremote = Nothing
-	, fromremote = Nothing
 	, limit = Left []
 	, forcetrust = []
 	, trustmap = Nothing
 	, ciphers = M.empty
+	, flags = M.empty
+	, fields = M.empty
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
@@ -134,6 +135,24 @@ getState = gets
 changeState :: (AnnexState -> AnnexState) -> Annex ()
 changeState = modify
 
+{- Sets a flag to True -}
+setFlag :: String -> Annex ()
+setFlag flag = changeState $ \s ->
+	s { flags = M.insert flag True $ flags s }
+
+{- Sets a field to a value -}
+setField :: String -> String -> Annex ()
+setField field value = changeState $ \s ->
+	s { fields = M.insert field value $ fields s }
+
+{- Checks if a flag was set. -}
+getFlag :: String -> Annex Bool
+getFlag flag = fromMaybe False . M.lookup flag <$> getState flags
+
+{- Gets the value of a field. -}
+getField :: String -> Annex (Maybe String)
+getField field = M.lookup field <$> getState fields
+
 {- Returns the annex's git repository. -}
 gitRepo :: Annex Git.Repo
 gitRepo = getState repo
diff --git a/Checks.hs b/Checks.hs
index e443811cdc..9d846842db 100644
--- a/Checks.hs
+++ b/Checks.hs
@@ -13,24 +13,13 @@ module Checks where
 import Common.Annex
 import Types.Command
 import Init
-import qualified Annex
 
 commonChecks :: [CommandCheck]
-commonChecks = [fromOpt, toOpt, repoExists]
+commonChecks = [repoExists]
 
 repoExists :: CommandCheck
 repoExists = CommandCheck 0 ensureInitialized
 
-fromOpt :: CommandCheck
-fromOpt = CommandCheck 1 $ do
-	v <- Annex.getState Annex.fromremote
-	unless (isNothing v) $ error "cannot use --from with this command"
-
-toOpt :: CommandCheck
-toOpt = CommandCheck 2 $ do
-	v <- Annex.getState Annex.toremote
-	unless (isNothing v) $ error "cannot use --to with this command"
-
 dontCheck :: CommandCheck -> Command -> Command
 dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
 
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 77beb4b4f4..d789d41f6b 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -12,15 +12,15 @@ import Command
 import qualified Command.Move
 
 def :: [Command]
-def = [dontCheck toOpt $ dontCheck fromOpt $ 
-	command "copy" paramPaths seek
+def = [withOptions Command.Move.options $ command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies $ \n -> whenAnnexed $ start n]
+seek = [withField "to" id $ \to -> withField "from" id $ \from ->
+		withNumCopies $ \n -> whenAnnexed $ start to from n]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start numcopies file (key, backend) = autoCopies key (<) numcopies $
-	Command.Move.start False file (key, backend)
+start :: Maybe String -> Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start to from numcopies file (key, backend) = autoCopies key (<) numcopies $
+	Command.Move.start to from False file (key, backend)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 89e7c8e42a..f76951f08c 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -18,15 +18,18 @@ import Annex.Content
 import Config
 
 def :: [Command]
-def = [dontCheck fromOpt $ command "drop" paramPaths seek
+def = [withOptions [fromOption] $ command "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
-seek :: [CommandSeek]
-seek = [withNumCopies $ \n -> whenAnnexed $ start n]
+fromOption :: Option
+fromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote"
 
-start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start numcopies file (key, _) = autoCopies key (>) numcopies $ do
-	from <- Annex.getState Annex.fromremote
+seek :: [CommandSeek]
+seek = [withField "from" id $ \from -> withNumCopies $ \n ->
+	whenAnnexed $ start from n]
+
+start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start from numcopies file (key, _) = autoCopies key (>) numcopies $ do
 	case from of
 		Nothing -> startLocal file numcopies key
 		Just name -> do
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 244f378d97..fd3e84fe5c 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -20,8 +20,9 @@ import Types.Key
 type UnusedMap = M.Map String Key
 
 def :: [Command]
-def = [dontCheck fromOpt $ command "dropunused" (paramRepeating paramNumber)
-	seek "drop unused file content"]
+def = [withOptions [Command.Drop.fromOption] $
+	command "dropunused" (paramRepeating paramNumber)
+		seek "drop unused file content"]
 
 seek :: [CommandSeek]
 seek = [withUnusedMaps]
@@ -50,7 +51,7 @@ start (unused, unusedbad, unusedtmp) s = search
 					next $ a key
 
 perform :: Key -> CommandPerform
-perform key = maybe droplocal dropremote =<< Annex.getState Annex.fromremote
+perform key = maybe droplocal dropremote =<< Annex.getField "from"
 	where
 		dropremote name = do
 			r <- Remote.byName name
diff --git a/Command/Find.hs b/Command/Find.hs
index c86db5fa65..eb0267c142 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -23,20 +23,25 @@ def = [withOptions [formatOption, print0Option] $
 	command "find" paramPaths seek "lists available files"]
 
 print0Option :: Option
-print0Option = Option [] ["print0"] (NoArg $ setFormat "${file}\0")
+print0Option = Option [] ["print0"] (NoArg $ Annex.setField "format" "${file}\0")
 	"terminate output with null"
 
-seek :: [CommandSeek]
-seek = [withFilesInGit $ whenAnnexed start]
+formatOption :: Option
+formatOption = fieldOption [] "format" paramFormat "control format of output"
 
-start :: FilePath -> (Key, Backend) -> CommandStart
-start file (key, _) = do
+seek :: [CommandSeek]
+seek = [withField "format" formatconverter $ \f ->
+		withFilesInGit $ whenAnnexed $ start f]
+	where
+		formatconverter = maybe Nothing (Just . Utility.Format.gen)
+
+start :: Maybe Utility.Format.Format -> FilePath -> (Key, Backend) -> CommandStart
+start format file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
 	whenM (liftM2 (||) limited (inAnnex key)) $
-		unlessM (showFullJSON vars) $ do
-			f <- Annex.getState Annex.format
-			case f of
+		unlessM (showFullJSON vars) $
+			case format of
 				Nothing -> liftIO $ putStrLn file
 				Just formatter -> liftIO $ putStr $
 					Utility.Format.format formatter $
diff --git a/Command/Get.hs b/Command/Get.hs
index f2b70baebd..4a50fe3fea 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -9,22 +9,21 @@ module Command.Get where
 
 import Common.Annex
 import Command
-import qualified Annex
 import qualified Remote
 import Annex.Content
 import qualified Command.Move
 
 def :: [Command]
-def = [dontCheck fromOpt $ command "get" paramPaths seek
+def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withNumCopies $ \n -> whenAnnexed $ start n]
+seek = [withField "from" id $ \from -> withNumCopies $ \n ->
+	whenAnnexed $ start from n]
 
-start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
+start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start from numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
 	autoCopies key (<) numcopies $ do
-		from <- Annex.getState Annex.fromremote
 		case from of
 			Nothing -> go $ perform key
 			Just name -> do
diff --git a/Command/Move.hs b/Command/Move.hs
index bd1490b0cd..66a0c16602 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -16,18 +16,25 @@ import qualified Remote
 import Annex.UUID
 
 def :: [Command]
-def = [dontCheck toOpt $ dontCheck fromOpt $
-	command "move" paramPaths seek
+def = [withOptions options $ command "move" paramPaths seek
 	"move content of files to/from another repository"]
 
-seek :: [CommandSeek]
-seek = [withFilesInGit $ whenAnnexed $ start True]
+fromOption :: Option
+fromOption = fieldOption ['f'] "from" paramRemote "source remote"
 
-start :: Bool -> FilePath -> (Key, Backend) -> CommandStart
-start move file (key, _) = do
+toOption :: Option
+toOption = fieldOption ['t'] "to" paramRemote "destination remote"
+
+options :: [Option]
+options = [fromOption, toOption]
+
+seek :: [CommandSeek]
+seek = [withField "to" id $ \to -> withField "from" id $ \from ->
+		withFilesInGit $ whenAnnexed $ start to from True]
+
+start :: Maybe String -> Maybe String -> Bool -> FilePath -> (Key, Backend) -> CommandStart
+start to from move file (key, _) = do
 	noAuto
-	to <- Annex.getState Annex.toremote
-	from <- Annex.getState Annex.fromremote
 	case (from, to) of
 		(Nothing, Nothing) -> error "specify either --from or --to"
 		(Nothing, Just name) -> do
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 8d45c51cbf..59efe64c80 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -30,16 +30,19 @@ import qualified Annex.Branch
 import Annex.CatFile
 
 def :: [Command]
-def = [dontCheck fromOpt $ command "unused" paramNothing seek
+def = [withOptions [fromOption] $ command "unused" paramNothing seek
 	"look for unused file content"]
 
+fromOption :: Option
+fromOption = fieldOption ['f'] "from" paramRemote "remote to check for unused content"
+
 seek :: [CommandSeek]
-seek = [withNothing start]
+seek = [withNothing $ start]
 
 {- Finds unused content in the annex. -} 
 start :: CommandStart
 start = do
-	from <- Annex.getState Annex.fromremote
+	from <- Annex.getField "from"
 	let (name, action) = case from of
 		Nothing -> (".", checkUnused)
 		Just "." -> (".", checkUnused)
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 3ce4518108..8af1d5d59e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -94,11 +94,7 @@ cmds = concat
 
 options :: [Option]
 options = commonOptions ++
-	[ Option ['t'] ["to"] (ReqArg setto paramRemote)
-		"specify to where to transfer content"
-	, Option ['f'] ["from"] (ReqArg setfrom paramRemote)
-		"specify from where to transfer content"
-	, Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
+	[ Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
 	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
 		"override trust setting"
@@ -120,8 +116,6 @@ options = commonOptions ++
 		"skip files not using a key-value backend"
 	] ++ matcherOptions
 	where
-		setto v = Annex.changeState $ \s -> s { Annex.toremote = Just v }
-		setfrom v = Annex.changeState $ \s -> s { Annex.fromremote = Just v }
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
diff --git a/Options.hs b/Options.hs
index fa008d0644..56f0bc0ee3 100644
--- a/Options.hs
+++ b/Options.hs
@@ -8,8 +8,8 @@
 module Options (
 	commonOptions,
 	matcherOptions,
-	formatOption,
-	setFormat,
+	flagOption,
+	fieldOption,
 	ArgDescr(..),
 	Option,
 	OptDescr(..),
@@ -23,7 +23,6 @@ import qualified Annex
 import Limit
 import Types.Option
 import Usage
-import qualified Utility.Format
 
 commonOptions :: [Option]
 commonOptions =
@@ -65,10 +64,12 @@ matcherOptions =
 		longopt o = Option [] [o] $ NoArg $ addToken o
 		shortopt o = Option o [] $ NoArg $ addToken o
 
-formatOption :: Option
-formatOption = Option [] ["format"] (ReqArg setFormat paramFormat)
-		"control format of output"
+{- An option that sets a flag. -}
+flagOption :: String -> String -> String -> Option
+flagOption short flag description = 
+	Option short [flag] (NoArg (Annex.setFlag flag)) description
 
-setFormat :: String -> Annex ()
-setFormat v = Annex.changeState $ \s ->
-	s { Annex.format = Just $ Utility.Format.gen v }
+{- An option that sets a field. -}
+fieldOption :: String -> String -> String -> String -> Option
+fieldOption short field paramdesc description = 
+	Option short [field] (ReqArg (Annex.setField field) paramdesc) description
diff --git a/Seek.hs b/Seek.hs
index 80f31dd969..af074c7c5b 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -87,6 +87,15 @@ withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
 
+{- Modifies a seek action using the value of a field, which is fed into
+ - a conversion function, and then is passed into the seek action.
+ - This ensures that the conversion function only runs once.
+ -}
+withField :: String -> (Maybe String -> a) -> (a -> CommandSeek) -> CommandSeek
+withField field converter a ps = do
+        f <- converter <$> Annex.getField field
+        a f ps
+
 withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
diff --git a/Usage.hs b/Usage.hs
index 428a53fded..308ade798f 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -66,6 +66,8 @@ paramGlob :: String
 paramGlob = "GLOB"
 paramName :: String
 paramName = "NAME"
+paramValue :: String
+paramValue = "VALUE"
 paramUUID :: String
 paramUUID = "UUID"
 paramType :: String

From e0d6010d3602bfa6a231642816d788d5bc1b6988 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 03:31:20 -0400
Subject: [PATCH 2930/8313] simplify

---
 Command/Copy.hs |  2 +-
 Command/Drop.hs |  2 +-
 Command/Find.hs |  4 ++--
 Command/Get.hs  |  2 +-
 Command/Move.hs |  2 +-
 Seek.hs         | 11 ++++-------
 6 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/Command/Copy.hs b/Command/Copy.hs
index d789d41f6b..a9491f56f4 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -16,7 +16,7 @@ def = [withOptions Command.Move.options $ command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withField "to" id $ \to -> withField "from" id $ \from ->
+seek = [withField "to" $ \to -> withField "from" $ \from ->
 		withNumCopies $ \n -> whenAnnexed $ start to from n]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Drop.hs b/Command/Drop.hs
index f76951f08c..3ad4077036 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -25,7 +25,7 @@ fromOption :: Option
 fromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote"
 
 seek :: [CommandSeek]
-seek = [withField "from" id $ \from -> withNumCopies $ \n ->
+seek = [withField "from" $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
 start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Command/Find.hs b/Command/Find.hs
index eb0267c142..bc7f200c2e 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -30,8 +30,8 @@ formatOption :: Option
 formatOption = fieldOption [] "format" paramFormat "control format of output"
 
 seek :: [CommandSeek]
-seek = [withField "format" formatconverter $ \f ->
-		withFilesInGit $ whenAnnexed $ start f]
+seek = [withField "format" $ \f ->
+		withFilesInGit $ whenAnnexed $ start $ formatconverter f]
 	where
 		formatconverter = maybe Nothing (Just . Utility.Format.gen)
 
diff --git a/Command/Get.hs b/Command/Get.hs
index 4a50fe3fea..c3a6630cee 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -18,7 +18,7 @@ def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withField "from" id $ \from -> withNumCopies $ \n ->
+seek = [withField "from" $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
 start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Command/Move.hs b/Command/Move.hs
index 66a0c16602..b6a72bf6bc 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -29,7 +29,7 @@ options :: [Option]
 options = [fromOption, toOption]
 
 seek :: [CommandSeek]
-seek = [withField "to" id $ \to -> withField "from" id $ \from ->
+seek = [withField "to" $ \to -> withField "from" $ \from ->
 		withFilesInGit $ whenAnnexed $ start to from True]
 
 start :: Maybe String -> Maybe String -> Bool -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Seek.hs b/Seek.hs
index af074c7c5b..2c121bd96c 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -87,13 +87,10 @@ withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
 
-{- Modifies a seek action using the value of a field, which is fed into
- - a conversion function, and then is passed into the seek action.
- - This ensures that the conversion function only runs once.
- -}
-withField :: String -> (Maybe String -> a) -> (a -> CommandSeek) -> CommandSeek
-withField field converter a ps = do
-        f <- converter <$> Annex.getField field
+{- Feeds the value of a field to a seek action. -}
+withField :: String -> (Maybe String -> CommandSeek) -> CommandSeek
+withField field a ps = do
+        f <- Annex.getField field
         a f ps
 
 withNothing :: CommandStart -> CommandSeek

From df21cbfdd2b7342c206ebd4aea32d989328374dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 04:02:35 -0400
Subject: [PATCH 2931/8313] look up --to and --from remote names only once

This will speed up commands like move and drop.
---
 Command/Copy.hs       |  8 +++++---
 Command/Drop.hs       |  7 +++----
 Command/DropUnused.hs |  5 ++---
 Command/Find.hs       |  2 +-
 Command/Get.hs        |  7 +++----
 Command/Move.hs       | 15 ++++++---------
 Command/Sync.hs       |  2 +-
 Command/Unused.hs     |  2 +-
 Remote.hs             | 11 ++++++-----
 Seek.hs               |  4 ++--
 10 files changed, 30 insertions(+), 33 deletions(-)

diff --git a/Command/Copy.hs b/Command/Copy.hs
index d789d41f6b..c83c724127 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -10,17 +10,19 @@ module Command.Copy where
 import Common.Annex
 import Command
 import qualified Command.Move
+import qualified Remote
 
 def :: [Command]
 def = [withOptions Command.Move.options $ command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withField "to" id $ \to -> withField "from" id $ \from ->
-		withNumCopies $ \n -> whenAnnexed $ start to from n]
+seek = [withField "to" Remote.byName $ \to ->
+		withField "from" Remote.byName $ \from ->
+			withNumCopies $ \n -> whenAnnexed $ start to from n]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: Maybe String -> Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start :: Maybe Remote -> Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start to from numcopies file (key, backend) = autoCopies key (<) numcopies $
 	Command.Move.start to from False file (key, backend)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index f76951f08c..07ea50df16 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -25,15 +25,14 @@ fromOption :: Option
 fromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote"
 
 seek :: [CommandSeek]
-seek = [withField "from" id $ \from -> withNumCopies $ \n ->
+seek = [withField "from" Remote.byName $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
-start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start from numcopies file (key, _) = autoCopies key (>) numcopies $ do
 	case from of
 		Nothing -> startLocal file numcopies key
-		Just name -> do
-			remote <- Remote.byName name
+		Just remote -> do
 			u <- getUUID
 			if Remote.uuid remote == u
 				then startLocal file numcopies key
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index fd3e84fe5c..1c5bf8b8c0 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -51,10 +51,9 @@ start (unused, unusedbad, unusedtmp) s = search
 					next $ a key
 
 perform :: Key -> CommandPerform
-perform key = maybe droplocal dropremote =<< Annex.getField "from"
+perform key = maybe droplocal dropremote =<< Remote.byName =<< Annex.getField "from"
 	where
-		dropremote name = do
-			r <- Remote.byName name
+		dropremote r = do
 			showAction $ "from " ++ Remote.name r
 			ok <- Remote.removeKey r key
 			next $ Command.Drop.cleanupRemote key r ok
diff --git a/Command/Find.hs b/Command/Find.hs
index eb0267c142..8760cc9475 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -33,7 +33,7 @@ seek :: [CommandSeek]
 seek = [withField "format" formatconverter $ \f ->
 		withFilesInGit $ whenAnnexed $ start f]
 	where
-		formatconverter = maybe Nothing (Just . Utility.Format.gen)
+		formatconverter = return . maybe Nothing (Just . Utility.Format.gen)
 
 start :: Maybe Utility.Format.Format -> FilePath -> (Key, Backend) -> CommandStart
 start format file (key, _) = do
diff --git a/Command/Get.hs b/Command/Get.hs
index 4a50fe3fea..1a0435c36c 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -18,17 +18,16 @@ def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withField "from" id $ \from -> withNumCopies $ \n ->
+seek = [withField "from" Remote.byName $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
-start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start from numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
 	autoCopies key (<) numcopies $ do
 		case from of
 			Nothing -> go $ perform key
-			Just name -> do
+			Just src -> do
 				-- get --from = copy --from
-				src <- Remote.byName name
 				stopUnless (Command.Move.fromOk src key) $
 					go $ Command.Move.fromPerform src False key
 	where
diff --git a/Command/Move.hs b/Command/Move.hs
index 66a0c16602..4978283bf1 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -29,20 +29,17 @@ options :: [Option]
 options = [fromOption, toOption]
 
 seek :: [CommandSeek]
-seek = [withField "to" id $ \to -> withField "from" id $ \from ->
-		withFilesInGit $ whenAnnexed $ start to from True]
+seek = [withField "to" Remote.byName $ \to ->
+		withField "from" Remote.byName $ \from ->
+			withFilesInGit $ whenAnnexed $ start to from True]
 
-start :: Maybe String -> Maybe String -> Bool -> FilePath -> (Key, Backend) -> CommandStart
+start :: Maybe Remote -> Maybe Remote -> Bool -> FilePath -> (Key, Backend) -> CommandStart
 start to from move file (key, _) = do
 	noAuto
 	case (from, to) of
 		(Nothing, Nothing) -> error "specify either --from or --to"
-		(Nothing, Just name) -> do
-			dest <- Remote.byName name
-			toStart dest move file key
-		(Just name, Nothing) -> do
-			src <- Remote.byName name
-			fromStart src move file key
+		(Nothing, Just dest) -> toStart dest move file key
+		(Just src, Nothing) -> fromStart src move file key
 		(_ ,  _) -> error "only one of --from or --to can be specified"
 	where
 		noAuto = when move $ whenM (Annex.getState Annex.auto) $ error
diff --git a/Command/Sync.hs b/Command/Sync.hs
index e5884cc4a7..3d541c4dea 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -61,7 +61,7 @@ syncRemotes rs = do
 		wanted
 			| null rs = good =<< available
 			| otherwise = listed
-		listed = mapM Remote.byName rs
+		listed = catMaybes <$> mapM (Remote.byName . Just) rs
 		available = filter nonspecial <$> Remote.enabledRemoteList
 		good = filterM $ Remote.Git.repoAvail . Types.Remote.repo
 		nonspecial r = Types.Remote.remotetype r == Remote.Git.remote
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 59efe64c80..a6883dce1a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -66,7 +66,7 @@ checkUnused = do
 
 checkRemoteUnused :: String -> CommandPerform
 checkRemoteUnused name = do
-	checkRemoteUnused' =<< Remote.byName name
+	checkRemoteUnused' =<< fromJust <$> Remote.byName (Just name)
 	next $ return True
 
 checkRemoteUnused' :: Remote -> Annex ()
diff --git a/Remote.hs b/Remote.hs
index 8046175d27..3f60ca3acf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -94,14 +94,15 @@ enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
 remoteMap :: Annex (M.Map UUID String)
 remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> remoteList
 
-{- Looks up a remote by name. (Or by UUID.) Only finds currently configured
- - git remotes. -}
-byName :: String -> Annex (Remote)
-byName n = do
+{- When a name is specified, looks up the remote matching that name.
+ - (Or it can be a UUID.) Only finds currently configured git remotes. -}
+byName :: Maybe String -> Annex (Maybe Remote)
+byName Nothing = return Nothing
+byName (Just n) = do
 	res <- byName' n
 	case res of
 		Left e -> error e
-		Right r -> return r
+		Right r -> return $ Just r
 byName' :: String -> Annex (Either String Remote)
 byName' "" = return $ Left "no remote specified"
 byName' n = do
diff --git a/Seek.hs b/Seek.hs
index af074c7c5b..53101b23e4 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -91,9 +91,9 @@ withKeys a params = return $ map (a . parse) params
  - a conversion function, and then is passed into the seek action.
  - This ensures that the conversion function only runs once.
  -}
-withField :: String -> (Maybe String -> a) -> (a -> CommandSeek) -> CommandSeek
+withField :: String -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> CommandSeek
 withField field converter a ps = do
-        f <- converter <$> Annex.getField field
+        f <- converter =<< Annex.getField field
         a f ps
 
 withNothing :: CommandStart -> CommandSeek

From 1f8a1058c96bd4ee11fcb353f0ede1842d79ab6a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 10:14:37 -0400
Subject: [PATCH 2932/8313] tweak

---
 Command.hs              |  1 -
 Command/Copy.hs         |  4 ++--
 Command/Drop.hs         |  5 +++--
 Command/DropUnused.hs   |  4 +++-
 Command/Find.hs         | 15 +++++++++------
 Command/Get.hs          |  4 ++--
 Command/Move.hs         |  9 +++++----
 Command/Unused.hs       |  5 +++--
 GitAnnex.hs             |  5 +++--
 Options.hs => Option.hs | 40 ++++++++++++++++++++++------------------
 Seek.hs                 |  9 +++++----
 Types.hs                |  4 +++-
 Types/Command.hs        |  1 -
 git-annex-shell.hs      |  3 ++-
 14 files changed, 62 insertions(+), 47 deletions(-)
 rename Options.hs => Option.hs (77%)

diff --git a/Command.hs b/Command.hs
index b287629ae4..82d6429bfa 100644
--- a/Command.hs
+++ b/Command.hs
@@ -30,7 +30,6 @@ import Types.Command as ReExported
 import Types.Option as ReExported
 import Seek as ReExported
 import Checks as ReExported
-import Options as ReExported
 import Usage as ReExported
 import Logs.Trust
 import Logs.Location
diff --git a/Command/Copy.hs b/Command/Copy.hs
index c83c724127..32b83a5262 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -17,8 +17,8 @@ def = [withOptions Command.Move.options $ command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withField "to" Remote.byName $ \to ->
-		withField "from" Remote.byName $ \from ->
+seek = [withField Command.Move.toOption Remote.byName $ \to ->
+		withField Command.Move.fromOption Remote.byName $ \from ->
 			withNumCopies $ \n -> whenAnnexed $ start to from n]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 07ea50df16..578ab62b97 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -16,16 +16,17 @@ import Logs.Location
 import Logs.Trust
 import Annex.Content
 import Config
+import qualified Option
 
 def :: [Command]
 def = [withOptions [fromOption] $ command "drop" paramPaths seek
 	"indicate content of files not currently wanted"]
 
 fromOption :: Option
-fromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote"
+fromOption = Option.field ['f'] "from" paramRemote "drop content from a remote"
 
 seek :: [CommandSeek]
-seek = [withField "from" Remote.byName $ \from -> withNumCopies $ \n ->
+seek = [withField fromOption Remote.byName $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
 start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 1c5bf8b8c0..0b2a602161 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -15,6 +15,7 @@ import qualified Annex
 import qualified Command.Drop
 import qualified Remote
 import qualified Git
+import qualified Option
 import Types.Key
 
 type UnusedMap = M.Map String Key
@@ -51,13 +52,14 @@ start (unused, unusedbad, unusedtmp) s = search
 					next $ a key
 
 perform :: Key -> CommandPerform
-perform key = maybe droplocal dropremote =<< Remote.byName =<< Annex.getField "from"
+perform key = maybe droplocal dropremote =<< Remote.byName =<< from
 	where
 		dropremote r = do
 			showAction $ "from " ++ Remote.name r
 			ok <- Remote.removeKey r key
 			next $ Command.Drop.cleanupRemote key r ok
 		droplocal = Command.Drop.performLocal key (Just 0) -- force drop
+		from = Annex.getField $ Option.name Command.Drop.fromOption
 
 performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
diff --git a/Command/Find.hs b/Command/Find.hs
index 8760cc9475..902f50d2e3 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -17,20 +17,23 @@ import qualified Annex
 import qualified Utility.Format
 import Utility.DataUnits
 import Types.Key
+import qualified Option
 
 def :: [Command]
 def = [withOptions [formatOption, print0Option] $
 	command "find" paramPaths seek "lists available files"]
 
-print0Option :: Option
-print0Option = Option [] ["print0"] (NoArg $ Annex.setField "format" "${file}\0")
-	"terminate output with null"
-
 formatOption :: Option
-formatOption = fieldOption [] "format" paramFormat "control format of output"
+formatOption = Option.field [] "format" paramFormat "control format of output"
+
+print0Option :: Option
+print0Option = Option.Option [] ["print0"] (Option.NoArg set)
+	"terminate output with null"
+	where
+		set = Annex.setField (Option.name formatOption) "${file}\0"
 
 seek :: [CommandSeek]
-seek = [withField "format" formatconverter $ \f ->
+seek = [withField formatOption formatconverter $ \f ->
 		withFilesInGit $ whenAnnexed $ start f]
 	where
 		formatconverter = return . maybe Nothing (Just . Utility.Format.gen)
diff --git a/Command/Get.hs b/Command/Get.hs
index 1a0435c36c..5d032e13c4 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -18,8 +18,8 @@ def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withField "from" Remote.byName $ \from -> withNumCopies $ \n ->
-	whenAnnexed $ start from n]
+seek = [withField Command.Move.fromOption Remote.byName $ \from ->
+	withNumCopies $ \n -> whenAnnexed $ start from n]
 
 start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
 start from numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
diff --git a/Command/Move.hs b/Command/Move.hs
index 4978283bf1..2efaebbcb1 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -14,23 +14,24 @@ import qualified Annex
 import Annex.Content
 import qualified Remote
 import Annex.UUID
+import qualified Option
 
 def :: [Command]
 def = [withOptions options $ command "move" paramPaths seek
 	"move content of files to/from another repository"]
 
 fromOption :: Option
-fromOption = fieldOption ['f'] "from" paramRemote "source remote"
+fromOption = Option.field ['f'] "from" paramRemote "source remote"
 
 toOption :: Option
-toOption = fieldOption ['t'] "to" paramRemote "destination remote"
+toOption = Option.field ['t'] "to" paramRemote "destination remote"
 
 options :: [Option]
 options = [fromOption, toOption]
 
 seek :: [CommandSeek]
-seek = [withField "to" Remote.byName $ \to ->
-		withField "from" Remote.byName $ \from ->
+seek = [withField toOption Remote.byName $ \to ->
+		withField fromOption Remote.byName $ \from ->
 			withFilesInGit $ whenAnnexed $ start to from True]
 
 start :: Maybe Remote -> Maybe Remote -> Bool -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Command/Unused.hs b/Command/Unused.hs
index a6883dce1a..ffd4bef455 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -27,6 +27,7 @@ import qualified Git.LsTree as LsTree
 import qualified Backend
 import qualified Remote
 import qualified Annex.Branch
+import qualified Option
 import Annex.CatFile
 
 def :: [Command]
@@ -34,7 +35,7 @@ def = [withOptions [fromOption] $ command "unused" paramNothing seek
 	"look for unused file content"]
 
 fromOption :: Option
-fromOption = fieldOption ['f'] "from" paramRemote "remote to check for unused content"
+fromOption = Option.field ['f'] "from" paramRemote "remote to check for unused content"
 
 seek :: [CommandSeek]
 seek = [withNothing $ start]
@@ -42,7 +43,7 @@ seek = [withNothing $ start]
 {- Finds unused content in the annex. -} 
 start :: CommandStart
 start = do
-	from <- Annex.getField "from"
+	from <- Annex.getField $ Option.name fromOption
 	let (name, action) = case from of
 		Nothing -> (".", checkUnused)
 		Just "." -> (".", checkUnused)
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 8af1d5d59e..64020754fc 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -18,6 +18,7 @@ import Types.TrustLevel
 import qualified Annex
 import qualified Remote
 import qualified Limit
+import qualified Option
 
 import qualified Command.Add
 import qualified Command.Unannex
@@ -93,7 +94,7 @@ cmds = concat
 	]
 
 options :: [Option]
-options = commonOptions ++
+options = Option.common ++
 	[ Option ['N'] ["numcopies"] (ReqArg setnumcopies paramNumber)
 		"override default number of copies"
 	, Option [] ["trust"] (ReqArg (Remote.forceTrust Trusted) paramRemote)
@@ -114,7 +115,7 @@ options = commonOptions ++
 		"skip files with fewer copies"
 	, Option ['B'] ["inbackend"] (ReqArg Limit.addInBackend paramName)
 		"skip files not using a key-value backend"
-	] ++ matcherOptions
+	] ++ Option.matcher
 	where
 		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
 		setgitconfig :: String -> Annex ()
diff --git a/Options.hs b/Option.hs
similarity index 77%
rename from Options.hs
rename to Option.hs
index 56f0bc0ee3..d6d8b44a32 100644
--- a/Options.hs
+++ b/Option.hs
@@ -5,13 +5,13 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Options (
-	commonOptions,
-	matcherOptions,
-	flagOption,
-	fieldOption,
+module Option (
+	common,
+	matcher,
+	flag,
+	field,
+	name,
 	ArgDescr(..),
-	Option,
 	OptDescr(..),
 ) where
 
@@ -21,11 +21,10 @@ import System.Log.Logger
 import Common.Annex
 import qualified Annex
 import Limit
-import Types.Option
 import Usage
 
-commonOptions :: [Option]
-commonOptions =
+common :: [Option]
+common =
 	[ Option [] ["force"] (NoArg (setforce True))
 		"allow actions that may lose annexed data"
 	, Option ['F'] ["fast"] (NoArg (setfast True))
@@ -51,9 +50,9 @@ commonOptions =
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG
-	
-matcherOptions :: [Option]
-matcherOptions =
+
+matcher :: [Option]
+matcher =
 	[ longopt "not" "negate next option"
 	, longopt "and" "both previous and next option must match"
 	, longopt "or" "either previous or next option must match"
@@ -65,11 +64,16 @@ matcherOptions =
 		shortopt o = Option o [] $ NoArg $ addToken o
 
 {- An option that sets a flag. -}
-flagOption :: String -> String -> String -> Option
-flagOption short flag description = 
-	Option short [flag] (NoArg (Annex.setFlag flag)) description
+flag :: String -> String -> String -> Option
+flag short opt description = 
+	Option short [opt] (NoArg (Annex.setFlag opt)) description
 
 {- An option that sets a field. -}
-fieldOption :: String -> String -> String -> String -> Option
-fieldOption short field paramdesc description = 
-	Option short [field] (ReqArg (Annex.setField field) paramdesc) description
+field :: String -> String -> String -> String -> Option
+field short opt paramdesc description = 
+	Option short [opt] (ReqArg (Annex.setField opt) paramdesc) description
+
+{- The flag or field name used for an option. -}
+name :: Option -> String
+name (Option _ o _ _) = Prelude.head o
+
diff --git a/Seek.hs b/Seek.hs
index 53101b23e4..fdb117de03 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -20,6 +20,7 @@ import qualified Git
 import qualified Git.LsFiles as LsFiles
 import qualified Git.CheckAttr
 import qualified Limit
+import qualified Option
 
 seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
 seekHelper a params = do
@@ -87,13 +88,13 @@ withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
 
-{- Modifies a seek action using the value of a field, which is fed into
+{- Modifies a seek action using the value of a field option, which is fed into
  - a conversion function, and then is passed into the seek action.
  - This ensures that the conversion function only runs once.
  -}
-withField :: String -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> CommandSeek
-withField field converter a ps = do
-        f <- converter =<< Annex.getField field
+withField :: Option -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> CommandSeek
+withField option converter a ps = do
+        f <- converter =<< Annex.getField (Option.name option)
         a f ps
 
 withNothing :: CommandStart -> CommandSeek
diff --git a/Types.hs b/Types.hs
index c8839b7ebb..4c16fb8f4b 100644
--- a/Types.hs
+++ b/Types.hs
@@ -11,7 +11,8 @@ module Types (
 	Key,
 	UUID(..),
 	Remote,
-	RemoteType
+	RemoteType,
+	Option
 ) where
 
 import Annex
@@ -19,6 +20,7 @@ import Types.Backend
 import Types.Key
 import Types.UUID
 import Types.Remote
+import Types.Option
 
 type Backend = BackendA Annex
 type Remote = RemoteA Annex
diff --git a/Types/Command.hs b/Types/Command.hs
index b173b61c9d..1233df2cd9 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -8,7 +8,6 @@
 module Types.Command where
 
 import Types
-import Types.Option
 
 {- A command runs in these stages.
  -
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 1ff0bba447..4fdeae1a87 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -13,6 +13,7 @@ import qualified Git.Construct
 import CmdLine
 import Command
 import Annex.UUID
+import qualified Option
 
 import qualified Command.ConfigList
 import qualified Command.InAnnex
@@ -41,7 +42,7 @@ cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
 			}
 
 options :: [OptDescr (Annex ())]
-options = commonOptions ++
+options = Option.common ++
 	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid"
 	]
 	where

From a3a9f87047d27306c27f4108ee58af3365f284af Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 15:40:04 -0400
Subject: [PATCH 2933/8313] log: New command that displays the location log for
 file, showing each repository they were added to and removed from.

This needs to run git log on the location log files to get at all past
versions of the file, which tends to be a bit slow.

It would be possible to make a version optimised for showing the location
logs for every key. That would only need to run git log once, so would be
faster, but it would need to process an enormous amount of data, so
would not speed up the individual file case.

In the future it would be nice to support log --format. log --json also
doesn't work right yet.
---
 Annex/Branch.hs    |  1 +
 Command/Log.hs     | 94 ++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  2 +
 Logs/Presence.hs   |  7 +++-
 debian/changelog   |  2 +
 debian/copyright   |  2 +-
 doc/git-annex.mdwn |  5 +++
 7 files changed, 111 insertions(+), 2 deletions(-)
 create mode 100644 Command/Log.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index d3a81d8e50..8f07b7aa23 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -6,6 +6,7 @@
  -}
 
 module Annex.Branch (
+	fullname,
 	name,
 	hasOrigin,
 	hasSibling,
diff --git a/Command/Log.hs b/Command/Log.hs
new file mode 100644
index 0000000000..486efdf113
--- /dev/null
+++ b/Command/Log.hs
@@ -0,0 +1,94 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Log where
+
+import qualified Data.Set as S
+import qualified Data.ByteString.Lazy.Char8 as L
+import Data.Time.Clock.POSIX
+import Data.Time
+import System.Locale
+import Data.Char
+
+import Common.Annex
+import Command
+import qualified Logs.Location
+import qualified Logs.Presence
+import Annex.CatFile
+import qualified Annex.Branch
+import qualified Git
+import Git.Command
+import qualified Remote
+
+def :: [Command]
+def = [command "log" paramPaths seek "shows location log"]
+
+seek :: [CommandSeek]
+seek = [withFilesInGit $ whenAnnexed $ start]
+
+start :: FilePath -> (Key, Backend) -> CommandStart
+start file (key, _) = do
+	showStart file ""
+	liftIO $ putStrLn ""
+	showLog =<< readLog key
+	stop
+
+showLog :: [(POSIXTime, Git.Ref)] -> Annex ()
+showLog v = go Nothing v =<< (liftIO getCurrentTimeZone)
+	where
+		go new [] zone = diff S.empty new zone
+		go new ((ts, ref):ls) zone = do
+			cur <- S.fromList <$> get ref
+			diff cur new zone
+			go (Just (ts, cur)) ls zone
+		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
+			catObject ref
+		diff _ Nothing _ = return ()
+		diff cur (Just (ts, new)) zone = do
+			let time = show $ utcToLocalTime zone $
+				posixSecondsToUTCTime ts
+			output time True added
+			output time False removed
+			where
+				added = S.difference new cur
+				removed = S.difference cur new
+		output time present s = do
+			rs <- map (dropWhile isSpace) . lines <$>
+				Remote.prettyPrintUUIDs "log" (S.toList s)
+			liftIO $ mapM_ (putStrLn . indent . format) rs
+				where
+					format r = unwords
+						[ time
+						, if present then "+" else "-"
+						, r
+						]
+
+getLog :: Key -> Annex [String]
+getLog key = do
+	top <- fromRepo Git.workTree
+	p <- liftIO $ relPathCwdToFile top
+	let logfile = p  Logs.Location.logFile key
+	inRepo $ pipeNullSplit
+		[ Params "log -z --pretty=format:%ct --raw --abbrev=40"
+		, Param $ show Annex.Branch.fullname
+		, Param "--"
+		, Param logfile
+		]
+
+readLog :: Key -> Annex [(POSIXTime, Git.Ref)]
+readLog key = mapMaybe (parse . lines) <$> getLog key
+	where
+		parse (ts:raw:[]) = Just (parseTimeStamp ts, parseRaw raw)
+		parse _ = Nothing
+
+-- Parses something like ":100644 100644 oldsha newsha M"
+parseRaw :: String -> Git.Ref
+parseRaw l = Git.Ref $ words l !! 3
+
+parseTimeStamp :: String -> POSIXTime
+parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (error "bad timestamp") .
+	parseTime defaultTimeLocale "%s"
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 64020754fc..78f20e9d14 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -41,6 +41,7 @@ import qualified Command.Lock
 import qualified Command.PreCommit
 import qualified Command.Find
 import qualified Command.Whereis
+import qualified Command.Log
 import qualified Command.Merge
 import qualified Command.Status
 import qualified Command.Migrate
@@ -85,6 +86,7 @@ cmds = concat
 	, Command.DropUnused.def
 	, Command.Find.def
 	, Command.Whereis.def
+	, Command.Log.def
 	, Command.Merge.def
 	, Command.Status.def
 	, Command.Migrate.def
diff --git a/Logs/Presence.hs b/Logs/Presence.hs
index f5e4f1ea94..372af37d5d 100644
--- a/Logs/Presence.hs
+++ b/Logs/Presence.hs
@@ -13,14 +13,15 @@
 
 module Logs.Presence (
 	LogStatus(..),
+	LogLine,
 	addLog,
 	readLog,
+	getLog,
 	parseLog,
 	showLog,
 	logNow,
 	compactLog,
 	currentLog,
-	LogLine
 ) where
 
 import Data.Time.Clock.POSIX
@@ -80,6 +81,10 @@ logNow s i = do
 currentLog :: FilePath -> Annex [String]
 currentLog file = map info . filterPresent <$> readLog file
 
+{- Given a log, returns only the info that is are still in effect. -}
+getLog :: String -> [String]
+getLog = map info . filterPresent . parseLog
+
 {- Returns the info from LogLines that are in effect. -}
 filterPresent :: [LogLine] -> [LogLine]
 filterPresent = filter (\l -> InfoPresent == status l) . compactLog
diff --git a/debian/changelog b/debian/changelog
index e5687aac14..707e804af0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20120106) UNRELEASED; urgency=low
 
   * Support unescaped repository urls, like git does.
+  * log: New command that displays the location log for file,
+    showing each repository they were added to and removed from.
 
  -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400
 
diff --git a/debian/copyright b/debian/copyright
index a8a38913e4..dd880f1428 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -2,7 +2,7 @@ Format: http://dep.debian.net/deps/dep5/
 Source: native package
 
 Files: *
-Copyright: © 2010-2011 Joey Hess 
+Copyright: © 2010-2012 Joey Hess 
 License: GPL-3+
  The full text of version 3 of the GPL is distributed as doc/GPL in
  this package's source, or in /usr/share/common-licenses/GPL-3 on
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9751560a96..87775ead9b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -273,6 +273,11 @@ subdirectories).
   Displays a list of repositories known to contain the content of the
   specified file or files.
 
+* log [path ...]
+
+  Displays the location log for the specified file or files,
+  showing each repository they were added to ("+") and removed from ("-").
+
 * status
 
   Displays some statistics and other information, including how much data

From 47646d44b7a391d9439998ba34498f2fb74b4259 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 16:24:40 -0400
Subject: [PATCH 2934/8313] use a zipper

---
 Command/Log.hs | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 486efdf113..3489c5ab0c 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -33,29 +33,30 @@ seek = [withFilesInGit $ whenAnnexed $ start]
 start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, _) = do
 	showStart file ""
-	liftIO $ putStrLn ""
 	showLog =<< readLog key
 	stop
 
 showLog :: [(POSIXTime, Git.Ref)] -> Annex ()
-showLog v = go Nothing v =<< (liftIO getCurrentTimeZone)
+showLog ps = do
+	zone <- liftIO getCurrentTimeZone
+	sets <- mapM getset ps
+	liftIO $ putStrLn ""
+	mapM_ (diff zone) $ zip sets (drop 1 sets ++ genesis)
 	where
-		go new [] zone = diff S.empty new zone
-		go new ((ts, ref):ls) zone = do
-			cur <- S.fromList <$> get ref
-			diff cur new zone
-			go (Just (ts, cur)) ls zone
+		genesis = [(0, S.empty)]
+		getset (ts, ref) = do
+			s <- S.fromList <$> get ref
+			return (ts, s)
 		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
 			catObject ref
-		diff _ Nothing _ = return ()
-		diff cur (Just (ts, new)) zone = do
+		diff zone ((ts, new), (_, old)) = do
 			let time = show $ utcToLocalTime zone $
 				posixSecondsToUTCTime ts
 			output time True added
 			output time False removed
 			where
-				added = S.difference new cur
-				removed = S.difference cur new
+				added = S.difference new old
+				removed = S.difference old new
 		output time present s = do
 			rs <- map (dropWhile isSpace) . lines <$>
 				Remote.prettyPrintUUIDs "log" (S.toList s)

From 9fb5f3edc7e0aec79e38cf588b66e66e4a2bdd3c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 17:24:03 -0400
Subject: [PATCH 2935/8313] log --after=date

---
 Command/Log.hs     | 54 ++++++++++++++++++++++++++++++----------------
 Git/Sha.hs         |  3 +++
 Git/UnionMerge.hs  |  3 +--
 Usage.hs           |  2 ++
 doc/git-annex.mdwn |  3 +++
 5 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 3489c5ab0c..51bdbc74c1 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -23,29 +23,39 @@ import qualified Annex.Branch
 import qualified Git
 import Git.Command
 import qualified Remote
+import qualified Option
 
 def :: [Command]
-def = [command "log" paramPaths seek "shows location log"]
+def = [withOptions [afterOption] $
+	command "log" paramPaths seek "shows location log"]
+
+afterOption :: Option
+afterOption = Option.field [] "after" paramDate "show log after date"
 
 seek :: [CommandSeek]
-seek = [withFilesInGit $ whenAnnexed $ start]
+seek = [withField afterOption return $ \afteropt ->
+	withFilesInGit $ whenAnnexed $ start afteropt]
 
-start :: FilePath -> (Key, Backend) -> CommandStart
-start file (key, _) = do
+start :: Maybe String -> FilePath -> (Key, Backend) -> CommandStart
+start afteropt file (key, _) = do
 	showStart file ""
-	showLog =<< readLog key
+	let ps = case afteropt of
+		Nothing -> []
+		Just date -> [Param "--after", Param date]
+	showLog =<< (readLog <$> getLog key ps)
 	stop
 
-showLog :: [(POSIXTime, Git.Ref)] -> Annex ()
+showLog :: [(POSIXTime, (Git.Ref, Git.Ref))] -> Annex ()
 showLog ps = do
 	zone <- liftIO getCurrentTimeZone
-	sets <- mapM getset ps
+	sets <- mapM (getset snd) ps
+	previous <- maybe (return genesis) (getset fst) (lastMaybe ps)
 	liftIO $ putStrLn ""
-	mapM_ (diff zone) $ zip sets (drop 1 sets ++ genesis)
+	mapM_ (diff zone) $ zip sets (drop 1 sets ++ [previous])
 	where
-		genesis = [(0, S.empty)]
-		getset (ts, ref) = do
-			s <- S.fromList <$> get ref
+		genesis = (0, S.empty)
+		getset select (ts, refs) = do
+			s <- S.fromList <$> get (select refs)
 			return (ts, s)
 		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
 			catObject ref
@@ -68,27 +78,33 @@ showLog ps = do
 						, r
 						]
 
-getLog :: Key -> Annex [String]
-getLog key = do
+getLog :: Key -> [CommandParam] -> Annex [String]
+getLog key ps = do
 	top <- fromRepo Git.workTree
 	p <- liftIO $ relPathCwdToFile top
 	let logfile = p  Logs.Location.logFile key
-	inRepo $ pipeNullSplit
+	inRepo $ pipeNullSplit $
 		[ Params "log -z --pretty=format:%ct --raw --abbrev=40"
-		, Param $ show Annex.Branch.fullname
+		, Param "--boundary"
+		] ++ ps ++
+		[ Param $ show Annex.Branch.fullname
 		, Param "--"
 		, Param logfile
 		]
 
-readLog :: Key -> Annex [(POSIXTime, Git.Ref)]
-readLog key = mapMaybe (parse . lines) <$> getLog key
+readLog :: [String] -> [(POSIXTime, (Git.Ref, Git.Ref))]
+readLog = mapMaybe (parse . lines)
 	where
 		parse (ts:raw:[]) = Just (parseTimeStamp ts, parseRaw raw)
 		parse _ = Nothing
 
 -- Parses something like ":100644 100644 oldsha newsha M"
-parseRaw :: String -> Git.Ref
-parseRaw l = Git.Ref $ words l !! 3
+parseRaw :: String -> (Git.Ref, Git.Ref)
+parseRaw l = (Git.Ref oldsha, Git.Ref newsha)
+	where
+		ws = words l
+		oldsha = ws !! 2
+		newsha = ws !! 3
 
 parseTimeStamp :: String -> POSIXTime
 parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (error "bad timestamp") .
diff --git a/Git/Sha.hs b/Git/Sha.hs
index 9b3a346505..2a01ede83e 100644
--- a/Git/Sha.hs
+++ b/Git/Sha.hs
@@ -34,3 +34,6 @@ extractSha s
 {- Size of a git sha. -}
 shaSize :: Int
 shaSize = 40
+
+nullSha :: Ref		
+nullSha = Ref $ replicate shaSize '0'
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index d5323af1d1..4b335e47b1 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -103,14 +103,13 @@ calc_merge ch differ repo streamer = gendiff >>= go
  - a line suitable for update_index that union merges the two sides of the
  - diff. -}
 mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
-mergeFile info file h repo = case filter (/= nullsha) [Ref asha, Ref bsha] of
+mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> use sha
 	shas -> use =<< either return (hashObject repo . L.unlines) =<<
 		calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
-		nullsha = Ref $ replicate shaSize '0'
 		getcontents s = L.lines <$> catObject h s
 		use sha = return $ Just $ update_index_line sha file
 
diff --git a/Usage.hs b/Usage.hs
index 308ade798f..36944053f0 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -72,6 +72,8 @@ paramUUID :: String
 paramUUID = "UUID"
 paramType :: String
 paramType = "TYPE"
+paramDate :: String
+paramDate = "Date"
 paramFormat :: String
 paramFormat = "FORMAT"
 paramKeyValue :: String
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 87775ead9b..b9704f3bd4 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -278,6 +278,9 @@ subdirectories).
   Displays the location log for the specified file or files,
   showing each repository they were added to ("+") and removed from ("-").
 
+  To only show location changes after a date, specify --after=date.
+  (The "date" can be any format accepted by git log, ie "last wednesday")
+
 * status
 
   Displays some statistics and other information, including how much data

From 078788a9e755809ac050fd83eb19c4398d7366d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 17:30:48 -0400
Subject: [PATCH 2936/8313] change log display

Including the file in the lines behaves better when limiting with --after,
since only files that changed in the time period are shown.

Still not fully happy with the line layout, but putting the +/- first
followed by the date seems a good change.
---
 Command/Log.hs | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 51bdbc74c1..2651b14be9 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -38,19 +38,17 @@ seek = [withField afterOption return $ \afteropt ->
 
 start :: Maybe String -> FilePath -> (Key, Backend) -> CommandStart
 start afteropt file (key, _) = do
-	showStart file ""
 	let ps = case afteropt of
 		Nothing -> []
 		Just date -> [Param "--after", Param date]
-	showLog =<< (readLog <$> getLog key ps)
+	showLog file =<< (readLog <$> getLog key ps)
 	stop
 
-showLog :: [(POSIXTime, (Git.Ref, Git.Ref))] -> Annex ()
-showLog ps = do
+showLog :: FilePath -> [(POSIXTime, (Git.Ref, Git.Ref))] -> Annex ()
+showLog file ps = do
 	zone <- liftIO getCurrentTimeZone
 	sets <- mapM (getset snd) ps
 	previous <- maybe (return genesis) (getset fst) (lastMaybe ps)
-	liftIO $ putStrLn ""
 	mapM_ (diff zone) $ zip sets (drop 1 sets ++ [previous])
 	where
 		genesis = (0, S.empty)
@@ -70,11 +68,13 @@ showLog ps = do
 		output time present s = do
 			rs <- map (dropWhile isSpace) . lines <$>
 				Remote.prettyPrintUUIDs "log" (S.toList s)
-			liftIO $ mapM_ (putStrLn . indent . format) rs
+			liftIO $ mapM_ (putStrLn . format) rs
 				where
 					format r = unwords
-						[ time
-						, if present then "+" else "-"
+						[ if present then "+" else "-"
+						, time
+						, file
+						, "|"
 						, r
 						]
 

From 3c88d573990d79a5a964567c4a16068ef5ecfa0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 17:48:02 -0400
Subject: [PATCH 2937/8313] log --max-count=n

---
 Command/Log.hs | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 2651b14be9..ff217e573b 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -26,23 +26,29 @@ import qualified Remote
 import qualified Option
 
 def :: [Command]
-def = [withOptions [afterOption] $
+def = [withOptions [afterOption, maxcountOption] $
 	command "log" paramPaths seek "shows location log"]
 
 afterOption :: Option
 afterOption = Option.field [] "after" paramDate "show log after date"
 
+maxcountOption :: Option
+maxcountOption = Option.field ['n'] "max-count" paramNumber "limit number of logs displayed"
+
 seek :: [CommandSeek]
 seek = [withField afterOption return $ \afteropt ->
-	withFilesInGit $ whenAnnexed $ start afteropt]
+	withField maxcountOption return $ \maxcount ->
+	withFilesInGit $ whenAnnexed $ start afteropt maxcount]
 
-start :: Maybe String -> FilePath -> (Key, Backend) -> CommandStart
-start afteropt file (key, _) = do
-	let ps = case afteropt of
-		Nothing -> []
-		Just date -> [Param "--after", Param date]
+start :: Maybe String -> Maybe String -> FilePath -> (Key, Backend) -> CommandStart
+start afteropt maxcount file (key, _) = do
 	showLog file =<< (readLog <$> getLog key ps)
 	stop
+	where
+		ps = concatMap (\(o, p) -> maybe [] p o)
+			[ (afteropt, \d -> [Param "--after", Param d])
+			, (maxcount, \c -> [Param "--max-count", Param c])
+			]
 
 showLog :: FilePath -> [(POSIXTime, (Git.Ref, Git.Ref))] -> Annex ()
 showLog file ps = do

From bc59da72501db1cfac69315798a7359037bb9002 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 17:50:58 -0400
Subject: [PATCH 2938/8313] Revert "simplify"

This reverts commit e0d6010d3602bfa6a231642816d788d5bc1b6988.
---
 Command/Copy.hs |  2 +-
 Command/Drop.hs |  2 +-
 Command/Find.hs |  4 ++--
 Command/Get.hs  |  2 +-
 Command/Move.hs |  2 +-
 Seek.hs         | 11 +++++++----
 6 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/Command/Copy.hs b/Command/Copy.hs
index a9491f56f4..d789d41f6b 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -16,7 +16,7 @@ def = [withOptions Command.Move.options $ command "copy" paramPaths seek
 	"copy content of files to/from another repository"]
 
 seek :: [CommandSeek]
-seek = [withField "to" $ \to -> withField "from" $ \from ->
+seek = [withField "to" id $ \to -> withField "from" id $ \from ->
 		withNumCopies $ \n -> whenAnnexed $ start to from n]
 
 -- A copy is just a move that does not delete the source file.
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 3ad4077036..f76951f08c 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -25,7 +25,7 @@ fromOption :: Option
 fromOption = fieldOption ['f'] "from" paramRemote "drop content from a remote"
 
 seek :: [CommandSeek]
-seek = [withField "from" $ \from -> withNumCopies $ \n ->
+seek = [withField "from" id $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
 start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Command/Find.hs b/Command/Find.hs
index bc7f200c2e..eb0267c142 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -30,8 +30,8 @@ formatOption :: Option
 formatOption = fieldOption [] "format" paramFormat "control format of output"
 
 seek :: [CommandSeek]
-seek = [withField "format" $ \f ->
-		withFilesInGit $ whenAnnexed $ start $ formatconverter f]
+seek = [withField "format" formatconverter $ \f ->
+		withFilesInGit $ whenAnnexed $ start f]
 	where
 		formatconverter = maybe Nothing (Just . Utility.Format.gen)
 
diff --git a/Command/Get.hs b/Command/Get.hs
index c3a6630cee..4a50fe3fea 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -18,7 +18,7 @@ def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek
 	"make content of annexed files available"]
 
 seek :: [CommandSeek]
-seek = [withField "from" $ \from -> withNumCopies $ \n ->
+seek = [withField "from" id $ \from -> withNumCopies $ \n ->
 	whenAnnexed $ start from n]
 
 start :: Maybe String -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Command/Move.hs b/Command/Move.hs
index b6a72bf6bc..66a0c16602 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -29,7 +29,7 @@ options :: [Option]
 options = [fromOption, toOption]
 
 seek :: [CommandSeek]
-seek = [withField "to" $ \to -> withField "from" $ \from ->
+seek = [withField "to" id $ \to -> withField "from" id $ \from ->
 		withFilesInGit $ whenAnnexed $ start to from True]
 
 start :: Maybe String -> Maybe String -> Bool -> FilePath -> (Key, Backend) -> CommandStart
diff --git a/Seek.hs b/Seek.hs
index 2c121bd96c..af074c7c5b 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -87,10 +87,13 @@ withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
 
-{- Feeds the value of a field to a seek action. -}
-withField :: String -> (Maybe String -> CommandSeek) -> CommandSeek
-withField field a ps = do
-        f <- Annex.getField field
+{- Modifies a seek action using the value of a field, which is fed into
+ - a conversion function, and then is passed into the seek action.
+ - This ensures that the conversion function only runs once.
+ -}
+withField :: String -> (Maybe String -> a) -> (a -> CommandSeek) -> CommandSeek
+withField field converter a ps = do
+        f <- converter <$> Annex.getField field
         a f ps
 
 withNothing :: CommandStart -> CommandSeek

From b59759e33cda492b6cff67559391b65039ca66f3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 17:52:16 -0400
Subject: [PATCH 2939/8313] typo

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 707e804af0..f9ce1e2651 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,7 +1,7 @@
 git-annex (3.20120106) UNRELEASED; urgency=low
 
   * Support unescaped repository urls, like git does.
-  * log: New command that displays the location log for file,
+  * log: New command that displays the location log for files,
     showing each repository they were added to and removed from.
 
  -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400

From 773af32ee4250ec4101ac64816c25c0f51aa118b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Fri, 6 Jan 2012 22:14:35 +0000
Subject: [PATCH 2940/8313]

---
 doc/forum/unlock__47__lock_always_gets_me.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/unlock__47__lock_always_gets_me.mdwn

diff --git a/doc/forum/unlock__47__lock_always_gets_me.mdwn b/doc/forum/unlock__47__lock_always_gets_me.mdwn
new file mode 100644
index 0000000000..0891eed0e5
--- /dev/null
+++ b/doc/forum/unlock__47__lock_always_gets_me.mdwn
@@ -0,0 +1,11 @@
+Several times now I've done something like:
+
+    $ git annex unlock movie.avi
+    $ mv /tmp/fixed.avi movie.avi
+    $ git annex lock movie.avi
+
+Oops, I just lost my fixed.avi!  That really feels like the right sequence of operations to me, so I'm always surprised when I make that mistake.  I would like to see the current `lock` renamed to something like `undo-unlock`, or have the behavior changed to be the same as `add`, or maybe warn and require a `--force` when the file has been changed.
+
+If changing current behavior is undesirable, maybe `unlock` could just print a reminder that `git annex add` is the correct next step after making changes?
+
+Failing that, I suppose I could slowly start to learn from my mistakes.

From d8d72781af094b6eced12eba357f03a7ebdea953 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 18:54:48 -0400
Subject: [PATCH 2941/8313] better data type

---
 Command/Log.hs | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index ff217e573b..388cd21560 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -25,6 +25,12 @@ import Git.Command
 import qualified Remote
 import qualified Option
 
+data RefChange = RefChange 
+	{ changetime :: POSIXTime
+	, oldref :: Git.Ref
+	, newref :: Git.Ref
+	}
+
 def :: [Command]
 def = [withOptions [afterOption, maxcountOption] $
 	command "log" paramPaths seek "shows location log"]
@@ -42,7 +48,7 @@ seek = [withField afterOption return $ \afteropt ->
 
 start :: Maybe String -> Maybe String -> FilePath -> (Key, Backend) -> CommandStart
 start afteropt maxcount file (key, _) = do
-	showLog file =<< (readLog <$> getLog key ps)
+	showLog file =<< readLog <$> getLog key ps
 	stop
 	where
 		ps = concatMap (\(o, p) -> maybe [] p o)
@@ -50,17 +56,17 @@ start afteropt maxcount file (key, _) = do
 			, (maxcount, \c -> [Param "--max-count", Param c])
 			]
 
-showLog :: FilePath -> [(POSIXTime, (Git.Ref, Git.Ref))] -> Annex ()
+showLog :: FilePath -> [RefChange] -> Annex ()
 showLog file ps = do
 	zone <- liftIO getCurrentTimeZone
-	sets <- mapM (getset snd) ps
-	previous <- maybe (return genesis) (getset fst) (lastMaybe ps)
+	sets <- mapM (getset newref) ps
+	previous <- maybe (return genesis) (getset oldref) (lastMaybe ps)
 	mapM_ (diff zone) $ zip sets (drop 1 sets ++ [previous])
 	where
 		genesis = (0, S.empty)
-		getset select (ts, refs) = do
-			s <- S.fromList <$> get (select refs)
-			return (ts, s)
+		getset select change = do
+			s <- S.fromList <$> get (select change)
+			return (changetime change, s)
 		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
 			catObject ref
 		diff zone ((ts, new), (_, old)) = do
@@ -76,13 +82,9 @@ showLog file ps = do
 				Remote.prettyPrintUUIDs "log" (S.toList s)
 			liftIO $ mapM_ (putStrLn . format) rs
 				where
+					addel = if present then "+" else "-"
 					format r = unwords
-						[ if present then "+" else "-"
-						, time
-						, file
-						, "|"
-						, r
-						]
+						[ addel, time, file, "|", r ]
 
 getLog :: Key -> [CommandParam] -> Annex [String]
 getLog key ps = do
@@ -98,10 +100,15 @@ getLog key ps = do
 		, Param logfile
 		]
 
-readLog :: [String] -> [(POSIXTime, (Git.Ref, Git.Ref))]
+readLog :: [String] -> [RefChange]
 readLog = mapMaybe (parse . lines)
 	where
-		parse (ts:raw:[]) = Just (parseTimeStamp ts, parseRaw raw)
+		parse (ts:raw:[]) = let (old, new) = parseRaw raw in
+			Just RefChange
+				{ changetime = parseTimeStamp ts
+				, oldref = old
+				, newref = new
+				}
 		parse _ = Nothing
 
 -- Parses something like ":100644 100644 oldsha newsha M"

From 539f8c6f144e7fa9cb76f34c06417e3c13ff3420 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 21:09:23 -0400
Subject: [PATCH 2942/8313] --boundry was not needed

---
 Command/Log.hs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 388cd21560..b28abfc18c 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -93,7 +93,6 @@ getLog key ps = do
 	let logfile = p  Logs.Location.logFile key
 	inRepo $ pipeNullSplit $
 		[ Params "log -z --pretty=format:%ct --raw --abbrev=40"
-		, Param "--boundary"
 		] ++ ps ++
 		[ Param $ show Annex.Branch.fullname
 		, Param "--"

From 8e7de0104702d59e327694451303e8fbe53c9ea5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 21:27:42 -0400
Subject: [PATCH 2943/8313] log --before=date

---
 Command/Log.hs | 40 ++++++++++++++++++++--------------------
 Seek.hs        | 10 +++++++---
 Usage.hs       |  2 +-
 3 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index b28abfc18c..3336df873d 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -24,6 +24,7 @@ import qualified Git
 import Git.Command
 import qualified Remote
 import qualified Option
+import qualified Annex
 
 data RefChange = RefChange 
 	{ changetime :: POSIXTime
@@ -32,29 +33,28 @@ data RefChange = RefChange
 	}
 
 def :: [Command]
-def = [withOptions [afterOption, maxcountOption] $
+def = [withOptions options $
 	command "log" paramPaths seek "shows location log"]
 
-afterOption :: Option
-afterOption = Option.field [] "after" paramDate "show log after date"
-
-maxcountOption :: Option
-maxcountOption = Option.field ['n'] "max-count" paramNumber "limit number of logs displayed"
+options :: [Option]
+options = 
+	[ Option.field [] "after" paramDate "show log after date"
+	, Option.field [] "before" paramDate "show log before date"
+	, Option.field ['n'] "max-count" paramNumber "limit number of logs displayed"
+	]
 
 seek :: [CommandSeek]
-seek = [withField afterOption return $ \afteropt ->
-	withField maxcountOption return $ \maxcount ->
-	withFilesInGit $ whenAnnexed $ start afteropt maxcount]
-
-start :: Maybe String -> Maybe String -> FilePath -> (Key, Backend) -> CommandStart
-start afteropt maxcount file (key, _) = do
-	showLog file =<< readLog <$> getLog key ps
-	stop
+seek = [withValue (concat <$> mapM getoption options) $ \os ->
+	withFilesInGit $ whenAnnexed $ start os]
 	where
-		ps = concatMap (\(o, p) -> maybe [] p o)
-			[ (afteropt, \d -> [Param "--after", Param d])
-			, (maxcount, \c -> [Param "--max-count", Param c])
-			]
+		getoption o = maybe [] (use o) <$>
+			Annex.getField (Option.name o)
+		use o v = [Param ("--" ++ Option.name o), Param v]
+
+start :: [CommandParam] -> FilePath -> (Key, Backend) -> CommandStart
+start os file (key, _) = do
+	showLog file =<< readLog <$> getLog key os
+	stop
 
 showLog :: FilePath -> [RefChange] -> Annex ()
 showLog file ps = do
@@ -87,13 +87,13 @@ showLog file ps = do
 						[ addel, time, file, "|", r ]
 
 getLog :: Key -> [CommandParam] -> Annex [String]
-getLog key ps = do
+getLog key os = do
 	top <- fromRepo Git.workTree
 	p <- liftIO $ relPathCwdToFile top
 	let logfile = p  Logs.Location.logFile key
 	inRepo $ pipeNullSplit $
 		[ Params "log -z --pretty=format:%ct --raw --abbrev=40"
-		] ++ ps ++
+		] ++ os ++
 		[ Param $ show Annex.Branch.fullname
 		, Param "--"
 		, Param logfile
diff --git a/Seek.hs b/Seek.hs
index fdb117de03..59a85be886 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -88,14 +88,18 @@ withKeys a params = return $ map (a . parse) params
 	where
 		parse p = fromMaybe (error "bad key") $ readKey p
 
+withValue :: Annex v -> (v -> CommandSeek) -> CommandSeek
+withValue v a params = do
+	r <- v
+	a r params
+
 {- Modifies a seek action using the value of a field option, which is fed into
  - a conversion function, and then is passed into the seek action.
  - This ensures that the conversion function only runs once.
  -}
 withField :: Option -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> CommandSeek
-withField option converter a ps = do
-        f <- converter =<< Annex.getField (Option.name option)
-        a f ps
+withField option converter = withValue $
+	converter =<< Annex.getField (Option.name option)
 
 withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
diff --git a/Usage.hs b/Usage.hs
index 36944053f0..34c344b183 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -73,7 +73,7 @@ paramUUID = "UUID"
 paramType :: String
 paramType = "TYPE"
 paramDate :: String
-paramDate = "Date"
+paramDate = "DATE"
 paramFormat :: String
 paramFormat = "FORMAT"
 paramKeyValue :: String

From 2557bb876426d9d72f195f041b4efb43153725bd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 21:48:30 -0400
Subject: [PATCH 2944/8313] complete set of log options

---
 Command/Log.hs     | 4 +++-
 doc/git-annex.mdwn | 5 +++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 3336df873d..d59c8efd8b 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -38,7 +38,9 @@ def = [withOptions options $
 
 options :: [Option]
 options = 
-	[ Option.field [] "after" paramDate "show log after date"
+	[ Option.field [] "since" paramDate "show log since date"
+	, Option.field [] "after" paramDate "show log after date"
+	, Option.field [] "until" paramDate "show log until date"
 	, Option.field [] "before" paramDate "show log before date"
 	, Option.field ['n'] "max-count" paramNumber "limit number of logs displayed"
 	]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index b9704f3bd4..1103ffaf69 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -278,8 +278,9 @@ subdirectories).
   Displays the location log for the specified file or files,
   showing each repository they were added to ("+") and removed from ("-").
 
-  To only show location changes after a date, specify --after=date.
-  (The "date" can be any format accepted by git log, ie "last wednesday")
+  To limit how far back to seach for location log changes, the options
+  --since, --after, --until, --before, and --max-count can be specified.
+  They are passed through to git log. For example, --since "1 month ago"
 
 * status
 

From 64f9d00bedd1d035ec3b465c24ee324184e0a792 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 21:51:39 -0400
Subject: [PATCH 2945/8313] tweak

---
 Command/Log.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index d59c8efd8b..4a2bb7a7f5 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -37,13 +37,13 @@ def = [withOptions options $
 	command "log" paramPaths seek "shows location log"]
 
 options :: [Option]
-options = 
-	[ Option.field [] "since" paramDate "show log since date"
-	, Option.field [] "after" paramDate "show log after date"
-	, Option.field [] "until" paramDate "show log until date"
-	, Option.field [] "before" paramDate "show log before date"
-	, Option.field ['n'] "max-count" paramNumber "limit number of logs displayed"
+options = map odate ["since", "after", "until", "before"] ++
+	[ Option.field ['n'] "max-count" paramNumber
+		"limit number of logs displayed"
 	]
+	where
+		odate n = Option.field [] n paramDate $
+			"show log " ++ n ++ " date"
 
 seek :: [CommandSeek]
 seek = [withValue (concat <$> mapM getoption options) $ \os ->

From d9fdc76a144588b64811d62ee144aa8dde49c8d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 21:53:47 -0400
Subject: [PATCH 2946/8313] layout

---
 Checks.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Checks.hs b/Checks.hs
index 9d846842db..94ee752917 100644
--- a/Checks.hs
+++ b/Checks.hs
@@ -24,8 +24,8 @@ dontCheck :: CommandCheck -> Command -> Command
 dontCheck check cmd = mutateCheck cmd $ \c -> filter (/= check) c
 
 addCheck :: Annex () -> Command -> Command
-addCheck check cmd = mutateCheck cmd $
-	\c -> CommandCheck (length c + 100) check : c
+addCheck check cmd = mutateCheck cmd $ \c ->
+	CommandCheck (length c + 100) check : c
 
 mutateCheck :: Command -> ([CommandCheck] -> [CommandCheck]) -> Command
 mutateCheck cmd@(Command { cmdcheck = c }) a = cmd { cmdcheck = a c }

From 24b35113cf67cf8effd1f70f81fc4ae43987e6a0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Jan 2012 23:43:18 -0400
Subject: [PATCH 2947/8313] tweak

---
 Command/Log.hs | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 4a2bb7a7f5..0e0c8d3a0c 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -63,7 +63,7 @@ showLog file ps = do
 	zone <- liftIO getCurrentTimeZone
 	sets <- mapM (getset newref) ps
 	previous <- maybe (return genesis) (getset oldref) (lastMaybe ps)
-	mapM_ (diff zone) $ zip sets (drop 1 sets ++ [previous])
+	mapM_ (diff file zone) $ zip sets (drop 1 sets ++ [previous])
 	where
 		genesis = (0, S.empty)
 		getset select change = do
@@ -71,15 +71,14 @@ showLog file ps = do
 			return (changetime change, s)
 		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
 			catObject ref
-		diff zone ((ts, new), (_, old)) = do
-			let time = show $ utcToLocalTime zone $
-				posixSecondsToUTCTime ts
-			output time True added
-			output time False removed
-			where
-				added = S.difference new old
-				removed = S.difference old new
-		output time present s = do
+
+diff :: FilePath -> TimeZone -> ((POSIXTime, S.Set UUID), (POSIXTime, S.Set UUID)) -> Annex ()
+diff file zone ((ts, new), (_, old)) = output True added >> output False removed
+	where
+		added = S.difference new old
+		removed = S.difference old new
+		time = showTimeStamp zone ts
+		output present s = do
 			rs <- map (dropWhile isSpace) . lines <$>
 				Remote.prettyPrintUUIDs "log" (S.toList s)
 			liftIO $ mapM_ (putStrLn . format) rs
@@ -123,3 +122,6 @@ parseRaw l = (Git.Ref oldsha, Git.Ref newsha)
 parseTimeStamp :: String -> POSIXTime
 parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (error "bad timestamp") .
 	parseTime defaultTimeLocale "%s"
+
+showTimeStamp :: TimeZone -> POSIXTime -> String
+showTimeStamp zone = show . utcToLocalTime zone . posixSecondsToUTCTime

From 945f56f348c9c4963dbd6b6c969498a28da068bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 00:11:15 -0400
Subject: [PATCH 2948/8313] cleanup

Broke out pure general functions etc.
---
 Command/Log.hs | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 0e0c8d3a0c..eba9226e54 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -63,7 +63,7 @@ showLog file ps = do
 	zone <- liftIO getCurrentTimeZone
 	sets <- mapM (getset newref) ps
 	previous <- maybe (return genesis) (getset oldref) (lastMaybe ps)
-	mapM_ (diff file zone) $ zip sets (drop 1 sets ++ [previous])
+	sequence_ $ compareChanges (output zone) $ sets ++ [previous]
 	where
 		genesis = (0, S.empty)
 		getset select change = do
@@ -71,22 +71,29 @@ showLog file ps = do
 			return (changetime change, s)
 		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
 			catObject ref
-
-diff :: FilePath -> TimeZone -> ((POSIXTime, S.Set UUID), (POSIXTime, S.Set UUID)) -> Annex ()
-diff file zone ((ts, new), (_, old)) = output True added >> output False removed
-	where
-		added = S.difference new old
-		removed = S.difference old new
-		time = showTimeStamp zone ts
-		output present s = do
+		output zone present ts s = do
 			rs <- map (dropWhile isSpace) . lines <$>
 				Remote.prettyPrintUUIDs "log" (S.toList s)
 			liftIO $ mapM_ (putStrLn . format) rs
 				where
+					time = showTimeStamp zone ts
 					addel = if present then "+" else "-"
 					format r = unwords
 						[ addel, time, file, "|", r ]
 
+{- Generates a display of the changes (which are ordered with newest first),
+ - by comparing each change with the previous change.
+ - Uses a formater to generate a display of items that are added and
+ - removed. -}
+compareChanges :: Ord a => (Bool -> POSIXTime -> S.Set a -> b) -> [(POSIXTime, S.Set a)] -> [b]
+compareChanges format changes = concatMap diff $ zip changes (drop 1 changes)
+	where
+		diff ((ts, new), (_, old)) =
+			[format True ts added, format False ts removed]
+			where
+				added = S.difference new old
+				removed = S.difference old new
+
 getLog :: Key -> [CommandParam] -> Annex [String]
 getLog key os = do
 	top <- fromRepo Git.workTree

From b8966433efab3e2e3f66744404c9aec0d36b0ff5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 00:15:01 -0400
Subject: [PATCH 2949/8313] sped up git annex log rather a lot

See comment! Isn't git fun, always interesting approaches to optimise
things that seemed unfixably slow.
---
 Command/Log.hs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/Command/Log.hs b/Command/Log.hs
index eba9226e54..32f03471a8 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -94,6 +94,17 @@ compareChanges format changes = concatMap diff $ zip changes (drop 1 changes)
 				added = S.difference new old
 				removed = S.difference old new
 
+{- Gets the git log for a given location log file.
+ -
+ - This is complicated by git log using paths relative to the current
+ - directory, even when looking at files in a different branch. A wacky
+ - relative path to the log file has to be used.
+ -
+ - The --remove-empty is a significant optimisation. It relies on location
+ - log files never being deleted in normal operation. Letting git stop
+ - once the location log file is gone avoids it checking all the way back
+ - to commit 0 to see if it used to exist, so generally speeds things up a
+ - *lot* for newish files. -}
 getLog :: Key -> [CommandParam] -> Annex [String]
 getLog key os = do
 	top <- fromRepo Git.workTree
@@ -101,6 +112,7 @@ getLog key os = do
 	let logfile = p  Logs.Location.logFile key
 	inRepo $ pipeNullSplit $
 		[ Params "log -z --pretty=format:%ct --raw --abbrev=40"
+		, Param "--remove-empty"
 		] ++ os ++
 		[ Param $ show Annex.Branch.fullname
 		, Param "--"

From dfa76069d45adff2c3edf79b184d6767b2bbe6bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 00:22:16 -0400
Subject: [PATCH 2950/8313] reap zombies

---
 Command/Log.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Log.hs b/Command/Log.hs
index 32f03471a8..8505fd16a5 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -56,6 +56,7 @@ seek = [withValue (concat <$> mapM getoption options) $ \os ->
 start :: [CommandParam] -> FilePath -> (Key, Backend) -> CommandStart
 start os file (key, _) = do
 	showLog file =<< readLog <$> getLog key os
+	liftIO Git.Command.reap
 	stop
 
 showLog :: FilePath -> [RefChange] -> Annex ()

From bdc49ddbdb32146356fe2040d08071d7ce963466 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 00:45:01 -0400
Subject: [PATCH 2951/8313] typo

---
 Command/Log.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Log.hs b/Command/Log.hs
index 8505fd16a5..9b0e38626e 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -84,7 +84,7 @@ showLog file ps = do
 
 {- Generates a display of the changes (which are ordered with newest first),
  - by comparing each change with the previous change.
- - Uses a formater to generate a display of items that are added and
+ - Uses a formatter to generate a display of items that are added and
  - removed. -}
 compareChanges :: Ord a => (Bool -> POSIXTime -> S.Set a -> b) -> [(POSIXTime, S.Set a)] -> [b]
 compareChanges format changes = concatMap diff $ zip changes (drop 1 changes)

From 60c1aeeb6fa106756c00e7bb18bbf311bd553fe1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 12:38:08 -0400
Subject: [PATCH 2952/8313] Fix overbroad gpg --no-tty fix from last release.

Only set --no-tty when GPG_AGENT_INFO is set and batch mode is used.

In the test suite, set GPG_AGENT_INFO to /dev/null to avoid the test suite
relying on /dev/tty.
---
 Utility/Gpg.hs   | 7 +++----
 debian/changelog | 1 +
 test.hs          | 2 ++
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index b50e775d03..8f7e27870a 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -25,12 +25,11 @@ stdParams params = do
 	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
 	-- gpg output about password prompts.
 	e <- getEnv "GPG_AGENT_INFO"
-	let batch = if isNothing e then [] else ["--batch"]
+	let batch = if isNothing e then [] else ["--batch", "--no-tty"]
 	return $ batch ++ defaults ++ toCommand params
 	where
-		-- be quiet, even about checking the trustdb,
-		-- and avoid using a tty
-		defaults = ["--quiet", "--trust-model", "always", "--no-tty"]
+		-- be quiet, even about checking the trustdb
+		defaults = ["--quiet", "--trust-model", "always"]
 
 {- Runs gpg with some params and returns its stdout, strictly. -}
 readStrict :: [CommandParam] -> IO String
diff --git a/debian/changelog b/debian/changelog
index f9ce1e2651..afca096b2b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20120106) UNRELEASED; urgency=low
   * Support unescaped repository urls, like git does.
   * log: New command that displays the location log for files,
     showing each repository they were added to and removed from.
+  * Fix overbroad gpg --no-tty fix from last release.
 
  -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400
 
diff --git a/test.hs b/test.hs
index 7350a07697..2ce3f8b0b3 100644
--- a/test.hs
+++ b/test.hs
@@ -663,6 +663,8 @@ test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ when Build.SysConfi
 -- gpg is not a build dependency, so only test when it's available
 test_crypto :: Test
 test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $ do
+	-- force gpg into batch mode for the tests
+	setEnv "GPG_AGENT_INFO" "/dev/null" True
 	Utility.Gpg.testTestHarness @? "test harness self-test failed"
 	Utility.Gpg.testHarness $ do
 		createDirectory "dir"

From ba233606d25ce9f1207ea74c5cb96d3931671d8b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 7 Jan 2012 17:15:31 +0000
Subject: [PATCH 2953/8313] Added a comment

---
 ...mment_1_dee73a7ea3e1a5154601adb59782831f._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment

diff --git a/doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment b/doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment
new file mode 100644
index 0000000000..c37561665f
--- /dev/null
+++ b/doc/forum/unlock__47__lock_always_gets_me/comment_1_dee73a7ea3e1a5154601adb59782831f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-07T17:15:31Z"
+ content="""
+Well, lock could check for modifications and require --force to lose them. But the check could be expensive for large files.
+
+But `git annex lock` is just a convenient way to run `git checkout`. And running `git checkout` or `git reset --hard` will lose your uncommitted file the same way obviously. 
+
+Perhaps the best fix would be to get rid of `lock` entirely, and let the user use the underlying git commands same as they would to drop modifications to other files. It would then also make sense to remove `unlock`, leaving only `edit`.
+"""]]

From 3da28cad07c78ab04b8fcfa477dacea728c4fab6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 13:50:35 -0400
Subject: [PATCH 2954/8313] releasing version 3.20120106

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index afca096b2b..9b1e901d69 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (3.20120106) UNRELEASED; urgency=low
+git-annex (3.20120106) unstable; urgency=low
 
   * Support unescaped repository urls, like git does.
   * log: New command that displays the location log for files,
     showing each repository they were added to and removed from.
   * Fix overbroad gpg --no-tty fix from last release.
 
- -- Joey Hess   Thu, 05 Jan 2012 14:29:30 -0400
+ -- Joey Hess   Sat, 07 Jan 2012 13:16:23 -0400
 
 git-annex (3.20120105) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index bc87b467c8..b0b6d76f97 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120105
+Version: 3.20120106
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 2f0c3befbd3c04fab474a8cec30f830e08828006 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 13:50:54 -0400
Subject: [PATCH 2955/8313] add news item for git-annex 3.20120106

---
 doc/news/version_3.20111122.mdwn | 22 ----------------------
 doc/news/version_3.20120106.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 22 deletions(-)
 delete mode 100644 doc/news/version_3.20111122.mdwn
 create mode 100644 doc/news/version_3.20120106.mdwn

diff --git a/doc/news/version_3.20111122.mdwn b/doc/news/version_3.20111122.mdwn
deleted file mode 100644
index 193394de98..0000000000
--- a/doc/news/version_3.20111122.mdwn
+++ /dev/null
@@ -1,22 +0,0 @@
-git-annex 3.20111122 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * merge: Improve commit messages to mention what was merged.
-   * Avoid doing auto-merging in commands that don't need fully current
-     information from the git-annex branch. In particular, git annex add
-     no longer needs to auto-merge.
-   * init: When run in an already initalized repository, and without
-     a description specified, don't delete the old description.
-   * Optimised union merging; now only runs git cat-file once, and runs
-     in constant space.
-   * status: Now displays trusted, untrusted, and semitrusted repositories
-     separately.
-   * status: Include all special remotes in the list of repositories.
-   * status: Fix --json mode.
-   * status: --fast is back
-   * Fix support for insteadOf url remapping. Closes: #[644278](http://bugs.debian.org/644278)
-   * When not run in a git repository, git-annex can still display a usage
-     message, and "git annex version" even works.
-   * migrate: Don't fall over a stale temp file.
-   * Avoid excessive escaping for rsync special remotes that are not accessed
-     over ssh.
-   * find: Support --print0"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120106.mdwn b/doc/news/version_3.20120106.mdwn
new file mode 100644
index 0000000000..076184aaf7
--- /dev/null
+++ b/doc/news/version_3.20120106.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20120106 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Support unescaped repository urls, like git does.
+   * log: New command that displays the location log for files,
+     showing each repository they were added to and removed from.
+   * Fix overbroad gpg --no-tty fix from last release."""]]
\ No newline at end of file

From a35278430ae2dd3ae2f0c5be291e49077bcac534 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 18:13:12 -0400
Subject: [PATCH 2956/8313] log: Add --gource mode, which generates output
 usable by gource.

As part of this, I fixed up how log was getting the descriptions of
remotes.
---
 Command/Log.hs                                | 69 +++++++++++++------
 Remote.hs                                     | 27 +++++---
 Seek.hs                                       |  3 +
 debian/changelog                              |  6 ++
 doc/git-annex.mdwn                            |  3 +
 .../visualizing_repositories_with_gource.mdwn | 20 ++++++
 6 files changed, 97 insertions(+), 31 deletions(-)
 create mode 100644 doc/tips/visualizing_repositories_with_gource.mdwn

diff --git a/Command/Log.hs b/Command/Log.hs
index 9b0e38626e..4013b535ef 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -8,6 +8,7 @@
 module Command.Log where
 
 import qualified Data.Set as S
+import qualified Data.Map as M
 import qualified Data.ByteString.Lazy.Char8 as L
 import Data.Time.Clock.POSIX
 import Data.Time
@@ -32,12 +33,17 @@ data RefChange = RefChange
 	, newref :: Git.Ref
 	}
 
+type Outputter = Bool -> POSIXTime -> [UUID] -> Annex ()
+
 def :: [Command]
 def = [withOptions options $
 	command "log" paramPaths seek "shows location log"]
 
 options :: [Option]
-options = map odate ["since", "after", "until", "before"] ++
+options = passthruOptions ++ [gourceOption]
+
+passthruOptions :: [Option]
+passthruOptions = map odate ["since", "after", "until", "before"] ++
 	[ Option.field ['n'] "max-count" paramNumber
 		"limit number of logs displayed"
 	]
@@ -45,26 +51,37 @@ options = map odate ["since", "after", "until", "before"] ++
 		odate n = Option.field [] n paramDate $
 			"show log " ++ n ++ " date"
 
+gourceOption :: Option
+gourceOption = Option.flag [] "gource" "format output for gource"
+
 seek :: [CommandSeek]
-seek = [withValue (concat <$> mapM getoption options) $ \os ->
-	withFilesInGit $ whenAnnexed $ start os]
+seek = [withValue (Remote.uuidDescriptions) $ \m ->
+	withValue (liftIO getCurrentTimeZone) $ \zone ->
+	withValue (concat <$> mapM getoption passthruOptions) $ \os ->
+	withFlag gourceOption $ \gource ->
+	withFilesInGit $ whenAnnexed $ start m zone os gource]
 	where
 		getoption o = maybe [] (use o) <$>
 			Annex.getField (Option.name o)
 		use o v = [Param ("--" ++ Option.name o), Param v]
 
-start :: [CommandParam] -> FilePath -> (Key, Backend) -> CommandStart
-start os file (key, _) = do
-	showLog file =<< readLog <$> getLog key os
+start :: (M.Map UUID String) -> TimeZone -> [CommandParam] -> Bool ->
+	FilePath -> (Key, Backend) -> CommandStart
+start m zone os gource file (key, _) = do
+	showLog output =<< readLog <$> getLog key os
 	liftIO Git.Command.reap
 	stop
+	where
+		output
+			| gource = gourceOutput lookupdescription file
+			| otherwise = normalOutput lookupdescription file zone
+		lookupdescription u = fromMaybe (fromUUID u) $ M.lookup u m
 
-showLog :: FilePath -> [RefChange] -> Annex ()
-showLog file ps = do
-	zone <- liftIO getCurrentTimeZone
+showLog :: Outputter -> [RefChange] -> Annex ()
+showLog outputter ps = do
 	sets <- mapM (getset newref) ps
 	previous <- maybe (return genesis) (getset oldref) (lastMaybe ps)
-	sequence_ $ compareChanges (output zone) $ sets ++ [previous]
+	sequence_ $ compareChanges outputter $ sets ++ [previous]
 	where
 		genesis = (0, S.empty)
 		getset select change = do
@@ -72,28 +89,36 @@ showLog file ps = do
 			return (changetime change, s)
 		get ref = map toUUID . Logs.Presence.getLog . L.unpack <$>
 			catObject ref
-		output zone present ts s = do
-			rs <- map (dropWhile isSpace) . lines <$>
-				Remote.prettyPrintUUIDs "log" (S.toList s)
-			liftIO $ mapM_ (putStrLn . format) rs
-				where
-					time = showTimeStamp zone ts
-					addel = if present then "+" else "-"
-					format r = unwords
-						[ addel, time, file, "|", r ]
+
+normalOutput :: (UUID -> String) -> FilePath -> TimeZone -> Outputter
+normalOutput lookupdescription file zone present ts us = do
+	liftIO $ mapM_ (putStrLn . format) us
+	where
+		time = showTimeStamp zone ts
+		addel = if present then "+" else "-"
+		format u = unwords [ addel, time, file, "|", 
+			fromUUID u ++ " -- " ++ lookupdescription u ]
+
+gourceOutput :: (UUID -> String) -> FilePath -> Outputter
+gourceOutput lookupdescription file present ts us = do
+	liftIO $ mapM_ (putStrLn . intercalate "|" . format) us
+	where
+		time = takeWhile isDigit $ show ts
+		addel = if present then "A" else "M"
+		format u = [ time, lookupdescription u, addel, file ]
 
 {- Generates a display of the changes (which are ordered with newest first),
  - by comparing each change with the previous change.
  - Uses a formatter to generate a display of items that are added and
  - removed. -}
-compareChanges :: Ord a => (Bool -> POSIXTime -> S.Set a -> b) -> [(POSIXTime, S.Set a)] -> [b]
+compareChanges :: Ord a => (Bool -> POSIXTime -> [a] -> b) -> [(POSIXTime, S.Set a)] -> [b]
 compareChanges format changes = concatMap diff $ zip changes (drop 1 changes)
 	where
 		diff ((ts, new), (_, old)) =
 			[format True ts added, format False ts removed]
 			where
-				added = S.difference new old
-				removed = S.difference old new
+				added = S.toList $ S.difference new old
+				removed = S.toList $ S.difference old new
 
 {- Gets the git log for a given location log file.
  -
diff --git a/Remote.hs b/Remote.hs
index 3f60ca3acf..63d32f4295 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -19,6 +19,7 @@ module Remote (
 	remoteList,
 	enabledRemoteList,
 	remoteMap,
+	uuidDescriptions,
 	byName,
 	prettyPrintUUIDs,
 	remotesWithUUID,
@@ -94,6 +95,18 @@ enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
 remoteMap :: Annex (M.Map UUID String)
 remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> remoteList
 
+{- Map of UUIDs and their descriptions.
+ - The names of Remotes are added to suppliment any description that has
+ - been set for a repository. -}
+uuidDescriptions :: Annex (M.Map UUID String)
+uuidDescriptions = M.unionWith addName <$> uuidMap <*> remoteMap
+
+addName :: String -> String -> String
+addName desc n
+	| desc == n = desc
+	| null desc = n
+	| otherwise = n ++ " (" ++ desc ++ ")"
+
 {- When a name is specified, looks up the remote matching that name.
  - (Or it can be a UUID.) Only finds currently configured git remotes. -}
 byName :: Maybe String -> Annex (Maybe Remote)
@@ -143,28 +156,24 @@ nameToUUID n = byName' n >>= go
 prettyPrintUUIDs :: String -> [UUID] -> Annex String
 prettyPrintUUIDs desc uuids = do
 	hereu <- getUUID
-	m <- M.unionWith addname <$> uuidMap <*> remoteMap
+	m <- uuidDescriptions
 	maybeShowJSON [(desc, map (jsonify m hereu) uuids)]
 	return $ unwords $ map (\u -> "\t" ++ prettify m hereu u ++ "\n") uuids
 	where
-		addname d n
-			| d == n = d
-			| null d = n
-			| otherwise = n ++ " (" ++ d ++ ")"
-		findlog m u = M.findWithDefault "" u m
+		finddescription m u = M.findWithDefault "" u m
 		prettify m hereu u
 			| not (null d) = fromUUID u ++ " -- " ++ d
 			| otherwise = fromUUID u
 			where
 				ishere = hereu == u
-				n = findlog m u
+				n = finddescription m u
 				d
 					| null n && ishere = "here"
-					| ishere = addname n "here"
+					| ishere = addName n "here"
 					| otherwise = n
 		jsonify m hereu u = toJSObject
 			[ ("uuid", toJSON $ fromUUID u)
-			, ("description", toJSON $ findlog m u)
+			, ("description", toJSON $ finddescription m u)
 			, ("here", toJSON $ hereu == u)
 			]
 
diff --git a/Seek.hs b/Seek.hs
index 59a85be886..bf0770f404 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -101,6 +101,9 @@ withField :: Option -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> Comman
 withField option converter = withValue $
 	converter =<< Annex.getField (Option.name option)
 
+withFlag :: Option -> (Bool -> CommandSeek) -> CommandSeek
+withFlag option = withValue $ Annex.getFlag (Option.name option)
+
 withNothing :: CommandStart -> CommandSeek
 withNothing a [] = return [a]
 withNothing _ _ = error "This command takes no parameters."
diff --git a/debian/changelog b/debian/changelog
index 9b1e901d69..f61a4c799a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120107) UNRELEASED; urgency=low
+
+  * log: Add --gource mode, which generates output usable by gource.
+
+ -- Joey Hess   Sat, 07 Jan 2012 18:12:09 -0400
+
 git-annex (3.20120106) unstable; urgency=low
 
   * Support unescaped repository urls, like git does.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 1103ffaf69..629e191b5b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -282,6 +282,9 @@ subdirectories).
   --since, --after, --until, --before, and --max-count can be specified.
   They are passed through to git log. For example, --since "1 month ago"
 
+  To generate output suitable for the gource visualisation program,
+  specify --gource.
+
 * status
 
   Displays some statistics and other information, including how much data
diff --git a/doc/tips/visualizing_repositories_with_gource.mdwn b/doc/tips/visualizing_repositories_with_gource.mdwn
new file mode 100644
index 0000000000..5d9aa4fc24
--- /dev/null
+++ b/doc/tips/visualizing_repositories_with_gource.mdwn
@@ -0,0 +1,20 @@
+[Gource](http://code.google.com/p/gource/) is an amazing animated
+visualisation of a git repository.
+
+Normally, gource shows files being added, removed, and changed in
+the repository, and the user(s) making the changes. Of course it can be
+used in this way in a repository using git-annex too; just run `gource`.
+
+The other way to use gource with git-annex is to visualise the movement of
+annexed file contents between repositories. In this view, the "users" are
+repositories, and they move around the file contents that are being added
+or removed from them with git-annex.
+
+To use gource this way, first go into the directory you want to visualize,
+and use `git annex log` to make an input file for `gource`:
+
+	git annex log --gource | tee gorce.log
+	sort gource.log | gource --log-format custom -
+
+The `git annex log` can take a while, to speed it up you can use something
+like `--after "4 monts ago" to limit how far back it goes.

From 27e3429667ac221a4163b90236c0bfaa9e3492cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 21:46:32 -0400
Subject: [PATCH 2957/8313] new tip

---
 .../visualizing_repositories_with_gource.mdwn   |   2 ++
 .../screenshot.jpg                              | Bin 0 -> 78509 bytes
 2 files changed, 2 insertions(+)
 create mode 100644 doc/tips/visualizing_repositories_with_gource/screenshot.jpg

diff --git a/doc/tips/visualizing_repositories_with_gource.mdwn b/doc/tips/visualizing_repositories_with_gource.mdwn
index 5d9aa4fc24..02cf159b2a 100644
--- a/doc/tips/visualizing_repositories_with_gource.mdwn
+++ b/doc/tips/visualizing_repositories_with_gource.mdwn
@@ -10,6 +10,8 @@ annexed file contents between repositories. In this view, the "users" are
 repositories, and they move around the file contents that are being added
 or removed from them with git-annex.
 
+[[!img screenshot.jpg]]
+
 To use gource this way, first go into the directory you want to visualize,
 and use `git annex log` to make an input file for `gource`:
 
diff --git a/doc/tips/visualizing_repositories_with_gource/screenshot.jpg b/doc/tips/visualizing_repositories_with_gource/screenshot.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9d3096b99bbaba15008aa0d1b675b9db64d20493
GIT binary patch
literal 78509
zcmb5VbyQRTA3uIKY9mKYq`RjeT{5~mgbfu0q`ON83_%)%i3pQ!kd#nbNs(4kL{daV
zM8bgI<^B16&-woQy&M*=*R8|h`KZ_Pp3j|s0ucJzdfEU40s(ZvAK?59fa&-I`8ztF
zF9I3>3G`yScqIWhQgYIZ4Mt8*Mh=HjQo>VMDqG3$eG76db
z)Ejz{=9ByUQW|%fE(r+wH?B%+IXMRg#S~Bdkkzzu%*-lc5t6a6v<|?RfTw1=IMe@q
zVE}+!oSK{hoK&U$fGThTP#2k!(?e!~P-;GsD%`aMQV2>b`8i~Q7XpOR
zMze5S!tEwdbJIcaUX(ze1Og)AgC9s4Ks0+pJ+vuF+1NT7ST^uw&}S^`xrpsSfjfew
z+NuI~B+BsDCI8O0EuM=0NUruv)z>v_U|F_(_!6HpxY=l(3-=CT#d^}ZX4VCx`}Wk{
z`MQqJRP#O!OlZ}gr*FX|J*=a3T>3!M>(l|MH!*8-Z|S!r}Q`oeZx1w9-7j9-1H&?;1Tnak=>y4#K8
z%2!|JAZ9;9+eZMdeSVkyI@heTO7Jx1POE{n-RHux2OTRU<%zC2th;jl)2a5!DZjJJ
zf9oAZ7JUf!MwwN5%SboRK9NqhXnEy~TT6ME-6F%?ssVWE74J9dW^9JA8LPLzLnKA;_8vstTNEb^NE4=C@pT$DHa#@FM^nH
zIB4?maI9=6>&fIfNkkJ1jB~#FrAM#eadgIXJsaB
znK=4YyjtXKjqoLqKOk>O0RTk+8TZAGRd6z7mT|$MEVTn0PorngZpcpUYrp%EcP;vZ
z$_t|_Y0n?se&bL&!}z9bQU7#>?h@1LSOA9Cv-~rSGI~p(pA;%P5MUART=ES$|243b
zJs4Z)m?Y}a$TJ0gb2a%WJZ1!OFZ^D+I5<7_|fpn?bn~B{@>jdsAgnr
z1v=@wY&>PI&dbZk@vE+?-|sPgs&!a#GZHa#bIO)_uOYQIT>j@}-tG};q-xbfGctX4
z#g*#iN`=pO%&1py_M`+|@_{4UXhQPFpgflZa%{(E{_}Y8;qK!U)u5^hFQ&V;MK#oZ
zl4-9on8_IO6TYCwvNx>@Rjgq%D&Za9HKotUGiqgRBUXyK-urnB6jaaruG^@*bL_fo
z{R#VRNFxXEz{tnnLMGQZ5k8nGR906K3b*{f!1@0U-?oc-7RZ@iFokf3S8VU>$;@X=
z>-Kzi%Zp{D?Ppt08CDfAnZ0i)eN@whO{q-B?rc@3vOX-bhy37r?Kjj0=w6p-Z=7%@
zUcHpN__0g1aMre{bW~^b-W&zq}YG&d4ryC_TjZZk!Z(D^9;Nyg4c#P<0*a4_wM3bUlrQ7v&Y8iww@V#P6CUNr%B
zpZ0G=#E$^CDPoFadyQ5PI{P;OFY6z118_U6VN1O;nh^vwos0M3V=l++&2|$fCysb?iUQ`tgwhq0L!95qX%f@*^rsdzy4;{f#!g?Hue3
z8H+H)j-TU0lA58osC-0GUz~58ibQgAKwmPc&6}%&$kb3XToOWF<#Bxk?=Ahg-rn7U
zly<&bTJ_8^uTpxoDV%z6O3!_ZRxSFJxFM?)MMQRgt>mjBJf`w0Sr3R6xV?uGkci9L
zY)c}myQOFfnAKI(Igl%sc4l<-U*_|D2a8u}r@7tdfcE3FUgLN*K9D%LnJ9-fWvP02
zUD2E*RT7F77cT{LkXaTEOGuU%42sr{Gw
zw4|2aGmND9OY?4!Pw3wIUqU5vWLk-=D^kNEv%PfU4Grxxj+yYM+D*SqtiOs%xn`z{
zD%!%1v5ZN{ISRdgUx?8`X}X=Bt!~RA24z0KYD=O>RNBfSOi1Q4(L-$XqxY$d4uv%5
zTL{&m-42JA^zgTrd!DSX)`t9fHT=DPrz`%$?@L-A0~fz#+HMz!I~;M5qEaT=J+V01
zA47FzNhVb@T05t+hbx`;1~{~Jshu0rHoc8r{TVC
zh-|5$@W0O2JxW#o#a89!^a7`d_iyd+oJxDn8NZo&D;C^hqFfM3S?G(6>>iaXbq$NZ
zS_t3#VkC`BtOxa8Ut_tCe+HRT)jBHmt}4n*e!&0_u#`sa2q;fyUw_Ib74cAwZ~gu6
z*(#?Gh#f`D?U%hqY?Uc&dA{4jWJ|_pN9Ywcd#`bojzy+^FLQwj3+gMqfRz=c@MlIRO_T
za_!Y4+snCQWO+=q#vbPzN1e?z&bVhBd??QSSU%OU@D+6i=Ps|xQdzJ6p}8-asqiT^
zO8dHp>#rN{msJ9%vk6bFe4kgep?~}GxlyM)C%q(}F4rqhQuRK{w5oN-3yb-~H?N$P
zvH#AocXR8ouP483|;PTTCx4@+FJU
zCr3~P^eK|OmfgRhB0lf;^VNczGVM*;7jO+_q)lD9UHiXMk+V;CG6!$D8_2q_MHPMj
zefm!I=mvuDqU%p^ZMWUcC)J#{cB!c!4TX{b;tx|GX?^9g#e(4$QB)4F{z9!ue5>~J
zoT+?rknG{Ot%XS@fJ-R+NglXk6GpiT_&LotU@&*3?lrv;G4#^NId3
z+*thdsxJuKt71z6N{joi9QSdIeK*?leJF69;69YABQdY{xH6CF@IWf!6$WI6&=wGj
z`(no7OY)`|90PV$hc0W6j*`k0O67}h?F(~fm;qu|VE?_qsOGi0Kj%crx
zCG$rhi0aU#lJwuO08;{|+eVJM`qd%P*MuV&%s9XlnVG>u$|L@L*an4G2lU*bj0x(t
z%OKT6r4xcocpx0$Aj>*FwNSpbieW+y1`#OP!?#u%^N=H;3F>N_j7ffDYAowG28^y*
zbgEuy2{T!7t~e=C*H_<DmRc&N9>C
z94fmH_bxU6bmQpm^~S`1sI>|#B&=xNbrx@IZu-t^A>yh$str(V-{f+W)@}PN
z=3dJz(|gv$#};bbQZqwKezJl;X~xZUIi>RVyV7Rw;lD(eZnVzt9aq6M?IhcloHMWa
z4NqS3MTFy^o4M7-&(wd#Axoxz(N(lsDon6Ahpu^R8_0L8Q^%&QE<3eQEND;t@p!7G
zhebJfUuLSIcD#A^v(f*$L1{_qW4+YS7QKLTfTLn-$GLLm-Q~dAgZtOZkueKi)8hs_
zZzqPfo%c8M=uhK*JxDoZ?Kdg?^0|BYIkOC750|v>l5-3!>45=eKpsl&_b6{o$%Sft
z0q*WzQt`anFEIPs6T7v)#{8N0uf5!e{YI~+<&^lyRKt2C!IAo!de6}adLS&Baqip3
z-PJJZ`eC7G8+VBUG=@J0d+xK!(5RJ)MDc#`^kUZ7@%mNcm$uyNnKo11)5W1O
z+x=E**8HuNla;rpO%^-Y&9<(_Z$Sy_LugMkX~f)O-naZ=p(nbZtgA$vN-Lt2+twU<
z2e0M=Z+eFnjEyZK@)NWr5|hK3-`-#rYp3#m03suy{S3C{8psVI>1N<8-ScGmU_vjXAk%30)B
zcD@z~b8z7wm`k3(4DT^FapA9|^`R{!k1oCyjl&G>L8NrS3*_w8ARVh>mVSY<_4K*#
z-|5iE=f0tJEp*Ye#3qp0X6tf3Gbh1^sfT^4InIjUZIpXh05P^
zz0k9(YVLp<2#yEiFoQ^LQe+`Y1Qfa;YMK&2S*wo0i4iJ3?Wo@;-=Q)k^Ix6-NB@(+
zXbchC%@1NCvhZS>!n|-yuU@qMX@z)S{bniQ#2{D^p$@%9)J7nknYf_sK5=9@%Qh$j
z9-0&$S`Y(A!-d8BG{p8p)G>ML7l?`=Yy}>?zEIu;+`h)B{emX;2`H7Gx-EW72JZIG
z7l-(S*v~qcKoE!t5VAIPOoY(>vAT|C%^fPKF*P4UAtXZqP7%DHM-68Eu^oP-DKQLh
zzt8eKM3EG6GUTV!kb4Eqf&?Z|KjWBF6D4#sr$E3p;RlMQ$3Saw?OHLu06^^&
zL!|J9s9rP`ihagjkPs6O0fd`_Z}q_AirLfmQ-=Ucd%m7qG-M>)(gA&$6KN!kM^INX
zd)h?wxU_To3RQ2R9*h%_AP0*Luk6_^dq68Ssq40>E^)0?kBlU_l1BUeiB|8t5jT>h
zas_kWVd?&%UiU6@=4xN@J&GgdK`@SZde&=Ri@yOua9HxOzxOY~E+aT@;4~?mASe#A
zW?>TH1;qk5goT3%$f3}LaD%W4TbKdSQrf=4ReJ)*^1fV3d1~$&qm3LoUh1@bAhUN;
zC4VM8iN4zG$jYW>AeY`bamES3Z1Y=;K2M}fFv(MNM!!TyRlqO5qml<%{O^YU;phiN
z8mY3PhLx>Mh^9R6lLm!v0aEom1?k#;qm8Q#VrGA_^cnLWQ#*}rLgUF}mD7&18jpB7
ze|PSrAYVlZ)Q#?qnvj0{QyxJTx)KApDt&^6h~4mD`Rgam?muM3ohZoOC@6jGfxh8r
zHbxA8`0CNlU_7<2nq$1Cjs2H6hm@h-K0^l^d(p<}qTd=jriW)$Wx189f8rcGh}-jz
zwSTlC$=4m9-F$!?-%)*T&DI%Hx9FUU8Oa^Wr}$kyFxUTp>Y*=b9tVyprQxN&p_lR%
zWo<#m6+CM0BsJ0|64se)E6w8X{G+5*R?;o=XUXRoiAalS`Nks2w^AguXCBbYXXuIp~apXb@R`*#&Z
zKJ7p2ZFT-MmQCHjoO}<#a&$wb8^i_v0rNmN?}xV~IfuQI>XHev;-FPO2RNc_!n{U5nvtPG3xvm>}T^C|^{YG14+5X?h
zZTBv|m=ac1DnvIBl{b^eb8BlF*F7bf(3%OS%5MG!9e`omyIWzg=72R4xsj_CVMGB
zD4mHSrN@)yTgAMqu$!3+5^Kl(L}TZb@nms4*-JBI?bcNOabuwKMQ`Kq2TKLwgVI3E
z(g7M9=Po*BZRp#h9zR?1a>)xSPBq?
z?_+ZK=g3_gbN>e1$F9ux^cHH|5~xEYNIU3yoB5s^LRP7kc(^DDEqZ{m3X|^0*k`0_7GCn#lpBK$pk`Ius%ttoeccBTZHvO*?r_wpAl;K*nr_1j&|u4h(Ic
z1NGJ>ab`0#Z(=s@A1HuuAv(vUj)V1>DilMUh52bRqUSZjib<+~bVWXrAb<}7AUMpT
zaRoG!(X)Vox;=-bv>>Dd_=(A(aHfIdF1+-xv8#v|{OxWu^n%rHH;X&G1CCuIxqbH8&CA88H?5y9i50Luyyg}n7HR=@CJJ$P;IcIC=cw5`7nHS^cwh_rF(WpD@ksB5hAM4!d
z{>MvYbCI_3g~T0GHk*O8zH{I^N-DO`mTzd=FBt#FojO8{oieH^6A$wI5lJwXKc^R|O_d%)snwKXWtm25}sI#RZ=;;_ZSbzn=3CBxXu#$J_*FdAd8V
zw!7SiU7Wvr($4~xM?Kq}y=Obke77Wn;S`>;&w(J!QJOR7N=zcXVS}2(X|-`$
zALHfS^pQNVH&wk3J8^G!qO(UWeNT*07ik8q4-V>SPbq_B$ak8
z5ce?e9V(@Y`3yHdfOBvYc^
zD{f{|@Bi#h|FRsuaJ6@tn+XGrksJ~vB=ZamalrlYGxEz2G%^#kDxZ$jF#*F!=%}$J
z`LQ8N0Ot0N5ipQ!+v)7fQ5zHhIEQia92w-e1ZX&K?`{bWSc)>E&vGcg+;@;k3CIQU
zygV>mrPQ#T=%v|Dkp8K8IZN$jH3G_-7T>|W#7EN60CcgFX_!f|hHsmk1C!vdeNBPu
zX2b84?!(|O=RhGiLS5O?L@_Y{nPlNWM9;FRzjlF#2j
zC$S9A%Dy^H^~^h{*R=yxudUkJ&w-D*cZ%LRfMJ==s%^4kZZ>WclK&`IkvpYw7e?Rg
z>ai~SyLF+?CaCL^N+l+oai|LO%*3d?uF*{CO2S{?SN;KXsP`hV6nvgn;(R`RP_$pz
zb{C9($=xQK;im!+-)SW-|{-w0|CPkC}gD$!{
zzh2-yCUn8cUw%bZ%D)*WqAznEUDo<>B61~7Wp=Bn%sih9oz6@hx{e>HW9iaYc@7b!
z68U4R$!q;u`3Q!;4h`ge+-R9&HBI|B=9&FkW}ukl*?8Oh<3#IuKI0?F>GMyHeqwO=vsphUZ4`p
zlM#~2@&^Si%3yV+nIga>Ss9EJX1Ng_4w;v41WxJff6A*9LWgb=h2m|IUEGA8Bgf2m{Vxe+${&?PZ2{6;UrSYoCueWS;e
zE`tY-p!t(`@y%JEfpr9ZS1(DSKotyLnxIQk1Wpv9M*KJcb!?YJbmA&36fIKv@`4r_
z(7X85UU^(N_rM&Q3Jj8CQ+0>>4ghO9=RJ1sC;eSJp~1#I*6{6D+6tsS(9D@0tlAdd
zMkT`@uhwaq4u$NN7P7c=FJ~t}P-GE!!Ion7Eb`=PMDJ?PJd4|IiCmRz^O1#`CXhf~
z)U%r)gNaA)pIG@=|L#~;3iIMn+K^auFMou0iIIK%d5g1$EK=sK@RjD^Z)~-d?($fa
zH?cNRh@O0xibTJZa!BuDbyZ@Q7n_@hJrO*gv1Zv0V8+{48zEZfPiE>Nrg6Y1uMI(*?~&U
z&=m$rUY3bvpTg}DC$nH4%B*}a2lG&Sr65^bW34u`P3sL_j+pGk
zB>I%>2ef+hX`-X#i=?k5Q}!E!l}e2=H)sa3Un>eh(aM9S4ZGfp<s~=TheD7L+DeyjgJyv#QOIi-n2R{eyp5ddcDbE4N%IOcbwb>|z
zlogT=g-*&=0lk3d&t4}k-o4W%*`{twd|JML)z`C<|AzQ}^xM5!kOFLqbSpLmREW%j
zQcjO00gC`-(eEyCQM>auBGi+h$tiH!yMM&SU9w`{=9LUsM8M|72dw%p?;`%(=dYmb
z`Vhx^_1LNqp+cV!DVTvAiD8^)wBN6TMGC&4IR_{+HL=2fxb}*UXcfe9^(hjaR(HoC)QM^3RNoF88^q!*lhb2IvD(rwus8F5u011P(kHiBvT*r_Ae+xZBTs_
zG9+T{6`>L31joQD?m*pUfw=PYsA3^s5m5ID-&7UmVo3{wfO#R?hH`U&uz^&E^|c0f
zhNVDjD!+O(cBM!IVVgzk9+9iSgBsWXZlK1cXQoyy<>o$hj2*bZQ}WZ
zK?A_g6QCpgc{)p=7!=0TJScu4f6#9W*z-4Mv%p^ofsh43LX*(y+KdUiczg|Mx}|;D
z{skN3Xc@<;mV~NY-TF8ga(Y&K=MB`x`JKxtF?7!Sl1}Ug<`7ttKKBZxEBZll@hS|r
zlF7|9v5lCVHv!m+R>>~Wp=;1o*hvH|Hj)=e0V|(oco-g?&-CC(o2{{hnw`DIyMpA8
zOz`44P%(ZE(2R<|2S?l#-M_!^Q!~AP>}|38`ZtRp-iyPuHPfEslJbawj^zLz1;=^f
z;KfxS1>?P7KrSOI=$sWmDmHXAonU081h~PXjqdPJ2MB5KT23%>{I9BAP2Yp%4R_to
zo$xO#5R_&DhI&jq%x$h&%)vlotzhk_o5m(4Sw>dz;P0_@(MUC0c(ot3gEa4w!XnSj
zB!DJczNxVtrX<3BrN4c>u7BEDpTTOVDZK=5mn6FGV41)fhfQ!PN=Fs}Vv_5Jx@mW6
z2J7dgsRPLMzj3+TH{NBf>u5Z!D0ViK-I7V4>fY~CmwNqU29iu$@_pFv0jf#uYjsy#
z+x5ETKSM5%rkeqZjHm!b#sb6Lso_Nmdjne!r_x`F$^P7dfom^4HaHVWE
zsxMm!zK(4BFWDFF8z*_!yLKvQ*q4{rmX_D$O-nwSG49#Y4-MHG^7d7_`uNvYom`5U
zXXdb1i-G0>A!m!AC;!nVtfh?wIQ}+&S93o8nCKh;q~P4vuid>SVI1XvWCCN{v)8c}pFJ
z`FCu%d^kH|V6+!^I4ik_`PLt=4x3Bki3%zRib`$L!K&WfmlYN%j$ibVUwV(j==iFy
zuDC|ba@;FvwwQW5Xu>GKOzk%8VQ1Sz=h&30e0nqTH|);zN#*y}ckg7M-hL`u`Z>$Z
z=ZB@gk7M7WbXrOHGX6F}!N8jXjE?HPg
zI6>7so-Wt1A?nThQX3I_Q<+5^HyfHg4d3W-#o7MYa4Nm~r)o+kHL;RW
zi=v5*zcb%zu$4?;Y%30GZ3TuHd7us>L112C8-3jwW|2@T^HHmKYHl#0C~N=|z_>9J
zlu%Kcj|#&MtO}doQSphdsD0OK7>+;a3%vFy_i_L1%|QM@{Xk96Z0f4b()OzUC#!I8
zSzXGFHWqaoA~x%aH-aog2I6DRoXWXPWzG`wZrdPS40KA^R6XiCES?tZ0;tE%6!?PH
z2|#1?tl3|awXNgC+6(+)>`R9xC;>2F!GSvpbZ3V71d;}Vl1p-kTU^Cr)NhRB={!#Y
zAIty_E>z)~OC~67k|of|nBK;~sVn*Xlq12?El%T
zRM!praA)=Oquu1DUDIvy&r|os{soKUv*bLQWVq>DU@V4^uwuXnd?T^IK!J1+&RZl4
zxJf{OM^9K_C;w-^{wL9D|0mHvjOT%XE{siGHjV~f%aKmnEZEHB3Tige-i`oFbfF43
zMo~sja>LOfBDFh)xXFN}T
zj#$M9wM&%?lC7A#u~Cgp4}#O)NNdzB%_gm+glL8C
zNo35+`h-LLNKgM%quQv^mjrERD?m{c1;pc7_R`<@o@l?1Q&wrr>7A6;B6>grST5x(
ztp%Pe4jUmHbXew9GLmcir@Bp?f6>&L`8(S_hkZnCR$jn
zMTeDR0`B+ey8{=mer3h%elFFyfW0Ot&K=3eBF=oOjH&?ZdP;Tx~_1P!g*YVcge
zDi;=_M#v|Dwjt&E@VuIh`(=_}^utHrWxldLo2mS_eh%D!x|~?~$!h>Cm;&(uCHlrf
z=fmICGbvKcJ{A6>Cd{U5UM~{O2XJ(K&i6L9JaG~fcElU!K#F5v)pzfe+ua(9!(qq}
zCH*c{lWvQ3>j_!6>!;G+^T(C#OqL$46WyKCh@-8yy2ihky`NDGegm$~&Rdo4N$Hb3
zGkm}I$uE`ne*9zNmiz2|7aBgYxO1RE^sB@3hfkcQ^*QI+!CV`Hi*%&QIH8{MvOWj8
zd-l47%G0j=i&Bg`IILhT!>JJ8XV(EKn)B#Ck|2s(sF`W
zPX{ybC1UZd&|nyZ7DgopE@(j@E|$2uO;8alj6oni>*?ES-XUm=f*5J20cueC0nb|%
z({zw4$O^m>l4KR^jH3Zr7hPqY6B+XgW9sVG&`{qHFbr3DJ18yC`C#~>i>yolXxyf3}8;Okbm#nxMav^p?%2|Xg=W+#$*m)y>#;q@Ndfq@?*^Nu2zBO2wWPJD~HhQjjqBYzASyJ}A
z#$K`&^_g#??{T}k0VF?Sf8AP}-=pYnOQRjHoHaVh
zFfYls(rRAG?w;avm%2`L6@b;g}2r;uNzijtwImjHV2h3J+Dsd9x(uYQFr~ER*(aU;h8ZwvWTUC`)a&uSG
zzav9k94^x(|0MG`-RsPnc3Yrm@BUp{DpLCk;a-hcYrZ+y(Kp@fqdeCP?YLjsxVIXsr~Q$D@HB8@!wB60=}Guqe#qcs+-3)tA_m?JZHvHOO6SFe0_OS_YN2q9g*MR
z#@e3=pQ{{^U?-l&nch2*tD@Y(zeb?+O3(O3sP_$h#>Bm5B~wQ-c?at4tIde|O#!Pm
z$=T9sPsd9a4vZ+Gf+BLZre+4e8NY|TCiiDoj-4L9$#o;D@bK@8%C+WE6)TS%TQ=$|
zj+nLCH*<3VJ@4*Y^*IVbWEuw69&7hSCf7Zn3ARJ5$KAh8^6Z&LysQa>M?i<`)n;aD
z*1M@I70b%ZzS;}EN9teLWXW04uKWy=FBA^2^~JvwjWqPCDr)xMxTogt5(q`UBkdYP
z%4%&WH+DXnfat9}&fC|wFQ*`DTbA;@35~3xIh)xTnocNJ)g*vZhhn>{Ik>T)s+BU=
zE$8Ip+dt7Ov`QO+hfe}r;=KMuMV{@hdF&4wJ^JhSHRS&Ns1&1xcUx6EqPyY#A1ti=
zN_@UQQTBeo8p09ZaqPby?5)?^v(xZxb|Gz=L&3yoM_euLN%9S|mTC0gObjgrBS%Ur
zi~QX8XqV}8z#{OCaZ&gD7nk_&yrqp!2(;E!
z8mn9k*HV)5^n`wjT$9c_x>3Ty0szcz*oiB4%X{xitoXRZad?pn@AfcG8t-nNn@3yuxC9V
zk|OC+wu7bpn?0z}LD0x#&}HByJ|m3skx7IYO{gJPL(+6=-iJpij}|7%SzC7g^_Pck
zHp}BC=e(XNoL>0=Tb=rB`-*=jyY<8O%3^XZ*2q>Jmc+*c!(};+EH|eu-Kxnpf2f(6
zvHLxXs>y0yJ;rwT_^q&?{=^zZaTFhURHO)~E|1D*bF>&cs$MDxj$oxz+HJ5%VIC|Smn>RRkyAe
z6X4?n&TOxI2Nn{;SU(Els1`m$+`f;iYCSXCeP0m>0&UW1bu+hcy(Yl8&q<4F!*>Aw_~s6=>pSh%juIfhBW^J&UY;P6#~a
zQ}jdBz&1Ohfu6iNsIb+*1>;{Ps2zktCqh!7Z5`hlH9e}!(sxN3Y>LK1({xP6$D#DNxS-h=XcXe!a8LEP67uIVP84208_{_$f2*
zF@Viw1%a9rns!Rg#u7{urWCo{v;1CJXtFyby%h^nuO>GbF;-C9_T9`lTPZ&%Tgvj}Ec2jfaWjZii1Tmb|6L
zR0gP+gS2hVSmutVJ9{801O3lbsk%##d>-!HvJv;DW5+(0d)XrQvm@}j_Vo2A*PWc@
z{kTp>R>kMs_8>)dB4j{P^5U=nGbcqs8Gy|KJwl!`&8h&h)P)BD=?wfNQVU~c#2_He
zOan1-g)$fHWaKK0xflXzjOoIyTui>FnUEH&=O&k_+}$Qj<}q!O0w+CxXLA}_*Q*ET
zZBn2po$0<-q?PVS3j$mc@6|0|2kn(N(rC}`V#$}h8W&U)-_I&?nX3gjtd*w!
zu`+$MKYV#^Ju<6UbpG&;>+F&)KYQ#vYH(!xV}olhmU)22r2HHRyVE7NQXZlGHf;H=
zs?}KD#$~~L$=Y5&{SYT_m8m=vbirJYQUKbR@-Y+H8*Z4AaPL9(GvS5RVd>YS)YmQ6
zeCI@ygDvL1b%>Oy747#fi
z7p=6ECnSil9UD6qI*BQmsz&3mTUEY6wTE~3sHtTJ?CIm#+cAk
z`}~d*QF)ug%f6cgTLZ7NV7G&Qk}EQeeaA1Q{N9Pbpb^knrN@Fgwxlu{vIjQV*`VCq$F|6v8JK-x^_&C=S<71zob8;C&t
zOnrOp_;m&tUzdV-huz7Kv6D`&jORB?h~=z!E{*U%y5_bbJKN7^8X85bD&I2tdhMmM
z|Dc~qhCDltk9)JOaPX}1>hCn0r6;#5+(|kc`Ua#R$v$_RL%xq$Mntb$VO~v)WskF#
zxJ@|_s6`orC(6&NUJgc0FGb)VQ`dMsWe`)TG)_~_E}F1pIWRApNqlgXQYAliWUFO~
z?HP?Ohl7?b4wb~geBA%S
z+%QzXLiiAcB_?+4xa%1)Cn9IAX}sPp83$Aj6#Q-Sl}dN&)@O}ZjkGuq4F5e)7EhVB
zwJ@FTdQ-x8RPJ00H*fb(uQfPb;run}W8wa?t=5TIAf}PR
z@0xt7R^96ZN#Je5vS>tW1Ei3ru_Hfc@~!YDsl;@%Fn<3txI^7S%;k)n4u*n
zr|H*5UUTR|s+aEIpJui-IXjzA=u?m$gT}-PeFHQjcSFFH2~&BnRk?x+v??|AH8QyC4iP)yk5(t=kBN^{Zt8$a
z4Ff!5>ev}luu%`WWe+~ta^YGy9wnKkEAcb&H)km;Z02G7W8+7}1HnWJqJ$?U1sx3K
zpglfz6kY{WSjRCZbLxZdIco22B*+^{ki>;HliG)vqv!R2S6*@QniTfs9MD`AA#mj&
z`o*e&_M#XfO`6o+r*WS|7}wVzNpxN2qV;70A0MH74Mca1mW9hjwk`_P6+eq>VI(M&
z{EW~J<`Uo9X|IzUW>XvDPS@d8h@1bkWE{z7OXI}c_Hl*TFzcq0QhY^jE`dfBG_VbA
znwkm&hTZ+Ru)I(IkPmA+v8^|w#AleSd@?DkMg=^tue{K`8c65;+AEpZhCyjDwfN9g
z)Di9ckJ;_er`ZP98f?-*n-Y@`uvLD~&Vkn(UxGd^e3emfc|^I9T=aSRNV$B3V!Xk*oj~a%X7%7gc$vljGmx0WWw8*?(q7HU_jd0O)9F
z3jVV_V0bl=HxYf|Xe0|j$8%wL$U%?uAB(U@Jz8MMRT8824l5v+a0$925h0)rnAk6q
zRy44z*9LYM@4^@y-USKD|a_)zYy;
zj5;{1zzAt`R{71ps(*&*{maH+?Ifd+z=_?xDw#72l2>J#jqFT6f=1B(S5$=|OHnhI
z1pJQ8-pm5Qw{^cC{k!Q{efY@Ikd4OC;M=2@K6gDmAzz;T9QI!}xYOnqv3&Ev7pmfl
zlJxr68Ip$IRZQ6}OfqKHQzP4((Vw*y%1Py!7hB>`aQ$
zA#+vYVcihvFScX5{O1Q9k1L4F&!(1ToUaBvv-~N-9l+;nGI=}EYV4|dUe%ZIyp2rX
zfk~p8M(Ma^tRRIn?PJlpI}0qQ%B7EKoyEIHV*dWr8*(ExaiYjQy76F#e@dC$jGWhX
zFL%JZ#3r34&2v?!GHYh^~6(G
z9rf0wX8vaE&|Hd`HXme;Qo1#tM(uyZF|Zj+y4Lk5@0N9i09<2ei7_Ps6Gm
ztpXu8GU=#_xb#~53jchG$&Zgunh)%Lb?QHGui89u^0tMI*AMd{ch)`VJC0ro3gNNh
z@n|6e&22P6q5r3-#MS6`$|yJfZ#adTU$i-eE#^rdE3?#qkMi)75*1n7;nhahP|exi
z?8hN=G4wATuUn@D*R-uvjYTco`T3}y^Byt{az)qen0Rn-aB>c)mvWZ4)0o|^&@C-<
zD0(s2>|4LaH}Xc}1#-XHp7hkVzn)9L_N(~8i2vr(C8LkK4$nBupP3O~W%7sDox)Z`
zuXERU&ayd*;-|L6riP5Qtz6n~h|E!Dzs`z>=6%u|cWPY9BQpqw8blNZSR|-pQ{BKO
z2!>HLkv$l@B9IP8jZi@F#{dGE>PD^R3Zn`yr_YS;<#`xjD(`5Y2j=yirq|ltHlxRc
zH2QCB>3R9+dAJ++aA1Xjc8D>GWf|fvrRz=`cH8xOcT8%9s4Z8F-PQ<)08SPY2fn~+_cOOqq&_~
z*m&48qBpQeCbZj5&g3es0ywmB6Dg5PZY^~q?KfY9it62&bD+g?z52jbZh$&@x)m~}{5&+P3MaENuX
zIi-*+;dI-b7b)hFJ|f*s!o6o4Dwp0`&s-6yB`M0z@Z+4a!0hf;x4e-o=X2VveO=Fy
zGD-1EgeFAO{?~}J7aiR}g8Wb2&*tpv!J79kW$*D51@`!H4k1Au;YK^mI^_#htj9#9
zAl)Oeg5YnNLZ1G}v^Q~N^Xh@SwE17t%G2GHmN4-qXU8VDpB`6W|RIhYga&%Pp6DX{KzUr$(uJIG5a|m;%rn{!rc;v
zuyYHEJMJNJ33_5I*d+$?OM1T!3TKWzIASE;H$I4tUheNX2cRuQaZB~3?=(j@{O;U2pfWod1U}m?6QEr=aQ_uMFNfKO_Z}5V2d6!q
z`SN|L-_YYuF*CgdLPFqJ!kr`{)pjpnJm|Azgior)=vdsr8##{Spsg1leupUiK7*sk
zvv=5QtGZFko@amG`F*K1Bw)E;EO|9ezWc%!rI@iMPFFFSVz5L|;QdxXBK$Rv-~#l5%{cXw@Zm*8HsSSa46xVuAeCs=WJf_rg?o_zng${r(oXXI+__0ILo
zxfE5{Zi{a?Z5wg4v&fawAjl3c^w&Ypnp{&Y?Z)QM0)JDXe8?r`5L0LNdJWOM_Rx{y
zekpgr+nU7A0QS6NlcZt;DN!kl?r&F56yOI$Fyhd0Ty~W(Yf(k$3b08@Mi(kS*l2Oj0$#5i%b!CaZj~P3iKp@Ky-p5`&8w9&P7=m8Oja}CF#q_QO>k+7u0Ee
zN9}v{OD%`jYTEnGOrP3ZOsKuPCTe>Pd<*3Seq|Cefh7!>z}x}MT38DyiFWZR9y;_B
z64ieQ1Nwm(nd5q%
zAQ|mCEFmX-2N0rRqnL)XF7PoaV_N+5>e@71y~qLjGq!ZRd9xa*iGWS{!!_lO9~D&t
zf!!Gymnds^=L_2qJwp#G9Bbow$fbqPiq+4H6EkvLPnY2KJh(pc?mHA0?^~;syaK8*
zPmL-Y1A?=d9i{?&REv)T(Mk@dIhvJ*bcdP5RsHnBkY!lK9ZAEQmZ`KQGa-Atd
z-{4qI3+1{?<;t`HC5%FT_i7XUZ3QSGQv87NjZysAd;>-&kepxu_$brt%5qI{ed`tG
zFn=4Um-6GA$gD*#H+MTpL6ig&oiL0T+n2-j2f++t_ZRuSTr#k^&s%eMb>V!(ye3x5
zZ2*CUmHci58~LrC%=Zs(kdH~r2VZbHsh?N0{sGWNnUSBG
zD!7^@pxZxm>0B-~>JEB`E!1?ZxnP)1JK5yo;P~`G5@i>*)JU8$OvkedZmuO3HvM4>
zwwmTAL|=_Wl99ix>5@g;k>u;
zbs_Jmw3B&82uQ><5aXv`+n7*Fb_%VyzfbOp=qqAxFiRy`Ibm*{6~~gN)SP&lqLiu*
z#cdsRWaNx{vn)+;r+%|vb1QFH=y8Mm`C3x+LxFtoyqW#wh_5MOenaK3L#O65*6RdG
zafepDjrgI8d?F+OhHJUT6@+b|U=!(l9W66u|MZf5cB8=j-
z6Rhlc9+>iEckOP(4Gh?(Rt^{nOJeZ)NJ`^c{(xqpVd6`&MF*SzqdGIrlyAL)A
zNC1udAdsHXsKLO8vs$lmPpA1ZA(lYleXu5?;Z*8WoRI(Yi0p*>?HO5&?(12G^P)&L
zbA||u+?Zg6hVjCmb`Yecj?UL&pX>GEia@oQMyRQY8E@Yt8c1M}N
zYZlyZ;%vxP4w_77frg)=lKF
znO-?x>CJcUlM5JWGr(ZyBw|$5Z>a@^tE4I#b*r76f_{6-2w1A#uFREz{j}x<&w`!Ylc&10^V?Omf)BQjM;N)P;xjECu
zka;w;1Rjb$nAxwt`1xhVs~S6Tpf)`4nuFArxt{a-Q0Sq6@!=w`PU0ue5FqvRSAzQrB%U49r|
z_h)ly-cGMxNo?ZNGxULte!+7xrupH%9k`wiSIL?BX`#b_q<%OJlOdSn3e{xzY`jVX
zCnUFo3){c|tyYF+c)$EnNmBkToQTF>gQ(CUAdrr0lB8qqqbhl=354JZ*f3!#pi)d{
z$-XiACMYrR10VsLwW-}jJ}4^is)xroCwc`{%aM26?eUl*l}i!FpPW}
zcLRPc^Gn4*K}v#_&vCxa#F0-d*l`Vs;l(^C9}k#mu%78CCgGj}H5p5NFHCpMZtNq@
zI802IJm@>$zr$Z|zC#l_;gLo4_FXVRD=-47ex*fILgzog(Q@^@svcu4@f0}O_XF=2
zM86MwmVz+IV-Y0D2wpueC9vSqfTF!aa2~)xjviVF@4x;2>%<$`zVX3d0sLSG&E
`5c^TkOH3GQ8&??ZJkl=mMSqM2vH$0r8*XT`6PZK_C3^}=b1y=|DB4>r{blwE;Q;~Xl=f4zD9XDAD%xw_*Vre$pw;rEqcX}A5nnxRl#(^_1p
zm0$(40hQp}y&9(XivaI|?e}KB_
zHoXLCaNPID{DGGmy>fhl6Z8nzW_b&vF6Cgd`pGd9;h=|iCR$%gQ{tK=UnfK^AeY9(
z>Wl1#U+)ae+a49Kjttf=3Zk$anCYIRP&GZe
zDI8m;7n`@QIksarsHt3fy>u{0V!1t`h-JXxjEGD-u-wSaLloxCFpC_7uk6
zh!=c=Ep&8woAqN$EQ1vV_DNYx84a{-P5x};qUCT$;THcN|m
zjV;m!3p_Hn^;XFx?Tgu|PZ{Vo-fER$V#oHP8TuDa8nA56N*o^&H3{CU+Fwh;5UHNSGBFz%)xtt(n#^BRKf
zh;#&A-J97h1OX*i2HF@l30UX53>^Bm79UBH&{1&qGQ%n8z_AFbh)Kl!b#~4hyhsIB
zv@t-}G&${Qo(EdGBxL$9xJH_?wVDeA4kP~e%XE1Tzr*RjeBh;zqhUJ&%f0`$@@j{v
z6=jK(U*{0X9TI0nLtxZ-fFljBb4KC+QGBGTw5$bO>;PJt{UkeQ-&~O&Tnz1#xEV$%
z*Z>+NT=#HDwszkb4)y_pToU3iUw2T6D_1xm51p77#@D@_^adOE&Aoyw6rZzz_bihf
z-PZPF9=(-OWJXr(?1g1SVkFiB8)r0c#2_51shf|d3+j}kPy4NX!y0?h5`Qmwnq_bv
zN(PL$!TymnELC`VZlpwx$h2sbutjf~ypxgCmcbGwWx`K7l4Qm+?JwNJb&)XxGPr~?
z-~cNV_@A-%yb
zQim|$QZD%pYikET>~fTFjiQPn?-{ww&x+kyZ?;4J@27ps|trupl|oWi;5a7H))U|kBl|u
zNC!--CyrHv(kJeU!%(*Gah|$)F2UftwT#u1pY!LL(~d2&QL_BH?AUqx6l$Urpq<6Emy!wVKY90DJX!FjT^
zDdKuc=C!im0@Eouv6j)*U#{uhzLm(;iEpqa|f`BpvL{
z=nT#wo1KS@EMA{>63xUMk}Swy`k-xuBR*f^9PUyT7JX|-XclY^kZG63=|@8h9y7gR
zvfQbbnvh$})NYo0EEalhU+sGFTf1&uZ*N-y)vg$ssc(M@ED2#pypP@y|J;Ng%h%~6
zn$|s(flPMG|2E+1MHhLD!PpaFRzt{?>w;h4t;g`;mG0SNo>DNOu+z+-MjY=53d$c`
zHT-h%m`iQ!zxCXkenA+57nq7G_NwYGzlUXwtD9ys4E>j)0s^=ds3bjHJe&tVW21uz
zVgrCb9`@-w_1q|~7+af+5~4lcuQ>s!Qj-G`Z9G#E1CHj6^G5*{i|^^@W!ad
z_HtikJ3jmi)2p_d!s(f_svt8?T+KFyZ|m|JVm3iyT1R-}7~NaEKZIoqlD*q?qeuEiSKNkK2Fp@QBt)X;$X~$_bHg_FWGJz0Jxt~~A(Rk}1%yv#vrBbW%xvW=;
zch219-nDs32)bZEHYXBxT3j&>v*oIFH;g(yRb0#IXhT;zIvKU~?JWhpd&RNUTy{FW
zI~S`AoiFshw#_K0>B+FJkLVofA~Y~f3EcO
z4fG2{x(izfo|jhs157%Q|9ve#JmQo?Wd6;Xb7Q`rF!4|X`5K7TcIu>Y;y{l4({ej>
z!VbhJz2NSJOd&V)182q%&Faix@=#Xk$6`=|3YFpgOQn|-cruApK7mSoe?K{Uv)ejY0
z)b6RaDEeQ9#Up0m?-6qb7_3parBfMCa@rhKPY(>lr@ALH6n;+US_#hWKm&8FU7JZ&XeF6!!M_ZA$QHZ@3vd>O}>`o{hGZ|
zC)u&#lsl^Txk$;-TBEqB8We1nRcKru57u%91QAA5z%5G$I%&^u1av!G^Ppb3Q0rBL
zlgXpf8hI!Yw{4AUQ|qHM#YyAi&kzRVUo(TkTrNlm2ysFZVp-3)kRh(vJT;_8RQKP7Z!$ncpMES39Qr;+K!8xdea*l>zlON5}
zQnDx|CCtgW87Y
z`L)~%jU!P>6~`n}27&fdX|+t3My~IC`=;HcW-Aev-wM7r6jVjD6j_L5`eK
zU_%*ghD`!Z|c{9vXI3mW^fxzw)8AkeVT)~5Khw&0xEddF|KFG~La
z`;eWVpXvao&#Y%B0>=-IV4QcfzqYYmse9#U<6E3Y(-|r!NKV>{z(q4b1-fG;Eap$E
zM2D1W%EOXa!0!$jm+ScyG>H3jXO&ZzD@O|7m3w=}}bjH-BGKPaNFk@^R=#qq-oex6d}!1O!@I#TC6uHoae#0|V-kj-(#
z+*Z@%pwgZ0j4Ul3Qid>fHgF%@m#tmXS}vAwQd}$8QP7tx;{H)q*!g#h&K!GeVV6m=
zK&DE4lsGz6MXmb;w>8x|uDYtSw79W+AU44&Wf~8*mVbqXF&|jR
zrEX}AJNSr|D}a4$P~4n^bSs5C>=PrL@#e&z#Qna%Uas-Dz3jpKgoK(-j_f2adkdDi
z<&forY#T3wC*2~PKO{pu65nlm(OZChH{)2;6L#{(er}s6;Wg^V7cz4Vu@=@X=(ufh
zA~#kFaQJplSQ+o+Rc4xCCy2pa(M4e?8`2Q#z9J*lWyI6OGh%A*PQO5|7oE`|i^_Tw
z{^(V~ID{n2S6q>Om$HSuoaUukmJhJ46%bEiRzKczZ&xuxd1-l?L#9m^9hqoi2%_w>
z9Kb>(G9dNOX#C13u|w%CI}ViY-Z5zG=v>6t*qC1_4IFf5vSKuU$Np-X2i~_W!YRr1
zhAfc$QFO{Z>-as6j_1eErad{vh2TMrCAR2_R8cG<>c>un7@HjZ0~ljPYUckt#
zO|sb7YidWemG$e@)6vXj%lF$W(JQ<@0PNs9rtPp~1EWGO_vUPZ!J-+pr4BJt%aQz{
zQfB+)z2iAXmw$k!TefC@!;7J~aQ#J=RIk=pe}gmRmMHVRjBneXMp58{UPZ;JOHr=`
zu~?SAlCZdRn^_HK+_a*^VFdDWjp31qW@lWyIo^Z_#7ORUS##e3#NR6
zDXhhc#+nerTE+{O2B50|%lpG$OEcGe(6j}=oa!=j5A_j>6Od_8#%t_>ohK#Sy6vSz
z-L&WCc7aXdJ;BP6>~u#(@n-i&N!7c7)0aKjhAUe)n7GUaY*m#^UO-i3v}`?=%f=^J
zsio=YSf%&Wvw7V~ove)b8wUYhY!rYuvn?a_54#!0>P#()D+G(Jy6TavaAuhqaXTFg
z%p~`Y-o=rv$S_#t#cX2LCPCGn7z^gx|Bf0ZuctJ{Q!7orHQ8xlaH#a`WJ_RpB(>K5
z1+ny6#i+EpX}^E<`5nK{x)82zJBsIo-N4Ly%Z9br!$Ox7!^jEEdzO#Zj#Hw=0)Fus{Jmh9-xRFoFJ?Xp+&RZzz!JkE{!Nev{0*{r%#mj}aMu+%z
zEB8;80czri+^F~#$6+uULHx@jYFlbWhU-2<`H%H%pugZY$IT1=kTZRgp~UPG<1QqC
zuXjT2T2)rC#xchnmYY@D8ks+0^LH^k!*v2wCdI=ZDZ83==?*
zQuLmJ?2leVbW}}|dYVRuVYX32bX86(H`%jKC+%(jX(3}9bfnLGDx
z7Sw(NAj0xBzmVi@L^OmG)7*x|wn2uX5szC1Y|PU-nB-BhZO`pqG+te3C$k)x243za
zTMVH;A^jGXEvVypR|Ek{a}eNwN*iG7*G_gguobU6co5=U#YQn9-n3YW7%mN~3p5x*
zete_)079NtRIfEiz+-Ip^C5$jB52$FY7&s&*WjlBOY?jxQs4E`)U4^t0`l$@lG*WHJx;;WZX$(%wmmv@2a!8<#Sr4@oIDBq)l!2
zAIv{%G>~6FI>!%*;Up^`=~)}xQBgON>`;w`-u|T3myp#m=BhIwLBl87Oq10zc~Qsv^EO1o
zMKf(P3pR#;hA9;(&ifWP;^wR_*s_yI%T6(w7{0}ci#G=N*od$P-l>CITVj1_X*olE
zKhj{r13XV#@r6I&H65XU?{CS!`Fy4TdV7MKwhLelt9vt(>Mmp>PR`i0zCA^tKu6dI
z`hrRsIy@$^_qVs4J9X8k_s!4mxxiMI#A)HWA$6nsp?h6>7Fmft?X6e>a=yLKdzrn%
z%g#Ew;DhpmolP(?3h%WTPK0&QszGvZZ9Okkw?fb=Qqk38{_qi$p&F)l%+Rgz^m%$)
zkOYT&Pvj2DbN!-=y?+O{Tq&2dwCp)#WY~N+ZwuG~h2`igaeA7JfKv;?=tD4BUm?v+
zfuBb{uKsn7q3IS1&T^Qv_ZhGMzO^^EvsalevrxX$LCnNY{K3*AsH}YYv;M&*{>&e#
zY+_=xEySOq7tf)YMn^~!T3SL2N>U8GnwyHoXN>J$yCFpAzhFILiCjs34Cb2%RGDmO)on~tSKQas
z7O1WC9k!^=qPSTRcwjHi^W=MlIS{xlm6yGq2#DV6W2l)c&}~79*>65S0Xx@>Y{6?9
z(#wJvv5YR55gY4tIq>Mj>s1|BwQJiB5k28DkB4dph|yi=x@<faoBLKVHKazwODP*n8aKYsG
zcZ}_Xz_MS_A(o(mjLL_~F5XWuDb^pdZ>tX)k2{=93tJU@Dt8oc>+{K{%s^sfduK)@
z_YS{q0>$g!E*2#jFdt=^i3~=ci7G-JWG&=2f^VGko*CH+W;}Yk9e!f)I^Dx{bt0YM
zUUv_hq$RMh?}x^Zm>(Lofyi*#2a=~1{6E3GKF=YzOiSnkO$$ZuPKxJlNPyDE(Nfs6(kBxUo*)muakNu$+)X#|l;y7GQ7PX=
z@l>_=>Uiua>;||R^bf$xk%FtFDij|TpK9RgTo^qV`k)C*=)JRKRffEIJ}V~`7y%Cw
zl!4$@_mW}$#IY5xwT_jeu}CanxxQC36t;8`TIYE4<~ev?OA{X3YErHdn+G9gApIJ4
z;?Ot1_v0|}#pUkPL1SM5^`T!-{GfRETvFo(8-GmX$+EaL>d_V71E`Cqlv*
z=4-%0<&_~UP056~hd~foe)Xhz$0l~D2g5Ymu#@DRErEO6V({FiD>$!jC9k4cS4oG*
z_^^z!=@b3^Nz3`nZst8G@y40<)Ejxqx-UKaD@dUuo7LBh#e^f(e?hzSy6BF7G$*v~
zlM%JX>9G3kc}Nf|Zxlr!g5KA>{D>SGW$5_{1}nolX{PA6)123uhBG+Q#=jS)byu?b53pAI#}=XEJHYb!cJYI@`p9B_%z{9EU!)Yt
z^g6}g5LtxdI^>x#iQ#>ag1vJ1(4IitKH~X(_agNef!hSID8@$2P_{XmjKt7XqK9;d
zCZN;k3Y{Dd^~srR_ivsHDD@XTcG|Ml>yDi;3xxb^cZ2WNCruu%sHHz0
zGUWx#-!}ycLnio90FqY|)G~^Pen~UE!yng}fbI*Nuwsg<*`W4G1L~kOh4P~kJ4_u+
z*zGPo{%R_8<{}e^08SeDff~=5;@k4&V4T?RRw+F--ysmeoKa1c0Xszzw
zD_B+t5_n9!h|BOYKy^yN>pKmc=f^Z}8ABQz*TrWpw_@v8aBI*OfS$|a10)ZCe-e`F
zb;{XlhNNA!)2IwwINp|1i6q8oG`XF-o#BdY2!)u@f#MnK(xOithu
zznvdt2eLgpN>DueVDk<(s#7-lTOvJt=RuFVp_BO9By9~801MGvdm)xeo*I*+clJc`
zB-A+Up#i+;au-i$x~UF|-wGGf707+hKBxxCi|uDk^m+RH1AOAY?phubrqre?=g}$f
z^8XSa|my?DwR7xU=Y*m~%{bk@i^c$Vt|GbI`d5F$VxgU3Zpo`%ylx9W&Hy*53I2f3o{rUu+P5J)RuKU*!^OMq;2+j
zDbVuzZq84@W1;d^f3u244(T>YjJR?2n-8_he9IBXQlI9bA-}y7qXt#MOdXE9P#H)3
zAM1_ie8#e{1P{7AKRr*#OHK!K#dWZ@HqI#$m?LOH`y4T2em~r1kZyaY)3$x-OepAB
z&$3NRD61|;Pl{J{ZvoqWFM0MZ@LlQ0MusdsDI=Gk@{{g7ASdszp@?hZYOvbHcrVwPq~qP2E7wHbSQ9cQzixgibaKVG__z9{^_
ze}LhD>CRtIiit&m@2;<8HK6k{ykG4|+?B`->7DuaGvcr2cs~S;OhtYJ#rU`vRsu7~
z=93h<0nUn#7}H%6PP)HYI@E4j3)3>il{dV|n7-rdcnzKSA+%gfVa^3oUgMu0(Rk%`
z&IUoJucJUkonLyXrvC7VygCRUWmoIAsg%B^`B5ser3txyEpOqsvmb(+_Xks}AE%CQ
z7G|ORN{8R6EvK%7+ahCVyKQV2iMNY{e2sJC9Gzh1-KeTf5}0nNJ%SA5|BcpC>A&EI
z3gw_em(;ycFPaDtTED+RCWzeJRplR+B~!(lL`CfV08gkD{LmP~_QU-xJc6IkO0XCevhx!zwuL1c{USKMZ($w7mL(E`ZwN^$S_~~u=GjU8P1w=x
zT3N3;IlES#lT&ENt+EV)bQgF9jA?q#xewApvW3h~j~vk$+4g@Oz#V_E^AtoBt=r
zT)@T!6?KjG?mRqUmgejCqi~QFS6?2AV;(PSTvFP3#%W#JXJn?8AY0VMBEb?IDLh~=bk?t#!;+LUhO3#0=-$HQ^g_&8wz&!4!|ybOpGq4VcEmlPi9w-J3FK^`
zD3v+gY-do6!8)?i*Dr2}oo`BQEyQpo%yj&$lx%aS=C7m|`-;9R33rovL}jTNyFap5
zW5ABTTe-ULDi|F25TpJB=>EBAaWAO63v{s9Rfu96%T7Q46p
zy;yCu%;Dq1(~?D@sCFRFGGIgq6t}*H$gVg!7G=$0S+P=Teyq2U7N8R9lv$9
z!|p+$0mU{fzi79`v+<2Lg(@t`B2u$DcY7x{ZHOaz_s`zMn)Z(L~az_u+Fy<3oiwH_6Y+i87h!W7!zgZ?cFgVCr4z@f{WU8
zOf&r|EG!V(F)e(#@&!nb|74HZrwhStfqFbz$rN=X`nT~Xt^z6Nma{uqncCzY9hkSF
zabo9>!@$#N=zjCIRe0Lw(Q%c4%^NG2%S
z{BvCnU2}L8#QE@Z=iv+C)u%_H5Q!dx7`rA?7WMpc7@cSB;*#ieD@GuXQo%1+4w4A4
z*h1;PavXSY$#E>hBF}z_7*cZHPk*iu@u_(&P{Fd$S5z!VzN5s!~92Xc+
z!QM1Gy=Wp==+Ou)EzGmFw~nQLm%!=AhMwCvcVpdpX!!Vx!rl0e_}xRbT;($z=Fu)s8=M5QcbmC9goL=?v3R3=~Cb3+u)m(1mzxu=Pt0iAW;j>B>
zPE#!Ezs<=oa7%C#PPG>MqMZP47?&vFzUQ7b5wl;F`&tUsDFOWh@F6=vM{BPhS$X3{d?bxIdkakhT)@GUUfoEoVkOqnZ0#w)>63(09I~c^uQSipm>-wg2k|-|Ru7r&_
zOxUT*2g^fMdWzNZtGm)o2c@`ctxIvGwB+<8i)(ru7qVC6CYxAiP5~Nsn>JPSH{7%Qp*pw#`eb_&!v|AREo#rmh1^GwAR?ZP1`>)J}
zsQ1PT9aC$={XTiy#}X+bVyezB9M#!;rzR{%U;paaE0%X)Fio_N_=@`&4RaVQv@us-Xb$)tkxM(>I;_oJf22`52T=ioTYW7pV4C3PaVabiR
z4psM<>RGtRl&fpkV!d%Qpvz=pr1q>b^JM#5JQ!5o%3q#8PDgshQAle4$-Xs$}KZ5CG@{X
z8oXL^-yTNW%Z{La59z;gcsYIcewkXxzcHJ^$X6!*2T)#yCd7jHp9RlROh#%cM^82a
zTxvxr=jZ=o7gkq{aJXs4BVq9OSjJN1
z)Ki*lcYb`q6NSa#)p$6Id>q&lpRDqjYEe2KMEZe3749EmC(-ZaG!&;__GKu4FgjIdwFxjylGH_<N+^xMm<3?ZKY*R;{2O6hiZzbq-Z=x}7v}5GKJDj!
zfEyn$8s&Xvv&c-7jd?uLrR$Pex!;W!_;ghI$&$R+90~0zq90fILtV2raGd5kAa8-6CrGj#~#`zzJF-P9mr5uY}&&z!fr-R4N(-1mW3GY
zG9@cm$&OXz*AWP%`cpOPNuJGJ+p!)tzYX2UES5a@&nCz?lcCHDn}B9;+8o;E2SlqR
z+(oN6ji*(Yoc{s1M)Q`$79G=`j*7CTn#c^wjw;ijl9a5h?B0JVFnqmtuh#;7&2k%F
zpjl%oTB>&#NXnkt*c_wztUIfjqi)|o-Cl;&rRp<&E(RVqEXoH$YbxbciA|_`czewH
z%K}FISM}u0yBE}`c+iIGB16QJEA!$ToOuKDimr|#rNGQ8Y_=e76ttyrlKnUwMytLK
zts8&6$38z#xOba`ZK~}BdyVvEmhCX>>`ZacAvUT+2zsYg&UZ~r%DgH8EGiCg>*fi8
zhH_2~0UBY$A5cO7=2$6Iwm-6LB|cx6YR)d=7Up}NIDf~Ql?Yi)*@IYNId5Cu)?5B&
zNk)o0%UEn9
z0*V+b?(#2#GS(lqi>0UFIc!a=ghq+l#nz*_tdTGNky)R~)$s6Dx4f{>Uz_HzX8oea
zioDov_nrvGD4G^i-NS^`B(+y55Lws_Vp#VD#Du(_(-M_8>}gha5|jK>Q-hP7O^rBq
zb|r%tMd~^emx4L&dES=;FmnQR7srW2M*Ily6@6Jg&gW37=%q$f{ElV*b4g*tLaMGX
zd|CWXS70ix+{?X|50Sp2;dERF?dve}&UhrhEVurg9XZU>Dgk(@WBCX*T>6##m5^|84V04Z`G}-~&;bLOVWy{W5C%;*F9E
zY!m9vLzQ&J4CyfHdpjFu#Zv4@PpS&@@|@_W>hFLxyQ>nxM8UXcop$Y-f$MfTu6?k?
zDG_WCe3|J_qvj~g(Y}H-L+>3=foi0&-WF3`aJp>!f=)|xe#YlbZitg$Rdt#hAtjP4
zZpS^B{Q8H#D|LG}@1E2aZ$6KfZ_|$XA{Enw4Vw>weG11=l=tm!3_R-h`xuG$n)ti4
zBCa>`0xyf>lcecG+}^C?+_}}P+3MW11W6jrLt4mh!y3$%podI2Yr8PP!HFI27Axu9
zv(r70D#y-2w
zZj&ExtcZ3F#*WP%x|9u0;_d!4|9TMD$ey#8d=?--sejFx|5UH5geVZ$`mvu4a49Oc
z*LcUY)4qVhh{#8apdnLYHQ$sfNj@TR>a$X_URVn6yZ$?k63-m>?Gftlzb}dW-o9l7wE0jfbS*B5@wCw%BEBRr^A0nXq
zUDJ^fA746BrV&RPTnW0_lO=USygV9K=1{@?HU7
zJnn;v0_e;VKFJ$}wFU>ZaU1>-^!U|Ik+|!20?(NHwc&|0vcXoBN4mQ3`KxzJma)K^
zb%V_OFi~>TpoHO&f$|?-Oe)p@gR6}FalJR~T5C?9hK_E;@;s`Nk;#aY6i$B#zUWfL
zYS@1yd-z!WZW`r`r0@LIIK6&LY|dhKA0z`R)pJ^M+6Rq5eiz2pLf$z6?wIe12mKRsO
zcHC>$OHDa$9#J$THT<#+-R#Rh@C#?rdJR&%mf8hY4umPCzrAo_
zx3}+F1#c`Yj9UB}5J=dr-x<$0-k78C3CQB
zP_E1z=d4Q?L_qeb*G3v*1cOM>#^R&TyE@C|lVdBzm9tvI
zguO&E3dj@(V70%-zLqzLq=UmY_8&`+>>_x!5#mW~PzOe$
zXPfA-*f8=_SNuUD5&=!C&+{U!0Wk_+je-Uzn>iXYI2dt;n)_So+V&cq9qP188K#>Q
zp4LvvjORyrX}eoVDamVBDL%qwItJBHL6X5?K@m+Vy5z2fk-FXKm=)lT!KKVtGqdiY
zFN^1+y1`yfn`6n@w?Sq*5^m*;vwV?RZ;jc~0*m4I#$M{awBw3N-kP!ooawD4<0Eo>
z?<2`*htYrd(Ma-xIbGHH!Sn!!(b1tdMq>(qD?ly^5?5S6wWC|@g#VC(;Z%~s4nP9K
zL!<3C!XtafL2;hY@$4^)(#U-iiy7@DkGa>Ud~;Iz?@(7Yo|S;qrbmZcpZZ0|uFKTp
zMn|vT6qxU%ToRwTXFUA(F+%_w5q|*8nw(BI0{|bU8RCgj(HM8Pb2+q3?UWSwk%x*
zkWm$dnT9(PKJS8eQX*=sKZhs6@!dk#;E)q^?REzYn|jo_RjNPK-?9?z)03yY^(HHL&lkvmyadhMP9L|5V6g))mv&(Olvf+
z7oH|>vy47$L^_(vWM#jwN)IO~X5RI}>O9uXdvY!|HkS{Zmn18Q)Hxboy%!%1{{afb
zMhS7)W{G0glRVx=q2+lBSj$rr2}F|)iWl=mP}`V&=Hq`1O1()V$w`0FcJJiui*EuB
ziS+z!?OqC+zl$l;T?tg0CR$TaoZlQi>5S7$%RxNapoaXPf@mt(b)pT7!4
zug*h+x5IIwb`wF0Y^h3H0c!;QEL_megnxir(@DY$;U~ePG1kkD;uXuk(FIh$06VXe
z7VP4dY=H`>Ni$xJ#?d0+D-zflMk}0{A~b5!_&8c8VCm*qF^JoHa3@|rSa$0=dCI|H
zMg+W*ix%*O*MT;Xv<}xvaOZ@u9UMP~VMf4d-u<9_Ix)dA%j41{r>Y{PAD2s_KXy{h
zr#ne032Ivs{^vR{lw8G|68v$dqw)*hO`ESIK_n%PT8Ef^Q{yGV+6AGq-pY=`UmRP}
zqw9|ht$+44BP6?xlJ9B>UJlf=2fPvqstt3Q7PbdX<xNN78anSpS-YmV3tn-{l2wjNMoyiVd%EwH8d{`)OgYrG*w>a
z%GC(>bFO};$4i2p)mBn;3Zm`28vrSh-3PvIPnoCiUUacM{(S~-|4vG!)7?c%1KoJc
z>xD_VdUmdqIaKo2$nlA|kHEBeyg1k_ENmKS02$}C9g2giwhi&1v~4}FX5Be3%c&?x
zfbDe^l%`N>rintNB#CLuF4%n^C1Vps?{sb*FRPouti@S5f9y1UOIppd&dlzvWpt+f
zsZ5_K|4UY0UR7eFTUd5Da<@`O@EQ%vf(jSbg1&=N_*PxfhJ982N-0fY0O!y)TV9fK
z7?T6pTA}Z8@~(X@(0YP4Dv|j_IvEV5neggE{MfrhEWWiD4sAEOZPuRbJtABsTq_Ze41!yTQ
z6)RDTCzY}gY1*C>GW`Twf0u_2O~ud2LEP)`Ay%ro+hW?ud@`w2;z=4LwE8|!MWxV0
zcU)O?Be+jHNrMts`P~ZvUNh>@tGrMFG(;4CV~+Py(VI6C;j>1Ttr}-CN!=&Kvy96W
z*#T*Dyd-r&o>s((1Gq>Jpn)zai{fUX3peY6uSaoh}@rsr6`Ccru>Qm`8*;IRicRAEYxJU)m)Lng@SahZ)ee?
zOiDmj#4X*eQC3Z@*rIqL$E%qgkg;m?sHs3tWCXdy
z9pfLf=hWw$q73->oTyRY;X_H{QHbIH0Akr-m8F1}XRh?Ljj3;RNG{raQPa#dN3-`DlN+DX7O>;;l~lwQqsUR4poZ>`e1paJ9e*NVT*>B-?*
zO)0{Cas{^xvJ1G~Ck=E3u3Wt@^2`x@c_%?tfO2&jqo8pWZn|aTaXfGsYOUlq2C+gqWZAmhEZ5?F!R}fS
zzxGjXH9TEhDW|h`d}O#gRJYcPHYWIl*-_|Bidj)HRoK7uc52xd1|PO5f~sh;Jk9-l
z__mE|HBfd|)bKC6-@2ydIXYfT=U97b%i%DKI_z&k$Y0&vOb|U(>?P^dwg6-Ig8F!7
zlG{)ThZ|olsJx=kb@Fu@CcLMM<{R=f)_=6R_3w*|aALM~dp%*UpfHv<%6Ok*Fp%el
zZoCr7-O|r+#uDLme|a*bc3Rldg1g2NV{F_HiH|WrT
z!M1znWJRf|DMv6|c2|T1HD6i4#?iTvrpm3EuBe3n-)@1L(O3|!Qm#kPEQA(eGZ0k6
zQCMlfZeOb|2PuVFiUlM7dUN(ViUf~(`TXiaY(%R2(uh406e2Lly@gA`k3uRonwSya
zWr*nik1S9z#mEL_t1p1dQxzMfS1xMU;0F>t$N1%))TJ=2gN@YAz+Yn-6M)Mz9eNK&
z{a~TcrwsaJ#!;JM!J5%CI?2x21M4j8Rr4j>B6Mslb|NLjO*bEFY<>=mw!#r87Vg)i
zg^M$VqxqD$rSJ5juQAz)gZN9@K6S5tL%ZJ}YP)r;&Q&2BjuDJ&Fr37lIgQ`ra5MRr
zqm&jDIU*m2zvZAanKSgC40XSXODojK;XS2gDh~AG6g)tX6QC6+Em*~(tZ+NUbd%yE
z%!je#RW?E?Ho^UJAj6Iz
z_s=u@@KlDxYvn@3S(2(eWvu?e%sr$p{~%6hN?Fp*2?jo6`eUM~w%e-oIKZE8axpQ2
z#&_bCI#29f0L}oNT7>cb0}$VO=pAUHw+ohj%(1wVXKNiB*5ab-jl5(3n!@%vykcj7
zgd(J87CkN7-i;uymW2-$6-6Gv>VJn!f(0W+Dwz|RS{Iom9y#A9-G+z+Y;0bneIXL4
zbkN*5jN2Vb`K@bfE1nVfX;oXB+UTx@Kjeh4ON1umVXE%c(Xu(~Qyg6*zRB)fb;#UH
zOAZbo<4j-g_Af3uQ0A9D8E}?_oeGH5y|v!@w@cbb*UdLh&mCd3qR^n>w-yEYovEA%
zj_Qc%vSL@2(SL6B8k#NX|Ga-Z*8l(r%p0J?rs<94LZuaBHf0vx6V*i93#<(B4H8iz
zKIXJ2%G=^g>Otd-zoSb##$YjESwb!m!9tI`U~lgb|L2!(JS{~#yRNjYe<7i=U>bHQ
zuJp>C1?LyozqSJc>c~o4V(+x(tZP*>$CeOt`qcBT`4;SUv5Ci_0d*jwOA65d!9|}r
zka=>YKzN~K2uOG9n(}l=+g$ds+RN>xRQJEn~{I4}xfn|p8L*$yZ4+U0@oisRw
zw~B!md#gUqKhE^zg(RwRkOLf_dl53pu~XLEp?~X)D^a=Dz&BrNN?Y
z3Ny!}5%YD99l78NG+}jJO>Na=qUN);s1qTCMr&9iOs{9-rVST<97;=R2$#
z=VOBp7P729yK$j;=VFvz|4gb!{k{TinJ8NMA7DP=R${+NcenX=5zLqGwMSq6OLmr_
zuG_r;UiGI^9xI@nbND3=#n7wixt=uOlcvQ;J?&w%eOA4V*fY|ORNDlauZ_vy@!rlu
z1asHvvC7`2$G9oUqp~?MjxqO`tcoHide1enP-8?x3unA8YpF}-;x+H*=`Y7pR$bp!
z*KwHZmM~$Tc?~UR|Jh0Ru(EoPt0r<~c2Rz3Yc+N-tQJ9p)r-AgjagNIjT6eIT}MG)
zwKjbR@gv`aY*{%}0-p<^EKd{+^r#6Gahaf|GjyjP}Rg*R)q(3m`*wXsh3i5n$epC+~)8Cn0EBZoiIS!h~tAyuq
z3uUJRuKP>O&5(;EMeROm?c!x!CS2O(=I298uRj4h_}>N%JJk)?);$!vZHnSWY+BYX
zFgyx0P_!>sdqU41BU2v-%Et~{a5Y!Ur-<|qjv^Aj`yCJXXlj$0_pEjp_b#%fi5vue
zZYs3DK-wp1wuB_k?RC9zf3iI+k3uBo7gv74*wRjMe(>FkwutM^e$JaskRZ0t9i4?5
zH}6KC#9db8$LAeuiWisXEQAQIMn|uCM=nPJuoix$$avlf*s$`}2NXzH%`dz+EAfqc
z^Y-6LZ>KRxfE>WZ(_m$S7e-T-YsYme8JXKT_3<>faLy4u!BqSBV4;ctA0?Ptjk`6{H4?;|lPqj|58+;>
zJ7Q}K=y~y~#0ZM52qrCYyBcx1=iER4Vx0&LVwXE2bU@N&WD%jdyAKv1a3UGe8p__2
zxWgUYK)N%bvU@N@Y0F~rRDhKmhRxHQ80Qqkfj9g@+$o{X%j!^bft5Z#Q9f$q>#<-x
zgB2wTI%V`_rD{F~J$>nX4YT7O7Xs1DE&Vonw48N3mzx9|9S(a`gKGxol%xgU6_zII
zAU}8y{|f0M#B$=4pj+FVVF!5eplbtTl=T;LiZaPkpju87nn?{WEQlTt(&V0Eg9wyD
z{77Js8iOOFxz3_OH|-9xZq_#1BVsyYw1T&fWW+ttl^gt_V4O$3UpH0^i^?d~1%7$_c=drgkprHG>L3`R(15yc2NH<6=zNN!6`p$KD=IA
zQl9X7m`r$uIM9cc?sRh3=meo*Bcb|*tHEN(4(ej)ER5+9y=7W6xoZX8C6-oY3@}aV
zniQRFOqXO;v`kML9JqDt8p#;p^FFS?$j8;X#(k73z?yS)jjF-+HJsp>lh9u+!9J~)_{!slpNju|+Y9|n;4jW$0;1H;
zJGUK$%00&AKd45$K(%*V)xbEslVXD!9{54s@IwVfvmFunFRmYWjTY=9p@PuC
z7jT`Pv3bFs?_R8&HMDdc^(w@9OcPf&-YPsl{SK8j=-=~W!T0?+8917}&Sg91g)Hi_
zt@r5dzaT`|J@@4#eVk8`sj8%fQ{NH|l`O8R7DgOZbcwo7R~3_fm-s0i(6LQ||9J}P
z72)Q%lU;cIxuj#@w5OvU0E!3#iEqk;S1D}k4EtED=ox@UpJEH0=2^P4FO19^q+PO-
z5qgF|f$f!Iz3T{aXEnPrg#zmxs;N=AVs^Y~FF*e12HO?u#Te8Y>RaOgl08Qnje`8n
z#8>)KcU!SI`PoybpxB|1N+FSD=1>RaU03qs3X%&9@&9=v*EJf;HhN+<-Fj)G-n(OA
z{trb01oAA9w%G1a`i|>F8{rxq@^gd4sa!H8iVP?L>{pWV0~dh@xmMPUCzO9EBw{ke
z{YPb0>&b8izhe%wNAjzLbYnm3xp-VqXMR>iEr5XQ!?(>
zVg%RO?LF5;*gGf5>u~fA@Eslr6&WyI$s)Zf=*xl43*P)51-!2e=|6z|%ba8Q#D7d{
zBqfn1;>=-Ny?1OL1ye5$ryW`*S2WaOyfi$a7`9Bo4K99(1H#5vPTj@n!;9f+fEWe;
z3uVXu0JH)Ell!f4#46a8{onNr*M9)0;Z4S=H3q(%V`DbHveLFJl(BRtr?b(IB_zA$
z@Ou3N_-$y3^tYT`h_46(U0BRo`vY0MHvgZC-|8)^+-~d{$(eS7tFc^e;$DR&qD8Kp
zorKR>poW<;_wfux6D>t(3Cxz`g@L~t-JKx)*{J1_FmgWoV`c$wu;dZW{p;z&W?c#-m{vw-TxZR<|;Rk-0c
zh!+Nj5Rxpd!xhgMI6oG$Pu43Yj;U$3KnxwJMiZJfA_;U2DT?cvW1w>6RK$K>L!Fda{>3bRh
z3S{h;Q~Ukj>w3?Ezs;)v*|4Iqk&D53BcqgZv@^Fi7(4gaoXxhhx>-(uYOLH0ps^hf}9_X3rLwoAog6#%oFTh*JV&*
z>ysFM;`!yrai2tZCu^PJIvJ^@AU0$$Q*2i|*3N+XL`y-F*t8itfV}mmTC1LL)LToW
z1**)kd1cKoQ{mzfc5kK8BZJ>&3*C`#;<#P8DDCYbwXIu^+&t@KR=#
zoz!LY58BR2v8GiW8yA{DBHjV+48<<{cRE&Ic80&CBD&wN({yPU4v|s}J*lNe~R2$S4B&?1Fbz
zPbPTH0n!iIbxjtkOAdF{|C@D57{lswT1_xs2iH=X=SBWc}rDB1J9fG
zA3$O`k7^I#@`r%DUDRJ{o(9HZ>m96xC&gYBm(fKN{8!e5N*@;sT+<~E8&*Y0w6Ur<
z=p6~gJqHbf`fjc-usk&0gQ%3Jr&q^=?TXed!674
zeN4`Zyl>Az#d)bPolZ5XSsH3ad_vnWvk(K-mpyXQNa;5%u$oDcaS248@8n-@LpSp^
z3L|PPr>M+el;}x=(C(t6iim01gn(e-$KBj`C%$nqB0>5!KogVcpgX8mWi$RYBu05B
zPkb^p9HGUBaEWcRqAz!+7yAqj*|3FN0f4ro}oPx?33&Gx$G+{8R~B?0qk84_FXkrqCMTN1E3;z
z;Sx8oIJI3Td@+%yT*I~*iVE3L0swybNMj*0Kn!bA2mh;cdWw!gIE#_0WzHMt!`q4v
zlQf?v|1`5A`5`ryIVCkylbDrYerddx*oGNLT2oBqkW!sq#L<_-OGQWu4e#Xn*@PUJ
zx2!l(S4$a%!+k0=#Nmf)AV(=0IQb4cOwgPxEl?=1&YQMkqVCf(${{U9+)~!TO
zS=kcx9`#MSwl-iTxM?f)2_2v)AVz@dN3eoLrcCgAQZTmq8jms(5xu)6Rc
z!1#eQX1^t>1)DMCoH>_ajTtt3khw$3DH5}-La|+uDDiOJaC>2*WzF}-aawZDt6bpA
zY)jh@lqe9OuyQJ8_hCo94DZBP)c|!5+zx+g
zM#Wk5q8z-hb2;wycZc&OQuKLjOc_q{oxf+2cCQSO2|y2aN~dZ$QnPHg*zzvToz|ic
z_Uh41C>-i*prIjHJ&5rAZS!L&S?lgD+aaNOpnQfSp_)(W)99fYaAM=%QINX2lme@{
z=-18AcMUxrH)S#h*Uoj20-<$PUyyF6i^X67h*QUG>#I<&-fbAYyL2+)sKp2SPuk2V
zz9OEA=bCQ5V}PmgMcZ%0KPc!6y}wItJSV_xn$f7Dx*~&H8e8Y+vbT4B+ifet#IvT`
z)Ctj@cW6=KqHXxm{au=KOHwh!_Sa8i6;EmJM_0&3X3io0L5pyX)vZ_*J3lQGpj*-xNyyB0D}+m4+iPp{0{sLfpJ
z2Q`YTAL5S?{Mq?dqkLDHE@<30>TIpbT8i~&wlr<2W`F-Om|ZZQx(~g+
z1e4OuB~28X5-onMGve}}*9-|1{UIRkoW5Zc$Du@hI3z8~BgGu@J(GHVumbWFI+L4`
z$+s|j88xoGCSU_nD07i_>49MD`N2er;`Dq|D_k1u@fl5^J{zG5s<=OxDo7e#w&MC2
zrx>OLTi}99t;IO*M#Fc+mu@Xdn)(lt8G#;$Jwvz)!UrR#_i&~=kvZG-wQ=!WeRO8}
z&uZS+B91UQbW$|!9ZnF(e3}&MBcaSxxrt?~`Yx`C%ri#UFMfhs)J-vftGAfoyjGaLCiF9~Fm4
zAp-FdERtyqbNzUSdhv3Wc>ChBtn;rYp6^7%H4Y0^=Rd;V0);Hf;DD2aP^Lm1NWo<<
zbGEvx8}wPRYVzN97h9znEb}tp0HLVohF@gqQnSU_@q9X)+v9nfh-8mQR0fLPA@j_H
zcg*k_aGel-$n_7WVX6VPSCqGwa-O`z=3wbXOM|))6~OI`NcAo;qRVR1n1xc!=uVFs
z&Z_A1=F7IfYx7=&@B7Nnx|HeWJMasKenLjC{z((X3lw?GRd(321c>{Wy)vCP}^^OvJ
z__K9nVTW;K4pZ(nhUva}nzFoazI$%)ZYPMbd=2t8=+|eB1(;|Iu#8enCI3+P+cqPz
z9dSqgfTlBK9Vc@%5352|2jzXsy$!Jy?=8QlDAIKug6oq`RmbXj&?ai-z-PEev?Yn2-B@D(M6rPc29p18MpKbk6WxV*qUcidnW%9G8
z?ShfXjtQPEL*v|)8N%4%$lcZ}bsdbJyt~AYUUnUGcM1PMhs&_Kq~9bQP&*BC(f3uvMab|VWF-raNKUS2pC5i?D?hFxTgw8+zr#?
zJHWN83cnO^=qsvnFM%q~oAIRI*+KE5Ti0|Jzn(vqBMIl~`{wQHsy&pL+Z~U|ES<6+
z1i$)%Q#_l^R>PZ6l&3WVQ-3B>r%I`RR&Z}$xmIp)=Ol8aH@dQJgm4Zwoj(417&e)=
zt3(d(KD_u!AEI~_(6zo;qW%-4t%WgJ`Kl;~dWdeo>%4XIuY|Hl$pyOY^8l}RRSCFf
zN_sB-xTdhzjsI>oQvk}jKY4NV;LVyvC>Aa_)-*$_$nsxwMV#
ziNoK8*pL1
ze=htlmz7MG?E-Z*)%7`^%5SYSaS1lg>~M`#J1sm&RKuQJEq-{)8?)MbTw9@W;cMYg
zK!r#h_!=0JYD?;3?&<_k5(_;s5140dObW9Y4(b@>GfwxjRQaE;vEN3r7l5KR3M80zSdE`^w+7MZRIhsjtfi$Wn%o)uOGmPp1l79
z0|JEdgo^P0r-}|#9C#DoOvMAX-fX=|{6Bx1t0aMqt!HkPP%uE~C&o@@zOOY6sx+qt7;Qpkty`l@7qO8mB1|mvQ#9SZ@1V4UOA0C9ePp_TF<)^
zl$=3R+1!z;AB228rjae$aV@ndN3ff`qh%&y`XR7;{h1WS!Ur>!7wUi)J*&{A3@_7S
zU!^Ag#jiPz;Z1{DCt+C*8X(4+TgXnd?4KO>@~O$_s$DN96#JL$;IWv)O1f9mh2qVU
z1K?({bN84_j`Wx-7j0h0VeM!H2%XobxWUBgMq<}=8?cLkCQL1UNID;lzYWq8Ygsg!
zT@eF}qBug(Zo2dW&ST*U?s=iQmiToYUTg$-#{c^C155c~cb(Y+(nOP^^oaJzV
z8Ll}i93}i7o8GB8m@wST&-hC3jkCCQnFpb_;#&xybys&rs7F&@v`-DVMc^yBj9lhV
z;FIcq_ho(O>ZGJISJ#6INULoS#IDn(j(;-=&ASpHdWuY3~G%~ND}ozB@NBlwpXk&Uu2AOk)2Y
znTTc|6Cly#&ka%LfzwR{dPW^)r&r6$i|Vaq`VJ-7`=H$Z-?`)X
zw2pG&s;~F&-;%+MJoNU+A9!nankvWK^;pUUgo`~PyVqI4$8yRa92yS#=zTKGT@9pd
z52MNxH)-aoYBk3Ae=0Iw=LA$GcT&zcHpCQDDK-VSu-d&5P;7W#(>ylzNl2c!ck4MO
z?lQGX(I1<(cNr(eOU5oEtpv53tY}v6;l>zvs2Uz$rCdSP)^i_@3t4@Nu+Yig^i-v$ak@!bj8ijg;vd0-DgmY?&QFA
z5f+ATDPH1IV9TL3$h*$MnPT-O2Y4OIAp;#w`N5$jZIOdSSIIuL>?ok}F4}g;Q3@OT
z?1Ah~;%OC<$)fS8#a~pP%QEO-k_`_2TQF1bb@hs-ud@(b%V?*)?>cd~7A06CayPT}
zc@3SP6I+|!ejcXl*qAaS(lLaFI(SttvfA~?q9<568VQtukCtEAlr+sB
z3d~3UOsp1aKLly~o#5Vt*?QCGb?~PC)z{j17>>@4TBu%B|Bb`4(XEzz2_D`1qn>VN
zg+`DFMwQwfZk5R{Sc?Dg(bL<#ExCAls3Yvx%IKMNZGt?}YxMk>l
zM|B_ktry%(<0L_nwCr-H2;&g3&>2h
zTYZ3eX818t&(&ht4F~U`wQ<&pG=fdPU@`rd?!)h+6Ebo5rnQT_5P%oERkM-qiZN5U
znXb7~3-TX8Bjv?CBfJ_PXCQsrv0)YyiSAvjvR3M_J?)F>29>H$qs50VYCk#6xcO!y
zd9Drm0{qW$u}yY33nw&0{RYhc;aJHq(aQBT8sNF{adB6TPicLg%KRzEnu@cb^DW54
z)>hv%Yf|9FZ(4u@QvR*080buWvmeR(D6P?To8R*7kcwOnOm#I4nVXRo+7kN@@UBtM
zCPEnLelnpkNTNFHjVtSIUw=zz+^mojcU3RX@~sD#@#ky*OPG!UED#FY^F&MIY~F#ScTyZ**m>IN%rxkNTF*
zrsH{%i5f3b=?!A#@ZI&vHe$NueDiWivb1KRM`6Q^>Bd>(HCl)YL0lrC>MYsPGGH24xEW%(=7R9u$+Q^=f|bSK2aZ=cXv90
z5<$dNz&Heb;!rM<3V~sXoZdd@WcrTj4oa~8Hu-YgrpjPFAiwMr1MkKWf~}iH8VV6P
z%1uJuB{Uq!W*#?P%{W};%z$m=SSy9%79&l+$xMcdZpt&I)nE$mVi8@ZsLD{jwv|96
zv@SnrQsiTe;X<$5Giq)Xq4^_#|I5>o7F1Ski4k95V_pB1J!DG|;xx>ea$9Dwn}5Lp
zD}^e@pZ0w7c;kX{ee>$q;@QSN
zB39rsT{}|STc%d4`W4lPJNt49-FE7*&_zpy)ycj!iXi@g33BUULa#qEp;Dubj@TBB
zlLMAgn-O64Trt@fNO*C3mM@a!+g?k4THVKx?O;@rxKZyea(;EZ;__%>q>|?{q3y|c
zPhQ&hJ_on#<--LpZl>ntjHlw#0RHZ!ZjESQbaQ&yXV~|0;uyvBLdU-Lrjb-lgC3Wkio7Qv!*auLYNe#F=U8x0l`gsPB3I
z1MGO=Dz@m>BtC3?AL?>D7CPhyw!DdI|H#
zcVF8>f*a}o16Vt=>})FiNo0(M)*oG1mYL-9k
zPOw|mggE?7{3ZlGAGI#A&`z7J*oIl(xNDo4;SOXbHLPK0+{OX1Oq5M@YzY$w(DEcajd?*~nAAAql@
z>@i&B9bX=RubBiq%AEUw_YV$leoZ_^DPh!5{MMV0GYmq@ThrhdroR6FJtU2#j{TtR
zNL!qAcLl|KeFdnTxMVrG(xDggeydzX>#mcdy?!=o5XG!kT1{0;=VriY&-Dr}^-n)W
z#IEd%D8rYLvM#Q?&s#JU!>Tr&RPKsV-*wDr-@wVap!N&nsZT<9id`u3z8HpKcYL+^
zQ2F6GeZ1=D28CNr#|zMk@rul=E;3i?4u8h4RoK5LKGyJdkzuMsxH7=7NS@qt3^EpM
zIBQ#_j?`^{39PStJybL-LrT}wTM{Hj6u0EX$Mc?ocz^OmnK>JRg6B$U({~9K4xIK(
zbPCKT^Rmv}b%4#N0xFh)%aC)jgb6C&N)xyE=&ppPPBMxILWRx9!(7u-*e*I~B1XGbu9y^hcZN|3vn$sS=m;&v%u)(=O2_!zRPA4U
z3KTfl4$nf67=^w$Jhg#Zr??uoaqeFTRoAd+I6_XVa{
zW6h;cPx}NHIRU5OnA|xZ8+;f00{EvL(_bo_?Xe+8F*Ncw42!I6-4XiEY_?4TCWba^
zZ8mz&vNI|-l1z)
zc8dzX(f#V@SOn)Dq|`InPHn=TVj;g^Gzw8|)KZ|>CSjt!0$avbb~?M(v*tv(qq)0T
zVGCbeakXnSlw8U~B8Zgo95Mc8U+8zJ$h*$iAtMbTV2DiBAss8+=qv(E>X)
zGig*KQ&E7lS&;;s5CYM7?Kr8x?|&)f9H!U(EHA)llUpr4}ta#
zAC4Hisrc!B3D4L*>KPoMc_0>j7FzT%a(m^NCN}Cy-0F9zN5_
z|GwOh`uFHv0v9loQ49@AfThp&*NgP*(TSio%4GMgi&(1@K3maD78!DRXytiTQGbm%
zP0+sYcUc_NbwcMytG`(vWJ`~4_#^tuAb`nSNFJ}
z@fy9NMYDY+LE^l3TEq>LY;(q*-iq$!83erCP|J`HpUGk{m5Iih-pR@dTLqb9e{6j%
z%{emW4Wnke;LEG9W=y$daSy@6%-*zNkUKGO@nP+#4Db1&onzGs(Z8squNR^L$2tcD
zv3o9Zct$4fZr?2f4W&gLWhSrA!I)kH9*Qpk$u6o=)I{E;D#}U1l)ksN(OM3`u=w>;
z5zbOE)v8$z9m=hu2r?zrPWlg+ek$tN3>0FmHkDdoN&``7OZB}gdKfosz=zc1^=gH5
zQYTbtVzK)>hbnb~l;XHflb37tZ;ji?@&gBts-ohhB`LT_c+O7so2ACRWYfz%zip#d
z=Jx=zA)E^xTV8@+wu{~6?{0ZXHSY+sLWZZdcVDlAK+mrtgcPa5&i#woWAqOaR2&Y<
zu%&qZ1BA6<`)eXI4|H>MnWalyYqjyzZ-6PpU7}xxTwhRz<|6a{lZXjDL13qW7
zJ{SiE7r1-MAHSq~TO>TY3&4p~?>~xD3{kt4c*z;@U@%P_xU*m&ujZ8I&z3pX{+aVk
zR)Xn^_AO5-v$SOvm)5!jP$MMvbeZ*?lr~y5?Q3?^Og>UAHuRIM6F^{wsDqSx0o$kV
z93L{*yvi9v^xcRH$aRno_vu3doi+CoY)+=`x#cZ%?FQv2>1SLa2zk@opFUi8pFRXE
zxZRdno|M+GC-RWWY^7EG3{^3g%LkcDkDJEu=FZ;}(m^4r_5W$#r~cJMR=
z6$)4=KR6!>Go4rs4NR>@!t4F%|5^0@*%hpa{tqCz*lbhT*-eu@sjeoq`t%YaOA3f=<9hBO2avvWf{QU~a9eH$%^M4O*^B(jmOsQb*
zQpiXec0T`OL*$~^-8kEUt|Z3B3cGlJ^5?@jN>@%=U((7qWMwAEOjKqo{Th|F>dLhq
z?B_7zJf25(lO(;hb*^kbqliPH43S@fVFoy6w*O@v$~ICb^7Apn8m1V`i=igJ2ICO?
zx0BqJAKw!XxP+&U{Tlm!=d~`(BzG=J=)f?Q%X_L~DoJSX5^S2M6Lpx;*&rFoeTs6k
z6lEE8(fPJ}Q}m83XREA70$xI-lk-Ymxt7F}HI`5NCu=N$qICcUaAD#6mh}7YKEv>0
z1>&j`URGrrWiw1a`LGOcSRHpX$V|vCIC@8OjvJXkTs;VprSL2}9N4KS^ceWqGNwP*
z?@_%FT&*~>kKR
z?u%zpG>T-7R21ggKv8W1b0iGM<
z?`UFs{gf)kPm-gFz}pYw*w^FDBcps#|Z{+yYywE4f4Yl@Z(pRc9Rn`eH9TUE0#!<9~FUp0=kI
zllwkpCCzvj)N*4gUVp)VGRPap1{r}NlK{b*R;^gXq3xM8|DRKK;1x
z?Rr60z0gVR=i4+G^WuB2P;8=_FHswrXQQIeWt135jraZ5*B!-YC4$@1;nqJR@g0!wT)B=Z4bAW{DBh|Je@2T+
zQTdsf>kST%7c1@_`^BC5Bv$!Q5e9C_Gy06n<0Qc-0L~9ZD9sn%quh{GA>a(HdF-JB
zfjd2Rt7vg<1Vhx1A2p(PRb)NDZL(HD6rXbjP6KafjT7JS;>4RaOW-Vv!B;jt)y)
z8EFZx#VYtf7hAn)V>Y$Ykg~?b+<8tX2rZ0QplDx<1#nx;UWt@bH@pC
zWMbAzEU@(jU>*CLNAFhIa>-D7jmCJp>XicTU{O2uU%s+cI`%Mw3!SW5bzKFS0y_aK
z1*aAn&jxL61b;h;z8X3RXm1;t>ymK@{I&d(?VA4OnJOkHZ~k>;tvh97SfI(Ze!q32
znG`7d3f<|F^JZ#pY6QedU1`Zji+XYG@>oqT;-TD97Z|B$biRnF%P6YbHI1T(>;*c?
zF?AV^@Fb;&I0|}~e#n|2Q{X*VR)yS=yIxupaNLs3&m}Ca@`l4!&`{|I_cjyohSdu8iZ{BEYc0wkvi{`4
ztXtaf6AI^vQ*0k@UVEQbD)OL#$uB^tCdJ`6Q#EI;R!D`?%UN91D
zf1V^r&F~s<@1q`fLU{KVgY`13J*&`Iw6XqRcRrSwNc*<&c8P{YV+%HRhjwwhe^1qW
zCJlM~4?ywsn&ihl0_2}vpHV1Mfxu>(Q14`c1yl9l=s|-rMtMuL9$IU
z;t$sGwiUER!l+fVh1JGLM8W&(uPvdu7q;_J58hEnF%#z7NjcLSBcS8ky!wNOoRrQ6
zm@#GCF|zB%-{HXb7FZ6EMgK&&EE+XwXd9q2<@D*WXKw^OXvt{xMJu$Q7m$8zQoj1`
zY4v}vhDUlg>BC#;yp8=iJU1&CNzg$*Yx#Fd_{AeoD$1-_n1WgCQv{&Le$d>T%q`YI5^
z+9+g7RtTUJWHwJ*r4${DtXDS=6YU^()>uc9zrhQ2Qxa1rx4;q|c}gqq&5RLMiywGp
zRkeDKfuY4X(AEUvlsgMGQ~N!yvALf00wy=@I?KS-j!RLC3Yt&kY=MQ9XB!A7BV6!x
z!VF^IO?>ulj!ve!nCO(UeJ+yM6>5gI&zqRAuN}bCIzK`+yk|JM=3q(+9=83*_VbIu
znB+pc2;<3TZjxMLeAyEv&UO*Gs_V!WoR
z*5sC+td|2cb8O44O1{TF)$sht^zkcO?cWBujo?9hi5K^AEBS>b$x=$Vo3RV2?eKbU
zP~KR#D#u*UCVuu$A45*ZDmDW(PF+ce6MNKtT=99AOL<4L3~mO>1oMpWH@Nhv&9e9^h$U4
zg%76)27h#l}Vnt8pETlu}lnUEE#Y`@mqpg(jx+UB661
zp07x2gMrp(GK>#LVKvRMpd_-`e*!+Fm2(itjvcXp^4}qPh|8K&XRgNyk(UUHF)+Ke
zB~oe50p#gT@2xcKF0>#q`K>$&wNnPY(zn5d>iz?P-frSG%OCp>%fKm^zdI=~&azye
z^9_7zeXY_|0$nyvSApZ3vV+av{Ls0R3$d~KF&8%E@vPv)FOo#$oVJ)Czr1$*u8U}A
zz_$_WhHiWHdEWRj`U+V*sllF_8vWM}&SYB4zT*>g%i$|tnparJkAcR=^E08Yc)p|iT;J{jr*_xUqOy}wK?
zAC}9`Prld_J3Xi4o73uxPqz3HimdDjPBvhk*09xKTbJny&W-$WG-G~Q2SqubyG>3M
z#&ZSyy8i1a^nCtnophIym8C)_NI=7|;%LKpJ|wdVY;jovjq7rj{Ac9`a{)h$-f;fO
zoe*%rp|Qw@UC6HfjW2&TgSrB(Nx<7D<#u6zJZiPE>a|Eq8?)9r{}pXL7O^u&eZ)FYv5{6#fneVJUj-L@EOp5mYY-z`6xf+dKVl84
zK-%RY|NEG^jG0YP1tLjCFH!9MqQAVe5)`!`K$auIs&0zO45<6dFudO|w=0uD7SYK#
z6ZfeQTl6N(XbT25k)LUPp>zh?m3KVvPZR5JdR;|ik=&#Dq!Aam%@+l6Npl2HG89+g
zC)`qnTcb@&e#SUcQqw#0ui?<&Fy}cq6OJ4icxt(
ze~5aksJ8m<`52J1>^sn`W#;J(Xwt@Y9Y|cGr-#yh*1In^?U9YDL_kCv{
zFlmS5_VCOQoGWER&dbxNRUN8dpZ?6{85_GuCbXL~`|mzeXu;eb8Vp*Mf(;Q&(Z3?B
zd;xUEX0SUmWYF?%&~uuqNu-od2_Ac#zkK1Kiyt5oe+zfh_Y)JbUT8Ypv>Mx<9(6|l
z3$M73GZpS+RB+NrHq`r9LUQrJ>1aw*L+RPM=QJ7FvfE4G&fmT@YjGT`A0!|OmlcsS
zAw8&PCIeW31%Huba3DTy*?^%GOkMWxK;zeD!a>3gV}t4lwnnHb>E_zk#iG9L;yBm0
z)GPvuBIa)$gzVsPRn%-4!_MEx0X${p{yrh1amBVf1gH41H!;{%j(^F*&BCDl=s*KmKFfyq^ege~i
z8MtiFxk0U6><+45GEagXG?536FVV9fX$8&{U8Q=FEPg+WH_Xu&G9|xdutUdo-X;K~i`-fdXBlB2gKC}S)}Ni2662!$^6DPfH}T7a
zV3EK$E8^K7{8vg0aw5E2Z3FVQ^+C&wvG0F?MAH4Of_xZkg`n1Iq+Jnt#cKRXL^-Ct
zr~^XFH-A4wG}554Ejp{Qe;(EM*5-b&C5lj2km{v4%YVuipkgacjk(mJv~dS=#C+v
z_Xxg%^}(LxpTq04Dn@s-b??RXneR>#KoAnZQib|jFYtbcm9Jbf
z$^hRm${*SIX~wBpM4IlSOq=Dh-z|-f<|2$!vscicLNA`GeYx=A0v!?oR&oyXeiJsS
zSr;-QE+rh2QN1q+TNq>ws$Zkx1FZkpyP(@iA}QYFsnhor?24iQOSW=9bnaM-I8bn6
zah&+kT|VSu5rF*(#JhI>5#&C#C0NBP@}s|+WTTs%`sDXr3rC!^Tf!w4A`wm*N}KV6
zuXz(zhRyiPBlv5A6cimt@Mc-eRPr$z$QszXJm5PV>HHV|Ne%pV#qL}C@J?8;=_ft`
zG);kWJbe}Sg|o10&f)p8e8byyekm0c4k9aK;z0hIh&p37V)>bH-T*^iDlu|5E1iS_
zy)sH`ikVQKfd83DO}72TF)gS${e4n6Vs+!vH7=$;Dmx-3OS)r*
zaITuvXJP&$sf@W9xA>~2fmP7)V$h17j9tfwrYzc5rd*tY%2f>n1XC#A7mcqL78NH}
zXh_`}A1PZ67-SSr3jQo4Q`U2eoR!BSv|SSDpQkY?()UpAMbThc+EqOZG7uPj>>NL;
zQ|ftg(ekMuQa>NaU%~O0lpCvL(`~#*Uhqd|s=03ejhUi+H8@bPX$z-qupz|BK6Dh>Onzdkx*`&*j$g
z8{<%-*TVWR}&b#X&a$nd!~v%v-UW6|H$T
z@|PTWJk{v+LX+K#LuxP|leyd4V{$ed>Bp_nB9G?tp?+~lP|WnDUDaQi8rnFmbH9o)xlloxs)K$@d^!T_QRxZsM(g=Yav)GZTM{u5*B~=a(o1rD6{Mo~Sq)0efR!
zD2}AdnR%pdteu5Bq^egwXkM26c`MmR=HOSWcT3s80h=%WE2cY?m*}5HoWuLQF*1;q
zVLzHIlD^LoHj2}>#9YK`R+?YsT`I#~yFq|qrzD3948(%q-+eHuHmUj{$`rI
zT@#Er4gTdg?Q<$R(UfT;d{ETHgWIHVI<
z!bAP^Ktjy-oNM5xm#d1Sj18UYUYTm7k)C;I1!3o2t^-YUKt_@{V1KbCCet}R@(#=H
z&iaWymSVGtXz!7|B*mt>RLRN~e3X;EN<5!$m=>!}=-$@-1UPmcA;E*gwsU1sFzb)&
zp!?>DKRDrA(6;v_ZCd-wLYR*3y*iy^a`nAO9*ap!t+pAKaETz5R6P?i`XE
z5jkI!ens3kAwg%i8A%7a1^;1iHpjd%d{j^Ff;CyS$yYzJ78EziTO1{%n(9zkETcNmX^19bns{m!BVM%L@W)H_OihDxPwGU^)2dOo%v;po55RMJO}0
z&W@2gBSvCw25J_{q$XXWQoU3B|1$gl@sj=Dtrf9yfI6edPrUhGn7^k4It
z(C}1zv__zSuC37rW$XXhf0-~u(3|a{Xi4A~EqZmM7_Alj{|dp|mr1k=2f{4F^rB%u
zev>iK>xYM^aFu@ZQ2$^Q9Xnze93sp8Fx{`1`eEnv6=(}>-zj@I3+&J<^}MUp2?ESk0g!<4ipcq$XOi(2#k!=Ot0-5!A2
zc<;4tP%Y#JAFLc04>KxCN+TX1u|o#}y@nrH=DXIr<2;Z!
zS^YevKb?q($@dbb;zK4!zewhPu;LH!tCQ`{_HEIwD#TZ4UvlDXU*W{=xgrJAJZ}J8
z;MN&+Sq~Jksq-+&F0c_-=1|?YiJ>ptwQQ{aqA5YSR4=9S%xm+D1#CW&S!ofaE!KQsAzvFn|uq>>ot;BmQ1n|i5=
z?_WzAKC6q;`(?j3iTbfRvnz*`VJg-(i!sm-w=6abV`JN*DX-qIQ=f46Bn4t`B|?iY
ziIPxwjVnjfGomIIZ3#8~g+8DMQ7tN6nW)uT@KtMGogHxjE7onI1J!cAbthHnO@GB0GH
zIPmh)6`>a33P{!bcmLG86R!pDB8$)x-M`?W(aE+g)&Hl=01WgAhWzGbCI9!Ch7c*@
z-tppWQJ6(2USgky8Iz?3FEj^2L4f@ysiomSNdXhph|KoSTDYWzwf%GW!-bP5%>Yg(
zh74-|hk*$&iSqF1H@JqS26jJk8xQ5hJ@%tJWY^{cSwRBEv)jm?8S9^1Q&WdHE=iD?^dEQAr
z<7_M2VGiYPeXbD}Qcn9xa!GB&jW~T;Au;OJ7ga*AVprN4TlJOxPtf@{Qo*j2m%1^B
zB3HNhHHEGcBx2u2-Q_>(*%9{h+X9B*Lwh#Kx-~Cf&b&56T396e>XIWh=3MUBh$#)~
z-Nh7_+=zI<@2SO^IP~N^tJrdRZ}r;lR(^g#)gSZ3362?AFAB>qVa9HIu%$C03tU@i75RI>}rvK_1?VTjYl+Q5VB)Kce8aO(D;zKn%R-s@{h=oA1moRqemS=
zQx%flBpub`zv#0CEQobbtQU8Cg{D`6#b4f&d!@)~^JTG(DDklpY7H2WI`kg&ObHxS
z)(`=Oi>DoWwLcVmchTvg!9{m4{E1rU8P}2`b1Vt*4jwv;cELDA0Hm9?0s?Y-y*bu2zwjaUE;t@mJ3{YF?CW?6tUf!tsn+f#Glg
zPkG_Bvh!cn&krYTpj&zuw>%BBZ17Z3s0~h=%*(|!Nrsmn5DMttqnsFlMT5wZKaV6M
zYF<|*zGHk*e-I?>n_P8AkNg%I&Uh5Y)6L}-t8
zP*JyQ3-`U?W;$=y1xn#`hc(|uT)x`nQ*gBtKprRpSWQH3SZ!m2M1@VAO#qk8jVjI4
zyrH@lmM_MG0%e`C&&1&iHOXZ>yrd1*Y$sEcFkQVKR!jp@IWbG-I+;BlJx^Cxb+A>L
z-}UjHT*I26z&B6dBzVJhlR+DY$<6W93}@r{_q|;_+Q6z>Yfy{b;Xc?DG$QTXYyywb
zI$C2_ikQ!PV84Bt4l#aXC;$wHV1<-vH*p9UyE|`-xcg2(mzyIccuTt^uocvuvGu54
zMmDJ%(bp`lbNYrkzZ#(#EX@AP%q(A*>zZLv()dO`zymm5sYW|6xQDFnb`L^rEUJn`hKJ+YPT3mo6E1+-mOX
z_}pk{niWpe)N9f)&JIAAx-De#w!ES*enj)B?)bJ1NS4F@>XdS**DT5MW5)(Aw6p>)sZHTr;nU85aZpi%m
z{7O0t!XGFMGRqwJ;inptx$ucqf7tIVE_T%GHzF8#%=J*(Ob9tN8E?2Nw2RfS9%MI>
zp|yzodK4NnAIrCF}8GZX$&o)Uw`XQgtaP6
zSNOP}YU9IJi!-g=3-9+e+h!*4Wy9I-od(f|o4E_MW>VST@*7D%CM$2xmO!y26m
zMOpqxQ-gnJ
zX<;96marl+XTk_KuDOo^$9OUjyW^!>ggPzfLm6Y41k3TqWK5Pr{(0r`(%qynA9hsM@_;(Jg`GiWsl+2Wl?QqY)Ii8t}Sr
z2y?eOv&Qdfg+EUyDFW&E?_=%MJzoDhdBlTCpPiYXjtJPx6wR^W`O|Di-A7vQ8?Ijn
zQ&ZpznVKfyMGXIkF?X5PuN3h(beFgnFvHAFx0dds1x;hFo=nE^#=^Vxa-3Y-L2v0r
z@aF&(l@x(0WKc0@URca$<s_Sb|qPaneKzRilD|El{Ip%-e1$x4d9^vDC
zF8#j)OVj^`{?EG6b|q2P3#<@;g>n6~@$ob+^wPVigpJ%facha8f;w#6gW#JVDZ&4e
z^$+VB$b7he@K~AJ*`Lu}wS9%Wx#0R&aHj^yMtv^3qg>OF?NgjHMtTMjkal!DoTfj=
z_bRK~14HW6tru)#o{Egis;XNR;mrz%K@WsMRkB~xT}1AHJ0@B!Hk5F>gQIja
?
z#gPMdP?f%zr{GDWWnn6CCWbdJ2dq!!dg??0EGRDZ)nZ#;30?y`a`&%Br*Hg0ZFq`!
z$&pOE?*$NO#21BH7R1Id=KEdQ`zj~h=UUtHZo!4^QDuuZFyGL|$W#wG>SlHhY7Dv{
zMpN_<>b!B8_f_JAN>w8SM~j^T2&iIbw2#y^itX;JdqMNm;*xk4ntw@UG-WGfgKE^U
zDMAdd4gw*RM}T$aQxG0{@nW&@FyZhrS8FfMDZ6rky}T$Yl1y8>ZK;DwM3&si``3=|
z)G6xA)&OJt;$-<{+t20&!}{FUiq@R5%ZI0e(8zKAa;?iA3+R-0L9LHllERB)&Y@VG
zA&ej5*c=@zSU+@>W=$*)>7)zIl9eJ;^#`qnQW#VE=ff%1oIKdB$+sI`g-i*BO|Tmo
z-Md57c=CD-UORCJZ;H^WtT?#@zFBt5T;Q74uuX-I&}dFWzEnateR0Dx5C<8jbN;rp
zbX@`pJ3hzOmlwGXo~Q?_W`gC#LV$rF9+P3h5f~om(RtJew&(aM(W7$JI!@fzjpRq(xIpuT_n9UGQ>)0mVYh*uWWBn7KKJ{-AH)NZq
zW3fASdZ3Fy;_@YPE3U}cvj(USyANJ>GQTTZeFRKq?&d&W+t3}hcV2t$ogdUfzm9Eb
z=cUF>XhvE-zZ!}FjnC2T>?9697hL%^`9UKDfa
zacMf8l>q%*4L&luW;GjsAzB-t6T
z;;?d-Jv;776(7~!wo-cb-{e>agdIPAsa>5CiQlqBKEki030{u%=|r(QOjyrc`@ql2
z(hge^z#BQwx##&Yu?1xB%c22EJ1^V5nKr1-93y~|2j%ixo3BfiZGArh0cP7(jg<_-
zmhu;+hmCrUsfx}!d}Wx7_D(~dzvc*vm|W^4+1bi}CD)wRSEM_|hg4WmWg$?hLUnc%
z$M$_ubgy^L4=?rnl(UcdX#zAXCTHet@=L@$W1<p?=8Fj
zN9VzzWaUAW;$LM`B(0Trc3S=q;kp-|dIIl({Tn}+3#2+GK$s^`A|H~}yrMrp1h5DO
zT+Y^Pvymf_YP-YvqPS)l&t96*=Q?4Mi|3Tf@u>7kZ<()RL2
zH81uWBmJzWeV?ynLRXDz2TY5G@C#q`ou4FTTUj$fZ03VZ*ce}-U0nD95w1!`Ez@Ni
zE;stv7y+JTc4tE$0k1xtdq29KjqH78zenqMy4I1{dnk@-Wmgy8UIl}0c~6wbrgnsm
zk_fJC{_g_<3Tqen{?o(@ru*+{J~Aql1{n(~pedHN07dkJG8R_rzS#G_
zJoa5&d-oK%-P|XAt2ViA`z_QTiZR9Eo!aS^2vknwJga9+Ys$mA^-~0~e#y~;Hk|wX
zciU=JD|alNt~gvsX#v->aU(zzq}Z1+LsX^=*6~Ok`Png&`U{`|4#g7WtyyOVd?FDq=}{zWkek-9b=!!>==>ErB
zx?}gRowl?V;e&rYL*Hoa&#Nd>kOIzU4W^b-4%%L<
zYYFiIV18=CPB4GwwRZjHwh_f-Uvh@Owu5o3;ofJ_Z&gUWPK=aS@%@5+ko{@qs$ND%
zj{UV|Jabs94K_m#D0be{C(r7ECi3nB;@H3;+sf3g42W?vu=32Vi}~Gz=uVZOu_#YM
zZq0%P)*-t`LD9InPcYp@(G8X{n=sC{l^JJ5nP{CsTk-Vlh@{rLnncfn@Kq^f=Oolt
z-U|1Jm9VJ}zum^^h}=4_gO(-)J7X-_@mwuFn5i%IcRx<}+H3a}rk8WlJ>wsF6x{o9
z>Wl{sBGkyg9iK}WHHHz7;6zG`sO%~jGn>6!G^K~gGi?8Y`oKmdCA_jMBRYvdm&Tw^
zO23yeJe#mS88)j(s6ij!$$AQxoBxF7Rm4n7Ipu~2rDs`R^XVJZVPHlva)|qD6`h(m-a8Y3@Yy-!_$lbyzY#k-~d;;lQ3ujIt?&=F?6{P*>ufS>hRqe6e-`Q>Ga
zlbWOJJLs%5)pS8VKZQ}c5Ur={y8z_D&y-p73Hh^IT2Ck%%3&;$oUWuY#F@{|Wqh14
zWry`kL`)7ph6)awk3e^5b13ttzB_kaY;(WO)gSopk^9Iq!@G~+xwx2r3^eex;=@y|rLD7ZM6MFBw}^Qngy9D
z)n^*PmE2(VE`@(E4oxm=*86jS_!Ay0_*8sn{zr{ylD)WBQSET&Oy!aq6l}UF)qVO_
z()v9ZZe3r+l~4(AVfLsobR8M);D6`*?da2{^|sJk?dw7}RyW{>K<`Q!{{r&BJnGl&
zPh=jtK3aqEx{FkWj$E&Fq*g8zlU>A*AT{2Rl>5BxlcTfJA{=uu4Q_8#NYo=-&DX@N
zj>)bsg_P}6gLJ;%;RNH>*kHTAkCs#QiA|kCRX_UWluo%_jDlsU9r(ng7B3!qgACPW
z&?{}9%f<*U^xK^b<=!jK!B9Vo#OI9y0Ea_#T`#T3ooKkcL#o~}WLj1|Qn*(3v22@|
ztlj-xQSuX<;H>q|RXz1dcep)xP8^ypAwuF6htGDd@D*MM=;L)`u3@!zx&Wz1OwoTB(*}P2
z<8Eix$|ljI3l86$kw?&goNZ|N+)=G4G3-Oh?|<{v+D`q;3u^~$@Ig)fU^c#n={hS>
z{%O9{uv&Pk54)==;s}Ti65Ypc-My
z|C;wd|13{l_5ZvXJO#W=u>Wh3(7h5`pTQ8#=gp_j=l$PjZdEKEwcO7EvDnYMM3^gK
zYITq*Dq{JVm>Br5ol->PlpTx0o35DV#FWJ~Tu`C~-w(UIKP8_Q*Ysl@BEH-BJWnK~
zqtMP3oz})>#TAX0GM`(HW(6s10XIMfp&64-Z!~gtkHv
zt9i!P0*ht~1YA;ClJso&F;w`a!+nV!BQwBSJJp8Nm+QOuyJD<>=}8o|!QlZyS}aWXXS-5?kF
z597gb%;2w@gbAY?hg~fBke9z)=CqDo8{@XpB&%>Ai8aGmSA@;ZClxQVC2uiH+qKr-
zEgD>hLNgDH!3Mc?*XS3P?Y+4|;-C{=?A91w<;UXvig8^?`TYJWp5GCf%Bn*?ZZ=
zB1+>u+5l->1SJhUm_Kssh4^^0>ji{8nVpIzCt5!nsfo01f=6$WVrGzx-gjj9`@Zz#
zP4c&+RYsX=FL4`~m}nzvCvFT}xuYB+xqb=YA>QqX^y~iwzw*Mh#zffSg9e!DkJ9J{
zjN?YqXnVo7ju$HuI^JOck8w(>3syg)N7PiePhW8h%(;P>P2*+}Q+mU8b;k-d7vA{y
z>@o&M*Fycl&JDRt!PGT#D8WrH{p!82}1|A}7QSy)#b#7Vpk~pU3X?e9hD#s^St$OH_Jtgu#|)HEZFDqg7(NTk_}aqZ2y@l0wxw=hG7^G3&4U~CeU
z-*6y9aMb}c!UmIzliN<_P>r57)RywaqW4ow?3A*RMgFAQ#L{N8x(@*`^-SFBj~vDk
zfT_KO`eI=X<)&H6YvP)d)A!FtKp=uSUSBA_?RV@Sy+fQ`nth|k?J6CV5Xwf8C2W%G
z!vP?_NqV4X;xclRNiui>F>-1MflE3H7QQuTIlE
zcHNvRmQCFcy{O2VoWla#i=r@Ts^b#Z9XwaS`xrgSD-GfKm0$-5RLT_R|n3A+q#%p}Jt
zkdeB8TdEM7r?|SO1D|-^T3ng)lDszj#SP}>_6)H?5)0zP;yODthKAh!go1`xqmr1s
za_sLMF81w}mW(mt>R00WWF|_y2j`_$7o$f)(nuB3tLX%mL8O{!hj8Y-to9e)GIC4|
zlKznbf0m(`NdX}!4Kl+rbV6T#(Tab08j3Sj3v(UEx3}E9CWAz#^*z
z6bv~ctGPkG7T$b||3om|h&5fzj+ewLn*)F6td5Vp7x}A6_z6+t@mN`lUvH
z6i&)peG$iV+|#>$ycH5&e2ITDs(In{A4bvf6H{0ztk*;L%>u-w4OOqUwqQB)`td=F96q7tS++)h_bJ^bps1yM{J!`grkqT
z&lvm-lPHb!U;$6-4QREpLA(bDXByasDR6!-^}O)*WC062l3ln}oIG*}Oo3$3G~lmK
zQ8jfp
zrE#Wv)mix@6Qpa^$9|>Q?L7TIGK=R@I~!&%ZzK%46Nu#Z#p0I_uZGMfoa&A_y=;+&
zu6p-ZWp;7(Sm$T;0!76^9J$3w89P6Bmt@iG)yo%Q1WlvIX@kk4EW(Zc92Kj&UfxtS
zs0tkloKb_o>LTSM?A`qKX{=QjpVt|7y2?~rpDgY-;@&q2lk4QEN-eo9G@l3GA
zkH7Z##?;61dimQhq0@G7lkb+?Tj{uHcX>h%G1YhkksPDocH9S-k=GtFFXuifEYm{_
z!=f4ky|Xz`Ia~@I5?e2}@ST1`j1bfps)o;ryn~-H%?U~w%}WYyhedpRMm$YGCY8%9
zAhj_<4^g?%DUv66!3Gkf)rz#Pr+yp3{_t!SiW9!C;0m40WedsZjD1BT
zSU#9`@tb7I8gB30X~?O0<3`8t?pkkqJ-_^d`Jml7E1xY}#+aNQa%g8p|y0PPj
zri3Nw5Gh44LcAaV-iT7#jfmZbg%YTko;keEk|7)wKm~(lvuxXsmklGM%p0vcm)Y}d
zFL_q^tUE%}S1tV)pG+Vfs`J|ovpsS+uU*UW=DhR*G*ILVku>b`>!w*hT=pCv8y<)TK<kH;D^y-u^;g_LN9sw7E%&^59qX``5%HP>{IB2t
zIY8cn;zN+ZmWaJ;p}0afbK3l%SD}Y*iN9oky<(@HYza%yK3-$Wdsl}f@J58Sr>slQ
z0|Sk7wX?{dZ6lxL?Y@ylRPEE>{N8z>k6BDlnRWmBGgiM`o(0t0CzN6TG0({3R3>`E
zHaGjMM0t0+uCOMpr1lp%RqszCoaJPLcUHd{^gQXY9E^lBZWj!b)uV
zfMKZ7$6Tq2sSS|&xw8HR6#$?isrfUpEYd0?Q1GqPzP|IJ-AvMzE-%$0jUDh^jb)o`
zQ|GMzA)wTL#jg+uY?|c1j;8Wyce^d2;m!**CVSpjaU}RwJACbAuXJ+v6Mx8Iv)(0s
zL|Gv6XJMsQ-C|L;b}pD(WdM9n#C$D>#Sc4c$WZHX4z6keyFylGdBRPz|Si~j#C
z7VCq;wEQ3U(w(Rao+rHFE3qWE)u5EpP*Dr~MLjwGHJ=fE9i7l%T`B^hRJ2t^cOI1g
z6KDQ+&{vdJboCp18Hyh{EU6RC>J(iNN2(XmmQzRN#n}C(3HL`a(TBPITwxOTPYv%Q
z*@`P=Dm3JyEOR1s%>v>K@MFe=G7V>1!;}||RDW8zA_P0)x(PU#Xoj&B+Ifo<1c5VNC@Jt;IF{aSQ
zcKBDVZwI6`yfH4}wv>h{bC}MN&?IkI8UR4y^UsO$mGV>qWGL
zqC^F1SaQ4!U2{)q*?eHz9bbGydGHy)QYLTL77#%W^gxQW)wQ<}-$-EExoQKG&13AK
z8uOf5>gLsE^UT}n^S&i6AJ5P)M;Vb(hY7(yrI*$0I>?i0)PCmd#`!FF79Jh79No%*
zqhtY}Xg-vg&p%5js#TyriXX)MB#-V5Fj;~aYuO_BJL8z0;S^-SP3O1p`5gSHk>oNB
zJx+5l(FWWq84j0UiF7sl%W1e;$mg&Hq~KfpSnKP;QUC^v6eu9b-acc@s72M%n9@f=
zk%~JGGd?N%_4+MMkvX;4DThlZh&vQ4Ocjw)#(Xpu3apjY_Ub*8@m0Bs7Z1xqii_z!
z)jhl$&VBQ(CGpv3hQBh$FP-IRy`l=VXxC7%`{k(!=d#Ez=0oP~=d;+UXNpCP-);NM
zebq*WggJP7^dc5na05bY$UJozaObBU2`_-S;PMQr
zSLhfrBml+VW~MIIa4ygar?Y40R?qYuwUlI>zs32qW97_M=YJT@SolAxV}l8fiKt@ELlF8k_K<=RWl@Cu2zRz)%3z}sJ^GVQrYavs`I%;65Kg`
zt;;;dCJDGGu>Ar8b{gu)hdeb)#GU$ArHmKzJkP>_x!J
zoabgAnX6#HRNWAmpe{S7KN6%*iG?pj7!B-9Sc9l-mM*M}M&|OrsQPq2dv<@DEE6bTlP?umEiJnrYuAGoRSS1nK{O(|yd@YFgN?
z)3dK}(j1+nDkx&7T~z6==1EU?78NWrJhiTHk#gR1Z>my3Z%0rtq*5`P5NK%Tmzc%a
zZaBR^(^-f0X~7nme0f$yjuhO5+2UDMo-uhf|KZ>4vFKiC$Ku`4|9X>zbv%d3gtLG3
zphUfD>G{2R8u)^m`dT+4FO;9Gx3~IQ%d%W_MhM;}xUAK$NRBn5ZAP3JR8qXtief2t
z7cODfve9JeuiL*%x}V5RcsiODOOwq
z9}<7c)@O$*k0o2C>bbvd=>Pbo8Sjr^zk-wz;#D9>YW;1p8U6Zh-20`~;0AmtS`Ok<7Rg-CLUiWYR${2h_;XW^A_^bYTZdA@ILJv)*WhtVAA`bG~
zPY}k8KSc7XhF^n4bU$Or^mC6(y1yY(r(l{JByw`jNT;U)$N;qGdF`Z=XBfbS3fl;mjy%7S%Y-PCIdtlN%btgq1duo!yJY1QkxTs>rQ&fIG64H9`W$7P
zsCQ}ZvO}tY`AL86SS+iaJC-^vW~efjx@z1z)7-^$5=3r>#CKDvZS>3V_D{Arq|M|g
zX6cFKQ1^>Vn0+VYqcX^=#6DJQBm3u^9B&=V6dD{9(r)Xwm}n$3VVXQG(yu@x#L=^_
zoT)WFsWCo{otF!Xe=LGEmq49kdj;_)6|{y8@j-PV
z+ZT6i3&Z0bfgI||-=e9L_DC*+>0HO$S@f`QHbDYArzlV)J8|Md^qOrvw3IYQTZm;K}$Jz?O+ukYaABM?eK
z=LAZ{?z-6kWAvu8tQV=z+fHauSG;O%;k%_{vON^}#LI4H7Zrkt`y`65{a2I#2Rzj1
zy&@y2?3)YC_??p-YeCijBX%cq+*xSOpx2AdyM?hq(qM~~Cmv3;i7+X8s0K|;%Gnbzj&IPc}adhzd#Ji;grfVn4qrO!)
zKwoc6$rwbLY?&s6zdf>k@fT*-{{^fm9op_hJ$BtoL;ERrLb`}W=KT!cp`+fBv
z(|6dhh2^LIJ-u(QZxuA0p!Vpe=o3rn3ppR@?Hr=%ESobUQh1Th;r}%*x_YBP5@+UK
zx$9EM&T}h~ne?t)Xm9dq$8mvHkb@Ij5*>Dh9P1p{(p*3-&_19HBl
zOv_Gu;f+vuxpw3m*tZv%pg#@6ya9w;d4F1rhB
z9ZT)rvO5--!^yE7yV!i>C&x}*#0|@nTjYO4$=eL7a4>5a6lZ%t=Ahhn94*Z7#AGtA
z6&g55jkB{0a<$Wr$-LQM0&8@>wHVhgC^%K(v4GDSFqkmlC?OGN08|TE4TYzF6*Y6$
zhHu^Ql(m)}bU=^#P=&pR`fGFiGZebkghuUHV%sYI`27Cj`~NHivJ(lD<5Y?mE{eA5
zl>aL+7tpol{~bD(bz^T?N`#BQ(iRc3ca!YOF)6++@8m~UlN=H&I1DPI1}v^?X0Qc5
z?L6&y8&dqi1GTk}uka9c02NX3M)a`FK52#ZJ&@eIsL028E*FcUPhFv%DnQGhO?`+7
zo7Jw6Lw7wYeVNgHr7AQVF}6o>Sc+$E=mvn~=4Nzb;k4?K-@S{iw#caio+0Un=D+LF7tNH
zQI){W<)epwX5VtGTP2suk!Rl7$vKl|E1ye~u~M@bzZ0D)c1A~;A87pLPmD3kb5XWy
z)u_m2{CF1i??79$$}U=Ii}aldF{e+f7Cp;#!xrls4hr9Ft|Rz;t0slJLS@qTR#gg-
zJc@S|BLAy=w;l&$JqF8<=&#>V2q0FIMV(_1P*iyQAm32pE0fpbzTZ>kBXg;Sz=H>0
z1Xnt<#>=S}xf^A_Y6aw33*ZG%oIRJ#Eho`WT6-_d2e6~PF~BOEnWRxRUy#LrBjh1K
zyh0U{H+ajOYEvj2KE#lb4rCDak4z|M@RzgvC>IshDSoNY2GP_fK5S5DMt;yD@Sq~n
zjA;9(!|2SE(C@SCtX8Y;@e;Vm;W21y_NF^}a==0c;i+tvR^np*Qr9f#&F)`jR?f;N
zt)jfW#y;7TlD?1F%lgr0OUOBUH=X?!rqn3fOT!#KGU{hgvl~pei$~7+!rB!F(&S+tO
zx=sD%KA?BdCenedR)^?AUQAn9#GK&3=$s(qVaUv*O1*bs_5oh0y=d(gxg&0kDrlR?
zef_I&%6a^zslz0aNImHR)_7d^K+c;|DDo$kOFiFaKY-iyVpho@D-cn84e~?55+R6;
z%h+rGbO+#h~8Jb?D=uvoLA}2dx{LPk2-x_sQ0EhZo24>T@q3_30FEDuvUa6Iw)YG-&O75UD^-4Dhz?@5vfW1V~=AEGkdhlv*<>
zPqgleoy|P14V&4yA2L>l33fXuh=O7euE)9;G<8yj)~J#y@goxO*YHgb!bk}&43Yl#
zw0_`ybA)%LLOxFoCzXwP?Y9nU=gv{?x0n_pW`tn0`Gcvy1#qciqFTHMV?P-p!D9=R
z{R`@27+v~?_X|^iR&f9xno8*p8?rgSM_F~?8P;&9J0=H&kQiuzVWp*u7$p?&PP*0~
z=Z3GlQA#v_Hw?W6*)Jc{D;!EY9ux@5&Z-n&`oB&|Z38xDi-H7~ffi-`y;>BuDjN~n
zt9sN-3z5U|jXhJ-#M2*aS}j?krzu3na%@6{@V#SM5+Jvdj1Bu0TFW0*P_8!7R{Y{j
zuig1246~X(C*={ktL~UF-J4>O?qT>Ed>VXyIF{`2r;DSvU#U*YXc{LYV}ktK>Sne}
z7!T9Dl)OY22P!w>nZIhXT~515w#%PWbDY?Oh!0A?)w+H1lTGDFZzOujXm=Glb`Z~x
zSp*`UnxVQ8UGTxNCZ~#McRoEzy&DO2
zaCgo~LC7kGj>|=m-okE4EMiJbpb)GhkC~2DPc!AgKz{e~e3@Mr<44IKIB7hyQy|(}gNH3vxk**XW
zG)0jGe&TIs`YcuAAo`Y@CyJm0tLPN>UUx&=?HNC6#JXXzJ_?nQO
z?yEmF%xE#(E2P8eYeup3G
z>NIQqf)pWwN^&qvN&%Bb$MTU@fXU%QGt03Tn8?{djzj%+?p6Efmje$5P{ULN)E~x|
z%f-H@Yu%Z@1FP@Z>ROBbPzl|#^FUJR<{-=`=N}crN>c0&dTr%o>Zh93~sF;s+;{VslBz~P<^iL?R^@Z`O&j!y6eSQ&6ddV3wJd*&Nro%D3XbD
z;pn9--QxXd{lo00gg8xmKL=Znout>xDXTIcoEW#1o8N_*qGd3$f8tm?VN+-si
zhp>fJh`q%ilyS$=RZc_T{D{ai)>Z0SO=m)JuX>bIvvzI3Ot>avs^Uu`BC|Cl9ft#DH-n#i7?(A`eYec`sMXD{tXp)^ar1tE1%WS@Rzk-
zdMS5A`UDKw;qnYJJxm-{Fg
zrQYES@Wc7f5t#1DF@OI#vB%gl*@ppPv_jr0v?=-e^L7zH>XEOk9?vqjnvLU*S4($JA8+GpL4UoIf^-vL-(#Vcqg3q#()Hs?}s@Ly9u(N%a5jY}S1
zLaet*l=)}jnNg0_9dSA4uAesdBW^v7DIi&bg=bq}xRJ^$Ojv91+i#!M9lWG7CEs`D
zjngp9X)!-CXo8o$_C2Kd+!s~*Qf0ua=|e(XuewsSwZhF~7VnSLPZN^9t-)-t-_PQ5
zW{gGeeNI^5tK43^om|1Hx;m9q|1B#Ol}nnIH!aIHcTeg;g&p=46I=TTOyh~11VhUz
z^J40*->1SuJ)><;KJA>CA(uD)rM{%&1J6k~Qff#M+p&vA+3_dwV&Sq<7rGDGgJr1I
zZ0q#iz71(|J`^$6)9{_SXS3>&s=6uHaMQkoW+({Cp>7o^vGTP{^?M?gR`1oQy7xnd
z;7$A)y1_(WLs?soTF~Wp~s&}+T~5$>uHYF
z{MoH}BQ|ab`aWupMHijHblks~%Az^N^&rX)seX0C9W*rGz0G;u2FoZzJB_8MY1`2AR03F*ltM)J?ql!rz85yn=`G;6yC
zVrOfZ8eC+G#JTk}=j+#nFO!mp&6YQAeWSTCU!!}Mvc$Qy(jjkJQvod=&7?Qs`+M=1
z@somhFm%eB1#(i`z=9Sw@b&j89pJ>{@dFzYj;ai`choV8CWQn-%a)Fr;EqNr
zKa`o0?BM!L#A-`K043hQ|`0YPL>G2-=+sYJML_K4EUA
z<9#`BCZdMxwhs9w3F`;-AJXw*KBq?7HLoSt52~>W6MT0eT_$l_En<%y?g^)jE|rr<
z*Y`5N7*Z1t7}j&R7&gS-WFFl;uC2M}85*0@h`ZWU=a8np3VM1S8ZCP~%D-i*_}lz!?AEHbA&FV7i?k-s1}MB3$KIjwMDD##a^Pp8NK7bFCBJOt0a5oiH2
z+rYt4phLhYUH14!L@ef4{e_nXF@W%@+VRkrsJxXaFjX1zZ-
zjjgsEB^Ri;mVf@d6S6y@f_N~-)WS$iG%;Vp&We}C#81whdw1S!*)na048MZDv}2ek
zt21u7r?wsXA#(8HB@MKDp}XumgafPApADE<{ApUPRN=9PzXmx%Ex4$%!A#4m%Jcr+
z&80x|O0LC%4eW^Mc2-F~VVk$(4)Rc~y%eS}L@+}aVjP}xGptt~U`9;32&Q41?KK?b
zyv@%m*bZh{G#k?$=(0}3N~Cn&6{}S0_p>Ps=1reup!^g>g*w<&1GK0Wk=>$k_ZfB|
z>(oAUyM?51eAJI-g~2J+fT2K*B=+66s{jSa#0F$D^4ez3
z#{$DuaXzOEfKc*N5ANToN!d~jdRl%WASy$f8lkzZnpaC`vwi=egDBdWp+iii;KKTa
z&lbIK-;&aryO)drHZ8STMMc~`P(NkxwStx2zo0u^T6S~C?N#BYnUjlhKVDm@LA76|
zC)hoQT)!zP*)uLvdtb$H$MTiyy~#DI3(HOydu+(x0y(&RoHA~7ZFp@FnqY?3*WPS>t)Sq!sTrz6cm`%EHp
zqtg~CdCD#fv3W+ey4$Zl!Q3D7#=FnAby{0+rGB;lR_*l}a5$eRz1N;(ro=yIDz#<*
zLL#0HNjjv9Aju`ZhllGz{#Y-Z`<F>Ie
zuuQnFJ={l(wSBf&Z7sN7r+DsJt$g=&d4yEW+Kd3}MPL<==t+g>&yhixa^V#VCRJUM
zodD7i>&tvyEzilEt}4}?du~6VK^L7^^B*0nlamJ7>|h?II=9MF0Dx}AXC~DMoH4Fx-8!37tort^4BeyJ^Was7gCkK{N$1S
zkBPh?uF(OjKH@HfYh^uPyS_5?Ov@tA)7QnWCa1dP+q3;RQR$0Kr|eO_tgU2^@R=5T
zvW^B+N5$yNcFxZr$5ljt{IBj0@XKPOqQMHB44>!W^m|
zS3$rfAjn)w)Xsks-DNvx)9zYz*z9H^R+$lTrtPD27A)W1
z{iTodb`+6qZFpe+sduE@KG$a(JAYYIvVU6Tda$^CK9zkTV)E_a%kgIS{7J);!*Sj-
z=F7OLk~*;rzre_oH%e3k%JWfEaY^R_GWIJ>?(NihXClth$md0e0*?N8^=CP68(w{$
zU-KSHen%5_U@PV->r~+%_crR-`+lHkY*}8?RdIgEkGv5sA2Y|YGtEV4W3%+EKXCB;
z%J=chou`ui^7q({Qohew+fhDy``R>Rq*e)mnvABL;Mpc
zu3jn-8hp!Jar51{d2n^I?8t{p7=`N5llO9E<{YKuI(}}I-Ahd-M`h$j!j(}HRFaL?
zT3>RVGqiB5USFC%Z(Gbv@}FGW&om5KJtw}UGKiZp_MG1jH=}h8sZR0WH6=z>6+9m_
zqyCBB+=patboirip2q@B7d|-)GO;>!+HQE~XhC5;HUu^-^V+bH6`P^9stWp)V(g74
zkP7^U?~^Uf90yBb)L6{XhkE`@bN61&;YJ7L-dnHu1G~G$*q`!hq!pbO;G@OW#N;P7
z5A~g7uyHt*7M&K+bm<(x-`LWJ#t}IatSKB-Y_I-xAmj6dj+T+(CS`1q&?W=Tizf~s
z^4!!$3?^tTB{=A7!rBW}(Pz1G@?)23578bk&I}xC@92Xza>?IqQiGz;dn}S4Xx8v~1ZecV_Fk!;
z5pGnQl9B4YIQS^*p9R9}>Cvn5QW?`fC0M=l1utP5Qx8cK^CkK|j2iOF;=-p{)>2$Q
zBn@5MvACz02vf8s#tKZ_HN3!TD&?w)(J4u`;G_0<92XrbR>@S7TmtnAJ1}Mp0XC~!
z03kGC0zuN9bi9QxgG3U;-U{wK8&?`!W@XefhMhtt!J8NseqVKa%IDXE>akbOZ*v@~
zr-oU*<@zme6aO4^H^La~ueI81Fi=EF*&YKA2xeW~ihs_o0QQqLM)}FF5hx<`ALJrw
z#O~rP0`q239M_$|eh5e^*vbFuI)^$tpm1BD@K)3_j+EoyjZef}+5LCOc{`3pRX9G8
ziIt0-SF&M@Ek@4~BA+2+u9W&LW^Im!A^0sE>bM$kT1df#p(BtDxW}~m$X`{T5L;cn
zHh3ikngOzQ@a44-ykX&OkfA@78t3z%eOSp}8m6ja5&!0}DX*X2xy
zA7=}Sxr+#tiPp>*&tLQ{F!@Xka@27Miyw2HlC^xl0HYesoi|?LjEMO0%04l%R7+RF`cR2`#I{C&$vBi{4#mz_6LYw_RzHe
zqS@4m9%z#A?O+0}H-LW1S(`23H5KM%uPTYl4xF8?j!1Ns9xQFqC@(LfYf0fQqZahu
zQokSnupqW%*+cB3<+jH>Bb8B~_sR#B0hYqLk_xdJfPuij5}w{T*9ltjb7E@hIe=F1
z#f4;A2rQZ;CUb~rsD5H1-MQ*{l>2itFE5G3&LP;E8rt{!M2P2G3vB-VX}6}V)BSHt
z5=w7zZ#2XS&BtN$Upm5KoLx6&mI}Nopr5KACHJI#*nJo1hHA3LaN9iiA@lQKxg5=_
zI-Apaxp@_hpGRFAqCnwB+@*Fh=WZF;Y9X2Odm%r(S>sQyxsM@dziUUUHbx2Di2(uWUJr8|swnCR<59DKcb=%})83G_(WR&4XTrh}-zDv&i^+J(
zi-gSb5lU8LjIwmkRB0SQlLk>4G{$y?D4=V^}#=g+Stc%!_@(v7(IsT^KBVOurZNK592K-j;sGvTPJoIdXZI99B9Yj1@K1
z&=U0*LeCF<`=*!+!|xVw-^n`NaxE*WNNP5>G5=}Tqaeh6&t7z_+c(Sj%kgFg!AXkI!Ob@Uhx{Oq`_eGmR7@*0<;~(e`qtVd4>4G
z(ui^EXjx9e-y~=E>eg~QmYX(!UEYk1H%XURtGHY9Y6QB~t+n7>IBo1%dNhzOvXpm*
zLagtluVqiM|4=G-YpN-$ZA;B=JD?^fd7fygJ`qNnIaFg=boL~&->u5|5#ngwi{5t)
zTO&5FzwDX1^!{-vy2IQL8IV%uXC%&ou{_k@sr>G@_fMH!gi|swHa1iBQl=P!V7l;zuW`hnIEcHofcDb?r^Ni(ffbxfDNfk`KOq
z0Y!R8AWg^ov!XP~XNHHr(l7M)$O}N58Q8Ox7R!s{JQUbk&FVXCj)JV{|ANj6A5u&I
z`sCT>R3UL;^teOI?rn#-y6r7Af)eyf2rYyhMUZ=wXqnaE`cUg1odz3ow$1!Cv|2nr
zcAhWrvC@;#God5YcZ)od_)Zpz{
z-3gmx%9Y@cT@Hg+CO|JMK3#3fJFUr-bw9PNki
z|NdO0NbEY`oBdCiTjO(V+hZ6il2S}zk7f;lvwdK^A!PVDgq^$yIjQE
z78l?fnB|G}{BM|z7KF2fD)7z4%tmDLGA{<3-Tk%cJ;vl78Kj-{5DUlkOq9MHlz-6}
z_OljY_3JV*5`O6`^CHon|E_SMtQieH|KW&y>dzmiW#qgo!=Klw-Z2xHtgD)UDv6zB
zu^XDCj}@pP{E
z9$I{!!Rf_Y*Rfu4OdVWF5{%v(vx+^1QqqkCniZzEX^t4Yj{I%G2gdEq-ijShIj1m_<0+;j#H6VMuWPkMV8Tva!5N
z#E#_t9DkcafT8`}L4Y6())K@={&<%<1a?$o5Z78gPi-3pjqDV5%cO#T*GUWIHU|9v!tyd=@|W6)10G&ns59ih7(FZUZ|2
z_?BkW<#Q@w{cL*&8S@1o>ta6~H==-ID&3FJqY_Yj@VA|nAE~biA
zVru^?qf**3wNK%ZA)|fxlxV8qQ1Dd?R!&mVUsF;N)bCB6}
zQ(rHAU|Jk;Vp)$?)dUBdJq{wx864POwtBvnZ~95t)H8<RLg
ztvcK)g`jvAo|k#WA|A*k(kVk)OD=P233bq>c@mVUs%9;yR(ceTESr{PK%AwVd8Y5A
zpZ(@6+eyjuIz&r)7$h!7A1IgWefpvz5M>HYm!W>F2F?S0XVB1!qD
zC=KB#2__gSE=dR^4Jxf&`||}FB^9n*%Qq8<6rA`?E`DV15U}{mq$B`ub6z6*4GwkS
z|Lj>SF~&s}bag^8h&j^7YQ$ySTM}W#`Lf!+Jby+`7Bj?<*Iwh({`$9Ax!%OX5M2%>`EEANtJHLlIs;J0Tylvh~>xr4I@*Q(POoBm-JaGCBMvT
zT=kCNmr@aNcr+BA|Xv0&xao$@7o#e#H(qO!SxvqmP{iOihcszFrh
zR#5Pc5ou;X8!6=|+|koY8S$*-@jNtr>OQIKpi4n?{zD!ao7?e)mIehp;
zPR&qg<*`hmN*ed~%suZwlceZM6(aQtC+UZ1$mCh1KgBiq;v_CRYln^lGnb7a3G)E;
zJFlEre`%-H&*&uG`MtN$ZMonq+N6__C0w7Wmt(p~ohk{M4`yWi6Ku<^U@!`E*sdx`
zitiU$4O%~0vRpY}bD!M#@=_R|dBi>El2r$eTfT1?Ug$r@7o@Wn@i1)no-H$0{
zOtMa2a~KX$QX=djonhI-m#sUz(k&HrqVzaE3(etje%6mC_
zE-(<{Al6ThrjMa4rohnS@e#*ioAV5kFM$Rmxe>-FuXck#>xxx8$R*%L)a8h^rsP>h
zoMcC>{(dPK0LEA)MkE2SnJ3+15OaC*!I^&w7qTz<*_)n~pg!V1$c_82Elwm6nv()Q
zM~}8w2;!Ba;a}Mld~6v2gB1fB=l^9exXj;yiQ27r$7nlQ0sqQO*IM6f*4oY?v=sIh
z&aD-xG?+mp2XC`k-N`^kXRZ?4sv=B%FNPd#L))a1~lTxWd$e&$Pt&yOWDpk4jkV6v(2&JG?w}Ee$rl1GwvHAT~=ez
z9VFqX^z!E*4SGlWV6l`A;YhD5Rmra2jI(p#8oI!k0cK3wnMbmQ|#@O+h60E>KET+qv(RzV9J#F=JfA9cMM+unRRQ;5;!C*9fZF{^dkg
zj{{55?*oQ|o9!n>-T}EfZeEzI%_4?PZ(>-&K|4I6a5b?{NgKggVr1&=*0zB^$%m7ss^nH&+ijkp#ZP>sr`IrUB!4
zj{P8g+Zdty1BGpTFECB3Djztwi5@zg}gLnLI!e^z=Gt!Mypr{G6qrc3k5wwW9w7Fk)t
z-~jBB;$AxkP*ccTE({ACFNyVj7CV>;v5?nO22N=obRsiQr0=xgi;v$AJQxEBbrhxW
z`)I8a3t0wvnUhwTjp_5%wc7QDa7%=hbul)hHU0cd+2vpVlPC6$~JDD;KGX9}vI
zOgZzj!u7YW4;~$m_sg#dpI-YHbYrPK!_cj=#&(OWeNU9%tLGO{=tt6>kG~5lX2k+3
zT%P1`yFN;;Z=Bz8aDX|FrcMV`&&P^$RnxJ9qL+%tag8u|(Zd2o){4Gx>1_}r&D
zS|o@S-TjU<_zPM_I2?886kO}_9fQFgU22+_O+q3{LlR_soY(CLxA;a!H`TaVy%X*b
z)eJa7c^y(r790#>Kr08Z4HV1)rW}LE)I)$WA4U|U933WIvH`bt0<&pQK{l(v9d6kaCvE@}
zWCwp4;0VCuYT3^4QUVWh6bIhG
z2=D;c;Rd+m>}>QE$@&qBfIu4?rMnP|jbdPl5(u=Z$T@|8MPR<&zCHT*7Vw;;gHvh@
z2th&aXGDGhqH^N?ta)%yPZmF*
z@u`@aZxsJ2QUO196J|10%X^O-DW*)MN>1KbHOtPrxRx(fW6DqGSIS-f%Y{zMH=s2{
zsp7}Se(&ST&CKN0FxvD~8Tt<=ma4j_k>KFa`|d&4UPkGGSLii0K70N#{RfiKyt0(W
zw;N5&Zm7dS_W~yvGt!eJ5}cl9na!}bFY76PsvJNK3|$iz2i^PlED{8XIyl$`VniO<
z(y$bvy91x5FRm&Xl%(Ang&-={T4^JFN#j;Q)P4MpUmd75Y&J8e}X-H;Z=}fY_Un>5*
zI~o>wb95;k*q72@1`B+SGVWfKCw9S5QdDjruogOcTO+6i
e`bEAsjSh?$nJzA731Bca$DHU~Na^g~h5rF(8_Q_`

literal 0
HcmV?d00001


From dc1d51b2df61c8080af9d77cae3a404db9f2231d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Jan 2012 21:54:51 -0400
Subject: [PATCH 2958/8313] formatting

---
 doc/tips/visualizing_repositories_with_gource.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/visualizing_repositories_with_gource.mdwn b/doc/tips/visualizing_repositories_with_gource.mdwn
index 02cf159b2a..884f70c444 100644
--- a/doc/tips/visualizing_repositories_with_gource.mdwn
+++ b/doc/tips/visualizing_repositories_with_gource.mdwn
@@ -19,4 +19,4 @@ and use `git annex log` to make an input file for `gource`:
 	sort gource.log | gource --log-format custom -
 
 The `git annex log` can take a while, to speed it up you can use something
-like `--after "4 monts ago" to limit how far back it goes.
+like `--after "4 monts ago"` to limit how far back it goes.

From fc669100e5fdf300e2cfbbc1a1ca5d99b03cc564 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 8 Jan 2012 14:37:57 -0400
Subject: [PATCH 2959/8313] avoid showing remotes with no uuid in status

The filtering of remotes with NoUUID is done in remoteMap, rather than
remoteList because a few things should still act on remotes that have no
uuid. Particularly sync.
---
 Remote.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Remote.hs b/Remote.hs
index 63d32f4295..2716658388 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -93,7 +93,8 @@ enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
 
 {- Map of UUIDs of Remotes and their names. -}
 remoteMap :: Annex (M.Map UUID String)
-remoteMap = M.fromList . map (\r -> (uuid r, name r)) <$> remoteList
+remoteMap = M.fromList . map (\r -> (uuid r, name r)) .
+	filter (\r -> uuid r /= NoUUID) <$> remoteList
 
 {- Map of UUIDs and their descriptions.
  - The names of Remotes are added to suppliment any description that has

From 7675b83efa77527c0b04b87e35f83df67e6af8d7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 8 Jan 2012 16:05:57 -0400
Subject: [PATCH 2960/8313] map: Fix display of remote repos

A change to break local cycles made remote repos be dropped entirely.
---
 Command/Map.hs   | 1 +
 debian/changelog | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Command/Map.hs b/Command/Map.hs
index 0f32e1130d..faaa893924 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -155,6 +155,7 @@ spider' (r:rs) known
 absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo
 absRepo reference r
 	| Git.repoIsUrl reference = return $ Git.Construct.localToUrl reference r
+	| Git.repoIsUrl r = return r
 	| otherwise = liftIO $ Git.Construct.fromAbsPath =<< absPath (Git.workTree r)
 
 {- Checks if two repos are the same. -}
diff --git a/debian/changelog b/debian/changelog
index f61a4c799a..5f9f7ee619 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20120107) UNRELEASED; urgency=low
 
   * log: Add --gource mode, which generates output usable by gource.
+  * map: Fix display of remote repos
 
  -- Joey Hess   Sat, 07 Jan 2012 18:12:09 -0400
 

From 9ffd97442b3f2e4e9de6895d843aee382ad10dfd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 9 Jan 2012 18:19:29 -0400
Subject: [PATCH 2961/8313] don't use GPG_AGENT_INFO to force batch mode in
 test suite

Fails with gpg 2. Instead, use a different environment variable.

The clean fix would instead be to add an annex.gpg-options configuration.
But, that would be rather a lot of work and it's unlikely it would be
useful for much else.
---
 Utility/Gpg.hs | 8 ++++++--
 test.hs        | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index 8f7e27870a..4c798f2732 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -23,9 +23,13 @@ newtype KeyIds = KeyIds [String]
 stdParams :: [CommandParam] -> IO [String]
 stdParams params = do
 	-- Enable batch mode if GPG_AGENT_INFO is set, to avoid extraneous
-	-- gpg output about password prompts.
+	-- gpg output about password prompts. GPG_BATCH is set by the test
+	-- suite for a similar reason.
 	e <- getEnv "GPG_AGENT_INFO"
-	let batch = if isNothing e then [] else ["--batch", "--no-tty"]
+	b <- getEnv "GPG_BATCH"
+	let batch = if isNothing e && isNothing b
+		then []
+		else ["--batch", "--no-tty"]
 	return $ batch ++ defaults ++ toCommand params
 	where
 		-- be quiet, even about checking the trustdb
diff --git a/test.hs b/test.hs
index 2ce3f8b0b3..5d01f11169 100644
--- a/test.hs
+++ b/test.hs
@@ -664,7 +664,7 @@ test_bup_remote = "git-annex bup remote" ~: intmpclonerepo $ when Build.SysConfi
 test_crypto :: Test
 test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $ do
 	-- force gpg into batch mode for the tests
-	setEnv "GPG_AGENT_INFO" "/dev/null" True
+	setEnv "GPG_BATCH" "1" True
 	Utility.Gpg.testTestHarness @? "test harness self-test failed"
 	Utility.Gpg.testHarness $ do
 		createDirectory "dir"

From 0d5c4022105a393a4eac76b09940f8b22fa0a56c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 9 Jan 2012 23:31:44 -0400
Subject: [PATCH 2962/8313] Add annex-trustlevel configuration settings, which
 can be used to  override the trust level of a remote.

This overrides the trust.log, and is overridden by the command-line trust
parameters.

It would have been nicer to have Logs.Trust.trustMap just look up the
configuration for all remotes, but a dependency loop prevented that
(Remotes depends on Logs.Trust in several ways). So instead, look up
the configuration when building remotes, storing it in the same forcetrust
field used for the command-line trust parameters.
---
 Annex.hs           |  5 ++---
 Config.hs          | 13 +++++++++----
 Git/Config.hs      |  4 ++++
 Logs/Trust.hs      | 31 ++++++++++++++++++++-----------
 Remote.hs          | 29 +++++++++++++++++++++--------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  6 ++++++
 7 files changed, 64 insertions(+), 26 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f1e46126a9..b365132e5d 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -40,7 +40,6 @@ import qualified Types.Remote
 import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
-import Types.UUID
 import qualified Utility.Matcher
 import qualified Data.Map as M
 
@@ -84,7 +83,7 @@ data AnnexState = AnnexState
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, limit :: Matcher (FilePath -> Annex Bool)
-	, forcetrust :: [(UUID, TrustLevel)]
+	, forcetrust :: TrustMap
 	, trustmap :: Maybe TrustMap
 	, ciphers :: M.Map EncryptedCipher Cipher
 	, flags :: M.Map String Bool
@@ -106,7 +105,7 @@ newState gitrepo = AnnexState
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, limit = Left []
-	, forcetrust = []
+	, forcetrust = M.empty
 	, trustmap = Nothing
 	, ciphers = M.empty
 	, flags = M.empty
diff --git a/Config.hs b/Config.hs
index aa88858738..0a7ac07897 100644
--- a/Config.hs
+++ b/Config.hs
@@ -12,6 +12,8 @@ import qualified Git
 import qualified Git.Config
 import qualified Git.Command
 import qualified Annex
+import qualified Logs.Trust
+import Types.TrustLevel
 
 type ConfigKey = String
 
@@ -30,7 +32,7 @@ getConfig r key def = do
 	def' <- fromRepo $ Git.Config.get ("annex." ++ key) def
 	fromRepo $ Git.Config.get (remoteConfig r key) def'
 
-{- Looks up a per-remote config setting in git config. -}
+{- A per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
 remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key
 
@@ -67,9 +69,7 @@ prop_cost_sane = False `notElem`
 	, semiCheapRemoteCost + encryptedRemoteCostAdj < expensiveRemoteCost
 	]
 
-{- Checks if a repo should be ignored, based either on annex-ignore
- - setting, or on command-line options. Allows command-line to override
- - annex-ignore. -}
+{- Checks if a repo should be ignored. -}
 repoNotIgnored :: Git.Repo -> Annex Bool
 repoNotIgnored r = not . Git.configTrue <$> getConfig r "ignore" "false"
 
@@ -83,3 +83,8 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 			readMaybe <$> fromRepo (Git.Config.get config "1")
 		perhaps fallback = maybe fallback (return . id)
 		config = "annex.numcopies"
+
+{- Gets the trust level set for a remote in git config. -}
+getTrustLevel :: Git.Repo -> Annex (Maybe TrustLevel)
+getTrustLevel r = maybe Nothing Logs.Trust.trustName <$>
+	fromRepo (Git.Config.getMaybe (remoteConfig r "trustlevel"))
diff --git a/Git/Config.hs b/Git/Config.hs
index d9109548b8..55ab8a6f15 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -20,6 +20,10 @@ import qualified Git.Construct
 get :: String -> String -> Repo -> String
 get key defaultValue repo = M.findWithDefault defaultValue key (config repo)
 
+{- Returns a single git config setting, if set. -}
+getMaybe :: String -> Repo -> Maybe String
+getMaybe key repo = M.lookup key (config repo)
+
 {- Runs git config and populates a repo with its config. -}
 read :: Repo -> IO Repo
 read repo@(Repo { location = Dir d }) = do
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 5d769bd247..f6ead87f1d 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -9,7 +9,8 @@ module Logs.Trust (
 	TrustLevel(..),
 	trustGet,
 	trustSet,
-	trustPartition
+	trustPartition,
+	trustName
 ) where
 
 import qualified Data.Map as M
@@ -32,6 +33,15 @@ trustLog = "trust.log"
 trustGet :: TrustLevel -> Annex [UUID]
 trustGet level = M.keys . M.filter (== level) <$> trustMap
 
+{- Changes the trust level for a uuid in the trustLog. -}
+trustSet :: UUID -> TrustLevel -> Annex ()
+trustSet uuid@(UUID _) level = do
+	ts <- liftIO getPOSIXTime
+	Annex.Branch.change trustLog $
+		showLog showTrust . changeLog ts uuid level . parseLog (Just . parseTrust)
+	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
+trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
+
 {- Partitions a list of UUIDs to those matching a TrustLevel and not. -}
 trustPartition :: TrustLevel -> [UUID] -> Annex ([UUID], [UUID])
 trustPartition level ls
@@ -53,9 +63,10 @@ trustMap = do
 	case cached of
 		Just m -> return m
 		Nothing -> do
-			overrides <- M.fromList <$> Annex.getState Annex.forcetrust
-			m <- (M.union overrides . simpleMap . parseLog (Just . parseTrust)) <$>
+			overrides <- Annex.getState Annex.forcetrust
+			logged <- simpleMap . parseLog (Just . parseTrust) <$>
 				Annex.Branch.get trustLog
+			let m = M.union overrides logged
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
 
@@ -75,11 +86,9 @@ showTrust UnTrusted = "0"
 showTrust DeadTrusted = "X"
 showTrust SemiTrusted = "?"
 
-{- Changes the trust level for a uuid in the trustLog. -}
-trustSet :: UUID -> TrustLevel -> Annex ()
-trustSet uuid@(UUID _) level = do
-	ts <- liftIO getPOSIXTime
-	Annex.Branch.change trustLog $
-		showLog showTrust . changeLog ts uuid level . parseLog (Just . parseTrust)
-	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
-trustSet NoUUID _ = error "unknown UUID; cannot modify trust level"
+trustName :: String -> Maybe TrustLevel
+trustName "trusted" = Just Trusted
+trustName "untrusted" = Just UnTrusted
+trustName "deadtrusted" = Just DeadTrusted
+trustName "semitrusted" = Just SemiTrusted
+trustName _ = Nothing
diff --git a/Remote.hs b/Remote.hs
index 2716658388..6a97c2da35 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -40,6 +40,7 @@ import Text.JSON.Generic
 import Common.Annex
 import Types.Remote
 import qualified Annex
+import qualified Git
 import Config
 import Annex.UUID
 import Logs.UUID
@@ -74,17 +75,15 @@ remoteList = do
 	if null rs
 		then do
 			m <- readRemoteLog
-			l <- mapM (process m) remoteTypes
-			let rs' = concat l
+			rs' <- concat <$> mapM (process m) remoteTypes
 			Annex.changeState $ \s -> s { Annex.remotes = rs' }
 			return rs'
 		else return rs
 	where
-		process m t = 
-			enumerate t >>=
-			mapM (gen m t)
+		process m t = enumerate t >>= mapM (gen m t)
 		gen m t r = do
 			u <- getRepoUUID r
+			checkTrust r u
 			generate t r u (M.lookup u m)
 
 {- All remotes that are not ignored. -}
@@ -242,10 +241,24 @@ showTriedRemotes remotes =
 		(join ", " $ map name remotes)
 
 forceTrust :: TrustLevel -> String -> Annex ()
-forceTrust level remotename = do
-	r <- nameToUUID remotename
+forceTrust level remotename = forceTrust' True level =<< nameToUUID remotename
+
+forceTrust' :: Bool -> TrustLevel -> UUID -> Annex ()
+forceTrust' overwrite level u = do
 	Annex.changeState $ \s ->
-		s { Annex.forcetrust = (r, level):Annex.forcetrust s }
+		s { Annex.forcetrust = change u level (Annex.forcetrust s) }
+	-- This change invalidated any cached trustmap.
+	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
+	where
+		change
+			| overwrite = M.insert
+			| otherwise = M.insertWith (\_new old -> old)
+
+checkTrust :: Git.Repo -> UUID -> Annex ()
+checkTrust r u = set =<< getTrustLevel r
+	where
+		set (Just level) = forceTrust' False level u
+		set Nothing = return ()
 
 {- Used to log a change in a remote's having a key. The change is logged
  - in the local repo, not on the remote. The process of transferring the
diff --git a/debian/changelog b/debian/changelog
index 5f9f7ee619..90026d89ef 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (3.20120107) UNRELEASED; urgency=low
 
   * log: Add --gource mode, which generates output usable by gource.
   * map: Fix display of remote repos
+  * Add annex-trustlevel configuration settings, which can be used to 
+    override the trust level of a remote.
 
  -- Joey Hess   Sat, 07 Jan 2012 18:12:09 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 629e191b5b..59b756de83 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -606,6 +606,12 @@ Here are all the supported configuration settings.
 
   git-annex caches UUIDs of remote repositories here.
 
+* `remote..annex-trustlevel`
+
+  Configures a local trust level for the remote. This overrides the value
+  configured by the trust and untrust commands. The value can be any of
+  "trusted", "semitrusted" or "untrusted".
+
 * `remote..annex-ssh-options`
 
   Options to use when using ssh to talk to this remote.

From 07cacbeee95b377e1bf4111e4d4b30190956c585 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 10 Jan 2012 13:11:16 -0400
Subject: [PATCH 2963/8313] break module dependancy loop

A PITA but worth it to clean up the trust configuration code.
---
 Command.hs         |  4 +-
 Command/Fsck.hs    |  4 +-
 Command/Whereis.hs |  1 -
 Config.hs          |  7 +---
 Limit.hs           |  5 +--
 Logs/Location.hs   | 14 ++-----
 Logs/Trust.hs      | 28 +++++++++-----
 Remote.hs          | 92 ++++++++++------------------------------------
 Remote/List.hs     | 58 +++++++++++++++++++++++++++++
 test.hs            |  3 +-
 10 files changed, 109 insertions(+), 107 deletions(-)
 create mode 100644 Remote/List.hs

diff --git a/Command.hs b/Command.hs
index 82d6429bfa..386efafde9 100644
--- a/Command.hs
+++ b/Command.hs
@@ -26,13 +26,13 @@ import Common.Annex
 import qualified Backend
 import qualified Annex
 import qualified Git
+import qualified Remote
 import Types.Command as ReExported
 import Types.Option as ReExported
 import Seek as ReExported
 import Checks as ReExported
 import Usage as ReExported
 import Logs.Trust
-import Logs.Location
 import Config
 
 {- Generates a normal command -}
@@ -110,5 +110,5 @@ autoCopies key vs numcopiesattr a = Annex.getState Annex.auto >>= auto
 		auto False = a
 		auto True = do
 			needed <- getNumCopies numcopiesattr
-			(_, have) <- trustPartition UnTrusted =<< keyLocations key
+			(_, have) <- trustPartition UnTrusted =<< Remote.keyLocations key
 			if length have `vs` needed then a else stop
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 723a2e7408..61107ebe18 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -93,7 +93,7 @@ verifyLocationLog key desc = do
 			preventWrite (parentDir f)
 
 	u <- getUUID
-        uuids <- keyLocations key
+	uuids <- Remote.keyLocations key
 
 	case (present, u `elem` uuids) of
 		(True, False) -> do
@@ -142,7 +142,7 @@ checkBackend = Types.Backend.fsckKey
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
 	needed <- getNumCopies numcopies
-	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< Remote.keyLocations key
 	let present = length safelocations
 	if present < needed
 		then do
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 9e57f361b8..1fbe707992 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -8,7 +8,6 @@
 module Command.Whereis where
 
 import Common.Annex
-import Logs.Location
 import Command
 import Remote
 import Logs.Trust
diff --git a/Config.hs b/Config.hs
index 0a7ac07897..83a84a1fe2 100644
--- a/Config.hs
+++ b/Config.hs
@@ -12,8 +12,6 @@ import qualified Git
 import qualified Git.Config
 import qualified Git.Command
 import qualified Annex
-import qualified Logs.Trust
-import Types.TrustLevel
 
 type ConfigKey = String
 
@@ -85,6 +83,5 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 		config = "annex.numcopies"
 
 {- Gets the trust level set for a remote in git config. -}
-getTrustLevel :: Git.Repo -> Annex (Maybe TrustLevel)
-getTrustLevel r = maybe Nothing Logs.Trust.trustName <$>
-	fromRepo (Git.Config.getMaybe (remoteConfig r "trustlevel"))
+getTrustLevel :: Git.Repo -> Annex (Maybe String)
+getTrustLevel r = fromRepo $ Git.Config.getMaybe $ remoteConfig r "trustlevel"
diff --git a/Limit.hs b/Limit.hs
index 26e5d689c9..128ea0a27b 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -15,7 +15,6 @@ import qualified Annex
 import qualified Utility.Matcher
 import qualified Remote
 import qualified Backend
-import Logs.Location
 import Annex.Content
 
 type Limit = Utility.Matcher.Token (FilePath -> Annex Bool)
@@ -78,7 +77,7 @@ addIn name = addLimit $ check $ if name == "." then inAnnex else inremote
 		handle a (Just (key, _)) = a key
 		inremote key = do
 			u <- Remote.nameToUUID name
-			us <- keyLocations key
+			us <- Remote.keyLocations key
 			return $ u `elem` us
 
 {- Adds a limit to skip files not believed to have the specified number
@@ -92,7 +91,7 @@ addCopies num =
 		check n = Backend.lookupFile >=> handle n
 		handle _ Nothing = return False
 		handle n (Just (key, _)) = do
-			us <- keyLocations key
+			us <- Remote.keyLocations key
 			return $ length us >= n
 
 {- Adds a limit to skip files not using a specified key-value backend. -}
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 588962bc57..b6d59b928c 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -16,8 +16,7 @@
 module Logs.Location (
 	LogStatus(..),
 	logChange,
-	readLog,
-	keyLocations,
+	loggedLocations,
 	loggedKeys,
 	loggedKeysFor,
 	logFile,
@@ -27,7 +26,6 @@ module Logs.Location (
 import Common.Annex
 import qualified Annex.Branch
 import Logs.Presence
-import Logs.Trust
 
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Key -> UUID -> LogStatus -> Annex ()
@@ -36,13 +34,9 @@ logChange _ NoUUID _ = return ()
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key.
- -
- - Dead repositories are skipped.
  -}
-keyLocations :: Key -> Annex [UUID]
-keyLocations key = do
-	l <- map toUUID <$> (currentLog . logFile) key
-	snd <$> trustPartition DeadTrusted l
+loggedLocations :: Key -> Annex [UUID]
+loggedLocations key = map toUUID <$> (currentLog . logFile) key
 
 {- Finds all keys that have location log information.
  - (There may be duplicate keys in the list.) -}
@@ -57,7 +51,7 @@ loggedKeysFor u = filterM isthere =<< loggedKeys
 		{- This should run strictly to avoid the filterM
 		 - building many thunks containing keyLocations data. -}
 		isthere k = do
-			us <- keyLocations k
+			us <- loggedLocations k
 			let !there = u `elem` us
 			return there
 
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index f6ead87f1d..4dd728a8bc 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -10,7 +10,6 @@ module Logs.Trust (
 	trustGet,
 	trustSet,
 	trustPartition,
-	trustName
 ) where
 
 import qualified Data.Map as M
@@ -21,6 +20,9 @@ import Types.TrustLevel
 import qualified Annex.Branch
 import qualified Annex
 import Logs.UUIDBased
+import Remote.List
+import Config
+import qualified Types.Remote
 
 {- Filename of trust.log. -}
 trustLog :: FilePath
@@ -56,7 +58,7 @@ trustPartition level ls
 		return $ partition (`elem` candidates) ls
 
 {- Read the trustLog into a map, overriding with any
- - values from forcetrust. The map is cached for speed. -}
+ - values from forcetrust or the git config. The map is cached for speed. -}
 trustMap :: Annex TrustMap
 trustMap = do
 	cached <- Annex.getState Annex.trustmap
@@ -66,9 +68,22 @@ trustMap = do
 			overrides <- Annex.getState Annex.forcetrust
 			logged <- simpleMap . parseLog (Just . parseTrust) <$>
 				Annex.Branch.get trustLog
-			let m = M.union overrides logged
+			configured <- M.fromList . catMaybes <$>
+				(mapM configuredtrust =<< remoteList)
+			let m = M.union overrides $ M.union configured logged
 			Annex.changeState $ \s -> s { Annex.trustmap = Just m }
 			return m
+	where
+		configuredtrust r =
+			maybe Nothing (\l -> Just (Types.Remote.uuid r, l)) <$>
+				(convert <$> getTrustLevel (Types.Remote.repo r))
+		convert :: Maybe String -> Maybe TrustLevel
+		convert Nothing = Nothing
+		convert (Just s)
+			| s ==  "trusted" = Just Trusted
+			| s == "untrusted" = Just UnTrusted
+			| s == "semitrusted" = Just SemiTrusted
+			| otherwise = Nothing
 
 {- The trust.log used to only list trusted repos, without a field for the
  - trust status, which is why this defaults to Trusted. -}
@@ -85,10 +100,3 @@ showTrust Trusted = "1"
 showTrust UnTrusted = "0"
 showTrust DeadTrusted = "X"
 showTrust SemiTrusted = "?"
-
-trustName :: String -> Maybe TrustLevel
-trustName "trusted" = Just Trusted
-trustName "untrusted" = Just UnTrusted
-trustName "deadtrusted" = Just DeadTrusted
-trustName "semitrusted" = Just SemiTrusted
-trustName _ = Nothing
diff --git a/Remote.hs b/Remote.hs
index 6a97c2da35..3caf5555bf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -24,6 +24,7 @@ module Remote (
 	prettyPrintUUIDs,
 	remotesWithUUID,
 	remotesWithoutUUID,
+	keyLocations,
 	keyPossibilities,
 	keyPossibilitiesTrusted,
 	nameToUUID,
@@ -40,55 +41,11 @@ import Text.JSON.Generic
 import Common.Annex
 import Types.Remote
 import qualified Annex
-import qualified Git
-import Config
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
 import Logs.Location
-import Logs.Remote
-
-import qualified Remote.Git
-import qualified Remote.S3
-import qualified Remote.Bup
-import qualified Remote.Directory
-import qualified Remote.Rsync
-import qualified Remote.Web
-import qualified Remote.Hook
-
-remoteTypes :: [RemoteType]
-remoteTypes =
-	[ Remote.Git.remote
-	, Remote.S3.remote
-	, Remote.Bup.remote
-	, Remote.Directory.remote
-	, Remote.Rsync.remote
-	, Remote.Web.remote
-	, Remote.Hook.remote
-	]
-
-{- Builds a list of all available Remotes.
- - Since doing so can be expensive, the list is cached. -}
-remoteList :: Annex [Remote]
-remoteList = do
-	rs <- Annex.getState Annex.remotes
-	if null rs
-		then do
-			m <- readRemoteLog
-			rs' <- concat <$> mapM (process m) remoteTypes
-			Annex.changeState $ \s -> s { Annex.remotes = rs' }
-			return rs'
-		else return rs
-	where
-		process m t = enumerate t >>= mapM (gen m t)
-		gen m t r = do
-			u <- getRepoUUID r
-			checkTrust r u
-			generate t r u (M.lookup u m)
-
-{- All remotes that are not ignored. -}
-enabledRemoteList :: Annex [Remote]
-enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
+import Remote.List
 
 {- Map of UUIDs of Remotes and their names. -}
 remoteMap :: Annex (M.Map UUID String)
@@ -185,27 +142,32 @@ remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
 remotesWithoutUUID :: [Remote] -> [UUID] -> [Remote]
 remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 
-{- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
+{- List of repository UUIDs that the location log indicates may have a key.
+ - Dead repositories are excluded. -}
+keyLocations :: Key -> Annex [UUID]
+keyLocations key = snd <$> (trustPartition DeadTrusted =<< loggedLocations key)
+
+{- Cost ordered lists of remotes that the location log indicates
+ - may have a key.
  -}
 keyPossibilities :: Key -> Annex [Remote]
-keyPossibilities key = fst <$> keyPossibilities' False key
+keyPossibilities key = fst <$> keyPossibilities' key []
 
-{- Cost ordered lists of remotes that the Logs.Location indicate may have a key.
+{- Cost ordered lists of remotes that the location log indicates
+ - may have a key.
  -
  - Also returns a list of UUIDs that are trusted to have the key
  - (some may not have configured remotes).
  -}
 keyPossibilitiesTrusted :: Key -> Annex ([Remote], [UUID])
-keyPossibilitiesTrusted = keyPossibilities' True
+keyPossibilitiesTrusted key = keyPossibilities' key =<< trustGet Trusted
 
-keyPossibilities' :: Bool -> Key -> Annex ([Remote], [UUID])
-keyPossibilities' withtrusted key = do
+keyPossibilities' :: Key -> [UUID] -> Annex ([Remote], [UUID])
+keyPossibilities' key trusted = do
 	u <- getUUID
-	trusted <- if withtrusted then trustGet Trusted else return []
 
-	-- get uuids of all remotes that are recorded to have the key
-	uuids <- keyLocations key
-	let validuuids = filter (/= u) uuids
+	-- uuids of all remotes that are recorded to have the key
+	validuuids <- filter (/= u) <$> keyLocations key
 
 	-- note that validuuids is assumed to not have dups
 	let validtrusteduuids = validuuids `intersect` trusted
@@ -241,24 +203,10 @@ showTriedRemotes remotes =
 		(join ", " $ map name remotes)
 
 forceTrust :: TrustLevel -> String -> Annex ()
-forceTrust level remotename = forceTrust' True level =<< nameToUUID remotename
-
-forceTrust' :: Bool -> TrustLevel -> UUID -> Annex ()
-forceTrust' overwrite level u = do
+forceTrust level remotename = do
+	u <- nameToUUID remotename
 	Annex.changeState $ \s ->
-		s { Annex.forcetrust = change u level (Annex.forcetrust s) }
-	-- This change invalidated any cached trustmap.
-	Annex.changeState $ \s -> s { Annex.trustmap = Nothing }
-	where
-		change
-			| overwrite = M.insert
-			| otherwise = M.insertWith (\_new old -> old)
-
-checkTrust :: Git.Repo -> UUID -> Annex ()
-checkTrust r u = set =<< getTrustLevel r
-	where
-		set (Just level) = forceTrust' False level u
-		set Nothing = return ()
+		s { Annex.forcetrust = M.insert u level (Annex.forcetrust s) }
 
 {- Used to log a change in a remote's having a key. The change is logged
  - in the local repo, not on the remote. The process of transferring the
diff --git a/Remote/List.hs b/Remote/List.hs
new file mode 100644
index 0000000000..e589b4401a
--- /dev/null
+++ b/Remote/List.hs
@@ -0,0 +1,58 @@
+{- git-annex remote list
+ -
+ - Copyright 2011 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.List where
+
+import qualified Data.Map as M
+
+import Common.Annex
+import qualified Annex
+import Logs.Remote
+import Types.Remote
+import Annex.UUID
+import Config
+
+import qualified Remote.Git
+import qualified Remote.S3
+import qualified Remote.Bup
+import qualified Remote.Directory
+import qualified Remote.Rsync
+import qualified Remote.Web
+import qualified Remote.Hook
+
+remoteTypes :: [RemoteType]
+remoteTypes =
+	[ Remote.Git.remote
+	, Remote.S3.remote
+	, Remote.Bup.remote
+	, Remote.Directory.remote
+	, Remote.Rsync.remote
+	, Remote.Web.remote
+	, Remote.Hook.remote
+	]
+
+{- Builds a list of all available Remotes.
+ - Since doing so can be expensive, the list is cached. -}
+remoteList :: Annex [Remote]
+remoteList = do
+	rs <- Annex.getState Annex.remotes
+	if null rs
+		then do
+			m <- readRemoteLog
+			rs' <- concat <$> mapM (process m) remoteTypes
+			Annex.changeState $ \s -> s { Annex.remotes = rs' }
+			return rs'
+		else return rs
+	where
+		process m t = enumerate t >>= mapM (gen m t)
+		gen m t r = do
+			u <- getRepoUUID r
+			generate t r u (M.lookup u m)
+
+{- All remotes that are not ignored. -}
+enabledRemoteList :: Annex [Remote]
+enabledRemoteList = filterM (repoNotIgnored . repo) =<< remoteList
diff --git a/test.hs b/test.hs
index 5d01f11169..ee88c5f080 100644
--- a/test.hs
+++ b/test.hs
@@ -32,7 +32,6 @@ import qualified Locations
 import qualified Types.Backend
 import qualified Types
 import qualified GitAnnex
-import qualified Logs.Location
 import qualified Logs.UUIDBased
 import qualified Logs.Trust
 import qualified Logs.Remote
@@ -847,7 +846,7 @@ checklocationlog f expected = do
 	r <- annexeval $ Backend.lookupFile f
 	case r of
 		Just (k, _) -> do
-			uuids <- annexeval $ Logs.Location.keyLocations k
+			uuids <- annexeval $ Remote.keyLocations k
 			assertEqual ("bad content in location log for " ++ f ++ " key " ++ (show k) ++ " uuid " ++ show thisuuid)
 				expected (thisuuid `elem` uuids)
 		_ -> assertFailure $ f ++ " failed to look up key"

From 16e7178f207b0472346c06f30aa210cebe373c36 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 10 Jan 2012 15:29:10 -0400
Subject: [PATCH 2964/8313] reorg

---
 Command/Map.hs                  | 2 +-
 Remote/Bup.hs                   | 2 +-
 Remote/Git.hs                   | 2 +-
 {Annex => Remote/Helper}/Ssh.hs | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)
 rename {Annex => Remote/Helper}/Ssh.hs (98%)

diff --git a/Command/Map.hs b/Command/Map.hs
index faaa893924..da7a048a4d 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -20,7 +20,7 @@ import qualified Annex
 import Annex.UUID
 import Logs.UUID
 import Logs.Trust
-import Annex.Ssh
+import Remote.Helper.Ssh
 import qualified Utility.Dot as Dot
 
 -- a link from the first repository to the second (its remote)
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 04cd490265..37f3e02e09 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -19,7 +19,7 @@ import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
 import Config
-import Annex.Ssh
+import Remote.Helper.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 7d034d2425..7964074496 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -13,7 +13,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Utility.CopyFile
 import Utility.RsyncFile
-import Annex.Ssh
+import Remote.Helper.Ssh
 import Types.Remote
 import qualified Git
 import qualified Git.Command
diff --git a/Annex/Ssh.hs b/Remote/Helper/Ssh.hs
similarity index 98%
rename from Annex/Ssh.hs
rename to Remote/Helper/Ssh.hs
index 81e488b41d..7c5eeddb8e 100644
--- a/Annex/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Annex.Ssh where
+module Remote.Helper.Ssh where
 
 import Common
 import qualified Git

From abdacf58ed5d0da0439819feb669b04b2368bb92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 10 Jan 2012 15:36:54 -0400
Subject: [PATCH 2965/8313] tweaks

---
 Annex/Branch.hs    |  6 +++---
 CmdLine.hs         |  2 +-
 Command/Fsck.hs    |  2 +-
 Logs/Trust.hs      | 18 ++++++++----------
 Remote.hs          |  2 +-
 Seek.hs            | 17 ++++-------------
 Upgrade.hs         | 14 ++++++--------
 git-union-merge.hs |  6 ++----
 8 files changed, 26 insertions(+), 41 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 8f07b7aa23..a653a49955 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -68,15 +68,15 @@ create = do
 	return ()
 
 {- Returns the ref of the branch, creating it first if necessary. -}
-getBranch :: Annex (Git.Ref)
-getBranch = maybe (hasOrigin >>= go >>= use) (return) =<< branchsha
+getBranch :: Annex Git.Ref
+getBranch = maybe (hasOrigin >>= go >>= use) return =<< branchsha
 	where
 		go True = do
 			inRepo $ Git.Command.run "branch"
 				[Param $ show name, Param $ show originname]
 			fromMaybe (error $ "failed to create " ++ show name)
 				<$> branchsha
-		go False = withIndex' True $ do
+		go False = withIndex' True $
 			inRepo $ Git.Branch.commit "branch created" fullname []
 		use sha = do
 			setIndexSha sha
diff --git a/CmdLine.hs b/CmdLine.hs
index 6ac0b423fd..68157a01a9 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -47,7 +47,7 @@ dispatch args cmds commonoptions header getgitrepo = do
  - the Command being run, and the remaining parameters for the command. -} 
 parseCmd :: Params -> [Command] -> [Option] -> String -> (Flags, Command, Params)
 parseCmd argv cmds commonoptions header
-	| name == Nothing = err "missing command"
+	| isNothing name = err "missing command"
 	| null matches = err $ "unknown command " ++ fromJust name
 	| otherwise = check $ getOpt Permute (commonoptions ++ cmdoptions cmd) args
 	where
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 61107ebe18..680828748d 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -52,7 +52,7 @@ withBarePresentKeys a params = isBareRepo >>= go
 		go True = do
 			unless (null params) $
 				error "fsck should be run without parameters in a bare repository"
-			prepStart a loggedKeys
+			map a <$> loggedKeys
 
 startBare :: Key -> CommandStart
 startBare key = case Backend.maybeLookupBackendName (Types.Key.keyBackendName key) of
diff --git a/Logs/Trust.hs b/Logs/Trust.hs
index 4dd728a8bc..1a6716d175 100644
--- a/Logs/Trust.hs
+++ b/Logs/Trust.hs
@@ -1,6 +1,6 @@
-{- git-annex trust
+{- git-annex trust log
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -76,14 +76,12 @@ trustMap = do
 	where
 		configuredtrust r =
 			maybe Nothing (\l -> Just (Types.Remote.uuid r, l)) <$>
-				(convert <$> getTrustLevel (Types.Remote.repo r))
-		convert :: Maybe String -> Maybe TrustLevel
-		convert Nothing = Nothing
-		convert (Just s)
-			| s ==  "trusted" = Just Trusted
-			| s == "untrusted" = Just UnTrusted
-			| s == "semitrusted" = Just SemiTrusted
-			| otherwise = Nothing
+				maybe Nothing convert <$>
+					getTrustLevel (Types.Remote.repo r)
+		convert "trusted" = Just Trusted
+		convert "untrusted" = Just UnTrusted
+		convert "semitrusted" = Just SemiTrusted
+		convert _ = Nothing
 
 {- The trust.log used to only list trusted repos, without a field for the
  - trust status, which is why this defaults to Trusted. -}
diff --git a/Remote.hs b/Remote.hs
index 3caf5555bf..7feb84d615 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -200,7 +200,7 @@ showTriedRemotes :: [Remote] -> Annex ()
 showTriedRemotes [] = return ()	
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
-		(join ", " $ map name remotes)
+		join ", " (map name remotes)
 
 forceTrust :: TrustLevel -> String -> Annex ()
 forceTrust level remotename = do
diff --git a/Seek.hs b/Seek.hs
index bf0770f404..8e935c90cd 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -23,9 +23,7 @@ import qualified Limit
 import qualified Option
 
 seekHelper :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> [FilePath] -> Annex [FilePath]
-seekHelper a params = do
-	g <- gitRepo
-	liftIO $ runPreserveOrder (`a` g) params
+seekHelper a params = inRepo $ \g -> runPreserveOrder (`a` g) params
 
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
@@ -41,9 +39,8 @@ withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 		go (file, v) = a (readMaybe v) file
 
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
-withBackendFilesInGit a params = do
-	files <- seekHelper LsFiles.inRepo params
-	prepBackendPairs a files
+withBackendFilesInGit a params =
+	prepBackendPairs a =<< seekHelper LsFiles.inRepo params
 
 withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
@@ -118,18 +115,12 @@ prepBackendPairs a fs = prepFilteredGen a snd (chooseBackends fs)
 prepFilteredGen :: (b -> CommandStart) -> (b -> FilePath) -> Annex [b] -> Annex [CommandStart]
 prepFilteredGen a d fs = do
 	matcher <- Limit.getMatcher
-	prepStart (proc matcher) fs
+	map (proc matcher) <$> fs
 	where
 		proc matcher v = do
 			let f = d v
 			ok <- matcher f
 			if ok then a v else return Nothing
 
-{- Generates a list of CommandStart actions that will be run to perform a
- - command, using a list (ie of files) coming from an action. The list
- - will be produced and consumed lazily. -}
-prepStart :: (b -> CommandStart) -> Annex [b] -> Annex [CommandStart]
-prepStart a = liftM (map a)
-
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f
diff --git a/Upgrade.hs b/Upgrade.hs
index 8b2e939dde..44ca6323eb 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -13,12 +13,10 @@ import qualified Upgrade.V0
 import qualified Upgrade.V1
 import qualified Upgrade.V2
 
-{- Uses the annex.version git config setting to automate upgrades. -}
 upgrade :: Annex Bool
-upgrade = do
-	version <- getVersion
-	case version of
-		Just "0" -> Upgrade.V0.upgrade
-		Just "1" -> Upgrade.V1.upgrade
-		Just "2" -> Upgrade.V2.upgrade
-		_ -> return True
+upgrade = go =<< getVersion
+	where
+		go (Just "0") = Upgrade.V0.upgrade
+		go (Just "1") = Upgrade.V1.upgrade
+		go (Just "2") = Upgrade.V2.upgrade
+		go _ = return True
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 6fd19c8dae..e439c4665b 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -28,9 +28,7 @@ setup :: Git.Repo -> IO ()
 setup = cleanup -- idempotency
 
 cleanup :: Git.Repo -> IO ()
-cleanup g = do
-	e' <- doesFileExist (tmpIndex g)
-	when e' $ removeFile (tmpIndex g)
+cleanup g = whenM (doesFileExist $ tmpIndex g) $ removeFile $ tmpIndex g
 
 parseArgs :: IO [String]
 parseArgs = do
@@ -43,7 +41,7 @@ main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
 	g <- Git.Config.read =<< Git.Construct.fromCwd
-	_ <- Git.Index.override (tmpIndex g)
+	_ <- Git.Index.override $ tmpIndex g
 	setup g
 	Git.UnionMerge.merge aref bref g
 	_ <- Git.Branch.commit "union merge" newref [aref, bref] g

From 9e2ab6f1aabec6ed9936ed5f3dd2605cb3df9ff0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Jan 2012 19:41:23 -0400
Subject: [PATCH 2966/8313] add

---
 doc/bugs/signal_weirdness.mdwn | 38 ++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 doc/bugs/signal_weirdness.mdwn

diff --git a/doc/bugs/signal_weirdness.mdwn b/doc/bugs/signal_weirdness.mdwn
new file mode 100644
index 0000000000..a31cc21486
--- /dev/null
+++ b/doc/bugs/signal_weirdness.mdwn
@@ -0,0 +1,38 @@
+For the record, there is a slight weirdness with how git-annex
+handles a signal like ctrl-c.
+
+For example:
+
+	joey@gnu:~/tmp/b>git annex copy a b --to origin
+	copy a (checking origin...) (to origin...) 
+	SHA256-s104857600--20492a4d0d84f8beb1767f6616229f85d44c2827b64bdbfb260ee12fa1109e0e
+	        3272   0%    0.00kB/s    0:00:00  ^C
+	zsh: interrupt  git annex copy a --to origin
+	joey@gnu:~/tmp/b>
+	rsync error: unexplained error (code 130) at rsync.c(549) [sender=3.0.9]
+
+Here git-annex exits before rsync has fully exited. Not a large problem
+but sorta weird.
+
+The culprit is `safeSystemEnv` in Utility.SafeCommand, which installs
+a default signal handler for SIGINT, which causes it to immediatly
+terminate git-annex. rsync, in turn, has its own SIGINT handler, which 
+prints the message, typically later.
+
+(Why it prints that message and not its more usual message about having
+received a signal, I'm not sure?)
+
+It's more usual for a `system` like thing to block SIGINT, letting the child
+catch it and exit, and then detecting the child's exit status and terminating. 
+However, since rsync *is* trapping SIGINT, and exiting nonzero explicitly,
+git-annex can't tell that rsync failed due to a SIGINT.
+And, git-annex typically doesn't stop when a single child fails. In the
+example above, it would go on to copy `b` after a ctrl-c!
+
+A further complication is that git-annex is itself a child process
+of git, which does not block SIGINT either. So if git-annex blocks sigint,
+it will be left running in the background after git exits, and continuing
+with further actions too. (Probably this is a bug in git.)
+
+Which is why the current behavior of not blocking SIGINT was chosen,
+as a less bad alternative. Still, I'd like to find a better one. --[[Joey]] 

From daff9029ba85cee772b707add61e8ce1d7fd953f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 12:09:02 -0400
Subject: [PATCH 2967/8313] close

---
 doc/bugs/conq:_invalid_command_syntax.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bugs/conq:_invalid_command_syntax.mdwn b/doc/bugs/conq:_invalid_command_syntax.mdwn
index ac7271fc9c..82cd51d8da 100644
--- a/doc/bugs/conq:_invalid_command_syntax.mdwn
+++ b/doc/bugs/conq:_invalid_command_syntax.mdwn
@@ -27,3 +27,4 @@ For example, the last two commands I ran are:
 
 *git-version:*  1.7.7.4
 
+> [[done]], apparently not a git-annex bug --[[Joey]] 

From 1ae780ee7990dc542a61714522c902754e76fb05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 12:52:09 -0400
Subject: [PATCH 2968/8313] git-annex, git-union-merge: Support GIT_DIR and
 GIT_WORK_TREE.

Note that GIT_WORK_TREE cannot influence GIT_DIR; that is necessary for
git-fake-bare and vcsh type things to work.
---
 Git/Construct.hs                              | 21 ++++++++++++++++++-
 GitAnnex.hs                                   |  2 +-
 debian/changelog                              |  1 +
 .../on--git-dir_and_--work-tree_options.mdwn  |  2 ++
 git-union-merge.hs                            |  2 +-
 test.hs                                       |  2 +-
 6 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/Git/Construct.hs b/Git/Construct.hs
index 51fa656bc3..7b77a72ba3 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -6,6 +6,7 @@
  -}
 
 module Git.Construct (
+	fromCurrent,
 	fromCwd,
 	fromAbsPath,
 	fromUrl,
@@ -19,6 +20,8 @@ module Git.Construct (
 ) where
 
 import System.Posix.User
+import System.Posix.Env (getEnv)
+import System.Posix.Directory (changeWorkingDirectory)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
 
@@ -27,7 +30,23 @@ import Git.Types
 import Git
 import qualified Git.Url as Url
 
-{- Finds the current git repository, which may be in a parent directory. -}
+{- Finds the current git repository.
+ -
+ - GIT_DIR can override the location of the .git directory.
+ -
+ - When GIT_WORK_TREE is set, chdir to it, so that anything using
+ - this repository runs in the right location. However, this chdir is
+ - done after determining GIT_DIR; git does not let GIT_WORK_TREE
+ - influence the git directory.
+ -}
+fromCurrent :: IO Repo
+fromCurrent = do
+	r <- maybe fromCwd fromAbsPath =<< getEnv "GIT_DIR"
+	maybe (return ()) changeWorkingDirectory =<< getEnv "GIT_WORK_TREE"
+	return r
+
+{- Finds the git repository used for the Cwd, which may be in a parent
+ - directory. -}
 fromCwd :: IO Repo
 fromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
 	where
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 78f20e9d14..bc3541676b 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -129,4 +129,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header Git.Construct.fromCwd
+run args = dispatch args cmds options header Git.Construct.fromCurrent
diff --git a/debian/changelog b/debian/changelog
index 90026d89ef..f626dc315b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20120107) UNRELEASED; urgency=low
   * map: Fix display of remote repos
   * Add annex-trustlevel configuration settings, which can be used to 
     override the trust level of a remote.
+  * git-annex, git-union-merge: Support GIT_DIR and GIT_WORK_TREE.
 
  -- Joey Hess   Sat, 07 Jan 2012 18:12:09 -0400
 
diff --git a/doc/bugs/on--git-dir_and_--work-tree_options.mdwn b/doc/bugs/on--git-dir_and_--work-tree_options.mdwn
index d76a42bfff..0bcefbb5df 100644
--- a/doc/bugs/on--git-dir_and_--work-tree_options.mdwn
+++ b/doc/bugs/on--git-dir_and_--work-tree_options.mdwn
@@ -27,3 +27,5 @@ regular git add works:
 
 git-annex version: 3.20110702
 
+> [[done]], git-annex now honors `GIT_DIR` and `GIT_WORK_TREE` like other
+> git commands do. --[[Joey]]
diff --git a/git-union-merge.hs b/git-union-merge.hs
index e439c4665b..f44136bfc1 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -40,7 +40,7 @@ parseArgs = do
 main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
-	g <- Git.Config.read =<< Git.Construct.fromCwd
+	g <- Git.Config.read =<< Git.Construct.fromCurrent
 	_ <- Git.Index.override $ tmpIndex g
 	setup g
 	Git.UnionMerge.merge aref bref g
diff --git a/test.hs b/test.hs
index ee88c5f080..3aef83e879 100644
--- a/test.hs
+++ b/test.hs
@@ -727,7 +727,7 @@ git_annex_expectoutput command params expected = do
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a
 annexeval a = do
-	g <- Git.Construct.fromCwd
+	g <- Git.Construct.fromCurrent
 	g' <- Git.Config.read g
 	s <- Annex.new g'
 	Annex.eval s { Annex.output = Annex.QuietOutput } a

From 16d920b301e5a30f8dc9342ab87b8ed8c6d40baa Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 13 Jan 2012 17:37:13 +0000
Subject: [PATCH 2969/8313] Added a comment: Not announced on Hackage?

---
 .../comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment

diff --git a/doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment b/doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment
new file mode 100644
index 0000000000..effd9c9a8f
--- /dev/null
+++ b/doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.134.113"
+ subject="Not announced on Hackage?"
+ date="2012-01-13T17:37:11Z"
+ content="""
+We have the [latest version in NixOS](http://hydra.nixos.org/job/nixpkgs/trunk/gitAndTools.gitAnnex), but we cannot advertise that fact on Hackage because it seems the corresponding Cabal file hasn't been uploaded to . Is there any particular reason why Hackage doesn't know about this release?
+"""]]

From 92b8e2a1b5730be9a1775979e791a5180939e008 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 13:47:56 -0400
Subject: [PATCH 2970/8313] limit news feed to only toplevel

---
 doc/news.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/news.mdwn b/doc/news.mdwn
index 3d9ece3bbb..95114cdcf5 100644
--- a/doc/news.mdwn
+++ b/doc/news.mdwn
@@ -3,7 +3,7 @@ This is where announcements of new releases, features, and other news is
 posted. git-annex users are recommended to subscribe to this page's RSS
 feed.
 
-[[!inline pages="./news/* and !*/Discussion" rootpage="news" show="30"]]
+[[!inline pages="./news/* and !./news/*/* and !*/Discussion" rootpage="news" show="30"]]
 
 """
 else="""

From bb97bc1ebfe89ea4b899197755bf850c34781673 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 13 Jan 2012 17:52:46 +0000
Subject: [PATCH 2971/8313] Added a comment

---
 ...comment_2_ae292ca7294b6790233e545086c3ac2f._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment

diff --git a/doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment b/doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment
new file mode 100644
index 0000000000..7e6920eb72
--- /dev/null
+++ b/doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-01-13T17:52:46Z"
+ content="""
+Uploading to hackage is a PITA (manual password entry and I am often on a slow link besides) and is not integrated with my regular release process, so I often forget to do it. I will try to upload the next release there again.
+
+You might add a page under [[install]] for your git-annex packages.
+"""]]

From f4ddecd32262e23a255766a51bccecc92f6298ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 13:55:14 -0400
Subject: [PATCH 2972/8313] move windows support to a todo item

not holding my breath on this one..
---
 doc/install.mdwn                                   |  1 +
 ...ent_3_cff163ea3e7cad926f4ed9e78b896598._comment | 10 ----------
 .../windows_support.mdwn}                          | 14 +++++---------
 3 files changed, 6 insertions(+), 19 deletions(-)
 delete mode 100644 doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
 rename doc/{install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment => todo/windows_support.mdwn} (87%)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 26af5a091d..a5d029a061 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -7,6 +7,7 @@
 * [[FreeBSD]]
 * [[openSUSE]]
 * [[ArchLinux]]
+* Windows: [[sorry, not possible yet|todo/windows_support]]
 
 ## Using cabal
 
diff --git a/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment b/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
deleted file mode 100644
index 6b47ed0e3e..0000000000
--- a/doc/install/comment_3_cff163ea3e7cad926f4ed9e78b896598._comment
+++ /dev/null
@@ -1,10 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawmByD9tmR48HuYgS4qWEGDDaoVTTC3m4kc"
- nickname="Jonas"
- subject="Any chance to get git-annex going on windows?"
- date="2011-06-10T18:08:36Z"
- content="""
-Would be great! :-)
-
-Jonas
-"""]]
diff --git a/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment b/doc/todo/windows_support.mdwn
similarity index 87%
rename from doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment
rename to doc/todo/windows_support.mdwn
index 678847ecae..8df792fd6e 100644
--- a/doc/install/comment_4_82a17eee4a076c6c79fddeda347e0c9a._comment
+++ b/doc/todo/windows_support.mdwn
@@ -1,24 +1,21 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="short answer: no"
- date="2011-06-10T19:55:38Z"
- content="""
+short answer: no
+
 Long answer, quoting from a mail to someone else:
 
 Well, I can tell you that it assumes a POSIX system, both in available
 utilities and system calls, So you'd need to use cygwin or something
 like that. (Perhaps you already are for git, I think git also assumes a
 POSIX system.) So you need a Haskell that can target that. What this
-page refers to as \"GHC-Cygwin\":
+page refers to as "GHC-Cygwin":
 
 I don't know where to get one. Did find this:
 
 
 (There are probably also still some places where it assumes / as a path
-separator, although I fixed some.)
+separator, although I fixed some. Probably almost all are fixed now.)
 
 FWIW, git-annex works fine on OS X and other fine proprietary unixen. ;P
+--[[Joey]]
 
 ----
 
@@ -66,4 +63,3 @@ sigCHLD
 sigINT
 unionFileModes
 

-"""]]

From 66aac77467fd897bb5a148c8126804464097b582 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 14:40:36 -0400
Subject: [PATCH 2973/8313] support relative GIT_DIR

---
 Git/Construct.hs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/Git/Construct.hs b/Git/Construct.hs
index 7b77a72ba3..0df59bed6f 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -20,7 +20,7 @@ module Git.Construct (
 ) where
 
 import System.Posix.User
-import System.Posix.Env (getEnv)
+import System.Posix.Env (getEnv, unsetEnv)
 import System.Posix.Directory (changeWorkingDirectory)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
@@ -38,11 +38,18 @@ import qualified Git.Url as Url
  - this repository runs in the right location. However, this chdir is
  - done after determining GIT_DIR; git does not let GIT_WORK_TREE
  - influence the git directory.
+ -
+ - Both environment variables are unset, to avoid confusing other git
+ - commands that also look at them. This would particularly be a problem
+ - when GIT_DIR is relative and we chdir for GIT_WORK_TREE. Instead,
+ - the Git module passes --work-tree and --git-dir to git commands it runs.
  -}
 fromCurrent :: IO Repo
 fromCurrent = do
-	r <- maybe fromCwd fromAbsPath =<< getEnv "GIT_DIR"
+	r <- maybe fromCwd fromPath =<< getEnv "GIT_DIR"
 	maybe (return ()) changeWorkingDirectory =<< getEnv "GIT_WORK_TREE"
+	unsetEnv "GIT_DIR"
+	unsetEnv "GIT_WORK_TREE"
 	return r
 
 {- Finds the git repository used for the Cwd, which may be in a parent
@@ -53,6 +60,10 @@ fromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
 		makerepo = return . newFrom . Dir
 		norepo = error "Not in a git repository."
 
+{- Local Repo constructor, accepts a relative or absolute path. -}
+fromPath :: FilePath -> IO Repo
+fromPath dir = fromAbsPath =<< absPath dir
+
 {- Local Repo constructor, requires an absolute path to the repo be
  - specified. -}
 fromAbsPath :: FilePath -> IO Repo

From 226d5c9bb16f52a472e023924fa88116852fb6f1 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 13 Jan 2012 18:48:29 +0000
Subject: [PATCH 2974/8313] Added a comment

---
 ...mment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment

diff --git a/doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment b/doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment
new file mode 100644
index 0000000000..1bb0c32cb1
--- /dev/null
+++ b/doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.134.113"
+ subject="comment 3"
+ date="2012-01-13T18:48:28Z"
+ content="""
+For what it's worth, the package `cabal-install` is pretty good for uploading packages to Hackages, among other things. It allows users to configure their username/password, and then making a release to Hackage is as simple as running:
+
+    cabal upload foo-version.tar.gz
+
+I use that to do releases of my stuff, too, and I'm quite happy. `cabal-install` has other features, too, of course.
+"""]]

From 3a638c7ba19773fb91f8e7eb47735da8924152ae Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 13 Jan 2012 18:49:56 +0000
Subject: [PATCH 2975/8313] Added NixOS to the list of distributions that
 support git-annex.

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index a5d029a061..7da46b351a 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -7,6 +7,7 @@
 * [[FreeBSD]]
 * [[openSUSE]]
 * [[ArchLinux]]
+* [[NixOS]]
 * Windows: [[sorry, not possible yet|todo/windows_support]]
 
 ## Using cabal

From cff772031dd1fc924f717b3d22b5c6ef2612f4de Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Fri, 13 Jan 2012 18:53:41 +0000
Subject: [PATCH 2976/8313] Document how to install git-annex in Nix

---
 doc/NixOS.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/NixOS.mdwn

diff --git a/doc/NixOS.mdwn b/doc/NixOS.mdwn
new file mode 100644
index 0000000000..864184a23c
--- /dev/null
+++ b/doc/NixOS.mdwn
@@ -0,0 +1,5 @@
+Users of the [Nix package manager](http://nixos.org/) can install it by running:
+
+    nix-env -i git-annex
+
+The build status of the package within Nix can be seen on the [Hydra Build Farm](http://hydra.nixos.org/job/nixpkgs/trunk/gitAndTools.gitAnnex).

From 9ad52ef67ccf42654110cde6735409c75fdc1daf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 15:08:28 -0400
Subject: [PATCH 2977/8313] add hackage target

---
 Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Makefile b/Makefile
index 93d4567ec5..c7e5e5e127 100644
--- a/Makefile
+++ b/Makefile
@@ -100,4 +100,8 @@ sdist: clean
 	@cabal sdist
 	@mv git-annex.cabal.orig git-annex.cabal
 
+# Upload to hackage.
+hackage: sdist
+	@cabal upload dist/*.tar.gz
+
 .PHONY: $(bins) test install

From b88ecbdc1bf1826e3361732484263bd359cef8d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 15:50:56 -0400
Subject: [PATCH 2978/8313] Add libghc-testpack-dev to build depends on all
 arches.

---
 debian/changelog | 5 +++--
 debian/control   | 4 ++--
 git-annex.cabal  | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index f626dc315b..d3c8f9a355 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,12 +1,13 @@
-git-annex (3.20120107) UNRELEASED; urgency=low
+git-annex (3.20120113) unstable; urgency=low
 
   * log: Add --gource mode, which generates output usable by gource.
   * map: Fix display of remote repos
   * Add annex-trustlevel configuration settings, which can be used to 
     override the trust level of a remote.
   * git-annex, git-union-merge: Support GIT_DIR and GIT_WORK_TREE.
+  * Add libghc-testpack-dev to build depends on all arches.
 
- -- Joey Hess   Sat, 07 Jan 2012 18:12:09 -0400
+ -- Joey Hess   Fri, 13 Jan 2012 15:35:17 -0400
 
 git-annex (3.20120106) unstable; urgency=low
 
diff --git a/debian/control b/debian/control
index 6a312c6b10..c292d8295a 100644
--- a/debian/control
+++ b/debian/control
@@ -12,11 +12,11 @@ Build-Depends:
 	libghc-http-dev,
 	libghc-utf8-string-dev,
 	libghc-hs3-dev (>= 0.5.6),
-	libghc-testpack-dev [any-i386 any-amd64],
+	libghc-testpack-dev,
+	libghc-quickcheck2-dev,
 	libghc-monad-control-dev (>= 0.3),
 	libghc-lifted-base-dev,
 	libghc-json-dev,
-	libghc-quickcheck2-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/git-annex.cabal b/git-annex.cabal
index b0b6d76f97..4aee161b6a 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120106
+Version: 3.20120113
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 54815280588fb9901bf21471e47bfc31ae5b1592 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 16:01:31 -0400
Subject: [PATCH 2979/8313] add news item for git-annex 3.20120113

---
 doc/news/version_3.20111203.mdwn | 19 -------------------
 doc/news/version_3.20111211.mdwn | 20 --------------------
 doc/news/version_3.20120113.mdwn |  8 ++++++++
 3 files changed, 8 insertions(+), 39 deletions(-)
 delete mode 100644 doc/news/version_3.20111203.mdwn
 delete mode 100644 doc/news/version_3.20111211.mdwn
 create mode 100644 doc/news/version_3.20120113.mdwn

diff --git a/doc/news/version_3.20111203.mdwn b/doc/news/version_3.20111203.mdwn
deleted file mode 100644
index 5be6e21424..0000000000
--- a/doc/news/version_3.20111203.mdwn
+++ /dev/null
@@ -1,19 +0,0 @@
-git-annex 3.20111203 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * The VFAT filesystem on recent versions of Linux, when mounted with
-     shortname=mixed, does not get along well with git-annex's mixed case
-     .git/annex/objects hash directories. To avoid this problem, new content
-     is now stored in all-lowercase hash directories. Except for non-bare
-     repositories which would be a pain to transition and cannot be put on FAT.
-     (Old mixed-case hash directories are still tried for backwards
-     compatibility.)
-   * Flush json output, avoiding a buffering problem that could result in
-     doubled output.
-   * Avoid needing haskell98 and other fixes for new ghc. Thanks, Mark Wright.
-   * Bugfix: dropunused did not drop keys with two spaces in their name.
-   * Support for storing .git/annex on a different device than the rest of the
-     git repository.
-   * --inbackend can be used to make git-annex only operate on files
-     whose content is stored using a specified key-value backend.
-   * dead: A command which says that a repository is gone for good
-     and you don't want git-annex to mention it again."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20111211.mdwn b/doc/news/version_3.20111211.mdwn
deleted file mode 100644
index 5d2c57e455..0000000000
--- a/doc/news/version_3.20111211.mdwn
+++ /dev/null
@@ -1,20 +0,0 @@
-git-annex 3.20111211 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix bug in last version in getting contents from bare repositories.
-   * Ensure that git-annex branch changes are merged into git-annex's index,
-     which fixes a bug that could cause changes that were pushed to the
-     git-annex branch to get reverted. As a side effect, it's now safe
-     for users to check out and commit changes directly to the git-annex
-     branch.
-   * map: Fix a failure to detect a loop when both repositories are local
-     and refer to each other with relative paths.
-   * Prevent key names from containing newlines.
-   * add: If interrupted, add can leave files converted to symlinks but not
-     yet added to git. Running the add again will now clean up this situtation.
-   * Fix caching of decrypted ciphers, which failed when drop had to check
-     multiple different encrypted special remotes.
-   * unannex: Can be run on files that have been added to the annex, but not
-     yet committed.
-   * sync: New command that synchronises the local repository and default
-     remote, by running git commit, pull, and push for you.
-   * Version monad-control dependency in cabal file."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120113.mdwn b/doc/news/version_3.20120113.mdwn
new file mode 100644
index 0000000000..74611e7ba2
--- /dev/null
+++ b/doc/news/version_3.20120113.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20120113 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * log: Add --gource mode, which generates output usable by gource.
+   * map: Fix display of remote repos
+   * Add annex-trustlevel configuration settings, which can be used to
+     override the trust level of a remote.
+   * git-annex, git-union-merge: Support GIT\_DIR and GIT\_WORK\_TREE.
+   * Add libghc-testpack-dev to build depends on all arches."""]]
\ No newline at end of file

From ff5703ce77ffd4f911a74de9c7f6d0dd332185b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Jan 2012 21:06:00 -0400
Subject: [PATCH 2980/8313] tweak

---
 Git/Construct.hs | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/Git/Construct.hs b/Git/Construct.hs
index 0df59bed6f..1ad631ca4b 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -57,7 +57,7 @@ fromCurrent = do
 fromCwd :: IO Repo
 fromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
 	where
-		makerepo = return . newFrom . Dir
+		makerepo = newFrom . Dir
 		norepo = error "Not in a git repository."
 
 {- Local Repo constructor, accepts a relative or absolute path. -}
@@ -88,7 +88,7 @@ fromAbsPath dir
 				else ret dir
 	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
 	where
-		ret = return . newFrom . Dir
+		ret = newFrom . Dir
 
 {- Remote Repo constructor. Throws exception on invalid url.
  -
@@ -103,14 +103,14 @@ fromUrl url
 fromUrlStrict :: String -> IO Repo
 fromUrlStrict url
 	| startswith "file://" url = fromAbsPath $ uriPath u
-	| otherwise = return $ newFrom $ Url u
+	| otherwise = newFrom $ Url u
 	where
 		u = fromMaybe bad $ parseURI url
 		bad = error $ "bad url " ++ url
 
 {- Creates a repo that has an unknown location. -}
 fromUnknown :: IO Repo
-fromUnknown = return $ newFrom Unknown
+fromUnknown = newFrom Unknown
 
 {- Converts a local Repo into a remote repo, using the reference repo
  - which is assumed to be on the same host. -}
@@ -248,12 +248,11 @@ isRepoTop dir = do
 			(doesDirectoryExist (dir ++ "/" ++ subdir))
 			(doesFileExist (dir ++ "/" ++ file))
 
-newFrom :: RepoLocation -> Repo
-newFrom l = 
-	Repo {
-		location = l,
-		config = M.empty,
-		fullconfig = M.empty,
-		remotes = [],
-		remoteName = Nothing
+newFrom :: RepoLocation -> IO Repo
+newFrom l = return Repo
+	{ location = l
+	, config = M.empty
+	, fullconfig = M.empty
+	, remotes = []
+	, remoteName = Nothing
 	}

From 1f66af2b53432802ad4a3a0f5a3b4f3863cd00ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Jan 2012 11:28:49 -0400
Subject: [PATCH 2981/8313] optimize away 3 stats

---
 Git/Construct.hs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/Git/Construct.hs b/Git/Construct.hs
index 1ad631ca4b..fdc56a20a2 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -239,14 +239,15 @@ seekUp want dir = do
 isRepoTop :: FilePath -> IO Bool
 isRepoTop dir = do
 	r <- isRepo
-	b <- isBareRepo
-	return (r || b)
+	if r
+		then return r
+		else isBareRepo
 	where
-		isRepo = gitSignature ".git" ".git/config"
-		isBareRepo = gitSignature "objects" "config"
-		gitSignature subdir file = liftM2 (&&)
-			(doesDirectoryExist (dir ++ "/" ++ subdir))
-			(doesFileExist (dir ++ "/" ++ file))
+		isRepo = gitSignature (".git"  "config")
+		isBareRepo = (&&)
+			<$> doesDirectoryExist (dir  "objects")
+			<*> gitSignature "config"
+		gitSignature file = doesFileExist (dir  file)
 
 newFrom :: RepoLocation -> IO Repo
 newFrom l = return Repo

From 92a4af8b2019de2fac6df9c1e92e3858fe644639 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Jan 2012 11:42:51 -0400
Subject: [PATCH 2982/8313] avoid unnecessary chdir

---
 Git/Config.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 55ab8a6f15..0d73a0b9a5 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -30,8 +30,11 @@ read repo@(Repo { location = Dir d }) = do
 	{- Cannot use pipeRead because it relies on the config having
 	   been already read. Instead, chdir to the repo. -}
 	cwd <- getCurrentDirectory
-	bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) $
-		pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
+	if dirContains d cwd
+		then go
+		else bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) go
+	where
+		go = pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
 			hRead repo
 read r = assertLocal r $
 	error $ "internal error; trying to read config of " ++ show r

From 8c87293b4886bd7b3389c3a3066e2c6cfa9f1a56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Jan 2012 11:48:10 -0400
Subject: [PATCH 2983/8313] avoid unnecessary stats when traversing to parent

---
 Git/Construct.hs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Git/Construct.hs b/Git/Construct.hs
index fdc56a20a2..bfb16164ff 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -244,9 +244,11 @@ isRepoTop dir = do
 		else isBareRepo
 	where
 		isRepo = gitSignature (".git"  "config")
-		isBareRepo = (&&)
-			<$> doesDirectoryExist (dir  "objects")
-			<*> gitSignature "config"
+		isBareRepo = do
+			e <- doesDirectoryExist (dir  "objects")
+			if not e
+				then return e
+				else gitSignature "config"
 		gitSignature file = doesFileExist (dir  file)
 
 newFrom :: RepoLocation -> IO Repo

From 5e2b4e16ba8f6cb32461b5c09e3872ce50aa13e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Jan 2012 12:07:36 -0400
Subject: [PATCH 2984/8313] avoid multiple unnecessary stats of the index file

Up to one per file processed.
---
 Annex/Branch.hs      |  2 +-
 Annex/BranchState.hs | 17 +++++++++++------
 Types/BranchState.hs |  3 ++-
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index a653a49955..b2b1ed3e40 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -268,7 +268,7 @@ withIndex' :: Bool -> Annex a -> Annex a
 withIndex' bootstrapping a = do
 	f <- fromRepo gitAnnexIndex
 	bracketIO (Git.Index.override f) id $ do
-		unlessM (liftIO $ doesFileExist f) $ do
+		checkIndexOnce $ unlessM (liftIO $ doesFileExist f) $ do
 			unless bootstrapping create
 			liftIO $ createDirectoryIfMissing True $ takeDirectory f
 			unless bootstrapping $ inRepo genIndex
diff --git a/Annex/BranchState.hs b/Annex/BranchState.hs
index 0950e9a967..977c4aa107 100644
--- a/Annex/BranchState.hs
+++ b/Annex/BranchState.hs
@@ -37,6 +37,14 @@ invalidateCache = do
 	state <- getState
 	setState state { cachedFile = Nothing, cachedContent = "" }
 
+{- Runs an action to check that the index file exists, if it's not been
+ - checked before in this run of git-annex. -}
+checkIndexOnce :: Annex () -> Annex ()
+checkIndexOnce a = unlessM (indexChecked <$> getState) $ do
+	a
+	state <- getState
+	setState state { indexChecked = True }
+
 {- Runs an action to update the branch, if it's not been updated before
  - in this run of git-annex. -}
 runUpdateOnce :: Annex () -> Annex ()
@@ -48,9 +56,6 @@ runUpdateOnce a = unlessM (branchUpdated <$> getState) $ do
  - is known to have not changed, or git-annex won't be relying on info
  - from it. -}
 disableUpdate :: Annex ()
-disableUpdate = Annex.changeState setupdated
-	where
-		setupdated s = s { Annex.branchstate = new }
-			where
-				new = old { branchUpdated = True }
-				old = Annex.branchstate s
+disableUpdate = do
+	state <- getState
+	setState state { branchUpdated = True }
diff --git a/Types/BranchState.hs b/Types/BranchState.hs
index 777edb32cb..e5b2291f3b 100644
--- a/Types/BranchState.hs
+++ b/Types/BranchState.hs
@@ -9,6 +9,7 @@ module Types.BranchState where
 
 data BranchState = BranchState {
 	branchUpdated :: Bool, -- has the branch been updated this run?
+	indexChecked :: Bool, -- has the index file been checked to exist?
 
 	-- the content of one file is cached
 	cachedFile :: Maybe FilePath,
@@ -16,4 +17,4 @@ data BranchState = BranchState {
 }
 
 startBranchState :: BranchState
-startBranchState = BranchState False Nothing ""
+startBranchState = BranchState False False Nothing ""

From a3d97e0c85cf6c1b6668939c02d07522d06f2d18 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Jan 2012 14:31:16 -0400
Subject: [PATCH 2985/8313] tweak

---
 Annex/BranchState.hs | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/Annex/BranchState.hs b/Annex/BranchState.hs
index 977c4aa107..2e60d12299 100644
--- a/Annex/BranchState.hs
+++ b/Annex/BranchState.hs
@@ -2,7 +2,7 @@
  -
  - Runtime state about the git-annex branch, including a small read cache.
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -19,31 +19,31 @@ getState = Annex.getState Annex.branchstate
 setState :: BranchState -> Annex ()
 setState state = Annex.changeState $ \s -> s { Annex.branchstate = state }
 
+changeState :: (BranchState -> BranchState) -> Annex ()
+changeState changer = setState =<< changer <$> getState
+
 setCache :: FilePath -> String -> Annex ()
-setCache file content = do
-	state <- getState
-	setState state { cachedFile = Just file, cachedContent = content }
+setCache file content = changeState $ \s -> s
+	{ cachedFile = Just file, cachedContent = content}
 
 getCache :: FilePath -> Annex (Maybe String)
-getCache file = getState >>= go
+getCache file = from <$> getState
 	where
-		go state
+		from state
 			| cachedFile state == Just file =
-				return $ Just $ cachedContent state
-			| otherwise = return Nothing
+				Just $ cachedContent state
+			| otherwise = Nothing
 
 invalidateCache :: Annex ()
-invalidateCache = do
-	state <- getState
-	setState state { cachedFile = Nothing, cachedContent = "" }
+invalidateCache = changeState $ \s -> s
+	{ cachedFile = Nothing, cachedContent = "" }
 
 {- Runs an action to check that the index file exists, if it's not been
  - checked before in this run of git-annex. -}
 checkIndexOnce :: Annex () -> Annex ()
 checkIndexOnce a = unlessM (indexChecked <$> getState) $ do
 	a
-	state <- getState
-	setState state { indexChecked = True }
+	changeState $ \s -> s { indexChecked = True }
 
 {- Runs an action to update the branch, if it's not been updated before
  - in this run of git-annex. -}
@@ -56,6 +56,4 @@ runUpdateOnce a = unlessM (branchUpdated <$> getState) $ do
  - is known to have not changed, or git-annex won't be relying on info
  - from it. -}
 disableUpdate :: Annex ()
-disableUpdate = do
-	state <- getState
-	setState state { branchUpdated = True }
+disableUpdate = changeState $ \s -> s { branchUpdated = True }

From 0eed604446ce09e375b8f409fc77d98df9459c23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Jan 2012 17:17:20 -0400
Subject: [PATCH 2986/8313] Add a sanity check for bad StatFS results.

git-annex FTBFS on s390, mips, powerpc, sparc. That StatFS code is failing
on all of them. At least on s390, the failure appears as:

Just (FileSystemStats {fsStatBlockSize = 4096, fsStatBlockCount = 0,
fsStatByteCount = 0, fsStatBytesFree = 0, fsStatBytesAvailable = 0,
fsStatBytesUsed = 0})

While I don't understand why this is happening, or how to fix it,
bandaid over it by checking for obviously bad values and returning Nothing.
That disables disk free space checking, but at least git-annex will work.

Upstream bug: http://code.google.com/p/xmobar/issues/detail?id=70
---
 Utility/StatFS.hsc | 5 ++++-
 debian/changelog   | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc
index d3e4a507e5..937571dfa0 100644
--- a/Utility/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -114,7 +114,7 @@ getFileSystemStats path =
         bfree <- (#peek struct statfs, f_bfree) vfs
         bavail <- (#peek struct statfs, f_bavail) vfs
         let bpb = toI bsize
-        return $ Just FileSystemStats
+        let stats = FileSystemStats
                        { fsStatBlockSize = bpb
                        , fsStatBlockCount = toI bcount
                        , fsStatByteCount = toI bcount * bpb
@@ -122,4 +122,7 @@ getFileSystemStats path =
                        , fsStatBytesAvailable = toI bavail * bpb
                        , fsStatBytesUsed = toI (bcount - bfree) * bpb
                        }
+        if fsStatBlockCount stats == 0 || fsStatBlockSize stats == 0
+          then return Nothing
+          else return $ Just stats
 #endif
diff --git a/debian/changelog b/debian/changelog
index d3c8f9a355..b8b0d7327e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120114) UNRELEASED; urgency=low
+
+  * Add a sanity check for bad StatFS results.
+
+ -- Joey Hess   Sat, 14 Jan 2012 17:12:04 -0400
+
 git-annex (3.20120113) unstable; urgency=low
 
   * log: Add --gource mode, which generates output usable by gource.

From 072a3f2c9956320c0f9387ddb37b3c61e90782c3 Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 17:22:49 +0000
Subject: [PATCH 2987/8313]

---
 ...kcheck_that_satisfies___62____61__2.1.mdwn | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn

diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
new file mode 100644
index 0000000000..6bc4b70a14
--- /dev/null
+++ b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
@@ -0,0 +1,33 @@
+Hi,
+
+I just wanted to install git-annex via cabal, as described in the install document. More specifically, I did this on my Ubuntu Lucid box:
+
+    andreas@antares:~$ sudo aptitude install cabal-install
+    [...]
+    andreas@antares:~$ cabal update
+    andreas@antares:~$ cabal install quickcheck --bindir=$HOME/bin
+    andreas@antares:~$ cabal install git-annex -v --bindir=$HOME/bin
+
+However, I got this error:
+
+    /usr/bin/ghc --numeric-version
+    looking for package tool: ghc-pkg near compiler in /usr/bin
+    found package tool in /usr/bin/ghc-pkg
+    /usr/bin/ghc-pkg --version
+    /usr/bin/ghc --supported-languages
+    Reading installed packages...
+    /usr/bin/ghc-pkg dump --global
+    /usr/bin/ghc-pkg dump --user
+    Reading available packages...
+    Resolving dependencies...
+    selecting
+    cabal: cannot configure git-annex-3.20120113. It requires quickcheck >=2.1
+    There is no available version of quickcheck that satisfies >=2.1
+
+which is really strange, because quickcheck 2.4.2 is installed:
+
+    andreas@antares:~$ ls -a .cabal/lib/
+    .  ..  QuickCheck-2.4.2
+
+Any help is greatly appreciated :)
+Andreas.

From 81856c3175fb011f4c9559a5be55c51ec6ad71cf Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 13:26:17 -0400
Subject: [PATCH 2988/8313] add a configure check for StatFS

This way, the build log will indicate whether StatFS can be relied on.
I've tested all the failing architectures now, and on all of them,
the StatFS code now returns Nothing, rather than Just nonsense.

Also, if annex.diskreserve is set on a platform where StatFS is not
working, git-annex will complain.

Also, the Makefile was missing the sources target used when building with
cabal.
---
 Annex/Content.hs | 13 ++++++++++++-
 Makefile         |  4 +++-
 configure.hs     |  8 ++++++++
 debian/changelog |  5 ++++-
 git-annex.cabal  |  2 +-
 5 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 1713b5e12d..ba67a2f151 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -177,6 +177,7 @@ checkDiskSpace' adjustment key = do
 	r <- getConfig g "diskreserve" ""
 	let reserve = fromMaybe megabyte $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
+	sanitycheck r stats
 	case (stats, keySize key) of
 		(Nothing, _) -> return ()
 		(_, Nothing) -> return ()
@@ -189,7 +190,17 @@ checkDiskSpace' adjustment key = do
 		needmorespace n = unlessM (Annex.getState Annex.force) $
 			error $ "not enough free space, need " ++ 
 				roughSize storageUnits True n ++
-				" more (use --force to override this check or adjust annex.diskreserve)"
+				" more" ++ forcemsg
+		forcemsg = " (use --force to override this check or adjust annex.diskreserve)"
+		sanitycheck r stats
+			| not (null r) && isNothing stats = do
+				unlessM (Annex.getState Annex.force) $
+					error $ "You have configured a diskreserve of "
+						++ r ++
+						" but disk space checking is not working"
+						++ forcemsg
+				return ()
+			| otherwise = return ()
 
 {- Moves a file into .git/annex/objects/
  -
diff --git a/Makefile b/Makefile
index c7e5e5e127..43d498f1c1 100644
--- a/Makefile
+++ b/Makefile
@@ -21,11 +21,13 @@ endif
 
 all: $(all)
 
+sources: $(sources)
+
 # Disables optimisation. Not for production use.
 fast: GHCFLAGS=-Wall $(IGNORE)
 fast: $(bins)
 
-Build/SysConfig.hs: configure.hs Build/TestConfig.hs
+Build/SysConfig.hs: configure.hs Build/TestConfig.hs Utility/StatFS.hs
 	$(GHCMAKE) configure
 	./configure
 
diff --git a/configure.hs b/configure.hs
index 3b3626dd22..772df3e38f 100644
--- a/configure.hs
+++ b/configure.hs
@@ -2,9 +2,11 @@
 
 import System.Directory
 import Data.List
+import Data.Maybe
 import System.Cmd.Utils
 
 import Build.TestConfig
+import Utility.StatFS
 
 tests :: [TestCase]
 tests =
@@ -21,6 +23,7 @@ tests =
 	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
+	, TestCase "StatFS" testStatFS
 	] ++ shaTestCases [1, 256, 512, 224, 384]
 
 shaTestCases :: [Int] -> [TestCase]
@@ -63,6 +66,11 @@ getGitVersion = do
 	let version = last $ words $ head $ lines s
 	return $ Config "gitversion" (StringConfig version)
 
+testStatFS :: Test
+testStatFS = do
+	s <- getFileSystemStats "."
+	return $ Config "statfs_sane" $ BoolConfig $ isJust s
+
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
 cabalSetup = do
diff --git a/debian/changelog b/debian/changelog
index b8b0d7327e..2e9468d8c4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,9 @@
 git-annex (3.20120114) UNRELEASED; urgency=low
 
-  * Add a sanity check for bad StatFS results.
+  * Add a sanity check for bad StatFS results. On architectures
+    where StatFS does not currently work (s390, mips, powerpc, sparc),
+    this disables the diskreserve checking code, and attempting to
+    configure an annex.diskreserve will result in an error.
 
  -- Joey Hess   Sat, 14 Jan 2012 17:12:04 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 4aee161b6a..aaa773a3d1 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120113
+Version: 3.20120114
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 37b5b1bf0d59cdb8068c0cfb76e194a70754cafa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 13:53:51 -0400
Subject: [PATCH 2989/8313] Fix QuickCheck dependency in cabal file.

---
 debian/changelog                                                | 1 +
 ...le_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn | 2 ++
 git-annex.cabal                                                 | 2 +-
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 2e9468d8c4..1136c37c78 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20120114) UNRELEASED; urgency=low
     where StatFS does not currently work (s390, mips, powerpc, sparc),
     this disables the diskreserve checking code, and attempting to
     configure an annex.diskreserve will result in an error.
+  * Fix QuickCheck dependency in cabal file.
 
  -- Joey Hess   Sat, 14 Jan 2012 17:12:04 -0400
 
diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
index 6bc4b70a14..15f2f15fcd 100644
--- a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
+++ b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
@@ -31,3 +31,5 @@ which is really strange, because quickcheck 2.4.2 is installed:
 
 Any help is greatly appreciated :)
 Andreas.
+
+> [[fixed|done]], QuickCheck has to be spelled in mised case. --[[Joey]] 
diff --git a/git-annex.cabal b/git-annex.cabal
index aaa773a3d1..54511d9fbc 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
-   base < 5, monad-control, transformers-base, lifted-base, quickcheck >= 2.1
+   base < 5, monad-control, transformers-base, lifted-base, QuickCheck >= 2.1
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From ce608303a333f61fbf49138f8774c0f43a8c75de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 14:02:32 -0400
Subject: [PATCH 2990/8313] releasing version 3.20120115

---
 debian/changelog | 5 +++--
 git-annex.cabal  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 1136c37c78..17816dc1a5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,12 +1,13 @@
-git-annex (3.20120114) UNRELEASED; urgency=low
+git-annex (3.20120115) unstable; urgency=low
 
   * Add a sanity check for bad StatFS results. On architectures
     where StatFS does not currently work (s390, mips, powerpc, sparc),
     this disables the diskreserve checking code, and attempting to
     configure an annex.diskreserve will result in an error.
   * Fix QuickCheck dependency in cabal file.
+  * Minor optimisations.
 
- -- Joey Hess   Sat, 14 Jan 2012 17:12:04 -0400
+ -- Joey Hess   Sun, 15 Jan 2012 13:54:20 -0400
 
 git-annex (3.20120113) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 54511d9fbc..a63d362d61 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120114
+Version: 3.20120115
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From f0c84686b222b29fc8683a5696545033e59684e8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 14:03:00 -0400
Subject: [PATCH 2991/8313] add news item for git-annex 3.20120115

---
 doc/news/version_3.20111231.mdwn | 24 ------------------------
 doc/news/version_3.20120115.mdwn |  8 ++++++++
 2 files changed, 8 insertions(+), 24 deletions(-)
 delete mode 100644 doc/news/version_3.20111231.mdwn
 create mode 100644 doc/news/version_3.20120115.mdwn

diff --git a/doc/news/version_3.20111231.mdwn b/doc/news/version_3.20111231.mdwn
deleted file mode 100644
index 35c38fb525..0000000000
--- a/doc/news/version_3.20111231.mdwn
+++ /dev/null
@@ -1,24 +0,0 @@
-git-annex 3.20111231 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * sync: Improved to work well without a central bare repository.
-     Thanks to Joachim Breitner.
-   * Rather than manually committing, pushing, pulling, merging, and git annex
-     merging, we encourage you to give "git annex sync" a try.
-   * sync --fast: Selects some of the remotes with the lowest annex.cost
-     and syncs those, in addition to any specified at the command line.
-   * Union merge now finds the least expensive way to represent the merge.
-   * reinject: Add a sanity check for using an annexed file as the source file.
-   * Properly handle multiline git config values.
-   * Fix the hook special remote, which bitrotted a while ago.
-   * map: --fast disables use of dot to display map
-   * Test suite improvements. Current top-level test coverage: 75%
-   * Improve deletion of files from rsync special remotes. Closes: #[652849](http://bugs.debian.org/652849)
-   * Add --include, which is the same as --not --exclude.
-   * Format strings can be specified using the new --format option, to control
-     what is output by git annex find.
-   * Support git annex find --json
-   * Fixed behavior when multiple insteadOf configs are provided for the
-     same url base.
-   * Can now be built with older git versions (before 1.7.7); the resulting
-     binary should only be used with old git.
-   * Updated to build with monad-control 0.3."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120115.mdwn b/doc/news/version_3.20120115.mdwn
new file mode 100644
index 0000000000..8aee547a31
--- /dev/null
+++ b/doc/news/version_3.20120115.mdwn
@@ -0,0 +1,8 @@
+git-annex 3.20120115 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Add a sanity check for bad StatFS results. On architectures
+     where StatFS does not currently work (s390, mips, powerpc, sparc),
+     this disables the diskreserve checking code, and attempting to
+     configure an annex.diskreserve will result in an error.
+   * Fix QuickCheck dependency in cabal file.
+   * Minor optimisations."""]]
\ No newline at end of file

From e3ea5fe938d3b66524336bc74e5efa6242587e78 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 14:53:38 -0400
Subject: [PATCH 2992/8313] debhelper v9

kills that ugly python message during build
---
 debian/compat  | 2 +-
 debian/control | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/compat b/debian/compat
index 7f8f011eb7..ec635144f6 100644
--- a/debian/compat
+++ b/debian/compat
@@ -1 +1 @@
-7
+9
diff --git a/debian/control b/debian/control
index c292d8295a..3f171a11ce 100644
--- a/debian/control
+++ b/debian/control
@@ -2,7 +2,7 @@ Source: git-annex
 Section: utils
 Priority: optional
 Build-Depends: 
-	debhelper (>= 7.0.50),
+	debhelper (>= 9),
 	ghc,
 	libghc-missingh-dev,
 	libghc-hslogger-dev,

From da6810674c1ee02b09fd5a28842749045fc0b1b4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 14:57:52 -0400
Subject: [PATCH 2993/8313] copyright

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index a63d362d61..6730916cea 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -5,7 +5,7 @@ License: GPL
 Maintainer: Joey Hess 
 Author: Joey Hess
 Stability: Stable
-Copyright: 2010-2011 Joey Hess
+Copyright: 2010-2012 Joey Hess
 License-File: GPL
 Extra-Source-Files: use-make-sdist-instead
 Homepage: http://git-annex.branchable.com/

From 14113f84b5391128c59aa0314599e6140d0137a2 Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 19:02:06 +0000
Subject: [PATCH 2994/8313] re-opened. proposed solution doesn't help.

---
 ...kcheck_that_satisfies___62____61__2.1.mdwn | 43 ++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
index 15f2f15fcd..d9b517fae7 100644
--- a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
+++ b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
@@ -32,4 +32,45 @@ which is really strange, because quickcheck 2.4.2 is installed:
 Any help is greatly appreciated :)
 Andreas.
 
-> [[fixed|done]], QuickCheck has to be spelled in mised case. --[[Joey]] 
+> QuickCheck has to be spelled in mised case. --[[Joey]] 
+
+Sorry to disagree, this doesn't fix my problem. cabal still complains that no version >= 2.1 is available, even though 2.4.2 is installed. This problem already occurred before I explicitly installed QuickCheck. According to [[install]], the `cabal install git-annex -v --bindir=$HOME/bin` should already take care of the dependencies.
+
+This is what I get:
+
+    andreas@antares:~/src/gitolite-admin$ cabal install QuickCheck --reinstall --bindir=$HOME/bin
+    Resolving dependencies...
+    Configuring QuickCheck-2.4.2...
+    Preprocessing library QuickCheck-2.4.2...
+    Building QuickCheck-2.4.2...
+    [ 1 of 13] Compiling Test.QuickCheck.Exception ( Test/QuickCheck/Exception.hs, dist/build/Test/QuickCheck/Exception.o )
+    [ 2 of 13] Compiling Test.QuickCheck.Text ( Test/QuickCheck/Text.hs, dist/build/Test/QuickCheck/Text.o )
+    [ 3 of 13] Compiling Test.QuickCheck.State ( Test/QuickCheck/State.hs, dist/build/Test/QuickCheck/State.o )
+    [ 4 of 13] Compiling Test.QuickCheck.Gen ( Test/QuickCheck/Gen.hs, dist/build/Test/QuickCheck/Gen.o )
+    [ 5 of 13] Compiling Test.QuickCheck.Arbitrary ( Test/QuickCheck/Arbitrary.hs, dist/build/Test/QuickCheck/Arbitrary.o )
+    [ 6 of 13] Compiling Test.QuickCheck.Poly ( Test/QuickCheck/Poly.hs, dist/build/Test/QuickCheck/Poly.o )
+    [ 7 of 13] Compiling Test.QuickCheck.Modifiers ( Test/QuickCheck/Modifiers.hs, dist/build/Test/QuickCheck/Modifiers.o )
+    [ 8 of 13] Compiling Test.QuickCheck.Function ( Test/QuickCheck/Function.hs, dist/build/Test/QuickCheck/Function.o )
+    [ 9 of 13] Compiling Test.QuickCheck.Property ( Test/QuickCheck/Property.hs, dist/build/Test/QuickCheck/Property.o )
+    [10 of 13] Compiling Test.QuickCheck.Test ( Test/QuickCheck/Test.hs, dist/build/Test/QuickCheck/Test.o )
+    [11 of 13] Compiling Test.QuickCheck.Monadic ( Test/QuickCheck/Monadic.hs, dist/build/Test/QuickCheck/Monadic.o )
+    [12 of 13] Compiling Test.QuickCheck  ( Test/QuickCheck.hs, dist/build/Test/QuickCheck.o )
+    [13 of 13] Compiling Test.QuickCheck.All ( Test/QuickCheck/All.hs, dist/build/Test/QuickCheck/All.o )
+    Registering QuickCheck-2.4.2...
+    Installing library in /home/andreas/.cabal/lib/QuickCheck-2.4.2/ghc-6.12.1
+    Registering QuickCheck-2.4.2...
+    andreas@antares:~/src/gitolite-admin$ cabal install git-annex -v --bindir=$HOME/bin
+    /usr/bin/ghc --numeric-version
+    looking for package tool: ghc-pkg near compiler in /usr/bin
+    found package tool in /usr/bin/ghc-pkg
+    /usr/bin/ghc-pkg --version
+    /usr/bin/ghc --supported-languages
+    Reading installed packages...
+    /usr/bin/ghc-pkg dump --global
+    /usr/bin/ghc-pkg dump --user
+    Reading available packages...
+    Resolving dependencies...
+    selecting
+    cabal: cannot configure git-annex-3.20120113. It requires quickcheck >=2.1
+    There is no available version of quickcheck that satisfies >=2.1
+    

From ba6ca1bc23ff28ffba3018adab307b394f91361b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 15 Jan 2012 15:07:01 -0400
Subject: [PATCH 2995/8313] no, really, I fixed it

---
 ...kcheck_that_satisfies___62____61__2.1.mdwn | 42 ++-----------------
 1 file changed, 3 insertions(+), 39 deletions(-)

diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
index d9b517fae7..a6e2423a82 100644
--- a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
+++ b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
@@ -32,45 +32,9 @@ which is really strange, because quickcheck 2.4.2 is installed:
 Any help is greatly appreciated :)
 Andreas.
 
-> QuickCheck has to be spelled in mised case. --[[Joey]] 
+> QuickCheck has to be spelled in mixed case. --[[Joey]] 
 
 Sorry to disagree, this doesn't fix my problem. cabal still complains that no version >= 2.1 is available, even though 2.4.2 is installed. This problem already occurred before I explicitly installed QuickCheck. According to [[install]], the `cabal install git-annex -v --bindir=$HOME/bin` should already take care of the dependencies.
 
-This is what I get:
-
-    andreas@antares:~/src/gitolite-admin$ cabal install QuickCheck --reinstall --bindir=$HOME/bin
-    Resolving dependencies...
-    Configuring QuickCheck-2.4.2...
-    Preprocessing library QuickCheck-2.4.2...
-    Building QuickCheck-2.4.2...
-    [ 1 of 13] Compiling Test.QuickCheck.Exception ( Test/QuickCheck/Exception.hs, dist/build/Test/QuickCheck/Exception.o )
-    [ 2 of 13] Compiling Test.QuickCheck.Text ( Test/QuickCheck/Text.hs, dist/build/Test/QuickCheck/Text.o )
-    [ 3 of 13] Compiling Test.QuickCheck.State ( Test/QuickCheck/State.hs, dist/build/Test/QuickCheck/State.o )
-    [ 4 of 13] Compiling Test.QuickCheck.Gen ( Test/QuickCheck/Gen.hs, dist/build/Test/QuickCheck/Gen.o )
-    [ 5 of 13] Compiling Test.QuickCheck.Arbitrary ( Test/QuickCheck/Arbitrary.hs, dist/build/Test/QuickCheck/Arbitrary.o )
-    [ 6 of 13] Compiling Test.QuickCheck.Poly ( Test/QuickCheck/Poly.hs, dist/build/Test/QuickCheck/Poly.o )
-    [ 7 of 13] Compiling Test.QuickCheck.Modifiers ( Test/QuickCheck/Modifiers.hs, dist/build/Test/QuickCheck/Modifiers.o )
-    [ 8 of 13] Compiling Test.QuickCheck.Function ( Test/QuickCheck/Function.hs, dist/build/Test/QuickCheck/Function.o )
-    [ 9 of 13] Compiling Test.QuickCheck.Property ( Test/QuickCheck/Property.hs, dist/build/Test/QuickCheck/Property.o )
-    [10 of 13] Compiling Test.QuickCheck.Test ( Test/QuickCheck/Test.hs, dist/build/Test/QuickCheck/Test.o )
-    [11 of 13] Compiling Test.QuickCheck.Monadic ( Test/QuickCheck/Monadic.hs, dist/build/Test/QuickCheck/Monadic.o )
-    [12 of 13] Compiling Test.QuickCheck  ( Test/QuickCheck.hs, dist/build/Test/QuickCheck.o )
-    [13 of 13] Compiling Test.QuickCheck.All ( Test/QuickCheck/All.hs, dist/build/Test/QuickCheck/All.o )
-    Registering QuickCheck-2.4.2...
-    Installing library in /home/andreas/.cabal/lib/QuickCheck-2.4.2/ghc-6.12.1
-    Registering QuickCheck-2.4.2...
-    andreas@antares:~/src/gitolite-admin$ cabal install git-annex -v --bindir=$HOME/bin
-    /usr/bin/ghc --numeric-version
-    looking for package tool: ghc-pkg near compiler in /usr/bin
-    found package tool in /usr/bin/ghc-pkg
-    /usr/bin/ghc-pkg --version
-    /usr/bin/ghc --supported-languages
-    Reading installed packages...
-    /usr/bin/ghc-pkg dump --global
-    /usr/bin/ghc-pkg dump --user
-    Reading available packages...
-    Resolving dependencies...
-    selecting
-    cabal: cannot configure git-annex-3.20120113. It requires quickcheck >=2.1
-    There is no available version of quickcheck that satisfies >=2.1
-    
+>> You need to `cabal update` to get the fixed version of git-annex which
+>> spells QuickCheck correctly. [[done]] --[[Joey]]

From 37d42ed1e7061741218ec644a26f024aa015484c Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 19:20:29 +0000
Subject: [PATCH 2996/8313] rename
 bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
 to
 forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn

---
 ...able_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/{bugs => forum}/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn (100%)

diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
similarity index 100%
rename from doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
rename to doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn

From bc2337c0d9f831904546fe993bf2761289c3b34e Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 19:22:46 +0000
Subject: [PATCH 2997/8313] syb dependency problem.

---
 ...uickcheck_that_satisfies___62____61__2.1.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
index a6e2423a82..0c9bf92ade 100644
--- a/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
+++ b/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
@@ -38,3 +38,19 @@ Sorry to disagree, this doesn't fix my problem. cabal still complains that no ve
 
 >> You need to `cabal update` to get the fixed version of git-annex which
 >> spells QuickCheck correctly. [[done]] --[[Joey]]
+
+I moved this over to the forum, because I'm probably just too much a Haskell/Cabal/whatever Noob to get it right ;)
+
+I started with a clean `~/.cabal` directory and did the following:
+
+    andreas@antares:~/src/gitolite-admin$ cabal update
+    andreas@antares:~/src/gitolite-admin$ cabal install git-annex -v --bindir=$HOME/
+
+However, again some error:
+
+    cabal: dependencies conflict: base-3.0.3.2 requires syb ==0.1.0.2 however
+    syb-0.1.0.2 was excluded because json-0.5 requires syb >=0.3.3
+
+Any ideas?
+
+Thanks for your help!

From 847407f8604798bf8d5c47cf239643801d9e1284 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 15 Jan 2012 19:53:36 +0000
Subject: [PATCH 2998/8313] Added a comment

---
 ...ment_1_fae6e88115d175239fc55cef4c33fb2c._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment

diff --git a/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment b/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment
new file mode 100644
index 0000000000..c04cc335b6
--- /dev/null
+++ b/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-15T19:53:35Z"
+ content="""
+This is now about different build failure than the bug you reported, which was already fixed. Conflating the two is just confusing.
+
+The error message about `syb` is because by using cabal-install on an Ubuntu system from 2010, you're mixing the very old versions of some haskell libraries in Ubuntu with the new versions cabal wants to install. The solution is to stop mixing two package management systems --
+
+* Either install git-annex without using cabal, and use apt-get to install all its dependencies from Ubuntu, assuming your distribution has all the necessary haskell libraries packaged.
+* Or `apt-get remove ghc`, and manually install a current version of [The Haskell Platform](http://hackage.haskell.org/platform/) and use cabal.
+"""]]

From 728d7e96e0d5267ebf2912d1b3f870e38a4e2c3d Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 20:48:26 +0000
Subject: [PATCH 2999/8313] un-tangle unrelated posts

---
 ...kcheck_that_satisfies___62____61__2.1.mdwn | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn

diff --git a/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
new file mode 100644
index 0000000000..a6e2423a82
--- /dev/null
+++ b/doc/bugs/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
@@ -0,0 +1,40 @@
+Hi,
+
+I just wanted to install git-annex via cabal, as described in the install document. More specifically, I did this on my Ubuntu Lucid box:
+
+    andreas@antares:~$ sudo aptitude install cabal-install
+    [...]
+    andreas@antares:~$ cabal update
+    andreas@antares:~$ cabal install quickcheck --bindir=$HOME/bin
+    andreas@antares:~$ cabal install git-annex -v --bindir=$HOME/bin
+
+However, I got this error:
+
+    /usr/bin/ghc --numeric-version
+    looking for package tool: ghc-pkg near compiler in /usr/bin
+    found package tool in /usr/bin/ghc-pkg
+    /usr/bin/ghc-pkg --version
+    /usr/bin/ghc --supported-languages
+    Reading installed packages...
+    /usr/bin/ghc-pkg dump --global
+    /usr/bin/ghc-pkg dump --user
+    Reading available packages...
+    Resolving dependencies...
+    selecting
+    cabal: cannot configure git-annex-3.20120113. It requires quickcheck >=2.1
+    There is no available version of quickcheck that satisfies >=2.1
+
+which is really strange, because quickcheck 2.4.2 is installed:
+
+    andreas@antares:~$ ls -a .cabal/lib/
+    .  ..  QuickCheck-2.4.2
+
+Any help is greatly appreciated :)
+Andreas.
+
+> QuickCheck has to be spelled in mixed case. --[[Joey]] 
+
+Sorry to disagree, this doesn't fix my problem. cabal still complains that no version >= 2.1 is available, even though 2.4.2 is installed. This problem already occurred before I explicitly installed QuickCheck. According to [[install]], the `cabal install git-annex -v --bindir=$HOME/bin` should already take care of the dependencies.
+
+>> You need to `cabal update` to get the fixed version of git-annex which
+>> spells QuickCheck correctly. [[done]] --[[Joey]]

From a7f1d8c3bc6f1f10acd7c384fcfd43b3970340a7 Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 20:53:09 +0000
Subject: [PATCH 3000/8313] rename
 forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
 to
 forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn

---
 ...excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn} | 0
 .../comment_1_fae6e88115d175239fc55cef4c33fb2c._comment           | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename doc/forum/{__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn => __91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn} (100%)
 rename doc/forum/{__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1 => __91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3}/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment (100%)

diff --git a/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn
similarity index 100%
rename from doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1.mdwn
rename to doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn
diff --git a/doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment
similarity index 100%
rename from doc/forum/__91__Installation__93___There_is_no_available_version_of_quickcheck_that_satisfies___62____61__2.1/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment
rename to doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3/comment_1_fae6e88115d175239fc55cef4c33fb2c._comment

From ba85dd41a6f6d045241abd740d7d36ff0f780dbb Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 20:53:40 +0000
Subject: [PATCH 3001/8313] un-tangle unrelated posts

---
 ...on-0.5_requires_syb___62____61__0.3.3.mdwn | 50 ++-----------------
 1 file changed, 5 insertions(+), 45 deletions(-)

diff --git a/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn
index 0c9bf92ade..950b8ee193 100644
--- a/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn
+++ b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3.mdwn
@@ -1,52 +1,12 @@
-Hi,
-
-I just wanted to install git-annex via cabal, as described in the install document. More specifically, I did this on my Ubuntu Lucid box:
-
-    andreas@antares:~$ sudo aptitude install cabal-install
-    [...]
-    andreas@antares:~$ cabal update
-    andreas@antares:~$ cabal install quickcheck --bindir=$HOME/bin
-    andreas@antares:~$ cabal install git-annex -v --bindir=$HOME/bin
-
-However, I got this error:
-
-    /usr/bin/ghc --numeric-version
-    looking for package tool: ghc-pkg near compiler in /usr/bin
-    found package tool in /usr/bin/ghc-pkg
-    /usr/bin/ghc-pkg --version
-    /usr/bin/ghc --supported-languages
-    Reading installed packages...
-    /usr/bin/ghc-pkg dump --global
-    /usr/bin/ghc-pkg dump --user
-    Reading available packages...
-    Resolving dependencies...
-    selecting
-    cabal: cannot configure git-annex-3.20120113. It requires quickcheck >=2.1
-    There is no available version of quickcheck that satisfies >=2.1
-
-which is really strange, because quickcheck 2.4.2 is installed:
-
-    andreas@antares:~$ ls -a .cabal/lib/
-    .  ..  QuickCheck-2.4.2
-
-Any help is greatly appreciated :)
-Andreas.
-
-> QuickCheck has to be spelled in mixed case. --[[Joey]] 
-
-Sorry to disagree, this doesn't fix my problem. cabal still complains that no version >= 2.1 is available, even though 2.4.2 is installed. This problem already occurred before I explicitly installed QuickCheck. According to [[install]], the `cabal install git-annex -v --bindir=$HOME/bin` should already take care of the dependencies.
-
->> You need to `cabal update` to get the fixed version of git-annex which
->> spells QuickCheck correctly. [[done]] --[[Joey]]
-
-I moved this over to the forum, because I'm probably just too much a Haskell/Cabal/whatever Noob to get it right ;)
+Hi, another installation issue on Ubuntu Lucid:
 
 I started with a clean `~/.cabal` directory and did the following:
 
-    andreas@antares:~/src/gitolite-admin$ cabal update
-    andreas@antares:~/src/gitolite-admin$ cabal install git-annex -v --bindir=$HOME/
+    andreas@antares:~$ sudo aptitude install cabal-install
+    andreas@antares:~$ cabal update
+    andreas@antares:~$ cabal install git-annex -v --bindir=$HOME/
 
-However, again some error:
+However, I got some dpendancy error:
 
     cabal: dependencies conflict: base-3.0.3.2 requires syb ==0.1.0.2 however
     syb-0.1.0.2 was excluded because json-0.5 requires syb >=0.3.3

From 0c4f12e8a2742dc6f7703822034152ecb518e2cc Mon Sep 17 00:00:00 2001
From: 
 "https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
 
Date: Sun, 15 Jan 2012 20:54:55 +0000
Subject: [PATCH 3002/8313] Added a comment: Thanks and sorry

---
 ...comment_2_4c7a75638e8717132ccde949018d6008._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3/comment_2_4c7a75638e8717132ccde949018d6008._comment

diff --git a/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3/comment_2_4c7a75638e8717132ccde949018d6008._comment b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3/comment_2_4c7a75638e8717132ccde949018d6008._comment
new file mode 100644
index 0000000000..3a3106a635
--- /dev/null
+++ b/doc/forum/__91__Installation__93___base-3.0.3.2_requires_syb___61____61__0.1.0.2_however_syb-0.1.0.2_was_excluded_because_json-0.5_requires_syb___62____61__0.3.3/comment_2_4c7a75638e8717132ccde949018d6008._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://openid.stackexchange.com/user/fd55c6e3-966a-4626-865f-5d0f73e1eb88"
+ nickname="Andreas H."
+ subject="Thanks and sorry"
+ date="2012-01-15T20:54:55Z"
+ content="""
+Joey, thanks for you quick help! I'll try the manual haskell-platform install once I have quicker internet again, i.e. tomorrow.
+
+And sorry for the mess-up; I splitted the post into two. Hope it's clearer now.
+"""]]

From f161b5eb596839d54c006a68e875088a9d66c105 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 Jan 2012 16:28:07 -0400
Subject: [PATCH 3003/8313] Fix data loss bug in directory special remote

When moving a file to the remote failed, and partially transferred content
was left behind in the directory, re-running the same move would think it
succeeded and delete the local copy.

I reproduced data loss when moving files to a partition that was almost
full. Interrupting a transfer could have similar results.

Easily fixed by using a temp file which is then moved atomically into place
once the transfer completes.

I've audited other calls to copyFileExternal, and other special remote
file transfer code; everything else seems to use temp files correctly
(rsync, git), or otherwise use atomic transfers (bup, S3).
---
 Remote/Directory.hs | 4 +++-
 debian/changelog    | 9 +++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 8ca2a28750..23265dabc9 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -98,11 +98,13 @@ storeEncrypted d (cipher, enck) k = do
 storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
 storeHelper d key a = do
 	let dest = Prelude.head $ locations d key
+	let tmpdest = dest ++ ".tmp"
 	let dir = parentDir dest
 	createDirectoryIfMissing True dir
 	allowWrite dir
-	ok <- a dest
+	ok <- a tmpdest
 	when ok $ do
+		renameFile tmpdest dest
 		preventWrite dest
 		preventWrite dir
 	return ok
diff --git a/debian/changelog b/debian/changelog
index 17816dc1a5..61922ae054 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+git-annex (3.20120116) UNRELEASED; urgency=low
+
+  * Fix data loss bug in directory special remote, when moving a file
+    to the remote failed, and partially transferred content was left
+    behind in the directory, re-running the same move would think it
+    succeeded and delete the local copy.
+
+ -- Joey Hess   Mon, 16 Jan 2012 16:21:51 -0400
+
 git-annex (3.20120115) unstable; urgency=low
 
   * Add a sanity check for bad StatFS results. On architectures

From 2837e8fef163c13bb6f6c4345154599c3bcbe173 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 Jan 2012 16:52:26 -0400
Subject: [PATCH 3004/8313] releasing version 3.20120116

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 61922ae054..d6c4419bb0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (3.20120116) UNRELEASED; urgency=low
+git-annex (3.20120116) unstable; urgency=medium
 
   * Fix data loss bug in directory special remote, when moving a file
     to the remote failed, and partially transferred content was left
     behind in the directory, re-running the same move would think it
     succeeded and delete the local copy.
 
- -- Joey Hess   Mon, 16 Jan 2012 16:21:51 -0400
+ -- Joey Hess   Mon, 16 Jan 2012 16:43:45 -0400
 
 git-annex (3.20120115) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 6730916cea..1a4bac9cfd 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120115
+Version: 3.20120116
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From a2428320de7d56dfba8ecca63cb71050cae2a89d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 Jan 2012 16:52:46 -0400
Subject: [PATCH 3005/8313] add news item for git-annex 3.20120116

---
 doc/news/version_3.20120105.mdwn | 14 --------------
 doc/news/version_3.20120116.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 14 deletions(-)
 delete mode 100644 doc/news/version_3.20120105.mdwn
 create mode 100644 doc/news/version_3.20120116.mdwn

diff --git a/doc/news/version_3.20120105.mdwn b/doc/news/version_3.20120105.mdwn
deleted file mode 100644
index 2bec87cdab..0000000000
--- a/doc/news/version_3.20120105.mdwn
+++ /dev/null
@@ -1,14 +0,0 @@
-git-annex 3.20120105 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Added annex-web-options configuration settings, which can be
-     used to provide parameters to whichever of wget or curl git-annex uses
-     (depends on which is available, but most of their important options
-     suitable for use here are the same).
-   * Dotfiles, and files inside dotdirs are not added by "git annex add"
-     unless the dotfile or directory is explicitly listed. So "git annex add ."
-     will add all untracked files in the current directory except for those in
-     dotdirs.
-   * Added quickcheck to build dependencies, and fail if test suite cannot be
-     built.
-   * fsck: Do backend-specific check before checking numcopies is satisfied.
-   * Run gpg with --no-tty. Closes: #[654721](http://bugs.debian.org/654721)"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120116.mdwn b/doc/news/version_3.20120116.mdwn
new file mode 100644
index 0000000000..0fd07df0d5
--- /dev/null
+++ b/doc/news/version_3.20120116.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20120116 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix data loss bug in directory special remote, when moving a file
+     to the remote failed, and partially transferred content was left
+     behind in the directory, re-running the same move would think it
+     succeeded and delete the local copy."""]]
\ No newline at end of file

From de96ee72089c6673f36b02cfdc440288cd0be206 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 Jan 2012 16:19:57 -0400
Subject: [PATCH 3006/8313] thought

---
 ...vent_repeated_password_prompts_for_one_command.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
index 6d1552fe4e..808b8496f7 100644
--- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
+++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
@@ -6,3 +6,13 @@ Simple, when performing various git annex command over ssh, in particular a mult
 > 
 > Combining multiple operations into a single ssh is on the todo list, but
 > very far down it. --[[Joey]]
+
+>> OTOH, automatically running ssh in ControlMaster mode (and stopping it
+>> at exit) would be useful and not hard thing for git-annex to do.
+>> 
+>> It'd just need to set the appropriate config options, setting
+>> ControlPath to a per-remote socket location that includes git-annex's
+>> pid. Then at shutdown, run `ssh -O exit` on each such socket.
+>> 
+>> Complicated slightly by not doing this if the user has already set up
+>> more broad ssh connection caching. --[[Joey]]

From 50c063df069682bdac2af3b1746933da70a519b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 Jan 2012 17:30:21 -0400
Subject: [PATCH 3007/8313] add

---
 doc/todo/fsck_special_remotes.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/todo/fsck_special_remotes.mdwn

diff --git a/doc/todo/fsck_special_remotes.mdwn b/doc/todo/fsck_special_remotes.mdwn
new file mode 100644
index 0000000000..c81c56c856
--- /dev/null
+++ b/doc/todo/fsck_special_remotes.mdwn
@@ -0,0 +1,11 @@
+`git annex fsck --from remote`
+
+Basically, this needs to receive each file in turn from the remote, to a
+temp file, and then run the existing fsck code on it. Could be quite
+expensive, but sometimes you really want to check.
+
+An unencrypted directory special remote could be optimised, by not actually
+copying the file, just dropping a symlink, etc.
+
+The WORM backend doesn't care about file content, so it would be nice to
+avoid transferring the content at all, and only send the size.

From d36525e9745b90cc04abfeac6500ff646cb9c89b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 13:51:30 -0400
Subject: [PATCH 3008/8313] convert fsckKey to a Maybe

This way it's clear when a backend does not implement its own fsck checks.
---
 Backend/SHA.hs   | 2 +-
 Backend/URL.hs   | 2 +-
 Backend/WORM.hs  | 2 +-
 Command/Fsck.hs  | 4 +++-
 Types/Backend.hs | 4 ++--
 5 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index a1124dfe2e..29f4e2e942 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -32,7 +32,7 @@ genBackend size
 		b = Backend
 			{ name = shaName size
 			, getKey = keyValue size
-			, fsckKey = checkKeyChecksum size
+			, fsckKey = Just $ checkKeyChecksum size
 			}
 
 genBackendE :: SHASize -> Maybe Backend
diff --git a/Backend/URL.hs b/Backend/URL.hs
index 7f621b00f2..6406095ca1 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -21,7 +21,7 @@ backend :: Backend
 backend = Backend {
 	name = "URL",
 	getKey = const (return Nothing),
-	fsckKey = const (return True)
+	fsckKey = Nothing
 }
 
 fromUrl :: String -> Key
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index ae9833e30c..c022fd413b 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -18,7 +18,7 @@ backend :: Backend
 backend = Backend {
 	name = "WORM",
 	getKey = keyValue,
-	fsckKey = const (return True)
+	fsckKey = Nothing
 }
 
 {- The key includes the file size, modification time, and the
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 680828748d..051a58fb47 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -137,7 +137,9 @@ checkKeySize key = do
 
 
 checkBackend :: Backend -> Key -> Annex Bool
-checkBackend = Types.Backend.fsckKey
+checkBackend backend key = case Types.Backend.fsckKey backend of
+	Nothing -> return True
+	Just a -> a key
 
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
diff --git a/Types/Backend.hs b/Types/Backend.hs
index 025293a906..1966d667f7 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -16,8 +16,8 @@ data BackendA a = Backend {
 	name :: String,
 	-- converts a filename to a key
 	getKey :: FilePath -> a (Maybe Key),
-	-- called during fsck to check a key
-	fsckKey :: Key -> a Bool
+	-- called during fsck to check a key, if the backend has its own checks
+	fsckKey :: Maybe (Key -> a Bool)
 }
 
 instance Show (BackendA a) where

From 90319afa41ca6d8a9ffe00d787dc3dcdff320f00 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 15:24:05 -0400
Subject: [PATCH 3009/8313] fsck --from

Fscking a remote is now supported. It's done by retrieving
the contents of the specified files from the remote, and checking them,
so can be an expensive operation.

(Several optimisations are possible, to speed it up, of course.. This is
the slow and stupid remote fsck to start with.)

Still, if the remote is a special remote, or a git repository that you
cannot run fsck in locally, it's nice to have the ability to fsck it.

If you have any directory special remotes, now would be a good time to
fsck them, in case you were hit by the data loss bug fixed in the
previous release!
---
 Backend/SHA.hs     |  15 ++----
 Command/Drop.hs    |   2 +-
 Command/Fsck.hs    | 132 +++++++++++++++++++++++++++++++++++----------
 Command/Move.hs    |   3 +-
 Remote.hs          |   6 +--
 Types/Backend.hs   |   2 +-
 debian/changelog   |  13 +++++
 doc/git-annex.mdwn |   2 +
 8 files changed, 131 insertions(+), 44 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 29f4e2e942..3adac65d8c 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -9,7 +9,6 @@ module Backend.SHA (backends) where
 
 import Common.Annex
 import qualified Annex
-import Annex.Content
 import Types.Backend
 import Types.Key
 import qualified Build.SysConfig as SysConfig
@@ -97,18 +96,14 @@ keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
 			| otherwise = naiveextension
 
 {- A key's checksum is checked during fsck. -}
-checkKeyChecksum :: SHASize -> Key -> Annex Bool
-checkKeyChecksum size key = do
+checkKeyChecksum :: SHASize -> Key -> FilePath -> Annex Bool
+checkKeyChecksum size key file = do
 	fast <- Annex.getState Annex.fast
-	file <- inRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
 	if not present || fast
 		then return True
-		else check =<< shaN size file
+		else check <$> shaN size file
 	where
 		check s
-			| s == dropExtension (keyName key) = return True
-			| otherwise = do
-				dest <- moveBad key
-				warning $ "Bad file content; moved to " ++ dest
-				return False
+			| s == dropExtension (keyName key) = True
+			| otherwise = False
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 578ab62b97..b40de00cb2 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -87,7 +87,7 @@ cleanupRemote key remote ok = do
 	-- better safe than sorry: assume the remote dropped the key
 	-- even if it seemed to fail; the failure could have occurred
 	-- after it really dropped it
-	Remote.logStatus remote key False
+	Remote.logStatus remote key InfoMissing
 	return ok
 
 {- Checks specified remotes to verify that enough copies of a key exist to
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 051a58fb47..aec29a39b8 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -20,20 +20,31 @@ import Annex.UUID
 import Utility.DataUnits
 import Utility.FileMode
 import Config
+import qualified Option
 
 def :: [Command]
-def = [command "fsck" paramPaths seek "check for problems"]
+def = [withOptions options $ command "fsck" paramPaths seek
+	"check for problems"]
+
+fromOption :: Option
+fromOption = Option.field ['f'] "from" paramRemote "check remote"
+
+options :: [Option]
+options = [fromOption]
 
 seek :: [CommandSeek]
 seek =
-	[ withNumCopies $ \n -> whenAnnexed $ start n
+	[ withField fromOption Remote.byName $ \from ->
+		withNumCopies $ \n -> whenAnnexed $ start from n
 	, withBarePresentKeys startBare
 	]
 
-start :: Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start numcopies file (key, backend) = do
+start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
+start from numcopies file (key, backend) = do
 	showStart "fsck" file
-	next $ perform key file backend numcopies
+	case from of
+		Nothing -> next $ perform key file backend numcopies
+		Just r -> next $ performRemote key file backend numcopies r
 
 perform :: Key -> FilePath -> Backend -> Maybe Int -> CommandPerform
 perform key file backend numcopies = check
@@ -44,6 +55,27 @@ perform key file backend numcopies = check
 	, checkKeyNumCopies key file numcopies
 	]
 
+{- To fsck a remote, the content is retrieved to a tmp file,
+ - and checked locally. -}
+performRemote :: Key -> FilePath -> Backend -> Maybe Int -> Remote -> CommandPerform
+performRemote key file backend numcopies remote = withTmp key $ \tmpfile -> do
+	v <- Remote.hasKey remote key
+	case v of
+		Left err -> do
+			showNote err
+			stop
+		Right True -> do
+			copied <- Remote.retrieveKeyFile remote key tmpfile
+			if copied then go True (Just tmpfile) else go False Nothing
+		Right False -> go False Nothing
+	where
+		go present localcopy = check
+			[ verifyLocationLogRemote key file remote present
+			, checkKeySizeRemote key remote localcopy
+			, checkBackendRemote backend key remote localcopy
+			, checkKeyNumCopies key file numcopies
+			]
+
 {- To fsck a bare repository, fsck each key in the location log. -}
 withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
 withBarePresentKeys a params = isBareRepo >>= go
@@ -93,26 +125,33 @@ verifyLocationLog key desc = do
 			preventWrite (parentDir f)
 
 	u <- getUUID
-	uuids <- Remote.keyLocations key
+	verifyLocationLog' key desc present u (logChange key u)
 
+verifyLocationLogRemote :: Key -> String -> Remote -> Bool -> Annex Bool
+verifyLocationLogRemote key desc remote present =
+	verifyLocationLog' key desc present (Remote.uuid remote)
+		(Remote.logStatus remote key)
+
+verifyLocationLog' :: Key -> String -> Bool -> UUID -> (LogStatus -> Annex ()) -> Annex Bool
+verifyLocationLog' key desc present u bad = do
+	uuids <- Remote.keyLocations key
 	case (present, u `elem` uuids) of
 		(True, False) -> do
-				fix u InfoPresent
+				fix InfoPresent
 				-- There is no data loss, so do not fail.
 				return True
 		(False, True) -> do
-				fix u InfoMissing
+				fix InfoMissing
 				warning $
 					"** Based on the location log, " ++ desc
 					++ "\n** was expected to be present, " ++
 					"but its content is missing."
 				return False
 		_ -> return True
-	
 	where
-		fix u s = do
+		fix s = do
 			showNote "fixing location log"
-			logChange key u s
+			bad s
 
 {- The size of the data for a key is checked against the size encoded in
  - the key's metadata, if available. -}
@@ -120,26 +159,49 @@ checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
 	file <- inRepo $ gitAnnexLocation key
 	present <- liftIO $ doesFileExist file
-	case (present, Types.Key.keySize key) of
-		(_, Nothing) -> return True
-		(False, _) -> return True
-		(True, Just size) -> do
-			stat <- liftIO $ getFileStatus file
-			let size' = fromIntegral (fileSize stat)
-			if size == size'
-				then return True
-				else do
-					dest <- moveBad key
-					warning $ "Bad file size (" ++
-						compareSizes storageUnits True size size' ++ 
-						"); moved to " ++ dest
-					return False
+	if present
+		then checkKeySize' key file badContent
+		else return True
 
+checkKeySizeRemote :: Key -> Remote -> Maybe FilePath -> Annex Bool
+checkKeySizeRemote _ _ Nothing = return True
+checkKeySizeRemote key remote (Just file) = checkKeySize' key file
+	(badContentRemote remote)
+
+checkKeySize' :: Key -> FilePath -> (Key -> Annex String) -> Annex Bool
+checkKeySize' key file bad = case Types.Key.keySize key of
+	Nothing -> return True
+	Just size -> do
+		stat <- liftIO $ getFileStatus file
+		let size' = fromIntegral (fileSize stat)
+		if size == size'
+			then return True
+			else do
+				msg <- bad key
+				warning $ "Bad file size (" ++
+					compareSizes storageUnits True size size' ++
+					"); " ++ msg
+				return False
 
 checkBackend :: Backend -> Key -> Annex Bool
-checkBackend backend key = case Types.Backend.fsckKey backend of
+checkBackend backend key = do
+	file <- inRepo (gitAnnexLocation key)
+	checkBackend' backend key (Just file) badContent
+
+checkBackendRemote :: Backend -> Key -> Remote -> Maybe FilePath -> Annex Bool
+checkBackendRemote backend key remote localcopy =
+	checkBackend' backend key localcopy (badContentRemote remote)
+
+checkBackend' :: Backend -> Key -> Maybe FilePath -> (Key -> Annex String) -> Annex Bool
+checkBackend' _ _ Nothing _ = return True
+checkBackend' backend key (Just file) bad = case Types.Backend.fsckKey backend of
 	Nothing -> return True
-	Just a -> a key
+	Just a -> do
+		ok <- a key file
+		unless ok $ do
+			msg <- bad key
+			warning $ "Bad file content; " ++ msg
+		return ok
 
 checkKeyNumCopies :: Key -> FilePath -> Maybe Int -> Annex Bool
 checkKeyNumCopies key file numcopies = do
@@ -168,3 +230,19 @@ missingNote file present needed untrusted =
 		missingNote file present needed [] ++
 		"\nThe following untrusted locations may also have copies: " ++
 		"\n" ++ untrusted
+
+{- Bad content is moved aside. -}
+badContent :: Key -> Annex String
+badContent key = do
+	dest <- moveBad key
+	return $ "moved to " ++ dest
+
+badContentRemote :: Remote -> Key -> Annex String
+badContentRemote remote key = do
+	ok <- Remote.removeKey remote key
+	-- better safe than sorry: assume the remote dropped the key
+	-- even if it seemed to fail; the failure could have occurred
+	-- after it really dropped it
+	Remote.logStatus remote key InfoMissing
+	return $ (if ok then "dropped from " else "failed to drop from ")
+		++ Remote.name remote
diff --git a/Command/Move.hs b/Command/Move.hs
index 2efaebbcb1..2f2cd1b5d6 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -15,6 +15,7 @@ import Annex.Content
 import qualified Remote
 import Annex.UUID
 import qualified Option
+import Logs.Presence
 
 def :: [Command]
 def = [withOptions options $ command "move" paramPaths seek
@@ -97,7 +98,7 @@ toPerform dest move key = moveLock move key $ do
 		Right True -> finish
 	where
 		finish = do
-			Remote.logStatus dest key True
+			Remote.logStatus dest key InfoPresent
 			if move
 				then do
 					whenM (inAnnex key) $ removeAnnex key
diff --git a/Remote.hs b/Remote.hs
index 7feb84d615..133d3e2742 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -212,7 +212,5 @@ forceTrust level remotename = do
  - in the local repo, not on the remote. The process of transferring the
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
-logStatus :: Remote -> Key -> Bool -> Annex ()
-logStatus remote key present = logChange key (uuid remote) status
-	where
-		status = if present then InfoPresent else InfoMissing
+logStatus :: Remote -> Key -> LogStatus -> Annex ()
+logStatus remote key present = logChange key (uuid remote) present
diff --git a/Types/Backend.hs b/Types/Backend.hs
index 1966d667f7..d52cec5471 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -17,7 +17,7 @@ data BackendA a = Backend {
 	-- converts a filename to a key
 	getKey :: FilePath -> a (Maybe Key),
 	-- called during fsck to check a key, if the backend has its own checks
-	fsckKey :: Maybe (Key -> a Bool)
+	fsckKey :: Maybe (Key -> FilePath -> a Bool)
 }
 
 instance Show (BackendA a) where
diff --git a/debian/changelog b/debian/changelog
index d6c4419bb0..6849931486 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,16 @@
+git-annex (3.20120117) UNRELEASED; urgency=low
+
+  * fsck --from: Fscking a remote is now supported. It's done by retrieving
+    the contents of the specified files from the remote, and checking them,
+    so can be an expensive operation. Still, if the remote is a special
+    remote, or a git repository that you cannot run fsck in locally, it's
+    nice to have the ability to fsck it.
+  * If you have any directory special remotes, now would be a good time to
+    fsck them, in case you were hit by the data loss bug fixed in the
+    previous release!
+
+ -- Joey Hess   Thu, 19 Jan 2012 15:12:03 -0400
+
 git-annex (3.20120116) unstable; urgency=medium
 
   * Fix data loss bug in directory special remote, when moving a file
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 59b756de83..edf300d8d7 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -212,6 +212,8 @@ subdirectories).
 
   To avoid expensive checksum calculations, specify --fast
 
+  To check a remote to fsck, specify --from.
+
 * unused
 
   Checks the annex for data that does not correspond to any files present

From 711c15456109f77a04832d8ca0871ce56ffaffe4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 15:26:43 -0400
Subject: [PATCH 3010/8313] update NEWS

Add news item recommending fscking directory special remotes.

Remote news item about URL backend being removed; it was later added back
to be used by git annex addurl --fast.

Link NEWS into top level.
---
 NEWS        |  1 +
 debian/NEWS | 11 ++++++++---
 2 files changed, 9 insertions(+), 3 deletions(-)
 create mode 120000 NEWS

diff --git a/NEWS b/NEWS
new file mode 120000
index 0000000000..798088bec2
--- /dev/null
+++ b/NEWS
@@ -0,0 +1 @@
+debian/NEWS
\ No newline at end of file
diff --git a/debian/NEWS b/debian/NEWS
index f807d05255..298ff1f7e0 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,8 +1,13 @@
-git-annex (3.20110702) unstable; urgency=low
+git-annex (3.20120119) unstable; urgency=low
 
-  The URL backend has been removed. Instead the new web remote can be used.
+  There was a bug in the handling of directory special remotes that
+  could cause partial file contents to be stored in them. If you use
+  a directory special remote, you should fsck it, to avoid potential
+  data loss.
 
- -- Joey Hess   Fri, 01 Jul 2011 15:40:51 -0400
+  Example: git annex fsck --from mydirectory
+
+ -- Joey Hess   Thu, 19 Jan 2012 15:24:23 -0400
 
 git-annex (3.20110624) experimental; urgency=low
 

From 94aa6b42b5bc5c37c7017fb3493010a56a9d211e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 15:49:55 -0400
Subject: [PATCH 3011/8313] optimise fsck --from rsync special remote

When a file is present locally, the remote's version can be rsynced to
a copy of it, which will avoid wasting a lot of bandwidth.
---
 Remote/Rsync.hs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index c7b60467c4..eeb116675f 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -19,6 +19,8 @@ import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 import Utility.RsyncFile
+import Utility.CopyFile
+import Utility.FileMode
 
 type RsyncUrl = String
 
@@ -103,13 +105,20 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	rsyncSend o enck tmp
 
 retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieve o k f = untilTrue (rsyncUrls o k) $ \u ->
+retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> do
+	unlessM (liftIO $ doesFileExist f) $ whenM (inAnnex k) $ preseed
 	rsyncRemote o
 		-- use inplace when retrieving to support resuming
 		[ Param "--inplace"
 		, Param u
 		, Param f
 		]
+	where
+		-- this speeds up fsck --from
+		preseed = do
+			s <- inRepo $ gitAnnexLocation k
+			liftIO $ whenM (copyFileExternal s f) $
+				allowWrite f
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do

From 06b0cb6224377fd2ea86e4e209e94a502f92716e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 16:07:36 -0400
Subject: [PATCH 3012/8313] add tmp flag parameter to retrieveKeyFile

---
 Command/Fsck.hs              |  2 +-
 Command/Get.hs               |  2 +-
 Command/Move.hs              |  2 +-
 Remote/Bup.hs                |  4 ++--
 Remote/Directory.hs          |  5 +++--
 Remote/Git.hs                |  4 ++--
 Remote/Helper/Encryptable.hs |  4 ++--
 Remote/Hook.hs               |  4 ++--
 Remote/Rsync.hs              | 17 +++++++++--------
 Remote/S3.hs                 |  4 ++--
 Remote/Web.hs                |  4 ++--
 Types/Remote.hs              |  4 ++--
 12 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index aec29a39b8..77e189f436 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -65,7 +65,7 @@ performRemote key file backend numcopies remote = withTmp key $ \tmpfile -> do
 			showNote err
 			stop
 		Right True -> do
-			copied <- Remote.retrieveKeyFile remote key tmpfile
+			copied <- Remote.retrieveKeyFile remote key True tmpfile
 			if copied then go True (Just tmpfile) else go False Nothing
 		Right False -> go False Nothing
 	where
diff --git a/Command/Get.hs b/Command/Get.hs
index 5d032e13c4..7f5c08a7e6 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -72,7 +72,7 @@ getKeyFile key file = do
 				else return True
 		docopy r continue = do
 			showAction $ "from " ++ Remote.name r
-			copied <- Remote.retrieveKeyFile r key file
+			copied <- Remote.retrieveKeyFile r key False file
 			if copied
 				then return True
 				else continue
diff --git a/Command/Move.hs b/Command/Move.hs
index 2f2cd1b5d6..003ca27b86 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -131,7 +131,7 @@ fromPerform src move key = moveLock move key $ do
 		then handle move True
 		else do
 			showAction $ "from " ++ Remote.name src
-			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
+			ok <- getViaTmp key $ Remote.retrieveKeyFile src key False
 			handle move ok
 	where
 		handle _ False = stop -- failed
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 37f3e02e09..9a20d9e60b 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -118,8 +118,8 @@ storeEncrypted r buprepo (cipher, enck) k = do
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
-retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
-retrieve buprepo k f = do
+retrieve :: BupRepo -> Key -> Bool -> FilePath -> Annex Bool
+retrieve buprepo k _ f = do
 	let params = bupParams "join" buprepo [Param $ show k]
 	liftIO $ catchBoolIO $ do
 		tofile <- openFile f WriteMode
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 23265dabc9..9705d58435 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -109,8 +109,9 @@ storeHelper d key a = do
 		preventWrite dir
 	return ok
 
-retrieve :: FilePath -> Key -> FilePath -> Annex Bool
-retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
+retrieve :: FilePath -> Key -> Bool -> FilePath -> Annex Bool
+retrieve d k _ f = do
+	liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 7964074496..5dae3334ec 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -198,8 +198,8 @@ dropKey r key
 		]
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
-copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
-copyFromRemote r key file
+copyFromRemote :: Git.Repo -> Key -> Bool -> FilePath -> Annex Bool
+copyFromRemote r key _ file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
 		loc <- liftIO $ gitAnnexLocation key r
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 3abea7bc6a..ad99c3092e 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -55,8 +55,8 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 		store k = cip k >>= maybe
 			(storeKey r k)
 			(`storeKeyEncrypted` k)
-		retrieve k f = cip k >>= maybe
-			(retrieveKeyFile r k f)
+		retrieve k t f = cip k >>= maybe
+			(retrieveKeyFile r k t f)
 			(`retrieveKeyFileEncrypted` f)
 		withkey a k = cip k >>= maybe (a k) (a . snd)
 		cip = cipherKey c
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 6c4a044ac9..88124133aa 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -106,8 +106,8 @@ storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
-retrieve :: String -> Key -> FilePath -> Annex Bool
-retrieve h k f = runHook h "retrieve" k (Just f) $ return True
+retrieve :: String -> Key -> Bool -> FilePath -> Annex Bool
+retrieve h k _ f = runHook h "retrieve" k (Just f) $ return True
 
 retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index eeb116675f..b4ff3d6f17 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -104,9 +104,9 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	rsyncSend o enck tmp
 
-retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> do
-	unlessM (liftIO $ doesFileExist f) $ whenM (inAnnex k) $ preseed
+retrieve :: RsyncOpts -> Key -> Bool -> FilePath -> Annex Bool
+retrieve o k tmp f = untilTrue (rsyncUrls o k) $ \u -> do
+	when tmp $ preseed
 	rsyncRemote o
 		-- use inplace when retrieving to support resuming
 		[ Param "--inplace"
@@ -115,14 +115,15 @@ retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> do
 		]
 	where
 		-- this speeds up fsck --from
-		preseed = do
-			s <- inRepo $ gitAnnexLocation k
-			liftIO $ whenM (copyFileExternal s f) $
-				allowWrite f
+		preseed = unlessM (liftIO $ doesFileExist f) $
+			whenM (inAnnex k) $ do
+				s <- inRepo $ gitAnnexLocation k
+				liftIO $ whenM (copyFileExternal s f) $
+					allowWrite f
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
-	res <- retrieve o enck tmp
+	res <- retrieve o enck False tmp
 	if res
 		then liftIO $ catchBoolIO $ do
 			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
diff --git a/Remote/S3.hs b/Remote/S3.hs
index bef89b5539..b879448244 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -149,8 +149,8 @@ storeHelper (conn, bucket) r k file = do
 		xheaders = filter isxheader $ M.assocs $ fromJust $ config r
 		isxheader (h, _) = "x-amz-" `isPrefixOf` h
 
-retrieve :: Remote -> Key -> FilePath -> Annex Bool
-retrieve r k f = s3Action r False $ \(conn, bucket) -> do
+retrieve :: Remote -> Key -> Bool -> FilePath -> Annex Bool
+retrieve r k _ f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket k
 	case res of
 		Right o -> do
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 4d6348e596..6db3429ebe 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -48,8 +48,8 @@ gen r _ _ =
 		remotetype = remote
 	}
 
-downloadKey :: Key -> FilePath -> Annex Bool
-downloadKey key file = get =<< getUrls key
+downloadKey :: Key -> Bool -> FilePath -> Annex Bool
+downloadKey key _ file = get =<< getUrls key
 	where
 		get [] = do
 			warning "no known url"
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 216b34857d..d524ea2ca5 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -43,8 +43,8 @@ data RemoteA a = Remote {
 	cost :: Int,
 	-- Transfers a key to the remote.
 	storeKey :: Key -> a Bool,
-	-- retrieves a key's contents to a file
-	retrieveKeyFile :: Key -> FilePath -> a Bool,
+	-- retrieves a key's contents to a file (possibly a tmp file)
+	retrieveKeyFile :: Key -> Bool -> FilePath -> a Bool,
 	-- removes a key's contents
 	removeKey :: Key -> a Bool,
 	-- Checks if a key is present in the remote; if the remote

From 71cb04bb6d7d787181f158cad15a67628b0b4402 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 16:14:40 -0400
Subject: [PATCH 3013/8313] optimize fsck --from directory special remote

No need to copy anything, just symlink to the file.
---
 Remote/Directory.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 9705d58435..5cdb89f33d 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -110,8 +110,10 @@ storeHelper d key a = do
 	return ok
 
 retrieve :: FilePath -> Key -> Bool -> FilePath -> Annex Bool
-retrieve d k _ f = do
-	liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
+retrieve d k tmp f = liftIO $ withStoredFile d k $ \file ->
+	if tmp
+		then catchBoolIO $ createSymbolicLink file f >> return True
+		else copyFileExternal file f
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =

From f35a84fac750d8e246f3fcd1f25054951eff8b7e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 16:46:15 -0400
Subject: [PATCH 3014/8313] use a different tmp file when fscking remote data

Since the content might be symlinked into place, it's not appropriate to
use withTmp here.
---
 Command/Fsck.hs | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 77e189f436..9d856ce889 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -58,13 +58,13 @@ perform key file backend numcopies = check
 {- To fsck a remote, the content is retrieved to a tmp file,
  - and checked locally. -}
 performRemote :: Key -> FilePath -> Backend -> Maybe Int -> Remote -> CommandPerform
-performRemote key file backend numcopies remote = withTmp key $ \tmpfile -> do
+performRemote key file backend numcopies remote = do
 	v <- Remote.hasKey remote key
 	case v of
 		Left err -> do
 			showNote err
 			stop
-		Right True -> do
+		Right True -> withtmp $ \tmpfile -> do
 			copied <- Remote.retrieveKeyFile remote key True tmpfile
 			if copied then go True (Just tmpfile) else go False Nothing
 		Right False -> go False Nothing
@@ -75,6 +75,14 @@ performRemote key file backend numcopies remote = withTmp key $ \tmpfile -> do
 			, checkBackendRemote backend key remote localcopy
 			, checkKeyNumCopies key file numcopies
 			]
+		withtmp a = do
+			pid <- liftIO getProcessID
+			t <- fromRepo gitAnnexTmpDir
+			let tmp = t  "fsck" ++ show pid ++ "." ++ keyFile key
+			liftIO $ createDirectoryIfMissing True t
+			let cleanup = liftIO $ catch (removeFile tmp) (const $ return ())
+			cleanup
+			cleanup `after` a tmp
 
 {- To fsck a bare repository, fsck each key in the location log. -}
 withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek

From effaa298fabed963ec8a616d206662682e70e61a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 17:05:39 -0400
Subject: [PATCH 3015/8313] optimise fsck --from normal git remotes

For a local git remote, can symlink the file.
For a git remote using rsync, can preseed any local content.

There are a few reasons to use fsck --from on a normal git remote.
One is if it's using gitosis or similar, and you don't have shell access
to run git annex locally. Another reason could be if you just want to
fsck certian files of a bare remote.
---
 Annex/Content.hs | 11 +++++++++++
 Remote/Git.hs    | 10 +++++++---
 Remote/Rsync.hs  | 11 +----------
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index ba67a2f151..efd360a09c 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -22,6 +22,7 @@ module Annex.Content (
 	getKeysPresent,
 	saveState,
 	downloadUrl,
+	preseedTmp,
 ) where
 
 import System.IO.Error (try)
@@ -40,6 +41,7 @@ import Utility.FileMode
 import qualified Utility.Url as Url
 import Types.Key
 import Utility.DataUnits
+import Utility.CopyFile
 import Config
 import Annex.Exception
 
@@ -301,3 +303,12 @@ downloadUrl urls file = do
 	g <- gitRepo
 	o <- map Param . words <$> getConfig g "web-options" ""
 	liftIO $ anyM (\u -> Url.download u o file) urls
+
+{- Copies a key's content, when present, to a temp file.
+ - This is used to speed up some rsyncs. -}
+preseedTmp :: Key -> FilePath -> Annex ()
+preseedTmp key file =
+	unlessM (liftIO $ doesFileExist file) $ whenM (inAnnex key) $ do
+		s <- inRepo $ gitAnnexLocation key
+		liftIO $ whenM (copyFileExternal s file) $
+			allowWrite file
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 5dae3334ec..2196292cdf 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -199,12 +199,16 @@ dropKey r key
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
 copyFromRemote :: Git.Repo -> Key -> Bool -> FilePath -> Annex Bool
-copyFromRemote r key _ file
+copyFromRemote r key tmp file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
 		loc <- liftIO $ gitAnnexLocation key r
-		rsyncOrCopyFile params loc file
-	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
+		if tmp
+			then liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True
+			else rsyncOrCopyFile params loc file
+	| Git.repoIsSsh r = do
+		when tmp $ Annex.Content.preseedTmp key file
+		rsyncHelper =<< rsyncParamsRemote r True key file
 	| Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
 
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index b4ff3d6f17..a1722fe17d 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -19,8 +19,6 @@ import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 import Utility.RsyncFile
-import Utility.CopyFile
-import Utility.FileMode
 
 type RsyncUrl = String
 
@@ -106,20 +104,13 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 
 retrieve :: RsyncOpts -> Key -> Bool -> FilePath -> Annex Bool
 retrieve o k tmp f = untilTrue (rsyncUrls o k) $ \u -> do
-	when tmp $ preseed
+	when tmp $ preseedTmp k f
 	rsyncRemote o
 		-- use inplace when retrieving to support resuming
 		[ Param "--inplace"
 		, Param u
 		, Param f
 		]
-	where
-		-- this speeds up fsck --from
-		preseed = unlessM (liftIO $ doesFileExist f) $
-			whenM (inAnnex k) $ do
-				s <- inRepo $ gitAnnexLocation k
-				liftIO $ whenM (copyFileExternal s f) $
-					allowWrite f
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do

From 7fa95eff5e0bae92b27db793f8e2f4972218acd4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 19 Jan 2012 23:10:25 +0000
Subject: [PATCH 3016/8313]

---
 doc/tips/visualizing_repositories_with_gource.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/visualizing_repositories_with_gource.mdwn b/doc/tips/visualizing_repositories_with_gource.mdwn
index 884f70c444..25a69c1b7a 100644
--- a/doc/tips/visualizing_repositories_with_gource.mdwn
+++ b/doc/tips/visualizing_repositories_with_gource.mdwn
@@ -15,8 +15,8 @@ or removed from them with git-annex.
 To use gource this way, first go into the directory you want to visualize,
 and use `git annex log` to make an input file for `gource`:
 
-	git annex log --gource | tee gorce.log
+	git annex log --gource | tee gource.log
 	sort gource.log | gource --log-format custom -
 
 The `git annex log` can take a while, to speed it up you can use something
-like `--after "4 monts ago"` to limit how far back it goes.
+like `--after "4 months ago"` to limit how far back it goes.

From 3783ccf2529b7e566183684579835bb9541a4596 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 20:41:20 -0400
Subject: [PATCH 3017/8313] design

---
 ...ated_password_prompts_for_one_command.mdwn | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
index 808b8496f7..e5b5e3c5cc 100644
--- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
+++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
@@ -16,3 +16,34 @@ Simple, when performing various git annex command over ssh, in particular a mult
 >> 
 >> Complicated slightly by not doing this if the user has already set up
 >> more broad ssh connection caching. --[[Joey]]
+
+---
+
+Slightly more elaborate design for using ssh connection caching:
+
+* Per-uuid ssh socket in `.git/annex/ssh/user@host.socket`
+* Can be shared amoung concurrent git-annex processes.
+* Run ssh like: `ssh -S .git/annex/ssh/user@host.socket -o ControlMaster=auto -o ControlPersist=yes user@host`
+* At shutdown, enumerate all existing sockets, and on each:
+  1. Rename to .old (prevents various races)
+  2. `ssh -q -S .git/annex/ssh/user@host.old -o ControlMaster=auto -o ControlPersist=yes -O stop user@host`
+     (Will exit nonzero if ssh is not running on that socket.)
+  3. And then remove the socket.
+* Do same *at startup*. Why? In case an old git-annex was interrupted
+  and left behind a ssh. May have moved to a different network
+  in the meantime, etc, and be stalled waiting for a response from the
+  network, or talking to the wrong interface or something.
+  (Ie, the reason why I don't use ssh connection caching by default.)
+* This would stop ssh's used by a concurrently running git-annex,
+  but only after they finish servicing their current connection.
+  Could use locks to detect if another git-annex is using a ssh
+  socket, but concurrent git-annex is rare enough, and the impact small
+  enough (next ssh it runs needs to do a full connect), that
+  the locks are probably not justified. Could be added later if needed tho.
+* Could also set ControlPersist to something like "1h", in order to
+  auto-terminate leftover ssh's when git-annex is ctrl-c'd or somehow
+  exits. When transferring big enough files that the next ssh doesn't
+  happen for an hour, the overhead of that ssh needing to reconnect is
+  not significant.
+* User should be able to override this, to use their own preferred
+  connection caching setup. `annex.sshcaching=false`

From e96726caa31fd76413b450790860611f71d13915 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 19 Jan 2012 21:15:13 -0400
Subject: [PATCH 3018/8313] better design

Avoids possible repeated password prompts, at the cost of a small bit of
locking complication.
---
 ...ated_password_prompts_for_one_command.mdwn | 24 +++++++------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
index e5b5e3c5cc..a047370ac7 100644
--- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
+++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
@@ -22,28 +22,22 @@ Simple, when performing various git annex command over ssh, in particular a mult
 Slightly more elaborate design for using ssh connection caching:
 
 * Per-uuid ssh socket in `.git/annex/ssh/user@host.socket`
-* Can be shared amoung concurrent git-annex processes.
+* Can be shared amoung concurrent git-annex processes as well as ssh
+  invocations inside the current git-annex.
+* Also a lock file, `.git/annex/ssh/user@host.lock`.
+  Open and take shared lock before running ssh; store lock in lock pool.
+  (Not locking socket directly, because ssh might want to.)
 * Run ssh like: `ssh -S .git/annex/ssh/user@host.socket -o ControlMaster=auto -o ControlPersist=yes user@host`
 * At shutdown, enumerate all existing sockets, and on each:
-  1. Rename to .old (prevents various races)
-  2. `ssh -q -S .git/annex/ssh/user@host.old -o ControlMaster=auto -o ControlPersist=yes -O stop user@host`
+  1. Drop any shared lock.
+  2. Attempt to take an exclusive lock (non-blocking).
+  3. `ssh -q -S .git/annex/ssh/user@host.socket -o ControlMaster=auto -o ControlPersist=yes -O stop user@host`
      (Will exit nonzero if ssh is not running on that socket.)
-  3. And then remove the socket.
+  4. And then remove the socket and the lock file.
 * Do same *at startup*. Why? In case an old git-annex was interrupted
   and left behind a ssh. May have moved to a different network
   in the meantime, etc, and be stalled waiting for a response from the
   network, or talking to the wrong interface or something.
   (Ie, the reason why I don't use ssh connection caching by default.)
-* This would stop ssh's used by a concurrently running git-annex,
-  but only after they finish servicing their current connection.
-  Could use locks to detect if another git-annex is using a ssh
-  socket, but concurrent git-annex is rare enough, and the impact small
-  enough (next ssh it runs needs to do a full connect), that
-  the locks are probably not justified. Could be added later if needed tho.
-* Could also set ControlPersist to something like "1h", in order to
-  auto-terminate leftover ssh's when git-annex is ctrl-c'd or somehow
-  exits. When transferring big enough files that the next ssh doesn't
-  happen for an hour, the overhead of that ssh needing to reconnect is
-  not significant.
 * User should be able to override this, to use their own preferred
   connection caching setup. `annex.sshcaching=false`

From 61dbad505d648f13394018c31ce2d718c175007e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jan 2012 13:23:11 -0400
Subject: [PATCH 3019/8313] fsck --from remote --fast

Avoids expensive file transfers, at the expense of checking file size
and/or contents.

Required some reworking of the remote code.
---
 Annex/Content.hs             | 21 +++++++++++++++------
 Command/Fsck.hs              | 14 ++++++++++++--
 Command/Get.hs               |  2 +-
 Command/Move.hs              |  2 +-
 Remote.hs                    |  1 +
 Remote/Bup.hs                |  8 ++++++--
 Remote/Directory.hs          | 12 +++++++-----
 Remote/Git.hs                | 25 +++++++++++++++++--------
 Remote/Helper/Encryptable.hs |  8 ++++++--
 Remote/Hook.hs               |  8 ++++++--
 Remote/Rsync.hs              | 26 ++++++++++++++++----------
 Remote/S3.hs                 |  8 ++++++--
 Remote/Web.hs                |  8 ++++++--
 Types/Remote.hs              |  6 ++++--
 debian/changelog             |  2 ++
 doc/git-annex.mdwn           |  5 +++--
 16 files changed, 109 insertions(+), 47 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index efd360a09c..c21ac405ea 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -306,9 +306,18 @@ downloadUrl urls file = do
 
 {- Copies a key's content, when present, to a temp file.
  - This is used to speed up some rsyncs. -}
-preseedTmp :: Key -> FilePath -> Annex ()
-preseedTmp key file =
-	unlessM (liftIO $ doesFileExist file) $ whenM (inAnnex key) $ do
-		s <- inRepo $ gitAnnexLocation key
-		liftIO $ whenM (copyFileExternal s file) $
-			allowWrite file
+preseedTmp :: Key -> FilePath -> Annex Bool
+preseedTmp key file = go =<< inAnnex key
+	where
+		go False = return False
+		go True = do
+			ok <- copy
+			when ok $ liftIO $ allowWrite file
+			return ok
+		copy = do
+			present <- liftIO $ doesFileExist file
+			if present
+				then return True
+				else do
+					s <- inRepo $ gitAnnexLocation key
+					liftIO $ copyFileExternal s file
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 9d856ce889..59af29edb1 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -9,6 +9,7 @@ module Command.Fsck where
 
 import Common.Annex
 import Command
+import qualified Annex
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
@@ -65,8 +66,8 @@ performRemote key file backend numcopies remote = do
 			showNote err
 			stop
 		Right True -> withtmp $ \tmpfile -> do
-			copied <- Remote.retrieveKeyFile remote key True tmpfile
-			if copied then go True (Just tmpfile) else go False Nothing
+			copied <- getfile tmpfile
+			if copied then go True (Just tmpfile) else go True Nothing
 		Right False -> go False Nothing
 	where
 		go present localcopy = check
@@ -83,6 +84,15 @@ performRemote key file backend numcopies remote = do
 			let cleanup = liftIO $ catch (removeFile tmp) (const $ return ())
 			cleanup
 			cleanup `after` a tmp
+		getfile tmp = do
+			ok <- Remote.retrieveKeyFileCheap remote key tmp
+			if ok
+				then return ok
+				else do
+					fast <- Annex.getState Annex.fast
+					if fast
+						then return False
+						else Remote.retrieveKeyFile remote key tmp
 
 {- To fsck a bare repository, fsck each key in the location log. -}
 withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
diff --git a/Command/Get.hs b/Command/Get.hs
index 7f5c08a7e6..5d032e13c4 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -72,7 +72,7 @@ getKeyFile key file = do
 				else return True
 		docopy r continue = do
 			showAction $ "from " ++ Remote.name r
-			copied <- Remote.retrieveKeyFile r key False file
+			copied <- Remote.retrieveKeyFile r key file
 			if copied
 				then return True
 				else continue
diff --git a/Command/Move.hs b/Command/Move.hs
index 003ca27b86..2f2cd1b5d6 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -131,7 +131,7 @@ fromPerform src move key = moveLock move key $ do
 		then handle move True
 		else do
 			showAction $ "from " ++ Remote.name src
-			ok <- getViaTmp key $ Remote.retrieveKeyFile src key False
+			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			handle move ok
 	where
 		handle _ False = stop -- failed
diff --git a/Remote.hs b/Remote.hs
index 133d3e2742..ffb53446b4 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -11,6 +11,7 @@ module Remote (
 	name,
 	storeKey,
 	retrieveKeyFile,
+	retrieveKeyFileCheap,
 	removeKey,
 	hasKey,
 	hasKeyCheap,
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 9a20d9e60b..7329167dae 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -50,6 +50,7 @@ gen r u c = do
 			name = Git.repoDescribe r,
  			storeKey = store r buprepo,
 			retrieveKeyFile = retrieve buprepo,
+			retrieveKeyFileCheap = retrieveCheap buprepo,
 			removeKey = remove,
 			hasKey = checkPresent r bupr',
 			hasKeyCheap = bupLocal buprepo,
@@ -118,13 +119,16 @@ storeEncrypted r buprepo (cipher, enck) k = do
 		withEncryptedHandle cipher (L.readFile src) $ \h ->
 			pipeBup params (Just h) Nothing
 
-retrieve :: BupRepo -> Key -> Bool -> FilePath -> Annex Bool
-retrieve buprepo k _ f = do
+retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
+retrieve buprepo k f = do
 	let params = bupParams "join" buprepo [Param $ show k]
 	liftIO $ catchBoolIO $ do
 		tofile <- openFile f WriteMode
 		pipeBup params Nothing (Just tofile)
 
+retrieveCheap :: BupRepo -> Key -> FilePath -> Annex Bool
+retrieveCheap _ _ _ = return False
+
 retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted buprepo (cipher, enck) f = do
 	let params = bupParams "join" buprepo [Param $ show enck]
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 5cdb89f33d..52f4263409 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -41,6 +41,7 @@ gen r u c = do
 			name = Git.repoDescribe r,
  			storeKey = store dir,
 			retrieveKeyFile = retrieve dir,
+			retrieveKeyFileCheap = retrieveCheap dir,
 			removeKey = remove dir,
 			hasKey = checkPresent dir,
 			hasKeyCheap = True,
@@ -109,11 +110,12 @@ storeHelper d key a = do
 		preventWrite dir
 	return ok
 
-retrieve :: FilePath -> Key -> Bool -> FilePath -> Annex Bool
-retrieve d k tmp f = liftIO $ withStoredFile d k $ \file ->
-	if tmp
-		then catchBoolIO $ createSymbolicLink file f >> return True
-		else copyFileExternal file f
+retrieve :: FilePath -> Key -> FilePath -> Annex Bool
+retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
+
+retrieveCheap :: FilePath -> Key -> FilePath -> Annex Bool
+retrieveCheap d k f = liftIO $ withStoredFile d k $ \file ->
+	catchBoolIO $ createSymbolicLink file f >> return True
 
 retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted d (cipher, enck) f =
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 2196292cdf..efe1829610 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -75,6 +75,7 @@ gen r u _ = do
 		name = Git.repoDescribe r',
 		storeKey = copyToRemote r',
 		retrieveKeyFile = copyFromRemote r',
+		retrieveKeyFileCheap = copyFromRemoteCheap r',
 		removeKey = dropKey r',
 		hasKey = inAnnex r',
 		hasKeyCheap = cheap,
@@ -198,20 +199,28 @@ dropKey r key
 		]
 
 {- Tries to copy a key's content from a remote's annex to a file. -}
-copyFromRemote :: Git.Repo -> Key -> Bool -> FilePath -> Annex Bool
-copyFromRemote r key tmp file
+copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool
+copyFromRemote r key file
 	| not $ Git.repoIsUrl r = do
 		params <- rsyncParams r
 		loc <- liftIO $ gitAnnexLocation key r
-		if tmp
-			then liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True
-			else rsyncOrCopyFile params loc file
-	| Git.repoIsSsh r = do
-		when tmp $ Annex.Content.preseedTmp key file
-		rsyncHelper =<< rsyncParamsRemote r True key file
+		rsyncOrCopyFile params loc file
+	| Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file
 	| Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) file
 	| otherwise = error "copying from non-ssh, non-http repo not supported"
 
+copyFromRemoteCheap :: Git.Repo -> Key -> FilePath -> Annex Bool
+copyFromRemoteCheap r key file
+	| not $ Git.repoIsUrl r = do
+		loc <- liftIO $ gitAnnexLocation key r
+		liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True
+	| Git.repoIsSsh r = do
+		ok <- Annex.Content.preseedTmp key file
+		if ok
+			then copyFromRemote r key file
+			else return False
+	| otherwise = return False
+
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index ad99c3092e..0569cb5551 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -47,6 +47,7 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 	r {
 		storeKey = store,
 		retrieveKeyFile = retrieve,
+		retrieveKeyFileCheap = retrieveCheap,
 		removeKey = withkey $ removeKey r,
 		hasKey = withkey $ hasKey r,
 		cost = cost r + encryptedRemoteCostAdj
@@ -55,9 +56,12 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 		store k = cip k >>= maybe
 			(storeKey r k)
 			(`storeKeyEncrypted` k)
-		retrieve k t f = cip k >>= maybe
-			(retrieveKeyFile r k t f)
+		retrieve k f = cip k >>= maybe
+			(retrieveKeyFile r k f)
 			(`retrieveKeyFileEncrypted` f)
+		retrieveCheap k f = cip k >>= maybe
+			(retrieveKeyFileCheap r k f)
+			(\_ -> return False)
 		withkey a k = cip k >>= maybe (a k) (a . snd)
 		cip = cipherKey c
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 88124133aa..a08c4011ef 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -41,6 +41,7 @@ gen r u c = do
 			name = Git.repoDescribe r,
  			storeKey = store hooktype,
 			retrieveKeyFile = retrieve hooktype,
+			retrieveKeyFileCheap = retrieveCheap hooktype,
 			removeKey = remove hooktype,
 			hasKey = checkPresent r hooktype,
 			hasKeyCheap = False,
@@ -106,8 +107,11 @@ storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	runHook h "store" enck (Just tmp) $ return True
 
-retrieve :: String -> Key -> Bool -> FilePath -> Annex Bool
-retrieve h k _ f = runHook h "retrieve" k (Just f) $ return True
+retrieve :: String -> Key -> FilePath -> Annex Bool
+retrieve h k f = runHook h "retrieve" k (Just f) $ return True
+
+retrieveCheap :: String -> Key -> FilePath -> Annex Bool
+retrieveCheap _ _ _ = return False
 
 retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index a1722fe17d..8de6ba6a74 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -48,6 +48,7 @@ gen r u c = do
 			name = Git.repoDescribe r,
  			storeKey = store o,
 			retrieveKeyFile = retrieve o,
+			retrieveKeyFileCheap = retrieveCheap o,
 			removeKey = remove o,
 			hasKey = checkPresent r o,
 			hasKeyCheap = False,
@@ -102,19 +103,24 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
 	liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp
 	rsyncSend o enck tmp
 
-retrieve :: RsyncOpts -> Key -> Bool -> FilePath -> Annex Bool
-retrieve o k tmp f = untilTrue (rsyncUrls o k) $ \u -> do
-	when tmp $ preseedTmp k f
-	rsyncRemote o
-		-- use inplace when retrieving to support resuming
-		[ Param "--inplace"
-		, Param u
-		, Param f
-		]
+retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool
+retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o
+	-- use inplace when retrieving to support resuming
+	[ Param "--inplace"
+	, Param u
+	, Param f
+	]
+
+retrieveCheap :: RsyncOpts -> Key -> FilePath -> Annex Bool
+retrieveCheap o k f = do
+	ok <- preseedTmp k f
+	if ok
+		then retrieve o k f
+		else return False
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
-	res <- retrieve o enck False tmp
+	res <- retrieve o enck tmp
 	if res
 		then liftIO $ catchBoolIO $ do
 			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
diff --git a/Remote/S3.hs b/Remote/S3.hs
index b879448244..1d23b7d6f0 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -53,6 +53,7 @@ gen' r u c cst =
 			name = Git.repoDescribe r,
 	 		storeKey = store this,
 			retrieveKeyFile = retrieve this,
+			retrieveKeyFileCheap = retrieveCheap this,
 			removeKey = remove this,
 			hasKey = checkPresent this,
 			hasKeyCheap = False,
@@ -149,8 +150,8 @@ storeHelper (conn, bucket) r k file = do
 		xheaders = filter isxheader $ M.assocs $ fromJust $ config r
 		isxheader (h, _) = "x-amz-" `isPrefixOf` h
 
-retrieve :: Remote -> Key -> Bool -> FilePath -> Annex Bool
-retrieve r k _ f = s3Action r False $ \(conn, bucket) -> do
+retrieve :: Remote -> Key -> FilePath -> Annex Bool
+retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket k
 	case res of
 		Right o -> do
@@ -158,6 +159,9 @@ retrieve r k _ f = s3Action r False $ \(conn, bucket) -> do
 			return True
 		Left e -> s3Warning e
 
+retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool
+retrieveCheap _ _ _ = return False
+
 retrieveEncrypted :: Remote -> (Cipher, Key) -> FilePath -> Annex Bool
 retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket enck
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 6db3429ebe..49c3f43f3a 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -40,6 +40,7 @@ gen r _ _ =
 		name = Git.repoDescribe r,
 		storeKey = uploadKey,
 		retrieveKeyFile = downloadKey,
+		retrieveKeyFileCheap = downloadKeyCheap,
 		removeKey = dropKey,
 		hasKey = checkKey,
 		hasKeyCheap = False,
@@ -48,8 +49,8 @@ gen r _ _ =
 		remotetype = remote
 	}
 
-downloadKey :: Key -> Bool -> FilePath -> Annex Bool
-downloadKey key _ file = get =<< getUrls key
+downloadKey :: Key -> FilePath -> Annex Bool
+downloadKey key file = get =<< getUrls key
 	where
 		get [] = do
 			warning "no known url"
@@ -58,6 +59,9 @@ downloadKey key _ file = get =<< getUrls key
 			showOutput -- make way for download progress bar
 			downloadUrl urls file
 
+downloadKeyCheap :: Key -> FilePath -> Annex Bool
+downloadKeyCheap _ _ = return False
+
 uploadKey :: Key -> Annex Bool
 uploadKey _ = do
 	warning "upload to web not supported"
diff --git a/Types/Remote.hs b/Types/Remote.hs
index d524ea2ca5..003dd5342a 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -43,8 +43,10 @@ data RemoteA a = Remote {
 	cost :: Int,
 	-- Transfers a key to the remote.
 	storeKey :: Key -> a Bool,
-	-- retrieves a key's contents to a file (possibly a tmp file)
-	retrieveKeyFile :: Key -> Bool -> FilePath -> a Bool,
+	-- retrieves a key's contents to a file
+	retrieveKeyFile :: Key -> FilePath -> a Bool,
+	-- retrieves a key's contents to a tmp file, if it can be done cheaply
+	retrieveKeyFileCheap :: Key -> FilePath -> a Bool,
 	-- removes a key's contents
 	removeKey :: Key -> a Bool,
 	-- Checks if a key is present in the remote; if the remote
diff --git a/debian/changelog b/debian/changelog
index 6849931486..5adba128f7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,8 @@ git-annex (3.20120117) UNRELEASED; urgency=low
   * If you have any directory special remotes, now would be a good time to
     fsck them, in case you were hit by the data loss bug fixed in the
     previous release!
+  * fsck --from remote --fast: Avoids expensive file transfers, at the
+    expense of checking file size and/or contents.
 
  -- Joey Hess   Thu, 19 Jan 2012 15:12:03 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index edf300d8d7..a377665c68 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -210,9 +210,10 @@ subdirectories).
 
   With parameters, only the specified files are checked.
 
-  To avoid expensive checksum calculations, specify --fast
-
   To check a remote to fsck, specify --from.
+  
+  To avoid expensive checksum calculations (and expensive transfers when
+  fscking a remote), specify --fast
 
 * unused
 

From 25f998679cd68cd4bb9b320998253f1b2ae23315 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jan 2012 15:06:17 -0400
Subject: [PATCH 3020/8313] typo

---
 Utility/Misc.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 1d3c0e6763..c9bfcb953a 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -25,7 +25,7 @@ readFileStrict = readFile >=> \s -> length s `seq` return s
  - in the second result list.
  -
  - separate (== ':') "foo:bar" = ("foo", "bar")
- - separate (== ':') "foobar" = ("foo, "")
+ - separate (== ':') "foobar" = ("foobar", "")
  -}
 separate :: (a -> Bool) -> [a] -> ([a], [a])
 separate c l = unbreak $ break c l

From 47250a153a6c5a2864fec15fb136290683aeb1c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jan 2012 15:34:52 -0400
Subject: [PATCH 3021/8313] ssh connection caching

Ssh connection caching is now enabled automatically by git-annex. Only one
ssh connection is made to each host per git-annex run, which can speed some
things up a lot, as well as avoiding repeated password prompts. Concurrent
git-annex processes also share ssh connections. Cached ssh connections are
shut down when git-annex exits.

Note: The rsync special remote does not yet participate in the ssh
connection caching.
---
 Annex.hs                                      |   3 +
 Annex/LockPool.hs                             |  43 +++++++
 Annex/Ssh.hs                                  | 107 ++++++++++++++++++
 CmdLine.hs                                    |   2 +
 Locations.hs                                  |   5 +
 Remote/Helper/Ssh.hs                          |  14 +--
 debian/changelog                              |   5 +
 ..._no_fixed_hostname_and_optimising_ssh.mdwn |  13 ---
 ...ated_password_prompts_for_one_command.mdwn |   4 +-
 9 files changed, 173 insertions(+), 23 deletions(-)
 create mode 100644 Annex/LockPool.hs
 create mode 100644 Annex/Ssh.hs

diff --git a/Annex.hs b/Annex.hs
index b365132e5d..3b79ea2700 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -29,6 +29,7 @@ module Annex (
 import Control.Monad.State
 import Control.Monad.Trans.Control (StM, MonadBaseControl, liftBaseWith, restoreM)
 import Control.Monad.Base (liftBase, MonadBase)
+import System.Posix.Types (Fd)
 
 import Common
 import qualified Git
@@ -86,6 +87,7 @@ data AnnexState = AnnexState
 	, forcetrust :: TrustMap
 	, trustmap :: Maybe TrustMap
 	, ciphers :: M.Map EncryptedCipher Cipher
+	, lockpool :: M.Map FilePath Fd
 	, flags :: M.Map String Bool
 	, fields :: M.Map String String
 	}
@@ -108,6 +110,7 @@ newState gitrepo = AnnexState
 	, forcetrust = M.empty
 	, trustmap = Nothing
 	, ciphers = M.empty
+	, lockpool = M.empty
 	, flags = M.empty
 	, fields = M.empty
 	}
diff --git a/Annex/LockPool.hs b/Annex/LockPool.hs
new file mode 100644
index 0000000000..3fede5739b
--- /dev/null
+++ b/Annex/LockPool.hs
@@ -0,0 +1,43 @@
+{- git-annex lock pool
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.LockPool where
+
+import qualified Data.Map as M
+import System.Posix.Types (Fd)
+
+import Common.Annex
+import Annex
+
+{- Create a specified lock file, and takes a shared lock. -}
+lockFile :: FilePath -> Annex ()
+lockFile file = go =<< fromPool file
+	where
+		go (Just _) = return () -- already locked
+		go Nothing = do
+			fd <- liftIO $ openFd file ReadOnly (Just stdFileMode) defaultFileFlags
+			liftIO $ waitToSetLock fd (ReadLock, AbsoluteSeek, 0, 0)
+			changePool $ M.insert file fd
+
+unlockFile :: FilePath -> Annex ()
+unlockFile file = go =<< fromPool file
+	where
+		go Nothing = return ()
+		go (Just fd) = do
+			liftIO $ closeFd fd
+			changePool $ M.delete file
+
+getPool :: Annex (M.Map FilePath Fd)
+getPool = getState lockpool
+
+fromPool :: FilePath -> Annex (Maybe Fd)
+fromPool file = M.lookup file <$> getPool
+
+changePool :: (M.Map FilePath Fd -> M.Map FilePath Fd) -> Annex ()
+changePool a = do
+	m <- getPool
+	changeState $ \s -> s { lockpool = a m }
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
new file mode 100644
index 0000000000..cd832a373f
--- /dev/null
+++ b/Annex/Ssh.hs
@@ -0,0 +1,107 @@
+{- git-annex ssh interface, with connection caching
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.Ssh (
+	sshParams,
+	sshCleanup,
+) where
+
+import qualified Data.Map as M
+import System.IO.Error (try)
+
+import Common.Annex
+import Annex.LockPool
+
+{- Generates parameters to ssh to a given host (or user@host) on a given
+ - port, with connection caching. -}
+sshParams :: (String, Maybe Integer) -> Annex [CommandParam]
+sshParams (host, port) = do
+	cleanstale
+	(socketfile, params) <- sshInfo (host, port)
+	liftIO $ createDirectoryIfMissing True $ parentDir socketfile
+	lockFile $ socket2lock socketfile
+	return params
+	where
+		-- If the lock pool is empty, this is the first ssh of this
+		-- run. There could be stale ssh connections hanging around
+		-- from a previous git-annex run that was interrupted.
+		cleanstale = whenM (null . filter isLock . M.keys <$> getPool) $
+			sshCleanup
+
+sshInfo :: (String, Maybe Integer) -> Annex (FilePath, [CommandParam])
+sshInfo (host, port) = do
+	dir <- fromRepo $ gitAnnexSshDir
+	let socketfile = dir  hostport2socket host port
+	return $ (socketfile, cacheParams socketfile ++ portParams port ++ [Param host])
+
+cacheParams :: FilePath -> [CommandParam]
+cacheParams socketfile =
+	[ Param "-S", Param socketfile
+	, Params "-o ControlMaster=auto -o ControlPersist=yes"
+	]
+
+portParams :: Maybe Integer -> [CommandParam]
+portParams Nothing = []
+portParams (Just port) = [Param "-p", Param $ show port]
+
+{- Stop any unused ssh processes. -}
+sshCleanup :: Annex ()
+sshCleanup = do
+	dir <- fromRepo $ gitAnnexSshDir
+	liftIO $ createDirectoryIfMissing True dir
+	sockets <- filter (not . isLock) <$> liftIO (dirContents dir)
+	forM_ sockets cleanup
+	where
+		cleanup socketfile = do
+			-- Drop any shared lock we have, and take an
+			-- exclusive lock, without blocking. If the lock
+			-- succeeds, nothing is using this ssh, and it can
+			-- be stopped.
+			let lockfile = socket2lock socketfile
+			unlockFile lockfile
+			fd <- liftIO $ openFd lockfile ReadWrite (Just stdFileMode) defaultFileFlags
+			v <- liftIO $ try $ setLock fd (WriteLock, AbsoluteSeek, 0, 0)
+			case v of
+				Left _ -> return ()
+				Right _ -> stopssh socketfile
+			liftIO $ closeFd fd
+		stopssh socketfile = do
+			(_, params) <- sshInfo $ socket2hostport socketfile
+			_ <- liftIO $ do
+				-- "ssh -O stop" is noisy on stderr even with -q
+				let cmd = unwords $ toCommand $
+					[ Params "-O stop"
+					] ++ params
+				_ <- boolSystem "sh"
+					[ Param "-c"
+					, Param $ "ssh " ++ cmd ++ " >/dev/null 2>/dev/null"
+					]
+				--try $ removeFile socketfile
+				return ()
+			-- Cannot remove the lock file; other processes may
+			-- be waiting on our exclusive lock to use it.
+			return ()
+
+hostport2socket :: String -> Maybe Integer -> FilePath
+hostport2socket host Nothing = host
+hostport2socket host (Just port) = host ++ "!" ++ show port
+
+socket2hostport :: FilePath -> (String, Maybe Integer)
+socket2hostport socket
+	| null p = (h, Nothing)
+	| otherwise = (h, readMaybe p)
+	where
+		(h, p) = separate (== '!') $ takeFileName socket
+
+socket2lock :: FilePath -> FilePath
+socket2lock socket = socket ++ lockExt
+
+isLock :: FilePath -> Bool
+isLock f = lockExt `isSuffixOf` f
+
+lockExt :: String
+lockExt = ".lock"
diff --git a/CmdLine.hs b/CmdLine.hs
index 68157a01a9..29b95d01bd 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -22,6 +22,7 @@ import qualified Annex.Queue
 import qualified Git
 import qualified Git.Command
 import Annex.Content
+import Annex.Ssh
 import Command
 
 type Params = [String]
@@ -92,4 +93,5 @@ shutdown :: Annex Bool
 shutdown = do
 	saveState
 	liftIO Git.Command.reap -- zombies from long-running git processes
+	sshCleanup -- ssh connection caching
 	return True
diff --git a/Locations.hs b/Locations.hs
index 73a2473b56..03d6deb1d7 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -22,6 +22,7 @@ module Locations (
 	gitAnnexJournalLock,
 	gitAnnexIndex,
 	gitAnnexIndexLock,
+	gitAnnexSshDir,
 	isLinkToAnnex,
 	annexHashes,
 	hashDirMixed,
@@ -142,6 +143,10 @@ gitAnnexIndex r = gitAnnexDir r  "index"
 gitAnnexIndexLock :: Git.Repo -> FilePath
 gitAnnexIndexLock r = gitAnnexDir r  "index.lck"
 
+{- .git/annex/ssh/ is used for ssh connection caching -}
+gitAnnexSshDir :: Git.Repo -> FilePath
+gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r  "ssh"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs
index 7c5eeddb8e..88b29fdb69 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -7,25 +7,21 @@
 
 module Remote.Helper.Ssh where
 
-import Common
+import Common.Annex
 import qualified Git
 import qualified Git.Url
-import Types
 import Config
 import Annex.UUID
+import Annex.Ssh
 
 {- Generates parameters to ssh to a repository's host and run a command.
  - Caller is responsible for doing any neccessary shellEscaping of the
  - passed command. -}
 sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
-	s <- getConfig repo "ssh-options" ""
-	let sshoptions = map Param (words s)
-	let sshport = case Git.Url.port repo of
-		Nothing -> []
-		Just p -> [Param "-p", Param (show p)]
-	let sshhost = Param $ Git.Url.hostuser repo
-	return $ sshoptions ++ sshport ++ [sshhost] ++ sshcmd
+	opts <- map Param . words <$> getConfig repo "ssh-options" ""
+	params <- sshParams (Git.Url.hostuser repo, Git.Url.port repo)
+	return $ opts ++ params ++ sshcmd
 
 {- Generates parameters to run a git-annex-shell command on a remote
  - repository. -}
diff --git a/debian/changelog b/debian/changelog
index 5adba128f7..f5fc107c7b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -10,6 +10,11 @@ git-annex (3.20120117) UNRELEASED; urgency=low
     previous release!
   * fsck --from remote --fast: Avoids expensive file transfers, at the
     expense of checking file size and/or contents.
+  * Ssh connection caching is now enabled automatically by git-annex.
+    Only one ssh connection is made to each host per git-annex run, which
+    can speed some things up a lot, as well as avoiding repeated password
+    prompts. Concurrent git-annex processes also share ssh connections.
+    Cached ssh connections are shut down when git-annex exits.
 
  -- Joey Hess   Thu, 19 Jan 2012 15:12:03 -0400
 
diff --git a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
index 8fb2bf9db1..594d8c480f 100644
--- a/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
+++ b/doc/tips/using_git_annex_with_no_fixed_hostname_and_optimising_ssh.mdwn
@@ -57,16 +57,3 @@ b) From the desktop add the remote
 So now you can work on the train, pop on the wifi at work upon arrival, and sync up with a `git pull && git annex get`.
 
 An alternative solution may be to use direct tunnels over Openvpn.
-
-## Optimising SSH
-
-Running a `git annex get .`, at least in the version I have, creates a new SSH connection for every file transfer (maybe this should be a feature request?)
-
-Lot's of new small files in an _annex_ cause lot's of connections to be made quickly: this is an relatively expensive overhead and is enough for connection limiting to start in my case.  The process can be made much faster by using SSH's connection sharing capabilities.  An SSH config like this should do it:
-
-    # Global Settings
-    ControlMaster auto
-    ControlPersist 30
-    ControlPath ~/.ssh/master-%r@%h:%p
-
-This will create a master connection for sharing if one isn't present, maintain it for 30 seconds after closing down the connection (just-in-cases') and automatically use the master connection for subsequent connections.  Wins all round!
diff --git a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
index a047370ac7..341a9afa45 100644
--- a/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
+++ b/doc/todo/wishlist:_Prevent_repeated_password_prompts_for_one_command.mdwn
@@ -15,7 +15,9 @@ Simple, when performing various git annex command over ssh, in particular a mult
 >> pid. Then at shutdown, run `ssh -O exit` on each such socket.
 >> 
 >> Complicated slightly by not doing this if the user has already set up
->> more broad ssh connection caching. --[[Joey]]
+>> more broad ssh connection caching.
+>> 
+>> [[done]]! --[[Joey]]
 
 ---
 

From 6ef82665de86195a7da6cffcba40874eba06424f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jan 2012 17:13:36 -0400
Subject: [PATCH 3022/8313] add annex.sshcaching config setting

---
 Annex/Ssh.hs       | 29 +++++++++++++++++++----------
 doc/git-annex.mdwn |  4 ++++
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index cd832a373f..7f5ba48d84 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -15,28 +15,37 @@ import System.IO.Error (try)
 
 import Common.Annex
 import Annex.LockPool
+import qualified Git
+import qualified Git.Config
 
 {- Generates parameters to ssh to a given host (or user@host) on a given
  - port, with connection caching. -}
 sshParams :: (String, Maybe Integer) -> Annex [CommandParam]
-sshParams (host, port) = do
-	cleanstale
-	(socketfile, params) <- sshInfo (host, port)
-	liftIO $ createDirectoryIfMissing True $ parentDir socketfile
-	lockFile $ socket2lock socketfile
-	return params
+sshParams (host, port) = go =<< sshInfo (host, port)
 	where
+		go (Nothing, params) = return params
+		go (Just socketfile, params) = do
+			cleanstale
+			liftIO $ createDirectoryIfMissing True $ parentDir socketfile
+			lockFile $ socket2lock socketfile
+			return params
 		-- If the lock pool is empty, this is the first ssh of this
 		-- run. There could be stale ssh connections hanging around
 		-- from a previous git-annex run that was interrupted.
 		cleanstale = whenM (null . filter isLock . M.keys <$> getPool) $
 			sshCleanup
 
-sshInfo :: (String, Maybe Integer) -> Annex (FilePath, [CommandParam])
+sshInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam])
 sshInfo (host, port) = do
-	dir <- fromRepo $ gitAnnexSshDir
-	let socketfile = dir  hostport2socket host port
-	return $ (socketfile, cacheParams socketfile ++ portParams port ++ [Param host])
+	caching <- Git.configTrue <$> fromRepo (Git.Config.get "annex.sshcaching" "true")
+	if caching
+		then do
+			dir <- fromRepo $ gitAnnexSshDir
+			let socketfile = dir  hostport2socket host port
+		 	return $ (Just socketfile, cacheParams socketfile ++ params)
+		else return (Nothing, params)
+	where
+		params = portParams port ++ [Param host]
 
 cacheParams :: FilePath -> [CommandParam]
 cacheParams socketfile =
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a377665c68..148b6336de 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -575,6 +575,10 @@ Here are all the supported configuration settings.
 
   Automatically maintained, and used to automate upgrades between versions.
 
+* `annex.sshcaching`
+
+  By default, git-annex caches ssh connections. To disable this, set to `false`.
+
 * `remote..annex-cost`
 
   When determining which repository to

From eb9001044f3db682236d1007aded58f47109d6a6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jan 2012 17:32:32 -0400
Subject: [PATCH 3023/8313] order user provided params after connection caching
 params

So the user can override them.
---
 Annex/Ssh.hs         | 15 +++++++--------
 Remote/Helper/Ssh.hs |  4 ++--
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 7f5ba48d84..c05e236040 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -20,15 +20,16 @@ import qualified Git.Config
 
 {- Generates parameters to ssh to a given host (or user@host) on a given
  - port, with connection caching. -}
-sshParams :: (String, Maybe Integer) -> Annex [CommandParam]
-sshParams (host, port) = go =<< sshInfo (host, port)
+sshParams :: (String, Maybe Integer) -> [CommandParam] -> Annex [CommandParam]
+sshParams (host, port) opts = go =<< sshInfo (host, port)
 	where
-		go (Nothing, params) = return params
+		go (Nothing, params) = ret params
 		go (Just socketfile, params) = do
 			cleanstale
 			liftIO $ createDirectoryIfMissing True $ parentDir socketfile
 			lockFile $ socket2lock socketfile
-			return params
+			ret params
+		ret ps = return $ ps ++ opts ++ portParams port ++ [Param host]
 		-- If the lock pool is empty, this is the first ssh of this
 		-- run. There could be stale ssh connections hanging around
 		-- from a previous git-annex run that was interrupted.
@@ -42,10 +43,8 @@ sshInfo (host, port) = do
 		then do
 			dir <- fromRepo $ gitAnnexSshDir
 			let socketfile = dir  hostport2socket host port
-		 	return $ (Just socketfile, cacheParams socketfile ++ params)
-		else return (Nothing, params)
-	where
-		params = portParams port ++ [Param host]
+		 	return $ (Just socketfile, cacheParams socketfile)
+		else return (Nothing, [])
 
 cacheParams :: FilePath -> [CommandParam]
 cacheParams socketfile =
diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs
index 88b29fdb69..c61d1b96f2 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -20,8 +20,8 @@ import Annex.Ssh
 sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
 	opts <- map Param . words <$> getConfig repo "ssh-options" ""
-	params <- sshParams (Git.Url.hostuser repo, Git.Url.port repo)
-	return $ opts ++ params ++ sshcmd
+	params <- sshParams (Git.Url.hostuser repo, Git.Url.port repo) opts
+	return $ params ++ sshcmd
 
 {- Generates parameters to run a git-annex-shell command on a remote
  - repository. -}

From 183bdacca219065e6a888385e3bf309708e827ec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Jan 2012 02:24:12 -0400
Subject: [PATCH 3024/8313] treak

---
 Utility/Format.hs         |  4 +++-
 Utility/Monad.hs          | 10 +++++-----
 Utility/PartialPrelude.hs |  2 +-
 Utility/TempFile.hs       |  4 +++-
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/Utility/Format.hs b/Utility/Format.hs
index 2c2042cc22..d8b7e45493 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -37,9 +37,11 @@ data Frag = Const String | Var String Justify
 data Justify = LeftJustified Int | RightJustified Int | UnJustified
 	deriving (Show)
 
+type Variables = M.Map String String
+
 {- Expands a Format using some variables, generating a formatted string.
  - This can be repeatedly called, efficiently. -}
-format :: Format -> M.Map String String -> String
+format :: Format -> Variables -> String
 format f vars = concatMap expand f
 	where
 		expand (Const s) = s
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 95964361e6..28aa33ee82 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -12,7 +12,7 @@ import Control.Monad (liftM)
 
 {- Return the first value from a list, if any, satisfying the given
  - predicate -}
-firstM :: (Monad m) => (a -> m Bool) -> [a] -> m (Maybe a)
+firstM :: Monad m => (a -> m Bool) -> [a] -> m (Maybe a)
 firstM _ [] = return Nothing
 firstM p (x:xs) = do
 	q <- p x
@@ -22,20 +22,20 @@ firstM p (x:xs) = do
 
 {- Returns true if any value in the list satisfies the predicate,
  - stopping once one is found. -}
-anyM :: (Monad m) => (a -> m Bool) -> [a] -> m Bool
+anyM :: Monad m => (a -> m Bool) -> [a] -> m Bool
 anyM p = liftM isJust . firstM p
 
 {- Runs an action on values from a list until it succeeds. -}
-untilTrue :: (Monad m) => [a] -> (a -> m Bool) -> m Bool
+untilTrue :: Monad m => [a] -> (a -> m Bool) -> m Bool
 untilTrue = flip anyM
 
 {- Runs an action, passing its value to an observer before returning it. -}
-observe :: (Monad m) => (a -> m b) -> m a -> m a
+observe :: Monad m => (a -> m b) -> m a -> m a
 observe observer a = do
 	r <- a
 	_ <- observer r
 	return r
 
 {- b `after` a runs first a, then b, and returns the value of a -}
-after :: (Monad m) => m b -> m a -> m a
+after :: Monad m => m b -> m a -> m a
 after = observe . const
diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs
index ad857196d6..507fc6252b 100644
--- a/Utility/PartialPrelude.hs
+++ b/Utility/PartialPrelude.hs
@@ -37,7 +37,7 @@ last = Prelude.last
  - Ignores leading/trailing whitespace, and throws away any trailing
  - text after the part that can be read.
  -}
-readMaybe :: (Read a) => String -> Maybe a
+readMaybe :: Read a => String -> Maybe a
 readMaybe s = case reads s of
 	((x,_):_) -> Just x
 	_ -> Nothing
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
index 3887b422b6..469d52e8ce 100644
--- a/Utility/TempFile.hs
+++ b/Utility/TempFile.hs
@@ -26,8 +26,10 @@ viaTmp a file content = do
 	a tmpfile content
 	renameFile tmpfile file
 
+type Template = String
+
 {- Runs an action with a temp file, then removes the file. -}
-withTempFile :: String -> (FilePath -> Handle -> IO a) -> IO a
+withTempFile :: Template -> (FilePath -> Handle -> IO a) -> IO a
 withTempFile template a = bracket create remove use
 	where
 		create = do

From ece239f08661929ae37afed547a8ad97bad2521d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlB7-aXsqwzOi2BIR_Q4sUF8sjj24H6F3c"
 
Date: Mon, 23 Jan 2012 18:38:14 +0000
Subject: [PATCH 3025/8313]

---
 ...in_directory_tree_below_objects__47__.mdwn | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn

diff --git a/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn
new file mode 100644
index 0000000000..d89bcee290
--- /dev/null
+++ b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__.mdwn
@@ -0,0 +1,77 @@
+Hello,
+
+I have the problem that, while git-annex preserves the file access rights (user, group, others) for the actual file, it does not make sure that others can access this file through the directory tree above said file:
+
+    /tmp $ mkdir test
+    /tmp $ chown claudius:media test
+    /tmp $ chmod 750 test
+    /tmp $ ls -dl test
+    drwxr-x--- 2 claudius media 40 2012-01-23 19:27 test/
+    /tmp $ cd test
+    /tmp/test $ git init --shared=all
+    Initialized empty shared Git repository in /tmp/test/.git/
+    /tmp/test $ git annex init "test"
+    init test ok
+    /tmp/test $ echo 123 > abc
+    /tmp/test $ chmod 640 abc
+    /tmp/test $ chown claudius:media abc
+    /tmp/test $ ls -l
+    total 4
+    -rw-r----- 1 claudius media 4 2012-01-23 19:27 abc
+    /tmp/test $ git annex add .
+    add abc (checksum...) ok
+    (Recording state in git...)
+    /tmp/test $ ls -l
+    total 4
+    lrwxrwxrwx 1 claudius claudius 176 2012-01-23 19:27 abc -> .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    /tmp/test $ ls -l .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    -r--r----- 1 claudius media 4 2012-01-23 19:27 .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    /tmp/test $ ls -lR .git/annex/objects/
+    .git/annex/objects/:
+    total 0
+    drwx--S--- 3 claudius claudius 60 2012-01-23 19:28 8F/
+    
+    .git/annex/objects/8F:
+    total 0
+    drwx--S--- 3 claudius claudius 60 2012-01-23 19:28 pj/
+    
+    .git/annex/objects/8F/pj:
+    total 0
+    dr-x--S--- 2 claudius claudius 60 2012-01-23 19:28 SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b/
+    
+    .git/annex/objects/8F/pj/SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b:
+    total 4
+    -r--r----- 1 claudius media 4 2012-01-23 19:27 SHA256-s4--181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b
+    /tmp/test $ stat .git/annex/objects/
+      File: `.git/annex/objects/'
+      Size: 60        	Blocks: 0          IO Block: 4096   directory
+    Device: 11h/17d	Inode: 2365970     Links: 3
+    Access: (2700/drwx--S---)  Uid: ( 1000/claudius)   Gid: ( 1000/claudius)
+    Access: 2012-01-23 19:28:10.614948386 +0100
+    Modify: 2012-01-23 19:28:10.614948386 +0100
+    Change: 2012-01-23 19:28:10.614948386 +0100
+     Birth: -
+
+The use case is that I have a rather large collection of music I would like to manage with git-annex in various locations (all of it on my external hard drive, some on my notebook etc. This music is played by MPD, which can access the collection because it is in the "media" group. After changing to git-annex, however, this fails.
+
+I tried to avoid this specific problem by declaring the git repository to be shared, which does appear to have some effect on the other files in .git:
+
+    /tmp/test $ ls -l .git
+    total 16
+    drwx--S---  5 claudius claudius 160 2012-01-23 19:28 annex/
+    drwxrwsr-x  2 claudius claudius  40 2012-01-23 19:27 branches/
+    -rw-rw-r--  1 claudius claudius 218 2012-01-23 19:27 config
+    -rw-rw-r--  1 claudius claudius  73 2012-01-23 19:27 description
+    -rw-rw-r--  1 claudius claudius  23 2012-01-23 19:27 HEAD
+    drwxrwsr-x  2 claudius claudius 220 2012-01-23 19:27 hooks/
+    -rw-rw-r--  1 claudius claudius 104 2012-01-23 19:28 index
+    drwxrwsr-x  2 claudius claudius  60 2012-01-23 19:27 info/
+    drwxrwsr-x  3 claudius claudius  60 2012-01-23 19:27 logs/
+    drwxrwsr-x 15 claudius claudius 300 2012-01-23 19:28 objects/
+    drwxrwsr-x  4 claudius claudius  80 2012-01-23 19:27 refs/
+
+I could obviously try to change the rights of annex/, annex/objects etc., but I would like to avoid having to adapt them each time a new folder is added somewhere below annex/objects/.
+
+My knowledge of git and especially git-annex is not too good, so it might well be that I missed something obvious. Any hints? :)
+
+(And thank you, of course, for taking the time to read all this)

From 549bf0cddb08663de4703d7b70d50a42936066d5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 23 Jan 2012 19:00:40 +0000
Subject: [PATCH 3026/8313] Added a comment

---
 ...ent_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment

diff --git a/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment
new file mode 100644
index 0000000000..d2da5e94df
--- /dev/null
+++ b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_1_5dd978f9b5a0771f44ab9e086bf5a07f._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-01-23T19:00:40Z"
+ content="""
+You say you started the repo with \"git init --shared\" .. but what that's really meant for is bare repositories, which can have several users pushing into it, not a non-bare repository.
+
+The strange mode on the directories \"dr-x--S---\" and files \"-r--r-----\" must be due to your umask setting though. My umask is 022 and the directories and files under `.git/annex/objects` are \"drwxr-xr-x\" and \"-r--r--r--\", which allows anyone to read them unless an upper directory blocks it -- and with this umask, none do unless I explicitly remove permissions from one to lock down a repository.
+
+About mpd, the obvious fix is to run mpd not as a system user but as yourself. I put \"@reboot mpd\" in my crontab to do this.
+
+
+"""]]

From 20d02888023e9707287742eb4e3568cb960c1fe8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 Jan 2012 15:09:50 -0400
Subject: [PATCH 3027/8313] releasing version 3.20120123

---
 debian/NEWS      | 2 +-
 debian/changelog | 8 +++++---
 git-annex.cabal  | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/debian/NEWS b/debian/NEWS
index 298ff1f7e0..1c95146912 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -1,4 +1,4 @@
-git-annex (3.20120119) unstable; urgency=low
+git-annex (3.20120123) unstable; urgency=low
 
   There was a bug in the handling of directory special remotes that
   could cause partial file contents to be stored in them. If you use
diff --git a/debian/changelog b/debian/changelog
index f5fc107c7b..2f573e6d3f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120117) UNRELEASED; urgency=low
+git-annex (3.20120123) unstable; urgency=low
 
   * fsck --from: Fscking a remote is now supported. It's done by retrieving
     the contents of the specified files from the remote, and checking them,
@@ -9,14 +9,16 @@ git-annex (3.20120117) UNRELEASED; urgency=low
     fsck them, in case you were hit by the data loss bug fixed in the
     previous release!
   * fsck --from remote --fast: Avoids expensive file transfers, at the
-    expense of checking file size and/or contents.
+    expense of not checking file size and/or contents.
   * Ssh connection caching is now enabled automatically by git-annex.
     Only one ssh connection is made to each host per git-annex run, which
     can speed some things up a lot, as well as avoiding repeated password
     prompts. Concurrent git-annex processes also share ssh connections.
     Cached ssh connections are shut down when git-annex exits.
+  * To disable the ssh caching (if for example you have your own broader
+    ssh caching configuration), set annex.sshcaching=false.
 
- -- Joey Hess   Thu, 19 Jan 2012 15:12:03 -0400
+ -- Joey Hess   Mon, 23 Jan 2012 13:48:48 -0400
 
 git-annex (3.20120116) unstable; urgency=medium
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 1a4bac9cfd..43901b6939 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120116
+Version: 3.20120123
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From bed495db644cc49f45ec2a4f49a572308fc586f7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 Jan 2012 15:10:10 -0400
Subject: [PATCH 3028/8313] add news item for git-annex 3.20120123

---
 doc/news/version_3.20120123.mdwn | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 doc/news/version_3.20120123.mdwn

diff --git a/doc/news/version_3.20120123.mdwn b/doc/news/version_3.20120123.mdwn
new file mode 100644
index 0000000000..4eb37ef2da
--- /dev/null
+++ b/doc/news/version_3.20120123.mdwn
@@ -0,0 +1,27 @@
+News for git-annex 3.20120123:
+
+   There was a bug in the handling of directory special remotes that
+   could cause partial file contents to be stored in them. If you use
+   a directory special remote, you should fsck it, to avoid potential
+   data loss.
+   Example: git annex fsck --from mydirectory
+
+git-annex 3.20120123 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * fsck --from: Fscking a remote is now supported. It's done by retrieving
+     the contents of the specified files from the remote, and checking them,
+     so can be an expensive operation. Still, if the remote is a special
+     remote, or a git repository that you cannot run fsck in locally, it's
+     nice to have the ability to fsck it.
+   * If you have any directory special remotes, now would be a good time to
+     fsck them, in case you were hit by the data loss bug fixed in the
+     previous release!
+   * fsck --from remote --fast: Avoids expensive file transfers, at the
+     expense of not checking file size and/or contents.
+   * Ssh connection caching is now enabled automatically by git-annex.
+     Only one ssh connection is made to each host per git-annex run, which
+     can speed some things up a lot, as well as avoiding repeated password
+     prompts. Concurrent git-annex processes also share ssh connections.
+     Cached ssh connections are shut down when git-annex exits.
+   * To disable the ssh caching (if for example you have your own broader
+     ssh caching configuration), set annex.sshcaching=false."""]]
\ No newline at end of file

From 5e172b43c4dc5d779e4f1af5c1376c9a8ebc60a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 Jan 2012 16:57:45 -0400
Subject: [PATCH 3029/8313] a few things available elsewhere...

---
 Utility/PartialPrelude.hs | 8 ++++++--
 Utility/Path.hs           | 5 ++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs
index 507fc6252b..dbdf4fa5c2 100644
--- a/Utility/PartialPrelude.hs
+++ b/Utility/PartialPrelude.hs
@@ -7,6 +7,8 @@
 
 module Utility.PartialPrelude where
 
+import qualified Data.Maybe
+
 {- read should be avoided, as it throws an error
  - Instead, use: readMaybe -}
 read :: Read a => String -> a
@@ -36,6 +38,9 @@ last = Prelude.last
  -
  - Ignores leading/trailing whitespace, and throws away any trailing
  - text after the part that can be read.
+ -
+ - readMaybe is available in Text.Read in new versions of GHC,
+ - but that one requires the entire string to be consumed.
  -}
 readMaybe :: Read a => String -> Maybe a
 readMaybe s = case reads s of
@@ -44,8 +49,7 @@ readMaybe s = case reads s of
 
 {- Like head but Nothing on empty list. -}
 headMaybe :: [a] -> Maybe a
-headMaybe [] = Nothing
-headMaybe v = Just $ Prelude.head v
+headMaybe = Data.Maybe.listToMaybe
 
 {- Like last but Nothing on empty list. -}
 lastMaybe :: [a] -> Maybe a
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 9f4fe29277..ed5e59cb5f 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -47,7 +47,10 @@ dirContains a b = a == b || a' == b' || (a'++"/") `isPrefixOf` b'
 		a' = norm a
 		b' = norm b
 
-{- Converts a filename into a normalized, absolute path. -}
+{- Converts a filename into a normalized, absolute path.
+ -
+ - Unlike Directory.canonicalizePath, this does not require the path
+ - already exists. -}
 absPath :: FilePath -> IO FilePath
 absPath file = do
 	cwd <- getCurrentDirectory

From ba6088b249902d456177af3c14f20f43b6def1fd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 Jan 2012 17:00:10 -0400
Subject: [PATCH 3030/8313] rename readMaybe to readish

a stricter (but also partial) readMaybe is getting added to base
---
 Annex/Ssh.hs              | 2 +-
 Config.hs                 | 4 ++--
 Git/Url.hs                | 2 +-
 GitAnnex.hs               | 2 +-
 Limit.hs                  | 2 +-
 Seek.hs                   | 2 +-
 Types/Key.hs              | 4 ++--
 Utility/Format.hs         | 2 +-
 Utility/PartialPrelude.hs | 6 +++---
 9 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index c05e236040..14ea74e53e 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -101,7 +101,7 @@ hostport2socket host (Just port) = host ++ "!" ++ show port
 socket2hostport :: FilePath -> (String, Maybe Integer)
 socket2hostport socket
 	| null p = (h, Nothing)
-	| otherwise = (h, readMaybe p)
+	| otherwise = (h, readish p)
 	where
 		(h, p) = separate (== '!') $ takeFileName socket
 
diff --git a/Config.hs b/Config.hs
index 83a84a1fe2..349ddf67f4 100644
--- a/Config.hs
+++ b/Config.hs
@@ -40,7 +40,7 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" +
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
 	cmd <- getConfig r "cost-command" ""
-	(fromMaybe def . readMaybe) <$>
+	(fromMaybe def . readish) <$>
 		if not $ null cmd
 			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
 			else getConfig r "cost" ""
@@ -78,7 +78,7 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
 		use Nothing = perhaps (return 1) =<< 
-			readMaybe <$> fromRepo (Git.Config.get config "1")
+			readish <$> fromRepo (Git.Config.get config "1")
 		perhaps fallback = maybe fallback (return . id)
 		config = "annex.numcopies"
 
diff --git a/Git/Url.hs b/Git/Url.hs
index 6a893d92fe..21b69dc7c0 100644
--- a/Git/Url.hs
+++ b/Git/Url.hs
@@ -45,7 +45,7 @@ port :: Repo -> Maybe Integer
 port r = 
 	case authpart uriPort r of
 		":" -> Nothing
-		(':':p) -> readMaybe p
+		(':':p) -> readish p
 		_ -> Nothing
 
 {- Hostname of an URL repo, including any username (ie, "user@host") -}
diff --git a/GitAnnex.hs b/GitAnnex.hs
index bc3541676b..4af10a9ce4 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -119,7 +119,7 @@ options = Option.common ++
 		"skip files not using a key-value backend"
 	] ++ Option.matcher
 	where
-		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readMaybe v }
+		setnumcopies v = Annex.changeState $ \s -> s {Annex.forcenumcopies = readish v }
 		setgitconfig :: String -> Annex ()
 		setgitconfig v = do
 			newg <- inRepo $ Git.Config.store v
diff --git a/Limit.hs b/Limit.hs
index 128ea0a27b..0db418e6c6 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -84,7 +84,7 @@ addIn name = addLimit $ check $ if name == "." then inAnnex else inremote
  - of copies. -}
 addCopies :: String -> Annex ()
 addCopies num =
-	case readMaybe num :: Maybe Int of
+	case readish num :: Maybe Int of
 		Nothing -> error "bad number for --copies"
 		Just n -> addLimit $ check n
 	where
diff --git a/Seek.hs b/Seek.hs
index 8e935c90cd..b4e1218e2b 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -36,7 +36,7 @@ withAttrFilesInGit attr a params = do
 withNumCopies :: (Maybe Int -> FilePath -> CommandStart) -> CommandSeek
 withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
 	where
-		go (file, v) = a (readMaybe v) file
+		go (file, v) = a (readish v) file
 
 withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
 withBackendFilesInGit a params =
diff --git a/Types/Key.hs b/Types/Key.hs
index 165f814d4b..f258f5c4ce 100644
--- a/Types/Key.hs
+++ b/Types/Key.hs
@@ -69,8 +69,8 @@ readKey s = if key == Just stubKey then Nothing else key
 		findfields _ v = v
 
 		addbackend k v = Just k { keyBackendName = v }
-		addfield 's' k v = Just k { keySize = readMaybe v }
-		addfield 'm' k v = Just k { keyMtime = readMaybe v }
+		addfield 's' k v = Just k { keySize = readish v }
+		addfield 'm' k v = Just k { keyMtime = readish v }
 		addfield _ _ _ = Nothing
 
 prop_idempotent_key_read_show :: Key -> Bool
diff --git a/Utility/Format.hs b/Utility/Format.hs
index d8b7e45493..79e94ae963 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -88,7 +88,7 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 			| c == '}' = foundvar f var (readjustify $ reverse p) cs
 			| otherwise = inpad (c:p) f var cs
 		inpad p f var [] = Const (novar $ p++";"++var) : f
-		readjustify = getjustify . fromMaybe 0 . readMaybe
+		readjustify = getjustify . fromMaybe 0 . readish
 		getjustify i
 			| i == 0 = UnJustified
 			| i < 0 = LeftJustified (-1 * i)
diff --git a/Utility/PartialPrelude.hs b/Utility/PartialPrelude.hs
index dbdf4fa5c2..6efa093fd3 100644
--- a/Utility/PartialPrelude.hs
+++ b/Utility/PartialPrelude.hs
@@ -10,7 +10,7 @@ module Utility.PartialPrelude where
 import qualified Data.Maybe
 
 {- read should be avoided, as it throws an error
- - Instead, use: readMaybe -}
+ - Instead, use: readish -}
 read :: Read a => String -> a
 read = Prelude.read
 
@@ -42,8 +42,8 @@ last = Prelude.last
  - readMaybe is available in Text.Read in new versions of GHC,
  - but that one requires the entire string to be consumed.
  -}
-readMaybe :: Read a => String -> Maybe a
-readMaybe s = case reads s of
+readish :: Read a => String -> Maybe a
+readish s = case reads s of
 	((x,_):_) -> Just x
 	_ -> Nothing
 

From ce5637498fd4158f98376009dee2d22bec2d1f68 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 24 Jan 2012 15:28:13 -0400
Subject: [PATCH 3031/8313] remove Utility.Conditional and use IfElse

This drops the >>! and >>? with the nice low fixity. IfElse does have
undocumented >>=>>! and >>=>>? operators, but I deem that too fishy.
Anyway, using whenM and unlessM is easier; I sometimes mixed the operators
up.
---
 Common.hs              |  2 +-
 Git/Command.hs         |  4 ++--
 Remote/Bup.hs          |  8 ++++----
 Remote/Directory.hs    |  4 ++--
 Remote/Rsync.hs        |  4 ++--
 Utility/Conditional.hs | 26 --------------------------
 Utility/CopyFile.hs    |  2 +-
 Utility/Directory.hs   |  2 +-
 debian/changelog       |  6 ++++++
 debian/control         |  1 +
 doc/install.mdwn       |  1 +
 git-annex.cabal        |  3 ++-
 12 files changed, 23 insertions(+), 40 deletions(-)
 delete mode 100644 Utility/Conditional.hs

diff --git a/Common.hs b/Common.hs
index 90895f08e8..385d1aba43 100644
--- a/Common.hs
+++ b/Common.hs
@@ -1,6 +1,7 @@
 module Common (module X) where
 
 import Control.Monad as X hiding (join)
+import Control.Monad.IfElse as X
 import Control.Applicative as X
 import Control.Monad.State as X (liftIO)
 import Control.Exception.Extensible as X (IOException)
@@ -20,7 +21,6 @@ import System.Posix.Process as X hiding (executeFile)
 import System.Exit as X
 
 import Utility.Misc as X
-import Utility.Conditional as X
 import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
diff --git a/Git/Command.hs b/Git/Command.hs
index 2350bb0ca3..61b7728dbe 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -30,8 +30,8 @@ runBool subcommand params repo = assertLocal repo $
 {- Runs git in the specified repo, throwing an error if it fails. -}
 run :: String -> [CommandParam] -> Repo -> IO ()
 run subcommand params repo = assertLocal repo $
-	runBool subcommand params repo
-		>>! error $ "git " ++ show params ++ " failed"
+	unlessM (runBool subcommand params repo) $
+		error $ "git " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns its output, lazily. 
  -
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 7329167dae..9b54d8c855 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -69,7 +69,7 @@ bupSetup u c = do
 	-- bup init will create the repository.
 	-- (If the repository already exists, bup init again appears safe.)
 	showAction "bup init"
-	bup "init" buprepo [] >>! error "bup init failed"
+	unlessM (bup "init" buprepo []) $ error "bup init failed"
 
 	storeBupUUID u buprepo
 
@@ -167,9 +167,9 @@ storeBupUUID u buprepo = do
 	if Git.repoIsUrl r
 		then do
 			showAction "storing uuid"
-			onBupRemote r boolSystem "git"
-				[Params $ "config annex.uuid " ++ v]
-					>>! error "ssh failed"
+			unlessM (onBupRemote r boolSystem "git"
+				[Params $ "config annex.uuid " ++ v]) $
+					error "ssh failed"
 		else liftIO $ do
 			r' <- Git.Config.read r
 			let olduuid = Git.Config.get "annex.uuid" "" r'
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 52f4263409..85f6446078 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -55,8 +55,8 @@ directorySetup u c = do
 	-- verify configuration is sane
 	let dir = fromMaybe (error "Specify directory=") $
 		M.lookup "directory" c
-	liftIO $ doesDirectoryExist dir
-		>>! error $ "Directory does not exist: " ++ dir
+	liftIO $ unlessM (doesDirectoryExist dir) $
+		error $ "Directory does not exist: " ++ dir
 	c' <- encryptionSetup c
 
 	-- The directory is stored in git config, not in this remote's
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 8de6ba6a74..c7efe42008 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -181,8 +181,8 @@ withRsyncScratchDir a = do
 	liftIO $ createDirectoryIfMissing True tmp
 	nuke tmp `after` a tmp
 	where
-		nuke d = liftIO $ 
-			doesDirectoryExist d >>? removeDirectoryRecursive d
+		nuke d = liftIO $ whenM (doesDirectoryExist d) $
+			removeDirectoryRecursive d
 
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
diff --git a/Utility/Conditional.hs b/Utility/Conditional.hs
deleted file mode 100644
index 85e39ec64c..0000000000
--- a/Utility/Conditional.hs
+++ /dev/null
@@ -1,26 +0,0 @@
-{- monadic conditional operators
- -
- - Copyright 2011 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-module Utility.Conditional where
-
-import Control.Monad (when, unless)
-
-whenM :: Monad m => m Bool -> m () -> m ()
-whenM c a = c >>= flip when a
-
-unlessM :: Monad m => m Bool -> m () -> m ()
-unlessM c a = c >>= flip unless a
-
-(>>?) :: Monad m => m Bool -> m () -> m ()
-(>>?) = whenM
-
-(>>!) :: Monad m => m Bool -> m () -> m ()
-(>>!) = unlessM
-
--- low fixity allows eg, foo bar >>! error $ "failed " ++ meep
-infixr 0 >>?
-infixr 0 >>!
diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 5d6855bf01..c42506485b 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -8,8 +8,8 @@
 module Utility.CopyFile (copyFileExternal) where
 
 import System.Directory (doesFileExist, removeFile)
+import Control.Monad.IfElse
 
-import Utility.Conditional
 import Utility.SafeCommand
 import qualified Build.SysConfig as SysConfig
 
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 249ed69356..b5fedb9c7d 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -12,9 +12,9 @@ import System.Posix.Files
 import System.Directory
 import Control.Exception (throw)
 import Control.Monad
+import Control.Monad.IfElse
 
 import Utility.SafeCommand
-import Utility.Conditional
 import Utility.TempFile
 
 {- Moves one filename to another.
diff --git a/debian/changelog b/debian/changelog
index 2f573e6d3f..e1c861d192 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120124) UNRELEASED; urgency=low
+
+  * Use the haskell IfElse library.
+
+ -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
+
 git-annex (3.20120123) unstable; urgency=low
 
   * fsck --from: Fscking a remote is now supported. It's done by retrieving
diff --git a/debian/control b/debian/control
index 3f171a11ce..c3ddad9322 100644
--- a/debian/control
+++ b/debian/control
@@ -17,6 +17,7 @@ Build-Depends:
 	libghc-monad-control-dev (>= 0.3),
 	libghc-lifted-base-dev,
 	libghc-json-dev,
+	libghc-ifelse-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 7da46b351a..b48914197f 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -34,6 +34,7 @@ To build and use git-annex, you will need:
   * [HTTP](http://hackage.haskell.org/package/HTTP)
   * [hS3](http://hackage.haskell.org/package/hS3)
   * [json](http://hackage.haskell.org/package/json)
+  * [IfElse](http://hackage.haskell.org/package/IfElse)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 43901b6939..3f152ea4b8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,8 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
-   base < 5, monad-control, transformers-base, lifted-base, QuickCheck >= 2.1
+   base < 5, monad-control, transformers-base, lifted-base, IfElse,
+   QuickCheck >= 2.1
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs

From bc130e355e41a85cb1115e16df89d93af88429d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 24 Jan 2012 16:22:24 -0400
Subject: [PATCH 3032/8313] close

---
 doc/todo/fsck_special_remotes.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/fsck_special_remotes.mdwn b/doc/todo/fsck_special_remotes.mdwn
index c81c56c856..7196baafe6 100644
--- a/doc/todo/fsck_special_remotes.mdwn
+++ b/doc/todo/fsck_special_remotes.mdwn
@@ -9,3 +9,5 @@ copying the file, just dropping a symlink, etc.
 
 The WORM backend doesn't care about file content, so it would be nice to
 avoid transferring the content at all, and only send the size.
+
+> [[done]] --[[Joey]] 

From 9b51ec9dbf48a54184aedf3f4cd5387347d67336 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jan 2012 17:39:38 -0400
Subject: [PATCH 3033/8313] add

---
 doc/forum/cloud_services_to_support.mdwn | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/cloud_services_to_support.mdwn

diff --git a/doc/forum/cloud_services_to_support.mdwn b/doc/forum/cloud_services_to_support.mdwn
new file mode 100644
index 0000000000..e268bc1d86
--- /dev/null
+++ b/doc/forum/cloud_services_to_support.mdwn
@@ -0,0 +1,16 @@
+git-annex can already be used to store data in several cloud services:
+Amazon S3, rsync.net, Tahoe-LAFFS, The Internet Archive.
+
+I would like to support as many other cloud services as possible/reasonable.
+
+* [[swift|todo/wishlist:_swift_backend]]
+* Dropbox (I had been reluctant to go there due to it using a non-free client,
+  which I have no interest in installing, but there is actually an API,
+  and already a
+  [haskell module to use it](http://hackage.haskell.org/package/dropbox-sdk).
+  Would need to register for an API key.
+  .
+  Annoyingly, Dropbox reviews each app before granting it production status.
+  Whoops my interest level dropped by 99%.)
+
+Post others in the comments. --[[Joey]]

From 3ca7cf5db1c90b6fce8d918647c5a689ea376ecd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jan 2012 20:42:01 -0400
Subject: [PATCH 3034/8313] export fromPath

Not used in git-annex, but I am using it in git-backup
---
 Git/Construct.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Git/Construct.hs b/Git/Construct.hs
index bfb16164ff..ef6094a21d 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -9,6 +9,7 @@ module Git.Construct (
 	fromCurrent,
 	fromCwd,
 	fromAbsPath,
+	fromPath,
 	fromUrl,
 	fromUnknown,
 	localToUrl,

From 97209ac08dcfc005c3da2fa889275e618d892d31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jan 2012 20:43:01 -0400
Subject: [PATCH 3035/8313] fix error message

---
 Git/Command.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Git/Command.hs b/Git/Command.hs
index 61b7728dbe..ec701c1f09 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -31,7 +31,7 @@ runBool subcommand params repo = assertLocal repo $
 run :: String -> [CommandParam] -> Repo -> IO ()
 run subcommand params repo = assertLocal repo $
 	unlessM (runBool subcommand params repo) $
-		error $ "git " ++ show params ++ " failed"
+		error $ "git " ++ subcommand ++ " " ++ show params ++ " failed"
 
 {- Runs a git subcommand and returns its output, lazily. 
  -

From 6da40100c99a097401fb130b63edc05b18f61249 Mon Sep 17 00:00:00 2001
From: Lauri Alanko 
Date: Fri, 20 Jan 2012 06:26:04 +0200
Subject: [PATCH 3036/8313] Avoid creating ~/.bup when initializing a bup
 remote

---
 Remote/Bup.hs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 9b54d8c855..583358f242 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -11,6 +11,8 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
 import qualified Data.Map as M
 import System.Process
+import System.Posix.Env (getEnvironment)
+import System.Path (brackettmpdir)
 
 import Common.Annex
 import Types.Remote
@@ -83,10 +85,21 @@ bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam]
 bupParams command buprepo params = 
 	Param command : [Param "-r", Param buprepo] ++ params
 
+isLocal :: BupRepo -> Bool
+isLocal buprepo = not (elem ':' buprepo)
+
 bup :: String -> BupRepo -> [CommandParam] -> Annex Bool
 bup command buprepo params = do
 	showOutput -- make way for bup output
-	liftIO $ boolSystem "bup" $ bupParams command buprepo params
+	liftIO action
+	where
+		action | isLocal buprepo = runBup lparams buprepo
+		       | otherwise = brackettmpdir "bupXXXXXX" $ runBup rparams
+		lparams = Param command : params
+		rparams = bupParams command buprepo params
+		runBup params bupdir = do
+			env <- getEnvironment
+			boolSystemEnv "bup" params (Just (("BUP_DIR", bupdir) : env))
 
 pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool
 pipeBup params inh outh = do

From 9b91db825484f8e16ce5d1bb3daee6e6a8151206 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
 
Date: Thu, 26 Jan 2012 22:13:19 +0000
Subject: [PATCH 3037/8313] Added a comment

---
 ...omment_5_519cda534c7aea7f5ad5acd3f76e21fa._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_5_519cda534c7aea7f5ad5acd3f76e21fa._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_5_519cda534c7aea7f5ad5acd3f76e21fa._comment b/doc/bugs/problems_with_utf8_names/comment_5_519cda534c7aea7f5ad5acd3f76e21fa._comment
new file mode 100644
index 0000000000..96b0ffed04
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_5_519cda534c7aea7f5ad5acd3f76e21fa._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
+ nickname="Lauri"
+ subject="comment 5"
+ date="2012-01-26T22:13:18Z"
+ content="""
+I also encountered Adam's bug. The problem seems to be that communication with the git process is done with `Char8`-bytestrings. So, when `L.unpack` is called, all filenames that git outputs (with `ls-files` or `ls-tree`) are interpreted to be in latin-1, which wreaks havoc if they are really in UTF-8.
+
+I suspect that it would be enough to just switch to standard `String`s (or `Data.Text.Text`) instead of bytestrings for textual data, and to `Word8`-bytestrings for pure binary data. GHC should nowadays handle locale-dependent encoding of `String`s transparently.
+
+"""]]

From ba6f01b137498b7df2191f050158f58052311624 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
 
Date: Fri, 27 Jan 2012 08:12:48 +0000
Subject: [PATCH 3038/8313]

---
 doc/bugs/copy_doesn__39__t_scale.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/bugs/copy_doesn__39__t_scale.mdwn

diff --git a/doc/bugs/copy_doesn__39__t_scale.mdwn b/doc/bugs/copy_doesn__39__t_scale.mdwn
new file mode 100644
index 0000000000..1a83ae548b
--- /dev/null
+++ b/doc/bugs/copy_doesn__39__t_scale.mdwn
@@ -0,0 +1,4 @@
+It seems that git-annex copies every individual file in a separate transaction. This is quite costly for mass transfers: each file involves a separate rsync invocation and the creation of a new commit. Even with a meager thousand files or so in the annex, I have to wait for fifteen minutes to copy the contents to another disk, simply because every individual file involves some disk thrashing. Also, it seems suspicious that the git-annex branch would get a thousands commits of history from the simple procedure of copying everything to a new repository. Surely it would be better to first copy everything and then create only a single commit that registers the changes to the files' availability?
+
+(I'm also not quite clear on why rsync is being used when both repositories are local. It seems to be just overhead.)
+

From 0bb3a31a6ec40b54d6fa5aaafecbccaa719a80db Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 Jan 2012 16:50:27 -0400
Subject: [PATCH 3039/8313] old version?

---
 doc/bugs/copy_doesn__39__t_scale.mdwn | 29 +++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/copy_doesn__39__t_scale.mdwn b/doc/bugs/copy_doesn__39__t_scale.mdwn
index 1a83ae548b..79479a3357 100644
--- a/doc/bugs/copy_doesn__39__t_scale.mdwn
+++ b/doc/bugs/copy_doesn__39__t_scale.mdwn
@@ -1,4 +1,29 @@
-It seems that git-annex copies every individual file in a separate transaction. This is quite costly for mass transfers: each file involves a separate rsync invocation and the creation of a new commit. Even with a meager thousand files or so in the annex, I have to wait for fifteen minutes to copy the contents to another disk, simply because every individual file involves some disk thrashing. Also, it seems suspicious that the git-annex branch would get a thousands commits of history from the simple procedure of copying everything to a new repository. Surely it would be better to first copy everything and then create only a single commit that registers the changes to the files' availability?
+It seems that git-annex copies every individual file in a separate
+transaction. This is quite costly for mass transfers: each file involves a
+separate rsync invocation and the creation of a new commit. Even with a
+meager thousand files or so in the annex, I have to wait for fifteen
+minutes to copy the contents to another disk, simply because every
+individual file involves some disk thrashing. Also, it seems suspicious
+that the git-annex branch would get a thousands commits of history from the
+simple procedure of copying everything to a new repository. Surely it would
+be better to first copy everything and then create only a single commit
+that registers the changes to the files' availability?
 
-(I'm also not quite clear on why rsync is being used when both repositories are local. It seems to be just overhead.)
+> git-annex is very careful to commit as infrequently as possible,
+> and the current version makes *1* commit after all the copies are
+> complete, even if it transferred a billion files. The only overhead
+> incurred for each file is writing a journal file.
+> You must have an old version.
+> --[[Joey]]
 
+(I'm also not quite clear on why rsync is being used when both repositories
+are local. It seems to be just overhead.)
+
+> Even when copying to another disk it's often on 
+> some slow bus, and the file is by definition large. So it's
+> nice to support resumes of interrupted transfers of files.
+> Also because rsync has a handy progress display that is hard to get with cp.
+> 
+> (However, if the copy is to another directory in the same disk, it does
+> use cp, and even supports really fast copies on COW filesystems.)
+> --[[Joey]]

From 0f8a8ef4a54a5daffcdb42aa60e09248e8e86235 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 27 Jan 2012 21:00:06 +0000
Subject: [PATCH 3040/8313] Added a comment

---
 .../comment_6_52e0bfff2b177b6f92e226b25d2f3ff1._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_6_52e0bfff2b177b6f92e226b25d2f3ff1._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_6_52e0bfff2b177b6f92e226b25d2f3ff1._comment b/doc/bugs/problems_with_utf8_names/comment_6_52e0bfff2b177b6f92e226b25d2f3ff1._comment
new file mode 100644
index 0000000000..093616d476
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_6_52e0bfff2b177b6f92e226b25d2f3ff1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 6"
+ date="2012-01-27T21:00:06Z"
+ content="""
+Lauri, what version of GHC do you have that behaves this way? 7.0.4 does not. 
+"""]]

From 13e78fbf88e892959dd65a59a77ac2b89bdc6396 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
 
Date: Sat, 28 Jan 2012 00:17:38 +0000
Subject: [PATCH 3041/8313] Added a comment

---
 ...comment_1_7c12499c9ac28a9883c029f8c659eb57._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment

diff --git a/doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment b/doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment
new file mode 100644
index 0000000000..749b3ba108
--- /dev/null
+++ b/doc/bugs/copy_doesn__39__t_scale/comment_1_7c12499c9ac28a9883c029f8c659eb57._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
+ nickname="Lauri"
+ subject="comment 1"
+ date="2012-01-28T00:17:37Z"
+ content="""
+To me it very much seems that a commit per file is indeed created at the remote end, although not at the local end. See the following transcript: .
+
+
+"""]]

From 141718da73173855ce08662b1131344e6c10ef46 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
 
Date: Sat, 28 Jan 2012 00:21:42 +0000
Subject: [PATCH 3042/8313] Added a comment

---
 .../comment_7_0cc588f787d6eecfa19a8f6cee4b07b5._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_7_0cc588f787d6eecfa19a8f6cee4b07b5._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_7_0cc588f787d6eecfa19a8f6cee4b07b5._comment b/doc/bugs/problems_with_utf8_names/comment_7_0cc588f787d6eecfa19a8f6cee4b07b5._comment
new file mode 100644
index 0000000000..5a929940d4
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_7_0cc588f787d6eecfa19a8f6cee4b07b5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
+ nickname="Lauri"
+ subject="comment 7"
+ date="2012-01-28T00:21:40Z"
+ content="""
+7.2. nomeata already explained the issue. I got utf-8 filenames to work on a utf-8 locale by switching from Char8-bytestrings to UTF8-bytestrings, and adding `hSetEncoding h localeEncoding` to suitable places. Making things work properly with an arbitrary locale encoding would be more complicated.
+"""]]

From 303666965ab5bc891c8ed44969553afb642c3f9c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 Jan 2012 15:23:28 -0400
Subject: [PATCH 3043/8313] Revert "Avoid creating ~/.bup when initializing a
 bup remote"

This reverts commit 6da40100c99a097401fb130b63edc05b18f61249.

On closer examinaton, this change is wrong. The bup special remote
can be configured with "buprepo=", which makes it use the default
~/.bup repo. This change makes it use a different temp dir each time,
which I'm sure would not be appreciated by anyone with that
configuration.

Bup insisting in creating ~/.bup even when using a different repo
does seem like a bug in *something*, but I'm leaning toward the bug
being in bup itself.
---
 Remote/Bup.hs | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 583358f242..9b54d8c855 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -11,8 +11,6 @@ import qualified Data.ByteString.Lazy.Char8 as L
 import System.IO.Error
 import qualified Data.Map as M
 import System.Process
-import System.Posix.Env (getEnvironment)
-import System.Path (brackettmpdir)
 
 import Common.Annex
 import Types.Remote
@@ -85,21 +83,10 @@ bupParams :: String -> BupRepo -> [CommandParam] -> [CommandParam]
 bupParams command buprepo params = 
 	Param command : [Param "-r", Param buprepo] ++ params
 
-isLocal :: BupRepo -> Bool
-isLocal buprepo = not (elem ':' buprepo)
-
 bup :: String -> BupRepo -> [CommandParam] -> Annex Bool
 bup command buprepo params = do
 	showOutput -- make way for bup output
-	liftIO action
-	where
-		action | isLocal buprepo = runBup lparams buprepo
-		       | otherwise = brackettmpdir "bupXXXXXX" $ runBup rparams
-		lparams = Param command : params
-		rparams = bupParams command buprepo params
-		runBup params bupdir = do
-			env <- getEnvironment
-			boolSystemEnv "bup" params (Just (("BUP_DIR", bupdir) : env))
+	liftIO $ boolSystem "bup" $ bupParams command buprepo params
 
 pipeBup :: [CommandParam] -> Maybe Handle -> Maybe Handle -> IO Bool
 pipeBup params inh outh = do

From 2a6be431d4cc4d8056f45f32e91b1b72a65fb107 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 28 Jan 2012 19:32:36 +0000
Subject: [PATCH 3044/8313] Added a comment

---
 ...nt_2_f85d8023cdbc203bb439644cf7245d4e._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment

diff --git a/doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment b/doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment
new file mode 100644
index 0000000000..9a2bd92fa3
--- /dev/null
+++ b/doc/bugs/copy_doesn__39__t_scale/comment_2_f85d8023cdbc203bb439644cf7245d4e._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-01-28T19:32:36Z"
+ content="""
+Ah, I see, I was not thinking about the location log update that's done on the remote side.
+
+For transfers over ssh, that's a separate git-annex-shell invoked per change. For local-local transfers, it's all done in a single process but it spins up a state to handle the remote and then immediately shuts it down, also generating a commit.
+
+In either case, I think there is a nice fix. Since git-annex *does* have a journal nowadays, and goes to all the bother to
+support recovery if a process was interrupted and journalled changes that did not get committed, there's really no reason in either of these cases for the remote end to do anything more than journal the change. The next time git-annex is actually run on the remote, and needs to look up location information, it will merge the journalled changes into the branch, in a single commit.
+
+My only real concern is that some remotes might *never* have git-annex run in them directly, and would just continue to accumulate journal files forever. Although due to the way the journal is structured, it can have, at a maximum, the number of files in the git-annex branch. However, the number of files in it is expected to be relatively smal and it might get a trifle innefficient, as it lacks directory hashing. These performance problems could certainly be dealt with if they do turn out to be a problem.
+"""]]

From a75eff4a8eb937b07c15f817d329647e4a4391fe Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 28 Jan 2012 19:40:34 +0000
Subject: [PATCH 3045/8313] Added a comment

---
 ...comment_8_ff5c6da9eadfee20c18c86b648a62c47._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/problems_with_utf8_names/comment_8_ff5c6da9eadfee20c18c86b648a62c47._comment

diff --git a/doc/bugs/problems_with_utf8_names/comment_8_ff5c6da9eadfee20c18c86b648a62c47._comment b/doc/bugs/problems_with_utf8_names/comment_8_ff5c6da9eadfee20c18c86b648a62c47._comment
new file mode 100644
index 0000000000..dcfd59bce9
--- /dev/null
+++ b/doc/bugs/problems_with_utf8_names/comment_8_ff5c6da9eadfee20c18c86b648a62c47._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 8"
+ date="2012-01-28T19:40:34Z"
+ content="""
+Lauri a scratch patch would be very helpful. Encoding stuff makes my head explode.
+
+However, I am very worried by haskell's changes WRT unicode and filenames. Based on user input, git-annex users like to use it on diverse sets of files, with diverse and ill-defined encodings. Faffing about with converting between encodings seems likely to speactacularly fail.
+"""]]

From b81d662cbf0036d0e2b632ed95a877feab2a4860 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 Jan 2012 15:41:52 -0400
Subject: [PATCH 3046/8313] Avoid repeated location log commits when a remote
 is receiving files.

Done by adding a oneshot mode, in which location log changes are written to
the journal, but not committed. Taking advantage of git-annex's existing
ability to recover in this situation.

This is used by git-annex-shell and other places where changes are made to
a remote's location log.
---
 Annex/Content.hs   | 13 +++++++++----
 CmdLine.hs         | 12 ++++++------
 Command/RecvKey.hs |  4 ++--
 Command/Uninit.hs  |  2 +-
 GitAnnex.hs        |  2 +-
 Remote/Git.hs      |  2 +-
 Upgrade/V2.hs      |  2 +-
 debian/changelog   |  6 ++++++
 git-annex-shell.hs |  2 +-
 9 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index c21ac405ea..dcfd43866b 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -291,11 +291,16 @@ getKeysPresent' dir = do
 			let files = concat contents
 			return $ mapMaybe (fileKey . takeFileName) files
 
-{- Things to do to record changes to content. -}
-saveState :: Annex ()
-saveState = do
+{- Things to do to record changes to content when shutting down.
+ -
+ - It's acceptable to avoid committing changes to the branch,
+ - especially if performing a short-lived action.
+ -}
+saveState :: Bool -> Annex ()
+saveState oneshot = do
 	Annex.Queue.flush False
-	Annex.Branch.commit "update"
+	unless oneshot $
+		Annex.Branch.commit "update"
 
 {- Downloads content from any of a list of urls. -}
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
diff --git a/CmdLine.hs b/CmdLine.hs
index 29b95d01bd..61e6c26bb8 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -29,8 +29,8 @@ type Params = [String]
 type Flags = [Annex ()]
 
 {- Runs the passed command line. -}
-dispatch :: Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
-dispatch args cmds commonoptions header getgitrepo = do
+dispatch :: Bool -> Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
+dispatch oneshot args cmds commonoptions header getgitrepo = do
 	setupConsole
 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
 	case r of
@@ -40,7 +40,7 @@ dispatch args cmds commonoptions header getgitrepo = do
 			(actions, state') <- Annex.run state $ do
 				sequence_ flags
 				prepCommand cmd params
-			tryRun state' cmd $ [startup] ++ actions ++ [shutdown]
+			tryRun state' cmd $ [startup] ++ actions ++ [shutdown oneshot]
 	where
 		(flags, cmd, params) = parseCmd args cmds commonoptions header
 
@@ -89,9 +89,9 @@ startup :: Annex Bool
 startup = return True
 
 {- Cleanup actions. -}
-shutdown :: Annex Bool
-shutdown = do
-	saveState
+shutdown :: Bool -> Annex Bool
+shutdown oneshot = do
+	saveState oneshot
 	liftIO Git.Command.reap -- zombies from long-running git processes
 	sshCleanup -- ssh connection caching
 	return True
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 5243fa9d4b..a27a5efdf6 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -28,7 +28,7 @@ start key = do
 	if ok
 		then do
 			-- forcibly quit after receiving one key,
-			-- and shutdown cleanly so queued git commands run
-			_ <- shutdown
+			-- and shutdown cleanly
+			_ <- shutdown True
 			liftIO exitSuccess
 		else liftIO exitFailure
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index cef89a5cf3..ec6d0abf39 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -57,7 +57,7 @@ cleanup = do
 	mapM_ removeAnnex =<< getKeysPresent
 	liftIO $ removeDirectoryRecursive annexdir
 	-- avoid normal shutdown
-	saveState
+	saveState False
 	inRepo $ Git.Command.run "branch"
 		[Param "-D", Param $ show Annex.Branch.name]
 	liftIO exitSuccess
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 4af10a9ce4..1ca89315a3 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -129,4 +129,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header Git.Construct.fromCurrent
+run args = dispatch False args cmds options header Git.Construct.fromCurrent
diff --git a/Remote/Git.hs b/Remote/Git.hs
index efe1829610..501a617c03 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -230,7 +230,7 @@ copyToRemote r key
 		-- run copy from perspective of remote
 		liftIO $ onLocal r $ do
 			ensureInitialized
-			Annex.Content.saveState `after`
+			Annex.Content.saveState True `after`
 				Annex.Content.getViaTmp key
 					(rsyncOrCopyFile params keysrc)
 	| Git.repoIsSsh r = do
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index ffc2f60022..c57b0bf685 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -50,7 +50,7 @@ upgrade = do
 		mapM_ (\(k, f) -> inject f $ logFile k) =<< locationLogs
 		mapM_ (\f -> inject f f) =<< logFiles old
 
-	saveState
+	saveState False
 	showProgress
 
 	when e $ do
diff --git a/debian/changelog b/debian/changelog
index e1c861d192..96234d9272 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,12 @@
 git-annex (3.20120124) UNRELEASED; urgency=low
 
   * Use the haskell IfElse library.
+  * Avoid repeated location log commits when a remote is receiving files.
+    Done by adding a oneshot mode, in which location log changes are
+    written to the journal, but not committed. Taking advantage of
+    git-annex's existing ability to recover in this situation. This is
+    used by git-annex-shell and other places where changes are made to
+    a remote's location log.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 4fdeae1a87..e747a447b4 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -82,7 +82,7 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
-	dispatch (cmd : filterparams params) cmds options header $
+	dispatch True (cmd : filterparams params) cmds options header $
 		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
 
 external :: [String] -> IO ()

From 775958b4dc9edba64ab78ed90ef3c2d11f252676 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 Jan 2012 15:54:42 -0400
Subject: [PATCH 3047/8313] faster local-local dropping

Dropping a key from a local remote ran git-annex-shell unnecessarily.

Now git-annex-shell is never used when acting on a local remote.
---
 Remote/Git.hs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 501a617c03..829ad1ccba 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -20,6 +20,7 @@ import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
 import qualified Annex
+import Logs.Presence
 import Annex.UUID
 import qualified Annex.Content
 import qualified Annex.BranchState
@@ -192,6 +193,14 @@ keyUrls r key = map tourl (annexLocations key)
 
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key
+	| not $ Git.repoIsUrl r = liftIO $ onLocal r $ do
+		ensureInitialized
+		whenM (Annex.Content.inAnnex key) $ do
+			Annex.Content.lockContent key $
+				Annex.Content.removeAnnex key
+			Annex.Content.logStatus key InfoMissing
+			Annex.Content.saveState True
+		return True
 	| Git.repoIsHttp r = error "dropping from http repo not supported"
 	| otherwise = onRemote r (boolSystem, False) "dropkey"
 		[ Params "--quiet --force"

From 6e89064d27d8f9a3170f8677b686bbb0b3d8ca50 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 Jan 2012 16:01:53 -0400
Subject: [PATCH 3048/8313] fixed

---
 doc/bugs/copy_doesn__39__t_scale.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/copy_doesn__39__t_scale.mdwn b/doc/bugs/copy_doesn__39__t_scale.mdwn
index 79479a3357..adbd0660af 100644
--- a/doc/bugs/copy_doesn__39__t_scale.mdwn
+++ b/doc/bugs/copy_doesn__39__t_scale.mdwn
@@ -27,3 +27,9 @@ are local. It seems to be just overhead.)
 > (However, if the copy is to another directory in the same disk, it does
 > use cp, and even supports really fast copies on COW filesystems.)
 > --[[Joey]]
+
+---
+
+Oneshot mode is now implemented, making git-annex-shell and other
+short lifetime processes not bother with committing changes.
+[[done]] --[[Joey]] 

From 0609e102396083afa6380f8b67a69fa849235d16 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 28 Jan 2012 18:09:28 -0400
Subject: [PATCH 3049/8313] reopen

People seem to want to post comments here with vague details about a new
bug, rather than opening a new bug report.
---
 doc/bugs/problems_with_utf8_names.mdwn | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index d6dc6ca3c3..b734ddecf7 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -1,3 +1,11 @@
+This bug is reopened to track some new UTF-8 filename issues caused by GHC
+7.4. Older versions of GHC, like the 7.0.4 in debian unstable, are not
+affected. See the comments for details about the new bug. --[[Joey]] 
+
+----
+
+Old, now fixed bug report follows:
+
 There are problems with displaying filenames in UTF8 encoding, as shown here:
 
     $ echo $LANG
@@ -45,7 +53,7 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu
 >    outputting a filename (assuming the filename is encoded using the
 >    user's configured encoding), and allow haskell's output encoding to then
 >    encode it according to the user's locale configuration.
->    > This is now [[implemented|done]]. I'm not very happy that I have to watch
+>    > This is now implemented. I'm not very happy that I have to watch
 >    > out for any place that a filename is output and call `filePathToString`
 >    > on it, but there are really not too many such places in git-annex.
 >    >

From b9a317cdbd5be8d2df151e0c9743ff184d3ce414 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
 
Date: Sun, 29 Jan 2012 01:51:37 +0000
Subject: [PATCH 3050/8313] Added a comment

---
 ...omment_3_4592765c3d77bb5664b8d16867e9d79c._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment

diff --git a/doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment b/doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment
new file mode 100644
index 0000000000..aa9bf1e45a
--- /dev/null
+++ b/doc/bugs/copy_doesn__39__t_scale/comment_3_4592765c3d77bb5664b8d16867e9d79c._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk6QAwUsFHpr3Km1yQbg8hf3S7RDYf7hX4"
+ nickname="Lauri"
+ subject="comment 3"
+ date="2012-01-29T01:51:35Z"
+ content="""
+That sounds just fine, but indeed my use case was a bare backup/transfer repository that is meant to always be only at the remote end of git-annex operations. So why not as well do a single commit after everything has been copied and journaled? That's what's done at the other end too, after all. Or, if commits are to be minimized, just stage the journal into the index before finishing, but don't commit it yet?
+
+(I would actually prefer this mode of usage for other git-annex operations, too. In git you can add stuff little by little and commit them all in one go. In git-annex the add immediately creates a commit, which is unexpected and a bit annoying.)
+
+"""]]

From a964012fc36d22e4554dd12e3594579fb3190501 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 29 Jan 2012 22:55:06 -0400
Subject: [PATCH 3051/8313] switch to the strict state monad

I had not realized what a memory leak the lazy state monad could be,
although I have not seen much evidence of actual leaking in git-annex.
However, if running git-annex on a great many files, this could matter.

The additional Utility.State.changeState adds even more strictness,
avoiding a problem I saw in github-backup where repeatedly modifying
state built up a huge pile of thunks.
---
 Annex.hs          | 15 ++-------------
 Command/Status.hs |  2 +-
 Common.hs         |  2 +-
 Git/CatFile.hs    |  1 -
 Utility/State.hs  | 26 ++++++++++++++++++++++++++
 5 files changed, 30 insertions(+), 16 deletions(-)
 create mode 100644 Utility/State.hs

diff --git a/Annex.hs b/Annex.hs
index 3b79ea2700..de36c816d8 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -26,7 +26,7 @@ module Annex (
 	fromRepo,
 ) where
 
-import Control.Monad.State
+import Control.Monad.State.Strict
 import Control.Monad.Trans.Control (StM, MonadBaseControl, liftBaseWith, restoreM)
 import Control.Monad.Base (liftBase, MonadBase)
 import System.Posix.Types (Fd)
@@ -41,6 +41,7 @@ import qualified Types.Remote
 import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
+import Utility.State
 import qualified Utility.Matcher
 import qualified Data.Map as M
 
@@ -125,18 +126,6 @@ run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
 eval s a = evalStateT (runAnnex a) s
 
-{- Gets a value from the internal state, selected by the passed value
- - constructor. -}
-getState :: (AnnexState -> a) -> Annex a
-getState = gets
-
-{- Applies a state mutation function to change the internal state. 
- -
- - Example: changeState $ \s -> s { output = QuietOutput }
- -}
-changeState :: (AnnexState -> AnnexState) -> Annex ()
-changeState = modify
-
 {- Sets a flag to True -}
 setFlag :: String -> Annex ()
 setFlag flag = changeState $ \s ->
diff --git a/Command/Status.hs b/Command/Status.hs
index d2d8d4c077..a1d4eea087 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -7,7 +7,7 @@
 
 module Command.Status where
 
-import Control.Monad.State
+import Control.Monad.State.Strict
 import qualified Data.Map as M
 import qualified Data.Set as S
 import Data.Set (Set)
diff --git a/Common.hs b/Common.hs
index 385d1aba43..fb998214bc 100644
--- a/Common.hs
+++ b/Common.hs
@@ -3,7 +3,7 @@ module Common (module X) where
 import Control.Monad as X hiding (join)
 import Control.Monad.IfElse as X
 import Control.Applicative as X
-import Control.Monad.State as X (liftIO)
+import Control.Monad.State.Strict as X (liftIO)
 import Control.Exception.Extensible as X (IOException)
 
 import Data.Maybe as X
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 16f0b11b95..2a2eb5e6f7 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -13,7 +13,6 @@ module Git.CatFile (
 	catObject
 ) where
 
-import Control.Monad.State
 import System.Cmd.Utils
 import System.IO
 import qualified Data.ByteString.Char8 as S
diff --git a/Utility/State.hs b/Utility/State.hs
new file mode 100644
index 0000000000..c27f3c2610
--- /dev/null
+++ b/Utility/State.hs
@@ -0,0 +1,26 @@
+{- state monad support
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.State where
+
+import Control.Monad.State.Strict
+
+{- Modifies Control.Monad.State's state, forcing a strict update.
+ - This avoids building thunks in the state and leaking.
+ - Why it's not the default, I don't know.
+ -
+ - Example: changeState $ \s -> s { foo = bar }
+ -}
+changeState :: MonadState s m => (s -> s) -> m ()
+changeState f = do
+	x <- get
+	put $! f x
+
+{- Gets a value from the internal state, selected by the passed value
+ - constructor. -}
+getState :: MonadState s m => (s -> a) -> m a
+getState = gets

From ed64bd8a4b542bcc37950c7c9a4c87ea2a96f4f5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 30 Jan 2012 13:20:36 -0400
Subject: [PATCH 3052/8313] remove; unused

---
 debian/manpages | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 debian/manpages

diff --git a/debian/manpages b/debian/manpages
deleted file mode 100644
index ca34203aa0..0000000000
--- a/debian/manpages
+++ /dev/null
@@ -1 +0,0 @@
-git-annex.1

From 6c64a214fa569dcf1fa8cc4c79efd90d01ff5705 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 31 Jan 2012 16:34:04 -0400
Subject: [PATCH 3053/8313] add a test case for adding a file with a unicode
 filename

This fails with ghc 4.7.
---
 test.hs | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 3aef83e879..7b25917a11 100644
--- a/test.hs
+++ b/test.hs
@@ -131,7 +131,7 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 		reponame = "test repo"
 
 test_add :: Test
-test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
+test_add = "git-annex add" ~: TestList [basic, sha1dup, sha1unicode, subdirs]
 	where
 		-- this test case runs in the main repo, to set up a basic
 		-- annexed file that later tests will use
@@ -158,6 +158,10 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 			git_annex "add" [sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
 			annexed_present sha1annexedfile
+		sha1unicode = TestCase $ intmpclonerepo $ do
+			writeFile sha1annexedfileunicode $ content sha1annexedfileunicode
+			git_annex "add" [sha1annexedfileunicode, "--backend=SHA1"] @? "add of unicode filename failed"
+			annexed_present sha1annexedfileunicode
 		subdirs = TestCase $ intmpclonerepo $ do
 			createDirectory "dir"
 			writeFile "dir/foo" $ content annexedfile
@@ -919,6 +923,9 @@ sha1annexedfile = "sha1foo"
 sha1annexedfiledup :: String
 sha1annexedfiledup = "sha1foodup"
 
+sha1annexedfileunicode :: String
+sha1annexedfileunicode = "foo¡"
+
 ingitfile :: String
 ingitfile = "bar"
 
@@ -928,6 +935,7 @@ content f
 	| f == ingitfile = "normal file content"
 	| f == sha1annexedfile ="sha1 annexed file content"
 	| f == sha1annexedfiledup = content sha1annexedfile
+	| f == sha1annexedfileunicode ="sha1 annexed file content ¡ünicodé!"
 	| f == wormannexedfile = "worm annexed file content"
 	| otherwise = "unknown file " ++ f
 

From 3d49258e5bed4d9a6ec9e24ddb776f277542664b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Feb 2012 16:05:02 -0400
Subject: [PATCH 3054/8313] attempt at a quick, utf-8 only fix to the ghc 7.4
 problem

If you have only utf-8 filenames, and need to build git-annex with ghc 7.4,
this will work. But, it will crash on non-utf-8 filenames.
---
 Command/Uninit.hs |  4 ++--
 Command/Unused.hs |  5 +++--
 Git/Branch.hs     |  2 +-
 Git/Command.hs    | 26 +++++++++++++++-----------
 Git/LsTree.hs     |  6 +++---
 Git/Queue.hs      |  5 +++--
 Git/Ref.hs        |  2 +-
 Git/UnionMerge.hs |  9 +++++----
 Messages.hs       |  6 +++---
 9 files changed, 36 insertions(+), 29 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index ec6d0abf39..878547bc36 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,7 +7,7 @@
 
 module Command.Uninit where
 
-import qualified Data.ByteString.Lazy.Char8 as B
+import qualified Data.Text.Lazy as L
 
 import Common.Annex
 import Command
@@ -29,7 +29,7 @@ check = do
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
-		current_branch = Git.Ref . Prelude.head . lines . B.unpack <$> revhead
+		current_branch = Git.Ref . Prelude.head . lines . L.unpack <$> revhead
 		revhead = inRepo $ Git.Command.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
diff --git a/Command/Unused.hs b/Command/Unused.hs
index ffd4bef455..67f743ab08 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -10,7 +10,8 @@
 module Command.Unused where
 
 import qualified Data.Set as S
-import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Text.Lazy as L
+import qualified Data.Text.Lazy.Encoding as L
 
 import Common.Annex
 import Command
@@ -202,7 +203,7 @@ getKeysReferencedInGit ref = do
 		findkeys c [] = return c
 		findkeys c (l:ls)
 			| isSymLink (LsTree.mode l) = do
-				content <- catFile ref $ LsTree.file l
+				content <- L.decodeUtf8 <$> catFile ref (LsTree.file l)
 				case fileKey (takeFileName $ L.unpack content) of
 					Nothing -> findkeys c ls
 					Just k -> findkeys (k:c) ls
diff --git a/Git/Branch.hs b/Git/Branch.hs
index 98811a9876..546d4a96b3 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -7,7 +7,7 @@
 
 module Git.Branch where
 
-import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Text.Lazy as L
 
 import Common
 import Git
diff --git a/Git/Command.hs b/Git/Command.hs
index ec701c1f09..1650efe13c 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -7,7 +7,10 @@
 
 module Git.Command where
 
-import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Text.Lazy as L
+import qualified Data.Text.Lazy.Encoding as L
+import qualified Data.Text.Lazy.IO as L
+import qualified Data.ByteString.Lazy as B
 
 import Common
 import Git
@@ -38,28 +41,27 @@ run subcommand params repo = assertLocal repo $
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
  -}
-pipeRead :: [CommandParam] -> Repo -> IO L.ByteString
+pipeRead :: [CommandParam] -> Repo -> IO L.Text
 pipeRead params repo = assertLocal repo $ do
 	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
-	hSetBinaryMode h True
-	L.hGetContents h
+	L.decodeUtf8 <$> B.hGetContents h
 
 {- Runs a git subcommand, feeding it input.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWrite :: [CommandParam] -> L.ByteString -> Repo -> IO PipeHandle
+pipeWrite :: [CommandParam] -> L.Text -> Repo -> IO PipeHandle
 pipeWrite params s repo = assertLocal repo $ do
 	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
-	L.hPut h s
+	L.hPutStr h s
 	hClose h
 	return p
 
 {- Runs a git subcommand, feeding it input, and returning its output.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: [CommandParam] -> L.ByteString -> Repo -> IO (PipeHandle, L.ByteString)
+pipeWriteRead :: [CommandParam] -> L.Text -> Repo -> IO (PipeHandle, L.Text)
 pipeWriteRead params s repo = assertLocal repo $ do
 	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
 	hSetBinaryMode from True
-	L.hPut to s
+	L.hPutStr to s
 	hClose to
 	c <- L.hGetContents from
 	return (p, c)
@@ -67,12 +69,14 @@ pipeWriteRead params s repo = assertLocal repo $ do
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it. -}
 pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
-pipeNullSplit params repo = map L.unpack <$> pipeNullSplitB params repo
+pipeNullSplit params repo = map L.unpack <$> pipeNullSplitT params repo
 
 {- For when Strings are not needed. -}
-pipeNullSplitB ::[CommandParam] -> Repo -> IO [L.ByteString]
-pipeNullSplitB params repo = filter (not . L.null) . L.split '\0' <$>
+pipeNullSplitT ::[CommandParam] -> Repo -> IO [L.Text]
+pipeNullSplitT params repo = filter (not . L.null) . L.splitOn sep <$>
 	pipeRead params repo
+	where
+		sep = L.pack "\0"
 
 {- Reaps any zombie git processes. -}
 reap :: IO ()
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index aae7f1263b..5c1541819a 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -14,7 +14,7 @@ module Git.LsTree (
 import Numeric
 import Control.Applicative
 import System.Posix.Types
-import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Text.Lazy as L
 
 import Common
 import Git
@@ -31,11 +31,11 @@ data TreeItem = TreeItem
 {- Lists the contents of a Ref -}
 lsTree :: Ref -> Repo -> IO [TreeItem]
 lsTree t repo = map parseLsTree <$>
-	pipeNullSplitB [Params "ls-tree --full-tree -z -r --", File $ show t] repo
+	pipeNullSplitT [Params "ls-tree --full-tree -z -r --", File $ show t] repo
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
-parseLsTree :: L.ByteString -> TreeItem
+parseLsTree :: L.Text -> TreeItem
 parseLsTree l = TreeItem 
 	{ mode = fst $ Prelude.head $ readOct $ L.unpack m
 	, typeobj = L.unpack t
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 25c5b073c7..63c3adee7c 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -18,8 +18,9 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Utility.SafeCommand
+import Codec.Binary.UTF8.String
 
+import Utility.SafeCommand
 import Common
 import Git
 import Git.Command
@@ -90,4 +91,4 @@ runAction repo action files =
 	where
 		params = toCommand $ gitCommandLine
 			(Param (getSubcommand action):getParams action) repo
-		feedxargs h = hPutStr h $ join "\0" files
+		feedxargs h = hPutStr h $ join "\0" $ map encodeString files
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 557d24a372..81560b0157 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -7,7 +7,7 @@
 
 module Git.Ref where
 
-import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Text.Lazy as L
 
 import Common
 import Git
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 4b335e47b1..19db328609 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -15,7 +15,8 @@ module Git.UnionMerge (
 ) where
 
 import System.Cmd.Utils
-import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.Text.Lazy as L
+import qualified Data.Text.Lazy.Encoding as L
 import qualified Data.Set as S
 
 import Common
@@ -110,11 +111,11 @@ mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 		calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
-		getcontents s = L.lines <$> catObject h s
+		getcontents s = L.lines . L.decodeUtf8 <$> catObject h s
 		use sha = return $ Just $ update_index_line sha file
 
 {- Injects some content into git, returning its Sha. -}
-hashObject :: Repo -> L.ByteString -> IO Sha
+hashObject :: Repo -> L.Text -> IO Sha
 hashObject repo content = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
 	L.length s `seq` do
@@ -130,7 +131,7 @@ hashObject repo content = getSha subcmd $ do
  - When possible, reuses the content of an existing ref, rather than
  - generating new content.
  -}
-calcMerge :: [(Ref, [L.ByteString])] -> Either Ref [L.ByteString]
+calcMerge :: [(Ref, [L.Text])] -> Either Ref [L.Text]
 calcMerge shacontents
 	| null reuseable = Right $ new
 	| otherwise = Left $ fst $ Prelude.head reuseable
diff --git a/Messages.hs b/Messages.hs
index 1294e44f69..844c6bfc51 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -128,9 +128,9 @@ showRaw s = handle q $ putStrLn s
  -
  - NB: Once git-annex gets localized, this will need a rethink. -}
 setupConsole :: IO ()
-setupConsole = do
-	hSetBinaryMode stdout True
-	hSetBinaryMode stderr True
+setupConsole = return ()
+	--hSetBinaryMode stdout True
+	--hSetBinaryMode stderr True
 
 handle :: IO () -> IO () -> Annex ()
 handle json normal = Annex.getState Annex.output >>= go

From b91569ba986ed6e85a6855c9ded07536d80d0d90 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Feb 2012 16:26:23 -0400
Subject: [PATCH 3055/8313] spent 3 hours on this bug; developed two incomplete
 fixes

---
 doc/bugs/problems_with_utf8_names.mdwn | 62 ++++++++++----------------
 1 file changed, 24 insertions(+), 38 deletions(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index b734ddecf7..c33420d2ab 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -1,6 +1,28 @@
 This bug is reopened to track some new UTF-8 filename issues caused by GHC
-7.4. Older versions of GHC, like the 7.0.4 in debian unstable, are not
-affected. See the comments for details about the new bug. --[[Joey]] 
+7.4. In this version of GHC, git-annex's hack to support filenames in any
+encoding no longer works. Even unicode filenames fail to work when
+git-annex is built with 7.4. --[[Joey]] 
+
+The new ghc requires a new data type, `RawFilePath` be used if you
+don't want to impose utf-8 filenames on your users. I have a `newghc` branch
+in git where I am trying to convert it to use `RawFilePath`. However, since
+there is no way to cast a `FilePath` to a `RawFilePath` or back (because
+the encoding of `RawFilePath` is not specified), this means changing
+essentially all of git-annex. Even the filenames used for keys in
+`.git/annex/objects` need to use the new data type. Worse, several utility
+libraries it uses are only available for `FilePath`.
+
+The current state of the branch is that it needs an implementation of
+`absNormPath` for `RawFilePath` to be added, as well as some other path
+manipulation functions like `parentDir`. Then the types can continue
+to be followed to get it to build and work. It could take days or weeks of
+work. --[[Joey]]
+
+**As a stopgap workaround**, I have made a branch `unicode-only`. This
+makes git-annex work with unicode filenames with ghc 7.4, but *only*
+unicode filenames. If you have filenames with some other encoding, you're
+out in the cold, and it will probably just crash with a error about wrong
+encoding. --[[Joey]]
 
 ----
 
@@ -74,39 +96,3 @@ It looks like the common latin1-to-UTF8 encoding. Functionality other than otupu
 >    > On second thought, I switched to this. Any decoding of a filename
 >    > is going to make someone unhappy; the previous approach broke
 >    > non-utf8 filenames.
-
-----
-
-Simpler test case:
-
-
-import Codec.Binary.UTF8.String
-import System.Environment
-
-main = do
-        args <- getArgs
-        let file = decodeString $ head args
-        putStrLn $ "file is: " ++ file
-        putStr =<< readFile file
-

-
-If I pass this a filename like 'ü', it will fail, and notice
-the bad encoding of the filename in the error message:
-
-
-$ echo hi > ü; runghc foo.hs ü
-file is: ü
-foo.hs: �: openFile: does not exist (No such file or directory)
-

-
-On the other hand, if I remove the decodeString, it prints the filename
-wrong, while accessing it right:
-
-
-$ runghc foo.hs ü
-file is: üa
-hi
-

-
-The only way that seems to consistently work is to delay decoding the
-filename to places where it's output. But then it's easy to miss some.

From f2081ad056306c663ac89c95f379b39135e682e5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Feb 2012 22:24:40 -0400
Subject: [PATCH 3056/8313] update

---
 doc/bugs/problems_with_utf8_names.mdwn | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index c33420d2ab..3c3fdd0ace 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -9,14 +9,7 @@ in git where I am trying to convert it to use `RawFilePath`. However, since
 there is no way to cast a `FilePath` to a `RawFilePath` or back (because
 the encoding of `RawFilePath` is not specified), this means changing
 essentially all of git-annex. Even the filenames used for keys in
-`.git/annex/objects` need to use the new data type. Worse, several utility
-libraries it uses are only available for `FilePath`.
-
-The current state of the branch is that it needs an implementation of
-`absNormPath` for `RawFilePath` to be added, as well as some other path
-manipulation functions like `parentDir`. Then the types can continue
-to be followed to get it to build and work. It could take days or weeks of
-work. --[[Joey]]
+`.git/annex/objects` need to use the new data type. --[[Joey]]
 
 **As a stopgap workaround**, I have made a branch `unicode-only`. This
 makes git-annex work with unicode filenames with ghc 7.4, but *only*

From 33fd49703cab632225b6ec36a797daef63c86012 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Feb 2012 10:31:56 -0400
Subject: [PATCH 3057/8313] update

---
 doc/bugs/problems_with_utf8_names.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index 3c3fdd0ace..cb5917ef8d 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -11,6 +11,14 @@ the encoding of `RawFilePath` is not specified), this means changing
 essentially all of git-annex. Even the filenames used for keys in
 `.git/annex/objects` need to use the new data type. --[[Joey]]
 
+> Actually it may not be that bad. A `RawFilePath` contains only bytes,
+> so it can be cast to a string, containing encoded characters. That
+> string can then be 1) output in binary mode or 2) manipulated
+> in ways that do not add characters larger than 255, and cast back to
+> a `RawFilePath`. While not type-safe, such casts should at least
+> help during bootstrapping, and might allow for a quick fix that only
+> changes to `RawFilePath` at the edges.
+
 **As a stopgap workaround**, I have made a branch `unicode-only`. This
 makes git-annex work with unicode filenames with ghc 7.4, but *only*
 unicode filenames. If you have filenames with some other encoding, you're

From 39e887e8e623acd011a9ae40f06b8f16548c019d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Feb 2012 10:56:23 -0400
Subject: [PATCH 3058/8313] update; unix-compat gets some of the way

---
 doc/todo/windows_support.mdwn | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/doc/todo/windows_support.mdwn b/doc/todo/windows_support.mdwn
index 8df792fd6e..c64e6fce5c 100644
--- a/doc/todo/windows_support.mdwn
+++ b/doc/todo/windows_support.mdwn
@@ -1,25 +1,16 @@
-short answer: no
+Can it be built on Windows?
 
-Long answer, quoting from a mail to someone else:
+short answer: not yet
 
-Well, I can tell you that it assumes a POSIX system, both in available
-utilities and system calls, So you'd need to use cygwin or something
-like that. (Perhaps you already are for git, I think git also assumes a
-POSIX system.) So you need a Haskell that can target that. What this
-page refers to as "GHC-Cygwin":
-
-I don't know where to get one. Did find this:
-
+First, you need to get some unix utilities for windows. Git of course.
+Also rsync, and a `cp` command that understands at least `cp -p`, and
+`uuid`, and `xargs` and `sha1sum`. Note that some of these could be
+replaced with haskell libraries to some degree.
 
-(There are probably also still some places where it assumes / as a path
-separator, although I fixed some. Probably almost all are fixed now.)
+There are probably still some places where it assumes / as a path
+separator, although I fixed probably almost all by now.
 
-FWIW, git-annex works fine on OS X and other fine proprietary unixen. ;P
---[[Joey]]
-
-----
-
-Alternatively, windows versions of these functions could be found,
+Then windows versions of these functions could be found,
 which are all the ones that need POSIX, I think. A fair amount of this,
 the stuff to do with signals and users, could be empty stubs in windows.
 The file manipulation, particularly symlinks, would probably be the main
@@ -63,3 +54,8 @@ sigCHLD
 sigINT
 unionFileModes
 

+
+A good starting point is
+. However, note
+that its implementations of stuff like `createSymbolicLink` are stubs.
+--[[Joey]] 

From fc8a1d213b3683253923529918c84c91a75448fa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Feb 2012 10:56:39 -0400
Subject: [PATCH 3059/8313] update

---
 doc/bugs/problems_with_utf8_names.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index cb5917ef8d..c9ca1e3b07 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -3,6 +3,12 @@ This bug is reopened to track some new UTF-8 filename issues caused by GHC
 encoding no longer works. Even unicode filenames fail to work when
 git-annex is built with 7.4. --[[Joey]] 
 
+> What's going on exactly? The new ghc, when presented with 
+> a String of raw bytes like "fo\194\161", and asked to do
+> something like `getSymbolicLinkStatus`, encodes it
+> as unicode, yielding "fo\303\202\302\241". Which is
+> not the same as the original filename.
+
 The new ghc requires a new data type, `RawFilePath` be used if you
 don't want to impose utf-8 filenames on your users. I have a `newghc` branch
 in git where I am trying to convert it to use `RawFilePath`. However, since

From 828df56453d9b0a1483d5c85e6ca739b158883d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Feb 2012 15:41:22 -0400
Subject: [PATCH 3060/8313] update; newghc-edges branch

---
 doc/bugs/problems_with_utf8_names.mdwn | 73 +++++++++++++++++---------
 1 file changed, 49 insertions(+), 24 deletions(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index c9ca1e3b07..ac110a6aed 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -1,35 +1,60 @@
 This bug is reopened to track some new UTF-8 filename issues caused by GHC
 7.4. In this version of GHC, git-annex's hack to support filenames in any
 encoding no longer works. Even unicode filenames fail to work when
-git-annex is built with 7.4. --[[Joey]] 
-
-> What's going on exactly? The new ghc, when presented with 
-> a String of raw bytes like "fo\194\161", and asked to do
-> something like `getSymbolicLinkStatus`, encodes it
-> as unicode, yielding "fo\303\202\302\241". Which is
-> not the same as the original filename.
-
-The new ghc requires a new data type, `RawFilePath` be used if you
-don't want to impose utf-8 filenames on your users. I have a `newghc` branch
-in git where I am trying to convert it to use `RawFilePath`. However, since
-there is no way to cast a `FilePath` to a `RawFilePath` or back (because
-the encoding of `RawFilePath` is not specified), this means changing
-essentially all of git-annex. Even the filenames used for keys in
-`.git/annex/objects` need to use the new data type. --[[Joey]]
-
-> Actually it may not be that bad. A `RawFilePath` contains only bytes,
-> so it can be cast to a string, containing encoded characters. That
-> string can then be 1) output in binary mode or 2) manipulated
-> in ways that do not add characters larger than 255, and cast back to
-> a `RawFilePath`. While not type-safe, such casts should at least
-> help during bootstrapping, and might allow for a quick fix that only
-> changes to `RawFilePath` at the edges.
+git-annex is built with 7.4. --[[Joey]]
 
 **As a stopgap workaround**, I have made a branch `unicode-only`. This
 makes git-annex work with unicode filenames with ghc 7.4, but *only*
 unicode filenames. If you have filenames with some other encoding, you're
 out in the cold, and it will probably just crash with a error about wrong
-encoding. --[[Joey]]
+encoding.
+
+## analysis
+
+What's going on exactly? The new ghc, when presented with 
+a String of raw bytes like "fo\194\161", and asked to do
+something like `getSymbolicLinkStatus`, encodes it
+as unicode, yielding "fo\303\202\302\241". Which is
+not the same as the original filename, assuming it was "fo¡".
+
+The new ghc requires a new data type, `RawFilePath` be used if you
+don't want to impose utf-8 filenames on your users.
+
+The available `RawFilePath` support is quite low-level, so all the nice
+readFile and writeFile code, etc has to be reimplemented. So do any utility
+libraries that do things with FilePaths, if you need them to use
+RawFilePaths. Until the haskell ecosystem adapts to `RawFilePath`
+(if it does), using it broadly, as git-annex needs to, will be difficult.
+
+## newghc branch
+
+I have a `newghc` branch in git where I am trying to convert it to use
+`RawFilePath`. However, since there is no way to cast a `FilePath` to a
+`RawFilePath` or back (because the encoding of `RawFilePath` is not
+specified), this means changing essentially all of git-annex. Even the
+filenames used for keys in `.git/annex/objects` need to use the new data
+type. I didn't get very far on this branch.
+
+## newghc-edges branch
+
+I have a `newghc-edges` branch in git, trying a different approach.
+
+A `RawFilePath` contains only bytes, so it can actually be cast to a string,
+containing encoded characters. That string can then be 1) output in binary
+mode or 2) manipulated in ways that do not add characters larger than 255,
+and cast back to a `RawFilePath`. While not type-safe, such casts should at
+least help during bootstrapping, and might allow for a quick fix that only
+changes to `RawFilePath` at the edges.
+
+The branch contains an almost complete, although probably also buggy
+conversion using this method. It is missing wrappers for a
+few things like `readFile` and `writeFile` but otherwise seems to
+basically work.
+
+Is this a suitable approach for merging into `master`? It's nasty,
+being not type safe, having to reimplent/copy+modify random bits of
+libraries, etc. The nastiness is contained, though, in a single file,
+of only a few hundred lines of code. --[[Joey]]
 
 ----
 

From 94caa268831e14134ee72a01eace8c9fff9a954a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Feb 2012 15:44:22 -0400
Subject: [PATCH 3061/8313] update

---
 doc/bugs/problems_with_utf8_names.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index ac110a6aed..b99b587839 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -26,9 +26,9 @@ libraries that do things with FilePaths, if you need them to use
 RawFilePaths. Until the haskell ecosystem adapts to `RawFilePath`
 (if it does), using it broadly, as git-annex needs to, will be difficult.
 
-## newghc branch
+## rawfilepath branch
 
-I have a `newghc` branch in git where I am trying to convert it to use
+I have a `rawfilepath` branch in git where I am trying to convert it to use
 `RawFilePath`. However, since there is no way to cast a `FilePath` to a
 `RawFilePath` or back (because the encoding of `RawFilePath` is not
 specified), this means changing essentially all of git-annex. Even the

From fb78107f85ffbd83c7c8dad6bfcaa8374387cccc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Feb 2012 01:38:23 -0400
Subject: [PATCH 3062/8313] add a check for not utf-8 console

---
 Messages.hs | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/Messages.hs b/Messages.hs
index 844c6bfc51..ff5287d80c 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -119,18 +119,16 @@ showHeader h = handle q $
 showRaw :: String -> Annex ()
 showRaw s = handle q $ putStrLn s
 
-{- By default, haskell honors the user's locale in its output to stdout
- - and stderr. While that's great for proper unicode support, for git-annex
- - all that's really needed is the ability to display simple messages
- - (currently untranslated), and importantly, to display filenames exactly
- - as they are written on disk, no matter what their encoding. So, force
- - raw mode. 
- -
- - NB: Once git-annex gets localized, this will need a rethink. -}
+{- This check is done because the code assumes filenames are utf8 encoded,
+ - using decodeUtf8 and Codec.Binary.UTF8.String.encodeString. So if
+ - run in a non unicode locale, it will crash or worse, possibly operate
+ - on the wrong file.
+ -}
 setupConsole :: IO ()
-setupConsole = return ()
-	--hSetBinaryMode stdout True
-	--hSetBinaryMode stderr True
+setupConsole
+	| show localeEncoding == show utf8 = return ()
+	| otherwise = error $
+		"Sorry, only UTF-8 locales are currently supported."
 
 handle :: IO () -> IO () -> Annex ()
 handle json normal = Annex.getState Annex.output >>= go

From d8fb97806c430be8358b2b77d67c02e876278d2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Feb 2012 15:12:41 -0400
Subject: [PATCH 3063/8313] support all filename encodings with ghc 7.4

Under ghc 7.4, this seems to be able to handle all filename encodings
again. Including filename encodings that do not match the LANG setting.
I think this will not work with earlier versions of ghc, it uses some ghc
internals.

Turns out that ghc 7.4 has a special filesystem encoding that it uses when
reading/writing filenames (as FilePaths). This encoding is documented
to allow  "arbitrary undecodable bytes to be round-tripped through it".

So, to get FilePaths from eg, git ls-files, set the Handle that is reading
from git to use this encoding. Then things basically just work.

However, I have not found a way to make Text read using this encoding.
Text really does assume unicode. So I had to switch back to using String
when reading/writing data to git. Which is a pity, because it's some
percent slower, but at least it works.

Note that stdout and stderr also have to be set to this encoding, or
printing out filenames that contain undecodable bytes causes a crash.
IMHO this is a misfeature in ghc, that the user can pass you a filename,
which you can readFile, etc, but that default, putStr of filename may
cause a crash!

Git.CheckAttr gave me special trouble, because the filenames I got back
from git, after feeding them in, had further encoding breakage.
Rather than try to deal with that, I just zip up the input filenames
with the attributes. Which must be returned in the same order queried
for this to work.

Also of note is an apparent GHC bug I worked around in Git.CheckAttr. It
used to forkProcess and feed git from the child process.  Unfortunatly,
after this forkProcess, accessing the `files` variable from the parent
returns []. Not the value that was passed into the function. This screams
of a bad bug, that's clobbering a variable, but for now I just avoid
forkProcess there to work around it. That forkProcess was itself only added
because of a ghc bug, #624389. I've confirmed that the test case for that
bug doesn't reproduce it with ghc 7.4. So that's ok, except for the new ghc
bug I have not isolated and reported. Why does this simple bit of code
magnet the ghc bugs? :)

Also, the symlink touching code is currently broken, when used on utf-8
filenames in a non-utf-8 locale, or probably on any filename containing
undecodable bytes, and I temporarily commented it out.
---
 Command/Add.hs    |  8 +++++---
 Command/Uninit.hs |  4 +---
 Command/Unused.hs |  2 +-
 Git/Branch.hs     | 15 ++++++---------
 Git/CheckAttr.hs  | 32 +++++---------------------------
 Git/Command.hs    | 26 +++++++++++---------------
 Git/LsTree.hs     | 21 ++++++++++-----------
 Git/Queue.hs      |  5 +++--
 Git/Ref.hs        |  6 ++----
 Git/UnionMerge.hs | 13 +++++++------
 Messages.hs       | 13 +++++--------
 Utility/Misc.hs   |  8 ++++++++
 12 files changed, 64 insertions(+), 89 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 9410601b8b..944525ea5e 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -82,9 +82,11 @@ cleanup file key hascontent = do
 	
 			-- touch the symlink to have the same mtime as the
 			-- file it points to
-			liftIO $ do
-				mtime <- modificationTime <$> getFileStatus file
-				touch file (TimeSpec mtime) False
+			-- XXX Currently broken on non-utf8 locales when
+			-- dealing with utf-8 filenames.
+			--liftIO $ do
+				--mtime <- modificationTime <$> getFileStatus file
+				--touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 878547bc36..d6283a77da 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -7,8 +7,6 @@
 
 module Command.Uninit where
 
-import qualified Data.Text.Lazy as L
-
 import Common.Annex
 import Command
 import qualified Git
@@ -29,7 +27,7 @@ check = do
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	where
-		current_branch = Git.Ref . Prelude.head . lines . L.unpack <$> revhead
+		current_branch = Git.Ref . Prelude.head . lines <$> revhead
 		revhead = inRepo $ Git.Command.pipeRead 
 			[Params "rev-parse --abbrev-ref HEAD"]
 
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 67f743ab08..1c82b9ae4a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -162,7 +162,7 @@ excludeReferenced l = do
 		refs = map (Git.Ref .  snd) .
 			nubBy uniqref .
 			filter ourbranches .
-			map (separate (== ' ')) . lines . L.unpack
+			map (separate (== ' ')) . lines
 		uniqref (a, _) (b, _) = a == b
 		ourbranchend = '/' : show Annex.Branch.name
 		ourbranches (_, b) = not $ ourbranchend `isSuffixOf` b
diff --git a/Git/Branch.hs b/Git/Branch.hs
index 546d4a96b3..cd91882283 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -7,8 +7,6 @@
 
 module Git.Branch where
 
-import qualified Data.Text.Lazy as L
-
 import Common
 import Git
 import Git.Sha
@@ -19,15 +17,15 @@ current :: Repo -> IO (Maybe Git.Ref)
 current r = parse <$> pipeRead [Param "symbolic-ref", Param "HEAD"] r
 	where
 		parse v
-			| L.null v = Nothing
-			| otherwise = Just $ Git.Ref $ firstLine $ L.unpack v
+			| null v = Nothing
+			| otherwise = Just $ Git.Ref $ firstLine v
 
 {- Checks if the second branch has any commits not present on the first
  - branch. -}
 changed :: Branch -> Branch -> Repo -> IO Bool
 changed origbranch newbranch repo
 	| origbranch == newbranch = return False
-	| otherwise = not . L.null <$> diffs
+	| otherwise = not . null <$> diffs
 	where
 		diffs = pipeRead
 			[ Param "log"
@@ -73,15 +71,14 @@ fastForward branch (first:rest) repo = do
  - with the specified parent refs, and returns the committed sha -}
 commit :: String -> Branch -> [Ref] -> Repo -> IO Sha
 commit message branch parentrefs repo = do
-	tree <- getSha "write-tree" $ asString $
+	tree <- getSha "write-tree" $
 		pipeRead [Param "write-tree"] repo
-	sha <- getSha "commit-tree" $ asString $
+	sha <- getSha "commit-tree" $
 		ignorehandle $ pipeWriteRead
 			(map Param $ ["commit-tree", show tree] ++ ps)
-			(L.pack message) repo
+			message repo
 	run "update-ref" [Param $ show branch, Param $ show sha] repo
 	return sha
 	where
 		ignorehandle a = snd <$> a
-		asString a = L.unpack <$> a
 		ps = concatMap (\r -> ["-p", show r]) parentrefs
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index eedaf66420..3e9375159a 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -7,12 +7,9 @@
 
 module Git.CheckAttr where
 
-import System.Exit
-
 import Common
 import Git
 import Git.Command
-import qualified Git.Filename
 import qualified Git.Version
 
 {- Efficiently looks up a gitattributes value for each file in a list. -}
@@ -20,13 +17,9 @@ lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
 lookup attr files repo = do
 	cwd <- getCurrentDirectory
 	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
-        _ <- forkProcess $ do
-		hClose fromh
-                hPutStr toh $ join "\0" $ input cwd
-                hClose toh
-                exitSuccess
-        hClose toh
-	output cwd . lines <$> hGetContents fromh
+	hPutStr toh $ join "\0" $ input cwd
+	hClose toh
+	zip files . map attrvalue . lines <$> hGetContents fromh
 	where
 		params = gitCommandLine 
 				[ Param "check-attr"
@@ -45,22 +38,7 @@ lookup attr files repo = do
 		input cwd
 			| oldgit = map (absPathFrom cwd) files
 			| otherwise = map (relPathDirToFile cwd . absPathFrom cwd) files
-		output cwd
-			| oldgit = map (torel cwd . topair)
-			| otherwise = map topair
-
-		topair l = (Git.Filename.decode file, value)
-			where 
-				file = join sep $ beginning bits
-				value = end bits !! 0
+		attrvalue l = end bits !! 0
+			where
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
-
-		torel cwd (file, value) = (relfile, value)
-			where
-				relfile
-					| startswith cwd' file = drop (length cwd') file
-					| otherwise = relPathDirToFile top' file
-				top = workTree repo
-				cwd' = cwd ++ "/"
-				top' = top ++ "/"
diff --git a/Git/Command.hs b/Git/Command.hs
index 1650efe13c..3d859ed28d 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -8,9 +8,7 @@
 module Git.Command where
 
 import qualified Data.Text.Lazy as L
-import qualified Data.Text.Lazy.Encoding as L
 import qualified Data.Text.Lazy.IO as L
-import qualified Data.ByteString.Lazy as B
 
 import Common
 import Git
@@ -41,10 +39,11 @@ run subcommand params repo = assertLocal repo $
  - Note that this leaves the git process running, and so zombies will
  - result unless reap is called.
  -}
-pipeRead :: [CommandParam] -> Repo -> IO L.Text
+pipeRead :: [CommandParam] -> Repo -> IO String
 pipeRead params repo = assertLocal repo $ do
 	(_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo
-	L.decodeUtf8 <$> B.hGetContents h
+	fileEncoding h
+	hGetContents h
 
 {- Runs a git subcommand, feeding it input.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
@@ -57,26 +56,23 @@ pipeWrite params s repo = assertLocal repo $ do
 
 {- Runs a git subcommand, feeding it input, and returning its output.
  - You should call either getProcessStatus or forceSuccess on the PipeHandle. -}
-pipeWriteRead :: [CommandParam] -> L.Text -> Repo -> IO (PipeHandle, L.Text)
+pipeWriteRead :: [CommandParam] -> String -> Repo -> IO (PipeHandle, String)
 pipeWriteRead params s repo = assertLocal repo $ do
 	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
-	hSetBinaryMode from True
-	L.hPutStr to s
+	fileEncoding to
+	fileEncoding from
+	hPutStr to s
 	hClose to
-	c <- L.hGetContents from
+	c <- hGetContents from
 	return (p, c)
 
 {- Reads null terminated output of a git command (as enabled by the -z 
  - parameter), and splits it. -}
 pipeNullSplit :: [CommandParam] -> Repo -> IO [String]
-pipeNullSplit params repo = map L.unpack <$> pipeNullSplitT params repo
-
-{- For when Strings are not needed. -}
-pipeNullSplitT ::[CommandParam] -> Repo -> IO [L.Text]
-pipeNullSplitT params repo = filter (not . L.null) . L.splitOn sep <$>
-	pipeRead params repo
+pipeNullSplit params repo =
+	filter (not . null) . split sep <$> pipeRead params repo
 	where
-		sep = L.pack "\0"
+		sep = "\0"
 
 {- Reaps any zombie git processes. -}
 reap :: IO ()
diff --git a/Git/LsTree.hs b/Git/LsTree.hs
index 5c1541819a..8f9066f0f3 100644
--- a/Git/LsTree.hs
+++ b/Git/LsTree.hs
@@ -14,7 +14,6 @@ module Git.LsTree (
 import Numeric
 import Control.Applicative
 import System.Posix.Types
-import qualified Data.Text.Lazy as L
 
 import Common
 import Git
@@ -31,22 +30,22 @@ data TreeItem = TreeItem
 {- Lists the contents of a Ref -}
 lsTree :: Ref -> Repo -> IO [TreeItem]
 lsTree t repo = map parseLsTree <$>
-	pipeNullSplitT [Params "ls-tree --full-tree -z -r --", File $ show t] repo
+	pipeNullSplit [Params "ls-tree --full-tree -z -r --", File $ show t] repo
 
 {- Parses a line of ls-tree output.
  - (The --long format is not currently supported.) -}
-parseLsTree :: L.Text -> TreeItem
+parseLsTree :: String -> TreeItem
 parseLsTree l = TreeItem 
-	{ mode = fst $ Prelude.head $ readOct $ L.unpack m
-	, typeobj = L.unpack t
-	, sha = L.unpack s
-	, file = Git.Filename.decode $ L.unpack f
+	{ mode = fst $ Prelude.head $ readOct m
+	, typeobj = t
+	, sha = s
+	, file = Git.Filename.decode f
 	}
 	where
 		-- l =  SP  SP  TAB 
 		-- All fields are fixed, so we can pull them out of
 		-- specific positions in the line.
-		(m, past_m) = L.splitAt 7 l
-		(t, past_t) = L.splitAt 4 past_m
-		(s, past_s) = L.splitAt 40 $ L.tail past_t
-		f = L.tail past_s
+		(m, past_m) = splitAt 7 l
+		(t, past_t) = splitAt 4 past_m
+		(s, past_s) = splitAt 40 $ Prelude.tail past_t
+		f = Prelude.tail past_s
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 63c3adee7c..c71605ad52 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -18,7 +18,6 @@ import qualified Data.Map as M
 import System.IO
 import System.Cmd.Utils
 import Data.String.Utils
-import Codec.Binary.UTF8.String
 
 import Utility.SafeCommand
 import Common
@@ -91,4 +90,6 @@ runAction repo action files =
 	where
 		params = toCommand $ gitCommandLine
 			(Param (getSubcommand action):getParams action) repo
-		feedxargs h = hPutStr h $ join "\0" $ map encodeString files
+		feedxargs h = do
+			fileEncoding h
+			hPutStr h $ join "\0" files
diff --git a/Git/Ref.hs b/Git/Ref.hs
index 81560b0157..f483aede03 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -7,8 +7,6 @@
 
 module Git.Ref where
 
-import qualified Data.Text.Lazy as L
-
 import Common
 import Git
 import Git.Command
@@ -40,7 +38,7 @@ exists ref = runBool "show-ref"
 
 {- Get the sha of a fully qualified git ref, if it exists. -}
 sha :: Branch -> Repo -> IO (Maybe Sha)
-sha branch repo = process . L.unpack <$> showref repo
+sha branch repo = process <$> showref repo
 	where
 		showref = pipeRead [Param "show-ref",
 			Param "--hash", -- get the hash
@@ -52,7 +50,7 @@ sha branch repo = process . L.unpack <$> showref repo
 matching :: Ref -> Repo -> IO [(Ref, Branch)]
 matching ref repo = do
 	r <- pipeRead [Param "show-ref", Param $ show ref] repo
-	return $ map (gen . L.unpack) (L.lines r)
+	return $ map gen (lines r)
 	where
 		gen l = let (r, b) = separate (== ' ') l in
 			(Ref r, Ref b)
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 19db328609..15bff60527 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -107,21 +107,22 @@ mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> use sha
-	shas -> use =<< either return (hashObject repo . L.unlines) =<<
+	shas -> use =<< either return (hashObject repo . unlines) =<<
 		calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
-		getcontents s = L.lines . L.decodeUtf8 <$> catObject h s
+		getcontents s = map L.unpack . L.lines .
+			L.decodeUtf8 <$> catObject h s
 		use sha = return $ Just $ update_index_line sha file
 
 {- Injects some content into git, returning its Sha. -}
-hashObject :: Repo -> L.Text -> IO Sha
+hashObject :: Repo -> String -> IO Sha
 hashObject repo content = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
-	L.length s `seq` do
+	length s `seq` do
 		forceSuccess h
 		reap -- XXX unsure why this is needed
-		return $ L.unpack s
+		return s
 	where
 		subcmd = "hash-object"
 		params = [subcmd, "-w", "--stdin"]
@@ -131,7 +132,7 @@ hashObject repo content = getSha subcmd $ do
  - When possible, reuses the content of an existing ref, rather than
  - generating new content.
  -}
-calcMerge :: [(Ref, [L.Text])] -> Either Ref [L.Text]
+calcMerge :: [(Ref, [String])] -> Either Ref [String]
 calcMerge shacontents
 	| null reuseable = Right $ new
 	| otherwise = Left $ fst $ Prelude.head reuseable
diff --git a/Messages.hs b/Messages.hs
index ff5287d80c..a0bd20ca3c 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -119,16 +119,13 @@ showHeader h = handle q $
 showRaw :: String -> Annex ()
 showRaw s = handle q $ putStrLn s
 
-{- This check is done because the code assumes filenames are utf8 encoded,
- - using decodeUtf8 and Codec.Binary.UTF8.String.encodeString. So if
- - run in a non unicode locale, it will crash or worse, possibly operate
- - on the wrong file.
+{- This avoids ghc's output layer crashing on invalid encoded characters in
+ - files when printing them out.
  -}
 setupConsole :: IO ()
-setupConsole
-	| show localeEncoding == show utf8 = return ()
-	| otherwise = error $
-		"Sorry, only UTF-8 locales are currently supported."
+setupConsole = do
+	fileEncoding stdout
+	fileEncoding stderr
 
 handle :: IO () -> IO () -> Annex ()
 handle json normal = Annex.getState Annex.output >>= go
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index c9bfcb953a..c4992e1428 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -11,6 +11,14 @@ import System.IO
 import System.IO.Error (try)
 import Control.Monad
 import Control.Applicative
+import GHC.IO.Encoding
+
+{- Sets a Handle to use the filesystem encoding. This causes data
+ - written or read from it to be encoded/decoded the same
+ - as ghc 7.4 does to filenames et. This special encoding
+ - allows "arbitrary undecodable bytes to be round-tripped through it". -}
+fileEncoding :: Handle -> IO ()
+fileEncoding h = hSetEncoding h =<< getFileSystemEncoding
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}

From 05f89123e08075cfbd136f37c60423c1ad38d1fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Feb 2012 16:25:34 -0400
Subject: [PATCH 3064/8313] update; ghc7.4 branch fixes this pretty well now

---
 doc/bugs/problems_with_utf8_names.mdwn | 56 ++------------------------
 1 file changed, 4 insertions(+), 52 deletions(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index b99b587839..fbdca41cd1 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -3,58 +3,10 @@ This bug is reopened to track some new UTF-8 filename issues caused by GHC
 encoding no longer works. Even unicode filenames fail to work when
 git-annex is built with 7.4. --[[Joey]]
 
-**As a stopgap workaround**, I have made a branch `unicode-only`. This
-makes git-annex work with unicode filenames with ghc 7.4, but *only*
-unicode filenames. If you have filenames with some other encoding, you're
-out in the cold, and it will probably just crash with a error about wrong
-encoding.
-
-## analysis
-
-What's going on exactly? The new ghc, when presented with 
-a String of raw bytes like "fo\194\161", and asked to do
-something like `getSymbolicLinkStatus`, encodes it
-as unicode, yielding "fo\303\202\302\241". Which is
-not the same as the original filename, assuming it was "fo¡".
-
-The new ghc requires a new data type, `RawFilePath` be used if you
-don't want to impose utf-8 filenames on your users.
-
-The available `RawFilePath` support is quite low-level, so all the nice
-readFile and writeFile code, etc has to be reimplemented. So do any utility
-libraries that do things with FilePaths, if you need them to use
-RawFilePaths. Until the haskell ecosystem adapts to `RawFilePath`
-(if it does), using it broadly, as git-annex needs to, will be difficult.
-
-## rawfilepath branch
-
-I have a `rawfilepath` branch in git where I am trying to convert it to use
-`RawFilePath`. However, since there is no way to cast a `FilePath` to a
-`RawFilePath` or back (because the encoding of `RawFilePath` is not
-specified), this means changing essentially all of git-annex. Even the
-filenames used for keys in `.git/annex/objects` need to use the new data
-type. I didn't get very far on this branch.
-
-## newghc-edges branch
-
-I have a `newghc-edges` branch in git, trying a different approach.
-
-A `RawFilePath` contains only bytes, so it can actually be cast to a string,
-containing encoded characters. That string can then be 1) output in binary
-mode or 2) manipulated in ways that do not add characters larger than 255,
-and cast back to a `RawFilePath`. While not type-safe, such casts should at
-least help during bootstrapping, and might allow for a quick fix that only
-changes to `RawFilePath` at the edges.
-
-The branch contains an almost complete, although probably also buggy
-conversion using this method. It is missing wrappers for a
-few things like `readFile` and `writeFile` but otherwise seems to
-basically work.
-
-Is this a suitable approach for merging into `master`? It's nasty,
-being not type safe, having to reimplent/copy+modify random bits of
-libraries, etc. The nastiness is contained, though, in a single file,
-of only a few hundred lines of code. --[[Joey]]
+I now have a `ghc7.4` branch in git that seems to solve this, 
+for all filename encodings, and all system encodings. It will
+only build with the new GHC. If you have this problem, give it a try!
+--[[Joey]] 
 
 ----
 

From 146c36ca545a297f1e44e3cf2c91f3c0e17c909f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Feb 2012 16:47:24 -0400
Subject: [PATCH 3065/8313] IO exception rework

ghc 7.4 comaplains about use of System.IO.Error to catch exceptions.
Ok, use Control.Exception, with variants specialized to only catch IO
exceptions.
---
 Annex/Content.hs     |  3 +--
 Annex/Journal.hs     |  2 +-
 Annex/Ssh.hs         |  4 ++--
 Backend.hs           |  3 +--
 CmdLine.hs           |  3 +--
 Command/Fsck.hs      |  2 +-
 Common.hs            |  1 +
 Remote/Bup.hs        |  3 +--
 Upgrade/V0.hs        |  4 +---
 Upgrade/V1.hs        |  5 ++---
 Utility/Directory.hs |  8 ++++----
 Utility/Exception.hs | 39 +++++++++++++++++++++++++++++++++++++++
 Utility/Misc.hs      | 21 ---------------------
 Utility/TempFile.hs  |  2 +-
 14 files changed, 56 insertions(+), 44 deletions(-)
 create mode 100644 Utility/Exception.hs

diff --git a/Annex/Content.hs b/Annex/Content.hs
index dcfd43866b..d10370bc9a 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -25,7 +25,6 @@ module Annex.Content (
 	preseedTmp,
 ) where
 
-import System.IO.Error (try)
 import Control.Exception (bracket_)
 import System.Posix.Types
 
@@ -79,7 +78,7 @@ lockContent key a = do
 	where
 		lock Nothing = return Nothing
 		lock (Just l) = do
-			v <- try $ setLock l (WriteLock, AbsoluteSeek, 0, 0)
+			v <- tryIO $ setLock l (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
 				Left _ -> error "content is locked"
 				Right _ -> return $ Just l
diff --git a/Annex/Journal.hs b/Annex/Journal.hs
index 9c5be89b19..34c4d98c88 100644
--- a/Annex/Journal.hs
+++ b/Annex/Journal.hs
@@ -91,4 +91,4 @@ lockJournal a = do
 {- Runs an action, catching failure and running something to fix it up, and
  - retrying if necessary. -}
 doRedo :: IO a -> IO b -> IO a
-doRedo a b = catch a $ const $ b >> a
+doRedo a b = catchIO a $ const $ b >> a
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 14ea74e53e..d6f36e8689 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -11,7 +11,6 @@ module Annex.Ssh (
 ) where
 
 import qualified Data.Map as M
-import System.IO.Error (try)
 
 import Common.Annex
 import Annex.LockPool
@@ -72,7 +71,8 @@ sshCleanup = do
 			let lockfile = socket2lock socketfile
 			unlockFile lockfile
 			fd <- liftIO $ openFd lockfile ReadWrite (Just stdFileMode) defaultFileFlags
-			v <- liftIO $ try $ setLock fd (WriteLock, AbsoluteSeek, 0, 0)
+			v <- liftIO $ tryIO $
+				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
 				Left _ -> return ()
 				Right _ -> stopssh socketfile
diff --git a/Backend.hs b/Backend.hs
index 003d62bfcd..e351bb3b27 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -16,7 +16,6 @@ module Backend (
 	maybeLookupBackendName
 ) where
 
-import System.IO.Error (try)
 import System.Posix.Files
 
 import Common.Annex
@@ -77,7 +76,7 @@ genKey' (b:bs) file = do
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile file = do
-	tl <- liftIO $ try getsymlink
+	tl <- liftIO $ tryIO getsymlink
 	case tl of
 		Left _ -> return Nothing
 		Right l -> makekey l
diff --git a/CmdLine.hs b/CmdLine.hs
index 61e6c26bb8..18bb5fe51b 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -11,7 +11,6 @@ module CmdLine (
 	shutdown
 ) where
 
-import qualified System.IO.Error as IO
 import qualified Control.Exception as E
 import Control.Exception (throw)
 import System.Console.GetOpt
@@ -74,7 +73,7 @@ tryRun' errnum _ cmd []
 	| otherwise = return ()
 tryRun' errnum state cmd (a:as) = run >>= handle
 	where
-		run = IO.try $ Annex.run state $ do
+		run = tryIO $ Annex.run state $ do
 			Annex.Queue.flushWhenFull
 			a
 		handle (Left err) = showerr err >> cont False state
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 59af29edb1..469fad749e 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -81,7 +81,7 @@ performRemote key file backend numcopies remote = do
 			t <- fromRepo gitAnnexTmpDir
 			let tmp = t  "fsck" ++ show pid ++ "." ++ keyFile key
 			liftIO $ createDirectoryIfMissing True t
-			let cleanup = liftIO $ catch (removeFile tmp) (const $ return ())
+			let cleanup = liftIO $ catchIO (removeFile tmp) (const $ return ())
 			cleanup
 			cleanup `after` a tmp
 		getfile tmp = do
diff --git a/Common.hs b/Common.hs
index fb998214bc..cc6cf92527 100644
--- a/Common.hs
+++ b/Common.hs
@@ -21,6 +21,7 @@ import System.Posix.Process as X hiding (executeFile)
 import System.Exit as X
 
 import Utility.Misc as X
+import Utility.Exception as X
 import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 9b54d8c855..50c3b10b39 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -8,7 +8,6 @@
 module Remote.Bup (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import System.IO.Error
 import qualified Data.Map as M
 import System.Process
 
@@ -200,7 +199,7 @@ getBupUUID :: Git.Repo -> UUID -> Annex (UUID, Git.Repo)
 getBupUUID r u
 	| Git.repoIsUrl r = return (u, r)
 	| otherwise = liftIO $ do
-		ret <- try $ Git.Config.read r
+		ret <- tryIO $ Git.Config.read r
 		case ret of
 			Right r' -> return (toUUID $ Git.Config.get "annex.uuid" "" r', r')
 			Left _ -> return (NoUUID, r)
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index c5310c641a..c439c7caa2 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -7,8 +7,6 @@
 
 module Upgrade.V0 where
 
-import System.IO.Error (try)
-
 import Common.Annex
 import Annex.Content
 import qualified Upgrade.V1
@@ -47,7 +45,7 @@ getKeysPresent0 dir = do
 			return $ map fileKey0 files
 	where
 		present d = do
-			result <- try $
+			result <- tryIO $
 				getFileStatus $ dir ++ "/" ++ takeFileName d
 			case result of
 				Right s -> return $ isRegularFile s
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index add50fcf3a..ca2bff6617 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -7,7 +7,6 @@
 
 module Upgrade.V1 where
 
-import System.IO.Error (try)
 import System.Posix.Types
 import Data.Char
 
@@ -183,7 +182,7 @@ readLog1 file = catchDefaultIO (parseLog <$> readFileStrict file) []
 
 lookupFile1 :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile1 file = do
-	tl <- liftIO $ try getsymlink
+	tl <- liftIO $ tryIO getsymlink
 	case tl of
 		Left _ -> return Nothing
 		Right l -> makekey l
@@ -216,7 +215,7 @@ getKeyFilesPresent1' dir = do
 			liftIO $ filterM present files
 	where
 		present f = do
-			result <- try $ getFileStatus f
+			result <- tryIO $ getFileStatus f
 			case result of
 				Right s -> return $ isRegularFile s
 				Left _ -> return False
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index b5fedb9c7d..e7b7c442b2 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -16,11 +16,12 @@ import Control.Monad.IfElse
 
 import Utility.SafeCommand
 import Utility.TempFile
+import Utility.Exception
 
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}
 moveFile :: FilePath -> FilePath -> IO ()
-moveFile src dest = try (rename src dest) >>= onrename
+moveFile src dest = tryIO (rename src dest) >>= onrename
 	where
 		onrename (Right _) = return ()
 		onrename (Left e)
@@ -40,11 +41,10 @@ moveFile src dest = try (rename src dest) >>= onrename
 						Param src, Param tmp]
 					unless ok $ do
 						-- delete any partial
-						_ <- try $
-							removeFile tmp
+						_ <- tryIO $ removeFile tmp
 						rethrow
 		isdir f = do
-			r <- try (getFileStatus f)
+			r <- tryIO $ getFileStatus f
 			case r of
 				(Left _) -> return False
 				(Right s) -> return $ isDirectory s
diff --git a/Utility/Exception.hs b/Utility/Exception.hs
new file mode 100644
index 0000000000..7b6c9c999f
--- /dev/null
+++ b/Utility/Exception.hs
@@ -0,0 +1,39 @@
+{- Simple IO exception handling
+ -
+ - Copyright 2011-2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Exception where
+
+import Prelude hiding (catch)
+import Control.Exception
+import Control.Applicative
+
+{- Catches IO errors and returns a Bool -}
+catchBoolIO :: IO Bool -> IO Bool
+catchBoolIO a = catchDefaultIO a False
+
+{- Catches IO errors and returns a Maybe -}
+catchMaybeIO :: IO a -> IO (Maybe a)
+catchMaybeIO a = catchDefaultIO (Just <$> a) Nothing
+
+{- Catches IO errors and returns a default value. -}
+catchDefaultIO :: IO a -> a -> IO a
+catchDefaultIO a def = catchIO a (const $ return def)
+
+{- Catches IO errors and returns the error message. -}
+catchMsgIO :: IO a -> IO (Either String a)
+catchMsgIO a = dispatch <$> tryIO a
+	where
+		dispatch (Left e) = Left $ show e
+		dispatch (Right v) = Right v
+
+{- catch specialized for IO errors only -}
+catchIO :: IO a -> (IOException -> IO a) -> IO a
+catchIO = catch
+
+{- try specialized for IO errors only -}
+tryIO :: IO a -> IO (Either IOException a)
+tryIO = try
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index c9bfcb953a..3ac5ca5c0b 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -8,9 +8,7 @@
 module Utility.Misc where
 
 import System.IO
-import System.IO.Error (try)
 import Control.Monad
-import Control.Applicative
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
@@ -37,22 +35,3 @@ separate c l = unbreak $ break c l
 {- Breaks out the first line. -}
 firstLine :: String-> String
 firstLine = takeWhile (/= '\n')
-
-{- Catches IO errors and returns a Bool -}
-catchBoolIO :: IO Bool -> IO Bool
-catchBoolIO a = catchDefaultIO a False
-
-{- Catches IO errors and returns a Maybe -}
-catchMaybeIO :: IO a -> IO (Maybe a)
-catchMaybeIO a = catchDefaultIO (Just <$> a) Nothing
-
-{- Catches IO errors and returns a default value. -}
-catchDefaultIO :: IO a -> a -> IO a
-catchDefaultIO a def = catch a (const $ return def)
-
-{- Catches IO errors and returns the error message. -}
-catchMsgIO :: IO a -> IO (Either String a)
-catchMsgIO a = dispatch <$> try a
-	where
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs
index 469d52e8ce..4dcbf1cca4 100644
--- a/Utility/TempFile.hs
+++ b/Utility/TempFile.hs
@@ -12,7 +12,7 @@ import System.IO
 import System.Posix.Process hiding (executeFile)
 import System.Directory
 
-import Utility.Misc
+import Utility.Exception
 import Utility.Path
 
 {- Runs an action like writeFile, writing to a temp file first and

From 97db2f945a4d1874e711defc3a855bb9ecada6c3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Feb 2012 16:57:07 -0400
Subject: [PATCH 3066/8313] exception update in test too

---
 test.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/test.hs b/test.hs
index 7b25917a11..245dd6706a 100644
--- a/test.hs
+++ b/test.hs
@@ -11,10 +11,8 @@ import Test.QuickCheck
 
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
-import Control.Exception (bracket_, bracket, throw)
-import System.IO.Error
 import System.Posix.Env
-import qualified Control.Exception.Extensible as E
+import Control.Exception.Extensible
 import qualified Data.Map as M
 import System.IO.HVFS (SystemFS(..))
 import Text.JSON
@@ -695,7 +693,7 @@ test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $
 git_annex :: String -> [String] -> IO Bool
 git_annex command params = do
 	-- catch all errors, including normally fatal errors
-	r <- E.try (run)::IO (Either E.SomeException ())
+	r <- try (run)::IO (Either SomeException ())
 	case r of
 		Right _ -> return True
 		Left _ -> return False
@@ -761,7 +759,7 @@ indir dir a = do
 	-- any type of error and change back to cwd before
 	-- rethrowing.
 	r <- bracket_ (changeToTmpDir dir) (changeWorkingDirectory cwd)
-		(E.try (a)::IO (Either E.SomeException ()))
+		(try (a)::IO (Either SomeException ()))
 	case r of
 		Right () -> return ()
 		Left e -> throw e
@@ -832,14 +830,14 @@ checkunwritable f = do
 
 checkwritable :: FilePath -> Assertion
 checkwritable f = do
-	r <- try $ writeFile f $ content f
+	r <- tryIO $ writeFile f $ content f
 	case r of
 		Left _ -> assertFailure $ "unable to modify " ++ f
 		Right _ -> return ()
 
 checkdangling :: FilePath -> Assertion
 checkdangling f = do
-	r <- try $ readFile f
+	r <- tryIO $ readFile f
 	case r of
 		Left _ -> return () -- expected; dangling link
 		Right _ -> assertFailure $ f ++ " was not a dangling link as expected"

From 2385fe3c4cd1a2c33cc27c4e24b6325d5917c1f6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 01:59:53 -0400
Subject: [PATCH 3067/8313] add news item

---
 doc/news/Presentation_at_FOSDEM.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/news/Presentation_at_FOSDEM.mdwn

diff --git a/doc/news/Presentation_at_FOSDEM.mdwn b/doc/news/Presentation_at_FOSDEM.mdwn
new file mode 100644
index 0000000000..48daf2678d
--- /dev/null
+++ b/doc/news/Presentation_at_FOSDEM.mdwn
@@ -0,0 +1,4 @@
+git-annex will be briefly presented at FOSDEM, on Sunday February 4th at 15:40.
+[Details](http://fosdem.org/2012/schedule/event/gitannex).
+
+Thanks to Richard Hartmann for making this presentation.

From f1c7dc12127fcbad411c28df57e9ce194bd66509 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 12:24:00 -0400
Subject: [PATCH 3068/8313] fix touch and statfs to work on any files in any
 locale

Use withCAString rather than withCString.

XXX Actually, this only works in non-unicode locales when presented with
unicode characters. Help?
---
 Command/Add.hs     | 6 +++---
 Utility/StatFS.hsc | 4 +---
 Utility/Touch.hsc  | 4 ++--
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 944525ea5e..f437d160d1 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -84,9 +84,9 @@ cleanup file key hascontent = do
 			-- file it points to
 			-- XXX Currently broken on non-utf8 locales when
 			-- dealing with utf-8 filenames.
-			--liftIO $ do
-				--mtime <- modificationTime <$> getFileStatus file
-				--touch file (TimeSpec mtime) False
+			liftIO $ do
+				mtime <- modificationTime <$> getFileStatus file
+				touch file (TimeSpec mtime) False
 
 	force <- Annex.getState Annex.force
 	if force
diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc
index 937571dfa0..6b96274637 100644
--- a/Utility/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -50,8 +50,6 @@ module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 import Foreign
 import Foreign.C.Types
 import Foreign.C.String
-import Data.ByteString (useAsCString)
-import Data.ByteString.Char8 (pack)
 
 #if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
 # include 
@@ -105,7 +103,7 @@ getFileSystemStats path =
   return Nothing
 #else
   allocaBytes (#size struct statfs) $ \vfs ->
-  useAsCString (pack path) $ \cpath -> do
+  withCAString path $ \cpath -> do
     res <- c_statfs cpath vfs
     if res == -1 then return Nothing
       else do
diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index fd3320cd1d..41d3e17b8e 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -64,7 +64,7 @@ foreign import ccall "utimensat"
 
 touchBoth file atime mtime follow = 
 	allocaArray 2 $ \ptr ->
-	withCString file $ \f -> do
+	withCAString file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- c_utimensat at_fdcwd f ptr flags
 		when (r /= 0) $ throwErrno "touchBoth"
@@ -101,7 +101,7 @@ foreign import ccall "lutimes"
 
 touchBoth file atime mtime follow = 
 	allocaArray 2 $ \ptr ->
-	withCString file $ \f -> do
+	withCAString file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- syscall f ptr
 		if (r /= 0)

From 586be399523a9a0ae1ed39d34b84c2c78296b457 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 12:58:42 -0400
Subject: [PATCH 3069/8313] fix file encoding of HashObject

---
 Git/HashObject.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index f5e6d50cdf..ae498278f4 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -16,6 +16,7 @@ import Git.Command
 hashFiles :: [FilePath] -> Repo -> IO ([Sha], IO ())
 hashFiles paths repo = do
 	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
+	fileEncoding toh
 	_ <- forkProcess (feeder toh)
 	hClose toh
 	shas <- map Ref . lines <$> hGetContentsStrict fromh

From dc682e53a2971a4e1ca89a997dcb44406aa5be75 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 13:03:33 -0400
Subject: [PATCH 3070/8313] use fileEncoding for git-update-index input handle

---
 Git/UnionMerge.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 15bff60527..be8eb10d99 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -57,6 +57,7 @@ update_index repo ls = stream_update_index repo [(`mapM_` ls)]
 stream_update_index :: Repo -> [Streamer] -> IO ()
 stream_update_index repo as = do
 	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
+	fileEncoding h
 	forM_ as (stream h)
 	hClose h
 	forceSuccess p

From 56470ce3e5a417dd81c985711b7927db0ce9015e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 14:30:28 -0400
Subject: [PATCH 3071/8313] really fix foreign C functions filename encodings

GHC should probably export withFilePath.
---
 Utility/StatFS.hsc | 7 ++++++-
 Utility/Touch.hsc  | 9 +++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc
index 6b96274637..51a6bda1e3 100644
--- a/Utility/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -50,6 +50,11 @@ module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 import Foreign
 import Foreign.C.Types
 import Foreign.C.String
+import GHC.IO.Encoding (getFileSystemEncoding)
+import GHC.Foreign as GHC
+
+withFilePath :: FilePath -> (CString -> IO a) -> IO a
+withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f
 
 #if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
 # include 
@@ -103,7 +108,7 @@ getFileSystemStats path =
   return Nothing
 #else
   allocaBytes (#size struct statfs) $ \vfs ->
-  withCAString path $ \cpath -> do
+  withFilePath path $ \cpath -> do
     res <- c_statfs cpath vfs
     if res == -1 then return Nothing
       else do
diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index 41d3e17b8e..24ccd17a62 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -16,6 +16,11 @@ module Utility.Touch (
 import Foreign
 import Foreign.C
 import Control.Monad (when)
+import GHC.IO.Encoding (getFileSystemEncoding)
+import GHC.Foreign as GHC
+
+withFilePath :: FilePath -> (CString -> IO a) -> IO a
+withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f
 
 newtype TimeSpec = TimeSpec CTime
 
@@ -64,7 +69,7 @@ foreign import ccall "utimensat"
 
 touchBoth file atime mtime follow = 
 	allocaArray 2 $ \ptr ->
-	withCAString file $ \f -> do
+	withFilePath file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- c_utimensat at_fdcwd f ptr flags
 		when (r /= 0) $ throwErrno "touchBoth"
@@ -101,7 +106,7 @@ foreign import ccall "lutimes"
 
 touchBoth file atime mtime follow = 
 	allocaArray 2 $ \ptr ->
-	withCAString file $ \f -> do
+	withFilePath file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- syscall f ptr
 		if (r /= 0)

From 91fc975964d94503fa932256bc8684de13cd9e1e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 14:44:03 -0400
Subject: [PATCH 3072/8313] note 7.4 needed

---
 debian/control   | 2 +-
 doc/install.mdwn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/control b/debian/control
index c3ddad9322..5d5956de8d 100644
--- a/debian/control
+++ b/debian/control
@@ -3,7 +3,7 @@ Section: utils
 Priority: optional
 Build-Depends: 
 	debhelper (>= 9),
-	ghc,
+	ghc (>= 7.4),
 	libghc-missingh-dev,
 	libghc-hslogger-dev,
 	libghc-pcre-light-dev,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index b48914197f..8de24d40db 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -21,7 +21,7 @@ As a haskell package, git-annex can be installed using cabal. For example:
 To build and use git-annex, you will need:
 
 * Haskell stuff
-  * [The Haskell Platform](http://haskell.org/platform/)
+  * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer)
   * [MissingH](http://github.com/jgoerzen/missingh/wiki)
   * [pcre-light](http://hackage.haskell.org/package/pcre-light)
   * [utf8-string](http://hackage.haskell.org/package/utf8-string)

From 90ab17e15394892c6b0a17682d317085bab3e711 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 16:34:13 -0400
Subject: [PATCH 3073/8313] remove old comment

---
 Command/Add.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index f437d160d1..9410601b8b 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -82,8 +82,6 @@ cleanup file key hascontent = do
 	
 			-- touch the symlink to have the same mtime as the
 			-- file it points to
-			-- XXX Currently broken on non-utf8 locales when
-			-- dealing with utf-8 filenames.
 			liftIO $ do
 				mtime <- modificationTime <$> getFileStatus file
 				touch file (TimeSpec mtime) False

From 10876ca59e296c40f1e992c8521f67a66ec41a1c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 16:37:55 -0400
Subject: [PATCH 3074/8313] wording

---
 Messages.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Messages.hs b/Messages.hs
index a0bd20ca3c..982b9313cf 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -120,7 +120,7 @@ showRaw :: String -> Annex ()
 showRaw s = handle q $ putStrLn s
 
 {- This avoids ghc's output layer crashing on invalid encoded characters in
- - files when printing them out.
+ - filenames when printing them out.
  -}
 setupConsole :: IO ()
 setupConsole = do

From e066fa3884b7085278636b7fe042c84c4b6eeafa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 14:19:44 -0400
Subject: [PATCH 3075/8313] use "known" instead of "visible"

I think it's clearer, also it's the same length as "local" :)
---
 Command/Status.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index a1d4eea087..5facaab9be 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -66,8 +66,8 @@ slow_stats =
 	, bad_data_size
 	, local_annex_keys
 	, local_annex_size
-	, visible_annex_keys
-	, visible_annex_size
+	, known_annex_keys
+	, known_annex_size
 	, backend_usage
 	]
 
@@ -128,12 +128,12 @@ local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $ json show $
 	S.size <$> cachedKeysPresent
 
-visible_annex_size :: Stat
-visible_annex_size = stat "visible annex size" $ json id $
+known_annex_size :: Stat
+known_annex_size = stat "known annex size" $ json id $
 	keySizeSum <$> cachedKeysReferenced
 
-visible_annex_keys :: Stat
-visible_annex_keys = stat "visible annex keys" $ json show $
+known_annex_keys :: Stat
+known_annex_keys = stat "known annex keys" $ json show $
 	S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat

From 5e59440533eb36074d4cc6500b4f51e7722af9a0 Mon Sep 17 00:00:00 2001
From: "http://jefferai.org/" 
Date: Tue, 7 Feb 2012 00:18:24 +0000
Subject: [PATCH 3076/8313]

---
 doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn

diff --git a/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn b/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn
new file mode 100644
index 0000000000..8dc075474e
--- /dev/null
+++ b/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn
@@ -0,0 +1 @@
+It'd be nice to be able to run "git annex version" -- and maybe some other commands, like "git annex" itself for the help text, without having to be inside a git repo. Right now it requires you to be in a git repo even if it's not a git-annex repo.

From e2bcf1717e4cf7f49dba82b27fc99ea70dc6a885 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 20:34:07 -0400
Subject: [PATCH 3077/8313] already done

---
 ...w_version_without_having_to_be_in_a_git_repo.mdwn | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn b/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn
index 8dc075474e..98b9ced228 100644
--- a/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn
+++ b/doc/bugs/show_version_without_having_to_be_in_a_git_repo.mdwn
@@ -1 +1,11 @@
-It'd be nice to be able to run "git annex version" -- and maybe some other commands, like "git annex" itself for the help text, without having to be inside a git repo. Right now it requires you to be in a git repo even if it's not a git-annex repo.
+It'd be nice to be able to run "git annex version" -- and maybe some other
+commands, like "git annex" itself for the help text, without having to be
+inside a git repo. Right now it requires you to be in a git repo even if
+it's not a git-annex repo.
+
+> You need a newer verison of git-annex. --[[Joey]] 
+
+	joey@gnu:/>git annex version
+	git-annex version: 3.20120124
+
+[[done]]

From edcd3123d5292e39071a6bd9f8efa86d71e2da2c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 20:18:20 -0400
Subject: [PATCH 3078/8313] list of git branches

---
 doc/download.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index e1257d2618..bfde849f80 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -15,3 +15,20 @@ From time to time, releases of git-annex are uploaded
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
+
+## git branches
+
+The git repository has some branches:
+
+* `debian-stable` contains the latest backport of git-annex to Debian
+  stable.
+* `no-s3` disables the S3 special remote, for systems that lack the
+ necessary haskell library.
+* `old-monad-control` is for systems that don't have a newer monad-control
+  library.
+* `tweak-fetch` adds support for the git tweak-fetch hook, which has
+  been proposed and implemented but not yet accepted into git.
+* `ghc7.4` is for use this that version of ghc.
+* `setup` contains configuration for this website
+* `pristine-tar` contains [pristine-tar](http://kitenet.net/~joey/code/pristine-tar)
+  data to create tarballs of any past git-annex release.

From 5a82c0dee77a2302fb7ddd0026c6d4e311eb1a07 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 11:52:51 -0400
Subject: [PATCH 3079/8313] add a tip about using git's assume-unchanged
 feature to optimize large trees

---
 doc/tips/assume-unstaged.mdwn | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/tips/assume-unstaged.mdwn

diff --git a/doc/tips/assume-unstaged.mdwn b/doc/tips/assume-unstaged.mdwn
new file mode 100644
index 0000000000..ef74d9bd40
--- /dev/null
+++ b/doc/tips/assume-unstaged.mdwn
@@ -0,0 +1,31 @@
+[[!meta title="using assume-unstages to speed up git with large trees of annexed files"]]
+
+Git update-index's assume-unstaged feature can be used to speed
+up `git status` and stuff by not statting the whole tree looking for changed
+files.
+
+This feature works quite well with git-annex. Especially because git
+annex's files are immutable, so arn't going to change out from under it,
+this is a nice fit. If you have a very large tree and `git status` is
+annoyingly slow, you can turn it on:
+
+	git config core.ignoreStat true
+
+When git mv and git rm are used, those changes *do* get noticed, even
+on assume-unchanged files. When new files are added, eg by `git annex add`,
+they are also noticed.
+
+There are two gotchas. Both occur because `git add` does not stage
+assume-unchanged files.
+
+1. When an annexed file is moved to a different directory, it updates
+   the symlink, and runs `git add` on it. So the file will move,
+   but the changed symlink will not be noticed by git and it will commit a
+   dangling symlink.
+2. When using `git annex migrate`, it changes the symlink and `git adds`
+   it. Again this won't be committed.
+
+These can be worked around by running `git update-index --really-refresh`
+after performing such operations. I hope that `git add` will be changed
+to stage changes to assume-unchanged files, which would remove this
+only complication. --[[Joey]] 

From a84d50a1edc268d4dbdaebeb4d23c76c48d9a506 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Feb 2012 16:57:07 -0400
Subject: [PATCH 3080/8313] exception update in test too

---
 test.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/test.hs b/test.hs
index 7b25917a11..245dd6706a 100644
--- a/test.hs
+++ b/test.hs
@@ -11,10 +11,8 @@ import Test.QuickCheck
 
 import System.Posix.Directory (changeWorkingDirectory)
 import System.Posix.Files
-import Control.Exception (bracket_, bracket, throw)
-import System.IO.Error
 import System.Posix.Env
-import qualified Control.Exception.Extensible as E
+import Control.Exception.Extensible
 import qualified Data.Map as M
 import System.IO.HVFS (SystemFS(..))
 import Text.JSON
@@ -695,7 +693,7 @@ test_crypto = "git-annex crypto" ~: intmpclonerepo $ when Build.SysConfig.gpg $
 git_annex :: String -> [String] -> IO Bool
 git_annex command params = do
 	-- catch all errors, including normally fatal errors
-	r <- E.try (run)::IO (Either E.SomeException ())
+	r <- try (run)::IO (Either SomeException ())
 	case r of
 		Right _ -> return True
 		Left _ -> return False
@@ -761,7 +759,7 @@ indir dir a = do
 	-- any type of error and change back to cwd before
 	-- rethrowing.
 	r <- bracket_ (changeToTmpDir dir) (changeWorkingDirectory cwd)
-		(E.try (a)::IO (Either E.SomeException ()))
+		(try (a)::IO (Either SomeException ()))
 	case r of
 		Right () -> return ()
 		Left e -> throw e
@@ -832,14 +830,14 @@ checkunwritable f = do
 
 checkwritable :: FilePath -> Assertion
 checkwritable f = do
-	r <- try $ writeFile f $ content f
+	r <- tryIO $ writeFile f $ content f
 	case r of
 		Left _ -> assertFailure $ "unable to modify " ++ f
 		Right _ -> return ()
 
 checkdangling :: FilePath -> Assertion
 checkdangling f = do
-	r <- try $ readFile f
+	r <- tryIO $ readFile f
 	case r of
 		Left _ -> return () -- expected; dangling link
 		Right _ -> assertFailure $ f ++ " was not a dangling link as expected"

From a6c4b107716c61a48fbf1b0431abbcc330c7fd44 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 01:59:53 -0400
Subject: [PATCH 3081/8313] add news item

---
 doc/news/Presentation_at_FOSDEM.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/news/Presentation_at_FOSDEM.mdwn

diff --git a/doc/news/Presentation_at_FOSDEM.mdwn b/doc/news/Presentation_at_FOSDEM.mdwn
new file mode 100644
index 0000000000..48daf2678d
--- /dev/null
+++ b/doc/news/Presentation_at_FOSDEM.mdwn
@@ -0,0 +1,4 @@
+git-annex will be briefly presented at FOSDEM, on Sunday February 4th at 15:40.
+[Details](http://fosdem.org/2012/schedule/event/gitannex).
+
+Thanks to Richard Hartmann for making this presentation.

From a81297065dd2f1a4fb59abcddec852414174200e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 14:19:44 -0400
Subject: [PATCH 3082/8313] use "known" instead of "visible"

I think it's clearer, also it's the same length as "local" :)
---
 Command/Status.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index a1d4eea087..5facaab9be 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -66,8 +66,8 @@ slow_stats =
 	, bad_data_size
 	, local_annex_keys
 	, local_annex_size
-	, visible_annex_keys
-	, visible_annex_size
+	, known_annex_keys
+	, known_annex_size
 	, backend_usage
 	]
 
@@ -128,12 +128,12 @@ local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $ json show $
 	S.size <$> cachedKeysPresent
 
-visible_annex_size :: Stat
-visible_annex_size = stat "visible annex size" $ json id $
+known_annex_size :: Stat
+known_annex_size = stat "known annex size" $ json id $
 	keySizeSum <$> cachedKeysReferenced
 
-visible_annex_keys :: Stat
-visible_annex_keys = stat "visible annex keys" $ json show $
+known_annex_keys :: Stat
+known_annex_keys = stat "known annex keys" $ json show $
 	S.size <$> cachedKeysReferenced
 
 tmp_size :: Stat

From 3f4f96228e85fc6762490b99b76026826917d95b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 14:23:10 -0400
Subject: [PATCH 3083/8313] changelog

---
 debian/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 96234d9272..30b7090bea 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     git-annex's existing ability to recover in this situation. This is
     used by git-annex-shell and other places where changes are made to
     a remote's location log.
+  * Modifications to support ghc 7.4's handling of filenames.
+    This version can only be built with ghc 7.4.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 679e2567d09667ef4f90902db8b493e01844e246 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 21:37:44 -0400
Subject: [PATCH 3084/8313] add a bug template

---
 doc/bugs.mdwn                  |  2 ++
 doc/templates/bugtemplate.mdwn | 12 ++++++++++++
 2 files changed, 14 insertions(+)
 create mode 100644 doc/templates/bugtemplate.mdwn

diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn
index 2786e5bf74..b0837eb10b 100644
--- a/doc/bugs.mdwn
+++ b/doc/bugs.mdwn
@@ -2,3 +2,5 @@ This is git-annex's bug list. Link bugs to [[bugs/done]] when done.
 
 [[!inline pages="./bugs/* and !./bugs/done and !link(done) 
 and !*/Discussion" actions=yes postform=yes show=0 archive=yes]]
+
+[[!edittemplate template=templates/bugtemplate match="bugs/*" silent=yes]]
diff --git a/doc/templates/bugtemplate.mdwn b/doc/templates/bugtemplate.mdwn
new file mode 100644
index 0000000000..2d35c8f6fb
--- /dev/null
+++ b/doc/templates/bugtemplate.mdwn
@@ -0,0 +1,12 @@
+What steps will reproduce the problem?
+
+
+What is the expected output? What do you see instead?
+
+
+What version of git-annex are you using? On what operating system?
+
+
+Please provide any additional information below.
+
+

From 0ad5d8168f59561827dfe42020ef952d6e0cd309 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 21:37:44 -0400
Subject: [PATCH 3085/8313] add a bug template

---
 doc/bugs.mdwn                  |  2 ++
 doc/templates/bugtemplate.mdwn | 12 ++++++++++++
 2 files changed, 14 insertions(+)
 create mode 100644 doc/templates/bugtemplate.mdwn

diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn
index 2786e5bf74..b0837eb10b 100644
--- a/doc/bugs.mdwn
+++ b/doc/bugs.mdwn
@@ -2,3 +2,5 @@ This is git-annex's bug list. Link bugs to [[bugs/done]] when done.
 
 [[!inline pages="./bugs/* and !./bugs/done and !link(done) 
 and !*/Discussion" actions=yes postform=yes show=0 archive=yes]]
+
+[[!edittemplate template=templates/bugtemplate match="bugs/*" silent=yes]]
diff --git a/doc/templates/bugtemplate.mdwn b/doc/templates/bugtemplate.mdwn
new file mode 100644
index 0000000000..2d35c8f6fb
--- /dev/null
+++ b/doc/templates/bugtemplate.mdwn
@@ -0,0 +1,12 @@
+What steps will reproduce the problem?
+
+
+What is the expected output? What do you see instead?
+
+
+What version of git-annex are you using? On what operating system?
+
+
+Please provide any additional information below.
+
+

From b9b72d22a9036fddbb34f70b85136f00cfe94b10 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 7 Feb 2012 01:40:14 -0400
Subject: [PATCH 3086/8313] refactor

Wow, triple monadic lift!
---
 Remote/S3.hs | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 1d23b7d6f0..2ef96dbdaf 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -272,26 +272,29 @@ s3Connection c = do
 {- S3 creds come from the environment if set. 
  - Otherwise, might be stored encrypted in the remote's config. -}
 s3GetCreds :: RemoteConfig -> Annex (Maybe (String, String))
-s3GetCreds c = do
-	ak <- getEnvKey s3AccessKey
-	sk <- getEnvKey s3SecretKey
-	if null ak || null sk
-		then do
+s3GetCreds c = maybe fromconfig (return . Just) =<< liftIO getenv
+	where
+		getenv = liftM2 (,)
+			<$> get s3AccessKey
+			<*> get s3SecretKey
+			where
+				get = catchMaybeIO . getEnv
+		setenv (ak, sk) = do
+			setEnv s3AccessKey ak True
+			setEnv s3SecretKey sk True
+		fromconfig = do
 			mcipher <- remoteCipher c
 			case (M.lookup "s3creds" c, mcipher) of
-				(Just encrypted, Just cipher) -> do
-					s <- liftIO $ withDecryptedContent cipher
-						(return $ L.pack $ fromB64 encrypted)
-						(return . L.unpack)
-					let [ak', sk', _rest] = lines s
-					liftIO $ do
-						setEnv s3AccessKey ak True
-						setEnv s3SecretKey sk True
-					return $ Just (ak', sk')
+				(Just s3creds, Just cipher) ->
+					liftIO $ decrypt s3creds cipher
 				_ -> return Nothing
-		else return $ Just (ak, sk)
-	where
-		getEnvKey s = liftIO $ catchDefaultIO (getEnv s) ""
+		decrypt s3creds cipher = do
+			[ak, sk, _rest] <- lines <$>
+				withDecryptedContent cipher
+					(return $ L.pack $ fromB64 s3creds)
+					(return . L.unpack)
+			setenv (ak, sk)
+			return $ Just (ak, sk)
 
 {- Stores S3 creds encrypted in the remote's config if possible. -}
 s3SetCreds :: RemoteConfig -> Annex RemoteConfig

From 1338922f9e327937c5cb07c3a6add1cf84c882db Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 6 Feb 2012 11:52:51 -0400
Subject: [PATCH 3087/8313] add a tip about using git's assume-unchanged
 feature to optimize large trees

---
 doc/tips/assume-unstaged.mdwn | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/tips/assume-unstaged.mdwn

diff --git a/doc/tips/assume-unstaged.mdwn b/doc/tips/assume-unstaged.mdwn
new file mode 100644
index 0000000000..ef74d9bd40
--- /dev/null
+++ b/doc/tips/assume-unstaged.mdwn
@@ -0,0 +1,31 @@
+[[!meta title="using assume-unstages to speed up git with large trees of annexed files"]]
+
+Git update-index's assume-unstaged feature can be used to speed
+up `git status` and stuff by not statting the whole tree looking for changed
+files.
+
+This feature works quite well with git-annex. Especially because git
+annex's files are immutable, so arn't going to change out from under it,
+this is a nice fit. If you have a very large tree and `git status` is
+annoyingly slow, you can turn it on:
+
+	git config core.ignoreStat true
+
+When git mv and git rm are used, those changes *do* get noticed, even
+on assume-unchanged files. When new files are added, eg by `git annex add`,
+they are also noticed.
+
+There are two gotchas. Both occur because `git add` does not stage
+assume-unchanged files.
+
+1. When an annexed file is moved to a different directory, it updates
+   the symlink, and runs `git add` on it. So the file will move,
+   but the changed symlink will not be noticed by git and it will commit a
+   dangling symlink.
+2. When using `git annex migrate`, it changes the symlink and `git adds`
+   it. Again this won't be committed.
+
+These can be worked around by running `git update-index --really-refresh`
+after performing such operations. I hope that `git add` will be changed
+to stage changes to assume-unchanged files, which would remove this
+only complication. --[[Joey]] 

From b8140f3fc2586603da9b92bad2346ecf4cba99c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 4 Feb 2012 20:18:20 -0400
Subject: [PATCH 3088/8313] list of git branches

---
 doc/download.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index e1257d2618..bfde849f80 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -15,3 +15,20 @@ From time to time, releases of git-annex are uploaded
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
+
+## git branches
+
+The git repository has some branches:
+
+* `debian-stable` contains the latest backport of git-annex to Debian
+  stable.
+* `no-s3` disables the S3 special remote, for systems that lack the
+ necessary haskell library.
+* `old-monad-control` is for systems that don't have a newer monad-control
+  library.
+* `tweak-fetch` adds support for the git tweak-fetch hook, which has
+  been proposed and implemented but not yet accepted into git.
+* `ghc7.4` is for use this that version of ghc.
+* `setup` contains configuration for this website
+* `pristine-tar` contains [pristine-tar](http://kitenet.net/~joey/code/pristine-tar)
+  data to create tarballs of any past git-annex release.

From 2dcce5a8bb1fcc2be5f844b782158406dc419221 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 7 Feb 2012 14:15:37 -0400
Subject: [PATCH 3089/8313] merged ghc 7.4 support into master

---
 doc/bugs/problems_with_utf8_names.mdwn | 5 ++---
 doc/download.mdwn                      | 3 ++-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/problems_with_utf8_names.mdwn b/doc/bugs/problems_with_utf8_names.mdwn
index fbdca41cd1..aeeb16be65 100644
--- a/doc/bugs/problems_with_utf8_names.mdwn
+++ b/doc/bugs/problems_with_utf8_names.mdwn
@@ -3,10 +3,9 @@ This bug is reopened to track some new UTF-8 filename issues caused by GHC
 encoding no longer works. Even unicode filenames fail to work when
 git-annex is built with 7.4. --[[Joey]]
 
-I now have a `ghc7.4` branch in git that seems to solve this, 
+This bug is now fixed in current master. Once again, git-annex will work
 for all filename encodings, and all system encodings. It will
-only build with the new GHC. If you have this problem, give it a try!
---[[Joey]] 
+only build with the new GHC. [[done]] --[[Joey]] 
 
 ----
 
diff --git a/doc/download.mdwn b/doc/download.mdwn
index bfde849f80..120e0a517d 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -28,7 +28,8 @@ The git repository has some branches:
   library.
 * `tweak-fetch` adds support for the git tweak-fetch hook, which has
   been proposed and implemented but not yet accepted into git.
-* `ghc7.4` is for use this that version of ghc.
+* `ghc7.0` supports versions of ghc older than 7.4, which
+  had a major change to filename encoding.
 * `setup` contains configuration for this website
 * `pristine-tar` contains [pristine-tar](http://kitenet.net/~joey/code/pristine-tar)
   data to create tarballs of any past git-annex release.

From 995bf51e10161c26ac6b0716080d3a0a75657314 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 7 Feb 2012 16:52:39 -0400
Subject: [PATCH 3090/8313] correction

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 30b7090bea..7165092a65 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,7 +8,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     used by git-annex-shell and other places where changes are made to
     a remote's location log.
   * Modifications to support ghc 7.4's handling of filenames.
-    This version can only be built with ghc 7.4.
+    This version can only be built with ghc 7.4 or newer.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 57a747d0819d587d8f7fb301c3c6c589c80c556f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 8 Feb 2012 11:41:15 -0400
Subject: [PATCH 3091/8313] S3: Fix irrefutable pattern failure when accessing
 encrypted S3 credentials.

---
 Remote/S3.hs     | 9 ++++++---
 debian/changelog | 2 ++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 2ef96dbdaf..c9527ba67a 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -289,12 +289,15 @@ s3GetCreds c = maybe fromconfig (return . Just) =<< liftIO getenv
 					liftIO $ decrypt s3creds cipher
 				_ -> return Nothing
 		decrypt s3creds cipher = do
-			[ak, sk, _rest] <- lines <$>
+			creds <- lines <$>
 				withDecryptedContent cipher
 					(return $ L.pack $ fromB64 s3creds)
 					(return . L.unpack)
-			setenv (ak, sk)
-			return $ Just (ak, sk)
+			case creds of
+				[ak, sk] -> do
+					setenv (ak, sk)
+					return $ Just (ak, sk)
+				_ -> do error "bad s3creds"
 
 {- Stores S3 creds encrypted in the remote's config if possible. -}
 s3SetCreds :: RemoteConfig -> Annex RemoteConfig
diff --git a/debian/changelog b/debian/changelog
index 7165092a65..ad7121da24 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     a remote's location log.
   * Modifications to support ghc 7.4's handling of filenames.
     This version can only be built with ghc 7.4 or newer.
+  * S3: Fix irrefutable pattern failure when accessing encrypted S3
+    credentials.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From ef013506cb8c82d547160195b66572b5774db14f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 8 Feb 2012 15:35:18 -0400
Subject: [PATCH 3092/8313] addurl: Added a --file option

Can be used to specify what file the url is added to. This can be used to
override the default filename that is used when adding an url, which is
based on the url. Or, when the file already exists, the url is recorded as
another location of the file.
---
 Command/AddUrl.hs  | 37 +++++++++++++++++++++++--------------
 Usage.hs           |  2 ++
 debian/changelog   |  5 +++++
 doc/git-annex.mdwn |  9 +++++++--
 4 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 46584f0d81..600a6169d8 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -17,26 +17,39 @@ import qualified Annex
 import qualified Backend.URL
 import Annex.Content
 import Logs.Web
+import qualified Option
 
 def :: [Command]
-def = [command "addurl" (paramRepeating paramUrl) seek "add urls to annex"]
+def = [withOptions [fileOption] $
+	command "addurl" (paramRepeating paramUrl) seek "add urls to annex"]
+
+fileOption :: Option
+fileOption = Option.field [] "file" paramFile "specify what file the url is added to"
 
 seek :: [CommandSeek]
-seek = [withStrings start]
+seek = [withField fileOption return $ \f ->
+	withStrings $ start f]
 
-start :: String -> CommandStart
-start s = notBareRepo $ go $ parseURI s
+start :: Maybe FilePath -> String -> CommandStart
+start optfile s = notBareRepo $ go $ parseURI s
 	where
 		go Nothing = error $ "bad url " ++ s
 		go (Just url) = do
-			file <- liftIO $ url2file url
+			let file = fromMaybe (url2file url) optfile
 			showStart "addurl" file
 			next $ perform s file
 
 perform :: String -> FilePath -> CommandPerform
-perform url file = do
-	fast <- Annex.getState Annex.fast
-	if fast then nodownload url file else download url file
+perform url file = ifAnnexed file addurl geturl
+	where
+		geturl = do
+			whenM (liftIO $ doesFileExist file) $
+				error $ "already have this url in " ++ file
+			fast <- Annex.getState Annex.fast
+			if fast then nodownload url file else download url file
+		addurl (key, _backend) = do
+			setUrlPresent key url
+			next $ return True
 
 download :: String -> FilePath -> CommandPerform
 download url file = do
@@ -60,12 +73,8 @@ nodownload url file = do
 	setUrlPresent key url
 	next $ Command.Add.cleanup file key False
 
-url2file :: URI -> IO FilePath
-url2file url = do
-	whenM (doesFileExist file) $
-		error $ "already have this url in " ++ file
-	return file
+url2file :: URI -> FilePath
+url2file url = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
 	where
-		file = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
 		escape = replace "/" "_" . replace "?" "_"
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url
diff --git a/Usage.hs b/Usage.hs
index 34c344b183..a33f6f311b 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -76,6 +76,8 @@ paramDate :: String
 paramDate = "DATE"
 paramFormat :: String
 paramFormat = "FORMAT"
+paramFile :: String
+paramFile = "FILE"
 paramKeyValue :: String
 paramKeyValue = "K=V"
 paramNothing :: String
diff --git a/debian/changelog b/debian/changelog
index ad7121da24..2f9d79939d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -11,6 +11,11 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     This version can only be built with ghc 7.4 or newer.
   * S3: Fix irrefutable pattern failure when accessing encrypted S3
     credentials.
+  * addurl: Added a --file option, which can be used to specify what
+    file the url is added to. This can be used to override the default
+    filename that is used when adding an url, which is based on the url.
+    Or, when the file already exists, the url is recorded as another
+    location of the file.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 148b6336de..9232bf0208 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -145,9 +145,14 @@ subdirectories).
 
 * addurl [url ...]
 
-  Downloads each url to a file, which is added to the annex.
+  Downloads each url to its own file, which is added to the annex.
 
-  To avoid immediately downloading the url, specify --fast
+  To avoid immediately downloading the url, specify --fast.
+
+  To specify what file the url is added to, specify --file. This changes
+  the behavior; now all the specified urls are recorded as alternate
+  locations from which the file can be downloaded. In this mode, addurl
+  can be used both to add new files, or to add urls to existing files.
 
 # REPOSITORY SETUP COMMANDS
 

From ac9745465954d77d7215e4d7411a3c218203643d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 8 Feb 2012 15:49:42 -0400
Subject: [PATCH 3093/8313] improve error message

---
 Command/AddUrl.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 600a6169d8..2f157c7fdd 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -44,7 +44,7 @@ perform url file = ifAnnexed file addurl geturl
 	where
 		geturl = do
 			whenM (liftIO $ doesFileExist file) $
-				error $ "already have this url in " ++ file
+				error $ "not overwriting existing " ++ file
 			fast <- Annex.getState Annex.fast
 			if fast then nodownload url file else download url file
 		addurl (key, _backend) = do

From 1c0bd81ba6aa6bd081042c81fcb6dca21ece0eec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 9 Feb 2012 14:19:58 -0400
Subject: [PATCH 3094/8313] addurl: Normalize badly encoded urls.

---
 Command/AddUrl.hs | 7 ++++---
 debian/changelog  | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 2f157c7fdd..496b9f2e8b 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -31,10 +31,11 @@ seek = [withField fileOption return $ \f ->
 	withStrings $ start f]
 
 start :: Maybe FilePath -> String -> CommandStart
-start optfile s = notBareRepo $ go $ parseURI s
+start optfile s = notBareRepo $ go $ fromMaybe bad $ parseURI s
 	where
-		go Nothing = error $ "bad url " ++ s
-		go (Just url) = do
+		bad = fromMaybe (error $ "bad url " ++ s) $
+			parseURI $ escapeURIString isUnescapedInURI s
+		go url = do
 			let file = fromMaybe (url2file url) optfile
 			showStart "addurl" file
 			next $ perform s file
diff --git a/debian/changelog b/debian/changelog
index 2f9d79939d..ad1fe1945a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     filename that is used when adding an url, which is based on the url.
     Or, when the file already exists, the url is recorded as another
     location of the file.
+  * addurl: Normalize badly encoded urls.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From df3be7308093f33fa96c7d0c8d4f03d0499b7e17 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Thu, 9 Feb 2012 18:31:03 +0000
Subject: [PATCH 3095/8313] How to expire old versions of files that have been
 edited?

---
 ..._old_versions_of_files_that_have_been_edited__63__.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn

diff --git a/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn
new file mode 100644
index 0000000000..f06135c24e
--- /dev/null
+++ b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__.mdwn
@@ -0,0 +1,7 @@
+My annex contains several large files that I have unlocked, edited, and committed again, i.e. the annex contains the version history of those files. However, I don't want the history -- keeping the latest version is good enough for me. Running `git annex unused` won't detect those old versions, though, because they aren't unused as old Git revisions still refer to them. So I wonder:
+
+1. What is the best way to get rid of the old versions of files in the annex?
+
+2. What is the best way to detect old versions of files in the annex?
+
+I guess, I could run `git rebase -i` to squash commits to those files into one commit, thereby getting rid of the references to the old copies, but that approach feels awkward and error prone. Is anyone aware of a better way?

From 4b4b887d8dcc997d0d93835f37ab48276d037da6 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Thu, 9 Feb 2012 18:53:04 +0000
Subject: [PATCH 3096/8313] Added a comment

---
 .../comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment

diff --git a/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment
new file mode 100644
index 0000000000..ee4fe2e6ce
--- /dev/null
+++ b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_1_dccf4dc4483d08e5e2936b2cadeafeaf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.179.173"
+ subject="comment 1"
+ date="2012-02-09T18:53:00Z"
+ content="""
+Sorry for commmenting on my own question ... I think I just figured out that `git annex unused` *does* in fact do what I want. When I tried it, it just didn't show the obsolete versions of the files I edited because I hadn't yet synchronized all repositories, so that was why the obsolete versions were still considered used.
+"""]]

From 4ccc01922fa9cbddeb56e2bfa40ddfd76fafee4b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 9 Feb 2012 19:42:28 +0000
Subject: [PATCH 3097/8313] Added a comment

---
 .../comment_2_5710294c1c8652c12b6df2233255a45e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment

diff --git a/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment
new file mode 100644
index 0000000000..576093a87f
--- /dev/null
+++ b/doc/forum/How_to_expire_old_versions_of_files_that_have_been_edited__63__/comment_2_5710294c1c8652c12b6df2233255a45e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-02-09T19:42:28Z"
+ content="""
+Yes, contents are still considered used while tags or refs refer to them. Including remote tracking branches like `remotes/origin/master`
+"""]]

From e4d09235446e13134e28aa4519c54ec14061d126 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 9 Feb 2012 17:35:36 -0400
Subject: [PATCH 3098/8313] wording

---
 Git/UnionMerge.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index be8eb10d99..90bbf5c4cc 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -85,13 +85,14 @@ merge_trees (Ref x) (Ref y) h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
 
 {- For merging a single tree into the index. -}
 merge_tree_index :: Ref -> CatFileHandle -> Repo -> Streamer
-merge_tree_index (Ref x) h = calc_merge h $ "diff-index":diff_opts ++ ["--cached", x]
+merge_tree_index (Ref x) h = calc_merge h $
+	"diff-index" : diff_opts ++ ["--cached", x]
 
 diff_opts :: [String]
 diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
 {- Calculates how to perform a merge, using git to get a raw diff,
- - and returning a list suitable for update_index. -}
+ - and generating update-index input. -}
 calc_merge :: CatFileHandle -> [String] -> Repo -> Streamer
 calc_merge ch differ repo streamer = gendiff >>= go
 	where
@@ -102,7 +103,7 @@ calc_merge ch differ repo streamer = gendiff >>= go
 		go (_:[]) = error "calc_merge parse error"
 
 {- Given an info line from a git raw diff, and the filename, generates
- - a line suitable for update_index that union merges the two sides of the
+ - a line suitable for update-index that union merges the two sides of the
  - diff. -}
 mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of

From d55f3c07167498aea8a41631f48fa7b4d5c7cb5e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 9 Feb 2012 21:49:46 -0400
Subject: [PATCH 3099/8313] Fix teardown of stale cached ssh connections.

---
 Annex/Ssh.hs     | 5 +++--
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index d6f36e8689..184eb92caa 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -78,12 +78,13 @@ sshCleanup = do
 				Right _ -> stopssh socketfile
 			liftIO $ closeFd fd
 		stopssh socketfile = do
-			(_, params) <- sshInfo $ socket2hostport socketfile
+			let (host, port) = socket2hostport socketfile
+			(_, params) <- sshInfo (host, port)
 			_ <- liftIO $ do
 				-- "ssh -O stop" is noisy on stderr even with -q
 				let cmd = unwords $ toCommand $
 					[ Params "-O stop"
-					] ++ params
+					] ++ params ++ [Param host]
 				_ <- boolSystem "sh"
 					[ Param "-c"
 					, Param $ "ssh " ++ cmd ++ " >/dev/null 2>/dev/null"
diff --git a/debian/changelog b/debian/changelog
index ad1fe1945a..fdc909e3e1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     Or, when the file already exists, the url is recorded as another
     location of the file.
   * addurl: Normalize badly encoded urls.
+  * Fix teardown of stale cached ssh connections.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 9030f684521ce8db3e9cd6a4e2a10f4edce7bfee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Feb 2012 19:17:41 -0400
Subject: [PATCH 3100/8313] When checking that an url has a key, verify that
 the Content-Length, if available, matches the size of the key.

If there's no Content-Length, or the key has no size, this check is not
done, but it should happen most of the time, and protect against web
content that has changed.
---
 Remote/Git.hs    |  4 +++-
 Remote/Web.hs    |  9 +++++----
 Utility/Url.hs   | 24 +++++++++++++++++++-----
 debian/changelog |  2 ++
 git-annex.cabal  |  2 +-
 5 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 829ad1ccba..3905247755 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -28,6 +28,7 @@ import qualified Utility.Url as Url
 import Utility.TempFile
 import Config
 import Init
+import Types.Key
 
 remote :: RemoteType
 remote = RemoteType {
@@ -143,7 +144,8 @@ inAnnex r key
 			where
 				go e [] = return $ Left e
 				go _ (u:us) = do
-					res <- catchMsgIO $ Url.exists u
+					res <- catchMsgIO $
+						Url.check u (keySize key)
 					case res of
 						Left e -> go e us
 						v -> return v
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 49c3f43f3a..6bd04d4b15 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -15,6 +15,7 @@ import Annex.Content
 import Config
 import Logs.Web
 import qualified Utility.Url as Url
+import Types.Key
 
 remote :: RemoteType
 remote = RemoteType {
@@ -77,8 +78,8 @@ checkKey key = do
 	us <- getUrls key
 	if null us
 		then return $ Right False
-		else return . Right =<< checkKey' us
-checkKey' :: [URLString] -> Annex Bool
-checkKey' us = untilTrue us $ \u -> do
+		else return . Right =<< checkKey' key us
+checkKey' :: Key -> [URLString] -> Annex Bool
+checkKey' key us = untilTrue us $ \u -> do
 	showAction $ "checking " ++ u
-	liftIO $ Url.exists u
+	liftIO $ Url.check u (keySize key)
diff --git a/Utility/Url.hs b/Utility/Url.hs
index e10b8a92a4..efd6ad16dd 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -7,6 +7,7 @@
 
 module Utility.Url (
 	URLString,
+	check,
 	exists,
 	canDownload,
 	download,
@@ -14,6 +15,7 @@ module Utility.Url (
 ) where
 
 import Control.Applicative
+import Control.Monad
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
@@ -23,16 +25,28 @@ import Utility.Path
 
 type URLString = String
 
-{- Checks that an url exists and could be successfully downloaded. -}
-exists :: URLString -> IO Bool
+{- Checks that an url exists and could be successfully downloaded,
+ - also checking that its size, if available, matches a specified size. -}
+check :: URLString -> Maybe Integer -> IO Bool
+check url expected_size = handle <$> exists url
+	where
+		handle (False, _) = False
+		handle (True, Nothing) = True
+		handle (True, s) = expected_size == s
+
+{- Checks that an url exists and could be successfully downloaded,
+ - also returning its size if available. -}
+exists :: URLString -> IO (Bool, Maybe Integer)
 exists url =
 	case parseURI url of
-		Nothing -> return False
+		Nothing -> return (False, Nothing)
 		Just u -> do
 			r <- request u HEAD
 			case rspCode r of
-				(2,_,_) -> return True
-				_ -> return False
+				(2,_,_) -> return (True, size r)
+				_ -> return (False, Nothing)
+	where
+		size = liftM read . lookupHeader HdrContentLength . rspHeaders
 
 canDownload :: IO Bool
 canDownload = (||) <$> inPath "wget" <*> inPath "curl"
diff --git a/debian/changelog b/debian/changelog
index fdc909e3e1..36034f2ae6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -18,6 +18,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     location of the file.
   * addurl: Normalize badly encoded urls.
   * Fix teardown of stale cached ssh connections.
+  * When checking that an url has a key, verify that the Content-Length,
+    if available, matches the size of the key.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 3f152ea4b8..0c343e42c9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120123
+Version: 3.20120124
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 17fed709c83de69c5bdf190b80eaa875fe6c9c7e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Feb 2012 19:23:23 -0400
Subject: [PATCH 3101/8313] addurl --fast: Verifies that the url can be
 downloaded (only getting its head), and records the size in the key.

---
 Backend/URL.hs    | 8 ++++++--
 Command/AddUrl.hs | 8 ++++++--
 debian/changelog  | 2 ++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/Backend/URL.hs b/Backend/URL.hs
index 6406095ca1..b3411bac5b 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -24,5 +24,9 @@ backend = Backend {
 	fsckKey = Nothing
 }
 
-fromUrl :: String -> Key
-fromUrl url = stubKey { keyName = url, keyBackendName = "URL" }
+fromUrl :: String -> Maybe Integer -> Key
+fromUrl url size = stubKey
+	{ keyName = url
+	, keyBackendName = "URL"
+	, keySize = size
+	}
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 496b9f2e8b..40e3a0e985 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -15,6 +15,7 @@ import qualified Backend
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
+import qualified Utility.Url as Url
 import Annex.Content
 import Logs.Web
 import qualified Option
@@ -55,7 +56,7 @@ perform url file = ifAnnexed file addurl geturl
 download :: String -> FilePath -> CommandPerform
 download url file = do
 	showAction $ "downloading " ++ url ++ " "
-	let dummykey = Backend.URL.fromUrl url
+	let dummykey = Backend.URL.fromUrl url Nothing
 	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	stopUnless (downloadUrl [url] tmp) $ do
@@ -70,7 +71,10 @@ download url file = do
 
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
-	let key = Backend.URL.fromUrl url
+	(exists, size) <- liftIO $ Url.exists url
+	unless exists $
+		error $ "unable to access url: " ++ url
+	let key = Backend.URL.fromUrl url size
 	setUrlPresent key url
 	next $ Command.Add.cleanup file key False
 
diff --git a/debian/changelog b/debian/changelog
index 36034f2ae6..f137972728 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -20,6 +20,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * Fix teardown of stale cached ssh connections.
   * When checking that an url has a key, verify that the Content-Length,
     if available, matches the size of the key.
+  * addurl --fast: Verifies that the url can be downloaded (only getting
+    its head), and records the size in the key.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From a3ebf16e62e4499401165eebc8cf3d7123dc4fe7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Feb 2012 19:40:36 -0400
Subject: [PATCH 3102/8313] also verify new urls when adding them to existing
 files

---
 Command/AddUrl.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 40e3a0e985..db73f14e93 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -19,6 +19,7 @@ import qualified Utility.Url as Url
 import Annex.Content
 import Logs.Web
 import qualified Option
+import Types.Key
 
 def :: [Command]
 def = [withOptions [fileOption] $
@@ -50,6 +51,8 @@ perform url file = ifAnnexed file addurl geturl
 			fast <- Annex.getState Annex.fast
 			if fast then nodownload url file else download url file
 		addurl (key, _backend) = do
+			unlessM (liftIO $ Url.check url (keySize key)) $
+				error $ "failed to verify url: " ++ url
 			setUrlPresent key url
 			next $ return True
 

From 6335abcab2c0b48132b04011acbd01fb99bc5b53 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Feb 2012 20:40:18 -0400
Subject: [PATCH 3103/8313] doc update

---
 doc/tips/using_the_web_as_a_special_remote.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/tips/using_the_web_as_a_special_remote.mdwn b/doc/tips/using_the_web_as_a_special_remote.mdwn
index 8009927a49..a151f99332 100644
--- a/doc/tips/using_the_web_as_a_special_remote.mdwn
+++ b/doc/tips/using_the_web_as_a_special_remote.mdwn
@@ -8,10 +8,10 @@ The web can be used as a [[special_remote|special_remotes]] too.
 Now the file is downloaded, and has been added to the annex like any other
 file. So it can be renamed, copied to other repositories, and so on.
 
-Note that git-annex assumes that, if the web site does not 404, the file is
-still present on the web, and this counts as one [[copy|copies]] of the
-file. So it will let you remove your last copy, trusting it can be
-downloaded again:
+Note that git-annex assumes that, if the web site does not 404, and has the
+right file size, the file is still present on the web, and this counts as
+one [[copy|copies]] of the file. So it will let you remove your last copy,
+trusting it can be downloaded again:
 
 	# git annex drop example.com_video.mpeg
 	drop example.com_video.mpeg (checking http://example.com/video.mpeg) ok

From ecfcb41abe5c2903ca80d26365afdf20faaf9989 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 10 Feb 2012 21:42:46 -0400
Subject: [PATCH 3104/8313] work around Network.Browser bug that converts a
 HEAD to a GET when following a redirect

The code explicitly switches from HEAD to GET for most redirects.
Possibly because someone misread a spec (which does require switching from
POST to GET for 303 redirects). Or possibly because the spec really is that
bad. Upstream bug: https://github.com/haskell/HTTP/issues/24

Since we absolutely don't want to download entire (large) files from
the web when checking that they exist with HEAD, I wrote my own redirect
follower, based closely on the one used by Network.Browser, but without
this misfeature.

Note that Network.Browser checks that the redirect url is a http url
and fails if not. I don't, because I want to not need to change this
code when it gets https support (related: I'm surprised to see it
doesn't support https yet..). The check does not seem security significant;
it doesn't support file:// urls for example. If a http url is redirected
to https, the Network.Browser will actually make a http connection again.
This could loop, but only up to 5 times.
---
 Utility/Url.hs | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/Utility/Url.hs b/Utility/Url.hs
index efd6ad16dd..dfdebaf06a 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -19,6 +19,7 @@ import Control.Monad
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
+import Data.Maybe
 
 import Utility.SafeCommand
 import Utility.Path
@@ -87,12 +88,32 @@ get url =
 
 {- Makes a http request of an url. For example, HEAD can be used to
  - check if the url exists, or GET used to get the url content (best for
- - small urls). -}
+ - small urls).
+ -
+ - This does its own redirect following because Browser's is buggy for HEAD
+ - requests.
+ -}
 request :: URI -> RequestMethod -> IO (Response String)
-request url requesttype = Browser.browse $ do
-	Browser.setErrHandler ignore
-	Browser.setOutHandler ignore
-	Browser.setAllowRedirects True
-	snd <$> Browser.request (mkRequest requesttype url :: Request_String)
+request url requesttype = go 5 url
 	where
+		go :: Int -> URI -> IO (Response String)
+		go 0 _ = error "Too many redirects "
+		go n u = do
+			rsp <- Browser.browse $ do
+				Browser.setErrHandler ignore
+				Browser.setOutHandler ignore
+				Browser.setAllowRedirects False
+				snd <$> Browser.request (mkRequest requesttype u :: Request_String)
+			case rspCode rsp of
+				(3,0,x) | x /= 5 -> redir (n - 1) u rsp
+				_ -> return rsp
 		ignore = const $ return ()
+		redir n u rsp = do
+			case retrieveHeaders HdrLocation rsp of
+				[] -> return rsp
+				(Header _ newu:_) ->
+					case parseURIReference newu of
+						Nothing -> return rsp
+						Just newURI -> go n newURI_abs
+							where
+								newURI_abs = fromMaybe newURI (newURI `relativeTo` u)

From 3ac2677e00661370f71250ed4cf53ad66bfcab2c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Feb 2012 16:58:26 -0400
Subject: [PATCH 3105/8313] comment typo

---
 Git/HashObject.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index ae498278f4..9c99dae268 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -12,7 +12,7 @@ import Git
 import Git.Command
 
 {- Injects a set of files into git, returning the shas of the objects
- - and an IO action to call ones the the shas have been used. -}
+ - and an IO action to call once the the shas have been used. -}
 hashFiles :: [FilePath] -> Repo -> IO ([Sha], IO ())
 hashFiles paths repo = do
 	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo

From 59b2adea4f006a391da5210394187f867c3e696b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Feb 2012 15:27:49 -0400
Subject: [PATCH 3106/8313] changelog for
 a964012fc36d22e4554dd12e3594579fb3190501

Turns out that commit really made some serious improvements to memory use.
With the lazy state monad, git-annex add in a huge tree grew seemingly
without bound until it overflowed the stack. With the strict monad,
it uses 42 mb max.

It's possible another change since the 3.20120123 release fixed that,
but a964012fc36d22e4554dd12e3594579fb3190501 seems most likely.
---
 debian/changelog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index f137972728..a5b0b31d14 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -22,6 +22,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     if available, matches the size of the key.
   * addurl --fast: Verifies that the url can be downloaded (only getting
     its head), and records the size in the key.
+  * Fixed to use the strict state monad, to avoid leaking all kinds of memory
+    due to lazy state update thunks when adding/fixing many files.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 0ef6d86873f55e487be5660a9bb24cb767a06993 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Feb 2012 16:59:00 -0400
Subject: [PATCH 3107/8313] force state strictly

When converting to the strict state monad, I missed this place where
thunks to the state could be built up, possibly. This seems to make
it run in some percentage less memory.
---
 CmdLine.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 18bb5fe51b..d2adb71bbd 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -71,7 +71,9 @@ tryRun' :: Integer -> Annex.AnnexState -> Command -> [CommandCleanup] -> IO ()
 tryRun' errnum _ cmd []
 	| errnum > 0 = error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
 	| otherwise = return ()
-tryRun' errnum state cmd (a:as) = run >>= handle
+tryRun' errnum state cmd (a:as) = do
+	r <- run
+	handle $! r
 	where
 		run = tryIO $ Annex.run state $ do
 			Annex.Queue.flushWhenFull

From cad8824852aa0623dc41eac02a9e2bae47d88ec4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Feb 2012 20:01:37 -0400
Subject: [PATCH 3108/8313] thinko

I removed the now unnecessary forkProcess, but forgot to change back to
pipeBoth, so there was no writer thread.
---
 Git/CheckAttr.hs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 3e9375159a..5c747a9519 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -16,10 +16,9 @@ import qualified Git.Version
 lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
 lookup attr files repo = do
 	cwd <- getCurrentDirectory
-	(_, fromh, toh) <- hPipeBoth "git" (toCommand params)
-	hPutStr toh $ join "\0" $ input cwd
-	hClose toh
-	zip files . map attrvalue . lines <$> hGetContents fromh
+	(_, r) <- pipeBoth "git" (toCommand params) $
+		join "\0" $ input cwd
+	return $ zip files $ map attrvalue $ lines r
 	where
 		params = gitCommandLine 
 				[ Param "check-attr"

From d35a8d85b5ee9ce3d6057300e21729183cce802b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Feb 2012 20:23:45 -0400
Subject: [PATCH 3109/8313] another place hGetBoth was used without a writer
 thread

---
 Git/Command.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Git/Command.hs b/Git/Command.hs
index 3d859ed28d..50d4455fe7 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -9,6 +9,8 @@ module Git.Command where
 
 import qualified Data.Text.Lazy as L
 import qualified Data.Text.Lazy.IO as L
+import Control.Concurrent
+import Control.Exception (finally)
 
 import Common
 import Git
@@ -61,8 +63,7 @@ pipeWriteRead params s repo = assertLocal repo $ do
 	(p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo)
 	fileEncoding to
 	fileEncoding from
-	hPutStr to s
-	hClose to
+	_ <- forkIO $ finally (hPutStr to s) (hClose to)
 	c <- hGetContents from
 	return (p, c)
 

From cbaebf538a8659193fb3dbb4f32e0f918a385af3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 13 Feb 2012 23:42:44 -0400
Subject: [PATCH 3110/8313] rework git check-attr interface

Now gitattributes are looked up, efficiently, in only the places that
really need them, using the same approach used for cat-file.

The old CheckAttr code seemed very fragile, in the way it streamed files
through git check-attr.
I actually found that cad8824852aa0623dc41eac02a9e2bae47d88ec4
was still deadlocking with ghc 7.4, at the end of adding a lot of files.
This should fix that problem, and avoid future ones.

The best part is that this removes withAttrFilesInGit and withNumCopies,
which were complicated Seek methods, as well as simplfying the types
for several other Seek methods that had a Backend tupled in.
---
 Annex.hs             |  3 +++
 Annex/CheckAttr.hs   | 35 ++++++++++++++++++++++++++
 Backend.hs           | 21 ++++++----------
 Command.hs           | 17 +++++++++----
 Command/Add.hs       | 13 +++++-----
 Command/AddUrl.hs    |  2 +-
 Command/Copy.hs      |  6 ++---
 Command/Drop.hs      |  8 +++---
 Command/Fsck.hs      |  7 +++---
 Command/Get.hs       |  8 +++---
 Command/Lock.hs      |  6 ++---
 Command/Migrate.hs   |  8 +++---
 Command/PreCommit.hs | 11 ++++----
 Git/CheckAttr.hs     | 60 +++++++++++++++++++++++++++++++-------------
 Remote/List.hs       |  4 +--
 Seek.hs              | 33 ++++++------------------
 16 files changed, 143 insertions(+), 99 deletions(-)
 create mode 100644 Annex/CheckAttr.hs

diff --git a/Annex.hs b/Annex.hs
index de36c816d8..5344152071 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -35,6 +35,7 @@ import Common
 import qualified Git
 import qualified Git.Config
 import Git.CatFile
+import Git.CheckAttr
 import qualified Git.Queue
 import Types.Backend
 import qualified Types.Remote
@@ -82,6 +83,7 @@ data AnnexState = AnnexState
 	, auto :: Bool
 	, branchstate :: BranchState
 	, catfilehandle :: Maybe CatFileHandle
+	, checkattrhandle :: Maybe CheckAttrHandle
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, limit :: Matcher (FilePath -> Annex Bool)
@@ -105,6 +107,7 @@ newState gitrepo = AnnexState
 	, auto = False
 	, branchstate = startBranchState
 	, catfilehandle = Nothing
+	, checkattrhandle = Nothing
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, limit = Left []
diff --git a/Annex/CheckAttr.hs b/Annex/CheckAttr.hs
new file mode 100644
index 0000000000..01779e8136
--- /dev/null
+++ b/Annex/CheckAttr.hs
@@ -0,0 +1,35 @@
+{- git check-attr interface, with handle automatically stored in the Annex monad
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.CheckAttr (
+	checkAttr,
+	checkAttrHandle
+) where
+
+import Common.Annex
+import qualified Git.CheckAttr as Git
+import qualified Annex
+
+{- All gitattributes used by git-annex. -}
+annexAttrs :: [Git.Attr]
+annexAttrs =
+	[ "annex.backend"
+	, "annex.numcopies"
+	]
+
+checkAttr :: Git.Attr -> FilePath -> Annex String
+checkAttr attr file = do
+	h <- checkAttrHandle
+	liftIO $ Git.checkAttr h attr file
+
+checkAttrHandle :: Annex Git.CheckAttrHandle
+checkAttrHandle = maybe startup return =<< Annex.getState Annex.checkattrhandle
+	where
+		startup = do
+			h <- inRepo $ Git.checkAttrStart annexAttrs
+			Annex.changeState $ \s -> s { Annex.checkattrhandle = Just h }
+			return h
diff --git a/Backend.hs b/Backend.hs
index e351bb3b27..50c89ecc55 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -6,12 +6,11 @@
  -}
 
 module Backend (
-	BackendFile,
 	list,
 	orderedList,
 	genKey,
 	lookupFile,
-	chooseBackends,
+	chooseBackend,
 	lookupBackendName,
 	maybeLookupBackendName
 ) where
@@ -22,6 +21,7 @@ import Common.Annex
 import qualified Git.Config
 import qualified Git.CheckAttr
 import qualified Annex
+import Annex.CheckAttr
 import Types.Key
 import qualified Types.Backend as B
 
@@ -93,20 +93,15 @@ lookupFile file = do
 						bname ++ ")"
 					return Nothing
 
-type BackendFile = (Maybe Backend, FilePath)
-
-{- Looks up the backends that should be used for each file in a list.
+{- Looks up the backend that should be used for a file.
  - That can be configured on a per-file basis in the gitattributes file.
  -}
-chooseBackends :: [FilePath] -> Annex [BackendFile]
-chooseBackends fs = Annex.getState Annex.forcebackend >>= go
+chooseBackend :: FilePath -> Annex (Maybe Backend)
+chooseBackend f = Annex.getState Annex.forcebackend >>= go
 	where
-		go Nothing = do
-			pairs <- inRepo $ Git.CheckAttr.lookup "annex.backend" fs
-			return $ map (\(f,b) -> (maybeLookupBackendName b, f)) pairs
-		go (Just _) = do
-			l <- orderedList
-			return $ map (\f -> (Just $ Prelude.head l, f)) fs
+		go Nothing =  maybeLookupBackendName <$>
+			checkAttr "annex.backend" f
+		go (Just _) = Just . Prelude.head <$> orderedList
 
 {- Looks up a backend by name. May fail if unknown. -}
 lookupBackendName :: String -> Backend
diff --git a/Command.hs b/Command.hs
index 386efafde9..e7ce335c71 100644
--- a/Command.hs
+++ b/Command.hs
@@ -18,6 +18,7 @@ module Command (
 	ifAnnexed,
 	notBareRepo,
 	isBareRepo,
+	numCopies,
 	autoCopies,
 	module ReExported
 ) where
@@ -34,6 +35,7 @@ import Checks as ReExported
 import Usage as ReExported
 import Logs.Trust
 import Config
+import Annex.CheckAttr
 
 {- Generates a normal command -}
 command :: String -> String -> [CommandSeek] -> String -> Command
@@ -98,17 +100,22 @@ notBareRepo a = do
 isBareRepo :: Annex Bool
 isBareRepo = fromRepo Git.repoIsLocalBare
 
+numCopies :: FilePath  -> Annex (Maybe Int)
+numCopies file = readish <$> checkAttr "annex.numcopies" file
+
 {- Used for commands that have an auto mode that checks the number of known
  - copies of a key.
  -
  - In auto mode, first checks that the number of known
  - copies of the key is > or < than the numcopies setting, before running
  - the action. -}
-autoCopies :: Key -> (Int -> Int -> Bool) -> Maybe Int -> CommandStart -> CommandStart
-autoCopies key vs numcopiesattr a = Annex.getState Annex.auto >>= auto
+autoCopies :: FilePath -> Key -> (Int -> Int -> Bool) -> (Maybe Int -> CommandStart) -> CommandStart
+autoCopies file key vs a = do
+	numcopiesattr <- numCopies file
+	Annex.getState Annex.auto >>= auto numcopiesattr
 	where
-		auto False = a
-		auto True = do
+		auto numcopiesattr False = a numcopiesattr
+		auto numcopiesattr True = do
 			needed <- getNumCopies numcopiesattr
 			(_, have) <- trustPartition UnTrusted =<< Remote.keyLocations key
-			if length have `vs` needed then a else stop
+			if length have `vs` needed then a numcopiesattr else stop
diff --git a/Command/Add.hs b/Command/Add.hs
index 9410601b8b..28971529a7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -16,7 +16,6 @@ import qualified Backend
 import Logs.Location
 import Annex.Content
 import Utility.Touch
-import Backend
 
 def :: [Command]
 def = [command "add" paramPaths seek "add files to annex"]
@@ -28,8 +27,8 @@ seek = [withFilesNotInGit start, withFilesUnlocked start]
 {- The add subcommand annexes a file, storing it in a backend, and then
  - moving it into the annex directory and setting up the symlink pointing
  - to its content. -}
-start :: BackendFile -> CommandStart
-start p@(_, file) = notBareRepo $ ifAnnexed file fixup add
+start :: FilePath -> CommandStart
+start file = notBareRepo $ ifAnnexed file fixup add
 	where
 		add = do
 			s <- liftIO $ getSymbolicLinkStatus file
@@ -37,7 +36,7 @@ start p@(_, file) = notBareRepo $ ifAnnexed file fixup add
 				then stop
 				else do
 					showStart "add" file
-					next $ perform p
+					next $ perform file
 		fixup (key, _) = do
 			-- fixup from an interrupted add; the symlink
 			-- is present but not yet added to git
@@ -45,8 +44,10 @@ start p@(_, file) = notBareRepo $ ifAnnexed file fixup add
 			liftIO $ removeFile file
 			next $ next $ cleanup file key =<< inAnnex key
 
-perform :: BackendFile -> CommandPerform
-perform (backend, file) = Backend.genKey file backend >>= go
+perform :: FilePath -> CommandPerform
+perform file = do
+	backend <- Backend.chooseBackend file
+	Backend.genKey file backend >>= go
 	where
 		go Nothing = stop
 		go (Just (key, _)) = do
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index db73f14e93..f91d6dd553 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -63,7 +63,7 @@ download url file = do
 	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	stopUnless (downloadUrl [url] tmp) $ do
-		[(backend, _)] <- Backend.chooseBackends [file]
+		backend <- Backend.chooseBackend file
 		k <- Backend.genKey tmp backend
 		case k of
 			Nothing -> stop
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 32b83a5262..a8ec225706 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -19,10 +19,10 @@ def = [withOptions Command.Move.options $ command "copy" paramPaths seek
 seek :: [CommandSeek]
 seek = [withField Command.Move.toOption Remote.byName $ \to ->
 		withField Command.Move.fromOption Remote.byName $ \from ->
-			withNumCopies $ \n -> whenAnnexed $ start to from n]
+			withFilesInGit $ whenAnnexed $ start to from]
 
 -- A copy is just a move that does not delete the source file.
 -- However, --auto mode avoids unnecessary copies.
-start :: Maybe Remote -> Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start to from numcopies file (key, backend) = autoCopies key (<) numcopies $
+start :: Maybe Remote -> Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart
+start to from file (key, backend) = autoCopies file key (<) $ \_numcopies ->
 	Command.Move.start to from False file (key, backend)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index b40de00cb2..9eb36a22fe 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -26,11 +26,11 @@ fromOption :: Option
 fromOption = Option.field ['f'] "from" paramRemote "drop content from a remote"
 
 seek :: [CommandSeek]
-seek = [withField fromOption Remote.byName $ \from -> withNumCopies $ \n ->
-	whenAnnexed $ start from n]
+seek = [withField fromOption Remote.byName $ \from ->
+	withFilesInGit $ whenAnnexed $ start from]
 
-start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start from numcopies file (key, _) = autoCopies key (>) numcopies $ do
+start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart
+start from file (key, _) = autoCopies file key (>) $ \numcopies -> do
 	case from of
 		Nothing -> startLocal file numcopies key
 		Just remote -> do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 469fad749e..94b3601043 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -36,12 +36,13 @@ options = [fromOption]
 seek :: [CommandSeek]
 seek =
 	[ withField fromOption Remote.byName $ \from ->
-		withNumCopies $ \n -> whenAnnexed $ start from n
+		withFilesInGit $ whenAnnexed $ start from
 	, withBarePresentKeys startBare
 	]
 
-start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start from numcopies file (key, backend) = do
+start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart
+start from file (key, backend) = do
+	numcopies <- numCopies file
 	showStart "fsck" file
 	case from of
 		Nothing -> next $ perform key file backend numcopies
diff --git a/Command/Get.hs b/Command/Get.hs
index 5d032e13c4..928ab0f1be 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -19,11 +19,11 @@ def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek
 
 seek :: [CommandSeek]
 seek = [withField Command.Move.fromOption Remote.byName $ \from ->
-	withNumCopies $ \n -> whenAnnexed $ start from n]
+	withFilesInGit $ whenAnnexed $ start from]
 
-start :: Maybe Remote -> Maybe Int -> FilePath -> (Key, Backend) -> CommandStart
-start from numcopies file (key, _) = stopUnless (not <$> inAnnex key) $
-	autoCopies key (<) numcopies $ do
+start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart
+start from file (key, _) = stopUnless (not <$> inAnnex key) $
+	autoCopies file key (<) $ \_numcopies -> do
 		case from of
 			Nothing -> go $ perform key
 			Just src -> do
diff --git a/Command/Lock.hs b/Command/Lock.hs
index 329fd3eff7..b8aedb252b 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -10,7 +10,6 @@ module Command.Lock where
 import Common.Annex
 import Command
 import qualified Annex.Queue
-import Backend
 	
 def :: [Command]
 def = [command "lock" paramPaths seek "undo unlock command"]
@@ -18,9 +17,8 @@ def = [command "lock" paramPaths seek "undo unlock command"]
 seek :: [CommandSeek]
 seek = [withFilesUnlocked start, withFilesUnlockedToBeCommitted start]
 
-{- Undo unlock -}
-start :: BackendFile -> CommandStart
-start (_, file) = do
+start :: FilePath -> CommandStart
+start file = do
 	showStart "lock" file
 	next $ perform file
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index f6467463d0..c6b0f086cf 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -19,12 +19,12 @@ def :: [Command]
 def = [command "migrate" paramPaths seek "switch data to different backend"]
 
 seek :: [CommandSeek]
-seek = [withBackendFilesInGit $ \(b, f) -> whenAnnexed (start b) f]
+seek = [withFilesInGit $ whenAnnexed start]
 
-start :: Maybe Backend -> FilePath -> (Key, Backend) -> CommandStart
-start b file (key, oldbackend) = do
+start :: FilePath -> (Key, Backend) -> CommandStart
+start file (key, oldbackend) = do
 	exists <- inAnnex key
-	newbackend <- choosebackend b
+	newbackend <- choosebackend =<< Backend.chooseBackend file
 	if (newbackend /= oldbackend || upgradableKey key) && exists
 		then do
 			showStart "migrate" file
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index 57bc7ac138..b0328ca190 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -10,7 +10,6 @@ module Command.PreCommit where
 import Command
 import qualified Command.Add
 import qualified Command.Fix
-import Backend
 
 def :: [Command]
 def = [command "pre-commit" paramPaths seek "run by git pre-commit hook"]
@@ -22,12 +21,12 @@ seek =
 	[ withFilesToBeCommitted $ whenAnnexed Command.Fix.start
 	, withFilesUnlockedToBeCommitted start]
 
-start :: BackendFile -> CommandStart
-start p = next $ perform p
+start :: FilePath -> CommandStart
+start file = next $ perform file
 
-perform :: BackendFile -> CommandPerform
-perform pair@(_, file) = do
-	ok <- doCommand $ Command.Add.start pair
+perform :: FilePath -> CommandPerform
+perform file = do
+	ok <- doCommand $ Command.Add.start file
 	if ok
 		then next $ return True
 		else error $ "failed to add " ++ file ++ "; canceling commit"
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 5c747a9519..669a9c54ef 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -1,6 +1,6 @@
 {- git check-attr interface
  -
- - Copyright 2010, 2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -12,20 +12,44 @@ import Git
 import Git.Command
 import qualified Git.Version
 
-{- Efficiently looks up a gitattributes value for each file in a list. -}
-lookup :: String -> [FilePath] -> Repo -> IO [(FilePath, String)]
-lookup attr files repo = do
-	cwd <- getCurrentDirectory
-	(_, r) <- pipeBoth "git" (toCommand params) $
-		join "\0" $ input cwd
-	return $ zip files $ map attrvalue $ lines r
-	where
-		params = gitCommandLine 
-				[ Param "check-attr"
-				, Param attr
-				, Params "-z --stdin"
-				] repo
+type CheckAttrHandle = (PipeHandle, Handle, Handle, [Attr], String)
 
+type Attr = String
+
+{- Starts git check-attr running to look up the specified gitattributes
+ - values and return a handle.  -}
+checkAttrStart :: [Attr] -> Repo -> IO CheckAttrHandle
+checkAttrStart attrs repo = do
+	cwd <- getCurrentDirectory
+	(pid, from, to) <- hPipeBoth "git" $ toCommand $
+		gitCommandLine params repo
+	return (pid, from, to, attrs, cwd)
+	where
+		params =
+			[ Param "check-attr" ]
+			++ map Param attrs ++
+			[ Params "-z --stdin" ]
+
+{- Stops git check-attr. -}
+checkAttrStop :: CheckAttrHandle -> IO ()
+checkAttrStop (pid, from, to, _, _) = do
+	hClose to
+	hClose from
+	forceSuccess pid
+
+{- Gets an attribute of a file. -}
+checkAttr :: CheckAttrHandle -> Attr -> FilePath -> IO String
+checkAttr (_, from, to, attrs, cwd) want file = do
+	hPutStr to $ file' ++ "\0"
+	hFlush to
+	pairs <- forM attrs $ \attr -> do
+		l <- hGetLine from
+		return (attr, attrvalue attr l)
+	let vals = map snd $ filter (\(attr, _) -> attr == want) pairs
+	case vals of
+		[v] -> return v
+		_ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file
+	where
 		{- Before git 1.7.7, git check-attr worked best with
 		 - absolute filenames; using them worked around some bugs
 		 - with relative filenames.
@@ -34,10 +58,10 @@ lookup attr files repo = do
 		 - filenames, and the bugs that necessitated them were fixed,
 		 - so use relative filenames. -}
 		oldgit = Git.Version.older "1.7.7"
-		input cwd
-			| oldgit = map (absPathFrom cwd) files
-			| otherwise = map (relPathDirToFile cwd . absPathFrom cwd) files
-		attrvalue l = end bits !! 0
+		file'
+			| oldgit = absPathFrom cwd file
+			| otherwise = relPathDirToFile cwd $ absPathFrom cwd file
+		attrvalue attr l = end bits !! 0
 			where
 				bits = split sep l
 				sep = ": " ++ attr ++ ": "
diff --git a/Remote/List.hs b/Remote/List.hs
index e589b4401a..7c419c75d6 100644
--- a/Remote/List.hs
+++ b/Remote/List.hs
@@ -17,7 +17,7 @@ import Annex.UUID
 import Config
 
 import qualified Remote.Git
-import qualified Remote.S3
+--import qualified Remote.S3
 import qualified Remote.Bup
 import qualified Remote.Directory
 import qualified Remote.Rsync
@@ -27,7 +27,7 @@ import qualified Remote.Hook
 remoteTypes :: [RemoteType]
 remoteTypes =
 	[ Remote.Git.remote
-	, Remote.S3.remote
+--	, Remote.S3.remote
 	, Remote.Bup.remote
 	, Remote.Directory.remote
 	, Remote.Rsync.remote
diff --git a/Seek.hs b/Seek.hs
index b4e1218e2b..a9c034d222 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -14,11 +14,9 @@ module Seek where
 import Common.Annex
 import Types.Command
 import Types.Key
-import Backend
 import qualified Annex
 import qualified Git
 import qualified Git.LsFiles as LsFiles
-import qualified Git.CheckAttr
 import qualified Limit
 import qualified Option
 
@@ -28,26 +26,12 @@ seekHelper a params = inRepo $ \g -> runPreserveOrder (`a` g) params
 withFilesInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
 
-withAttrFilesInGit :: String -> ((FilePath, String) -> CommandStart) -> CommandSeek
-withAttrFilesInGit attr a params = do
-	files <- seekHelper LsFiles.inRepo params
-	prepFilteredGen a fst $ inRepo $ Git.CheckAttr.lookup attr files
-
-withNumCopies :: (Maybe Int -> FilePath -> CommandStart) -> CommandSeek
-withNumCopies a params = withAttrFilesInGit "annex.numcopies" go params
-	where
-		go (file, v) = a (readish v) file
-
-withBackendFilesInGit :: (BackendFile -> CommandStart) -> CommandSeek
-withBackendFilesInGit a params =
-	prepBackendPairs a =<< seekHelper LsFiles.inRepo params
-
-withFilesNotInGit :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesNotInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	{- dotfiles are not acted on unless explicitly listed -}
 	files <- filter (not . dotfile) <$> seek ps
 	dotfiles <- if null dotps then return [] else seek dotps
-	prepBackendPairs a $ preserveOrder params (files++dotfiles)
+	prepFiltered a $ return $ preserveOrder params (files++dotfiles)
 	where
 		(dotps, ps) = partition dotfile params
 		seek l = do
@@ -65,20 +49,20 @@ withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = prepFiltered a $
 	seekHelper LsFiles.stagedNotDeleted params
 
-withFilesUnlocked :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked :: (FilePath -> CommandStart) -> CommandSeek
 withFilesUnlocked = withFilesUnlocked' LsFiles.typeChanged
 
-withFilesUnlockedToBeCommitted :: (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlockedToBeCommitted :: (FilePath -> CommandStart) -> CommandSeek
 withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 
-withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (BackendFile -> CommandStart) -> CommandSeek
+withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (FilePath -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
 	top <- fromRepo Git.workTree
 	typechangedfiles <- seekHelper typechanged params
-	unlockedfiles <- liftIO $ filterM notSymlink $
+	let unlockedfiles = liftIO $ filterM notSymlink $
 		map (\f -> top ++ "/" ++ f) typechangedfiles
-	prepBackendPairs a unlockedfiles
+	prepFiltered a unlockedfiles
 
 withKeys :: (Key -> CommandStart) -> CommandSeek
 withKeys a params = return $ map (a . parse) params
@@ -109,9 +93,6 @@ withNothing _ _ = error "This command takes no parameters."
 prepFiltered :: (FilePath -> CommandStart) -> Annex [FilePath] -> Annex [CommandStart]
 prepFiltered a = prepFilteredGen a id
 
-prepBackendPairs :: (BackendFile -> CommandStart) -> CommandSeek
-prepBackendPairs a fs = prepFilteredGen a snd (chooseBackends fs)
-
 prepFilteredGen :: (b -> CommandStart) -> (b -> FilePath) -> Annex [b] -> Annex [CommandStart]
 prepFilteredGen a d fs = do
 	matcher <- Limit.getMatcher

From a2f241d50344726831ece2a354a599f389b20b54 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 00:22:42 -0400
Subject: [PATCH 3111/8313] fix LsFiles.typeChanged paths

Passing absolute paths to Command.Add used to work, but after recent
changes doesn't. All LsFiles should use relative paths anyway, so fix it
there.
---
 Backend.hs     | 1 -
 Git/LsFiles.hs | 8 +++++++-
 Seek.hs        | 4 +---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 50c89ecc55..4c28f1c779 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -19,7 +19,6 @@ import System.Posix.Files
 
 import Common.Annex
 import qualified Git.Config
-import qualified Git.CheckAttr
 import qualified Annex
 import Annex.CheckAttr
 import Types.Key
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 0de86383d3..201d76d1d4 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -65,7 +65,13 @@ typeChanged :: [FilePath] -> Repo -> IO [FilePath]
 typeChanged = typeChanged' []
 
 typeChanged' :: [CommandParam] -> [FilePath] -> Repo -> IO [FilePath]
-typeChanged' ps l = pipeNullSplit $ prefix ++ ps ++ suffix
+typeChanged' ps l repo = do
+	fs <- pipeNullSplit (prefix ++ ps ++ suffix) repo
+	-- git diff returns filenames relative to the top of the git repo;
+	-- convert to filenames relative to the cwd, like git ls-files.
+	let top = workTree repo
+	cwd <- getCurrentDirectory
+	return $ map (\f -> relPathDirToFile cwd $ top  f) fs
 	where
 		prefix = [Params "diff --name-only --diff-filter=T -z"]
 		suffix = Param "--" : map File l
diff --git a/Seek.hs b/Seek.hs
index a9c034d222..7f55063169 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -58,10 +58,8 @@ withFilesUnlockedToBeCommitted = withFilesUnlocked' LsFiles.typeChangedStaged
 withFilesUnlocked' :: ([FilePath] -> Git.Repo -> IO [FilePath]) -> (FilePath -> CommandStart) -> CommandSeek
 withFilesUnlocked' typechanged a params = do
 	-- unlocked files have changed type from a symlink to a regular file
-	top <- fromRepo Git.workTree
 	typechangedfiles <- seekHelper typechanged params
-	let unlockedfiles = liftIO $ filterM notSymlink $
-		map (\f -> top ++ "/" ++ f) typechangedfiles
+	let unlockedfiles = liftIO $ filterM notSymlink typechangedfiles
 	prepFiltered a unlockedfiles
 
 withKeys :: (Key -> CommandStart) -> CommandSeek

From 2b28c70f5fd9c03cadd39615a4abd4d12f4a9c35 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 01:01:38 -0400
Subject: [PATCH 3112/8313] add, and immediately close bug. useful
 documentation though

---
 doc/bugs/git_annex_add_memory_leak.mdwn | 38 +++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_memory_leak.mdwn

diff --git a/doc/bugs/git_annex_add_memory_leak.mdwn b/doc/bugs/git_annex_add_memory_leak.mdwn
new file mode 100644
index 0000000000..846e764369
--- /dev/null
+++ b/doc/bugs/git_annex_add_memory_leak.mdwn
@@ -0,0 +1,38 @@
+For the record, `git annex add` has had a series of memory leaks.
+Mostly these are minor -- until you need to check in a few
+million files in a single operation. 
+
+If this happens to you, git-annex will run out of memory and stop.
+(Generally well before your system runs out of memory, since it has some
+built-in ulimits.) You can recover by just re-running the `git annex add`
+-- it will automatically pick up where it left off.
+
+A history of the leaks:
+
+* Originally, `git annex add` remembered all the files
+  it had added, and fed them to git at the end. Of course
+  that made its memory use grow, so it was fixed to periodically
+  flush its buffer. Affected versions: before 0.20110417
+
+* Something called a "lazy state monad" caused "thunks" to build
+  up and memory to leak. Also affected other git annex commands
+  than `add`. Adding files using a SHA* backend hit the worst.
+  Fixed in versions afer 3.20120123.
+
+* A strange GHC bug seemed to be responsible for another leak.
+  (In particular, a child process was forked. All the child did 
+  was read filenames from one pipe and shove them reformatted out
+  another pipe. For some reason, it steadily grew in size.)
+  Code was rewritten in a way that happens to avoid that leak.
+  Apparently fixed in versions afer 3.20120123, but this one is not
+  well understood.
+
+* (Note that `git ls-files --others`, which is used to find files to add,
+  also uses surpsisingly large amounts
+  of memory when you have a lot of files. It buffers
+  the entire list, so it can compare it with the files in the index,
+  before outputting anything.
+  This is Not Our Problem, but I'm sure the git developers
+  would appreciate a patch that fixes it.)
+
+[[done]] ... for now --[[Joey]] 

From afd33b0236b4875897c01a07a753584df8c5a2cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 01:11:02 -0400
Subject: [PATCH 3113/8313] simplify

---
 Seek.hs | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/Seek.hs b/Seek.hs
index 7f55063169..6f56f30f4a 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -89,17 +89,13 @@ withNothing _ _ = error "This command takes no parameters."
 
 
 prepFiltered :: (FilePath -> CommandStart) -> Annex [FilePath] -> Annex [CommandStart]
-prepFiltered a = prepFilteredGen a id
-
-prepFilteredGen :: (b -> CommandStart) -> (b -> FilePath) -> Annex [b] -> Annex [CommandStart]
-prepFilteredGen a d fs = do
+prepFiltered a fs = do
 	matcher <- Limit.getMatcher
 	map (proc matcher) <$> fs
 	where
-		proc matcher v = do
-			let f = d v
+		proc matcher f = do
 			ok <- matcher f
-			if ok then a v else return Nothing
+			if ok then a f else return Nothing
 
 notSymlink :: FilePath -> IO Bool
 notSymlink f = liftIO $ not . isSymbolicLink <$> getSymbolicLinkStatus f

From 8fbc529d68feb1f40ac7dd44514ac387ba1237ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 03:10:01 -0400
Subject: [PATCH 3114/8313] oops

---
 Remote/List.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Remote/List.hs b/Remote/List.hs
index 7c419c75d6..e589b4401a 100644
--- a/Remote/List.hs
+++ b/Remote/List.hs
@@ -17,7 +17,7 @@ import Annex.UUID
 import Config
 
 import qualified Remote.Git
---import qualified Remote.S3
+import qualified Remote.S3
 import qualified Remote.Bup
 import qualified Remote.Directory
 import qualified Remote.Rsync
@@ -27,7 +27,7 @@ import qualified Remote.Hook
 remoteTypes :: [RemoteType]
 remoteTypes =
 	[ Remote.Git.remote
---	, Remote.S3.remote
+	, Remote.S3.remote
 	, Remote.Bup.remote
 	, Remote.Directory.remote
 	, Remote.Rsync.remote

From cb631ce518b715e36cb3c476d576696f0630738f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 03:49:48 -0400
Subject: [PATCH 3115/8313] whereis: Prints the urls of files that the web
 special remote knows about.

---
 Command/Status.hs   |  2 +-
 Command/Whereis.hs  | 28 +++++++++++++++++++++-------
 Remote.hs           |  9 +++++----
 Remote/Bup.hs       |  1 +
 Remote/Directory.hs |  1 +
 Remote/Git.hs       |  1 +
 Remote/Hook.hs      |  1 +
 Remote/Rsync.hs     |  1 +
 Remote/S3.hs        |  1 +
 Remote/Web.hs       |  1 +
 Types/Remote.hs     |  2 ++
 debian/changelog    |  1 +
 12 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 5facaab9be..dfe847bb8e 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -113,7 +113,7 @@ supported_remote_types = stat "supported remote types" $ json unwords $
 
 remote_list :: TrustLevel -> String -> Stat
 remote_list level desc = stat n $ nojson $ lift $ do
-	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap)
+	us <- M.keys <$> (M.union <$> uuidMap <*> remoteMap Remote.name)
 	rs <- fst <$> trustPartition level us
 	s <- prettyPrintUUIDs n rs
 	return $ if null s then "0" else show (length rs) ++ "\n" ++ beginning s
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index 1fbe707992..f62d34642f 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -7,6 +7,8 @@
 
 module Command.Whereis where
 
+import qualified Data.Map as M
+
 import Common.Annex
 import Command
 import Remote
@@ -17,24 +19,36 @@ def = [command "whereis" paramPaths seek
 	"lists repositories that have file content"]
 
 seek :: [CommandSeek]
-seek = [withFilesInGit $ whenAnnexed start]
+seek = [withValue (remoteMap id) $ \m ->
+	withFilesInGit $ whenAnnexed $ start m]
 
-start :: FilePath -> (Key, Backend) -> CommandStart
-start file (key, _) = do
+start :: (M.Map UUID Remote) -> FilePath -> (Key, Backend) -> CommandStart
+start remotemap file (key, _) = do
 	showStart "whereis" file
-	next $ perform key
+	next $ perform remotemap key
 
-perform :: Key -> CommandPerform
-perform key = do
-	(untrustedlocations, safelocations) <- trustPartition UnTrusted =<< keyLocations key
+perform :: (M.Map UUID Remote) -> Key -> CommandPerform
+perform remotemap key = do
+	locations <- keyLocations key
+	(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
 	let num = length safelocations
 	showNote $ show num ++ " " ++ copiesplural num
 	pp <- prettyPrintUUIDs "whereis" safelocations
 	unless (null safelocations) $ showLongNote pp
 	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
 	unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp'
+	forM_ (catMaybes $ map (`M.lookup` remotemap) locations) $
+		performRemote key
 	if null safelocations then stop else next $ return True
 	where
 		copiesplural 1 = "copy"
 		copiesplural _ = "copies"
 		untrustedheader = "The following untrusted locations may also have copies:\n"
+
+performRemote :: Key -> Remote -> Annex () 
+performRemote key remote = case whereisKey remote of
+	Nothing -> return ()
+	Just a -> do
+		ls <- a key
+		unless (null ls) $ showLongNote $
+			unlines $ map (\l -> name remote ++ ": " ++ l) ls
diff --git a/Remote.hs b/Remote.hs
index ffb53446b4..861319e083 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -15,6 +15,7 @@ module Remote (
 	removeKey,
 	hasKey,
 	hasKeyCheap,
+	whereisKey,
 
 	remoteTypes,
 	remoteList,
@@ -48,16 +49,16 @@ import Logs.Trust
 import Logs.Location
 import Remote.List
 
-{- Map of UUIDs of Remotes and their names. -}
-remoteMap :: Annex (M.Map UUID String)
-remoteMap = M.fromList . map (\r -> (uuid r, name r)) .
+{- Map from UUIDs of Remotes to a calculated value. -}
+remoteMap :: (Remote -> a) -> Annex (M.Map UUID a)
+remoteMap c = M.fromList . map (\r -> (uuid r, c r)) .
 	filter (\r -> uuid r /= NoUUID) <$> remoteList
 
 {- Map of UUIDs and their descriptions.
  - The names of Remotes are added to suppliment any description that has
  - been set for a repository. -}
 uuidDescriptions :: Annex (M.Map UUID String)
-uuidDescriptions = M.unionWith addName <$> uuidMap <*> remoteMap
+uuidDescriptions = M.unionWith addName <$> uuidMap <*> remoteMap name
 
 addName :: String -> String -> String
 addName desc n
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 50c3b10b39..a4f43a3f3e 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -53,6 +53,7 @@ gen r u c = do
 			removeKey = remove,
 			hasKey = checkPresent r bupr',
 			hasKeyCheap = bupLocal buprepo,
+			whereisKey = Nothing,
 			config = c,
 			repo = r,
 			remotetype = remote
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 85f6446078..ee2a0d75aa 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -45,6 +45,7 @@ gen r u c = do
 			removeKey = remove dir,
 			hasKey = checkPresent dir,
 			hasKeyCheap = True,
+			whereisKey = Nothing,
 			config = Nothing,
 			repo = r,
 			remotetype = remote
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 3905247755..c07ae3237b 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -81,6 +81,7 @@ gen r u _ = do
 		removeKey = dropKey r',
 		hasKey = inAnnex r',
 		hasKeyCheap = cheap,
+		whereisKey = Nothing,
 		config = Nothing,
 		repo = r',
 		remotetype = remote
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index a08c4011ef..c7d710f196 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -45,6 +45,7 @@ gen r u c = do
 			removeKey = remove hooktype,
 			hasKey = checkPresent r hooktype,
 			hasKeyCheap = False,
+			whereisKey = Nothing,
 			config = Nothing,
 			repo = r,
 			remotetype = remote
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index c7efe42008..54fb890cae 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -52,6 +52,7 @@ gen r u c = do
 			removeKey = remove o,
 			hasKey = checkPresent r o,
 			hasKeyCheap = False,
+			whereisKey = Nothing,
 			config = Nothing,
 			repo = r,
 			remotetype = remote
diff --git a/Remote/S3.hs b/Remote/S3.hs
index c9527ba67a..812345b00a 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -57,6 +57,7 @@ gen' r u c cst =
 			removeKey = remove this,
 			hasKey = checkPresent this,
 			hasKeyCheap = False,
+			whereisKey = Nothing,
 			config = c,
 			repo = r,
 			remotetype = remote
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 6bd04d4b15..81e6ca321c 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -45,6 +45,7 @@ gen r _ _ =
 		removeKey = dropKey,
 		hasKey = checkKey,
 		hasKeyCheap = False,
+		whereisKey = Just getUrls,
 		config = Nothing,
 		repo = r,
 		remotetype = remote
diff --git a/Types/Remote.hs b/Types/Remote.hs
index 003dd5342a..9bac2ca0f8 100644
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -55,6 +55,8 @@ data RemoteA a = Remote {
 	-- Some remotes can check hasKey without an expensive network
 	-- operation.
 	hasKeyCheap :: Bool,
+	-- Some remotes can provide additional details for whereis.
+	whereisKey :: Maybe (Key -> a [String]),
 	-- a Remote can have a persistent configuration store
 	config :: Maybe RemoteConfig,
 	-- git configuration for the remote
diff --git a/debian/changelog b/debian/changelog
index a5b0b31d14..8df49d925f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,6 +24,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     its head), and records the size in the key.
   * Fixed to use the strict state monad, to avoid leaking all kinds of memory
     due to lazy state update thunks when adding/fixing many files.
+  * whereis: Prints the urls of files that the web special remote knows about.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 8f76d66f329a9bc431f0c16da529b9573756c755 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 04:31:39 -0400
Subject: [PATCH 3116/8313] set fileEncoding on CheckAttr handles

Seemed to work without it, but this is correct.
---
 Git/CheckAttr.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 669a9c54ef..8de19390f1 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -23,6 +23,8 @@ checkAttrStart attrs repo = do
 	cwd <- getCurrentDirectory
 	(pid, from, to) <- hPipeBoth "git" $ toCommand $
 		gitCommandLine params repo
+	fileEncoding from
+	fileEncoding to
 	return (pid, from, to, attrs, cwd)
 	where
 		params =

From 2f1f1e6b136611d844caf5b304251b4f024ccde6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 10:59:48 -0400
Subject: [PATCH 3117/8313] avoid version saving state

This is not the place to commit journal files.
---
 Command/Version.hs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Version.hs b/Command/Version.hs
index 9fb7fe5bdb..8761d2a2e0 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Command
 import qualified Build.SysConfig as SysConfig
 import Annex.Version
+import CmdLine
 
 def :: [Command]
 def = [noRepo showPackageVersion $ dontCheck repoExists $
@@ -28,7 +29,9 @@ start = do
 		putStrLn $ "default repository version: " ++ defaultVersion
 		putStrLn $ "supported repository versions: " ++ vs supportedVersions
 		putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
-	stop
+	-- avoid normal cleanup
+	_ <- shutdown True
+	liftIO exitSuccess
 	where
 		vs = join " "
 

From 82ae30d82099782708abd8e3a6fe27abbde04e71 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 11:02:31 -0400
Subject: [PATCH 3118/8313] don't close yet

---
 doc/bugs/git_annex_add_memory_leak.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/bugs/git_annex_add_memory_leak.mdwn b/doc/bugs/git_annex_add_memory_leak.mdwn
index 846e764369..57ce4c0f97 100644
--- a/doc/bugs/git_annex_add_memory_leak.mdwn
+++ b/doc/bugs/git_annex_add_memory_leak.mdwn
@@ -34,5 +34,3 @@ A history of the leaks:
   before outputting anything.
   This is Not Our Problem, but I'm sure the git developers
   would appreciate a patch that fixes it.)
-
-[[done]] ... for now --[[Joey]] 

From a40ec5e03e980b7337bf01eca1661a088ee476c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 11:20:30 -0400
Subject: [PATCH 3119/8313] Fixed a memory leak due to excessive strictness
 when committing journal files.

When hashing the files, the entire list of shas was read strictly.
That was entirely unnecessary, since there's a cleanup action run
after they're consumed.
---
 Annex/Branch.hs                         | 2 +-
 Git/HashObject.hs                       | 2 +-
 debian/changelog                        | 2 ++
 doc/bugs/git_annex_add_memory_leak.mdwn | 5 +++++
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index b2b1ed3e40..72a98ac167 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -190,7 +190,7 @@ commit message = whenM journalDirty $ lockJournal $ do
 {- Commits the staged changes in the index to the branch.
  - 
  - Ensures that the branch's index file is first updated to the state
- - of the brannch at branchref, before running the commit action. This
+ - of the branch at branchref, before running the commit action. This
  - is needed because the branch may have had changes pushed to it, that
  - are not yet reflected in the index.
  -
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 9c99dae268..ac74f02577 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -19,7 +19,7 @@ hashFiles paths repo = do
 	fileEncoding toh
 	_ <- forkProcess (feeder toh)
 	hClose toh
-	shas <- map Ref . lines <$> hGetContentsStrict fromh
+	shas <- map Ref . lines <$> hGetContents fromh
 	return (shas, ender fromh pid)
 	where
 		git_hash_object = gitCommandLine
diff --git a/debian/changelog b/debian/changelog
index 8df49d925f..23ade624c1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,6 +24,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     its head), and records the size in the key.
   * Fixed to use the strict state monad, to avoid leaking all kinds of memory
     due to lazy state update thunks when adding/fixing many files.
+  * Fixed a memory leak due to excessive strictness when committing journal
+    files.
   * whereis: Prints the urls of files that the web special remote knows about.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
diff --git a/doc/bugs/git_annex_add_memory_leak.mdwn b/doc/bugs/git_annex_add_memory_leak.mdwn
index 57ce4c0f97..b6ae60f7bd 100644
--- a/doc/bugs/git_annex_add_memory_leak.mdwn
+++ b/doc/bugs/git_annex_add_memory_leak.mdwn
@@ -27,6 +27,11 @@ A history of the leaks:
   Apparently fixed in versions afer 3.20120123, but this one is not
   well understood.
 
+* Committing journal files turned out to have another memory leak.
+  After adding a lot of files ran out of memory, this left the journal
+  behind and could affect other git-anne commands. Fixed in versions afer
+  3.20120123.
+
 * (Note that `git ls-files --others`, which is used to find files to add,
   also uses surpsisingly large amounts
   of memory when you have a lot of files. It buffers

From 0e3f7b64b6a9d92e86801a09b98c54c976fad864 Mon Sep 17 00:00:00 2001
From: antymat 
Date: Tue, 14 Feb 2012 16:34:27 +0000
Subject: [PATCH 3120/8313]

---
 doc/forum/fsck_gives_false_positives.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/fsck_gives_false_positives.mdwn

diff --git a/doc/forum/fsck_gives_false_positives.mdwn b/doc/forum/fsck_gives_false_positives.mdwn
new file mode 100644
index 0000000000..ac3bd1b462
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+I use git-annex 3.20120123 on a debian-testing amd-64 machine with software RAID6 and LVM2 on it. I needed to move the whole `/home` directory to another LV (the new LV is on encrypted PV, the old LV is encrypted and not properly aligned; I'm changing from encrypted `/home` only to encrypted everything except `/boot`), so I have user the `rsync -aAXH` from a `ro` mounted `/home` to a new LV mounted on `/mnt/home_2`. After the move was complete I run the `git annex fsck` on my (4TB of) data. The fsck finds some files bad, and moves them to the `..../bad` directory. So far so good, this is how it should be, right? But then- I have a file with sha1sum of all my files. So - I checked the 'bad' file against that. It was OK. Then I computed the SHA256 of the file - this is used by `git annex fsck`. It was OK, too. So how did it happen, that the file was marked as bad? Do I miss something here? Could it be related to the hardware (HDDs) and silent data corruption? Or is it the undesirable effect of rsync? Or maybe the fsck is at fault here?
+
+Any ideas?
+

From 33e03d58ae2a351b137ca8e32fa704d240e626e0 Mon Sep 17 00:00:00 2001
From: antymat 
Date: Tue, 14 Feb 2012 16:39:17 +0000
Subject: [PATCH 3121/8313] spelling

---
 doc/forum/fsck_gives_false_positives.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/fsck_gives_false_positives.mdwn b/doc/forum/fsck_gives_false_positives.mdwn
index ac3bd1b462..2fae57c4ed 100644
--- a/doc/forum/fsck_gives_false_positives.mdwn
+++ b/doc/forum/fsck_gives_false_positives.mdwn
@@ -1,6 +1,6 @@
 Hi,
 
-I use git-annex 3.20120123 on a debian-testing amd-64 machine with software RAID6 and LVM2 on it. I needed to move the whole `/home` directory to another LV (the new LV is on encrypted PV, the old LV is encrypted and not properly aligned; I'm changing from encrypted `/home` only to encrypted everything except `/boot`), so I have user the `rsync -aAXH` from a `ro` mounted `/home` to a new LV mounted on `/mnt/home_2`. After the move was complete I run the `git annex fsck` on my (4TB of) data. The fsck finds some files bad, and moves them to the `..../bad` directory. So far so good, this is how it should be, right? But then- I have a file with sha1sum of all my files. So - I checked the 'bad' file against that. It was OK. Then I computed the SHA256 of the file - this is used by `git annex fsck`. It was OK, too. So how did it happen, that the file was marked as bad? Do I miss something here? Could it be related to the hardware (HDDs) and silent data corruption? Or is it the undesirable effect of rsync? Or maybe the fsck is at fault here?
+I use git-annex 3.20120123 on a debian-testing amd-64 machine with software RAID6 and LVM2 on it. I needed to move the whole `/home` directory to another LV (the new LV is on encrypted PV, the old LV is encrypted and not properly aligned; I'm changing from encrypted `/home` only to encrypted everything except `/boot`), so I have used the `rsync -aAXH` from a `ro` mounted `/home` to a new LV mounted on `/mnt/home_2`. After the move was complete I run the `git annex fsck` on my (4TB of) data. The fsck finds some files bad, and moves them to the `..../bad` directory. So far so good, this is how it should be, right? But then- I have a file with sha1sum of all my files. So - I checked the 'bad' file against that. It was OK. Then I computed the SHA256 of the file - this is used by `git annex fsck`. It was OK, too. So how did it happen, that the file was marked as bad? Do I miss something here? Could it be related to the hardware (HDDs) and silent data corruption? Or is it the undesirable effect of rsync? Or maybe the fsck is at fault here?
 
 Any ideas?
 

From 90a8b38ac048d2a9a7caeb68d70d5148fb4148b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 12:40:40 -0400
Subject: [PATCH 3122/8313] set oneshot mode on a per-command basis

Avoids ugly (and test suite failing) hack in Command.Version
---
 CmdLine.hs            | 6 +++---
 Command.hs            | 7 ++++++-
 Command/ConfigList.hs | 2 +-
 Command/DropKey.hs    | 2 +-
 Command/InAnnex.hs    | 2 +-
 Command/RecvKey.hs    | 2 +-
 Command/SendKey.hs    | 2 +-
 Command/Version.hs    | 7 ++-----
 GitAnnex.hs           | 2 +-
 Types/Command.hs      | 1 +
 git-annex-shell.hs    | 2 +-
 11 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index d2adb71bbd..0bb3459124 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -28,8 +28,8 @@ type Params = [String]
 type Flags = [Annex ()]
 
 {- Runs the passed command line. -}
-dispatch :: Bool -> Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
-dispatch oneshot args cmds commonoptions header getgitrepo = do
+dispatch :: Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
+dispatch args cmds commonoptions header getgitrepo = do
 	setupConsole
 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
 	case r of
@@ -39,7 +39,7 @@ dispatch oneshot args cmds commonoptions header getgitrepo = do
 			(actions, state') <- Annex.run state $ do
 				sequence_ flags
 				prepCommand cmd params
-			tryRun state' cmd $ [startup] ++ actions ++ [shutdown oneshot]
+			tryRun state' cmd $ [startup] ++ actions ++ [shutdown $ cmdoneshot cmd]
 	where
 		(flags, cmd, params) = parseCmd args cmds commonoptions header
 
diff --git a/Command.hs b/Command.hs
index e7ce335c71..13ea167bbc 100644
--- a/Command.hs
+++ b/Command.hs
@@ -8,6 +8,7 @@
 module Command (
 	command,
 	noRepo,
+	oneShot,
 	withOptions,
 	next,
 	stop,
@@ -39,7 +40,11 @@ import Annex.CheckAttr
 
 {- Generates a normal command -}
 command :: String -> String -> [CommandSeek] -> String -> Command
-command = Command [] Nothing commonChecks
+command = Command [] Nothing commonChecks False
+
+{- Makes a command run in oneshot mode. -}
+oneShot :: Command -> Command
+oneShot c = c { cmdoneshot = True }
 
 {- Adds a fallback action to a command, that will be run if it's used
  - outside a git repository. -}
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index dcf4d15093..fc4ba91022 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.UUID
 
 def :: [Command]
-def = [command "configlist" paramNothing seek
+def = [oneShot $ command "configlist" paramNothing seek
 	"outputs relevant git configuration"]
 
 seek :: [CommandSeek]
diff --git a/Command/DropKey.hs b/Command/DropKey.hs
index aaaa224661..68fdbfdd96 100644
--- a/Command/DropKey.hs
+++ b/Command/DropKey.hs
@@ -14,7 +14,7 @@ import Logs.Location
 import Annex.Content
 
 def :: [Command]
-def = [command "dropkey" (paramRepeating paramKey) seek
+def = [oneShot $ command "dropkey" (paramRepeating paramKey) seek
 	"drops annexed content for specified keys"] 
 
 seek :: [CommandSeek]
diff --git a/Command/InAnnex.hs b/Command/InAnnex.hs
index c41f9a92c1..ad0a4d5c7c 100644
--- a/Command/InAnnex.hs
+++ b/Command/InAnnex.hs
@@ -12,7 +12,7 @@ import Command
 import Annex.Content
 
 def :: [Command]
-def = [command "inannex" (paramRepeating paramKey) seek
+def = [oneShot $ command "inannex" (paramRepeating paramKey) seek
 	"checks if keys are present in the annex"]
 
 seek :: [CommandSeek]
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index a27a5efdf6..9744a56d4a 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -14,7 +14,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 def :: [Command]
-def = [command "recvkey" paramKey seek
+def = [oneShot $ command "recvkey" paramKey seek
 	"runs rsync in server mode to receive content"]
 
 seek :: [CommandSeek]
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index 7b1cd3ecae..686a31caa7 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -13,7 +13,7 @@ import Annex.Content
 import Utility.RsyncFile
 
 def :: [Command]
-def = [command "sendkey" paramKey seek
+def = [oneShot $ command "sendkey" paramKey seek
 	"runs rsync in server mode to send content"]
 
 seek :: [CommandSeek]
diff --git a/Command/Version.hs b/Command/Version.hs
index 8761d2a2e0..af08d3d709 100644
--- a/Command/Version.hs
+++ b/Command/Version.hs
@@ -11,10 +11,9 @@ import Common.Annex
 import Command
 import qualified Build.SysConfig as SysConfig
 import Annex.Version
-import CmdLine
 
 def :: [Command]
-def = [noRepo showPackageVersion $ dontCheck repoExists $
+def = [oneShot $ noRepo showPackageVersion $ dontCheck repoExists $
 	command "version" paramNothing seek "show version info"]
 
 seek :: [CommandSeek]
@@ -29,9 +28,7 @@ start = do
 		putStrLn $ "default repository version: " ++ defaultVersion
 		putStrLn $ "supported repository versions: " ++ vs supportedVersions
 		putStrLn $ "upgrade supported from repository versions: " ++ vs upgradableVersions
-	-- avoid normal cleanup
-	_ <- shutdown True
-	liftIO exitSuccess
+	stop
 	where
 		vs = join " "
 
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 1ca89315a3..4af10a9ce4 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -129,4 +129,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch False args cmds options header Git.Construct.fromCurrent
+run args = dispatch args cmds options header Git.Construct.fromCurrent
diff --git a/Types/Command.hs b/Types/Command.hs
index 1233df2cd9..6dbcf48d16 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -36,6 +36,7 @@ data Command = Command
 	{ cmdoptions :: [Option]     -- command-specific options
 	, cmdnorepo :: Maybe (IO ()) -- an action to run when not in a repo
 	, cmdcheck :: [CommandCheck] -- check stage
+	, cmdoneshot :: Bool         -- don't save state after running
 	, cmdname :: String
 	, cmdparamdesc :: String     -- description of params for usage
 	, cmdseek :: [CommandSeek]   -- seek stage
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index e747a447b4..4fdeae1a87 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -82,7 +82,7 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
-	dispatch True (cmd : filterparams params) cmds options header $
+	dispatch (cmd : filterparams params) cmds options header $
 		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
 
 external :: [String] -> IO ()

From fa7ffd1cc3e7fb9b79820c82125f10a6349ce901 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 14 Feb 2012 16:58:33 +0000
Subject: [PATCH 3123/8313] Added a comment

---
 ...nt_1_b91070218b9d5fb687eeee1f244237ad._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment

diff --git a/doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment b/doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment
new file mode 100644
index 0000000000..c65eaf51d9
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_1_b91070218b9d5fb687eeee1f244237ad._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-02-14T16:58:33Z"
+ content="""
+Well, it should only move files to `.git/annex/bad/` if their filesize is wrong, or their checksum is wrong.
+
+You can try moving a file out of `.git/annex/bad/` and re-run fsck and see if it fails it again. (And if it does, paste in a log!) 
+
+To do that -- 
+Suppose you have a file `.git/annex/bad/SHA256-s33--5dc45521382f1c7974d9dbfcff1246370404b952` and you know that file `foobar` was supposed to have that content (you can check that `foobar` is a symlink to that SHA value). Then reinject it:
+
+`git annex reinject .git/annex/bad/SHA256-s33--5dc45521382f1c7974d9dbfcff1246370404b952 foobar`
+"""]]

From 7ebd98d8d829005c7dae38b789146d98e6800e5b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 14:35:52 -0400
Subject: [PATCH 3124/8313] fix memory leak when staging the journal

The list of files had to be retained until the end so it could be deleted.
Also, a list of update-index lines was generated and only then fed into it.
Now everything streams in constant space.
---
 Annex/Branch.hs                         | 23 ++++++------
 Git/HashObject.hs                       | 47 +++++++++++++++----------
 debian/changelog                        |  3 +-
 doc/bugs/git_annex_add_memory_leak.mdwn | 21 +++++------
 4 files changed, 52 insertions(+), 42 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 72a98ac167..f20c87b4a3 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -1,6 +1,6 @@
 {- management of the git-annex branch
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -32,7 +32,7 @@ import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.UnionMerge
-import qualified Git.HashObject
+import Git.HashObject
 import qualified Git.Index
 import Annex.CatFile
 
@@ -307,13 +307,14 @@ stageJournal = do
 	fs <- getJournalFiles
 	g <- gitRepo
 	withIndex $ liftIO $ do
-		let dir = gitAnnexJournalDir g
-		let paths = map (dir ) fs
-		(shas, cleanup) <- Git.HashObject.hashFiles paths g
-		Git.UnionMerge.update_index g $
-			index_lines shas (map fileJournal fs)
-		cleanup
-		mapM_ removeFile paths
+		h <- hashObjectStart g
+		Git.UnionMerge.stream_update_index g
+			[genstream (gitAnnexJournalDir g) h fs]
+		hashObjectStop h
 	where
-		index_lines shas = map genline . zip shas
-		genline (sha, file) = Git.UnionMerge.update_index_line sha file
+		genstream dir h fs streamer = forM_ fs $ \file -> do
+			let path = dir  file
+			sha <- hashFile h path
+			streamer $ Git.UnionMerge.update_index_line
+				sha (fileJournal file)
+			removeFile path
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index ac74f02577..200fedbd27 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -1,6 +1,6 @@
 {- git hash-object interface
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -11,23 +11,32 @@ import Common
 import Git
 import Git.Command
 
-{- Injects a set of files into git, returning the shas of the objects
- - and an IO action to call once the the shas have been used. -}
-hashFiles :: [FilePath] -> Repo -> IO ([Sha], IO ())
-hashFiles paths repo = do
-	(pid, fromh, toh) <- hPipeBoth "git" $ toCommand $ git_hash_object repo
+type HashObjectHandle = (PipeHandle, Handle, Handle)
+
+{- Starts git hash-object and returns a handle.  -}
+hashObjectStart :: Repo -> IO HashObjectHandle
+hashObjectStart repo = do
+	r@(_, _, toh) <- hPipeBoth "git" $
+		toCommand $ gitCommandLine params repo
 	fileEncoding toh
-	_ <- forkProcess (feeder toh)
-	hClose toh
-	shas <- map Ref . lines <$> hGetContents fromh
-	return (shas, ender fromh pid)
+	return r
 	where
-		git_hash_object = gitCommandLine
-			[Param "hash-object", Param "-w", Param "--stdin-paths"]
-		feeder toh = do
-			hPutStr toh $ unlines paths
-			hClose toh
-			exitSuccess
-		ender fromh pid = do
-			hClose fromh
-			forceSuccess pid
+		params =
+			[ Param "hash-object"
+			, Param "-w"
+			, Param "--stdin-paths"
+			]
+
+{- Stops git hash-object. -}
+hashObjectStop :: HashObjectHandle -> IO ()
+hashObjectStop (pid, from, to) = do
+	hClose to
+	hClose from
+	forceSuccess pid
+
+{- Injects a file into git, returning the shas of the objects. -}
+hashFile :: HashObjectHandle -> FilePath -> IO Sha
+hashFile (_, from, to) file = do
+	hPutStrLn to file
+	hFlush to
+	Ref <$> hGetLine from
diff --git a/debian/changelog b/debian/changelog
index 23ade624c1..9317a52915 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -24,8 +24,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     its head), and records the size in the key.
   * Fixed to use the strict state monad, to avoid leaking all kinds of memory
     due to lazy state update thunks when adding/fixing many files.
-  * Fixed a memory leak due to excessive strictness when committing journal
-    files.
+  * Fixed some memory leaks that occurred when committing journal files.
   * whereis: Prints the urls of files that the web special remote knows about.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
diff --git a/doc/bugs/git_annex_add_memory_leak.mdwn b/doc/bugs/git_annex_add_memory_leak.mdwn
index b6ae60f7bd..891ba318f6 100644
--- a/doc/bugs/git_annex_add_memory_leak.mdwn
+++ b/doc/bugs/git_annex_add_memory_leak.mdwn
@@ -12,26 +12,27 @@ A history of the leaks:
 * Originally, `git annex add` remembered all the files
   it had added, and fed them to git at the end. Of course
   that made its memory use grow, so it was fixed to periodically
-  flush its buffer. Affected versions: before 0.20110417
+  flush its buffer. Fixed in version 0.20110417.
 
 * Something called a "lazy state monad" caused "thunks" to build
   up and memory to leak. Also affected other git annex commands
   than `add`. Adding files using a SHA* backend hit the worst.
   Fixed in versions afer 3.20120123.
 
-* A strange GHC bug seemed to be responsible for another leak.
-  (In particular, a child process was forked. All the child did 
-  was read filenames from one pipe and shove them reformatted out
-  another pipe. For some reason, it steadily grew in size.)
-  Code was rewritten in a way that happens to avoid that leak.
-  Apparently fixed in versions afer 3.20120123, but this one is not
-  well understood.
-
 * Committing journal files turned out to have another memory leak.
   After adding a lot of files ran out of memory, this left the journal
-  behind and could affect other git-anne commands. Fixed in versions afer
+  behind and could affect other git-annex commands. Fixed in versions afer
   3.20120123.
 
+* Something is still causing a slow leak when adding files.
+  I tested by adding many copies of the whole linux kernel
+  tree into the annex using the WORM backend, and once
+  it had added 1 million files, git-annex used ~100 mb of ram.
+  That's 100 bytes leaked per file on average .. roughly the
+  size of a filename? It's worth noting that `git add` uses more memory
+  than that in such a large tree.
+  **not fixed yet**
+
 * (Note that `git ls-files --others`, which is used to find files to add,
   also uses surpsisingly large amounts
   of memory when you have a lot of files. It buffers

From 03c559f8d63f5f4e9f3472d7d2de7342a21c0a90 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 14:51:26 -0400
Subject: [PATCH 3125/8313] tweak

---
 Annex/Branch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index f20c87b4a3..388cbc12d5 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -315,6 +315,6 @@ stageJournal = do
 		genstream dir h fs streamer = forM_ fs $ \file -> do
 			let path = dir  file
 			sha <- hashFile h path
-			streamer $ Git.UnionMerge.update_index_line
+			_ <- streamer $ Git.UnionMerge.update_index_line
 				sha (fileJournal file)
 			removeFile path

From e76988f6c24277770fe1b7143e245cadab1e0ba7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 16:28:16 -0400
Subject: [PATCH 3126/8313] add

---
 doc/todo/redundancy_stats_in_status.mdwn | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/todo/redundancy_stats_in_status.mdwn

diff --git a/doc/todo/redundancy_stats_in_status.mdwn b/doc/todo/redundancy_stats_in_status.mdwn
new file mode 100644
index 0000000000..56095fd333
--- /dev/null
+++ b/doc/todo/redundancy_stats_in_status.mdwn
@@ -0,0 +1,23 @@
+Currently, `git annex status` only shows the size of 1 copy of each file.
+If numcopies is being used for redundancy, much more disk can actually be
+in use than status shows.
+
+One idea:
+
+	known annex size: 2 terabytes (plus 4 terabytes of redundant copies)
+
+But, to get that number, it would have to walk every location log, 
+counting how many copies currently exist of each file. That would make
+status a lot slower than it is.
+
+One option is to just put it at the end of the status:
+
+	redundancy: 300% (4 terabytes of copies)
+
+And ctrl-c if it's taking too long.
+
+Hmm, fsck looks at that same info. Maybe it could cache the redundancy
+level it discovers? Since fsck can be run incrementally, it would be tricky
+to get an overall number. And the number would tend to be stale, but
+then again it might also be nice if status shows how long ago the last fsck
+was.

From 29dede039ce07ce8fc2533401546658bb06d8eba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 17:19:48 -0400
Subject: [PATCH 3127/8313] add video tag with RichiH's talk

---
 doc/index.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index e0dbb69438..7e99214d94 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -42,6 +42,10 @@ files with git.
 
 ## documentation
 
+[[!template id="note" text="""
+
+]]
+
 * [[git-annex man page|git-annex]]
 * [[key-value backends|backends]] for data storage
 * [[special_remotes]] (including [[special_remotes/S3]] and [[special_remotes/bup]])

From 9da8bb2846a39fbfac7a0c248be2f34051dde346 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 17:22:56 -0400
Subject: [PATCH 3128/8313] typo

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 7e99214d94..183920b9b7 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -44,7 +44,7 @@ files with git.
 
 [[!template id="note" text="""
 
-]]
+"""]]
 
 * [[git-annex man page|git-annex]]
 * [[key-value backends|backends]] for data storage

From 7371209d13b595c427ae250ac22384d527127bbb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 17:27:13 -0400
Subject: [PATCH 3129/8313] layout

---
 doc/index.mdwn | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 183920b9b7..8bbffab4ad 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -42,10 +42,6 @@ files with git.
 
 ## documentation
 
-[[!template id="note" text="""
-
-"""]]
-
 * [[git-annex man page|git-annex]]
 * [[key-value backends|backends]] for data storage
 * [[special_remotes]] (including [[special_remotes/S3]] and [[special_remotes/bup]])
@@ -57,6 +53,11 @@ files with git.
 * [[what git annex is not|not]]
 * [[sitemap]]
 
+## talks
+
+  
+A [15 minute introduction to git-annex](http://video.fosdem.org/2012/lightningtalks/git_annex___manage_files_with_git,_without_checking_their_contents_into_git.webm), presented by Richard Hartmann.
+
 

 
 ----

From 586e937ad0e5eb0074fd188cce728bc92a1e4ef9 Mon Sep 17 00:00:00 2001
From: antymat 
Date: Tue, 14 Feb 2012 22:48:38 +0000
Subject: [PATCH 3130/8313] Added a comment

---
 ...comment_2_f51c53f3f6e6ee1ad463992657db5828._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment

diff --git a/doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment b/doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment
new file mode 100644
index 0000000000..1e0111e672
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_2_f51c53f3f6e6ee1ad463992657db5828._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="antymat"
+ ip="77.190.74.127"
+ subject="comment 2"
+ date="2012-02-14T22:48:37Z"
+ content="""
+Thanks, joey, but I still do not know, why the file that has been (and *is*) OK according to separate sha1 and sha256 checks, has been marked 'bad' by `fsck` and moved to `.git/annex/bad`. What could be a reason for that? Could have `rsync` caused it? I know too little about internal workings of `git-annex` to answer this question.
+
+But one thing I know for certain - the false positives should *not* happen, unless something *is* wrong with the file. Otherwise, if it is unreliable, if I have to check twice, it is useless. I might as well just keep checksums of all the files and do all checks by hand...
+"""]]

From c26db2625934e315163077d19d814bc71df7ef6e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 14 Feb 2012 18:50:25 -0400
Subject: [PATCH 3131/8313] add scalability page

---
 doc/index.mdwn       |  1 +
 doc/scalability.mdwn | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 doc/scalability.mdwn

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 8bbffab4ad..9ba5d5c316 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -49,6 +49,7 @@ files with git.
 * [[encryption]]
 * [[bare_repositories]]
 * [[internals]]
+* [[scalability]]
 * [[design]]
 * [[what git annex is not|not]]
 * [[sitemap]]
diff --git a/doc/scalability.mdwn b/doc/scalability.mdwn
new file mode 100644
index 0000000000..71e21ac4c2
--- /dev/null
+++ b/doc/scalability.mdwn
@@ -0,0 +1,31 @@
+git-annex is designed for scalability. The key points are:
+
+* Arbitrarily large files can be managed. The only constraint
+  on file size are how large a file your filesystem can hold.
+
+  While git-annex does checksum files by default, there
+  is a [[WORM_backend|backends]] available that avoids the checksumming
+  overhead, so you can add new, enormous files, very fast. This also
+  allows it to be used on systems with very slow disk IO.
+
+* Memory usage should be constant. This is a "should", because there
+  can sometimes be leaks (and this is one of haskell's weak spots),
+  but git-annex is designed so that it does not need to hold all
+  the details about your repository in memory.
+
+  The one exception is that [[todo/git-annex_unused_eats_memory]],
+  because it *does* need to hold the whole repo state in memory. But
+  that is still considered a bug, and hoped to be solved one day.
+  Luckily, that command is not often used.
+
+* Many files can be managed. The limiting factor is git's own
+  limitations in scaling to repositories with a lot of files, and as git
+  improves this will improve. Scaling to hundreds of thousands of files
+  is not a problem, scaling beyond that and git will start to get slow.
+
+  To some degree, git-annex works around innefficiencies in git; for
+  example it batches input sent to certian git commands that are slow
+  when run in an emormous repository.
+
+* It can use as much, or as little bandwidth as is available. In
+  particular, any interrupted file transfer can be resumed by git-annex.

From e04e05ef1b4e4dbf738698d5365c8f0507e93465 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 14 Feb 2012 22:57:29 +0000
Subject: [PATCH 3132/8313] Added a comment

---
 ...comment_3_692d6d4cd2f75a497e7d314041a768d2._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment

diff --git a/doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment b/doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment
new file mode 100644
index 0000000000..bd807f4048
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_3_692d6d4cd2f75a497e7d314041a768d2._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-02-14T22:57:29Z"
+ content="""
+All that git annex fsck does is checksum the file and move it away if the checksum fails. 
+
+If bad data was somehow read from the disk that one time, what you describe could occur. I cannot think of any other way it could happen.
+"""]]

From d380c18c1e0ebb708a94eccd6bcd688475d0190e Mon Sep 17 00:00:00 2001
From: antymat 
Date: Wed, 15 Feb 2012 07:13:12 +0000
Subject: [PATCH 3133/8313] Added a comment

---
 ...comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment

diff --git a/doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment b/doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment
new file mode 100644
index 0000000000..038d41620c
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_4_7ceb395bf8a2e6a041ccd8de63b1b6eb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="antymat"
+ ip="77.190.74.127"
+ subject="comment 4"
+ date="2012-02-15T07:13:12Z"
+ content="""
+OK, thanks. I was just wondering - since there are links in git(-annex), and a hard links too, that maybe the issue has been caused by `rsync`.
+
+I will keep my eye on that and run checks with my own checksum and `fsck` from time to time, and see what happens. I will post my results here, but the whole run (`fsck` or checksum) takes almost 2 days, so I will not do it too often... ;)
+"""]]

From 52c5b164d8dee3761b6ad96e3d636f862a2344e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 11:13:13 -0400
Subject: [PATCH 3134/8313] Added a annex.queuesize setting

useful when adding hundreds of thousands of files on a system with plenty
of memory.

git add gets quite slow in such a large repository, so if the system has
more than the ~32 mb of memory the queue can use by default, it's a useful
optimisation to increase the queue size, in order to decrease the number
of times git add is run.
---
 Annex.hs           |  4 ++--
 Annex/Queue.hs     | 22 +++++++++++++++++-----
 Git/Queue.hs       | 32 +++++++++++++++++---------------
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  8 ++++++++
 5 files changed, 46 insertions(+), 22 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 5344152071..123c9facf9 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -76,12 +76,12 @@ data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [BackendA Annex]
 	, remotes :: [Types.Remote.RemoteA Annex]
-	, repoqueue :: Git.Queue.Queue
 	, output :: OutputType
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
 	, branchstate :: BranchState
+	, repoqueue :: Maybe Git.Queue.Queue
 	, catfilehandle :: Maybe CatFileHandle
 	, checkattrhandle :: Maybe CheckAttrHandle
 	, forcebackend :: Maybe String
@@ -100,12 +100,12 @@ newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, repoqueue = Git.Queue.new
 	, output = NormalOutput
 	, force = False
 	, fast = False
 	, auto = False
 	, branchstate = startBranchState
+	, repoqueue = Nothing
 	, catfilehandle = Nothing
 	, checkattrhandle = Nothing
 	, forcebackend = Nothing
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index f611cf02eb..df6ba12a28 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -12,30 +12,42 @@ module Annex.Queue (
 ) where
 
 import Common.Annex
-import Annex
+import Annex hiding (new)
 import qualified Git.Queue
+import qualified Git.Config
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
 add command params files = do
-	q <- getState repoqueue
+	q <- get
 	store $ Git.Queue.add q command params files
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
 flushWhenFull = do
-	q <- getState repoqueue
+	q <- get
 	when (Git.Queue.full q) $ flush False
 
 {- Runs (and empties) the queue. -}
 flush :: Bool -> Annex ()
 flush silent = do
-	q <- getState repoqueue
+	q <- get
 	unless (0 == Git.Queue.size q) $ do
 		unless silent $
 			showSideAction "Recording state in git"
 		q' <- inRepo $ Git.Queue.flush q
 		store q'
 
+get :: Annex Git.Queue.Queue
+get = maybe new return =<< getState repoqueue
+
+new :: Annex Git.Queue.Queue
+new = do
+	q <- Git.Queue.new <$> fromRepo queuesize
+	store q
+	return q
+	where
+		queuesize r = readish =<< Git.Config.getMaybe "annex.queuesize" r
+
 store :: Git.Queue.Queue -> Annex ()
-store q = changeState $ \s -> s { repoqueue = q }
+store q = changeState $ \s -> s { repoqueue = Just q }
diff --git a/Git/Queue.hs b/Git/Queue.hs
index c71605ad52..b8055ab445 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -5,13 +5,15 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Git.Queue (
 	Queue,
 	new,
 	add,
 	size,
 	full,
-	flush
+	flush,
 ) where
 
 import qualified Data.Map as M
@@ -34,7 +36,11 @@ data Action = Action
 {- A queue of actions to perform (in any order) on a git repository,
  - with lists of files to perform them on. This allows coalescing 
  - similar git commands. -}
-data Queue = Queue Int (M.Map Action [FilePath])
+data Queue = Queue
+	{ size :: Int
+	, _limit :: Int
+	, _items :: M.Map Action [FilePath]
+	}
 	deriving (Show, Eq)
 
 {- A recommended maximum size for the queue, after which it should be
@@ -46,37 +52,33 @@ data Queue = Queue Int (M.Map Action [FilePath])
  - above 20k, so this is a fairly good balance -- the queue will buffer
  - only a few megabytes of stuff and a minimal number of commands will be
  - run by xargs. -}
-maxSize :: Int
-maxSize = 10240
+defaultLimit :: Int
+defaultLimit = 10240
 
 {- Constructor for empty queue. -}
-new :: Queue
-new = Queue 0 M.empty
+new :: Maybe Int -> Queue
+new lim = Queue 0 (fromMaybe defaultLimit lim) M.empty
 
 {- Adds an action to a queue. -}
 add :: Queue -> String -> [CommandParam] -> [FilePath] -> Queue
-add (Queue n m) subcommand params files = Queue (n + 1) m'
+add (Queue cur lim m) subcommand params files = Queue (cur + 1) lim m'
 	where
 		action = Action subcommand params
 		-- There are probably few items in the map, but there
 		-- can be a lot of files per item. So, optimise adding
 		-- files.
 		m' = M.insertWith' const action fs m
-		fs = files ++ M.findWithDefault [] action m
-
-{- Number of items in a queue. -}
-size :: Queue -> Int
-size (Queue n _) = n
+		!fs = files ++ M.findWithDefault [] action m
 
 {- Is a queue large enough that it should be flushed? -}
 full :: Queue -> Bool
-full (Queue n _) = n > maxSize
+full (Queue cur lim  _) = cur > lim
 
 {- Runs a queue on a git repository. -}
 flush :: Queue -> Repo -> IO Queue
-flush (Queue _ m) repo = do
+flush (Queue _ lim m) repo = do
 	forM_ (M.toList m) $ uncurry $ runAction repo
-	return new
+	return $ Queue 0 lim M.empty
 
 {- Runs an Action on a list of files in a git repository.
  -
diff --git a/debian/changelog b/debian/changelog
index 9317a52915..28d253704c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -26,6 +26,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     due to lazy state update thunks when adding/fixing many files.
   * Fixed some memory leaks that occurred when committing journal files.
   * whereis: Prints the urls of files that the web special remote knows about.
+  * Added a annex.queuesize setting, useful when adding hundreds of thousands
+    of files on a system with plenty of memory.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 9232bf0208..d4e62568f8 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -576,6 +576,14 @@ Here are all the supported configuration settings.
 
   The default reserve is 1 megabyte.
 
+* `annex.queuesize`
+
+  git-annex builds a queue of git commands, in order to combine similar
+  commands for speed. By default the size of the queue is limited to
+  10240 commands; this can be used to change the size. If you have plenty
+  of memory and are working with very large numbers of files, increasing
+  the queue size can speed it up.
+
 * `annex.version`
 
   Automatically maintained, and used to automate upgrades between versions.

From 623a42b0e922d08fd8672892508d7ac32f2d8225 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 15 Feb 2012 15:22:56 +0000
Subject: [PATCH 3135/8313] Added a comment

---
 ...ment_5_86484a504c3bbcecd5876982b9c95688._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment

diff --git a/doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment b/doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment
new file mode 100644
index 0000000000..7f0fbf96a0
--- /dev/null
+++ b/doc/forum/fsck_gives_false_positives/comment_5_86484a504c3bbcecd5876982b9c95688._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2012-02-15T15:22:56Z"
+ content="""
+The symlinks are in the git repository. So if the rsync damanged one, git would see the change. And nothing that happens to the symlinks can affect fsck.
+
+git-annex does not use hard links at all.
+
+fsck corrects mangled file permissions. It is possible to screw up the permissions so badly that it cannot see the files at all (ie, chmod 000 on a file under .git/annex/objects), but then fsck will complain and give up, not move the files to bad.
+So I don't see how a botched rsync could result in fsck moving a file with correct content to bad.
+"""]]

From f0f07db01de13a6da8f0fd50532c5cb004e82d81 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 13:59:09 -0400
Subject: [PATCH 3136/8313] reorder prams and put -- after atrributes, for
 compatability with old git (cherry picked from commit
 c8ec0e233e9d47a7e69b3de2952099c221c79ac1)

---
 Git/CheckAttr.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 8de19390f1..696d8aafd3 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -28,9 +28,10 @@ checkAttrStart attrs repo = do
 	return (pid, from, to, attrs, cwd)
 	where
 		params =
-			[ Param "check-attr" ]
-			++ map Param attrs ++
-			[ Params "-z --stdin" ]
+			[ Param "check-attr" 
+			, Params "-z --stdin"
+			] ++ map Param attrs ++
+			[ Param "--" ]
 
 {- Stops git check-attr. -}
 checkAttrStop :: CheckAttrHandle -> IO ()

From 4645f836781fd1db91c2b64138d441508a2bb847 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 14:34:40 -0400
Subject: [PATCH 3137/8313] add tips

---
 doc/scalability.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/doc/scalability.mdwn b/doc/scalability.mdwn
index 71e21ac4c2..232a84cc67 100644
--- a/doc/scalability.mdwn
+++ b/doc/scalability.mdwn
@@ -29,3 +29,16 @@ git-annex is designed for scalability. The key points are:
 
 * It can use as much, or as little bandwidth as is available. In
   particular, any interrupted file transfer can be resumed by git-annex.
+
+## scalability tips
+
+* If the files are so big that checksumming becomes a bottleneck, consider
+  using the [[WORM_backend|backends]]. You can always `git annex migrate`
+  files to a checksumming backend later on.
+
+* If you're adding a huge number of files at once (hundreds of thousands),
+  you'll soon notice that git-annex periodically stops and say
+  "Recording state in git" while it runs a `git add` command that
+  becomes increasingly expensive. Consider adjusting the `annex.queuesize`
+  to a higher value, at the expense of it using more memory.
+

From 505d6b1a06df2d84fd26d9f29025050b7944618a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 14:35:49 -0400
Subject: [PATCH 3138/8313] fix failure count memory leak

This is the last memory leak that prevents git-annex from running
in constant space, as far as I can see. I can now run git annex find
dummied up to repeatedly find the same file over and over, on millions
olf files, and memory stays entirely constant.
---
 CmdLine.hs                              |  4 +++-
 doc/bugs/git_annex_add_memory_leak.mdwn | 13 +++++--------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 0bb3459124..fbc1eaeca0 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -80,7 +80,9 @@ tryRun' errnum state cmd (a:as) = do
 			a
 		handle (Left err) = showerr err >> cont False state
 		handle (Right (success, state')) = cont success state'
-		cont success s = tryRun' (if success then errnum else errnum + 1) s cmd as
+		cont success s = do
+			let errnum' = if success then errnum else errnum + 1
+			(tryRun' $! errnum') s cmd as
 		showerr err = Annex.eval state $ do
 			showErr err
 			showEndFail
diff --git a/doc/bugs/git_annex_add_memory_leak.mdwn b/doc/bugs/git_annex_add_memory_leak.mdwn
index 891ba318f6..4bcffb17bf 100644
--- a/doc/bugs/git_annex_add_memory_leak.mdwn
+++ b/doc/bugs/git_annex_add_memory_leak.mdwn
@@ -24,14 +24,9 @@ A history of the leaks:
   behind and could affect other git-annex commands. Fixed in versions afer
   3.20120123.
 
-* Something is still causing a slow leak when adding files.
-  I tested by adding many copies of the whole linux kernel
-  tree into the annex using the WORM backend, and once
-  it had added 1 million files, git-annex used ~100 mb of ram.
-  That's 100 bytes leaked per file on average .. roughly the
-  size of a filename? It's worth noting that `git add` uses more memory
-  than that in such a large tree.
-  **not fixed yet**
+* The count of the number of failed commands was updated lazily, which
+  caused a slow leak when running on a lot of files. Fixed in versions afer
+  3.20120123.
 
 * (Note that `git ls-files --others`, which is used to find files to add,
   also uses surpsisingly large amounts
@@ -40,3 +35,5 @@ A history of the leaks:
   before outputting anything.
   This is Not Our Problem, but I'm sure the git developers
   would appreciate a patch that fixes it.)
+
+[[done]]

From 63152428e9cdb7ed7d4e0ae209d539fe2afd23f0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 17:33:21 -0400
Subject: [PATCH 3139/8313] changelog

---
 debian/changelog | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 28d253704c..dc792a6e71 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,7 +8,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     used by git-annex-shell and other places where changes are made to
     a remote's location log.
   * Modifications to support ghc 7.4's handling of filenames.
-    This version can only be built with ghc 7.4 or newer.
+    This version can only be built with ghc 7.4 or newer. See the ghc7.0
+    branch for older ghcs.
   * S3: Fix irrefutable pattern failure when accessing encrypted S3
     credentials.
   * addurl: Added a --file option, which can be used to specify what

From 4d8afc1713cdd530f2cd2a42746e81da69a28203 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 19:43:15 -0400
Subject: [PATCH 3140/8313] tweak wording

---
 debian/control   | 2 +-
 doc/summary.mdwn | 2 +-
 git-annex.cabal  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/control b/debian/control
index 5d5956de8d..983c3da1ed 100644
--- a/debian/control
+++ b/debian/control
@@ -42,7 +42,7 @@ Description: manage files with git, without checking their contents into git
  git-annex allows managing files with git, without checking the file
  contents into git. While that may seem paradoxical, it is useful when
  dealing with files larger than git can currently easily handle, whether due
- to limitations in memory, checksumming time, or disk space.
+ to limitations in memory, time, or disk space.
  .
  Even without file content tracking, being able to manage files with git,
  move files around and delete files with versioned directory trees, and use
diff --git a/doc/summary.mdwn b/doc/summary.mdwn
index 458eaab56d..25c95b3b5c 100644
--- a/doc/summary.mdwn
+++ b/doc/summary.mdwn
@@ -1,7 +1,7 @@
 git-annex allows managing files with git, without checking the file
 contents into git. While that may seem paradoxical, it is useful when
 dealing with files larger than git can currently easily handle, whether due
-to limitations in memory, checksumming time, or disk space.
+to limitations in memory, time, or disk space.
 
 Even without file content tracking, being able to manage files with git,
 move files around and delete files with versioned directory trees, and use
diff --git a/git-annex.cabal b/git-annex.cabal
index 0c343e42c9..8bfb3ac0c9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -16,7 +16,7 @@ Description:
  git-annex allows managing files with git, without checking the file
  contents into git. While that may seem paradoxical, it is useful when
  dealing with files larger than git can currently easily handle, whether due
- to limitations in memory, checksumming time, or disk space.
+ to limitations in memory, time, or disk space.
  .
  Even without file content tracking, being able to manage files with git,
  move files around and delete files with versioned directory trees, and use

From 7d1c09fe4a4b54b68c7a446659cc6b7f6cec7252 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 15 Feb 2012 19:46:29 -0400
Subject: [PATCH 3141/8313] update

---
 doc/future_proofing.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/future_proofing.mdwn b/doc/future_proofing.mdwn
index a7bcce37c9..afaf0b5213 100644
--- a/doc/future_proofing.mdwn
+++ b/doc/future_proofing.mdwn
@@ -21,7 +21,8 @@ problem:
   formats.
 
 * What filesystem is used on the drive? Will that filesystem still be
-  available?
+  available? Whatever you choose to use, git-annex can put files on it.
+  Even if you choose (ugh) FAT.
 
 * What is the hardware interface of the drive? Will hardware still exist
   to talk to it?

From e7aaa55c53fb54c6dd5a1a1aeb0955b05227676b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 00:05:17 -0400
Subject: [PATCH 3142/8313] create parent directories as needed for addurl
 --file

---
 Command/AddUrl.hs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index f91d6dd553..e23f4262b9 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -46,8 +46,7 @@ perform :: String -> FilePath -> CommandPerform
 perform url file = ifAnnexed file addurl geturl
 	where
 		geturl = do
-			whenM (liftIO $ doesFileExist file) $
-				error $ "not overwriting existing " ++ file
+			liftIO $ createDirectoryIfMissing True (parentDir file)
 			fast <- Annex.getState Annex.fast
 			if fast then nodownload url file else download url file
 		addurl (key, _backend) = do

From a1e52f0ce5984058c737ed709fc5d4b6398e019a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 00:41:30 -0400
Subject: [PATCH 3143/8313] hlint

---
 Annex/Branch.hs     |  4 ++--
 Annex/Ssh.hs        |  8 ++++----
 Backend.hs          |  2 +-
 Command/Drop.hs     |  2 +-
 Command/Find.hs     |  2 +-
 Command/Get.hs      |  4 ++--
 Command/Log.hs      |  8 ++++----
 Command/Reinject.hs |  2 +-
 Command/Sync.hs     |  6 +++---
 Command/Unused.hs   |  2 +-
 Command/Whereis.hs  |  4 ++--
 Init.hs             |  2 +-
 Logs/Remote.hs      |  2 +-
 Messages.hs         |  2 +-
 Option.hs           |  2 +-
 Remote.hs           |  2 +-
 Utility/Format.hs   |  2 +-
 Utility/Url.hs      | 17 ++++++++---------
 18 files changed, 36 insertions(+), 37 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 388cbc12d5..acab417fb3 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -116,7 +116,7 @@ updateTo pairs = do
 	-- check what needs updating before taking the lock
 	dirty <- journalDirty
 	(refs, branches) <- unzip <$> filterM isnewer pairs
-	if (not dirty && null refs)
+	if not dirty && null refs
 		then updateIndex branchref
 		else withIndex $ lockJournal $ do
 			when dirty stageJournal
@@ -172,7 +172,7 @@ get' staleok file = fromcache =<< getCache file
  - modifes the current content of the file on the branch.
  -}
 change :: FilePath -> (String -> String) -> Annex ()
-change file a = lockJournal $ getStale file >>= return . a >>= set file
+change file a = lockJournal $ a <$> getStale file >>= set file
 
 {- Records new content of a file into the journal and cache. -}
 set :: FilePath -> String -> Annex ()
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 184eb92caa..df9f0e410c 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -32,7 +32,7 @@ sshParams (host, port) opts = go =<< sshInfo (host, port)
 		-- If the lock pool is empty, this is the first ssh of this
 		-- run. There could be stale ssh connections hanging around
 		-- from a previous git-annex run that was interrupted.
-		cleanstale = whenM (null . filter isLock . M.keys <$> getPool) $
+		cleanstale = whenM (not . any isLock . M.keys <$> getPool) $
 			sshCleanup
 
 sshInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam])
@@ -40,9 +40,9 @@ sshInfo (host, port) = do
 	caching <- Git.configTrue <$> fromRepo (Git.Config.get "annex.sshcaching" "true")
 	if caching
 		then do
-			dir <- fromRepo $ gitAnnexSshDir
+			dir <- fromRepo gitAnnexSshDir
 			let socketfile = dir  hostport2socket host port
-		 	return $ (Just socketfile, cacheParams socketfile)
+		 	return (Just socketfile, cacheParams socketfile)
 		else return (Nothing, [])
 
 cacheParams :: FilePath -> [CommandParam]
@@ -58,7 +58,7 @@ portParams (Just port) = [Param "-p", Param $ show port]
 {- Stop any unused ssh processes. -}
 sshCleanup :: Annex ()
 sshCleanup = do
-	dir <- fromRepo $ gitAnnexSshDir
+	dir <- fromRepo gitAnnexSshDir
 	liftIO $ createDirectoryIfMissing True dir
 	sockets <- filter (not . isLock) <$> liftIO (dirContents dir)
 	forM_ sockets cleanup
diff --git a/Backend.hs b/Backend.hs
index 4c28f1c779..6810c3a44b 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -60,7 +60,7 @@ genKey file trybackend = do
 genKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend))
 genKey' [] _ = return Nothing
 genKey' (b:bs) file = do
-	r <- (B.getKey b) file
+	r <- B.getKey b file
 	case r of
 		Nothing -> genKey' bs file
 		Just k -> return $ Just (makesane k, b)
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 9eb36a22fe..28a52d6262 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -30,7 +30,7 @@ seek = [withField fromOption Remote.byName $ \from ->
 	withFilesInGit $ whenAnnexed $ start from]
 
 start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart
-start from file (key, _) = autoCopies file key (>) $ \numcopies -> do
+start from file (key, _) = autoCopies file key (>) $ \numcopies ->
 	case from of
 		Nothing -> startLocal file numcopies key
 		Just remote -> do
diff --git a/Command/Find.hs b/Command/Find.hs
index 902f50d2e3..33f512e39f 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -36,7 +36,7 @@ seek :: [CommandSeek]
 seek = [withField formatOption formatconverter $ \f ->
 		withFilesInGit $ whenAnnexed $ start f]
 	where
-		formatconverter = return . maybe Nothing (Just . Utility.Format.gen)
+		formatconverter = return . fmap Utility.Format.gen
 
 start :: Maybe Utility.Format.Format -> FilePath -> (Key, Backend) -> CommandStart
 start format file (key, _) = do
diff --git a/Command/Get.hs b/Command/Get.hs
index 928ab0f1be..9b12b95994 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -23,7 +23,7 @@ seek = [withField Command.Move.fromOption Remote.byName $ \from ->
 
 start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart
 start from file (key, _) = stopUnless (not <$> inAnnex key) $
-	autoCopies file key (<) $ \_numcopies -> do
+	autoCopies file key (<) $ \_numcopies ->
 		case from of
 			Nothing -> go $ perform key
 			Just src -> do
@@ -36,7 +36,7 @@ start from file (key, _) = stopUnless (not <$> inAnnex key) $
 			next a	
 
 perform :: Key -> CommandPerform
-perform key = stopUnless (getViaTmp key $ getKeyFile key) $ do
+perform key = stopUnless (getViaTmp key $ getKeyFile key) $
 	next $ return True -- no cleanup needed
 
 {- Try to find a copy of the file in one of the remotes,
diff --git a/Command/Log.hs b/Command/Log.hs
index 4013b535ef..d78b602067 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -55,7 +55,7 @@ gourceOption :: Option
 gourceOption = Option.flag [] "gource" "format output for gource"
 
 seek :: [CommandSeek]
-seek = [withValue (Remote.uuidDescriptions) $ \m ->
+seek = [withValue Remote.uuidDescriptions $ \m ->
 	withValue (liftIO getCurrentTimeZone) $ \zone ->
 	withValue (concat <$> mapM getoption passthruOptions) $ \os ->
 	withFlag gourceOption $ \gource ->
@@ -65,7 +65,7 @@ seek = [withValue (Remote.uuidDescriptions) $ \m ->
 			Annex.getField (Option.name o)
 		use o v = [Param ("--" ++ Option.name o), Param v]
 
-start :: (M.Map UUID String) -> TimeZone -> [CommandParam] -> Bool ->
+start :: M.Map UUID String -> TimeZone -> [CommandParam] -> Bool ->
 	FilePath -> (Key, Backend) -> CommandStart
 start m zone os gource file (key, _) = do
 	showLog output =<< readLog <$> getLog key os
@@ -91,7 +91,7 @@ showLog outputter ps = do
 			catObject ref
 
 normalOutput :: (UUID -> String) -> FilePath -> TimeZone -> Outputter
-normalOutput lookupdescription file zone present ts us = do
+normalOutput lookupdescription file zone present ts us =
 	liftIO $ mapM_ (putStrLn . format) us
 	where
 		time = showTimeStamp zone ts
@@ -100,7 +100,7 @@ normalOutput lookupdescription file zone present ts us = do
 			fromUUID u ++ " -- " ++ lookupdescription u ]
 
 gourceOutput :: (UUID -> String) -> FilePath -> Outputter
-gourceOutput lookupdescription file present ts us = do
+gourceOutput lookupdescription file present ts us =
 	liftIO $ mapM_ (putStrLn . intercalate "|" . format) us
 	where
 		time = takeWhile isDigit $ show ts
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 480806e11c..bb277af2c6 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -23,7 +23,7 @@ seek = [withWords start]
 start :: [FilePath] -> CommandStart
 start (src:dest:[])
 	| src == dest = stop
-	| otherwise = do
+	| otherwise =
 		ifAnnexed src
 			(error $ "cannot used annexed file as src: " ++ src)
 			go
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 3d541c4dea..8e237ae84d 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -33,7 +33,7 @@ seek :: CommandSeek
 seek rs = do
 	!branch <- fromMaybe nobranch <$> inRepo Git.Branch.current
 	remotes <- syncRemotes rs
-	return $ concat $
+	return $ concat
 		[ [ commit ]
 		, [ mergeLocal branch ]
 		, [ pullRemote remote branch | remote <- remotes ]
@@ -137,9 +137,9 @@ pushRemote remote branch = go =<< needpush
 			showStart "push" (Remote.name remote)
 			next $ next $ do
 				showOutput
-				inRepo $ Git.Command.runBool "push" $
+				inRepo $ Git.Command.runBool "push"
 					[ Param (Remote.name remote)
-					, Param (show $ Annex.Branch.name)
+					, Param (show Annex.Branch.name)
 					, Param refspec
 					]
 		refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch)
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1c82b9ae4a..58a99882eb 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -39,7 +39,7 @@ fromOption :: Option
 fromOption = Option.field ['f'] "from" paramRemote "remote to check for unused content"
 
 seek :: [CommandSeek]
-seek = [withNothing $ start]
+seek = [withNothing start]
 
 {- Finds unused content in the annex. -} 
 start :: CommandStart
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index f62d34642f..d4d268d937 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -22,12 +22,12 @@ seek :: [CommandSeek]
 seek = [withValue (remoteMap id) $ \m ->
 	withFilesInGit $ whenAnnexed $ start m]
 
-start :: (M.Map UUID Remote) -> FilePath -> (Key, Backend) -> CommandStart
+start :: M.Map UUID Remote -> FilePath -> (Key, Backend) -> CommandStart
 start remotemap file (key, _) = do
 	showStart "whereis" file
 	next $ perform remotemap key
 
-perform :: (M.Map UUID Remote) -> Key -> CommandPerform
+perform :: M.Map UUID Remote -> Key -> CommandPerform
 perform remotemap key = do
 	locations <- keyLocations key
 	(untrustedlocations, safelocations) <- trustPartition UnTrusted locations
diff --git a/Init.hs b/Init.hs
index c8deadf3b4..c9d5bb909a 100644
--- a/Init.hs
+++ b/Init.hs
@@ -68,7 +68,7 @@ gitPreCommitHookUnWrite = unlessBare $ do
 				" Edit it to remove call to git annex."
 
 unlessBare :: Annex () -> Annex ()
-unlessBare = unlessM $ fromRepo $ Git.repoIsLocalBare
+unlessBare = unlessM $ fromRepo Git.repoIsLocalBare
 
 preCommitHook :: Annex FilePath
 preCommitHook = () <$> fromRepo Git.gitDir <*> pure "hooks/pre-commit"
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index d9b41d8c47..c38a05c189 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -79,7 +79,7 @@ configUnEscape = unescape
 				num = takeWhile isNumber s
 				r = drop (length num) s
 				rest = drop 1 r
-				ok = not (null num) && take 1 r == ";"
+				ok = not (null num) && ":" `isPrefixOf` r
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool
diff --git a/Messages.hs b/Messages.hs
index 982b9313cf..1b51cf23ea 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -132,7 +132,7 @@ handle json normal = Annex.getState Annex.output >>= go
 	where
 		go Annex.NormalOutput = liftIO normal
 		go Annex.QuietOutput = q
-		go Annex.JSONOutput = liftIO $ flushed $ json
+		go Annex.JSONOutput = liftIO $ flushed json
 
 q :: Monad m => m ()
 q = return ()
diff --git a/Option.hs b/Option.hs
index d6d8b44a32..2f0d00744d 100644
--- a/Option.hs
+++ b/Option.hs
@@ -37,7 +37,7 @@ common =
 		"allow verbose output (default)"
 	, Option ['j'] ["json"] (NoArg (setoutput Annex.JSONOutput))
 		"enable JSON output"
-	, Option ['d'] ["debug"] (NoArg (setdebug))
+	, Option ['d'] ["debug"] (NoArg setdebug)
 		"show debug messages"
 	, Option ['b'] ["backend"] (ReqArg setforcebackend paramName)
 		"specify key-value backend to use"
diff --git a/Remote.hs b/Remote.hs
index 861319e083..f676ddddcf 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -215,4 +215,4 @@ forceTrust level remotename = do
  - key to the remote, or removing the key from it *may* log the change
  - on the remote, but this cannot always be relied on. -}
 logStatus :: Remote -> Key -> LogStatus -> Annex ()
-logStatus remote key present = logChange key (uuid remote) present
+logStatus remote key = logChange key (uuid remote)
diff --git a/Utility/Format.hs b/Utility/Format.hs
index 79e94ae963..1d96695ed5 100644
--- a/Utility/Format.hs
+++ b/Utility/Format.hs
@@ -94,7 +94,7 @@ gen = filter (not . empty) . fuse [] . scan [] . decode_c
 			| i < 0 = LeftJustified (-1 * i)
 			| otherwise = RightJustified i
 		novar v = "${" ++ reverse v
-		foundvar f v p cs = scan (Var (reverse v) p : f) cs
+		foundvar f v p = scan (Var (reverse v) p : f)
 
 empty :: Frag -> Bool
 empty (Const "") = True
diff --git a/Utility/Url.hs b/Utility/Url.hs
index dfdebaf06a..8a43cf788d 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -108,12 +108,11 @@ request url requesttype = go 5 url
 				(3,0,x) | x /= 5 -> redir (n - 1) u rsp
 				_ -> return rsp
 		ignore = const $ return ()
-		redir n u rsp = do
-			case retrieveHeaders HdrLocation rsp of
-				[] -> return rsp
-				(Header _ newu:_) ->
-					case parseURIReference newu of
-						Nothing -> return rsp
-						Just newURI -> go n newURI_abs
-							where
-								newURI_abs = fromMaybe newURI (newURI `relativeTo` u)
+		redir n u rsp = case retrieveHeaders HdrLocation rsp of
+			[] -> return rsp
+			(Header _ newu:_) ->
+				case parseURIReference newu of
+					Nothing -> return rsp
+					Just newURI -> go n newURI_abs
+						where
+							newURI_abs = fromMaybe newURI (newURI `relativeTo` u)

From 8f9b501515d215ac2befab51773b63bf0f180d5d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 02:05:06 -0400
Subject: [PATCH 3144/8313] handle really long urls

Using the whole url as a key can make the filename too long. Truncate
and use a md5sum for uniqueness if necessary.
---
 Backend/URL.hs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/Backend/URL.hs b/Backend/URL.hs
index b3411bac5b..b98974cb45 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -10,6 +10,8 @@ module Backend.URL (
 	fromUrl
 ) where
 
+import Data.Hash.MD5
+
 import Common.Annex
 import Types.Backend
 import Types.Key
@@ -26,7 +28,14 @@ backend = Backend {
 
 fromUrl :: String -> Maybe Integer -> Key
 fromUrl url size = stubKey
-	{ keyName = url
+	{ keyName = key
 	, keyBackendName = "URL"
 	, keySize = size
 	}
+	where
+		-- when it's not too long, use the url as the key name
+		-- 256 is the absolute filename max, but use a shorter
+		-- length because this is not the entire key filename.
+		key
+			| length url < 128 = url
+			| otherwise = take 128 url ++ "-" ++ md5s (Str url)

From a86d937b5b4dd8348bdf3d08ceea7cfe1aa43668 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 02:09:09 -0400
Subject: [PATCH 3145/8313] avoid too long filename when making up a filename
 for addurl too

---
 Command/AddUrl.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index e23f4262b9..981af2f7e6 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -81,7 +81,7 @@ nodownload url file = do
 	next $ Command.Add.cleanup file key False
 
 url2file :: URI -> FilePath
-url2file url = escape $ uriRegName auth ++ uriPath url ++ uriQuery url
+url2file url = take 255 $ escape $ uriRegName auth ++ uriPath url ++ uriQuery url
 	where
 		escape = replace "/" "_" . replace "?" "_"
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url

From 39c3f56b339fcaad3f91530f1f2dce0a0783d782 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 12:25:19 -0400
Subject: [PATCH 3146/8313] addurl: Add --pathdepth option.

---
 Command/AddUrl.hs  | 37 ++++++++++++++++++++++++++-----------
 debian/changelog   |  1 +
 doc/git-annex.mdwn | 13 +++++++++----
 3 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 981af2f7e6..a6c89542e0 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -22,34 +22,40 @@ import qualified Option
 import Types.Key
 
 def :: [Command]
-def = [withOptions [fileOption] $
+def = [withOptions [fileOption, pathdepthOption] $
 	command "addurl" (paramRepeating paramUrl) seek "add urls to annex"]
 
 fileOption :: Option
 fileOption = Option.field [] "file" paramFile "specify what file the url is added to"
 
+pathdepthOption :: Option
+pathdepthOption = Option.field [] "pathdepth" paramFile "number of path components to use in filename"
+
 seek :: [CommandSeek]
 seek = [withField fileOption return $ \f ->
-	withStrings $ start f]
+	withField pathdepthOption (return . maybe Nothing readish) $ \d ->
+	withStrings $ start f d]
 
-start :: Maybe FilePath -> String -> CommandStart
-start optfile s = notBareRepo $ go $ fromMaybe bad $ parseURI s
+start :: Maybe FilePath -> Maybe Int -> String -> CommandStart
+start optfile pathdepth s = notBareRepo $ go $ fromMaybe bad $ parseURI s
 	where
 		bad = fromMaybe (error $ "bad url " ++ s) $
 			parseURI $ escapeURIString isUnescapedInURI s
 		go url = do
-			let file = fromMaybe (url2file url) optfile
+			let file = fromMaybe (url2file url pathdepth) optfile
 			showStart "addurl" file
-			next $ perform s file
+			next $ perform s file pathdepth
 
-perform :: String -> FilePath -> CommandPerform
-perform url file = ifAnnexed file addurl geturl
+perform :: String -> FilePath -> Maybe Int -> CommandPerform
+perform url file pathdepth = ifAnnexed file addurl geturl
 	where
 		geturl = do
 			liftIO $ createDirectoryIfMissing True (parentDir file)
 			fast <- Annex.getState Annex.fast
 			if fast then nodownload url file else download url file
 		addurl (key, _backend) = do
+			when (pathdepth /= Nothing) $
+				error $ file ++ " already exists"
 			unlessM (liftIO $ Url.check url (keySize key)) $
 				error $ "failed to verify url: " ++ url
 			setUrlPresent key url
@@ -80,8 +86,17 @@ nodownload url file = do
 	setUrlPresent key url
 	next $ Command.Add.cleanup file key False
 
-url2file :: URI -> FilePath
-url2file url = take 255 $ escape $ uriRegName auth ++ uriPath url ++ uriQuery url
+url2file :: URI -> Maybe Int -> FilePath
+url2file url pathdepth = case pathdepth of
+	Nothing -> filesize $ escape fullurl
+	Just depth
+		| depth > 0 -> filesize $ join "/" $
+			fromend depth $ map escape $
+			filter (not . null) $ split "/" fullurl
+		| otherwise -> error "bad --pathdepth value"
 	where
-		escape = replace "/" "_" . replace "?" "_"
+		fullurl = uriRegName auth ++ uriPath url ++ uriQuery url
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url
+		filesize = take 255
+		escape = replace "/" "_" . replace "?" "_"
+		fromend n = reverse . take n . reverse
diff --git a/debian/changelog b/debian/changelog
index dc792a6e71..9ca8752170 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -29,6 +29,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * whereis: Prints the urls of files that the web special remote knows about.
   * Added a annex.queuesize setting, useful when adding hundreds of thousands
     of files on a system with plenty of memory.
+  * addurl: Add --pathdepth option.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d4e62568f8..d5814a2c24 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -149,10 +149,15 @@ subdirectories).
 
   To avoid immediately downloading the url, specify --fast.
 
-  To specify what file the url is added to, specify --file. This changes
-  the behavior; now all the specified urls are recorded as alternate
-  locations from which the file can be downloaded. In this mode, addurl
-  can be used both to add new files, or to add urls to existing files.
+  Normally the filename is based on the full url, so will look like
+  "www.example.com_subdir_bigfile". For a shorter filename, specify
+  --pathdepth=N. For example, --pathdepth=2 will use "subdir/bigfile",
+  while --parhdepth=1 will use "bigfile".
+
+  Or, to directly specify what file the url is added to, specify --file.
+  This changes the behavior; now all the specified urls are recorded as
+  alternate locations from which the file can be downloaded. In this mode,
+  addurl can be used both to add new files, or to add urls to existing files.
 
 # REPOSITORY SETUP COMMANDS
 

From c2245260b1e6e3bd9a8827449fea2c0eafe7e0f2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 12:37:30 -0400
Subject: [PATCH 3147/8313] improve usage

---
 Command/AddUrl.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index a6c89542e0..9867369438 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -29,7 +29,7 @@ fileOption :: Option
 fileOption = Option.field [] "file" paramFile "specify what file the url is added to"
 
 pathdepthOption :: Option
-pathdepthOption = Option.field [] "pathdepth" paramFile "number of path components to use in filename"
+pathdepthOption = Option.field [] "pathdepth" paramNumber "path components to use in filename"
 
 seek :: [CommandSeek]
 seek = [withField fileOption return $ \f ->

From 346c9344094635edbcf85ac6ddb27b8235038d8a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 14:26:53 -0400
Subject: [PATCH 3148/8313] allow pathdepth to drop from the front or take from
 the end (negative)

---
 Command/AddUrl.hs  | 9 ++++-----
 doc/git-annex.mdwn | 5 +++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 9867369438..0b41310677 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -90,13 +90,12 @@ url2file :: URI -> Maybe Int -> FilePath
 url2file url pathdepth = case pathdepth of
 	Nothing -> filesize $ escape fullurl
 	Just depth
-		| depth > 0 -> filesize $ join "/" $
-			fromend depth $ map escape $
-			filter (not . null) $ split "/" fullurl
-		| otherwise -> error "bad --pathdepth value"
+		| depth > 0 -> frombits $ drop depth
+		| otherwise -> frombits $ reverse . take (negate depth) . reverse
 	where
 		fullurl = uriRegName auth ++ uriPath url ++ uriQuery url
+		frombits a = filesize $ join "/" $ a urlbits
+		urlbits = map escape $ filter (not . null) $ split "/" fullurl
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url
 		filesize = take 255
 		escape = replace "/" "_" . replace "?" "_"
-		fromend n = reverse . take n . reverse
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index d5814a2c24..f13baf79ca 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -150,9 +150,10 @@ subdirectories).
   To avoid immediately downloading the url, specify --fast.
 
   Normally the filename is based on the full url, so will look like
-  "www.example.com_subdir_bigfile". For a shorter filename, specify
+  "www.example.com_dir_subdir_bigfile". For a shorter filename, specify
   --pathdepth=N. For example, --pathdepth=2 will use "subdir/bigfile",
-  while --parhdepth=1 will use "bigfile".
+  while --pathdepth=3 will use "bigfile". It can also be negative;
+  --pathdepth=-2 will use the last two parts of the url.
 
   Or, to directly specify what file the url is added to, specify --file.
   This changes the behavior; now all the specified urls are recorded as

From d05550e8039dc38b6c83c88aef7da8a593b7c5aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 14:28:17 -0400
Subject: [PATCH 3149/8313] zero still bad

---
 Command/AddUrl.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 0b41310677..f87417d5d1 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -91,7 +91,8 @@ url2file url pathdepth = case pathdepth of
 	Nothing -> filesize $ escape fullurl
 	Just depth
 		| depth > 0 -> frombits $ drop depth
-		| otherwise -> frombits $ reverse . take (negate depth) . reverse
+		| depth < 0 -> frombits $ reverse . take (negate depth) . reverse
+		| otherwise -> error "bad --pathdepth"
 	where
 		fullurl = uriRegName auth ++ uriPath url ++ uriQuery url
 		frombits a = filesize $ join "/" $ a urlbits

From aeaaa0ff87e22c3a5b4c9657544224c9db006cde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 15:07:59 -0400
Subject: [PATCH 3150/8313] reorder

---
 debian/changelog | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 9ca8752170..24f40fddc2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,10 @@
 git-annex (3.20120124) UNRELEASED; urgency=low
 
+  * Modifications to support ghc 7.4's handling of filenames.
+    This version can only be built with ghc 7.4 or newer. See the ghc7.0
+    branch for older ghcs.
+  * S3: Fix irrefutable pattern failure when accessing encrypted S3
+    credentials.
   * Use the haskell IfElse library.
   * Avoid repeated location log commits when a remote is receiving files.
     Done by adding a oneshot mode, in which location log changes are
@@ -7,28 +12,23 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     git-annex's existing ability to recover in this situation. This is
     used by git-annex-shell and other places where changes are made to
     a remote's location log.
-  * Modifications to support ghc 7.4's handling of filenames.
-    This version can only be built with ghc 7.4 or newer. See the ghc7.0
-    branch for older ghcs.
-  * S3: Fix irrefutable pattern failure when accessing encrypted S3
-    credentials.
+  * Fix teardown of stale cached ssh connections.
+  * Fixed to use the strict state monad, to avoid leaking all kinds of memory
+    due to lazy state update thunks when adding/fixing many files.
+  * Fixed some memory leaks that occurred when committing journal files.
+  * Added a annex.queuesize setting, useful when adding hundreds of thousands
+    of files on a system with plenty of memory.
+  * whereis: Prints the urls of files that the web special remote knows about.
+  * addurl --fast: Verifies that the url can be downloaded (only getting
+    its head), and records the size in the key.
+  * When checking that an url has a key, verify that the Content-Length,
+    if available, matches the size of the key.
   * addurl: Added a --file option, which can be used to specify what
     file the url is added to. This can be used to override the default
     filename that is used when adding an url, which is based on the url.
     Or, when the file already exists, the url is recorded as another
     location of the file.
   * addurl: Normalize badly encoded urls.
-  * Fix teardown of stale cached ssh connections.
-  * When checking that an url has a key, verify that the Content-Length,
-    if available, matches the size of the key.
-  * addurl --fast: Verifies that the url can be downloaded (only getting
-    its head), and records the size in the key.
-  * Fixed to use the strict state monad, to avoid leaking all kinds of memory
-    due to lazy state update thunks when adding/fixing many files.
-  * Fixed some memory leaks that occurred when committing journal files.
-  * whereis: Prints the urls of files that the web special remote knows about.
-  * Added a annex.queuesize setting, useful when adding hundreds of thousands
-    of files on a system with plenty of memory.
   * addurl: Add --pathdepth option.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400

From db6b4cdfcf67cfc3fce5e5f4323aa96a6ebd3e59 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 16:36:35 -0400
Subject: [PATCH 3151/8313] rekey: New plumbing level command, can be used to
 change the keys used for files en masse.

---
 Command/Migrate.hs | 39 ++++++++++++++++++++++-----------------
 Command/ReKey.hs   | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  2 ++
 Seek.hs            |  7 +++++++
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  8 ++++++++
 6 files changed, 87 insertions(+), 17 deletions(-)
 create mode 100644 Command/ReKey.hs

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index c6b0f086cf..795ecc265d 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -58,22 +58,27 @@ perform file oldkey newbackend = do
 	cleantmp tmpfile
 	case k of
 		Nothing -> stop
-		Just (newkey, _) -> stopUnless (link src newkey) $ do
-			-- Update symlink to use the new key.
-			liftIO $ removeFile file
-
-			-- If the old key had some
-			-- associated urls, record them for
-			-- the new key as well.
-			urls <- getUrls oldkey
-			unless (null urls) $
-				mapM_ (setUrlPresent newkey) urls
-
-			next $ Command.Add.cleanup file newkey True
+		Just (newkey, _) -> stopUnless (linkKey src newkey) $
+			next $ cleanup file oldkey newkey
 	where
 		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
-		link src newkey = getViaTmpUnchecked newkey $ \t -> do
-			-- Make a hard link to the old backend's
-			-- cached key, to avoid wasting disk space.
-			liftIO $ unlessM (doesFileExist t) $ createLink src t
-			return True	
+
+linkKey :: FilePath -> Key -> Annex Bool
+linkKey src newkey = getViaTmpUnchecked newkey $ \t -> do
+	-- Make a hard link to the old backend's
+	-- cached key, to avoid wasting disk space.
+	liftIO $ unlessM (doesFileExist t) $ createLink src t
+	return True
+
+cleanup :: FilePath -> Key -> Key -> CommandCleanup
+cleanup file oldkey newkey = do
+	-- Update symlink to use the new key.
+	liftIO $ removeFile file
+
+	-- If the old key had some associated urls, record them for
+	-- the new key as well.
+	urls <- getUrls oldkey
+	unless (null urls) $
+		mapM_ (setUrlPresent newkey) urls
+
+	Command.Add.cleanup file newkey True
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
new file mode 100644
index 0000000000..9c3625aef8
--- /dev/null
+++ b/Command/ReKey.hs
@@ -0,0 +1,46 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.ReKey where
+
+import Common.Annex
+import Command
+import qualified Annex
+import Types.Key
+import Annex.Content
+import qualified Command.Migrate
+
+def :: [Command]
+def = [command "rekey"
+	(paramOptional $ paramRepeating $ paramPair paramPath paramKey)
+	seek "change keys used for files"]
+
+seek :: [CommandSeek]
+seek = [withPairs start]
+
+start :: (FilePath, String) -> CommandStart
+start (file, keyname) = ifAnnexed file go stop
+	where
+		newkey = fromMaybe (error "bad key") $ readKey keyname
+		go (oldkey, _)
+			| oldkey == newkey = stop
+			| otherwise = do
+				showStart "rekey" file
+				next $ perform file oldkey newkey
+
+perform :: FilePath -> Key -> Key -> CommandPerform
+perform file oldkey newkey = do
+	present <- inAnnex oldkey
+	_ <- if present
+		then do
+			src <- inRepo $ gitAnnexLocation oldkey
+			Command.Migrate.linkKey src newkey
+		else do
+			unlessM (Annex.getState Annex.force) $
+				error $ file ++ " is not available (use --force to override)"
+			return True
+	next $ Command.Migrate.cleanup file oldkey newkey
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 4af10a9ce4..4a0888b53c 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -28,6 +28,7 @@ import qualified Command.Copy
 import qualified Command.Get
 import qualified Command.FromKey
 import qualified Command.DropKey
+import qualified Command.ReKey
 import qualified Command.Reinject
 import qualified Command.Fix
 import qualified Command.Init
@@ -80,6 +81,7 @@ cmds = concat
 	, Command.Dead.def
 	, Command.FromKey.def
 	, Command.DropKey.def
+	, Command.ReKey.def
 	, Command.Fix.def
 	, Command.Fsck.def
 	, Command.Unused.def
diff --git a/Seek.hs b/Seek.hs
index 6f56f30f4a..7a53fc04b2 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -45,6 +45,13 @@ withWords a params = return [a params]
 withStrings :: (String -> CommandStart) -> CommandSeek
 withStrings a params = return $ map a params
 
+withPairs :: ((String, String) -> CommandStart) -> CommandSeek
+withPairs a params = return $ map a $ pairs [] params
+	where
+		pairs c [] = reverse c
+		pairs c (x:y:xs) = pairs ((x,y):c) xs
+		pairs _ _ = error "expected pairs"
+
 withFilesToBeCommitted :: (String -> CommandStart) -> CommandSeek
 withFilesToBeCommitted a params = prepFiltered a $
 	seekHelper LsFiles.stagedNotDeleted params
diff --git a/debian/changelog b/debian/changelog
index 24f40fddc2..3315f85130 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -30,6 +30,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     location of the file.
   * addurl: Normalize badly encoded urls.
   * addurl: Add --pathdepth option.
+  * rekey: New plumbing level command, can be used to change the keys used
+    for files en masse.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index f13baf79ca..992e31c859 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -399,6 +399,14 @@ subdirectories).
 
 	git annex dropkey SHA1-s10-7da006579dd64330eb2456001fd01948430572f2
 
+* rekey [file key ...]
+  
+  This plumbing-level command is similar to migrate, but you specify
+  both the file, and the new key to use for it.
+
+  With --force, even files whose content is not currently available will
+  be rekeyed. Use with caution.
+
 # OPTIONS
 
 * --force

From 990fcad9787ef5fa3b7c039c3ee01ef298b60a27 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 16:53:44 -0400
Subject: [PATCH 3152/8313] order commands in usage by name

---
 Types/Command.hs | 9 +++++++++
 Usage.hs         | 3 ++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/Types/Command.hs b/Types/Command.hs
index 6dbcf48d16..c5020d112f 100644
--- a/Types/Command.hs
+++ b/Types/Command.hs
@@ -7,6 +7,8 @@
 
 module Types.Command where
 
+import Data.Ord
+
 import Types
 
 {- A command runs in these stages.
@@ -46,3 +48,10 @@ data Command = Command
 {- CommandCheck functions can be compared using their unique id. -}
 instance Eq CommandCheck where
 	a == b = idCheck a == idCheck b
+
+instance Eq Command where
+	a == b = cmdname a == cmdname b
+
+{- Order commands by name -}
+instance Ord Command where
+	compare = comparing cmdname
diff --git a/Usage.hs b/Usage.hs
index a33f6f311b..a1b47252d7 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -7,6 +7,7 @@
 
 module Usage where
 
+import Common.Annex
 import System.Console.GetOpt
 
 import Types.Command
@@ -30,7 +31,7 @@ usage header cmds commonoptions = unlines $
 		alloptlines = filter (not . null) $
 			lines $ usageInfo "" $
 				concatMap cmdoptions cmds ++ commonoptions
-		(cmdlines, optlines) = go cmds alloptlines []
+		(cmdlines, optlines) = go (sort cmds) alloptlines []
 		go [] os ls = (ls, os)
 		go (c:cs) os ls = go cs os' (ls++(l:o))
 			where

From 69a0161c3afb9d6f7062ffac206044d95d3ee85a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 19:37:02 -0400
Subject: [PATCH 3153/8313] fix filename limit when using --pathdepth

---
 Command/AddUrl.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index f87417d5d1..3d484a1d3c 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -95,8 +95,8 @@ url2file url pathdepth = case pathdepth of
 		| otherwise -> error "bad --pathdepth"
 	where
 		fullurl = uriRegName auth ++ uriPath url ++ uriQuery url
-		frombits a = filesize $ join "/" $ a urlbits
-		urlbits = map escape $ filter (not . null) $ split "/" fullurl
+		frombits a = join "/" $ a urlbits
+		urlbits = map (filesize . escape) $ filter (not . null) $ split "/" fullurl
 		auth = fromMaybe (error $ "bad url " ++ show url) $ uriAuthority url
 		filesize = take 255
 		escape = replace "/" "_" . replace "?" "_"

From 156a631f63eb0b2cee6a1b223d8e12ff8c62cb16 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 22:36:56 -0400
Subject: [PATCH 3154/8313] make Migrate use ReKey rather than the other way
 around

as ReKey is plumbing, this makes sense
---
 Command/Migrate.hs | 28 ++++------------------------
 Command/ReKey.hs   | 29 ++++++++++++++++++++++++-----
 Usage.hs           |  1 -
 3 files changed, 28 insertions(+), 30 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 795ecc265d..b837ef8275 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -12,8 +12,7 @@ import Command
 import qualified Backend
 import qualified Types.Key
 import Annex.Content
-import qualified Command.Add
-import Logs.Web
+import qualified Command.ReKey
 
 def :: [Command]
 def = [command "migrate" paramPaths seek "switch data to different backend"]
@@ -58,27 +57,8 @@ perform file oldkey newbackend = do
 	cleantmp tmpfile
 	case k of
 		Nothing -> stop
-		Just (newkey, _) -> stopUnless (linkKey src newkey) $
-			next $ cleanup file oldkey newkey
+		Just (newkey, _) ->
+			stopUnless (Command.ReKey.linkKey oldkey newkey) $
+				next $ Command.ReKey.cleanup file oldkey newkey
 	where
 		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
-
-linkKey :: FilePath -> Key -> Annex Bool
-linkKey src newkey = getViaTmpUnchecked newkey $ \t -> do
-	-- Make a hard link to the old backend's
-	-- cached key, to avoid wasting disk space.
-	liftIO $ unlessM (doesFileExist t) $ createLink src t
-	return True
-
-cleanup :: FilePath -> Key -> Key -> CommandCleanup
-cleanup file oldkey newkey = do
-	-- Update symlink to use the new key.
-	liftIO $ removeFile file
-
-	-- If the old key had some associated urls, record them for
-	-- the new key as well.
-	urls <- getUrls oldkey
-	unless (null urls) $
-		mapM_ (setUrlPresent newkey) urls
-
-	Command.Add.cleanup file newkey True
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 9c3625aef8..644e84382e 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -12,7 +12,8 @@ import Command
 import qualified Annex
 import Types.Key
 import Annex.Content
-import qualified Command.Migrate
+import qualified Command.Add
+import Logs.Web
 
 def :: [Command]
 def = [command "rekey"
@@ -36,11 +37,29 @@ perform :: FilePath -> Key -> Key -> CommandPerform
 perform file oldkey newkey = do
 	present <- inAnnex oldkey
 	_ <- if present
-		then do
-			src <- inRepo $ gitAnnexLocation oldkey
-			Command.Migrate.linkKey src newkey
+		then linkKey oldkey newkey
 		else do
 			unlessM (Annex.getState Annex.force) $
 				error $ file ++ " is not available (use --force to override)"
 			return True
-	next $ Command.Migrate.cleanup file oldkey newkey
+	next $ cleanup file oldkey newkey
+
+{- Make a hard link to the old key content, to avoid wasting disk space. -}
+linkKey :: Key -> Key -> Annex Bool
+linkKey oldkey newkey = getViaTmpUnchecked newkey $ \t -> do
+	src <- inRepo $ gitAnnexLocation oldkey
+	liftIO $ unlessM (doesFileExist t) $ createLink src t
+	return True
+
+cleanup :: FilePath -> Key -> Key -> CommandCleanup
+cleanup file oldkey newkey = do
+	-- Update symlink to use the new key.
+	liftIO $ removeFile file
+
+	-- If the old key had some associated urls, record them for
+	-- the new key as well.
+	urls <- getUrls oldkey
+	unless (null urls) $
+		mapM_ (setUrlPresent newkey) urls
+
+	Command.Add.cleanup file newkey True
diff --git a/Usage.hs b/Usage.hs
index a1b47252d7..59223daae4 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import System.Console.GetOpt
 
 import Types.Command
-import Types.Option
 
 {- Usage message with lists of commands and options. -}
 usage :: String -> [Command] -> [Option] -> String

From ba5515d4226a47bc89ad141bd8029a4f252ad28d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 16 Feb 2012 22:38:08 -0400
Subject: [PATCH 3155/8313] reorder for clarity

---
 Command/ReKey.hs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 644e84382e..2bd4541c61 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -53,13 +53,12 @@ linkKey oldkey newkey = getViaTmpUnchecked newkey $ \t -> do
 
 cleanup :: FilePath -> Key -> Key -> CommandCleanup
 cleanup file oldkey newkey = do
-	-- Update symlink to use the new key.
-	liftIO $ removeFile file
-
 	-- If the old key had some associated urls, record them for
 	-- the new key as well.
 	urls <- getUrls oldkey
 	unless (null urls) $
 		mapM_ (setUrlPresent newkey) urls
 
+	-- Update symlink to use the new key.
+	liftIO $ removeFile file
 	Command.Add.cleanup file newkey True

From f3c75b601f7a8a46d1e5a064f5c3eb5d828998ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 17 Feb 2012 00:19:47 -0400
Subject: [PATCH 3156/8313] reorg

---
 Command/Migrate.hs | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index b837ef8275..6e28c4b52e 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -47,18 +47,18 @@ upgradableKey key = isNothing $ Types.Key.keySize key
  - generate.
  -}
 perform :: FilePath -> Key -> Backend -> CommandPerform
-perform file oldkey newbackend = do
-	src <- inRepo $ gitAnnexLocation oldkey
-	tmp <- fromRepo gitAnnexTmpDir
-	let tmpfile = tmp  takeFileName file
-	cleantmp tmpfile
-	liftIO $ createLink src tmpfile
-	k <- Backend.genKey tmpfile $ Just newbackend
-	cleantmp tmpfile
-	case k of
-		Nothing -> stop
-		Just (newkey, _) ->
-			stopUnless (Command.ReKey.linkKey oldkey newkey) $
-				next $ Command.ReKey.cleanup file oldkey newkey
+perform file oldkey newbackend = maybe stop go =<< genkey
 	where
+		go newkey = stopUnless (Command.ReKey.linkKey oldkey newkey) $
+			next $ Command.ReKey.cleanup file oldkey newkey
+		genkey = do
+			src <- inRepo $ gitAnnexLocation oldkey
+			tmp <- fromRepo gitAnnexTmpDir
+			let tmpfile = tmp  takeFileName file
+			cleantmp tmpfile
+			liftIO $ createLink src tmpfile
+			newkey <- liftM fst <$>
+				Backend.genKey tmpfile (Just newbackend)
+			cleantmp tmpfile
+			return newkey
 		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t

From 1ed5e4d9e3084f5371eb318c500254e1547240e7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 17 Feb 2012 00:21:35 -0400
Subject: [PATCH 3157/8313] variable name

---
 Command/ReKey.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 2bd4541c61..6de7e45e32 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -46,9 +46,9 @@ perform file oldkey newkey = do
 
 {- Make a hard link to the old key content, to avoid wasting disk space. -}
 linkKey :: Key -> Key -> Annex Bool
-linkKey oldkey newkey = getViaTmpUnchecked newkey $ \t -> do
+linkKey oldkey newkey = getViaTmpUnchecked newkey $ \tmp -> do
 	src <- inRepo $ gitAnnexLocation oldkey
-	liftIO $ unlessM (doesFileExist t) $ createLink src t
+	liftIO $ unlessM (doesFileExist tmp) $ createLink src tmp
 	return True
 
 cleanup :: FilePath -> Key -> Key -> CommandCleanup

From 5bf07b3b5c9a798c3a50a775f42888e497f48732 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 17 Feb 2012 23:15:29 -0400
Subject: [PATCH 3158/8313] Store web special remote url info in a more
 efficient location.

storing it in remotes/web/xx/yy/foo.log meant lots of extra directory
objects in git. Now I use xx/yy/foo.log.web, which is just as unique, but
more efficient since foo.log is there anyway.

Of course, it still looks in the old location too.
---
 Logs/Web.hs      | 27 ++++++++++++++++-----------
 debian/changelog |  1 +
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/Logs/Web.hs b/Logs/Web.hs
index 62656b7ed8..bd87772503 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -23,21 +23,26 @@ type URLString = String
 webUUID :: UUID
 webUUID = UUID "00000000-0000-0000-0000-000000000001"
 
-{- The urls for a key are stored in remote/web/hash/key.log 
- - in the git-annex branch. -}
 urlLog :: Key -> FilePath
-urlLog key = "remote/web"  hashDirLower key  keyFile key ++ ".log"
-oldurlLog :: Key -> FilePath
-{- A bug used to store the urls elsewhere. -}
-oldurlLog key = "remote/web"  hashDirLower key  show key ++ ".log"
+urlLog key = hashDirLower key  keyFile key ++ ".log.web"
+
+{- Used to store the urls elsewhere. -}
+oldurlLogs :: Key -> [FilePath]
+oldurlLogs key = 
+	[ "remote/web"  hashDirLower key  show key ++ ".log"
+	, "remote/web"  hashDirLower key  keyFile key ++ ".log"
+	]
 
 {- Gets all urls that a key might be available from. -}
 getUrls :: Key -> Annex [URLString]
-getUrls key = do
-	us <- currentLog (urlLog key)
-	if null us
-		then currentLog (oldurlLog key)
-		else return us
+getUrls key = go $ urlLog key : oldurlLogs key
+	where
+		go [] = return []
+		go (l:ls) = do
+			us <- currentLog l
+			if null us
+				then go ls
+				else return us
 
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
diff --git a/debian/changelog b/debian/changelog
index 3315f85130..46d65df380 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -32,6 +32,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * addurl: Add --pathdepth option.
   * rekey: New plumbing level command, can be used to change the keys used
     for files en masse.
+  * Store web special remote url info in a more efficient location.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 0fada4380884758e1e51bff6f8ca708f07425e40 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 17 Feb 2012 23:58:56 -0400
Subject: [PATCH 3159/8313] avoid unnecessary log changes when re-adding the
 same url

---
 Logs/Web.hs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Logs/Web.hs b/Logs/Web.hs
index bd87772503..5a2f6e57e9 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -47,11 +47,13 @@ getUrls key = go $ urlLog key : oldurlLogs key
 {- Records a change in an url for a key. -}
 setUrl :: Key -> URLString -> LogStatus -> Annex ()
 setUrl key url status = do
-	addLog (urlLog key) =<< logNow status url
-
-	-- update location log to indicate that the web has the key, or not
 	us <- getUrls key
-	logChange key webUUID (if null us then InfoMissing else InfoPresent)
+	unless (status == InfoPresent && url `elem` us) $ do
+		addLog (urlLog key) =<< logNow status url
+
+		-- update location log to indicate that the web has the key, or not
+		us <- getUrls key
+		logChange key webUUID (if null us then InfoMissing else InfoPresent)
 
 setUrlPresent :: Key -> URLString -> Annex ()
 setUrlPresent key url = setUrl key url InfoPresent

From 00340dfe493ad71a55f4de2c352fc186792363cd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 18 Feb 2012 11:44:21 -0400
Subject: [PATCH 3160/8313] don't error out entirely if an url cannot be
 downloaded

---
 Command/AddUrl.hs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 3d484a1d3c..459fbc623d 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -80,11 +80,14 @@ download url file = do
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
 	(exists, size) <- liftIO $ Url.exists url
-	unless exists $
-		error $ "unable to access url: " ++ url
-	let key = Backend.URL.fromUrl url size
-	setUrlPresent key url
-	next $ Command.Add.cleanup file key False
+	if exists
+		then do
+			let key = Backend.URL.fromUrl url size
+			setUrlPresent key url
+			next $ Command.Add.cleanup file key False
+		else do
+			warning $ "unable to access url: " ++ url
+			stop
 
 url2file :: URI -> Maybe Int -> FilePath
 url2file url pathdepth = case pathdepth of

From 0cbbf0da793f6bbffe989007a70953a7593e6841 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 18 Feb 2012 11:54:47 -0400
Subject: [PATCH 3161/8313] warning

---
 Logs/Web.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Logs/Web.hs b/Logs/Web.hs
index 5a2f6e57e9..607c81c5bf 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -52,8 +52,8 @@ setUrl key url status = do
 		addLog (urlLog key) =<< logNow status url
 
 		-- update location log to indicate that the web has the key, or not
-		us <- getUrls key
-		logChange key webUUID (if null us then InfoMissing else InfoPresent)
+		us' <- getUrls key
+		logChange key webUUID (if null us' then InfoMissing else InfoPresent)
 
 setUrlPresent :: Key -> URLString -> Annex ()
 setUrlPresent key url = setUrl key url InfoPresent

From abd50e01fbf08ddf4a71c243febe6474dfb566df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 18 Feb 2012 12:05:13 -0400
Subject: [PATCH 3162/8313] don't fail with --pathdepth when file already
 exists

---
 Command/AddUrl.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 459fbc623d..b27cb7f0f8 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -54,8 +54,6 @@ perform url file pathdepth = ifAnnexed file addurl geturl
 			fast <- Annex.getState Annex.fast
 			if fast then nodownload url file else download url file
 		addurl (key, _backend) = do
-			when (pathdepth /= Nothing) $
-				error $ file ++ " already exists"
 			unlessM (liftIO $ Url.check url (keySize key)) $
 				error $ "failed to verify url: " ++ url
 			setUrlPresent key url

From 779ec919086a47ca0293a72f1fb452220651d0c5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 18 Feb 2012 12:08:02 -0400
Subject: [PATCH 3163/8313] more robustness fixes

---
 Command/AddUrl.hs | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index b27cb7f0f8..6c945baf93 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -44,20 +44,24 @@ start optfile pathdepth s = notBareRepo $ go $ fromMaybe bad $ parseURI s
 		go url = do
 			let file = fromMaybe (url2file url pathdepth) optfile
 			showStart "addurl" file
-			next $ perform s file pathdepth
+			next $ perform s file
 
-perform :: String -> FilePath -> Maybe Int -> CommandPerform
-perform url file pathdepth = ifAnnexed file addurl geturl
+perform :: String -> FilePath -> CommandPerform
+perform url file = ifAnnexed file addurl geturl
 	where
 		geturl = do
 			liftIO $ createDirectoryIfMissing True (parentDir file)
 			fast <- Annex.getState Annex.fast
 			if fast then nodownload url file else download url file
 		addurl (key, _backend) = do
-			unlessM (liftIO $ Url.check url (keySize key)) $
-				error $ "failed to verify url: " ++ url
-			setUrlPresent key url
-			next $ return True
+			ok <- liftIO $ Url.check url (keySize key)
+			if ok
+				then do
+					setUrlPresent key url
+					next $ return True
+				else do
+					warning $ "failed to verify url: " ++ url
+					stop
 
 download :: String -> FilePath -> CommandPerform
 download url file = do

From ddb8edcf2b45c788a7595a1fbd70ad2414317ce3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 18 Feb 2012 18:03:24 -0400
Subject: [PATCH 3164/8313] improve interface

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 992e31c859..bd91391737 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -151,7 +151,7 @@ subdirectories).
 
   Normally the filename is based on the full url, so will look like
   "www.example.com_dir_subdir_bigfile". For a shorter filename, specify
-  --pathdepth=N. For example, --pathdepth=2 will use "subdir/bigfile",
+  --pathdepth=N. For example, --pathdepth=1 will use "dir/subdir/bigfile",
   while --pathdepth=3 will use "bigfile". It can also be negative;
   --pathdepth=-2 will use the last two parts of the url.
 

From ac5cff3668ad1fc529d32058c31b30dd341c2547 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Feb 2012 11:08:50 -0400
Subject: [PATCH 3165/8313] quiet expected compiler warning

---
 test.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test.hs b/test.hs
index 245dd6706a..69837f9eda 100644
--- a/test.hs
+++ b/test.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
 import Test.HUnit
 import Test.HUnit.Tools
 import Test.QuickCheck

From 6c0155efb7fd1ff7259095655093e0eab0e37e51 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Feb 2012 15:20:36 -0400
Subject: [PATCH 3166/8313] refactor

---
 Git/CatFile.hs       | 43 ++++++++++++++++++++-----------------------
 Git/CheckAttr.hs     | 27 ++++++++++++---------------
 Git/HashObject.hs    | 23 ++++++++++-------------
 Utility/CoProcess.hs | 35 +++++++++++++++++++++++++++++++++++
 4 files changed, 77 insertions(+), 51 deletions(-)
 create mode 100644 Utility/CoProcess.hs

diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 2a2eb5e6f7..2987a9d9d4 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -13,7 +13,6 @@ module Git.CatFile (
 	catObject
 ) where
 
-import System.Cmd.Utils
 import System.IO
 import qualified Data.ByteString.Char8 as S
 import qualified Data.ByteString.Lazy.Char8 as L
@@ -22,20 +21,18 @@ import Common
 import Git
 import Git.Sha
 import Git.Command
+import qualified Utility.CoProcess as CoProcess
 
-type CatFileHandle = (PipeHandle, Handle, Handle)
+type CatFileHandle = CoProcess.CoProcessHandle
 
 {- Starts git cat-file running in batch mode in a repo and returns a handle. -}
 catFileStart :: Repo -> IO CatFileHandle
-catFileStart repo = hPipeBoth "git" $ toCommand $
+catFileStart repo = CoProcess.start "git" $ toCommand $
 	gitCommandLine [Param "cat-file", Param "--batch"] repo
 
 {- Stops git cat-file. -}
 catFileStop :: CatFileHandle -> IO ()
-catFileStop (pid, from, to) = do
-	hClose to
-	hClose from
-	forceSuccess pid
+catFileStop = CoProcess.stop
 
 {- Reads a file from a specified branch. -}
 catFile :: CatFileHandle -> Branch -> FilePath -> IO L.ByteString
@@ -44,23 +41,23 @@ catFile h branch file = catObject h $ Ref $ show branch ++ ":" ++ file
 {- Uses a running git cat-file read the content of an object.
  - Objects that do not exist will have "" returned. -}
 catObject :: CatFileHandle -> Ref -> IO L.ByteString
-catObject (_, from, to) object = do
-	hPutStrLn to $ show object
-	hFlush to
-	header <- hGetLine from
-	case words header of
-		[sha, objtype, size]
-			| length sha == shaSize &&
-			  validobjtype objtype -> handle size
-			| otherwise -> dne
-		_
-			| header == show object ++ " missing" -> dne
-			| otherwise -> error $ "unknown response from git cat-file " ++ header
+catObject h object = CoProcess.query h send receive
 	where
-		handle size = case reads size of
-			[(bytes, "")] -> readcontent bytes
-			_ -> dne
-		readcontent bytes = do
+		send to = hPutStrLn to $ show object
+		receive from = do
+			header <- hGetLine from
+			case words header of
+				[sha, objtype, size]
+					| length sha == shaSize &&
+					  validobjtype objtype -> 
+						case reads size of
+							[(bytes, "")] -> readcontent bytes from
+							_ -> dne
+					| otherwise -> dne
+				_
+					| header == show object ++ " missing" -> dne
+					| otherwise -> error $ "unknown response from git cat-file " ++ header
+		readcontent bytes from = do
 			content <- S.hGet from bytes
 			c <- hGetChar from
 			when (c /= '\n') $
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 696d8aafd3..65e8e03d8e 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -11,8 +11,9 @@ import Common
 import Git
 import Git.Command
 import qualified Git.Version
+import qualified Utility.CoProcess as CoProcess
 
-type CheckAttrHandle = (PipeHandle, Handle, Handle, [Attr], String)
+type CheckAttrHandle = (CoProcess.CoProcessHandle, [Attr], String)
 
 type Attr = String
 
@@ -21,11 +22,10 @@ type Attr = String
 checkAttrStart :: [Attr] -> Repo -> IO CheckAttrHandle
 checkAttrStart attrs repo = do
 	cwd <- getCurrentDirectory
-	(pid, from, to) <- hPipeBoth "git" $ toCommand $
+	h <- CoProcess.start "git" $ toCommand $
 		gitCommandLine params repo
-	fileEncoding from
-	fileEncoding to
-	return (pid, from, to, attrs, cwd)
+	CoProcess.query h fileEncoding fileEncoding
+	return (h, attrs, cwd)
 	where
 		params =
 			[ Param "check-attr" 
@@ -35,24 +35,21 @@ checkAttrStart attrs repo = do
 
 {- Stops git check-attr. -}
 checkAttrStop :: CheckAttrHandle -> IO ()
-checkAttrStop (pid, from, to, _, _) = do
-	hClose to
-	hClose from
-	forceSuccess pid
+checkAttrStop (h, _, _) = CoProcess.stop h
 
 {- Gets an attribute of a file. -}
 checkAttr :: CheckAttrHandle -> Attr -> FilePath -> IO String
-checkAttr (_, from, to, attrs, cwd) want file = do
-	hPutStr to $ file' ++ "\0"
-	hFlush to
-	pairs <- forM attrs $ \attr -> do
-		l <- hGetLine from
-		return (attr, attrvalue attr l)
+checkAttr (h, attrs, cwd) want file = do
+	pairs <- CoProcess.query h send receive
 	let vals = map snd $ filter (\(attr, _) -> attr == want) pairs
 	case vals of
 		[v] -> return v
 		_ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file
 	where
+		send to = hPutStr to $ file' ++ "\0"
+		receive from = forM attrs $ \attr -> do
+			l <- hGetLine from
+			return (attr, attrvalue attr l)
 		{- Before git 1.7.7, git check-attr worked best with
 		 - absolute filenames; using them worked around some bugs
 		 - with relative filenames.
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 200fedbd27..5848d0144a 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -10,16 +10,16 @@ module Git.HashObject where
 import Common
 import Git
 import Git.Command
+import qualified Utility.CoProcess as CoProcess
 
-type HashObjectHandle = (PipeHandle, Handle, Handle)
+type HashObjectHandle = CoProcess.CoProcessHandle
 
 {- Starts git hash-object and returns a handle.  -}
 hashObjectStart :: Repo -> IO HashObjectHandle
 hashObjectStart repo = do
-	r@(_, _, toh) <- hPipeBoth "git" $
-		toCommand $ gitCommandLine params repo
-	fileEncoding toh
-	return r
+	h <- CoProcess.start "git" $ toCommand $ gitCommandLine params repo
+	CoProcess.query h fileEncoding (const $ return ())
+	return h
 	where
 		params =
 			[ Param "hash-object"
@@ -29,14 +29,11 @@ hashObjectStart repo = do
 
 {- Stops git hash-object. -}
 hashObjectStop :: HashObjectHandle -> IO ()
-hashObjectStop (pid, from, to) = do
-	hClose to
-	hClose from
-	forceSuccess pid
+hashObjectStop = CoProcess.stop
 
 {- Injects a file into git, returning the shas of the objects. -}
 hashFile :: HashObjectHandle -> FilePath -> IO Sha
-hashFile (_, from, to) file = do
-	hPutStrLn to file
-	hFlush to
-	Ref <$> hGetLine from
+hashFile h file = CoProcess.query h send receive
+	where
+		send to = hPutStrLn to file
+		receive from = Ref <$> hGetLine from
diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs
new file mode 100644
index 0000000000..9fa8d864fe
--- /dev/null
+++ b/Utility/CoProcess.hs
@@ -0,0 +1,35 @@
+{- Interface for running a shell command as a coprocess,
+ - sending it queries and getting back results.
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.CoProcess (
+	CoProcessHandle,
+	start,
+	stop,
+	query
+) where
+
+import System.Cmd.Utils
+
+import Common
+
+type CoProcessHandle = (PipeHandle, Handle, Handle)
+
+start :: FilePath -> [String] -> IO CoProcessHandle
+start command params = hPipeBoth command params
+
+stop :: CoProcessHandle -> IO ()
+stop (pid, from, to) = do
+	hClose to
+	hClose from
+	forceSuccess pid
+
+query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b
+query (_, from, to) send receive = do
+	_ <- send to
+	hFlush to
+	receive from

From 7e17151e69fcd86fd5cb90dd61ff55d2d017fee7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 20 Feb 2012 15:28:57 -0400
Subject: [PATCH 3167/8313] revert hlint change

broke a test
---
 Logs/Remote.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index c38a05c189..ccfb4bb31a 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -79,7 +79,7 @@ configUnEscape = unescape
 				num = takeWhile isNumber s
 				r = drop (length num) s
 				rest = drop 1 r
-				ok = not (null num) && ":" `isPrefixOf` r
+				ok = not (null num) && ";" `isPrefixOf` r
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool

From cac130b205fca9fc45833296dc326b3fd6385163 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 21 Feb 2012 00:16:24 -0400
Subject: [PATCH 3168/8313] cleanup

---
 Git/CatFile.hs    |  8 ++++----
 Git/CheckAttr.hs  | 12 ++++++------
 Git/HashObject.hs | 21 ++++++++-------------
 3 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 2987a9d9d4..41ecb362c1 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -25,12 +25,12 @@ import qualified Utility.CoProcess as CoProcess
 
 type CatFileHandle = CoProcess.CoProcessHandle
 
-{- Starts git cat-file running in batch mode in a repo and returns a handle. -}
 catFileStart :: Repo -> IO CatFileHandle
-catFileStart repo = CoProcess.start "git" $ toCommand $
-	gitCommandLine [Param "cat-file", Param "--batch"] repo
+catFileStart = CoProcess.start "git" . toCommand . gitCommandLine
+	[ Param "cat-file"
+	, Param "--batch"
+	]
 
-{- Stops git cat-file. -}
 catFileStop :: CatFileHandle -> IO ()
 catFileStop = CoProcess.stop
 
diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs
index 65e8e03d8e..6b321f8b8f 100644
--- a/Git/CheckAttr.hs
+++ b/Git/CheckAttr.hs
@@ -18,13 +18,11 @@ type CheckAttrHandle = (CoProcess.CoProcessHandle, [Attr], String)
 type Attr = String
 
 {- Starts git check-attr running to look up the specified gitattributes
- - values and return a handle.  -}
+ - values and returns a handle.  -}
 checkAttrStart :: [Attr] -> Repo -> IO CheckAttrHandle
 checkAttrStart attrs repo = do
 	cwd <- getCurrentDirectory
-	h <- CoProcess.start "git" $ toCommand $
-		gitCommandLine params repo
-	CoProcess.query h fileEncoding fileEncoding
+	h <- CoProcess.start "git" $ toCommand $ gitCommandLine params repo
 	return (h, attrs, cwd)
 	where
 		params =
@@ -33,7 +31,6 @@ checkAttrStart attrs repo = do
 			] ++ map Param attrs ++
 			[ Param "--" ]
 
-{- Stops git check-attr. -}
 checkAttrStop :: CheckAttrHandle -> IO ()
 checkAttrStop (h, _, _) = CoProcess.stop h
 
@@ -46,8 +43,11 @@ checkAttr (h, attrs, cwd) want file = do
 		[v] -> return v
 		_ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file
 	where
-		send to = hPutStr to $ file' ++ "\0"
+		send to = do
+			fileEncoding to
+			hPutStr to $ file' ++ "\0"
 		receive from = forM attrs $ \attr -> do
+			fileEncoding from
 			l <- hGetLine from
 			return (attr, attrvalue attr l)
 		{- Before git 1.7.7, git check-attr worked best with
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 5848d0144a..617e5ac28f 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -14,20 +14,13 @@ import qualified Utility.CoProcess as CoProcess
 
 type HashObjectHandle = CoProcess.CoProcessHandle
 
-{- Starts git hash-object and returns a handle.  -}
 hashObjectStart :: Repo -> IO HashObjectHandle
-hashObjectStart repo = do
-	h <- CoProcess.start "git" $ toCommand $ gitCommandLine params repo
-	CoProcess.query h fileEncoding (const $ return ())
-	return h
-	where
-		params =
-			[ Param "hash-object"
-			, Param "-w"
-			, Param "--stdin-paths"
-			]
+hashObjectStart = CoProcess.start "git" . toCommand . gitCommandLine
+	[ Param "hash-object"
+	, Param "-w"
+	, Param "--stdin-paths"
+	]
 
-{- Stops git hash-object. -}
 hashObjectStop :: HashObjectHandle -> IO ()
 hashObjectStop = CoProcess.stop
 
@@ -35,5 +28,7 @@ hashObjectStop = CoProcess.stop
 hashFile :: HashObjectHandle -> FilePath -> IO Sha
 hashFile h file = CoProcess.query h send receive
 	where
-		send to = hPutStrLn to file
+		send to = do
+			fileEncoding to
+			hPutStrLn to file
 		receive from = Ref <$> hGetLine from

From 6dca07e255b0aad254bd8f94f9bf3f3706590e4d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkCoDbzG4_biL7Y9IvUiRsBH_GchCKAaW4"
 
Date: Fri, 24 Feb 2012 17:47:02 +0000
Subject: [PATCH 3169/8313]

---
 ...o_results_in_errors_on_drop__47__move.mdwn | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn

diff --git a/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn b/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn
new file mode 100644
index 0000000000..aa0c3cc29a
--- /dev/null
+++ b/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn
@@ -0,0 +1,45 @@
+I'm on an nfs mounted filesystem (some netapp somewhere).  This is repeatable, every time.
+
+    git init repo; cd repo;
+    git annex init repo
+    truncate -s 20M big
+    git annex add big
+    git commit -m "annexed file"
+    cd ..
+    git clone repo repo_copy
+    cd repo_copy;
+    git annex get .
+    git annex whereis big
+
+    #whereis big (2 copies) 
+    #   	9310b242-6021-4621-8cef-4548a00907ff -- here
+    #   	b3526e4d-38d7-4781-a9c3-436007899f1b -- origin (repo)
+    #ok
+
+    git annex drop big
+
+    #git-annex: /nfspath/repo_copy/.git/annex/objects/fM/4k/SHA1-s20971520--9674344c90c2f0646f0b78026e127c9b86e3ad77: removeDirectory: unsatisified constraints (Directory not empty)
+    #failed
+    #git-annex: drop: 1 failed
+
+    git annex drop big  # no error second time, I suspect nfs has caught up by now.
+    git annex fsck      # Doesn't know that the second drop succeeded.
+
+    #fsck big (fixing location log) 
+    #  ** Based on the location log, big
+    #  ** was expected to be present, but its content is missing.
+    #failed
+    #git-annex: fsck: 1 failed
+
+    git annex fsck
+
+    #fsck big ok
+
+    git annex whereis big
+
+    #whereis big (1 copy) 
+    #  	b3526e4d-38d7-4781-a9c3-436007899f1b -- origin (repo)
+    #ok
+
+I suspect git-annex is just too fast and optimistic for big slow nfs directories.
+

From c9b48520ccf7c9dbda0ad08150178868cc69f660 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Feb 2012 15:48:01 -0400
Subject: [PATCH 3170/8313] move misplaced bug report

---
 ...d_repo_results_in_errors_on_drop_move.mdwn | 45 +++++++++++++++++
 ...o_results_in_errors_on_drop__47__move.mdwn | 49 ++-----------------
 2 files changed, 49 insertions(+), 45 deletions(-)
 create mode 100644 doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn

diff --git a/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn b/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn
new file mode 100644
index 0000000000..aa0c3cc29a
--- /dev/null
+++ b/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn
@@ -0,0 +1,45 @@
+I'm on an nfs mounted filesystem (some netapp somewhere).  This is repeatable, every time.
+
+    git init repo; cd repo;
+    git annex init repo
+    truncate -s 20M big
+    git annex add big
+    git commit -m "annexed file"
+    cd ..
+    git clone repo repo_copy
+    cd repo_copy;
+    git annex get .
+    git annex whereis big
+
+    #whereis big (2 copies) 
+    #   	9310b242-6021-4621-8cef-4548a00907ff -- here
+    #   	b3526e4d-38d7-4781-a9c3-436007899f1b -- origin (repo)
+    #ok
+
+    git annex drop big
+
+    #git-annex: /nfspath/repo_copy/.git/annex/objects/fM/4k/SHA1-s20971520--9674344c90c2f0646f0b78026e127c9b86e3ad77: removeDirectory: unsatisified constraints (Directory not empty)
+    #failed
+    #git-annex: drop: 1 failed
+
+    git annex drop big  # no error second time, I suspect nfs has caught up by now.
+    git annex fsck      # Doesn't know that the second drop succeeded.
+
+    #fsck big (fixing location log) 
+    #  ** Based on the location log, big
+    #  ** was expected to be present, but its content is missing.
+    #failed
+    #git-annex: fsck: 1 failed
+
+    git annex fsck
+
+    #fsck big ok
+
+    git annex whereis big
+
+    #whereis big (1 copy) 
+    #  	b3526e4d-38d7-4781-a9c3-436007899f1b -- origin (repo)
+    #ok
+
+I suspect git-annex is just too fast and optimistic for big slow nfs directories.
+
diff --git a/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn b/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn
index aa0c3cc29a..bf84f75c0b 100644
--- a/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn
+++ b/doc/forum/nfs_mounted_repo_results_in_errors_on_drop__47__move.mdwn
@@ -1,45 +1,4 @@
-I'm on an nfs mounted filesystem (some netapp somewhere).  This is repeatable, every time.
-
-    git init repo; cd repo;
-    git annex init repo
-    truncate -s 20M big
-    git annex add big
-    git commit -m "annexed file"
-    cd ..
-    git clone repo repo_copy
-    cd repo_copy;
-    git annex get .
-    git annex whereis big
-
-    #whereis big (2 copies) 
-    #   	9310b242-6021-4621-8cef-4548a00907ff -- here
-    #   	b3526e4d-38d7-4781-a9c3-436007899f1b -- origin (repo)
-    #ok
-
-    git annex drop big
-
-    #git-annex: /nfspath/repo_copy/.git/annex/objects/fM/4k/SHA1-s20971520--9674344c90c2f0646f0b78026e127c9b86e3ad77: removeDirectory: unsatisified constraints (Directory not empty)
-    #failed
-    #git-annex: drop: 1 failed
-
-    git annex drop big  # no error second time, I suspect nfs has caught up by now.
-    git annex fsck      # Doesn't know that the second drop succeeded.
-
-    #fsck big (fixing location log) 
-    #  ** Based on the location log, big
-    #  ** was expected to be present, but its content is missing.
-    #failed
-    #git-annex: fsck: 1 failed
-
-    git annex fsck
-
-    #fsck big ok
-
-    git annex whereis big
-
-    #whereis big (1 copy) 
-    #  	b3526e4d-38d7-4781-a9c3-436007899f1b -- origin (repo)
-    #ok
-
-I suspect git-annex is just too fast and optimistic for big slow nfs directories.
-
+Bug reports should be posted in [[bugs]], not in the forum. I have moved
+this misplaced bug report to
+[[bugs/nfs_mounted_repo_results_in_errors_on_drop_move]].
+--[[Joey]] 

From 0b19f7c70901bf3c2f21d19c01ee7ad80f7629fa Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 24 Feb 2012 20:28:20 +0000
Subject: [PATCH 3171/8313]

---
 doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn

diff --git a/doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn b/doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn
new file mode 100644
index 0000000000..60b8e034d0
--- /dev/null
+++ b/doc/forum/Automatic_commit_messages_for_git_annex_sync.mdwn
@@ -0,0 +1 @@
+Not really important (who reads git annex commit messages anyways ;-)), but nice to have and maybe a nice task for someone who wants to play around with Haskell and git annex: It would be shiny if the auto-commit done by git annex sync would automatically create a sensible commmit message and description. E.g. if just one file is added, it could say „Added blubb“. If a few files are added, it could say „Added blubb, bla and n other files“, based on the file name length, and list the files in the long description. Lots of room for playing around :-)

From bd66f962d33e5480ed66fdafc9502926f60ed13b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Feb 2012 16:30:47 -0400
Subject: [PATCH 3172/8313] Deal with NFS problem that caused a failure to
 remove a directory when removing content from the annex.

I was able to reproduce this on linux using the kernel's nfs server and
mounting localhost:/. Determined that removing the directory fails when
the just-deleted file in it was locked. Considered dropping the lock
before removing the directory, but this would complicate parts of the code
that should not need to worry about locking. So instead, ignore the failure
to remove the directory in this case.

While I was at it, made it attempt to remove both levels of hash
directories, in case they're empty.
---
 Annex/Content.hs                              | 33 +++++++++++++------
 debian/changelog                              |  2 ++
 ...d_repo_results_in_errors_on_drop_move.mdwn | 14 ++++++++
 3 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index d10370bc9a..fc5f46af98 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -244,20 +244,33 @@ withObjectLoc key a = do
 	let dir = parentDir file
 	a (dir, file)
 
+cleanObjectLoc :: Key -> Annex ()
+cleanObjectLoc key = do
+	file <- inRepo $ gitAnnexLocation key
+	liftIO $ removeparents file (3 :: Int)
+	where
+		removeparents _ 0 = return ()
+		removeparents file n = do
+			let dir = parentDir file
+			maybe (return ()) (const $ removeparents dir (n-1))
+				=<< catchMaybeIO (removeDirectory dir)
+
 {- Removes a key's file from .git/annex/objects/ -}
 removeAnnex :: Key -> Annex ()
-removeAnnex key = withObjectLoc key $ \(dir, file) -> liftIO $ do
-	allowWrite dir
-	removeFile file
-	removeDirectory dir
+removeAnnex key = withObjectLoc key $ \(dir, file) -> do
+	liftIO $ do
+		allowWrite dir
+		removeFile file
+	cleanObjectLoc key
 
 {- Moves a key's file out of .git/annex/objects/ -}
 fromAnnex :: Key -> FilePath -> Annex ()
-fromAnnex key dest = withObjectLoc key $ \(dir, file) -> liftIO $ do
-	allowWrite dir
-	allowWrite file
-	moveFile file dest
-	removeDirectory dir
+fromAnnex key dest = withObjectLoc key $ \(dir, file) -> do
+	liftIO $ do
+		allowWrite dir
+		allowWrite file
+		moveFile file dest
+	cleanObjectLoc key
 
 {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
  - returns the file it was moved to. -}
@@ -270,7 +283,7 @@ moveBad key = do
 		createDirectoryIfMissing True (parentDir dest)
 		allowWrite (parentDir src)
 		moveFile src dest
-		removeDirectory (parentDir src)
+	cleanObjectLoc key
 	logStatus key InfoMissing
 	return dest
 
diff --git a/debian/changelog b/debian/changelog
index 46d65df380..8c30c25005 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -33,6 +33,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * rekey: New plumbing level command, can be used to change the keys used
     for files en masse.
   * Store web special remote url info in a more efficient location.
+  * Deal with NFS problem that caused a failure to remove a directory
+    when removing content from the annex.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn b/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn
index aa0c3cc29a..761ee5b25b 100644
--- a/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn
+++ b/doc/bugs/nfs_mounted_repo_results_in_errors_on_drop_move.mdwn
@@ -43,3 +43,17 @@ I'm on an nfs mounted filesystem (some netapp somewhere).  This is repeatable, e
 
 I suspect git-annex is just too fast and optimistic for big slow nfs directories.
 
+> git-annex locks files while it is operating on their content
+> to avoid race conditions with other git-annex processes. 
+> Quite likely this problem (which I can reproduce) is due to
+> NFS having bad (non-POSIX) locking semantics. 
+> 
+> Probably the
+> lock is represented on the NFS server as some form of lock file
+> next to the file being locked, and so when that file is deleted, with
+> the lock still held, the directory, which should then be empty, still
+> contains this lock file.
+> 
+> So, this can be worked around by it not failing when the directory
+> unexpectedly cannot be removed. I've made that change. [[done]]
+> --[[Joey]]

From 0aa7f19c93e40c1cca11d7ab131c6ebb7b0a71ed Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 24 Feb 2012 20:52:51 +0000
Subject: [PATCH 3173/8313] Added a comment

---
 ...nt_1_ea2ec57bc695da4df8a30a35d433959d._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment

diff --git a/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment
new file mode 100644
index 0000000000..c07dd49999
--- /dev/null
+++ b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_1_ea2ec57bc695da4df8a30a35d433959d._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-02-24T20:52:51Z"
+ content="""
+Took me a minute to see this is not about [[bugs/wishlist:_more_descriptive_commit_messages_in_git-annex_branch]], but about the \"git-annex automatic sync\" message that is used when committing any changes currently on the master branch before doing the rest of the sync.
+
+So.. It would be pretty easy to `ls-files` the relevant files before the commit and make a message. Although this would roughly double the commit time in a large tree, since that would walk the whole tree again (git commit -a already does it once). Smarter approaches could be faster.. perhaps it could find unstaged files, stage them, generate the message, and then `git commit` the staged changes.
+
+But, would this really be useful? It's already easy to get `git log` to show a summary of the changes made in such a commit. So it's often seen as bad form to unnecessarily mention which files a commit changes in the commit message.
+
+Perhaps more useful would be to expand the current message with details like where the sync is being committed, or what
+remotes it's going to sync from, or something like that.
+"""]]

From 39f3e9e068632705ff4b24edf8e82a9fb70e12a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 24 Feb 2012 17:53:21 -0400
Subject: [PATCH 3174/8313] update

---
 doc/bugs/signal_weirdness.mdwn | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/doc/bugs/signal_weirdness.mdwn b/doc/bugs/signal_weirdness.mdwn
index a31cc21486..a7cc146422 100644
--- a/doc/bugs/signal_weirdness.mdwn
+++ b/doc/bugs/signal_weirdness.mdwn
@@ -25,14 +25,24 @@ received a signal, I'm not sure?)
 It's more usual for a `system` like thing to block SIGINT, letting the child
 catch it and exit, and then detecting the child's exit status and terminating. 
 However, since rsync *is* trapping SIGINT, and exiting nonzero explicitly,
-git-annex can't tell that rsync failed due to a SIGINT.
+git-annex can't tell that rsync failed due to a SIGINT by examining the
+`waitpid` result.
 And, git-annex typically doesn't stop when a single child fails. In the
 example above, it would go on to copy `b` after a ctrl-c!
 
 A further complication is that git-annex is itself a child process
-of git, which does not block SIGINT either. So if git-annex blocks sigint,
+of git, which does not block SIGINT either. So if git-annex blocks SIGINT,
 it will be left running in the background after git exits, and continuing
-with further actions too. (Probably this is a bug in git.)
+with further actions too. (Perhaps its SIGINT handling is a bug in git.)
+
+Now, rsync does have a documented exit code it uses after a SIGINT.
+But other programs git-annex runs generally do not. So it would be possible
+to special case in support for rsync, blocking SIGINT while running it,
+noticing it exited with 20, and git-annex then stopping. But this is
+ugly and failure prone if rsync's code 20 changes. And it only
+would fix the rsync case, not helping with other commands like wget, unless
+it assumes they never trap SIGINT on their own.
 
 Which is why the current behavior of not blocking SIGINT was chosen,
-as a less bad alternative. Still, I'd like to find a better one. --[[Joey]] 
+as a less bad alternative. Still, I'd like to find a better one.
+--[[Joey]] 

From 92236dbdaf97324c84cb48c34189e9f011c77fee Mon Sep 17 00:00:00 2001
From: "http://www.joachim-breitner.de/" 
Date: Fri, 24 Feb 2012 23:09:04 +0000
Subject: [PATCH 3175/8313] Added a comment

---
 .../comment_2_af71f53dbbca35d5a5c66ff131887ada._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment

diff --git a/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment
new file mode 100644
index 0000000000..a71f72b9e4
--- /dev/null
+++ b/doc/forum/Automatic_commit_messages_for_git_annex_sync/comment_2_af71f53dbbca35d5a5c66ff131887ada._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://www.joachim-breitner.de/"
+ nickname="nomeata"
+ subject="comment 2"
+ date="2012-02-24T23:09:03Z"
+ content="""
+Yes, it is really a minor point. And indeed, \"git log --summary\" is pretty good already. But I’d still think that at least the title could deserves some love. Including the Hostname or the name of the repository there is a good idea as well.
+"""]]

From df3a310b836f8b59cab18441250d0cf3820ed02f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 10:40:05 -0400
Subject: [PATCH 3176/8313] update copyright format url

---
 debian/copyright | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/copyright b/debian/copyright
index dd880f1428..85fd174fc0 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -1,4 +1,4 @@
-Format: http://dep.debian.net/deps/dep5/
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Source: native package
 
 Files: *

From 638741f49e1c4a52b09e6dbf83f1dfbfcbd466d4 Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Sat, 25 Feb 2012 15:02:18 +0000
Subject: [PATCH 3177/8313] Added a comment: very nice

---
 .../comment_1_59681be5568f568f5c54eb0445163dd2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/sync/comment_1_59681be5568f568f5c54eb0445163dd2._comment

diff --git a/doc/sync/comment_1_59681be5568f568f5c54eb0445163dd2._comment b/doc/sync/comment_1_59681be5568f568f5c54eb0445163dd2._comment
new file mode 100644
index 0000000000..fa228c5aed
--- /dev/null
+++ b/doc/sync/comment_1_59681be5568f568f5c54eb0445163dd2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="very nice"
+ date="2012-02-25T15:02:18Z"
+ content="""
+Here's a way to get from a starting point of two or more peer directory trees *not* tracked by git or git-annex, to the point where they can be synced in the manner described above: [[forum/syncing_non-git_trees_with_git-annex/]]
+"""]]

From b49c0c2633d662a7c91a609aec74f95f189129ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 15:23:17 -0400
Subject: [PATCH 3178/8313] add annex.alwayscommit option

To avoid commits of data to the git-annex branch after each command
is run, set annex.alwayscommit=false. Its data will then be committed
less frequently, when a merge or sync is done.
---
 Annex/Content.hs   |  4 +++-
 debian/changelog   | 15 +++++++++------
 doc/git-annex.mdwn |  7 +++++++
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index fc5f46af98..3a388129e2 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -32,6 +32,7 @@ import Common.Annex
 import Logs.Location
 import Annex.UUID
 import qualified Git
+import qualified Git.Config
 import qualified Annex
 import qualified Annex.Queue
 import qualified Annex.Branch
@@ -311,7 +312,8 @@ getKeysPresent' dir = do
 saveState :: Bool -> Annex ()
 saveState oneshot = do
 	Annex.Queue.flush False
-	unless oneshot $
+	alwayscommit <- Git.configTrue <$> fromRepo (Git.Config.get "annex.alwayscommit" "true")
+	unless (oneshot || not alwayscommit) $ do
 		Annex.Branch.commit "update"
 
 {- Downloads content from any of a list of urls. -}
diff --git a/debian/changelog b/debian/changelog
index 8c30c25005..44cdb37828 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,12 +6,6 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * S3: Fix irrefutable pattern failure when accessing encrypted S3
     credentials.
   * Use the haskell IfElse library.
-  * Avoid repeated location log commits when a remote is receiving files.
-    Done by adding a oneshot mode, in which location log changes are
-    written to the journal, but not committed. Taking advantage of
-    git-annex's existing ability to recover in this situation. This is
-    used by git-annex-shell and other places where changes are made to
-    a remote's location log.
   * Fix teardown of stale cached ssh connections.
   * Fixed to use the strict state monad, to avoid leaking all kinds of memory
     due to lazy state update thunks when adding/fixing many files.
@@ -35,6 +29,15 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * Store web special remote url info in a more efficient location.
   * Deal with NFS problem that caused a failure to remove a directory
     when removing content from the annex.
+  * Avoid repeated location log commits when a remote is receiving files.
+    Done by adding a oneshot mode, in which location log changes are
+    written to the journal, but not committed. Taking advantage of
+    git-annex's existing ability to recover in this situation. This is
+    used by git-annex-shell and other places where changes are made to
+    a remote's location log.
+  * To avoid commits of data to the git-annex branch after each command
+    is run, set annex.alwayscommit=false. Its data will then be committed
+    less frequently, when a merge or sync is done. 
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index bd91391737..0dc22b88d8 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -606,6 +606,13 @@ Here are all the supported configuration settings.
 
   By default, git-annex caches ssh connections. To disable this, set to `false`.
 
+* `annex.alwayscommit`
+
+  By default, git-annex automatically commits data to the git-annex branch
+  after each command is run. To disable these commits,
+  set to `false`. Then data will only be committed when
+  running `git annex merge` (or by automatic merges) or `git annex sync`.
+
 * `remote..annex-cost`
 
   When determining which repository to

From 1f73db3469e29448bcb1520893de11b23da6fb1f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 16:11:47 -0400
Subject: [PATCH 3179/8313] improve alwayscommit=false mode

Now changes are staged into the branch's index, but not committed,
which avoids growing a large journal. And sync and merge always
explicitly commit, ensuring that even when they do nothing else,
they commit the staged changes.

Added a flag file to indicate that the branch's journal contains
uncommitted changes. (Could use git ls-files, but don't want to run
that every time.)

In the future, this ability to have uncommitted changes staged in the
journal might be used on remotes after a series of oneshot commands.
---
 Annex/Branch.hs  | 33 ++++++++++++++++++++++++++++++---
 Annex/Content.hs |  8 +++++---
 Command/Merge.hs |  2 ++
 Command/Sync.hs  |  1 +
 Locations.hs     |  5 +++++
 debian/changelog |  2 +-
 6 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index acab417fb3..52089ac97d 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -18,6 +18,7 @@ module Annex.Branch (
 	get,
 	change,
 	commit,
+	stage,
 	files,
 ) where
 
@@ -114,14 +115,14 @@ updateTo pairs = do
 	-- ensure branch exists, and get its current ref
 	branchref <- getBranch
 	-- check what needs updating before taking the lock
-	dirty <- journalDirty
+	dirty <- unCommitted
 	(refs, branches) <- unzip <$> filterM isnewer pairs
 	if not dirty && null refs
 		then updateIndex branchref
 		else withIndex $ lockJournal $ do
 			when dirty stageJournal
 			let merge_desc = if null branches
-				then "update" 
+				then "update"
 				else "merging " ++
 					unwords (map Git.Ref.describe branches) ++ 
 					" into " ++ show name
@@ -182,11 +183,17 @@ set file content = do
 
 {- Stages the journal, and commits staged changes to the branch. -}
 commit :: String -> Annex ()
-commit message = whenM journalDirty $ lockJournal $ do
+commit message = whenM unCommitted $ lockJournal $ do
 	stageJournal
 	ref <- getBranch
 	withIndex $ commitBranch ref message [fullname]
 
+{- Stages the journal, not making a commit to the branch. -}
+stage :: Annex ()
+stage = whenM journalDirty $ lockJournal $ do
+	stageJournal
+	setUnCommitted
+
 {- Commits the staged changes in the index to the branch.
  - 
  - Ensures that the branch's index file is first updated to the state
@@ -213,6 +220,7 @@ commitBranch branchref message parents = do
 	parentrefs <- commitparents <$> catObject committedref
 	when (racedetected branchref parentrefs) $
 		fixrace committedref parentrefs
+	setCommitted
 	where
 		-- look for "parent ref" lines and return the refs
 		commitparents = map (Git.Ref . snd) . filter isparent .
@@ -301,6 +309,25 @@ setIndexSha ref = do
         lock <- fromRepo gitAnnexIndexLock
 	liftIO $ writeFile lock $ show ref ++ "\n"
 
+{- Checks if there are uncommitted changes in the branch's index or journal. -}
+unCommitted :: Annex Bool
+unCommitted = do
+	d <- liftIO . doesFileExist =<< fromRepo gitAnnexIndexDirty
+	if d
+		then return d
+		else journalDirty
+
+setUnCommitted :: Annex ()
+setUnCommitted = do
+	file <- fromRepo gitAnnexIndexDirty
+	liftIO $ writeFile file "1"
+
+setCommitted :: Annex ()
+setCommitted = do
+	file <- fromRepo gitAnnexIndexDirty
+	_ <- liftIO $ tryIO $ removeFile file
+	return ()
+
 {- Stages the journal into the index. -}
 stageJournal :: Annex ()
 stageJournal = do
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 3a388129e2..f328051e3a 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -312,9 +312,11 @@ getKeysPresent' dir = do
 saveState :: Bool -> Annex ()
 saveState oneshot = do
 	Annex.Queue.flush False
-	alwayscommit <- Git.configTrue <$> fromRepo (Git.Config.get "annex.alwayscommit" "true")
-	unless (oneshot || not alwayscommit) $ do
-		Annex.Branch.commit "update"
+	unless oneshot $ do
+		alwayscommit <- Git.configTrue <$> fromRepo (Git.Config.get "annex.alwayscommit" "true")
+		if alwayscommit
+			then Annex.Branch.commit "update"
+			else Annex.Branch.stage
 
 {- Downloads content from any of a list of urls. -}
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
diff --git a/Command/Merge.hs b/Command/Merge.hs
index c1f7e899af..0f46614971 100644
--- a/Command/Merge.hs
+++ b/Command/Merge.hs
@@ -26,4 +26,6 @@ start = do
 perform :: CommandPerform
 perform = do
 	Annex.Branch.update
+	-- commit explicitly, in case no remote branches were merged
+	Annex.Branch.commit "update"
 	next $ return True
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 8e237ae84d..f7ebba6f54 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -75,6 +75,7 @@ commit = do
 	showStart "commit" ""
 	next $ next $ do
 		showOutput
+		Annex.Branch.commit "update"
 		-- Commit will fail when the tree is clean, so ignore failure.
 		_ <- inRepo $ Git.Command.runBool "commit"
 			[Param "-a", Param "-m", Param "git-annex automatic sync"]
diff --git a/Locations.hs b/Locations.hs
index 03d6deb1d7..244388e0e4 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -22,6 +22,7 @@ module Locations (
 	gitAnnexJournalLock,
 	gitAnnexIndex,
 	gitAnnexIndexLock,
+	gitAnnexIndexDirty,
 	gitAnnexSshDir,
 	isLinkToAnnex,
 	annexHashes,
@@ -143,6 +144,10 @@ gitAnnexIndex r = gitAnnexDir r  "index"
 gitAnnexIndexLock :: Git.Repo -> FilePath
 gitAnnexIndexLock r = gitAnnexDir r  "index.lck"
 
+{- Flag file for .git/annex/index. -}
+gitAnnexIndexDirty :: Git.Repo -> FilePath
+gitAnnexIndexDirty r = gitAnnexDir r  "index.dirty"
+
 {- .git/annex/ssh/ is used for ssh connection caching -}
 gitAnnexSshDir :: Git.Repo -> FilePath
 gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r  "ssh"
diff --git a/debian/changelog b/debian/changelog
index 44cdb37828..bb838ed603 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -37,7 +37,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     a remote's location log.
   * To avoid commits of data to the git-annex branch after each command
     is run, set annex.alwayscommit=false. Its data will then be committed
-    less frequently, when a merge or sync is done. 
+    less frequently, when a merge or sync is done.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From a3c9d06a265b2d6d3003af805b8345e4ddd3d87c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 16:31:38 -0400
Subject: [PATCH 3180/8313] add git-annex-shell commit

Eventually, git-annex might try running this after making changes to
a remote. I have not yet thought of a good way for it to tell which
remotes it needs to run it on though. It can't just do it when
shutting down a cached ssh connection, because ssh connection caching
is optional, and that would not handle local remotes not accessed over ssh
either.
---
 Command/Commit.hs        | 23 +++++++++++++++++++++++
 doc/git-annex-shell.mdwn |  4 ++++
 git-annex-shell.hs       |  2 ++
 3 files changed, 29 insertions(+)
 create mode 100644 Command/Commit.hs

diff --git a/Command/Commit.hs b/Command/Commit.hs
new file mode 100644
index 0000000000..2bb016ea05
--- /dev/null
+++ b/Command/Commit.hs
@@ -0,0 +1,23 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Commit where
+
+import Command
+import qualified Annex.Branch
+
+def :: [Command]
+def = [command "commit" paramNothing seek
+	"commits any staged changes to the git-annex branch"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStart
+start = next $ next $ do
+	Annex.Branch.commit "update"
+	return True
diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 7a65f10775..0fd77a8115 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -46,6 +46,10 @@ first "/~/" or "/~user/" is expanded to the specified home directory.
 
   This runs rsync in server mode to transfer out the content of a key.
 
+* commit
+
+  This commits any staged changes to the git-annex branch.
+
 # OPTIONS
 
 Most options are the same as in git-annex. The ones specific
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 4fdeae1a87..396b7b7905 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -20,6 +20,7 @@ import qualified Command.InAnnex
 import qualified Command.DropKey
 import qualified Command.RecvKey
 import qualified Command.SendKey
+import qualified Command.Commit
 
 cmds_readonly :: [Command]
 cmds_readonly = concat
@@ -32,6 +33,7 @@ cmds_notreadonly :: [Command]
 cmds_notreadonly = concat
 	[ Command.RecvKey.def
 	, Command.DropKey.def
+	, Command.Commit.def
 	]
 
 cmds :: [Command]

From c3fbe07d7ad03944d0967ebfa2b8f65cbc2ad5dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 18:02:49 -0400
Subject: [PATCH 3181/8313] do a cleanup commit after moving data from or to a
 git remote

Added Annex.cleanup, which is a general purpose interface for adding
actions to run at the end.

Remotes with the old git-annex-shell will commit every time, and have no
commit command, so hide stderr when running the commit command.
---
 Annex.hs                              | 12 +++++++++--
 CmdLine.hs                            |  2 ++
 Remote/Git.hs                         | 31 ++++++++++++++++++++++-----
 debian/changelog                      |  8 ++-----
 doc/bugs/copy_doesn__39__t_scale.mdwn |  3 +++
 5 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 123c9facf9..ef95ff174c 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -21,6 +21,7 @@ module Annex (
 	setField,
 	getFlag,
 	getField,
+	addCleanup,
 	gitRepo,
 	inRepo,
 	fromRepo,
@@ -93,6 +94,7 @@ data AnnexState = AnnexState
 	, lockpool :: M.Map FilePath Fd
 	, flags :: M.Map String Bool
 	, fields :: M.Map String String
+	, cleanup :: M.Map String (Annex ())
 	}
 
 newState :: Git.Repo -> AnnexState
@@ -117,6 +119,7 @@ newState gitrepo = AnnexState
 	, lockpool = M.empty
 	, flags = M.empty
 	, fields = M.empty
+	, cleanup = M.empty
 	}
 
 {- Create and returns an Annex state object for the specified git repo. -}
@@ -132,12 +135,17 @@ eval s a = evalStateT (runAnnex a) s
 {- Sets a flag to True -}
 setFlag :: String -> Annex ()
 setFlag flag = changeState $ \s ->
-	s { flags = M.insert flag True $ flags s }
+	s { flags = M.insertWith' const flag True $ flags s }
 
 {- Sets a field to a value -}
 setField :: String -> String -> Annex ()
 setField field value = changeState $ \s ->
-	s { fields = M.insert field value $ fields s }
+	s { fields = M.insertWith' const field value $ fields s }
+
+{- Adds a cleanup action to perform. -}
+addCleanup :: String -> Annex () -> Annex ()
+addCleanup uid a = changeState $ \s ->
+	s { cleanup = M.insertWith' const uid a $ cleanup s }
 
 {- Checks if a flag was set. -}
 getFlag :: String -> Annex Bool
diff --git a/CmdLine.hs b/CmdLine.hs
index fbc1eaeca0..05f7bfe2eb 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -12,6 +12,7 @@ module CmdLine (
 ) where
 
 import qualified Control.Exception as E
+import qualified Data.Map as M
 import Control.Exception (throw)
 import System.Console.GetOpt
 
@@ -95,6 +96,7 @@ startup = return True
 shutdown :: Bool -> Annex Bool
 shutdown oneshot = do
 	saveState oneshot
+	sequence_ =<< M.elems <$> Annex.getState Annex.cleanup
 	liftIO Git.Command.reap -- zombies from long-running git processes
 	sshCleanup -- ssh connection caching
 	return True
diff --git a/Remote/Git.hs b/Remote/Git.hs
index c07ae3237b..29c50e87fd 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -1,6 +1,6 @@
 {- Standard git remotes.
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -24,6 +24,7 @@ import Logs.Presence
 import Annex.UUID
 import qualified Annex.Content
 import qualified Annex.BranchState
+import qualified Annex.Branch
 import qualified Utility.Url as Url
 import Utility.TempFile
 import Config
@@ -196,7 +197,7 @@ keyUrls r key = map tourl (annexLocations key)
 
 dropKey :: Git.Repo -> Key -> Annex Bool
 dropKey r key
-	| not $ Git.repoIsUrl r = liftIO $ onLocal r $ do
+	| not $ Git.repoIsUrl r = commitOnCleanup r $ liftIO $ onLocal r $ do
 		ensureInitialized
 		whenM (Annex.Content.inAnnex key) $ do
 			Annex.Content.lockContent key $
@@ -205,7 +206,7 @@ dropKey r key
 			Annex.Content.saveState True
 		return True
 	| Git.repoIsHttp r = error "dropping from http repo not supported"
-	| otherwise = onRemote r (boolSystem, False) "dropkey"
+	| otherwise = commitOnCleanup r $ onRemote r (boolSystem, False) "dropkey"
 		[ Params "--quiet --force"
 		, Param $ show key
 		]
@@ -236,7 +237,7 @@ copyFromRemoteCheap r key file
 {- Tries to copy a key's content to a remote's annex. -}
 copyToRemote :: Git.Repo -> Key -> Annex Bool
 copyToRemote r key
-	| not $ Git.repoIsUrl r = do
+	| not $ Git.repoIsUrl r = commitOnCleanup r $ do
 		keysrc <- inRepo $ gitAnnexLocation key
 		params <- rsyncParams r
 		-- run copy from perspective of remote
@@ -245,7 +246,7 @@ copyToRemote r key
 			Annex.Content.saveState True `after`
 				Annex.Content.getViaTmp key
 					(rsyncOrCopyFile params keysrc)
-	| Git.repoIsSsh r = do
+	| Git.repoIsSsh r = commitOnCleanup r $ do
 		keysrc <- inRepo $ gitAnnexLocation key
 		rsyncHelper =<< rsyncParamsRemote r False key keysrc
 	| otherwise = error "copying to non-ssh repo not supported"
@@ -301,3 +302,23 @@ rsyncParams r = do
 	where
  		-- --inplace to resume partial files
 		options = [Params "-p --progress --inplace"]
+
+commitOnCleanup :: Git.Repo -> Annex a -> Annex a
+commitOnCleanup r a = go `after` a
+	where
+		go = Annex.addCleanup (Git.repoLocation r) cleanup
+		cleanup
+			| not $ Git.repoIsUrl r = liftIO $ onLocal r $
+				Annex.Branch.commit "update"
+			| otherwise = do
+				Just (shellcmd, shellparams) <-
+					git_annex_shell r "commit" []
+				-- Throw away stderr, since the remote may not
+				-- have a new enough git-annex shell to
+				-- support committing.
+				let cmd = shellcmd ++ " "
+					++ unwords (map shellEscape $ toCommand shellparams)
+					++ ">/dev/null 2>/dev/null"
+				_ <- liftIO $
+					boolSystem "sh" [Param "-c", Param cmd]
+				return ()
diff --git a/debian/changelog b/debian/changelog
index bb838ed603..9436d2e6d7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -29,12 +29,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * Store web special remote url info in a more efficient location.
   * Deal with NFS problem that caused a failure to remove a directory
     when removing content from the annex.
-  * Avoid repeated location log commits when a remote is receiving files.
-    Done by adding a oneshot mode, in which location log changes are
-    written to the journal, but not committed. Taking advantage of
-    git-annex's existing ability to recover in this situation. This is
-    used by git-annex-shell and other places where changes are made to
-    a remote's location log.
+  * Make a single location log commit after a remote has received or
+    dropped files. Uses a new "git-annex-shell commit" command.
   * To avoid commits of data to the git-annex branch after each command
     is run, set annex.alwayscommit=false. Its data will then be committed
     less frequently, when a merge or sync is done.
diff --git a/doc/bugs/copy_doesn__39__t_scale.mdwn b/doc/bugs/copy_doesn__39__t_scale.mdwn
index adbd0660af..a5ca9d9ee6 100644
--- a/doc/bugs/copy_doesn__39__t_scale.mdwn
+++ b/doc/bugs/copy_doesn__39__t_scale.mdwn
@@ -33,3 +33,6 @@ are local. It seems to be just overhead.)
 Oneshot mode is now implemented, making git-annex-shell and other
 short lifetime processes not bother with committing changes.
 [[done]] --[[Joey]] 
+
+Update: Now it makes one commit at the very end of such a mass transfer.
+--[[Joey]]

From 12b89a3eb81fdac92ec3ea9633bbd9a7d6a72adb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 19:15:29 -0400
Subject: [PATCH 3182/8313] configure: Check if ssh connection caching is
 supported by the installed version of ssh and default annex.sshcaching
 accordingly.

---
 Annex/Content.hs   |  3 ++-
 Annex/Ssh.hs       |  4 +++-
 Config.hs          |  2 +-
 Git.hs             | 12 +++++++++---
 configure.hs       |  7 +++++++
 debian/changelog   |  2 ++
 debian/control     |  1 +
 doc/git-annex.mdwn |  3 ++-
 8 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index f328051e3a..fdd03f320e 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -313,7 +313,8 @@ saveState :: Bool -> Annex ()
 saveState oneshot = do
 	Annex.Queue.flush False
 	unless oneshot $ do
-		alwayscommit <- Git.configTrue <$> fromRepo (Git.Config.get "annex.alwayscommit" "true")
+		alwayscommit <- fromMaybe True . Git.configTrue
+			<$> fromRepo (Git.Config.get "annex.alwayscommit" "")
 		if alwayscommit
 			then Annex.Branch.commit "update"
 			else Annex.Branch.stage
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index df9f0e410c..47f0ee4f60 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -16,6 +16,7 @@ import Common.Annex
 import Annex.LockPool
 import qualified Git
 import qualified Git.Config
+import qualified Build.SysConfig as SysConfig
 
 {- Generates parameters to ssh to a given host (or user@host) on a given
  - port, with connection caching. -}
@@ -37,7 +38,8 @@ sshParams (host, port) opts = go =<< sshInfo (host, port)
 
 sshInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam])
 sshInfo (host, port) = do
-	caching <- Git.configTrue <$> fromRepo (Git.Config.get "annex.sshcaching" "true")
+	caching <- fromMaybe SysConfig.sshconnectioncaching . Git.configTrue
+		<$> fromRepo (Git.Config.get "annex.sshcaching" "")
 	if caching
 		then do
 			dir <- fromRepo gitAnnexSshDir
diff --git a/Config.hs b/Config.hs
index 349ddf67f4..a93e2610e7 100644
--- a/Config.hs
+++ b/Config.hs
@@ -69,7 +69,7 @@ prop_cost_sane = False `notElem`
 
 {- Checks if a repo should be ignored. -}
 repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = not . Git.configTrue <$> getConfig r "ignore" "false"
+repoNotIgnored r = not . fromMaybe False . Git.configTrue <$> getConfig r "ignore" ""
 
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}
diff --git a/Git.hs b/Git.hs
index d8db471e90..284bf331c1 100644
--- a/Git.hs
+++ b/Git.hs
@@ -85,7 +85,8 @@ assertLocal repo action =
 		else error $ "acting on non-local git repo " ++  repoDescribe repo ++ 
 				" not supported"
 configBare :: Repo -> Bool
-configBare repo = maybe unknown configTrue $ M.lookup "core.bare" $ config repo
+configBare repo = maybe unknown (fromMaybe False . configTrue) $
+	M.lookup "core.bare" $ config repo
 	where
 		unknown = error $ "it is not known if git repo " ++
 			repoDescribe repo ++
@@ -112,5 +113,10 @@ workTree Repo { location = Dir d } = d
 workTree Repo { location = Unknown } = undefined
 
 {- Checks if a string from git config is a true value. -}
-configTrue :: String -> Bool
-configTrue s = map toLower s == "true"
+configTrue :: String -> Maybe Bool
+configTrue s
+	| s' == "true" = Just True
+	| s' == "false" = Just False
+	| otherwise = Nothing
+	where
+		s' = map toLower s
diff --git a/configure.hs b/configure.hs
index 772df3e38f..9dcc6a5017 100644
--- a/configure.hs
+++ b/configure.hs
@@ -4,9 +4,11 @@ import System.Directory
 import Data.List
 import Data.Maybe
 import System.Cmd.Utils
+import Control.Applicative
 
 import Build.TestConfig
 import Utility.StatFS
+import Utility.SafeCommand
 
 tests :: [TestCase]
 tests =
@@ -23,6 +25,7 @@ tests =
 	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
+	, TestCase "ssh connection caching" getSshConnectionCaching
 	, TestCase "StatFS" testStatFS
 	] ++ shaTestCases [1, 256, 512, 224, 384]
 
@@ -66,6 +69,10 @@ getGitVersion = do
 	let version = last $ words $ head $ lines s
 	return $ Config "gitversion" (StringConfig version)
 
+getSshConnectionCaching :: Test
+getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$>
+	boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"]
+
 testStatFS :: Test
 testStatFS = do
 	s <- getFileSystemStats "."
diff --git a/debian/changelog b/debian/changelog
index 9436d2e6d7..94bc093891 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -34,6 +34,8 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * To avoid commits of data to the git-annex branch after each command
     is run, set annex.alwayscommit=false. Its data will then be committed
     less frequently, when a merge or sync is done.
+  * configure: Check if ssh connection caching is supported by the installed
+    version of ssh and default annex.sshcaching accordingly.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 
diff --git a/debian/control b/debian/control
index 983c3da1ed..28f171c75f 100644
--- a/debian/control
+++ b/debian/control
@@ -23,6 +23,7 @@ Build-Depends:
 	git,
 	uuid,
 	rsync,
+	openssh-client,
 Maintainer: Joey Hess 
 Standards-Version: 3.9.2
 Vcs-Git: git://git.kitenet.net/git-annex
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0dc22b88d8..0928c36f6e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -604,7 +604,8 @@ Here are all the supported configuration settings.
 
 * `annex.sshcaching`
 
-  By default, git-annex caches ssh connections. To disable this, set to `false`.
+  By default, git-annex caches ssh connections
+  (if built using a new enough ssh). To disable this, set to `false`.
 
 * `annex.alwayscommit`
 

From b889581945f0444127b2e0aac29d0e101dcafca6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 25 Feb 2012 19:31:46 -0400
Subject: [PATCH 3183/8313] version dependency on openssh-client

This is only to ensure that it's as new a version as it was built with, so
partial upgrades work.
---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 28f171c75f..8ea1a6259e 100644
--- a/debian/control
+++ b/debian/control
@@ -37,7 +37,7 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	uuid,
 	rsync,
 	wget | curl,
-	openssh-client
+	openssh-client (>= 1:5.6p1)
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git
  git-annex allows managing files with git, without checking the file

From 00d814aecc7368bbc0fe1722b27340255772d2f2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 26 Feb 2012 14:11:50 -0400
Subject: [PATCH 3184/8313] fix filename encoding for git cat-file

The filename sent to git cat-file needs to be sent on a File encoded handle.

Also set the read handle to use the File encoding, so that any error
message mentioning the filename is received properly.

The actual file content is read using Data.ByteString.Char8, which
will ignore the read handle's encoding, so this won't change that.
(Whether that is entirely correct remains to be seen.)
---
 Git/CatFile.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index 41ecb362c1..c598d7aa4b 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -43,8 +43,11 @@ catFile h branch file = catObject h $ Ref $ show branch ++ ":" ++ file
 catObject :: CatFileHandle -> Ref -> IO L.ByteString
 catObject h object = CoProcess.query h send receive
 	where
-		send to = hPutStrLn to $ show object
+		send to = do
+			fileEncoding to
+			hPutStrLn to $ show object
 		receive from = do
+			fileEncoding from
 			header <- hGetLine from
 			case words header of
 				[sha, objtype, size]
@@ -56,7 +59,7 @@ catObject h object = CoProcess.query h send receive
 					| otherwise -> dne
 				_
 					| header == show object ++ " missing" -> dne
-					| otherwise -> error $ "unknown response from git cat-file " ++ header
+					| otherwise -> error $ "unknown response from git cat-file " ++ show (header, object)
 		readcontent bytes from = do
 			content <- S.hGet from bytes
 			c <- hGetChar from

From 2fd294d06f11c81a29cbf94a78952d1ad74dbf22 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 26 Feb 2012 14:59:12 -0400
Subject: [PATCH 3185/8313] move --from, copy --from: 10 times faster scanning
 remote on local disk

Rather than go through the location log to see which files are present on
the remote, it simply looks at the disk contents directly.

I benchmarked this speeding up scanning 834 files, from an annex on my
phone's SSD, from 11.39 seconds to 1.31 seconds. (No files actually moved.)

Also benchmarked 8139 files, from an annex on spinning storage,
speeding up from 103.17 to 13.39 seconds.

Note that benchmarking with an encrypted annex on flash actually showed a
minor slowdown with this optimisation -- from 13.93 to 14.50 seconds. Seems
the overhead of doing the crypto needed to get the filenames to directly
check can be higher than the overhead of looking up data in the location
log. (Which says good things about how well the location log and git have
been optimised!) It *may* make sense to make encrypted local remotes not
have hasKeyCheap set; further benchmarking is called for.
---
 Command/Move.hs  | 13 +++++++++----
 debian/changelog |  4 ++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/Command/Move.hs b/Command/Move.hs
index 2f2cd1b5d6..6b58f711aa 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -120,10 +120,15 @@ fromStart src move file key
 			showMoveAction move file
 			next $ fromPerform src move key
 fromOk :: Remote -> Key -> Annex Bool
-fromOk src key = do
-	u <- getUUID
-	remotes <- Remote.keyPossibilities key
-	return $ u /= Remote.uuid src && any (== src) remotes
+fromOk src key
+	| Remote.hasKeyCheap src =
+		either (const expensive) return =<< Remote.hasKey src key
+	| otherwise = expensive
+	where
+		expensive = do
+			u <- getUUID
+			remotes <- Remote.keyPossibilities key
+			return $ u /= Remote.uuid src && any (== src) remotes
 fromPerform :: Remote -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
 	ishere <- inAnnex key
diff --git a/debian/changelog b/debian/changelog
index 94bc093891..1d401149d3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -36,6 +36,10 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     less frequently, when a merge or sync is done.
   * configure: Check if ssh connection caching is supported by the installed
     version of ssh and default annex.sshcaching accordingly.
+  * move --from, copy --from: Now 10 times faster when scanning to find
+    files in a remote on a local disk; rather than go through the location log
+    to see which files are present on the remote, it simply looks at the 
+    disk contents directly.
 
  -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
 

From 8cae4115a8d10a67a618d08abc3971f520f8d66a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 27 Feb 2012 13:07:04 -0400
Subject: [PATCH 3186/8313] releasing version 3.20120227

---
 debian/changelog | 7 ++++---
 git-annex.cabal  | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 1d401149d3..6ba8696c0c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120124) UNRELEASED; urgency=low
+git-annex (3.20120227) unstable; urgency=low
 
   * Modifications to support ghc 7.4's handling of filenames.
     This version can only be built with ghc 7.4 or newer. See the ghc7.0
@@ -27,10 +27,11 @@ git-annex (3.20120124) UNRELEASED; urgency=low
   * rekey: New plumbing level command, can be used to change the keys used
     for files en masse.
   * Store web special remote url info in a more efficient location.
+    (Urls stored with this version will not be visible to older versions.)
   * Deal with NFS problem that caused a failure to remove a directory
     when removing content from the annex.
   * Make a single location log commit after a remote has received or
-    dropped files. Uses a new "git-annex-shell commit" command.
+    dropped files. Uses a new "git-annex-shell commit" command when available.
   * To avoid commits of data to the git-annex branch after each command
     is run, set annex.alwayscommit=false. Its data will then be committed
     less frequently, when a merge or sync is done.
@@ -41,7 +42,7 @@ git-annex (3.20120124) UNRELEASED; urgency=low
     to see which files are present on the remote, it simply looks at the 
     disk contents directly.
 
- -- Joey Hess   Tue, 24 Jan 2012 16:21:55 -0400
+ -- Joey Hess   Mon, 27 Feb 2012 12:58:21 -0400
 
 git-annex (3.20120123) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 8bfb3ac0c9..40f928dbf3 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120124
+Version: 3.20120227
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From ed0f5cd2d3625007b966470d76f2f031134e9a63 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 27 Feb 2012 13:07:09 -0400
Subject: [PATCH 3187/8313] add news item for git-annex 3.20120227

---
 doc/news/version_3.20120106.mdwn |  6 -----
 doc/news/version_3.20120227.mdwn | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 6 deletions(-)
 delete mode 100644 doc/news/version_3.20120106.mdwn
 create mode 100644 doc/news/version_3.20120227.mdwn

diff --git a/doc/news/version_3.20120106.mdwn b/doc/news/version_3.20120106.mdwn
deleted file mode 100644
index 076184aaf7..0000000000
--- a/doc/news/version_3.20120106.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 3.20120106 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Support unescaped repository urls, like git does.
-   * log: New command that displays the location log for files,
-     showing each repository they were added to and removed from.
-   * Fix overbroad gpg --no-tty fix from last release."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120227.mdwn b/doc/news/version_3.20120227.mdwn
new file mode 100644
index 0000000000..61a9fc8580
--- /dev/null
+++ b/doc/news/version_3.20120227.mdwn
@@ -0,0 +1,43 @@
+git-annex 3.20120227 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Modifications to support ghc 7.4's handling of filenames.
+     This version can only be built with ghc 7.4 or newer. See the ghc7.0
+     branch for older ghcs.
+   * S3: Fix irrefutable pattern failure when accessing encrypted S3
+     credentials.
+   * Use the haskell IfElse library.
+   * Fix teardown of stale cached ssh connections.
+   * Fixed to use the strict state monad, to avoid leaking all kinds of memory
+     due to lazy state update thunks when adding/fixing many files.
+   * Fixed some memory leaks that occurred when committing journal files.
+   * Added a annex.queuesize setting, useful when adding hundreds of thousands
+     of files on a system with plenty of memory.
+   * whereis: Prints the urls of files that the web special remote knows about.
+   * addurl --fast: Verifies that the url can be downloaded (only getting
+     its head), and records the size in the key.
+   * When checking that an url has a key, verify that the Content-Length,
+     if available, matches the size of the key.
+   * addurl: Added a --file option, which can be used to specify what
+     file the url is added to. This can be used to override the default
+     filename that is used when adding an url, which is based on the url.
+     Or, when the file already exists, the url is recorded as another
+     location of the file.
+   * addurl: Normalize badly encoded urls.
+   * addurl: Add --pathdepth option.
+   * rekey: New plumbing level command, can be used to change the keys used
+     for files en masse.
+   * Store web special remote url info in a more efficient location.
+     (Urls stored with this version will not be visible to older versions.)
+   * Deal with NFS problem that caused a failure to remove a directory
+     when removing content from the annex.
+   * Make a single location log commit after a remote has received or
+     dropped files. Uses a new "git-annex-shell commit" command when available.
+   * To avoid commits of data to the git-annex branch after each command
+     is run, set annex.alwayscommit=false. Its data will then be committed
+     less frequently, when a merge or sync is done.
+   * configure: Check if ssh connection caching is supported by the installed
+     version of ssh and default annex.sshcaching accordingly.
+   * move --from, copy --from: Now 10 times faster when scanning to find
+     files in a remote on a local disk; rather than go through the location log
+     to see which files are present on the remote, it simply looks at the
+     disk contents directly."""]]
\ No newline at end of file

From 6d3fb5cba71bb066e7493085f34b3f5e5aa46cd2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 27 Feb 2012 14:36:40 -0400
Subject: [PATCH 3188/8313] remove -fspec-constr-count workaround

not needed with ghc 7.4
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 43d498f1c1..6adbdc2df0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
-GHCFLAGS=-O2 -Wall $(IGNORE) -fspec-constr-count=8
+GHCFLAGS=-O2 -Wall $(IGNORE)
 
 ifdef PROFILE
 GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)

From 8428703d78c36ec859e0b920a73b3d1441773257 Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich 
Date: Tue, 28 Feb 2012 13:18:42 +0300
Subject: [PATCH 3189/8313] git-union-merge: add 'text' to dependencies

Fixes following build failure on ghc-7.4/Cabal-1.14:

    $ runhaskell Setup.hs build
    Building git-annex-3.20120227...
    Preprocessing executable 'git-union-merge' for git-annex-3.20120227...
    Git/Command.hs:11:18:
        Could not find module `Data.Text.Lazy.IO'
        It is a member of the hidden package `text-0.11.1.13'.
        Perhaps you need to add `text' to the build-depends in your .cabal file.
        Use -v to see a list of the files searched for.

Signed-off-by: Sergei Trofimovich 
---
 git-annex.cabal | 1 +
 1 file changed, 1 insertion(+)

diff --git a/git-annex.cabal b/git-annex.cabal
index 40f928dbf3..69a1d5b769 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -39,6 +39,7 @@ Executable git-annex-shell
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs
+  Build-Depends: text
 
 source-repository head
   type: git

From ea6cb182e0325a02b693b038b6a04ba58974dfc6 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Tue, 28 Feb 2012 17:40:00 +0000
Subject: [PATCH 3190/8313] Added a comment: Test-suite won't compile with GHC
 7.4.x

---
 .../comment_1_f8fc894680f2a2e5b5e757a677414b42._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment

diff --git a/doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment b/doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment
new file mode 100644
index 0000000000..aa3c5ce6d1
--- /dev/null
+++ b/doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.184.15.65"
+ subject="Test-suite won't compile with GHC 7.4.x"
+ date="2012-02-28T17:39:59Z"
+ content="""
+The recent version requires GHC 7.4.x, but some dependencies for the test suite don't build with that compiler, i.e. the `testpack` library. Do you have any recommendation how to deal with that situation? I would like to update, but I would very much like to run the regression test suite, too.
+"""]]

From d56da1b9cb30fc157927964dc233145562e00058 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 29 Feb 2012 04:27:47 +0000
Subject: [PATCH 3191/8313] Added a comment

---
 ...comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment

diff --git a/doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment b/doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment
new file mode 100644
index 0000000000..d45e66026f
--- /dev/null
+++ b/doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-02-29T04:27:47Z"
+ content="""
+Here's the patch that was used to make testpack build with 7.4 on Debian:
+
+
+"""]]

From e5fee3f35257d4a27e573398c114fa58f511da9d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Feb 2012 02:32:05 -0400
Subject: [PATCH 3192/8313] Fix test suite to not require a unicode locale.

Without a unicode locale, it will fail to print a unicode filename to
console, and fails.
---
 debian/changelog |  6 ++++++
 test.hs          | 10 +---------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 6ba8696c0c..f85ebb94db 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120229) unstable; urgency=low
+
+  * Fix test suite to not require a unicode locale.
+
+ -- Joey Hess   Wed, 29 Feb 2012 02:31:31 -0400
+
 git-annex (3.20120227) unstable; urgency=low
 
   * Modifications to support ghc 7.4's handling of filenames.
diff --git a/test.hs b/test.hs
index 69837f9eda..bbb8001d6e 100644
--- a/test.hs
+++ b/test.hs
@@ -131,7 +131,7 @@ test_init = "git-annex init" ~: TestCase $ innewrepo $ do
 		reponame = "test repo"
 
 test_add :: Test
-test_add = "git-annex add" ~: TestList [basic, sha1dup, sha1unicode, subdirs]
+test_add = "git-annex add" ~: TestList [basic, sha1dup, subdirs]
 	where
 		-- this test case runs in the main repo, to set up a basic
 		-- annexed file that later tests will use
@@ -158,10 +158,6 @@ test_add = "git-annex add" ~: TestList [basic, sha1dup, sha1unicode, subdirs]
 			git_annex "add" [sha1annexedfiledup, "--backend=SHA1"] @? "add of second file with same SHA1 failed"
 			annexed_present sha1annexedfiledup
 			annexed_present sha1annexedfile
-		sha1unicode = TestCase $ intmpclonerepo $ do
-			writeFile sha1annexedfileunicode $ content sha1annexedfileunicode
-			git_annex "add" [sha1annexedfileunicode, "--backend=SHA1"] @? "add of unicode filename failed"
-			annexed_present sha1annexedfileunicode
 		subdirs = TestCase $ intmpclonerepo $ do
 			createDirectory "dir"
 			writeFile "dir/foo" $ content annexedfile
@@ -923,9 +919,6 @@ sha1annexedfile = "sha1foo"
 sha1annexedfiledup :: String
 sha1annexedfiledup = "sha1foodup"
 
-sha1annexedfileunicode :: String
-sha1annexedfileunicode = "foo¡"
-
 ingitfile :: String
 ingitfile = "bar"
 
@@ -935,7 +928,6 @@ content f
 	| f == ingitfile = "normal file content"
 	| f == sha1annexedfile ="sha1 annexed file content"
 	| f == sha1annexedfiledup = content sha1annexedfile
-	| f == sha1annexedfileunicode ="sha1 annexed file content ¡ünicodé!"
 	| f == wormannexedfile = "worm annexed file content"
 	| otherwise = "unknown file " ++ f
 

From 6571831b92cebe6fc28483608d3ea4eb0adcf4ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Feb 2012 02:39:44 -0400
Subject: [PATCH 3193/8313] releasing version 3.20120229

---
 debian/changelog | 1 +
 git-annex.cabal  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index f85ebb94db..0c07cb6ff3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20120229) unstable; urgency=low
 
   * Fix test suite to not require a unicode locale.
+  * Fix cabal build failure. Thanks, Sergei Trofimovich
 
  -- Joey Hess   Wed, 29 Feb 2012 02:31:31 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 69a1d5b769..e135771844 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120227
+Version: 3.20120229
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 478fdc8a5d8b87b6ccbdff61e1e2e88032d34b48 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 29 Feb 2012 02:39:56 -0400
Subject: [PATCH 3194/8313] add news item for git-annex 3.20120229

---
 doc/news/version_3.20120113.mdwn | 8 --------
 doc/news/version_3.20120115.mdwn | 8 --------
 doc/news/version_3.20120229.mdwn | 4 ++++
 3 files changed, 4 insertions(+), 16 deletions(-)
 delete mode 100644 doc/news/version_3.20120113.mdwn
 delete mode 100644 doc/news/version_3.20120115.mdwn
 create mode 100644 doc/news/version_3.20120229.mdwn

diff --git a/doc/news/version_3.20120113.mdwn b/doc/news/version_3.20120113.mdwn
deleted file mode 100644
index 74611e7ba2..0000000000
--- a/doc/news/version_3.20120113.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20120113 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * log: Add --gource mode, which generates output usable by gource.
-   * map: Fix display of remote repos
-   * Add annex-trustlevel configuration settings, which can be used to
-     override the trust level of a remote.
-   * git-annex, git-union-merge: Support GIT\_DIR and GIT\_WORK\_TREE.
-   * Add libghc-testpack-dev to build depends on all arches."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120115.mdwn b/doc/news/version_3.20120115.mdwn
deleted file mode 100644
index 8aee547a31..0000000000
--- a/doc/news/version_3.20120115.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-git-annex 3.20120115 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Add a sanity check for bad StatFS results. On architectures
-     where StatFS does not currently work (s390, mips, powerpc, sparc),
-     this disables the diskreserve checking code, and attempting to
-     configure an annex.diskreserve will result in an error.
-   * Fix QuickCheck dependency in cabal file.
-   * Minor optimisations."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120229.mdwn b/doc/news/version_3.20120229.mdwn
new file mode 100644
index 0000000000..9b6d663a05
--- /dev/null
+++ b/doc/news/version_3.20120229.mdwn
@@ -0,0 +1,4 @@
+git-annex 3.20120229 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix test suite to not require a unicode locale.
+   * Fix cabal build failure. Thanks, Sergei Trofimovich"""]]
\ No newline at end of file

From 432dde91567b6984324d56c2ca8736a09bea910e Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Wed, 29 Feb 2012 19:20:21 +0000
Subject: [PATCH 3195/8313] Added a comment: How do you build the Crypto
 library with GHC 7.4.1?

---
 .../comment_1_18158b9be2313f49509d59295c7d3c90._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment

diff --git a/doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment b/doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment
new file mode 100644
index 0000000000..c3d2cab59a
--- /dev/null
+++ b/doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.152.146"
+ subject="How do you build the Crypto library with GHC 7.4.1?"
+ date="2012-02-29T19:20:20Z"
+ content="""
+`Crypto 4.2.4` doesn't seem to compile with GHC 7.4.1. How did you build that package?
+"""]]

From 98747ae86befc2bcbbd30fac97527014b664cabf Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 29 Feb 2012 22:54:01 +0000
Subject: [PATCH 3196/8313] Added a comment

---
 ...ent_2_03436ddda42decf8cb1b4d5316d88a75._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment

diff --git a/doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment b/doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment
new file mode 100644
index 0000000000..b08712ee8f
--- /dev/null
+++ b/doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-02-29T22:54:01Z"
+ content="""
+Probably this patch will help with Crypto:
+
+
+
+Or, there's the `ghc7.0` branch of git-annex in git, which can be used to build with the older, stable ghc.
+
+BTW, when asking, for help, a log of the build failure is always a good idea..
+"""]]

From 1098bc37ab2aea2510ef41f9d9f6a6909b252cdd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 1 Mar 2012 22:35:10 -0400
Subject: [PATCH 3197/8313] "here" can be used to refer to the current
 repository, which can read better than the old "." (which still works too).

---
 Command/Unused.hs  | 1 +
 Remote.hs          | 1 +
 debian/changelog   | 7 +++++++
 doc/git-annex.mdwn | 8 ++++----
 4 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 58a99882eb..bbef835f73 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -48,6 +48,7 @@ start = do
 	let (name, action) = case from of
 		Nothing -> (".", checkUnused)
 		Just "." -> (".", checkUnused)
+		Just "here" -> (".", checkUnused)
 		Just n -> (n, checkRemoteUnused n)
 	showStart "unused" name
 	next action
diff --git a/Remote.hs b/Remote.hs
index f676ddddcf..b3f464f5c2 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -90,6 +90,7 @@ byName' n = do
  - .git/config. -}
 nameToUUID :: String -> Annex UUID
 nameToUUID "." = getUUID -- special case for current repo
+nameToUUID "here" = getUUID
 nameToUUID "" = error "no remote specified"
 nameToUUID n = byName' n >>= go
 	where
diff --git a/debian/changelog b/debian/changelog
index 0c07cb6ff3..47713e2269 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120230) UNRELEASED; urgency=low
+
+  * "here" can be used to refer to the current repository,
+    which can read better than the old "." (which still works too).
+
+ -- Joey Hess   Thu, 01 Mar 2012 22:34:27 -0400
+
 git-annex (3.20120229) unstable; urgency=low
 
   * Fix test suite to not require a unicode locale.
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0928c36f6e..85d972259b 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -177,7 +177,7 @@ subdirectories).
 
   The repository to describe can be specified by git remote name or
   by uuid. To change the description of the current repository, use
-  "."
+  "here".
 
 * initremote name [param=value ...]
 
@@ -196,7 +196,7 @@ subdirectories).
   Records that a repository is trusted to not unexpectedly lose
   content. Use with care.
 
-  To trust the current repository, use "."
+  To trust the current repository, use "here".
 
 * untrust [repository ...]
 
@@ -273,7 +273,7 @@ subdirectories).
   By default, only lists annexed files whose content is currently present.
   This can be changed by specifying file matching options. To list all
   annexed files, present or not, specify --include "*". To list all
-  annexed files whose content is not present, specify --not --in="."
+  annexed files whose content is not present, specify --not --in=here
 
   To output filenames terminated with nulls, for use with xargs -0,
   specify --print0. Or, a custom output formatting can be specified using
@@ -527,7 +527,7 @@ file contents are present at either of two repositories.
 
   The repository should be specified using the name of a configured remote,
   or the UUID or description of a repository. For the current repository,
-  use "--in=."
+  use --in=here
 
 * --copies=number
 

From 3436aba6de8cad844691ca979ab5825f7011fb7e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Mar 2012 18:05:55 -0400
Subject: [PATCH 3198/8313] Directory special remotes now support chunking
 files written to them

Avoiding writing files larger than a specified size is useful on certian
things. For example, box.com has a file size limit of 100 mb. Could also
be useful on really crappy removable media.
---
 Remote/Directory.hs                | 222 +++++++++++++++++++++--------
 debian/changelog                   |   2 +
 doc/special_remotes/directory.mdwn |  19 +++
 3 files changed, 185 insertions(+), 58 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index ee2a0d75aa..6f7ce8cd62 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -1,6 +1,6 @@
 {- A "remote" that is just a filesystem directory.
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -19,6 +19,8 @@ import Utility.FileMode
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
+import Utility.DataUnits
+import Data.Int
 
 remote :: RemoteType
 remote = RemoteType {
@@ -32,24 +34,39 @@ gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
 	dir <- getConfig r "directory" (error "missing directory")
 	cst <- remoteCost r cheapRemoteCost
+	let chunksize = chunkSize c
 	return $ encryptableRemote c
-		(storeEncrypted dir)
-		(retrieveEncrypted dir)
+		(storeEncrypted dir chunksize)
+		(retrieveEncrypted dir chunksize)
 		Remote {
 			uuid = u,
 			cost = cst,
 			name = Git.repoDescribe r,
- 			storeKey = store dir,
-			retrieveKeyFile = retrieve dir,
-			retrieveKeyFileCheap = retrieveCheap dir,
-			removeKey = remove dir,
-			hasKey = checkPresent dir,
+ 			storeKey = store dir chunksize,
+			retrieveKeyFile = retrieve dir chunksize,
+			retrieveKeyFileCheap = retrieveCheap dir chunksize,
+			removeKey = remove dir chunksize,
+			hasKey = checkPresent dir chunksize,
 			hasKeyCheap = True,
 			whereisKey = Nothing,
 			config = Nothing,
 			repo = r,
 			remotetype = remote
 		}
+	where
+
+type ChunkSize = Maybe Int64
+
+chunkSize :: Maybe RemoteConfig -> ChunkSize
+chunkSize Nothing = Nothing
+chunkSize (Just m) =
+	case M.lookup "chunksize" m of
+		Nothing -> Nothing
+		Just v -> case readSize dataUnits v of
+			Nothing -> error "bad chunksize"
+			Just size
+				| size <= 0 -> error "bad chunksize"
+				| otherwise -> Just $ fromInteger size
 
 directorySetup :: UUID -> RemoteConfig -> Annex RemoteConfig
 directorySetup u c = do
@@ -69,69 +86,158 @@ directorySetup u c = do
 locations :: FilePath -> Key -> [FilePath]
 locations d k = map (d ) (keyPaths k)
 
-withCheckedFile :: (FilePath -> IO Bool) -> FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
-withCheckedFile _ [] _ _ = return False
-withCheckedFile check d k a = go $ locations d k
+{- An infinite stream of chunks to use for a given file. -}
+chunkStream :: FilePath -> [FilePath]
+chunkStream f = map tochunk [1 :: Integer ..]
+	where
+		tochunk n = f ++ ".chunk" ++ show n
+
+{- A file that records the number of chunks used. -}
+chunkCount :: FilePath -> FilePath
+chunkCount f = f ++ ".chunkcount"
+
+withCheckedFiles :: (FilePath -> IO Bool) -> ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool
+withCheckedFiles _ _ [] _ _ = return False
+withCheckedFiles check Nothing d k a = go $ locations d k
 	where
 		go [] = return False
 		go (f:fs) = do
 			use <- check f
 			if use
-				then a f
+				then a [f]
 				else go fs
-
-withStoredFile :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
-withStoredFile = withCheckedFile doesFileExist
-
-store :: FilePath -> Key -> Annex Bool
-store d k = do
-	src <- inRepo $ gitAnnexLocation k
-	liftIO $ catchBoolIO $ storeHelper d k $ copyFileExternal src
-
-storeEncrypted :: FilePath -> (Cipher, Key) -> Key -> Annex Bool
-storeEncrypted d (cipher, enck) k = do
-	src <- inRepo $ gitAnnexLocation k
-	liftIO $ catchBoolIO $ storeHelper d enck $ encrypt src
+withCheckedFiles check (Just _) d k a = go $ locations d k
 	where
-		encrypt src dest = do
-			withEncryptedContent cipher (L.readFile src) $ L.writeFile dest
-			return True
+		go [] = return False
+		go (f:fs) = do
+			let chunkcount = chunkCount f
+			use <- check chunkcount
+			if use
+				then do
+					count <- readcount chunkcount
+					let chunks = take count $ chunkStream f
+					ok <- all id <$> mapM check chunks
+					if ok
+						then a chunks
+						else return False
+				else go fs
+		readcount f = fromMaybe (error $ "cannot parse " ++ f)
+			. (readish :: String -> Maybe Int)
+			<$> readFile f
 
-storeHelper :: FilePath -> Key -> (FilePath -> IO Bool) -> IO Bool
-storeHelper d key a = do
-	let dest = Prelude.head $ locations d key
-	let tmpdest = dest ++ ".tmp"
-	let dir = parentDir dest
+withStoredFiles :: ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool
+withStoredFiles = withCheckedFiles doesFileExist
+
+store :: FilePath -> ChunkSize -> Key -> Annex Bool
+store d chunksize k = do
+	src <- inRepo $ gitAnnexLocation k
+	liftIO $ catchBoolIO $ storeHelper d chunksize k $ \dests ->
+		case chunksize of
+			Nothing -> do
+				let dest = Prelude.head dests
+				ok <- copyFileExternal src dest
+				return $ if ok then [dest] else []
+			Just _ -> storeSplit chunksize dests =<< L.readFile src
+
+storeEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> Annex Bool
+storeEncrypted d chunksize (cipher, enck) k = do
+	src <- inRepo $ gitAnnexLocation k
+	liftIO $ catchBoolIO $ storeHelper d chunksize enck $ encrypt src
+	where
+		encrypt src dests = withEncryptedContent cipher (L.readFile src) $ \s ->
+			case chunksize of
+				Nothing -> do
+					let dest = Prelude.head dests
+					L.writeFile dest s
+					return [dest]
+				Just _ -> storeSplit chunksize dests s
+
+{- Splits a ByteString into chunks and writes to dests.
+ - Note: Must always write at least one file, even for empty ByteString. -}
+storeSplit :: ChunkSize -> [FilePath] -> L.ByteString -> IO [FilePath]
+storeSplit Nothing _ _ = error "bad storeSplit call"
+storeSplit _ [] _ = error "bad storeSplit call"
+storeSplit (Just chunksize) alldests@(firstdest:_) s
+	| L.null s = do
+		-- must always write at least one file, even for empty
+		L.writeFile firstdest s
+		return [firstdest]
+	| otherwise = storeSplit' chunksize alldests s []
+storeSplit' :: Int64 -> [FilePath] -> L.ByteString -> [FilePath] -> IO [FilePath]
+storeSplit' _ [] _ _ = error "expected an infinite list"
+storeSplit' chunksize (d:dests) s c
+	| L.null s = return $ reverse c
+	| otherwise = do
+		let (chunk, rest) = L.splitAt chunksize s
+		L.writeFile d chunk
+		storeSplit' chunksize dests rest (d:c)
+
+{- Generates a list of destinations to write to in order to store a key.
+ - When chunksize is specified, this list will be a list of chunks.
+ - The action should store the file, and return a list of the destinations
+ - it stored it to, or [] on error.
+ - The stored files are only put into their final place once storage is
+ - complete.
+ -}
+storeHelper :: FilePath -> ChunkSize -> Key -> ([FilePath] -> IO [FilePath]) -> IO Bool
+storeHelper d chunksize key a = do
+	let dir = parentDir desttemplate
 	createDirectoryIfMissing True dir
 	allowWrite dir
-	ok <- a tmpdest
-	when ok $ do
-		renameFile tmpdest dest
+	stored <- a tmpdests
+	forM_ stored $ \f -> do
+		let dest = detmpprefix f
+		renameFile f dest
 		preventWrite dest
-		preventWrite dir
-	return ok
+	when (chunksize /= Nothing) $ do
+		let chunkcount = chunkCount desttemplate
+		_ <- tryIO $ allowWrite chunkcount
+		writeFile chunkcount (show $ length stored)
+		preventWrite chunkcount
+	preventWrite dir
+	return (not $ null stored)
+	where
+		desttemplate = Prelude.head $ locations d key
+		tmpdests = case chunksize of
+			Nothing -> [desttemplate ++ tmpprefix]
+			Just _ -> map (++ tmpprefix) (chunkStream desttemplate)
+		tmpprefix = ".tmp"
+		detmpprefix f = take (length f - tmpprefixlen) f
+		tmpprefixlen = length tmpprefix
 
-retrieve :: FilePath -> Key -> FilePath -> Annex Bool
-retrieve d k f = liftIO $ withStoredFile d k $ \file -> copyFileExternal file f
+retrieve :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
+retrieve d chunksize k f = liftIO $ withStoredFiles chunksize d k go
+	where
+		go [file] = copyFileExternal file f
+		go files = catchBoolIO $ do
+			L.writeFile f =<< (L.concat <$> mapM L.readFile files)
+			return True
 
-retrieveCheap :: FilePath -> Key -> FilePath -> Annex Bool
-retrieveCheap d k f = liftIO $ withStoredFile d k $ \file ->
-	catchBoolIO $ createSymbolicLink file f >> return True
-
-retrieveEncrypted :: FilePath -> (Cipher, Key) -> FilePath -> Annex Bool
-retrieveEncrypted d (cipher, enck) f =
-	liftIO $ withStoredFile d enck $ \file -> catchBoolIO $ do
-		withDecryptedContent cipher (L.readFile file) $ L.writeFile f
+retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> FilePath -> Annex Bool
+retrieveEncrypted d chunksize (cipher, enck) f =
+	liftIO $ withStoredFiles chunksize d enck $ \files -> catchBoolIO $ do
+		withDecryptedContent cipher (L.concat <$> mapM L.readFile files) $
+			L.writeFile f
 		return True
 
-remove :: FilePath -> Key -> Annex Bool
-remove d k = liftIO $ withStoredFile d k $ \file -> catchBoolIO $ do
-	let dir = parentDir file
-	allowWrite dir
-	removeFile file
-	removeDirectory dir
-	return True
+retrieveCheap :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
+retrieveCheap _ (Just _) _ _ = return False -- no cheap retrieval for chunks
+retrieveCheap d _ k f = liftIO $ withStoredFiles Nothing d k go
+	where
+		go [file] = catchBoolIO $ createSymbolicLink file f >> return True
+		go _files = return False
 
-checkPresent :: FilePath -> Key -> Annex (Either String Bool)
-checkPresent d k = liftIO $ catchMsgIO $ withStoredFile d k $
-	const $ return True -- withStoredFile checked that it exists
+remove :: FilePath -> ChunkSize -> Key -> Annex Bool
+remove d chunksize k = liftIO $ withStoredFiles chunksize d k go
+	where
+		go files = all id <$> mapM removefile files
+		removefile file = catchBoolIO $ do
+			let dir = parentDir file
+			allowWrite dir
+			removeFile file
+			_ <- tryIO $ removeDirectory dir
+			return True
+
+checkPresent :: FilePath -> ChunkSize -> Key -> Annex (Either String Bool)
+checkPresent d chunksize k = liftIO $ catchMsgIO $ withStoredFiles chunksize d k $
+	const $ return True -- withStoredFiles checked that it exists
diff --git a/debian/changelog b/debian/changelog
index 47713e2269..d5706d7be9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (3.20120230) UNRELEASED; urgency=low
 
   * "here" can be used to refer to the current repository,
     which can read better than the old "." (which still works too).
+  * Directory special remotes now support chunking files written to them,
+    avoiding writing files larger than a specified size.
 
  -- Joey Hess   Thu, 01 Mar 2012 22:34:27 -0400
 
diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn
index 0a38c763cc..100e068789 100644
--- a/doc/special_remotes/directory.mdwn
+++ b/doc/special_remotes/directory.mdwn
@@ -5,6 +5,25 @@ you want to use it to sneakernet files between systems (possibly with
 [[encrypted|encryption]] contents). Just set up both systems to use
 the drive's mountpoint as a directory remote.
 
+## configuration
+
+These parameters can be passed to `git annex initremote` to configure the
+remote:
+
+* `encryption` - Required. Either "none" to disable encryption of content
+  stored in the directory,
+  or a value that can be looked up (using gpg -k) to find a gpg encryption
+  key that will be given access to the remote. Note that additional gpg
+  keys can be given access to a remote by rerunning initremote with
+  the new key id. See [[encryption]].
+* `chunksize` - Avoid storing files larger than the specified size in the
+  directory. For use on directories on mount points that have file size
+  limitations. The default is to never chunk files.
+  Note that changing the chunk size of an existing remote is 
+  not recommended.
+  The value can use specified using any commonly used units.
+  Example: `chunksize=100 megabytes`
+
 Setup example:
 
 	# git annex initremote usbdrive type=directory directory=/media/usbdrive/ encryption=none

From 50c897c082f766777134626b7e8571f0adc2a473 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Mar 2012 20:02:48 -0400
Subject: [PATCH 3199/8313] tweak

---
 Remote/Directory.hs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 6f7ce8cd62..d5394fe519 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -88,9 +88,7 @@ locations d k = map (d ) (keyPaths k)
 
 {- An infinite stream of chunks to use for a given file. -}
 chunkStream :: FilePath -> [FilePath]
-chunkStream f = map tochunk [1 :: Integer ..]
-	where
-		tochunk n = f ++ ".chunk" ++ show n
+chunkStream f = map (\n -> f ++ ".chunk" ++ show n) [1 :: Integer ..]
 
 {- A file that records the number of chunks used. -}
 chunkCount :: FilePath -> FilePath

From 8fc533643d0acd5cddbdfede1a438a84c57329ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 3 Mar 2012 20:06:27 -0400
Subject: [PATCH 3200/8313] instructions for using Box.com as a special remote

I was sucked in by the 50 gb free lifetime storage offer.
Happily, it was pretty easy to get it to work with git-annex.
---
 doc/special_remotes.mdwn                      | 10 +++-
 .../using_box.com_as_a_special_remote.mdwn    | 50 +++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 doc/tips/using_box.com_as_a_special_remote.mdwn

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index ddb2fd125d..2d0474ffe5 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -13,7 +13,15 @@ They cannot be used by other git commands though.
 * [[rsync]]
 * [[web]]
 * [[hook]]
-* [[tahoe-lafs|forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]] - limited testing
+
+The above special remotes can be used to tie git-annex 
+into many cloud services. Here are specific instructions
+for various cloud things:
+
+* [[tips/using_Amazon_S3]]
+* [[tips/Internet_Archive_via_S3]]
+* [[tahoe-lafs|forum/tips:_special__95__remotes__47__hook_with_tahoe-lafs]]
+* [[tips/using_box.com_as_a_special_remote]]
 
 ## Unused content on special remotes
 
diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn
new file mode 100644
index 0000000000..2e6f738668
--- /dev/null
+++ b/doc/tips/using_box.com_as_a_special_remote.mdwn
@@ -0,0 +1,50 @@
+[Box.com](http://box.com/) is a file storage service, currently notable 
+for providing 50 gb of free storage if you sign up with its Android client.
+(Or a few GB free otherwise.)
+
+With a little setup, git-annex can use Box as a
+[[special remote|special_remotes]]. 
+
+## davfs2 setup
+
+* First, install
+  the [davfs2](http://savannah.nongnu.org/projects/davfs2) program,
+  which can mount Box using WebDAV. On Debian, just `sudo apt-get install davfs2`
+* Allow users to mount davfs filesystems, by ensuring that
+  `/sbin/mount.davfs` is setuid root. On Debian, just `sudo dpkg-reconfigure davfs2`
+* Add yourself to the davfs2 group.
+	sudo adduser $(whoami) davfs2
+* Edit `/etc/fstab`, and add a line to mount Box using davfs.
+	sudo mkdir -p /media/box.com
+	echo "https://www.box.com/dav/	/media/box.com	davfs	noauto,user	0 0" | sudo tee -a /etc/fstab
+* Create `~/.davfs2/davfs2.conf`:
+	mkdir ~/.davfs2/
+	echo use_locks 0 >> ~/.davfs2/davfs2.conf
+	echo delay_upload 0 >> ~/.davfs2/davfs2.conf
+* Create `~/.davfs2/secrets`. This file contains your Box.com login and password.
+  Your login is probably the email address you signed up with.
+	echo "/media/box.com joey@kitenet.net mypassword" > ~/.davfs2/secrets
+	chmod 600 ~/.davfs2/secrets
+* Now you should be able to mount Box, as a non-root user:
+	mount /media/box.com
+
+## git-annex setup
+
+You need git-annex version 3.20120303 or newer, which adds support for chunking
+files larger than Box's 100 mb limit.
+
+Create the special remote, in your git-annex repository.
+** This example is non-encrypted; fill in your gpg key ID for a securely
+encrypted special remote! **
+
+	git annex initremote box.com type=directory directory=/media/box.com chunksize=100mb encryption=none
+
+Now git-annex can copy files to box.com, get files from it, etc, just like
+with any other special remote.
+
+	% git annex copy bigfile --to box.com
+	bigfile (to box.com...) ok
+	% git annex drop bigfile
+	bigfile (checking box.com...) ok
+	% git annex get bigfile
+	bigfile (from box.com...) ok

From 9856c24a5996f2d493c559cd9ea6b27b8127694a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 03:17:03 -0400
Subject: [PATCH 3201/8313] Add progress bar display to the directory special
 remote.

So far I've only written progress bars for sending files, not yet
receiving.

No longer uses external cp at all. ByteString IO is fast enough.
---
 Messages.hs         | 25 +++++++++++++
 Remote/Directory.hs | 91 +++++++++++++++++++++++++++++----------------
 Remote/S3.hs        |  4 +-
 debian/changelog    |  1 +
 4 files changed, 86 insertions(+), 35 deletions(-)

diff --git a/Messages.hs b/Messages.hs
index 1b51cf23ea..e8bbfed133 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -10,6 +10,8 @@ module Messages (
 	showNote,
 	showAction,
 	showProgress,
+	metered,
+	MeterUpdate,
 	showSideAction,
 	showOutput,
 	showLongNote,
@@ -29,9 +31,13 @@ module Messages (
 ) where
 
 import Text.JSON
+import Data.Progress.Meter
+import Data.Progress.Tracker
+import Data.Quantity
 
 import Common
 import Types
+import Types.Key
 import qualified Annex
 import qualified Messages.JSON as JSON
 
@@ -46,10 +52,29 @@ showNote s = handle (JSON.note s) $
 showAction :: String -> Annex ()
 showAction s = showNote $ s ++ "..."
 
+{- Progress dots. -}
 showProgress :: Annex ()
 showProgress = handle q $
 	flushed $ putStr "."
 
+{- Shows a progress meter while performing a transfer of a key.
+ - The action is passed a callback to use to update the meter. -}
+type MeterUpdate = Integer -> IO ()
+metered :: Key -> (MeterUpdate -> Annex a) -> Annex a
+metered key a = Annex.getState Annex.output >>= go (keySize key)
+	where
+		go (Just size) Annex.NormalOutput = do
+			progress <- liftIO $ newProgress "" size
+			meter <- liftIO $ newMeter progress "B" 20 (renderNums binaryOpts 1)
+			showOutput
+			liftIO $ displayMeter stdout meter
+			r <- a $ \n -> liftIO $ do
+				incrP progress n
+				displayMeter stdout meter
+			liftIO $ clearMeter stdout meter
+			return r	
+                go _ _ = a (const $ return ())
+
 showSideAction :: String -> Annex ()
 showSideAction s = handle q $
 	putStrLn $ "(" ++ s ++ "...)"
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index d5394fe519..ab2a064ec1 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -8,7 +8,9 @@
 module Remote.Directory (remote) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
+import qualified Data.ByteString.Char8 as S
 import qualified Data.Map as M
+import Control.Exception (bracket)
 
 import Common.Annex
 import Utility.CopyFile
@@ -129,46 +131,71 @@ withStoredFiles = withCheckedFiles doesFileExist
 store :: FilePath -> ChunkSize -> Key -> Annex Bool
 store d chunksize k = do
 	src <- inRepo $ gitAnnexLocation k
-	liftIO $ catchBoolIO $ storeHelper d chunksize k $ \dests ->
-		case chunksize of
-			Nothing -> do
-				let dest = Prelude.head dests
-				ok <- copyFileExternal src dest
-				return $ if ok then [dest] else []
-			Just _ -> storeSplit chunksize dests =<< L.readFile src
+	metered k $ \meterupdate -> 
+		liftIO $ catchBoolIO $ storeHelper d chunksize k $ \dests ->
+			case chunksize of
+				Nothing -> do
+					let dest = Prelude.head dests
+					meteredWriteFile meterupdate dest
+						=<< L.readFile src
+					return [dest]
+				Just _ ->
+					storeSplit meterupdate chunksize dests
+						=<< L.readFile src
 
 storeEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d chunksize (cipher, enck) k = do
 	src <- inRepo $ gitAnnexLocation k
-	liftIO $ catchBoolIO $ storeHelper d chunksize enck $ encrypt src
-	where
-		encrypt src dests = withEncryptedContent cipher (L.readFile src) $ \s ->
-			case chunksize of
-				Nothing -> do
-					let dest = Prelude.head dests
-					L.writeFile dest s
-					return [dest]
-				Just _ -> storeSplit chunksize dests s
+	metered k $ \meterupdate ->
+		liftIO $ catchBoolIO $ storeHelper d chunksize enck $ \dests ->
+			withEncryptedContent cipher (L.readFile src) $ \s ->
+				case chunksize of
+					Nothing -> do
+						let dest = Prelude.head dests
+						meteredWriteFile meterupdate dest s
+						return [dest]
+					Just _ -> storeSplit meterupdate chunksize dests s
 
-{- Splits a ByteString into chunks and writes to dests.
+{- Splits a ByteString into chunks and writes to dests, obeying configured
+ - chunk size (not to be confused with the L.ByteString chunk size).
  - Note: Must always write at least one file, even for empty ByteString. -}
-storeSplit :: ChunkSize -> [FilePath] -> L.ByteString -> IO [FilePath]
-storeSplit Nothing _ _ = error "bad storeSplit call"
-storeSplit _ [] _ = error "bad storeSplit call"
-storeSplit (Just chunksize) alldests@(firstdest:_) s
-	| L.null s = do
+storeSplit :: MeterUpdate -> ChunkSize -> [FilePath] -> L.ByteString -> IO [FilePath]
+storeSplit _ Nothing _ _ = error "bad storeSplit call"
+storeSplit _ _ [] _ = error "bad storeSplit call"
+storeSplit meterupdate (Just chunksize) alldests@(firstdest:_) b
+	| L.null b = do
 		-- must always write at least one file, even for empty
-		L.writeFile firstdest s
+		L.writeFile firstdest b
 		return [firstdest]
-	| otherwise = storeSplit' chunksize alldests s []
-storeSplit' :: Int64 -> [FilePath] -> L.ByteString -> [FilePath] -> IO [FilePath]
-storeSplit' _ [] _ _ = error "expected an infinite list"
-storeSplit' chunksize (d:dests) s c
-	| L.null s = return $ reverse c
-	| otherwise = do
-		let (chunk, rest) = L.splitAt chunksize s
-		L.writeFile d chunk
-		storeSplit' chunksize dests rest (d:c)
+	| otherwise = storeSplit' meterupdate chunksize alldests (L.toChunks b) []
+storeSplit' :: MeterUpdate -> Int64 -> [FilePath] -> [S.ByteString] -> [FilePath] -> IO [FilePath]
+storeSplit' _ _ [] _ _ = error "ran out of dests"
+storeSplit' _ _  _ [] c = return $ reverse c
+storeSplit' meterupdate chunksize (d:dests) bs c = do
+	bs' <- bracket (openFile d WriteMode) hClose (feed chunksize bs)
+	storeSplit' meterupdate chunksize dests bs' (d:c)
+	where
+		feed _ [] _ = return []
+		feed sz (l:ls) h = do
+			let s = fromIntegral $ S.length l
+			if s <= sz
+				then do
+					S.hPut h l
+					meterupdate $ toInteger s
+					feed (sz - s) ls h
+				else return (l:ls)
+
+{- Write a L.ByteString to a file, updating a progress meter
+ - after each chunk of the L.ByteString, typically every 64 kb or so. -}
+meteredWriteFile :: MeterUpdate -> FilePath -> L.ByteString -> IO ()
+meteredWriteFile meterupdate dest b =
+	bracket (openFile dest WriteMode) hClose (feed $ L.toChunks b)
+	where
+		feed [] _ = return ()
+		feed (l:ls) h = do
+			S.hPut h l
+			meterupdate $ toInteger $ S.length l
+			feed ls h
 
 {- Generates a list of destinations to write to in order to store a key.
  - When chunksize is specified, this list will be a list of chunks.
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 812345b00a..523edef65e 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -144,9 +144,7 @@ storeHelper (conn, bucket) r k file = do
 			case fromJust $ M.lookup "storageclass" $ fromJust $ config r of
 				"REDUCED_REDUNDANCY" -> REDUCED_REDUNDANCY
 				_ -> STANDARD
-		getsize = do
-			s <- liftIO $ getFileStatus file
-			return $ fileSize s
+		getsize = fileSize <$> (liftIO $ getFileStatus file)
 		
 		xheaders = filter isxheader $ M.assocs $ fromJust $ config r
 		isxheader (h, _) = "x-amz-" `isPrefixOf` h
diff --git a/debian/changelog b/debian/changelog
index d5706d7be9..7b9db54184 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,7 @@ git-annex (3.20120230) UNRELEASED; urgency=low
     which can read better than the old "." (which still works too).
   * Directory special remotes now support chunking files written to them,
     avoiding writing files larger than a specified size.
+  * Add progress bar display to the directory special remote.
 
  -- Joey Hess   Thu, 01 Mar 2012 22:34:27 -0400
 

From 46383140015fb607ba7d9c7bf5a3c2c435dfb548 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 03:25:41 -0400
Subject: [PATCH 3202/8313] add progress display when receiving files

That was actually really easy. But, when getting a file from an encrypted
directory special remote, no meter can be shown, because the total file
size is not known.
---
 Remote/Directory.hs | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index ab2a064ec1..40e6dc72de 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -13,7 +13,6 @@ import qualified Data.Map as M
 import Control.Exception (bracket)
 
 import Common.Annex
-import Utility.CopyFile
 import Types.Remote
 import qualified Git
 import Config
@@ -231,11 +230,11 @@ storeHelper d chunksize key a = do
 		tmpprefixlen = length tmpprefix
 
 retrieve :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
-retrieve d chunksize k f = liftIO $ withStoredFiles chunksize d k go
-	where
-		go [file] = copyFileExternal file f
-		go files = catchBoolIO $ do
-			L.writeFile f =<< (L.concat <$> mapM L.readFile files)
+retrieve d chunksize k f = metered k $ \meterupdate ->
+	liftIO $ withStoredFiles chunksize d k $ \files ->
+		catchBoolIO $ do
+			meteredWriteFile meterupdate f =<<
+				(L.concat <$> mapM L.readFile files)
 			return True
 
 retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> FilePath -> Annex Bool

From 7ba79cfb8c5c2a0fcb1472ab67bb777999e8e88a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 03:36:39 -0400
Subject: [PATCH 3203/8313] thread through original key to retrieveEnctypted

Allows showing progress bar for this last case of the directory special
remote.
---
 Remote/Bup.hs                |  4 ++--
 Remote/Directory.hs          | 13 +++++++------
 Remote/Helper/Encryptable.hs |  4 ++--
 Remote/Hook.hs               |  4 ++--
 Remote/Rsync.hs              |  4 ++--
 Remote/S3.hs                 |  4 ++--
 6 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index a4f43a3f3e..4ac91e945f 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -129,8 +129,8 @@ retrieve buprepo k f = do
 retrieveCheap :: BupRepo -> Key -> FilePath -> Annex Bool
 retrieveCheap _ _ _ = return False
 
-retrieveEncrypted :: BupRepo -> (Cipher, Key) -> FilePath -> Annex Bool
-retrieveEncrypted buprepo (cipher, enck) f = do
+retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
+retrieveEncrypted buprepo (cipher, enck) _ f = do
 	let params = bupParams "join" buprepo [Param $ show enck]
 	liftIO $ catchBoolIO $ do
 		(pid, h) <- hPipeFrom "bup" $ toCommand params
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 40e6dc72de..e23891e5a4 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -237,12 +237,13 @@ retrieve d chunksize k f = metered k $ \meterupdate ->
 				(L.concat <$> mapM L.readFile files)
 			return True
 
-retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> FilePath -> Annex Bool
-retrieveEncrypted d chunksize (cipher, enck) f =
-	liftIO $ withStoredFiles chunksize d enck $ \files -> catchBoolIO $ do
-		withDecryptedContent cipher (L.concat <$> mapM L.readFile files) $
-			L.writeFile f
-		return True
+retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
+retrieveEncrypted d chunksize (cipher, enck) k f = metered k $ \meterupdate ->
+	liftIO $ withStoredFiles chunksize d enck $ \files ->
+		catchBoolIO $ do
+			withDecryptedContent cipher (L.concat <$> mapM L.readFile files) $
+				meteredWriteFile meterupdate f
+			return True
 
 retrieveCheap :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
 retrieveCheap _ (Just _) _ _ = return False -- no cheap retrieval for chunks
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index 0569cb5551..bcecb30cc4 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -40,7 +40,7 @@ encryptionSetup c =
 encryptableRemote
 	:: Maybe RemoteConfig
 	-> ((Cipher, Key) -> Key -> Annex Bool)
-	-> ((Cipher, Key) -> FilePath -> Annex Bool)
+	-> ((Cipher, Key) -> Key -> FilePath -> Annex Bool)
 	-> Remote
 	-> Remote
 encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = 
@@ -58,7 +58,7 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r =
 			(`storeKeyEncrypted` k)
 		retrieve k f = cip k >>= maybe
 			(retrieveKeyFile r k f)
-			(`retrieveKeyFileEncrypted` f)
+			(\enck -> retrieveKeyFileEncrypted enck k f)
 		retrieveCheap k f = cip k >>= maybe
 			(retrieveKeyFileCheap r k f)
 			(\_ -> return False)
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index c7d710f196..b37d5e2156 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -114,8 +114,8 @@ retrieve h k f = runHook h "retrieve" k (Just f) $ return True
 retrieveCheap :: String -> Key -> FilePath -> Annex Bool
 retrieveCheap _ _ _ = return False
 
-retrieveEncrypted :: String -> (Cipher, Key) -> FilePath -> Annex Bool
-retrieveEncrypted h (cipher, enck) f = withTmp enck $ \tmp ->
+retrieveEncrypted :: String -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
+retrieveEncrypted h (cipher, enck) _ f = withTmp enck $ \tmp ->
 	runHook h "retrieve" enck (Just tmp) $ liftIO $ catchBoolIO $ do
 		withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
 		return True
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 54fb890cae..577ea0b049 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -119,8 +119,8 @@ retrieveCheap o k f = do
 		then retrieve o k f
 		else return False
 
-retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> FilePath -> Annex Bool
-retrieveEncrypted o (cipher, enck) f = withTmp enck $ \tmp -> do
+retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
+retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do
 	res <- retrieve o enck tmp
 	if res
 		then liftIO $ catchBoolIO $ do
diff --git a/Remote/S3.hs b/Remote/S3.hs
index 523edef65e..a688ffcf34 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -161,8 +161,8 @@ retrieve r k f = s3Action r False $ \(conn, bucket) -> do
 retrieveCheap :: Remote -> Key -> FilePath -> Annex Bool
 retrieveCheap _ _ _ = return False
 
-retrieveEncrypted :: Remote -> (Cipher, Key) -> FilePath -> Annex Bool
-retrieveEncrypted r (cipher, enck) f = s3Action r False $ \(conn, bucket) -> do
+retrieveEncrypted :: Remote -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
+retrieveEncrypted r (cipher, enck) _ f = s3Action r False $ \(conn, bucket) -> do
 	res <- liftIO $ getObject conn $ bucketKey r bucket enck
 	case res of
 		Right o -> liftIO $ 

From 89cc2d128f833e65def1066fdf2dbd1c66c7dd11 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 10:49:28 -0400
Subject: [PATCH 3204/8313] make meter slightly wider

---
 Messages.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Messages.hs b/Messages.hs
index e8bbfed133..73a7d976fd 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -65,7 +65,7 @@ metered key a = Annex.getState Annex.output >>= go (keySize key)
 	where
 		go (Just size) Annex.NormalOutput = do
 			progress <- liftIO $ newProgress "" size
-			meter <- liftIO $ newMeter progress "B" 20 (renderNums binaryOpts 1)
+			meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1)
 			showOutput
 			liftIO $ displayMeter stdout meter
 			r <- a $ \n -> liftIO $ do

From 013c7ea44170023bf9d8471fb01312a7a3c789cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 10:51:25 -0400
Subject: [PATCH 3205/8313] clarification

---
 doc/special_remotes/directory.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/special_remotes/directory.mdwn b/doc/special_remotes/directory.mdwn
index 100e068789..7194e0d8e9 100644
--- a/doc/special_remotes/directory.mdwn
+++ b/doc/special_remotes/directory.mdwn
@@ -18,11 +18,11 @@ remote:
   the new key id. See [[encryption]].
 * `chunksize` - Avoid storing files larger than the specified size in the
   directory. For use on directories on mount points that have file size
-  limitations. The default is to never chunk files.
-  Note that changing the chunk size of an existing remote is 
-  not recommended.
+  limitations. The default is to never chunk files.  
   The value can use specified using any commonly used units.
-  Example: `chunksize=100 megabytes`
+  Example: `chunksize=100 megabytes`  
+  Note that enabling chunking on an existing remote with non-chunked
+  files is not recommended.
 
 Setup example:
 

From 896fce38472e32e73550b5113f00c9de2147982e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 11:02:41 -0400
Subject: [PATCH 3206/8313] box.com seems to behave better with smaller chunks

Seeing some weird failures in the davfs2 or box.com layer with 100 mb
chunks.
---
 doc/tips/using_box.com_as_a_special_remote.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn
index 2e6f738668..b216dd98de 100644
--- a/doc/tips/using_box.com_as_a_special_remote.mdwn
+++ b/doc/tips/using_box.com_as_a_special_remote.mdwn
@@ -37,7 +37,7 @@ Create the special remote, in your git-annex repository.
 ** This example is non-encrypted; fill in your gpg key ID for a securely
 encrypted special remote! **
 
-	git annex initremote box.com type=directory directory=/media/box.com chunksize=100mb encryption=none
+	git annex initremote box.com type=directory directory=/media/box.com chunksize=2mb encryption=none
 
 Now git-annex can copy files to box.com, get files from it, etc, just like
 with any other special remote.

From 612ca3cf2e217120d92061ef471b7242829ef229 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 11:12:52 -0400
Subject: [PATCH 3207/8313] tweak davfs2 settings

---
 doc/tips/using_box.com_as_a_special_remote.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn
index b216dd98de..7ef856178e 100644
--- a/doc/tips/using_box.com_as_a_special_remote.mdwn
+++ b/doc/tips/using_box.com_as_a_special_remote.mdwn
@@ -17,9 +17,10 @@ With a little setup, git-annex can use Box as a
 * Edit `/etc/fstab`, and add a line to mount Box using davfs.
 	sudo mkdir -p /media/box.com
 	echo "https://www.box.com/dav/	/media/box.com	davfs	noauto,user	0 0" | sudo tee -a /etc/fstab
-* Create `~/.davfs2/davfs2.conf`:
+* Create `~/.davfs2/davfs2.conf` with some important settings:
 	mkdir ~/.davfs2/
 	echo use_locks 0 >> ~/.davfs2/davfs2.conf
+	echo cache_size 1 >> ~/.davfs2/davfs2.conf
 	echo delay_upload 0 >> ~/.davfs2/davfs2.conf
 * Create `~/.davfs2/secrets`. This file contains your Box.com login and password.
   Your login is probably the email address you signed up with.

From 3960825ceffe2cd82c2fa91e55bff1f7bc75bdf9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 11:48:23 -0400
Subject: [PATCH 3208/8313] better chunked file retrieval

Avoids opening every chunk at once, instead streaming them in.

Not done for encrypted file retrieval yet.
---
 Remote/Directory.hs | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index e23891e5a4..80c45a6913 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -188,13 +188,23 @@ storeSplit' meterupdate chunksize (d:dests) bs c = do
  - after each chunk of the L.ByteString, typically every 64 kb or so. -}
 meteredWriteFile :: MeterUpdate -> FilePath -> L.ByteString -> IO ()
 meteredWriteFile meterupdate dest b =
-	bracket (openFile dest WriteMode) hClose (feed $ L.toChunks b)
+	meteredWriteFile' meterupdate dest (L.toChunks b) feeder
 	where
-		feed [] _ = return ()
-		feed (l:ls) h = do
-			S.hPut h l
-			meterupdate $ toInteger $ S.length l
-			feed ls h
+		feeder chunks = return ([], chunks)
+
+{- Writes a series of S.ByteString chunks to a file, updating a progress
+ - meter after each chunk. The feeder is called to get more chunks. -}
+meteredWriteFile' :: MeterUpdate -> FilePath -> s -> (s -> IO (s, [S.ByteString])) -> IO ()
+meteredWriteFile' meterupdate dest startstate feeder =
+	bracket (openFile dest WriteMode) hClose (feed startstate [])
+	where
+		feed state [] h = do
+			(state', cs) <- feeder state
+			if null cs then return () else feed state' cs h
+		feed state (c:cs) h = do
+			S.hPut h c
+			meterupdate $ toInteger $ S.length c
+			feed state cs h
 
 {- Generates a list of destinations to write to in order to store a key.
  - When chunksize is specified, this list will be a list of chunks.
@@ -233,9 +243,13 @@ retrieve :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
 retrieve d chunksize k f = metered k $ \meterupdate ->
 	liftIO $ withStoredFiles chunksize d k $ \files ->
 		catchBoolIO $ do
-			meteredWriteFile meterupdate f =<<
-				(L.concat <$> mapM L.readFile files)
+			meteredWriteFile' meterupdate f files feeder
 			return True
+	where
+		feeder [] = return ([], [])
+		feeder (x:xs) = do
+			chunks <- L.toChunks <$> L.readFile x
+			return (xs, chunks)
 
 retrieveEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted d chunksize (cipher, enck) k f = metered k $ \meterupdate ->

From fba66c55ed3e060aae90fe90b5ea8e3ec4132bb7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 13:16:35 -0400
Subject: [PATCH 3209/8313] foo

---
 doc/tips/using_box.com_as_a_special_remote.mdwn | 2 +-
 git-annex.cabal                                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn
index 7ef856178e..f9e2fbba5d 100644
--- a/doc/tips/using_box.com_as_a_special_remote.mdwn
+++ b/doc/tips/using_box.com_as_a_special_remote.mdwn
@@ -1,6 +1,6 @@
 [Box.com](http://box.com/) is a file storage service, currently notable 
 for providing 50 gb of free storage if you sign up with its Android client.
-(Or a few GB free otherwise.)
+(Or a few gb free otherwise.)
 
 With a little setup, git-annex can use Box as a
 [[special remote|special_remotes]]. 
diff --git a/git-annex.cabal b/git-annex.cabal
index e135771844..719e3d3fcb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120229
+Version: 3.20120230
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 52e88f3ebf974c3802e951e17593ce5768c04b92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 4 Mar 2012 16:00:24 -0400
Subject: [PATCH 3210/8313] add remote start and stop hooks

Locking is used, so that, if there are multiple git-annex processes
using a remote concurrently, the stop hook is only run by the last
process that uses it.
---
 Locations.hs           |  5 +++
 Remote/Helper/Hooks.hs | 93 ++++++++++++++++++++++++++++++++++++++++++
 Remote/List.hs         |  3 +-
 debian/changelog       |  3 ++
 doc/git-annex.mdwn     | 15 +++++++
 5 files changed, 118 insertions(+), 1 deletion(-)
 create mode 100644 Remote/Helper/Hooks.hs

diff --git a/Locations.hs b/Locations.hs
index 244388e0e4..18df060046 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -24,6 +24,7 @@ module Locations (
 	gitAnnexIndexLock,
 	gitAnnexIndexDirty,
 	gitAnnexSshDir,
+	gitAnnexRemotesDir,
 	isLinkToAnnex,
 	annexHashes,
 	hashDirMixed,
@@ -152,6 +153,10 @@ gitAnnexIndexDirty r = gitAnnexDir r  "index.dirty"
 gitAnnexSshDir :: Git.Repo -> FilePath
 gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r  "ssh"
 
+{- .git/annex/remotes/ is used for remote-specific state. -}
+gitAnnexRemotesDir :: Git.Repo -> FilePath
+gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r  "remotes"
+
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
 isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs
new file mode 100644
index 0000000000..5929b17935
--- /dev/null
+++ b/Remote/Helper/Hooks.hs
@@ -0,0 +1,93 @@
+{- Adds hooks to remotes.
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Remote.Helper.Hooks (addHooks) where
+
+import qualified Data.Map as M
+
+import Common.Annex
+import Types.Remote
+import qualified Annex
+import Annex.LockPool
+import Config
+
+{- Modifies a remote's access functions to first run the
+ - annex-start-command hook, and trigger annex-stop-command on shutdown.
+ - This way, the hooks are only run when a remote is actively being used.
+ -}
+addHooks :: Remote -> Annex Remote
+addHooks r = addHooks' r <$> lookupHook r "start" <*> lookupHook r "stop"
+addHooks' :: Remote -> Maybe String -> Maybe String -> Remote
+addHooks' r Nothing Nothing = r
+addHooks' r starthook stophook = r'
+	where
+		r' = r
+			{ storeKey = \k -> wrapper $ storeKey r k
+			, retrieveKeyFile = \k f -> wrapper $ retrieveKeyFile r k f
+			, retrieveKeyFileCheap = \k f -> wrapper $ retrieveKeyFileCheap r k f
+			, removeKey = \k -> wrapper $ removeKey r k
+			, hasKey = \k -> wrapper $ hasKey r k
+			}
+			where
+				wrapper = runHooks r' starthook stophook
+
+runHooks :: Remote -> Maybe String -> Maybe String -> Annex a -> Annex a
+runHooks r starthook stophook a = do
+	dir <- fromRepo gitAnnexRemotesDir
+	let lck = dir  remoteid ++ ".lck"
+	whenM (not . any (== lck) . M.keys <$> getPool) $ do
+		liftIO $ createDirectoryIfMissing True dir
+		firstrun lck
+	a
+	where
+		remoteid = show (uuid r)
+		run Nothing = return ()
+		run (Just command) = liftIO $ do
+			_ <- boolSystem "sh" [Param "-c", Param command]
+			return ()
+		firstrun lck = do
+			-- Take a shared lock; This indicates that git-annex
+			-- is using the remote, and prevents other instances
+			-- of it from running the stophook. If another
+			-- instance is shutting down right now, this
+			-- will block waiting for its exclusive lock to clear.
+			lockFile lck
+
+			-- The starthook is run even if some other git-annex
+			-- is already running, and ran it before.
+			-- It would be difficult to use locking to ensure
+			-- it's only run once, and it's also possible for
+			-- git-annex to be interrupted before it can run the
+			-- stophook, in which case the starthook
+			-- would be run again by the next git-annex.
+			-- So, requiring idempotency is the right approach.
+			run starthook
+
+			Annex.addCleanup (remoteid ++ "-stop-command") $
+				runstop lck
+		runstop lck = do
+			-- Drop any shared lock we have, and take an
+			-- exclusive lock, without blocking. If the lock
+			-- succeeds, we're the only process using this remote,
+			-- so can stop it.
+			unlockFile lck
+			fd <- liftIO $ openFd lck ReadWrite (Just stdFileMode) defaultFileFlags
+			v <- liftIO $ tryIO $
+				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
+			case v of
+				Left _ -> return ()
+				Right _ -> run stophook
+			liftIO $ closeFd fd
+
+lookupHook :: Remote -> String -> Annex (Maybe String)
+lookupHook r n = do
+	command <- getConfig (repo r) hookname ""
+	if null command
+		then return Nothing
+		else return $ Just command
+	where
+		hookname =  n ++ "-command"
diff --git a/Remote/List.hs b/Remote/List.hs
index e589b4401a..57dfa43ebf 100644
--- a/Remote/List.hs
+++ b/Remote/List.hs
@@ -15,6 +15,7 @@ import Logs.Remote
 import Types.Remote
 import Annex.UUID
 import Config
+import Remote.Helper.Hooks
 
 import qualified Remote.Git
 import qualified Remote.S3
@@ -51,7 +52,7 @@ remoteList = do
 		process m t = enumerate t >>= mapM (gen m t)
 		gen m t r = do
 			u <- getRepoUUID r
-			generate t r u (M.lookup u m)
+			addHooks =<< generate t r u (M.lookup u m)
 
 {- All remotes that are not ignored. -}
 enabledRemoteList :: Annex [Remote]
diff --git a/debian/changelog b/debian/changelog
index 7b9db54184..8ac34487b0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,9 @@ git-annex (3.20120230) UNRELEASED; urgency=low
   * Directory special remotes now support chunking files written to them,
     avoiding writing files larger than a specified size.
   * Add progress bar display to the directory special remote.
+  * Add configurable hooks that are run when git-annex starts and stops
+    using a remote: remote.name.annex-start-command and
+    remote.name.annex-stop-command
 
  -- Joey Hess   Thu, 01 Mar 2012 22:34:27 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 85d972259b..a941d4420e 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -627,6 +627,21 @@ Here are all the supported configuration settings.
   This allows varying the cost based on eg, the current network. The
   cost-command can be any shell command line.
 
+* `remote..annex-start-command`
+
+  A command to run when git-annex begins to use the remote. This can
+  be used to, for example, mount the directory containing the remote.
+
+  The command may be run repeatedly in multiple git-annex processes
+  are running concurrently.
+
+* `remote..annex-stop-command`
+
+  A command to run when git-annex is done using the remote.
+
+  The command will only be run once *all* running git-annex processes
+  are finished using the remote.
+
 * `remote..annex-ignore`
 
   If set to `true`, prevents git-annex

From 51338486dcf9ab86de426e41b1eb31af1d3a6c87 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Mar 2012 12:42:52 -0400
Subject: [PATCH 3211/8313] Fix a bug in symlink calculation code, that
 triggered in rare cases where an annexed file is in a subdirectory that
 nearly matched to the .git/annex/object/xx/yy subdirectories.

This is a straight up pure-code stinker. The relative path calculation
looked for common subdirectories in the two paths, but failed to stop
after the paths diverged. When a later pair of subdirectories were the
same, the resulting relative path was wrong.

Added regression test for this.
---
 Utility/Path.hs  | 11 ++++++++++-
 debian/changelog |  3 +++
 test.hs          |  1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Utility/Path.hs b/Utility/Path.hs
index ed5e59cb5f..eb530442b1 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -82,7 +82,7 @@ relPathDirToFile from to = join s $ dotdots ++ uncommon
 		s = [pathSeparator]
 		pfrom = split s from
 		pto = split s to
-		common = map fst $ filter same $ zip pfrom pto
+		common = map fst $ takeWhile same $ zip pfrom pto
 		same (c,d) = c == d
 		uncommon = drop numcommon pto
 		dotdots = replicate (length pfrom - numcommon) ".."
@@ -95,6 +95,15 @@ prop_relPathDirToFile_basics from to
 	where
 		r = relPathDirToFile from to 
 
+prop_relPathDirToFile_regressionTest :: Bool
+prop_relPathDirToFile_regressionTest = same_dir_shortcurcuits_at_difference
+	where
+		{- Two paths have the same directory component at the same
+		 - location, but it's not really the same directory.
+		 - Code used to get this wrong. -}
+		same_dir_shortcurcuits_at_difference =
+			relPathDirToFile "/tmp/r/lll/xxx/yyy/18" "/tmp/r/.git/annex/objects/18/gk/SHA256-foo/SHA256-foo" == "../../../../.git/annex/objects/18/gk/SHA256-foo/SHA256-foo"
+
 {- Given an original list of files, and an expanded list derived from it,
  - ensures that the original list's ordering is preserved. 
  -
diff --git a/debian/changelog b/debian/changelog
index 8ac34487b0..cbcb46e83c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,9 @@ git-annex (3.20120230) UNRELEASED; urgency=low
   * Add configurable hooks that are run when git-annex starts and stops
     using a remote: remote.name.annex-start-command and
     remote.name.annex-stop-command
+  * Fix a bug in symlink calculation code, that triggered in rare
+    cases where an annexed file is in a subdirectory that nearly
+    matched to the .git/annex/object/xx/yy subdirectories.
 
  -- Joey Hess   Thu, 01 Mar 2012 22:34:27 -0400
 
diff --git a/test.hs b/test.hs
index bbb8001d6e..431517114b 100644
--- a/test.hs
+++ b/test.hs
@@ -82,6 +82,7 @@ quickcheck = TestLabel "quickcheck" $ TestList
 	, qctest "prop_parentDir_basics" Utility.Path.prop_parentDir_basics
 
 	, qctest "prop_relPathDirToFile_basics" Utility.Path.prop_relPathDirToFile_basics
+	, qctest "prop_relPathDirToFile_regressionTest" Utility.Path.prop_relPathDirToFile_regressionTest
 	, qctest "prop_cost_sane" Config.prop_cost_sane
 	, qctest "prop_hmacWithCipher_sane" Crypto.prop_hmacWithCipher_sane
 	, qctest "prop_TimeStamp_sane" Logs.UUIDBased.prop_TimeStamp_sane

From 0d41899304810cc36205ad66d7e93455444f19ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Mar 2012 13:47:20 -0400
Subject: [PATCH 3212/8313] releasing version 3.20120230

---
 debian/changelog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index cbcb46e83c..97cf38348b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120230) UNRELEASED; urgency=low
+git-annex (3.20120230) unstable; urgency=low
 
   * "here" can be used to refer to the current repository,
     which can read better than the old "." (which still works too).
@@ -12,7 +12,7 @@ git-annex (3.20120230) UNRELEASED; urgency=low
     cases where an annexed file is in a subdirectory that nearly
     matched to the .git/annex/object/xx/yy subdirectories.
 
- -- Joey Hess   Thu, 01 Mar 2012 22:34:27 -0400
+ -- Joey Hess   Mon, 05 Mar 2012 13:38:13 -0400
 
 git-annex (3.20120229) unstable; urgency=low
 

From ee806c1175b6dcb59fb22aaa9b23480955e7630e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 5 Mar 2012 13:47:35 -0400
Subject: [PATCH 3213/8313] add news item for git-annex 3.20120230

---
 doc/news/version_3.20120116.mdwn |  6 ------
 doc/news/version_3.20120123.mdwn | 27 ---------------------------
 doc/news/version_3.20120230.mdwn | 13 +++++++++++++
 3 files changed, 13 insertions(+), 33 deletions(-)
 delete mode 100644 doc/news/version_3.20120116.mdwn
 delete mode 100644 doc/news/version_3.20120123.mdwn
 create mode 100644 doc/news/version_3.20120230.mdwn

diff --git a/doc/news/version_3.20120116.mdwn b/doc/news/version_3.20120116.mdwn
deleted file mode 100644
index 0fd07df0d5..0000000000
--- a/doc/news/version_3.20120116.mdwn
+++ /dev/null
@@ -1,6 +0,0 @@
-git-annex 3.20120116 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix data loss bug in directory special remote, when moving a file
-     to the remote failed, and partially transferred content was left
-     behind in the directory, re-running the same move would think it
-     succeeded and delete the local copy."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120123.mdwn b/doc/news/version_3.20120123.mdwn
deleted file mode 100644
index 4eb37ef2da..0000000000
--- a/doc/news/version_3.20120123.mdwn
+++ /dev/null
@@ -1,27 +0,0 @@
-News for git-annex 3.20120123:
-
-   There was a bug in the handling of directory special remotes that
-   could cause partial file contents to be stored in them. If you use
-   a directory special remote, you should fsck it, to avoid potential
-   data loss.
-   Example: git annex fsck --from mydirectory
-
-git-annex 3.20120123 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * fsck --from: Fscking a remote is now supported. It's done by retrieving
-     the contents of the specified files from the remote, and checking them,
-     so can be an expensive operation. Still, if the remote is a special
-     remote, or a git repository that you cannot run fsck in locally, it's
-     nice to have the ability to fsck it.
-   * If you have any directory special remotes, now would be a good time to
-     fsck them, in case you were hit by the data loss bug fixed in the
-     previous release!
-   * fsck --from remote --fast: Avoids expensive file transfers, at the
-     expense of not checking file size and/or contents.
-   * Ssh connection caching is now enabled automatically by git-annex.
-     Only one ssh connection is made to each host per git-annex run, which
-     can speed some things up a lot, as well as avoiding repeated password
-     prompts. Concurrent git-annex processes also share ssh connections.
-     Cached ssh connections are shut down when git-annex exits.
-   * To disable the ssh caching (if for example you have your own broader
-     ssh caching configuration), set annex.sshcaching=false."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120230.mdwn b/doc/news/version_3.20120230.mdwn
new file mode 100644
index 0000000000..52ac369e01
--- /dev/null
+++ b/doc/news/version_3.20120230.mdwn
@@ -0,0 +1,13 @@
+git-annex 3.20120230 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * "here" can be used to refer to the current repository,
+     which can read better than the old "." (which still works too).
+   * Directory special remotes now support chunking files written to them,
+     avoiding writing files larger than a specified size.
+   * Add progress bar display to the directory special remote.
+   * Add configurable hooks that are run when git-annex starts and stops
+     using a remote: remote.name.annex-start-command and
+     remote.name.annex-stop-command
+   * Fix a bug in symlink calculation code, that triggered in rare
+     cases where an annexed file is in a subdirectory that nearly
+     matched to the .git/annex/object/xx/yy subdirectories."""]]
\ No newline at end of file

From 6a1e334a78c94014f3aee8b383caf20c81f281d1 Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Mon, 5 Mar 2012 21:10:48 +0000
Subject: [PATCH 3214/8313] Added a comment

---
 ..._8f7f8d4758804f1b695925934219745a._comment | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment

diff --git a/doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment b/doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment
new file mode 100644
index 0000000000..a31a182cca
--- /dev/null
+++ b/doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.165.208"
+ subject="comment 3"
+ date="2012-03-05T21:10:47Z"
+ content="""
+Unfortunately, the patch you mentioned doesn't seem to address the problem. I'm getting the following compile error:
+
+    Data/Digest/SHA2.hs:111:4:
+        Could not deduce (Show a) arising from a use of `showHex'
+        from the context (Integral a)
+          bound by the instance declaration at Data/Digest/SHA2.hs:109:10-39
+        Possible fix:
+          add (Show a) to the context of the instance declaration
+        In the first argument of `(.)', namely `(showHex a)'
+        In the expression:
+          (showHex a)
+          . (' ' :)
+            . (showHex b)
+              . (' ' :)
+                . (showHex c)
+                  . (' ' :)
+                    . (showHex d)
+                      . (' ' :)
+                        . (showHex e)
+                          . (' ' :)
+                            . (showHex f) . (' ' :) . (showHex g) . (' ' :) . (showHex h)
+        In an equation for `showsPrec':
+            showsPrec _ (Hash8 a b c d e f g h)
+              = (showHex a)
+                . (' ' :)
+                  . (showHex b)
+                    . (' ' :)
+                      . (showHex c)
+                        . (' ' :)
+                          . (showHex d)
+                            . (' ' :)
+                              . (showHex e)
+                                . (' ' :)
+                                  . (showHex f) . (' ' :) . (showHex g) . (' ' :) . (showHex h)
+
+"""]]

From eedc774c8a48659c7091aa03263573ffc7a3df94 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 5 Mar 2012 21:29:46 +0000
Subject: [PATCH 3215/8313] Added a comment

---
 .../comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment

diff --git a/doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment b/doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment
new file mode 100644
index 0000000000..e9b2d3ab9c
--- /dev/null
+++ b/doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2012-03-05T21:29:46Z"
+ content="""
+Ah, the trouble is that's a patch nested inside a patch. So after you apply it, you need to `patch -p1 < patches/class-constraints.diff`
+"""]]

From 614208ad52ccaabd2f6c8f66522b69caf37f4294 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 5 Mar 2012 21:30:08 +0000
Subject: [PATCH 3216/8313] removed

---
 .../comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment   | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment

diff --git a/doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment b/doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment
deleted file mode 100644
index e9b2d3ab9c..0000000000
--- a/doc/news/version_3.20120229/comment_4_70e9cbbc47e02ce5641e1963a6d735f8._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 4"
- date="2012-03-05T21:29:46Z"
- content="""
-Ah, the trouble is that's a patch nested inside a patch. So after you apply it, you need to `patch -p1 < patches/class-constraints.diff`
-"""]]

From d2835d4304fdb31805ce14f92f0b4109cf94e652 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 5 Mar 2012 21:32:00 +0000
Subject: [PATCH 3217/8313] Added a comment

---
 .../comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment

diff --git a/doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment b/doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment
new file mode 100644
index 0000000000..a931c7874a
--- /dev/null
+++ b/doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2012-03-05T21:32:00Z"
+ content="""
+Hmm, I was able to produce exactly the same build error, and then I downloaded the patch I linked to before, and did
+`patch -p1 < debian/patches/class-constraints.diff` and that fixed the build nicely.
+"""]]

From b4b36b6ebe27bee5c7ece442865d4474a2667dca Mon Sep 17 00:00:00 2001
From: "http://peter-simons.myopenid.com/"
 
Date: Mon, 5 Mar 2012 23:29:42 +0000
Subject: [PATCH 3218/8313] Added a comment

---
 .../comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment

diff --git a/doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment b/doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment
new file mode 100644
index 0000000000..afddb3fb0a
--- /dev/null
+++ b/doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://peter-simons.myopenid.com/"
+ ip="77.186.165.208"
+ subject="comment 5"
+ date="2012-03-05T23:29:41Z"
+ content="""
+I didn't realize that the patch adds a patch file to the source distribution (instead of, well, patching it). That additional level of indirection surprised me. Anyway, now I figured it out and `Crypto` compiles fine. Thanks!
+"""]]

From ca936cd2d83b6c2c8519e676554603702eef6cf5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk_LOahrm_Cdg7io-_H0CNKkaxsRRQgRFo"
 
Date: Tue, 6 Mar 2012 11:20:36 +0000
Subject: [PATCH 3219/8313] Added a comment: Test suite failure

---
 ..._b975cbd3a01ba5c2fa0f24fe739d3433._comment | 128 ++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100644 doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment

diff --git a/doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment b/doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment
new file mode 100644
index 0000000000..eb5c7d257b
--- /dev/null
+++ b/doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment
@@ -0,0 +1,128 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk_LOahrm_Cdg7io-_H0CNKkaxsRRQgRFo"
+ nickname="Peter"
+ subject="Test suite failure"
+ date="2012-03-06T11:20:35Z"
+ content="""
+I managed to compile this version of git-annex with GHC 7.4.1 on NixOS, but unfortunately the test suite fails during the `addurl` test:
+
+    Testing 0:quickcheck:0:prop_idempotent_deencode_git                           
+    Testing 0:quickcheck:1:prop_idempotent_deencode                               
+    Testing 0:quickcheck:2:prop_idempotent_fileKey                                
+    Testing 0:quickcheck:3:prop_idempotent_key_read_show                          
+    Testing 0:quickcheck:4:prop_idempotent_shellEscape                            
+    Testing 0:quickcheck:5:prop_idempotent_shellEscape_multiword                  
+    Testing 0:quickcheck:6:prop_idempotent_configEscape                           
+    Testing 0:quickcheck:7:prop_parentDir_basics                                  
+    Testing 0:quickcheck:8:prop_relPathDirToFile_basics                           
+    Testing 0:quickcheck:9:prop_relPathDirToFile_regressionTest                   
+    Testing 0:quickcheck:10:prop_cost_sane                                        
+    Testing 0:quickcheck:11:prop_hmacWithCipher_sane                              
+    Testing 0:quickcheck:12:prop_TimeStamp_sane                                   
+    Testing 0:quickcheck:13:prop_addLog_sane                                      
+    Testing 1:blackbox:0:git-annex init                                           
+    Testing 1:blackbox:1:git-annex add:0                                          
+    Testing 1:blackbox:1:git-annex add:1                                          
+    Testing 1:blackbox:1:git-annex add:2                                          
+    Testing 1:blackbox:2:git-annex reinject/fromkey                               
+    Testing 1:blackbox:3:git-annex unannex:0:no content                           
+    Testing 1:blackbox:3:git-annex unannex:1:with content                         
+    Testing 1:blackbox:4:git-annex drop:0:no remotes                              
+    Testing 1:blackbox:4:git-annex drop:1:with remote                             
+    Testing 1:blackbox:4:git-annex drop:2:untrusted remote                        
+    Testing 1:blackbox:5:git-annex get                                            
+    Testing 1:blackbox:6:git-annex move                                           
+    Testing 1:blackbox:7:git-annex copy                                           
+    Testing 1:blackbox:8:git-annex unlock/lock                                    
+    Testing 1:blackbox:9:git-annex edit/commit:0                                  
+    Cases: 55  Tried: 28  Errors: 0  Failures: 0add foo (checksum...) ok
+    ok
+    (Recording state in git...)
+    Testing 1:blackbox:9:git-annex edit/commit:1                                  
+    Testing 1:blackbox:10:git-annex fix                                           
+    Testing 1:blackbox:11:git-annex trust/untrust/semitrust/dead                  
+    Testing 1:blackbox:12:git-annex fsck:0                                        
+    Cases: 55  Tried: 32  Errors: 0  Failures: 0  Only 1 of 2 trustworthy copies exist of foo
+      Back it up with git-annex copy.
+      Only 1 of 2 trustworthy copies exist of sha1foo
+      Back it up with git-annex copy.
+      Bad file size (11 B larger); moved to /tmp/nix-build-jzvhzrdysy619y4vgmafryy9ck8mz7z7-git-annex-3.20120230.drv-0/git-annex/.t/tmprepo/.git/annex/bad/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+      Bad file size (11 B larger); moved to /tmp/nix-build-jzvhzrdysy619y4vgmafryy9ck8mz7z7-git-annex-3.20120230.drv-0/git-annex/.t/tmprepo/.git/annex/bad/SHA1-s25--ee80d2cec57a3810db83b80e1b320df3a3721ffa
+    Testing 1:blackbox:12:git-annex fsck:1                                        
+    Testing 1:blackbox:12:git-annex fsck:2                                        
+    Cases: 55  Tried: 34  Errors: 0  Failures: 0  Only these untrusted locations may have copies of foo
+            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
+            17aab099-fcde-413a-8ef1-6acc09d7d081 -- here (.t/tmprepo)
+      Back it up to trusted locations with git-annex copy.
+      Only these untrusted locations may have copies of sha1foo
+            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
+      Back it up to trusted locations with git-annex copy.
+    Testing 1:blackbox:12:git-annex fsck:3                                        
+    Cases: 55  Tried: 35  Errors: 0  Failures: 0  Only 1 of 2 trustworthy copies exist of foo
+      Back it up with git-annex copy.
+      The following untrusted locations may also have copies: 
+            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
+      Only 1 of 2 trustworthy copies exist of sha1foo
+      Back it up with git-annex copy.
+      The following untrusted locations may also have copies: 
+            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
+    Testing 1:blackbox:13:git-annex migrate:0                                     
+    Testing 1:blackbox:13:git-annex migrate:1                                     
+    Testing 1:blackbox:14:git-annex unused/dropunused                             
+    Testing 1:blackbox:15:git-annex addurl                                        
+    Cases: 55  Tried: 39  Errors: 0  Failures: 0git-annex: connect: timeout (Connection timed out)
+    ### Failure in: 1:blackbox:15:git-annex addurl
+    addurl failed
+    Testing 1:blackbox:16:git-annex describe                                      
+    Testing 1:blackbox:17:git-annex find                                          
+    Cases: 55  Tried: 41  Errors: 0  Failures: 1foo
+    foo
+    sha1foo
+    sha1foo
+    Testing 1:blackbox:18:git-annex merge                                         
+    Testing 1:blackbox:19:git-annex status                                        
+    Cases: 55  Tried: 43  Errors: 0  Failures: 1{\"command\":\"status\",\"supported backends\":[\"SHA256\",\"SHA1\",\"SHA512\",\"SHA224\",\"SHA384\",\"SHA256E\",\"SHA1E\",\"SHA512E\",\"SHA224E\",\"SHA384E\",\"WORM\",\"URL\"],\"supported remote types\":[\"git\",\"S3\",\"bup\",\"directory\",\"rsync\",\"web\",\"hook\"],\"trusted repositories\":[],\"semitrusted repositories\":[{\"uuid\":\"00000000-0000-0000-0000-000000000001\",\"description\":\"web\",\"here\":false},{\"uuid\":\"17575c68-d5cc-4e18-bc96-fdafe716d488\",\"description\":\"origin (test repo)\",\"here\":false},{\"uuid\":\"5b9fe416-d6ed-4df7-af67-14fc5f2ea631\",\"description\":\".t/tmprepo\",\"here\":true}],\"untrusted repositories\":[],\"dead repositories\":[],\"local annex keys\":0,\"local annex size\":\"0 bytes\",\"known annex keys\":2,\"known annex size\":\"45 bytes\",\"success\":true}
+    Testing 1:blackbox:20:git-annex version                                       
+    Cases: 55  Tried: 44  Errors: 0  Failures: 1git-annex version: 3.20120230
+    local repository version: 3
+    default repository version: 3
+    supported repository versions: 3
+    upgrade supported from repository versions: 0 1 2
+    Testing 1:blackbox:21:git-annex sync                                          
+    Cases: 55  Tried: 45  Errors: 0  Failures: 1# On branch master
+    nothing to commit (working directory clean)
+    To /tmp/nix-build-jzvhzrdysy619y4vgmafryy9ck8mz7z7-git-annex-3.20120230.drv-0/git-annex/.t/repo
+       34fa270..7242932  git-annex -> git-annex
+     * [new branch]      master -> synced/master
+    Testing 1:blackbox:22:git-annex map                                           
+    Testing 1:blackbox:23:git-annex uninit                                        
+    Cases: 55  Tried: 47  Errors: 0  Failures: 1Switched to branch 'git-annex'
+    Switched to branch 'master'
+    Deleted branch git-annex (was e636789).
+    Testing 1:blackbox:24:git-annex upgrade                                       
+    Testing 1:blackbox:25:git-annex whereis                                       
+    Testing 1:blackbox:26:git-annex hook remote                                   
+    Testing 1:blackbox:27:git-annex directory remote                              
+    Testing 1:blackbox:28:git-annex rsync remote                                  
+    Cases: 55  Tried: 52  Errors: 0  Failures: 1sending incremental file list
+    af4/
+    af4/74c/
+    af4/74c/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77/
+    af4/74c/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+              20 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/5)
+
+    sent 300 bytes  received 43 bytes  686.00 bytes/sec
+    total size is 20  speedup is 0.06
+    SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+              20 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)
+
+    sent 160 bytes  received 31 bytes  382.00 bytes/sec
+    total size is 20  speedup is 0.10
+    Testing 1:blackbox:29:git-annex bup remote                                    
+    Testing 1:blackbox:30:git-annex crypto                                        
+    Cases: 55  Tried: 55  Errors: 0  Failures: 1
+    test: failed
+    ** test suite failed!
+
+Apparently, there is a network timeout? I see from the comments in `test.hs` that the test suite tries to avoid depending on network traffic, but is it possible, maybe, that the test tries to resolve a DNS name?
+"""]]

From b927dfd970d53418feeddf6c423ca43a81900155 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Mar 2012 13:21:46 -0400
Subject: [PATCH 3220/8313] remove addurl test

addurl --fast used to avoid network, but it always uses it now, getting at
least size. Thus not appropriate for test suite without a lot of work.
---
 test.hs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/test.hs b/test.hs
index 431517114b..5f4b0ad40e 100644
--- a/test.hs
+++ b/test.hs
@@ -107,7 +107,6 @@ blackbox = TestLabel "blackbox" $ TestList
 	, test_fsck
 	, test_migrate
 	, test_unused
-	, test_addurl
 	, test_describe
 	, test_find
 	, test_merge
@@ -503,13 +502,6 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 			r <- Backend.lookupFile f
 			return $ fst $ fromJust r
 
-test_addurl :: Test
-test_addurl = "git-annex addurl" ~: intmpclonerepo $ do
-	annexed_notpresent annexedfile
-	-- can't check download; test suite should not access network,
-	-- and starting up a web server seems excessive
-	git_annex "addurl" ["--fast", "http://example.com/nosuchfile"] @? "addurl failed"
-
 test_describe :: Test
 test_describe = "git-annex describe" ~: intmpclonerepo $ do
 	git_annex "describe" [".", "this repo"] @? "describe 1 failed"

From a78f699190520242c4f44a1eb5bb329ade1deefd Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 6 Mar 2012 17:22:54 +0000
Subject: [PATCH 3221/8313] Added a comment

---
 .../comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment

diff --git a/doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment b/doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment
new file mode 100644
index 0000000000..a4cc3073d7
--- /dev/null
+++ b/doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-03-06T17:22:54Z"
+ content="""
+My mistake, addurl --fast used to avoid the network, so the test suite ran it, but then it was changed to always look up the file size. Removed from test suite.
+"""]]

From d08ee1a9d20adbc6ead0bba50aadf4c99a6bce46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Mar 2012 13:56:20 -0400
Subject: [PATCH 3222/8313] syscall optimisation

---
 Annex/Ssh.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 47f0ee4f60..39983ab250 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -61,8 +61,8 @@ portParams (Just port) = [Param "-p", Param $ show port]
 sshCleanup :: Annex ()
 sshCleanup = do
 	dir <- fromRepo gitAnnexSshDir
-	liftIO $ createDirectoryIfMissing True dir
-	sockets <- filter (not . isLock) <$> liftIO (dirContents dir)
+	sockets <- filter (not . isLock) <$>
+		liftIO (catchDefaultIO (dirContents dir) [])
 	forM_ sockets cleanup
 	where
 		cleanup socketfile = do

From dc9049373e8af94f42f31027d963dbd612d678fc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Mar 2012 14:12:15 -0400
Subject: [PATCH 3223/8313] cleanup

---
 Command/Add.hs | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 28971529a7..b6b5753af7 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -58,14 +58,12 @@ perform file = do
  - This can be called before or after the symlink is in place. -}
 undo :: FilePath -> Key -> IOException -> Annex a
 undo file key e = do
-	unlessM (inAnnex key) rethrow -- no cleanup to do
-	liftIO $ whenM (doesFileExist file) $ removeFile file
-	handle tryharder $ fromAnnex key file
-	logStatus key InfoMissing
-	rethrow
+	whenM (inAnnex key) $ do
+		liftIO $ whenM (doesFileExist file) $ removeFile file
+		handle tryharder $ fromAnnex key file
+		logStatus key InfoMissing
+	throw e
 	where
-		rethrow = throw e
-
 		-- fromAnnex could fail if the file ownership is weird
 		tryharder :: IOException -> Annex ()
 		tryharder _ = do

From 581dc819e1025d00a6979a245a5c839fad116ea7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 6 Mar 2012 17:32:18 -0400
Subject: [PATCH 3224/8313] version base dependency for ghc 7.4

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 719e3d3fcb..46eacbaaf3 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -31,7 +31,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
-   base < 5, monad-control, transformers-base, lifted-base, IfElse,
+   base >= 4.5, base < 5, monad-control, transformers-base, lifted-base, IfElse,
    QuickCheck >= 2.1
 
 Executable git-annex-shell

From 789254747bceeaac004236275a6c1906f859945a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Mar 2012 18:52:03 -0400
Subject: [PATCH 3225/8313] refactor

---
 Utility/FileSystemEncoding.hs | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 Utility/FileSystemEncoding.hs

diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
new file mode 100644
index 0000000000..6970a10ded
--- /dev/null
+++ b/Utility/FileSystemEncoding.hs
@@ -0,0 +1,18 @@
+{- File system encoding handling.
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.FileSystemEncoding where
+
+import GHC.IO.Encoding (getFileSystemEncoding)
+import GHC.Foreign as GHC
+
+{- Marshal a Haskell FilePath into a NUL terminated C string using temporary
+ - storage. The FilePath is encoded using the filesystem encoding,
+ - reversing the decoding that should have been done when the FilePath
+ - was obtained. -}
+withFilePath :: FilePath -> (CString -> IO a) -> IO a
+withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f

From d6e77595ba45762b3c2dfdcd47a2d6b5b70154ae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Mar 2012 19:08:10 -0400
Subject: [PATCH 3226/8313] factor out Utility.FileSystemEncoding

---
 Common.hs                     |  1 +
 Utility/FileSystemEncoding.hs | 23 +++++++++++++++++++++--
 Utility/Misc.hs               |  8 --------
 Utility/StatFS.hsc            |  5 ++---
 Utility/Touch.hsc             |  5 ++---
 5 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/Common.hs b/Common.hs
index cc6cf92527..3475024601 100644
--- a/Common.hs
+++ b/Common.hs
@@ -26,5 +26,6 @@ import Utility.SafeCommand as X
 import Utility.Path as X
 import Utility.Directory as X
 import Utility.Monad as X
+import Utility.FileSystemEncoding as X
 
 import Utility.PartialPrelude as X
diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
index 6970a10ded..048323ee3c 100644
--- a/Utility/FileSystemEncoding.hs
+++ b/Utility/FileSystemEncoding.hs
@@ -1,4 +1,4 @@
-{- File system encoding handling.
+{- GHC File system encoding handling.
  -
  - Copyright 2012 Joey Hess 
  -
@@ -7,8 +7,17 @@
 
 module Utility.FileSystemEncoding where
 
-import GHC.IO.Encoding (getFileSystemEncoding)
+import System.IO
+import Foreign.C
 import GHC.Foreign as GHC
+import GHC.IO.Encoding
+
+{- Sets a Handle to use the filesystem encoding. This causes data
+ - written or read from it to be encoded/decoded the same
+ - as ghc 7.4 does to filenames etc. This special encoding
+ - allows "arbitrary undecodable bytes to be round-tripped through it". -}
+fileEncoding :: Handle -> IO ()
+fileEncoding h = hSetEncoding h =<< getFileSystemEncoding
 
 {- Marshal a Haskell FilePath into a NUL terminated C string using temporary
  - storage. The FilePath is encoded using the filesystem encoding,
@@ -16,3 +25,13 @@ import GHC.Foreign as GHC
  - was obtained. -}
 withFilePath :: FilePath -> (CString -> IO a) -> IO a
 withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f
+
+{- Encodes a FilePath into a String of encoded bytes, applying the
+ - filesystem encoding.
+ -
+ - This does not do any IO, beyond allocating a C buffer. GHC does not
+ - seem to provide a pure way to do this conversion. -}
+encodeFilePath :: FilePath -> IO String
+encodeFilePath fp = do
+	enc <- getFileSystemEncoding
+	GHC.withCString enc fp $ GHC.peekCString enc
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index 9c284c826f..3ac5ca5c0b 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -9,14 +9,6 @@ module Utility.Misc where
 
 import System.IO
 import Control.Monad
-import GHC.IO.Encoding
-
-{- Sets a Handle to use the filesystem encoding. This causes data
- - written or read from it to be encoded/decoded the same
- - as ghc 7.4 does to filenames et. This special encoding
- - allows "arbitrary undecodable bytes to be round-tripped through it". -}
-fileEncoding :: Handle -> IO ()
-fileEncoding h = hSetEncoding h =<< getFileSystemEncoding
 
 {- A version of hgetContents that is not lazy. Ensures file is 
  - all read before it gets closed. -}
diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc
index 51a6bda1e3..58d0b3e023 100644
--- a/Utility/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -47,15 +47,14 @@
 
 module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
 
+import Utility.FileSystemEncoding
+
 import Foreign
 import Foreign.C.Types
 import Foreign.C.String
 import GHC.IO.Encoding (getFileSystemEncoding)
 import GHC.Foreign as GHC
 
-withFilePath :: FilePath -> (CString -> IO a) -> IO a
-withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f
-
 #if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
 # include 
 # include 
diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index 24ccd17a62..b84054cbcd 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -13,15 +13,14 @@ module Utility.Touch (
 	touch
 ) where
 
+import Utility.FileSystemEncoding
+
 import Foreign
 import Foreign.C
 import Control.Monad (when)
 import GHC.IO.Encoding (getFileSystemEncoding)
 import GHC.Foreign as GHC
 
-withFilePath :: FilePath -> (CString -> IO a) -> IO a
-withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f
-
 newtype TimeSpec = TimeSpec CTime
 
 {- Changes the access and modification times of an existing file.

From bca3fd65b9cd3c2224144349505ddda20361b59e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Mar 2012 19:26:02 -0400
Subject: [PATCH 3227/8313] fix key directory hash calculation code

Fix Key directory hash calculation code to behave as it did before version
3.20120227 when a key contains non-ascii.

The hash directories for a given Key are based on its md5sum.
Prior to ghc 7.4, Keys contained raw, undecoded bytes, so the md5sum was
taken of each byte in turn. With the ghc 7.4 filename encoding change,
keys contains decoded unicode characters (possibly with surrigates for
undecodable bytes). This changes the result of the md5sum, since the md5sum
used is pure haskell and supports unicode. And that won't do, as git-annex
will start looking in a different hash directory for the content of a key.

The surrigates are particularly bad, since that's essentially a ghc
implementation detail, so could change again at any time. Also, changing
the locale changes how the bytes are decoded, which can also change
the md5sum.

Symptoms would include things like:

* git annex fsck would complain that no copies existed of a file,
  despite its symlink pointing to the content that was locally present
* git annex fix would change the symlink to use the wrong hash
  directory.

Only WORM backend is likely to have been affected, since only it tends
to include much filename data (SHA1E could in theory also be affected).

I have not tried to support the hash directories used by git-annex versions
3.20120227 to 3.20120308, so things added with those versions with WORM
will require manual fixups. Sorry for the inconvenience!
---
 Locations.hs                  |  4 ++--
 Utility/FileSystemEncoding.hs | 28 +++++++++++++++++-----------
 Utility/StatFS.hsc            |  2 --
 debian/changelog              |  8 ++++++++
 4 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 18df060046..2e33725514 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -218,12 +218,12 @@ hashDirMixed :: Hasher
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
 	where
 		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]
-		ABCD (a,b,c,d) = md5 $ Str $ show k
+		ABCD (a,b,c,d) = md5 $ Str $ encodeFilePath $ show k
 
 hashDirLower :: Hasher
 hashDirLower k = addTrailingPathSeparator $ take 3 dir  drop 3 dir
 	where
-		dir = take 6 $ md5s $ Str $ show k
+		dir = take 6 $ md5s $ Str $ encodeFilePath $ show k
 
 {- modified version of display_32bits_as_hex from Data.Hash.MD5
  -   Copyright (C) 2001 Ian Lynagh 
diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
index 048323ee3c..fe4202a752 100644
--- a/Utility/FileSystemEncoding.hs
+++ b/Utility/FileSystemEncoding.hs
@@ -7,31 +7,37 @@
 
 module Utility.FileSystemEncoding where
 
-import System.IO
+import qualified GHC.Foreign as GHC
+import qualified GHC.IO.Encoding as Encoding
 import Foreign.C
-import GHC.Foreign as GHC
-import GHC.IO.Encoding
+import System.IO
+import System.IO.Unsafe
 
 {- Sets a Handle to use the filesystem encoding. This causes data
  - written or read from it to be encoded/decoded the same
  - as ghc 7.4 does to filenames etc. This special encoding
  - allows "arbitrary undecodable bytes to be round-tripped through it". -}
 fileEncoding :: Handle -> IO ()
-fileEncoding h = hSetEncoding h =<< getFileSystemEncoding
+fileEncoding h = hSetEncoding h =<< Encoding.getFileSystemEncoding
 
 {- Marshal a Haskell FilePath into a NUL terminated C string using temporary
  - storage. The FilePath is encoded using the filesystem encoding,
  - reversing the decoding that should have been done when the FilePath
  - was obtained. -}
 withFilePath :: FilePath -> (CString -> IO a) -> IO a
-withFilePath fp f = getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f
+withFilePath fp f = Encoding.getFileSystemEncoding
+	>>= \enc -> GHC.withCString enc fp f
 
 {- Encodes a FilePath into a String of encoded bytes, applying the
  - filesystem encoding.
  -
- - This does not do any IO, beyond allocating a C buffer. GHC does not
- - seem to provide a pure way to do this conversion. -}
-encodeFilePath :: FilePath -> IO String
-encodeFilePath fp = do
-	enc <- getFileSystemEncoding
-	GHC.withCString enc fp $ GHC.peekCString enc
+ - This use of unsafePerformIO is belived to be safe; GHC's interface
+ - only allows doing this conversion with CStrings, and the CString buffer
+ - is allocated, used, and deallocated within the call, with no side
+ - effects.
+ -}
+{-# NOINLINE encodeFilePath #-}
+encodeFilePath :: FilePath -> String
+encodeFilePath fp = unsafePerformIO $ do
+	enc <- Encoding.getFileSystemEncoding
+	GHC.withCString enc fp $ GHC.peekCString Encoding.char8
diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc
index 58d0b3e023..ed4c9f1cb6 100644
--- a/Utility/StatFS.hsc
+++ b/Utility/StatFS.hsc
@@ -52,8 +52,6 @@ import Utility.FileSystemEncoding
 import Foreign
 import Foreign.C.Types
 import Foreign.C.String
-import GHC.IO.Encoding (getFileSystemEncoding)
-import GHC.Foreign as GHC
 
 #if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
 # include 
diff --git a/debian/changelog b/debian/changelog
index 97cf38348b..a7eb552727 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,11 @@
+git-annex (3.20120231) UNRELEASED; urgency=low
+
+  * Fix key directory hash calculation code to behave as it did before 
+    version 3.20120227 when a key contains non-ascii characters (only
+    WORM backend is likely to have been affected).
+
+ -- Joey Hess   Fri, 09 Mar 2012 19:19:44 -0400
+
 git-annex (3.20120230) unstable; urgency=low
 
   * "here" can be used to refer to the current repository,

From 433b5fe59e9d3475704ddc2cc17e50526af0f707 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Mar 2012 20:14:34 -0400
Subject: [PATCH 3228/8313] releasing version 3.20120309

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index a7eb552727..478de23b87 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,10 @@
-git-annex (3.20120231) UNRELEASED; urgency=low
+git-annex (3.20120309) unstable; urgency=low
 
   * Fix key directory hash calculation code to behave as it did before 
     version 3.20120227 when a key contains non-ascii characters (only
     WORM backend is likely to have been affected).
 
- -- Joey Hess   Fri, 09 Mar 2012 19:19:44 -0400
+ -- Joey Hess   Fri, 09 Mar 2012 20:05:09 -0400
 
 git-annex (3.20120230) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 46eacbaaf3..b2a07cb110 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120230
+Version: 3.20120309
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 41c0d9e969c24cf726d81b1884f2cdc18e48e2f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Mar 2012 20:15:29 -0400
Subject: [PATCH 3229/8313] add news item for git-annex 3.20120309

---
 doc/news/version_3.20120227.mdwn | 43 --------------------------------
 doc/news/version_3.20120309.mdwn |  5 ++++
 2 files changed, 5 insertions(+), 43 deletions(-)
 delete mode 100644 doc/news/version_3.20120227.mdwn
 create mode 100644 doc/news/version_3.20120309.mdwn

diff --git a/doc/news/version_3.20120227.mdwn b/doc/news/version_3.20120227.mdwn
deleted file mode 100644
index 61a9fc8580..0000000000
--- a/doc/news/version_3.20120227.mdwn
+++ /dev/null
@@ -1,43 +0,0 @@
-git-annex 3.20120227 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Modifications to support ghc 7.4's handling of filenames.
-     This version can only be built with ghc 7.4 or newer. See the ghc7.0
-     branch for older ghcs.
-   * S3: Fix irrefutable pattern failure when accessing encrypted S3
-     credentials.
-   * Use the haskell IfElse library.
-   * Fix teardown of stale cached ssh connections.
-   * Fixed to use the strict state monad, to avoid leaking all kinds of memory
-     due to lazy state update thunks when adding/fixing many files.
-   * Fixed some memory leaks that occurred when committing journal files.
-   * Added a annex.queuesize setting, useful when adding hundreds of thousands
-     of files on a system with plenty of memory.
-   * whereis: Prints the urls of files that the web special remote knows about.
-   * addurl --fast: Verifies that the url can be downloaded (only getting
-     its head), and records the size in the key.
-   * When checking that an url has a key, verify that the Content-Length,
-     if available, matches the size of the key.
-   * addurl: Added a --file option, which can be used to specify what
-     file the url is added to. This can be used to override the default
-     filename that is used when adding an url, which is based on the url.
-     Or, when the file already exists, the url is recorded as another
-     location of the file.
-   * addurl: Normalize badly encoded urls.
-   * addurl: Add --pathdepth option.
-   * rekey: New plumbing level command, can be used to change the keys used
-     for files en masse.
-   * Store web special remote url info in a more efficient location.
-     (Urls stored with this version will not be visible to older versions.)
-   * Deal with NFS problem that caused a failure to remove a directory
-     when removing content from the annex.
-   * Make a single location log commit after a remote has received or
-     dropped files. Uses a new "git-annex-shell commit" command when available.
-   * To avoid commits of data to the git-annex branch after each command
-     is run, set annex.alwayscommit=false. Its data will then be committed
-     less frequently, when a merge or sync is done.
-   * configure: Check if ssh connection caching is supported by the installed
-     version of ssh and default annex.sshcaching accordingly.
-   * move --from, copy --from: Now 10 times faster when scanning to find
-     files in a remote on a local disk; rather than go through the location log
-     to see which files are present on the remote, it simply looks at the
-     disk contents directly."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120309.mdwn b/doc/news/version_3.20120309.mdwn
new file mode 100644
index 0000000000..869b96ccec
--- /dev/null
+++ b/doc/news/version_3.20120309.mdwn
@@ -0,0 +1,5 @@
+git-annex 3.20120309 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix key directory hash calculation code to behave as it did before
+     version 3.20120227 when a key contains non-ascii characters (only
+     WORM backend is likely to have been affected)."""]]
\ No newline at end of file

From 10d9315b59ce1c0d1be91ab7034f87e1f58a0710 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 9 Mar 2012 20:43:50 -0400
Subject: [PATCH 3230/8313] cleanup

---
 Utility/Touch.hsc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index b84054cbcd..b53eab634e 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -18,8 +18,6 @@ import Utility.FileSystemEncoding
 import Foreign
 import Foreign.C
 import Control.Monad (when)
-import GHC.IO.Encoding (getFileSystemEncoding)
-import GHC.Foreign as GHC
 
 newtype TimeSpec = TimeSpec CTime
 

From f9d44cccd993926ce0180437ebcb6d08fdb1efd7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 11:38:38 -0400
Subject: [PATCH 3231/8313] perhaps more clear type

---
 Locations.hs                  | 4 ++--
 Utility/FileSystemEncoding.hs | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index 2e33725514..d263f3d2ac 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -218,12 +218,12 @@ hashDirMixed :: Hasher
 hashDirMixed k = addTrailingPathSeparator $ take 2 dir  drop 2 dir
 	where
 		dir = take 4 $ display_32bits_as_dir =<< [a,b,c,d]
-		ABCD (a,b,c,d) = md5 $ Str $ encodeFilePath $ show k
+		ABCD (a,b,c,d) = md5 $ encodeFilePath $ show k
 
 hashDirLower :: Hasher
 hashDirLower k = addTrailingPathSeparator $ take 3 dir  drop 3 dir
 	where
-		dir = take 6 $ md5s $ Str $ encodeFilePath $ show k
+		dir = take 6 $ md5s $ encodeFilePath $ show k
 
 {- modified version of display_32bits_as_hex from Data.Hash.MD5
  -   Copyright (C) 2001 Ian Lynagh 
diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
index fe4202a752..cf1a6a731e 100644
--- a/Utility/FileSystemEncoding.hs
+++ b/Utility/FileSystemEncoding.hs
@@ -12,6 +12,7 @@ import qualified GHC.IO.Encoding as Encoding
 import Foreign.C
 import System.IO
 import System.IO.Unsafe
+import qualified Data.Hash.MD5 as MD5
 
 {- Sets a Handle to use the filesystem encoding. This causes data
  - written or read from it to be encoded/decoded the same
@@ -28,8 +29,7 @@ withFilePath :: FilePath -> (CString -> IO a) -> IO a
 withFilePath fp f = Encoding.getFileSystemEncoding
 	>>= \enc -> GHC.withCString enc fp f
 
-{- Encodes a FilePath into a String of encoded bytes, applying the
- - filesystem encoding.
+{- Encodes a FilePath into a Str, applying the filesystem encoding.
  -
  - This use of unsafePerformIO is belived to be safe; GHC's interface
  - only allows doing this conversion with CStrings, and the CString buffer
@@ -37,7 +37,7 @@ withFilePath fp f = Encoding.getFileSystemEncoding
  - effects.
  -}
 {-# NOINLINE encodeFilePath #-}
-encodeFilePath :: FilePath -> String
-encodeFilePath fp = unsafePerformIO $ do
+encodeFilePath :: FilePath -> MD5.Str
+encodeFilePath fp = MD5.Str $ unsafePerformIO $ do
 	enc <- Encoding.getFileSystemEncoding
 	GHC.withCString enc fp $ GHC.peekCString Encoding.char8

From 13598d94327c6fd7b11af9cb81c82100cf7cc2f4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 12:47:57 -0400
Subject: [PATCH 3232/8313] add other-modules for hsc files

---
 git-annex.cabal | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/git-annex.cabal b/git-annex.cabal
index b2a07cb110..d8a3b980cb 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -33,9 +33,11 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base, IfElse,
    QuickCheck >= 2.1
+  Other-Modules: Utility.StatFS, Utility.Touch
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
+  Other-Modules: Utility.StatFS
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs

From eaa80be91758e534aec022638829879d82895b76 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 14:00:06 -0400
Subject: [PATCH 3233/8313] move text dependency into same block with the other
 dependencies

---
 git-annex.cabal | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index d8a3b980cb..6efebc66e8 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,9 +30,9 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
-   base >= 4.5, base < 5, monad-control, transformers-base, lifted-base, IfElse,
-   QuickCheck >= 2.1
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hs3, json, HTTP,
+   base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
+   IfElse, text, QuickCheck >= 2.1
   Other-Modules: Utility.StatFS, Utility.Touch
 
 Executable git-annex-shell
@@ -41,7 +41,6 @@ Executable git-annex-shell
 
 Executable git-union-merge
   Main-Is: git-union-merge.hs
-  Build-Depends: text
 
 source-repository head
   type: git

From 468fecc31561064be2fe05928f9c866395c60aa8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 14:00:26 -0400
Subject: [PATCH 3234/8313] Setup.hs: import configure

Rather than running make, which runs configure, let Setup.hs just include
the configure code. The standalone configure is retained for use by the
Makefile.

This may work better with cabal-dev, since it avoids the Makefile running
ghc, and lets cabal handle all the compiler running, with whatever
flags it uses to expose dependencies.
---
 Build/Configure.hs | 107 +++++++++++++++++++++++++++++++++++++++++++++
 Setup.hs           |  15 +++----
 configure.hs       | 106 +++-----------------------------------------
 3 files changed, 119 insertions(+), 109 deletions(-)
 create mode 100644 Build/Configure.hs

diff --git a/Build/Configure.hs b/Build/Configure.hs
new file mode 100644
index 0000000000..341b8840dc
--- /dev/null
+++ b/Build/Configure.hs
@@ -0,0 +1,107 @@
+{- Checks system configuration and generates SysConfig.hs. -}
+
+module Build.Configure where
+
+import System.Directory
+import Data.List
+import System.Cmd.Utils
+import Control.Applicative
+
+import Build.TestConfig
+import Utility.SafeCommand
+
+tests :: [TestCase]
+tests =
+	[ TestCase "version" getVersion
+	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
+	, TestCase "git version" getGitVersion
+	, testCp "cp_a" "-a"
+	, testCp "cp_p" "-p"
+	, testCp "cp_reflink_auto" "--reflink=auto"
+	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] ""
+	, TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
+	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"
+	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
+	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
+	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
+	, TestCase "ssh connection caching" getSshConnectionCaching
+	] ++ shaTestCases [1, 256, 512, 224, 384]
+
+shaTestCases :: [Int] -> [TestCase]
+shaTestCases l = map make l
+	where make n =
+		let
+			cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"]
+			key = "sha" ++ show n
+		in TestCase key $ maybeSelectCmd key cmds " String -> TestCase
+testCp k option = TestCase cmd $ testCmd k cmdline
+	where
+		cmd = "cp " ++ option
+		cmdline = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new"
+
+{- Pulls package version out of the changelog. -}
+getVersion :: Test
+getVersion = do
+	version <- getVersionString
+	return $ Config "packageversion" (StringConfig version)
+	
+getVersionString :: IO String
+getVersionString = do
+	changelog <- readFile "CHANGELOG"
+	let verline = head $ lines changelog
+	return $ middle (words verline !! 1)
+	where
+		middle = drop 1 . init
+
+getGitVersion :: Test
+getGitVersion = do
+	(_, s) <- pipeFrom "git" ["--version"]
+	let version = last $ words $ head $ lines s
+	return $ Config "gitversion" (StringConfig version)
+
+getSshConnectionCaching :: Test
+getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$>
+	boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"]
+
+{- Set up cabal file with version. -}
+cabalSetup :: IO ()
+cabalSetup = do
+	version <- getVersionString
+	cabal <- readFile cabalfile
+	writeFile tmpcabalfile $ unlines $ 
+		map (setfield "Version" version) $
+		lines cabal
+	renameFile tmpcabalfile cabalfile
+	where
+		cabalfile = "git-annex.cabal"
+		tmpcabalfile = cabalfile++".tmp"
+		setfield field value s
+			| fullfield `isPrefixOf` s = fullfield ++ value
+			| otherwise = s
+			where
+				fullfield = field ++ ": "
+
+setup :: IO ()
+setup = do
+	createDirectoryIfMissing True tmpDir
+	writeFile testFile "test file contents"
+
+cleanup :: IO ()
+cleanup = removeDirectoryRecursive tmpDir
+
+run :: [TestCase] -> IO ()
+run ts = do
+	setup
+	config <- runTests ts
+	writeSysConfig config
+	cleanup
+	cabalSetup
diff --git a/Setup.hs b/Setup.hs
index 547d6a156e..14e6a4ea71 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -3,15 +3,10 @@
 import Distribution.Simple
 import System.Cmd
 
-main = defaultMainWithHooks simpleUserHooks {
-	preConf = makeSources,
-	postClean = makeClean
-}
+import qualified Build.Configure as Configure
 
-makeSources _ _ = do
-	system "make sources"
+main = defaultMainWithHooks simpleUserHooks { preConf = configure }
+
+configure _ _ = do
+	Configure.run Configure.tests
 	return (Nothing, [])
-
-makeClean _ _ _ _ = do
-	system "make clean"
-	return ()
diff --git a/configure.hs b/configure.hs
index 9dcc6a5017..3fb0671e78 100644
--- a/configure.hs
+++ b/configure.hs
@@ -1,113 +1,21 @@
-{- Checks system configuration and generates SysConfig.hs. -}
+{- configure program -}
 
-import System.Directory
-import Data.List
 import Data.Maybe
-import System.Cmd.Utils
-import Control.Applicative
 
+import qualified Build.Configure as Configure
 import Build.TestConfig
 import Utility.StatFS
-import Utility.SafeCommand
 
 tests :: [TestCase]
-tests =
-	[ TestCase "version" getVersion
-	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
-	, TestCase "git version" getGitVersion
-	, testCp "cp_a" "-a"
-	, testCp "cp_p" "-p"
-	, testCp "cp_reflink_auto" "--reflink=auto"
-	, TestCase "uuid generator" $ selectCmd "uuid" ["uuid", "uuidgen"] ""
-	, TestCase "xargs -0" $ requireCmd "xargs_0" "xargs -0 /dev/null"
-	, TestCase "curl" $ testCmd "curl" "curl --version >/dev/null"
-	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
-	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
-	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
-	, TestCase "ssh connection caching" getSshConnectionCaching
-	, TestCase "StatFS" testStatFS
-	] ++ shaTestCases [1, 256, 512, 224, 384]
-
-shaTestCases :: [Int] -> [TestCase]
-shaTestCases l = map make l
-	where make n =
-		let
-			cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"]
-			key = "sha" ++ show n
-		in TestCase key $ maybeSelectCmd key cmds " String -> TestCase
-testCp k option = TestCase cmd $ testCmd k run
-	where
-		cmd = "cp " ++ option
-		run = cmd ++ " " ++ testFile ++ " " ++ testFile ++ ".new"
-
-{- Pulls package version out of the changelog. -}
-getVersion :: Test
-getVersion = do
-	version <- getVersionString
-	return $ Config "packageversion" (StringConfig version)
-	
-getVersionString :: IO String
-getVersionString = do
-	changelog <- readFile "CHANGELOG"
-	let verline = head $ lines changelog
-	return $ middle (words verline !! 1)
-	where
-		middle = drop 1 . init
-
-getGitVersion :: Test
-getGitVersion = do
-	(_, s) <- pipeFrom "git" ["--version"]
-	let version = last $ words $ head $ lines s
-	return $ Config "gitversion" (StringConfig version)
-
-getSshConnectionCaching :: Test
-getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$>
-	boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"]
+tests = [ TestCase "StatFS" testStatFS
+	] ++ Configure.tests
 
+{- This test cannot be included in Build.Configure due to needing
+ - Utility/StatFS.hs to be built. -}
 testStatFS :: Test
 testStatFS = do
 	s <- getFileSystemStats "."
 	return $ Config "statfs_sane" $ BoolConfig $ isJust s
 
-{- Set up cabal file with version. -}
-cabalSetup :: IO ()
-cabalSetup = do
-	version <- getVersionString
-	cabal <- readFile cabalfile
-	writeFile tmpcabalfile $ unlines $ 
-		map (setfield "Version" version) $
-		lines cabal
-	renameFile tmpcabalfile cabalfile
-	where
-		cabalfile = "git-annex.cabal"
-		tmpcabalfile = cabalfile++".tmp"
-		setfield field value s
-			| fullfield `isPrefixOf` s = fullfield ++ value
-			| otherwise = s
-			where
-				fullfield = field ++ ": "
-
-setup :: IO ()
-setup = do
-	createDirectoryIfMissing True tmpDir
-	writeFile testFile "test file contents"
-
-cleanup :: IO ()
-cleanup = removeDirectoryRecursive tmpDir
-
 main :: IO ()
-main = do
-	setup
-	config <- runTests tests
-	writeSysConfig config
-	cleanup
-	cabalSetup
+main = Configure.run tests

From 5ab82230f7668897a62f40fa100e51bb53d0c38e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 14:46:21 -0400
Subject: [PATCH 3235/8313] fsck: Fix up any broken links and misplaced content
 caused by the directory hash calculation bug fixed in the last release.

---
 Command/Fsck.hs  | 30 +++++++++++++++++++++++++++++-
 debian/changelog |  7 +++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 94b3601043..d8d0db23be 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -10,6 +10,7 @@ module Command.Fsck where
 import Common.Annex
 import Command
 import qualified Annex
+import qualified Annex.Queue
 import qualified Remote
 import qualified Types.Backend
 import qualified Types.Key
@@ -51,7 +52,8 @@ start from file (key, backend) = do
 perform :: Key -> FilePath -> Backend -> Maybe Int -> CommandPerform
 perform key file backend numcopies = check
 	-- order matters
-	[ verifyLocationLog key file
+	[ fixLink key file
+	, verifyLocationLog key file
 	, checkKeySize key
 	, checkBackend backend key
 	, checkKeyNumCopies key file numcopies
@@ -129,6 +131,32 @@ check = sequence >=> dispatch
 			| all (== True) vs = next $ return True
 			| otherwise = stop
 
+
+{- Checks that the file's symlink points correctly to the content. -}
+fixLink :: Key -> FilePath -> Annex Bool
+fixLink key file = do
+	want <- calcGitLink file key
+	have <- liftIO $ readSymbolicLink file
+	when (want /= have) $ do
+		{- Version 3.20120227 had a bug that could cause content
+		 - to be stored in the wrong hash directory. Clean up
+		 - after the bug by moving the content.
+		 -}
+		whenM (liftIO $ doesFileExist file) $
+			unlessM (inAnnex key) $ do
+				showNote $ "fixing content location"
+				dir <- liftIO $ parentDir <$> absPath file
+				let content = absPathFrom dir have
+				liftIO $ allowWrite (parentDir content)
+				moveAnnex key content
+
+		showNote $ "fixing link"
+		liftIO $ createDirectoryIfMissing True (parentDir file)
+		liftIO $ removeFile file
+		liftIO $ createSymbolicLink want file
+		Annex.Queue.add "add" [Param "--force", Param "--"] [file]
+	return True
+
 {- Checks that the location log reflects the current status of the key,
    in this repository only. -}
 verifyLocationLog :: Key -> String -> Annex Bool
diff --git a/debian/changelog b/debian/changelog
index 478de23b87..1955f8b3b7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120310) UNRELEASED; urgency=low
+
+  * fsck: Fix up any broken links and misplaced content caused by the
+    directory hash calculation bug fixed in the last release.
+
+ -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
+
 git-annex (3.20120309) unstable; urgency=low
 
   * Fix key directory hash calculation code to behave as it did before 

From 997e29f29491868f7d83235cdc2d312263509167 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 15:37:38 -0400
Subject: [PATCH 3236/8313] sync: Sync to lower cost remotes first.

This has two benefits.

1. When a lot of refs are going to be received, get them via lower cost
   connection when possible.
2. Allows ctrl-c of sync after the cheaper remotes have been pulled from
   (or pushed to).
---
 Command/Sync.hs  | 6 +++---
 debian/changelog | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index f7ebba6f54..51b6d6f636 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -59,14 +59,14 @@ syncRemotes rs = do
 	where
 		pickfast = (++) <$> listed <*> (good =<< fastest <$> available)
 		wanted
-			| null rs = good =<< available
+			| null rs = good =<< concat . byspeed <$> available
 			| otherwise = listed
 		listed = catMaybes <$> mapM (Remote.byName . Just) rs
 		available = filter nonspecial <$> Remote.enabledRemoteList
 		good = filterM $ Remote.Git.repoAvail . Types.Remote.repo
 		nonspecial r = Types.Remote.remotetype r == Remote.Git.remote
-		fastest = fromMaybe [] . headMaybe .
-			map snd . sort . M.toList . costmap
+		fastest = fromMaybe [] . headMaybe . byspeed
+		byspeed = map snd . sort . M.toList . costmap
 		costmap = M.fromListWith (++) . map costpair
 		costpair r = (Types.Remote.cost r, [r])
 
diff --git a/debian/changelog b/debian/changelog
index 1955f8b3b7..10dc1c50c6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20120310) UNRELEASED; urgency=low
 
   * fsck: Fix up any broken links and misplaced content caused by the
     directory hash calculation bug fixed in the last release.
+  * sync: Sync to lower cost remotes first.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 

From 94d7b323ee1971430b645d068bb0695fc07c77f1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 10 Mar 2012 23:02:17 -0400
Subject: [PATCH 3237/8313] remove cruft

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6adbdc2df0..08c5b08c00 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,6 @@ Build/SysConfig.hs: configure.hs Build/TestConfig.hs Utility/StatFS.hs
 
 %.hs: %.hsc
 	hsc2hs $<
-	perl -i -pe 's/^{-# INCLUDE.*//' $@
 
 $(bins): $(sources)
 	$(GHCMAKE) $@

From 43dff01dff061dfc55d7c00638e4e948765e8373 Mon Sep 17 00:00:00 2001
From: "http://claimid.com/FooBarWidget" 
Date: Sun, 11 Mar 2012 09:22:58 +0000
Subject: [PATCH 3238/8313]

---
 doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn

diff --git a/doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn b/doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn
new file mode 100644
index 0000000000..d49081bf34
--- /dev/null
+++ b/doc/forum/Please_fix_compatibility_with_ghc_7.0.mdwn
@@ -0,0 +1 @@
+I'm having trouble installing the latest git-annex. It depends on 'base >= 4.5 && base < 5', but ghc 7.0 only ships base 3.0. I've tried upgrading ghc to 7.4 but that breaks a whole bunch of other things; for example, the Crypto module fails to compile, preventing me from installing ghc. Please fix compatibility with ghc 7.0.

From c90e4fdb669f1c7944b395267dba419eb1168a98 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 11 Mar 2012 15:50:11 +0000
Subject: [PATCH 3239/8313] Added a comment

---
 .../comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment

diff --git a/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
new file mode 100644
index 0000000000..fe84228361
--- /dev/null
+++ b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-11T15:50:11Z"
+ content="""
+There is a `ghc7.0` branch in [[git]] that is being maintained to work with that version.
+"""]]

From f2e4187323be9433789d604d6ab568aa1a9424b0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 11:52:02 -0400
Subject: [PATCH 3240/8313] fix link

---
 .../comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
index fe84228361..032969bf2f 100644
--- a/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
+++ b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
@@ -4,5 +4,5 @@
  subject="comment 1"
  date="2012-03-11T15:50:11Z"
  content="""
-There is a `ghc7.0` branch in [[git]] that is being maintained to work with that version.
+There is a `ghc7.0` branch in [[git|install]] that is being maintained to work with that version.
 """]]

From a13949bf3794a33248611aca9ef821a6088b1b3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 11:52:26 -0400
Subject: [PATCH 3241/8313] fix link

---
 .../comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
index 032969bf2f..04b8ce6750 100644
--- a/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
+++ b/doc/forum/Please_fix_compatibility_with_ghc_7.0/comment_1_d1d10217ebd0151e947b3a6cd37399ce._comment
@@ -4,5 +4,5 @@
  subject="comment 1"
  date="2012-03-11T15:50:11Z"
  content="""
-There is a `ghc7.0` branch in [[git|install]] that is being maintained to work with that version.
+There is a `ghc7.0` branch in [[git|download]] that is being maintained to work with that version.
 """]]

From b086e32c63a4932fc5916bedae7abe0690da4eb0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 15:19:07 -0400
Subject: [PATCH 3242/8313] unused: Reduce memory usage significantly.

Much of the memory bloat turned out to be due to getKeysReferenced
containing a mapM, which is strict and buffered the whole list
rather than streaming it.

The other half of the bloat was due to building a temporary Set
in order to call S.difference. While that is more cpu efficient,
I switched to successive S.delete, since with it, I can run a whole
git annex unused in less than 8 mb of memory.

The whole Set of keys with content available is still stored in memory,
so running unused in a repo with a whole lot of file content will still
use more memory. In a repo containing 6000 files, it needed 40 mb.

Note that the status command still uses the bloatful getKeysReferenced.
---
 Command/Unused.hs                          | 52 ++++++++++++++--------
 debian/changelog                           |  3 ++
 doc/todo/git-annex_unused_eats_memory.mdwn | 16 +++++--
 3 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index bbef835f73..ba14bfc4af 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -155,9 +155,9 @@ unusedKeys = do
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
 excludeReferenced l = do
-	c <- inRepo $ Git.Command.pipeRead [Param "show-ref"]
-	removewith (getKeysReferenced : map getKeysReferencedInGit (refs c))
-		(S.fromList l)
+	let s = S.fromList l
+	!s' <- withKeysReferenced s S.delete
+	go s' =<< refs <$> (inRepo $ Git.Command.pipeRead [Param "show-ref"])
 	where
 		-- Skip the git-annex branches, and get all other unique refs.
 		refs = map (Git.Ref .  snd) .
@@ -167,13 +167,12 @@ excludeReferenced l = do
 		uniqref (a, _) (b, _) = a == b
 		ourbranchend = '/' : show Annex.Branch.name
 		ourbranches (_, b) = not $ ourbranchend `isSuffixOf` b
-		removewith [] s = return $ S.toList s
-		removewith (a:as) s
+		go s [] = return $ S.toList s
+		go s (r:rs)
 			| s == S.empty = return [] -- optimisation
 			| otherwise = do
-				referenced <- a
-				let !s' = s `S.difference` S.fromList referenced
-				removewith as s'
+				!s' <- withKeysReferencedInGit r s S.delete
+				go s' rs
 
 {- Finds items in the first, smaller list, that are not
  - present in the second, larger list.
@@ -195,20 +194,37 @@ getKeysReferenced = do
 	keypairs <- mapM Backend.lookupFile files
 	return $ map fst $ catMaybes keypairs
 
-{- List of keys referenced by symlinks in a git ref. -}
-getKeysReferencedInGit :: Git.Ref -> Annex [Key]
-getKeysReferencedInGit ref = do
-	showAction $ "checking " ++ Git.Ref.describe ref
-	findkeys [] =<< inRepo (LsTree.lsTree ref)
+{- Given an initial value, mutates it using an action for each
+ - key referenced by symlinks in the git repo. -}
+withKeysReferenced :: v -> (Key -> v -> v) -> Annex v
+withKeysReferenced initial a = do
+	top <- fromRepo Git.workTree
+	go initial =<< inRepo (LsFiles.inRepo [top])
 	where
-		findkeys c [] = return c
-		findkeys c (l:ls)
+		go v [] = return v
+		go v (f:fs) = do
+			x <- Backend.lookupFile f
+			case x of
+				Nothing -> go v fs
+				Just (k, _) -> do
+					let !v' = a k v
+					go v' fs
+
+withKeysReferencedInGit :: Git.Ref -> v -> (Key -> v -> v) -> Annex v
+withKeysReferencedInGit ref initial a = do
+	showAction $ "checking " ++ Git.Ref.describe ref
+	go initial =<< inRepo (LsTree.lsTree ref)
+	where
+		go v [] = return v
+		go v (l:ls)
 			| isSymLink (LsTree.mode l) = do
 				content <- L.decodeUtf8 <$> catFile ref (LsTree.file l)
 				case fileKey (takeFileName $ L.unpack content) of
-					Nothing -> findkeys c ls
-					Just k -> findkeys (k:c) ls
-			| otherwise = findkeys c ls
+					Nothing -> go v ls
+					Just k -> do
+						let !v' = a k v
+						go v' ls
+			| otherwise = go v ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable. 
diff --git a/debian/changelog b/debian/changelog
index 10dc1c50c6..2cb5a1aea2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (3.20120310) UNRELEASED; urgency=low
   * fsck: Fix up any broken links and misplaced content caused by the
     directory hash calculation bug fixed in the last release.
   * sync: Sync to lower cost remotes first.
+  * unused: Reduce memory usage significantly. Still not constant
+    space, but now only needs to store the set of file contents that
+    are present in the annex in memory.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 
diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 3e9942e98e..91f9a33f7c 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -1,12 +1,20 @@
 `git-annex unused` has to compare large sets of data
 (all keys with content present in the repository,
 with all keys used by files in the repository), and so
-uses more memory than git-annex typically needs; around
-50 mb when run in a repository with 80 thousand files.
+uses more memory than git-annex typically needs.
 
-(Used to be 80 mb, but implementation improved.)
+It used to be a lot worse (hundreds of megabytes).
 
-I would like to reduce this. One idea is to use a bloom filter. 
+Now it only needs enough memory to store a Set of all Keys that currently
+have content in the annex. On a lightly populated repository, it runs in
+quite low memory use (like 8 mb) even if the git repo has 100 thousand
+files. On a repository with lots of file contents, it will use more.
+
+Still, I would like to reduce this to a purely constant memory use,
+as running in constant memory no matter the repo size is a git-annex design
+goal.
+
+One idea is to use a bloom filter. 
 For example, construct a bloom filter of all keys used by files in
 the repository. Then for each key with content present, check if it's
 in the bloom filter. Since there can be false positives, this might

From ff3644ad38d210c5ce0ebfb5a2cf5e84bb3b47da Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 17:15:58 -0400
Subject: [PATCH 3243/8313] status: Fixed to run in nearly constant space.

Before, it leaked space due to caching lists of keys. Now all necessary
data about keys is calculated as they stream in.

The "nearly constant" is due to getKeysPresent, which builds up a lot
of [] thunks as it traverses .git/annex/objects/. Will deal with it later.
---
 Annex/Content.hs  |  2 +-
 Command/Status.hs | 94 +++++++++++++++++++++++++++++------------------
 Command/Unused.hs | 19 +++-------
 debian/changelog  |  1 +
 4 files changed, 67 insertions(+), 49 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index fdd03f320e..bf5a6c3a7e 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -300,7 +300,7 @@ getKeysPresent' dir = do
 			-- 2 levels of hashing
 			levela <- dirContents dir
 			levelb <- mapM dirContents levela
-			contents <- mapM dirContents (concat levelb)
+			contents <- unsafeInterleaveIO $ mapM dirContents (concat levelb)
 			let files = concat contents
 			return $ mapMaybe (fileKey . takeFileName) files
 
diff --git a/Command/Status.hs b/Command/Status.hs
index dfe847bb8e..0b1741dc0a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -5,12 +5,12 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Status where
 
 import Control.Monad.State.Strict
 import qualified Data.Map as M
-import qualified Data.Set as S
-import Data.Set (Set)
 import Text.JSON
 
 import Common.Annex
@@ -32,10 +32,18 @@ import Remote
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
 
--- cached info that multiple Stats may need
+-- data about a set of keys
+data KeyData = KeyData
+	{ countKeys :: Integer
+	, sizeKeys :: Integer
+	, unknownSizeKeys :: Integer
+	, backendsKeys :: M.Map String Integer
+	}
+
+-- cached info that multiple Stats use
 data StatInfo = StatInfo
-	{ keysPresentCache :: Maybe (Set Key)
-	, keysReferencedCache :: Maybe (Set Key)
+	{ presentData :: Maybe KeyData
+	, referencedData :: Maybe KeyData
 	}
 
 -- a state monad for running Stats in
@@ -122,19 +130,19 @@ remote_list level desc = stat n $ nojson $ lift $ do
 
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $ json id $
-	keySizeSum <$> cachedKeysPresent
+	showSizeKeys <$> cachedPresentData
 
 local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $ json show $
-	S.size <$> cachedKeysPresent
+	countKeys <$> cachedPresentData
 
 known_annex_size :: Stat
 known_annex_size = stat "known annex size" $ json id $
-	keySizeSum <$> cachedKeysReferenced
+	showSizeKeys <$> cachedReferencedData
 
 known_annex_keys :: Stat
 known_annex_keys = stat "known annex keys" $ json show $
-	S.size <$> cachedKeysReferenced
+	countKeys <$> cachedReferencedData
 
 tmp_size :: Stat
 tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
@@ -144,46 +152,62 @@ bad_data_size = staleSize "bad keys size" gitAnnexBadDir
 
 backend_usage :: Stat
 backend_usage = stat "backend usage" $ nojson $
-	calc <$> cachedKeysReferenced <*> cachedKeysPresent
+	calc
+		<$> (backendsKeys <$> cachedReferencedData)
+		<*> (backendsKeys <$> cachedPresentData)
 	where
-		calc a b = pp "" $ reverse . sort $ map swap $ splits $ S.toList $ S.union a b
-		splits :: [Key] -> [(String, Integer)]
-		splits ks = M.toList $ M.fromListWith (+) $ map tcount ks
-		tcount k = (keyBackendName k, 1)
-		swap (a, b) = (b, a)
+		calc a b = pp "" $ reverse . sort $ map swap $ M.toList $ M.unionWith (+) a b
 		pp c [] = c
 		pp c ((n, b):xs) = "\n\t" ++ b ++ ": " ++ show n ++ pp c xs
+		swap (a, b) = (b, a)
 
-cachedKeysPresent :: StatState (Set Key)
-cachedKeysPresent = do
+cachedPresentData :: StatState KeyData
+cachedPresentData = do
 	s <- get
-	case keysPresentCache s of
+	case presentData s of
 		Just v -> return v
 		Nothing -> do
-			keys <- S.fromList <$> lift getKeysPresent
-			put s { keysPresentCache = Just keys }
-			return keys
+			v <- foldKeys <$> lift getKeysPresent
+			put s { presentData = Just v }
+			return v
 
-cachedKeysReferenced :: StatState (Set Key)
-cachedKeysReferenced = do
+cachedReferencedData :: StatState KeyData
+cachedReferencedData = do
 	s <- get
-	case keysReferencedCache s of
+	case referencedData s of
 		Just v -> return v
 		Nothing -> do
-			keys <- S.fromList <$> lift Command.Unused.getKeysReferenced
-			put s { keysReferencedCache = Just keys }
-			return keys
+			!v <- lift $ Command.Unused.withKeysReferenced
+				emptyKeyData addKey
+			put s { referencedData = Just v }
+			return v
 
-keySizeSum :: Set Key -> String
-keySizeSum s = total ++ missingnote
+emptyKeyData :: KeyData
+emptyKeyData = KeyData 0 0 0 M.empty
+
+foldKeys :: [Key] -> KeyData
+foldKeys = foldl' (flip addKey) emptyKeyData
+
+addKey :: Key -> KeyData -> KeyData
+addKey key (KeyData count size unknownsize backends) =
+	KeyData count' size' unknownsize' backends'
 	where
-		knownsizes = mapMaybe keySize $ S.toList s
-		total = roughSize storageUnits False $ sum knownsizes
-		missing = S.size s - genericLength knownsizes
+		{- All calculations strict to avoid thunks when repeatedly
+		 - applied to many keys. -}
+		!count' = count + 1
+		!backends' = M.insertWith' (+) (keyBackendName key) 1 backends
+		!size' = maybe size (+ size) ks
+		!unknownsize' = maybe (unknownsize + 1) (const unknownsize) ks
+		ks = keySize key
+
+showSizeKeys :: KeyData -> String
+showSizeKeys d = total ++ missingnote
+	where
+		total = roughSize storageUnits False $ sizeKeys d
 		missingnote
-			| missing == 0 = ""
+			| unknownSizeKeys d == 0 = ""
 			| otherwise = aside $
-				"+ " ++ show missing ++
+				"+ " ++ show (unknownSizeKeys d) ++
 				" keys of unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
@@ -192,7 +216,7 @@ staleSize label dirspec = do
 	if null keys
 		then nostat
 		else stat label $ json (++ aside "clean up with git-annex unused") $
-			return $ keySizeSum $ S.fromList keys
+			return $ showSizeKeys $ foldKeys keys
 
 aside :: String -> String
 aside s = " (" ++ s ++ ")"
diff --git a/Command/Unused.hs b/Command/Unused.hs
index ba14bfc4af..69b58c5e70 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010-2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -171,7 +171,7 @@ excludeReferenced l = do
 		go s (r:rs)
 			| s == S.empty = return [] -- optimisation
 			| otherwise = do
-				!s' <- withKeysReferencedInGit r s S.delete
+				s' <- withKeysReferencedInGit r s S.delete
 				go s' rs
 
 {- Finds items in the first, smaller list, that are not
@@ -186,21 +186,14 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 	where
 		remove a b = foldl (flip S.delete) b a
 
-{- List of keys referenced by symlinks in the git repo. -}
-getKeysReferenced :: Annex [Key]
-getKeysReferenced = do
-	top <- fromRepo Git.workTree
-	files <- inRepo $ LsFiles.inRepo [top]
-	keypairs <- mapM Backend.lookupFile files
-	return $ map fst $ catMaybes keypairs
-
 {- Given an initial value, mutates it using an action for each
  - key referenced by symlinks in the git repo. -}
 withKeysReferenced :: v -> (Key -> v -> v) -> Annex v
-withKeysReferenced initial a = do
-	top <- fromRepo Git.workTree
-	go initial =<< inRepo (LsFiles.inRepo [top])
+withKeysReferenced initial a = go initial =<< files
 	where
+		files = do
+			top <- fromRepo Git.workTree
+			inRepo $ LsFiles.inRepo [top]
 		go v [] = return v
 		go v (f:fs) = do
 			x <- Backend.lookupFile f
diff --git a/debian/changelog b/debian/changelog
index 2cb5a1aea2..6da54056c7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20120310) UNRELEASED; urgency=low
   * unused: Reduce memory usage significantly. Still not constant
     space, but now only needs to store the set of file contents that
     are present in the annex in memory.
+  * status: Fixed to run in nearly constant space.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 

From b3256946457ec8a2da056573bf49593b225adbd8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 18:04:58 -0400
Subject: [PATCH 3244/8313] getKeysPresent is now fully lazy

.. Allowing it to be used by things in constant space!

Random statistics: git annex status has gone from taking 239 mb
of memory and 26 seconds in a repo, to 8 mb and 13 seconds.

The trick here is the unsafeInterleaveIO, and the form of the function's
recursion, which I cribbed heavily from System.IO.HVFS.Utils.recurseDirStat.
The difference is, this one goes to a limited depth and avoids statting
everything.
---
 Annex/Content.hs | 28 +++++++++++++++-------------
 debian/changelog |  2 +-
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index bf5a6c3a7e..ccaff5c564 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -27,6 +27,7 @@ module Annex.Content (
 
 import Control.Exception (bracket_)
 import System.Posix.Types
+import System.IO.Unsafe (unsafeInterleaveIO)
 
 import Common.Annex
 import Logs.Location
@@ -290,19 +291,20 @@ moveBad key = do
 
 {- List of keys whose content exists in .git/annex/objects/ -}
 getKeysPresent :: Annex [Key]
-getKeysPresent = getKeysPresent' =<< fromRepo gitAnnexObjectDir
-getKeysPresent' :: FilePath -> Annex [Key]
-getKeysPresent' dir = do
-	exists <- liftIO $ doesDirectoryExist dir
-	if not exists
-		then return []
-		else liftIO $ do
-			-- 2 levels of hashing
-			levela <- dirContents dir
-			levelb <- mapM dirContents levela
-			contents <- unsafeInterleaveIO $ mapM dirContents (concat levelb)
-			let files = concat contents
-			return $ mapMaybe (fileKey . takeFileName) files
+getKeysPresent = liftIO . traverse (2 :: Int) =<< fromRepo gitAnnexObjectDir
+	where
+		traverse depth dir = do
+			contents <- catchDefaultIO (dirContents dir) []
+			if depth == 0
+				then continue (mapMaybe (fileKey . takeFileName) contents) []
+				else do
+					let deeper = traverse (depth - 1)
+					continue [] (map deeper contents)
+		continue keys [] = return keys
+		continue keys (a:as) = do
+			{- Force lazy traversal with unsafeInterleaveIO. -}
+			morekeys <- unsafeInterleaveIO a
+			continue (morekeys++keys) as
 
 {- Things to do to record changes to content when shutting down.
  -
diff --git a/debian/changelog b/debian/changelog
index 6da54056c7..120513806e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,7 +6,7 @@ git-annex (3.20120310) UNRELEASED; urgency=low
   * unused: Reduce memory usage significantly. Still not constant
     space, but now only needs to store the set of file contents that
     are present in the annex in memory.
-  * status: Fixed to run in nearly constant space.
+  * status: Fixed to run in constant space.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 

From 6fd0c0bfecb8e3585aecfa946515d89c8d443441 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 18:12:36 -0400
Subject: [PATCH 3245/8313] move

---
 Utility/Directory.hs | 11 +++++++++++
 Utility/Path.hs      |  9 ---------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index e7b7c442b2..40e65d6349 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -13,11 +13,22 @@ import System.Directory
 import Control.Exception (throw)
 import Control.Monad
 import Control.Monad.IfElse
+import System.FilePath
+import Control.Applicative
 
 import Utility.SafeCommand
 import Utility.TempFile
 import Utility.Exception
 
+{- Lists the contents of a directory.
+ - Unlike getDirectoryContents, paths are not relative to the directory. -}
+dirContents :: FilePath -> IO [FilePath]
+dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
+	where
+		notcruft "." = False
+		notcruft ".." = False
+		notcruft _ = True
+
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}
 moveFile :: FilePath -> FilePath -> IO ()
diff --git a/Utility/Path.hs b/Utility/Path.hs
index eb530442b1..76fbc6c4a4 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -128,15 +128,6 @@ preserveOrder (l:ls) new = found ++ preserveOrder ls rest
 runPreserveOrder :: ([FilePath] -> IO [FilePath]) -> [FilePath] -> IO [FilePath]
 runPreserveOrder a files = preserveOrder files <$> a files
 
-{- Lists the contents of a directory.
- - Unlike getDirectoryContents, paths are not relative to the directory. -}
-dirContents :: FilePath -> IO [FilePath]
-dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
-	where
-		notcruft "." = False
-		notcruft ".." = False
-		notcruft _ = True
-
 {- Current user's home directory. -}
 myHomeDir :: IO FilePath
 myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)

From 5df18b311a691c7b7c213468034f13a52eaf0874 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 12:38:59 -0400
Subject: [PATCH 3246/8313] avoid needing to keep list of present keys

Stale and bad files are rare, so it's more efficient to use inAnnex to see
if they can be deleted, rather than keeping the list of all present keys
around for them.
---
 Command/Unused.hs | 43 +++++++++++++++----------------------------
 1 file changed, 15 insertions(+), 28 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 69b58c5e70..1d14b837c0 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -55,12 +55,20 @@ start = do
 
 checkUnused :: CommandPerform
 checkUnused = do
-	(unused, stalebad, staletmp) <- unusedKeys
+	unused <- findunused =<< Annex.getState Annex.fast
+	stalebad <- staleKeysPrune gitAnnexBadDir
+	staletmp <- staleKeysPrune gitAnnexTmpDir
 	_ <- list "" unusedMsg unused 0 >>=
 		list "bad" staleBadMsg stalebad >>=
 			list "tmp" staleTmpMsg staletmp
 	next $ return True
 	where
+		findunused True = do
+			showNote "fast mode enabled; only finding stale files"
+			return []
+		findunused False = do
+			showAction "checking for unused data"
+			excludeReferenced =<< getKeysPresent
 		list file msg l c = do
 			let unusedlist = number c l
 			unless (null l) $ showLongNote $ msg unusedlist
@@ -131,26 +139,6 @@ dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
 dropMsg' :: String -> String
 dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n"
 
-{- Finds keys whose content is present, but that do not seem to be used
- - by any files in the git repo, or that are only present as bad or tmp
- - files. -}
-unusedKeys :: Annex ([Key], [Key], [Key])
-unusedKeys = do
-	fast <- Annex.getState Annex.fast
-	if fast
-		then do
-			showNote "fast mode enabled; only finding stale files"
-			tmp <- staleKeys gitAnnexTmpDir
-			bad <- staleKeys gitAnnexBadDir
-			return ([], bad, tmp)
-		else do
-			showAction "checking for unused data"
-			present <- getKeysPresent
-			unused <- excludeReferenced present
-			staletmp <- staleKeysPrune gitAnnexTmpDir present
-			stalebad <- staleKeysPrune gitAnnexBadDir present
-			return (unused, stalebad, staletmp)
-
 {- Finds keys in the list that are not referenced in the git repository. -}
 excludeReferenced :: [Key] -> Annex [Key]
 excludeReferenced [] = return [] -- optimisation
@@ -220,17 +208,16 @@ withKeysReferencedInGit ref initial a = do
 			| otherwise = go v ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
- - of those that might still have value, or might be stale and removable. 
+ - of those that might still have value, or might be stale and removable.
  - 
- - When a list of presently available keys is provided, stale keys
- - that no longer have value are deleted.
+ - Also, stale keys that can be proven to have no value are deleted.
  -}
-staleKeysPrune :: (Git.Repo -> FilePath) -> [Key] -> Annex [Key]
-staleKeysPrune dirspec present = do
+staleKeysPrune :: (Git.Repo -> FilePath) -> Annex [Key]
+staleKeysPrune dirspec = do
 	contents <- staleKeys dirspec
 	
-	let stale = contents `exclude` present
-	let dups = contents `exclude` stale
+	dups <- filterM inAnnex contents
+	let stale = contents `exclude` dups
 
 	dir <- fromRepo dirspec
 	liftIO $ forM_ dups $ \t -> removeFile $ dir  keyFile t

From 83bbb3bc9330de7c4d2b8c07ef48da3bea82433d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 21:08:48 -0400
Subject: [PATCH 3247/8313] prettify

---
 Command/Unused.hs | 56 +++++++++++++++++++++++------------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1d14b837c0..71cbfd4708 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -54,14 +54,11 @@ start = do
 	next action
 
 checkUnused :: CommandPerform
-checkUnused = do
-	unused <- findunused =<< Annex.getState Annex.fast
-	stalebad <- staleKeysPrune gitAnnexBadDir
-	staletmp <- staleKeysPrune gitAnnexTmpDir
-	_ <- list "" unusedMsg unused 0 >>=
-		list "bad" staleBadMsg stalebad >>=
-			list "tmp" staleTmpMsg staletmp
-	next $ return True
+checkUnused = chain 0
+	[ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast
+	, check "bad" staleBadMsg $ staleKeysPrune gitAnnexBadDir
+	, check "tmp" staleTmpMsg $ staleKeysPrune gitAnnexTmpDir
+	]
 	where
 		findunused True = do
 			showNote "fast mode enabled; only finding stale files"
@@ -69,25 +66,32 @@ checkUnused = do
 		findunused False = do
 			showAction "checking for unused data"
 			excludeReferenced =<< getKeysPresent
-		list file msg l c = do
-			let unusedlist = number c l
-			unless (null l) $ showLongNote $ msg unusedlist
-			writeUnusedFile file unusedlist
-			return $ c + length l
+		chain _ [] = next $ return True
+		chain v (a:as) = do
+			v' <- a v
+			chain v' as
 
 checkRemoteUnused :: String -> CommandPerform
-checkRemoteUnused name = do
-	checkRemoteUnused' =<< fromJust <$> Remote.byName (Just name)
-	next $ return True
+checkRemoteUnused name = go =<< fromJust <$> Remote.byName (Just name)
+	where
+		go r = do
+			showAction "checking for unused data"
+			_ <- check "" (remoteUnusedMsg r) (remoteunused r) 0
+			next $ return True
+		remoteunused r =
+			excludeReferenced =<< loggedKeysFor (Remote.uuid r)
 
-checkRemoteUnused' :: Remote -> Annex ()
-checkRemoteUnused' r = do
-	showAction "checking for unused data"
-	remotehas <- loggedKeysFor (Remote.uuid r)
-	remoteunused <- excludeReferenced remotehas
-	let list = number 0 remoteunused
-	writeUnusedFile "" list
-	unless (null remoteunused) $ showLongNote $ remoteUnusedMsg r list
+check :: FilePath -> ([(Int, Key)] -> String) -> Annex [Key] -> Int -> Annex Int
+check file msg a c = do
+	l <- a
+	let unusedlist = number c l
+	unless (null l) $ showLongNote $ msg unusedlist
+	writeUnusedFile file unusedlist
+	return $ c + length l
+
+number :: Int -> [a] -> [(Int, a)]
+number _ [] = []
+number n (x:xs) = (n+1, x) : number (n+1) xs
 
 writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
 writeUnusedFile prefix l = do
@@ -101,10 +105,6 @@ table l = "  NUMBER  KEY" : map cols l
 		cols (n,k) = "  " ++ pad 6 (show n) ++ "  " ++ show k
 		pad n s = s ++ replicate (n - length s) ' '
 
-number :: Int -> [a] -> [(Int, a)]
-number _ [] = []
-number n (x:xs) = (n+1, x) : number (n+1) xs
-
 staleTmpMsg :: [(Int, Key)] -> String
 staleTmpMsg t = unlines $ 
 	["Some partially transferred data exists in temporary files:"]

From 89ee70c43afd12f15c2ee796036f96c620d8becb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 00:41:48 -0400
Subject: [PATCH 3248/8313] status: More accurate display of sizes of tmp and
 bad keys.

Can't trust the key size to be accurate for tmp and bad keys, so check
actual file size. In the wild I saw the old code be wrong by a factor
of about 100!

If all tmp/bad keys are empty, they're not shown in status at all.
Showing 0 bytes and suggesting to clean it up seemed weird..
---
 Command/Status.hs | 19 +++++++++++++------
 debian/changelog  |  1 +
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 0b1741dc0a..96345e92b5 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -211,12 +211,19 @@ showSizeKeys d = total ++ missingnote
 				" keys of unknown size"
 
 staleSize :: String -> (Git.Repo -> FilePath) -> Stat
-staleSize label dirspec = do
-	keys <- lift (Command.Unused.staleKeys dirspec)
-	if null keys
-		then nostat
-		else stat label $ json (++ aside "clean up with git-annex unused") $
-			return $ showSizeKeys $ foldKeys keys
+staleSize label dirspec = go =<< lift (Command.Unused.staleKeys dirspec)
+	where
+		go [] = nostat
+		go keys = onsize =<< sum <$> keysizes keys
+		onsize 0 = nostat
+		onsize size = stat label $
+			json (++ aside "clean up with git-annex unused") $
+				return $ roughSize storageUnits False size
+		keysizes keys = map (fromIntegral . fileSize) <$> stats keys
+		stats keys = do
+			dir <- lift $ fromRepo dirspec
+			liftIO $ forM keys $ \k ->
+				getFileStatus (dir  keyFile k)
 
 aside :: String -> String
 aside s = " (" ++ s ++ ")"
diff --git a/debian/changelog b/debian/changelog
index 120513806e..8fc2cc3304 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,7 @@ git-annex (3.20120310) UNRELEASED; urgency=low
     space, but now only needs to store the set of file contents that
     are present in the annex in memory.
   * status: Fixed to run in constant space.
+  * status: More accurate display of sizes of tmp and bad keys.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 

From 98ff74a85158bbc8bf1cea9d642cee25e6e02eb3 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawne9wwsAaMzo0kGyidj6PW_3_IA8eeDv7Y"
 
Date: Mon, 12 Mar 2012 05:16:28 +0000
Subject: [PATCH 3249/8313]

---
 doc/forum/windows_port__63__.mdwn | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 doc/forum/windows_port__63__.mdwn

diff --git a/doc/forum/windows_port__63__.mdwn b/doc/forum/windows_port__63__.mdwn
new file mode 100644
index 0000000000..092b8f8e1c
--- /dev/null
+++ b/doc/forum/windows_port__63__.mdwn
@@ -0,0 +1,2 @@
+Any progress on Windows port? That would be very nice to have!
+Depending on the scale of it, I might be able to help.

From 160715166b95f17b76b4b0bd47bbec4fdc6c1aac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 11 Mar 2012 22:00:49 -0400
Subject: [PATCH 3250/8313] try at using bloom filters

leaks memory
---
 Command/Unused.hs | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 71cbfd4708..2fb2781262 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -12,6 +12,9 @@ module Command.Unused where
 import qualified Data.Set as S
 import qualified Data.Text.Lazy as L
 import qualified Data.Text.Lazy.Encoding as L
+import Data.BloomFilter
+import Data.BloomFilter.Easy
+import Data.BloomFilter.Hash
 
 import Common.Annex
 import Command
@@ -53,6 +56,18 @@ start = do
 	showStart "unused" name
 	next action
 
+genBloomFilter :: [Key] -> Annex (Bloom String)
+genBloomFilter ks = do
+	-- A bloom filter capable of holding one million keys with a
+	-- false positive rate of 0.1% uses 16 mb of memory.
+	-- TODO: make this configurable, for the really large repos,
+	-- or really low false positive rates.
+	let (numbits, numhashes) = suggestSizing 1000000 0.0001
+	return $ fromListB (cheapHashes numhashes) numbits $ map show ks
+
+bloomFilter :: Bloom String -> [Key] -> [Key]
+bloomFilter b l = filter (\k -> show k `notElemB` b) l
+
 checkUnused :: CommandPerform
 checkUnused = chain 0
 	[ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast
@@ -65,7 +80,9 @@ checkUnused = chain 0
 			return []
 		findunused False = do
 			showAction "checking for unused data"
-			excludeReferenced =<< getKeysPresent
+			b <- genBloomFilter =<< withKeysReferenced [] (:)
+			bloomFilter b <$> getKeysPresent
+			-- TODO: check branches
 		chain _ [] = next $ return True
 		chain v (a:as) = do
 			v' <- a v

From a886fe16015ef3859e311435456691bb28ebdad5 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 12 Mar 2012 06:43:03 +0000
Subject: [PATCH 3251/8313] Added a comment

---
 .../comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment

diff --git a/doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment b/doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
new file mode 100644
index 0000000000..95323ff997
--- /dev/null
+++ b/doc/forum/windows_port__63__/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-12T06:43:02Z"
+ content="""
+[[todo/windows_support]] has everything I know about making a windows port. This badly needs someone who understand Windows to dive into it. The question of how to create a symbolic link (or the relevant Windows equivilant) from haskell on Windows
+is a good starting point..
+"""]]

From b6caf8997d59817eebdc5750ddbe4a31b3f3eb5f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl-J5N9y-JBa_GcOQ4VQXIF8MjAtxgN67w"
 
Date: Mon, 12 Mar 2012 11:22:21 +0000
Subject: [PATCH 3252/8313] Formatting

---
 .../using_box.com_as_a_special_remote.mdwn    | 29 ++++++++++++-------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn
index f9e2fbba5d..917c7a93bd 100644
--- a/doc/tips/using_box.com_as_a_special_remote.mdwn
+++ b/doc/tips/using_box.com_as_a_special_remote.mdwn
@@ -13,21 +13,30 @@ With a little setup, git-annex can use Box as a
 * Allow users to mount davfs filesystems, by ensuring that
   `/sbin/mount.davfs` is setuid root. On Debian, just `sudo dpkg-reconfigure davfs2`
 * Add yourself to the davfs2 group.
-	sudo adduser $(whoami) davfs2
+
+        sudo adduser $(whoami) davfs2
+
 * Edit `/etc/fstab`, and add a line to mount Box using davfs.
-	sudo mkdir -p /media/box.com
-	echo "https://www.box.com/dav/	/media/box.com	davfs	noauto,user	0 0" | sudo tee -a /etc/fstab
+
+        sudo mkdir -p /media/box.com
+        echo "https://www.box.com/dav/	/media/box.com	davfs	noauto,user	0 0" | sudo tee -a /etc/fstab
+
 * Create `~/.davfs2/davfs2.conf` with some important settings:
-	mkdir ~/.davfs2/
-	echo use_locks 0 >> ~/.davfs2/davfs2.conf
-	echo cache_size 1 >> ~/.davfs2/davfs2.conf
-	echo delay_upload 0 >> ~/.davfs2/davfs2.conf
+
+        mkdir ~/.davfs2/
+        echo use_locks 0 >> ~/.davfs2/davfs2.conf
+        echo cache_size 1 >> ~/.davfs2/davfs2.conf
+        echo delay_upload 0 >> ~/.davfs2/davfs2.conf
+
 * Create `~/.davfs2/secrets`. This file contains your Box.com login and password.
   Your login is probably the email address you signed up with.
-	echo "/media/box.com joey@kitenet.net mypassword" > ~/.davfs2/secrets
-	chmod 600 ~/.davfs2/secrets
+
+        echo "/media/box.com joey@kitenet.net mypassword" > ~/.davfs2/secrets
+        chmod 600 ~/.davfs2/secrets
+
 * Now you should be able to mount Box, as a non-root user:
-	mount /media/box.com
+
+        mount /media/box.com
 
 ## git-annex setup
 

From 32f9742a883c2a9f6ed29f1261c64c7590ecc8ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 14:09:43 -0400
Subject: [PATCH 3253/8313] fixed bloom filter creation space leak

it works!
---
 Command/Unused.hs | 49 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index 2fb2781262..cc7ff7c71a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -15,6 +15,7 @@ import qualified Data.Text.Lazy.Encoding as L
 import Data.BloomFilter
 import Data.BloomFilter.Easy
 import Data.BloomFilter.Hash
+import Control.Monad.ST
 
 import Common.Annex
 import Command
@@ -56,18 +57,6 @@ start = do
 	showStart "unused" name
 	next action
 
-genBloomFilter :: [Key] -> Annex (Bloom String)
-genBloomFilter ks = do
-	-- A bloom filter capable of holding one million keys with a
-	-- false positive rate of 0.1% uses 16 mb of memory.
-	-- TODO: make this configurable, for the really large repos,
-	-- or really low false positive rates.
-	let (numbits, numhashes) = suggestSizing 1000000 0.0001
-	return $ fromListB (cheapHashes numhashes) numbits $ map show ks
-
-bloomFilter :: Bloom String -> [Key] -> [Key]
-bloomFilter b l = filter (\k -> show k `notElemB` b) l
-
 checkUnused :: CommandPerform
 checkUnused = chain 0
 	[ check "" unusedMsg $ findunused =<< Annex.getState Annex.fast
@@ -80,7 +69,7 @@ checkUnused = chain 0
 			return []
 		findunused False = do
 			showAction "checking for unused data"
-			b <- genBloomFilter =<< withKeysReferenced [] (:)
+			b <- genBloomFilter show withKeysReferenced'
 			bloomFilter b <$> getKeysPresent
 			-- TODO: check branches
 		chain _ [] = next $ return True
@@ -191,10 +180,40 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 	where
 		remove a b = foldl (flip S.delete) b a
 
+{- Creates a bloom filter, and runs an action, such as withKeysReferenced',
+ - to populate it.
+ -
+ - The action is passed a callback that it can use to feed values into the
+ - bloom filter. 
+ -
+ - Once the action completes, the mutable filter is frozen
+ - for later use.
+ -}
+genBloomFilter :: Hashable t => (v -> t) -> (() -> (v -> () -> Annex ()) -> Annex b) -> Annex (Bloom t)
+genBloomFilter convert populate = do
+	-- A bloom filter capable of holding one million keys with a
+	-- false positive rate of 0.1% uses 16 mb of memory.
+	-- TODO: make this configurable, for the really large repos,
+	-- or really low false positive rates.
+	let (numbits, numhashes) = suggestSizing 1000000 0.0001
+
+	bloom <- lift $ newMB (cheapHashes numhashes) numbits
+	_ <- populate () $ \v _ -> lift $ insertMB bloom (convert v)
+	lift $ unsafeFreezeMB bloom
+	where
+		lift = liftIO . stToIO
+
+bloomFilter :: Bloom String -> [Key] -> [Key]
+bloomFilter b l = filter (\k -> show k `notElemB` b) l
+
 {- Given an initial value, mutates it using an action for each
  - key referenced by symlinks in the git repo. -}
 withKeysReferenced :: v -> (Key -> v -> v) -> Annex v
-withKeysReferenced initial a = go initial =<< files
+withKeysReferenced initial a = withKeysReferenced' initial reta
+	where
+		reta k v = return $ a k v
+withKeysReferenced' :: v -> (Key -> v -> Annex v) -> Annex v
+withKeysReferenced' initial a = go initial =<< files
 	where
 		files = do
 			top <- fromRepo Git.workTree
@@ -205,7 +224,7 @@ withKeysReferenced initial a = go initial =<< files
 			case x of
 				Nothing -> go v fs
 				Just (k, _) -> do
-					let !v' = a k v
+					!v' <- a k v
 					go v' fs
 
 withKeysReferencedInGit :: Git.Ref -> v -> (Key -> v -> v) -> Annex v

From faf3a94fa7dfaaf7f95477895c645ff793dcf2f4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 15:21:20 -0400
Subject: [PATCH 3254/8313] added second stage bloom filter

---
 Command/Unused.hs | 114 ++++++++++++++++++++++++++++------------------
 1 file changed, 69 insertions(+), 45 deletions(-)

diff --git a/Command/Unused.hs b/Command/Unused.hs
index cc7ff7c71a..028e20445d 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -69,9 +69,7 @@ checkUnused = chain 0
 			return []
 		findunused False = do
 			showAction "checking for unused data"
-			b <- genBloomFilter show withKeysReferenced'
-			bloomFilter b <$> getKeysPresent
-			-- TODO: check branches
+			excludeReferenced =<< getKeysPresent
 		chain _ [] = next $ return True
 		chain v (a:as) = do
 			v' <- a v
@@ -145,28 +143,32 @@ dropMsg (Just r) = dropMsg' $ " --from " ++ Remote.name r
 dropMsg' :: String -> String
 dropMsg' s = "\nTo remove unwanted data: git-annex dropunused" ++ s ++ " NUMBER\n"
 
-{- Finds keys in the list that are not referenced in the git repository. -}
+{- Finds keys in the list that are not referenced in the git repository.
+ -
+ - Strategy:
+ -
+ - * Build a bloom filter of all keys referenced by symlinks. This 
+ -   is the fastest one to build and will filter out most keys.
+ - * If keys remain, build a second bloom filter of keys referenced by
+ -   all branches.
+ - * The list is streamed through these bloom filters lazily, so both will
+ -   exist at the same time. This means that twice the memory is used,
+ -   but they're relatively small, so the added complexity of using a
+ -   mutable bloom filter does not seem worthwhile.
+ - * Generating the second bloom filter can take quite a while, since
+ -   it needs enumerating all keys in all git branches. But, the common
+ -   case, if the second filter is needed, is for some keys to be globally
+ -   unused, and in that case, no short-circuit is possible.
+ -   Short-circuiting if the first filter filters all the keys handles the
+ -   other common case.
+ -}
 excludeReferenced :: [Key] -> Annex [Key]
-excludeReferenced [] = return [] -- optimisation
-excludeReferenced l = do
-	let s = S.fromList l
-	!s' <- withKeysReferenced s S.delete
-	go s' =<< refs <$> (inRepo $ Git.Command.pipeRead [Param "show-ref"])
+excludeReferenced ks = runfilter firstlevel ks >>= runfilter secondlevel
 	where
-		-- Skip the git-annex branches, and get all other unique refs.
-		refs = map (Git.Ref .  snd) .
-			nubBy uniqref .
-			filter ourbranches .
-			map (separate (== ' ')) . lines
-		uniqref (a, _) (b, _) = a == b
-		ourbranchend = '/' : show Annex.Branch.name
-		ourbranches (_, b) = not $ ourbranchend `isSuffixOf` b
-		go s [] = return $ S.toList s
-		go s (r:rs)
-			| s == S.empty = return [] -- optimisation
-			| otherwise = do
-				s' <- withKeysReferencedInGit r s S.delete
-				go s' rs
+		runfilter _ [] = return [] -- optimisation
+		runfilter a l = bloomFilter show l <$> genBloomFilter show a
+		firstlevel = withKeysReferencedM
+		secondlevel = withKeysReferencedInGit
 
 {- Finds items in the first, smaller list, that are not
  - present in the second, larger list.
@@ -180,7 +182,7 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 	where
 		remove a b = foldl (flip S.delete) b a
 
-{- Creates a bloom filter, and runs an action, such as withKeysReferenced',
+{- Creates a bloom filter, and runs an action, such as withKeysReferenced,
  - to populate it.
  -
  - The action is passed a callback that it can use to feed values into the
@@ -189,29 +191,36 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
  - Once the action completes, the mutable filter is frozen
  - for later use.
  -}
-genBloomFilter :: Hashable t => (v -> t) -> (() -> (v -> () -> Annex ()) -> Annex b) -> Annex (Bloom t)
+genBloomFilter :: Hashable t => (v -> t) -> ((v -> Annex ()) -> Annex b) -> Annex (Bloom t)
 genBloomFilter convert populate = do
-	-- A bloom filter capable of holding one million keys with a
-	-- false positive rate of 0.1% uses 16 mb of memory.
+	-- A bloom filter capable of holding half a million keys with a
+	-- false positive rate of 0.1% uses around 8 mb of memory.
 	-- TODO: make this configurable, for the really large repos,
 	-- or really low false positive rates.
-	let (numbits, numhashes) = suggestSizing 1000000 0.0001
+	let (numbits, numhashes) = suggestSizing 500000 0.001
 
 	bloom <- lift $ newMB (cheapHashes numhashes) numbits
-	_ <- populate () $ \v _ -> lift $ insertMB bloom (convert v)
+	_ <- populate $ \v -> lift $ insertMB bloom (convert v)
 	lift $ unsafeFreezeMB bloom
 	where
 		lift = liftIO . stToIO
 
-bloomFilter :: Bloom String -> [Key] -> [Key]
-bloomFilter b l = filter (\k -> show k `notElemB` b) l
+bloomFilter :: Hashable t => (v -> t) -> [v] -> Bloom t -> [v]
+bloomFilter convert l bloom = filter (\k -> convert k `notElemB` bloom) l
 
-{- Given an initial value, mutates it using an action for each
- - key referenced by symlinks in the git repo. -}
+{- Given an initial value, folds it with each key referenced by
+ - symlinks in the git repo. -}
 withKeysReferenced :: v -> (Key -> v -> v) -> Annex v
-withKeysReferenced initial a = withKeysReferenced' initial reta
+withKeysReferenced initial a = withKeysReferenced' initial folda
 	where
-		reta k v = return $ a k v
+		folda k v = return $ a k v
+
+{- Runs an action on each referenced key in the git repo. -}
+withKeysReferencedM :: (Key -> Annex ()) -> Annex ()
+withKeysReferencedM a = withKeysReferenced' () calla
+	where
+		calla k _ = a k
+
 withKeysReferenced' :: v -> (Key -> v -> Annex v) -> Annex v
 withKeysReferenced' initial a = go initial =<< files
 	where
@@ -227,21 +236,36 @@ withKeysReferenced' initial a = go initial =<< files
 					!v' <- a k v
 					go v' fs
 
-withKeysReferencedInGit :: Git.Ref -> v -> (Key -> v -> v) -> Annex v
-withKeysReferencedInGit ref initial a = do
-	showAction $ "checking " ++ Git.Ref.describe ref
-	go initial =<< inRepo (LsTree.lsTree ref)
+
+withKeysReferencedInGit :: (Key -> Annex ()) -> Annex ()
+withKeysReferencedInGit a = do
+	rs <- relevantrefs <$> showref
+	forM_ rs (withKeysReferencedInGitRef a)
 	where
-		go v [] = return v
-		go v (l:ls)
+		showref = inRepo $ Git.Command.pipeRead [Param "show-ref"]
+		relevantrefs = map (Git.Ref .  snd) .
+			nubBy uniqref .
+			filter ourbranches .
+			map (separate (== ' ')) . lines
+		uniqref (x, _) (y, _) = x == y
+		ourbranchend = '/' : show Annex.Branch.name
+		ourbranches (_, b) = not $ ourbranchend `isSuffixOf` b
+
+withKeysReferencedInGitRef :: (Key -> Annex ()) -> Git.Ref -> Annex ()
+withKeysReferencedInGitRef a ref = do
+	showAction $ "checking " ++ Git.Ref.describe ref
+	go =<< inRepo (LsTree.lsTree ref)
+	where
+		go [] = return ()
+		go (l:ls)
 			| isSymLink (LsTree.mode l) = do
 				content <- L.decodeUtf8 <$> catFile ref (LsTree.file l)
 				case fileKey (takeFileName $ L.unpack content) of
-					Nothing -> go v ls
+					Nothing -> go ls
 					Just k -> do
-						let !v' = a k v
-						go v' ls
-			| otherwise = go v ls
+						a k
+						go ls
+			| otherwise = go ls
 
 {- Looks in the specified directory for bad/tmp keys, and returns a list
  - of those that might still have value, or might be stale and removable.

From 25809ce2e0861a54ec63a414037b95fe29acc6df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 16:18:14 -0400
Subject: [PATCH 3255/8313] finish bloom filters

Add tuning, docs, etc.

Not sure if status is the right place to remote size.. perhaps unused
should report the size and also warn if it sees more keys than the bloom
filter allows?
---
 Command/Status.hs  | 19 ++++++++++++++++++-
 Command/Unused.hs  | 24 ++++++++++++++++++------
 debian/changelog   |  8 ++++++++
 debian/control     |  1 +
 doc/git-annex.mdwn | 17 +++++++++++++++++
 doc/install.mdwn   |  1 +
 git-annex.cabal    |  2 +-
 7 files changed, 64 insertions(+), 8 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 0b1741dc0a..eadb4f1634 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -76,6 +76,7 @@ slow_stats =
 	, local_annex_size
 	, known_annex_keys
 	, known_annex_size
+	, bloom_info
 	, backend_usage
 	]
 
@@ -127,7 +128,7 @@ remote_list level desc = stat n $ nojson $ lift $ do
 	return $ if null s then "0" else show (length rs) ++ "\n" ++ beginning s
 	where
 		n = desc ++ " repositories"
-
+	
 local_annex_size :: Stat
 local_annex_size = stat "local annex size" $ json id $
 	showSizeKeys <$> cachedPresentData
@@ -136,6 +137,22 @@ local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $ json show $
 	countKeys <$> cachedPresentData
 
+bloom_info :: Stat
+bloom_info = stat "bloom filter size" $ json id $ do
+	localkeys <- countKeys <$> cachedPresentData
+	capacity <- fromIntegral <$> lift Command.Unused.bloomCapacity
+	let note = aside $
+		if localkeys >= capacity
+		then "appears too small for this repository; adjust annex.bloomcapacity"
+		else "has room for " ++ show (capacity - localkeys) ++ " more local annex keys"
+
+	-- Two bloom filters are used at the same time, so double the size
+	-- of one.
+	size <- roughSize memoryUnits True . (* 2) . fromIntegral . fst <$>
+		lift Command.Unused.bloomBitsHashes
+
+	return $ size ++ note
+
 known_annex_size :: Stat
 known_annex_size = stat "known annex size" $ json id $
 	showSizeKeys <$> cachedReferencedData
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 028e20445d..b878ab2650 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -29,6 +29,7 @@ import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
+import qualified Git.Config
 import qualified Backend
 import qualified Remote
 import qualified Annex.Branch
@@ -182,6 +183,22 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
 	where
 		remove a b = foldl (flip S.delete) b a
 
+{- A bloom filter capable of holding half a million keys with a
+ - false positive rate of 1 in 1000 uses around 8 mb of memory,
+ - so will easily fit on even my lowest memory systems.
+ -}
+bloomCapacity :: Annex Int
+bloomCapacity = fromMaybe 500000 . readish
+	<$> fromRepo (Git.Config.get "annex.bloomcapacity" "")
+bloomAccuracy :: Annex Int
+bloomAccuracy = fromMaybe 1000 . readish
+	<$> fromRepo (Git.Config.get "annex.bloomaccuracy" "")
+bloomBitsHashes :: Annex (Int, Int)
+bloomBitsHashes = do
+	capacity <- bloomCapacity
+	accuracy <- bloomAccuracy
+	return $ suggestSizing capacity (1/ fromIntegral accuracy)
+
 {- Creates a bloom filter, and runs an action, such as withKeysReferenced,
  - to populate it.
  -
@@ -193,12 +210,7 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
  -}
 genBloomFilter :: Hashable t => (v -> t) -> ((v -> Annex ()) -> Annex b) -> Annex (Bloom t)
 genBloomFilter convert populate = do
-	-- A bloom filter capable of holding half a million keys with a
-	-- false positive rate of 0.1% uses around 8 mb of memory.
-	-- TODO: make this configurable, for the really large repos,
-	-- or really low false positive rates.
-	let (numbits, numhashes) = suggestSizing 500000 0.001
-
+	(numbits, numhashes) <- bloomBitsHashes
 	bloom <- lift $ newMB (cheapHashes numhashes) numbits
 	_ <- populate $ \v -> lift $ insertMB bloom (convert v)
 	lift $ unsafeFreezeMB bloom
diff --git a/debian/changelog b/debian/changelog
index 120513806e..8d73371162 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,14 @@ git-annex (3.20120310) UNRELEASED; urgency=low
     space, but now only needs to store the set of file contents that
     are present in the annex in memory.
   * status: Fixed to run in constant space.
+  * unused: Now uses a bloom filter, and runs in constant space.
+    Use of a bloom filter does mean it will not notice a small
+    number of unused keys. For repos with up to half a million keys,
+    it will miss one key in 1000.
+  * Added annex.bloomcapacity and annex.bloomaccuracy, which can be
+    adjusted as desired to tune the bloom filter.
+  * status: Display about of memory used by bloom filter, and
+    detect then it's too small for the number of keys in a repository.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 
diff --git a/debian/control b/debian/control
index 8ea1a6259e..a73433c2ac 100644
--- a/debian/control
+++ b/debian/control
@@ -18,6 +18,7 @@ Build-Depends:
 	libghc-lifted-base-dev,
 	libghc-json-dev,
 	libghc-ifelse-dev,
+	libghc-bloomfilter-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index a941d4420e..10899d12cd 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -598,6 +598,23 @@ Here are all the supported configuration settings.
   of memory and are working with very large numbers of files, increasing
   the queue size can speed it up.
 
+* `annex.bloomcapacity`
+
+  The `git annex unused` command uses a bloom filter to determine
+  what data is no longer used. The default bloom filter is sized to handle
+  up to 500000 keys. If your repository is larger than that,
+  you can adjust this to avoid `git annex unused` not noticing some unused
+  data files. Increasing this will make `git-annex unused` consume more memory;
+  run `git annex status` for memory usage numbers.
+
+* `annex.bloomaccuracy`
+
+  Adjusts the accuracy of the bloom filter used by
+  `git annex unused`. The default accuracy is 1000 -- 
+  1 unused file out of 1000 will be missed by `git annex unused`. Increasing
+  the accuracy will make `git annex unused` consume more memory;
+  run `git annex status` for memory usage numbers.
+
 * `annex.version`
 
   Automatically maintained, and used to automate upgrades between versions.
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 8de24d40db..0698a8bc4a 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -35,6 +35,7 @@ To build and use git-annex, you will need:
   * [hS3](http://hackage.haskell.org/package/hS3)
   * [json](http://hackage.haskell.org/package/json)
   * [IfElse](http://hackage.haskell.org/package/IfElse)
+  * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 6efebc66e8..278d87555a 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -32,7 +32,7 @@ Executable git-annex
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hs3, json, HTTP,
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1
+   IfElse, text, QuickCheck >= 2.1, bloomfilter
   Other-Modules: Utility.StatFS, Utility.Touch
 
 Executable git-annex-shell

From 77fb50e01a8c66f87fdd79c9c4235074a0f55fa7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 16:20:17 -0400
Subject: [PATCH 3256/8313] bloom branch

---
 doc/download.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 120e0a517d..2727cc1918 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -28,6 +28,8 @@ The git repository has some branches:
   library.
 * `tweak-fetch` adds support for the git tweak-fetch hook, which has
   been proposed and implemented but not yet accepted into git.
+* `bloom` makes `git annex unused` use a bloom filter, thus running
+  in truely constant memory. Waiting on the haskell library being packaged.
 * `ghc7.0` supports versions of ghc older than 7.4, which
   had a major change to filename encoding.
 * `setup` contains configuration for this website

From 8540183a02345843a7869521cd65abbe658b4e51 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 16:31:41 -0400
Subject: [PATCH 3257/8313] close

---
 doc/todo/git-annex_unused_eats_memory.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 91f9a33f7c..d02638d0ee 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -21,6 +21,8 @@ in the bloom filter. Since there can be false positives, this might
 miss finding some unused keys. The probability/size of filter
 could be tunable.
 
+> [[done]]! --[[Joey]] 
+
 Another way might be to scan the git log for files that got removed
 or changed what key they pointed to. Correlate with keys with content
 currently present in the repository (possibly using a bloom filter again),

From 6a95240dff4799669ebc14e610f2555c32a86ef3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 16:32:54 -0400
Subject: [PATCH 3258/8313] note fixed

---
 doc/todo/git-annex_unused_eats_memory.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/git-annex_unused_eats_memory.mdwn b/doc/todo/git-annex_unused_eats_memory.mdwn
index 91f9a33f7c..caaecfa8cf 100644
--- a/doc/todo/git-annex_unused_eats_memory.mdwn
+++ b/doc/todo/git-annex_unused_eats_memory.mdwn
@@ -21,6 +21,8 @@ in the bloom filter. Since there can be false positives, this might
 miss finding some unused keys. The probability/size of filter
 could be tunable.
 
+> Fixed in `bloom` branch in git. --[[Joey]] 
+
 Another way might be to scan the git log for files that got removed
 or changed what key they pointed to. Correlate with keys with content
 currently present in the repository (possibly using a bloom filter again),

From b27760aa68d8c4cf1a1b0be3c3a5582539f857b5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 12 Mar 2012 22:53:43 -0400
Subject: [PATCH 3259/8313] Work around a bug in rsync (IMHO) introduced by
 openSUSE's SIP patch.

openSUSE patches rsync with a patch adding SIP protocol support.
https://gist.github.com/2026167

With this patch, running rsync with no hostname parameter is apparently
supposed to list SIP hosts on the network. Practically, it does nothing
and exits 0.

git-annex uses rsync in a very special way to allow git-annex-shell to be
run on the remote host, and so did not need to specify a hostname, or a
file to transfer as a rsync parameter. So it sent ":", a degenerate case of
"host:file".

But the patch cannot differentiate ":" with no host parameter
(a bug in the SIP patch surely).

Results were that getting files failed, as rsync seemed to succeed, but the
requested file failed to arrive. Also I think that sending files will
make git-annex think a file has been transferred to the remote when
really rsync does nothing.

The workaround for this buggy rsync patch is to use "dummy:" as the
hostname.
---
 Remote/Git.hs    | 7 +++++--
 debian/changelog | 1 +
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Remote/Git.hs b/Remote/Git.hs
index 29c50e87fd..12a7f18448 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -290,10 +290,13 @@ rsyncParamsRemote r sending key file = do
 		then return $ o ++ eparam ++ [dummy, File file]
 		else return $ o ++ eparam ++ [File file, dummy]
 	where
-		-- the rsync shell parameter controls where rsync
+		-- The rsync shell parameter controls where rsync
 		-- goes, so the source/dest parameter can be a dummy value,
 		-- that just enables remote rsync mode.
-		dummy = Param ":"
+		-- For maximum compatability with some patched rsyncs,
+		-- the dummy value needs to still contain a hostname,
+		-- even though this hostname will never be used.
+		dummy = Param "dummy:"
 
 rsyncParams :: Git.Repo -> Annex [CommandParam]
 rsyncParams r = do
diff --git a/debian/changelog b/debian/changelog
index 8fc2cc3304..63797f2ad1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,7 @@ git-annex (3.20120310) UNRELEASED; urgency=low
     are present in the annex in memory.
   * status: Fixed to run in constant space.
   * status: More accurate display of sizes of tmp and bad keys.
+  * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 

From f047e04179a7366e9e20b7c5805242f97e6ae6c7 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm2MUhwzcOSnZfYnmWu7_2dMrH4064OKyQ"
 
Date: Wed, 14 Mar 2012 11:09:32 +0000
Subject: [PATCH 3260/8313]

---
 doc/forum/post-copy__47__sync_hook.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/post-copy__47__sync_hook.mdwn

diff --git a/doc/forum/post-copy__47__sync_hook.mdwn b/doc/forum/post-copy__47__sync_hook.mdwn
new file mode 100644
index 0000000000..05fbc0b291
--- /dev/null
+++ b/doc/forum/post-copy__47__sync_hook.mdwn
@@ -0,0 +1,14 @@
+Hi,
+
+I have the following setup:
+- normal git repository with website code.
+- git annex repository to hold large set of binary data (pdfs, flashmovies, etc) that belongs to the site.
+
+I use git annex so I (and other developers) don't need to copy 1.4Gb+ of binary data for every working copy. (Data that is mostly left untouched.) Using git annex copy --to=origin I can simply only add new additions to this media/binary repository, without first pulling all the data. So far so good.
+
+When commits are pushed to a certain branch on the normal git repository, a post-receive hook exports (GIT_WORK_TREE=/data/site/ git checkout $branch -f) the updated repository to an apache documentroot. Thereby updating the staging server of the website.
+
+My question is, how can I do the same thing for my git annex repository? Since post-receive fires on receiving the annex hashes, and not the actual files. Those are rsynced, and I cannot find a way to trigger an action after all files are copied by git annex via rsync.
+
+Any tips?
+

From caf97fcffd0a1e934fef60c0cae878ee3813f81f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:01:56 -0400
Subject: [PATCH 3261/8313] git-annex-shell: Runs hooks/annex-content after
 content is received or dropped.

---
 Command/Commit.hs        |  7 ++++++-
 Git.hs                   | 14 ++++++++++++--
 debian/changelog         |  2 ++
 doc/git-annex-shell.mdwn |  8 ++++++++
 4 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/Command/Commit.hs b/Command/Commit.hs
index 2bb016ea05..19b0bc250b 100644
--- a/Command/Commit.hs
+++ b/Command/Commit.hs
@@ -7,8 +7,10 @@
 
 module Command.Commit where
 
+import Common.Annex
 import Command
 import qualified Annex.Branch
+import qualified Git
 
 def :: [Command]
 def = [command "commit" paramNothing seek
@@ -20,4 +22,7 @@ seek = [withNothing start]
 start :: CommandStart
 start = next $ next $ do
 	Annex.Branch.commit "update"
-	return True
+	runhook =<< (inRepo $ Git.hookPath "annex-content")
+	where
+		runhook (Just hook) = liftIO $ boolSystem hook []
+		runhook Nothing = return True
diff --git a/Git.hs b/Git.hs
index 284bf331c1..043cda5ea1 100644
--- a/Git.hs
+++ b/Git.hs
@@ -24,12 +24,14 @@ module Git (
 	gitDir,
 	configTrue,
 	attributes,
+	hookPath,
 	assertLocal,
 ) where
 
 import qualified Data.Map as M
 import Data.Char
 import Network.URI (uriPath, uriScheme, unEscapeString)
+import System.Directory
 
 import Common
 import Git.Types
@@ -93,17 +95,25 @@ configBare repo = maybe unknown (fromMaybe False . configTrue) $
 			" is a bare repository; config not read"
 
 {- Path to a repository's gitattributes file. -}
-attributes :: Repo -> String
+attributes :: Repo -> FilePath
 attributes repo
 	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
 {- Path to a repository's .git directory. -}
-gitDir :: Repo -> String
+gitDir :: Repo -> FilePath
 gitDir repo
 	| configBare repo = workTree repo
 	| otherwise = workTree repo  ".git"
 
+{- Path to a given hook script in a repository, only if the hook exists
+ - and is executable. -}
+hookPath :: String -> Repo -> IO (Maybe FilePath)
+hookPath script repo = do
+	let hook = gitDir repo  "hooks"  script
+	ok <- doesFileExist hook
+	return $ if ok then Just hook else Nothing
+
 {- Path to a repository's --work-tree, that is, its top.
  -
  - Note that for URL repositories, this is the path on the remote host. -}
diff --git a/debian/changelog b/debian/changelog
index fd58386834..938aba130c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -16,6 +16,8 @@ git-annex (3.20120310) UNRELEASED; urgency=low
     adjusted as desired to tune the bloom filter.
   * status: Display about of memory used by bloom filter, and
     detect then it's too small for the number of keys in a repository.
+  * git-annex-shell: Runs hooks/annex-content after content is received
+    or dropped.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 
diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 0fd77a8115..89a05b1d68 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -49,6 +49,7 @@ first "/~/" or "/~user/" is expanded to the specified home directory.
 * commit
 
   This commits any staged changes to the git-annex branch.
+  It also runs the annex-content hook.
 
 # OPTIONS
 
@@ -60,6 +61,13 @@ to git-annex-shell are:
   git-annex uses this to specify the UUID of the repository it was expecting
   git-annex-shell to access, as a sanity check.
 
+# HOOK
+
+After content is received or dropped from the repository by git-annex-shell,
+it runs a hook, `.git/hooks/annex-content` (or `hooks/annex-content` on a bare
+repository). The hook is not currently passed any information about what
+changed.
+
 # ENVIRONMENT
 
 * GIT_ANNEX_SHELL_READONLY

From 95a1f6b2accca9b7c6a6c30c92380dc0de57d3a0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:17:38 -0400
Subject: [PATCH 3262/8313] check hook executability

---
 Git.hs              | 10 ++++++++--
 Utility/FileMode.hs |  7 +++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Git.hs b/Git.hs
index 043cda5ea1..9b7dccfebd 100644
--- a/Git.hs
+++ b/Git.hs
@@ -32,9 +32,11 @@ import qualified Data.Map as M
 import Data.Char
 import Network.URI (uriPath, uriScheme, unEscapeString)
 import System.Directory
+import System.Posix.Files
 
 import Common
 import Git.Types
+import Utility.FileMode
 
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
@@ -111,8 +113,12 @@ gitDir repo
 hookPath :: String -> Repo -> IO (Maybe FilePath)
 hookPath script repo = do
 	let hook = gitDir repo  "hooks"  script
-	ok <- doesFileExist hook
-	return $ if ok then Just hook else Nothing
+	e <- doesFileExist hook
+	if e
+		then do
+			m <- fileMode <$> getFileStatus hook
+			return $ if isExecutable m then Just hook else Nothing
+		else return Nothing
 
 {- Path to a repository's --work-tree, that is, its top.
  -
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index 6c1c06e82a..571b035039 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -34,3 +34,10 @@ allowWrite f = do
 {- Checks if a file mode indicates it's a symlink. -}
 isSymLink :: FileMode -> Bool
 isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode
+
+{- Checks if a file has any executable bits set. -}
+isExecutable :: FileMode -> Bool
+isExecutable mode = ebits `intersectFileModes` mode /= 0
+	where
+		ebits = ownerExecuteMode `unionFileModes`
+			groupExecuteMode `unionFileModes` otherExecuteMode

From 5b869eef911d36627174ed5a02452011effc2a14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:01:56 -0400
Subject: [PATCH 3263/8313] git-annex-shell: Runs hooks/annex-content after
 content is received or dropped.

---
 Command/Commit.hs        |  7 ++++++-
 Git.hs                   | 14 ++++++++++++--
 debian/changelog         |  2 ++
 doc/git-annex-shell.mdwn |  8 ++++++++
 4 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/Command/Commit.hs b/Command/Commit.hs
index 2bb016ea05..19b0bc250b 100644
--- a/Command/Commit.hs
+++ b/Command/Commit.hs
@@ -7,8 +7,10 @@
 
 module Command.Commit where
 
+import Common.Annex
 import Command
 import qualified Annex.Branch
+import qualified Git
 
 def :: [Command]
 def = [command "commit" paramNothing seek
@@ -20,4 +22,7 @@ seek = [withNothing start]
 start :: CommandStart
 start = next $ next $ do
 	Annex.Branch.commit "update"
-	return True
+	runhook =<< (inRepo $ Git.hookPath "annex-content")
+	where
+		runhook (Just hook) = liftIO $ boolSystem hook []
+		runhook Nothing = return True
diff --git a/Git.hs b/Git.hs
index 284bf331c1..043cda5ea1 100644
--- a/Git.hs
+++ b/Git.hs
@@ -24,12 +24,14 @@ module Git (
 	gitDir,
 	configTrue,
 	attributes,
+	hookPath,
 	assertLocal,
 ) where
 
 import qualified Data.Map as M
 import Data.Char
 import Network.URI (uriPath, uriScheme, unEscapeString)
+import System.Directory
 
 import Common
 import Git.Types
@@ -93,17 +95,25 @@ configBare repo = maybe unknown (fromMaybe False . configTrue) $
 			" is a bare repository; config not read"
 
 {- Path to a repository's gitattributes file. -}
-attributes :: Repo -> String
+attributes :: Repo -> FilePath
 attributes repo
 	| configBare repo = workTree repo ++ "/info/.gitattributes"
 	| otherwise = workTree repo ++ "/.gitattributes"
 
 {- Path to a repository's .git directory. -}
-gitDir :: Repo -> String
+gitDir :: Repo -> FilePath
 gitDir repo
 	| configBare repo = workTree repo
 	| otherwise = workTree repo  ".git"
 
+{- Path to a given hook script in a repository, only if the hook exists
+ - and is executable. -}
+hookPath :: String -> Repo -> IO (Maybe FilePath)
+hookPath script repo = do
+	let hook = gitDir repo  "hooks"  script
+	ok <- doesFileExist hook
+	return $ if ok then Just hook else Nothing
+
 {- Path to a repository's --work-tree, that is, its top.
  -
  - Note that for URL repositories, this is the path on the remote host. -}
diff --git a/debian/changelog b/debian/changelog
index 63797f2ad1..b6b62708e5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,8 @@ git-annex (3.20120310) UNRELEASED; urgency=low
   * status: Fixed to run in constant space.
   * status: More accurate display of sizes of tmp and bad keys.
   * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch.
+  * git-annex-shell: Runs hooks/annex-content after content is received
+    or dropped.
 
  -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
 
diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn
index 0fd77a8115..89a05b1d68 100644
--- a/doc/git-annex-shell.mdwn
+++ b/doc/git-annex-shell.mdwn
@@ -49,6 +49,7 @@ first "/~/" or "/~user/" is expanded to the specified home directory.
 * commit
 
   This commits any staged changes to the git-annex branch.
+  It also runs the annex-content hook.
 
 # OPTIONS
 
@@ -60,6 +61,13 @@ to git-annex-shell are:
   git-annex uses this to specify the UUID of the repository it was expecting
   git-annex-shell to access, as a sanity check.
 
+# HOOK
+
+After content is received or dropped from the repository by git-annex-shell,
+it runs a hook, `.git/hooks/annex-content` (or `hooks/annex-content` on a bare
+repository). The hook is not currently passed any information about what
+changed.
+
 # ENVIRONMENT
 
 * GIT_ANNEX_SHELL_READONLY

From f974a200871d11edc44af5e5acad491f9653ad12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:17:38 -0400
Subject: [PATCH 3264/8313] check hook executability

---
 Git.hs              | 10 ++++++++--
 Utility/FileMode.hs |  7 +++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Git.hs b/Git.hs
index 043cda5ea1..9b7dccfebd 100644
--- a/Git.hs
+++ b/Git.hs
@@ -32,9 +32,11 @@ import qualified Data.Map as M
 import Data.Char
 import Network.URI (uriPath, uriScheme, unEscapeString)
 import System.Directory
+import System.Posix.Files
 
 import Common
 import Git.Types
+import Utility.FileMode
 
 {- User-visible description of a git repo. -}
 repoDescribe :: Repo -> String
@@ -111,8 +113,12 @@ gitDir repo
 hookPath :: String -> Repo -> IO (Maybe FilePath)
 hookPath script repo = do
 	let hook = gitDir repo  "hooks"  script
-	ok <- doesFileExist hook
-	return $ if ok then Just hook else Nothing
+	e <- doesFileExist hook
+	if e
+		then do
+			m <- fileMode <$> getFileStatus hook
+			return $ if isExecutable m then Just hook else Nothing
+		else return Nothing
 
 {- Path to a repository's --work-tree, that is, its top.
  -
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index 6c1c06e82a..571b035039 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -34,3 +34,10 @@ allowWrite f = do
 {- Checks if a file mode indicates it's a symlink. -}
 isSymLink :: FileMode -> Bool
 isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode
+
+{- Checks if a file has any executable bits set. -}
+isExecutable :: FileMode -> Bool
+isExecutable mode = ebits `intersectFileModes` mode /= 0
+	where
+		ebits = ownerExecuteMode `unionFileModes`
+			groupExecuteMode `unionFileModes` otherExecuteMode

From c26fbecaf48b60f0bf9cb4e8e7880adcf19ceee3 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 14 Mar 2012 16:23:26 +0000
Subject: [PATCH 3265/8313] Added a comment

---
 ...omment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment

diff --git a/doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment b/doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment
new file mode 100644
index 0000000000..04a6f4668e
--- /dev/null
+++ b/doc/forum/post-copy__47__sync_hook/comment_1_c8322d4b9bbf5eac80b48c312a42fbcf._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-14T16:23:25Z"
+ content="""
+I've made git-annex-shell run the git `hooks/annex-content` after content is received or dropped. 
+
+Note that the clients need to be running at least git-annex version 3.20120227 , which runs git-annex-shell commit, which runs the hook.
+
+"""]]

From 6cb4743cfb105427a00b551ebd51fc5c9626bfea Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:39:09 -0400
Subject: [PATCH 3266/8313] ignore hook exit status

---
 Command/Commit.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/Commit.hs b/Command/Commit.hs
index 19b0bc250b..1c82ed7df8 100644
--- a/Command/Commit.hs
+++ b/Command/Commit.hs
@@ -22,7 +22,8 @@ seek = [withNothing start]
 start :: CommandStart
 start = next $ next $ do
 	Annex.Branch.commit "update"
-	runhook =<< (inRepo $ Git.hookPath "annex-content")
+	_ <- runhook =<< (inRepo $ Git.hookPath "annex-content")
+	return True
 	where
 		runhook (Just hook) = liftIO $ boolSystem hook []
 		runhook Nothing = return True

From bfa15bd66500d041a72b5ddb03117d434b1ecfde Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:43:34 -0400
Subject: [PATCH 3267/8313] no-bloom branch

---
 doc/download.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 2727cc1918..30307f3088 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -23,13 +23,12 @@ The git repository has some branches:
 * `debian-stable` contains the latest backport of git-annex to Debian
   stable.
 * `no-s3` disables the S3 special remote, for systems that lack the
- necessary haskell library.
+ necessary haskell library. (merge it into master if you need it)
+* `no-bloom` avoids using bloom filters. (merge it into master if you need it)
 * `old-monad-control` is for systems that don't have a newer monad-control
   library.
 * `tweak-fetch` adds support for the git tweak-fetch hook, which has
   been proposed and implemented but not yet accepted into git.
-* `bloom` makes `git annex unused` use a bloom filter, thus running
-  in truely constant memory. Waiting on the haskell library being packaged.
 * `ghc7.0` supports versions of ghc older than 7.4, which
   had a major change to filename encoding.
 * `setup` contains configuration for this website

From a4f72c9625486786a4549cf4db1b542ea89da7c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 12:44:17 -0400
Subject: [PATCH 3268/8313] update

---
 debian/changelog | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 393234ba55..0f8fc001e1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,9 +3,6 @@ git-annex (3.20120310) UNRELEASED; urgency=low
   * fsck: Fix up any broken links and misplaced content caused by the
     directory hash calculation bug fixed in the last release.
   * sync: Sync to lower cost remotes first.
-  * unused: Reduce memory usage significantly. Still not constant
-    space, but now only needs to store the set of file contents that
-    are present in the annex in memory.
   * status: Fixed to run in constant space.
   * status: More accurate display of sizes of tmp and bad keys.
   * unused: Now uses a bloom filter, and runs in constant space.
@@ -14,7 +11,7 @@ git-annex (3.20120310) UNRELEASED; urgency=low
     it will miss one key in 1000.
   * Added annex.bloomcapacity and annex.bloomaccuracy, which can be
     adjusted as desired to tune the bloom filter.
-  * status: Display about of memory used by bloom filter, and
+  * status: Display amount of memory used by bloom filter, and
     detect then it's too small for the number of keys in a repository.
   * git-annex-shell: Runs hooks/annex-content after content is received
     or dropped.

From 60ab3d84e188b8dd3a284d962df25bbee41ff1cb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 17:43:34 -0400
Subject: [PATCH 3269/8313] added ifM and nuked 11 lines of code

no behavior changes
---
 Annex/Content.hs     | 35 +++++++++++-----------
 Annex/Ssh.hs         | 20 +++++++------
 Annex/UUID.hs        |  6 ++--
 Command.hs           |  4 +--
 Command/Add.hs       |  9 +++---
 Command/AddUrl.hs    | 14 ++++-----
 Command/Fsck.hs      | 70 ++++++++++++++++++++++++--------------------
 Command/Get.hs       | 38 ++++++++++--------------
 Command/Map.hs       | 10 +++----
 Command/Move.hs      |  8 ++---
 Command/PreCommit.hs |  8 ++---
 Command/Sync.hs      | 30 ++++++++-----------
 Command/Unannex.hs   |  8 ++---
 Command/Unused.hs    |  8 ++---
 Init.hs              | 17 +++++------
 Remote.hs            | 14 +++------
 Utility/Monad.hs     | 14 +++++----
 17 files changed, 151 insertions(+), 162 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index ccaff5c564..fad5f51349 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -150,16 +150,16 @@ prepTmp key = do
 getViaTmpUnchecked :: Key -> (FilePath -> Annex Bool) -> Annex Bool
 getViaTmpUnchecked key action = do
 	tmp <- prepTmp key
-	success <- action tmp
-	if success
-		then do
+	ifM (action tmp)
+		( do
 			moveAnnex key tmp
 			logStatus key InfoPresent
 			return True
-		else do
+		, do
 			-- the tmp file is left behind, in case caller wants
 			-- to resume its transfer
 			return False
+		)
 
 {- Creates a temp file, runs an action on it, and cleans up the temp file. -}
 withTmp :: Key -> (FilePath -> Annex a) -> Annex a
@@ -230,15 +230,15 @@ moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
 	dest <- inRepo $ gitAnnexLocation key
 	let dir = parentDir dest
-	e <- liftIO $ doesFileExist dest
-	if e
-		then liftIO $ removeFile src
-		else liftIO $ do
+	liftIO $ ifM (doesFileExist dest)
+		( removeFile src
+		, do
 			createDirectoryIfMissing True dir
 			allowWrite dir -- in case the directory already exists
 			moveFile src dest
 			preventWrite dest
 			preventWrite dir
+		)
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
 withObjectLoc key a = do
@@ -314,12 +314,12 @@ getKeysPresent = liftIO . traverse (2 :: Int) =<< fromRepo gitAnnexObjectDir
 saveState :: Bool -> Annex ()
 saveState oneshot = do
 	Annex.Queue.flush False
-	unless oneshot $ do
-		alwayscommit <- fromMaybe True . Git.configTrue
+	unless oneshot $
+		ifM alwayscommit
+			( Annex.Branch.commit "update" , Annex.Branch.stage)
+	where
+		alwayscommit = fromMaybe True . Git.configTrue
 			<$> fromRepo (Git.Config.get "annex.alwayscommit" "")
-		if alwayscommit
-			then Annex.Branch.commit "update"
-			else Annex.Branch.stage
 
 {- Downloads content from any of a list of urls. -}
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
@@ -338,10 +338,9 @@ preseedTmp key file = go =<< inAnnex key
 			ok <- copy
 			when ok $ liftIO $ allowWrite file
 			return ok
-		copy = do
-			present <- liftIO $ doesFileExist file
-			if present
-				then return True
-				else do
+		copy = ifM (liftIO $ doesFileExist file)
+				( return True
+				, do
 					s <- inRepo $ gitAnnexLocation key
 					liftIO $ copyFileExternal s file
+				)
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 39983ab250..79cfbe908b 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -37,15 +37,17 @@ sshParams (host, port) opts = go =<< sshInfo (host, port)
 			sshCleanup
 
 sshInfo :: (String, Maybe Integer) -> Annex (Maybe FilePath, [CommandParam])
-sshInfo (host, port) = do
-	caching <- fromMaybe SysConfig.sshconnectioncaching . Git.configTrue
-		<$> fromRepo (Git.Config.get "annex.sshcaching" "")
-	if caching
-		then do
-			dir <- fromRepo gitAnnexSshDir
-			let socketfile = dir  hostport2socket host port
-		 	return (Just socketfile, cacheParams socketfile)
-		else return (Nothing, [])
+sshInfo (host, port) = ifM caching
+	( do
+		dir <- fromRepo gitAnnexSshDir
+		let socketfile = dir  hostport2socket host port
+	 	return (Just socketfile, cacheParams socketfile)
+	, return (Nothing, [])
+	)
+	where
+		caching = fromMaybe SysConfig.sshconnectioncaching 
+			. Git.configTrue
+			<$> fromRepo (Git.Config.get "annex.sshcaching" "")
 
 cacheParams :: FilePath -> [CommandParam]
 cacheParams socketfile =
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 48bf71f104..0ab2e7e521 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -34,11 +34,11 @@ genUUID :: IO UUID
 genUUID = pOpen ReadFromPipe command params $ liftM toUUID . hGetLine
 	where
 		command = SysConfig.uuid
-		params = if command == "uuid"
+		params
 			-- request a random uuid be generated
-			then ["-m"]
+			| command == "uuid" = ["-m"]
 			-- uuidgen generates random uuid by default
-			else []
+			| otherwise = []
 
 {- Get current repository's UUID. -}
 getUUID :: Annex UUID
diff --git a/Command.hs b/Command.hs
index 13ea167bbc..0dff0c862e 100644
--- a/Command.hs
+++ b/Command.hs
@@ -65,9 +65,7 @@ stop = return Nothing
 
 {- Stops unless a condition is met. -}
 stopUnless :: Annex Bool -> Annex (Maybe a) -> Annex (Maybe a)
-stopUnless c a = do
-	ok <- c
-	if ok then a else stop
+stopUnless c a = ifM c ( a , stop )
 
 {- Prepares to run a command via the check and seek stages, returning a
  - list of actions to perform to run the command. -}
diff --git a/Command/Add.hs b/Command/Add.hs
index b6b5753af7..ef839b2a30 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -85,8 +85,9 @@ cleanup file key hascontent = do
 				mtime <- modificationTime <$> getFileStatus file
 				touch file (TimeSpec mtime) False
 
-	force <- Annex.getState Annex.force
-	if force
-		then Annex.Queue.add "add" [Param "-f", Param "--"] [file]
-		else Annex.Queue.add "add" [Param "--"] [file]
+	params <- ifM (Annex.getState Annex.force)
+		( return [Param "-f"]
+		, return []
+		)
+	Annex.Queue.add "add" (params++[Param "--"]) [file]
 	return True
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 6c945baf93..c87399f5dc 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -51,17 +51,17 @@ perform url file = ifAnnexed file addurl geturl
 	where
 		geturl = do
 			liftIO $ createDirectoryIfMissing True (parentDir file)
-			fast <- Annex.getState Annex.fast
-			if fast then nodownload url file else download url file
-		addurl (key, _backend) = do
-			ok <- liftIO $ Url.check url (keySize key)
-			if ok
-				then do
+			ifM (Annex.getState Annex.fast)
+				( nodownload url file , download url file )
+		addurl (key, _backend) =
+			ifM (liftIO $ Url.check url $ keySize key)
+				( do
 					setUrlPresent key url
 					next $ return True
-				else do
+				, do
 					warning $ "failed to verify url: " ++ url
 					stop
+				)
 
 download :: String -> FilePath -> CommandPerform
 download url file = do
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index d8d0db23be..dac3bfac96 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -62,17 +62,18 @@ perform key file backend numcopies = check
 {- To fsck a remote, the content is retrieved to a tmp file,
  - and checked locally. -}
 performRemote :: Key -> FilePath -> Backend -> Maybe Int -> Remote -> CommandPerform
-performRemote key file backend numcopies remote = do
-	v <- Remote.hasKey remote key
-	case v of
-		Left err -> do
+performRemote key file backend numcopies remote =
+	dispatch =<< Remote.hasKey remote key
+	where
+		dispatch (Left err) = do
 			showNote err
 			stop
-		Right True -> withtmp $ \tmpfile -> do
-			copied <- getfile tmpfile
-			if copied then go True (Just tmpfile) else go True Nothing
-		Right False -> go False Nothing
-	where
+		dispatch (Right True) = withtmp $ \tmpfile ->
+			ifM (getfile tmpfile)
+				( go True (Just tmpfile)
+				, go True Nothing
+				)
+		dispatch (Right False) = go False Nothing
 		go present localcopy = check
 			[ verifyLocationLogRemote key file remote present
 			, checkKeySizeRemote key remote localcopy
@@ -87,15 +88,14 @@ performRemote key file backend numcopies remote = do
 			let cleanup = liftIO $ catchIO (removeFile tmp) (const $ return ())
 			cleanup
 			cleanup `after` a tmp
-		getfile tmp = do
-			ok <- Remote.retrieveKeyFileCheap remote key tmp
-			if ok
-				then return ok
-				else do
-					fast <- Annex.getState Annex.fast
-					if fast
-						then return False
-						else Remote.retrieveKeyFile remote key tmp
+		getfile tmp =
+			ifM (Remote.retrieveKeyFileCheap remote key tmp)
+				( return True
+				, ifM (Annex.getState Annex.fast)
+					( return False
+					, Remote.retrieveKeyFile remote key tmp
+					)
+				)
 
 {- To fsck a bare repository, fsck each key in the location log. -}
 withBarePresentKeys :: (Key -> CommandStart) -> CommandSeek
@@ -205,10 +205,10 @@ verifyLocationLog' key desc present u bad = do
 checkKeySize :: Key -> Annex Bool
 checkKeySize key = do
 	file <- inRepo $ gitAnnexLocation key
-	present <- liftIO $ doesFileExist file
-	if present
-		then checkKeySize' key file badContent
-		else return True
+	ifM (liftIO $ doesFileExist file)
+		( checkKeySize' key file badContent
+		, return True
+		)
 
 checkKeySizeRemote :: Key -> Remote -> Maybe FilePath -> Annex Bool
 checkKeySizeRemote _ _ Nothing = return True
@@ -219,16 +219,22 @@ checkKeySize' :: Key -> FilePath -> (Key -> Annex String) -> Annex Bool
 checkKeySize' key file bad = case Types.Key.keySize key of
 	Nothing -> return True
 	Just size -> do
-		stat <- liftIO $ getFileStatus file
-		let size' = fromIntegral (fileSize stat)
-		if size == size'
-			then return True
-			else do
-				msg <- bad key
-				warning $ "Bad file size (" ++
-					compareSizes storageUnits True size size' ++
-					"); " ++ msg
-				return False
+		size' <- fromIntegral . fileSize
+			<$> (liftIO $ getFileStatus file)
+		comparesizes size size'
+	where
+		comparesizes a b = do
+			let same = a == b
+			unless same $ badsize a b
+			return same
+		badsize a b = do
+			msg <- bad key
+			warning $ concat
+				[ "Bad file size ("
+				, compareSizes storageUnits True a b
+				, "); "
+				, msg
+				]
 
 checkBackend :: Backend -> Key -> Annex Bool
 checkBackend backend key = do
diff --git a/Command/Get.hs b/Command/Get.hs
index 9b12b95994..772fbd90c2 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -42,37 +42,29 @@ perform key = stopUnless (getViaTmp key $ getKeyFile key) $
 {- Try to find a copy of the file in one of the remotes,
  - and copy it to here. -}
 getKeyFile :: Key -> FilePath -> Annex Bool
-getKeyFile key file = do
-	remotes <- Remote.keyPossibilities key
-	if null remotes
-		then do
+getKeyFile key file = dispatch =<< Remote.keyPossibilities key
+	where
+		dispatch [] = do
 			showNote "not available"
 			Remote.showLocations key []
 			return False
-		else trycopy remotes remotes
-	where
+		dispatch remotes = trycopy remotes remotes
 		trycopy full [] = do
 			Remote.showTriedRemotes full
 			Remote.showLocations key []
 			return False
-		trycopy full (r:rs) = do
-			probablythere <- probablyPresent r
-			if probablythere
-				then docopy r (trycopy full rs)
-				else trycopy full rs
+		trycopy full (r:rs) =
+			ifM (probablyPresent r)
+				( docopy r (trycopy full rs)
+				, trycopy full rs
+				)
 		-- This check is to avoid an ugly message if a remote is a
 		-- drive that is not mounted.
-		probablyPresent r =
-			if Remote.hasKeyCheap r
-				then do
-					res <- Remote.hasKey r key
-					case res of
-						Right b -> return b
-						Left _ -> return False
-				else return True
+		probablyPresent r
+			| Remote.hasKeyCheap r =
+				either (const False) id <$> Remote.hasKey r key
+			| otherwise = return True
 		docopy r continue = do
 			showAction $ "from " ++ Remote.name r
-			copied <- Remote.retrieveKeyFile r key file
-			if copied
-				then return True
-				else continue
+			ifM (Remote.retrieveKeyFile r key file)
+				( return True , continue)
diff --git a/Command/Map.hs b/Command/Map.hs
index da7a048a4d..bdb86f95a5 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -41,14 +41,14 @@ start = do
 	trusted <- trustGet Trusted
 
 	liftIO $ writeFile file (drawMap rs umap trusted)
-	next $ next $ do
-		fast <- Annex.getState Annex.fast
-		if fast
-			then return True
-			else do
+	next $ next $
+		ifM (Annex.getState Annex.fast)
+			( return True
+			, do
 				showLongNote $ "running: dot -Tx11 " ++ file
 				showOutput
 				liftIO $ boolSystem "dot" [Param "-Tx11", File file]
+			)
 	where
 		file = "map.dot"
 
diff --git a/Command/Move.hs b/Command/Move.hs
index 6b58f711aa..8612c9f2db 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -131,13 +131,13 @@ fromOk src key
 			return $ u /= Remote.uuid src && any (== src) remotes
 fromPerform :: Remote -> Bool -> Key -> CommandPerform
 fromPerform src move key = moveLock move key $ do
-	ishere <- inAnnex key
-	if ishere
-		then handle move True
-		else do
+	ifM (inAnnex key)
+		( handle move True
+		, do
 			showAction $ "from " ++ Remote.name src
 			ok <- getViaTmp key $ Remote.retrieveKeyFile src key
 			handle move ok
+		)
 	where
 		handle _ False = stop -- failed
 		handle False True = next $ return True -- copy complete
diff --git a/Command/PreCommit.hs b/Command/PreCommit.hs
index b0328ca190..06140fa52a 100644
--- a/Command/PreCommit.hs
+++ b/Command/PreCommit.hs
@@ -7,6 +7,7 @@
 
 module Command.PreCommit where
 
+import Common.Annex
 import Command
 import qualified Command.Add
 import qualified Command.Fix
@@ -26,7 +27,6 @@ start file = next $ perform file
 
 perform :: FilePath -> CommandPerform
 perform file = do
-	ok <- doCommand $ Command.Add.start file
-	if ok
-		then next $ return True
-		else error $ "failed to add " ++ file ++ "; canceling commit"
+	unlessM (doCommand $ Command.Add.start file) $
+		error $ "failed to add " ++ file ++ "; canceling commit"
+	next $ return True
diff --git a/Command/Sync.hs b/Command/Sync.hs
index 51b6d6f636..b9ef0bc979 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -51,11 +51,7 @@ remoteBranch :: Remote -> Git.Ref -> Git.Ref
 remoteBranch remote = Git.Ref.under $ "refs/remotes/" ++ Remote.name remote
 
 syncRemotes :: [String] -> Annex [Remote]
-syncRemotes rs = do
-	fast <- Annex.getState Annex.fast
-	if fast
-		then nub <$> pickfast
-		else wanted
+syncRemotes rs = ifM (Annex.getState Annex.fast) ( nub <$> pickfast , wanted )
 	where
 		pickfast = (++) <$> listed <*> (good =<< fastest <$> available)
 		wanted
@@ -113,11 +109,11 @@ pullRemote remote branch = do
 	showStart "pull" (Remote.name remote)
 	next $ do
 		showOutput
-		fetched <- inRepo $ Git.Command.runBool "fetch"
+		stopUnless fetch $
+			next $ mergeRemote remote branch
+	where
+		fetch = inRepo $ Git.Command.runBool "fetch"
 			[Param $ Remote.name remote]
-		if fetched
-			then next $ mergeRemote remote branch
-			else stop
 
 {- The remote probably has both a master and a synced/master branch.
  - Which to merge from? Well, the master has whatever latest changes
@@ -159,15 +155,15 @@ mergeFrom branch = do
 changed :: Remote -> Git.Ref -> Annex Bool
 changed remote b = do
 	let r = remoteBranch remote b
-	e <- inRepo $ Git.Ref.exists r
-	if e
-		then inRepo $ Git.Branch.changed b r
-		else return False
+	ifM (inRepo $ Git.Ref.exists r)
+		( inRepo $ Git.Branch.changed b r
+		, return False
+		)
 
 newer :: Remote -> Git.Ref -> Annex Bool
 newer remote b = do
 	let r = remoteBranch remote b
-	e <- inRepo $ Git.Ref.exists r
-	if e
-		then inRepo $ Git.Branch.changed r b
-		else return True
+	ifM (inRepo $ Git.Ref.exists r)
+		( inRepo $ Git.Branch.changed r b
+		, return True
+		)
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index fee67429df..1e7313711c 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -47,16 +47,16 @@ cleanup file key = do
 			Params "-m", Param "content removed from git annex",
 			Param "--", File file]
 
-	fast <- Annex.getState Annex.fast
-	if fast
-		then do
+	ifM (Annex.getState Annex.fast)
+		( do
 			-- fast mode: hard link to content in annex
 			src <- inRepo $ gitAnnexLocation key
 			liftIO $ do
 				createLink src file
 				allowWrite file
-		else do
+		, do
 			fromAnnex key file
 			logStatus key InfoMissing
+		)
 
 	return True
diff --git a/Command/Unused.hs b/Command/Unused.hs
index b878ab2650..246929f715 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -299,11 +299,11 @@ staleKeysPrune dirspec = do
 staleKeys :: (Git.Repo -> FilePath) -> Annex [Key]
 staleKeys dirspec = do
 	dir <- fromRepo dirspec
-	exists <- liftIO $ doesDirectoryExist dir
-	if not exists
-		then return []
-		else do
+	ifM (liftIO $ doesDirectoryExist dir)
+		( do
 			contents <- liftIO $ getDirectoryContents dir
 			files <- liftIO $ filterM doesFileExist $
 				map (dir ) contents
 			return $ mapMaybe (fileKey . takeFileName) files
+		, return []
+		)
diff --git a/Init.hs b/Init.hs
index c9d5bb909a..f3d8bd0171 100644
--- a/Init.hs
+++ b/Init.hs
@@ -38,23 +38,22 @@ uninitialize = gitPreCommitHookUnWrite
 ensureInitialized :: Annex ()
 ensureInitialized = getVersion >>= maybe needsinit checkVersion
 	where
-		needsinit = do
-			annexed <- Annex.Branch.hasSibling
-			if annexed
-				then initialize Nothing
-				else error "First run: git-annex init"
+		needsinit = ifM Annex.Branch.hasSibling
+				( initialize Nothing
+				, error "First run: git-annex init"
+				)
 
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Annex ()
 gitPreCommitHookWrite = unlessBare $ do
 	hook <- preCommitHook
-	exists <- liftIO $ doesFileExist hook
-	if exists
-		then warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
-		else liftIO $ do
+	ifM (liftIO $ doesFileExist hook)
+		( warning $ "pre-commit hook (" ++ hook ++ ") already exists, not configuring"
+		, liftIO $ do
 			viaTmp writeFile hook preCommitScript
 			p <- getPermissions hook
 			setPermissions hook $ p {executable = True}
+		)
 
 gitPreCommitHookUnWrite :: Annex ()
 gitPreCommitHookUnWrite = unlessBare $ do
diff --git a/Remote.hs b/Remote.hs
index b3f464f5c2..aac45fae9d 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -70,19 +70,13 @@ addName desc n
  - (Or it can be a UUID.) Only finds currently configured git remotes. -}
 byName :: Maybe String -> Annex (Maybe Remote)
 byName Nothing = return Nothing
-byName (Just n) = do
-	res <- byName' n
-	case res of
-		Left e -> error e
-		Right r -> return $ Just r
+byName (Just n) = either error Just <$> byName' n
 byName' :: String -> Annex (Either String Remote)
 byName' "" = return $ Left "no remote specified"
-byName' n = do
-	match <- filter matching <$> remoteList
-	if null match
-		then return $ Left $ "there is no git remote named \"" ++ n ++ "\""
-		else return $ Right $ Prelude.head match
+byName' n = handle . filter matching <$> remoteList
 	where
+		handle [] = Left $ "there is no git remote named \"" ++ n ++ "\""
+		handle match = Right $ Prelude.head match
 		matching r = n == name r || toUUID n == uuid r
 
 {- Looks up a remote by name (or by UUID, or even by description),
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 28aa33ee82..23c0c4c194 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -1,6 +1,6 @@
 {- monadic stuff
  -
- - Copyright 2010-2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -14,11 +14,7 @@ import Control.Monad (liftM)
  - predicate -}
 firstM :: Monad m => (a -> m Bool) -> [a] -> m (Maybe a)
 firstM _ [] = return Nothing
-firstM p (x:xs) = do
-	q <- p x
-	if q
-		then return (Just x)
-		else firstM p xs
+firstM p (x:xs) = ifM (p x) (return $ Just x , firstM p xs)
 
 {- Returns true if any value in the list satisfies the predicate,
  - stopping once one is found. -}
@@ -29,6 +25,12 @@ anyM p = liftM isJust . firstM p
 untilTrue :: Monad m => [a] -> (a -> m Bool) -> m Bool
 untilTrue = flip anyM
 
+{- if with a monadic conditional. -}
+ifM :: Monad m => m Bool -> (m a, m a) -> m a
+ifM cond (thenclause, elseclause) = do
+	c <- cond
+	if c then thenclause else elseclause
+
 {- Runs an action, passing its value to an observer before returning it. -}
 observe :: Monad m => (a -> m b) -> m a -> m a
 observe observer a = do

From d820099c8ff3ac208e90dc676eb3d7ea0b8bfd9f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 14 Mar 2012 23:50:28 -0400
Subject: [PATCH 3270/8313] makefile tweaks

Put build cruft in a subdir
---
 .gitignore      |  1 +
 Makefile        | 14 +++++++-------
 git-annex.cabal |  2 +-
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index 250d0381c4..fe65fc5747 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+tmp
 *.hi
 *.o
 test
diff --git a/Makefile b/Makefile
index 08c5b08c00..fbdefc2726 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,10 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
-GHCFLAGS=-O2 -Wall $(IGNORE)
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp
+GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
-GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(IGNORE)
+GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS)
 endif
 
 GHCMAKE=ghc $(GHCFLAGS) --make
@@ -24,7 +25,7 @@ all: $(all)
 sources: $(sources)
 
 # Disables optimisation. Not for production use.
-fast: GHCFLAGS=-Wall $(IGNORE)
+fast: GHCFLAGS=$(BASEFLAGS)
 fast: $(bins)
 
 Build/SysConfig.hs: configure.hs Build/TestConfig.hs Utility/StatFS.hs
@@ -65,7 +66,7 @@ test:
 
 testcoverage:
 	rm -f test.tix test
-	ghc -odir build/test -hidir build/test $(GHCFLAGS) --make -fhpc test
+	ghc $(GHCFLAGS) -outputdir tmp/testcoverage --make -fhpc test
 	./test
 	@echo ""
 	@hpc report test --exclude=Main --exclude=QC
@@ -89,9 +90,8 @@ docs: $(mans)
 		--exclude='news/.*'
 
 clean:
-	rm -rf build $(bins) $(mans) test configure  *.tix .hpc $(sources)
-	rm -rf doc/.ikiwiki html dist
-	find . \( -name \*.o -or -name \*.hi \) -exec rm {} \;
+	rm -rf tmp $(bins) $(mans) test configure  *.tix .hpc $(sources) \
+		doc/.ikiwiki html dist
 
 # Workaround for cabal sdist not running Setup hooks, so I cannot
 # generate a file list there.
diff --git a/git-annex.cabal b/git-annex.cabal
index 278d87555a..aa86b44416 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120309
+Version: 3.20120310
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From 7a65df32236df42d49758ee861237613f501e3c2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Mar 2012 11:00:29 -0400
Subject: [PATCH 3271/8313] oh darn, I lost the crazy capitalization of hS3 a
 while ago

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index aa86b44416..7d69b9a919 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -30,7 +30,7 @@ Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hs3, json, HTTP,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter
   Other-Modules: Utility.StatFS, Utility.Touch

From d2769cf7953657ac9ff6ba2acc27cb71a6543c5d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Mar 2012 12:00:19 -0400
Subject: [PATCH 3272/8313] shave some 12 mb from the installed size

* git-annex now behaves as git-annex-shell if symlinked to and run by that
  name. The Makefile sets this up, saving some 8 mb of installed size.
* git-union-merge is a demo program, so it is no longer built by default.
---
 GitAnnexShell.hs   | 116 +++++++++++++++++++++++++++++++++++++++++++++
 Makefile           |   5 +-
 debian/changelog   |   9 ++--
 git-annex-shell.hs | 110 ++----------------------------------------
 git-annex.cabal    |   5 +-
 git-annex.hs       |  14 ++++--
 6 files changed, 140 insertions(+), 119 deletions(-)
 create mode 100644 GitAnnexShell.hs

diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs
new file mode 100644
index 0000000000..3394bc477d
--- /dev/null
+++ b/GitAnnexShell.hs
@@ -0,0 +1,116 @@
+{- git-annex-shell main program
+ -
+ - Copyright 2010 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module GitAnnexShell where
+
+import System.Environment
+import System.Console.GetOpt
+
+import Common.Annex
+import qualified Git.Construct
+import CmdLine
+import Command
+import Annex.UUID
+import qualified Option
+
+import qualified Command.ConfigList
+import qualified Command.InAnnex
+import qualified Command.DropKey
+import qualified Command.RecvKey
+import qualified Command.SendKey
+import qualified Command.Commit
+
+cmds_readonly :: [Command]
+cmds_readonly = concat
+	[ Command.ConfigList.def
+	, Command.InAnnex.def
+	, Command.SendKey.def
+	]
+
+cmds_notreadonly :: [Command]
+cmds_notreadonly = concat
+	[ Command.RecvKey.def
+	, Command.DropKey.def
+	, Command.Commit.def
+	]
+
+cmds :: [Command]
+cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
+	where
+		adddirparam c = c
+			{ cmdparamdesc = "DIRECTORY " ++ cmdparamdesc c
+			}
+
+options :: [OptDescr (Annex ())]
+options = Option.common ++
+	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid"
+	]
+	where
+		checkuuid expected = getUUID >>= check
+			where
+				check u | u == toUUID expected = return ()
+				check NoUUID = unexpected "uninitialized repository"
+				check u = unexpected $ "UUID " ++ fromUUID u
+				unexpected s = error $
+					"expected repository UUID " ++
+					expected ++ " but found " ++ s
+
+header :: String
+header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
+
+run :: [String] -> IO ()
+run [] = failure
+-- skip leading -c options, passed by eg, ssh
+run ("-c":p) = run p
+-- a command can be either a builtin or something to pass to git-shell
+run c@(cmd:dir:params)
+	| cmd `elem` builtins = builtin cmd dir params
+	| otherwise = external c
+run c@(cmd:_)
+	-- Handle the case of being the user's login shell. It will be passed
+	-- a single string containing all the real parameters.
+	| "git-annex-shell " `isPrefixOf` cmd = run $ drop 1 $ shellUnEscape cmd
+	| cmd `elem` builtins = failure
+	| otherwise = external c
+
+builtins :: [String]
+builtins = map cmdname cmds
+
+builtin :: String -> String -> [String] -> IO ()
+builtin cmd dir params = do
+	checkNotReadOnly cmd
+	dispatch (cmd : filterparams params) cmds options header $
+		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
+
+external :: [String] -> IO ()
+external params = do
+	checkNotLimited
+	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
+		error "git-shell failed"
+
+-- Drop all args after "--".
+-- These tend to be passed by rsync and not useful.
+filterparams :: [String] -> [String]
+filterparams [] = []
+filterparams ("--":_) = []
+filterparams (a:as) = a:filterparams as
+
+failure :: IO ()
+failure = error $ "bad parameters\n\n" ++ usage header cmds options
+
+checkNotLimited :: IO ()
+checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED"
+
+checkNotReadOnly :: String -> IO ()
+checkNotReadOnly cmd
+	| cmd `elem` map cmdname cmds_readonly = return ()
+	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
+
+checkEnv :: String -> IO ()
+checkEnv var =
+	whenM (not . null <$> catchDefaultIO (getEnv var) "") $
+		error $ "Action blocked by " ++ var
diff --git a/Makefile b/Makefile
index fbdefc2726..ddb5e3ff6a 100644
--- a/Makefile
+++ b/Makefile
@@ -9,8 +9,8 @@ endif
 
 GHCMAKE=ghc $(GHCFLAGS) --make
 
-bins=git-annex git-annex-shell git-union-merge
-mans=git-annex.1 git-annex-shell.1 git-union-merge.1
+bins=git-annex
+mans=git-annex.1 git-annex-shell.1
 sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs
 
 all=$(bins) $(mans) docs
@@ -48,6 +48,7 @@ git-union-merge.1: doc/git-union-merge.mdwn
 install: all
 	install -d $(DESTDIR)$(PREFIX)/bin
 	install $(bins) $(DESTDIR)$(PREFIX)/bin
+	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
 	install -d $(DESTDIR)$(PREFIX)/share/man/man1
 	install -m 0644 $(mans) $(DESTDIR)$(PREFIX)/share/man/man1
 	install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex
diff --git a/debian/changelog b/debian/changelog
index 0f8fc001e1..cf957deb38 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120310) UNRELEASED; urgency=low
+git-annex (3.20120315) unstable; urgency=low
 
   * fsck: Fix up any broken links and misplaced content caused by the
     directory hash calculation bug fixed in the last release.
@@ -12,12 +12,15 @@ git-annex (3.20120310) UNRELEASED; urgency=low
   * Added annex.bloomcapacity and annex.bloomaccuracy, which can be
     adjusted as desired to tune the bloom filter.
   * status: Display amount of memory used by bloom filter, and
-    detect then it's too small for the number of keys in a repository.
+    detect when it's too small for the number of keys in a repository.
   * git-annex-shell: Runs hooks/annex-content after content is received
     or dropped.
   * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch.
+  * git-annex now behaves as git-annex-shell if symlinked to and run by that
+    name. The Makefile sets this up, saving some 8 mb of installed size.
+  * git-union-merge is a demo program, so it is no longer built by default.
 
- -- Joey Hess   Sat, 10 Mar 2012 14:03:22 -0400
+ -- Joey Hess   Thu, 15 Mar 2012 11:05:28 -0400
 
 git-annex (3.20120309) unstable; urgency=low
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
index 396b7b7905..08c1f9664d 100644
--- a/git-annex-shell.hs
+++ b/git-annex-shell.hs
@@ -1,117 +1,13 @@
 {- git-annex-shell main program
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
 import System.Environment
-import System.Console.GetOpt
 
-import Common.Annex
-import qualified Git.Construct
-import CmdLine
-import Command
-import Annex.UUID
-import qualified Option
-
-import qualified Command.ConfigList
-import qualified Command.InAnnex
-import qualified Command.DropKey
-import qualified Command.RecvKey
-import qualified Command.SendKey
-import qualified Command.Commit
-
-cmds_readonly :: [Command]
-cmds_readonly = concat
-	[ Command.ConfigList.def
-	, Command.InAnnex.def
-	, Command.SendKey.def
-	]
-
-cmds_notreadonly :: [Command]
-cmds_notreadonly = concat
-	[ Command.RecvKey.def
-	, Command.DropKey.def
-	, Command.Commit.def
-	]
-
-cmds :: [Command]
-cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly
-	where
-		adddirparam c = c
-			{ cmdparamdesc = "DIRECTORY " ++ cmdparamdesc c
-			}
-
-options :: [OptDescr (Annex ())]
-options = Option.common ++
-	[ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid"
-	]
-	where
-		checkuuid expected = getUUID >>= check
-			where
-				check u | u == toUUID expected = return ()
-				check NoUUID = unexpected "uninitialized repository"
-				check u = unexpected $ "UUID " ++ fromUUID u
-				unexpected s = error $
-					"expected repository UUID " ++
-					expected ++ " but found " ++ s
-
-header :: String
-header = "Usage: git-annex-shell [-c] command [parameters ...] [option ..]"
+import GitAnnexShell
 
 main :: IO ()
-main = main' =<< getArgs
-
-main' :: [String] -> IO ()
-main' [] = failure
--- skip leading -c options, passed by eg, ssh
-main' ("-c":p) = main' p
--- a command can be either a builtin or something to pass to git-shell
-main' c@(cmd:dir:params)
-	| cmd `elem` builtins = builtin cmd dir params
-	| otherwise = external c
-main' c@(cmd:_)
-	-- Handle the case of being the user's login shell. It will be passed
-	-- a single string containing all the real parameters.
-	| "git-annex-shell " `isPrefixOf` cmd = main' $ drop 1 $ shellUnEscape cmd
-	| cmd `elem` builtins = failure
-	| otherwise = external c
-
-builtins :: [String]
-builtins = map cmdname cmds
-
-builtin :: String -> String -> [String] -> IO ()
-builtin cmd dir params = do
-	checkNotReadOnly cmd
-	dispatch (cmd : filterparams params) cmds options header $
-		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
-
-external :: [String] -> IO ()
-external params = do
-	checkNotLimited
-	unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $
-		error "git-shell failed"
-
--- Drop all args after "--".
--- These tend to be passed by rsync and not useful.
-filterparams :: [String] -> [String]
-filterparams [] = []
-filterparams ("--":_) = []
-filterparams (a:as) = a:filterparams as
-
-failure :: IO ()
-failure = error $ "bad parameters\n\n" ++ usage header cmds options
-
-checkNotLimited :: IO ()
-checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED"
-
-checkNotReadOnly :: String -> IO ()
-checkNotReadOnly cmd
-	| cmd `elem` map cmdname cmds_readonly = return ()
-	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
-
-checkEnv :: String -> IO ()
-checkEnv var =
-	whenM (not . null <$> catchDefaultIO (getEnv var) "") $
-		error $ "Action blocked by " ++ var
+main = run =<< getArgs
diff --git a/git-annex.cabal b/git-annex.cabal
index 7d69b9a919..881e4d212b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120310
+Version: 3.20120315
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 
@@ -39,9 +39,6 @@ Executable git-annex-shell
   Main-Is: git-annex-shell.hs
   Other-Modules: Utility.StatFS
 
-Executable git-union-merge
-  Main-Is: git-union-merge.hs
-
 source-repository head
   type: git
   location: git://git-annex.branchable.com/
diff --git a/git-annex.hs b/git-annex.hs
index a53697cdbb..f5f2f22d74 100644
--- a/git-annex.hs
+++ b/git-annex.hs
@@ -1,13 +1,21 @@
 {- git-annex main program stub
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
 import System.Environment
+import System.FilePath
 
-import GitAnnex
+import qualified GitAnnex
+import qualified GitAnnexShell
 
 main :: IO ()
-main = run =<< getArgs
+main = run =<< getProgName
+	where
+		run n
+			| isshell n = go GitAnnexShell.run
+			| otherwise = go GitAnnex.run
+		isshell n = takeFileName n == "git-annex-shell"
+		go a = a =<< getArgs

From faf8cede245d19a9af3f62ea9fd576f3b2ba294c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Mar 2012 12:14:45 -0400
Subject: [PATCH 3273/8313] fix option order in usage display

---
 Usage.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Usage.hs b/Usage.hs
index 59223daae4..b1de930ef2 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -29,8 +29,8 @@ usage header cmds commonoptions = unlines $
 		-- be displayed after the command.
 		alloptlines = filter (not . null) $
 			lines $ usageInfo "" $
-				concatMap cmdoptions cmds ++ commonoptions
-		(cmdlines, optlines) = go (sort cmds) alloptlines []
+				concatMap cmdoptions scmds ++ commonoptions
+		(cmdlines, optlines) = go scmds alloptlines []
 		go [] os ls = (ls, os)
 		go (c:cs) os ls = go cs os' (ls++(l:o))
 			where
@@ -46,6 +46,7 @@ usage header cmds commonoptions = unlines $
 		namepad = pad $ longest cmdname + 1
 		descpad = pad $ longest cmdparamdesc + 2
 		longest f = foldl max 0 $ map (length . f) cmds
+		scmds = sort cmds
 
 {- Descriptions of params used in usage messages. -}
 paramPaths :: String

From d1e136193b74d332d6c00697dbc86e98848ab8ba Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Mar 2012 12:23:34 -0400
Subject: [PATCH 3274/8313] releasing version 3.20120315

---
 debian/control | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index a73433c2ac..d8ba09f7ce 100644
--- a/debian/control
+++ b/debian/control
@@ -26,7 +26,7 @@ Build-Depends:
 	rsync,
 	openssh-client,
 Maintainer: Joey Hess 
-Standards-Version: 3.9.2
+Standards-Version: 3.9.3
 Vcs-Git: git://git.kitenet.net/git-annex
 Homepage: http://git-annex.branchable.com/
 

From ff8b6c1bab519f243b67219cc2b43d037b3fa4e2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Mar 2012 12:25:20 -0400
Subject: [PATCH 3275/8313] add news item for git-annex 3.20120315

---
 doc/news/version_3.20120315.mdwn | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/news/version_3.20120315.mdwn

diff --git a/doc/news/version_3.20120315.mdwn b/doc/news/version_3.20120315.mdwn
new file mode 100644
index 0000000000..a3ccb4cf47
--- /dev/null
+++ b/doc/news/version_3.20120315.mdwn
@@ -0,0 +1,21 @@
+git-annex 3.20120315 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * fsck: Fix up any broken links and misplaced content caused by the
+     directory hash calculation bug fixed in the last release.
+   * sync: Sync to lower cost remotes first.
+   * status: Fixed to run in constant space.
+   * status: More accurate display of sizes of tmp and bad keys.
+   * unused: Now uses a bloom filter, and runs in constant space.
+     Use of a bloom filter does mean it will not notice a small
+     number of unused keys. For repos with up to half a million keys,
+     it will miss one key in 1000.
+   * Added annex.bloomcapacity and annex.bloomaccuracy, which can be
+     adjusted as desired to tune the bloom filter.
+   * status: Display amount of memory used by bloom filter, and
+     detect when it's too small for the number of keys in a repository.
+   * git-annex-shell: Runs hooks/annex-content after content is received
+     or dropped.
+   * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch.
+   * git-annex now behaves as git-annex-shell if symlinked to and run by that
+     name. The Makefile sets this up, saving some 8 mb of installed size.
+   * git-union-merge is a demo program, so it is no longer built by default."""]]
\ No newline at end of file

From c0c9991c9f5322aef05f4c97d2c3f3bdc3101e46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 15 Mar 2012 20:39:25 -0400
Subject: [PATCH 3276/8313] nukes another 15 lines thanks to ifM

---
 Remote/Directory.hs    | 12 +++---------
 Remote/Git.hs          | 43 ++++++++++++++++++++++--------------------
 Remote/Helper/Hooks.hs |  8 +++-----
 Remote/Hook.hs         | 12 ++++++------
 Remote/Rsync.hs        | 24 ++++++++++-------------
 Utility/Url.hs         | 32 ++++++++++++-------------------
 6 files changed, 57 insertions(+), 74 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 80c45a6913..ecbf511d64 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -100,11 +100,7 @@ withCheckedFiles _ _ [] _ _ = return False
 withCheckedFiles check Nothing d k a = go $ locations d k
 	where
 		go [] = return False
-		go (f:fs) = do
-			use <- check f
-			if use
-				then a [f]
-				else go fs
+		go (f:fs) = ifM (check f) ( a [f] , go fs )
 withCheckedFiles check (Just _) d k a = go $ locations d k
 	where
 		go [] = return False
@@ -115,10 +111,8 @@ withCheckedFiles check (Just _) d k a = go $ locations d k
 				then do
 					count <- readcount chunkcount
 					let chunks = take count $ chunkStream f
-					ok <- all id <$> mapM check chunks
-					if ok
-						then a chunks
-						else return False
+					ifM (all id <$> mapM check chunks)
+						( a chunks , return False )
 				else go fs
 		readcount f = fromMaybe (error $ "cannot parse " ++ f)
 			. (readish :: String -> Maybe Int)
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 12a7f18448..5c10c0fc9f 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -127,10 +127,11 @@ tryGitConfigRead r
 			Annex.changeState $ \s -> s { Annex.repo = g' }
 
 		exchange [] _ = []
-		exchange (old:ls) new =
-			if Git.remoteName old == Git.remoteName new
-				then new : exchange ls new
-				else old : exchange ls new
+		exchange (old:ls) new
+			| Git.remoteName old == Git.remoteName new =
+				new : exchange ls new
+			| otherwise =
+				old : exchange ls new
 
 {- Checks if a given remote has the content for a key inAnnex.
  - If the remote cannot be accessed, or if it cannot determine
@@ -227,11 +228,11 @@ copyFromRemoteCheap r key file
 	| not $ Git.repoIsUrl r = do
 		loc <- liftIO $ gitAnnexLocation key r
 		liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True
-	| Git.repoIsSsh r = do
-		ok <- Annex.Content.preseedTmp key file
-		if ok
-			then copyFromRemote r key file
-			else return False
+	| Git.repoIsSsh r =
+		ifM (Annex.Content.preseedTmp key file)
+			( copyFromRemote r key file
+			, return False
+			)
 	| otherwise = return False
 
 {- Tries to copy a key's content to a remote's annex. -}
@@ -254,22 +255,24 @@ copyToRemote r key
 rsyncHelper :: [CommandParam] -> Annex Bool
 rsyncHelper p = do
 	showOutput -- make way for progress bar
-	res <- liftIO $ rsync p
-	if res
-		then return res
-		else do
+	ifM (liftIO $ rsync p)
+		( return True
+		, do
 			showLongNote "rsync failed -- run git annex again to resume file transfer"
-			return res
+			return False
+		)
 
 {- Copys a file with rsync unless both locations are on the same
  - filesystem. Then cp could be faster. -}
 rsyncOrCopyFile :: [CommandParam] -> FilePath -> FilePath -> Annex Bool
-rsyncOrCopyFile rsyncparams src dest = do
-	ss <- liftIO $ getFileStatus $ parentDir src
-	ds <- liftIO $ getFileStatus $ parentDir dest
-	if deviceID ss == deviceID ds
-		then liftIO $ copyFileExternal src dest
-		else rsyncHelper $ rsyncparams ++ [Param src, Param dest]
+rsyncOrCopyFile rsyncparams src dest =
+	ifM (sameDeviceIds src dest)
+		( liftIO $ copyFileExternal src dest
+		, rsyncHelper $ rsyncparams ++ [Param src, Param dest]
+		)
+	where
+		sameDeviceIds a b = (==) <$> (getDeviceId a) <*> (getDeviceId b)
+		getDeviceId f = deviceID <$> liftIO (getFileStatus $ parentDir f)
 
 {- Generates rsync parameters that ssh to the remote and asks it
  - to either receive or send the key's content. -}
diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs
index 5929b17935..ed329b914c 100644
--- a/Remote/Helper/Hooks.hs
+++ b/Remote/Helper/Hooks.hs
@@ -84,10 +84,8 @@ runHooks r starthook stophook a = do
 			liftIO $ closeFd fd
 
 lookupHook :: Remote -> String -> Annex (Maybe String)
-lookupHook r n = do
-	command <- getConfig (repo r) hookname ""
-	if null command
-		then return Nothing
-		else return $ Just command
+lookupHook r n = go =<< getConfig (repo r) hookname ""
 	where
+		go "" = return Nothing
+		go command = return $ Just command
 		hookname =  n ++ "-command"
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index b37d5e2156..1e5c27b91f 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -89,13 +89,13 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 	where
 		run command = do
 			showOutput -- make way for hook output
-			res <- liftIO $ boolSystemEnv
-				"sh" [Param "-c", Param command] $ hookEnv k f
-			if res
-				then a
-				else do
+			ifM (liftIO $ boolSystemEnv
+				"sh" [Param "-c", Param command] $ hookEnv k f)
+				( a
+				, do
 					warning $ hook ++ " hook exited nonzero!"
-					return res
+					return False
+				)
 
 store :: String -> Key -> Annex Bool
 store h k = do
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 577ea0b049..03c9911d78 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -113,20 +113,16 @@ retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o
 	]
 
 retrieveCheap :: RsyncOpts -> Key -> FilePath -> Annex Bool
-retrieveCheap o k f = do
-	ok <- preseedTmp k f
-	if ok
-		then retrieve o k f
-		else return False
+retrieveCheap o k f = ifM (preseedTmp k f) ( retrieve o k f , return False )
 
 retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do
-	res <- retrieve o enck tmp
-	if res
-		then liftIO $ catchBoolIO $ do
+	ifM (retrieve o enck tmp)
+		( liftIO $ catchBoolIO $ do
 			withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f
 			return True
-		else return res
+		, return False
+		)
 
 remove :: RsyncOpts -> Key -> Annex Bool
 remove o k = withRsyncScratchDir $ \tmp -> liftIO $ do
@@ -188,12 +184,12 @@ withRsyncScratchDir a = do
 rsyncRemote :: RsyncOpts -> [CommandParam] -> Annex Bool
 rsyncRemote o params = do
 	showOutput -- make way for progress bar
-	res <- liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params
-	if res
-		then return res
-		else do
+	ifM (liftIO $ rsync $ rsyncOptions o ++ defaultParams ++ params)
+		( return True
+		, do
 			showLongNote "rsync failed -- run git annex again to resume file transfer"
-			return res
+			return False
+		)
 	where
 		defaultParams = [Params "--progress"]
 
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 8a43cf788d..8a8d732a34 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -14,15 +14,10 @@ module Utility.Url (
 	get
 ) where
 
-import Control.Applicative
-import Control.Monad
+import Common
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
-import Data.Maybe
-
-import Utility.SafeCommand
-import Utility.Path
 
 type URLString = String
 
@@ -47,7 +42,7 @@ exists url =
 				(2,_,_) -> return (True, size r)
 				_ -> return (False, Nothing)
 	where
-		size = liftM read . lookupHeader HdrContentLength . rspHeaders
+		size = liftM Prelude.read . lookupHeader HdrContentLength . rspHeaders
 
 canDownload :: IO Bool
 canDownload = (||) <$> inPath "wget" <*> inPath "curl"
@@ -60,20 +55,17 @@ canDownload = (||) <$> inPath "wget" <*> inPath "curl"
  - for only one in.
  -}
 download :: URLString -> [CommandParam] -> FilePath -> IO Bool
-download url options file = do
-	e <- inPath "wget"
-	if e
-		then
-			go "wget" [Params "-c -O", File file, File url]
-		else
-			-- Uses the -# progress display, because the normal
-			-- one is very confusing when resuming, showing
-			-- the remainder to download as the whole file,
-			-- and not indicating how much percent was
-			-- downloaded before the resume.
-			go "curl" [Params "-L -C - -# -o", File file, File url]
+download url options file = ifM (inPath "wget") (wget , curl)
 	where
-		go cmd opts = boolSystem cmd (options++opts)
+		wget = go "wget" [Params "-c -O"]
+		{- Uses the -# progress display, because the normal
+		 - one is very confusing when resuming, showing
+		 - the remainder to download as the whole file,
+		 - and not indicating how much percent was
+		 - downloaded before the resume. -}
+		curl = go "curl" [Params "-L -C - -# -o"]
+		go cmd opts = boolSystem cmd $
+			options++opts++[File file, File url]
 
 {- Downloads a small file. -}
 get :: URLString -> IO String

From 184a69171d5d983ee2f08cce28011d235f44cc5c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Mar 2012 01:59:07 -0400
Subject: [PATCH 3277/8313] removed another 10 lines via ifM

---
 Git.hs           | 24 ++++++++++----------
 Git/Branch.hs    | 10 ++++----
 Git/Config.hs    | 13 +++++------
 Git/Construct.hs | 59 ++++++++++++++++++++----------------------------
 Init.hs          | 10 ++++----
 Logs/Presence.hs |  7 +++---
 Logs/Remote.hs   |  9 ++++----
 Upgrade/V0.hs    | 13 ++++-------
 Upgrade/V1.hs    | 56 +++++++++++++++++++++++----------------------
 9 files changed, 95 insertions(+), 106 deletions(-)

diff --git a/Git.hs b/Git.hs
index 9b7dccfebd..4278e9fcf2 100644
--- a/Git.hs
+++ b/Git.hs
@@ -31,7 +31,6 @@ module Git (
 import qualified Data.Map as M
 import Data.Char
 import Network.URI (uriPath, uriScheme, unEscapeString)
-import System.Directory
 import System.Posix.Files
 
 import Common
@@ -83,11 +82,14 @@ repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
 repoIsLocalBare _ = False
 
 assertLocal :: Repo -> a -> a
-assertLocal repo action = 
-	if not $ repoIsUrl repo
-		then action
-		else error $ "acting on non-local git repo " ++  repoDescribe repo ++ 
-				" not supported"
+assertLocal repo action
+	| repoIsUrl repo = error $ unwords
+		[ "acting on non-local git repo"
+		, repoDescribe repo
+		, "not supported"
+		]
+	| otherwise = action
+
 configBare :: Repo -> Bool
 configBare repo = maybe unknown (fromMaybe False . configTrue) $
 	M.lookup "core.bare" $ config repo
@@ -113,12 +115,10 @@ gitDir repo
 hookPath :: String -> Repo -> IO (Maybe FilePath)
 hookPath script repo = do
 	let hook = gitDir repo  "hooks"  script
-	e <- doesFileExist hook
-	if e
-		then do
-			m <- fileMode <$> getFileStatus hook
-			return $ if isExecutable m then Just hook else Nothing
-		else return Nothing
+	ifM (catchBoolIO $ isexecutable hook)
+		( return $ Just hook , return Nothing )
+	where
+		isexecutable f = isExecutable . fileMode <$> getFileStatus f
 
 {- Path to a repository's --work-tree, that is, its top.
  -
diff --git a/Git/Branch.hs b/Git/Branch.hs
index cd91882283..6edc1c306d 100644
--- a/Git/Branch.hs
+++ b/Git/Branch.hs
@@ -41,14 +41,14 @@ changed origbranch newbranch repo
  -}
 fastForward :: Branch -> [Ref] -> Repo -> IO Bool
 fastForward _ [] _ = return True
-fastForward branch (first:rest) repo = do
+fastForward branch (first:rest) repo =
 	-- First, check that the branch does not contain any
 	-- new commits that are not in the first ref. If it does,
 	-- cannot fast-forward.
-	diverged <- changed first branch repo
-	if diverged
-		then no_ff
-		else maybe no_ff do_ff =<< findbest first rest
+	ifM (changed first branch repo)
+		( no_ff
+		, maybe no_ff do_ff =<< findbest first rest
+		)
 	where
 		no_ff = return False
 		do_ff to = do
diff --git a/Git/Config.hs b/Git/Config.hs
index 0d73a0b9a5..8190a62ad3 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -26,16 +26,15 @@ getMaybe key repo = M.lookup key (config repo)
 
 {- Runs git config and populates a repo with its config. -}
 read :: Repo -> IO Repo
-read repo@(Repo { location = Dir d }) = do
+read repo@(Repo { location = Dir d }) = bracketcd d $
 	{- Cannot use pipeRead because it relies on the config having
 	   been already read. Instead, chdir to the repo. -}
-	cwd <- getCurrentDirectory
-	if dirContains d cwd
-		then go
-		else bracket_ (changeWorkingDirectory d) (changeWorkingDirectory cwd) go
+	pOpen ReadFromPipe "git" ["config", "--null", "--list"] $ hRead repo
 	where
-		go = pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
-			hRead repo
+		bracketcd to a = bracketcd' to a =<< getCurrentDirectory
+		bracketcd' to a cwd 
+			| dirContains to cwd = a
+			| otherwise = bracket_ (changeWorkingDirectory to) (changeWorkingDirectory cwd) a
 read r = assertLocal r $
 	error $ "internal error; trying to read config of " ++ show r
 
diff --git a/Git/Construct.hs b/Git/Construct.hs
index ef6094a21d..49905f818b 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -69,27 +69,25 @@ fromPath dir = fromAbsPath =<< absPath dir
  - specified. -}
 fromAbsPath :: FilePath -> IO Repo
 fromAbsPath dir
-	| "/" `isPrefixOf` dir = do
- 		-- Git always looks for "dir.git" in preference to
-		-- to "dir", even if dir ends in a "/".
-		let canondir = dropTrailingPathSeparator dir
-		let dir' = canondir ++ ".git"
-		e <- doesDirectoryExist dir'
-		if e
-			then ret dir'
-			else if "/.git" `isSuffixOf` canondir
-				then do
-					-- When dir == "foo/.git", git looks
-					-- for "foo/.git/.git", and failing
-					-- that, uses "foo" as the repository.
-					e' <- doesDirectoryExist $ dir  ".git"
-					if e'
-						then ret dir
-						else ret $ takeDirectory canondir
-				else ret dir
-	| otherwise = error $ "internal error, " ++ dir ++ " is not absolute"
+	| "/" `isPrefixOf` dir =
+		ifM (doesDirectoryExist dir') ( ret dir' , hunt )
+	| otherwise =
+		error $ "internal error, " ++ dir ++ " is not absolute"
 	where
 		ret = newFrom . Dir
+ 		{- Git always looks for "dir.git" in preference to
+		 - to "dir", even if dir ends in a "/". -}
+		canondir = dropTrailingPathSeparator dir
+		dir' = canondir ++ ".git"
+		{- When dir == "foo/.git", git looks for "foo/.git/.git",
+		 - and failing that, uses "foo" as the repository. -}
+		hunt
+			| "/.git" `isSuffixOf` canondir =
+				ifM (doesDirectoryExist $ dir  ".git")
+					( ret dir
+					, ret $ takeDirectory canondir
+					)
+			| otherwise = ret dir
 
 {- Remote Repo constructor. Throws exception on invalid url.
  -
@@ -229,27 +227,20 @@ expandTilde = expandt True
 			| otherwise = findname (n++[c]) cs
 
 seekUp :: (FilePath -> IO Bool) -> FilePath -> IO (Maybe FilePath)
-seekUp want dir = do
-	ok <- want dir
-	if ok
-		then return $ Just dir
-		else case parentDir dir of
+seekUp want dir =
+	ifM (want dir)
+		( return $ Just dir
+		, case parentDir dir of
 			"" -> return Nothing
 			d -> seekUp want d
+		)
 
 isRepoTop :: FilePath -> IO Bool
-isRepoTop dir = do
-	r <- isRepo
-	if r
-		then return r
-		else isBareRepo
+isRepoTop dir = ifM isRepo ( return True , isBareRepo )
 	where
 		isRepo = gitSignature (".git"  "config")
-		isBareRepo = do
-			e <- doesDirectoryExist (dir  "objects")
-			if not e
-				then return e
-				else gitSignature "config"
+		isBareRepo = ifM (doesDirectoryExist $ dir  "objects")
+			( gitSignature "config" , return False )
 		gitSignature file = doesFileExist (dir  file)
 
 newFrom :: RepoLocation -> IO Repo
diff --git a/Init.hs b/Init.hs
index f3d8bd0171..9f1988a394 100644
--- a/Init.hs
+++ b/Init.hs
@@ -58,13 +58,13 @@ gitPreCommitHookWrite = unlessBare $ do
 gitPreCommitHookUnWrite :: Annex ()
 gitPreCommitHookUnWrite = unlessBare $ do
 	hook <- preCommitHook
-	whenM (liftIO $ doesFileExist hook) $ do
-		c <- liftIO $ readFile hook
-		if c == preCommitScript
-			then liftIO $ removeFile hook
-			else warning $ "pre-commit hook (" ++ hook ++ 
+	whenM (liftIO $ doesFileExist hook) $
+		ifM (liftIO $ (==) preCommitScript <$> readFile hook)
+			( liftIO $ removeFile hook
+			, warning $ "pre-commit hook (" ++ hook ++ 
 				") contents modified; not deleting." ++
 				" Edit it to remove call to git annex."
+			)
 
 unlessBare :: Annex () -> Annex ()
 unlessBare = unlessM $ fromRepo Git.repoIsLocalBare
diff --git a/Logs/Presence.hs b/Logs/Presence.hs
index 372af37d5d..933426718b 100644
--- a/Logs/Presence.hs
+++ b/Logs/Presence.hs
@@ -99,10 +99,9 @@ type LogMap = M.Map String LogLine
 {- Inserts a log into a map of logs, if the log has better (ie, newer)
  - information than the other logs in the map -}
 mapLog :: LogLine -> LogMap -> LogMap
-mapLog l m = 
-	if better
-		then M.insert i l m
-		else m
+mapLog l m
+	| better = M.insert i l m
+	| otherwise = m
 	where
 		better = maybe True newer $ M.lookup i m
 		newer l' = date l' <= date l
diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index ccfb4bb31a..5c9d67df03 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -72,14 +72,15 @@ configUnEscape = unescape
 		unescape (c:rest)
 			| c == '&' = entity rest
 			| otherwise = c : unescape rest
-		entity s = if ok
-				then chr (Prelude.read num) : unescape rest
-				else '&' : unescape s
+		entity s
+			| not (null num) && ";" `isPrefixOf` r =
+				chr (Prelude.read num) : unescape rest
+			| otherwise =
+				'&' : unescape s
 			where
 				num = takeWhile isNumber s
 				r = drop (length num) s
 				rest = drop 1 r
-				ok = not (null num) && ";" `isPrefixOf` r
 
 {- for quickcheck -}
 prop_idempotent_configEscape :: String -> Bool
diff --git a/Upgrade/V0.hs b/Upgrade/V0.hs
index c439c7caa2..8f3af337ed 100644
--- a/Upgrade/V0.hs
+++ b/Upgrade/V0.hs
@@ -35,14 +35,11 @@ lookupFile0 :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile0 = Upgrade.V1.lookupFile1
 
 getKeysPresent0 :: FilePath -> Annex [Key]
-getKeysPresent0 dir = do
-	exists <- liftIO $ doesDirectoryExist dir
-	if not exists
-		then return []
-		else do
-			contents <- liftIO $ getDirectoryContents dir
-			files <- liftIO $ filterM present contents
-			return $ map fileKey0 files
+getKeysPresent0 dir = ifM (liftIO $ doesDirectoryExist dir)
+	( liftIO $ map fileKey0
+		<$> (filterM present =<< getDirectoryContents dir)
+	, return []
+	)
 	where
 		present d = do
 			result <- tryIO $
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index ca2bff6617..62e3b3b313 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -50,18 +50,18 @@ upgrade :: Annex Bool
 upgrade = do
 	showAction "v1 to v2"
 	
-	bare <- fromRepo Git.repoIsLocalBare
-	if bare
-		then do
+	ifM (fromRepo Git.repoIsLocalBare)
+		( do
 			moveContent
 			setVersion
-		else do
+		, do
 			moveContent
 			updateSymlinks
 			moveLocationLogs
 	
 			Annex.Queue.flush True
 			setVersion
+		)
 	
 	Upgrade.V2.upgrade
 
@@ -104,12 +104,11 @@ moveLocationLogs = do
 		where
 			oldlocationlogs = do
 				dir <- fromRepo Upgrade.V2.gitStateDir
-				exists <- liftIO $ doesDirectoryExist dir
-				if exists
-					then do
-						contents <- liftIO $ getDirectoryContents dir
-						return $ mapMaybe oldlog2key contents
-					else return []
+				ifM (liftIO $ doesDirectoryExist dir)
+					( mapMaybe oldlog2key
+						<$> (liftIO $ getDirectoryContents dir)
+					, return []
+					)
 			move (l, k) = do
 				dest <- fromRepo $ logFile2 k
 				dir <- fromRepo Upgrade.V2.gitStateDir
@@ -127,14 +126,13 @@ moveLocationLogs = do
 				Annex.Queue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
 		
 oldlog2key :: FilePath -> Maybe (FilePath, Key)
-oldlog2key l = 
-	let len = length l - 4 in
-		if drop len l == ".log"
-		then let k = readKey1 (take len l) in
-			if null (keyName k) || null (keyBackendName k)
-			then Nothing
-			else Just (l, k)
-		else Nothing
+oldlog2key l
+	| drop len l == ".log" && sane = Just (l, k)
+	| otherwise = Nothing
+	where
+		len = length l - 4
+		k = readKey1 (take len l)
+		sane = (not . null $ keyName k) && (not . null $ keyBackendName k)
 
 -- WORM backend keys: "WORM:mtime:size:filename"
 -- all the rest: "backend:key"
@@ -143,10 +141,14 @@ oldlog2key l =
 -- v2 and v1; that infelicity is worked around by treating the value
 -- as the v2 key that it is.
 readKey1 :: String -> Key
-readKey1 v = 
-	if mixup
-		then fromJust $ readKey $ join ":" $ Prelude.tail bits
-		else Key { keyName = n , keyBackendName = b, keySize = s, keyMtime = t }
+readKey1 v
+	| mixup = fromJust $ readKey $ join ":" $ Prelude.tail bits
+	| otherwise = Key
+		{ keyName = n
+		, keyBackendName = b
+		, keySize = s
+		, keyMtime = t
+		}
 	where
 		bits = split ":" v
 		b = Prelude.head bits
@@ -205,14 +207,14 @@ lookupFile1 file = do
 getKeyFilesPresent1 :: Annex [FilePath]
 getKeyFilesPresent1  = getKeyFilesPresent1' =<< fromRepo gitAnnexObjectDir
 getKeyFilesPresent1' :: FilePath -> Annex [FilePath]
-getKeyFilesPresent1' dir = do
-	exists <- liftIO $ doesDirectoryExist dir
-	if not exists
-		then return []
-		else do
+getKeyFilesPresent1' dir =
+	ifM (liftIO $ doesDirectoryExist dir)
+		(  do
 			dirs <- liftIO $ getDirectoryContents dir
 			let files = map (\d -> dir ++ "/" ++ d ++ "/" ++ takeFileName d) dirs
 			liftIO $ filterM present files
+		, return []
+		)
 	where
 		present f = do
 			result <- tryIO $ getFileStatus f

From b06336fa3a146e9a0ef1a1307b1fc219570795c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Mar 2012 02:12:56 -0400
Subject: [PATCH 3278/8313] simplify

---
 Utility/Exception.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Utility/Exception.hs b/Utility/Exception.hs
index 7b6c9c999f..a6726945cb 100644
--- a/Utility/Exception.hs
+++ b/Utility/Exception.hs
@@ -25,10 +25,7 @@ catchDefaultIO a def = catchIO a (const $ return def)
 
 {- Catches IO errors and returns the error message. -}
 catchMsgIO :: IO a -> IO (Either String a)
-catchMsgIO a = dispatch <$> tryIO a
-	where
-		dispatch (Left e) = Left $ show e
-		dispatch (Right v) = Right v
+catchMsgIO a = either (Left . show) Right <$> tryIO a
 
 {- catch specialized for IO errors only -}
 catchIO :: IO a -> (IOException -> IO a) -> IO a

From 771052a85e3a000911e8a438012e61b3caf9c1a8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Mar 2012 12:28:17 -0400
Subject: [PATCH 3279/8313] optimize monadic ||

(||) used applicative style runs both conditions rather than short
circuiting. Add an orM that properly short-circuits.
---
 Command/Find.hs    |  2 +-
 Utility/Matcher.hs |  6 +++---
 Utility/Monad.hs   | 12 +++++++++++-
 Utility/Url.hs     |  4 ----
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index 33f512e39f..f5bd2734b4 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -42,7 +42,7 @@ start :: Maybe Utility.Format.Format -> FilePath -> (Key, Backend) -> CommandSta
 start format file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (liftM2 (||) limited (inAnnex key)) $
+	whenM (orM limited (inAnnex key)) $
 		unlessM (showFullJSON vars) $
 			case format of
 				Nothing -> liftIO $ putStrLn file
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 01500a2111..71e1e17f4c 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -25,7 +25,7 @@ module Utility.Matcher (
 	matchesAny
 ) where
 
-import Control.Monad
+import Common
 
 {- A Token can be an Operation of an arbitrary type, or one of a few
  - predefined peices of syntax. -}
@@ -87,8 +87,8 @@ matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool
 matchM m v = go m
 	where
 		go MAny = return True
-		go (MAnd m1 m2) = liftM2 (&&) (go m1) (go m2)
-		go (MOr m1 m2) =  liftM2 (||) (go m1) (go m2)
+		go (MAnd m1 m2) = andM (go m1) (go m2)
+		go (MOr m1 m2) =  orM  (go m1) (go m2)
 		go (MNot m1) = liftM not (go m1)
 		go (MOp o) = o v
 
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 23c0c4c194..5cc2432903 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -8,7 +8,7 @@
 module Utility.Monad where
 
 import Data.Maybe
-import Control.Monad (liftM)
+import Control.Monad (liftM, liftM2)
 
 {- Return the first value from a list, if any, satisfying the given
  - predicate -}
@@ -31,6 +31,16 @@ ifM cond (thenclause, elseclause) = do
 	c <- cond
 	if c then thenclause else elseclause
 
+{- monadic ||
+ -
+ - Compare with (||) <$> ma <*> mb, which always runs both ma and mb. -}
+orM :: Monad m => m Bool -> m Bool -> m Bool
+orM ma mb = ifM ma ( return True , mb )
+
+{- monadic &&  (for completeness) -}
+andM :: Monad m => m Bool -> m Bool -> m Bool
+andM  = liftM2 (&&)
+
 {- Runs an action, passing its value to an observer before returning it. -}
 observe :: Monad m => (a -> m b) -> m a -> m a
 observe observer a = do
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 8a8d732a34..86d66d83b5 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -9,7 +9,6 @@ module Utility.Url (
 	URLString,
 	check,
 	exists,
-	canDownload,
 	download,
 	get
 ) where
@@ -44,9 +43,6 @@ exists url =
 	where
 		size = liftM Prelude.read . lookupHeader HdrContentLength . rspHeaders
 
-canDownload :: IO Bool
-canDownload = (||) <$> inPath "wget" <*> inPath "curl"
-
 {- Used to download large files, such as the contents of keys.
  -
  - Uses wget or curl program for its progress bar. (Wget has a better one,

From 28698e54aff0346221c8ecb625364e79c784d97b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Mar 2012 16:00:08 -0400
Subject: [PATCH 3280/8313] document rsyncurl setting

---
 doc/git-annex.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 10899d12cd..3ee2579c30 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -713,6 +713,12 @@ Here are all the supported configuration settings.
   Default ssh, rsync, wget/curl, and bup options to use if a remote does not
   have specific options.
 
+* `remote..rsyncurl`
+
+  Used by rsunc special remotes, this configures
+  the location of the rsync repository to use. Normally this is automaticaly
+  set up by `git annex initremote`, but you can change it if needed.
+
 * `remote..buprepo`
 
   Used by bup special remotes, this configures

From d6624b6c798df401eb9e715810537d2b93935a76 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 16 Mar 2012 16:03:04 -0400
Subject: [PATCH 3281/8313] typo

---
 doc/git-annex.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 3ee2579c30..dd94ccc0c3 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -649,7 +649,7 @@ Here are all the supported configuration settings.
   A command to run when git-annex begins to use the remote. This can
   be used to, for example, mount the directory containing the remote.
 
-  The command may be run repeatedly in multiple git-annex processes
+  The command may be run repeatedly when multiple git-annex processes
   are running concurrently.
 
 * `remote..annex-stop-command`

From a362c46b70c45872ff8c479ba5a6716cf13cc8d8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 17 Mar 2012 00:22:05 -0400
Subject: [PATCH 3282/8313] fun with symbols

Nothing at all on hackage is using <&&> or <||>.

(Also, <&&> should short-circuit on failure.)
---
 Command/Find.hs    |  2 +-
 Remote/Git.hs      |  9 +++------
 Utility/Matcher.hs |  8 ++++----
 Utility/Monad.hs   | 16 +++++++---------
 4 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/Command/Find.hs b/Command/Find.hs
index f5bd2734b4..e568c35106 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -42,7 +42,7 @@ start :: Maybe Utility.Format.Format -> FilePath -> (Key, Backend) -> CommandSta
 start format file (key, _) = do
 	-- only files inAnnex are shown, unless the user has requested
 	-- others via a limit
-	whenM (orM limited (inAnnex key)) $
+	whenM (limited <||> inAnnex key) $
 		unlessM (showFullJSON vars) $
 			case format of
 				Nothing -> liftIO $ putStrLn file
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 5c10c0fc9f..3725edd3a2 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -7,8 +7,8 @@
 
 module Remote.Git (remote, repoAvail) where
 
-import Control.Exception.Extensible
 import qualified Data.Map as M
+import Control.Exception.Extensible
 
 import Common.Annex
 import Utility.CopyFile
@@ -102,11 +102,8 @@ tryGitConfigRead r
 	where
 		-- Reading config can fail due to IO error or
 		-- for other reasons; catch all possible exceptions.
-		safely a = do
-			result <- liftIO (try a :: IO (Either SomeException Git.Repo))
-			case result of
-				Left _ -> return r
-				Right r' -> return r'
+		safely a = either (const $ return r) return
+				=<< liftIO (try a :: IO (Either SomeException Git.Repo))
 
 		pipedconfig cmd params = safely $
 			pOpen ReadFromPipe cmd (toCommand params) $
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index 71e1e17f4c..9b60057674 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -78,8 +78,8 @@ match a m v = go m
 	where
 		go MAny = True
 		go (MAnd m1 m2) = go m1 && go m2
-		go (MOr m1 m2) = go m1 || go m2
-		go (MNot m1) = not (go m1)
+		go (MOr m1 m2) =  go m1 || go m2
+		go (MNot m1) = not $ go m1
 		go (MOp o) = a o v
 
 {- Runs a monadic Matcher, where Operations are actions in the monad. -}
@@ -87,8 +87,8 @@ matchM :: Monad m => Matcher (v -> m Bool) -> v -> m Bool
 matchM m v = go m
 	where
 		go MAny = return True
-		go (MAnd m1 m2) = andM (go m1) (go m2)
-		go (MOr m1 m2) =  orM  (go m1) (go m2)
+		go (MAnd m1 m2) = go m1 <&&> go m2
+		go (MOr m1 m2) =  go m1 <||> go m2
 		go (MNot m1) = liftM not (go m1)
 		go (MOp o) = o v
 
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 5cc2432903..9c85d31ca8 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -8,7 +8,7 @@
 module Utility.Monad where
 
 import Data.Maybe
-import Control.Monad (liftM, liftM2)
+import Control.Monad (liftM)
 
 {- Return the first value from a list, if any, satisfying the given
  - predicate -}
@@ -31,15 +31,13 @@ ifM cond (thenclause, elseclause) = do
 	c <- cond
 	if c then thenclause else elseclause
 
-{- monadic ||
- -
- - Compare with (||) <$> ma <*> mb, which always runs both ma and mb. -}
-orM :: Monad m => m Bool -> m Bool -> m Bool
-orM ma mb = ifM ma ( return True , mb )
+{- short-circuiting monadic || -}
+(<||>) :: Monad m => m Bool -> m Bool -> m Bool
+ma <||> mb = ifM ma ( return True , mb )
 
-{- monadic &&  (for completeness) -}
-andM :: Monad m => m Bool -> m Bool -> m Bool
-andM  = liftM2 (&&)
+{- short-circuiting monadic && -}
+(<&&>) :: Monad m => m Bool -> m Bool -> m Bool
+ma <&&> mb = ifM ma ( mb , return False )
 
 {- Runs an action, passing its value to an observer before returning it. -}
 observe :: Monad m => (a -> m b) -> m a -> m a

From 27df491bae5d725e71e7eded4d3e37e7aa4ddec9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
 
Date: Mon, 19 Mar 2012 18:06:30 +0000
Subject: [PATCH 3283/8313]

---
 doc/forum/Sharing_annex_with_local_clones.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/Sharing_annex_with_local_clones.mdwn

diff --git a/doc/forum/Sharing_annex_with_local_clones.mdwn b/doc/forum/Sharing_annex_with_local_clones.mdwn
new file mode 100644
index 0000000000..756075ef90
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones.mdwn
@@ -0,0 +1 @@
+Hi, is there any particular problem with symlinking one .git/annex to share between multiple repos?

From a90c34bdd78635177e1bfbc8bb2707d0be19fbea Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 19 Mar 2012 18:23:13 +0000
Subject: [PATCH 3284/8313] Added a comment: don't do that

---
 ...mment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment

diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment b/doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment
new file mode 100644
index 0000000000..892bc50f52
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_1_2b60e13e5f7b8cee56cf2ddc6c47f64d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="don't do that"
+ date="2012-03-19T18:23:13Z"
+ content="""
+Suppose you do that to repos A and B. Now, in A, you `git annex drop` a file that is only present in those repositories. A checks B to make sure it still has a copy of the file. It sees the (same) file there, so assumes it's safe to drop. The file is removed from A, also removing it from B, and losing data.
+
+It is possible to configure A and B to mutually distrust one-another and avoid this problem, but there will be other problems too.
+
+Instead, git-annex supports using `cp --reflink=auto`, which on filesystems supporting Copy On Write (eg, btrfs), avoids duplicating contents when A and B are on the same filesystem.
+"""]]

From bd3dfbd64b222288449d68859df5b2fdee146a1e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
 
Date: Mon, 19 Mar 2012 18:46:14 +0000
Subject: [PATCH 3285/8313] Added a comment

---
 .../comment_2_24ff2c1eb643077daa37c01644cebcd2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment

diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment b/doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment
new file mode 100644
index 0000000000..88f495da63
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_2_24ff2c1eb643077daa37c01644cebcd2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
+ nickname="Bryon"
+ subject="comment 2"
+ date="2012-03-19T18:46:13Z"
+ content="""
+Ah, OK. Is there a configuration step to set this up, or is this included magic in a new enough git-annex client?
+"""]]

From 3126dfc45e96c14a5844000a5aba6021f03ff30d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
 
Date: Mon, 19 Mar 2012 18:55:03 +0000
Subject: [PATCH 3286/8313] Added a comment

---
 .../comment_3_5359b8eada24d27be83214ac0ae62f23._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment

diff --git a/doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment b/doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment
new file mode 100644
index 0000000000..3e5fd11545
--- /dev/null
+++ b/doc/forum/Sharing_annex_with_local_clones/comment_3_5359b8eada24d27be83214ac0ae62f23._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnhsaESlYphzLTzpJy5IxxGFxxctIhWYfo"
+ nickname="Bryon"
+ subject="comment 3"
+ date="2012-03-19T18:55:03Z"
+ content="""
+Nevermind, found it. (git-annex 0.08)
+"""]]

From 181d2ccd20a41b1785569acb3efb76deb8cbdf00 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Mar 2012 21:21:20 -0400
Subject: [PATCH 3287/8313] Improve detection of inability to check free disk
 space.

Don't check if configure indicated checks won't work. This should fix a
FTBFS on mipsel, where configure correctly detects the checks won't work,
while garbage is returned for disk space info at git-annex runtime. It also
means that, when built via cabal, disk space checks are not enabled,
unfortunatly.
---
 Annex/Content.hs    | 17 ++++++++++-------
 Build/Configure.hs  | 17 +++++++++++++++--
 Build/TestConfig.hs |  8 +++++++-
 Setup.hs            |  2 +-
 configure.hs        |  8 +++++---
 debian/changelog    |  6 ++++++
 6 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index fad5f51349..6bf5391df3 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -45,6 +45,7 @@ import Utility.DataUnits
 import Utility.CopyFile
 import Config
 import Annex.Exception
+import qualified Build.SysConfig
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
@@ -178,13 +179,14 @@ checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
 	g <- gitRepo
 	r <- getConfig g "diskreserve" ""
+	sanitycheck r
 	let reserve = fromMaybe megabyte $ readSize dataUnits r
 	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
-	sanitycheck r stats
-	case (stats, keySize key) of
-		(Nothing, _) -> return ()
-		(_, Nothing) -> return ()
-		(Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
+	case (cancheck, stats, keySize key) of
+		(False, _, _) -> return ()
+		(_, Nothing, _) -> return ()
+		(_, _, Nothing) -> return ()
+		(_, Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
 			when (need + reserve > have + adjustment) $
 				needmorespace (need + reserve - have - adjustment)
 	where
@@ -195,8 +197,8 @@ checkDiskSpace' adjustment key = do
 				roughSize storageUnits True n ++
 				" more" ++ forcemsg
 		forcemsg = " (use --force to override this check or adjust annex.diskreserve)"
-		sanitycheck r stats
-			| not (null r) && isNothing stats = do
+		sanitycheck r
+			| not (null r) && not cancheck = do
 				unlessM (Annex.getState Annex.force) $
 					error $ "You have configured a diskreserve of "
 						++ r ++
@@ -204,6 +206,7 @@ checkDiskSpace' adjustment key = do
 						++ forcemsg
 				return ()
 			| otherwise = return ()
+		cancheck = Build.SysConfig.statfs_sanity_checked == Just True
 
 {- Moves a file into .git/annex/objects/
  -
diff --git a/Build/Configure.hs b/Build/Configure.hs
index 341b8840dc..14667ba860 100644
--- a/Build/Configure.hs
+++ b/Build/Configure.hs
@@ -10,8 +10,12 @@ import Control.Applicative
 import Build.TestConfig
 import Utility.SafeCommand
 
-tests :: [TestCase]
-tests =
+tests :: Bool -> [TestCase]
+tests True = cabaltests ++ common
+tests False = common
+
+common :: [TestCase]
+common =
 	[ TestCase "version" getVersion
 	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
 	, TestCase "git version" getGitVersion
@@ -28,6 +32,11 @@ tests =
 	, TestCase "ssh connection caching" getSshConnectionCaching
 	] ++ shaTestCases [1, 256, 512, 224, 384]
 
+cabaltests :: [TestCase]
+cabaltests =
+	[ TestCase "StatFS" testStatFSDummy
+	]
+
 shaTestCases :: [Int] -> [TestCase]
 shaTestCases l = map make l
 	where make n =
@@ -72,6 +81,10 @@ getSshConnectionCaching :: Test
 getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$>
 	boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"]
 
+testStatFSDummy :: Test
+testStatFSDummy = 
+	return $ Config "statfs_sanity_checked" $ MaybeBoolConfig Nothing
+
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
 cabalSetup = do
diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs
index e8a0d13368..0cc2019cfe 100644
--- a/Build/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -10,7 +10,8 @@ type ConfigKey = String
 data ConfigValue =
 	BoolConfig Bool |
 	StringConfig String |
-	MaybeStringConfig (Maybe String)
+	MaybeStringConfig (Maybe String) |
+	MaybeBoolConfig (Maybe Bool)
 data Config = Config ConfigKey ConfigValue
 
 type Test = IO Config
@@ -21,6 +22,7 @@ instance Show ConfigValue where
 	show (BoolConfig b) = show b
 	show (StringConfig s) = show s
 	show (MaybeStringConfig s) = show s
+	show (MaybeBoolConfig s) = show s
 
 instance Show Config where
 	show (Config key value) = unlines
@@ -31,6 +33,7 @@ instance Show Config where
 			valuetype (BoolConfig _) = "Bool"
 			valuetype (StringConfig _) = "String"
 			valuetype (MaybeStringConfig _) = "Maybe String"
+			valuetype (MaybeBoolConfig _) = "Maybe Bool"
 
 writeSysConfig :: [Config] -> IO ()
 writeSysConfig config = writeFile "Build/SysConfig.hs" body
@@ -109,6 +112,9 @@ testEnd (Config _ (BoolConfig False)) = status "no"
 testEnd (Config _ (StringConfig s)) = status s
 testEnd (Config _ (MaybeStringConfig (Just s))) = status s
 testEnd (Config _ (MaybeStringConfig Nothing)) = status "not available"
+testEnd (Config _ (MaybeBoolConfig (Just True))) = status "yes"
+testEnd (Config _ (MaybeBoolConfig (Just False))) = status "no"
+testEnd (Config _ (MaybeBoolConfig Nothing)) = status "unknown"
 
 status :: String -> IO ()
 status s = putStrLn $ ' ':s
diff --git a/Setup.hs b/Setup.hs
index 14e6a4ea71..239a6217f0 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -8,5 +8,5 @@ import qualified Build.Configure as Configure
 main = defaultMainWithHooks simpleUserHooks { preConf = configure }
 
 configure _ _ = do
-	Configure.run Configure.tests
+	Configure.run $ Configure.tests True
 	return (Nothing, [])
diff --git a/configure.hs b/configure.hs
index 3fb0671e78..6fdc5fcb03 100644
--- a/configure.hs
+++ b/configure.hs
@@ -8,14 +8,16 @@ import Utility.StatFS
 
 tests :: [TestCase]
 tests = [ TestCase "StatFS" testStatFS
-	] ++ Configure.tests
+	] ++ Configure.tests False
 
 {- This test cannot be included in Build.Configure due to needing
- - Utility/StatFS.hs to be built. -}
+ - Utility/StatFS.hs to be built, which it is not when "cabal configure"
+ - is run. -}
 testStatFS :: Test
 testStatFS = do
 	s <- getFileSystemStats "."
-	return $ Config "statfs_sane" $ BoolConfig $ isJust s
+	return $ Config "statfs_sanity_checked" $
+		MaybeBoolConfig $ Just $ isJust s
 
 main :: IO ()
 main = Configure.run tests
diff --git a/debian/changelog b/debian/changelog
index cf957deb38..cf732ab34d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120316) UNRELEASED; urgency=low
+
+  * Improve detection of inability to check free disk space.
+
+ -- Joey Hess   Wed, 21 Mar 2012 21:19:16 -0400
+
 git-annex (3.20120315) unstable; urgency=low
 
   * fsck: Fix up any broken links and misplaced content caused by the

From 188e2edc41551fa145d6cb8b36838fcb85132088 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Mar 2012 21:55:02 -0400
Subject: [PATCH 3288/8313] status: Prints available local disk space, or shows
 if git-annex doesn't know.

---
 Annex/Content.hs  | 18 ++----------------
 Command/Status.hs | 21 +++++++++++++++++++++
 Config.hs         | 21 +++++++++++++++++++++
 debian/changelog  |  2 ++
 4 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 6bf5391df3..1794fb5d93 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -177,11 +177,8 @@ checkDiskSpace = checkDiskSpace' 0
 
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
-	g <- gitRepo
-	r <- getConfig g "diskreserve" ""
-	sanitycheck r
-	let reserve = fromMaybe megabyte $ readSize dataUnits r
-	stats <- liftIO $ getFileSystemStats (gitAnnexDir g)
+	reserve <- getDiskReserve True
+	stats <- inRepo $ getFileSystemStats .gitAnnexDir
 	case (cancheck, stats, keySize key) of
 		(False, _, _) -> return ()
 		(_, Nothing, _) -> return ()
@@ -190,22 +187,11 @@ checkDiskSpace' adjustment key = do
 			when (need + reserve > have + adjustment) $
 				needmorespace (need + reserve - have - adjustment)
 	where
-		megabyte :: Integer
-		megabyte = 1000000
 		needmorespace n = unlessM (Annex.getState Annex.force) $
 			error $ "not enough free space, need " ++ 
 				roughSize storageUnits True n ++
 				" more" ++ forcemsg
 		forcemsg = " (use --force to override this check or adjust annex.diskreserve)"
-		sanitycheck r
-			| not (null r) && not cancheck = do
-				unlessM (Annex.getState Annex.force) $
-					error $ "You have configured a diskreserve of "
-						++ r ++
-						" but disk space checking is not working"
-						++ forcemsg
-				return ()
-			| otherwise = return ()
 		cancheck = Build.SysConfig.statfs_sanity_checked == Just True
 
 {- Moves a file into .git/annex/objects/
diff --git a/Command/Status.hs b/Command/Status.hs
index 39e71e750c..576c3bba6d 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -22,12 +22,15 @@ import qualified Git
 import qualified Annex
 import Command
 import Utility.DataUnits
+import Utility.StatFS
 import Annex.Content
 import Types.Key
 import Backend
 import Logs.UUID
 import Logs.Trust
 import Remote
+import Config
+import qualified Build.SysConfig
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -76,6 +79,7 @@ slow_stats =
 	, local_annex_size
 	, known_annex_keys
 	, known_annex_size
+	, disk_size
 	, bloom_info
 	, backend_usage
 	]
@@ -157,6 +161,23 @@ known_annex_size :: Stat
 known_annex_size = stat "known annex size" $ json id $
 	showSizeKeys <$> cachedReferencedData
 
+disk_size :: Stat
+disk_size = stat "available local disk space" $ json id $ lift go
+	where
+		go
+			| Build.SysConfig.statfs_sanity_checked == Just True =
+				calcfree
+					<$> getDiskReserve False
+					<*> inRepo (getFileSystemStats . gitAnnexDir)
+			| otherwise = return unknown
+		calcfree reserve (Just (FileSystemStats { fsStatBytesAvailable = have })) =
+			roughSize storageUnits True $ unreserved reserve have
+		calcfree _ _ = unknown
+		unreserved reserve have
+			| have >= reserve = have - reserve
+			| otherwise = 0
+		unknown = "unknown"
+
 known_annex_keys :: Stat
 known_annex_keys = stat "known annex keys" $ json show $
 	countKeys <$> cachedReferencedData
diff --git a/Config.hs b/Config.hs
index a93e2610e7..aecf77a2a7 100644
--- a/Config.hs
+++ b/Config.hs
@@ -12,6 +12,8 @@ import qualified Git
 import qualified Git.Config
 import qualified Git.Command
 import qualified Annex
+import qualified Build.SysConfig
+import Utility.DataUnits
 
 type ConfigKey = String
 
@@ -85,3 +87,22 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 {- Gets the trust level set for a remote in git config. -}
 getTrustLevel :: Git.Repo -> Annex (Maybe String)
 getTrustLevel r = fromRepo $ Git.Config.getMaybe $ remoteConfig r "trustlevel"
+
+{- Gets annex.diskreserve setting. -}
+getDiskReserve :: Bool -> Annex Integer
+getDiskReserve sanitycheck = do
+	g <- gitRepo
+	r <- getConfig g "diskreserve" ""
+	when sanitycheck $ check r
+	return $ fromMaybe megabyte $ readSize dataUnits r
+	where
+		megabyte = 1000000
+		check r
+			| not (null r) && not cancheck = do
+				unlessM (Annex.getState Annex.force) $
+					error $ "You have configured a diskreserve of "
+						++ r ++
+						" but disk space checking is not working"
+				return ()
+			| otherwise = return ()
+		cancheck = Build.SysConfig.statfs_sanity_checked == Just True
diff --git a/debian/changelog b/debian/changelog
index cf732ab34d..fe91ee4e9e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20120316) UNRELEASED; urgency=low
 
   * Improve detection of inability to check free disk space.
+  * status: Prints available local disk space, or shows if git-annex
+    doesn't know.
 
  -- Joey Hess   Wed, 21 Mar 2012 21:19:16 -0400
 

From d315798609fe82bf753172a807a19cc7203797b6 Mon Sep 17 00:00:00 2001
From: "http://schnouki.net/" 
Date: Thu, 22 Mar 2012 18:42:09 +0000
Subject: [PATCH 3289/8313]

---
 doc/bugs/configurable_path_to_git-annex-shell.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/configurable_path_to_git-annex-shell.mdwn

diff --git a/doc/bugs/configurable_path_to_git-annex-shell.mdwn b/doc/bugs/configurable_path_to_git-annex-shell.mdwn
new file mode 100644
index 0000000000..5ef479160f
--- /dev/null
+++ b/doc/bugs/configurable_path_to_git-annex-shell.mdwn
@@ -0,0 +1,5 @@
+On one of my ssh remotes I had to install git-annex using cabal. No system-wide installation possible. Hence `git-annex` and `git-annex-shell` are not in the default `$PATH` but in `$HOME/.cabal/bin`.
+
+Right now the command run by git-annex when ssh'ing to a remote is hardcoded to "`git-annex-shell`", which doesn't work for me. It would be nice to be able to change this per remote, for example with an option named `annex..annex-shell-command`. Changing "`git-annex-shell`" in `Remote/Helper/Ssh.hs` to "`~/.cabal/bin/git-annex-shell`" worked for me, but it's obviously very ugly :)
+
+Could you do that please? I'll try to hack it myself and send you a patch in the next few days, but I'm pretty new to Haskell so it may take me a while... Thanks!

From 52b90e5d4c2a22415d48a8e572eab328dfcc4407 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Mar 2012 23:23:23 -0400
Subject: [PATCH 3290/8313] tweak

---
 Command/Status.hs | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 576c3bba6d..0003748714 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -141,6 +141,20 @@ local_annex_keys :: Stat
 local_annex_keys = stat "local annex keys" $ json show $
 	countKeys <$> cachedPresentData
 
+known_annex_size :: Stat
+known_annex_size = stat "known annex size" $ json id $
+	showSizeKeys <$> cachedReferencedData
+
+known_annex_keys :: Stat
+known_annex_keys = stat "known annex keys" $ json show $
+	countKeys <$> cachedReferencedData
+
+tmp_size :: Stat
+tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
+
+bad_data_size :: Stat
+bad_data_size = staleSize "bad keys size" gitAnnexBadDir
+
 bloom_info :: Stat
 bloom_info = stat "bloom filter size" $ json id $ do
 	localkeys <- countKeys <$> cachedPresentData
@@ -157,10 +171,6 @@ bloom_info = stat "bloom filter size" $ json id $ do
 
 	return $ size ++ note
 
-known_annex_size :: Stat
-known_annex_size = stat "known annex size" $ json id $
-	showSizeKeys <$> cachedReferencedData
-
 disk_size :: Stat
 disk_size = stat "available local disk space" $ json id $ lift go
 	where
@@ -171,23 +181,13 @@ disk_size = stat "available local disk space" $ json id $ lift go
 					<*> inRepo (getFileSystemStats . gitAnnexDir)
 			| otherwise = return unknown
 		calcfree reserve (Just (FileSystemStats { fsStatBytesAvailable = have })) =
-			roughSize storageUnits True $ unreserved reserve have
+			roughSize storageUnits True $ nonneg $ have - reserve
 		calcfree _ _ = unknown
-		unreserved reserve have
-			| have >= reserve = have - reserve
+		nonneg x
+			| x >= 0 = x
 			| otherwise = 0
 		unknown = "unknown"
 
-known_annex_keys :: Stat
-known_annex_keys = stat "known annex keys" $ json show $
-	countKeys <$> cachedReferencedData
-
-tmp_size :: Stat
-tmp_size = staleSize "temporary directory size" gitAnnexTmpDir
-
-bad_data_size :: Stat
-bad_data_size = staleSize "bad keys size" gitAnnexBadDir
-
 backend_usage :: Stat
 backend_usage = stat "backend usage" $ nojson $
 	calc

From 4eb51126819fe01a003688267f481c6d8014ef47 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 21 Mar 2012 23:41:01 -0400
Subject: [PATCH 3291/8313] rationalize getConfig

getConfig got a remote-specific config, and this confusing name caused it
to be used a couple of places that only were interested in global configs.
Rename to getRemoteConfig and make getConfig only get global configs.

There are no behavior changes here, but remote..annex-web-options
never actually worked (and per-remote web options is a very unlikely to be
useful case so I didn't make it work), so fix the documentation for it.
---
 Annex/Content.hs       |  3 +--
 Command/Status.hs      | 13 ++++++-------
 Config.hs              | 20 ++++++++++++--------
 Remote/Bup.hs          |  4 ++--
 Remote/Directory.hs    |  2 +-
 Remote/Git.hs          |  2 +-
 Remote/Helper/Hooks.hs |  2 +-
 Remote/Helper/Ssh.hs   |  2 +-
 Remote/Hook.hs         |  7 +++----
 Remote/Rsync.hs        |  4 ++--
 doc/git-annex.mdwn     | 16 ++++++++--------
 11 files changed, 38 insertions(+), 37 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 1794fb5d93..e0cfa7227f 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -313,8 +313,7 @@ saveState oneshot = do
 {- Downloads content from any of a list of urls. -}
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
 downloadUrl urls file = do
-	g <- gitRepo
-	o <- map Param . words <$> getConfig g "web-options" ""
+	o <- map Param . words <$> getConfig "annex.web-options" ""
 	liftIO $ anyM (\u -> Url.download u o file) urls
 
 {- Copies a key's content, when present, to a temp file.
diff --git a/Command/Status.hs b/Command/Status.hs
index 0003748714..aaf8489054 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -172,14 +172,13 @@ bloom_info = stat "bloom filter size" $ json id $ do
 	return $ size ++ note
 
 disk_size :: Stat
-disk_size = stat "available local disk space" $ json id $ lift go
+disk_size = stat "available local disk space" $ json id $ lift $
+	if Build.SysConfig.statfs_sanity_checked == Just True
+		then calcfree
+			<$> getDiskReserve False
+			<*> inRepo (getFileSystemStats . gitAnnexDir)
+		else return unknown
 	where
-		go
-			| Build.SysConfig.statfs_sanity_checked == Just True =
-				calcfree
-					<$> getDiskReserve False
-					<*> inRepo (getFileSystemStats . gitAnnexDir)
-			| otherwise = return unknown
 		calcfree reserve (Just (FileSystemStats { fsStatBytesAvailable = have })) =
 			roughSize storageUnits True $ nonneg $ have - reserve
 		calcfree _ _ = unknown
diff --git a/Config.hs b/Config.hs
index aecf77a2a7..39fe3333b0 100644
--- a/Config.hs
+++ b/Config.hs
@@ -25,11 +25,15 @@ setConfig k value = do
 	newg <- inRepo Git.Config.read
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
+{- Looks up a git config setting in git config. -}
+getConfig :: ConfigKey -> String -> Annex String
+getConfig key def = fromRepo $ Git.Config.get key def
+
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
-getConfig :: Git.Repo -> ConfigKey -> String -> Annex String
-getConfig r key def = do
-	def' <- fromRepo $ Git.Config.get ("annex." ++ key) def
+getRemoteConfig :: Git.Repo -> ConfigKey -> String -> Annex String
+getRemoteConfig r key def = do
+	def' <- getConfig key def
 	fromRepo $ Git.Config.get (remoteConfig r key) def'
 
 {- A per-remote config setting in git config. -}
@@ -41,11 +45,11 @@ remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" +
  - is set and prints a number, that is used. -}
 remoteCost :: Git.Repo -> Int -> Annex Int
 remoteCost r def = do
-	cmd <- getConfig r "cost-command" ""
+	cmd <- getRemoteConfig r "cost-command" ""
 	(fromMaybe def . readish) <$>
 		if not $ null cmd
 			then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd]
-			else getConfig r "cost" ""
+			else getRemoteConfig r "cost" ""
 
 cheapRemoteCost :: Int
 cheapRemoteCost = 100
@@ -71,7 +75,8 @@ prop_cost_sane = False `notElem`
 
 {- Checks if a repo should be ignored. -}
 repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = not . fromMaybe False . Git.configTrue <$> getConfig r "ignore" ""
+repoNotIgnored r = not . fromMaybe False . Git.configTrue
+	<$> getRemoteConfig r "ignore" ""
 
 {- If a value is specified, it is used; otherwise the default is looked up
  - in git config. forcenumcopies overrides everything. -}
@@ -91,8 +96,7 @@ getTrustLevel r = fromRepo $ Git.Config.getMaybe $ remoteConfig r "trustlevel"
 {- Gets annex.diskreserve setting. -}
 getDiskReserve :: Bool -> Annex Integer
 getDiskReserve sanitycheck = do
-	g <- gitRepo
-	r <- getConfig g "diskreserve" ""
+	r <- getConfig "diskreserve" ""
 	when sanitycheck $ check r
 	return $ fromMaybe megabyte $ readSize dataUnits r
 	where
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 4ac91e945f..54aff75058 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -35,7 +35,7 @@ remote = RemoteType {
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
-	buprepo <- getConfig r "buprepo" (error "missing buprepo")
+	buprepo <- getRemoteConfig r "buprepo" (error "missing buprepo")
 	cst <- remoteCost r (if bupLocal buprepo then semiCheapRemoteCost else expensiveRemoteCost)
 	bupr <- liftIO $ bup2GitRemote buprepo
 	(u', bupr') <- getBupUUID bupr u
@@ -99,7 +99,7 @@ pipeBup params inh outh = do
 
 bupSplitParams :: Git.Repo -> BupRepo -> Key -> CommandParam -> Annex [CommandParam]
 bupSplitParams r buprepo k src = do
-	o <- getConfig r "bup-split-options" ""
+	o <- getRemoteConfig r "bup-split-options" ""
 	let os = map Param $ words o
 	showOutput -- make way for bup output
 	return $ bupParams "split" buprepo 
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index ecbf511d64..3627d9a9ad 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -33,7 +33,7 @@ remote = RemoteType {
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
-	dir <- getConfig r "directory" (error "missing directory")
+	dir <- getRemoteConfig r "directory" (error "missing directory")
 	cst <- remoteCost r cheapRemoteCost
 	let chunksize = chunkSize c
 	return $ encryptableRemote c
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 3725edd3a2..541b050994 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -300,7 +300,7 @@ rsyncParamsRemote r sending key file = do
 
 rsyncParams :: Git.Repo -> Annex [CommandParam]
 rsyncParams r = do
-	o <- getConfig r "rsync-options" ""
+	o <- getRemoteConfig r "rsync-options" ""
 	return $ options ++ map Param (words o)
 	where
  		-- --inplace to resume partial files
diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs
index ed329b914c..2864a8ed58 100644
--- a/Remote/Helper/Hooks.hs
+++ b/Remote/Helper/Hooks.hs
@@ -84,7 +84,7 @@ runHooks r starthook stophook a = do
 			liftIO $ closeFd fd
 
 lookupHook :: Remote -> String -> Annex (Maybe String)
-lookupHook r n = go =<< getConfig (repo r) hookname ""
+lookupHook r n = go =<< getRemoteConfig (repo r) hookname ""
 	where
 		go "" = return Nothing
 		go command = return $ Just command
diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs
index c61d1b96f2..4c5eef0e6c 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -19,7 +19,7 @@ import Annex.Ssh
  - passed command. -}
 sshToRepo :: Git.Repo -> [CommandParam] -> Annex [CommandParam]
 sshToRepo repo sshcmd = do
-	opts <- map Param . words <$> getConfig repo "ssh-options" ""
+	opts <- map Param . words <$> getRemoteConfig repo "ssh-options" ""
 	params <- sshParams (Git.Url.hostuser repo, Git.Url.port repo) opts
 	return $ params ++ sshcmd
 
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 1e5c27b91f..1c87823caa 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -30,7 +30,7 @@ remote = RemoteType {
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
-	hooktype <- getConfig r "hooktype" (error "missing hooktype")
+	hooktype <- getRemoteConfig r "hooktype" (error "missing hooktype")
 	cst <- remoteCost r expensiveRemoteCost
 	return $ encryptableRemote c
 		(storeEncrypted hooktype)
@@ -74,15 +74,14 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
-	g <- gitRepo
-	command <- getConfig g hookname ""
+	command <- getConfig hookname ""
 	if null command
 		then do
 			warning $ "missing configuration for " ++ hookname
 			return Nothing
 		else return $ Just command
 	where
-		hookname =  hooktype ++ "-" ++ hook ++ "-hook"
+		hookname =  "annex." ++ hooktype ++ "-" ++ hook ++ "-hook"
 
 runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
 runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 03c9911d78..571cd8f5ee 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -60,8 +60,8 @@ gen r u c = do
 
 genRsyncOpts :: Git.Repo -> Annex RsyncOpts
 genRsyncOpts r = do
-	url <- getConfig r "rsyncurl" (error "missing rsyncurl")
-	opts <- getConfig r "rsync-options" ""
+	url <- getRemoteConfig r "rsyncurl" (error "missing rsyncurl")
+	opts <- getRemoteConfig r "rsync-options" ""
 	return $ RsyncOpts url $ map Param $ filter safe $ words opts
 	where
 		safe o
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dd94ccc0c3..72301c0719 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -696,26 +696,26 @@ Here are all the supported configuration settings.
   to or from this remote. For example, to force ipv6, and limit
   the bandwidth to 100Kbyte/s, set it to "-6 --bwlimit 100"
 
-* `remote..annex-web-options`
-
-  Options to use when using wget or curl to download a file from the web.
-  (wget is always used in preference to curl if available).
-  For example, to force ipv4 only, set it to "-4"
-
 * `remote..annex-bup-split-options`
 
   Options to pass to bup split when storing content in this remote.
   For example, to limit the bandwidth to 100Kbye/s, set it to "--bwlimit 100k"
   (There is no corresponding option for bup join.)
 
-* `annex.ssh-options`, `annex.rsync-options`, `annex.web-options, `annex.bup-split-options`
+* `annex.ssh-options`, `annex.rsync-options`, `annex.bup-split-options`
 
   Default ssh, rsync, wget/curl, and bup options to use if a remote does not
   have specific options.
 
+* `annex.web-options`
+
+  Options to use when using wget or curl to download a file from the web.
+  (wget is always used in preference to curl if available).
+  For example, to force ipv4 only, set it to "-4"
+
 * `remote..rsyncurl`
 
-  Used by rsunc special remotes, this configures
+  Used by rsync special remotes, this configures
   the location of the rsync repository to use. Normally this is automaticaly
   set up by `git annex initremote`, but you can change it if needed.
 

From f1398b558316a936690a8f3b01493f498d15b659 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Mar 2012 00:23:15 -0400
Subject: [PATCH 3292/8313] use new getConfig

---
 Annex/Content.hs  | 3 +--
 Annex/Queue.hs    | 6 +++---
 Annex/Ssh.hs      | 4 ++--
 Annex/UUID.hs     | 3 +--
 Annex/Version.hs  | 3 +--
 Backend.hs        | 4 ++--
 Command/Unused.hs | 6 +++---
 Config.hs         | 8 +++-----
 8 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index e0cfa7227f..7bb94aec2c 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -33,7 +33,6 @@ import Common.Annex
 import Logs.Location
 import Annex.UUID
 import qualified Git
-import qualified Git.Config
 import qualified Annex
 import qualified Annex.Queue
 import qualified Annex.Branch
@@ -308,7 +307,7 @@ saveState oneshot = do
 			( Annex.Branch.commit "update" , Annex.Branch.stage)
 	where
 		alwayscommit = fromMaybe True . Git.configTrue
-			<$> fromRepo (Git.Config.get "annex.alwayscommit" "")
+			<$> getConfig "annex.alwayscommit" ""
 
 {- Downloads content from any of a list of urls. -}
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index df6ba12a28..f49a220690 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -14,7 +14,7 @@ module Annex.Queue (
 import Common.Annex
 import Annex hiding (new)
 import qualified Git.Queue
-import qualified Git.Config
+import Config
 
 {- Adds a git command to the queue. -}
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
@@ -43,11 +43,11 @@ get = maybe new return =<< getState repoqueue
 
 new :: Annex Git.Queue.Queue
 new = do
-	q <- Git.Queue.new <$> fromRepo queuesize
+	q <- Git.Queue.new <$> queuesize
 	store q
 	return q
 	where
-		queuesize r = readish =<< Git.Config.getMaybe "annex.queuesize" r
+		queuesize = readish <$> getConfig "annex.queuesize" ""
 
 store :: Git.Queue.Queue -> Annex ()
 store q = changeState $ \s -> s { repoqueue = Just q }
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 79cfbe908b..e6cd6a9263 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -15,7 +15,7 @@ import qualified Data.Map as M
 import Common.Annex
 import Annex.LockPool
 import qualified Git
-import qualified Git.Config
+import Config
 import qualified Build.SysConfig as SysConfig
 
 {- Generates parameters to ssh to a given host (or user@host) on a given
@@ -47,7 +47,7 @@ sshInfo (host, port) = ifM caching
 	where
 		caching = fromMaybe SysConfig.sshconnectioncaching 
 			. Git.configTrue
-			<$> fromRepo (Git.Config.get "annex.sshcaching" "")
+			<$> getConfig "annex.sshcaching" ""
 
 cacheParams :: FilePath -> [CommandParam]
 cacheParams socketfile =
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 0ab2e7e521..e8306de903 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -47,7 +47,7 @@ getUUID = getRepoUUID =<< gitRepo
 {- Looks up a repo's UUID, caching it in .git/config if it's not already. -}
 getRepoUUID :: Git.Repo -> Annex UUID
 getRepoUUID r = do
-	c <- fromRepo cached
+	c <- toUUID <$> getConfig cachekey ""
 	let u = getUncachedUUID r
 	
 	if c /= u && u /= NoUUID
@@ -56,7 +56,6 @@ getRepoUUID r = do
 			return u
 		else return c
 	where
-		cached = toUUID . Git.Config.get cachekey ""
 		updatecache u = do
 			g <- gitRepo
 			when (g /= r) $ storeUUID cachekey u
diff --git a/Annex/Version.hs b/Annex/Version.hs
index 917859eae4..cf5d224842 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -8,7 +8,6 @@
 module Annex.Version where
 
 import Common.Annex
-import qualified Git.Config
 import Config
 
 type Version = String
@@ -26,7 +25,7 @@ versionField :: String
 versionField = "annex.version"
 
 getVersion :: Annex (Maybe Version)
-getVersion = handle <$> fromRepo (Git.Config.get versionField "")
+getVersion = handle <$> getConfig versionField ""
 	where
 		handle [] = Nothing
 		handle v = Just v
diff --git a/Backend.hs b/Backend.hs
index 6810c3a44b..19562205c8 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -18,7 +18,7 @@ module Backend (
 import System.Posix.Files
 
 import Common.Annex
-import qualified Git.Config
+import Config
 import qualified Annex
 import Annex.CheckAttr
 import Types.Key
@@ -46,7 +46,7 @@ orderedList = do
 			l' <- (lookupBackendName name :) <$> standard
 			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		standard = fromRepo $ parseBackendList . Git.Config.get "annex.backends" ""
+		standard = parseBackendList <$> getConfig "annex.backends" ""
 		parseBackendList [] = list
 		parseBackendList s = map lookupBackendName $ words s
 
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 246929f715..bc721635b7 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -23,13 +23,13 @@ import Annex.Content
 import Utility.FileMode
 import Utility.TempFile
 import Logs.Location
+import Config
 import qualified Annex
 import qualified Git
 import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.LsFiles as LsFiles
 import qualified Git.LsTree as LsTree
-import qualified Git.Config
 import qualified Backend
 import qualified Remote
 import qualified Annex.Branch
@@ -189,10 +189,10 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
  -}
 bloomCapacity :: Annex Int
 bloomCapacity = fromMaybe 500000 . readish
-	<$> fromRepo (Git.Config.get "annex.bloomcapacity" "")
+	<$> getConfig "annex.bloomcapacity" ""
 bloomAccuracy :: Annex Int
 bloomAccuracy = fromMaybe 1000 . readish
-	<$> fromRepo (Git.Config.get "annex.bloomaccuracy" "")
+	<$> getConfig "annex.bloomaccuracy" ""
 bloomBitsHashes :: Annex (Int, Int)
 bloomBitsHashes = do
 	capacity <- bloomCapacity
diff --git a/Config.hs b/Config.hs
index 39fe3333b0..f202984852 100644
--- a/Config.hs
+++ b/Config.hs
@@ -32,9 +32,8 @@ getConfig key def = fromRepo $ Git.Config.get key def
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
 getRemoteConfig :: Git.Repo -> ConfigKey -> String -> Annex String
-getRemoteConfig r key def = do
-	def' <- getConfig key def
-	fromRepo $ Git.Config.get (remoteConfig r key) def'
+getRemoteConfig r key def =
+	getConfig (remoteConfig r key) =<< getConfig key def
 
 {- A per-remote config setting in git config. -}
 remoteConfig :: Git.Repo -> ConfigKey -> String
@@ -85,9 +84,8 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
 		use Nothing = perhaps (return 1) =<< 
-			readish <$> fromRepo (Git.Config.get config "1")
+			readish <$> getConfig "annex.numcopies" "1"
 		perhaps fallback = maybe fallback (return . id)
-		config = "annex.numcopies"
 
 {- Gets the trust level set for a remote in git config. -}
 getTrustLevel :: Git.Repo -> Annex (Maybe String)

From e38a839a80ae70eba13b6fd0e7ee08be8a62c513 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Mar 2012 17:09:54 -0400
Subject: [PATCH 3293/8313] Rewrote free disk space checking code

Moving the portability handling into a small C library cleans up things
a lot, avoiding the pain of unpacking structs from inside haskell code.
---
 Annex/Content.hs    |  16 +++---
 Build/Configure.hs  |  17 +-----
 Command/Status.hs   |  16 +++---
 Config.hs           |  18 ++-----
 Makefile            |  14 ++---
 Utility/DiskFree.hs |  32 +++++++++++
 Utility/StatFS.hsc  | 128 --------------------------------------------
 Utility/diskfree.c  |  61 +++++++++++++++++++++
 Utility/diskfree.h  |   1 +
 configure.hs        |  21 +-------
 debian/changelog    |   5 +-
 debian/copyright    |  30 -----------
 git-annex.cabal     |   2 +-
 13 files changed, 124 insertions(+), 237 deletions(-)
 create mode 100644 Utility/DiskFree.hs
 delete mode 100644 Utility/StatFS.hsc
 create mode 100644 Utility/diskfree.c
 create mode 100644 Utility/diskfree.h

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 7bb94aec2c..8542d8775d 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -36,7 +36,7 @@ import qualified Git
 import qualified Annex
 import qualified Annex.Queue
 import qualified Annex.Branch
-import Utility.StatFS
+import Utility.DiskFree
 import Utility.FileMode
 import qualified Utility.Url as Url
 import Types.Key
@@ -44,7 +44,6 @@ import Utility.DataUnits
 import Utility.CopyFile
 import Config
 import Annex.Exception
-import qualified Build.SysConfig
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
@@ -176,22 +175,19 @@ checkDiskSpace = checkDiskSpace' 0
 
 checkDiskSpace' :: Integer -> Key -> Annex ()
 checkDiskSpace' adjustment key = do
-	reserve <- getDiskReserve True
-	stats <- inRepo $ getFileSystemStats .gitAnnexDir
-	case (cancheck, stats, keySize key) of
-		(False, _, _) -> return ()
-		(_, Nothing, _) -> return ()
-		(_, _, Nothing) -> return ()
-		(_, Just (FileSystemStats { fsStatBytesAvailable = have }), Just need) ->
+	reserve <- getDiskReserve
+	free <- inRepo $ getDiskFree . gitAnnexDir
+	case (free, keySize key) of
+		(Just have, Just need) ->
 			when (need + reserve > have + adjustment) $
 				needmorespace (need + reserve - have - adjustment)
+		_ -> return ()
 	where
 		needmorespace n = unlessM (Annex.getState Annex.force) $
 			error $ "not enough free space, need " ++ 
 				roughSize storageUnits True n ++
 				" more" ++ forcemsg
 		forcemsg = " (use --force to override this check or adjust annex.diskreserve)"
-		cancheck = Build.SysConfig.statfs_sanity_checked == Just True
 
 {- Moves a file into .git/annex/objects/
  -
diff --git a/Build/Configure.hs b/Build/Configure.hs
index 14667ba860..341b8840dc 100644
--- a/Build/Configure.hs
+++ b/Build/Configure.hs
@@ -10,12 +10,8 @@ import Control.Applicative
 import Build.TestConfig
 import Utility.SafeCommand
 
-tests :: Bool -> [TestCase]
-tests True = cabaltests ++ common
-tests False = common
-
-common :: [TestCase]
-common =
+tests :: [TestCase]
+tests =
 	[ TestCase "version" getVersion
 	, TestCase "git" $ requireCmd "git" "git --version >/dev/null"
 	, TestCase "git version" getGitVersion
@@ -32,11 +28,6 @@ common =
 	, TestCase "ssh connection caching" getSshConnectionCaching
 	] ++ shaTestCases [1, 256, 512, 224, 384]
 
-cabaltests :: [TestCase]
-cabaltests =
-	[ TestCase "StatFS" testStatFSDummy
-	]
-
 shaTestCases :: [Int] -> [TestCase]
 shaTestCases l = map make l
 	where make n =
@@ -81,10 +72,6 @@ getSshConnectionCaching :: Test
 getSshConnectionCaching = Config "sshconnectioncaching" . BoolConfig <$>
 	boolSystem "sh" [Param "-c", Param "ssh -o ControlPersist=yes -V >/dev/null 2>/dev/null"]
 
-testStatFSDummy :: Test
-testStatFSDummy = 
-	return $ Config "statfs_sanity_checked" $ MaybeBoolConfig Nothing
-
 {- Set up cabal file with version. -}
 cabalSetup :: IO ()
 cabalSetup = do
diff --git a/Command/Status.hs b/Command/Status.hs
index aaf8489054..40cefb5cc2 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -22,7 +22,7 @@ import qualified Git
 import qualified Annex
 import Command
 import Utility.DataUnits
-import Utility.StatFS
+import Utility.DiskFree
 import Annex.Content
 import Types.Key
 import Backend
@@ -30,7 +30,6 @@ import Logs.UUID
 import Logs.Trust
 import Remote
 import Config
-import qualified Build.SysConfig
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -173,19 +172,16 @@ bloom_info = stat "bloom filter size" $ json id $ do
 
 disk_size :: Stat
 disk_size = stat "available local disk space" $ json id $ lift $
-	if Build.SysConfig.statfs_sanity_checked == Just True
-		then calcfree
-			<$> getDiskReserve False
-			<*> inRepo (getFileSystemStats . gitAnnexDir)
-		else return unknown
+	calcfree
+		<$> getDiskReserve
+		<*> inRepo (getDiskFree . gitAnnexDir)
 	where
-		calcfree reserve (Just (FileSystemStats { fsStatBytesAvailable = have })) =
+		calcfree reserve (Just have) =
 			roughSize storageUnits True $ nonneg $ have - reserve
-		calcfree _ _ = unknown
+		calcfree _ _ = "unknown"
 		nonneg x
 			| x >= 0 = x
 			| otherwise = 0
-		unknown = "unknown"
 
 backend_usage :: Stat
 backend_usage = stat "backend usage" $ nojson $
diff --git a/Config.hs b/Config.hs
index f202984852..10a66e47b1 100644
--- a/Config.hs
+++ b/Config.hs
@@ -12,7 +12,6 @@ import qualified Git
 import qualified Git.Config
 import qualified Git.Command
 import qualified Annex
-import qualified Build.SysConfig
 import Utility.DataUnits
 
 type ConfigKey = String
@@ -92,19 +91,8 @@ getTrustLevel :: Git.Repo -> Annex (Maybe String)
 getTrustLevel r = fromRepo $ Git.Config.getMaybe $ remoteConfig r "trustlevel"
 
 {- Gets annex.diskreserve setting. -}
-getDiskReserve :: Bool -> Annex Integer
-getDiskReserve sanitycheck = do
-	r <- getConfig "diskreserve" ""
-	when sanitycheck $ check r
-	return $ fromMaybe megabyte $ readSize dataUnits r
+getDiskReserve :: Annex Integer
+getDiskReserve = fromMaybe megabyte . readSize dataUnits
+	<$> getConfig "diskreserve" ""
 	where
 		megabyte = 1000000
-		check r
-			| not (null r) && not cancheck = do
-				unlessM (Annex.getState Annex.force) $
-					error $ "You have configured a diskreserve of "
-						++ r ++
-						" but disk space checking is not working"
-				return ()
-			| otherwise = return ()
-		cancheck = Build.SysConfig.statfs_sanity_checked == Just True
diff --git a/Makefile b/Makefile
index ddb5e3ff6a..a0447f2d21 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
@@ -11,7 +11,8 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 
 bins=git-annex
 mans=git-annex.1 git-annex-shell.1
-sources=Build/SysConfig.hs Utility/StatFS.hs Utility/Touch.hs
+sources=Build/SysConfig.hs Utility/Touch.hs
+clibs=Utility/diskfree.o
 
 all=$(bins) $(mans) docs
 
@@ -28,15 +29,16 @@ sources: $(sources)
 fast: GHCFLAGS=$(BASEFLAGS)
 fast: $(bins)
 
-Build/SysConfig.hs: configure.hs Build/TestConfig.hs Utility/StatFS.hs
+Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs
 	$(GHCMAKE) configure
 	./configure
 
 %.hs: %.hsc
 	hsc2hs $<
 
-$(bins): $(sources)
-	$(GHCMAKE) $@
+
+git-annex: $(sources) $(clibs)
+	$(GHCMAKE) $@ $(clibs)
 
 git-annex.1: doc/git-annex.mdwn
 	./mdwn2man git-annex 1 doc/git-annex.mdwn > git-annex.1
@@ -92,7 +94,7 @@ docs: $(mans)
 
 clean:
 	rm -rf tmp $(bins) $(mans) test configure  *.tix .hpc $(sources) \
-		doc/.ikiwiki html dist
+		doc/.ikiwiki html dist $(clibs)
 
 # Workaround for cabal sdist not running Setup hooks, so I cannot
 # generate a file list there.
diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs
new file mode 100644
index 0000000000..e02794954d
--- /dev/null
+++ b/Utility/DiskFree.hs
@@ -0,0 +1,32 @@
+{- disk free space checking 
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE ForeignFunctionInterface #-}
+
+module Utility.DiskFree ( getDiskFree ) where
+
+import Common
+
+import Foreign.C.Types
+import Foreign.C.String
+import Foreign.C.Error
+
+foreign import ccall unsafe "diskfree.h diskfree" c_diskfree
+	:: CString -> IO CULLong
+
+getDiskFree :: String -> IO (Maybe Integer)
+getDiskFree path = withFilePath path $ \c_path -> do
+	free <- c_diskfree c_path
+	ifM (safeErrno <$> getErrno)
+		( return $ Just $ toInteger free
+		, do
+			Errno i <- getErrno
+			print i
+			return Nothing
+		)
+	where
+		safeErrno (Errno v) = v == 0
diff --git a/Utility/StatFS.hsc b/Utility/StatFS.hsc
deleted file mode 100644
index ed4c9f1cb6..0000000000
--- a/Utility/StatFS.hsc
+++ /dev/null
@@ -1,128 +0,0 @@
------------------------------------------------------------------------------
--- |
---
--- (This code originally comes from xmobar)
--- 
--- Module      :  StatFS
--- Copyright   :  (c) Jose A Ortega Ruiz
--- License     :  BSD-3-clause
---
--- All rights reserved.
--- 
--- Redistribution and use in source and binary forms, with or without
--- modification, are permitted provided that the following conditions
--- are met:
--- 
--- 1. Redistributions of source code must retain the above copyright
---    notice, this list of conditions and the following disclaimer.
--- 2. Redistributions in binary form must reproduce the above copyright
---    notice, this list of conditions and the following disclaimer in the
---    documentation and/or other materials provided with the distribution.
--- 3. Neither the name of the author nor the names of his contributors
---    may be used to endorse or promote products derived from this software
---    without specific prior written permission.
--- 
--- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
--- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
--- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
--- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
--- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
--- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
--- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
--- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
--- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
--- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
--- SUCH DAMAGE.
---
--- Maintainer  :  Jose A Ortega Ruiz 
--- Stability   :  unstable
--- Portability :  unportable
---
---  A binding to C's statvfs(2)
---
------------------------------------------------------------------------------
-
-{-# LANGUAGE CPP, ForeignFunctionInterface, EmptyDataDecls #-}
-
-
-module Utility.StatFS ( FileSystemStats(..), getFileSystemStats ) where
-
-import Utility.FileSystemEncoding
-
-import Foreign
-import Foreign.C.Types
-import Foreign.C.String
-
-#if defined (__FreeBSD__) || defined (__FreeBSD_kernel__) || defined (__APPLE__)
-# include 
-# include 
-#else
-#if defined (__linux__)
-#include 
-#else
-#define UNKNOWN
-#endif
-#endif
-
-data FileSystemStats = FileSystemStats {
-  fsStatBlockSize :: Integer
-  -- ^ Optimal transfer block size.
-  , fsStatBlockCount :: Integer
-  -- ^ Total data blocks in file system.
-  , fsStatByteCount :: Integer
-  -- ^ Total bytes in file system.
-  , fsStatBytesFree :: Integer
-  -- ^ Free bytes in file system.
-  , fsStatBytesAvailable :: Integer
-  -- ^ Free bytes available to non-superusers.
-  , fsStatBytesUsed :: Integer
-  -- ^ Bytes used.
-  } deriving (Show, Eq)
-
-data CStatfs
-
-#ifdef UNKNOWN
-#warning free space checking code not available for this OS
-#else
-#if defined(__APPLE__)
-foreign import ccall unsafe "sys/mount.h statfs64"
-#else
-#if defined(__FreeBSD__) || defined (__FreeBSD_kernel__)
-foreign import ccall unsafe "sys/mount.h statfs"
-#else
-foreign import ccall unsafe "sys/vfs.h statfs64"
-#endif
-#endif
-  c_statfs :: CString -> Ptr CStatfs -> IO CInt
-#endif
-
-toI :: CULong -> Integer
-toI = toInteger
-
-getFileSystemStats :: String -> IO (Maybe FileSystemStats)
-getFileSystemStats path =
-#ifdef UNKNOWN
-  return Nothing
-#else
-  allocaBytes (#size struct statfs) $ \vfs ->
-  withFilePath path $ \cpath -> do
-    res <- c_statfs cpath vfs
-    if res == -1 then return Nothing
-      else do
-        bsize <- (#peek struct statfs, f_bsize) vfs
-        bcount <- (#peek struct statfs, f_blocks) vfs
-        bfree <- (#peek struct statfs, f_bfree) vfs
-        bavail <- (#peek struct statfs, f_bavail) vfs
-        let bpb = toI bsize
-        let stats = FileSystemStats
-                       { fsStatBlockSize = bpb
-                       , fsStatBlockCount = toI bcount
-                       , fsStatByteCount = toI bcount * bpb
-                       , fsStatBytesFree = toI bfree * bpb
-                       , fsStatBytesAvailable = toI bavail * bpb
-                       , fsStatBytesUsed = toI (bcount - bfree) * bpb
-                       }
-        if fsStatBlockCount stats == 0 || fsStatBlockSize stats == 0
-          then return Nothing
-          else return $ Just stats
-#endif
diff --git a/Utility/diskfree.c b/Utility/diskfree.c
new file mode 100644
index 0000000000..9ac31a752d
--- /dev/null
+++ b/Utility/diskfree.c
@@ -0,0 +1,61 @@
+/* disk free space checking, C mini-library
+ *
+ * Copyright 2012 Joey Hess 
+ *
+ * Licensed under the GNU GPL version 3 or higher.
+ */
+
+/* Include appropriate headers for the OS, and define what will be used to
+ * check the free space. */
+#if defined(__APPLE__)
+# include 
+# include 
+# define STATSTRUCT statfs
+# define STATCALL statfs64
+#else
+#if defined (__FreeBSD__) || defined (__FreeBSD_kernel__)
+# include 
+# include 
+# define STATSTRUCT statfs
+# define STATCALL statfs
+#else
+#if defined (__linux__)
+# include 
+# define STATSTRUCT statvfs
+# define STATCALL statvfs
+#else
+# warning free space checking code not available for this OS
+# define UNKNOWN
+#endif
+#endif
+#endif
+
+#include 
+
+/* Checks the amount of disk that is available to regular (non-root) users.
+ * (If there's an error, or this is not supported,
+ * returns 0 and sets errno to nonzero.)
+ */
+unsigned long long int diskfree(const char *path) {
+#ifdef UNKNOWN
+	errno = 1;
+	return 0;
+#else
+	unsigned long long int available, blocksize;
+	struct STATSTRUCT buf;
+
+	errno = 0;
+	if (STATCALL(path, &buf) != 0)
+		return 0; /* errno is set */
+
+	available = buf.f_bavail;
+	blocksize = buf.f_bsize;
+	return available * blocksize;
+#endif
+}
+
+/*
+main () {
+	printf("%lli\n", diskfree("."));
+}
+*/
diff --git a/Utility/diskfree.h b/Utility/diskfree.h
new file mode 100644
index 0000000000..e5b84754fe
--- /dev/null
+++ b/Utility/diskfree.h
@@ -0,0 +1 @@
+unsigned long long int diskfree(const char *path);
diff --git a/configure.hs b/configure.hs
index 6fdc5fcb03..15833e62a7 100644
--- a/configure.hs
+++ b/configure.hs
@@ -1,23 +1,6 @@
 {- configure program -}
 
-import Data.Maybe
-
-import qualified Build.Configure as Configure
-import Build.TestConfig
-import Utility.StatFS
-
-tests :: [TestCase]
-tests = [ TestCase "StatFS" testStatFS
-	] ++ Configure.tests False
-
-{- This test cannot be included in Build.Configure due to needing
- - Utility/StatFS.hs to be built, which it is not when "cabal configure"
- - is run. -}
-testStatFS :: Test
-testStatFS = do
-	s <- getFileSystemStats "."
-	return $ Config "statfs_sanity_checked" $
-		MaybeBoolConfig $ Just $ isJust s
+import Build.Configure
 
 main :: IO ()
-main = Configure.run tests
+main = run tests
diff --git a/debian/changelog b/debian/changelog
index fe91ee4e9e..66e7b83a71 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,7 @@
 git-annex (3.20120316) UNRELEASED; urgency=low
 
-  * Improve detection of inability to check free disk space.
-  * status: Prints available local disk space, or shows if git-annex
-    doesn't know.
+  * Rewrote free disk space checking code, moving the portability
+    handling into a small C library.
 
  -- Joey Hess   Wed, 21 Mar 2012 21:19:16 -0400
 
diff --git a/debian/copyright b/debian/copyright
index 85fd174fc0..332c1e71d2 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -7,33 +7,3 @@ License: GPL-3+
  The full text of version 3 of the GPL is distributed as doc/GPL in
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
-
-Files: Utility/StatFS.hsc
-Copyright: Jose A Ortega Ruiz 
-License: BSD-3-clause
- -- All rights reserved.
- -- 
- -- Redistribution and use in source and binary forms, with or without
- -- modification, are permitted provided that the following conditions
- -- are met:
- -- 
- -- 1. Redistributions of source code must retain the above copyright
- --    notice, this list of conditions and the following disclaimer.
- -- 2. Redistributions in binary form must reproduce the above copyright
- --    notice, this list of conditions and the following disclaimer in the
- --    documentation and/or other materials provided with the distribution.
- -- 3. Neither the name of the author nor the names of his contributors
- --    may be used to endorse or promote products derived from this software
- --    without specific prior written permission.
- -- 
- -- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- -- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- -- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- -- ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
- -- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- -- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- -- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- -- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- -- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- -- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- -- SUCH DAMAGE.
diff --git a/git-annex.cabal b/git-annex.cabal
index 881e4d212b..184f6323a0 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120315
+Version: 3.20120316
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From d7d2fefbf2e670f9ba160769620b3e268712dab8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Mar 2012 17:30:35 -0400
Subject: [PATCH 3294/8313] break out kfreebsd

Now tested on linux and freebsd (amd64).
---
 Utility/diskfree.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/Utility/diskfree.c b/Utility/diskfree.c
index 9ac31a752d..b8d8aa3b31 100644
--- a/Utility/diskfree.c
+++ b/Utility/diskfree.c
@@ -13,11 +13,17 @@
 # define STATSTRUCT statfs
 # define STATCALL statfs64
 #else
-#if defined (__FreeBSD__) || defined (__FreeBSD_kernel__)
+#if defined (__FreeBSD__)
 # include 
 # include 
 # define STATSTRUCT statfs
-# define STATCALL statfs
+# define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */
+#else
+#if defined (__FreeBSD_kernel__) /* Debian kFreeBSD */
+# include 
+# include 
+# define STATSTRUCT statfs
+# define STATCALL statfs64
 #else
 #if defined (__linux__)
 # include 
@@ -29,8 +35,10 @@
 #endif
 #endif
 #endif
+#endif
 
 #include 
+#include 
 
 /* Checks the amount of disk that is available to regular (non-root) users.
  * (If there's an error, or this is not supported,

From 49e8e294991d855552a65f1b9755cae96972e84e Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 22 Mar 2012 22:26:02 +0000
Subject: [PATCH 3295/8313] Added a comment

---
 ...comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment

diff --git a/doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment b/doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment
new file mode 100644
index 0000000000..50349363b1
--- /dev/null
+++ b/doc/bugs/configurable_path_to_git-annex-shell/comment_1_fb6771f902b57f2b690e7cc46fdac47e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-22T22:26:02Z"
+ content="""
+It doesn't need to be installed into the system PATH; just the user PATH. Which you should be able to control.
+
+Exactly how to do this surely varies, but here I have a `~/.bashrc` containing `PATH=$HOME/bin:$PATH; export PATH` and I keep git-annex-shell in bin and it's available to eg \"ssh mybox git-annex-shell\"
+"""]]

From 6f514155f194862e9616a73d8881253ed19eb5b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Mar 2012 22:48:19 -0400
Subject: [PATCH 3296/8313] update

---
 Utility/diskfree.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Utility/diskfree.c b/Utility/diskfree.c
index b8d8aa3b31..bb4e8eed36 100644
--- a/Utility/diskfree.c
+++ b/Utility/diskfree.c
@@ -11,6 +11,9 @@
 # include 
 # include 
 # define STATSTRUCT statfs
+/* In newer OSX versions, statfs64 is deprecated, in favor of statfs,
+ * which is 64 bit with a built option -- but statfs64 still works,
+ * and this keeps older OSX also supported. */
 # define STATCALL statfs64
 #else
 #if defined (__FreeBSD__)
@@ -26,6 +29,7 @@
 # define STATCALL statfs64
 #else
 #if defined (__linux__)
+/* This is a POSIX standard, so might also work elsewhere. */
 # include 
 # define STATSTRUCT statvfs
 # define STATCALL statvfs

From 0f2052446f90f806961a6430bacc1b98c1ae7585 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 22 Mar 2012 22:49:33 -0400
Subject: [PATCH 3297/8313] tweak

---
 Utility/DiskFree.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs
index e02794954d..bde6ef8caa 100644
--- a/Utility/DiskFree.hs
+++ b/Utility/DiskFree.hs
@@ -23,10 +23,7 @@ getDiskFree path = withFilePath path $ \c_path -> do
 	free <- c_diskfree c_path
 	ifM (safeErrno <$> getErrno)
 		( return $ Just $ toInteger free
-		, do
-			Errno i <- getErrno
-			print i
-			return Nothing
+		, return Nothing
 		)
 	where
 		safeErrno (Errno v) = v == 0

From 688fdc0acb3be7db80061d941e9e28e640afe62c Mon Sep 17 00:00:00 2001
From: "http://schnouki.net/" 
Date: Fri, 23 Mar 2012 13:27:13 +0000
Subject: [PATCH 3298/8313] Added a comment

---
 .../comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment

diff --git a/doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment b/doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment
new file mode 100644
index 0000000000..adea95ecf2
--- /dev/null
+++ b/doc/bugs/configurable_path_to_git-annex-shell/comment_2_2b856f4f0b65c2331be7d565f0e4e8a8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://schnouki.net/"
+ nickname="Schnouki"
+ subject="comment 2"
+ date="2012-03-23T13:27:12Z"
+ content="""
+Hmm, ok, solved. I'm using zsh, which is a little different: `.zshrc` is only read for interactive shells, so `ssh mybox 'echo $PATH'` displayed `/usr/bin:/bin:/usr/sbin:/sbin`. Using `.zshenv`, which is used even for non-interactive shells, did the trick. Thanks!
+"""]]

From 42cc85a82f03e0e449af630032e42f58f1829cc9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Mar 2012 12:05:05 -0400
Subject: [PATCH 3299/8313] use struct statfs64 on OSX

---
 Utility/diskfree.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Utility/diskfree.c b/Utility/diskfree.c
index bb4e8eed36..6e9eda930f 100644
--- a/Utility/diskfree.c
+++ b/Utility/diskfree.c
@@ -10,29 +10,29 @@
 #if defined(__APPLE__)
 # include 
 # include 
-# define STATSTRUCT statfs
 /* In newer OSX versions, statfs64 is deprecated, in favor of statfs,
- * which is 64 bit with a built option -- but statfs64 still works,
+ * which is 64 bit only with a build option -- but statfs64 still works,
  * and this keeps older OSX also supported. */
 # define STATCALL statfs64
+# define STATSTRUCT statfs64
 #else
 #if defined (__FreeBSD__)
 # include 
 # include 
-# define STATSTRUCT statfs
 # define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */
+# define STATSTRUCT statfs
 #else
 #if defined (__FreeBSD_kernel__) /* Debian kFreeBSD */
 # include 
 # include 
-# define STATSTRUCT statfs
 # define STATCALL statfs64
+# define STATSTRUCT statfs
 #else
 #if defined (__linux__)
 /* This is a POSIX standard, so might also work elsewhere. */
 # include 
-# define STATSTRUCT statvfs
 # define STATCALL statvfs
+# define STATSTRUCT statvfs
 #else
 # warning free space checking code not available for this OS
 # define UNKNOWN

From e72c2b9be2d76887a91ef7eaf645e43c5fcaaf04 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Mar 2012 12:14:42 -0400
Subject: [PATCH 3300/8313] close

---
 doc/bugs/configurable_path_to_git-annex-shell.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/configurable_path_to_git-annex-shell.mdwn b/doc/bugs/configurable_path_to_git-annex-shell.mdwn
index 5ef479160f..af2474451c 100644
--- a/doc/bugs/configurable_path_to_git-annex-shell.mdwn
+++ b/doc/bugs/configurable_path_to_git-annex-shell.mdwn
@@ -3,3 +3,5 @@ On one of my ssh remotes I had to install git-annex using cabal. No system-wide
 Right now the command run by git-annex when ssh'ing to a remote is hardcoded to "`git-annex-shell`", which doesn't work for me. It would be nice to be able to change this per remote, for example with an option named `annex..annex-shell-command`. Changing "`git-annex-shell`" in `Remote/Helper/Ssh.hs` to "`~/.cabal/bin/git-annex-shell`" worked for me, but it's obviously very ugly :)
 
 Could you do that please? I'll try to hack it myself and send you a patch in the next few days, but I'm pretty new to Haskell so it may take me a while... Thanks!
+
+> [[closing|done]], see comments --[[Joey]] 

From ff62bc515170f728d4418a56d91ac9a71d5dd522 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Mar 2012 12:24:40 -0400
Subject: [PATCH 3301/8313] fix use of Configure

---
 Setup.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Setup.hs b/Setup.hs
index 239a6217f0..14e6a4ea71 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -8,5 +8,5 @@ import qualified Build.Configure as Configure
 main = defaultMainWithHooks simpleUserHooks { preConf = configure }
 
 configure _ _ = do
-	Configure.run $ Configure.tests True
+	Configure.run Configure.tests
 	return (Nothing, [])

From 981e1ab43dd4dc2ae99b69e34c5f2dabd35ec1ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Mar 2012 12:36:06 -0400
Subject: [PATCH 3302/8313] avoid rewriting SysConfig.hs with identical
 contents

This avoids some compliation when when reconfiguring.
---
 Build/TestConfig.hs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs
index 0cc2019cfe..03d87cd8bf 100644
--- a/Build/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -5,6 +5,7 @@ module Build.TestConfig where
 import System.IO
 import System.Cmd
 import System.Exit
+import System.Directory
 
 type ConfigKey = String
 data ConfigValue =
@@ -36,8 +37,14 @@ instance Show Config where
 			valuetype (MaybeBoolConfig _) = "Maybe Bool"
 
 writeSysConfig :: [Config] -> IO ()
-writeSysConfig config = writeFile "Build/SysConfig.hs" body
+writeSysConfig config = do
+	e <- doesFileExist dest
+	old <- if e then readFile dest else return []
+	if (old /= body)
+		then writeFile dest body
+		else return ()
 	where
+		dest = "Build/SysConfig.hs"
 		body = unlines $ header ++ map show config ++ footer
 		header = [
 			  "{- Automatically generated. -}"

From b9098a3e3795f7efc3dc1e9ce2a1aa40e3690df1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Mar 2012 12:42:22 -0400
Subject: [PATCH 3303/8313] update cabal file for recent changes

also wired the test suite into cabal
---
 git-annex.cabal | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 184f6323a0..9e37a6cb3c 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -33,11 +33,18 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter
-  Other-Modules: Utility.StatFS, Utility.Touch
+  Other-Modules: Utility.Touch
+  C-Sources: Utility/diskfree.c
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  Other-Modules: Utility.StatFS
+  C-Sources: Utility/diskfree.c
+
+Test-Suite test
+  Type: exitcode-stdio-1.0
+  Main-Is: test.hs
+  Build-Depends: testpack, HUnit
+  C-Sources: Utility/diskfree.c
 
 source-repository head
   type: git

From 0e6e840a2ae10b517c3ca19d323d329eb935fc5d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 23 Mar 2012 12:43:21 -0400
Subject: [PATCH 3304/8313] Revert "avoid rewriting SysConfig.hs with identical
 contents"

That made the Makefile want to rebuild the file each time if it's
dependencies were newer, as it was not updated.
---
 Build/TestConfig.hs | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/Build/TestConfig.hs b/Build/TestConfig.hs
index 03d87cd8bf..0cc2019cfe 100644
--- a/Build/TestConfig.hs
+++ b/Build/TestConfig.hs
@@ -5,7 +5,6 @@ module Build.TestConfig where
 import System.IO
 import System.Cmd
 import System.Exit
-import System.Directory
 
 type ConfigKey = String
 data ConfigValue =
@@ -37,14 +36,8 @@ instance Show Config where
 			valuetype (MaybeBoolConfig _) = "Maybe Bool"
 
 writeSysConfig :: [Config] -> IO ()
-writeSysConfig config = do
-	e <- doesFileExist dest
-	old <- if e then readFile dest else return []
-	if (old /= body)
-		then writeFile dest body
-		else return ()
+writeSysConfig config = writeFile "Build/SysConfig.hs" body
 	where
-		dest = "Build/SysConfig.hs"
 		body = unlines $ header ++ map show config ++ footer
 		header = [
 			  "{- Automatically generated. -}"

From a398db7885b5e5d2f97bd496f7c354e52145a441 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Mar 2012 11:58:22 -0400
Subject: [PATCH 3305/8313] update

---
 debian/changelog | 1 +
 1 file changed, 1 insertion(+)

diff --git a/debian/changelog b/debian/changelog
index 66e7b83a71..5aecd7b7e0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20120316) UNRELEASED; urgency=low
 
   * Rewrote free disk space checking code, moving the portability
     handling into a small C library.
+  * status: Display amount of free disk space.
 
  -- Joey Hess   Wed, 21 Mar 2012 21:19:16 -0400
 

From bb71305088ba509cae57367974e6033c24db20ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Mar 2012 17:21:03 -0400
Subject: [PATCH 3306/8313] add a .ghci file to make ghci easier to use in
 git-annex

---
 .ghci | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .ghci

diff --git a/.ghci b/.ghci
new file mode 100644
index 0000000000..37258aefe3
--- /dev/null
+++ b/.ghci
@@ -0,0 +1,3 @@
+-- make ghci use precompiled modules, and C library
+:set -outputdir=tmp
+:set -IUtility

From 63d3b2a62e9cdb4c75d6ee3762546d937f58606c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Mar 2012 17:25:47 -0400
Subject: [PATCH 3307/8313] autoload Common too in ghci

---
 .ghci | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ghci b/.ghci
index 37258aefe3..318bac2028 100644
--- a/.ghci
+++ b/.ghci
@@ -1,3 +1,4 @@
 -- make ghci use precompiled modules, and C library
 :set -outputdir=tmp
 :set -IUtility
+:load Common

From ef74908fe795989afb0c1a9115916ae9c9380b06 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 24 Mar 2012 19:46:11 -0400
Subject: [PATCH 3308/8313] expand

---
 .../using_the_web_as_a_special_remote.mdwn    | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/doc/tips/using_the_web_as_a_special_remote.mdwn b/doc/tips/using_the_web_as_a_special_remote.mdwn
index a151f99332..24ded90c08 100644
--- a/doc/tips/using_the_web_as_a_special_remote.mdwn
+++ b/doc/tips/using_the_web_as_a_special_remote.mdwn
@@ -30,3 +30,28 @@ With the result that it will hang onto files:
 	  	00000000-0000-0000-0000-000000000001  -- web
 	  (Use --force to override this check, or adjust annex.numcopies.)
 	failed
+
+You can also add urls to any file already in the annex:
+
+	# git annex addurl --file my_cool_big_file http://example.com/cool_big_file
+	addurl my_cool_big_file ok
+	# git annex whereis my_cool_big_file
+	whereis my_cool_big_file (2 copies) 
+  	00000000-0000-0000-0000-000000000001 -- web
+   	27a9510c-760a-11e1-b9a0-c731d2b77df9 -- here
+
+To add a lot of urls at once, just list them all as parameters to
+`git annex addurl`. 
+
+If you're adding a bunch of related files to a directory, or just don't
+like the default filenames generated by `addurl`, you can use `--pathdepth`
+to specify how many parts of the url are put in the filename.
+A positive number drops that many paths from the beginning, while a negative
+number takes that many paths from the end.
+	
+	# git annex addurl http://example.com/videos/2012/01/video.mpeg
+	addurl example.com_videos_2012_01_video.mpeg (downloading http://example.com/videos/2012/01/video.mpeg)
+	# git annex addurl http://example.com/videos/2012/01/video.mpeg --pathdepth=2
+	addurl 2012_01_video.mpeg (downloading http://example.com/videos/2012/01/video.mpeg)
+	# git annex addurl http://example.com/videos/2012/video.mpeg --pathdepth=-2
+	addurl 01_video.mpeg (downloading http://example.com/videos/2012/01/video.mpeg)

From 180a6ef3da9bf05321cc40a897ff2a6691c29f02 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkEUhIcw37X2Kh-dznSMIb9Vgcq0frfdWs"
 
Date: Wed, 28 Mar 2012 19:06:52 +0000
Subject: [PATCH 3309/8313] Added a comment: GHC 7

---
 ..._0327c64b15249596add635d26f4ce67f._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment

diff --git a/doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment b/doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment
new file mode 100644
index 0000000000..5768d8b932
--- /dev/null
+++ b/doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkEUhIcw37X2Kh-dznSMIb9Vgcq0frfdWs"
+ nickname="Ethan"
+ subject="GHC 7"
+ date="2012-03-28T19:06:51Z"
+ content="""
+The Haskell Platform installer for OSX uses GHC 7.0.4, which doesn't seem able to support the current version of git-annex.
+
+Cabal throws a very cryptic error about not being able to use the proper base package.
+
+I was able to install it by 
+
+1.  cloning the repo
+2.  merging the ghc7.0 branch
+3.  resolving merge conflicts in git-annex.cabal
+4.  cabal install git-annex.cabal
+
+(Note I also tried this with homebrew and had similar results)
+"""]]

From 20313b32922566b3fa43d3cf8d91214a47ecfa3d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Wed, 28 Mar 2012 19:18:58 +0000
Subject: [PATCH 3310/8313] Added a comment: ghc 7.0

---
 .../comment_3_47c682a779812dda77601c24a619923c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment

diff --git a/doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment b/doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment
new file mode 100644
index 0000000000..251375cbec
--- /dev/null
+++ b/doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="ghc 7.0"
+ date="2012-03-28T19:18:58Z"
+ content="""
+You did the right thing, although just checking out the ghc-7.0 branch will avoid merge conflicts. I am trying to keep it fairly close to up-to-date.
+"""]]

From ae1734e7e847f3434b85c224fcdc57c677bf030b Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 29 Mar 2012 01:05:12 +0000
Subject: [PATCH 3311/8313]

---
 .../tell_us_how_you__39__re_using_git-annex.mdwn     | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn

diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
new file mode 100644
index 0000000000..14ca838cf1
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
@@ -0,0 +1,12 @@
+Tell your git-annex stories here. Feel free to give as little or as much detail as appropriate about how you're using it. How's it working out for you?
+
+I'll start with a comment a user just posted to IRC:
+
+
+oh my god, git-annex is amazing
+this is the revolution in fucking with gigantic piles of files that I've been waiting for
+

+
+And then my own story: I have a ton of drives. I have a lot of servers. I live in a cabin on **dialup** and often have 1 hour on broadband in a week to get everything I need. Without git-annex, managing all this would not be possible. It works perfectly for me, not a surprise since I wrote it, but still, it's a different level of "perfect" than anything I could put together before.
+
+[[!meta author=Joey]]

From e418cc92f47a22f70125ea28b3d6509ebd52dc1a Mon Sep 17 00:00:00 2001
From: "http://adamspiers.myopenid.com/" 
Date: Thu, 29 Mar 2012 21:41:55 +0000
Subject: [PATCH 3312/8313] Added a comment: re-annexing previously annexed
 files

---
 ...ment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment

diff --git a/doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment b/doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment
new file mode 100644
index 0000000000..1793e686fa
--- /dev/null
+++ b/doc/forum/syncing_non-git_trees_with_git-annex/comment_3_6d8f399f0549eddd1d1f5c9c9a10c654._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://adamspiers.myopenid.com/"
+ nickname="Adam"
+ subject="re-annexing previously annexed files"
+ date="2012-03-29T21:41:54Z"
+ content="""
+Here's another handy command-line which annexes all files in repo B which have already been annexed in repo A:
+
+    git status --porcelain | sed -n '/^ T /{s///;p}' | xargs git annex add
+
+The 'T' outputted by git status for these files indicates a type change: it's a symlink to the annex in repo A, but a normal file in repo B.
+
+"""]]

From 69d58892e87623cc04c56917a4b9760e24558b9b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 1 Apr 2012 20:34:09 -0400
Subject: [PATCH 3313/8313] update sharebox urls to new project

---
 doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn | 2 +-
 doc/not.mdwn                                           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
index 7386841b2b..8eab9a34ac 100644
--- a/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
+++ b/doc/news/sharebox_a_FUSE_filesystem_for_git-annex.mdwn
@@ -1,7 +1,7 @@
 [[!meta title="sharebox: a FUSE filesystem for git-annex"]]
 
 Christophe-Marie Duquesne has just announced 
-[Sharebox](https://github.com/chmduquesne/sharebox), a FUSE filesystem
+[Sharebox](https://github.com/chmduquesne/sharebox-fs), a FUSE filesystem
 relying on git-annex:
 
 

diff --git a/doc/not.mdwn b/doc/not.mdwn
index ad278da0dd..a794cf721c 100644
--- a/doc/not.mdwn
+++ b/doc/not.mdwn
@@ -7,7 +7,7 @@
 
 * git-annex is not a filesystem or DropBox clone. But there
   is a FUSE filesystem built on top of git-annex, called 
-  [ShareBox](https://github.com/chmduquesne/sharebox), and there is
+  [ShareBox](https://github.com/chmduquesne/sharebox-fs), and there is
   interest in making it easy to use and covering some of the use
   cases supported by DropBox.
 

From 9924e44c112838506f9e2a46532765a931f167bb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 3 Apr 2012 12:30:41 -0400
Subject: [PATCH 3314/8313] close old bug with no followup

---
 doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
index df0fb50cc1..f2d301cd7c 100644
--- a/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
+++ b/doc/bugs/git_annex_unused_seems_to_check_for_current_path.mdwn
@@ -35,3 +35,5 @@ I am using git-annex version 836e71297b8e3b5bd6f89f7eb1198f59af985b0b
 >    but I don't know how to construct such an ignore.
 > 
 > --[[Joey]] 
+
+>> Closing as there is no followup. [[done]] --[[Joey]] 

From 62005d294e30285dee4645bf6633247214426573 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Apr 2012 14:53:53 -0400
Subject: [PATCH 3315/8313] room for a few more feed items

---
 doc/index.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index 9ba5d5c316..a54a1073e7 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -24,7 +24,7 @@ To get a feel for it, see the [[walkthrough]] or read about [[how_it_works]].
 [[Feeds]]:
 
 
-[[!inline pages="internal(feeds/*)" archive=yes show=5 feeds=no]]
+[[!inline pages="internal(feeds/*)" archive=yes show=8 feeds=no]]
 
 """]]
 

From 16acc507f3b814d01b30bc1f0e08e790785327ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Apr 2012 16:37:44 -0400
Subject: [PATCH 3316/8313] releasing version 3.20120405

---
 Makefile         | 2 +-
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index a0447f2d21..b935140576 100644
--- a/Makefile
+++ b/Makefile
@@ -59,7 +59,7 @@ install: all
 	fi
 
 test:
-	@if ! $(GHCMAKE) -O0 test; then \
+	@if ! $(GHCMAKE) -O0 test $(clibs); then \
 		echo "** failed to build the test suite" >&2; \
 		exit 1; \
 	elif ! ./test; then \
diff --git a/debian/changelog b/debian/changelog
index 5aecd7b7e0..6306bc859d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,10 @@
-git-annex (3.20120316) UNRELEASED; urgency=low
+git-annex (3.20120405) unstable; urgency=low
 
   * Rewrote free disk space checking code, moving the portability
     handling into a small C library.
   * status: Display amount of free disk space.
 
- -- Joey Hess   Wed, 21 Mar 2012 21:19:16 -0400
+ -- Joey Hess   Thu, 05 Apr 2012 16:19:10 -0400
 
 git-annex (3.20120315) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 9e37a6cb3c..f542f57541 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120316
+Version: 3.20120405
 Cabal-Version: >= 1.6
 License: GPL
 Maintainer: Joey Hess 

From e3150ead42ba0efaa1dfee20062c33116d70439e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Apr 2012 16:38:46 -0400
Subject: [PATCH 3317/8313] add news item for git-annex 3.20120405

---
 doc/news/version_3.20120229.mdwn | 4 ----
 doc/news/version_3.20120405.mdwn | 5 +++++
 2 files changed, 5 insertions(+), 4 deletions(-)
 delete mode 100644 doc/news/version_3.20120229.mdwn
 create mode 100644 doc/news/version_3.20120405.mdwn

diff --git a/doc/news/version_3.20120229.mdwn b/doc/news/version_3.20120229.mdwn
deleted file mode 100644
index 9b6d663a05..0000000000
--- a/doc/news/version_3.20120229.mdwn
+++ /dev/null
@@ -1,4 +0,0 @@
-git-annex 3.20120229 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix test suite to not require a unicode locale.
-   * Fix cabal build failure. Thanks, Sergei Trofimovich"""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120405.mdwn b/doc/news/version_3.20120405.mdwn
new file mode 100644
index 0000000000..c5a7ecc22a
--- /dev/null
+++ b/doc/news/version_3.20120405.mdwn
@@ -0,0 +1,5 @@
+git-annex 3.20120405 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Rewrote free disk space checking code, moving the portability
+     handling into a small C library.
+   * status: Display amount of free disk space."""]]
\ No newline at end of file

From 1424186c0b9b90cc8e80998148e10ba8d626d369 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 5 Apr 2012 16:39:50 -0400
Subject: [PATCH 3318/8313] cabal says I need a tighter version of cabal to use
 test-suite

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index f542f57541..805f804eb0 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,6 +1,6 @@
 Name: git-annex
 Version: 3.20120405
-Cabal-Version: >= 1.6
+Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
 Author: Joey Hess

From fcc08c59ec827bf6672b4288998cab4a98ebd20a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 6 Apr 2012 14:54:41 -0400
Subject: [PATCH 3319/8313] use unabbreviated size units in status

---
 Command/Status.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 40cefb5cc2..1ee36d8b47 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -165,7 +165,7 @@ bloom_info = stat "bloom filter size" $ json id $ do
 
 	-- Two bloom filters are used at the same time, so double the size
 	-- of one.
-	size <- roughSize memoryUnits True . (* 2) . fromIntegral . fst <$>
+	size <- roughSize memoryUnits False . (* 2) . fromIntegral . fst <$>
 		lift Command.Unused.bloomBitsHashes
 
 	return $ size ++ note
@@ -177,7 +177,7 @@ disk_size = stat "available local disk space" $ json id $ lift $
 		<*> inRepo (getDiskFree . gitAnnexDir)
 	where
 		calcfree reserve (Just have) =
-			roughSize storageUnits True $ nonneg $ have - reserve
+			roughSize storageUnits False $ nonneg $ have - reserve
 		calcfree _ _ = "unknown"
 		nonneg x
 			| x >= 0 = x

From 62c69e7e25c3d60098914e6cc0be4ff2b9a8db00 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Apr 2012 15:50:34 -0400
Subject: [PATCH 3320/8313] Disable diskfree on kfreebsd, as I have a build
 failure on kfreebsd-i386 that is quite likely caused by it.

---
 Utility/diskfree.c | 1 +
 debian/changelog   | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/Utility/diskfree.c b/Utility/diskfree.c
index 6e9eda930f..968d2b8b50 100644
--- a/Utility/diskfree.c
+++ b/Utility/diskfree.c
@@ -27,6 +27,7 @@
 # include 
 # define STATCALL statfs64
 # define STATSTRUCT statfs
+# define UNKNOWN /* temporarily disabled; trying to nail down a build failure */
 #else
 #if defined (__linux__)
 /* This is a POSIX standard, so might also work elsewhere. */
diff --git a/debian/changelog b/debian/changelog
index 6306bc859d..581c7d04fb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120406) UNRELEASED; urgency=low
+
+  * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386
+    that is quite likely caused by it.
+
+ -- Joey Hess   Sat, 07 Apr 2012 15:49:41 -0400
+
 git-annex (3.20120405) unstable; urgency=low
 
   * Rewrote free disk space checking code, moving the portability

From 29acf62ba3db22b6764db164f221dec4e0a41d3b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Apr 2012 15:58:13 -0400
Subject: [PATCH 3321/8313] releasing version 3.20120406

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 581c7d04fb..3266d85c03 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,9 +1,9 @@
-git-annex (3.20120406) UNRELEASED; urgency=low
+git-annex (3.20120406) unstable; urgency=low
 
   * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386
     that is quite likely caused by it.
 
- -- Joey Hess   Sat, 07 Apr 2012 15:49:41 -0400
+ -- Joey Hess   Sat, 07 Apr 2012 15:50:36 -0400
 
 git-annex (3.20120405) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 805f804eb0..e047bb2858 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120405
+Version: 3.20120406
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From 713ce1e623a8ccc7b6d8f581c04d6432b916797b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 7 Apr 2012 15:59:05 -0400
Subject: [PATCH 3322/8313] add news item for git-annex 3.20120406

---
 doc/news/version_3.20120406.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/news/version_3.20120406.mdwn

diff --git a/doc/news/version_3.20120406.mdwn b/doc/news/version_3.20120406.mdwn
new file mode 100644
index 0000000000..b3df304e27
--- /dev/null
+++ b/doc/news/version_3.20120406.mdwn
@@ -0,0 +1,4 @@
+git-annex 3.20120406 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386
+     that is quite likely caused by it."""]]
\ No newline at end of file

From e49f7ea62fce9efd66b681eed5b7c0d04eb35bfa Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 8 Apr 2012 15:47:29 +0000
Subject: [PATCH 3323/8313]

---
 doc/bugs/git_annex_add_..._adds_too_much.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/git_annex_add_..._adds_too_much.mdwn

diff --git a/doc/bugs/git_annex_add_..._adds_too_much.mdwn b/doc/bugs/git_annex_add_..._adds_too_much.mdwn
new file mode 100644
index 0000000000..1079c5585a
--- /dev/null
+++ b/doc/bugs/git_annex_add_..._adds_too_much.mdwn
@@ -0,0 +1,15 @@
+When a hidden file (starting with a dot) is git-annex add'ed, other non-tracked files are also added 
+
+What steps will reproduce the problem?
+$ touch a .b
+$ git annex add .b
+add a (checksum...) ok
+add .b (checksum...) ok
+(Recording state in git...)
+
+What is the expected output? What do you see instead?
+Only file .b should be added.
+
+What version of git-annex are you using? On what operating system?
+3.20120406
+(same problem with version 3.20120123)

From 4309f363fb2e779856092f10b53fa7c6e88a6ea3 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Sun, 8 Apr 2012 15:49:07 +0000
Subject: [PATCH 3324/8313]

---
 doc/bugs/git_annex_add_..._adds_too_much.mdwn | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/git_annex_add_..._adds_too_much.mdwn b/doc/bugs/git_annex_add_..._adds_too_much.mdwn
index 1079c5585a..5ef9bc577e 100644
--- a/doc/bugs/git_annex_add_..._adds_too_much.mdwn
+++ b/doc/bugs/git_annex_add_..._adds_too_much.mdwn
@@ -1,15 +1,23 @@
 When a hidden file (starting with a dot) is git-annex add'ed, other non-tracked files are also added 
 
 What steps will reproduce the problem?
+
 $ touch a .b
+
 $ git annex add .b
+
 add a (checksum...) ok
+
 add .b (checksum...) ok
+
 (Recording state in git...)
 
+
 What is the expected output? What do you see instead?
+
 Only file .b should be added.
 
 What version of git-annex are you using? On what operating system?
+
 3.20120406
-(same problem with version 3.20120123)
+(same problem with version 3.20120123) on Debian.

From 182778d664f5e7fb66e1422a6d429a604d083fa3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 8 Apr 2012 12:25:54 -0400
Subject: [PATCH 3325/8313] bugfix: Adding a dotfile also caused all
 non-dotfiles to be added.

When only a dotfile was specified, the list of non-dotfiles was empty,
triggering the fallback behavior of finding all files.
---
 Seek.hs                                       | 8 +++++---
 debian/changelog                              | 6 ++++++
 doc/bugs/git_annex_add_..._adds_too_much.mdwn | 2 ++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/Seek.hs b/Seek.hs
index 7a53fc04b2..8d4f917e72 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -29,12 +29,14 @@ withFilesInGit a params = prepFiltered a $ seekHelper LsFiles.inRepo params
 withFilesNotInGit :: (FilePath -> CommandStart) -> CommandSeek
 withFilesNotInGit a params = do
 	{- dotfiles are not acted on unless explicitly listed -}
-	files <- filter (not . dotfile) <$> seek ps
-	dotfiles <- if null dotps then return [] else seek dotps
+	files <- filter (not . dotfile) <$>
+		seekunless (null ps && not (null params)) ps
+	dotfiles <- seekunless (null dotps) dotps
 	prepFiltered a $ return $ preserveOrder params (files++dotfiles)
 	where
 		(dotps, ps) = partition dotfile params
-		seek l = do
+		seekunless True _ = return []
+		seekunless _ l = do
 			force <- Annex.getState Annex.force
 			g <- gitRepo
 			liftIO $ (\p -> LsFiles.notInRepo force p g) l
diff --git a/debian/changelog b/debian/changelog
index 3266d85c03..2372820cc5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120407) UNRELEASED; urgency=low
+
+  * bugfix: Adding a dotfile also caused all non-dotfiles to be added.
+
+ -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
+
 git-annex (3.20120406) unstable; urgency=low
 
   * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386
diff --git a/doc/bugs/git_annex_add_..._adds_too_much.mdwn b/doc/bugs/git_annex_add_..._adds_too_much.mdwn
index 5ef9bc577e..4eb46455f9 100644
--- a/doc/bugs/git_annex_add_..._adds_too_much.mdwn
+++ b/doc/bugs/git_annex_add_..._adds_too_much.mdwn
@@ -21,3 +21,5 @@ What version of git-annex are you using? On what operating system?
 
 3.20120406
 (same problem with version 3.20120123) on Debian.
+
+> Thanks for reporting this bug, I've fixed it in git. [[done]] --[[Joey]] 

From 83ab75fd48438aaf0983ef9586f4b770d5b696a6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Tue, 10 Apr 2012 15:01:35 +0000
Subject: [PATCH 3326/8313]

---
 doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn

diff --git a/doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn b/doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn
new file mode 100644
index 0000000000..49c732e226
--- /dev/null
+++ b/doc/forum/How_to_handle_the_git-annex_branch__63__.mdwn
@@ -0,0 +1,5 @@
+Hi!
+
+When I update one repository to/from another, am I to push/pull only the master branch (and other branches I'd have created myself) or also the git-annex branch?
+
+From what I understand, the git-annex branch is local to one repository and has no interest in being merged in another, but some examples make use of git pull --all (which merges distant git-annex branch into local).

From dce77e9e2bc1be01a901c27c4a1772f72f6e7c4f Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 10 Apr 2012 16:05:40 +0000
Subject: [PATCH 3327/8313] Added a comment

---
 .../comment_1_800bd55b322e72f229882d7fd3888b14._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment

diff --git a/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment
new file mode 100644
index 0000000000..6c5a5144f3
--- /dev/null
+++ b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_1_800bd55b322e72f229882d7fd3888b14._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-10T16:05:40Z"
+ content="""
+The git-annex branch is how the annex information for the different repositories is communicated around, so yes, you need to push/pull it. 
+"""]]

From 767ab5fc732893595ce81506fc7e1bcff9400c1b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Tue, 10 Apr 2012 16:29:35 +0000
Subject: [PATCH 3328/8313] Added a comment

---
 .../comment_2_c7ff652d459b3a23c1386dc56863ca98._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment

diff --git a/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment
new file mode 100644
index 0000000000..e4da8cbe6e
--- /dev/null
+++ b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
+ nickname="Yves"
+ subject="comment 2"
+ date="2012-04-10T16:29:33Z"
+ content="""
+Okay, I was under the impression that the git-annex branch was updated when getting/dropping files in local repository.
+Are those relevant information for distant repos?
+"""]]

From 75ff4f77b25536fe50560ce6627bac321ca5fe4e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Tue, 10 Apr 2012 16:30:09 +0000
Subject: [PATCH 3329/8313] removed

---
 .../comment_2_c7ff652d459b3a23c1386dc56863ca98._comment  | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment

diff --git a/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment b/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment
deleted file mode 100644
index e4da8cbe6e..0000000000
--- a/doc/forum/How_to_handle_the_git-annex_branch__63__/comment_2_c7ff652d459b3a23c1386dc56863ca98._comment
+++ /dev/null
@@ -1,9 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
- nickname="Yves"
- subject="comment 2"
- date="2012-04-10T16:29:33Z"
- content="""
-Okay, I was under the impression that the git-annex branch was updated when getting/dropping files in local repository.
-Are those relevant information for distant repos?
-"""]]

From af61a5f986d1acb39901c78f23a2859ae22d5098 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:36:34 +0000
Subject: [PATCH 3330/8313]

---
 .../Problem_with_bup:_cannot_lock_refs.mdwn   | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
new file mode 100644
index 0000000000..8a773c4cb1
--- /dev/null
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -0,0 +1,29 @@
+Hi!
+
+Using bup for storing seems a good idea to save space, but I still have a problem when trying to copy files to my local git repo.
+I have two partitions:
+- /Data (NTFS)
+- / (ext4)
+
+I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone
+I added the remote bup to AudioClone by doing:
+git annex initremote mybup type=bup encryption=none buprepo=
+
+But when I try to copy some files that I have previously got by "git annex get" by doing:
+git annex copy /home/me/AudioClone/someartist/somealbum --to mybup
+it fails and tells me:
+
+copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) 
+fatal: Cannot lock the ref 'refs/heads/WORM-s7351771-m1318841909--01 Morituri Te Salutant.flac'.
+Traceback (most recent call last):
+  File "/usr/lib/bup/cmd/bup-split", line 170, in 
+    git.update_ref(refname, commit, oldref)
+  File "/usr/lib/bup/bup/git.py", line 835, in update_ref
+    _git_wait('git update-ref', p)
+  File "/usr/lib/bup/bup/git.py", line 930, in _git_wait
+    raise GitError('%s returned %d' % (cmd, rv))
+bup.git.GitError: git update-ref returned 128
+
+for each file, **except for the album cover file**, which is a simple JPG that bup doesn't try to split. This one gets copied nicely but the big FLAC files don't.
+
+I tried to restart my session, in case bup adds my username to a group or something.

From 6b07fb19dacccea09d69d6ed66b0fa01043ac039 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:37:06 +0000
Subject: [PATCH 3331/8313]

---
 .../Problem_with_bup:_cannot_lock_refs.mdwn   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index 8a773c4cb1..63eda36b97 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -13,16 +13,16 @@ But when I try to copy some files that I have previously got by "git annex get"
 git annex copy /home/me/AudioClone/someartist/somealbum --to mybup
 it fails and tells me:
 
-copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) 
-fatal: Cannot lock the ref 'refs/heads/WORM-s7351771-m1318841909--01 Morituri Te Salutant.flac'.
-Traceback (most recent call last):
-  File "/usr/lib/bup/cmd/bup-split", line 170, in 
-    git.update_ref(refname, commit, oldref)
-  File "/usr/lib/bup/bup/git.py", line 835, in update_ref
-    _git_wait('git update-ref', p)
-  File "/usr/lib/bup/bup/git.py", line 930, in _git_wait
-    raise GitError('%s returned %d' % (cmd, rv))
-bup.git.GitError: git update-ref returned 128
+    copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) 
+    fatal: Cannot lock the ref 'refs/heads/WORM-s7351771-m1318841909--01 Morituri Te Salutant.flac'.
+    Traceback (most recent call last):
+      File "/usr/lib/bup/cmd/bup-split", line 170, in 
+        git.update_ref(refname, commit, oldref)
+      File "/usr/lib/bup/bup/git.py", line 835, in update_ref
+        _git_wait('git update-ref', p)
+      File "/usr/lib/bup/bup/git.py", line 930, in _git_wait
+        raise GitError('%s returned %d' % (cmd, rv))
+    bup.git.GitError: git update-ref returned 128
 
 for each file, **except for the album cover file**, which is a simple JPG that bup doesn't try to split. This one gets copied nicely but the big FLAC files don't.
 

From 14ec54118b7f93ac20d41593b5480d4f58add65f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:37:26 +0000
Subject: [PATCH 3332/8313]

---
 doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index 63eda36b97..858569a81a 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -2,12 +2,15 @@ Hi!
 
 Using bup for storing seems a good idea to save space, but I still have a problem when trying to copy files to my local git repo.
 I have two partitions:
+
 - /Data (NTFS)
+
 - / (ext4)
 
 I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone
 I added the remote bup to AudioClone by doing:
-git annex initremote mybup type=bup encryption=none buprepo=
+
+    git annex initremote mybup type=bup encryption=none buprepo=
 
 But when I try to copy some files that I have previously got by "git annex get" by doing:
 git annex copy /home/me/AudioClone/someartist/somealbum --to mybup

From e27934c42e729cd0a6dad17c988a10e9cf37f05d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:37:52 +0000
Subject: [PATCH 3333/8313]

---
 doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index 858569a81a..3badb33be9 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -7,7 +7,7 @@ I have two partitions:
 
 - / (ext4)
 
-I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone
+I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone.
 I added the remote bup to AudioClone by doing:
 
     git annex initremote mybup type=bup encryption=none buprepo=

From 7b8d8daeceb292e3daecb9a222ab3ca463761f32 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:38:26 +0000
Subject: [PATCH 3334/8313]

---
 doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index 3badb33be9..1778e63a73 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -13,7 +13,9 @@ I added the remote bup to AudioClone by doing:
     git annex initremote mybup type=bup encryption=none buprepo=
 
 But when I try to copy some files that I have previously got by "git annex get" by doing:
-git annex copy /home/me/AudioClone/someartist/somealbum --to mybup
+
+    git annex copy /home/me/AudioClone/someartist/somealbum --to mybup
+
 it fails and tells me:
 
     copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) 

From 6e128936bc3cfc8aaa5b95c95cceb74449a5dbdd Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:39:34 +0000
Subject: [PATCH 3335/8313]

---
 doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index 1778e63a73..cb217eb1ea 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -32,3 +32,5 @@ it fails and tells me:
 for each file, **except for the album cover file**, which is a simple JPG that bup doesn't try to split. This one gets copied nicely but the big FLAC files don't.
 
 I tried to restart my session, in case bup adds my username to a group or something.
+
+(I'm using Ubuntu 11.10)

From 2d3efab70d9e59554738921e1a1ae14d35222f18 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlxg9HfyLFn0E87p8Em9P5G9YBUr8JYrkE"
 
Date: Wed, 11 Apr 2012 09:42:13 +0000
Subject: [PATCH 3336/8313]

---
 doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index cb217eb1ea..a9b411c3a7 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -14,7 +14,7 @@ I added the remote bup to AudioClone by doing:
 
 But when I try to copy some files that I have previously got by "git annex get" by doing:
 
-    git annex copy /home/me/AudioClone/someartist/somealbum --to mybup
+    [~/AudioClone]$ git annex copy someartist/somealbum --to mybup
 
 it fails and tells me:
 

From 028972c80f01122350dfc7520ffa8461d403aaa1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 12:00:39 -0400
Subject: [PATCH 3337/8313] move bug report to right place; analysis

---
 .../Problem_with_bup:_cannot_lock_refs.mdwn   | 50 +++++++++++++++++++
 .../Problem_with_bup:_cannot_lock_refs.mdwn   | 37 ++------------
 2 files changed, 53 insertions(+), 34 deletions(-)
 create mode 100644 doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn

diff --git a/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn
new file mode 100644
index 0000000000..ea34075779
--- /dev/null
+++ b/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -0,0 +1,50 @@
+Hi!
+
+Using bup for storing seems a good idea to save space, but I still have a problem when trying to copy files to my local git repo.
+I have two partitions:
+
+- /Data (NTFS)
+
+- / (ext4)
+
+I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone.
+I added the remote bup to AudioClone by doing:
+
+    git annex initremote mybup type=bup encryption=none buprepo=
+
+But when I try to copy some files that I have previously got by "git annex get" by doing:
+
+    [~/AudioClone]$ git annex copy someartist/somealbum --to mybup
+
+it fails and tells me:
+
+    copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) 
+    fatal: Cannot lock the ref 'refs/heads/WORM-s7351771-m1318841909--01 Morituri Te Salutant.flac'.
+    Traceback (most recent call last):
+      File "/usr/lib/bup/cmd/bup-split", line 170, in 
+        git.update_ref(refname, commit, oldref)
+      File "/usr/lib/bup/bup/git.py", line 835, in update_ref
+        _git_wait('git update-ref', p)
+      File "/usr/lib/bup/bup/git.py", line 930, in _git_wait
+        raise GitError('%s returned %d' % (cmd, rv))
+    bup.git.GitError: git update-ref returned 128
+
+for each file, **except for the album cover file**, which is a simple JPG that bup doesn't try to split. This one gets copied nicely but the big FLAC files don't.
+
+I tried to restart my session, in case bup adds my username to a group or something.
+
+(I'm using Ubuntu 11.10)
+
+> Apparently bup-split does not allow storing data using filenames with
+> spaces in them. I can reproduce the same bug using the same filename;
+> if I remove the spaces all is well.
+> 
+> Since bup-split -n uses git branches, I guess git-annex needs to avoid
+> giving it any names containing spaces, or anything else not allowed
+> in a git branch name. The rules for legal git branch names are quite complex
+> (see git-check-ref-format(1)) so it will take me some times to code
+> this up.
+> 
+> A workaround is to switch to the SHA256 backend
+> (`git annex migrate --backend=SHA256`), which avoids spaces in its keys.
+> --[[Joey]]
diff --git a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
index a9b411c3a7..897d3b4098 100644
--- a/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/forum/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -1,36 +1,5 @@
 Hi!
 
-Using bup for storing seems a good idea to save space, but I still have a problem when trying to copy files to my local git repo.
-I have two partitions:
-
-- /Data (NTFS)
-
-- / (ext4)
-
-I turned the directory /Data/Audio into a git-annex repo, and cloned it into /home/me/AudioClone.
-I added the remote bup to AudioClone by doing:
-
-    git annex initremote mybup type=bup encryption=none buprepo=
-
-But when I try to copy some files that I have previously got by "git annex get" by doing:
-
-    [~/AudioClone]$ git annex copy someartist/somealbum --to mybup
-
-it fails and tells me:
-
-    copy Order To Die/01 Morituri Te Salutant.flac (to mybup...) 
-    fatal: Cannot lock the ref 'refs/heads/WORM-s7351771-m1318841909--01 Morituri Te Salutant.flac'.
-    Traceback (most recent call last):
-      File "/usr/lib/bup/cmd/bup-split", line 170, in 
-        git.update_ref(refname, commit, oldref)
-      File "/usr/lib/bup/bup/git.py", line 835, in update_ref
-        _git_wait('git update-ref', p)
-      File "/usr/lib/bup/bup/git.py", line 930, in _git_wait
-        raise GitError('%s returned %d' % (cmd, rv))
-    bup.git.GitError: git update-ref returned 128
-
-for each file, **except for the album cover file**, which is a simple JPG that bup doesn't try to split. This one gets copied nicely but the big FLAC files don't.
-
-I tried to restart my session, in case bup adds my username to a group or something.
-
-(I'm using Ubuntu 11.10)
+> Hi.. Please file bug reports in [[bugs]], not in the forum. 
+> Moved this post to [[bugs/Problem_with_bup:_cannot_lock_refs]]
+--[[Joey]] 

From 0be6ebb0aad2cc9135479f3a3725e7c21add4989 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 12:21:54 -0400
Subject: [PATCH 3338/8313] added a git ref legality checker

git-check-ref-format is .. wow. Good design on one level, but what a mess.
---
 Git/Ref.hs | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/Git/Ref.hs b/Git/Ref.hs
index f483aede03..c7abdb00aa 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -11,6 +11,8 @@ import Common
 import Git
 import Git.Command
 
+import Data.Char (chr)
+
 {- Converts a fully qualified git ref into a user-visible string. -}
 describe :: Ref -> String
 describe = show . base
@@ -61,3 +63,24 @@ matchingUniq :: Ref -> Repo -> IO [(Ref, Branch)]
 matchingUniq ref repo = nubBy uniqref <$> matching ref repo
 	where
 		uniqref (a, _) (b, _) = a == b
+
+{- Checks if a String is a legal git ref name.
+ -
+ - The rules for this are complex; see git-check-ref-format(1) -}
+legalRef :: Bool -> String -> Bool
+legalRef allowonelevel s
+	| any ("." `isPrefixOf`) pathbits = False
+	| any (".lock" `isSuffixOf`) pathbits = False
+	| not allowonelevel && length pathbits < 2 = False
+	| ".." `isInfixOf` s = False
+	| any (\c -> [c] `isInfixOf` s) illegalchars = False
+	| "/" `isPrefixOf` s = False
+	| "/" `isSuffixOf` s = False
+	| "//" `isInfixOf` s = False
+	| "." `isSuffixOf` s = False
+	| "@{" `isInfixOf` s = False
+	| otherwise = True
+	where
+		pathbits = split "/" s
+		illegalchars = " ~^:?*[\\" ++ controlchars
+		controlchars = chr 0o177 : [chr 0 .. chr (0o40-1)]

From 378f61d0ef5564aab28442f09b2462ce7013ce1b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 12:29:31 -0400
Subject: [PATCH 3339/8313] nicer style; also empty refs are implicitly not
 allowed

---
 Git/Ref.hs | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/Git/Ref.hs b/Git/Ref.hs
index c7abdb00aa..29b69de9b5 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -68,19 +68,25 @@ matchingUniq ref repo = nubBy uniqref <$> matching ref repo
  -
  - The rules for this are complex; see git-check-ref-format(1) -}
 legalRef :: Bool -> String -> Bool
-legalRef allowonelevel s
-	| any ("." `isPrefixOf`) pathbits = False
-	| any (".lock" `isSuffixOf`) pathbits = False
-	| not allowonelevel && length pathbits < 2 = False
-	| ".." `isInfixOf` s = False
-	| any (\c -> [c] `isInfixOf` s) illegalchars = False
-	| "/" `isPrefixOf` s = False
-	| "/" `isSuffixOf` s = False
-	| "//" `isInfixOf` s = False
-	| "." `isSuffixOf` s = False
-	| "@{" `isInfixOf` s = False
-	| otherwise = True
+legalRef allowonelevel s = all (== False) illegal
 	where
+		illegal =
+			[ any ("." `isPrefixOf`) pathbits
+			, any (".lock" `isSuffixOf`) pathbits
+			, not allowonelevel && length pathbits < 2
+			, contains ".."
+			, any (\c -> contains [c]) illegalchars
+			, begins "/"
+			, ends "/"
+			, contains "//"
+			, ends "."
+			, contains "@{"
+			, null s
+			]
+		contains v = v `isInfixOf` s
+		ends v = v `isSuffixOf` s
+		begins v = v `isPrefixOf` s
+
 		pathbits = split "/" s
 		illegalchars = " ~^:?*[\\" ++ controlchars
 		controlchars = chr 0o177 : [chr 0 .. chr (0o40-1)]

From c924542e61607d725fbfa51ffbf88d825b4d3382 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 12:45:05 -0400
Subject: [PATCH 3340/8313] bup: Properly handle key names with spaces or other
 things that are not legal git refs.

Continue using the key name as bup ref name, to preserve backwards
compatability, unless it is an illegal git ref. In that case, use a sha256
of the key name instead.
---
 Git/Ref.hs                                    |  4 ++--
 Remote/Bup.hs                                 | 20 +++++++++++++++----
 debian/changelog                              |  2 ++
 .../Problem_with_bup:_cannot_lock_refs.mdwn   |  2 ++
 4 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/Git/Ref.hs b/Git/Ref.hs
index 29b69de9b5..ee2f021871 100644
--- a/Git/Ref.hs
+++ b/Git/Ref.hs
@@ -67,8 +67,8 @@ matchingUniq ref repo = nubBy uniqref <$> matching ref repo
 {- Checks if a String is a legal git ref name.
  -
  - The rules for this are complex; see git-check-ref-format(1) -}
-legalRef :: Bool -> String -> Bool
-legalRef allowonelevel s = all (== False) illegal
+legal :: Bool -> String -> Bool
+legal allowonelevel s = all (== False) illegal
 	where
 		illegal =
 			[ any ("." `isPrefixOf`) pathbits
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 54aff75058..1081815944 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -17,11 +17,14 @@ import qualified Git
 import qualified Git.Command
 import qualified Git.Config
 import qualified Git.Construct
+import qualified Git.Ref
 import Config
 import Remote.Helper.Ssh
 import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
+import Data.ByteString.Lazy.UTF8 (fromString)
+import Data.Digest.Pure.SHA
 
 type BupRepo = String
 
@@ -103,7 +106,7 @@ bupSplitParams r buprepo k src = do
 	let os = map Param $ words o
 	showOutput -- make way for bup output
 	return $ bupParams "split" buprepo 
-		(os ++ [Param "-n", Param (show k), src])
+		(os ++ [Param "-n", Param (bupRef k), src])
 
 store :: Git.Repo -> BupRepo -> Key -> Annex Bool
 store r buprepo k = do
@@ -121,7 +124,7 @@ storeEncrypted r buprepo (cipher, enck) k = do
 
 retrieve :: BupRepo -> Key -> FilePath -> Annex Bool
 retrieve buprepo k f = do
-	let params = bupParams "join" buprepo [Param $ show k]
+	let params = bupParams "join" buprepo [Param $ bupRef k]
 	liftIO $ catchBoolIO $ do
 		tofile <- openFile f WriteMode
 		pipeBup params Nothing (Just tofile)
@@ -131,7 +134,7 @@ retrieveCheap _ _ _ = return False
 
 retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool
 retrieveEncrypted buprepo (cipher, enck) _ f = do
-	let params = bupParams "join" buprepo [Param $ show enck]
+	let params = bupParams "join" buprepo [Param $ bupRef enck]
 	liftIO $ catchBoolIO $ do
 		(pid, h) <- hPipeFrom "bup" $ toCommand params
 		withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f
@@ -158,7 +161,7 @@ checkPresent r bupr k
 	where
 		params = 
 			[ Params "show-ref --quiet --verify"
-			, Param $ "refs/heads/" ++ show k]
+			, Param $ "refs/heads/" ++ bupRef k]
 
 {- Store UUID in the annex.uuid setting of the bup repository. -}
 storeBupUUID :: UUID -> BupRepo -> Annex ()
@@ -230,5 +233,14 @@ bup2GitRemote r
 				| "/" `isPrefixOf` d = d
 				| otherwise = "/~/" ++ d
 
+{- Converts a key into a git ref name, which bup-split -n will use to point
+ - to it. -}
+bupRef :: Key -> String
+bupRef k
+	| Git.Ref.legal True shown = shown
+	| otherwise = "git-annex-" ++ showDigest (sha256 (fromString shown))
+	where
+		shown = show k
+
 bupLocal :: BupRepo -> Bool
 bupLocal = notElem ':'
diff --git a/debian/changelog b/debian/changelog
index 2372820cc5..274879f6e4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 git-annex (3.20120407) UNRELEASED; urgency=low
 
   * bugfix: Adding a dotfile also caused all non-dotfiles to be added.
+  * bup: Properly handle key names with spaces or other things that are
+    not legal git refs.
 
  -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
 
diff --git a/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn b/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn
index ea34075779..f8df1f0822 100644
--- a/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn
+++ b/doc/bugs/Problem_with_bup:_cannot_lock_refs.mdwn
@@ -48,3 +48,5 @@ I tried to restart my session, in case bup adds my username to a group or someth
 > A workaround is to switch to the SHA256 backend
 > (`git annex migrate --backend=SHA256`), which avoids spaces in its keys.
 > --[[Joey]]
+
+>> Now fixed in git. [[done]] --[[Joey]] 

From 037961aa5d5ac3c4b81642a8bf0962abedd35f26 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 14:22:59 -0400
Subject: [PATCH 3341/8313] update

---
 doc/special_remotes.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 2d0474ffe5..2f6020461f 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -1,6 +1,6 @@
-Most [[backends]] can transfer data to and from configured git remotes.
+git-annex can transfer data to and from configured git remotes.
 Normally those remotes are normal git repositories (bare and non-bare;
-local and remote), that store the file contents in their own git annex
+local and remote), that store the file contents in their own git-annex
 directory.
 
 But, git-annex also extends git's concept of remotes, with these special

From b133a76f96a340c98f5698ca991e9433165c0817 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 20:03:49 -0400
Subject: [PATCH 3342/8313] recursive inotify thing

Recursive inotify has beaten me before, with its bad design and races,
but not this time! (I think.) This is able to follow the strongest
filesystem traffic I can throw at it, and robustly notices every file
add and delete. Mostly that's down to Haskell having a quite nice threaded
inotify library (that does its own buffering). A key insight was realizing
that the inotify directory add race could be dealt with by scanning for
files inside newly added directories.

TODO: Add support for freebsd/osx kqueue; see
http://hackage.haskell.org/package/kqueue

Can a git-annex-monitor be far off?
---
 Utility/Inotify.hs | 66 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 Utility/Inotify.hs

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
new file mode 100644
index 0000000000..bf36814685
--- /dev/null
+++ b/Utility/Inotify.hs
@@ -0,0 +1,66 @@
+module Utility.Inotify where
+
+import Common hiding (isDirectory)
+import System.INotify
+import qualified System.Posix.Files as Files
+
+demo :: IO String
+demo = withINotify $ \i -> do
+	watchDir i add del "/home/joey/tmp/me"
+	putStrLn "started"
+	getLine -- wait for exit
+	where
+		add file = putStrLn $ "add " ++ file
+		del file = putStrLn $ "del " ++ file
+
+{- Watches for changes to files in a directory, and all its subdirectories,
+ - using inotify. This function returns after its initial setup is
+ - complete, leaving a thread running. Then callbacks are made for adding
+ - and deleting files.
+ -
+ - Inotify is weak at recursive directory watching; the whole directory
+ - tree must be walked and watches set explicitly for each subdirectory.
+ -
+ - To notice newly created subdirectories, inotify is used, and
+ - watches are registered for those directories. There is a race there;
+ - things can be added to a directory before the watch gets registered.
+ -
+ - To close the inotify race, each time a new directory is found, it also 
+ - recursively scans it, assuming all files in it were just added,
+ - and registering each subdirectory.
+ -
+ - Note: Due to the race amelioration, multiple add events may occur
+ - for the same file.
+ - 
+ - Note: Moving a file may involve deleting it from its old location and
+ - adding it to the new location. 
+ - 
+ - Note: Modification of files is not detected, and it's assumed that when
+ - a file that was open for write is closed, it's done being written
+ - to, and can be added.
+ -
+ - Note: inotify has a limit to the number of watches allowed,
+ - /proc/sys/fs/inotify/max_user_watches (default 8192).
+ - So This will fail if there are too many subdirectories.
+ -}
+watchDir :: INotify -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
+watchDir i add del dir = watchDir' False i add del dir
+watchDir' :: Bool -> INotify -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
+watchDir' scan i add del dir = do
+	_ <- addWatch i [MoveIn, MoveOut, Create, Delete, CloseWrite] dir go
+	_ <- mapM walk =<< dirContents dir
+	return ()
+	where
+		recurse = watchDir' scan i add del
+		walk f = ifM (Files.isDirectory <$> getFileStatus f)
+			( recurse f
+			, if scan then add f else return ()
+			)
+		a <@> f = a $ dir  f
+		go (Created { isDirectory = False }) = return ()
+		go (Created { filePath = subdir }) = recurse <@> subdir
+		go (Closed { maybeFilePath = Just f }) = add <@> f
+		go (MovedIn { isDirectory = False, filePath = f }) = add <@> f
+		go (MovedOut { isDirectory = False, filePath = f }) = del <@> f
+		go (Deleted { isDirectory = False, filePath = f }) = del <@> f
+		go _ = return ()

From 1f34bf9acb3efa191e4f09f7e2f300cd759262a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 11 Apr 2012 20:28:01 -0400
Subject: [PATCH 3343/8313] add waitForTermination

---
 Utility/Inotify.hs | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index bf36814685..56906466b9 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -1,14 +1,19 @@
+{-# LANGUAGE CPP #-}
+
 module Utility.Inotify where
 
 import Common hiding (isDirectory)
 import System.INotify
 import qualified System.Posix.Files as Files
+import System.Posix.Terminal
+import Control.Concurrent.MVar
+import System.Posix.Signals
 
-demo :: IO String
+demo :: IO ()
 demo = withINotify $ \i -> do
 	watchDir i add del "/home/joey/tmp/me"
 	putStrLn "started"
-	getLine -- wait for exit
+	waitForTermination
 	where
 		add file = putStrLn $ "add " ++ file
 		del file = putStrLn $ "del " ++ file
@@ -64,3 +69,16 @@ watchDir' scan i add del dir = do
 		go (MovedOut { isDirectory = False, filePath = f }) = del <@> f
 		go (Deleted { isDirectory = False, filePath = f }) = del <@> f
 		go _ = return ()
+
+{- Pauses the main thread, letting children run until program termination. -}
+waitForTermination :: IO ()
+waitForTermination = do
+	mv <- newEmptyMVar
+	check softwareTermination mv
+	whenM (queryTerminal stdInput) $
+		check keyboardSignal mv
+	takeMVar mv
+	where
+		check sig mv = do
+			installHandler sig (CatchOnce $ putMVar mv ()) Nothing
+			return ()

From 1625fed6e854bec83b03342b7d20a99c939ae290 Mon Sep 17 00:00:00 2001
From: "http://mildred.pip.verisignlabs.com/"
 
Date: Thu, 12 Apr 2012 15:42:26 +0000
Subject: [PATCH 3344/8313]

---
 ...oes_not_exist__40__v_3.20111231__41__.mdwn | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__.mdwn

diff --git a/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__.mdwn b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__.mdwn
new file mode 100644
index 0000000000..633ff6f9bc
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__.mdwn
@@ -0,0 +1,26 @@
+First, I'm using a 2011 version because i'm getting this kind of errors from cabal install (on Fedora 16 and 17):
+
+    $ cabal install git-annex --bindir=$HOME/.local/bin
+    Resolving dependencies...
+    cabal: cannot configure git-annex-3.20120406. It requires base >=4.5 && <5
+    For the dependency on base >=4.5 && <5 there are these packages: base-4.5.0.0.
+    However none of them are available.
+    base-4.5.0.0 was excluded because of the top level dependency base -any
+
+So I installed a 2011 version and it worked.
+
+Now, when I add some files in the git annex repository I get an error:
+
+    $ git annex add Photo\ Library/2010/06/28/IMG_4926.JPG
+    add Photo Library/2010/06/28/IMG_4926.JPG (checksum...)
+    git-annex: Photo Library/2010/06/28/IMG_4926.JPG: getFileStatus: does not exist (No such file or directory)
+    failed
+    git-annex: add: 1 failed
+
+None of the other files in the same directory are a problem. The file content is not a problem either as I can move the file elsewhere and git annex add it w/o any problem. It's this file in this directory that causes the problem.
+
+Something interesting though. If I move the file elsewhere, and git annex add it, there is no problem. Now, if I git mv the file back into its original location, and git annex fix the file, the symbolic link is wrong: instead of pointing to `../../../../.git/annex/objects/somefile` it points to  `../../../annex/objects/somefile` (notice the missing `../.git/` part of the path).
+
+I can fix that by hand, and it works well, but that's very annoying. There are not much files having that bug though.
+
+[Mildred](http://mildred.fr)

From 5d0d9c791cedcf49c6483be29c4bee953d03962c Mon Sep 17 00:00:00 2001
From: "http://mildred.pip.verisignlabs.com/"
 
Date: Thu, 12 Apr 2012 15:46:54 +0000
Subject: [PATCH 3345/8313] Added a comment: Manually fixing links doesn't even
 work

---
 .../comment_1_990197bf01351dc1ccbe1940d5084adb._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment

diff --git a/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment
new file mode 100644
index 0000000000..7164de760d
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_1_990197bf01351dc1ccbe1940d5084adb._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://mildred.pip.verisignlabs.com/"
+ subject="Manually fixing links doesn't even work"
+ date="2012-04-12T15:46:54Z"
+ content="""
+I'm just answering myself: manually fixing symlinks doesn't always works. Sometimes the pre-commit hook will just rewrite the link to some wrong path.
+"""]]

From 08d9a4df0ff2c1c439d28641ce4d95ceb5b3dd08 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 12:25:09 -0400
Subject: [PATCH 3346/8313] encourage bugs reports be posted to the right place

---
 doc/forum.mdwn | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/forum.mdwn b/doc/forum.mdwn
index ce5787adcb..e7cd73b7fa 100644
--- a/doc/forum.mdwn
+++ b/doc/forum.mdwn
@@ -1,3 +1,5 @@
-This is a place to discuss using git-annex. If you need help, advice, or anything, post about it here.
+This is a place to discuss using git-annex.
+If you need help, advice, or anything, post about it here.
+(But [[post_bug_reports_over_here|bugs]].)
 
 [[!inline pages="forum/* and !*/Discussion" archive=yes rootpage=forum postformtext="Add a new thread titled:"]]

From d819c305b143ab3c2ae173a349c2ec499321111c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Thu, 12 Apr 2012 16:29:58 +0000
Subject: [PATCH 3347/8313] Added a comment

---
 ...mment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment

diff --git a/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
new file mode 100644
index 0000000000..642ae305b6
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-04-12T16:29:58Z"
+ content="""
+This bug was fixed in git-annex 3.20120230. You have a few options to get the fix:
+
+* Upgrade to ghc 4.7, the need for which is the cause of the cabal error message you pasted.
+* Manually [[download]] from git, and `git checkout ghc7.0` -- that branch will build with your old ghc and has the fix.
+* cherry-pick commit 51338486dcf9ab86de426e41b1eb31af1d3a6c87
+"""]]

From fa45175210319bfe771b15c539cf39905eb1a3c1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 12:37:19 -0400
Subject: [PATCH 3348/8313] version dyslexia

---
 .../comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
index 642ae305b6..2fa6cc01a6 100644
--- a/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
+++ b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_2_3bb1d21b7f0d0bd6d59190ae9d246d46._comment
@@ -6,7 +6,7 @@
  content="""
 This bug was fixed in git-annex 3.20120230. You have a few options to get the fix:
 
-* Upgrade to ghc 4.7, the need for which is the cause of the cabal error message you pasted.
+* Upgrade to ghc 7.4, the need for which is the cause of the cabal error message you pasted.
 * Manually [[download]] from git, and `git checkout ghc7.0` -- that branch will build with your old ghc and has the fix.
 * cherry-pick commit 51338486dcf9ab86de426e41b1eb31af1d3a6c87
 """]]

From 52a158a7c6b9b7df93db30dfc802c8c350524951 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 15:34:41 -0400
Subject: [PATCH 3349/8313] autocorrection

git-annex (but not git-annex-shell) supports the git help.autocorrect
configuration setting, doing fuzzy matching using the restricted
Damerau-Levenshtein edit distance, just as git does. This adds a build
dependency on the haskell edit-distance library.
---
 CmdLine.hs         | 51 ++++++++++++++++++++++-----------
 Git/AutoCorrect.hs | 71 ++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  2 +-
 GitAnnexShell.hs   |  2 +-
 debian/changelog   |  4 +++
 debian/control     |  1 +
 doc/install.mdwn   |  1 +
 git-annex.cabal    |  4 +--
 8 files changed, 115 insertions(+), 21 deletions(-)
 create mode 100644 Git/AutoCorrect.hs

diff --git a/CmdLine.hs b/CmdLine.hs
index 05f7bfe2eb..5330f40fc9 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -1,6 +1,6 @@
 {- git-annex command line parsing and dispatch
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -21,6 +21,7 @@ import qualified Annex
 import qualified Annex.Queue
 import qualified Git
 import qualified Git.Command
+import qualified Git.AutoCorrect
 import Annex.Content
 import Annex.Ssh
 import Command
@@ -29,8 +30,8 @@ type Params = [String]
 type Flags = [Annex ()]
 
 {- Runs the passed command line. -}
-dispatch :: Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
-dispatch args cmds commonoptions header getgitrepo = do
+dispatch :: Bool -> Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO ()
+dispatch fuzzyok allargs allcmds commonoptions header getgitrepo = do
 	setupConsole
 	r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo)
 	case r of
@@ -38,30 +39,46 @@ dispatch args cmds commonoptions header getgitrepo = do
 		Right g -> do
 			state <- Annex.new g
 			(actions, state') <- Annex.run state $ do
+				checkfuzzy
 				sequence_ flags
 				prepCommand cmd params
-			tryRun state' cmd $ [startup] ++ actions ++ [shutdown $ cmdoneshot cmd]
+		 	tryRun state' cmd $ [startup] ++ actions ++ [shutdown $ cmdoneshot cmd]
 	where
-		(flags, cmd, params) = parseCmd args cmds commonoptions header
+		err msg = msg ++ "\n\n" ++ usage header allcmds commonoptions
+		cmd = Prelude.head cmds
+		(cmds, name, args) = findCmd fuzzyok allargs allcmds err
+		(flags, params) = getOptCmd args cmd commonoptions err
+		checkfuzzy = when (length cmds > 1) $
+			inRepo $ Git.AutoCorrect.prepare name cmdname cmds
 
-{- Parses command line, and returns actions to run to configure flags,
- - the Command being run, and the remaining parameters for the command. -} 
-parseCmd :: Params -> [Command] -> [Option] -> String -> (Flags, Command, Params)
-parseCmd argv cmds commonoptions header
-	| isNothing name = err "missing command"
-	| null matches = err $ "unknown command " ++ fromJust name
-	| otherwise = check $ getOpt Permute (commonoptions ++ cmdoptions cmd) args
+{- Parses command line params far enough to find the Command to run, and
+ - returns the remaining params.
+ - Does fuzzy matching if necessary, which may result in multiple Commands. -}
+findCmd :: Bool -> Params -> [Command] -> (String -> String) -> ([Command], String, Params)
+findCmd fuzzyok argv cmds err
+	| isNothing name = error $ err "missing command"
+	| not (null exactcmds) = (exactcmds, fromJust name, args)
+	| fuzzyok && not (null inexactcmds) = (inexactcmds, fromJust name, args)
+	| otherwise = error $ err $ "unknown command " ++ fromJust name
 	where
 		(name, args) = findname argv []
 		findname [] c = (Nothing, reverse c)
 		findname (a:as) c
 			| "-" `isPrefixOf` a = findname as (a:c)
 			| otherwise = (Just a, reverse c ++ as)
-		matches = filter (\c -> name == Just (cmdname c)) cmds
-		cmd = Prelude.head matches
-		check (flags, rest, []) = (flags, cmd, rest)
-		check (_, _, errs) = err $ concat errs
-		err msg = error $ msg ++ "\n\n" ++ usage header cmds commonoptions
+		exactcmds = filter (\c -> name == Just (cmdname c)) cmds
+		inexactcmds = case name of
+			Nothing -> []
+			Just n -> Git.AutoCorrect.fuzzymatches n cmdname cmds
+
+{- Parses command line options, and returns actions to run to configure flags
+ - and the remaining parameters for the command. -}
+getOptCmd :: Params -> Command -> [Option] -> (String -> String) -> (Flags, Params)
+getOptCmd argv cmd commonoptions err = check $
+	getOpt Permute (commonoptions ++ cmdoptions cmd) argv
+	where
+		check (flags, rest, []) = (flags, rest)
+		check (_, _, errs) = error $ err $ concat errs
 
 {- Runs a list of Annex actions. Catches IO errors and continues
  - (but explicitly thrown errors terminate the whole command).
diff --git a/Git/AutoCorrect.hs b/Git/AutoCorrect.hs
new file mode 100644
index 0000000000..a18bf56197
--- /dev/null
+++ b/Git/AutoCorrect.hs
@@ -0,0 +1,71 @@
+{- git autocorrection using Damerau-Levenshtein edit distance
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.AutoCorrect where
+
+import Common
+import Git.Types
+import qualified Git.Config
+
+import Text.EditDistance
+import Control.Concurrent
+
+{- These are the same cost values as used in git. -}
+gitEditCosts :: EditCosts
+gitEditCosts = EditCosts
+	{ deletionCosts = ConstantCost 4
+	, insertionCosts = ConstantCost 1
+	, substitutionCosts = ConstantCost 2
+	, transpositionCosts = ConstantCost 0
+	}
+		
+{- Git's source calls this "an empirically derived magic number" -}
+similarityFloor :: Int
+similarityFloor = 7
+
+{- Finds inexact matches for the input amoung the choices.
+ - Returns an ordered list of good enough matches, or an empty list if
+ - nothing matches well. -}
+fuzzymatches :: String -> (c -> String) -> [c] -> [c]
+fuzzymatches input showchoice choices = fst $ unzip $
+	sortBy comparecost $ filter similarEnough $ zip choices costs
+        where
+                distance v = restrictedDamerauLevenshteinDistance gitEditCosts v input
+                costs = map (distance . showchoice) choices
+                comparecost a b = compare (snd a) (snd b)
+                similarEnough (_, cst) = cst < similarityFloor
+
+{- Takes action based on git's autocorrect configuration, in preparation for
+ - an autocorrected command being run. -}
+prepare :: String -> (c -> String) -> [c] -> Repo -> IO ()
+prepare input showmatch matches r =
+	case readish $ Git.Config.get "help.autocorrect" "0" r of
+		Just n
+			| n == 0 -> list
+			| n < 0 -> warn
+			| otherwise -> sleep n
+		Nothing -> list
+	where
+		list = error $ unlines $
+			[ "Unknown command '" ++ input ++ "'"
+			, ""
+			, "Did you mean one of these?"
+			] ++ map (\m -> "\t" ++ showmatch m) matches
+		warn = 
+			hPutStr stderr $ unlines
+				[ "WARNING: You called a command named '" ++
+				  input ++ "', which does not exist."
+				, "Continuing under the assumption that you meant '" ++
+				  showmatch (Prelude.head matches) ++ "'"
+				]
+		sleep n = do
+			warn
+			hPutStrLn stderr $ unwords
+				[ "in"
+				, show (fromIntegral n / 10 :: Float)
+				, "seconds automatically..."]
+			threadDelay (n * 100000) -- deciseconds to microseconds
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 4a0888b53c..52886c3087 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -131,4 +131,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch args cmds options header Git.Construct.fromCurrent
+run args = dispatch True args cmds options header Git.Construct.fromCurrent
diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs
index 3394bc477d..0cf81f0e21 100644
--- a/GitAnnexShell.hs
+++ b/GitAnnexShell.hs
@@ -83,7 +83,7 @@ builtins = map cmdname cmds
 builtin :: String -> String -> [String] -> IO ()
 builtin cmd dir params = do
 	checkNotReadOnly cmd
-	dispatch (cmd : filterparams params) cmds options header $
+	dispatch False (cmd : filterparams params) cmds options header $
 		Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath
 
 external :: [String] -> IO ()
diff --git a/debian/changelog b/debian/changelog
index 274879f6e4..0884f75744 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,10 @@ git-annex (3.20120407) UNRELEASED; urgency=low
   * bugfix: Adding a dotfile also caused all non-dotfiles to be added.
   * bup: Properly handle key names with spaces or other things that are
     not legal git refs.
+  * git-annex (but not git-annex-shell) supports the git help.autocorrect
+    configuration setting, doing fuzzy matching using the restricted
+    Damerau-Levenshtein edit distance, just as git does. This adds a build
+    dependency on the haskell edit-distance library.
 
  -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
 
diff --git a/debian/control b/debian/control
index d8ba09f7ce..2510e2b336 100644
--- a/debian/control
+++ b/debian/control
@@ -19,6 +19,7 @@ Build-Depends:
 	libghc-json-dev,
 	libghc-ifelse-dev,
 	libghc-bloomfilter-dev,
+	libghc-edit-distance-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 0698a8bc4a..04d961e006 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -36,6 +36,7 @@ To build and use git-annex, you will need:
   * [json](http://hackage.haskell.org/package/json)
   * [IfElse](http://hackage.haskell.org/package/IfElse)
   * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
+  * [edit-distance](http://hackage.haskell.org/package/edit-distance)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index e047bb2858..0f28589859 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120406
+Version: 3.20120407
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -32,7 +32,7 @@ Executable git-annex
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   Other-Modules: Utility.Touch
   C-Sources: Utility/diskfree.c
 

From d08a67a78e4c950e1be4f1fd57a74744f443705f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 15:48:59 -0400
Subject: [PATCH 3350/8313] put in a warning so I remember to look at this

---
 Utility/diskfree.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Utility/diskfree.c b/Utility/diskfree.c
index 968d2b8b50..b68abd0c47 100644
--- a/Utility/diskfree.c
+++ b/Utility/diskfree.c
@@ -27,7 +27,8 @@
 # include 
 # define STATCALL statfs64
 # define STATSTRUCT statfs
-# define UNKNOWN /* temporarily disabled; trying to nail down a build failure */
+# warning free space checking code temporarily disabled due to build failure
+# define UNKNOWN
 #else
 #if defined (__linux__)
 /* This is a POSIX standard, so might also work elsewhere. */

From 4ccc86669a3c125ca357e4f37643fbe436988602 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 16:46:57 -0400
Subject: [PATCH 3351/8313] don't fall over broken links

---
 Utility/Inotify.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 56906466b9..25514438df 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -57,7 +57,7 @@ watchDir' scan i add del dir = do
 	return ()
 	where
 		recurse = watchDir' scan i add del
-		walk f = ifM (Files.isDirectory <$> getFileStatus f)
+		walk f = ifM (catchBoolIO $ Files.isDirectory <$> getFileStatus f)
 			( recurse f
 			, if scan then add f else return ()
 			)
@@ -80,5 +80,5 @@ waitForTermination = do
 	takeMVar mv
 	where
 		check sig mv = do
-			installHandler sig (CatchOnce $ putMVar mv ()) Nothing
+			_ <- installHandler sig (CatchOnce $ putMVar mv ()) Nothing
 			return ()

From be4edbaaf1395b9378664bc57c2bfd7a33fa80a1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 16:59:33 -0400
Subject: [PATCH 3352/8313] allow excluding directories from being watched

---
 Utility/Inotify.hs | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 25514438df..c278a51aee 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -11,17 +11,17 @@ import System.Posix.Signals
 
 demo :: IO ()
 demo = withINotify $ \i -> do
-	watchDir i add del "/home/joey/tmp/me"
+	watchDir i (const True) add del "/home/joey/tmp/me"
 	putStrLn "started"
 	waitForTermination
 	where
 		add file = putStrLn $ "add " ++ file
 		del file = putStrLn $ "del " ++ file
 
-{- Watches for changes to files in a directory, and all its subdirectories,
- - using inotify. This function returns after its initial setup is
- - complete, leaving a thread running. Then callbacks are made for adding
- - and deleting files.
+{- Watches for changes to files in a directory, and all its subdirectories
+ - that match a test, using inotify. This function returns after its initial
+ - setup is complete, leaving a thread running. Then callbacks are made for
+ - adding and deleting files.
  -
  - Inotify is weak at recursive directory watching; the whole directory
  - tree must be walked and watches set explicitly for each subdirectory.
@@ -48,15 +48,18 @@ demo = withINotify $ \i -> do
  - /proc/sys/fs/inotify/max_user_watches (default 8192).
  - So This will fail if there are too many subdirectories.
  -}
-watchDir :: INotify -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
-watchDir i add del dir = watchDir' False i add del dir
-watchDir' :: Bool -> INotify -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
-watchDir' scan i add del dir = do
-	_ <- addWatch i [MoveIn, MoveOut, Create, Delete, CloseWrite] dir go
-	_ <- mapM walk =<< dirContents dir
-	return ()
+watchDir :: INotify -> (FilePath -> Bool) -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
+watchDir i test add del dir = watchDir' False i test add del dir
+watchDir' :: Bool -> INotify -> (FilePath -> Bool) -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
+watchDir' scan i test add del dir = do
+	if test dir
+		then do
+			_ <- addWatch i [MoveIn, MoveOut, Create, Delete, CloseWrite] dir go
+			_ <- mapM walk =<< dirContents dir
+			return ()
+		else return ()
 	where
-		recurse = watchDir' scan i add del
+		recurse = watchDir' scan i test add del
 		walk f = ifM (catchBoolIO $ Files.isDirectory <$> getFileStatus f)
 			( recurse f
 			, if scan then add f else return ()

From 6464a576cd0c4063fb6cedf32a5dca224db3de6d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 17:28:40 -0400
Subject: [PATCH 3353/8313] allow add or del events to be ignored

---
 Utility/Inotify.hs | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index c278a51aee..049737c08a 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -11,7 +11,7 @@ import System.Posix.Signals
 
 demo :: IO ()
 demo = withINotify $ \i -> do
-	watchDir i (const True) add del "/home/joey/tmp/me"
+	watchDir i (const True) (Just add) (Just del) "/home/joey/tmp/me"
 	putStrLn "started"
 	waitForTermination
 	where
@@ -48,30 +48,40 @@ demo = withINotify $ \i -> do
  - /proc/sys/fs/inotify/max_user_watches (default 8192).
  - So This will fail if there are too many subdirectories.
  -}
-watchDir :: INotify -> (FilePath -> Bool) -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
+watchDir :: INotify -> (FilePath -> Bool) -> Maybe (FilePath -> IO ()) -> Maybe (FilePath -> IO ()) -> FilePath -> IO ()
 watchDir i test add del dir = watchDir' False i test add del dir
-watchDir' :: Bool -> INotify -> (FilePath -> Bool) -> (FilePath -> IO ()) -> (FilePath -> IO ()) -> FilePath -> IO ()
+watchDir' :: Bool -> INotify -> (FilePath -> Bool) -> Maybe (FilePath -> IO ()) -> Maybe (FilePath -> IO ()) -> FilePath -> IO ()
 watchDir' scan i test add del dir = do
 	if test dir
 		then do
-			_ <- addWatch i [MoveIn, MoveOut, Create, Delete, CloseWrite] dir go
+			_ <- addWatch i watchevents dir go
 			_ <- mapM walk =<< dirContents dir
 			return ()
 		else return ()
 	where
+		watchevents
+			| isJust add && isJust del =
+				[Create, MoveIn, MoveOut, Delete, CloseWrite]
+			| isJust add = [Create, MoveIn, CloseWrite]
+			| isJust del = [Create, MoveOut, Delete]
+			| otherwise = [Create]
+
 		recurse = watchDir' scan i test add del
 		walk f = ifM (catchBoolIO $ Files.isDirectory <$> getFileStatus f)
 			( recurse f
-			, if scan then add f else return ()
+			, if scan && isJust add then fromJust add f else return ()
 			)
-		a <@> f = a $ dir  f
+
 		go (Created { isDirectory = False }) = return ()
-		go (Created { filePath = subdir }) = recurse <@> subdir
+		go (Created { filePath = subdir }) = Just recurse <@> subdir
 		go (Closed { maybeFilePath = Just f }) = add <@> f
 		go (MovedIn { isDirectory = False, filePath = f }) = add <@> f
 		go (MovedOut { isDirectory = False, filePath = f }) = del <@> f
 		go (Deleted { isDirectory = False, filePath = f }) = del <@> f
 		go _ = return ()
+		
+		Just a <@> f = a $ dir  f
+		Nothing <@> _ = return ()
 
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()

From d5ffd2d99d0da587e55b31994dae658c2bb6d9d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 17:29:43 -0400
Subject: [PATCH 3354/8313] watch subcommand

So far this only handles auto-annexing new files that are created inside
the repository while it's running. To make this really useful,
it needs to at least:

- notice deleted files and stage the deletion
  (tricky; there's a race with add..)
- notice renamed files, auto-fix the symlink, and stage the new file location
- periodically auto-commit staged changes
- honor .gitignore, not adding files it excludes

Also nice to have would be:

- Somehow sync remotes, possibly using a push sync like dvcs-autosync
  does, so they are immediately updated.
- Somehow get content that is unavilable. This is problimatic with inotify,
  since we only get an event once the user has tried (and failed) to read
  from the file. Perhaps instead, automatically copy content that is added
  out to remotes, with the goal of all repos eventually getting a copy,
  if df allows.
- Drop files that have not been used lately, or meet some other criteria
  (as long as there's a copy elsewhere).
- Perhaps automatically dropunused files that have been deleted,
  although I cannot see a way to do that, since by the time the inotify
  deletion event arrives, the file is deleted, and we cannot see what
  its symlink pointed to! Alternatievely, perhaps automatically
  do an expensive unused/dropunused cleanup process.

Some of this probably needs the currently stateless threads to maintain
a common state.
---
 Command/Watch.hs | 55 ++++++++++++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs      |  2 ++
 2 files changed, 57 insertions(+)
 create mode 100644 Command/Watch.hs

diff --git a/Command/Watch.hs b/Command/Watch.hs
new file mode 100644
index 0000000000..e0fa97ac57
--- /dev/null
+++ b/Command/Watch.hs
@@ -0,0 +1,55 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Watch where
+
+import Common.Annex
+import Command
+import qualified Annex
+import CmdLine
+import Utility.Inotify
+import Control.Exception as E
+import qualified Command.Add as Add
+
+import System.INotify
+
+def :: [Command]
+def = [command "watch" paramPaths seek "watch for changes"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStart
+start = notBareRepo $ do
+	showStart "watch" "."
+	showAction "scanning"
+	state <- Annex.getState id
+	next $ next $ liftIO $ withINotify $ \i -> do
+		watchDir i notgit (Just $ run state onAdd) Nothing "."
+		putStrLn "(started)"
+		waitForTermination
+		return True
+	where
+		notgit dir = takeFileName dir /= ".git"
+
+{- Inotify events are run in separate threads, and so each is a
+ - self-contained Annex monad. Exceptions by the handlers are ignored,
+ - otherwise a whole watcher thread could be crashed. -}
+run :: Annex.AnnexState -> (FilePath -> Annex a) -> FilePath -> IO ()
+run startstate a f = do
+	r <- E.try go :: IO (Either E.SomeException ())
+	case r of
+		Left e -> putStrLn (show e)
+		_ -> return ()
+	where
+		go = Annex.eval startstate $ do
+			_ <- a f
+			_ <- shutdown True
+			return ()
+
+onAdd :: FilePath -> Annex Bool
+onAdd file = doCommand $ Add.start file
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 52886c3087..5caa93499e 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -56,6 +56,7 @@ import qualified Command.AddUrl
 import qualified Command.Map
 import qualified Command.Upgrade
 import qualified Command.Version
+import qualified Command.Watch
 
 cmds :: [Command]
 cmds = concat
@@ -95,6 +96,7 @@ cmds = concat
 	, Command.Map.def
 	, Command.Upgrade.def
 	, Command.Version.def
+	, Command.Watch.def
 	]
 
 options :: [Option]

From 49a95f151ce31f44417eee0463e9d334a581155f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 12 Apr 2012 17:51:11 -0400
Subject: [PATCH 3355/8313] add a todo item

---
 .../automatic_bookkeeping_watch_command.mdwn  | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 doc/todo/automatic_bookkeeping_watch_command.mdwn

diff --git a/doc/todo/automatic_bookkeeping_watch_command.mdwn b/doc/todo/automatic_bookkeeping_watch_command.mdwn
new file mode 100644
index 0000000000..d7a3517a19
--- /dev/null
+++ b/doc/todo/automatic_bookkeeping_watch_command.mdwn
@@ -0,0 +1,42 @@
+A "git annex watch" command would help make git-annex usable by users who
+don't know how to use git, or don't want to bother typing the git commands. 
+It would run, in the background, watching via inotify for changes, and
+automatically annexing new files, etc.
+
+The blue sky goal would be something automated like dropbox, except fully
+distributed. All files put into the repository would propigate out
+to all the other clones of it, as network links allow. Note that while
+dropbox allows modifying files, git-annex freezes them upon creation,
+so this would not be 100% equivilant to dropbox. --[[Joey]]
+
+----
+
+There is a `watch` branch in git that adds such a command, although currently
+it only handles adding new files, and nothing else. To make this really
+useful, it needs to:
+
+- notice deleted files and stage the deletion
+  (tricky; there's a race with add..)
+- notice renamed files, auto-fix the symlink, and stage the new file location
+- periodically auto-commit staged changes
+- honor .gitignore, not adding files it excludes
+    
+Also nice to have would be:
+    
+- Somehow sync remotes, possibly using a push sync like dvcs-autosync
+  does, so they are immediately updated.
+- Somehow get content that is unavilable. This is problimatic with inotify,
+  since we only get an event once the user has tried (and failed) to read
+  from the file. Perhaps instead, automatically copy content that is added
+  out to remotes, with the goal of all repos eventually getting a copy,
+  if df allows.
+- Drop files that have not been used lately, or meet some other criteria
+  (as long as there's a copy elsewhere).
+- Perhaps automatically dropunused files that have been deleted,
+  although I cannot see a way to do that, since by the time the inotify
+  deletion event arrives, the file is deleted, and we cannot see what
+  its symlink pointed to! Alternatievely, perhaps automatically
+  do an expensive unused/dropunused cleanup process.
+- Support OSes other than Linux; it only uses inotify currently.
+
+

From 8000fae9b259153c921b682afe3388791db20149 Mon Sep 17 00:00:00 2001
From: "http://mildred.pip.verisignlabs.com/"
 
Date: Fri, 13 Apr 2012 07:28:11 +0000
Subject: [PATCH 3356/8313] Added a comment: Thank you a lot

---
 ...t_3_692f268218690437138ae0540c879425._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment

diff --git a/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment
new file mode 100644
index 0000000000..d5c84b431f
--- /dev/null
+++ b/doc/forum/fail_to_git_annex_add_some_files:_getFileStatus:_does_not_exist__40__v_3.20111231__41__/comment_3_692f268218690437138ae0540c879425._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://mildred.pip.verisignlabs.com/"
+ subject="Thank you a lot"
+ date="2012-04-13T07:28:10Z"
+ content="""
+Thank you,
+
+I imagined it was something like that.
+I 'm just sorry I posted that on the forum and not on the bugs section (I hadn't discovered it at that time). but now, if people search for this error, they should find this.
+
+Note for Fedora users: unfortunately GHC 7.4 will not be shipped with Fedora 17 (which is still not released). The [feature page](https://fedoraproject.org/wiki/Features/GHC74) mention it for Fedora 18. I feel like I am using debian ... outdated packages the day of the release.
+
+And many thanks for this wonderful piece of software.
+
+Mildred
+"""]]

From ebc91c7637b8f245d467742fc33a577ca9516105 Mon Sep 17 00:00:00 2001
From: "http://mildred.pip.verisignlabs.com/"
 
Date: Fri, 13 Apr 2012 07:57:28 +0000
Subject: [PATCH 3357/8313] Installation by hand using cabal / Setup.hs

---
 doc/install.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 04d961e006..b85700e661 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -50,3 +50,13 @@ To build and use git-annex, you will need:
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`
+
+## Installation by hand using cabal / Setup.hs
+
+You can fetch the dependencies using `cabal install`. Then, you can build git-annex by running:
+
+    runhaskell Setup.hs configure --user
+    runhaskell Setup.hs build
+    runhaskell Setup.hs install --bindir=$HOME/bin
+
+The `--user` option configures the build so that it uses the packages you already have in `~/.cabal`.

From 4837ad6dd3ed877513446cb948a696a90ccfdf16 Mon Sep 17 00:00:00 2001
From: "http://mildred.pip.verisignlabs.com/"
 
Date: Fri, 13 Apr 2012 08:19:27 +0000
Subject: [PATCH 3358/8313]

---
 doc/install.mdwn | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index b85700e661..33426d637f 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -57,6 +57,7 @@ You can fetch the dependencies using `cabal install`. Then, you can build git-an
 
     runhaskell Setup.hs configure --user
     runhaskell Setup.hs build
-    runhaskell Setup.hs install --bindir=$HOME/bin
+    runhaskell Setup.hs install
 
-The `--user` option configures the build so that it uses the packages you already have in `~/.cabal`.
+The `--user` option configures the build so that it uses the packages you already have in `~/.cabal`. Binaries
+will be installed in `~/.cabal/bin`, you'll need it in your PATH.

From d712f5b0c4d96fc63a2a42c6e7af6139f4357733 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm2AOTJmbCbGvmW5fxACaREraMnEVrCofo"
 
Date: Fri, 13 Apr 2012 13:26:09 +0000
Subject: [PATCH 3359/8313]

---
 doc/bugs/sensitive.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/sensitive.mdwn

diff --git a/doc/bugs/sensitive.mdwn b/doc/bugs/sensitive.mdwn
new file mode 100644
index 0000000000..21c2c4ec47
--- /dev/null
+++ b/doc/bugs/sensitive.mdwn
@@ -0,0 +1,17 @@
+What steps will reproduce the problem?
+
+> Building git-annex on the ghc7.0 branch on a Mac with the default case-insensitive file system
+
+What is the expected output? What do you see instead?
+
+> Expected: build successfully; instead: 
+     
+    ld: duplicate symbol _UtilityziDiskFree_zdwa_info in dist/build/git-annex/git-annex-tmp/Utility/diskfree.o and dist/build/git-annex/git-annex-tmp/Utility/DiskFree.o for architecture x86_64
+
+What version of git-annex are you using? On what operating system?
+
+> commit `0bd5c90ef0518f75d52f0c5889422d8233df847d` on a Mac OS 10.6 and 10.7, using the Haskell Platform 2012.04
+
+Please provide any additional information below.
+
+> The problem is that since `DiskFree.hs` generates `DiskFree.o` and `diskfree.c` generates `diskfree.o`, a case-insensitive file system overwrites one object file with the other.  Renaming `diskfree.c` to `diskfreec.c` and changing the corresponding filenames in `git-annex.cabal` fixes the problem.

From 45a3cbc55bcc1462241e44ffa2b22d37895cfcd9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawm2AOTJmbCbGvmW5fxACaREraMnEVrCofo"
 
Date: Fri, 13 Apr 2012 13:47:14 +0000
Subject: [PATCH 3360/8313] rename bugs/sensitive.mdwn to
 bugs/case-insensitive.mdwn

---
 doc/bugs/{sensitive.mdwn => case-insensitive.mdwn} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/bugs/{sensitive.mdwn => case-insensitive.mdwn} (100%)

diff --git a/doc/bugs/sensitive.mdwn b/doc/bugs/case-insensitive.mdwn
similarity index 100%
rename from doc/bugs/sensitive.mdwn
rename to doc/bugs/case-insensitive.mdwn

From cc70792772f3254fec97bdaaeda8a11bac43907a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 11:13:58 -0400
Subject: [PATCH 3361/8313] update with manual git clone, and ghc7.0 build

Seems that Fedora is going to ship with an outdated GHC. Pity.
---
 doc/install/Fedora.mdwn | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 7e983597b2..97ccf38ef4 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -1,7 +1,15 @@
-Installation recipe for Fedora 14.
+Installation recipe for Fedora 14 thruough 17.
 
 
 sudo yum install ghc cabal-install
-sudo cabal update
-cabal install git-annex --bindir=$HOME/bin
+git clone git://git.kitenet.net/git-annex
+cd git-annex
+git checkout ghc 7.0
+cabal update
+cabal configure
+cabal build
+cabal install --bindir=$HOME/bin
 

+
+Note: You can't just use `cabal install git-annex`, because Fedora does
+not yet ship ghc 7.4.

From 64c00933479a20e57f11f0cf20b88a6a68b16d2b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 11:15:27 -0400
Subject: [PATCH 3362/8313] move manual cabal install into its own page, and
 simplify it

---
 doc/install.mdwn       | 14 +++-----------
 doc/install/cabal.mdwn | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 11 deletions(-)
 create mode 100644 doc/install/cabal.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 33426d637f..a02d9d2c74 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -16,6 +16,9 @@ As a haskell package, git-annex can be installed using cabal. For example:
 
 	cabal install git-annex --bindir=$HOME/bin
 
+The above downloads the latest release. Alternatively, you can [[download]]
+it yourself and [[manually_build_with_cabal|install/cabal]].
+
 ## Installation by hand
 
 To build and use git-annex, you will need:
@@ -50,14 +53,3 @@ To build and use git-annex, you will need:
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`
-
-## Installation by hand using cabal / Setup.hs
-
-You can fetch the dependencies using `cabal install`. Then, you can build git-annex by running:
-
-    runhaskell Setup.hs configure --user
-    runhaskell Setup.hs build
-    runhaskell Setup.hs install
-
-The `--user` option configures the build so that it uses the packages you already have in `~/.cabal`. Binaries
-will be installed in `~/.cabal/bin`, you'll need it in your PATH.
diff --git a/doc/install/cabal.mdwn b/doc/install/cabal.mdwn
new file mode 100644
index 0000000000..fe7b025e12
--- /dev/null
+++ b/doc/install/cabal.mdwn
@@ -0,0 +1,15 @@
+As a haskell package, git-annex can be installed using cabal. For example:
+
+	cabal update
+        cabal install git-annex --bindir=$HOME/bin
+
+The above downloads the latest release and installs it into a ~/bin/
+directory, which you can put in your PATH.
+
+But maybe you want something newer (or older). Then [[download]] the version
+you want, and use cabal as follows inside its source tree:
+
+	cabal update
+	cabal configure
+	cabal build
+	cabal install --bindir=$HOME/bin

From c56cfeba9ec134d61ce8da8fe351722ddb7ec00f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 11:19:56 -0400
Subject: [PATCH 3363/8313] word wrap

---
 doc/install/openSUSE.mdwn | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/doc/install/openSUSE.mdwn b/doc/install/openSUSE.mdwn
index 0383cbbf2a..73cbe585f6 100644
--- a/doc/install/openSUSE.mdwn
+++ b/doc/install/openSUSE.mdwn
@@ -1,4 +1,12 @@
-Unfortunately there is currently no git-annex rpm available for openSUSE; however it is possible to build it via cabal or from source as described on the [[install]] page.  Fulfilling the dependencies listed on that page should not be a problem, except for obtaining a suitable version of the Haskell library.
+Unfortunately there is currently no git-annex rpm available for openSUSE;
+however it is possible to build it via cabal or from source as described on
+the [[install]] page.  Fulfilling the dependencies listed on that page
+should not be a problem, except for obtaining a suitable version of the
+Haskell library.
 
-The last [official release of Haskell for openSUSE](https://build.opensuse.org/project/show?project=devel:languages:haskell) is quite old, and may not satisfy the dependencies needed by git-annex.  Fortunately [searching the openSUSE build service](http://software.opensuse.org/search?q=cabal&baseproject=openSUSE%3A11.4&lang=en&include_home=true&exclude_debug=true) reveals that Peter Trommler has built a [newer Haskell suite](https://build.opensuse.org/project/show?project=home%3Aptrommler%3Adevel%3Alanguages%3Ahaskell) based on ghc 7.2.
-To install this, simply click on the relevant "1-Click Install" link in the openSUSE build service search results.
+The last [official release of Haskell for openSUSE](https://build.opensuse.org/project/show?project=devel:languages:haskell)
+is quite old, and may not satisfy the dependencies needed by git-annex.
+Fortunately [searching the openSUSE build service](http://software.opensuse.org/search?q=cabal&baseproject=openSUSE%3A11.4&lang=en&include_home=true&exclude_debug=true)
+reveals that Peter Trommler has built a [newer Haskell suite](https://build.opensuse.org/project/show?project=home%3Aptrommler%3Adevel%3Alanguages%3Ahaskell)
+based on ghc 7.2. To install this, simply click on the relevant
+"1-Click Install" link in the openSUSE build service search results.

From fdb246044c7b64802a9d0e90b92a978105a3704f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 11:24:25 -0400
Subject: [PATCH 3364/8313] download updates

---
 doc/download.mdwn                             | 26 +++++++++----------
 ..._fbd8b6d39e9d3c71791551358c863966._comment |  8 ------
 ..._f85f72b33aedc3425f0c0c47867d02f3._comment |  8 ------
 ..._cf6044ebe99f71158034e21197228abd._comment | 10 -------
 ..._10fc013865c7542c2ed9d6c0963bb391._comment |  9 -------
 ..._c6b1bc40226fc2c8ba3e558150856992._comment |  8 ------
 ..._3a52993d3553deb9a413debec9a5f92d._comment | 11 --------
 ..._a5eebd214b135f34b18274a682211943._comment |  8 ------
 ..._59a976de6c7d333709b92f7cd5830850._comment |  8 ------
 doc/install/Fedora.mdwn                       |  2 +-
 10 files changed, 13 insertions(+), 85 deletions(-)
 delete mode 100644 doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
 delete mode 100644 doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
 delete mode 100644 doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
 delete mode 100644 doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
 delete mode 100644 doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
 delete mode 100644 doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
 delete mode 100644 doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
 delete mode 100644 doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 30307f3088..64eee64c80 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -7,11 +7,9 @@ Other mirrors of the git repository:
 * `git://git.kitenet.net/git-annex` [[gitweb](http://git.kitenet.net/?p=git-annex.git;a=summary)]
 * [at github](https://github.com/joeyh/git-annex)
 
-To download a tarball of a particular release, use an url like
-
-
-From time to time, releases of git-annex are uploaded
-[to hackage](http://hackage.haskell.org/package/git-annex).
+Releases of git-annex are uploaded
+[to hackage](http://hackage.haskell.org/package/git-annex). Get your
+tarballs there, if you need them.
 
 Some operating systems include git-annex in easily prepackaged form and
 others need some manual work. See [[install]] for details.
@@ -20,17 +18,17 @@ others need some manual work. See [[install]] for details.
 
 The git repository has some branches:
 
-* `debian-stable` contains the latest backport of git-annex to Debian
-  stable.
-* `no-s3` disables the S3 special remote, for systems that lack the
- necessary haskell library. (merge it into master if you need it)
-* `no-bloom` avoids using bloom filters. (merge it into master if you need it)
-* `old-monad-control` is for systems that don't have a newer monad-control
-  library.
-* `tweak-fetch` adds support for the git tweak-fetch hook, which has
-  been proposed and implemented but not yet accepted into git.
 * `ghc7.0` supports versions of ghc older than 7.4, which
   had a major change to filename encoding.
+* `old-monad-control` is for systems that don't have a newer monad-control
+  library.
+* `no-s3` disables the S3 special remote, for systems that lack the
+  necessary haskell library. (merge it into master if you need it)
+* `no-bloom` avoids using bloom filters. (merge it into master if you need it)
+* `debian-stable` contains the latest backport of git-annex to Debian
+  stable.
+* `tweak-fetch` adds support for the git tweak-fetch hook, which has
+  been proposed and implemented but not yet accepted into git.
 * `setup` contains configuration for this website
 * `pristine-tar` contains [pristine-tar](http://kitenet.net/~joey/code/pristine-tar)
   data to create tarballs of any past git-annex release.
diff --git a/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment b/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
deleted file mode 100644
index 488e005278..0000000000
--- a/doc/download/comment_1_fbd8b6d39e9d3c71791551358c863966._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="84.189.2.244"
- subject="Please provide stable tarballs or zipfiles"
- date="2011-03-22T13:06:58Z"
- content="""
-I'm trying to package git annex for ArchLinux and NixOS. That task would be a *lot* easier, if there were proper release archives available for download. The Gitweb site offers to create snapshot tarballs on the fly, but those tarballs have a different SHA hash every time they're generated, so they cannot be used for the purposes of a distribution. A simple solution for this problem would be to enable snapshots in zip format (because zip files look the same every time they're generated).
-"""]]
diff --git a/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment b/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
deleted file mode 100644
index 5441c3e4ce..0000000000
--- a/doc/download/comment_2_f85f72b33aedc3425f0c0c47867d02f3._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
- nickname="Jimmy"
- subject="comment 2"
- date="2011-03-22T14:01:37Z"
- content="""
-maybe snag tarballs from  ? 
-"""]]
diff --git a/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment b/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
deleted file mode 100644
index b72b848f80..0000000000
--- a/doc/download/comment_3_cf6044ebe99f71158034e21197228abd._comment
+++ /dev/null
@@ -1,10 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 3"
- date="2011-03-22T18:09:21Z"
- content="""
-The tarballs produced by gitweb are actually stable. They are wrapped in a gz file with a varying timestamp however. It might be nice if gitweb passed --no-name to gzip to avoid that inconsistency.
-
-git-annex also has a [pristine-tar](http://kitenet.net/~joey/code/pristine-tar/) branch in git that can be used to recreate the tarballs I upload to Debian.
-"""]]
diff --git a/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment b/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
deleted file mode 100644
index 9bb9aa8ae3..0000000000
--- a/doc/download/comment_4_10fc013865c7542c2ed9d6c0963bb391._comment
+++ /dev/null
@@ -1,9 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
- nickname="Fraser"
- subject="gitweb supplies --no-name as of 1.7.5.1"
- date="2011-05-19T08:19:02Z"
- content="""
-git v1.7.5.1 fixes the gitweb gzip issue.  If the git instance is updated we
-can have stable distributions (and I can finally write a FreeBSD port ^_^)
-"""]]
diff --git a/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment b/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
deleted file mode 100644
index 76ba75edc4..0000000000
--- a/doc/download/comment_5_c6b1bc40226fc2c8ba3e558150856992._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 5"
- date="2011-05-19T16:10:35Z"
- content="""
-Hmm, I've upgraded to that version, but I see nothing in its changelog, commit log, code, or runtime behavior to indicate that it's producing stable gzip output.
-"""]]
diff --git a/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment b/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
deleted file mode 100644
index 0dbd88b1e5..0000000000
--- a/doc/download/comment_6_3a52993d3553deb9a413debec9a5f92d._comment
+++ /dev/null
@@ -1,11 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
- nickname="Fraser"
- subject="comment 6"
- date="2011-05-22T23:02:39Z"
- content="""
-Whups, the fix landed in git's `maint' branch just after 1.7.5 but 1.7.5.1 was
-tagged on a different branch.
-
-Will look closer in future, and let you know when it's really released.
-"""]]
diff --git a/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment b/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
deleted file mode 100644
index 9960e0ea85..0000000000
--- a/doc/download/comment_7_a5eebd214b135f34b18274a682211943._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawnOvt3TwSSDOLnoVzDNbOP1qO9OmNH5s0s"
- nickname="Fraser"
- subject="comment 7"
- date="2011-05-27T01:27:37Z"
- content="""
-v1.7.5.3 has it.
-"""]]
diff --git a/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment b/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment
deleted file mode 100644
index 5aa4f8c94a..0000000000
--- a/doc/download/comment_8_59a976de6c7d333709b92f7cd5830850._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 8"
- date="2011-05-28T16:04:51Z"
- content="""
-And that is now installed on kitenet.net and verified to work.
-"""]]
diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 97ccf38ef4..cfdeeb8826 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -2,7 +2,7 @@ Installation recipe for Fedora 14 thruough 17.
 
 
 sudo yum install ghc cabal-install
-git clone git://git.kitenet.net/git-annex
+git clone git://git-annex.branchable.com/ git-annex
 cd git-annex
 git checkout ghc 7.0
 cabal update

From 3642c723209bfb117b16c12f0ad83ef0fe025613 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 11:26:39 -0400
Subject: [PATCH 3365/8313] Renamed diskfree.c to avoid OSX case insensativity
 bug.

---
 Makefile                              | 2 +-
 Utility/DiskFree.hs                   | 2 +-
 Utility/{diskfree.c => libdiskfree.c} | 0
 Utility/{diskfree.h => libdiskfree.h} | 0
 debian/changelog                      | 1 +
 doc/bugs/case-insensitive.mdwn        | 3 +++
 git-annex.cabal                       | 6 +++---
 7 files changed, 9 insertions(+), 5 deletions(-)
 rename Utility/{diskfree.c => libdiskfree.c} (100%)
 rename Utility/{diskfree.h => libdiskfree.h} (100%)

diff --git a/Makefile b/Makefile
index b935140576..eb30a3833c 100644
--- a/Makefile
+++ b/Makefile
@@ -12,7 +12,7 @@ GHCMAKE=ghc $(GHCFLAGS) --make
 bins=git-annex
 mans=git-annex.1 git-annex-shell.1
 sources=Build/SysConfig.hs Utility/Touch.hs
-clibs=Utility/diskfree.o
+clibs=Utility/libdiskfree.o
 
 all=$(bins) $(mans) docs
 
diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs
index bde6ef8caa..8d07afaf2d 100644
--- a/Utility/DiskFree.hs
+++ b/Utility/DiskFree.hs
@@ -15,7 +15,7 @@ import Foreign.C.Types
 import Foreign.C.String
 import Foreign.C.Error
 
-foreign import ccall unsafe "diskfree.h diskfree" c_diskfree
+foreign import ccall unsafe "libdiskfree.h diskfree" c_diskfree
 	:: CString -> IO CULLong
 
 getDiskFree :: String -> IO (Maybe Integer)
diff --git a/Utility/diskfree.c b/Utility/libdiskfree.c
similarity index 100%
rename from Utility/diskfree.c
rename to Utility/libdiskfree.c
diff --git a/Utility/diskfree.h b/Utility/libdiskfree.h
similarity index 100%
rename from Utility/diskfree.h
rename to Utility/libdiskfree.h
diff --git a/debian/changelog b/debian/changelog
index 0884f75744..abc3ae5120 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,7 @@ git-annex (3.20120407) UNRELEASED; urgency=low
     configuration setting, doing fuzzy matching using the restricted
     Damerau-Levenshtein edit distance, just as git does. This adds a build
     dependency on the haskell edit-distance library.
+  * Renamed diskfree.c to avoid OSX case insensativity bug.
 
  -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
 
diff --git a/doc/bugs/case-insensitive.mdwn b/doc/bugs/case-insensitive.mdwn
index 21c2c4ec47..a917f64c28 100644
--- a/doc/bugs/case-insensitive.mdwn
+++ b/doc/bugs/case-insensitive.mdwn
@@ -15,3 +15,6 @@ What version of git-annex are you using? On what operating system?
 Please provide any additional information below.
 
 > The problem is that since `DiskFree.hs` generates `DiskFree.o` and `diskfree.c` generates `diskfree.o`, a case-insensitive file system overwrites one object file with the other.  Renaming `diskfree.c` to `diskfreec.c` and changing the corresponding filenames in `git-annex.cabal` fixes the problem.
+
+>> Man, not this again. The 80's called, they want their 
+>> unix portability wars back. [[fixed|done]]. --[[Joey]]
diff --git a/git-annex.cabal b/git-annex.cabal
index 0f28589859..df40011895 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -34,17 +34,17 @@ Executable git-annex
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   Other-Modules: Utility.Touch
-  C-Sources: Utility/diskfree.c
+  C-Sources: Utility/libdiskfree.c
 
 Executable git-annex-shell
   Main-Is: git-annex-shell.hs
-  C-Sources: Utility/diskfree.c
+  C-Sources: Utility/libdiskfree.c
 
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs
   Build-Depends: testpack, HUnit
-  C-Sources: Utility/diskfree.c
+  C-Sources: Utility/libdiskfree.c
 
 source-repository head
   type: git

From befda6f9d26ade107f0b9cec3ea7919fda64444f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 11:37:49 -0400
Subject: [PATCH 3366/8313] add cabal command to install build deps

---
 doc/install/Fedora.mdwn | 1 +
 doc/install/cabal.mdwn  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index cfdeeb8826..68b0056f1a 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -6,6 +6,7 @@ git clone git://git-annex.branchable.com/ git-annex
 cd git-annex
 git checkout ghc 7.0
 cabal update
+cabal install --only-dependencies
 cabal configure
 cabal build
 cabal install --bindir=$HOME/bin
diff --git a/doc/install/cabal.mdwn b/doc/install/cabal.mdwn
index fe7b025e12..180208211c 100644
--- a/doc/install/cabal.mdwn
+++ b/doc/install/cabal.mdwn
@@ -10,6 +10,7 @@ But maybe you want something newer (or older). Then [[download]] the version
 you want, and use cabal as follows inside its source tree:
 
 	cabal update
+	cabal install --only-dependencies
 	cabal configure
 	cabal build
 	cabal install --bindir=$HOME/bin

From 7a36d8ed7cdece92058a25b30e4056992d96c715 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 12:22:20 -0400
Subject: [PATCH 3367/8313] link to fedora ITP bug

---
 doc/install/Fedora.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index 68b0056f1a..a4dd4f5c87 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -14,3 +14,5 @@ cabal install --bindir=$HOME/bin
 
 Note: You can't just use `cabal install git-annex`, because Fedora does
 not yet ship ghc 7.4.
+
+[Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259)

From a4bbdffd6ae5dbb7b57dbaf6c4263ba353f9239b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:19:04 -0400
Subject: [PATCH 3368/8313] wrap

---
 doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
index 14ca838cf1..17605ee0d0 100644
--- a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
@@ -7,6 +7,11 @@ oh my god, git-annex is amazing
 this is the revolution in fucking with gigantic piles of files that I've been waiting for
 

 
-And then my own story: I have a ton of drives. I have a lot of servers. I live in a cabin on **dialup** and often have 1 hour on broadband in a week to get everything I need. Without git-annex, managing all this would not be possible. It works perfectly for me, not a surprise since I wrote it, but still, it's a different level of "perfect" than anything I could put together before.
+And then my own story: I have a ton of drives. I have a lot of servers. I
+live in a cabin on **dialup** and often have 1 hour on broadband in a week
+to get everything I need. Without git-annex, managing all this would not be
+possible. It works perfectly for me, not a surprise since I wrote it, but
+still, it's a different level of "perfect" than anything I could put
+together before.
 
 [[!meta author=Joey]]

From 1efc317a1da1c308c2cc8b2efeb019563308d833 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:33:44 -0400
Subject: [PATCH 3369/8313] add some embedded tweets

---
 ...ll_us_how_you__39__re_using_git-annex.mdwn | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
index 17605ee0d0..ce4ccee31d 100644
--- a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
@@ -1,6 +1,19 @@
-Tell your git-annex stories here. Feel free to give as little or as much detail as appropriate about how you're using it. How's it working out for you?
+Tell your git-annex stories here. Feel free to give as little or as much detail
+as appropriate about how you're using it. How's it working out for you?
 
-I'll start with a comment a user just posted to IRC:
+I'll start with some tweets and comments from IRC:
+
+
+
+
+
+
+
+
+
I want #git-annex whereis for all the stuff (not) in my room.
— topr (@derwelle) February 22, 2012

+
 
 
 oh my god, git-annex is amazing
@@ -12,6 +25,6 @@ live in a cabin on **dialup** and often have 1 hour on broadband in a week
 to get everything I need. Without git-annex, managing all this would not be
 possible. It works perfectly for me, not a surprise since I wrote it, but
 still, it's a different level of "perfect" than anything I could put
-together before.
+together before. --[[Joey]]
 
 [[!meta author=Joey]]

From df367c92e8dc35df474057b37bba45d37e7045fc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:35:12 -0400
Subject: [PATCH 3370/8313] move

---
 ...ll_us_how_you__39__re_using_git-annex.mdwn | 26 +------------------
 doc/testimonials.mdwn                         | 23 ++++++++++++++++
 2 files changed, 24 insertions(+), 25 deletions(-)
 create mode 100644 doc/testimonials.mdwn

diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
index ce4ccee31d..d289b9f50a 100644
--- a/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex.mdwn
@@ -1,30 +1,6 @@
 Tell your git-annex stories here. Feel free to give as little or as much detail
 as appropriate about how you're using it. How's it working out for you?
 
-I'll start with some tweets and comments from IRC:
-
-
-
-
-
-
-
-
-
I want #git-annex whereis for all the stuff (not) in my room.
— topr (@derwelle) February 22, 2012
-
-
-
-oh my god, git-annex is amazing
-this is the revolution in fucking with gigantic piles of files that I've been waiting for
-

-
-And then my own story: I have a ton of drives. I have a lot of servers. I
-live in a cabin on **dialup** and often have 1 hour on broadband in a week
-to get everything I need. Without git-annex, managing all this would not be
-possible. It works perfectly for me, not a surprise since I wrote it, but
-still, it's a different level of "perfect" than anything I could put
-together before. --[[Joey]]
+See [[testimonials]] for some other stories.
 
 [[!meta author=Joey]]
diff --git a/doc/testimonials.mdwn b/doc/testimonials.mdwn
new file mode 100644
index 0000000000..b43fc76688
--- /dev/null
+++ b/doc/testimonials.mdwn
@@ -0,0 +1,23 @@
+
+
+
+
+
+
+
+
I want #git-annex whereis for all the stuff (not) in my room.
— topr (@derwelle) February 22, 2012
+
+
+
+oh my god, git-annex is amazing
+this is the revolution in fucking with gigantic piles of files that I've been waiting for
+

+
+And then my own story: I have a ton of drives. I have a lot of servers. I
+live in a cabin on **dialup** and often have 1 hour on broadband in a week
+to get everything I need. Without git-annex, managing all this would not be
+possible. It works perfectly for me, not a surprise since I wrote it, but
+still, it's a different level of "perfect" than anything I could put
+together before. --[[Joey]]

From a077b75a80440c18b4229c2dc4028dce961df83d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:37:31 -0400
Subject: [PATCH 3371/8313] markup

---
 doc/testimonials.mdwn | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/testimonials.mdwn b/doc/testimonials.mdwn
index b43fc76688..a340e6ebc7 100644
--- a/doc/testimonials.mdwn
+++ b/doc/testimonials.mdwn
@@ -1,14 +1,13 @@
+

 
 
-
 
 
-
 
 
I want #git-annex whereis for all the stuff (not) in my room.
— topr (@derwelle) February 22, 2012
 
+

 
 
 oh my god, git-annex is amazing

From 3d2cb43d1be45e9417707989752735a1d34b1c4e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:38:37 -0400
Subject: [PATCH 3372/8313] link

---
 doc/index.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/index.mdwn b/doc/index.mdwn
index a54a1073e7..2de17df3ce 100644
--- a/doc/index.mdwn
+++ b/doc/index.mdwn
@@ -13,6 +13,7 @@ To get a feel for it, see the [[walkthrough]] or read about [[how_it_works]].
 * [[forum]]
 * [[comments]]
 * [[contact]]
+* [[testimonials]]
 * Flattr this
 
 [[News]]:

From 46a9f902fe5400e1d841b98f66d66e401868aaec Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:43:47 -0400
Subject: [PATCH 3373/8313] pasto

---
 doc/testimonials.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/testimonials.mdwn b/doc/testimonials.mdwn
index a340e6ebc7..60d53f0509 100644
--- a/doc/testimonials.mdwn
+++ b/doc/testimonials.mdwn
@@ -4,7 +4,7 @@
 
 
 
-
 
 
 

From becf0628b7bd86e93f8d84f2456460b4c198f17a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:44:12 -0400
Subject: [PATCH 3374/8313] update

---
 doc/testimonials.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/testimonials.mdwn b/doc/testimonials.mdwn
index 60d53f0509..65bc5a5fe2 100644
--- a/doc/testimonials.mdwn
+++ b/doc/testimonials.mdwn
@@ -9,6 +9,7 @@
 
 
 
+Seen on IRC:
 
 oh my god, git-annex is amazing
 this is the revolution in fucking with gigantic piles of files that I've been waiting for

From 2cc13d1a4ad69943069e0eb91f6172eeba51733d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 13 Apr 2012 15:49:13 -0400
Subject: [PATCH 3375/8313] add a missing step

---
 doc/bare_repositories.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bare_repositories.mdwn b/doc/bare_repositories.mdwn
index bf56d81446..f40277d61d 100644
--- a/doc/bare_repositories.mdwn
+++ b/doc/bare_repositories.mdwn
@@ -26,6 +26,7 @@ Here is a quick example of how to set this up, using `origin` as the remote name
 On the server:
 
     mkdir bare-annex
+    cd bare-annex
     git init --bare
     git annex init origin
 

From 65d393a4bcff7bb5f406c18a45c3e8c9fbd28736 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Fri, 13 Apr 2012 19:49:23 +0000
Subject: [PATCH 3376/8313] Comment moderation

---
 ..._2_9f51947b35ee04e473655e20d56c740a._comment | 16 ++++++++++++++++
 ..._1_4884803ddee7f642a3ac995a19967a6a._comment | 17 +++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment
 create mode 100644 doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment

diff --git a/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment
new file mode 100644
index 0000000000..5d632ab860
--- /dev/null
+++ b/doc/forum/Preserving_file_access_rights_in_directory_tree_below_objects__47__/comment_2_9f51947b35ee04e473655e20d56c740a._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlB7-aXsqwzOi2BIR_Q4sUF8sjj24H6F3c"
+ nickname="Claudius"
+ subject="comment 2"
+ date="2012-01-23T19:39:17Z"
+ content="""
+Thank you for your comment! Indeed, setting the umask to, for example, 022 has the desired effect that annex/objects etc. are executable (and in this special case also writable), my previous umask setting was 077; the \"strange\" permissions on the git directories was probably due to --shared=all, and the mode of \"440\" on the files within the git-annex tree is correct (the original file was 640 and stripped of its write permission).
+
+Using this umask setting and newgrp to switch the default group, I was successfully able to set up the repositories.
+
+However, I would like to suggest adding the execute bit to the directories below .git/annex/objects/ per default, even if the umask of the current shell differs. As the correct rights are already preserved in the actual files (minus their write permission) together with correct owner and group, the files are still protected the same way as previously, and because +x does not allow directory listings, no additional information can leak out either. Not having to set the umask to something \"sensible\" before operating git-annex would be a huge plus, too :)
+
+The reason why I am not running MPD as my user is that I am a bit wary of running an application even exposed to the local network as my main user, and I see nothing wrong with running it as its own user.
+
+Thank you again for your help and the time you put into this project!
+"""]]
diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment
new file mode 100644
index 0000000000..2351378bca
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_1_4884803ddee7f642a3ac995a19967a6a._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="http://mildred.fr/"
+ subject="just amazing"
+ date="2012-04-12T17:12:41Z"
+ content="""
+git-annex is just amazing. I just started using it and for once, I have hope to be able to organize my files a little better than now.
+
+Currently, I have a huge homedir. From time to time, I move file away in external hard drives, then forget about them. When I want to look at them back, I just can't because I have forgotten where they are. I have also a ton of files on those drives that I can't access because they are not indexed. With git-annex I have hope to put all of these files on a git repository. I will be able to see them everywhere, and find them when I need to.
+
+I might stop loosing files for once.
+
+I might avoid having multiple copies of the same things over and over again, without knowing so. and regain some more disk space.
+
+For the moment, I'm archiving my photographs. But there is one thing that might not go very well: directory hierarchies where everything is important (file owner, specific permissions, symlinks). I won't just be able to blindly annex all of these files. But for the moment I'll stick at archiving ocuments and it should be amazing.
+
+[Mildred](http://mildred.fr)
+"""]]

From dd610c913f5276d51f5f0d299757fb6f1a582d5c Mon Sep 17 00:00:00 2001
From: "http://cgray.myopenid.com/" 
Date: Sat, 14 Apr 2012 01:18:54 +0000
Subject: [PATCH 3377/8313] Added a comment

---
 ...comment_2_61f5054918e7b36c191454365bc7f3b7._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment

diff --git a/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment
new file mode 100644
index 0000000000..3bd981c5db
--- /dev/null
+++ b/doc/forum/tell_us_how_you__39__re_using_git-annex/comment_2_61f5054918e7b36c191454365bc7f3b7._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://cgray.myopenid.com/"
+ nickname="cgray"
+ subject="comment 2"
+ date="2012-04-14T01:18:53Z"
+ content="""
+Git-annex has really helped me with my media files.  I have a big NAS drive where I keep all my music, tv, and movies files, each in their own git annex.  I tend to keep the media that I want to watch or listen to on my laptop and then drop it when it is done.  This way I don't have too much on my laptop at any one time, but I have a nice selection for when I'm traveling and don't have access to my NAS.
+
+Additionally, I have a mp3 player that will format itself randomly every few months or so.  I keep my podcasts on it in a git annex and in a git annex on my laptop.  When I am done with a podcast, I can delete it from the mp3 player and then sync that information with my laptop.  With this method, I have a backup of what should be on my mp3 player, so I don't need to worry about losing it all when the mp3 player decides it's had enough.
+"""]]

From 142bde13cd33481f415a5355a9cd9e74a716e3a8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Apr 2012 12:33:32 -0400
Subject: [PATCH 3378/8313] import juggling

---
 Utility/CopyFile.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index c42506485b..01639ef2a7 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -7,10 +7,7 @@
 
 module Utility.CopyFile (copyFileExternal) where
 
-import System.Directory (doesFileExist, removeFile)
-import Control.Monad.IfElse
-
-import Utility.SafeCommand
+import Common
 import qualified Build.SysConfig as SysConfig
 
 {- The cp command is used, because I hate reinventing the wheel,

From 1ca41044e8fab2b2e859a482e2293582c04db81f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Apr 2012 14:01:14 -0400
Subject: [PATCH 3379/8313] cabal now installs git-annex-shell as a symlink to
 git-annex.

---
 Setup.hs           | 17 ++++++++++++++++-
 debian/changelog   |  1 +
 git-annex-shell.hs | 13 -------------
 git-annex.cabal    |  4 ----
 4 files changed, 17 insertions(+), 18 deletions(-)
 delete mode 100644 git-annex-shell.hs

diff --git a/Setup.hs b/Setup.hs
index 14e6a4ea71..c36d6e4fe1 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -1,12 +1,27 @@
 {- cabal setup file -}
 
 import Distribution.Simple
+import Distribution.Simple.LocalBuildInfo
+import Distribution.Simple.Setup
 import System.Cmd
+import System.FilePath
 
 import qualified Build.Configure as Configure
 
-main = defaultMainWithHooks simpleUserHooks { preConf = configure }
+main = defaultMainWithHooks simpleUserHooks
+	{ preConf = configure
+	, instHook = install
+	}
 
 configure _ _ = do
 	Configure.run Configure.tests
 	return (Nothing, [])
+
+install pkg_descr lbi userhooks flags = do
+	r <- (instHook simpleUserHooks) pkg_descr lbi userhooks flags
+	_ <- rawSystem "ln" ["-sf", "git-annex", 
+		bindir installDirs  "git-annex-shell"]
+	return r
+	where
+		installDirs = absoluteInstallDirs pkg_descr lbi $
+			fromFlag (copyDest defaultCopyFlags)
diff --git a/debian/changelog b/debian/changelog
index abc3ae5120..73b4f31b08 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,7 @@ git-annex (3.20120407) UNRELEASED; urgency=low
     Damerau-Levenshtein edit distance, just as git does. This adds a build
     dependency on the haskell edit-distance library.
   * Renamed diskfree.c to avoid OSX case insensativity bug.
+  * cabal now installs git-annex-shell as a symlink to git-annex.
 
  -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
 
diff --git a/git-annex-shell.hs b/git-annex-shell.hs
deleted file mode 100644
index 08c1f9664d..0000000000
--- a/git-annex-shell.hs
+++ /dev/null
@@ -1,13 +0,0 @@
-{- git-annex-shell main program
- -
- - Copyright 2012 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
- -}
-
-import System.Environment
-
-import GitAnnexShell
-
-main :: IO ()
-main = run =<< getArgs
diff --git a/git-annex.cabal b/git-annex.cabal
index df40011895..6b1ebb42e6 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -36,10 +36,6 @@ Executable git-annex
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
 
-Executable git-annex-shell
-  Main-Is: git-annex-shell.hs
-  C-Sources: Utility/libdiskfree.c
-
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs

From 626697b459669d934da6339117f6f4abfce16f38 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Apr 2012 14:22:33 -0400
Subject: [PATCH 3380/8313] cabal file now autodetects whether S3 support is
 available.

---
 Locations.hs     |  2 +-
 Makefile         |  2 +-
 Remote/List.hs   |  4 ++++
 debian/changelog |  1 +
 git-annex.cabal  | 10 +++++++++-
 5 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/Locations.hs b/Locations.hs
index d263f3d2ac..67abf2166a 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -124,7 +124,7 @@ gitAnnexBadDir r = addTrailingPathSeparator $ gitAnnexDir r  "bad"
 gitAnnexBadLocation :: Key -> Git.Repo -> FilePath
 gitAnnexBadLocation key r = gitAnnexBadDir r  keyFile key
 
-{- .git/annex/*unused is used to number possibly unused keys -}
+{- .git/annex/foounused is used to number possibly unused keys -}
 gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath
 gitAnnexUnusedLog prefix r = gitAnnexDir r  (prefix ++ "unused")
 
diff --git a/Makefile b/Makefile
index eb30a3833c..87aa8c0767 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -cpp -DWITH_S3
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
diff --git a/Remote/List.hs b/Remote/List.hs
index 57dfa43ebf..c09341fb53 100644
--- a/Remote/List.hs
+++ b/Remote/List.hs
@@ -18,7 +18,9 @@ import Config
 import Remote.Helper.Hooks
 
 import qualified Remote.Git
+#ifdef WITH_S3
 import qualified Remote.S3
+#endif
 import qualified Remote.Bup
 import qualified Remote.Directory
 import qualified Remote.Rsync
@@ -28,7 +30,9 @@ import qualified Remote.Hook
 remoteTypes :: [RemoteType]
 remoteTypes =
 	[ Remote.Git.remote
+#ifdef WITH_S3
 	, Remote.S3.remote
+#endif
 	, Remote.Bup.remote
 	, Remote.Directory.remote
 	, Remote.Rsync.remote
diff --git a/debian/changelog b/debian/changelog
index 73b4f31b08..7b9fcde3fc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -9,6 +9,7 @@ git-annex (3.20120407) UNRELEASED; urgency=low
     dependency on the haskell edit-distance library.
   * Renamed diskfree.c to avoid OSX case insensativity bug.
   * cabal now installs git-annex-shell as a symlink to git-annex.
+  * cabal file now autodetects whether S3 support is available.
 
  -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 6b1ebb42e6..58370daf4b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -26,15 +26,23 @@ Description:
  etc that are associated with annexed files but that benefit from full
  revision control.
 
+Flag S3
+  Description: Enable S3 support
+
 Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
-   pcre-light, extensible-exceptions, dataenc, SHA, process, hS3, json, HTTP,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
+  Extensions: CPP
+
+  if flag(S3)
+    Build-Depends: hS3
+    CPP-Options: -DWITH_S3
 
 Test-Suite test
   Type: exitcode-stdio-1.0

From 61e5663097a2cf779d136b5a28500460293e9e3c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Apr 2012 15:23:13 -0400
Subject: [PATCH 3381/8313] make cabal test work again

---
 git-annex.cabal | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 58370daf4b..395042c9a6 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -47,8 +47,13 @@ Executable git-annex
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs
-  Build-Depends: testpack, HUnit
+  Build-Depends: testpack, HUnit, MissingH, hslogger, directory, filepath,
+   unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
+   pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
+   base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   C-Sources: Utility/libdiskfree.c
+  Extensions: CPP
 
 source-repository head
   type: git

From 70538dac844f00ccbdd0f2d0283fbf4e707cb84a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Apr 2012 16:01:08 -0400
Subject: [PATCH 3382/8313] compute distance in correct direction

---
 Git/AutoCorrect.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Git/AutoCorrect.hs b/Git/AutoCorrect.hs
index a18bf56197..a1ef14779b 100644
--- a/Git/AutoCorrect.hs
+++ b/Git/AutoCorrect.hs
@@ -34,7 +34,7 @@ fuzzymatches :: String -> (c -> String) -> [c] -> [c]
 fuzzymatches input showchoice choices = fst $ unzip $
 	sortBy comparecost $ filter similarEnough $ zip choices costs
         where
-                distance v = restrictedDamerauLevenshteinDistance gitEditCosts v input
+                distance = restrictedDamerauLevenshteinDistance gitEditCosts input
                 costs = map (distance . showchoice) choices
                 comparecost a b = compare (snd a) (snd b)
                 similarEnough (_, cst) = cst < similarityFloor

From 42cbb41ada884b21d7bbd369003b71752260c44e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 14 Apr 2012 16:01:22 -0400
Subject: [PATCH 3383/8313] always run autocorrect code on fuzzy matches, even
 if there is just 1

---
 CmdLine.hs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/CmdLine.hs b/CmdLine.hs
index 5330f40fc9..ebaef5369d 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -46,19 +46,19 @@ dispatch fuzzyok allargs allcmds commonoptions header getgitrepo = do
 	where
 		err msg = msg ++ "\n\n" ++ usage header allcmds commonoptions
 		cmd = Prelude.head cmds
-		(cmds, name, args) = findCmd fuzzyok allargs allcmds err
+		(fuzzy, cmds, name, args) = findCmd fuzzyok allargs allcmds err
 		(flags, params) = getOptCmd args cmd commonoptions err
-		checkfuzzy = when (length cmds > 1) $
+		checkfuzzy = when fuzzy $
 			inRepo $ Git.AutoCorrect.prepare name cmdname cmds
 
 {- Parses command line params far enough to find the Command to run, and
  - returns the remaining params.
  - Does fuzzy matching if necessary, which may result in multiple Commands. -}
-findCmd :: Bool -> Params -> [Command] -> (String -> String) -> ([Command], String, Params)
+findCmd :: Bool -> Params -> [Command] -> (String -> String) -> (Bool, [Command], String, Params)
 findCmd fuzzyok argv cmds err
 	| isNothing name = error $ err "missing command"
-	| not (null exactcmds) = (exactcmds, fromJust name, args)
-	| fuzzyok && not (null inexactcmds) = (inexactcmds, fromJust name, args)
+	| not (null exactcmds) = (False, exactcmds, fromJust name, args)
+	| fuzzyok && not (null inexactcmds) = (True, inexactcmds, fromJust name, args)
 	| otherwise = error $ err $ "unknown command " ++ fromJust name
 	where
 		(name, args) = findname argv []

From 4d221a7a80d9b02b523536b146fe94ce6448cfa5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 16 Apr 2012 16:35:47 -0400
Subject: [PATCH 3384/8313] the no-s3 branch is not needed, and there is a new
 no-ifelse branch

---
 doc/download.mdwn | 4 ++--
 doc/install.mdwn  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/download.mdwn b/doc/download.mdwn
index 64eee64c80..f0f17e141d 100644
--- a/doc/download.mdwn
+++ b/doc/download.mdwn
@@ -22,8 +22,8 @@ The git repository has some branches:
   had a major change to filename encoding.
 * `old-monad-control` is for systems that don't have a newer monad-control
   library.
-* `no-s3` disables the S3 special remote, for systems that lack the
-  necessary haskell library. (merge it into master if you need it)
+* `no-ifelse` avoids using the IFelse library
+  (merge it into master if you need it)
 * `no-bloom` avoids using bloom filters. (merge it into master if you need it)
 * `debian-stable` contains the latest backport of git-annex to Debian
   stable.
diff --git a/doc/install.mdwn b/doc/install.mdwn
index a02d9d2c74..2590060e69 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -35,7 +35,7 @@ To build and use git-annex, you will need:
   * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack)
   * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck)
   * [HTTP](http://hackage.haskell.org/package/HTTP)
-  * [hS3](http://hackage.haskell.org/package/hS3)
+  * [hS3](http://hackage.haskell.org/package/hS3) (optional)
   * [json](http://hackage.haskell.org/package/json)
   * [IfElse](http://hackage.haskell.org/package/IfElse)
   * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)

From aa353d1400512174ff3c5ccce89d1ebdd1a5be12 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 17 Apr 2012 18:37:40 -0400
Subject: [PATCH 3385/8313] use LANGUAGE CPP pragma, avoids running cpp on all
 the other sources

---
 Makefile       | 2 +-
 Remote/List.hs | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 87aa8c0767..94dc05a819 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd
-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -cpp -DWITH_S3
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
diff --git a/Remote/List.hs b/Remote/List.hs
index c09341fb53..14a1771b48 100644
--- a/Remote/List.hs
+++ b/Remote/List.hs
@@ -1,3 +1,5 @@
+{-# LANGUAGE CPP #-}
+
 {- git-annex remote list
  -
  - Copyright 2011 Joey Hess 

From 840315c3507d6a3df0d747f423244c132e41e3fa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 Apr 2012 12:22:22 -0400
Subject: [PATCH 3386/8313] releasing version 3.20120418

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7b9fcde3fc..e63e82f0e5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120407) UNRELEASED; urgency=low
+git-annex (3.20120418) unstable; urgency=low
 
   * bugfix: Adding a dotfile also caused all non-dotfiles to be added.
   * bup: Properly handle key names with spaces or other things that are
@@ -11,7 +11,7 @@ git-annex (3.20120407) UNRELEASED; urgency=low
   * cabal now installs git-annex-shell as a symlink to git-annex.
   * cabal file now autodetects whether S3 support is available.
 
- -- Joey Hess   Sun, 08 Apr 2012 12:23:42 -0400
+ -- Joey Hess   Wed, 18 Apr 2012 12:11:32 -0400
 
 git-annex (3.20120406) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 395042c9a6..041b077dc5 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120407
+Version: 3.20120418
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From f08943016708f8d4f25c557f255ef81e387bc522 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 Apr 2012 12:23:22 -0400
Subject: [PATCH 3387/8313] add news item for git-annex 3.20120418

---
 doc/news/version_3.20120230.mdwn | 13 -------------
 doc/news/version_3.20120418.mdwn | 12 ++++++++++++
 2 files changed, 12 insertions(+), 13 deletions(-)
 delete mode 100644 doc/news/version_3.20120230.mdwn
 create mode 100644 doc/news/version_3.20120418.mdwn

diff --git a/doc/news/version_3.20120230.mdwn b/doc/news/version_3.20120230.mdwn
deleted file mode 100644
index 52ac369e01..0000000000
--- a/doc/news/version_3.20120230.mdwn
+++ /dev/null
@@ -1,13 +0,0 @@
-git-annex 3.20120230 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * "here" can be used to refer to the current repository,
-     which can read better than the old "." (which still works too).
-   * Directory special remotes now support chunking files written to them,
-     avoiding writing files larger than a specified size.
-   * Add progress bar display to the directory special remote.
-   * Add configurable hooks that are run when git-annex starts and stops
-     using a remote: remote.name.annex-start-command and
-     remote.name.annex-stop-command
-   * Fix a bug in symlink calculation code, that triggered in rare
-     cases where an annexed file is in a subdirectory that nearly
-     matched to the .git/annex/object/xx/yy subdirectories."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120418.mdwn b/doc/news/version_3.20120418.mdwn
new file mode 100644
index 0000000000..93968a83e6
--- /dev/null
+++ b/doc/news/version_3.20120418.mdwn
@@ -0,0 +1,12 @@
+git-annex 3.20120418 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * bugfix: Adding a dotfile also caused all non-dotfiles to be added.
+   * bup: Properly handle key names with spaces or other things that are
+     not legal git refs.
+   * git-annex (but not git-annex-shell) supports the git help.autocorrect
+     configuration setting, doing fuzzy matching using the restricted
+     Damerau-Levenshtein edit distance, just as git does. This adds a build
+     dependency on the haskell edit-distance library.
+   * Renamed diskfree.c to avoid OSX case insensativity bug.
+   * cabal now installs git-annex-shell as a symlink to git-annex.
+   * cabal file now autodetects whether S3 support is available."""]]
\ No newline at end of file

From d75771b0ab88ebec9b573fb1b3ef257e3b19c3ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 Apr 2012 13:17:30 -0400
Subject: [PATCH 3388/8313] clear errno after successful call

When preparing the debian stable backport, I am seeing a call to statvfs()
succeed, but also set errno to 2 (ENOENT). Not sure why this happens;
I am in a schroot when it does happen, or perhaps stable's libc is a little
broken and sets errno incorrectly. Anyway, it should be perfectly fine to
clear errno after the successful call, rather than before it.
---
 Utility/libdiskfree.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Utility/libdiskfree.c b/Utility/libdiskfree.c
index b68abd0c47..54e8c08942 100644
--- a/Utility/libdiskfree.c
+++ b/Utility/libdiskfree.c
@@ -58,9 +58,10 @@ unsigned long long int diskfree(const char *path) {
 	unsigned long long int available, blocksize;
 	struct STATSTRUCT buf;
 
-	errno = 0;
-	if (STATCALL(path, &buf) != 0)
+	if (STATCALL(path, &buf) != 0) {
 		return 0; /* errno is set */
+	}
+	errno = 0;
 
 	available = buf.f_bavail;
 	blocksize = buf.f_bsize;

From 37061c019d816ff9ddd30cf23833539102741844 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 18 Apr 2012 13:23:33 -0400
Subject: [PATCH 3389/8313] tweak

---
 Utility/DiskFree.hs   | 2 +-
 Utility/libdiskfree.c | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs
index 8d07afaf2d..ff70705621 100644
--- a/Utility/DiskFree.hs
+++ b/Utility/DiskFree.hs
@@ -18,7 +18,7 @@ import Foreign.C.Error
 foreign import ccall unsafe "libdiskfree.h diskfree" c_diskfree
 	:: CString -> IO CULLong
 
-getDiskFree :: String -> IO (Maybe Integer)
+getDiskFree :: FilePath -> IO (Maybe Integer)
 getDiskFree path = withFilePath path $ \c_path -> do
 	free <- c_diskfree c_path
 	ifM (safeErrno <$> getErrno)
diff --git a/Utility/libdiskfree.c b/Utility/libdiskfree.c
index 54e8c08942..a37cb75713 100644
--- a/Utility/libdiskfree.c
+++ b/Utility/libdiskfree.c
@@ -58,10 +58,10 @@ unsigned long long int diskfree(const char *path) {
 	unsigned long long int available, blocksize;
 	struct STATSTRUCT buf;
 
-	if (STATCALL(path, &buf) != 0) {
+	if (STATCALL(path, &buf) != 0)
 		return 0; /* errno is set */
-	}
-	errno = 0;
+	else
+		errno = 0;
 
 	available = buf.f_bavail;
 	blocksize = buf.f_bsize;

From c908bd3b97eb84bbd8399951a1cf0ece4d824ea2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Apr 2012 11:31:30 -0400
Subject: [PATCH 3390/8313] some updates

---
 doc/internals.mdwn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index b2fd1e5545..a69a747e5d 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -39,6 +39,9 @@ space and then the description, followed by a timestamp. Example:
 	e605dca6-446a-11e0-8b2a-002170d25c55 laptop timestamp=1317929189.157237s
 	26339d22-446b-11e0-9101-002170d25c55 usb disk timestamp=1317929330.769997s
 
+If there are multiple lines for the same uuid, the one with the most recent
+timestamp wins. git-annex union merges this and other files.
+
 ## `remotes.log`
 
 Holds persistent configuration settings for [[special_remotes]] such as
@@ -80,7 +83,7 @@ Example:
 These files are designed to be auto-merged using git's [[union merge driver|git-union-merge]].
 The timestamps allow the most recent information to be identified.
 
-## `remote/web/aaa/bbb/*.log`
+## `aaa/bbb/*.log.web`
 
 These log files record urls used by the
 [[web_special_remote|special_remotes/web]]. Their format is similar

From 262017e17d466599f5b17313d69c995e844d59c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Apr 2012 14:57:57 -0400
Subject: [PATCH 3391/8313] export a more generalized checkDiskSpace

---
 Annex/Content.hs  | 45 +++++++++++++++++++++++----------------------
 Command/Unlock.hs |  3 +--
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 8542d8775d..494c9c10c1 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -127,15 +127,15 @@ getViaTmp key action = do
 	-- When the temp file already exists, count the space
 	-- it is using as free.
 	e <- liftIO $ doesFileExist tmp
-	if e
-		then do
-			stat <- liftIO $ getFileStatus tmp
-			checkDiskSpace' (fromIntegral $ fileSize stat) key
-		else checkDiskSpace key
-
-	when e $ liftIO $ allowWrite tmp
-
-	getViaTmpUnchecked key action
+	alreadythere <- if e
+		then fromIntegral . fileSize <$> liftIO (getFileStatus tmp)
+		else return 0
+	ifM (checkDiskSpace Nothing key alreadythere)
+		( do
+			when e $ liftIO $ allowWrite tmp
+			getViaTmpUnchecked key action
+		, return False
+		)
 
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
@@ -169,22 +169,23 @@ withTmp key action = do
 	return res
 
 {- Checks that there is disk space available to store a given key,
- - throwing an error if not. -}
-checkDiskSpace :: Key -> Annex ()
-checkDiskSpace = checkDiskSpace' 0
-
-checkDiskSpace' :: Integer -> Key -> Annex ()
-checkDiskSpace' adjustment key = do
+ - in a destination (or the annex) printing a warning if not. -}
+checkDiskSpace :: Maybe FilePath -> Key -> Integer -> Annex Bool
+checkDiskSpace destination key alreadythere = do
 	reserve <- getDiskReserve
-	free <- inRepo $ getDiskFree . gitAnnexDir
+	free <- liftIO . getDiskFree =<< dir
+	force <- Annex.getState Annex.force
 	case (free, keySize key) of
-		(Just have, Just need) ->
-			when (need + reserve > have + adjustment) $
-				needmorespace (need + reserve - have - adjustment)
-		_ -> return ()
+		(Just have, Just need) -> do
+			let ok = need + reserve > have + alreadythere || force
+			unless ok $
+				needmorespace (need + reserve - have - alreadythere)
+			return ok
+		_ -> return True
 	where
-		needmorespace n = unlessM (Annex.getState Annex.force) $
-			error $ "not enough free space, need " ++ 
+		dir = maybe (fromRepo gitAnnexDir) return destination
+		needmorespace n =
+			warning $ "not enough free space, need " ++ 
 				roughSize storageUnits True n ++
 				" more" ++ forcemsg
 		forcemsg = " (use --force to override this check or adjust annex.diskreserve)"
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index afee101459..aeb2705160 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -34,8 +34,7 @@ start file (key, _) = do
 perform :: FilePath -> Key -> CommandPerform
 perform dest key = do
 	unlessM (inAnnex key) $ error "content not present"
-	
-	checkDiskSpace key
+	unlessM (checkDiskSpace Nothing key 0) $ error "cannot unlock"
 
 	src <- inRepo $ gitAnnexLocation key
 	tmpdest <- fromRepo $ gitAnnexTmpLocation key

From e807502666b2fd649f133766be7a605a6338d9b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Apr 2012 16:14:29 -0400
Subject: [PATCH 3392/8313] had the wrong name for this

---
 Config.hs        | 2 +-
 debian/changelog | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/Config.hs b/Config.hs
index 10a66e47b1..087cb4043b 100644
--- a/Config.hs
+++ b/Config.hs
@@ -93,6 +93,6 @@ getTrustLevel r = fromRepo $ Git.Config.getMaybe $ remoteConfig r "trustlevel"
 {- Gets annex.diskreserve setting. -}
 getDiskReserve :: Annex Integer
 getDiskReserve = fromMaybe megabyte . readSize dataUnits
-	<$> getConfig "diskreserve" ""
+	<$> getConfig "annex.diskreserve" ""
 	where
 		megabyte = 1000000
diff --git a/debian/changelog b/debian/changelog
index e63e82f0e5..4520a05469 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120419) UNRELEASED; urgency=low
+
+  * Fix use of annex.diskreserve config setting.
+
+ -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
+
 git-annex (3.20120418) unstable; urgency=low
 
   * bugfix: Adding a dotfile also caused all non-dotfiles to be added.

From b65e257b13773bd44ccf9cc734c42927e94ed929 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Apr 2012 16:16:13 -0400
Subject: [PATCH 3393/8313] inverted logic

---
 Annex/Content.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 494c9c10c1..932f1b7553 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -177,8 +177,9 @@ checkDiskSpace destination key alreadythere = do
 	force <- Annex.getState Annex.force
 	case (free, keySize key) of
 		(Just have, Just need) -> do
-			let ok = need + reserve > have + alreadythere || force
-			unless ok $
+			let ok = (need + reserve <= have + alreadythere) || force
+			unless ok $ do
+				liftIO $ print (need, reserve, have, alreadythere)
 				needmorespace (need + reserve - have - alreadythere)
 			return ok
 		_ -> return True

From 5cc76098ca7b702772ccf37a47f03da088148003 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Apr 2012 16:24:44 -0400
Subject: [PATCH 3394/8313] Directory special remotes now check
 annex.diskreserve.

---
 Remote/Directory.hs | 52 ++++++++++++++++++++++++++-------------------
 debian/changelog    |  1 +
 2 files changed, 31 insertions(+), 22 deletions(-)

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 3627d9a9ad..fd5a6f0b19 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -10,7 +10,7 @@ module Remote.Directory (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.ByteString.Char8 as S
 import qualified Data.Map as M
-import Control.Exception (bracket)
+import qualified Control.Exception as E
 
 import Common.Annex
 import Types.Remote
@@ -22,6 +22,7 @@ import Remote.Helper.Encryptable
 import Crypto
 import Utility.DataUnits
 import Data.Int
+import Annex.Content
 
 remote :: RemoteType
 remote = RemoteType {
@@ -125,7 +126,7 @@ store :: FilePath -> ChunkSize -> Key -> Annex Bool
 store d chunksize k = do
 	src <- inRepo $ gitAnnexLocation k
 	metered k $ \meterupdate -> 
-		liftIO $ catchBoolIO $ storeHelper d chunksize k $ \dests ->
+		storeHelper d chunksize k $ \dests ->
 			case chunksize of
 				Nothing -> do
 					let dest = Prelude.head dests
@@ -140,7 +141,7 @@ storeEncrypted :: FilePath -> ChunkSize -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted d chunksize (cipher, enck) k = do
 	src <- inRepo $ gitAnnexLocation k
 	metered k $ \meterupdate ->
-		liftIO $ catchBoolIO $ storeHelper d chunksize enck $ \dests ->
+		storeHelper d chunksize enck $ \dests ->
 			withEncryptedContent cipher (L.readFile src) $ \s ->
 				case chunksize of
 					Nothing -> do
@@ -165,7 +166,7 @@ storeSplit' :: MeterUpdate -> Int64 -> [FilePath] -> [S.ByteString] -> [FilePath
 storeSplit' _ _ [] _ _ = error "ran out of dests"
 storeSplit' _ _  _ [] c = return $ reverse c
 storeSplit' meterupdate chunksize (d:dests) bs c = do
-	bs' <- bracket (openFile d WriteMode) hClose (feed chunksize bs)
+	bs' <- E.bracket (openFile d WriteMode) hClose (feed chunksize bs)
 	storeSplit' meterupdate chunksize dests bs' (d:c)
 	where
 		feed _ [] _ = return []
@@ -190,7 +191,7 @@ meteredWriteFile meterupdate dest b =
  - meter after each chunk. The feeder is called to get more chunks. -}
 meteredWriteFile' :: MeterUpdate -> FilePath -> s -> (s -> IO (s, [S.ByteString])) -> IO ()
 meteredWriteFile' meterupdate dest startstate feeder =
-	bracket (openFile dest WriteMode) hClose (feed startstate [])
+	E.bracket (openFile dest WriteMode) hClose (feed startstate [])
 	where
 		feed state [] h = do
 			(state', cs) <- feeder state
@@ -207,31 +208,38 @@ meteredWriteFile' meterupdate dest startstate feeder =
  - The stored files are only put into their final place once storage is
  - complete.
  -}
-storeHelper :: FilePath -> ChunkSize -> Key -> ([FilePath] -> IO [FilePath]) -> IO Bool
-storeHelper d chunksize key a = do
-	let dir = parentDir desttemplate
-	createDirectoryIfMissing True dir
-	allowWrite dir
-	stored <- a tmpdests
-	forM_ stored $ \f -> do
-		let dest = detmpprefix f
-		renameFile f dest
-		preventWrite dest
-	when (chunksize /= Nothing) $ do
-		let chunkcount = chunkCount desttemplate
-		_ <- tryIO $ allowWrite chunkcount
-		writeFile chunkcount (show $ length stored)
-		preventWrite chunkcount
-	preventWrite dir
-	return (not $ null stored)
+storeHelper :: FilePath -> ChunkSize -> Key -> ([FilePath] -> IO [FilePath]) -> Annex Bool
+storeHelper d chunksize key a = prep <&&> check <&&> go
 	where
 		desttemplate = Prelude.head $ locations d key
+		dir = parentDir desttemplate
 		tmpdests = case chunksize of
 			Nothing -> [desttemplate ++ tmpprefix]
 			Just _ -> map (++ tmpprefix) (chunkStream desttemplate)
 		tmpprefix = ".tmp"
 		detmpprefix f = take (length f - tmpprefixlen) f
 		tmpprefixlen = length tmpprefix
+		prep = liftIO $ catchBoolIO $ do
+			createDirectoryIfMissing True dir
+			allowWrite dir
+			return True
+		{- The size is not exactly known when encrypting the key;
+		 - this assumes that at least the size of the key is
+		 - needed as free space. -}
+		check = checkDiskSpace (Just dir) key 0
+		go = liftIO $ catchBoolIO $ do
+			stored <- a tmpdests
+			forM_ stored $ \f -> do
+				let dest = detmpprefix f
+				renameFile f dest
+				preventWrite dest
+			when (chunksize /= Nothing) $ do
+				let chunkcount = chunkCount desttemplate
+				_ <- tryIO $ allowWrite chunkcount
+				writeFile chunkcount (show $ length stored)
+				preventWrite chunkcount
+			preventWrite dir
+			return (not $ null stored)
 
 retrieve :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool
 retrieve d chunksize k f = metered k $ \meterupdate ->
diff --git a/debian/changelog b/debian/changelog
index 4520a05469..eeb4bdfe47 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20120419) UNRELEASED; urgency=low
 
   * Fix use of annex.diskreserve config setting.
+  * Directory special remotes now check annex.diskreserve.
 
  -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
 

From c8940c02e3d5577bbe3acdb79d0641226fa6d529 Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Sat, 21 Apr 2012 15:07:47 +0000
Subject: [PATCH 3395/8313] getting "permission denied" in fsck, looking for
 cause

---
 ...ion_denied__34___in_fsck_on_shared_repo.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn

diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn
new file mode 100644
index 0000000000..f13aed2c2f
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo.mdwn
@@ -0,0 +1,17 @@
+i'm getting errors in ``git annex fsck`` on a shared bare git repo with git-annex 3.20120418 local repo version 3:
+
+``git-annex: ${PATH}/${MYREPO}.git/annex/objects/${HA}/${SH}/SHA1-${HASH}/SHA1-${HASH}: setFileMode: permission denied (Operation not permitted)``
+
+the repository is shared among several users in a common group, and the repo is set up with sticky group, and with appropriate umasks, everything should work.
+
+however, even with the file having permissions -rw-rw-r-- in the directory with permissions drwxrwsr-x, owned by someone else but by a group i'm currently in (as verified by issuing `groups`), i get said error message.
+
+a strace reveals that the failing syscall is:
+
+``[pid 17626] chmod("${FILENAME}", 0100555) = -1 EPERM (Operation not permitted)``
+
+(maybe related: git annex looks for the file in another ${HA}/${SH} combination (of three digits instead of two digits each) before, i take it this is just a new feature not used by the data in my repo? also, i should add that the repository dates back to git-annex 0.13.)
+
+as a workaround, i'm currently ``sudo chown``ing all files to me before the check.
+
+why does fsck try to set permissions even if they are ok? is this a bug in my setup, and if yes, how is a shared repository set up correctly?

From cbf9a9420e5d0fd9af2ad7d2864618386a75cfc3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 11:57:17 -0400
Subject: [PATCH 3396/8313] avoid chown call if the current file mode is same
 as new

Not only an optimisation. fsck always tried to preventWrite to make sure
file modes are good, and in a shared repo, that will fail on directories
not owned by the current user. Although there may be other problems with
such a setup.
---
 Utility/FileMode.hs | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index 571b035039..4992690c6a 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -7,16 +7,24 @@
 
 module Utility.FileMode where
 
-import System.Posix.Files
+import Common
+
 import System.Posix.Types
 import Foreign (complement)
 
+modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO ()
+modifyFileMode f convert = do
+	s <- getFileStatus f
+	let cur = fileMode s
+	let new = convert cur
+	when (new /= cur) $
+		setFileMode f new
+
 {- Removes a FileMode from a file.
  - For example, call with otherWriteMode to chmod o-w -}
 unsetFileMode :: FilePath -> FileMode -> IO ()
-unsetFileMode f m = do
-	s <- getFileStatus f
-	setFileMode f $ fileMode s `intersectFileModes` complement m
+unsetFileMode f m = modifyFileMode f $
+	\cur -> cur `intersectFileModes` complement m
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
@@ -27,9 +35,8 @@ preventWrite f = unsetFileMode f writebits
 
 {- Turns a file's write bit back on. -}
 allowWrite :: FilePath -> IO ()
-allowWrite f = do
-	s <- getFileStatus f
-	setFileMode f $ fileMode s `unionFileModes` ownerWriteMode
+allowWrite f = modifyFileMode f $
+	\cur -> cur `unionFileModes` ownerWriteMode
 
 {- Checks if a file mode indicates it's a symlink. -}
 isSymLink :: FileMode -> Bool

From 9c87e3014050742faf0b32ad9a4190df062ce94c Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 21 Apr 2012 16:09:19 +0000
Subject: [PATCH 3397/8313] Added a comment

---
 ...comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment

diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment
new file mode 100644
index 0000000000..5a5cafa721
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_1_3a5202ef2116ebb5559b6f4d920755fc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-21T16:09:19Z"
+ content="""
+Well, the modes you show are wrong. Nothing in the annex should be writable. fsck needs to fix those. (It's true that it also always chmods even correct mode files/directories.. I've made a change avoiding that.)
+
+I have not thought or tried shared git annex repos with multiple unix users writing to them. ([[tips/Using_gitolite_with_git-annex]] would be an alternative.) Seems to me that removing content from the annex would also be a problem, since the directory will need to be chmodded to allow deleting the content from it, and that will fail if it's owned by someone else.  Perhaps git-annex needs to honor core.sharedRepository and avoid these nice safeguards on file modes then.
+"""]]

From b4a5e39ee62020380fc0dcf7aecaaf593d44dba5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 14:06:36 -0400
Subject: [PATCH 3398/8313] Support git's core.sharedRepository configuration

This is incomplete, it does not honor it yet for hash directories
and other annex bookkeeping files. Some of that is not needed for a bare
repo; some of it may be.
---
 Annex/Content.hs        | 104 +++++++++++++++++++++++++++-------------
 Command/Fsck.hs         |   7 ++-
 Command/Unannex.hs      |   6 +--
 Command/Unlock.hs       |   2 +-
 Git/SharedRepository.hs |  27 +++++++++++
 Utility/FileMode.hs     |  60 +++++++++++++++++++----
 debian/changelog        |   1 +
 7 files changed, 155 insertions(+), 52 deletions(-)
 create mode 100644 Git/SharedRepository.hs

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 932f1b7553..616e4128af 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -23,10 +23,11 @@ module Annex.Content (
 	saveState,
 	downloadUrl,
 	preseedTmp,
+	freezeContent,
+	thawContent,
+	freezeContentDir,
 ) where
 
-import Control.Exception (bracket_)
-import System.Posix.Types
 import System.IO.Unsafe (unsafeInterleaveIO)
 
 import Common.Annex
@@ -44,6 +45,7 @@ import Utility.DataUnits
 import Utility.CopyFile
 import Config
 import Annex.Exception
+import Git.SharedRepository
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
@@ -57,8 +59,10 @@ inAnnex' a key = do
 {- A safer check; the key's content must not only be present, but
  - is not in the process of being removed. -}
 inAnnexSafe :: Key -> Annex (Maybe Bool)
-inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
+inAnnexSafe = inAnnex' $ \f -> openforlock f >>= check
 	where
+		openforlock f = catchMaybeIO $
+			openFd f ReadOnly Nothing defaultFileFlags
 		check Nothing = return is_missing
 		check (Just h) = do
 			v <- getLock h (ReadLock, AbsoluteSeek, 0, 0)
@@ -75,30 +79,27 @@ inAnnexSafe = inAnnex' $ \f -> openForLock f False >>= check
 lockContent :: Key -> Annex a -> Annex a
 lockContent key a = do
 	file <- inRepo $ gitAnnexLocation key
-	bracketIO (openForLock file True >>= lock) unlock a
+	bracketIO (openforlock file >>= lock) unlock a
 	where
+		{- Since files are stored with the write bit disabled, have
+		 - to fiddle with permissions to open for an exclusive lock. -}
+		openforlock f = catchMaybeIO $ ifM (doesFileExist f)
+			( withModifiedFileMode f
+				(\cur -> cur `unionFileModes` ownerWriteMode)
+				open
+			, open
+			)
+			where
+				open = openFd f ReadWrite Nothing defaultFileFlags
 		lock Nothing = return Nothing
-		lock (Just l) = do
-			v <- tryIO $ setLock l (WriteLock, AbsoluteSeek, 0, 0)
+		lock (Just fd) = do
+			v <- tryIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
 				Left _ -> error "content is locked"
-				Right _ -> return $ Just l
+				Right _ -> return $ Just fd
 		unlock Nothing = return ()
 		unlock (Just l) = closeFd l
 
-openForLock :: FilePath -> Bool -> IO (Maybe Fd)
-openForLock file writelock = bracket_ prep cleanup go
-	where
-		go = catchMaybeIO $ openFd file mode Nothing defaultFileFlags
-		mode = if writelock then ReadWrite else ReadOnly
-		{- Since files are stored with the write bit disabled,
-		 - have to fiddle with permissions to open for an
-		 - exclusive lock. -}
-		forwritelock a = 
-			when writelock $ whenM (doesFileExist file) a
-		prep = forwritelock $ allowWrite file
-		cleanup = forwritelock $ preventWrite file
-
 {- Calculates the relative path to use to link a file to a key. -}
 calcGitLink :: FilePath -> Key -> Annex FilePath
 calcGitLink file key = do
@@ -132,7 +133,7 @@ getViaTmp key action = do
 		else return 0
 	ifM (checkDiskSpace Nothing key alreadythere)
 		( do
-			when e $ liftIO $ allowWrite tmp
+			when e $ thawContent tmp
 			getViaTmpUnchecked key action
 		, return False
 		)
@@ -216,14 +217,15 @@ moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
 	dest <- inRepo $ gitAnnexLocation key
 	let dir = parentDir dest
-	liftIO $ ifM (doesFileExist dest)
-		( removeFile src
+	ifM (liftIO $ doesFileExist dest)
+		( liftIO $ removeFile src
 		, do
-			createDirectoryIfMissing True dir
-			allowWrite dir -- in case the directory already exists
-			moveFile src dest
-			preventWrite dest
-			preventWrite dir
+			liftIO $ do
+				createDirectoryIfMissing True dir
+				allowWrite dir -- in case the directory already exists
+				moveFile src dest
+			freezeContent dest
+			freezeContentDir dest
 		)
 
 withObjectLoc :: Key -> ((FilePath, FilePath) -> Annex a) -> Annex a
@@ -254,10 +256,9 @@ removeAnnex key = withObjectLoc key $ \(dir, file) -> do
 {- Moves a key's file out of .git/annex/objects/ -}
 fromAnnex :: Key -> FilePath -> Annex ()
 fromAnnex key dest = withObjectLoc key $ \(dir, file) -> do
-	liftIO $ do
-		allowWrite dir
-		allowWrite file
-		moveFile file dest
+	liftIO $ allowWrite dir
+	thawContent file
+	liftIO $ moveFile file dest
 	cleanObjectLoc key
 
 {- Moves a key out of .git/annex/objects/ into .git/annex/bad, and
@@ -321,7 +322,7 @@ preseedTmp key file = go =<< inAnnex key
 		go False = return False
 		go True = do
 			ok <- copy
-			when ok $ liftIO $ allowWrite file
+			when ok $ thawContent file
 			return ok
 		copy = ifM (liftIO $ doesFileExist file)
 				( return True
@@ -329,3 +330,40 @@ preseedTmp key file = go =<< inAnnex key
 					s <- inRepo $ gitAnnexLocation key
 					liftIO $ copyFileExternal s file
 				)
+
+{- Blocks writing to an annexed file. The file is made unwritable
+ - to avoid accidental edits. core.sharedRepository may change
+ - who can read it. -}
+freezeContent :: FilePath -> Annex ()
+freezeContent file = liftIO . go =<< fromRepo getSharedRepository
+	where
+		go GroupShared = do
+			preventWrite file
+			groupRead file
+		go AllShared = do
+			preventWrite file
+			allRead file
+		go _ = preventWrite file
+
+{- Allows writing to an annexed file that freezeContent was called on
+ - before. -}
+thawContent :: FilePath -> Annex ()
+thawContent file = liftIO . go =<< fromRepo getSharedRepository
+	where
+		go GroupShared = groupWriteRead file
+		go AllShared = groupWriteRead file
+		go _ = allowWrite file
+
+{- Blocks writing to the directory an annexed file is in, to prevent the
+ - file accidentially being deleted. However, if core.sharedRepository
+ - is set, this is not done, since the group must be allowed to delete the
+ - file.
+ -}
+freezeContentDir :: FilePath -> Annex ()
+freezeContentDir file = liftIO . go =<< fromRepo getSharedRepository
+	where
+		dir = parentDir file
+		go GroupShared = groupWriteRead dir
+		go AllShared = groupWriteRead dir
+		go _ = preventWrite dir
+
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index dac3bfac96..c60101fc79 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -166,10 +166,9 @@ verifyLocationLog key desc = do
 	-- Since we're checking that a key's file is present, throw
 	-- in a permission fixup here too.
 	when present $ do
-		f <- inRepo $ gitAnnexLocation key
-		liftIO $ do
-			preventWrite f
-			preventWrite (parentDir f)
+		file <- inRepo $ gitAnnexLocation key
+		freezeContent file
+		freezeContentDir file
 
 	u <- getUUID
 	verifyLocationLog' key desc present u (logChange key u)
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index 1e7313711c..bf931adfd4 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -10,7 +10,6 @@ module Command.Unannex where
 import Common.Annex
 import Command
 import qualified Annex
-import Utility.FileMode
 import Logs.Location
 import Annex.Content
 import qualified Git.Command
@@ -51,9 +50,8 @@ cleanup file key = do
 		( do
 			-- fast mode: hard link to content in annex
 			src <- inRepo $ gitAnnexLocation key
-			liftIO $ do
-				createLink src file
-				allowWrite file
+			liftIO $ createLink src file
+			thawContent file
 		, do
 			fromAnnex key file
 			logStatus key InfoMissing
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index aeb2705160..3ac50a0ebb 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -46,6 +46,6 @@ perform dest key = do
 			liftIO $ do
 				removeFile dest
 				moveFile tmpdest dest
-				allowWrite dest
+			thawContent dest
 			next $ return True
                 else error "copy failed!"
diff --git a/Git/SharedRepository.hs b/Git/SharedRepository.hs
new file mode 100644
index 0000000000..f3efa8fde9
--- /dev/null
+++ b/Git/SharedRepository.hs
@@ -0,0 +1,27 @@
+{- git core.sharedRepository handling
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.SharedRepository where
+
+import Data.Char
+
+import Common
+import Git
+import qualified Git.Config
+
+data SharedRepository = UnShared | GroupShared | AllShared | UmaskShared Int
+
+getSharedRepository :: Repo -> SharedRepository
+getSharedRepository r =
+	case map toLower $ Git.Config.get "core.sharedrepository" "" r of
+		"1" -> GroupShared
+		"group" -> GroupShared
+		"true" -> GroupShared
+		"all" -> AllShared
+		"world" -> AllShared
+		"everybody" -> AllShared
+		v -> maybe UnShared UmaskShared (readish v)
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index 4992690c6a..98c7124c2a 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -1,6 +1,6 @@
 {- File mode utilities.
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -9,16 +9,36 @@ module Utility.FileMode where
 
 import Common
 
+import Control.Exception (bracket)
 import System.Posix.Types
 import Foreign (complement)
 
+combineModes :: [FileMode] -> FileMode
+combineModes [] = undefined
+combineModes [m] = m
+combineModes (m:ms) = foldl unionFileModes m ms
+
+{- Applies a conversion function to a file's mode. -}
 modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO ()
 modifyFileMode f convert = do
+	_ <- modifyFileMode' f convert
+	return ()
+modifyFileMode' :: FilePath -> (FileMode -> FileMode) -> IO FileMode
+modifyFileMode' f convert = do
 	s <- getFileStatus f
-	let cur = fileMode s
-	let new = convert cur
-	when (new /= cur) $
+	let old = fileMode s
+	let new = convert old
+	when (new /= old) $
 		setFileMode f new
+	return old
+
+{- Runs an action after changing a file's mode, then restores the old mode. -}
+withModifiedFileMode :: FilePath -> (FileMode -> FileMode) -> IO a -> IO a
+withModifiedFileMode file convert a = bracket setup cleanup go
+	where
+		setup = modifyFileMode' file convert
+		cleanup oldmode = modifyFileMode file (const oldmode)
+		go _ = a
 
 {- Removes a FileMode from a file.
  - For example, call with otherWriteMode to chmod o-w -}
@@ -28,23 +48,43 @@ unsetFileMode f m = modifyFileMode f $
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
-preventWrite f = unsetFileMode f writebits
+preventWrite f = unsetFileMode f $ combineModes writebits
 	where
-		writebits = foldl unionFileModes ownerWriteMode
-					[groupWriteMode, otherWriteMode]
+		writebits = [ownerWriteMode, groupWriteMode, otherWriteMode]
 
 {- Turns a file's write bit back on. -}
 allowWrite :: FilePath -> IO ()
 allowWrite f = modifyFileMode f $
 	\cur -> cur `unionFileModes` ownerWriteMode
 
+{- Allows owner and group to read and write to a file. -}
+groupWriteRead :: FilePath -> IO ()
+groupWriteRead f = modifyFileMode f $ \cur -> combineModes
+	[ cur
+	, ownerWriteMode, groupWriteMode
+	, ownerReadMode, groupReadMode
+	]
+
+{- Allows group to read a file. -}
+groupRead :: FilePath -> IO ()
+groupRead f = modifyFileMode f $ \cur -> combineModes
+	[ cur
+	, ownerReadMode, groupReadMode
+	]
+
+{- Allows all to read a file. -}
+allRead :: FilePath -> IO ()
+allRead f = modifyFileMode f $ \cur -> combineModes
+	[ cur
+	, ownerReadMode, groupReadMode, otherReadMode
+	]
+
 {- Checks if a file mode indicates it's a symlink. -}
 isSymLink :: FileMode -> Bool
 isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode
 
 {- Checks if a file has any executable bits set. -}
 isExecutable :: FileMode -> Bool
-isExecutable mode = ebits `intersectFileModes` mode /= 0
+isExecutable mode = combineModes ebits `intersectFileModes` mode /= 0
 	where
-		ebits = ownerExecuteMode `unionFileModes`
-			groupExecuteMode `unionFileModes` otherExecuteMode
+		ebits = [ownerExecuteMode, groupExecuteMode, otherExecuteMode]
diff --git a/debian/changelog b/debian/changelog
index eeb4bdfe47..3c56fda51a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20120419) UNRELEASED; urgency=low
 
   * Fix use of annex.diskreserve config setting.
   * Directory special remotes now check annex.diskreserve.
+  * Support git's core.sharedRepository configuration.
 
  -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
 

From 7e45712d194aa2b231083c3ccee3668f053e5717 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 16:01:56 -0400
Subject: [PATCH 3399/8313] better file mode setting code

---
 Annex/Content.hs    | 12 +++++-----
 Command/Unlock.hs   |  1 -
 Utility/FileMode.hs | 57 +++++++++++++++++++--------------------------
 3 files changed, 30 insertions(+), 40 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 616e4128af..b216b861d9 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -337,12 +337,12 @@ preseedTmp key file = go =<< inAnnex key
 freezeContent :: FilePath -> Annex ()
 freezeContent file = liftIO . go =<< fromRepo getSharedRepository
 	where
-		go GroupShared = do
-			preventWrite file
-			groupRead file
-		go AllShared = do
-			preventWrite file
-			allRead file
+		go GroupShared = modifyFileMode file $
+			removeModes writeModes .
+			addModes [ownerReadMode, groupReadMode]
+		go AllShared = modifyFileMode file $
+			removeModes writeModes .
+			addModes readModes
 		go _ = preventWrite file
 
 {- Allows writing to an annexed file that freezeContent was called on
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index 3ac50a0ebb..f3ffd31ba6 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -11,7 +11,6 @@ import Common.Annex
 import Command
 import Annex.Content
 import Utility.CopyFile
-import Utility.FileMode
 
 def :: [Command]
 def =
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index 98c7124c2a..f3db709231 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -13,11 +13,6 @@ import Control.Exception (bracket)
 import System.Posix.Types
 import Foreign (complement)
 
-combineModes :: [FileMode] -> FileMode
-combineModes [] = undefined
-combineModes [m] = m
-combineModes (m:ms) = foldl unionFileModes m ms
-
 {- Applies a conversion function to a file's mode. -}
 modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO ()
 modifyFileMode f convert = do
@@ -32,6 +27,15 @@ modifyFileMode' f convert = do
 		setFileMode f new
 	return old
 
+{- Adds the specified FileModes to the input mode, leaving the rest
+ - unchanged. -}
+addModes :: [FileMode] -> FileMode -> FileMode
+addModes ms m = combineModes (m:ms)
+
+{- Removes the specified FileModes from the input mode. -}
+removeModes :: [FileMode] -> FileMode -> FileMode
+removeModes ms m = m `intersectFileModes` complement (combineModes ms)
+
 {- Runs an action after changing a file's mode, then restores the old mode. -}
 withModifiedFileMode :: FilePath -> (FileMode -> FileMode) -> IO a -> IO a
 withModifiedFileMode file convert a = bracket setup cleanup go
@@ -40,45 +44,27 @@ withModifiedFileMode file convert a = bracket setup cleanup go
 		cleanup oldmode = modifyFileMode file (const oldmode)
 		go _ = a
 
-{- Removes a FileMode from a file.
- - For example, call with otherWriteMode to chmod o-w -}
-unsetFileMode :: FilePath -> FileMode -> IO ()
-unsetFileMode f m = modifyFileMode f $
-	\cur -> cur `intersectFileModes` complement m
+writeModes :: [FileMode]
+writeModes = [ownerWriteMode, groupWriteMode, otherWriteMode]
+
+readModes :: [FileMode]
+readModes = [ownerReadMode, groupReadMode, otherReadMode]
 
 {- Removes the write bits from a file. -}
 preventWrite :: FilePath -> IO ()
-preventWrite f = unsetFileMode f $ combineModes writebits
-	where
-		writebits = [ownerWriteMode, groupWriteMode, otherWriteMode]
+preventWrite f = modifyFileMode f $ removeModes writeModes
 
-{- Turns a file's write bit back on. -}
+{- Turns a file's owner write bit back on. -}
 allowWrite :: FilePath -> IO ()
-allowWrite f = modifyFileMode f $
-	\cur -> cur `unionFileModes` ownerWriteMode
+allowWrite f = modifyFileMode f $ addModes [ownerWriteMode]
 
 {- Allows owner and group to read and write to a file. -}
 groupWriteRead :: FilePath -> IO ()
-groupWriteRead f = modifyFileMode f $ \cur -> combineModes
-	[ cur
-	, ownerWriteMode, groupWriteMode
+groupWriteRead f = modifyFileMode f $ addModes
+	[ ownerWriteMode, groupWriteMode
 	, ownerReadMode, groupReadMode
 	]
 
-{- Allows group to read a file. -}
-groupRead :: FilePath -> IO ()
-groupRead f = modifyFileMode f $ \cur -> combineModes
-	[ cur
-	, ownerReadMode, groupReadMode
-	]
-
-{- Allows all to read a file. -}
-allRead :: FilePath -> IO ()
-allRead f = modifyFileMode f $ \cur -> combineModes
-	[ cur
-	, ownerReadMode, groupReadMode, otherReadMode
-	]
-
 {- Checks if a file mode indicates it's a symlink. -}
 isSymLink :: FileMode -> Bool
 isSymLink mode = symbolicLinkMode `intersectFileModes` mode == symbolicLinkMode
@@ -88,3 +74,8 @@ isExecutable :: FileMode -> Bool
 isExecutable mode = combineModes ebits `intersectFileModes` mode /= 0
 	where
 		ebits = [ownerExecuteMode, groupExecuteMode, otherExecuteMode]
+
+combineModes :: [FileMode] -> FileMode
+combineModes [] = undefined
+combineModes [m] = m
+combineModes (m:ms) = foldl unionFileModes m ms

From b98b69e8c6d9b873a864b79cff857882f67ee576 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 16:59:49 -0400
Subject: [PATCH 3400/8313] honor core.sharedRepository when making all the
 other files in the annex

Lock files, directories, etc.
---
 Annex/Branch.hs        |  2 ++
 Annex/Content.hs       | 22 ++++++++++-----
 Annex/Journal.hs       | 41 +++++++++++++---------------
 Annex/LockPool.hs      |  5 +++-
 Annex/Perms.hs         | 61 ++++++++++++++++++++++++++++++++++++++++++
 Annex/Ssh.hs           |  5 +++-
 Remote/Helper/Hooks.hs |  5 +++-
 Utility/FileMode.hs    | 11 ++++++++
 8 files changed, 119 insertions(+), 33 deletions(-)
 create mode 100644 Annex/Perms.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 52089ac97d..e5976c2c01 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -36,6 +36,7 @@ import qualified Git.UnionMerge
 import Git.HashObject
 import qualified Git.Index
 import Annex.CatFile
+import Annex.Perms
 
 {- Name of the branch that is used to store git-annex's information. -}
 name :: Git.Ref
@@ -308,6 +309,7 @@ setIndexSha :: Git.Ref -> Annex ()
 setIndexSha ref = do
         lock <- fromRepo gitAnnexIndexLock
 	liftIO $ writeFile lock $ show ref ++ "\n"
+	setAnnexPerm lock
 
 {- Checks if there are uncommitted changes in the branch's index or journal. -}
 unCommitted :: Annex Bool
diff --git a/Annex/Content.hs b/Annex/Content.hs
index b216b861d9..7022364d09 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -46,6 +46,7 @@ import Utility.CopyFile
 import Config
 import Annex.Exception
 import Git.SharedRepository
+import Annex.Perms
 
 {- Checks if a given key's content is currently present. -}
 inAnnex :: Key -> Annex Bool
@@ -141,7 +142,7 @@ getViaTmp key action = do
 prepTmp :: Key -> Annex FilePath
 prepTmp key = do
 	tmp <- fromRepo $ gitAnnexTmpLocation key
-	liftIO $ createDirectoryIfMissing True (parentDir tmp)
+	createAnnexDirectory (parentDir tmp)
 	return tmp
 
 {- Like getViaTmp, but does not check that there is enough disk space
@@ -216,14 +217,11 @@ checkDiskSpace destination key alreadythere = do
 moveAnnex :: Key -> FilePath -> Annex ()
 moveAnnex key src = do
 	dest <- inRepo $ gitAnnexLocation key
-	let dir = parentDir dest
 	ifM (liftIO $ doesFileExist dest)
 		( liftIO $ removeFile src
 		, do
-			liftIO $ do
-				createDirectoryIfMissing True dir
-				allowWrite dir -- in case the directory already exists
-				moveFile src dest
+			createContentDir dest
+			liftIO $ moveFile src dest
 			freezeContent dest
 			freezeContentDir dest
 		)
@@ -268,8 +266,8 @@ moveBad key = do
 	src <- inRepo $ gitAnnexLocation key
 	bad <- fromRepo gitAnnexBadDir
 	let dest = bad  takeFileName src
+	createAnnexDirectory (parentDir dest)
 	liftIO $ do
-		createDirectoryIfMissing True (parentDir dest)
 		allowWrite (parentDir src)
 		moveFile src dest
 	cleanObjectLoc key
@@ -367,3 +365,13 @@ freezeContentDir file = liftIO . go =<< fromRepo getSharedRepository
 		go AllShared = groupWriteRead dir
 		go _ = preventWrite dir
 
+{- Makes the directory tree to store an annexed file's content,
+ - with appropriate permissions on each level. -}
+createContentDir :: FilePath -> Annex ()
+createContentDir dest = do
+	unlessM (liftIO $ doesDirectoryExist dir) $
+		createAnnexDirectory dir 
+	-- might have already existed with restricted perms
+	liftIO $ allowWrite dir
+	where
+		dir = parentDir dest
diff --git a/Annex/Journal.hs b/Annex/Journal.hs
index 34c4d98c88..ff103180ee 100644
--- a/Annex/Journal.hs
+++ b/Annex/Journal.hs
@@ -16,6 +16,7 @@ import System.IO.Binary
 import Common.Annex
 import Annex.Exception
 import qualified Git
+import Annex.Perms
 
 {- Records content for a file in the branch to the journal.
  -
@@ -23,22 +24,20 @@ import qualified Git
  - avoids git needing to rewrite the index after every change. -}
 setJournalFile :: FilePath -> String -> Annex ()
 setJournalFile file content = do
-	g <- gitRepo
-	liftIO $ doRedo (write g) $ do
-		createDirectoryIfMissing True $ gitAnnexJournalDir g
-		createDirectoryIfMissing True $ gitAnnexTmpDir g
-	where
-		-- journal file is written atomically
-		write g = do
-			let jfile = journalFile g file
-			let tmpfile = gitAnnexTmpDir g  takeFileName jfile
-			writeBinaryFile tmpfile content
-			moveFile tmpfile jfile
+	createAnnexDirectory =<< fromRepo gitAnnexJournalDir
+	createAnnexDirectory =<< fromRepo gitAnnexTmpDir
+	-- journal file is written atomically
+	jfile <- fromRepo $ journalFile file
+	tmp <- fromRepo gitAnnexTmpDir
+	let tmpfile = tmp  takeFileName jfile
+	liftIO $ do
+		writeBinaryFile tmpfile content
+		moveFile tmpfile jfile
 
 {- Gets any journalled content for a file in the branch. -}
 getJournalFile :: FilePath -> Annex (Maybe String)
 getJournalFile file = inRepo $ \g -> catchMaybeIO $
-	readFileStrict $ journalFile g file
+	readFileStrict $ journalFile file g
 
 {- List of files that have updated content in the journal. -}
 getJournalledFiles :: Annex [FilePath]
@@ -62,8 +61,8 @@ journalDirty = not . null <$> getJournalFiles
  - used in the branch is not necessary, and all the files are put directly
  - in the journal directory.
  -}
-journalFile :: Git.Repo -> FilePath -> FilePath
-journalFile repo file = gitAnnexJournalDir repo  concatMap mangle file
+journalFile :: FilePath -> Git.Repo -> FilePath
+journalFile file repo = gitAnnexJournalDir repo  concatMap mangle file
 	where
 		mangle '/' = "_"
 		mangle '_' = "__"
@@ -79,16 +78,12 @@ fileJournal = replace "//" "_" . replace "_" "/"
 lockJournal :: Annex a -> Annex a
 lockJournal a = do
 	file <- fromRepo gitAnnexJournalLock
-	bracketIO (lock file) unlock a
+	createAnnexDirectory $ takeDirectory file
+	mode <- annexFileMode
+	bracketIO (lock file mode) unlock a
 	where
-		lock file = do
-			l <- doRedo (createFile file stdFileMode) $
-				createDirectoryIfMissing True $ takeDirectory file
+		lock file mode = do
+			l <- noUmask mode $ createFile file mode
 			waitToSetLock l (WriteLock, AbsoluteSeek, 0, 0)
 			return l
 		unlock = closeFd
-
-{- Runs an action, catching failure and running something to fix it up, and
- - retrying if necessary. -}
-doRedo :: IO a -> IO b -> IO a
-doRedo a b = catchIO a $ const $ b >> a
diff --git a/Annex/LockPool.hs b/Annex/LockPool.hs
index 3fede5739b..3eb1363eed 100644
--- a/Annex/LockPool.hs
+++ b/Annex/LockPool.hs
@@ -12,6 +12,7 @@ import System.Posix.Types (Fd)
 
 import Common.Annex
 import Annex
+import Annex.Perms
 
 {- Create a specified lock file, and takes a shared lock. -}
 lockFile :: FilePath -> Annex ()
@@ -19,7 +20,9 @@ lockFile file = go =<< fromPool file
 	where
 		go (Just _) = return () -- already locked
 		go Nothing = do
-			fd <- liftIO $ openFd file ReadOnly (Just stdFileMode) defaultFileFlags
+			mode <- annexFileMode
+			fd <- liftIO $ noUmask mode $
+				openFd file ReadOnly (Just mode) defaultFileFlags
 			liftIO $ waitToSetLock fd (ReadLock, AbsoluteSeek, 0, 0)
 			changePool $ M.insert file fd
 
diff --git a/Annex/Perms.hs b/Annex/Perms.hs
new file mode 100644
index 0000000000..2b54077ca6
--- /dev/null
+++ b/Annex/Perms.hs
@@ -0,0 +1,61 @@
+{- git-annex file permissions
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Annex.Perms (
+	setAnnexPerm,
+	annexFileMode,
+	createAnnexDirectory,
+	noUmask,
+) where
+
+import Common.Annex
+import Utility.FileMode
+import Git.SharedRepository
+
+import System.Posix.Types
+
+{- Sets appropriate file mode for a file or directory in the annex,
+ - other than the content files and content directory. Normally,
+ - use the default mode, but with core.sharedRepository set,
+ - allow the group to write, etc. -}
+setAnnexPerm :: FilePath -> Annex ()
+setAnnexPerm file = liftIO . go =<< fromRepo getSharedRepository
+	where
+		go GroupShared = groupWriteRead file
+		go AllShared = modifyFileMode file $ addModes $
+			[ ownerWriteMode, groupWriteMode ] ++ readModes
+		go _ = return ()
+
+{- Gets the appropriate mode to use for creating a file in the annex
+ - (other than content files, which are locked down more). -}
+annexFileMode :: Annex FileMode
+annexFileMode = go <$> fromRepo getSharedRepository
+	where
+		go GroupShared = sharedmode
+		go AllShared = combineModes (sharedmode:readModes)
+		go _ = stdFileMode
+		sharedmode = combineModes
+			[ ownerWriteMode, groupWriteMode
+			, ownerReadMode, groupReadMode
+			]
+
+{- Creates a directory inside the gitAnnexDir, including any parent
+ - directories. Makes directories with appropriate permissions. -}
+createAnnexDirectory :: FilePath -> Annex ()
+createAnnexDirectory dir = traverse dir [] =<< top
+	where
+		top = parentDir <$> fromRepo gitAnnexDir
+		traverse d below stop
+			| d `equalFilePath` stop = done
+			| otherwise = ifM (liftIO $ doesDirectoryExist d)
+				( done
+				, traverse (parentDir d) (d:below) stop
+				)
+			where
+				done = forM_ below $ \p -> do
+					liftIO $ createDirectory p
+					setAnnexPerm p
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index e6cd6a9263..c9e6e29515 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -17,6 +17,7 @@ import Annex.LockPool
 import qualified Git
 import Config
 import qualified Build.SysConfig as SysConfig
+import Annex.Perms
 
 {- Generates parameters to ssh to a given host (or user@host) on a given
  - port, with connection caching. -}
@@ -74,7 +75,9 @@ sshCleanup = do
 			-- be stopped.
 			let lockfile = socket2lock socketfile
 			unlockFile lockfile
-			fd <- liftIO $ openFd lockfile ReadWrite (Just stdFileMode) defaultFileFlags
+			mode <- annexFileMode
+			fd <- liftIO $ noUmask mode $
+				openFd lockfile ReadWrite (Just mode) defaultFileFlags
 			v <- liftIO $ tryIO $
 				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs
index 2864a8ed58..de731bd6e6 100644
--- a/Remote/Helper/Hooks.hs
+++ b/Remote/Helper/Hooks.hs
@@ -14,6 +14,7 @@ import Types.Remote
 import qualified Annex
 import Annex.LockPool
 import Config
+import Annex.Perms
 
 {- Modifies a remote's access functions to first run the
  - annex-start-command hook, and trigger annex-stop-command on shutdown.
@@ -75,7 +76,9 @@ runHooks r starthook stophook a = do
 			-- succeeds, we're the only process using this remote,
 			-- so can stop it.
 			unlockFile lck
-			fd <- liftIO $ openFd lck ReadWrite (Just stdFileMode) defaultFileFlags
+			mode <- annexFileMode
+			fd <- liftIO $ noUmask mode $
+				openFd lck ReadWrite (Just mode) defaultFileFlags
 			v <- liftIO $ tryIO $
 				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index f3db709231..c0f2ad589c 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -75,6 +75,17 @@ isExecutable mode = combineModes ebits `intersectFileModes` mode /= 0
 	where
 		ebits = [ownerExecuteMode, groupExecuteMode, otherExecuteMode]
 
+{- Runs an action without that pesky umask influencing it, unless the
+ - passed FileMode is the standard one. -}
+noUmask :: FileMode -> IO a -> IO a
+noUmask mode a
+	| mode == stdFileMode = a
+	| otherwise = bracket setup cleanup go
+	where
+		setup = setFileCreationMask nullFileMode
+		cleanup = setFileCreationMask
+		go _ = a
+
 combineModes :: [FileMode] -> FileMode
 combineModes [] = undefined
 combineModes [m] = m

From cab63b89f2470d0874e72a4a9e088206fb554c94 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 19:42:49 -0400
Subject: [PATCH 3401/8313] cache parsed core.sharedrepository

---
 Annex.hs       |  3 +++
 Annex/Perms.hs | 13 +++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index ef95ff174c..2b58c2bd49 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -37,6 +37,7 @@ import qualified Git
 import qualified Git.Config
 import Git.CatFile
 import Git.CheckAttr
+import Git.SharedRepository
 import qualified Git.Queue
 import Types.Backend
 import qualified Types.Remote
@@ -88,6 +89,7 @@ data AnnexState = AnnexState
 	, forcebackend :: Maybe String
 	, forcenumcopies :: Maybe Int
 	, limit :: Matcher (FilePath -> Annex Bool)
+	, shared :: Maybe SharedRepository
 	, forcetrust :: TrustMap
 	, trustmap :: Maybe TrustMap
 	, ciphers :: M.Map EncryptedCipher Cipher
@@ -113,6 +115,7 @@ newState gitrepo = AnnexState
 	, forcebackend = Nothing
 	, forcenumcopies = Nothing
 	, limit = Left []
+	, shared = Nothing
 	, forcetrust = M.empty
 	, trustmap = Nothing
 	, ciphers = M.empty
diff --git a/Annex/Perms.hs b/Annex/Perms.hs
index 2b54077ca6..12dfdd6679 100644
--- a/Annex/Perms.hs
+++ b/Annex/Perms.hs
@@ -15,15 +15,24 @@ module Annex.Perms (
 import Common.Annex
 import Utility.FileMode
 import Git.SharedRepository
+import qualified Annex
 
 import System.Posix.Types
 
+withShared :: (SharedRepository -> Annex a) -> Annex a
+withShared a = maybe startup a =<< Annex.getState Annex.shared
+	where
+		startup = do
+			shared <- fromRepo getSharedRepository
+			Annex.changeState $ \s -> s { Annex.shared = Just shared }
+			a shared
+
 {- Sets appropriate file mode for a file or directory in the annex,
  - other than the content files and content directory. Normally,
  - use the default mode, but with core.sharedRepository set,
  - allow the group to write, etc. -}
 setAnnexPerm :: FilePath -> Annex ()
-setAnnexPerm file = liftIO . go =<< fromRepo getSharedRepository
+setAnnexPerm file = withShared $ liftIO . go
 	where
 		go GroupShared = groupWriteRead file
 		go AllShared = modifyFileMode file $ addModes $
@@ -33,7 +42,7 @@ setAnnexPerm file = liftIO . go =<< fromRepo getSharedRepository
 {- Gets the appropriate mode to use for creating a file in the annex
  - (other than content files, which are locked down more). -}
 annexFileMode :: Annex FileMode
-annexFileMode = go <$> fromRepo getSharedRepository
+annexFileMode = withShared $ return . go
 	where
 		go GroupShared = sharedmode
 		go AllShared = combineModes (sharedmode:readModes)

From be36aaca5bccecdff7e48b020ee62b0db3bc38f1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sat, 21 Apr 2012 23:46:42 +0000
Subject: [PATCH 3402/8313] Added a comment

---
 .../comment_2_86663eeb75b0477f53c45f26c8e4b051._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment

diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment
new file mode 100644
index 0000000000..1c9bfbfe40
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_2_86663eeb75b0477f53c45f26c8e4b051._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-04-21T23:46:42Z"
+ content="""
+All right, I've made all the changes so it supports `core.sharedRepository`.
+"""]]

From bee420bd2d0cbe16489b061b208083e2b8ba9d0e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 23:04:59 -0400
Subject: [PATCH 3403/8313] in which I discover void

void :: Functor f => f a -> f () -- ah, of course that's useful :)
---
 Annex/Branch.hs        |  9 +++------
 Annex/Ssh.hs           | 11 ++++-------
 Remote/Git.hs          |  6 ++----
 Remote/Helper/Hooks.hs |  5 ++---
 Utility/FileMode.hs    |  4 +---
 Utility/Inotify.hs     | 10 ++++------
 6 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index e5976c2c01..ce1dd58cec 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -65,9 +65,7 @@ siblingBranches = inRepo $ Git.Ref.matchingUniq name
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = do
-	_ <- getBranch
-	return ()
+create = void $ getBranch
 
 {- Returns the ref of the branch, creating it first if necessary. -}
 getBranch :: Annex Git.Ref
@@ -325,10 +323,9 @@ setUnCommitted = do
 	liftIO $ writeFile file "1"
 
 setCommitted :: Annex ()
-setCommitted = do
+setCommitted = void $ do
 	file <- fromRepo gitAnnexIndexDirty
-	_ <- liftIO $ tryIO $ removeFile file
-	return ()
+	liftIO $ tryIO $ removeFile file
 
 {- Stages the journal into the index. -}
 stageJournal :: Annex ()
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index c9e6e29515..02a1ee705a 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -87,20 +87,17 @@ sshCleanup = do
 		stopssh socketfile = do
 			let (host, port) = socket2hostport socketfile
 			(_, params) <- sshInfo (host, port)
-			_ <- liftIO $ do
+			void $ liftIO $ do
 				-- "ssh -O stop" is noisy on stderr even with -q
 				let cmd = unwords $ toCommand $
 					[ Params "-O stop"
 					] ++ params ++ [Param host]
-				_ <- boolSystem "sh"
+				boolSystem "sh"
 					[ Param "-c"
 					, Param $ "ssh " ++ cmd ++ " >/dev/null 2>/dev/null"
 					]
-				--try $ removeFile socketfile
-				return ()
-			-- Cannot remove the lock file; other processes may
-			-- be waiting on our exclusive lock to use it.
-			return ()
+				-- Cannot remove the lock file; other processes may
+				-- be waiting on our exclusive lock to use it.
 
 hostport2socket :: String -> Maybe Integer -> FilePath
 hostport2socket host Nothing = host
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 541b050994..d71872b277 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -313,7 +313,7 @@ commitOnCleanup r a = go `after` a
 		cleanup
 			| not $ Git.repoIsUrl r = liftIO $ onLocal r $
 				Annex.Branch.commit "update"
-			| otherwise = do
+			| otherwise = void $ do
 				Just (shellcmd, shellparams) <-
 					git_annex_shell r "commit" []
 				-- Throw away stderr, since the remote may not
@@ -322,6 +322,4 @@ commitOnCleanup r a = go `after` a
 				let cmd = shellcmd ++ " "
 					++ unwords (map shellEscape $ toCommand shellparams)
 					++ ">/dev/null 2>/dev/null"
-				_ <- liftIO $
-					boolSystem "sh" [Param "-c", Param cmd]
-				return ()
+				liftIO $ boolSystem "sh" [Param "-c", Param cmd]
diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs
index de731bd6e6..40484b2a7c 100644
--- a/Remote/Helper/Hooks.hs
+++ b/Remote/Helper/Hooks.hs
@@ -47,9 +47,8 @@ runHooks r starthook stophook a = do
 	where
 		remoteid = show (uuid r)
 		run Nothing = return ()
-		run (Just command) = liftIO $ do
-			_ <- boolSystem "sh" [Param "-c", Param command]
-			return ()
+		run (Just command) = void $ liftIO $
+			boolSystem "sh" [Param "-c", Param command]
 		firstrun lck = do
 			-- Take a shared lock; This indicates that git-annex
 			-- is using the remote, and prevents other instances
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index c0f2ad589c..353de7b92a 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -15,9 +15,7 @@ import Foreign (complement)
 
 {- Applies a conversion function to a file's mode. -}
 modifyFileMode :: FilePath -> (FileMode -> FileMode) -> IO ()
-modifyFileMode f convert = do
-	_ <- modifyFileMode' f convert
-	return ()
+modifyFileMode f convert = void $ modifyFileMode' f convert
 modifyFileMode' :: FilePath -> (FileMode -> FileMode) -> IO FileMode
 modifyFileMode' f convert = do
 	s <- getFileStatus f
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 049737c08a..0a261ecfe0 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -53,10 +53,9 @@ watchDir i test add del dir = watchDir' False i test add del dir
 watchDir' :: Bool -> INotify -> (FilePath -> Bool) -> Maybe (FilePath -> IO ()) -> Maybe (FilePath -> IO ()) -> FilePath -> IO ()
 watchDir' scan i test add del dir = do
 	if test dir
-		then do
+		then void $ do
 			_ <- addWatch i watchevents dir go
-			_ <- mapM walk =<< dirContents dir
-			return ()
+			mapM walk =<< dirContents dir
 		else return ()
 	where
 		watchevents
@@ -92,6 +91,5 @@ waitForTermination = do
 		check keyboardSignal mv
 	takeMVar mv
 	where
-		check sig mv = do
-			_ <- installHandler sig (CatchOnce $ putMVar mv ()) Nothing
-			return ()
+		check sig mv = void $
+			installHandler sig (CatchOnce $ putMVar mv ()) Nothing

From ed79596b758935a3f22bf6803bc082a6bbe10f58 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 21 Apr 2012 23:32:33 -0400
Subject: [PATCH 3404/8313] noop

---
 Annex/Content.hs       |  6 +++---
 Annex/LockPool.hs      |  7 +++----
 Annex/Perms.hs         |  2 +-
 Annex/Ssh.hs           |  2 +-
 Annex/Version.hs       |  2 +-
 CmdLine.hs             |  2 +-
 Command/Fsck.hs        |  2 +-
 Command/Status.hs      |  5 ++---
 Command/Unused.hs      |  2 +-
 Command/Whereis.hs     | 12 ++++++------
 Git/Command.hs         |  4 ++--
 Git/Construct.hs       |  2 +-
 Git/UnionMerge.hs      |  2 +-
 GitAnnexShell.hs       |  4 ++--
 Logs/Location.hs       |  2 +-
 Logs/UUID.hs           |  2 +-
 Messages.hs            |  6 +++---
 Remote.hs              |  2 +-
 Remote/Directory.hs    |  3 ++-
 Remote/Helper/Hooks.hs |  4 ++--
 Remote/S3.hs           |  6 +++---
 Upgrade/V1.hs          |  2 +-
 Utility/Directory.hs   |  3 ++-
 Utility/Inotify.hs     | 10 +++++-----
 Utility/Monad.hs       |  4 ++++
 Utility/Touch.hsc      |  7 +++----
 Utility/Url.hs         |  3 ++-
 27 files changed, 56 insertions(+), 52 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 7022364d09..c5771af28e 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -98,7 +98,7 @@ lockContent key a = do
 			case v of
 				Left _ -> error "content is locked"
 				Right _ -> return $ Just fd
-		unlock Nothing = return ()
+		unlock Nothing = noop
 		unlock (Just l) = closeFd l
 
 {- Calculates the relative path to use to link a file to a key. -}
@@ -237,10 +237,10 @@ cleanObjectLoc key = do
 	file <- inRepo $ gitAnnexLocation key
 	liftIO $ removeparents file (3 :: Int)
 	where
-		removeparents _ 0 = return ()
+		removeparents _ 0 = noop
 		removeparents file n = do
 			let dir = parentDir file
-			maybe (return ()) (const $ removeparents dir (n-1))
+			maybe noop (const $ removeparents dir (n-1))
 				=<< catchMaybeIO (removeDirectory dir)
 
 {- Removes a key's file from .git/annex/objects/ -}
diff --git a/Annex/LockPool.hs b/Annex/LockPool.hs
index 3eb1363eed..b99a8ec4df 100644
--- a/Annex/LockPool.hs
+++ b/Annex/LockPool.hs
@@ -18,7 +18,7 @@ import Annex.Perms
 lockFile :: FilePath -> Annex ()
 lockFile file = go =<< fromPool file
 	where
-		go (Just _) = return () -- already locked
+		go (Just _) = noop -- already locked
 		go Nothing = do
 			mode <- annexFileMode
 			fd <- liftIO $ noUmask mode $
@@ -27,10 +27,9 @@ lockFile file = go =<< fromPool file
 			changePool $ M.insert file fd
 
 unlockFile :: FilePath -> Annex ()
-unlockFile file = go =<< fromPool file
+unlockFile file = maybe noop go =<< fromPool file
 	where
-		go Nothing = return ()
-		go (Just fd) = do
+		go fd = do
 			liftIO $ closeFd fd
 			changePool $ M.delete file
 
diff --git a/Annex/Perms.hs b/Annex/Perms.hs
index 12dfdd6679..c54908b439 100644
--- a/Annex/Perms.hs
+++ b/Annex/Perms.hs
@@ -37,7 +37,7 @@ setAnnexPerm file = withShared $ liftIO . go
 		go GroupShared = groupWriteRead file
 		go AllShared = modifyFileMode file $ addModes $
 			[ ownerWriteMode, groupWriteMode ] ++ readModes
-		go _ = return ()
+		go _ = noop
 
 {- Gets the appropriate mode to use for creating a file in the annex
  - (other than content files, which are locked down more). -}
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 02a1ee705a..6a230312ab 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -81,7 +81,7 @@ sshCleanup = do
 			v <- liftIO $ tryIO $
 				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
-				Left _ -> return ()
+				Left _ -> noop
 				Right _ -> stopssh socketfile
 			liftIO $ closeFd fd
 		stopssh socketfile = do
diff --git a/Annex/Version.hs b/Annex/Version.hs
index cf5d224842..a1d0402445 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -35,7 +35,7 @@ setVersion = setConfig versionField defaultVersion
 
 checkVersion :: Version -> Annex ()
 checkVersion v
-	| v `elem` supportedVersions = return ()
+	| v `elem` supportedVersions = noop
 	| v `elem` upgradableVersions = err "Upgrade this repository: git-annex upgrade"
 	| otherwise = err "Upgrade git-annex."
 	where
diff --git a/CmdLine.hs b/CmdLine.hs
index ebaef5369d..910f228b60 100644
--- a/CmdLine.hs
+++ b/CmdLine.hs
@@ -88,7 +88,7 @@ tryRun = tryRun' 0
 tryRun' :: Integer -> Annex.AnnexState -> Command -> [CommandCleanup] -> IO ()
 tryRun' errnum _ cmd []
 	| errnum > 0 = error $ cmdname cmd ++ ": " ++ show errnum ++ " failed"
-	| otherwise = return ()
+	| otherwise = noop
 tryRun' errnum state cmd (a:as) = do
 	r <- run
 	handle $! r
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index c60101fc79..38b1bbbacd 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -85,7 +85,7 @@ performRemote key file backend numcopies remote =
 			t <- fromRepo gitAnnexTmpDir
 			let tmp = t  "fsck" ++ show pid ++ "." ++ keyFile key
 			liftIO $ createDirectoryIfMissing True t
-			let cleanup = liftIO $ catchIO (removeFile tmp) (const $ return ())
+			let cleanup = liftIO $ catchIO (removeFile tmp) (const noop)
 			cleanup
 			cleanup `after` a tmp
 		getfile tmp =
diff --git a/Command/Status.hs b/Command/Status.hs
index 1ee36d8b47..0c6eda0b2a 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -108,12 +108,11 @@ nojson :: StatState String -> String -> StatState String
 nojson a _ = a
 
 showStat :: Stat -> StatState ()
-showStat s = calc =<< s
+showStat s = maybe noop calc =<< s
 	where
-		calc (Just (desc, a)) = do
+		calc (desc, a) = do
 			(lift . showHeader) desc
 			lift . showRaw =<< a
-		calc Nothing = return ()
 
 supported_backends :: Stat
 supported_backends = stat "supported backends" $ json unwords $
diff --git a/Command/Unused.hs b/Command/Unused.hs
index bc721635b7..5bdadcf44a 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -268,7 +268,7 @@ withKeysReferencedInGitRef a ref = do
 	showAction $ "checking " ++ Git.Ref.describe ref
 	go =<< inRepo (LsTree.lsTree ref)
 	where
-		go [] = return ()
+		go [] = noop
 		go (l:ls)
 			| isSymLink (LsTree.mode l) = do
 				content <- L.decodeUtf8 <$> catFile ref (LsTree.file l)
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index d4d268d937..eb6ea7c56d 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -46,9 +46,9 @@ perform remotemap key = do
 		untrustedheader = "The following untrusted locations may also have copies:\n"
 
 performRemote :: Key -> Remote -> Annex () 
-performRemote key remote = case whereisKey remote of
-	Nothing -> return ()
-	Just a -> do
-		ls <- a key
-		unless (null ls) $ showLongNote $
-			unlines $ map (\l -> name remote ++ ": " ++ l) ls
+performRemote key remote = maybe noop go $ whereisKey remote
+	where
+		go a = do
+			ls <- a key
+			unless (null ls) $ showLongNote $ unlines $
+				map (\l -> name remote ++ ": " ++ l) ls
diff --git a/Git/Command.hs b/Git/Command.hs
index 50d4455fe7..bb82d13395 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -79,5 +79,5 @@ pipeNullSplit params repo =
 reap :: IO ()
 reap = do
 	-- throws an exception when there are no child processes
-	r <- catchDefaultIO (getAnyProcessStatus False True) Nothing
-	maybe (return ()) (const reap) r
+	catchDefaultIO (getAnyProcessStatus False True) Nothing
+		>>= maybe noop (const reap)
diff --git a/Git/Construct.hs b/Git/Construct.hs
index 49905f818b..3f3ea97476 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -48,7 +48,7 @@ import qualified Git.Url as Url
 fromCurrent :: IO Repo
 fromCurrent = do
 	r <- maybe fromCwd fromPath =<< getEnv "GIT_DIR"
-	maybe (return ()) changeWorkingDirectory =<< getEnv "GIT_WORK_TREE"
+	maybe noop changeWorkingDirectory =<< getEnv "GIT_WORK_TREE"
 	unsetEnv "GIT_DIR"
 	unsetEnv "GIT_WORK_TREE"
 	return r
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 90bbf5c4cc..d68bb61ab1 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -97,7 +97,7 @@ calc_merge :: CatFileHandle -> [String] -> Repo -> Streamer
 calc_merge ch differ repo streamer = gendiff >>= go
 	where
 		gendiff = pipeNullSplit (map Param differ) repo
-		go [] = return ()
+		go [] = noop
 		go (info:file:rest) = mergeFile info file ch repo >>=
 			maybe (go rest) (\l -> streamer l >> go rest)
 		go (_:[]) = error "calc_merge parse error"
diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs
index 0cf81f0e21..6633037138 100644
--- a/GitAnnexShell.hs
+++ b/GitAnnexShell.hs
@@ -52,7 +52,7 @@ options = Option.common ++
 	where
 		checkuuid expected = getUUID >>= check
 			where
-				check u | u == toUUID expected = return ()
+				check u | u == toUUID expected = noop
 				check NoUUID = unexpected "uninitialized repository"
 				check u = unexpected $ "UUID " ++ fromUUID u
 				unexpected s = error $
@@ -107,7 +107,7 @@ checkNotLimited = checkEnv "GIT_ANNEX_SHELL_LIMITED"
 
 checkNotReadOnly :: String -> IO ()
 checkNotReadOnly cmd
-	| cmd `elem` map cmdname cmds_readonly = return ()
+	| cmd `elem` map cmdname cmds_readonly = noop
 	| otherwise = checkEnv "GIT_ANNEX_SHELL_READONLY"
 
 checkEnv :: String -> IO ()
diff --git a/Logs/Location.hs b/Logs/Location.hs
index b6d59b928c..e27ece5d46 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -30,7 +30,7 @@ import Logs.Presence
 {- Log a change in the presence of a key's value in a repository. -}
 logChange :: Key -> UUID -> LogStatus -> Annex ()
 logChange key (UUID u) s = addLog (logFile key) =<< logNow s u
-logChange _ NoUUID _ = return ()
+logChange _ NoUUID _ = noop
 
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key.
diff --git a/Logs/UUID.hs b/Logs/UUID.hs
index 18cbee61e4..d825e11273 100644
--- a/Logs/UUID.hs
+++ b/Logs/UUID.hs
@@ -73,7 +73,7 @@ recordUUID u = go . M.lookup u =<< uuidMap
 	where
 		go (Just "") = set
 		go Nothing = set
-		go _ = return ()
+		go _ = noop
 		set = describeUUID u ""
 
 {- Read the uuidLog into a simple Map.
diff --git a/Messages.hs b/Messages.hs
index 73a7d976fd..af7eb88b43 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -72,8 +72,8 @@ metered key a = Annex.getState Annex.output >>= go (keySize key)
 				incrP progress n
 				displayMeter stdout meter
 			liftIO $ clearMeter stdout meter
-			return r	
-                go _ _ = a (const $ return ())
+			return r
+                go _ _ = a (const noop)
 
 showSideAction :: String -> Annex ()
 showSideAction s = handle q $
@@ -160,7 +160,7 @@ handle json normal = Annex.getState Annex.output >>= go
 		go Annex.JSONOutput = liftIO $ flushed json
 
 q :: Monad m => m ()
-q = return ()
+q = noop
 
 flushed :: IO () -> IO ()
 flushed a = a >> hFlush stdout
diff --git a/Remote.hs b/Remote.hs
index aac45fae9d..e9e66990c5 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -194,7 +194,7 @@ showLocations key exclude = do
 		message rs us = message rs [] ++ message [] us
 
 showTriedRemotes :: [Remote] -> Annex ()
-showTriedRemotes [] = return ()	
+showTriedRemotes [] = noop
 showTriedRemotes remotes =
 	showLongNote $ "Unable to access these remotes: " ++
 		join ", " (map name remotes)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index fd5a6f0b19..7521e70135 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -195,7 +195,8 @@ meteredWriteFile' meterupdate dest startstate feeder =
 	where
 		feed state [] h = do
 			(state', cs) <- feeder state
-			if null cs then return () else feed state' cs h
+			unless (null cs) $
+				feed state' cs h
 		feed state (c:cs) h = do
 			S.hPut h c
 			meterupdate $ toInteger $ S.length c
diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs
index 40484b2a7c..d85959062e 100644
--- a/Remote/Helper/Hooks.hs
+++ b/Remote/Helper/Hooks.hs
@@ -46,7 +46,7 @@ runHooks r starthook stophook a = do
 	a
 	where
 		remoteid = show (uuid r)
-		run Nothing = return ()
+		run Nothing = noop
 		run (Just command) = void $ liftIO $
 			boolSystem "sh" [Param "-c", Param command]
 		firstrun lck = do
@@ -81,7 +81,7 @@ runHooks r starthook stophook a = do
 			v <- liftIO $ tryIO $
 				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 			case v of
-				Left _ -> return ()
+				Left _ -> noop
 				Right _ -> run stophook
 			liftIO $ closeFd fd
 
diff --git a/Remote/S3.hs b/Remote/S3.hs
index a688ffcf34..18d4915dcb 100644
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -93,7 +93,7 @@ s3Setup u c = handlehost $ M.lookup "host" c
 
 		archiveorg = do
 			showNote "Internet Archive mode"
-			maybe (error "specify bucket=") (const $ return ()) $
+			maybe (error "specify bucket=") (const noop) $
 				M.lookup "bucket" archiveconfig
 			use archiveconfig
 			where
@@ -237,13 +237,13 @@ genBucket c = do
 	showAction "checking bucket"
 	loc <- liftIO $ getBucketLocation conn bucket 
 	case loc of
-		Right _ -> return ()
+		Right _ -> noop
 		Left err@(NetworkError _) -> s3Error err
 		Left (AWSError _ _) -> do
 			showAction $ "creating bucket in " ++ datacenter
 			res <- liftIO $ createBucketIn conn bucket datacenter
 			case res of
-				Right _ -> return ()
+				Right _ -> noop
 				Left err -> s3Error err
 	where
 		bucket = fromJust $ M.lookup "bucket" c
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 62e3b3b313..a8005b2644 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -89,7 +89,7 @@ updateSymlinks = do
 		fixlink f = do
 			r <- lookupFile1 f
 			case r of
-				Nothing -> return ()
+				Nothing -> noop
 				Just (k, _) -> do
 					link <- calcGitLink f k
 					liftIO $ removeFile f
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 40e65d6349..e6622d31ee 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -19,6 +19,7 @@ import Control.Applicative
 import Utility.SafeCommand
 import Utility.TempFile
 import Utility.Exception
+import Utility.Monad
 
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
@@ -34,7 +35,7 @@ dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
 moveFile :: FilePath -> FilePath -> IO ()
 moveFile src dest = tryIO (rename src dest) >>= onrename
 	where
-		onrename (Right _) = return ()
+		onrename (Right _) = noop
 		onrename (Left e)
 			| isPermissionError e = rethrow
 			| isDoesNotExistError e = rethrow
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 0a261ecfe0..d41e997d61 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -56,7 +56,7 @@ watchDir' scan i test add del dir = do
 		then void $ do
 			_ <- addWatch i watchevents dir go
 			mapM walk =<< dirContents dir
-		else return ()
+		else noop
 	where
 		watchevents
 			| isJust add && isJust del =
@@ -68,19 +68,19 @@ watchDir' scan i test add del dir = do
 		recurse = watchDir' scan i test add del
 		walk f = ifM (catchBoolIO $ Files.isDirectory <$> getFileStatus f)
 			( recurse f
-			, if scan && isJust add then fromJust add f else return ()
+			, when (scan && isJust add) $ fromJust add f
 			)
 
-		go (Created { isDirectory = False }) = return ()
+		go (Created { isDirectory = False }) = noop
 		go (Created { filePath = subdir }) = Just recurse <@> subdir
 		go (Closed { maybeFilePath = Just f }) = add <@> f
 		go (MovedIn { isDirectory = False, filePath = f }) = add <@> f
 		go (MovedOut { isDirectory = False, filePath = f }) = del <@> f
 		go (Deleted { isDirectory = False, filePath = f }) = del <@> f
-		go _ = return ()
+		go _ = noop
 		
 		Just a <@> f = a $ dir  f
-		Nothing <@> _ = return ()
+		Nothing <@> _ = noop
 
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()
diff --git a/Utility/Monad.hs b/Utility/Monad.hs
index 9c85d31ca8..2c9b9e9e07 100644
--- a/Utility/Monad.hs
+++ b/Utility/Monad.hs
@@ -49,3 +49,7 @@ observe observer a = do
 {- b `after` a runs first a, then b, and returns the value of a -}
 after :: Monad m => m b -> m a -> m a
 after = observe . const
+
+{- do nothing -}
+noop :: Monad m => m ()
+noop = return ()
diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index b53eab634e..e2dba79ab2 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -106,9 +106,8 @@ touchBoth file atime mtime follow =
 	withFilePath file $ \f -> do
 		pokeArray ptr [atime, mtime]
 		r <- syscall f ptr
-		if (r /= 0)
-			then throwErrno "touchBoth"
-			else return ()
+		when (r /= 0) $
+			throwErrno "touchBoth"
 	where
 		syscall = if follow
 			then c_lutimes
@@ -116,6 +115,6 @@ touchBoth file atime mtime follow =
 
 #else
 #warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
-touchBoth _ _ _ _ = return ()
+touchBoth _ _ _ _ = noop
 #endif
 #endif
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 86d66d83b5..20c5db574d 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -17,6 +17,7 @@ import Common
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
+import Utility.Monad
 
 type URLString = String
 
@@ -95,7 +96,7 @@ request url requesttype = go 5 url
 			case rspCode rsp of
 				(3,0,x) | x /= 5 -> redir (n - 1) u rsp
 				_ -> return rsp
-		ignore = const $ return ()
+		ignore = const noop
 		redir n u rsp = case retrieveHeaders HdrLocation rsp of
 			[] -> return rsp
 			(Header _ newu:_) ->

From 5fbe83f595bf5957376544ee83b3cc46cc2323ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 00:07:21 -0400
Subject: [PATCH 3405/8313] add

---
 doc/todo/http_headers.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/todo/http_headers.mdwn

diff --git a/doc/todo/http_headers.mdwn b/doc/todo/http_headers.mdwn
new file mode 100644
index 0000000000..99f083b7ae
--- /dev/null
+++ b/doc/todo/http_headers.mdwn
@@ -0,0 +1,6 @@
+The IA would find it useful to be able to control the http headers
+git-annex get, addurl, etc uses. This will allow setting cookies, for
+example.
+
+* annex-web-headers=blah 
+* Perhaps also annex-web-headers-command=blah

From 84ac8c58db30637db4fb88566530b6293f12dab0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 01:13:09 -0400
Subject: [PATCH 3406/8313] Add annex.httpheaders and annex.httpheader-command
 config settings

Allow custom headers to be sent with all HTTP requests.

(Requested by the Internet Archive)
---
 Annex/Content.hs   |  3 ++-
 Command/AddUrl.hs  |  9 ++++++---
 Config.hs          |  9 +++++++++
 Git/Config.hs      |  4 ++++
 Remote/Git.hs      | 14 ++++++++------
 Remote/Web.hs      |  3 ++-
 Utility/Url.hs     | 38 ++++++++++++++++++++++----------------
 debian/changelog   |  3 +++
 doc/git-annex.mdwn | 10 ++++++++++
 9 files changed, 66 insertions(+), 27 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index c5771af28e..01ee7d83d9 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -310,7 +310,8 @@ saveState oneshot = do
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
 downloadUrl urls file = do
 	o <- map Param . words <$> getConfig "annex.web-options" ""
-	liftIO $ anyM (\u -> Url.download u o file) urls
+	headers <- getHttpHeaders
+	liftIO $ anyM (\u -> Url.download u headers o file) urls
 
 {- Copies a key's content, when present, to a temp file.
  - This is used to speed up some rsyncs. -}
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index c87399f5dc..089606e85d 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -20,6 +20,7 @@ import Annex.Content
 import Logs.Web
 import qualified Option
 import Types.Key
+import Config
 
 def :: [Command]
 def = [withOptions [fileOption, pathdepthOption] $
@@ -53,8 +54,9 @@ perform url file = ifAnnexed file addurl geturl
 			liftIO $ createDirectoryIfMissing True (parentDir file)
 			ifM (Annex.getState Annex.fast)
 				( nodownload url file , download url file )
-		addurl (key, _backend) =
-			ifM (liftIO $ Url.check url $ keySize key)
+		addurl (key, _backend) = do
+			headers <- getHttpHeaders
+			ifM (liftIO $ Url.check url headers $ keySize key)
 				( do
 					setUrlPresent key url
 					next $ return True
@@ -81,7 +83,8 @@ download url file = do
 
 nodownload :: String -> FilePath -> CommandPerform
 nodownload url file = do
-	(exists, size) <- liftIO $ Url.exists url
+	headers <- getHttpHeaders
+	(exists, size) <- liftIO $ Url.exists url headers
 	if exists
 		then do
 			let key = Backend.URL.fromUrl url size
diff --git a/Config.hs b/Config.hs
index 087cb4043b..065ee48f39 100644
--- a/Config.hs
+++ b/Config.hs
@@ -96,3 +96,12 @@ getDiskReserve = fromMaybe megabyte . readSize dataUnits
 	<$> getConfig "annex.diskreserve" ""
 	where
 		megabyte = 1000000
+
+{- Gets annex.httpheaders or annex.httpheaders-command setting,
+ - splitting it into lines. -}
+getHttpHeaders :: Annex [String]
+getHttpHeaders = do
+	cmd <- getConfig "annex.httpheaders-command" ""
+	if (null cmd)
+		then fromRepo $ Git.Config.getList "annex.httpheaders"
+		else lines . snd <$> liftIO (pipeFrom "sh" ["-c", cmd])
diff --git a/Git/Config.hs b/Git/Config.hs
index 8190a62ad3..38b9ade455 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -20,6 +20,10 @@ import qualified Git.Construct
 get :: String -> String -> Repo -> String
 get key defaultValue repo = M.findWithDefault defaultValue key (config repo)
 
+{- Returns a list with each line of a multiline config setting. -}
+getList :: String -> Repo -> [String]
+getList key repo = M.findWithDefault [] key (fullconfig repo)
+
 {- Returns a single git config setting, if set. -}
 getMaybe :: String -> Repo -> Maybe String
 getMaybe key repo = M.lookup key (config repo)
diff --git a/Remote/Git.hs b/Remote/Git.hs
index d71872b277..35928b96cb 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -94,7 +94,9 @@ tryGitConfigRead :: Git.Repo -> Annex Git.Repo
 tryGitConfigRead r 
 	| not $ M.null $ Git.config r = return r -- already read
 	| Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" []
-	| Git.repoIsHttp r = store $ safely geturlconfig
+	| Git.repoIsHttp r = do
+		headers <- getHttpHeaders
+		store $ safely $ geturlconfig headers
 	| Git.repoIsUrl r = return r
 	| otherwise = store $ safely $ onLocal r $ do 
 		ensureInitialized
@@ -109,8 +111,8 @@ tryGitConfigRead r
 			pOpen ReadFromPipe cmd (toCommand params) $
 				Git.Config.hRead r
 
-		geturlconfig = do
-			s <- Url.get (Git.repoLocation r ++ "/config")
+		geturlconfig headers = do
+			s <- Url.get (Git.repoLocation r ++ "/config") headers
 			withTempFile "git-annex.tmp" $ \tmpfile h -> do
 				hPutStr h s
 				hClose h
@@ -136,16 +138,16 @@ tryGitConfigRead r
  -}
 inAnnex :: Git.Repo -> Key -> Annex (Either String Bool)
 inAnnex r key
-	| Git.repoIsHttp r = checkhttp
+	| Git.repoIsHttp r = checkhttp =<< getHttpHeaders
 	| Git.repoIsUrl r = checkremote
 	| otherwise = checklocal
 	where
-		checkhttp = liftIO $ go undefined $ keyUrls r key
+		checkhttp headers = liftIO $ go undefined $ keyUrls r key
 			where
 				go e [] = return $ Left e
 				go _ (u:us) = do
 					res <- catchMsgIO $
-						Url.check u (keySize key)
+						Url.check u headers (keySize key)
 					case res of
 						Left e -> go e us
 						v -> return v
diff --git a/Remote/Web.hs b/Remote/Web.hs
index 81e6ca321c..5fc592326c 100644
--- a/Remote/Web.hs
+++ b/Remote/Web.hs
@@ -83,4 +83,5 @@ checkKey key = do
 checkKey' :: Key -> [URLString] -> Annex Bool
 checkKey' key us = untilTrue us $ \u -> do
 	showAction $ "checking " ++ u
-	liftIO $ Url.check u (keySize key)
+	headers <- getHttpHeaders
+	liftIO $ Url.check u headers (keySize key)
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 20c5db574d..465ef855c8 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -17,14 +17,16 @@ import Common
 import qualified Network.Browser as Browser
 import Network.HTTP
 import Network.URI
-import Utility.Monad
+import Data.Either
 
 type URLString = String
 
+type Headers = [String]
+
 {- Checks that an url exists and could be successfully downloaded,
  - also checking that its size, if available, matches a specified size. -}
-check :: URLString -> Maybe Integer -> IO Bool
-check url expected_size = handle <$> exists url
+check :: URLString -> Headers -> Maybe Integer -> IO Bool
+check url headers expected_size = handle <$> exists url headers
 	where
 		handle (False, _) = False
 		handle (True, Nothing) = True
@@ -32,12 +34,12 @@ check url expected_size = handle <$> exists url
 
 {- Checks that an url exists and could be successfully downloaded,
  - also returning its size if available. -}
-exists :: URLString -> IO (Bool, Maybe Integer)
-exists url =
+exists :: URLString -> Headers -> IO (Bool, Maybe Integer)
+exists url headers =
 	case parseURI url of
 		Nothing -> return (False, Nothing)
 		Just u -> do
-			r <- request u HEAD
+			r <- request u headers HEAD
 			case rspCode r of
 				(2,_,_) -> return (True, size r)
 				_ -> return (False, Nothing)
@@ -51,26 +53,27 @@ exists url =
  - would not be appropriate to test at configure time and build support
  - for only one in.
  -}
-download :: URLString -> [CommandParam] -> FilePath -> IO Bool
-download url options file = ifM (inPath "wget") (wget , curl)
+download :: URLString -> Headers -> [CommandParam] -> FilePath -> IO Bool
+download url headers options file = ifM (inPath "wget") (wget , curl)
 	where
-		wget = go "wget" [Params "-c -O"]
+		headerparams = map (\h -> Param $ "--header=" ++ h) headers
+		wget = go "wget" $ Params "-c -O" : headerparams
 		{- Uses the -# progress display, because the normal
 		 - one is very confusing when resuming, showing
 		 - the remainder to download as the whole file,
 		 - and not indicating how much percent was
 		 - downloaded before the resume. -}
-		curl = go "curl" [Params "-L -C - -# -o"]
+		curl = go "curl" $ Params "-L -C - -# -o" : headerparams
 		go cmd opts = boolSystem cmd $
 			options++opts++[File file, File url]
 
 {- Downloads a small file. -}
-get :: URLString -> IO String
-get url =
+get :: URLString -> Headers -> IO String
+get url headers =
 	case parseURI url of
 		Nothing -> error "url parse error"
 		Just u -> do
-			r <- request u GET
+			r <- request u headers GET
 			case rspCode r of
 				(2,_,_) -> return $ rspBody r
 				_ -> error $ rspReason r
@@ -82,8 +85,8 @@ get url =
  - This does its own redirect following because Browser's is buggy for HEAD
  - requests.
  -}
-request :: URI -> RequestMethod -> IO (Response String)
-request url requesttype = go 5 url
+request :: URI -> Headers -> RequestMethod -> IO (Response String)
+request url headers requesttype = go 5 url
 	where
 		go :: Int -> URI -> IO (Response String)
 		go 0 _ = error "Too many redirects "
@@ -92,7 +95,8 @@ request url requesttype = go 5 url
 				Browser.setErrHandler ignore
 				Browser.setOutHandler ignore
 				Browser.setAllowRedirects False
-				snd <$> Browser.request (mkRequest requesttype u :: Request_String)
+				let req = mkRequest requesttype u :: Request_String
+				snd <$> Browser.request (addheaders req)
 			case rspCode rsp of
 				(3,0,x) | x /= 5 -> redir (n - 1) u rsp
 				_ -> return rsp
@@ -105,3 +109,5 @@ request url requesttype = go 5 url
 					Just newURI -> go n newURI_abs
 						where
 							newURI_abs = fromMaybe newURI (newURI `relativeTo` u)
+		addheaders req = setHeaders req (rqHeaders req ++ userheaders)
+		userheaders = rights $ map parseHeader headers
diff --git a/debian/changelog b/debian/changelog
index 3c56fda51a..c55932e3e0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,9 @@ git-annex (3.20120419) UNRELEASED; urgency=low
   * Fix use of annex.diskreserve config setting.
   * Directory special remotes now check annex.diskreserve.
   * Support git's core.sharedRepository configuration.
+  * Add annex.httpheaders and annex.httpheader-command config
+    settings, to allow custom headers to be sent with all HTTP requests.
+    (Requested by the Internet Archive)
 
  -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 72301c0719..098d520010 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -713,6 +713,16 @@ Here are all the supported configuration settings.
   (wget is always used in preference to curl if available).
   For example, to force ipv4 only, set it to "-4"
 
+* `annex.http-headers`
+
+  HTTP headers to send when downloading from the web. Multiple lines of
+  this option can be set, one per header.
+
+* `annex.http-headers-command`
+
+  If set, the command is run and each line of its output is used as a HTTP
+  header. This overrides annex.http-headers.
+
 * `remote..rsyncurl`
 
   Used by rsync special remotes, this configures

From f8fc79cf58fa8c3acf9948a162a81850ffbbccb5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 01:16:08 -0400
Subject: [PATCH 3407/8313] fixed

---
 doc/todo/http_headers.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/todo/http_headers.mdwn b/doc/todo/http_headers.mdwn
index 99f083b7ae..9f61bdc931 100644
--- a/doc/todo/http_headers.mdwn
+++ b/doc/todo/http_headers.mdwn
@@ -4,3 +4,5 @@ example.
 
 * annex-web-headers=blah 
 * Perhaps also annex-web-headers-command=blah
+
+[[done]]

From 42e4145a170a12f557fbe58344bb115061452d2a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 01:20:17 -0400
Subject: [PATCH 3408/8313] bugfixes

---
 Config.hs      | 4 ++--
 Utility/Url.hs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Config.hs b/Config.hs
index 065ee48f39..3feb246e53 100644
--- a/Config.hs
+++ b/Config.hs
@@ -101,7 +101,7 @@ getDiskReserve = fromMaybe megabyte . readSize dataUnits
  - splitting it into lines. -}
 getHttpHeaders :: Annex [String]
 getHttpHeaders = do
-	cmd <- getConfig "annex.httpheaders-command" ""
+	cmd <- getConfig "annex.http-headers-command" ""
 	if (null cmd)
-		then fromRepo $ Git.Config.getList "annex.httpheaders"
+		then fromRepo $ Git.Config.getList "annex.http-headers"
 		else lines . snd <$> liftIO (pipeFrom "sh" ["-c", cmd])
diff --git a/Utility/Url.hs b/Utility/Url.hs
index 465ef855c8..b75229e1bc 100644
--- a/Utility/Url.hs
+++ b/Utility/Url.hs
@@ -57,13 +57,13 @@ download :: URLString -> Headers -> [CommandParam] -> FilePath -> IO Bool
 download url headers options file = ifM (inPath "wget") (wget , curl)
 	where
 		headerparams = map (\h -> Param $ "--header=" ++ h) headers
-		wget = go "wget" $ Params "-c -O" : headerparams
+		wget = go "wget" $ headerparams ++ [Params "-c -O"]
 		{- Uses the -# progress display, because the normal
 		 - one is very confusing when resuming, showing
 		 - the remainder to download as the whole file,
 		 - and not indicating how much percent was
 		 - downloaded before the resume. -}
-		curl = go "curl" $ Params "-L -C - -# -o" : headerparams
+		curl = go "curl" $ headerparams ++ [Params "-L -C - -# -o"]
 		go cmd opts = boolSystem cmd $
 			options++opts++[File file, File url]
 

From 2d0290bda0b62b03a65659d5c238a5d173cdf7f5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 22 Apr 2012 05:28:21 +0000
Subject: [PATCH 3409/8313]

---
 .../error_in_installation_of_base-4.5.0.0.mdwn     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/error_in_installation_of_base-4.5.0.0.mdwn

diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0.mdwn b/doc/forum/error_in_installation_of_base-4.5.0.0.mdwn
new file mode 100644
index 0000000000..673222ed6d
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0.mdwn
@@ -0,0 +1,14 @@
+Hi,
+
+I was trying to install git-annex, but then, I got a warning saying that I need to install base-4.5.0.0 first.
+
+So, I did "sudo cabal install base-4.5.0.0". Everything was going well, until I got this error:
+
+config.status: error: cannot find input file: `base.buildinfo.in'
+cabal: Error: some packages failed to install:
+base-4.5.0.0 failed during the configure step. The exception was:
+ExitFailure 1
+
+I tried to look for information on the internet, but I did not find anything useful.
+
+I know that this is not totally related to git-annex, but anyone has any thoughts on this?

From e36b2c7b3d72161019ac2295a4f658b4dc2ab837 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 01:31:19 -0400
Subject: [PATCH 3410/8313] typo

---
 doc/install/Fedora.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn
index a4dd4f5c87..50f1d78180 100644
--- a/doc/install/Fedora.mdwn
+++ b/doc/install/Fedora.mdwn
@@ -4,7 +4,7 @@ Installation recipe for Fedora 14 thruough 17.
 sudo yum install ghc cabal-install
 git clone git://git-annex.branchable.com/ git-annex
 cd git-annex
-git checkout ghc 7.0
+git checkout ghc7.0
 cabal update
 cabal install --only-dependencies
 cabal configure

From 744b33c240ec476de4de616738d192fa919ae003 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 22 Apr 2012 05:39:28 +0000
Subject: [PATCH 3411/8313] Added a comment

---
 ..._0b2f79c014e0dd9badd52b8b6aa47e0c._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment

diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment
new file mode 100644
index 0000000000..5a52a28ab1
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_1_0b2f79c014e0dd9badd52b8b6aa47e0c._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-22T05:39:28Z"
+ content="""
+git-annex needs ghc 7.4, that's why it depends on that base version that comes with it. So you either need to upgrade your ghc, or you can build from the `ghc7.0` branch in [[git|download]], like this:
+
+
+git clone git://git-annex.branchable.com/ git-annex
+cd git-annex
+git checkout ghc7.0
+cabal update
+cabal install --only-dependencies
+cabal configure
+cabal build
+cabal install --bindir=$HOME/bin
+

+"""]]

From 54c2cb8558e8e9747c8f721604cd47b060a69e90 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 22 Apr 2012 14:09:33 +0000
Subject: [PATCH 3412/8313] Added a comment

---
 ..._3badd64e48fbb174cd7de1ac9589bedf._comment | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment

diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment
new file mode 100644
index 0000000000..3ad1a11388
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_2_3badd64e48fbb174cd7de1ac9589bedf._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 2"
+ date="2012-04-22T14:09:33Z"
+ content="""
+Thanks for the fast response!
+
+Unfortunately, I had another problem:
+
+==================================
+Building git-annex-3.20120419...
+Utility/libdiskfree.c: In function ‘diskfree’:
+
+Utility/libdiskfree.c:61:0:
+     warning: ‘statfs64’ is deprecated (declared at /usr/include/sys/mount.h:379)
+[  6 of 157] Compiling Build.SysConfig  ( Build/SysConfig.hs, dist/build/git-annex/git-annex-tmp/Build/SysConfig.o )
+[ 15 of 157] Compiling Utility.Touch    ( dist/build/git-annex/git-annex-tmp/Utility/Touch.hs, dist/build/git-annex/git-annex-tmp/Utility/Touch.o )
+
+Utility/Touch.hsc:118:21: Not in scope: `noop'
+cabal: Error: some packages failed to install:
+git-annex-3.20120419 failed during the building phase. The exception was:
+ExitFailure 1
+==================================
+
+I also tried to look for information on the internet, and I did not find anything useful.
+Any idea of what happened?
+
+Thanks again!
+
+"""]]

From 94e479dd1b71c0495baae690cf0eabb282919417 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 01:49:15 -0400
Subject: [PATCH 3413/8313] simplfy versioned base dependency

---
 git-annex.cabal | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 041b077dc5..0635de6b17 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -34,7 +34,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
-   base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
+   base == 4.5.*, base < 5, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
@@ -50,7 +50,7 @@ Test-Suite test
   Build-Depends: testpack, HUnit, MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
-   base >= 4.5, base < 5, monad-control, transformers-base, lifted-base,
+   base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP

From f7667742098e8147f54e15e57b130dafda156017 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 11:21:23 -0400
Subject: [PATCH 3414/8313] unbreak code inside ifdef

---
 Utility/Touch.hsc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/Touch.hsc b/Utility/Touch.hsc
index e2dba79ab2..0b1ca3d9bd 100644
--- a/Utility/Touch.hsc
+++ b/Utility/Touch.hsc
@@ -115,6 +115,6 @@ touchBoth file atime mtime follow =
 
 #else
 #warning "utimensat and lutimes not available; building without symlink timestamp preservation support"
-touchBoth _ _ _ _ = noop
+touchBoth _ _ _ _ = return ()
 #endif
 #endif

From b5fc3746a8f3d390aad9d7966815e9b72e8f2eee Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 22 Apr 2012 15:23:26 +0000
Subject: [PATCH 3415/8313] Added a comment

---
 .../comment_3_d8190061ac1c683a7b699cf42e9db694._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment

diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment
new file mode 100644
index 0000000000..d9de5089e9
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_3_d8190061ac1c683a7b699cf42e9db694._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-04-22T15:23:26Z"
+ content="""
+That's my fault, I made a change last night that caused the noop problem. Fixed now.
+"""]]

From 1b876daad10b05dde49d8fef4befa70a11e8d9ed Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 11:24:39 -0400
Subject: [PATCH 3416/8313] remove unneeded dependency

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 0635de6b17..da0e9f4de7 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -34,7 +34,7 @@ Executable git-annex
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
-   base == 4.5.*, base < 5, monad-control, transformers-base, lifted-base,
+   base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c

From 1db09af14cf41dff96be23935e4a690b95c0f6b8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 22 Apr 2012 11:42:38 -0400
Subject: [PATCH 3417/8313] fix names

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index c55932e3e0..1962a6b669 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,7 @@ git-annex (3.20120419) UNRELEASED; urgency=low
   * Fix use of annex.diskreserve config setting.
   * Directory special remotes now check annex.diskreserve.
   * Support git's core.sharedRepository configuration.
-  * Add annex.httpheaders and annex.httpheader-command config
+  * Add annex.http-headers and annex.http-headers-command config
     settings, to allow custom headers to be sent with all HTTP requests.
     (Requested by the Internet Archive)
 

From cd8934e4a2268431d4cc5dc8e665cf5303dfdb46 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 22 Apr 2012 16:08:56 +0000
Subject: [PATCH 3418/8313] Added a comment

---
 .../comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment

diff --git a/doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment
new file mode 100644
index 0000000000..ba99334511
--- /dev/null
+++ b/doc/forum/error_in_installation_of_base-4.5.0.0/comment_4_49a4fcd2dc4f97d4055b5051feea5e3b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 4"
+ date="2012-04-22T16:08:55Z"
+ content="""
+Thanks, it worked now!
+"""]]

From 450201dcee2bd0900cf29da4ddf04a41457e3662 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Mon, 23 Apr 2012 04:03:17 +0000
Subject: [PATCH 3419/8313]

---
 doc/forum/What_can_be_done_in_case_of_conflict.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/What_can_be_done_in_case_of_conflict.mdwn

diff --git a/doc/forum/What_can_be_done_in_case_of_conflict.mdwn b/doc/forum/What_can_be_done_in_case_of_conflict.mdwn
new file mode 100644
index 0000000000..42167a6091
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+How can I resolve the conflict when it occurs?
+
+Suppose I have 2 branches (master, current), When I merge these branches or while doing cherry-pick, if I get conflict how can I resolve it?
+
+Thank You

From c3e6fb165ee21aa944f034288b23ca6e0c0b9a1f Mon Sep 17 00:00:00 2001
From: "http://christian.amsuess.com/chrysn" 
Date: Mon, 23 Apr 2012 14:14:29 +0000
Subject: [PATCH 3420/8313] Added a comment

---
 ...comment_3_c336b2b07cd006d378e5be9639ff17ec._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment

diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment
new file mode 100644
index 0000000000..fd75f2f856
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_3_c336b2b07cd006d378e5be9639ff17ec._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/chrysn"
+ nickname="chrysn"
+ subject="comment 3"
+ date="2012-04-23T14:14:28Z"
+ content="""
+thanks, that's great. will there be a way to have sharedRepository work for shared remotes (rsync, directory) too, or is that better taken care of by acls?
+
+@not thought of shared repos: we're having our family photo archive spread over our laptops, and backed up on our home storage server and on an rsync+encryption off-site server, with everyone naturally having their own accounts on all systems -- just if you need a use case.
+"""]]

From 7de646596faf4533aca65787539b5e1e244f9e68 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 23 Apr 2012 14:29:03 +0000
Subject: [PATCH 3421/8313] Added a comment

---
 ...mment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment

diff --git a/doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment b/doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment
new file mode 100644
index 0000000000..6f1d241300
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict/comment_1_5ca86b099dfa08a50f656ea03bf1dcd9._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-23T14:29:03Z"
+ content="""
+You handle conflicts in annexed files the same as you would handle them in other binary files checked into git.
+
+For example, you might choose to `git rm` or `git add` the file to resolve the conflict.
+
+[[Previous_discussion|forum/A_really_stupid_question]]
+"""]]

From 5cabab81540268aaf4da56c4fd897951483756b1 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Mon, 23 Apr 2012 14:35:40 +0000
Subject: [PATCH 3422/8313] Added a comment

---
 ...comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment

diff --git a/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment
new file mode 100644
index 0000000000..568f11330c
--- /dev/null
+++ b/doc/forum/__34__permission_denied__34___in_fsck_on_shared_repo/comment_4_1339cd27ca2955f30b01ecf4da7d6fe8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 4"
+ date="2012-04-23T14:35:39Z"
+ content="""
+I'm not currently planning to support sharedRepository perms on special remotes. I suppose I could be convinced otherwise, it's perhaps doable for the ones you mention (rsync might be tricky). (bup special remote already supports it of course.)
+
+thanks for the use case!
+"""]]

From eedde3454901887f1a29766b15f60e26be64583a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 23 Apr 2012 10:37:05 -0400
Subject: [PATCH 3423/8313] show amount of reserved space

---
 Command/Status.hs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 0c6eda0b2a..16bcec7cce 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -69,6 +69,7 @@ fast_stats =
 	, remote_list SemiTrusted "semitrusted"
 	, remote_list UnTrusted "untrusted"
 	, remote_list DeadTrusted "dead"
+	, disk_size
 	]
 slow_stats :: [Stat]
 slow_stats = 
@@ -78,7 +79,6 @@ slow_stats =
 	, local_annex_size
 	, known_annex_keys
 	, known_annex_size
-	, disk_size
 	, bloom_info
 	, backend_usage
 	]
@@ -175,8 +175,12 @@ disk_size = stat "available local disk space" $ json id $ lift $
 		<$> getDiskReserve
 		<*> inRepo (getDiskFree . gitAnnexDir)
 	where
-		calcfree reserve (Just have) =
-			roughSize storageUnits False $ nonneg $ have - reserve
+		calcfree reserve (Just have) = unwords
+			[ roughSize storageUnits False $ nonneg $ have - reserve
+			, "(+" ++ roughSize storageUnits False reserve
+			, "reserved)"
+			]
+			
 		calcfree _ _ = "unknown"
 		nonneg x
 			| x >= 0 = x

From 6050c7cca789311eaa12e561c23e2589df1fa6b5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Tue, 24 Apr 2012 21:10:28 +0000
Subject: [PATCH 3424/8313]

---
 doc/forum/retrieving_previous_versions.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/retrieving_previous_versions.mdwn

diff --git a/doc/forum/retrieving_previous_versions.mdwn b/doc/forum/retrieving_previous_versions.mdwn
new file mode 100644
index 0000000000..7626b7935b
--- /dev/null
+++ b/doc/forum/retrieving_previous_versions.mdwn
@@ -0,0 +1,7 @@
+Hi,
+
+This might be a stupid question, but I did not find any information about it.
+Can I retrieve previous versions of a file?
+Let's say, I wanna do a "git annex get file", but considering a specific commit id. Is it possible? Are all the versions of the files kept inside .git/annex/objects?
+
+Thanks!

From 64e0e9d2cd68c453a02d71f21043312312aa099d Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Tue, 24 Apr 2012 21:14:15 +0000
Subject: [PATCH 3425/8313] Added a comment

---
 ...omment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment

diff --git a/doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment b/doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment
new file mode 100644
index 0000000000..ab2ecee317
--- /dev/null
+++ b/doc/forum/retrieving_previous_versions/comment_1_a4e83f688d4ec9177e7bf520f12ed26d._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-04-24T21:14:15Z"
+ content="""
+To get to a specific version of a file, you need to have a tag or a branch that includes that version of the file. Check out the branch and `git annex get $file`. 
+
+(Of course, even without a tag or branch, old file versions are retained, unless dropped with `unused`/`dropunused`.
+So you could even `git checkout $COMMITID`.)
+"""]]

From a2ee4386bb8f64046a5838038c89492dddcce94f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkO9tsPZkAxEulq2pGCdwz4md-LqB0RcMw"
 
Date: Wed, 25 Apr 2012 22:56:20 +0000
Subject: [PATCH 3426/8313] Added a comment: Problems with Base & Crypto

---
 ..._e6109a964064a2a799768a370e57801d._comment | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment

diff --git a/doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment b/doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment
new file mode 100644
index 0000000000..be3ba2be44
--- /dev/null
+++ b/doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment
@@ -0,0 +1,30 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkO9tsPZkAxEulq2pGCdwz4md-LqB0RcMw"
+ nickname="Reimund"
+ subject="Problems with Base & Crypto"
+ date="2012-04-25T22:56:18Z"
+ content="""
+I got the following error message trying to install git-annex:
+
+    cabal: cannot configure git-annex-3.20120418. It requires base >=4.5 && <5
+    For the dependency on base >=4.5 && <5 there are these packages: base-4.5.0.0.
+    However none of them are available.
+    base-4.5.0.0 was excluded because of the top level dependency base -any
+
+These are the steps I performed to make it work
+
+1. Download [Ghc 7.4](http://www.haskell.org/ghc/download).
+2. Run `sudo cabal install git-annex --bindir=$HOME/bin`.
+3. Compilation of the Crypto-4.2.4 dependency failed since it's not updated to work with Ghc 7.4. You need to patch SHA2.hs (steps below).
+4. Run `sudo cabal install git-annex --bindir=$HOME/bin` a second time.
+
+The steps I did to patch the SHA2.hs file in Crypto-4.2.4:
+
+1. `cabal unpack crypto-4.2.4`
+2. `cd Crypto-4.2.4`
+3. `patch -p1 < crypto-4.2.4-ghc-7.4.patch`
+4. `sudo cabal install`.
+
+PS: I used [this patchfile](http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/dev-haskell/crypto/files/crypto-4.2.4-ghc-7.4.patch?revision=1.1).
+Then I did the last step a third time.
+"""]]

From e0b7012ccc405dedb556b8c940eb66e42304bc73 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 Apr 2012 12:21:38 -0400
Subject: [PATCH 3427/8313] uninit: Clear annex.uuid from .git/config. Closes:
 #670639

---
 Annex/UUID.hs    | 8 +++++++-
 Init.hs          | 4 +++-
 debian/changelog | 1 +
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index e8306de903..5459cc7fee 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -16,12 +16,14 @@ module Annex.UUID (
 	getRepoUUID,
 	getUncachedUUID,
 	prepUUID,
-	genUUID
+	genUUID,
+	removeRepoUUID,
 ) where
 
 import Common.Annex
 import qualified Git
 import qualified Git.Config
+import qualified Git.Command
 import qualified Build.SysConfig as SysConfig
 import Config
 
@@ -61,6 +63,10 @@ getRepoUUID r = do
 			when (g /= r) $ storeUUID cachekey u
 		cachekey = remoteConfig r "uuid"
 
+removeRepoUUID :: Annex ()
+removeRepoUUID = inRepo $ Git.Command.run "config"
+	[Param "--unset", Param configkey]
+
 getUncachedUUID :: Git.Repo -> UUID
 getUncachedUUID = toUUID . Git.Config.get configkey ""
 
diff --git a/Init.hs b/Init.hs
index 9f1988a394..a0e16e8815 100644
--- a/Init.hs
+++ b/Init.hs
@@ -29,7 +29,9 @@ initialize mdescription = do
 	maybe (recordUUID u) (describeUUID u) mdescription
 
 uninitialize :: Annex ()
-uninitialize = gitPreCommitHookUnWrite
+uninitialize = do
+	gitPreCommitHookUnWrite
+	removeRepoUUID
 
 {- Will automatically initialize if there is already a git-annex
    branch from somewhere. Otherwise, require a manual init
diff --git a/debian/changelog b/debian/changelog
index 1962a6b669..06990a0f8a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,7 @@ git-annex (3.20120419) UNRELEASED; urgency=low
   * Add annex.http-headers and annex.http-headers-command config
     settings, to allow custom headers to be sent with all HTTP requests.
     (Requested by the Internet Archive)
+  * uninit: Clear annex.uuid from .git/config. Closes: #670639
 
  -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
 

From 76102c1c7541e7b10c3a3fbe242e9856fef955b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 27 Apr 2012 13:23:52 -0400
Subject: [PATCH 3428/8313] display "Recording state in git..." when staging
 the journal

A bit tricky to avoid printing it twice in a row when there are queued git
commands to run and journal to stage.

Added a generic way to run an action that may output multiple side
messages, with only the first displayed.
---
 Annex.hs          |  8 +++-----
 Annex/Branch.hs   |  1 +
 Annex/Content.hs  |  4 ++--
 Annex/Queue.hs    |  9 ++++-----
 Messages.hs       | 45 +++++++++++++++++++++++++++++++++++----------
 Option.hs         | 10 ++++++----
 Types/Messages.hs | 20 ++++++++++++++++++++
 Upgrade/V1.hs     |  2 +-
 8 files changed, 72 insertions(+), 27 deletions(-)
 create mode 100644 Types/Messages.hs

diff --git a/Annex.hs b/Annex.hs
index 2b58c2bd49..441e460beb 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -10,7 +10,6 @@
 module Annex (
 	Annex,
 	AnnexState(..),
-	OutputType(..),
 	new,
 	newState,
 	run,
@@ -44,6 +43,7 @@ import qualified Types.Remote
 import Types.Crypto
 import Types.BranchState
 import Types.TrustLevel
+import Types.Messages
 import Utility.State
 import qualified Utility.Matcher
 import qualified Data.Map as M
@@ -69,8 +69,6 @@ instance MonadBaseControl IO Annex where
 		where
 			unStAnnex (StAnnex st) = st
 
-data OutputType = NormalOutput | QuietOutput | JSONOutput
-
 type Matcher a = Either [Utility.Matcher.Token a] (Utility.Matcher.Matcher a)
 
 -- internal state storage
@@ -78,7 +76,7 @@ data AnnexState = AnnexState
 	{ repo :: Git.Repo
 	, backends :: [BackendA Annex]
 	, remotes :: [Types.Remote.RemoteA Annex]
-	, output :: OutputType
+	, output :: MessageState
 	, force :: Bool
 	, fast :: Bool
 	, auto :: Bool
@@ -104,7 +102,7 @@ newState gitrepo = AnnexState
 	{ repo = gitrepo
 	, backends = []
 	, remotes = []
-	, output = NormalOutput
+	, output = defaultMessageState
 	, force = False
 	, fast = False
 	, auto = False
diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index ce1dd58cec..706522f3b3 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -330,6 +330,7 @@ setCommitted = void $ do
 {- Stages the journal into the index. -}
 stageJournal :: Annex ()
 stageJournal = do
+	showStoringStateAction
 	fs <- getJournalFiles
 	g <- gitRepo
 	withIndex $ liftIO $ do
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 01ee7d83d9..b5754e15be 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -297,8 +297,8 @@ getKeysPresent = liftIO . traverse (2 :: Int) =<< fromRepo gitAnnexObjectDir
  - especially if performing a short-lived action.
  -}
 saveState :: Bool -> Annex ()
-saveState oneshot = do
-	Annex.Queue.flush False
+saveState oneshot = doSideAction $ do
+	Annex.Queue.flush
 	unless oneshot $
 		ifM alwayscommit
 			( Annex.Branch.commit "update" , Annex.Branch.stage)
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index f49a220690..728e29645e 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -26,15 +26,14 @@ add command params files = do
 flushWhenFull :: Annex ()
 flushWhenFull = do
 	q <- get
-	when (Git.Queue.full q) $ flush False
+	when (Git.Queue.full q) flush
 
 {- Runs (and empties) the queue. -}
-flush :: Bool -> Annex ()
-flush silent = do
+flush :: Annex ()
+flush = do
 	q <- get
 	unless (0 == Git.Queue.size q) $ do
-		unless silent $
-			showSideAction "Recording state in git"
+		showStoringStateAction
 		q' <- inRepo $ Git.Queue.flush q
 		store q'
 
diff --git a/Messages.hs b/Messages.hs
index af7eb88b43..4330f7c09f 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -13,6 +13,8 @@ module Messages (
 	metered,
 	MeterUpdate,
 	showSideAction,
+	doSideAction,
+	showStoringStateAction,
 	showOutput,
 	showLongNote,
 	showEndOk,
@@ -37,6 +39,7 @@ import Data.Quantity
 
 import Common
 import Types
+import Types.Messages
 import Types.Key
 import qualified Annex
 import qualified Messages.JSON as JSON
@@ -61,9 +64,9 @@ showProgress = handle q $
  - The action is passed a callback to use to update the meter. -}
 type MeterUpdate = Integer -> IO ()
 metered :: Key -> (MeterUpdate -> Annex a) -> Annex a
-metered key a = Annex.getState Annex.output >>= go (keySize key)
+metered key a = withOutputType $ go (keySize key)
 	where
-		go (Just size) Annex.NormalOutput = do
+		go (Just size) NormalOutput = do
 			progress <- liftIO $ newProgress "" size
 			meter <- liftIO $ newMeter progress "B" 25 (renderNums binaryOpts 1)
 			showOutput
@@ -76,8 +79,27 @@ metered key a = Annex.getState Annex.output >>= go (keySize key)
                 go _ _ = a (const noop)
 
 showSideAction :: String -> Annex ()
-showSideAction s = handle q $
-	putStrLn $ "(" ++ s ++ "...)"
+showSideAction m = Annex.getState Annex.output >>= go
+	where
+		go (MessageState v StartBlock) = do
+			p
+	 		Annex.changeState $ \s -> s { Annex.output = MessageState v InBlock }
+		go (MessageState _ InBlock) = return ()
+		go _ = p
+		p = handle q $ putStrLn $ "(" ++ m ++ "...)"
+			
+showStoringStateAction :: Annex ()
+showStoringStateAction = showSideAction "Recording state in git"
+
+{- Performs an action, that may call showSideAction multiple times.
+ - Only the first will be displayed. -}
+doSideAction :: Annex a -> Annex a
+doSideAction a = do
+	o <- Annex.getState Annex.output
+	set $ o { sideActionBlock = StartBlock }
+	set o `after` a
+	where
+		set o = Annex.changeState $ \s -> s {  Annex.output = o }
 
 showOutput :: Annex ()
 showOutput = handle q $
@@ -122,9 +144,9 @@ maybeShowJSON v = handle (JSON.add v) q
 
 {- Shows a complete JSON value, only when in json mode. -}
 showFullJSON :: JSON a => [(String, a)] -> Annex Bool
-showFullJSON v = Annex.getState Annex.output >>= liftIO . go
+showFullJSON v = withOutputType $ liftIO . go
 	where
-		go Annex.JSONOutput = JSON.complete v >> return True
+		go JSONOutput = JSON.complete v >> return True
 		go _ = return False
 
 {- Performs an action that outputs nonstandard/customized output, and
@@ -153,14 +175,17 @@ setupConsole = do
 	fileEncoding stderr
 
 handle :: IO () -> IO () -> Annex ()
-handle json normal = Annex.getState Annex.output >>= go
+handle json normal = withOutputType $ go
 	where
-		go Annex.NormalOutput = liftIO normal
-		go Annex.QuietOutput = q
-		go Annex.JSONOutput = liftIO $ flushed json
+		go NormalOutput = liftIO normal
+		go QuietOutput = q
+		go JSONOutput = liftIO $ flushed json
 
 q :: Monad m => m ()
 q = noop
 
 flushed :: IO () -> IO ()
 flushed a = a >> hFlush stdout
+
+withOutputType :: (OutputType -> Annex a) -> Annex a
+withOutputType a = outputType <$> Annex.getState Annex.output >>= a
diff --git a/Option.hs b/Option.hs
index 2f0d00744d..7696220554 100644
--- a/Option.hs
+++ b/Option.hs
@@ -20,6 +20,7 @@ import System.Log.Logger
 
 import Common.Annex
 import qualified Annex
+import Types.Messages
 import Limit
 import Usage
 
@@ -31,11 +32,11 @@ common =
 		"avoid slow operations"
 	, Option ['a'] ["auto"] (NoArg (setauto True))
 		"automatic mode"
-	, Option ['q'] ["quiet"] (NoArg (setoutput Annex.QuietOutput))
+	, Option ['q'] ["quiet"] (NoArg (setoutput QuietOutput))
 		"avoid verbose output"
-	, Option ['v'] ["verbose"] (NoArg (setoutput Annex.NormalOutput))
+	, Option ['v'] ["verbose"] (NoArg (setoutput NormalOutput))
 		"allow verbose output (default)"
-	, Option ['j'] ["json"] (NoArg (setoutput Annex.JSONOutput))
+	, Option ['j'] ["json"] (NoArg (setoutput JSONOutput))
 		"enable JSON output"
 	, Option ['d'] ["debug"] (NoArg setdebug)
 		"show debug messages"
@@ -46,7 +47,8 @@ common =
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setauto v = Annex.changeState $ \s -> s { Annex.auto = v }
-		setoutput v = Annex.changeState $ \s -> s { Annex.output = v }
+		setoutput v = Annex.changeState $ \s ->
+			s { Annex.output = (Annex.output s) { outputType = v } }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG
diff --git a/Types/Messages.hs b/Types/Messages.hs
new file mode 100644
index 0000000000..75653d574c
--- /dev/null
+++ b/Types/Messages.hs
@@ -0,0 +1,20 @@
+{- git-annex Messages data types
+ - 
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Types.Messages where
+
+data OutputType = NormalOutput | QuietOutput | JSONOutput
+
+data SideActionBlock = NoBlock | StartBlock | InBlock
+
+data MessageState = MessageState
+	{ outputType :: OutputType
+	, sideActionBlock :: SideActionBlock
+	}
+
+defaultMessageState :: MessageState
+defaultMessageState = MessageState NormalOutput NoBlock
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index a8005b2644..ddf0728b61 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -59,7 +59,7 @@ upgrade = do
 			updateSymlinks
 			moveLocationLogs
 	
-			Annex.Queue.flush True
+			Annex.Queue.flush
 			setVersion
 		)
 	

From 090d50a7f7741e1b8d69a416563e86739f747a0b Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Sat, 28 Apr 2012 22:53:38 +0000
Subject: [PATCH 3429/8313]

---
 doc/forum/wishlist:_simpler_gpg_usage.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_simpler_gpg_usage.mdwn

diff --git a/doc/forum/wishlist:_simpler_gpg_usage.mdwn b/doc/forum/wishlist:_simpler_gpg_usage.mdwn
new file mode 100644
index 0000000000..f09f817af1
--- /dev/null
+++ b/doc/forum/wishlist:_simpler_gpg_usage.mdwn
@@ -0,0 +1,10 @@
+This is my current understanding on how one must use gpg with git-annex:
+
+ * Generate(or copy around) a gpg key on every machine that needs to access the encrypted remote.
+ * git annex initremote myremote encryption=KEY for each key that you generated
+
+What I'm trying to figure out is if I can generate a no-passphrase gpg key and commit it to the repository, and have git-annex use that. That way any new clones of the annex automatically have access to any encrypted remotes, without having to do any key management.
+
+I think I can generate a no-passphrase key, but then I still have to manually copy it around to each machine.
+
+I'm not a huge gpg user so part of this is me wanting to avoid having to manage and keeping track of the keys.  This would probably be a non-issue if I used gpg on more machines and was more comfortable with it.

From 20a25d2f20f255f434193bab31d0981d74b3aee3 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Sun, 29 Apr 2012 01:41:58 +0000
Subject: [PATCH 3430/8313] Added a comment

---
 ..._6923fa6ebc0bbe7d93edb1d01d7c46c5._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/forum/wishlist:_simpler_gpg_usage/comment_1_6923fa6ebc0bbe7d93edb1d01d7c46c5._comment

diff --git a/doc/forum/wishlist:_simpler_gpg_usage/comment_1_6923fa6ebc0bbe7d93edb1d01d7c46c5._comment b/doc/forum/wishlist:_simpler_gpg_usage/comment_1_6923fa6ebc0bbe7d93edb1d01d7c46c5._comment
new file mode 100644
index 0000000000..f96f5c3777
--- /dev/null
+++ b/doc/forum/wishlist:_simpler_gpg_usage/comment_1_6923fa6ebc0bbe7d93edb1d01d7c46c5._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-04-29T01:41:57Z"
+ content="""
+Thinking about this more, I think minimally git-annex could support a
+
+    remote..gpg-options
+
+or
+
+    remote..gpg-keyring
+
+for options to be passed to gpg.  I'm not sure how automatically setting it to $ANNEX_ROOT/.gnupg/.. would work.
+
+
+I need to read the encryption code to fully understand it, but I also wonder if there is not also a way to just bypass gpg entirely and store the remote-encryption keys locally in plain text.
+"""]]

From 44defe8b70a299a66050e8c60fe7f46daeed6661 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 29 Apr 2012 02:39:20 +0000
Subject: [PATCH 3431/8313] Added a comment

---
 ...comment_2_6fc874b6c391df242bd2592c4a65eae8._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_simpler_gpg_usage/comment_2_6fc874b6c391df242bd2592c4a65eae8._comment

diff --git a/doc/forum/wishlist:_simpler_gpg_usage/comment_2_6fc874b6c391df242bd2592c4a65eae8._comment b/doc/forum/wishlist:_simpler_gpg_usage/comment_2_6fc874b6c391df242bd2592c4a65eae8._comment
new file mode 100644
index 0000000000..29eb11622a
--- /dev/null
+++ b/doc/forum/wishlist:_simpler_gpg_usage/comment_2_6fc874b6c391df242bd2592c4a65eae8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 2"
+ date="2012-04-29T02:39:20Z"
+ content="""
+The encryption uses a symmetric cipher that is stored in the git repository already. It's just stored encrypted to the various gpg keys that have been configured to use it. It would certianly be possible to store the symmetric cipher unencrypted in the git repo. 
+
+I don't see your idea of gpg-options saving any work. It would still require you to do key distribution and run commands in each repo to set it up.
+"""]]

From 0828f31a8bbf6f3bb27b2b2c0b0802269ceb3b58 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 29 Apr 2012 02:41:38 +0000
Subject: [PATCH 3432/8313] Added a comment

---
 .../comment_3_012f340c8c572fe598fc860c1046dabd._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_simpler_gpg_usage/comment_3_012f340c8c572fe598fc860c1046dabd._comment

diff --git a/doc/forum/wishlist:_simpler_gpg_usage/comment_3_012f340c8c572fe598fc860c1046dabd._comment b/doc/forum/wishlist:_simpler_gpg_usage/comment_3_012f340c8c572fe598fc860c1046dabd._comment
new file mode 100644
index 0000000000..051f17a24b
--- /dev/null
+++ b/doc/forum/wishlist:_simpler_gpg_usage/comment_3_012f340c8c572fe598fc860c1046dabd._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 3"
+ date="2012-04-29T02:41:38Z"
+ content="""
+BTW re your Tweet.. I was so happy to be able to use 'c i a' in Crypto.hs. :)
+"""]]

From d7a4a9a66bd51b18a9e5f4427c3492db1adec40d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 
Date: Sun, 29 Apr 2012 03:09:04 +0000
Subject: [PATCH 3433/8313] Added a comment

---
 ...comment_4_e0c2a13217b795964f3b630c001661ef._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/wishlist:_simpler_gpg_usage/comment_4_e0c2a13217b795964f3b630c001661ef._comment

diff --git a/doc/forum/wishlist:_simpler_gpg_usage/comment_4_e0c2a13217b795964f3b630c001661ef._comment b/doc/forum/wishlist:_simpler_gpg_usage/comment_4_e0c2a13217b795964f3b630c001661ef._comment
new file mode 100644
index 0000000000..c9e3375414
--- /dev/null
+++ b/doc/forum/wishlist:_simpler_gpg_usage/comment_4_e0c2a13217b795964f3b630c001661ef._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 4"
+ date="2012-04-29T03:09:03Z"
+ content="""
+I got a good laugh out of it :-)
+
+Storing the key unencrypted would make things easier..  I think at least for my use-cases I don't require another layer of protection on top of the ssh keys that provide access to the encrypted remotes themselves.
+"""]]

From 1c16f616df9a8469d24cefb6007333df3a35a449 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 29 Apr 2012 14:02:18 -0400
Subject: [PATCH 3434/8313] Added shared cipher mode to encryptable special
 remotes.

This option avoids gpg key distribution, at the expense of flexability, and
with the requirement that all clones of the git repository be equally
trusted.
---
 Annex.hs                     |  2 +-
 Crypto.hs                    | 75 +++++++++++++++---------------------
 Remote/Helper/Encryptable.hs | 25 +++++++-----
 Types/Crypto.hs              |  6 +--
 Utility/Gpg.hs               | 13 ++++++-
 debian/changelog             |  3 ++
 doc/encryption.mdwn          | 13 +++++++
 7 files changed, 79 insertions(+), 58 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index 441e460beb..f2fb1f4019 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -90,7 +90,7 @@ data AnnexState = AnnexState
 	, shared :: Maybe SharedRepository
 	, forcetrust :: TrustMap
 	, trustmap :: Maybe TrustMap
-	, ciphers :: M.Map EncryptedCipher Cipher
+	, ciphers :: M.Map StorableCipher Cipher
 	, lockpool :: M.Map FilePath Fd
 	, flags :: M.Map String Bool
 	, fields :: M.Map String String
diff --git a/Crypto.hs b/Crypto.hs
index cb1ca40d14..e530bd0e65 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -3,16 +3,17 @@
  - Currently using gpg; could later be modified to support different
  - crypto backends if neccessary.
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Crypto (
 	Cipher,
-	EncryptedCipher,
-	genCipher,
-	updateCipher,
+	StorableCipher(..),
+	genEncryptedCipher,
+	genSharedCipher,
+	updateEncryptedCipher,
 	describeCipher,
 	storeCipher,
 	extractCipher,
@@ -60,59 +61,55 @@ cipherPassphrase (Cipher c) = drop cipherHalf c
 cipherHmac :: Cipher -> String
 cipherHmac (Cipher c) = take cipherHalf c
 
-{- Creates a new Cipher, encrypted as specified in the remote's configuration -}
-genCipher :: RemoteConfig -> IO EncryptedCipher
-genCipher c = do
-	ks <- configKeyIds c
-	random <- genrandom
+{- Creates a new Cipher, encrypted to the specificed key id. -}
+genEncryptedCipher :: String -> IO StorableCipher
+genEncryptedCipher keyid = do
+	ks <- Gpg.findPubKeys keyid
+	random <- Gpg.genRandom cipherSize
 	encryptCipher (Cipher random) ks
-	where
-		genrandom = Gpg.readStrict
-			-- Armor the random data, to avoid newlines,
-			-- since gpg only reads ciphers up to the first
-			-- newline.
-			[ Params "--gen-random --armor"
-			, Param $ show randomquality
-			, Param $ show cipherSize
-			]
-		-- 1 is /dev/urandom; 2 is /dev/random
-		randomquality = 1 :: Int
 
-{- Updates an existing Cipher, re-encrypting it to add KeyIds specified in
- - the remote's configuration. -}
-updateCipher :: RemoteConfig -> EncryptedCipher -> IO EncryptedCipher
-updateCipher c encipher@(EncryptedCipher _ ks) = do
-	ks' <- configKeyIds c
-	cipher <- decryptCipher c encipher
+{- Creates a new, shared Cipher. -}
+genSharedCipher :: IO StorableCipher
+genSharedCipher = SharedCipher <$> Gpg.genRandom cipherSize
+
+{- Updates an existing Cipher, re-encrypting it to add a keyid. -}
+updateEncryptedCipher :: String -> StorableCipher -> IO StorableCipher
+updateEncryptedCipher _ (SharedCipher _) = undefined
+updateEncryptedCipher keyid encipher@(EncryptedCipher _ ks) = do
+	ks' <- Gpg.findPubKeys keyid
+	cipher <- decryptCipher encipher
 	encryptCipher cipher (merge ks ks')
 	where
 		merge (KeyIds a) (KeyIds b) = KeyIds $ a ++ b
 
-describeCipher :: EncryptedCipher -> String
+describeCipher :: StorableCipher -> String
+describeCipher (SharedCipher _) = "shared cipher"
 describeCipher (EncryptedCipher _ (KeyIds ks)) =
 	"with gpg " ++ keys ks ++ " " ++ unwords ks
 	where
 		keys [_] = "key"
 		keys _ = "keys"
 
-{- Stores an EncryptedCipher in a remote's configuration. -}
-storeCipher :: RemoteConfig -> EncryptedCipher -> RemoteConfig
+{- Stores an StorableCipher in a remote's configuration. -}
+storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig
+storeCipher c (SharedCipher t) = M.insert "cipher" (toB64 t) c
 storeCipher c (EncryptedCipher t ks) = 
 	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c
 	where
 		showkeys (KeyIds l) = join "," l
 
-{- Extracts an EncryptedCipher from a remote's configuration. -}
-extractCipher :: RemoteConfig -> Maybe EncryptedCipher
+{- Extracts an StorableCipher from a remote's configuration. -}
+extractCipher :: RemoteConfig -> Maybe StorableCipher
 extractCipher c = 
 	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of
 		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks)
+		(Just t, Nothing) -> Just $ SharedCipher (fromB64 t)
 		_ -> Nothing
 	where
 		readkeys = KeyIds . split ","
 
 {- Encrypts a Cipher to the specified KeyIds. -}
-encryptCipher :: Cipher -> KeyIds -> IO EncryptedCipher
+encryptCipher :: Cipher -> KeyIds -> IO StorableCipher
 encryptCipher (Cipher c) (KeyIds ks) = do
 	let ks' = nub $ sort ks -- gpg complains about duplicate recipient keyids
 	encipher <- Gpg.pipeStrict (encrypt++recipients ks') c
@@ -126,9 +123,9 @@ encryptCipher (Cipher c) (KeyIds ks) = do
 		force_recipients = Params "--no-encrypt-to --no-default-recipient"
 
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
-decryptCipher :: RemoteConfig -> EncryptedCipher -> IO Cipher
-decryptCipher _ (EncryptedCipher encipher _) = 
-	Cipher <$> Gpg.pipeStrict decrypt encipher
+decryptCipher :: StorableCipher -> IO Cipher
+decryptCipher (SharedCipher t) = return $ Cipher t
+decryptCipher (EncryptedCipher t _) = Cipher <$> Gpg.pipeStrict decrypt t
 	where
 		decrypt = [ Param "--decrypt" ]
 
@@ -165,14 +162,6 @@ pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a)
       -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
 pass to c i a = to c i $ \h -> a =<< L.hGetContents h
 
-configKeyIds :: RemoteConfig -> IO KeyIds
-configKeyIds c = Gpg.findPubKeys $ configGet c "encryption"
-
-configGet :: RemoteConfig -> String -> String
-configGet c key = fromMaybe missing $ M.lookup key c
-	where
-		missing = error $ "missing " ++ key ++ " in remote config"
-
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
 hmacWithCipher' :: String -> String -> String
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index bcecb30cc4..a44e6e4539 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -17,17 +17,22 @@ import Config
 
 {- Encryption setup for a remote. The user must specify whether to use
  - an encryption key, or not encrypt. An encrypted cipher is created, or is
- - updated to be accessible to an additional encryption key. -}
+ - updated to be accessible to an additional encryption key. Or the user
+ - could opt to use a shared cipher, which is stored unencrypted. -}
 encryptionSetup :: RemoteConfig -> Annex RemoteConfig
-encryptionSetup c =
-	case (M.lookup "encryption" c, extractCipher c) of
-		(Nothing, Nothing) -> error "Specify encryption=key or encryption=none"
-		(Just "none", Nothing) -> return c
-		(Just "none", Just _) -> error "Cannot change encryption type of existing remote."
-		(Nothing, Just _) -> return c
-		(Just _, Nothing) -> use "encryption setup" $ genCipher c
-		(Just _, Just v) -> use "encryption updated" $ updateCipher c v
+encryptionSetup c = case (M.lookup "encryption" c, extractCipher c) of
+	(Nothing, Nothing) -> error "Specify encryption=key or encryption=none or encryption=shared"
+	(Just "none", Nothing) -> return c
+	(Nothing, Just _) -> return c
+	(Just "shared", Just (SharedCipher _)) -> return c
+	(Just "none", Just _) -> cannotchange
+	(Just "shared", Just (EncryptedCipher _ _)) -> cannotchange
+	(Just _, Just (SharedCipher _)) -> cannotchange
+	(Just "shared", Nothing) -> use "encryption setup" $ genSharedCipher
+	(Just keyid, Nothing) -> use "encryption setup" $ genEncryptedCipher keyid
+	(Just keyid, Just v) -> use "encryption updated" $ updateEncryptedCipher keyid v
 	where
+		cannotchange = error "Cannot change encryption type of existing remote."
 		use m a = do
 			cipher <- liftIO a
 			showNote $ m ++ " " ++ describeCipher cipher
@@ -78,7 +83,7 @@ remoteCipher c = go $ extractCipher c
 				Nothing -> decrypt encipher cache
 		decrypt encipher cache = do
 			showNote "gpg"
-			cipher <- liftIO $ decryptCipher c encipher
+			cipher <- liftIO $ decryptCipher encipher
 			Annex.changeState (\s -> s { Annex.ciphers = M.insert encipher cipher cache })
 			return $ Just cipher
 
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index 686bf5c1a6..135522ba11 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -1,13 +1,13 @@
 {- git-annex crypto types
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Types.Crypto (
 	Cipher(..),
-	EncryptedCipher(..),
+	StorableCipher(..),
 	KeyIds(..),
 ) where
 
@@ -16,5 +16,5 @@ import Utility.Gpg (KeyIds(..))
 -- XXX ideally, this would be a locked memory region
 newtype Cipher = Cipher String
 
-data EncryptedCipher = EncryptedCipher String KeyIds
+data StorableCipher = EncryptedCipher String KeyIds | SharedCipher String
 	deriving (Ord, Eq)
diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index 4c798f2732..ff6735ba57 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -94,7 +94,18 @@ findPubKeys for = KeyIds . parse <$> readStrict params
 		pubKey = isPrefixOf "pub:"
 		keyIdField s = split ":" s !! 4
 
-
+{- Creates a block of high-quality random data suitable to use as a cipher.
+ - It is armored, to avoid newlines, since gpg only reads ciphers up to the
+ - first newline. -}
+genRandom :: Int -> IO String
+genRandom size = readStrict
+	[ Params "--gen-random --armor"
+	, Param $ show randomquality
+	, Param $ show size
+	]
+	where
+		-- 1 is /dev/urandom; 2 is /dev/random
+		randomquality = 1 :: Int
 
 {- A test key. This is provided pre-generated since generating a new gpg
  - key is too much work (requires too much entropy) for a test suite to
diff --git a/debian/changelog b/debian/changelog
index 06990a0f8a..7a9b5c6eab 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -7,6 +7,9 @@ git-annex (3.20120419) UNRELEASED; urgency=low
     settings, to allow custom headers to be sent with all HTTP requests.
     (Requested by the Internet Archive)
   * uninit: Clear annex.uuid from .git/config. Closes: #670639
+  * Added shared cipher mode to encryptable special remotes. This option
+    avoids gpg key distribution, at the expense of flexability, and with
+    the requirement that all clones of the git repository be equally trusted.
 
  -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
 
diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn
index 0f83bb7f90..cc61fea6f7 100644
--- a/doc/encryption.mdwn
+++ b/doc/encryption.mdwn
@@ -33,3 +33,16 @@ Note that once a key has been given access to a remote, it's not
 possible to revoke that access, short of deleting the remote. See
 [[encryption_design|design/encryption]] for other security risks
 associated with encryption.
+
+## shared cipher mode
+
+Alternatively, you can configure git-annex to use a shared cipher to
+encrypt data stored in a remote. This shared cipher is stored,
+**unencrypted** in the git repository. So it's shared amoung every
+clone of the git repository. The advantage is you don't need to set up gpg
+keys. The disadvantage is that this is **insecure** unless you
+trust every clone of the git repository with access to the encrypted data
+stored in the special remote.
+
+To use shared encryption, specify "encryption=shared" when first setting
+up a special remote.

From 9f04367336965fcfc0b78aeacefe70970e77db19 Mon Sep 17 00:00:00 2001
From: "http://joey.kitenet.net/" 
Date: Sun, 29 Apr 2012 18:04:13 +0000
Subject: [PATCH 3435/8313] Added a comment

---
 .../comment_5_9668b58eb71901e1db8da7db38e068ca._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/wishlist:_simpler_gpg_usage/comment_5_9668b58eb71901e1db8da7db38e068ca._comment

diff --git a/doc/forum/wishlist:_simpler_gpg_usage/comment_5_9668b58eb71901e1db8da7db38e068ca._comment b/doc/forum/wishlist:_simpler_gpg_usage/comment_5_9668b58eb71901e1db8da7db38e068ca._comment
new file mode 100644
index 0000000000..60b98bde5d
--- /dev/null
+++ b/doc/forum/wishlist:_simpler_gpg_usage/comment_5_9668b58eb71901e1db8da7db38e068ca._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 5"
+ date="2012-04-29T18:04:13Z"
+ content="""
+encryption=shared is now supported
+"""]]

From bd592d1450e52a99e7507a211ad1c36414d3d869 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 29 Apr 2012 14:31:34 -0400
Subject: [PATCH 3436/8313] refactor

---
 Crypto.hs                    | 26 ++------------------------
 Remote/Helper/Encryptable.hs | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/Crypto.hs b/Crypto.hs
index e530bd0e65..58c0e6d008 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -10,13 +10,12 @@
 
 module Crypto (
 	Cipher,
+	KeyIds(..),
 	StorableCipher(..),
 	genEncryptedCipher,
 	genSharedCipher,
 	updateEncryptedCipher,
 	describeCipher,
-	storeCipher,
-	extractCipher,
 	decryptCipher,		
 	encryptKey,
 	withEncryptedHandle,
@@ -28,7 +27,6 @@ module Crypto (
 ) where
 
 import qualified Data.ByteString.Lazy.Char8 as L
-import qualified Data.Map as M
 import Data.ByteString.Lazy.UTF8 (fromString)
 import Data.Digest.Pure.SHA
 import Control.Applicative
@@ -36,8 +34,6 @@ import Control.Applicative
 import Common.Annex
 import qualified Utility.Gpg as Gpg
 import Types.Key
-import Types.Remote
-import Utility.Base64
 import Types.Crypto
 
 {- The first half of a Cipher is used for HMAC; the remainder
@@ -90,24 +86,6 @@ describeCipher (EncryptedCipher _ (KeyIds ks)) =
 		keys [_] = "key"
 		keys _ = "keys"
 
-{- Stores an StorableCipher in a remote's configuration. -}
-storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig
-storeCipher c (SharedCipher t) = M.insert "cipher" (toB64 t) c
-storeCipher c (EncryptedCipher t ks) = 
-	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c
-	where
-		showkeys (KeyIds l) = join "," l
-
-{- Extracts an StorableCipher from a remote's configuration. -}
-extractCipher :: RemoteConfig -> Maybe StorableCipher
-extractCipher c = 
-	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of
-		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks)
-		(Just t, Nothing) -> Just $ SharedCipher (fromB64 t)
-		_ -> Nothing
-	where
-		readkeys = KeyIds . split ","
-
 {- Encrypts a Cipher to the specified KeyIds. -}
 encryptCipher :: Cipher -> KeyIds -> IO StorableCipher
 encryptCipher (Cipher c) (KeyIds ks) = do
@@ -160,7 +138,7 @@ withDecryptedContent = pass withDecryptedHandle
 
 pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a) 
       -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
-pass to c i a = to c i $ \h -> a =<< L.hGetContents h
+pass to n s a = to n s $ \h -> a =<< L.hGetContents h
 
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index a44e6e4539..789a1d9964 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -14,6 +14,7 @@ import Types.Remote
 import Crypto
 import qualified Annex
 import Config
+import Utility.Base64
 
 {- Encryption setup for a remote. The user must specify whether to use
  - an encryption key, or not encrypt. An encrypted cipher is created, or is
@@ -93,3 +94,21 @@ cipherKey Nothing _ = return Nothing
 cipherKey (Just c) k = maybe Nothing encrypt <$> remoteCipher c
 	where
 		encrypt ciphertext = Just (ciphertext, encryptKey ciphertext k)
+
+{- Stores an StorableCipher in a remote's configuration. -}
+storeCipher :: RemoteConfig -> StorableCipher -> RemoteConfig
+storeCipher c (SharedCipher t) = M.insert "cipher" (toB64 t) c
+storeCipher c (EncryptedCipher t ks) = 
+	M.insert "cipher" (toB64 t) $ M.insert "cipherkeys" (showkeys ks) c
+	where
+		showkeys (KeyIds l) = join "," l
+
+{- Extracts an StorableCipher from a remote's configuration. -}
+extractCipher :: RemoteConfig -> Maybe StorableCipher
+extractCipher c = 
+	case (M.lookup "cipher" c, M.lookup "cipherkeys" c) of
+		(Just t, Just ks) -> Just $ EncryptedCipher (fromB64 t) (readkeys ks)
+		(Just t, Nothing) -> Just $ SharedCipher (fromB64 t)
+		_ -> Nothing
+	where
+		readkeys = KeyIds . split ","

From d2bfba6324ca54253be62716b1bbdc86e0e1aafe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 29 Apr 2012 16:10:47 -0400
Subject: [PATCH 3437/8313] show percent the bloom filter is full

---
 Command/Status.hs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Command/Status.hs b/Command/Status.hs
index 16bcec7cce..057ab72bfa 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -12,6 +12,7 @@ module Command.Status where
 import Control.Monad.State.Strict
 import qualified Data.Map as M
 import Text.JSON
+import Data.Ratio
 
 import Common.Annex
 import qualified Types.Backend as B
@@ -160,7 +161,7 @@ bloom_info = stat "bloom filter size" $ json id $ do
 	let note = aside $
 		if localkeys >= capacity
 		then "appears too small for this repository; adjust annex.bloomcapacity"
-		else "has room for " ++ show (capacity - localkeys) ++ " more local annex keys"
+		else show (floor (percentage capacity localkeys) :: Integer) ++ "% full"
 
 	-- Two bloom filters are used at the same time, so double the size
 	-- of one.
@@ -169,6 +170,10 @@ bloom_info = stat "bloom filter size" $ json id $ do
 
 	return $ size ++ note
 
+	where
+		percentage :: Integer -> Integer -> Double
+		percentage full have = 100 * (fromRational $ have % full)
+
 disk_size :: Stat
 disk_size = stat "available local disk space" $ json id $ lift $
 	calcfree

From 0c9c14b52fcdd9684da70c8dd763661acdbd3843 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 29 Apr 2012 17:48:07 -0400
Subject: [PATCH 3438/8313] percentage library

---
 Command/Status.hs     |  8 ++------
 Utility/Percentage.hs | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 6 deletions(-)
 create mode 100644 Utility/Percentage.hs

diff --git a/Command/Status.hs b/Command/Status.hs
index 057ab72bfa..2540a92da8 100644
--- a/Command/Status.hs
+++ b/Command/Status.hs
@@ -12,7 +12,6 @@ module Command.Status where
 import Control.Monad.State.Strict
 import qualified Data.Map as M
 import Text.JSON
-import Data.Ratio
 
 import Common.Annex
 import qualified Types.Backend as B
@@ -31,6 +30,7 @@ import Logs.UUID
 import Logs.Trust
 import Remote
 import Config
+import Utility.Percentage
 
 -- a named computation that produces a statistic
 type Stat = StatState (Maybe (String, StatState String))
@@ -161,7 +161,7 @@ bloom_info = stat "bloom filter size" $ json id $ do
 	let note = aside $
 		if localkeys >= capacity
 		then "appears too small for this repository; adjust annex.bloomcapacity"
-		else show (floor (percentage capacity localkeys) :: Integer) ++ "% full"
+		else showPercentage 1 (percentage capacity localkeys) ++ " full"
 
 	-- Two bloom filters are used at the same time, so double the size
 	-- of one.
@@ -170,10 +170,6 @@ bloom_info = stat "bloom filter size" $ json id $ do
 
 	return $ size ++ note
 
-	where
-		percentage :: Integer -> Integer -> Double
-		percentage full have = 100 * (fromRational $ have % full)
-
 disk_size :: Stat
 disk_size = stat "available local disk space" $ json id $ lift $
 	calcfree
diff --git a/Utility/Percentage.hs b/Utility/Percentage.hs
new file mode 100644
index 0000000000..309e00181d
--- /dev/null
+++ b/Utility/Percentage.hs
@@ -0,0 +1,38 @@
+{- percentages
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Percentage (
+	Percentage,
+	percentage,
+	showPercentage
+) where
+
+import Data.Ratio
+
+newtype Percentage = Percentage (Ratio Integer)
+
+instance Show Percentage where
+	show = showPercentage 0
+
+{- Normally the big number comes first. But 110% is allowed if desired. :) -}
+percentage :: Integer -> Integer -> Percentage
+percentage 0 _ = Percentage 0
+percentage full have = Percentage $ have * 100 % full
+
+{- Pretty-print a Percentage, with a specified level of precision. -}
+showPercentage :: Int -> Percentage -> String
+showPercentage precision (Percentage p)
+	| precision == 0 || remainder == 0 = go $ show int
+	| otherwise = go $ show int ++ "." ++ strip0s (show remainder)
+	where
+		go v = v ++ "%"
+		int :: Integer
+		(int, frac) = properFraction (fromRational p)
+		remainder = floor (frac * multiplier) :: Integer
+		strip0s = reverse . dropWhile (== '0') . reverse
+		multiplier :: Float
+		multiplier = 10 ** (fromIntegral precision)

From bf70bbdbfc875836cf880ba8a03cac74948a30a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 30 Apr 2012 13:59:05 -0400
Subject: [PATCH 3439/8313] fix test suite build

---
 Annex.hs  | 6 ++++++
 Option.hs | 8 +++-----
 test.hs   | 9 +++++----
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index f2fb1f4019..d1509d4bd2 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -18,6 +18,7 @@ module Annex (
 	changeState,
 	setFlag,
 	setField,
+	setOutput,
 	getFlag,
 	getField,
 	addCleanup,
@@ -148,6 +149,11 @@ addCleanup :: String -> Annex () -> Annex ()
 addCleanup uid a = changeState $ \s ->
 	s { cleanup = M.insertWith' const uid a $ cleanup s }
 
+{- Sets the type of output to emit. -}
+setOutput :: OutputType -> Annex ()
+setOutput o = changeState $ \s ->
+	s { output = (output s) { outputType = o } }
+
 {- Checks if a flag was set. -}
 getFlag :: String -> Annex Bool
 getFlag flag = fromMaybe False . M.lookup flag <$> getState flags
diff --git a/Option.hs b/Option.hs
index 7696220554..1bac2cd050 100644
--- a/Option.hs
+++ b/Option.hs
@@ -32,11 +32,11 @@ common =
 		"avoid slow operations"
 	, Option ['a'] ["auto"] (NoArg (setauto True))
 		"automatic mode"
-	, Option ['q'] ["quiet"] (NoArg (setoutput QuietOutput))
+	, Option ['q'] ["quiet"] (NoArg (Annex.setOutput QuietOutput))
 		"avoid verbose output"
-	, Option ['v'] ["verbose"] (NoArg (setoutput NormalOutput))
+	, Option ['v'] ["verbose"] (NoArg (Annex.setOutput NormalOutput))
 		"allow verbose output (default)"
-	, Option ['j'] ["json"] (NoArg (setoutput JSONOutput))
+	, Option ['j'] ["json"] (NoArg (Annex.setOutput JSONOutput))
 		"enable JSON output"
 	, Option ['d'] ["debug"] (NoArg setdebug)
 		"show debug messages"
@@ -47,8 +47,6 @@ common =
 		setforce v = Annex.changeState $ \s -> s { Annex.force = v }
 		setfast v = Annex.changeState $ \s -> s { Annex.fast = v }
 		setauto v = Annex.changeState $ \s -> s { Annex.auto = v }
-		setoutput v = Annex.changeState $ \s ->
-			s { Annex.output = (Annex.output s) { outputType = v } }
 		setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v }
 		setdebug = liftIO $ updateGlobalLogger rootLoggerName $
 			setLevel DEBUG
diff --git a/test.hs b/test.hs
index 5f4b0ad40e..8c24c3a23d 100644
--- a/test.hs
+++ b/test.hs
@@ -38,6 +38,7 @@ import qualified Logs.Remote
 import qualified Remote
 import qualified Command.DropUnused
 import qualified Types.Key
+import qualified Types.Messages
 import qualified Config
 import qualified Crypto
 import qualified Utility.Path
@@ -720,10 +721,10 @@ git_annex_expectoutput command params expected = do
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a
 annexeval a = do
-	g <- Git.Construct.fromCurrent
-	g' <- Git.Config.read g
-	s <- Annex.new g'
-	Annex.eval s { Annex.output = Annex.QuietOutput } a
+	s <- Annex.new =<< Git.Config.read =<< Git.Construct.fromCurrent
+	Annex.eval s $ do
+		Annex.setOutput Types.Messages.QuietOutput
+		a
 
 innewrepo :: Assertion -> Assertion
 innewrepo a = withgitrepo $ \r -> indir r a

From 76b80d6af08f3e758e2180813392860578ff9756 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 30 Apr 2012 13:59:28 -0400
Subject: [PATCH 3440/8313] releasing version 3.20120430

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7a9b5c6eab..27a6bab2b8 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120419) UNRELEASED; urgency=low
+git-annex (3.20120430) unstable; urgency=low
 
   * Fix use of annex.diskreserve config setting.
   * Directory special remotes now check annex.diskreserve.
@@ -11,7 +11,7 @@ git-annex (3.20120419) UNRELEASED; urgency=low
     avoids gpg key distribution, at the expense of flexability, and with
     the requirement that all clones of the git repository be equally trusted.
 
- -- Joey Hess   Fri, 20 Apr 2012 16:14:08 -0400
+ -- Joey Hess   Mon, 30 Apr 2012 13:16:10 -0400
 
 git-annex (3.20120418) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index da0e9f4de7..6299944698 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120418
+Version: 3.20120430
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From f377cbc12908c173ba1fde782cae3aa80b9dffef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 30 Apr 2012 14:10:10 -0400
Subject: [PATCH 3441/8313] update

---
 .gitignore | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index fe65fc5747..b6b8d606d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,12 +1,8 @@
 tmp
-*.hi
-*.o
 test
 configure
 Build/SysConfig.hs
 git-annex
-git-annex-shell
-git-union-merge
 git-annex.1
 git-annex-shell.1
 git-union-merge.1
@@ -15,5 +11,5 @@ html
 *.tix
 .hpc
 Utility/Touch.hs
-Utility/StatFS.hs
+Utility/libdiskfree.o
 dist

From 1f5965fa0d5a869568aa4579bb996b6b04a236f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 30 Apr 2012 14:34:08 -0400
Subject: [PATCH 3442/8313] add news item for git-annex 3.20120430

---
 doc/news/version_3.20120309.mdwn |  5 -----
 doc/news/version_3.20120430.mdwn | 12 ++++++++++++
 2 files changed, 12 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_3.20120309.mdwn
 create mode 100644 doc/news/version_3.20120430.mdwn

diff --git a/doc/news/version_3.20120309.mdwn b/doc/news/version_3.20120309.mdwn
deleted file mode 100644
index 869b96ccec..0000000000
--- a/doc/news/version_3.20120309.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 3.20120309 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix key directory hash calculation code to behave as it did before
-     version 3.20120227 when a key contains non-ascii characters (only
-     WORM backend is likely to have been affected)."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120430.mdwn b/doc/news/version_3.20120430.mdwn
new file mode 100644
index 0000000000..07e1e85ad2
--- /dev/null
+++ b/doc/news/version_3.20120430.mdwn
@@ -0,0 +1,12 @@
+git-annex 3.20120430 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Fix use of annex.diskreserve config setting.
+   * Directory special remotes now check annex.diskreserve.
+   * Support git's core.sharedRepository configuration.
+   * Add annex.http-headers and annex.http-headers-command config
+     settings, to allow custom headers to be sent with all HTTP requests.
+     (Requested by the Internet Archive)
+   * uninit: Clear annex.uuid from .git/config. Closes: #[670639](http://bugs.debian.org/670639)
+   * Added shared cipher mode to encryptable special remotes. This option
+     avoids gpg key distribution, at the expense of flexability, and with
+     the requirement that all clones of the git repository be equally trusted."""]]
\ No newline at end of file

From 153912633e3aa1f0f4e5413e1786ea9b5770715f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Wed, 2 May 2012 00:27:52 +0000
Subject: [PATCH 3443/8313]

---
 doc/forum/Windows_support.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/forum/Windows_support.mdwn

diff --git a/doc/forum/Windows_support.mdwn b/doc/forum/Windows_support.mdwn
new file mode 100644
index 0000000000..0e9e8dcb6e
--- /dev/null
+++ b/doc/forum/Windows_support.mdwn
@@ -0,0 +1,6 @@
+Hi,
+
+Do you have any news about Windows support?
+Is this something you're currently working on?
+
+Thanks!

From 8f8f7ecf6fed3afa189225d961ba6c232745be93 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 1 May 2012 20:34:28 -0400
Subject: [PATCH 3444/8313] recycled comment

---
 .../comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment

diff --git a/doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment b/doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
new file mode 100644
index 0000000000..95323ff997
--- /dev/null
+++ b/doc/forum/Windows_support/comment_1_23fa9aa3b00940a1c1b3876c35eef019._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joey.kitenet.net/"
+ nickname="joey"
+ subject="comment 1"
+ date="2012-03-12T06:43:02Z"
+ content="""
+[[todo/windows_support]] has everything I know about making a windows port. This badly needs someone who understand Windows to dive into it. The question of how to create a symbolic link (or the relevant Windows equivilant) from haskell on Windows
+is a good starting point..
+"""]]

From 834e73cde5f1669384641a8df3b8c542e240db29 Mon Sep 17 00:00:00 2001
From: "http://id.wgnr.me/" 
Date: Wed, 2 May 2012 14:09:28 +0000
Subject: [PATCH 3445/8313]

---
 ...___versions_3.20120315_and_3.20120430.mdwn | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn

diff --git a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
new file mode 100644
index 0000000000..d8c15ed260
--- /dev/null
+++ b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
@@ -0,0 +1,49 @@
+What steps will reproduce the problem?
+
+	$ git annex initremote rsyncremote type=rsync rsyncurl=myuser@rsync.hidrive.strato.com:/users/myuser/git-annex/Music/ encryption=0xC597DECC177AFD7C
+	$ git annex get --from rsyncremote "file"
+
+What is the expected output? What do you see instead?
+
+I expect that the requested file is copied as for every other remote, but instead I get this error:
+
+----------------------------------------
+	get  (from rsyncremote...) (gpg) 
+	rsync: change_dir "/users/myuser/git-annex/Music/0e5/a5b/'GPGHMACSHA1--3afd32ab8e70ac329262adeb770c330b0845b1e0" failed: No such file or directory (2)
+
+	sent 8 bytes  received 10 bytes  7.20 bytes/sec
+	total size is 0  speedup is 0.00
+	rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1518) [Receiver=3.0.9]
+
+	  rsync failed -- run git annex again to resume file transfer
+
+	rsync: change_dir "/users/myuser/git-annex/Music/8k/QZ/'GPGHMACSHA1--3afd32ab8e70ac329262adeb770c330b0845b1e0" failed: No such file or directory (2)
+
+	sent 8 bytes  received 10 bytes  36.00 bytes/sec
+	total size is 0  speedup is 0.00
+	rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1518) [Receiver=3.0.9]
+
+	  rsync failed -- run git annex again to resume file transfer
+failed
+	git-annex: get: 1 failed
+----------------------------------------
+
+I can verify that the directory /users/myuser/git-annex/Music/0e5/a5b/GPGHMACSHA1--3afd32ab8e70ac329262adeb770c330b0845b1e0 exists in the rsync remote, without the ' character.
+
+What version of git-annex are you using? On what operating system?
+
+I tried versions 3.20120315 and 3.20120430 on Gentoo linux.
+
+	$ uname -a
+	Linux odin 3.3.1-gentoo-odin #1 SMP Sat Apr 7 21:18:11 CEST 2012 x86_64 Intel(R) Core(TM) i5 CPU M 560 @ 2.67GHz GenuineIntel GNU/Linux
+
+	$ ghc --version
+	The Glorious Glasgow Haskell Compilation System, version 7.4.1
+
+Please provide any additional information below.
+
+The rsync remote config in .git/config:
+
+	[remote "rsyncremote"]
+		annex-rsyncurl = myuser@rsync.hidrive.strato.com:/users/myuser/git-annex/Music/
+		annex-uuid = "UUID"

From 5337c4e0c4e9f7a803a77532057a20f7c54fe28a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 11:38:27 -0400
Subject: [PATCH 3446/8313] rsync protocol?

---
 Utility/RsyncFile.hs                                |  2 +-
 ...te__44___versions_3.20120315_and_3.20120430.mdwn | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index a691d0a0e6..fae23d4ceb 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -58,7 +58,7 @@ rsyncUrlIsShell s
 	| "rsync://" `isPrefixOf` s = False
 	| otherwise = go s
 	where
-		-- host:dir is rsync protocol, while host:dir is ssh/rsh
+		-- host:dir is rsync protocol, while host/dir is ssh/rsh
 		go [] = False
 		go (c:cs)
 			| c == '/' = False -- got to directory with no colon
diff --git a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
index d8c15ed260..ad97a82b2c 100644
--- a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
+++ b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
@@ -47,3 +47,16 @@ The rsync remote config in .git/config:
 	[remote "rsyncremote"]
 		annex-rsyncurl = myuser@rsync.hidrive.strato.com:/users/myuser/git-annex/Music/
 		annex-uuid = "UUID"
+
+> Here's what the --debug flag shows is being run:
+
+	Running: rsync ["--progress","--inplace","joey@localhost:/tmp/Music/d98/a3c/'GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c/GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c'","/home/joey/tmp/x/.git/annex/tmp/GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c"] 
+
+> But, this works for me, here, despite containing the quoting!
+> That's because here it's using rsync over ssh, which actually requires
+> that quoting. Are you using rsync
+> over the rsync protocol? If so, the workaround is to explicitly make
+> the rsyncurl start with `rsync://` -- and if this is the case, I need
+> to adjust the code in git-annex that determines if it's using ssh or
+> the rsync protocol.
+>  --[[Joey]]

From 913775410bb1b30116c3550c43389747c3b0cf14 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 11:43:30 -0400
Subject: [PATCH 3447/8313] update

---
 Utility/RsyncFile.hs                                      | 2 +-
 ...c_remote__44___versions_3.20120315_and_3.20120430.mdwn | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs
index fae23d4ceb..db9057843c 100644
--- a/Utility/RsyncFile.hs
+++ b/Utility/RsyncFile.hs
@@ -58,7 +58,7 @@ rsyncUrlIsShell s
 	| "rsync://" `isPrefixOf` s = False
 	| otherwise = go s
 	where
-		-- host:dir is rsync protocol, while host/dir is ssh/rsh
+		-- host::dir is rsync protocol, while host:dir is ssh/rsh
 		go [] = False
 		go (c:cs)
 			| c == '/' = False -- got to directory with no colon
diff --git a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
index ad97a82b2c..c57c645a01 100644
--- a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
+++ b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
@@ -56,7 +56,11 @@ The rsync remote config in .git/config:
 > That's because here it's using rsync over ssh, which actually requires
 > that quoting. Are you using rsync
 > over the rsync protocol? If so, the workaround is to explicitly make
-> the rsyncurl start with `rsync://` -- and if this is the case, I need
+> the rsyncurl start with `rsync://`
+> 
+> And if this is the case, I need
 > to adjust the code in git-annex that determines if it's using ssh or
-> the rsync protocol.
+> the rsync protocol. It assumes that (and this is what the rsync man 
+> says AFAICS) that the rsync protocol is only used if the url starts
+> with `rsync://` or contains `::`.
 >  --[[Joey]]

From 17fd57bd818931532eda1892f23e0998ad0110d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 11:50:57 -0400
Subject: [PATCH 3448/8313] hidrive.strato.com

---
 ...emote__44___versions_3.20120315_and_3.20120430.mdwn | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
index c57c645a01..ab5eb78435 100644
--- a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
+++ b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
@@ -48,7 +48,7 @@ The rsync remote config in .git/config:
 		annex-rsyncurl = myuser@rsync.hidrive.strato.com:/users/myuser/git-annex/Music/
 		annex-uuid = "UUID"
 
-> Here's what the --debug flag shows is being run:
+> Here's what the --debug flag shows is being run: --[[Joey]] 
 
 	Running: rsync ["--progress","--inplace","joey@localhost:/tmp/Music/d98/a3c/'GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c/GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c'","/home/joey/tmp/x/.git/annex/tmp/GPGHMACSHA1--878c3a3f59965bd87b4738ab29562efd215b954c"] 
 
@@ -63,4 +63,12 @@ The rsync remote config in .git/config:
 > the rsync protocol. It assumes that (and this is what the rsync man 
 > says AFAICS) that the rsync protocol is only used if the url starts
 > with `rsync://` or contains `::`.
+> 
+> Hmm, I see that `hidrive.strato.com` is some kind of rsync provider?
+> Perhaps they do something with rsync over ssh that
+> avoids the need for shell quoting. For example, they might pass incoming
+> ssh connections directly into rsync, bypassing the shell
+> -- which avoids the need for this quoting. Any details you can provide
+> about them would probably be useful then. Ie, do they really use rsync
+> over ssh, is it really a `rsync.net` type rsync provider? 
 >  --[[Joey]]

From 6d61067599142af3eb520527589b0adfbf45867d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 13:08:31 -0400
Subject: [PATCH 3449/8313] rsync shellescape disable option

Rsync special remotes can be configured with shellescape=no to avoid shell
quoting that is normally done when using rsync over ssh. This is known to
be needed for certian rsync hosting providers (specificially
hidrive.strato.com) that use rsync over ssh but do not pass it through the
shell.
---
 Remote/Rsync.hs                | 21 ++++++++++++---------
 debian/changelog               | 10 ++++++++++
 doc/special_remotes/rsync.mdwn |  8 ++++++++
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 571cd8f5ee..60cbf4595f 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -22,9 +22,10 @@ import Utility.RsyncFile
 
 type RsyncUrl = String
 
-data RsyncOpts = RsyncOpts {
-	rsyncUrl :: RsyncUrl,
-	rsyncOptions :: [CommandParam]
+data RsyncOpts = RsyncOpts
+	{ rsyncUrl :: RsyncUrl
+	, rsyncOptions :: [CommandParam]
+	, rsyncShellEscape :: Bool
 }
 
 remote :: RemoteType
@@ -37,7 +38,7 @@ remote = RemoteType {
 
 gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote
 gen r u c = do
-	o <- genRsyncOpts r
+	o <- genRsyncOpts r c
 	cst <- remoteCost r expensiveRemoteCost
 	return $ encryptableRemote c
 		(storeEncrypted o)
@@ -58,11 +59,13 @@ gen r u c = do
 			remotetype = remote
 		}
 
-genRsyncOpts :: Git.Repo -> Annex RsyncOpts
-genRsyncOpts r = do
+genRsyncOpts :: Git.Repo -> Maybe RemoteConfig -> Annex RsyncOpts
+genRsyncOpts r c = do
 	url <- getRemoteConfig r "rsyncurl" (error "missing rsyncurl")
-	opts <- getRemoteConfig r "rsync-options" ""
-	return $ RsyncOpts url $ map Param $ filter safe $ words opts
+	opts <- map Param . filter safe . words
+		<$> getRemoteConfig r "rsync-options" ""
+	let escape = maybe True (\m -> M.lookup "shellescape" m /= Just "no") c
+	return $ RsyncOpts url opts escape
 	where
 		safe o
 			-- Don't allow user to pass --delete to rsync;
@@ -86,7 +89,7 @@ rsyncSetup u c = do
 
 rsyncEscape :: RsyncOpts -> String -> String
 rsyncEscape o s
-	| rsyncUrlIsShell (rsyncUrl o) = shellEscape s
+	| rsyncShellEscape o && rsyncUrlIsShell (rsyncUrl o) = shellEscape s
 	| otherwise = s
 
 rsyncUrls :: RsyncOpts -> Key -> [String]
diff --git a/debian/changelog b/debian/changelog
index 27a6bab2b8..033b739de5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,13 @@
+git-annex (3.20120431) UNRELEASED; urgency=low
+
+  * Rsync special remotes can be configured with shellescape=no
+    to avoid shell quoting that is normally done when using rsync over ssh.
+    This is known to be needed for certian rsync hosting providers
+    (specificially hidrive.strato.com) that use rsync over ssh but do not
+    pass it through the shell.
+
+ -- Joey Hess   Wed, 02 May 2012 13:06:18 -0400
+
 git-annex (3.20120430) unstable; urgency=low
 
   * Fix use of annex.diskreserve config setting.
diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn
index 90d544a1e1..286615460c 100644
--- a/doc/special_remotes/rsync.mdwn
+++ b/doc/special_remotes/rsync.mdwn
@@ -24,5 +24,13 @@ These parameters can be passed to `git annex initremote` to configure rsync:
 * `rsyncurl` - Required. This is the url or `hostname:/directory` to 
   pass to rsync to tell it where to store content.
 
+* `shellescape` - Optional. Set to "no" to avoid shell escaping normally
+  done when using rsync over ssh. That escaping is needed with typical
+  setups, but not with some hosting providers that do not expose rsynced
+  filenames to the shell. You'll know you need this option if `git annex get`
+  from the special remote fails with an error message containing a single
+  quote (`'`) character. If that happens, you can re-run initremote
+  setting shellescape=no.
+
 The `annex-rsync-options` git configuration setting can be used to pass
 parameters to rsync.

From 8f453004791217e23268728129c197e49c19f67b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 13:15:19 -0400
Subject: [PATCH 3450/8313] dropunused: Allow specifying ranges to drop.

Sort of by popular demand, but the last straw for not using seq
was that it can run into command line length limits.
---
 Command/DropUnused.hs | 40 +++++++++++++++++++++++++++-------------
 debian/changelog      |  1 +
 doc/git-annex.mdwn    |  2 ++
 3 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 0b2a602161..9c9513ca9d 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -18,7 +18,7 @@ import qualified Git
 import qualified Option
 import Types.Key
 
-type UnusedMap = M.Map String Key
+type UnusedMap = M.Map Integer Key
 
 def :: [Command]
 def = [withOptions [Command.Drop.fromOption] $
@@ -34,10 +34,20 @@ withUnusedMaps params = do
 	unused <- readUnusedLog ""
 	unusedbad <- readUnusedLog "bad"
 	unusedtmp <- readUnusedLog "tmp"
-	return $ map (start (unused, unusedbad, unusedtmp)) params
+	return $ map (start (unused, unusedbad, unusedtmp)) $
+		concatMap unusedSpec params
 
-start :: (UnusedMap, UnusedMap, UnusedMap) -> FilePath -> CommandStart
-start (unused, unusedbad, unusedtmp) s = search
+unusedSpec :: String -> [Integer]
+unusedSpec spec
+	| "-" `isInfixOf` spec = range $ separate (== '-') spec
+	| otherwise = catMaybes [readish spec]
+	where
+		range (a, b) = case (readish a, readish b) of
+			(Just x, Just y) -> [x..y]
+			_ -> []
+
+start :: (UnusedMap, UnusedMap, UnusedMap) -> Integer -> CommandStart
+start (unused, unusedbad, unusedtmp) n = search
 	[ (unused, perform)
 	, (unusedbad, performOther gitAnnexBadLocation)
 	, (unusedtmp, performOther gitAnnexTmpLocation)
@@ -45,10 +55,10 @@ start (unused, unusedbad, unusedtmp) s = search
 	where
 		search [] = stop
 		search ((m, a):rest) =
-			case M.lookup s m of
+			case M.lookup n m of
 				Nothing -> search rest
 				Just key -> do
-					showStart "dropunused" s
+					showStart "dropunused" (show n)
 					next $ a key
 
 perform :: Key -> CommandPerform
@@ -70,11 +80,15 @@ performOther filespec key = do
 readUnusedLog :: FilePath -> Annex UnusedMap
 readUnusedLog prefix = do
 	f <- fromRepo $ gitAnnexUnusedLog prefix
-	e <- liftIO $ doesFileExist f
-	if e
-		then M.fromList . map parse . lines <$> liftIO (readFile f)
-		else return M.empty
+	ifM (liftIO $ doesFileExist f)
+		( M.fromList . catMaybes . map parse . lines
+			<$> liftIO (readFile f)
+		, return M.empty
+		)
 	where
-		parse line = (num, fromJust $ readKey rest)
+		parse line =
+			case (readish tag, readKey rest) of
+				(Just num, Just key) -> Just (num, key)
+				_ -> Nothing
 			where
-				(num, rest) = separate (== ' ') line
+				(tag, rest) = separate (== ' ') line
diff --git a/debian/changelog b/debian/changelog
index 033b739de5..72bce85511 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -5,6 +5,7 @@ git-annex (3.20120431) UNRELEASED; urgency=low
     This is known to be needed for certian rsync hosting providers
     (specificially hidrive.strato.com) that use rsync over ssh but do not
     pass it through the shell.
+  * dropunused: Allow specifying ranges to drop.
 
  -- Joey Hess   Wed, 02 May 2012 13:06:18 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 098d520010..998e1fa266 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -240,6 +240,8 @@ subdirectories).
   Drops the data corresponding to the numbers, as listed by the last
   `git annex unused`
 
+  You can also specify ranges of numbers, such as "1-1000".
+
   To drop the data from a remote, specify --from.
 
 * merge

From c058cdf46165af3ede7f09155ce99dc2bde9a17f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 13:17:51 -0400
Subject: [PATCH 3451/8313] move

---
 doc/{ => install}/NixOS.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
 rename doc/{ => install}/NixOS.mdwn (67%)

diff --git a/doc/NixOS.mdwn b/doc/install/NixOS.mdwn
similarity index 67%
rename from doc/NixOS.mdwn
rename to doc/install/NixOS.mdwn
index 864184a23c..115f9fa532 100644
--- a/doc/NixOS.mdwn
+++ b/doc/install/NixOS.mdwn
@@ -2,4 +2,5 @@ Users of the [Nix package manager](http://nixos.org/) can install it by running:
 
     nix-env -i git-annex
 
-The build status of the package within Nix can be seen on the [Hydra Build Farm](http://hydra.nixos.org/job/nixpkgs/trunk/gitAndTools.gitAnnex).
+The build status of the package within Nix can be seen on the [Hydra Build
+Farm](http://hydra.nixos.org/job/nixpkgs/trunk/gitAndTools.gitAnnex).

From ea0c48bfcd1d376a2f25573cca8c7876671dfd2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 13:18:21 -0400
Subject: [PATCH 3452/8313] gentoo

---
 doc/install.mdwn        | 1 +
 doc/install/Gentoo.mdwn | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 doc/install/Gentoo.mdwn

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 2590060e69..fe0522aa05 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -8,6 +8,7 @@
 * [[openSUSE]]
 * [[ArchLinux]]
 * [[NixOS]]
+* [[Gentoo]]
 * Windows: [[sorry, not possible yet|todo/windows_support]]
 
 ## Using cabal
diff --git a/doc/install/Gentoo.mdwn b/doc/install/Gentoo.mdwn
new file mode 100644
index 0000000000..b2061ad7a1
--- /dev/null
+++ b/doc/install/Gentoo.mdwn
@@ -0,0 +1,3 @@
+Gentoo users can:
+
+	`emerge git-annex`

From d8606a327d5b7dc34708405634baab1aade3c96c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 13:20:21 -0400
Subject: [PATCH 3453/8313] portage overlay

---
 doc/install/Gentoo.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/install/Gentoo.mdwn b/doc/install/Gentoo.mdwn
index b2061ad7a1..c508309c6e 100644
--- a/doc/install/Gentoo.mdwn
+++ b/doc/install/Gentoo.mdwn
@@ -1,3 +1,5 @@
 Gentoo users can:
 
 	`emerge git-annex`
+
+A possibly more up-to-date version is in the haskell portage overlay.

From 7d6b36dffbb11837a6fcfea3317b7d24ccbeeff7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 13:22:04 -0400
Subject: [PATCH 3454/8313] deseq

---
 ...dd_range_argument_to___34__git_annex_dropunused__34___.mdwn | 3 +++
 doc/walkthrough/unused_data.mdwn                               | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn
index bbe6007a87..471a698a0e 100644
--- a/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn
+++ b/doc/bugs/add_range_argument_to___34__git_annex_dropunused__34___.mdwn
@@ -16,3 +16,6 @@ I work around this lack as I want to drop all unused files anyway by something l
 
 > I don't see adding my own range operations to be an improvement worth
 > making; it'd arguably only be a complication. --[[Joey]] [[done]]
+
+>> Actually, this did get implemented, since using seq could fall afoul 
+>> of command-line length limits in extreme cases.
diff --git a/doc/walkthrough/unused_data.mdwn b/doc/walkthrough/unused_data.mdwn
index 518550ac02..63fb9f66d9 100644
--- a/doc/walkthrough/unused_data.mdwn
+++ b/doc/walkthrough/unused_data.mdwn
@@ -27,4 +27,4 @@ data anymore, you can easily remove it:
 
 Hint: To drop a lot of unused data, use a command like this:
 
-	# git annex dropunused `seq 1 1000`
+	# git annex dropunused 1-1000

From 392931eca9191117ae5c9d479fabab1e8ecaf8df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 14:59:05 -0400
Subject: [PATCH 3455/8313] addunused: New command, the opposite of dropunused,
 it relinks unused content into the git repository.

---
 Command/AddUnused.hs  | 34 ++++++++++++++++
 Command/DropUnused.hs | 60 +++-------------------------
 Command/Unused.hs     | 10 +----
 GitAnnex.hs           |  2 +
 Logs/Unused.hs        | 91 +++++++++++++++++++++++++++++++++++++++++++
 Usage.hs              |  2 +
 debian/changelog      |  2 +
 doc/git-annex.mdwn    |  8 +++-
 8 files changed, 145 insertions(+), 64 deletions(-)
 create mode 100644 Command/AddUnused.hs
 create mode 100644 Logs/Unused.hs

diff --git a/Command/AddUnused.hs b/Command/AddUnused.hs
new file mode 100644
index 0000000000..c498216dc3
--- /dev/null
+++ b/Command/AddUnused.hs
@@ -0,0 +1,34 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.AddUnused where
+
+import Common.Annex
+import Logs.Unused
+import Command
+import qualified Command.Add
+
+def :: [Command]
+def = [command "addunused" (paramRepeating paramNumRange)
+	seek "add back unused files"]
+
+seek :: [CommandSeek]
+seek = [withUnusedMaps start]
+
+start :: UnusedMaps -> Int -> CommandStart
+start = startUnused "addunused" perform (performOther "bad") (performOther "tmp")
+
+perform :: Key -> CommandPerform
+perform key = next $ Command.Add.cleanup file key True
+	where
+		file = "unused." ++ show key
+
+{- The content is not in the annex, but in another directory, and
+ - it seems better to error out, rather than moving bad/tmp content into
+ - the annex. -}
+performOther :: String -> Key -> CommandPerform
+performOther other _ = error $ "cannot addunused " ++ other ++ "content"
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index 9c9513ca9d..a94c2873dd 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -7,8 +7,7 @@
 
 module Command.DropUnused where
 
-import qualified Data.Map as M
-
+import Logs.Unused
 import Common.Annex
 import Command
 import qualified Annex
@@ -16,50 +15,17 @@ import qualified Command.Drop
 import qualified Remote
 import qualified Git
 import qualified Option
-import Types.Key
-
-type UnusedMap = M.Map Integer Key
 
 def :: [Command]
 def = [withOptions [Command.Drop.fromOption] $
-	command "dropunused" (paramRepeating paramNumber)
+	command "dropunused" (paramRepeating paramNumRange)
 		seek "drop unused file content"]
 
 seek :: [CommandSeek]
-seek = [withUnusedMaps]
+seek = [withUnusedMaps start]
 
-{- Read unused logs once, and pass the maps to each start action. -}
-withUnusedMaps :: CommandSeek
-withUnusedMaps params = do
-	unused <- readUnusedLog ""
-	unusedbad <- readUnusedLog "bad"
-	unusedtmp <- readUnusedLog "tmp"
-	return $ map (start (unused, unusedbad, unusedtmp)) $
-		concatMap unusedSpec params
-
-unusedSpec :: String -> [Integer]
-unusedSpec spec
-	| "-" `isInfixOf` spec = range $ separate (== '-') spec
-	| otherwise = catMaybes [readish spec]
-	where
-		range (a, b) = case (readish a, readish b) of
-			(Just x, Just y) -> [x..y]
-			_ -> []
-
-start :: (UnusedMap, UnusedMap, UnusedMap) -> Integer -> CommandStart
-start (unused, unusedbad, unusedtmp) n = search
-	[ (unused, perform)
-	, (unusedbad, performOther gitAnnexBadLocation)
-	, (unusedtmp, performOther gitAnnexTmpLocation)
-	]
-	where
-		search [] = stop
-		search ((m, a):rest) =
-			case M.lookup n m of
-				Nothing -> search rest
-				Just key -> do
-					showStart "dropunused" (show n)
-					next $ a key
+start :: UnusedMaps -> Int -> CommandStart
+start = startUnused "dropunused" perform (performOther gitAnnexBadLocation) (performOther gitAnnexTmpLocation)
 
 perform :: Key -> CommandPerform
 perform key = maybe droplocal dropremote =<< Remote.byName =<< from
@@ -76,19 +42,3 @@ performOther filespec key = do
 	f <- fromRepo $ filespec key
 	liftIO $ whenM (doesFileExist f) $ removeFile f
 	next $ return True
-
-readUnusedLog :: FilePath -> Annex UnusedMap
-readUnusedLog prefix = do
-	f <- fromRepo $ gitAnnexUnusedLog prefix
-	ifM (liftIO $ doesFileExist f)
-		( M.fromList . catMaybes . map parse . lines
-			<$> liftIO (readFile f)
-		, return M.empty
-		)
-	where
-		parse line =
-			case (readish tag, readKey rest) of
-				(Just num, Just key) -> Just (num, key)
-				_ -> Nothing
-			where
-				(tag, rest) = separate (== ' ') line
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 5bdadcf44a..6b319ee72c 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -19,9 +19,9 @@ import Control.Monad.ST
 
 import Common.Annex
 import Command
+import Logs.Unused
 import Annex.Content
 import Utility.FileMode
-import Utility.TempFile
 import Logs.Location
 import Config
 import qualified Annex
@@ -91,19 +91,13 @@ check file msg a c = do
 	l <- a
 	let unusedlist = number c l
 	unless (null l) $ showLongNote $ msg unusedlist
-	writeUnusedFile file unusedlist
+	writeUnusedLog file unusedlist
 	return $ c + length l
 
 number :: Int -> [a] -> [(Int, a)]
 number _ [] = []
 number n (x:xs) = (n+1, x) : number (n+1) xs
 
-writeUnusedFile :: FilePath -> [(Int, Key)] -> Annex ()
-writeUnusedFile prefix l = do
-	logfile <- fromRepo $ gitAnnexUnusedLog prefix
-	liftIO $ viaTmp writeFile logfile $
-		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
-
 table :: [(Int, Key)] -> [String]
 table l = "  NUMBER  KEY" : map cols l
 	where
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 52886c3087..0e707b1868 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -37,6 +37,7 @@ import qualified Command.InitRemote
 import qualified Command.Fsck
 import qualified Command.Unused
 import qualified Command.DropUnused
+import qualified Command.AddUnused
 import qualified Command.Unlock
 import qualified Command.Lock
 import qualified Command.PreCommit
@@ -86,6 +87,7 @@ cmds = concat
 	, Command.Fsck.def
 	, Command.Unused.def
 	, Command.DropUnused.def
+	, Command.AddUnused.def
 	, Command.Find.def
 	, Command.Whereis.def
 	, Command.Log.def
diff --git a/Logs/Unused.hs b/Logs/Unused.hs
new file mode 100644
index 0000000000..7d240cfe33
--- /dev/null
+++ b/Logs/Unused.hs
@@ -0,0 +1,91 @@
+{- git-annex unused log file
+ -
+ - Copyright 2010,2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Logs.Unused (
+	UnusedMap,
+	UnusedMaps(..),
+	writeUnusedLog,
+	readUnusedLog,
+	withUnusedMaps,
+	startUnused,
+) where
+
+import qualified Data.Map as M
+
+import Common.Annex
+import Command
+import Types.Key
+import Utility.TempFile
+
+writeUnusedLog :: FilePath -> [(Int, Key)] -> Annex ()
+writeUnusedLog prefix l = do
+	logfile <- fromRepo $ gitAnnexUnusedLog prefix
+	liftIO $ viaTmp writeFile logfile $
+		unlines $ map (\(n, k) -> show n ++ " " ++ show k) l
+
+readUnusedLog :: FilePath -> Annex UnusedMap
+readUnusedLog prefix = do
+	f <- fromRepo $ gitAnnexUnusedLog prefix
+	ifM (liftIO $ doesFileExist f)
+		( M.fromList . catMaybes . map parse . lines
+			<$> liftIO (readFile f)
+		, return M.empty
+		)
+	where
+		parse line =
+			case (readish tag, readKey rest) of
+				(Just num, Just key) -> Just (num, key)
+				_ -> Nothing
+			where
+				(tag, rest) = separate (== ' ') line
+
+type UnusedMap = M.Map Int Key
+
+data UnusedMaps = UnusedMaps
+	{ unusedMap :: UnusedMap
+	, unusedBadMap :: UnusedMap
+	, unusedTmpMap :: UnusedMap
+	}
+
+{- Read unused logs once, and pass the maps to each start action. -}
+withUnusedMaps :: (UnusedMaps -> Int -> CommandStart) -> CommandSeek
+withUnusedMaps a params = do
+	unused <- readUnusedLog ""
+	unusedbad <- readUnusedLog "bad"
+	unusedtmp <- readUnusedLog "tmp"
+	return $ map (a $ UnusedMaps unused unusedbad unusedtmp) $
+		concatMap unusedSpec params
+
+unusedSpec :: String -> [Int]
+unusedSpec spec
+	| "-" `isInfixOf` spec = range $ separate (== '-') spec
+	| otherwise = catMaybes [readish spec]
+	where
+		range (a, b) = case (readish a, readish b) of
+			(Just x, Just y) -> [x..y]
+			_ -> []
+
+{- Start action for unused content. Finds the number in the maps, and
+ - calls either of 3 actions, depending on the type of unused file. -}
+startUnused :: String
+	-> (Key -> CommandPerform)
+	-> (Key -> CommandPerform) 
+	-> (Key -> CommandPerform)
+	-> UnusedMaps -> Int -> CommandStart
+startUnused message unused badunused tmpunused maps n = search
+	[ (unusedMap maps, unused)
+	, (unusedBadMap maps, badunused)
+	, (unusedTmpMap maps, tmpunused)
+	]
+	where
+		search [] = stop
+		search ((m, a):rest) =
+			case M.lookup n m of
+				Nothing -> search rest
+				Just key -> do
+					showStart message (show n)
+					next $ a key
diff --git a/Usage.hs b/Usage.hs
index b1de930ef2..e74c1490d0 100644
--- a/Usage.hs
+++ b/Usage.hs
@@ -61,6 +61,8 @@ paramUrl :: String
 paramUrl = "URL"
 paramNumber :: String
 paramNumber = "NUMBER"
+paramNumRange :: String
+paramNumRange = "NUM|RANGE"
 paramRemote :: String
 paramRemote = "REMOTE"
 paramGlob :: String
diff --git a/debian/changelog b/debian/changelog
index 72bce85511..b419b46227 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ git-annex (3.20120431) UNRELEASED; urgency=low
     (specificially hidrive.strato.com) that use rsync over ssh but do not
     pass it through the shell.
   * dropunused: Allow specifying ranges to drop.
+  * addunused: New command, the opposite of dropunused, it relinks unused
+    content into the git repository.
 
  -- Joey Hess   Wed, 02 May 2012 13:06:18 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 998e1fa266..5d41f86e9c 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -235,7 +235,7 @@ subdirectories).
 
   To check for annexed data on a remote, specify --from.
 
-* dropunused [number ...]
+* dropunused [number|range ...]
 
   Drops the data corresponding to the numbers, as listed by the last
   `git annex unused`
@@ -244,6 +244,12 @@ subdirectories).
 
   To drop the data from a remote, specify --from.
 
+* addunused [number|range ...]
+
+  Adds back files for the content corresponding to the numbers or ranges,
+  as listed by the last `git annex unused`. The files will have names
+  starting with "unused."
+
 * merge
 
   Automatically merges remote tracking branches */git-annex into

From 441b138592c243beb3627c8aab5aafb12523ad05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 15:03:44 -0400
Subject: [PATCH 3456/8313] followup and close

---
 ...sync_remote__44___versions_3.20120315_and_3.20120430.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
index ab5eb78435..85e2433829 100644
--- a/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
+++ b/doc/bugs/Error___39__get__39__ting_files_from_rsync_remote__44___versions_3.20120315_and_3.20120430.mdwn
@@ -64,6 +64,8 @@ The rsync remote config in .git/config:
 > says AFAICS) that the rsync protocol is only used if the url starts
 > with `rsync://` or contains `::`.
 > 
+>> Nope, it is indeed using rsync over ssh as git-annex thought.
+> 
 > Hmm, I see that `hidrive.strato.com` is some kind of rsync provider?
 > Perhaps they do something with rsync over ssh that
 > avoids the need for shell quoting. For example, they might pass incoming
@@ -72,3 +74,6 @@ The rsync remote config in .git/config:
 > about them would probably be useful then. Ie, do they really use rsync
 > over ssh, is it really a `rsync.net` type rsync provider? 
 >  --[[Joey]]
+> 
+>> This was the case, and the shellescape=no config option has been added
+>> to rsync special remotes to deal with it. [[done]] --[[Joey]] 

From 32de288c35b5cc728821c1c465c398f9bd8ba8d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 2 May 2012 19:51:41 -0400
Subject: [PATCH 3457/8313] syntax tweaks

Although I hate to lose one of the only places I've ever used the list monad..
---
 Logs/Remote.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Logs/Remote.hs b/Logs/Remote.hs
index 5c9d67df03..b75573a411 100644
--- a/Logs/Remote.hs
+++ b/Logs/Remote.hs
@@ -36,7 +36,7 @@ configSet u c = do
 
 {- Map of remotes by uuid containing key/value config maps. -}
 readRemoteLog :: Annex (M.Map UUID RemoteConfig)
-readRemoteLog = (simpleMap . parseLog parseConfig) <$> Annex.Branch.get remoteLog
+readRemoteLog = simpleMap . parseLog parseConfig <$> Annex.Branch.get remoteLog
 
 parseConfig :: String -> Maybe RemoteConfig
 parseConfig = Just . keyValToConfig . words
@@ -59,7 +59,7 @@ configToKeyVal m = map toword $ sort $ M.toList m
 		toword (k, v) = k ++ "=" ++ configEscape v
 
 configEscape :: String -> String
-configEscape = (>>= escape)
+configEscape = concatMap escape
 	where
 		escape c
 			| isSpace c || c `elem` "&" = "&" ++ show (ord c) ++ ";"

From 8c09c17f6bcc1d9aaeab38eb4a6f7682139e8c65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 4 May 2012 00:44:11 -0400
Subject: [PATCH 3458/8313] use strict insertWith

---
 Logs/UUIDBased.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index b09d93f903..847d499237 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -83,7 +83,7 @@ changeLog t u v = M.insert u $ LogEntry (Date t) v
 {- Only add an LogEntry if it's newer (or at least as new as) than any
  - existing LogEntry for a UUID. -}
 addLog :: UUID -> LogEntry a -> Log a -> Log a
-addLog = M.insertWith best
+addLog = M.insertWith' best
 
 {- Converts a Log into a simple Map without the timestamp information.
  - This is a one-way trip, but useful for code that never needs to change

From 657d09d49990af85c1a91b1154a195a30438477c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 5 May 2012 20:11:08 -0400
Subject: [PATCH 3459/8313] fix build

---
 test.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test.hs b/test.hs
index 8c24c3a23d..1952a39a9b 100644
--- a/test.hs
+++ b/test.hs
@@ -35,8 +35,8 @@ import qualified GitAnnex
 import qualified Logs.UUIDBased
 import qualified Logs.Trust
 import qualified Logs.Remote
+import qualified Logs.Unused
 import qualified Remote
-import qualified Command.DropUnused
 import qualified Types.Key
 import qualified Types.Messages
 import qualified Config
@@ -495,7 +495,7 @@ test_unused = "git-annex unused/dropunused" ~: intmpclonerepo $ do
 	where
 		checkunused expectedkeys = do
 			git_annex "unused" [] @? "unused failed"
-			unusedmap <- annexeval $ Command.DropUnused.readUnusedLog ""
+			unusedmap <- annexeval $ Logs.Unused.readUnusedLog ""
 			let unusedkeys = M.elems unusedmap
 			assertEqual "unused keys differ"
 				(sort expectedkeys) (sort unusedkeys)

From f7d8982672fd330a466c2cb22f34388e7cc429c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 5 May 2012 20:15:32 -0400
Subject: [PATCH 3460/8313] Fix use of several config settings

annex.ssh-options, annex.rsync-options, annex.bup-split-options.

And adjust types to avoid the bugs that broke several config settings
recently. Now "annex." prefixing is enforced at the type level.
---
 Annex/Content.hs  |  4 ++--
 Annex/Queue.hs    |  2 +-
 Annex/Ssh.hs      |  2 +-
 Annex/UUID.hs     | 14 +++++++-------
 Annex/Version.hs  |  4 ++--
 Backend.hs        |  2 +-
 Command/Unused.hs |  4 ++--
 Config.hs         | 43 ++++++++++++++++++++++++++++---------------
 Remote/Hook.hs    |  4 ++--
 debian/changelog  |  3 +++
 10 files changed, 49 insertions(+), 33 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index b5754e15be..2142d1f09d 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -304,12 +304,12 @@ saveState oneshot = doSideAction $ do
 			( Annex.Branch.commit "update" , Annex.Branch.stage)
 	where
 		alwayscommit = fromMaybe True . Git.configTrue
-			<$> getConfig "annex.alwayscommit" ""
+			<$> getConfig (annexConfig "alwayscommit") ""
 
 {- Downloads content from any of a list of urls. -}
 downloadUrl :: [Url.URLString] -> FilePath -> Annex Bool
 downloadUrl urls file = do
-	o <- map Param . words <$> getConfig "annex.web-options" ""
+	o <- map Param . words <$> getConfig (annexConfig "web-options") ""
 	headers <- getHttpHeaders
 	liftIO $ anyM (\u -> Url.download u headers o file) urls
 
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 728e29645e..24575e9068 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -46,7 +46,7 @@ new = do
 	store q
 	return q
 	where
-		queuesize = readish <$> getConfig "annex.queuesize" ""
+		queuesize = readish <$> getConfig (annexConfig "queuesize") ""
 
 store :: Git.Queue.Queue -> Annex ()
 store q = changeState $ \s -> s { repoqueue = Just q }
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index 6a230312ab..f0824b1191 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -48,7 +48,7 @@ sshInfo (host, port) = ifM caching
 	where
 		caching = fromMaybe SysConfig.sshconnectioncaching 
 			. Git.configTrue
-			<$> getConfig "annex.sshcaching" ""
+			<$> getConfig (annexConfig "sshcaching") ""
 
 cacheParams :: FilePath -> [CommandParam]
 cacheParams socketfile =
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 5459cc7fee..517840fbad 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -23,12 +23,11 @@ module Annex.UUID (
 import Common.Annex
 import qualified Git
 import qualified Git.Config
-import qualified Git.Command
 import qualified Build.SysConfig as SysConfig
 import Config
 
-configkey :: String
-configkey = "annex.uuid"
+configkey :: ConfigKey
+configkey = annexConfig "uuid"
 
 {- Generates a UUID. There is a library for this, but it's not packaged,
  - so use the command line tool. -}
@@ -64,16 +63,17 @@ getRepoUUID r = do
 		cachekey = remoteConfig r "uuid"
 
 removeRepoUUID :: Annex ()
-removeRepoUUID = inRepo $ Git.Command.run "config"
-	[Param "--unset", Param configkey]
+removeRepoUUID = unsetConfig configkey
 
 getUncachedUUID :: Git.Repo -> UUID
-getUncachedUUID = toUUID . Git.Config.get configkey ""
+getUncachedUUID = toUUID . Git.Config.get key ""
+	where
+		(ConfigKey key) = configkey
 
 {- Make sure that the repo has an annex.uuid setting. -}
 prepUUID :: Annex ()
 prepUUID = whenM ((==) NoUUID <$> getUUID) $
 	storeUUID configkey =<< liftIO genUUID
 
-storeUUID :: String -> UUID -> Annex ()
+storeUUID :: ConfigKey -> UUID -> Annex ()
 storeUUID configfield = setConfig configfield . fromUUID
diff --git a/Annex/Version.hs b/Annex/Version.hs
index a1d0402445..7c909ae05b 100644
--- a/Annex/Version.hs
+++ b/Annex/Version.hs
@@ -21,8 +21,8 @@ supportedVersions = [defaultVersion]
 upgradableVersions :: [Version]
 upgradableVersions = ["0", "1", "2"]
 
-versionField :: String
-versionField = "annex.version"
+versionField :: ConfigKey
+versionField = annexConfig "version"
 
 getVersion :: Annex (Maybe Version)
 getVersion = handle <$> getConfig versionField ""
diff --git a/Backend.hs b/Backend.hs
index 19562205c8..8071b9b835 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -46,7 +46,7 @@ orderedList = do
 			l' <- (lookupBackendName name :) <$> standard
 			Annex.changeState $ \s -> s { Annex.backends = l' }
 			return l'
-		standard = parseBackendList <$> getConfig "annex.backends" ""
+		standard = parseBackendList <$> getConfig (annexConfig "backends") ""
 		parseBackendList [] = list
 		parseBackendList s = map lookupBackendName $ words s
 
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 6b319ee72c..f5ee452a80 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -183,10 +183,10 @@ exclude smaller larger = S.toList $ remove larger $ S.fromList smaller
  -}
 bloomCapacity :: Annex Int
 bloomCapacity = fromMaybe 500000 . readish
-	<$> getConfig "annex.bloomcapacity" ""
+	<$> getConfig (annexConfig "bloomcapacity") ""
 bloomAccuracy :: Annex Int
 bloomAccuracy = fromMaybe 1000 . readish
-	<$> getConfig "annex.bloomaccuracy" ""
+	<$> getConfig (annexConfig "bloomaccuracy") ""
 bloomBitsHashes :: Annex (Int, Int)
 bloomBitsHashes = do
 	capacity <- bloomCapacity
diff --git a/Config.hs b/Config.hs
index 3feb246e53..5f1ac8bb20 100644
--- a/Config.hs
+++ b/Config.hs
@@ -1,6 +1,6 @@
 {- Git configuration
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -14,29 +14,40 @@ import qualified Git.Command
 import qualified Annex
 import Utility.DataUnits
 
-type ConfigKey = String
+type UnqualifiedConfigKey = String
+data ConfigKey = ConfigKey String
 
 {- Changes a git config setting in both internal state and .git/config -}
 setConfig :: ConfigKey -> String -> Annex ()
-setConfig k value = do
-	inRepo $ Git.Command.run "config" [Param k, Param value]
+setConfig (ConfigKey key) value = do
+	inRepo $ Git.Command.run "config" [Param key, Param value]
 	-- re-read git config and update the repo's state
 	newg <- inRepo Git.Config.read
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
-{- Looks up a git config setting in git config. -}
+{- Unsets a git config setting. (Leaves it in state currently.) -}
+unsetConfig :: ConfigKey -> Annex ()
+unsetConfig (ConfigKey key) = inRepo $ Git.Command.run "config"
+        [Param "--unset", Param key]
+
+{- Looks up a setting in git config. -}
 getConfig :: ConfigKey -> String -> Annex String
-getConfig key def = fromRepo $ Git.Config.get key def
+getConfig (ConfigKey key) def = fromRepo $ Git.Config.get key def
 
 {- Looks up a per-remote config setting in git config.
  - Failing that, tries looking for a global config option. -}
-getRemoteConfig :: Git.Repo -> ConfigKey -> String -> Annex String
-getRemoteConfig r key def =
-	getConfig (remoteConfig r key) =<< getConfig key def
+getRemoteConfig :: Git.Repo -> UnqualifiedConfigKey -> String -> Annex String
+getRemoteConfig r key def = 
+	getConfig (remoteConfig r key) =<< getConfig (annexConfig key) def
 
 {- A per-remote config setting in git config. -}
-remoteConfig :: Git.Repo -> ConfigKey -> String
-remoteConfig r key = "remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key
+remoteConfig :: Git.Repo -> UnqualifiedConfigKey -> ConfigKey
+remoteConfig r key = ConfigKey $
+	"remote." ++ fromMaybe "" (Git.remoteName r) ++ ".annex-" ++ key
+
+{- A global annex setting in git config. -}
+annexConfig :: UnqualifiedConfigKey -> ConfigKey
+annexConfig key = ConfigKey $ "annex." ++ key
 
 {- Calculates cost for a remote. Either the default, or as configured 
  - by remote..annex-cost, or if remote..annex-cost-command
@@ -83,17 +94,19 @@ getNumCopies v = perhaps (use v) =<< Annex.getState Annex.forcenumcopies
 	where
 		use (Just n) = return n
 		use Nothing = perhaps (return 1) =<< 
-			readish <$> getConfig "annex.numcopies" "1"
+			readish <$> getConfig (annexConfig "numcopies") "1"
 		perhaps fallback = maybe fallback (return . id)
 
 {- Gets the trust level set for a remote in git config. -}
 getTrustLevel :: Git.Repo -> Annex (Maybe String)
-getTrustLevel r = fromRepo $ Git.Config.getMaybe $ remoteConfig r "trustlevel"
+getTrustLevel r = fromRepo $ Git.Config.getMaybe key
+	where
+		(ConfigKey key) = remoteConfig r "trustlevel"
 
 {- Gets annex.diskreserve setting. -}
 getDiskReserve :: Annex Integer
 getDiskReserve = fromMaybe megabyte . readSize dataUnits
-	<$> getConfig "annex.diskreserve" ""
+	<$> getConfig (annexConfig "diskreserve") ""
 	where
 		megabyte = 1000000
 
@@ -101,7 +114,7 @@ getDiskReserve = fromMaybe megabyte . readSize dataUnits
  - splitting it into lines. -}
 getHttpHeaders :: Annex [String]
 getHttpHeaders = do
-	cmd <- getConfig "annex.http-headers-command" ""
+	cmd <- getConfig (annexConfig "http-headers-command") ""
 	if (null cmd)
 		then fromRepo $ Git.Config.getList "annex.http-headers"
 		else lines . snd <$> liftIO (pipeFrom "sh" ["-c", cmd])
diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index 1c87823caa..dcac9da889 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -74,14 +74,14 @@ hookEnv k f = Just $ fileenv f ++ keyenv
 
 lookupHook :: String -> String -> Annex (Maybe String)
 lookupHook hooktype hook =do
-	command <- getConfig hookname ""
+	command <- getConfig (annexConfig hookname) ""
 	if null command
 		then do
 			warning $ "missing configuration for " ++ hookname
 			return Nothing
 		else return $ Just command
 	where
-		hookname =  "annex." ++ hooktype ++ "-" ++ hook ++ "-hook"
+		hookname = hooktype ++ "-" ++ hook ++ "-hook"
 
 runHook :: String -> String -> Key -> Maybe FilePath -> Annex Bool -> Annex Bool
 runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype hook
diff --git a/debian/changelog b/debian/changelog
index b419b46227..049531dad0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,9 @@ git-annex (3.20120431) UNRELEASED; urgency=low
   * dropunused: Allow specifying ranges to drop.
   * addunused: New command, the opposite of dropunused, it relinks unused
     content into the git repository.
+  * Fix use of several config settings annex.ssh-options,
+    annex.rsync-options, annex.bup-split-options. (And adjust types to avoid
+    the bugs that broke several config settings.)
 
  -- Joey Hess   Wed, 02 May 2012 13:06:18 -0400
 

From bbfa74e7ac365124e5aa8d4797d9f1c957b81eda Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 7 May 2012 13:18:45 -0400
Subject: [PATCH 3461/8313] format

---
 debian/changelog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/changelog b/debian/changelog
index 049531dad0..7d10b560b1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,7 +8,7 @@ git-annex (3.20120431) UNRELEASED; urgency=low
   * dropunused: Allow specifying ranges to drop.
   * addunused: New command, the opposite of dropunused, it relinks unused
     content into the git repository.
-  * Fix use of several config settings annex.ssh-options,
+  * Fix use of several config settings: annex.ssh-options,
     annex.rsync-options, annex.bup-split-options. (And adjust types to avoid
     the bugs that broke several config settings.)
 

From 3770bc5d4181e230159b493a2f6bef7a2f19a7b7 Mon Sep 17 00:00:00 2001
From: "https://me.yahoo.com/a/IAg3idYGk.joxsJb2WCxl20gig.0.8hS#d5165"
 
Date: Mon, 7 May 2012 21:57:58 +0000
Subject: [PATCH 3462/8313]

---
 doc/forum/Making_git-annex_less_necessary.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/Making_git-annex_less_necessary.mdwn

diff --git a/doc/forum/Making_git-annex_less_necessary.mdwn b/doc/forum/Making_git-annex_less_necessary.mdwn
new file mode 100644
index 0000000000..086c051398
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary.mdwn
@@ -0,0 +1,5 @@
+http://git-annex.branchable.com/walkthrough/ says "Git wants to first stage the entire contents of the file in its index. That can be slow for big files (sorta why git-annex exists in the first place)."

+What is git doing that git-annex isn't, other than copying the file to .git/objects rather than just moving it to .git/annex/objects, prepending it with "blob"+length, and compressing it? If git were changed to store the "blob"+length as part of the object filename rather than as part of the object file content, have a config option to use uncompressed objects for large files (and not try to pack them when creating pack files), and were used on a filesystem such as zfs or btrfs which does COW so the copy would be as fast as a move, then what speed advantage would git-annex still have over git? I realize git-annex has more features than just big file handling, and has the worm backend for even faster handling, but I'm just talking about the case with the default sha backend.

+Have such changes been proposed for git? It seems that for anybody already familiar with the git codebase, adding the config option for uncompressed objects and moving the storage location for "blob"+length would be easy changes to make, and I see no downside to them. It wouldn't break backwards compatibility because the object filename being hash."blob".length rather than just hash would indicate that the new object format is in use, and a ".raw" filename extension could be used for uncompressed objects (or more sensibly, in the new format, no additional extension for uncompressed, and ".compressed" for compressed).

+This would also eliminate the need for a git-annex object store separate from the git object store, and the complexities involved with having them separate, and the need for symlinks, and the complexities they cause. I don't think that relying on COW for speed is unreasonable once btrfs becomes the default in major Linux distros (the bsds already have zfs and hammerfs); right now part of what git-annex is doing is just working around the functional deficiency of non-COW filesystems.

+P.S. I recommend a "plain" option for the page type when submitting comments on your wiki, so I don't have to put HTML line break markup at the end of my lines.

From ac08c37649e25cd3647d3ca97bda91ffc9232889 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Tue, 8 May 2012 18:22:12 +0000
Subject: [PATCH 3463/8313] Added a comment

---
 ...mment_1_03faaa3866778d24cd03887b85dc9954._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment

diff --git a/doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment b/doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment
new file mode 100644
index 0000000000..3396643f66
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_1_03faaa3866778d24cd03887b85dc9954._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.245"
+ subject="comment 1"
+ date="2012-05-08T18:22:12Z"
+ content="""
+git's code base makes lots of assumptions hardcoding the size of the hash, etc. (grep its source for magic numbers 40 and 42...) I'd like to see git get parameratised hashes. SHA1 insecurity may evenutally push it in that direction. However, when I asked the git developers about this at the Gittogether last year, there were several ideas floated that would avoid parameterisation, and a lot of good thoughts about problems parameterised hashes would cause.
+
+Moving data into git proper would still leave the problems unique to large data of not being able to store it all on every clone. Which means a git-annex like thing is needed to track where the data resides and move it around.
+
+(BTW, in markdown, you separate paragraphs with blank lines. Like in email.)
+"""]]

From ef247f3da7f9b767180a2985922cf3d0c391f278 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Thu, 10 May 2012 13:14:50 +0000
Subject: [PATCH 3464/8313]

---
 doc/forum/Git_Annex_Transfer_Protocols.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols.mdwn

diff --git a/doc/forum/Git_Annex_Transfer_Protocols.mdwn b/doc/forum/Git_Annex_Transfer_Protocols.mdwn
new file mode 100644
index 0000000000..d6a660a2d8
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols.mdwn
@@ -0,0 +1,9 @@
+Hi,
+
+May I know, which processes git-annex is using to move/copy/get file content between repositories?
+Is it using same processes which git uses, Like send-pack, receive-pack.
+I want to use Aspera to move file contents between repositories.
+Is it enough if I customize send-pack, receive-pack of git code to do Aspera file transfer or git-annex uses any other transfer mechanism.
+
+Many Thanks,
+Royal Pinto

From ab84b612ead4bea1c1e5fdf090d7371283a296a7 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 10 May 2012 18:18:01 +0000
Subject: [PATCH 3465/8313] Added a comment: rsync over ssh

---
 .../comment_1_a870ec991078c95a6bb683d6962ab56e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment

diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment
new file mode 100644
index 0000000000..31c463470c
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_1_a870ec991078c95a6bb683d6962ab56e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.186"
+ subject="rsync over ssh"
+ date="2012-05-10T18:18:01Z"
+ content="""
+Some other protocols such as S3 for special remotes.
+"""]]

From 32a41f8af18a067f59e43f6fdc0be312ade8b69e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 10 May 2012 14:18:35 -0400
Subject: [PATCH 3466/8313] add a favicon

---
 debian/copyright |   6 ++++++
 doc/favicon.ico  | Bin 0 -> 405 bytes
 2 files changed, 6 insertions(+)
 create mode 100644 doc/favicon.ico

diff --git a/debian/copyright b/debian/copyright
index 332c1e71d2..de1e08e1cd 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -7,3 +7,9 @@ License: GPL-3+
  The full text of version 3 of the GPL is distributed as doc/GPL in
  this package's source, or in /usr/share/common-licenses/GPL-3 on
  Debian systems.
+
+Files: doc/logo.png doc/logo_small.png doc/favicon.png
+Copyright: 2007 Henrik Nyh 
+           2010 Joey Hess 
+License: other
+  Free to modify and redistribute with due credit, and obviously free to use.
diff --git a/doc/favicon.ico b/doc/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..5bb405931fcc8f8194694e5d6cb728e50658891e
GIT binary patch
literal 405
zcmex=bI)35R>*e`CsU}f<4#mC*t-skh=9nPFAvL)f4AHU1X%imZ2>GMq7
z9Qkz8nG&C%#d}NA9x5}4Y=}=u%=tanUZi)Dv@7qlRhOEXZ#_NRE_mW`HTUk>SB@)n
MY&muAb@%_9027yGs{jB1

literal 0
HcmV?d00001


From 660f5de5512e25654e84802317686b21419a4148 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Thu, 10 May 2012 18:48:41 +0000
Subject: [PATCH 3467/8313] Added a comment: Protocols to transfer file content

---
 .../comment_2_71419376ef50a679ea8f0f9e16991c17._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment

diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment
new file mode 100644
index 0000000000..2e07aef088
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_2_71419376ef50a679ea8f0f9e16991c17._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Protocols to transfer file content"
+ date="2012-05-10T18:48:40Z"
+ content="""
+Thanks, Is git annex is using same protocols as normal git to transfer content between normal git repositories?
+"""]]

From daaeba3afb3be5b5c72269d999b7286261fac0c3 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 10 May 2012 18:51:56 +0000
Subject: [PATCH 3468/8313] Added a comment

---
 .../comment_3_fea43664a500111ca99f4043e0dadb14._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment

diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment
new file mode 100644
index 0000000000..6e7b36e311
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_3_fea43664a500111ca99f4043e0dadb14._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.186"
+ subject="comment 3"
+ date="2012-05-10T18:51:56Z"
+ content="""
+git-annex doesn't transfer git content between git repositories. You use git for that. Well, git-annex sync can run a few git commands for you to do it.
+"""]]

From 8734e7cba518034a08f85eeef153bf64bb09ed0c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Thu, 10 May 2012 19:13:48 +0000
Subject: [PATCH 3469/8313] Added a comment: Git annex content transfer
 protocols

---
 .../comment_4_56fb2dab1d4030c9820be32b495afdf0._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment

diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment
new file mode 100644
index 0000000000..fef10cd813
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_4_56fb2dab1d4030c9820be32b495afdf0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Git annex content transfer protocols"
+ date="2012-05-10T19:13:48Z"
+ content="""
+Sorry if I am not clear. Actually i meant to ask, if i have 2 git repositories which are not special remotes and I am transferring annexed file content between these repositories using git annex command (move or copy) then, which protocol it uses to transfer content? Is it uses git-send-pack git-recieve-pack or some other protocols. 
+"""]]

From 891684f85dd8691bbea2557d645bf194b76abd24 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 10 May 2012 19:17:22 +0000
Subject: [PATCH 3470/8313] Added a comment

---
 .../comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment

diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment
new file mode 100644
index 0000000000..be25737c1d
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_5_a6ec9c5a4a3c0bac1df87f1df9be140b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.186"
+ subject="comment 5"
+ date="2012-05-10T19:17:22Z"
+ content="""
+rsync over ssh is used to transfer file contents between repositories. (You can use the -d option to see the commands git-annex runs.)
+"""]]

From f02db7bfce54563de9a42588ba926d22d4fe3ace Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Thu, 10 May 2012 19:21:08 +0000
Subject: [PATCH 3471/8313] Added a comment

---
 .../comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment

diff --git a/doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment b/doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment
new file mode 100644
index 0000000000..b7ef8f33c4
--- /dev/null
+++ b/doc/forum/Git_Annex_Transfer_Protocols/comment_6_1678452fb7114aeabcf0cc3d5f6c69b0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="comment 6"
+ date="2012-05-10T19:21:08Z"
+ content="""
+Ok. This helped me a lot. Thank you
+"""]]

From 61a5df33d4d24ccbedf8395c2218e1e5a3f241e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 May 2012 12:37:26 -0400
Subject: [PATCH 3472/8313] releasing version 3.20120511

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 7d10b560b1..6ebd6fabf0 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120431) UNRELEASED; urgency=low
+git-annex (3.20120511) unstable; urgency=low
 
   * Rsync special remotes can be configured with shellescape=no
     to avoid shell quoting that is normally done when using rsync over ssh.
@@ -12,7 +12,7 @@ git-annex (3.20120431) UNRELEASED; urgency=low
     annex.rsync-options, annex.bup-split-options. (And adjust types to avoid
     the bugs that broke several config settings.)
 
- -- Joey Hess   Wed, 02 May 2012 13:06:18 -0400
+ -- Joey Hess   Fri, 11 May 2012 12:29:30 -0400
 
 git-annex (3.20120430) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 6299944698..f87e49a200 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120430
+Version: 3.20120511
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From 1c942c898d8f738652a5f4239fc7cd0b009e4903 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 May 2012 12:38:08 -0400
Subject: [PATCH 3473/8313] add news item for git-annex 3.20120511

---
 doc/news/version_3.20120315.mdwn | 21 ---------------------
 doc/news/version_3.20120511.mdwn | 13 +++++++++++++
 2 files changed, 13 insertions(+), 21 deletions(-)
 delete mode 100644 doc/news/version_3.20120315.mdwn
 create mode 100644 doc/news/version_3.20120511.mdwn

diff --git a/doc/news/version_3.20120315.mdwn b/doc/news/version_3.20120315.mdwn
deleted file mode 100644
index a3ccb4cf47..0000000000
--- a/doc/news/version_3.20120315.mdwn
+++ /dev/null
@@ -1,21 +0,0 @@
-git-annex 3.20120315 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * fsck: Fix up any broken links and misplaced content caused by the
-     directory hash calculation bug fixed in the last release.
-   * sync: Sync to lower cost remotes first.
-   * status: Fixed to run in constant space.
-   * status: More accurate display of sizes of tmp and bad keys.
-   * unused: Now uses a bloom filter, and runs in constant space.
-     Use of a bloom filter does mean it will not notice a small
-     number of unused keys. For repos with up to half a million keys,
-     it will miss one key in 1000.
-   * Added annex.bloomcapacity and annex.bloomaccuracy, which can be
-     adjusted as desired to tune the bloom filter.
-   * status: Display amount of memory used by bloom filter, and
-     detect when it's too small for the number of keys in a repository.
-   * git-annex-shell: Runs hooks/annex-content after content is received
-     or dropped.
-   * Work around a bug in rsync (IMHO) introduced by openSUSE's SIP patch.
-   * git-annex now behaves as git-annex-shell if symlinked to and run by that
-     name. The Makefile sets this up, saving some 8 mb of installed size.
-   * git-union-merge is a demo program, so it is no longer built by default."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120511.mdwn b/doc/news/version_3.20120511.mdwn
new file mode 100644
index 0000000000..19e8355224
--- /dev/null
+++ b/doc/news/version_3.20120511.mdwn
@@ -0,0 +1,13 @@
+git-annex 3.20120511 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Rsync special remotes can be configured with shellescape=no
+     to avoid shell quoting that is normally done when using rsync over ssh.
+     This is known to be needed for certian rsync hosting providers
+     (specificially hidrive.strato.com) that use rsync over ssh but do not
+     pass it through the shell.
+   * dropunused: Allow specifying ranges to drop.
+   * addunused: New command, the opposite of dropunused, it relinks unused
+     content into the git repository.
+   * Fix use of several config settings: annex.ssh-options,
+     annex.rsync-options, annex.bup-split-options. (And adjust types to avoid
+     the bugs that broke several config settings.)"""]]
\ No newline at end of file

From 2701800cda086782f40dd615c23d605667b8a81b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 11 May 2012 12:38:56 -0400
Subject: [PATCH 3474/8313] cleanup

---
 ..._fb1a3135e2d9f39f2c372ccc2c50c85a._comment |   8 --
 ..._ae292ca7294b6790233e545086c3ac2f._comment |  10 --
 ..._29ccda9ac458fd5cc9ec5508c62df6ea._comment |  12 --
 ..._f8fc894680f2a2e5b5e757a677414b42._comment |   8 --
 ..._ea5075cfecc50d5da2364931ef7a02d1._comment |  10 --
 ..._18158b9be2313f49509d59295c7d3c90._comment |   8 --
 ..._03436ddda42decf8cb1b4d5316d88a75._comment |  14 --
 ..._8f7f8d4758804f1b695925934219745a._comment |  42 ------
 ..._cd90223f78571e5bdd3dfc07ab1369d7._comment |   9 --
 ..._7dbf131ff4611abbfc8fbf1ee0f66dbe._comment |   8 --
 ..._b975cbd3a01ba5c2fa0f24fe739d3433._comment | 128 ------------------
 ..._899de1196cd1ba4a393e4ef574d7aa5e._comment |   8 --
 12 files changed, 265 deletions(-)
 delete mode 100644 doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment
 delete mode 100644 doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment
 delete mode 100644 doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment
 delete mode 100644 doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment
 delete mode 100644 doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment
 delete mode 100644 doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment
 delete mode 100644 doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment
 delete mode 100644 doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment
 delete mode 100644 doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment
 delete mode 100644 doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment
 delete mode 100644 doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment
 delete mode 100644 doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment

diff --git a/doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment b/doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment
deleted file mode 100644
index effd9c9a8f..0000000000
--- a/doc/news/version_3.20120106/comment_1_fb1a3135e2d9f39f2c372ccc2c50c85a._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="77.186.134.113"
- subject="Not announced on Hackage?"
- date="2012-01-13T17:37:11Z"
- content="""
-We have the [latest version in NixOS](http://hydra.nixos.org/job/nixpkgs/trunk/gitAndTools.gitAnnex), but we cannot advertise that fact on Hackage because it seems the corresponding Cabal file hasn't been uploaded to . Is there any particular reason why Hackage doesn't know about this release?
-"""]]
diff --git a/doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment b/doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment
deleted file mode 100644
index 7e6920eb72..0000000000
--- a/doc/news/version_3.20120106/comment_2_ae292ca7294b6790233e545086c3ac2f._comment
+++ /dev/null
@@ -1,10 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 2"
- date="2012-01-13T17:52:46Z"
- content="""
-Uploading to hackage is a PITA (manual password entry and I am often on a slow link besides) and is not integrated with my regular release process, so I often forget to do it. I will try to upload the next release there again.
-
-You might add a page under [[install]] for your git-annex packages.
-"""]]
diff --git a/doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment b/doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment
deleted file mode 100644
index 1bb0c32cb1..0000000000
--- a/doc/news/version_3.20120106/comment_3_29ccda9ac458fd5cc9ec5508c62df6ea._comment
+++ /dev/null
@@ -1,12 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="77.186.134.113"
- subject="comment 3"
- date="2012-01-13T18:48:28Z"
- content="""
-For what it's worth, the package `cabal-install` is pretty good for uploading packages to Hackages, among other things. It allows users to configure their username/password, and then making a release to Hackage is as simple as running:
-
-    cabal upload foo-version.tar.gz
-
-I use that to do releases of my stuff, too, and I'm quite happy. `cabal-install` has other features, too, of course.
-"""]]
diff --git a/doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment b/doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment
deleted file mode 100644
index aa3c5ce6d1..0000000000
--- a/doc/news/version_3.20120227/comment_1_f8fc894680f2a2e5b5e757a677414b42._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="77.184.15.65"
- subject="Test-suite won't compile with GHC 7.4.x"
- date="2012-02-28T17:39:59Z"
- content="""
-The recent version requires GHC 7.4.x, but some dependencies for the test suite don't build with that compiler, i.e. the `testpack` library. Do you have any recommendation how to deal with that situation? I would like to update, but I would very much like to run the regression test suite, too.
-"""]]
diff --git a/doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment b/doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment
deleted file mode 100644
index d45e66026f..0000000000
--- a/doc/news/version_3.20120227/comment_2_ea5075cfecc50d5da2364931ef7a02d1._comment
+++ /dev/null
@@ -1,10 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 2"
- date="2012-02-29T04:27:47Z"
- content="""
-Here's the patch that was used to make testpack build with 7.4 on Debian:
-
-
-"""]]
diff --git a/doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment b/doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment
deleted file mode 100644
index c3d2cab59a..0000000000
--- a/doc/news/version_3.20120229/comment_1_18158b9be2313f49509d59295c7d3c90._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="77.186.152.146"
- subject="How do you build the Crypto library with GHC 7.4.1?"
- date="2012-02-29T19:20:20Z"
- content="""
-`Crypto 4.2.4` doesn't seem to compile with GHC 7.4.1. How did you build that package?
-"""]]
diff --git a/doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment b/doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment
deleted file mode 100644
index b08712ee8f..0000000000
--- a/doc/news/version_3.20120229/comment_2_03436ddda42decf8cb1b4d5316d88a75._comment
+++ /dev/null
@@ -1,14 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 2"
- date="2012-02-29T22:54:01Z"
- content="""
-Probably this patch will help with Crypto:
-
-
-
-Or, there's the `ghc7.0` branch of git-annex in git, which can be used to build with the older, stable ghc.
-
-BTW, when asking, for help, a log of the build failure is always a good idea..
-"""]]
diff --git a/doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment b/doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment
deleted file mode 100644
index a31a182cca..0000000000
--- a/doc/news/version_3.20120229/comment_3_8f7f8d4758804f1b695925934219745a._comment
+++ /dev/null
@@ -1,42 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="77.186.165.208"
- subject="comment 3"
- date="2012-03-05T21:10:47Z"
- content="""
-Unfortunately, the patch you mentioned doesn't seem to address the problem. I'm getting the following compile error:
-
-    Data/Digest/SHA2.hs:111:4:
-        Could not deduce (Show a) arising from a use of `showHex'
-        from the context (Integral a)
-          bound by the instance declaration at Data/Digest/SHA2.hs:109:10-39
-        Possible fix:
-          add (Show a) to the context of the instance declaration
-        In the first argument of `(.)', namely `(showHex a)'
-        In the expression:
-          (showHex a)
-          . (' ' :)
-            . (showHex b)
-              . (' ' :)
-                . (showHex c)
-                  . (' ' :)
-                    . (showHex d)
-                      . (' ' :)
-                        . (showHex e)
-                          . (' ' :)
-                            . (showHex f) . (' ' :) . (showHex g) . (' ' :) . (showHex h)
-        In an equation for `showsPrec':
-            showsPrec _ (Hash8 a b c d e f g h)
-              = (showHex a)
-                . (' ' :)
-                  . (showHex b)
-                    . (' ' :)
-                      . (showHex c)
-                        . (' ' :)
-                          . (showHex d)
-                            . (' ' :)
-                              . (showHex e)
-                                . (' ' :)
-                                  . (showHex f) . (' ' :) . (showHex g) . (' ' :) . (showHex h)
-
-"""]]
diff --git a/doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment b/doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment
deleted file mode 100644
index a931c7874a..0000000000
--- a/doc/news/version_3.20120229/comment_4_cd90223f78571e5bdd3dfc07ab1369d7._comment
+++ /dev/null
@@ -1,9 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 4"
- date="2012-03-05T21:32:00Z"
- content="""
-Hmm, I was able to produce exactly the same build error, and then I downloaded the patch I linked to before, and did
-`patch -p1 < debian/patches/class-constraints.diff` and that fixed the build nicely.
-"""]]
diff --git a/doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment b/doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment
deleted file mode 100644
index afddb3fb0a..0000000000
--- a/doc/news/version_3.20120229/comment_5_7dbf131ff4611abbfc8fbf1ee0f66dbe._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://peter-simons.myopenid.com/"
- ip="77.186.165.208"
- subject="comment 5"
- date="2012-03-05T23:29:41Z"
- content="""
-I didn't realize that the patch adds a patch file to the source distribution (instead of, well, patching it). That additional level of indirection surprised me. Anyway, now I figured it out and `Crypto` compiles fine. Thanks!
-"""]]
diff --git a/doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment b/doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment
deleted file mode 100644
index eb5c7d257b..0000000000
--- a/doc/news/version_3.20120230/comment_1_b975cbd3a01ba5c2fa0f24fe739d3433._comment
+++ /dev/null
@@ -1,128 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawk_LOahrm_Cdg7io-_H0CNKkaxsRRQgRFo"
- nickname="Peter"
- subject="Test suite failure"
- date="2012-03-06T11:20:35Z"
- content="""
-I managed to compile this version of git-annex with GHC 7.4.1 on NixOS, but unfortunately the test suite fails during the `addurl` test:
-
-    Testing 0:quickcheck:0:prop_idempotent_deencode_git                           
-    Testing 0:quickcheck:1:prop_idempotent_deencode                               
-    Testing 0:quickcheck:2:prop_idempotent_fileKey                                
-    Testing 0:quickcheck:3:prop_idempotent_key_read_show                          
-    Testing 0:quickcheck:4:prop_idempotent_shellEscape                            
-    Testing 0:quickcheck:5:prop_idempotent_shellEscape_multiword                  
-    Testing 0:quickcheck:6:prop_idempotent_configEscape                           
-    Testing 0:quickcheck:7:prop_parentDir_basics                                  
-    Testing 0:quickcheck:8:prop_relPathDirToFile_basics                           
-    Testing 0:quickcheck:9:prop_relPathDirToFile_regressionTest                   
-    Testing 0:quickcheck:10:prop_cost_sane                                        
-    Testing 0:quickcheck:11:prop_hmacWithCipher_sane                              
-    Testing 0:quickcheck:12:prop_TimeStamp_sane                                   
-    Testing 0:quickcheck:13:prop_addLog_sane                                      
-    Testing 1:blackbox:0:git-annex init                                           
-    Testing 1:blackbox:1:git-annex add:0                                          
-    Testing 1:blackbox:1:git-annex add:1                                          
-    Testing 1:blackbox:1:git-annex add:2                                          
-    Testing 1:blackbox:2:git-annex reinject/fromkey                               
-    Testing 1:blackbox:3:git-annex unannex:0:no content                           
-    Testing 1:blackbox:3:git-annex unannex:1:with content                         
-    Testing 1:blackbox:4:git-annex drop:0:no remotes                              
-    Testing 1:blackbox:4:git-annex drop:1:with remote                             
-    Testing 1:blackbox:4:git-annex drop:2:untrusted remote                        
-    Testing 1:blackbox:5:git-annex get                                            
-    Testing 1:blackbox:6:git-annex move                                           
-    Testing 1:blackbox:7:git-annex copy                                           
-    Testing 1:blackbox:8:git-annex unlock/lock                                    
-    Testing 1:blackbox:9:git-annex edit/commit:0                                  
-    Cases: 55  Tried: 28  Errors: 0  Failures: 0add foo (checksum...) ok
-    ok
-    (Recording state in git...)
-    Testing 1:blackbox:9:git-annex edit/commit:1                                  
-    Testing 1:blackbox:10:git-annex fix                                           
-    Testing 1:blackbox:11:git-annex trust/untrust/semitrust/dead                  
-    Testing 1:blackbox:12:git-annex fsck:0                                        
-    Cases: 55  Tried: 32  Errors: 0  Failures: 0  Only 1 of 2 trustworthy copies exist of foo
-      Back it up with git-annex copy.
-      Only 1 of 2 trustworthy copies exist of sha1foo
-      Back it up with git-annex copy.
-      Bad file size (11 B larger); moved to /tmp/nix-build-jzvhzrdysy619y4vgmafryy9ck8mz7z7-git-annex-3.20120230.drv-0/git-annex/.t/tmprepo/.git/annex/bad/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
-      Bad file size (11 B larger); moved to /tmp/nix-build-jzvhzrdysy619y4vgmafryy9ck8mz7z7-git-annex-3.20120230.drv-0/git-annex/.t/tmprepo/.git/annex/bad/SHA1-s25--ee80d2cec57a3810db83b80e1b320df3a3721ffa
-    Testing 1:blackbox:12:git-annex fsck:1                                        
-    Testing 1:blackbox:12:git-annex fsck:2                                        
-    Cases: 55  Tried: 34  Errors: 0  Failures: 0  Only these untrusted locations may have copies of foo
-            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
-            17aab099-fcde-413a-8ef1-6acc09d7d081 -- here (.t/tmprepo)
-      Back it up to trusted locations with git-annex copy.
-      Only these untrusted locations may have copies of sha1foo
-            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
-      Back it up to trusted locations with git-annex copy.
-    Testing 1:blackbox:12:git-annex fsck:3                                        
-    Cases: 55  Tried: 35  Errors: 0  Failures: 0  Only 1 of 2 trustworthy copies exist of foo
-      Back it up with git-annex copy.
-      The following untrusted locations may also have copies: 
-            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
-      Only 1 of 2 trustworthy copies exist of sha1foo
-      Back it up with git-annex copy.
-      The following untrusted locations may also have copies: 
-            17575c68-d5cc-4e18-bc96-fdafe716d488 -- origin (test repo)
-    Testing 1:blackbox:13:git-annex migrate:0                                     
-    Testing 1:blackbox:13:git-annex migrate:1                                     
-    Testing 1:blackbox:14:git-annex unused/dropunused                             
-    Testing 1:blackbox:15:git-annex addurl                                        
-    Cases: 55  Tried: 39  Errors: 0  Failures: 0git-annex: connect: timeout (Connection timed out)
-    ### Failure in: 1:blackbox:15:git-annex addurl
-    addurl failed
-    Testing 1:blackbox:16:git-annex describe                                      
-    Testing 1:blackbox:17:git-annex find                                          
-    Cases: 55  Tried: 41  Errors: 0  Failures: 1foo
-    foo
-    sha1foo
-    sha1foo
-    Testing 1:blackbox:18:git-annex merge                                         
-    Testing 1:blackbox:19:git-annex status                                        
-    Cases: 55  Tried: 43  Errors: 0  Failures: 1{\"command\":\"status\",\"supported backends\":[\"SHA256\",\"SHA1\",\"SHA512\",\"SHA224\",\"SHA384\",\"SHA256E\",\"SHA1E\",\"SHA512E\",\"SHA224E\",\"SHA384E\",\"WORM\",\"URL\"],\"supported remote types\":[\"git\",\"S3\",\"bup\",\"directory\",\"rsync\",\"web\",\"hook\"],\"trusted repositories\":[],\"semitrusted repositories\":[{\"uuid\":\"00000000-0000-0000-0000-000000000001\",\"description\":\"web\",\"here\":false},{\"uuid\":\"17575c68-d5cc-4e18-bc96-fdafe716d488\",\"description\":\"origin (test repo)\",\"here\":false},{\"uuid\":\"5b9fe416-d6ed-4df7-af67-14fc5f2ea631\",\"description\":\".t/tmprepo\",\"here\":true}],\"untrusted repositories\":[],\"dead repositories\":[],\"local annex keys\":0,\"local annex size\":\"0 bytes\",\"known annex keys\":2,\"known annex size\":\"45 bytes\",\"success\":true}
-    Testing 1:blackbox:20:git-annex version                                       
-    Cases: 55  Tried: 44  Errors: 0  Failures: 1git-annex version: 3.20120230
-    local repository version: 3
-    default repository version: 3
-    supported repository versions: 3
-    upgrade supported from repository versions: 0 1 2
-    Testing 1:blackbox:21:git-annex sync                                          
-    Cases: 55  Tried: 45  Errors: 0  Failures: 1# On branch master
-    nothing to commit (working directory clean)
-    To /tmp/nix-build-jzvhzrdysy619y4vgmafryy9ck8mz7z7-git-annex-3.20120230.drv-0/git-annex/.t/repo
-       34fa270..7242932  git-annex -> git-annex
-     * [new branch]      master -> synced/master
-    Testing 1:blackbox:22:git-annex map                                           
-    Testing 1:blackbox:23:git-annex uninit                                        
-    Cases: 55  Tried: 47  Errors: 0  Failures: 1Switched to branch 'git-annex'
-    Switched to branch 'master'
-    Deleted branch git-annex (was e636789).
-    Testing 1:blackbox:24:git-annex upgrade                                       
-    Testing 1:blackbox:25:git-annex whereis                                       
-    Testing 1:blackbox:26:git-annex hook remote                                   
-    Testing 1:blackbox:27:git-annex directory remote                              
-    Testing 1:blackbox:28:git-annex rsync remote                                  
-    Cases: 55  Tried: 52  Errors: 0  Failures: 1sending incremental file list
-    af4/
-    af4/74c/
-    af4/74c/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77/
-    af4/74c/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77/SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
-              20 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/5)
-
-    sent 300 bytes  received 43 bytes  686.00 bytes/sec
-    total size is 20  speedup is 0.06
-    SHA256-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
-              20 100%    0.00kB/s    0:00:00 (xfer#1, to-check=0/1)
-
-    sent 160 bytes  received 31 bytes  382.00 bytes/sec
-    total size is 20  speedup is 0.10
-    Testing 1:blackbox:29:git-annex bup remote                                    
-    Testing 1:blackbox:30:git-annex crypto                                        
-    Cases: 55  Tried: 55  Errors: 0  Failures: 1
-    test: failed
-    ** test suite failed!
-
-Apparently, there is a network timeout? I see from the comments in `test.hs` that the test suite tries to avoid depending on network traffic, but is it possible, maybe, that the test tries to resolve a DNS name?
-"""]]
diff --git a/doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment b/doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment
deleted file mode 100644
index a4cc3073d7..0000000000
--- a/doc/news/version_3.20120230/comment_2_899de1196cd1ba4a393e4ef574d7aa5e._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://joey.kitenet.net/"
- nickname="joey"
- subject="comment 2"
- date="2012-03-06T17:22:54Z"
- content="""
-My mistake, addurl --fast used to avoid the network, so the test suite ran it, but then it was changed to always look up the file size. Removed from test suite.
-"""]]

From b261d09637a3c128a636a7b6741615b5d47b90f1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkRITTYYsN0TFKN7G5sZ6BWGZOTQ88Pz4s"
 
Date: Tue, 15 May 2012 00:14:10 +0000
Subject: [PATCH 3475/8313] Added a comment: cygwin

---
 ...comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment

diff --git a/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment b/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment
new file mode 100644
index 0000000000..fd5b6f5cd3
--- /dev/null
+++ b/doc/todo/windows_support/comment_1_3cc26ad8101a22e95a8c60cf0c4dedcc._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkRITTYYsN0TFKN7G5sZ6BWGZOTQ88Pz4s"
+ nickname="Zoltán"
+ subject="cygwin"
+ date="2012-05-15T00:14:08Z"
+ content="""
+What about [Cygwin](http://cygwin.com/)? It emulates POSIX fairly well under Windows (including signals, forking, fs (also things like /dev/null, /proc), unix file permissions), has all standard gnu utilities. It also emulates symlinks, but they are unfortunately incompatible with NTFS symlinks introduced in Vista [due to some stupid restrictions on Windows](http://cygwin.com/ml/cygwin/2009-10/msg00756.html).
+
+If git-annex could be modified to not require symlinks to work, the it would be a pretty neat solution (and you get a real shell, not some command.com on drugs (aka cmd.exe))
+"""]]

From 461367f23be5e5ff7925be96ad1291b794adfda1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Tue, 15 May 2012 02:42:31 +0000
Subject: [PATCH 3476/8313]

---
 ...er_version__39__s_file_content_without_doing_checkout.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn

diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn
new file mode 100644
index 0000000000..3ed022b481
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout.mdwn
@@ -0,0 +1,4 @@
+Hi,
+Is there any way I can move or copy file content of older version without doing checkout to that version, by passing commit hash as parameter in move command itself? 
+
+Thank you

From 1e590dcc47253b48d54807717d0cf747c659d93f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 15 May 2012 07:36:25 +0000
Subject: [PATCH 3477/8313] Added a comment

---
 .../comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment

diff --git a/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment b/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment
new file mode 100644
index 0000000000..6ea2677289
--- /dev/null
+++ b/doc/bugs/git_rename_detection_on_file_move/comment_10_5ec2f965c80cc5dd31ee3c4edb695664._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafael"
+ subject="comment 10"
+ date="2012-05-15T07:36:25Z"
+ content="""
+Won't git itself be fixed on this issue? It was on my plans to look into that, however I don't know how difficult it will be.
+"""]]

From 300d3cbdef2450b21c9eb22361161af37ca2f266 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
 
Date: Tue, 15 May 2012 07:59:58 +0000
Subject: [PATCH 3478/8313] Added a comment

---
 .../comment_1_f114b75b29123453758b493fae7f5167._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment

diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment
new file mode 100644
index 0000000000..a53c6bbd69
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_1_f114b75b29123453758b493fae7f5167._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnpdM9F8VbtQ_H5PaPMpGSxPe_d5L1eJ6w"
+ nickname="Rafael"
+ subject="comment 1"
+ date="2012-05-15T07:59:57Z"
+ content="""
+I had a similiar question in forum/new_microfeatures/. I would like to fetch/copy all the annexed content from a repo, be it on the current branch, another branch, or corresponds to an old version of a file. A command like \"git annex copy --all --from=source [path]\" would then ensure I have access to all the content I need even if I have later no longer access to source. Sure I could use rsync.
+"""]]

From a25237d2055c147d626267a099a70cc1498f7a8e Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Tue, 15 May 2012 17:00:10 +0000
Subject: [PATCH 3479/8313] Added a comment

---
 ...comment_2_e377b7614c2961b460a10e285f3db274._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment

diff --git a/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment
new file mode 100644
index 0000000000..18ff153ad0
--- /dev/null
+++ b/doc/forum/Moving_older_version__39__s_file_content_without_doing_checkout/comment_2_e377b7614c2961b460a10e285f3db274._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.51"
+ subject="comment 2"
+ date="2012-05-15T17:00:10Z"
+ content="""
+Yes, I think that [[todo/add_-all_option]] is the right approach for this. Seems unlikely you'd have some files' hashes handy without having them checked out, but operating on all content makes sense.
+
+That page discusses some problems implementing it for some commands, but should not pose a problem for `move`. It would also be possible to support `get` and `copy`, except `--auto` couldn't be used with `--all`. Even `fsck` could support it.
+"""]]

From e36808e167815eaa70a827ea703ed98ebb36da64 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 15 May 2012 14:18:51 -0400
Subject: [PATCH 3480/8313] Pass -a to cp even when it supports --reflink=auto,
 to preserve permissions.

Amoung other things, this makes unlocking a WORM backed file and then
re-adding it without making any changes not add a new object, as the
timestamp is preserved.
---
 Utility/CopyFile.hs | 14 +++++++-------
 debian/changelog    |  7 +++++++
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/Utility/CopyFile.hs b/Utility/CopyFile.hs
index 01639ef2a7..66b88e4f0c 100644
--- a/Utility/CopyFile.hs
+++ b/Utility/CopyFile.hs
@@ -1,6 +1,6 @@
 {- git-annex file copying
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -16,10 +16,10 @@ copyFileExternal :: FilePath -> FilePath -> IO Bool
 copyFileExternal src dest = do
 	whenM (doesFileExist dest) $
 		removeFile dest
-	boolSystem "cp" [params, File src, File dest]
+	boolSystem "cp" $ params ++ [File src, File dest]
 	where
-		params
-			| SysConfig.cp_reflink_auto = Params "--reflink=auto"
-			| SysConfig.cp_a = Params "-a"
-			| SysConfig.cp_p = Params "-p"
-			| otherwise = Params ""
+		params = map snd $ filter fst
+			[ (SysConfig.cp_reflink_auto, Param "--reflink=auto")
+			, (SysConfig.cp_a, Param "-a")
+			, (SysConfig.cp_p && not SysConfig.cp_a, Param "-p")
+			]
diff --git a/debian/changelog b/debian/changelog
index 6ebd6fabf0..a4e2b8b3e9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120512) UNRELEASED; urgency=low
+
+  * Pass -a to cp even when it supports --reflink=auto, to preserve
+    permissions.
+
+ -- Joey Hess   Tue, 15 May 2012 14:17:49 -0400
+
 git-annex (3.20120511) unstable; urgency=low
 
   * Rsync special remotes can be configured with shellescape=no

From a2be4265bf8207c785c7e996e2cc563ac91eb82e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 16 May 2012 15:19:39 -0400
Subject: [PATCH 3481/8313] add

---
 doc/bugs/GIT_DIR_support_incomplete.mdwn | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/GIT_DIR_support_incomplete.mdwn

diff --git a/doc/bugs/GIT_DIR_support_incomplete.mdwn b/doc/bugs/GIT_DIR_support_incomplete.mdwn
new file mode 100644
index 0000000000..d52871df29
--- /dev/null
+++ b/doc/bugs/GIT_DIR_support_incomplete.mdwn
@@ -0,0 +1,15 @@
+`GIT_DIR` support isn't right. Git does not look for `GIT_DIR/.git`;
+git-annex does.
+
+Also, to support this scenario, support for core.worktree needs to be added
+as well:
+
+	mkdir repo workdir
+	git --work-tree=$PWD/workdir --git-dir=$PWD/repo init
+	export GIT_DIR=$PWD/repo
+	git status
+	# ok
+	git annex init "new repo"
+	# fail
+
+--[[Joey]] 

From bb4f31a0ee496ffb83d31cc56f8827e47605d763 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 May 2012 16:38:26 -0400
Subject: [PATCH 3482/8313] Clean up handling of git directory and git
 worktree.

Baked into the code was an assumption that a repository's git directory
could be determined by adding ".git" to its work tree (or nothing for bare
repos). That fails when core.worktree, or GIT_DIR and GIT_WORK_TREE are
used to separate the two.

This was attacked at the type level, by storing the gitdir and worktree
separately, so Nothing for the worktree means a bare repo.

A complication arose because we don't learn where a repository is bare
until its configuration is read. So another Location type handles
repositories that have not had their config read yet. I am not entirely
happy with this being a Location type, rather than representing them
entirely separate from the Git type. The new code is not worse than the
old, but better types could enforce more safety.

Added support for core.worktree. Overriding it with -c isn't supported
because it's not really clear what to do if a git repo's config is read, is
not bare, and is then overridden to bare. What is the right git directory
in this case? I will worry about this if/when someone has a use case for
overriding core.worktree with -c. (See Git.Config.updateLocation)

Also removed and renamed some functions like gitDir and workTree that
misused git's terminology.

One minor regression is known: git annex add in a bare repository does not
print a nice error message, but runs git ls-files in a way that fails
earlier with a less nice error message. This is because before --work-tree
was always passed to git commands, even in a bare repo, while now it's not.
---
 Annex/Content.hs     |  3 +-
 Annex/Ssh.hs         |  4 +--
 Command/Log.hs       |  2 +-
 Command/Map.hs       |  8 ++---
 Command/Unused.hs    |  2 +-
 Config.hs            |  2 +-
 Git.hs               | 77 ++++++++++++++++++--------------------------
 Git/Command.hs       | 13 ++++----
 Git/Config.hs        | 62 +++++++++++++++++++++++++----------
 Git/Construct.hs     | 10 +++---
 Git/LsFiles.hs       |  2 +-
 Git/Types.hs         | 20 +++++++++---
 Init.hs              |  2 +-
 Locations.hs         | 14 +++-----
 Remote/Bup.hs        |  2 +-
 Remote/Helper/Ssh.hs |  2 +-
 Upgrade/V1.hs        |  4 +--
 Upgrade/V2.hs        |  2 +-
 Utility/Directory.hs | 14 ++++++++
 debian/changelog     |  1 +
 git-union-merge.hs   |  2 +-
 21 files changed, 144 insertions(+), 104 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 2142d1f09d..26b332e24c 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -34,6 +34,7 @@ import Common.Annex
 import Logs.Location
 import Annex.UUID
 import qualified Git
+import qualified Git.Config
 import qualified Annex
 import qualified Annex.Queue
 import qualified Annex.Branch
@@ -303,7 +304,7 @@ saveState oneshot = doSideAction $ do
 		ifM alwayscommit
 			( Annex.Branch.commit "update" , Annex.Branch.stage)
 	where
-		alwayscommit = fromMaybe True . Git.configTrue
+		alwayscommit = fromMaybe True . Git.Config.isTrue
 			<$> getConfig (annexConfig "alwayscommit") ""
 
 {- Downloads content from any of a list of urls. -}
diff --git a/Annex/Ssh.hs b/Annex/Ssh.hs
index f0824b1191..8bd4fe33ab 100644
--- a/Annex/Ssh.hs
+++ b/Annex/Ssh.hs
@@ -14,7 +14,7 @@ import qualified Data.Map as M
 
 import Common.Annex
 import Annex.LockPool
-import qualified Git
+import qualified Git.Config
 import Config
 import qualified Build.SysConfig as SysConfig
 import Annex.Perms
@@ -47,7 +47,7 @@ sshInfo (host, port) = ifM caching
 	)
 	where
 		caching = fromMaybe SysConfig.sshconnectioncaching 
-			. Git.configTrue
+			. Git.Config.isTrue
 			<$> getConfig (annexConfig "sshcaching") ""
 
 cacheParams :: FilePath -> [CommandParam]
diff --git a/Command/Log.hs b/Command/Log.hs
index d78b602067..aa39aea9c7 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -133,7 +133,7 @@ compareChanges format changes = concatMap diff $ zip changes (drop 1 changes)
  - *lot* for newish files. -}
 getLog :: Key -> [CommandParam] -> Annex [String]
 getLog key os = do
-	top <- fromRepo Git.workTree
+	top <- fromRepo Git.repoPath
 	p <- liftIO $ relPathCwdToFile top
 	let logfile = p  Logs.Location.logFile key
 	inRepo $ pipeNullSplit $
diff --git a/Command/Map.hs b/Command/Map.hs
index bdb86f95a5..86e9609a7e 100644
--- a/Command/Map.hs
+++ b/Command/Map.hs
@@ -156,14 +156,14 @@ absRepo :: Git.Repo -> Git.Repo -> Annex Git.Repo
 absRepo reference r
 	| Git.repoIsUrl reference = return $ Git.Construct.localToUrl reference r
 	| Git.repoIsUrl r = return r
-	| otherwise = liftIO $ Git.Construct.fromAbsPath =<< absPath (Git.workTree r)
+	| otherwise = liftIO $ Git.Construct.fromAbsPath =<< absPath (Git.repoPath r)
 
 {- Checks if two repos are the same. -}
 same :: Git.Repo -> Git.Repo -> Bool
 same a b
-	| both Git.repoIsSsh = matching Git.Url.authority && matching Git.workTree
+	| both Git.repoIsSsh = matching Git.Url.authority && matching Git.repoPath
 	| both Git.repoIsUrl && neither Git.repoIsSsh = matching show
-	| neither Git.repoIsSsh = matching Git.workTree
+	| neither Git.repoIsSsh = matching Git.repoPath
 	| otherwise = False
 		
 	where
@@ -210,7 +210,7 @@ tryScan r
 			where
 				sshcmd = cddir ++ " && " ++
 					"git config --null --list"
-				dir = Git.workTree r
+				dir = Git.repoPath r
 				cddir
 					| "/~" `isPrefixOf` dir =
 						let (userhome, reldir) = span (/= '/') (drop 1 dir)
diff --git a/Command/Unused.hs b/Command/Unused.hs
index f5ee452a80..1224d05457 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -231,7 +231,7 @@ withKeysReferenced' :: v -> (Key -> v -> Annex v) -> Annex v
 withKeysReferenced' initial a = go initial =<< files
 	where
 		files = do
-			top <- fromRepo Git.workTree
+			top <- fromRepo Git.repoPath
 			inRepo $ LsFiles.inRepo [top]
 		go v [] = return v
 		go v (f:fs) = do
diff --git a/Config.hs b/Config.hs
index 5f1ac8bb20..bb57ab675e 100644
--- a/Config.hs
+++ b/Config.hs
@@ -84,7 +84,7 @@ prop_cost_sane = False `notElem`
 
 {- Checks if a repo should be ignored. -}
 repoNotIgnored :: Git.Repo -> Annex Bool
-repoNotIgnored r = not . fromMaybe False . Git.configTrue
+repoNotIgnored r = not . fromMaybe False . Git.Config.isTrue
 	<$> getRemoteConfig r "ignore" ""
 
 {- If a value is specified, it is used; otherwise the default is looked up
diff --git a/Git.hs b/Git.hs
index 4278e9fcf2..7d64205634 100644
--- a/Git.hs
+++ b/Git.hs
@@ -3,7 +3,7 @@
  - This is written to be completely independant of git-annex and should be
  - suitable for other uses.
  -
- - Copyright 2010, 2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -17,19 +17,17 @@ module Git (
 	repoIsUrl,
 	repoIsSsh,
 	repoIsHttp,
+	repoIsLocal,
 	repoIsLocalBare,
 	repoDescribe,
 	repoLocation,
-	workTree,
-	gitDir,
-	configTrue,
+	repoPath,
+	localGitDir,
 	attributes,
 	hookPath,
 	assertLocal,
 ) where
 
-import qualified Data.Map as M
-import Data.Char
 import Network.URI (uriPath, uriScheme, unEscapeString)
 import System.Posix.Files
 
@@ -41,15 +39,34 @@ import Utility.FileMode
 repoDescribe :: Repo -> String
 repoDescribe Repo { remoteName = Just name } = name
 repoDescribe Repo { location = Url url } = show url
-repoDescribe Repo { location = Dir dir } = dir
+repoDescribe Repo { location = Local { worktree = Just dir } } = dir
+repoDescribe Repo { location = Local { gitdir = dir } } = dir
+repoDescribe Repo { location = LocalUnknown dir } = dir
 repoDescribe Repo { location = Unknown } = "UNKNOWN"
 
 {- Location of the repo, either as a path or url. -}
 repoLocation :: Repo -> String
 repoLocation Repo { location = Url url } = show url
-repoLocation Repo { location = Dir dir } = dir
+repoLocation Repo { location = Local { worktree = Just dir } } = dir
+repoLocation Repo { location = Local { gitdir = dir } } = dir
+repoLocation Repo { location = LocalUnknown dir } = dir
 repoLocation Repo { location = Unknown } = undefined
 
+{- Path to a repository. For non-bare, this is the worktree, for bare, 
+ - it's the gitdir, and for URL repositories, is the path on the remote
+ - host. -}
+repoPath :: Repo -> FilePath
+repoPath Repo { location = Url u } = unEscapeString $ uriPath u
+repoPath Repo { location = Local { worktree = Just d } } = d
+repoPath Repo { location = Local { gitdir = d } } = d
+repoPath Repo { location = LocalUnknown dir } = dir
+repoPath Repo { location = Unknown } = undefined
+
+{- Path to a local repository's .git directory. -}
+localGitDir :: Repo -> FilePath
+localGitDir Repo { location = Local { gitdir = d } } = d
+localGitDir _ = undefined
+
 {- Some code needs to vary between URL and normal repos,
  - or bare and non-bare, these functions help with that. -}
 repoIsUrl :: Repo -> Bool
@@ -74,11 +91,12 @@ repoIsHttp Repo { location = Url url }
 	| otherwise = False
 repoIsHttp _ = False
 
-configAvail ::Repo -> Bool
-configAvail Repo { config = c } = c /= M.empty
+repoIsLocal :: Repo -> Bool
+repoIsLocal Repo { location = Local { } } = True
+repoIsLocal _ = False
 
 repoIsLocalBare :: Repo -> Bool
-repoIsLocalBare r@(Repo { location = Dir _ }) = configAvail r && configBare r
+repoIsLocalBare Repo { location = Local { worktree = Nothing } } = True
 repoIsLocalBare _ = False
 
 assertLocal :: Repo -> a -> a
@@ -90,49 +108,18 @@ assertLocal repo action
 		]
 	| otherwise = action
 
-configBare :: Repo -> Bool
-configBare repo = maybe unknown (fromMaybe False . configTrue) $
-	M.lookup "core.bare" $ config repo
-	where
-		unknown = error $ "it is not known if git repo " ++
-			repoDescribe repo ++
-			" is a bare repository; config not read"
-
 {- Path to a repository's gitattributes file. -}
 attributes :: Repo -> FilePath
 attributes repo
-	| configBare repo = workTree repo ++ "/info/.gitattributes"
-	| otherwise = workTree repo ++ "/.gitattributes"
-
-{- Path to a repository's .git directory. -}
-gitDir :: Repo -> FilePath
-gitDir repo
-	| configBare repo = workTree repo
-	| otherwise = workTree repo  ".git"
+	| repoIsLocalBare repo = repoPath repo ++ "/info/.gitattributes"
+	| otherwise = repoPath repo ++ "/.gitattributes"
 
 {- Path to a given hook script in a repository, only if the hook exists
  - and is executable. -}
 hookPath :: String -> Repo -> IO (Maybe FilePath)
 hookPath script repo = do
-	let hook = gitDir repo  "hooks"  script
+	let hook = localGitDir repo  "hooks"  script
 	ifM (catchBoolIO $ isexecutable hook)
 		( return $ Just hook , return Nothing )
 	where
 		isexecutable f = isExecutable . fileMode <$> getFileStatus f
-
-{- Path to a repository's --work-tree, that is, its top.
- -
- - Note that for URL repositories, this is the path on the remote host. -}
-workTree :: Repo -> FilePath
-workTree Repo { location = Url u } = unEscapeString $ uriPath u
-workTree Repo { location = Dir d } = d
-workTree Repo { location = Unknown } = undefined
-
-{- Checks if a string from git config is a true value. -}
-configTrue :: String -> Maybe Bool
-configTrue s
-	| s' == "true" = Just True
-	| s' == "false" = Just False
-	| otherwise = Nothing
-	where
-		s' = map toLower s
diff --git a/Git/Command.hs b/Git/Command.hs
index bb82d13395..35f0838ba9 100644
--- a/Git/Command.hs
+++ b/Git/Command.hs
@@ -1,6 +1,6 @@
 {- running git commands
  -
- - Copyright 2010, 2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -18,11 +18,12 @@ import Git.Types
 
 {- Constructs a git command line operating on the specified repo. -}
 gitCommandLine :: [CommandParam] -> Repo -> [CommandParam]
-gitCommandLine params repo@(Repo { location = Dir _ } ) =
-	-- force use of specified repo via --git-dir and --work-tree
-	[ Param ("--git-dir=" ++ gitDir repo)
-	, Param ("--work-tree=" ++ workTree repo)
-	] ++ params
+gitCommandLine params Repo { location = l@(Local _ _ ) } = setdir : settree ++ params
+	where
+		setdir = Param $ "--git-dir=" ++ gitdir l
+		settree = case worktree l of
+			Nothing -> []
+			Just t -> [Param $ "--work-tree=" ++ t]
 gitCommandLine _ repo = assertLocal repo $ error "internal"
 
 {- Runs git in the specified repo. -}
diff --git a/Git/Config.hs b/Git/Config.hs
index 38b9ade455..e37b437071 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -1,15 +1,14 @@
 {- git repository configuration handling
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Git.Config where
 
-import System.Posix.Directory
-import Control.Exception (bracket_)
 import qualified Data.Map as M
+import Data.Char
 
 import Common
 import Git
@@ -30,17 +29,14 @@ getMaybe key repo = M.lookup key (config repo)
 
 {- Runs git config and populates a repo with its config. -}
 read :: Repo -> IO Repo
-read repo@(Repo { location = Dir d }) = bracketcd d $
-	{- Cannot use pipeRead because it relies on the config having
-	   been already read. Instead, chdir to the repo. -}
+read repo@(Repo { location = Local { gitdir = d } }) = read' repo d
+read repo@(Repo { location = LocalUnknown d }) = read' repo d
+read r = assertLocal r $ error "internal"
+{- Cannot use pipeRead because it relies on the config having
+   been already read. Instead, chdir to the repo. -}
+read' :: Repo -> FilePath -> IO Repo
+read' repo d = bracketCd d $
 	pOpen ReadFromPipe "git" ["config", "--null", "--list"] $ hRead repo
-	where
-		bracketcd to a = bracketcd' to a =<< getCurrentDirectory
-		bracketcd' to a cwd 
-			| dirContains to cwd = a
-			| otherwise = bracket_ (changeWorkingDirectory to) (changeWorkingDirectory cwd) a
-read r = assertLocal r $
-	error $ "internal error; trying to read config of " ++ show r
 
 {- Reads git config from a handle and populates a repo with it. -}
 hRead :: Repo -> Handle -> IO Repo
@@ -48,19 +44,42 @@ hRead repo h = do
 	val <- hGetContentsStrict h
 	store val repo
 
-{- Stores a git config into a repo, returning the new version of the repo.
- - The git config may be multiple lines, or a single line. Config settings
- - can be updated inrementally. -}
+{- Stores a git config into a Repo, returning the new version of the Repo.
+ - The git config may be multiple lines, or a single line.
+ - Config settings can be updated incrementally.
+ -}
 store :: String -> Repo -> IO Repo
 store s repo = do
 	let c = parse s
-	let repo' = repo
+	let repo' = updateLocation $ repo
 		{ config = (M.map Prelude.head c) `M.union` config repo
 		, fullconfig = M.unionWith (++) c (fullconfig repo)
 		}
+	print repo'
 	rs <- Git.Construct.fromRemotes repo'
 	return $ repo' { remotes = rs }
 
+{- Updates the location of a repo, based on its configuration.
+ -
+ - Git.Construct makes LocalUknown repos, of which only a directory is
+ - known. Once the config is read, this can be fixed up to a Local repo, 
+ - based on the core.bare and core.worktree settings.
+ -}
+updateLocation :: Repo -> Repo
+updateLocation r = go $ location r
+	where
+		go (LocalUnknown d)
+			| isbare = ret $ Local d Nothing
+			| otherwise = ret $ Local (d  ".git") (Just d)
+		go l@(Local {}) = ret l
+		go _ = r
+		isbare = fromMaybe False $ isTrue =<< getMaybe "core.bare" r
+		ret l = r { location = l' }
+			where
+				l' = maybe l (setworktree l) $
+					getMaybe "core.worktree" r
+		setworktree l t = l { worktree = Just t }
+
 {- Parses git config --list or git config --null --list output into a
  - config map. -}
 parse :: String -> M.Map String [String]
@@ -74,3 +93,12 @@ parse s
 		ls = lines s
 		sep c = M.fromListWith (++) . map (\(k,v) -> (k, [v])) .
 			map (separate (== c))
+
+{- Checks if a string from git config is a true value. -}
+isTrue :: String -> Maybe Bool
+isTrue s
+	| s' == "true" = Just True
+	| s' == "false" = Just False
+	| otherwise = Nothing
+	where
+		s' = map toLower s
diff --git a/Git/Construct.hs b/Git/Construct.hs
index 3f3ea97476..45ea0f64d8 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -1,6 +1,6 @@
 {- Construction of Git Repo objects
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -58,7 +58,7 @@ fromCurrent = do
 fromCwd :: IO Repo
 fromCwd = getCurrentDirectory >>= seekUp isRepoTop >>= maybe norepo makerepo
 	where
-		makerepo = newFrom . Dir
+		makerepo = newFrom . LocalUnknown
 		norepo = error "Not in a git repository."
 
 {- Local Repo constructor, accepts a relative or absolute path. -}
@@ -74,7 +74,7 @@ fromAbsPath dir
 	| otherwise =
 		error $ "internal error, " ++ dir ++ " is not absolute"
 	where
-		ret = newFrom . Dir
+		ret = newFrom . LocalUnknown
  		{- Git always looks for "dir.git" in preference to
 		 - to "dir", even if dir ends in a "/". -}
 		canondir = dropTrailingPathSeparator dir
@@ -122,7 +122,7 @@ localToUrl reference r
 		absurl =
 			Url.scheme reference ++ "//" ++
 			Url.authority reference ++
-			workTree r
+			repoPath r
 
 {- Calculates a list of a repo's configured remotes, by parsing its config. -}
 fromRemotes :: Repo -> IO [Repo]
@@ -191,7 +191,7 @@ fromRemoteLocation s repo = gen $ calcloc s
 fromRemotePath :: FilePath -> Repo -> IO Repo
 fromRemotePath dir repo = do
 	dir' <- expandTilde dir
-	fromAbsPath $ workTree repo  dir'
+	fromAbsPath $ repoPath repo  dir'
 
 {- Git remotes can have a directory that is specified relative
  - to the user's home directory, or that contains tilde expansions.
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index 201d76d1d4..06d4b9f44f 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -69,7 +69,7 @@ typeChanged' ps l repo = do
 	fs <- pipeNullSplit (prefix ++ ps ++ suffix) repo
 	-- git diff returns filenames relative to the top of the git repo;
 	-- convert to filenames relative to the cwd, like git ls-files.
-	let top = workTree repo
+	let top = repoPath repo
 	cwd <- getCurrentDirectory
 	return $ map (\f -> relPathDirToFile cwd $ top  f) fs
 	where
diff --git a/Git/Types.hs b/Git/Types.hs
index 6063ad213f..deb14ebd48 100644
--- a/Git/Types.hs
+++ b/Git/Types.hs
@@ -1,6 +1,6 @@
 {- git data types
  -
- - Copyright 2010,2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -10,9 +10,21 @@ module Git.Types where
 import Network.URI
 import qualified Data.Map as M
 
-{- There are two types of repositories; those on local disk and those
- - accessed via an URL. -}
-data RepoLocation = Dir FilePath | Url URI | Unknown
+{- Support repositories on local disk, and repositories accessed via an URL.
+ -
+ - Repos on local disk have a git directory, and unless bare, a worktree.
+ -
+ - A local repo may not have had its config read yet, in which case all
+ - that's known about it is its path.
+ -
+ - Finally, an Unknown repository may be known to exist, but nothing
+ - else known about it.
+ -}
+data RepoLocation
+	= Local { gitdir :: FilePath, worktree :: Maybe FilePath }
+	| LocalUnknown FilePath
+	| Url URI
+	| Unknown
 	deriving (Show, Eq)
 
 data Repo = Repo {
diff --git a/Init.hs b/Init.hs
index a0e16e8815..bddcc696e0 100644
--- a/Init.hs
+++ b/Init.hs
@@ -72,7 +72,7 @@ unlessBare :: Annex () -> Annex ()
 unlessBare = unlessM $ fromRepo Git.repoIsLocalBare
 
 preCommitHook :: Annex FilePath
-preCommitHook = () <$> fromRepo Git.gitDir <*> pure "hooks/pre-commit"
+preCommitHook = () <$> fromRepo Git.localGitDir <*> pure "hooks/pre-commit"
 
 preCommitScript :: String
 preCommitScript = 
diff --git a/Locations.hs b/Locations.hs
index 67abf2166a..46a85e0ee1 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -85,28 +85,24 @@ gitAnnexLocation key r
 	| Git.repoIsLocalBare r =
 		{- Bare repositories default to hashDirLower for new
 		 - content, as it's more portable. -}
-		check (map inrepo $ annexLocations key)
+		check $ map inrepo $ annexLocations key
 	| otherwise =
 		{- Non-bare repositories only use hashDirMixed, so
 		 - don't need to do any work to check if the file is
 		 - present. -}
-		return $ inrepo ".git"  annexLocation key hashDirMixed
+		return $ inrepo $ annexLocation key hashDirMixed
 	where
-		inrepo d = Git.workTree r  d
+		inrepo d = Git.localGitDir r  d
 		check locs@(l:_) = fromMaybe l <$> firstM doesFileExist locs
 		check [] = error "internal"
 
 {- The annex directory of a repository. -}
 gitAnnexDir :: Git.Repo -> FilePath
-gitAnnexDir r
-	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  annexDir
-	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  annexDir
+gitAnnexDir r = addTrailingPathSeparator $ Git.localGitDir r  annexDir
 
 {- The part of the annex directory where file contents are stored. -}
 gitAnnexObjectDir :: Git.Repo -> FilePath
-gitAnnexObjectDir r
-	| Git.repoIsLocalBare r = addTrailingPathSeparator $ Git.workTree r  objectDir
-	| otherwise = addTrailingPathSeparator $ Git.workTree r  ".git"  objectDir
+gitAnnexObjectDir r = addTrailingPathSeparator $ Git.localGitDir r  objectDir
 
 {- .git/annex/tmp/ is used for temp files -}
 gitAnnexTmpDir :: Git.Repo -> FilePath
diff --git a/Remote/Bup.hs b/Remote/Bup.hs
index 1081815944..3e7e9211f9 100644
--- a/Remote/Bup.hs
+++ b/Remote/Bup.hs
@@ -184,7 +184,7 @@ storeBupUUID u buprepo = do
 
 onBupRemote :: Git.Repo -> (FilePath -> [CommandParam] -> IO a) -> FilePath -> [CommandParam] -> Annex a
 onBupRemote r a command params = do
-	let dir = shellEscape (Git.workTree r)
+	let dir = shellEscape (Git.repoPath r)
 	sshparams <- sshToRepo r [Param $
 			"cd " ++ dir ++ " && " ++ unwords (command : toCommand params)]
 	liftIO $ a "ssh" sshparams
diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs
index 4c5eef0e6c..f6742b89f6 100644
--- a/Remote/Helper/Ssh.hs
+++ b/Remote/Helper/Ssh.hs
@@ -34,7 +34,7 @@ git_annex_shell r command params
 		return $ Just ("ssh", sshparams)
 	| otherwise = return Nothing
 	where
-		dir = Git.workTree r
+		dir = Git.repoPath r
 		shellcmd = "git-annex-shell"
 		shellopts = Param command : File dir : params
 		sshcmd uuid = unwords $
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index ddf0728b61..280742f062 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -82,7 +82,7 @@ moveContent = do
 updateSymlinks :: Annex ()
 updateSymlinks = do
 	showAction "updating symlinks"
-	top <- fromRepo Git.workTree
+	top <- fromRepo Git.repoPath
 	files <- inRepo $ LsFiles.inRepo [top]
 	forM_ files fixlink
 	where
@@ -236,4 +236,4 @@ stateDir :: FilePath
 stateDir = addTrailingPathSeparator ".git-annex"
 
 gitStateDir :: Git.Repo -> FilePath
-gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
+gitStateDir repo = addTrailingPathSeparator $ Git.repoPath repo  stateDir
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index c57b0bf685..202ba5b167 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -134,4 +134,4 @@ gitAttributesUnWrite repo = do
 stateDir :: FilePath
 stateDir = addTrailingPathSeparator ".git-annex"
 gitStateDir :: Git.Repo -> FilePath
-gitStateDir repo = addTrailingPathSeparator $ Git.workTree repo  stateDir
+gitStateDir repo = addTrailingPathSeparator $ Git.repoPath repo  stateDir
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index e6622d31ee..3041361dfd 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -15,11 +15,14 @@ import Control.Monad
 import Control.Monad.IfElse
 import System.FilePath
 import Control.Applicative
+import Control.Exception (bracket_)
+import System.Posix.Directory
 
 import Utility.SafeCommand
 import Utility.TempFile
 import Utility.Exception
 import Utility.Monad
+import Utility.Path
 
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
@@ -60,3 +63,14 @@ moveFile src dest = tryIO (rename src dest) >>= onrename
 			case r of
 				(Left _) -> return False
 				(Right s) -> return $ isDirectory s
+
+{- Runs an action in another directory. -}
+bracketCd :: FilePath -> IO a -> IO a
+bracketCd dir a = go =<< getCurrentDirectory
+	where
+		go cwd
+			| dirContains dir cwd = a
+			| otherwise = bracket_
+				(changeWorkingDirectory dir)
+				(changeWorkingDirectory cwd)
+				a
diff --git a/debian/changelog b/debian/changelog
index a4e2b8b3e9..4e61445c8d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20120512) UNRELEASED; urgency=low
 
   * Pass -a to cp even when it supports --reflink=auto, to preserve
     permissions.
+  * Clean up handling of git directory and git worktree.
 
  -- Joey Hess   Tue, 15 May 2012 14:17:49 -0400
 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index f44136bfc1..182d8cf790 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -22,7 +22,7 @@ usage :: IO a
 usage = error $ "bad parameters\n\n" ++ header
 
 tmpIndex :: Git.Repo -> FilePath
-tmpIndex g = Git.gitDir g  "index.git-union-merge"
+tmpIndex g = Git.localGitDir g  "index.git-union-merge"
 
 setup :: Git.Repo -> IO ()
 setup = cleanup -- idempotency

From eb6cb1b87f2d7016ddd4386e2a3bb20d8ea3c036 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 May 2012 18:20:53 -0400
Subject: [PATCH 3483/8313] Add support for core.worktree, and fix support for
 GIT_WORK_TREE and GIT_DIR.

The environment needs to override git-config. Changed when git config is
read, and avoid rereading it once it's been read.

chdir for both worktree settings.
---
 Annex.hs                                 |  3 +-
 Git/Config.hs                            | 15 ++++---
 Git/Construct.hs                         | 27 +-----------
 Git/CurrentRepo.hs                       | 54 ++++++++++++++++++++++++
 GitAnnex.hs                              |  4 +-
 Remote/Git.hs                            |  8 +---
 debian/changelog                         |  2 +
 doc/bugs/GIT_DIR_support_incomplete.mdwn |  2 +
 git-union-merge.hs                       |  4 +-
 test.hs                                  |  4 +-
 10 files changed, 79 insertions(+), 44 deletions(-)
 create mode 100644 Git/CurrentRepo.hs

diff --git a/Annex.hs b/Annex.hs
index d1509d4bd2..a9cc680125 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -124,7 +124,8 @@ newState gitrepo = AnnexState
 	, cleanup = M.empty
 	}
 
-{- Create and returns an Annex state object for the specified git repo. -}
+{- Makes an Annex state object for the specified git repo.
+ - Ensures the config is read, if it was not already. -}
 new :: Git.Repo -> IO AnnexState
 new gitrepo = newState <$> Git.Config.read gitrepo
 
diff --git a/Git/Config.hs b/Git/Config.hs
index e37b437071..2fa685a11d 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -27,16 +27,20 @@ getList key repo = M.findWithDefault [] key (fullconfig repo)
 getMaybe :: String -> Repo -> Maybe String
 getMaybe key repo = M.lookup key (config repo)
 
-{- Runs git config and populates a repo with its config. -}
+{- Runs git config and populates a repo with its config.
+ - Cannot use pipeRead because it relies on the config having been already
+ - read. Instead, chdir to the repo.
+ -}
 read :: Repo -> IO Repo
 read repo@(Repo { location = Local { gitdir = d } }) = read' repo d
 read repo@(Repo { location = LocalUnknown d }) = read' repo d
 read r = assertLocal r $ error "internal"
-{- Cannot use pipeRead because it relies on the config having
-   been already read. Instead, chdir to the repo. -}
 read' :: Repo -> FilePath -> IO Repo
-read' repo d = bracketCd d $
-	pOpen ReadFromPipe "git" ["config", "--null", "--list"] $ hRead repo
+read' repo@(Repo { config = c}) d
+	| c == M.empty = bracketCd d $
+		pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
+			hRead repo
+	| otherwise = return repo -- config already read
 
 {- Reads git config from a handle and populates a repo with it. -}
 hRead :: Repo -> Handle -> IO Repo
@@ -55,7 +59,6 @@ store s repo = do
 		{ config = (M.map Prelude.head c) `M.union` config repo
 		, fullconfig = M.unionWith (++) c (fullconfig repo)
 		}
-	print repo'
 	rs <- Git.Construct.fromRemotes repo'
 	return $ repo' { remotes = rs }
 
diff --git a/Git/Construct.hs b/Git/Construct.hs
index 45ea0f64d8..b809d7318a 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -6,7 +6,6 @@
  -}
 
 module Git.Construct (
-	fromCurrent,
 	fromCwd,
 	fromAbsPath,
 	fromPath,
@@ -21,8 +20,6 @@ module Git.Construct (
 ) where
 
 import System.Posix.User
-import System.Posix.Env (getEnv, unsetEnv)
-import System.Posix.Directory (changeWorkingDirectory)
 import qualified Data.Map as M hiding (map, split)
 import Network.URI
 
@@ -31,28 +28,6 @@ import Git.Types
 import Git
 import qualified Git.Url as Url
 
-{- Finds the current git repository.
- -
- - GIT_DIR can override the location of the .git directory.
- -
- - When GIT_WORK_TREE is set, chdir to it, so that anything using
- - this repository runs in the right location. However, this chdir is
- - done after determining GIT_DIR; git does not let GIT_WORK_TREE
- - influence the git directory.
- -
- - Both environment variables are unset, to avoid confusing other git
- - commands that also look at them. This would particularly be a problem
- - when GIT_DIR is relative and we chdir for GIT_WORK_TREE. Instead,
- - the Git module passes --work-tree and --git-dir to git commands it runs.
- -}
-fromCurrent :: IO Repo
-fromCurrent = do
-	r <- maybe fromCwd fromPath =<< getEnv "GIT_DIR"
-	maybe noop changeWorkingDirectory =<< getEnv "GIT_WORK_TREE"
-	unsetEnv "GIT_DIR"
-	unsetEnv "GIT_WORK_TREE"
-	return r
-
 {- Finds the git repository used for the Cwd, which may be in a parent
  - directory. -}
 fromCwd :: IO Repo
@@ -251,3 +226,5 @@ newFrom l = return Repo
 	, remotes = []
 	, remoteName = Nothing
 	}
+
+
diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs
new file mode 100644
index 0000000000..4325f452c0
--- /dev/null
+++ b/Git/CurrentRepo.hs
@@ -0,0 +1,54 @@
+{- The current git repository.
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.CurrentRepo where
+
+import System.Posix.Directory (changeWorkingDirectory)
+import System.Posix.Env (getEnv, unsetEnv)
+
+import Common
+import Git.Types
+import Git.Construct
+import qualified Git.Config
+
+{- Gets the current git repository.
+ -
+ - Honors GIT_DIR and GIT_WORK_TREE.
+ - Both environment variables are unset, to avoid confusing other git
+ - commands that also look at them. Instead, the Git module passes
+ - --work-tree and --git-dir to git commands it runs.
+ -
+ - When GIT_WORK_TREE or core.worktree are set, changes the working
+ - directory if necessary to ensure it is within the repository's work
+ - tree. While not needed for git commands, this is useful for anything
+ - else that looks for files in the worktree.
+ -}
+get :: IO Repo
+get = do
+	gd <- takeenv "GIT_DIR"
+	r <- configure gd =<< maybe fromCwd fromPath gd
+	wt <- maybe (worktree $ location r) Just <$> takeenv "GIT_WORK_TREE"
+	case wt of
+		Nothing -> return r
+		Just d -> do
+			changeWorkingDirectory d
+			return $ addworktree wt r
+	where
+		takeenv s = do
+			v <- getEnv s
+			when (isJust v) $
+				unsetEnv s
+			return v
+		configure Nothing r = Git.Config.read r
+		configure (Just d) r = do
+			r' <- Git.Config.read r
+			-- Let GIT_DIR override the default gitdir.
+			return $ changelocation r' $
+				Local { gitdir = d, worktree = worktree (location r') }
+		addworktree w r = changelocation r $
+			Local { gitdir = gitdir (location r), worktree = w }
+		changelocation r l = r { location = l }
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 0e707b1868..9910e33d21 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -11,7 +11,7 @@ import System.Console.GetOpt
 
 import Common.Annex
 import qualified Git.Config
-import qualified Git.Construct
+import qualified Git.CurrentRepo
 import CmdLine
 import Command
 import Types.TrustLevel
@@ -133,4 +133,4 @@ header :: String
 header = "Usage: git-annex command [option ..]"
 
 run :: [String] -> IO ()
-run args = dispatch True args cmds options header Git.Construct.fromCurrent
+run args = dispatch True args cmds options header Git.CurrentRepo.get
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 35928b96cb..79439b784f 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -179,12 +179,8 @@ repoAvail r
  - monad using that repository. -}
 onLocal :: Git.Repo -> Annex a -> IO a
 onLocal r a = do
-	-- Avoid re-reading the repository's configuration if it was
-	-- already read.
-	state <- if M.null $ Git.config r
-		then Annex.new r
-		else return $ Annex.newState r
-	Annex.eval state $ do
+	s <- Annex.new r
+	Annex.eval s $ do
 		-- No need to update the branch; its data is not used
 		-- for anything onLocal is used to do.
 		Annex.BranchState.disableUpdate
diff --git a/debian/changelog b/debian/changelog
index 4e61445c8d..586077878b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,8 @@ git-annex (3.20120512) UNRELEASED; urgency=low
   * Pass -a to cp even when it supports --reflink=auto, to preserve
     permissions.
   * Clean up handling of git directory and git worktree.
+  * Add support for core.worktree, and fix support for GIT_WORK_TREE and
+    GIT_DIR.
 
  -- Joey Hess   Tue, 15 May 2012 14:17:49 -0400
 
diff --git a/doc/bugs/GIT_DIR_support_incomplete.mdwn b/doc/bugs/GIT_DIR_support_incomplete.mdwn
index d52871df29..1b9738c4f7 100644
--- a/doc/bugs/GIT_DIR_support_incomplete.mdwn
+++ b/doc/bugs/GIT_DIR_support_incomplete.mdwn
@@ -13,3 +13,5 @@ as well:
 	# fail
 
 --[[Joey]] 
+
+> [[fixed|done]] --[[Joey]] 
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 182d8cf790..2c2e7a46bd 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -10,7 +10,7 @@ import System.Environment
 import Common
 import qualified Git.UnionMerge
 import qualified Git.Config
-import qualified Git.Construct
+import qualified Git.CurrentRepo
 import qualified Git.Branch
 import qualified Git.Index
 import qualified Git
@@ -40,7 +40,7 @@ parseArgs = do
 main :: IO ()
 main = do
 	[aref, bref, newref] <- map Git.Ref <$> parseArgs
-	g <- Git.Config.read =<< Git.Construct.fromCurrent
+	g <- Git.Config.read =<< Git.CurrentRepo.get
 	_ <- Git.Index.override $ tmpIndex g
 	setup g
 	Git.UnionMerge.merge aref bref g
diff --git a/test.hs b/test.hs
index 1952a39a9b..c52a88d66f 100644
--- a/test.hs
+++ b/test.hs
@@ -26,7 +26,7 @@ import qualified Annex
 import qualified Annex.UUID
 import qualified Backend
 import qualified Git.Config
-import qualified Git.Construct
+import qualified Git.CurrentRepo
 import qualified Git.Filename
 import qualified Locations
 import qualified Types.Backend
@@ -721,7 +721,7 @@ git_annex_expectoutput command params expected = do
 -- are not run; this should only be used for actions that query state.
 annexeval :: Types.Annex a -> IO a
 annexeval a = do
-	s <- Annex.new =<< Git.Config.read =<< Git.Construct.fromCurrent
+	s <- Annex.new =<< Git.CurrentRepo.get
 	Annex.eval s $ do
 		Annex.setOutput Types.Messages.QuietOutput
 		a

From a1885bd11607d8668d70f81eaafa25e5341e8e8c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 18 May 2012 18:30:50 -0400
Subject: [PATCH 3484/8313] make GIT_DIR, GIT_WORK_TREE absolute

GIT_DIR is set to something relative, like ".git" in the pre-commit hook.
But internally all the directories are assumed to be absolute.
---
 Git/CurrentRepo.hs | 10 ++++++----
 test.hs            |  1 -
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs
index 4325f452c0..746904caf2 100644
--- a/Git/CurrentRepo.hs
+++ b/Git/CurrentRepo.hs
@@ -29,20 +29,22 @@ import qualified Git.Config
  -}
 get :: IO Repo
 get = do
-	gd <- takeenv "GIT_DIR"
+	gd <- pathenv "GIT_DIR"
 	r <- configure gd =<< maybe fromCwd fromPath gd
-	wt <- maybe (worktree $ location r) Just <$> takeenv "GIT_WORK_TREE"
+	wt <- maybe (worktree $ location r) Just <$> pathenv "GIT_WORK_TREE"
 	case wt of
 		Nothing -> return r
 		Just d -> do
 			changeWorkingDirectory d
 			return $ addworktree wt r
 	where
-		takeenv s = do
+		pathenv s = do
 			v <- getEnv s
 			when (isJust v) $
 				unsetEnv s
-			return v
+			case v of
+				Nothing -> return Nothing
+				Just d -> Just <$> absPath d
 		configure Nothing r = Git.Config.read r
 		configure (Just d) r = do
 			r' <- Git.Config.read r
diff --git a/test.hs b/test.hs
index c52a88d66f..9a0fce873e 100644
--- a/test.hs
+++ b/test.hs
@@ -25,7 +25,6 @@ import qualified Utility.SafeCommand
 import qualified Annex
 import qualified Annex.UUID
 import qualified Backend
-import qualified Git.Config
 import qualified Git.CurrentRepo
 import qualified Git.Filename
 import qualified Locations

From 0093a456e869b5250735ef4ec790faac115d9142 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 May 2012 10:22:43 -0400
Subject: [PATCH 3485/8313] test suite saved my bacon

git config reading memoization shouldn't be used when changing config
---
 Config.hs     |  3 +--
 Git/Config.hs | 31 ++++++++++++++++++++-----------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/Config.hs b/Config.hs
index bb57ab675e..f579e40b21 100644
--- a/Config.hs
+++ b/Config.hs
@@ -21,8 +21,7 @@ data ConfigKey = ConfigKey String
 setConfig :: ConfigKey -> String -> Annex ()
 setConfig (ConfigKey key) value = do
 	inRepo $ Git.Command.run "config" [Param key, Param value]
-	-- re-read git config and update the repo's state
-	newg <- inRepo Git.Config.read
+	newg <- inRepo Git.Config.reRead
 	Annex.changeState $ \s -> s { Annex.repo = newg }
 
 {- Unsets a git config setting. (Leaves it in state currently.) -}
diff --git a/Git/Config.hs b/Git/Config.hs
index 2fa685a11d..38e1ca4be7 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -28,19 +28,28 @@ getMaybe :: String -> Repo -> Maybe String
 getMaybe key repo = M.lookup key (config repo)
 
 {- Runs git config and populates a repo with its config.
- - Cannot use pipeRead because it relies on the config having been already
+ - Avoids re-reading config when run repeatedly. -}
+read :: Repo -> IO Repo
+read repo@(Repo { config = c })
+	| c == M.empty = read' repo
+	| otherwise = return repo
+
+{- Reads config even if it was read before. -}
+reRead :: Repo -> IO Repo
+reRead = read'
+
+{- Cannot use pipeRead because it relies on the config having been already
  - read. Instead, chdir to the repo.
  -}
-read :: Repo -> IO Repo
-read repo@(Repo { location = Local { gitdir = d } }) = read' repo d
-read repo@(Repo { location = LocalUnknown d }) = read' repo d
-read r = assertLocal r $ error "internal"
-read' :: Repo -> FilePath -> IO Repo
-read' repo@(Repo { config = c}) d
-	| c == M.empty = bracketCd d $
-		pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
-			hRead repo
-	| otherwise = return repo -- config already read
+read' :: Repo -> IO Repo
+read' repo = go repo
+	where
+		go Repo { location = Local { gitdir = d } } = git_config d
+		go Repo { location = LocalUnknown d } = git_config d
+		go _ = assertLocal repo $ error "internal"
+		git_config d = bracketCd d $
+			pOpen ReadFromPipe "git" ["config", "--null", "--list"] $
+				hRead repo
 
 {- Reads git config from a handle and populates a repo with it. -}
 hRead :: Repo -> Handle -> IO Repo

From 9d9814477601217c8d39d75cc03e27ee4d734de3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 May 2012 10:37:28 -0400
Subject: [PATCH 3486/8313] avoid chdir when already inside worktree

---
 Git/CurrentRepo.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs
index 746904caf2..a40b412f5f 100644
--- a/Git/CurrentRepo.hs
+++ b/Git/CurrentRepo.hs
@@ -35,7 +35,9 @@ get = do
 	case wt of
 		Nothing -> return r
 		Just d -> do
-			changeWorkingDirectory d
+			cwd <- getCurrentDirectory
+			unless (d `dirContains` cwd) $
+				changeWorkingDirectory d
 			return $ addworktree wt r
 	where
 		pathenv s = do

From ebbd24e5ed6bbb41305d8105d7fac355dcf15bbc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 19 May 2012 10:51:22 -0400
Subject: [PATCH 3487/8313] more worktree improvements

Avoid more expensive code path when no core.worktree is configured.

Don't change worktree when reading config if one is already set.
This could happen if GIT_CORE_WORKTREE is set, and the repo also has
core.worktree, and the config is reread. Now GIT_CORE_WORKTREE will
prevail.
---
 Git/Config.hs      | 26 ++++++++++++++------------
 Git/CurrentRepo.hs |  2 +-
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/Git/Config.hs b/Git/Config.hs
index 38e1ca4be7..dab1cdf5ea 100644
--- a/Git/Config.hs
+++ b/Git/Config.hs
@@ -78,19 +78,15 @@ store s repo = do
  - based on the core.bare and core.worktree settings.
  -}
 updateLocation :: Repo -> Repo
-updateLocation r = go $ location r
+updateLocation r@(Repo { location = LocalUnknown d })
+	| isBare r = newloc $ Local d Nothing
+	| otherwise = newloc $ Local (d  ".git") (Just d)
 	where
-		go (LocalUnknown d)
-			| isbare = ret $ Local d Nothing
-			| otherwise = ret $ Local (d  ".git") (Just d)
-		go l@(Local {}) = ret l
-		go _ = r
-		isbare = fromMaybe False $ isTrue =<< getMaybe "core.bare" r
-		ret l = r { location = l' }
-			where
-				l' = maybe l (setworktree l) $
-					getMaybe "core.worktree" r
-		setworktree l t = l { worktree = Just t }
+		newloc l = r { location = getworktree l }
+		getworktree l = case workTree r of
+			Nothing -> l
+			wt -> l { worktree = wt }
+updateLocation r = r
 
 {- Parses git config --list or git config --null --list output into a
  - config map. -}
@@ -114,3 +110,9 @@ isTrue s
 	| otherwise = Nothing
 	where
 		s' = map toLower s
+
+isBare :: Repo -> Bool
+isBare r = fromMaybe False $ isTrue =<< getMaybe "core.bare" r
+
+workTree :: Repo -> Maybe FilePath
+workTree = getMaybe "core.worktree"
diff --git a/Git/CurrentRepo.hs b/Git/CurrentRepo.hs
index a40b412f5f..de11ce217c 100644
--- a/Git/CurrentRepo.hs
+++ b/Git/CurrentRepo.hs
@@ -31,7 +31,7 @@ get :: IO Repo
 get = do
 	gd <- pathenv "GIT_DIR"
 	r <- configure gd =<< maybe fromCwd fromPath gd
-	wt <- maybe (worktree $ location r) Just <$> pathenv "GIT_WORK_TREE"
+	wt <- maybe (Git.Config.workTree r) Just <$> pathenv "GIT_WORK_TREE"
 	case wt of
 		Nothing -> return r
 		Just d -> do

From 37ef39c9295fd5f036264791e201262b0e641f85 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 20 May 2012 00:14:56 -0400
Subject: [PATCH 3488/8313] suppress "(Recording state in git)" message when
 committing change to remote state

This was shown redundantly for a tricky reason -- while it runs
inside a doSideAction block that would appear to supress it,
the action being run is in a different state monad; for the remote,
and so the suppression doesn't work.

Always suppressing the message when committing to a local remote is
ok do to though -- it mirrors the /dev/nulling of the git annex shell commit
output. And it turns out that any time there is a git-annex branch state
change to commit on the remote, the local repo has also had a similar
change made, and so the message has been shown already.
---
 Messages.hs   | 12 ++++++++++--
 Remote/Git.hs |  3 ++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/Messages.hs b/Messages.hs
index 4330f7c09f..96bf3ae4b5 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -14,6 +14,7 @@ module Messages (
 	MeterUpdate,
 	showSideAction,
 	doSideAction,
+	doQuietSideAction,
 	showStoringStateAction,
 	showOutput,
 	showLongNote,
@@ -91,12 +92,19 @@ showSideAction m = Annex.getState Annex.output >>= go
 showStoringStateAction :: Annex ()
 showStoringStateAction = showSideAction "Recording state in git"
 
+{- Performs an action, supressing showSideAction messages. -}
+doQuietSideAction :: Annex a -> Annex a
+doQuietSideAction = doSideAction' InBlock
+
 {- Performs an action, that may call showSideAction multiple times.
  - Only the first will be displayed. -}
 doSideAction :: Annex a -> Annex a
-doSideAction a = do
+doSideAction = doSideAction' StartBlock
+
+doSideAction' :: SideActionBlock -> Annex a -> Annex a
+doSideAction' b a = do
 	o <- Annex.getState Annex.output
-	set $ o { sideActionBlock = StartBlock }
+	set $ o { sideActionBlock = b }
 	set o `after` a
 	where
 		set o = Annex.changeState $ \s -> s {  Annex.output = o }
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 79439b784f..cf7542d74e 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -310,7 +310,8 @@ commitOnCleanup r a = go `after` a
 		go = Annex.addCleanup (Git.repoLocation r) cleanup
 		cleanup
 			| not $ Git.repoIsUrl r = liftIO $ onLocal r $
-				Annex.Branch.commit "update"
+				doQuietSideAction $
+					Annex.Branch.commit "update"
 			| otherwise = void $ do
 				Just (shellcmd, shellparams) <-
 					git_annex_shell r "commit" []

From ab07762ddba62a2fcf04585a3549b503bc4bf1a9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 May 2012 11:27:22 -0400
Subject: [PATCH 3489/8313] releasing version 3.20120522

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 586077878b..99594c351c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120512) UNRELEASED; urgency=low
+git-annex (3.20120522) unstable; urgency=low
 
   * Pass -a to cp even when it supports --reflink=auto, to preserve
     permissions.
@@ -6,7 +6,7 @@ git-annex (3.20120512) UNRELEASED; urgency=low
   * Add support for core.worktree, and fix support for GIT_WORK_TREE and
     GIT_DIR.
 
- -- Joey Hess   Tue, 15 May 2012 14:17:49 -0400
+ -- Joey Hess   Tue, 22 May 2012 11:16:13 -0400
 
 git-annex (3.20120511) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index f87e49a200..e12cbb1777 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120511
+Version: 3.20120522
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From d0da2ce5eaf35a9895a80a08359df6bb9cbb451a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 22 May 2012 11:28:09 -0400
Subject: [PATCH 3490/8313] add news item for git-annex 3.20120522

---
 doc/news/version_3.20120405.mdwn | 5 -----
 doc/news/version_3.20120522.mdwn | 7 +++++++
 2 files changed, 7 insertions(+), 5 deletions(-)
 delete mode 100644 doc/news/version_3.20120405.mdwn
 create mode 100644 doc/news/version_3.20120522.mdwn

diff --git a/doc/news/version_3.20120405.mdwn b/doc/news/version_3.20120405.mdwn
deleted file mode 100644
index c5a7ecc22a..0000000000
--- a/doc/news/version_3.20120405.mdwn
+++ /dev/null
@@ -1,5 +0,0 @@
-git-annex 3.20120405 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Rewrote free disk space checking code, moving the portability
-     handling into a small C library.
-   * status: Display amount of free disk space."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120522.mdwn b/doc/news/version_3.20120522.mdwn
new file mode 100644
index 0000000000..55c45900c1
--- /dev/null
+++ b/doc/news/version_3.20120522.mdwn
@@ -0,0 +1,7 @@
+git-annex 3.20120522 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * Pass -a to cp even when it supports --reflink=auto, to preserve
+     permissions.
+   * Clean up handling of git directory and git worktree.
+   * Add support for core.worktree, and fix support for GIT\_WORK\_TREE and
+     GIT\_DIR."""]]
\ No newline at end of file

From 1b1703c84cd3ee9c58bff6a7f586938670ac3b5f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk5cj-itfFHq_yhJHdzk3QOPp-PNW_MjPU"
 
Date: Wed, 23 May 2012 19:30:22 +0000
Subject: [PATCH 3491/8313] Added a comment: +1 Cygwin

---
 ...mment_2_8acae818ce468967499050bbe3c532ea._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment

diff --git a/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment b/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment
new file mode 100644
index 0000000000..e37a555756
--- /dev/null
+++ b/doc/todo/windows_support/comment_2_8acae818ce468967499050bbe3c532ea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk5cj-itfFHq_yhJHdzk3QOPp-PNW_MjPU"
+ nickname="Michael"
+ subject="+1 Cygwin"
+ date="2012-05-23T19:30:21Z"
+ content="""
+Windows support is a must. In my experience, binary file means proprietary editor, which means Windows.
+
+Unfortunately, there's not much overlap between people who use graphical editors in Windows all day vs. people who are willing to tolerate Cygwin's setup.exe, compile a Haskell program, learn git and git-annex's 90-odd subcommands, and use a mintty terminal to manage their repository, especially now that there's a sexy GitHub app for Windows.
+
+That aside, I think Windows-based content producers are still *the* audience for git-annex. First Windows support, then a GUI, then the world.
+"""]]

From 66580a8b7ab4aad30a3d615063ddd0efbd35df65 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 23 May 2012 19:36:14 -0400
Subject: [PATCH 3492/8313] fix link

---
 doc/design/encryption.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/encryption.mdwn b/doc/design/encryption.mdwn
index 647683bd9f..b7acbb732a 100644
--- a/doc/design/encryption.mdwn
+++ b/doc/design/encryption.mdwn
@@ -20,7 +20,7 @@ with more than one encryption backend in mind helps future-proofing.
 
 [[!template id=note text="""
 The basis of this scheme was originally developed by Lars Wirzenius et al
-[for Obnam](http://braawi.org/obnam/encryption/).
+[for Obnam](http://liw.fi/obnam/encryption/).
 """]]
 
 Data is encrypted by gpg, using a symmetric cipher.

From 09fbc215e8f7cf7aec85c3a68e8ad8c5c3b84b66 Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~ojwb" 
Date: Thu, 24 May 2012 05:07:07 +0000
Subject: [PATCH 3493/8313] typo fixes

---
 doc/todo/automatic_bookkeeping_watch_command.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/todo/automatic_bookkeeping_watch_command.mdwn b/doc/todo/automatic_bookkeeping_watch_command.mdwn
index d7a3517a19..4b688b839e 100644
--- a/doc/todo/automatic_bookkeeping_watch_command.mdwn
+++ b/doc/todo/automatic_bookkeeping_watch_command.mdwn
@@ -4,10 +4,10 @@ It would run, in the background, watching via inotify for changes, and
 automatically annexing new files, etc.
 
 The blue sky goal would be something automated like dropbox, except fully
-distributed. All files put into the repository would propigate out
+distributed. All files put into the repository would propagate out
 to all the other clones of it, as network links allow. Note that while
 dropbox allows modifying files, git-annex freezes them upon creation,
-so this would not be 100% equivilant to dropbox. --[[Joey]]
+so this would not be 100% equivalent to dropbox. --[[Joey]]
 
 ----
 
@@ -25,7 +25,7 @@ Also nice to have would be:
     
 - Somehow sync remotes, possibly using a push sync like dvcs-autosync
   does, so they are immediately updated.
-- Somehow get content that is unavilable. This is problimatic with inotify,
+- Somehow get content that is unavailable. This is problematic with inotify,
   since we only get an event once the user has tried (and failed) to read
   from the file. Perhaps instead, automatically copy content that is added
   out to remotes, with the goal of all repos eventually getting a copy,
@@ -35,7 +35,7 @@ Also nice to have would be:
 - Perhaps automatically dropunused files that have been deleted,
   although I cannot see a way to do that, since by the time the inotify
   deletion event arrives, the file is deleted, and we cannot see what
-  its symlink pointed to! Alternatievely, perhaps automatically
+  its symlink pointed to! Alternatively, perhaps automatically
   do an expensive unused/dropunused cleanup process.
 - Support OSes other than Linux; it only uses inotify currently.
 

From c270ca5453382e2ccaf70d96bf7ddff4a5a014b8 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 24 May 2012 07:52:54 +0000
Subject: [PATCH 3494/8313] fix typo

---
 doc/forum/cloud_services_to_support.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/cloud_services_to_support.mdwn b/doc/forum/cloud_services_to_support.mdwn
index e268bc1d86..4d6bde1b54 100644
--- a/doc/forum/cloud_services_to_support.mdwn
+++ b/doc/forum/cloud_services_to_support.mdwn
@@ -1,5 +1,5 @@
 git-annex can already be used to store data in several cloud services:
-Amazon S3, rsync.net, Tahoe-LAFFS, The Internet Archive.
+Amazon S3, rsync.net, Tahoe-LAFS, The Internet Archive.
 
 I would like to support as many other cloud services as possible/reasonable.
 

From 3d208ee569470f816ab6def94ee2b13ad9fe6423 Mon Sep 17 00:00:00 2001
From: "https://rmunn.myopenid.com/" 
Date: Thu, 24 May 2012 18:14:13 +0000
Subject: [PATCH 3495/8313]

---
 doc/forum/What_happened_to_the_walkthrough__63__.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/What_happened_to_the_walkthrough__63__.mdwn

diff --git a/doc/forum/What_happened_to_the_walkthrough__63__.mdwn b/doc/forum/What_happened_to_the_walkthrough__63__.mdwn
new file mode 100644
index 0000000000..e8098d29a1
--- /dev/null
+++ b/doc/forum/What_happened_to_the_walkthrough__63__.mdwn
@@ -0,0 +1 @@
+As of right now (2012-05-24 at 18:00 UTC), the [[Walkthrough]] page is basically empty. Its entire contents are "A walkthrough of the basic features of git-annex." No links (other than the autogenerated "what links to this page" list at the bottom) and no contents. Any idea what happened?

From 164b55e99209bfb9ec0d92064027ab6e604232d0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Thu, 24 May 2012 19:18:55 +0000
Subject: [PATCH 3496/8313] Added a comment

---
 .../comment_1_70db0e3cfb1318e95671c23726e5541d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment

diff --git a/doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment b/doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment
new file mode 100644
index 0000000000..cbf852dbfb
--- /dev/null
+++ b/doc/forum/What_happened_to_the_walkthrough__63__/comment_1_70db0e3cfb1318e95671c23726e5541d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-05-24T19:18:55Z"
+ content="""
+It seems the pages that are supposed to be inlined are not being found even though they are in `doc/walkthrough/`.
+"""]]

From b8f12b0dbe4d0304fab0c3eecbb38e52815e2f3e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 24 May 2012 16:10:21 -0400
Subject: [PATCH 3497/8313] fix inline pagenames in walkthrough

I should read my own ikiwiki changelog when upgrading, apparently.
---
 doc/walkthrough.mdwn | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index f93e28393e..a64b175f95 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -3,22 +3,22 @@ A walkthrough of the basic features of git-annex.
 [[!toc]]
 
 [[!inline feeds=no show=0 template=walkthrough pagenames="""
-	creating_a_repository
-	adding_a_remote
-	adding_files
-	renaming_files
-	getting_file_content
-	syncing
-	transferring_files:_When_things_go_wrong
-	removing_files
-	removing_files:_When_things_go_wrong
-	modifying_annexed_files
-	using_ssh_remotes
-	moving_file_content_between_repositories
-	unused_data
-	fsck:_verifying_your_data
-	fsck:_when_things_go_wrong
-	backups
-	automatically_managing_content
-	more
+	walkthrough/creating_a_repository
+	walkthrough/adding_a_remote
+	walkthrough/adding_files
+	walkthrough/renaming_files
+	walkthrough/getting_file_content
+	walkthrough/syncing
+	walkthrough/transferring_files:_When_things_go_wrong
+	walkthrough/removing_files
+	walkthrough/removing_files:_When_things_go_wrong
+	walkthrough/modifying_annexed_files
+	walkthrough/using_ssh_remotes
+	walkthrough/moving_file_content_between_repositories
+	walkthrough/unused_data
+	walkthrough/fsck:_verifying_your_data
+	walkthrough/fsck:_when_things_go_wrong
+	walkthrough/backups
+	walkthrough/automatically_managing_content
+	walkthrough/more
 """]]

From 570fe8af6f385503c3aab4ef4c1ef606685c194c Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 24 May 2012 20:15:19 +0000
Subject: [PATCH 3498/8313] Added a comment

---
 .../comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment

diff --git a/doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment b/doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment
new file mode 100644
index 0000000000..9720adfbc6
--- /dev/null
+++ b/doc/forum/What_happened_to_the_walkthrough__63__/comment_2_f9305dd19b9b5f35e66d915b8c30374b._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-05-24T20:15:19Z"
+ content="""
+Broken last night during upgrade, fixed now, thanks for noticing.
+"""]]

From f7524811e2e47ca8455d3038b6d0a8102e5a5044 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 25 May 2012 08:52:55 -0400
Subject: [PATCH 3499/8313] bug

---
 doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn

diff --git a/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn b/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn
new file mode 100644
index 0000000000..69704bdd07
--- /dev/null
+++ b/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn
@@ -0,0 +1,5 @@
+Add a file, then unlock it, and then lock it. There is an error and the
+symlink gets deleted. 
+
+The file will still be staged in the index, and the file content is still
+in the annex. --[[Joey]] 

From 45a01db6add3399ff6ca93f2e7c7d83dbf59992d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:11:19 -0400
Subject: [PATCH 3500/8313] add preliminary design

---
 doc/design/assistant.mdwn               | 19 +++++++++++++++
 doc/design/assistant/android.mdwn       | 31 +++++++++++++++++++++++++
 doc/design/assistant/configurators.mdwn | 21 +++++++++++++++++
 doc/design/assistant/inotify.mdwn       | 18 ++++++++++++++
 doc/design/assistant/progressbars.mdwn  | 14 +++++++++++
 doc/design/assistant/syncing.mdwn       | 31 +++++++++++++++++++++++++
 doc/design/assistant/webapp.mdwn        | 27 +++++++++++++++++++++
 doc/design/assistant/windows.mdwn       | 17 ++++++++++++++
 8 files changed, 178 insertions(+)
 create mode 100644 doc/design/assistant.mdwn
 create mode 100644 doc/design/assistant/android.mdwn
 create mode 100644 doc/design/assistant/configurators.mdwn
 create mode 100644 doc/design/assistant/inotify.mdwn
 create mode 100644 doc/design/assistant/progressbars.mdwn
 create mode 100644 doc/design/assistant/syncing.mdwn
 create mode 100644 doc/design/assistant/webapp.mdwn
 create mode 100644 doc/design/assistant/windows.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
new file mode 100644
index 0000000000..a9444c3673
--- /dev/null
+++ b/doc/design/assistant.mdwn
@@ -0,0 +1,19 @@
+The git-annex assistant is being
+[crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/).
+
+This is my design and plan for developing it.
+Still being fleshed out, still many ideas and use cases to add. --[[Joey]]
+
+## roadmap
+
+### Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
+
+### Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars||
+
+### Month 3 "easy setup": [[!traillink configurators]]
+
+### Month 4: polishing and overflow
+
+### Month 5 & 6 "9k bonus round": [[!traillink Android]]
+
+### In my overfunded nighmares: [[!traillink Windows]]
diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
new file mode 100644
index 0000000000..24791501da
--- /dev/null
+++ b/doc/design/assistant/android.mdwn
@@ -0,0 +1,31 @@
+Porting git-annex to Android will use the Android native SDK.
+
+A hopefully small Java app will be developed, which runs the webapp
+daemon, and a web browser to display it.
+
+### programs to port
+
+These will probably need to be bundled into the Android app, unless already
+available in the App Store.
+
+* ssh (native ssh needed for scp, not a client like ConnectBot)
+* rsync
+* gpg
+* git (not all git commands are needed, but a core plumbing and a few like `git-add` are.)
+
+### FAT sucks
+
+The main media partition will use some awful FAT filesystem format from
+1982 that cannot support git-annex's symlinks. Hopefully it can at least
+handle all of git's filenames. Possible approaches to this:
+
+* Keep only a bare git repo on Android. The app would then need to include
+  a file browser to access the files in there, and adding a file would move
+  it into the repo. Not ideal.
+* Implement [[smudge]] filters to avoid needing symlinks. Difficult.
+* Use a bare git repo but don't keep files in `annex/objects`, instead
+  leave them outside the repo, and add some local mapping to find them.
+  Seems best?
+* Use a `LD_PRELOAD` wrapper to do Something Crazy.
+
+(May want to consider which of these would make a Windows port easier too.)
diff --git a/doc/design/assistant/configurators.mdwn b/doc/design/assistant/configurators.mdwn
new file mode 100644
index 0000000000..e03723e946
--- /dev/null
+++ b/doc/design/assistant/configurators.mdwn
@@ -0,0 +1,21 @@
+Add to the [[webapp]] some configuration of git-annex.
+
+There are some basic settings that pass through to `git config`, things
+like how much disk space to leave free, how many copies to ensure are kept
+of files, etc.
+
+The meat of the configuration will be in configuration assistants that walk
+through setting up common use cases.
+
+* Clone this repo to a USB drive.
+* Clone this repo to another host:
+  1. Prompt for the hostname (or do avahi local machine discovery). 
+  2. Enable the two hosts to ssh to one-another and run git-annex shell.
+     (A tricky problem, if ssh keys need to be added to do that.)
+  3. Push over a clone of the repository. (Using git-annex-shell?)
+  4. Start [[syncing]].
+* Set up Amazon S3.
+* Set up rsync remote.
+* Set up encryption.
+* I lost my USB drive!
+* etc -- many more possibilities
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
new file mode 100644
index 0000000000..be38ef13cd
--- /dev/null
+++ b/doc/design/assistant/inotify.mdwn
@@ -0,0 +1,18 @@
+Finish "git annex watch" command, which runs, in the background, watching via
+inotify for changes, and automatically annexing new files, etc.
+
+There is a `watch` branch in git that adds such a command, although currently
+it only handles adding new files, and nothing else. To make this really
+useful, it needs to:
+
+- notice deleted files and stage the deletion
+  (tricky; there's a race with add..)
+- notice renamed files, auto-fix the symlink, and stage the new file location
+- periodically auto-commit staged changes
+- honor .gitignore, not adding files it excludesa
+
+Also to do:
+
+- Support OSes other than Linux; it only uses inotify currently.
+  OSX and FreeBSD use the same mechanism, and there is a Haskell interface
+  for it,
diff --git a/doc/design/assistant/progressbars.mdwn b/doc/design/assistant/progressbars.mdwn
new file mode 100644
index 0000000000..f76b42d73a
--- /dev/null
+++ b/doc/design/assistant/progressbars.mdwn
@@ -0,0 +1,14 @@
+Currently, git-annex takes a very lazy approch to displaying
+progress into. It just lets rsync or whatever display the progress
+for it, in the terminal.
+
+Something better is needed for the [[webapp]]. There needs to be a
+way for the web app to know what the current progress is of all transfers.
+
+To get this info for downloads, git-annex can watch the file as it arrives
+and use its size.
+
+TODO: What about uploads? Will i have to parse rsync's progresss output?
+Ugh.
+
+This is one of those potentially hidden but time consuming problems.
diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn
new file mode 100644
index 0000000000..0a081c1010
--- /dev/null
+++ b/doc/design/assistant/syncing.mdwn
@@ -0,0 +1,31 @@
+Once files are added (or removed or moved), need to send those changes to
+all the other git clones, at both the git level and the key/value level.
+
+## git syncing
+
+1. At regular intervals, just run `git annex sync`, which already handles
+   bidirectional syncing.
+2. Investigate the XMPP approach like dvcs-autosync does, or other ways of
+   signaling a change out of band.
+3. Add a hook, so when there's a change to sync, a program can be run.
+
+## data syncing
+
+There are two parts to data syncing. First, map the network and second,
+decide what to sync when.
+
+Mapping the network can reuse code in `git annex map`. Once the map is
+built, we want to find paths through the network that reach all nodes
+eventually, with the least cost. This is a minimum spanning tree problem,
+except with a directed graph, so really a Arborescence problem.
+
+With the map, we can determine which nodes to push new content to. Then we
+need to control those data transfers, sending to the cheapest nodes first,
+and with appropriate rate limiting and control facilities.
+
+This probably will need lots of refinements to get working well.
+
+## other considerations
+
+This assumes the network is connected. It's often not, so the
+cloud needs to be used to bridge between LANs.
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
new file mode 100644
index 0000000000..c0f6b893e1
--- /dev/null
+++ b/doc/design/assistant/webapp.mdwn
@@ -0,0 +1,27 @@
+The webapp is a web server that displays a shiny interface.
+
+## security
+
+* Listen only to localhost.
+* Instruct the user's web browser to open an url that contains a secret
+  token. This guards against other users on the same system.
+* I would like to avoid passwords or other authentication methods,
+  it's your local system.
+
+## interface
+
+* list of files uploading and downloading
+* progress bars for each file
+* drag and drop to reorder
+* cancel and pause
+
+## implementation
+
+Hope to use Yesod.
+
+TODO: Ensure that Yesod will work on arm. Necessary for later Android port.
+Will its template haskell cause a problem? Does new GHC support TH on ARM?
+Will it use too much memory or be too slow?
+
+Hopefully Yesod comes with some good UI widgets. Otherwise, need to use
+Jquery or similar.
diff --git a/doc/design/assistant/windows.mdwn b/doc/design/assistant/windows.mdwn
new file mode 100644
index 0000000000..0b176934b9
--- /dev/null
+++ b/doc/design/assistant/windows.mdwn
@@ -0,0 +1,17 @@
+See [[todo/windows_support]]..
+
+## symlinks
+
+Apparently new versions of Windows have something very like symlinks.
+(Or really, 3 or so things not entirely unlike symlinks and all different.)
+Stackoverflow has some details.
+
+Make git use them, as it (apparently) does not yet.
+
+(What **does** git do on Windows when it clones a repo with symlinks?)
+
+## POSIX
+
+Lots of ifdefs and pain to deal with POSIX calls in the code base.
+
+Or I could try to use Cywin.

From 931b85c4817d12ad84f3be2b3649d4b550b9214b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:14:27 -0400
Subject: [PATCH 3501/8313] update

---
 doc/design/assistant.mdwn | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index a9444c3673..316f62ba1d 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -2,18 +2,14 @@ The git-annex assistant is being
 [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/).
 
 This is my design and plan for developing it.
-Still being fleshed out, still many ideas and use cases to add. --[[Joey]]
+Still being fleshed out, still many ideas and use cases to add.
+Feel free to chip in with comments! --[[Joey]]
 
 ## roadmap
 
-### Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
-
-### Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars||
-
-### Month 3 "easy setup": [[!traillink configurators]]
-
-### Month 4: polishing and overflow
-
-### Month 5 & 6 "9k bonus round": [[!traillink Android]]
-
-### In my overfunded nighmares: [[!traillink Windows]]
+* Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
+* Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars||
+* Month 3 "easy setup": [[!traillink configurators]]
+* Month 4: polishing and overflow
+* Month 5 & 6 "9k bonus round": [[!traillink Android]]
+* In my overfunded nighmares: [[!traillink Windows]]

From 573f1c03a84ccc285490f6964b2bbfef7205bf09 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:15:07 -0400
Subject: [PATCH 3502/8313] fix

---
 doc/design/assistant.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 316f62ba1d..6320925e89 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -8,7 +8,7 @@ Feel free to chip in with comments! --[[Joey]]
 ## roadmap
 
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
-* Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars||
+* Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
 * Month 3 "easy setup": [[!traillink configurators]]
 * Month 4: polishing and overflow
 * Month 5 & 6 "9k bonus round": [[!traillink Android]]

From 147f63ccabc12b95d28f995b776eb61bde141b37 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:15:40 -0400
Subject: [PATCH 3503/8313] tweak

---
 doc/design/assistant.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 6320925e89..3ab18e0a3f 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -11,5 +11,5 @@ Feel free to chip in with comments! --[[Joey]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
 * Month 3 "easy setup": [[!traillink configurators]]
 * Month 4: polishing and overflow
-* Month 5 & 6 "9k bonus round": [[!traillink Android]]
+* Months 5-6 "9k bonus round": [[!traillink Android]]
 * In my overfunded nighmares: [[!traillink Windows]]

From 08aeec8f1e3d30435177b8f0756630e488ec3ede Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:17:21 -0400
Subject: [PATCH 3504/8313] alternative

---
 doc/design/assistant/progressbars.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/progressbars.mdwn b/doc/design/assistant/progressbars.mdwn
index f76b42d73a..2ade05aa57 100644
--- a/doc/design/assistant/progressbars.mdwn
+++ b/doc/design/assistant/progressbars.mdwn
@@ -9,6 +9,6 @@ To get this info for downloads, git-annex can watch the file as it arrives
 and use its size.
 
 TODO: What about uploads? Will i have to parse rsync's progresss output?
-Ugh.
+Feed it via a named pipe? Ugh.
 
 This is one of those potentially hidden but time consuming problems.

From ec9f793db1426126673ce4a5e9380e089628bc7d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:18:27 -0400
Subject: [PATCH 3505/8313] update

---
 doc/design/assistant.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 3ab18e0a3f..f0805594e7 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -10,6 +10,6 @@ Feel free to chip in with comments! --[[Joey]]
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
 * Month 3 "easy setup": [[!traillink configurators]]
-* Month 4: polishing and overflow
+* Month 4: polishing and overflow; release
 * Months 5-6 "9k bonus round": [[!traillink Android]]
 * In my overfunded nighmares: [[!traillink Windows]]

From 6703892e83c89a27a2e2766e5984618e7449bdb8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:24:43 -0400
Subject: [PATCH 3506/8313] reord

---
 doc/design/assistant.mdwn                     | 10 +++++-
 doc/design/assistant/leftovers.mdwn           | 14 ++++++++
 .../automatic_bookkeeping_watch_command.mdwn  | 32 +------------------
 3 files changed, 24 insertions(+), 32 deletions(-)
 create mode 100644 doc/design/assistant/leftovers.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index f0805594e7..5cbdb20d19 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -5,11 +5,19 @@ This is my design and plan for developing it.
 Still being fleshed out, still many ideas and use cases to add.
 Feel free to chip in with comments! --[[Joey]]
 
+## the pitch
+
+The blue sky goal would be something automated like dropbox, except fully
+distributed. All files put into the repository would propagate out
+to all the other clones of it, as network links allow. Note that while
+dropbox allows modifying files, git-annex freezes them upon creation,
+so this would not be 100% equivalent to dropbox.
+
 ## roadmap
 
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
 * Month 3 "easy setup": [[!traillink configurators]]
-* Month 4: polishing and overflow; release
+* Month 4 "release": [[!traillink leftovers]]; release
 * Months 5-6 "9k bonus round": [[!traillink Android]]
 * In my overfunded nighmares: [[!traillink Windows]]
diff --git a/doc/design/assistant/leftovers.mdwn b/doc/design/assistant/leftovers.mdwn
new file mode 100644
index 0000000000..313544d384
--- /dev/null
+++ b/doc/design/assistant/leftovers.mdwn
@@ -0,0 +1,14 @@
+Things that don't fit anywhere else:
+
+* Somehow get content that is unavailable. This is problematic with inotify,
+  since we only get an event once the user has tried (and failed) to read
+  from the file. This is only needed if all the files in the directory 
+  are not kept synced, but in some situations (ie, low disk space phones),
+  that is likely.
+* Drop files that have not been used lately, or meet some other criteria
+  (as long as there's a copy elsewhere).
+* Perhaps automatically dropunused files that have been deleted,
+  although I cannot see a way to do that, since by the time the inotify
+  deletion event arrives, the file is deleted, and we cannot see what
+  its symlink pointed to! Alternatively, perhaps automatically
+  do an expensive unused/dropunused cleanup process.
diff --git a/doc/todo/automatic_bookkeeping_watch_command.mdwn b/doc/todo/automatic_bookkeeping_watch_command.mdwn
index 4b688b839e..0bb86e4a13 100644
--- a/doc/todo/automatic_bookkeeping_watch_command.mdwn
+++ b/doc/todo/automatic_bookkeeping_watch_command.mdwn
@@ -9,34 +9,4 @@ to all the other clones of it, as network links allow. Note that while
 dropbox allows modifying files, git-annex freezes them upon creation,
 so this would not be 100% equivalent to dropbox. --[[Joey]]
 
-----
-
-There is a `watch` branch in git that adds such a command, although currently
-it only handles adding new files, and nothing else. To make this really
-useful, it needs to:
-
-- notice deleted files and stage the deletion
-  (tricky; there's a race with add..)
-- notice renamed files, auto-fix the symlink, and stage the new file location
-- periodically auto-commit staged changes
-- honor .gitignore, not adding files it excludes
-    
-Also nice to have would be:
-    
-- Somehow sync remotes, possibly using a push sync like dvcs-autosync
-  does, so they are immediately updated.
-- Somehow get content that is unavailable. This is problematic with inotify,
-  since we only get an event once the user has tried (and failed) to read
-  from the file. Perhaps instead, automatically copy content that is added
-  out to remotes, with the goal of all repos eventually getting a copy,
-  if df allows.
-- Drop files that have not been used lately, or meet some other criteria
-  (as long as there's a copy elsewhere).
-- Perhaps automatically dropunused files that have been deleted,
-  although I cannot see a way to do that, since by the time the inotify
-  deletion event arrives, the file is deleted, and we cannot see what
-  its symlink pointed to! Alternatively, perhaps automatically
-  do an expensive unused/dropunused cleanup process.
-- Support OSes other than Linux; it only uses inotify currently.
-
-
+This is a big project with its own [[design pages|design/assistant]].

From 76720a6d0df1dd8f3d4405bf4e7cd1a6ce31005d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 21:38:25 -0400
Subject: [PATCH 3507/8313] update

---
 doc/design/assistant.mdwn           | 15 ++++++---------
 doc/design/assistant/deltas.mdwn    |  9 +++++++++
 doc/design/assistant/desymlink.mdwn |  5 +++++
 3 files changed, 20 insertions(+), 9 deletions(-)
 create mode 100644 doc/design/assistant/deltas.mdwn
 create mode 100644 doc/design/assistant/desymlink.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 5cbdb20d19..3bbd27c581 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -5,14 +5,6 @@ This is my design and plan for developing it.
 Still being fleshed out, still many ideas and use cases to add.
 Feel free to chip in with comments! --[[Joey]]
 
-## the pitch
-
-The blue sky goal would be something automated like dropbox, except fully
-distributed. All files put into the repository would propagate out
-to all the other clones of it, as network links allow. Note that while
-dropbox allows modifying files, git-annex freezes them upon creation,
-so this would not be 100% equivalent to dropbox.
-
 ## roadmap
 
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
@@ -20,4 +12,9 @@ so this would not be 100% equivalent to dropbox.
 * Month 3 "easy setup": [[!traillink configurators]]
 * Month 4 "release": [[!traillink leftovers]]; release
 * Months 5-6 "9k bonus round": [[!traillink Android]]
-* In my overfunded nighmares: [[!traillink Windows]]
+
+## not yet on the map:
+
+* [[desymlink]]
+* [[deltas]]
+* In my overfunded nighmares: [[Windows]]
diff --git a/doc/design/assistant/deltas.mdwn b/doc/design/assistant/deltas.mdwn
new file mode 100644
index 0000000000..cf2d9f6c34
--- /dev/null
+++ b/doc/design/assistant/deltas.mdwn
@@ -0,0 +1,9 @@
+Speed up syncing of modified versions of to existing files. 
+
+One simple way is to find the key of the old version of a file that's
+being transferred, so it can be used as the basis for rsync, or any
+other similar transfer protocol.
+
+For remotes that don't use rsync, a poor man's version could be had by
+chunking each object into multiple parts. Only modified parts need be
+transferred. Sort of sub-keys to the main key being stored.
diff --git a/doc/design/assistant/desymlink.mdwn b/doc/design/assistant/desymlink.mdwn
new file mode 100644
index 0000000000..e12cd52dc7
--- /dev/null
+++ b/doc/design/assistant/desymlink.mdwn
@@ -0,0 +1,5 @@
+While dropbox allows modifying files in the folder, git-annex freezes
+them upon creation.
+
+To allow editing files in its folder, something like [[todo/smudge]] is
+needed, to get rid of the symlinks that stand in for the files.

From 697298718cbc45f1a521584e6757acc69d0f612d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 22:25:25 -0400
Subject: [PATCH 3508/8313] update

---
 doc/design/assistant/syncing.mdwn | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn
index 0a081c1010..9cbdddcb96 100644
--- a/doc/design/assistant/syncing.mdwn
+++ b/doc/design/assistant/syncing.mdwn
@@ -5,9 +5,11 @@ all the other git clones, at both the git level and the key/value level.
 
 1. At regular intervals, just run `git annex sync`, which already handles
    bidirectional syncing.
-2. Investigate the XMPP approach like dvcs-autosync does, or other ways of
+2. Use a git merge driver that adds both conflicting files,
+   so conflicts never break a sync.
+3. Investigate the XMPP approach like dvcs-autosync does, or other ways of
    signaling a change out of band.
-3. Add a hook, so when there's a change to sync, a program can be run.
+4. Add a hook, so when there's a change to sync, a program can be run.
 
 ## data syncing
 
@@ -29,3 +31,6 @@ This probably will need lots of refinements to get working well.
 
 This assumes the network is connected. It's often not, so the
 cloud needs to be used to bridge between LANs.
+
+It would be nice if, when a USB drive is connected,
+syncing starts automatically.

From 8bf2d1902b5bff01ee9343d6fdcd09254f960d79 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 26 May 2012 23:47:55 -0400
Subject: [PATCH 3509/8313] format

---
 doc/install/Gentoo.mdwn | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/doc/install/Gentoo.mdwn b/doc/install/Gentoo.mdwn
index c508309c6e..feeaad739b 100644
--- a/doc/install/Gentoo.mdwn
+++ b/doc/install/Gentoo.mdwn
@@ -1,5 +1,3 @@
-Gentoo users can:
-
-	`emerge git-annex`
+Gentoo users can: `emerge git-annex`
 
 A possibly more up-to-date version is in the haskell portage overlay.

From 737a273ea7da564bdae8feaae2385766307e68b4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 20:33:01 +0000
Subject: [PATCH 3510/8313] Added a comment: cannot get files

---
 ...t_1_95cadaad486dc32b4503cfddf2500bc1._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment b/doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment
new file mode 100644
index 0000000000..604c1fb6d7
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="cannot get files"
+ date="2012-05-27T20:33:00Z"
+ content="""
+Hi,
+
+I could successfully clone my ssh repo's annex to my laptop, following these instructions.
+I'm also able to sync the repositories (laptop and ssh) when I commit new files in the ssh repo.
+
+However, every time I try to get files from the ssh repo (using 'git annex get some_file'), nothing happens.
+Do you know what can be happening?
+
+Thanks!
+"""]]

From 085e02012fd029bcf6709cc3d30d8c0886d4eb52 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 20:33:10 +0000
Subject: [PATCH 3511/8313] Added a comment: cannot get files

---
 ...t_2_451fd0c6a25ee61ef137e8e5be0c286b._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment b/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment
new file mode 100644
index 0000000000..2121401968
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_2_451fd0c6a25ee61ef137e8e5be0c286b._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="cannot get files"
+ date="2012-05-27T20:33:09Z"
+ content="""
+Hi,
+
+I could successfully clone my ssh repo's annex to my laptop, following these instructions.
+I'm also able to sync the repositories (laptop and ssh) when I commit new files in the ssh repo.
+
+However, every time I try to get files from the ssh repo (using 'git annex get some_file'), nothing happens.
+Do you know what can be happening?
+
+Thanks!
+"""]]

From d39c0d542005c01c6653d7a9ab2cc748f1e0206f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 20:33:22 +0000
Subject: [PATCH 3512/8313] removed

---
 ...t_1_95cadaad486dc32b4503cfddf2500bc1._comment | 16 ----------------
 1 file changed, 16 deletions(-)
 delete mode 100644 doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment b/doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment
deleted file mode 100644
index 604c1fb6d7..0000000000
--- a/doc/walkthrough/using_ssh_remotes/comment_1_95cadaad486dc32b4503cfddf2500bc1._comment
+++ /dev/null
@@ -1,16 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
- nickname="Fernando Seabra"
- subject="cannot get files"
- date="2012-05-27T20:33:00Z"
- content="""
-Hi,
-
-I could successfully clone my ssh repo's annex to my laptop, following these instructions.
-I'm also able to sync the repositories (laptop and ssh) when I commit new files in the ssh repo.
-
-However, every time I try to get files from the ssh repo (using 'git annex get some_file'), nothing happens.
-Do you know what can be happening?
-
-Thanks!
-"""]]

From 9ccf933ffa7d11e7abefb51f0ffd8c652847899f Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sun, 27 May 2012 20:53:06 +0000
Subject: [PATCH 3513/8313] Added a comment

---
 ...ment_2_365db5820d96d5daa62c19fd76fcdf1e._comment | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment b/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment
new file mode 100644
index 0000000000..8973978ad8
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_2_365db5820d96d5daa62c19fd76fcdf1e._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-27T20:53:05Z"
+ content="""
+When `git annex get` does nothing, it's because it doesn't know a place to get the file from.
+
+This can happen if the `git-annex` branch has not propigated from the place where the file was added.
+For example, if on the laptop you had run `git pull ssh master`, that would only pull the master branch, not the git-annex branch. 
+
+An easy way to ensure the git-annex branch is kept in sync is to run `git annex sync`
+"""]]

From 402b4c81ba2eed3b5fd9a9f9db6800fa7a128b0b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 May 2012 16:56:20 -0400
Subject: [PATCH 3514/8313] add trail

---
 doc/walkthrough.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/walkthrough.mdwn b/doc/walkthrough.mdwn
index a64b175f95..c288b71ded 100644
--- a/doc/walkthrough.mdwn
+++ b/doc/walkthrough.mdwn
@@ -2,7 +2,7 @@ A walkthrough of the basic features of git-annex.
 
 [[!toc]]
 
-[[!inline feeds=no show=0 template=walkthrough pagenames="""
+[[!inline feeds=no trail=yes show=0 template=walkthrough pagenames="""
 	walkthrough/creating_a_repository
 	walkthrough/adding_a_remote
 	walkthrough/adding_files

From e96975eb3af2cb5da49099a4e87013af0534749f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 May 2012 16:56:28 -0400
Subject: [PATCH 3515/8313] update

---
 doc/design/assistant/leftovers.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/design/assistant/leftovers.mdwn b/doc/design/assistant/leftovers.mdwn
index 313544d384..c322a27812 100644
--- a/doc/design/assistant/leftovers.mdwn
+++ b/doc/design/assistant/leftovers.mdwn
@@ -1,5 +1,6 @@
 Things that don't fit anywhere else:
 
+* Automatically start daemon on boot or when user logs in.
 * Somehow get content that is unavailable. This is problematic with inotify,
   since we only get an event once the user has tried (and failed) to read
   from the file. This is only needed if all the files in the directory 

From 4127e92b3aedd265e5f9676a9775e637c3f79f6d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 May 2012 16:57:32 -0400
Subject: [PATCH 3516/8313] update

---
 doc/design/assistant/configurators.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/design/assistant/configurators.mdwn b/doc/design/assistant/configurators.mdwn
index e03723e946..b9e3a66328 100644
--- a/doc/design/assistant/configurators.mdwn
+++ b/doc/design/assistant/configurators.mdwn
@@ -7,6 +7,8 @@ of files, etc.
 The meat of the configuration will be in configuration assistants that walk
 through setting up common use cases.
 
+* Create a repository (run when the web app is started without a configured
+  repository too).
 * Clone this repo to a USB drive.
 * Clone this repo to another host:
   1. Prompt for the hostname (or do avahi local machine discovery). 

From 94649c12704d02314e89e792df90547ed6e96cae Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 21:13:50 +0000
Subject: [PATCH 3517/8313] Added a comment

---
 ...nt_3_b2f15a46620385da26d5fe8f11ebfc1a._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment b/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment
new file mode 100644
index 0000000000..75a133d840
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_3_b2f15a46620385da26d5fe8f11ebfc1a._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 3"
+ date="2012-05-27T21:13:50Z"
+ content="""
+Thanks for the quick replay!
+
+I already did 'git annex sync', but it didn't work. The steps were: 'git clone ssh...', then 'cd annex', then 'git annex init \"laptop\"'
+
+After that, I did a 'git annex sync', and tried to get the file, but nothing happens. That's why I found it weird.
+Any other thing that might have happened?
+
+Thanks again!
+"""]]

From dd6f92bf6374457c4e371766c0cccfe9573abf77 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sun, 27 May 2012 21:33:11 +0000
Subject: [PATCH 3518/8313] Added a comment

---
 .../comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment b/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment
new file mode 100644
index 0000000000..3df03abc2c
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_4_433ccc87fbb0a13e32d59d77f0b4e56c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 4"
+ date="2012-05-27T21:33:11Z"
+ content="""
+Try running `git annex whereis` on the file and see where it says it is.
+"""]]

From 7b09d83db2db5b67c98854902dc5a639d0202a2c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 21:41:58 +0000
Subject: [PATCH 3519/8313] Added a comment

---
 ...5_da860d0f8c8772062d28d063942e1af7._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment b/doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment
new file mode 100644
index 0000000000..d1091eb8c8
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 5"
+ date="2012-05-27T21:41:57Z"
+ content="""
+Hi,
+
+I guess the problem is with git-annex-shell. I tried to do 'git annex get file --from name_ssh_repo', and I got the following:
+
+bash: git-annex-shell: command not found; failed; exit code 127
+
+The same thing happens if I try to do 'git annex whereis'
+
+git-annex-shell it is indeed installed. How can I make my shell recognize this command?
+
+Thanks a lot!
+"""]]

From 3f5b0fc6cd89db6e2bf44b8bc1f44c1b1fb1346e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 21:42:40 +0000
Subject: [PATCH 3520/8313] removed

---
 ...5_da860d0f8c8772062d28d063942e1af7._comment | 18 ------------------
 1 file changed, 18 deletions(-)
 delete mode 100644 doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment b/doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment
deleted file mode 100644
index d1091eb8c8..0000000000
--- a/doc/walkthrough/using_ssh_remotes/comment_5_da860d0f8c8772062d28d063942e1af7._comment
+++ /dev/null
@@ -1,18 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
- nickname="Fernando Seabra"
- subject="comment 5"
- date="2012-05-27T21:41:57Z"
- content="""
-Hi,
-
-I guess the problem is with git-annex-shell. I tried to do 'git annex get file --from name_ssh_repo', and I got the following:
-
-bash: git-annex-shell: command not found; failed; exit code 127
-
-The same thing happens if I try to do 'git annex whereis'
-
-git-annex-shell it is indeed installed. How can I make my shell recognize this command?
-
-Thanks a lot!
-"""]]

From b8d36b66678672958382d19d83754a4d65691e57 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 21:42:57 +0000
Subject: [PATCH 3521/8313] Added a comment

---
 ...5_a9805c7965da0b88a1c9f7f207c450a1._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment b/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment
new file mode 100644
index 0000000000..703b89ebfa
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_5_a9805c7965da0b88a1c9f7f207c450a1._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 5"
+ date="2012-05-27T21:42:56Z"
+ content="""
+Hi,
+
+I guess the problem is with git-annex-shell. I tried to do 'git annex get file --from name_ssh_repo', and I got the following:
+
+bash: git-annex-shell: command not found; failed; exit code 127
+
+The same thing happens if I try to do 'git annex whereis'
+
+git-annex-shell is indeed installed. How can I make my shell recognize this command?
+
+Thanks a lot!
+"""]]

From 6de910312cf8a9f089cc7dda89e0ebc95b8a87a2 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sun, 27 May 2012 22:08:50 +0000
Subject: [PATCH 3522/8313] Added a comment

---
 ...mment_6_9d5c12c056892b706cf100ea01866685._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment b/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment
new file mode 100644
index 0000000000..4d5961ca90
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_6_9d5c12c056892b706cf100ea01866685._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 6"
+ date="2012-05-27T22:08:50Z"
+ content="""
+git-annex-shell needs to be installed in the `PATH` on any host that will hold annexed files. 
+
+If you installed with cabal, it might be `.cabal/bin/`. Whereever it was installed to is apparently not on the PATH that is set when you ssh into that host. 
+
+
+"""]]

From 56468d41f292779dbdaac6c21a0b1938dcda1a68 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
 
Date: Sun, 27 May 2012 23:35:19 +0000
Subject: [PATCH 3523/8313] Added a comment

---
 ...t_7_725e7dbb2d0a74a035127cb01ee0442c._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment

diff --git a/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment b/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment
new file mode 100644
index 0000000000..700b170ad6
--- /dev/null
+++ b/doc/walkthrough/using_ssh_remotes/comment_7_725e7dbb2d0a74a035127cb01ee0442c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkaT0B6s9jQuMzQUYRVBgWqtO7BhT_ZSaE"
+ nickname="Fernando Seabra"
+ subject="comment 7"
+ date="2012-05-27T23:35:17Z"
+ content="""
+Hi,
+
+It was already installed in PATH. In fact, I can call it from the command line, and it is recognized (e.g. calling 'git-annex-shell' gives me 'git-annex-shell: bad parameters'). However, every time I do a 'git annex whereis' or 'git annex get file --from repo', it gives me the following error:
+
+bash: git-annex-shell: command not found
+Command ssh [\"-S\",\"/Users/username/annex/.git/annex/ssh/username@example.edu\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"username@example.edu\",\"git-annex-shell 'configlist' '/~/annex'\"] failed; exit code 127
+
+I tried to run this ssh command, but it gives me the same 'command not found' error. It seems that the problem is with the ssh repo?
+The ssh repo has a git-annex-shell working and installed in PATH.
+"""]]

From f945c5231d0281c7161323d1d8905a0568f07da4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 May 2012 20:06:36 -0400
Subject: [PATCH 3524/8313] format

---
 doc/special_remotes/rsync.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn
index 286615460c..273469258b 100644
--- a/doc/special_remotes/rsync.mdwn
+++ b/doc/special_remotes/rsync.mdwn
@@ -16,7 +16,7 @@ These parameters can be passed to `git annex initremote` to configure rsync:
 
 * `encryption` - Required. Either "none" to disable encryption of content
   stored in rsync, 
-  or a value that can be looked up (using gpg -k) to find a gpg encryption
+  or a value that can be looked up (using `gpg -k`) to find a gpg encryption
   key that will be given access to the remote. Note that additional gpg
   keys can be given access to a remote by rerunning initremote with
   the new key id. See [[encryption]].

From f120dbc73587bbff42910dce9eb26b79cebe424f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 May 2012 20:36:01 -0400
Subject: [PATCH 3525/8313] add manual cabal instuctions, I'm told the AUR may
 have dependency issues

---
 doc/install/ArchLinux.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/install/ArchLinux.mdwn b/doc/install/ArchLinux.mdwn
index e531fc968e..68e8b81f1c 100644
--- a/doc/install/ArchLinux.mdwn
+++ b/doc/install/ArchLinux.mdwn
@@ -7,3 +7,13 @@ such as yaourt:
 
 $ yaourt -Sy git-annex
 

+
+----
+
+I'm told the AUR has some dependency problems currently.
+If it doesn't work, you can just use cabal:
+
+
+pacman -S git rsync curl wget gpg openssh cabal-install
+cabal install git-annex --bindir=$HOME/bin
+


From 6e213d04f1233af3faffcd49acf479be086c71f4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 27 May 2012 20:55:56 -0400
Subject: [PATCH 3526/8313] sync: Show a nicer message if a user tries to sync
 to a special remote.

---
 Command/Sync.hs  | 9 ++++++++-
 debian/changelog | 6 ++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/Command/Sync.hs b/Command/Sync.hs
index b9ef0bc979..5fb49d30c5 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -57,10 +57,17 @@ syncRemotes rs = ifM (Annex.getState Annex.fast) ( nub <$> pickfast , wanted )
 		wanted
 			| null rs = good =<< concat . byspeed <$> available
 			| otherwise = listed
-		listed = catMaybes <$> mapM (Remote.byName . Just) rs
+		listed = do
+			l <- catMaybes <$> mapM (Remote.byName . Just) rs
+			let s = filter special l
+			unless (null s) $
+				error $ "cannot sync special remotes: " ++
+					unwords (map Types.Remote.name s)
+			return l
 		available = filter nonspecial <$> Remote.enabledRemoteList
 		good = filterM $ Remote.Git.repoAvail . Types.Remote.repo
 		nonspecial r = Types.Remote.remotetype r == Remote.Git.remote
+		special = not . nonspecial
 		fastest = fromMaybe [] . headMaybe . byspeed
 		byspeed = map snd . sort . M.toList . costmap
 		costmap = M.fromListWith (++) . map costpair
diff --git a/debian/changelog b/debian/changelog
index 99594c351c..9c38d1727f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+git-annex (3.20120523) UNRELEASED; urgency=low
+
+  * sync: Show a nicer message if a user tries to sync to a special remote.
+
+ -- Joey Hess   Sun, 27 May 2012 20:55:29 -0400
+
 git-annex (3.20120522) unstable; urgency=low
 
   * Pass -a to cp even when it supports --reflink=auto, to preserve

From 7a505a807b2f05046bc8d0093066a915af44279b Mon Sep 17 00:00:00 2001
From: "https://launchpad.net/~gdr-go2" 
Date: Mon, 28 May 2012 18:12:49 +0000
Subject: [PATCH 3527/8313] Added a comment: FAT symlinks

---
 .../comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment

diff --git a/doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment b/doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment
new file mode 100644
index 0000000000..389eac026d
--- /dev/null
+++ b/doc/design/assistant/android/comment_1_8be9a74e5fc4641c2bf2e1bb7673dd59._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://launchpad.net/~gdr-go2"
+ nickname="gdr-go2"
+ subject="FAT symlinks"
+ date="2012-05-28T18:12:10Z"
+ content="""
+It's a linux kernel so perhaps another option would be to create a big file and mount -o loop
+"""]]

From 5a270415ef8ce94fee3ed1eb6ddc00f6e579dfcc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 14:25:33 -0400
Subject: [PATCH 3528/8313] update

---
 doc/design/assistant/android.mdwn | 64 +++++++++++++++++++++++++------
 1 file changed, 52 insertions(+), 12 deletions(-)

diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index 24791501da..e8f192748c 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -3,6 +3,8 @@ Porting git-annex to Android will use the Android native SDK.
 A hopefully small Java app will be developed, which runs the webapp
 daemon, and a web browser to display it.
 
+[[!toc]]
+
 ### programs to port
 
 These will probably need to be bundled into the Android app, unless already
@@ -11,21 +13,59 @@ available in the App Store.
 * ssh (native ssh needed for scp, not a client like ConnectBot)
 * rsync
 * gpg
-* git (not all git commands are needed, but a core plumbing and a few like `git-add` are.)
+* git (not all git commands are needed,
+  but core plumbing and a few like `git-add` are.)
 
 ### FAT sucks
 
 The main media partition will use some awful FAT filesystem format from
-1982 that cannot support git-annex's symlinks. Hopefully it can at least
-handle all of git's filenames. Possible approaches to this:
-
-* Keep only a bare git repo on Android. The app would then need to include
-  a file browser to access the files in there, and adding a file would move
-  it into the repo. Not ideal.
-* Implement [[smudge]] filters to avoid needing symlinks. Difficult.
-* Use a bare git repo but don't keep files in `annex/objects`, instead
-  leave them outside the repo, and add some local mapping to find them.
-  Seems best?
-* Use a `LD_PRELOAD` wrapper to do Something Crazy.
+1982 that cannot support git-annex's symlinks. (Hopefully it can at least
+handle all of git's filenames.) Possible approaches to this follow.
 
 (May want to consider which of these would make a Windows port easier too.)
+
+#### bare git repo with file browser
+
+Keep only a bare git repo on Android. The app would then need to include
+a file browser to access the files in there, and adding a file would move
+it into the repo. 
+
+Not ideal.
+
+#### implement git smudge filters
+
+See [[smudge]].
+
+Difficult. Would make git-annex generally better.
+
+#### keep files outside bare git repo
+
+Use a bare git repo but don't keep files in `annex/objects`, instead
+leave them outside the repo, and add some local mapping to find them.
+
+Problem: Would leave files unlocked to modification, which might lose a
+version git-annex dependend upon existing on the phone. (Maybe the phone
+would have to be always considered an untrusted repo, which probably
+makes sense anyway.)
+
+Problem: 
+
+#### crazy `LD_PRELOAD` wrapper
+
+Need I say more? (Also, Android's linker may not even support it.)
+
+### partial content
+
+On a regular system, a reasonable simplifying assumption is that all the
+files in the folder will be synced to the system. A user might want to
+disable syncing of some subdirectories, for eg, archived files. But in
+general, things are simpler to understand and implement if all files sync.
+
+But, a phone probably cannot hold all a user's files. Indeed, it's likely
+that old files will be aggressively dropped from the phone after syncing to
+elsewhere, in order to keep enough free space on it for new files. 
+
+There needs to be a way for the user to browse files not on the phone and
+request they be transferred to it. This could be done as a browser in the
+web app, or using a subdirectory full of placeholder files (not symlinks;
+see above) that start transfer of the real file when accessed.

From 5d178d2aca257347fa0d402a933a45a350bd8ad5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 14:29:21 -0400
Subject: [PATCH 3529/8313] reorg

---
 doc/design/assistant.mdwn                 |  2 +-
 doc/design/assistant/android.mdwn         | 18 ------------------
 doc/design/assistant/partial_content.mdwn | 14 ++++++++++++++
 3 files changed, 15 insertions(+), 19 deletions(-)
 create mode 100644 doc/design/assistant/partial_content.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 3bbd27c581..9a9e4fd9fa 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -11,7 +11,7 @@ Feel free to chip in with comments! --[[Joey]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
 * Month 3 "easy setup": [[!traillink configurators]]
 * Month 4 "release": [[!traillink leftovers]]; release
-* Months 5-6 "9k bonus round": [[!traillink Android]]
+* Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]]
 
 ## not yet on the map:
 
diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index e8f192748c..d936d53cc1 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -3,8 +3,6 @@ Porting git-annex to Android will use the Android native SDK.
 A hopefully small Java app will be developed, which runs the webapp
 daemon, and a web browser to display it.
 
-[[!toc]]
-
 ### programs to port
 
 These will probably need to be bundled into the Android app, unless already
@@ -53,19 +51,3 @@ Problem:
 #### crazy `LD_PRELOAD` wrapper
 
 Need I say more? (Also, Android's linker may not even support it.)
-
-### partial content
-
-On a regular system, a reasonable simplifying assumption is that all the
-files in the folder will be synced to the system. A user might want to
-disable syncing of some subdirectories, for eg, archived files. But in
-general, things are simpler to understand and implement if all files sync.
-
-But, a phone probably cannot hold all a user's files. Indeed, it's likely
-that old files will be aggressively dropped from the phone after syncing to
-elsewhere, in order to keep enough free space on it for new files. 
-
-There needs to be a way for the user to browse files not on the phone and
-request they be transferred to it. This could be done as a browser in the
-web app, or using a subdirectory full of placeholder files (not symlinks;
-see above) that start transfer of the real file when accessed.
diff --git a/doc/design/assistant/partial_content.mdwn b/doc/design/assistant/partial_content.mdwn
new file mode 100644
index 0000000000..5572811d48
--- /dev/null
+++ b/doc/design/assistant/partial_content.mdwn
@@ -0,0 +1,14 @@
+On a regular system, a reasonable simplifying assumption is that all the
+files in the folder will be synced to the system. A user might want to
+disable syncing of some subdirectories, for eg, archived files. But in
+general, things are simpler to understand and implement if all files sync.
+
+But, an Android gadget probably cannot hold all a user's files. Indeed,
+it's likely that old files will be aggressively dropped from the Android
+after syncing to elsewhere, in order to keep enough free space on it for
+new files. 
+
+There needs to be a way for the user to browse files not on the gadget and
+request they be transferred to it. This could be done as a browser in the
+web app, or using a subdirectory full of placeholder files (not symlinks;
+see [[Android]]) that start transfer of the real file when accessed.

From a5da677b9e37747dfb86eda998ce92bd5ff80d03 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 14:41:23 -0400
Subject: [PATCH 3530/8313] update

---
 doc/design/assistant.mdwn         |  2 +-
 doc/design/assistant/cloud.mdwn   | 28 ++++++++++++++++++++++++++++
 doc/design/assistant/syncing.mdwn |  6 +++---
 3 files changed, 32 insertions(+), 4 deletions(-)
 create mode 100644 doc/design/assistant/cloud.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 9a9e4fd9fa..63f3c56d6f 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -10,7 +10,7 @@ Feel free to chip in with comments! --[[Joey]]
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
 * Month 3 "easy setup": [[!traillink configurators]]
-* Month 4 "release": [[!traillink leftovers]]; release
+* Month 4 "polishing": [[!traillink cloud]] [[!traillink leftovers]]
 * Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]]
 
 ## not yet on the map:
diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn
new file mode 100644
index 0000000000..1d612feac3
--- /dev/null
+++ b/doc/design/assistant/cloud.mdwn
@@ -0,0 +1,28 @@
+The [[syncing]] design assumes the network is connected. But it's often
+not in these pre-IPV6 days, so the cloud needs to be used to bridge between
+LANS.
+
+## more cloud providers
+
+Git-annex already supports several cloud providers via [[special_remotes].
+More should be added, such as:
+
+* Google drive (attractive because it's free)
+* OpenStack Swift
+* Box.com (it's free, and current method is hard to set up and a sorta
+  shakey)
+* Dropbox? That would be ironic..
+
+## limited space
+
+When syncing via the cloud, space there is probably limited, so
+users with more files than cloud space will want to be able to use the
+cloud as a temporary transfer point, which files are removed from after
+they've propigated out.
+
+Other users will want to use the cloud as the canonical or backup location
+of their data, and would want a copy of all their files to be kept there.
+That's also ok.
+
+git-annex will need a way to tell the difference between these, either
+heuristically, or via configuration.
diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn
index 9cbdddcb96..0813b8b70b 100644
--- a/doc/design/assistant/syncing.mdwn
+++ b/doc/design/assistant/syncing.mdwn
@@ -29,8 +29,8 @@ This probably will need lots of refinements to get working well.
 
 ## other considerations
 
-This assumes the network is connected. It's often not, so the
-cloud needs to be used to bridge between LANs.
-
 It would be nice if, when a USB drive is connected,
 syncing starts automatically.
+
+This assumes the network is connected. It's often not, so the
+[[cloud]] needs to be used to bridge between LANs.

From 98adee4369b97036dd69a9c145a10c631fbae69d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 14:43:27 -0400
Subject: [PATCH 3531/8313] typos

---
 doc/design/assistant/cloud.mdwn  | 2 +-
 doc/design/assistant/deltas.mdwn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn
index 1d612feac3..7759bb7993 100644
--- a/doc/design/assistant/cloud.mdwn
+++ b/doc/design/assistant/cloud.mdwn
@@ -4,7 +4,7 @@ LANS.
 
 ## more cloud providers
 
-Git-annex already supports several cloud providers via [[special_remotes].
+Git-annex already supports several cloud providers via [[special_remotes]].
 More should be added, such as:
 
 * Google drive (attractive because it's free)
diff --git a/doc/design/assistant/deltas.mdwn b/doc/design/assistant/deltas.mdwn
index cf2d9f6c34..ff4185a18f 100644
--- a/doc/design/assistant/deltas.mdwn
+++ b/doc/design/assistant/deltas.mdwn
@@ -1,4 +1,4 @@
-Speed up syncing of modified versions of to existing files. 
+Speed up syncing of modified versions of existing files. 
 
 One simple way is to find the key of the old version of a file that's
 being transferred, so it can be used as the basis for rsync, or any

From 8139731708290a516a844dc2cdbeab031e185119 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 14:47:16 -0400
Subject: [PATCH 3532/8313] update

---
 doc/design/assistant/cloud.mdwn | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn
index 7759bb7993..d8e7f09204 100644
--- a/doc/design/assistant/cloud.mdwn
+++ b/doc/design/assistant/cloud.mdwn
@@ -7,11 +7,11 @@ LANS.
 Git-annex already supports several cloud providers via [[special_remotes]].
 More should be added, such as:
 
-* Google drive (attractive because it's free)
-* OpenStack Swift
+* Google drive (attractive because it's free, only 5 gb tho)
+* OpenStack Swift (teh future)
 * Box.com (it's free, and current method is hard to set up and a sorta
-  shakey)
-* Dropbox? That would be ironic..
+  shakey; a better method would be to use its API)
+* Dropbox? That would be ironic.. Via its API, presumably.
 
 ## limited space
 

From 71c3196356539bccb533828b6cb01fdffa2bedc6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 15:07:32 -0400
Subject: [PATCH 3533/8313] update OSX instructions to use ghc7.0 branch

Needed until the next Haskell Platform provides a new ghc.
---
 doc/install/OSX.mdwn | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index f65e0bb4fa..ffd9424372 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,4 +1,6 @@
-Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]]. The version provided by Macports is too old to work with current versions of git-annex. Then execute
+Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]].
+The version provided by Macports is too old to work with current versions of git-annex.
+Then execute
 
 
 sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
@@ -8,8 +10,15 @@ sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that al
 # optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
+git clone git://git-annex.branchable.com/ git-annex
+cd git-annex
+git checkout ghc7.0
+
 sudo cabal update
-cabal install git-annex --bindir=$HOME/bin
+cabal install --only-dependencies
+cabal configure
+cabal build
+cabal install --bindir=$HOME/bin
 

 
 Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]

From 4ad541b1c8f0051690b81a01e64c3c696a5b1aea Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 28 May 2012 18:25:54 -0400
Subject: [PATCH 3534/8313] note

---
 doc/design/assistant/cloud.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn
index d8e7f09204..ae850c7203 100644
--- a/doc/design/assistant/cloud.mdwn
+++ b/doc/design/assistant/cloud.mdwn
@@ -26,3 +26,5 @@ That's also ok.
 
 git-annex will need a way to tell the difference between these, either
 heuristically, or via configuration.
+
+Also needed for USB keys and Android gadgets.

From a1195e92a33ce535502168eaa7ee44c95d20b544 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Tue, 29 May 2012 12:24:27 +0000
Subject: [PATCH 3535/8313] Added a comment: ANNEX_S3 vs AWS for keys

---
 .../comment_1_666a26f95024760c99c627eed37b1966._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment

diff --git a/doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment b/doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment
new file mode 100644
index 0000000000..60d96cb44e
--- /dev/null
+++ b/doc/tips/using_Amazon_S3/comment_1_666a26f95024760c99c627eed37b1966._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="ANNEX_S3 vs AWS for keys"
+ date="2012-05-29T12:24:25Z"
+ content="""
+The instructions state ANNEX_S3_ACCESS_KEY_ID and ANNEX_SECRET_ACCESS_KEY but git-annex cannot connect with those constants. git-annex tells me to set both \"AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY\" instead, which works. This is with Xubuntu 12.04.
+"""]]

From f1bcd7c218899ec455b454e2affc0235ab969e6f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Tue, 29 May 2012 12:40:25 +0000
Subject: [PATCH 3536/8313] Added a comment: environment variables

---
 .../comment_1_4a1f7a230dad6caa84831685b236fd73._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/special_remotes/S3/comment_1_4a1f7a230dad6caa84831685b236fd73._comment

diff --git a/doc/special_remotes/S3/comment_1_4a1f7a230dad6caa84831685b236fd73._comment b/doc/special_remotes/S3/comment_1_4a1f7a230dad6caa84831685b236fd73._comment
new file mode 100644
index 0000000000..17e35e7d99
--- /dev/null
+++ b/doc/special_remotes/S3/comment_1_4a1f7a230dad6caa84831685b236fd73._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="environment variables"
+ date="2012-05-29T12:40:25Z"
+ content="""
+Just noting that the environment variables `ANNEX_S3_ACCESS_KEY_ID` and `ANNEX_S3_SECRET_ACCESS_KEY` seem to have been changed to `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
+"""]]

From 21ca84cebf825f98d5521bbec2288f04e75e1505 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Tue, 29 May 2012 13:23:57 +0000
Subject: [PATCH 3537/8313]

---
 doc/forum/Getting_started_with_Amazon_S3.mdwn | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/forum/Getting_started_with_Amazon_S3.mdwn

diff --git a/doc/forum/Getting_started_with_Amazon_S3.mdwn b/doc/forum/Getting_started_with_Amazon_S3.mdwn
new file mode 100644
index 0000000000..1ee86b57fa
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3.mdwn
@@ -0,0 +1,28 @@
+I'm just getting started with git-annex and trying to wrap my head around using it with Amazon S3. I am familiar with using git, but things are a bit different as we can't init a repo at S3 directly.
+
+I've followed http://git-annex.branchable.com/tips/using_Amazon_S3/, and performed:
+
+`git init`

+Initialized empty Git repository in /home/

+`git annex init`

+init  ok

+`git annex initremote s3 type=S3 encryption=FOOBAR bucket=foo`

+initremote s3 (encryption setup with gpg key YGTVT51715TFR) (checking bucket...) (gpg) ok

+`git annex describe s3 "Amazon S3"`

+describe s3 ok

+`git annexx add foo/`

+add foo/bar.txt

+add foo/bar.png

+...etc

+`git annex sync`

+51 files changed, 51 insertions(+)

+create mode 120000 foo/bar.txt

+create mode 120000 foo/bar.png

+...etc

+
+
+Looking at http://git-annex.branchable.com/git-annex/, I thought the files added would then be pushed to S3 by git annex sync, but that doesn't seem to be the case. I've also tried variations of got annex copy, like `git annex copy . --to s3`, without any luck.
+
+Is there a way to push to s3?
+
+Any help is appreciated!

From f97009fffe18570a01ecda6af25807f33999dd1f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 May 2012 15:01:36 -0400
Subject: [PATCH 3538/8313] correct S3 environment variable names

---
 doc/special_remotes/S3.mdwn   | 4 ++--
 doc/tips/using_Amazon_S3.mdwn | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index d4d3d02388..f34b078ae1 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -6,8 +6,8 @@ See [[tips/using_Amazon_S3]] and
 
 ## configuration
 
-The standard environment variables `ANNEX_S3_ACCESS_KEY_ID` and 
-`ANNEX_S3_SECRET_ACCESS_KEY` are used to supply login credentials
+The standard environment variables `AWS_S3_ACCESS_KEY_ID` and 
+`AWS_S3_SECRET_ACCESS_KEY` are used to supply login credentials
 for Amazon. When encryption is enabled, they are stored in encrypted form
 by `git annex initremote`, so you do not need to keep the environment
 variables set after the initial initalization of the remote.
diff --git a/doc/tips/using_Amazon_S3.mdwn b/doc/tips/using_Amazon_S3.mdwn
index b59ca9b4f8..d941a19f01 100644
--- a/doc/tips/using_Amazon_S3.mdwn
+++ b/doc/tips/using_Amazon_S3.mdwn
@@ -4,8 +4,8 @@ Amazon S3, and use git-annex to transfer files into the cloud.
 
 First, export your S3 credentials:
 
-	# export ANNEX_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	# export ANNEX_S3_SECRET_ACCESS_KEY="s3kr1t"
+	# export AWS_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export AWS_S3_SECRET_ACCESS_KEY="s3kr1t"
 
 Now, create a gpg key, if you don't already have one. This will be used
 to encrypt everything stored in S3, for your privacy. Once you have

From f9b24d1a47a863ea10ada18c4db88ccdc9c5dac7 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Tue, 29 May 2012 19:09:50 +0000
Subject: [PATCH 3539/8313] Added a comment

---
 ...comment_1_f50883133d5d4903cc95c0dcaa52d052._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment

diff --git a/doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment b/doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment
new file mode 100644
index 0000000000..b2211fa6c1
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3/comment_1_f50883133d5d4903cc95c0dcaa52d052._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 1"
+ date="2012-05-29T19:09:50Z"
+ content="""
+`git annex sync` only syncs git metadata, not file contents, and metadata is not stored on S3, so it does notthing (much).
+
+`git annex move . --to s3` or `git annex copy . --to s3` is the right way to send the files to S3. I'm not sure why you say it's not working. I'd try it but Amazon is not letting me sign up for S3 again right now. Can you show what goes wrong with copy?
+"""]]

From 5c82bcf886eb71f773ec42d9648368499707fc14 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Tue, 29 May 2012 19:10:42 +0000
Subject: [PATCH 3540/8313] Added a comment

---
 .../comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment

diff --git a/doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment b/doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment
new file mode 100644
index 0000000000..dc809cb126
--- /dev/null
+++ b/doc/tips/using_Amazon_S3/comment_2_f5a0883be7dbb421b584c6dc0165f1ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-29T19:10:42Z"
+ content="""
+Thanks, I've fixed that. (You could have too.. this is a wiki ;)
+"""]]

From bb393ea24a8687532164569ca1aa58ac73f264da Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Tue, 29 May 2012 19:10:46 +0000
Subject: [PATCH 3541/8313] Added a comment

---
 .../comment_2_5b22d67de946f4d34a4a3c7449d32988._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/special_remotes/S3/comment_2_5b22d67de946f4d34a4a3c7449d32988._comment

diff --git a/doc/special_remotes/S3/comment_2_5b22d67de946f4d34a4a3c7449d32988._comment b/doc/special_remotes/S3/comment_2_5b22d67de946f4d34a4a3c7449d32988._comment
new file mode 100644
index 0000000000..f535559aeb
--- /dev/null
+++ b/doc/special_remotes/S3/comment_2_5b22d67de946f4d34a4a3c7449d32988._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 2"
+ date="2012-05-29T19:10:46Z"
+ content="""
+Thanks, I've fixed that. (You could have too.. this is a wiki ;)
+"""]]

From 0fcbf22ed6d4bbf7ac0df7a546a00e164e23b135 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 May 2012 19:17:38 -0400
Subject: [PATCH 3542/8313] updates

---
 doc/design/assistant/cloud.mdwn  | 15 ++++++++++++++-
 doc/design/assistant/webapp.mdwn |  1 +
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/cloud.mdwn b/doc/design/assistant/cloud.mdwn
index ae850c7203..bec3bc36b5 100644
--- a/doc/design/assistant/cloud.mdwn
+++ b/doc/design/assistant/cloud.mdwn
@@ -4,7 +4,8 @@ LANS.
 
 ## more cloud providers
 
-Git-annex already supports several cloud providers via [[special_remotes]].
+Git-annex already supports storing large files in 
+several cloud providers via [[special_remotes]].
 More should be added, such as:
 
 * Google drive (attractive because it's free, only 5 gb tho)
@@ -28,3 +29,15 @@ git-annex will need a way to tell the difference between these, either
 heuristically, or via configuration.
 
 Also needed for USB keys and Android gadgets.
+
+## storing git repos in the cloud
+
+Of course, one option is to just use github etc to store the git repo.
+
+Two things can store git repos in Anazon S3:
+* 
+* 
+
+Another option is to not store the git repo in the cloud, but push/pull
+peer-to-peer. When peers cannot directly talk to one-another, this could be
+bounced through something like XMPP.
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index c0f6b893e1..5322fe8a5f 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -14,6 +14,7 @@ The webapp is a web server that displays a shiny interface.
 * progress bars for each file
 * drag and drop to reorder
 * cancel and pause
+* keep it usable w/o javascript, and accessible to blind, etc
 
 ## implementation
 

From 8cca7616ae3a84e64bd3d0a880f09f18b4394b8e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Wed, 30 May 2012 00:25:22 +0000
Subject: [PATCH 3543/8313]

---
 doc/special_remotes/S3.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index f34b078ae1..195693b3b3 100644
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -6,8 +6,8 @@ See [[tips/using_Amazon_S3]] and
 
 ## configuration
 
-The standard environment variables `AWS_S3_ACCESS_KEY_ID` and 
-`AWS_S3_SECRET_ACCESS_KEY` are used to supply login credentials
+The standard environment variables `AWS_ACCESS_KEY_ID` and 
+`AWS_SECRET_ACCESS_KEY` are used to supply login credentials
 for Amazon. When encryption is enabled, they are stored in encrypted form
 by `git annex initremote`, so you do not need to keep the environment
 variables set after the initial initalization of the remote.

From 66d885aabc5e4afe94e303a2bd723970c6f36e48 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Wed, 30 May 2012 00:26:36 +0000
Subject: [PATCH 3544/8313] Added a comment

---
 .../comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/special_remotes/S3/comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment

diff --git a/doc/special_remotes/S3/comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment b/doc/special_remotes/S3/comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment
new file mode 100644
index 0000000000..abb6aacc96
--- /dev/null
+++ b/doc/special_remotes/S3/comment_3_bcab2bd0f168954243aa9bcc9671bd94._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 3"
+ date="2012-05-30T00:26:33Z"
+ content="""
+Thanks! Being new here, I didn't want to overstep my boundaries. I've gone ahead and made a small edit and will do so elsewhere as needed.
+"""]]

From 2150b8790645be32c8dd79b413e147daa8816c24 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Wed, 30 May 2012 00:27:23 +0000
Subject: [PATCH 3545/8313]

---
 doc/tips/using_Amazon_S3.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/tips/using_Amazon_S3.mdwn b/doc/tips/using_Amazon_S3.mdwn
index d941a19f01..128819fcbb 100644
--- a/doc/tips/using_Amazon_S3.mdwn
+++ b/doc/tips/using_Amazon_S3.mdwn
@@ -4,8 +4,8 @@ Amazon S3, and use git-annex to transfer files into the cloud.
 
 First, export your S3 credentials:
 
-	# export AWS_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	# export AWS_S3_SECRET_ACCESS_KEY="s3kr1t"
+	# export AWS_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
+	# export AWS_SECRET_ACCESS_KEY="s3kr1t"
 
 Now, create a gpg key, if you don't already have one. This will be used
 to encrypt everything stored in S3, for your privacy. Once you have

From c5412ca7fed26dcc294c5873a07a28f174b255ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 29 May 2012 20:27:52 -0400
Subject: [PATCH 3546/8313] fix

---
 doc/tips/using_Amazon_S3.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_Amazon_S3.mdwn b/doc/tips/using_Amazon_S3.mdwn
index d941a19f01..380326d6f7 100644
--- a/doc/tips/using_Amazon_S3.mdwn
+++ b/doc/tips/using_Amazon_S3.mdwn
@@ -5,7 +5,7 @@ Amazon S3, and use git-annex to transfer files into the cloud.
 First, export your S3 credentials:
 
 	# export AWS_S3_ACCESS_KEY_ID="08TJMT99S3511WOZEP91"
-	# export AWS_S3_SECRET_ACCESS_KEY="s3kr1t"
+	# export AWS_SECRET_ACCESS_KEY="s3kr1t"
 
 Now, create a gpg key, if you don't already have one. This will be used
 to encrypt everything stored in S3, for your privacy. Once you have

From 2a61c5f2869c562ebd4433802c132b9bfac49ec4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
 
Date: Wed, 30 May 2012 00:40:48 +0000
Subject: [PATCH 3547/8313] Added a comment

---
 .../comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment

diff --git a/doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment b/doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment
new file mode 100644
index 0000000000..742e8d4469
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3/comment_2_e90aa3259d9a12cd67daa27d42d69ab5._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnoUOqs_lbuWyZBqyU6unHgUduJwDDgiKY"
+ nickname="Matt"
+ subject="comment 2"
+ date="2012-05-30T00:40:45Z"
+ content="""
+It's strange. I've done some testing on another machine, and this one, and the issue seems to be with adding only certain sub-directories of the git-annex directory. Would it cause an issue with git-annex if a sub-directory was a git repo?
+"""]]

From 1209972509a6014df2b3d85c1f3fb6f418c7c27d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlBSuP2uCSUr5ONRVoyL7jg0ZhBc5us_w0"
 
Date: Wed, 30 May 2012 00:51:29 +0000
Subject: [PATCH 3548/8313] smudge broken link

---
 doc/design/assistant/android.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index d936d53cc1..a33029b9db 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -32,7 +32,7 @@ Not ideal.
 
 #### implement git smudge filters
 
-See [[smudge]].
+See [[todo/smudge]].
 
 Difficult. Would make git-annex generally better.
 

From bdb3c573934728c59c9dab6295cab8d65056bad5 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Wed, 30 May 2012 00:54:38 +0000
Subject: [PATCH 3549/8313] Added a comment

---
 .../comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment

diff --git a/doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment b/doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment
new file mode 100644
index 0000000000..450a1513ce
--- /dev/null
+++ b/doc/forum/Getting_started_with_Amazon_S3/comment_3_c3adce7c0f29e71ed9dd07103ede2c1a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.81.112"
+ subject="comment 3"
+ date="2012-05-30T00:54:38Z"
+ content="""
+If the subdirectory has a .git, then it's a separate git repo, and inside the directory, all git (and git-annex) commands in it will operate on that nested repo and ignore the outside one.
+"""]]

From 65977a558446fd4208b04614191b68bae838b044 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 May 2012 17:01:22 -0400
Subject: [PATCH 3550/8313] lock: Reset unlocked file to index, rather than to
 branch head.

Resetting an unlocked file to the branch head failed if it had just been
added, not committed, and unlocked, since the branch didbn't have it.

The code was concerned about dropping any changes that might be staged in the
index, but I cannot see why.
---
 Command/Lock.hs                                             | 6 +-----
 debian/changelog                                            | 1 +
 doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn | 6 ++++--
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/Command/Lock.hs b/Command/Lock.hs
index b8aedb252b..ab97b14bcc 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -24,9 +24,5 @@ start file = do
 
 perform :: FilePath -> CommandPerform
 perform file = do
-	liftIO $ removeFile file
-	-- Checkout from HEAD to get rid of any changes that might be 
-	-- staged in the index, and get back to the previous symlink to
-	-- the content.
-	Annex.Queue.add "checkout" [Param "HEAD", Param "--"] [file]
+	Annex.Queue.add "checkout" [Param "--"] [file]
 	next $ return True -- no cleanup needed
diff --git a/debian/changelog b/debian/changelog
index 9c38d1727f..6b57a5580c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 git-annex (3.20120523) UNRELEASED; urgency=low
 
   * sync: Show a nicer message if a user tries to sync to a special remote.
+  * lock: Reset unlocked file to index, rather than to branch head.
 
  -- Joey Hess   Sun, 27 May 2012 20:55:29 -0400
 
diff --git a/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn b/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn
index 69704bdd07..9c093de389 100644
--- a/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn
+++ b/doc/bugs/unlock_then_lock_of_uncommitted_file_loses_it.mdwn
@@ -1,5 +1,7 @@
-Add a file, then unlock it, and then lock it. There is an error and the
-symlink gets deleted. 
+Add a file (do not commit), then unlock it, and then lock it.
+There is an error and the symlink gets deleted.
 
 The file will still be staged in the index, and the file content is still
 in the annex. --[[Joey]] 
+
+[[done]]

From 29b43cfa395c87f722be35fb45cfea94cd3338ea Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 30 May 2012 20:50:53 -0400
Subject: [PATCH 3551/8313] pairing

---
 doc/design/assistant.mdwn               |  2 +-
 doc/design/assistant/configurators.mdwn |  7 +------
 doc/design/assistant/pairing.mdwn       | 13 +++++++++++++
 3 files changed, 15 insertions(+), 7 deletions(-)
 create mode 100644 doc/design/assistant/pairing.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 63f3c56d6f..7a720a5e08 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -9,7 +9,7 @@ Feel free to chip in with comments! --[[Joey]]
 
 * Month 1 "like dropbox": [[!traillink inotify]] [[!traillink syncing]]
 * Month 2 "shiny webapp": [[!traillink webapp]] [[!traillink progressbars]]
-* Month 3 "easy setup": [[!traillink configurators]]
+* Month 3 "easy setup": [[!traillink configurators]] [[!traillink pairing]]
 * Month 4 "polishing": [[!traillink cloud]] [[!traillink leftovers]]
 * Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]]
 
diff --git a/doc/design/assistant/configurators.mdwn b/doc/design/assistant/configurators.mdwn
index b9e3a66328..e0e938efdd 100644
--- a/doc/design/assistant/configurators.mdwn
+++ b/doc/design/assistant/configurators.mdwn
@@ -10,12 +10,7 @@ through setting up common use cases.
 * Create a repository (run when the web app is started without a configured
   repository too).
 * Clone this repo to a USB drive.
-* Clone this repo to another host:
-  1. Prompt for the hostname (or do avahi local machine discovery). 
-  2. Enable the two hosts to ssh to one-another and run git-annex shell.
-     (A tricky problem, if ssh keys need to be added to do that.)
-  3. Push over a clone of the repository. (Using git-annex-shell?)
-  4. Start [[syncing]].
+* Clone this repo to another host. (Needs [[pairing]])
 * Set up Amazon S3.
 * Set up rsync remote.
 * Set up encryption.
diff --git a/doc/design/assistant/pairing.mdwn b/doc/design/assistant/pairing.mdwn
new file mode 100644
index 0000000000..f33c5e11de
--- /dev/null
+++ b/doc/design/assistant/pairing.mdwn
@@ -0,0 +1,13 @@
+For git-annex to be able to clone its repo to another host, it'd be good to
+have some way of pairing devices.
+
+It could work like this:
+
+1. Prompt for the hostname (or do avahi local machine discovery). 
+2. Enable the two hosts to ssh to one-another and run git-annex shell.
+   (A tricky problem, if ssh keys need to be added to do that.)
+3. Push over a clone of the repository. (Using git-annex-shell?)
+4. Start [[syncing]].
+
+Also look into the method used by
+

From 8c759893384d78bde6247ab200fab696113186dc Mon Sep 17 00:00:00 2001
From: "http://dlaxalde.myopenid.com/" 
Date: Thu, 31 May 2012 14:36:33 +0000
Subject: [PATCH 3552/8313] Added a comment

---
 ...omment_1_cf19b8dc304dc37c26717174c4a98aa4._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment

diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment
new file mode 100644
index 0000000000..a7fce26ef8
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_1_cf19b8dc304dc37c26717174c4a98aa4._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://dlaxalde.myopenid.com/"
+ nickname="dl"
+ subject="comment 1"
+ date="2012-05-31T14:36:33Z"
+ content="""
+Is there a way to have git-annex completely ignore a repository? I see that
+the `dead` command adds the uuid of the repository to `trust.log` but does
+not change `uuid.log`. Is it enough to remove the corresponding line in
+`uuid.log` and `trust.log`?
+"""]]

From 4f3b83d33043f07cbf94a6260052c1e20737bc7b Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 31 May 2012 17:00:59 +0000
Subject: [PATCH 3553/8313] Added a comment

---
 .../comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment

diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment
new file mode 100644
index 0000000000..0e19b806c3
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.243"
+ subject="comment 2"
+ date="2012-05-31T17:00:59Z"
+ content="""
+`dead
+"""]]

From 9c72d42cf4881840b51535091c23126857260f98 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 31 May 2012 17:01:37 +0000
Subject: [PATCH 3554/8313] Added a comment

---
 .../comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment

diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment
new file mode 100644
index 0000000000..a8d044c287
--- /dev/null
+++ b/doc/tips/what_to_do_when_you_lose_a_repository/comment_3_fa9ca81668f5faebf2f61b10f82c97d2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.243"
+ subject="comment 3"
+ date="2012-05-31T17:01:37Z"
+ content="""
+`dead` is the best we can do. The automatic merging used on the git-annex branch tends to re-add lines that are deleted in one repo when merging with another that still has them.
+"""]]

From cbace05372b05d0b7a73561e497edcae9b4abd6d Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 31 May 2012 17:01:51 +0000
Subject: [PATCH 3555/8313] removed

---
 .../comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment   | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment

diff --git a/doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment b/doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment
deleted file mode 100644
index 0e19b806c3..0000000000
--- a/doc/tips/what_to_do_when_you_lose_a_repository/comment_2_e2f5d7d3efaaa8833a2cf2e9f1bd9962._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="http://joeyh.name/"
- ip="4.153.8.243"
- subject="comment 2"
- date="2012-05-31T17:00:59Z"
- content="""
-`dead
-"""]]

From 595d13020b4004b1813cc3bc31ed1a8edf2fe511 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 15:25:26 -0400
Subject: [PATCH 3556/8313] updates

---
 doc/design/assistant/android.mdwn | 13 ++++++++++++-
 doc/design/assistant/inotify.mdwn |  4 +++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index a33029b9db..90dc551794 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -14,6 +14,14 @@ available in the App Store.
 * git (not all git commands are needed,
   but core plumbing and a few like `git-add` are.)
 
+### Android specific features
+
+The app should be aware of power status, and avoid expensive background
+jobs when low on battery or run flat out when plugged in.
+
+The app should be aware of network status, and avoid expensive data
+transfers when not on wifi. This may need to be configurable.
+
 ### FAT sucks
 
 The main media partition will use some awful FAT filesystem format from
@@ -26,10 +34,13 @@ handle all of git's filenames.) Possible approaches to this follow.
 
 Keep only a bare git repo on Android. The app would then need to include
 a file browser to access the files in there, and adding a file would move
-it into the repo. 
+it into the repo.
 
 Not ideal.
 
+Could be improved some by registering git-annex as a file handling app on
+Android, allowing you to "send to" git-annex.
+
 #### implement git smudge filters
 
 See [[todo/smudge]].
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index be38ef13cd..6bb810a755 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -8,7 +8,9 @@ useful, it needs to:
 - notice deleted files and stage the deletion
   (tricky; there's a race with add..)
 - notice renamed files, auto-fix the symlink, and stage the new file location
-- periodically auto-commit staged changes
+- periodically auto-commit staged changes (avoid autocommitting when
+  lots of changes are coming in)
+- tunable delays before adding new files, etc
 - honor .gitignore, not adding files it excludesa
 
 Also to do:

From 6ea9a88ede48fa6d37a0ac476ca775c0edaa56a4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 15:28:04 -0400
Subject: [PATCH 3557/8313] update

---
 doc/design/assistant/webapp.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 5322fe8a5f..857b5972bf 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -16,6 +16,11 @@ The webapp is a web server that displays a shiny interface.
 * cancel and pause
 * keep it usable w/o javascript, and accessible to blind, etc
 
+## other features
+
+* there could be a UI to export a file, which would make it be served up
+  over http by the web app
+
 ## implementation
 
 Hope to use Yesod.

From 2941d30bda3adc7420dd8f800192d7154e960bc2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 15:48:26 -0400
Subject: [PATCH 3558/8313] update

---
 doc/design/assistant/webapp.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 857b5972bf..abf7b38c94 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -7,6 +7,9 @@ The webapp is a web server that displays a shiny interface.
   token. This guards against other users on the same system.
 * I would like to avoid passwords or other authentication methods,
   it's your local system.
+* Alternative for Linux at least would be to write a small program using
+  GTK+ Webkit, that runs the webapp, and can know what user ran it, avoiding
+  needing authentication.
 
 ## interface
 

From 263ee1d067c8ef7361ecdb096ee227011f8d25dd Mon Sep 17 00:00:00 2001
From: josh 
Date: Thu, 31 May 2012 20:02:48 +0000
Subject: [PATCH 3559/8313] Add explanation of how git on Windows handles
 symlinks.

---
 doc/design/assistant/windows.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/windows.mdwn b/doc/design/assistant/windows.mdwn
index 0b176934b9..89c3082a85 100644
--- a/doc/design/assistant/windows.mdwn
+++ b/doc/design/assistant/windows.mdwn
@@ -8,7 +8,8 @@ Stackoverflow has some details.
 
 Make git use them, as it (apparently) does not yet.
 
-(What **does** git do on Windows when it clones a repo with symlinks?)
+Currently, on Windows, git checks out symlinks as files containing the symlink
+target as their contents.
 
 ## POSIX
 

From 233d4020755429785688ab91e99bd260ffbc1490 Mon Sep 17 00:00:00 2001
From: josh 
Date: Thu, 31 May 2012 20:03:02 +0000
Subject: [PATCH 3560/8313] Fix typo.

---
 doc/design/assistant/windows.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/windows.mdwn b/doc/design/assistant/windows.mdwn
index 89c3082a85..850f98194e 100644
--- a/doc/design/assistant/windows.mdwn
+++ b/doc/design/assistant/windows.mdwn
@@ -15,4 +15,4 @@ target as their contents.
 
 Lots of ifdefs and pain to deal with POSIX calls in the code base.
 
-Or I could try to use Cywin.
+Or I could try to use Cygwin.

From c47b1ecda00cea6d9922e0733c3ab365bc2f004d Mon Sep 17 00:00:00 2001
From: josh 
Date: Thu, 31 May 2012 20:11:43 +0000
Subject: [PATCH 3561/8313] Discussion of NTFS symlinks, junction points, and
 reparse points.

---
 doc/design/assistant/windows.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/design/assistant/windows.mdwn b/doc/design/assistant/windows.mdwn
index 850f98194e..da669ad82c 100644
--- a/doc/design/assistant/windows.mdwn
+++ b/doc/design/assistant/windows.mdwn
@@ -6,6 +6,8 @@ Apparently new versions of Windows have something very like symlinks.
 (Or really, 3 or so things not entirely unlike symlinks and all different.)
 Stackoverflow has some details.
 
+NTFS supports symbolic links two different ways: an [[!wikipedia NTFS symbolic link]] and an [[!wikipedia NTFS_junction_point]].  The former seems like the closest analogue to POSIX symlinks.
+
 Make git use them, as it (apparently) does not yet.
 
 Currently, on Windows, git checks out symlinks as files containing the symlink
@@ -16,3 +18,7 @@ target as their contents.
 Lots of ifdefs and pain to deal with POSIX calls in the code base.
 
 Or I could try to use Cygwin.
+
+## Deeper system integration
+
+[NTFS Reparse Points](http://msdn.microsoft.com/en-us/library/aa365503%28v=VS.85%29.aspx) allow a program to define how the OS will interpret a file or directory in arbitrary ways.  This requires writing a file system filter.

From 3b09281b442e794213f2e296e42e2d74fddec733 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 19:25:33 -0400
Subject: [PATCH 3562/8313] add dirContentsRecursive

---
 Utility/Directory.hs | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 3041361dfd..5bfd49a9c1 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -17,6 +17,7 @@ import System.FilePath
 import Control.Applicative
 import Control.Exception (bracket_)
 import System.Posix.Directory
+import System.IO.Unsafe (unsafeInterleaveIO)
 
 import Utility.SafeCommand
 import Utility.TempFile
@@ -24,14 +25,37 @@ import Utility.Exception
 import Utility.Monad
 import Utility.Path
 
+dirCruft :: FilePath -> Bool
+dirCruft "." = True
+dirCruft ".." = True
+dirCruft _ = False
+
 {- Lists the contents of a directory.
  - Unlike getDirectoryContents, paths are not relative to the directory. -}
 dirContents :: FilePath -> IO [FilePath]
-dirContents d = map (d ) . filter notcruft <$> getDirectoryContents d
+dirContents d = map (d ) . filter (not . dirCruft) <$> getDirectoryContents d
+
+{- Gets contents of directory, and then its subdirectories, recursively,
+ - and lazily. -}
+dirContentsRecursive :: FilePath -> IO [FilePath]
+dirContentsRecursive topdir = dirContentsRecursive' topdir [""]
+
+dirContentsRecursive' :: FilePath -> [FilePath] -> IO [FilePath]
+dirContentsRecursive' _ [] = return []
+dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do
+	(files, dirs') <- collect [] [] =<< dirContents (topdir  dir)
+	files' <- dirContentsRecursive' topdir (dirs' ++ dirs)
+	return (files ++ files')
 	where
-		notcruft "." = False
-		notcruft ".." = False
-		notcruft _ = True
+		collect files dirs' [] = return (reverse files, reverse dirs')
+		collect files dirs' (entry:entries)
+			| dirCruft entry = collect files dirs' entries
+			| otherwise = do
+				let dirEntry = dir  entry
+				ifM (doesDirectoryExist $ topdir  dirEntry)
+					( collect files (dirEntry:dirs') entries
+					, collect (dirEntry:files) dirs' entries
+					)			
 
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}

From 3a10095d40cf9a9e0380b6b10e1ebe304f1537c0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 19:47:18 -0400
Subject: [PATCH 3563/8313] import: New subcommand, pulls files from a
 directory outside the annex and adds them

Use case for this was developed somewhere on the Transiberian Railroad.
---
 Command/Import.hs  | 39 +++++++++++++++++++++++++++++++++++++++
 GitAnnex.hs        |  2 ++
 Seek.hs            | 10 +++++++++-
 debian/changelog   |  2 ++
 doc/git-annex.mdwn |  9 +++++++++
 5 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 Command/Import.hs

diff --git a/Command/Import.hs b/Command/Import.hs
new file mode 100644
index 0000000000..e27a421f27
--- /dev/null
+++ b/Command/Import.hs
@@ -0,0 +1,39 @@
+{- git-annex command
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.Import where
+
+import Common.Annex
+import Command
+import qualified Annex
+import qualified Command.Add
+
+def :: [Command]
+def = [command "import" paramPaths seek "move and add files from outside git working copy"]
+
+seek :: [CommandSeek]
+seek = [withPathContents start]
+
+start :: (FilePath, FilePath) -> CommandStart
+start (srcfile, destfile) = notBareRepo $
+	ifM (liftIO $ isRegularFile <$> getSymbolicLinkStatus srcfile)
+		( do
+			showStart "import" destfile
+			next $ perform srcfile destfile
+		, stop
+		)
+
+perform :: FilePath -> FilePath -> CommandPerform
+perform srcfile destfile = do
+	whenM (liftIO $ doesFileExist destfile) $
+		unlessM (Annex.getState Annex.force) $
+			error $ "not overwriting existing " ++ destfile ++
+				" (use --force to override)"
+ 
+	liftIO $ createDirectoryIfMissing True (parentDir destfile)
+	liftIO $ moveFile srcfile destfile
+	Command.Add.perform destfile
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 9910e33d21..149b37f930 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -54,6 +54,7 @@ import qualified Command.Semitrust
 import qualified Command.Dead
 import qualified Command.Sync
 import qualified Command.AddUrl
+import qualified Command.Import
 import qualified Command.Map
 import qualified Command.Upgrade
 import qualified Command.Version
@@ -69,6 +70,7 @@ cmds = concat
 	, Command.Lock.def
 	, Command.Sync.def
 	, Command.AddUrl.def
+	, Command.Import.def
 	, Command.Init.def
 	, Command.Describe.def
 	, Command.InitRemote.def
diff --git a/Seek.hs b/Seek.hs
index 8d4f917e72..eed4a81558 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -4,7 +4,7 @@
  - the values a user passes to a command, and prepare actions operating
  - on them.
  -
- - Copyright 2010-2011 Joey Hess 
+ - Copyright 2010-2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -41,6 +41,14 @@ withFilesNotInGit a params = do
 			g <- gitRepo
 			liftIO $ (\p -> LsFiles.notInRepo force p g) l
 
+withPathContents :: ((FilePath, FilePath) -> CommandStart) -> CommandSeek
+withPathContents a params = map a . concat <$> liftIO (mapM get params)
+	where
+		get p = ifM (isDirectory <$> getFileStatus p)
+			( map (\f -> (f, makeRelative p f)) <$> dirContentsRecursive p
+			, return [(p, takeFileName p)]
+			)
+
 withWords :: ([String] -> CommandStart) -> CommandSeek
 withWords a params = return [a params]
 
diff --git a/debian/changelog b/debian/changelog
index 6b57a5580c..a110e94ce9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ git-annex (3.20120523) UNRELEASED; urgency=low
 
   * sync: Show a nicer message if a user tries to sync to a special remote.
   * lock: Reset unlocked file to index, rather than to branch head.
+  * import: New subcommand, pulls files from a directory outside the annex
+    and adds them.
 
  -- Joey Hess   Sun, 27 May 2012 20:55:29 -0400
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 5d41f86e9c..c7de59cd2a 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -160,6 +160,15 @@ subdirectories).
   alternate locations from which the file can be downloaded. In this mode,
   addurl can be used both to add new files, or to add urls to existing files.
 
+* import [path ...]
+
+  Moves files from somewhere outside the git working copy, and adds them to
+  the annex. Individual files to import can be specified. 
+  If a directory is specified, all files in it are imported, and any
+  subdirectory structure inside it is preserved.
+
+	git annex import /media/camera/DCIM/
+
 # REPOSITORY SETUP COMMANDS
 
 * init [description]

From 6fd83851c13232b7810a33e8bb1d83e8a46bd354 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 21:03:24 -0400
Subject: [PATCH 3564/8313] Fix display of warning message when encountering a
 file that uses an unsupported backend.

---
 Backend.hs       | 8 ++++----
 Locations.hs     | 4 +++-
 debian/changelog | 2 ++
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index 8071b9b835..fa32669449 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -75,16 +75,16 @@ genKey' (b:bs) file = do
  - by examining what the file symlinks to. -}
 lookupFile :: FilePath -> Annex (Maybe (Key, Backend))
 lookupFile file = do
-	tl <- liftIO $ tryIO getsymlink
+	tl <- liftIO $ tryIO $ readSymbolicLink file
 	case tl of
 		Left _ -> return Nothing
 		Right l -> makekey l
 	where
-		getsymlink = takeFileName <$> readSymbolicLink file
-		makekey l = maybe (return Nothing) (makeret l) (fileKey l)
+		makekey l = maybe (return Nothing) (makeret l) (fileKey $ takeFileName l)
 		makeret l k = let bname = keyBackendName k in
 			case maybeLookupBackendName bname of
-				Just backend -> return $ Just (k, backend)
+				Just backend -> do
+					return $ Just (k, backend)
 				Nothing -> do
 					when (isLinkToAnnex l) $ warning $
 						"skipping " ++ file ++
diff --git a/Locations.hs b/Locations.hs
index 46a85e0ee1..db456388a6 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -155,7 +155,9 @@ gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r  "remotes"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = ("/.git/" ++ objectDir) `isInfixOf` s
+isLinkToAnnex s = ("/" ++ d) `isInfixOf` s || d `isPrefixOf` s
+	where
+		d = ".git"  objectDir
 
 {- Converts a key into a filename fragment without any directory.
  -
diff --git a/debian/changelog b/debian/changelog
index a110e94ce9..61290b1aaf 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,8 @@ git-annex (3.20120523) UNRELEASED; urgency=low
   * lock: Reset unlocked file to index, rather than to branch head.
   * import: New subcommand, pulls files from a directory outside the annex
     and adds them.
+  * Fix display of warning message when encountering a file that uses an
+    unsupported backend.
 
  -- Joey Hess   Sun, 27 May 2012 20:55:29 -0400
 

From 1f951de014d0bfb3d2cf2b58e607cabefdf30d84 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 21:31:25 -0400
Subject: [PATCH 3565/8313] add a nice one

---
 doc/testimonials.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/testimonials.mdwn b/doc/testimonials.mdwn
index 65bc5a5fe2..f053c58398 100644
--- a/doc/testimonials.mdwn
+++ b/doc/testimonials.mdwn
@@ -9,6 +9,13 @@
 
 
 
+

+What excites me about GIT ANNEX is how it fundamentally tracks the
+backup and availability of any data you own, and allows you to share
+data with a large or small audience, ensuring that the data survives.
+

+-- Jason Scott 
+
 Seen on IRC:
 
 oh my god, git-annex is amazing

From f5de183c7168627e9bd969aa48f8215f411bc507 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 21:51:42 -0400
Subject: [PATCH 3566/8313] sha256sum not so optional

---
 doc/install/OSX.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index ffd9424372..08904aef9f 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,7 +7,7 @@ sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 
-# optional: this will enable the gnu tools, (to give sha224sum etc..., it does not override the BSD userland)
+# this will enable the gnu tools, (to give sha256sum etc..., it does not override the BSD userland)
 export PATH=$PATH:/opt/local/libexec/gnubin
 
 git clone git://git-annex.branchable.com/ git-annex

From 2183fd2abd95b6deaa9baef47e2f9c5f865123e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 23:15:40 -0400
Subject: [PATCH 3567/8313] Require that the SHA256 backend can be used when
 building, since it's the default.

---
 Backend/SHA.hs     | 2 +-
 Build/Configure.hs | 9 +++++----
 debian/changelog   | 2 ++
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index 3adac65d8c..c2a6cf9761 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -45,7 +45,7 @@ genBackendE size =
 
 shaCommand :: SHASize -> Maybe String
 shaCommand 1 = SysConfig.sha1
-shaCommand 256 = SysConfig.sha256
+shaCommand 256 = Just SysConfig.sha256
 shaCommand 224 = SysConfig.sha224
 shaCommand 384 = SysConfig.sha384
 shaCommand 512 = SysConfig.sha512
diff --git a/Build/Configure.hs b/Build/Configure.hs
index 341b8840dc..86a3479244 100644
--- a/Build/Configure.hs
+++ b/Build/Configure.hs
@@ -26,15 +26,16 @@ tests =
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
 	, TestCase "ssh connection caching" getSshConnectionCaching
-	] ++ shaTestCases [1, 256, 512, 224, 384]
+	] ++ shaTestCases False [1, 512, 224, 384] ++ shaTestCases True [256]
 
-shaTestCases :: [Int] -> [TestCase]
-shaTestCases l = map make l
+shaTestCases :: Bool -> [Int] -> [TestCase]
+shaTestCases required l = map make l
 	where make n =
 		let
 			cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"]
 			key = "sha" ++ show n
-		in TestCase key $ maybeSelectCmd key cmds "  Sun, 27 May 2012 20:55:29 -0400
 

From 665c0fbdaafab7b3ea2737f302a8951f26e4f01a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 31 May 2012 23:33:07 -0400
Subject: [PATCH 3568/8313] check at configure time for sha commands in Mac OSX
 location

---
 Build/Configure.hs   | 18 ++++++++++++------
 doc/install/OSX.mdwn |  3 ---
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/Build/Configure.hs b/Build/Configure.hs
index 86a3479244..2f79297ee9 100644
--- a/Build/Configure.hs
+++ b/Build/Configure.hs
@@ -6,6 +6,7 @@ import System.Directory
 import Data.List
 import System.Cmd.Utils
 import Control.Applicative
+import System.FilePath
 
 import Build.TestConfig
 import Utility.SafeCommand
@@ -30,12 +31,17 @@ tests =
 
 shaTestCases :: Bool -> [Int] -> [TestCase]
 shaTestCases required l = map make l
-	where make n =
-		let
-			cmds = map (\x -> "sha" ++ show n ++ x) ["", "sum"]
-			key = "sha" ++ show n
-			selector = if required then selectCmd else maybeSelectCmd
-		in TestCase key $ selector key cmds " [x, osxpath  x]) $
+			map (\x -> "sha" ++ show n ++ x) ["", "sum"]
+		-- Max OSX puts GNU tools outside PATH, so look in
+		-- the location it uses, and remember where to run them
+		-- from.
+		osxpath = "/opt/local/libexec/gnubin"
 
 tmpDir :: String
 tmpDir = "tmp"
diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 08904aef9f..1d8a3b9437 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -7,9 +7,6 @@ sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 
-# this will enable the gnu tools, (to give sha256sum etc..., it does not override the BSD userland)
-export PATH=$PATH:/opt/local/libexec/gnubin
-
 git clone git://git-annex.branchable.com/ git-annex
 cd git-annex
 git checkout ghc7.0

From 3812e67a054d5bd479b7a4be6d28027664a89bcb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
 
Date: Fri, 1 Jun 2012 16:05:06 +0000
Subject: [PATCH 3569/8313] Added a comment: sha256 ?

---
 .../comment_5_69f71b49e6a434cc3426989102c65edf._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment

diff --git a/doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment b/doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment
new file mode 100644
index 0000000000..b37e9732fb
--- /dev/null
+++ b/doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
+ nickname="Damien"
+ subject="sha256 ?"
+ date="2012-06-01T16:05:05Z"
+ content="""
+homebrew's `md5sha1sum` does not provide a `sha256` command that seems to be required… where does yours come from ?
+"""]]

From ca755617d6fe0835c36371a90db75b3a2b4ce4a4 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
 
Date: Fri, 1 Jun 2012 16:10:17 +0000
Subject: [PATCH 3570/8313] removed

---
 .../comment_5_69f71b49e6a434cc3426989102c65edf._comment   | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment

diff --git a/doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment b/doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment
deleted file mode 100644
index b37e9732fb..0000000000
--- a/doc/install/OSX/comment_5_69f71b49e6a434cc3426989102c65edf._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
- nickname="Damien"
- subject="sha256 ?"
- date="2012-06-01T16:05:05Z"
- content="""
-homebrew's `md5sha1sum` does not provide a `sha256` command that seems to be required… where does yours come from ?
-"""]]

From be45ac2fa0d201ec3a7270c33c59be05ec43bab3 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
 
Date: Fri, 1 Jun 2012 16:13:06 +0000
Subject: [PATCH 3571/8313] Added a comment: sha256

---
 ...comment_5_50777853f808d57b957f8ce9a0f84b3d._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment

diff --git a/doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment b/doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment
new file mode 100644
index 0000000000..eca1761786
--- /dev/null
+++ b/doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c"
+ nickname="Damien"
+ subject="sha256"
+ date="2012-06-01T16:13:05Z"
+ content="""
+If you're missing the `sha256sum` command with Homebrew, it's provided by `coreutils`. You have to change your `$PATH` before running `cabal install git-annex.cabal`:
+
+    PATH=\"$(brew --prefix coreutils)/libexec/gnubin:$PATH\"
+"""]]

From f74c287e7eea1900c2155ef200f90ba5f58c7caf Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 1 Jun 2012 17:24:29 +0000
Subject: [PATCH 3572/8313] Added a comment

---
 .../comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment

diff --git a/doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment b/doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment
new file mode 100644
index 0000000000..5cb813776b
--- /dev/null
+++ b/doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 6"
+ date="2012-06-01T17:24:29Z"
+ content="""
+Last night I made it look in /opt/local/libexec/gnubin .. if there's another directory it could look in, let me know. I am reluctant to make it run the brew command directly.
+"""]]

From 0064fc0beb98e7254dc026b614bad2e5518ea5dc Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sat, 2 Jun 2012 12:06:38 +0000
Subject: [PATCH 3573/8313] Added a comment

---
 .../comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment

diff --git a/doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment b/doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment
new file mode 100644
index 0000000000..646a03398a
--- /dev/null
+++ b/doc/design/assistant/comment_1_a48fcfbf97f0a373ea375cd8f07f0fc8._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-02T12:06:37Z"
+ content="""
+Will statically linked binaries be provided for say Linux, OSX and *BSD?  I think having some statically linked binaries will certainly help and appeal to a lot of users.
+"""]]

From 619d765646a23d7f22ac8c0dd256be10a5a278f7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 3 Jun 2012 14:27:11 -0400
Subject: [PATCH 3574/8313] simplify OSX installation instructions

The new Haskell Platform was released today, with the new ghc git-annex's
master branch needs, so cabal should be usable again on OSX.
---
 doc/install/OSX.mdwn | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/doc/install/OSX.mdwn b/doc/install/OSX.mdwn
index 1d8a3b9437..3c24609684 100644
--- a/doc/install/OSX.mdwn
+++ b/doc/install/OSX.mdwn
@@ -1,4 +1,4 @@
-Install Haskel Platform from [[http://hackage.haskell.org/platform/mac.html]].
+Install the Haskell Platform from [[http://hackage.haskell.org/platform/mac.html]].
 The version provided by Macports is too old to work with current versions of git-annex.
 Then execute
 
@@ -7,19 +7,10 @@ sudo port install git-core ossp-uuid md5sha1sum coreutils pcre
 
 sudo ln -s /opt/local/include/pcre.h  /usr/include/pcre.h # This is hack that allows pcre-light to find pcre
 
-git clone git://git-annex.branchable.com/ git-annex
-cd git-annex
-git checkout ghc7.0
-
 sudo cabal update
-cabal install --only-dependencies
-cabal configure
-cabal build
-cabal install --bindir=$HOME/bin
+cabal install git-annex --bindir=$HOME/bin
 

 
-Originally posted by Jon at  --[[Joey]], modified by [[kristianrumberg]]
-
 See also:
 
 * [[forum/OSX__39__s_haskell-platform_statically_links_things]]

From 2a75bbdaa071ecc59f8e36c57705256bb73bf5c8 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlu7K3h7Ry1uDAU_ERYGuqt0LoGNJqGuRo"
 
Date: Mon, 4 Jun 2012 03:41:57 +0000
Subject: [PATCH 3575/8313] Added a forum post asking how to make locally
 cached files stay up to date across both rename and modify.

---
 ...es_due_to_external_modification__63__.mdwn | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn

diff --git a/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn
new file mode 100644
index 0000000000..50518e33ad
--- /dev/null
+++ b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__.mdwn
@@ -0,0 +1,46 @@
+## Use case
+
+A laptop with a relatively small hard drive has copies of a subset of
+all annexed files.  When annexed files are changed externally and `git
+annex sync` is run on the laptop, the stale local copies are
+invalidated and their symlinks break.  How can I automatically fetch
+the updated versions of these previously locally-cached files?
+
+Because I only want a subset of files, I can't do
+
+    git annex add --not --in here --and --in superset.
+
+Because files may be renamed, the
+[[tips/automatically_getting_files_on_checkout/]] solution, by making
+`dir` specify the subset, will require manually and redundantly
+tracking renames.
+
+## Simple ( (?) ) feature addition to git-annex to support this
+
+When locally-cached files are invalidated by `git-annex sync`,
+git-annex could notify the user, and give them the option to
+`git-annex get` the invalidated files.  Bonus points if the mechanism
+allows this to be done at any point in the future, not just when
+running `git-annex sync`.  The idea is that git-annex could track
+which files, previously cached locally, have been invalidated
+*unintentionally* by syncs, and treat them differently from files,
+previously cached locally, that have been *intentionally* dropped
+using `git-annex drop` or `git-annex move`.
+
+## More generally
+
+The ability to specify a collection of files to always cache locally
+(something like a numcopies.here=1), which is robust to renames, would
+work.  The "robust to renames" part seems tricky in git: whereas svn
+attaches properties to files, and so properties are propagated by `svn
+mv`, I believe git attributes are only specified by patterns in
+.gitattributes files.
+
+## Related questions / possible approaches
+
+Other forum posts mention [[`git
+subtree`|forum/git-subtree_support__63__/]] and [[sparse git
+checkouts|forum/sparse_git_checkouts_with_annex/]], but I'm not
+familiar with these features and from reading those questions it's
+unclear if those approaches will work for me.  Does anyone more
+familiar see how to adapt one of those features to my use case?

From 759b55a674a012c9c1c553e6b401ff1be14ef1b9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 4 Jun 2012 09:21:36 +0000
Subject: [PATCH 3576/8313] nothing too crazy, but it's the usual distro
 specific problems or architecture specific problems

---
 ...ave_the_32bit_version_installed__41__.mdwn | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn

diff --git a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn
new file mode 100644
index 0000000000..0414823f8b
--- /dev/null
+++ b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn
@@ -0,0 +1,48 @@
+I've just come across a subtle build issue (as haskell-platform just
+got updated, I thought I might give it a try) The scenario is
+
+* OSX 10.7 (everything is up to date with xcode etc... the usual)
+* The 32bit version of Haskell Platform 2012.2
+
+The issue is when libdiskfree.c is compiled and linked to git-annex, 
+OSX defaults to a 64bit binary, thus...
+
+        Linking git-annex ...
+        ld: warning: ignoring file Utility/libdiskfree.o, file was built for unsupported file format which is not the architecture being linked (i386)
+        Undefined symbols for architecture i386:
+          "_diskfree", referenced from:
+              _UtilityziDiskFree_zdwa_info in DiskFree.o
+        ld: symbol(s) not found for architecture i386
+        collect2: ld returned 1 exit status
+        make: *** [git-annex] Error 1
+
+You can either compile up the c library in a 32bit mode if you have the 32bit 
+version of Haskell Platform installed as in the following example
+
+        laplace:git-annex jtang$ cc  -m32  -c -o Utility/libdiskfree.o Utility/libdiskfree.c
+        Utility/libdiskfree.c: In function ‘diskfree’:
+        Utility/libdiskfree.c:61: warning: ‘statfs64’ is deprecated (declared at /usr/include/sys/mount.h:379)
+        laplace:git-annex jtang$ make
+        ghc -O2 -Wall -ignore-package monads-fd -outputdir tmp -IUtility -DWITH_S3 --make git-annex Utility/libdiskfree.o
+
+        Utility/Touch.hs:1:12:
+            Warning: -#include and INCLUDE pragmas are deprecated: They no longer have any effect
+
+        Utility/Touch.hs:2:12:
+            Warning: -#include and INCLUDE pragmas are deprecated: They no longer have any effect
+
+        Utility/Touch.hs:3:12:
+            Warning: -#include and INCLUDE pragmas are deprecated: They no longer have any effect
+
+        Utility/Touch.hs:4:12:
+            Warning: -#include and INCLUDE pragmas are deprecated: They no longer have any effect
+        Linking git-annex ...
+
+Or else just install the 64bit haskell platform. I'm not too sure where
+you would but the intelligence to detect 32 or 64 outputs from the
+different compilers. I suspect checking what ghc outputs then putting in
+the appropriate -m32 or -m64 for the c compiler is the right thing to do. 
+Or just telling users to use the 64bit version of the haskell platform?
+It may also be possible to get osx's c compiler to output a universal binary 
+to give you everything, but that be going down the _being too platform 
+specific route_.

From 23dbff4b4381628fab2827e693e8234243a63511 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 13:22:56 -0400
Subject: [PATCH 3577/8313] add events for symlink creation and directory
 removal

Improved the inotify code, so it will also notice directory removal
and symlink creation.

In the watch code, optimised away a stat of a file that's being added,
that's done by Command.Add.start. This is the reason symlink creation is
handled separately from file creation, since during initial tree walk
at startup, a stat was already done, and can be reused.
---
 Command/Watch.hs   |  22 +++++++--
 Utility/Inotify.hs | 111 +++++++++++++++++++++++++++------------------
 2 files changed, 86 insertions(+), 47 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index e0fa97ac57..a7553a677a 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -29,12 +29,15 @@ start = notBareRepo $ do
 	showAction "scanning"
 	state <- Annex.getState id
 	next $ next $ liftIO $ withINotify $ \i -> do
-		watchDir i notgit (Just $ run state onAdd) Nothing "."
+		let hook a = Just $ run state a
+		watchDir i "." (not . gitdir)
+			(hook onAdd) (hook onAddSymlink)
+			(hook onDel) (hook onDelDir)
 		putStrLn "(started)"
 		waitForTermination
 		return True
 	where
-		notgit dir = takeFileName dir /= ".git"
+		gitdir dir = takeFileName dir /= ".git"
 
 {- Inotify events are run in separate threads, and so each is a
  - self-contained Annex monad. Exceptions by the handlers are ignored,
@@ -51,5 +54,16 @@ run startstate a f = do
 			_ <- shutdown True
 			return ()
 
-onAdd :: FilePath -> Annex Bool
-onAdd file = doCommand $ Add.start file
+onAdd :: FilePath -> Annex ()
+onAdd file = void $ doCommand $ do
+	showStart "add" file
+	next $ Add.perform file
+
+onAddSymlink :: FilePath -> Annex ()
+onAddSymlink link = liftIO $ print $ "add symlink " ++ link
+
+onDel :: FilePath -> Annex ()
+onDel file = liftIO $ print $ "del " ++ file
+
+onDelDir :: FilePath -> Annex ()
+onDelDir dir = liftIO $ print $ "del dir " ++ dir
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index d41e997d61..dc4c352bf4 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -1,3 +1,10 @@
+{- higher-level inotify interface
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
 {-# LANGUAGE CPP #-}
 
 module Utility.Inotify where
@@ -9,19 +16,12 @@ import System.Posix.Terminal
 import Control.Concurrent.MVar
 import System.Posix.Signals
 
-demo :: IO ()
-demo = withINotify $ \i -> do
-	watchDir i (const True) (Just add) (Just del) "/home/joey/tmp/me"
-	putStrLn "started"
-	waitForTermination
-	where
-		add file = putStrLn $ "add " ++ file
-		del file = putStrLn $ "del " ++ file
+type Hook = Maybe (FilePath -> IO ())
 
 {- Watches for changes to files in a directory, and all its subdirectories
- - that match a test, using inotify. This function returns after its initial
- - setup is complete, leaving a thread running. Then callbacks are made for
- - adding and deleting files.
+ - that are not ignored, using inotify. This function returns after
+ - its initial scan is complete, leaving a thread running. Callbacks are
+ - made for different events.
  -
  - Inotify is weak at recursive directory watching; the whole directory
  - tree must be walked and watches set explicitly for each subdirectory.
@@ -37,51 +37,76 @@ demo = withINotify $ \i -> do
  - Note: Due to the race amelioration, multiple add events may occur
  - for the same file.
  - 
- - Note: Moving a file may involve deleting it from its old location and
- - adding it to the new location. 
+ - Note: Moving a file will cause events deleting it from its old location
+ - and adding it to the new location. 
  - 
  - Note: Modification of files is not detected, and it's assumed that when
- - a file that was open for write is closed, it's done being written
+ - a file that was open for write is closed, it's finished being written
  - to, and can be added.
  -
  - Note: inotify has a limit to the number of watches allowed,
  - /proc/sys/fs/inotify/max_user_watches (default 8192).
- - So This will fail if there are too many subdirectories.
+ - So this will fail if there are too many subdirectories.
  -}
-watchDir :: INotify -> (FilePath -> Bool) -> Maybe (FilePath -> IO ()) -> Maybe (FilePath -> IO ()) -> FilePath -> IO ()
-watchDir i test add del dir = watchDir' False i test add del dir
-watchDir' :: Bool -> INotify -> (FilePath -> Bool) -> Maybe (FilePath -> IO ()) -> Maybe (FilePath -> IO ()) -> FilePath -> IO ()
-watchDir' scan i test add del dir = do
-	if test dir
-		then void $ do
-			_ <- addWatch i watchevents dir go
-			mapM walk =<< dirContents dir
-		else noop
+watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook -> Hook -> IO ()
+watchDir i dir ignored add addsymlink del deldir
+	| ignored dir = noop
+	| otherwise = void $ do
+		_ <- addWatch i watchevents dir go
+		mapM walk =<< filter (not . dirCruft) <$>
+			getDirectoryContents dir
 	where
-		watchevents
-			| isJust add && isJust del =
-				[Create, MoveIn, MoveOut, Delete, CloseWrite]
-			| isJust add = [Create, MoveIn, CloseWrite]
-			| isJust del = [Create, MoveOut, Delete]
-			| otherwise = [Create]
+		recurse d = watchDir i d ignored add addsymlink del deldir
 
-		recurse = watchDir' scan i test add del
-		walk f = ifM (catchBoolIO $ Files.isDirectory <$> getFileStatus f)
-			( recurse f
-			, when (scan && isJust add) $ fromJust add f
-			)
+		-- Select only inotify events required by the enabled
+		-- hooks, but always include Create so new directories can
+		-- be walked.
+		watchevents = Create : addevents ++ delevents
+		addevents
+			| isJust add || isJust addsymlink = [MoveIn, CloseWrite]
+			| otherwise = []
+		delevents
+			| isJust del || isJust deldir = [MoveOut, Delete]
+			| otherwise = []
 
-		go (Created { isDirectory = False }) = noop
-		go (Created { filePath = subdir }) = Just recurse <@> subdir
-		go (Closed { maybeFilePath = Just f }) = add <@> f
-		go (MovedIn { isDirectory = False, filePath = f }) = add <@> f
-		go (MovedOut { isDirectory = False, filePath = f }) = del <@> f
-		go (Deleted { isDirectory = False, filePath = f }) = del <@> f
+		walk f = do
+			let fullf = indir f
+			r <- catchMaybeIO $ getSymbolicLinkStatus fullf
+			case r of
+				Nothing -> return ()
+				Just s
+					| Files.isDirectory s -> recurse fullf
+					| Files.isSymbolicLink s -> addsymlink <@> f
+					| Files.isRegularFile s -> add <@> f
+					| otherwise -> return ()
+
+		-- Ignore creation events for regular files, which won't be
+		-- done being written when initially created, but handle for
+		-- directories and symlinks.
+		go (Created { isDirectory = True, filePath = subdir }) = recurse $ indir subdir
+		go (Created { isDirectory = False, filePath = f })
+			| isJust addsymlink =
+				ifM (catchBoolIO $ Files.isSymbolicLink <$> getSymbolicLinkStatus (indir f))
+					( addsymlink <@> f
+					, noop
+					)
+			| otherwise = noop
+		-- Closing a file is assumed to mean it's done being written.
+		go (Closed { isDirectory = False, maybeFilePath = Just f }) = add <@> f
+		-- When a file or directory is moved in, walk it to add new
+		-- stuff.
+		go (MovedIn { filePath = f }) = walk f
+		go (MovedOut { isDirectory = True, filePath = d }) = deldir <@> d
+		go (MovedOut { filePath = f }) = del <@> f
+		go (Deleted { isDirectory = True, filePath = d }) = deldir <@> d
+		go (Deleted { filePath = f }) = del <@> f
 		go _ = noop
-		
-		Just a <@> f = a $ dir  f
+
+		Just a <@> f = a $ indir f
 		Nothing <@> _ = noop
 
+		indir f = dir  f
+
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()
 waitForTermination = do

From bc99cf6a5ad73ebc5036e46e9ef89a4cf806ac56 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
 
Date: Mon, 4 Jun 2012 17:28:38 +0000
Subject: [PATCH 3578/8313]

---
 doc/forum/Debugging_Git_Annex.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/forum/Debugging_Git_Annex.mdwn

diff --git a/doc/forum/Debugging_Git_Annex.mdwn b/doc/forum/Debugging_Git_Annex.mdwn
new file mode 100644
index 0000000000..fc4d829a2a
--- /dev/null
+++ b/doc/forum/Debugging_Git_Annex.mdwn
@@ -0,0 +1,4 @@
+Hi, 
+May I know, how can I debug git-annex code.
+I am new to Haskell Platform, I would like to know which IDE can be used to debug haskell code.
+Thank You.

From 7053f5f94730cc171f677b1c267d479ba046b8b5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 13:30:30 -0400
Subject: [PATCH 3579/8313] handle directory deletion

When a directory is deleted, or moved away, git rm -r it to stage
the deletion.
---
 Command/Watch.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index a7553a677a..abbc98ad57 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -14,6 +14,7 @@ import CmdLine
 import Utility.Inotify
 import Control.Exception as E
 import qualified Command.Add as Add
+import qualified Git.Command
 
 import System.INotify
 
@@ -66,4 +67,4 @@ onDel :: FilePath -> Annex ()
 onDel file = liftIO $ print $ "del " ++ file
 
 onDelDir :: FilePath -> Annex ()
-onDelDir dir = liftIO $ print $ "del dir " ++ dir
+onDelDir dir = inRepo $ Git.Command.run "rm" [Params "--quiet -r", File dir]

From 59ce18d7572a088408ec3477999612d9f2b3b043 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 13:33:56 -0400
Subject: [PATCH 3580/8313] add hinotify dependencies

---
 debian/control   | 1 +
 doc/install.mdwn | 1 +
 git-annex.cabal  | 3 ++-
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 2510e2b336..bfb0017bc8 100644
--- a/debian/control
+++ b/debian/control
@@ -20,6 +20,7 @@ Build-Depends:
 	libghc-ifelse-dev,
 	libghc-bloomfilter-dev,
 	libghc-edit-distance-dev,
+	libghc-hinotify-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index fe0522aa05..ec3a7b013c 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -41,6 +41,7 @@ To build and use git-annex, you will need:
   * [IfElse](http://hackage.haskell.org/package/IfElse)
   * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
   * [edit-distance](http://hackage.haskell.org/package/edit-distance)
+  * [hinotify](http://hackage.haskell.org/package/hinotify)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index e12cbb1777..e5ad27d9d9 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -35,7 +35,8 @@ Executable git-annex
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
+   hinotify
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP

From fa9d479fd1102efeebaafca66d7747bf07604ecb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 14:31:06 -0400
Subject: [PATCH 3581/8313] add explict test that a closed file even is on a
 regular file

There are two reasons for this test. First, there could be a fifo or other
non-regular file that was closed.

Second, this test avoids ugliness when a subdirectory is moved out of the
top of the watch directory to elsewhere, and a file added to it.
Since the subdirectory has moved, the file won't be present under the
old location, and nothing will be done.

I cannot find a way to stop watching such directories, at least not without
a lot of pain. The inotify interface in Haskell doesn't make it easy to
stop watching a given subdirectory when it's moved out; it would require
keeping a map of all watch handles that is shared between threads.
This workaround avoids the problem in most cases; the only remaining case
being deletion of a file from a moved subdirectory.
---
 Utility/Inotify.hs | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index dc4c352bf4..c5caf0655a 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -86,13 +86,13 @@ watchDir i dir ignored add addsymlink del deldir
 		go (Created { isDirectory = True, filePath = subdir }) = recurse $ indir subdir
 		go (Created { isDirectory = False, filePath = f })
 			| isJust addsymlink =
-				ifM (catchBoolIO $ Files.isSymbolicLink <$> getSymbolicLinkStatus (indir f))
-					( addsymlink <@> f
-					, noop
-					)
+				whenM (filetype Files.isSymbolicLink f) $
+					addsymlink <@> f
 			| otherwise = noop
 		-- Closing a file is assumed to mean it's done being written.
-		go (Closed { isDirectory = False, maybeFilePath = Just f }) = add <@> f
+		go (Closed { isDirectory = False, maybeFilePath = Just f }) =
+			whenM (filetype Files.isRegularFile f) $
+				add <@> f
 		-- When a file or directory is moved in, walk it to add new
 		-- stuff.
 		go (MovedIn { filePath = f }) = walk f
@@ -106,6 +106,7 @@ watchDir i dir ignored add addsymlink del deldir
 		Nothing <@> _ = noop
 
 		indir f = dir  f
+		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
 
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()

From 47f8f43715f355e141fc2d57ce7f72bdda75b5d6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 14:46:09 -0400
Subject: [PATCH 3582/8313] workaround other part of moved directory problem

This fixes the scenario where:

* directory foo is moved away (and still watched)
* a new directory foo is made
* file (or directory) foo/bar is created
* the old directory's file (or directory) "bar" is deleted

We don't want a deletion event for foo/bar in this case.
---
 Utility/Inotify.hs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index c5caf0655a..320f45525e 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -98,16 +98,25 @@ watchDir i dir ignored add addsymlink del deldir
 		go (MovedIn { filePath = f }) = walk f
 		go (MovedOut { isDirectory = True, filePath = d }) = deldir <@> d
 		go (MovedOut { filePath = f }) = del <@> f
-		go (Deleted { isDirectory = True, filePath = d }) = deldir <@> d
-		go (Deleted { filePath = f }) = del <@> f
+		go (Deleted { isDirectory = True, filePath = d }) =
+			notexist d $ deldir <@> d
+		go (Deleted { filePath = f }) =
+			notexist f $ del <@> f
 		go _ = noop
 
 		Just a <@> f = a $ indir f
 		Nothing <@> _ = noop
 
 		indir f = dir  f
+
 		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
 
+		-- Check that a file or directory does not exist.
+		-- This is used when there could be a spurious deletion
+		-- event for an item in a directory that has been moved away
+		-- but is still being watched.
+		notexist f = unlessM (filetype (const True) f)
+
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()
 waitForTermination = do

From 677ad7468700ae77cbf66a18d047de011b2d173d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 15:10:43 -0400
Subject: [PATCH 3583/8313] add handling of symlink addition events

And just like that, annexed files can be moved and copies around within
the tree, and are automatically fixed to point to the content, and staged
in git. Huzzah!

Delete still remains TODO, with its troublesome race during add..
---
 Command/Watch.hs | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index abbc98ad57..0f60bd6361 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -7,15 +7,18 @@
 
 module Command.Watch where
 
+import CmdLine
 import Common.Annex
 import Command
-import qualified Annex
-import CmdLine
 import Utility.Inotify
-import Control.Exception as E
+import qualified Annex
 import qualified Command.Add as Add
 import qualified Git.Command
+import qualified Annex.Queue
+import qualified Backend
+import Annex.Content
 
+import Control.Exception as E
 import System.INotify
 
 def :: [Command]
@@ -55,16 +58,41 @@ run startstate a f = do
 			_ <- shutdown True
 			return ()
 
+{- Adding a file is the same as git-annex add.
+ - The git queue is immediately flushed, so the file is added to git
+ - now, rather than later (when it may have been already moved or deleted!) -}
 onAdd :: FilePath -> Annex ()
-onAdd file = void $ doCommand $ do
-	showStart "add" file
-	next $ Add.perform file
+onAdd file = do
+	void $ doCommand $ do
+		showStart "add" file
+		next $ Add.perform file
+	Annex.Queue.flush
 
+{- A symlink might be an arbitrary symlink, which is just added.
+ - Or, if it is a git-annex symlink, ensure it points to the content
+ - before adding it.
+ -}
 onAddSymlink :: FilePath -> Annex ()
-onAddSymlink link = liftIO $ print $ "add symlink " ++ link
+onAddSymlink file = go =<< Backend.lookupFile file
+	where
+		go Nothing = addlink
+		go (Just (key, _)) = do
+			link <- calcGitLink file key
+			ifM ((==) link <$> liftIO (readSymbolicLink file))
+				( addlink
+				, do
+					liftIO $ removeFile file
+					liftIO $ createSymbolicLink link file
+					addlink
+				)
+		addlink = inRepo $ Git.Command.run "add"
+			[Params "--force --", File file]
 
 onDel :: FilePath -> Annex ()
 onDel file = liftIO $ print $ "del " ++ file
 
+{- A directory has been deleted, so tell git to remove anything that
+   was inside it from its cache. -}
 onDelDir :: FilePath -> Annex ()
-onDelDir dir = inRepo $ Git.Command.run "rm" [Params "--quiet -r", File dir]
+onDelDir dir = inRepo $ Git.Command.run "rm"
+	[Params "--quiet -r --cached --", File dir]

From 9791d1dfe7083d1564d64a35cb2a37350dcc184f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 15:14:45 -0400
Subject: [PATCH 3584/8313] update; worked on watch branch today

---
 doc/design/assistant/inotify.mdwn | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 6bb810a755..fd8d56e057 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -8,10 +8,15 @@ useful, it needs to:
 - notice deleted files and stage the deletion
   (tricky; there's a race with add..)
 - notice renamed files, auto-fix the symlink, and stage the new file location
+  **done**
+- handle cases where directories are moved outside the repo, and stop
+  watching them **done**
+- when a whole directory is deleted, stage removal of its
+  contents in the index **done**
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in)
 - tunable delays before adding new files, etc
-- honor .gitignore, not adding files it excludesa
+- honor .gitignore, not adding files it excludes
 
 Also to do:
 

From 9b40cd7461cc7b384f21a428a6b99f33301d67dd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 15:23:56 -0400
Subject: [PATCH 3585/8313] update

---
 doc/design/assistant/inotify.mdwn | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index fd8d56e057..166e5b7cd4 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -1,22 +1,33 @@
 Finish "git annex watch" command, which runs, in the background, watching via
 inotify for changes, and automatically annexing new files, etc.
 
-There is a `watch` branch in git that adds such a command, although currently
-it only handles adding new files, and nothing else. To make this really
-useful, it needs to:
+There is a `watch` branch in git that adds such a command. To make this
+really useful, it needs to:
 
-- notice deleted files and stage the deletion
-  (tricky; there's a race with add..)
+- on startup, add any files that have appeared since last run **done**
+- on startup, fix the symlinks for any renamed links **done**
+- on startup, stage any files that have been deleted since last run
+  (seems to require a `git commit -a` on startup, or at least a
+  `git add --update`, which will notice deleted files)
+- notice new files, and git annex add **done**
 - notice renamed files, auto-fix the symlink, and stage the new file location
   **done**
 - handle cases where directories are moved outside the repo, and stop
   watching them **done**
-- when a whole directory is deleted, stage removal of its
-  contents in the index **done**
+- when a whole directory is deleted or moved, stage removal of its
+  contents from the index **done**
+- notice deleted files and stage the deletion
+  (tricky; there's a race with add since it replaces the file with a symlink..)
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in)
 - tunable delays before adding new files, etc
-- honor .gitignore, not adding files it excludes
+- don't annex `.gitignore` and `.gitattributes` files, but do auto-stage
+  changes to them
+- configurable option to only annex files meeting certian size or
+  filename criteria
+- honor .gitignore, not adding files it excludes (difficult, probably
+  needs my own .gitignore parser to avoid excessive running of git commands
+  to check for ignored files)
 
 Also to do:
 

From 2b269f32a10c63e089151d66020c7295b991393a Mon Sep 17 00:00:00 2001
From: "http://ciffer.net/~svend/" 
Date: Mon, 4 Jun 2012 19:40:08 +0000
Subject: [PATCH 3586/8313] Added a comment

---
 .../comment_2_e65cb478c57e70031b986ace2aafbfc0._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment

diff --git a/doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment b/doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment
new file mode 100644
index 0000000000..6c07b0e63b
--- /dev/null
+++ b/doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://ciffer.net/~svend/"
+ subject="comment 2"
+ date="2012-06-04T19:40:06Z"
+ content="""
+I would find it useful if the watch command could 'git add' new files (instead of 'git annex add') for certain repositories.
+"""]]

From 0accc932cb88a79829cf0cfb3f27b16b46e33140 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 15:40:11 -0400
Subject: [PATCH 3587/8313] update

---
 doc/design/assistant.mdwn                     |  7 +++
 doc/design/assistant/blog.mdwn                |  1 +
 doc/design/assistant/blog/day_1__inotify.mdwn | 57 +++++++++++++++++++
 doc/design/assistant/inotify.mdwn             |  2 +
 4 files changed, 67 insertions(+)
 create mode 100644 doc/design/assistant/blog.mdwn
 create mode 100644 doc/design/assistant/blog/day_1__inotify.mdwn

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 7a720a5e08..69bf1da341 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -18,3 +18,10 @@ Feel free to chip in with comments! --[[Joey]]
 * [[desymlink]]
 * [[deltas]]
 * In my overfunded nighmares: [[Windows]]
+
+## blog
+
+I'll be blogging about my progress here on a semi-daily basis. Follow
+along!
+
+[[!inline pages="page(design/assistant/blog/*)" show=30]]
diff --git a/doc/design/assistant/blog.mdwn b/doc/design/assistant/blog.mdwn
new file mode 100644
index 0000000000..f51da3db41
--- /dev/null
+++ b/doc/design/assistant/blog.mdwn
@@ -0,0 +1 @@
+See [[assistant]].
diff --git a/doc/design/assistant/blog/day_1__inotify.mdwn b/doc/design/assistant/blog/day_1__inotify.mdwn
new file mode 100644
index 0000000000..9935698e06
--- /dev/null
+++ b/doc/design/assistant/blog/day_1__inotify.mdwn
@@ -0,0 +1,57 @@
+First day of [Kickstarter funded work](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/)! 
+
+Worked on [[inotify]] today. The `watch` branch in git now does a pretty
+good job of following changes made to the directory, annexing files
+as they're added and staging other changes into git. Here's a quick
+transcript of it in action:
+
+	joey@gnu:~/tmp>mkdir demo
+	joey@gnu:~/tmp>cd demo
+	joey@gnu:~/tmp/demo>git init
+	Initialized empty Git repository in /home/joey/tmp/demo/.git/
+	joey@gnu:~/tmp/demo>git annex init demo
+	init demo ok
+	(Recording state in git...)
+	joey@gnu:~/tmp/demo>git annex watch &
+	[1] 3284
+	watch . (scanning...) (started)
+	joey@gnu:~/tmp/demo>dd if=/dev/urandom of=bigfile bs=1M count=2
+	add ./bigfile 2+0 records in
+	2+0 records out
+	2097152 bytes (2.1 MB) copied, 0.835976 s, 2.5 MB/s
+	(checksum...) ok
+	(Recording state in git...)
+	joey@gnu:~/tmp/demo>ls -la bigfile
+	lrwxrwxrwx 1 joey joey 188 Jun  4 15:36 bigfile -> .git/annex/objects/Wx/KQ/SHA256-s2097152--e5ced5836a3f9be782e6da14446794a1d22d9694f5c85f3ad7220b035a4b82ee/SHA256-s2097152--e5ced5836a3f9be782e6da14446794a1d22d9694f5c85f3ad7220b035a4b82ee
+	joey@gnu:~/tmp/demo>git status -s
+	A  bigfile
+	joey@gnu:~/tmp/demo>mkdir foo
+	joey@gnu:~/tmp/demo>mv bigfile foo
+	"del ./bigfile"
+	joey@gnu:~/tmp/demo>git status -s
+	AD bigfile
+	A  foo/bigfile
+
+Due to Linux's inotify interface, this is suely some of the most subtle,
+race-heavy code that I'll need to deal with while developing the git annex
+assistant. But I can't start wading, need to jump off the deep end to make
+progress!
+
+The hardest problem today involved the case where a directory is moved
+outside of the tree that's being watched. Inotify will still send events
+for such directories, but it doesn't make sense to continue to handle them.
+
+Ideally I'd stop inotify watching such directories, but a lot of state
+would need to be maintained to know which inotify handle to stop watching.
+(Seems like Haskell's inotify API makes this harder than it needs to be...)
+
+Instead, I put in a hack that will make it detect inotify events from
+directories moved away, and ignore them. This is probably acceptable,
+since this is an unusual edge case.
+
+----
+
+The notable omission in the inotify code, which I'll work on next, is
+staging deleting of files. This is tricky because adding a file to the
+annex happens to cause a deletion event. I need to make sure there are no
+races where that deletion event causes data loss.
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 166e5b7cd4..e6335e0451 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -28,6 +28,8 @@ really useful, it needs to:
 - honor .gitignore, not adding files it excludes (difficult, probably
   needs my own .gitignore parser to avoid excessive running of git commands
   to check for ignored files)
+- Possibly, when a directory is moved out of the annex location,
+  unannex its contents.
 
 Also to do:
 

From e33f6bdfde536d0393b0857bca044150d756bc35 Mon Sep 17 00:00:00 2001
From: "http://ciffer.net/~svend/" 
Date: Mon, 4 Jun 2012 19:40:57 +0000
Subject: [PATCH 3588/8313] removed

---
 .../comment_2_e65cb478c57e70031b986ace2aafbfc0._comment    | 7 -------
 1 file changed, 7 deletions(-)
 delete mode 100644 doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment

diff --git a/doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment b/doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment
deleted file mode 100644
index 6c07b0e63b..0000000000
--- a/doc/design/assistant/comment_2_e65cb478c57e70031b986ace2aafbfc0._comment
+++ /dev/null
@@ -1,7 +0,0 @@
-[[!comment format=mdwn
- username="http://ciffer.net/~svend/"
- subject="comment 2"
- date="2012-06-04T19:40:06Z"
- content="""
-I would find it useful if the watch command could 'git add' new files (instead of 'git annex add') for certain repositories.
-"""]]

From dfb3182877e0072e8fffb2c1df1a01ad343cfdb3 Mon Sep 17 00:00:00 2001
From: "http://ciffer.net/~svend/" 
Date: Mon, 4 Jun 2012 19:42:08 +0000
Subject: [PATCH 3589/8313] Added a comment

---
 .../comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment

diff --git a/doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment b/doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment
new file mode 100644
index 0000000000..d042806a17
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_1_3d3ff74447452d65c10ccc3dbfc323cd._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://ciffer.net/~svend/"
+ subject="comment 1"
+ date="2012-06-04T19:42:07Z"
+ content="""
+I would find it useful if the watch command could 'git add' new files (instead of 'git annex add') for certain repositories.
+"""]]

From d08274a941ef92b7644293e050744e6c7af55fc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 15:42:26 -0400
Subject: [PATCH 3590/8313] move

---
 doc/design/assistant.mdwn      | 6 ++----
 doc/design/assistant/blog.mdwn | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn
index 69bf1da341..1069cd8c6d 100644
--- a/doc/design/assistant.mdwn
+++ b/doc/design/assistant.mdwn
@@ -21,7 +21,5 @@ Feel free to chip in with comments! --[[Joey]]
 
 ## blog
 
-I'll be blogging about my progress here on a semi-daily basis. Follow
-along!
-
-[[!inline pages="page(design/assistant/blog/*)" show=30]]
+I'll be blogging about my progress in the [[blog]] on a semi-daily basis.
+Follow along!
diff --git a/doc/design/assistant/blog.mdwn b/doc/design/assistant/blog.mdwn
index f51da3db41..f099e48be0 100644
--- a/doc/design/assistant/blog.mdwn
+++ b/doc/design/assistant/blog.mdwn
@@ -1 +1 @@
-See [[assistant]].
+[[!inline pages="page(design/assistant/blog/*)" show=30]]

From ecac13a11391443c2ac61f9b036be9f0da66ddcd Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Mon, 4 Jun 2012 19:45:00 +0000
Subject: [PATCH 3591/8313] Added a comment

---
 .../comment_2_6d3552414fdcc2ed3244567e6c67989d._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment

diff --git a/doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment b/doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment
new file mode 100644
index 0000000000..8056eec168
--- /dev/null
+++ b/doc/design/assistant/comment_2_6d3552414fdcc2ed3244567e6c67989d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-06-04T19:45:00Z"
+ content="""
+Jimmy, I hope to make it as easy as possible to install. I've been focusing on getting it directly into popular Linux distributions, rather than shipping my own binary. The OSX binary is static, and while I lack a OSX machine, I would like to get it easier to distribute to OSX users.
+"""]]

From 02dce20325a28973240733036c5ef00ead14ac04 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Mon, 4 Jun 2012 19:46:04 +0000
Subject: [PATCH 3592/8313] Added a comment

---
 .../comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment

diff --git a/doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment b/doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment
new file mode 100644
index 0000000000..13ee1523c3
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_2_a3c0fa6d97397c508b4b8aafdcee8f6f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 2"
+ date="2012-06-04T19:46:03Z"
+ content="""
+I think it's already on the list: \"configurable option to only annex files meeting certian size or filename criteria\" -- files not meeting those criteria would just be git added.
+"""]]

From 39fb9db0433d66efff2085d3447bc465d8d29dc8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 15:47:19 -0400
Subject: [PATCH 3593/8313] add

---
 doc/design/assistant/inotify.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index e6335e0451..fe4c26d412 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -30,6 +30,8 @@ really useful, it needs to:
   to check for ignored files)
 - Possibly, when a directory is moved out of the annex location,
   unannex its contents.
+- Gracefully handle when the default limit of 8192 inotified directories
+  is exceeded. This can be tuned by root, so help the user fix it.
 
 Also to do:
 

From db2e6c8239d8f0e1c1593f95617b11db761dd27c Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Mon, 4 Jun 2012 19:49:46 +0000
Subject: [PATCH 3594/8313] Added a comment: This is not an easy question to
 answer...

---
 .../comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment

diff --git a/doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment b/doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment
new file mode 100644
index 0000000000..84781a00a4
--- /dev/null
+++ b/doc/forum/Debugging_Git_Annex/comment_1_ce63b2ee641a2338f1ad5ded9e6f09a8._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="This is not an easy question to answer..."
+ date="2012-06-04T19:49:46Z"
+ content="""
+Do you have a bug in git-annex that you need fixed, or are you just curious?
+"""]]

From 04ddceece0db7aa0c9f1c2f45d1675133fdba6b1 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Mon, 4 Jun 2012 19:56:05 +0000
Subject: [PATCH 3595/8313] Added a comment

---
 .../comment_1_dab1099ee56541c194de319c593f1268._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment

diff --git a/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment
new file mode 100644
index 0000000000..fa561ef04c
--- /dev/null
+++ b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_1_dab1099ee56541c194de319c593f1268._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-06-04T19:56:05Z"
+ content="""
+Personally, I deal with this problem by having a directory, or directories where I put files that I want to have on my partial checkout laptop, and run `git annex get` in that directory. 
+
+It's not a perfect solution, but I don't know that a perfect solution exists.
+"""]]

From fc66d642054175ef93359eff2ade6215374b66ee Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Mon, 4 Jun 2012 20:02:23 +0000
Subject: [PATCH 3596/8313] Added a comment

---
 .../comment_1_6208e70a21a048d5423926d16e32d421._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_1_6208e70a21a048d5423926d16e32d421._comment

diff --git a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_1_6208e70a21a048d5423926d16e32d421._comment b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_1_6208e70a21a048d5423926d16e32d421._comment
new file mode 100644
index 0000000000..11d135251d
--- /dev/null
+++ b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_1_6208e70a21a048d5423926d16e32d421._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-06-04T20:02:23Z"
+ content="""
+Seems you built it using `make` .. could you try instead building with cabal, ie run `cabal install git-annex` or `cabal build` in the source tree. I think cabal will probably do the right thing.
+
+I could fix the Makefile, I suppose. What does this say: `ghc -e 'print System.Info.arch'
+"""]]

From e43e5134142344c4870b62af0efa6321d7783dcb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 16:16:50 -0400
Subject: [PATCH 3597/8313] typo

---
 doc/design/assistant/blog/day_1__inotify.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_1__inotify.mdwn b/doc/design/assistant/blog/day_1__inotify.mdwn
index 9935698e06..ca27a45861 100644
--- a/doc/design/assistant/blog/day_1__inotify.mdwn
+++ b/doc/design/assistant/blog/day_1__inotify.mdwn
@@ -32,7 +32,7 @@ transcript of it in action:
 	AD bigfile
 	A  foo/bigfile
 
-Due to Linux's inotify interface, this is suely some of the most subtle,
+Due to Linux's inotify interface, this is surely some of the most subtle,
 race-heavy code that I'll need to deal with while developing the git annex
 assistant. But I can't start wading, need to jump off the deep end to make
 progress!

From 88107724d7dcf564322660a0cb77e7d1e5ffee70 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlu7K3h7Ry1uDAU_ERYGuqt0LoGNJqGuRo"
 
Date: Mon, 4 Jun 2012 21:01:52 +0000
Subject: [PATCH 3598/8313] Added a comment

---
 .../comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment

diff --git a/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment
new file mode 100644
index 0000000000..2cd1001ef3
--- /dev/null
+++ b/doc/forum/Automatic___96__git_annex_get__96___after_invalidation_of_local_files_due_to_external_modification__63__/comment_2_b5faccf132fb47e3cda778a6600fd9ef._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlu7K3h7Ry1uDAU_ERYGuqt0LoGNJqGuRo"
+ nickname="Nathan"
+ subject="comment 2"
+ date="2012-06-04T21:01:52Z"
+ content="""
+Joey, that sounds reasonable; I'll try it. Thanks!
+"""]]

From 529a3721e1bd396694786a56630bd2d40c368c23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 17:17:05 -0400
Subject: [PATCH 3599/8313] refactor

---
 Utility/Inotify.hs | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 320f45525e..2dcc1ed64c 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -83,8 +83,8 @@ watchDir i dir ignored add addsymlink del deldir
 		-- Ignore creation events for regular files, which won't be
 		-- done being written when initially created, but handle for
 		-- directories and symlinks.
-		go (Created { isDirectory = True, filePath = subdir }) = recurse $ indir subdir
-		go (Created { isDirectory = False, filePath = f })
+		go (Created { isDirectory = isd, filePath = f })
+			| isd = recurse $ indir f
 			| isJust addsymlink =
 				whenM (filetype Files.isSymbolicLink f) $
 					addsymlink <@> f
@@ -96,12 +96,18 @@ watchDir i dir ignored add addsymlink del deldir
 		-- When a file or directory is moved in, walk it to add new
 		-- stuff.
 		go (MovedIn { filePath = f }) = walk f
-		go (MovedOut { isDirectory = True, filePath = d }) = deldir <@> d
-		go (MovedOut { filePath = f }) = del <@> f
-		go (Deleted { isDirectory = True, filePath = d }) =
-			notexist d $ deldir <@> d
-		go (Deleted { filePath = f }) =
-			notexist f $ del <@> f
+		go (MovedOut { isDirectory = isd, filePath = f })
+			| isd = deldir <@> f
+			| otherwise = del <@> f
+		-- Verify that the deleted item really doesn't exist,
+		-- since there can be spurious deletion events for items
+		-- in a directory that has been moved out, but is still
+		-- being watched.
+		go (Deleted { isDirectory = isd, filePath = f })
+			| isd = guarded $ deldir <@> f
+			| otherwise = guarded $ del <@> f
+			where
+				guarded = unlessM (filetype (const True) f)
 		go _ = noop
 
 		Just a <@> f = a $ indir f
@@ -111,12 +117,6 @@ watchDir i dir ignored add addsymlink del deldir
 
 		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
 
-		-- Check that a file or directory does not exist.
-		-- This is used when there could be a spurious deletion
-		-- event for an item in a directory that has been moved away
-		-- but is still being watched.
-		notexist f = unlessM (filetype (const True) f)
-
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()
 waitForTermination = do

From 659e6b13249ec9d8f3c01607b0eb819b8a5690fe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 17:18:54 -0400
Subject: [PATCH 3600/8313] suppress "recording state in git" message during
 add

---
 Command/Watch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 0f60bd6361..b38c04d2c9 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -62,7 +62,7 @@ run startstate a f = do
  - The git queue is immediately flushed, so the file is added to git
  - now, rather than later (when it may have been already moved or deleted!) -}
 onAdd :: FilePath -> Annex ()
-onAdd file = do
+onAdd file = doQuietSideAction $ do
 	void $ doCommand $ do
 		showStart "add" file
 		next $ Add.perform file

From 5b4e5ce7e5141eac7f0bc81033d7d7676ff0008f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 17:32:46 -0400
Subject: [PATCH 3601/8313] deletion

When a new file is annexed, a deletion event occurs when it's moved away
to be replaced by a symlink. Most of the time, there is no problimatic
race, because the same thread runs the add event as the deletion event.
So, once the symlink is in place, the deletion code won't run at all,
due to existing checks that a deleted file is really gone.

But there is a race at startup, as then the inotify thread is running
at the same time as the main thread, which does the initial tree walking
and annexing. It would be possible for the deletion inotify to run
in a perfect race with the addition, and remove the newly added symlink
from the git cache.

To solve this race, added event serialization via a MVar. We putMVar
before running each event, which blocks if an event is already running.
And when an event finishes (or crashes!), we takeMVar to free the lock.

Also, make rm -rf not spew warnings by passing --ignore-unmatch when
deleting directories.
---
 Command/Watch.hs   | 16 ++++++++++------
 Utility/Inotify.hs | 32 ++++++++++++++++++++++----------
 2 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index b38c04d2c9..0467776854 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -44,8 +44,11 @@ start = notBareRepo $ do
 		gitdir dir = takeFileName dir /= ".git"
 
 {- Inotify events are run in separate threads, and so each is a
- - self-contained Annex monad. Exceptions by the handlers are ignored,
- - otherwise a whole watcher thread could be crashed. -}
+ - self-contained Annex monad.
+ -
+ - Exceptions by the handlers are ignored,
+ - otherwise a whole watcher thread could be crashed.
+ -}
 run :: Annex.AnnexState -> (FilePath -> Annex a) -> FilePath -> IO ()
 run startstate a f = do
 	r <- E.try go :: IO (Either E.SomeException ())
@@ -89,10 +92,11 @@ onAddSymlink file = go =<< Backend.lookupFile file
 			[Params "--force --", File file]
 
 onDel :: FilePath -> Annex ()
-onDel file = liftIO $ print $ "del " ++ file
+onDel file = inRepo $ Git.Command.run "rm"
+	[Params "--quiet --cached --", File file]
 
-{- A directory has been deleted, so tell git to remove anything that
-   was inside it from its cache. -}
+{- A directory has been deleted, or moved, so tell git to remove anything
+ - that was inside it from its cache. -}
 onDelDir :: FilePath -> Annex ()
 onDelDir dir = inRepo $ Git.Command.run "rm"
-	[Params "--quiet -r --cached --", File dir]
+	[Params "--quiet -r --cached --ignore-unmatch --", File dir]
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 2dcc1ed64c..ff3de81b1f 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -15,6 +15,7 @@ import qualified System.Posix.Files as Files
 import System.Posix.Terminal
 import Control.Concurrent.MVar
 import System.Posix.Signals
+import Control.Exception as E
 
 type Hook = Maybe (FilePath -> IO ())
 
@@ -51,10 +52,13 @@ type Hook = Maybe (FilePath -> IO ())
 watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook -> Hook -> IO ()
 watchDir i dir ignored add addsymlink del deldir
 	| ignored dir = noop
-	| otherwise = void $ do
-		_ <- addWatch i watchevents dir go
-		mapM walk =<< filter (not . dirCruft) <$>
-			getDirectoryContents dir
+	| otherwise = do
+		mvar <- newEmptyMVar
+		void $ addWatch i watchevents dir $ \event ->
+			serialized mvar (void $ go event)
+		serialized mvar $
+			mapM_ walk =<< filter (not . dirCruft) <$>
+				getDirectoryContents dir
 	where
 		recurse d = watchDir i d ignored add addsymlink del deldir
 
@@ -117,14 +121,22 @@ watchDir i dir ignored add addsymlink del deldir
 
 		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
 
+{- Uses an MVar to serialize an action, so that only one thread at a time
+ - runs it. -}
+serialized :: MVar () -> IO () -> IO ()
+serialized mvar a = void $ do
+	putMVar mvar () -- blocks if action already running
+	_ <- E.try a :: IO (Either E.SomeException ())
+	takeMVar mvar -- allow next action to run
+
 {- Pauses the main thread, letting children run until program termination. -}
 waitForTermination :: IO ()
 waitForTermination = do
-	mv <- newEmptyMVar
-	check softwareTermination mv
+	mvar <- newEmptyMVar
+	check softwareTermination mvar
 	whenM (queryTerminal stdInput) $
-		check keyboardSignal mv
-	takeMVar mv
+		check keyboardSignal mvar
+	takeMVar mvar
 	where
-		check sig mv = void $
-			installHandler sig (CatchOnce $ putMVar mv ()) Nothing
+		check sig mvar = void $
+			installHandler sig (CatchOnce $ putMVar mvar ()) Nothing

From ec98581112070244d5cdd69d4228aeab856ce3eb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 18:14:42 -0400
Subject: [PATCH 3602/8313] notice deleted files on startup

---
 Command/Watch.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 0467776854..4a0ee66404 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -31,6 +31,7 @@ start :: CommandStart
 start = notBareRepo $ do
 	showStart "watch" "."
 	showAction "scanning"
+	inRepo $ Git.Command.run "add" [Param "--update"]
 	state <- Annex.getState id
 	next $ next $ liftIO $ withINotify $ \i -> do
 		let hook a = Just $ run state a

From b51520b0d5cd13ce43421b0d156a8e889ba27900 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 18:12:25 -0400
Subject: [PATCH 3603/8313] update

---
 doc/design/assistant/inotify.mdwn | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index fe4c26d412..82867f1040 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -8,7 +8,7 @@ really useful, it needs to:
 - on startup, fix the symlinks for any renamed links **done**
 - on startup, stage any files that have been deleted since last run
   (seems to require a `git commit -a` on startup, or at least a
-  `git add --update`, which will notice deleted files)
+  `git add --update`, which will notice deleted files) **done**
 - notice new files, and git annex add **done**
 - notice renamed files, auto-fix the symlink, and stage the new file location
   **done**
@@ -18,6 +18,7 @@ really useful, it needs to:
   contents from the index **done**
 - notice deleted files and stage the deletion
   (tricky; there's a race with add since it replaces the file with a symlink..)
+  **done**
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in)
 - tunable delays before adding new files, etc
@@ -32,9 +33,6 @@ really useful, it needs to:
   unannex its contents.
 - Gracefully handle when the default limit of 8192 inotified directories
   is exceeded. This can be tuned by root, so help the user fix it.
-
-Also to do:
-
 - Support OSes other than Linux; it only uses inotify currently.
   OSX and FreeBSD use the same mechanism, and there is a Haskell interface
   for it,

From b1619fbd0b41caddf6caf216c10440b3f62f0a0a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 19:04:55 -0400
Subject: [PATCH 3604/8313] header

---
 doc/design/assistant/blog.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/design/assistant/blog.mdwn b/doc/design/assistant/blog.mdwn
index f099e48be0..ca9ba59193 100644
--- a/doc/design/assistant/blog.mdwn
+++ b/doc/design/assistant/blog.mdwn
@@ -1 +1,5 @@
+The git-annex assistant is being
+[crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/).
+I'll be blogging about my progress here on a semi-daily basis.
+
 [[!inline pages="page(design/assistant/blog/*)" show=30]]

From cbf16f1967c247dfd41bc264fc994b6dae2620f9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 19:43:29 -0400
Subject: [PATCH 3605/8313] refactor

---
 Command/Watch.hs      |  1 +
 Utility/Inotify.hs    | 32 +++++---------------------------
 Utility/ThreadLock.hs | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 27 deletions(-)
 create mode 100644 Utility/ThreadLock.hs

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 4a0ee66404..31f1716696 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -11,6 +11,7 @@ import CmdLine
 import Common.Annex
 import Command
 import Utility.Inotify
+import Utility.ThreadLock
 import qualified Annex
 import qualified Command.Add as Add
 import qualified Git.Command
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index ff3de81b1f..3c69a7ee2e 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -10,12 +10,10 @@
 module Utility.Inotify where
 
 import Common hiding (isDirectory)
+import Utility.ThreadLock
+
 import System.INotify
 import qualified System.Posix.Files as Files
-import System.Posix.Terminal
-import Control.Concurrent.MVar
-import System.Posix.Signals
-import Control.Exception as E
 
 type Hook = Maybe (FilePath -> IO ())
 
@@ -53,10 +51,10 @@ watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook ->
 watchDir i dir ignored add addsymlink del deldir
 	| ignored dir = noop
 	| otherwise = do
-		mvar <- newEmptyMVar
+		lock <- newLock
 		void $ addWatch i watchevents dir $ \event ->
-			serialized mvar (void $ go event)
-		serialized mvar $
+			withLock lock (void $ go event)
+		withLock lock $
 			mapM_ walk =<< filter (not . dirCruft) <$>
 				getDirectoryContents dir
 	where
@@ -120,23 +118,3 @@ watchDir i dir ignored add addsymlink del deldir
 		indir f = dir  f
 
 		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
-
-{- Uses an MVar to serialize an action, so that only one thread at a time
- - runs it. -}
-serialized :: MVar () -> IO () -> IO ()
-serialized mvar a = void $ do
-	putMVar mvar () -- blocks if action already running
-	_ <- E.try a :: IO (Either E.SomeException ())
-	takeMVar mvar -- allow next action to run
-
-{- Pauses the main thread, letting children run until program termination. -}
-waitForTermination :: IO ()
-waitForTermination = do
-	mvar <- newEmptyMVar
-	check softwareTermination mvar
-	whenM (queryTerminal stdInput) $
-		check keyboardSignal mvar
-	takeMVar mvar
-	where
-		check sig mvar = void $
-			installHandler sig (CatchOnce $ putMVar mvar ()) Nothing
diff --git a/Utility/ThreadLock.hs b/Utility/ThreadLock.hs
new file mode 100644
index 0000000000..4285c0ec54
--- /dev/null
+++ b/Utility/ThreadLock.hs
@@ -0,0 +1,35 @@
+{- locking between threads
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.ThreadLock where
+
+import Common
+
+import System.Posix.Terminal
+import Control.Concurrent.MVar
+import System.Posix.Signals
+
+type Lock = MVar ()
+
+newLock :: IO Lock
+newLock = newMVar ()
+
+{- Runs an action with a lock held, so only one thread at a time can run it. -}
+withLock :: Lock -> IO a -> IO a
+withLock lock = withMVar lock . const
+
+{- Pauses the main thread, letting children run until program termination. -}
+waitForTermination :: IO ()
+waitForTermination = do
+	lock <- newEmptyMVar
+	check softwareTermination lock
+	whenM (queryTerminal stdInput) $
+		check keyboardSignal lock
+	takeMVar lock
+	where
+		check sig lock = void $
+			installHandler sig (CatchOnce $ putMVar lock ()) Nothing

From bd7857d903a602bc5f1d5e01f0b936bb5b41b5c6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 20:13:25 -0400
Subject: [PATCH 3606/8313] ignore-unmatch when removing a staged file

When a file is added, and then deleted before the add action runs,
the delete event was unhappy that the file never did get staged.
---
 Command/Watch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 31f1716696..024d2c568a 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -95,7 +95,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 
 onDel :: FilePath -> Annex ()
 onDel file = inRepo $ Git.Command.run "rm"
-	[Params "--quiet --cached --", File file]
+	[Params "--quiet --cached --ignore-unmatch --", File file]
 
 {- A directory has been deleted, or moved, so tell git to remove anything
  - that was inside it from its cache. -}

From 7a6fb8ae4e455ea311213da50cc5e8cd6d5667b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 20:41:22 -0400
Subject: [PATCH 3607/8313] flush the git queue when a new type of action is
 being added to it

This allows the queue to be used in a single process for multiple possibly
conflicting commands, like add and rm, without running them out of order.

This assumes that running the same git subcommand with different parameters
cannot itself conflict.
---
 Annex/Queue.hs |  2 +-
 Git/Queue.hs   | 28 +++++++++++++++++++---------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 24575e9068..d4a2c592e5 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -20,7 +20,7 @@ import Config
 add :: String -> [CommandParam] -> [FilePath] -> Annex ()
 add command params files = do
 	q <- get
-	store $ Git.Queue.add q command params files
+	store =<< inRepo (Git.Queue.add q command params files)
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
diff --git a/Git/Queue.hs b/Git/Queue.hs
index b8055ab445..956e9adb1d 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -1,6 +1,6 @@
 {- git repository command queue
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -33,6 +33,12 @@ data Action = Action
 	, getParams :: [CommandParam]
 	} deriving (Show, Eq, Ord)
 
+{- Compares two actions by subcommand. -}
+(===) :: Action -> Action -> Bool
+a === b = getSubcommand a == getSubcommand b
+(/==) :: Action -> Action -> Bool
+a /== b = not $ a === b
+
 {- A queue of actions to perform (in any order) on a git repository,
  - with lists of files to perform them on. This allows coalescing 
  - similar git commands. -}
@@ -59,16 +65,20 @@ defaultLimit = 10240
 new :: Maybe Int -> Queue
 new lim = Queue 0 (fromMaybe defaultLimit lim) M.empty
 
-{- Adds an action to a queue. -}
-add :: Queue -> String -> [CommandParam] -> [FilePath] -> Queue
-add (Queue cur lim m) subcommand params files = Queue (cur + 1) lim m'
+{- Adds an action to a queue. If the queue already contains a different
+ - action, it will be flushed; this is to ensure that conflicting actions,
+ - like add and rm, are run in the right order. -}
+add :: Queue -> String -> [CommandParam] -> [FilePath] -> Repo -> IO Queue
+add q@(Queue _ _ m) subcommand params files repo
+	| null (filter (/== action) (M.keys m)) = go q
+	| otherwise = go =<< flush q repo
 	where
 		action = Action subcommand params
-		-- There are probably few items in the map, but there
-		-- can be a lot of files per item. So, optimise adding
-		-- files.
-		m' = M.insertWith' const action fs m
-		!fs = files ++ M.findWithDefault [] action m
+		go (Queue cur lim m') =
+			return $ Queue (cur + 1) lim $
+				M.insertWith' const action fs m'
+			where
+				!fs = files ++ M.findWithDefault [] action m'
 
 {- Is a queue large enough that it should be flushed? -}
 full :: Queue -> Bool

From 48efa2d2d34ec532345054d0054dd81cfc597895 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 20:44:15 -0400
Subject: [PATCH 3608/8313] avoid explicit queue flush

The queue is still flushed on add, because each add event is handled by a
separate Annex monad. That needs to be fixed to speed up add a lot.
---
 Command/Watch.hs | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 024d2c568a..a3dc48b01e 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -15,7 +15,6 @@ import Utility.ThreadLock
 import qualified Annex
 import qualified Command.Add as Add
 import qualified Git.Command
-import qualified Annex.Queue
 import qualified Backend
 import Annex.Content
 
@@ -67,11 +66,9 @@ run startstate a f = do
  - The git queue is immediately flushed, so the file is added to git
  - now, rather than later (when it may have been already moved or deleted!) -}
 onAdd :: FilePath -> Annex ()
-onAdd file = doQuietSideAction $ do
-	void $ doCommand $ do
-		showStart "add" file
-		next $ Add.perform file
-	Annex.Queue.flush
+onAdd file = void $ doCommand $ do
+	showStart "add" file
+	next $ Add.perform file
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content

From cbdaccd44aa8f0ca30afba23fc06dd244c242075 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 21:21:52 -0400
Subject: [PATCH 3609/8313] run event handlers all in the same Annex monad

Uses a MVar again, as there seems no other way to thread the state through
inotify events.

This is a rather unsatisfactory result. I had wanted to run them in
the same monad so that the git queue could be used to coleasce git commands
and speed things up. But, that led to fragility: If several files are
added, and one is removed before queue flush, git add will fail to add
any of them. So, the queue is still explicitly flushed after each add for
now.

TODO: Investigate using git add --ignore-errors. This would need to be done
in Command.Add. And, git add still exits nonzero with it, so would need
to avoid crashing on queue flush.
---
 Annex.hs         |  3 +++
 Command/Watch.hs | 44 +++++++++++++++++++++++++-------------------
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/Annex.hs b/Annex.hs
index a9cc680125..38168334dd 100644
--- a/Annex.hs
+++ b/Annex.hs
@@ -14,6 +14,7 @@ module Annex (
 	newState,
 	run,
 	eval,
+	exec,
 	getState,
 	changeState,
 	setFlag,
@@ -134,6 +135,8 @@ run :: AnnexState -> Annex a -> IO (a, AnnexState)
 run s a = runStateT (runAnnex a) s
 eval :: AnnexState -> Annex a -> IO a
 eval s a = evalStateT (runAnnex a) s
+exec :: AnnexState -> Annex a -> IO AnnexState
+exec s a = execStateT (runAnnex a) s
 
 {- Sets a flag to True -}
 setFlag :: String -> Annex ()
diff --git a/Command/Watch.hs b/Command/Watch.hs
index a3dc48b01e..15c862bec5 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -5,14 +5,16 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Command.Watch where
 
-import CmdLine
 import Common.Annex
 import Command
 import Utility.Inotify
 import Utility.ThreadLock
 import qualified Annex
+import qualified Annex.Queue
 import qualified Command.Add as Add
 import qualified Git.Command
 import qualified Backend
@@ -20,6 +22,7 @@ import Annex.Content
 
 import Control.Exception as E
 import System.INotify
+import Control.Concurrent.MVar
 
 def :: [Command]
 def = [command "watch" paramPaths seek "watch for changes"]
@@ -33,8 +36,9 @@ start = notBareRepo $ do
 	showAction "scanning"
 	inRepo $ Git.Command.run "add" [Param "--update"]
 	state <- Annex.getState id
+	mvar <- liftIO $ newMVar state
 	next $ next $ liftIO $ withINotify $ \i -> do
-		let hook a = Just $ run state a
+		let hook a = Just $ runAnnex mvar a
 		watchDir i "." (not . gitdir)
 			(hook onAdd) (hook onAddSymlink)
 			(hook onDel) (hook onDelDir)
@@ -44,31 +48,33 @@ start = notBareRepo $ do
 	where
 		gitdir dir = takeFileName dir /= ".git"
 
-{- Inotify events are run in separate threads, and so each is a
- - self-contained Annex monad.
+{- Runs a handler, inside the Annex monad.
  -
- - Exceptions by the handlers are ignored,
- - otherwise a whole watcher thread could be crashed.
+ - Exceptions by the handlers are ignored, otherwise a whole watcher
+ - thread could be crashed.
  -}
-run :: Annex.AnnexState -> (FilePath -> Annex a) -> FilePath -> IO ()
-run startstate a f = do
-	r <- E.try go :: IO (Either E.SomeException ())
+runAnnex :: MVar Annex.AnnexState -> (FilePath -> Annex a) -> FilePath -> IO ()
+runAnnex mvar a f = do
+	startstate <- takeMVar mvar
+	r <- E.try (go startstate) :: IO (Either E.SomeException Annex.AnnexState)
 	case r of
-		Left e -> putStrLn (show e)
-		_ -> return ()
+		Left e -> do
+			putStrLn (show e)
+			putMVar mvar startstate
+		Right !newstate ->
+			putMVar mvar newstate
 	where
-		go = Annex.eval startstate $ do
-			_ <- a f
-			_ <- shutdown True
-			return ()
+		go state = Annex.exec state $ a f
 
 {- Adding a file is the same as git-annex add.
  - The git queue is immediately flushed, so the file is added to git
  - now, rather than later (when it may have been already moved or deleted!) -}
 onAdd :: FilePath -> Annex ()
-onAdd file = void $ doCommand $ do
-	showStart "add" file
-	next $ Add.perform file
+onAdd file = do
+	void $ doCommand $ do
+		showStart "add" file
+		next $ Add.perform file
+	Annex.Queue.flush
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
@@ -92,7 +98,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 
 onDel :: FilePath -> Annex ()
 onDel file = inRepo $ Git.Command.run "rm"
-	[Params "--quiet --cached --ignore-unmatch --", File file]
+	[Params "--quiet --cached --ignore-unmatch --", File file] 
 
 {- A directory has been deleted, or moved, so tell git to remove anything
  - that was inside it from its cache. -}

From b86825e42b227c3645fe93c123c38010115b2b5a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 21:29:26 -0400
Subject: [PATCH 3610/8313] update

---
 doc/design/assistant/inotify.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 82867f1040..6692b7d94b 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -22,6 +22,9 @@ really useful, it needs to:
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in)
 - tunable delays before adding new files, etc
+- Coleasce related add/rm events. See commit
+  cbdaccd44aa8f0ca30afba23fc06dd244c242075 for some details of the problems
+  with doing this.
 - don't annex `.gitignore` and `.gitattributes` files, but do auto-stage
   changes to them
 - configurable option to only annex files meeting certian size or

From 13118136c07a621ecd1272de3103207ed55b1da7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 4 Jun 2012 21:52:36 -0400
Subject: [PATCH 3611/8313] Preserve parent environment when running hooks of
 the hook special remote.

---
 Remote/Hook.hs   | 17 +++++++++++------
 debian/changelog |  1 +
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/Remote/Hook.hs b/Remote/Hook.hs
index dcac9da889..1202c6087e 100644
--- a/Remote/Hook.hs
+++ b/Remote/Hook.hs
@@ -10,6 +10,7 @@ module Remote.Hook (remote) where
 import qualified Data.ByteString.Lazy.Char8 as L
 import qualified Data.Map as M
 import System.Exit
+import System.Environment
 
 import Common.Annex
 import Types.Remote
@@ -59,9 +60,12 @@ hookSetup u c = do
 	gitConfigSpecialRemote u c' "hooktype" hooktype
 	return c'
 
-hookEnv :: Key -> Maybe FilePath -> Maybe [(String, String)]
-hookEnv k f = Just $ fileenv f ++ keyenv
+hookEnv :: Key -> Maybe FilePath -> IO (Maybe [(String, String)])
+hookEnv k f = Just <$> mergeenv (fileenv f ++ keyenv)
 	where
+		mergeenv l = M.toList .
+			M.union (M.fromList l) 
+				<$> M.fromList <$> getEnvironment
 		env s v = ("ANNEX_" ++ s, v)
 		keyenv =
 			[ env "KEY" (show k)
@@ -88,8 +92,9 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h
 	where
 		run command = do
 			showOutput -- make way for hook output
-			ifM (liftIO $ boolSystemEnv
-				"sh" [Param "-c", Param command] $ hookEnv k f)
+			ifM (liftIO $
+				boolSystemEnv "sh" [Param "-c", Param command]
+					=<< hookEnv k f)
 				( a
 				, do
 					warning $ hook ++ " hook exited nonzero!"
@@ -129,14 +134,14 @@ checkPresent r h k = do
 	liftIO $ catchMsgIO $ check v
 	where
 		findkey s = show k `elem` lines s
-		env = hookEnv k Nothing
 		check Nothing = error "checkpresent hook misconfigured"
 		check (Just hook) = do
 			(frompipe, topipe) <- createPipe
 			pid <- forkProcess $ do
 				_ <- dupTo topipe stdOutput
 				closeFd frompipe
-				executeFile "sh" True ["-c", hook] env
+				executeFile "sh" True ["-c", hook]
+					=<< hookEnv k Nothing
 			closeFd topipe
 			fromh <- fdToHandle frompipe
 			reply <- hGetContentsStrict fromh
diff --git a/debian/changelog b/debian/changelog
index 52f6c3b971..fa57391a85 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -8,6 +8,7 @@ git-annex (3.20120523) UNRELEASED; urgency=low
     unsupported backend.
   * Require that the SHA256 backend can be used when building, since it's the
     default.
+  * Preserve parent environment when running hooks of the hook special remote.
 
  -- Joey Hess   Sun, 27 May 2012 20:55:29 -0400
 

From a71b09a4fe9146b60adacbf52f6b9db3fba88f01 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 5 Jun 2012 14:07:27 +0000
Subject: [PATCH 3612/8313] Added a comment

---
 ..._8765b6190e79251637bb59ba28f245c1._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_2_8765b6190e79251637bb59ba28f245c1._comment

diff --git a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_2_8765b6190e79251637bb59ba28f245c1._comment b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_2_8765b6190e79251637bb59ba28f245c1._comment
new file mode 100644
index 0000000000..cad802a88a
--- /dev/null
+++ b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__/comment_2_8765b6190e79251637bb59ba28f245c1._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-06-05T14:07:26Z"
+ content="""
+FYI, (the follow is on OSX 10.7 on two different machines)
+
+On my 64bit install of haskell platform...
+
+    laplace:~ jtang$ ghc -e 'print System.Info.arch'
+    \"x86_64\"
+
+On my 32bit install of haskell platform...
+
+     x00:git-annex jtang$ ghc -e 'print System.Info.arch'
+    \"i386\"
+
+Running _cabal build_ or _cabal install git-annex_ as you suggest with the 32bit install does do the right thing.
+
+"""]]

From 8511957c68942e92b7574b5598e8ad183c59a19b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 14:14:45 -0400
Subject: [PATCH 3613/8313] releasing version 3.20120605

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index fa57391a85..fd4f7b98bc 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-git-annex (3.20120523) UNRELEASED; urgency=low
+git-annex (3.20120605) unstable; urgency=low
 
   * sync: Show a nicer message if a user tries to sync to a special remote.
   * lock: Reset unlocked file to index, rather than to branch head.
@@ -10,7 +10,7 @@ git-annex (3.20120523) UNRELEASED; urgency=low
     default.
   * Preserve parent environment when running hooks of the hook special remote.
 
- -- Joey Hess   Sun, 27 May 2012 20:55:29 -0400
+ -- Joey Hess   Tue, 05 Jun 2012 14:03:39 -0400
 
 git-annex (3.20120522) unstable; urgency=low
 
diff --git a/git-annex.cabal b/git-annex.cabal
index e12cbb1777..c151678319 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120522
+Version: 3.20120605
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From 1ab7b2b8eaa1de567e20c36edf10affdd7f6ac8d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 14:15:00 -0400
Subject: [PATCH 3614/8313] add news item for git-annex 3.20120605

---
 doc/news/version_3.20120406.mdwn |  4 ----
 doc/news/version_3.20120605.mdwn | 11 +++++++++++
 2 files changed, 11 insertions(+), 4 deletions(-)
 delete mode 100644 doc/news/version_3.20120406.mdwn
 create mode 100644 doc/news/version_3.20120605.mdwn

diff --git a/doc/news/version_3.20120406.mdwn b/doc/news/version_3.20120406.mdwn
deleted file mode 100644
index b3df304e27..0000000000
--- a/doc/news/version_3.20120406.mdwn
+++ /dev/null
@@ -1,4 +0,0 @@
-git-annex 3.20120406 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Disable diskfree on kfreebsd, as I have a build failure on kfreebsd-i386
-     that is quite likely caused by it."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120605.mdwn b/doc/news/version_3.20120605.mdwn
new file mode 100644
index 0000000000..ed0a091771
--- /dev/null
+++ b/doc/news/version_3.20120605.mdwn
@@ -0,0 +1,11 @@
+git-annex 3.20120605 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * sync: Show a nicer message if a user tries to sync to a special remote.
+   * lock: Reset unlocked file to index, rather than to branch head.
+   * import: New subcommand, pulls files from a directory outside the annex
+     and adds them.
+   * Fix display of warning message when encountering a file that uses an
+     unsupported backend.
+   * Require that the SHA256 backend can be used when building, since it's the
+     default.
+   * Preserve parent environment when running hooks of the hook special remote."""]]
\ No newline at end of file

From e771c645595fb6cedf0f0c5f2af393712013deb2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 14:53:30 -0400
Subject: [PATCH 3615/8313] races

---
 doc/design/assistant/inotify.mdwn | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 6692b7d94b..938aa23558 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -39,3 +39,34 @@ really useful, it needs to:
 - Support OSes other than Linux; it only uses inotify currently.
   OSX and FreeBSD use the same mechanism, and there is a Haskell interface
   for it,
+
+## the races
+
+Many races need to be dealt with by this code. Here are some of them.
+
+* File is added and then removed before the annex add finishes.
+  
+  Currently unfixed; The annex add re-adds the file as a symlink and then
+  the remove event does nothing since the file exists.
+
+* File is added and then replaced with another file before the annex add
+  makes its symlink.
+
+  Currently unfixed; The annex add will fail creating its symlink since
+  the file exists. The second add event will add the new file.
+
+* File is added and then replaced with another file before the annex add
+  moves its content into the annex.
+
+  Currently unfixed; The new content will be moved to the annex under the
+  old checksum, and fsck will later catch this inconsistency.
+
+* File is removed and then re-added before the removal event finishes.
+
+  Not a problem; The removal event removes the old file from the index, and
+  the add event adds the new one.
+
+* File is removed and then re-added before the removal event starts.
+
+  Not a problem; The removal event does nothing since the file exists,
+  and the add event replaces it in git with the new one.

From 73ab0e642c3ab3f35886d9d111e6490d66769d07 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 14:53:30 -0400
Subject: [PATCH 3616/8313] races

---
 doc/design/assistant/inotify.mdwn | 43 +++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 6692b7d94b..f6febeac82 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -39,3 +39,46 @@ really useful, it needs to:
 - Support OSes other than Linux; it only uses inotify currently.
   OSX and FreeBSD use the same mechanism, and there is a Haskell interface
   for it,
+
+## the races
+
+Many races need to be dealt with by this code. Here are some of them.
+
+* File is added and then removed before the add event starts.
+
+  Not a problem; The add event does nothing since the file is not present.
+
+* File is added and then removed before the add event has finished
+  processing it.
+  
+  Minor problem; When the add's processing of the file (checksum and so
+  on) fails due to it going away, there is an ugly error message, but
+  things are otherwise ok.
+
+* File is added and then removed before the add event finishes.
+  
+  Currently unfixed; The annex add re-adds the file as a symlink and then
+  the remove event does nothing since the symlink exists.
+
+* File is added and then replaced with another file before the annex add
+  makes its symlink.
+
+  Minor problem; The annex add will fail creating its symlink since
+  the file exists. There is an ugly error message, but the second add
+  event will add the new file.
+
+* File is added and then replaced with another file before the annex add
+  moves its content into the annex.
+
+  Currently unfixed; The new content will be moved to the annex under the
+  old checksum, and fsck will later catch this inconsistency.
+
+* File is removed and then re-added before the removal event finishes.
+
+  Not a problem; The removal event removes the old file from the index, and
+  the add event adds the new one.
+
+* File is removed and then re-added before the removal event starts.
+
+  Not a problem; The removal event does nothing since the file exists,
+  and the add event replaces it in git with the new one.

From 0eec65c81509210ed50d7980223da574dd9d98f7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 14:53:30 -0400
Subject: [PATCH 3617/8313] races

---
 doc/design/assistant/inotify.mdwn | 45 +++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 6692b7d94b..3a836a04c0 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -39,3 +39,48 @@ really useful, it needs to:
 - Support OSes other than Linux; it only uses inotify currently.
   OSX and FreeBSD use the same mechanism, and there is a Haskell interface
   for it,
+
+## the races
+
+Many races need to be dealt with by this code. Here are some of them.
+
+* File is added and then removed before the add event starts.
+
+  Not a problem; The add event does nothing since the file is not present.
+
+* File is added and then removed before the add event has finished
+  processing it.
+  
+  Minor problem; When the add's processing of the file (checksum and so
+  on) fails due to it going away, there is an ugly error message, but
+  things are otherwise ok.
+
+* File is added and then removed before the add event finishes.
+  
+  Currently unfixed; The annex add re-adds the file as a symlink and then
+  the remove event does nothing since the symlink exists.
+
+* File is added and then replaced with another file before the annex add
+  makes its symlink.
+
+  Minor problem; The annex add will fail creating its symlink since
+  the file exists. There is an ugly error message, but the second add
+  event will add the new file.
+
+* File is added and then replaced with another file before the annex add
+  moves its content into the annex.
+
+  Currently unfixed; The new content will be moved to the annex under the
+  old checksum, and fsck will later catch this inconsistency.
+
+  Possible fix: Move content someplace before doing checksumming.
+
+* File is removed and then re-added before the removal event starts.
+
+  Not a problem; The removal event does nothing since the file exists,
+  and the add event replaces it in git with the new one.
+
+* File is removed and then re-added before the removal event finishes.
+
+  Not a problem; The removal event removes the old file from the index, and
+  the add event adds the new one.

From 40456953c27a39e68f663e3a0ad08e95bbaf2f1c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 15:20:13 -0400
Subject: [PATCH 3618/8313] another one

---
 doc/design/assistant/inotify.mdwn | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 3a836a04c0..3b3e5dc56d 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -75,6 +75,14 @@ Many races need to be dealt with by this code. Here are some of them.
 
   Possible fix: Move content someplace before doing checksumming.
 
+* File is added and then replaced with another file before the annex add
+  stages the symlink in git.
+
+  Currently unfixed; `git add` will be run on the new file, which is
+  not at all good when it's big. Could be dealt with by using `git
+  update-index` to manually put the symlink into the index without git
+  looking at what's currently on disk.
+
 * File is removed and then re-added before the removal event starts.
 
   Not a problem; The removal event does nothing since the file exists,

From b5c617b6232715f8109fa9eb2b77edae099f3a92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 18:54:25 -0400
Subject: [PATCH 3619/8313] one more

---
 doc/design/assistant/inotify.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 3b3e5dc56d..2e79ee95d7 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -83,6 +83,12 @@ Many races need to be dealt with by this code. Here are some of them.
   update-index` to manually put the symlink into the index without git
   looking at what's currently on disk.
 
+* Link is moved, fixed link is written by fix event, but then that is
+  removed by the user and replaced with a file before the event finishes.
+
+  Currently unfixed: `git add` will be run on the file. Basically same
+  effect as previous race above.
+
 * File is removed and then re-added before the removal event starts.
 
   Not a problem; The removal event does nothing since the file exists,
@@ -92,3 +98,4 @@ Many races need to be dealt with by this code. Here are some of them.
 
   Not a problem; The removal event removes the old file from the index, and
   the add event adds the new one.
+

From 77188ff04d8d4d42b25daac9eeffbf10d8b663ac Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 19:03:39 -0400
Subject: [PATCH 3620/8313] update

---
 doc/design/assistant/inotify.mdwn | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 2e79ee95d7..3263c476da 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -51,34 +51,29 @@ Many races need to be dealt with by this code. Here are some of them.
 * File is added and then removed before the add event has finished
   processing it.
   
-  Minor problem; When the add's processing of the file (checksum and so
+  **Minor problem**; When the add's processing of the file (checksum and so
   on) fails due to it going away, there is an ugly error message, but
   things are otherwise ok.
 
-* File is added and then removed before the add event finishes.
-  
-  Currently unfixed; The annex add re-adds the file as a symlink and then
-  the remove event does nothing since the symlink exists.
-
-* File is added and then replaced with another file before the annex add
-  makes its symlink.
-
-  Minor problem; The annex add will fail creating its symlink since
-  the file exists. There is an ugly error message, but the second add
-  event will add the new file.
-
 * File is added and then replaced with another file before the annex add
   moves its content into the annex.
 
-  Currently unfixed; The new content will be moved to the annex under the
+  **Currently unfixed**; The new content will be moved to the annex under the
   old checksum, and fsck will later catch this inconsistency.
 
   Possible fix: Move content someplace before doing checksumming.
 
+* File is added and then replaced with another file before the annex add
+  makes its symlink.
+
+  **Minor problem**; The annex add will fail creating its symlink since
+  the file exists. There is an ugly error message, but the second add
+  event will add the new file.
+
 * File is added and then replaced with another file before the annex add
   stages the symlink in git.
 
-  Currently unfixed; `git add` will be run on the new file, which is
+  **Currently unfixed**; `git add` will be run on the new file, which is
   not at all good when it's big. Could be dealt with by using `git
   update-index` to manually put the symlink into the index without git
   looking at what's currently on disk.
@@ -86,7 +81,7 @@ Many races need to be dealt with by this code. Here are some of them.
 * Link is moved, fixed link is written by fix event, but then that is
   removed by the user and replaced with a file before the event finishes.
 
-  Currently unfixed: `git add` will be run on the file. Basically same
+  **Currently unfixed**: `git add` will be run on the file. Basically same
   effect as previous race above.
 
 * File is removed and then re-added before the removal event starts.

From d3cee987caf20b309334b37bd1b89e8b9115cf0a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 19:51:03 -0400
Subject: [PATCH 3621/8313] separate source of content from the filename
 associated with the key when generating a key

This already made migrate's code a lot simpler.
---
 Backend.hs                        | 18 ++++++++++--------
 Backend/SHA.hs                    | 13 +++++++------
 Backend/URL.hs                    | 10 +++++-----
 Backend/WORM.hs                   | 18 +++++++++---------
 Command/Add.hs                    |  7 ++++---
 Command/AddUrl.hs                 |  7 ++++---
 Command/Migrate.hs                | 26 +++++++-------------------
 Types/Backend.hs                  | 13 ++++++++++---
 doc/design/assistant/inotify.mdwn |  4 +++-
 9 files changed, 59 insertions(+), 57 deletions(-)

diff --git a/Backend.hs b/Backend.hs
index fa32669449..bde1aad78e 100644
--- a/Backend.hs
+++ b/Backend.hs
@@ -6,6 +6,7 @@
  -}
 
 module Backend (
+	B.KeySource(..),
 	list,
 	orderedList,
 	genKey,
@@ -51,18 +52,19 @@ orderedList = do
 		parseBackendList s = map lookupBackendName $ words s
 
 {- Generates a key for a file, trying each backend in turn until one
- - accepts it. -}
-genKey :: FilePath -> Maybe Backend -> Annex (Maybe (Key, Backend))
-genKey file trybackend = do
+ - accepts it.
+ -}
+genKey :: B.KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend))
+genKey source trybackend = do
 	bs <- orderedList
 	let bs' = maybe bs (: bs) trybackend
-	genKey' bs' file
-genKey' :: [Backend] -> FilePath -> Annex (Maybe (Key, Backend))
+	genKey' bs' source
+genKey' :: [Backend] -> B.KeySource -> Annex (Maybe (Key, Backend))
 genKey' [] _ = return Nothing
-genKey' (b:bs) file = do
-	r <- B.getKey b file
+genKey' (b:bs) source = do
+	r <- B.getKey b source
 	case r of
-		Nothing -> genKey' bs file
+		Nothing -> genKey' bs source
 		Just k -> return $ Just (makesane k, b)
 	where
 		-- keyNames should not contain newline characters.
diff --git a/Backend/SHA.hs b/Backend/SHA.hs
index c2a6cf9761..df613bbcdd 100644
--- a/Backend/SHA.hs
+++ b/Backend/SHA.hs
@@ -69,9 +69,10 @@ shaN size file = do
 		command = fromJust $ shaCommand size
 
 {- A key is a checksum of its contents. -}
-keyValue :: SHASize -> FilePath -> Annex (Maybe Key)
-keyValue size file = do
-	s <- shaN size file	
+keyValue :: SHASize -> KeySource -> Annex (Maybe Key)
+keyValue size source = do
+	let file = contentLocation source
+	s <- shaN size file
 	stat <- liftIO $ getFileStatus file
 	return $ Just $ stubKey
 		{ keyName = s
@@ -80,14 +81,14 @@ keyValue size file = do
 		}
 
 {- Extension preserving keys. -}
-keyValueE :: SHASize -> FilePath -> Annex (Maybe Key)
-keyValueE size file = keyValue size file >>= maybe (return Nothing) addE
+keyValueE :: SHASize -> KeySource -> Annex (Maybe Key)
+keyValueE size source = keyValue size source >>= maybe (return Nothing) addE
 	where
 		addE k = return $ Just $ k
 			{ keyName = keyName k ++ extension
 			, keyBackendName = shaNameE size
 			}
-		naiveextension = takeExtension file
+		naiveextension = takeExtension $ keyFilename source
 		extension
 			-- long or newline containing extensions are 
 			-- probably not really an extension
diff --git a/Backend/URL.hs b/Backend/URL.hs
index b98974cb45..cc9112a362 100644
--- a/Backend/URL.hs
+++ b/Backend/URL.hs
@@ -20,11 +20,11 @@ backends :: [Backend]
 backends = [backend]
 
 backend :: Backend
-backend = Backend {
-	name = "URL",
-	getKey = const (return Nothing),
-	fsckKey = Nothing
-}
+backend = Backend
+	{ name = "URL"
+	, getKey = const $ return Nothing
+	, fsckKey = Nothing
+	}
 
 fromUrl :: String -> Maybe Integer -> Key
 fromUrl url size = stubKey
diff --git a/Backend/WORM.hs b/Backend/WORM.hs
index c022fd413b..630000fa2b 100644
--- a/Backend/WORM.hs
+++ b/Backend/WORM.hs
@@ -15,11 +15,11 @@ backends :: [Backend]
 backends = [backend]
 
 backend :: Backend
-backend = Backend {
-	name = "WORM",
-	getKey = keyValue,
-	fsckKey = Nothing
-}
+backend = Backend
+	{ name = "WORM"
+	, getKey = keyValue
+	, fsckKey = Nothing
+	}
 
 {- The key includes the file size, modification time, and the
  - basename of the filename.
@@ -28,11 +28,11 @@ backend = Backend {
  - while also allowing a file to be moved around while retaining the
  - same key.
  -}
-keyValue :: FilePath -> Annex (Maybe Key)
-keyValue file = do
-	stat <- liftIO $ getFileStatus file
+keyValue :: KeySource -> Annex (Maybe Key)
+keyValue source = do
+	stat <- liftIO $ getFileStatus $ contentLocation source
 	return $ Just Key {
-		keyName = takeFileName file,
+		keyName = takeFileName $ keyFilename source,
 		keyBackendName = name backend,
 		keySize = Just $ fromIntegral $ fileSize stat,
 		keyMtime = Just $ modificationTime stat
diff --git a/Command/Add.hs b/Command/Add.hs
index ef839b2a30..7029a9c167 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -12,7 +12,7 @@ import Annex.Exception
 import Command
 import qualified Annex
 import qualified Annex.Queue
-import qualified Backend
+import Backend
 import Logs.Location
 import Annex.Content
 import Utility.Touch
@@ -46,8 +46,9 @@ start file = notBareRepo $ ifAnnexed file fixup add
 
 perform :: FilePath -> CommandPerform
 perform file = do
-	backend <- Backend.chooseBackend file
-	Backend.genKey file backend >>= go
+	let source = KeySource { keyFilename = file, contentLocation = file}
+	backend <- chooseBackend file
+	genKey source backend >>= go
 	where
 		go Nothing = stop
 		go (Just (key, _)) = do
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 089606e85d..87b24149d8 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -11,7 +11,7 @@ import Network.URI
 
 import Common.Annex
 import Command
-import qualified Backend
+import Backend
 import qualified Command.Add
 import qualified Annex
 import qualified Backend.URL
@@ -72,8 +72,9 @@ download url file = do
 	tmp <- fromRepo $ gitAnnexTmpLocation dummykey
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	stopUnless (downloadUrl [url] tmp) $ do
-		backend <- Backend.chooseBackend file
-		k <- Backend.genKey tmp backend
+		backend <- chooseBackend file
+		let source = KeySource { keyFilename = file, contentLocation = file}
+		k <- genKey source backend
 		case k of
 			Nothing -> stop
 			Just (key, _) -> do
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 6e28c4b52e..29e664ce23 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -9,7 +9,7 @@ module Command.Migrate where
 
 import Common.Annex
 import Command
-import qualified Backend
+import Backend
 import qualified Types.Key
 import Annex.Content
 import qualified Command.ReKey
@@ -23,14 +23,14 @@ seek = [withFilesInGit $ whenAnnexed start]
 start :: FilePath -> (Key, Backend) -> CommandStart
 start file (key, oldbackend) = do
 	exists <- inAnnex key
-	newbackend <- choosebackend =<< Backend.chooseBackend file
+	newbackend <- choosebackend =<< chooseBackend file
 	if (newbackend /= oldbackend || upgradableKey key) && exists
 		then do
 			showStart "migrate" file
 			next $ perform file key newbackend
 		else stop
 	where
-		choosebackend Nothing = Prelude.head <$> Backend.orderedList
+		choosebackend Nothing = Prelude.head <$> orderedList
 		choosebackend (Just backend) = return backend
 
 {- Checks if a key is upgradable to a newer representation. -}
@@ -40,25 +40,13 @@ upgradableKey key = isNothing $ Types.Key.keySize key
 
 {- Store the old backend's key in the new backend
  - The old backend's key is not dropped from it, because there may
- - be other files still pointing at that key.
- -
- - Use the same filename as the file for the temp file name, to support
- - backends that allow the filename to influence the keys they
- - generate.
- -}
+ - be other files still pointing at that key. -}
 perform :: FilePath -> Key -> Backend -> CommandPerform
 perform file oldkey newbackend = maybe stop go =<< genkey
 	where
 		go newkey = stopUnless (Command.ReKey.linkKey oldkey newkey) $
 			next $ Command.ReKey.cleanup file oldkey newkey
 		genkey = do
-			src <- inRepo $ gitAnnexLocation oldkey
-			tmp <- fromRepo gitAnnexTmpDir
-			let tmpfile = tmp  takeFileName file
-			cleantmp tmpfile
-			liftIO $ createLink src tmpfile
-			newkey <- liftM fst <$>
-				Backend.genKey tmpfile (Just newbackend)
-			cleantmp tmpfile
-			return newkey
-		cleantmp t = liftIO $ whenM (doesFileExist t) $ removeFile t
+			content <- inRepo $ gitAnnexLocation oldkey
+			let source = KeySource { keyFilename = file, contentLocation = content }
+			liftM fst <$> genKey source (Just newbackend)
diff --git a/Types/Backend.hs b/Types/Backend.hs
index d52cec5471..5abb0896dc 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -2,7 +2,7 @@
  -
  - Most things should not need this, using Types instead
  -
- - Copyright 2010 Joey Hess 
+ - Copyright 2010,2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
@@ -11,11 +11,18 @@ module Types.Backend where
 
 import Types.Key
 
+{- The source used to generate a key. The location of the content
+ - may be different from the filename associated with the key. -}
+data KeySource = KeySource
+	{ keyFilename :: FilePath
+	, contentLocation :: FilePath
+	}
+
 data BackendA a = Backend {
 	-- name of this backend
 	name :: String,
-	-- converts a filename to a key
-	getKey :: FilePath -> a (Maybe Key),
+	-- gets the key to use for a given content
+	getKey :: KeySource -> a (Maybe Key),
 	-- called during fsck to check a key, if the backend has its own checks
 	fsckKey :: Maybe (Key -> FilePath -> a Bool)
 }
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 3263c476da..ca63a1c823 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -61,7 +61,9 @@ Many races need to be dealt with by this code. Here are some of them.
   **Currently unfixed**; The new content will be moved to the annex under the
   old checksum, and fsck will later catch this inconsistency.
 
-  Possible fix: Move content someplace before doing checksumming.
+  Possible fix: Move content someplace before doing checksumming. Perhaps
+  using a hard link and removing the write bit to prevent modification 
+  while checksumming.
 
 * File is added and then replaced with another file before the annex add
   makes its symlink.

From f8d422fe24e425676a928959a2489f277c3026d3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 19:54:44 -0400
Subject: [PATCH 3622/8313] update test suite

---
 test.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test.hs b/test.hs
index 9a0fce873e..1a7c382c01 100644
--- a/test.hs
+++ b/test.hs
@@ -171,7 +171,8 @@ test_reinject :: Test
 test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do
 	git_annex "drop" ["--force", sha1annexedfile] @? "drop failed"
 	writeFile tmp $ content sha1annexedfile
-	r <- annexeval $ Types.Backend.getKey backendSHA1 tmp
+	r <- annexeval $ Types.Backend.getKey backendSHA1 $
+		Types.Backend.KeySource { Types.Backend.keyFilename = tmp, Types.Backend.contentLocation = tmp }
 	let key = show $ fromJust r
 	git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed"
 	git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed"

From 5809f33f8b3c2aa3cb8207bc775339c533a914ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 20:25:32 -0400
Subject: [PATCH 3623/8313] use createAnnexDirectory when setting up tmp dir

---
 Command/Fsck.hs | 3 ++-
 Remote/Rsync.hs | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 38b1bbbacd..ae21acf8af 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -16,6 +16,7 @@ import qualified Types.Backend
 import qualified Types.Key
 import qualified Backend
 import Annex.Content
+import Annex.Perms
 import Logs.Location
 import Logs.Trust
 import Annex.UUID
@@ -83,8 +84,8 @@ performRemote key file backend numcopies remote =
 		withtmp a = do
 			pid <- liftIO getProcessID
 			t <- fromRepo gitAnnexTmpDir
+			createAnnexDirectory t
 			let tmp = t  "fsck" ++ show pid ++ "." ++ keyFile key
-			liftIO $ createDirectoryIfMissing True t
 			let cleanup = liftIO $ catchIO (removeFile tmp) (const noop)
 			cleanup
 			cleanup `after` a tmp
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index 60cbf4595f..df4e0a44f2 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -19,6 +19,7 @@ import Remote.Helper.Special
 import Remote.Helper.Encryptable
 import Crypto
 import Utility.RsyncFile
+import Annex.Perms
 
 type RsyncUrl = String
 
@@ -176,6 +177,7 @@ withRsyncScratchDir :: (FilePath -> Annex Bool) -> Annex Bool
 withRsyncScratchDir a = do
 	pid <- liftIO getProcessID
 	t <- fromRepo gitAnnexTmpDir
+	createAnnexDirectory t
 	let tmp = t  "rsynctmp"  show pid
 	nuke tmp
 	liftIO $ createDirectoryIfMissing True tmp

From c981ccc0773a02ca60eb6456f04de14cd758ee7b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 20:28:34 -0400
Subject: [PATCH 3624/8313] add: Prevent (most) modifications from being made
 to a file while it is being added to the annex.

Anything that tries to open the file for write, or delete the file,
or replace it with something else, will not affect the add.

Only if a process has the file open for write before add starts
can it still change it while (or after) it's added to the annex.
(fsck will catch this later of course)
---
 Command/Add.hs   | 29 +++++++++++++++++++++++------
 debian/changelog |  7 +++++++
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 7029a9c167..2c671eea29 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -15,7 +15,9 @@ import qualified Annex.Queue
 import Backend
 import Logs.Location
 import Annex.Content
+import Annex.Perms
 import Utility.Touch
+import Utility.FileMode
 
 def :: [Command]
 def = [command "add" paramPaths seek "add files to annex"]
@@ -44,23 +46,38 @@ start file = notBareRepo $ ifAnnexed file fixup add
 			liftIO $ removeFile file
 			next $ next $ cleanup file key =<< inAnnex key
 
+{- The file that's being added is locked down before a key is generated,
+ - to prevent it from being modified in between. It's hard linked into a
+ - temporary location, and its writable bits are removed. It could still be
+ - written to by a process that already has it open for writing. -}
 perform :: FilePath -> CommandPerform
 perform file = do
-	let source = KeySource { keyFilename = file, contentLocation = file}
+	liftIO $ preventWrite file
+	tmp <- fromRepo gitAnnexTmpDir
+	createAnnexDirectory tmp
+	pid <- liftIO getProcessID
+	let tmpfile = tmp  "add" ++ show pid ++ "." ++ takeFileName file
+	nuke tmpfile
+	liftIO $ createLink file tmpfile
+	let source = KeySource { keyFilename = file, contentLocation = tmpfile }
 	backend <- chooseBackend file
-	genKey source backend >>= go
+	genKey source backend >>= go tmpfile
 	where
-		go Nothing = stop
-		go (Just (key, _)) = do
-			handle (undo file key) $ moveAnnex key file
+		go _ Nothing = stop
+		go tmpfile (Just (key, _)) = do
+			handle (undo file key) $ moveAnnex key tmpfile
+			nuke file
 			next $ cleanup file key True
 
+nuke :: FilePath -> Annex ()
+nuke file = liftIO $ whenM (doesFileExist file) $ removeFile file
+
 {- On error, put the file back so it doesn't seem to have vanished.
  - This can be called before or after the symlink is in place. -}
 undo :: FilePath -> Key -> IOException -> Annex a
 undo file key e = do
 	whenM (inAnnex key) $ do
-		liftIO $ whenM (doesFileExist file) $ removeFile file
+		nuke file
 		handle tryharder $ fromAnnex key file
 		logStatus key InfoMissing
 	throw e
diff --git a/debian/changelog b/debian/changelog
index fd4f7b98bc..9a010327df 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120606) UNRELEASED; urgency=low
+
+  * add: Prevent (most) modifications from being made to a file while it
+    is being added to the annex.
+
+ -- Joey Hess   Tue, 05 Jun 2012 20:25:51 -0400
+
 git-annex (3.20120605) unstable; urgency=low
 
   * sync: Show a nicer message if a user tries to sync to a special remote.

From d05a22b5f6004abeef0878d164f90afc59ede9ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 20:34:25 -0400
Subject: [PATCH 3625/8313] closed a race, although a less likely similar one
 remains

---
 doc/design/assistant/inotify.mdwn | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index ca63a1c823..5d903a9b04 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -58,12 +58,18 @@ Many races need to be dealt with by this code. Here are some of them.
 * File is added and then replaced with another file before the annex add
   moves its content into the annex.
 
-  **Currently unfixed**; The new content will be moved to the annex under the
-  old checksum, and fsck will later catch this inconsistency.
+  Fixed this problem; Now it hard links the file to a temp directory and
+  operates on the hard link, which is also made unwritable.
 
-  Possible fix: Move content someplace before doing checksumming. Perhaps
-  using a hard link and removing the write bit to prevent modification 
-  while checksumming.
+* A process has a file open for write, another one closes it, and so it's
+  added. Then the first process modifies it.
+
+  **Currently unfixed**; This changes content in the annex, and fsck will
+  later catch the inconsistency.
+
+  Possible fixes: Somehow track or detect if a file is open for write
+  by any processes. Or, when possible, making a copy on write copy
+  before adding the file would avoid this.
 
 * File is added and then replaced with another file before the annex add
   makes its symlink.

From d41c1134cb1f49529b294407efd24061c8219347 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 5 Jun 2012 20:54:56 -0400
Subject: [PATCH 3626/8313] daily blog

---
 doc/design/assistant/blog/day_2__races.mdwn | 45 +++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_2__races.mdwn

diff --git a/doc/design/assistant/blog/day_2__races.mdwn b/doc/design/assistant/blog/day_2__races.mdwn
new file mode 100644
index 0000000000..fadedb5fb2
--- /dev/null
+++ b/doc/design/assistant/blog/day_2__races.mdwn
@@ -0,0 +1,45 @@
+Last night I got `git annex watch` to also handle deletion of files.
+This was not as tricky as feared; the key is using `git rm --ignore-unmatch`,
+which avoids most problimatic situations (such as a just deleted file
+being added back before git is run).
+
+Also fixed some races when `git annex watch` is doing its startup scan of
+the tree, which might be changed as it's being traversed. Now only one
+thread performs actions at a time, so inotify events are queued up during
+the scan, and dealt with once it completes. It's worth noting that inotify
+can only buffer so many events .. Which might have been a problem except
+for a very nice feature of Haskell's inotify interface: It has a thread
+that drains the limited inotify buffer and does its own buffering.
+
+----
+
+Right now, `git annex watch` is not as fast as it could be when doing
+something like adding a lot of files, or deleting a lot of files.
+For each file, it currently runs a git command that updates the index.
+I did some work toward coalescing these into one command (which `git annex`
+already does normally). It's not quite ready to be turned on yet,
+because of some races involving `git add` that become much worse
+if it's delayed by event coalescing.
+
+----
+
+And races were the theme of today. Spent most of the day really
+getting to grips with all the fun races that can occur between
+modification happening to files, and `git annex watch`. The [[inotify]]
+page now has a long list of known races, some benign, and several,
+all involving adding files, that are quite nasty.
+
+I fixed one of those races this evening. The rest will probably involve
+moving away from using `git add`, which necessarily examines the file
+on disk, to directly shoving the symlink into git's index.
+
+BTW, it turns out that `dvcs-autosync` has grappled with some of these same
+races: 
+I hope that `git annex watch` will be in a better place to deal with them,
+since it's only dealing with git, and with a restricted portion of it
+relevant to git-annex.
+
+It's important that `git annex watch` be rock solid. It's the foundation
+of the git annex assistant. Users should not need to worry about races
+when using it. Most users won't know what race conditions are. If only I
+could be so lucky!

From f5261f60c3d4462088768959014189a3aa61ff30 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 3 Jun 2012 15:05:59 -0700
Subject: [PATCH 3627/8313] Make standalone man-page installation possible

The `cabal install git-annex` doesn't install the man pages, and the
Makefile only installed the man pages as part of a full build/install.
So, I factored out the documentation parts of the Makefile.
---
 Makefile         | 12 ++++++++----
 doc/install.mdwn | 22 ++++++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 94dc05a819..26367aa46d 100644
--- a/Makefile
+++ b/Makefile
@@ -47,17 +47,21 @@ git-annex-shell.1: doc/git-annex-shell.mdwn
 git-union-merge.1: doc/git-union-merge.mdwn
 	./mdwn2man git-union-merge 1 doc/git-union-merge.mdwn > git-union-merge.1
 
-install: all
-	install -d $(DESTDIR)$(PREFIX)/bin
-	install $(bins) $(DESTDIR)$(PREFIX)/bin
-	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
+install-mans: $(mans)
 	install -d $(DESTDIR)$(PREFIX)/share/man/man1
 	install -m 0644 $(mans) $(DESTDIR)$(PREFIX)/share/man/man1
+
+install-docs: docs install-mans
 	install -d $(DESTDIR)$(PREFIX)/share/doc/git-annex
 	if [ -d html ]; then \
 		rsync -a --delete html/ $(DESTDIR)$(PREFIX)/share/doc/git-annex/html/; \
 	fi
 
+install: all install-docs
+	install -d $(DESTDIR)$(PREFIX)/bin
+	install $(bins) $(DESTDIR)$(PREFIX)/bin
+	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
+
 test:
 	@if ! $(GHCMAKE) -O0 test $(clibs); then \
 		echo "** failed to build the test suite" >&2; \
diff --git a/doc/install.mdwn b/doc/install.mdwn
index fe0522aa05..1f4b54f542 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -20,6 +20,28 @@ As a haskell package, git-annex can be installed using cabal. For example:
 The above downloads the latest release. Alternatively, you can [[download]]
 it yourself and [[manually_build_with_cabal|install/cabal]].
 
+### Man pages
+
+Cabal does not install the man pages.  Do:
+
+	cd /tmp
+	cabal unpack git-annex
+	cd git-annex*
+	make install-mans PREFIX=~/.cabal
+
+to install man pages under ~/.cabal/share/man (or $PREFIX/share/man
+generally).  If ~/.cabal/bin is on your PATH, and you're on a Debian
+derivative, you're probably set.
+
+On Debian systems, `manpath` will print the paths searched for manual
+pages.  The paths searched are inferred from your PATH.  Experiments
+indicate that, on Ubuntu, D/share/man will be searched for manual
+pages whenever D/bin is on your PATH.  So, having ~/.cabal/bin on your
+PATH is enough to make the above example work.  The `man manpath` does
+not document this, but see
+http://linux.derkeiler.com/Mailing-Lists/Debian/2003-08/0956.html for
+some other special cases.
+
 ## Installation by hand
 
 To build and use git-annex, you will need:

From 141fa3c94d9b5d9404b8e875b6806f27340f2cf3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 00:01:17 -0400
Subject: [PATCH 3628/8313] update

---
 doc/design/assistant/inotify.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 5d903a9b04..e7c61c68b1 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -69,7 +69,8 @@ Many races need to be dealt with by this code. Here are some of them.
 
   Possible fixes: Somehow track or detect if a file is open for write
   by any processes. Or, when possible, making a copy on write copy
-  before adding the file would avoid this.
+  before adding the file would avoid this.  Or, as a last resort, make
+  an expensive copy of the file and add that.
 
 * File is added and then replaced with another file before the annex add
   makes its symlink.

From f1bd72ea546be705334ba8f6d01d9dcfb0c33cf9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 00:03:08 -0400
Subject: [PATCH 3629/8313] factor out generic update-index code from
 unionmerge code

---
 Annex/Branch.hs    |  9 +++++----
 Git/UnionMerge.hs  | 41 ++------------------------------------
 Git/UpdateIndex.hs | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 43 deletions(-)
 create mode 100644 Git/UpdateIndex.hs

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 706522f3b3..c8d0719b0b 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -33,6 +33,7 @@ import qualified Git.Command
 import qualified Git.Ref
 import qualified Git.Branch
 import qualified Git.UnionMerge
+import qualified Git.UpdateIndex
 import Git.HashObject
 import qualified Git.Index
 import Annex.CatFile
@@ -258,8 +259,8 @@ files = withIndexUpdate $ do
  - in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UnionMerge.stream_update_index g
-	[Git.UnionMerge.ls_tree fullname g]
+genIndex g = Git.UpdateIndex.stream_update_index g
+	[Git.UpdateIndex.ls_tree fullname g]
 
 {- Merges the specified refs into the index.
  - Any changes staged in the index will be preserved. -}
@@ -335,13 +336,13 @@ stageJournal = do
 	g <- gitRepo
 	withIndex $ liftIO $ do
 		h <- hashObjectStart g
-		Git.UnionMerge.stream_update_index g
+		Git.UpdateIndex.stream_update_index g
 			[genstream (gitAnnexJournalDir g) h fs]
 		hashObjectStop h
 	where
 		genstream dir h fs streamer = forM_ fs $ \file -> do
 			let path = dir  file
 			sha <- hashFile h path
-			_ <- streamer $ Git.UnionMerge.update_index_line
+			_ <- streamer $ Git.UpdateIndex.update_index_line
 				sha (fileJournal file)
 			removeFile path
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index d68bb61ab1..9ff820dc91 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -7,11 +7,7 @@
 
 module Git.UnionMerge (
 	merge,
-	merge_index,
-	update_index,
-	stream_update_index,
-	update_index_line,
-	ls_tree
+	merge_index
 ) where
 
 import System.Cmd.Utils
@@ -24,8 +20,7 @@ import Git
 import Git.Sha
 import Git.CatFile
 import Git.Command
-
-type Streamer = (String -> IO ()) -> IO ()
+import Git.UpdateIndex
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
@@ -47,38 +42,6 @@ merge_index :: CatFileHandle -> Repo -> [Ref] -> IO ()
 merge_index h repo bs =
 	stream_update_index repo $ map (\b -> merge_tree_index b h repo) bs
 
-{- Feeds content into update-index. Later items in the list can override
- - earlier ones, so the list can be generated from any combination of
- - ls_tree, merge_trees, and merge_tree_index. -}
-update_index :: Repo -> [String] -> IO ()
-update_index repo ls = stream_update_index repo [(`mapM_` ls)]
-
-{- Streams content into update-index. -}
-stream_update_index :: Repo -> [Streamer] -> IO ()
-stream_update_index repo as = do
-	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
-	fileEncoding h
-	forM_ as (stream h)
-	hClose h
-	forceSuccess p
-	where
-		params = map Param ["update-index", "-z", "--index-info"]
-		stream h a = a (streamer h)
-		streamer h s = do
-			hPutStr h s
-			hPutStr h "\0"
-
-{- Generates a line suitable to be fed into update-index, to add
- - a given file with a given sha. -}
-update_index_line :: Sha -> FilePath -> String
-update_index_line sha file = "100644 blob " ++ show sha ++ "\t" ++ file
-
-{- Gets the current tree for a ref. -}
-ls_tree :: Ref -> Repo -> Streamer
-ls_tree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
-	where
-		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
-
 {- For merging two trees. -}
 merge_trees :: Ref -> Ref -> CatFileHandle -> Repo -> Streamer
 merge_trees (Ref x) (Ref y) h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
new file mode 100644
index 0000000000..04bc4da5ba
--- /dev/null
+++ b/Git/UpdateIndex.hs
@@ -0,0 +1,49 @@
+{- git-update-index library
+ -
+ - Copyright 2011, 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.UpdateIndex (
+	Streamer,
+	stream_update_index,
+	update_index_line,
+	ls_tree
+) where
+
+import System.Cmd.Utils
+
+import Common
+import Git
+import Git.Command
+
+{- Streamers are passed a callback and should feed it lines in the form
+ - read by update-index, and generated by ls-tree. -}
+type Streamer = (String -> IO ()) -> IO ()
+
+{- Streams content into update-index from a list of Streamers. -}
+stream_update_index :: Repo -> [Streamer] -> IO ()
+stream_update_index repo as = do
+	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
+	fileEncoding h
+	forM_ as (stream h)
+	hClose h
+	forceSuccess p
+	where
+		params = map Param ["update-index", "-z", "--index-info"]
+		stream h a = a (streamer h)
+		streamer h s = do
+			hPutStr h s
+			hPutStr h "\0"
+
+{- Generates a line suitable to be fed into update-index, to add
+ - a given file with a given sha. -}
+update_index_line :: Sha -> FilePath -> String
+update_index_line sha file = "100644 blob " ++ show sha ++ "\t" ++ file
+
+{- Gets the current tree for a ref. -}
+ls_tree :: Ref -> Repo -> Streamer
+ls_tree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
+	where
+		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]

From f596084a599fb363dcbb425dce7c4ca46bb56ca0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 02:31:31 -0400
Subject: [PATCH 3630/8313] move hashObject to HashObject library and
 generalize it to support all git object types

---
 Git/CatFile.hs    |  8 ++------
 Git/HashObject.hs | 18 ++++++++++++++++--
 Git/Types.hs      | 15 +++++++++++++++
 Git/UnionMerge.hs | 17 +++--------------
 4 files changed, 36 insertions(+), 22 deletions(-)

diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index c598d7aa4b..d5b367945f 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -21,6 +21,7 @@ import Common
 import Git
 import Git.Sha
 import Git.Command
+import Git.Types
 import qualified Utility.CoProcess as CoProcess
 
 type CatFileHandle = CoProcess.CoProcessHandle
@@ -52,7 +53,7 @@ catObject h object = CoProcess.query h send receive
 			case words header of
 				[sha, objtype, size]
 					| length sha == shaSize &&
-					  validobjtype objtype -> 
+					  isJust (readObjectType objtype) -> 
 						case reads size of
 							[(bytes, "")] -> readcontent bytes from
 							_ -> dne
@@ -67,8 +68,3 @@ catObject h object = CoProcess.query h send receive
 				error "missing newline from git cat-file"
 			return $ L.fromChunks [content]
 		dne = return L.empty
-		validobjtype t
-			| t == "blob" = True
-			| t == "commit" = True
-			| t == "tree" = True
-			| otherwise = False
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index 617e5ac28f..b052413fdb 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -9,7 +9,9 @@ module Git.HashObject where
 
 import Common
 import Git
+import Git.Sha
 import Git.Command
+import Git.Types
 import qualified Utility.CoProcess as CoProcess
 
 type HashObjectHandle = CoProcess.CoProcessHandle
@@ -24,11 +26,23 @@ hashObjectStart = CoProcess.start "git" . toCommand . gitCommandLine
 hashObjectStop :: HashObjectHandle -> IO ()
 hashObjectStop = CoProcess.stop
 
-{- Injects a file into git, returning the shas of the objects. -}
+{- Injects a file into git, returning the Sha of the object. -}
 hashFile :: HashObjectHandle -> FilePath -> IO Sha
 hashFile h file = CoProcess.query h send receive
 	where
 		send to = do
 			fileEncoding to
 			hPutStrLn to file
-		receive from = Ref <$> hGetLine from
+		receive from = getSha "hash-object" $ hGetLine from
+
+{- Injects some content into git, returning its Sha. -}
+hashObject :: Repo -> ObjectType -> String -> IO Sha
+hashObject repo objtype content = getSha subcmd $ do
+	(h, s) <- pipeWriteRead (map Param params) content repo
+	length s `seq` do
+		forceSuccess h
+		reap -- XXX unsure why this is needed
+		return s
+	where
+		subcmd = "hash-object"
+		params = [subcmd, "-t", show objtype, "-w", "--stdin"]
diff --git a/Git/Types.hs b/Git/Types.hs
index deb14ebd48..64d418a041 100644
--- a/Git/Types.hs
+++ b/Git/Types.hs
@@ -48,3 +48,18 @@ instance Show Ref where
 type Branch = Ref
 type Sha = Ref
 type Tag = Ref
+
+{- Types of objects that can be stored in git. -}
+data ObjectType = BlobObject | CommitObject | TreeObject
+
+instance Show ObjectType where
+	show BlobObject = "blob"
+	show CommitObject = "commit"
+	show TreeObject = "tree"
+
+readObjectType :: String -> Maybe ObjectType
+readObjectType "blob" = Just BlobObject
+readObjectType "commit" = Just CommitObject
+readObjectType "tree" = Just TreeObject
+readObjectType _ = Nothing
+
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 9ff820dc91..822e6abbf0 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -10,7 +10,6 @@ module Git.UnionMerge (
 	merge_index
 ) where
 
-import System.Cmd.Utils
 import qualified Data.Text.Lazy as L
 import qualified Data.Text.Lazy.Encoding as L
 import qualified Data.Set as S
@@ -21,6 +20,8 @@ import Git.Sha
 import Git.CatFile
 import Git.Command
 import Git.UpdateIndex
+import Git.HashObject
+import Git.Types
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
@@ -72,7 +73,7 @@ mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> use sha
-	shas -> use =<< either return (hashObject repo . unlines) =<<
+	shas -> use =<< either return (hashObject repo BlobObject . unlines) =<<
 		calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
@@ -80,18 +81,6 @@ mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 			L.decodeUtf8 <$> catObject h s
 		use sha = return $ Just $ update_index_line sha file
 
-{- Injects some content into git, returning its Sha. -}
-hashObject :: Repo -> String -> IO Sha
-hashObject repo content = getSha subcmd $ do
-	(h, s) <- pipeWriteRead (map Param params) content repo
-	length s `seq` do
-		forceSuccess h
-		reap -- XXX unsure why this is needed
-		return s
-	where
-		subcmd = "hash-object"
-		params = [subcmd, "-w", "--stdin"]
-
 {- Calculates a union merge between a list of refs, with contents.
  -
  - When possible, reuses the content of an existing ref, rather than

From 1cfd2e1c0572acf13f75897af3f5921cf0950e75 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Wed, 6 Jun 2012 01:57:31 -0700
Subject: [PATCH 3631/8313] Remove INSTALL note about man pages.

---
 doc/install.mdwn | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 1f4b54f542..fe0522aa05 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -20,28 +20,6 @@ As a haskell package, git-annex can be installed using cabal. For example:
 The above downloads the latest release. Alternatively, you can [[download]]
 it yourself and [[manually_build_with_cabal|install/cabal]].
 
-### Man pages
-
-Cabal does not install the man pages.  Do:
-
-	cd /tmp
-	cabal unpack git-annex
-	cd git-annex*
-	make install-mans PREFIX=~/.cabal
-
-to install man pages under ~/.cabal/share/man (or $PREFIX/share/man
-generally).  If ~/.cabal/bin is on your PATH, and you're on a Debian
-derivative, you're probably set.
-
-On Debian systems, `manpath` will print the paths searched for manual
-pages.  The paths searched are inferred from your PATH.  Experiments
-indicate that, on Ubuntu, D/share/man will be searched for manual
-pages whenever D/bin is on your PATH.  So, having ~/.cabal/bin on your
-PATH is enough to make the above example work.  The `man manpath` does
-not document this, but see
-http://linux.derkeiler.com/Mailing-Lists/Debian/2003-08/0956.html for
-some other special cases.
-
 ## Installation by hand
 
 To build and use git-annex, you will need:

From 6f17b2fb39f389b01accee420548b54e22f41e0a Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Wed, 6 Jun 2012 02:35:32 -0700
Subject: [PATCH 3632/8313] WIP: Add man page installation to Setup.hs

This works with `cabal-dev install .`, but `cabal sdist` does not yet
include the man pages (tried adding a `make $(mans)` before `cabal
sdist` in `make sdist`, but no luck).

XXX: Need to go back and replace spaces with tabs.
---
 Setup.hs | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/Setup.hs b/Setup.hs
index c36d6e4fe1..c722077eb5 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -3,6 +3,9 @@
 import Distribution.Simple
 import Distribution.Simple.LocalBuildInfo
 import Distribution.Simple.Setup
+import Distribution.Simple.Utils (installOrdinaryFiles)
+import Distribution.PackageDescription (PackageDescription(..))
+import Distribution.Verbosity (Verbosity)
 import System.Cmd
 import System.FilePath
 
@@ -11,6 +14,7 @@ import qualified Build.Configure as Configure
 main = defaultMainWithHooks simpleUserHooks
 	{ preConf = configure
 	, instHook = install
+        , postInst = const installManpages
 	}
 
 configure _ _ = do
@@ -25,3 +29,23 @@ install pkg_descr lbi userhooks flags = do
 	where
 		installDirs = absoluteInstallDirs pkg_descr lbi $
 			fromFlag (copyDest defaultCopyFlags)
+
+-- See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages.
+--
+-- Based on pandoc's 'Setup.installManpages' and 'postInst' hook.
+-- Would be easier to just use 'rawSystem' as above.
+--
+-- XXX: fix tabs!
+installManpages :: InstallFlags -> PackageDescription -> LocalBuildInfo -> IO ()
+installManpages flags pkg lbi =
+  installOrdinaryFiles verbosity dstManDir manpages
+  where
+    srcManDir = ""
+    -- The 'NoCopyDest' means "don't add an additional path prefix".
+    -- The pandoc Setup.hs uses 'NoCopyDest' in the post install hook
+    -- and the 'CopyDest' from the copy flags in the post copy hook.
+    dstManDir = mandir (absoluteInstallDirs pkg lbi NoCopyDest)  "man1"
+    manpages  = zip (repeat srcManDir)
+                    [ "git-annex.1"
+                    , "git-annex-shell.1" ]
+    verbosity = fromFlag $ installVerbosity flags

From 455fca65bfb9ca4270fa7f89986d09ee62188d43 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 11:58:08 -0400
Subject: [PATCH 3633/8313] layout

---
 Types/Backend.hs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/Types/Backend.hs b/Types/Backend.hs
index 5abb0896dc..97f7cef907 100644
--- a/Types/Backend.hs
+++ b/Types/Backend.hs
@@ -18,14 +18,11 @@ data KeySource = KeySource
 	, contentLocation :: FilePath
 	}
 
-data BackendA a = Backend {
-	-- name of this backend
-	name :: String,
-	-- gets the key to use for a given content
-	getKey :: KeySource -> a (Maybe Key),
-	-- called during fsck to check a key, if the backend has its own checks
-	fsckKey :: Maybe (Key -> FilePath -> a Bool)
-}
+data BackendA a = Backend
+	{ name :: String
+	, getKey :: KeySource -> a (Maybe Key) 
+	, fsckKey :: Maybe (Key -> FilePath -> a Bool)
+	}
 
 instance Show (BackendA a) where
 	show backend = "Backend { name =\"" ++ name backend ++ "\" }"

From 723eb19bbf30d502cb6979655b8013de49ce0b5e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 13:07:30 -0400
Subject: [PATCH 3634/8313] split out utility functions

---
 Command/Add.hs | 52 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 2c671eea29..7fbfa6e7f9 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -50,8 +50,8 @@ start file = notBareRepo $ ifAnnexed file fixup add
  - to prevent it from being modified in between. It's hard linked into a
  - temporary location, and its writable bits are removed. It could still be
  - written to by a process that already has it open for writing. -}
-perform :: FilePath -> CommandPerform
-perform file = do
+lockDown :: FilePath -> Annex FilePath
+lockDown file = do
 	liftIO $ preventWrite file
 	tmp <- fromRepo gitAnnexTmpDir
 	createAnnexDirectory tmp
@@ -59,18 +59,27 @@ perform file = do
 	let tmpfile = tmp  "add" ++ show pid ++ "." ++ takeFileName file
 	nuke tmpfile
 	liftIO $ createLink file tmpfile
+	return tmpfile
+
+nuke :: FilePath -> Annex ()
+nuke file = liftIO $ whenM (doesFileExist file) $ removeFile file
+
+{- Moves the file into the annex. -}
+ingest :: FilePath -> Annex (Maybe Key)
+ingest file = do
+	tmpfile <- lockDown file
 	let source = KeySource { keyFilename = file, contentLocation = tmpfile }
 	backend <- chooseBackend file
 	genKey source backend >>= go tmpfile
 	where
-		go _ Nothing = stop
+		go _ Nothing = return Nothing
 		go tmpfile (Just (key, _)) = do
 			handle (undo file key) $ moveAnnex key tmpfile
 			nuke file
-			next $ cleanup file key True
+			return $ Just key
 
-nuke :: FilePath -> Annex ()
-nuke file = liftIO $ whenM (doesFileExist file) $ removeFile file
+perform :: FilePath -> CommandPerform
+perform file = maybe stop (\key -> next $ cleanup file key True) =<< ingest file
 
 {- On error, put the file back so it doesn't seem to have vanished.
  - This can be called before or after the symlink is in place. -}
@@ -88,21 +97,26 @@ undo file key e = do
 			src <- inRepo $ gitAnnexLocation key
 			liftIO $ moveFile src file
 
+{- Creates the symlink to the annexed content. -}
+link :: FilePath -> Key -> Bool -> Annex ()
+link file key hascontent = handle (undo file key) $ do
+	l <- calcGitLink file key
+	liftIO $ createSymbolicLink l file
+
+	when hascontent $ do
+		logStatus key InfoPresent
+	
+		-- touch the symlink to have the same mtime as the
+		-- file it points to
+		liftIO $ do
+			mtime <- modificationTime <$> getFileStatus file
+			touch file (TimeSpec mtime) False
+
+{- Note: Several other commands call this, and expect it to 
+ - create the symlink and add it. -}
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
 cleanup file key hascontent = do
-	handle (undo file key) $ do
-		link <- calcGitLink file key
-		liftIO $ createSymbolicLink link file
-
-		when hascontent $ do
-			logStatus key InfoPresent
-	
-			-- touch the symlink to have the same mtime as the
-			-- file it points to
-			liftIO $ do
-				mtime <- modificationTime <$> getFileStatus file
-				touch file (TimeSpec mtime) False
-
+	link file key hascontent
 	params <- ifM (Annex.getState Annex.force)
 		( return [Param "-f"]
 		, return []

From 993e6459a38817a9062aafae7552a668c2db7a31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 13:13:13 -0400
Subject: [PATCH 3635/8313] factor out nukeFile

---
 Annex/Content.hs      | 2 +-
 Command/Add.hs        | 9 +++------
 Command/DropUnused.hs | 2 +-
 Utility/Directory.hs  | 7 +++++++
 git-union-merge.hs    | 2 +-
 5 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 26b332e24c..232b43b2c3 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -168,7 +168,7 @@ withTmp :: Key -> (FilePath -> Annex a) -> Annex a
 withTmp key action = do
 	tmp <- prepTmp key
 	res <- action tmp
-	liftIO $ whenM (doesFileExist tmp) $ liftIO $ removeFile tmp
+	liftIO $ nukeFile tmp
 	return res
 
 {- Checks that there is disk space available to store a given key,
diff --git a/Command/Add.hs b/Command/Add.hs
index 7fbfa6e7f9..d83817d729 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -57,13 +57,10 @@ lockDown file = do
 	createAnnexDirectory tmp
 	pid <- liftIO getProcessID
 	let tmpfile = tmp  "add" ++ show pid ++ "." ++ takeFileName file
-	nuke tmpfile
+	liftIO $ nukeFile tmpfile
 	liftIO $ createLink file tmpfile
 	return tmpfile
 
-nuke :: FilePath -> Annex ()
-nuke file = liftIO $ whenM (doesFileExist file) $ removeFile file
-
 {- Moves the file into the annex. -}
 ingest :: FilePath -> Annex (Maybe Key)
 ingest file = do
@@ -75,7 +72,7 @@ ingest file = do
 		go _ Nothing = return Nothing
 		go tmpfile (Just (key, _)) = do
 			handle (undo file key) $ moveAnnex key tmpfile
-			nuke file
+			liftIO $ nukeFile file
 			return $ Just key
 
 perform :: FilePath -> CommandPerform
@@ -86,7 +83,7 @@ perform file = maybe stop (\key -> next $ cleanup file key True) =<< ingest file
 undo :: FilePath -> Key -> IOException -> Annex a
 undo file key e = do
 	whenM (inAnnex key) $ do
-		nuke file
+		liftIO $ nukeFile file
 		handle tryharder $ fromAnnex key file
 		logStatus key InfoMissing
 	throw e
diff --git a/Command/DropUnused.hs b/Command/DropUnused.hs
index a94c2873dd..597a4eec07 100644
--- a/Command/DropUnused.hs
+++ b/Command/DropUnused.hs
@@ -40,5 +40,5 @@ perform key = maybe droplocal dropremote =<< Remote.byName =<< from
 performOther :: (Key -> Git.Repo -> FilePath) -> Key -> CommandPerform
 performOther filespec key = do
 	f <- fromRepo $ filespec key
-	liftIO $ whenM (doesFileExist f) $ removeFile f
+	liftIO $ nukeFile f
 	next $ return True
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 5bfd49a9c1..52f2396d75 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -88,6 +88,13 @@ moveFile src dest = tryIO (rename src dest) >>= onrename
 				(Left _) -> return False
 				(Right s) -> return $ isDirectory s
 
+{- Removes a file, which may or may not exist.
+ -
+ - Note that an exception is thrown if the file exists but
+ - cannot be removed. -}
+nukeFile :: FilePath -> IO ()
+nukeFile file = whenM (doesFileExist file) $ removeFile file
+
 {- Runs an action in another directory. -}
 bracketCd :: FilePath -> IO a -> IO a
 bracketCd dir a = go =<< getCurrentDirectory
diff --git a/git-union-merge.hs b/git-union-merge.hs
index 2c2e7a46bd..0e4cd644ce 100644
--- a/git-union-merge.hs
+++ b/git-union-merge.hs
@@ -28,7 +28,7 @@ setup :: Git.Repo -> IO ()
 setup = cleanup -- idempotency
 
 cleanup :: Git.Repo -> IO ()
-cleanup g = whenM (doesFileExist $ tmpIndex g) $ removeFile $ tmpIndex g
+cleanup g = nukeFile $ tmpIndex g
 
 parseArgs :: IO [String]
 parseArgs = do

From 91db54076964979b6c50bd7efd0b895c4d416978 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 14:26:15 -0400
Subject: [PATCH 3636/8313] add support for staging other types of blobs, like
 symlinks, into the index

Also added a utility TopFilePath type, which could stand to be used more
widely.
---
 Git/FilePath.hs    | 34 ++++++++++++++++++++++++++++++++++
 Git/Types.hs       |  8 ++++++++
 Git/UnionMerge.hs  |  3 ++-
 Git/UpdateIndex.hs | 15 +++++++++------
 4 files changed, 53 insertions(+), 7 deletions(-)
 create mode 100644 Git/FilePath.hs

diff --git a/Git/FilePath.hs b/Git/FilePath.hs
new file mode 100644
index 0000000000..6344353d6f
--- /dev/null
+++ b/Git/FilePath.hs
@@ -0,0 +1,34 @@
+{- git FilePath library
+ -
+ - Different git commands use different types of FilePaths to refer to
+ - files in the repository. Some commands use paths relative to the
+ - top of the repository even when run in a subdirectory. Adding some
+ - types helps keep that straight.
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Git.FilePath (
+	TopFilePath,
+	getTopFilePath,
+	toTopFilePath,
+	asTopFilePath,
+) where
+
+import Common
+import Git
+
+{- A FilePath, relative to the top of the git repository. -}
+newtype TopFilePath = TopFilePath { getTopFilePath :: FilePath }
+
+{- The input FilePath can be absolute, or relative to the CWD. -}
+toTopFilePath :: FilePath -> Git.Repo -> IO TopFilePath
+toTopFilePath file repo = TopFilePath <$>
+	relPathDirToFile (repoPath repo) <$> absPath file
+
+{- The input FilePath must already be relative to the top of the git
+ - repository -}
+asTopFilePath :: FilePath -> TopFilePath
+asTopFilePath file = TopFilePath file
diff --git a/Git/Types.hs b/Git/Types.hs
index 64d418a041..1df6e343b8 100644
--- a/Git/Types.hs
+++ b/Git/Types.hs
@@ -63,3 +63,11 @@ readObjectType "commit" = Just CommitObject
 readObjectType "tree" = Just TreeObject
 readObjectType _ = Nothing
 
+{- Types of blobs. -}
+data BlobType = FileBlob | ExecutableBlob | SymlinkBlob
+
+{- Git uses magic numbers to denote the type of a blob. -}
+instance Show BlobType where
+	show FileBlob = "100644"
+	show ExecutableBlob = "100755"
+	show SymlinkBlob = "120000"
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index 822e6abbf0..f65b59c53a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -22,6 +22,7 @@ import Git.Command
 import Git.UpdateIndex
 import Git.HashObject
 import Git.Types
+import Git.FilePath
 
 {- Performs a union merge between two branches, staging it in the index.
  - Any previously staged changes in the index will be lost.
@@ -79,7 +80,7 @@ mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		getcontents s = map L.unpack . L.lines .
 			L.decodeUtf8 <$> catObject h s
-		use sha = return $ Just $ update_index_line sha file
+		use sha = return $ Just $ update_index_line sha FileBlob $ asTopFilePath file
 
 {- Calculates a union merge between a list of refs, with contents.
  -
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index 04bc4da5ba..8c003dd13b 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -8,15 +8,17 @@
 module Git.UpdateIndex (
 	Streamer,
 	stream_update_index,
+	ls_tree,
 	update_index_line,
-	ls_tree
 ) where
 
 import System.Cmd.Utils
 
 import Common
 import Git
+import Git.Types
 import Git.Command
+import Git.FilePath
 
 {- Streamers are passed a callback and should feed it lines in the form
  - read by update-index, and generated by ls-tree. -}
@@ -37,13 +39,14 @@ stream_update_index repo as = do
 			hPutStr h s
 			hPutStr h "\0"
 
-{- Generates a line suitable to be fed into update-index, to add
- - a given file with a given sha. -}
-update_index_line :: Sha -> FilePath -> String
-update_index_line sha file = "100644 blob " ++ show sha ++ "\t" ++ file
-
 {- Gets the current tree for a ref. -}
 ls_tree :: Ref -> Repo -> Streamer
 ls_tree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
+
+{- Generates a line suitable to be fed into update-index, to add
+ - a given file with a given sha. -}
+update_index_line :: Sha -> BlobType -> TopFilePath -> String
+update_index_line sha filetype file =
+	show filetype ++ " blob " ++ show sha ++ "\t" ++ getTopFilePath file

From b819f644ad00c5ad13ba5b249d3e127fd59d8694 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 14:29:10 -0400
Subject: [PATCH 3637/8313] close the git add race

There's a race adding a new file to the annex: The file is moved to the
annex and replaced with a symlink, and then we git add the symlink. If
someone comes along in the meantime and replaces the symlink with
something else, such as a new large file, we add that instead. Which could
be bad..

This race is fixed by avoiding using git add, instead the symlink is
directly staged into the index.

It would be nice to make `git annex add` use this same technique.
I have not done so yet because it currently runs git update-index once per
file, which would slow does `git annex add`. A future enhancement would be
to extend the Git.Queue to include the ability to run update-index with
a list of Streamers.
---
 Annex/Branch.hs  |  4 +++-
 Command/Add.hs   |  9 +++++---
 Command/Watch.hs | 54 ++++++++++++++++++++++++++++++++++--------------
 3 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index c8d0719b0b..1dacd5f32a 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -35,6 +35,8 @@ import qualified Git.Branch
 import qualified Git.UnionMerge
 import qualified Git.UpdateIndex
 import Git.HashObject
+import Git.Types
+import Git.FilePath
 import qualified Git.Index
 import Annex.CatFile
 import Annex.Perms
@@ -344,5 +346,5 @@ stageJournal = do
 			let path = dir  file
 			sha <- hashFile h path
 			_ <- streamer $ Git.UpdateIndex.update_index_line
-				sha (fileJournal file)
+				sha FileBlob (asTopFilePath $ fileJournal file)
 			removeFile path
diff --git a/Command/Add.hs b/Command/Add.hs
index d83817d729..ea0f850332 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -94,8 +94,9 @@ undo file key e = do
 			src <- inRepo $ gitAnnexLocation key
 			liftIO $ moveFile src file
 
-{- Creates the symlink to the annexed content. -}
-link :: FilePath -> Key -> Bool -> Annex ()
+{- Creates the symlink to the annexed content, and also returns the link's
+ - text. -}
+link :: FilePath -> Key -> Bool -> Annex FilePath
 link file key hascontent = handle (undo file key) $ do
 	l <- calcGitLink file key
 	liftIO $ createSymbolicLink l file
@@ -109,11 +110,13 @@ link file key hascontent = handle (undo file key) $ do
 			mtime <- modificationTime <$> getFileStatus file
 			touch file (TimeSpec mtime) False
 
+	return l
+
 {- Note: Several other commands call this, and expect it to 
  - create the symlink and add it. -}
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
 cleanup file key hascontent = do
-	link file key hascontent
+	_ <- link file key hascontent
 	params <- ifM (Annex.getState Annex.force)
 		( return [Param "-f"]
 		, return []
diff --git a/Command/Watch.hs b/Command/Watch.hs
index 15c862bec5..7b714ac186 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -14,9 +14,13 @@ import Command
 import Utility.Inotify
 import Utility.ThreadLock
 import qualified Annex
-import qualified Annex.Queue
-import qualified Command.Add as Add
+import qualified Command.Add
+import qualified Git
 import qualified Git.Command
+import qualified Git.UpdateIndex
+import Git.HashObject
+import Git.Types
+import Git.FilePath
 import qualified Backend
 import Annex.Content
 
@@ -39,14 +43,14 @@ start = notBareRepo $ do
 	mvar <- liftIO $ newMVar state
 	next $ next $ liftIO $ withINotify $ \i -> do
 		let hook a = Just $ runAnnex mvar a
-		watchDir i "." (not . gitdir)
+		watchDir i "." (not . pruned)
 			(hook onAdd) (hook onAddSymlink)
 			(hook onDel) (hook onDelDir)
 		putStrLn "(started)"
 		waitForTermination
 		return True
 	where
-		gitdir dir = takeFileName dir /= ".git"
+		pruned dir = takeFileName dir /= ".git"
 
 {- Runs a handler, inside the Annex monad.
  -
@@ -66,15 +70,20 @@ runAnnex mvar a f = do
 	where
 		go state = Annex.exec state $ a f
 
-{- Adding a file is the same as git-annex add.
- - The git queue is immediately flushed, so the file is added to git
- - now, rather than later (when it may have been already moved or deleted!) -}
+{- Adding a file is tricky; the file has to be replaced with a symlink
+ - but this is race prone, as the symlink could be changed immediately
+ - after creation. To avoid that race, git add is not used to stage the
+ - symlink. -}
 onAdd :: FilePath -> Annex ()
 onAdd file = do
-	void $ doCommand $ do
-		showStart "add" file
-		next $ Add.perform file
-	Annex.Queue.flush
+	showStart "add" file
+	Command.Add.ingest file >>= go
+	where
+		go Nothing = showEndFail
+		go (Just key) = do
+			link <- Command.Add.link file key True
+			inRepo $ stageSymlink file link
+			showEndOk
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
@@ -83,19 +92,20 @@ onAdd file = do
 onAddSymlink :: FilePath -> Annex ()
 onAddSymlink file = go =<< Backend.lookupFile file
 	where
-		go Nothing = addlink
+		go Nothing = addlink =<< liftIO (readSymbolicLink file)
 		go (Just (key, _)) = do
 			link <- calcGitLink file key
 			ifM ((==) link <$> liftIO (readSymbolicLink file))
-				( addlink
+				( addlink link
 				, do
 					liftIO $ removeFile file
 					liftIO $ createSymbolicLink link file
-					addlink
+					addlink link
 				)
-		addlink = inRepo $ Git.Command.run "add"
-			[Params "--force --", File file]
+		addlink link = inRepo $ stageSymlink file link
 
+{- The file could reappear at any time, so --cached is used, to only delete
+ - it from the index. -}
 onDel :: FilePath -> Annex ()
 onDel file = inRepo $ Git.Command.run "rm"
 	[Params "--quiet --cached --ignore-unmatch --", File file] 
@@ -105,3 +115,15 @@ onDel file = inRepo $ Git.Command.run "rm"
 onDelDir :: FilePath -> Annex ()
 onDelDir dir = inRepo $ Git.Command.run "rm"
 	[Params "--quiet -r --cached --ignore-unmatch --", File dir]
+
+{- Adds a symlink to the index, without ever accessing the actual symlink
+ - on disk. -}
+stageSymlink :: FilePath -> String -> Git.Repo -> IO ()
+stageSymlink file linktext repo = Git.UpdateIndex.stream_update_index repo [stage]
+	where
+		stage streamer = do
+			line <- Git.UpdateIndex.update_index_line
+				<$> (hashObject repo BlobObject linktext)
+				<*> pure SymlinkBlob
+				<*> toTopFilePath file repo
+			streamer line

From 899334223f4c46307af6c34ae103971b5f37ce8f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 14:33:12 -0400
Subject: [PATCH 3638/8313] fixed 2 races! Only 1 serious race to go!

---
 doc/design/assistant/inotify.mdwn | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index e7c61c68b1..079941f597 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -82,16 +82,14 @@ Many races need to be dealt with by this code. Here are some of them.
 * File is added and then replaced with another file before the annex add
   stages the symlink in git.
 
-  **Currently unfixed**; `git add` will be run on the new file, which is
-  not at all good when it's big. Could be dealt with by using `git
-  update-index` to manually put the symlink into the index without git
+  Now fixed; `git annex watch` avoids running `git add` because of this
+  race. Instead, it stages symlinks directly into the index, without
   looking at what's currently on disk.
 
 * Link is moved, fixed link is written by fix event, but then that is
   removed by the user and replaced with a file before the event finishes.
 
-  **Currently unfixed**: `git add` will be run on the file. Basically same
-  effect as previous race above.
+  Now fixed; same fix as previous race above.
 
 * File is removed and then re-added before the removal event starts.
 

From 81d4991fa3b9b9a303d37eef407cc7dcdc997a94 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 15:13:18 -0400
Subject: [PATCH 3639/8313] thoughts

---
 doc/design/assistant/inotify.mdwn | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 079941f597..28b3130e33 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -67,10 +67,18 @@ Many races need to be dealt with by this code. Here are some of them.
   **Currently unfixed**; This changes content in the annex, and fsck will
   later catch the inconsistency.
 
-  Possible fixes: Somehow track or detect if a file is open for write
-  by any processes. Or, when possible, making a copy on write copy
-  before adding the file would avoid this.  Or, as a last resort, make
-  an expensive copy of the file and add that.
+  Possible fixes: 
+
+  * Somehow track or detect if a file is open for write by any processes.
+  * Or, when possible, making a copy on write copy before adding the file
+    would avoid this.
+  * Or, as a last resort, make an expensive copy of the file and add that.
+  * Tracking file opens and closes with inotify could tell if any other
+    processes have the file open. But there are problems.. It doesn't
+    seem to differentiate between files opened for read and for write.
+    And there would still be a race after the last close and before it's
+    injected into the annex, where it could be opened for write again.
+    Would need to detect that and undo the annex injection or something.
 
 * File is added and then replaced with another file before the annex add
   makes its symlink.

From e17bc40c312ee04019bdd65cf992e0caa1819df1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 15:24:51 -0400
Subject: [PATCH 3640/8313] update

---
 doc/design/assistant/inotify.mdwn | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 28b3130e33..b963597c83 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -22,9 +22,7 @@ really useful, it needs to:
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in)
 - tunable delays before adding new files, etc
-- Coleasce related add/rm events. See commit
-  cbdaccd44aa8f0ca30afba23fc06dd244c242075 for some details of the problems
-  with doing this.
+- coleasce related add/rm events for speed and less disk IO
 - don't annex `.gitignore` and `.gitattributes` files, but do auto-stage
   changes to them
 - configurable option to only annex files meeting certian size or

From db8effb8f3e861c61bc4c640d712688a8ed342e1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 15:50:12 -0400
Subject: [PATCH 3641/8313] ignore .gitignore and .gitattributes

---
 Command/Watch.hs   |  7 +++++--
 Utility/Inotify.hs | 20 +++++++++-----------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 7b714ac186..d6f77b6ae0 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -43,14 +43,17 @@ start = notBareRepo $ do
 	mvar <- liftIO $ newMVar state
 	next $ next $ liftIO $ withINotify $ \i -> do
 		let hook a = Just $ runAnnex mvar a
-		watchDir i "." (not . pruned)
+		watchDir i "." (ignored . takeFileName)
 			(hook onAdd) (hook onAddSymlink)
 			(hook onDel) (hook onDelDir)
 		putStrLn "(started)"
 		waitForTermination
 		return True
 	where
-		pruned dir = takeFileName dir /= ".git"
+		ignored ".git" = True
+		ignored ".gitignore" = True
+		ignored ".gitattributes" = True
+		ignored _ = False
 
 {- Runs a handler, inside the Annex monad.
  -
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 3c69a7ee2e..1896a2a268 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -48,15 +48,13 @@ type Hook = Maybe (FilePath -> IO ())
  - So this will fail if there are too many subdirectories.
  -}
 watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook -> Hook -> IO ()
-watchDir i dir ignored add addsymlink del deldir
-	| ignored dir = noop
-	| otherwise = do
-		lock <- newLock
-		void $ addWatch i watchevents dir $ \event ->
-			withLock lock (void $ go event)
-		withLock lock $
-			mapM_ walk =<< filter (not . dirCruft) <$>
-				getDirectoryContents dir
+watchDir i dir ignored add addsymlink del deldir = unless (ignored dir) $ do
+	lock <- newLock
+	void $ addWatch i watchevents dir $ \event ->
+		withLock lock (void $ go event)
+	withLock lock $
+		mapM_ walk =<< filter (not . dirCruft) <$>
+			getDirectoryContents dir
 	where
 		recurse d = watchDir i d ignored add addsymlink del deldir
 
@@ -71,7 +69,7 @@ watchDir i dir ignored add addsymlink del deldir
 			| isJust del || isJust deldir = [MoveOut, Delete]
 			| otherwise = []
 
-		walk f = do
+		walk f = unless (ignored f) $ do
 			let fullf = indir f
 			r <- catchMaybeIO $ getSymbolicLinkStatus fullf
 			case r of
@@ -112,7 +110,7 @@ watchDir i dir ignored add addsymlink del deldir
 				guarded = unlessM (filetype (const True) f)
 		go _ = noop
 
-		Just a <@> f = a $ indir f
+		Just a <@> f = unless (ignored f) $ a $ indir f
 		Nothing <@> _ = noop
 
 		indir f = dir  f

From 8aa194bf806c4620f0794ff7a13084e328061cc8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 15:51:34 -0400
Subject: [PATCH 3642/8313] update

---
 doc/design/assistant/inotify.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index b963597c83..0f4e8d28db 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -23,10 +23,10 @@ really useful, it needs to:
   lots of changes are coming in)
 - tunable delays before adding new files, etc
 - coleasce related add/rm events for speed and less disk IO
-- don't annex `.gitignore` and `.gitattributes` files, but do auto-stage
-  changes to them
+- don't annex `.gitignore` and `.gitattributes` files **done**
 - configurable option to only annex files meeting certian size or
   filename criteria
+- option to check files not meeting annex criteria into git directly
 - honor .gitignore, not adding files it excludes (difficult, probably
   needs my own .gitignore parser to avoid excessive running of git commands
   to check for ignored files)

From c5b11561f0110c70454b6123ab64ac044c81a5c3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 16:50:28 -0400
Subject: [PATCH 3643/8313] handle running out of watch descriptors

---
 Command/Watch.hs   | 27 +++++++++++++++++++++++++++
 Utility/Inotify.hs | 35 +++++++++++++++++++++++++----------
 2 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index d6f77b6ae0..dcd411d43a 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -44,6 +44,7 @@ start = notBareRepo $ do
 	next $ next $ liftIO $ withINotify $ \i -> do
 		let hook a = Just $ runAnnex mvar a
 		watchDir i "." (ignored . takeFileName)
+			(hook onTooMany)
 			(hook onAdd) (hook onAddSymlink)
 			(hook onDel) (hook onDelDir)
 		putStrLn "(started)"
@@ -119,6 +120,32 @@ onDelDir :: FilePath -> Annex ()
 onDelDir dir = inRepo $ Git.Command.run "rm"
 	[Params "--quiet -r --cached --ignore-unmatch --", File dir]
 
+{- There are too many directories for inotify to watch them all. -}
+onTooMany :: FilePath -> Annex ()
+onTooMany dir = do
+	sysctlval <- liftIO $ runsysctl [Param maxwatches]
+	warning $ unlines $
+		basewarning : maybe withoutsysctl withsysctl sysctlval
+	where
+		maxwatches = "fs.inotify.max_user_watches"
+		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"
+		withoutsysctl = ["Increase the value in /proc/sys/fs/inotify/max_user_watches"]
+		withsysctl n = let new = n * 10 in
+			[ "Increase the limit by running:"
+			, "  echo " ++ maxwatches ++ "=" ++ show new ++
+			  " | sudo tee -a /etc/sysctl.conf; sudo sysctl -p"
+			]
+		runsysctl ps = do
+			v <- catchMaybeIO $ hPipeFrom "sysctl" $ toCommand ps
+			case v of
+				Nothing -> return Nothing
+				Just (pid, h) -> do
+					val <- parsesysctl <$> liftIO (hGetContentsStrict h)
+					void $ getProcessStatus True False $ processID pid
+					return val
+		parsesysctl :: String -> Maybe Integer
+		parsesysctl s = readish =<< lastMaybe (words s)
+
 {- Adds a symlink to the index, without ever accessing the actual symlink
  - on disk. -}
 stageSymlink :: FilePath -> String -> Git.Repo -> IO ()
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 1896a2a268..b74fbd18d5 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -14,6 +14,8 @@ import Utility.ThreadLock
 
 import System.INotify
 import qualified System.Posix.Files as Files
+import System.IO.Error
+import Control.Exception (throw)
 
 type Hook = Maybe (FilePath -> IO ())
 
@@ -45,18 +47,22 @@ type Hook = Maybe (FilePath -> IO ())
  -
  - Note: inotify has a limit to the number of watches allowed,
  - /proc/sys/fs/inotify/max_user_watches (default 8192).
- - So this will fail if there are too many subdirectories.
+ - So this will fail if there are too many subdirectories. The
+ - toomany hook is called when this happens.
  -}
-watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook -> Hook -> IO ()
-watchDir i dir ignored add addsymlink del deldir = unless (ignored dir) $ do
-	lock <- newLock
-	void $ addWatch i watchevents dir $ \event ->
-		withLock lock (void $ go event)
-	withLock lock $
-		mapM_ walk =<< filter (not . dirCruft) <$>
-			getDirectoryContents dir
+watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook -> Hook -> Hook -> IO ()
+watchDir i dir ignored toomany add addsymlink del deldir
+	| ignored dir = noop
+	| otherwise = do
+		lock <- newLock
+		let handler event = withLock lock (void $ go event)
+		void (addWatch i watchevents dir handler)
+			`catchIO` failedaddwatch
+		withLock lock $
+			mapM_ walk =<< filter (not . dirCruft) <$>
+				getDirectoryContents dir
 	where
-		recurse d = watchDir i d ignored add addsymlink del deldir
+		recurse d = watchDir i d ignored toomany add addsymlink del deldir
 
 		-- Select only inotify events required by the enabled
 		-- hooks, but always include Create so new directories can
@@ -116,3 +122,12 @@ watchDir i dir ignored add addsymlink del deldir = unless (ignored dir) $ do
 		indir f = dir  f
 
 		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
+
+		-- Inotify fails when there are too many watches with a
+		-- disk full error.
+		failedaddwatch e
+			| isFullError e =
+				case toomany of
+					Nothing -> throw e
+					Just hook -> hook dir
+			| otherwise = throw e

From a79aebbe2a15a05f7475b08d98279bb88ef07305 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 16:54:39 -0400
Subject: [PATCH 3644/8313] update

---
 doc/design/assistant/inotify.mdwn | 5 +++--
 doc/design/assistant/webapp.mdwn  | 2 ++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 0f4e8d28db..ab88210b2b 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -19,6 +19,9 @@ really useful, it needs to:
 - notice deleted files and stage the deletion
   (tricky; there's a race with add since it replaces the file with a symlink..)
   **done**
+- Gracefully handle when the default limit of 8192 inotified directories
+  is exceeded. This can be tuned by root, so help the user fix it.
+  **done**
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in)
 - tunable delays before adding new files, etc
@@ -32,8 +35,6 @@ really useful, it needs to:
   to check for ignored files)
 - Possibly, when a directory is moved out of the annex location,
   unannex its contents.
-- Gracefully handle when the default limit of 8192 inotified directories
-  is exceeded. This can be tuned by root, so help the user fix it.
 - Support OSes other than Linux; it only uses inotify currently.
   OSX and FreeBSD use the same mechanism, and there is a Haskell interface
   for it,
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index abf7b38c94..598c1ff3a4 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -23,6 +23,8 @@ The webapp is a web server that displays a shiny interface.
 
 * there could be a UI to export a file, which would make it be served up
   over http by the web app
+* Display any relevant warning messages. One is the `inotify max_user_watches`
+  exceeded message.
 
 ## implementation
 

From baf9c7102ec58b3b31ef9d6a9424cf5e749bcc04 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 16:56:48 -0400
Subject: [PATCH 3645/8313] blog for the day

---
 .../assistant/blog/day_3__more_races.mdwn     | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_3__more_races.mdwn

diff --git a/doc/design/assistant/blog/day_3__more_races.mdwn b/doc/design/assistant/blog/day_3__more_races.mdwn
new file mode 100644
index 0000000000..9c11828420
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races.mdwn
@@ -0,0 +1,26 @@
+Today I worked on the race conditions, and fixed two of them. Both
+were fixed by avoiding using `git add`, which looks at the files currently
+on disk. Instead, `git annex watch` injects symlinks directly into git's
+index, using `git update-index`.
+
+There is one bad race condition remaining. If multiple processes have a
+file open for write, one can close it, and it will be added to the annex.
+But then the other can still write to it.
+
+----
+
+Getting away from race conditions for a while, I made `git annex watch`
+not annex `.gitignore` and `.gitattributes` files.
+
+And, I made it handle running out of inotify descriptors. By default,
+`/proc/sys/fs/inotify/max_user_watches` is 8192, and that's how many
+directories inotify can watch. Now when it needs more, it will print
+a nice message showing how to increase it with `sysctl`.
+
+FWIW, DropBox also uses inotify and has the same limit. It seems to not
+tell the user how to fix it when it goes over. Here's what `git annex
+watch` will say:
+
+	Too many directories to watch! (Not watching ./dir4299)
+	Increase the limit by running:
+	  echo fs.inotify.max_user_watches=81920 | sudo tee -a /etc/sysctl.conf; sudo sysctl -p

From 9ee59f62d56ae8bdbe6a4600a6274108d89b0187 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkmtR6oVColYKoU0SjBORLDGrwR10G-mKo"
 
Date: Wed, 6 Jun 2012 22:03:29 +0000
Subject: [PATCH 3646/8313] Added a comment: Dropbox Inotify

---
 .../comment_1_d6015338f602b574a3805de5481fc45e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment

diff --git a/doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment b/doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment
new file mode 100644
index 0000000000..2d330f3327
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_1_d6015338f602b574a3805de5481fc45e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkmtR6oVColYKoU0SjBORLDGrwR10G-mKo"
+ nickname="Jo-Herman"
+ subject="Dropbox Inotify"
+ date="2012-06-06T22:03:29Z"
+ content="""
+Actually, Dropbox giver you a warning via libnotify inotify. It tends to go away too quickly to properly read though, much less actually copy down the command...
+"""]]

From 54f5462ad4a6f3e95a76d008f5eabbff988e7f9f Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Wed, 6 Jun 2012 23:25:57 +0000
Subject: [PATCH 3647/8313] Added a comment

---
 .../comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment

diff --git a/doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment b/doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment
new file mode 100644
index 0000000000..523e6d85ff
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_2_4d6b23fc6442e0ee0303523cb69d0fba._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.8.36"
+ subject="comment 2"
+ date="2012-06-06T23:25:57Z"
+ content="""
+When I work on the [[webapp]], I'm planning to make it display this warning, and any other similar warning messages that might come up.
+"""]]

From b8f85f7a82009f3480abaedae510b4db1c3b3f3a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 22:49:32 -0400
Subject: [PATCH 3648/8313] build watch on non-linux, just don't do anything

---
 Command/Watch.hs   | 11 ++++++++++-
 Utility/Inotify.hs |  2 --
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index dcd411d43a..4786311002 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -5,6 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE CPP #-}
 {-# LANGUAGE BangPatterns #-}
 
 module Command.Watch where
@@ -25,9 +26,12 @@ import qualified Backend
 import Annex.Content
 
 import Control.Exception as E
-import System.INotify
 import Control.Concurrent.MVar
 
+#if defined linux_HOST_OS
+import System.INotify
+#endif
+
 def :: [Command]
 def = [command "watch" paramPaths seek "watch for changes"]
 
@@ -35,6 +39,7 @@ seek :: [CommandSeek]
 seek = [withNothing start]
 
 start :: CommandStart
+#if defined linux_HOST_OS
 start = notBareRepo $ do
 	showStart "watch" "."
 	showAction "scanning"
@@ -56,6 +61,10 @@ start = notBareRepo $ do
 		ignored ".gitattributes" = True
 		ignored _ = False
 
+#else
+start = error "watch mode is so far only available on Linux"
+#endif
+
 {- Runs a handler, inside the Annex monad.
  -
  - Exceptions by the handlers are ignored, otherwise a whole watcher
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index b74fbd18d5..8504b21d03 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -5,8 +5,6 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE CPP #-}
-
 module Utility.Inotify where
 
 import Common hiding (isDirectory)

From b8ae9528ab6d18f8b16c71e455d9f9077fc6e97b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 23:20:09 -0400
Subject: [PATCH 3649/8313] refactor

---
 Command/Watch.hs   | 40 +++++++-----------------
 Utility/Inotify.hs | 77 ++++++++++++++++++++++++++++++++++------------
 2 files changed, 68 insertions(+), 49 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 4786311002..b812691bf7 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -48,10 +48,14 @@ start = notBareRepo $ do
 	mvar <- liftIO $ newMVar state
 	next $ next $ liftIO $ withINotify $ \i -> do
 		let hook a = Just $ runAnnex mvar a
-		watchDir i "." (ignored . takeFileName)
-			(hook onTooMany)
-			(hook onAdd) (hook onAddSymlink)
-			(hook onDel) (hook onDelDir)
+		let hooks = WatchHooks
+			{ addHook = hook onAdd
+			, delHook = hook onDel
+			, addSymlinkHook = hook onAddSymlink
+			, delDirHook = hook onDelDir
+			, errHook = hook onErr
+			}
+		watchDir i "." (ignored . takeFileName) hooks
 		putStrLn "(started)"
 		waitForTermination
 		return True
@@ -129,31 +133,9 @@ onDelDir :: FilePath -> Annex ()
 onDelDir dir = inRepo $ Git.Command.run "rm"
 	[Params "--quiet -r --cached --ignore-unmatch --", File dir]
 
-{- There are too many directories for inotify to watch them all. -}
-onTooMany :: FilePath -> Annex ()
-onTooMany dir = do
-	sysctlval <- liftIO $ runsysctl [Param maxwatches]
-	warning $ unlines $
-		basewarning : maybe withoutsysctl withsysctl sysctlval
-	where
-		maxwatches = "fs.inotify.max_user_watches"
-		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"
-		withoutsysctl = ["Increase the value in /proc/sys/fs/inotify/max_user_watches"]
-		withsysctl n = let new = n * 10 in
-			[ "Increase the limit by running:"
-			, "  echo " ++ maxwatches ++ "=" ++ show new ++
-			  " | sudo tee -a /etc/sysctl.conf; sudo sysctl -p"
-			]
-		runsysctl ps = do
-			v <- catchMaybeIO $ hPipeFrom "sysctl" $ toCommand ps
-			case v of
-				Nothing -> return Nothing
-				Just (pid, h) -> do
-					val <- parsesysctl <$> liftIO (hGetContentsStrict h)
-					void $ getProcessStatus True False $ processID pid
-					return val
-		parsesysctl :: String -> Maybe Integer
-		parsesysctl s = readish =<< lastMaybe (words s)
+{- Called when there's an error with inotify. -}
+onErr :: String -> Annex ()
+onErr = warning
 
 {- Adds a symlink to the index, without ever accessing the actual symlink
  - on disk. -}
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 8504b21d03..c6faddadba 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -15,7 +15,15 @@ import qualified System.Posix.Files as Files
 import System.IO.Error
 import Control.Exception (throw)
 
-type Hook = Maybe (FilePath -> IO ())
+type Hook a = Maybe (a -> IO ())
+
+data WatchHooks = WatchHooks
+	{ addHook :: Hook FilePath
+	, addSymlinkHook :: Hook FilePath
+	, delHook :: Hook FilePath
+	, delDirHook :: Hook FilePath
+	, errHook :: Hook String -- error message
+	}
 
 {- Watches for changes to files in a directory, and all its subdirectories
  - that are not ignored, using inotify. This function returns after
@@ -46,10 +54,10 @@ type Hook = Maybe (FilePath -> IO ())
  - Note: inotify has a limit to the number of watches allowed,
  - /proc/sys/fs/inotify/max_user_watches (default 8192).
  - So this will fail if there are too many subdirectories. The
- - toomany hook is called when this happens.
+ - errHook is called when this happens.
  -}
-watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> Hook -> Hook -> Hook -> Hook -> Hook -> IO ()
-watchDir i dir ignored toomany add addsymlink del deldir
+watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> WatchHooks -> IO ()
+watchDir i dir ignored hooks
 	| ignored dir = noop
 	| otherwise = do
 		lock <- newLock
@@ -60,17 +68,17 @@ watchDir i dir ignored toomany add addsymlink del deldir
 			mapM_ walk =<< filter (not . dirCruft) <$>
 				getDirectoryContents dir
 	where
-		recurse d = watchDir i d ignored toomany add addsymlink del deldir
+		recurse d = watchDir i d ignored hooks
 
 		-- Select only inotify events required by the enabled
 		-- hooks, but always include Create so new directories can
 		-- be walked.
 		watchevents = Create : addevents ++ delevents
 		addevents
-			| isJust add || isJust addsymlink = [MoveIn, CloseWrite]
+			| hashook addHook || hashook addSymlinkHook = [MoveIn, CloseWrite]
 			| otherwise = []
 		delevents
-			| isJust del || isJust deldir = [MoveOut, Delete]
+			| hashook delHook || hashook delDirHook = [MoveOut, Delete]
 			| otherwise = []
 
 		walk f = unless (ignored f) $ do
@@ -80,8 +88,8 @@ watchDir i dir ignored toomany add addsymlink del deldir
 				Nothing -> return ()
 				Just s
 					| Files.isDirectory s -> recurse fullf
-					| Files.isSymbolicLink s -> addsymlink <@> f
-					| Files.isRegularFile s -> add <@> f
+					| Files.isSymbolicLink s -> addSymlinkHook <@> f
+					| Files.isRegularFile s -> addHook <@> f
 					| otherwise -> return ()
 
 		-- Ignore creation events for regular files, which won't be
@@ -89,33 +97,36 @@ watchDir i dir ignored toomany add addsymlink del deldir
 		-- directories and symlinks.
 		go (Created { isDirectory = isd, filePath = f })
 			| isd = recurse $ indir f
-			| isJust addsymlink =
+			| hashook addSymlinkHook =
 				whenM (filetype Files.isSymbolicLink f) $
-					addsymlink <@> f
+					addSymlinkHook <@> f
 			| otherwise = noop
 		-- Closing a file is assumed to mean it's done being written.
 		go (Closed { isDirectory = False, maybeFilePath = Just f }) =
 			whenM (filetype Files.isRegularFile f) $
-				add <@> f
+				addHook <@> f
 		-- When a file or directory is moved in, walk it to add new
 		-- stuff.
 		go (MovedIn { filePath = f }) = walk f
 		go (MovedOut { isDirectory = isd, filePath = f })
-			| isd = deldir <@> f
-			| otherwise = del <@> f
+			| isd = delDirHook <@> f
+			| otherwise = delHook <@> f
 		-- Verify that the deleted item really doesn't exist,
 		-- since there can be spurious deletion events for items
 		-- in a directory that has been moved out, but is still
 		-- being watched.
 		go (Deleted { isDirectory = isd, filePath = f })
-			| isd = guarded $ deldir <@> f
-			| otherwise = guarded $ del <@> f
+			| isd = guarded $ delDirHook <@> f
+			| otherwise = guarded $ delHook <@> f
 			where
 				guarded = unlessM (filetype (const True) f)
 		go _ = noop
 
-		Just a <@> f = unless (ignored f) $ a $ indir f
-		Nothing <@> _ = noop
+		hashook h = isJust $ h hooks
+
+		h <@> f
+			| ignored f = noop
+			| otherwise = maybe noop (\a -> a $ indir f) (h hooks)
 
 		indir f = dir  f
 
@@ -125,7 +136,33 @@ watchDir i dir ignored toomany add addsymlink del deldir
 		-- disk full error.
 		failedaddwatch e
 			| isFullError e =
-				case toomany of
+				case errHook hooks of
 					Nothing -> throw e
-					Just hook -> hook dir
+					Just hook -> tooManyWatches hook dir
 			| otherwise = throw e
+
+tooManyWatches :: (String -> IO ()) -> FilePath -> IO ()
+tooManyWatches hook dir = do
+	sysctlval <- querySysctl [Param maxwatches] :: IO (Maybe Integer)
+	hook $ unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval
+	where
+		maxwatches = "fs.inotify.max_user_watches"
+		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"
+		withoutsysctl = ["Increase the value in /proc/sys/fs/inotify/max_user_watches"]
+		withsysctl n = let new = n * 10 in
+			[ "Increase the limit by running:"
+			, "  echo " ++ maxwatches ++ "=" ++ show new ++
+			  " | sudo tee -a /etc/sysctl.conf; sudo sysctl -p"
+			]
+
+querySysctl :: Read a => [CommandParam] -> IO (Maybe a)
+querySysctl ps = do
+	v <- catchMaybeIO $ hPipeFrom "sysctl" $ toCommand ps
+	case v of
+		Nothing -> return Nothing
+		Just (pid, h) -> do
+			val <- parsesysctl <$> hGetContentsStrict h
+			void $ getProcessStatus True False $ processID pid
+			return val
+	where
+		parsesysctl s = readish =<< lastMaybe (words s)

From c56812980c2676a9b78f57c3c5259985805df3cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 23:27:20 -0400
Subject: [PATCH 3650/8313] document watch

---
 debian/changelog   | 3 +++
 doc/git-annex.mdwn | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 9a010327df..05919cb812 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,8 @@
 git-annex (3.20120606) UNRELEASED; urgency=low
 
+  * watch: New subcommand, which uses inotify to watch for changes to
+    files and automatically annexes new files, etc, so you don't need
+    to manually run git commands when manipulating files.
   * add: Prevent (most) modifications from being made to a file while it
     is being added to the annex.
 
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c7de59cd2a..c1d8015ab7 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -169,6 +169,13 @@ subdirectories).
 
 	git annex import /media/camera/DCIM/
 
+* watch
+
+  Watches for changes to files in the current directory and its subdirectories,
+  and takes care of automatically adding new files, as well as dealing with
+  deleted, copied, and moved files. Run this in the background, and you
+  no longer need to manually run git commands when manipulating your files.
+
 # REPOSITORY SETUP COMMANDS
 
 * init [description]

From d5de27ff40544bfb4d526eebce5e698a5efe35d1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 6 Jun 2012 23:30:38 -0400
Subject: [PATCH 3651/8313] tweak

---
 Command/Watch.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index b812691bf7..046fca7d14 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -59,16 +59,16 @@ start = notBareRepo $ do
 		putStrLn "(started)"
 		waitForTermination
 		return True
-	where
-		ignored ".git" = True
-		ignored ".gitignore" = True
-		ignored ".gitattributes" = True
-		ignored _ = False
-
 #else
 start = error "watch mode is so far only available on Linux"
 #endif
 
+ignored :: FilePath -> Bool
+ignored ".git" = True
+ignored ".gitignore" = True
+ignored ".gitattributes" = True
+ignored _ = False
+
 {- Runs a handler, inside the Annex monad.
  -
  - Exceptions by the handlers are ignored, otherwise a whole watcher

From c7efb2888cb612ea7635d8840d22ab1b38a8cb80 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
 
Date: Thu, 7 Jun 2012 03:43:19 +0000
Subject: [PATCH 3652/8313] Added a comment: Wording

---
 ...ent_3_03f5b2344c2a47dea60086f217d60f9b._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment

diff --git a/doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment b/doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment
new file mode 100644
index 0000000000..92f5dcbd62
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_3_03f5b2344c2a47dea60086f217d60f9b._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBJ6Dv1glxzzi4qIzGFNa6F-mfHIvv9Ck"
+ nickname="Jim"
+ subject="Wording"
+ date="2012-06-07T03:43:19Z"
+ content="""
+For the unfamiliar, it's hard to tell if a command like that would persist.  I'd suggest being as clear as possible, e.g.:
+
+    Increase the limit for now by running:
+      sudo sysctl fs.inotify.max_user_watches=81920
+    Increase the limit now and automatically at every boot by running:
+      echo fs.inotify.max_user_watches=81920 | sudo tee -a /etc/sysctl.conf; sudo sysctl -p
+
+"""]]

From 5d4e09199c757a34d0a9bf9b073dd13481b88cc1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 00:47:09 -0400
Subject: [PATCH 3653/8313] update message based on user feedback

---
 Utility/Inotify.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index c6faddadba..7329b51229 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -150,9 +150,11 @@ tooManyWatches hook dir = do
 		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"
 		withoutsysctl = ["Increase the value in /proc/sys/fs/inotify/max_user_watches"]
 		withsysctl n = let new = n * 10 in
-			[ "Increase the limit by running:"
+			[ "Increase the limit permanently by running:"
 			, "  echo " ++ maxwatches ++ "=" ++ show new ++
 			  " | sudo tee -a /etc/sysctl.conf; sudo sysctl -p"
+			, "Or temporarily by running:"
+			, "  sudo sysctl -w " ++ maxwatches ++ "=" ++ show new
 			]
 
 querySysctl :: Read a => [CommandParam] -> IO (Maybe a)

From 2478aca3c531d6d60fed87d9aa5eb42f6dbe6517 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 7 Jun 2012 04:48:15 +0000
Subject: [PATCH 3654/8313] Added a comment

---
 .../comment_4_860e90e989ec022100001c65e353a91e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment

diff --git a/doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment b/doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment
new file mode 100644
index 0000000000..05b601eafe
--- /dev/null
+++ b/doc/design/assistant/blog/day_3__more_races/comment_4_860e90e989ec022100001c65e353a91e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.252.8.36"
+ subject="comment 4"
+ date="2012-06-07T04:48:15Z"
+ content="""
+Good thought Jim. I've done something like that.
+"""]]

From 8408a91082f440e139884117b2698bb8e0bd3fe9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 10:37:06 -0400
Subject: [PATCH 3655/8313] fixie

---
 doc/tips/using_box.com_as_a_special_remote.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/using_box.com_as_a_special_remote.mdwn b/doc/tips/using_box.com_as_a_special_remote.mdwn
index 917c7a93bd..cafbc033c4 100644
--- a/doc/tips/using_box.com_as_a_special_remote.mdwn
+++ b/doc/tips/using_box.com_as_a_special_remote.mdwn
@@ -24,7 +24,7 @@ With a little setup, git-annex can use Box as a
 * Create `~/.davfs2/davfs2.conf` with some important settings:
 
         mkdir ~/.davfs2/
-        echo use_locks 0 >> ~/.davfs2/davfs2.conf
+        echo use_locks 0 > ~/.davfs2/davfs2.conf
         echo cache_size 1 >> ~/.davfs2/davfs2.conf
         echo delay_upload 0 >> ~/.davfs2/davfs2.conf
 

From 4d1c114e4d27ae339f6fb408d398945fa68c2435 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 11:16:48 -0400
Subject: [PATCH 3656/8313] initremote: Automatically describe a remote when
 creating it.

This ensures that all special remotes show up in git annex status.
Before, a special remote that was not manually described, and was not
a current git remote, did not show up there, although initremote did list
it.
---
 Command/InitRemote.hs | 2 ++
 Remote.hs             | 7 ++-----
 debian/changelog      | 1 +
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index 698d604552..a78505a197 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -15,6 +15,7 @@ import qualified Remote
 import qualified Logs.Remote
 import qualified Types.Remote as R
 import Annex.UUID
+import Logs.UUID
 
 def :: [Command]
 def = [command "initremote"
@@ -60,6 +61,7 @@ findByName name = do
 	where
 		generate = do
 			uuid <- liftIO genUUID
+			describeUUID uuid name
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
diff --git a/Remote.hs b/Remote.hs
index e9e66990c5..839c6ddb09 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -54,9 +54,9 @@ remoteMap :: (Remote -> a) -> Annex (M.Map UUID a)
 remoteMap c = M.fromList . map (\r -> (uuid r, c r)) .
 	filter (\r -> uuid r /= NoUUID) <$> remoteList
 
-{- Map of UUIDs and their descriptions.
+{- Map of UUIDs of remotes and their descriptions.
  - The names of Remotes are added to suppliment any description that has
- - been set for a repository. -}
+ - been set for a repository.  -}
 uuidDescriptions :: Annex (M.Map UUID String)
 uuidDescriptions = M.unionWith addName <$> uuidMap <*> remoteMap name
 
@@ -101,9 +101,6 @@ nameToUUID n = byName' n >>= go
 		double (a, _) = (a, a)
 
 {- Pretty-prints a list of UUIDs of remotes, for human display.
- -
- - Shows descriptions from the uuid log, falling back to remote names,
- - as some remotes may not be in the uuid log.
  -
  - When JSON is enabled, also generates a machine-readable description
  - of the UUIDs. -}
diff --git a/debian/changelog b/debian/changelog
index 9a010327df..8a734e0aa7 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,7 @@ git-annex (3.20120606) UNRELEASED; urgency=low
 
   * add: Prevent (most) modifications from being made to a file while it
     is being added to the annex.
+  * initremote: Automatically describe a remote when creating it.
 
  -- Joey Hess   Tue, 05 Jun 2012 20:25:51 -0400
 

From 0a11b35d89104fa0b9653f15963d273a0d3585c3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 15:19:44 -0400
Subject: [PATCH 3657/8313] extend Git.Queue to be able to queue more than
 simple git commands

While I was in there, I noticed and fixed a bug in the queue size
calculations. It was never encountered only because Queue.add was
only ever run with 1 file in the list.
---
 Annex/Queue.hs     |   8 ++--
 Command/Add.hs     |   2 +-
 Command/Fix.hs     |   2 +-
 Command/FromKey.hs |   2 +-
 Command/Fsck.hs    |   2 +-
 Command/Lock.hs    |   2 +-
 Git/Queue.hs       | 111 +++++++++++++++++++++++++++++++++------------
 Upgrade/V1.hs      |   8 ++--
 8 files changed, 94 insertions(+), 43 deletions(-)

diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index d4a2c592e5..9f2ad67911 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -6,7 +6,7 @@
  -}
 
 module Annex.Queue (
-	add,
+	addCommand,
 	flush,
 	flushWhenFull
 ) where
@@ -17,10 +17,10 @@ import qualified Git.Queue
 import Config
 
 {- Adds a git command to the queue. -}
-add :: String -> [CommandParam] -> [FilePath] -> Annex ()
-add command params files = do
+addCommand :: String -> [CommandParam] -> [FilePath] -> Annex ()
+addCommand command params files = do
 	q <- get
-	store =<< inRepo (Git.Queue.add q command params files)
+	store =<< inRepo (Git.Queue.addCommand command params files q)
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
diff --git a/Command/Add.hs b/Command/Add.hs
index ea0f850332..3f39f8713e 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -121,5 +121,5 @@ cleanup file key hascontent = do
 		( return [Param "-f"]
 		, return []
 		)
-	Annex.Queue.add "add" (params++[Param "--"]) [file]
+	Annex.Queue.addCommand "add" (params++[Param "--"]) [file]
 	return True
diff --git a/Command/Fix.hs b/Command/Fix.hs
index c4f9813811..227e08cd2a 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -36,5 +36,5 @@ perform file link = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	Annex.Queue.add "add" [Param "--force", Param "--"] [file]
+	Annex.Queue.addCommand "add" [Param "--force", Param "--"] [file]
 	return True
diff --git a/Command/FromKey.hs b/Command/FromKey.hs
index ec194e06e8..f7841c9770 100644
--- a/Command/FromKey.hs
+++ b/Command/FromKey.hs
@@ -39,5 +39,5 @@ perform key file = do
 
 cleanup :: FilePath -> CommandCleanup
 cleanup file = do
-	Annex.Queue.add "add" [Param "--"] [file]
+	Annex.Queue.addCommand "add" [Param "--"] [file]
 	return True
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index ae21acf8af..1fc656207f 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -155,7 +155,7 @@ fixLink key file = do
 		liftIO $ createDirectoryIfMissing True (parentDir file)
 		liftIO $ removeFile file
 		liftIO $ createSymbolicLink want file
-		Annex.Queue.add "add" [Param "--force", Param "--"] [file]
+		Annex.Queue.addCommand "add" [Param "--force", Param "--"] [file]
 	return True
 
 {- Checks that the location log reflects the current status of the key,
diff --git a/Command/Lock.hs b/Command/Lock.hs
index ab97b14bcc..8aadf3f590 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -24,5 +24,5 @@ start file = do
 
 perform :: FilePath -> CommandPerform
 perform file = do
-	Annex.Queue.add "checkout" [Param "--"] [file]
+	Annex.Queue.addCommand "checkout" [Param "--"] [file]
 	next $ return True -- no cleanup needed
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 956e9adb1d..5870bf866b 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -10,7 +10,8 @@
 module Git.Queue (
 	Queue,
 	new,
-	add,
+	addCommand,
+	addUpdateIndex,
 	size,
 	full,
 	flush,
@@ -25,19 +26,31 @@ import Utility.SafeCommand
 import Common
 import Git
 import Git.Command
+import qualified Git.UpdateIndex
+	
+{- Queable actions that can be performed in a git repository.
+ -}
+data Action
+	{- Updating the index file, using a list of streamers that can
+	 - be added to as the queue grows. -}
+	= UpdateIndexAction
+		{ getStreamers :: [Git.UpdateIndex.Streamer]
+		}
+	{- A git command to run, on a list of files that can be added to
+	 - as the queue grows. -}
+	| CommandAction 
+		{ getSubcommand :: String
+		, getParams :: [CommandParam]
+		, getFiles :: [FilePath]
+		} 
 
-{- An action to perform in a git repository. The file to act on
- - is not included, and must be able to be appended after the params. -}
-data Action = Action
-	{ getSubcommand :: String
-	, getParams :: [CommandParam]
-	} deriving (Show, Eq, Ord)
+{- A key that can uniquely represent an action in a Map. -}
+data ActionKey = UpdateIndexActionKey | CommandActionKey String
+	deriving (Eq, Ord)
 
-{- Compares two actions by subcommand. -}
-(===) :: Action -> Action -> Bool
-a === b = getSubcommand a == getSubcommand b
-(/==) :: Action -> Action -> Bool
-a /== b = not $ a === b
+actionKey :: Action -> ActionKey
+actionKey (UpdateIndexAction _) = UpdateIndexActionKey
+actionKey CommandAction { getSubcommand = s } = CommandActionKey s
 
 {- A queue of actions to perform (in any order) on a git repository,
  - with lists of files to perform them on. This allows coalescing 
@@ -45,9 +58,8 @@ a /== b = not $ a === b
 data Queue = Queue
 	{ size :: Int
 	, _limit :: Int
-	, _items :: M.Map Action [FilePath]
+	, items :: M.Map ActionKey Action
 	}
-	deriving (Show, Eq)
 
 {- A recommended maximum size for the queue, after which it should be
  - run.
@@ -65,20 +77,58 @@ defaultLimit = 10240
 new :: Maybe Int -> Queue
 new lim = Queue 0 (fromMaybe defaultLimit lim) M.empty
 
-{- Adds an action to a queue. If the queue already contains a different
+{- Adds a command to a queue. If the queue already contains a different
  - action, it will be flushed; this is to ensure that conflicting actions,
- - like add and rm, are run in the right order. -}
-add :: Queue -> String -> [CommandParam] -> [FilePath] -> Repo -> IO Queue
-add q@(Queue _ _ m) subcommand params files repo
-	| null (filter (/== action) (M.keys m)) = go q
-	| otherwise = go =<< flush q repo
+ - like add and rm, are run in the right order. 
+ -
+ - Actions with the same subcommand but different parameters are
+ - roughly equivilant; assumed equivilant enough to perform in any order
+ - with the same result.
+ -}
+addCommand :: String -> [CommandParam] -> [FilePath] -> Queue -> Repo -> IO Queue
+addCommand subcommand params files q repo =
+	updateQueue action different (length newfiles) q repo
 	where
-		action = Action subcommand params
-		go (Queue cur lim m') =
-			return $ Queue (cur + 1) lim $
-				M.insertWith' const action fs m'
-			where
-				!fs = files ++ M.findWithDefault [] action m'
+		key = actionKey action
+		action = CommandAction
+			{ getSubcommand = subcommand
+			, getParams = params
+			, getFiles = newfiles
+			}
+		newfiles = files ++ maybe [] getFiles (M.lookup key $ items q)
+		
+		different (CommandAction { getSubcommand = s }) = s /= subcommand
+		different _ = True
+
+addUpdateIndex :: Git.UpdateIndex.Streamer -> Queue -> Repo -> IO Queue
+addUpdateIndex streamer q repo =
+	updateQueue action different 0 q repo
+	where
+		key = actionKey action
+		-- streamer is added to the end of the list, since
+		-- order does matter for update-index input
+		action = UpdateIndexAction $ streamers ++ [streamer]
+		streamers = maybe [] getStreamers $ M.lookup key $ items q
+
+		different (UpdateIndexAction _) = False
+		different _ = True
+
+{- Updates or adds an action in the queue. If the queue already contains a
+ - different action, it will be flushed; this is to ensure that conflicting
+ - actions, like add and rm, are run in the right order.-}
+updateQueue :: Action -> (Action -> Bool) -> Int -> Queue -> Repo -> IO Queue
+updateQueue action different sizeincrease q repo
+	| null (filter different (M.elems (items q))) = return $ go q
+	| otherwise = go <$> flush q repo
+	where
+		go q' = newq
+			where		
+				!newq = q'
+					{ size = newsize
+					, items = newitems
+					}
+				!newsize = size q' + sizeincrease
+				!newitems = M.insertWith' const (actionKey action) action (items q')
 
 {- Is a queue large enough that it should be flushed? -}
 full :: Queue -> Bool
@@ -87,7 +137,7 @@ full (Queue cur lim  _) = cur > lim
 {- Runs a queue on a git repository. -}
 flush :: Queue -> Repo -> IO Queue
 flush (Queue _ lim m) repo = do
-	forM_ (M.toList m) $ uncurry $ runAction repo
+	forM_ (M.elems m) $ runAction repo
 	return $ Queue 0 lim M.empty
 
 {- Runs an Action on a list of files in a git repository.
@@ -96,12 +146,13 @@ flush (Queue _ lim m) repo = do
  -
  - Intentionally runs the command even if the list of files is empty;
  - this allows queueing commands that do not need a list of files. -}
-runAction :: Repo -> Action -> [FilePath] -> IO ()
-runAction repo action files =
+runAction :: Repo -> Action -> IO ()
+runAction _repo _action@(UpdateIndexAction {}) = error "TODO"
+runAction repo action@(CommandAction {}) =
 	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
 		params = toCommand $ gitCommandLine
 			(Param (getSubcommand action):getParams action) repo
 		feedxargs h = do
 			fileEncoding h
-			hPutStr h $ join "\0" files
+			hPutStr h $ join "\0" $ getFiles action
diff --git a/Upgrade/V1.hs b/Upgrade/V1.hs
index 280742f062..31c0210c07 100644
--- a/Upgrade/V1.hs
+++ b/Upgrade/V1.hs
@@ -94,7 +94,7 @@ updateSymlinks = do
 					link <- calcGitLink f k
 					liftIO $ removeFile f
 					liftIO $ createSymbolicLink link f
-					Annex.Queue.add "add" [Param "--"] [f]
+					Annex.Queue.addCommand "add" [Param "--"] [f]
 
 moveLocationLogs :: Annex ()
 moveLocationLogs = do
@@ -121,9 +121,9 @@ moveLocationLogs = do
 				old <- liftIO $ readLog1 f
 				new <- liftIO $ readLog1 dest
 				liftIO $ writeLog1 dest (old++new)
-				Annex.Queue.add "add" [Param "--"] [dest]
-				Annex.Queue.add "add" [Param "--"] [f]
-				Annex.Queue.add "rm" [Param "--quiet", Param "-f", Param "--"] [f]
+				Annex.Queue.addCommand "add" [Param "--"] [dest]
+				Annex.Queue.addCommand "add" [Param "--"] [f]
+				Annex.Queue.addCommand "rm" [Param "--quiet", Param "-f", Param "--"] [f]
 		
 oldlog2key :: FilePath -> Maybe (FilePath, Key)
 oldlog2key l

From 20f425be19dafda17c904945dfbf069c496a4ff8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 15:40:44 -0400
Subject: [PATCH 3658/8313] make watch use the queue

May not work. Certianly needs to flush the queue from time to time
when only symlink changes are being made.
---
 Annex/Queue.hs    | 10 +++++++++-
 Command/Watch.hs  | 22 ++++++++++------------
 Git/HashObject.hs |  4 ++--
 Git/Queue.hs      | 18 +++++++++++-------
 Git/UnionMerge.hs |  5 +++--
 5 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index 9f2ad67911..a7d4e153bf 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -1,12 +1,13 @@
 {- git-annex command queue
  -
- - Copyright 2011 Joey Hess 
+ - Copyright 2011, 2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Annex.Queue (
 	addCommand,
+	addUpdateIndex,
 	flush,
 	flushWhenFull
 ) where
@@ -14,6 +15,7 @@ module Annex.Queue (
 import Common.Annex
 import Annex hiding (new)
 import qualified Git.Queue
+import qualified Git.UpdateIndex
 import Config
 
 {- Adds a git command to the queue. -}
@@ -22,6 +24,12 @@ addCommand command params files = do
 	q <- get
 	store =<< inRepo (Git.Queue.addCommand command params files q)
 
+{- Adds an update-index stream to the queue. -}
+addUpdateIndex :: Git.UpdateIndex.Streamer -> Annex ()
+addUpdateIndex streamer = do
+	q <- get
+	store =<< inRepo (Git.Queue.addUpdateIndex streamer q)
+
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
 flushWhenFull = do
diff --git a/Command/Watch.hs b/Command/Watch.hs
index 046fca7d14..4447d4ffe6 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -15,8 +15,8 @@ import Command
 import Utility.Inotify
 import Utility.ThreadLock
 import qualified Annex
+import qualified Annex.Queue
 import qualified Command.Add
-import qualified Git
 import qualified Git.Command
 import qualified Git.UpdateIndex
 import Git.HashObject
@@ -99,7 +99,7 @@ onAdd file = do
 		go Nothing = showEndFail
 		go (Just key) = do
 			link <- Command.Add.link file key True
-			inRepo $ stageSymlink file link
+			stageSymlink file link
 			showEndOk
 
 {- A symlink might be an arbitrary symlink, which is just added.
@@ -119,7 +119,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 					liftIO $ createSymbolicLink link file
 					addlink link
 				)
-		addlink link = inRepo $ stageSymlink file link
+		addlink link = stageSymlink file link
 
 {- The file could reappear at any time, so --cached is used, to only delete
  - it from the index. -}
@@ -139,12 +139,10 @@ onErr = warning
 
 {- Adds a symlink to the index, without ever accessing the actual symlink
  - on disk. -}
-stageSymlink :: FilePath -> String -> Git.Repo -> IO ()
-stageSymlink file linktext repo = Git.UpdateIndex.stream_update_index repo [stage]
-	where
-		stage streamer = do
-			line <- Git.UpdateIndex.update_index_line
-				<$> (hashObject repo BlobObject linktext)
-				<*> pure SymlinkBlob
-				<*> toTopFilePath file repo
-			streamer line
+stageSymlink :: FilePath -> String -> Annex ()
+stageSymlink file linktext = do
+	line <- Git.UpdateIndex.update_index_line
+		<$> inRepo (hashObject BlobObject linktext)
+		<*> pure SymlinkBlob
+		<*> inRepo (toTopFilePath file)
+	Annex.Queue.addUpdateIndex $ \streamer -> streamer line
diff --git a/Git/HashObject.hs b/Git/HashObject.hs
index b052413fdb..9f37de5ba9 100644
--- a/Git/HashObject.hs
+++ b/Git/HashObject.hs
@@ -36,8 +36,8 @@ hashFile h file = CoProcess.query h send receive
 		receive from = getSha "hash-object" $ hGetLine from
 
 {- Injects some content into git, returning its Sha. -}
-hashObject :: Repo -> ObjectType -> String -> IO Sha
-hashObject repo objtype content = getSha subcmd $ do
+hashObject :: ObjectType -> String -> Repo -> IO Sha
+hashObject objtype content repo = getSha subcmd $ do
 	(h, s) <- pipeWriteRead (map Param params) content repo
 	length s `seq` do
 		forceSuccess h
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 5870bf866b..f2312cfaa2 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -77,13 +77,11 @@ defaultLimit = 10240
 new :: Maybe Int -> Queue
 new lim = Queue 0 (fromMaybe defaultLimit lim) M.empty
 
-{- Adds a command to a queue. If the queue already contains a different
- - action, it will be flushed; this is to ensure that conflicting actions,
- - like add and rm, are run in the right order. 
+{- Adds an git command to the queue.
  -
- - Actions with the same subcommand but different parameters are
- - roughly equivilant; assumed equivilant enough to perform in any order
- - with the same result.
+ - Git commands with the same subcommand but different parameters are
+ - assumed to be equivilant enough to perform in any order with the same
+ - result.
  -}
 addCommand :: String -> [CommandParam] -> [FilePath] -> Queue -> Repo -> IO Queue
 addCommand subcommand params files q repo =
@@ -100,6 +98,11 @@ addCommand subcommand params files q repo =
 		different (CommandAction { getSubcommand = s }) = s /= subcommand
 		different _ = True
 
+{- Adds an update-index streamer to the queue. 
+ -
+ - Note that this does not increase the queue size, because data is
+ - streamed into update-index, so command-line length limits are not
+ - involved. -}
 addUpdateIndex :: Git.UpdateIndex.Streamer -> Queue -> Repo -> IO Queue
 addUpdateIndex streamer q repo =
 	updateQueue action different 0 q repo
@@ -147,7 +150,8 @@ flush (Queue _ lim m) repo = do
  - Intentionally runs the command even if the list of files is empty;
  - this allows queueing commands that do not need a list of files. -}
 runAction :: Repo -> Action -> IO ()
-runAction _repo _action@(UpdateIndexAction {}) = error "TODO"
+runAction repo (UpdateIndexAction streamers) =
+	Git.UpdateIndex.stream_update_index repo streamers
 runAction repo action@(CommandAction {}) =
 	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index f65b59c53a..d38bdfe222 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -74,8 +74,9 @@ mergeFile :: String -> FilePath -> CatFileHandle -> Repo -> IO (Maybe String)
 mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 	[] -> return Nothing
 	(sha:[]) -> use sha
-	shas -> use =<< either return (hashObject repo BlobObject . unlines) =<<
-		calcMerge . zip shas <$> mapM getcontents shas
+	shas -> use
+		=<< either return (\s -> hashObject BlobObject (unlines s) repo)
+		=<< calcMerge . zip shas <$> mapM getcontents shas
 	where
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		getcontents s = map L.unpack . L.lines .

From 021325ce7107be71e067f964e4a57864eac59ea6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 7 Jun 2012 20:22:58 +0000
Subject: [PATCH 3659/8313] Added a comment

---
 .../comment_3_05223be50c889b2ed6bc4abf74116450._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment

diff --git a/doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment b/doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment
new file mode 100644
index 0000000000..a78fa33439
--- /dev/null
+++ b/doc/design/assistant/comment_3_05223be50c889b2ed6bc4abf74116450._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-06-07T20:22:55Z"
+ content="""
+I'd agree getting it into the main distros is the way to go, if you need OSX binaries, I could volunteer to setup an autobuilder to generate binaries for OSX users, however it would rely on users to have macports with the correct ports installed to use it (things like coreutils etc...)
+
+"""]]

From b778b9b345dc9686a2721fa946d46313bfd66876 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 7 Jun 2012 20:23:09 +0000
Subject: [PATCH 3660/8313] Added a comment

---
 .../comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment

diff --git a/doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment b/doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment
new file mode 100644
index 0000000000..73b01947c0
--- /dev/null
+++ b/doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2012-06-07T20:23:08Z"
+ content="""
+I'd agree getting it into the main distros is the way to go, if you need OSX binaries, I could volunteer to setup an autobuilder to generate binaries for OSX users, however it would rely on users to have macports with the correct ports installed to use it (things like coreutils etc...)
+
+"""]]

From 3f03e58dc681a82fef156d97316ac8035304e306 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Thu, 7 Jun 2012 20:25:14 +0000
Subject: [PATCH 3661/8313] removed

---
 .../comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment  | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment

diff --git a/doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment b/doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment
deleted file mode 100644
index 73b01947c0..0000000000
--- a/doc/design/assistant/comment_4_5643ebf22a0f3764a88f5d2e66562e59._comment
+++ /dev/null
@@ -1,9 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
- nickname="Jimmy"
- subject="comment 4"
- date="2012-06-07T20:23:08Z"
- content="""
-I'd agree getting it into the main distros is the way to go, if you need OSX binaries, I could volunteer to setup an autobuilder to generate binaries for OSX users, however it would rely on users to have macports with the correct ports installed to use it (things like coreutils etc...)
-
-"""]]

From 7d78cbf97ca9d5a579f05b4ba735be69bafdef97 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 21:17:10 -0400
Subject: [PATCH 3662/8313] use git queue for rm too

---
 Command/Watch.hs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 4447d4ffe6..bf544679da 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -124,14 +124,14 @@ onAddSymlink file = go =<< Backend.lookupFile file
 {- The file could reappear at any time, so --cached is used, to only delete
  - it from the index. -}
 onDel :: FilePath -> Annex ()
-onDel file = inRepo $ Git.Command.run "rm"
-	[Params "--quiet --cached --ignore-unmatch --", File file] 
+onDel file = Annex.Queue.addCommand "rm"
+	[Params "--quiet --cached --ignore-unmatch --"] [file]
 
 {- A directory has been deleted, or moved, so tell git to remove anything
  - that was inside it from its cache. -}
 onDelDir :: FilePath -> Annex ()
-onDelDir dir = inRepo $ Git.Command.run "rm"
-	[Params "--quiet -r --cached --ignore-unmatch --", File dir]
+onDelDir dir = Annex.Queue.addCommand "rm"
+	[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
 
 {- Called when there's an error with inotify. -}
 onErr :: String -> Annex ()

From 109bd9c08b2f7d26720ebb26ed7e165d38accb11 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 21:37:59 -0400
Subject: [PATCH 3663/8313] blog for the day

---
 doc/design/assistant/blog/day_4__speed.mdwn | 46 +++++++++++++++++++++
 doc/design/assistant/inotify.mdwn           |  3 ++
 2 files changed, 49 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_4__speed.mdwn

diff --git a/doc/design/assistant/blog/day_4__speed.mdwn b/doc/design/assistant/blog/day_4__speed.mdwn
new file mode 100644
index 0000000000..151f4af2a9
--- /dev/null
+++ b/doc/design/assistant/blog/day_4__speed.mdwn
@@ -0,0 +1,46 @@
+Only had a few hours to work today, but my current focus is speed, and I
+have indeed sped up parts of `git annex watch`.
+
+One thing folks don't realize about git is that despite a rep for being
+fast, it can be rather slow in one area: Writing the index. You don't
+notice it until you have a lot of files, and the index gets big. So I've
+put a lot of effort into git-annex in the past to avoid writing the index
+repeatedly, and queue up big index changes that can happen all at once. The
+new `git annex watch` was not able to use that queue. Today I reworked the
+queue machinery to support the types of direct index writes it needs, and
+now repeated index writes are eliminated.
+
+... Eliminated too far, it turns out, since it doesn't yet *ever* flush
+that queue until shutdown! So the next step here will be to have a worker
+thread that wakes up periodically, flushes the queue, and autocommits.
+There's lots of room here for smart behavior. Like, if a lot of changes are
+being made close together, wait for them to die down before committing. Or,
+if it's been idle and a single file appears, commit it immediatly, since
+this is probably something the user wants synced out right away. I'll start
+with something stupid and then add the smarts.
+
+(BTW, in all my years of programming, I have avoided threads like the nasty
+bug-prone plague they are. Here I already have three threads, and am going to
+add probably 4 or 5 more before I'm done with the git annex assistant. So
+far, it's working well -- I give credit to Haskell for making it easy to
+manage state in ways that make it possible to reason about how the threads
+will interact.)
+
+What about the races I've been stressing over? Well, I have an ulterior
+motive in speeding up `git annex watch`, and that's to also be able to
+**slow it down**. Running in slow-mo makes it easy to try things that might
+cause a race and watch how it reacts. I'll be using this technique when
+I circle back around to dealing with the races.
+
+Another tricky speed problem came up today that I also need to fix. On
+startup, `git annex watch` scans the whole tree to find files that have
+been added or moved etc while it was not running, and take care of them.
+Currently, this scan involves re-staging every symlink in the tree. That's
+slow! I need to find a way to avoid re-staging symlinks; I may use `git
+cat-file` to check if the currently staged symlink is correct, or I may
+come up with some better and faster solution. Sleeping on this problem.
+
+----
+
+Oh yeah, I also found one more race bug today. It only happens at startup
+and could only make it miss staging file deletions.
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index ab88210b2b..7cdde33ac6 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -108,3 +108,6 @@ Many races need to be dealt with by this code. Here are some of them.
   Not a problem; The removal event removes the old file from the index, and
   the add event adds the new one.
 
+* At startup, `git add --update` is run, to notice deleted files.
+  Then inotify starts up. Files deleted in between won't have their
+  removals staged.

From 4f6b522d8cf25993bc5827b6bbce9b14a1ffdf2d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 7 Jun 2012 21:40:31 -0400
Subject: [PATCH 3664/8313] update

---
 doc/design/assistant/blog/day_4__speed.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/design/assistant/blog/day_4__speed.mdwn b/doc/design/assistant/blog/day_4__speed.mdwn
index 151f4af2a9..badc6b7b18 100644
--- a/doc/design/assistant/blog/day_4__speed.mdwn
+++ b/doc/design/assistant/blog/day_4__speed.mdwn
@@ -13,6 +13,7 @@ now repeated index writes are eliminated.
 ... Eliminated too far, it turns out, since it doesn't yet *ever* flush
 that queue until shutdown! So the next step here will be to have a worker
 thread that wakes up periodically, flushes the queue, and autocommits.
+(This will, in fact, be the start of the [[syncing]] phase of my roadmap!)
 There's lots of room here for smart behavior. Like, if a lot of changes are
 being made close together, wait for them to die down before committing. Or,
 if it's been idle and a single file appears, commit it immediatly, since

From 63290903aa5c92013dd7383464fe3c6061aa835d Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 8 Jun 2012 01:56:52 +0000
Subject: [PATCH 3665/8313] Added a comment

---
 .../comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment

diff --git a/doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment b/doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment
new file mode 100644
index 0000000000..cd3b5aaef7
--- /dev/null
+++ b/doc/design/assistant/comment_4_fbbd93b55803ae21e6ba4b6568c2fafd._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 4"
+ date="2012-06-08T01:56:52Z"
+ content="""
+I always appreciate your OSX work Jimmy...
+
+Could it be put into macports?
+"""]]

From 817a42056b228bccae2c3111214944bde1286325 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Thu, 7 Jun 2012 19:01:49 -0700
Subject: [PATCH 3666/8313] Add note about lhs2tex's man page installation.

---
 Setup.hs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Setup.hs b/Setup.hs
index c722077eb5..9d2162f47f 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -35,6 +35,10 @@ install pkg_descr lbi userhooks flags = do
 -- Based on pandoc's 'Setup.installManpages' and 'postInst' hook.
 -- Would be easier to just use 'rawSystem' as above.
 --
+-- XXX: lhs2tex installs man pages with the 'postCopy' hook. 
+-- I chose 'postInst'.  Pandoc uses both :P  So, probably
+-- to use the 'postCopy' hook.
+--
 -- XXX: fix tabs!
 installManpages :: InstallFlags -> PackageDescription -> LocalBuildInfo -> IO ()
 installManpages flags pkg lbi =

From d45a9a7831a567c548f2552e1470d72f1966fffd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 8 Jun 2012 00:29:39 -0400
Subject: [PATCH 3667/8313] refactor and function name cleanup

(oops, I had a calcMerge and a calc_merge!)
---
 Annex/Branch.hs    | 10 +++++-----
 Command/Watch.hs   | 12 +++---------
 Git/Queue.hs       |  2 +-
 Git/UnionMerge.hs  | 43 ++++++++++++++++++++++---------------------
 Git/UpdateIndex.hs | 37 +++++++++++++++++++++++++++----------
 5 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 1dacd5f32a..7b433cc6ea 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -261,15 +261,15 @@ files = withIndexUpdate $ do
  - in changes from other branches.
  -}
 genIndex :: Git.Repo -> IO ()
-genIndex g = Git.UpdateIndex.stream_update_index g
-	[Git.UpdateIndex.ls_tree fullname g]
+genIndex g = Git.UpdateIndex.streamUpdateIndex g
+	[Git.UpdateIndex.lsTree fullname g]
 
 {- Merges the specified refs into the index.
  - Any changes staged in the index will be preserved. -}
 mergeIndex :: [Git.Ref] -> Annex ()
 mergeIndex branches = do
 	h <- catFileHandle
-	inRepo $ \g -> Git.UnionMerge.merge_index h g branches
+	inRepo $ \g -> Git.UnionMerge.mergeIndex h g branches
 
 {- Runs an action using the branch's index file. -}
 withIndex :: Annex a -> Annex a
@@ -338,13 +338,13 @@ stageJournal = do
 	g <- gitRepo
 	withIndex $ liftIO $ do
 		h <- hashObjectStart g
-		Git.UpdateIndex.stream_update_index g
+		Git.UpdateIndex.streamUpdateIndex g
 			[genstream (gitAnnexJournalDir g) h fs]
 		hashObjectStop h
 	where
 		genstream dir h fs streamer = forM_ fs $ \file -> do
 			let path = dir  file
 			sha <- hashFile h path
-			_ <- streamer $ Git.UpdateIndex.update_index_line
+			_ <- streamer $ Git.UpdateIndex.updateIndexLine
 				sha FileBlob (asTopFilePath $ fileJournal file)
 			removeFile path
diff --git a/Command/Watch.hs b/Command/Watch.hs
index bf544679da..c5d824864e 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -19,9 +19,6 @@ import qualified Annex.Queue
 import qualified Command.Add
 import qualified Git.Command
 import qualified Git.UpdateIndex
-import Git.HashObject
-import Git.Types
-import Git.FilePath
 import qualified Backend
 import Annex.Content
 
@@ -140,9 +137,6 @@ onErr = warning
 {- Adds a symlink to the index, without ever accessing the actual symlink
  - on disk. -}
 stageSymlink :: FilePath -> String -> Annex ()
-stageSymlink file linktext = do
-	line <- Git.UpdateIndex.update_index_line
-		<$> inRepo (hashObject BlobObject linktext)
-		<*> pure SymlinkBlob
-		<*> inRepo (toTopFilePath file)
-	Annex.Queue.addUpdateIndex $ \streamer -> streamer line
+stageSymlink file linktext =
+	Annex.Queue.addUpdateIndex =<<
+		inRepo (Git.UpdateIndex.stageSymlink file linktext)
diff --git a/Git/Queue.hs b/Git/Queue.hs
index f2312cfaa2..78b52a2bc3 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -151,7 +151,7 @@ flush (Queue _ lim m) repo = do
  - this allows queueing commands that do not need a list of files. -}
 runAction :: Repo -> Action -> IO ()
 runAction repo (UpdateIndexAction streamers) =
-	Git.UpdateIndex.stream_update_index repo streamers
+	Git.UpdateIndex.streamUpdateIndex repo streamers
 runAction repo action@(CommandAction {}) =
 	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs
index d38bdfe222..0987f9131a 100644
--- a/Git/UnionMerge.hs
+++ b/Git/UnionMerge.hs
@@ -7,7 +7,7 @@
 
 module Git.UnionMerge (
 	merge,
-	merge_index
+	mergeIndex
 ) where
 
 import qualified Data.Text.Lazy as L
@@ -32,40 +32,40 @@ import Git.FilePath
 merge :: Ref -> Ref -> Repo -> IO ()
 merge x y repo = do
 	h <- catFileStart repo
-	stream_update_index repo
-		[ ls_tree x repo
-		, merge_trees x y h repo
+	streamUpdateIndex repo
+		[ lsTree x repo
+		, mergeTrees x y h repo
 		]
 	catFileStop h
 
-{- Merges a list of branches into the index. Previously staged changed in
+{- Merges a list of branches into the index. Previously staged changes in
  - the index are preserved (and participate in the merge). -}
-merge_index :: CatFileHandle -> Repo -> [Ref] -> IO ()
-merge_index h repo bs =
-	stream_update_index repo $ map (\b -> merge_tree_index b h repo) bs
+mergeIndex :: CatFileHandle -> Repo -> [Ref] -> IO ()
+mergeIndex h repo bs =
+	streamUpdateIndex repo $ map (\b -> mergeTreeIndex b h repo) bs
 
 {- For merging two trees. -}
-merge_trees :: Ref -> Ref -> CatFileHandle -> Repo -> Streamer
-merge_trees (Ref x) (Ref y) h = calc_merge h $ "diff-tree":diff_opts ++ [x, y]
+mergeTrees :: Ref -> Ref -> CatFileHandle -> Repo -> Streamer
+mergeTrees (Ref x) (Ref y) h = doMerge h $ "diff-tree":diffOpts ++ [x, y]
 
 {- For merging a single tree into the index. -}
-merge_tree_index :: Ref -> CatFileHandle -> Repo -> Streamer
-merge_tree_index (Ref x) h = calc_merge h $
-	"diff-index" : diff_opts ++ ["--cached", x]
+mergeTreeIndex :: Ref -> CatFileHandle -> Repo -> Streamer
+mergeTreeIndex (Ref x) h = doMerge h $
+	"diff-index" : diffOpts ++ ["--cached", x]
 
-diff_opts :: [String]
-diff_opts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
+diffOpts :: [String]
+diffOpts = ["--raw", "-z", "-r", "--no-renames", "-l0"]
 
-{- Calculates how to perform a merge, using git to get a raw diff,
- - and generating update-index input. -}
-calc_merge :: CatFileHandle -> [String] -> Repo -> Streamer
-calc_merge ch differ repo streamer = gendiff >>= go
+{- Streams update-index changes to perform a merge,
+ - using git to get a raw diff. -}
+doMerge :: CatFileHandle -> [String] -> Repo -> Streamer
+doMerge ch differ repo streamer = gendiff >>= go
 	where
 		gendiff = pipeNullSplit (map Param differ) repo
 		go [] = noop
 		go (info:file:rest) = mergeFile info file ch repo >>=
 			maybe (go rest) (\l -> streamer l >> go rest)
-		go (_:[]) = error "calc_merge parse error"
+		go (_:[]) = error $ "parse error " ++ show differ
 
 {- Given an info line from a git raw diff, and the filename, generates
  - a line suitable for update-index that union merges the two sides of the
@@ -81,7 +81,8 @@ mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of
 		[_colonmode, _bmode, asha, bsha, _status] = words info
 		getcontents s = map L.unpack . L.lines .
 			L.decodeUtf8 <$> catObject h s
-		use sha = return $ Just $ update_index_line sha FileBlob $ asTopFilePath file
+		use sha = return $ Just $
+			updateIndexLine sha FileBlob $ asTopFilePath file
 
 {- Calculates a union merge between a list of refs, with contents.
  -
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index 8c003dd13b..a32db8b9db 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -7,9 +7,11 @@
 
 module Git.UpdateIndex (
 	Streamer,
-	stream_update_index,
-	ls_tree,
-	update_index_line,
+	pureStreamer,
+	streamUpdateIndex,
+	lsTree,
+	updateIndexLine,
+	stageSymlink
 ) where
 
 import System.Cmd.Utils
@@ -19,14 +21,19 @@ import Git
 import Git.Types
 import Git.Command
 import Git.FilePath
+import Git.HashObject
 
 {- Streamers are passed a callback and should feed it lines in the form
  - read by update-index, and generated by ls-tree. -}
 type Streamer = (String -> IO ()) -> IO ()
 
+{- A streamer with a precalculated value. -}
+pureStreamer :: String -> Streamer
+pureStreamer s = \streamer -> streamer s
+
 {- Streams content into update-index from a list of Streamers. -}
-stream_update_index :: Repo -> [Streamer] -> IO ()
-stream_update_index repo as = do
+streamUpdateIndex :: Repo -> [Streamer] -> IO ()
+streamUpdateIndex repo as = do
 	(p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo)
 	fileEncoding h
 	forM_ as (stream h)
@@ -39,14 +46,24 @@ stream_update_index repo as = do
 			hPutStr h s
 			hPutStr h "\0"
 
-{- Gets the current tree for a ref. -}
-ls_tree :: Ref -> Repo -> Streamer
-ls_tree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
+{- A streamer that adds the current tree for a ref. Useful for eg, copying
+ - and modifying branches. -}
+lsTree :: Ref -> Repo -> Streamer
+lsTree (Ref x) repo streamer = mapM_ streamer =<< pipeNullSplit params repo
 	where
 		params = map Param ["ls-tree", "-z", "-r", "--full-tree", x]
 
 {- Generates a line suitable to be fed into update-index, to add
  - a given file with a given sha. -}
-update_index_line :: Sha -> BlobType -> TopFilePath -> String
-update_index_line sha filetype file =
+updateIndexLine :: Sha -> BlobType -> TopFilePath -> String
+updateIndexLine sha filetype file =
 	show filetype ++ " blob " ++ show sha ++ "\t" ++ getTopFilePath file
+
+{- A streamer that adds a symlink to the index. -}
+stageSymlink :: FilePath -> String -> Repo -> IO Streamer
+stageSymlink file linktext repo = do
+	line <- updateIndexLine
+		<$> hashObject BlobObject linktext repo
+		<*> pure SymlinkBlob
+		<*> toTopFilePath file repo
+	return $ pureStreamer line

From 12afa8fb560ae877dcf8b78d352806e50e1f215e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Fri, 8 Jun 2012 07:22:34 +0000
Subject: [PATCH 3668/8313] Added a comment

---
 ...mment_5_f4e9af3fed6c27e8ff39badb9794064d._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment

diff --git a/doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment b/doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment
new file mode 100644
index 0000000000..bf8d9709e8
--- /dev/null
+++ b/doc/design/assistant/comment_5_f4e9af3fed6c27e8ff39badb9794064d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-06-08T07:22:34Z"
+ content="""
+In relation to macports, I often found that haskell in macports are often behind other distros, and I'm not willing to put much effort into maintaining or updating those ports. I found that to build git-annex, installing macports manually and then installing haskell-platform from the upstream to be the best way to get the most up to date dependancies for git-annex.
+
+fyi in macports ghc is at version 6.10.4 and haskell platform is at version 2009.2, so there are a significant number of ports to update.
+
+I was thinking about this a bit more and I reckon it might be easier to try and build a self contained .pkg package and have all the needed binaries in a .app styled package, that would work well when the webapp comes along. I will take a look at it in a week or two (currently moving house so I dont have much time)
+"""]]

From c388e90dc9f0f09ea5ef639cb4c9aabb9568ae0e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Fri, 8 Jun 2012 15:21:21 +0000
Subject: [PATCH 3669/8313] Added a comment

---
 ...comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment

diff --git a/doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment b/doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment
new file mode 100644
index 0000000000..9fa66d6d31
--- /dev/null
+++ b/doc/design/assistant/comment_6_c7ad07cade1f44f9a8b61f92225bb9c5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 6"
+ date="2012-06-08T15:21:18Z"
+ content="""
+It's not much for now... but see  I'm ignoring the debian-stable and pristine-tar branches for now, as I am just building and testing on osx 10.7.
+
+Hope the autobuilder will help you develop the OSX side of things without having direct access to an osx machine! I will try and get gitbuilder to spit out appropriately named tarballs of the compiled binaries in a few days when I have more time.
+"""]]

From 7f823b56af88f28eee450fa39b7aa691a6745e92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 9 Jun 2012 14:06:56 -0400
Subject: [PATCH 3670/8313] fix non-linux build

---
 Command/Watch.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index c5d824864e..f9daa1a99c 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -12,7 +12,9 @@ module Command.Watch where
 
 import Common.Annex
 import Command
+#if defined linux_HOST_OS
 import Utility.Inotify
+#endif
 import Utility.ThreadLock
 import qualified Annex
 import qualified Annex.Queue

From 3141c36f2e5cefb67eb200f604ac5b38a04b3e3d Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sat, 9 Jun 2012 18:07:51 +0000
Subject: [PATCH 3671/8313] Added a comment

---
 .../comment_7_609d38e993267195a80fecd84c93d1e2._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment

diff --git a/doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment b/doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment
new file mode 100644
index 0000000000..6685c6548e
--- /dev/null
+++ b/doc/design/assistant/comment_7_609d38e993267195a80fecd84c93d1e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.126"
+ subject="comment 7"
+ date="2012-06-09T18:07:51Z"
+ content="""
+Thanks, that's already been useful to me. You might as well skip the debian-specific \"bpo\" tags too.
+"""]]

From 6c8f76ca28859b1e025189e4f669508e2c516a9b Mon Sep 17 00:00:00 2001
From: "http://rmunn.myopenid.com/" 
Date: Sat, 9 Jun 2012 18:54:14 +0000
Subject: [PATCH 3672/8313] New idea: using youtube-dl to download video URLs

---
 ...cial-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn

diff --git a/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
new file mode 100644
index 0000000000..a368c98a01
--- /dev/null
+++ b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
@@ -0,0 +1 @@
+The [Web special remote](http://git-annex.branchable.com/special_remotes/web/) could possibly be improved by detecting when URLs reference a Youtube video page and using [youtube-dl](http://rg3.github.com/youtube-dl/) instead of wget to download the page. Youtube-dl can also handle several other video sites such as vimeo.com and blip.tv, so if this idea were to be implemented, it might make sense to borrow the regular expressions that youtube-dl uses to identify video URLs. A quick grep through the youtube-dl source for the identifier _VALID_URL should find those regexes (in Python's regex format).

From affd52be0e3e2dae1ffa633551cff919a44c624a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 9 Jun 2012 17:07:40 -0400
Subject: [PATCH 3673/8313] response

---
 ...andling_of_Youtube_URLs_in_Web_special_remote.mdwn | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
index a368c98a01..e11989e521 100644
--- a/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
+++ b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote.mdwn
@@ -1 +1,12 @@
 The [Web special remote](http://git-annex.branchable.com/special_remotes/web/) could possibly be improved by detecting when URLs reference a Youtube video page and using [youtube-dl](http://rg3.github.com/youtube-dl/) instead of wget to download the page. Youtube-dl can also handle several other video sites such as vimeo.com and blip.tv, so if this idea were to be implemented, it might make sense to borrow the regular expressions that youtube-dl uses to identify video URLs. A quick grep through the youtube-dl source for the identifier _VALID_URL should find those regexes (in Python's regex format).
+
+> This is something I've thought about doing for a while.. 
+> Two things I have not figured out:
+> 
+> * Seems that this should really be user-configurable or a plugin system,
+>   to handle more than just this one case.
+> * Youtube-dl breaks from time to time, I really trust these urls a lot
+>   less than regular urls. Perhaps per-url trust levels are called for by
+>   this.
+> 
+> --[[Joey]] 

From 3a213ced1e6167f3b595fb1327954d147fd5a34d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E"
 
Date: Sat, 9 Jun 2012 22:34:31 +0000
Subject: [PATCH 3674/8313] Added a comment: open source?

---
 .../comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment

diff --git a/doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment b/doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment
new file mode 100644
index 0000000000..fb5b95490f
--- /dev/null
+++ b/doc/design/assistant/blog/day_4__speed/comment_1_bf3c9c33cc0dea5eaeb6f2af110b924b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E"
+ nickname="Matt"
+ subject="open source?"
+ date="2012-06-09T22:34:30Z"
+ content="""
+Are you publishing the source code for git-annex assistant somewhere?
+"""]]

From 6a71a9729fbe02aa6c179bb6c617278257edf71c Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sat, 9 Jun 2012 23:01:29 +0000
Subject: [PATCH 3675/8313] Added a comment

---
 .../comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment

diff --git a/doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment b/doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment
new file mode 100644
index 0000000000..1fcc197ab7
--- /dev/null
+++ b/doc/design/assistant/blog/day_4__speed/comment_2_33aba4c9abaa3e6a05a2c87ab7df9d0e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.126"
+ subject="comment 2"
+ date="2012-06-09T23:01:29Z"
+ content="""
+Yes, it's in [[git|download]] with the rest of git-annex. Currently in the `watch` branch.
+"""]]

From 2b29a0228527e00d112770cb89b07f28c6bb0a14 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sat, 9 Jun 2012 20:46:41 -0700
Subject: [PATCH 3676/8313] Make man pages when making sdist.

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 26367aa46d..9ac06e1d70 100644
--- a/Makefile
+++ b/Makefile
@@ -103,6 +103,7 @@ clean:
 # Workaround for cabal sdist not running Setup hooks, so I cannot
 # generate a file list there.
 sdist: clean
+	@make $(mans)
 	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
 	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print | perl -ne 'print unless length >= 100')!i" < git-annex.cabal.orig > git-annex.cabal
 	@cabal sdist

From 6c8507ee1b0a5bd361c38ec0fa342e616b990357 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sat, 9 Jun 2012 23:32:25 -0700
Subject: [PATCH 3677/8313] Combine post install commands in 'postInst' and add
 'postCopy' hook.

The creation of the 'git-annex-shell' symlink was in 'postInst' hook.
I combined it with the man-page installation in a 'postInst' hook and
a 'postCopy' hook.  I don't understand how to use the `cabal copy`
command, but the examples I looked at defined both hooks.

Relevant comments from the source:

* man-page installation:

  See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages.

  Based on pandoc's and lhs2tex's 'Setup.installManpages' and
  'postInst' hooks.

  My understanding: 'postCopy' is run for `cabal copy`, 'postInst' is
  run for `cabal inst`, and copy is not a generalized install, so you
  have to write two nearly identical hooks.

  Summary of hooks:
  http://www.haskell.org/cabal/release/cabal-latest/doc/API/Cabal/Distribution-Simple-UserHooks.htm--
  Other people are also confused:

  * Bug: 'postCopy' and 'postInst' are confusing:
  http://hackage.haskell.org/trac/hackage/ticket/718

  * A cabal maintainer suggests using 'postCopy' instead of
  'postInst', because `cabal install` is `cabal copy` followed by
  `cabal register`:
  http://www.haskell.org/pipermail/libraries/2008-March/009416.html
  Although that sounds desirable, it's not true, as the reply and
  experiments indicate.

* the `cabal copy` command:

  ???: Not sure how you're supposed to use this.  E.g., when I do

     cabal install --prefix=/tmp/git-annex-install
     cabal copy --deistdir=/tmp/git-annex-copy

  I get the copy under

    /tmp/git-annex-copy/tmp/git-annex-install

  Also, `cabal install` fails when given a relative --prefix.
---
 Setup.hs | 95 +++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 66 insertions(+), 29 deletions(-)

diff --git a/Setup.hs b/Setup.hs
index 9d2162f47f..b2370a837d 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -1,55 +1,92 @@
+{-# LANGUAGE NamedFieldPuns #-}
 {- cabal setup file -}
 
 import Distribution.Simple
 import Distribution.Simple.LocalBuildInfo
 import Distribution.Simple.Setup
-import Distribution.Simple.Utils (installOrdinaryFiles)
+import Distribution.Simple.Utils (installOrdinaryFiles, rawSystemExit)
 import Distribution.PackageDescription (PackageDescription(..))
 import Distribution.Verbosity (Verbosity)
-import System.Cmd
 import System.FilePath
 
 import qualified Build.Configure as Configure
 
 main = defaultMainWithHooks simpleUserHooks
 	{ preConf = configure
-	, instHook = install
-        , postInst = const installManpages
+	, postInst = myPostInst
+        , postCopy = myPostCopy
 	}
 
 configure _ _ = do
 	Configure.run Configure.tests
 	return (Nothing, [])
 
-install pkg_descr lbi userhooks flags = do
-	r <- (instHook simpleUserHooks) pkg_descr lbi userhooks flags
-	_ <- rawSystem "ln" ["-sf", "git-annex", 
-		bindir installDirs  "git-annex-shell"]
-	return r
-	where
-		installDirs = absoluteInstallDirs pkg_descr lbi $
-			fromFlag (copyDest defaultCopyFlags)
+myPostInst :: Args -> InstallFlags -> PackageDescription
+           -> LocalBuildInfo -> IO ()
+myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do
+  installGitAnnexShell dest verbosity pkg lbi
+  installManpages      dest verbosity pkg lbi
+  where
+    dest      = NoCopyDest
+    verbosity = fromFlag installVerbosity
+
+-- ???: Not sure how you're supposed to use this.  E.g., when I do
+--
+--    cabal install --prefix=/tmp/git-annex-install
+--    cabal copy --deistdir=/tmp/git-annex-copy
+--
+-- I get the copy under
+--
+--   /tmp/git-annex-copy/tmp/git-annex-install
+--
+-- Also, `cabal install` fails when given a relative --prefix.
+myPostCopy :: Args -> CopyFlags -> PackageDescription
+           -> LocalBuildInfo -> IO ()
+myPostCopy _ (CopyFlags { copyDest, copyVerbosity }) pkg lbi = do
+  installGitAnnexShell dest verbosity pkg lbi
+  installManpages      dest verbosity pkg lbi
+  where
+    dest      = fromFlag copyDest
+    verbosity = fromFlag copyVerbosity
+
+installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription
+                     -> LocalBuildInfo -> IO ()
+installGitAnnexShell copyDest verbosity pkg lbi =
+  rawSystemExit verbosity "ln"
+    ["-sf", "git-annex", dstBinDir  "git-annex-shell"]
+  where
+    dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest
 
 -- See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages.
 --
--- Based on pandoc's 'Setup.installManpages' and 'postInst' hook.
--- Would be easier to just use 'rawSystem' as above.
+-- Based on pandoc's and lhs2tex's 'Setup.installManpages' and
+-- 'postInst' hooks.
 --
--- XXX: lhs2tex installs man pages with the 'postCopy' hook. 
--- I chose 'postInst'.  Pandoc uses both :P  So, probably
--- to use the 'postCopy' hook.
+-- My understanding: 'postCopy' is run for `cabal copy`, 'postInst' is
+-- run for `cabal inst`, and copy is not a generalized install, so you
+-- have to write two nearly identical hooks. 
+--
+-- Summary of hooks:
+-- http://www.haskell.org/cabal/release/cabal-latest/doc/API/Cabal/Distribution-Simple-UserHooks.htm--
+-- Other people are also confused:
+--
+-- * Bug: 'postCopy' and 'postInst' are confusing:
+-- http://hackage.haskell.org/trac/hackage/ticket/718
+--
+-- * A cabal maintainer suggests using 'postCopy' instead of
+-- 'postInst', because `cabal install` is `cabal copy` followed by
+-- `cabal register`:
+-- http://www.haskell.org/pipermail/libraries/2008-March/009416.html
+-- Although that sounds desirable, it's not true, as the reply and
+-- experiments indicate.
 --
 -- XXX: fix tabs!
-installManpages :: InstallFlags -> PackageDescription -> LocalBuildInfo -> IO ()
-installManpages flags pkg lbi =
-  installOrdinaryFiles verbosity dstManDir manpages
+installManpages :: CopyDest -> Verbosity -> PackageDescription
+                -> LocalBuildInfo -> IO ()
+installManpages copyDest verbosity pkg lbi =
+  installOrdinaryFiles verbosity dstManDir srcManpages
   where
-    srcManDir = ""
-    -- The 'NoCopyDest' means "don't add an additional path prefix".
-    -- The pandoc Setup.hs uses 'NoCopyDest' in the post install hook
-    -- and the 'CopyDest' from the copy flags in the post copy hook.
-    dstManDir = mandir (absoluteInstallDirs pkg lbi NoCopyDest)  "man1"
-    manpages  = zip (repeat srcManDir)
-                    [ "git-annex.1"
-                    , "git-annex-shell.1" ]
-    verbosity = fromFlag $ installVerbosity flags
+    dstManDir   = mandir (absoluteInstallDirs pkg lbi copyDest)  "man1"
+    srcManpages = zip (repeat srcManDir) manpages
+    srcManDir   = ""
+    manpages    = ["git-annex.1", "git-annex-shell.1"]

From ba62741e5a33c2df59c4c2bcf5eb71edf8677f0c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 10 Jun 2012 08:58:48 +0000
Subject: [PATCH 3678/8313]

---
 ...ders_for_git-annex_to_aid_development.mdwn | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn

diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
new file mode 100644
index 0000000000..5a4654bc16
--- /dev/null
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
@@ -0,0 +1,34 @@
+This is a continuation of the conversation from [[the comments|design/assistant/#comment-77e54e7ebfbd944c370173014b535c91]] section in the design of git-assistant. In summary, I've setup an auto builder which should help [[Joey]] have an easier time developing on git-annex on non-linux/debian platforms. This builder is currently running on OSX 10.7 with the 64bit version of Haskell Platform.
+
+The builder output can be found at , the CGI on this site does not work as my OSX workstation is pushing the output from another location.
+
+The builder currently tries to build all branches except 
+
+* debian-stable
+* pristine-tar
+* setup
+
+It also does not build any of the tags as well, Joey had suggested to ignore the bpo named tags, but for now it's easier for me to not build any tags. To continue on this discussion, if anyone wants to setup a gitbuilder instance, here is the build.sh script that I am using.
+
+
+#!/bin/bash -x
+
+# Macports
+export PATH=/opt/local/bin:$PATH
+
+# Haskell userland
+export PATH=$PATH:$HOME/.cabal/bin
+
+# Macports gnu
+export PATH=/opt/local/libexec/gnubin:$PATH
+
+make || exit 3
+
+make -q test
+if [ "$?" = 1 ]; then
+        # run "make test", but give it a time limit in case a test gets stuck
+        ../maxtime 1800 make test || exit 4
+fi
+

+
+It's also using the branches-local script for sorting and prioritising the branches to build, this branches-local script can be found at the [autobuild-ceph](https://github.com/ceph/autobuild-ceph/blob/master/branches-local) repository. If there are other people interested in setting up their own instances of gitbuilder for git-annex, please let me know and I will setup an aggregator page to collect status of the builds.

From 3d6dc335636f1414c3d4ff8f86ea413ed5a83169 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 10 Jun 2012 09:03:28 +0000
Subject: [PATCH 3679/8313]

---
 doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
index 5a4654bc16..8cd370937c 100644
--- a/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
@@ -31,4 +31,4 @@ if [ "$?" = 1 ]; then
 fi
 

 
-It's also using the branches-local script for sorting and prioritising the branches to build, this branches-local script can be found at the [autobuild-ceph](https://github.com/ceph/autobuild-ceph/blob/master/branches-local) repository. If there are other people interested in setting up their own instances of gitbuilder for git-annex, please let me know and I will setup an aggregator page to collect status of the builds.
+It's also using the branches-local script for sorting and prioritising the branches to build, this branches-local script can be found at the [autobuild-ceph](https://github.com/ceph/autobuild-ceph/blob/master/branches-local) repository. If there are other people interested in setting up their own instances of gitbuilder for git-annex, please let me know and I will setup an aggregator page to collect status of the builds. The builder runs and updates the webpage every 30mins.

From 4453351ca4b3dfe313b475bbf6e1de9a9b250fb5 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 10 Jun 2012 02:11:40 -0700
Subject: [PATCH 3680/8313] Add a .dir-locals.el that configures emacs'
 treatment of tabs.

The Haskell code uses tabs for indentation, and displays well with
tab-width set to 2.  So, I created a .dir-locals.el (applies to all
files opened in emacs at or below ./), which sets 'tab-width' to 2,
turns on 'indent-tabs-mode', and highlights leading spaces in
'haskell-mode'.
---
 .dir-locals.el | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 .dir-locals.el

diff --git a/.dir-locals.el b/.dir-locals.el
new file mode 100644
index 0000000000..e523528e2f
--- /dev/null
+++ b/.dir-locals.el
@@ -0,0 +1,22 @@
+;; Configure emacs' treatment of tabs.
+;;
+;; See
+;; https://www.gnu.org/software/emacs/manual/html_node/emacs/Directory-Variables.html
+;; for a description of this file.
+;;
+;; The 'nil' below applies to all modes.
+((nil . ((indent-tabs-mode . t)
+        (tab-width . 2)))
+ (haskell-mode . (
+        ;; Highlight leading space characters, to avoid indenting with
+        ;; spaces.
+        ;;
+        ;; Emacs will prompt you about this, saying it's unsafe, but
+        ;; you can permanently store an exception by pressing "!",
+        ;; which inserts
+        ;;
+        ;;   (safe-local-variable-values . (quote ((eval highlight-regexp "^ *"))))
+        ;;
+        ;; in your ~/.emacs ... except the exception doesn't work, and
+        ;; emacs still asks you on each file you open :P
+        (eval . (highlight-regexp "^ *")))))

From d76afc8152cbade2dbfe68c7ef786b2fa5a79f2f Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 10 Jun 2012 02:21:13 -0700
Subject: [PATCH 3681/8313] Replace indentation spaces by tabs in Setup.hs.

---
 Setup.hs | 44 +++++++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/Setup.hs b/Setup.hs
index b2370a837d..3f282e0aa9 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -14,7 +14,7 @@ import qualified Build.Configure as Configure
 main = defaultMainWithHooks simpleUserHooks
 	{ preConf = configure
 	, postInst = myPostInst
-        , postCopy = myPostCopy
+	, postCopy = myPostCopy
 	}
 
 configure _ _ = do
@@ -24,11 +24,11 @@ configure _ _ = do
 myPostInst :: Args -> InstallFlags -> PackageDescription
            -> LocalBuildInfo -> IO ()
 myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do
-  installGitAnnexShell dest verbosity pkg lbi
-  installManpages      dest verbosity pkg lbi
-  where
-    dest      = NoCopyDest
-    verbosity = fromFlag installVerbosity
+	installGitAnnexShell dest verbosity pkg lbi
+	installManpages      dest verbosity pkg lbi
+	where
+		dest      = NoCopyDest
+		verbosity = fromFlag installVerbosity
 
 -- ???: Not sure how you're supposed to use this.  E.g., when I do
 --
@@ -43,19 +43,19 @@ myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do
 myPostCopy :: Args -> CopyFlags -> PackageDescription
            -> LocalBuildInfo -> IO ()
 myPostCopy _ (CopyFlags { copyDest, copyVerbosity }) pkg lbi = do
-  installGitAnnexShell dest verbosity pkg lbi
-  installManpages      dest verbosity pkg lbi
-  where
-    dest      = fromFlag copyDest
-    verbosity = fromFlag copyVerbosity
+	installGitAnnexShell dest verbosity pkg lbi
+	installManpages      dest verbosity pkg lbi
+	where
+		dest      = fromFlag copyDest
+		verbosity = fromFlag copyVerbosity
 
 installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription
                      -> LocalBuildInfo -> IO ()
 installGitAnnexShell copyDest verbosity pkg lbi =
-  rawSystemExit verbosity "ln"
-    ["-sf", "git-annex", dstBinDir  "git-annex-shell"]
-  where
-    dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest
+	rawSystemExit verbosity "ln"
+		["-sf", "git-annex", dstBinDir  "git-annex-shell"]
+	where
+		dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest
 
 -- See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages.
 --
@@ -79,14 +79,12 @@ installGitAnnexShell copyDest verbosity pkg lbi =
 -- http://www.haskell.org/pipermail/libraries/2008-March/009416.html
 -- Although that sounds desirable, it's not true, as the reply and
 -- experiments indicate.
---
--- XXX: fix tabs!
 installManpages :: CopyDest -> Verbosity -> PackageDescription
                 -> LocalBuildInfo -> IO ()
 installManpages copyDest verbosity pkg lbi =
-  installOrdinaryFiles verbosity dstManDir srcManpages
-  where
-    dstManDir   = mandir (absoluteInstallDirs pkg lbi copyDest)  "man1"
-    srcManpages = zip (repeat srcManDir) manpages
-    srcManDir   = ""
-    manpages    = ["git-annex.1", "git-annex-shell.1"]
+	installOrdinaryFiles verbosity dstManDir srcManpages
+	where
+		dstManDir   = mandir (absoluteInstallDirs pkg lbi copyDest)  "man1"
+		srcManpages = zip (repeat srcManDir) manpages
+		srcManDir   = ""
+		manpages    = ["git-annex.1", "git-annex-shell.1"]

From 3bb58afd594c7208a34749202a858644498acb6f Mon Sep 17 00:00:00 2001
From: Sergei Trofimovich 
Date: Sun, 10 Jun 2012 12:03:11 +0300
Subject: [PATCH 3682/8313] Makefile: ignore monads-tf in favour of mtl

Fixes build breakage when both 'mtl' and 'monads-tf' are present:
$ make git-annex
> ghc -O2 -Wall -ignore-package monads-fd -outputdir tmp -IUtility -DWITH_S3 --make git-annex Utility/libdiskfree.o
>
> Common.hs:6:8:
>    Ambiguous module name `Control.Monad.State.Strict':
>      it was found in multiple packages: monads-tf-0.1.0.0 mtl-2.1.1
> make: *** [git-annex] Error 1

Signed-off-by: Sergei Trofimovich 
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 94dc05a819..2d8146c857 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 PREFIX=/usr
-IGNORE=-ignore-package monads-fd
+IGNORE=-ignore-package monads-fd -ignore-package monads-tf
 BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
 GHCFLAGS=-O2 $(BASEFLAGS)
 

From 7f39415600a808c7664520a435eb45cf8d82f7ce Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 12:50:06 -0400
Subject: [PATCH 3683/8313] force thunk for precalculated value

---
 Git/UpdateIndex.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index a32db8b9db..faffeec667 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Git.UpdateIndex (
 	Streamer,
 	pureStreamer,
@@ -29,7 +31,7 @@ type Streamer = (String -> IO ()) -> IO ()
 
 {- A streamer with a precalculated value. -}
 pureStreamer :: String -> Streamer
-pureStreamer s = \streamer -> streamer s
+pureStreamer !s = \streamer -> streamer s
 
 {- Streams content into update-index from a list of Streamers. -}
 streamUpdateIndex :: Repo -> [Streamer] -> IO ()

From 5308b51ec0dce12849d8f4e5bc3f0adf6bf09a5f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 13:05:58 -0400
Subject: [PATCH 3684/8313] stage deletions directly using update-index

no need to run git-rm separately
---
 Command/Watch.hs   | 11 ++++++++---
 Git/UpdateIndex.hs |  8 ++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index f9daa1a99c..d50a581a18 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -123,11 +123,16 @@ onAddSymlink file = go =<< Backend.lookupFile file
 {- The file could reappear at any time, so --cached is used, to only delete
  - it from the index. -}
 onDel :: FilePath -> Annex ()
-onDel file = Annex.Queue.addCommand "rm"
-	[Params "--quiet --cached --ignore-unmatch --"] [file]
+onDel file = Annex.Queue.addUpdateIndex =<<
+	inRepo (Git.UpdateIndex.unstageFile file)
 
 {- A directory has been deleted, or moved, so tell git to remove anything
- - that was inside it from its cache. -}
+ - that was inside it from its cache. Since it could reappear at any time,
+ - use --cached to only delete it from the index. 
+ -
+ - Note: This could use unstageFile, but would need to run another git
+ - command to get the recursive list of files in the directory, so rm is
+ - just as good. -}
 onDelDir :: FilePath -> Annex ()
 onDelDir dir = Annex.Queue.addCommand "rm"
 	[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index faffeec667..07057ed988 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -13,6 +13,7 @@ module Git.UpdateIndex (
 	streamUpdateIndex,
 	lsTree,
 	updateIndexLine,
+	unstageFile,
 	stageSymlink
 ) where
 
@@ -24,6 +25,7 @@ import Git.Types
 import Git.Command
 import Git.FilePath
 import Git.HashObject
+import Git.Sha
 
 {- Streamers are passed a callback and should feed it lines in the form
  - read by update-index, and generated by ls-tree. -}
@@ -61,6 +63,12 @@ updateIndexLine :: Sha -> BlobType -> TopFilePath -> String
 updateIndexLine sha filetype file =
 	show filetype ++ " blob " ++ show sha ++ "\t" ++ getTopFilePath file
 
+{- A streamer that removes a file from the index. -}
+unstageFile :: FilePath -> Repo -> IO Streamer
+unstageFile file repo = do
+	p <- toTopFilePath file repo
+	return $ pureStreamer $ "0 " ++ show nullSha ++ "\t" ++ getTopFilePath p
+
 {- A streamer that adds a symlink to the index. -}
 stageSymlink :: FilePath -> String -> Repo -> IO Streamer
 stageSymlink file linktext repo = do

From e5f855b7f8e887f169c1bb086ef9a4f595dc767e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 13:23:10 -0400
Subject: [PATCH 3685/8313] generalize and improve state MVar code

---
 Command/Watch.hs | 48 ++++++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index d50a581a18..8961379e77 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -24,7 +24,6 @@ import qualified Git.UpdateIndex
 import qualified Backend
 import Annex.Content
 
-import Control.Exception as E
 import Control.Concurrent.MVar
 
 #if defined linux_HOST_OS
@@ -43,10 +42,8 @@ start = notBareRepo $ do
 	showStart "watch" "."
 	showAction "scanning"
 	inRepo $ Git.Command.run "add" [Param "--update"]
-	state <- Annex.getState id
-	mvar <- liftIO $ newMVar state
-	next $ next $ liftIO $ withINotify $ \i -> do
-		let hook a = Just $ runAnnex mvar a
+	next $ next $ withStateMVar $ \mvar -> liftIO $ withINotify $ \i -> do
+		let hook a = Just $ runHook mvar a
 		let hooks = WatchHooks
 			{ addHook = hook onAdd
 			, delHook = hook onDel
@@ -68,23 +65,36 @@ ignored ".gitignore" = True
 ignored ".gitattributes" = True
 ignored _ = False
 
-{- Runs a handler, inside the Annex monad.
+{- Stores the Annex state in a MVar, so that threaded actions can access
+ - it.
  -
- - Exceptions by the handlers are ignored, otherwise a whole watcher
- - thread could be crashed.
+ - Once the action is finished, retrieves the state from the MVar.
  -}
-runAnnex :: MVar Annex.AnnexState -> (FilePath -> Annex a) -> FilePath -> IO ()
-runAnnex mvar a f = do
+withStateMVar :: (MVar Annex.AnnexState -> Annex a) -> Annex a
+withStateMVar a = do
+	state <- Annex.getState id
+	mvar <- liftIO $ newMVar state
+	r <- a mvar
+	newstate <- liftIO $ takeMVar mvar
+	Annex.changeState (const newstate)
+	return r
+
+{- Runs an Annex action, using the state from the MVar. -}
+runStateMVar :: MVar Annex.AnnexState -> Annex () -> IO ()
+runStateMVar mvar a = do
 	startstate <- takeMVar mvar
-	r <- E.try (go startstate) :: IO (Either E.SomeException Annex.AnnexState)
-	case r of
-		Left e -> do
-			putStrLn (show e)
-			putMVar mvar startstate
-		Right !newstate ->
-			putMVar mvar newstate
+	!newstate <- Annex.exec startstate a
+	putMVar mvar newstate
+
+{- Runs a hook, inside the Annex monad.
+ -
+ - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
+ -}
+runHook :: MVar Annex.AnnexState -> (FilePath -> Annex ()) -> FilePath -> IO ()
+runHook mvar a f = handle =<< tryIO (runStateMVar mvar $ a f)
 	where
-		go state = Annex.exec state $ a f
+		handle (Right ()) = return ()
+		handle (Left e) = putStrLn $ show e
 
 {- Adding a file is tricky; the file has to be replaced with a symlink
  - but this is race prone, as the symlink could be changed immediately
@@ -120,8 +130,6 @@ onAddSymlink file = go =<< Backend.lookupFile file
 				)
 		addlink link = stageSymlink file link
 
-{- The file could reappear at any time, so --cached is used, to only delete
- - it from the index. -}
 onDel :: FilePath -> Annex ()
 onDel file = Annex.Queue.addUpdateIndex =<<
 	inRepo (Git.UpdateIndex.unstageFile file)

From c5707c84d372fc668b957c5d0b224bcf524e04f1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 13:56:04 -0400
Subject: [PATCH 3686/8313] queue size fix

Increase queue size for update-index actions, because otherwise they'll
never be flushed.
---
 Annex/Queue.hs | 7 ++++++-
 Git/Queue.hs   | 8 ++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index a7d4e153bf..c019aed6cc 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -9,7 +9,8 @@ module Annex.Queue (
 	addCommand,
 	addUpdateIndex,
 	flush,
-	flushWhenFull
+	flushWhenFull,
+	size
 ) where
 
 import Common.Annex
@@ -45,6 +46,10 @@ flush = do
 		q' <- inRepo $ Git.Queue.flush q
 		store q'
 
+{- Gets the size of the queue. -}
+size :: Annex Int
+size = Git.Queue.size <$> get
+
 get :: Annex Git.Queue.Queue
 get = maybe new return =<< getState repoqueue
 
diff --git a/Git/Queue.hs b/Git/Queue.hs
index 78b52a2bc3..acf6cd0918 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -98,14 +98,10 @@ addCommand subcommand params files q repo =
 		different (CommandAction { getSubcommand = s }) = s /= subcommand
 		different _ = True
 
-{- Adds an update-index streamer to the queue. 
- -
- - Note that this does not increase the queue size, because data is
- - streamed into update-index, so command-line length limits are not
- - involved. -}
+{- Adds an update-index streamer to the queue. -}
 addUpdateIndex :: Git.UpdateIndex.Streamer -> Queue -> Repo -> IO Queue
 addUpdateIndex streamer q repo =
-	updateQueue action different 0 q repo
+	updateQueue action different 1 q repo
 	where
 		key = actionKey action
 		-- streamer is added to the end of the list, since

From 6e54907e3570f23b50d97f26c7c0580b77ecf81d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 13:56:39 -0400
Subject: [PATCH 3687/8313] add a thread to commit changes

Currently the stupidest possible version, just wakes up every second,
and may make empty commits sometimes.
---
 Command/Watch.hs | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 8961379e77..7c1bc5c177 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -24,7 +24,7 @@ import qualified Git.UpdateIndex
 import qualified Backend
 import Annex.Content
 
-import Control.Concurrent.MVar
+import Control.Concurrent
 
 #if defined linux_HOST_OS
 import System.INotify
@@ -52,6 +52,7 @@ start = notBareRepo $ do
 			, errHook = hook onErr
 			}
 		watchDir i "." (ignored . takeFileName) hooks
+		_ <- forkIO $ commitThread mvar
 		putStrLn "(started)"
 		waitForTermination
 		return True
@@ -91,8 +92,11 @@ runStateMVar mvar a = do
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
 runHook :: MVar Annex.AnnexState -> (FilePath -> Annex ()) -> FilePath -> IO ()
-runHook mvar a f = handle =<< tryIO (runStateMVar mvar $ a f)
+runHook mvar a f = handle =<< tryIO (runStateMVar mvar go)
 	where
+		go = do
+			a f
+			Annex.Queue.flushWhenFull
 		handle (Right ()) = return ()
 		handle (Left e) = putStrLn $ show e
 
@@ -155,3 +159,21 @@ stageSymlink :: FilePath -> String -> Annex ()
 stageSymlink file linktext =
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.stageSymlink file linktext)
+
+{- This thread wakes up periodically and makes git commits. -}
+commitThread :: MVar Annex.AnnexState -> IO ()
+commitThread mvar = forever $ do
+	threadDelay 1000000 -- 1 second
+	commit
+	where
+		commit = tryIO $ runStateMVar mvar $
+			whenM ((>) <$> Annex.Queue.size <*> pure 0) $ do
+				Annex.Queue.flush
+				{- Empty commits may be made if tree
+				 - changes cancel each other out, etc. -}
+				inRepo $ Git.Command.run "commit"
+					[ Param "--allow-empty-message"
+					, Param "-m", Param ""
+					, Param "--allow-empty"
+					, Param "--quiet"
+					]

From 2de50f733a01ce5b282ff0f6eb8a1101bd496216 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 16:07:48 -0400
Subject: [PATCH 3688/8313] smart commit thread

The commit thread now has access to a channel containing the times of
all uncommitted changes. This lets it be smart about detecting busy times
when a batch job is running (such as rm -rf, or untarring something, etc),
and avoid committing until it's done. While at the same time, instantly
committing one-off changes that the user is going to expect to see
immediately.

I had to use STM to implement the channel, because of
http://hackage.haskell.org/trac/ghc/ticket/4154
While this adds a dependency, I always wanted to use STM, so this actually
makes me happy. ;)

Also happy that shouldCommit is a pure function, so other commit smartness
strategies can easily be played with. Although the current one seems pretty
good.

There is one bug, for some reason it does double commits, every time.
---
 Command/Watch.hs | 105 ++++++++++++++++++++++++++++++++++++-----------
 debian/control   |   1 +
 git-annex.cabal  |   5 ++-
 3 files changed, 84 insertions(+), 27 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 7c1bc5c177..34282e46c7 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -12,9 +12,6 @@ module Command.Watch where
 
 import Common.Annex
 import Command
-#if defined linux_HOST_OS
-import Utility.Inotify
-#endif
 import Utility.ThreadLock
 import qualified Annex
 import qualified Annex.Queue
@@ -25,11 +22,16 @@ import qualified Backend
 import Annex.Content
 
 import Control.Concurrent
+import Control.Concurrent.STM
+import Data.Time.Clock
 
 #if defined linux_HOST_OS
+import Utility.Inotify
 import System.INotify
 #endif
 
+type ChangeChan = TChan UTCTime
+
 def :: [Command]
 def = [command "watch" paramPaths seek "watch for changes"]
 
@@ -42,8 +44,10 @@ start = notBareRepo $ do
 	showStart "watch" "."
 	showAction "scanning"
 	inRepo $ Git.Command.run "add" [Param "--update"]
-	next $ next $ withStateMVar $ \mvar -> liftIO $ withINotify $ \i -> do
-		let hook a = Just $ runHook mvar a
+	next $ next $ withStateMVar $ \st -> liftIO $ withINotify $ \i -> do
+		changechan <- atomically newTChan
+		_ <- forkIO $ commitThread st changechan
+		let hook a = Just $ runHook st changechan a
 		let hooks = WatchHooks
 			{ addHook = hook onAdd
 			, delHook = hook onDel
@@ -52,7 +56,6 @@ start = notBareRepo $ do
 			, errHook = hook onErr
 			}
 		watchDir i "." (ignored . takeFileName) hooks
-		_ <- forkIO $ commitThread mvar
 		putStrLn "(started)"
 		waitForTermination
 		return True
@@ -91,12 +94,12 @@ runStateMVar mvar a = do
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
-runHook :: MVar Annex.AnnexState -> (FilePath -> Annex ()) -> FilePath -> IO ()
-runHook mvar a f = handle =<< tryIO (runStateMVar mvar go)
+runHook :: MVar Annex.AnnexState -> ChangeChan -> (FilePath -> Annex ()) -> FilePath -> IO ()
+runHook st changetimes a f = handle =<< tryIO (runStateMVar st go)
 	where
 		go = do
 			a f
-			Annex.Queue.flushWhenFull
+			signalChange changetimes
 		handle (Right ()) = return ()
 		handle (Left e) = putStrLn $ show e
 
@@ -160,20 +163,72 @@ stageSymlink file linktext =
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.stageSymlink file linktext)
 
-{- This thread wakes up periodically and makes git commits. -}
-commitThread :: MVar Annex.AnnexState -> IO ()
-commitThread mvar = forever $ do
-	threadDelay 1000000 -- 1 second
-	commit
+{- Signals that a change has been made, that needs to get committed. -}
+signalChange :: ChangeChan -> Annex ()
+signalChange chan = do
+	liftIO $ (atomically . writeTChan chan) =<< getCurrentTime
+	-- Just in case the commit thread is not flushing
+	-- the queue fast enough.
+	Annex.Queue.flushWhenFull
+
+{- Gets the times of all unhandled changes.
+ - Blocks until at least one change is made. -}
+getChanges :: ChangeChan -> IO [UTCTime]
+getChanges chan = atomically $ do
+	c <- readTChan chan
+	go [c]
 	where
-		commit = tryIO $ runStateMVar mvar $
-			whenM ((>) <$> Annex.Queue.size <*> pure 0) $ do
-				Annex.Queue.flush
-				{- Empty commits may be made if tree
-				 - changes cancel each other out, etc. -}
-				inRepo $ Git.Command.run "commit"
-					[ Param "--allow-empty-message"
-					, Param "-m", Param ""
-					, Param "--allow-empty"
-					, Param "--quiet"
-					]
+		go l = do
+			v <- tryReadTChan chan
+			case v of
+				Nothing -> return l
+				Just c -> go (c:l)
+
+{- Puts unhandled changes back into the channel.
+ - Note: Original order is not preserved. -}
+refillChanges :: ChangeChan -> [UTCTime] -> IO ()
+refillChanges chan cs = atomically $ mapM_ (writeTChan chan) cs
+
+{- This thread makes git commits. -}
+commitThread :: MVar Annex.AnnexState -> ChangeChan -> IO ()
+commitThread st changechan = forever $ do
+	-- First, a simple rate limiter.
+	threadDelay $ oneSecond
+	liftIO $ putStrLn "running"
+	-- Next, wait until at least one change has been made.
+	cs <- getChanges changechan
+	-- Now see if now's a good time to commit.
+	ifM (shouldCommit <$> getCurrentTime <*> pure cs) $
+		( commit
+		, do
+			liftIO $ putStrLn $ "no commit now " ++ show (length cs)
+			refillChanges changechan cs
+		)
+	where
+		commit = void $ tryIO $ runStateMVar st $ do
+			Annex.Queue.flush
+			{- Empty commits may be made if tree
+			 - changes cancel each other out, etc. -}
+			inRepo $ Git.Command.run "commit"
+				[ Param "--allow-empty-message"
+				, Param "-m", Param ""
+				, Param "--allow-empty"
+				, Param "--quiet"
+				]
+		oneSecond = 1000000 -- microseconds
+
+{- Decide if now is a good time to make a commit.
+ - Note that the list of change times has an undefined order.
+ -
+ - Current strategy: If there have been 10 commits within the past second,
+ - a batch activity is taking place, so wait for later.
+ -}
+shouldCommit :: UTCTime -> [UTCTime] -> Bool
+shouldCommit now changetimes
+	| len == 0 = False
+	| len > 4096 = True -- avoid bloating queue too much
+	| length (filter thisSecond changetimes) < 10 = True
+	| otherwise = False -- batch activity
+	where
+		len = length changetimes
+		thisSecond t = now `diffUTCTime` t <= 1
diff --git a/debian/control b/debian/control
index bfb0017bc8..6534fef317 100644
--- a/debian/control
+++ b/debian/control
@@ -21,6 +21,7 @@ Build-Depends:
 	libghc-bloomfilter-dev,
 	libghc-edit-distance-dev,
 	libghc-hinotify-dev,
+	libghc-stm-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/git-annex.cabal b/git-annex.cabal
index 114a4069fa..e1ad164535 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -36,7 +36,7 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify
+   hinotify, STM
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
@@ -52,7 +52,8 @@ Test-Suite test
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
+   hinotify, STM
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
 

From a0e29b214f9cb0d3772f9d97d152e2dd9e40adf5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 16:33:42 -0400
Subject: [PATCH 3689/8313] blog for the day

---
 .../assistant/blog/day_5__committing.mdwn     | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_5__committing.mdwn

diff --git a/doc/design/assistant/blog/day_5__committing.mdwn b/doc/design/assistant/blog/day_5__committing.mdwn
new file mode 100644
index 0000000000..3840138c60
--- /dev/null
+++ b/doc/design/assistant/blog/day_5__committing.mdwn
@@ -0,0 +1,57 @@
+After a few days otherwise engaged, back to work today.
+
+My focus was on adding the committing thread mentioned in [[day_4__speed]].
+I got rather further than expected!
+
+First, I implemented a really dumb thread, that woke up once per second,
+checked if any changes had been made, and committed them. Of course, this
+rather sucked. In the middle of a large operation like untarring a tarball,
+or `rm -r` of a large directory tree, it made lots of commits and made
+things slow and ugly. This was not unexpected.
+
+So next, I added some smarts to it. First, I wanted to stop it waking up
+every second when there was nothing to do, and instead blocking wait on a
+change occuring. Secondly, I wanted it to know when past changes happened,
+so it could detect batch mode scenarios, and avoid committing too
+frequently. 
+
+I played around with combinations of various Haskell thread communications
+tools to get that information to the committer thread: `MVar`, `Chan`,
+`QSem`, `QSemN`. Eventually, I realized all I needed was a simple channel
+through which the timestamps of changes could be sent. However, `Chan`
+wasn't quite suitable, and I had to add a dependency on 
+[Software Transactional Memory](http://en.wikipedia.org/wiki/Software_Transactional_Memory),
+and use a `TChan`. Now I'm cooking with gas!
+
+With that data channel available to the committer thread, it quickly got
+some very nice smart behavior. Playing around with it, I find it commits
+*instantly* when I'm making some random change that I'd want the
+git-annex assistant to sync out instantly; and that its batch job detection
+works pretty well too.
+
+There's surely room for improvement, and I made this part of the code
+be an entirely pure function, so it's really easy to change the strategy.
+This part of the committer thread is so nice and clean, that here's the
+current code, for your viewing pleasure:
+
+[[format haskell """
+{- Decide if now is a good time to make a commit.
+ - Note that the list of change times has an undefined order.
+ -
+ - Current strategy: If there have been 10 commits within the past second,
+ - a batch activity is taking place, so wait for later.
+ -}
+shouldCommit :: UTCTime -> [UTCTime] -> Bool
+shouldCommit now changetimes
+       | len == 0 = False
+       | len > 4096 = True -- avoid bloating queue too much
+       | length (filter thisSecond changetimes) < 10 = True
+       | otherwise = False -- batch activity
+       where
+               len = length changetimes
+               thisSecond t = now `diffUTCTime` t <= 1
+"""]]
+
+Still some polishing to do to eliminate minor innefficiencies and deal
+with more races, but this part of the git-annex assistant is now very usable,
+and will be going out to my beta testers soon!

From 39b9aaa7a99e3ab7af926a64dd276d746e54e74a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 16:44:22 -0400
Subject: [PATCH 3690/8313] typo

---
 doc/design/assistant/blog/day_5__committing.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_5__committing.mdwn b/doc/design/assistant/blog/day_5__committing.mdwn
index 3840138c60..7d6b52199d 100644
--- a/doc/design/assistant/blog/day_5__committing.mdwn
+++ b/doc/design/assistant/blog/day_5__committing.mdwn
@@ -34,7 +34,7 @@ be an entirely pure function, so it's really easy to change the strategy.
 This part of the committer thread is so nice and clean, that here's the
 current code, for your viewing pleasure:
 
-[[format haskell """
+[[!format haskell """
 {- Decide if now is a good time to make a commit.
  - Note that the list of change times has an undefined order.
  -

From f4aacd5c8659d7638eb98fcdd9b3909de782569f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 16:49:36 -0400
Subject: [PATCH 3691/8313] update

---
 doc/install.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index ec3a7b013c..5aec2e914d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -41,7 +41,8 @@ To build and use git-annex, you will need:
   * [IfElse](http://hackage.haskell.org/package/IfElse)
   * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
   * [edit-distance](http://hackage.haskell.org/package/edit-distance)
-  * [hinotify](http://hackage.haskell.org/package/hinotify)
+  * [stm](http://hackage.haskell.org/package/stm)
+  * [hinotify](http://hackage.haskell.org/package/hinotify) (on Linux only)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)

From cda6c4dff5cfa4733198ab87096235d81fce65d6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 17:40:35 -0400
Subject: [PATCH 3692/8313] tweak

---
 Command/Watch.hs | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 34282e46c7..92ebb6d70a 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -39,12 +39,17 @@ seek :: [CommandSeek]
 seek = [withNothing start]
 
 start :: CommandStart
-#if defined linux_HOST_OS
 start = notBareRepo $ do
 	showStart "watch" "."
+	watch
+	stop
+
+watch :: Annex ()
+#if defined linux_HOST_OS
+watch = do
 	showAction "scanning"
 	inRepo $ Git.Command.run "add" [Param "--update"]
-	next $ next $ withStateMVar $ \st -> liftIO $ withINotify $ \i -> do
+	withStateMVar $ \st -> liftIO $ withINotify $ \i -> do
 		changechan <- atomically newTChan
 		_ <- forkIO $ commitThread st changechan
 		let hook a = Just $ runHook st changechan a
@@ -58,9 +63,8 @@ start = notBareRepo $ do
 		watchDir i "." (ignored . takeFileName) hooks
 		putStrLn "(started)"
 		waitForTermination
-		return True
 #else
-start = error "watch mode is so far only available on Linux"
+watch = error "watch mode is so far only available on Linux"
 #endif
 
 ignored :: FilePath -> Bool

From fc0dd7977490917a1a87968ba117799bf04891bd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 17:53:17 -0400
Subject: [PATCH 3693/8313] avoid running pre-commit hook from watch commits

---
 Command/Watch.hs | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 92ebb6d70a..f468d7271e 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -197,26 +197,28 @@ refillChanges chan cs = atomically $ mapM_ (writeTChan chan) cs
 commitThread :: MVar Annex.AnnexState -> ChangeChan -> IO ()
 commitThread st changechan = forever $ do
 	-- First, a simple rate limiter.
-	threadDelay $ oneSecond
-	liftIO $ putStrLn "running"
+	threadDelay oneSecond
 	-- Next, wait until at least one change has been made.
 	cs <- getChanges changechan
 	-- Now see if now's a good time to commit.
-	ifM (shouldCommit <$> getCurrentTime <*> pure cs) $
-		( commit
-		, do
-			liftIO $ putStrLn $ "no commit now " ++ show (length cs)
-			refillChanges changechan cs
-		)
+	time <- getCurrentTime
+	if shouldCommit time cs
+		then commit
+		else refillChanges changechan cs
 	where
 		commit = void $ tryIO $ runStateMVar st $ do
 			Annex.Queue.flush
-			{- Empty commits may be made if tree
-			 - changes cancel each other out, etc. -}
 			inRepo $ Git.Command.run "commit"
 				[ Param "--allow-empty-message"
 				, Param "-m", Param ""
+				-- Empty commits may be made if tree
+				-- changes cancel each other out, etc
 				, Param "--allow-empty"
+				-- Avoid running the usual git-annex
+				-- pre-commit hook; watch does the same
+				-- symlink fixing, and we don't want to
+				-- deal with unlocked files in these
+				-- commits.
 				, Param "--quiet"
 				]
 		oneSecond = 1000000 -- microseconds

From aae0ba1995e258c4f4b83b40eb6324ec1f9baa05 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 18:29:05 -0400
Subject: [PATCH 3694/8313] fixed the double commits problem

---
 Command/Add.hs   |   7 +--
 Command/Watch.hs | 136 ++++++++++++++++++++++++++++-------------------
 2 files changed, 84 insertions(+), 59 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 3f39f8713e..ccdff67ec9 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -94,9 +94,8 @@ undo file key e = do
 			src <- inRepo $ gitAnnexLocation key
 			liftIO $ moveFile src file
 
-{- Creates the symlink to the annexed content, and also returns the link's
- - text. -}
-link :: FilePath -> Key -> Bool -> Annex FilePath
+{- Creates the symlink to the annexed content. -}
+link :: FilePath -> Key -> Bool -> Annex ()
 link file key hascontent = handle (undo file key) $ do
 	l <- calcGitLink file key
 	liftIO $ createSymbolicLink l file
@@ -110,8 +109,6 @@ link file key hascontent = handle (undo file key) $ do
 			mtime <- modificationTime <$> getFileStatus file
 			touch file (TimeSpec mtime) False
 
-	return l
-
 {- Note: Several other commands call this, and expect it to 
  - create the symlink and add it. -}
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup
diff --git a/Command/Watch.hs b/Command/Watch.hs
index f468d7271e..ab0c0ce796 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -30,7 +30,16 @@ import Utility.Inotify
 import System.INotify
 #endif
 
-type ChangeChan = TChan UTCTime
+type ChangeChan = TChan Change
+
+type Handler = FilePath -> Annex (Maybe Change)
+
+data Change = Change
+	{ changeTime :: UTCTime
+	, changeFile :: FilePath
+	, changeDesc :: String
+	}
+	deriving (Show)
 
 def :: [Command]
 def = [command "watch" paramPaths seek "watch for changes"]
@@ -52,7 +61,7 @@ watch = do
 	withStateMVar $ \st -> liftIO $ withINotify $ \i -> do
 		changechan <- atomically newTChan
 		_ <- forkIO $ commitThread st changechan
-		let hook a = Just $ runHook st changechan a
+		let hook a = Just $ runHandler st changechan a
 		let hooks = WatchHooks
 			{ addHook = hook onAdd
 			, delHook = hook onDel
@@ -94,56 +103,70 @@ runStateMVar mvar a = do
 	!newstate <- Annex.exec startstate a
 	putMVar mvar newstate
 
-{- Runs a hook, inside the Annex monad.
+{- Runs an action handler, inside the Annex monad.
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
-runHook :: MVar Annex.AnnexState -> ChangeChan -> (FilePath -> Annex ()) -> FilePath -> IO ()
-runHook st changetimes a f = handle =<< tryIO (runStateMVar st go)
+runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO ()
+runHandler st changechan hook file = handle =<< tryIO (runStateMVar st go)
 	where
-		go = do
-			a f
-			signalChange changetimes
+		go = maybe noop (signalChange changechan) =<< hook file
 		handle (Right ()) = return ()
 		handle (Left e) = putStrLn $ show e
 
+{- Handlers call this when they made a change that needs to get committed. -}
+madeChange :: FilePath -> String -> Annex (Maybe Change)
+madeChange file desc = liftIO $ 
+	Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
+
 {- Adding a file is tricky; the file has to be replaced with a symlink
  - but this is race prone, as the symlink could be changed immediately
  - after creation. To avoid that race, git add is not used to stage the
- - symlink. -}
-onAdd :: FilePath -> Annex ()
+ - symlink.
+ -
+ - Inotify will notice the new symlink, so this Handler does not stage it
+ - or return a Change, leaving that to onAddSymlink.
+ -}
+onAdd :: Handler
 onAdd file = do
 	showStart "add" file
-	Command.Add.ingest file >>= go
+	handle =<< Command.Add.ingest file
+	return Nothing
 	where
-		go Nothing = showEndFail
-		go (Just key) = do
-			link <- Command.Add.link file key True
-			stageSymlink file link
+		handle Nothing = showEndFail
+		handle (Just key) = do
+			Command.Add.link file key True
 			showEndOk
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
  - before adding it.
  -}
-onAddSymlink :: FilePath -> Annex ()
+onAddSymlink :: Handler
 onAddSymlink file = go =<< Backend.lookupFile file
 	where
-		go Nothing = addlink =<< liftIO (readSymbolicLink file)
+		go Nothing = do
+			addlink =<< liftIO (readSymbolicLink file)
+			madeChange file "add"
 		go (Just (key, _)) = do
 			link <- calcGitLink file key
 			ifM ((==) link <$> liftIO (readSymbolicLink file))
-				( addlink link
+				( do
+					addlink link
+					madeChange file "add"
 				, do
 					liftIO $ removeFile file
 					liftIO $ createSymbolicLink link file
 					addlink link
+					madeChange file "fix"
 				)
 		addlink link = stageSymlink file link
 
-onDel :: FilePath -> Annex ()
-onDel file = Annex.Queue.addUpdateIndex =<<
-	inRepo (Git.UpdateIndex.unstageFile file)
+onDel :: Handler
+onDel file = do
+	Annex.Queue.addUpdateIndex =<<
+		inRepo (Git.UpdateIndex.unstageFile file)
+	madeChange file "rm"
 
 {- A directory has been deleted, or moved, so tell git to remove anything
  - that was inside it from its cache. Since it could reappear at any time,
@@ -152,13 +175,17 @@ onDel file = Annex.Queue.addUpdateIndex =<<
  - Note: This could use unstageFile, but would need to run another git
  - command to get the recursive list of files in the directory, so rm is
  - just as good. -}
-onDelDir :: FilePath -> Annex ()
-onDelDir dir = Annex.Queue.addCommand "rm"
-	[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
+onDelDir :: Handler
+onDelDir dir = do
+	Annex.Queue.addCommand "rm"
+		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
+	madeChange dir "rmdir"
 
 {- Called when there's an error with inotify. -}
-onErr :: String -> Annex ()
-onErr = warning
+onErr :: Handler
+onErr msg = do
+	warning msg
+	return Nothing
 
 {- Adds a symlink to the index, without ever accessing the actual symlink
  - on disk. -}
@@ -168,16 +195,17 @@ stageSymlink file linktext =
 		inRepo (Git.UpdateIndex.stageSymlink file linktext)
 
 {- Signals that a change has been made, that needs to get committed. -}
-signalChange :: ChangeChan -> Annex ()
-signalChange chan = do
-	liftIO $ (atomically . writeTChan chan) =<< getCurrentTime
+signalChange :: ChangeChan -> Change -> Annex ()
+signalChange chan change = do
+	liftIO $ atomically $ writeTChan chan change
+
 	-- Just in case the commit thread is not flushing
 	-- the queue fast enough.
 	Annex.Queue.flushWhenFull
 
-{- Gets the times of all unhandled changes.
+{- Gets all unhandled changes.
  - Blocks until at least one change is made. -}
-getChanges :: ChangeChan -> IO [UTCTime]
+getChanges :: ChangeChan -> IO [Change]
 getChanges chan = atomically $ do
 	c <- readTChan chan
 	go [c]
@@ -190,10 +218,10 @@ getChanges chan = atomically $ do
 
 {- Puts unhandled changes back into the channel.
  - Note: Original order is not preserved. -}
-refillChanges :: ChangeChan -> [UTCTime] -> IO ()
+refillChanges :: ChangeChan -> [Change] -> IO ()
 refillChanges chan cs = atomically $ mapM_ (writeTChan chan) cs
 
-{- This thread makes git commits. -}
+{- This thread makes git commits at appropriate times. -}
 commitThread :: MVar Annex.AnnexState -> ChangeChan -> IO ()
 commitThread st changechan = forever $ do
 	-- First, a simple rate limiter.
@@ -203,38 +231,38 @@ commitThread st changechan = forever $ do
 	-- Now see if now's a good time to commit.
 	time <- getCurrentTime
 	if shouldCommit time cs
-		then commit
+		then void $ tryIO $ runStateMVar st $ commitStaged
 		else refillChanges changechan cs
 	where
-		commit = void $ tryIO $ runStateMVar st $ do
-			Annex.Queue.flush
-			inRepo $ Git.Command.run "commit"
-				[ Param "--allow-empty-message"
-				, Param "-m", Param ""
-				-- Empty commits may be made if tree
-				-- changes cancel each other out, etc
-				, Param "--allow-empty"
-				-- Avoid running the usual git-annex
-				-- pre-commit hook; watch does the same
-				-- symlink fixing, and we don't want to
-				-- deal with unlocked files in these
-				-- commits.
-				, Param "--quiet"
-				]
 		oneSecond = 1000000 -- microseconds
 
+commitStaged :: Annex ()
+commitStaged = do
+	Annex.Queue.flush
+	inRepo $ Git.Command.run "commit"
+		[ Param "--allow-empty-message"
+		, Param "-m", Param ""
+		-- Empty commits may be made if tree changes cancel
+		-- each other out, etc
+		, Param "--allow-empty"
+		-- Avoid running the usual git-annex pre-commit hook;
+		-- watch does the same symlink fixing, and we don't want
+		-- to deal with unlocked files in these commits.
+		, Param "--quiet"
+		]
+
 {- Decide if now is a good time to make a commit.
  - Note that the list of change times has an undefined order.
  -
  - Current strategy: If there have been 10 commits within the past second,
  - a batch activity is taking place, so wait for later.
  -}
-shouldCommit :: UTCTime -> [UTCTime] -> Bool
-shouldCommit now changetimes
+shouldCommit :: UTCTime -> [Change] -> Bool
+shouldCommit now changes
 	| len == 0 = False
 	| len > 4096 = True -- avoid bloating queue too much
-	| length (filter thisSecond changetimes) < 10 = True
+	| length (filter thisSecond changes) < 10 = True
 	| otherwise = False -- batch activity
 	where
-		len = length changetimes
-		thisSecond t = now `diffUTCTime` t <= 1
+		len = length changes
+		thisSecond c = now `diffUTCTime` changeTime c <= 1

From c1b432ee54424c3943dee97ff2dd90c4cc533e9b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 19:08:03 -0400
Subject: [PATCH 3695/8313] run git add --update after inotify is started

This way, there's no window where deleted files won't be noticed.
---
 Command/Watch.hs | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index ab0c0ce796..c57b21ac69 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -57,10 +57,8 @@ watch :: Annex ()
 #if defined linux_HOST_OS
 watch = do
 	showAction "scanning"
-	inRepo $ Git.Command.run "add" [Param "--update"]
 	withStateMVar $ \st -> liftIO $ withINotify $ \i -> do
 		changechan <- atomically newTChan
-		_ <- forkIO $ commitThread st changechan
 		let hook a = Just $ runHandler st changechan a
 		let hooks = WatchHooks
 			{ addHook = hook onAdd
@@ -69,7 +67,17 @@ watch = do
 			, delDirHook = hook onDelDir
 			, errHook = hook onErr
 			}
+		-- The commit thread is started early, so that the user
+		-- can immediately begin adding files and having them
+		-- committed, even while the inotify scan is taking place.
+		_ <- forkIO $ commitThread st changechan
+		-- This does not return until the inotify scan is done.
+		-- That can take some time for large trees.
 		watchDir i "." (ignored . takeFileName) hooks
+		-- Notice any files that were deleted before inotify
+		-- was started.
+		runStateMVar st $
+			inRepo $ Git.Command.run "add" [Param "--update"]
 		putStrLn "(started)"
 		waitForTermination
 #else

From b20c270d0061d4ae9f1b143ef0dab70318b37fe8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 19:11:04 -0400
Subject: [PATCH 3696/8313] update

---
 doc/design/assistant/inotify.mdwn | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 7cdde33ac6..a01bf51e78 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -23,9 +23,9 @@ really useful, it needs to:
   is exceeded. This can be tuned by root, so help the user fix it.
   **done**
 - periodically auto-commit staged changes (avoid autocommitting when
-  lots of changes are coming in)
+  lots of changes are coming in) **done**
 - tunable delays before adding new files, etc
-- coleasce related add/rm events for speed and less disk IO
+- coleasce related add/rm events for speed and less disk IO **done**
 - don't annex `.gitignore` and `.gitattributes` files **done**
 - configurable option to only annex files meeting certian size or
   filename criteria
@@ -107,7 +107,3 @@ Many races need to be dealt with by this code. Here are some of them.
 
   Not a problem; The removal event removes the old file from the index, and
   the add event adds the new one.
-
-* At startup, `git add --update` is run, to notice deleted files.
-  Then inotify starts up. Files deleted in between won't have their
-  removals staged.

From ca9ee21bd771e7f94ecd3916f55b10fb3cc8dcbe Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 10 Jun 2012 19:58:34 -0400
Subject: [PATCH 3697/8313] crazy optimisation

Crazy like a fox..
---
 Annex/CatFile.hs   |  7 +++++++
 Command/Watch.hs   | 45 +++++++++++++++++++++++++++++++++------------
 Git/CatFile.hs     | 17 +++++++++++------
 Git/UpdateIndex.hs |  7 +++----
 4 files changed, 54 insertions(+), 22 deletions(-)

diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs
index bcf44551e2..afb14c67f0 100644
--- a/Annex/CatFile.hs
+++ b/Annex/CatFile.hs
@@ -8,6 +8,7 @@
 module Annex.CatFile (
 	catFile,
 	catObject,
+	catObjectDetails,
 	catFileHandle
 ) where
 
@@ -17,6 +18,7 @@ import Common.Annex
 import qualified Git
 import qualified Git.CatFile
 import qualified Annex
+import Git.Types
 
 catFile :: Git.Branch -> FilePath -> Annex L.ByteString
 catFile branch file = do
@@ -28,6 +30,11 @@ catObject ref = do
 	h <- catFileHandle
 	liftIO $ Git.CatFile.catObject h ref
 
+catObjectDetails :: Git.Ref -> Annex (Maybe (L.ByteString, Sha))
+catObjectDetails ref = do
+	h <- catFileHandle
+	liftIO $ Git.CatFile.catObjectDetails h ref
+
 catFileHandle :: Annex Git.CatFile.CatFileHandle
 catFileHandle = maybe startup return =<< Annex.getState Annex.catfilehandle
 	where
diff --git a/Command/Watch.hs b/Command/Watch.hs
index c57b21ac69..e2ff8d7f9a 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -18,12 +18,17 @@ import qualified Annex.Queue
 import qualified Command.Add
 import qualified Git.Command
 import qualified Git.UpdateIndex
+import qualified Git.HashObject
 import qualified Backend
 import Annex.Content
+import Annex.CatFile
+import Git.Types
 
 import Control.Concurrent
 import Control.Concurrent.STM
 import Data.Time.Clock
+import Data.Bits.Utils
+import qualified Data.ByteString.Lazy as L
 
 #if defined linux_HOST_OS
 import Utility.Inotify
@@ -127,6 +132,9 @@ madeChange :: FilePath -> String -> Annex (Maybe Change)
 madeChange file desc = liftIO $ 
 	Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
 
+noChange :: Annex (Maybe Change)
+noChange = return Nothing
+
 {- Adding a file is tricky; the file has to be replaced with a symlink
  - but this is race prone, as the symlink could be changed immediately
  - after creation. To avoid that race, git add is not used to stage the
@@ -139,7 +147,7 @@ onAdd :: Handler
 onAdd file = do
 	showStart "add" file
 	handle =<< Command.Add.ingest file
-	return Nothing
+	noChange
 	where
 		handle Nothing = showEndFail
 		handle (Just key) = do
@@ -153,22 +161,35 @@ onAdd file = do
 onAddSymlink :: Handler
 onAddSymlink file = go =<< Backend.lookupFile file
 	where
-		go Nothing = do
-			addlink =<< liftIO (readSymbolicLink file)
-			madeChange file "add"
+		go Nothing = addlink =<< liftIO (readSymbolicLink file)
 		go (Just (key, _)) = do
 			link <- calcGitLink file key
 			ifM ((==) link <$> liftIO (readSymbolicLink file))
-				( do
-					addlink link
-					madeChange file "add"
+				( addlink link
 				, do
 					liftIO $ removeFile file
 					liftIO $ createSymbolicLink link file
 					addlink link
-					madeChange file "fix"
 				)
-		addlink link = stageSymlink file link
+		{- This is often called on symlinks that are already staged
+		 - correctly, especially during the startup scan. A symlink
+		 - may have been deleted and re-added, or added when
+		 - the watcher was not running; so it always stages
+		 - even symlinks that already exist.
+		 -
+		 - So for speed, tries to reuse the existing blob for
+		 - the symlink target. -}
+		addlink link = do
+			v <- catObjectDetails $ Ref $ ":" ++ file
+			case v of
+				Just (currlink, sha)
+					| s2w8 link == L.unpack currlink ->
+						stageSymlink file sha
+				_ -> do
+					sha <- inRepo $
+						Git.HashObject.hashObject BlobObject link
+					stageSymlink file sha
+			madeChange file "link"
 
 onDel :: Handler
 onDel file = do
@@ -197,10 +218,10 @@ onErr msg = do
 
 {- Adds a symlink to the index, without ever accessing the actual symlink
  - on disk. -}
-stageSymlink :: FilePath -> String -> Annex ()
-stageSymlink file linktext =
+stageSymlink :: FilePath -> Sha -> Annex ()
+stageSymlink file sha =
 	Annex.Queue.addUpdateIndex =<<
-		inRepo (Git.UpdateIndex.stageSymlink file linktext)
+		inRepo (Git.UpdateIndex.stageSymlink file sha)
 
 {- Signals that a change has been made, that needs to get committed. -}
 signalChange :: ChangeChan -> Change -> Annex ()
diff --git a/Git/CatFile.hs b/Git/CatFile.hs
index d5b367945f..8a320a7120 100644
--- a/Git/CatFile.hs
+++ b/Git/CatFile.hs
@@ -10,7 +10,8 @@ module Git.CatFile (
 	catFileStart,
 	catFileStop,
 	catFile,
-	catObject
+	catObject,
+	catObjectDetails,
 ) where
 
 import System.IO
@@ -42,7 +43,11 @@ catFile h branch file = catObject h $ Ref $ show branch ++ ":" ++ file
 {- Uses a running git cat-file read the content of an object.
  - Objects that do not exist will have "" returned. -}
 catObject :: CatFileHandle -> Ref -> IO L.ByteString
-catObject h object = CoProcess.query h send receive
+catObject h object = maybe L.empty fst <$> catObjectDetails h object
+
+{- Gets both the content of an object, and its Sha. -}
+catObjectDetails :: CatFileHandle -> Ref -> IO (Maybe (L.ByteString, Sha))
+catObjectDetails h object = CoProcess.query h send receive
 	where
 		send to = do
 			fileEncoding to
@@ -55,16 +60,16 @@ catObject h object = CoProcess.query h send receive
 					| length sha == shaSize &&
 					  isJust (readObjectType objtype) -> 
 						case reads size of
-							[(bytes, "")] -> readcontent bytes from
+							[(bytes, "")] -> readcontent bytes from sha
 							_ -> dne
 					| otherwise -> dne
 				_
 					| header == show object ++ " missing" -> dne
 					| otherwise -> error $ "unknown response from git cat-file " ++ show (header, object)
-		readcontent bytes from = do
+		readcontent bytes from sha = do
 			content <- S.hGet from bytes
 			c <- hGetChar from
 			when (c /= '\n') $
 				error "missing newline from git cat-file"
-			return $ L.fromChunks [content]
-		dne = return L.empty
+			return $ Just (L.fromChunks [content], Ref sha)
+		dne = return Nothing
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index 07057ed988..31e8a45b27 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -24,7 +24,6 @@ import Git
 import Git.Types
 import Git.Command
 import Git.FilePath
-import Git.HashObject
 import Git.Sha
 
 {- Streamers are passed a callback and should feed it lines in the form
@@ -70,10 +69,10 @@ unstageFile file repo = do
 	return $ pureStreamer $ "0 " ++ show nullSha ++ "\t" ++ getTopFilePath p
 
 {- A streamer that adds a symlink to the index. -}
-stageSymlink :: FilePath -> String -> Repo -> IO Streamer
-stageSymlink file linktext repo = do
+stageSymlink :: FilePath -> Sha -> Repo -> IO Streamer
+stageSymlink file sha repo = do
 	line <- updateIndexLine
-		<$> hashObject BlobObject linktext repo
+		<$> pure sha
 		<*> pure SymlinkBlob
 		<*> toTopFilePath file repo
 	return $ pureStreamer line

From f4b82e31b47ac17bbd55dfcdb681cb82174dda00 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 10 Jun 2012 18:59:16 -0700
Subject: [PATCH 3698/8313] Fix Makefile dependencies for `make test`.

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9ac06e1d70..03bada30ef 100644
--- a/Makefile
+++ b/Makefile
@@ -62,7 +62,7 @@ install: all install-docs
 	install $(bins) $(DESTDIR)$(PREFIX)/bin
 	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
 
-test:
+test: $(sources) $(clibs)
 	@if ! $(GHCMAKE) -O0 test $(clibs); then \
 		echo "** failed to build the test suite" >&2; \
 		exit 1; \

From 721de7067c43b8693aa6b4c163ecf9ec39d907d2 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 10 Jun 2012 19:03:35 -0700
Subject: [PATCH 3699/8313] Add cabal-dev to .gitignore.

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index b6b8d606d6..1de0aaf722 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@ html
 Utility/Touch.hs
 Utility/libdiskfree.o
 dist
+cabal-dev

From d271383d319606e71974f45e70f4880256057b04 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkq0-zRhubO6kR9f85-5kALszIzxIokTUw"
 
Date: Mon, 11 Jun 2012 02:15:06 +0000
Subject: [PATCH 3700/8313] Added a comment: Cloud Service Limitations

---
 ...t_1_4997778abc171999499487b71b31c9ba._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment

diff --git a/doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment b/doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment
new file mode 100644
index 0000000000..1a01afaa33
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_1_4997778abc171999499487b71b31c9ba._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkq0-zRhubO6kR9f85-5kALszIzxIokTUw"
+ nickname="James"
+ subject="Cloud Service Limitations"
+ date="2012-06-11T02:15:04Z"
+ content="""
+Hey Joey!
+
+I'm not very tech savvy, but here is my question. 
+I think for all cloud service providers, there is an upload limitation on how big one file may be.
+For example, I can't upload a file bigger than 100 MB on box.net. 
+Does this affect git-annex at all? Will git-annex automatically split the file depending on the cloud provider or will I have to create small RAR archives of one large file to upload them?
+
+Thanks!
+James
+"""]]

From b86f201bdf5bba6da51647486ac5cf2dbbf206bd Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 10 Jun 2012 19:17:51 -0700
Subject: [PATCH 3701/8313] Rename git-annex.cabal.

---
 git-annex.cabal => git-annex.cabal.template.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename git-annex.cabal => git-annex.cabal.template.sh (100%)

diff --git a/git-annex.cabal b/git-annex.cabal.template.sh
similarity index 100%
rename from git-annex.cabal
rename to git-annex.cabal.template.sh

From d5884388b09347835df599d8a0dcea77e6795c10 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 00:39:09 -0400
Subject: [PATCH 3702/8313] daemonize git annex watch

---
 Command/Watch.hs   | 75 ++++++++++++++++++++++++++--------------------
 Locations.hs       |  5 ++++
 Utility/Daemon.hs  | 39 ++++++++++++++++++++++++
 Utility/LogFile.hs | 31 +++++++++++++++++++
 doc/git-annex.mdwn |  7 +++--
 5 files changed, 123 insertions(+), 34 deletions(-)
 create mode 100644 Utility/Daemon.hs
 create mode 100644 Utility/LogFile.hs

diff --git a/Command/Watch.hs b/Command/Watch.hs
index e2ff8d7f9a..480bd3ede1 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -12,6 +12,8 @@ module Command.Watch where
 
 import Common.Annex
 import Command
+import Utility.Daemon
+import Utility.LogFile
 import Utility.ThreadLock
 import qualified Annex
 import qualified Annex.Queue
@@ -23,6 +25,7 @@ import qualified Backend
 import Annex.Content
 import Annex.CatFile
 import Git.Types
+import Option
 
 import Control.Concurrent
 import Control.Concurrent.STM
@@ -47,44 +50,52 @@ data Change = Change
 	deriving (Show)
 
 def :: [Command]
-def = [command "watch" paramPaths seek "watch for changes"]
+def = [withOptions [foregroundOption] $ 
+	command "watch" paramPaths seek "watch for changes"]
 
 seek :: [CommandSeek]
-seek = [withNothing start]
+seek = [withFlag foregroundOption $ withNothing . start]
 
-start :: CommandStart
-start = notBareRepo $ do
-	showStart "watch" "."
-	watch
+foregroundOption :: Option
+foregroundOption = Option.flag [] "foreground" "do not daemonize"
+
+start :: Bool -> CommandStart
+start foreground = notBareRepo $ withStateMVar $ \st -> do
+	if foreground
+		then do
+			showStart "watch" "."
+			liftIO $ watch st
+		else do
+			logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
+			liftIO $ daemonize logfd False $ watch st
 	stop
 
-watch :: Annex ()
+watch :: MVar Annex.AnnexState -> IO ()
 #if defined linux_HOST_OS
-watch = do
-	showAction "scanning"
-	withStateMVar $ \st -> liftIO $ withINotify $ \i -> do
-		changechan <- atomically newTChan
-		let hook a = Just $ runHandler st changechan a
-		let hooks = WatchHooks
-			{ addHook = hook onAdd
-			, delHook = hook onDel
-			, addSymlinkHook = hook onAddSymlink
-			, delDirHook = hook onDelDir
-			, errHook = hook onErr
-			}
-		-- The commit thread is started early, so that the user
-		-- can immediately begin adding files and having them
-		-- committed, even while the inotify scan is taking place.
-		_ <- forkIO $ commitThread st changechan
-		-- This does not return until the inotify scan is done.
-		-- That can take some time for large trees.
-		watchDir i "." (ignored . takeFileName) hooks
-		-- Notice any files that were deleted before inotify
-		-- was started.
-		runStateMVar st $
-			inRepo $ Git.Command.run "add" [Param "--update"]
-		putStrLn "(started)"
-		waitForTermination
+watch st = withINotify $ \i -> do
+	changechan <- atomically newTChan
+	let hook a = Just $ runHandler st changechan a
+	let hooks = WatchHooks
+		{ addHook = hook onAdd
+		, delHook = hook onDel
+		, addSymlinkHook = hook onAddSymlink
+		, delDirHook = hook onDelDir
+		, errHook = hook onErr
+		}
+	-- The commit thread is started early, so that the user
+	-- can immediately begin adding files and having them
+	-- committed, even while the inotify scan is taking place.
+	_ <- forkIO $ commitThread st changechan
+	-- This does not return until the inotify scan is done.
+	-- That can take some time for large trees.
+	watchDir i "." (ignored . takeFileName) hooks
+	runStateMVar st $ showAction "scanning"
+	-- Notice any files that were deleted before inotify
+	-- was started.
+	runStateMVar st $ do
+		inRepo $ Git.Command.run "add" [Param "--update"]
+		showAction "started"
+	waitForTermination
 #else
 watch = error "watch mode is so far only available on Linux"
 #endif
diff --git a/Locations.hs b/Locations.hs
index db456388a6..1dcfdc0fff 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -23,6 +23,7 @@ module Locations (
 	gitAnnexIndex,
 	gitAnnexIndexLock,
 	gitAnnexIndexDirty,
+	gitAnnexLogFile,
 	gitAnnexSshDir,
 	gitAnnexRemotesDir,
 	isLinkToAnnex,
@@ -145,6 +146,10 @@ gitAnnexIndexLock r = gitAnnexDir r  "index.lck"
 gitAnnexIndexDirty :: Git.Repo -> FilePath
 gitAnnexIndexDirty r = gitAnnexDir r  "index.dirty"
 
+{- Log file for daemon mode. -}
+gitAnnexLogFile :: Git.Repo -> FilePath
+gitAnnexLogFile r = gitAnnexDir r  "daemon.log"
+
 {- .git/annex/ssh/ is used for ssh connection caching -}
 gitAnnexSshDir :: Git.Repo -> FilePath
 gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r  "ssh"
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
new file mode 100644
index 0000000000..be3df17b77
--- /dev/null
+++ b/Utility/Daemon.hs
@@ -0,0 +1,39 @@
+{- daemon functions
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Daemon where
+
+import System.Posix
+import System.Directory
+import System.Exit
+import Control.Monad
+
+{- Run an action as a daemon, with all output sent to a file descriptor.
+ -
+ - Does not return. -}
+daemonize :: Fd -> Bool -> IO () -> IO ()
+daemonize logfd changedirectory a = do
+	_ <- forkProcess child1
+	end
+	where
+		child1 = do
+			_ <- createSession
+			_ <- forkProcess child2
+			end
+		child2 = do
+			when changedirectory $
+				setCurrentDirectory "/"
+			nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags
+			_ <- redir nullfd stdInput
+			mapM_ (redir logfd) [stdOutput, stdError]
+			closeFd logfd
+			a
+			end
+		redir newh h = do
+			closeFd h
+			dupTo newh h
+		end = exitImmediately ExitSuccess
diff --git a/Utility/LogFile.hs b/Utility/LogFile.hs
new file mode 100644
index 0000000000..7ffb63f524
--- /dev/null
+++ b/Utility/LogFile.hs
@@ -0,0 +1,31 @@
+{- log files
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.LogFile where
+
+import Common
+
+import System.Posix
+
+openLog :: FilePath -> IO Fd
+openLog logfile = do
+	rotateLog logfile 0
+	openFd logfile WriteOnly (Just stdFileMode)
+		defaultFileFlags { append = True }
+
+rotateLog :: FilePath -> Int -> IO ()
+rotateLog logfile num
+	| num >= 10 = return ()
+	| otherwise = whenM (doesFileExist currfile) $ do
+		rotateLog logfile (num + 1)
+		renameFile currfile nextfile
+	where
+		currfile = filename num
+		nextfile = filename (num + 1)
+		filename n
+			| n == 0 = logfile
+			| otherwise = logfile ++ "." ++ show n
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index c1d8015ab7..8ff005d8d0 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -173,8 +173,11 @@ subdirectories).
 
   Watches for changes to files in the current directory and its subdirectories,
   and takes care of automatically adding new files, as well as dealing with
-  deleted, copied, and moved files. Run this in the background, and you
-  no longer need to manually run git commands when manipulating your files.
+  deleted, copied, and moved files. With this running as a daemon in the
+  background, you no longer need to manually run git commands when
+  manipulating your files.
+
+  To not daemonize, run with --foreground
 
 # REPOSITORY SETUP COMMANDS
 

From d2dbb5953915007b52601bf3836c7c1060785b9c Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Mon, 11 Jun 2012 04:48:08 +0000
Subject: [PATCH 3703/8313] Added a comment: re: cloud

---
 .../comment_2_08da8bc74a4845e354dca99184cffd70._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment

diff --git a/doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment b/doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment
new file mode 100644
index 0000000000..a9b377ea5f
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_2_08da8bc74a4845e354dca99184cffd70._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.126"
+ subject="re: cloud"
+ date="2012-06-11T04:48:08Z"
+ content="""
+Yes, git-annex has to split files for certian providers. I already added support for this as part of my first pass at supporting box.com, see [[tips/using_box.com_as_a_special_remote]].
+"""]]

From 0b3e2bed783ade691baf60a4198aaa1034b28440 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 01:20:19 -0400
Subject: [PATCH 3704/8313] add a pid file

Writes pid to a file. Is supposed to take an exclusive lock, but that's not
working, and it's too late for me to understand why.
---
 Command/Watch.hs  |  3 ++-
 Locations.hs      |  5 +++++
 Utility/Daemon.hs | 45 ++++++++++++++++++++++++++++++---------------
 3 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 480bd3ede1..8b8e5578d2 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -67,7 +67,8 @@ start foreground = notBareRepo $ withStateMVar $ \st -> do
 			liftIO $ watch st
 		else do
 			logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
-			liftIO $ daemonize logfd False $ watch st
+			pidfile <- fromRepo gitAnnexPidFile
+			liftIO $ daemonize logfd (Just pidfile) False $ watch st
 	stop
 
 watch :: MVar Annex.AnnexState -> IO ()
diff --git a/Locations.hs b/Locations.hs
index 1dcfdc0fff..9d27fbdae1 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -23,6 +23,7 @@ module Locations (
 	gitAnnexIndex,
 	gitAnnexIndexLock,
 	gitAnnexIndexDirty,
+	gitAnnexPidFile,
 	gitAnnexLogFile,
 	gitAnnexSshDir,
 	gitAnnexRemotesDir,
@@ -146,6 +147,10 @@ gitAnnexIndexLock r = gitAnnexDir r  "index.lck"
 gitAnnexIndexDirty :: Git.Repo -> FilePath
 gitAnnexIndexDirty r = gitAnnexDir r  "index.dirty"
 
+{- Pid file for daemon mode. -}
+gitAnnexPidFile :: Git.Repo -> FilePath
+gitAnnexPidFile r = gitAnnexDir r  "daemon.pid"
+
 {- Log file for daemon mode. -}
 gitAnnexLogFile :: Git.Repo -> FilePath
 gitAnnexLogFile r = gitAnnexDir r  "daemon.log"
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index be3df17b77..eeec0bfccd 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -1,4 +1,4 @@
-{- daemon functions
+{- daemon support
  -
  - Copyright 2012 Joey Hess 
  -
@@ -7,24 +7,28 @@
 
 module Utility.Daemon where
 
+import Common
+
 import System.Posix
-import System.Directory
-import System.Exit
-import Control.Monad
 
 {- Run an action as a daemon, with all output sent to a file descriptor.
  -
- - Does not return. -}
-daemonize :: Fd -> Bool -> IO () -> IO ()
-daemonize logfd changedirectory a = do
-	_ <- forkProcess child1
-	end
+ - Can write its pid to a file, to guard against multiple instances
+ - running and allow easy termination.
+ -
+ - When successful, does not return. -}
+daemonize :: Fd -> Maybe FilePath -> Bool -> IO () -> IO ()
+daemonize logfd pidfile changedirectory a = do
+	pidfd <- lockpidfile
+	_ <- forkProcess $ child1 pidfd
+	out
 	where
-		child1 = do
+		child1 pidfd = do
 			_ <- createSession
-			_ <- forkProcess child2
-			end
-		child2 = do
+			_ <- forkProcess $ child2 pidfd
+			out
+		child2 pidfd = do
+			writepidfile pidfd
 			when changedirectory $
 				setCurrentDirectory "/"
 			nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags
@@ -32,8 +36,19 @@ daemonize logfd changedirectory a = do
 			mapM_ (redir logfd) [stdOutput, stdError]
 			closeFd logfd
 			a
-			end
+			out
+		lockpidfile = case pidfile of
+			Just file -> do
+				fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
+				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
+				return $ Just fd
+			Nothing -> return Nothing
+		writepidfile pidfd = 
+			case pidfd of
+				Just fd -> void $
+					fdWrite fd =<< show <$> getProcessID
+				Nothing -> return ()
 		redir newh h = do
 			closeFd h
 			dupTo newh h
-		end = exitImmediately ExitSuccess
+		out = exitImmediately ExitSuccess

From 8539a7bde8e20758b7f7d70af93fe92aa4be1e7f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 01:37:25 -0400
Subject: [PATCH 3705/8313] fix pid file locking

Ok, that's odd.. opening it before fork breaks the locking.
I don't understand why.
---
 Utility/Daemon.hs | 30 +++++++++++++-----------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index eeec0bfccd..b41ebe2285 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -19,16 +19,15 @@ import System.Posix
  - When successful, does not return. -}
 daemonize :: Fd -> Maybe FilePath -> Bool -> IO () -> IO ()
 daemonize logfd pidfile changedirectory a = do
-	pidfd <- lockpidfile
-	_ <- forkProcess $ child1 pidfd
+	_ <- forkProcess child1
 	out
 	where
-		child1 pidfd = do
+		child1 = do
 			_ <- createSession
-			_ <- forkProcess $ child2 pidfd
+			_ <- forkProcess child2
 			out
-		child2 pidfd = do
-			writepidfile pidfd
+		child2 = do
+			maybe noop lockPidFile pidfile
 			when changedirectory $
 				setCurrentDirectory "/"
 			nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags
@@ -37,18 +36,15 @@ daemonize logfd pidfile changedirectory a = do
 			closeFd logfd
 			a
 			out
-		lockpidfile = case pidfile of
-			Just file -> do
-				fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
-				setLock fd (WriteLock, AbsoluteSeek, 0, 0)
-				return $ Just fd
-			Nothing -> return Nothing
-		writepidfile pidfd = 
-			case pidfd of
-				Just fd -> void $
-					fdWrite fd =<< show <$> getProcessID
-				Nothing -> return ()
 		redir newh h = do
 			closeFd h
 			dupTo newh h
 		out = exitImmediately ExitSuccess
+
+lockPidFile :: FilePath -> IO ()
+lockPidFile file = void $ do
+	fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
+	catchIO
+		(setLock fd (WriteLock, AbsoluteSeek, 0, 0))
+		(const $ error "Daemon is already running.")
+	fdWrite fd =<< show <$> getProcessID

From d0a0a6ae21e71c83c6365500a4f9b78f38477ac0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 02:01:20 -0400
Subject: [PATCH 3706/8313] git annex watch --stop

---
 Command/Watch.hs   | 33 +++++++++++++++++++++------------
 Utility/Daemon.hs  | 33 ++++++++++++++++++++++++++-------
 doc/git-annex.mdwn |  3 ++-
 3 files changed, 49 insertions(+), 20 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 8b8e5578d2..f01a9616f0 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -50,27 +50,36 @@ data Change = Change
 	deriving (Show)
 
 def :: [Command]
-def = [withOptions [foregroundOption] $ 
+def = [withOptions [foregroundOption, stopOption] $ 
 	command "watch" paramPaths seek "watch for changes"]
 
 seek :: [CommandSeek]
-seek = [withFlag foregroundOption $ withNothing . start]
+seek = [withFlag stopOption $ \stopdaemon -> 
+	withFlag foregroundOption $ \foreground ->
+	withNothing $ start foreground stopdaemon]
 
 foregroundOption :: Option
 foregroundOption = Option.flag [] "foreground" "do not daemonize"
 
-start :: Bool -> CommandStart
-start foreground = notBareRepo $ withStateMVar $ \st -> do
-	if foreground
-		then do
-			showStart "watch" "."
-			liftIO $ watch st
-		else do
-			logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
-			pidfile <- fromRepo gitAnnexPidFile
-			liftIO $ daemonize logfd (Just pidfile) False $ watch st
+stopOption :: Option
+stopOption = Option.flag [] "stop" "stop daemon"
+
+start :: Bool -> Bool -> CommandStart
+start foreground stopdaemon = notBareRepo $ do
+	if stopdaemon
+		then liftIO . stopDaemon =<< fromRepo gitAnnexPidFile
+		else withStateMVar $ startDaemon (not foreground)
 	stop
 
+startDaemon :: Bool -> MVar Annex.AnnexState -> Annex ()
+startDaemon False st = do
+	showStart "watch" "."
+	liftIO $ watch st
+startDaemon True st = do
+	logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
+	pidfile <- fromRepo gitAnnexPidFile
+	liftIO $ daemonize logfd (Just pidfile) False $ watch st
+
 watch :: MVar Annex.AnnexState -> IO ()
 #if defined linux_HOST_OS
 watch st = withINotify $ \i -> do
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index b41ebe2285..b09dce7399 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -27,7 +27,7 @@ daemonize logfd pidfile changedirectory a = do
 			_ <- forkProcess child2
 			out
 		child2 = do
-			maybe noop lockPidFile pidfile
+			maybe noop (lockPidFile True alreadyrunning) pidfile 
 			when changedirectory $
 				setCurrentDirectory "/"
 			nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags
@@ -39,12 +39,31 @@ daemonize logfd pidfile changedirectory a = do
 		redir newh h = do
 			closeFd h
 			dupTo newh h
+		alreadyrunning = error "Daemon is already running."
 		out = exitImmediately ExitSuccess
 
-lockPidFile :: FilePath -> IO ()
-lockPidFile file = void $ do
+lockPidFile :: Bool -> IO () -> FilePath -> IO ()
+lockPidFile write onfailure file = do
 	fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
-	catchIO
-		(setLock fd (WriteLock, AbsoluteSeek, 0, 0))
-		(const $ error "Daemon is already running.")
-	fdWrite fd =<< show <$> getProcessID
+	when (write) $ void $
+		fdWrite fd =<< show <$> getProcessID
+	catchIO (setLock fd (locktype, AbsoluteSeek, 0, 0)) (const onfailure)
+	where
+		locktype
+			| write = WriteLock
+			| otherwise = ReadLock
+
+{- Stops the daemon.
+ -
+ - The pid file is used to get the daemon's pid.
+ -
+ - To guard against a stale pid, try to take a nonblocking shared lock
+ - of the pid file. If this *fails*, the daemon must be running,
+ - and have the exclusive lock, so the pid file is trustworthy.
+ -}
+stopDaemon :: FilePath -> IO ()
+stopDaemon pidfile = lockPidFile False go pidfile
+	where
+		go = do
+			pid <- readish <$> readFile pidfile
+			maybe noop (signalProcess sigTERM) pid
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 8ff005d8d0..39fad04882 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -177,7 +177,8 @@ subdirectories).
   background, you no longer need to manually run git commands when
   manipulating your files.
 
-  To not daemonize, run with --foreground
+  To not daemonize, run with --foreground ; to stop a running daemon,
+  run with --stop
 
 # REPOSITORY SETUP COMMANDS
 

From 433ff41496b073c71e465af8b38b2ecafe27d8dd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 02:06:22 -0400
Subject: [PATCH 3707/8313] bugfix

---
 Utility/Daemon.hs | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index b09dce7399..3d2faed676 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -45,9 +45,11 @@ daemonize logfd pidfile changedirectory a = do
 lockPidFile :: Bool -> IO () -> FilePath -> IO ()
 lockPidFile write onfailure file = do
 	fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
-	when (write) $ void $
-		fdWrite fd =<< show <$> getProcessID
-	catchIO (setLock fd (locktype, AbsoluteSeek, 0, 0)) (const onfailure)
+	locked <- catchMaybeIO $ setLock fd (locktype, AbsoluteSeek, 0, 0)
+	case locked of
+		Nothing -> onfailure
+		_ -> when write $ void $
+			fdWrite fd =<< show <$> getProcessID
 	where
 		locktype
 			| write = WriteLock

From aba425fb2bab8e991e714931703c14b3fbcc7f10 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 02:09:22 -0400
Subject: [PATCH 3708/8313] update

---
 doc/design/assistant/inotify.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index a01bf51e78..9fe6938c4e 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -24,9 +24,10 @@ really useful, it needs to:
   **done**
 - periodically auto-commit staged changes (avoid autocommitting when
   lots of changes are coming in) **done**
-- tunable delays before adding new files, etc
 - coleasce related add/rm events for speed and less disk IO **done**
 - don't annex `.gitignore` and `.gitattributes` files **done**
+- run as a daemon **done**
+- tunable delays before adding new files, etc
 - configurable option to only annex files meeting certian size or
   filename criteria
 - option to check files not meeting annex criteria into git directly

From 28242b3bf8d75bf5b789830b66b90cec92e296d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 02:13:04 -0400
Subject: [PATCH 3709/8313] note

---
 doc/design/assistant/windows.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/design/assistant/windows.mdwn b/doc/design/assistant/windows.mdwn
index da669ad82c..26ff2c1c66 100644
--- a/doc/design/assistant/windows.mdwn
+++ b/doc/design/assistant/windows.mdwn
@@ -22,3 +22,15 @@ Or I could try to use Cygwin.
 ## Deeper system integration
 
 [NTFS Reparse Points](http://msdn.microsoft.com/en-us/library/aa365503%28v=VS.85%29.aspx) allow a program to define how the OS will interpret a file or directory in arbitrary ways.  This requires writing a file system filter.
+
+## Developement environment
+
+Someone wrote in to say:
+
+> For Windows Development you can easily qualify
+> for Bizspark - http://www.microsoft.com/bizspark/
+> 
+> This will get you 100% free Windows OS licenses and
+> Dev tools, plus a free Azure account for cloud testing.
+> (You can also now deploy Linux VMs to Azure as well)
+> No money required at all.

From 7753b33c8860486edf37936e30f35da9db7c8f8b Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Sun, 10 Jun 2012 19:24:15 -0700
Subject: [PATCH 3710/8313] Refactor generation of git-annex.cabal and
 incorporate man pages.

The existing `sed | find | perl` hack in the Makefile was not
including the man pages in the generated git-annex.cabal.  I couldn't
figure out why it didn't work; running the `find | perl` part of the
command *did* list the man pages ...

So, I set up a new hack. It produces a cleaner .cabal file and
includes the man pages in the sdist.  I changed git-annex.cabal and
its generation as follows:

- git-annex.cabal is now generated by a here document in
  git-annex.cabal.template.sh.  The here document has inline file list
  insertion, whereas before the file lists were inserted with sed.

- The 'Extra-Source-Files:' field now only includes the non-source
  files: the man pages, plain text documentation, and license.

- The source dependencies are now listed in 'Other-Modules' sections
  in the 'Executable' and 'Test-Suite' sections.  The list of
  dependencies is generated by `gen-other-modules.sh`.

- The ./debian and ./doc are no longer included in the sdist package.
  These were not installed anywhere by `cabal install`.  A user that
  wants them could clone the git repo.

Running the tests with cabal is not yet working, i.e.

  cabal configure --enable-tests && cabal build && cabal test

and

  cabal install --enable-tests

fail to find Utility.Touch.  However, I did not break this: it doesn't
work for the git-annex package on Hackage either.  Next step is to
figure out how to deal with HSC in cabal ... or not bother, because
`make test` works.  I'm worried this is a cabal bug.

To test building from sdist, I've been running

  cd ../.. ; cabal sdist ; cd dist ; tar xf git-annex-3.20120605.tar.gz && cd git-annex-3.20120605 && rm -fr /tmp/git-annex && cabal install --prefix=/tmp/git-annex && tree -A /tmp/git-annex

in the dist directory.  Using `cabal-dev install` is a better test,
but is very slow.
---
 .gitignore                  |  1 +
 Makefile                    | 18 +++++++++++-------
 gen-other-modules.sh        | 32 ++++++++++++++++++++++++++++++++
 git-annex.cabal.template.sh | 20 ++++++++++++++++++--
 4 files changed, 62 insertions(+), 9 deletions(-)
 create mode 100755 gen-other-modules.sh
 mode change 100644 => 100755 git-annex.cabal.template.sh

diff --git a/.gitignore b/.gitignore
index 1de0aaf722..74c0e672aa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@ test
 configure
 Build/SysConfig.hs
 git-annex
+git-annex.cabal
 git-annex.1
 git-annex-shell.1
 git-union-merge.1
diff --git a/Makefile b/Makefile
index 03bada30ef..131daade70 100644
--- a/Makefile
+++ b/Makefile
@@ -98,16 +98,20 @@ docs: $(mans)
 
 clean:
 	rm -rf tmp $(bins) $(mans) test configure  *.tix .hpc $(sources) \
-		doc/.ikiwiki html dist $(clibs)
+		doc/.ikiwiki html dist $(clibs) git-annex.cabal
 
 # Workaround for cabal sdist not running Setup hooks, so I cannot
 # generate a file list there.
-sdist: clean
-	@make $(mans)
-	@if [ ! -e git-annex.cabal.orig ]; then cp git-annex.cabal git-annex.cabal.orig; fi
-	@sed -e "s!\(Extra-Source-Files: \).*!\1$(shell find . -name .git -prune -or -not -name \\*.orig -not -type d -print | perl -ne 'print unless length >= 100')!i" < git-annex.cabal.orig > git-annex.cabal
-	@cabal sdist
-	@mv git-annex.cabal.orig git-annex.cabal
+sdist: clean $(mans)
+  # Could make this a .PHONY, but it needs to be rerun each time,
+  # unless we want to list a *lot* of dependencies.
+	./git-annex.cabal.template.sh > git-annex.cabal
+  # Complains about not running 'configure' first, but adding
+  #
+  #   cabal configure
+  #
+  # does not help.
+	cabal sdist
 
 # Upload to hackage.
 hackage: sdist
diff --git a/gen-other-modules.sh b/gen-other-modules.sh
new file mode 100755
index 0000000000..3c50d91e73
--- /dev/null
+++ b/gen-other-modules.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Generate module list for 'Other-Modules:' field in git-annex.cabal.
+# This would be simpler if the code were under ./src.
+
+find Annex Annex.hs \
+     Backend Backend.hs \
+     Build \
+     Checks.hs \
+     CmdLine.hs \
+     Command Command.hs \
+     Common Common.hs \
+     Config.hs \
+     Crypto.hs \
+     Git Git.hs \
+     GitAnnex.hs \
+     GitAnnexShell.hs \
+     Init.hs \
+     Limit.hs \
+     Locations.hs \
+     Logs \
+     Messages Messages.hs \
+     Option.hs \
+     Remote Remote.hs \
+     Seek.hs \
+     Setup.hs \
+     Types Types.hs \
+     Upgrade Upgrade.hs \
+     Usage.hs \
+     Utility \
+     -name '*.hs' \
+| sed -r -e 's!.hs!!' -e 's!/!.!g'
diff --git a/git-annex.cabal.template.sh b/git-annex.cabal.template.sh
old mode 100644
new mode 100755
index c151678319..46eaad7458
--- a/git-annex.cabal.template.sh
+++ b/git-annex.cabal.template.sh
@@ -1,3 +1,9 @@
+#!/bin/bash
+
+# Template for git-annex.cabal: the 'Other-Module:' fields are
+# dynamically generated.
+
+cat <= 1.8
@@ -7,7 +13,6 @@ Author: Joey Hess
 Stability: Stable
 Copyright: 2010-2012 Joey Hess
 License-File: GPL
-Extra-Source-Files: use-make-sdist-instead
 Homepage: http://git-annex.branchable.com/
 Build-type: Custom
 Category: Utility
@@ -25,6 +30,9 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
+Extra-Source-Files:
+  git-annex.1 git-annex-shell.1
+  INSTALL README CHANGELOG NEWS GPL
 
 Flag S3
   Description: Enable S3 support
@@ -36,7 +44,10 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
-  Other-Modules: Utility.Touch
+  Other-Modules:
+   Utility.Touch
+   -- Auto-generated list of all Haskell modules:
+`./gen-other-modules.sh | xargs -n1 -i echo '   '{}`
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
 
@@ -53,8 +64,13 @@ Test-Suite test
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
   C-Sources: Utility/libdiskfree.c
+  Other-Modules:
+   Utility.Touch
+   -- Auto-generated list of all Haskell modules:
+`./gen-other-modules.sh | xargs -n1 -i echo '   '{}`
   Extensions: CPP
 
 source-repository head
   type: git
   location: git://git-annex.branchable.com/
+EOF

From 7f70767bfb2ada474274c1dfad0e1a6d60955402 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 10:33:58 -0400
Subject: [PATCH 3711/8313] uninit: Refuse to run in a subdirectory. Closes:
 #677076

---
 Command/Uninit.hs | 6 +++++-
 debian/changelog  | 5 +++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index d6283a77da..5724bffd03 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -23,9 +23,13 @@ def = [addCheck check $ command "uninit" paramPaths seek
 
 check :: Annex ()
 check = do
-	b <- current_branch	
+	b <- current_branch
 	when (b == Annex.Branch.name) $ error $
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
+	top <- fromRepo Git.repoPath
+	cwd <- liftIO getCurrentDirectory
+	whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath cwd)) $ error $
+		"can only run uninit from the top of the git repository"
 	where
 		current_branch = Git.Ref . Prelude.head . lines <$> revhead
 		revhead = inRepo $ Git.Command.pipeRead 
diff --git a/debian/changelog b/debian/changelog
index 8a734e0aa7..a058a17961 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,10 +1,11 @@
-git-annex (3.20120606) UNRELEASED; urgency=low
+git-annex (3.20120611) unstable; urgency=medium
 
   * add: Prevent (most) modifications from being made to a file while it
     is being added to the annex.
   * initremote: Automatically describe a remote when creating it.
+  * uninit: Refuse to run in a subdirectory. Closes: #677076
 
- -- Joey Hess   Tue, 05 Jun 2012 20:25:51 -0400
+ -- Joey Hess   Mon, 11 Jun 2012 10:32:01 -0400
 
 git-annex (3.20120605) unstable; urgency=low
 

From 5a7b7d67f7b0bea262ffcafa1baea786ee68d65d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 10:44:16 -0400
Subject: [PATCH 3712/8313] releasing version 3.20120611

---
 git-annex.cabal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index c151678319..e74804b58f 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120605
+Version: 3.20120611
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From 5642e189b76070b43a8e24f9f49d36b950f83c8d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 10:45:33 -0400
Subject: [PATCH 3713/8313] add news item for git-annex 3.20120611

---
 doc/news/version_3.20120418.mdwn | 12 ------------
 doc/news/version_3.20120611.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 12 deletions(-)
 delete mode 100644 doc/news/version_3.20120418.mdwn
 create mode 100644 doc/news/version_3.20120611.mdwn

diff --git a/doc/news/version_3.20120418.mdwn b/doc/news/version_3.20120418.mdwn
deleted file mode 100644
index 93968a83e6..0000000000
--- a/doc/news/version_3.20120418.mdwn
+++ /dev/null
@@ -1,12 +0,0 @@
-git-annex 3.20120418 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * bugfix: Adding a dotfile also caused all non-dotfiles to be added.
-   * bup: Properly handle key names with spaces or other things that are
-     not legal git refs.
-   * git-annex (but not git-annex-shell) supports the git help.autocorrect
-     configuration setting, doing fuzzy matching using the restricted
-     Damerau-Levenshtein edit distance, just as git does. This adds a build
-     dependency on the haskell edit-distance library.
-   * Renamed diskfree.c to avoid OSX case insensativity bug.
-   * cabal now installs git-annex-shell as a symlink to git-annex.
-   * cabal file now autodetects whether S3 support is available."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120611.mdwn b/doc/news/version_3.20120611.mdwn
new file mode 100644
index 0000000000..e17456e236
--- /dev/null
+++ b/doc/news/version_3.20120611.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20120611 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * add: Prevent (most) modifications from being made to a file while it
+     is being added to the annex.
+   * initremote: Automatically describe a remote when creating it.
+   * uninit: Refuse to run in a subdirectory. Closes: #[677076](http://bugs.debian.org/677076)"""]]
\ No newline at end of file

From 129f6123fe933310829986fd5a99a9fd6911ca0f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 12:20:38 -0400
Subject: [PATCH 3714/8313] Build with ghc's threaded runtime, so threaded code
 does not busy-wait.

Sort of a work around for http://bugs.debian.org/677096
---
 Makefile         | 2 +-
 debian/changelog | 1 +
 git-annex.cabal  | 7 ++++---
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 2d8146c857..0819998d15 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd -ignore-package monads-tf
-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
+BASEFLAGS=-threaded -Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
diff --git a/debian/changelog b/debian/changelog
index 4ea0f46ac9..fcf8be9a94 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20120612) UNRELEASED; urgency=low
   * watch: New subcommand, which uses inotify to watch for changes to
     files and automatically annexes new files, etc, so you don't need
     to manually run git commands when manipulating files.
+  * Build with ghc's threaded runtime, so threaded code does not busy-wait.
 
  -- Joey Hess   Tue, 05 Jun 2012 20:25:51 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 3459eaae03..9bfb5eec60 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120611
+Version: 3.20120612
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -36,10 +36,11 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, STM
+   hinotify, stm
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
+  GHC-options: -threaded
 
   if flag(S3)
     Build-Depends: hS3
@@ -53,7 +54,7 @@ Test-Suite test
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, STM
+   hinotify, stm
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
 

From 0847a300fc65a832171af53272439d18f6e9ed6b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 12:46:35 -0400
Subject: [PATCH 3715/8313] Revert "Build with ghc's threaded runtime, so
 threaded code does not busy-wait."

This reverts commit 129f6123fe933310829986fd5a99a9fd6911ca0f.

Saw hang during batch add with -threaded, so deferred for now.
---
 Makefile         | 2 +-
 debian/changelog | 1 -
 git-annex.cabal  | 7 +++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 0819998d15..2d8146c857 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd -ignore-package monads-tf
-BASEFLAGS=-threaded -Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
diff --git a/debian/changelog b/debian/changelog
index fcf8be9a94..4ea0f46ac9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,7 +3,6 @@ git-annex (3.20120612) UNRELEASED; urgency=low
   * watch: New subcommand, which uses inotify to watch for changes to
     files and automatically annexes new files, etc, so you don't need
     to manually run git commands when manipulating files.
-  * Build with ghc's threaded runtime, so threaded code does not busy-wait.
 
  -- Joey Hess   Tue, 05 Jun 2012 20:25:51 -0400
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 9bfb5eec60..3459eaae03 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120612
+Version: 3.20120611
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -36,11 +36,10 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, stm
+   hinotify, STM
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
-  GHC-options: -threaded
 
   if flag(S3)
     Build-Depends: hS3
@@ -54,7 +53,7 @@ Test-Suite test
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, stm
+   hinotify, STM
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
 

From f7dbcd58ffb65258d914f1ecd6c0ff8f6bf2d3aa Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 14:24:13 -0400
Subject: [PATCH 3716/8313] tweak

---
 Command/Watch.hs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index f01a9616f0..76f03a0973 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -142,11 +142,10 @@ runStateMVar mvar a = do
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
 runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO ()
-runHandler st changechan hook file = handle =<< tryIO (runStateMVar st go)
+runHandler st changechan handler file = 
+	either (putStrLn . show) return =<< tryIO (runStateMVar st go)
 	where
-		go = maybe noop (signalChange changechan) =<< hook file
-		handle (Right ()) = return ()
-		handle (Left e) = putStrLn $ show e
+		go = maybe noop (signalChange changechan) =<< handler file
 
 {- Handlers call this when they made a change that needs to get committed. -}
 madeChange :: FilePath -> String -> Annex (Maybe Change)

From 7f3934520a8aa877be17e5d9a0d5dbc77b274c5e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 15:08:04 -0400
Subject: [PATCH 3717/8313] avoid using STM while the MVar is held

I thought this might be a lock conflict that explains the deadlock when
built with -threaded, but it seems not.. it still locks! It even locks
without the committer thread.

Indeed, it locks when running "git annex add"! -threaded is exposing some
other problem.

Still, this seems conceptually cleaner and did not add any inneficiencies.
Also added some high-level documentation about the threads used.
---
 Command/Watch.hs | 76 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 51 insertions(+), 25 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 76f03a0973..5c354cace9 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -1,13 +1,36 @@
-{- git-annex command
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE BangPatterns #-}
+
+{- git-annex watch daemon
+ -
+ - Overview of threads and MVars, etc:
+ -
+ - Thread 1: Parent
+ - 	The initial thread run, double forks to background, starts other
+ - 	threads, and then stops, waiting for them to terminate.
+ - Thread 2: inotify
+ - 	Notices new files, and calls handlers for events, queuing changes.
+ - Thread 3: inotify internal
+ - 	Used by haskell inotify library to ensure inotify event buffer is
+ - 	kept drained.
+ - Thread 4: committer
+ - 	Waits for changes to occur, and runs the git queue to update its
+ - 	index, then commits.
+ -
+ - State MVar:
+ - 	The Annex state is stored here, which allows recuscitating the
+ - 	Annex monad in IO actions run by the inotify and committer
+ - 	threads. Thus, a single state is shared amoung the threads, and
+ - 	only one at a time can access it.
+ - ChangeChan STM TChan:
+ - 	Changes are indicated by writing to this channel. The committer
+ - 	reads from it.
  -
  - Copyright 2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE CPP #-}
-{-# LANGUAGE BangPatterns #-}
-
 module Command.Watch where
 
 import Common.Annex
@@ -83,7 +106,7 @@ startDaemon True st = do
 watch :: MVar Annex.AnnexState -> IO ()
 #if defined linux_HOST_OS
 watch st = withINotify $ \i -> do
-	changechan <- atomically newTChan
+	changechan <- runChangeChan newTChan
 	let hook a = Just $ runHandler st changechan a
 	let hooks = WatchHooks
 		{ addHook = hook onAdd
@@ -131,26 +154,38 @@ withStateMVar a = do
 	return r
 
 {- Runs an Annex action, using the state from the MVar. -}
-runStateMVar :: MVar Annex.AnnexState -> Annex () -> IO ()
+runStateMVar :: MVar Annex.AnnexState -> Annex a -> IO a
 runStateMVar mvar a = do
+	liftIO $ putStrLn "takeMVar"
 	startstate <- takeMVar mvar
-	!newstate <- Annex.exec startstate a
+	!(r, newstate) <- Annex.run startstate a
+	liftIO $ putStrLn "putMVar"
 	putMVar mvar newstate
+	return r
 
-{- Runs an action handler, inside the Annex monad.
+runChangeChan :: STM a -> IO a
+runChangeChan = atomically
+
+{- Runs an action handler, inside the Annex monad, and if there was a
+ - change, adds it to the ChangeChan.
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
 runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO ()
-runHandler st changechan handler file = 
-	either (putStrLn . show) return =<< tryIO (runStateMVar st go)
-	where
-		go = maybe noop (signalChange changechan) =<< handler file
+runHandler st changechan handler file = void $ do
+	r <- tryIO (runStateMVar st $ handler file)
+	case r of
+		Left e -> putStrLn $ show e
+		Right Nothing -> noop
+		Right (Just change) -> void $
+			runChangeChan $ writeTChan changechan change
 
 {- Handlers call this when they made a change that needs to get committed. -}
 madeChange :: FilePath -> String -> Annex (Maybe Change)
-madeChange file desc = liftIO $ 
-	Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
+madeChange file desc = do
+	-- Just in case the commit thread is not flushing the queue fast enough.
+	Annex.Queue.flushWhenFull
+	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
 
 noChange :: Annex (Maybe Change)
 noChange = return Nothing
@@ -243,19 +278,10 @@ stageSymlink file sha =
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.stageSymlink file sha)
 
-{- Signals that a change has been made, that needs to get committed. -}
-signalChange :: ChangeChan -> Change -> Annex ()
-signalChange chan change = do
-	liftIO $ atomically $ writeTChan chan change
-
-	-- Just in case the commit thread is not flushing
-	-- the queue fast enough.
-	Annex.Queue.flushWhenFull
-
 {- Gets all unhandled changes.
  - Blocks until at least one change is made. -}
 getChanges :: ChangeChan -> IO [Change]
-getChanges chan = atomically $ do
+getChanges chan = runChangeChan $ do
 	c <- readTChan chan
 	go [c]
 	where
@@ -268,7 +294,7 @@ getChanges chan = atomically $ do
 {- Puts unhandled changes back into the channel.
  - Note: Original order is not preserved. -}
 refillChanges :: ChangeChan -> [Change] -> IO ()
-refillChanges chan cs = atomically $ mapM_ (writeTChan chan) cs
+refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs
 
 {- This thread makes git commits at appropriate times. -}
 commitThread :: MVar Annex.AnnexState -> ChangeChan -> IO ()

From d3a6f04abfd6cc1eb51b9311382fe143579fbed2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 15:41:26 -0400
Subject: [PATCH 3718/8313] update

---
 Command/Watch.hs | 15 ++++++++-------
 git-annex.cabal  |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 5c354cace9..e049591e9c 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -2,12 +2,17 @@
 {-# LANGUAGE BangPatterns #-}
 
 {- git-annex watch daemon
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
  -
  - Overview of threads and MVars, etc:
  -
- - Thread 1: Parent
+ - Thread 1: parent
  - 	The initial thread run, double forks to background, starts other
- - 	threads, and then stops, waiting for them to terminate.
+ - 	threads, and then stops, waiting for them to terminate,
+ - 	or for a ctrl-c.
  - Thread 2: inotify
  - 	Notices new files, and calls handlers for events, queuing changes.
  - Thread 3: inotify internal
@@ -18,17 +23,13 @@
  - 	index, then commits.
  -
  - State MVar:
- - 	The Annex state is stored here, which allows recuscitating the
+ - 	The Annex state is stored here, which allows resuscitating the
  - 	Annex monad in IO actions run by the inotify and committer
  - 	threads. Thus, a single state is shared amoung the threads, and
  - 	only one at a time can access it.
  - ChangeChan STM TChan:
  - 	Changes are indicated by writing to this channel. The committer
  - 	reads from it.
- -
- - Copyright 2012 Joey Hess 
- -
- - Licensed under the GNU GPL version 3 or higher.
  -}
 
 module Command.Watch where
diff --git a/git-annex.cabal b/git-annex.cabal
index 3459eaae03..5f94392f53 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120611
+Version: 3.20120612
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From 48dd2ff264ace9b725bb880822a3a3eefc1fc58a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 11 Jun 2012 16:44:45 -0400
Subject: [PATCH 3719/8313] blog for the day

---
 doc/design/assistant/blog/day_6__polish.mdwn | 50 ++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_6__polish.mdwn

diff --git a/doc/design/assistant/blog/day_6__polish.mdwn b/doc/design/assistant/blog/day_6__polish.mdwn
new file mode 100644
index 0000000000..dd1239626b
--- /dev/null
+++ b/doc/design/assistant/blog/day_6__polish.mdwn
@@ -0,0 +1,50 @@
+Since my last blog, I've been polishing the `git annex watch` command.
+
+First, I fixed the double commits problem. There's still some extra
+committing going on in the `git-annex` branch that I don't understand. It
+seems like a shutdown event is somehow being triggered whenever
+a git command is run by the commit thread.
+
+I also made `git annex watch` run as a proper daemon, with locking to
+prevent multiple copies running, and a pid file, and everything.
+I made `git annex watch --stop` stop it.
+
+---
+
+Then I managed to greatly increase its startup speed. At startup, it
+generates "add" events for every symlink in the tree. This is necessary
+because it doesn't really know if a symlink is already added, or was
+manually added before it starter, or indeed was added while it started up.
+Problem was that these events were causing a lot of work staging the
+symlinks -- most of which were already correctly staged.
+
+You'd think it could just check if the same symlink was in the index.
+But it can't, because the index is in a constant state of flux. The
+symlinks might have just been deleted and re-added, or changed, and
+the index still have the old value.
+
+Instead, I got creative. :) We can't trust what the index says about the
+symlink, but if the index happens to contian a symlink that looks right,
+we can trust that the SHA1 of its blob is the right SHA1, and reuse it
+when re-staging the symlink. Wham! Massive speedup!
+
+---
+
+Then I started running `git annex watch` on my own real git annex repos,
+and noticed some problems.. Like it turns normal files already checked into
+git into symlinks. And it leaks memory scanning a big tree. Oops..
+
+---
+
+I put together a quick screencast demoing `git annex watch`.
+
+
+
+While making the screencast, I noticed that `git-annex watch` was spinning
+in strace, which is bad news for powertop and battery usage. This seems to
+be a [GHC bug](http://bugs.debian.org/677096) also affecting Xmonad. I
+tried switching to GHC's threaded runtime, which solves that problem, but
+causes git-annex to hang under heavy load. Tried to debug that for quite a
+while, but didn't get far. Will need to investigate this further.. 
+Am seeing indications that this problem only affects ghc 7.4.1; in
+particular 7.4.2 does not seem to have the problem.

From 236ae1fbc4fa114cfcef5a7d71c40cb1c2ef9d8f Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Mon, 11 Jun 2012 23:27:00 -0700
Subject: [PATCH 3720/8313] Add link COPYRIGHT -> debian/copyright; add
 COPYRIGHT to sdist.

---
 COPYRIGHT                   | 1 +
 git-annex.cabal.template.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 120000 COPYRIGHT

diff --git a/COPYRIGHT b/COPYRIGHT
new file mode 120000
index 0000000000..9060ce8208
--- /dev/null
+++ b/COPYRIGHT
@@ -0,0 +1 @@
+debian/copyright
\ No newline at end of file
diff --git a/git-annex.cabal.template.sh b/git-annex.cabal.template.sh
index 46eaad7458..5ff8155e90 100755
--- a/git-annex.cabal.template.sh
+++ b/git-annex.cabal.template.sh
@@ -32,7 +32,7 @@ Description:
  revision control.
 Extra-Source-Files:
   git-annex.1 git-annex-shell.1
-  INSTALL README CHANGELOG NEWS GPL
+  INSTALL README CHANGELOG NEWS GPL COPYRIGHT
 
 Flag S3
   Description: Enable S3 support

From 9fe433390d3860bdb740c54b015889805578c992 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Tue, 12 Jun 2012 00:39:21 -0700
Subject: [PATCH 3721/8313] Remove .dir-locals.el and add
 doc/contributing.mdwn.

Add link CONTRIBUTING -> doc/contributing.mdwn, so that it's easy to
find (many files in doc/).

Add .dir-locals.el to .gitignore, now that it's no longer versioned.

The CONTRIBUTING file gives a reference to a page on the Emacs wiki
that shows how to set up a .dir-locals.el that sets up tabs for
indentation.  I updated the wiki page to include the
`(highlight-regexp "^ *")` part, which had been the hardest to
discover.
---
 .dir-locals.el        | 22 ----------------------
 .gitignore            |  3 +++
 CONTRIBUTING          |  1 +
 doc/contributing.mdwn |  7 +++++++
 4 files changed, 11 insertions(+), 22 deletions(-)
 delete mode 100644 .dir-locals.el
 create mode 120000 CONTRIBUTING
 create mode 100644 doc/contributing.mdwn

diff --git a/.dir-locals.el b/.dir-locals.el
deleted file mode 100644
index e523528e2f..0000000000
--- a/.dir-locals.el
+++ /dev/null
@@ -1,22 +0,0 @@
-;; Configure emacs' treatment of tabs.
-;;
-;; See
-;; https://www.gnu.org/software/emacs/manual/html_node/emacs/Directory-Variables.html
-;; for a description of this file.
-;;
-;; The 'nil' below applies to all modes.
-((nil . ((indent-tabs-mode . t)
-        (tab-width . 2)))
- (haskell-mode . (
-        ;; Highlight leading space characters, to avoid indenting with
-        ;; spaces.
-        ;;
-        ;; Emacs will prompt you about this, saying it's unsafe, but
-        ;; you can permanently store an exception by pressing "!",
-        ;; which inserts
-        ;;
-        ;;   (safe-local-variable-values . (quote ((eval highlight-regexp "^ *"))))
-        ;;
-        ;; in your ~/.emacs ... except the exception doesn't work, and
-        ;; emacs still asks you on each file you open :P
-        (eval . (highlight-regexp "^ *")))))
diff --git a/.gitignore b/.gitignore
index 74c0e672aa..e176e59c2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,7 @@ html
 Utility/Touch.hs
 Utility/libdiskfree.o
 dist
+# Sandboxed builds
 cabal-dev
+# Project-local emacs configuration
+.dir-locals.el
diff --git a/CONTRIBUTING b/CONTRIBUTING
new file mode 120000
index 0000000000..8ad193e227
--- /dev/null
+++ b/CONTRIBUTING
@@ -0,0 +1 @@
+doc/contributing.mdwn
\ No newline at end of file
diff --git a/doc/contributing.mdwn b/doc/contributing.mdwn
new file mode 100644
index 0000000000..11c6c19871
--- /dev/null
+++ b/doc/contributing.mdwn
@@ -0,0 +1,7 @@
+## Style
+
+This project uses tabs for indentation and the code looks fine with
+any tab width. If you are using Emacs, and have it configured to use
+spaces for indentation, then you can add a ./.dir-locals.el to use
+tabs for files in this project. See
+http://www.emacswiki.org/emacs/DirectoryVariables.

From 72b0054931cd0d41477fa7270154cb53d8e94e4d Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Tue, 12 Jun 2012 00:53:35 -0700
Subject: [PATCH 3722/8313] Get ready for a simple git-annex.cabal.

I have a new idea: instead of the template-based approaches that work
around cabals requirement that you list all files to put in the sdist,
we can simply generate the sdist ourselves, with the files we
want. Take that cabal!
---
 gen-other-modules.sh                          | 32 -------------------
 ...annex.cabal.template.sh => git-annex.cabal |  0
 2 files changed, 32 deletions(-)
 delete mode 100755 gen-other-modules.sh
 rename git-annex.cabal.template.sh => git-annex.cabal (100%)

diff --git a/gen-other-modules.sh b/gen-other-modules.sh
deleted file mode 100755
index 3c50d91e73..0000000000
--- a/gen-other-modules.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Generate module list for 'Other-Modules:' field in git-annex.cabal.
-# This would be simpler if the code were under ./src.
-
-find Annex Annex.hs \
-     Backend Backend.hs \
-     Build \
-     Checks.hs \
-     CmdLine.hs \
-     Command Command.hs \
-     Common Common.hs \
-     Config.hs \
-     Crypto.hs \
-     Git Git.hs \
-     GitAnnex.hs \
-     GitAnnexShell.hs \
-     Init.hs \
-     Limit.hs \
-     Locations.hs \
-     Logs \
-     Messages Messages.hs \
-     Option.hs \
-     Remote Remote.hs \
-     Seek.hs \
-     Setup.hs \
-     Types Types.hs \
-     Upgrade Upgrade.hs \
-     Usage.hs \
-     Utility \
-     -name '*.hs' \
-| sed -r -e 's!.hs!!' -e 's!/!.!g'
diff --git a/git-annex.cabal.template.sh b/git-annex.cabal
similarity index 100%
rename from git-annex.cabal.template.sh
rename to git-annex.cabal

From 79a71d9ba68818a62cbf96f3cf46d01ac662f836 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Tue, 12 Jun 2012 02:36:05 -0700
Subject: [PATCH 3723/8313] Simplify git-annex.cabal and generate sdist with
 make-sdist.sh.

The `cabal install` is happy as long as the files it needs are
present, but `cabal sdist` will only package up files you tell it to.
So, generate the source tarball ourselves.

The source tarball is generated by make-sdist.sh, which uses cabal
sdist to calculate the package name.  Could also generate the name
from the 'Version:' field in git-annex.cabal.
---
 Makefile        | 16 ++++------------
 git-annex.cabal | 24 +++++-------------------
 make-sdist.sh   | 15 +++++++++++++++
 3 files changed, 24 insertions(+), 31 deletions(-)
 mode change 100755 => 100644 git-annex.cabal
 create mode 100755 make-sdist.sh

diff --git a/Makefile b/Makefile
index ca9ea2036b..d23f08a7d1 100644
--- a/Makefile
+++ b/Makefile
@@ -98,20 +98,12 @@ docs: $(mans)
 
 clean:
 	rm -rf tmp $(bins) $(mans) test configure  *.tix .hpc $(sources) \
-		doc/.ikiwiki html dist $(clibs) git-annex.cabal
+		doc/.ikiwiki html dist $(clibs)
 
-# Workaround for cabal sdist not running Setup hooks, so I cannot
-# generate a file list there.
+# Workaround for `cabal sdist` requiring all included files to be listed
+# in .cabal.
 sdist: clean $(mans)
-  # Could make this a .PHONY, but it needs to be rerun each time,
-  # unless we want to list a *lot* of dependencies.
-	./git-annex.cabal.template.sh > git-annex.cabal
-  # Complains about not running 'configure' first, but adding
-  #
-  #   cabal configure
-  #
-  # does not help.
-	cabal sdist
+	./make-sdist.sh
 
 # Upload to hackage.
 hackage: sdist
diff --git a/git-annex.cabal b/git-annex.cabal
old mode 100755
new mode 100644
index 5ff8155e90..ac039bc636
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,11 +1,5 @@
-#!/bin/bash
-
-# Template for git-annex.cabal: the 'Other-Module:' fields are
-# dynamically generated.
-
-cat <= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -30,9 +24,6 @@ Description:
  versioned files, which is convenient for maintaining documents, Makefiles,
  etc that are associated with annexed files but that benefit from full
  revision control.
-Extra-Source-Files:
-  git-annex.1 git-annex-shell.1
-  INSTALL README CHANGELOG NEWS GPL COPYRIGHT
 
 Flag S3
   Description: Enable S3 support
@@ -44,10 +35,8 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
-  Other-Modules:
-   Utility.Touch
-   -- Auto-generated list of all Haskell modules:
-`./gen-other-modules.sh | xargs -n1 -i echo '   '{}`
+  -- Need to list this because it's generated from a .hsc file.
+  Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
 
@@ -55,6 +44,7 @@ Executable git-annex
     Build-Depends: hS3
     CPP-Options: -DWITH_S3
 
+-- XXX: Broken.
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs
@@ -63,14 +53,10 @@ Test-Suite test
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance
+  Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
-  Other-Modules:
-   Utility.Touch
-   -- Auto-generated list of all Haskell modules:
-`./gen-other-modules.sh | xargs -n1 -i echo '   '{}`
   Extensions: CPP
 
 source-repository head
   type: git
   location: git://git-annex.branchable.com/
-EOF
diff --git a/make-sdist.sh b/make-sdist.sh
new file mode 100755
index 0000000000..debc5195e4
--- /dev/null
+++ b/make-sdist.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Create target directory
+cabal sdist
+sdist_dir=$(basename dist/*.tar.gz .tar.gz)
+rm -f dist/*.tar.gz
+mkdir dist/$sdist_dir
+
+find . \( -name .git -or -name dist -or -name cabal-dev \) -prune \
+	-or -not -name \\*.orig -not -type d -print \
+| perl -ne 'print unless length >= 100' \
+| xargs cp --parents --target-directory dist/$sdist_dir
+
+cd dist
+tar -caf $sdist_dir.tar.gz $sdist_dir

From 2e0965eba9fa024f0df308d663fc584eb43c2726 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Tue, 12 Jun 2012 02:45:53 -0700
Subject: [PATCH 3724/8313] Generate sdist tarball name from git-annex.cabal
 'Version:'.

Instead of generating with `cabal sdist`.
---
 make-sdist.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/make-sdist.sh b/make-sdist.sh
index debc5195e4..4098719160 100755
--- a/make-sdist.sh
+++ b/make-sdist.sh
@@ -1,10 +1,8 @@
 #!/bin/bash
 
 # Create target directory
-cabal sdist
-sdist_dir=$(basename dist/*.tar.gz .tar.gz)
-rm -f dist/*.tar.gz
-mkdir dist/$sdist_dir
+sdist_dir=git-annex-$(grep '^Version:' git-annex.cabal | sed -re 's/Version: *//')
+mkdir --parents dist/$sdist_dir
 
 find . \( -name .git -or -name dist -or -name cabal-dev \) -prune \
 	-or -not -name \\*.orig -not -type d -print \

From 000bbba3c0fe0e2abbee9646428720edf765ae01 Mon Sep 17 00:00:00 2001
From: Nathan Collins 
Date: Tue, 12 Jun 2012 02:50:09 -0700
Subject: [PATCH 3725/8313] Clean up Setup.hs.

Remove post-copy hook 'myPostCopy': it's easy to write one based on
'myPostInst', so just wait until someone complains that it's missing.

Remove most comments.

Put long type sigs on one line like in the other source files.
---
 Setup.hs | 50 ++++----------------------------------------------
 1 file changed, 4 insertions(+), 46 deletions(-)

diff --git a/Setup.hs b/Setup.hs
index 3f282e0aa9..faca87cd24 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -14,15 +14,13 @@ import qualified Build.Configure as Configure
 main = defaultMainWithHooks simpleUserHooks
 	{ preConf = configure
 	, postInst = myPostInst
-	, postCopy = myPostCopy
 	}
 
 configure _ _ = do
 	Configure.run Configure.tests
 	return (Nothing, [])
 
-myPostInst :: Args -> InstallFlags -> PackageDescription
-           -> LocalBuildInfo -> IO ()
+myPostInst :: Args -> InstallFlags -> PackageDescription -> LocalBuildInfo -> IO ()
 myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do
 	installGitAnnexShell dest verbosity pkg lbi
 	installManpages      dest verbosity pkg lbi
@@ -30,27 +28,7 @@ myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do
 		dest      = NoCopyDest
 		verbosity = fromFlag installVerbosity
 
--- ???: Not sure how you're supposed to use this.  E.g., when I do
---
---    cabal install --prefix=/tmp/git-annex-install
---    cabal copy --deistdir=/tmp/git-annex-copy
---
--- I get the copy under
---
---   /tmp/git-annex-copy/tmp/git-annex-install
---
--- Also, `cabal install` fails when given a relative --prefix.
-myPostCopy :: Args -> CopyFlags -> PackageDescription
-           -> LocalBuildInfo -> IO ()
-myPostCopy _ (CopyFlags { copyDest, copyVerbosity }) pkg lbi = do
-	installGitAnnexShell dest verbosity pkg lbi
-	installManpages      dest verbosity pkg lbi
-	where
-		dest      = fromFlag copyDest
-		verbosity = fromFlag copyVerbosity
-
-installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription
-                     -> LocalBuildInfo -> IO ()
+installGitAnnexShell :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO ()
 installGitAnnexShell copyDest verbosity pkg lbi =
 	rawSystemExit verbosity "ln"
 		["-sf", "git-annex", dstBinDir  "git-annex-shell"]
@@ -59,28 +37,8 @@ installGitAnnexShell copyDest verbosity pkg lbi =
 
 -- See http://www.haskell.org/haskellwiki/Cabal/Developer-FAQ#Installing_manpages.
 --
--- Based on pandoc's and lhs2tex's 'Setup.installManpages' and
--- 'postInst' hooks.
---
--- My understanding: 'postCopy' is run for `cabal copy`, 'postInst' is
--- run for `cabal inst`, and copy is not a generalized install, so you
--- have to write two nearly identical hooks. 
---
--- Summary of hooks:
--- http://www.haskell.org/cabal/release/cabal-latest/doc/API/Cabal/Distribution-Simple-UserHooks.htm--
--- Other people are also confused:
---
--- * Bug: 'postCopy' and 'postInst' are confusing:
--- http://hackage.haskell.org/trac/hackage/ticket/718
---
--- * A cabal maintainer suggests using 'postCopy' instead of
--- 'postInst', because `cabal install` is `cabal copy` followed by
--- `cabal register`:
--- http://www.haskell.org/pipermail/libraries/2008-March/009416.html
--- Although that sounds desirable, it's not true, as the reply and
--- experiments indicate.
-installManpages :: CopyDest -> Verbosity -> PackageDescription
-                -> LocalBuildInfo -> IO ()
+-- Based on pandoc's Setup.hs.
+installManpages :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO ()
 installManpages copyDest verbosity pkg lbi =
 	installOrdinaryFiles verbosity dstManDir srcManpages
 	where

From 942d8f72984377c4e69d7c55877621d434e5d687 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 11:32:06 -0400
Subject: [PATCH 3726/8313] hlint

---
 Annex/Branch.hs      | 2 +-
 Annex/Content.hs     | 2 +-
 Command/Commit.hs    | 2 +-
 Command/Fsck.hs      | 6 +++---
 Command/Get.hs       | 2 +-
 Command/Move.hs      | 4 ++--
 Command/Uninit.hs    | 4 ++--
 Command/Watch.hs     | 6 +++---
 Command/Whereis.hs   | 2 +-
 Config.hs            | 2 +-
 Crypto.hs            | 2 +-
 Locations.hs         | 2 +-
 Messages.hs          | 2 +-
 Utility/Directory.hs | 3 +--
 14 files changed, 20 insertions(+), 21 deletions(-)

diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index 7b433cc6ea..8e7f45a4ad 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -68,7 +68,7 @@ siblingBranches = inRepo $ Git.Ref.matchingUniq name
 
 {- Creates the branch, if it does not already exist. -}
 create :: Annex ()
-create = void $ getBranch
+create = void getBranch
 
 {- Returns the ref of the branch, creating it first if necessary. -}
 getBranch :: Annex Git.Ref
diff --git a/Annex/Content.hs b/Annex/Content.hs
index 232b43b2c3..3e3e958686 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -87,7 +87,7 @@ lockContent key a = do
 		 - to fiddle with permissions to open for an exclusive lock. -}
 		openforlock f = catchMaybeIO $ ifM (doesFileExist f)
 			( withModifiedFileMode f
-				(\cur -> cur `unionFileModes` ownerWriteMode)
+				(`unionFileModes` ownerWriteMode)
 				open
 			, open
 			)
diff --git a/Command/Commit.hs b/Command/Commit.hs
index 1c82ed7df8..f53ab7e097 100644
--- a/Command/Commit.hs
+++ b/Command/Commit.hs
@@ -22,7 +22,7 @@ seek = [withNothing start]
 start :: CommandStart
 start = next $ next $ do
 	Annex.Branch.commit "update"
-	_ <- runhook =<< (inRepo $ Git.hookPath "annex-content")
+	_ <- runhook =<< inRepo (Git.hookPath "annex-content")
 	return True
 	where
 		runhook (Just hook) = liftIO $ boolSystem hook []
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index 1fc656207f..7bfc46f4a6 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -145,13 +145,13 @@ fixLink key file = do
 		 -}
 		whenM (liftIO $ doesFileExist file) $
 			unlessM (inAnnex key) $ do
-				showNote $ "fixing content location"
+				showNote "fixing content location"
 				dir <- liftIO $ parentDir <$> absPath file
 				let content = absPathFrom dir have
 				liftIO $ allowWrite (parentDir content)
 				moveAnnex key content
 
-		showNote $ "fixing link"
+		showNote "fixing link"
 		liftIO $ createDirectoryIfMissing True (parentDir file)
 		liftIO $ removeFile file
 		liftIO $ createSymbolicLink want file
@@ -220,7 +220,7 @@ checkKeySize' key file bad = case Types.Key.keySize key of
 	Nothing -> return True
 	Just size -> do
 		size' <- fromIntegral . fileSize
-			<$> (liftIO $ getFileStatus file)
+			<$> liftIO (getFileStatus file)
 		comparesizes size size'
 	where
 		comparesizes a b = do
diff --git a/Command/Get.hs b/Command/Get.hs
index 772fbd90c2..c4ba483126 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -26,7 +26,7 @@ start from file (key, _) = stopUnless (not <$> inAnnex key) $
 	autoCopies file key (<) $ \_numcopies ->
 		case from of
 			Nothing -> go $ perform key
-			Just src -> do
+			Just src ->
 				-- get --from = copy --from
 				stopUnless (Command.Move.fromOk src key) $
 					go $ Command.Move.fromPerform src False key
diff --git a/Command/Move.hs b/Command/Move.hs
index 8612c9f2db..6ec7cd90ab 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -128,9 +128,9 @@ fromOk src key
 		expensive = do
 			u <- getUUID
 			remotes <- Remote.keyPossibilities key
-			return $ u /= Remote.uuid src && any (== src) remotes
+			return $ u /= Remote.uuid src && elem src remotes
 fromPerform :: Remote -> Bool -> Key -> CommandPerform
-fromPerform src move key = moveLock move key $ do
+fromPerform src move key = moveLock move key $
 	ifM (inAnnex key)
 		( handle move True
 		, do
diff --git a/Command/Uninit.hs b/Command/Uninit.hs
index 5724bffd03..46a2480e64 100644
--- a/Command/Uninit.hs
+++ b/Command/Uninit.hs
@@ -28,8 +28,8 @@ check = do
 		"cannot uninit when the " ++ show b ++ " branch is checked out"
 	top <- fromRepo Git.repoPath
 	cwd <- liftIO getCurrentDirectory
-	whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath cwd)) $ error $
-		"can only run uninit from the top of the git repository"
+	whenM ((/=) <$> liftIO (absPath top) <*> liftIO (absPath cwd)) $
+		error "can only run uninit from the top of the git repository"
 	where
 		current_branch = Git.Ref . Prelude.head . lines <$> revhead
 		revhead = inRepo $ Git.Command.pipeRead 
diff --git a/Command/Watch.hs b/Command/Watch.hs
index e049591e9c..0ee932dba2 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -176,7 +176,7 @@ runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO (
 runHandler st changechan handler file = void $ do
 	r <- tryIO (runStateMVar st $ handler file)
 	case r of
-		Left e -> putStrLn $ show e
+		Left e -> print e
 		Right Nothing -> noop
 		Right (Just change) -> void $
 			runChangeChan $ writeTChan changechan change
@@ -236,7 +236,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 		 - So for speed, tries to reuse the existing blob for
 		 - the symlink target. -}
 		addlink link = do
-			v <- catObjectDetails $ Ref $ ":" ++ file
+			v <- catObjectDetails $ Ref $ ':':file
 			case v of
 				Just (currlink, sha)
 					| s2w8 link == L.unpack currlink ->
@@ -307,7 +307,7 @@ commitThread st changechan = forever $ do
 	-- Now see if now's a good time to commit.
 	time <- getCurrentTime
 	if shouldCommit time cs
-		then void $ tryIO $ runStateMVar st $ commitStaged
+		then void $ tryIO $ runStateMVar st commitStaged
 		else refillChanges changechan cs
 	where
 		oneSecond = 1000000 -- microseconds
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index eb6ea7c56d..b697bf5541 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -37,7 +37,7 @@ perform remotemap key = do
 	unless (null safelocations) $ showLongNote pp
 	pp' <- prettyPrintUUIDs "untrusted" untrustedlocations
 	unless (null untrustedlocations) $ showLongNote $ untrustedheader ++ pp'
-	forM_ (catMaybes $ map (`M.lookup` remotemap) locations) $
+	forM_ (mapMaybe (`M.lookup` remotemap) locations) $
 		performRemote key
 	if null safelocations then stop else next $ return True
 	where
diff --git a/Config.hs b/Config.hs
index f579e40b21..e66947e2cc 100644
--- a/Config.hs
+++ b/Config.hs
@@ -114,6 +114,6 @@ getDiskReserve = fromMaybe megabyte . readSize dataUnits
 getHttpHeaders :: Annex [String]
 getHttpHeaders = do
 	cmd <- getConfig (annexConfig "http-headers-command") ""
-	if (null cmd)
+	if null cmd
 		then fromRepo $ Git.Config.getList "annex.http-headers"
 		else lines . snd <$> liftIO (pipeFrom "sh" ["-c", cmd])
diff --git a/Crypto.hs b/Crypto.hs
index 58c0e6d008..8941f7637b 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -138,7 +138,7 @@ withDecryptedContent = pass withDecryptedHandle
 
 pass :: (Cipher -> IO L.ByteString -> (Handle -> IO a) -> IO a) 
       -> Cipher -> IO L.ByteString -> (L.ByteString -> IO a) -> IO a
-pass to n s a = to n s $ \h -> a =<< L.hGetContents h
+pass to n s a = to n s $ a <=< L.hGetContents
 
 hmacWithCipher :: Cipher -> String -> String
 hmacWithCipher c = hmacWithCipher' (cipherHmac c) 
diff --git a/Locations.hs b/Locations.hs
index 9d27fbdae1..0c9935614d 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -165,7 +165,7 @@ gitAnnexRemotesDir r = addTrailingPathSeparator $ gitAnnexDir r  "remotes"
 
 {- Checks a symlink target to see if it appears to point to annexed content. -}
 isLinkToAnnex :: FilePath -> Bool
-isLinkToAnnex s = ("/" ++ d) `isInfixOf` s || d `isPrefixOf` s
+isLinkToAnnex s = ('/':d) `isInfixOf` s || d `isPrefixOf` s
 	where
 		d = ".git"  objectDir
 
diff --git a/Messages.hs b/Messages.hs
index 96bf3ae4b5..1b48c119b2 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -183,7 +183,7 @@ setupConsole = do
 	fileEncoding stderr
 
 handle :: IO () -> IO () -> Annex ()
-handle json normal = withOutputType $ go
+handle json normal = withOutputType go
 	where
 		go NormalOutput = liftIO normal
 		go QuietOutput = q
diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 52f2396d75..78bb6e7016 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -10,12 +10,11 @@ module Utility.Directory where
 import System.IO.Error
 import System.Posix.Files
 import System.Directory
-import Control.Exception (throw)
+import Control.Exception (throw, bracket_)
 import Control.Monad
 import Control.Monad.IfElse
 import System.FilePath
 import Control.Applicative
-import Control.Exception (bracket_)
 import System.Posix.Directory
 import System.IO.Unsafe (unsafeInterleaveIO)
 

From 66a961b1b70c17e84379168d713cfa19728ad446 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 11:35:24 -0400
Subject: [PATCH 3727/8313] remove cabal file

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index e176e59c2f..d628f23b7d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,6 @@ test
 configure
 Build/SysConfig.hs
 git-annex
-git-annex.cabal
 git-annex.1
 git-annex-shell.1
 git-union-merge.1

From 7b3b6e4654f51b0eec62af912d24e7e7c331c716 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 11:35:32 -0400
Subject: [PATCH 3728/8313] cleanup

---
 doc/contributing.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/contributing.mdwn b/doc/contributing.mdwn
index 11c6c19871..c3ab3d69df 100644
--- a/doc/contributing.mdwn
+++ b/doc/contributing.mdwn
@@ -2,6 +2,6 @@
 
 This project uses tabs for indentation and the code looks fine with
 any tab width. If you are using Emacs, and have it configured to use
-spaces for indentation, then you can add a ./.dir-locals.el to use
+spaces for indentation, then you can add a `./.dir-locals.el` to use
 tabs for files in this project. See
-http://www.emacswiki.org/emacs/DirectoryVariables.
+.

From 0e944fd0e9b868929f54623848bd2473709595a0 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 11:36:42 -0400
Subject: [PATCH 3729/8313] Install man page when run by cabal, in a location
 where man will find it, even when installing under $HOME. Thanks, Nathan
 Collins

---
 debian/changelog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index a058a17961..67ecdda45d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120612) UNRELEASED; urgency=low
+
+  * Install man page when run by cabal, in a location where man will
+    find it, even when installing under $HOME. Thanks, Nathan Collins
+
+ -- Joey Hess   Tue, 12 Jun 2012 11:35:59 -0400
+
 git-annex (3.20120611) unstable; urgency=medium
 
   * add: Prevent (most) modifications from being made to a file while it

From 5f5f7b1a9f6d5c4361e44dcd46d3da6edc6a625a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 11:38:16 -0400
Subject: [PATCH 3730/8313] tweak

---
 Setup.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Setup.hs b/Setup.hs
index faca87cd24..80d23cc878 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -1,4 +1,5 @@
 {-# LANGUAGE NamedFieldPuns #-}
+
 {- cabal setup file -}
 
 import Distribution.Simple

From 4ebb0b51d77484fcee12fd92a71b737b1aaca283 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 11:48:31 -0400
Subject: [PATCH 3731/8313] remove xxx ... cabal test works for me

---
 git-annex.cabal | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index 7b74cd2527..9703b61f03 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120611
+Version: 3.20120612
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -44,7 +44,6 @@ Executable git-annex
     Build-Depends: hS3
     CPP-Options: -DWITH_S3
 
--- XXX: Broken.
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs

From d3b9b32f2193520242fecf973fd1d3c1f2254003 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 13:54:00 -0400
Subject: [PATCH 3732/8313] cleanup

---
 Command/Watch.hs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 0ee932dba2..1c4f61a55f 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -157,10 +157,8 @@ withStateMVar a = do
 {- Runs an Annex action, using the state from the MVar. -}
 runStateMVar :: MVar Annex.AnnexState -> Annex a -> IO a
 runStateMVar mvar a = do
-	liftIO $ putStrLn "takeMVar"
 	startstate <- takeMVar mvar
 	!(r, newstate) <- Annex.run startstate a
-	liftIO $ putStrLn "putMVar"
 	putMVar mvar newstate
 	return r
 

From 535d9e49984daa7669f8bbc6e83ef2f7989c21c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 14:34:09 -0400
Subject: [PATCH 3733/8313] add a flag indicating if an event was synthesized
 during initial dir scan

---
 Command/Watch.hs   |  5 ++++-
 Utility/Inotify.hs | 32 +++++++++++++++++++-------------
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 1c4f61a55f..5564df7bcd 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -18,7 +18,10 @@
  - Thread 3: inotify internal
  - 	Used by haskell inotify library to ensure inotify event buffer is
  - 	kept drained.
- - Thread 4: committer
+ - Thread 4: inotify initial scan
+ -	A MVar lock is used to prevent other inotify handlers from running
+ -	until this is complete.
+ - Thread 5: committer
  - 	Waits for changes to occur, and runs the git queue to update its
  - 	index, then commits.
  -
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 7329b51229..ad0c21b227 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -15,7 +15,11 @@ import qualified System.Posix.Files as Files
 import System.IO.Error
 import Control.Exception (throw)
 
-type Hook a = Maybe (a -> IO ())
+{- A hook is passed some value to act on.
+ -
+ - The Bool is False when we're in the intial scan of a directory tree,
+ - rather than having received a genuine inotify event. -}
+type Hook a = Maybe (a -> Bool -> IO ())
 
 data WatchHooks = WatchHooks
 	{ addHook :: Hook FilePath
@@ -31,7 +35,7 @@ data WatchHooks = WatchHooks
  - made for different events.
  -
  - Inotify is weak at recursive directory watching; the whole directory
- - tree must be walked and watches set explicitly for each subdirectory.
+ - tree must be scanned and watches set explicitly for each subdirectory.
  -
  - To notice newly created subdirectories, inotify is used, and
  - watches are registered for those directories. There is a race there;
@@ -65,14 +69,14 @@ watchDir i dir ignored hooks
 		void (addWatch i watchevents dir handler)
 			`catchIO` failedaddwatch
 		withLock lock $
-			mapM_ walk =<< filter (not . dirCruft) <$>
+			mapM_ scan =<< filter (not . dirCruft) <$>
 				getDirectoryContents dir
 	where
 		recurse d = watchDir i d ignored hooks
 
 		-- Select only inotify events required by the enabled
 		-- hooks, but always include Create so new directories can
-		-- be walked.
+		-- be scanned.
 		watchevents = Create : addevents ++ delevents
 		addevents
 			| hashook addHook || hashook addSymlinkHook = [MoveIn, CloseWrite]
@@ -81,15 +85,15 @@ watchDir i dir ignored hooks
 			| hashook delHook || hashook delDirHook = [MoveOut, Delete]
 			| otherwise = []
 
-		walk f = unless (ignored f) $ do
+		scan f = unless (ignored f) $ do
 			let fullf = indir f
 			r <- catchMaybeIO $ getSymbolicLinkStatus fullf
 			case r of
 				Nothing -> return ()
 				Just s
 					| Files.isDirectory s -> recurse fullf
-					| Files.isSymbolicLink s -> addSymlinkHook <@> f
-					| Files.isRegularFile s -> addHook <@> f
+					| Files.isSymbolicLink s -> addSymlinkHook <@?> f
+					| Files.isRegularFile s -> addHook <@?> f
 					| otherwise -> return ()
 
 		-- Ignore creation events for regular files, which won't be
@@ -105,9 +109,9 @@ watchDir i dir ignored hooks
 		go (Closed { isDirectory = False, maybeFilePath = Just f }) =
 			whenM (filetype Files.isRegularFile f) $
 				addHook <@> f
-		-- When a file or directory is moved in, walk it to add new
+		-- When a file or directory is moved in, scan it to add new
 		-- stuff.
-		go (MovedIn { filePath = f }) = walk f
+		go (MovedIn { filePath = f }) = scan f
 		go (MovedOut { isDirectory = isd, filePath = f })
 			| isd = delDirHook <@> f
 			| otherwise = delHook <@> f
@@ -124,9 +128,11 @@ watchDir i dir ignored hooks
 
 		hashook h = isJust $ h hooks
 
-		h <@> f
+		runhook h f inscan
 			| ignored f = noop
-			| otherwise = maybe noop (\a -> a $ indir f) (h hooks)
+			| otherwise = maybe noop (\a -> a (indir f) inscan) (h hooks)
+		h <@> f = runhook h f False
+		h <@?> f = runhook h f True
 
 		indir f = dir  f
 
@@ -141,10 +147,10 @@ watchDir i dir ignored hooks
 					Just hook -> tooManyWatches hook dir
 			| otherwise = throw e
 
-tooManyWatches :: (String -> IO ()) -> FilePath -> IO ()
+tooManyWatches :: (String -> Bool -> IO ()) -> FilePath -> IO ()
 tooManyWatches hook dir = do
 	sysctlval <- querySysctl [Param maxwatches] :: IO (Maybe Integer)
-	hook $ unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval
+	hook (unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval) False
 	where
 		maxwatches = "fs.inotify.max_user_watches"
 		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"

From 7d2c8133967d2f12cd18cf8f57e91a107e17bedb Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 14:50:54 -0400
Subject: [PATCH 3734/8313] fix bug that turned files already in git into
 symlinks

This requires a relatively expensive test at file add time to see if it's
in git already. But it can be optimised to only happen during the startup
scan.
---
 Command/Watch.hs   | 29 ++++++++++++++++++++---------
 Utility/Inotify.hs |  2 ++
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 5564df7bcd..1b728f2547 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -48,6 +48,7 @@ import qualified Command.Add
 import qualified Git.Command
 import qualified Git.UpdateIndex
 import qualified Git.HashObject
+import qualified Git.LsFiles
 import qualified Backend
 import Annex.Content
 import Annex.CatFile
@@ -67,7 +68,7 @@ import System.INotify
 
 type ChangeChan = TChan Change
 
-type Handler = FilePath -> Annex (Maybe Change)
+type Handler = FilePath -> Bool -> Annex (Maybe Change)
 
 data Change = Change
 	{ changeTime :: UTCTime
@@ -173,9 +174,9 @@ runChangeChan = atomically
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
-runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO ()
-runHandler st changechan handler file = void $ do
-	r <- tryIO (runStateMVar st $ handler file)
+runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> Bool -> IO ()
+runHandler st changechan handler file inscan = void $ do
+	r <- tryIO (runStateMVar st $ handler file inscan)
 	case r of
 		Left e -> print e
 		Right Nothing -> noop
@@ -201,7 +202,7 @@ noChange = return Nothing
  - or return a Change, leaving that to onAddSymlink.
  -}
 onAdd :: Handler
-onAdd file = do
+onAdd file False = do
 	showStart "add" file
 	handle =<< Command.Add.ingest file
 	noChange
@@ -210,13 +211,23 @@ onAdd file = do
 		handle (Just key) = do
 			Command.Add.link file key True
 			showEndOk
+{- During initial directory scan, this will be run for any files that
+ - are already checked into git. We don't want to turn those into symlinks,
+ - so do a check. This is rather expensive, but only happens during
+ - startup, and when a directory is moved into the tree. -}
+onAdd file True = do
+	liftIO $ putStrLn $ "expensive check for " ++ file
+	ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
+		( noChange
+		, onAdd file False
+		)
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
  - before adding it.
  -}
 onAddSymlink :: Handler
-onAddSymlink file = go =<< Backend.lookupFile file
+onAddSymlink file _inscan = go =<< Backend.lookupFile file
 	where
 		go Nothing = addlink =<< liftIO (readSymbolicLink file)
 		go (Just (key, _)) = do
@@ -249,7 +260,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 			madeChange file "link"
 
 onDel :: Handler
-onDel file = do
+onDel file _inscan = do
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.unstageFile file)
 	madeChange file "rm"
@@ -262,14 +273,14 @@ onDel file = do
  - command to get the recursive list of files in the directory, so rm is
  - just as good. -}
 onDelDir :: Handler
-onDelDir dir = do
+onDelDir dir _inscan = do
 	Annex.Queue.addCommand "rm"
 		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
 	madeChange dir "rmdir"
 
 {- Called when there's an error with inotify. -}
 onErr :: Handler
-onErr msg = do
+onErr msg _inscan = do
 	warning msg
 	return Nothing
 
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index ad0c21b227..5ed016c449 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -64,6 +64,8 @@ watchDir :: INotify -> FilePath -> (FilePath -> Bool) -> WatchHooks -> IO ()
 watchDir i dir ignored hooks
 	| ignored dir = noop
 	| otherwise = do
+		-- Use a lock to make sure events generated during initial
+		-- scan come before real inotify events.
 		lock <- newLock
 		let handler event = withLock lock (void $ go event)
 		void (addWatch i watchevents dir handler)

From b240418acc99d5cacc2fdcfe655979517eda9fd4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 16:20:56 -0400
Subject: [PATCH 3735/8313] better optimisation of add check

Now really only done in the startup scan.

It turns out to be quite hard for event handlers to know when the startup
scan is complete. I tried to make addWatch pass that info, but found
threading the state very difficult. For now, a quick hack, using the fast
flag.

Note that it's actually possible for inotify events to come in while the
startup scan is still ongoing. Due to my hack, the expensive check will
be done for files added in such inotify events.
---
 Command/Watch.hs   | 61 +++++++++++++++++++++++++++-------------------
 Utility/Inotify.hs | 21 ++++++----------
 2 files changed, 44 insertions(+), 38 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 1b728f2547..26875b9a75 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -68,7 +68,7 @@ import System.INotify
 
 type ChangeChan = TChan Change
 
-type Handler = FilePath -> Bool -> Annex (Maybe Change)
+type Handler = FilePath -> Annex (Maybe Change)
 
 data Change = Change
 	{ changeTime :: UTCTime
@@ -122,18 +122,25 @@ watch st = withINotify $ \i -> do
 		}
 	-- The commit thread is started early, so that the user
 	-- can immediately begin adding files and having them
-	-- committed, even while the inotify scan is taking place.
+	-- committed, even while the startup scan is taking place.
 	_ <- forkIO $ commitThread st changechan
-	-- This does not return until the inotify scan is done.
+	-- The fast flag is abused somewhat, to tell when the startup
+	-- scan is still running.
+	runStateMVar st $ do
+		setfast False
+		showAction "scanning"
+	-- This does not return until the startup scan is done.
 	-- That can take some time for large trees.
 	watchDir i "." (ignored . takeFileName) hooks
-	runStateMVar st $ showAction "scanning"
+	runStateMVar st $ setfast True
 	-- Notice any files that were deleted before inotify
 	-- was started.
 	runStateMVar st $ do
 		inRepo $ Git.Command.run "add" [Param "--update"]
 		showAction "started"
 	waitForTermination
+	where
+		setfast v= Annex.changeState $ \s -> s { Annex.fast = v }
 #else
 watch = error "watch mode is so far only available on Linux"
 #endif
@@ -174,9 +181,9 @@ runChangeChan = atomically
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
-runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> Bool -> IO ()
-runHandler st changechan handler file inscan = void $ do
-	r <- tryIO (runStateMVar st $ handler file inscan)
+runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO ()
+runHandler st changechan handler file = void $ do
+	r <- tryIO (runStateMVar st $ handler file)
 	case r of
 		Left e -> print e
 		Right Nothing -> noop
@@ -200,34 +207,38 @@ noChange = return Nothing
  -
  - Inotify will notice the new symlink, so this Handler does not stage it
  - or return a Change, leaving that to onAddSymlink.
+ -
+ - During initial directory scan, this will be run for any files that
+ - are already checked into git. We don't want to turn those into symlinks,
+ - so do a check. This is rather expensive, but only happens during
+ - startup.
  -}
 onAdd :: Handler
-onAdd file False = do
-	showStart "add" file
-	handle =<< Command.Add.ingest file
-	noChange
+onAdd file = do
+	ifM (Annex.getState Annex.fast)
+		( go -- initial directory scan is complete
+		, do -- expensive check done only during startup scan
+			ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
+				( noChange
+				, go
+				)
+		)
 	where
+		go = do
+			showStart "add" file
+			handle =<< Command.Add.ingest file
+			noChange
 		handle Nothing = showEndFail
 		handle (Just key) = do
 			Command.Add.link file key True
 			showEndOk
-{- During initial directory scan, this will be run for any files that
- - are already checked into git. We don't want to turn those into symlinks,
- - so do a check. This is rather expensive, but only happens during
- - startup, and when a directory is moved into the tree. -}
-onAdd file True = do
-	liftIO $ putStrLn $ "expensive check for " ++ file
-	ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
-		( noChange
-		, onAdd file False
-		)
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
  - before adding it.
  -}
 onAddSymlink :: Handler
-onAddSymlink file _inscan = go =<< Backend.lookupFile file
+onAddSymlink file = go =<< Backend.lookupFile file
 	where
 		go Nothing = addlink =<< liftIO (readSymbolicLink file)
 		go (Just (key, _)) = do
@@ -260,7 +271,7 @@ onAddSymlink file _inscan = go =<< Backend.lookupFile file
 			madeChange file "link"
 
 onDel :: Handler
-onDel file _inscan = do
+onDel file = do
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.unstageFile file)
 	madeChange file "rm"
@@ -273,14 +284,14 @@ onDel file _inscan = do
  - command to get the recursive list of files in the directory, so rm is
  - just as good. -}
 onDelDir :: Handler
-onDelDir dir _inscan = do
+onDelDir dir = do
 	Annex.Queue.addCommand "rm"
 		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
 	madeChange dir "rmdir"
 
 {- Called when there's an error with inotify. -}
 onErr :: Handler
-onErr msg _inscan = do
+onErr msg = do
 	warning msg
 	return Nothing
 
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 5ed016c449..6eb7be31c6 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -15,11 +15,7 @@ import qualified System.Posix.Files as Files
 import System.IO.Error
 import Control.Exception (throw)
 
-{- A hook is passed some value to act on.
- -
- - The Bool is False when we're in the intial scan of a directory tree,
- - rather than having received a genuine inotify event. -}
-type Hook a = Maybe (a -> Bool -> IO ())
+type Hook a = Maybe (a -> IO ())
 
 data WatchHooks = WatchHooks
 	{ addHook :: Hook FilePath
@@ -94,8 +90,8 @@ watchDir i dir ignored hooks
 				Nothing -> return ()
 				Just s
 					| Files.isDirectory s -> recurse fullf
-					| Files.isSymbolicLink s -> addSymlinkHook <@?> f
-					| Files.isRegularFile s -> addHook <@?> f
+					| Files.isSymbolicLink s -> addSymlinkHook <@> f
+					| Files.isRegularFile s -> addHook <@> f
 					| otherwise -> return ()
 
 		-- Ignore creation events for regular files, which won't be
@@ -130,11 +126,10 @@ watchDir i dir ignored hooks
 
 		hashook h = isJust $ h hooks
 
-		runhook h f inscan
+		runhook h f
 			| ignored f = noop
-			| otherwise = maybe noop (\a -> a (indir f) inscan) (h hooks)
-		h <@> f = runhook h f False
-		h <@?> f = runhook h f True
+			| otherwise = maybe noop (\a -> a $ indir f) (h hooks)
+		h <@> f = runhook h f
 
 		indir f = dir  f
 
@@ -149,10 +144,10 @@ watchDir i dir ignored hooks
 					Just hook -> tooManyWatches hook dir
 			| otherwise = throw e
 
-tooManyWatches :: (String -> Bool -> IO ()) -> FilePath -> IO ()
+tooManyWatches :: (String -> IO ()) -> FilePath -> IO ()
 tooManyWatches hook dir = do
 	sysctlval <- querySysctl [Param maxwatches] :: IO (Maybe Integer)
-	hook (unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval) False
+	hook $ unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval
 	where
 		maxwatches = "fs.inotify.max_user_watches"
 		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"

From cb2255e93aa30425da3213b742745832a0067622 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 16:24:19 -0400
Subject: [PATCH 3736/8313] do fewer commits during long batch jobs

10 thousand queue size does not use appreciable memory in my testing.
---
 Command/Watch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 26875b9a75..8b4e1b65cd 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -359,7 +359,7 @@ commitStaged = do
 shouldCommit :: UTCTime -> [Change] -> Bool
 shouldCommit now changes
 	| len == 0 = False
-	| len > 4096 = True -- avoid bloating queue too much
+	| len > 10000 = True -- avoid bloating queue too much
 	| length (filter thisSecond changes) < 10 = True
 	| otherwise = False -- batch activity
 	where

From 74aa310ad6f68cc670135f12e1023a4f9dfaf3c4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 17:01:52 -0400
Subject: [PATCH 3737/8313] update

---
 doc/design/assistant/inotify.mdwn | 116 +++++++++++++++++-------------
 1 file changed, 66 insertions(+), 50 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 9fe6938c4e..60c598673a 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -1,44 +1,53 @@
 Finish "git annex watch" command, which runs, in the background, watching via
 inotify for changes, and automatically annexing new files, etc.
 
-There is a `watch` branch in git that adds such a command. To make this
-really useful, it needs to:
+There is a `watch` branch in git that adds the command.
 
-- on startup, add any files that have appeared since last run **done**
-- on startup, fix the symlinks for any renamed links **done**
-- on startup, stage any files that have been deleted since last run
-  (seems to require a `git commit -a` on startup, or at least a
-  `git add --update`, which will notice deleted files) **done**
-- notice new files, and git annex add **done**
-- notice renamed files, auto-fix the symlink, and stage the new file location
-  **done**
-- handle cases where directories are moved outside the repo, and stop
-  watching them **done**
-- when a whole directory is deleted or moved, stage removal of its
-  contents from the index **done**
-- notice deleted files and stage the deletion
-  (tricky; there's a race with add since it replaces the file with a symlink..)
-  **done**
-- Gracefully handle when the default limit of 8192 inotified directories
-  is exceeded. This can be tuned by root, so help the user fix it.
-  **done**
-- periodically auto-commit staged changes (avoid autocommitting when
-  lots of changes are coming in) **done**
-- coleasce related add/rm events for speed and less disk IO **done**
-- don't annex `.gitignore` and `.gitattributes` files **done**
-- run as a daemon **done**
-- tunable delays before adding new files, etc
+## known bugs
+
+* A process has a file open for write, another one closes it,
+  and so it's added. Then the first process modifies it.
+
+  Or, a process has a file open for write when `git annex watch` starts
+  up, it will be added to the annex. If the process later continues
+  writing, it will change content in the annex.
+
+  This changes content in the annex, and fsck will later catch
+  the inconsistency.
+
+  Possible fixes: 
+
+  * Somehow track or detect if a file is open for write by any processes.
+  * Or, when possible, making a copy on write copy before adding the file
+    would avoid this.
+  * Or, as a last resort, make an expensive copy of the file and add that.
+  * Tracking file opens and closes with inotify could tell if any other
+    processes have the file open. But there are problems.. It doesn't
+    seem to differentiate between files opened for read and for write.
+    And there would still be a race after the last close and before it's
+    injected into the annex, where it could be opened for write again.
+    Would need to detect that and undo the annex injection or something.
+
+* If a file is checked into git as a normal file and gets modified
+  (or merged, etc), it will be converted into an annexed file.
+  See [[blog/day_7__bugfixes]]
+
+## todo
+
+- Support OSes other than Linux; it only uses inotify currently.
+  OSX and FreeBSD use the same mechanism, and there is a Haskell interface
+  for it,
+- Run niced and ioniced? Seems to make sense, this is a background job.
 - configurable option to only annex files meeting certian size or
   filename criteria
-- option to check files not meeting annex criteria into git directly
+- option to check files not meeting annex criteria into git directly,
+  automatically
 - honor .gitignore, not adding files it excludes (difficult, probably
   needs my own .gitignore parser to avoid excessive running of git commands
   to check for ignored files)
 - Possibly, when a directory is moved out of the annex location,
-  unannex its contents.
-- Support OSes other than Linux; it only uses inotify currently.
-  OSX and FreeBSD use the same mechanism, and there is a Haskell interface
-  for it,
+  unannex its contents. (Does inotify tell us where the directory moved
+  to so we can access it?)
 
 ## the races
 
@@ -61,25 +70,6 @@ Many races need to be dealt with by this code. Here are some of them.
   Fixed this problem; Now it hard links the file to a temp directory and
   operates on the hard link, which is also made unwritable.
 
-* A process has a file open for write, another one closes it, and so it's
-  added. Then the first process modifies it.
-
-  **Currently unfixed**; This changes content in the annex, and fsck will
-  later catch the inconsistency.
-
-  Possible fixes: 
-
-  * Somehow track or detect if a file is open for write by any processes.
-  * Or, when possible, making a copy on write copy before adding the file
-    would avoid this.
-  * Or, as a last resort, make an expensive copy of the file and add that.
-  * Tracking file opens and closes with inotify could tell if any other
-    processes have the file open. But there are problems.. It doesn't
-    seem to differentiate between files opened for read and for write.
-    And there would still be a race after the last close and before it's
-    injected into the annex, where it could be opened for write again.
-    Would need to detect that and undo the annex injection or something.
-
 * File is added and then replaced with another file before the annex add
   makes its symlink.
 
@@ -108,3 +98,29 @@ Many races need to be dealt with by this code. Here are some of them.
 
   Not a problem; The removal event removes the old file from the index, and
   the add event adds the new one.
+
+## done
+
+- on startup, add any files that have appeared since last run **done**
+- on startup, fix the symlinks for any renamed links **done**
+- on startup, stage any files that have been deleted since last run
+  (seems to require a `git commit -a` on startup, or at least a
+  `git add --update`, which will notice deleted files) **done**
+- notice new files, and git annex add **done**
+- notice renamed files, auto-fix the symlink, and stage the new file location
+  **done**
+- handle cases where directories are moved outside the repo, and stop
+  watching them **done**
+- when a whole directory is deleted or moved, stage removal of its
+  contents from the index **done**
+- notice deleted files and stage the deletion
+  (tricky; there's a race with add since it replaces the file with a symlink..)
+  **done**
+- Gracefully handle when the default limit of 8192 inotified directories
+  is exceeded. This can be tuned by root, so help the user fix it.
+  **done**
+- periodically auto-commit staged changes (avoid autocommitting when
+  lots of changes are coming in) **done**
+- coleasce related add/rm events for speed and less disk IO **done**
+- don't annex `.gitignore` and `.gitattributes` files **done**
+- run as a daemon **done**

From 7d458c40db7778f677c00cbc076d37b5e4abf60d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 19:36:11 -0400
Subject: [PATCH 3738/8313] tweak

---
 Command/Watch.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 8b4e1b65cd..22a534d458 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -79,7 +79,7 @@ data Change = Change
 
 def :: [Command]
 def = [withOptions [foregroundOption, stopOption] $ 
-	command "watch" paramPaths seek "watch for changes"]
+	command "watch" paramNothing seek "watch for changes"]
 
 seek :: [CommandSeek]
 seek = [withFlag stopOption $ \stopdaemon -> 

From da62edb42af02ce34e9ea69edcd8ec2f30cdd625 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 21:13:15 -0400
Subject: [PATCH 3739/8313] optimisation and memory leak fix

---
 Git/Queue.hs       | 12 ++++++------
 Git/UpdateIndex.hs |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Git/Queue.hs b/Git/Queue.hs
index acf6cd0918..ddcf135197 100644
--- a/Git/Queue.hs
+++ b/Git/Queue.hs
@@ -34,7 +34,7 @@ data Action
 	{- Updating the index file, using a list of streamers that can
 	 - be added to as the queue grows. -}
 	= UpdateIndexAction
-		{ getStreamers :: [Git.UpdateIndex.Streamer]
+		{ getStreamers :: [Git.UpdateIndex.Streamer] -- in reverse order
 		}
 	{- A git command to run, on a list of files that can be added to
 	 - as the queue grows. -}
@@ -104,9 +104,8 @@ addUpdateIndex streamer q repo =
 	updateQueue action different 1 q repo
 	where
 		key = actionKey action
-		-- streamer is added to the end of the list, since
-		-- order does matter for update-index input
-		action = UpdateIndexAction $ streamers ++ [streamer]
+		-- the list is built in reverse order
+		action = UpdateIndexAction $ streamer : streamers
 		streamers = maybe [] getStreamers $ M.lookup key $ items q
 
 		different (UpdateIndexAction _) = False
@@ -116,7 +115,7 @@ addUpdateIndex streamer q repo =
  - different action, it will be flushed; this is to ensure that conflicting
  - actions, like add and rm, are run in the right order.-}
 updateQueue :: Action -> (Action -> Bool) -> Int -> Queue -> Repo -> IO Queue
-updateQueue action different sizeincrease q repo
+updateQueue !action different sizeincrease q repo
 	| null (filter different (M.elems (items q))) = return $ go q
 	| otherwise = go <$> flush q repo
 	where
@@ -147,7 +146,8 @@ flush (Queue _ lim m) repo = do
  - this allows queueing commands that do not need a list of files. -}
 runAction :: Repo -> Action -> IO ()
 runAction repo (UpdateIndexAction streamers) =
-	Git.UpdateIndex.streamUpdateIndex repo streamers
+	-- list is stored in reverse order
+	Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers
 runAction repo action@(CommandAction {}) =
 	pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs
 	where
diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs
index 31e8a45b27..abdc4bcbe3 100644
--- a/Git/UpdateIndex.hs
+++ b/Git/UpdateIndex.hs
@@ -71,7 +71,7 @@ unstageFile file repo = do
 {- A streamer that adds a symlink to the index. -}
 stageSymlink :: FilePath -> Sha -> Repo -> IO Streamer
 stageSymlink file sha repo = do
-	line <- updateIndexLine
+	!line <- updateIndexLine
 		<$> pure sha
 		<*> pure SymlinkBlob
 		<*> toTopFilePath file repo

From c1566757972e4774476a10bc6f865a88823e8938 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 12 Jun 2012 21:29:01 -0400
Subject: [PATCH 3740/8313] blog for the day

---
 .../assistant/blog/day_7__bugfixes.mdwn       |  45 ++++++++++++++++++
 .../blog/day_7__bugfixes/profile.png          | Bin 0 -> 47098 bytes
 .../blog/day_7__bugfixes/profile2.png         | Bin 0 -> 230937 bytes
 3 files changed, 45 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_7__bugfixes.mdwn
 create mode 100644 doc/design/assistant/blog/day_7__bugfixes/profile.png
 create mode 100644 doc/design/assistant/blog/day_7__bugfixes/profile2.png

diff --git a/doc/design/assistant/blog/day_7__bugfixes.mdwn b/doc/design/assistant/blog/day_7__bugfixes.mdwn
new file mode 100644
index 0000000000..3704969e30
--- /dev/null
+++ b/doc/design/assistant/blog/day_7__bugfixes.mdwn
@@ -0,0 +1,45 @@
+Kickstarter is over. Yay!
+
+Today I worked on the bug where `git annex watch` turned regular files
+that were already checked into git into symlinks. So I made it check
+if a file is already in git before trying to add it to the annex.
+
+The tricky part was doing this check quickly. Unless I want to write my
+own git index parser (or use one from Hackage), this check requires running
+`git ls-files`, once per file to be added. That won't fly if a huge
+tree of files is being moved or unpacked into the watched directory.
+
+Instead, I made it only do the check during `git annex watch`'s initial
+scan of the tree. This should be ok, because once it's running, you
+won't be adding new  files to git anyway, since it'll automatically annex
+new files. This is good enough for now, but there are at least two problems
+with it:
+
+* Someone might `git merge` in a branch that has some regular files,
+  and it would add the merged in files to the annex.
+* Once `git annex watch` is running, if you modify a file that was
+  checked into git as a regular file, the new version will be added
+  to the annex.
+
+I'll probably come back to this issue, and may well find myself directly
+querying git's index.
+
+---
+
+I've started work to fix the memory leak I see when running `git annex
+watch` in a large repository (40 thousand files). As always with a Haskell
+memory leak, I crack open [Real World Haskell's chapter on profiling](http://book.realworldhaskell.org/read/profiling-and-optimization.html).
+
+Eventually this yields a nice graph of the problem:
+
+[[!img profile.png alt="memory profile"]]
+
+So, looks like a few minor memory leaks, and one huge leak. Stared
+at this for a while and trying a few things, and got a much better result:
+
+[[!img profile2.png alt="memory profile"]]
+
+I may come back later and try to improve this further, but it's not bad memory
+usage. But, it's still rather slow to start up in such a large repository,
+and its initial scan is still doing too much work. I need to optimize
+more..
diff --git a/doc/design/assistant/blog/day_7__bugfixes/profile.png b/doc/design/assistant/blog/day_7__bugfixes/profile.png
new file mode 100644
index 0000000000000000000000000000000000000000..702af1ca00367f5e4b6b8242b195ee3531b193f7
GIT binary patch
literal 47098
zcmaI82|QHq+dn>1c2P-VPYF{Z+1DwhLUzK)lF2fL(Af7PW&gy4>}1F?_9bLXQDiJr
zV-HDV-l8tGI29OAP`oKJ2wp=
zkfY$~5mSaE-~scP)*U?1*=XIq2|1wt%WTMt2d^-?-Z8^~$2*7rZ+$_lLLk8qjhokv
zyocr~x=~^pfdbwLtb8F##>h01)3GYVk##uR^!Y`?@_5-me<%Wi9@USu*mxGK5PG6n
zg~3%|`5cGDqKJ$Krir60DRRUGVZFs3rXo{&%>}CPoGX(lQu^A_6-~9W4`CntI^nF1
z(phKkJ%AM;*_t%be)|&V&qKLi7xiULFmSUw1rrd$)j#8%v
zGHsxlR>x>hOB>vPzgK7|ceqnasgy3A(a?$~GSrViAZL!!9zF&$KmB{q=3^~~FF?kh
zTAx_9z@dC+g{d#x=XC0Ld;Eyp|Jkp4Gh~^SoGKY6
z$=SY=KTN$|Y{zsVI@Yo&DEtf>8CA9U?Xs_E{kr&j9(qH~F7n!-ljkh|Sc;HFNBPJc
zwp`4q-y)*6W_YdyQNj0C9+E&6#8p%
zq@nQiFoEdN_O7h_v_ph1Ejq#qs-ii3$
ztVz83Qt_Y*ET-lF>Y3nB-0dqH)UAqCCs3x}rmoL1x8b|IE*eo>Qc^SQtk2%o5K1%fm};MXlhtVwN;YbALVhm@jk!Axp&a_QK5^A
zO}1)G$!dOQM=|1@z;BYr!mzUU8*n1dHRf$l5BrM2`&L$3FrqZ;jp1^!3b8lJuUq^c
z&#{RYJ=j4Yci`u_BmP7_Cb%>()<&nxeh
ztp;_y^lvok3W8NJeh&&!=D=aAstYa>6v%tQ>+Y}%X$0t{h^Gzdp675mypJ*}Roq6P
zZ>lWu_pl;2UJG&{POD#MzP6^!c19%h%&HkLVot2{tTH%W*W;%xdkD=?Aim4wBa=l
zDlOw1bB`q(8*lfN3eZJ=>Ii9vEzf?;<@llhD_Mw9g>^cENBxBM9mAv)+rh`RIWs}9
z3m&d@`Hdk)uS!U8u6HtvOC!8(rew?QI0TyD<6RSQ(w~e#Zv7$&q?SyO(9ttn{R(G(
zq&C=%DZz_em#@9nSM3c-#>SB&chdRvMJgk;cpKFX02m|14mB5Njk=A42Lmp
zk&g}%`;7fV`xu9zL8|ukO~xcj`XC(R|b7Qzsbi?yw85UJXn@r
zPkuAzi%}fYf!0qCsMK~A7twpU-bTz_S!;Lk)2!7kl6#!Fei^^94rxXVSL4}Q-iStI
z-_fkSFaF9Pjh=y$@u|h@jo~X@6{gM+TMcP2V=Nya<5{i*CG)NkglbfuXhUdUi(;+WXb@+cElcUfv-H>}rFf4IcYl2T$ZO>Ig&3q@Xvs(D
zq}*oKXVuQD+_Wrqr61kqQ`dT(In43%ESQ
z6fWEHP)Y1DyPCB24dct4%=nD})@X;?H9Ru-yW=N?Ef?nWZZZNyl>k=*VhemxG-5h%
zvGFMv80F^G-JfgZyhp8r-sI7(-IOyB#brvck8wL#-bMm_2Ez2^RA1?0?q|
zIgrusA{qX1xik6Zhu?&$ZvToml$&_}0OD#gcCf!C@`HM1A&@O*Oo>-0$64z0|7Sr!
zeFGf&`jx*={*Qw=(e9KERewh)k+VfR%I`9inMmS2hN}s+@-dOw47z0`l7}J!H
zq$y^&S`!B1hda(_xcb4Df&wVg)@<{VSxNsyOI9`uce=ObLyX_>J+~eiXOCPfoIzB7O!2oldlc;i9rX3k*s|B*_O{VVQnAm%M0B
zom5vs-T}$$uJq$`7okqejTT`c6O7HY7dx+ZV+3zhjgOXZ;#1+ugoGD@fv;%mgQu+O#e_HIb-rP9_!OO^a
zAcDk46mBC^bTsf)>^~PzF(NmjED{4T8=dB8l)SXJ7QFx(iVS5>UDGL9Dnvsdx?h}R
z*Mc@Yr8iUji}pt|Aa~&`}Q4gY?9fXFmQ=ZJ5N`l-N;>L?1%v?{$=%CYVxdJ{aG&MFZ(i0z4
zsl5QPJVYOTne1Oju{Dda1P7|hguKarq{XDo9kJAP}KiR(GS-%11bd3r5X}ampat4h`FjDS7
zEtK8XOjyGp!S*17{?pcjo71VxNIm3Nf7OgGyU0%;+B5lW=e2(^Z$^%mwOE=Z6n=NC
zAH4@^4RF^OXm*%oA3>mDIQFNxta}nKrM1&ADP4B;qnzq5L18q60LxOJV|c-{Vz5Us
zlyDk{HA)|ScxhAQwD2s~dvU(l?CBlq1`4DDzYOupK_^gj;jpOWo#I|x&G3S6!#&Ms
zK?y(Qa|`6j?*55j7MjJ|0^`R<$0at8M-Y_a7nN?EK<4Q_r%A;`e10Rk`s7_zwV0OT
zz|4pLYYLE=B37wuGym+6Y#xlYK8C_dHKnwO>d>LBVvQnm&htCeDaMU5x>*sjs;a1lD6EsqlNmZ;^=X7n7zi
zb(1VE2EmVxRjU;zva&_rDsdRF+0C<%q&zL^+>MPZ-#wkp5fN;IUhMw2fJ)v&h|Zp+
z=uI11y$Tw6kL(ATYiPh_)i2+F`OI#kXVf`0;seMh$^$2Dw)nZaT)5o+pBTZGnCOuJ
z#e-n+R?AZ6p4RLj1!}=g+P>{^txPY$cD*E9G-B$pS-(e;kVxp;f{v1xxYNq}L|3WM
zTvU{#^YG5wI%jjjHoo^)%qLH$(te-K+Y5OwujzBFq%n=U*3@jOg6x-R>*;e0svEnIA5X{KUUq;}PAb%VBW!hAd-$$2z$Kek
zH&ga3FOH#4e6SwgWM%v84(+M2EJ?#S5C8Vz-*+3Ea-p3sIPSyXRJo?4Lo9=iFCWV
zyYvB^>{AEp%4^>-;aUx%*xzh_HWK6pUJlYk@kwIR{Ztc_+6dFGE
zE34W$b`R~@E^6~t`~jrPw6|Y+lwbU`wNmI&Vsz>YCjRWsQjqre^^$W9ItYfHrsgfIsX{B8upN^wltxTitL5OZJTZ?`p@)Fy+JFE_sZYc2grLk#3HwUR-8a006tVr-~Y{-TilH%CK>5BKx;zmb%3)
zXd%J>WZ11XMYIRTd-eD^eef4NI8$InZm7hFMGSnp32lB9M$a)~j;4$~s*7N2Y*%|~
z!L4xOw6b2}0|`^wWxc(7S0-9O09J)mCS$Te9;+06j#)Ip8qjE(f=DS%yw7n=efdF`
zHnB&QPI`3FtNH|}$0pZ~;H%a@cgy*?4)Bqb@689(ZK>=3inKNR9oY-XT&r)DIQh1~
zu?m4hDQ0w2F1$6Y{5{b0F{fz>+Cbw5qE=dWxmfpFVPRqE-2~gid=w`^x%D=%rPa3~
zm-Q~nu?dwtLlb=9(;IYgpKI|ZAaktszUF(Ft{HzkxY*YoaH-vnj3q
zpgT-;Rym&qz3CUY5hSo3>lyU?Ta5FMnw9|RPkmqRnqqsmjf8jX&T>PVpF#Id-SFH#|NV7jZ54r-4Dx=Z{Q*7z{`IDj>lyD8
z1AGr4rO$V{A)>e_Nu@^BL>@{9A-5yy?~BjXJpzPD^?mWqa%GP<;9K6My;6{o
zW8nR~Vl-^(`}fmR6cTN#8lrJWlZt!PZ4K
z2>Ced8aYR-lUm()fO;ty173hY5`TXZaRCG%#5DoTxrni`bIours<^ng`bn2v2v{)9
zQ+geuwoSXgy>v&}A@W*xXcXaQ3Dj?Z&9!*>==k0@k2JCZ9&$)JQbQp5w&oWFU7~xL
zDb_aJ1e{HCtHP>(d=*K-Ys)tkgFxdwOnDj<>`Q;OoauYwhgus=IqG_e1wx2r&n_tF
zva3Mgp~y~v3h!4t`$;rOf>FFL7i)!!10YS8OQ@ZCJh<&p>YA4$DXb9(D8Af$wywE4
z)y%Ys3mxV6#W+0xj?#{|-GBVxODQY8mAFP+x!BCao|A0nn*UC>TN4c>fwxB}xB*Ok
zu>*IAo#ymtbL1yAn8&Oq80)F)K@JYFDCFJYYUknO31XdBQ_V6&c@x|Lfd{0Ft728fp=Ma|P59vMHGcG9JD?PT%yWCo!|1(9^G(U#1|8GZL{B
zxHJMoMY$19tG56O=|elkKvUoXQE3U4kY-1UmP92BYy3=X?HyC?2(KblJJ%!dd!3SR
zL>-fb_jd2pqjv3&KWL1`#E2)7Yh4`kSSO|w`F^F!X;0Pwv2orbJ_v-jl^pa(jxB%@4`yLqM*H!u
ziwfG=ECCxg%gVW!7zHYy?7HGAlC9Y-{hQ{9cDK8e^u9OGhKfc!E%%zfLj_wskL3%V
z6Mm*%!u^zAbc)t)g+}13NTi``(NK1cxSphzSh+Z%iB!WWjW~Jojx&Qqe-ldWg=mCm
z!D9dK?tsZ?xg9=6b2U-_?(!m0CY+Ida%@MUt%d~MC~gHg>OXyuFkaRon+X=7h7|j{
z#63-oCOvsF#i|Re*XO;6vE-L=WFKSWkgR3=1qHG}N4eOfJP83NBvz`esEVVDc>ajs
z(P*Z7g{P0Lt1&Po1{ifgc_M8oW4DuY@>biRIPAfT*}Qyd$CJ^ViePwhFEBdMOZ~TA
z`tr6DY6EZYL`kYKa67T=ixpLwn3zbf4pPUJ6V@`-3@(08tLjelwGclT?w|qOLaw~h
zZ@wzzeesC6M{jjk!8GowDX76=HkRUv>vw}CZ^chf_${KiQ$}-2qMSLUWjR)u**?53
z7mfI!>0p~?6`*CN=su&vlWT@XIa@Z#3fN-^c}i+ISk3%xfBtqi-u4iQBiZ6hKmN4D
zK_U0K!9AvH(QlTp?UYeaz!|BYV4Jj04+)-JtLWE+5w<@LUsSq)uc|DQssEC?vNjhY
zDap#es_c>+c7!zOM1tLjZ#xrU!#5((=H-b0=xK#oQt2Qq^YBeUz+trQM
z*sEsnAa|UT-U(J}(NB@sNSq0&iwrO-ZZEjpd3DFFr{bB;mr))#>57%J>A`r_IOE|p
zXtF=JtkT)K4u>?W`)p5XZ49&qYBq=-V{IY0=KO
zFjg#miV&kVnP_-S@FQTXNV3|e+E@Hlb>l)UU!8gsCm5R9;Pv5BoM=S3n7J6I`#BZF
zQcV^{q0RIVomw7YZbN^m#B%^I|E(A1A=H
zoZ$!Y{7M_qee8W&@1I7|h-zoA!JXfY4)|Cf1+?=!T3GY5X0T7l_>3vOc(F{jNrK$V
z>r-$kBKwVsGsIiDl;M!H0?HHhQ{W!D{?8Fmg182%{`n!*rn(@Y+myktH#~dFiJRtM
z5=tIk(&)RL=E54Ve8Kz+C41Y5@2VXncJGWibYx${h*FFAv
zmPA@=O%aUA#y*G&*qaEk`mA6d6+W3HX%~f}CbO0}j?o<#qV#DsCs`hXPjYUQB-%MS
zS6v?{A?cAuWKf@=2F!%N7Mm?Uel}!n(w;
zl4Q_9^yCr;^%&T$WICHi>~kSCVYqMBk~kx1iD9vpv2p5JK|v*ngur|7wi}rn8yi_N
zoEYmt*qd>xpFrN#)VS_ETy4~cFi*G;pncd*D|);OPZqLUZ-`pr0@
zkkJW{v2LiWZcCAds-1T$X3_xf;TT&oSkuFnnmf1d^|-XD#?z;?$I94
zAheYvj+YfFjK~B0?W|eHDsJUS41arBRf6#DjI~9-+6$cvn)%tH)5G#HFr$sK5`%(b
zG*j9u1L?LqaY7N<*`j+Gy1GZa?WvfH)D-psI=S|6TFs4@2n(QQq-ogE=o{0KuEwuff7+)7EwVN8JT
z;D<2?zDuJ7uKQL+e_dhPxL@7cUE3a1>$
z&hFJiG9^D=75DD7L71pAel9ut0p25>?8aNtKL@a&IZy2_of3ABsOvdo0`Ec<g1o?j9ed9DOR7zsyufquf+%FAOfdHNpi6
z{UT+cN#+gE>Sld=1^1a~>``{SN(QlJ!R%)I`t`Nf5Sv2gL-AJ0Zd$~s+d*;)DT?|pNY=UIBc+lak7
zGB+-%MO?q143I)S-~*$|7Gg~c24=Ub*HAlwb#c3tRbAscUscx#Qe8$`)A?_9#D7VYWHJj(~Al|<&k%&YKA-y%2z8eXlLK4CfSlWrp*9Jzj7!Rrej}?
zl=v6ujM_lt#d9EO(Jfoc6I1@wos^zofyCg&w0xfkNVesm6jQqPA}M<<)Nl-IAf-qq
z2x*YNlo&8ckZA#&&S=_M|E3KImI4WI1&SQ1W}WmP-;Wd0=K!EdJ|Ik{=9R%Z|55?qR{?0#P&Mc%
zrNd#des;c8d8XMGAbamaAThyXoDE9)u*TfM0KrhwP-0eMAlq>W*1+2wTxDfxOvk7>g}2?H
zBixEcpJQ9>EMA5h$CooZ%2^IBdL)#ASVF~FqGIH{6uWG~r2tf>H+!utb
z27%OiU4a}WDLwejs*Q8@V+rMhSpuFm6_dQ2{iS4XZca49p(};`1HzA!XJkP}PeEa9
zNHl^pw~?PhD*~oH{DQ{M)L|!CvxMH4@-eM52?T5~w%pmg<6t5+qi!24i-+xzQNz_6
zvB|rg1IkqVv~-43?W0^J$t(eAEuf1(HIY)R%Km8MxA^DCaL|1DqDhT73nd-r0}%1w?6Mz
zD0=vw=FD^UwQc|Tap{6o;OLR;Dpw-YU7D$Ra*P(jUDzXlo_#^8Ta!!PPbNn3Uyjba
znud-x{w}SZP6jese7w$WIv0IVGSsrG!KTdTx(zfL1^22zO&%oHgSsfpn;``5rptS_
znq`4aEYa`rRbUN|6$~#_2i^1bwa?lA`6<$H3zWx;V-@x;0U4C_Z+Po@I|~>&)#mol
zu{lbGs!(;2PaYM*c5J`
zPSz&sDQmNRRSM#T=?5-bTR~g4-fXvM(hc`qk6-s2tt_KsyJw5X3Em4*4NUwU-SyLJ
z5n_4SSHF~c6OTKIdc3#TM%_pcKZ2dpogNOK&e-^2ePa;D5oW;Dc!2d3$W~|I7}|MO
zDy_48{;1=`_;t?;OjwhRMZfA-nYKS-K^?ETiXN7-3J#@pZ`r73ER|=G$E(^3I@VIF
znbr$E@>0&+E5>%Ly&r>UT4rBuOENbM_;6`wm?w~x3`|>KvxeZ(M*2Tqab7)j>$O<|
z7c!v}$Nm9&W2QvxPVAR*r`r1lnk*`!nHP@ki*>wOFq&e!+LoNF5%;jn^iNAv5JEik
z4iPJ^jgobr3G51S&g4$$v?T%$3=sG=&AIOqmw246*lv3jWYdeet0RuyVLpcH9XDO8
zBsD(n;M~M-!0*FwP21YUyRln)rzvM9THfq!F>Im!6IktXdB++i=aT!3fF|DSB_+e4$q3qqWMs9q~&*y!oa&ZOE0CDlsj=qbheZzxn&R2x~vz
zJ>#HA{_yHClAhvru&OW}Bvo(^=pRv34
zFN&h#f?%TJZwM$Da#8jrv*+)`{mL3MQm2KyXPXOefmnO}&21tXfjfq>Wek71YKR&0
zW|JvzGC0(Gs~5&)ud;=1{tlWhbho3>h_fEn;=1j#@w+3mLra}L((S-Q@`rN7+o+V8
zoqApV598k1W5A&|K6A%Dl9qsQmdkr258-9|yMvC1g~VGKR|95+X`A&0shX7
z03r6TurDv-65tmIY=6x@t<;ytQ}u!(t*
z06Kz8lT(2!&hoDN!jnhBsX*od8zIDC_Ol7#+M>>Omh-Ymr(rs$`hp%tX%O92GxAVF
zjMtQ8gjejld{a
zRCUU5oOql^*~qxcvP3SW3wYJ4WVX8#_sIsSbD?7MT%-uj{EVZhnC*C80S^gcwT^9kxjm8h`z2>>=HAvYf&LE@0q>)=p
z=~vz}QcFf4t-EwR`LdQOyi0jGN{*djyk^l4(!}15Zv!~C^C3nWkfCBE446D7OOS|E
zGTGJKZEV_1E&APf4K)NqGdS|-kGt{ehSsKH(3hic`5PXBB!j|Gc8;gS=xlpoO5|pW
zrbi$CO`8LLQa<-buQS)2Jfl**U%x{zFfygpn~GJEu+!gNWR)KrR3D}EXP*0VJkR
zNX5Jdc1MkC*eK{ELhpo40bB{>}sx2&GiPZS-lO$hBzc~$E(
zK0|xYtKVduK62_pN^VvhaGcVyi6t@EM`^&)Fzdj!Mo@?dfk~wka9n6#Cc#is@kHqa
z09kAA_Yq`Iby-INziUd@^aY3;?-(NKr+4z7t;EmePRrRy?FDAT*fvG})wEF@x?)fa
z5X~_-?6(g>8p*jQ)L3MPy`=UOl^N`v|1(>(HXP@#kk;Qg?GF{R&HddfG
zg(Zy^@iLTX?H*g`i)z*5neN*^Nr^lzTV-V~zgjz%;yr7=BIzzv*qt5R-=gDl=bKI-}d}BCJnug
zLt^Nu9|~Su_G{X5@zS7)E2MZjnxLvpd?50FQ`&!+&;AasEiYsL5MWh2m!TfvN(U74
z)r7_rtw27t#P3NkLwlj(B-?6TeuP)|Yhi$jmeeA5Ql6ZOAsRpH-`EU5myqf(ztX&Z
zZ2ydu_$i<6@WVne5M8y5wWjV4(h9K6d9|Htf%A&wIHCLk$86W+Tus=)jAy-V6c-ZI
zvhOMiE;1R<)fE>w7HPC4)fEFZRXo~@;6?~=0K$rJ7$-3QdPGZV#OX0BRg$ESymrVk
z=z!kXa3$m$8gz&aBg=rcrqA&VPuZ|K*cjXE8XI|3bE>OetK9|9nSS!n2;Us+syP59
zQq`p;iJRBoA9$9QVu}IP#@d!tC981~Hf2|FLM;c?0X?N=ipE!M^&PAG`ThI%jSZP9
zOtz~or?;)r@@kbskzyfnZ5tZ@T~1U~78#+0uaJ=PlN%d82Cqca%7#@f?VOe3L1-*8~1
zC1plw-#%#5{!~Ml2;Jd|fo4{1=5I>B8R>psLjqPU1pY$^?}M#bNY3;8(kre)aEvCXp1#a*9T74
zl0xiBwa^RHx*-l@$#Dz`?&`7>g?n+gjv>%vd~n$-ZPPq$P_%PUfvU#If_Myn`tZ`9
zfv)w@?fRgtrQ@62=|V-+Dj}E#F=vj^Sye76a6H$PsweBu+YWLK@0wl3O9NpIwbrO=
z=YZdA-)|K@p{5yecC|mtiI^YKtVppKBr?h%!e9X=$~Me<>sRI1;KoIhxk+oQMijSW
zMNrOotGgAzs)9$2dD>TO%6brN#cyyet=1Z|xl203)zWwLG`C>~q7gga`>uTH*SK*E
zHf7pQ?~=PDZ&5gg_Ky2|QE%u)eOAaD=!HsElkf7jW1|SrwfJ>)ePDmBmB#Y@65!S&
zF)euC_;Pt-#Pg4nV8l7{eQLA^K8S5~)rr6c%dJVfCE}FOs1`sfgRy_1r8V7%o2RkF$7di*Nk1Pv641ys{JDWWs^*Cy^j3>
z)tyrp0H9l(9}t4!N{9+_8Qmfzc5q&v6+g4rJiOTnCl1Rt`^h9)NJQ@vu0X6al)ZBb
za2LQuX>1G#(Ad5f5~C&uya1t{i)u@U+{Mm&*|*notLrD&Mt(Y{5jXYtwe8@?-QG6eh5EJj
zpuP4G5uoJKv+U0;pG|!;cqRi5_uwJ4DJ?Zh;O6H&mK{KojkxwPNU#MkTCg>4&2I%w
z8t%vfr1E}uD0hn}(D6O+@$lzW?1^WmZP~MO7bV;ij7{Uug?MQ8)zrVo)L6wmP5{buo>KiVWfm!i
zshhMH`ifQqSqM#yq=e>`8r!JJch2U%dGifM^3c&uHg4per%L5{?WTv`{_~$hTO9L{
z)627IarH;Xh!?gT_hLL22j&_agGK(T6e8+4zb1FjyZRh~CLNQRc-dIb+IS@=lE8S(
z;X0S+0d$*tU9?{j;1`3{cLUJ*NTc(vY%9+oMf>F9qNpGefvDf2BI
zEdSH>H2nZNe2ezqchflr7teGR&JXZJ9ce}h9-DSzY4qe-`bpi@p%uUhg!W7C!v9;2
z!94nD9Ve7l(=3Wx7Jmc@j(jr14|yj5h498o<~k4Sya&DmcAJVpS3-b4-1{HN->J;x
z-71o|hX8d9^T%j|C-s3E^5J|NlzjiM5+a&mzkK#pqCuF9C@u$^w+P%Kys`dE118dK
zylan1h=b`}dBu42#L99wbY2Tb$qmzbOLe`z-A$pn?
zLvtsAV`PNNbnH#&mzXBlLSwwjO;yMvk89`sIXJ~TA8*!`iw)lCKwI<^wd=-I4F*Qx
zkD@fJqwe^f8w}{;<5$jyOe!a={P&Z0Vx85?ed5AB$1$0ODe^codN;@
z$d+SXXnU_(p4hCA1JIw-FsWO5qrz0>f3tSd39A=z>u#5w
zy#R~w;RTGYi>8%B*hJN+gu+lL(LMk0JUkr#30VmxAlpcjR>tNkFPfd`g3dTeuDoN{ev2p&2^sR1aIcTI^&J-AT0<%PZGkIWu~e^w%r%CI45LHE3HZk)Z)wiV0VhsWS(z|!FZqV)
zB>k*`kAw6oid0FSfGqAR?(^(?XGc#W811;<-JPYIl_a7&h4;egQ
zd`rI?5w1Yy@G*R^S_YTVK9JMKpu5)3{8Mg8UQ?Jl?qSzhR5fw*HT#E>m^XrgFRSxs
zDgEHb6vhB0m8y}!<{@&+)iy;~N#(8{HTMw{uA3!zaNp26=D=bdHFJAGdO`bGDrRka
z#4562br^M1|CiQ9+@5%9K;@$+gk(Zap3%7iVj{0oT=&2Vxvv7-P^@HHu{1uO#xkZb
z-W2%a%!%gnp(4~+b_IxAfkM7lhDM~KdfN>wDenP%1pEy2M*VtEP8GXN?R}~jq6`Zh
zTYcyHb5V-t6aIBU7{Q%YK0>wPfYwn^P*n9zspC~%0bRU{isMP!p&lU%i~f+<6Xjx3
zZF*^qO-OB^aCqLFN;NE0Oyhh1NS~vz*}BWag7}?&DW{2TqJQGL_lN%!yUOEHYyX`X
z&Gkv`o44WcxQo@kjSC!p2ha1JwsxmV%GbAJ>2X|ihItuE`Fpf
zNumd)JEl{lk|f%rL6hyQ(VD=LEvzrSG%V|F0T&QxUjaQ1`R|e1+0P*pE2R}7{b`Ln
z59oKjR%aNc-cUs_h&3BE4K?TU20@ykT10Ghi-MhQypYckrSCoU4AYCfg(uUmUIyH4$KNP0mZFHCcy%pPjudY~>mbB=c&(cKtkQmFWilgOPLP2PM-URhY_y9O|m%3A{_t^Rl^_tIr#bo#dK@
zoB^ae2w-c=q?)GI0Ri1apW?=ZqEURKi)g}vF+YUBXAlfNz+d0`dxHi-_ZNl#7tiOy3OM_!;6GEDn*tmN3g`tI5rfXHSwHW8
z>hh~p=VvLy9=~?!=v%>&AdoQ7u5=6dv5$v&{V$;@#@M`ngW~zXhGjBUi|Af^pX6bY
z0Gd|wK2X`d7|#r)*4WW0pLC%wru6i?i)_I1W-2xdvX^h|{og);uDg^4#nph2kbk*^
zRDcrYW4MrQNdRzpdWZDtWDZKC+72os4dCB
z(J9#s_C36R_t`UcHHMu!u}-nhY*FsQdzzplfGEu|{+KXaJ(YS!{rc21c4K2V0(ES4
z#c3Ed1}i)qdh0u(wpPimIfAR|0qoGm=GcScBi71Cf=Fci!(H6GKqAT@zUhuu;`@
zAU)&>G*l%NgND}+CR)fi!-IC_n&wl~6QRO18XIHdxs9QqIp8Zhr=TofQYwNx@Nd;Hap1nWbYN;P>a4?_TID4A#Cb7<&7RxZeE
zGO-rL#efn?unoB|T#d^NDrI#)?MQ2DpigsA`CgNYC14$iyGo>`-SO*G$cf#(dHZdk
zV^~m5PxWP$e_roj4~38Ic8x!!fBH8jT?2ZQ;_Y;Dn!-2=v?9L#Eg2}D3W~5b%rFVT
zbgh-~_jgD2F(yILbtj-kXQ}QY7ZpIsXxE9FmCkkSmW75^Yk6H
z$<{#w%?ecW1pgx6!ze~6^F3}l(ARf4Yl@Xp%?i%5+2@}nzKzF~
zg55W1I>trT7u@}I73S*i4=t%mN8f|#{WD7>y%9Cg)E8RtdY1=tR-8_Kv!IH)@F+P~
zb$(GIqhaGQvW8$q0Lh5TNT?(NOIBwwGP0@Rj
zT(=wST>5Muo?ccm5xNY#U%<#srKj2j_Sch4a&E58h;5Jd-rp5}5MXcMuSQK)adD0g
zeQ6{p6bMllSgqd{{F|V#_kV3M111}x)js2Vzfx;vF_Fi~F~PQx)jgDRbaNQ=ZAjd3
zQ%W)T^GDbltv{p2|APrD*iOr>OOjWe?~;Zfa4h6
z9=gk|8w_6yI76{RPAvWX9*mre!lm`Mg&?v-|W{82wrNi*#v
zJN&37#CbZ_vyHp9zNL|k|6x?`jo|&MKI!PmKjR`0krTqU|EKO0pVVd7o!;Q_iw1&>
z5`Go?ep*T+4iog6qk%o~5nqGIM3gUR!IEj2;5kVEh^zhv6Fa!p@!10nT|<4cn}Ngo
z=a>C#(yK%xl5?}Z?uteb(5eI%lDlOu9H1X?QGl1jN&Mi7KF2~*OpsOjSJLWw%n!qw
zpFrnK>s@1#XF2Rbm8X@k+W$^v7IlyQMd1+e<#!T?!k4#Vzbi>Z!ZNjP0`l*V
zT`2QeHTu`O{H#^>*6vUrSjrNfh`h@%(S*}Q
zd7s6tv(fyOp42d8X|hGa1&dy2d@d?Z$R~aG#0&UatmANkrXtYj^5XViU>s@M)zJ9wX##0FnmE%o%n)+#Jg7?$Ony|VAgNSsGdGBJAewpY@
zI=|abRAo#VuG7!J=wIbjyuCOUw$BRR2nt!&`@1HA1F=RJ_shr9_&eTAv<*r{?*Foh
zEB4<#7VO6cyNoKUNYERS;^b6>np*)IG2lon`_pW4wKLEI$C(5=gE=nl2C3
zb4mRfk_B5cCc){Ogj#Hz@Mi_GW_~Ve`tka)s%UY^B=)J!p0w~u`TNJOyKRHk6b~ZX
z#oX?VCxNE(v$KvxUa6oEVEqc3I-|r0wd@}f0Zapl@5Vt`tKqZ5)auI6aT56N-O*88
z!y3XdlXG)u&6PX_6ukZ}@c{e^#ncy?db2vptdMB2y_#fjn9q1TI>mZe^llbhx+xx
zgm&LEs!zhRUlj9fsxwT;No3339dq^8{5QJ$@6W!y6zp~8_Z)_xl%<^Yt~SC
z_F)_XL}lpx$W}^hN~iV6N&HTS=AO-
zEYLW~~6iRuSCU`GBeHvU=(YxhGN`J#Zy
z6@k{gVK0}${H!x_*Kpn__=s(CECBsPC?fkBO*-$;r-e-H)lL=cUK(RIovR~5D
z3})2j;!Pbn7OfkRy<8Zvq}mLf=c^x#3?CIkWqu+x3N;u9$NZ6H`LVowfpLFMsZoJO
z=lH)~im8}b=l0q}u}-W3NM*UHl)%i&#%d|ZGNun$A`-O_aO+3`_9b#
zu6uMopL0In@9**Z`J?lAoYQgNGjq@T{k~q;>-D^BpQe;b6=$&r*cMVa@0xZt=4iyG5Kgo})0
zNl?YB45vp^GFv5>!)JZCw>y9b83G)QbW}t$Z9{NGDdo(TLrLz$SZ%i@8IK}ZNn$g$
zK*;ky?XiV5r%yLpN|S1*TkfuF2%o4=b|^);w2)6)_u3M;tn%pqFi0Pl%?gs
zZ9z=Qk4lnUa9H7dDIxY?#f>W&7B*D1zf=SqHorMerPm$|Fs3ev*Kh%-$xz#xV{_(_haxsx2A
z1pPFrxhZ^$cBAaOoba@fAMQ1Z6qeo1|5p+Ntjcn3ur)7!2~q4z|K*OK=DSXP+##4b
zb>u%5jRv(P0i($mbSeEq1?DIzddOvQ2RQWAy|ph|nZLx2XexXoK;tegO+W=^H2doJ2cn)Y9C_{GZuiN~1&kXV_VYU%tnHRX5P
zU|W1E)e@#n9?>@GDsMfO%x7GAvEF5Bjv0u<4pzPbktQhm%9{(q#3g5Q#UaD4X>1dY
zY-npqWGJo4{sItsBjBsf6%P8HHad^n*M!g*TZ503)`+Az8u>l5ta2a)gCcLBCLt%
zE>?|Fm*B11jU{$j97*83nc35W*cqQ|4<0FcqMgl}yUfIN_$N2g$%C@X!r?y#63T{(
zu<%+DA@~*J&YDrP-jX6|6HEHa=HDI^vt*KPl)}lTsGe()hu)wLzFfQc_Br(^w~`iK
z!sVygvtwi3>32S#(--b2!HPl^XX$zxw4(-`n&TAm790-L4y6d_*lY4MP~4Azk6OJ6
zwIN(2m-@~I5pb&mc6PqjL+ml+(qsStG_z(rEe
zqT7`+uDs0?bi~JLdoH#*)7#h~4O3|C6=&dH7Ip9*T&Nh+db`1I7{xEKF_5UEC-I5U
zeJ7IgyS;>me2ke;=&fRU%+?W@ARCBjPbB+(ffcn0{+f^gLVx|M
zjO`)^8Yo|;;?U{DU9o|^D5=ou-fQaT-1!)PcH}^bfrb;Hqu{eGhYsW4)
z%1DqP!7L!V^YDfbefZW*;qg%KWK0#X10r|U4qmX_l5vI?9_;s~(RQ`cnXYf@<7ONT
z_`c~OsWr!DG6yt?;%fz$P^uirZ)i{G^s
zJClFaAuqXxVKG3!SIiE(c$3B9r~u=NgLU0`RgMfn6nx|yD8!Sc;V{!VyfA$o76#>7
z!BU}gDH2-^%e`L7jp$<(a;dq9Y|@=}YSBD9-XS)Y1WqJ2Z1dehKRxY6$Md!JQSnx{
zBl`G9+V@*iyz^AZUT&hJ>T*{{U0&>CIL50gw_yChd9%kK*(ZuyRBOPEe4IG_@)qVI
zQR6Z-XL&n!Xr}KmFR5IfDTT|*GLLVN4T`j1xV5D|>BOCXM&P)f6tK*gi<3h#?toIO6MC0AH6($KmwL+e-V8M<(6kO6
zOk2nz`;3T`-Dr%@ugBU`E57m(89tqk;1Pop1mQ#5MXC7qCFkBEj#vUN7@V|lb~SJm
zPUkz`8ZV=oY7u5(0GVK6_n>>J`x$jD@e*2W@Ne`*goZw!OSR_mV(}-&M$X--kC(X7
zG9>^ewa~Qe<3uLRFNRov}7Yp$^j(=)zN6~B5XaEvUxt2n)`&i~m{wI}(oe=H|r
z=8_4%GnJM&DtC%oL}n;9*`Xo`zubMVWn)6u?k0VWKcW0t0d>zB(;zH!)*i%NZiR=Q
zS7FD!iVd}@{NlD>$}M4zZD|YGcKpgbf$dZ9+le)1#+Gy)DFh=o+>~@I-1Sawl!ymY
zf%B<|A#k0;G1c_*qHsXX%RF^RLe}D5NrvS`U4OzTZi~oi4u%
zQ}%fcaR~fS)^dq8_#o&LJ78i4mqE>RRj!QFoIYQ?xUYtq;fBJ{m
zgOVvr#t$m$PMwFp8`<|Z*Bv6Ns2@9#TZ9Mqk3l7KsDh~Y>+FoELQTfrKPW^0%%S1f
zurKnjJY7G>(V?2zX#(iK!tt~Uw&80sU>0Y-o6&wFRG6D)nEYqcNfYg0Z>-X8G;D%)
z(sY+3=@tn*XN>tn!$pLoqu&kC(423UUb`K;eZsHr(Y+++=CZzCjM%-ps#2!pU4JUC
zWO?uLAr?*NQP{B>-gp^F0EkH{{ksPeDqH(#LZ?d!1_%`
zwF4}{;LhuuyfxMsuW$k3}R?4W{Ur7V(g=P#RT+ZO4WXE*r=eV
zgNbdjNG2`ook?V(9Z<~9OKI8t9T0;jVa7M-j~zt@93VUm;E&c~OaI%ALcTkENs41+
ze*YCM{`A+mg(AIg6U^Tta>|>aiWJD!yZZWmJ65+aTD1pT96dLHch{vAr4)NfZfvpk
zSk>{6!p*$V&?5MTlE&Jh9+TLg{#`%pP4wLC+GU9P#8}KSyFF9X0T%}(eEF8ugoV4(
z4_`KjaruEDZIFsn^I8#0Oc9|S6yc~D>$*dvTNti>Y0~(_LM+1WN{bAGV6?V*`Hf6b
zmsofiQLmkIg~(qpsz*a-7W{?>_dnN$>%Lt`P&c@ufPIWOMo71WhrR8vl?}7oW!gkt5;rS56U?OIxt6T5J~|irsABc;#@tZhIY(U&SX!E^h2n-FC1McRx&f
zxD)>@JWo{DL>P*DFunUnv>p_z|K*u%VW7pv*1C=-dJKglHw`_e5Ki)4{CVIULCH!^
zzyHFyS+QtG$etN~^Sy{xv8xpV&O4qV0q4?p-3S*8oP}u*f0uxW4ADN&5qKI5$1|+o
zh;`h>E^3LQbN<`yv%)fTwrO+p39EFpp9f-`s&?Ami&FFI=%^YQ=>KVOoxJ`|Ly8Ro
zg^f8kPTWh_^Pqm8x#?9nL1_9A_JnJWxa~M>Rdil)i=qUeuTD51vMULxUq87Nc8-J7
zUSewB#nv)yRWGxPn)Dly^vPi2ho6BxY-N8n3;So}xjL$GsxD;a))`Ee*yZNtN|OjZ
z!+(tqIW(s4cw0aSZY$h#+ZH8u<8;^rz=m{I^z_sx<}Zbb6Abyrb$E9(qX^@!R2S6`
z`2>yE#}}2={iuAx>Du=cQgq67--D?#*yVU8_L&dH!A?@1b!LcfP5OqOQs-o(-oR)L
z?M}4T7w{BoHBV;7+Wk6
z_$qTNp(j7+^UNKZfQs6c%ItY;r;FPnn9^(+c(KJT3CjsgI}S%{Q_Q5CiNs?m2ew}~
zZ|vbqNaHj)pVHAdbsxgEWrAuGCwr-kxfp&Z}k(|0NtK
zR>R?tbR4FWezfpt9MldN*YOQ4y#MO1bG2BH0e@URKJ|KDP20yup?
zHnl+j`c!PR$0g@^Xx{xqk2MV#m^ZR^oM#ez!$i5U@fb5JY9;zgW0<0$&Aia2UkF|y
zqYGbdUSura$}C+Ap({i{Ju*YP!EaS78>?Bp;0y9x_7%3U%neXJp^kU9K8`|#z2u3U
zjNGoEy5C5i=+=ZwIbB8)#XPr9VM)B$TSpwaeZ6P7uu#L_jk=4ay-_zlFhdg_n-c-D
zK@Bdd+NZvOGvUozo;&xnpQ~5n1@<{3Ue~pDoOF`!vyDgQMDpX%x<+erRMLK=2?*M1
zsbm807MBQcqZ_z7Ua7S`@v#&yhU%h6ltsrof;o3md*$(;O})P
z>|3~JzNXS=?Mjt2SnjZrj2fOWO*~9
z8aA`h%BsR2f1@0!G$GXEHT=UL!rZj${D~3_Xn8MoG$G*NWH#FOnYZ|+Y06(HvoPu3
z!#TyxWtH)rqSGw*u(B!lu+!S}Cd`Edg&)O<&Mvtb-XKxm52&Em1Y9Kih~>bdDh5be
zv4==Fnzjixm&rl-3X8tqb#afpcjJ|YU3_UP0p`W9s8@XdAuL9QZ1b`!R$=EzxHw)O
zfK{7yp$6VXGGVg+xJNDR*8S<4rI8&uOdJ7wp!3m!eg{PoGlbE$h24PWzXY|JoXyrG`EIVgU%{GBuz+5LsRn;4DL{|mRNX`iw%Pq)`PN+W6!3)
zx?^xGkVEb>;`CoqbVW!fN?KZMiOycPke;3}!`+F2tu}Vah`Zirbg<4$Ai&NnOl$fl
zc>e`eE)Jgd?R#OGIbwh~`gs~yE6u_d-|oBk9a3VLWRlC{XzhhYQBfQ-
zrOne&B=Mlp<=xpEdUy%XWM!>iDT>#_`f*>9-maWi6mfHPJ?nO|d^M>TimOK$nfLl4
zKHnUxHlLeIt;hA7rH8{2^kLFgj!5UJ5LWP_Z)v4mXmov)_SsO)`nYFuh!H&}dZTdL
zPPc~L8cclGL2X{WN`|Tt8fjLge0+ECZ!%K+@SI1(7jNffgVmL1S<`UjOZ9;I6yL4f
zPD*ezEZLyeb_^5)6@_?!bjw3=v<>1sBrt3vHQ0C}p;r5+l+w&WiH2!hy30Jczlw<;
z&$P^ESPLa5f4i$YQOtd3va6;_?7_71e8BTvB`*VxjV_m4qnUYg8Z&?E{5rUscWFN$
zi98o`JBVjpXH3GwX9vRmY(R`h=PvkpbH;xH+pE}`$bOl;9jX
zEzG9$RJ$>9&zqnC22L~-dS{L_`Y*@M#AXaPBiB33pT1xzP5M2*dWzd?w~0+owp%C>nP%`uAmlQS27ztM?zl)Yat4Pl_7
zpZ*=Wt~{f+<+&r63B!QW;in;#JkiZ{J1ZAQ@gnN<`_0=eJ(F+obY6IdWCSLsrB)rc
zpL_qoJ!#bL(EF2rFcZpSER+FFC}3=IbxeHbe)4kfQEGd7w03Ah)N{ixY1@yOru+Zi
zks>?)76WwxXik_08zNn`J8qp!|8DN*Cy+OVX!53IfE#Htr-n}s)Gzk&;U>{;x^@38
z)dClGwt^Mx?+}BLv=sk(Y+0ZONaONXvH?m~ff
zs7W*}2$y$Mjt-Fp5io3l2%S1tZ~Q1l8ec5W&hEH3oH)8}N9fdo0v12@kPX9sC?r(ju;GZ5o#
z+Bc@y?0Uf_->ZAa;6QVHruFgk*S|WNFHivzMVaOzcg&Cf;6Q^QSJn*Q^QHpZ_ze7U
zY;j5tk$lv2{-d%>n_SDyw;d|>Qq^sU$n3K$j0jK=Rr`gjo;;T*kF&apaBGUCy
zxncltB8B=Qwc9F2VUB--AHWjLwPyaOse3Z;hDV5fC2=7TG$f^
zsP~q?W1uQT_S9{bV-B|SaLO`{dr
zck`bM5ZTY)d12GHmGZ0I&?bHnKHDR^Qo&9dn#QW_zWX;c7xglC`Hs`urxdrq4*s66
z0s69#m*r_7=OpJxMr)gUf%b_$kswX#GNVtH19eYy7daDLh1J9FU5y^1(@vfk8UJ*9
zZ`H%#yKAzAD@g%GyW^ZSwBC!?Q(rJy{y8k2XE6OPc$nN3o3aZ
zO>^<5TNqR6aj|!7+F2jBr*>LP+_#lUacAegm9yIq4$f*655D85z(^1{{G0C6G(vDc
zqDhG@ZJD=H*NqRU8JVS&ze2Gy&^mL-(cSM^e#(Srdjgt|mdMr5_;zr0c^9hssX211
zGcJh7L!NLzsjbfLk#kp%DeArKzvp4_3Vw(@fa2Xslc16#B-B}?Y>Ew&{^52IUcWLQ
z63&Sl@S$aK2WF$DlghWDW`3E`F5v;T?<^zmPwb|hq9dPb|AxI(qi-k5;{ojugIM@z
z6b;lXSZkI~o2r{kL3}K6LBE2PMB~J-rUEO7KT(
zv8f1;LT^RG7d>FU`4L{N^jWthHpkmy6di~w04#b1@Q
zaO=UTUyro_XS|?=%hS%bGrFgb%Nh*Yn2>!L%!wXhqkjKnIDTRlnc?8oE^)R#+M>*x
z4y5F&=N4t_=b4f@{%qm+-KZjh!g|W@M>Mqx(#^TNv=-%BkF+H=-FD9JOP(L%LQqgn
zskN~=DKOc}1NWCWpC%+lLWp}KE{C@QN@Cgj0)3Rm74I(gbT($)k-dSK2T&n9sJvQx
zu5A#*e89e!5b~Xzq3x4`h*T1~scJfPZ+N++w7zSUY3uIL8<~Dd=;3?4rJh6aPQ1{7
z!`?;th@nb?VYSW!rD}L;;kYj!qdPS0siCt%VFZJm+D=ah-Of~+7T}h?cLJaFYLB5e
zg6hmXd_hkyEFaFyuF3Zz(yWd==`hsHwX};54(9W_?FTKGS1gAwOTEb>XtTo~Jsd~V
zUt1Y5=Ie`apaay?pI5OtuUCv6b@l4{IsTG#5?q#FAg24po0Q0BYG5-O?PxrW-KGzi
z{A#M#fD`R7ijvx@^c9fS>fz!Fbit=Jp^0SXlJhH0DAb289dL*WikQcu|pi8vnP5bE_(yyHn_2TN|={
z8hA$c$2vX~)NL!NwW=Q|-}M~~AqFea;g{6*Zr5CXMox<0S(vBe@ZVg#{4bIoZSkY$
zY_!gTgwp9o`^*pVqfrvyG@WojliI3Vg)^><))rP0&v$Ni7%o#24NV)dJu{Zb-Rm84
z<5+7Zx`H?8`TC93+5mMR>a~D~9euj#v<+6fu@sVJm{7>OWp3<6hY{93bISUuyR9J3
zfd6*i$+9JG%HqCKP9U%j7r3|nR$O@R-#$LNGVrzm83AAmg7n~{(Rqc&H!q#7jpw{7
zc@sXgAb5!OG^67>oo6lDSq?}dpnXiN)sD-!ti*X>@rMm2a--ujW>5G-9ioao+i|J%
z0YRg6?{Unyhpp(vhXAgT_||22E$Sn6U;|v^Qa1lSy>m{P&BXm3r&0)gS*pl&@(Il}
z&ijKJOYIVd@a}gygLB?5HxeBWJ0MV`8tjPg=@zg!|~x)t6lQ*3_et5s*@6ib=%?oIJ`?0WQXLxO9wyj56Nqr{?g-fzm&5Joaz`CTcdO*LAki29X*=w08H8It5J
zYURD2zcnsoz`b6MX`!87l@OW6$kIP}i?&?OxcqR~#ii6Xm9@0|b3!sRRyzVKxZR`Z
z{~ZE>@20cF-sq)o-r|2I2+?<{pA}3W_g^Hh!X;&}v4Ngt2{^pv?2#D&a0kNkH_+1yEF4kmuPs}kjMd{h^#
z;l^BYO90JNiJ|}#y|Ex?T&4v^LB)`yi#>BTu=^vPTf;}#c<%AtZ^O;Vdnie$WB^f1~lj(AHv4ek^A#cYFhdh{2
zt+~OmtV9hH2mAHCJeU~~)+yYX4x@lTlQ*=1D&q>H7Ok@Z^rx(nntR~#`VQ2Gvqo@y
z+|{sa;`dW+g`B_8?A~oF>0n9n?Wn(UESbBb{BH@f(9M-Y;i;9
zgwcqy6y9}BR=NuSVRBu>%5t9`YX3kSZTt6lZQ|q7_|^ssl1i?1Z5L)->rMI2e)$8*
zpQIy)x(qjM08gU&g7%<0)&}oK`aahN{@h4Tbei~fd2ImS0q4OB(Ry^~xLi5VKFJVs
zy>e*u^Lk4lcsbj27`Z}P*h*zBfSmjn&qsPeW|rWF4QR@51P>mob?Yas=efTBhC|`192n@gXmEX4=pOtZLCc&Ik*LVbWRhQGSxqqf2xi(nD#w;HP3*V5fn6~&pV1^)Du>E
zYX3eV+?40q=!v4!D+C$g>hzwm@&gyIdl+_}RjT_Ft#yF;RW3ml4tbZpC~C3wIZ^Q%
za`NK1sgRwlXfV?d_
zvHL9Imi>O*Fvaozg?=Xz*j0&RHD0P@swhLp4u*aGcYol0XAL?gW3)_SKMExyRD)iB
zm35kiXPBySDVd(8NLak>D8bwUQD&}UdE%4h^9iY8zT<%5cN&+WYmCes{v>gIwqki@
zyxs3Y$k`#2iFSX-YUhSiDIai|`!kdf|MVuCz+d}%qBFzgX9|zWDBvFn=5>#su<=Fp
zl}&w_?{T-To(lg^+$1{LT}Y>wZNJNUNav3JhN
z(6pnP$ET~@nNd8uXt0O>X!((9D_l2`5nXJlg)fVza5!l+FSIeE_+i-B)GKi~?|Ox&VC;5QUYZ>nYfMRxPK$M)C@=zX2xoLU&_Y1U-RufI&37P#*E%2!eL+7u(
z(>opNUKLf&`Yy0uG-q*dL20sr0|wS_+2^!{R(78!xV
zgJEz-uAMEV}NF$r)escMA>|Zu%=<|2s6-7h!N-3KzOIiUJ|v
zKMZk@dggs36M%Rr?>~Ajl^8`IzZD?}YDD^760eWemdNF!OpSciyDAK(!;O@gGWdSW
zF^PoMTJl5-Lu{rUaNZ-C($|3TFHwi`{j0TP5@xG7-_c?_tIoRHc&uC_iXR+>@?T4xn%jow&@19SxHf#1+G8DbJ`;|*
z9041AD1X7R{H=EeGuR~1!#J|1k++e-^=frfeEHeK9|tH00tA3N!+$=K(dx9Ji~rI|1G!Ce0JU
zp{HuqK=aYxgz>^)L1cFK$`!7`nSd(?op`um^d04Bb_yjAGE?JT*y1F(Yi$-)D`x`)#8K}hXy?-E!HtPx?NDdn
zA_$bWdYP_0ycYMZ|_A`YLO%u3vDL#gS^lBrI|Y(-h|raUssh+c1PiZDKsc=4j-K0%4FT2XXx<@heYH0M3}RVHUvP>b6$?7ZUncmbbUg
zGyD%K{0!dn8FtbUj7xQP(c>D4FY+OhMD|T6l_(
z=Tz;ryjd6f?x#2wv9)g
zFdA{5Uq#5oGOWYZS6_$RY1a!AaA8MB9h0hZ5T*>tU?q0jU^^(YodQ
z+UBzCY{EI8TuyYtHwc&Tt62TjXJ|9o?|FSN4-O%4u4nJDm4pJEt=HVj$%YPGWPi;7
z8r26^>*WKUpxm2p^^!@0abpoh1nNc#)fv+#{@+W>9Z8`M65o)<0;c6RKPG?Eo9Ho(l#F$Q9Zed2)u!#|^X0JrlY))E?t`rGm
zCpz=S?67H`PrM?Uq$(3TBP5?i6eID-^NidQx*Whzev7(~V8|s!^VUmc6cE;Gle);3
zitC)lu^OF*>%q*F4;j5E_AQ$oT@s>xm+t^5b)8lG#VOJ4d4zTsZ(5*<0LP}*n6XYn
zod>EQ6B_Zspr_o;ob7R7CL8f-3#$CA^YXGk-x)d-M_{@92Mxj0R(({2l>h@J81|aq
zrwDKWj{V-CSsl36?01Zh{&+|0zf(p{AEf!vTxpCL?T9mSGSVbBJR%dAMy>rmVc2Ykb>c(iI(G`5n^(Q{2dU
zElg~&QMcT$w6_`NR*g37g_c0g%za9P9fz9Z13{Qk4P&=v67qI1N#OLw^jA1u*HqH6A8I|4MzqJu67wh8XM9>ror2{1C{vD
z^J5fW&c?ehbA@LPdgmQVw~fD>Bae*3-6&*&5K~E^Ya{_Aj|^+8+TLkZ38zPnvfl&T
zg~K7g^o}A>L_n)ZB@JDC7*J}NWdC=>&!5ixvcj9fx0*2bq)ice(g*Jr;eTI@cKa#V
z2^uFga3>>p&0J!LI#*RkS!AGx?`hfJV_CRz6zI0sq)#fQHZ7J)a(&o3d~{V7@6csZ
z?`0k~d+wuXZAK3NahRF*F*#2@gwrN~&_n$}MmP;y{znN3
zp+^vKWi$rB=%A|#+bNc+KQeigig
zgrw}mHp|kI!z%j}vu6=U*^W5h+0}WS(m~q~^Hb|EUe1Ie^2Q$s>{WbF^iNYVi>VK(
zgF4mGDe6Z(XHVQFM8m<#OC?OMzg4ubE$&;2#4oKb#~rzt`+D+2eC%K3x`3j;&NZFx
zF?t53pU0c2rJioZ#}TdXYfDoqDaQu#cF8-Z?hR_eu`s!u+MDs?nuKkRcz{f7t05#s
zp&?LrY$vgkJ08dqK0#TxpuZ2pz!;EiW8m=S7Vb-Np#1Km{=`tfs1-LI83^G;U9YnO
z_jNyW&+dnPABziDX}V(L))T3H<}3czUVb
z5vNuk7d_`Rb&a}*S(efl{5XiF=?gA@Lk)aV7S-t6hw@n4OzwS*KHY04nNU2j+ZmD~
zvbCFDkBhz;Fsy?%E-YD!)|P{ka)Lo|MQL&4-83pWJtT*&UmlW)+F=a;XoqR7Y_L#&BLIxes<~QNIQ{j;s+zvL=QbFDW^4xUBD9iK$#T=C
zzHXgJ{a#wj1%LYEbsFt4MM>nQ?kw(hi!jgBQXRx;l<184xVi1xNj!aqLY>>Ea`}QT
zY{4$B_4=6rh=bV=A8T|Hos(?ZGC^w@(#@pGe7H_MRL#06yJrqhnS7#$gVY0S668if
z##Gt2^Zh6uNg(k+d-_a05L{`c=h^SBler!LA~1nZ%?2oZ9IY_bh%Y=TEjsO}b^LEn
z4O5+wlue^I%MMTjO-*+pvfyjGULEQl)D#~r@s=3#Onb%HpO+>6R&I7EK!s_Dsv^4t
z{x3*l3^u=pe9@E)Xiz!#D~`Z$Q~TaOIrrbmvQ
zG=to4jxF|Vy@Yz5H1rpQcE(4s{|Ct~;qj#Xuh2BJFqnYu1WS{u3JXjVZI(_MyTqO*
zgv-t@v9r#udS{dExfWe_RoK)h@?~`N+;_#T{tTP_S-~p~bm-ivApz8~vW@y-+2J4X
zG6)j0#7BkLV%01d7?B?UV;!_~JE2c7v=!NcKwWJ;5}G#rmx|_oqze%?_9*QQ;$eN8
zy)*Z;Q{&bY-(M$|eW8!)sbPFWHC$c@<+0`O=eZ!4vSFENuM|s=@!4`-XEC~wXkcN~
z>55sBm-`G%k+ltPzASv^f`nP6iKPKBmrljPAXiaHXzJ+=1JR^NC{*mE<f?N-zfW@1NSo$%!qDIwCrG#Aj$b-MPU>E6zVOdP
zTC~Ne;FUe+N-(
zkHV!xu&^^S42-_`_hR31&vh(#f`8LMT>VKEA1(eW+W1Qcu6%7}8H0{A_SdGRRRano
zEY>M=n)u09q#49!SKVS?yH@dmv()qw(hkxn*u!MfTJe}(%xcs#gAMA8=IF-s#2ERoT^VGbdPCJ$_#%;)S=rP7PcKHE<{TwDs7O2@9$k
zQhM+sm46dMDO6lR7n9qvdg)NHP)s3mp=b(CE+hq+(EEtC2OL0|bxQd^egM-HIMG0K
zX3t<#8eCR-H*RQiu)zs0QACCYgch5zvGJ<&b8YMR14>TgB@M8G9%j?AC7MzC3A^a4
zJ*yvAYkZMbHYdAPgbzun?DHbs{+puBLi%fD@)5--o}!54MF%vZ3q%#XoW^d@=;Ppr
zN*^xT!KF?;0dp=FPF>n;jPNa}36CZ{y(}DbbN?2U99@>G;-4Mj6bxF-1n(iH?#Q<(
zro%-9zm~H|2Rncs)yO5|3iO=;zYh=-LK3gBTVQJX&qSRZ2Pl*IBhbR>&ac*MVCHC{
zzg19mmN&dh%R?QV@a9)PMHEVewFCK@g0G6qaRYo?4pA9U13R9~<%3O>H9d|M!M#b3FwYl{@W8c)hntXGYq2nuG(R;&^Q+b5
z1mfFK&>iQ@IsntdfF$=zeG2D&i4#HMCLwe=gO$oQv?
z?e64zhyi*%>8#>mxLoA;w8G9UasF^2m<_=cZK0AtMa&9d(Iv}M(xdA+knCq76HL7<
z6HtF9VZ26MUD!I%kwWY>s-C(D=7)kuiF%!NYO%VjF7PL=R=K?ywySsr5DpbRF
zEdX1E^_Rt}WwB^1o6735I*vMZrDQn-E`7(+#!n3=-r6l+%?AV6P?*o8Z
zN(Zs&RMY8{Q9~Hw`{w~y;;>(N<60^GPNEyrMw!XLkJ@g&{(9TP)fOLkP)4>e;c_z
zFCmlc#Dkly=hpS#t^|p!_{V7@E}qvi#fEJx3~sP(3umr%533ETh9lm-ixcZnxy_G%
zuy}LJp-|VDj$sd??L``6wR3KC_ZKT3GlXobym4@{Zpfn?)-#Q4LTBnT&Q+%NY~&zp
z2$GM;v*g=0T-L3ITfCZ6qq$@)AS}pvIIyK?wDVH8ti;yPw9k=_jO26Y>4Bn^B}e*k
zF0L;uHB8)PKYACln7-J1F556ZRBK+KD#TaW6a-jwbR5XiZaeXLA4_5Wk(p@?Qy?fH
z5XDchj%J9QBl^uxy5r_0(NOy~c)#**S0*jEH9eYh;&CW7YJ38QYDJ-uSHq4Ai&$Bc
zT?e3E*mcjc@P+8LELrkou*@P%m5MwqlhS^4%UaA|u5ZF}8>B%=3)%zlA$UD@ltCTO
z1};{@c~gACrO_MH#Y|T-^NJ-Wm@55Yey$oa&=hs6DB(P&^K}wTfq6cylv|}R^jclf
z0UMomUOLfhnYBMmJoL2iLW+hw=KV((?Ac*Dh*D%Hf(FOBDukcP>immc5
zfAM7(zOyM`n+hN8#IPszRyfqt$1Ov_j7Cv>|ypJV#9azrouoDpET2|ATlBNr__=w09hAs
ziZ;vH13ombu-w54kviWk-K$V5&teVaef>mbxprKCCw@!;VcdGHJDnUfgw!k)vU(_zm{i&koxMvr=TkXt_65>D5bD9Zf$0aX_g^O^jBonwsR
zFLPZ!?62@Q*-t7z;XG0NN?9}ULvI~9xA;PxLfi0{+=^oF
zz=Yb$I{2@sE8*Wp_(%c%2o8%MlXm#1gXRujonhfknPK2-lF>B`8ufKkXIs`S)^
zQ7YAR$Q_uqvFRT+Nus8w-Kr(eqa0O`YrQ{|^!&YtCP78W;`!QLfdz!bK6wt~HFkim
z+5)-wfc4I00?)O+3#rue?Yhe>YY$VjdRWlNSvZ&S)3T-CZ)vW{yNODN>VoYWdXqWN
z%IkvL7xs=2a~&!Dd^>Q=L*8(z)kM@7+h2O7S*qgj5|1J;Wk7D%-Dxw2Z{D3RaMN3t
zCyrH7p`Z({dOrPSGE$+cX}_tD%a{Uvbr9`FoN9I_u-t1{_2wfwU?k>Duq;zq@+nz5
z#3*o0N35O_qj?(@Zh+?l+n25aIw_Cu$-q6gJI|@e0%Rc|mmjJNQpVezx#N=^(7Gk`
zT`q9%_ar?Qt%B(RX`b$*E7*=&9i#xR{(_X=I%CDSg{ysKkNqfQ?5cF#{PHpzF{J`_
z^M_BWCL10i!W=Y0A#&@j0>6w!rO9-=2Z@LDBc$YQ$GzKD
zT7XXKL{hzOvSLekLp;Vs^sbIN>C6=))fX1t8_v*J3r))_#!<#`lmy&rLhq#8
zzweAb{&SnLl7R2)rJaG-@Pw;{NvN#uTKTQZo&c>Z8
zQ84xK@MTzqu?GtZ7))(Sdrf`#V8v+sL-8!d
zjodCuZqJh0y_wN+FK+6O-zvRU8n00^{fi`4ui&-R*1&hnaUS3C#^A_tu>9tb}(j*95)5STz
z6H&`96bhxh5uR8X__lBmLnSW@dNFU_v+R!kM~^2q=od-T-F7-Fb+htw&JiFUwncItZA;?%Xi4($@6Ea8l5NGofMl0s|s6!Z@iL-JlNTJ}e
zYVlq{7I)@p%ac;P^Z4&&TPq5$uuS9&!Wuz$U9LqI-!Cx?z9$F`^8&8&@|?bk;@M_t
zQ|t{1(#gO#<2@-?kA_G^go{hKp9)M(m9ur0>W3*U$}Qx3zC066c*i3VLl6gGo-Lfv
zg`;bHbLP>t!1WSbfI`|vYvnKzO`&9^ay)g`$UmCC05i+pmD}}ZXJ_23ig+G|WT#~N
ze|g$tnU_0uJiB9MFWYoF{X2XJ`|GP<@MvFTg0*C&sU1!qSDeJd4d4zSok^nNgu179
z`}P^ow-VVu!|7=;zY5`J{gYJhpDFse-<;KYjd8hqoOPAFx-M3QnQeQVA8S;jH0%6D
zf;Gj0^Q7NmmI!OTs&*g%5kK-X(PJOScE3kBXbn0)n-(
zMk>OBUS7*40Lp^JL2lcHhs5w{1JV1WDExwZ_qLQYAAS-~1I?m$)R$EZ>zMS=s?KMj
z+xE&CLQ$Yg>>HOfemfJ$U;|G+Puo1LIK%OWbSKhNPYV|nE_ZG26KUkTB0WpKmnodS
zrg`S4lm7755zgIm(*l8=6F_qR}{wU%nkLEb=T
zj|8S0F8r^{bNBuBp?~H`-f@Eom5w
zPeHtUzq=j;F3%BlogPD7XlCZ`0kUs<)p~v0=wO3car6(|ty2xn%ncW-__0PXZsmI%
zknQa|FMsiKcl*%sH~j*A{D*+`vdQ|?Mj>XYWxM9<^g2K0bH2#ENv!{zCuQ2#b$gKd
zmeZpvSiWg)W_iyyMBrE7b_`eL7o2&jSV>r-sfZ>VG1&{7BJ6ITEr>z0zc155QjN2B
zITlmiw>$CT`d?FV=6^*t7A*9x0BpCmQiQCbK#@vPQhrm7%~B5NXkTZj*I?1{mk}U8
zX0kshvZ$ur++YSYUw4e_NHy>GNJWkHQWLD`6Ot1*ZuIU29!@
zJ}z5=Wb%Ff{)cEylWcar)zjCuVV>PAwX3DH(xdA2<1>m*eifev6Jf=Nk!9;ty6WMD
znlb`KF%p4!E^Bc79e-;`%GuqEnT1Wx2wkwbQDeJcCUPr)ta#DJXa3!~XF_
zJvH*&K=AUN0oR+ha$1(_uTG+E4^=g|y=W-Dc}?*^Ylwo!x`TMViSH^C5q_-#W>x(>
zRdT)L&QJ7`Z+(|Tz9!{~@6u|M>B>Npd+Ee_nE2B0Ia0I2`m0pBF!5u=-Yj;s^J>VW
zw2{w)qf?wSnsu|&Vkl(i&T#$9waeoQW%$~!>4K-b$0r|BXG8c{GzX71VyEb1l}i(Gp8RNW@<%y&?nk4-sg_~BlsVj;
z3)iT7*Bz{at884}QSW;-ez%;s*}u_iw1inbx%9}3f;157vz50+0V{g433LV~(Fs|dMf#D`W7k1Quxfw_vvK_`_>gA{udEj)RDu>O
z#ualt;9Z_5l+B;>S9UCr3Ye4g*XjL$WU&3Wo-6SZBl1=MO*zCtR#dfTbfsW^6{nEaM5_iic~X?c0sH&XMw^HaBhM4px?`mkFJ1FRu9jUmPrbU5
zDIkn@f4c*9iVe<+EK0)8JGg)69W42D=aRMboctG#OFiVkx{`{~7QDN*JqH>9p>5x8
zEKnzBiudB_QF6xcNyg`2}Jy$@*s^{sJ15Lmh3#!-1`aAz&>swDGEpC3ww5
z`vt`=$%xTTrvr``q8-*xxN9b-YMQT^{Z!XCdyndTKhAx
z_tb^qOq|5@2hk-GcRTTp*Dm$;qZyb5A0b8KyPs|!zd~IPa3FVLC|K$SJBFlK7`ya~
z9tLh3L({$e?Y(eugBXC_LUgU|O*IFc1rM3PS06_B@E3jRh{v
zZORfHR>1n(0=7b^HUM~?`tIY5oEuy`C$BHOxnAhQ?Z0he69%;Z&wUWOlX{N45QJFX|*VJDV$^k3~^i
zH64O~yOfzwj;3M&;!%3)yN8Isv-+KqaPHY!#i?KEGsdb;%hDbMrk}zOsMwSZf`;^h
zrmwwU?yc2`N9zAib>AJ;ME9*dfPje7e+Wtw1yO-er3q1y-V{(!dhY}Tqyz{cQUpPy
z2}qGDqEx8@p@*(W2|XavOQ;DXklexd{nod>b=P8
zNHIc^?9yc%@HJiz+)uA@ThBwB67}f}=Q}mP>kwi$1Z?^_UYhd#1u?i2Boi$^!-V9;
zV6gO6jzCUQ!NcZ6b!FT*#$Mh*HY7+?j4vhS#HrV7u7v5iTx-2N1=J`>y
ztA|=l;{MjBmKn=ecFNV3`_En*=uDIjim)E3&895p;>tb@7SP2UFh+RZSsy?l557*E
zUVMLj6M|TN;{n&~=|Dl+TbeF*Al~qitY*^1yj(^GU#Np|K0*6!rj~7!$Tadnym^qM
z_0jeMh6Evv>isH<$1~+1yHbA!ggAlJZvmKA@V)K+M^J+$1x#9$*VUsCWugH|IofJn
zj_yLw?_h)Y$(EWERa%wVSuc&UZyaTD^_8T})^+K1MKa%6?cDitxfd}ojyA*=!+xes
zxlA3-RN0ZX6%Z%LqczyU^%+be^ms_8t$!;kplz}=D<~=GkT65kBcDWW_{P=lkd1dw
zIyRco8v7s+wGHp9wt{xMr%V2E;y?>k>nRq!Wf6B#lD*74$;Pkj2_mmwtcYwByvloL
zy1=QytQq2)=IubVZD&1)u?Rle-IT%LJ|07UU=C{$B=UyZb}X*MUql{dd;D@-APi&Y
zxt5Jshe~#J?b<2@SdA+#)wSPiCZ23zYr!$JNk$<_h=Ed*F}Uk4Q+|@&(`PR2!hGqC
z(EIkfTqqwi=>B6slpT~#Kp89znp{j~QrK*qeP`y|eEswTYdiXx4{r2P_KkE22ZbV3
z`BY8jj?)&~7sPQ7%GI_(@t6QLMXbBwHe2PLvRmUP@Q9coi4VH0TUuV+$B+$sRC4jV
z=%(k>6Y7V-X~hzHFEx?=-bZN!v#xRYE-!H)tP$LcCOd2=Me>D%*bzFz2LYSD6uo;~
zCSH_xD8L;~@y6!tB`5X;93xJP)C)Ozi9J4%8DFP=0hf9dmJ#0+91+m=8KlMh&|;q`crgYwN>Uweu0I-$EGKTIjjg@
zd>e*XjW|Re$CLNKdm4WF_wO3!qz4X7f)a*BY_w1IcxgsShxLRCQ*Iw-F`F+=>6OAD
zat9Y`U%ot7XseF;;q@VC+ydy>LsS_#lk~rFGXc|J>iJx7c=3^|MDulg9Y-xM2LF=i
z@tKrC!)52cMNT_;iLoQAb#*Nq%;b92jW)SrwvNV6#y$o4b|=#JiBij&UW(
zV@AX>_N2_>!m^EL2qd655-AoiFuC!yewpJD|fBChs4z}Fy+(7{#%dyH3
zg0&pYh*{0Fc&SF<-F(alOoIWvmN1aGXVT+>s6wxVDesE!Zz5yxc1&<(^fvpX)LO>(U=&GAB$KWq_7gFruX!>#J28!9P&MUOU9+R)QJ3
z`&Ke)_jCk$air`0NC3S`2}tmD!j`9CpKt6TmM6@>@?;j8GW_T;2djb{liMB5CM}{7
zxKs=QLux7`Z$NyD=M{17j1#b`p34LTu?j)xJ^s#H0kw;*K+B9D+l${#2|jQ@v~mzL
zHZg(g*r2_k4ulHCyb*N85;OC-a+4XjxwJ&Z`&#XgSV)V{@>QsN
zJG0ih+eH7ZUq0Ed$)ucN4magX{*NCv8_RIXIU5-&-u}jN;@j+jgGa=bc|R>v2Fv9I
z=uL>o{uacd!DhCZh|`E%VoA-`{oxzAXLEeWx_mPC0)945m5m|Cgs(YVA;Zss&KDS=
zTK5gTGH6zNu$9fV(;;<4yNqG{!7=L};tROuic|149gldHDtVHH*I_~7*(&B+ikDtT
zeJbnbt>O9L!`;<|hfPcA?9Dd89n2CQrlh8}8&!*c!KefU1tAdVqv|p5g?5$8Rk_fPUohJ6?Z0pH*!Krq&bxM|X&B
zM|nFj@O5GB3d`(19>Lfp%9FbDM`X?c2sxBKS!1ouHx)|8+jPYTzXzvtn>cW*>3A-4
z=TMVhp1L*rgH7PnW!4ai4{nqoBx~tgH}Qp
z&VI2_TCt9r*;-Rfy0c>3wqvt;#KXVqE`K?b&={mJh_YhfZ^gsw;5Aq3@+mUj_%*;R
zgIIlOE;I|gPvK|~DCZf$H)uOgb}slg3SVCQC{M;v2G5Q=?(l50fscw{)N`jIW!SR<
zmXwTu-eb*Iga9ioSo#t{@k}ATBn98DIK|m`x)wlJgu;XCdTQT9O{TUWdTh*gqFVi$
zHHYf9?~q-GZtdeVq9E{V9UDoK$VXM0J{6Y?_F021AkgAJ8Fj-qWq%2d%aBL+egNuR
zeCudfy=={J{nxC75h!N62DCK%V
zyE9sO4IeUPWpWNE!d2IDHpuDg8;JVt_&YvoWO-Fh2hd$O3V=q9aYP=PEvS*&q4nfe
zr|JmDk^lS1#-^rHI+BS!-9O6v*18kaAD|E3yt#lxBJAusYutvVBGrLob42|OPqIsL
zwq^&?RU|`ZU=C|ctzpy@wkj*yn}y>ux88Y6HBFlyNm$=4dbIg$TNEKn9QqO1ckVJN
zitRj4663`%UsIiew}xa8n{pvfbN9ieWCkPcYe>#sSiO!+|;jP
z2s6@q{<{MCt0m#t(;({9$@t-`&akKR%y(;xKXK%*o8|813CqL}_>lf^h8oI^C1yp7s|&0^u7W|h^}bR5qwF-%mL!I-zEyUbRdnxhqD=Zz>Qm
z_NTe=TlZqxZc*rPk4J_nuwDuY*nqR-^Uj!lhJ9c_hDI~pdQK2H@!HKiqclJj!rqA<
znkdV*sNd~GTMV+fU}Cq}b9-O4U;KSxbB+hu5xw>v2uFd*21SWL_Z%--hS{LlP%u$L
zFfl>>59x$>>((uabD$5{SNX^J?%|o1coK!}o$sLKzOgmTqQh*`13#bh=YaG6PnJ~Prpr^=qEGMn)4V=*<9z(9
zn$A!05(lODcrkrP$IN}(AVQ@3flNC$(yPSE5CB?N%6^#^hNB8Ts;ax(WgB;Om6E}E
zG^(Xmc~;JJr9b~@lQi8>6?mvH+!~lOazpv7O4~hXVAAIHqg^MO@0-<|S`y)NIxy{E
ziFRJh$|5tz`nXz;vZ2DXJMv=i&$8mRQg85k86(S~EP1%2R;i9m9nuzj;ghuD?T_4n
z-73(3q+@qiO}D~$<3*|!N2m3W56B^A7ZI`^VfNsc#c#=kI!e7CaR0JDa(YVMF+4z8
z5Zr9E_f;;nZ%<&={)*>*aw^5~Gwc0I;d{Ouy-67N6Y=zS_&OLH&xi~G9UH#<$4(j7
ztz(!@fOh7)%QK%0`v4E(dC3FFY
zXXigxoJo4d&UfMVXDwQPE%0+ij3gq!yp``;OQQ4Lf}!wPwFFBeD$SPRAo{uh=!azJoiJCTMwYEqK^G<^{$4Svr`~&2e?QIps68W__T0@7
zhbn_I$e-FA5oRRN^Q{my41R(G_1=n6!{sSqw_dqP?u#`w&udPIi-Ph96rRVix
z@w*Bn*63Sadi#ux1mAIUugabBIhAUcM-ODa`sPhaL&(gOt-f!U|1=%9RM`uJj0G@Q
zq~z3-&_`0QT?FnV)IndnJP!}JB$kx2f?S+Jj(GJ)&|de0v2Dv!;|B(u0-+`6F0K2W
zw>1cOtiX^9+-HzHG$@n&#qo6>-yI+TWNi%P0*#@R-?vsZjn;hnQS*xmu7`Mr?R|g8
z=zwgO@{AfAwp%j9H9s1fJ0hg|O(r4%*3lX^B&s1CpYGv-b<6%|k4M~<
zFx?cBCD4}d3lk+}N>jZh)NG|xc8t_)fDpWQ@Pxin9TAO?W>=*JT!F
zZfi=h=kI#*-FQh|DGO4a;QpE@1hU?cr7dXZlQSEvO>3{h+*z
z!wVa(J%vU$Y*gir3rpEG;UNLsi`@PFd4EP3r6m~wfigR9)Vu~=7f!@MgvD{=;XTk*RGR8h_pk6JJ>If&)w0S|O$MommiWZb9U
zXSBy$%v_h0i7e7ytnPGfvTJbbOhCU(^Q(x(X7XHAX=DDOJi
z`UM~K3Z;s>_QgkEI{TKGRVK?s`yzxpM__8)X;78;KGUtCzBY2=X_c4cYXcf-F!N&seQ
z4~Bod2Iszz(y}s#s;BK@rf4u85zIObJ;RqV0P3gvADKWDM!JymI~P;zg2>-1pvM(H
z@~xfqeN~WNulMk{nM({yg*XQ1y@X;jU~$sbzx4KjHB5=c|F!}Z08iFxsX`vJE5R-l
zB{_e+qv*r`COe}uMzMe3_k^T2wP2$Tg#nPGYau-146=@W%^hQ(=@}{vSz*(Z4lfRt
z4s*_Th%O$J^G7|b?4&HG-|(44(r(c7ngZ9lCh1+fsj2Ia&ex5M*E7}CGxp2yRHIk<
zs?6L8h@s+G#Bs`|7~pG|3{_R@TP%*!SL{KvZkHAec`jOPi`iI)lW)B!NZc#rKP3(eiDf^J#AWV
z`s9+FBZwfCl~wwr1Sl*jI#H~=VwWFc;R=K4@oZs}c17UFHZ>dj^jgv&hFl-TOVke5
z8PQ;w`7)k+aBFMpDYE84Uf92`TRQU0#%V%WgjG1_GPt<5pybi{LQcDk7c3_^Y;j42
zMs=NK4cV5ZwS2JJutlD`$PAVtPr0;5Tn|o*)Vucp)9E=KCPCou8P@Q`5L%D)#E|Hy
zs3@U3x|9s@MgVvi6(x9<72pR;E?wK1kkb>6VPX_~{QQ@k;uhFqI`eoVqtwMHIN)&$
z!VpNw=Z40{>7C2Duu#b_FRrm@6zJEu#luWqcmDorQhWclRsNcVFuT!{Cx(x83iRU^
z!oSR~NVR%?FuW-s&H0@G4lA+4fylMU*$7EjjXjPUgEc&jaFP@J^hU^AJL0$G}s<|-y=QPuPFF=}#|^*mM7*B}ars;!qiKL3aIb338E6cUv_k&npc%ke)Fy>j;Zd$%inT(iqC$
zqklySYMxJ@*Kjy+oj6bbfo`g%YGUJrh+iyx@SH^^*2qM-@0o=~tsC6N=4zTYu5?~~
z6yq~vQtPf^2mE<1H!LL(dpRbxrDoIcrjLgQpUl;SZX6D0JYpfCit^l=`JSpR_irkpcpcBGYW6S4=g?hU9eLeKd`Apnk&Mbf7g@q=e8fUW#dGR8%KmI%DTc<
z;s!VD^l{q7#=vDE$ky5GJ3n6C=`#ZcKxm(ZqdoH1Ajt_Juw-As@aj$g-T8L0^H;%Z
zvYv0}NF|h&mb$yT9*(X&DM@hCf}e{E)(#M#W*EdwhFQ1RIfmP)q4`Fz;%^ny2D!hi1dQXA(jAn)xiYDlH#{!Zn4L(w8qj;)qrsQ^sm~w
zy1H<<+ePYG^`7Jhj2%(~pmbPQMEXu|nwM{sj}Bx4*r%25@c}gGju*OpSSA==2ml^j
zD9qL9d#Gi_b;)}V7GBe}_-P3yw#I$KaeRV>R`I8H*Vfh!ukIh#2!Jg2&*i`;o2Z%j
z9&zUw<1L7v!PxSPs%SRYe@|mG8w3iFJ#I9j#pV**N?m>t0o{&S;cTI)Zeyx53
zgY&p8)0b40KR$Hi&4E+|_Z7su5A!_3X_4>$eJ_-(Av^XQ5#G`i{-Uc=iNW>23ccshc@^f5~-T1Q?1$zHkCcJ(K~FsSo!yLj2St4%cA
z+5CsbWEB`JH*Xo&{vlD^vx!;O2Fc{b%LNj<>{*R
z5R+=-y~Z2U36Ar8iMR!I9HphHd6>8SrtBc7lDy4Q30i#8q6ds?#w9xhM@D=!PbPu!
zW~MHt!ADDVRW`1wrpBH_RX)?))HG?Sn^tpaqVK!mq)^q>WBlgiWcu{z&W&T=x<6Xn
z+4wI%&M?2Z^?$TSUFj_h(@aD-PDD(8@SaJ)KXVEkgtD`ivF2f(WeCi(H%=yPXpjAM
z%o|e~DyA=iD4Be1niZ2Tc{tRIKy!W;QyQr|f&dlA#9!6x*mnAn!|Q+6%l!Q5e5N$s
zVkvgiMeqMA`8OBMPZjoQh4z2z%S(CnpQ=X6yMIl?z5i|+=zzcWYti`m<(H3F5UZo;e?^e#(7Hk{=ra=i;1hIe`Jz_x&*YI^
z(BHH$$MpZoR6hGzE}7~<@KFre@*nYdGgPaQKb+A%m8Q3nPYg+p?)~4+jjPNjUMymN
zWm*60Dq;DW<*Dd%y$XEye`vJ)wZqG5n3e2*tZ>v|^xGlmbqdzg4sV*ahsbUBWB+{>
z9Prg$X=pPt@Muj&ROS1Osc{s=A!2jdvF!0DhC%=U7`#=Cy=`D_Zg%e8fS<3ekDEsv
z_38;&BXC+{YvpYxZ}Z#^`~f7y#3e+;Zi|RZ7)prAi%H0fON)p}$%~1tx8BMAe>%9q
bY@O`=|NkA*KZIWeI{+Fgx(`b4TZa7?h~I)p

literal 0
HcmV?d00001

diff --git a/doc/design/assistant/blog/day_7__bugfixes/profile2.png b/doc/design/assistant/blog/day_7__bugfixes/profile2.png
new file mode 100644
index 0000000000000000000000000000000000000000..4d487d02a2b664eee6edbf3fc5135732eac25364
GIT binary patch
literal 230937
zcmagG2{e`M+dgdAA(@g8Vkb!wl8~7sNk~HGBuPTZT&5&Rl4J@ANs=U)GKD0`l+05S
zGS4&Lu|4m*{_Fp(^{sFJhNnl{-uHc9=XIXPc^t=i1!-%Zq^ISeB_kuFKc%LkLqgUaIcNfojU(tn;8rH11tTOHIcIN_gaoBva-z9vsb=0kQ$MM>AK
zeYER}`MHzrqN7QsZ-g&Yo*&=o>VMr>fGW0|@(zny+Zl=Eu#bgi5=EnSkN3CTc|>4l
zHXN$_qU)bUpSRF`z3r)G^}nK#mBV%EEt&K}r-%%qEg~y!&f-pdrw8NnuTV9&G(A*g
zA}EpmJ8Q0}<+_DPSA%k^RqmhRFTm#%Vi`D3@On
zW&iHnLh;|*3l66?4KM9_>*Taelq}E0K**)$=^@tO%LNB1Hou2(=5Bf4{CL-hgVf7H
z-UTvtvQ-M!;DNE+9xGAyCX?;|UBJ+WuW}*-k31cvPX{@rkJ;Ta^LK5d!*93zcd_ET
zXai!#1v@!=`ON?4iamV(uPfd%)WXTlZFlu*n0m_pe(H1TU`67+;NWdEG*RRCJHyS%
zCP#8Ue0cES!KB-#;kPuKpIz^N+0qh!!gWcDk}b|9>l_HJGAiC
zQ2|rwWyi?=_u-NGl9T;+X#_b%b2763ZdHkqNs796;jgpwn2Su9+OzTdt9x2)B?w9m
zQl0jl`Q;W_c$QWD4MQt?Uf#?%b}8KLUuK?_yTU`;tuGu|<3tEh?wfL>qw*)->#}o|
z;`rzkQmSyW~u
zEcBN7$Hm1>e41g9+Oru7-d>2GqS~m#{XHA&l72+Z)a-ZfM0t66<>l9l>BtHS3hoC7
zZztrRAS080?cKj4=i|p82?`syhtPZXqQ+Hwc0EtHZ6RMbCVQvM{LkQE$@=n0@4ZTk
zrP(H}lEvS_Z{EDYS}Dxd9~X|i=HTG4GC$@~bbIhe9F5=e25aV^JH)TSBE>`f{WJ3&
z_JM(cdubXwI#vcE&A5*_^2y7a3b3rVyk4JY+fEAW=RNdvWaBStB1K$XUH8&3v$0(f
zi>|kJwzGTNU_Cl9QTO$$)a_L_d;82vi?r9TV6TW!K4j&JLZ8)==eR&VK*?
z^XJc#ZnAzv$NW!k-@eVx4p-q#c}aTl(loZVwl*|`MMP|oXt1`m<>KOc5E8N_*N4Wh
zb!>iZWg+nHUH<+1!>j+?zkk1>p&>6X@9S5!5Y?vER)O$p3ENilECX6vT0Z8Wnt|U_
zZu$B7Uh*YrX~X}T)3H#-!jcja64KJ`*5a%?cLr5js1un=cOWgWu(8Q-
zJ@oPM>FhM74RDOrZ|!qzq5MwAkmE
zUyOCC;_?2JoLo^;b8yd|vW^ZDuV&>6w|C9Y0>~w`ChHoOkcubLY<0B`7F}ikfG)eJ(Fo`a$PM#I@sP?oSh0
zvoB1|=GrwUCnx-}l9H0O^{X=TSLx|a`Nn(p>^XMq81}SXu2E703k%DuW*f2S@8+VT
zE-NCpN)j8aDSQ=Eo60hE1#kX%cFc2Od~AID;lqc>!M#N8(5n8acnd{W*M(mhMwAp3
zrBise`%XXo@ZrPPfoUUZGDX~ZbGm+p-ob=Av-g%k`=4*wps>n5cKGn&W5?VZ!!^a_
z<+~djKM&4a7Q1li(q8{EH4P07b@k$@Pss31r>y6gpNHP{#Q5yWIO~;1&um#P$rYf<_%F3uWCm(X16^s~Z$rKR~
z5D*lkrK9^`AawlraW-LN?KE9ZNxPrJ&I#V@`DNy!v9&IRMepCgM`jsLP0-ZPu(h-6
zF7fuha^;Gx?VB?5RzxPUNpwO&-QbME`r@S=!y>yb={QZ-KVP@A&_;xZ_ow;S47`5*
z`rq*IRz^mIU8-iBprD|~R3+Qqy?aldJSk%O>7H`p-`d!PZqJFFoSc;8Wb>@n7cY3j
z)Mh8k8Dnd$!mBNF3~Q~jD=l_$a;}UQPJH|J4N-GhtgSMb&CGjk?!f~(f)cUsUoA;)~dg9njZpB^%L8WGXl)3c$lF{6MdURzt6uAf=#GG?fxRQ6!kk#`28(MPU0
zIy!QDj%Gc7{`|*}AL0A8JSfP*)O2-q#jpJm5{&Rzp1m4bvy+9zQX=*QZ$rM*;7^;m
zmuF&~rMpgx6p4$9K6>`dRxBD-Ls~}0zF~4^u&%@Qm47@tg^+1$!V4So#EHr;A)LDzs)mM!kkj^t%2E_Os`>Wq
z?%lgcf9j`CdwO_q-(Kv)3NhXEns2+}H27n{K??t0!{v{cc>ivacL@B<`3KzaP;e%Rn$f
zes!{Z`}XZByjibasb#jRvw)iz{Z_llTuFia?j*br3vG%#|N9vN3lksT?
zBt2c-A!liV(s!$D10mH!ZIPnc$6aUSoP7SA7+#&M
zEg&0#o!@)@IFm}6ZcOdpPoF;FIggKzpG(y=DY<>C^8QXX5mO-%5fVB`dhvpbp(?lM
zc1L&j?GZ`#A-x24vZ&g>NaaPI3%oSH9rG>4+7PmOnWnp0SZ2-icX$j$=s)C9SVy(r
zE#o4DZ7(h^e&E0XLBVfF)Sb5J3LZt;8~7ICn9t>3_OqpBFO8VE_=hWX@h5^=Yom_@
zs_}74JHN`v$jHw>CL^(B=C
zCe|bX4VgHOwI4-siin7ain^zogp7Li>Tm@GI~~8MXnjo$l7XkECuhhP)Xb@=8w4dF
zjNiY1D=RCTy7dJ-dc+itiIp7f1fwJ`sU4>xQKWsl%vXu
zibvI!5$ZRmexJxOloJzcY-k|28`w^mnx2l4^PFE_TftK8*s(+Zt#MmJLzd@)9n&Mr
z%a_j>XHf@N{v90jo~f18NprTeyrZ00QCYbwrVB#9K3?y@Mg4Ki
zZ?G2U2vJXIw8?pIAY^ZEFBn0q7nPQgaXGsU;Db8x2dDifU)3aFTR&o0SQrWdii!~H
zH59kEZ|xc@f7;|AA&EuPDLtyMug8W;OiX2?gaB>#jTABa&
z@#FNgqY$gil^@S0CmmR5k4j5RPvro2S65eOTh&T?Exzj*GtII%0p!XWe6z3%lbn);s7_!LYrgT_8yUI^sE9PGUUrC%99L%
zm%O~VKJh%$Ox55Y85%MGnjGt_5HTypwqk!E`_=M2uBa(hOT;+OVXQr;ySw}OgN`vh
z!HAtZcj~3@RC@H`gSfZ1H$lnI&ksr2!oq?;;j6|cC?KH7l>Fht&jH8UI2krN`mk!t
zd}D5Q_91uG$eN?_^0yIIqRnWt*h9V?bNVY}Qt+f^fNIN@>Vau3g*9huYXNELWxI`X
zQiiOj+(Jd>8lb^{Da*i8Y?shOVJ4b9QDpRpi{l1I2h
z6Y?}_TU*(&kTXlYvO=ueH1yeb?II^93$3CYF$6mL=Pd1CMmDauop3h282~Cn?=&AX
zVi?gVoR^s?5?DSZyTw;=64gdJ&fnjk!nd@egVXtKU_ij(Xbps+PMY0Xx49^Ld?aIF
zd33E+U^$Q&R_vY%ubYm7LP^@p=j#$3_4Q2mRR4{P;E!eg)NQsOOG*^J?%2yjgElRg
zO@wM7L^Y|_D*f$SQ?#WSM*?u0h8dctNiWV_yTa7?P%sC98@<;*WoAxH57fv7;8{62
zWj9!>F!9OCauLW-oy>b_{L9>)I_=)I3;DrWdbe@Hy?gggo%>)S45W_zMs|@-ZDC<*
zihSb$@?lF$q-s*xf26>;6gyexA-(kG?EBkV?+&vD6n*-{ykm#b#w%?B>`(nnQ#qGW
z1H2p@M8m7mJ+K8=mb~CVJE+EoT`x+}>zI#Dv*6|kveY|+H$X;ed|vWB1CAjNAGVLp
zpBgFNL38KNpFeo!*xo;X>LzNtP5rL?_wOI}($J7K9o0D_qm{LV9z0pRvJBp!vs_pI-|x}>#IMPRSRACD)eKA<+eFrE
z`Ir+^P)orOxR*JI^b}aZUyO|%Vxh&`wFRCu+sp}Fb1H9P2vFW0R1s8ZD-m1X7Zr1r
z@-LZ2T>s&%Q3Tm3+jEg*j`_x_NjEmB111UdDF}MzAi~>2d1CQgoT*J|I%(bRgS(4N
zy@OL-@{IvfWn4zx=9=}_*4B>WDJUy9Ha5}_K8Ni+iFCtHMdmDh3%I5Cb+`1(6YD+R
z7+^x;;3%DU|AuU(5Zq!x=F?e9NlW+oD$zJFSZi?=Y7OEYJbw-xFPjSgMUYm
zN-uw-6IW=}5q7%V4wJKIm55BZOEmuo}o!
zv$^Q=9@mBOF6=i>30rn9E?F!vo+;9rTJ6HuZ{L=uYebZawav}Vv9XnuP&n>Acu-+3
z3VKR~7a%K6ga2T}x9)D!6rHVBG)QvUZL(rwza6B|b)m(~1~z>5j4MUQeRik;^~PtP
z`owcQi+AtdA;tSsGyaG=oRprPo|bmC;l&9&+r~2vii(OL4@^wD`pN_QfB#lhRmBJO
zkBxr_&H1h8RONKn~N9=5xt~keT
zE-tT;qtZ8|xM%|&tMTE%?dUPzaZxnt)vH&CcXW4gnrxtn%FJQ{
z!&pfkDRgYUloGKaH3PtBz8bFl;agef$*+AU(H4@Dfp_j`d{XG2a>LFv-=~ctwl#dfKMu?49h)!n#pqfUNh^xEY(QnEe=wy!ShQdQ|2;}jiGX8Bi}
zUS7zy>RSGd-(#WOqO*a@mYXZdeh9RZNXhE-{8$GgmsGE_^j5;>*XPq8hlktW!u|b1
zhc?`}kD#P{{CKWy8%5+Mb^lz30R;bgN8L2>-h+odc{loTt>dS1TXU|l#nw{Q6lsYe
z0!&QC1_uY7rK>F+kn`R=&&`zSG(-q9g
z&yQA3T3TG}d~jGqdiu*^(M3f?*5d6|Rn&pyr&4q>
z^hWyov46)Lq(G}jRWq@%xh~j^BCTS@t1UJ7XMTB78fEAK#!E<`^&=I8yTCF{{sKQ
z$IKgslr|cD#`Cy&$5DIxiH-|>gY&q?11z+G<;WiR1-mT!mg|pe2AG2?EV2kxRI~&?
zqbIi3)^5Wok)-Y%1Q^&?D*x?FOk@-5d%&5-9=q}Tj!v4_@^I>Y?qr>`_<_aVL*s`^
zxBC<-3rB9OO?Z3%jlRMvWZ+X~4(9r%bXT%1JqzuLq=r)KQu?!LKLOqAGR;#W&G3Ad
zmY2IL3R|OQX%NT
z`xh5?TV!>9ZbU!BaKL?R(rxwU*3SC4ojg3=^P?9Q6*h&T63WG!p1GiG-6Wkt{W7~I7(-P3
z^LzjM)awqVyEhwRn*Wy?_1$T4?t$v1bLW~s2K*m|21GX}E2~6TPM&@lS#Jx-*%G^c
z!A+1v9LlX9nREEuok=CqnR$}P1o(|&l3K{S`Bh4ctRNJ01yc)I7X+oK|9>?wia|2P
zrn&(~uv57m^tpb^nOqEk)s`=G(gav+3Px~ivw@eg@wc3<-)=F3x}Y9>AkD>)-8MQf
z5TIIKtUJ_A3G(R&sU1{Bq$XjHtOUc
zNNqaB$4n~VcJhT?!eMF)=1yp46{pG$B4H@CcFzov>76UqTI^E9|>EM}0(`%6_1f%`yeL9N8cX(wuOhpyB{+h!ijWo)>weleP*CD$}G;BJJ*iWO}oqzWze%rO@HI<>9y6
zzyewRXvcoEC)uDVL8N(({L1ho{p*#w?jldfGNrkV_Ja;m$ax7K{|1kE@&C3#MJ#3vk@CvF>VYYz#7H(a{W0AYdDt>=w`vi;Ig_
z@WzTATG$b3J3(&+GV*li-MF-=M0~3eDe3I$LQtL;1gm%JsQ3CR8qPyTAB_u~tB0LE
zxIsvv6NA(U-c*hn5ZV=jjYZ}CsHiC5orNB6Z)@uzO_Ph(yzIJNCg>3gUBS4I$Af;S
z-{u7@1b&{>;O$^$M%i|q{&R{p;JsBX$E1F6B}o67A2Jgl(Du++h_e6J8E#bXPk_kE
zbIkEiYQk+tDK0J9sfy&L@wV*2pK0d~>b+?NtVOs98|Q)4+L%pM82I})Ue>+w+?&Ee
zPXHM}cNW^qXmMZfzz+L?UdG5Wj4giSDP{=HrZwHCQFE>w8(xBNwm6!$<85`?5+ByZ@
zms{4Ye!%h8Y{N+?Ezo$5affa!k7Rl;5AO$1fAU0pst#t=HfhpQh?BJMra$P7Jsl_G`uFc&EU%N7*J?&?x5sq#e$Z``(4ZrqK82E;j$Ro3
z%Nn+0l!ld+6}U;O?EJTH#}P#z+H7}d<>1mmUxMPkTK#w*8s?5%d++`bNsz|!U72;a
zXh5F<(=I~u0rH8hExr(q9i3rXB)xxsVBMg?%4l{o_VU}0AFM$t;ma*Ze(aKDCEymvyNg!hT)PvShEIlaHxD}_
zbk3YPbArio=5K9}*UTBv#J6wX2KYm_D%s-|{_NSaM~_6s#dQr03oNT2pCb3EwtStQ
zUg*$ALFGTSuwa&Jx0^du!E+wG)EPB3a6O0uzQ-VG1C;lKsezGzuI%pa4wZ#F)E)x!
zn>V_G5r7eLa&n+NjgMy;7#kS8Y8nP|gjT!cwO%o07Nnx1uOCSK3N~vyA@b?
zNVvUxia$K8`p?qLAX1V)9S>lV6XXzN9gt4$Uj$$)K!(-P(c$)9y#Xq7Yz#19eYvIN
zV{R@onI~^Fwirlh)CpvUD9uzSDWK-bOz*YyqN2FEK{TFZvt>{!!>e_*wd*@OpFMd3
zG94BUPzVq=m1wnXY;0UyByjg`^jM3IF%%nwdr8~JRP=gy=7<)NPm>g2L+9p5Nd$?|
z;KBv>$=tm0o)4I)e3Ab2B(Yt1DM7iAI5T#Xf)z;L$h^O%?fr
zf`S6@4h;em0reEY$HhSXXVP~oCGZh{IKWd~G!#L|aNw2cTm*|AOrl=b*Q1Lg{;FV5_6Uj4rj$0WccHB0i>qR0ncP)(#yz)@GA4j)&;%
zJll39;e47dgkT7ybmkt#8%^7sK%!Joj3
zft!s(i;evM(Y|Ngr4Y?$Y)lLZB9Of;GbeRk=*6@JkYm9~PW=1l0dX{S#e@;Cx{8#;
z;nngXQ%uo0udjb3`a6i?NA8a3ghh=@ZjSsKjQ6x#Tb}Fg?#^{;$3+1=Ymq_s6o^pA
zrb0KyX6AJf6c0mS)37WG*y+>o2u()<1_Uvt>ZGOXWyC0W-@bWsZl+Fwk)hH012jRu
ziQSW3XZXV*<|cnKla7-IZh)SOjds8)x1-j|$^}@)=7Ept*IiZCH
z;Dv~*lLiYzZmxc66G>w7^hCrze@;Xmatv>^okx2GwM+Z#*~3%Uwuv7j63N7)6^TCl
zJ!ut(08zl#A3b^$mIXuDHIO7d=Kj&w_%H?lcxm#7qYLH;`DoV#M*bIQuN0jNKvZDZ
zkpK>Bo-}!IkgbrY!P}O;I(tx#OQbL*2n!;=;qT
z@pEjg_m7cg*|@kmySv|N3m_{gC2B()0(J@qB`lvXG2DV2vUTfL5C^0JKbF%1S}XfZ
zti;mFiq`01kO#Y8+H1_oLw6-n+2>8NfYXj+5U}=sw6e&rQiG=jom;`<*%A_
zq1X$=Ea;Z>j0|u__Qgvt2h;Er1X$D^9mUwAD4XWPw(gjOc|=j6j2>b;Rt-%QUf
z9_Gj#?iigNBHI4bl&Yd)X+`%fWLj3%r;(8hK+)?c2*`Q2N3*H~SiV)%aiFrn8mDs=
z8YUi*hev@TvDZNgNk6jMlERlPKzSG4T_9ZEuvXaZK=-t_b1*T9Uirc0a2{3?6978X
z$jdcPYh#XXqkA{IumFRePu%yj0=iJvU>%5z)Q2_>Im0l6o0IbhM+hisKqeFyV_{ct
z=IFnlFM&zR&OQPw885E|J_y{qW~$bLCyyU-h#T2gaY9E?k!(g}
zrxsUlin7z?UE9x0>Q|9rVQt8rQIzFoESLw@)l85#F*SWy-e(HS)v1HDa3ablgc8+&
zr_Y{kKys|MoIX+~3(M0{j@k+WC$wa{vh#uw+5&{FuQXG4bRL3J3E~=Uh$~ysWG&i0_%R
ze##KnSgHIkX_c6uR(|0P4-e<@7)rz@{r%fwQ#%=+m{7QC;IF|C5DvbB8_Bt7
zp&ROHtZr#j(OF|-FKDn-{)f>vKr2BS1`z}kor|-x&ts3G?w%eqXex6a41xVui9}3mJOc
zp^{VUoB_(<*-F;N5YT6d6OkJHEnt~uXMX~Yoj-pdLOsXO#Ml^-L?;#w!|x*cIS~ud
zJ389h#4t7BNPHQ7EDXcTRLxJ|Yav$J0I?gqzv*L*O-*n2ecn=JngHD`KAyqKTG5pY
zT>vQi?K^j(H;Rpo#nXoZ3bhLx3WQ7r8Ig|1+gj0ufxuQEcfAb0xSp6fUxNATNAVeIuV*>dXA&GL;D_i>;!&ve2#F!!UF9_
zbSXryTwhlgT4{l}3JQ*+XJB}(D_FHb&d_cv0bTcoVWlqewndiD>H9<)SR57%TwGi<
zG`@qN1RDV%3H;zIT|xG6lG0#z9sZs)-S=&_TnrdC03A1Yy4E7gA-@w>6*S|6=15&#
z-QvM>S^(k#{3c~jjuDjL7HF{k
z0PBSmmw}L8qP7oRN6@88;;F5pPkV<=;6Iayt+mM!i`L){ZE0${BoPY@39V3OCiT^|
z9VPjOMn<9Ba)7naCeVlop#dT&t*))%<61*$LaU%I`2Y?eteeNs0VbP_qqg
zJHGB7k_C8}x>o~G-my@m4O0qrWE9-9V1%F7&PVHz$zT)56F_MDru3i$2g;_=cARF}ujZ(PIYXtJQni>!e?=RQHgX|6D
z_OON_IwfV_fdfH`M8cLWhfE5b@zCMHB(|N!rY2B?sSzAs!BqsSBAnPm;5|Uq$jCtO
z$1}nx(6wvZFbs0ym;AkA^sHz}%5DoI%bRQG;Grfbs}PxNJM(UU$;NI0S>#Y;31n=9
zhf8^6pdEt~D=XUxaIhg0sqh}&v*#kd9c(@mZ2)9R;DkV>+9Fo#^#==r{L~r11ykPVM1gLKEcNf?m~0e9>wp{Kfn8LwZXdBi-%+qYmu2Jkiu
ze2_BJpAgCJ-YGYdknY3?Z?v?G46=ixlT(>Oe0d)@11KENo_O=1LD1JHu^_;^*lfrO
zI}hqPNO6gC+#2sZ7Og?`lpS7R*ha-;AeaQ5!Tv!X4IW!3jU}`SvnU8yv@`uvi`WQI
zLoiJOfhe|C@BI0Q$VkZp*SFqL#o!S{11Jn=#4wbAC42rDZ2&MS$btBvbBMYSRYnM0
zv$J>Z-o^A29CzK;;(j%aU0w8#T~Jx4`m11agg*#I=A;H}>K*4opQvAk4g@L{4GVTX
zM&rO}lS>?*vh@Mr#jqAI>85@NgARl{6=!@eDL96m0SDj#hR|^OsSx>yCw$Ug3tg_m
z!^1#W;50*Jr`2q2Z4n8eTv&tYw{E?9{W^T$U|Qk2x0UFb41qz3g_G0Bz+e&F4!qzY
z4<5h?=|@&-lLPkQRGKa<#IWDN&dD80eTD2e6QN5P93W^9MVHH`Pj4Ga$5LZ^KxZzT
zD2`R|miHr4GjfBd3U|EeCL51>2(|>N-&Vp1cMG0;Q5hUyMrI}yO%49+moNW59BppJ
zjdXW2u(}uCUXzwmOVN?M-i{0Vv+E_s^)QbHMGkTxUcK;kcfPxL+}ykuJTMH6p=IVk
zo{)<$?POuW>qmq+?D=zuA!{RM8^33kux-JsQHA>i2M1pjH?*;N4HSK~{5g2Z}n00t!=umVlRB6%VEu0HzBHq*231M&QZ$Vkrji@WwR~$%D52tw|vjiTy#F
z4;oO4aq$PqXE!)968rcu#rux!1aO5&&S_~|SC_$*k!&y^$?S3==nf7H_-gSm3b7(X
zgET55#GK^0VOWqqyGnIcLtnz7Tr_!6@i!GY|cF_C*%%liHH`XKj!mHsCH7`2CC>Fi?
zJ6L4AJ!b??8>{RKL?)8G#jBn`Ajra&54=it{0?#%ZZA&i?MeP{An{V23Wbf;BYs2}
zmTDz?X8#;mURug(8wI$klddp77>)t0hYuMNwHLuXkPPh@lJYMD6NCIB0@qu$C8Vi$
zu$&GxBq=94I6I3UIdVin!P{azR5b~lJwg=;4oS!8qb=4MoDyalv`HIaF
zt?^Pb)d;c`L;>($Xp5YN>JuM7W`1W`_UxDw1fWul%##=cz^DQImco%&tgPM|WPOHW
zGF8Q+;Lxjx(04O|NLYIJs0uGB+hNwwnIG^JJhSNS_vU330Hc5>9UUW%`M=(poaPOi
zbn`;FK|OGHU;6s>UehpNcr|udP;hYU_<{^c?}XDA&e*6APGi`!SUgW)QZ&;{G{7F%
zj=ws~v2P*l!!5g?$zDFIVd_unU6+BQvLZeVQDHD%wnjPPL60=4YCo?iQy%cDf`8w`{j
zA*d3NWoKr;b-k-bX4MH#l%asZNWSZ4o}`Lum3?R46o}G=(SO)8!V<9%+5VY1bEh}A
zH#WZa-dHEedcF4=tXYCoz%pUN%u6@QH_tbvdap36$X%>dKr3dBN?!oCC<(252B
zP%J`2>nhgQV-ItLfDEb$zH2HvtbeTgI(h^V`mC=%Z{~Thn~Ek6)aEo&uCzT2nRPC|7qaw*zJiW8s9K
zmhb4Y)l=Yi%w6P&`f4iy$JbSH4(T`sJ;iCD5l~q0<<|}N{B^turwOAN=5
zC5CM*cnYFzc@w8F*LSWYwe-fMuNEIOlrJ6|C5fa4DR`Q)3;?$5!@LzFB&a^_*+x&l
zb|U+keeF#1DCQlm7XP+)#Z+>N4rw@4OBP90?i%A~qfR3q^3t(_bL8;B6*~{>pl@uw
z{znt`2#n6bG93-xbgXk^gSRS_r^+G=v~)rLM$yL3)>$^&OC)Ha6-Q3DIhRWb1+!#}
z2KjQ4Z!_n_;?*?W*Qu_n4~_6ET}1+tlCrX^UpVd2;gF`V)TRH&taEz{JDH-lH42_N
zqsjl+Zyx&8_-syHk;V!LRAm3zZkT5OuMz5AE*~=yjL;+|_K{@i|KsfOEAPW#@k^3B
z2NT*JxChjl^kz-h6Pp%FKW-ruySh62f@?Q2k)|_oS9V#cRGao$7*fdpKWDaSN!Q_g
zkRwDjvjy`TWyvQ2|5hlu$n*4%aWSBLw2Z`Q4*o1bXqC2AqcUzsf!?yEDmKREKkra|
z966hHaSQPn3Xwj=nyO#35K@NYw-RilM6I{I<|=
zS)O|a)mp6}tJ+F*yf_gIB2?I<0CXT=I5d@QvfK(?IsU|<+n1G?YMfihNnX)Qy)>mo
z$@%%%?Qbt+-&Ntocmc^2hJpZ@AS>%eK{}OE{Vm|XeB;0W4bD4PVI-Wpe83Sla?IuD
z8#`=lqw%{Z3uuQ41GTnR5#lgUwhqP`UawGz{B
zXMM`e6@$B11V&DP+dQOyFr`%f4a3eF{7$G-vfTEY3%2%+bvOMLvYyiz#)Mr`P$TZ*r=UkX!-}WqpzCsXS^wPUfH4M)8qGSlF#OLOB{$y)
z>_O@eAseHi#;ESIO*0uO4J~Z05e&X?rNd2Qm_eYpSddPsRKF%7AAg9c9~rOU3QmuW;;xc$|la2ZAG^j(Rsq9UX9-
zS_&TwaJ_kR3pF*$)Z|EOYm^G_p73g4UtcKqN=)bz%*?>yy17Y#mPu-WMGFuwM78eJ
zD_t9O?Pg{mQAm`PSBuOE!3Y?(`SDK)CVI>y5Bh=N!1yF=_^+B0A
zCdDByY?EQ-5g^6Ao2RD&2z_9!VNBo;4G9PfV@3-9TK(e(#-8HG+extu-4SiE<|*Go
zAGvyxlTBf5SPKfX5=KpR3=A;JOfvXrkq1{!gLj2^2!hkoSeQ-e5tN|>>6@U8`W*7$
zg1!>(4#Z4yqk0s@*9|T#ERaqr0K1N8D25_5IVmay0GOzg25=7W0p)B1^Uz9vCWE=;
zy`swd5Y!uYy>|vzV#O!t=Hgh$xTGA2M~;MR@Z*FDh#IiUkiHyS7X~|&;ozd-+Q)k7
zW1ExM8<{&nF@W~~yo8_V?tbUOuKil^UJ#}1rko|OiGlw`nfti|)J(!LCo9qf3#AV%
zL+)^_fub33Y;I}c0W=~h-I*8ZoO@6+iwGdB2?P+uU>0&az_qj>^|ZBJfX!_8?zPC0
z)ykThYXu`T2)*N}7;t3Gqo1U#8Fqg9Oz990n`L|FzTmt-vti-4lyCTl9h3ja}X<%^aPjMiN7*Nm)bZf}Q@
zmt&Y=n1P8XctgQ%qZ82|JMvijO$JT2pEO~DtMJ={DZA&*Hq_MAATZc3f){ySgkOh|lgNZ~83;<4_Q90akt>?c
z&L2{%BSp+iKetv4mbkTcNMyVX@S>_h6`Fz6L|4>zU4H+LRKl2o#p0WnL&%Ap*ab65YzcJtp?1}}4Q3f-qduWu2OaK{jSC2u}hS3WRxV3meP7dr?n3nF@GamQdcSmyxri9N!!A9?A5Jp)0Gw|kF+mnuaM%&*N`KD5
z#&Q09rMV~wY$&N}Oc+V6x5)w5X=i6A50wWRD4J^{Gc$N7mPokKW*Umr2gB<~dcXq^
z0fGI&CXBksU@|%17_%LeM>}(_MQRIR03X``#y+&WlD3Z5uU`idPe;bhG&KB{Y<3$e
z1hxjQgM}7pAIo2SqyGy?F-U1P_^x*=JOCE2N=iynQck4koN+zz{3^m05(F4E2Avx~
z+E_EXf<{ngXLWV&R9YZ24V^NEhX|7+Q1&2i8{m`!!N?j&LxZ)k`e)B(A!wlR;3xq6
zRh>H+L%iVRBIo~zGzrEHe~BFnk5YzS2b2+0)9&VG%$WRaYda<-1?37-O-FmX)sJV<
zj~^pIAvR$uG`yOQh6XqUj~ta2O(&vULIPH<1x&o4trv`F9G-<>j@eO+Kf=VAMz5u&
z25|>}m)U~nh)5O3FMRxn$r~teXN1xktigE$H^78|IKVjzNHBjK;S>#k*drp6nVudi
z=jj4z9dlCBalLo2u<$xU6vUK4U7ZLxK&xzYnaAYhcEh1h^Exr{qD8n8ed80h7=2vWJMs
z82W@5Ze)S4YJ&}twZwAljE3qB8|8<@-RC`qylEzWF$*E~fB#(gjfEzQl~Z4@OM
zl3&8;1AmxfzU)c3#dLLlVv!F#;k3_sP}L9L2h1aUtid};!)&lCK(YoK3D%DW)*S;O
zNQP01OmtMR8^6c&21m%treQ2A@*nC9yqMNvA*z&TxOsTcm!@b7ScK
zz2IZMD9SFxI^rx1VpUhL8ApBayQGjt09;5KzIsYxVjc-<$-$nF2Tp3LpXtKqiF?Et
z+$JF-BP;s}gf1j103HmBXSXqt%!R^{7&!xwf1&*f_DUF@Fhc=~w~nZYvxqeKw{6`D
zg{}8HcDxGjRq+^DGB9uW@+Ch(3HO18yL@?Hkjl5}Y9*$MVP~{3II0M&VT1L(ij6Dp
zF_nqQErjW&#btHXi=YIl9SITMLn{BWJ_mEF8TwcNV~h*$Zwo3j?{&zF?ONMQ^SOTt
zPzklwT$H$o!yK?bAw@jLz+!3gv07(oIry(S-9dX}-hn?n{PE-4t5a3DDx4$}6-5(+
z@-&F-ms7n%>s8B0t<{lNdjgapdOH<%L5xsNe59G`Z`ftw0WlOa5t4<(`Za17_MX!y
z*Rb`$0gn@E`r!xp+aIcc7Dwd3fn6{HZT2W;Na1O4<6gM3XO^g#)PTkF^6CV8{0<6K
zO4G0qYdZMD#6(z|;Dg2p#})zs3q7eAp?+B`+D*0_XG(!!0S|&gg`hjZ8)lkGJqW%|
zk|QKsA`kmzv%>}#3$v?*XcU+=E`}D273Ub@K0&seI#rFN4c~~cB#PsRkFa-k
z+XLdg@;+FyhyMMexLAEC&-g9)E2Kq`Z%5c4a6Y{u9f#)_-`Z=1!vo+R^-@B)kCj7z
z0!)lI!DoW9P;Utz`_=cgR;OXIJ0pMdAC4$tx1O%>xueDh_9eafKJi|gt(~nc34X?L
z-@;)BQc`X(N}zRwOrxDmkd7#cH2r&MrOpLIDXt;Jw^S@ne!0eg-u>(X2Sh!cm4WYr)jLlOIV0;sWU>+GFC)_;_QpPy$h&vS9h@DQA$h0|>6N{5Q*w|p2f$2@{Ax@z{1DB?Y
z_5~JxJXTbBIDz5ReDmVPZ>wYDIK>S={qSKHI1V;W$UU{+zdwWb*gB^j$LGL1+_gL(
zqf9!x171phJ(x6XBQw_4QR0`du(9Br3u7tV`5>M3B_|
zUqTS)U@h!R)+8d-&!;zI^nk-)QHMVq+Zq!?hkdK;aw8)maE_b58XwLZ181vp;sjbY
zC`S;Tp#E|yF>!=EjgG#cr{@B`IxK8E>L^SR5)zoyD*bB=JP6S9HY4NKMA3rc0HB3X
zm>PWhrJ2}n9UZY{=D@5JzNB%3y)>9hLQU?*L^4hT61_`gqJfUi5ik}!Tva-G^vV@ABpJPJq&!Q11#aSJ+RiVpBoH%4V9
zCKz270Z~Dz3r$ms#qb|JCLzHJHSNh0OiLo*bkcqT+rwUnXGIxW<YzRsi{fXB2duV(woWch)KfQm$RYoo|)D~Wgf{u>N2*nn`W18&&5_{_n<1+zM;7}!j?ckZmT6azfL
z31eY!4FrUSTvAP9&Zo_nCHxm=nqv&Y&@>v?W0zmM3j$l*%hv_2@U>s#JFmV2Ul<<=}e|j`DKCD
z^bs~3`w_19WIq!Pc$kejUohu5A2Q?%W+03v2skhZU2UronbM=YJgLd@inBKW4o_uo
zyL|;Vh3tZcTGYah{FMdqHmA=pl%F~S^a6~~)M`3pN@7C&OG-8>;!g5Uzxjlt9cR;#
zB8o6claj(#vgR^>f2XpD2vXPnpspnAiBC-%!K?uvJ4`NKT#Q|tONlgvyaVOcYIm#a
z(=Iyb`G~vFELv2V#TSXjrmelo)x_T
z;K+Pc6cE?Gk(LSLv<;`~TD3K%mJ~(zwcNS=%q*SkRUG+@B%CvH_g&L6E1NH$3PQt1
zt!v#g>;We3qMDPLr#_trnJHhZM?c7)NcZ{Rks;j&{B!BpZ_7V<)@_->i2@b;!j1nz
zj@xQW81sLnb%cRTQGK%Hd7Y<G`y|ZCRaYrIneL#nay6
zMB86_kH}L3Yq!F$cN&1^FLLLa0eF>evPN=TbnDh%3XhtG&4DCMIB($`0cm+iuQ|B
zw<8J#G74Bb_VQ9#wQ_Pa2zu@Ov7M&x(W6IiyXS^9`{?!-u=qfN3x}$V-8yBf$Mtru
z>yHl}S(UgGu>Svzt6e;Iz5W4{bI2VA4;r*kJD%)QI!U4htP{(Fg3nY#Q-)(%t8+W;
z`=u~?^=@Ko=)zyPA
Gp407zX-8-~L9v=Ye#EJcv>RpvMI+#h6PmTEEnb#2J>fq0
zgLhw7hV2cWwpU;zLASgbYN&_<#8(
zDC}@+h-!8H5%
zJEM~zbq4(g@bvxevE?4)PH?WAy!`vxTC~xK#J~iAgor}TtD=7a)Z$sbfdn5jI8lF5
zg?rTYziI-6qZ#1M5Z%H^6*p1ID7gZE14Ux?S)i3;<|LKEUGb)AM1pgBFtoeBOEU*b
z?WLb0Ln=HR-~*=thCQU#`S*t2>)YQqn-g&Ol&l?1DkC-@*l$JvG{hvXnoE~HVty2>
zn}+gBU^7>4&h9xvwzpu(*z^TLZv@L)}BlQL`E}zd>n;W;V|eJM>^i;pQH~9
z*2gIyG4x|6=uT&}c;Tit5%U@fD!iH#XUyot5YoW4gY;TD_A#nI*!ruhbGo2JcuQC}
zkf4L`G5da^KbPA39xwH~4CluudpCFGzCf*sM8@-1$5xPN8l=UGbJyzqCO;q5LH`N2
z`Gh?nkm!7N;v@d#-|d5p5+M*dvV`s(*Kv60s4KwxoJ$xB2w&4z=|n9-l4dRjS`6$JZykvRnd*OUCghs|>!
z{JyWRp~|5yd{}yhhK&skjb2pGkU0P$f|P5n`aQ2L=CM>;Snvp{2f@DmkRDA}U1dzyAGJE06bzt3pk_
z2zjTxG2!m_oJ^IV2X04c0&Yc-(ARIJKcCmNNc0(dld7Q?xr*6*7hrF))-Z5P^J%+6
zE?}l-9l90VOE%`6ZnsONLeW*K11VjbL1y!5cWA06;io{JV0Q|78YrXsn=bdJ#rY~XEITRPqMYd1S_&(%RfZ2HsS
z8Y=fq+aJ-|{lfPN4VS&ZEfBhu7F~LKZ`oez6l2r1_kU%q3X~+)`
zCsG@W9Cm$vGC24tNUHOxdqIJl+JuQbJRGyqRRYpnv(Om<>&ZzB49izJPbWmpdOp56D=RBZpP1vy
zU9m5K)$ag_)6*kGkrAJ&!|;wD%cENI#AFh0NCnDwqSrnn4mfIlfuW9`gPv)PkZbq%
zPU|iBKYmwP$alOS48y~Hiz(FD#F2tWxtPv1fH{?qn)9*?AE4BjHp4-Mq-3rG10;>V@Sy#n$k{WX>Pc_?|TWNh#Gh
zr_C1GVD2pp?fN?uL~oSk7PT0GaUP51mgSzbH%S;#+np)-!
zsDHG-(M6cdYX6QlQtRNu^~8ZyQCW%626r8oLMg2GMhRs9FjrG`b+>?U8{F0e=%MK%
zsfn`^!^*oo939Hi7qhZfp0%>DUtu=(SlhR6d-v@_lP7-2H2vmH%NbRKSpw$j#o&$N
z-Jv^F9zGl;8DbdUnWrtgb5{6b^*m@s0fhX69nlpVMS(|5X`cr)C?g{S7or)pA^p`h
z4-W{Yl`CI?QM{rDXy%w@vIxsJ*N_j%%l*(oi%5dh*RKTVmUbTBBj`C8?<%`ryQ7va
zrOnq@p$ob*$z_+1cP_$v8*cpJ;j)sFkWwJq^VClExv^qDPOeT)ot=utfLp8t~$z$Gx43!WdTFDTrbK7OQYWsZQHe(a2MyeeeG%(mx$
zD5Oh@))bP;Lc2eXBn5r_qtVf%Mj^>ST%Y?pr+k0zvm5INhMMpefe4Zxtn~os#fRS~
z_k$&n!tM;6qk|aT?k1s(LS6auIgKq$5uYmzS?77=b`Z1iB+P2pMEEA9q
zC6;4ojEIdLHhA#73)Z>MzCky)-iH%I?+IC?Cbwty8t|s*BAlQMTJGuWJpjn()T4J+
zm7%ouZZ1RgY-%caNaQ3?L%<2Vy|F>zm^nln2RDW$bXgdjFxGo}W8{i20ce6Bb)nDP
zp^Y*F&k9~D{zkzEir`tR6R?{xF%VV!DmeFyHPfKtpIaDq;2P04lUdf$L1GPZIq#Qj
zj0jN^>|D&qD1RxIJ+1(H^hP@r+C0649S~@?bS%afC@vn8T*rXTzSCk4o5mAvdg}|3
z1Z1dD`z`*wW|ssh^A1g{1$>2-7G_l|tw%*gd{4a?Z9~I9i)-O)iQkPcfoCA4l|Y8j
z(2V6Cdu9^dUwmTF?fmTCOII)Yxp`Jmu-gAe9VPE{zXKkvMDL9+2SQ51xD->E=gyt8
zzLhPp+w?Ya)V4qVh{94DVBDtw9e>2|4YZv
z6S}6sB!?GZgJlZqQ9^LZgL6e;$5BS4*&#FKWokD6WOd7`3>zloKRa8RPG!>08l|P)Y%yUje7v+CwF`PIwNbQ{p
zSk0n;}YVeh6pAVjW)_uw$;z$E3pNhDB
z5q!E-T3}IVXqeuzAFGGUBw#2+Ix`Gva&rxypO6j9x6Bkz7ikW+m@)I4w86N3|Lc!D
zXH7cwI4$k>Z{KR(yy=)XV!+q7JN}tJ&I~{b!15
zb8~Oe5{3$_u5M0~OC07VlyZI#D0ptW{|IaaJ-rmEKZlRf2E_G21Z+0o3
zdm%C!=@5{pmLi{y6KX@E=D^K8(?$6Bbx35>uW#J3eNLX^s<;XE-yu09XQLF=2QTUI
zTS54h)U}d9O5|m(BcL66Rb}NZ(_+N~)rj=bR{HchW72QGts`70ZqPr>Dy*a#!L)$4
zy3Son!r0%+{KC2O)pSJvkDz1l9K!m`h>g0F<5}zYsOv?lgY-L3dJi?%%s;aj~
z@6ZTPTc1gQRZvj>Z{K>2@Qnxpu~B{W-B(w6?dH)7#6RY|8#ujT;C<&mn|J+B6L|w(
zm6(q{~)q~SV;G5DIl_6P0dIrI`|6{6o?mPlro2jia(eubrJXauQaf|z
z#oE)0&RM;CKTA+Qb@%QSP#0(#pS$}v!qardk4N%AEQ-0A8O^tZ_*rx)M3bSup@~I_
zupgh39IJ)8TI)*XLoulox~4(IW!TH~5=jPKR2eob8)V4e|Lc+8a=W@#Q5^)xGMsGr
z(!br)KfO$Y9r%qyNDxeF_G#QGsC{j1N&7<*1=QEiMr=06;8~Oyod39#sIhz%3hL1>
zB()>9Zf;0C5H`x$`HTP0egHmaeo9M=E+P|7ymAEr*n2QR!_99DmBPq??QsS2mdYEx
z(atWgu(GQRE
zX!;UL3!PHv4?x7s2}?t77_|BC>GTXBZk()(u7HnhAc}n!BP3uiUh;r_AHV&8Vr0=l
zL&MK3hb=J{`(~eIY+;7`u{`}%lJ#YzJ}6CQYz%}{O0i`=1zpxWmcvJnDz~hn|E{~a
zBFA0OTu7F-qon!$cick(!Qg{%tvn`DdxLTvS$_l>iP}CY$~X|8F74KaG&@!UIj1IM
z4HU8+c`$mu!Q>~g3u2~5O=p;F?#mO${OCB)HLG&f;>DCIm)$Bo>^s)7(Q#hluh-*&
zrvW6=g-BK+Z?{y<=Iq)*sAMNbD-bKwgzisc=R(Q~i9bzEbLH-%I+``>bY2zQ6m75|
zQ#H8V>rNUZ-PmK?Q+|0l2lC$w3@iA_`vG{fYl$gSK*<#+bI24g`c+iuJ#tC(QOvh*
zCyBVm2lwy4^;~vAt@ES-L@8MO=hR6=ov^lJ-MYZo
z0Ny3~iqUb{{}=957%BN9n|QbD*ZaeQp0j$it#)5u2}IbYPqUHhA;;TJUO@R6=m&bu
z!>Uv1v1DKnKCR#lJ_k1+4IkeqLH!
z#3iw}Cqoid5({eP&FdRpP*hwDhDRwvO4ozL@o6wzOg_c(1&&VG2o+TI2o!@y_D+Ab
z@T?X0YT?>x_=rvy3#$RrZ`&j;(}_lzftyRhxJSE}Zhk?f$J)hVj7Zkk_crq+0&yfU
ze|~253mQlsW~M#n=`A>N2zkFNKP<}qB1(I^q?<3?9DC2|QE*yc=8+^(dI@OeBcb74
z+vBf~wM^*AX+Z{-SdP}7;2v{sDuNGSqd4giw&+B&>OhF`t`z+5Qo7=tb-WtzFHRzy
zZu>$)c!M0N4d2Me$ae}%EiWtca-YP~hcMH4V>uLm-haFFs8VebkMRrd+>ANxQHs=mTQ$n|n6jq~_BIkqlGyN6kA=Lw^8np_U9r7Kry4S6%Vc_OopwI-}p3>PdKgc-Bf!BC5i1T?|R@D7c6LPFm|rdjq^p_MrSvMO06##{wjhL){x
z?L%C8p`oAn#H5Cpk{c+
zTO*Ly=d?aspb&l=__x$y3>KQ@h!?hR|4k|EO~i#nwnHIM>R-!H-NND&l(tL+>1Ye2};fkd!1Bf}1z53SyExGl-QdE7O~KX5zl4
zr=@Z7drkxW0tP{7h9oFRZHlVS=+Q33vLQutP2&RFEMG3{I1&tup`onRnc?->arf>!
zw%UT5i~T*{eRmrhL-j+;!gIN&2d}IL(Wh|!rlCQ$7VbL~DX*MUDNEQbTU$LN{Hod1
zIoXhmTI{O3vRB2I*k^i%7Oq7eMEOUq#)2akr$P`nRw5SMzfZ_KzYhr$Wp*BFp14#X
zIvNltBJ4L%jn6RR78fVqz73s<&6pH-iiqz|%n4{Y;-*Ke_`n2+e!(sY&lRg28B=>=K@jjo6>^1-nrR4xW!XhGUUGQg
zxn$^;lib7MAs%=Qbc7ejYdVteM6mm+43Bjz94NiGP+^{KPwyc3g1qqsV{9BTEPI;&
z%<#U>&`+M${l&_
zc;!;mrLWfziANr^qSQAa`+uveIaA=!DCfKRxg>gV_RGjXLHC%^;qR4(4qcE?3TFYv
zDx~QuN)dd*5~$s~yJ(Ube}PBXRZ&t_hO(zE-2Yde+hjg^^3Ev9FaYF!(feIWP5sK9
z#<3drdT99FyGG!vaLn`2s}QjxVDj@7KGW_KBFJ?oiv#BfWgy?Fg~bR&FsuM@@AzDW
z%oYjbkt5e#%g6Q!4lMY<>msR?rmS!+$wvM^U%l-~+!3}dmDZ7y>)TNkg7}L{Ny&tV
zg(ttflxLlTiN_8Ghu~|AaDg?!eQ}LAQT`3LF;6~hR^Rgq!5j803H(zifxwrndLzcK
zllz-qMQ7~Tb0zD!b{<@^ty^2}-o1;}1%KHe|Jmwl{op_M?k!MIpgYc;J#srnp-Nev
zE_)rK2(w>Ukdfxg>bvw;GeC=o;q2L0anCsww6ABk={pVg^jA^C^f3}yk39xDAsnOd
zk7Vu?z+FpKr_zaH+rQN>%c_ivh+uaJY{_HiuYh(&!=Id&
zV6X?x;L{x_L9&Suk{P6iZjO79j4MKILhqmYec=UbMCK7Gy$&N;z;1B4Xft~bI$#7D
znl_sDQuh(t3BjKB1XDPD!C$aq2P=({@RY_y65T=0Kldv27=bKqfjmG5HSW
z4IyF7H%RRPx5CA*v3)yr-aYxuc+h?ugm&NFHE(x~8#Ag`+jHZceQZPFBZ5|bzfABr
zQZ#xNvUpo5Jscn~L+I`I7S}SxUR!iwm9X43ZfdWnfSO(0#XQhfS{C!?w={14u!9Ii
zPs~-Qc3@UvN}fE?l#^thgUT5{{&CO7&Oq<2bP=>IQO!?;0Fb@n$NI((K?&!)DF#v>
z@EP~s%GahoI{l-II>`iF?N!x#cz8M-CCJ;#q<=ye(dMB17~$6nIC}Ev(Hpd)RH&hG
zrA<5d=YTSD$AQLSXP!2kj*Vp_5Y!VQbWq{VSOvR^h`y0@L(p7WCs8#-@|Dq$9@VnZ
zP-Dry=0W1=UJH!cStASvZ7iohw_~=4i~wNplWU
zYJLHQ0*D@k;PO&Vp32YC;I?Yv7>YrRgWri6%hBVPyZ+(iBq=c)xDDalt-5(~=TX@w
zmhaoM2lN#h2wjB`;(Z!6v258!=78`5z}7i(_t)#+!KF!Xm;c>o2I-FJ!aj*>gdl=%
zTg6&hHIa%vRupmLQBtiDddq87lKT15r_;DC9*M4aDKjCU&z!ZMzu0s+i=yP?)c5Mnw|)i_Cdjr%8{s#^Ou`uNU4J&38kMw#)g2Q
zVKeFXc-e@*$;Mz)Ri*nPFdj)slFb=W9yLn!;wm1WZ1xQ
zL#Z7yg=#0c8FvJW5^j=c8)I?&#EFkFF|d{RQy{1j2M<-jX+l1jzRI$(gCqyh*+VH~
z_d!K{iMVflWohCGs#Tc&*-BmaGO4-}D0M`(WI34$oxbL)e`EqpUQj(IAva+PO
zaDt(p{O-$IM6$Z&FZprd};AVu^JN<1KYe^7=RFBTVajF`F#G;q+(Wdc%
zM*?xErdqEDu%O4-hP!=!VOm2PBIm`)Cf+Akn^G0UV8{rY8AWRa?*~ZTU2uo>^5@
z+OF@*K@?C$MMBsJuHdtkZEdZ=A0)8CB@*D_kxJJTv^ru;)afIGPLE!-Y0EdNhl`x(
z-~}%`(&lESIe^Xo3ESp#aXls(sEt#PvDX>@e!}8n!l}2V)kEXtGHV5ow%W%yShpvC
z{@OI4RU`@!%{MV#I_5~Bor6w6eYvr?*!T3)3YjB1G(Ii)=`&<^?x-L8%_M93
z!Vh{H80<8eTe4^N*uc;8R1b&j_8;1Pqsh5+$7bg~K{P6_8t<3p@@
zgqcZT;89kc?*93rM@l=OaT*{b>+XIIT9NZKn^PQpYFp*b?uQTFb+@Goo_1L^LEqbX
z2G*V3XPFztI)@`xfyY{X=unfQl~9>wi88c#re<^zhP^RD2CDJ$!Vu)^xf1Wx2R7?zEVQgl&v)o+3y+Azmj>nOAsgL`+T
z!B@}$P!lA%zxHH&${i?@%p3u5ilctv&hclmuHOFjEMwa&_t(n5>Jzv8n9w@?#T~z_
zS519(wjnCv^cOsHws+ko`?K$XhAG9a)4$?Cnyoft+%&JxY8}hQ9%FVqwYqx*eS%$*
zl5qs{r(0#FkUZ|!kd(d(ze|QaPfVTX52DcBZ=PuE4+!C@JF@)NuB`@D0Ri1tyc;o%e5f$5Q4xm571Vh`?a>HLp4>)u
z4=7a+hy`BOw7LznlO^2%$dn*}Eob;Jw&wQ^Ru!TnnH^=V6p%UCcJ)a${^eMj;v)d1G+0UK(s#*H9{{)TH7x(L~
z%6N8j+Mu}uHz4C;?`%Q)%qK|qLvjChl<}9f2B=(vlarH+YQ+1ZIq^g+j7+SW46IQc
zqzs@~-rg-|U))+f#1#yii1n1l+X)Upfg4BVR#wu2@
zNG~t6By?9O6#0`BTb!_*qa=V$49iC-!(t;BmxR2k&n+$3592nTvq@xyCTG!b7L8z*
z5f9YW(<4k5V?=lX`&cPw$Z#+;GRh*nds0{!!}e<|Wi4KkL0GETvB7%dKFdgT>FAg;
z3S7eaXuKdu(0g)(V2?61v`~}7D9d+7;NH|Uqvx+~CSrZfwOs*U(`jL&enK_%oBF;2^
zq@+9m1aP(Bs2BDA4}7Mv6JiD_FvY!i?>xw?5HOw(`m?p57mAJCOhy@-#8P
z=9eyM>BYP*FJH!fv2{gC>X9eN-bJfIo`R4z*4D<%!;)16g;e~j0st%!OPD;#Zstis
z*msiw?bPF&q(Q(;qdZTs+DWxPQ^zUA$0|L!iBKJp1BpBf_lx56Sr
z+6%Tb!p9Rha>FTK3KDU*w#cY@Y$NfCy*s(4ju;roc-s
zwf9`PEB;3MMHbi%Xg9){Y`F&rO)!q2CC)f^PCUGT(UPehz?Kcwq~>5tB_rgl
z6*Ah%?ysl#cOjIffOYuJykvOEA5SrZJK_k6jZAQ=%#Fgc%~gb9
zQogLKatKX55sHYgK$&~Q*)y5J2qtGxAUA?gCm~G2H2y`_&do$U6t2DJlp^qTf_ZAE
zx@T=J29L~I@U=z1>i0G;cW}t2)Jym2YWH`Q>g|%=z^=!HpDIQ<;}S{*Nixl?d?>1H
zjEYP*_ZznW0+BKWvvK8$wsu?hjxhi=1O*As?A0{GJhpa8;Hr)F^_a?pl~gDidMw0_
z{r<4()4GoS9ehWq7;SBb4;~b@rLMX?1raWya6n&Uxzrb|pF$ZV%*TTVw=;K8Dmh8$
z+R;IQU+^)~a!DAs?jkBQ9}GQQb?O%uIih;!kLo=$tE#Htc=(%487LvSbVwvlp2ToY
z*srmM{C~P>(&Rt`QB-ss1VsNtmLHQcq3~C@{@D}e>7@83Nz96g;9G&sL
zu8v_QZX0j^=&@sjfbmaAqQn5odr_BkN`FOd@gBmnzEc~$Heq@b``9a|;u3*jHfIeIM5oB?OE#1GS4%vhA9gsDNI-)Z`S
zopf6ok*cNN8rkRzN~o>vmN}tU>-4P|_C0a~@Qw(_NugXwP<}KDfL^|Np5C(L=M%II
z)D?WEn&W4t_4?BYkAkts@cm4bSM1_tZ<$(+w9!uhY=6MfHxpW&jIU}l{1B_DY$mRnK-IDB<`l5AL
zHf+<_V;n#h)1N$kOynX>EJ2#2&I1$E2{`^>r8trt+7EOryaELEjM20(tVxA7AxS|1
z(f9&A9gsW+*qw!vNu|Enp}7P>JE(NW|pHnKeUBKyN#qoDi0Oa!T&5
z_fzMpFGjnLk0Qe!TP>8efk8!`C&LTJZTfuk&3SjNc-qEKL!AG4=`$!k
zGOuc_vojlY|6pA&@Tl*qz7Lxu>P2VHyzXNwGRg|5{8#_?@-fF8h75VLVZ*A52~8?k
zQ1Ne;76s^Sy{p@}WeS?(g3V2Wd!2lzTmPZQ6e7k{6yM*3ctcvZH)br`LQbxtazG*%
zPClzSb3u{~fH2`k$t)s@FX7upn(bNoB-c@w#PH;bo`v!5@gJsBbrVG=1S>VG-3`qQV)
z>92;#^cA6e@r0?N>v_=BtYZGbSldfu>k!eqrlv#Z$5FEgIU*bZ#UIv7MnKlhj6Jq=
z?4}JH(6u7EIHy2=g4`QOeajS$hP$PoOpcxsuNQeKqvA@l1F?^_+*wfzz-s0VTRBIK
zm&4Ux}l@B
zQ-ZlR*YUyH>kG=SVrSweWy1tl7>3j}?O_cvpWRhvJq|NgGY;%4$`~9XWCzhTZLcpQ
zAs&}`VmW_LOr$Ff%1yX1U#nKd0d2;`JzhRtp8K0v4;u+22BF7sey9
zT@;X4#eLg_o>f>m(*I{Kl1`nsF&f~<;I0ca(i%IKd?y%enn45(MryoGR$!C4Ck*i-
z7N@bL02eawJv~qlbWuiGJ7k(=+O`Ep6ObvHgyOuD{$8?q}RK)|I;zu4b9z$
zkm)1IaeZ4XI@{gGswn+slK5vMihv?jPd01CJ(_o%Kk
z4557JT3uz42soK|Res-W**W5Zj~0Ablk%KGfU@!jERe1C&UtD!HN-A`KDWVSp#A#LKok>CTg
zzgY1giM5r+O`ZCp^wW}~8^uaX%Xf8JH%q9;D-dRrhS_Q7#CHw3&Z)7}w%4!j{!ziA
z((K!}=?n!&JUtky=C9}zpRLzt$KGZ03op@ASZJ=y@Tehrj>7+$z%7ue()BlRdD52*
zpX#A8X;NvW^EyLF^WBKm?J^tM0FrS<62!>fVOr0bxO+4-YqgSGuvs&DSlr
z|Hk4hcBcxfMNXc?qDbzvPwW9EwB}}aR=yuSS}o(!TO=(j%VS6Tp0!98p#y6ve_^s6
zdDG3`e-7t-fj?M
zfi75EX_WDI;)uqKsk?VEgX`gxqBa@F7Jo!=XF&myoKO=zhS3i#JRmwJ|C&Dh{9C0X
z?-poA@yw8UGY``?m6WVP9#}#s5(oSRYe(4I{=%7WTyBx12$}?sPCv!)T3?UVk_X{|
zC#fj|(`*0Po9jp-NVQAZ{cP!v{Q?TuPC`Ge!Eu^lPxFpVT=z9g9rRV`K}n=V1%CLj
z;IzX;DJBvWVSoKKTgCu@8vmQU{htY=1K&@)V$(oX|Ngc+&T1@+Eq%qs#@G40!moYu
z)bxv%Y2FWi^c7iYd8rKOymM>r-8I{%B-Fbe-smP0Y5#k#!OL(r8Qi^l_vXWV5u7tT
z8xoF9ID0F4iHk?9S}rLkF*0rAMT?lAY9U%LbKc=-GVdbJC>khA{id$|-u&oUE3M`u
zHpC8Qr;UcRoLx0{GsigQ_~hUJuKHWx%)Gq)|1dPG)oV}YsPEBaX-37IJh^Mr#s)C5
zC&{-AIq`iDM)-7h875h|`#hEPEH+Y)jCe3Bq4cs%qJKwK-M|>JfSTUuq(fQ(bAMHuc^LR?YhMIu%fU3{tA{K_lgqOG^Iyyyc&7VRK{t6h!7S
z^ER99zZ*QQx~8|BWc7jygLyF~?|ONi{t%MfcGXt%o3+2Nly8kP!ma&NccUz{N)kXDWU5*oL+aBW=0i^09ui5m?Qg#+%a@~#<_W+xb7
z5p|B5=#TB-^1*%5Mh8xg#QV^AV?!xtFN6t>8Z^Q6qSq-z)FXb|ob%Xh{y>e5R+Jtz%ko&yR@%@07>Gxab8JWv^+x0&>
z{qXdU{%+Q=e{G-MzOAT8XSFSp0ygF9p`j+~oWcM6<2?BkKorP8M}0oZ7N{ohTDQs5
z`HV`Vj9LW8U|4uU?>>6_O&aUV*QC1qbjYzdS{Q0sp)h!!eCS87+aa^A##%;KHmA(I
zkLgHD%RsJG;_jsXj7hW#XieS>``*rTNj^QOL(|)wZnoaSzYBm%PUlU#B-1`*++Y#W
zj3PpG`phYCC)ND3;S>BF9q*PdG07OCmXSkY&^{Pa@P|m`+cj*UrjH8-TnFoCw{~nY
zGCwd{L1g7V{vZF&z5hCm|6@Y2-?rM?i!QtDb}A|!?I*rnWY}x(Y>cixoDc=+W7v8@
zH-;*jWq<0COn$Fl3yvCiNRXoQ(q9o+xTo1M9D=kW;yu_ZMkx@c&lODp9XQ<=ijCCdn4S63`9Cr4BQ4W5Z(eHNy%TPe>wc^|XW>D?9N}Y8
zU9)$WU{mHI%9n{58E5p?J@pt?owt6cL8s(>zPeQJ&f@IHfonRd3!}uvChHwpn4oZV
zwHm0Ul+=NtWe0_S!%~;mnctR6*84Uj(J0QeYyEnY_;SIlJ#*&g>bjf-2?`S8`pR{d
zCW+;UMdt`t&OBf@co^Q8}WYI_!Wn?BCCjU!y
zXWd|(r${6dvMn|{BpD~2yIzc$#xw6I+#^gm^u~p^>*jhUgsN$W3a^s9TNK7oL7z;cv
z8wBdEIPklVcVynZr~68Gssp2e0H{Yw(hkX$J$ojew}pPjhl2|OmG1;Awe!^#ISGI;
z^pP7%FJcWQwg@eB=4OBcKx-Bd;RfzmJcBp^3p~;KYQ3))&q=c3%r%AQ
zy3`N#8@4`9#M8AROm?8Btdz0WU-Rj2039^IOBOF?Rzxx{6GDD3UC$q>FU&8{Y;cuV
z*S{f$S}&&Y!-s8`#ZV%`q0lK^>J{+b`MPf!XqLSq7*k0JXtmOy{M}y)*Dp#8_9zWh
zGt-Ig#%0gCCIf?jycmn?QB8TyZRBbU4BOu`@-4iG^g02)_q{<;7wwLCXn#jVWO!74U
z{nr_BZtd6hrdL2DBm;Zw;-N%tBs924yjcjqos+rUXj(1wvhqG)5NwNfNj(dh)K!@9
z;4(a?t?5voafEffY#Bj+SyaRQ!(A}sqD>-Q2|Pczk+SrD@#bU3rjXAj^Q%J%20Ay2
zaVxKiF<(wXOhiDaM|0q>dXEgI&sDxHbHnmIDjas=0eRtelYo=>t5=`6r=MmYBCJa|
z5oD&Br^B?ssxiJ}dknWwTA>r18(?M%MjfEGpV(o09gH=lb*NMm-r8#+S?RslN^`=F
ztZXQ{?yjcb;NQHx#o}!Gu2W~%A)gFeKndUVwK*;u_Is#Q06Vu3ydqQdvKx4`-@Q6d
z7IBQ5IFVqTULqD7vGHZGwe|H~-(1&HmDV1$OGMFU
zW&OOI5GN%q&F|q55!gEO3s}Gy?ok-=g^C8tNlFTv2I(_FDcI@@v4r!Ngf+gQlG0?t
zR@uxWlB6*9>xcf19uCD;imhZ2iziWW*d_hDWs0E16MN}c{iOGHkrmgRBT)dnt~yK~
ztSxt^RsLmsZ5z}a`|Xhbt|T3aG&gxk&E=iKjyg`Hb24{hgV`s{Mia&fU@T-3z>f>qL?8X?~rte^&5Q7ROdp2>WnLN?N!l
zQ7v#Y-9hzC8lr-SNXP}J%ewl|G-cnPFjpxe0i|ESL7f6-j)_PNZ*aU&n|K
zU}kFC%t(cfhkg$^7M>T@l`y*+4yXKlYHw{-o#wgUOyiy|siK4(m(vHFwK6+vMRdh4
zUIpPQ(_*{u_`lzBii_zl=P&~G7L$A+5iah-V^{U3>J4^<1D<}j}
z9LU2UG%%98oc22&8M)Zj7LJ^)k%Of2N1yg(8zEjOZU{P_<*WT_>e7`D6wn?h9xW_7
zJ12z1gE5NFE?RyQ&^QHS0v0SSWsV(ixNUpGD`mgQJz6ovqZ|7Koq6`mI?(7{v#tMd
z&?O~Z_25A=B4M(soChwxV&@C9cVv$iI{wZfKb#{Wb_JA=OU4KU_?~ZS2S^K
zdZ8@w%Jim9`+*dkNdkW!k}}%tx7}UoCF`l&qa}?<OO|7kI**4s{ScmZ&5hpZZxzncWC0c
z%kW6Bu00LT;rHL+eLBvD499MQHIF;$)9$YCE^HPOS~xUu0vtyy_md|=ayGCEFa-(z
zA$i_(ERn?_0x`lp=XD!6la(%PQy^%Q9X)`0XwOPfg(KKzgatGawwy1DoyLXQPQDCJ
zQd4CXBruN5F?%#w?ET^+jy_hjiZp#fSlqRWJw
z0{mh6M|D962T&k@po`0B`Cme|j5r6NkDp(schx6`y?iGa@*Ztgz|x7kh|HL%Y=OKK
zQ3YjwFA6xVrDFvb==vL|`&j|8p;R+_l|Q+a$Q(sR$aYvb0bVEM1TE~r>roR@PH_uE
zIAL|7C_H-x89Hvr(@#fl^JWfT(
zR}h~Wol0|BH0X_j^yY7Hb*&dt$OvnWGo3Ro_iY!p34eYrO+{{yk=~HBuEmDZ)>Bxo
zx9TsY1mG(+L=lU`rYTre(|B~aeBVV3p>qY+mYqOoSV`kl(?j1vtHL@nwAQDK(`~gW
zWe%8oM553Yq~;4jMFdh3ZzK36%FDmK3zj_`-Oo6J$5uO5L}Ifi_g17vRBX#Q_(ErWl@Y$5bJ1*)OG@W)n<|2X#r%SN
z#;_q@AyPSqG%%TheBYLzZ+3FR>Ed~s{qwf4=$)evDTVzMB&8Lre3Cd)p?jDnP^{q1
zWn*)6!rS#)abi6O;wb8j&X}og>EY|vdcEDY=<~P!CNaOjpO!B_UHJIh=~=5kZg9}q
zJ1Ah{qJ{uHW8{c^f)}q|4bw7V8p1{=eA5gJb)LY<&0uguiA7(wfvW($_38M$Mye*c
z`Z%0A)*IAH96QKk!ar<;we>B>{hHGFaz_@Bt1RUg9bhN&)4GM4oAivTzdQR
zrAq+kAme63#e_ZNER<-fuO|RHaLp|)0x5rWp6cOrA#@{98#87MBgWwScUIx483?y0
zP|Uf%Q!EoUx(g{WcQ}&3O%X@E>N|y{YO2c0&&$jA{;~rhFRf!%B#JIPd=Y1W^)j^g
z2tURi6V4bN+iQ2}I||o=?&H`l_)xj=f=wb@wZVo2?*veNS=pYl-X3`$T4;{@1amJM
zADBnir7+qO>Q24Of@z5uGj`wowJedo4M`FjXRH)F%RFG@kPsaFYwa7Hm>uy&Y&OP7bmY)d#FJMdrNhp!-^G+%U3Ua=P|k>y7}Fi
z#ba@QeiRE3U0a=FbaZ8JT$#g$LDvF~>S&KFcGQh{462M*1fdM3FrD%&*1q0INA&J_
z{AB%Xy1ecqSCD|6U#)YMQ%g$N=$KH6*CM^&;(F>3kz%s-V{;}@_rKcP~Sp9k?kj+4aN=_
zj9iYwPqZOHIRYgPn5D1JCP9D!)Nx<{FfraTcPhc)^74cyiqzN(zqJjjp0CaM3IU`Hu6B2k{(b9&Uc1CO
zA~T3uhu>VXdGm)WD_#@Q>(c++Yw59L|6(K7@z}>F|BQ|MU1g~!St&VF(cjkXf^KT1nUOSy-n2%$rBT>0a_>4cEAcgb|xKORmR}XXdN_C
zZtjema+1_~`7^U{l86GbVwFSz`A*}Rj(Yg*%+$cj#YW7Qv1S~wHdky(3fL!!(2ipg
zvM>WzfByaZA`x9JQ}W%rMwr|C1dq|u!XHa%c1n3(P%tl{l!XD9NUdqKAT0URXUm}n
ze=R<6Xa9X`*tFEpkZ6+_d94fO0i?2;+{HzwhZD+!TGnj`N)on;7S+7}`W+OWDZ5!s
zowPT`^7H34$kT~0LD=;r(~$A*K>*{>;O`@+Pk&XqdB)LGGJ^cVLgzr(BDNkGsp;}v
zRR?Yz=R_={#UEf}~55drdI0SHLCXe(1EGG+SU^uqu@K1B!u~E*!%X5)>
zWK}-(-o~|^Tm1aPw9N({Q
znxW7$p3T3HU~hrB>v8R>SuG}>Yc*b#)}MK8U*`HPa@0ILTfxEK=FeXfXH$E`?1JXy
zGS5yAE6sh&ZIZK7#79qwJ!qtM%b;TO_Rm7d@8@-6N1Z+6bs*={tB{}|J+ZsGhEU8j
z${};ocWu{l?R?ut82k0>rPFs3;D7aNd&0iGXRB8Az_bbyJx|^kqC=3&qB1QxXS8~C
z>%or6h{=^85fpz_)h5JFejM^RuI$3({SvB?0h2Ye)98b2Ax`TyI*%_>-qB^Ll0iPWB158
zAz+@uk4ZCg}_I7i#!l&k;cCz`ecq^26q`EZw0)yvAH>>|cJ>151vMiAt&O-h0W^
zKF&rAT%vc4X^^H^{24Fh;M^^(A5#=Y+x`2vzhlaN<%0=>W@i*K%_ALhcKP{w?6FmA
z1_vx2>}YLmV{_B{t256;P>@GMU>pw<|J<0Dw%I1}kE(k{V+VCc-|l3L+G%sOdfvQk
zTW+n)d)2>YpSVbL%?B9nD=0ES#ie#aCEa!ApMA9$f&ix~|CZ%`1Zf;1Qe!KVg|K6a
zX_79Qsb^>ZyNo|z!xZ)Nr>8y~f50+f`nHR$t#v2sO8p4WRr1+ryGN@6CcL{oG5pff
z{OtC>e|EgqON;Xe@(_vSrqpeiYt=OZL?I!e;J=@JP!vK-DamDGx8@=%pvZJwrP+JltT_SDL_T
z7wA3tP=O(}U0WZGA`DGyGDeY{!!Y{BnpTexWFgtAF9r;
zxyTY0m-GugU(xw(>(}SmKAwW9;m4JBNiKl+=4(wFQHu>hv9t#y_pvBKSLz_^8~NfhYr2(S)aQ*uJjEt_he|p#ai?GjiW5e!WTT2s@|9-tadjD+WSw4%ehS6BF
zx#F+#`gia4pK4b`@`Hr}P4<1rL-#H>7@aHEgB*X0i&!dXzvbjaA{Y*sx?S0;#vZ$;`v_m$6qaZszogL{
zqZH<#Q^CAOZVyB*y0E&|mGM)=9&QiX-Orm)Bm==0_>0u+mA1B1HoeqOvAvD^Ktz&`FHY5n)c%;&3C!yg{KC!D
zecp}&PKEUlhqNphBzpcNq^xsO(;d)hO|;7V%I%0DTd*5b9kJ9F7cRh`7K;C
z>~ZlG5mNu_*Xy@M59A5E;1-KCJ7`KaIs6$AF+mTg3lqx;3%s3`C4gEB2#v*?@xl$w
zGNCf@@xel_jW_8X3CYRf+%~gz`!a_S$-vbh5s9n7jMMn>;}ll)VHS=ULuM11bNVX(
z<;3s-2zCL>SORuf^MkoDI2TFh&dTkBMPw3DrT}g(mAj(2+g`sE0e$;X9?8lj#Co6}
zGaeQ&U82=zy$6!~A<|^-V_UJrOTFWNP9^}O8Zf?un`~6
za6Z2E?&nOPFAkusUboS63h(vX71_+
zUjEs$Bh!cO^LP~;C$iAdke?6AVx;B;ST=B8x6tqqr7aR+&CaW$;9^-;l^LxRh7-du
zX~M(b&n-1?s2|(n#W1iaJiK-lP*veUc*sJ<`1ty=vX$DwH*?+^Y07sPKKvFoTj*j3
z8AoUsRGcxc6gwvA$dbiUu$3!21v2M2UC|Ywx8(Uc0Kpu_u(H{@eetbmjpyuI(CMDTE{$N}44h6^bO4R6>*~
znJP(=gp4H`5J?h}qzp+rNl21Zri3JfBqWuDqEgCG=l6C#f1R_xy|r5FUC(ph*Z3QF
zO~X-;g3d)GP(Qrp(zX=9Qc*t
z3N0+|VdUf!ox5NGO}F=Zp=m{k{ZImdJh6ulI2}2wEi87y!zzbEhgtia=>>oUxC_#_
zovPrG2$$!)G8L(^BdV$fKOnEk??gWj~^TX_2mQzwek5IoQ-~5AvcZi8P$DxHO_N}ttMcIz!Z{C33`!IqDJTxfvVUL8RySb?wOJr$_~N#;xfY5h5m-OuUCkOeS8k+Iwlmk=b5qPi0h#
z7?aBE+N^8?{TIL7OeI6}p1WTq{wSlzHf5ye%%~FlLI)s<1rVAOr=_WBFt&gj
zn^yBtf!bMXFii3NL0lBPtd$K`BvSqJ%6j8d%b`Q|^s$!qf=DJN1oR{3=uzyc@JqTv
zFvsO3I@Dw=y6hrkyz5NBgOIr8-_cH-<*!Rz
z{VP|lbeb1LL#U39;LVf`;c0+!Ao+toSfZn$pdqLcLJwRGs7$VPPthr=qN!)y5Bzx+
z4FKsm7!^K|y?fH+pHSa7af=Ss;_muF1{?R6o_OxujU^WeD!O;?9@7Vc_(uHd)?My{o4xj{RgB;YM{=fK
zqrn;^yb@9xqiqD+4=1b8eg%FENxh~#=U=F9j%tK8@7532U36Vf^-hX-EBEId
zn>o+JF1%a+;Z}fo#9zT6@KbyCla+Wr5IjG4lis)E_k!bzn=*||+Rj!)IT>vl()ugS
z^k={2TeNZyPM_)E=|Bj7;{2=)TnO9h(woCgrtkQCTF+Q{Lt~Zuh7GT0=x!gkq%BUjgdIEGs*?M$c{qys9#^)OB`tZaB(g1M)g7sR5Qmc{b@k|F!n+%u^q>0r=g3?
z&>qtVDJbAaM<_-k7^5f4rL~$rPb60?U0ilrSUu`CZ{rC#)R*&s%pJZ}MzPji)wBmr
z+>r<;ja5k_rL=bABJF{QcKqaqSpGyXwkQQ7f`$4s;s6TWEEEpSqoR<=9aYKUEr2E?
zM~raLw&%8_My;C=QXCktYXL}Sr%SiSB$kLC?1pH^9#-{Aq=%CohL+arOX_(TsEH#I
zX^9SFh^4d@v8!Stv@wkQo#ERsp3wA<2}4C)6`D6D$EUP)`0DgUTru}FVPn`dC5B}s
zeyjBl$`u=aaNpnk$FE0;O%gk{U-jhFxK~}OyJ4((Ra`vfw0mRY-!o?nOLS#cG^X){
zhYc>DF)cH1Ux(_vdHbz3tc>ES1=KM7`Sayn0;g$J&9a%a&{)}@r!$i`;*V}BzeP=zE
zLpr?6%T;RUj$<+7)!%j|N%P8#nVnVeZGJfg)#l29cPrldaw`8?q7xZG0fg>j39>02
z9sR3+_LJPz^l7(4QGEms!SUlU7~*1X$|-%9#tOP@SqE}L4O!i>)!fwZsh#x^Q1$n1
z(E;;}kX0`q+*j#-u1;mm7{_lu`-?`(n?9bj;z*fzCy}7a4h^ZQKIEacy?*q5<}
zHdWo}va{K2sj-L6b#78Mz#w*IMZt+#iSN8}&%bk0l-@GDUsEGDH!c9Y>fVMYa^Llu
z;$78#Jg>vV%P(ax4SQ=m-fA7(XXQWf)0k!-ofB6Uf7m@Y=hKR*nZ;(NZxDnA1qXKv
zvA9z?Va^=Czkl6@&+|}mdAa|W7sC^-Of;A;({Dq8Vyj{7x=rb_qDyNQM=fdZaOs$8
zrLCm&X2w~&Gd2=zaNe0?CQQ)L`{CWKdWP+!^}RQ#!sqEWCz8mfF~`s49Mumzm*iS0
ztUeIr-PFl-gnS@jo4ikz4#}lgG|06YoKoJ7gp``PiLukAYwXXr*RH!Rt)lGdZf7ul
zb=(Y}q{-XI9f;B!;%+6UH97{OsIa)_`)B+okE4fB4>>m}>QDVSQhYJlNN_d4v(+@eNdiTzWPPXMVl`j
zqbH3wFfsVz@tq%-Y00(O|47UeDx_i0O@C|Y>-R5Ayty-AS)v03YW2VeqH)nD=Ffkm
zo9ijt);MsTa>LBf`(6B0$LfU~uUhreuKnYm@3||-C}HF?`nNVHZ%g~SUFyAuh~1B*
za#PgR9rK4kS9sS17>Q8T$A%3a->v)d!-w0nU%Bd@3ighNekZ7Z$=jK!3Vu`UMvTfo
z&^tZj%g%@EJFI=g_e@@%I!sCEK1;MYy~L!o`|WdkhHT^l9&sRUqgk)h6-%|Ww#d4t
zOyhT;T>`9>1>DxS?+DqYdBKX0Ql1FmU2Zprs-uZoxh?D5S%=%2Kdu~`G5W*D=AqLx
zzLnQcihn!rQva>JO*1osV{9U*`k^O^?2WQn$
zhFi_khNbpSdg1r+`bJk#mZ3|-hT5W~8~5jRsaPlP?|(iqGP_@(>VH5%d3R8@)z<9E
zkq}G;qMs6?uGMaU210nt@#9xA$`AZ%vFbIpeDb$Idyb1rY|icnoALc-lbiRLrs*I35vbW+l=xXJ(Y5*A25
znI}+7H+|}{O8=*So%Eer#jwyJ_wLVXX;qO7vW-2oLf
z$0q3d{61FqS>XkM#k||wy)SnnJO_wYkac2ptOnDWg<29$-a;L_c9nXdS!-f#F
zsi*%0zVjl+-Ar}KbrBNmcD
z7{5a-6gqyNu^DF`xGo#rv3sXl9`ams`8r_WAL
z+Pd!4MX!yH?vMJ-jr(!cNJ}o?Ym1`MJY%EZGAI296mDOu3N+5iuGJrfq$t4rG!Ukk
zTx-y%QKJy3RMve1o+U37u^ka|-Ma%8=ZD))n^v)|J#}gvS-9@{48!AM&~gnB9Xxhy
z>%VMbjurzzg4s{$GihVeXdH9!spK7S1R*{m>c9>%e~C`&g$o#=d-v>lJiwBEbnb2b
zLk-!bDK#570_yAQK`zfFB!uNk-~GV}PoW!v(5^upBd4A#`P%8+e8)ck6~;6#U_!57
zBi%*u0jImU_guoM`Ne3g06SyxZ%y=G^6Xy8*_;CBtqp?V0LVv^-1Y7R
zs0!jaMql_lNr^zSAH*+}dX7IeCZf>o>$h*(T3Xj%1hB3M;}dn(_)mrIgFKgq1qHwl
zZ_yoq!Y$D`dCc;fgEq;HO$`m1i@rJ~%~Lj9_kpSSUE0dUuadw&;W(=^2=I%1#J3Yt=#Tb@q^^$4Zq*&zT!$J7y5SV
zk?FP3W2f%5AWc*?EcoQHxliH9)ElCvD*N0kPRlL6T>3)fFlj)_`k|h0UK~9>;&sMa
z=Z^Nk6>7;Qam}L->zBTZoJi1?*X51{=meb0L_B}V
zlKp}afbWxyCWHW~(LBeRhXG6x4xy(~lcZsn4Fn9@O9@RELyDQ|k6*tM=X5+e3}gkS
zhSRd|@CSKNc41cNgZP2eqKibr3m7%(d(Iy(E30lK$3A|1wX6K52XW#!Hj2MNT&!GH@+1|WaPk+&lJ7H~1_Z1%n`fNIT2r_;@Y?AtK}pn1It5AU|ij^2OF={@
zSXfEC;~7^%i2KrGmHr#VC}?B^aEh8KprN|DrSs>f4CsqX9x;rc(RCtm)C?6>3jAsN01gXRp=VT
z=m^!rS<1Dv3^#o_Vy23G47L~)V{z8zQ6DED^{Z@XWp{bxdIc#Hn=ZH&y_c*;ybhFW
z`U+wqenSMBU?bLhJBTpC));8!oL0(njmj5f;;fw(w~CNXOGzjnCLRd%Y`poHR8@$u
zan|EVCHVvtb_?t^YPmo&h^tEup5WEfr}u1p19t(@&jB!W*syq$Tdv)Di%=7xqq!C
z2#O`x3sh(*8u*amMv&vI9GJfyv^V!5vkkO>Hv-^3JA0CjPCdJWbdMgy*P%J43)cxM
z4_=;JUxLR0WDkIm%g(M1zR{(vcidtYERcYcC(FrFC9Db^
z2i`I$IyflA@^q$hVIb!Fjj$Pb=T}P$4bV?2)hfjcW6TnZbr%h^N|+OQ
z^eD*Hg+=L)l=8g?p{qn%4P6CceQ?NbzD9&Yz@Cu%cUHYO#!UiB;Elm>2@-sCWH>0i
z`Ds1W`aDrMa*(!rdP;@n^^^#rH8r~M6J!gopPKIUMc4__7u9k6u?JH?f*^q(wtoHP
zmmbg~q~y~L$ak7S%v$d;$tjS%LRT{ChJy!Ly20uL%^;5!UUCkclZ3eoBL&kNqp#>z
zwKqfvJTz!6*Cq#J-;%DSQ;P}AFk}#2WIwoqX&idAi{$$EdHQaW
zv)(b|lswCTWlx`<@me)oWs-KC=VhINDL>;S+MaGVPLX?*M+wHeOBLm@vCSjxB~T_B
zpAnb3t0#pXcgZ6YxKm(kueEFQE7s9|OaD6anGo%1aV9j%ByCX&AB6e021sO9xt*VSKi6fddB~vL#m0`A1Wwn;4F8-N}
zE`@HmtEr!QP|?&zFte-cJu+8l8L=TIf;hMV%F6Y1b%oA)FpG<)MHRfnMMgqm|NeZB
zWNHYQQVS)roezQ`3Y~RPCu_THJr07-CZfH*+^N(UMco1334tuv2iS?`d(Bfyi7Ogf
zlFc$7xLlTr_A@E3SO=^w9HhJ!i7sM6ryAns@Q6fM6IKc(j-|;hjqy`$;fn_%j!=u^
z4_!$`+YeKU)ilZnOL(4J88J6|Y`5b;I9Qwb)q^
zv2{cH``&lS>OU(nIc;vnvMF=t9yU5CA3f~YwGoYK($PEbo+*EOxGy3k^b4^QJNAdy
znEZCXk@y1RsZ=-mEo4e7a%2XE@n0^G2B${x$x)<(@1ZDWysh>?`0caqeUQ@Jc^fer0eom4`9|da
zz(i3OZKs6?9B!St(m|UB(qtwMf^L?*cGs^jO-qDsFDUqWXBcX%`S$jYgkF^>Iy4bJ
zsGl~JGgpsbGM$rTv4fIZt9=SkL3CaY3?(EI5Y$2)EgZ>Xlf92G)P4n+aOesqP~;n}
zBy63-u33V*3iTJo*&-xGJig~sQfAuPP91Uaj^4K&ZOo_w`;~&6$JEHEryv?;3&8*b
z(8-|15ewynSS&Hmg##|>;zbkRpZ{5FOeY)bTEZ;x%(AbSFTNIa$8Cd>
zvEilF3v`tVe4Ky3`<4GLy#Al98!r$!R&jAQp-|MG7$~xQQT|sr>pr1l{2{IT-L}_)Ie>sAkuwGaI0RJD1g(lhe4hv_c3coRgXVjT
z7a;u1ceDvH9QNEz(dgN;n03ekfCeY(m8Xfz9zzUNvRMiqpGhL&Y-`-RzwpUI>q(;d^+jmwwU
zvu|I%JUg|71CRnvDx}fsQCkxiM-p)6ojU?8h-6YBY@mvYe^Lm~)7+#Nn4;9$imkmi
zQr{J5_TR;e?=#lIMwxwV6%+A+LFS_TAbv~pFtx30lw=Unk|FH9yk_&$bTRY&lj_!Q
z;6NVd7jNG(esd{-E*_fZm+o11Z4~qE(4o6Y96NfHG)*d`k)j}+x!`d5$R!|Zz3N($yW~b`*zv7Xav3+C?p_xQm7Bg
zY!;c>*gW<`~yk=eR+dg1vb3W-Fzx~U|A_1ry%cw9OmSnD`4xjt;pz4RE+b2rx)7lDqJ78^e{
zF}?(VBFyunMvi1?)RWR(BJ`wp5pmHXb--_j@a2xP9vj%^%`G%5K*=!(hv$onRI@7i
zdeHI5o~s^#V3XDMY<~1ZSC=u-5dC|bc0S&X!$U)Z(H#j2P%6pIwBX*YfC)xQ6k*Pd
zLIN1sZ2IHIsc*mz1B?7BrFVFiSXKki%T?!@zop?O>L6y=+UIr(PqIui<0Fh;y=!Od}QP_
zl7XC<_@}Mya_Kh^NbsI6B6JW8nWW;#n39CSs~b|k(8gx}OApHf25hg&;Zya=Zz0Cd
zU0=Pnr%^1&!$g4;XQ1$&a@_dTl0L`cF526EUAJ0hQ0yhTB|u%(Rk|NOft$&c6}2tb
zGFm>T{dZbpqY{VVC?{p(23r+{G0GFhxQt7@`Sbb8ILDBQM_2=+ZQiN)pXfjE)8$cR
zz32~1NjbjdeNv&RcF3=glX7FexBaZyK4`&q^X65hiKZq0z7_n)+jj|(S6tkyHF%r<
zOG;vu;Ky$oJo@mT(Mr>?fi
z_ZztReHw+z4043mC#C!wEG>1H>n$P5kpPLMq@Y*#5p$LY+G22HjlkDSv6Y27=}RQ6
z&Ph5uXE;q@F%Lum#C)8cWl9x`(>0Rdk<)3^Kj22*P{)!`7fuQy&l-_ZrkFo9dkZN-m;Ou
z9hj^&_>08A$9ay9#>%Gt?SG-6S?v+>Mz^*P20vE_3x4QI)MWb_NtsTuPQ79DhYuZP
z{*(#GdA$ZbTRKl#R%*jgq0WFZ4A0l%qWiyw{PCahWh4C=UYz@|>7oG0UV-=BF8AE6
zVA9r`6Qo6w>_AddH{%Rgej&Q$9Eo$Te0MtZqMGOwI@Ysi2}Wd@EnoD?^ikwaBT*a^
zBD8wLpWdBD+^B-xRVAE41X{l6z0SYgNKyLi)7z2#ou*_avFsPd5dgI!QlzNgPZcC8M(?F&)jR5>Dk0Wi=`{-jVLE5fw9Qz@y7nPsXka&pFZ1
zXpsNolC}1@=rw!LBQDh3I&m4nDfVA6$G>rcE}VnPgr)m{Vt6aTQxS>6kvC^XOd_83tBp)BtDTzCu(oelCK7LCYq&YeB`__#qq
z>`}p+$Q2>E`%ZTizo*5AuNNo%r)BE?zlQTktai-@oq(~hT?
zm(O(DAbCu`rQyS=)90s`zQ33We7@_rf_%UVUcg16~db345
zW-TXp=E^#?Kq)_}wta$J;$snY77+4Q3TFxcUNyErK?c&r1RUrI5(y@y!
z1&m*Pv|~l%eXnsc5>sTp)#hFAhbpYCt!rD`Od0)<_CdCBN087@S)QUZ)o?>ecGi#A
zUz=?%$wR2@aC+;^t=3ET<|VYfG`-@6CgLn;XSDKPd4UAnaLnS*{4EL+U6ujPkj1-AB};v?l_#iJBM(eT;abJ10ai=Bv60am!pp?)AxsE7
zB@^23fvdh{<18?Qs`L`aG>32Bb4_;_o$TLW+O_KxC;xYEU!8T5o!Iejzpli*+sy?Q
znbZDz`DJ;2i&kXgS&jNLmVuKek3SXo!L+e)9*4n~M+^Tv8X2m*^;Yp$>_LLRDIgyW
zG8jF7tShJN!hw)tU&i|W{CUt%Wq%Y_5vGN4ld@>HLY%3mcPS%7P9)Q}@A@^_6YRYD
z3n9^1Dz<&!t*P0V>hz{rPv7<1m*-2*id@o
z6W2MPy6NX?Z}xuiAeDWkF1D+8c~7JLnaIl}ifD}P#m8%oi?PsnAFzE_+_h_XPLijq
z*Ne=?8hxp4HX{?vkW4Ud&M`u89=bLH&m8RXm^Z8q^L`(Z_4Mf@w6uT(fsa85AYsf-iG^%wsFkxQu-*uJAq;}#Dbzu=zk82sn2>3wgoL{
z`ml4dnwm|~`PuUmE+kDZm+H4~dcp{8?Z-vsgZqjpZ`1Z3v$3PFDC%$P^2OWI)Jj%P
zdvfHhyO@Hy;WTn01@+U~+%**qHoVy<4p&aqtgO#jO!jQsmcBiEK2i#*$+%#5pAZsq
zF;{1lMO+Zc2j`EMd;I7T+2vZ=+Ninln#S3;9OxFjl=6-}s!Oec<)Xe;xV&7t^hUi`
zY+_WEG?+)FDSaI6|0YhEQnB)6SdX4ATNe#^S{z??BRwtUp^RA%N7?nihD-~$>1{SR
z&{pV=B{@qyQhN6;y_H2RQ=jjdGpE(WRb@^?d)KbuI-sI;a8SlO
z{FhY3;UNS<&Z;+Hv{1x#SX3W3lZ~6bU99H*Y?0@>4l*@mPHC?bO42
zq~AzLsJ_|trYwU-XPIxwO?#~^n(mSZYXtm?Cg8@gv^3eGo_YyL+L>39vM`Mdyi|wF
z)J|{p@mT>XLKcaoB}Mhvqy>;7;GygJ)KnGSxtxu@i!#Qta{j1LTE2Wr!O_B+24kmI
z%lOL^zFjihJ7LhZGi$ef`I=nt=)hs?8^h<`l$G6z=M!-F>ib={wwc?Hl6KNQbvjG@
zFj>SmR+qF85pQX6DN3ie?~mJyD^-7ih$kf_;U*S+*l2`IP$c5}TQ4>%F%2vp$}4VL
z{jxC~J*h=tlmvn$GGwp_FdelJ(xVIZBcpBNMlZJ_G%~^(Pe0o?0y)<)6Ik!Yj_tSS
zE^jX(I0DLK@zLPi0#w;(FXNwS$%g0{32M5Bu1wLP?agGQ5Pt$=bf&oDoX+8SBPNk3
zMJCvw{UiUij?&U1*Fq^bi~KY`Ja!^{DeJB$&yZN_Ko4->;X4XHJExQD`I04_#%r+?kTElP0B<4T>rB
z{`~aGJ&SQ6RTX{g)8AN~{tz6ze_+}SZ5IMtAmFa(9D<*8u9aQipB(wbGNof$sJU8M+RFT#?f&=j)~onx8BcmwP^|E8k{)D1
zA4c4jZf=O9ljgl4SB1TmfaMHvtYTb!IjtW<+08h!?}>2`!Ub_xcc=Jo(l$6UAa+<0
ztcd01xoK$x$!C4fiOF+On&f*GW~I=Lq(5ZuWT_rH1T%I(#nc2=L04Bd>)ou=c0aZn
zOH9V+v-38w1Fm-m2QLfFGf$W^y#Kx|{~%$NPvLvRrTf59UKmQdiOhs5Y${M`j9Z-H
z#elsu+t(m+`X)m~Sg_z_NX&M<8>g>N2{S>k<}$A7i>MVXsOD3ly`XIlW()-6A_nX?
ze4Aop!;Eo6A#-D&jDMylzyFy#>(ql$L`#R}p;V(0@$kcq6&`GIogI@PO?fVX%YY*@
zm#WgbqOU+534(4J$@(l*Gh{4q29H^<9>M&lCV6Ojf*|F^3it2tUqSznNS6=aJoD4n
z*OF62vC*@qE|*E_N7zYd`~-lnP0C)&Iqb(E>C;JHwSG2H&W`B)8Wz0w@7Qi@`|({`g+nRHJ`n33pQEq!}?ud)f^+AKT0qy(xMtChgGM#4M#Ji^p3&R>N
zH1`obtyLuPD06(d&5e%sxkkNze%$G!-ZJ0M>y(WKSQI@fh%VRYA752^<>|_}kcO*P
z3wGVDc%8mFYTJ;T2?;9@@uC~>Yj@fmzD)kr*Jdy-kJYO;I7x_57?JMu3pRuAsz}rs
znz)|H3Qa2Be*k&F9O5RCB*U*!+X{I|RX0Y_v{R>44Gbc0I|Cg6e4uZDO3lyDCpj1)
zH;*_(YX6>?LBQeOd6jsi>nn^RIi^RblCrXwhlly&?f8w3rLjSyZh*J8P*)oQ=-(M&
zxR7jo2W^BC^g`#H
zW20@jvPpd-)Y)C1xH^K;(8D`9mGg7%-Nw`hMlE_unPYJ=(cJF(q@AM7UBIV`dGiA~IGdplJn`-bbZr+ZXpHGJz4koWPze|9fkhE2HOV7gYnwdP#>OAr4)bs6pL
zUkp~O*JrMr^P(i9bct@R-z(>J_k4!xS%2Q6lJn<#ibx+J>;g7^>S;S6%7q7q7I2K>
zK|x){#R&bubYE}Tw22RaHyDfv(S>W_aRUmz5Qy+hFosfHa@r0Qfm@GeB}eUn@G>MI
z{ei7?y5_l%5GJemXJ4B+)o>3H%7S$wzmiB&{Ivzvk7e)RAj4%zUML^WnsJK(2f=dw
zr+MZW!>e=PK<<}}H+3xx@JgmT6h|{CHc`yZWG`^oDk`bPwL*p-@adADy9O@2k0FLa
zvJFD~7`5QU+^k+bGSJK+&21*B20Y523TZitA_a^88%7*S!ZeLJmX=+K-!@dl4#q)X
zi#6uQk4sTeB^=e*JBZ-K9w|3!kq}#hDEPG9Yxp`v#lK{YPPMRr{1ghIc`%7tBDRiu
z5!M#jGar<7?E2Go*Ab0Cr`qQBp6%7MQ>Hh7Z)o=KFJDtq!bkxcCtdSU(f9B}pD#y#
z{#f=fbHfNH<%xO$mgnAeDczN!PcC6GTit`280)hVwqLtAlS-``nzwN4oNkkUe4pVT
z-f&}K%%gFMh0>)dvyu~sZBGUxEh{co8uSpwNWoj)i09m(D2PL1i1X3XP|ap;?i-(hfFw?M)#=1niSxaZs35`l8sFm_N^F=n}y!
zAIwRDjZ}yQM1%vx$#lf#Iz(0VL+qCK?06i2zHry?8jL0P4rzEaGF2<`(YvfICns&Q
zPc0dyskws}C<+j+08U0xKub#t=~}qD3|7D5-$a+gx>Lsz{9j56@1HGkiF6<(BDTzF
zuCXNl`42ejxy}s!46NzgijmWqdhnkTeZ3*3a5Itseyxdm#;99def$6P;(42<3Z&4A_>|H?tuTq*JX7Q!wYWn#4KIfwr{}gE$^x;96~eR
zpD)eH-sbtZr${fxIkfgo!HQE$)E>yyRE8&}IQ0FtYH5J|YSWGGKR!QGGYwYh=KSM&
zOzX0ojP2h84~YMJv%}W*w|&Ct)9i%p^;zO;BQH2fka-X}odtbIlIlUIORAd4tVG>3
zP)^Rt@pzug>=BXR?%bb%1?7HCg)8#gd!tKe4
zht=L4n@c(S8EUYfL{7Jq6~Sk43QbQiKXSFylb1Nfo@Qh&CdwA!qH$1MVe}%c3BIqg
ztCgi?%F?WBmoBa2vmjkjLw5MEVN)re2nz!?qY+c$&eARKNd2^%J6C-2Q-+VwefvZr
z@CD0W2g>hvGDVTkpFS0O6}B$wMJ-Ro?tMxVeH~ms=_Yr)es+E0BAL4HQ_IHwj!kU7
z5!;3_14z4{?AB$=wE1R>MB?JXC638wWdAb=f2sZY@+~2Ep*Pv}|7l%IOuuf#^Aox$
zb4!-_X+vzWEc2Y+N`f;!Bs6s5`i~ZroBIr*$~84rmoNyTqNF~d)cN-Ft
z8}BW{C*Cj0TUz_s7}){YPW$$C5!swHA&-xuoU@ww9GVm7e^5Y0gM@}tV#*O#R<1Xw
zJ02F&C$`?)^4=#gJNqBUBo=d!PIQ1>4-FQH{yTr(mmY4GRw2f!VjXI*70OkWm1D_P
z!!If_qdv#%)>*9$`@P4S{XzD%WH=99Qj+%@eHGi0U8K5Awb1Iwx_Qzq^ixEGYtJ}y
zHJhWGB{GHznihf*B)WV}PSvccp1E%=>ef-3QR53^W>nNWWja01sh!YcNzd_nm8xoe
z$`9#}pKq`bVi)(kBdrhKF^zXm7HkocnArxbDiyK5T4IQ
z!1Z)P7kBNF_>r5L$KZm+gXuNv9asV#h<*Pnu2oP_
zKnx)Snl7trR@$5K3@0?qYuO4itx0w)7;(X12lnnnwQ
zw)&5V`(0GqaaJ~I=$9!Xkw{T=Y*O~T3^n*(-qqEwc4=t*_`LB>bFZ7qN2V79iiLO96v4>$2e8_V7TcQ(rL#;J@t0JaiVt~l7M@^-uNW=_B50AoozowbV
zFFi8Ks7&iCqMP}``SWbE`}Xe#KVWpju7-ywn#79KmPpsPUR=1>S!&u
zGilONOO~?e115HJo6VvC2W{bo(`~Y-3UJ}15Y&A!g{0XC%_V_m9LEnHKR#PrYonP4
z2h2svU1(hw9}plSI3@nsHP2F61%*ZA#Gt6Riz*m%#8|OA`$mj)H8H%0r
zmF33GEiXz=#+sIoR0|v|W0tieuq5D}boy2I^XIMl_Y9Sbte`B+OkP(?Bx?7brYMq6
z{U@f?p~7}T!ZIRZ7Mlmw_y4v9x?HXqStr#e=76%&`P8{9yM@m6n@bVFqe9y2Px|`q
zv%lECs;bp@QPgUT$_&TJ=g!?PDk@US%^?U~uCaEkmxGJTN4EBDj{E=c@kX3dx~Cxs
z&)7i-FX5v`MuaI^Iot`2GgU@Qbnxz@zUGT1Jb3GwLl7iISVMT|^4ThcQIQKbiZ;
kP0H_e#dq+
zE(WbvkGn!>>(1K_^wXM|-9o+*ZNufl7DW`Pfx&zsazfEUGjh6c{6uLl{OCAz%k7y=
zotXMjM}7a^M@TPUL=r8_{^5r}yz@q+XJ;#g<`FomI%LRBYYm$BictR`M1zh|-g+uN
z9z_t?18H_-+pROih&i~kdKGwsrw0&&Oavt*a%`~8X~;4oEFT!!wNob08ONtg-VHM8
z$mv7`q2dnOxl>F8O~N#J-a(s;UcN9U&o_Udl?!V{R3F(X(T_Xt_VOW71!xq`+18je
zT5BkWWBswY8CKZs<;$=~uAekco)x)?Vp}^F&Fd-Q+F{mB!&36HjYg@=Z4S4BfaH@$0m*OzpJYpWyUYS7i&VfFxb5V={F
zWXM8?a6q7CslO-AMNahMV|r@qzn^zh=I^!md9VNE4QYvT
zU&r04_BLP{Z*Jbb>*+b$UwtJX#JybUe9La}#GVuMTo%3P^)OXOGUobQr-W%q*L*b_
z>jqZmcKlnra%JQ@ote}3{cJcr>ql{x==%Bb$)99P@RK9Lb#Xy{&vF`7bl~4&bcks2
z^q!bnO%y78}Gh7VA8->-va
zi)x2#u2$`xsdIBy!koVOmBI8jK3Tk{p?3k_(a=1FT=MtR*BhSr<5;y1x~8E4iIVk`
zDWsMoAe9M}+aC761i$LF?yFZ%BscrM_Oe=b2B{J$#J&ESC7GNg_~yXqv_3c^#5HTO-H%bdA*3NeMN
z1-IS{xSF>V;b+3%-l$?YGOtdlYlZ_R9j$AZ$@tRAaShGmOJ4KR&O9@_(~ws{)Fy2O
zIlq55X4N-7deUOk|NDQEAhk}
zHvbw6j{g3AD677wLx#@2{H7|$%Gbx{YmQwoZ@=Qi5s|%1Q?7=1{nj+F-qU<{NzamQ
zr5Byv)P)t-DgtaECeb)l|EIp+qy~$tsoyQ~JJ8IW`a7!ga-BwpBFn(rH*bFY{@P7{$m
zr|QFh7nAequYdXSZL*12j&W*8$v~j!A)QBa9@{Di>huKtA$La(5ApxkCAGJrH1CCO
zs8j3umi#!!Lle7(S;|oTdw#x}vaz8;
z*K-k3A1BNF8ByPM?Tv(D+#GqK?5XpCo*}kePjz^Rio%ybt6}Ns@q46uE1MW=ech=t
zZMv^&y_jw4eI+reyKJ^tMwTru`|v1IEpGn(>LA+_@=r{)XNQ;O-uNT_;`hnNB-|2V
z@gU^xvdgL6o1KeS`xREt@pssYI^(%T8>K>QE
zo@vq&lVw95*M57K>p$-P-Bm+`gj4@Mf&y^KuHwW#1+G`Bs$TQ3N|^qRS-WU+-4}!Z
zChJYmITZasGmS>WEr@Q1>K}o&S+BhIPqlYo{M0?4EpEu=O*PBDp`0-OXv^izNjv1j
zr{rqugzvn&zUOYw@QhIEu_kBAJ}uhV|47GOkMtb`7c%JV>2p%1CrSGILyD>fE02Rx
zS~hmm%ujDuiMiK*zc<_7e%-g%{)41~@_S{(6h6_-F-){OCb#R!S(iz3e-_W&dGqv)
zchw7(G$-(Zns(gGt5#r@aJBvFJr{A^Fxe1d0!RZDkzyuw*RQ=T{TU)e7RvmL@AP|P
z{|8leOG6d`yb-2CyB`0jZ(z^eK^E$5fBujMkoC^Q&`?r@OdqqofAV`vh#^)i;?S`Q
zlxQwv4FYk?%%m^a5(+gxA6fmjZwA-(QRCHaBm|Rg_w-*7uhBNIh)G}hL{Z9jg%mXGtH^KAGPc!y7<-3tY)93D
z)GmP@pKMOAG{C|YGk(j7QFBN4_%`)pd&QJrHXWj0qwk8nL-~=8UYZ38{?v^q;Gmgkb#2_3vQX5|R*!0Igk6>Bm0ICh505wR-0r
zw>01`C8Ld6%n@g#&AL*PJiOiOwy;)D>f
zj=>4xtiAnaDneLN$uk-gE!aa;vA7v1^geSM+QMBQf#QTwOz&=K3zJ`VlaRPRm88L*
zyXgtRDByk3Tz#oV!RoDBi7;gtM?8180;l5`N|i}RiinrJ=Spi&7uGm_cjh((fI`FQ
zwB(m)V=ayoF)kF0mHgArcGQh3STiohF%`Rko|L$Vbhf(96GZl%-l?@}a!YJet#$U)*zEH_!gS
z-<Z
zq-0SN_c-G4L+~rgXinOy%21p(MzBXjK4P0k0?JvLJ2u2XDq_lL&
z5)!SE5y=F?DzE@+lPxx41Y!lIWFZZA8T$tgI|e?$r-7jqbY5}E+ebth*@G=Do0H}p
z$AI3y|GYVKI6o}npMmzm8Q?^RJV+}ti8*m%zs01GJeQb4k~?s*96EOF_51gZOPBtG
zI-|-XI9RB8M(>_tzj)!oReJK&irHzMFjze>aym)BTBAk@-GrANE}(k=^=J3T6HCCJ
zeKLSJxH+rXB;7aL(y12(ZyF^V!q0=@kDSg~&Jl%-
zjn*CeWG26K-0mRP;2e2f?p}K4e!^&z2Y?aYIxjDs5hH-t0mVrX?(>f9q+7QH%J6IR
zg9ku1P$-B>NeFtkp=c7=KMWRUrCZu47I8R@GZ%Y3O6l_DmX9*dulELn02s~8M{P2C
z$7u%f_x2?GG(MUI3rb2$YgaX|Re9{z#ORuJ!)VHsRTdw2wXc5d`4m7dYs)Y3iK~2m
zdi&o`w>aL?JZQwoshV@@$^rn%a5N3jNt~kc>+TU!X!X2pfQb&
z-Won;c>D|uFyx}#<3FOP98FP|KpV}n8=I2Jdj9XfY9J$Ho~R7Bta$@a;khytd$`S8Q4rZ)6lQx7P@+wxq&
z7gLwE@C08FQNTMd6?u6>+^3MI*abK-sV|q8;zEEv2aktTA?dTAfX4fl=4Lg=w}8jU
zI=Cfx%s%#wcu1YQ75edc9yFB^2noj?9$o|q43mPeGO;{PtE8)Mmk-OJckvRhe0(fW
zvF{Cnj(89OEgpE?qrEKh`0*QnYiqyfZq_2mlk&1ba&mOe`Xb>N$96W}Aa-tY!bp3I
znXWyo4?kSEM7yXcN@jFb^|O(2Gq=4uIbqP;LN8MY2IxNph3RfZpR^rhK9{~N&oTK|
z-_cTHpS1n6iF#n2g#fsTb=2%mE
zzorR}V&RzR6A=;XizfXthET+LiLsX|6>6m!27s=DR55t~|Ki(BeyJcMqo=Qr`Ubce
zm>x5oM3>tPG^i=u+|H3T?4T_`r5HGLj8=43KI)veh3FkbR_`CSmX(J1-nvayT
zCUoE^t~3ZA_&)@Bb9$w3u3L230r$_(H>*=OZP~(|o;1&hGaN0o-Dx|Alvlu&@B@R!
zx~UFR=CtFnNU?{}V1_KHZsd-p-~|;~^w^c^F1i0&%FttkSeyyjG55?{1f{o@vwOdP
z{~qy+@J<^VZoMfE!UUw6^RH#GQLNCYgG=$-Yvxs&o+t#~ecKrznx~duO;P%)&!65B
znlhfDa-ng)BZ42ExNdLak=~rIQucNA%Yx-c{Y`2%m_+*j8#7yn))WOisi2z4kW(J9L{1DJ4;SUY0{3>{>Jp9Qhvu!Vdh`|6V9*Ux{Z%I;k?Bvr85iz^Cnz=vO2CfXd5e29TR@rnHeL(L36xNrIm5%l8nfCOf;ww5RkmQ
znLghBy3G*38D)fms{0vB?-38KxP5!FWDp~fItS*7U#bGJv1^%wdW2!@tZ180JJds)
zJ8t;Vm;v+Z>cOE^|5q&Q-GRUR?^a+B#dEyInM>Pw+GS4;_dH{X72MkBHb8IdXebV;9Xa
zPVVaLQAn2r4ouE=Et?0^J;@Nb?Bi}b%j0Q-5XPviQ!Qfl<#N$peC=`U3DTyaY~uXe
zeY)*LG)3*lP}v@h!(5w)QZ0fZ}XMS=tEZkE1rk@=aU8n;npWkz3GsV$hi7nxo`5hD}Md=#C{GI
z-=d1$cJMgQ03rCCzQS$SIs(NvB-;LJIAStx@Y!dt!4m$p#Mqzuxm5nf)8dgE*PrVi
zR2vv)?=ptwT<^bIlkE1!ns!`EZ@=B)bvGkDUF}-sZgH38%V|_tL`dtxg@S6H4J-T9
zDOE_YGtUq16Q;B@w?gk>;es>h^`&LpK5)-i7@VC^{|)%N)@wZ(o^!m(ph%u~9@Ssf
zV7}~n^4z{9Wo09y3c{5pYEoeBN*g+x1w8OPt^Vk>Sj}+8B)^1uS<8vU00f8Nk*?lt
zbhn+C7p1@|6xHB{Bda0MEa0sPLLW4^Lew|WtvXSZqzN^hs7TRh0CX^2?YRpDMF-=Y
zBp7d$1#m{h3e&`Un($<4CLSp%UY4SCVsY)k2(DG0bq$SOFjOFgh+E`u&#EMkWf|bOLMmXlwEK!k&2sv&&jona_XODlFABPK$p3bl)M{>u6!fekHYis32OsUCmEM%J#w(_3t^
z9e@?dN64f&30f7)nVp3Ur=Gjj$@XZw3D`1d(B#7=Yeho9@A)rZp6^ladDmsw#XNb|jqait-jd?lpH0t2tRUOc%00(rT)T!*pPH+zk=E#!`BP6@Hw_P9}_
z>Ht5x2~oS)l^GhSqarB*{TGVm{=-H4{+oQ-&RJflg(bzoMspi580b98JoHfG(HJpP
zAWuE~5Z##~K{yD9aL&2rE>Vy8`Rf|Bt0HfrL2MHUZ
zmOR>Bkf1a?N?G!tjJd4pM$h7y_209%6wJS(k-TM9^S{e&{Z(YlFOL%c^Jj)*;giQP
zkqN)t92edsI|Y=CuOj7V33CR?$jGRu^qJ^F_RuPh0Pt9(D3E@#|3}k#$7A{T@BcJH
z$w){PDp8VpIW(`!$Z^c?4a7eZ%4BGu%*P3oAzr-vX@a2LHxPLXZ6J@%*cB>a
zpt(^!0aSpu1&_sDtv!{wXMKCgABabDE9BN2K(=DVLTmjng|A5o#eC@`*8Ax}*W~~wK)CJdLUyB&aL5(n^Iq*4h|fWi4$5lQqXv%W;vkKHr!58t
z5G3j7k#u)w=^|ir8Z(1sj}5d(s7Ha$f;%KZ0}@*_Rd~)xz+K2!@em`77=G2>DU!zY
z>(}Sn{5Wx;tTSVH1{6P@7}5h-Wd=VrI%^2Mp>{PEV1>pVGb}KikT*=@hie@1d9ZLN
zXdtL52a=GXflHEf))2gz^E}cx
zaR#9aD1+*ytPE%d+fg1qK6n(i1u!8iMVb~B9r6JF)r_X@Q({@Vw5P?swzs3tAeqr;
z@j-OsCusYH9|?~$>dW8BFfU?r11sfO!6>}Zv;m4)x}`V=@ZQQokBJXW5vxUJR4L5c
z&@FVM(tle}P+VFHHXDX$qU@gE-v>e2j_%mS1&AM1qgYJuLZ`;W1VtGd9ymNq3=NTL
zhXM*YPkSh4v9%&6aNNKSTEif}UA`QEgHF4cS#O%VJp}0~#5au3!#sBhPL+U`K}4qG
z2!IYG`8>X4So-i4;Y_ZtrxqsaD3>IiMnAQTHvf)RovL`M@W#`9C*_IrM8K)AD(`1C
z!sYJ1f2j?!R(gwT2%qKOvgz7UTXK!dq?YP9Miv$xq))e5PruS``Fiw>p;4K*_kD_G
zv(WE=2vSg#1v0e6iUf>~7Zas@Uc=gqGb7|5r))7EG9;-!R&>S_@mgpD%JhC2L|Mc}oABvc~3d{q87
zU0edQ=+l&uQU_qxA?E7!nswaE^nu5bUZ^i+H45{oha6XcVkqYh=UBvIl9EtSoWO;F
zIUYKD?E8D=k=b+oaOE3USJx1e%S=JcVA+k1QipA+*b=A$H&m70U@nqT6}aJn`D95D
z5)y@QWP2|IdMkXTyEoBJUD*
z02QjK{w*>)nf5kU4;jt$4?6T#%9Q3ig|#E;F`H6-nl7it@B@4fw&G!0LusuxqD!0X
zlE=#aJZ?)Hi@kds2_T0L2UVFtBx&`^i?8E)QcB#3QueIXhhCn|p%U5-Ilq2~Q5N?_
z3PFPbaJC{wACR+R1CQw_mN!F%=MQ}yOD%dq-KlcR1PC4Yya{6eBUjJ5#0jea#pHNB
zm7Jlm4%MGYt6Sd_(MFP5(V$%bWQJd0Tc(jQva>^*hPk@Of1~4#{e1daROBb);wJ@}
zH9q@#S*7tx8^%XTJb8a^^Ypm9!HB9@;nr%qLc^?kU$wn#Z0?szU)#^%k-d-4>hp^O
z+2>5-ro-(5GKsFsNq?+GW1$DTGElCC4=De#5@XVg+h)?ccgY_=3d)(fHZPbZ&vepL
zo-Wsjo`*OEBJJ1ulU(TyxE1@glA@Q@g`k~BQF7=Ia6Zs9@rK{AlK=bKly%8JnvRn4
zJ>eir(sM3c+X>YwRKMtm!JG%(4CAAJVmPnq;!y$CU=RdA9|NpuX#ZiH4p{GSc$OXy
zXRkhLRy!QJ^B1kxisMXcim~;P6aVdHs|lOC`mideKx68B<0G2ExYn+&mwB3@9KVR~-m@0X{kj^yjp(#p|9rta;tA9Up9t5+6$vBYb9y?!hz
zE;cSgJPru95?GF|?jJ#Z@29Vk643CNR-}2!lLgN9U*&_QzLDjj!%jwu$_2@@6QNXL
z^jgJl|PWY#3
z1RV%!8%=E)zTir^-S_lv3QcqIhg_B0-zd`UJWyM9-#}JIRJ+(bz}{A%zQ#h4g`H3L
zNWQ&kPvL<0qZ_@y`rG=_r?X91?i!lIwH#%-cscigeHn}Nnf%S~=GQD=6lI+|%4{+*
z^}9-<;$=bM4HkizgxE;QlcxT_=L_PToTdg@5MRY<)5ew-mxbrkbXdyOaJuL^0984;{63G#iEh<-ZwepXfC-AI^vID18x~)DkGSY-s
z|K^c5y*1*eom1k=(Q_pmET6Uf
z($d^?)3#dgFUP-lejy_)qL*9aekC+$O5Kw`A?(+C8f3pzf|r=1Sh!1%c3$C4Q1JYZ&IflZ0&vKMPhmT8mpXQOI=iZ>?ouAFB1=Z*lej-VN@?Ns
zd_L0Adu2Yj=!(5eBrZ*#0I%oP?L5|F&M}{|+6{7yI0YKDWLY+&c3EtmtUqp4#zRx6
zj5`u2p0jF#H~QGaYM%*3pLx{1RvqA#`E{l?T~q7O6E}^rj5L2AABP!VHiCvh5GP4<
zGo4Yk(~x4$9Ore{&LWG`DWwOi>-!Rd=QzXmSyRSJ*Y4%+aqU1nA~^qIvA+5FqI%7j
z%@iuugCFkBylR+cTk~utM%!)bz?4Xt`q;{?!fb@F?L`RmOn(lSyTD+Km7=k1!a%W`
zlTh!<*ZKcOcC8AQxC!FP`OA|Xc12Oafdv@9R~e;DyCin>Z`MC_+BJGuZ<~a-^yccF
zjo)<^_uZ~}T3-K2*>J1n5hPm?PQCSktSlq!yXedMzx6(U>^vx^zP`fwWdj>VLyd2A
z9mQ^}f}pvs%??qdr%#)D#e5$xUeoZdFv$s+e&ZlXPF_txe~a8_)@Of&+CO9K86%_b
zu3;b?lrNVwHZxW8<{2OT>#qt|O+)&)sF2~+63VW3T&><9Ki?L^fp4x8x#qKd3i^ua^LvMv*LZB53JFS&2*y)Cp
zmzWsYV`cg>NB4WMTlr?}8qQWo@tNRHEA~MW6sMFUW1y1Oq42TKDrtF5S6WsrHL)r}+6FM?z!ju?_=GYM6(4(O9V)xfpM{k-Y0z7-8a-(O
zzik1atP&_YG4U1>LPo`1#eMO=OF*%MiWV`O4rCZrzO(Im1F8(DV&~5TB#l87I&Z94
zATK4AtOU0BI87NTvA#$Dg+2u#<^SA*E(GR-pppWRf~pD2CP7;sNvl%Vx$me<;Sh%Y
zEnb1UQbhw7E=&pl>(X7PIKbU2!A(U}`>SFhf>-TJYJZq4tB_a9*uu91r%NzHW_aUI
zC(OaRbaY5-CiMNWzdQLfHxMS&a`-Tu$P7tKt)-=65fLJBTh3?s{i_JsT;6Z$9sc>B
zZx4mm1!jcBTqJ*bKWy~fyxNnDwBh&&SohQIu6@3-)i7p%G*zy791GgLVw)VER+Yp&y&X1CjP(Xqxsxd5;cJAJNNk?Z30Z6gEwe=Db1+a63
zg!I6Y^0TK;kxo)uU*Be(kG-XsxVR!?KDIItXGVQMMhN@>SP`HJA_*iQ_53<{`FKip
z_B7N5*gBxK#-0Fu!wSNUfHkWb(YWh!h}G3M-!o==HW
z{@jesVO}@7ulI-C3|c=`bfURg)-q2<{F~T@d|zMpB5N6SV9JI545wH+-H2;z*PAuO
z`>_PU&taA;PP}v(=+U41W;i0h(OGv_W@>G#2m(NQun2?v7lE$G*uu~g6(nkJ%n2}d
z27DWFdVjz?uL?zKQ_7DYR~Q(=u#Lqr*tzB9n>X`#JTN^ZIdTy&wT?jo=u%aM|H>8^
zEy^PyJ|bOQM^7=8UH{h7`0NbxDL2Ehta0jQq7zaTpmomirl=|hg(fI3bcd*v)|w}c
z$nsSlbud@K83!o{Ha~FNVrzx1vyjkd%RE31NM_O62P|k$NUro2#Xnl+C1BHPEo$_>hWC?&yRGjY=fGhT+43R^(1j~JSDoW-
zI1JYfrfc1?{LQ15X?St}DZ`h$<#zXf+$yC3uN<~L;FIlmqzdQ3<_5717-
zp*rMx$-inA)~)6w`2ke0={ghyx>Zagt(s#`rC!A7lavbj_%RdS&WVb8X^>ne3*OpFx*nzu|u$7qxibs
z?`-4d#`sI#p!8bJV8E~hPa7)lTzx7kDm04FbhLJK!0@|?
zkyIHxHJFZL3lFSFR$>1ww+;;RsHiZGX}EF=iwwo&hQ>w^T3}>VhtPl0g8@bZ$cS#2
z_F3k^UI;}6f!)J7blwofnQicmA}tnMDWf8XA$aky;k?tlOmK!l1%-9ACM-0dZkk~F
z1Cs_|H;C{f2!Jn{x7i;msu_CS!}n+?&_{@qbh%KxuMOUXZ)V>k(Ypc54wAzo%01n$
zRyJ0D_V(@*3|L+8%c+sPoZ>yj$a@d0zu%+BR#&=@i`*5BrF~PBpZ}8kgq7v|BzNKK
z(wXLwrDEUNRau@lAPbm1Jt<&kSomr5nD>upF!qw@Rx9SD@q@hz1pvK7&>Q)E+#x6KYl-~tlfpc;BV>D1
zQQZbTL!1r2&VP}yht!SsZZ>|ijRF_w8Wg9a|emyG?EcVVh%f<&E7Q9%Xx3ynl;HO3<<}0;K>U~8CJ(XAwjd_t7vht@2TK-7_J$~CS1Q}eLkDP
zOGR}d`t=TI!a`Lh#>RXKexsG%|HxWA?y@JIj4J*&Dk&$R>48o!yEPg!2!!c^Aw|j#
zCqX8=fZquV3p94zf8fC7zx#mU;0+(jhiHx57juzaSQ%iWfrsMddd=HoRHUW`@)4*oXXftJk+L8D5^NAK`>5xgPe^a9&+Fi
ze|`RbO9P{uYegyTb>mCJf9mhn4+YsKT7`%298bV4=`
ze6)1ejnv<2OM=VRF9G?%8McdwX+QHe1b1Vl^e@#RTG}PM%Xa$LFMNbp
zfTZjk#mfpEJw&4TUxEm$H!QGWS=xN+hi3;Ke$2H9A7#?M)S-K$haA?VEn8sCT|@;d
zZEG2Vhwog1Q~?T#_d%#qb`u1RAN^h_ZJdRo+K~T26G0G|=phdS))n!%8?LUBfLY_Y
z?x5u4=I*s3y>aUZtoZctBRGi9<2drYJYR>kxK2%kghU@RnUr^a-q9Dm*7>!*TwwWD
z^_3?aKm5B-KL13hU%2o{Tj~(Ai1D*^J?(~)QpZNWe4Y!BGxWY^AL2L^KK?vAv6d3=
z(T!Yv-K=@j3b}?Jcc#=0c$PyKZ}iysv0YZ4RQ663X8QJ^bYP*>uWy?OKI>(WikABUc#5CrUGSe1m60CQO2)1)2Q
zYkW-~n#id|Y(j7hkyIWhsQkCm)0-et?;JfBBjYp!I_V)fVDkaYx58VmqthHWn(J0j
zp(<*!Ds}a0P2eKkg7zbFj}zK5Vj7I2Hov&eYb44jrjAE{?CICqvs2J$%>LSE`^8H#
zJ@#)&A6KVdUe2l3@8g$3q#a&0L2L5GdXeRb2VdNxy+~K^10D?jr)c%sW<$1e!VLB)
zNq!TkAcvSN?3EB=8KZs==y(~<%=^HO8in8FBB2QoRDH)Y5X)kcsC%q1{~c5mP)Svp
z-MxDkQ?h@*>?FgXsH;2%`46&FMVG`N4;6_4Da$*)IT2g_*6=X+UxQbt+?Pl>otIni
zC&PKZLU~Mp-0#ClH14a23dO-Zv-sz0bMz4B&KlnE`ibN6lGY7POh@AGwD4xmUvVwb
z>XmqJ>rn8AVL#^16cn=icT^ormzS0n^YnCk@*p%%BR^aGZT^w&4^$tXsHFBh5Axf~
zhnmoNG3fhut;WXftRJ?VR*^WJDl*gGr&!6P8<1npX!lF$R3*7z$==R_!Y@mkdM5+Q
zF1$v7bz?(Akt;}8)_1vS9&Te(8gf<{6|XJn#-
z8auLWzCtFaOGIMeczGLt_Dg`@atn{luuUvVs^@}Ae$_)Y20x1
zug5c?*7hSe`4s41YW9s14>x~orfGGZSUDQEH+s7C>wQK_3>zmWG2OTMw1+|5-H(5=
zf4^m3S)rc&XQ?k`Z8xJ0qo^F9tUq8vB01(HjFkSY2j?{*tZ5m%XhSOz`q>$a7d
zB*n*n98!N~bDf5Ufd(C{gM3?8*K!p#7h_QL)ej!8JLj72AWyhrHG6rK%WYCEvO)8Q
zO){6`cB*mM-<80{&()0Hhbp1U7yN*lq`-pnSL`A
zJ;cmyM$pp{!~tGzUEewW#L;b;x)bQCy7IaL0wPLXG~=Z8lv6dHeGFp_E-NvZ`xX~L
zvGC;Q=%>Dw&Qm5-qaG@i%701@-j^)pUz)A!zQ(54*Mqsx>Dt)E2esy)MR8sUKALFO&dbejgnJ|olf4qf5}dSh2_iz
z@#>YkI$=~XDgysb6MpSc5-AWf;BSn>=p~Rqo>VtD6D4wrR$X4Oy&lJ&5Of
zHej(DOIJgeW9lZW>ERS<`u_6(8ND&}32tVh-f)fL(kn?{fsFQctvy_S+-_gHcGi2-
zg6GSR6)}OskIENxPkEKU5BpWYZRY#G(Nu+Sarr!+RYn?-wqGyAuCEo|h!gQjvF*|Q
zUBi%Nuj09OzS3q-x0v+((3;S5Z+N4&%ie+J11lXqO%w1y*rfWlQ2DfPi&0+;i}Dm{
zzRte;Z22lZU&zTRGhIVnubBwO?2*9K0_J!Mf_m-S{GgoxyCP0Qg5mR%)tAoC)p@0z
z#h;0Tt(N)v=#F8`rMNz6ffKF
z+i^W{gtqg_hcTBkl5xMqwmm&>w!g_hlRy=|>zJjUUed>gLAkZpT3CX1xo9@hiSUvP-_pu)Imf0o~=kRyEk3(Bmw|sNswDzlw8E=M^Hoiv_@&wFB
zRha_0r_M%yKQR&UUSj5R;&XBb!wW)!lE;-46dVrTCmVEwObsuHQ7xlNMms$n-ATsx
zBe(i`dm&)9v&#m>2P%@7wH|Sjw$am5Hn{dyr1D7z;2NQWO>S-I$fjdYd*AO|YEg49
zsMTV95KB(Pvw6*oK-qdr_LKjri-nJb2
zg7@m3V)bMpr~HZMo}Wd-b{~7cv7VU6htpx21q1(Dw$M6Q^|O)CacOrhDj1l%;JH!d
z0ht_WT|HBaq@;7il9AXXZN#C&hr0@{>me|~oG{X~vN~&McuJMVsXPDXVL=PgZm*T}
z^@c~;S?M#Kd!1cgUFZL_UlJu&)G^bL&{V-o-_P-Ayp%80u#$FCGtxQO(IoU)_Azh8
zss1lQz1*#5ny5eAINx%QZB~_y=3PD0uD%n-#-Tbo#U>_S@%k;Vo!ODq+MBuZGp?>$
z@i2X8+-TWSbEhL&KVOq&*_iZKpDHDgf4^|C2_u
zz`!Nt$sha&oMdl25ZAoNM+_R(7oW_E4$=BZD`fwG>=DB~I%&*jdW}rApB!Zj$-K1l
z=a8`A07K!~>lvV+)LGP*998*xUi4&bRq
zssW4<*Rv6Q0c%8mi4RlI^XzQ9KP<>F8X3VH4b*;<9O$7WmJ5y^KJ42;N%+*7lju9Z
z+bqQu>-;ZGMwESOc=+~uDd3j@ikQ5_x!jU~d=s9l&dx(oA24f4Y6%Mpsv6>rlHOBY
zsJgM+KvFJ994W2b2emh7
zQZFj;7`)IZ3+3`Ld~f#J$=Sx{`e|mSLv^zBVO77=!=L%dCMaBe#CD*rhkxTv60AWF
z9?VzQ^hFBHC92MNCwww6ka~CBD)?6C$(pteroX$WZqB>;o$yJy{rtB3@}lcWj|!FE
zj2+>*O(XkWiap|9#)9_*c{yer7zjd@3K2Wtu$UF%;BZ?QNA(DQ2nH+=VJxk2
zc(fS^M8b&zrh$Vbd{B6T|DpI@Wj5Y!_d&l-5-M|ZSEf5a4T0=$p!zXRgn)T)a4?RI
zz={&$NkHlRs}OPp$OHZ~(1X|1L30cjp8_{5XG3#K9P9+9Y@j-H#ObT0V%qBpU5?A9
zdTB`sG+B_EQ-&nUhh*=(see{~-r-5A*AcV{3@6V+Gk)$|Ge)tX<|A|uNF>a3
zaUCX5k<1FPGGo9Dl@Lh}!a+DZ>4tSNhJ#_E9w7eq|HFz1g=KXaVz>ghpj|n@(qrnZxCXW+#@I{8BKD2J*1$f
z2QD1;ZT^UDhP0M>yZfW@567!$YP&hI?Tm2084=M%DAJ{{zqe?@dN=$3w|cptl(}^=Y=oROb9*3?c2XGd}rUa
zi<2urxWAV8t5%Pyi*(s6aj%Jd*r?*xZLCk*;_7U);4@0
ztkmt`1|!Fn30ff9hk|!e+ltrs&q#moY}_iN9`=elEXJ{=#xf6kPDwerM3UPGu3BKgplXGFYLJRP
zP$~5y=&h_9I=A{eA-hLfhN&roy%5y05Hg`H1A@t^W?x6L6_rnN5
zrg(g@$AZua>;kBNfn=-Jyvx=Zv&v8v;)g*M%2h&HJZbBYXe=2%z0HzwCh#JmkX|On>?EOgE#nwDb%Q6Xi*q
zA3uLDL$?e6*=x;5?KU6;1JnzwY=C0w^XHXmWVrp%kJ=SRMMS_vm(r@5()v#@ri-Tv
z&J1YKl3c$B0T=l5CnP}_Iir%;x)tkZT#K(>*}%(CWroKjLTv$yC@`(CVTn=ag+`5)
z^(SUyLTu2#{fk~NQRTTMxd&oH5||H*1Q2i7L+N2R$kj)&0WIB!4@Ynmw9RP@y8b+gFssB08N{$axIR(l2e
zF+5}#{AqkT<>C^m`d&kI5XIfptHv?MZ_wAc3}o>i3*T|`?$zA9LaVjT8*-d-3mAtJ`PAL(RNNtV!H69iuScSk+(
zIY5MnFC88oyx9NT7$djtVBU*mZmacnT3QN{lL>zpD9>Yfe(`LP?3gzHsDGZgg&^JA
zPSw`iy9R0Er&zHGmEcSgLKlYygoZSf5H>YuxgkTTuyM^}`Uz
zj3B|CLGPhHCtvkZ@Ok6`nz1=ePpVGxC9esP69V4XHJc
zQMW`p3xNwXqKp)S<&-4ZBFaJZ2pB?vrejj!B?}L$LDnZ1{DuC$zM~OpaBX1AgMXYI
z-8wEfDk>Wg=%sk>o_L#(p&c2^e#$Uh?tKjh+avGzGrq7cv$GgwiyyGqlOb!QvEoW?&fRH$}h9wJtIBlduH`Fu%X{N
zV1sNg-zdq6+SOw1c!%mPS-|cJ)FYg5)$%&xFU*BMrIA^si=Y&
zDVHIA+$PH!jO9x==IkV<)}OJ~ij9RFsEMKQ19%vlXcA-!*{h)akeq2Km~aE)RKkpy
z)K8(shvX9?Tp_>+2p_dnH+XMwqv8}JVOFtF>m*=D0H7cCNMCo?pD|Vfk(-AzzI1c|
z?N?`>+wV4UG^I5f>m*3es}nB}KCo4WH{PuuhVQP+HW6muu=+T8?+p;x;3-Ff67U@;
zy|~=QlCkc-24o&Yg7iWkU5gQw8QH&Q!KlrcIC}bwVj^ByG0l`Li^GMTjhfYznYIhN_)wvLYYKhlYj*1WX|^6|jQ;NwJ@{j=em@!_mIJ^;x~T
zAZ{PNO`yTq6q%;wS#_~O&jN}oNu-~2Z1xw{2J^Xcmu{KvJ>82RP3=6&FK0f1|I2#p{TW
z(0?^UX1SgCz~CtQ*gp(B>Ni9CP-ikf3s5qF30hdVCK?M8**20(2jhwdK
zEwrqpB(UY>^mZGdXrW?;TL9l%$itX>bF;JPwNOWz=T>s+5=2~c9sd38b>KE-8x_BS
zHy^A$Ou?T#VIBn+lo!#7%kv}9UtwK}mxq)+&mHxFj!Fq`p?8CLT~L_duvgwcH`ba$
zN)7z{?Hem2Ba#{s(T8AGpldrI5I%q20XGF6Kj{CFx%OCjKS~D_&FF@}fZ!2W^4dZ=
z!Vpah>k%Ixn0@`eIbgxV&0d{@ZvhZD1PUG(VS9o8iVL|lV#Y@e@*CQ}sEY#hZ+^4B
zO>f%C-Xp(#XtjE%6V+pmLq!Cn^V8xc5oX8C9@$=~@}
zm+mYmqAFIe_G3Q`05Zw-o}QMdrm0CyR94kXAeRZQVTWQP*pfzWk@7+?A%Oi2QBtO5
zZVM#+Iu6sfc>mH8-Nqq&*En|$N+0(ZXTI)g$%_S
z*?~|PU_cpzpgbg|j*bEue~2|0mN!zn!SaA{TAn0Ig#|&fLK#74*q}>wPf_>NAOQ2O
z79}V$ql}c0Vu-S@V-{a&NDc`$OohAqMiIm?X_LzvOL91@&UiOAJUwbOCiYgAOZwY-
zBVPfQIa@Y_H9xt1A$@NQS}$f3J)Qmb@HKt73WeDluUR9-`3Ij_vRT&|u00k@n2>aN
zHKpCCu$6`3C$*2}Pag@vVmE_q<%wGqgR?`~$0{;sriE@DHZpauP7HPDORnD(!(@g(
z!r=UQGx0e4PVRmC@Hx|p$2Hp&u(GgBfk}@2KXToR%ddi`0vj-174OYeQ=$^DGInrq
zr-nEF%Xp(8aQkEAlc|J?y}$r8e7Tj`DkRig5kW0TK>*2tP7FRldr4v5@N0Qp_-c@;
zSyO`pB2nI3ZcP_RjnAL|D=c(5eLAqpY<97{-JSs&DAcuuJ@B-{?urT$2Qg&xIr7s4
zp_+OzM*S^7mR5;(sQjMdn=lk5J|)VcTrvVe2Y5~#JfL98;EE#mefs+K=a}CV;sm(1
zRIBIr%D6I$6aPeoxLL>u>w<15tWwUCdU?=!V5Y@31KsAcXZsP62<^8hJ1l>19flMk
z6LXLpytv&%E=hqKWbA|T@)b{?nqd($dj|=AHaBkIU4~x|w=XIkOo4K9cL;{fo~Bl&
z3CG;nK{h4ad&r@PW4h>dS4HuJ6vH+$LL(2`OlhSkGrOe-M{f%u$H0Kgh|b_NoY`Di
zc@)m)-pz6Dc-c=mmeM=7blv7HY09D%W4zwC?~Ji_b3e__LEZ4_cUDYUv;qu2csmRX_zS3cfElVs|D-^7_^~{sn}S_s5Qr48hoGtX=Ko1
z23$KlhZB;16gz(K5kNvk79p&+z&(QbT{M)B1L66_U+CRQ7idvt=i3`SgN49p!ur!<
znTK?0hz+scwYzy3dB+ec;V>voJsTwwqh4%Oj-yt7B^_y`Fo5AVKeC4xJ0oaT218*V
zj}H&OUGqK`B%X|nv;XWgP>MksL`p8BB!`EKiRt^u2=slRRYQ!27y7MT-wKWokiSD!
zVDs|u@X*cF-@(edh_VhZ3T}6xniZ2-1&#Fe^hla4P!y2~fL;vr6v3#t#KivYZYleI
zna%^qin-SQN(X}~lpzBHNg8SJr@+5cNsjsrpWDcXV-+DmL8QPYR3(zA{YlyoIN5P|
z?A*pDNJ)En`lQ|X6iE@3V0%pw|ZT*a&c`f=z
zESl@psMH+bcl0h!OFNy;3`o^Xd4_zooy(6=hJ$>iqd&HDN`1Zd^=aR=-c!E8mdnj0
ztDoxQUnRe?Z>qQ!DfWSG{^ZnC>gIRCRBH$6i=fE!@$Q(oa#E?z{c{jD-{=f_rp~)c
zo&`Qr^TWB9*^v@rhXbzU)cT&aT>hB+-S~K@G=WWYv@j?}@A1YLGc2IJkX~<1XlB~E
z6BVSE730y5Xb6VR8vrHz-+!~f$vQeZV&sok8=wNV{;BTQmz8X6#s>$veHD;*gCm8S
z01yE$JwD@Y0rQJJ_8+c@0B*#?H2N`y)O8|uRwV`QFq9|>7E=QiKuuIX6ja_XEGuh#
zh}qf6iHyKS0Ce_{CD#0S`SI*QhY2KPb{e2t;2q6gw+TF@th|OyLO$RAjIR!t~5Rd90}rN6wAEfmkbQH?GpDwnTe+l4F?jE
zhW4aDqyPaYw6xZuD?xmTJ~inm6gTF=ypf}XF_d5w$3LzZ8t;mgacEp3@S$Li2GIuw
zo*=2Q1|xU5W{8G?!F!=y51w(#ldlVqgD6EsMcBmvS{ZUcQ6nfQC@#**%7?C@$G(D&h6X%p029FDgj?LcVjaOFWj})s
z^eV|Xs8Lj}*Q5K=x)d`rO`m1Bi;YE0Jmy^aYWCGC@|HJmF!@>CqQ4mdJWxKww#x^g92`15LlB=Oo`-o+wXMuAb=z+eS+Idf$F1T@YYeWx8-@f+wi@WM7HG!fLJ*L-e6*Vf-+x$hJY%XaGzA@_HGL^1r
zhm|fsG3epL#oxb??~QLT#?GjI=>IjW=p1%!BRZdI(%kPbOpO$(i-urOZr>g1PD
zouCzgi5U}7wbY0GJ{7!|0OAEg;@)pPpn{4Fmz-ef!Z3Sy&Y}1Ssj%eCDzB+*FI6%1
z{XK^_UWpN`r1;bOVcP1*;Yez6xqTaSE{C)K^0k04#q*EgD5ICOcH6DU1NcauQunWn
zLGO_e04FRIbx|wi#OGgsFaWBdIs;+@cu0>_aV0xnLCi14gaA1}e^GKVJu^N15|lBl
z!C>k*;&+K_c8C@c5J8X&4bAMIKVN@2zD^Ss(7=RUNy%TG_pGL-3#&YmqckkIg@t!`
z?Od2^$rKQfAnm)<++!G`AHW!;lUSRS2IEbM+sZ{3i
zt~-A{A95K4S6jOeZu2cDaO$4&zdcG7Ho!b1%TqB){Flz!N>
z+AC{E2y<0iCqv>RRWhX{XU_}HYY*$yH`isT<{KKaoLdQZL@ZB%z
zU0a%aOXAeIn})MdvlZ$e5AXMH5OUmIc_V;rtcq@(V@L*Iylc3tvUj*k>|9NGX--na
zZpf81TS6;#sodj4QVYa6Y}+mWltM>w$?xw^qc)}h#Vyp-uu!vyig*8Be2dqi(Dm51
z0QRH1f)@-EZjt*s>`Ys3%{~dWo?wc&8yTLI$#q(3qpFUCd(fQSH^@Q~5IMnf@F3XM
zv~np${n5o<_k=La&B!={Rm8!ABZ6EK)jSxZF-~8G0u2)v9UT_dj7vpApWAK=v?1*F
zi&&?HvBUwVla=Ipv@3kkwr+XnG;41c{n*NXZ$uHML0C3hUtY5locmo97!W#aDMNW;
zeByLTNxq%vxat0+OPp_dTW*BxRPUpc3KlFqy4pH>T2nO1`iYpE@K;*$$swb&nS`tG
zbor_~M_yMvKHdl?er`hct(|XSZC9S>fLG6xq+?xDVmx0w{T6qac&|8>o%AksNqxV5
z`REjS7fb}>o-cD5BIG2l`5yJ&8oLXJv~6to-`$ZH_ESe}y_lSw2{$8>RRLrJy{xyo
z!U;DX_+6;?_x1)m*#G|riUi*(D7eu%Sm>B8kt84a>%r=ZW9vt4t`}h$sqb_eB;Rmc
z`%?TO{kFuP*skV(0p$mdP$cxSHGt#{bZK|AiZ~Q))*Z6Q0xes-
z<5tw(ard{=!p!6|D@{@c0|{J2>It=b{Les#j~UPs0?)@*@)SZD7vbAYh0A9K_fPY?Sov&ggO;M-$NPg4H_ku1m}3$s{{6>
zrw`xKxFOynE%o(IF_TClv%qX$`7FoUbM@sMk#s>PJM$
zmM*bfQ6P3qX)=Db|ZpBs1
zbhpTR1J}@TZdMyo2BR(J1b1&Z22sh$`$
zpg3LLvz5Dc=UC=b^Us!V<7UbW8$bRXm;oJEU31sN8^scQQV9jwc0a}*CB85y%ZW1;
zI>EWch*f^&wEn(~y3Ai}zYa|uyUbLzTwk?jT6y+NrIk_S50^d8G;g|b$0nia8=tt&
z@*q;rZ;j?4-zW%|*E#(N~
z;ei1E*TS2x+KWG%
ziRNk5=ImOp>EhaU?Qh*Nm#qA6R8Dt3&xCgM3nTd@2_^VY4WIX#@Xh#q@6x{3uJm6!
zIkZwJQiN`kHmZbVyVon5mk&;vUoEg?SzvKHesY?MAeP+cmmc=;P@EGX&e@D`A(VYv
z_LZxsW%s#Ge3Oh)fBH%-G2R2$&5qpnGgh3I;q@;B_W@~I9qmv0U7nd)6f1Q;ylBQ)Hp8At*-ANOCQ8_(3
za;eVoNGYuw?W=vDGQ&+C()t4YmEor4C82&v
zm3((`LO8^cl)hzwz`Ptfgtv3RBm6;Ch%Xs;`0n4o168e?y6-zN@40%pQnC7Z$g0PG
zjZIA#GurLnJz`=JTP%GA<<@jxPxsg7U#tbrlD{Z>yvENt?DzhWMB0UKoiD5>E;<(+
zw%JA5xyeMm;pXOd@7|tDDzZ`vb?t*Mr4#rl=a%1JYcm>l_FkT^4CkQrJC!(nNd5|h
zcx0C!|NCIraP)q!&1-LudZD>18R9r|%frkm39&lr7%`rWqB#;|FL}D_L9fLl+zJ8Sc+qraYR#w)!+S;uF
z-rt`AD}*W(ntM{Vgi)knGJ3<*i^$5roCo_Wj<;Q%;_et^2Us0gXiEE+eEW>
z^X5mFfzZE;oWr5*Twd0D**-@_(hu+9k(a1o^9sqREwj7VE-%`|{=0U$Y}|xNV^l@t
zjcR2anN)<2SH-Uwmrr06$S?oc$*CC^J?6)~
z#sVzU0+Bg(aRml9^12XbOM-~`D)`h7yG(U2;2HW))%*KA^~v)1GcduOQi
zFeL&C2&mLnDg_;#h=1l&ZyQXiz5o>lI)@Sk;rA3_PNOh!T*;t_g{rI`pcuqFV0Lj4()`zAI+!3D8kWL-DV=A-
zIL$#ipE+x6zh!9q+rIV8WI7qE?p*e*>V4kpx9-2bZlrZ5ZF8e-qH3CD6Ahe<98Wje
zbNAY#Qa1yO2O4P^FJD4x_Czfe3m*A{wF#|$kc(+DvEec@NGQ=lbOA=cg@yLZM
z6XQaB63D~2w+a&lO_Ly#X}Wa^rF#5EII?ICd76{1X4`Hhuvu9fU)(j!p#VI2#A|(
zY-~rLz=W}TH{{=-8=>tpMjRh-Ugqn%tEfb4Z3-}M2v8&|BqLCQZESd9>OABCG#f~+
zn3Tdy@j^3+o`WCwVZkU6mHwf9ktn9li~SWSVaewalnuTvK_ro^ou3--lOVa4LTreX
z-bdqi3WYUVeVCmcBr$Z-%|Ntkcq6QX7z*I=t{K7~6`OpfAmp&zkx6W~0nyenFGiK8
zn)P8+)Gp#_MTIv&J6L4jqkE1?$3t*ABUw91`YlXFu$le`9DUVpv$PN96S$#=9O5lm
z))?ui{L|9-LRFMf_>3Q1x!XfdDryhHC>+Kme@%q$m@&KK-?9@`Pd^JV9f{|dwpq4|
zSYKFmc{0QH)p@rqzhI1i&C_>mb%U!%hS3NK9y`|3)FkCHt!^v8cQ8QyZ8t!@d-*h-
z^>iAU5W>Uv(FE?JixHkadp`TFP{d@#+<3sBaH>G~o@B?sr!;cWak5t{9!HEy9Dm`y
zLoavd&xH!H>*TyIx}(B3ywLVC54esqEGwd7J`~{
z(KNuRWk;OE3hFaNKg>IJ0Qx%i4yiA2MSzNokDc%_$jZ*hKUQPXm!8gb$#QrUP*aG$1PiFavSZKZ7Qeo!C-hcU}QYC!lQb%(wmG
zc|5=`030U>YoE+>(MqMXl9Dkoc|s+P#Rg_N!OG8I~QYEoaZPdi_v#U3GDkyjFc1Ww5$RIQ3MuIKtL5A
z3*-c_TEEu8v=qw_q^rV5iHI%yfNr?Jq@{KESqKPuz`7yf8nGx&K2MTF;NJndl0O2w
zeJL!)RNqC8gB!G<8Bdo;Qy(GI^Rr&sQ657H3D!wB&YY1#&PX
z-H<qYC=4^gWj#S@zmM<^l=j@SEd&by6I
z#O)LhyvY|e3SZ39i^;h0<
z0pl=v&Z0ZF^_B1TaEbk%OBQCc;OaEiOOtd`Q}4Jd!(I{_a-h^rxQ-L3BFH4+yMfG<
zprNPtLZXL-g||xuaDjLp-Wp^pqA1Z8-t0fOnzK4sK;oSV8`ULKa^%?7XmGKm>z?J80~N9FR-QCF_Ql4v)~I#vhoL
zf@lh{Ud<4e>DcVJkAFQ(_y88TEdXmnKYu|;yRb#Z`t_ezOE~HS=E4%}q1e;=GpImW
zP0uL<&Jt2Whr_U~#Yp1NAsRcZm!SxkI4A>%NLRm6d-aVBMRKa-*S6_cdovnF2B_Y@_&vc2k(bX&%2F_S3B;R
z?ydB#?_}MwTlaLQoM3(zw_b~FCI*LDSNQ?2&%hL`1OwY8`I
zn*1s%&hLNea*O62xoDPgo12@|!F^v+CvEidB^ucMx;U>N$Q_dTu+SW>EYxG~_ku@)
zC~j?^xOD(31H8X4)RQ;XXD$p4*-2cBdHns8W!TM+%1#9`(xruIZTWQ)*KTxn-y5PL
z9gq|!87E_r_+-x(0_D+>BS)Z+=*Tg(7h<#f{u4%VEb?##0Xm6DDz^pxJsg!+L{NP;
zG|(XSiETS0WmO~yV$Nz3Oo1VITP2REpg4LPu~Jb%^r%EaZm$K4wytx
zuKoEl`Rf;UPZ!f2A%XjXBH5CLL<3K*t3Db7i{FUbsSBpiQmj20VB!yN7;
zWsJclz=4=M0jdy$OCF44R2^9CK?uqe1SWE`rL^N1V_5wMJm|m(u(9DKj3XhY3*!l3;*xG!J(mKZGLY0h1-T2>jo|a6o`AUE=#e9a`PLVloC*+|jIxC!BfAcP%k7^ZeT_r@
zpF1Y;q`Ay)0s0C!<{*2*d1i5*=70_*KfwFAB2zQXn7s4M)r0!
z_PiHQ3kS7((wnN(!kd+=&Int7pfh}YliT*$Pr2t5N(Mf$SE%do?gr9NiN)Oe`H8Nm
z7>)1jU#{Vg8Vd^tAU{fy6&v#pjvW^kYm((}lC}B1i|m$ybAy~lLe2C73M_BtNGVXK
zSN{A;OiaXX7BFx8Ror1YCS@xtgQ!7J)ihk0M+7pEI9UIgc+AxvJd3=<$M~laiG)l&
z?d0?EvhIseR|EwGp_fC48leW`dR`aOKt5aJ1d({hmD4eSIEwgss!$c&eASDv#iG
zF_5}B{QQW{qz7zGMFM;eaG5bo46Cn6sOevH@Z*-@1lo;(z#kRSRPYR91w#;69IZ})
zAcuf0{KQyDE32rG6Hn9*qS{3H?}A%XKzfgtkd&8{u<%VN&LQ7J=A@h_)W^`Z?4Y@e
zVbPK@x!uo^kyHC04bo6!Q(>M92_B^R23hF$Vm_dL&&{~fH#RaGBLXI@vBdJihpmzPZ$-FCH^70~_5KV*Zu-oNwZle)bwiS%l~x
z^pYN(QaiIHoDlJ-tR@yb7ai&!!0Z9U(_v<(+oQMY__->xQqD&^PumJg=Fslvc_*|G
zaJA}#Q31VkoM--q#9I`!QVDV*BIbnyoppd1ida+05)IIL$p%xVRtzLwI{j3%*g*d+TUMW{>J-M5v^|YlL>r65=pyMUdH;
zl|@IO$%fbDw!PPtE0oS;KqERZ3lKQwemP1J1omr;H28qU*`V>ByulzX0PGuDv%;Qz
zG)Pp@=5O!o^TfZ{F9|W^?7Xv!3lx?YdBd|Vb6cy=jM-2YN`}S4!^%bzkgA#1;&CklW@`Snp@5s_P{D&XgVSO|KXye{M|kP+wkBM=gROZCMID!EW}EqL(4
z>Y+T_LqGa;3Agi@qg4D(iYJrLJ2cGSNIk9??MiAQ`=6GVm!zh?
zZ)p+My(Fg*KZf9I%cnnNL`Rah_|h639pHKYdb-KkTdYp=NBJuy2*5fzN{4J&(!{_nhtt$RSb6kGMKEC+Ufs2DBt8jxLWC=(IdBYbCN}(5|eb
z$DV?KCGg}d@;mkP^w6rq)q+H*#>RUtAFUs4=M(x8J4cI;$??YlZ{80jT4}_=KhiMDGL!W^W3eRql_%8@_
z0EwUYIE2=HzWD})81Of|(Xc~73yf+E=Y59-^oJ1EO&~si$m+O$9cw5YVMAi%2;k<~
zjhhn14P>xn#~*ZCL6nB01f?8mejGvnSp`1WNa_hK{~q5ye-I8s$c8aa!mWa5pP%{Y
zfdi=+IoTA$LQtwV(A(QIY=;*a?GqTvVD1oAjvdzxE9P5d@Vr0^=VZq8{w~famQb81
zh`QwA`Gv+D=c^nym~KazX$rgPuJbTblRK0WW3n*-x|p%Hx~gAdmx|ov(7KtyDp~B1
zg~&lEfY1Wk&?E%mJMzI7qN-GTLHg$i))
zVq%vC4QGHjG2Isj=0{Iy;Xc25>KCM!1IOy)!lSc>VmoY_nZ_!-7N;jtj~?YzT_!DK
zDRWng-h<~ODai{(1mFO0SdGo|^@PB}Xz1PjwL9Bwdw4akW9u$U{{#DvChYl4u)0$E
zVS_q6H1rIL*Y*rx+-u6F(lK^`3iYWv33^N0Crs4-7@MI<+=g5L*AN~dgw_Y5373(%|!gg~={zc;vUZ{sk3IJxX&7&LP(89N>d2}ZY&F7XD
zBrw7UN9<=Eetp6hirq94o$+~c&{LHwGZ1`lQy-^*?6-XhF%&+?^gui`8%nug1u#!h
z>_7sB4pIW8V?r|!9kg_G{$lgE0^x;_1#KWB)&L>k-zOJe&+qO#z`;*f25A>Y9~g?$
z1OTTBH$jb&6xU9SI*3fTg6=8&jG-{9!b0Bjj3K&4{JdY~=Hg97mH`e-9EU)XFOBPf
z_XWYUWqyTXs`^KhzmQ%U&;Oq2$U#UIAki5aqr~;mjvc5Bko!SIU8J-s!FWUWGCjAg
z!?EN9^^Q;jd=wz&ZYdp>o!MK~PoI(|oe$x}%BMZJyVd!r??cRpNs389i`iT~12)9^
z&TWk{v*lv81Le=3^ISjPyBF9sU?>u7D_e#iTlKTKEbs5J`if-ksUK}q#-ZMaM`I4l
ztLPp387`s|zVl6BzDm_r#>{OFO4Vgb)~^0no6FaRvaKGXV2X|&nJ67R0G5+Q5so=l
zk7e(TFJh_3t4^8m5g32*3q)6J>D;*!C2GKE_&U3~dH)VD?*e8IjmD^k%BR$muA1
zX^%5=$_`kI4J{uVEI#NIkn`lj%xkVYr~Y9hdGF3Wgx>7ie3CWxWot{AU}JgfsO+}q
zbklLe^*v|{(1A%oOAVFYq{{ZdQsVRy#?|f7*3KJF-!fXD1<#q+r9vUz0^E<`EqL;`yA>a
zVkigq?qEzih6bC72RJh`vIzKic@dud>>2l1p!=08yY9)M6%RDUiw*0Fk&#@k7Rto5
zwm+ahNpY#66~xrg(-i-Uh_rG;`pHe75IrH=J9jXNJ8+;X>p-B`em7TFNX34%|8~a?
zdBt<9zpNWuJ)oB%ecZJRPlPl06YcGVz`{!=>||$9gAz~|`U;)gz>46lw-*X?PyoDM
zW+f1=*xQmikPf-ymcZ!dk6$^C1rVtSayTLb&Z3IkTpddp`S}w%)*(hNf*f8{HMIzJ
ze!U9!qMXBf4J(nU*wdp^rl0uqGehjprBsU8pSIdXq!3$rT~N
zwf=lAdMRqD=*p}9&^<32-OB+zCfJoY(`#fefac!x<%?d4`dg>$*rAYaQ&;!SJHR%xl4>6Z*=rd&DLMWU^5S38fcoj#+_VjiK^}Y
z0*zBF%Yz|eNol1bNQ&O;x4BMq>m5CLa^D$eU#7?pYcC4-#az6kqT;wUPA>Goqxn2e
zA&>Wl+bo=s;9cVvr`LY9A+a?Y%WQRM
zLrsL6*`+ss
zdn_~1q?frA(=GkToQp2U_iTG2s6J~iv$f`_iNE}FgQWz=Dy|IT+z;Y=fk`Vx
z%9SR=gKef`V{g4YMmlrhcHSSv`|rgmI(H2#y**2NlA-;5w`rA-7Oo~-DO|T@_LM~C
zNXi?AoulWdPP4IE5_zl!J{X`Nt-a&=5zjMjUvkd%k!O4>zoqgQR9|uC^)G?7jcGT#
zygfK$jKDIvD;AC0)Y31a?bI56@t`VHs+799vl-
z0fNtz=936=)wZ5&c|MuU~a;sp@0%1VQyF*q?y3x1%P
zg6#?UUEZr)yi9>#)pt{hQD)DyB#v*6XE~nqq=L35M%eMDd@A*=`aP6}Eaf00Mtg$+Q5Y*(efe0B&GOx8ekIH1+tYmq7Xvj&D
zTIxSKB&)cJ_%o3PJt_+UUyH>}MYv(A?t_P8{PPG#Y7L2Z;})T#L*#a4=hhu!~WZJi1gf=JCphD-TCw@(AlI!9Vjp46l8hvOzu&N
z?DxU^tX(gBLS0=7x-Rm1KYDVd|4QYE`ba99@=ki|AIy`k-ZXYueO~#|`%AbpY7gw9
zX;=x$%29!$<$y`B=TX_VL-s1)SgtqGK+nB4Ofv1;_w`7c?g6R`^OOR7*AM?3Y_uX-
zt;kD8Tj`4r4U%_`NF~`NZSH%*_8!|uV@5YtE2n!C{*d>i#sqD~-xJ*3RHS16dTZ$6
zyVXW*&d@yeXW?aek~I^E^LoFY9;i#Awk;c#cFCL9Y;+ed8Gsj}MlOn&@w9c{9}Y9M
z6QYpHK6|d#w&O6ke^``ZAE+610rT)3#1Sk)O7otWdb%OgAkU!LXXa#n0Bwa-qRgtv
zqozw9OK<7u*+j)WUTcf9eej6B&zHMYwfg;c;!L8!rTlI)Z^>O&5mW|DY(M{qhJ@x5
zzE;_%A5V%Hb7&A<^im0ZQ(*6Zd9-29^bh&_lwO6H){0%uK^0cdUU@#-(C9iGt~_t?
z(k)q5s+-i6nfuv$JA)IZ7pRz{pV8TY@>UGV)emhk(?H**!x5=LKT^JRWVmf_PRvNQ
zk6bn5qu#w^FRSH+hJ?uLDj{PlDq4a2*suLD{!*k;03(SE|2J_=jR3Oh3JLV}6*lIk
zto}vXLQQU4Tz!}f(~u3cKz)Ez;lf381ZF`{t$cdhas|mlAYhH+n3zguW?Wt{M})<>
zOj}SA78b{uDoi+Oy$(Ly1A~H%@BTL!6O0sOn$o2Iv~7HLiR6(&^2NiOo|)gn}xXl_h?p(W{0uf5-f}Cry`NfN&gJbMFzH-(#HA0oS3?&h!kM?duU_gMG7
z^l%3o>CE}ZqP1e1QLmLKH1FjcE@Me=Ua_prH+<}wDq{J<^)(Omkg%^z@Jr3`@R?fuGwa(Y9Xf+|EczH<2XEsC
zxs5lB9^C8%!r6w*xBXHkZ=PKe3YVOiojoE{S$%UtOthT!*uST*FDMjfdko1bolo`l
zuHL9RylEZMz>;D4(#K}w-Xf$KuNm5z=MmRRUd(C)!lgkiA$>A}Cy1!H9Ld%m@bd-tN)ZN*Rp
zm!A~*?KeV0Gk}5(tTQ@g&Z+xo>-XgECLUq^V|_(&)4z3;r*GG#NHjkEBxU}DYe++G
z<@t4*h7bs=Z{ONH$o$l-W#RAN(&?8^d=%vG{pz(%0NKot$9*CAVKdHJh{Wn{?W*QY
z$kx%=Jl{BGakVgo{qDOI>H2;(`MyZod*JZHA+WagCV;k|MR1OeI)OvCsA~GCTOdo5uaGaHr;I$pBNcBZs)TPd($N^qt%{{Yp7i9dt2p55mR2H@}8+`stsel^9R`duth0i;5$#;!@PC9l>Q)ls?47Ke+cUNOq#
zay>z2H}icyD3EFZ4ksTHZxFDYTzCBduoqp`R
zkNtaZekR?(qK9$ki{xq*;}n
z4n@zNDQXnv4#}#fnHqENBQOz`lb2YiejL;KN-fSL>!hM`?)Ggj?KoYBe;-TBK{_4q
zsd_1BcKJS_&K~nnS+N?ElJh~=xwFDAQQyQ|NytgR&
zX;;$QOhwZK-T~rOSUmV%VzmePInXD~cF@R9ExEN5hO~sh2wKb6gvBkS%x(5S3^bRmodAf1S
z=&IH~x=oppN2l9(ut1W}%3yandFWGx-+A3OdC{HIuYQ!~FDadPWG(M&@{~1hnCwBc
zdBjMlVmdBa7f3<1@0u6OtgSE7Q@JkpXUt5okb#XBWVz!#!6`p_mHff!d^t8+HxQz2#CEvy6hWhp$
zOso!bO8?RSu1l&X{t>C>(V4BR?ae?YS5?&@G1HlwdII!zTiflFeP@x|c1%zZIjQN#
z;ZrGk`?lJrXL<>_M7SVvp0YACW6M_7KV#JmG{N`pmO)up9h$6tE~c}F1Q(aPm4=R8oYN>=8fJQ*M9v8x$32hbS+2dmf6zE965W|
zD}AntWO+OL!^8ozFn_Yld?KsgoYg_IpB3BTio^{uC>iMIUwtPIGR4Fk7w9C|nlV$G
zL+>rqp%C2&U+DsVe4Hm8$SV)VHdVoE=Cd`Q(h$kzGv9myVrYsJlJTn0C1Nv0MbF`@
z`_oQ}KB&-T)*wl347&{apbm>7ZNb;4u^0kN8lMn1_5I~prB>(w;WS8SMj#tEH5}Cl
z4XvE9fCe?B^yKm)T@`LJ1k!vrNUPn8Tc$uTP)2yEO5jBah{e4o5Y~$!BQ)opqhV
zp9fy1DD;&7rR|w7&0#e4W-L+$6Hl~#-vVgQL!L3!7hi7@UehQ_elkokA+nW(%KvF?
zf*OfX-Xi76XXu~m)Ej@3_pFQ61X1KAwIp2?ppa(Wug@X%I;&{$XWDtD>-6c*&~)xenc~W}l5NNBqFZLgw`3EjKQS5jSEKX9$9?oDEIgzHbWTKf
zB3`1~DPEo3$W6uK2)!rw(W9T;M6e_uYe|Co5}Fvlb>cJ>DzS?sonqZ#b9L#Bn}3|aHC~t?59_P&IvOyLbb@tQz3OGAe}*U
zhMfckv4SjkjPJ=|!J4VgkI4$$G&@re=d5<|;s+yPe8xx<#4=%F0lBF-P=46c-hYPH
zG+UEq><}jDuSq1kIm~y}Bd$m;j_3}(Wo9NRv1j&&h;_H9Ep~Dqz3jC`MKsgY50)R$
zK3C|UsbG@pmg`=3n;i>r%yrJ$^QL~TriWj+c6~;M^0st!q(mOeZ0EzKQ3!TMM@v1H
zzGzqW1zp@vmXTR}eR(41aF*I)&ve9M+vJwSreUS3VOPj%XG}5NBBvf$FFkK8`BmWd
znSpI_aX;b3!GpXu(=;?ZEE0d$GX`G;YmZor`<%0FOPQ78dw$O+v-Fwd1BeLjPJGAe
z8u`Km0#+(S$=aX2VmNt0V1=N-rvLpSo{yGEhSLjQAzeT`!XeinUEXLsX5bSb*o!zx
zT)`HL{Kv8vY}lAR6N4UxXqJWlm#N1pqzBwS5s3zMUaC5d=*ZJcKcJCZbl*g0*W1$r
zDIj*~?_uj80zPhvuEL%Qci->7uCQHVSp}^P)<-#7xo}di1H}vFtDdf|N>)2DAPSy%
z3&0Ja&9NwYdVwbyJ5wX0p^D`JSPoGWV@N&?m-@q)7<|DzF&DvGQ02umzpva4=DC4)
z)==gAoL_~mN3D1TP?&TiHegR`ZvF{588AD}RX~Sx$cjJ=Iutr`1mp#DVEXT@x+glS
zW-lYQ8k)`hGS0w0U~_6v>TvkvNjI3P5zvW8HluwUpp#rJOg;!iqd7680wYW;+JMKx
zPGfF$1G6_6tMIcu2gzt_bMt08w_Jht1`h2<7kK`I`mcDN@vC|9>Xlxl=c?@1Uk14G
z+myIQ$HtHk7_9IpDyqP^78|>70|PU&v$Aa|r~LmC9a_8;*nmTO+FQ1qusR>0^2Bv8
z{_>}p{$}*+;$JDOBQEK)m4E3F?f2b3Dg)<;38S6kef!3P(X3<`;`Ts8*Z2?>kPPwo(ygylW50czB&d>>3qkn33tzbbHg(40J#O!+;d=pSGkVt~gz+<-J1oG>#F(~G!I7B~)b;K6dtBk=4!13`VyB@?M
z2aog|XSqQ>jEw6L^5H0gG!o-a(4w6dCpz#)SPeZzjvUE`rVGh+nVEr2qwk7~q0adq
z#Yp1BiT^y27~LaP05JlbksCNgD3}*1E$1a0U&uZ5zBp{SlRWdLQEL2>jhSZlmMP4Xcd@d0bhr;
z8k!fxgCJWK4pA(E0Vt&pSA@*&TYvv%;N7#vtcL_y5Wc&Qt{(1n26k7FP2fHUXaL~b
znCk|?VPRG_Hbknr$C8z{_K#1W9?aWO{;%B9Q-+eRx>^pO7Iugj*8Fa{Rk3O|m2@Nk
znj35x`ubjl7}Y7*9Xe7Xlf7RuZ|c>L8-0d?&m58yO!W8hW{r^0idw)OXhx&x(yMk+10f
zvyD%(%j@EsO`LxQ=={Clrsul)`iPpmL$9=d-;s|ad>@`PHL+V*UTqM`T)%Rbx3*ZR
z==oacqvIFXL+}3>ns@U%c-{XXgb&%8Y`b>JAB?j3Lyo;W;B4gbU!e%arrYLQOzrU3
z$wh3-(j(7+IfnSwAEl02>il655tYUsQ0I^Q#RHJ38c(FMKmzo!u8x9|5@!LTwedXR
z_WREl`*VDJhs*>Pu#TmASc+rqcKPb7UJJJkiZeti
z86ib4UTwR!ZcnfR2?@!;s0R%rqhL>%nIX&@j~op-DXKuc1S{Xq^6bvlE=Yq`5|AaZ
zFcCs5B7(q{4>fdyU)Mo+)ev$VXt55TZ%*Fa!`%2kT$;O=0YzbK+`lJV?LRewQL
zEPefo9usdEcE?bOV#3~G(e(NASMvqTBv3x+k^aFEg?tt>X!HQcfQAnE3*(>{3Dy4*
zeetqDfv3*zAD#jkBfc-_$i9C^pcVKf{LB{>6vzY4mM%`6qh4l&bz|rvWTB7?F1p9m
zd}ZQ{h(E3CM0I1EgfQsUrJ-|lKd0oD#8N=Vt{nZDE&J1DN1dnh`m77G=k_MBw+%JN
z&G}D6`{6zQ`BPwg`~yTqE`z$)Vm8LS_FIQ5z6q$}HMvyIy*0C5?!gww9wD)R^4f`9
zY6{Nf^EJE_!nMbF1zUdM=^t}A4EGo%Rip=9UgOQ}pP_3^eD2iz-_{=N+*0kzia5r_
zzYQJk)H|yKmo{ge5@nmF9^JvRRqEESKVOoGq4a=G0tJPm6a>1gtlQH)MZ^UzXb=*|
z0LB602`WBBveO5_EwG7}=Ve=;V1Xb4xU~gA6|83teHrzu~ham|hgPKS0N^#Ipn!a_iqtwW%Jsouc
zM(_XR5=bBe*0IJ2lN160P><+4V5H%3H4=`)2aJ*2WV<_TB4<)sNB$1!BC3dqi|aT`
zs1mJ8dfQNAtphq`bP=aqLGZ&MDDdBhjJ~EY?&44X}z$c=!
zxD5-jKzzuBa$mia(z&B`llss{XOg*&hcyGM>M&cx~Y_Y7BO%##5Rqn^0TSnaRLh_5$~LcvY1%l0AOHKT>SQJB9Jeru$()KICetE
z@!)|J^qh%KKCZ4mvR@4{{%7>Sn}WUz3M`KI?|rttQ5h(0{Q-I}J9|egqyCyKo(htvUANWqu9QiK_j#s6WXN+yC=s!#IUj`+ko-z(h
zOG^Rz+cZ0OCW@TMJP^iOx4dh{o#AOizl#KKqK&NAtMRz{KCx>z
z8Ajf_OD^*w**mHB0IJ5!w)+08OZmk~XZkV5{2(=aSz#mK@~#qvU-(Ok_Y69RIS~lV
zis!p7u%CEupP!FUOW`0AJ9wjQIcN|K_3iuj
z$4N>0T3VE4zgtseQ<}}-k~1uI7=>JBczzYSc7A4PW*`A&2}LNM+o@!P(zcHS`w1Xq
zpyQ*6mJjhUF&&+q=z`#>!F!1CYuwuhs6Ut#;19eeK>*wVYBmlI&~C5^1^LF#(b4bF
zVGL{%F>H;Bx`H|H#)>^>WLH-gA0zdUseghnt}r}5Q2e92pLCLj7KWMTj<^mK?rv_Y
zD9T|>hj|LRd!sQf`1gMN_<_g$)hof0E#y^qzqu7!Sc`b{&9-kfsz|X$)FXm}JkR(5
z4q9g-&-lE<0y_L*O6jB-Fm5o$ypBa8dWaaX&yrDbEnph40YTA@fMuO~wN#vF(d_oU
zUIMWX;wgi|`fYj(Z~2Gm8RzjQcUA|6@A7YEh?{LrRA|&fY-!_@=o`5=XleE+Q(kGdKJE;WR&d2!fb?kvpqLOubQT79Ak1_xkNJ!(7K7skK-=-GhLlu7Glmv
z#D|ZVNGUK!`)b7VecA-yMJznLwCPDf?T5G#>4yp?pIT#y`7SZ{=DcLQ-oOrsecK2rzSGpCI#A7WIOP^-3OvRD|V=y?hvhqC@73(D$
zM-Y4n3ZQA^Y3ED9-H;m9dY^$59MN-|Ym4Ze;8PeJ^nrAYv}X1v2(9Gg8#Xfl(I7UN
z7g>T04WRzO(eR?Di;*jP?{uiI^nrR
zU4UP}!C>OV7j?nzq6e%MC`Y*&h4-qOm;g8U;6F1F#y!`vHSJy|q9laN6hK&>-KbHC
z`ySi~;%vZehSohr*@}A&F9N7*U?AsoD5i@5-rC=vgl*UA1~Lobo&!)D?v86YT6p@7vvYbEWVygOHJia$Yho2CcuXpZb
z_uH`21i*-gyrng_Co7#
z(Q$NTN1k>8X)+JecS1uCF;Zi)bw?7P3^*sa&lJX9K@`Z#Y?K;i=Y6NKIaplSOxw?Lo+TE0Ob44!F
zpNxJV|E22D9$5mxuSQVk?H@!mLBDjDjeyUxyfPm7@OUmIy0R{BSM25bFZ-wlA2FUH
zkhufS?wj^x7%TC=x9NjXPO(+@&*|w3*e|j*EfD^8DjA*`f^YLy?q@110y$(aK6qF$
zVS@V)cc@vR@reXH>M~g((yL~uoL*7k7s`-B!zV_bS@YQ9?(!NOQ$RQ=frrRc`YpDxH1
z?MlgT2clk=8jW()Le>)ACXkRI%j@soAMq6~iGacI;9z_Asq)RG-Y(g%mdQlL7OZtU
zcLr>K!9a_a7BEgcTNn&j6h#Qtt7d3GGfY#3<}hoRl@@ldxX6S5Gtmn@%glt|N36S>
zN|~qb-{_UAnSH4_{{8f~ZZEq(^m?+I=PvbY7w4@vfApYTNSXBBO8Z4C$QCm^vQ*4-
zEuZpdc*2=>9S|f2pk2KWTE2N2NQ^$Pn`yq9v)
z@0Z0Y;g>JvTQZ4r60z@FORg%Ny7SflS8hD@FUn_yeO`FW`f)L?-BeN<5o?`uE0_68
zS$@~u{cU&m&wBT3Z>xF*!T);Yc=I*e`^WkxPoJckV8G_U;q|wmJH;}DeN^Ad7ZRA{
zzG^>yos*jj>s9&U(p=ugVi!Wa9UT6FfCevCzfDz>kvf{DQHRn#+0_q_$Y0DrFsv|O
zT1$dKUOF}u^g)j2xSmh$^U}UoV}v74qSR){8*S}{1=)SHus5oIEsFXHTk`nPk?&pk
z!3|)kd2UVDmG-~h5&~W?LgF{Pelk=Q1LL%4RNdZF3k^Y8+)C7gJzW2Ku0UDYMVq?I
z=71tSPX)Ud+%5!xe
zDEz#%@b5nbOT%CViCt)Q#3JOk&n-tTGBGh>m2m7S6QVLX&}<2}J*fK$G%iiudvC9y
zja8SDaAV0{OG~8z_nbuE+T2is6K-#Pif-sjXx@E0ZWo#+?Mq=0N17Z#^@pk?Yz?N;
zNY2v}v$p#0zO2s_`Q&z56w3F$>mV}P_wIcoe5?FjjSX!=|C8e_*W)!wUz&sjHPe$!
zZ2n{faANndI6_A2P0?Go_UGo?a!0AVsFQB6i0rijfnAcjn};}=Pifh7+6W*c5CX1T
z=`qwfq1ScL7P&2S0i>Fz;znud&ur+5Jd|jtw|d2D;A(8eH#;el8XWW2b>ojK;kvz-
z=v8w6^&P}%EpfS_C*(5{enRu*FmaXK3b7IZu_FjaIinuIQqxmumxe0mpP+k
z&Yy(DJ}qc@85?@{8(!M7;r?}|UqpdSSS_R!7$>&fW22k@zMey2W>iq$PUr^9H~UaeK^jP+z-{)gZkB6XkO
zzXQq#v~kz~ECFop_`3w5cFkjh&OKavsU1Gu%*;7?PW7Oehw=cq((LB&D-QMaS0ZJ{
z3XMz5@?@np<8t`Kcf8jAf;oTT?W>4wO;9+!y86g#y@zLat>EAd6~8Bna|iYKlMrHg
zR;%D*`RK+IvFY1s{S>ZYbiV7t%>uwZa5xd_51pqcmp^?v31n9>n4bz;%@~76?hY6g
zorN9-8SmzIv@S2lZ~gqv+Thr{6+}F?GN;(HD+a-`9fZ6h@{Ej}%)9}={pYx$^k(0^oAmY`>^l7)K9p&tZ2v0!Z@}Z4IO%j_2_Ij*
z+p^Y+=uR)a75mVQx%6RQkD$!v&3Y=X@+;S`uYNiN0gCC+8Ux`|E7zloI|;r4_h@gg
zH#_G#oqM0^RkS|P=+$6ROiFlk`Y_Lwf6V73^{2iXbQ~j(Yu^FMdX
zGSkyh?&V=>ne+61mcqY-(F)2V(tQ$NcU1e)Cx?(R1Vxlx9umk`hzz6ao6(=_^b*C|
zjj5EE$N>W8n~|3^<|4&d6oK$1Nx;GJP@jEIc#XW5S!PM)_kUI&3T@SV93Sl^fPKlu
z;)$uK#ksJ}lskF$YYk-LHszk;GmHtBVjfn{{Y#5begse)Erve352;$Cn`(Nt6e3O@oYKYPjXbdFYD`rDBbwwi{Pa6fXqgmK0~2G>G$+x`$+rc=$U7~SygTG
zXR=ayoM+7$OzJm}t=fv0)4p8Ny9}*G5Cs>@a*B^=%YwshBSw8;ssj|aGSb&2qWSw)
zFkRan$_~bVwT#+=_*0+FoS%Q%@@8zT#rbTm_GRuEsNCN?zSJ%w;H*Wb6*p~0!^!oT
zLRJ>qJ<^HIebYPT`Ts0!D%(V1`razLo!DIOz0k%?XB;_RPSUZy&gyk#$d;Xcy)Yq~;9$9?rn4m>#7nS-E2JR$Z!+K@+$JyFU_to2W^7
zDU^1pH&lG;YzW|D&#nKQdND?MLo;zLcrVq@eGj0zb0eN1_DZ>0P^cRVzh53mVBJ;o
z-qiMC)3^P8$a`2Wu`IAGyzqFZ){qzemYOdh|7TMB;9z9!Fcd9Vv=d%v9u2A`VJ+K#
z2{tl>X+4R&h7_6rQokBdGNhBdF9l(*YGHUzLy(R>m*@jXJDKG{eZ8f7~_9PFCckCZ+%SE(8X{5QH8DT8|S0
z4{j8F-nMbidG$C&=Ct`EVY9U_YRk$lx#qkHzxmGq)g18M;Y=
zZ{tmABnmsWo>t=S)41{g_);xm63-V%#Q|L{?8Xlk&US3NwFQyRjqm})9AJEibq_hI
zF(yO!o8dAA1pKQ-5tcm|{$Hac5)c5dM&JOSnN*TmS7#?BG3N{a7=t%7AwVGx+dYnr
z#V!lMlxR#w9d?lo{f7gr$3m(T+r!aczvfmyjCmxku)
z76Iw;MfBL$T!Nyk>GYq7PM02Oy;MFZ`^#5nPcB9-TAz9{rf7!ccm(5SHlLYqpZAhW
zV~s3pK_0Au|{J7A_x&_z%&aH^`R>LJz`vakYe
z2=g(UIvDlITpWS)h=4{|!_7DUL3&W9ZTd_l-8Is?)>++JV#zCMzg*v*X+7~az&(_i
zzeT9!gAgs(yOLwCosQ7!vS3b;C6ry)ug@nMA0M{pkZSIc1p`F
z3Ck^|%MW5%qZ@gsP0Y>B!TLhLP$E#97c)5M8^)dBL<!CA6crmzJQGH4ie!-BLX~
z1sL4nO_1H*a71tvT*+vlvNfT4(8{@%--U+)qdw8gNjkaC(n;W({0Et(tDSN#YZ`@6
z2!oC~6Re3I-o|Et`TVNq8R&S9UgKsY5bkqJC$&sFudl4&)y0C{xopwBqG=Q>?b^Yaa)Bi&Y+ZRSXF;mB_v2b&XggvWRXuqF$w`!MqNqlrkw
z2&OEcj7rXtr{tt`m9z7we?XBcKz4NydHf#JnxZ?ksKeHp1Qz0iIGNNEI{7%2pGd(%o}@9Wi5Moxkqx0(XMkuA
z_wuDnJ8`EVMS+xGS(Tf!OU}lsV2>*oko>WM$M;
zrU(848WkWKzFSD7v|>7mv?c8E`2_@^;RhEO{&zV0%?fX%w_)svh;0OXaWq^zdQC}5
z3Heyq+;X}OkvOESczRHwwza=FQb_4+2xV*&iI_3ROmoW+NIFF>YPa*Ox5{6YSxpmhu1^LiP^V~;AZ3*
zX^*hnqPKD08wIr%JaT(Vyq9iTZkxrfIxUY!s>mh}pGdcQ7N&N-&dTk5)#MTnkJpFj
z22~L$lA{8Qgl+!7^!-K-_sFYhwe_CesG$_@WUqYtR`Xz#_=yvaQcp5eZin5vGjtRv
z1?6Dr6W80mnFlY>R|vSZ`&(=izTTec?S4IP*IP+$D<4vI=ps$@lfRB<<*zJY0$M!a
zE&vmr?-cQuRaUz)3wY_FbsFmGuo=4{{>NspGU*im?qDJZfA{YD#&HPN=(fU5XjX_I
z6Hu-C+{TiK85c8YfF8>dHz}=2GO?NsB)&C9Uyw=;@y5cxe``=7VF?SR3Dj=baljSK
z13EI
z9(SCHc%p#mf`uJh4jjP%el1|Ffq^#u3S9rtNqsaaNJ%k7U=%!vPmUXMGBVoXVT784
zm^A>eF099RGNH;s8U@zNc%^^`!R|^QC$9|#GYaQPsVgTVAKKT~UI@Fy$8-E)hDpa{};YXzx
z(sr6#T6p=0q}6*=oRA2ttxe{4<>`Y5zt!rq<%lm6f;M=Q7JziN^!wreydyVDpD$e#m?u?pPfNO
zykp>(o{^Q61#|!D(+e#!>AQFHn3^}FY4(Vm=m#QBT@XcLiteP(57R$9MzbGacb<4D>5>Dl6r
zuE}QU`y&esztrU4^95Y_dt>$XZTFVMYl(ly{K9Cd2j;uF=VUs!`uE1^6JXXVwSHDu
zJ8tv3?1d(`dbJ35oSZ3%^%SL?ib@k4>N@=*9`9@&{`vc#I!Hb6&BSvw=JOkk>H04*
zHD9C-oI7QIHB0!&nyOdPM32L*YzBskLydFaA9uXidz<<`YI8gq018RHT{gjujKwmN
zCTyxUo#}(b^aex_Q6VU(nvWe^SBk8ULAIvQjW^GMF-Mk#OP@VeY*R*?&?#vQ{NTnH
zp-VVnYhY=aW021m**=Z|`2s#=OcS1}X3B01GeB==fq$Bt3psYx@&FR)<^erKSbUSN
zZ3U4h>50HBPOpV4u}yco??GLFr}tZ6so>r~V924yZXzJJ5Cd+&bK)L75`{zv)dZzq
zTW6;vD0WaVIhO^T9X8xvyUrHQ8*{M=Q}fy&hWXVE#BuP8i2NO@@q2C!4hWuo5jqdR
zjg(`Pi4e-n1e=2~i!iK$3+Qkaq|V93HFld4W8kxljg5&`%>RH|#X7lIFaab7Q6nmD
zQ=HO3abY5H0(ll1zhQ_F<>5h|9~3ZoHwt+rC6}NXoSd5CkzHI`@&
z6bzWCnfH@<(2RcnuK7>qqO$T4etuT~5i-{l=Y=I)>3qinfUHMFLrV*J)-sf&m8-uK
z5dCb^^L7W)cM-knqpiI+w#nMc3fADCSN6EaJ828*DEQk6_EhOiyUtcT^;&n|t9FMV
z>r2};_CpIX>Rf~h4TyWO`(Ak@oy=9~-P;OTg>f>LY6*eljodv~$>(dbd0
zdPL%M^y6o44e9v%s*-H%9Mc4LQhv&@vg}*^bj)P?zJ?7skp8BmqxeAj>BnY2>c|>D
zOtweifHZMh@E6>+ENZC@T05!|9y}0icx<5Mmg>-ldyRM2UaX}1JlSF*luOxJ9^9~b
z3$V1kUQP!DyCV7Oe8iRFr>H~J_x%F`kh9cl^$xD_U1R{9y>!S)??K8WEQl&VP%x}^81ZJn
z-@rkQG+2V~f1#|0_ECeN3a|A*5jU<|D0`dD$b2=ABFSaUp;Rp^x^?U-KQr!BXhpTO
zK(r*r0pFGrWZ~oCfqc2R_magNZ0vy9KxsorOHWtM(nDL--yfdFf8;1J*8_4^qOP7O
zrWO!TQB@5K34!5(nFjnLWFE}S*x)Jn=5_}(Fv1*a@V~2`kT(#ZhR0?Q{46pAVX`nW
zA(HS@G-R@kpMD!3ILV>*ALD_q;wARls9A$*OL?Xr1#R1(qO4xH-!uw^7pfFsFDU&0
zU0tkr_|^uL9n5lJyg6+Z+&FEv=g9>(nj#=a|GytIGCGXL64U-$Wp4{38ZE;BZeD5Qr&|u+Hrjb_tW)_Gh7=lJ^UiQFXAcV5?I|bZdwY!F$Tl^v
z%1=#CbI2;kFU^1^)`*;J+lJbNq=@nRPR%6N8kSwJykCc_7D;G^VvJDU-2`vlmN%#wnACEj$_~y+&oHmdULn#j%
zM74b%T4D0Gt}d59Uk||J^7D~375w~AI6181Hs5hyj`1cgSUga%P0e4vYz+A^d@&Ql
z>8g=UE)R)1k=%#-4omON#lsLy;zSrgxeJ9aG*jzqYkA+%k2lzJG}sg5t~_2aNz=%_
z>Cahe6v=%7n^COIq4SMIj1!*o{xTQXDXjZSu4HI5jyezlOi~fiS1$!>KiqzF%ye&H
z+Mi#RbHBcR+Rx5>9vmp3kr@3I)5C?&)%J4|-x2J|AM4~MTSVnwPz{qhGA_LneOv^`_+2S=bh&ybnssHaVbNRe
zw$
zmmKw|>@ZK+;SRT!)I!M-!jF$5-}NtsTvzz0kl~p7;_I1h&l|TTrrQ&e6a2FeDgR4)
zvwMh%Cgvh`evlencgR(~d|B51CpkVr2e9*+6HX9gXB5$I@wvccXlsOP
zdoL$vC3Gb@dkN6sqN97xhlfflnf~G#gFO`!Ydxhl2A@2JD<7|?
zm8T`2TKjeVn&IDg#sl|se-3BzuM~Jcx)tAv1^`S|a*8Y0Up??x>V8}xzBry09uQzb
zC^@~B%qlvV_~nWwy%y&k*>4%M7M+rgdxgaaoUNq1`?VBFtlg#D!;f>Z)K1CaaM(O%
z_l$y>mw$$eq$RAl@w%MwXqwSKiM4$e<|k?E?Way3KF_j8;=rwWW}3-+WTz=8UhY-y
z`}b*-NjhkjwQ^g3+NHnVhPQdqoVSm8$K9tprtK`Cm_ktmmKxHzt~-#gNdPTcUhav)
zqjvlj9)T*K?X9TgQ0`A4h@x%~k|f~hf%=zt`XEp+w1FtoL5pJPzmtaiJ{%Fh2Rvqp
znp{kJlFTHiI6sr!j>o?zNUg0^slYReAYXWkR|4<{BzBR4^I{
zD!KlThK5F`+|?dPFEg`Dz}O}x)^H$!9YlM)95#871~2ipho2bWkGcREB(6^gMvoQu
zJxWL@a~Opz=0*EB5m}g+n2410SzKN?65ws)W+d7~M^;AB;)33bvOmY4BoU}xtaXTP
zfY3UGNw;GjK*V9AX8VIe{XL2vAAP!(T3hN!65$GSYwPojH<4+2^9(se7u9484=JH8
zZE9&j$BzdcnXlsFPEXlUAO%RePb=p-moI8bL@;?QITfn^-EQ{cK9^kcJba-}%s`5<
zA*hZ^^v;*|{;YV-uG)QbA5|HJ?=|_3jj6p6Y=4S|;#`oLxwtikTBb+VbhRYk`uJQ{
z4z8cB*Sh)o$6Mu#&$Rmt6T4zuUQZj9RqB7dIqZ9Ehu^Uiq@nbL>Xjtv?Gp7D9R6wE
zKL$x9j>_3T9GYV1w%fI<+IPF>pAn-tTUame1!d)69RPXvYEJn7xE2ub(puIjU&UZc
zzGaMVf;{h%xB5cOiHpt}<<
zRXh?4f$h(vKnlOOxltWKEr>p`|M?__Z91=wsnICvoR0l@kw+5y;4Rlk4#
z{!GaT{Tq2oKM{Q3(4#Y-Px1VxjL<3LIK^~uPB4LvniGl5{g*J_`9^U1=k@^uN-o~wgH%#p
zOX+tXi%&x0@s!=lYR*Z&M5}`*xSIX>Zf|LPRg%0_`XDcmR8YrGjCYfD+TqUO-B;Dr
z=K5VCv(0YW7rf7D+AW}(sqw<|_s#O_|2*bfmlLnwxORKb?cTf7LSODWo|+zZ3h||=
z8u@I45lZiF5`y;&vZL|^z2j{*QPgpsM&&!=cbDi4OA7ez=f3HGT0i3ceNtOnq+cgL
zYwMmi7nGUrPi(G98w!3Z?_0&XRUzMQ;$2`;ZXT*KvXQCY(n1oJ6*&RJ4qYt3%fCDgH~1LXnSqeU)OSI
zq`ukW1E9Oj#P*Inkm<*^yZuwebP95Ei2lCCCe@il#Uc(n>cYb0JrLEPyn~Mrx!04E
zp(9dA9AhzC`2s&9v6I6*0vN8!%4D^yhPpZuLUbd1gODyuMPN1{ntb5k!Gm~yJ?o-l
z>HlWtStSP}G
zuFlR&b`#eK+)W_Ys`GOQX-n!k{kiCUmD5{ygEK|D>K4Wxm@#5z*xuP0I8vgKT|SoV
z5h#nML5b1-%dYyQ_la>#;l1IpF?WQ+T=_>*&2NbNNlF+5n#JDuASwGWBsV;1X7p68
z+z_dGR__fu6VV(^Vs16xWgMJ&bDg^9e*F{ae|Y}9MNY($QT^I^?Mm?BS-?^V47E&;Y>uG-j);DHx_{;68dXkOp^X5fKq!
zEy6`aS=HQ-;8p}Xuf2z%58E|ey=5f}eM*1!8+*WNXFr&~O&@Ij-whhNM;fB@Ea?MS
zSXdln_ki#LAjXt_WTrg81VP6I37bFzg|Q_3JA8<~78|3AdK_9c*C4xO%(TaI3y(!aA7RIVqI_t37E6$Fyf6`)b+dn<|>4tq%>u-xm
z{2Aj3X3LA2Fbyl}QRwwU5{4<``pBR}UfEj}zymt3jSyM6Xg=q+v*{Romh$My
z!ySj2C@Z_v^76*2sa2i*%ynie;@eTadyCEmS~+OJrBO0MtMv-WHwg5MRqI0NJi87?rU1@#$Zv%th8Ut6uM
zti*_&h`w^jb!qaiEJmUV9uZOcz%R^*|MDZ%#;+lW`nDG13W^%!g!aj*?RCV@BD424
z^AiYjh9^ef$ofD73*{<%-@ic1Dm~xvseX8jdwwMxzgf|*51anH-xU#=mzE?BXX7;P
z_a(uW8qeSoPK<*vQd;SzJWbKTyuG{(vA0zlQ2C$i?H#ESw{_RpopM}xcPus@~zRtICUaHuuJ40T)R;OM#(Vh^w4kOTRoOP>oED%-?G5X
z3|z}^$A?yCoqQUK^X0EEPj)o^{y4k!#~^w^aQCJ0{0~eG#ZMnDYU8wS({6G78Pk)^
zxQT6bJT;q3{NaNO?II@Ir{DOG9y_dP%j=}*Bko0^VUgJmvp)>7d%d^aE69KrdUh-2_1@r;FzuGuZv|6^^dRn)
z{hzH5SP2SR<-Sk{%#y}`L!BGrIZi}t_TJgtdUteitDsPMVPUcmE1w;jeiFXgH6bBH
zzpYoltj*n7sWg1F{iWZ@%haLU9nOZ=NKYhwj}G54o~7H652eK~H+pzCQUX)xO?&tF)BzHJ#Y`
z$aIJqZM|<`w@5R<&0eEbU?tTv(0}*{sfNkc>gE%RV4kN}iS0$~^QXQ_B{(~Cw-f$4
zoRf3sVYJwzxepn}Z07iqR#n>iP8_pL%dtUz&sP=i@8`3eUtwm4AS^mitD!zQah7B?E3gjV*t)YzM{aYBv36U>9gui>G%{==U7T+Z(YNhiA@YM(>oA9Ks?x{waon
z$g*)VqHQz$^14*oGk!qOf{J~ehkcW4e_8PI)EnXJbrrSB)6Qm8_r|AVSww6ejhe;|
zKLP#udqg{B$tlZ`ZCElwrJ|L(V`J$XRfqicWbL?;KA+#oyr`mLqUii81ErV?4X>Xc
z+R#u>0h8AU;gzv3?ZbGKYmnh)H&GvKdgfJ0y1k7^5ATBo^xl&+x8yR@{oVUBpGkbC8akHKT?h9@{f*pIj_JzYJiXx|Vw42{Yoe{;EOo$%t
z{a}2O9}1;6iSLNfN1%%&{5#)7jepSd6LsLbq#IrMEo_KSZ-c*i$vt($kDN*@)C}*M
z%KEkSMLgLX7PDd-ZI|seV$f
ze7w}zRdjgnkL8~sqNIfF*WzfcKuP^nPv7AIm#vN3+|2$*-GWQB(4KM*?Nu&y99G+h4U>M!J`}
zq%EY}E_itYVRwbHOaG{!WL9ytzkLukg6_&IB~|mHCj~1m;v1?w)4FKan0Mb+bglH8
z+6TUJM5y|leY;`ncZhKjP}E_U=`!Am1ENZz}VJiifD`I1lwm}=kPf<&h
z{$aMJ9b0O4d_K5o>^SN2Zq~a=Qp=u7+vv3oR$}X2B6=y`3^QMR1kp`q%_ecuy-
ztajQ5?eZz-
zzOYasfV&J$)sbXxx+l3Pa%FYb`-swjlK~PalDqxCHb_7kK+fQs%N37qm-iRAa5-}jdR69(?@?UdAmmZ
znU`*-{u8f;@?k5JQYWTI>*zYw1^eE_M5g~Q3{|r3#gO*OH*(##l!VZ1HS>?Ph|vkf
zRK$vDck)>IQ7*;q;#sINHrym^=*$!f*=LxloGbY%u2v0l5I&>?cW^Ks_$S&^|IO%fPN
zI7CELxHS^+qarMrm0Hm41^7*0zHk(N4V?-KioDieY5P@euX}BhwFo$}2p~of3*nMr
zcyN%&5EI+u7th=-8Zs(~jd10Tw6*D~WPN@`|3@hi$-4#K2qmQ&`4=!Hi6vp7qnkW6
zf|i@QI)1#@EuTX$DVduy{3l{y99U$+hIBbs!AxYHTuV!Is#EGc2w8{7CAhIdU7Dk5
z3-QE=!3=@NbMKM~@bSq&I$&!HJ{3X+82N)8ob60BQ3c%mMo2t_VWJAG%bUc+{EvG=
zZr)QQF3QiBXCZ-xWn!`5K{qk_*zByawsy7CG?>cc?l$q`L)smX4`4))qMJ3{wMUYS
zvSgVJ_P7B+BlIuZp}gch!&ZK_(tEC%yS@1F9o|PbS)+er{~G@aZ?Zg0Z;8az=z2Cx
zsw$cE!|`jH>QtsKbGI7R2l+A*J$OjO%@|J;E6Jzu+t{1TaY?%wYwSrN*edS87jiQD~;CqzuMG8C)r8Ew#V
za2>5I{8q(|_l@DU!80Or&RsSwNt*->TpjjTvX>iLU2Kqr0WDup+#y-pM}PjHg#xQ1
zFZds{EchuCpn(e)xg^F|MZyY%`4`yce?P=W=zIq7gnM0d
zZyJv=+(FZLO(l%Nq%U|6C=v8M$R73TyQ(&;n!2ZF#Z=^UgxlMXp14#pi5i3%f67X~
z@_2G|kN5F+qLYIHGu`$QKdpE5UVbdUK1Qui(PT5?`ok-$r{A}SfsTcRi>M|csKTHl
zqO`I=B^-QqeQQMw5CdR3)lAA$@1b`_$hlNn4Wva^YC5XD$xwc*p^#`BKcMs7;QQn4
zXU{t{9#pfEpQ>k^%#NH-J4XibXn!N}#O=E8kL>sims*l%$zCzB`!raU^2U-_K}yoRt(6r)dA`poUAER(jSW4PoUo?m|cx
z5CABnj$gfKh$X>bv`5!$WMs-jbP{C5!~hiFAo~hKaB#rA$jb|vb#5TYTk&q*+T5&n
znFqtFdebi0gFv^^-P6+s^mQ=cL(gZi!PVT!X&Z0_;5uW1=`+l$VfsAq>nWLXd|Vu)
z;EA}*i9vJnXnl+!wc&d}C4*KBIZ
zuIjH}?}anM#sES=z?OoCcWq|}Jh~*bwD=IxQCwCAkwfra2T~IF7~zZ%_K|w8
z;@vx7AEw_c21QZTWV{j+bnR05i^ezLV{CZ(BPCcMGlCx>$^?k~)ARTvZ6vmO${Owz~g4Y54{_B=0jKDUF+~8fKc@G
zQ?^PW-1A|o?hTDQ=!x_5L&g*+z7EeW;O#1F8<0{bKpw=tfZ%^oUte!vu3;(2H=&G=
zIaxcBMo1v;mOa&SQF!wOSLFMHGAvKJZP)4ckN%E%Wseo5{;fF$^m!g+o$^T>X@AH1
zRG~eJ$BTcF^Rr#C$98lbW1E6;0%d*mGysM%O-&MjSJ%-w9Td8|^&`qn?5Mi>0h!0p
z(8Iug_K!_-WRxg>_hP?Z4^+O5cH8B~^MeNJ47f7N2PGkgJ+;%$IeSkOaZrh~@I3oKZzz~oxX@d3{+Lhb(jKL-aW
zp_a*h_+0>XKr&i@(fxv&4FQu&W^57|azB3#3mYVLVCHtMorVS!*!rO}hwMNCDJrEO
z32B5G%*qN7k|47Jc8(`cZX;l60&TRVMe3tbtKC=u%$E%G^><-(1=BCk68Qi;26SN;
zJv6RjIm2*zP{||F4xrM73{uc1gVGVcB^X{n{{;qF!3W(G|7kd!rVmfrujzW-EA-F%
z+48_t3p_V4*usZK0oy06kgF>StFJhZBBVg523jVFg@-`*U3eZ4Xn^u0p_;i2=wgOqd!&T^5U%_u(65EBOL6DGwY))KKvKa(#
z&CSNeIxsVipF4o@4`gNjdfEy@j=(En|4`oy+W$%>(0wT>xs89buBl<7-l3q8tMCpy
zh8WMnTT_!IIQ95iSdz$%ygf=GDWx0%>J=w84yFu^9S&ph^~`ic=re3o*kfQo1Hu-Goa2Q63MCa>)sGkL
zAt57cd7@rEGe3iVru9$8;4m9Uv$D*-r13C#S1$A`b8oeMrT4_~+pGB;ry1hwfYQIS
z@1qV;)<+dL5Z?(#15GcwiK~;a3BH`V18FsLNyxqR990#UTRNfq12;L{(5LfVH@+(O
zDUY)9Z^2i7*D<@}pbMvgjN7mc7@3N2pBcD+Xcru{HaB$#6z*&xgM=temy%46S
zu)Bk-8O#Eprh{nfH?1HOLYKK{_&Ugq;i!CGhKEh!Pr#06@{vk!P{aBNLN^
zl~YrndF6aoBq*aT_+cUq5^YGYgt88n
zHkFVK3maPs$-Uxlph*zP3L4;Pm~yIzYKC6YTb-L6A_%L@+k@d0g5LteUUst
zl;e;00+je5=Ko)y59s}Xxd&ZyhzPQ7N4QOz`atx^eVKYSJ9t|#}A6-FWy$yr0*lco=ZY}76i1JHFb-67Tup-4?^D=d&c|X{t>j%`>tn+9lD3
zRJ2=QBgozfGb3Hk2^A~*)3Jr47Mu0nNmlAja(!Tind`WN#X&e8yyrtAaF8xZ)o|S
z2wJm*c_$mn4IKp-dFye)(SuB7brVx!x{=Zk#vmx3uP)9&M}Uu6_+Q4b*B0)B!G$
z%@Wpc&7!-@+FAM(D-Kz8$eCubo+j
z8e>7uT8GAoc3fnCP0>X=jaLZI*wT{GPa6}d2W^gvi_XsqH*J=A?wNhC?|IAbKGm2|
z+&fkK&VF(vQFdji=UJ{DpCN@UMUqzC=RK@&Mr|?zZ0trI&Rp18pov>wSLd|Y)(QnF
z?0b75*XQO}hZH*F;PpVr@fMt)ogMf}0eZs>zI6Cw)jq-j2M9S*n_6)sgHT#pMur{%
z>izg`AiBc&37A+Y`2c2CbPn?AOl|gTAzQ*|OQVW3&jDD?FxMp!H|0&AcVS!c+3@@a
z$|#gKrKNqny)nY}P^_dX_=2?v#;5V!OsXgdh=l9300ArL=>DCg9D+C|1i%MHk+3^@
zO2Ef4VIdjHku<2#Pk;5w*Ut|&dh&R$XE&ZS%$o92lm&W-WrHf71l=8EzfUXG4tS|<
zgfm+2qs4cFKor>3S|AAqL>bHz<%vPYrBf!%!2#pz7nd-}h7Gm{-b`Ya^yb7kCgwhFWtk>#adl|G0n?*e9(kX71adE}i`13VN+dqyJwwhZT
zS{3x#l*IGL^A_25n)s;0@thRjf#!X3?b(%$w9!1}QPwRbbk*6;pO}QB=LE01eV3T9
z@iAV|o19ud48jb(H+meFc#(y|PZEBp{~)LK4y3r4LEEw^8~F>)bXlF)t}4Wo(Ne%`hL*B1lM?Y^}?DJdx+Y@2WZb{Koy
zjFSj$FKkQ@t1TlY7F$rNSppnQICJ5&1>L;+CljDkeflKM$$5{F7(&=!>OSZ25vj11
zmJSjtyDd))nwtJ)+U6n|
zM=R!h+IZ#PrG6HsXu5;Kbx)N;ED4
zmYzAcC7hPZ%H0qW0J+Xu*s(Wsp1A7E`Q0c{A
zW~H$P{GB)&Gc$W2Q0M2-S}^ceX*k+552x(IkH_0*mB-DbIc!ad6%TYkM|InB^nFyb
zP3WTQID)_V%^M0$FU2@MJwjAG%Xw`1%#Jp^0znHMx-~V;E;*VE6|ZOAR;zB@wKj>0
z{|3WQjGZfXF+0Dmt;xt9|ETkJZvV9S^F!U%?8(RNdSWR{AUJ1N5D-78Ztk{7A7WcGBAu-)j*m61R%wO
zU=&Ituq6JM=nHzek48MvmQO#KaHc{;*MAIc02JYbL%j-p8<;bLss)frNYxkrcJp2v
z;QCti{Ljd&{adf#q=V8-EJ;*E1hoT}U4I&%__&h@93;}93GGsaTO4$IxO4!)Nv@*+
zn=7H<4jZ`|MwK6FQ8S-%*Zf=Aj5`Pkilf$QR7X=Ll1}B{(qUjpihmq^ppsKo4T|#
zY})XV09y-IfjlvHD=01?WE7cU1}P-D&z~cMFf-NDa&mac;~_mA8jw&49Bh3cq7U9e
z&JiAC$UooQ`~Y{!?fe)f`VgrHylbeT)7!%XG-6L1tJ+6yY~1v-{g#u$n%w=QhUo{0
zJ)=Gp42(XkDl5_5Sdgk>6a4FQoAkoIJ}&=bSH8%SKt5c;Low=}5Fd$H>xYTpqu=E(
z@XWMvGq$*hpn{$K>guEc>Rd&l#3nrtSW`JZ4sp2AIxMw{-T9=#0o3o_ni{Y!*S-G}
z<#%h}juj#IiA1Sg!B?FL#3hMQ1iE4KZuah5J3j7lnNc*SRC&ox-@
zt0(v-w0ij%OGv1DM#>%+)$q)ai{xmeMeUCIu=CuL6s%@R@VX!F=ry=HwCH+6A_q{s
zhY+&+jW0@yl9UwMO?)zm3i-g22YT5DUx@yNR__xtvjylc0m}#+T!5K{ENbV5dGEtj
zB)|^Rpt=Y|`>X}S(#1Jbz3o@Zu0GB8Nb&4oK@N=Ak+
z)bcNIq%lxCp$PycH?VZPo7{qL`qM1&&6~w~oRSg}NcJ1712|wH$Pv7_QXTVhj+D?7Jz>Xv5F;dFc2KymA_}#Z&3J+dg
z#6h$=z<6#4P|yc6XiLmcml#DaV%8k=Lsro_q%p7lx-(2aNZJV@ivV)|09Q2)PgTn3
zzek?e2gwbh!osPq-ct<|LSLX`MAoOi!7!29f!5<#izfJWT}?XSEB$xnchNew1t-e5
zILL@=-fZ1n(h%BN+>G|X^M%|+9E_)wXr(dKpPC(
zQ-Y~TtJyWQQZA0hRiVR|fW?qNcNQh4>FNDC&+qN(_p&g%y#0~1;FO&_p&PYpa&%Zm
zgk;C9ULg*O<%>#3{C;8&MkU{r5-sqAmdh`pt%$@TLK_&zvEB;G>;;ud6JGA9pEFKP
zmuB59{xbAB2l}GLf53qz0~a`n*25Tv9zEj#iw?-KBaq!&ziTlRWZgan@t#1qMo9QAozjzlS}_sveD
z8Z*hnHExt;@roxEK$L(g1oANWujen5WirE&3}JXWA8&&U5sW{jI%N?1>wYtxB%Bcf
z*`Pp82W=ytkkFsW8%0yp51wtmRKkKu8#;Q~)i3+UdF3+55#hF+rKZqChQO?z2yh
z3%DsIiJ-4!!0!^K{+q9OJ}}_8*U0>GA!evoPc@C6s$qfv9
zBHni#Y5df=hZx?ws2yR>C_TK}q)Cgt%C(f}p)G>&e6xg@UiyQ^ebmYu<%*rnUnw3s
zoGTr)s^+2+eNqa}88@*}*~>C^K39Yy$MlEph4;^JKe~*6%vqO;UK(mQ-C2Bwnux1O
zR*tpo%eH!=Eg>u{qYdZ&BtL7aPvElOd0b}_!LjL|LFfR0p>qIblMmqjJqEo{A2E+z^&dd1!?L8%@+KhoQ;dlb{N*|(_?RoNXJn_V)LCg3*ctC9
zpoBtag_ldk#Xn)C&i8a^SpXqrA`8cCN0{{5@TKnn&+_%Dsz`dk#!lTWm##FL2KBdU
zE`vKtIXTgU6|IupT4X!L*A^TXg_Qn@y3&9Q?n3EAVP0|~YVjlFd_^Xr9T6srI6l9n
zj~Fl5WskOWY2aaxef>fJx7{Ngf!?XX3b`0EoKe)gH~}E+Q`-=V*Gnp9r1X#FZbO!?B2inWVN73VqZv3*m+n+zE6S8h>Z|MNE9$y
zSS56`6QhTm@4U{d_lc$a(2&;NCUbavK&@6Jyzcs^V!kKo@y>IOywhh{WY~zG27iWb
zJe9c-A~^F_$B@I|-+E0iJsJWo?Ea_f!t8P2)Pot;?&eH8Ouc);LC)cJx$e1%h80|1
zX3s!7Cp_`N#KQ6K5Ab8@+A{cZ$De%-l=j;{`?TMeUzWBlt5?R@fMMh{>Ky4!a3^};
zsKIY1qS#e#LIV3qqTupB^s%Wa-=z+L6nlOyN3xoB|5h&11~yRx5S|9MKlY)|iXD_i
z>{peCQ01t^8;2;KcIegy2Y^K+dp8=bg5Np<_qHdEJOW?bjd>@3uQO8VeliAgV?|*s
zzX0mGN*eF_VYbwC$dVjUOwYC50o#D0e&ULp-VfdsQiO#!@th#xZQQpQyd?t;wPsKE
z84|B47tK`Kl%#o~%?%B>{_U}dm}_}t3B3jF@L#{_WrCVaI9pq|Y!eLBUuu0T>i^@?
z*}U`b_~jMY`>TqGm!^=#5EjWvbiq8Z!3?~L~~o0%a=pv45GWAhj#^X3)GE4
z0mo|+94-m}_tSe&lOxfVF_9T&!QwnZ9z~J}hmR}pzJtkr!ToO+6_KM8wMdWxIo^G&
zdqj2gp!CTgT|HT?8&6aHx;QLXL5+>7O^aBfj}4|1>sy-8z4s0d_I9o+oL8(!Tpk%OQ#O
zZ%F-4opp?j&xb^fAr-->o_qZR+i01mgx8@9zoeCHl~@%&hfZLPY*E+i_?H3N?2H+k
zIja|DV)F#eS#KWVIc497#ATAtAPB~LhTt7V2^FQ6RiRG(O7Br`Q!>txuA9#DXJqi`
z?YJtLc)s3D$VCm(ze6kXIhpimCVWzx_V03HpN4l`p#Ie(c~_J~hY`X}Q_6od1M};j
zmx{+jtwt>oY`A!GxoE-4Z*0&sP<4Zn%}_36acd*wJp?Ol3jXu4QIUPoxSx{ve#hKR
zq6Q7i9|6X|w&`zAh(vmmWo(kxo^wWnpxqU7a{oJ8Mscs{X|O6^Yf4UaAw6D>
z^v>7Jfc+)Qe}5$X$^A2wz|9O_SKLb4N5^s3CJtYRsI!3O#R`|LtF%jOo7WWT%^sK^
zSTT#n3+u$=6y^Dd5mh-6B0T)=5?OJJF}-tIU);iY(OG>0Ln6q&PHnxvBPxoA{$tfz
zJWIL&W8XY$0s_n&%*2zV#U`6A^s+0&{=;7e-9g>nU#`}iP9IZC%zu+Gr7I*|
z;0k4MxD`8O(QJ?JwurjC#VCw^I{xMF*1f6d?@S1en{|_YX?yqtPlmH7ggMbV>^v#R
zN{4eFz8ro`#Of*c8J7AfxA)0W*Fz-{F|YOMFFm3GXa}W4A_ZO!AX_11kO@3o?Js3N{_rGB
zXM?QD#HDJnvc}$%aGvXrr5_79_iNsn_D&r0R}nKvad9Ck8?NivOCjt%HQ-SDrZN4&
zOcJf`SCV^deX`3CA3suM*qIcclHi$vY49lXZKWmyN8*K3x2^q^J*hw@Xp6HKv&;+2
z1wQ?IViJ2C9%`>Uw6lpnE$@s$MN$9iv(LxS>8ZweX;M|>pBmP>|pAt?i%Z_fuQZkm5D
za}-b@WwohH?jmYHi9+rCS(f|N{imB`pc&yAZgpel**nhG9NXdd^punjIeteVqLt&{
z`WqreQBvTc^!&2n3FEBhdES(MFxYFBK!Q2=C=#DN1Ce_R#Z6pg4-dG7F9ZanSwtvcws@e%#m5_5Jc2q6vE9oQ&&5PsC)KRTwFq(mwi2o!GVwEwAa*=UL}Z}
zMATBCbH(&~84Wk9n@7(|LkmcZLEkRH*dgO;HdyK@hM7aZc{79AR6^43_!MH4vr9c-
z-{#|Nvzu)2pjywfQFzH$lyI*(lZi9cLKV@uJa8}eAe`B2+vk04qukmXPd|lFBpTJf=<-
z6DLAt>hqJanU7%qv<*n&BaD$uk#qKldKRV}LnTAZo4YAaD-`frzAt=UWAr`7t;8N%
zx1dMuf>EZGVkJVJ!b9Y!h`6NRxS+hcs0V$*coN)S!Dj@))Zon|#ujm!0xt)Y+AmX6
z|AIA=D18NP=jVV30Ouv^G@`V!64Vl=HXZ=EWP>hZD$W$VwZlC*u#nUXXYyX-v<1d8
zD1ZHm^JOV=e13mKr_TU!lRc~>ASqK0tH%NRFWPRSj{(7qCVygU4iq}!;ZiY*ep?k+
z%s$JdWn*;GGm}+rtns>gnr#7rF*d5(iHeC2BOb!9pUdA{609MUHKZbVG~UJGOXaSDordsQILp|(0K2{3HnVzLB>x%*hLzLQzx#B<}dKI*>UnQ
z{1j~b6Rw-5b{;?EY)n=kJJ5KQ_v~}gY1viDhcKb|uH{eS(CQ##W$X%R_0wg*6IL1fpt?eZtCOolQ}znlxrVxA@_@
z@Mp5o$BMG6FK?-cWpt8LpLH)q9-{O)x4yZI52ZpYn4Pa7rTAHf+RG@0Et92D(#ihq
zj@eh3Y3b_ed9EZb+)o4l2J_4Mdi@4lAFXBa9FJIr<#gfQVOeO4`i$=?I>#7!Rg?w0
z>5{RWE0p?%vp$t7qejo6!}VYG#oEtUQYaNpSxP1rop@?WB#aqxZxlrtg{?AIu6Hth
z<(F6DcP%tD5&Gn@MOR5rH4sN@c@=l2@6qtZn^V@iS*~ZqVWEY7E}hbE-#MRgrK-y<
zvF&UjMr816MY}J1iheIe47NNWuEPzt4RfU%G1AH+?^cuomFp~sUDf%(fe$g}YAncv
zE9k%(RqSqVGRZ=K6%5M2&`t8J=+mcZJK>~y{_`nu2;Q|^?*FdWBYmQzm4Cb^M|<;Y
zTnO29;iFkf&S=S(yzlolc$_8HMI(AJ&ss-+^uET@VY3iln#9xp>YKLSiVuB@4fIrb
zte^h~LV;HUQF!26I^5sCAwKCS3>qa#E!v4leiBy=b1638VX-8;QHd$3@9*o&vxE~l
z9f^`rlAm-s<8)jnDqS9DT87}&Idty6sijrtd-Li2dqZNak0z67iTy=ZN7tt@CSwRn
zYU-bV1k4v58--F+?cwSKT6NjWx(CIc-CwO-H5%`Xu?d)v$f$@XYOeHGrZi{O|HEt2
z@9lV5Q!l!6T0@vrk2ci(8tp1s&P3w5WbO|Eku8u%Uuz&8<^?4sG2CHDntFWq&#kSj
zloVZKV>|>{iIL;{`1lkx7WgX!M%m$(FK8H(x*oVpJ*TDS%qZvhznKN?
zGj%;yI&{Jz#&Ga`DP?*2_eT%6&-%G?U;q6sM%eqa{?$=>pz&qJLjKJU&jWM~^&2il
zi!M`+GUc4}F;Unszvx)p?r?Rz<&T|B*ug8|_K_|W9QX~h>}1hqtwTwKwVG?3T;oS4
zvw^-cqRR6}hb`&7yyc&Z#KwbuTNqcqDh##};byo#9M{D}%5?AHwRlI9Y`;{wHAur~
zF{k*fGO6Dh#0-FuRVhT?Oc9~Wp>>wPBY=5{PKe(ekc4P^pa@wlV
zTn5U*LJEyoZeAV(un__11kyo#M;8}XY$oZ;birlHXb!SC@B{v*^rN6Z7fWiWuZIYD
zXeohn5L&d*id()9T$yMB8Zc1S899Se2l^9PLrHOQOz#L`uJ7aN2}Zk@osH2Er1Hct
z4+K;OdQ*1+@Sea$(#L#z+HKlVY75nIH*h7T>=8
z>w_uuomG^TYYfvvoHqad1N;awP!rH5n`!vrjL?`v20y~kt;#+vAu22k1*8bpf?Hc#
z;17O%$C0&?k7%40>Dz_e8j&m>8JR2Bkn^m07vv}zke-(Zy$on_|4~dpJPW8vaC<_&
zxQnhJaZ^x$2KOSx@pN;
zboE5;Aw`y@jWl+R)@N(i@x&VdxvoYd?#;iJF5N&Edi>?Z){7Sfj=KaR!jv;y9GwNH
zMQ(c!$KJUlG`T0BrW3u|vtRIdzhYQb#wLf&{o1{}#EsVN@n~g?2;J$yvJ(;6u{Lf1
zdr0UgEjBiK{vP@&Nh8UBfB^W*N3wFD$7LtXnfi6F{o0H*g@6W055SKGF_uBT_8`cL
zgkdV5qEj{pL3H2|zT=oSzTpJvCcM0?d>|@;D-E~{4o^;GK|+_W7~=-W2opwPsgpSm
z5WB$S3n?pwIRlJOq2n^Xq4WTmRZEHglmZpu29wquu_OpYCMP4C3;Z)wgtGMX6AU}I
z{=#x8s@h)eis2JMK$JFY27nuZ;f4S{bJl0~>`n{(SsIZn5F>(C`N4x_=#Eh%*yEtf
z3lCz!4|2*On*dlwAe1!9^3l+u8!{*Bq&qlsqAd@&g@u+^FyZc0brD!9Ba-`OQxFu+V&$3O=`YO7F&zxt`=x
zMiX9;H8#!ff(r8iI~fcTL9YnvwPn}mbMJ0pFk?eU4@7mu=p2|HgMV*Jt1JG;MHyLB
zRZ;@P3~*jVvT{;V{xq~-I-ib%ZZIDBMWni92^DG=2aZA47
zH1d_$w3w1~lYnUAJ!b3z_tFEmbIao5zrTIQS~>n@m;JcX&=gxhncaL-m-o5$&t%wQ
z5OFPH6|p4y%+-zrH`PDrP6si1|ImEMvD0Ju&chqQpSrs@qcOX}pGP^7$(yd|ca|$W
z%ID|Mrgf$+|0Q4%jJ@6Fb;#<*-6+uKjDZlFA?%f43>)}F@$4~TO?34r8Px@Y46z+5
zgA+wbY%6bHY$bPbBJG7i#0aBwP95gu35+V4jN_%LC*GF46q}seJW>np??>M6FLyG0
zTl(dvE0MgSRDwcS_qkKQl=4k%S%NjM=#6z(`83Lmlx$&gtFHBW@CdEIrj({MH001Fr#Pkm4_nI6Xa`1063?bce8@JO(o%NUix;SdfjF
zwY8GsV(Gu%bwQSdTq0p;SkTW0297!_+^t%cqiM9sKoter3Ya$lu8KDuUJ-P8pdP@W
z33|oRmgw>bggmkSyN@7^FC0U3REG!LhS?WnJa6P$e>8${4N&I6YXo^!92}ke_`L^~
zi)t}lV5OKJ8|$>H0eBg7kYT3;^q*#)%+AURGkH86p{lLzT1N*WZv>*L7#5Nku>lwf
z#q${;EQiy;Y7V{3|0t+P^Xi>D56bl3{rx-)h(LP>hi_YdpVhO3V1_2O=ts(sj_<(T*@C-%E`Jmhl3$+l
zf*){OAnyS+4ME6jSlz&pNPYFn+S(elnJ{tBmePE^LD3Mga$5SYRU+hkb}{&$y@5J@
zP-NvbrHaLAW2y+}E*~K|#@2QdxB;n!z2dv*u8npm18OI+b?}DH;tkd-RpM+$MnRp^?Ur616%%1i
z4(tcCJR5u*bnMR54kY}>^h1%28!x_lv50AlsZk^7mOORMy?IS=!*qGRPjrO6>EQd#
zM5sitz7SzVi;cd|T!#{_KrGhz_hsYz`WM26CMI*BfEpUtChP6p=nq}$E34FntE%b@
z(HMJ@9migq-j;c9J8t)C*iQ;3Znsc2}?J{V=BKpIitVL#ZS
zLAZdNHv^&tJngl$DMw?XSwk@ScW`&FQD^OPpK}u7Giy2JEPPj5dJ0WP7;g7J`~zza
znLx?R{1{?I(d8iq1>#Shnnp3k0vagf^6w<9OFhny#l^qCw$I7QsqnxP=EgvlA;*n^
zKBJ`D61e7-9sqO*Q!jo2fL7I+Ce6^|pbps&YpypukTYUjH&0ToI!A${*vw;DC
z&uDRzuZST54Zs}+j|>PQpw3wUNHM|a~V;ZrSsAjrsV
zZ*L#7W5NFUanfvHa1ioTxb)xeA0HDE5(XnN`aypmMsz}vq0Xx7V%rkk46|RvT_t%v
zmr9T>8C3wW6RKL+h+sabrmhY=zyOdU!>q5~b#Y;42ErHt?(pi~ubxC2ydB%@F3$xPwLXrQnl3&JSQJ8}|#eFRtsK2dh7$bZ+d;R%(Dg@pZRCVr^GKycCfz`xATAJqaQnjw*&t!Yr(XkO#
zY=>ojWzr*YGn^zD?yLQQIKK12AhED(d)Ag3>{8M!ar?RI#%04^zUwyZvS<9??iuPT
z3S2>`msyRO@CE9b_g33#X5AJaWrih{7C4)3|O^q_2v)~2Y-(IFz1
zAAewE1U9y|m!z=@`LIr*`n){8p;85B@AY~6bu*AJ`e}baSunDe@&hAp1WYn;^Fzp$
zjDv{!8diycfx*BpWXgyXRF1%S1M5GOoB$XHrvV$=8oWHxQB~
zG6zv!5tdq^!I%-z+L6
zbpZrXz;;7a$0K+T!8-&iwpeEa=qvxh#(>`qL7_mf0;RVtKcy@vqyeHLPYm}4NC*l9
zr)~k`q8UlX=Hc-_FhsX6oY`2|$7Y-4u2~Wu+!>I{5y8-bmEN
z4^)7X@$2J;_7}6e0Ko#idrM;u9NWo-dX&-9m!97SGi8KlVqbkYJj_ob6iP@z3B57Z
zUVb%UzaA(yf4QAqrjFrJK)Nf7V5Qi0LdQbQclzz!tI>v@9zFhrznk|Zj)ZB(A6j4!;ks$dbkN=BgKfOR$KkslWjTqm6ib-8fjWGfpY;FI*
zQU_*9pgbz6s)BVXYY1WjfwiVLA4$(j2-nda(1?J#5Y9QVB=B4v`q0w`b}nE3{A#YV
z0dispJa9ioLs0fU&$+nBBPGqWv_$>U_v(XdyE6~Zgx#yiyQ&=-ldHQ6J}0o?6pwG+
zh`-6Ye0HE(M=!n`Xn+@x6{Z=50(p!BLox&dd!1SVkEO&D?q+2M(h_22S`ooT%13Nl
ziav;C8I3)z{Xhk%GQI9Uhm>81@u(PFw?HdrCVlQulAnSa&Hs$62T!LY$Vws#XPE1a
z3I?6O_9OOAC(N7qmlkH9doy1!7HGIl$JDH#I?;Hph8LhlbINaF(zc)v9x0i*g|e@H
z-u;`t;YTOXO@%^os`3F_j(RdL*d@r78C$W60Wcy3kr5NXq*QD2rWImrvDBU{AMr3
zJx`rwq%t8bxZnCEpW{=qRWeYHDk~{L1SA3*7nf9
z8L*BHi8zlnBQg`PnG~1Ld_Ia18^0^McVO9#7`>A}q2HPii^|-LmZ%fAvT?cI-}lIeI7
z>eTdFPeYN>Qn8l2ts?AOFn@DsM;T7!s?ekp8T+FQqM%K3$Et9GQ7dhVF1~|XSn^|d
z`iP}3QA3Sh%&98QvM(uodS^m2+rK|0$LvQ5-MsDDloQ$I_4j7fzkgxtc1kQwN8B-J
zT)g@>ZiGBnN!ba_0R}F4TBWB=t0vzj^K)g6-7`xolszBYB~?c7uaw&
z;4Z^;$F5-(+*qyf&oYIHr{@t|7NKos!b?H*Qu1*RKA;=vkP!-jldcM
zF1JYwT>GtR8jFiD5MYrmD}xpU0u0$u&w5GFA)YrwRClb2Iruymo4mFV5MK(kSujAR
z2*S|_Fa_v2pvy~0O7@E_&CD1HzOQU;owpW5fe^QJ%@U|tO`5$cG)o*^UE8hgAXpnH
ziLeC$o(+2KJg2TgH?PvjuHD1u#F^<60EF|y)+LZ?flN3D-#JOiT-RG_Gv-$5+=A){I8haS+=Rf
zk0>HeyKHLH@OzZiRf^%i-=3m+qcQ=FFA7z#Z%2&-}MqPqXQh?=hv{Ivj0{(El2>U)js
zxqDC2_H$H(y5r2|*@dX-G}nqBmO6_RH5#U5z3Us=>2$Y{u-0MST26>rLr-k7tNT$|
zLHk~6*2CMjOA{A^#mU!C2Kx~)Q|{&?dg&ei(+{Zi6A{CN0#-)lL)SJ;Kk?b|P;0EaZq
zN&n*jB!Gnn*pH)DHPF;R#`GbeLsDK28Q$^Fo`HV(ePyMKgTv;=2H>m(Gr6JOzjNm=
z{C|NP#pz%ZQz^VP0Xyg#Og>$pKnBJ-XzYB!Rq|nPpzA^ML&oB<6?+3q$q(
z5R~zH<$Zm9Aj$?&8UX6x_y%PwjL6`U%mxl3pwL6F^eXNq7bJv6TY{s_ruGwPN88);
z246RUTod}}aDz-8S%WQM9@?7VE9tie=L=-kzYvV={t1d#cr99>_iDl`Dk~e(|ME7}
zB}J
zF2iWoBcGZ@Cn3Br#Nv^_F5XXHug&*D0k}^N9
zL7iy-cT@X!DqSfS*3M$c;_>B)v)TWVbl%}u{%sh4?9CIh_f9rtBztG?Y%(&k_ujH+
zr0m%sqeS)yp==S^gk+PE@m_wfKRPN72hVfg_xBp-`8kOP&lQ{As1y=Q
z_Oqngx%o=@Hw+Zek{%8FAW8Wd#L!d_42W?imGAlV`+Mmz{I4i53}sq(5_~NSBoWxC
z@78^yKHcCH7$wc`dj4%YF0K$H7-we{EiE4tW2v*sxJ$L{4`hE~#VcfW)WeW;ttWrz
zZ_d^ouY1GNk`m3IkYv%9wamg7eFmYlD5z5QVPxEY9)~jC?vCjkbnV-YUiq=`64%Vg
zb+l}rkQjntKRmuc8L1#vifD)QY2Lnjip3)^|DuC5f(IY2;JHuK)H(L}1DqL?W&w!<
ztAqj1s(<`}{|8qScvI-<=||n@udMi)Ce0SCE&
z17fHJf~^2tC&0ub
z5dI0kMHr%}vJLe0)fiO*+56?)C!E~eUShdm5d6RS#=9v9Jb9{Yg5u&POsSAg87l`%
zLg$9XXqZ#N$%KI6`=c5|=*_@;edpyjPcg82U4qwI*3OPSt_Um!@XdjM2_aQ3Po6+B
z7~CAyGg~mNVq{zdk>Tsg%5J_t*X9El;KEC!#KLqt1*V{IAEnwLr=0%6F$;57$PI_I
z63G90$fXbAEgOzP6ZLOmv<_zc!u_9{{PF$=TMT#y@`L?9YWgNVMGX)AuD=ZaWg^1q
zogp*7fb(4Uut1g;P~Ed7-dgVG@r;{#8S{c#h~VhOne7t~Zw7xw;@<-?k9kwfH5mVn
zgjo#Rhf@5&Aljx)Y)AcF5c?}?EK_Qs1(}XWqpPOq$im7V#=mf#$
zQhUf!+x#Y?Yf+Az#yLhE?Sn%YimtFaA777nN>y~q_+Fd`z2sSYKbC^z6k+HR;_ySb
zTgq>~xaSG3rxtoG;{FQ1FY42&u3ouDV0_vyhjbg5a3Rv-i5!Fm0WJqHuTY(38@FUT
ziM;Ln!J_18Ws5HofFJt;{kM%QCUG-k51y}T13QL7X7U2LjGWE}muCfu%2^mg`%;Mo
z_BX4-Gepqc#ef)YXS$QI&@|jy(p_RTFP<=SS&;5lnO#7)56T+5Da3QQIatfJUIYK3
z!2I5owe`6+8*F@_jF6KvY4k=y?ft8YhR~K78yt}E0lwpFumb?6>x>9vm`QQL69Vy1
z2d4l-`8eZDO+m5XDgwP(7umJX3UWnq)XSAvsJsIq3IQ5f0R18uh+~09yh|3T$?>(n
z->AX05G;dPrzi`*Ub?{NOT@cSykMaT8!dqFSE$|P<6}rw+HSiF2Cw2YF&LU54cJ88G`@`{QunS@&$xTA|wt+02MS5Sb^ZLlXIvB87boew%=
z*j?tH2b>T=10B(UK)_G4&yUmJ<(v)an}Q=jBs>1ffrSPqS7ma#4l(Sq
zef9B|ar(fpW5wF=$h~w6J$xlLN6V=sDd*c?`G6wnFg{!qgZV%PZVelF<%zQ)XSYTAtoS(%>#zV|jd%
z4-6i7{7w^9&V9J834hj0z8Tkv|0TS}kVz0ARUsL;B4K3JSTv
zR~44FDY$|Tmc&H6Owx!WfS4Z@W8P@aP8J^#9&Rm=4R*o?8w5p?JYBSUIT8CDBJ|q*
zyl>frjrcMftCg7ByS(R{nY#pN_9QfE%+S%hbUzj<=5iaN&C~6
z!KP+ndX56+Lwf$g?IEwONM5&c#~)h
zg$^6f&!705e^k{vH>uePY$$ld(n!(Khm>jngkK*sLb@kboe7~Pk@Q$KUs7gU%pcYw6M|v6A;&ev4R-wL8i))a)@Q;nEp|nrwSnSKFJyYI28Xx|r&g
zh>{Tu;TwyL&n-F*r>@*kY!O*>CeH2VVg1CKO(^C3lr*p9*fL}MVsA-UCF#3N
z>6!jC4{SbAiu7Da-bk+}5h24T1k-z8YjxYrM*M=0o$U<5!>l=2mz>q*;eYm&ap)U&
zjwXBSJKnPV&Bms{$NbbPF#@9_(^{!*P(Bh~d;M44#8Qij1}EOizH?79+xb~je6S#m
zk=}!kM7PGW!82r@?Syjrvj?B85@^E1p%cRXdGi$jm66-UDNRov}D1kza1aE3}QPI1P
z+j@qt{C8T!)UX_bznt?aQ^X8DqN0oj^#e*4}ni
z6eo;bWDp^W81Lgm7HV?v;EdAZ2xL2b*uD)C=##CbYf}y2pAP-GqfqDO{@lLl)5|CG
z`UAaK2?x)(ZNK1E+6B4GyJPH`$$N)>vwpqzl5Qu&x#DEW$MD6juKJ~H6CtDu5m};s
zbs6A8x?tj{svmQW*Y9T;M4ppfFm^rAOZ4B(C{f{sfiztQA*
zSC|p^s|@z1$?C={ZMBnA14Zx2qIen!McXRQ_d%oeEb|f@M^Z^{OjK3pzDFAsFRnK_
zCuX53)*5NbBmuF7LIlT6j*h#RKGiAw_ubz&peP~~QHHQ4)`qPyeQ><6nO!YiL%lRT
z4hDGxiLH1f%o&bJzOX;?dS6jtcjRDZ!9av?y88hhjYdU<+V9vkND(5)n8Ct-73c5I
z=R69ne3|gJRq`UGP1yB?MUVw!p$cJi%sK1m@zintw{TkJQuHz-b?8E;T#<DK9(HDlk9MQmzCKJ<=MgTdC?w{BU3g@xKr>&ZGQq$p}lhc8h=)gtcjw?DUN
zbH&H1t)+;A#9sSxUBWA6$M0&$W@4G1?|(msBa#r8#2Jd=)dZ>+7Sxr&u&M~Pj4mm8
zJ`ikVS~*)^RbJS`Bww7>KRK7@`OhmpZs{@T`9e$QU*aX(8s)#hU>CCt=+YfpkQdPc
ze8o?Fk)N1om{hc{vlIy7d#8J5jmljQbM7vBwuMQB(URD-XskyVtTBi
z@pLV7pBmd$6{)FuvfsHOOtH7dd^c>cE43M2loYK_MZmmo3IzV(}Cmbf)-_A8K@
zI&{E3^b#C8T7Dx=Kis=k@P1#v)?L@Hbp{-QZRe{$Uc8vrkjxEuhP}`s9iSH_ahD}@
zgTlTRl~j@FBdz$9ziY_mC10$#OkCvC!`H!l>=X+{hG=B8`tn_J2J}^vQZXoqf}p2M
zAIDfJ#-7;BqIZlnuZyCdNNc7j({YC)q$sch!``C(^Yzj)tsMBK_q1Z<_-o@%=7}++
zEC#!@PQM|0zxd17kcJT9Dti(oS%%}XD%(V;f+b}cxn|j;jr|96z!ap6?kFgqwbqT%
z-6i<$$_8<-V|#oxU@8P&#=W|kUwPuRvIRbw
zgO;_dDBgfSyN?2l{s#hQKmO&Eg|ZiTDY0@ngZM-S@s%M`y^
zw*$EUcliph%abd4NbQDMkeQB7v%Af1|Fez{x2BT)=@NyZw(6GMTc=3%Y
z7ed?rls9ZmU-9q}lw`jxTBezg*FX6xOeCC^G6Ad9XZj#m(8Vie3UWmkMExn$BqaW!=
zG2V^|ZS&b#Q83Da@7mrnZSN6b3$;O82-;6#A`?smB1+)q?V_uWg-EKKT=*D>lLI#p
zPdy{xqw%E}BU)#&u#IpbI~gtu8;wSz2{WT+xKl7gFx!7s;#;&oWi_Ajv=oo5b&m^)
z{!3w9wWloo-Vdh5DY0h!Ao-j6&oB|4D4L;j1G(v0S$0BHD3=2l$ts1T9LyJa#$J{5
zq`?+$#&lnUhW`lN(^{)_T^jn4@=KC(C~T9tjXddN#CJj)wBa`qA|{QY7Q*3+3fM*%
zjKWe3nCSErV|sztINs|*5ldGQdWj;_Dl(ZY-Y`ggQ&4EckHl1!Durod&
zk;XwRX4`tQ?-6aUuKrB~{O-xN;q>$b$Z!ky&AFVNC{lA|twq57KrNIy2iHO*;V3y{
zE{blI{l;Dh9p-@L7wRd+f-0&H4z38q&`i8yX-{R_4G+ezMRyH!(x;prp@=B_Vs=zBpNqYV!8tB{0kW^0K?kTx01b)^Fd9!v~%j%22{3guJu6`)_;xH>9
zZ3l-GGpVBHsxwSv-+#Zg>cVVico|)Sed^4DD)UbOEq5Y&5F5ujS|Y)^?mEAj2nX6n
zaw&A|F=;Xs8MfgvSz3MChQBgH%IlH@Dy&nQ44qjkn*XqLwwt*9R|!ZtZ@P(0$)L=K
za&GUCJ(Ur%DWYT5bX5l8m=`tIJvk8oPIqwdyo;CA
z@b0nVGr|nEGnTs!R4~wYpl|@(xqHDcDU`4H1=tT0nrxAPI|SaBllkid^CSvD6KQF(9=}e?N=$;a
z+KRiHL)wZezC2&9U(tAfNy^e}T@#Vrb2-=0vE%AeY-X2nbs+HV=%!Wz9;9KmO~=eqw})FhS=
zEoTd}6r(Yry+Hf?;RiFUe)Qi>e1gQ0H4Ha2UjME_Y$B$F7C11}MDZsg%pQ0Y-w-b|
zTOuLLIQopnmD+w=%w$1QD>}pUtw9d2{z}#oN9NUEGCeOelt{H=o9a(On0GrBrf1aP
z8mj+Dg_9Fye#-QI;O8s*3Vl6#r~8j(okT3v_rrcj?`n!k7dBj+>?*OH>?uyIMHYVy
z8^JLVbE*uZx`{+L{8>=jsXBgj_o*QoW1xj*7!v{$?bRaRrJ3O*Iy*K#^@fk%y}eC{kI&+GoCbA1seIpPGfx)C%7$#47QV8j_kqH_4-?G!
z3dPb|ytyDk@^!C=h7k0r=|7vV07Jy@5*<
zV(LESbQgAQAjfNHFn{yCkOMfcgfv&IaT4&hFfv*;c<)evjFTe{6!VD+vZdN2#60GX
zOuW1<77JNYbg^=2DJi>~zrWC3`ux}WY3k|K`Rp8t7#VMvo(zUQOlWMc3DmSIY0ftjdcpV@62jQ#RnAFsBKj
zWsQUpW1pXFXniL~X{Hgnx$qvZ@s-pDx(ENOQX!r}+lFkKHb5LwlSlF7&^B5mnY%Z0
zi0n27jl-;Ydgg5=@A1n!o1;EMz4XGmdIT*QN1karuSlxaX&T?(xAVFCr$RGxV&pz!D&yPW&_n$&$?%9aa=(1gxcFwmodUto8N@q%
zzP}MquxY#s>~kkLoL_X36yc2Wq}rda@vAY2Ey+pMwSA7Fg107xk4qM`}V
z>SuyLnD}<0(SD2#0ZJ;!r^5Mxjj*<{;eJ^A88nS8E!PReAU+1`0z_fK{E-kFyPfaP
zT7F0{q}fjW84+cIM-QZu0X^(>Ma7@?Ff>wJ*8jk^WH)cZsz&dvK0ieQ0G+>O3$Cfu
z^&rttL2*V8Vcm2)`vK-+x!W*i1$W+Nl_WsaK=Xqd#=ye^x<*(S%6f?zXisxDeC2Dq1{%uI5|P@
z7{gZn>qU(HbzX;S7^rd1&JK=$E@!>)KNa51yFUvCPyY0fv2?+F)nkn|e@8&xmN#hx
zxcOP#73`qC6vtahZxSo>3GF68M{&uE@wksbgvt1lrQz*{-9d`^5PP_r)7}x`w0KN-
zQYdv!>!2akM&ugZzW7TZ($YO#fgm7_OKG0yugi`_e~n4NpccGh
z<>Hze+G)#pc0ZsIk~oB^X8FzdnXA^MMvI<^j(PlQUOK=0-V%m-St0eOL+J6O?9;~%
zG%eA6J5d+hkZH8#KRgwR_Ca&ul*Hs#rx9-0O9qF19ib&Dp)h_-Irf4p#Nl{olIp6p
z)s3g2t0bhEnQp}zG708J+De08`H#J|+Bn`k+1O2rFFIGe_K{B*T`1o+T?@+z`|Vz)S`0aY~956+)qS4?yKr&HK21GN2WY?}l*M
z>ty;re^gaf;lctoV_6v+i10a|eS`>bPfvjSK(tn=c5KlrDFz1YaL9ZF*^CrYu2LT&
z>LHAlFa|VTAB?s@kpXb&NEu6r#Zpx41%n2pqk}sVu>fjH^G46tW>r_<{(;pkXb+?q
zhBuGk8Q*8E@*j-V61dnWz>c*0_5$ENoz?=dmV&gxJzwJ3$e%w!{soKUp|)zRQcaHk
zupofeT4BSC>8hF8f|uf27Xb@9*tLN&1eR{2V`C4w$PhI5{Xc*<9~Ra^2q+4SDix5Q
z=jS1g{iQ0~=*mZ4FU;KYj1)bhH={d5-gW6LhFw6Kuke3o{P7=o*;1wi2ezh1B-
zaH_>C0UtX*9|A&8?boCTDr#zYz=9GF-Uc@sdKwy_W&*xioNlMl#}q8g*JlCZ`l}#3
zQARfQ@Nj@5(Qb4RRPv_^vAcUbpO
zuijuH3J`KSavCyA_;O_EBz1ZI_eaAI5Qon@W>I?-_!gw_T}Pfz+~!B$d!W4V&p%!U
zl{wRlyR^|ub8DDdaWu=Q5@L^uaR%nEMoas-jj7Hbr(Ry-^Bap#GkQ`s5`Lh0JXY6t
zy6btD?p7VDREFn=Zwb_Jc_Y{G8^Y@R$TbX{#RXl>-C01o8Pq6uVYrSZ}G&}ZI1gr
z(J}6=5lq{)h}Vt~GYyKeiGF8R-mxL#3>_^DIAytx_7nx6NU?8
z2=BvK1q+vkvsC_p{m8OqdkE%IdWRzpx==VH=4Tx}>TP_E%24aDA+7Zb%0{a8g3hg5
zQ|LX20**#d%zpi<8YYDRcNO4$vj*;%C?k9KVC@eaESxBrrVDaqR~i6Wg8g&RHRTGN
zO!!@ZI0cDWXIB?UYk`ptYI{dVN3a@z;Ml6p2tce|iy%=44FLrGLGCKZe;|4l_6wjz
z`(|6r0hdRU@17NG7V`7wM@Ku?4%}$WTl~GGq@+Mer=yeJXN`-8Cz!1YVTXKteB$CX
zU3`GCDj!oWAGE6(L{c*{Ry@TXJa_;OsQ+?nA*`H*I|h^y;4+AbiUN0GTpaFx
z=lc3OthS((d8Q)2?GBNKa9iFc#pTHwu;7EewWp_0I0HpAh?e0v=;`SJlKuR(Odm-}
zDQY)JGwKAmY(USF#baDYkwbYE5CAF9%-B`{wg$l{tjL_}({ED4@(%Xqo6F0v5mV0<
zyTixVzIL$X{qI_W4NFByGR?e?bzJeiQ1
z&LrbolO0oZa(WPP=T4)6?S&psO3VX=MdCMgjsa>dH~Vp;UJoSU590LXeW^`2-`~UN
zhPXXOrjtfU^?XcQM0%{5OW2>tV27x~xXf=kNJ=hlJiBzU_5OHldNlXe=Arw}EGG4g
zMciwgu*ggmr2u;D6mc9@BJ?~`Vr*q>bTmx-?YrB0nUz*YX8NOl=LRy4lk1}Rw=sOf
zTj{SDhX#M%^Re~$l@l;@N_%{)DS;8YIeZ-T@XA^4p{v+a6uEY)g`KeDNBBySOPFE8
zNXuScBgAXh1w%*}{!J+vwDSA!_MeTNA0OG54q|e)f7NZ?k6_O1(GRkh9^Q^S3m5N=
zwT{b(`yJV--8Mn$es93woifi`n)}VJdjtp*wEHab<}190Y#B4C;==2cUv+J@
zS?cDi=T`)6IQWI}gad7aE=B1g=&avVzbA@`sV*jGmSB+;(<$I(n38zjD|<|#p2-Ze
zp|xpLYCd25kEcraxvD4ai*l4$lT}#z9Yu;eYXAJ*n%RMn5npjtz*_6+f!m7G3pjO9
z_ymA0(WPnYBNS7PhwUJwhg^m7G0Nn7L@`}t@lneSAWkMCAW$wH1;tNBRuI*sCV8en>fn^tXVU*F?5SSP`3*5q1ehSv1w#Qm9e^8ZO;H9%z4_JI
zBv3k5-A_SL1uk5`1j-gdu{SU+7=-#}d+sQzR;Yw++56Jfftf!zLWIHaD4--9cLjnWm?^T}12}c_8u@elpFH-gj;4
zfNzqko&^!Id%ge?H03dHci#svx-0mjfUsUyUk@*?l!jf`|DUKud0
z!&;DILB|ZGuOV^$)4r=UWu#2bN=ke~CjDi$$wP_bjr>;v+Ewp1={~y6$Ss3Aot-4n89lwZ
zUvwAi{1YB9yn5>DE~G`c!MQTbponb1_$2Z97xNL`y&9f|2n*bT-;s)Fi_~&H@xznv
znD3y5NmQW*p>-2gD06U+SI%&>y}@anvn(F6uKVasNGo>NbDaWlmrQWt&RfCvdN`ul
zBy0Qlz5EqY9GoVBvX{-JJhJ)k+t^(Omptx>1a7ycrLo8?_HFE%$h&2iX|rA)V$oYN
z9sN3+PMvl!2x=OdtFN>(X81LxBqz0k*Mzsm)Y(oQrI29Bd)M2_iu0pq7i~ei6zW&S
z-^tJ2xLuE!4{a*RbEVP#MC=}Q{VON6)=juQr2c5N==oUjkW!Pv{e5wIZ1g|gO-7Yu
zIhEs{9Hb7;&T~yZJJ5(IrHO0&Kt@}Fv7M7MZD19ohH$M3Qzc%%=`CheV;cY;;mNv9
zfe(=qf~2^WMs>jXg;W%9FFcuRe#nz`og)l567chc!0mid7A@b3@p&NR8q7
zpZ7!W3~Q{Q76bA#s9#_3KT;46^93gjq)I!(scdDnbnU+z%)LHe6BRth)P!y9?5Goa
z04W3FKUi!Rk4(+aw}BQA90=N^*WQA0@OAzD`_8EDO>OO{)7@)WJ}?b@=KRVhH!CYD
z66Babh5Pl(7vF_2EO-RK<`rUf6nbA~XN!W{50dPz-K@cF^irm#xp2DMixI~{t_Xa4
zTU%S8tUD5=JF+pFGM{*&b2!
zIM*a@pTIbcK`U1KiBa%
zV{_fL{-bX&|7i?WU$;XaQ3IA-#;ZZyW20F|%%L#+o9=b5l;Q$Y!_3Mb;aS+x6WbWh
zClzCIcD}z+^=deO|6(eVGq6mx4)qY-#A)&O?T)7mc&#@GNy3k+uPC3y$0pRTcG-B?
zCe!Crdo~`B4e?LhxizNq;q-@K$X?69W>kE)$%X86cuw^IiF>0+GzRhrtMTqOg}Cdu
zORY!~7DpkHK@OKynwVtAulnpA*603ob3R3La``cYQ1+)6UmuX6E&8>BoU}-WJAK5=
zMTo|BhokM$!^K((YBZ!x@nFjPp^4*v%xylAH1dr_MAMT`w&sem`f$CpD-Q>J5L_rU
zd<9A|iKZft+ZAaaQ!tV|+DFG$VQjtmi6oUTvYog@GBoY>&bD^!;;~y19CeZbuOFR8+;GYW)+z$EBqjqETF$
z_W-h?X>J~}yU@X&=esn2b?F1`U?BY`6S`z7(5S;0?)pPA_Le%udsS5O@1W@M0Jqc5T8LLdlysFb>om8tFB_sgSK8>d^Ae0hDdTt6R<{e|sO6YN5}n6o?!KMPrw5Z)
z_MZ7mdJlN3_6#{_@S;n^)V^%Z9&eJbT^$bv)NgOkcrh#yA*$;<{bSV7m5EtyeShrf
zhd~_0^SXI|O7haP@rXz&D?rq$s@Rq0cxL|cF47g{p<hcx|e?`kAD9Vb$(ZMCJfLk(<
zB1&k4_@o(lYbcSU!P)n0@oo5A(r!p`NAi^Pr+Z%88A^N^LA?ZLY2hb%mvbGCK2+$^
zpPBy+v7&+Y6iPoRn&<;}I{-J~{wa7rJ-_+t@}HgIC_*G@S;Se_fL&NPi>0rDKNkiP
zX=zNPxIn3g=kN99HXmd25UwJ!_+_mlMCYQbc;E9P4p4iM#jkzM?m7tMqu}D^zHh2k
z-qaKT))Ppsum9=(0JITCy=!xTwp^XG*9;jV`9kWb!zAgnN
zEGYfKoCHGup@Y2|9a8x;7Ui_P#RKu3Jzp4s1O&jLERkh2vZk!2R{84H(=`O-I{Ti5
z$uTLbsJwurt)G;)fxG1RT%lMiZ_WP2FFy$gv$>xR!$jiP9`VR4{ObyAczj$)zzT^y
z7KNRB4l|7g`W=?6pJ+zn@x?EFEf8`uKrePx`fP+J__gc0Xae-PEoLc@l
zj3^>89539DR57nNW4>UK*{WSyu7f1@Y=u%XAhKW0*|^?qlzG1Yxx|fj(|dEDI@NJ<
zbN~RAs7*TY-Ij{dO__x~kvD0%%knObt>|`90!^l!u-B6pS@)AyrzU@!9VvYEe=T!6
z74?#Le;O6Lr~P0djJD;(4(_jUd=hOjqkMF3Mq0EY0`Z@{U^PMF775A)5!Y=aCo#q{15&?-$X`Jb$F$YrEvbTlf
z&F6A~$IFG($-GKgLbyq28WAfUKAji*n2wgLh`1svs}7#$51bo=M?8{+FAq!$&#eer
zZ%48yXRCTc0tbw-G_LMHih)MB_mR6*%~Wg9KQsT0Qt&x|(Su9N8+r?H_y6A73Idxk
zxa>bZF^0?ol~T@Ix4?OC2dSwg%IP4kh6nOe{~=8;gn8}n@58^qbph=`rTz>^(Z|Np
zGcrmh9KgH^!wWoIT;DuN&vi{uwnGa7rbcnahvj2_EpyNxzt$;Fa`?9^o}ZnqJSHTK
zTy$+{Y6AARTTcQdaJk`@v7}(TzVHarSkS_xK%xtElS1(=fQbW3d3_yRQg8$`fs+Jf
zSO}iUEj9yW@!hBZ_+(Znb{lVk864RI2CAFy8QHrfni`hFgH
zPP9wk{v2)h8QbO8RX>T#-i@L8_s-2u{ry7n@^-DdGK{}c6$yW@uN?VQ@{R`CX#bVK
z8(8k0StNY4!$2d=rO>aTbt>QEmw)rN485i6I+S}Jzt#PoEts6;zI}DzYh}3U%GK2|NLlVZ_O^@c+JBf
zaAqR!?$8mwiIe7VaYC!+eHl5kgT{c2jU$_t!sY2jLCSE3azwv3+6oM8zjNoi)#CX6
z=lzrK#a)W@^A{(8R<5Hs)~{7Czc9~~Z$waAzKYUQjdvIDn0Wh375&%!3kvRZ
zC_`;96HMt)!z{qz?A>(#)-8LX{5ulVK|Ois(t&dvG_!)NtTuZ~-C&~H1v_z-_G=5jqAEB_hfdzjZ4YK!knVal^(YzAz90(XU=*;@`LJXS_-swJEoGVsw6Sad82GB50qCw&*W+W6aN&$YEx1U9kD(*->1F
zi@MG14FOLQ@gvn%gK@;W4U}efBp*)4IcXX->-jeV0ZYySqi;M2m<~vfSm#C)N~CE
zAM;Vn0Y(KLbkGvXz80sP82-B)BbS%$h#I6?CP1OU-en>N{oUBGrA=k%nj0b
zk>)m0cL=m67Es1Md5Z*|xB~K_ym-TFIIW)|tyk%@76NnG#4Eo?}jc=i`c
z_IJbNfu?&e(bhedzVKn*|DuajPN|UYH|2O9X1#dJrl(tpJg|l{Q45kO45PmW@txkd
zhdQ80u^uH{>R__>rZgiZ2}5QqoFFgZ#JZx;rYOmziWgM)M4*VxKZ@ll8qxMLe-lcqA9+6JBcYiPgeq8Mj)^
z2hXhM8V>GmG@iw=-IL}aL0(=*3kwX=NU&kMCBA4ttG4bVuAiYDx@k+%
zV=Vy8I9slduguI$-*XLk=86+uLfFBz=^w^N{r&!@pofT##>WI8CzSB;1##?x
z*R>WM9UTt`$8)%pLqa5b-M|w9M*t)+g71SR4KCShPr2$}m}v2uGz?n{Y2j{m|Cr*6&y?e9-z8%
z0oB_B-j?z2-^pU-;J|_!l89cy1XKV}`Zmxw=<8>yl%i_t>K=CD@xobzff|;UmgbAQ
zv+n-@;A`g&y^r1^u%kM`1ErQPaete!8L|xE7Ao^dpcdW+R%fJBzPAL{*9CZBN7{d|m71S&ZX%j!fPm@t}J{%oNs$bKkRM5*veZ
zt5dzt3msQ93JG0}cTb89`|O}3Q6p88A0>A>&+TA1zsKK`oo^*Y)k0YYsS7*!zCw8Bin~kjW#5iQK+`{iwl_1MJkS}GG#gS-NNqds
z{qwG+pi6PJBQtl9~3*B
z+ZT8eoVrrzRk88krYW(Wb=u&uOS)IhI~`F(N7X*KO)_v0c=ZmAa*TaMJ9f|aQS&M>(Ea=y$Flz_01?8Ood2m)+8G@i0JUZGJKhm9;
zt~1qld^%W$vC>DEC{Y|K(Yi_3Jc}QNEletNSgKCGEm+VI*>7FakIgFYAFlgGco8Lo
zORn)Zo;erlFu@T0Fem06tXuc^JHxLi(3r9Lv>BO6)O~kv7%^>}SUga%$!21Y&=1b;
za6%1R007Q)fKHpk#2YQFg^CG3NNQQz>Kr?UIZyG3YM!|0yBR^Tn`&d85G&QZ=gY&}
z>qeIhnFtUZUfk65FXZYt1O~M^xw%0rUv8u-2|534@O3n*^oHA#rWYrwa|CDO@bJ(#
z3MH-xoZjFC92j6$*5)8h8jM*lGo>6VRg)U90djuQ
zt0BzE!#jKOaCYoBSda}#m2qiCj*X+?F%M$(~3|1(-K{Gyv~+ThZd@btnYt{Ws%Aq
zqk}m{gTgWi?gPa0Z%}g))CEMu3uMv?(luF_XNInRiWqwGdwK=`9chw6k{ZQ%ud0I0@Xl`6tKBwhB)%MgJYQspdUY
zODPc-)vC}M5r}2l8ojM?5O|zvT4V8LZe*;^-mcvV1Bn-vt(n1q9#^t5zDMFm$0G0G
zf#+e%jqHEdnr(RXciw4sYfVBrc)m5MJYU_CBFRQW47=%{&(81p39BcJv6`7w+EnNa
zSHwH=Ifx0uNBuYAXh#6z9sr#4axHGgB3q_7LFWwgExdT9_|~vc}g87hkRz8a2I%4
zYNr#Nu!d6g@O`x9b0rq{sH4KAgKzos!Hs4Tp@v+)JjI~?k$$EEG#qa*TYrG9g|IMO
zEE;*@pQ@{O#VfT)?>Lb39$jt>eQGU#*v9U2h
zxHuqz2hOFk{Es~<$jIHLvVg(rS%|R-?g>Bf0kg8#yVqqZ&KthLwTc32~Q>l2DM5iNvw-#$<8Tb=HI
zlYO=G59R&ViTM>PGT!y*9b<1bTxAQVc~1fe?;o?r>VNB|h}9Qd@ziej6WjlHar0(T
z0={VI;-Zvn%X`lHlh=D~lzWNim6tAS8*ambCvA#X$Cbb8|418$k&&W=jE6j+)N#kT
zxyVCej-E@n;;?&+iKxcta0hKcF407m{88Q-Cs6ks&7@Y
zPhR8VvXYve*CzXX@jb6{F;f3fs+K)`x%j`T<8RkKBsaVVSqTga+$tva?{4(
z*IVxqNET&Wbe-0tpzLkV`Q*lWv$p}8RT5*^JNNAJVg|gl=@o8i$6;6@l2RVcx3|*S
z4dghLeAyfR%zNI0K%nx|P|3s)7<($+I&@OtF&;8TIK+NVRC
zyim1vM&fV{02Kq42&rUOr?I?_3GeJ=G1eVcoV)cCRr)Fa-@(1*NwpYs@`OwTjdp
zWlgVDgsgcYvIULLFE)@PIH@1YbwQCLE0p)_Sv#6n^^?wp;O7n&_E7^&t1?ysc9#|V
zGcFnItkz$juLv7Aef(3=dZljUUs_K7*Egt(P2OLnYt!+b!xUKfv;@ESgpzK`HW$-WRh_N4@j)3ZbrSjBdu~f(!>tA913+
z8=`MegwIPW)Df57L1wI#u|BiSn)P_!$oFsGAE!zDB7pGD5Y4Bb@QcGu7XgcI_zZk#@9ajSE`gnZs6QO!zn~#7dv3
zCeq0BI1N9hkZPun9yu2)XQz7a*+_{;-_S7~8}29;Tg_&CG1lMzZrf0zYBNMF)}Vyy
z0wRc-d`ky-DWYWDCU}z+JZe}U_E0RVP9d$ob1Qio(&j6tX
zlMab|z`=Cu1@JaiT@yWQH_hL(IiBv9j2R%DKJNb8Ies#8S5BvsKH*=8K*wq%8Hc<;
zwx+)qE#f`BvHx!H?LzEZ`tpLdOsAOEE#V)tl`@00=B)+86)jDpzp{yh#7C?9S}iCb
zIU3e_?8n~RGAiKZ{`%b8eOjkPH=*rc=GPxORn|AVzkgr-vr!%?1JzQYiT9tOE6*d!
z_5-F*OrNX*Kd(J{O%cHmRj}LacK+i>T&K)471j?wI=8lXmX$a-exYFBS`@zBc8fS?
zHizDmXnTE@tt=M*X5Nd4O+2Y|5pmBa9%9CoFn;hRlDgdy8t>X@e`lBU&9aZ^&05a&
z^j2h6GDU_b3}u3L;=OTHuB
z7*Fm6U>02>5N&-+^;0tH^(`%cQ`itRLTA$Yn}IPKR{0_!XKB+pV}rQ1Y-q%GEyjc}
z*-eRoG}rmU94Gi~_9IFjGRK+2%l6bDv$2%duLmb)HmUx2BpzzzZ$=q<$|kL)gU(GM1EtqoD82vV^7a;r%ttStt=Kv~uc(
zK9c5Ql&}3X1r~ziRDRT%_xmu=V64gi8G2k9(SnlX-5}g4)z7M(l)+Z|Z1!XA!r7@E
zjr;>vzxXv1vt;Hvf;mg(&_(8>t~`wZr-~2ZcZ8@lP$xcvoiFMEu*i=JIHBs
zt8LgbP)PJ)i8c<(;gR3w=2m_l3uGU+gWB^EAqC+${v`Hm
zUflz`sdyf&zXVGuD4B@x8v^S$5+k<7tTSZZBkfyu6*^uZRiiwAiavAaB@qngk*h
hV<)C)(=-$aN$s>IW7|}5Hscv{6tq=63)GRnoyj4{lX6?2
z`s&MAhn3$)G$438XrVm}LQbKjy-7+6HJFJRulNwGN$P597GKV*
zsw#2*_?@ml7w}S{sAP=zu-ngI
z?nY=wkfK!~UudR}#7Chz6zpl5IZF~{t;XhXx7+&bf;|M2^xs~#yt*3BYZi@1@!Xz5
z9-X38vvj1}wV5j{Kb_w}(?%Tr^@r2C{MP88!*Op7))uvxFqB(WINCEuQO(*bA-mI9
z9`u4mU4%EH6a;U?r^F&NW!|GclbAxFdE_g41}p}dn-6e%X5E%q>xO!)6wUp)UHX1D8q5pOR6Ju_XEHPYvY56(a_6C+
zs32~dN^dI5{nbl)gc8@MqObt9E|ey2o1f@+sl2GqOl2<+;~NiyUq;17DpEPz?}`Y1
z`(t!&
zVqnPnO9=E9V3kT=itbzIQ-49$2h`Z*$II#Qg8tT%9
zf^WP6?t)u_cSy!UXpwAKT0ZiGuClFxH$9bJ%Dr-gonwj_Cw?{#J6z*l4Eu{B2iu)*
za>=2H+4LH(y1P*JgDQwV@@*TJJ9R4H1S-E#7@<}9LWDHWS++po~JJv6L(yXH^-uJxtk_U$h7
z7{!b14>^$j+S-O>CmP6MzllHDcCMZi2^dU>w}|ehVbs=vXL6;ZMCBzpoj|w28Jyp~
z;c=IOW|s^ugr_%S?TO6%%kEhVZ?gHEO3d7*O7Sgbak9zyR~ehl^pgityR;Poel!Gz
zSglN3*n^j?lge#q-?Q7qrP;tDuD{;&RP9uA;IV2BLDu)?-go}V!3llywv!@>+GvY~
zM_V@|CZlwwvZvfUUQ}uTFV9qIn9LTNjOX=J`9TJ~c3Uwo2Mx3Tm%(T~f}N9LB+Gr6
zm*^X$LuZqPuWLfrtqqrrN;rOTcuopY4f4WzRHKj)83LDRV;g34y}2Dd
zv?Y{Bce0QWTUV6C{%w1MOzAot4suE6ReS5c8ZCE}fLFf1h%F$mY{`S1P#y*8=T~-V
z9*Ago!$YqFIBGJJq{j+B+A#^7g^_c&T!`MDhb2?MkyQ%}MU{%Cy~}Rq-FGB^=-Lxa
z8&{jwDL2#nTO#@4(2EP~1=9xPqH(Q$)+ZuoHBBqV;GOC7=6RzgNR+JsUD7=fFA#nd
z7vBPc+WI^Q5si*|fqFC(t5Gl(eKzczcRjaiI@FMbNpRaNRhy|)%3tm*y6{u&O@_rm+=Z|A>=8=q$55&??QAEW?jKYMYKjTK@UE3ZL+TGL8t{
zO^3O&+`32SL3KMf9-%mif0ML}mxaj!^zxEIzZ6|u<(iX?0Y4tjWF?(lPe{Yx)<4cK
z4~OrDdEISk5A)sS%!J)K|1`7v8_mr9ueUVtXK%wFzg7aHguPkCT5149B*2nO4C_`P
zR%ZIIN~C{;U%aS+_%RW%&Ec&eHU8XjHNN%I2F
zr^k0nkI~fqGT2`0MgpfdEmEAKaVPFp!2$2*mzIHr2qlgnw1PB`sZ^|{g;agav*ZtFW=vz-GyJ@pYT0`
zQzGfpiS_RtQlU8grbBMlPmPY&jrjdDLB4
z1=lIc`yDR~BZWaG5((%3xu)D-pI~_|__?AMp@pd_$|P}+2z9U7?@ap>^J3@VDR+E`
zH5vYH$3C`a;DMk^ODH-9|Mu6=&$77T;tfrf{Sy}V!xLklN7I{EJb&<5DDI}_#zyP*
zzJsgptN{I_`Q$7Kye!8yccd#DAfOgcx8Nyk~_d!+OQ(Ancej
z4bvqMX$+kg3sfi{MQz@9hptfZQbSHqgS!aKDt>GSOMdXVKKLQrE@cdETSO>K?vEQ1
zgWKe$K~X)O)~up9)e;ymBDAZ+nWm>{R#a*uNm)FG?-1AE$RvC#${&>p_TKk>IRSp=
zh#o1%zwidc!Rz&-ySgU`z?1?k@8p#6{nBw5Aze%Z0C_*E1!Wot7Rz?!(~e%h_x9l;
zMHCsYR(tc0mxT6rE~AX`!PLV)f~`fg2ap}uForVvX$vcqW~McYD)OE
zapY4|B$#Kg-g`eCE(#UjC$`_rFHx1hf4jbxSE)0}EB8;|&9FSWdE;ZMfeu3@Y
z9l50>J5+@9@#3_btu($1edT2vblhdN;n|*u$hsWbtN>mRuG`J1~*J!CDM?t)S$kZc+%MhJlCz
zEp6fVwVB_-i8EFRCGDDB?jQq^mL`{1Agh#a>+Iw{gGLJ~4Jos7?FWUY&=K8&j=nu$BLNv3xLFl^7NX=MR~($a$c
z=NdQGOb9QZ?sw#Fn8%{c%J?4~BzQOvuAB+D2uYJUPw%t0v`d(Y+HOcEG4>r+*A*>y
zyJHQH1t}o**$aMqZ|nc$(`y_Hx)KSZzZQ8R_>z4FA^>4EYLbapej|hOp;@{_apKc(
z8vDPyE1D?b)WeYm!w^B59UbN~GGt$v3`B51Q=9+t>GrBxBE8K~$T!ofsfp(Z%cB{F
z;-USOpD|#IFKd)?oX(=^%*Nv&*Q~wnX6vhG7!3(I65n4>DRAvqxg9sr-cmK^Q{zoA
z!rP2AA=Fu2pkbAH2gv`TGuO&U&CvO1ezenkwydp}8Mj_XT>5HAo(s(v{Fi1?j>V+$
zRdoC&|Emvxe}Fl|!9-72esg-a;bQq!NyP0hHuZnzgmY6+h%RFp?u;blNnY~o6`FUi
z^;)>7;M}-3K>VmfuW4s}`6-F}wPS8aB)7n$hcNGJG8g+>AM7M`QJ;fDd)DSO1A~rX
zvD3EEDxwxL#U)LwX0@NVp^{;Q`9<-J=t+MWo1A*`Ylo$Bb<>0xnhj6>>fdu`SGP}1
zErQF9OPeTlJY(F4NfJKzY&JKY$$V~)*Cnr*U%qZ(CB<;`szikhS3vScOpF`JWDrx7
zpuay_PEdOAzlOH^F?SlBk_sN<|C8a8K~R>1HF3K)dnsCOi=+iHC8528lSREHwvN*a
zsMzlf#jc<#w+s|T{f4*~Ry>$9eC_n_;vFHIBMSX*k<oPU5CN!_-T#?S05ImK`Y-D*U*>@;X*(0-
zoMv0Sn)=pn03s11INlm$GJs$9c0m8}v?}Xi(0~p2sDP~z7^2)pb0mShpj~JEiiD)$
z*`PQvIr;7pRNn*zxhy&*gYPk(^~lA>cHBk&$i~311Hx&*#CyAn?epW~&qeDiK#f@f
zt;=g7R?U<^34)RU%}%0pRK!(UC;MWy<@=vJL}!5Y_6!N=q0@)
zW|}%7wi7=&i_+=YyMdOrg}Iiuy}fZ8u4lB`jd9~TvVOF|UC0MBW0;Ul2x+85kwhHs
zNCf=G+5H5@@1lV*iWXw66$1Y^ffmGmn8bn*P8pT(!3mPD?>5Z#PBcqblcKLPZ`b_g
ztON}l&JSh;T+4|BhXW}E2M)b%5nY~?J*_q9W0mR;ygzA4-4$S;Bhny*TpfBJqGqlS
zmPRq*BMc+ph||0mBNig5POP^gv(AA*Vb?m8frzCGd|f^e-*v$c1!jYr20m@9R0^qu
zqpIDh9co~qxOM5h7F6(W9HF6OeaNhyJCC(_YCUnyGYUFK|&E2VMy;o^9(VDe)Y
z+jq?W!lR!OaXkp
zw)%Pv+IXO()qI&&%oMxk53!$tb?-mE;b_g=q3CsRi(Ob+@&@@Yg>PJ_zS7k2?Unn$=nO0@(cRtgq}Cu9DIUCL?K^)k
zwRZs(CF8RP&A#!}H#cTE;GWX<7x;NED2Xu+7CkIG{D%X>Cnjg_ZaoRLQpH@&;Ryl)6wh?aeO=)-?C^aFk%01Hh7C)8mTZ4Y`XcRrKBxHv`L-$u|C3k^
zTkrxtS+OjF`cL8G%46lcI(s(C%Lw-Da!O_r5wV?o0mw&>yZ7_AF0VtMTnvs3+xjUJ
z7&f2wdBpp$zf{`Od7T+L$Wh@(ag}(z+rMzMenNq~jDU|3sUmJW@A@ny$6t)jrjOQw
zvlZE3h-=)DzGY%z;qG(EIn?g!V$eP;5vH-hss6S*FmvH-1NngbeoyLYl_F0+&iPt$
z#Gp6p>uNhp5Z9G*#V8>uh9;z&<$Br(4wC~)yER5Iz?wwMd)dhqDS}V--Z07!eLTke
z;zUVCD~8?JeyLMahA=<5@E>XU95bC(uAlPLVL--9XOrQy*XV)t6d8POA9t_AL_>7b
zwCTa&#o?4MiSQ3gX%6?Work`w`ya1m7tBj%uS$lsE7j_esYwfKa}3W!3&*-X_$4D8Gxci0W
z#PptujZ_t)m-PnQuEfVwI5#}jNs)UvBHctfak_=0+%~eg7B=rlzn3vJZ~>I5gQQXH
z?B2UyTXipn=J}^>1g`JeXcGwDj4R4U$X4&3RqwDzR+s3KE5QqS-L15C{aq4u5RZ_~
z$LrrP3PM6d>0Se)1n>`9?arx8Z1eHz8y6<(C#aS{^G*6=wo~&buz{L5cowh&5wHcn
zs}=zp7ZhF;AakUtW##2{THN14f;t0iuAu4)ZG3Rj+X~>-+k3W9^Y7dO64V){8n(x*nRL2L#t&gh$ow
zv;a5EMcS?1K(AAA8{V;42na%Al^Un#?~mk^rA(6v9ve^
zN#q7^a=u4p;lKI<){Ci&Fg{wA>zSA7k;u#ejkIb;Vyc>&OmyME#|7GHntG%=jG7Rn
zFP9+;ELS5U-CGo*Dnj9qRKKwCiMX|egi|uJoll7
zZ)=)Uw%Z!F32)>+q}9Tvd5rMBOvr<|`F6_cG5d0;W}aF~bQVfOOZbGzd($lCH`AZD
zl$y7)C&I%r=7pn&=ibDPv$lHLbnmf~378yMre3z^GVd)%-+Br5U69YcDo=%z5Em8t
z9S)Q0t#7!tgSOIorbM2S4dUrl4Vps?TpM&Kabm6OuptALAi2HrE^}LafZV#O_UW^b
zgIlqFR^eyjkl^xK5+v3Q+?_?84$=)Y(Qz0YiYSqOqoCTWeHICbW0!cqU~mrVU!KuP
zxGU{#XFS|&oz}0%u7NoTGXFHuG&h@pm4)aHr>eq`Rl}L-;PnmFmc{Js;?YZXhOrMd
zH?tHWU#@?j#LfltBrhCpZ}+FL_fL#;Ld7BN_)w)k7e#zkM0p?8_Rb>UaHwg(x*(Z_
z%2-sMLMi)qG^_M+HIf#=$?n+`2`DBV9UU4>mq$KNch49;sa$aS17|9I&pHFP>p{+8
zCJ>S2=
z5nbSB2BZ-HEtz~q5dujE?1I4AAE`7?M@tLf*`AAyPQ$`ap6>3qAb|5NCZ_$&6Qt$A
zn5q?&j(|=o0s?|aa$#?9y8-NyE%IqlC4h+(AWw?`=;>gUaT|6*A*w}=J7f9Jhu7yO
z4HpOqaIS6f^%6fBJ_`u~^x^c(4-;#A?h}AABAZvqY4Rei(PP2&K5_lm{`Mk8UcHgm
zadudW`wnX3QOz3gjV^>X#ppv45i91e7mPl7O!mEmI>1L?FKZjjpy%Rj$c!tCN!Apn
zyKxqi*K6m+d=S$rZmY&;o8;L;bMUA=I3bpTV2~G8)jHZ@tdo#s_oza^AHNE{@ZGt_
zUBT5nck7FCZLiO{!$xcH?06>dXc6VrgfBm@#2&a-;zZc9i9-<~xG-T4#Ust~o>!2}
zzAcQeS5&bHx^D&d&!yv)Z*ll0!BV
zoRZHDeYkU94PyfIf(NN}6x698o4l4$wu1hZrlTnsJM1(3D_DLE`c>h0W3R}q1~NIV
z{(NoC;4ZSi(a%DTdFR4U)7+l$wyzvqie!rE50)K%xNJS92up_r9(o6y%%EnC;-QB~
z+G{cq_ooPUx#n0w8O*>RHcekOSSGS|Tb>Q8kJ{ISwZ
zy_qTm{`AaaN1Xf4A&Z1E(v3#Qfvd-V>^vv@;CN=}|0A3FSQ$1=ha*h{|LUh*7aEu41ZAxsgYt=H6A%Ih84^yqJ+!}?
z)(c|`*1WE2>M>uw?2!s8bC%!544Yl1SwSfEe10FQnQr{I;F&DcR&TY=U!nT$MteEY
z;=M1JvUPq)&*hNYiYip-zFlDkCirvoufHr`aNxf*(XC
z4qyba1&>9Pl$302eoa~|0>GP<)d9dy=qM|Ts7wT+;)7@o2w|O_odLH2rY{U7Ou`=A
zfj|w~Ptek2$(_Y(m($ST1};y4K*Ia3`w0UC1d^CEL3jodDv>kV`HeF6M`Poh0b9z@
zW^1bu_-&{6Bwt+Jy#msl_VbFO8vK0L8%6$6&GYu~=6c=FY40{T82&18C^enqz@Hhz
z9Vu;4;bkND{^4?mZJCWEpbWFTvB2BM6<&`lWN`IsK^$QRp
zQzAN%hfAtc(Jy?R@lMeble=$2x+bxGQ?bzQDk!Fy8mm+3`uEzyS|)?{@pSI!&cx^U
zj`ibJNYbV=M4o<-6UN{N@G;LB8e;nCFg}ukOYA%785|rF7^cx>3N`VhJ1|L>z
z>wzb0&yKz_t}LUUq)CSK_pQUs$OtU?>;#d#MocRuA-wl-Ju>V9R(2pi-9^q1OJxQHFgt)T?T?#W$s6j+m
zjody7;>Zf-075-H_3Nci3Em|6eHw?n2y~D3%($rA>QxIvrEE1Vi
zgy*DU)6;O=^e98c)1w8oxMFo1HciIQ0XgqXQd?Ev5_viv$m?0;QdI!Li2r+%ybA27K7Q%b5IU8wnb9Rd@N{l54Ll=}=iTg(UFQ
znR}Q!SFYYGK9=IEj>6m+E6GJBMo)n|H8;Ej~|Lo-6FyC&gWFiSiPpa#m7
zD#9ObM>0OS%@&uOSNG#5>@YyiC2#EZp6+>@?%=r#X<2_7KTKtd*EB?IYJ8hNc=fJ0
zQmt*aK)yfH_1y}Jm$#j)q|=GhPy5ema_t1%0
zUJ6$Ja=1hCliuGxvM0HPlW2aO1&k$eqRGB%MofsX_hKxct@z1896b-@ob<|N=^7GfQx+-_W^eA>93oBjSaL^ns)EMl`@p{x~h}GiR8v)Bw
zCu+3jZ{p&9wW~z>J`e{S*xGa{GwL`8k{q+M_n=6~hH~M{0xz1PDo8i5Tb=f=SxYh!
zWsQb1$*{_eA%xxrJTzg`{+9r;b-cXXpZQ{KhWj%iL7LzKF$IReLgsuoK{6YjHvv4I
zf+tHa5znkqn!MZ`lBJ#8Aq<4-ACN2vf(>(Gzx-F0_FR23ySzNihT#kF%2e;8yN%d0
zx7J5x?Hrc3&a7rD{{dQbzHnO6!pI0tm;{KEK>*{uZaxDf!@d@ROmWrxL98?sOoIUG
zRxF#1D59pe_VViLE4i>9=$@V2O;1mMZebcVbL8R$eTEM6%#jS*
zC9W{JL8%l5603lfIt+&i918&^x~mIpYWL+IJ*=juz|Cpk*;7u6i-838wx7j{bNj#A
zmh8@dBu1coB+}K-uos#)Yk!AhNafVq=)?W^f%=E89*REnjmrML#Yy2hma#Eirf}p(
zMw9PB?W7wID1eQ5jz89oq0{;wR3KTK=Wb;#pUe=W9W@FQ(10HZW$ATGD
z*bPelXW(>seiupf)o|nZNDudC;#5*}`GCW9P#elW`{D1g)QDi%9x1~q#!{GH6N?&>
z>z(ECgB;R>isrHPx#~Yb*HS8hm*fAgxt7)ZHX?Q&{v5;a$5JKDX?=+`JLO%3Lb{e#
z>(Z9J*SoXMxJE+lv}Z%N|5%i`QEl59bw?0~aKD^8fWQJ^DnD9-WI!2;-5$9J9<@5h
zgN?5+)R_uNMg-mN#R9XfQ>^$X71bGeS499v3uO1=fXXQB4K|a6dC(
zGRC?N)0;}FKLO$JAEwJwOhd)c)Lj2r^UqvVhsQWFx}YV<5u!yHH)B?LZ;RE=qcLd2|S!s~CY
z%f~=Y7XGyL6flpt&fRv(`5Sog3g<%%s&rKb3y#InCc4kpO#x+PyB%BwMF~rkJ6ujm
zNc-0rcMJ$ycn(ROF5QgFj8*j(ORBhS`i|>|)4)@P<+dY#D|Za4vR2UwEpx#o6TVUR
z?HgeL`f;gH>7C@EAE*7ts*$6>SkPlni+3>J)M7tWhvJ`d9Tn24hfPDU`Bw*~qOW}x
zYnb8+SMmi;6vwC{rNBIEI5{Rp8Nen|Qb5ui_iNv}a}y>~*Zs{IRcvU7LHVpD2;1;_
z-7bTxA~yEk#Vw$QL*677*Vh9>s{)|qBuP{A+pS3eWlj*j&!_?no}#33CoMpT7>kS-
zaB_cjKR^RIViFVoQ&xt>raxp;5TF6Mk2m`6Zuuz_ur9}|CoRBb2-Lc5@hpIGRa@55}eMk*7V*+`RJj;dz?h=CnSVeQo5Pl^O9bh}S@bEtuqm3O5X=D+akQ
z>N<jXJq!93MxQt3AwyG}
zF4{{&|3#30BYvzu>+F2m>U-HCitxA?X^m{egrWEbuKM+wrgSQSLj7x!B6TWZClqg@
zdu!tP)j6m*PNXj$aTtsd&um<^3S_zyg9w;Y9j5PRJcVxu)`i#flUD=713%og&ScRvn#hpQhl_k7;13T
zdk7`4HyARD6&Vl5?BZn+hYx}HqEP1geIBJ3;8;^sOt{I>i{3g+^A#NGajtfOOfmp3
zODXM)KU+hQydSR;ZB{p(`g=TUu}c(qQWVx&zV>isoJjOoELZ-O-QYUp*Z$OQVdViR
z44h-V3Rx--14r(Mcd>nhOnMbFd(LmJ2ALLVZn_Aa7&ST*lUHj3(
ziFQwa?H4-fiCHI`b{ib+sOifE$%f|;vFa@>E@s5Vfe2Fo{%vR?&He*<=K$wx*>m271*5XSz0bK>(-
z0zWr3<~a5J!|A_)XN>v)0mMMFwxpz_zWy2%l+mJ4GBaC&p;uT~7*HrGG%P_bUs7`V
z_+vDCZ*u4J5);e*p3E||!t1>^MZqnmr`~)VX0DX5!-9iwrZ^zLJe9Odu%PEFV=kR!
ztVl$befDZCs(l~1uM4kTfcXar5NPd3cqr~P`
z(sx8!QRxCI-&;v}zQIjHO)*QPpu2{`Xg~2m%J(l5^I-hIy+Qa#rjJ`DNeiyJo=+#s
z^%rMR-C`891lbZ|?@g6FBbF`K`C4~|oP=`uT7&YSaA};o=z+R^r~iFvmZEZPHS0(F
z@ZsIdsKM*Q;ftG|SYLhW_^h{@od2rwTZ5xt)mCS~r0wflKOg`0R`^_2Hy`u*XYucb
z$sx?&lygG+S>ZNx-;W`Bv=lL79;NthN^}_5i2N8?v2mZ+~84L&xnxyu(_XYC5VF-EBwx~`NWTGjaKMa9%M9V3PpNEz6hSsfy&gFt(
zc*{;l?Z+T0Z#538$ZP+6%rfqJDID9FQ8>|<_>tns>WSTO)j0Vq4#R~E9@Kvv6(ZXS
z=p+>Re}~d-pI)sW*2hLkPzkkZl%i=^dPgW^<_Vu{@`G?sO@pz)5v~^P=;jvG3d_Nm
zQ8cbvqF{jWPYPEq&|s|6rpz7C{#e&b#@?IfQR0W0C}uJw^vBD-f#iKuY$S3SnwtG1
zK{>bI9MkvK)=X*HPd9ogx_XwgtX~}pf#IM4MW7NSJwYjGgN~5ZTyeTCTC;WE9XQDY}z}7vD456l_
zU9#(YHIT#{p}hsv5BrAu^SBnU_vK2qN@?pgI_`i@LL_8lz(2OPuT(62=|o)o{PFi*
z>fOyoyy+`TP0d-enqR<(Y2J|U$<^hUNR`O9}yWY70vcbko={(vAJCLg+oT_u(b`YC=0^zllL>D
zjeCl2P*+*ZLe>ej>=|+07V413Zf?mNmJX6BqW4mDn~Ri5h+%2-n#`p0cIhbK4ZcWY
zY7w~n{P!E)F9(70E!&a9eT}xsg%PoV4Gu4V4(6fJ@h*1D3G%^a!%LANAqVky3^+JU
zslQ)*GD3^cfJZB~b7@w9PuQ5F*M&#vVhl_WO%a{&nu_Q;0|O3wKJdjLxmQ-J42#F^
z^`G~sYa2hs!a>ZD#XW2|zc*Eas*-Yr;eW241=f4*CR9`zX(<_G^%HgyG_{Dgq%5v@2-iDc5{#Tbz!|**k1B>4~|8=
za+yRi9)s3~i80dG2sFtiYRY3tTs2r2!pmSfC=Y~PO$Ex!#_brE&6Zn(o&|x4^x{;m
zxZ8#8_$F!??Yn8bL=qI;S@6w)HXSCBuhNVhAL
zwH*F3A9FxUv=!V!;eWmONI%@S4>x}MHYMI^RS)SM%-4r&jXdhRj5NLD94fO!d<+|*
zOj~n(LrX*;qHtt;AMs33tjnke!N(Vr{c{^7k%BkZ{*~J58~QHxET)q$S+m+0Xz}cv
zuUu@F%mux34jvSMr8zbi6RN}>TQ2fOk%9b|>_BZb55kJZ6c3l`**!WW*&j_z6dhqr
zbCldfI^ykj8tqK%Q0+;POEnJT^29f~{#fp$RdZb}WVU~Aeo&qoPE?
z1|P#EO-Gq#FcY>aNdLtk%6}1)iXl|JnoyeSO{^;WNwy8auSTqPO=CVO)TWjSk#f_!
zr75SECaC;Hit}xE%J)>NFo>vfH5)~nMu{rIkNTTK)VgIJ$9>zU4qXUHQ%+U$y>QqZ
zRkppVdP)@dB6{?gA=C5k?_*~9u<@St83u_-9JGdual58~Z9S>pn`BuSAs$Y?PyFO5
zx{q0Xb-<34DF-75X@YcAEoeR$<$T>h7)uiOcP9a-mNCld3ZJbtMm<|Q0zArO3?cl_
z4g@R+Ih;5I;Tv)vK8h-}IHb`2GjDX=Je8H7*N3s|v+einWKy0pi(eZwc|UjeDyDei
z1pM)%WT({(9?V+FAsZpCC*~zJLDvx$9kPx*ZkNJ{BcM@Ihzt6MS}iMJ+!^RG9r?Jc
zWNNUy>Kn<_^1$t-l4;Y8(QZr(oR?qn4xyqrP?^-dIgC3C*Uae`L@MyH{O>$2bKBC(
zHWPhuZI;dkmNChNGT)kN9bI|>
z=cM4U!~KNKQ~SagudT_iX%jRL&!j1Qs5Blt;n<(@9LzN9jxV}pN`W!vu@5&%=F;|T
zduZ?fQ1((t`EGPL7|fNUieO)s7yikQ^~K}IW-|Q%{|^odkI{7;KVGm=>>yzIig}92
zAErv!-Ml}5`s(t%UAr%jqVM{s1)>-wDyuTv2>WS+bVd{bM9z-SezVVDTqFUrhtbS~
ztWPmInkX~@O;ow0e~4NspZM?Hg;K;vCcZ7gh+b(e5gdu`Et~s30R#_h0S{!$Qe99G
zmnYKhKn6)=Vc&P6BS_K%gFNY9j}vZGz{(;Kj^5!w*4OQtX=+^YW4TX?YO$&I{MEu_
z#f>-~yuxP|e6;@W;~}RT+1)4{lJzD5`{}v?>m9K;$16rCs1(My2(Cozcm&vg!?6V1
zb%?ogoj4WK`NNO;{Pgy5;n2*)P=auiAgFZWzjJ6pGc#RE;M0A{%voH;k42afW!g7B
zYjWeD*b17SXiLL?QK!=ON{ZibmC%fIJ&Di_Ex-POz!l~dP>#%wJExzsi=@*X2@ipx
zm7!ZB-0#FZK;Jg8!TaKkkqs0%a3r)-wZ(8zgd|&k6)TFc3k}ms>nfbCI
zBI{%B?;3ZA^&Fj&8)B=*DG*zcWO`fh`>XDd{IUw^YD(Zm?vnyNb0m`g?tUbLaQ#tS
zi4R2lty7JU>njl^;@#QCm%gp#uwC?<(#_alt*l9C`|PrUc(I7;Jr1tP*)cvA6K21}
zf&()WuPpiH)p`u6x%bSQiq8n%p1oM8zdynYuJE0WTyr1p6z#`0tJE_j)Vd}or
zDq$wAa5RiQ7nr;p;LXa~_oWJ?5-*1;=TVTq__*+|wD}FyJ1lL?5(vK%W)R+(c|6@h
z!@w76x&@PQ>Xf
zikU1Bu9^$%f3e@gX~f3;8lC^(A>^`rRC+5K^W=bI+kq{fV}#o=aW&XTXF2SE9csG%
zUj5%2xZ{rfUS-+>=<1wb;m4I8y?sL*l6J#a_Fm=n2Daaa{cAa~)F4n>WHyZCS11UG
zA`Avd{C=FOw_9*$zFow;NVa6CbI26P-$8Rwa+_d7LM&5X!G$mQWhac@^M#*ZIrHh~yM*27hYu#k4`
z8n?n`A@1Aa^O7$FLq6U*0banz@LBWori;((a_u^yM=wZ*0Jr>~r>T{d)u6+tWDkqa
zhA7CseyooeeISkewD-R;xCT*6DF)9&Cr=jTZ<5silm@Nk6LeuI3F{{TRx>kUxa3I`
zXDVuHVpoEsi=mH)4u0&(9m^xJh&_~f7hQFj;(dlyRD!off(+SfWs^NTdPj#mKR8%x
z>Fv(ign54kGRUEAJG+QoxNEP8)SLiuEG1@quIWbSpcCMG?OMnQw$
zAiGz#twGniF!aI5#*nw{OTy)Xl;_exLP&SSdA-SOx?RSpDA1CzBFSNX4#OKBM9i|K
zrEg91xBokhpQ=)%{3%bAM8nUz(=CP}w1epTwP^(-M_T+bD9x$;z*hH!(*!xV5O!j-
zea$$Bd5jPfkzaI5z#^{hes(JJ)$;>=;vjfE292HRS`3f;ZY|V<(O0fCBF_7qo5(r^
zm6i@n+CX4&j3PL=S?VQfge-dO2Li(GNAo;e!`l339&I@Ni3!TC9}oWW%2QV!gpo5{
z`wUMs-+DMn)y|J(aefgU`S-)?0edQ%HrjBF5inEe<5d5J^>-)^6mJg{{eXnhdcyq0
z0wO7yP7Jo+c!@&V(LW&9KHt0}DX%@$I78E0;oJQp=m=3D6zvE>BAmy}BG6FCV$qSj
zrwk45$m&ASRUVh*X*nuyFx3CVN-R7xPA{8~2d${egmnUA{g+cYQu$A8>ab!H9HIkK
z?NxreJ6{;nCzkt+B0m}gdpb&O_H%ZmUBefno!*E9Yt(T>8>|`R&P~&v=h(Wdg+bEk
ziw~nBF;P;OA<^dO_AdA5>aX_doNShcB#m$5Jh{oZCk&&ZfluYFa=+X^dWzfP
zTjToohvDi;UszQmWQf2B0iSO(ZWQ9aX?YS<8OUEVqG{_
z`bN3Y)ImGp`eu8-=R6*9A?GHEi_D|v#l^w?
zvSz9dQ~q$#=_*=oPxv2d!B*ri+AC)0*S{m~o<~OR#A_|-2?KdNxl|LJ?LQXK;QOBiWnv!+#OY|Vs4{$8y8Z4m4R;n+
zd&t~G!(T&7CtzR~s1to{O*Vk^%DF=0-Qsn|;7p#gGdf};0Z%CGg%~&TS3DTpNc|7g
zowN0B5drR@b{9UjrV~r#S}EKdim>)RSY$kfND205(rnAXwN5AYdWbVAxK$bF;otKm
zwBH1wkMr}VQ~1P$8R_mdzG@NpND!2n`QjrAD~Kg+^nLKCXh4UQc$W)WvS_tO1U9e4
z6*?%bN}Zcez|91T!Uv+`YP})kpw#Em@ZQ3#@_y9SgW(iy7$xGhlQ=w0Z92!QX8+;e
z*$*Ns%N%N&o3euUI~;0yAkuC>Bnbb6m(QL3qk3(#>D9Z8%cZ|TPZ#OZWie*eE^TLF
z%CyNA7f`nji*Zk`mHE3Sx&W3C({8pgjrsaR%*;>X*2v7U4An)30r9~s2>1uluY>3e
zHTTtJ=P|vG>!_WoUxDvZ^ppb}WZ~UT|9gYNcGGuhL4zL!v?Zj&55|uT+jDwN|C&MKv<{
z;>})GW3YRn@Of2vpOevITf1luALm8}QI@$zJiMsj&4!l4^>EvlN3&8&b{ZnI#`j3M
za6Visrkxa4YqUV_;P0RDM{{;8^
zo<;}lTjM)}f8u&sw3(GegMtAgf8$-M0X-2K@`0i(J&3nM
zpePL05DLS$^1Mvz)fPhH(NM?kKID6mu}lr%a^;$?5reVVNQpsJF6o`$+6S!
zJdFGuqrbM@$fhXlL{~Q59F!xLB$BQ%0hT&sHf7~B4$*A(KndTz9Yy_d`;9TUvQaaT>rG4T0v`>m<
zzNj*q)GJ|uEnL*_A5+L!ih47!&i@%&C!oGIdi){gzHo~i6B>6-P7U427dCgZMLs^I
z_8dp-f?0^!W(bO>M4Fv;g~!XR(v?1?`BJC2u`Tp>8NxSGeBO8jb0k``PSCXR!ouSyR0KX`H4=r}#q;gZ1
zrHHYwhWdv|)AOeB6CpoB2JZ~f0v6%BaPB3-6=PG?%N_M5yX{nbME;9mmK~9=zp6F%
zxqr_@P~^5Vd2WtxRhrTD_9FW4Qp`MpZ;pAkd`DksXAlogg29^ZvI=z}!-p|MO4OI1
zXx=dMDrUyy46`;&2-w%4%#?z^5|YKG_MBq3WIje87T;YX0#;g8n|8JwA@W8f6~vZf
zn^1a+DoZpEDpwRiG+FwyDl&2bD^F+R5#sCgQysar!o!Af*04ONnGCxYpJMQnaam&PO&5`5|Yre}H-Q}{$V2^Z2l
zGyawPr49vGiU=M|iqzBiy-pPFnr$SHJxwx(xrmPvv_{kO61Z1CrXZ6;R%d%i!-;RG
z$jl-6a~DhuM56B1GRn
zYZVVu(d1L(n#aB6D3F)QRAZVk|D0|6n~Y%IBhT$n5-R-r4pbIs@e~OYpumEl(EaLP
z*|I;}96n=P#u`#FvmZI4uAZW2dT_#UpENEt4la}K4?RKMA6MVDGn|i?NziU^>CGW+
z1?vF4c6W7*+MFKQ!3^%ms>mf_*a@RP(w^=!iWx{kQH7P*s?qGTLX2{
zjApDr#?~6=U?wpK?A5V#wEX{bpF^q0L$oF2+S!>$o=vsRSgQ}#89==i`JIG*>fT3S
zR6RL@1dV6q@m6ruyX?5jqa0BTE5a{iqSDBOX7k5HXhqXzIZ8|vn=1x#LyrQ=e(-M8
zz&Dh@sb6p$$eng^Dy5T~>Ax1K%vw6*bxzt@*NhJ4n{oczn7
z)%W%bC+39lcC{hRzw7>UENVrku)?(PzPz>u2ny84Y+TWVpyB_Q$+qYW`Y;`h6oS*_
z4nDq@BIf$G4mA<*M<_{LK@97pi@-M>R%0GX1g*$VNd(Hakg@9bG^kPgil5o>(`D(Y
zzEKT=WCtCM6dNhReNTKS)|vOjj`p9d0f7=q${LcF20MoNUi34ImI?VSW%;wUtB{Bf
zq6~K4cUwc}3DHGRWX>LiLU$tFPUqm1r%okjSZ1SnJR$-qnE2^Q?!m^=
zvar5>#HQxvJC~D5q)aL^4g%l-&A52cem;}YdOMQZ3i^2?uUMRqrdMWYRw8qQYM`M_(b3Gexjf%N3?DgGi*4VZ%nuD&o~zK^EUDc_+f$dQvE
z`@Ca@h_*0BU1%X6DH-!U;wc~8Uw?$|!1^fUV|`Oli8Dyp^8CQOguCDZ`QIz
z!vugf$X6~IB+JfVdt|X!X;dNc%H8%HF%N9;UlB37S0R!x2VZRo*SOz^VHu;Bo5zV$
zvr%wYlPfYYY1DImwa*uz4*f0$e?^MKmsSI-d%c~$9h5Nn`g4cDjPB~G_IK=hu6BzQ
zvGSR1JhR$!VPPe|kDNyZ{L`g{CDXN>$vZmGE$?%nI!7k(E8$G6&DWCR{Y^P=5G5LL2+H&=TbW&-5wooVP3_w9)&
zzh0?i9GsGW3b}$0%$LSBcKxDYLq*@$E?Q8($&+*m*1IYPLfyoO@r_yP=?xMM&K0^3(NPaO>iGGjfp56@5CMIFUU*vB
zU|rdu3M6+{n|&A(q2>Db6U+8iKr=2eykhSA6pRF2*ID01*E$KHBzw{kL1*pa$wDZC
zM~6_0phvsCRp5t2^GpwKLosy02>KFWLQG}1!P181T!*ew4i2(ZRQP_?1yIQ>S($e<
zb8@{oho-eT$?6ZgBn3;d(jqRxVbocOn9>vPqc^grMLkbvmRBUkS0}KbN`nRiiULfg
zhCh`gGVwQ9KmMZawE1e@SbrT~shK|{=%zJX7w(s`dXV5@WtveLP>i^W29-*q~z
zBqc@N4*Tyar6_!)#T%v5G%`>M_K&iVsnT(!DEdw3pmysRnMoIe#EvWi-}>Bdp9w1M
z=ngJ#e!Z#a);lrgTmxpn-DvGl{=3istBB%0CNCaLe@g-D8OlP;aM1r8<7(9Mg~OLY
zS?{V_%f)8g?-yd=`)mGMNaOFmup+Du!9|Z4gyha#1hy7d0fx{QzKWbHBI$J!G8HtV
z95b;jj9lf&4osN{_ZqB`FN&@AECOa2fD2Tk;$x${H-gGx#El5$h(BO45tZH;vm-}3
z*c(_k3^an!oj}O+d69#Sf(Ew=CG5q1P%x6%4W7J3`O5z
zol`Wxys6C1#n2-&vheFGO{BuZ4-6c3;~xF$zLyEN+NAg+wU~uIntJ!w_i}Jni~gF2
z=<=4D?|Z2WCM<@w>xF#h`!{5@ph`FbV9mLJ+Ia8
zw=uqd9{>l#(A9zDh!Exvo$I?$mF60d7#u!c>>nSYn{eV2{{|tK(ey&&D!sSei@tUP
zTa8tWBYn)pf9Jq&1{*vpD2ioIT>U4_nrhf{E9YqCct-l{Qp?L1l=M^9+AT?PDND9Xcjxo1=UU*z|R9;NeO*Fq@hKKPSy@wZT(5KjIM5M)NhCcmTu
zMqW%I(k7WMs6oF_5dv07oF7#d>YAU9h>uoQ!+|kA?T*>Tx`?g_Eu_m%%S!DdH8h5+
zuq-TCi2|Oxk+FoUizX*JYt{@NYk}8>g5vI*=^P#KFxHN|6NZwt$!44rlMv*r2j*es
zy_CMLuYGH~KHNR55fES76fL@Gq(eD3ZU4K}!?R^F7A%+Osd?^9zU#S{0wYqK
zCE0HVmR6Q_NqP60?NIj*j_`+Z>>R=pK03LHQ_rocw0ORw&-&c$%YF&HP7>K+-8=bM
zCzhh1;e7Msd2$O6QsvY*K)wFwO+cFP&wx?oUn5kdu;g)yT)3-J>jQB9k*7`K;X3d!
zjw(`F34hc2V>x$1nofFle=5BF$v(x$f20vuzl^b
zP(H#cK`5e8=^(VxP<^)Sv$XJz(C8wNi_F?S__W>eyJlTBA12bGIEmSH0!QE2{#DgVPP^5Fhgt9EVDS*f4N>9w%_Do
zKngx|LTf@)Fvr9#P?u>4Dn0}e{1!@zK(r3(j*<`1|v`=xQk2m9^Y#rEV0^_i54MiN$)nzyxw<2h%<
zlP>GTq>(FBqSwxow25_LycW|utEInM3A}b(7%l%^PP_`{qF{>(cgWl@is84-_=Wm?
z423VxE9JoGx6uc|@1VqadFTyN{KKf?^{O<)Sg!Jo+F&in7_nRfY>Ap_-hnm=)c<{?
z-2Y2yor<-!ESN!sP?g5`3{B_XL{(SKcVf`jzhur@l>MgJiMSJuEHNM?g#Jnq%m{is
z5uZO8$BL-P4Ay2RhvhL;KW)_Y9*Xq^YoMkDVUl7l3KCT-4l6g9UBb8^e3kqBC-fI#
zTz)8Dn9%v-X!QXg8oeBJj%F;@T)m0SVU#_*8q2sTgwEDrR25%-4Ua`(9JgyDff$9B
z4hgeMCkcg`=NxRY0-Ky2P{LyHf1K%_uS|8a$*@XxKxS{KlD5WS}
zCD_*Jd@Go*m8aFwTMz^d$J4B2WCV9K&@l{^@hv3QkaGDF>ZtN${e}0ie?%Is@vS|f
z5lJ*&^pyVkD4Wd6{P|8SM#-8-W8|k1cDl(jDK|E3jDi+jycC>K_PFuhpAK$78N+$k
zRB3W!CJ6D*GrkM-hZ~`LpTh59s8cz;G9ANSQDVE!o-BmqD^@NzehW)_6|y?rZs{<6
zZ&NbG?&S8ZV4F|b;mvK8WXc2J`0nCVhTthSzFyyFj5d0N+e^{kMPJ)yjc+JQ}f+Myb0(J
zBE(%VoS?ZcG*{2^w1pLoiByK;M@3lhWv!fMSsyM2vYL*wG#Si{aG~^TL_!rgg)u4G
zpE~oAX?#Q?AuPTw)?-wL2QjR>*cT6)L_CA~K#YmrXGdR8Ia^`Sq&9|1N)czk4)FeV7Dp}Whqvt0k2p#N
zNd|!TI>@k1&ju(#G(7Qezl)Ghys)1W0s{lVf_UiWKf2x)9p+f7FYL){C|I3D8zWy1
zUi;tz;H-CN<_6FI`niDaHSqf`kvqA6c|reX{?J*wTn!vQbxViQ#0^<-Dc~&p*`k*o
z_w>BlDD<(wSmbuJo?mX~m7P@o;dzDk+*
z4`Pw8N|){1mR<}oD1HNc6nCXf)_W`?ay$ig3Vzz~V|%5qYC;6B5aDpV+@awSTp@ae
zF^o()4~x}vnnI2aNZVgBz04%Yy!8VP+e5^P-tH!R_QbvLFf?YNoa~B9!UDClUSA%j
zXQ1DmGSVn?j6zBgOZ!$($-T8a9TRLcSH1-}EUwl{?sUv>8QkSaY31_DVH&(dMJ0}U
z2T#g;932xSny5V}@n&ykx{mj@*89YDWZ*cyQs)h3CA6m^BtRul&}L_syf&hJrVQ>N6_(xiKrf{I4e(C&jf!65Rwf
z_sBo@NoLLi<21W~+stXiB%k{PUf@Wl(ss)WQCl9NsJETHE>
zru|rrTBeoieuz+FLXkkchS5Tx@eBy6pxwJrk~S8|ji9iv!wL}))wspPO?+c#NKLa25@1c2hL5)qfMH;zNTRR_LIlWI-NlpgUXI_46
zWK`o&9Cqa6EgBb|1AvQ1Wg@n={rgqc=JrW>xHMO&QlWxQ*W01X(BZ;4fw_}<*B+U<
zKMQ|obBF%MR+Lgm|G0xx)y10_Y#6OL=8Oq)W=W58Hm0AX1bvW`oZsN8pH&x3Aw9s9KYugA`rVPxElj_^b!9cqcG4Oby)t
zZxoALmiBIV%FI4vFJmu_;aUHpj*)iur0aR$Y}cJFGvnXC){@ea(y``or;6|^tjMse
z-2CKg1fNRoP(V6j)~XxabCizF+rIgncY#LBW`%4${QBbJ0AQ6mJN3bWODv$5>AL)R2Tr|&$xlH}3SjikeCc|XbDX2x
z(hwRNnim-pFe1IP)LVa5NjD6n#Q^*CqF4utjy7v(Rv{taA@gYm#1AE^HAg<+;o1iL
z=SM|*Utsu_1L`60C(_U-&(IZV8^Zj7z3Y*Yk?ZS*qL{!N^CD70{`Mk`0;=N`@LadH
zl!@X2wb%7-uTWcCTlg#{>fs0q+}n8n?R(8i=F?_+SEoK&`48E_G5!eAf&P~A)4793A)-Fj6@t%z9ySYmO=|TctUn$hChVlT{?fzM()MGo
zWSBFVz~gzLwdX~+S|du5_v2#Ut}7RbQ^F5O3L`_PN}h}DePJi3{f(G+*V-QN2xEnl
zM&YVcJ7`VK#nOE8jQ(J=Zh4FQ@ayV1!|hW>356l)T9@LJe8i;o?VMrc(GEXa+45MT
zD)twHH#zSKRXC`9`S87;QYMWtyODW6z9-K{O-xvnw7@$1XlI){>@ovR5TV
z2;vozNM@i#)ZtG5NBK#b>+V%hK@vgYW6tOUZtGPrx^}Nl57s7(1PiBwoim@(E7#qOpEf
zx8Ctc+lC?Y9d4yMOU6znHv{v)R!nr(I(
z)BRAdDE5WAc7QjLnmxb?_m|G#Qk#m0TQj?o*bJvP#<@C-VD~)5Q)$MQ*MaVE>#K(I9$+JuSsH&86hhE7{29kn?rOzYwE6X`(`1={OOW|^lDwJXV#guIZI2vP9CHlS+`j{=A$)_ytRJ#
zo7+`${V;lbt@}j2`B#xT{cKi7*&#XRmE^w!y&vBL(VynHe;8BRuGe7>NS4b?sVZ$d
z$1mC@PU}=O{6_tih`FypRN#Zhy=58kj?wjeNp+|g&JVn3EYPi~Hi+a#X`8~-bp2nT
zG&xD8p35(Kj(yEj{6q}yuMWXRa_xc@xYwzyJU$&Dt1K^XkB1NhvGMZWg5Oy|Ohg@z
z!}?oT10$mx@Hg-~Z@vv|4ADY>a(4B8_rML(;!xiy!R3>n)4NtBfp>PMP5pcn(I
z%|4gvK0wsdU_M=NxjzRqX~_l2L&lu44$V5wt-7i+AA-@rV)fp>{&UM5{}v}%)Bsv9
zI2%F_exvUX?9w%iAb$z(&nfwr?=H@luBJM(ZaZ(qrK-veJ`2zhmTWq`009ob=m0_C
z0oc$EkG-#3YZ$?eH`vrh^e-uCsn}75)}5o48aAm#5AI#J%~PV6od!XLN-At%V`n_`
z`u-7wM>g4MPFYn|T=D9q!(x?Tqw1zV)`~b_{SE-M&Q9IE;llfk8kcazfeLj8LHI@#
zP#tAxfE&wu8a$|VBMdlt052rj{&*Dmx1V+aGzSmTQO{&Nz<
z7hQ{oF_Y&tlZDr}tkIS3_Ndq17cwUi(>%vPH7OwF{g&%Y_t
zACB7(mD~yA*5=Jye7woX8GOR_O@eIf-t>3lANnZC-HXgN-wM|!KXuCNMxZY1y3R&ksKpj2TP6{86*241I+frw}wCe_k(`lc$xw``p!Y
z`OT~eUVW5U*_YjHAZeGnIc~B&LVaCE^8zJ*J1v=^6`>_T;KsLqFX=k6HLQezBl{yC
z5B%eE}(4x5N0@Uf1o+62qSCsH2D
z<&^Adt~S?R*?Z_;cb^QamY7qD6fHG7J%cmx>ty4GXh^9sJWtY_$^J;gNcy~<3!W-s
zV_gMgyjY}I>NXO{wmu^~jD58Hu!DgJBvSOv?_TmvWP!{yx}+R~6dIb3l-3MRdMfGN
zJeL3VB2Nxe&Ahhc?K!3B0>;N`h;Y)Ew5vPp@{(o{3#vkXraiTKBOTrnVNNpKZ6-LJEyHQEP9WwkZl@
z3$ja->ncjd{wm4TS`MK2IneH`-iugJMiUTw?=fT^dhPXl?uX#cFBBW=NjPvV
z^HP@HD8&0ww=r}#g*2sgy2qu0?iwOuh+FwMPwp$nB4el@%b1dWNa*jB^!=4k^b`gm
zEi*?h4|XK#diW(JPuGAE12l?fhF(B|40vVD;3Nx37+E^~%~OVq&+-%X0Q;q~%(FSt
z5u8oYW_rH-F90MVS^IbEy}^yZS8p-%
zRuQLN&qh{u6gUcApk>+re>n1iV}5>+ap^F-TYVv}I1CGFln>js?!xpFgGo;Xx5}>LCGH8G%
z0P=JIjWVg7nVV~@tNR%12yQt+VBdGA!Uka)zZ~43ItT4|RIo`q=-+RRTgnUI4=asJO(Ai+ops4v>yk
zdTud93Ei1Jl~-3+gCv`uBoY8cqaq@7QXXzHqo*#b821ZkO7;>*imP?3)SoLv-mX~rT;$eJQT~A}clCAE<3?K;HBBHh
zis*b$_}w2B4ZGPIGq&ThPAm91@ja!L$JT2#$sbZw$vlJCm0N~Tlm3s8RR@K5qnfC8
zMffukR5-@904^T1>D}s{`jZl^O48*CjIF9yTwS&qu^3Px0p!slhz8r8+ga1nTHWT^
zeW9!Ar5DkP^bNVGI&*!rEOGVK+IpufYqc9SNdFeNfrmqlG%$)_F1ZVbir9um;_gFvc0p_{*T
z6gF)BsZ#4A@;WM=uYkcu!#O(6?!&a)(?2$S-Ny(gZi1HA#IJW(8*_nw&o7`A?L+PQ
zY|PA%$`1j4$#My6`eVh9@>5O>&naY^x)0Z>Z4plK>7Gu4J}&Xk9O0?KJbw%8Bl88@
z9R2W-6Eh8r&}~ITxJ9W{!Vbve;qc={iQ3u@sAJ$zb|)~{lfD&1B7YQRLESNbj73D%
z7H6akJ1V-^-I9A3PffAmGO6@4OpE<(u*CcYdWGAAli1Tpx^f^{uNZfe`yzu*D-Nlm
z$P8Q8TYJxV8?x@t8?}}<7tUv$l^@;&)`pP#Arbj5eKpTebZox+y&_68)h4`>aE!P(
zHi$Pe@0NIg*>+SCM%?h^x>VR%ceX(mUpAYQBCQ?6?^sQ{ynqrCONE&8XTLN}u+wmc
zZ<12?lH2Dx{N5(TWsd)`c5!8s_b$MYsHMUJua$IKWh-swh+w{xBT)vGj6WJNV$^L!
zuppc4_S6TKe~jd=Qc_$jd}d+F{X;%4zs-#LKrFSuT48+{S60C27tZnZ*@v~c^JebF
z3cre&>1e0;Ps5+R4*raCm+=V5V6iE~((Gcgr4!I>8t^g}G?EczGYVJ2w6|0|(X0A8
z>aMoAozDSDwI{Ug2w+#)k1%E4FYnkl7t>&hD~_e}Ujr9`&skjN!=a6j$6f&U%8R!e
ztf)LcjOXU&`j(UcMQ>5jE?`c+guvk+4C>`tFKS5h_k1|WfI-QdKBjHm3|FO94#KwA
z)9nt3R2?VZSb&jt(W+r`a|+TR8{j!{!L6w{7UAStJKdNHR(M2QHhFy0c6*@ZLW5`L1-`KxT@+5
zpipuF$*N&5eCwYTV4-$9H&MiidM%hDL-T*?%*hE<-_!)C%%GTBZgzO_)PQ%Iu<&p)
za7757c78>LwUyO@dgtAY@Z+@b)&ejQngI}BkRhNoAFOshx2`@OfoB2IL9VZhoHi?M
zeQ6FH!2tkF2Q37qOMZNV(o7=|E?*L(8QcH~;D(`bQ4T9+7sEzmZ%8GTh2EY?C
z>onN{u-Wsa{<9N!J4s;D1_iS-n03Ap~M3ghR`vMTms;hGZ
zWx{9*K+WGGKlRvBL*1PFlOu-Xvy5HCFIY7ITA^nvE}$R=frOTuiwpQYudlAO%4cuS
zcRG_ns!l>Aoz8<*a{AlaY!R|sU`_DT)W!QM^t>~2i?Xq9jm6O+*?@q6#xA*BFa%l{
zapjc3eI>8F(l0qGSngE=`B;8mAS8K*aCxjK=+>WgSw~Sg0lBa(VX5QvJmIFegAA~m;2Jj79pk%aQCVRjDEMan0MXimg|&d
z5Sta6OAxf7t6<pUw~ApJWzCI<6`IVClxKNqbG)S}SmE4+;KCpt1b{N2}49EJY!9U*j8qn;zj
zLj8OuE0x!Vk$nm)-D6qNud^%(Rx;YaL4}brcWig{z0Z|r|1aKKw3%P}XMEZ$fjsw}
z=auU#71bOAq~aYYsouoIM`XetJ(?<73q0fcLbRNAc5KId{XR=Aq{KG^Va3f(@Vp0Z
z-%@*spEX!zWS}NRIM^IjV7w5XQ3d)Dq|Fd*tr7H*rakt|v~6$9FBR6mP=?zH3aYZg
z<9A~?7!uqWvL$78p|HXPWuZuu%Vx(%3#Z;P483I^4KAoH3JWrd{J!<{WN~D+Rakg4
zrl>Y|UY?2^{8J3OsW>@DT?;db=03=bY$uLEYS8LfGB8!U>Jy?t5(#PU@1Z@2RC4l`
z=QUk5iM;-%sVOECB6f>_su{Eu3oWEY8ZdW9Y{wyS^g=(oGbiYCT`(DJto3diT(oq@
z#=f&4`@3#%cbxPDKa0vyA-IqY_hXMu8KssYD&vTCt11bd2QAHdot-Db|KjIuOOCtT
zHY5r%@2E-b=Tv^>H#ByRrmp+BNO~uHe6(%7p~H(m?v-E6Rlrb}#(3l?Wo#&tMwLhL
zMbVSTxYy2iF1A!{69<%}+x;EYW1`(el8!jomU-tG{9xXdp8DHwj|by15UQ~)*|`HA
zhN2>|_P8q4@m`{NxbSRPN5uEEu+e5*-bmwh0Gwi8Q1{z-
z?Q2(Sl;!4rzPs2141|yzI?y^zO-gDK8)*)-;vz;5?!0b4#>K}UPvw4Tee~xexPvmF
zI{@L)`x(%A4xC#De*S!UGT^Il=?_D|0r$(d{1Q+>3k1~dn-fjY~pZ}?MYU$`O
zB7(A^r2@p#%vq<$`RdOv@$mzOASdXyJe-gFoTsjO9#jwbf_EkmmmAB=%Xh}o@4@GS
zM$)Ma4T!wn;4VPu;iMPzU*rW&`!IZNdR0r+NCn+5!0YiGNy-nj5lbsq4WL780pw@E
zuq^N9rQ7|S*c1zSaLWd%EkN4I^9g`&|GM+}v9bN|>uqC2>rqQ#hWlFr)R$;`aX}C?
z35HL_F8^Gz3QgHbA}ci3@g@EVxBSfwtFZdFc=7%kMczf=YusO#(Z6w}O{_C@LuUT?
zExfE^(xTW+C0aFb-#TZO7;WUPy~zADaWLZUkZBR5M)QpWFBEAwWk^3@!phe_Hw4J@
z4oidDkEC)VC68+Q!##Rq+n3&iD50q;ll~{4N9(5AafO+_!T5qgy)eCB`gu+8t6T5T7!+{(6tk}c#{D(I_zA*4VrENush)^U-)cw#7
z$b0nhy@lUI2T3EoN1!sO>x!O0+4P2?bEt0cLy3#${UKjQRb1uaE3ez{fbVwvzTx8J
zu~EUEaNYjt*t4ed_jr!QaKS9T-egu0nPvmO9;Y
zor5gnPN{_5{p=$fHLF9pk(6Owj
zyt_fnO^E4m+J6{vK44J^a+L_;HICwqf3d;?Q8lLC(+#7Rp{bZK(<`j=a_n}Q`+6(%
zA!2yU+uD46y;Ohn_&C)5ZCA0~(;ctO1e#17#@D|o63blZ=o|?t!>}@#a5{JeiPIYL
z%RP(ZlV^C|Es;Yg-`3CY6OlHJofuE-WvbevcK9aQ(eW3P-)oADZo^-bycTj?p1AO-
zG=t&PWKL((9T^s}JG>a?MX|D$AMN>a`KdDOJE~5{M(OK^O)}j|hm(^G86L;m*uI;O
zb62tfl2@5eqb!c9hl>q%+n0@(hf9h|fAV%zvLdM&ZbeciOrENP;Ew+$wuO2aB;4>6
z7Sw;_ejqXYN1kF%gTQN?Vt17p>Fzg+)zSpDgg?AZ=r`arnT
zt6IY?$Kgu-3JhN017wSp@3SXBBp@Otc4$r;*(xb6h5{rBwMsqU-h@(B1)_s}=hm0V
z28PbPtl)0~F!Tc`^2fc79LR15B|$}h4Q}s2+jb|z<0!H7Ve4R_R#g>{Z%K)X8R+SU
zTow}Ifp3j>dgBz}(o#nooDUZTyq>>nnS_Uhsa5Fgo!>$-$7#cqg{J`c6Qt0;=jJ+(
zy^cZS*Ox8~JV~L+63cCVq+q>8^k%X6%r$!0SY+;*O5jboBJicmGPyGtb=H
z!B==`gMtDA#B7!q%4B@`SHIU(#O9S-lcwW*pfTX{=!(C%^^^K
zW)}M$-<<4~2_A~UM`+*g+)Mm7@8p`RP-FPyLE9(2PwrLBr0!Ne6L%T0;PTT^dMABJ
zDHbx;uLXUf!xFkwyWWxUPt-qJZXUQk%&`cy=WjHoI#i3T$WH5RHV`qt`|)+K+%r7w
zc=w&?N9?8;793@k4y0Fqbsv_DR`znQaOAIQg@?Ngy{-C%1{GiR`j>}o^N`XM5DOxF
zpBPlJ{OWzduAz?z%M3@~YYQ)sB|_fWL3hJw^l{Qo>$}gNu5U;x0oP19`8fR|*e45Y
zGS=x*)d6?7u2u(LEYy+eeFScx)+9J3sn=`iWh^uDAY1ZaqbNOFLJ6U~4<-oPdyVk(
zx1uaf90JBs^Im~DzjK7|#I}u_>*cKznzFRmBU{6jtr-F8WH=N3Tgz{6_xIEPQYcZ_
zBmZ2Qj}aZofg-H)ib!-1vw2*6(`!2RmibeMy0#tPrg6#rH`4U+_WAbPd_P?TI8g5+)^`qzUN{Wyt&GVI~=^ZQU@n6I?nRcV!xyDO}tlk_3d$@FJEdCbo
zJjf~-B-PI7xJY2$k(s_oQT+sce1cRT=g7pi`h{DNYuWNP4{h#Jz{r5d)$P}+-B(nH
zviJh3ZYrp(U|kpzL|eHxiPTd=;e274npp6tbrQd*f>2eB+YoVCF=|*tCUtiY$1=Of
zFGNs-$RWrN5J$x-_rsuGBV>i69AOjY%I}GMyu4#Ud!E+m%+aIx&dX|HRu8>IWt7bO
zBSD^OSfwBD34$eWPuSVtvN+l)bKD2buY6L1O466oGW>*bxIvdl-@9Fz%+y{tV{lG6
zl*c|VFdsHiBevmaQq)eAD)P*H<@HVD`3sq&0rWz#_VAb5J$`qjFSiNJ|IFbtvdOAi
z(k%D{?VEHHaTQ}p#HpVa&P~hyOn4rSdUT%mAK`PO_|LaJ{oLEHXq-QY{iCSzhz$Mb
z`McUUvEj+Yio$XK`UXO9Bxwi+1|+DWUor?j>IeoFHnUO4zF;ZEInTo%PxovsbCik6
zif3fHU&eClj=RP+HBY%mZ8)AzDRN#jqkV98?jb%>oB$#a&u1&oZjgnd5O~F0)m4ib
zWNFAeZvgcR9!VJ%Hz*rd&!4l%wyEp6+0pw+3{#wKb>&V%`M%rW1ZYzD?f&-FjtixZ
zY+;A?fm$Ef9Fn5WSc3bFia~fQDjHPB1~IOjZkYOP3b+hw7Y=4XdPOe=f5x#4h;#Ro
zQQj|2#Idol0l?`2oVvskSYO7G`|IQ9--SX>JLxVB_4RgtW4gl=Qp1E2Cv*5l!H05nu9I_@ko$GJh^1eN-hAHr)MkE?@}cb2mix=Vnl_2-Wu
zXdZYrG2%q8x_fV~DqasVptL)~wf=eH_BrRq3*Q=yc{7s8k_loESlTI-rn@y*JNpph
zlRoYSz6qu3v!H
z?%liF4zezbdS>kgD+7b9+}z$nXVA)MPlg4z-{6B-L$mP=xRlg+K5_YJFyU
z%YM-A-ec+TqiIBdVO#I{IMd}OEMBFhDLY~0$j+Yq2MNYDe$(TYbLRC)qfb1#jP>o{
zn`GPHnp&Rpt%%$Bdm8k(obBCp
z+9owmZJqnaBPuWu@OdND$D5CgFL>eKMIfGG(o4l#v~St#npf1)mX;v#tNBRH)WD5R
z{k@sk?K%o*U%Hk_Yu8{iH34@D7JRt{9A7w0dEgYKj0JFKP>y|8(;AoGXqCmz%r@E-
z8rS2gs==rgj{HkQE+2c?-U%vjE8U03IOc6U4b@>kqOZJ|d522+r_yA~Hyk@T53Moq
zEB&!v-l|PEU;0dwYrexs@}7pc3D^T@=%e(@o9XGHN=t?SI1K~
z^>Hnw3N?AMZw&J^K_
z9n1ZF4fd$0DBn3|H~FoqfO?Fg>JL`
z76_VX;Aa^Sp3UpQ^a&)YTO*0WCtWaLjN&H#3=~vmoJ6SbegOCfygE>L;a_IUG>N&b
zby^(F!7SR;)YQYHb$n-kcXxMxA56QzOzM~X&cMcLSQxS*PG$Smq9l;Zf>;WsLtqaQ
z2^RVxMnO$&c6nIJY)>K
z36o}Z?d^grWw0dt&Wt4Ji3thLE-tABdtfA_S6Q)tyAtt{ss{9QLH=z6IKD0W6(Iiv
z(VmQeKm!b_fS1AhereU&$w^jD4s?S%IywkBOg6PeK|QaktgNi1#mCBOP_G5L3oAtu
zj&MF>|Ci7d`rjDo>D@rHLtS4#Qyr@H;s|iB#By=EaBuO0T@c3jQUY@L2_BtpK}R7*&Wk4bWtk-=?K%V0lo
z!bY&vSEF4ht&Z;G%!&Lg$>>sbmP^&SUmX@hYeUnYgXP@!_pSQ%e3lqqBYhr_PvqjJ
z9soy`eQOyXU|2}2n$k|Yu_i5KBY-gOH~k)JfjJFY>CtaRhU8p&UhXg6Cb~kAzMOHy
zjBR(baRy~kUaxdGC+_A`=@o7{#P}CzjAEqts@&O#6fGtnKBki!1Pw9!|H$%({(>1)
z3&%fW_%Q%piq#PNmr~&eZk6yNAFF(35!TThiv*QqLe>4yDU}Edg}-9%pU=7sBFY5?
zns7$fh=b>a`_hn|uh?P@6yM>+L}JntcD-Hi7&IWsT9hB|`}bkgC~(rD$ih#|w_?9q
zLjX_N2;H31$yif%-ZFE_s+qv(Gi|V>0&X|WqWW6pPu1d%_o#(&8qo(1s6wJFSq6{-
z-vfu`v-Zl}l9r8iAFOh`((%s{IbQtN={{$-V2+eLR24gx_mh@tX#K5|jCR9YViXpi
z)m8V>g3F^7HJ!ON=4+VRReSkAQ>DzS%e$eyxaD6byE}#|Q1&Lo_;5)09Vp+ip%;m&fyID&*^TIArQe=BHxQVla8Q;c+uhRB0g$VIwHy
z|J9)9e6`_#In0{;GjQ0R)9_!9>U0s2h}svWZe>|>j#s8qk+x6Vs0Y-u^33RKSF!c6
zh%obxCjChmv#4Fn=^-*7-xTT#jT|(v)uMK>-u@H~`dh~NAV2MsJuzZH^s_Esz4Ebd
zCw3zjU*B^~*dt>=f9ow1zS!hCQ*!kYjVPw)yGmN#hk<|;DGXAPr@fN3Ft*N{TEm*%uD_+;?kJ@&&jnc+$-X<+i7!CP+}iu$
z2_e|(V;#Vqs%;jhPBh91PBd2XI)IaLOwNmoMLZ(fyR{J9?-34u|sqt@^QDtk$@i&)h2DQP52H%NJOngFl+
z_R)LavW(yy)=NZnMuTzt(jD~NJm||hN%SfktE*4JP@e=;(yf06B1v2KD_YN{|ACdG
zP_Y-@W8K9UWntOrmlE3J%Bpkq8Myr%S3A76wzfe3?{?O(bcQHYEW>R(In}zw{ORHL
z#iIlUL10)95R>o$VB{6fIP%iS1N~PjDd~KrSSro(Sz?>EQ=po8DSJUv7PJ^aP2Ft$
zhc9*X3K03*o*$1ZOG`&gYC(bevhtLgl0qeAVQI-BTU%UAFW-Or=C!I+EF<10;P3!-
zZv)sfu-o`)#>wY(cOh(5q`}zWerXXk0J^prS65e_U)KH{XT)}#xC*><769QK><$2n
z)Xr$?+|*PJm{EcDtvBe;j&dss+^huvwV@Y?ex|hxZFlv$9#n_QETH0)d1|d3DS=*h9pK%?xzF>oW
zZ~Vg>5(3P*;pr7=OSs4%H@FeOnyleQr{9w+&Fgd{4h=2{;&Su8_KV|93O)OISSB(t
zmXyl%eNHJWyBXB)bVhbCnDcu|zyB$phLX`9O@d}Xsl_Dy_NyJwMcUt*c+DwiOp8r8
zwX`_L(Ji$W5xgQa~J*p)Jgggp8E1;qds6ArJ~0(l`JOL#qTG@GmgCr-t+YP*eL
zS+;8!i!EEvlR(F1fUBSyc<2Ztr)f{v*Ne%Bg?*VCttn?2TLtt~azB@0Csz8H~&))W)J#
zbjQP%ZcVUMDUBinrGJj6<@jq1TiW^SkN%aw|MKU6U@OlTr-0>nC_gt3iMr6y1#(bD
zf{N`$z>)P@qAP4$lEd&_U9T7gu1Xh-o!$U+FC=&=p?0=if)ffIK6(5ybF6ib|_0ot{n>>_x^?{^Xku!Xh<`ZSS1&
zzjHk8GCto7uFWT3-RvrG;e}o}?Wd%WkjcVIKv4_~-{@z~X8!t-x4mus$&~iK-pf1J
z3C`zhzv|BsMfG=;396Ww_Ea=`HS2^M7lz)jj&yk@%fqs3ixX;bkNV
zC|iS;HGjP=f+uu5@kgX@LfjDzl%j)%g2oKs%wx(i!TlUm3~fy;+dtIVx|nUY7$u+L
z+8XGl>FSKdO5Hwwsk(=
zcXrmvy>?mka-oTX0z)A{krsa3lgB_qD=ujPu?39mx3`$V(EGUa`KfJWJ|6?DZGylA
zDrrW&HjUW4m>4WoReTn`HnT~}%<)EpZhs7~-4D*KKzO3>G^2Ve(vY1EdZU-u*Xru(
z<4J6}+1a9CopV~?wLDiUSm`V&`S*eBZsK%fz@d2=%-3HK+hCHVr>6&U&ZVVPxk73w
zt#)ucOEjtsAvt^T1c%q!t<(8L?^JN_dFZ0fluTc-n2hX--(4&sfg4M?ori
z>^i_tB{j@);VJ;yupOHW0-yyD1tKM=*EwX*K~I2=iOGqF1T-E?%FAasnSMxbrFVh`
z<9VpOXAT&P)z{Z&XG5LbKQ{hx*7lADJ4erJ2*ORvg;rpR0Bj{#SXrsyQ0t>rEggbx
z
z$h5(xtG<$wQOY*hUhV#*JBA^2vj$qar04@*8~>OzE}hGwtDkhDt7H2Lv&du$J%Xp+
zl(zJ_aS80#KR>QMx98*lZF$EA=`lEA0@kDx_jXWH$Um*Tyt#Q@EsP8hl992r{i4AR
zm62e%;#HFV_p2{i#9}icp_qQJg4O(oAdJAYAOC)HP|Fg^>r1XaIGeHypTN)^!aB;=
zhDI7POYP!ZVY8lBpDWI42>gNPJFf=ZxG-&8JRZ0;9`?!uM+$@%nN0=u3ls--~G
z>ZH0o%HdUf>&-8Vrwzo$vEL8Y2`Zf=U0dr_(Mt2nqnFdSTbhdx7h*%^kO2v)xA8o?
zQ8lK~9dvPo9WD`m^S@1D;Dk5gS#ceCJ)0hj2!HoP7`_d_Jx~iINqL)M@QPVkVupf(
ze4Z+G$s$=}85&hvh=
z_``Df=djK_ckF%bEg_E`TYGVz^QQzGSv--f(}xS|s)aR=CC~6;05#_5*=|qFZgf}l
zR%gb*|CVXHCbz@&_sg%F2a~yx+^Eo0?fG3-5XGvIwvm{JN8nX|)~ah@_tZeNYJV}^
zjHwa1ceIk#S9%r-&2=zoZ%j8HSJQR}&z!tbK^$JQP6$pOlu6-jK_NuR2t|
zuy`@}8>H5}*9{CK#SP+*Obn+Oq(KzvK&}WgM-31nku4?K;HOS=9yi<+Fql63J@ZWt
zpT%qEyL&)#GZtKB-?wSkUxwfBT%GX?@?yt_aNnSM8i()7HBzu>lR%@Uw$M16D3XcB
z!qW%%@i>&ca;-)!xB^o=h#^FB$}~)H^mQ;>ja*CT6cN*xIGIb*ZX^1BZ`4L3NG0{_7sM^l^&
zlb;KRC=X4p5$Di}E|LJhUtYvw-`SLyL7N;oMR=@7=pcWyB({BL9BtO}
zZLYw!piqtusd7UpSwz@0w>st=g-r;wG;X@i`iUUi1lTdgi(LX&jet5Jic_`k#eu^|
zUK0TK``4I01G6e;4G`Yp2gHDe$!nM6Uh#s-wz1<*Z9alm0$zF
zyFPi^*_{AKB;QBI?3^5~12C|Uh?0{b97;~cuUZ1+P&bIqt95<+YCwLEH!UJyQd+4o
z$k^VPpVz8d@&hjGhgB3nJ_2xu1C*?0oEbNMU5`8euIZ6lZ~wXr9kV@{M6jXv7Lc5t
zo!#inz5j_w{UG&ks2R|&+Wlmpb5l;{Y{uAk%VG}3{=n(LSZqyCz9!acqw9MSkrB7f
z0rv?-gnP0!AbhtI@m1#;N2k*Q`X?Fg5AQ$B_6Y{>Sq@<*27w7;^&aP`;`{AiS88fZ
z2?gR+rSk`k25<~%*q$BbzR#r*tuPdlij#4%IdCA5O8?o(nYh&!J?9Wn9%ZCpvvBdPApvph}h
zW3+G0*eH5KO75$h8A&kc4M~HHVTxRpKCM~yV|l2d`pG#3g1GKmnfzI(6GR~@<&yh|
z$yL?->k#kO)uCFR=OlJ9NtBbdZEWn)0nnI+59fU2^Zb0=JGb~S7`EMsEBv|0=1g@i
zKX^WCq2zN2aSoPrU583Fr{YV)k^pA<3a7DIz1Vp9jH6zgZpz1n73ak2<-bG@L45fbQ~
zhrEIFqrpzWg?Gru`}Mv2GT-SWCOepap;MThKg0C}Br20zpk3=^`H9PWro0mJ^{feh
z+LXXe4;-1a)C%5pOmv~xJx6+qD6d5|r43mLo*8(=wMUU2ys%l5y$z&(OBByq+-1w_
zENP$afxFPlKQ^dTVf=<)XBMsAZ9dTj#SKPnFCC#UTgdTp%^?ZTyYH8BWplC49LuzR=^GhV?`=~4D=yhGJxJS7
zlaT^{nMcmOy
z`m4#`73Y?trO+UtST68)09CW21L#?S;|w5L;=)LsDe?*!koFx&0Izw0un}M|($u+gr@V7FpaDu0%lkH1
zkEx+D+&}!Co5#!Ur^|_qT!~MDue5|JVKOWXs=U^rjX$>#xzgGOu@>M_%ndUO`VkJ+n8D2Nyd{HX$5f>F`@{o
za~5k%#^jqS?z}FKO8|G>Sk{Ii6j`E>5Q3N4<7fu(J
z=WfjK1EYuy7BLKfzq~=0(rjWefYV=Y?_df_&9-M({~*7&!>vOF+^Pd)T9=KxRci9e
zi4jSgoMIg<3l}&lkYWWMBBeZk*VtCshFNvU+1d0fd1&2l??UTmlCRlJ!`}
zGPi_3-}2;J^D8P8Pr!u~iEdIM;YHKH8EGKPbL+ZVPVj-%k3R8Xj!GfasXkCxF8b56
zm+Isa|M)S?^d4POLS8GYowYJXPje!UPgqhJ5(d#YV<<2{)e>Kjc1cwcI${zL?{C5W
z=nT6dkR$B=ggqcPLHmKn1C&siH9FY^nui3hBWUs#
zqPy(VO`yZ1u~RQ(@+l>KiU?4?aG6Yz7rHHp1_bW!)*qXG%T>WB_SqlV?ryzX`)cyp
zJA!BgRQo$M6$QB_^sk?;u=|hfO_*Q0$(w#igjydv9$7LHk{orjs0!1@-!+#Zv%Q2GD>5S?h;_Jj*&i*C$Km+AfwPfEdi1)l8g^QFh
z@)5|4?)rYVGgSy=Wdd=Uj?T{0_vHLSLOr@guYqzv&@C`O1FhD-`@cJYBpRqe@bQvm
z?{A5oe+{J}?Ql`-?ftH=&)mLv?|GLMZzDQj^UZEBbYMVyd*~}1W2@ZPO%H9;Bcf_`
z=%m?vb8Dbf1Qw6dpH2q62IfnA17)?co>AR59ab+SZ;(y12O>5k{$0Pk{@pquUoM9c
zeBSsYJfYH%WLs(jndftTtSjS>(qm`nROAN@6FcwQ^pa<>dBG^>S_o5~308W50H
z6yn1A2tp#yH;Xs2#4Jo4Lw$KX3PEO{Mf%adt$7U32tPDJ))BLIqzq>Pc8>n
zZD(pZV9d)LfZ@p|c)-B_ab?MUZEj&C7yv^D-O8CBA9p*lBa|XaqgQQR)VMbN#n+Km
z9f8)qS59!VrYvACW075jm1NrSjw!cmH(*Ee$~m>(UrpH`M(Hu0P#ihN%Oz{V(+wqJ
z@d{6TwH@RIvF$K0M+nYSxdv%w&Jjz6s1B39V`&)RT`DO)o@A{kkNn4M#VH77O17f!
zbCsb|-L1dwW6*BZsYrWF+Z2g0&HnkI{r3Ap|J%1bxd1u7ScI4Lh|%W{ErH!8wnFk3
z*6s6BYl-r38!l^+D98uwIRr+OQNek2dU7wWEbb5K1Fx}(bYd!AgoW^xOz7a!UTobj
zDCD>O?8hir2w_oyOrN
zHZfsD5h?gta(pYbKP~EyuecS@^xveAYd2u9^vb=(z0yjd1@D`)60ny$2ZfN*Qn8|l
z5SnSwP!?|lZ>c%Q-Y9?R-IM1Gx0*#^GKEty(+nm;5yr5ygc8ydEb%|l)3Bj6*r8P8
zH*gBGf3GI%aETA#?LEiHR$Y495atf0cRz#jvi{0cH+(QU
zC*SSm=2N*^{c?BRxBw^b9FIEITb)KYGKOWA4J(SA-hEHmrHY>z$o~A=Hnd<~{|ry_
zba3_P4>%)?JAHmri}%Np@?9?+y?!#Q*SqFefA5O?Ee04zUI{Wac}o^G{~mOi)Ly&J
zII73rP)CG0Z54m{fHnD;s`K-9r}LRk^iViU4a8Bl3nq}Dr>|IQIiyN-D7qUFnY9llL&WQ+2`{lWy*tkXX9}^oVkseAA|(@e&d)InQ_pQCw{3+
z7N!j_Xh0s$5-s_?b$u*6QSF#1Sqo7Q)86-nrf~ZNtrJb>%;^*ocL6Hm6$sj<1bN}~
z1V6P~RyP_8QA}?=50#j*%%LO{JN{})mi^11+0Sz%`^sAAwmwyPQe
za4_f=p%|jaFviWl)axuAbiy#nK43vt<2u5An0j+|Ln!tTv4o`M5%xHTu0jE3tU$;u
z7_^cXSY02;f-oVn*U%)(3!s$>zX^ACb&Tbs*%jOexhI>84W-G2Xty+6?+r#gPAzCJ
z(Nu6ZWf+yxF^%O9;%c%-Vyu;<%3z2;j#=sJP=X7+bH?
zyliCEm`ZzdS;U?vh158OYR;B}I3yh?A7%FFTRv~+Pqm{ac~PN%?|b++1Gq(3kBXay
zyT4TW{c}Bc6`2}Z)OW4kBlvHj<;5n%Y-yQn*DuZ4*t^eq-jek|LKu(GhoNeh%Ag5*M^P@!90Q>a0SZ}{sp~jRF0IHdsg|>#zp{TA=czK
z;~(2)omR_cv#0Aoeb^Mu7oOqg5^5bB5wz8R{UL~``S}^dgh;O)
zz6(aMQ1^&pFux2Xd#s
zqDT?Og58OYf(jP8GJeM`Ck3Um<%kRzyzjdGMV0Ng=Vxkq-&b!@QAskjai8XrAs~cx
z4&3dJTwiMu%ge<+Gx(|F)ucnYi%Z`jBx89abErpitZt77qG#M)|7kGjKkakrg9jgL
zq9}^!jGa??c{W(YrjUu?@$>;H)~Fvvt3#Yy(HwT{xJ}6{O$lX836SAsywEGsQV}3g
zO{rQ2gQtG^zkuC>!3PbG4->8B=%&Jb{9H@&&rk4gTCg>HZEXC|MdC<4J^XV+G55Lf
z7)kU5<#psMG$0siXcR{Zo_i&$6XrD#Ki`~uc4LL-=2y7CnG68o_VK{-`;nM?4ks1bb~xJ1!kl+_T!iP*&=jab6&tGxOSCgI
z%tc3c13Iu_HTafb8m+zMOKW>k(b;BJ`fm3A>~Z6-7oR2
zv}6~@R)ei5NMzjilyQ9tp=5eYlUY~Xiu31DEY8i+-PAAUje^Inv=rEf$Cf`vr6AN<
zSRdibs0Ej3FlNbGv)uL~O@<4fBb%ZQ_69j6L&!7BqejBvl#!JOe-T?}Y^b9)mG;Nt
z((5EiXSRINl}07V37$|Hme++n0)YO0EpzPSATYDWP%3Ep
z%@_7tl|i_xV1jRQ>W~+VOGT~2FiuY5YR^1|GIFZNmh!4^Cr}9+&#rDWT`F0{`wq5e
zBEF+_3HhMf(e}y@=kbQLj7)pwW+1BdoH%)?tR#?Gu$oK7kUy`;%Q-Gr{+|0r_t7K)
zTb!z_BFmu|WH>`7ugy~Wk=`Q}Jiw|SXUO#_FBRlwn@HDnPP&^YGv2GMEW%?H#-)$s
z#ry(e=G)$YPIKeQJ
zHYHlcpJVfxJlW#>g%|S;Y-udz9-rFW6er0TDw}ZUQ2)MV|IO{JY=-5|ws`mR0axHv
z2=#N%r$fU&BE!B{vj0{C$!@2MUoS5&%@f#|hH(<8y!4p8t_+!;P{L3KUf7mj!@J3|
z1T&IV(R-Osn2sgXe29lk3Ms!BHAVK_UP%5^csXVHP*Yo}!kUM!NI+-t^>-9tyK)Nf
zRhE}!6+DU35xz>In`X|`H56cPEg0e2t_R_v#}H-20~%=p4{h#uveWp~MMjES6GA%s
z&yL?o3c}gV^kaEKJn}sZFjyVKAemR%`6P7cXs{_8l4^Xu2|c}`C^y19z4)h%hke#v
z-uEjhn)(`J8Wd9y8jG~!sH+DS?AW^Q-~=aly^Y=kb&XTTR5Rvq+p=-Bj|49~yk^mr
zzn6=0%XU8utMAo9=3}P9EUor&OCoT4MTx?HB1-N{!309&3emDnk%py^aV+~#H_Wcm
za&KACO0VCgKl&QFu8w`+R-p+2XJ|0v9udi-hsjjh=SouRYfelDbb4uFl+&b67&FCl
zno|xM5|8ATz2|z!Nbs)h{%D&`_qQz&XqJK+PWbYM%4*0jD~(#ADIbYyJ_fflu&FI*
zH`?Va_6t&
zGYLuw#%9aFd4eV6HIYwfN(nu8P;q>Aa9bm6VLJg2%Oy5Zq3&m&l{s5Pb{gkxR%#TvH8`Ly`uaW)$Ny}?68SNsHor~
zH8nY>Q!bB(rOC{t)1xN}&c*Ua3^axx!}!esG%h(4PA?Q-)1H_HL1(dn6z6fq<47J*
z?Cg(;OJR$kpiBn->9%*zT1fCk;cIEF3zinP*qe=-jh0V+=sFrG1G5b1Ay>ta*sO9x
zjrY)H!UNV#d*%UC(6_iXB0af}GoP5eZyQU=2Bd^oysoMm9WTs2(_v$}Wle2)gzMj=
ze3i9MsZ(~mJkU-o0tanHY{?3S*-EEgEox{390qKBYQ}d{*)zNstkeIJF?;%Unedqe
zdv~0lojPyPU{dChjgk;bh(t8DAW8iD`-E_rB0z$qK!@#}w
z?_a+Ib|Z@ci)E!SRXtq`=U;f%Z5KkNWSiv&Dacz6=zYD=VlTYy!C4x!enKdS9ro}i
zPEz7&q9Ue}4NGmsT|{<)PU0nR#!2jPHIVx0@~vw2DLlONps0oW3%J60r+`ck{ro4n
zk>j^Qok4`mOyJu-Cir`T3khFM*cn3Znc4f$fAHAI!X}Ugz3j}55fL0wQ(P`&4i{IX
z0avgk7(teh8(KI?J4`SAm9^hV=TLW6Ykb%KGBb?cm1gi${!dv|O{{-*B-F;!)z>U|
zDU_cLSL8mjY)gh)C+e*Ih_Hr4cdjOwai8=a4OH#rO0W5A?FmC0%#hIEBh!(>-om6x
z3E0r~G838dH!ukPhPmc)zU7_e&IKg3Gn^zNsFO)U2EstxZzBDIq
z>ztqGditwz^pb(Hj#>tnO_{^uoH;AFFcb6sThTtLv2`m^qeS}~H?k6CQMgRs0A8Gga?39fvs
znzxps=lwG`p+-*o?JiRqCO{
z$HG*=;UIct5Dc3|2_1@o09mK^ZXQ8K#I9n?o!iviqAkw3w$Ysk?24q$+bJT^(;7%{
z-ZQ~TTx#_rbNNyV1Cv>dkgh9!!Bk|4!t2j-d?IX@nbCDbX?AKQ^uN%S
zCGL;R%C%Gj9I{{9{`ca4{!f`~MQ4b6Up{)6dBo+yp8q_pS)P?1XsP+z3Z>mC_07yq
zv!kf;s9E#MQ>84i0C}e2<(EeE;vOL1;4YR@via%e^+(Scr2K@Pjm611`VUBCx5_OY<5cQOX0%5v)Ix!u(0rpm-yL)-qb807_n6-P4l$A9A4M_&cz
ziCNR40cO(Ix@;G!!)GLsI6!nEA|j5lQ`g{1ruf(Hi+mOR>R6{YcR;e9WPJ*fdT47LpvmzN11M^jF|zZO0jGFZL+
zd{r}V^?bg1B_+yl#&v!9c$m>_SZ=eDp2hWJuF)Z;=q$X_I{uO?e_8L)MICwAa}l4c
zyuLJz9C?tI6?u?RSBxbf8^^o8E4uB-_szIQP-9S(Tln^2m^zmer?evv*tOm?(E~qQ
zDx|hwlf_h+Bl`jlk|i2(hvL`RZrOfmSx<_>YXjcj~1$iadfiUwao>&etEVd1?#>A4FCa
z^TryKQJ|bbh}xqt&`}|QsC0z%sPth2n>4X&bm$F}CiCEZ+D?e3o|-@Ld6;b2B!>)YQQ&#fVt-D@3VM6g_!L_x!%2%lKCm0TaD)
z`|n|L;11UR)0H!)OB@l+W^ImP80H)!>(YHmTy^8^7GNAx|4uPwuf&}d@A87b?}Uw$
zwFaBIQ$wyS^g{ak3+!pTf8}|SxIAF&w!xb#VVLv|n0naGXRK%iEggyy59u6eHsCPL
z#Nb$jky#^Gk#2X&oKY&$w3r0P^t98S@46nXPOv2R%89e1Mc$~Sfb%2Nnt^vF>CE3%DJPa$QNaNlBXbH2O$d7L)4De)J6q3G@|$f}Y8hD7t+&`H&oeGlzCHDtVb
z$`?yQ{*ZNGl)T|3`lyVoksOV0BO!$jo9pbg#O
zFEzvKMB?p4V|+%5?u}MMO*)^j5AquGT54)bb|s75LJZ|KRD48gbH)$IJy_ji5Ug+3
zjJKBQqV!}-*Q4;7GjH3QD=>^cqW!$JC1wJa3=}1gxxH8mw>zy|@3KfK9}ViUiZqN+JrT9>uq#X{ov0Z!?CT?@AIBj9
zOnn6R4PRrtC1t~Ms>j>V8_0sp4pkN4+R=Ad3+#{Jh`Xq(QY1{uTjAddAa*DrS2T39
zWRALV*9UJ6A~G;4w^9b6L@lJ+89K~kRzMMO!zGX(i+D2OC_!kenl~h&1n&@uSjQHN
z%H85qe)oTd5MKAAWWo{cY9KB^kQsB4@mPztWPkiNI+ppT+Tkfagaj+$}aNAU2D@F+xz
zLAc9n%WA2l4IlB~;$|tA^ZYa;W9dq0!j@xilzGPUw>fZh4i=QOK@<;&{wbW0Cx<`w
zWt2=(Vv1{q_ACHplMj
zJsd(s)a`s1DM2cIP9f$XHNFTpq+)6eWrdj_?9czad<=3oIuuSkf_D;
z^3#v?I%li_vV~r?;Ay`4n1}ocN;97L`3xaCUp)`
z9b2E)wrBoPM7{YVPYT72oPDqI4!MMdaag1(S%8P%5^D_$!SYx?K_u822JfXLgYZY1
z%$PZ{na$MOKwv159rmwfB8}X-kWAZmvdKJMM1*I0h?}F#$MJbbcnBNVODrY~^#68q
zCu^5h7@}<`#E8jz6Xp|{EP`g|uF2RaJmLC?=4oo1DY|#mC%2;f*7S6fy+ju7ly4C&
zR*EM!N7!Sy59^|k{9Scwu*|I}I&m<5Qe8YtVcXnhBC=T0Ql=eLXi=IZwjZMv?dB&1
z1&mWE7>Opa7a0$XNF53RQD)?s5#gH__N>zZ8?oP(if=wt*6TZooRm_j7pL#l(fmm!
zbdUdA9DOEvgVW=C91u8xuh{^HZNkGn=eB&vf^p|%yxH#d?ATFsmYqu=uj=bt#bqd9
zOSPQHM^}{fSJvXt4j{RclyOZ*#%I%b1RwbhaUS|FR_;fmbwHphbOzy6Orn%1eWA>Q
z(jkTc3f=u`QL=AGA1JP;ruy#LcMKxxBhl&tGmC^puoB|%nJPk_|5A{1;ZI*XG9jE(
zK{9uZW)ig`59%TxI!&4G$<#br3LG;Vbj@FUDiaBsUP;i7dFKLl2jI%FcWNQzL$#VH
z4Ll6r7f`cgUWb4B=T~Z>RG7gf7Y4(JhtkuYtae%&n9+#8ZJ@9~cMRzq^pRY&sFf9%
z)#64|G?O?FVFiboDM=H3;YJy@`tIgCU>qIInyao!s0PV`mM>tKwio~;nEo)#*OP+o
zhsPPo>*jBtIb4Y7CmL$c0P;@Sf1iKD!^4M%)@tv#3|s5`AFtnjK7AK%=HcN1kiiW(
zg@(ded+n_Hp3ALX+tzfnoIieIPCVpNbku*F?3ST_INK@zfb;tWMfa|8TXu-gpI=!+>S~u{j{cUyUZPrI4W(#8kWe9p
zh(FpKB3#oxQyZdnxl4KDbLw9><--1jS3S0|MhI^@Qvz>+)vN_~yzXmEg|_kONN8#Q
zB|IZvY6jk9vTl7F?$$BOEmA{}9;^lSDNxOn>lB@l%iJ>S#kqlJvQdv>5S8cJ3ya$q
zUTVgSIhMi_iZE+QkIaLK>>%2rtQo)2>zi&vsDM$HCr&1XgpMsu{1-{(bmg!1;qg9!v^$)B_T!DVqb#C9J$c^kjn
zu8Mg*^Y_xY&$X6roDTJw2(dN0QLv3Q03+Wlc|jVQLh?z3cic2rBRhshAw^Gq+{lzJmELYZCl0S-bTFw%Qei;U
zJ%UGomw?$Wm9RXVeNUJ9D?MLySE{dBI0R{kOqn?tDU`+udY5Z06#?!^sWx8CbZc!81k3@7?CMeMkKah8O$;XDhQ$QfT6h?kJRHT4cF*KM>Epl
zkuq~}ksz4?XeH|B9n^wPmu)11F3U|UH|%ON$Lmtf+ug;iErFRvFO$kT0^nUuMyp$9
zUPy@n$OyM|H@qA;_I&$0yRA<-O_%VZ$Vh=OtWK$_$hZU*<#}UXN+c#Pm`l8%S)liQ
z^SSEHABx5|oOZeyn!CjwnHB>SF+7faYa;<{>S<604r#x0!l+XICjKJ)SaT&=GKrXc
zBqE9TXj|o8TMk0!#Pb+JLXE}gQxb+7mgyZ$MZz0TdPWIqrERIM>9OO1R0l1TgYHOL
zKIacz^j5gDonX5LU3y6^xB6D}pX8p-g?VGUo<@)7aQk4lq
zJa=DNtT1C?*reRjC|L7hQLCDxQOCr$tR6)8wj9SxnTnd7br_pF17AYmm5AEK=)n;s
znWU4y5zK7iXcJitL85wnuA&T8OGy(4{%;D*z)4D_ot+2|)85@1t9k_q!$5}(FbNv~
zx?y^H`hVVo|GZ@H-by5?S5W&M7Xj{yfL(%u)edi$w{PFFrs2y2MzFYgy-D+W@ta9L
zz^?${1ponVxgBeCvU-Kug1=5=!9r-zE_43!AUTBR&3cRXr=#yjDA?!83u7N={(0-%
zJZsz!F`SAqo_37TRm5ohqpg&a9mgvu&&D+^k
z`KhJM?M3hGoDS<@@~J|`9N{VEyh@pdiLEI$TO_GVaQkLzkKz|sBO@(@It)T(IY&JY
z-h0#apKvExnZ%hv%F6C~>{fxNiptzz|C6!|z~1)W&~S5k`8$A@1f0(R%?3d6%F`yj
zuki<@I(fow#cgeEWn~OK^!QP?cXw@uwSWouOC`Yf`llVpVB+?8eE{6=FVD|6n;ljT
zCP+9wGyxQ{C-ReFsd{py>Y@bT0RB@#bDY;lR)5oMX>PPA0rLrJ#i5#6l$@b=u;DO
z;xxUK!>KJ0_TX%H5Xc{j5c?B@Xf$?^N@-C1fHNAmQd>sbz{yye^MvXh5S3MAg*%6&@uR@!X`c>ei%UgUYXQTBJ|b{+FFRNzZ8
z+mSn+VpF6W2Q(Hb>Z!brZbR)(k5<7p9j`INSh6
zmeyu~Bc4Yhn<=hz@AmMhG+|US*}QTCO|dLts0JR#s`$P+oM0FQxT|`MH_39=6h&_b
ztcN)iNeGwCyTpo%z-N+<@2aPyKj+CkcUpw1GBy@!ZEnr;!3>v?rzA57hMgO(i6#Z&
zO+-Luho~kyHzFfR>n!D(4FUwc?ni*+2|UGbm6fdj^KEtmtS7)`dNWAS^P}lx4Uoha
z%mDN$tCkHM3gJYRvLJwOj|#}_0keNuSsCEa0+4gI;#4?cG611o%yp;t-^&vaB*0F5
zc6kX<`)-eqfqzAfj>#)#R-m;T;MNu(2N^3dKK9|@%D?%;6J&@j?~9E7#bdZyAi@8v
zxh4COlFNtHp~>*O_snAT^rrF?{P*)}?PW5~1
z;dNzHI4yV(B4jZhc^@gBUMqkW5=z;C@YSudF_JY%TX=)DRGP!Jl@N@YGswITf)bRW
zS;6}h5`_20>%shq+T7UbGUv7L7H1)7A&CN@EpDyAcBaVMGhpuW18_2c&+P2%Oh@PQ
zZWI$>CI0iLrH2wsPL7F$0Awlv4;t_zcz0~UAfpY>=Z!Pn2OKVKsc=wr^~<%!kze;m
z1VOJ439ou>?p6Q*oLPN~`zAmKaC`xz#K3Nk6RnDmYq!!90EUxN)nPZly&xx)o4eq`yubbRy6(Lz
zOSk4aNxL?OD_8HKus8Yd?5X)>LI0in23$n@Dt7EFRk@HVy&`X4BqBv2T@HAypIyXM
z{=2MKbOkKys6lTi)%uuqiU1qk0pX3g*M!~!W_)XmaZ_XToolI*JC9NzrG07_h4P6kld0g
zjpWgY10z9gbmS{33E?6{-W^!8t(cj}2a@pUS{P0-H^Q4p>Gs+0V%P>|ol*|tO370{
zN3*OIbaWyH%(82N3HaA`PvFbnI}%RAOdyb9$sS-@g_{lT0mcsN>0EsyBg!}h05E(9
z@QCSirpgyi01uzj#`io?PkVRw`NO8M_0(T)8EdfP=wpzds-q{DsTlbjc@((tSGaXR
zRSP_yx9-$kd{@4cuzN+6kCrgSQ@pO>h_*;%fXT{$z;!IYSnSJBviQDHeoN4lwG
zjoXqLMhgx*Neo6xg&06|aMxEbytIJVJx?7TUoZzr3hzJX
zzMh@}o}LQ;c*EoezyfXA4tP?0bDabXA^-gi${}dMy>&~hsTmn;(Y!o7Nh)Ok)krXB
z%DS-^crk*;$5nvnwKxTfdOb}|JlhU0Cu}4I394PU=$OYrCYY7g2KA@d-)`d*Z9h@Y
z8m#&+x5h0Jl*`J#i+rs6fR2KP#E&H}y~343j@+xSwd}Jb@a^UI-NiQP>FKNhgXkhx
zsoMLdRW-YGrS;rYbFD;RDa1%;l%1|du$*Wmg8*J7oPT6-f!8t(vcy9kSj=0t{fJ(3
zKu@$jG7DN~9kr*R{Ont06EZ3qOjy~c1Z!3J0CC_Z9MTPC-@C&5D$gNwBK3WzW^gfy
zsm{kW)%;z3Thcp_#L(o%SSXLSyJi!+>?id6eA+W6MUh8~SUhVPI4unX#z_>?;a(Kc
z<3wa4!W73)?qdz4Y^8abX$b0XM&_`TLepq^Ro5C?rXot%tadOOW@`}{5{Rvh(AJjc
zIGh{WudKA;SU&J`wyE0e`A5Eq3x2uXuz9^p;L`>s31{bFwgbS;x6}tn)07B>=T%DSe4K7^0%3!U@Myfi}?h~%FPm?MVl;pC6HCcmeri$f?1BVAL(6R
z5iJ}-nTc;YXo
zE>#D0Fn16o5}Pw=f_iE1EORiuWpS%CD*3)C1t^oy6%7DQlaCHwn4+fL`hN*>bM5EP
zgk!SoMJM_~CO21&8_ENYAcVpUa1%?-)nd?Lr+3-LF7$cJx$5^9E!#i-i%G}=Dt8$>
zlc=cZ@;8$nfLD|7`P>2(o8n7NjRcTUAWkI&qDxZ&ym6}+e{A{vE;rjOd3^zn8Gv#H
zAdMATEDQWV@Lv|6jf$!&2l0S5C{#rGj%+a2fc<{z!K*rkTTU?N8Apc?{@(oBjSR~Qd`Up&)9V!RzJ
zw3L9bzHK&C=B1y#RG9UPu^0=3NKbN+a1wi(bGB;K5s2F(MM|aS9)B!3bxzGPPtAy{
zK(T`)BMl1Hp+7CFhRxA|$qW?~-Y1uiXlM;aRD|-3RbgPkJ@7{>j0Tf9hP@y{5u0aO
z^WLht6aPi#rEMR!eIy)WI-+@x*rkDq9;=F8jky5(T09oUxzLGhAUT?df_W?5Z{0{9
z&Zztc@M~-9EWH5epgfVrpG6UGDl6CdW}e%dn+3q2%k7??&dyGt(ZL7w>gkoNCT{fv
z?$O1^?f)~!d*}}m`9M&
zS3kWj{J^NSw1fQ@@@3b=ol*K0CVPjWF|=t4LA61x4&8m2_q*jqz7-_vqmz;MS}m$vi0i?eo;|&Eq@Bx+SYR+?9eS=V|I#H
zkT8;K<{)iO$3rbCwhl*r{@dwqCkbOi9V=`mbRXLqDVQAIR)1aQ1eZeirE>9+NM^|$
zt#}D{tY+|{x@S;B9DRvT=2~14k3w~AjZp>j0Ar1E5P4&;I}O2y
zn1jgD-AAuoZCR4x9F>!GJdr386#w`fO8%Ye>AsmZF>)Vc0ICrXjN(R7q5YR{8e!S<
zGeEls=zae|MgjwSHbQ@kca~A^{u04b%0usBjkb|
z@z%Hx0tWrLyUPQ%6qk}>Vq!5vyY2xc5VSxUS~#Dzb)Yf={cKoE|2GxPp*9`u&X{S$
z{mZoSVOrh#4#D;1Ofu4ydD8p9fa0pwO-0*&--f;qjY%yoEj`(J`M&*sFF|idgg?+T
zBw@-TafLP(nx7DUE)2n@DzFFZ{<&}nx$*rolO%sL9={!7Qq~gw7t%vj+QYc5FK>eN
zPM;#eF)J@cTmp6Heb#u1@;(%0_t>}p4$St~KR(W)h}^PKTN~FIBH8l*U<9R*0LC;j
zGUA$E=fx#^TADHp2SWo^7h9H?R)o@w5)VTcNDShZOV7Uu`kq?GcM$(e8lEZyAGmWz
zNl9rgu>gn#x4o+MjSV0*G2k`#au{!{O?s2wn04r>Ju0(NT{dD5M
zmNp59>Pok=YSiEO4p3t=RLTyINXVQ034oL=fDfjo9#1KHD^~=Ei=CnrD
zJQhm*&S<(U!_}k8e?}AGyW&8T8{E*@iQgU3kevVdoKvNw26VU?$|&LkLT%I&-F_Ub
z-JQbLg(X$AZIz~4iH`YXonJri9;g?Fg0zyFV)x!#y-Vgv_oFeka?{zuphi8iOGsu<
zVlUEp<=p5!&tE=he49pP9t&*&}OL}@@UV_c>RyiBiF
z`Les44x(cs#Cu*`**i&)vePsXoff$_FUGi0k0%)-mm#>%9xmWaNwpA;pV@H!PKOA;
zTpt6)=H}5~m?EA2Z1bZKz{Y8BKb)IOs;~cP%fCLWQNrwJWhJkwnpb<)X2_YRJ~KU?
zK5+nS6M%Z8L=U9Dy6?L}cBrWffbZhJKne4lhm@Ok8gNi(3GNC|aJGxQdV&2Td#+;Y^
zUuFh)3e=idU6e#C|A1$+CJ|Te5IxcFQAq>!b00HBCXwOf83~I}zyS)md*VPx_i5F}
z(t~t^eh)qd-ut#Blq+Z#8!6PSQtU5V{0ju|F1{r#en
zb;iN?qdEXa2I81RgoTgqpUBA#w*P#;r;-r$xZ3$qS!w$AZN~O57khgZdi?OBvEUh36Oth5->J6IJm!W`R?7q
zwTt(~x(;W?*8V?R{&s+I93PJjNm6nI*mFKUtDn~w7Z(9I{j$vnItk?V>28g=r?g=YS(t26bI*+CzDP@z-RH1p&5{h-Ma+6EGZ@>$LGUROR9r%PhsE^a
zO#gI+RP|2N5%A{|LR&COBJ>EMi|l1^;c~|PzopSW@UV@}$Y4+;X5a@kmFc(1%m4Q4
zGS>HEpP2V#q7dBJASK8A{#}rhJr)VUQ%6VV^5WNjc)j%|mMmZ(jYnu;Zvo<|?(dmW
z#sHu^0IvVL)ZZ^dn*;96hC-J;q%1cn
z>|<)8mSdzm54$cuMfA#WP!*lxUIR(LkK3kwm@A+-}{e><=F
zxh)k8E*|ZSf0Px!U5OJpsl=L%6@$v{Dag#*n<*2v=V8l9dJ{0zL{VFZO^?q5$g03g
zsG-~1)AQVYJw;ARiu&fsv>u=a0^wTjyQ8T9MJqP8G1ui^jg7(3kUF(krv$F#Qg$H+
zDRNY3%{!P)U*Grv%)e*vgJn^B=B0a04Qezq@nt|G+x|ZH@VlBghn$B%Qv=N>2cNaB
z2gg8a)Zso`U9*zhN7UhWW(azRK*<0heV56dR(
zf!>Na+zgUJKU7Hyj{}kM&O$T@64K*WE~9ipG@$<_EvPifHb+rFGwY*sz72z&ftQJX
z6qX1^pA?hIBo~JuSww2l7odBg{T$F6ki6YO+hl+#D%*uxm)c!0krKB>)LFvJJyo{!
zy&6&L&n$}ZS3oQPoOooec3mk(GNI{lmdT=vEL=E(W(Q(-`!Ma-KldbK@vk-B7h8`ia
zPcTqX9Nz}OtIl_J5+9@=Tt5KsK6J+&>VEa`1QZsouC7dk|9JSda80W}z+E)itd?t}
z@UCc$VX|B$2wV#}pR<&xr(KB5Mpi~QRK@V;Dv{wdg-cCX6_E(ODm{5dbk4GYhwK|@wyui;+~8sdf5f_c`$M~Ho~y5csA_|)o@ei@
zH|UjxLU<7#{>eOAqdV)LmWv!YXpEQ!9XrZsjvmfsOSXMyvyMj(LFZ@iy8lmw*(&wp$d60WnA(NQ$_OkxVMtGg@KE<
zBk%~iOufTY1qfK2*nj5MF#qcH7t`GT4haDlrsTG9ND+ZFdX|3*I%v>f$~I&cm?5;|
z)WC}uzK1cGla2s!eu2HefFtWZFo5^10x130FGyr$-cFUdf6w}_Uxuq~K;-{4@Pb*J
zOv}B~jSSmRwNa~_|NZ+nfVMllxv}OJGGjTaQPTYhpirtcVuYvG0D>sY10N0AR$aT>
z4_~7B?=d^8pR=lZth#3Yr!G-)WzAQ6XDw{17U9+1I2J~HceeN=u4IVTz+E~bkq#kL
zSvP`hyMBu=`9fj(!sZx*k83XI34@JdQ&QygeA~pi#d|(6l4pB@>!W;r9D11o5i^2#
zAdwDSyXT{e&$*2&QV_=rrRX~;G}Ykst=I6r?$(qTRnbN|>fCE&nWP>4k=TP=e4Zd2
z$+5D*%1p~P3;no+T1yr|2KY$(m|CTpFm#RQGH8mf7r{!rRy0wRXi(xZtq;X^nRUt&
zuO2#qHPLE69bP&o#S_0L*}|I&rLlO!AB`zPq-1NTKfuiF0ObjD=WkI)%%4%ITKq#A
z8Lw)ZdK8w%*5SFwcx5IfEG&4K7CI979K^-|LW%ijz}zi}>Oz2j{8F%j%+=|nw!R)H
z+va$k0j3oor2#ZomCAYWZiF=O2OS;S`*sc1jMF7Q`2+?&9nM-Pf}K`LDJZ@_>~rLK
zCS@p0)EV&nxfE<9{qm)`S_hj`HBR;=(AuD`s*3dOTj6T^`W;2;2f{T~*s_UV*w$@scLpyy~s<`L+;EE#)F0whJJ)udrdAp2X)=S1Jvw;j48
zg&QHkj0e~4bo5W!yL^vTfS&m3=;4y-M*PMvt&KV{t+C1^F*A|8TKIg&Tn$&Xv{J
zz0>eQx&D|^da5;o#jj&UEJR)>>lc~|xeH4z^%T)=eY=^_G|_~??YB1AI+8xjjl;)u
zhef$%<{JA&Z&-WNq@_IGwuxxyATrouHO^rKvC^Q;O^_
zYWq?=R#)Do!-O~2|K9m=VzR+R1H2B9NPs%d9vrcu&a~U^e4OaY`yAxbl^Flt(knd*
zxyc~lvtl_DA!_7BfMO*HIY~TTJRZ7&t)^I@3JO@UyNZQa(009xTZWGGCPIb?Cvjwz
zC7*GhguCcSSHJELhYVE5H@JQ{*qUd<6#KoW%P(=n$3e&r~nF>e3xSkVeBnlV4o2r4ZRRnqApj
zi(^8D*@_0y^nJB_o-vJbcQG24Ow>SkjJW
zdJ_mBHW}xjo*1s>tXBT@6RmLQy=$!p*q}fO0z{GmkdJDen$Da^q&0Tn9#dQl5Az6d
z;@_j%lZggHWZw&5`1MeTce_r$1u!AP!U)jh<7&c8Gc_}_vwz2)B*y(_FE-LkK(m53
zHo{uh`yC#XOQ&a7C6Tpy$3ho!J)Y-$pZ{SK-N7^d%So;D!Hx7t%%jckt|&KwJ;6(c
zoPUSpw%@b&pHE)j6vp6ny?^%AEO}i1sC=5g_FxpWs`fc1CGr;M32}3iVz=5;XgdB=
z3H824-oxhHf{{e6QJ`5_z$ezKZ!*a7rL+%9CoMI?m=X{KOF2CHH}
zcN!BSw*;`YAr%>4$j
=mC?X&R<-N(sZ;14HYf5E&W5Gr1j?ny8W#WaGOYoT0F}0
zJa9-d!)__6`;u4Ne2v|LQlhdTjISX$)@buRvj;|boyQBKHhrfcMJUm*opR?64J@f-
zLNLiDXSAt;$qHhl`|%^@W&y>YBR!0Tf?|(kOT2V8jmL=g-cttKl6bD=WaU
zQ=+l#KgoA>+P)Nc*K9NI^>S<0aaeM+Wno(Zf`LVAu7QXA$^|600)VYAtIq8JG0F>w
zF?1PGW=Y*$2LK#Z4Y_MX&sQ2la_q4U(BNrOs
z@C@lx?CjAAuJCUgt5YKpRv8;WBRnc>U-nqVOSl~p*S|9VP&nmB=SrsulDGtEA6C5iUp@XYOh^VM1gV{-o
zO_tK1T9#147GjlC0o~F3iImK!&qVkXG~cA5%S>2v6F`TBH4CT>?RirMcKP1-7r-p8)Jx9nqW?9hRxtZcH!5rG#2O;H({WRm!#jcFv37DvO&
z<;L&M+M1ybV6`-r5+
zvR1I&b!e#D7wsSmwOf<~m38?Kl^R1eLboxew(HT|_X|&-1@>5<*S_b5lG?OS(}#}N
zA*o>Wmx9wSqN*@b-)qIteom43krv;|6Ym}}@nhT2Fa#oZ`7V??485F5?PTFA>4TK_
zb(Kv_KZ76XlZN4x7kIA;G^wa14WTzE;8yi5DI!2+v(MSPu1<}~fC;tb0~hfMGC$kS
zus85bA=`LC-}m^$db7zjR>)toz8czNz(kKfv&b2%U9b*K3CYcNt;D2-Y6
zonMb2c9V+m_es~?Q!7KWp(%GD!*}$6H?ZhXoUYT>P`r<$Df*tofamH+#v0^Tx);01;{LmpITW0C23+L53jwG27;(ohJ(;
zk~dp_Y5v(JSMX%w
znhz?yyt+Um0#sXp8$3~mWQ~)GdwV+pMeB5&QHUUq!7?{Id7g^RNBw9e*RVgTmhacl
zn;B{47uG_o@F~TN;*5`$u~_j9NY2SED`jBd*gW*(;7SCs8U|REDGI@S}
zPD@J@@ewTnCPK!OMY%a(S3iEtTUjMl%r2AG)o5KkBvjxf&#zCBuLEi(-$5b3dFA@r
zR|sHfEX_lrsYgr%9DE?DcyRO9$9iqfw^>;%O4BUNhDT-1i)(pllkD$r_Vyc(HE{56
z=sV@ImD0Y3qBF^e)glwU5Cy@j58xd#{MEFNuJ8BF)XD@gk$fwrTPX>3RvvBQz|CHN
zEwee?bNc0;=k}!(%hrGg_7+@L-I1bcVm?)UnaauK%q>j%Fh1J^wDYhJeSIs^I4*NR9o`2qGjR@;D)}
zDQOD%9jf2lG^dxour-8H!{Y7z^F5pWo<~g+_`%=Mh&j`p!jelRZG7B|Q^ApSi?g)!
zQ22OJiH=T3mcC&JqbcHM)>j9@{P}XR)qz~3dqQ9rx*&|63(oC@{yp0>#X42(c7V=O
zy2Vq!+VV>bYnymI3%snhUA1kbw|}dU4^zL2-MGj_8|k^HI=Cb@Y_c994IM6#C;t9>
zOV}^L`9Nbyst$X%98@VTNNtVU5ABa{jvUHlQ_U8S#qxwBy&(UNABP+5(x65Jmww%W
z^75c<+hMH4QHMxWJW!225=4iP;pjw6(3+h@0793-fJ}f2!RHVZ7R~
zdH|kL|K-%Ce<3{>VD~gOTH00=Ii$nLhDN`OsSnM}A4RZvZuTSQMAbFBZtPUVmVmsz
ztQ}*?l`i{OThN-A_4y4>OkUP}yj-@D;p59<;Nw%kkCT=cp&8y*Q5Di3KOua5CJ=q%
zM4AMH7_fC-ZbGP;G+DG3wAaT|l&TDkE^Z}Hw)#0v&0K7bZHyP?3xyoTc^_x$rThN4
z1-vwzM@d9XA9MlN{Fj5*r^3RQn@Uy|7AGJC{}Eu+`!98^w;F#l)BOpIs|v)A0eeL2
zlTrW9<%f580vTrDGJq-nAC&#SRC-9^1>kwS-l2Z=qrt7Jt250$B|wkdKR8fVlEeoX
zrqXZ41Ma=fhm?o@wC4T?C*5!x!s6o#zOx}Q`yTv|8T@eO+b6=ucY9){$fVoy)gFzw
zHm%d_YfTl_)sNWJTR8isKg+~|ZyeDCG`n*4JqIi<)GB;llAf#V`$=f|pFW?q)NEdB
zUr6J6q=X|Y6p&sUt)MF!3b8HB%mC)7cd0QI4ohaiJvB9eP3;;0g2>CuqaY*e^`ch+
zk^IE*&(B}>n#4IdXeQb7VLoVzydRu%2%)$$6#IFl?K#OAzMGVTUn`5{Hg@~Q(f=l%
zaKHMGMd0IHylKBi9u8;*gj%|QN6NT?$}`=knhe!8GJ33~5(TO&h(pfVX--CR*Y5yD
z%bzu!exikHnK?Nu)&@BRQ{Q84Vd}1*-VvAf6_UH|-mL=8FnhX`+#ua=GH~CZ{3^m>upk2Z
zUvA862t5r1Ih$P6SC0c0H-RxJlGoFEU;FHv
z{K9|zv)_EbP29gy4r19mOxVfAhL-KUqy2>JFeRZ)GAx0x(V!-B^|1RUYOh|{FWSe
z3}>D3IKf$yxk7#^b?<-u8nt$#Xfj}6G-%F=riQ>F`C9h&h(`Vzarxw<*XsYpZ{FZ>
z2CplWzy@dlR&YpA($Y}pe3&}qb5feSg{smXU4
zU5gtep98oFFkj+|MFs}iG`jD=dDJs#d*0BL?)NZ*cm`
z&w~#cl%WbBAX}Ir&B4{R?_v-m_~p8g4
zm-EO0&%u7&&JKz+t)Zdcb*?|m=zbIB@?o5h@6pgO&)lu9cpy6`E_e$=&5ABOoI*_L
zix?Kx#Dw{3*}T(=vG0^6sWBa!n
z6Sf#U4~yt+s@2})pC$ZYh#MXIfu%Zcw07uR?531wiH+||
z1UcD;+BTD?dZ>&Gw&dkpL+le?X?i6l?dZ2G|1PJ-odZd#uaW~V
z`91&fnt;4!UjS@8{i55L<>KJ5+){02X<4gL)J6<@N4yhGy$dlqJS_UkDlUE&A#Lof
zt2Xbq3E$btM5v#HaCH4wLE7e=F|$V
zd_T|M>m#$hw|w852Xj5R%RhqZou7-qgg&$42n#k(ENb^9e?i1Nv6#k{fePM{2kmSg
z9(Jw#Rz4TMGtjpZB`PnIA`_Mh=gh;QU&iUVfS(X(JfQN!5)I9d@GPbb!ocZdT^O*a
z3jH~QnK?+Z$(4oTz}&TWe8HXqK{ynF1oR#U_JTJgKC<}h{KSs-q{1l&YqM!-%P3>?
zJ)drq--O~cd=_YKKCAY|*ItUE`aUmSE!ibMV&Sef+8jw7Q^&v+?PCd7Fvp|yKwST#
zJr6n4g7fn6QBzm{sH1Zq_@JZ2N*wzEkfl1Z&s>rycVwuyKW{X{_P(g9Vq$B>t*Q%SrB`E9&V3&etu#?!zkWC>wU8`
zd7J7Cb|O|z9NUuoQ+0`WCKDmaVD1sC*4m`YCTrl{X?IV)&!%#qgZ170#$@mf+mr`I
z`-jum@6!YR!4IxKhmVH4GynGH`f@b5C0HS$uP)!ot^0BOU5EAfdiqK{&#NSJ*;=hu
zeEAl{9vn0|ISE8*_cya5poLv%fq>$>xy?S;`M5gJ7@?=Gq`dh{z+Y3zk;Q_Zwq$_`
zgI64^+~o(GPmf%2ge1!|{og*O6b!Uw2T-`8{!A`lgbPwwiLYL-OEEg_3}{S?4rM7F
zwVyu0{lP{Yy|tD!Ja-y|-o(3i%NL4BnZplY6w}lyv4-FfLdVa}Ojj4zZX3=3TWUtG
z7ZiaK8Y_o}d>dFWr55n{wC6JyCO6q)J&L`(pvTkq-y7>Ifn
zHoLmrndPi5Z`yw=lWWAZ{rj%^{vDHPH{N*mu#Z0SHHsS>1qPoam)e5dHkqvGfXM+`
z!Oust?MpuZd^@^RkxA)}P
zvIU>O{A_5621}ziYx{4x#?eNrge2jLSZB}2)1;kEwWiv7=-2UlzSPp|?10KGzHSEV
z=yijD12{y99lK1C%xK~#0jO*Dn*St#1Z_3wG84GY*$~8;g4z<7J!G{u1-6#wm-mTxhT+(G3@`GZ#epM>8Z(+R{2=KVX1SB#dy
zV{?=Qw)Z6o)GWlVkNUoYL>C}XKMG4aM32emW9fGk3Z_9B4U$;4Hwg5;PvV($g-ef*
z9ru=SeT9>`lS{pjZ$=&W+jPY#?23>bg%GyYot=~AP>}l={ZsHO_3H&reDqeTd<^vH
zkS3oYRo_3v<+I$IhC
z3(jM%@Gq&sPbs&>lqZ4${&%zoGd@SDOM9lQt5@}TB7Lr3Ho~DJ*aYk2i#Qjd$ZvHP
z5GmZ;e&u5q5>2FRj4wj{*OFhA>bk+K5->aMvI1*g$eQbY;v>TZ5N6y%D}CVk^1Mb3
z(l665`ySVZNnm4J;)2Jh>wKASOk3I8`;x4X>;Lj87Yt%%J)S}jyTE4zmVD(7G+Dk>
zPmALR0X?kEMm9Ar@+y@U4o&3Hv!*6Hek+#dbHnu^eFDCG(7?sBbtUtAHsYGmPh})-
z=R{e=ZH@QN&dj9Tq>L4QO$!18?5nQuS^U$z(+&6N!MtzhZY`CSOU?bz#+Tub@4yy>hQhEid{w~A=k0fuipLNaAdgm?;wL9NFGpu4n1p54RJD%MC9H60dW*yV5B%;bfSlA2czsxDTQgu@$Lmo;OU8;n~tPe@20g
zS`P1>ymIAuC*sYv)vO%p)T6rclBB>SU1w_#>jEw$NmvEc*Cm;nT6B2!%lezfTgN}p
zZJvj2BW##ENtYW=)kd#(%4C{(a{x9~deJ->0uQX;aEJj4KI
z=(lKS{vTWJ^0CsE-ND7>MaJwM>Y#EaDx7IS(@HJKpv_vxFfQV4GprZ%u($iw$0ZI{@P+J$P@}^wERA#llKxGn7+#xH0ozAY(ux6<-mP
zsj70}C+b8?F4xultHJgVkO5fdBH$E@yc3&Jm^hZSDZ!$Mi7C$Gw6#Nn>Gir$B$q#R
z@@h9NuLXB+X$*9Eza8S
zJsO;eu@NSAnR4&HFb)d+esi6jf4_(1MXL|ukPoDl5#asS(zJ1K3DBT2tgPY@MSXw>RaY_b^G(v@+uGRNm3?qg9dggg<{~DZ;RWtxIq!;91`*MmD;RXZ
z2-pTq!v9tYsCyxfDBbGx+-bSXUSDtL>+FstPrAHi
z@F$HPO1Cv9pO8WjjeGa-jFxQotBK0tQyhI-f1oRexA)1K!Q5?v$d9wXjwEJzOSd;x
z$waHal)&ki0bi2EDqNR0nZ_-RSz%1*JUl8YoD%Bd(`Ub}xhcd|_#J^|9XYt&;PYPI
zn@?ABd){6)k2m?{Esap}DO_zdQcQm?{CCPvo55_92X64w6l96@!^@`o*>ulGv9tMR
zuYG^6bd7rAPHwixagri$6N2n$`a2E~K=>`Ks#EWm{k0K{mM8^K&7Rme-Il21hdD8I
zpiXsq+}B&PjAFNj&_CS)c{}8jiLX}M!4o{M4&q;mgNh@pCo9(5FV_D_S2~P_PzT3o
zq!gj3LuyIRab@g0?*_J|uqOM3<2~y<>&Pt+(Y`FL=C$|4I`XxAs#(pygJL9`^&`>!
zqsOKqIOdFgJaVlN`cRd7M%vibL4Hrvhd{L}McqqBF+0m+ZTaJ_sXgZpCK*lv=av-m
zajY7*=p?GfJCWh4Fw+P!TXJ}XQ1nUA3tBJM>Ocr;BdU;Fp#5Z)5Gk}fKLJ@}i_<9_
zPw9~ctENT|*5tgpZZ{dJ-)uWc{V++3I>;J@3ODuPAu!0r1=}8&x_Jn^^m_u7@Y9nM
zV*qgG@w){W835Yx04qHHedpnd5wKGAYt12EI*L34?kbhtg8b(pb$Bk3{?fPUTJf@m
zYEshJmlN1jg648SjE|tGh{)>%3kBTi>ExE1Tab8xLYecbTV7gp*yF#>5ojeV>z5?$
zDeKzt@jOr}%za7-*xQGrsm0&4wDWM9|Gjm863mHWqk}C@q;6S-O|?7(7$F}T}*q~)(%v|&d!)vMAq3Oa)yRg-(=?-oaN-0G8>(*
z{23UU+En?!lAtfQ{WexNfX;9#?P{w~$gyrR!>Le7hG|i7_!VPcnGXcmhp=Qc&pWAd
zA<$tkX`p$hL|hJIJ~q8=Tj~qAeiSD*`T6&2-)rvLN%}4mdQIZqrUqUky&adqsCx=!
zipG~WM$E}2tBkgR;;dgJ!KiQ2b8`M)#3IwF*8jZ-D&*zjU_2@
z_UG^o@9Aldh5ngq{O~YY=UAElu7Vn?V$cKGfm~L@(gi|>Eqw}Tg}Yh!~AN}@5>3>(9Clg
zCrPpI%L@28kob%p(kZ6b@c6bbiN8R;2?fE!v#8H3A
zbZIS3*G_CQRxD4*1Fc978RYFH)%GPVo!SM9A4Do~Y%$AO=7}yIJ@n$jf@A#ro^jGV)}
z&reQKH!Lkz+lH3f+MMv-8YrF$*m4-|e;;)#PBg~fkqct~$gHqj<;0N)wNg$lds4z3
zvRfsC%~A+$URw98zh3xJK*z{O?z=Wl@1NI#9!I^G@RFmWqHJyLCdbBZgMauY(4f})
zy@M0n!NkFF{oQ#xno2id-cV7o|KjG9=$zksz+L!c4|07e*=Y_99@GcXkG
zz$lVIxB~qfD6K(Co=yQ3RWTvSu3*U~GoD@ICO5lKau$p&$h{lhf!^NTIoXu+h`?@m
z_qeMG%kf?09vZE7Jtn3(_$Aw76H~{xLnP!6MEM#JJ28wuZdMZbZ3^3m$T%crWg$zU
zRjL$?8Wat>=r^~J4x;W#B4asag|OYL;yb9|h*!arixySyChQqgYG2MSd~ANueGIz#
zPg*Uvh8A^`4k3|qVKos5^5c)51+y=Rf(b7EjoXOewyZs3nWRCs#2^9(}6!mZ5NmMyE7H`Nb%k;u_hZn2D(zBBIFs{khTYlTTihirbq~42+tD0_5`7{w*;|
zQ%WEGxuPzK;cMcVG~m&7rU+0_{(%<;^Bb6lbn$iFgYcxMEeIPitl0xM;U&DnsJ$*4
zv(oqb=Q-hy39v8H=O7=qHz2A_
zz`KQckJNni`jYd5GfCFL9b0|o9P8h+%{OBf$n@UY@%Jf5Wp4-FLft^AW@8-~k}jH}
zS!-{>yKCb0rqo0>)ja#+JsOnhgQp#e*Nvxhfy#~UXfdveP+kWb#VZS{7kCL$8!zx!
z*LTMRE)2({4!BSx;wJ0T52XNnhQV^HJUle$EPcQYs
z_#EREb%;6Bm9nLy+@79oHSb$mpLyTPf=MaLbaI+&Df}oZHwB}1BAmN6Z%6dLfw8Hv
z4T$OJ;tQiX2u;y{ZiVT@J0DKxc5$GFxhVnbzByf~JYU3D{N;WJF!liB$Uvrt!>3OG
zIy4Y431UPJc5OVLRa7v@YVPmvV^at=xNh_Ux|+5NV=gwI(kRYg|c8c!=FNhz$7^XX0D*6m;o#B80X{?!=>I@$3xHvV@NT#eWN|nb}2H-
z>OyY~qsX@MVWx1G)Kdu)>qWItbbcW8v_6M^K00gudWvPB0bfMkkZx1J;PE<X0=LqUDRDc
z@^KERs$;^Dh5ZPhxYbhO(rxfra&QQw&BN$Y4HE-}#^87~^fj-nJBkX?6VLcQA8cm@
zYtm*8WrYuWF_r4){rdZ(*XRXNQkTNfp|gFvyvpJz{)>qGRkfCG1`kQNaYf&Bc0L!(
zrt({&WvW*|qdNJ8Vr#|~7n=c?MMvF2uaUjFk=c-??54#z-N}skq>(x8Zs)wLdE-z*
z*rO%vC@$Nl-WW_^QO#*fo|HEq+*YI?3AD
zVv?2ASX*NG>~Z(buJh+c)QY+t!DW7UYT4|RXmFwvAz=?L?`1}+$Ebe@`uCFl_X;nV
zpR4t4kaWsph&A}K`r9QXFs1#g!F}XO`pWbBJw*ySU;Sv!DvU?qw54z?6?O+gx&3HJ
zqp;H7587T{8WnnzK?m6VcgN-~R?~uwv_`8B8C8#OBuRK{nN(N`c?*Z+=33d(d^xW!
z#-X|S`^_hbT#Wkze~H)UBoUFX!hG3(%XH9^F~aSpXe>6qSMthpr%~otY)Tz*AK|fX
zYuIC{du#WEN0n{Y3m)g(qUdlRsr;yeAv~^0JRL_sA4T`!E!scMgH(ba(lk;$yrIbO
zVQBcC$|Sm3=JZ7Vai%BS55#Vngm?b};~XpvYM|QiofZ>et$yjX)#aI)e~!njM~?rj
zbp?)g;v`@Xwb_p>VdQ6sLxHz+~2TtbZ8VQs4F5W-sX6h=zX;%
z)hJ`$+eLq%n0O*4E4k>;S9lp_?9l#=B!&AT`QKVqQU4Eo%qJiayh}%|_+Pld+UN70
z^WfLVP)QoMo5NXP90|~C1_lNIRHdkA+dfI$gJreW-FVw>9xv9CJJUgT%~JoBAoyee
z4x`(Yg-%DO$bk^OrPY12Xm?iTYbTFPc#_eFVJu8gtn@&Njf}LGrYO~XzRdlmm^k8)
z+A@6#f&%{D)%FRmz_=SlMa3aRKMQ(enNB~XZ_K^$&HdoPC9oIKt-opX;P+Gya;frh##=UGI7a-E>VhUY8TL!i(FIjSZF*AZa@gK#an5w>=uk_$j9#HP97=cg=^H!9|u-z802`D4X?)~oD6+bv=KPPo_b37Q`#1X9=RfjF-Fi@
z5QFDQz^|p;ET7~ls`7n~G9w!Iz8H?1ZAriYbtPSCxlIZWh?NS#{mvR=2-VOr}f?L!I3E)WQirTE-nJ=F-sXkZ#0@>j6=Rq35_}#v4pmkygc{7-P5;K
zI~lnzU$c1!w=L2*ZWpP?IS?DNs5vk|bU;3l#3tz%1YD=fS#YhZ%
zKqA7qzL3HuW=dx+Y}r5&YO8
z&_d}<>8+F~t!;Yw7kP-7$HR0hegh3Eht>broT_RDY!4U^zYpwn6I`30_k^*>AOALW
zA0`5hN`rP66L0TZfDTS1YtZ-d0G3delhXEvkvFJH*y8<+UXteuM_|}$NE_2I`F13{
z!ln?n?LKJHRSZ85qN?oGgA~o05oqxl%=I%9G7)Cw!lQn~MWV`f%1JezDaCnlO!QB|CliIgvSDhpEm0~1={fJ;i=ua4
z`HbmU$fv2aKixr!9t=sJ^|YG{ZZmWk6on(r+i(#mjJW45Bv?}NnfPb6hgo+N)4a5O
z;|L4)kLWOO<{Sp3ux86m;%5O+qoe_yaqNf$}~%(i9~e>CoiXc
zF&57VM{Ju-p`$eVGYft~Fx9Z4JKojjNXPk}%PZjibkLGuo@%aOm=lQx-EPPTKBmzt
z6Y*;LaOM-|ooHrjFX4Te1wzon+H|ddqVR@JUe=K3sZn^C6eE_{v=F;U>?3h?1eand5NZ4tt8UBt{qjj6jgCIxgOodXmv3vcH+MgFm$`6SEzCT-b37F#q
zTv_^t;$JKWCwKR^<2hb{1R^m3fu*%IpyzXZ{Q$s}KSE{CDf(U9Uy!oOrFtg}DT`#=1tKfWP@vj3
z)n>qI90CgI;LIJktkz^uA_8ytwi_GSZa_Yt+|fIs%~^N2VZ;~rV{#-YKkW3JT*Q%t
zhmnZ&@))_9uvajNxNB5QOAxeDw3$D|qCnSxHVERu|NLSA(CbKyGg%ZThF*OOXFE
z0Tz6gUC$rJ=GEBmNh%}+2S*{;Tf#n;L5(LIzw@+C)M!j~l@b~8SX|8%RPQ@f_#%lB
zcHsf&mpt($OkhyQ@VWCUzbX%|{zjXgQ}zUVVQAl)6B%W$=0h>6knW1!r68zg|}O=JZTfPM*2d+(*{~B?hX|q
z%UK2CnL58}e|u6sYxvrw@oo4ri9xQC;m@C#51gLG)^oYcyG|J7c|<)M5aN-MILPn>
zW?S?KQ-5At`hqDSMrgN;vy~bVddCTQusBeE*WO@b;@QTo^G*4-6}w6$3WFtX*KK
zBxfr72%nT?x{`<#+LX_dGRN-tP$jgC93S6HB&Kbn*rclM>y=wR*;Sh5qNYt2NC7n=
zP0G%B+m28LnRe{!!doA<)UHrONEo$;
z_>Y0?
zI{$U`Bllf;>_Ko1ua+mSqjI@M)B$92NZ)Q6{p5a2PpwZ)}9WqN0Hf3vv~ePmqStqk$t7
zBaLh+KWpJM@oZBptl}Bt;ls7(=YQeowudi`HQ^ohrj+wC&^2+N(}B|f?VB>r&oD%?
zEPZs2f}jEU{xF%~mz^iOo#nYFDTn>@y@eDsFul<9N$s2RlsLw7YpgLQmf_MV%@s>X
zIKRe{^gAg}9?p+*2O7-#V`WnC&=EuZIOM3(pMFnebNa$bi4Gx&)H$>W{0TB5Ol}J%
z%+7ys9AvO~!_?Fc%>9OUt~rBix|pt6uMW*(R)_>>Bebu#Zk5NS?o`Gb%VpdxA#P{B
zy}cK;t%b^*9XFo;l2@xQCC+RrJ#K`kUc=SQfdRoxI6SyK10aS7A+&#MC>k)lb7zYN
z-d_xpm~}s`#&h)EZ@ipt6!>k&0{z6I`uhKJqD1!Oe9qUMbD8ai9skKfX&<
zOIK?7$itJ|#*55v9dEYZIum3mqV!%J~Mi6-~SAA@u2sTJ&*a17@y-8vG7R1BZ!cOFiquVGG53$xO`19
zH&(8xZa0hrBi4(s*sjHW^WPUB1-^hgAik6o6ck)ux({&zlM)&R2L3?SC^i<>&&5S0
z@C^X+LwE|?YYu9Lk4Iv>-}n1|-}iODuIs+9=Oy(N17osgfkIK$Kz?h@
zda3~drhWA25iy2ppHqd+owhd`dW^e#>y&CoBb6$TQcuc=GvZ4lMz7+H?d_d_9aAKU_m=YaIEZQC|labaqFK2VCF#@r
z0^*OY&_#S5mFNY0SBvA#&X+n0P1&^3TqcJjePZmlFR!@Rl1;B&94rH|r%8N4%=5^>VwrSP0UZ^1N?zJk0mN;B>;m->9yh8r5yx;j%Ouk?`
zY{T7#UX#PwNAztBl^2JaXBxU^&t;T0^2V(W)XL
zPS!UKMMpL&T-_xM?=Z4q2C&NJ>^<5QI7VC2IN~)Po#&vVa;=*c5
zYrBUH?bT^XO*bcA@7zdQb4EB6V5!liy$JPlhXZ;okfTQDQXAcxGU+e$?)X(YZBAOb
z_f*J{F5|8dA*F`HGZt4a;WolkblZCr-1Jo6U6M?-hD|99z|^ss^1yW)5HIFG$0ekc
z`Bj>mnuZ{AkWqT_?&*i2f`KI0sPD3mD66ZhuU-vY3X;GJPTn;$q+257IyJ=ViwG&m
z@8ojn7hb9CJ$fv!?eSGb5DbwIg<*nkroF4HkMc8-NEE+*znt)GhqXxLPF|_5
zS@WfWJeq-2n_S^rATPpar-+Rpl2;&LA}uXP>-4XHXoF;F5N#Nxt5&Uo6~F6odjI~0
zf%by+@z2M1ZAvXew1^Z48swY5m2Y&DA0u(U8Y
zFU#Y)r`OWb!v3_oyE_`dPm&~2f-d;;xBT0O`5Xz
zMAeSP_$-EvZf}QuB6imt=@F7~W~Mibuuas-9F6+3Tj^Ylg^^MTS5^ZI4tYehY4F|s
zK8CBbl=e0snJ_a01+z0Vn$|PTGI28?BDN>m!z`%vU)zTvAb2?oaEr}JGf$Li?*ZtI
zbNJY=r-~b>)#HuY@X!wX>^ao>^1K1kXJ|uZtmL_JtTKW}a@VnZMj|DfNs`N}c|Ni!
zpZ)vxk-*l7kB^5PI&Uz|9*GDN474-YI;^r0x=iZmQ?o7Oa6$fQdwU@PiKWqI9~d>3
z4eid>-?>wghbjQwQ5pb3Dnv}(&l+OWn7Z5BJ39viq3Eriu?or&;K7jJVmbS+xpp|9
z*x&uNS?qoA!qRj
z@8@dx7XJ(M>>u>KrE(8;%Wv?6su@x
z$h!J0%5RPlUHzcET=qeU{dkVM&ERKfPYpOUxP_SdTBF90evy9DT21x`=`mY8~;9Mm0
zOk#ByB+&!$&-wY<&7Q4BYZ{KObekTCm@-_`P~z>Fpvfw;w|IA1t<~tuIw*lIkRBc~
zT^yaAC4-i_FxXLMlDgYZG;~uKL!6o&@!yL-VqNZDN)6^E*@L*sZR(p~*nY2-hyesh
zbFGj=p(FD@ASj6I$I#8C6x-^?f8WMrasg%y>bl%;x12M>5!S+vAGWcPjHl4Wvvk=k
zGM6SmHz#W)zN-zV&4xKM6%`e!f&y1IRVJZ}>h+`_eWV|=qtsPauQp{55h>&vGq0m8
zM30v?H8pLqG=cHN^_{2^K@caTZv%Q3g8*}31oSOtfJ8bwBEwyS#_E;0!96j<7pt^U
zsYdkvDiVr1b_%;DZtI0(?)DQRzu(gi(oH(U*Mp+U&i$PvlfBt_9;3nD
zb+(gPrO3n8BSs%9Ql}ZNwZfW_aD3hH>{(ZzyFscmEzmU9UnUV!`*v_c+6!C)dSZ;X
z_zczeavFmN@^Wh@GDf<{tv1N(x=*oE59y7^fm~M+@m#TwH7{0L|_p$9aC!h>Z%lRN2*~79t!x`%uL%q7Y4JXmAzp8^VXcEzFu}
z%O$Qwn3SaS?V*0d2U7pD_~F=N!Zqq&9~=+qsUxx|}Y?
z%V_f+=StYzFPAPY#sb;@{_$0qGC04STZfG{ubP{=LELbAmk=J&{gut(TUIXgepkwi
z9CXY+Vgm9v);QD->e^rWZje($(V&XQ4r$#VPX(H>6S=^>y}je>aViPqHSn$*&l^Ch
zRyvU`Uc49@O5}suT1@v&PEL+*2UW1vzpp7#I*A36_)pknfYrp=PuE;Yxd&4?Ha6zp
zWV&Jb^^0j!n1M4~nS^T}xGpPn00>lLQkU}L`IkIBXJ%%Iv7R|xV=>QIn^{^2YZn0c
z&_F@4OF3!Hy#Rg(P_hTfK(^Ym;Cw>Hb*0N!>x#pU#=Q=1s$A$=L;L}O}$
zY@;Wdc8u%?chVAfFl+kdxbDwk78)`82-6LdeqoWTXezQ-4b@*G`2@L+cPF|^+;)HB?L`ph;
z{&(04^*Gni{@YZUkV>b@P)de4JF9fRI|WXi=Fr
z9mb$AFK=4hXQ@Y|rm<4$f)AD2L?FV*;QTH(8e(R6#`#X#*I)#xp(eh?)8%vqUZBN-%N7P
z8@y|4LswtHU^`o4Vqoy*?b}a*J9nzv`uk5{cBQy`$^%5Xav`832NPB>WS72Dt>CnAO}j7JMx#YNSTUA*zd@A5h{VC#Np_Gg+tEwF#ruH1_^@a%7LmSa9{-yWW*AQV)G3&yxcO
z3*O!O_3Me8VjPyZl}ms?bHI6;%nbv=IEUo_gC
zln&$7a?wa7=0HNPI$vqITo=p$EQt?=#oF%O`2f?vYTcNqCk{+uAc-h;J-Aas#JEejnI=
z3M9`o@vV5x7cs91mmlC6B!O2GhR5B5=2Jnj>WZ!}EliAMH$J%So)x8?EMCYaAnt}WRX@8dVi%-ads|gM8_xI1UyRk24_~%ELTqe>wI!-z>
z^=N~on1Ao_HVJLy1Hs?jeJ1FD(8JUSo@m>3x%c1dEL(s*O}owY$EKyrMtwd>{2NdIdI7_EPtm(F28(&`Vt6ODNm08k(%CF9$pAME*
zb>%%8=JjwM`syXa6ON;bd(eegg3zg4OeP9vLftskW*HDF8?@%o{|ELLwOYPhytrb`
zQ-Zq+yDJQ>X^7LX)m)zJ77!SicE{Af;MkUw*8a(tFJFFqm5wM-))Sl$%?RT3A0FMd
z_*`gcRb}O=WL}J_aco?io{r9McT7!9^EMs`3JTInT)ldAyjvxmPK8{I-2M3OwYHX)
z%gRNfffwcCs)IR$q|lH}g2*CN(n(+cz>
z*%__Ev#4(OGGPyyoWUgtb<$P%yVrKdjVs}}!J9__585$#DRFzCejyOCnj?CquS`yN
z(pd$cBPCtD&T#QL%v;RF3e+*+z~AxVRGX^Al$DnsOdyLzV}6x4UtNIGZ!+yuJC#hV
z>Ff8$Y9?AUN@F!FhnIYk2e*?<20EU@;qbIH%!6Nhk4JO=fM6SeE^NJHLmgA1UVc1J
zseCc#wg`7WYg(#dag)Jo!r;Q#yo>pkdT3uV532EAN>?-h7BKnTue^G`#%bRjC>2>2nc!!~4RrsuMIPCb8mZ%#Hz{v^$Rc)sS;wrQ;0Y
z6nsinTbq%YnM!p1VMfbW3fZxA&&?)1=n!k?4eu}Bu-~p4PpjyhPIw9_#8P~S3+2PGQg-!wOC*0eVrw>4f_rse-
zf8k&>L^yV>#V7<25gd#)yUv(RKCZ9a1$|DH6wBIeyed$aSg*1-f!M?-FAwtdMRu=*
zq?MSOnHeFQg)V1Z-i==%{EzTi1o&`TE9H<@aplLfAM`6o>?*t{AXhFy3Bf56s&Ylo
zAZQ4(iAB~3VJ#3qLpzB03qZ%jace|&Om+7nR!Wor`*E(~W^L^h{sHH3K|7*T7`h^?
z@7}B_$W-dIo@{S2Przql(8~thEwO5_A6~MzwQZge@PCxd3JE_L@sQb$B4?5&WqXq@Hy`8r`q^~Ih?IR_+h6w}$C^#s7lQ5BQiG!M|Dy5mi
zxsMn%v_AaN9f)>z!V|^~-?p~)tTGK2bw?2~FA!Buajit02h5Iexaf{Gzi
z!|Vc&PE6_T=}_3*nf)UUl}=Lh-_U$@tcU1bXmMiXB76bBH+$Q6=TFs&#~FCKJfbE^
z9Qao-u*7nO;0hwR5Do9Isgiuv+%Zx{B80S*k%YT#Z{Cz5$cZy{N)U7~he30%_*I6%
zh{9SvEH7_<`O@y>$;@#c5!APx7I9_leoj=%HJ=FnN4OnMct^Q>sr3G)8_SnOu}?2c
zb`wlaR>M0+c(tys4q^?O9;_~vm@0+`03
zjRu6Xh`%4ao#5#I5eRI-vlvxCkH8Jt`9zAgT4Hg7E794IEV}sN~B@TrJ1vCs0
zgo$k#Nnc;L!b55$T5B#(&_s>^BrdM<5KR~hc4~cqMM%4-;@189t$e{rJG@eiY2{>?R?ZbeH!
ziCRa5?)Og#Zy;bAHgw6A7BX>xRR)hthXLgh+?vi=8MBXFzXFW}yX4|ULZc#{P@R^h
z(s21sZtgalOP4ND5PPaLXL6m}7GuW(Lmw{}oN#!a`T3a$li@pX8lVv?oth-ActuLC
zVPmi~^`3G+-5VrAO5-cnw$z8KzZq?*(+krsRw!VSLWMMLIN-?@2jAQ@U0esTO03*+
zBCp%WUJUtVl+5jA%$gFlN1Ch|&Ta;&SZHGQWo2~ttn$FwRB%8^EeVFHtT;s5E-N+5
zcTK7j{OYR1(+Ouwegi&!uc_|UM*49)*zIH8zQJzGD=9~oU$??=zwfUK32KlUmZ_EH
zg1gy-n|hQT+MIMiD-o}NWFWEjVj<$x+}_*Tr2h-74>mM`x+%-QZGa)g%zFfo>
z-m}|`m{s~6)UYe{E5`UU%P7jrJ0(?UHS#8ra6^>J?9(`k<^T{&D9YTRN+%Zw2j7X|
zt~2txmL)?C^5(@`DG5VVHnEcXYab9=-o=0Yp6`6W6w{#VTZSi0
zdJa6vdZEMcOF8@1=gXvQjF;E^yaV^2m*1-R-QSnp^>mHgpLfN!?Yyt%T>0Fd>Ysj=
z`{y(Ny_x@P@GX%0>$VErUt9hA{-*r@lgsZNReVBIXKnfCd8Q{6pDeWuj_Wb`=hdZD
z@;I_iLHa*Ersu9rkv$9$4D@p}K6msosnf|RX?oh}y7)gT=Kehb^lUc>q-%cPK^3g{
zO78D|WxT#mk)0)uqy9N)s?B%WkGLL^U-kDZPVeiNL|Z@lJfcQR$IWY~TfcDl`|Y00
zqzm=5<}CW_=T8_7BZL3`af30+l(b$G$2b1GUhs=q2}PEabmHHh=U!*k9iBAY*?G@K
zr!gm4VPT=Xl$th?+pyb9v{p_|ZV7p>1Noefx3?RKEEgPfKET^Ic3!^(KlzsU$@w#6
zH?wnoZum#eK!2-&uKsr2tp;`mTg?oN%(fcq>KmEq>rYngeE1)~;O%qX<3h-P|AolD
S$S?2
Date: Wed, 13 Jun 2012 02:14:29 +0000
Subject: [PATCH 3741/8313] Added a comment: librsync

---
 .../comment_1_3ea263b1f334e8e38e14f00a96202988._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment

diff --git a/doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment b/doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment
new file mode 100644
index 0000000000..4a011f61be
--- /dev/null
+++ b/doc/design/assistant/progressbars/comment_1_3ea263b1f334e8e38e14f00a96202988._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://abhidg.myopenid.com/"
+ ip="129.67.132.87"
+ subject="librsync"
+ date="2012-06-13T02:14:29Z"
+ content="""
+There's librsync which might support reporting the progress through its API, but it seems to be in beta.
+"""]]

From ab076b2e818e616c1049ec1119d77ca2ff2e3a17 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 00:57:48 -0400
Subject: [PATCH 3742/8313] move comment

---
 Command/Watch.hs | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 22a534d458..6f85c124cf 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -236,6 +236,11 @@ onAdd file = do
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
  - before adding it.
+ - 
+ - This is often called on symlinks that are already staged correctly.
+ - A symlink may have been deleted and being re-added, or added when
+ - the watcher was not running; so it always stages even symlinks that
+ - already exist.
  -}
 onAddSymlink :: Handler
 onAddSymlink file = go =<< Backend.lookupFile file
@@ -250,13 +255,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 					liftIO $ createSymbolicLink link file
 					addlink link
 				)
-		{- This is often called on symlinks that are already staged
-		 - correctly, especially during the startup scan. A symlink
-		 - may have been deleted and re-added, or added when
-		 - the watcher was not running; so it always stages
-		 - even symlinks that already exist.
-		 -
-		 - So for speed, tries to reuse the existing blob for
+		{- For speed, tries to reuse the existing blob for
 		 - the symlink target. -}
 		addlink link = do
 			v <- catObjectDetails $ Ref $ ':':file

From 12dbb9d1d0162d5417805503525f30faf9aa2fc2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 01:20:37 -0400
Subject: [PATCH 3743/8313] plumb file status through to event handlers

The idea, not yet done, is to use this to detect when a file
has an old change time, and avoid expensive restaging of the file.

If git-annex watch keeps track of the last time it finished a full scan,
then any symlink that is older than that time must have been scanned
before, so need not be added. (Relying on moving, copying, etc of a file
all updating its change time.)

Anyway, this info is available for free since inotify already checks it,
so it might as well make it available.
---
 Command/Watch.hs   | 18 ++++++++---------
 Utility/Inotify.hs | 49 ++++++++++++++++++++++++++--------------------
 2 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index 6f85c124cf..b97a4212d2 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -68,7 +68,7 @@ import System.INotify
 
 type ChangeChan = TChan Change
 
-type Handler = FilePath -> Annex (Maybe Change)
+type Handler = FilePath -> Maybe FileStatus -> Annex (Maybe Change)
 
 data Change = Change
 	{ changeTime :: UTCTime
@@ -181,9 +181,9 @@ runChangeChan = atomically
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
-runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> IO ()
-runHandler st changechan handler file = void $ do
-	r <- tryIO (runStateMVar st $ handler file)
+runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO ()
+runHandler st changechan handler file filestatus = void $ do
+	r <- tryIO (runStateMVar st $ handler file filestatus)
 	case r of
 		Left e -> print e
 		Right Nothing -> noop
@@ -214,7 +214,7 @@ noChange = return Nothing
  - startup.
  -}
 onAdd :: Handler
-onAdd file = do
+onAdd file _filestatus = do
 	ifM (Annex.getState Annex.fast)
 		( go -- initial directory scan is complete
 		, do -- expensive check done only during startup scan
@@ -243,7 +243,7 @@ onAdd file = do
  - already exist.
  -}
 onAddSymlink :: Handler
-onAddSymlink file = go =<< Backend.lookupFile file
+onAddSymlink file filestatus = go =<< Backend.lookupFile file
 	where
 		go Nothing = addlink =<< liftIO (readSymbolicLink file)
 		go (Just (key, _)) = do
@@ -270,7 +270,7 @@ onAddSymlink file = go =<< Backend.lookupFile file
 			madeChange file "link"
 
 onDel :: Handler
-onDel file = do
+onDel file _filestatus = do
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.unstageFile file)
 	madeChange file "rm"
@@ -283,14 +283,14 @@ onDel file = do
  - command to get the recursive list of files in the directory, so rm is
  - just as good. -}
 onDelDir :: Handler
-onDelDir dir = do
+onDelDir dir _filestatus = do
 	Annex.Queue.addCommand "rm"
 		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
 	madeChange dir "rmdir"
 
 {- Called when there's an error with inotify. -}
 onErr :: Handler
-onErr msg = do
+onErr msg _ = do
 	warning msg
 	return Nothing
 
diff --git a/Utility/Inotify.hs b/Utility/Inotify.hs
index 6eb7be31c6..9ad947f315 100644
--- a/Utility/Inotify.hs
+++ b/Utility/Inotify.hs
@@ -15,7 +15,7 @@ import qualified System.Posix.Files as Files
 import System.IO.Error
 import Control.Exception (throw)
 
-type Hook a = Maybe (a -> IO ())
+type Hook a = Maybe (a -> Maybe FileStatus -> IO ())
 
 data WatchHooks = WatchHooks
 	{ addHook :: Hook FilePath
@@ -84,15 +84,18 @@ watchDir i dir ignored hooks
 			| otherwise = []
 
 		scan f = unless (ignored f) $ do
-			let fullf = indir f
-			r <- catchMaybeIO $ getSymbolicLinkStatus fullf
-			case r of
+			ms <- getstatus f
+			case ms of
 				Nothing -> return ()
 				Just s
-					| Files.isDirectory s -> recurse fullf
-					| Files.isSymbolicLink s -> addSymlinkHook <@> f
-					| Files.isRegularFile s -> addHook <@> f
-					| otherwise -> return ()
+					| Files.isDirectory s ->
+						recurse $ indir f
+					| Files.isSymbolicLink s ->
+						runhook addSymlinkHook f ms
+					| Files.isRegularFile s ->
+						runhook addHook f ms
+					| otherwise ->
+						noop
 
 		-- Ignore creation events for regular files, which won't be
 		-- done being written when initially created, but handle for
@@ -100,39 +103,43 @@ watchDir i dir ignored hooks
 		go (Created { isDirectory = isd, filePath = f })
 			| isd = recurse $ indir f
 			| hashook addSymlinkHook =
-				whenM (filetype Files.isSymbolicLink f) $
-					addSymlinkHook <@> f
+				checkfiletype Files.isSymbolicLink addSymlinkHook f
 			| otherwise = noop
 		-- Closing a file is assumed to mean it's done being written.
 		go (Closed { isDirectory = False, maybeFilePath = Just f }) =
-			whenM (filetype Files.isRegularFile f) $
-				addHook <@> f
+				checkfiletype Files.isRegularFile addHook f
 		-- When a file or directory is moved in, scan it to add new
 		-- stuff.
 		go (MovedIn { filePath = f }) = scan f
 		go (MovedOut { isDirectory = isd, filePath = f })
-			| isd = delDirHook <@> f
-			| otherwise = delHook <@> f
+			| isd = runhook delDirHook f Nothing
+			| otherwise = runhook delHook f Nothing
 		-- Verify that the deleted item really doesn't exist,
 		-- since there can be spurious deletion events for items
 		-- in a directory that has been moved out, but is still
 		-- being watched.
 		go (Deleted { isDirectory = isd, filePath = f })
-			| isd = guarded $ delDirHook <@> f
-			| otherwise = guarded $ delHook <@> f
+			| isd = guarded $ runhook delDirHook f Nothing
+			| otherwise = guarded $ runhook delHook f Nothing
 			where
 				guarded = unlessM (filetype (const True) f)
 		go _ = noop
 
 		hashook h = isJust $ h hooks
 
-		runhook h f
+		runhook h f s
 			| ignored f = noop
-			| otherwise = maybe noop (\a -> a $ indir f) (h hooks)
-		h <@> f = runhook h f
+			| otherwise = maybe noop (\a -> a (indir f) s) (h hooks)
 
 		indir f = dir  f
 
+		getstatus f = catchMaybeIO $ getSymbolicLinkStatus $ indir f
+		checkfiletype check h f = do
+			ms <- getstatus f
+			case ms of
+				Just s
+					| check s -> runhook h f ms
+				_ -> noop
 		filetype t f = catchBoolIO $ t <$> getSymbolicLinkStatus (indir f)
 
 		-- Inotify fails when there are too many watches with a
@@ -144,10 +151,10 @@ watchDir i dir ignored hooks
 					Just hook -> tooManyWatches hook dir
 			| otherwise = throw e
 
-tooManyWatches :: (String -> IO ()) -> FilePath -> IO ()
+tooManyWatches :: (String -> Maybe FileStatus -> IO ()) -> FilePath -> IO ()
 tooManyWatches hook dir = do
 	sysctlval <- querySysctl [Param maxwatches] :: IO (Maybe Integer)
-	hook $ unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval
+	hook (unlines $ basewarning : maybe withoutsysctl withsysctl sysctlval) Nothing
 	where
 		maxwatches = "fs.inotify.max_user_watches"
 		basewarning = "Too many directories to watch! (Not watching " ++ dir ++")"

From c31ddeda84542414dd58e03473a23a6de8890390 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 02:48:52 -0400
Subject: [PATCH 3744/8313] optimise link staging at startup

Now it starts really, really fast! Down from 15 minutes or so on my big
tree to around 1 minute.

The trick is to remember the last time the daemon was running. Links with a
ctime from before that point don't need to be restaged on startup (as long
as they are correct), since the old daemon would have handled them already.

We also assume that if the daemon has never run before, any links that
already exist are good. The pre-commit hook fixes links, so this should be
a safe assumption.

Adds another MVar holding a DaemonStatus data structure. Also
allowed getting rid of the Annex.Fast hack. This data structure will
probably grow a lot of details about the daemon's status, that will
later be used by the webapp's UI.

The code to actually track when the daemon was last running is not written
yet. It's 3 am.
---
 Command/Watch.hs | 144 +++++++++++++++++++++++++++++++----------------
 1 file changed, 97 insertions(+), 47 deletions(-)

diff --git a/Command/Watch.hs b/Command/Watch.hs
index b97a4212d2..54be556c9e 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -24,12 +24,18 @@
  - Thread 5: committer
  - 	Waits for changes to occur, and runs the git queue to update its
  - 	index, then commits.
+ - Thread 6: status logger
+ - 	Wakes up periodically and records the daemon's status to disk.
  -
  - State MVar:
  - 	The Annex state is stored here, which allows resuscitating the
  - 	Annex monad in IO actions run by the inotify and committer
  - 	threads. Thus, a single state is shared amoung the threads, and
  - 	only one at a time can access it.
+ - DaemonStatus MVar:
+ - 	The daemon's current status. This MVar should only be manipulated
+ - 	from inside the Annex monad, which ensures it's accessed only
+ - 	after the State MVar.
  - ChangeChan STM TChan:
  - 	Changes are indicated by writing to this channel. The committer
  - 	reads from it.
@@ -59,6 +65,7 @@ import Control.Concurrent
 import Control.Concurrent.STM
 import Data.Time.Clock
 import Data.Bits.Utils
+import System.Posix.Types
 import qualified Data.ByteString.Lazy as L
 
 #if defined linux_HOST_OS
@@ -66,9 +73,28 @@ import Utility.Inotify
 import System.INotify
 #endif
 
+data DaemonStatus = DaemonStatus
+	-- False when the daemon is performing its startup scan
+	{ scanComplete :: Bool
+	-- Time when a previous process of the daemon was running ok
+	, lastRunning :: Maybe EpochTime
+	}
+
+newDaemonStatus :: Annex DaemonStatus
+newDaemonStatus = return $ DaemonStatus
+	{ scanComplete = False
+	, lastRunning = Nothing
+	}
+
+getDaemonStatus :: MVar DaemonStatus -> Annex DaemonStatus
+getDaemonStatus = liftIO . readMVar
+
+modifyDaemonStatus :: MVar DaemonStatus -> (DaemonStatus -> DaemonStatus) -> Annex ()
+modifyDaemonStatus status a = liftIO $ modifyMVar_ status (return . a)
+
 type ChangeChan = TChan Change
 
-type Handler = FilePath -> Maybe FileStatus -> Annex (Maybe Change)
+type Handler = FilePath -> Maybe FileStatus -> MVar DaemonStatus -> Annex (Maybe Change)
 
 data Change = Change
 	{ changeTime :: UTCTime
@@ -96,43 +122,40 @@ start :: Bool -> Bool -> CommandStart
 start foreground stopdaemon = notBareRepo $ do
 	if stopdaemon
 		then liftIO . stopDaemon =<< fromRepo gitAnnexPidFile
-		else withStateMVar $ startDaemon (not foreground)
+		else withStateMVar $ startDaemon foreground
 	stop
 
 startDaemon :: Bool -> MVar Annex.AnnexState -> Annex ()
-startDaemon False st = do
-	showStart "watch" "."
-	liftIO $ watch st
-startDaemon True st = do
-	logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
-	pidfile <- fromRepo gitAnnexPidFile
-	liftIO $ daemonize logfd (Just pidfile) False $ watch st
+startDaemon foreground st
+	| foreground = do
+		showStart "watch" "."
+		go id
+	| otherwise = do
+		logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
+		pidfile <- fromRepo gitAnnexPidFile
+		go $ daemonize logfd (Just pidfile) False
+	where
+		go a = do
+			daemonstatus <- newDaemonStatus
+			liftIO $ a $ do
+				dstatus <- newMVar daemonstatus
+				changechan <- runChangeChan newTChan
+				watch st dstatus changechan
 
-watch :: MVar Annex.AnnexState -> IO ()
+watch :: MVar Annex.AnnexState -> MVar DaemonStatus -> ChangeChan -> IO ()
 #if defined linux_HOST_OS
-watch st = withINotify $ \i -> do
-	changechan <- runChangeChan newTChan
-	let hook a = Just $ runHandler st changechan a
-	let hooks = WatchHooks
-		{ addHook = hook onAdd
-		, delHook = hook onDel
-		, addSymlinkHook = hook onAddSymlink
-		, delDirHook = hook onDelDir
-		, errHook = hook onErr
-		}
+watch st dstatus changechan = withINotify $ \i -> do
 	-- The commit thread is started early, so that the user
 	-- can immediately begin adding files and having them
 	-- committed, even while the startup scan is taking place.
 	_ <- forkIO $ commitThread st changechan
-	-- The fast flag is abused somewhat, to tell when the startup
-	-- scan is still running.
-	runStateMVar st $ do
-		setfast False
+	runStateMVar st $
 		showAction "scanning"
 	-- This does not return until the startup scan is done.
 	-- That can take some time for large trees.
 	watchDir i "." (ignored . takeFileName) hooks
-	runStateMVar st $ setfast True
+	runStateMVar st $
+		modifyDaemonStatus dstatus $ \s -> s { scanComplete = True }
 	-- Notice any files that were deleted before inotify
 	-- was started.
 	runStateMVar st $ do
@@ -140,7 +163,14 @@ watch st = withINotify $ \i -> do
 		showAction "started"
 	waitForTermination
 	where
-		setfast v= Annex.changeState $ \s -> s { Annex.fast = v }
+		hook a = Just $ runHandler st dstatus changechan a
+		hooks = WatchHooks
+			{ addHook = hook onAdd
+			, delHook = hook onDel
+			, addSymlinkHook = hook onAddSymlink
+			, delDirHook = hook onDelDir
+			, errHook = hook onErr
+			}
 #else
 watch = error "watch mode is so far only available on Linux"
 #endif
@@ -181,14 +211,16 @@ runChangeChan = atomically
  -
  - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
  -}
-runHandler :: MVar Annex.AnnexState -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO ()
-runHandler st changechan handler file filestatus = void $ do
-	r <- tryIO (runStateMVar st $ handler file filestatus)
+runHandler :: MVar Annex.AnnexState -> MVar DaemonStatus -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO ()
+runHandler st dstatus changechan handler file filestatus = void $ do
+	r <- tryIO go
 	case r of
 		Left e -> print e
 		Right Nothing -> noop
 		Right (Just change) -> void $
 			runChangeChan $ writeTChan changechan change
+	where
+		go = runStateMVar st $ handler file filestatus dstatus
 
 {- Handlers call this when they made a change that needs to get committed. -}
 madeChange :: FilePath -> String -> Annex (Maybe Change)
@@ -214,14 +246,13 @@ noChange = return Nothing
  - startup.
  -}
 onAdd :: Handler
-onAdd file _filestatus = do
-	ifM (Annex.getState Annex.fast)
-		( go -- initial directory scan is complete
-		, do -- expensive check done only during startup scan
-			ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
-				( noChange
-				, go
-				)
+onAdd file _filestatus dstatus = do
+	ifM (scanComplete <$> getDaemonStatus dstatus)
+		( go
+		, ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
+			( noChange
+			, go
+			)
 		)
 	where
 		go = do
@@ -237,24 +268,43 @@ onAdd file _filestatus = do
  - Or, if it is a git-annex symlink, ensure it points to the content
  - before adding it.
  - 
- - This is often called on symlinks that are already staged correctly.
- - A symlink may have been deleted and being re-added, or added when
- - the watcher was not running; so it always stages even symlinks that
- - already exist.
  -}
 onAddSymlink :: Handler
-onAddSymlink file filestatus = go =<< Backend.lookupFile file
+onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file
 	where
-		go Nothing = addlink =<< liftIO (readSymbolicLink file)
 		go (Just (key, _)) = do
 			link <- calcGitLink file key
 			ifM ((==) link <$> liftIO (readSymbolicLink file))
-				( addlink link
+				( ensurestaged link =<< getDaemonStatus dstatus
 				, do
 					liftIO $ removeFile file
 					liftIO $ createSymbolicLink link file
 					addlink link
 				)
+		go Nothing = do -- other symlink
+			link <- liftIO (readSymbolicLink file)
+			ensurestaged link =<< getDaemonStatus dstatus
+
+		{- This is often called on symlinks that are already
+		 - staged correctly. A symlink may have been deleted
+		 - and being re-added, or added when the watcher was
+		 - not running. So they're normally restaged to make sure.
+		 -
+		 - As an optimisation, during the status scan, avoid
+		 - restaging everything. Only links that were created since
+		 - the last time the daemon was running are staged.
+		 - (If the daemon has never ran before, avoid staging
+		 - links too.)
+		 -}
+		ensurestaged link daemonstatus
+			| scanComplete daemonstatus = addlink link
+			| otherwise = case filestatus of
+				Just s
+					| safe (statusChangeTime s) -> noChange
+				_ -> addlink link
+			where
+				safe t = maybe True (> t) (lastRunning daemonstatus)
+
 		{- For speed, tries to reuse the existing blob for
 		 - the symlink target. -}
 		addlink link = do
@@ -270,7 +320,7 @@ onAddSymlink file filestatus = go =<< Backend.lookupFile file
 			madeChange file "link"
 
 onDel :: Handler
-onDel file _filestatus = do
+onDel file _ _dstatus = do
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.unstageFile file)
 	madeChange file "rm"
@@ -283,14 +333,14 @@ onDel file _filestatus = do
  - command to get the recursive list of files in the directory, so rm is
  - just as good. -}
 onDelDir :: Handler
-onDelDir dir _filestatus = do
+onDelDir dir _ _dstatus = do
 	Annex.Queue.addCommand "rm"
 		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
 	madeChange dir "rmdir"
 
 {- Called when there's an error with inotify. -}
 onErr :: Handler
-onErr msg _ = do
+onErr msg _ _dstatus = do
 	warning msg
 	return Nothing
 

From ccc50052453ccaf2db0c371c5c36b5eea3e9191a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 12:36:33 -0400
Subject: [PATCH 3745/8313] reorganize

---
 Assistant.hs               |  74 +++++++
 Assistant/Committer.hs     | 104 ++++++++++
 Assistant/DaemonStatus.hs  |  35 ++++
 Assistant/ThreadedMonad.hs |  40 ++++
 Assistant/Watcher.hs       | 206 ++++++++++++++++++++
 Command/Watch.hs           | 386 +------------------------------------
 6 files changed, 463 insertions(+), 382 deletions(-)
 create mode 100644 Assistant.hs
 create mode 100644 Assistant/Committer.hs
 create mode 100644 Assistant/DaemonStatus.hs
 create mode 100644 Assistant/ThreadedMonad.hs
 create mode 100644 Assistant/Watcher.hs

diff --git a/Assistant.hs b/Assistant.hs
new file mode 100644
index 0000000000..bc394bd998
--- /dev/null
+++ b/Assistant.hs
@@ -0,0 +1,74 @@
+{- git-annex assistant daemon
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -
+ - Overview of threads and MVars, etc:
+ -
+ - Thread 1: parent
+ - 	The initial thread run, double forks to background, starts other
+ - 	threads, and then stops, waiting for them to terminate,
+ - 	or for a ctrl-c.
+ - Thread 2: inotify
+ - 	Notices new files, and calls handlers for events, queuing changes.
+ - Thread 3: inotify internal
+ - 	Used by haskell inotify library to ensure inotify event buffer is
+ - 	kept drained.
+ - Thread 4: inotify initial scan
+ -	A MVar lock is used to prevent other inotify handlers from running
+ -	until this is complete.
+ - Thread 5: committer
+ - 	Waits for changes to occur, and runs the git queue to update its
+ - 	index, then commits.
+ - Thread 6: status logger
+ - 	Wakes up periodically and records the daemon's status to disk.
+ -
+ - ThreadState: (MVar)
+ - 	The Annex state is stored here, which allows resuscitating the
+ - 	Annex monad in IO actions run by the inotify and committer
+ - 	threads. Thus, a single state is shared amoung the threads, and
+ - 	only one at a time can access it.
+ - DaemonStatusHandle: (MVar)
+ - 	The daemon's current status. This MVar should only be manipulated
+ - 	from inside the Annex monad, which ensures it's accessed only
+ - 	after the ThreadState MVar.
+ - ChangeChan: (STM TChan)
+ - 	Changes are indicated by writing to this channel. The committer
+ - 	reads from it.
+ -}
+
+module Assistant where
+
+import Common.Annex
+import Assistant.ThreadedMonad
+import Assistant.DaemonStatus
+import Assistant.Watcher
+import Assistant.Committer
+import qualified Utility.Daemon
+import Utility.LogFile
+
+import Control.Concurrent
+
+startDaemon :: Bool -> Annex ()
+startDaemon foreground
+	| foreground = do
+		showStart "watch" "."
+		go id
+	| otherwise = do
+		logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
+		pidfile <- fromRepo gitAnnexPidFile
+		go $ Utility.Daemon.daemonize logfd (Just pidfile) False
+	where
+		go a = withThreadState $ \st -> liftIO $ a $ do
+			dstatus <- startDaemonStatus
+			changechan <- newChangeChan
+			-- The commit thread is started early, so that the user
+			-- can immediately begin adding files and having them
+			-- committed, even while the startup scan is taking
+			-- place.
+			_ <- forkIO $ commitThread st changechan
+			watchThread st dstatus changechan
+
+stopDaemon :: Annex ()
+stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
new file mode 100644
index 0000000000..d6fc085793
--- /dev/null
+++ b/Assistant/Committer.hs
@@ -0,0 +1,104 @@
+{- git-annex assistant change tracking and committing
+ -
+ - Copyright 2012 Joey Hess 
+ -}
+
+module Assistant.Committer where
+
+import Common.Annex
+import Assistant.ThreadedMonad
+import qualified Annex.Queue
+import qualified Git.Command
+
+import Control.Concurrent
+import Control.Concurrent.STM
+import Data.Time.Clock
+
+type ChangeChan = TChan Change
+
+data Change = Change
+	{ changeTime :: UTCTime
+	, changeFile :: FilePath
+	, changeDesc :: String
+	}
+	deriving (Show)
+
+runChangeChan :: STM a -> IO a
+runChangeChan = atomically
+
+newChangeChan :: IO ChangeChan
+newChangeChan = atomically newTChan
+
+{- Handlers call this when they made a change that needs to get committed. -}
+madeChange :: FilePath -> String -> Annex (Maybe Change)
+madeChange file desc = do
+	-- Just in case the commit thread is not flushing the queue fast enough.
+	Annex.Queue.flushWhenFull
+	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
+
+noChange :: Annex (Maybe Change)
+noChange = return Nothing
+
+{- Gets all unhandled changes.
+ - Blocks until at least one change is made. -}
+getChanges :: ChangeChan -> IO [Change]
+getChanges chan = runChangeChan $ do
+	c <- readTChan chan
+	go [c]
+	where
+		go l = do
+			v <- tryReadTChan chan
+			case v of
+				Nothing -> return l
+				Just c -> go (c:l)
+
+{- Puts unhandled changes back into the channel.
+ - Note: Original order is not preserved. -}
+refillChanges :: ChangeChan -> [Change] -> IO ()
+refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs
+
+{- This thread makes git commits at appropriate times. -}
+commitThread :: ThreadState -> ChangeChan -> IO ()
+commitThread st changechan = forever $ do
+	-- First, a simple rate limiter.
+	threadDelay oneSecond
+	-- Next, wait until at least one change has been made.
+	cs <- getChanges changechan
+	-- Now see if now's a good time to commit.
+	time <- getCurrentTime
+	if shouldCommit time cs
+		then void $ tryIO $ runThreadState st commitStaged
+		else refillChanges changechan cs
+	where
+		oneSecond = 1000000 -- microseconds
+
+commitStaged :: Annex ()
+commitStaged = do
+	Annex.Queue.flush
+	inRepo $ Git.Command.run "commit"
+		[ Param "--allow-empty-message"
+		, Param "-m", Param ""
+		-- Empty commits may be made if tree changes cancel
+		-- each other out, etc
+		, Param "--allow-empty"
+		-- Avoid running the usual git-annex pre-commit hook;
+		-- watch does the same symlink fixing, and we don't want
+		-- to deal with unlocked files in these commits.
+		, Param "--quiet"
+		]
+
+{- Decide if now is a good time to make a commit.
+ - Note that the list of change times has an undefined order.
+ -
+ - Current strategy: If there have been 10 commits within the past second,
+ - a batch activity is taking place, so wait for later.
+ -}
+shouldCommit :: UTCTime -> [Change] -> Bool
+shouldCommit now changes
+	| len == 0 = False
+	| len > 10000 = True -- avoid bloating queue too much
+	| length (filter thisSecond changes) < 10 = True
+	| otherwise = False -- batch activity
+	where
+		len = length changes
+		thisSecond c = now `diffUTCTime` changeTime c <= 1
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
new file mode 100644
index 0000000000..43a8c7e2f0
--- /dev/null
+++ b/Assistant/DaemonStatus.hs
@@ -0,0 +1,35 @@
+{- git-annex assistant daemon status
+ -
+ - Copyright 2012 Joey Hess 
+ -}
+
+module Assistant.DaemonStatus where
+
+import Common.Annex
+
+import Control.Concurrent
+import System.Posix.Types
+
+data DaemonStatus = DaemonStatus
+	-- False when the daemon is performing its startup scan
+	{ scanComplete :: Bool
+	-- Time when a previous process of the daemon was running ok
+	, lastRunning :: Maybe EpochTime
+	}
+
+type DaemonStatusHandle = MVar DaemonStatus
+
+newDaemonStatus :: DaemonStatus
+newDaemonStatus = DaemonStatus
+	{ scanComplete = False
+	, lastRunning = Nothing
+	}
+
+startDaemonStatus :: IO DaemonStatusHandle
+startDaemonStatus = newMVar newDaemonStatus
+
+getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
+getDaemonStatus = liftIO . readMVar
+
+modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex ()
+modifyDaemonStatus status a = liftIO $ modifyMVar_ status (return . a)
diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs
new file mode 100644
index 0000000000..c4d331f61b
--- /dev/null
+++ b/Assistant/ThreadedMonad.hs
@@ -0,0 +1,40 @@
+{- making the Annex monad available across threads
+ -
+ - Copyright 2012 Joey Hess 
+ -}
+
+{-# LANGUAGE BangPatterns #-}
+
+module Assistant.ThreadedMonad where
+
+import Common.Annex
+import qualified Annex
+
+import Control.Concurrent
+
+{- The Annex state is stored in a MVar, so that threaded actions can access
+ - it. -}
+type ThreadState = MVar Annex.AnnexState
+
+{- Stores the Annex state in a MVar.
+ -
+ - Once the action is finished, retrieves the state from the MVar.
+ -}
+withThreadState :: (ThreadState -> Annex a) -> Annex a
+withThreadState a = do
+	state <- Annex.getState id
+	mvar <- liftIO $ newMVar state
+	r <- a mvar
+	newstate <- liftIO $ takeMVar mvar
+	Annex.changeState (const newstate)
+	return r
+
+{- Runs an Annex action, using the state from the MVar. 
+ -
+ - This serializes calls by threads. -}
+runThreadState :: ThreadState -> Annex a -> IO a
+runThreadState mvar a = do
+	startstate <- takeMVar mvar
+	!(r, newstate) <- Annex.run startstate a
+	putMVar mvar newstate
+	return r
diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
new file mode 100644
index 0000000000..19a65db6e8
--- /dev/null
+++ b/Assistant/Watcher.hs
@@ -0,0 +1,206 @@
+{- git-annex assistant tree watcher
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Assistant.Watcher where
+
+import Common.Annex
+import Assistant.ThreadedMonad
+import Assistant.DaemonStatus
+import Assistant.Committer
+import Utility.ThreadLock
+import qualified Annex.Queue
+import qualified Command.Add
+import qualified Git.Command
+import qualified Git.UpdateIndex
+import qualified Git.HashObject
+import qualified Git.LsFiles
+import qualified Backend
+import Annex.Content
+import Annex.CatFile
+import Git.Types
+
+import Control.Concurrent.STM
+import Data.Bits.Utils
+import qualified Data.ByteString.Lazy as L
+
+#if defined linux_HOST_OS
+import Utility.Inotify
+import System.INotify
+#endif
+
+type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Maybe Change)
+
+watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
+#if defined linux_HOST_OS
+watchThread st dstatus changechan = withINotify $ \i -> do
+	runThreadState st $
+		showAction "scanning"
+	-- This does not return until the startup scan is done.
+	-- That can take some time for large trees.
+	watchDir i "." (ignored . takeFileName) hooks
+	runThreadState st $
+		modifyDaemonStatus dstatus $ \s -> s { scanComplete = True }
+	-- Notice any files that were deleted before inotify
+	-- was started.
+	runThreadState st $ do
+		inRepo $ Git.Command.run "add" [Param "--update"]
+		showAction "started"
+	waitForTermination
+	where
+		hook a = Just $ runHandler st dstatus changechan a
+		hooks = WatchHooks
+			{ addHook = hook onAdd
+			, delHook = hook onDel
+			, addSymlinkHook = hook onAddSymlink
+			, delDirHook = hook onDelDir
+			, errHook = hook onErr
+			}
+#else
+watchThread = error "so far only available on Linux"
+#endif
+
+ignored :: FilePath -> Bool
+ignored ".git" = True
+ignored ".gitignore" = True
+ignored ".gitattributes" = True
+ignored _ = False
+
+{- Runs an action handler, inside the Annex monad, and if there was a
+ - change, adds it to the ChangeChan.
+ -
+ - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
+ -}
+runHandler :: ThreadState -> DaemonStatusHandle -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO ()
+runHandler st dstatus changechan handler file filestatus = void $ do
+	r <- tryIO go
+	case r of
+		Left e -> print e
+		Right Nothing -> noop
+		Right (Just change) -> void $
+			runChangeChan $ writeTChan changechan change
+	where
+		go = runThreadState st $ handler file filestatus dstatus
+
+{- Adding a file is tricky; the file has to be replaced with a symlink
+ - but this is race prone, as the symlink could be changed immediately
+ - after creation. To avoid that race, git add is not used to stage the
+ - symlink.
+ -
+ - Inotify will notice the new symlink, so this Handler does not stage it
+ - or return a Change, leaving that to onAddSymlink.
+ -
+ - During initial directory scan, this will be run for any files that
+ - are already checked into git. We don't want to turn those into symlinks,
+ - so do a check. This is rather expensive, but only happens during
+ - startup.
+ -}
+onAdd :: Handler
+onAdd file _filestatus dstatus = do
+	ifM (scanComplete <$> getDaemonStatus dstatus)
+		( go
+		, ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
+			( noChange
+			, go
+			)
+		)
+	where
+		go = do
+			showStart "add" file
+			handle =<< Command.Add.ingest file
+			noChange
+		handle Nothing = showEndFail
+		handle (Just key) = do
+			Command.Add.link file key True
+			showEndOk
+
+{- A symlink might be an arbitrary symlink, which is just added.
+ - Or, if it is a git-annex symlink, ensure it points to the content
+ - before adding it.
+ -}
+onAddSymlink :: Handler
+onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file
+	where
+		go (Just (key, _)) = do
+			link <- calcGitLink file key
+			ifM ((==) link <$> liftIO (readSymbolicLink file))
+				( ensurestaged link =<< getDaemonStatus dstatus
+				, do
+					liftIO $ removeFile file
+					liftIO $ createSymbolicLink link file
+					addlink link
+				)
+		go Nothing = do -- other symlink
+			link <- liftIO (readSymbolicLink file)
+			ensurestaged link =<< getDaemonStatus dstatus
+
+		{- This is often called on symlinks that are already
+		 - staged correctly. A symlink may have been deleted
+		 - and being re-added, or added when the watcher was
+		 - not running. So they're normally restaged to make sure.
+		 -
+		 - As an optimisation, during the status scan, avoid
+		 - restaging everything. Only links that were created since
+		 - the last time the daemon was running are staged.
+		 - (If the daemon has never ran before, avoid staging
+		 - links too.)
+		 -}
+		ensurestaged link daemonstatus
+			| scanComplete daemonstatus = addlink link
+			| otherwise = case filestatus of
+				Just s
+					| safe (statusChangeTime s) -> noChange
+				_ -> addlink link
+			where
+				safe t = maybe True (> t) (lastRunning daemonstatus)
+
+		{- For speed, tries to reuse the existing blob for
+		 - the symlink target. -}
+		addlink link = do
+			v <- catObjectDetails $ Ref $ ':':file
+			case v of
+				Just (currlink, sha)
+					| s2w8 link == L.unpack currlink ->
+						stageSymlink file sha
+				_ -> do
+					sha <- inRepo $
+						Git.HashObject.hashObject BlobObject link
+					stageSymlink file sha
+			madeChange file "link"
+
+onDel :: Handler
+onDel file _ _dstatus = do
+	Annex.Queue.addUpdateIndex =<<
+		inRepo (Git.UpdateIndex.unstageFile file)
+	madeChange file "rm"
+
+{- A directory has been deleted, or moved, so tell git to remove anything
+ - that was inside it from its cache. Since it could reappear at any time,
+ - use --cached to only delete it from the index. 
+ -
+ - Note: This could use unstageFile, but would need to run another git
+ - command to get the recursive list of files in the directory, so rm is
+ - just as good. -}
+onDelDir :: Handler
+onDelDir dir _ _dstatus = do
+	Annex.Queue.addCommand "rm"
+		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
+	madeChange dir "rmdir"
+
+{- Called when there's an error with inotify. -}
+onErr :: Handler
+onErr msg _ _dstatus = do
+	warning msg
+	return Nothing
+
+{- Adds a symlink to the index, without ever accessing the actual symlink
+ - on disk. -}
+stageSymlink :: FilePath -> Sha -> Annex ()
+stageSymlink file sha =
+	Annex.Queue.addUpdateIndex =<<
+		inRepo (Git.UpdateIndex.stageSymlink file sha)
diff --git a/Command/Watch.hs b/Command/Watch.hs
index 54be556c9e..5681b38619 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -1,108 +1,20 @@
 {-# LANGUAGE CPP #-}
 {-# LANGUAGE BangPatterns #-}
 
-{- git-annex watch daemon
+{- git-annex watch command
  -
  - Copyright 2012 Joey Hess 
  -
  - Licensed under the GNU GPL version 3 or higher.
- -
- - Overview of threads and MVars, etc:
- -
- - Thread 1: parent
- - 	The initial thread run, double forks to background, starts other
- - 	threads, and then stops, waiting for them to terminate,
- - 	or for a ctrl-c.
- - Thread 2: inotify
- - 	Notices new files, and calls handlers for events, queuing changes.
- - Thread 3: inotify internal
- - 	Used by haskell inotify library to ensure inotify event buffer is
- - 	kept drained.
- - Thread 4: inotify initial scan
- -	A MVar lock is used to prevent other inotify handlers from running
- -	until this is complete.
- - Thread 5: committer
- - 	Waits for changes to occur, and runs the git queue to update its
- - 	index, then commits.
- - Thread 6: status logger
- - 	Wakes up periodically and records the daemon's status to disk.
- -
- - State MVar:
- - 	The Annex state is stored here, which allows resuscitating the
- - 	Annex monad in IO actions run by the inotify and committer
- - 	threads. Thus, a single state is shared amoung the threads, and
- - 	only one at a time can access it.
- - DaemonStatus MVar:
- - 	The daemon's current status. This MVar should only be manipulated
- - 	from inside the Annex monad, which ensures it's accessed only
- - 	after the State MVar.
- - ChangeChan STM TChan:
- - 	Changes are indicated by writing to this channel. The committer
- - 	reads from it.
  -}
 
 module Command.Watch where
 
 import Common.Annex
+import Assistant
 import Command
-import Utility.Daemon
-import Utility.LogFile
-import Utility.ThreadLock
-import qualified Annex
-import qualified Annex.Queue
-import qualified Command.Add
-import qualified Git.Command
-import qualified Git.UpdateIndex
-import qualified Git.HashObject
-import qualified Git.LsFiles
-import qualified Backend
-import Annex.Content
-import Annex.CatFile
-import Git.Types
 import Option
 
-import Control.Concurrent
-import Control.Concurrent.STM
-import Data.Time.Clock
-import Data.Bits.Utils
-import System.Posix.Types
-import qualified Data.ByteString.Lazy as L
-
-#if defined linux_HOST_OS
-import Utility.Inotify
-import System.INotify
-#endif
-
-data DaemonStatus = DaemonStatus
-	-- False when the daemon is performing its startup scan
-	{ scanComplete :: Bool
-	-- Time when a previous process of the daemon was running ok
-	, lastRunning :: Maybe EpochTime
-	}
-
-newDaemonStatus :: Annex DaemonStatus
-newDaemonStatus = return $ DaemonStatus
-	{ scanComplete = False
-	, lastRunning = Nothing
-	}
-
-getDaemonStatus :: MVar DaemonStatus -> Annex DaemonStatus
-getDaemonStatus = liftIO . readMVar
-
-modifyDaemonStatus :: MVar DaemonStatus -> (DaemonStatus -> DaemonStatus) -> Annex ()
-modifyDaemonStatus status a = liftIO $ modifyMVar_ status (return . a)
-
-type ChangeChan = TChan Change
-
-type Handler = FilePath -> Maybe FileStatus -> MVar DaemonStatus -> Annex (Maybe Change)
-
-data Change = Change
-	{ changeTime :: UTCTime
-	, changeFile :: FilePath
-	, changeDesc :: String
-	}
-	deriving (Show)
-
 def :: [Command]
 def = [withOptions [foregroundOption, stopOption] $ 
 	command "watch" paramNothing seek "watch for changes"]
@@ -121,296 +33,6 @@ stopOption = Option.flag [] "stop" "stop daemon"
 start :: Bool -> Bool -> CommandStart
 start foreground stopdaemon = notBareRepo $ do
 	if stopdaemon
-		then liftIO . stopDaemon =<< fromRepo gitAnnexPidFile
-		else withStateMVar $ startDaemon foreground
+		then stopDaemon
+		else startDaemon foreground -- does not return
 	stop
-
-startDaemon :: Bool -> MVar Annex.AnnexState -> Annex ()
-startDaemon foreground st
-	| foreground = do
-		showStart "watch" "."
-		go id
-	| otherwise = do
-		logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
-		pidfile <- fromRepo gitAnnexPidFile
-		go $ daemonize logfd (Just pidfile) False
-	where
-		go a = do
-			daemonstatus <- newDaemonStatus
-			liftIO $ a $ do
-				dstatus <- newMVar daemonstatus
-				changechan <- runChangeChan newTChan
-				watch st dstatus changechan
-
-watch :: MVar Annex.AnnexState -> MVar DaemonStatus -> ChangeChan -> IO ()
-#if defined linux_HOST_OS
-watch st dstatus changechan = withINotify $ \i -> do
-	-- The commit thread is started early, so that the user
-	-- can immediately begin adding files and having them
-	-- committed, even while the startup scan is taking place.
-	_ <- forkIO $ commitThread st changechan
-	runStateMVar st $
-		showAction "scanning"
-	-- This does not return until the startup scan is done.
-	-- That can take some time for large trees.
-	watchDir i "." (ignored . takeFileName) hooks
-	runStateMVar st $
-		modifyDaemonStatus dstatus $ \s -> s { scanComplete = True }
-	-- Notice any files that were deleted before inotify
-	-- was started.
-	runStateMVar st $ do
-		inRepo $ Git.Command.run "add" [Param "--update"]
-		showAction "started"
-	waitForTermination
-	where
-		hook a = Just $ runHandler st dstatus changechan a
-		hooks = WatchHooks
-			{ addHook = hook onAdd
-			, delHook = hook onDel
-			, addSymlinkHook = hook onAddSymlink
-			, delDirHook = hook onDelDir
-			, errHook = hook onErr
-			}
-#else
-watch = error "watch mode is so far only available on Linux"
-#endif
-
-ignored :: FilePath -> Bool
-ignored ".git" = True
-ignored ".gitignore" = True
-ignored ".gitattributes" = True
-ignored _ = False
-
-{- Stores the Annex state in a MVar, so that threaded actions can access
- - it.
- -
- - Once the action is finished, retrieves the state from the MVar.
- -}
-withStateMVar :: (MVar Annex.AnnexState -> Annex a) -> Annex a
-withStateMVar a = do
-	state <- Annex.getState id
-	mvar <- liftIO $ newMVar state
-	r <- a mvar
-	newstate <- liftIO $ takeMVar mvar
-	Annex.changeState (const newstate)
-	return r
-
-{- Runs an Annex action, using the state from the MVar. -}
-runStateMVar :: MVar Annex.AnnexState -> Annex a -> IO a
-runStateMVar mvar a = do
-	startstate <- takeMVar mvar
-	!(r, newstate) <- Annex.run startstate a
-	putMVar mvar newstate
-	return r
-
-runChangeChan :: STM a -> IO a
-runChangeChan = atomically
-
-{- Runs an action handler, inside the Annex monad, and if there was a
- - change, adds it to the ChangeChan.
- -
- - Exceptions are ignored, otherwise a whole watcher thread could be crashed.
- -}
-runHandler :: MVar Annex.AnnexState -> MVar DaemonStatus -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO ()
-runHandler st dstatus changechan handler file filestatus = void $ do
-	r <- tryIO go
-	case r of
-		Left e -> print e
-		Right Nothing -> noop
-		Right (Just change) -> void $
-			runChangeChan $ writeTChan changechan change
-	where
-		go = runStateMVar st $ handler file filestatus dstatus
-
-{- Handlers call this when they made a change that needs to get committed. -}
-madeChange :: FilePath -> String -> Annex (Maybe Change)
-madeChange file desc = do
-	-- Just in case the commit thread is not flushing the queue fast enough.
-	Annex.Queue.flushWhenFull
-	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
-
-noChange :: Annex (Maybe Change)
-noChange = return Nothing
-
-{- Adding a file is tricky; the file has to be replaced with a symlink
- - but this is race prone, as the symlink could be changed immediately
- - after creation. To avoid that race, git add is not used to stage the
- - symlink.
- -
- - Inotify will notice the new symlink, so this Handler does not stage it
- - or return a Change, leaving that to onAddSymlink.
- -
- - During initial directory scan, this will be run for any files that
- - are already checked into git. We don't want to turn those into symlinks,
- - so do a check. This is rather expensive, but only happens during
- - startup.
- -}
-onAdd :: Handler
-onAdd file _filestatus dstatus = do
-	ifM (scanComplete <$> getDaemonStatus dstatus)
-		( go
-		, ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
-			( noChange
-			, go
-			)
-		)
-	where
-		go = do
-			showStart "add" file
-			handle =<< Command.Add.ingest file
-			noChange
-		handle Nothing = showEndFail
-		handle (Just key) = do
-			Command.Add.link file key True
-			showEndOk
-
-{- A symlink might be an arbitrary symlink, which is just added.
- - Or, if it is a git-annex symlink, ensure it points to the content
- - before adding it.
- - 
- -}
-onAddSymlink :: Handler
-onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file
-	where
-		go (Just (key, _)) = do
-			link <- calcGitLink file key
-			ifM ((==) link <$> liftIO (readSymbolicLink file))
-				( ensurestaged link =<< getDaemonStatus dstatus
-				, do
-					liftIO $ removeFile file
-					liftIO $ createSymbolicLink link file
-					addlink link
-				)
-		go Nothing = do -- other symlink
-			link <- liftIO (readSymbolicLink file)
-			ensurestaged link =<< getDaemonStatus dstatus
-
-		{- This is often called on symlinks that are already
-		 - staged correctly. A symlink may have been deleted
-		 - and being re-added, or added when the watcher was
-		 - not running. So they're normally restaged to make sure.
-		 -
-		 - As an optimisation, during the status scan, avoid
-		 - restaging everything. Only links that were created since
-		 - the last time the daemon was running are staged.
-		 - (If the daemon has never ran before, avoid staging
-		 - links too.)
-		 -}
-		ensurestaged link daemonstatus
-			| scanComplete daemonstatus = addlink link
-			| otherwise = case filestatus of
-				Just s
-					| safe (statusChangeTime s) -> noChange
-				_ -> addlink link
-			where
-				safe t = maybe True (> t) (lastRunning daemonstatus)
-
-		{- For speed, tries to reuse the existing blob for
-		 - the symlink target. -}
-		addlink link = do
-			v <- catObjectDetails $ Ref $ ':':file
-			case v of
-				Just (currlink, sha)
-					| s2w8 link == L.unpack currlink ->
-						stageSymlink file sha
-				_ -> do
-					sha <- inRepo $
-						Git.HashObject.hashObject BlobObject link
-					stageSymlink file sha
-			madeChange file "link"
-
-onDel :: Handler
-onDel file _ _dstatus = do
-	Annex.Queue.addUpdateIndex =<<
-		inRepo (Git.UpdateIndex.unstageFile file)
-	madeChange file "rm"
-
-{- A directory has been deleted, or moved, so tell git to remove anything
- - that was inside it from its cache. Since it could reappear at any time,
- - use --cached to only delete it from the index. 
- -
- - Note: This could use unstageFile, but would need to run another git
- - command to get the recursive list of files in the directory, so rm is
- - just as good. -}
-onDelDir :: Handler
-onDelDir dir _ _dstatus = do
-	Annex.Queue.addCommand "rm"
-		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
-	madeChange dir "rmdir"
-
-{- Called when there's an error with inotify. -}
-onErr :: Handler
-onErr msg _ _dstatus = do
-	warning msg
-	return Nothing
-
-{- Adds a symlink to the index, without ever accessing the actual symlink
- - on disk. -}
-stageSymlink :: FilePath -> Sha -> Annex ()
-stageSymlink file sha =
-	Annex.Queue.addUpdateIndex =<<
-		inRepo (Git.UpdateIndex.stageSymlink file sha)
-
-{- Gets all unhandled changes.
- - Blocks until at least one change is made. -}
-getChanges :: ChangeChan -> IO [Change]
-getChanges chan = runChangeChan $ do
-	c <- readTChan chan
-	go [c]
-	where
-		go l = do
-			v <- tryReadTChan chan
-			case v of
-				Nothing -> return l
-				Just c -> go (c:l)
-
-{- Puts unhandled changes back into the channel.
- - Note: Original order is not preserved. -}
-refillChanges :: ChangeChan -> [Change] -> IO ()
-refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs
-
-{- This thread makes git commits at appropriate times. -}
-commitThread :: MVar Annex.AnnexState -> ChangeChan -> IO ()
-commitThread st changechan = forever $ do
-	-- First, a simple rate limiter.
-	threadDelay oneSecond
-	-- Next, wait until at least one change has been made.
-	cs <- getChanges changechan
-	-- Now see if now's a good time to commit.
-	time <- getCurrentTime
-	if shouldCommit time cs
-		then void $ tryIO $ runStateMVar st commitStaged
-		else refillChanges changechan cs
-	where
-		oneSecond = 1000000 -- microseconds
-
-commitStaged :: Annex ()
-commitStaged = do
-	Annex.Queue.flush
-	inRepo $ Git.Command.run "commit"
-		[ Param "--allow-empty-message"
-		, Param "-m", Param ""
-		-- Empty commits may be made if tree changes cancel
-		-- each other out, etc
-		, Param "--allow-empty"
-		-- Avoid running the usual git-annex pre-commit hook;
-		-- watch does the same symlink fixing, and we don't want
-		-- to deal with unlocked files in these commits.
-		, Param "--quiet"
-		]
-
-{- Decide if now is a good time to make a commit.
- - Note that the list of change times has an undefined order.
- -
- - Current strategy: If there have been 10 commits within the past second,
- - a batch activity is taking place, so wait for later.
- -}
-shouldCommit :: UTCTime -> [Change] -> Bool
-shouldCommit now changes
-	| len == 0 = False
-	| len > 10000 = True -- avoid bloating queue too much
-	| length (filter thisSecond changes) < 10 = True
-	| otherwise = False -- batch activity
-	where
-		len = length changes
-		thisSecond c = now `diffUTCTime` changeTime c <= 1

From ff2414427b21324722ed74b754d72307084fc6a5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 13:35:15 -0400
Subject: [PATCH 3746/8313] implement daemon status serialization to a file

Also afterLastDaemonRun, with 10 minute slop to handle majority of clock
skew issues.
---
 Assistant/DaemonStatus.hs | 51 +++++++++++++++++++++++++++++++++++++--
 Assistant/Watcher.hs      |  4 +--
 Locations.hs              |  5 ++++
 3 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 43a8c7e2f0..3615d0e5c3 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -6,16 +6,21 @@
 module Assistant.DaemonStatus where
 
 import Common.Annex
+import Utility.TempFile
 
 import Control.Concurrent
 import System.Posix.Types
+import Data.Time.Clock.POSIX
+import Data.Time
+import System.Locale
 
 data DaemonStatus = DaemonStatus
 	-- False when the daemon is performing its startup scan
 	{ scanComplete :: Bool
 	-- Time when a previous process of the daemon was running ok
-	, lastRunning :: Maybe EpochTime
+	, lastRunning :: Maybe POSIXTime
 	}
+	deriving (Show)
 
 type DaemonStatusHandle = MVar DaemonStatus
 
@@ -32,4 +37,46 @@ getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
 getDaemonStatus = liftIO . readMVar
 
 modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex ()
-modifyDaemonStatus status a = liftIO $ modifyMVar_ status (return . a)
+modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a)
+
+{- Don't just dump out the structure, because it will change over time,
+ - and parts of it are not relevant. -}
+writeDaemonStatusFile :: FilePath -> DaemonStatus -> IO ()
+writeDaemonStatusFile file status = 
+	viaTmp writeFile file =<< serialized <$> getPOSIXTime
+	where
+		serialized now = unlines
+			[ "lastRunning:" ++ show now
+			, "scanComplete:" ++ show (scanComplete status)
+			]
+
+readDaemonStatusFile :: FilePath -> IO DaemonStatus
+readDaemonStatusFile file = parse <$> readFile file
+	where
+		parse = foldr parseline newDaemonStatus . lines
+		parseline line status
+			| key == "lastRunning" = parseval readtime $ \v ->
+				status { lastRunning = Just v }
+			| key == "scanComplete" = parseval readish $ \v ->
+				status { scanComplete = v }
+			| otherwise = status -- unparsable line
+			where
+				(key, value) = separate (== ':') line
+				parseval parser a = maybe status a (parser value)
+				readtime s = do
+					d <- parseTime defaultTimeLocale "%s%Qs" s
+					Just $ utcTimeToPOSIXSeconds d
+
+{- Checks if a time stamp was made after the daemon was lastRunning.
+ -
+ - Some slop is built in; this really checks if the time stamp was made
+ - at least ten minutes after the daemon was lastRunning. This is to
+ - ensure the daemon shut down cleanly, and deal with minor clock skew.
+ -
+ - If the daemon has never ran before, this always returns False.
+ -}
+afterLastDaemonRun :: EpochTime -> DaemonStatus -> Bool
+afterLastDaemonRun timestamp status = maybe True (< t) (lastRunning status)
+	where
+		t = realToFrac (timestamp + slop) :: POSIXTime
+		slop = 10 * 60
diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 19a65db6e8..ee5bc13af0 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -154,10 +154,8 @@ onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file
 			| scanComplete daemonstatus = addlink link
 			| otherwise = case filestatus of
 				Just s
-					| safe (statusChangeTime s) -> noChange
+					| not (afterLastDaemonRun (statusChangeTime s) daemonstatus) -> noChange
 				_ -> addlink link
-			where
-				safe t = maybe True (> t) (lastRunning daemonstatus)
 
 		{- For speed, tries to reuse the existing blob for
 		 - the symlink target. -}
diff --git a/Locations.hs b/Locations.hs
index 0c9935614d..cd3f55d466 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -24,6 +24,7 @@ module Locations (
 	gitAnnexIndexLock,
 	gitAnnexIndexDirty,
 	gitAnnexPidFile,
+	gitAnnexDaemonStatusFile,
 	gitAnnexLogFile,
 	gitAnnexSshDir,
 	gitAnnexRemotesDir,
@@ -151,6 +152,10 @@ gitAnnexIndexDirty r = gitAnnexDir r  "index.dirty"
 gitAnnexPidFile :: Git.Repo -> FilePath
 gitAnnexPidFile r = gitAnnexDir r  "daemon.pid"
 
+{- Status file for daemon mode. -}
+gitAnnexDaemonStatusFile :: Git.Repo -> FilePath
+gitAnnexDaemonStatusFile r = gitAnnexDir r  "daemon.status"
+
 {- Log file for daemon mode. -}
 gitAnnexLogFile :: Git.Repo -> FilePath
 gitAnnexLogFile r = gitAnnexDir r  "daemon.log"

From 59a7b3a51a7cdfb8528ebc44a26a7577f28254d4 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 14:02:40 -0400
Subject: [PATCH 3747/8313] finish daemon status thread

---
 Assistant.hs              | 19 +++++++++++--------
 Assistant/DaemonStatus.hs | 29 +++++++++++++++++++++++++----
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index bc394bd998..eb8fd7054c 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -60,15 +60,18 @@ startDaemon foreground
 		pidfile <- fromRepo gitAnnexPidFile
 		go $ Utility.Daemon.daemonize logfd (Just pidfile) False
 	where
-		go a = withThreadState $ \st -> liftIO $ a $ do
+		go a = withThreadState $ \st -> do
 			dstatus <- startDaemonStatus
-			changechan <- newChangeChan
-			-- The commit thread is started early, so that the user
-			-- can immediately begin adding files and having them
-			-- committed, even while the startup scan is taking
-			-- place.
-			_ <- forkIO $ commitThread st changechan
-			watchThread st dstatus changechan
+			liftIO $ a $ do
+				changechan <- newChangeChan
+				-- The commit thread is started early,
+				-- so that the user can immediately
+				-- begin adding files and having them
+				-- committed, even while the startup scan
+				-- is taking place.
+				_ <- forkIO $ commitThread st changechan
+				_ <- forkIO $ daemonStatusThread st dstatus
+				watchThread st dstatus changechan
 
 stopDaemon :: Annex ()
 stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 3615d0e5c3..eb8ff256b5 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -7,6 +7,7 @@ module Assistant.DaemonStatus where
 
 import Common.Annex
 import Utility.TempFile
+import Assistant.ThreadedMonad
 
 import Control.Concurrent
 import System.Posix.Types
@@ -30,15 +31,35 @@ newDaemonStatus = DaemonStatus
 	, lastRunning = Nothing
 	}
 
-startDaemonStatus :: IO DaemonStatusHandle
-startDaemonStatus = newMVar newDaemonStatus
-
 getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
 getDaemonStatus = liftIO . readMVar
 
 modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex ()
 modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a)
 
+{- Load any previous daemon status file, and store it in the MVar for this
+ - process to use as its DaemonStatus. -}
+startDaemonStatus :: Annex DaemonStatusHandle
+startDaemonStatus = do
+	file <- fromRepo gitAnnexDaemonStatusFile
+	status <- liftIO $
+		catchDefaultIO (readDaemonStatusFile file) newDaemonStatus
+	liftIO $ newMVar status { scanComplete = False }
+
+{- This thread wakes up periodically and writes the daemon status to disk. -}
+daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
+daemonStatusThread st handle = do
+	checkpoint
+	forever $ do
+		threadDelay tenMinutes
+		checkpoint
+	where
+		checkpoint = runThreadState st $ do
+			file <- fromRepo gitAnnexDaemonStatusFile
+			status <- getDaemonStatus handle
+			liftIO $ writeDaemonStatusFile file status
+		tenMinutes = 10 * 60 * 1000000 -- microseconds
+
 {- Don't just dump out the structure, because it will change over time,
  - and parts of it are not relevant. -}
 writeDaemonStatusFile :: FilePath -> DaemonStatus -> IO ()
@@ -76,7 +97,7 @@ readDaemonStatusFile file = parse <$> readFile file
  - If the daemon has never ran before, this always returns False.
  -}
 afterLastDaemonRun :: EpochTime -> DaemonStatus -> Bool
-afterLastDaemonRun timestamp status = maybe True (< t) (lastRunning status)
+afterLastDaemonRun timestamp status = maybe False (< t) (lastRunning status)
 	where
 		t = realToFrac (timestamp + slop) :: POSIXTime
 		slop = 10 * 60

From 7575c5bb561bd04d6106ac09d97ea813c9ee46f3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 14:19:21 -0400
Subject: [PATCH 3748/8313] tweak

---
 Assistant/DaemonStatus.hs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index eb8ff256b5..dfd3c44f36 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -51,14 +51,14 @@ daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
 daemonStatusThread st handle = do
 	checkpoint
 	forever $ do
-		threadDelay tenMinutes
+		threadDelay (tenMinutes * oneSecond)
 		checkpoint
 	where
 		checkpoint = runThreadState st $ do
 			file <- fromRepo gitAnnexDaemonStatusFile
 			status <- getDaemonStatus handle
 			liftIO $ writeDaemonStatusFile file status
-		tenMinutes = 10 * 60 * 1000000 -- microseconds
+		oneSecond = 1000000 -- microseconds
 
 {- Don't just dump out the structure, because it will change over time,
  - and parts of it are not relevant. -}
@@ -100,4 +100,7 @@ afterLastDaemonRun :: EpochTime -> DaemonStatus -> Bool
 afterLastDaemonRun timestamp status = maybe False (< t) (lastRunning status)
 	where
 		t = realToFrac (timestamp + slop) :: POSIXTime
-		slop = 10 * 60
+		slop = fromIntegral tenMinutes
+
+tenMinutes :: Int
+tenMinutes = 10 * 60

From f9cc2acc71de32d0f4b8d85b5eb39f1ace5f9333 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 14:32:11 -0400
Subject: [PATCH 3749/8313] bug

---
 doc/design/assistant/inotify.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 60c598673a..c9b1d60bfd 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -32,6 +32,8 @@ There is a `watch` branch in git that adds the command.
   (or merged, etc), it will be converted into an annexed file.
   See [[blog/day_7__bugfixes]]
 
+* When you `git annex unlock` a file, it will immediately be re-locked.
+
 ## todo
 
 - Support OSes other than Linux; it only uses inotify currently.

From 24370fa3ac4fcb5c1d2a8e727fb4730f0d2d9789 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 15:03:26 -0400
Subject: [PATCH 3750/8313] lsof

---
 doc/design/assistant/inotify.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 60c598673a..8f0aebcb1f 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -18,6 +18,8 @@ There is a `watch` branch in git that adds the command.
   Possible fixes: 
 
   * Somehow track or detect if a file is open for write by any processes.
+    `lsof` could be used, although it would be a little slow, and not
+    avoid every possible race.
   * Or, when possible, making a copy on write copy before adding the file
     would avoid this.
   * Or, as a last resort, make an expensive copy of the file and add that.

From 36d73b00171aa26bf5379be7dbd66611834a0459 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 17:53:19 -0400
Subject: [PATCH 3751/8313] slightly higher-level thread scheduling code

Including support for unbound thread sleeping. Haskell's max thread sleep
is 37 minutes, due to maxBound Int!
---
 Utility/ThreadScheduler.hs | 42 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 Utility/ThreadScheduler.hs

diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs
new file mode 100644
index 0000000000..9204cd9b9e
--- /dev/null
+++ b/Utility/ThreadScheduler.hs
@@ -0,0 +1,42 @@
+{- thread scheduling
+ -
+ - Copyright 2012 Joey Hess 
+ - Copyright 2011 Bas van Dijk & Roel van Dijk
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.ThreadScheduler where
+
+import Common
+import Control.Concurrent
+
+newtype Seconds = Seconds { fromSeconds :: Int }
+	deriving (Eq, Ord, Show)
+
+{- Runs an action repeatedly forever, sleeping at least the specified number
+ - of seconds in between. -}
+runEvery :: Seconds -> IO a -> IO a
+runEvery n a = forever $ do
+	threadDelaySeconds n
+	a
+
+threadDelaySeconds :: Seconds -> IO ()
+threadDelaySeconds (Seconds n) = unboundDelay (fromIntegral n * oneSecond)
+	where
+		oneSecond = 1000000 -- microseconds
+
+{- Like threadDelay, but not bounded by an Int.
+ -
+ - There is no guarantee that the thread will be rescheduled promptly when the
+ - delay has expired, but the thread will never continue to run earlier than
+ - specified.
+ - 
+ - Taken from the unbounded-delay package to avoid a dependency for 4 lines
+ - of code.
+ -}
+unboundDelay :: Integer -> IO ()
+unboundDelay time = do
+	let maxWait = min time $ toInteger (maxBound :: Int)
+	threadDelay $ fromInteger maxWait
+	when (maxWait /= time) $ unboundDelay (time - maxWait)

From 4b9b9b494757e04ec5c449666d5a0a063378cdb3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 17:54:23 -0400
Subject: [PATCH 3752/8313] add sanity checker thread

Currently wakes up once a day, and does nothing. :)
---
 Assistant.hs               |  2 ++
 Assistant/Committer.hs     |  9 ++----
 Assistant/DaemonStatus.hs  | 25 +++++++++++++----
 Assistant/SanityChecker.hs | 56 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 80 insertions(+), 12 deletions(-)
 create mode 100644 Assistant/SanityChecker.hs

diff --git a/Assistant.hs b/Assistant.hs
index eb8fd7054c..3d819a018f 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -45,6 +45,7 @@ import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.Watcher
 import Assistant.Committer
+import Assistant.SanityChecker
 import qualified Utility.Daemon
 import Utility.LogFile
 
@@ -71,6 +72,7 @@ startDaemon foreground
 				-- is taking place.
 				_ <- forkIO $ commitThread st changechan
 				_ <- forkIO $ daemonStatusThread st dstatus
+				_ <- forkIO $ sanityCheckerThread st dstatus
 				watchThread st dstatus changechan
 
 stopDaemon :: Annex ()
diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index d6fc085793..a572556de5 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -9,8 +9,8 @@ import Common.Annex
 import Assistant.ThreadedMonad
 import qualified Annex.Queue
 import qualified Git.Command
+import Utility.ThreadScheduler
 
-import Control.Concurrent
 import Control.Concurrent.STM
 import Data.Time.Clock
 
@@ -59,9 +59,8 @@ refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs
 
 {- This thread makes git commits at appropriate times. -}
 commitThread :: ThreadState -> ChangeChan -> IO ()
-commitThread st changechan = forever $ do
-	-- First, a simple rate limiter.
-	threadDelay oneSecond
+commitThread st changechan = runEvery (Seconds 1) $ do
+	-- We already waited one second as a simple rate limiter.
 	-- Next, wait until at least one change has been made.
 	cs <- getChanges changechan
 	-- Now see if now's a good time to commit.
@@ -69,8 +68,6 @@ commitThread st changechan = forever $ do
 	if shouldCommit time cs
 		then void $ tryIO $ runThreadState st commitStaged
 		else refillChanges changechan cs
-	where
-		oneSecond = 1000000 -- microseconds
 
 commitStaged :: Annex ()
 commitStaged = do
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index dfd3c44f36..1bc6031eec 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -6,8 +6,9 @@
 module Assistant.DaemonStatus where
 
 import Common.Annex
-import Utility.TempFile
 import Assistant.ThreadedMonad
+import Utility.ThreadScheduler
+import Utility.TempFile
 
 import Control.Concurrent
 import System.Posix.Types
@@ -20,6 +21,10 @@ data DaemonStatus = DaemonStatus
 	{ scanComplete :: Bool
 	-- Time when a previous process of the daemon was running ok
 	, lastRunning :: Maybe POSIXTime
+	-- True when the sanity checker is running
+	, sanityCheckRunning :: Bool
+	-- Last time the sanity checker ran
+	, lastSanityCheck :: Maybe POSIXTime
 	}
 	deriving (Show)
 
@@ -29,6 +34,8 @@ newDaemonStatus :: DaemonStatus
 newDaemonStatus = DaemonStatus
 	{ scanComplete = False
 	, lastRunning = Nothing
+	, sanityCheckRunning = False
+	, lastSanityCheck = Nothing
 	}
 
 getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
@@ -44,21 +51,21 @@ startDaemonStatus = do
 	file <- fromRepo gitAnnexDaemonStatusFile
 	status <- liftIO $
 		catchDefaultIO (readDaemonStatusFile file) newDaemonStatus
-	liftIO $ newMVar status { scanComplete = False }
+	liftIO $ newMVar status
+		{ scanComplete = False
+		, sanityCheckRunning = False
+		}
 
 {- This thread wakes up periodically and writes the daemon status to disk. -}
 daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
 daemonStatusThread st handle = do
 	checkpoint
-	forever $ do
-		threadDelay (tenMinutes * oneSecond)
-		checkpoint
+	runEvery (Seconds tenMinutes) checkpoint
 	where
 		checkpoint = runThreadState st $ do
 			file <- fromRepo gitAnnexDaemonStatusFile
 			status <- getDaemonStatus handle
 			liftIO $ writeDaemonStatusFile file status
-		oneSecond = 1000000 -- microseconds
 
 {- Don't just dump out the structure, because it will change over time,
  - and parts of it are not relevant. -}
@@ -69,6 +76,8 @@ writeDaemonStatusFile file status =
 		serialized now = unlines
 			[ "lastRunning:" ++ show now
 			, "scanComplete:" ++ show (scanComplete status)
+			, "sanityCheckRunning:" ++ show (sanityCheckRunning status)
+			, "lastSanityCheck:" ++ show (lastSanityCheck status)
 			]
 
 readDaemonStatusFile :: FilePath -> IO DaemonStatus
@@ -80,6 +89,10 @@ readDaemonStatusFile file = parse <$> readFile file
 				status { lastRunning = Just v }
 			| key == "scanComplete" = parseval readish $ \v ->
 				status { scanComplete = v }
+			| key == "sanityCheckRunning" = parseval readish $ \v ->
+				status { sanityCheckRunning = v }
+			| key == "lastSanityCheck" = parseval readtime $ \v ->
+				status { lastSanityCheck = Just v }
 			| otherwise = status -- unparsable line
 			where
 				(key, value) = separate (== ':') line
diff --git a/Assistant/SanityChecker.hs b/Assistant/SanityChecker.hs
new file mode 100644
index 0000000000..9567b11885
--- /dev/null
+++ b/Assistant/SanityChecker.hs
@@ -0,0 +1,56 @@
+{- git-annex assistant sanity checker
+ -
+ - Copyright 2012 Joey Hess 
+ -}
+
+module Assistant.SanityChecker (
+	sanityCheckerThread
+) where
+
+import Common.Annex
+import Assistant.DaemonStatus
+import Assistant.ThreadedMonad
+import Utility.ThreadScheduler
+
+import Data.Time.Clock.POSIX
+
+{- This thread wakes up occasionally to make sure the tree is in good shape. -}
+sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> IO ()
+sanityCheckerThread st status = forever $ do
+	waitForNextCheck st status
+
+	runThreadState st $
+		modifyDaemonStatus status $ \s -> s
+				{ sanityCheckRunning = True }
+
+	now <- getPOSIXTime -- before check started
+	ok <- catchBoolIO $ runThreadState st check
+
+	runThreadState st $ do
+		modifyDaemonStatus status $ \s -> s
+			{ sanityCheckRunning = False
+			, lastSanityCheck =
+				if ok
+					then Just now
+					else lastSanityCheck s
+			}
+
+{- Only run one check per day, from the time of the last check. -}
+waitForNextCheck :: ThreadState -> DaemonStatusHandle -> IO ()
+waitForNextCheck st status = do
+	v <- runThreadState st $
+		lastSanityCheck <$> getDaemonStatus status
+	now <- getPOSIXTime
+	threadDelaySeconds $ Seconds $ calcdelay now v
+	where
+		calcdelay _ Nothing = oneDay
+		calcdelay now (Just lastcheck)
+			| lastcheck < now = oneDay - truncate (now - lastcheck)
+			| otherwise = oneDay
+
+check  :: Annex Bool
+check = do
+	return True
+
+oneDay :: Int
+oneDay = 24 * 60 * 60

From 8919c2e4da5e17b8127d738ded733a1a01996194 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 19:25:47 -0400
Subject: [PATCH 3753/8313] check for unstaged old symlinks in the sanity
 checker

---
 Assistant.hs               |  2 +-
 Assistant/DaemonStatus.hs  |  2 +-
 Assistant/SanityChecker.hs | 49 ++++++++++++++++++++++++++++----------
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 3d819a018f..3a3bcf7e08 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -72,7 +72,7 @@ startDaemon foreground
 				-- is taking place.
 				_ <- forkIO $ commitThread st changechan
 				_ <- forkIO $ daemonStatusThread st dstatus
-				_ <- forkIO $ sanityCheckerThread st dstatus
+				_ <- forkIO $ sanityCheckerThread st dstatus changechan
 				watchThread st dstatus changechan
 
 stopDaemon :: Annex ()
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 1bc6031eec..e5ba3d1512 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -77,7 +77,7 @@ writeDaemonStatusFile file status =
 			[ "lastRunning:" ++ show now
 			, "scanComplete:" ++ show (scanComplete status)
 			, "sanityCheckRunning:" ++ show (sanityCheckRunning status)
-			, "lastSanityCheck:" ++ show (lastSanityCheck status)
+			, "lastSanityCheck:" ++ maybe "" show (lastSanityCheck status)
 			]
 
 readDaemonStatusFile :: FilePath -> IO DaemonStatus
diff --git a/Assistant/SanityChecker.hs b/Assistant/SanityChecker.hs
index 9567b11885..a5f1380248 100644
--- a/Assistant/SanityChecker.hs
+++ b/Assistant/SanityChecker.hs
@@ -8,15 +8,18 @@ module Assistant.SanityChecker (
 ) where
 
 import Common.Annex
+import qualified Git.LsFiles
 import Assistant.DaemonStatus
 import Assistant.ThreadedMonad
+import Assistant.Committer
 import Utility.ThreadScheduler
+import qualified Assistant.Watcher
 
 import Data.Time.Clock.POSIX
 
 {- This thread wakes up occasionally to make sure the tree is in good shape. -}
-sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> IO ()
-sanityCheckerThread st status = forever $ do
+sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
+sanityCheckerThread st status changechan = forever $ do
 	waitForNextCheck st status
 
 	runThreadState st $
@@ -24,15 +27,13 @@ sanityCheckerThread st status = forever $ do
 				{ sanityCheckRunning = True }
 
 	now <- getPOSIXTime -- before check started
-	ok <- catchBoolIO $ runThreadState st check
+	catchIO (check st status changechan)
+		(runThreadState st . warning . show)
 
 	runThreadState st $ do
 		modifyDaemonStatus status $ \s -> s
 			{ sanityCheckRunning = False
-			, lastSanityCheck =
-				if ok
-					then Just now
-					else lastSanityCheck s
+			, lastSanityCheck = Just now
 			}
 
 {- Only run one check per day, from the time of the last check. -}
@@ -45,12 +46,36 @@ waitForNextCheck st status = do
 	where
 		calcdelay _ Nothing = oneDay
 		calcdelay now (Just lastcheck)
-			| lastcheck < now = oneDay - truncate (now - lastcheck)
+			| lastcheck < now = max oneDay $
+				oneDay - truncate (now - lastcheck)
 			| otherwise = oneDay
 
-check  :: Annex Bool
-check = do
-	return True
-
 oneDay :: Int
 oneDay = 24 * 60 * 60
+
+{- It's important to stay out of the Annex monad as much as possible while
+ - running potentially expensive parts of this check, since remaining in it
+ - will block the watcher. -}
+check :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () 
+check st status changechan = do
+	g <- runThreadState st $ do
+		showSideAction "Running daily check"
+		fromRepo id
+	-- Find old unstaged symlinks, and add them to git.
+	unstaged <- Git.LsFiles.notInRepo False ["."] g
+	now <- getPOSIXTime
+	forM_ unstaged $ \file -> do
+		ms <- catchMaybeIO $ getSymbolicLinkStatus file
+		case ms of
+			Just s	| toonew (statusChangeTime s) now -> noop
+				| isSymbolicLink s ->
+					addsymlink file ms
+			_ -> noop
+	where
+		toonew timestamp now = now < (realToFrac (timestamp + slop) :: POSIXTime)
+		slop = fromIntegral tenMinutes
+		insanity m = runThreadState st $ warning m
+		addsymlink file s = do
+			insanity $ "found unstaged symlink: " ++ file
+			Assistant.Watcher.runHandler st status changechan
+				Assistant.Watcher.onAddSymlink file s

From 6be8cc18024ffa49545a03cbdd977f00a6fc0c2f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 19:26:22 -0400
Subject: [PATCH 3754/8313] blog for the day

---
 doc/design/assistant/blog/day_8__speed.mdwn | 67 +++++++++++++++++++++
 1 file changed, 67 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_8__speed.mdwn

diff --git a/doc/design/assistant/blog/day_8__speed.mdwn b/doc/design/assistant/blog/day_8__speed.mdwn
new file mode 100644
index 0000000000..52c4de7a22
--- /dev/null
+++ b/doc/design/assistant/blog/day_8__speed.mdwn
@@ -0,0 +1,67 @@
+Since last post, I've worked on speeding up `git annex watch`'s startup time
+in a large repository.
+
+The problem was that its initial scan was naively staging every symlink in
+the repository, even though most of them are, presumably, staged correctly
+already. This was done in case the user copied or moved some symlinks
+around while `git annex watch` was not running -- we want to notice and
+commit such changes at startup.
+
+Since I already had the `stat` info for the symlink, it can look at the
+`ctime` to see if the symlink was made recently, and only stage it if so.
+This sped up startup in my big repo from longer than I cared to wait (10+
+minutes, or half an hour while profiling) to a minute or so. Of course,
+inotify events are already serviced during startup, so making it scan
+quickly is really only important so people don't think it's a resource hog.
+First impressions are important. :)
+
+But what does "made recently" mean exactly? Well, my answer is possibly
+overengineered, but most of it is really groundwork for things I'll need
+later anyway. I added a new data structure for tracking the status of the
+daemon, which is periodically written to disk by another thread (thread #6!)
+to `.git/annex/daemon.status` Currently it looks like this; I anticipate
+adding lots more info as I move into the [[syncing]] stage:
+
+	lastRunning:1339610482.47928s
+	scanComplete:True
+
+So, only symlinks made after the daemon was last running need to be
+expensively staged on startup. Although, as RichiH pointed out,
+this fails if the clock is changed. But I have been planning to have a
+cleanup thread anyway, that will handle this, and other
+potential problems, so I think that's ok.
+
+Stracing its startup scan, it's fairly tight now. There are some repeated 
+`getcwd` syscalls that could be optimised out for a minor speedup.
+
+----
+
+Added the sanity check thread. Thread #8! It currently only does one sanity
+check per day, but the sanity check is a fairly lightweight job,
+so I may make it run more frequently. OTOH, it may never ever find a
+problem, so once per day seems a good compromise. 
+
+Currently it's only checking that all files in the tree are properly staged
+in git. I might make it `git annex fsck` later, but fscking the whole tree
+once per day is a bit much. Perhaps it should only fsck a few files per
+day? TBD
+
+Currently any problems found in the sanity check are just fixed and logged.
+It would be good to do something about getting problems that might indicate
+bugs fed back to me, in a privacy-respecting way. TBD
+
+----
+
+I also refactored the code, which was getting far too large to all be in
+one module. 
+
+I have been thinking about renaming `git annex watch` to `git annex assistant`,
+but I think I'll leave the command name as-is. Some users might
+want a simple watcher and stager, without the assistant's other features
+like syncing and the webapp. So the next stage of the
+[[roadmap|design/assistant]] will be a different command that also runs
+`watch`.
+
+At this point, I feel I'm done with the first phase of [[inotify]].
+It has a couple known bugs, but it's ready for brave beta testers to try.
+I trust it enough to be running it on my live data.

From a40dc2d390e5b2ba09614477737845aad6b6bb1c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 19:30:07 -0400
Subject: [PATCH 3755/8313] typo

---
 doc/design/assistant/blog/day_8__speed.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_8__speed.mdwn b/doc/design/assistant/blog/day_8__speed.mdwn
index 52c4de7a22..56b1e9c079 100644
--- a/doc/design/assistant/blog/day_8__speed.mdwn
+++ b/doc/design/assistant/blog/day_8__speed.mdwn
@@ -36,7 +36,7 @@ Stracing its startup scan, it's fairly tight now. There are some repeated
 
 ----
 
-Added the sanity check thread. Thread #8! It currently only does one sanity
+Added the sanity check thread. Thread #7! It currently only does one sanity
 check per day, but the sanity check is a fairly lightweight job,
 so I may make it run more frequently. OTOH, it may never ever find a
 problem, so once per day seems a good compromise. 

From 89dad12b3510f2540c6678e68145d922c7db5a7b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 13 Jun 2012 19:32:09 -0400
Subject: [PATCH 3756/8313] update

---
 Assistant.hs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 3a3bcf7e08..b72f9a7e7d 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -10,12 +10,13 @@
  - 	The initial thread run, double forks to background, starts other
  - 	threads, and then stops, waiting for them to terminate,
  - 	or for a ctrl-c.
- - Thread 2: inotify
+ - Thread 2: watcher
  - 	Notices new files, and calls handlers for events, queuing changes.
  - Thread 3: inotify internal
  - 	Used by haskell inotify library to ensure inotify event buffer is
  - 	kept drained.
- - Thread 4: inotify initial scan
+ - Thread 4: inotify startup scanner
+ - 	Scans the tree and registers inotify watches for each directory.
  -	A MVar lock is used to prevent other inotify handlers from running
  -	until this is complete.
  - Thread 5: committer
@@ -23,6 +24,8 @@
  - 	index, then commits.
  - Thread 6: status logger
  - 	Wakes up periodically and records the daemon's status to disk.
+ - Thread 7: sanity checker
+ - 	Wakes up periodically (rarely) and does sanity checks.
  -
  - ThreadState: (MVar)
  - 	The Annex state is stored here, which allows resuscitating the

From e0095b0bdc88147a081f3539975f20096fd0ea3c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 00:01:48 -0400
Subject: [PATCH 3757/8313] fishy commit

---
 Annex/Content.hs  | 2 +-
 Annex/Queue.hs    | 4 ++--
 Command/Commit.hs | 2 +-
 Command/Unused.hs | 4 ++--
 Remote/Rsync.hs   | 2 +-
 Seek.hs           | 2 +-
 Upgrade/V2.hs     | 2 +-
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/Annex/Content.hs b/Annex/Content.hs
index 3e3e958686..0bce5633fe 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -242,7 +242,7 @@ cleanObjectLoc key = do
 		removeparents file n = do
 			let dir = parentDir file
 			maybe noop (const $ removeparents dir (n-1))
-				=<< catchMaybeIO (removeDirectory dir)
+				<=< catchMaybeIO $ removeDirectory dir
 
 {- Removes a key's file from .git/annex/objects/ -}
 removeAnnex :: Key -> Annex ()
diff --git a/Annex/Queue.hs b/Annex/Queue.hs
index c019aed6cc..97a759d103 100644
--- a/Annex/Queue.hs
+++ b/Annex/Queue.hs
@@ -23,13 +23,13 @@ import Config
 addCommand :: String -> [CommandParam] -> [FilePath] -> Annex ()
 addCommand command params files = do
 	q <- get
-	store =<< inRepo (Git.Queue.addCommand command params files q)
+	store <=< inRepo $ Git.Queue.addCommand command params files q
 
 {- Adds an update-index stream to the queue. -}
 addUpdateIndex :: Git.UpdateIndex.Streamer -> Annex ()
 addUpdateIndex streamer = do
 	q <- get
-	store =<< inRepo (Git.Queue.addUpdateIndex streamer q)
+	store <=< inRepo $ Git.Queue.addUpdateIndex streamer q
 
 {- Runs the queue if it is full. Should be called periodically. -}
 flushWhenFull :: Annex ()
diff --git a/Command/Commit.hs b/Command/Commit.hs
index f53ab7e097..d3ce3d7bb1 100644
--- a/Command/Commit.hs
+++ b/Command/Commit.hs
@@ -22,7 +22,7 @@ seek = [withNothing start]
 start :: CommandStart
 start = next $ next $ do
 	Annex.Branch.commit "update"
-	_ <- runhook =<< inRepo (Git.hookPath "annex-content")
+	_ <- runhook <=< inRepo $ Git.hookPath "annex-content"
 	return True
 	where
 		runhook (Just hook) = liftIO $ boolSystem hook []
diff --git a/Command/Unused.hs b/Command/Unused.hs
index 1224d05457..03a709534e 100644
--- a/Command/Unused.hs
+++ b/Command/Unused.hs
@@ -84,7 +84,7 @@ checkRemoteUnused name = go =<< fromJust <$> Remote.byName (Just name)
 			_ <- check "" (remoteUnusedMsg r) (remoteunused r) 0
 			next $ return True
 		remoteunused r =
-			excludeReferenced =<< loggedKeysFor (Remote.uuid r)
+			excludeReferenced <=< loggedKeysFor $ Remote.uuid r
 
 check :: FilePath -> ([(Int, Key)] -> String) -> Annex [Key] -> Int -> Annex Int
 check file msg a c = do
@@ -260,7 +260,7 @@ withKeysReferencedInGit a = do
 withKeysReferencedInGitRef :: (Key -> Annex ()) -> Git.Ref -> Annex ()
 withKeysReferencedInGitRef a ref = do
 	showAction $ "checking " ++ Git.Ref.describe ref
-	go =<< inRepo (LsTree.lsTree ref)
+	go <=< inRepo $ LsTree.lsTree ref
 	where
 		go [] = noop
 		go (l:ls)
diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs
index df4e0a44f2..3c449b5de0 100644
--- a/Remote/Rsync.hs
+++ b/Remote/Rsync.hs
@@ -100,7 +100,7 @@ rsyncUrls o k = map use annexHashes
                 f = keyFile k
 
 store :: RsyncOpts -> Key -> Annex Bool
-store o k = rsyncSend o k =<< inRepo (gitAnnexLocation k)
+store o k = rsyncSend o k <=< inRepo $ gitAnnexLocation k
 
 storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool
 storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do
diff --git a/Seek.hs b/Seek.hs
index eed4a81558..2cf0d8d460 100644
--- a/Seek.hs
+++ b/Seek.hs
@@ -95,7 +95,7 @@ withValue v a params = do
  -}
 withField :: Option -> (Maybe String -> Annex a) -> (a -> CommandSeek) -> CommandSeek
 withField option converter = withValue $
-	converter =<< Annex.getField (Option.name option)
+	converter <=< Annex.getField $ Option.name option
 
 withFlag :: Option -> (Bool -> CommandSeek) -> CommandSeek
 withFlag option = withValue $ Annex.getFlag (Option.name option)
diff --git a/Upgrade/V2.hs b/Upgrade/V2.hs
index 202ba5b167..c001bc5a95 100644
--- a/Upgrade/V2.hs
+++ b/Upgrade/V2.hs
@@ -84,7 +84,7 @@ inject source dest = do
 
 logFiles :: FilePath -> Annex [FilePath]
 logFiles dir = return . filter (".log" `isSuffixOf`)
-		=<< liftIO (getDirectoryContents dir)
+		<=< liftIO $ getDirectoryContents dir
 
 push :: Annex ()
 push = do

From 686760befc3f145b8a490c337af9bcba289c42b2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 12:27:30 -0400
Subject: [PATCH 3758/8313] typo

---
 doc/design/assistant/android.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index 90dc551794..897ebd7606 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -57,8 +57,6 @@ version git-annex dependend upon existing on the phone. (Maybe the phone
 would have to be always considered an untrusted repo, which probably
 makes sense anyway.)
 
-Problem: 
-
 #### crazy `LD_PRELOAD` wrapper
 
 Need I say more? (Also, Android's linker may not even support it.)

From 6b56abf215701b294895c175499fe0493360427c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 12:43:23 -0400
Subject: [PATCH 3759/8313] hard problems are fun?

---
 doc/design/assistant/android.mdwn | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index 897ebd7606..f2711a4794 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -14,6 +14,18 @@ available in the App Store.
 * git (not all git commands are needed,
   but core plumbing and a few like `git-add` are.)
 
+## GHC Android?
+
+Android's native SDK does not use glibc. GHC's runtime links with glibc.
+This could be an enormous problem. Other people want to see GHC able to
+target Android, of course, so someone may solve it before I get stuck on
+it.
+
+References:
+
+* 
+* 
+
 ### Android specific features
 
 The app should be aware of power status, and avoid expensive background

From 06caf52f03de118e08dbe368d431835b5a8e6bd6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 20:06:43 -0400
Subject: [PATCH 3760/8313] blog for the day

---
 .../assistant/blog/day_9__correctness.mdwn    | 30 +++++++++++++++++++
 doc/design/assistant/inotify.mdwn             |  1 +
 2 files changed, 31 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness.mdwn

diff --git a/doc/design/assistant/blog/day_9__correctness.mdwn b/doc/design/assistant/blog/day_9__correctness.mdwn
new file mode 100644
index 0000000000..1fa4c09d0d
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness.mdwn
@@ -0,0 +1,30 @@
+	git merge watch_
+
+My cursor has been mentally poised here all day, but I've been reluctant to
+merge watch into master. It seems solid, but is it correct? I was able to
+think up a lot of races it'd be subject to, and deal with them, but did I
+find them all?
+
+Perhaps I need to do some automated fuzz testing to reassure myself.
+I looked into using [genbackupdata](http://liw.fi/genbackupdata/) to that
+end. It's not quite what I need, but could be
+[moved in that direction](http://bugs.debian.org/677542). Or I could write
+my own fuzz tester, but it seems better to use someone else's, because 
+a) laziness and b) they're less likely to have the same blind spots I do.
+
+My reluctance to merge isn't helped by the known bugs with files that are
+either already open before `git annex watch` starts, or are opened by two
+processes at once, and confuse it into annexing the still-open file when one
+process closes it.
+
+I've been thinking about just running `lsof` on every file as it's being
+annexed to check for that, but in the end, `lsof` is too slow. Since its
+check involves trawling through all of /proc, it takes it a good half a
+second to check a file, and adding 25 seconds to the time it takes to
+process 100 files is just not acceptable.
+
+But an option that could work is to run `lsof` after a bunch of new files
+have been annexed. It can check a lot of files nearly as fast as a single
+one. In the rare case that an annexed file is indeed still open, it could
+be moved back out of the annex. Then when its remaining writer finally
+closes it, another inotify event would re-annex it.
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index c9b1d60bfd..9d8857ab65 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -18,6 +18,7 @@ There is a `watch` branch in git that adds the command.
   Possible fixes: 
 
   * Somehow track or detect if a file is open for write by any processes.
+    `lsof` could be used, although it would be a little slow.
   * Or, when possible, making a copy on write copy before adding the file
     would avoid this.
   * Or, as a last resort, make an expensive copy of the file and add that.

From ca9d94a0add47446eca1ab2474faccc48a0cadf2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 20:20:03 -0400
Subject: [PATCH 3761/8313] addurl: Was broken by a typo introduced 2 released
 ago, now fixed. Closes: #677576

---
 Command/AddUrl.hs | 2 +-
 debian/changelog  | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 87b24149d8..369940bdfe 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -73,7 +73,7 @@ download url file = do
 	liftIO $ createDirectoryIfMissing True (parentDir tmp)
 	stopUnless (downloadUrl [url] tmp) $ do
 		backend <- chooseBackend file
-		let source = KeySource { keyFilename = file, contentLocation = file}
+		let source = KeySource { keyFilename = file, contentLocation = tmp }
 		k <- genKey source backend
 		case k of
 			Nothing -> stop
diff --git a/debian/changelog b/debian/changelog
index 67ecdda45d..624f1e132b 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,7 @@
 git-annex (3.20120612) UNRELEASED; urgency=low
 
+  * addurl: Was broken by a typo introduced 2 released ago, now fixed.
+    Closes: #677576
   * Install man page when run by cabal, in a location where man will
     find it, even when installing under $HOME. Thanks, Nathan Collins
 

From 8492f1c1826d1f194c594bfc3b1597b8d03e23fc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 20:32:06 -0400
Subject: [PATCH 3762/8313] releasing version 3.20120614

---
 debian/changelog | 4 ++--
 git-annex.cabal  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/debian/changelog b/debian/changelog
index 624f1e132b..adacaa8df6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,11 +1,11 @@
-git-annex (3.20120612) UNRELEASED; urgency=low
+git-annex (3.20120614) unstable; urgency=medium
 
   * addurl: Was broken by a typo introduced 2 released ago, now fixed.
     Closes: #677576
   * Install man page when run by cabal, in a location where man will
     find it, even when installing under $HOME. Thanks, Nathan Collins
 
- -- Joey Hess   Tue, 12 Jun 2012 11:35:59 -0400
+ -- Joey Hess   Thu, 14 Jun 2012 20:21:29 -0400
 
 git-annex (3.20120611) unstable; urgency=medium
 
diff --git a/git-annex.cabal b/git-annex.cabal
index 9703b61f03..7bbf683a7b 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120612
+Version: 3.20120614
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 

From c24e86c7e1704cbf39b9ca3e213caf4951d376c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 14 Jun 2012 20:34:10 -0400
Subject: [PATCH 3763/8313] add news item for git-annex 3.20120614

---
 doc/news/version_3.20120430.mdwn | 12 ------------
 doc/news/version_3.20120614.mdwn |  6 ++++++
 2 files changed, 6 insertions(+), 12 deletions(-)
 delete mode 100644 doc/news/version_3.20120430.mdwn
 create mode 100644 doc/news/version_3.20120614.mdwn

diff --git a/doc/news/version_3.20120430.mdwn b/doc/news/version_3.20120430.mdwn
deleted file mode 100644
index 07e1e85ad2..0000000000
--- a/doc/news/version_3.20120430.mdwn
+++ /dev/null
@@ -1,12 +0,0 @@
-git-annex 3.20120430 released with [[!toggle text="these changes"]]
-[[!toggleable text="""
-   * Fix use of annex.diskreserve config setting.
-   * Directory special remotes now check annex.diskreserve.
-   * Support git's core.sharedRepository configuration.
-   * Add annex.http-headers and annex.http-headers-command config
-     settings, to allow custom headers to be sent with all HTTP requests.
-     (Requested by the Internet Archive)
-   * uninit: Clear annex.uuid from .git/config. Closes: #[670639](http://bugs.debian.org/670639)
-   * Added shared cipher mode to encryptable special remotes. This option
-     avoids gpg key distribution, at the expense of flexability, and with
-     the requirement that all clones of the git repository be equally trusted."""]]
\ No newline at end of file
diff --git a/doc/news/version_3.20120614.mdwn b/doc/news/version_3.20120614.mdwn
new file mode 100644
index 0000000000..33a06a0bf9
--- /dev/null
+++ b/doc/news/version_3.20120614.mdwn
@@ -0,0 +1,6 @@
+git-annex 3.20120614 released with [[!toggle text="these changes"]]
+[[!toggleable text="""
+   * addurl: Was broken by a typo introduced 2 released ago, now fixed.
+     Closes: #[677576](http://bugs.debian.org/677576)
+   * Install man page when run by cabal, in a location where man will
+     find it, even when installing under $HOME. Thanks, Nathan Collins"""]]
\ No newline at end of file

From 1d5246fb20e17fc8285083c4380afcdaf3e36b79 Mon Sep 17 00:00:00 2001
From: "http://wiggy.net/" 
Date: Fri, 15 Jun 2012 07:19:24 +0000
Subject: [PATCH 3764/8313] Added a comment: os compatibility

---
 .../comment_1_564a39cb976767e2c0a9c74fabe10be4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment

diff --git a/doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment b/doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment
new file mode 100644
index 0000000000..8236002ccd
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_1_564a39cb976767e2c0a9c74fabe10be4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://wiggy.net/"
+ nickname="Wichert"
+ subject="os compatibility"
+ date="2012-06-15T07:19:23Z"
+ content="""
+A downside of relying on lsof is that you might be painting yourself into a linux corner: other operating systems might not have a lsof or alternative you can rely on. Especially for Windows this might be a worry. 
+"""]]

From c51c4685f2b1b753541615eef2b18cb37018781e Mon Sep 17 00:00:00 2001
From: "http://dieter-be.myopenid.com/" 
Date: Fri, 15 Jun 2012 08:21:37 +0000
Subject: [PATCH 3765/8313] Added a comment: filesystem number of open file
 handles on a file

---
 .../comment_2_77924e9d50b40f05e792e427a25849a6._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment

diff --git a/doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment b/doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment
new file mode 100644
index 0000000000..8744882c97
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_2_77924e9d50b40f05e792e427a25849a6._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="filesystem number of open file handles on a file"
+ date="2012-06-15T08:21:37Z"
+ content="""
+wasn't there some filesystem functionality that could tell you the amount of open file handles on a certain file?  I thought this was tracked per-file too.
+Or maybe i'm just confusing it with the number of hard links (which stat can tell you), anyway something to look into.
+"""]]

From fb33e55a6e5ac4c40e6f6cc5a40afb7a229c2c3d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Fri, 15 Jun 2012 08:58:18 +0000
Subject: [PATCH 3766/8313] Added a comment

---
 .../comment_3_92bd86cd06d579e23800af2e5c66a291._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment

diff --git a/doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment b/doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment
new file mode 100644
index 0000000000..680f16ecc1
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_3_92bd86cd06d579e23800af2e5c66a291._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-06-15T08:58:17Z"
+ content="""
+I would also be reluctant to use lsof for the sake of non-linux systems or systems that don't have lsof. I've only been playing around with the watch branch of my \"other\" laptop under archlinux. It looks usable, however I would prefer support for OSX before the watch branch gets merged to master ;)
+"""]]

From 20744d31463cf9e91991a3f4af2251054bbec55a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 15 Jun 2012 09:57:34 +0000
Subject: [PATCH 3767/8313] Added a comment: Battery usage

---
 ...comment_1_a3dba537b276d5737abc8cb93f1965f4._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment

diff --git a/doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment b/doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment
new file mode 100644
index 0000000000..d0a207c82a
--- /dev/null
+++ b/doc/design/assistant/blog/day_8__speed/comment_1_a3dba537b276d5737abc8cb93f1965f4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="Battery usage"
+ date="2012-06-15T09:57:33Z"
+ content="""
+Complete fsck is good, but once a week probably enough.
+
+But please see if you can make fsck optional depending on if the machine is running on battery.
+"""]]

From c3eb00271cf423d8e7f5077cc77fa0b03d7dc052 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 
Date: Fri, 15 Jun 2012 10:21:17 +0000
Subject: [PATCH 3768/8313] Added a comment

---
 ...comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment

diff --git a/doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment b/doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment
new file mode 100644
index 0000000000..7218754916
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_4_0d12b51ccdfc2a94d3e59a5628521e0a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 4"
+ date="2012-06-15T10:21:17Z"
+ content="""
+Corner case, but if the other program finishes writing while you are annexing and your check shows no open files, you are left with bad checksum on a correct file. This \"broken\" file with propagate and the next round of fsck will show that all copies are \"bad\".
+
+Without verifying if this is viable, could you set the file RO and thus block future writes before starting to annex?
+"""]]

From 5e0f1790777b6a4c7799511824db70c871ba6c0f Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 15 Jun 2012 15:14:53 +0000
Subject: [PATCH 3769/8313] Added a comment

---
 ...ent_5_208f9dd3e1d92555b05c29159538a901._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment

diff --git a/doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment b/doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment
new file mode 100644
index 0000000000..b7fbecc398
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_5_208f9dd3e1d92555b05c29159538a901._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 5"
+ date="2012-06-15T15:14:52Z"
+ content="""
+@wichert All this inotify stuff is entirely linux specific AFAIK anyway, so it's find for workarounds to limitations in inotify functionality to also be linux specific.
+
+@dieter I think you're thinking of hard links, filesystems don't track number of open file handles afaik.
+
+@Jimmy, I'm planning to get watch going on freebsd (and hopefully that will also cover OSX), after merging it :)
+
+@Richard, the file is set RO while it's being annexed, so any lsof would come after that point.
+"""]]

From 37a13de8ff931574174a7b64ba6a487b6ef2f754 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 15 Jun 2012 15:23:21 +0000
Subject: [PATCH 3770/8313] Added a comment

---
 .../comment_6_90cc6b60718896fb175919417600fdf9._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment

diff --git a/doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment b/doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment
new file mode 100644
index 0000000000..622b141fdb
--- /dev/null
+++ b/doc/design/assistant/blog/day_9__correctness/comment_6_90cc6b60718896fb175919417600fdf9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 6"
+ date="2012-06-15T15:23:21Z"
+ content="""
+But Rich is right, and I was thinking the same thing earlier this morning, that delaying the lsof allows the writer to change the file and exit, and only fsck can detect the problem then. Setting file permissions doesn't help once a process already has it open for write. Which has put me off the delayed lsof idea unfortunately. lsof *could* be run safely during the intial annexing.
+"""]]

From a95149ac0f05d462edceac867b66b6498f1c7474 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 12:22:28 -0400
Subject: [PATCH 3771/8313] quarantine idea to avoid repeated lsof calls

---
 doc/design/assistant/inotify.mdwn | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 9d8857ab65..baa420b4e9 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -19,6 +19,19 @@ There is a `watch` branch in git that adds the command.
 
   * Somehow track or detect if a file is open for write by any processes.
     `lsof` could be used, although it would be a little slow.
+
+    Here's one way to avoid the slowdown: When a file is being added,
+    set it read-only, and hard-link it into a quarantine directory,
+    remembering both filenames.
+    Then use the batch change mode code to detect batch adds and bundle
+    them together.
+    Just before committing, lsof the quarantine directory. Any files in
+    it that are still open for write can just have their write bit turned
+    back on and be deleted from quarantine, to be handled when their writer
+    closes. Files that pass quarantine get added as usual. This avoids
+    repeated lsof calls slowing down adds, but does add a constant factor
+    overhead (0.25 seconds lsof call) before any add gets committed.
+
   * Or, when possible, making a copy on write copy before adding the file
     would avoid this.
   * Or, as a last resort, make an expensive copy of the file and add that.

From a6dafe9f72cc1b80dcbafba5aef3406fe80fe01a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmURXBzaYE1gmVc-X9eLAyDat_6rHPl670"
 
Date: Fri, 15 Jun 2012 17:39:46 +0000
Subject: [PATCH 3772/8313] Added a comment: Error when installing from Hackage

---
 ...t_1_30651a7c1c0d0879a13ce3380219beaa._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/news/version_3.20120614/comment_1_30651a7c1c0d0879a13ce3380219beaa._comment

diff --git a/doc/news/version_3.20120614/comment_1_30651a7c1c0d0879a13ce3380219beaa._comment b/doc/news/version_3.20120614/comment_1_30651a7c1c0d0879a13ce3380219beaa._comment
new file mode 100644
index 0000000000..3ef89632d0
--- /dev/null
+++ b/doc/news/version_3.20120614/comment_1_30651a7c1c0d0879a13ce3380219beaa._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmURXBzaYE1gmVc-X9eLAyDat_6rHPl670"
+ nickname="Bram"
+ subject="Error when installing from Hackage"
+ date="2012-06-15T17:39:45Z"
+ content="""
+I get this error when trying to install (actually upgrade) from Hackage:
+
+    bram@falafel% cabal install git-annex
+    Resolving dependencies...
+    Downloading git-annex-3.20120614...
+    cabal: Error: some packages failed to install:
+    git-annex-3.20120614 failed while unpacking the package. The exception was:
+    user error (File in tar archive is not in the expected directory
+    \"git-annex-3.20120614\")
+"""]]

From 4b2b5e820edda56ed7c6bb0910c41e8bfff5a06b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 14:11:09 -0400
Subject: [PATCH 3773/8313] add

---
 Utility/Lsof.hs | 71 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 Utility/Lsof.hs

diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs
new file mode 100644
index 0000000000..1aa620e2d1
--- /dev/null
+++ b/Utility/Lsof.hs
@@ -0,0 +1,71 @@
+{- lsof interface
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE BangPatterns #-}
+
+module Utility.Lsof where
+
+import Common
+
+import System.Cmd.Utils
+import System.Posix.Types
+
+data OpenMode = ReadWrite | ReadOnly | WriteOnly | Unknown
+
+type CmdLine = String
+
+data ProcessInfo = ProcessInfo ProcessID CmdLine
+
+query :: FilePath -> IO [(FilePath, OpenMode, ProcessInfo)]
+query p = do
+	(h, s) <- pipeFrom "lsof" ["-F0can", "--", p]
+	!r <- parse s
+	forceSuccess h
+	return r
+
+{- Parsing null-delimited output like:
+ -
+ - pPID\0cCMDLINE\0
+ - aMODE\0nFILE\0
+ - aMODE\0nFILE\0
+ - pPID\0[...]
+ -
+ - Where each new process block is started by a pid, and a process can
+ - have multiple files open.
+ -}
+parse :: String -> [(FilePath, OpenMode, ProcessInfo)]
+parse s = go [] $ lines s
+	where
+		go c [] = c
+		go c (l@(t:r):ls)
+			| t == 'p' = parseprocess r
+			| otherwise = parsefail
+		go _ _ = parsefail
+
+		parseprocess l =
+			case splitnull l of
+				[pid, 'c':cmdline] ->
+					case readish pid of
+						(Just n) -> ProcessInfo n cmdline
+						Nothing -> parsefail
+				_ -> parsefail
+
+		parsefile l =
+			case splitnull l of
+				['a':mode, 'n':file] -> (file, parsemode mode)
+				_ -> parsefail
+
+		parsemode ('r':_) = ReadOnly
+		parsemode ('w':_) = WriteOnly
+		parsemode ('u':_) = ReadWrite
+		parsemode _ = Unknown
+
+		ls = lines s
+
+		splitnull = split "\0"
+
+		parsefail = error "failed to parse lsof output: " ++ show s

From 01beef445963c5c92db1d36b7c816c2d0a373ae7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 14:14:50 -0400
Subject: [PATCH 3774/8313] fix make-sdist to omit too long filenames

---
 Makefile      |  2 --
 make-sdist.sh | 10 +++++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index d23f08a7d1..023ea5d5e1 100644
--- a/Makefile
+++ b/Makefile
@@ -100,8 +100,6 @@ clean:
 	rm -rf tmp $(bins) $(mans) test configure  *.tix .hpc $(sources) \
 		doc/.ikiwiki html dist $(clibs)
 
-# Workaround for `cabal sdist` requiring all included files to be listed
-# in .cabal.
 sdist: clean $(mans)
 	./make-sdist.sh
 
diff --git a/make-sdist.sh b/make-sdist.sh
index 4098719160..3222cad7e3 100755
--- a/make-sdist.sh
+++ b/make-sdist.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
+#
+# Workaround for `cabal sdist` requiring all included files to be listed
+# in .cabal.
 
 # Create target directory
 sdist_dir=git-annex-$(grep '^Version:' git-annex.cabal | sed -re 's/Version: *//')
@@ -6,8 +9,13 @@ mkdir --parents dist/$sdist_dir
 
 find . \( -name .git -or -name dist -or -name cabal-dev \) -prune \
 	-or -not -name \\*.orig -not -type d -print \
-| perl -ne 'print unless length >= 100' \
+| perl -ne "print unless length >= 100 - length q{$sdist_dir}" \
 | xargs cp --parents --target-directory dist/$sdist_dir
 
 cd dist
 tar -caf $sdist_dir.tar.gz $sdist_dir
+
+# Check that tarball can be unpacked by cabal.
+# It's picky about tar longlinks etc.
+rm -rf $sdist_dir
+cabal unpack $sdist_dir.tar.gz

From 4ecbf78eca4f8f0ea6dfd1a55f7eb62f6cbee527 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 15 Jun 2012 18:21:03 +0000
Subject: [PATCH 3775/8313] Added a comment

---
 .../comment_2_2b309b763d70cfc03fb9e7f6142ba477._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/news/version_3.20120614/comment_2_2b309b763d70cfc03fb9e7f6142ba477._comment

diff --git a/doc/news/version_3.20120614/comment_2_2b309b763d70cfc03fb9e7f6142ba477._comment b/doc/news/version_3.20120614/comment_2_2b309b763d70cfc03fb9e7f6142ba477._comment
new file mode 100644
index 0000000000..70a11542f1
--- /dev/null
+++ b/doc/news/version_3.20120614/comment_2_2b309b763d70cfc03fb9e7f6142ba477._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 2"
+ date="2012-06-15T18:21:03Z"
+ content="""
+My, cabal is picky about the tarballs it will accept. Doesn't understand longlinks in tarballs. I've uploaded a new release.
+"""]]

From 9c519ba955d9ff8080558aa37d5e50c266c6e2d9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 14:22:00 -0400
Subject: [PATCH 3776/8313] not bash specific

---
 make-sdist.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/make-sdist.sh b/make-sdist.sh
index 3222cad7e3..9503345327 100755
--- a/make-sdist.sh
+++ b/make-sdist.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Workaround for `cabal sdist` requiring all included files to be listed
 # in .cabal.

From 98148bece8beea3ffe9c1b464aaa32293b23f1e3 Mon Sep 17 00:00:00 2001
From: "http://denis.laxalde.org/" 
Date: Fri, 15 Jun 2012 18:57:08 +0000
Subject: [PATCH 3777/8313]

---
 doc/forum/exporting_annexed_files.mdwn | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/forum/exporting_annexed_files.mdwn

diff --git a/doc/forum/exporting_annexed_files.mdwn b/doc/forum/exporting_annexed_files.mdwn
new file mode 100644
index 0000000000..0b8d6f36b2
--- /dev/null
+++ b/doc/forum/exporting_annexed_files.mdwn
@@ -0,0 +1,4 @@
+Is there an easy way to export annexed files out of the repository? (e.g. to make a copy elsewhere, send a file by email...)
+
+Thanks,
+Denis.

From 9e5c5bb6e15a26c955c47fb895c0d3421ae80966 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Fri, 15 Jun 2012 19:06:45 +0000
Subject: [PATCH 3778/8313]

---
 doc/install.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index fe0522aa05..fef146544a 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -10,6 +10,7 @@
 * [[NixOS]]
 * [[Gentoo]]
 * Windows: [[sorry, not possible yet|todo/windows_support]]
+* [[ScientificLinux5]] - This should cover RHEL5 clones such as CentOS5 and so on
 
 ## Using cabal
 

From df56fc370217ddfdc435982d2a4590da77fe08f1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Fri, 15 Jun 2012 19:10:36 +0000
Subject: [PATCH 3779/8313]

---
 doc/ScientificLinux5.mdwn | 70 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 doc/ScientificLinux5.mdwn

diff --git a/doc/ScientificLinux5.mdwn b/doc/ScientificLinux5.mdwn
new file mode 100644
index 0000000000..4292813701
--- /dev/null
+++ b/doc/ScientificLinux5.mdwn
@@ -0,0 +1,70 @@
+I was waiting for my backups to be done hence this article, as I was using
+_git-annex_ to manage my files and I decided I needed to have
+git-annex on a SL5 based machine. SL5 is just an opensource
+clone/recompile of RHEL5.
+
+I haven't tried to install the newer versions of Haskell Platform and
+GHC in a while on SL5 to install git-annex. But the last time I checked
+when GHC7 was out, it was a pain to install GHC on SL5.
+
+However I have discovered that someone has gone through the trouble of
+packaging up GHC and Haskell Platform for RHEL based distros.
+
+*  - Packaged GHC and Haskell Platform
+  RPM's for RHEL based systems.
+
+I'm primarily interested in installing _git-annex_ on SL5 based
+systems. The installation process goes as such...
+
+First install GHC and Haskell Platform (you need root for these
+following steps)
+
+    $ wget http://sherkin.justhub.org/el5/RPMS/x86_64/justhub-release-2.0-4.0.el5.x86_64.rpm
+    $ rpm -ivh justhub-release-2.0-4.0.el5.x86_64.rpm
+    $ yum install haskell
+
+The RPM's don't place the files in /usr/bin, so you must add the
+following to your .bashrc (from here on you don't need root if you
+don't want things to be system wide)
+
+    $ export PATH=/usr/hs/bin:$PATH
+
+On SL5 pcre is at version 6.6 which is far too old for one of the
+dependancies that git-annex requires. Therefore the user must install
+an updated version of _pcre_ either from source or another method, I
+chose to install it from source and by hand into /usr/local
+
+    $ wget http://sourceforge.net/projects/pcre/files/pcre/8.30/pcre-8.30.tar.gz/download
+    $ tar zxvf pcre-8.30.tar.gz
+    $ cd pcre-8.30
+    $ ./configure
+    $ make && make install
+
+Once the packages are installed and are in your execution path, using
+cabal to configure and build git-annex just makes life easier, it
+should install all the needed dependancies.
+
+    $ cabal update
+    $ cabal install pcre-light --extra-include-dirs=/usr/local/include
+    $ git clone git://git.kitenet.net/git-annex
+    $ cd git-annex
+    $ make git-annex.1
+    $ cabal configure
+    $ cabal build
+    $ cabal install
+
+Or if you want to install it globallly for everyone (otherwise it will
+get installed into $HOME/.cabal/bin)
+
+    $ cabal install --global
+
+The above will take a while to compile and install the needed
+dependancies. I would suggest any user who does should run the tests
+that comes with git-annex to make sure everything is functioning as
+expected.
+
+I haven't had a chance or need to install git-annex on a SL6 based
+system yet, but I would assume something similar to the above steps
+would be required to do so.
+
+The above is almost a cut and paste of , the above could probably be refined, it was what worked for me on SL5. Please feel free to re-edit and chop out or add useless bits of text in the above!

From af7b6319d725e8090b32f74b397a9eba79e22978 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 15:11:18 -0400
Subject: [PATCH 3780/8313] move

---
 doc/{ => install}/ScientificLinux5.mdwn | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/{ => install}/ScientificLinux5.mdwn (100%)

diff --git a/doc/ScientificLinux5.mdwn b/doc/install/ScientificLinux5.mdwn
similarity index 100%
rename from doc/ScientificLinux5.mdwn
rename to doc/install/ScientificLinux5.mdwn

From 85db1720e9cb3e06951554ffa1079b1b8a814f92 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 15:13:31 -0400
Subject: [PATCH 3781/8313] add lsof interface

Uses lsof -F0 to get machine-readable output
---
 Utility/Lsof.hs | 42 ++++++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs
index 1aa620e2d1..0491487bc1 100644
--- a/Utility/Lsof.hs
+++ b/Utility/Lsof.hs
@@ -14,17 +14,20 @@ import Common
 import System.Cmd.Utils
 import System.Posix.Types
 
-data OpenMode = ReadWrite | ReadOnly | WriteOnly | Unknown
+data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown
+	deriving (Show)
 
 type CmdLine = String
 
 data ProcessInfo = ProcessInfo ProcessID CmdLine
+	deriving (Show)
 
-query :: FilePath -> IO [(FilePath, OpenMode, ProcessInfo)]
+query :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
 query p = do
-	(h, s) <- pipeFrom "lsof" ["-F0can", "--", p]
-	!r <- parse s
-	forceSuccess h
+	(pid, s) <- pipeFrom "lsof" ["-F0can", "--", p]
+	let !r = parse s
+	-- ignore nonzero exit code; lsof returns that when no files are open
+	void $ getProcessStatus True False $ processID pid
 	return r
 
 {- Parsing null-delimited output like:
@@ -37,35 +40,42 @@ query p = do
  - Where each new process block is started by a pid, and a process can
  - have multiple files open.
  -}
-parse :: String -> [(FilePath, OpenMode, ProcessInfo)]
-parse s = go [] $ lines s
+parse :: String -> [(FilePath, LsofOpenMode, ProcessInfo)]
+parse s = bundle $ go [] $ lines s
 	where
+		bundle = concatMap (\(fs, p) -> map (\(f, m) -> (f, m, p)) fs)
+
 		go c [] = c
 		go c (l@(t:r):ls)
-			| t == 'p' = parseprocess r
+			| t == 'p' =
+				let (fs, ls') = parsefiles [] ls
+				in go ((fs, parseprocess r):c) ls'
 			| otherwise = parsefail
 		go _ _ = parsefail
 
 		parseprocess l =
 			case splitnull l of
-				[pid, 'c':cmdline] ->
+				[pid, 'c':cmdline, ""] ->
 					case readish pid of
 						(Just n) -> ProcessInfo n cmdline
 						Nothing -> parsefail
 				_ -> parsefail
 
-		parsefile l =
+		parsefiles c [] = (c, [])
+		parsefiles c (l:ls) = 
 			case splitnull l of
-				['a':mode, 'n':file] -> (file, parsemode mode)
+				['a':mode, 'n':file, ""] ->
+					parsefiles ((file, parsemode mode):c) ls
+				(('p':_):_) -> (c, l:ls)
 				_ -> parsefail
 
-		parsemode ('r':_) = ReadOnly
-		parsemode ('w':_) = WriteOnly
-		parsemode ('u':_) = ReadWrite
-		parsemode _ = Unknown
+		parsemode ('r':_) = OpenReadOnly
+		parsemode ('w':_) = OpenWriteOnly
+		parsemode ('u':_) = OpenReadWrite
+		parsemode _ = OpenUnknown
 
 		ls = lines s
 
 		splitnull = split "\0"
 
-		parsefail = error "failed to parse lsof output: " ++ show s
+		parsefail = error $ "failed to parse lsof output: " ++ show s

From 3a05b66cf9197ab3fcac7753f38d1d92429f6f16 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 15 Jun 2012 19:25:59 +0000
Subject: [PATCH 3782/8313] Added a comment

---
 ...t_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment

diff --git a/doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment b/doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment
new file mode 100644
index 0000000000..69fc46245f
--- /dev/null
+++ b/doc/forum/exporting_annexed_files/comment_1_e08e4c79588e17fb2f1cdf53d9fab7ea._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 1"
+ date="2012-06-15T19:25:59Z"
+ content="""
+Sure, you can simply:
+
+    cp annexedfile ~
+
+Or just attach the file right from the git repository to an email, like any other file. Should work fine.
+
+If you wanted to copy a whole directory to export, you'd need to use the -L flag to make cp follow the symlinks and copy the real contents:
+
+    cp -r -L annexeddirectory /media/usbdrive/
+"""]]

From 0641d6053321b1ef3049554929065402e3d25f6d Mon Sep 17 00:00:00 2001
From: "http://denis.laxalde.org/" 
Date: Fri, 15 Jun 2012 19:57:31 +0000
Subject: [PATCH 3783/8313] Added a comment: nautilus

---
 .../comment_2_15dc3024417b5b2ff3544a08beacab34._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment

diff --git a/doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment b/doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment
new file mode 100644
index 0000000000..3621f9b895
--- /dev/null
+++ b/doc/forum/exporting_annexed_files/comment_2_15dc3024417b5b2ff3544a08beacab34._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://denis.laxalde.org/"
+ nickname="dlax"
+ subject="nautilus"
+ date="2012-06-15T19:57:31Z"
+ content="""
+Ah! I was fooled by nautilus which is not able to properly handle symlinks when copying. It copies links instead of target [[!gnomebug 623580]].
+"""]]

From 8c7dfc93b523957334c3b93530a85b7dcfa5cfb1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 18:31:18 -0400
Subject: [PATCH 3784/8313] catch IO exceptions in runThreadState

A few places catch IO errors after calling runThreadState,
but since the MVar was not restored, it'd later deadlock trying to read
from it.

I'd like to catch all exceptions here, but I could not get the types
to unify.
---
 Assistant/ThreadedMonad.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs
index c4d331f61b..51f579d07f 100644
--- a/Assistant/ThreadedMonad.hs
+++ b/Assistant/ThreadedMonad.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import qualified Annex
 
 import Control.Concurrent
+import Control.Exception (throw)
 
 {- The Annex state is stored in a MVar, so that threaded actions can access
  - it. -}
@@ -35,6 +36,9 @@ withThreadState a = do
 runThreadState :: ThreadState -> Annex a -> IO a
 runThreadState mvar a = do
 	startstate <- takeMVar mvar
-	!(r, newstate) <- Annex.run startstate a
+	-- catch IO errors and rethrow after restoring the MVar
+	!(r, newstate) <- catchIO (Annex.run startstate a) $ \e -> do
+		putMVar mvar startstate
+		throw e
 	putMVar mvar newstate
 	return r

From c27c751b34af993a67e04e079d6f860a69f25038 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 16:27:44 -0400
Subject: [PATCH 3785/8313] preliminary deferring of file adds to commit time

Defer adding files to the annex until commit time, when during a batch
operation, a bundle of files will be available. This will allow for
checking a them all with a single lsof call.

The tricky part is that adding the file causes a symlink change inotify.
So I made it wait for an appropriate number of symlink changes to be
received before continuing with the commit. This avoids any delay
in the commit process. It is possible that some unrelated symlink change is
made; if that happens it'll commit it and delay committing the newly added
symlink for 1 second. This seems ok. I do rely on the expected symlink
change event always being received, but only when the add succeeds.

Another way to do it might be to directly stage the symlink, and then
ignore the redundant symlink change event. That would involve some
redundant work, and perhaps an empty commit, but if this code turns
out to have some bug, that'd be the best way to avoid it.

FWIW, this change seems to, as a bonus, have produced better grouping
of batch changes into single commits. Before, a large batch change would
result in a series of commits, with the first containing only one file,
and each of the rest bundling a number of files. Now, the added wait for
the symlink changes to arrive gives time for additional add changes to
be processed, all within the same commit.
---
 Assistant/Committer.hs | 68 +++++++++++++++++++++++++++++++++++++-----
 Assistant/Watcher.hs   | 42 ++++++++++++--------------
 2 files changed, 80 insertions(+), 30 deletions(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index a572556de5..6e56c22357 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -9,17 +9,21 @@ import Common.Annex
 import Assistant.ThreadedMonad
 import qualified Annex.Queue
 import qualified Git.Command
+import qualified Command.Add
 import Utility.ThreadScheduler
 
 import Control.Concurrent.STM
 import Data.Time.Clock
 
+data ChangeType = PendingAddChange | LinkChange | RmChange | RmDirChange
+	deriving (Show, Eq)
+
 type ChangeChan = TChan Change
 
 data Change = Change
 	{ changeTime :: UTCTime
 	, changeFile :: FilePath
-	, changeDesc :: String
+	, changeType :: ChangeType
 	}
 	deriving (Show)
 
@@ -30,11 +34,12 @@ newChangeChan :: IO ChangeChan
 newChangeChan = atomically newTChan
 
 {- Handlers call this when they made a change that needs to get committed. -}
-madeChange :: FilePath -> String -> Annex (Maybe Change)
-madeChange file desc = do
+madeChange :: FilePath -> ChangeType -> Annex (Maybe Change)
+madeChange f t = do
 	-- Just in case the commit thread is not flushing the queue fast enough.
-	Annex.Queue.flushWhenFull
-	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure file <*> pure desc)
+	when (t /= PendingAddChange) $
+		Annex.Queue.flushWhenFull
+	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure f <*> pure t)
 
 noChange :: Annex (Maybe Change)
 noChange = return Nothing
@@ -66,9 +71,58 @@ commitThread st changechan = runEvery (Seconds 1) $ do
 	-- Now see if now's a good time to commit.
 	time <- getCurrentTime
 	if shouldCommit time cs
-		then void $ tryIO $ runThreadState st commitStaged
+		then do
+			handleAdds st changechan cs
+			void $ tryIO $ runThreadState st commitStaged
 		else refillChanges changechan cs
 
+{- If there are PendingAddChanges, the files have not yet actually been
+ - added to the annex, and that has to be done now, before committing.
+ -
+ - Deferring the adds to this point causes batches to be bundled together,
+ - which allows faster checking with lsof that the files are not still open
+ - for write by some other process.
+ -
+ - When a file is added, Inotify will notice the new symlink. So this waits
+ - for one new LinkChange to be received per file that's successfully
+ - added, to ensure that its symlink has been staged before returning.
+ -}
+handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
+handleAdds st changechan cs
+	| null added = noop
+	| otherwise = do
+		numadded <- length . filter id <$>
+			runThreadState st (forM added add)
+		waitforlinkchanges numadded
+	where
+		added = filter isPendingAdd cs
+
+		isPendingAdd (Change { changeType = PendingAddChange }) = True
+		isPendingAdd _ = False
+		isLinkChange (Change { changeType = LinkChange }) = True
+		isLinkChange _ = False
+
+		add (Change { changeFile = file }) = do
+			showStart "add" file
+			handle file =<< Command.Add.ingest file
+
+		handle _ Nothing = do
+			showEndFail
+			return False
+		handle file (Just key) = do
+			Command.Add.link file key True
+			showEndOk
+			return True
+
+		waitforlinkchanges 0 = noop
+		waitforlinkchanges n = do
+			c <- runChangeChan $ readTChan changechan
+			if (isLinkChange c)
+				then waitforlinkchanges (n-1)
+				else do
+					handleAdds st changechan [c]
+					waitforlinkchanges n
+
 commitStaged :: Annex ()
 commitStaged = do
 	Annex.Queue.flush
@@ -87,7 +141,7 @@ commitStaged = do
 {- Decide if now is a good time to make a commit.
  - Note that the list of change times has an undefined order.
  -
- - Current strategy: If there have been 10 commits within the past second,
+ - Current strategy: If there have been 10 changes within the past second,
  - a batch activity is taking place, so wait for later.
  -}
 shouldCommit :: UTCTime -> [Change] -> Bool
diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index ee5bc13af0..4aac33fd1f 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -15,7 +15,6 @@ import Assistant.DaemonStatus
 import Assistant.Committer
 import Utility.ThreadLock
 import qualified Annex.Queue
-import qualified Command.Add
 import qualified Git.Command
 import qualified Git.UpdateIndex
 import qualified Git.HashObject
@@ -87,18 +86,20 @@ runHandler st dstatus changechan handler file filestatus = void $ do
 	where
 		go = runThreadState st $ handler file filestatus dstatus
 
-{- Adding a file is tricky; the file has to be replaced with a symlink
- - but this is race prone, as the symlink could be changed immediately
- - after creation. To avoid that race, git add is not used to stage the
- - symlink.
+{- During initial directory scan, this will be run for any regular files
+ - that are already checked into git. We don't want to turn those into
+ - symlinks, so do a check. This is rather expensive, but only happens
+ - during startup.
  -
- - Inotify will notice the new symlink, so this Handler does not stage it
- - or return a Change, leaving that to onAddSymlink.
+ - It's possible for the file to still be open for write by some process.
+ - This can happen in a few ways; one is if two processes had the file open
+ - and only one has just closed it. We want to avoid adding a file to the
+ - annex that is open for write, to avoid anything being able to change it.
  -
- - During initial directory scan, this will be run for any files that
- - are already checked into git. We don't want to turn those into symlinks,
- - so do a check. This is rather expensive, but only happens during
- - startup.
+ - We could run lsof on the file here to check for other writer.
+ - But, that's slow. Instead, a Change is returned that indicates this file
+ - still needs to be added. The committer will handle bundles of these
+ - Changes at once.
  -}
 onAdd :: Handler
 onAdd file _filestatus dstatus = do
@@ -110,14 +111,7 @@ onAdd file _filestatus dstatus = do
 			)
 		)
 	where
-		go = do
-			showStart "add" file
-			handle =<< Command.Add.ingest file
-			noChange
-		handle Nothing = showEndFail
-		handle (Just key) = do
-			Command.Add.link file key True
-			showEndOk
+		go = madeChange file PendingAddChange
 
 {- A symlink might be an arbitrary symlink, which is just added.
  - Or, if it is a git-annex symlink, ensure it points to the content
@@ -169,13 +163,13 @@ onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file
 					sha <- inRepo $
 						Git.HashObject.hashObject BlobObject link
 					stageSymlink file sha
-			madeChange file "link"
+			madeChange file LinkChange
 
 onDel :: Handler
 onDel file _ _dstatus = do
 	Annex.Queue.addUpdateIndex =<<
 		inRepo (Git.UpdateIndex.unstageFile file)
-	madeChange file "rm"
+	madeChange file RmChange
 
 {- A directory has been deleted, or moved, so tell git to remove anything
  - that was inside it from its cache. Since it could reappear at any time,
@@ -188,7 +182,7 @@ onDelDir :: Handler
 onDelDir dir _ _dstatus = do
 	Annex.Queue.addCommand "rm"
 		[Params "--quiet -r --cached --ignore-unmatch --"] [dir]
-	madeChange dir "rmdir"
+	madeChange dir RmDirChange
 
 {- Called when there's an error with inotify. -}
 onErr :: Handler
@@ -197,7 +191,9 @@ onErr msg _ _dstatus = do
 	return Nothing
 
 {- Adds a symlink to the index, without ever accessing the actual symlink
- - on disk. -}
+ - on disk. This avoids a race if git add is used, where the symlink is
+ - changed to something else immediately after creation.
+ -}
 stageSymlink :: FilePath -> Sha -> Annex ()
 stageSymlink file sha =
 	Annex.Queue.addUpdateIndex =<<

From 679ef4c85896b6830bc6ae335995558577f4e435 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 18:56:23 -0400
Subject: [PATCH 3786/8313] continued work on deferred addding

---
 Assistant/Committer.hs | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index 6e56c22357..64ea300e26 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -91,18 +91,18 @@ handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
 handleAdds st changechan cs
 	| null added = noop
 	| otherwise = do
-		numadded <- length . filter id <$>
-			runThreadState st (forM added add)
-		waitforlinkchanges numadded
+		r <- forM added $ catchBoolIO . runThreadState st . add
+		let numadded = length $ filter id r
+		handleAdds st changechan =<< waitforlinkchanges [] numadded
 	where
-		added = filter isPendingAdd cs
+		added = map changeFile $ filter isPendingAdd cs
 
 		isPendingAdd (Change { changeType = PendingAddChange }) = True
 		isPendingAdd _ = False
 		isLinkChange (Change { changeType = LinkChange }) = True
 		isLinkChange _ = False
 
-		add (Change { changeFile = file }) = do
+		add file = do
 			showStart "add" file
 			handle file =<< Command.Add.ingest file
 
@@ -114,14 +114,13 @@ handleAdds st changechan cs
 			showEndOk
 			return True
 
-		waitforlinkchanges 0 = noop
-		waitforlinkchanges n = do
-			c <- runChangeChan $ readTChan changechan
-			if (isLinkChange c)
-				then waitforlinkchanges (n-1)
-				else do
-					handleAdds st changechan [c]
-					waitforlinkchanges n
+		waitforlinkchanges c n
+			| n < 1 = return $ concat c
+			| otherwise = do
+				(done, rest) <- partition isLinkChange 
+					<$> getChanges changechan
+				let n' = (n - length done)
+				waitforlinkchanges (rest:c) n'
 
 commitStaged :: Annex ()
 commitStaged = do

From 59abd787c948ef7a6bda3b62be9024212eb69a46 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 19:17:21 -0400
Subject: [PATCH 3787/8313] can't wait for LinkChanges specifically

There is indeed a race waiting for LinkChanges:

1. file annexed, link made
2. link deleted
3. inotify event for link creation runs, but as link is gone, handler is not run
---
 Assistant/Committer.hs | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index 64ea300e26..71152e5e97 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -84,23 +84,20 @@ commitThread st changechan = runEvery (Seconds 1) $ do
  - for write by some other process.
  -
  - When a file is added, Inotify will notice the new symlink. So this waits
- - for one new LinkChange to be received per file that's successfully
- - added, to ensure that its symlink has been staged before returning.
+ - for additional Changes to arrive, so that the symlink has hopefully been
+ - staged before returning.
  -}
 handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
 handleAdds st changechan cs
 	| null added = noop
 	| otherwise = do
-		r <- forM added $ catchBoolIO . runThreadState st . add
-		let numadded = length $ filter id r
-		handleAdds st changechan =<< waitforlinkchanges [] numadded
+		forM_ added $ catchBoolIO . runThreadState st . add
+		handleAdds st changechan =<< getChanges changechan
 	where
 		added = map changeFile $ filter isPendingAdd cs
 
 		isPendingAdd (Change { changeType = PendingAddChange }) = True
 		isPendingAdd _ = False
-		isLinkChange (Change { changeType = LinkChange }) = True
-		isLinkChange _ = False
 
 		add file = do
 			showStart "add" file
@@ -114,14 +111,6 @@ handleAdds st changechan cs
 			showEndOk
 			return True
 
-		waitforlinkchanges c n
-			| n < 1 = return $ concat c
-			| otherwise = do
-				(done, rest) <- partition isLinkChange 
-					<$> getChanges changechan
-				let n' = (n - length done)
-				waitforlinkchanges (rest:c) n'
-
 commitStaged :: Annex ()
 commitStaged = do
 	Annex.Queue.flush

From 96ac25094b04548812ec3d8e0860a62c9438b197 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 20:41:28 -0400
Subject: [PATCH 3788/8313] fix pid file writing

need to truncate, or part of previous longer pid may be left after writing
---
 Utility/Daemon.hs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index 3d2faed676..192340cef7 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -44,7 +44,8 @@ daemonize logfd pidfile changedirectory a = do
 
 lockPidFile :: Bool -> IO () -> FilePath -> IO ()
 lockPidFile write onfailure file = do
-	fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
+	fd <- openFd file ReadWrite (Just stdFileMode)
+		defaultFileFlags { trunc = write }
 	locked <- catchMaybeIO $ setLock fd (locktype, AbsoluteSeek, 0, 0)
 	case locked of
 		Nothing -> onfailure

From 7ee300a47fe8a35019ae26e1dc787a136f53213c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 20:44:34 -0400
Subject: [PATCH 3789/8313] race avoidance

When there are duplicate add events for the same file, only add it once.
---
 Assistant/Committer.hs | 80 ++++++++++++++++++++++++------------------
 1 file changed, 45 insertions(+), 35 deletions(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index 71152e5e97..1348456cca 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -76,41 +76,6 @@ commitThread st changechan = runEvery (Seconds 1) $ do
 			void $ tryIO $ runThreadState st commitStaged
 		else refillChanges changechan cs
 
-{- If there are PendingAddChanges, the files have not yet actually been
- - added to the annex, and that has to be done now, before committing.
- -
- - Deferring the adds to this point causes batches to be bundled together,
- - which allows faster checking with lsof that the files are not still open
- - for write by some other process.
- -
- - When a file is added, Inotify will notice the new symlink. So this waits
- - for additional Changes to arrive, so that the symlink has hopefully been
- - staged before returning.
- -}
-handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
-handleAdds st changechan cs
-	| null added = noop
-	| otherwise = do
-		forM_ added $ catchBoolIO . runThreadState st . add
-		handleAdds st changechan =<< getChanges changechan
-	where
-		added = map changeFile $ filter isPendingAdd cs
-
-		isPendingAdd (Change { changeType = PendingAddChange }) = True
-		isPendingAdd _ = False
-
-		add file = do
-			showStart "add" file
-			handle file =<< Command.Add.ingest file
-
-		handle _ Nothing = do
-			showEndFail
-			return False
-		handle file (Just key) = do
-			Command.Add.link file key True
-			showEndOk
-			return True
-
 commitStaged :: Annex ()
 commitStaged = do
 	Annex.Queue.flush
@@ -141,3 +106,48 @@ shouldCommit now changes
 	where
 		len = length changes
 		thisSecond c = now `diffUTCTime` changeTime c <= 1
+
+{- If there are PendingAddChanges, the files have not yet actually been
+ - added to the annex (probably), and that has to be done now, before
+ - committing.
+ -
+ - Deferring the adds to this point causes batches to be bundled together,
+ - which allows faster checking with lsof that the files are not still open
+ - for write by some other process.
+ -
+ - When a file is added, Inotify will notice the new symlink. So this waits
+ - for additional Changes to arrive, so that the symlink has hopefully been
+ - staged before returning, and will be committed.
+ -}
+handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
+handleAdds st changechan cs
+	| null toadd = noop
+	| otherwise = do
+		added <- filter id <$> forM toadd go
+		unless (null added) $
+			handleAdds st changechan =<< getChanges changechan
+	where
+		toadd = map changeFile $ filter isPendingAdd cs
+
+		isPendingAdd (Change { changeType = PendingAddChange }) = True
+		isPendingAdd _ = False
+
+		go file = do
+			ms <- catchMaybeIO $ getSymbolicLinkStatus file
+			case ms of
+				Just s
+					| isRegularFile s -> catchBoolIO $
+						runThreadState st $ add file
+				_ -> return False
+
+		add file = do
+			showStart "add" file
+			handle file =<< Command.Add.ingest file
+
+		handle _ Nothing = do
+			showEndFail
+			return False
+		handle file (Just key) = do
+			Command.Add.link file key True
+			showEndOk
+			return True

From 1bae56e4a09533b777b8c2b58c551a11e36749de Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:06:59 -0400
Subject: [PATCH 3790/8313] tweak

---
 Command/Add.hs | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index ccdff67ec9..323d1ed1e8 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -50,7 +50,7 @@ start file = notBareRepo $ ifAnnexed file fixup add
  - to prevent it from being modified in between. It's hard linked into a
  - temporary location, and its writable bits are removed. It could still be
  - written to by a process that already has it open for writing. -}
-lockDown :: FilePath -> Annex FilePath
+lockDown :: FilePath -> Annex KeySource
 lockDown file = do
 	liftIO $ preventWrite file
 	tmp <- fromRepo gitAnnexTmpDir
@@ -59,24 +59,27 @@ lockDown file = do
 	let tmpfile = tmp  "add" ++ show pid ++ "." ++ takeFileName file
 	liftIO $ nukeFile tmpfile
 	liftIO $ createLink file tmpfile
-	return tmpfile
+	return $ KeySource { keyFilename = file , contentLocation = tmpfile }
 
-{- Moves the file into the annex. -}
-ingest :: FilePath -> Annex (Maybe Key)
-ingest file = do
-	tmpfile <- lockDown file
-	let source = KeySource { keyFilename = file, contentLocation = tmpfile }
-	backend <- chooseBackend file
-	genKey source backend >>= go tmpfile
+{- Moves a locked down file into the annex. -}
+ingest :: KeySource -> Annex (Maybe Key)
+ingest source = do
+	backend <- chooseBackend $ keyFilename source
+	genKey source backend >>= go
 	where
-		go _ Nothing = return Nothing
-		go tmpfile (Just (key, _)) = do
-			handle (undo file key) $ moveAnnex key tmpfile
-			liftIO $ nukeFile file
+		go Nothing = do
+			liftIO $ nukeFile $ contentLocation source
+			return Nothing
+		go (Just (key, _)) = do
+			handle (undo (keyFilename source) key) $
+				moveAnnex key $ contentLocation source
+			liftIO $ nukeFile $ keyFilename source
 			return $ Just key
 
 perform :: FilePath -> CommandPerform
-perform file = maybe stop (\key -> next $ cleanup file key True) =<< ingest file
+perform file = 
+	maybe stop (\key -> next $ cleanup file key True)
+		=<< ingest =<< lockDown file
 
 {- On error, put the file back so it doesn't seem to have vanished.
  - This can be called before or after the symlink is in place. -}

From e32dda07ca739d2fcab7a764916643d247e94a1b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:16:00 -0400
Subject: [PATCH 3791/8313] better temp file handling

---
 Command/Add.hs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/Command/Add.hs b/Command/Add.hs
index 323d1ed1e8..df57ba6cdf 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -55,11 +55,10 @@ lockDown file = do
 	liftIO $ preventWrite file
 	tmp <- fromRepo gitAnnexTmpDir
 	createAnnexDirectory tmp
-	pid <- liftIO getProcessID
-	let tmpfile = tmp  "add" ++ show pid ++ "." ++ takeFileName file
-	liftIO $ nukeFile tmpfile
-	liftIO $ createLink file tmpfile
-	return $ KeySource { keyFilename = file , contentLocation = tmpfile }
+	liftIO $ do
+		(tmpfile, _handle) <- openTempFile tmp (takeFileName file)
+		createLink file tmpfile
+		return $ KeySource { keyFilename = file , contentLocation = tmpfile }
 
 {- Moves a locked down file into the annex. -}
 ingest :: KeySource -> Annex (Maybe Key)

From d91950ecbad5f33e4d9ce80fc4256c55e0879cc2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:16:13 -0400
Subject: [PATCH 3792/8313] cleanup

---
 Utility/Lsof.hs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs
index 0491487bc1..1c0e934fd7 100644
--- a/Utility/Lsof.hs
+++ b/Utility/Lsof.hs
@@ -11,11 +11,10 @@ module Utility.Lsof where
 
 import Common
 
-import System.Cmd.Utils
 import System.Posix.Types
 
 data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown
-	deriving (Show)
+	deriving (Show, Eq)
 
 type CmdLine = String
 
@@ -46,7 +45,7 @@ parse s = bundle $ go [] $ lines s
 		bundle = concatMap (\(fs, p) -> map (\(f, m) -> (f, m, p)) fs)
 
 		go c [] = c
-		go c (l@(t:r):ls)
+		go c ((t:r):ls)
 			| t == 'p' =
 				let (fs, ls') = parsefiles [] ls
 				in go ((fs, parseprocess r):c) ls'
@@ -74,8 +73,6 @@ parse s = bundle $ go [] $ lines s
 		parsemode ('u':_) = OpenReadWrite
 		parsemode _ = OpenUnknown
 
-		ls = lines s
-
 		splitnull = split "\0"
 
 		parsefail = error $ "failed to parse lsof output: " ++ show s

From 3dac81d3450da25581ad8f4bcfb615da5050767d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:19:12 -0400
Subject: [PATCH 3793/8313] remove newly created tmp file before linking

---
 Command/Add.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Command/Add.hs b/Command/Add.hs
index df57ba6cdf..7a66960632 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -57,6 +57,7 @@ lockDown file = do
 	createAnnexDirectory tmp
 	liftIO $ do
 		(tmpfile, _handle) <- openTempFile tmp (takeFileName file)
+		nukeFile tmpfile
 		createLink file tmpfile
 		return $ KeySource { keyFilename = file , contentLocation = tmpfile }
 

From bb6074dfea4c87037abe096e3b9a0a6d746f5437 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:34:42 -0400
Subject: [PATCH 3794/8313] work around a wrinkle in how lsof handles hard
 links to files that are open elsewhere

+d is probably more expensive, but I need it
---
 Utility/Lsof.hs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs
index 1c0e934fd7..25581cc237 100644
--- a/Utility/Lsof.hs
+++ b/Utility/Lsof.hs
@@ -21,9 +21,14 @@ type CmdLine = String
 data ProcessInfo = ProcessInfo ProcessID CmdLine
 	deriving (Show)
 
-query :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
-query p = do
-	(pid, s) <- pipeFrom "lsof" ["-F0can", "--", p]
+{- Checks each of the files in a directory to find open files.
+ - Note that this will find hard links to files elsewhere that are open. -}
+queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
+queryDir path = query ["+d", path]
+
+query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
+query opts = do
+	(pid, s) <- pipeFrom "lsof" ("-F0can" : opts)
 	let !r = parse s
 	-- ignore nonzero exit code; lsof returns that when no files are open
 	void $ getProcessStatus True False $ processID pid

From 5d63c2a4bbf910c859e873f6e27aef3453992b43 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:35:29 -0400
Subject: [PATCH 3795/8313] check files with lsof in batches before adding

I've tested both cases where this is necessary, and it works great!
A file with multiple writers is not added until the last one closes.
---
 Assistant/Committer.hs | 65 +++++++++++++++++++++++++++++++++---------
 1 file changed, 51 insertions(+), 14 deletions(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index 1348456cca..a2b65dae5a 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -11,9 +11,13 @@ import qualified Annex.Queue
 import qualified Git.Command
 import qualified Command.Add
 import Utility.ThreadScheduler
+import qualified Utility.Lsof as Lsof
+import Types.Backend
 
 import Control.Concurrent.STM
 import Data.Time.Clock
+import Data.Tuple.Utils
+import qualified Data.Set as S
 
 data ChangeType = PendingAddChange | LinkChange | RmChange | RmDirChange
 	deriving (Show, Eq)
@@ -123,26 +127,21 @@ handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
 handleAdds st changechan cs
 	| null toadd = noop
 	| otherwise = do
-		added <- filter id <$> forM toadd go
-		unless (null added) $
-			handleAdds st changechan =<< getChanges changechan
+		toadd' <- safeToAdd st toadd
+		unless (null toadd') $ do
+			added <- filter id <$> forM toadd' add
+			unless (null added) $
+				handleAdds st changechan =<< getChanges changechan
 	where
 		toadd = map changeFile $ filter isPendingAdd cs
 
 		isPendingAdd (Change { changeType = PendingAddChange }) = True
 		isPendingAdd _ = False
 
-		go file = do
-			ms <- catchMaybeIO $ getSymbolicLinkStatus file
-			case ms of
-				Just s
-					| isRegularFile s -> catchBoolIO $
-						runThreadState st $ add file
-				_ -> return False
-
-		add file = do
-			showStart "add" file
-			handle file =<< Command.Add.ingest file
+		add keysource = catchBoolIO $ runThreadState st $ do
+			showStart "add" $ keyFilename keysource
+			handle (keyFilename keysource)
+				=<< Command.Add.ingest keysource
 
 		handle _ Nothing = do
 			showEndFail
@@ -151,3 +150,41 @@ handleAdds st changechan cs
 			Command.Add.link file key True
 			showEndOk
 			return True
+
+{- Checks which of a set of files can safely be added.
+ - Files are locked down as hard links in a temp directory,
+ - with their write bits disabled. But some may have already
+ - been opened for write, so lsof is run on the temp directory
+ - to check them.
+ -}
+safeToAdd :: ThreadState -> [FilePath] -> IO [KeySource]
+safeToAdd st files = do
+	locked <- catMaybes <$> lockdown files
+	runThreadState st $ do
+		tmpdir <- fromRepo gitAnnexTmpDir
+		open <- S.fromList . map fst3 . filter openwrite <$>
+			liftIO (Lsof.queryDir tmpdir)
+		catMaybes <$> forM locked (go open)
+	where
+		go open keysource
+			| S.member (contentLocation keysource) open = do
+				warning $ keyFilename keysource
+					++ " still has writers, not adding"
+				-- remove the hard link
+				--_ <- liftIO $ tryIO $
+				--	removeFile $ contentLocation keysource
+				return Nothing
+			| otherwise = return $ Just keysource
+
+		lockdown = mapM $ \file -> do
+			ms <- catchMaybeIO $ getSymbolicLinkStatus file
+			case ms of
+				Just s
+					| isRegularFile s -> 
+						catchMaybeIO $ runThreadState st $
+							Command.Add.lockDown file
+				_ -> return Nothing
+	
+
+		openwrite (_file, mode, _pid) =
+			mode == Lsof.OpenWriteOnly || mode == Lsof.OpenReadWrite

From bd8319e78ccf13d52ecf14b4f0c86ebf141671ab Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 22:59:32 -0400
Subject: [PATCH 3796/8313] update and blog for the day

the last of the bad bugs is fixed!
---
 doc/design/assistant/blog/day_10__lsof.mdwn | 54 +++++++++++++++
 doc/design/assistant/inotify.mdwn           | 74 ++++++++++-----------
 2 files changed, 91 insertions(+), 37 deletions(-)
 create mode 100644 doc/design/assistant/blog/day_10__lsof.mdwn

diff --git a/doc/design/assistant/blog/day_10__lsof.mdwn b/doc/design/assistant/blog/day_10__lsof.mdwn
new file mode 100644
index 0000000000..32b6705714
--- /dev/null
+++ b/doc/design/assistant/blog/day_10__lsof.mdwn
@@ -0,0 +1,54 @@
+A rather frustrating and long day coding went like this:
+
+## 1-3 pm
+
+Wrote a single function, of which all any Haskell programmer needs to know
+is its type signature:
+
+	Lsof.queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
+
+When I'm spending another hour or two taking a unix utility like lsof and
+parsing its output, which in this case is in a rather complicated
+machine-parsable output format, I often wish unix streams were strongly
+typed, which would avoid this bother.
+
+## 3-9 pm
+
+Six hours spent making it defer annexing files until the commit thread
+wakes up and is about to make a commit. Why did it take so horribly long?
+Well, there were a number of complications, and some really bad bugs
+involving races that were hard to reproduce reliably enough to deal with.
+
+In other words, I was lost in the weeds for a lot of those hours...
+
+At one point, something glorious happened, and it was always making exactly
+one commit for batch mode modifications of a lot of files (like untarring
+them). Unfortunatly, I had to lose that gloriousness due to another
+potential race, which, while unlikely, would have made the program deadlock
+if it happened. 
+
+So, it's back to making 2 or 3 commits per batch mode change. I also have a
+buglet that causes sometimes a second empty commit after a file is added.
+I know why (the inotify event for the symlink gets in late,
+after the commit); will try to improve commit frequency later.
+
+## 9-11 pm
+
+Put the capstone on the day's work, by calling lsof on a directory full
+of hardlinks to the files that are about to be annexed, to check if any
+are still open for write.
+
+This works great! Starting up `git annex watch` when processes have files
+open is no longer a problem, and even if you're evil enough to try having
+muliple processes open the same file, it will complain and not annex it
+until all the writers close it.
+
+(Well, someone really evil could turn the write bit back on after git annex
+clears it, and open the file again, but then really evil people can do
+that to files in `.git/annex/objects` too, and they'll get their just
+deserts when `git annex fsck` runs. So, that's ok..)
+
+----
+
+Anyway, will beat on it more tomorrow, and if all is well, this will finally
+go out to the beta testers.
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index baa420b4e9..0b0eb430c0 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -5,43 +5,6 @@ There is a `watch` branch in git that adds the command.
 
 ## known bugs
 
-* A process has a file open for write, another one closes it,
-  and so it's added. Then the first process modifies it.
-
-  Or, a process has a file open for write when `git annex watch` starts
-  up, it will be added to the annex. If the process later continues
-  writing, it will change content in the annex.
-
-  This changes content in the annex, and fsck will later catch
-  the inconsistency.
-
-  Possible fixes: 
-
-  * Somehow track or detect if a file is open for write by any processes.
-    `lsof` could be used, although it would be a little slow.
-
-    Here's one way to avoid the slowdown: When a file is being added,
-    set it read-only, and hard-link it into a quarantine directory,
-    remembering both filenames.
-    Then use the batch change mode code to detect batch adds and bundle
-    them together.
-    Just before committing, lsof the quarantine directory. Any files in
-    it that are still open for write can just have their write bit turned
-    back on and be deleted from quarantine, to be handled when their writer
-    closes. Files that pass quarantine get added as usual. This avoids
-    repeated lsof calls slowing down adds, but does add a constant factor
-    overhead (0.25 seconds lsof call) before any add gets committed.
-
-  * Or, when possible, making a copy on write copy before adding the file
-    would avoid this.
-  * Or, as a last resort, make an expensive copy of the file and add that.
-  * Tracking file opens and closes with inotify could tell if any other
-    processes have the file open. But there are problems.. It doesn't
-    seem to differentiate between files opened for read and for write.
-    And there would still be a race after the last close and before it's
-    injected into the annex, where it could be opened for write again.
-    Would need to detect that and undo the annex injection or something.
-
 * If a file is checked into git as a normal file and gets modified
   (or merged, etc), it will be converted into an annexed file.
   See [[blog/day_7__bugfixes]]
@@ -140,3 +103,40 @@ Many races need to be dealt with by this code. Here are some of them.
 - coleasce related add/rm events for speed and less disk IO **done**
 - don't annex `.gitignore` and `.gitattributes` files **done**
 - run as a daemon **done**
+- A process has a file open for write, another one closes it,
+  and so it's added. Then the first process modifies it.
+
+  Or, a process has a file open for write when `git annex watch` starts
+  up, it will be added to the annex. If the process later continues
+  writing, it will change content in the annex.
+
+  This changes content in the annex, and fsck will later catch
+  the inconsistency.
+
+  Possible fixes: 
+
+  * Somehow track or detect if a file is open for write by any processes.
+    `lsof` could be used, although it would be a little slow.
+
+    Here's one way to avoid the slowdown: When a file is being added,
+    set it read-only, and hard-link it into a quarantine directory,
+    remembering both filenames.
+    Then use the batch change mode code to detect batch adds and bundle
+    them together.
+    Just before committing, lsof the quarantine directory. Any files in
+    it that are still open for write can just have their write bit turned
+    back on and be deleted from quarantine, to be handled when their writer
+    closes. Files that pass quarantine get added as usual. This avoids
+    repeated lsof calls slowing down adds, but does add a constant factor
+    overhead (0.25 seconds lsof call) before any add gets committed. **done**
+
+  * Or, when possible, making a copy on write copy before adding the file
+    would avoid this.
+  * Or, as a last resort, make an expensive copy of the file and add that.
+  * Tracking file opens and closes with inotify could tell if any other
+    processes have the file open. But there are problems.. It doesn't
+    seem to differentiate between files opened for read and for write.
+    And there would still be a race after the last close and before it's
+    injected into the annex, where it could be opened for write again.
+    Would need to detect that and undo the annex injection or something.
+

From eaf4cbec541673c25cfbb2104eb67ac5818de1b1 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sat, 16 Jun 2012 03:26:37 +0000
Subject: [PATCH 3797/8313] Added a comment

---
 .../comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment

diff --git a/doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment b/doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment
new file mode 100644
index 0000000000..db6f90d881
--- /dev/null
+++ b/doc/forum/exporting_annexed_files/comment_3_86f0e0f767a84a0f583e121d36cb7d48._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.6.135"
+ subject="comment 3"
+ date="2012-06-16T03:26:37Z"
+ content="""
+That nautilous behavior is a bad thing when trying to export files out, but it's a good thing when just moving files around inside your repository...
+"""]]

From 0052cec2b7e39652bd52cd4978ed47d2daa5af68 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 15 Jun 2012 23:24:01 -0400
Subject: [PATCH 3798/8313] add lsof build deps

Check for it in configure; and add a --force option for people without it
who want to live dangerously.
---
 Assistant.hs       | 21 ++++++++++++++++++++-
 Build/Configure.hs |  1 +
 Utility/Lsof.hs    |  7 ++++++-
 debian/control     |  4 +++-
 doc/install.mdwn   |  2 ++
 5 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index b72f9a7e7d..4042c6ede5 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -49,8 +49,10 @@ import Assistant.DaemonStatus
 import Assistant.Watcher
 import Assistant.Committer
 import Assistant.SanityChecker
+import qualified Annex
 import qualified Utility.Daemon
 import Utility.LogFile
+import qualified Build.SysConfig as SysConfig
 
 import Control.Concurrent
 
@@ -64,7 +66,12 @@ startDaemon foreground
 		pidfile <- fromRepo gitAnnexPidFile
 		go $ Utility.Daemon.daemonize logfd (Just pidfile) False
 	where
-		go a = withThreadState $ \st -> do
+		go a
+			| SysConfig.lsof = start a
+			| otherwise =
+				ifM (Annex.getState Annex.force)
+					(start a, needlsof)
+		start a = withThreadState $ \st -> do
 			dstatus <- startDaemonStatus
 			liftIO $ a $ do
 				changechan <- newChangeChan
@@ -78,5 +85,17 @@ startDaemon foreground
 				_ <- forkIO $ sanityCheckerThread st dstatus changechan
 				watchThread st dstatus changechan
 
+		-- this message is optimised away when lsof is available
+		needlsof = error $ unlines
+			[ "The lsof command is needed for watch mode to be safe."
+			, "But this build of git-annex was made without lsof available. Giving up..."
+			, ""
+			, "You can use --force if lsof is available now. Please make very sure it is."
+			, "If run with --force and without lsof available, files can be added to the"
+			, "annex while a process still has them opened for writing. This can"
+			, "corrupt data in the annex, and make fsck complain."
+			, "Use the --force with caution, Luke!"
+			]
+
 stopDaemon :: Annex ()
 stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
diff --git a/Build/Configure.hs b/Build/Configure.hs
index 2f79297ee9..7af53cf10f 100644
--- a/Build/Configure.hs
+++ b/Build/Configure.hs
@@ -26,6 +26,7 @@ tests =
 	, TestCase "wget" $ testCmd "wget" "wget --version >/dev/null"
 	, TestCase "bup" $ testCmd "bup" "bup --version >/dev/null"
 	, TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null"
+	, TestCase "lsof" $ testCmd "lsof" "lsof -v >/dev/null 2>&1"
 	, TestCase "ssh connection caching" getSshConnectionCaching
 	] ++ shaTestCases False [1, 512, 224, 384] ++ shaTestCases True [256]
 
diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs
index 25581cc237..0061dfe574 100644
--- a/Utility/Lsof.hs
+++ b/Utility/Lsof.hs
@@ -26,11 +26,16 @@ data ProcessInfo = ProcessInfo ProcessID CmdLine
 queryDir :: FilePath -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
 queryDir path = query ["+d", path]
 
+{- Runs lsof with some parameters.
+ -
+ - Ignores nonzero exit code; lsof returns that when no files are open.
+ -
+ - Note: If lsof is not available, this always returns [] !
+ -}
 query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)]
 query opts = do
 	(pid, s) <- pipeFrom "lsof" ("-F0can" : opts)
 	let !r = parse s
-	-- ignore nonzero exit code; lsof returns that when no files are open
 	void $ getProcessStatus True False $ processID pid
 	return r
 
diff --git a/debian/control b/debian/control
index 6534fef317..c587c29ec2 100644
--- a/debian/control
+++ b/debian/control
@@ -28,6 +28,7 @@ Build-Depends:
 	uuid,
 	rsync,
 	openssh-client,
+	lsof,
 Maintainer: Joey Hess 
 Standards-Version: 3.9.3
 Vcs-Git: git://git.kitenet.net/git-annex
@@ -41,7 +42,8 @@ Depends: ${misc:Depends}, ${shlibs:Depends},
 	uuid,
 	rsync,
 	wget | curl,
-	openssh-client (>= 1:5.6p1)
+	openssh-client (>= 1:5.6p1),
+	lsof
 Suggests: graphviz, bup, gnupg
 Description: manage files with git, without checking their contents into git
  git-annex allows managing files with git, without checking the file
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 471f95c1fa..43b5bba20d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -54,6 +54,8 @@ To build and use git-annex, you will need:
   * [sha1sum](ftp://ftp.gnu.org/gnu/coreutils/) (optional, but recommended;
     a sha1 command will also do)
   * [gpg](http://gnupg.org/) (optional; needed for encryption)
+  * [lsof](ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/)
+    (needed for watch mode)
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`

From 7d6329e5e4e4dd488b24e0002edcb2220073d8e9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sat, 16 Jun 2012 02:49:26 -0400
Subject: [PATCH 3799/8313] check lsof at runtime

---
 Assistant.hs     | 24 +++++++++---------------
 debian/control   |  1 -
 doc/install.mdwn |  2 +-
 3 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 4042c6ede5..54556616df 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -52,7 +52,6 @@ import Assistant.SanityChecker
 import qualified Annex
 import qualified Utility.Daemon
 import Utility.LogFile
-import qualified Build.SysConfig as SysConfig
 
 import Control.Concurrent
 
@@ -66,11 +65,11 @@ startDaemon foreground
 		pidfile <- fromRepo gitAnnexPidFile
 		go $ Utility.Daemon.daemonize logfd (Just pidfile) False
 	where
-		go a
-			| SysConfig.lsof = start a
-			| otherwise =
-				ifM (Annex.getState Annex.force)
-					(start a, needlsof)
+		go a = ifM (liftIO $ inPath "lsof")
+			( go a
+			, ifM (Annex.getState Annex.force)
+				(start a, needlsof)	
+			)
 		start a = withThreadState $ \st -> do
 			dstatus <- startDaemonStatus
 			liftIO $ a $ do
@@ -85,16 +84,11 @@ startDaemon foreground
 				_ <- forkIO $ sanityCheckerThread st dstatus changechan
 				watchThread st dstatus changechan
 
-		-- this message is optimised away when lsof is available
 		needlsof = error $ unlines
-			[ "The lsof command is needed for watch mode to be safe."
-			, "But this build of git-annex was made without lsof available. Giving up..."
-			, ""
-			, "You can use --force if lsof is available now. Please make very sure it is."
-			, "If run with --force and without lsof available, files can be added to the"
-			, "annex while a process still has them opened for writing. This can"
-			, "corrupt data in the annex, and make fsck complain."
-			, "Use the --force with caution, Luke!"
+			[ "The lsof command is needed for watch mode to be safe, and is not in PATH."
+			, "To override lsof checks to ensure that files are not open for writing"
+			, "when added to the annex, you can use --force"
+			, "Be warned: This can corrupt data in the annex, and make fsck complain."
 			]
 
 stopDaemon :: Annex ()
diff --git a/debian/control b/debian/control
index c587c29ec2..6741ef9433 100644
--- a/debian/control
+++ b/debian/control
@@ -28,7 +28,6 @@ Build-Depends:
 	uuid,
 	rsync,
 	openssh-client,
-	lsof,
 Maintainer: Joey Hess 
 Standards-Version: 3.9.3
 Vcs-Git: git://git.kitenet.net/git-annex
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 43b5bba20d..54b52d4162 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -55,7 +55,7 @@ To build and use git-annex, you will need:
     a sha1 command will also do)
   * [gpg](http://gnupg.org/) (optional; needed for encryption)
   * [lsof](ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/)
-    (needed for watch mode)
+    (optional; recommended for watch mode)
   * [ikiwiki](http://ikiwiki.info) (optional; used to build the docs)
 
 Then just [[download]] git-annex and run: `make; make install`

From 2a9c5ebfaa16799c2d45555be6414f4a45b893e7 Mon Sep 17 00:00:00 2001
From: "http://dieter-be.myopenid.com/" 
Date: Sat, 16 Jun 2012 09:14:27 +0000
Subject: [PATCH 3800/8313] Added a comment

---
 .../comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment

diff --git a/doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment b/doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment
new file mode 100644
index 0000000000..9d970da22e
--- /dev/null
+++ b/doc/design/assistant/blog/day_10__lsof/comment_1_9b8c28c85c979f32e5c295b6a03c048e._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://dieter-be.myopenid.com/"
+ nickname="dieter"
+ subject="comment 1"
+ date="2012-06-16T09:14:26Z"
+ content="""
+maybe at some point, your tool could show \"warning, the following files are still open and are hence not being annexed\"
+to avoid any nasty surprises of a file not being annexed and the user not realizing it.
+"""]]

From 0b7047f7500734e7d6297f9e36f22873f323718d Mon Sep 17 00:00:00 2001
From: Ben Gamari 
Date: Sat, 16 Jun 2012 13:59:20 +0200
Subject: [PATCH 3801/8313] stm package name is lowercase

---
 git-annex.cabal | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index b02a9af4db..0fc4abaeb7 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -35,7 +35,7 @@ Executable git-annex
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, STM
+   hinotify, stm
   -- Need to list this because it's generated from a .hsc file.
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
@@ -53,7 +53,7 @@ Test-Suite test
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, STM
+   hinotify, stm
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP

From 9356f11c6077f61e6bda9758f12b3f76b68991e3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 01:25:48 -0400
Subject: [PATCH 3802/8313] surveyed the OSX and BSD options for file
 monitoring

---
 doc/design/assistant/inotify.mdwn | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index baa420b4e9..02c30752dc 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -3,6 +3,8 @@ inotify for changes, and automatically annexing new files, etc.
 
 There is a `watch` branch in git that adds the command.
 
+[[!toc]]
+
 ## known bugs
 
 * A process has a file open for write, another one closes it,
@@ -48,6 +50,35 @@ There is a `watch` branch in git that adds the command.
 
 * When you `git annex unlock` a file, it will immediately be re-locked.
 
+## beyond Linux
+
+I'd also like to support OSX and if possible the BSDs.
+
+* kqueue ([haskell bindings](http://hackage.haskell.org/package/kqueue))
+  is supported by FreeBSD, OSX, and other BSDs.
+
+  From what I can find, kqueue does not provide full directory watching
+  capabilities. To watch a file, you have to have an open file descriptor
+  to the file. This wouldn't scale.
+
+  Gamin does the best it can with just kqueue, supplimented by polling.
+  The source file `server/gam_kqueue.c` makes for interesting reading.
+  Using gamin to do the heavy lifting is one option. 
+  ([haskell bindings](http://hackage.haskell.org/package/hlibfam) for FAM;
+  gamin shares the API)
+
+* hfsevents ([haskell bindings](http://hackage.haskell.org/package/hfsevents))
+  is OSX specific.
+
+  Originally it was only directory level, and you were only told a
+  directory had changed and not which file. Based on the haskell
+  binding's code, from OSX 10.7.0, file level events were added.
+
+  This will be harder for me to develop for, since I don't have access to
+  OSX machines..
+
+* Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
+
 ## todo
 
 - Support OSes other than Linux; it only uses inotify currently.

From da261b31de349f99e518f590017f3a10078bc6dd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 01:25:48 -0400
Subject: [PATCH 3803/8313] surveyed the OSX and BSD options for file
 monitoring

---
 doc/design/assistant/inotify.mdwn | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 0b0eb430c0..2cd6654488 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -3,6 +3,8 @@ inotify for changes, and automatically annexing new files, etc.
 
 There is a `watch` branch in git that adds the command.
 
+[[!toc]]
+
 ## known bugs
 
 * If a file is checked into git as a normal file and gets modified
@@ -11,6 +13,35 @@ There is a `watch` branch in git that adds the command.
 
 * When you `git annex unlock` a file, it will immediately be re-locked.
 
+## beyond Linux
+
+I'd also like to support OSX and if possible the BSDs.
+
+* kqueue ([haskell bindings](http://hackage.haskell.org/package/kqueue))
+  is supported by FreeBSD, OSX, and other BSDs.
+
+  From what I can find, kqueue does not provide full directory watching
+  capabilities. To watch a file, you have to have an open file descriptor
+  to the file. This wouldn't scale.
+
+  Gamin does the best it can with just kqueue, supplimented by polling.
+  The source file `server/gam_kqueue.c` makes for interesting reading.
+  Using gamin to do the heavy lifting is one option. 
+  ([haskell bindings](http://hackage.haskell.org/package/hlibfam) for FAM;
+  gamin shares the API)
+
+* hfsevents ([haskell bindings](http://hackage.haskell.org/package/hfsevents))
+  is OSX specific.
+
+  Originally it was only directory level, and you were only told a
+  directory had changed and not which file. Based on the haskell
+  binding's code, from OSX 10.7.0, file level events were added.
+
+  This will be harder for me to develop for, since I don't have access to
+  OSX machines..
+
+* Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
+
 ## todo
 
 - Support OSes other than Linux; it only uses inotify currently.

From 31c15aa9b9285733c6874bf5c7a89fccdd01b5d2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 01:34:10 -0400
Subject: [PATCH 3804/8313] update

---
 doc/design/assistant/inotify.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 2cd6654488..e549c4d9d9 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -40,6 +40,9 @@ I'd also like to support OSX and if possible the BSDs.
   This will be harder for me to develop for, since I don't have access to
   OSX machines..
 
+  [This perl module](http://search.cpan.org/~agrundma/Mac-FSEvents-0.02/lib/Mac/FSEvents/Event.pm)
+  has the best description I've found of what the flags mean.
+
 * Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
 
 ## todo

From ec197feec062c59760a931aafb5d3087b921999a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 02:08:28 -0400
Subject: [PATCH 3805/8313] update

---
 doc/design/assistant/inotify.mdwn | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index e549c4d9d9..cae6298094 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -20,9 +20,12 @@ I'd also like to support OSX and if possible the BSDs.
 * kqueue ([haskell bindings](http://hackage.haskell.org/package/kqueue))
   is supported by FreeBSD, OSX, and other BSDs.
 
-  From what I can find, kqueue does not provide full directory watching
-  capabilities. To watch a file, you have to have an open file descriptor
-  to the file. This wouldn't scale.
+  In kqueue, to watch for changes to a file, you have to have an open file
+  descriptor to the file. This wouldn't scale. 
+
+  Apparently, a directory can be watched, and events are generated when
+  files are added/removed from it. You then have to scan to find which
+  files changed. [example](https://developer.apple.com/library/mac/#samplecode/FileNotification/Listings/Main_c.html#//apple_ref/doc/uid/DTS10003143-Main_c-DontLinkElementID_3)
 
   Gamin does the best it can with just kqueue, supplimented by polling.
   The source file `server/gam_kqueue.c` makes for interesting reading.
@@ -30,6 +33,12 @@ I'd also like to support OSX and if possible the BSDs.
   ([haskell bindings](http://hackage.haskell.org/package/hlibfam) for FAM;
   gamin shares the API)
 
+  kqueue does not seem to provide a way to tell when a file gets closed,
+  only when it's initially created. Poses problems..
+
+  * [man page](http://www.freebsd.org/cgi/man.cgi?query=kqueue&apropos=0&sektion=0&format=html)
+  *  (good example program)
+
 * hfsevents ([haskell bindings](http://hackage.haskell.org/package/hfsevents))
   is OSX specific.
 
@@ -40,8 +49,12 @@ I'd also like to support OSX and if possible the BSDs.
   This will be harder for me to develop for, since I don't have access to
   OSX machines..
 
-  [This perl module](http://search.cpan.org/~agrundma/Mac-FSEvents-0.02/lib/Mac/FSEvents/Event.pm)
-  has the best description I've found of what the flags mean.
+  hfsevents does not seem to provide a way to tell when a file gets closed,
+  only when it's initially created. Poses problems..
+
+  * 
+  *  (good example program)
+  *  (good example program)
 
 * Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
 

From bdf66c589ac537142d9e490b562467fc1dfc2af9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Sun, 17 Jun 2012 08:52:33 +0000
Subject: [PATCH 3806/8313] Added a comment

---
 .../comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment

diff --git a/doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment b/doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment
new file mode 100644
index 0000000000..c1f22c4b8a
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_3_b346e870c1cd80e4b0a313c3a9fed6b3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 3"
+ date="2012-06-17T08:52:32Z"
+ content="""
+In relation to OSX support, hfsevents (or supporting hfs is probably a bad idea), its very osx specific and users who are moving usb keys and disks between systems will probably end up using fat32/exfat/vfat disks around. Also if you want I can lower the turn around time for the OSX auto-builder that I have setup to every 1 or 2mins? would that help?
+"""]]

From 1a6f7ae00bd17aed180581c7acec25af0c80c9eb Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Sun, 17 Jun 2012 16:39:43 +0000
Subject: [PATCH 3807/8313] Added a comment

---
 ...mment_4_32be58b4c3b17a4ea539690d2fb45159._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment

diff --git a/doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment b/doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment
new file mode 100644
index 0000000000..d854d91c2b
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_4_32be58b4c3b17a4ea539690d2fb45159._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.2.147"
+ subject="comment 4"
+ date="2012-06-17T16:39:43Z"
+ content="""
+hfsevents seems usable, git-annex does not need to watch for file changes on remotes on other media.
+
+But, trying kqueue first.
+
+You could perhaps run the autobuilder on a per-commit basis..
+"""]]

From e84b78f40c5ae20031c1d5cdde524081774de656 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 14:02:40 -0400
Subject: [PATCH 3808/8313] reorg

---
 Utility/ThreadLock.hs      | 16 ----------------
 Utility/ThreadScheduler.hs | 15 +++++++++++++++
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/Utility/ThreadLock.hs b/Utility/ThreadLock.hs
index 4285c0ec54..c029a2b0c8 100644
--- a/Utility/ThreadLock.hs
+++ b/Utility/ThreadLock.hs
@@ -7,11 +7,7 @@
 
 module Utility.ThreadLock where
 
-import Common
-
-import System.Posix.Terminal
 import Control.Concurrent.MVar
-import System.Posix.Signals
 
 type Lock = MVar ()
 
@@ -21,15 +17,3 @@ newLock = newMVar ()
 {- Runs an action with a lock held, so only one thread at a time can run it. -}
 withLock :: Lock -> IO a -> IO a
 withLock lock = withMVar lock . const
-
-{- Pauses the main thread, letting children run until program termination. -}
-waitForTermination :: IO ()
-waitForTermination = do
-	lock <- newEmptyMVar
-	check softwareTermination lock
-	whenM (queryTerminal stdInput) $
-		check keyboardSignal lock
-	takeMVar lock
-	where
-		check sig lock = void $
-			installHandler sig (CatchOnce $ putMVar lock ()) Nothing
diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs
index 9204cd9b9e..6557398fd7 100644
--- a/Utility/ThreadScheduler.hs
+++ b/Utility/ThreadScheduler.hs
@@ -9,7 +9,10 @@
 module Utility.ThreadScheduler where
 
 import Common
+
 import Control.Concurrent
+import System.Posix.Terminal
+import System.Posix.Signals
 
 newtype Seconds = Seconds { fromSeconds :: Int }
 	deriving (Eq, Ord, Show)
@@ -40,3 +43,15 @@ unboundDelay time = do
 	let maxWait = min time $ toInteger (maxBound :: Int)
 	threadDelay $ fromInteger maxWait
 	when (maxWait /= time) $ unboundDelay (time - maxWait)
+
+{- Pauses the main thread, letting children run until program termination. -}
+waitForTermination :: IO ()
+waitForTermination = do
+	lock <- newEmptyMVar
+	check softwareTermination lock
+	whenM (queryTerminal stdInput) $
+		check keyboardSignal lock
+	takeMVar lock
+	where
+		check sig lock = void $
+			installHandler sig (CatchOnce $ putMVar lock ()) Nothing

From 1863185693f407a35f1b9e92d402e43409966928 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 14:02:58 -0400
Subject: [PATCH 3809/8313] startup check fixes

Move lsof check, and display a message before daemon startup if on an
unsupported OS.
---
 Assistant.hs         | 16 ++--------------
 Assistant/Watcher.hs | 22 ++++++++++++++++++++--
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 54556616df..f5acc2ef8a 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -49,7 +49,6 @@ import Assistant.DaemonStatus
 import Assistant.Watcher
 import Assistant.Committer
 import Assistant.SanityChecker
-import qualified Annex
 import qualified Utility.Daemon
 import Utility.LogFile
 
@@ -65,12 +64,8 @@ startDaemon foreground
 		pidfile <- fromRepo gitAnnexPidFile
 		go $ Utility.Daemon.daemonize logfd (Just pidfile) False
 	where
-		go a = ifM (liftIO $ inPath "lsof")
-			( go a
-			, ifM (Annex.getState Annex.force)
-				(start a, needlsof)	
-			)
-		start a = withThreadState $ \st -> do
+		go a = withThreadState $ \st -> do
+			checkCanWatch
 			dstatus <- startDaemonStatus
 			liftIO $ a $ do
 				changechan <- newChangeChan
@@ -84,12 +79,5 @@ startDaemon foreground
 				_ <- forkIO $ sanityCheckerThread st dstatus changechan
 				watchThread st dstatus changechan
 
-		needlsof = error $ unlines
-			[ "The lsof command is needed for watch mode to be safe, and is not in PATH."
-			, "To override lsof checks to ensure that files are not open for writing"
-			, "when added to the annex, you can use --force"
-			, "Be warned: This can corrupt data in the annex, and make fsck complain."
-			]
-
 stopDaemon :: Annex ()
 stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 4aac33fd1f..5af39ea88f 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -13,13 +13,14 @@ import Common.Annex
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.Committer
-import Utility.ThreadLock
+import Utility.ThreadScheduler
 import qualified Annex.Queue
 import qualified Git.Command
 import qualified Git.UpdateIndex
 import qualified Git.HashObject
 import qualified Git.LsFiles
 import qualified Backend
+import qualified Annex
 import Annex.Content
 import Annex.CatFile
 import Git.Types
@@ -35,6 +36,23 @@ import System.INotify
 
 type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Maybe Change)
 
+checkCanWatch :: Annex ()
+checkCanWatch = do
+#if defined linux_HOST_OS
+	unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) $
+		needLsof
+#else
+		error "watch mode is currently only available in Linux"
+#endif
+
+needLsof :: Annex ()
+needLsof = error $ unlines
+	[ "The lsof command is needed for watch mode to be safe, and is not in PATH."
+	, "To override lsof checks to ensure that files are not open for writing"
+	, "when added to the annex, you can use --force"
+	, "Be warned: This can corrupt data in the annex, and make fsck complain."
+	]
+
 watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
 #if defined linux_HOST_OS
 watchThread st dstatus changechan = withINotify $ \i -> do
@@ -61,7 +79,7 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			, errHook = hook onErr
 			}
 #else
-watchThread = error "so far only available on Linux"
+watchThread = undefined
 #endif
 
 ignored :: FilePath -> Bool

From c373f6e9546d615fb7c3f2c77a35136c9ccf654a Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 14:25:02 -0400
Subject: [PATCH 3810/8313] note

---
 Assistant.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Assistant.hs b/Assistant.hs
index f5acc2ef8a..880d3eb5e3 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -77,6 +77,7 @@ startDaemon foreground
 				_ <- forkIO $ commitThread st changechan
 				_ <- forkIO $ daemonStatusThread st dstatus
 				_ <- forkIO $ sanityCheckerThread st dstatus changechan
+				-- Does not return.
 				watchThread st dstatus changechan
 
 stopDaemon :: Annex ()

From 91567ab8f6fae75d1590bfd05567e84157887c4b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 17:15:56 -0400
Subject: [PATCH 3811/8313] make inotify a build flag etc

---
 Assistant/Watcher.hs | 11 +++++++----
 Makefile             |  2 +-
 debian/control       |  2 +-
 doc/install.mdwn     |  3 ++-
 git-annex.cabal      | 15 ++++++++++-----
 5 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 5af39ea88f..1d35b5c1e3 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -29,7 +29,7 @@ import Control.Concurrent.STM
 import Data.Bits.Utils
 import qualified Data.ByteString.Lazy as L
 
-#if defined linux_HOST_OS
+#ifdef WITH_INOTIFY
 import Utility.Inotify
 import System.INotify
 #endif
@@ -38,11 +38,14 @@ type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Mayb
 
 checkCanWatch :: Annex ()
 checkCanWatch = do
-#if defined linux_HOST_OS
+#ifdef WITH_INOTIFY
 	unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) $
 		needLsof
 #else
-		error "watch mode is currently only available in Linux"
+#if defined linux_HOST_OS
+#warning "Building without inotify support; watch mode will be disabled."
+#endif
+	error "watch mode is not available on this system"
 #endif
 
 needLsof :: Annex ()
@@ -54,7 +57,7 @@ needLsof = error $ unlines
 	]
 
 watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
-#if defined linux_HOST_OS
+#ifdef WITH_INOTIFY
 watchThread st dstatus changechan = withINotify $ \i -> do
 	runThreadState st $
 		showAction "scanning"
diff --git a/Makefile b/Makefile
index 023ea5d5e1..6d36e8b8bf 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd -ignore-package monads-tf
-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 -DWITH_INOTIFY
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE
diff --git a/debian/control b/debian/control
index 6741ef9433..3b142dc5f2 100644
--- a/debian/control
+++ b/debian/control
@@ -20,7 +20,7 @@ Build-Depends:
 	libghc-ifelse-dev,
 	libghc-bloomfilter-dev,
 	libghc-edit-distance-dev,
-	libghc-hinotify-dev,
+	libghc-hinotify-dev [linux-any],
 	libghc-stm-dev,
 	ikiwiki,
 	perlmagick,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 54b52d4162..a009ee00d3 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -43,7 +43,8 @@ To build and use git-annex, you will need:
   * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
   * [edit-distance](http://hackage.haskell.org/package/edit-distance)
   * [stm](http://hackage.haskell.org/package/stm)
-  * [hinotify](http://hackage.haskell.org/package/hinotify) (on Linux only)
+  * [hinotify](http://hackage.haskell.org/package/hinotify)
+    (optional; Linux only)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 0fc4abaeb7..1416a381dd 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120614
+Version: 3.20120616
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess 
@@ -28,14 +28,16 @@ Description:
 Flag S3
   Description: Enable S3 support
 
+Flag Inotify
+  Description: Enable inotify support
+
 Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: MissingH, hslogger, directory, filepath,
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, stm
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, stm
   -- Need to list this because it's generated from a .hsc file.
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
@@ -45,6 +47,10 @@ Executable git-annex
     Build-Depends: hS3
     CPP-Options: -DWITH_S3
 
+  if flag(Inotify)
+    Build-Depends: hinotify
+    CPP-Options: -DWITH_INOTIFY
+
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs
@@ -52,8 +58,7 @@ Test-Suite test
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance,
-   hinotify, stm
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, stm
   Other-Modules: Utility.Touch
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP

From 9afdf7c954a4e630cf1d13ea780fd1bb72fdb533 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 17:24:07 -0400
Subject: [PATCH 3812/8313] blog for the day

---
 .../assistant/blog/day_11__freebsd.mdwn       | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_11__freebsd.mdwn

diff --git a/doc/design/assistant/blog/day_11__freebsd.mdwn b/doc/design/assistant/blog/day_11__freebsd.mdwn
new file mode 100644
index 0000000000..92a3ef2896
--- /dev/null
+++ b/doc/design/assistant/blog/day_11__freebsd.mdwn
@@ -0,0 +1,50 @@
+I've been investigating how to make `git annex watch` work on
+FreeBSD, and by extension, OSX.
+
+One option is kqueue, which works on both operating systems, and allows
+very basic monitoring of file changes. There's also an OSX specific
+hfsevents interface.
+
+Kqueue is far from optimal for `git annex watch`, because it provides even
+less information than inotify (which didn't really provide everything I
+needed, thus the lsof hack). Kqueue doesn't have events for files being
+closed, only an event when a file is created. So it will be difficult for
+`git annex watch` to know when a file is done being written to and can be
+annexed. git annex will probably need to run lsof periodically to check when
+recently added files are complete. (hsevents shares this limitation)
+
+Kqueue also doesn't provide specific events when a file or directory is
+moved. Indeed, it doesn't provide specific events about what changed at
+all. All you get with kqueue is a generic "oh hey, the directory you're
+watching changed in some way", and it's up to you to scan it to work out
+how. So git annex will probably need to run `git ls-tree --others`
+to find changes in the directory tree. This could be expensive with large
+trees. (hsevents has per-file events on current versions of OSX)
+
+Despite these warts, I want to try kqueue first, since it's more portable
+than hfsevents, and will surely be easier for me to develop support for,
+since I don't have direct access to OSX.
+
+So I went to a handy Debian kFreeBSD porter box, and tried some kqueue
+stuff to get a feel for it. I got a python program that does basic
+directory monitoring with kqueue to work, so I know it's usable there.
+
+Next step was getting kqueue working from Haskell. Should be easy, there's
+a Haskell library already. I spent a while trying to get it to work on
+Debian kFreeBSD, but ran into a
+[problem](https://github.com/hesselink/kqueue/issues/1) that could be
+caused by the Debian kFreeBSD being different, or just a bug in the Haskell
+library. I didn't want to spend too long shaving this yak; I might install
+"real" FreeBSD on a spare laptop and try to get it working there instead.
+
+But for now, I've dropped down to C instead, and have a simple C program
+that can monitor a directory with kqueue. Next I'll turn it into a simple
+library, which can easily be linked into my Haskell code. The Haskell code
+will pass it a set of open directory descriptors, and it'll return the
+one that it gets an event on. This is necessary because kqueue doesn't
+recurse into subdirectories on its own.
+
+I've generally had good luck with this approach to adding stuff in Haskell;
+rather than writing a bit-banging and structure packing low level interface
+in Haskell, write it in C, with a simpler interface between C and
+Haskell.

From 66344a3613cf12720585f9afa98d2422b699b9c8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Sun, 17 Jun 2012 18:10:57 -0400
Subject: [PATCH 3813/8313] Enable diskfree on kfreebsd, using statvfs.

Could not reproduce the build failure I had seen related to this,
but the numbers were wrong with statfs64. Probably pulling from the wrong
place in the structure. statvfs seems to work..
---
 Utility/libdiskfree.c | 14 +++-----------
 debian/changelog      |  1 +
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/Utility/libdiskfree.c b/Utility/libdiskfree.c
index a37cb75713..5e84d4bded 100644
--- a/Utility/libdiskfree.c
+++ b/Utility/libdiskfree.c
@@ -22,16 +22,9 @@
 # define STATCALL statfs /* statfs64 not yet tested on a real FreeBSD machine */
 # define STATSTRUCT statfs
 #else
-#if defined (__FreeBSD_kernel__) /* Debian kFreeBSD */
-# include 
-# include 
-# define STATCALL statfs64
-# define STATSTRUCT statfs
-# warning free space checking code temporarily disabled due to build failure
-# define UNKNOWN
-#else
-#if defined (__linux__)
-/* This is a POSIX standard, so might also work elsewhere. */
+#if defined (__linux__) || defined (__FreeBSD_kernel__)
+/* Linux or Debian kFreeBSD */
+/* This is a POSIX standard, so might also work elsewhere too. */
 # include 
 # define STATCALL statvfs
 # define STATSTRUCT statvfs
@@ -41,7 +34,6 @@
 #endif
 #endif
 #endif
-#endif
 
 #include 
 #include 
diff --git a/debian/changelog b/debian/changelog
index 366854156b..9a47447ced 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ git-annex (3.20120616) UNRELEASED; urgency=low
   * watch: New subcommand, which uses inotify to watch for changes to
     files and automatically annexes new files, etc, so you don't need
     to manually run git commands when manipulating files.
+  * Enable diskfree on kfreebsd, using statvfs.
 
  -- Joey Hess   Tue, 12 Jun 2012 11:35:59 -0400
 

From 123df55d614c55d48b14b8152dd14d1548bf00d6 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 18 Jun 2012 08:24:24 +0000
Subject: [PATCH 3814/8313]

---
 ...relates_to_the_new_inotify_flag__41__.mdwn | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn

diff --git a/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn b/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn
new file mode 100644
index 0000000000..9805478f53
--- /dev/null
+++ b/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn
@@ -0,0 +1,34 @@
+Since the watch branch is now merged into master, it doesn't quite build cleanly on non-linux systems anymore. So here's a small change to the Makefile to work around the problem for now.
+
+
+From 707cb47744775c324060febe11987db5f10ed9ff Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Mon, 18 Jun 2012 09:20:35 +0100
+Subject: [PATCH] Teach _Makefile_ to only do _-DWITH_INOTIFY_ when on a Linux
+ machine.
+
+---
+ Makefile |    8 +++++++-
+ 1 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index 6d36e8b..8884b5c 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,6 +1,12 @@
++OS:=$(shell uname | sed 's/[-_].*//')
++
++ifeq ($(OS),Linux)
++BASEFLAGS_OPTS+=-DWITH_INOTIFY
++endif
++
+ PREFIX=/usr
+ IGNORE=-ignore-package monads-fd -ignore-package monads-tf
+-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 -DWITH_INOTIFY
++BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 $(BASEFLAGS_OPTS)
+ GHCFLAGS=-O2 $(BASEFLAGS)
+ 
+ ifdef PROFILE
+-- 
+1.7.10.4
+


From 762b67c163b4e6d836caa09e0b7ad6404d57b615 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Mon, 18 Jun 2012 08:39:42 +0000
Subject: [PATCH 3815/8313]

---
 doc/install/ScientificLinux5.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/install/ScientificLinux5.mdwn b/doc/install/ScientificLinux5.mdwn
index 4292813701..15ffb928b1 100644
--- a/doc/install/ScientificLinux5.mdwn
+++ b/doc/install/ScientificLinux5.mdwn
@@ -68,3 +68,5 @@ system yet, but I would assume something similar to the above steps
 would be required to do so.
 
 The above is almost a cut and paste of , the above could probably be refined, it was what worked for me on SL5. Please feel free to re-edit and chop out or add useless bits of text in the above!
+
+Note: from the minor testing, it appears the compiled binaries from SL5 will work on SL6.

From 89f6f820bf4aefada124219ff53509db0c85bee7 Mon Sep 17 00:00:00 2001
From: Jimmy Tang 
Date: Mon, 18 Jun 2012 09:20:35 +0100
Subject: [PATCH 3816/8313] Teach _Makefile_ to only do _-DWITH_INOTIFY_ when
 on a Linux machine.

---
 Makefile | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6d36e8b8bf..8884b5c649 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,12 @@
+OS:=$(shell uname | sed 's/[-_].*//')
+
+ifeq ($(OS),Linux)
+BASEFLAGS_OPTS+=-DWITH_INOTIFY
+endif
+
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd -ignore-package monads-tf
-BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 -DWITH_INOTIFY
+BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 $(BASEFLAGS_OPTS)
 GHCFLAGS=-O2 $(BASEFLAGS)
 
 ifdef PROFILE

From 0ecc7dc8927b3840d6a7ba4d39c344f3e962580e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 10:09:56 -0400
Subject: [PATCH 3817/8313] close

---
 ...inux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn b/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn
index 9805478f53..fee00855eb 100644
--- a/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn
+++ b/doc/bugs/fix_for_makefile_to_check_if_OS_is_linux_or_not___40__relates_to_the_new_inotify_flag__41__.mdwn
@@ -32,3 +32,5 @@ index 6d36e8b..8884b5c 100644
 -- 
 1.7.10.4
 

+
+[[done]], thanks --[[Joey]] 

From 3c8a9043b6fc8fafbeac16e8f9199a0d12870549 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 12:25:20 -0400
Subject: [PATCH 3818/8313] skeleton C library for calling kqueue

---
 .gitignore           |  2 +-
 Assistant/Watcher.hs | 12 ++++++++++--
 Makefile             | 18 ++++++++++--------
 Utility/Kqueue.hs    | 31 +++++++++++++++++++++++++++++++
 Utility/libkqueue.c  | 22 ++++++++++++++++++++++
 Utility/libkqueue.h  |  1 +
 6 files changed, 75 insertions(+), 11 deletions(-)
 create mode 100644 Utility/Kqueue.hs
 create mode 100644 Utility/libkqueue.c
 create mode 100644 Utility/libkqueue.h

diff --git a/.gitignore b/.gitignore
index d628f23b7d..afb5f314e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,7 +11,7 @@ html
 *.tix
 .hpc
 Utility/Touch.hs
-Utility/libdiskfree.o
+Utility/*.o
 dist
 # Sandboxed builds
 cabal-dev
diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 1d35b5c1e3..7c913d98c3 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -33,12 +33,15 @@ import qualified Data.ByteString.Lazy as L
 import Utility.Inotify
 import System.INotify
 #endif
+#ifdef WITH_KQUEUE
+import Utility.Kqueue
+#endif
 
 type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Maybe Change)
 
 checkCanWatch :: Annex ()
 checkCanWatch = do
-#ifdef WITH_INOTIFY
+#if (WITH_INOTIFY || WITH_KQUEUE)
 	unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) $
 		needLsof
 #else
@@ -82,8 +85,13 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			, errHook = hook onErr
 			}
 #else
+#ifdef WITH_KQUEUE
+watchThread st dstatus changechan = do
+	print =<< waitChange [stdError, stdOutput]
+#else
 watchThread = undefined
-#endif
+#endif /* WITH_KQUEUE */
+#endif /* WITH_INOTIFY */
 
 ignored :: FilePath -> Bool
 ignored ".git" = True
diff --git a/Makefile b/Makefile
index 8884b5c649..73fbc4140d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,13 +1,22 @@
-OS:=$(shell uname | sed 's/[-_].*//')
+bins=git-annex
+mans=git-annex.1 git-annex-shell.1
+sources=Build/SysConfig.hs Utility/Touch.hs
+all=$(bins) $(mans) docs
 
+OS:=$(shell uname | sed 's/[-_].*//')
 ifeq ($(OS),Linux)
 BASEFLAGS_OPTS+=-DWITH_INOTIFY
+clibs=Utility/libdiskfree.o
+else
+BASEFLAGS_OPTS+=-DWITH_KQUEUE
+clibs=Utility/libdiskfree.o Utility/libkqueue.o
 endif
 
 PREFIX=/usr
 IGNORE=-ignore-package monads-fd -ignore-package monads-tf
 BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 $(BASEFLAGS_OPTS)
 GHCFLAGS=-O2 $(BASEFLAGS)
+CFLAGS=-Wall
 
 ifdef PROFILE
 GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS)
@@ -15,13 +24,6 @@ endif
 
 GHCMAKE=ghc $(GHCFLAGS) --make
 
-bins=git-annex
-mans=git-annex.1 git-annex-shell.1
-sources=Build/SysConfig.hs Utility/Touch.hs
-clibs=Utility/libdiskfree.o
-
-all=$(bins) $(mans) docs
-
 # Am I typing :make in vim? Do a fast build.
 ifdef VIM
 all=fast
diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
new file mode 100644
index 0000000000..bfc6ee9fc8
--- /dev/null
+++ b/Utility/Kqueue.hs
@@ -0,0 +1,31 @@
+{- BSD kqueue file modification notification interface
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE ForeignFunctionInterface #-}
+
+module Utility.Kqueue ( waitChange ) where
+
+import Common
+
+import System.Posix.Types
+import Foreign.C.Types
+import Foreign.C.Error
+import Foreign.Ptr
+import Foreign.Marshal
+
+foreign import ccall unsafe "libkqueue.h waitchange" c_waitchange
+	:: Ptr Fd -> IO Fd
+
+waitChange :: [Fd] -> IO (Maybe Fd)
+waitChange fds = withArray fds $ \c_fds -> do
+	ret <- c_waitchange c_fds
+	ifM (safeErrno <$> getErrno)
+		( return $ Just ret
+		, return Nothing
+		)
+	where
+		safeErrno (Errno v) = v == 0
diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
new file mode 100644
index 0000000000..0ef42b801b
--- /dev/null
+++ b/Utility/libkqueue.c
@@ -0,0 +1,22 @@
+/* kqueue interface, C mini-library
+ *
+ * Copyright 2012 Joey Hess 
+ *
+ * Licensed under the GNU GPL version 3 or higher.
+ */
+
+#include 
+#include 
+
+/* Waits for a change event on one of the array of directory fds,
+ * and returns the one that changed. */
+int waitchange(const int *fds) {
+//	if (kqueue(blah, &fds) != 0)
+//		return 0; /* errno is set */
+//	else
+		errno = 0;
+
+	printf("in waitchange!, %i %i\n", fds[0], fds[1]);
+
+	return fds[0];
+}
diff --git a/Utility/libkqueue.h b/Utility/libkqueue.h
new file mode 100644
index 0000000000..75af9eeba7
--- /dev/null
+++ b/Utility/libkqueue.h
@@ -0,0 +1 @@
+int waitchange(const int *fds);

From dc3d9d1e982f7342dd3e2b3fc14fbbe85e7acd3e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 12:53:57 -0400
Subject: [PATCH 3819/8313] added dirTree

---
 Utility/Directory.hs | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index 78bb6e7016..b8ed63a36f 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -34,7 +34,7 @@ dirCruft _ = False
 dirContents :: FilePath -> IO [FilePath]
 dirContents d = map (d ) . filter (not . dirCruft) <$> getDirectoryContents d
 
-{- Gets contents of directory, and then its subdirectories, recursively,
+{- Gets files in a directory, and then its subdirectories, recursively,
  - and lazily. -}
 dirContentsRecursive :: FilePath -> IO [FilePath]
 dirContentsRecursive topdir = dirContentsRecursive' topdir [""]
@@ -56,6 +56,33 @@ dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do
 					, collect (dirEntry:files) dirs' entries
 					)			
 
+{- Gets the subdirectories in a directory, and their subdirectories,
+ - recursively, and lazily. Prunes sections of the tree matching a
+ - condition. -}
+dirTree :: FilePath -> (FilePath -> Bool) -> IO [FilePath]
+dirTree topdir prune
+	| prune topdir = return []
+	| otherwise = (:) topdir <$> dirTree' topdir prune [""]
+
+dirTree' :: FilePath -> (FilePath -> Bool) -> [FilePath] -> IO [FilePath]
+dirTree' _ _ [] = return []
+dirTree' topdir prune (dir:dirs)
+	| prune dir = dirTree' topdir prune dirs
+	| otherwise = unsafeInterleaveIO $ do
+		subdirs <- collect [] =<< dirContents (topdir  dir)
+		subdirs' <- dirTree' topdir prune (subdirs ++ dirs)
+		return $ subdirs ++ subdirs'
+	where
+		collect dirs' [] = return dirs'
+		collect dirs' (entry:entries)
+			| dirCruft entry || prune entry = collect dirs' entries
+			| otherwise = do
+				let dirEntry = dir  entry
+				ifM (doesDirectoryExist $ topdir  dirEntry)
+					( collect (dirEntry:dirs') entries
+					, collect dirs' entries
+					)			
+
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}
 moveFile :: FilePath -> FilePath -> IO ()

From a39b73d118c18707e6549d57a902fca9966119f8 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 13:01:58 -0400
Subject: [PATCH 3820/8313] recurse dirTree and open the directories for kqueue
 to watch

---
 Assistant/Watcher.hs | 22 +++++++++++++---------
 Utility/Kqueue.hs    | 23 ++++++++++++++++++-----
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 7c913d98c3..52c3780ab2 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -37,8 +37,6 @@ import System.INotify
 import Utility.Kqueue
 #endif
 
-type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Maybe Change)
-
 checkCanWatch :: Annex ()
 checkCanWatch = do
 #if (WITH_INOTIFY || WITH_KQUEUE)
@@ -66,7 +64,7 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 		showAction "scanning"
 	-- This does not return until the startup scan is done.
 	-- That can take some time for large trees.
-	watchDir i "." (ignored . takeFileName) hooks
+	watchDir i "." ignored hooks
 	runThreadState st $
 		modifyDaemonStatus dstatus $ \s -> s { scanComplete = True }
 	-- Notice any files that were deleted before inotify
@@ -86,18 +84,24 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			}
 #else
 #ifdef WITH_KQUEUE
-watchThread st dstatus changechan = do
-	print =<< waitChange [stdError, stdOutput]
+watchThread st dstatus changechan = forever $ do
+	dirs <- scanRecursive "." ignored
+	changeddir <- waitChange dirs
+	print $ "detected a change in " ++ show changeddir
 #else
 watchThread = undefined
 #endif /* WITH_KQUEUE */
 #endif /* WITH_INOTIFY */
 
 ignored :: FilePath -> Bool
-ignored ".git" = True
-ignored ".gitignore" = True
-ignored ".gitattributes" = True
-ignored _ = False
+ignored = ig . takeFileName
+	where
+	ig ".git" = True
+	ig ".gitignore" = True
+	ig ".gitattributes" = True
+	ig _ = False
+
+type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Maybe Change)
 
 {- Runs an action handler, inside the Annex monad, and if there was a
  - change, adds it to the ChangeChan.
diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index bfc6ee9fc8..aabea7d038 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -7,7 +7,10 @@
 
 {-# LANGUAGE ForeignFunctionInterface #-}
 
-module Utility.Kqueue ( waitChange ) where
+module Utility.Kqueue (
+	waitChange,
+	scanRecursive
+) where
 
 import Common
 
@@ -16,16 +19,26 @@ import Foreign.C.Types
 import Foreign.C.Error
 import Foreign.Ptr
 import Foreign.Marshal
+import qualified Data.Map as M
+
+type DirMap = M.Map Fd FilePath
 
 foreign import ccall unsafe "libkqueue.h waitchange" c_waitchange
 	:: Ptr Fd -> IO Fd
 
-waitChange :: [Fd] -> IO (Maybe Fd)
-waitChange fds = withArray fds $ \c_fds -> do
-	ret <- c_waitchange c_fds
+waitChange :: DirMap -> IO (Maybe FilePath)
+waitChange dirmap = withArray (M.keys dirmap) $ \c_fds -> do
+	changed <- c_waitchange c_fds
 	ifM (safeErrno <$> getErrno)
-		( return $ Just ret
+		( return $ M.lookup changed dirmap
 		, return Nothing
 		)
 	where
 		safeErrno (Errno v) = v == 0
+
+scanRecursive :: FilePath -> (FilePath -> Bool) -> IO DirMap
+scanRecursive dir prune = M.fromList <$> (mapM opendir =<< dirTree dir prune)
+	where
+		opendir d = (,)
+			<$> openFd d ReadOnly Nothing defaultFileFlags
+			<*> pure d

From 89fcee03d0f542c25d1afa9962839916f70994b3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 13:19:40 -0400
Subject: [PATCH 3821/8313] add some utility functions for later

Will need to update the DirMap to add or remove subdirs.
---
 Utility/Kqueue.hs | 39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index aabea7d038..e8ce73b263 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -8,8 +8,10 @@
 {-# LANGUAGE ForeignFunctionInterface #-}
 
 module Utility.Kqueue (
+	scanRecursive,
+	addSubDir,
+	removeSubDir,
 	waitChange,
-	scanRecursive
 ) where
 
 import Common
@@ -23,9 +25,37 @@ import qualified Data.Map as M
 
 type DirMap = M.Map Fd FilePath
 
+{- Builds a map of directories in a tree, possibly pruning some.
+ - Opens each directory in the tree. -}
+scanRecursive :: FilePath -> (FilePath -> Bool) -> IO DirMap
+scanRecursive dir prune = M.fromList <$> (mapM opendir =<< dirTree dir prune)
+	where
+		opendir d = (,)
+			<$> openFd d ReadOnly Nothing defaultFileFlags
+			<*> pure d
+
+{- Adds a subdirectory (and all its subdirectories, unless pruned) to a
+ - directory map. -}
+addSubDir :: DirMap -> FilePath -> (FilePath -> Bool) -> IO DirMap
+addSubDir dirmap dir prune = M.union dirmap <$> scanRecursive dir prune
+
+{- Removes a subdirectory (and all its subdirectories) from a directory map. -}
+removeSubDir :: FilePath -> DirMap -> DirMap
+removeSubDir dir = M.filter (not . dirContains dir)
+
 foreign import ccall unsafe "libkqueue.h waitchange" c_waitchange
 	:: Ptr Fd -> IO Fd
 
+{- Waits for a change in a map of directories, and returns the directory
+ - where the change took place. 
+ -
+ - The kqueue interface does not tell what type of change took place in
+ - the directory; it could be an added file, a deleted file, a renamed
+ - file, a new subdirectory, or a deleted subdirectory, or a moved
+ - subdirectory.
+ -
+ - Note that if subdirectories have changed, the caller will want to
+ - update the map before calling this again. -}
 waitChange :: DirMap -> IO (Maybe FilePath)
 waitChange dirmap = withArray (M.keys dirmap) $ \c_fds -> do
 	changed <- c_waitchange c_fds
@@ -35,10 +65,3 @@ waitChange dirmap = withArray (M.keys dirmap) $ \c_fds -> do
 		)
 	where
 		safeErrno (Errno v) = v == 0
-
-scanRecursive :: FilePath -> (FilePath -> Bool) -> IO DirMap
-scanRecursive dir prune = M.fromList <$> (mapM opendir =<< dirTree dir prune)
-	where
-		opendir d = (,)
-			<$> openFd d ReadOnly Nothing defaultFileFlags
-			<*> pure d

From 90d565149abd7d752e22beb4aa57bf99522e5851 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 16:18:59 -0400
Subject: [PATCH 3822/8313] flesh out kqueue library

Have not tried to build this yet. But barring minor mistakes, I think it's
good.
---
 Utility/Kqueue.hs   | 40 +++++++++++++++++-----------
 Utility/libkqueue.c | 64 ++++++++++++++++++++++++++++++++++++++-------
 Utility/libkqueue.h |  3 ++-
 3 files changed, 81 insertions(+), 26 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index e8ce73b263..a3d8aff2de 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -25,6 +25,8 @@ import qualified Data.Map as M
 
 type DirMap = M.Map Fd FilePath
 
+data Kqueue = Kqueue Fd DirMap
+
 {- Builds a map of directories in a tree, possibly pruning some.
  - Opens each directory in the tree. -}
 scanRecursive :: FilePath -> (FilePath -> Bool) -> IO DirMap
@@ -43,25 +45,33 @@ addSubDir dirmap dir prune = M.union dirmap <$> scanRecursive dir prune
 removeSubDir :: FilePath -> DirMap -> DirMap
 removeSubDir dir = M.filter (not . dirContains dir)
 
-foreign import ccall unsafe "libkqueue.h waitchange" c_waitchange
-	:: Ptr Fd -> IO Fd
+foreign import ccall unsafe "libkqueue.h init_kqueue" c_init_kqueue
+	:: CInt -> Ptr Fd -> IO Fd
+foreign import ccall unsafe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
+	:: Fd -> IO Fd
 
-{- Waits for a change in a map of directories, and returns the directory
- - where the change took place. 
+{- Initializes a Kqueue to watch a map of directories. -}
+initKqueue :: DirMap -> IO Kqueue
+initKqueue dirmap = withArrayLen (M.keys dirmap) $ \fdcnt c_fds ->
+	h <- c_init_kqueue (fromIntegral fdcnt) c_fds
+	return $ Kqueue h dirmap
+
+{- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap,
+ - so it can be reused.  -}
+stopKqueue :: Kqueue -> IO
+stopKqueue (Kqueue h _) = closeFd h
+
+{- Waits for a change on a Kqueue, and returns the directory
+ - or directories where a change took place.
  -
  - The kqueue interface does not tell what type of change took place in
  - the directory; it could be an added file, a deleted file, a renamed
  - file, a new subdirectory, or a deleted subdirectory, or a moved
  - subdirectory.
  -
- - Note that if subdirectories have changed, the caller will want to
- - update the map before calling this again. -}
-waitChange :: DirMap -> IO (Maybe FilePath)
-waitChange dirmap = withArray (M.keys dirmap) $ \c_fds -> do
-	changed <- c_waitchange c_fds
-	ifM (safeErrno <$> getErrno)
-		( return $ M.lookup changed dirmap
-		, return Nothing
-		)
-	where
-		safeErrno (Errno v) = v == 0
+ - Note that if subdirectories have changed, the caller should re-run
+ - initKqueue to get them watched. -}
+waitChange :: Kqueue -> IO [FilePath]
+waitChange (Kqueue h dirmap) = do
+	changed <- c_waitchange_kqueue h
+	return $ M.lookup changed dirmap
diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index 0ef42b801b..a919a60c77 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -5,18 +5,62 @@
  * Licensed under the GNU GPL version 3 or higher.
  */
 
-#include 
 #include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
 
-/* Waits for a change event on one of the array of directory fds,
- * and returns the one that changed. */
-int waitchange(const int *fds) {
-//	if (kqueue(blah, &fds) != 0)
-//		return 0; /* errno is set */
-//	else
-		errno = 0;
+/* Initializes a kqueue, with a list of fds to watch for changes.
+ * Returns the kqueue's handle. */
+int init_kqueue(const int fdcnt, const int *fdlist) {
+	struct nodelay = {0, 0};
+	int kq;
 
-	printf("in waitchange!, %i %i\n", fds[0], fds[1]);
+	if ((kq = kqueue()) == -1) {
+		perror("kqueue");
+		exit(1);
+	}
 
-	return fds[0];
+	/* Prime the pump with the list of fds, but don't wait for any
+	 * change events. */
+	helper(kq, fdcnt, fdlist, &nodelay);
+
+	return kq;
+}
+
+/* Waits for a change event on a kqueue.
+ *
+ * Returns the fd that changed, or -1 on error.
+ */
+signed int waitchange_kqueue(const int kq) {
+	helper(kq, 0, NULL, NULL);
+}
+
+/* The specified fds are added to the set of fds being watched for changes.
+ * Fds passed to prior calls still take effect, so it's most efficient to
+ * not pass the same fds repeatedly.
+ */
+signed int helper(const int kq, const int fdcnt, const int *fdlist, cont struct *timeout) {
+	int i, nev;
+	struct kevent evlist[1];
+	struct kevent chlist[fdcnt];
+	
+	for (i = 0; i < fdcnt; i++) {
+		EV_SET(&chlist[i], fdlist[i], EVFILT_VNODE,
+			EV_ADD | EV_ENABLE | EV_CLEAR,
+			NOTE_WRITE,
+			1,
+			timeout);
+	}
+
+	nev = kevent(info->kq, info->chlist, info->cnt, info->evlist,
+			1, NULL);
+
+	if (nev == 1)
+		return evlist[0].ident;
+	else
+		return -1;
 }
diff --git a/Utility/libkqueue.h b/Utility/libkqueue.h
index 75af9eeba7..1a285b8dad 100644
--- a/Utility/libkqueue.h
+++ b/Utility/libkqueue.h
@@ -1 +1,2 @@
-int waitchange(const int *fds);
+int init_kqueue(const int fdcnt, const int *fdlist);
+signed int waitchange_kqueue(const int kq);

From d680ff7ef06a3b0c8310836b03446e89d0ff9764 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 20:33:27 +0000
Subject: [PATCH 3823/8313] kqueue code compiles on debian kfreebsd

---
 Assistant/Watcher.hs |  8 ++++---
 Utility/Kqueue.hs    | 13 +++++++----
 Utility/libkqueue.c  | 55 ++++++++++++++++++++++----------------------
 3 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 52c3780ab2..13c27d0802 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -84,10 +84,12 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			}
 #else
 #ifdef WITH_KQUEUE
-watchThread st dstatus changechan = forever $ do
+watchThread st dstatus changechan = do
 	dirs <- scanRecursive "." ignored
-	changeddir <- waitChange dirs
-	print $ "detected a change in " ++ show changeddir
+	kqueue <- initKqueue dirs
+	forever $ do
+		changeddir <- waitChange kqueue
+		print $ "detected a change in " ++ show changeddir
 #else
 watchThread = undefined
 #endif /* WITH_KQUEUE */
diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index a3d8aff2de..6da97d3fa4 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -11,6 +11,10 @@ module Utility.Kqueue (
 	scanRecursive,
 	addSubDir,
 	removeSubDir,
+
+	initKqueue,
+	stopKqueue,
+
 	waitChange,
 ) where
 
@@ -18,7 +22,6 @@ import Common
 
 import System.Posix.Types
 import Foreign.C.Types
-import Foreign.C.Error
 import Foreign.Ptr
 import Foreign.Marshal
 import qualified Data.Map as M
@@ -52,17 +55,17 @@ foreign import ccall unsafe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
 
 {- Initializes a Kqueue to watch a map of directories. -}
 initKqueue :: DirMap -> IO Kqueue
-initKqueue dirmap = withArrayLen (M.keys dirmap) $ \fdcnt c_fds ->
+initKqueue dirmap = withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do
 	h <- c_init_kqueue (fromIntegral fdcnt) c_fds
 	return $ Kqueue h dirmap
 
 {- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap,
  - so it can be reused.  -}
-stopKqueue :: Kqueue -> IO
+stopKqueue :: Kqueue -> IO ()
 stopKqueue (Kqueue h _) = closeFd h
 
 {- Waits for a change on a Kqueue, and returns the directory
- - or directories where a change took place.
+ - where a change took place.
  -
  - The kqueue interface does not tell what type of change took place in
  - the directory; it could be an added file, a deleted file, a renamed
@@ -71,7 +74,7 @@ stopKqueue (Kqueue h _) = closeFd h
  -
  - Note that if subdirectories have changed, the caller should re-run
  - initKqueue to get them watched. -}
-waitChange :: Kqueue -> IO [FilePath]
+waitChange :: Kqueue -> IO (Maybe FilePath)
 waitChange (Kqueue h dirmap) = do
 	changed <- c_waitchange_kqueue h
 	return $ M.lookup changed dirmap
diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index a919a60c77..999508f7e5 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -13,10 +13,35 @@
 #include 
 #include 
 
+/* The specified fds are added to the set of fds being watched for changes.
+ * Fds passed to prior calls still take effect, so it's most efficient to
+ * not pass the same fds repeatedly.
+ */
+signed int helper(const int kq, const int fdcnt, const int *fdlist,
+		struct timespec *timeout) {
+	int i, nev;
+	struct kevent evlist[1];
+	struct kevent chlist[fdcnt];
+	
+	for (i = 0; i < fdcnt; i++) {
+		EV_SET(&chlist[i], fdlist[i], EVFILT_VNODE,
+			EV_ADD | EV_ENABLE | EV_CLEAR,
+			NOTE_WRITE,
+			0, 0);
+	}
+
+	nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout);
+
+	if (nev == 1)
+		return evlist[0].ident;
+	else
+		return -1;
+}
+
 /* Initializes a kqueue, with a list of fds to watch for changes.
  * Returns the kqueue's handle. */
 int init_kqueue(const int fdcnt, const int *fdlist) {
-	struct nodelay = {0, 0};
+	struct timespec nodelay = {0, 0};
 	int kq;
 
 	if ((kq = kqueue()) == -1) {
@@ -36,31 +61,5 @@ int init_kqueue(const int fdcnt, const int *fdlist) {
  * Returns the fd that changed, or -1 on error.
  */
 signed int waitchange_kqueue(const int kq) {
-	helper(kq, 0, NULL, NULL);
-}
-
-/* The specified fds are added to the set of fds being watched for changes.
- * Fds passed to prior calls still take effect, so it's most efficient to
- * not pass the same fds repeatedly.
- */
-signed int helper(const int kq, const int fdcnt, const int *fdlist, cont struct *timeout) {
-	int i, nev;
-	struct kevent evlist[1];
-	struct kevent chlist[fdcnt];
-	
-	for (i = 0; i < fdcnt; i++) {
-		EV_SET(&chlist[i], fdlist[i], EVFILT_VNODE,
-			EV_ADD | EV_ENABLE | EV_CLEAR,
-			NOTE_WRITE,
-			1,
-			timeout);
-	}
-
-	nev = kevent(info->kq, info->chlist, info->cnt, info->evlist,
-			1, NULL);
-
-	if (nev == 1)
-		return evlist[0].ident;
-	else
-		return -1;
+	return helper(kq, 0, NULL, NULL);
 }

From a11825a1f153fc7fe9aa469055b3935f254a4e9d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 20:55:06 +0000
Subject: [PATCH 3824/8313] add test stub

---
 Utility/libkqueue.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index 999508f7e5..cc001045bb 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -63,3 +63,13 @@ int init_kqueue(const int fdcnt, const int *fdlist) {
 signed int waitchange_kqueue(const int kq) {
 	return helper(kq, 0, NULL, NULL);
 }
+
+/*
+main () {
+	int list[1];
+	int kq;
+	list[0]=open(".", O_RDONLY);
+	kq = init_kqueue(1, list);
+	printf("change: %i\n", waitchange_kqueue(kq));
+}
+*/

From b141d9bcc8b5a4647e3c5f106115c8bf5a67f6cd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 22:02:57 +0000
Subject: [PATCH 3825/8313] retry interrupted kevent calls

Many thanks to geekosaur in #haskell for help with this.
---
 Utility/libkqueue.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index cc001045bb..5b38cdd339 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -12,6 +12,7 @@
 #include 
 #include 
 #include 
+#include 
 
 /* The specified fds are added to the set of fds being watched for changes.
  * Fds passed to prior calls still take effect, so it's most efficient to
@@ -30,7 +31,11 @@ signed int helper(const int kq, const int fdcnt, const int *fdlist,
 			0, 0);
 	}
 
-	nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout);
+	while ((nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout))) {
+		if (!(nev == -1 && errno == EINTR)) {
+			break;
+		}
+	}
 
 	if (nev == 1)
 		return evlist[0].ident;

From 1f6d80007c0cb9bca21bc744c8e2388e2f0fa8bc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 18:07:29 -0400
Subject: [PATCH 3826/8313] blog for the day

---
 .../assistant/blog/day_12__freebsd_redux.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_12__freebsd_redux.mdwn

diff --git a/doc/design/assistant/blog/day_12__freebsd_redux.mdwn b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn
new file mode 100644
index 0000000000..ba397788a5
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn
@@ -0,0 +1,23 @@
+Followed my plan from yesterday, and wrote a simple C library to interface
+to `kqueue`, and Haskell code to use that library. By now I think I
+understand kqueue fairly well -- there are some very tricky parts to the
+interface.
+
+But... it still did't work. After building all this, my code was
+failing the same way that the
+[haskell kqueue library failed](https://github.com/hesselink/kqueue/issues/1)
+yesterday. I filed a [bug report with a testcase]().
+
+Then I thought to ask on #haskell. Got sorted out in quick order! The
+problem turns out to be that haskell's runtime has a peridic SIGALARM,
+that is interrupting my kevent call. It can be worked around with `+RTS -V0`,
+but I put in a fix to retry to kevent when it's interrupted.
+
+And now `git-annex watch` can detect changes to directories on BSD and OSX!
+
+Note: I said "detect", not "do something useful in response to". Getting
+from the limited kqueue events to actually staging changes in the git repo
+is going to be another day's work. Still, brave FreeBSD or OSX users
+might want to check out the `watch` branch from git and see if 
+`git annex watch` will at least *say* it sees changes you make to your
+repository.

From f684f282f288a464e4d7d6d88865256b86a67c84 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 18:07:29 -0400
Subject: [PATCH 3827/8313] blog for the day

---
 .../assistant/blog/day_12__freebsd_redux.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_12__freebsd_redux.mdwn

diff --git a/doc/design/assistant/blog/day_12__freebsd_redux.mdwn b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn
new file mode 100644
index 0000000000..ba397788a5
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn
@@ -0,0 +1,23 @@
+Followed my plan from yesterday, and wrote a simple C library to interface
+to `kqueue`, and Haskell code to use that library. By now I think I
+understand kqueue fairly well -- there are some very tricky parts to the
+interface.
+
+But... it still did't work. After building all this, my code was
+failing the same way that the
+[haskell kqueue library failed](https://github.com/hesselink/kqueue/issues/1)
+yesterday. I filed a [bug report with a testcase]().
+
+Then I thought to ask on #haskell. Got sorted out in quick order! The
+problem turns out to be that haskell's runtime has a peridic SIGALARM,
+that is interrupting my kevent call. It can be worked around with `+RTS -V0`,
+but I put in a fix to retry to kevent when it's interrupted.
+
+And now `git-annex watch` can detect changes to directories on BSD and OSX!
+
+Note: I said "detect", not "do something useful in response to". Getting
+from the limited kqueue events to actually staging changes in the git repo
+is going to be another day's work. Still, brave FreeBSD or OSX users
+might want to check out the `watch` branch from git and see if 
+`git annex watch` will at least *say* it sees changes you make to your
+repository.

From ae7d07ddcb5768cf477410e019d42601d8c2b744 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 19:14:58 -0400
Subject: [PATCH 3828/8313] close fds

---
 Utility/Kqueue.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index 6da97d3fa4..d0b3c8a99e 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -45,8 +45,12 @@ addSubDir :: DirMap -> FilePath -> (FilePath -> Bool) -> IO DirMap
 addSubDir dirmap dir prune = M.union dirmap <$> scanRecursive dir prune
 
 {- Removes a subdirectory (and all its subdirectories) from a directory map. -}
-removeSubDir :: FilePath -> DirMap -> DirMap
-removeSubDir dir = M.filter (not . dirContains dir)
+removeSubDir :: FilePath -> DirMap -> IO DirMap
+removeSubDir dir dirmap = do
+	mapM_ closeFd $ M.keys toremove) $ closeFd
+	return rest
+	where
+		(toremove, rest) = M.partition (dirContains dir) dirmap
 
 foreign import ccall unsafe "libkqueue.h init_kqueue" c_init_kqueue
 	:: CInt -> Ptr Fd -> IO Fd

From 2bfcc0b09c5dd37c5e0ab65cb089232bfcc31934 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 21:29:30 -0400
Subject: [PATCH 3829/8313] kqueue: add directory content tracking, and change
 determination

This *may* now return Add or Delete Changes as appropriate. All I know
for sure is that it compiles.

I had hoped to avoid maintaining my own state about the content of the
directory tree, and rely on git to check what was changed. But I can't;
I need to know about new and deleted subdirectories to add them to the
watch list, and git doesn't deal with (empty) directories.

So, wrote all the code to scan directories, remember their past contents,
compare with current contents, generate appropriate Change events, and
update bookkeeping info appropriately.
---
 Utility/Directory.hs |  27 --------
 Utility/Kqueue.hs    | 158 ++++++++++++++++++++++++++++++++-----------
 2 files changed, 119 insertions(+), 66 deletions(-)

diff --git a/Utility/Directory.hs b/Utility/Directory.hs
index b8ed63a36f..2f2960a9d8 100644
--- a/Utility/Directory.hs
+++ b/Utility/Directory.hs
@@ -56,33 +56,6 @@ dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do
 					, collect (dirEntry:files) dirs' entries
 					)			
 
-{- Gets the subdirectories in a directory, and their subdirectories,
- - recursively, and lazily. Prunes sections of the tree matching a
- - condition. -}
-dirTree :: FilePath -> (FilePath -> Bool) -> IO [FilePath]
-dirTree topdir prune
-	| prune topdir = return []
-	| otherwise = (:) topdir <$> dirTree' topdir prune [""]
-
-dirTree' :: FilePath -> (FilePath -> Bool) -> [FilePath] -> IO [FilePath]
-dirTree' _ _ [] = return []
-dirTree' topdir prune (dir:dirs)
-	| prune dir = dirTree' topdir prune dirs
-	| otherwise = unsafeInterleaveIO $ do
-		subdirs <- collect [] =<< dirContents (topdir  dir)
-		subdirs' <- dirTree' topdir prune (subdirs ++ dirs)
-		return $ subdirs ++ subdirs'
-	where
-		collect dirs' [] = return dirs'
-		collect dirs' (entry:entries)
-			| dirCruft entry || prune entry = collect dirs' entries
-			| otherwise = do
-				let dirEntry = dir  entry
-				ifM (doesDirectoryExist $ topdir  dirEntry)
-					( collect (dirEntry:dirs') entries
-					, collect dirs' entries
-					)			
-
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}
 moveFile :: FilePath -> FilePath -> IO ()
diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index d0b3c8a99e..911eb71a95 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -8,14 +8,10 @@
 {-# LANGUAGE ForeignFunctionInterface #-}
 
 module Utility.Kqueue (
-	scanRecursive,
-	addSubDir,
-	removeSubDir,
-
 	initKqueue,
 	stopKqueue,
-
 	waitChange,
+	Change(..),
 ) where
 
 import Common
@@ -25,60 +21,144 @@ import Foreign.C.Types
 import Foreign.Ptr
 import Foreign.Marshal
 import qualified Data.Map as M
+import qualified Data.Set as S
 
-type DirMap = M.Map Fd FilePath
+data Change
+	= Deleted FilePath
+	| Added FilePath
+	deriving (Show)
 
-data Kqueue = Kqueue Fd DirMap
+isAdd :: Change -> Bool
+isAdd (Added _) = True
+isAdd (Deleted _) = False
+
+isDelete :: Change -> Bool
+isDelete = not . isAdd
+
+changedFile :: Change -> FilePath
+changedFile (Added f) = f
+changedFile (Deleted f) = f
+
+data Kqueue = Kqueue Fd DirMap Pruner
+
+type Pruner = FilePath -> Bool
+
+type DirMap = M.Map Fd DirInfo
+
+{- A directory, and its last known contents (with filenames relative to it) -}
+data DirInfo = DirInfo
+	{ dirName :: FilePath
+	, dirCache :: S.Set FilePath
+	}
+	deriving (Show)
+
+getDirInfo :: FilePath -> IO DirInfo
+getDirInfo dir = do
+	contents <- S.fromList . filter (not . dirCruft)
+		<$> getDirectoryContents dir
+	return $ DirInfo dir contents
+
+{- Difference between the dirCaches of two DirInfos. -}
+(//) :: DirInfo -> DirInfo -> [Change]
+old // new = deleted ++ added
+	where
+		deleted = calc Deleted old new
+		added   = calc Added new old
+		calc a x y = map a . map (dirName x ) $
+			S.toList $ S.difference (dirCache x) (dirCache y)
 
 {- Builds a map of directories in a tree, possibly pruning some.
- - Opens each directory in the tree. -}
-scanRecursive :: FilePath -> (FilePath -> Bool) -> IO DirMap
-scanRecursive dir prune = M.fromList <$> (mapM opendir =<< dirTree dir prune)
+ - Opens each directory in the tree, and records its current contents. -}
+scanRecursive :: FilePath -> Pruner -> IO DirMap
+scanRecursive topdir prune = M.fromList <$> walk [] [topdir]
 	where
-		opendir d = (,)
-			<$> openFd d ReadOnly Nothing defaultFileFlags
-			<*> pure d
+		walk c [] = return c
+		walk c (dir:rest)
+			| prune dir = walk c rest
+			| otherwise = do
+				info <- getDirInfo dir
+				fd <- openFd dir ReadOnly Nothing defaultFileFlags
+				dirs <- filterM (\d -> doesDirectoryExist $ dir  d)
+					(S.toList $ dirCache info)
+				walk ((fd, info):c) (dirs++rest)
 
-{- Adds a subdirectory (and all its subdirectories, unless pruned) to a
- - directory map. -}
-addSubDir :: DirMap -> FilePath -> (FilePath -> Bool) -> IO DirMap
-addSubDir dirmap dir prune = M.union dirmap <$> scanRecursive dir prune
+{- Adds a list of subdirectories (and all their children), unless pruned to a
+ - directory map. Adding a subdirectory that's already in the map will
+ - cause its contents to be refreshed. -}
+addSubDirs :: DirMap -> Pruner -> [FilePath] -> IO DirMap
+addSubDirs dirmap prune dirs = do
+	newmap <- foldr M.union M.empty <$>
+		mapM (\d -> scanRecursive d prune) dirs
+	return $ M.union newmap dirmap -- prefer newmap
 
-{- Removes a subdirectory (and all its subdirectories) from a directory map. -}
-removeSubDir :: FilePath -> DirMap -> IO DirMap
-removeSubDir dir dirmap = do
-	mapM_ closeFd $ M.keys toremove) $ closeFd
+{- Removes a subdirectory (and all its children) from a directory map. -}
+removeSubDir :: DirMap -> FilePath -> IO DirMap
+removeSubDir dirmap dir = do
+	mapM_ closeFd $ M.keys toremove
 	return rest
 	where
-		(toremove, rest) = M.partition (dirContains dir) dirmap
+		(toremove, rest) = M.partition (dirContains dir . dirName) dirmap
 
 foreign import ccall unsafe "libkqueue.h init_kqueue" c_init_kqueue
 	:: CInt -> Ptr Fd -> IO Fd
 foreign import ccall unsafe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
 	:: Fd -> IO Fd
 
-{- Initializes a Kqueue to watch a map of directories. -}
-initKqueue :: DirMap -> IO Kqueue
-initKqueue dirmap = withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do
-	h <- c_init_kqueue (fromIntegral fdcnt) c_fds
-	return $ Kqueue h dirmap
+{- Initializes a Kqueue to watch a directory, and all its subdirectories. -}
+initKqueue :: FilePath -> Pruner -> IO Kqueue
+initKqueue dir pruned = do
+	dirmap <- scanRecursive dir pruned
+	withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do
+		h <- c_init_kqueue (fromIntegral fdcnt) c_fds
+		return $ Kqueue h dirmap pruned
 
 {- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap,
  - so it can be reused.  -}
 stopKqueue :: Kqueue -> IO ()
-stopKqueue (Kqueue h _) = closeFd h
+stopKqueue (Kqueue h _ _) = closeFd h
 
-{- Waits for a change on a Kqueue, and returns the directory
- - where a change took place.
- -
- - The kqueue interface does not tell what type of change took place in
+{- Waits for a change on a Kqueue.
+ - May update the Kqueue.
+ -}
+waitChange :: Kqueue -> IO (Kqueue, [Change])
+waitChange kq@(Kqueue h dirmap _) = do
+	changedfd <- c_waitchange_kqueue h
+	case M.lookup changedfd dirmap of
+		Nothing -> return (kq, [])
+		Just info -> handleChange kq changedfd info
+
+{- The kqueue interface does not tell what type of change took place in
  - the directory; it could be an added file, a deleted file, a renamed
  - file, a new subdirectory, or a deleted subdirectory, or a moved
- - subdirectory.
+ - subdirectory. 
  -
- - Note that if subdirectories have changed, the caller should re-run
- - initKqueue to get them watched. -}
-waitChange :: Kqueue -> IO (Maybe FilePath)
-waitChange (Kqueue h dirmap) = do
-	changed <- c_waitchange_kqueue h
-	return $ M.lookup changed dirmap
+ - So to determine this, the contents of the directory are compared
+ - with its last cached contents. The Kqueue is updated to watch new
+ - directories as necessary.
+ -}
+handleChange :: Kqueue -> Fd -> DirInfo -> IO (Kqueue, [Change])
+handleChange kq@(Kqueue h dirmap pruner) fd olddirinfo =
+	go =<< catchMaybeIO (getDirInfo $ dirName olddirinfo)
+	where
+		go (Just newdirinfo) = do
+			let changes = olddirinfo // newdirinfo
+			let (added, deleted) = partition isAdd changes
+
+			-- Scan newly added directories to add to the map.
+			-- (Newly added files will fail getDirInfo.)
+			newdirinfos <- catMaybes <$>
+				mapM (catchMaybeIO . getDirInfo . changedFile) added
+			newmap <- addSubDirs dirmap pruner $ map dirName newdirinfos
+
+			-- Remove deleted directories from the map.
+			newmap' <- foldM removeSubDir newmap (map changedFile deleted)
+
+			-- Update the cached dirinfo just looked up.
+			let newmap'' = M.insertWith' const fd newdirinfo newmap'
+			ret (newmap'', changes)
+		go Nothing = do
+			-- The directory has been moved or deleted, so
+			-- remove it from our map.
+			newmap <- removeSubDir dirmap (dirName olddirinfo)
+			ret (newmap, [])
+		ret (newmap, changes) = return $ (Kqueue h newmap pruner, changes)

From 5e9fdac92fe91a60d8f4570ec5d785976fe6c3ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 21:46:04 -0400
Subject: [PATCH 3830/8313] update kqueue when new directories are added

---
 Utility/Kqueue.hs   | 27 +++++++++++++++++++++------
 Utility/libkqueue.c | 25 ++++++++++++-------------
 2 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index 911eb71a95..08029d703e 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -100,7 +100,9 @@ removeSubDir dirmap dir = do
 		(toremove, rest) = M.partition (dirContains dir . dirName) dirmap
 
 foreign import ccall unsafe "libkqueue.h init_kqueue" c_init_kqueue
-	:: CInt -> Ptr Fd -> IO Fd
+	:: IO Fd
+foreign import ccall unsafe "libkqueue.h addfds_kqueue" c_addfds_kqueue
+	:: Fd -> CInt -> Ptr Fd -> IO ()
 foreign import ccall unsafe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
 	:: Fd -> IO Fd
 
@@ -108,9 +110,16 @@ foreign import ccall unsafe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
 initKqueue :: FilePath -> Pruner -> IO Kqueue
 initKqueue dir pruned = do
 	dirmap <- scanRecursive dir pruned
+	h <- c_init_kqueue
+	let kq = Kqueue h dirmap pruned
+	updateKqueue kq
+	return kq
+
+{- Updates a Kqueue, adding watches for its map. -}
+updateKqueue :: Kqueue -> IO ()
+updateKqueue (Kqueue h dirmap _) =
 	withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do
-		h <- c_init_kqueue (fromIntegral fdcnt) c_fds
-		return $ Kqueue h dirmap pruned
+		c_addfds_kqueue h (fromIntegral fdcnt) c_fds
 
 {- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap,
  - so it can be reused.  -}
@@ -155,10 +164,16 @@ handleChange kq@(Kqueue h dirmap pruner) fd olddirinfo =
 
 			-- Update the cached dirinfo just looked up.
 			let newmap'' = M.insertWith' const fd newdirinfo newmap'
-			ret (newmap'', changes)
+
+			-- When new directories were added, need to update
+			-- the kqueue to watch them.
+			let kq' = Kqueue h newmap'' pruner
+			unless (null newdirinfos) $
+				updateKqueue kq'
+
+			return (kq', changes)
 		go Nothing = do
 			-- The directory has been moved or deleted, so
 			-- remove it from our map.
 			newmap <- removeSubDir dirmap (dirName olddirinfo)
-			ret (newmap, [])
-		ret (newmap, changes) = return $ (Kqueue h newmap pruner, changes)
+			return (Kqueue h newmap pruner, [])
diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index 5b38cdd339..b7f9595dc1 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -18,11 +18,12 @@
  * Fds passed to prior calls still take effect, so it's most efficient to
  * not pass the same fds repeatedly.
  */
-signed int helper(const int kq, const int fdcnt, const int *fdlist,
-		struct timespec *timeout) {
+signed int helper(const int kq, const int fdcnt, const int *fdlist, int nodelay) {
 	int i, nev;
 	struct kevent evlist[1];
 	struct kevent chlist[fdcnt];
+	struct timespec avoiddelay = {0, 0};
+	struct timespec *timeout = nodelay ? &avoiddelay : NULL;
 	
 	for (i = 0; i < fdcnt; i++) {
 		EV_SET(&chlist[i], fdlist[i], EVFILT_VNODE,
@@ -43,30 +44,27 @@ signed int helper(const int kq, const int fdcnt, const int *fdlist,
 		return -1;
 }
 
-/* Initializes a kqueue, with a list of fds to watch for changes.
- * Returns the kqueue's handle. */
+/* Initializes a new, empty kqueue. */
 int init_kqueue(const int fdcnt, const int *fdlist) {
-	struct timespec nodelay = {0, 0};
 	int kq;
-
 	if ((kq = kqueue()) == -1) {
 		perror("kqueue");
 		exit(1);
 	}
-
-	/* Prime the pump with the list of fds, but don't wait for any
-	 * change events. */
-	helper(kq, fdcnt, fdlist, &nodelay);
-
 	return kq;
 }
 
+/* Adds fds to the set that should be watched. */
+void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist) {
+	helper(kq, fdcnt, fdlist, 1);
+}
+
 /* Waits for a change event on a kqueue.
  *
  * Returns the fd that changed, or -1 on error.
  */
 signed int waitchange_kqueue(const int kq) {
-	return helper(kq, 0, NULL, NULL);
+	return helper(kq, 0, NULL, 0);
 }
 
 /*
@@ -74,7 +72,8 @@ main () {
 	int list[1];
 	int kq;
 	list[0]=open(".", O_RDONLY);
-	kq = init_kqueue(1, list);
+	kq = init_kqueue();
+	addfds_kqueue(kq, 1, list)
 	printf("change: %i\n", waitchange_kqueue(kq));
 }
 */

From 3d163f5ff9cae58e8a1e27215e58dab91180faff Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 01:52:07 +0000
Subject: [PATCH 3831/8313] fix build

---
 Assistant/Watcher.hs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 13c27d0802..e2dd5cd2ae 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -34,7 +34,7 @@ import Utility.Inotify
 import System.INotify
 #endif
 #ifdef WITH_KQUEUE
-import Utility.Kqueue
+import qualified Utility.Kqueue as Kqueue
 #endif
 
 checkCanWatch :: Annex ()
@@ -84,12 +84,12 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			}
 #else
 #ifdef WITH_KQUEUE
-watchThread st dstatus changechan = do
-	dirs <- scanRecursive "." ignored
-	kqueue <- initKqueue dirs
-	forever $ do
-		changeddir <- waitChange kqueue
-		print $ "detected a change in " ++ show changeddir
+watchThread st dstatus changechan = go =<< Kqueue.initKqueue "." ignored
+	where
+		go kq = do
+			(kq', changes) <- Kqueue.waitChange kq
+			print $ "detected a change in " ++ show changes
+			go kq'
 #else
 watchThread = undefined
 #endif /* WITH_KQUEUE */

From 2d457bf8dfa9e69050d213df664d0407072304ad Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 01:52:07 +0000
Subject: [PATCH 3832/8313] fix build

---
 Assistant/Watcher.hs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 13c27d0802..e2dd5cd2ae 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -34,7 +34,7 @@ import Utility.Inotify
 import System.INotify
 #endif
 #ifdef WITH_KQUEUE
-import Utility.Kqueue
+import qualified Utility.Kqueue as Kqueue
 #endif
 
 checkCanWatch :: Annex ()
@@ -84,12 +84,12 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			}
 #else
 #ifdef WITH_KQUEUE
-watchThread st dstatus changechan = do
-	dirs <- scanRecursive "." ignored
-	kqueue <- initKqueue dirs
-	forever $ do
-		changeddir <- waitChange kqueue
-		print $ "detected a change in " ++ show changeddir
+watchThread st dstatus changechan = go =<< Kqueue.initKqueue "." ignored
+	where
+		go kq = do
+			(kq', changes) <- Kqueue.waitChange kq
+			print $ "detected a change in " ++ show changes
+			go kq'
 #else
 watchThread = undefined
 #endif /* WITH_KQUEUE */

From e16455327247656bc47e331be710d6bd58b2675f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 02:13:26 +0000
Subject: [PATCH 3833/8313] robustness fixes

---
 Utility/Kqueue.hs | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index 08029d703e..5b4920f2f2 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -12,6 +12,9 @@ module Utility.Kqueue (
 	stopKqueue,
 	waitChange,
 	Change(..),
+	changedFile,
+	isAdd,
+	isDelete,
 ) where
 
 import Common
@@ -60,10 +63,10 @@ getDirInfo dir = do
 
 {- Difference between the dirCaches of two DirInfos. -}
 (//) :: DirInfo -> DirInfo -> [Change]
-old // new = deleted ++ added
+oldc // newc = deleted ++ added
 	where
-		deleted = calc Deleted old new
-		added   = calc Added new old
+		deleted = calc Deleted oldc newc
+		added   = calc Added newc oldc
 		calc a x y = map a . map (dirName x ) $
 			S.toList $ S.difference (dirCache x) (dirCache y)
 
@@ -76,11 +79,18 @@ scanRecursive topdir prune = M.fromList <$> walk [] [topdir]
 		walk c (dir:rest)
 			| prune dir = walk c rest
 			| otherwise = do
-				info <- getDirInfo dir
-				fd <- openFd dir ReadOnly Nothing defaultFileFlags
-				dirs <- filterM (\d -> doesDirectoryExist $ dir  d)
-					(S.toList $ dirCache info)
-				walk ((fd, info):c) (dirs++rest)
+				minfo <- catchMaybeIO $ getDirInfo dir
+				case minfo of
+					Nothing -> walk c rest
+					Just info -> do
+						mfd <- catchMaybeIO $
+							openFd dir ReadOnly Nothing defaultFileFlags
+						case mfd of
+							Nothing -> walk c rest
+							Just fd -> do
+								let subdirs = map (dir ) $
+									S.toList $ dirCache info
+								walk ((fd, info):c) (subdirs ++ rest)
 
 {- Adds a list of subdirectories (and all their children), unless pruned to a
  - directory map. Adding a subdirectory that's already in the map will
@@ -146,7 +156,7 @@ waitChange kq@(Kqueue h dirmap _) = do
  - directories as necessary.
  -}
 handleChange :: Kqueue -> Fd -> DirInfo -> IO (Kqueue, [Change])
-handleChange kq@(Kqueue h dirmap pruner) fd olddirinfo =
+handleChange (Kqueue h dirmap pruner) fd olddirinfo =
 	go =<< catchMaybeIO (getDirInfo $ dirName olddirinfo)
 	where
 		go (Just newdirinfo) = do

From 22b563408bf08158872ddb8e65c16f36b0ab712d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 02:13:39 +0000
Subject: [PATCH 3834/8313] refactor

---
 Assistant/Watcher.hs | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index e2dd5cd2ae..7e0a16f405 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -60,18 +60,9 @@ needLsof = error $ unlines
 watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
 #ifdef WITH_INOTIFY
 watchThread st dstatus changechan = withINotify $ \i -> do
-	runThreadState st $
-		showAction "scanning"
-	-- This does not return until the startup scan is done.
-	-- That can take some time for large trees.
-	watchDir i "." ignored hooks
-	runThreadState st $
-		modifyDaemonStatus dstatus $ \s -> s { scanComplete = True }
-	-- Notice any files that were deleted before inotify
-	-- was started.
-	runThreadState st $ do
-		inRepo $ Git.Command.run "add" [Param "--update"]
-		showAction "started"
+	statupScan st dstatus $
+		watchDir i "." ignored hooks
+	-- Let the inotify thread run.
 	waitForTermination
 	where
 		hook a = Just $ runHandler st dstatus changechan a
@@ -84,7 +75,10 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			}
 #else
 #ifdef WITH_KQUEUE
-watchThread st dstatus changechan = go =<< Kqueue.initKqueue "." ignored
+watchThread st dstatus changechan = do
+	kq <- statupScan st dstatus $
+		Kqueue.initKqueue "." ignored
+	go kq
 	where
 		go kq = do
 			(kq', changes) <- Kqueue.waitChange kq
@@ -95,6 +89,22 @@ watchThread = undefined
 #endif /* WITH_KQUEUE */
 #endif /* WITH_INOTIFY */
 
+{- Initial scartup scan. The action should return once the scan is complete. -}
+statupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
+statupScan st dstatus scanner = do
+	runThreadState st $
+		showAction "scanning"
+	r <- scanner
+	runThreadState st $
+		modifyDaemonStatus dstatus $ \s -> s { scanComplete = True }
+
+	-- Notice any files that were deleted before watching was started.
+	runThreadState st $ do
+		inRepo $ Git.Command.run "add" [Param "--update"]
+		showAction "started"
+
+	return r
+
 ignored :: FilePath -> Bool
 ignored = ig . takeFileName
 	where

From 7a09d74319c0e68dddfa2cf1979731a030e8881e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Mon, 18 Jun 2012 23:47:48 -0400
Subject: [PATCH 3835/8313] lifted out the kqueue and inotify to a generic
 DirWatcher interface

Kqueue code for dispatching events is not tested and probably doesn't
build.
---
 Assistant/Watcher.hs               | 49 +++++----------------------
 Utility/DirWatcher.hs              | 53 ++++++++++++++++++++++++++++++
 Utility/{Inotify.hs => INotify.hs} | 13 ++------
 Utility/Kqueue.hs                  | 25 ++++++++++++++
 Utility/Types/DirWatcher.hs        | 22 +++++++++++++
 debian/changelog                   |  7 ++--
 6 files changed, 115 insertions(+), 54 deletions(-)
 create mode 100644 Utility/DirWatcher.hs
 rename Utility/{Inotify.hs => INotify.hs} (95%)
 create mode 100644 Utility/Types/DirWatcher.hs

diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index 7e0a16f405..a2ca2396e4 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -13,7 +13,8 @@ import Common.Annex
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.Committer
-import Utility.ThreadScheduler
+import Utility.DirWatcher
+import Utility.Types.DirWatcher
 import qualified Annex.Queue
 import qualified Git.Command
 import qualified Git.UpdateIndex
@@ -29,25 +30,12 @@ import Control.Concurrent.STM
 import Data.Bits.Utils
 import qualified Data.ByteString.Lazy as L
 
-#ifdef WITH_INOTIFY
-import Utility.Inotify
-import System.INotify
-#endif
-#ifdef WITH_KQUEUE
-import qualified Utility.Kqueue as Kqueue
-#endif
-
 checkCanWatch :: Annex ()
-checkCanWatch = do
-#if (WITH_INOTIFY || WITH_KQUEUE)
-	unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) $
-		needLsof
-#else
-#if defined linux_HOST_OS
-#warning "Building without inotify support; watch mode will be disabled."
-#endif
-	error "watch mode is not available on this system"
-#endif
+checkCanWatch
+	| canWatch = 
+		unlessM (liftIO (inPath "lsof") <||> Annex.getState Annex.force) $
+			needLsof
+	| otherwise = error "watch mode is not available on this system"
 
 needLsof :: Annex ()
 needLsof = error $ unlines
@@ -58,13 +46,9 @@ needLsof = error $ unlines
 	]
 
 watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO ()
-#ifdef WITH_INOTIFY
-watchThread st dstatus changechan = withINotify $ \i -> do
-	statupScan st dstatus $
-		watchDir i "." ignored hooks
-	-- Let the inotify thread run.
-	waitForTermination
+watchThread st dstatus changechan = watchDir "." ignored hooks startup
 	where
+		startup = statupScan st dstatus
 		hook a = Just $ runHandler st dstatus changechan a
 		hooks = WatchHooks
 			{ addHook = hook onAdd
@@ -73,21 +57,6 @@ watchThread st dstatus changechan = withINotify $ \i -> do
 			, delDirHook = hook onDelDir
 			, errHook = hook onErr
 			}
-#else
-#ifdef WITH_KQUEUE
-watchThread st dstatus changechan = do
-	kq <- statupScan st dstatus $
-		Kqueue.initKqueue "." ignored
-	go kq
-	where
-		go kq = do
-			(kq', changes) <- Kqueue.waitChange kq
-			print $ "detected a change in " ++ show changes
-			go kq'
-#else
-watchThread = undefined
-#endif /* WITH_KQUEUE */
-#endif /* WITH_INOTIFY */
 
 {- Initial scartup scan. The action should return once the scan is complete. -}
 statupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs
new file mode 100644
index 0000000000..5750361903
--- /dev/null
+++ b/Utility/DirWatcher.hs
@@ -0,0 +1,53 @@
+{- generic directory watching interface
+ -
+ - Uses either inotify or kqueue to watch a directory (and subdirectories)
+ - for changes, and runs hooks for different sorts of events as they occur.
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.DirWatcher where
+
+import Utility.Types.DirWatcher
+
+#if WITH_INOTIFY
+import qualified Utility.INotify as INotify
+import qualified System.INotify as INotify
+import Utility.ThreadScheduler
+#endif
+#if WITH_KQUEUE
+import qualified Utility.Kqueue as Kqueue
+#endif
+
+type Pruner = FilePath -> Bool
+
+canWatch :: Bool
+#if (WITH_INOTIFY || WITH_KQUEUE)
+canWatch = True
+#else
+#if defined linux_HOST_OS
+#warning "Building without inotify support"
+#endif
+canWatch = False
+#endif
+
+#if WITH_INOTIFY
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
+watchDir dir prune hooks runstartup = INotify.withINotify $ \i -> do
+	runstartup $ INotify.watchDir i dir prune hooks
+	waitForTermination -- Let the inotify thread run.
+#else
+#if WITH_KQUEUE
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO ()
+watchDir dir ignored hooks runstartup = do
+	kq <- runstartup $ Kqueue.initKqueue dir ignored
+	Kqueue.runHooks kq hooks
+#else
+watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
+watchDir = undefined
+#endif
+#endif
diff --git a/Utility/Inotify.hs b/Utility/INotify.hs
similarity index 95%
rename from Utility/Inotify.hs
rename to Utility/INotify.hs
index 9ad947f315..bf87f4e71b 100644
--- a/Utility/Inotify.hs
+++ b/Utility/INotify.hs
@@ -5,26 +5,17 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.Inotify where
+module Utility.INotify where
 
 import Common hiding (isDirectory)
 import Utility.ThreadLock
+import Utility.Types.DirWatcher
 
 import System.INotify
 import qualified System.Posix.Files as Files
 import System.IO.Error
 import Control.Exception (throw)
 
-type Hook a = Maybe (a -> Maybe FileStatus -> IO ())
-
-data WatchHooks = WatchHooks
-	{ addHook :: Hook FilePath
-	, addSymlinkHook :: Hook FilePath
-	, delHook :: Hook FilePath
-	, delDirHook :: Hook FilePath
-	, errHook :: Hook String -- error message
-	}
-
 {- Watches for changes to files in a directory, and all its subdirectories
  - that are not ignored, using inotify. This function returns after
  - its initial scan is complete, leaving a thread running. Callbacks are
diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index 5b4920f2f2..30218bc29e 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -15,9 +15,11 @@ module Utility.Kqueue (
 	changedFile,
 	isAdd,
 	isDelete,
+	runHooks,
 ) where
 
 import Common
+import Utility.Types.DirWatcher
 
 import System.Posix.Types
 import Foreign.C.Types
@@ -187,3 +189,26 @@ handleChange (Kqueue h dirmap pruner) fd olddirinfo =
 			-- remove it from our map.
 			newmap <- removeSubDir dirmap (dirName olddirinfo)
 			return (Kqueue h newmap pruner, [])
+
+{- Processes changes on the Kqueue, calling the hooks as appropriate.
+ - Never returns. -}
+runHooks :: Kqueue -> WatchHooks -> IO ()
+runHooks kq hooks = do
+	(kq', changes) <- Kqueue.waitChange kq
+	forM_ changes $ dispatch kq'
+	runHooks kq' hooks
+	where
+		-- Kqueue returns changes for both whole directories
+		-- being added and deleted, and individual files being
+		-- added and deleted.
+		dispatch q change status
+			| isAdd change = withstatus s (dispatchadd q)
+			| isDelete change = callhook delDirHook change
+		dispatchadd q change s
+			| Files.isSymbolicLink = callhook addSymlinkHook change
+			| Files.isDirectory = print $ "TODO: recursive directory add: " ++ show change
+			| Files.isRegularFile = callhook addHook change
+			| otherwise = noop
+		callhook h change = hooks h $ changedFile change
+		withstatus change a = maybe noop (a change) =<<
+			(catchMaybeIO (getSymbolicLinkStatus (changedFile change)
diff --git a/Utility/Types/DirWatcher.hs b/Utility/Types/DirWatcher.hs
new file mode 100644
index 0000000000..c828a05938
--- /dev/null
+++ b/Utility/Types/DirWatcher.hs
@@ -0,0 +1,22 @@
+{- generic directory watching types
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+
+module Utility.Types.DirWatcher where
+
+import Common
+
+type Hook a = Maybe (a -> Maybe FileStatus -> IO ())
+
+data WatchHooks = WatchHooks
+	{ addHook :: Hook FilePath
+	, addSymlinkHook :: Hook FilePath
+	, delHook :: Hook FilePath
+	, delDirHook :: Hook FilePath
+	, errHook :: Hook String -- error message
+	}
diff --git a/debian/changelog b/debian/changelog
index 9a47447ced..f756a8538e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,9 @@
 git-annex (3.20120616) UNRELEASED; urgency=low
 
-  * watch: New subcommand, which uses inotify to watch for changes to
-    files and automatically annexes new files, etc, so you don't need
-    to manually run git commands when manipulating files.
+  * watch: New subcommand, a daemon which notices changes to
+    files and automatically annexes new files, etc, so you don't
+    need to manually run git commands when manipulating files.
+    Available on Linux, BSDs, and OSX!
   * Enable diskfree on kfreebsd, using statvfs.
 
  -- Joey Hess   Tue, 12 Jun 2012 11:35:59 -0400

From 02e9fdb0a5940a1c059445c616338dc147a32544 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 04:04:40 +0000
Subject: [PATCH 3836/8313] kqueue build fix

new event dispatch seems a bit broken though
---
 Utility/Kqueue.hs | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index 30218bc29e..da43a2d865 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -8,6 +8,7 @@
 {-# LANGUAGE ForeignFunctionInterface #-}
 
 module Utility.Kqueue (
+	Kqueue,
 	initKqueue,
 	stopKqueue,
 	waitChange,
@@ -27,6 +28,7 @@ import Foreign.Ptr
 import Foreign.Marshal
 import qualified Data.Map as M
 import qualified Data.Set as S
+import qualified System.Posix.Files as Files
 
 data Change
 	= Deleted FilePath
@@ -194,21 +196,25 @@ handleChange (Kqueue h dirmap pruner) fd olddirinfo =
  - Never returns. -}
 runHooks :: Kqueue -> WatchHooks -> IO ()
 runHooks kq hooks = do
-	(kq', changes) <- Kqueue.waitChange kq
-	forM_ changes $ dispatch kq'
+	(kq', changes) <- waitChange kq
+	forM_ changes $ \c -> do
+		print c
+		dispatch kq' c
 	runHooks kq' hooks
 	where
 		-- Kqueue returns changes for both whole directories
 		-- being added and deleted, and individual files being
 		-- added and deleted.
-		dispatch q change status
-			| isAdd change = withstatus s (dispatchadd q)
-			| isDelete change = callhook delDirHook change
+		dispatch q change
+			| isAdd change = withstatus change $ dispatchadd q
+			| otherwise = callhook delDirHook Nothing change
 		dispatchadd q change s
-			| Files.isSymbolicLink = callhook addSymlinkHook change
-			| Files.isDirectory = print $ "TODO: recursive directory add: " ++ show change
-			| Files.isRegularFile = callhook addHook change
+			| Files.isSymbolicLink s = callhook addSymlinkHook (Just s) change
+			| Files.isDirectory s = print $ "TODO: recursive directory add: " ++ show change
+			| Files.isRegularFile s = callhook addHook (Just s) change
 			| otherwise = noop
-		callhook h change = hooks h $ changedFile change
+		callhook h s change = case h hooks of
+			Nothing -> noop
+			Just a -> a (changedFile change) s
 		withstatus change a = maybe noop (a change) =<<
-			(catchMaybeIO (getSymbolicLinkStatus (changedFile change)
+			(catchMaybeIO (getSymbolicLinkStatus (changedFile change)))

From a5cceb7d4ff83b11da95cac204e99d1bfdbaecc9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 00:23:14 -0400
Subject: [PATCH 3837/8313] make --force really bypass lsof check

---
 Assistant/Committer.hs | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index a2b65dae5a..b482e5e7a0 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -7,6 +7,7 @@ module Assistant.Committer where
 
 import Common.Annex
 import Assistant.ThreadedMonad
+import qualified Annex
 import qualified Annex.Queue
 import qualified Git.Command
 import qualified Command.Add
@@ -153,18 +154,21 @@ handleAdds st changechan cs
 
 {- Checks which of a set of files can safely be added.
  - Files are locked down as hard links in a temp directory,
- - with their write bits disabled. But some may have already
- - been opened for write, so lsof is run on the temp directory
+ - with their write bits disabled. But some may still be
+ - opened for write, so lsof is run on the temp directory
  - to check them.
  -}
 safeToAdd :: ThreadState -> [FilePath] -> IO [KeySource]
 safeToAdd st files = do
 	locked <- catMaybes <$> lockdown files
-	runThreadState st $ do
-		tmpdir <- fromRepo gitAnnexTmpDir
-		open <- S.fromList . map fst3 . filter openwrite <$>
-			liftIO (Lsof.queryDir tmpdir)
-		catMaybes <$> forM locked (go open)
+	runThreadState st $ ifM (Annex.getState Annex.force)
+		( return locked -- force bypasses lsof check
+		, do
+			tmpdir <- fromRepo gitAnnexTmpDir
+			open <- S.fromList . map fst3 . filter openwrite <$>
+				liftIO (Lsof.queryDir tmpdir)
+			catMaybes <$> forM locked (go open)
+		)
 	where
 		go open keysource
 			| S.member (contentLocation keysource) open = do

From 9232bc860f5b570876da5fbd740a1c33e140615e Mon Sep 17 00:00:00 2001
From: "http://www.davidhaslem.com/" 
Date: Tue, 19 Jun 2012 04:41:28 +0000
Subject: [PATCH 3838/8313] Added a comment

---
 ...ent_7_2ce7acab15403d3f993cec94ec7f3bc6._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment

diff --git a/doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment b/doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment
new file mode 100644
index 0000000000..32093ee51f
--- /dev/null
+++ b/doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://www.davidhaslem.com/"
+ nickname="David"
+ subject="comment 7"
+ date="2012-06-19T04:41:27Z"
+ content="""
+$(brew --prefix) should, in most cases, be /usr/local.  That's the recommended install location for homebrew.
+
+I already had git installed and homebrew as my package manager - my install steps were as follows:
+
+1. brew install haskell-platform ossp-uuid md5sha1sum coreutils pcre
+2. PATH=\"$(brew --prefix coreutils)/libexec/gnubin:$PATH\" cabal install git-annex
+
+"""]]

From 03b9341356c8d4eabfec5864957a4e49e7fcac67 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 04:52:55 +0000
Subject: [PATCH 3839/8313] fix scheduling

Handle kevent interruptions in the haskell code, so it can yield to other threads
---
 Utility/Kqueue.hs   | 17 ++++++++++++-----
 Utility/libkqueue.c | 14 ++++----------
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index da43a2d865..1f65b2dbaa 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -24,11 +24,13 @@ import Utility.Types.DirWatcher
 
 import System.Posix.Types
 import Foreign.C.Types
+import Foreign.C.Error
 import Foreign.Ptr
 import Foreign.Marshal
 import qualified Data.Map as M
 import qualified Data.Set as S
 import qualified System.Posix.Files as Files
+import Control.Concurrent
 
 data Change
 	= Deleted FilePath
@@ -146,9 +148,14 @@ stopKqueue (Kqueue h _ _) = closeFd h
 waitChange :: Kqueue -> IO (Kqueue, [Change])
 waitChange kq@(Kqueue h dirmap _) = do
 	changedfd <- c_waitchange_kqueue h
-	case M.lookup changedfd dirmap of
-		Nothing -> return (kq, [])
-		Just info -> handleChange kq changedfd info
+	if changedfd == -1
+		then ifM ((==) eINTR <$> getErrno)
+			(yield >> waitChange kq, nochange)
+		else case M.lookup changedfd dirmap of
+			Nothing -> nochange
+			Just info -> handleChange kq changedfd info
+	where
+		nochange = return (kq, [])
 
 {- The kqueue interface does not tell what type of change took place in
  - the directory; it could be an added file, a deleted file, a renamed
@@ -212,9 +219,9 @@ runHooks kq hooks = do
 			| Files.isSymbolicLink s = callhook addSymlinkHook (Just s) change
 			| Files.isDirectory s = print $ "TODO: recursive directory add: " ++ show change
 			| Files.isRegularFile s = callhook addHook (Just s) change
-			| otherwise = noop
+			| otherwise = print "not a file??"
 		callhook h s change = case h hooks of
-			Nothing -> noop
+			Nothing -> print "missing hook??"
 			Just a -> a (changedFile change) s
 		withstatus change a = maybe noop (a change) =<<
 			(catchMaybeIO (getSymbolicLinkStatus (changedFile change)))
diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index b7f9595dc1..643a63b978 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -17,6 +17,8 @@
 /* The specified fds are added to the set of fds being watched for changes.
  * Fds passed to prior calls still take effect, so it's most efficient to
  * not pass the same fds repeatedly.
+ *
+ * Returns the fd that changed, or -1 on error.
  */
 signed int helper(const int kq, const int fdcnt, const int *fdlist, int nodelay) {
 	int i, nev;
@@ -32,12 +34,7 @@ signed int helper(const int kq, const int fdcnt, const int *fdlist, int nodelay)
 			0, 0);
 	}
 
-	while ((nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout))) {
-		if (!(nev == -1 && errno == EINTR)) {
-			break;
-		}
-	}
-
+	nev = kevent(kq, chlist, fdcnt, evlist, 1, timeout);
 	if (nev == 1)
 		return evlist[0].ident;
 	else
@@ -59,10 +56,7 @@ void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist) {
 	helper(kq, fdcnt, fdlist, 1);
 }
 
-/* Waits for a change event on a kqueue.
- *
- * Returns the fd that changed, or -1 on error.
- */
+/* Waits for a change event on a kqueue. */
 signed int waitchange_kqueue(const int kq) {
 	return helper(kq, 0, NULL, 0);
 }

From fd3e94593224fe0e656c7bb1dc117db057575f4e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 01:56:36 -0400
Subject: [PATCH 3840/8313] fix prototype

---
 Utility/libkqueue.c | 2 +-
 Utility/libkqueue.h | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Utility/libkqueue.c b/Utility/libkqueue.c
index 643a63b978..b5a19a1350 100644
--- a/Utility/libkqueue.c
+++ b/Utility/libkqueue.c
@@ -42,7 +42,7 @@ signed int helper(const int kq, const int fdcnt, const int *fdlist, int nodelay)
 }
 
 /* Initializes a new, empty kqueue. */
-int init_kqueue(const int fdcnt, const int *fdlist) {
+int init_kqueue() {
 	int kq;
 	if ((kq = kqueue()) == -1) {
 		perror("kqueue");
diff --git a/Utility/libkqueue.h b/Utility/libkqueue.h
index 1a285b8dad..692b47f14e 100644
--- a/Utility/libkqueue.h
+++ b/Utility/libkqueue.h
@@ -1,2 +1,3 @@
-int init_kqueue(const int fdcnt, const int *fdlist);
+int init_kqueue();
+void addfds_kqueue(const int kq, const int fdcnt, const int *fdlist);
 signed int waitchange_kqueue(const int kq);

From 424e58d960e8b307e39e2856f99ed4635f3f7d31 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 02:16:31 -0400
Subject: [PATCH 3841/8313] update

---
 doc/design/assistant/inotify.mdwn | 40 +++++++------------------------
 1 file changed, 8 insertions(+), 32 deletions(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 9d3db9192c..7fe2a89ce3 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -13,6 +13,14 @@ There is a `watch` branch in git that adds the command.
 
 * When you `git annex unlock` a file, it will immediately be re-locked.
 
+* With kqueue, added files are not staged because the symlink change event
+  it's expecting rarely arrives. I think due to a race.
+
+* With kqueue, if a while is created and still has a writer, it'll
+  give up adding it, and it will never get added. This is because kqueue
+  cannot track file closes. Need to go back and check these files every
+  second or something.
+
 ## beyond Linux
 
 I'd also like to support OSX and if possible the BSDs.
@@ -58,40 +66,8 @@ I'd also like to support OSX and if possible the BSDs.
 
 * Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
 
-## beyond Linux
-
-I'd also like to support OSX and if possible the BSDs.
-
-* kqueue ([haskell bindings](http://hackage.haskell.org/package/kqueue))
-  is supported by FreeBSD, OSX, and other BSDs.
-
-  From what I can find, kqueue does not provide full directory watching
-  capabilities. To watch a file, you have to have an open file descriptor
-  to the file. This wouldn't scale.
-
-  Gamin does the best it can with just kqueue, supplimented by polling.
-  The source file `server/gam_kqueue.c` makes for interesting reading.
-  Using gamin to do the heavy lifting is one option. 
-  ([haskell bindings](http://hackage.haskell.org/package/hlibfam) for FAM;
-  gamin shares the API)
-
-* hfsevents ([haskell bindings](http://hackage.haskell.org/package/hfsevents))
-  is OSX specific.
-
-  Originally it was only directory level, and you were only told a
-  directory had changed and not which file. Based on the haskell
-  binding's code, from OSX 10.7.0, file level events were added.
-
-  This will be harder for me to develop for, since I don't have access to
-  OSX machines..
-
-* Windows has a Win32 ReadDirectoryChangesW, and perhaps other things.
-
 ## todo
 
-- Support OSes other than Linux; it only uses inotify currently.
-  OSX and FreeBSD use the same mechanism, and there is a Haskell interface
-  for it,
 - Run niced and ioniced? Seems to make sense, this is a background job.
 - configurable option to only annex files meeting certian size or
   filename criteria

From 4ab9449cee0cb1377a768b44fe832282ac1f88b9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 02:23:45 -0400
Subject: [PATCH 3842/8313] add eventsCoalesce

---
 Utility/DirWatcher.hs | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs
index 5750361903..bf184ff8a4 100644
--- a/Utility/DirWatcher.hs
+++ b/Utility/DirWatcher.hs
@@ -35,6 +35,24 @@ canWatch = True
 canWatch = False
 #endif
 
+/* With inotify, discrete events will be received when making multiple changes
+ * to the same filename. For example, adding it, deleting it, and adding it
+ * again will be three events.
+ * 
+ * OTOH, with kqueue, often only one event is received, indicating the most
+ * recent state of the file.
+ */
+eventsCoalesce :: Bool
+#if WITH_INOTIFY
+eventsCoalesce = False
+#else
+#if WITH_KQUEUE
+eventsCoalesce = True
+#else
+eventsCoalesce = undefined
+#endif
+#endif
+
 #if WITH_INOTIFY
 watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
 watchDir dir prune hooks runstartup = INotify.withINotify $ \i -> do

From 57cf65eb6d811ba7fd19eb62a54e3b83a0c2dfa7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 02:40:21 -0400
Subject: [PATCH 3843/8313] fix kevent symlink creation

---
 Assistant.hs               |  1 +
 Assistant/Changes.hs       | 59 ++++++++++++++++++++++++++++++++
 Assistant/Committer.hs     | 70 +++++++++-----------------------------
 Assistant/SanityChecker.hs |  2 +-
 Assistant/Watcher.hs       |  2 +-
 Command/Add.hs             |  6 ++--
 6 files changed, 83 insertions(+), 57 deletions(-)
 create mode 100644 Assistant/Changes.hs

diff --git a/Assistant.hs b/Assistant.hs
index 880d3eb5e3..e924d94777 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -46,6 +46,7 @@ module Assistant where
 import Common.Annex
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
+import Assistant.Changes
 import Assistant.Watcher
 import Assistant.Committer
 import Assistant.SanityChecker
diff --git a/Assistant/Changes.hs b/Assistant/Changes.hs
new file mode 100644
index 0000000000..1cad423265
--- /dev/null
+++ b/Assistant/Changes.hs
@@ -0,0 +1,59 @@
+{- git-annex assistant change tracking
+ -
+ - Copyright 2012 Joey Hess 
+ -}
+
+module Assistant.Changes where
+
+import Common.Annex
+import qualified Annex.Queue
+
+import Control.Concurrent.STM
+import Data.Time.Clock
+
+data ChangeType = PendingAddChange | LinkChange | RmChange | RmDirChange
+	deriving (Show, Eq)
+
+type ChangeChan = TChan Change
+
+data Change = Change
+	{ changeTime :: UTCTime
+	, changeFile :: FilePath
+	, changeType :: ChangeType
+	}
+	deriving (Show)
+
+runChangeChan :: STM a -> IO a
+runChangeChan = atomically
+
+newChangeChan :: IO ChangeChan
+newChangeChan = atomically newTChan
+
+{- Handlers call this when they made a change that needs to get committed. -}
+madeChange :: FilePath -> ChangeType -> Annex (Maybe Change)
+madeChange f t = do
+	-- Just in case the commit thread is not flushing the queue fast enough.
+	when (t /= PendingAddChange) $
+		Annex.Queue.flushWhenFull
+	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure f <*> pure t)
+
+noChange :: Annex (Maybe Change)
+noChange = return Nothing
+
+{- Gets all unhandled changes.
+ - Blocks until at least one change is made. -}
+getChanges :: ChangeChan -> IO [Change]
+getChanges chan = runChangeChan $ do
+	c <- readTChan chan
+	go [c]
+	where
+		go l = do
+			v <- tryReadTChan chan
+			case v of
+				Nothing -> return l
+				Just c -> go (c:l)
+
+{- Puts unhandled changes back into the channel.
+ - Note: Original order is not preserved. -}
+refillChanges :: ChangeChan -> [Change] -> IO ()
+refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs
diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index b482e5e7a0..d3f7f15c52 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -1,4 +1,4 @@
-{- git-annex assistant change tracking and committing
+{- git-annex assistant commit thread
  -
  - Copyright 2012 Joey Hess 
  -}
@@ -6,67 +6,24 @@
 module Assistant.Committer where
 
 import Common.Annex
+import Assistant.Changes
 import Assistant.ThreadedMonad
+import Assistant.Watcher
 import qualified Annex
 import qualified Annex.Queue
 import qualified Git.Command
+import qualified Git.HashObject
+import Git.Types
 import qualified Command.Add
 import Utility.ThreadScheduler
 import qualified Utility.Lsof as Lsof
+import qualified Utility.DirWatcher as DirWatcher
 import Types.Backend
 
-import Control.Concurrent.STM
 import Data.Time.Clock
 import Data.Tuple.Utils
 import qualified Data.Set as S
 
-data ChangeType = PendingAddChange | LinkChange | RmChange | RmDirChange
-	deriving (Show, Eq)
-
-type ChangeChan = TChan Change
-
-data Change = Change
-	{ changeTime :: UTCTime
-	, changeFile :: FilePath
-	, changeType :: ChangeType
-	}
-	deriving (Show)
-
-runChangeChan :: STM a -> IO a
-runChangeChan = atomically
-
-newChangeChan :: IO ChangeChan
-newChangeChan = atomically newTChan
-
-{- Handlers call this when they made a change that needs to get committed. -}
-madeChange :: FilePath -> ChangeType -> Annex (Maybe Change)
-madeChange f t = do
-	-- Just in case the commit thread is not flushing the queue fast enough.
-	when (t /= PendingAddChange) $
-		Annex.Queue.flushWhenFull
-	liftIO $ Just <$> (Change <$> getCurrentTime <*> pure f <*> pure t)
-
-noChange :: Annex (Maybe Change)
-noChange = return Nothing
-
-{- Gets all unhandled changes.
- - Blocks until at least one change is made. -}
-getChanges :: ChangeChan -> IO [Change]
-getChanges chan = runChangeChan $ do
-	c <- readTChan chan
-	go [c]
-	where
-		go l = do
-			v <- tryReadTChan chan
-			case v of
-				Nothing -> return l
-				Just c -> go (c:l)
-
-{- Puts unhandled changes back into the channel.
- - Note: Original order is not preserved. -}
-refillChanges :: ChangeChan -> [Change] -> IO ()
-refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs
-
 {- This thread makes git commits at appropriate times. -}
 commitThread :: ThreadState -> ChangeChan -> IO ()
 commitThread st changechan = runEvery (Seconds 1) $ do
@@ -122,7 +79,9 @@ shouldCommit now changes
  -
  - When a file is added, Inotify will notice the new symlink. So this waits
  - for additional Changes to arrive, so that the symlink has hopefully been
- - staged before returning, and will be committed.
+ - staged before returning, and will be committed immediately. OTOH, for
+ - kqueue, eventsCoalesce, so instead the symlink is directly created and
+ - staged.
  -}
 handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO ()
 handleAdds st changechan cs
@@ -131,8 +90,9 @@ handleAdds st changechan cs
 		toadd' <- safeToAdd st toadd
 		unless (null toadd') $ do
 			added <- filter id <$> forM toadd' add
-			unless (null added) $
-				handleAdds st changechan =<< getChanges changechan
+			when (DirWatcher.eventsCoalesce && not (null added)) $
+				handleAdds st changechan
+					=<< getChanges changechan
 	where
 		toadd = map changeFile $ filter isPendingAdd cs
 
@@ -148,7 +108,11 @@ handleAdds st changechan cs
 			showEndFail
 			return False
 		handle file (Just key) = do
-			Command.Add.link file key True
+			link <- Command.Add.link file key True
+			when DirWatcher.eventsCoalesce $ do
+				sha <- inRepo $
+					Git.HashObject.hashObject BlobObject link
+				stageSymlink file sha
 			showEndOk
 			return True
 
diff --git a/Assistant/SanityChecker.hs b/Assistant/SanityChecker.hs
index a5f1380248..e2ca9da740 100644
--- a/Assistant/SanityChecker.hs
+++ b/Assistant/SanityChecker.hs
@@ -11,7 +11,7 @@ import Common.Annex
 import qualified Git.LsFiles
 import Assistant.DaemonStatus
 import Assistant.ThreadedMonad
-import Assistant.Committer
+import Assistant.Changes
 import Utility.ThreadScheduler
 import qualified Assistant.Watcher
 
diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs
index a2ca2396e4..cb7ede9209 100644
--- a/Assistant/Watcher.hs
+++ b/Assistant/Watcher.hs
@@ -12,7 +12,7 @@ module Assistant.Watcher where
 import Common.Annex
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
-import Assistant.Committer
+import Assistant.Changes
 import Utility.DirWatcher
 import Utility.Types.DirWatcher
 import qualified Annex.Queue
diff --git a/Command/Add.hs b/Command/Add.hs
index 7a66960632..43f186fbf0 100644
--- a/Command/Add.hs
+++ b/Command/Add.hs
@@ -97,8 +97,8 @@ undo file key e = do
 			src <- inRepo $ gitAnnexLocation key
 			liftIO $ moveFile src file
 
-{- Creates the symlink to the annexed content. -}
-link :: FilePath -> Key -> Bool -> Annex ()
+{- Creates the symlink to the annexed content, returns the link target. -}
+link :: FilePath -> Key -> Bool -> Annex String
 link file key hascontent = handle (undo file key) $ do
 	l <- calcGitLink file key
 	liftIO $ createSymbolicLink l file
@@ -112,6 +112,8 @@ link file key hascontent = handle (undo file key) $ do
 			mtime <- modificationTime <$> getFileStatus file
 			touch file (TimeSpec mtime) False
 
+	return l
+
 {- Note: Several other commands call this, and expect it to 
  - create the symlink and add it. -}
 cleanup :: FilePath -> Key -> Bool -> CommandCleanup

From d4e5affec03fce1f6a169a5dc29403393422514d Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 19 Jun 2012 07:08:34 +0000
Subject: [PATCH 3844/8313]

---
 doc/bugs/watch_command_on_OSX_10.7.mdwn | 30 +++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 doc/bugs/watch_command_on_OSX_10.7.mdwn

diff --git a/doc/bugs/watch_command_on_OSX_10.7.mdwn b/doc/bugs/watch_command_on_OSX_10.7.mdwn
new file mode 100644
index 0000000000..244c624f22
--- /dev/null
+++ b/doc/bugs/watch_command_on_OSX_10.7.mdwn
@@ -0,0 +1,30 @@
+Running the tip of the watch branch on OSX in an annex'ed directory.
+
+The watch command detects the changes, does _something_, see the output below.
+
+Output from watch command
+
+
+(Recording state in git...)
+Added "./KeePass2.18.dmg"
+Added "./KeePassX-0.4.3.dmg"
+add ./KeePass2.18.dmg (checksum...) ok
+add ./KeePassX-0.4.3.dmg (checksum...) ok
+

+
+State of the annex
+
+
+laplace:annex jtang$ git status
+# On branch master
+# Untracked files:
+#   (use "git add ..." to include in what will be committed)
+#
+#	KeePass2.18.dmg
+#	KeePassX-0.4.3.dmg
+nothing added to commit but untracked files present (use "git add" to track)
+

+
+It seems to not do a git add and commit afterwards of the symlinks, manually doing this makes it all happy again till more files are added.
+
+note: i had posted a commend in the blog post, but posting the issue here is probably more appropriate.

From e986d1acb1c335958805797e40a2ef79bf1e1e43 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo"
 
Date: Tue, 19 Jun 2012 07:08:44 +0000
Subject: [PATCH 3845/8313]

---
 doc/forum/Wishlist:_automatic_reinject.mdwn | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/forum/Wishlist:_automatic_reinject.mdwn

diff --git a/doc/forum/Wishlist:_automatic_reinject.mdwn b/doc/forum/Wishlist:_automatic_reinject.mdwn
new file mode 100644
index 0000000000..f975c75214
--- /dev/null
+++ b/doc/forum/Wishlist:_automatic_reinject.mdwn
@@ -0,0 +1,14 @@
+I think it would be useful to supplement the `reinject` command with an automatic
+mode which calculates the checksum of the source file and injects the file if it
+is known to the repository (without the need to provide a destination filename).
+In addition, this could be done recursively if the user provides a directory to
+inject. All this can probably be done already with some plumbing, but a simple
+`reinject --auto` (or `scour`, or `scavenge`, if you like) would be a nice addition.
+Of course this would only work for the checksum backends.
+
+Example use cases would be:
+
+* Recovering data from lost+found easily
+* Making use of old (pre-git-annex) archival volumes with useful files
+  scattered among non-useful files
+* Sneaker-netting files between disconnected git-annex repositories

From 99a3d6a00c9490d28d5df5603086798b2a7d4a3e Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 19 Jun 2012 07:09:09 +0000
Subject: [PATCH 3846/8313]

---
 doc/bugs/watch_command_on_OSX_10.7.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/watch_command_on_OSX_10.7.mdwn b/doc/bugs/watch_command_on_OSX_10.7.mdwn
index 244c624f22..aa7c341a5f 100644
--- a/doc/bugs/watch_command_on_OSX_10.7.mdwn
+++ b/doc/bugs/watch_command_on_OSX_10.7.mdwn
@@ -25,6 +25,6 @@ laplace:annex jtang$ git status
 nothing added to commit but untracked files present (use "git add" to track)
 

 
-It seems to not do a git add and commit afterwards of the symlinks, manually doing this makes it all happy again till more files are added.
+It seems to not do a git add and commit afterw the creation of the symlinks, manually doing this makes it all happy again till more files are added.
 
 note: i had posted a commend in the blog post, but posting the issue here is probably more appropriate.

From 27b5fc9fd4b86ce206f925a95717a13de457dbf2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 09:11:04 -0400
Subject: [PATCH 3847/8313] yeah

---
 doc/bugs/watch_command_on_OSX_10.7.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/watch_command_on_OSX_10.7.mdwn b/doc/bugs/watch_command_on_OSX_10.7.mdwn
index aa7c341a5f..2f8aac7d5f 100644
--- a/doc/bugs/watch_command_on_OSX_10.7.mdwn
+++ b/doc/bugs/watch_command_on_OSX_10.7.mdwn
@@ -28,3 +28,7 @@ nothing added to commit but untracked files present (use "git add" to track)
 It seems to not do a git add and commit afterw the creation of the symlinks, manually doing this makes it all happy again till more files are added.
 
 note: i had posted a commend in the blog post, but posting the issue here is probably more appropriate.
+
+> Yeah, this is the issue I was struggling with last night. 
+> I think it's fixed in 57cf65eb6d811ba7fd19eb62a54e3b83a0c2dfa7,
+> but the kqueue watch still needs a lot of work. --[[Joey]]

From 627504744c80c8a7b3f4b43e3646a5ad5c35d92f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 09:17:06 -0400
Subject: [PATCH 3848/8313] inverted logic

---
 Assistant/Committer.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs
index d3f7f15c52..74f0922b73 100644
--- a/Assistant/Committer.hs
+++ b/Assistant/Committer.hs
@@ -90,7 +90,7 @@ handleAdds st changechan cs
 		toadd' <- safeToAdd st toadd
 		unless (null toadd') $ do
 			added <- filter id <$> forM toadd' add
-			when (DirWatcher.eventsCoalesce && not (null added)) $
+			unless (DirWatcher.eventsCoalesce || null added) $
 				handleAdds st changechan
 					=<< getChanges changechan
 	where

From d832a474e9f0f9bda2bc2438e7e037fb16a637c7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 09:19:40 -0400
Subject: [PATCH 3849/8313] fixed

---
 doc/bugs/watch_command_on_OSX_10.7.mdwn | 3 +++
 doc/design/assistant/inotify.mdwn       | 3 ---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/bugs/watch_command_on_OSX_10.7.mdwn b/doc/bugs/watch_command_on_OSX_10.7.mdwn
index 2f8aac7d5f..a522c504a6 100644
--- a/doc/bugs/watch_command_on_OSX_10.7.mdwn
+++ b/doc/bugs/watch_command_on_OSX_10.7.mdwn
@@ -32,3 +32,6 @@ note: i had posted a commend in the blog post, but posting the issue here is pro
 > Yeah, this is the issue I was struggling with last night. 
 > I think it's fixed in 57cf65eb6d811ba7fd19eb62a54e3b83a0c2dfa7,
 > but the kqueue watch still needs a lot of work. --[[Joey]]
+
+>> Confirmed this is fixed, but do note the known kqueue bugs in 
+>> [[design/assistant/inotify]]! [[done]] --[[Joey]]
diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 7fe2a89ce3..2520ddcf53 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -13,9 +13,6 @@ There is a `watch` branch in git that adds the command.
 
 * When you `git annex unlock` a file, it will immediately be re-locked.
 
-* With kqueue, added files are not staged because the symlink change event
-  it's expecting rarely arrives. I think due to a race.
-
 * With kqueue, if a while is created and still has a writer, it'll
   give up adding it, and it will never get added. This is because kqueue
   cannot track file closes. Need to go back and check these files every

From 2a61df23e72ed4880f8927e6094acd9b256bb13b Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 09:56:03 -0400
Subject: [PATCH 3850/8313] kqueue recursive directory adding

---
 Utility/Kqueue.hs | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index 1f65b2dbaa..a0edcb5a99 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -48,7 +48,11 @@ changedFile :: Change -> FilePath
 changedFile (Added f) = f
 changedFile (Deleted f) = f
 
-data Kqueue = Kqueue Fd DirMap Pruner
+data Kqueue = Kqueue 
+	{ kqueueFd :: Fd
+	, kqueueMap :: DirMap
+	, kqueuePruner :: Pruner
+	}
 
 type Pruner = FilePath -> Bool
 
@@ -115,6 +119,13 @@ removeSubDir dirmap dir = do
 	where
 		(toremove, rest) = M.partition (dirContains dir . dirName) dirmap
 
+findDirContents :: DirMap -> FilePath -> [FilePath]
+findDirContents dirmap dir = concatMap absolutecontents $ search
+	where
+		absolutecontents i = map (dirName i ) (S.toList $ dirCache i)
+		search = map snd $ M.toList $
+			M.filter (\i -> dirName i == dir) dirmap
+
 foreign import ccall unsafe "libkqueue.h init_kqueue" c_init_kqueue
 	:: IO Fd
 foreign import ccall unsafe "libkqueue.h addfds_kqueue" c_addfds_kqueue
@@ -140,7 +151,7 @@ updateKqueue (Kqueue h dirmap _) =
 {- Stops a Kqueue. Note: Does not directly close the Fds in the dirmap,
  - so it can be reused.  -}
 stopKqueue :: Kqueue -> IO ()
-stopKqueue (Kqueue h _ _) = closeFd h
+stopKqueue = closeFd . kqueueFd
 
 {- Waits for a change on a Kqueue.
  - May update the Kqueue.
@@ -206,22 +217,30 @@ runHooks kq hooks = do
 	(kq', changes) <- waitChange kq
 	forM_ changes $ \c -> do
 		print c
-		dispatch kq' c
+		dispatch (kqueueMap kq') c
 	runHooks kq' hooks
 	where
 		-- Kqueue returns changes for both whole directories
 		-- being added and deleted, and individual files being
 		-- added and deleted.
-		dispatch q change
-			| isAdd change = withstatus change $ dispatchadd q
+		dispatch dirmap change
+			| isAdd change = withstatus change $ dispatchadd dirmap
 			| otherwise = callhook delDirHook Nothing change
-		dispatchadd q change s
-			| Files.isSymbolicLink s = callhook addSymlinkHook (Just s) change
-			| Files.isDirectory s = print $ "TODO: recursive directory add: " ++ show change
-			| Files.isRegularFile s = callhook addHook (Just s) change
-			| otherwise = print "not a file??"
+		dispatchadd dirmap change s
+			| Files.isSymbolicLink s =
+				callhook addSymlinkHook (Just s) change
+			| Files.isDirectory s = do
+				-- Recursively add directory contents.
+				let contents = findDirContents dirmap $
+					changedFile change
+				forM_ contents $ \f ->
+					withstatus (Added f) $
+						dispatchadd dirmap
+			| Files.isRegularFile s =
+				callhook addHook (Just s) change
+			| otherwise = noop
 		callhook h s change = case h hooks of
-			Nothing -> print "missing hook??"
+			Nothing -> noop
 			Just a -> a (changedFile change) s
 		withstatus change a = maybe noop (a change) =<<
 			(catchMaybeIO (getSymbolicLinkStatus (changedFile change)))

From e68b3c99f44a00cb6e5c405115746b6bbad1e2cc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 10:08:06 -0400
Subject: [PATCH 3851/8313] kqueue synthetic add events on startup

---
 Utility/Kqueue.hs | 40 +++++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index a0edcb5a99..7e7e653ec3 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -50,8 +50,9 @@ changedFile (Deleted f) = f
 
 data Kqueue = Kqueue 
 	{ kqueueFd :: Fd
+	, kqueueTop :: FilePath
 	, kqueueMap :: DirMap
-	, kqueuePruner :: Pruner
+	, _kqueuePruner :: Pruner
 	}
 
 type Pruner = FilePath -> Bool
@@ -138,13 +139,13 @@ initKqueue :: FilePath -> Pruner -> IO Kqueue
 initKqueue dir pruned = do
 	dirmap <- scanRecursive dir pruned
 	h <- c_init_kqueue
-	let kq = Kqueue h dirmap pruned
+	let kq = Kqueue h dir dirmap pruned
 	updateKqueue kq
 	return kq
 
 {- Updates a Kqueue, adding watches for its map. -}
 updateKqueue :: Kqueue -> IO ()
-updateKqueue (Kqueue h dirmap _) =
+updateKqueue (Kqueue h _ dirmap _) =
 	withArrayLen (M.keys dirmap) $ \fdcnt c_fds -> do
 		c_addfds_kqueue h (fromIntegral fdcnt) c_fds
 
@@ -157,7 +158,7 @@ stopKqueue = closeFd . kqueueFd
  - May update the Kqueue.
  -}
 waitChange :: Kqueue -> IO (Kqueue, [Change])
-waitChange kq@(Kqueue h dirmap _) = do
+waitChange kq@(Kqueue h _ dirmap _) = do
 	changedfd <- c_waitchange_kqueue h
 	if changedfd == -1
 		then ifM ((==) eINTR <$> getErrno)
@@ -178,7 +179,7 @@ waitChange kq@(Kqueue h dirmap _) = do
  - directories as necessary.
  -}
 handleChange :: Kqueue -> Fd -> DirInfo -> IO (Kqueue, [Change])
-handleChange (Kqueue h dirmap pruner) fd olddirinfo =
+handleChange kq@(Kqueue _ _ dirmap pruner) fd olddirinfo =
 	go =<< catchMaybeIO (getDirInfo $ dirName olddirinfo)
 	where
 		go (Just newdirinfo) = do
@@ -199,7 +200,7 @@ handleChange (Kqueue h dirmap pruner) fd olddirinfo =
 
 			-- When new directories were added, need to update
 			-- the kqueue to watch them.
-			let kq' = Kqueue h newmap'' pruner
+			let kq' = kq { kqueueMap = newmap'' }
 			unless (null newdirinfos) $
 				updateKqueue kq'
 
@@ -208,18 +209,21 @@ handleChange (Kqueue h dirmap pruner) fd olddirinfo =
 			-- The directory has been moved or deleted, so
 			-- remove it from our map.
 			newmap <- removeSubDir dirmap (dirName olddirinfo)
-			return (Kqueue h newmap pruner, [])
+			return (kq { kqueueMap = newmap }, [])
 
 {- Processes changes on the Kqueue, calling the hooks as appropriate.
  - Never returns. -}
 runHooks :: Kqueue -> WatchHooks -> IO ()
 runHooks kq hooks = do
-	(kq', changes) <- waitChange kq
-	forM_ changes $ \c -> do
-		print c
-		dispatch (kqueueMap kq') c
-	runHooks kq' hooks
+	-- First, synthetic add events for the whole directory tree contents,
+	-- to catch any files created beforehand.
+	recursiveadd (kqueueMap kq) (Added $ kqueueTop kq)
+	loop kq
 	where
+		loop q = do
+			(q', changes) <- waitChange q
+			forM_ changes $ dispatch (kqueueMap q')
+			loop q'
 		-- Kqueue returns changes for both whole directories
 		-- being added and deleted, and individual files being
 		-- added and deleted.
@@ -229,16 +233,14 @@ runHooks kq hooks = do
 		dispatchadd dirmap change s
 			| Files.isSymbolicLink s =
 				callhook addSymlinkHook (Just s) change
-			| Files.isDirectory s = do
-				-- Recursively add directory contents.
-				let contents = findDirContents dirmap $
-					changedFile change
-				forM_ contents $ \f ->
-					withstatus (Added f) $
-						dispatchadd dirmap
+			| Files.isDirectory s = recursiveadd dirmap change
 			| Files.isRegularFile s =
 				callhook addHook (Just s) change
 			| otherwise = noop
+		recursiveadd dirmap change = do
+			let contents = findDirContents dirmap $ changedFile change
+			forM_ contents $ \f ->
+				withstatus (Added f) $ dispatchadd dirmap
 		callhook h s change = case h hooks of
 			Nothing -> noop
 			Just a -> a (changedFile change) s

From 39746e30dc09bd3c6124d9f72f189d4e8c4b8de3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 10:11:28 -0400
Subject: [PATCH 3852/8313] typo

---
 doc/design/assistant/inotify.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 2520ddcf53..234a52a65a 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -13,7 +13,7 @@ There is a `watch` branch in git that adds the command.
 
 * When you `git annex unlock` a file, it will immediately be re-locked.
 
-* With kqueue, if a while is created and still has a writer, it'll
+* With kqueue, if a file is created and still has a writer, it'll
   give up adding it, and it will never get added. This is because kqueue
   cannot track file closes. Need to go back and check these files every
   second or something.

From 8bdb331b60d44ad0ea1d01fe355044463e0b9af1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 10:17:06 -0400
Subject: [PATCH 3853/8313] bug

---
 doc/design/assistant/inotify.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn
index 234a52a65a..0b55298a30 100644
--- a/doc/design/assistant/inotify.mdwn
+++ b/doc/design/assistant/inotify.mdwn
@@ -18,6 +18,9 @@ There is a `watch` branch in git that adds the command.
   cannot track file closes. Need to go back and check these files every
   second or something.
 
+* Kqueue has to open every directory it watches, so too many directories
+  will run it out of the max number of open files (typically 1024), and fail.
+
 ## beyond Linux
 
 I'd also like to support OSX and if possible the BSDs.

From 5580af5789427fc5fd7cd74fd4a2529668621a68 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 19 Jun 2012 10:22:36 -0400
Subject: [PATCH 3854/8313] add closingTracked flag

---
 Utility/DirWatcher.hs | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs
index bf184ff8a4..baab244c71 100644
--- a/Utility/DirWatcher.hs
+++ b/Utility/DirWatcher.hs
@@ -53,6 +53,25 @@ eventsCoalesce = undefined
 #endif
 #endif
 
+/* With inotify, file closing is tracked to some extent, so an add event
+ * will always be received for a file once its writer closes it, and
+ * (typically) not before. This may mean multiple add events for the same file.
+ *
+ * OTOH, with kqueue, add events will often be received while a file is
+ * still being written to, and then no add event will be received once the
+ * writer closes it.
+ */
+closingTracked :: Bool
+#if WITH_INOTIFY
+closingTracked = True
+#else
+#if WITH_KQUEUE
+closingTracked = False
+#else
+eventsCoalesce = undefined
+#endif
+#endif
+
 #if WITH_INOTIFY
 watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO ()
 watchDir dir prune hooks runstartup = INotify.withINotify $ \i -> do

From 23b08efa426a1aff010a2af13e648d704b5a92bf Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 19 Jun 2012 22:28:08 +0000
Subject: [PATCH 3855/8313]

---
 doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
index 8cd370937c..2c1280e514 100644
--- a/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development.mdwn
@@ -31,4 +31,4 @@ if [ "$?" = 1 ]; then
 fi
 

 
-It's also using the branches-local script for sorting and prioritising the branches to build, this branches-local script can be found at the [autobuild-ceph](https://github.com/ceph/autobuild-ceph/blob/master/branches-local) repository. If there are other people interested in setting up their own instances of gitbuilder for git-annex, please let me know and I will setup an aggregator page to collect status of the builds. The builder runs and updates the webpage every 30mins.
+It's also using the branches-local script for sorting and prioritising the branches to build, this branches-local script can be found at the [autobuild-ceph](https://github.com/ceph/autobuild-ceph/blob/master/branches-local) repository. If there are other people interested in setting up their own instances of gitbuilder for git-annex, please let me know and I will setup an aggregator page to collect status of the builds. The builder runs and updates on a very regular basis.

From c838a35d832c9f570ea3cbaf08abc53cd713ea80 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 19 Jun 2012 22:28:50 +0000
Subject: [PATCH 3856/8313]

---
 doc/bugs/watch_command_on_OSX_10.7.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/bugs/watch_command_on_OSX_10.7.mdwn b/doc/bugs/watch_command_on_OSX_10.7.mdwn
index a522c504a6..1348c1155d 100644
--- a/doc/bugs/watch_command_on_OSX_10.7.mdwn
+++ b/doc/bugs/watch_command_on_OSX_10.7.mdwn
@@ -25,9 +25,9 @@ laplace:annex jtang$ git status
 nothing added to commit but untracked files present (use "git add" to track)
-It seems to not do a git add and commit afterw the creation of the symlinks, manually doing this makes it all happy again till more files are added. +It seems to not do a git add and commit after the creation of the symlinks, manually doing this makes it all happy again till more files are added. -note: i had posted a commend in the blog post, but posting the issue here is probably more appropriate. +note: i had posted a comment in the blog post, but posting the issue here is probably more appropriate. > Yeah, this is the issue I was struggling with last night. > I think it's fixed in 57cf65eb6d811ba7fd19eb62a54e3b83a0c2dfa7, From 2a079a9d378b7c9fd9bab4b6c75efb1cb39dddf4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 19 Jun 2012 18:32:30 -0400 Subject: [PATCH 3857/8313] blog for the day --- .../blog/day_13__kqueue_continued.mdwn | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 doc/design/assistant/blog/day_13__kqueue_continued.mdwn diff --git a/doc/design/assistant/blog/day_13__kqueue_continued.mdwn b/doc/design/assistant/blog/day_13__kqueue_continued.mdwn new file mode 100644 index 0000000000..fd0cbb372b --- /dev/null +++ b/doc/design/assistant/blog/day_13__kqueue_continued.mdwn @@ -0,0 +1,34 @@ +Good news! My beta testers report that the new kqueue code works on OSX. +At least "works" as well as it does on Debian kFreeBSD. My crazy +development strategy of developing on Debian kFreeBSD while targeting Mac +OSX is vindicated. ;-) + +So, I've been beating the kqueue code into shape for the last 12 hours, +minus a few hours sleep. + +First, I noticed it was seeming to starve the other threads. I'm using +Haskell's non-threaded runtime, which does cooperative multitasking between +threads, and my C code was never returning to let the other threads run. +Changed that around, so the C code runs until SIGALARMed, and then that +thread calls `yield` before looping back into the C code. Wow, cooperative +multitasking.. I last dealt with that when programming for Windows 3.1! +(Should try to use Haskell's -threaded runtime sometime, but git-annex +doesn't work under it, and I have not tried to figure out why not.) + +Then I made a [single commit](http://source.git-annex.branchable.com/?p=source.git;a=commitdiff;h=2bfcc0b09c5dd37c5e0ab65cb089232bfcc31934), +with no testing, in which I made the kqueue code maintain a cache of what +it expects in the directory tree, and use that to determine what files +changed how when a change is detected. Serious code. It worked on the +first go. If you were wondering why I'm writing in Haskell ... yeah, +that's why. + +And I've continued to hammer on the kqueue code, making lots of little +fixes, and at this point it seems *almost* able to handle the changes I +throw at it. It does have one big remaining problem; kqueue doesn't tell me +when a writer closes a file, so it will sometimes miss adding files. To fix +this, I'm going to need to make it maintain a queue of new files, and +periodically check them, with `lsof`, to see when they're done being +written to, and add them to the annex. So while a file is being written +to, `git annex watch` will have to wake up every second or so, and run +`lsof` ... and it'll take it at least 1 second to notice a file's complete. +Not ideal, but the best that can be managed with kqueue. From 8dbd36ab0aae3f3bcc864f9f79ff72c553ad2b8d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" Date: Wed, 20 Jun 2012 11:12:11 +0000 Subject: [PATCH 3858/8313] --- ...d_aborts_due_to_filename_encoding_problems.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn new file mode 100644 index 0000000000..eb87eaacf1 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn @@ -0,0 +1,14 @@ +What steps will reproduce the problem? +I don't know exactly when it started + +What is the expected output? What do you see instead? +When I run git annex unused I get + +unused . (checking for unused data...) (checking master...) git-annex: Cannot decode byte '\xb4': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + +Most likely I have added some file with a strange encoding that git-annex can't decode. The problem is that the unused process aborts because of this. + +What version of git-annex are you using? On what operating system? + 3.20120522, Debian testing + +Please provide any additional information below. From befa378ef7b9cf71e5c497dd35e2cdce68941bee Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" Date: Wed, 20 Jun 2012 11:12:48 +0000 Subject: [PATCH 3859/8313] --- ...t_annex_unused_aborts_due_to_filename_encoding_problems.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn index eb87eaacf1..d134013a48 100644 --- a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn @@ -4,7 +4,7 @@ I don't know exactly when it started What is the expected output? What do you see instead? When I run git annex unused I get -unused . (checking for unused data...) (checking master...) git-annex: Cannot decode byte '\xb4': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + unused . (checking for unused data...) (checking master...) git-annex: Cannot decode byte '\xb4': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream Most likely I have added some file with a strange encoding that git-annex can't decode. The problem is that the unused process aborts because of this. From 5976271390bab560fad08d06bd8f029c874f3c41 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 20 Jun 2012 14:30:27 +0000 Subject: [PATCH 3860/8313] Added a comment --- .../comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment new file mode 100644 index 0000000000..ddea8225e2 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_1_8ba4fdb9f2d3bd44db5e910526cb9124._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 1" + date="2012-06-20T14:30:27Z" + content=""" +Try running `git annex unused --debug`; this will tell us the git command that's outputing the data it cannot process. Then you can try running that git command and see what the problem filename is. +"""]] From 5774f1495864e3bd96b5811c58b3d1416fa97645 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 20 Jun 2012 14:34:23 +0000 Subject: [PATCH 3861/8313] Added a comment --- .../comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment new file mode 100644 index 0000000000..8afe3143ce --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_2_2a4a2b3e287a0444a1c8e8d98768a206._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 2" + date="2012-06-20T14:34:23Z" + content=""" +Your `locale` setting may also be relevant. FWIW, I've tried to create a file with `\xb4` in its name and have not gotten git-annex unused to crash on it. +"""]] From e711f20c8f8211767eb2d0763703b71e6636c2ae Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" Date: Wed, 20 Jun 2012 14:37:10 +0000 Subject: [PATCH 3862/8313] Added a comment --- ..._3_dacfdb8322045fc4ceefc9128bf7c505._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment new file mode 100644 index 0000000000..8e2aa285a9 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_3_dacfdb8322045fc4ceefc9128bf7c505._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" + nickname="Kristian" + subject="comment 3" + date="2012-06-20T14:37:09Z" + content=""" +This is what happens when I add the debug parameter + +git annex unused --debug + +unused . (checking for unused data...) git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"/home/kristian/AnnexMedia\"] +git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"show-ref\"] +(checking master...) git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"ls-tree\",\"--full-tree\",\"-z\",\"-r\",\"--\",\"refs/heads/master\"] +git [\"--git-dir=/home/kristian/AnnexMedia/.git\",\"--work-tree=/home/kristian/AnnexMedia\",\"cat-file\",\"--batch\"] +git-annex: Cannot decode byte '\xb4': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + +"""]] From 568f8d02733111b193f39947b5fd1ed889f1ff60 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 20 Jun 2012 14:49:09 +0000 Subject: [PATCH 3863/8313] Added a comment --- ...comment_4_7889a3ff5ce80c6322448aa674df8525._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment new file mode 100644 index 0000000000..da97b12f7d --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_4_7889a3ff5ce80c6322448aa674df8525._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 4" + date="2012-06-20T14:49:09Z" + content=""" +Ah, reproduced it; need to use the WORM backend and have the file present in another branch.. + + +"""]] From 77bcc76e87bb745955b18d6aed6d1d53c9b45735 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" Date: Wed, 20 Jun 2012 14:55:33 +0000 Subject: [PATCH 3864/8313] Added a comment --- ..._6d28c2537ce24eeb3496ca349823defd._comment | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment new file mode 100644 index 0000000000..fafd1d248c --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_5_6d28c2537ce24eeb3496ca349823defd._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" + nickname="Kristian" + subject="comment 5" + date="2012-06-20T14:55:33Z" + content=""" +I checkout out the git annex branch and using + + find * | grep -P \"[\xb4]\" + +I found a file + + 43e/b16/WORM-s4118528-m1245167306--Jerry Lee Lewis - Whole Lotta Shakin\302\264 Going\302\264 On.mp3.log + +The corresponding file also existed in the master branch (as a link). + +I moved both these files to a folder outside my repository and synched my git-annex branch with by master server. I still get the same error. Is there any other place where information about this file is stored? + +"""]] From ee2acd474fbefb3851d168dec474c83475cc917c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 12:51:25 -0400 Subject: [PATCH 3865/8313] [Word8] to filesystem encoded String My, GHC makes this hard. --- Utility/FileSystemEncoding.hs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs index cf1a6a731e..d027ede48d 100644 --- a/Utility/FileSystemEncoding.hs +++ b/Utility/FileSystemEncoding.hs @@ -13,6 +13,8 @@ import Foreign.C import System.IO import System.IO.Unsafe import qualified Data.Hash.MD5 as MD5 +import Data.Word +import Data.Bits.Utils {- Sets a Handle to use the filesystem encoding. This causes data - written or read from it to be encoded/decoded the same @@ -29,7 +31,7 @@ withFilePath :: FilePath -> (CString -> IO a) -> IO a withFilePath fp f = Encoding.getFileSystemEncoding >>= \enc -> GHC.withCString enc fp f -{- Encodes a FilePath into a Str, applying the filesystem encoding. +{- Encodes a FilePath into a Md5.Str, applying the filesystem encoding. - - This use of unsafePerformIO is belived to be safe; GHC's interface - only allows doing this conversion with CStrings, and the CString buffer @@ -41,3 +43,15 @@ encodeFilePath :: FilePath -> MD5.Str encodeFilePath fp = MD5.Str $ unsafePerformIO $ do enc <- Encoding.getFileSystemEncoding GHC.withCString enc fp $ GHC.peekCString Encoding.char8 + +{- Converts a [Word8] to a FilePath, encoding using the filesystem encoding. + - + - w82c produces a String, which may contain Chars that are invalid + - unicode. From there, this is really a simple matter of applying the + - file system encoding, only complicated by GHC's interface to doing so. + -} +{-# NOINLINE encodeW8 #-} +encodeW8 :: [Word8] -> FilePath +encodeW8 w8 = unsafePerformIO $ do + enc <- Encoding.getFileSystemEncoding + GHC.withCString Encoding.char8 (w82s w8) $ GHC.peekCString enc From dfccee26168d1ecca34dc504812f24a23b7633ed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 12:57:00 -0400 Subject: [PATCH 3866/8313] unused: Fix crash when file names contain invalid utf8. Was decoding the git-cat-file of the symlink target as utf8, but that can't do, unix filenames are from the 70's and need this shiny disco fileSystemEncoding. --- Command/Unused.hs | 8 ++++---- debian/changelog | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Command/Unused.hs b/Command/Unused.hs index 03a709534e..b115eee831 100644 --- a/Command/Unused.hs +++ b/Command/Unused.hs @@ -10,8 +10,7 @@ module Command.Unused where import qualified Data.Set as S -import qualified Data.Text.Lazy as L -import qualified Data.Text.Lazy.Encoding as L +import qualified Data.ByteString.Lazy as L import Data.BloomFilter import Data.BloomFilter.Easy import Data.BloomFilter.Hash @@ -265,8 +264,9 @@ withKeysReferencedInGitRef a ref = do go [] = noop go (l:ls) | isSymLink (LsTree.mode l) = do - content <- L.decodeUtf8 <$> catFile ref (LsTree.file l) - case fileKey (takeFileName $ L.unpack content) of + content <- encodeW8 . L.unpack + <$> catFile ref (LsTree.file l) + case fileKey (takeFileName content) of Nothing -> go ls Just k -> do a k diff --git a/debian/changelog b/debian/changelog index 9a47447ced..bc9932d522 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,7 @@ git-annex (3.20120616) UNRELEASED; urgency=low files and automatically annexes new files, etc, so you don't need to manually run git commands when manipulating files. * Enable diskfree on kfreebsd, using statvfs. + * unused: Fix crash when key names contain invalid utf8. -- Joey Hess Tue, 12 Jun 2012 11:35:59 -0400 From 9dc512a3ca1cb981d357e9d23872004edf8bd026 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 12:58:31 -0400 Subject: [PATCH 3867/8313] fixed --- ..._annex_unused_aborts_due_to_filename_encoding_problems.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn index d134013a48..fb0bdb0936 100644 --- a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems.mdwn @@ -11,4 +11,5 @@ Most likely I have added some file with a strange encoding that git-annex can't What version of git-annex are you using? On what operating system? 3.20120522, Debian testing -Please provide any additional information below. +> I've just fixed this bug in git, will be in the next release. --[[Joey]] +> [[done]] From 6d641f1ac0059fc03c06e582739a96198bb4d5ee Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 20 Jun 2012 16:59:53 +0000 Subject: [PATCH 3868/8313] Added a comment --- ...comment_6_4bf14ecef622988e80976c0fb55c24b9._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment new file mode 100644 index 0000000000..b35e31da62 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_6_4bf14ecef622988e80976c0fb55c24b9._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.2.6" + subject="comment 6" + date="2012-06-20T16:59:53Z" + content=""" +git-annex was not crashing due to content in the git-annex branch, but due to a symlink in one of your regular git branches, probably master and origin/master. + +This bug is fixed in git master, if you need the fix before the next release. +"""]] From 75b6ee81f9d9b921106c829380e30445415ec2f7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 13:13:40 -0400 Subject: [PATCH 3869/8313] avoid ByteString.Char8 where not needed Its truncation behavior is a red flag, so avoid using it in these places where only raw ByteStrings are used, without looking at the data inside. --- Annex/CatFile.hs | 2 +- Crypto.hs | 2 +- Git/CatFile.hs | 4 ++-- Remote/Bup.hs | 2 +- Remote/Directory.hs | 4 ++-- Remote/Hook.hs | 2 +- Remote/Rsync.hs | 2 +- Utility/Gpg.hs | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Annex/CatFile.hs b/Annex/CatFile.hs index afb14c67f0..88c498d315 100644 --- a/Annex/CatFile.hs +++ b/Annex/CatFile.hs @@ -12,7 +12,7 @@ module Annex.CatFile ( catFileHandle ) where -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Lazy as L import Common.Annex import qualified Git diff --git a/Crypto.hs b/Crypto.hs index 8941f7637b..01322c403c 100644 --- a/Crypto.hs +++ b/Crypto.hs @@ -26,7 +26,7 @@ module Crypto ( prop_hmacWithCipher_sane ) where -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Lazy as L import Data.ByteString.Lazy.UTF8 (fromString) import Data.Digest.Pure.SHA import Control.Applicative diff --git a/Git/CatFile.hs b/Git/CatFile.hs index 8a320a7120..e667b20879 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -15,8 +15,8 @@ module Git.CatFile ( ) where import System.IO -import qualified Data.ByteString.Char8 as S -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString as S +import qualified Data.ByteString.Lazy as L import Common import Git diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 3e7e9211f9..f1a36e468e 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -7,7 +7,7 @@ module Remote.Bup (remote) where -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Lazy as L import qualified Data.Map as M import System.Process diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 7521e70135..a5b0ff2a25 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -7,8 +7,8 @@ module Remote.Directory (remote) where -import qualified Data.ByteString.Lazy.Char8 as L -import qualified Data.ByteString.Char8 as S +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString as S import qualified Data.Map as M import qualified Control.Exception as E diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 1202c6087e..5fb793e65f 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -7,7 +7,7 @@ module Remote.Hook (remote) where -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Lazy as L import qualified Data.Map as M import System.Exit import System.Environment diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 3c449b5de0..6207e14253 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -7,7 +7,7 @@ module Remote.Rsync (remote) where -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Lazy as L import qualified Data.Map as M import Common.Annex diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs index ff6735ba57..e13afe5d48 100644 --- a/Utility/Gpg.hs +++ b/Utility/Gpg.hs @@ -7,7 +7,7 @@ module Utility.Gpg where -import qualified Data.ByteString.Lazy.Char8 as L +import qualified Data.ByteString.Lazy as L import System.Posix.Types import Control.Applicative import Control.Concurrent From 88e26046d7916e306e819c33aba1d6365167b6c5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 15:27:54 -0400 Subject: [PATCH 3870/8313] typo --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index a0e15946f9..5d8d973c67 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,7 +4,7 @@ git-annex (3.20120616) UNRELEASED; urgency=low files and automatically annexes new files, etc, so you don't need to manually run git commands when manipulating files. Available on Linux, BSDs, and OSX! - * Enable diskfree on kfreebsd, using statvfs. + * Enable diskfree on kfreebsd, using kqueue. * unused: Fix crash when key names contain invalid utf8. -- Joey Hess Tue, 12 Jun 2012 11:35:59 -0400 From 4afefb48f1fd57824770deb2a07f7628005122b7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 15:29:08 -0400 Subject: [PATCH 3871/8313] thought --- doc/design/assistant/inotify.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn index 0b55298a30..fb9ed592bb 100644 --- a/doc/design/assistant/inotify.mdwn +++ b/doc/design/assistant/inotify.mdwn @@ -20,6 +20,7 @@ There is a `watch` branch in git that adds the command. * Kqueue has to open every directory it watches, so too many directories will run it out of the max number of open files (typically 1024), and fail. + I may need to fork off multiple watcher processes to handle this. ## beyond Linux From ad11de94e54d17c765d980bfe249eca1c9b6cabd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 15:53:56 -0400 Subject: [PATCH 3872/8313] typo --- Utility/DirWatcher.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs index baab244c71..11ce7baef1 100644 --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -68,7 +68,7 @@ closingTracked = True #if WITH_KQUEUE closingTracked = False #else -eventsCoalesce = undefined +closingTracked = undefined #endif #endif From e0fdfb2e706da2cb1451193c658dc676b0530968 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 16:07:14 -0400 Subject: [PATCH 3873/8313] maintain set of files pendingAdd Kqueue needs to remember which files failed to be added due to being open, and retry them. This commit gets the data in place for such a retry thread. Broke KeySource out into its own file, and added Eq and Ord instances so it can be stored in a Set. --- Assistant.hs | 2 +- Assistant/Committer.hs | 55 ++++++++++++++++++++++++++------------- Assistant/DaemonStatus.hs | 10 ++++++- Backend.hs | 6 ++--- Backend/SHA.hs | 1 + Backend/WORM.hs | 1 + Command/Add.hs | 1 + Command/AddUrl.hs | 1 + Command/Migrate.hs | 1 + Types/Backend.hs | 8 +----- Types/KeySource.hs | 33 +++++++++++++++++++++++ 11 files changed, 89 insertions(+), 30 deletions(-) create mode 100644 Types/KeySource.hs diff --git a/Assistant.hs b/Assistant.hs index e924d94777..554c372903 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -75,8 +75,8 @@ startDaemon foreground -- begin adding files and having them -- committed, even while the startup scan -- is taking place. - _ <- forkIO $ commitThread st changechan _ <- forkIO $ daemonStatusThread st dstatus + _ <- forkIO $ commitThread st dstatus changechan _ <- forkIO $ sanityCheckerThread st dstatus changechan -- Does not return. watchThread st dstatus changechan diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs index 74f0922b73..600034a0ad 100644 --- a/Assistant/Committer.hs +++ b/Assistant/Committer.hs @@ -7,6 +7,7 @@ module Assistant.Committer where import Common.Annex import Assistant.Changes +import Assistant.DaemonStatus import Assistant.ThreadedMonad import Assistant.Watcher import qualified Annex @@ -18,15 +19,15 @@ import qualified Command.Add import Utility.ThreadScheduler import qualified Utility.Lsof as Lsof import qualified Utility.DirWatcher as DirWatcher -import Types.Backend +import Types.KeySource import Data.Time.Clock import Data.Tuple.Utils import qualified Data.Set as S {- This thread makes git commits at appropriate times. -} -commitThread :: ThreadState -> ChangeChan -> IO () -commitThread st changechan = runEvery (Seconds 1) $ do +commitThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () +commitThread st dstatus changechan = runEvery (Seconds 1) $ do -- We already waited one second as a simple rate limiter. -- Next, wait until at least one change has been made. cs <- getChanges changechan @@ -34,7 +35,7 @@ commitThread st changechan = runEvery (Seconds 1) $ do time <- getCurrentTime if shouldCommit time cs then do - handleAdds st changechan cs + handleAdds st dstatus changechan cs void $ tryIO $ runThreadState st commitStaged else refillChanges changechan cs @@ -79,19 +80,20 @@ shouldCommit now changes - - When a file is added, Inotify will notice the new symlink. So this waits - for additional Changes to arrive, so that the symlink has hopefully been - - staged before returning, and will be committed immediately. OTOH, for - - kqueue, eventsCoalesce, so instead the symlink is directly created and - - staged. + - staged before returning, and will be committed immediately. + - + - OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly + - created and staged, if the file is not open. -} -handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO () -handleAdds st changechan cs +handleAdds :: ThreadState -> DaemonStatusHandle -> ChangeChan -> [Change] -> IO () +handleAdds st dstatus changechan cs | null toadd = noop | otherwise = do - toadd' <- safeToAdd st toadd + toadd' <- safeToAdd st dstatus toadd unless (null toadd') $ do added <- filter id <$> forM toadd' add unless (DirWatcher.eventsCoalesce || null added) $ - handleAdds st changechan + handleAdds st dstatus changechan =<< getChanges changechan where toadd = map changeFile $ filter isPendingAdd cs @@ -122,8 +124,8 @@ handleAdds st changechan cs - opened for write, so lsof is run on the temp directory - to check them. -} -safeToAdd :: ThreadState -> [FilePath] -> IO [KeySource] -safeToAdd st files = do +safeToAdd :: ThreadState -> DaemonStatusHandle -> [FilePath] -> IO [KeySource] +safeToAdd st dstatus files = do locked <- catMaybes <$> lockdown files runThreadState st $ ifM (Annex.getState Annex.force) ( return locked -- force bypasses lsof check @@ -134,16 +136,33 @@ safeToAdd st files = do catMaybes <$> forM locked (go open) ) where + {- When a file is still open, it can be put into pendingAdd + - to be checked again later. However when closingTracked + - is supported, another event will be received once it's + - closed, so there's no point in doing so. -} go open keysource | S.member (contentLocation keysource) open = do - warning $ keyFilename keysource - ++ " still has writers, not adding" - -- remove the hard link - --_ <- liftIO $ tryIO $ - -- removeFile $ contentLocation keysource + if DirWatcher.closingTracked + then do + warning $ keyFilename keysource + ++ " still has writers, not adding" + void $ liftIO $ canceladd keysource + else void $ addpending keysource return Nothing | otherwise = return $ Just keysource + canceladd keysource = tryIO $ + -- remove the hard link + removeFile $ contentLocation keysource + + {- The same file (or a file with the same name) + - could already be pending add; if so this KeySource + - superscedes the old one. -} + addpending keysource = modifyDaemonStatusM dstatus $ \s -> do + let set = pendingAdd s + mapM_ canceladd $ S.toList $ S.filter (== keysource) set + return $ s { pendingAdd = S.insert keysource set } + lockdown = mapM $ \file -> do ms <- catchMaybeIO $ getSymbolicLinkStatus file case ms of diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index e5ba3d1512..289a97bb25 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -9,12 +9,14 @@ import Common.Annex import Assistant.ThreadedMonad import Utility.ThreadScheduler import Utility.TempFile +import Types.KeySource import Control.Concurrent import System.Posix.Types import Data.Time.Clock.POSIX import Data.Time import System.Locale +import qualified Data.Set as S data DaemonStatus = DaemonStatus -- False when the daemon is performing its startup scan @@ -25,6 +27,8 @@ data DaemonStatus = DaemonStatus , sanityCheckRunning :: Bool -- Last time the sanity checker ran , lastSanityCheck :: Maybe POSIXTime + -- Files that are in the process of being added to the annex. + , pendingAdd :: S.Set KeySource } deriving (Show) @@ -36,13 +40,17 @@ newDaemonStatus = DaemonStatus , lastRunning = Nothing , sanityCheckRunning = False , lastSanityCheck = Nothing + , pendingAdd = S.empty } getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus getDaemonStatus = liftIO . readMVar modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex () -modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a) +modifyDaemonStatus handle a = modifyDaemonStatusM handle (return . a) + +modifyDaemonStatusM :: DaemonStatusHandle -> (DaemonStatus -> IO DaemonStatus) -> Annex () +modifyDaemonStatusM handle a = liftIO $ modifyMVar_ handle a {- Load any previous daemon status file, and store it in the MVar for this - process to use as its DaemonStatus. -} diff --git a/Backend.hs b/Backend.hs index bde1aad78e..d1dfdef3c5 100644 --- a/Backend.hs +++ b/Backend.hs @@ -6,7 +6,6 @@ -} module Backend ( - B.KeySource(..), list, orderedList, genKey, @@ -23,6 +22,7 @@ import Config import qualified Annex import Annex.CheckAttr import Types.Key +import Types.KeySource import qualified Types.Backend as B -- When adding a new backend, import it here and add it to the list. @@ -54,12 +54,12 @@ orderedList = do {- Generates a key for a file, trying each backend in turn until one - accepts it. -} -genKey :: B.KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend)) +genKey :: KeySource -> Maybe Backend -> Annex (Maybe (Key, Backend)) genKey source trybackend = do bs <- orderedList let bs' = maybe bs (: bs) trybackend genKey' bs' source -genKey' :: [Backend] -> B.KeySource -> Annex (Maybe (Key, Backend)) +genKey' :: [Backend] -> KeySource -> Annex (Maybe (Key, Backend)) genKey' [] _ = return Nothing genKey' (b:bs) source = do r <- B.getKey b source diff --git a/Backend/SHA.hs b/Backend/SHA.hs index df613bbcdd..838a97ab8b 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -11,6 +11,7 @@ import Common.Annex import qualified Annex import Types.Backend import Types.Key +import Types.KeySource import qualified Build.SysConfig as SysConfig type SHASize = Int diff --git a/Backend/WORM.hs b/Backend/WORM.hs index 630000fa2b..5232037136 100644 --- a/Backend/WORM.hs +++ b/Backend/WORM.hs @@ -10,6 +10,7 @@ module Backend.WORM (backends) where import Common.Annex import Types.Backend import Types.Key +import Types.KeySource backends :: [Backend] backends = [backend] diff --git a/Command/Add.hs b/Command/Add.hs index 43f186fbf0..73edb5eaaf 100644 --- a/Command/Add.hs +++ b/Command/Add.hs @@ -12,6 +12,7 @@ import Annex.Exception import Command import qualified Annex import qualified Annex.Queue +import Types.KeySource import Backend import Logs.Location import Annex.Content diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs index 369940bdfe..bef1d68752 100644 --- a/Command/AddUrl.hs +++ b/Command/AddUrl.hs @@ -20,6 +20,7 @@ import Annex.Content import Logs.Web import qualified Option import Types.Key +import Types.KeySource import Config def :: [Command] diff --git a/Command/Migrate.hs b/Command/Migrate.hs index 29e664ce23..c7c0d7af39 100644 --- a/Command/Migrate.hs +++ b/Command/Migrate.hs @@ -11,6 +11,7 @@ import Common.Annex import Command import Backend import qualified Types.Key +import Types.KeySource import Annex.Content import qualified Command.ReKey diff --git a/Types/Backend.hs b/Types/Backend.hs index 97f7cef907..d79787c27c 100644 --- a/Types/Backend.hs +++ b/Types/Backend.hs @@ -10,13 +10,7 @@ module Types.Backend where import Types.Key - -{- The source used to generate a key. The location of the content - - may be different from the filename associated with the key. -} -data KeySource = KeySource - { keyFilename :: FilePath - , contentLocation :: FilePath - } +import Types.KeySource data BackendA a = Backend { name :: String diff --git a/Types/KeySource.hs b/Types/KeySource.hs new file mode 100644 index 0000000000..9d1fa173f5 --- /dev/null +++ b/Types/KeySource.hs @@ -0,0 +1,33 @@ +{- KeySource data type + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Types.KeySource where + +import Data.Ord + +{- When content is in the process of being added to the annex, + - and a Key generated from it, this data type is used. + - + - The contentLocation may be different from the filename + - associated with the key. For example, the add command + - temporarily puts the content into a lockdown directory + - for checking. The migrate command uses the content + - of a different Key. -} +data KeySource = KeySource + { keyFilename :: FilePath + , contentLocation :: FilePath + } + deriving (Show) + +{- KeySources are assumed to be equal when the same filename is associated + - with the key. The contentLocation can be a random temp file. + -} +instance Eq KeySource where + x == y = keyFilename x == keyFilename y + +instance Ord KeySource where + compare = comparing keyFilename From a27f3af51f91b3981641825fde5739ec8ef472a1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg" Date: Wed, 20 Jun 2012 20:49:07 +0000 Subject: [PATCH 3874/8313] Added a comment --- .../comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment new file mode 100644 index 0000000000..65a02fed96 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_7_d2e5382fe0f38fb9dd9ee69901c68151._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmL8pteP2jbYJUn1M3CbeLDvz2SWAA1wtg" + nickname="Kristian" + subject="comment 7" + date="2012-06-20T20:49:06Z" + content=""" +Thank you +"""]] From 33b914bcf1f277aecccb4194e296f17f4708e434 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 19:04:16 -0400 Subject: [PATCH 3875/8313] pending adds now retried for kqueue Rethought how to keep track of pending adds that need to be retried later. The commit thread already run up every second when there are changes, so let's keep pending adds queued as changes until they're safe to add. Also, the committer is now smarter about avoiding empty commits when all the adds are currently unsafe, or in the rare case that an add event for a symlink is not received in time. It may avoid them entirely. This seems to work as before for inotify, and is untested for kqueue. (Actually commit batching seems to be improved for inotify, although I'm not sure why. I'm seeing only two commits made during large batch operations, and the first of those is the non-batch mode commit.) --- Assistant.hs | 2 +- Assistant/Changes.hs | 38 +++++++-- Assistant/Committer.hs | 160 ++++++++++++++++++++------------------ Assistant/DaemonStatus.hs | 10 +-- Assistant/Watcher.hs | 32 ++++---- 5 files changed, 135 insertions(+), 107 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 554c372903..e924d94777 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -75,8 +75,8 @@ startDaemon foreground -- begin adding files and having them -- committed, even while the startup scan -- is taking place. + _ <- forkIO $ commitThread st changechan _ <- forkIO $ daemonStatusThread st dstatus - _ <- forkIO $ commitThread st dstatus changechan _ <- forkIO $ sanityCheckerThread st dstatus changechan -- Does not return. watchThread st dstatus changechan diff --git a/Assistant/Changes.hs b/Assistant/Changes.hs index 1cad423265..173ba19220 100644 --- a/Assistant/Changes.hs +++ b/Assistant/Changes.hs @@ -7,20 +7,26 @@ module Assistant.Changes where import Common.Annex import qualified Annex.Queue +import Types.KeySource import Control.Concurrent.STM import Data.Time.Clock -data ChangeType = PendingAddChange | LinkChange | RmChange | RmDirChange +data ChangeType = AddChange | LinkChange | RmChange | RmDirChange deriving (Show, Eq) type ChangeChan = TChan Change -data Change = Change - { changeTime :: UTCTime - , changeFile :: FilePath - , changeType :: ChangeType - } +data Change + = Change + { changeTime :: UTCTime + , changeFile :: FilePath + , changeType :: ChangeType + } + | PendingAddChange + { changeTime ::UTCTime + , keySource :: KeySource + } deriving (Show) runChangeChan :: STM a -> IO a @@ -33,13 +39,29 @@ newChangeChan = atomically newTChan madeChange :: FilePath -> ChangeType -> Annex (Maybe Change) madeChange f t = do -- Just in case the commit thread is not flushing the queue fast enough. - when (t /= PendingAddChange) $ - Annex.Queue.flushWhenFull + Annex.Queue.flushWhenFull liftIO $ Just <$> (Change <$> getCurrentTime <*> pure f <*> pure t) noChange :: Annex (Maybe Change) noChange = return Nothing +{- Indicates an add is in progress. -} +pendingAddChange :: KeySource -> Annex (Maybe Change) +pendingAddChange ks = + liftIO $ Just <$> (PendingAddChange <$> getCurrentTime <*> pure ks) + +isPendingAddChange :: Change -> Bool +isPendingAddChange (PendingAddChange {}) = True +isPendingAddChange _ = False + +finishedChange :: Change -> Change +finishedChange c@(PendingAddChange { keySource = ks }) = Change + { changeTime = changeTime c + , changeFile = keyFilename ks + , changeType = AddChange + } +finishedChange c = c + {- Gets all unhandled changes. - Blocks until at least one change is made. -} getChanges :: ChangeChan -> IO [Change] diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs index 600034a0ad..46fee1b747 100644 --- a/Assistant/Committer.hs +++ b/Assistant/Committer.hs @@ -7,7 +7,6 @@ module Assistant.Committer where import Common.Annex import Assistant.Changes -import Assistant.DaemonStatus import Assistant.ThreadedMonad import Assistant.Watcher import qualified Annex @@ -24,20 +23,25 @@ import Types.KeySource import Data.Time.Clock import Data.Tuple.Utils import qualified Data.Set as S +import Data.Either {- This thread makes git commits at appropriate times. -} -commitThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () -commitThread st dstatus changechan = runEvery (Seconds 1) $ do +commitThread :: ThreadState -> ChangeChan -> IO () +commitThread st changechan = runEvery (Seconds 1) $ do -- We already waited one second as a simple rate limiter. - -- Next, wait until at least one change has been made. - cs <- getChanges changechan + -- Next, wait until at least one change is available for + -- processing. + changes <- getChanges changechan -- Now see if now's a good time to commit. time <- getCurrentTime - if shouldCommit time cs + if shouldCommit time changes then do - handleAdds st dstatus changechan cs - void $ tryIO $ runThreadState st commitStaged - else refillChanges changechan cs + readychanges <- handleAdds st changechan changes + if shouldCommit time readychanges + then do + void $ tryIO $ runThreadState st commitStaged + else refillChanges changechan readychanges + else refillChanges changechan changes commitStaged :: Annex () commitStaged = do @@ -83,95 +87,99 @@ shouldCommit now changes - staged before returning, and will be committed immediately. - - OTOH, for kqueue, eventsCoalesce, so instead the symlink is directly - - created and staged, if the file is not open. + - created and staged. + - + - Returns a list of all changes that are ready to be committed. + - Any pending adds that are not ready yet are put back into the ChangeChan, + - where they will be retried later. -} -handleAdds :: ThreadState -> DaemonStatusHandle -> ChangeChan -> [Change] -> IO () -handleAdds st dstatus changechan cs - | null toadd = noop - | otherwise = do - toadd' <- safeToAdd st dstatus toadd - unless (null toadd') $ do - added <- filter id <$> forM toadd' add - unless (DirWatcher.eventsCoalesce || null added) $ - handleAdds st dstatus changechan +handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO [Change] +handleAdds st changechan cs = returnWhen (null pendingadds) $ do + (postponed, toadd) <- partitionEithers <$> + safeToAdd st pendingadds + + unless (null postponed) $ + refillChanges changechan postponed + + returnWhen (null toadd) $ do + added <- catMaybes <$> forM toadd add + if (DirWatcher.eventsCoalesce || null added) + then return $ added ++ otherchanges + else do + r <- handleAdds st changechan =<< getChanges changechan + return $ r ++ added ++ otherchanges where - toadd = map changeFile $ filter isPendingAdd cs + (pendingadds, otherchanges) = partition isPendingAddChange cs - isPendingAdd (Change { changeType = PendingAddChange }) = True - isPendingAdd _ = False + returnWhen c a + | c = return otherchanges + | otherwise = a - add keysource = catchBoolIO $ runThreadState st $ do - showStart "add" $ keyFilename keysource - handle (keyFilename keysource) - =<< Command.Add.ingest keysource + add :: Change -> IO (Maybe Change) + add change@(PendingAddChange { keySource = ks }) = do + r <- catchMaybeIO $ runThreadState st $ do + showStart "add" $ keyFilename ks + handle (finishedChange change) (keyFilename ks) + =<< Command.Add.ingest ks + return $ maybeMaybe r + add _ = return Nothing - handle _ Nothing = do + maybeMaybe (Just j@(Just _)) = j + maybeMaybe _ = Nothing + + handle _ _ Nothing = do showEndFail - return False - handle file (Just key) = do + return Nothing + handle change file (Just key) = do link <- Command.Add.link file key True when DirWatcher.eventsCoalesce $ do sha <- inRepo $ Git.HashObject.hashObject BlobObject link stageSymlink file sha showEndOk - return True + return $ Just change -{- Checks which of a set of files can safely be added. - - Files are locked down as hard links in a temp directory, - - with their write bits disabled. But some may still be - - opened for write, so lsof is run on the temp directory - - to check them. +{- PendingAddChanges can Either be Right to be added now, + - or are unsafe, and must be Left for later. + - + - Check by running lsof on the temp directory, which + - the KeySources are locked down in. -} -safeToAdd :: ThreadState -> DaemonStatusHandle -> [FilePath] -> IO [KeySource] -safeToAdd st dstatus files = do - locked <- catMaybes <$> lockdown files - runThreadState st $ ifM (Annex.getState Annex.force) - ( return locked -- force bypasses lsof check +safeToAdd :: ThreadState -> [Change] -> IO [Either Change Change] +safeToAdd st changes = runThreadState st $ + ifM (Annex.getState Annex.force) + ( allRight changes -- force bypasses lsof check , do tmpdir <- fromRepo gitAnnexTmpDir - open <- S.fromList . map fst3 . filter openwrite <$> + openfiles <- S.fromList . map fst3 . filter openwrite <$> liftIO (Lsof.queryDir tmpdir) - catMaybes <$> forM locked (go open) + + let checked = map (check openfiles) changes + + {- If new events are received when files are closed, + - there's no need to retry any changes that cannot + - be done now. -} + if DirWatcher.closingTracked + then do + mapM_ canceladd $ lefts checked + allRight $ rights checked + else return checked ) where - {- When a file is still open, it can be put into pendingAdd - - to be checked again later. However when closingTracked - - is supported, another event will be received once it's - - closed, so there's no point in doing so. -} - go open keysource - | S.member (contentLocation keysource) open = do - if DirWatcher.closingTracked - then do - warning $ keyFilename keysource - ++ " still has writers, not adding" - void $ liftIO $ canceladd keysource - else void $ addpending keysource - return Nothing - | otherwise = return $ Just keysource + check openfiles change@(PendingAddChange { keySource = ks }) + | S.member (contentLocation ks) openfiles = Left change + check _ change = Right change - canceladd keysource = tryIO $ + canceladd (PendingAddChange { keySource = ks }) = do + warning $ keyFilename ks + ++ " still has writers, not adding" -- remove the hard link - removeFile $ contentLocation keysource - - {- The same file (or a file with the same name) - - could already be pending add; if so this KeySource - - superscedes the old one. -} - addpending keysource = modifyDaemonStatusM dstatus $ \s -> do - let set = pendingAdd s - mapM_ canceladd $ S.toList $ S.filter (== keysource) set - return $ s { pendingAdd = S.insert keysource set } - - lockdown = mapM $ \file -> do - ms <- catchMaybeIO $ getSymbolicLinkStatus file - case ms of - Just s - | isRegularFile s -> - catchMaybeIO $ runThreadState st $ - Command.Add.lockDown file - _ -> return Nothing - + void $ liftIO $ tryIO $ + removeFile $ contentLocation ks + canceladd _ = noop openwrite (_file, mode, _pid) = mode == Lsof.OpenWriteOnly || mode == Lsof.OpenReadWrite + + allRight = return . map Right diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 289a97bb25..e5ba3d1512 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -9,14 +9,12 @@ import Common.Annex import Assistant.ThreadedMonad import Utility.ThreadScheduler import Utility.TempFile -import Types.KeySource import Control.Concurrent import System.Posix.Types import Data.Time.Clock.POSIX import Data.Time import System.Locale -import qualified Data.Set as S data DaemonStatus = DaemonStatus -- False when the daemon is performing its startup scan @@ -27,8 +25,6 @@ data DaemonStatus = DaemonStatus , sanityCheckRunning :: Bool -- Last time the sanity checker ran , lastSanityCheck :: Maybe POSIXTime - -- Files that are in the process of being added to the annex. - , pendingAdd :: S.Set KeySource } deriving (Show) @@ -40,17 +36,13 @@ newDaemonStatus = DaemonStatus , lastRunning = Nothing , sanityCheckRunning = False , lastSanityCheck = Nothing - , pendingAdd = S.empty } getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus getDaemonStatus = liftIO . readMVar modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex () -modifyDaemonStatus handle a = modifyDaemonStatusM handle (return . a) - -modifyDaemonStatusM :: DaemonStatusHandle -> (DaemonStatus -> IO DaemonStatus) -> Annex () -modifyDaemonStatusM handle a = liftIO $ modifyMVar_ handle a +modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a) {- Load any previous daemon status file, and store it in the MVar for this - process to use as its DaemonStatus. -} diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs index cb7ede9209..db58f01e8b 100644 --- a/Assistant/Watcher.hs +++ b/Assistant/Watcher.hs @@ -15,13 +15,14 @@ import Assistant.DaemonStatus import Assistant.Changes import Utility.DirWatcher import Utility.Types.DirWatcher +import qualified Annex import qualified Annex.Queue import qualified Git.Command import qualified Git.UpdateIndex import qualified Git.HashObject import qualified Git.LsFiles import qualified Backend -import qualified Annex +import qualified Command.Add import Annex.Content import Annex.CatFile import Git.Types @@ -110,22 +111,27 @@ runHandler st dstatus changechan handler file filestatus = void $ do - and only one has just closed it. We want to avoid adding a file to the - annex that is open for write, to avoid anything being able to change it. - - - We could run lsof on the file here to check for other writer. - - But, that's slow. Instead, a Change is returned that indicates this file - - still needs to be added. The committer will handle bundles of these - - Changes at once. + - We could run lsof on the file here to check for other writers. + - But, that's slow, and even if there is currently a writer, we will want + - to add the file *eventually*. Instead, the file is locked down as a hard + - link in a temp directory, with its write bits disabled, for later + - checking with lsof, and a Change is returned containing a KeySource + - using that hard link. The committer handles running lsof and finishing + - the add. -} onAdd :: Handler -onAdd file _filestatus dstatus = do - ifM (scanComplete <$> getDaemonStatus dstatus) - ( go - , ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file])) - ( noChange - , go +onAdd file filestatus dstatus + | maybe False isRegularFile filestatus = do + ifM (scanComplete <$> getDaemonStatus dstatus) + ( go + , ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file])) + ( noChange + , go + ) ) - ) + | otherwise = noChange where - go = madeChange file PendingAddChange + go = pendingAddChange =<< Command.Add.lockDown file {- A symlink might be an arbitrary symlink, which is just added. - Or, if it is a git-annex symlink, ensure it points to the content From 75dba7f7bc50b75e08ee49cb4b758a375ef70d68 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 20 Jun 2012 20:05:40 -0400 Subject: [PATCH 3876/8313] belt and suspenders check It's possible for there to be multiple queued changes all adding the same file, and for those changes to be reordered. Maybe. This check will guard against that ending up adding the wrong version of the file last. --- Assistant/Committer.hs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs index 46fee1b747..63df8cafcc 100644 --- a/Assistant/Committer.hs +++ b/Assistant/Committer.hs @@ -118,7 +118,7 @@ handleAdds st changechan cs = returnWhen (null pendingadds) $ do add :: Change -> IO (Maybe Change) add change@(PendingAddChange { keySource = ks }) = do - r <- catchMaybeIO $ runThreadState st $ do + r <- catchMaybeIO $ sanitycheck ks $ runThreadState st $ do showStart "add" $ keyFilename ks handle (finishedChange change) (keyFilename ks) =<< Command.Add.ingest ks @@ -140,6 +140,16 @@ handleAdds st changechan cs = returnWhen (null pendingadds) $ do showEndOk return $ Just change + {- Check that the keysource's keyFilename still exists, + - and is still a hard link to its contentLocation, + - before ingesting it. -} + sanitycheck keysource a = do + fs <- getSymbolicLinkStatus $ keyFilename keysource + ks <- getSymbolicLinkStatus $ contentLocation keysource + if deviceID ks == deviceID fs && fileID ks == fileID fs + then a + else return Nothing + {- PendingAddChanges can Either be Right to be added now, - or are unsafe, and must be Left for later. - From 7db83a1b0ff49ddbc316556d416ce67418428d13 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Jun 2012 00:28:56 -0400 Subject: [PATCH 3877/8313] demote lsof not available on kfreebsd, and only used by watch --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 3b142dc5f2..bcecbec3d9 100644 --- a/debian/control +++ b/debian/control @@ -41,8 +41,8 @@ Depends: ${misc:Depends}, ${shlibs:Depends}, uuid, rsync, wget | curl, - openssh-client (>= 1:5.6p1), - lsof + openssh-client (>= 1:5.6p1) +Recommends: lsof Suggests: graphviz, bup, gnupg Description: manage files with git, without checking their contents into git git-annex allows managing files with git, without checking the file From 9f1a29762b074e668d3d6e7bea273c1f14f69b90 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Jun 2012 01:07:37 -0400 Subject: [PATCH 3878/8313] blog for the day --- .../blog/day_14__kqueue_kqueue_kqueue.mdwn | 23 +++++++++++++++++++ doc/design/assistant/inotify.mdwn | 5 ---- 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn diff --git a/doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn b/doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn new file mode 100644 index 0000000000..d7dab36117 --- /dev/null +++ b/doc/design/assistant/blog/day_14__kqueue_kqueue_kqueue.mdwn @@ -0,0 +1,23 @@ +... I'm getting tired of kqueue. + +But the end of the tunnel is in sight. Today I made git-annex handle files +that are still open for write after a kqueue creation event is received. +Unlike with inotify, which has a new event each time a file is closed, +kqueue only gets one event when a file is first created, and so git-annex +needs to retry adding files until there are no writers left. + +Eventually I found an elegant way to do that. The committer thread already +wakes up every second as long as there's a pending change to commit. So for +adds that need to be retried, it can just push them back onto the change +queue, and the committer thread will wait one second and retry the add. One +second might be too frequent to check, but it will do for now. + +This means that `git annex watch` should now be usable on OSX, FreeBSD, and +NetBSD! (It'll also work on Debian kFreeBSD once [lsof is ported to it](http://bugs.debian.org/589103).) +I've meged kqueue support to `master`. + +I also think I've squashed the empty commits that were sometimes made. + +Incidentally, I'm 50% through my first month, and finishing [[inotify]] +was the first half of my roadmap for this month. Seem to be right on +schedule.. Now I need to start thinking about [[syncing]]. diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn index fb9ed592bb..bea21bff7d 100644 --- a/doc/design/assistant/inotify.mdwn +++ b/doc/design/assistant/inotify.mdwn @@ -13,11 +13,6 @@ There is a `watch` branch in git that adds the command. * When you `git annex unlock` a file, it will immediately be re-locked. -* With kqueue, if a file is created and still has a writer, it'll - give up adding it, and it will never get added. This is because kqueue - cannot track file closes. Need to go back and check these files every - second or something. - * Kqueue has to open every directory it watches, so too many directories will run it out of the max number of open files (typically 1024), and fail. I may need to fork off multiple watcher processes to handle this. From 680b0c84057ed22ed9ad816b6e333c7a8eb00c1a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 21 Jun 2012 06:36:02 +0000 Subject: [PATCH 3879/8313] --- ...dfb2e706da2cb1451193c658dc676b0530968.mdwn | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn diff --git a/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn b/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn new file mode 100644 index 0000000000..1426f34882 --- /dev/null +++ b/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn @@ -0,0 +1,22 @@ +I only saw this just now, but the tests fail to link/build on OSX + +
+[181 of 181] Compiling Main ( git-annex.hs, tmp/Main.o )
+Linking git-annex ...
++ make -q test
++ '[' 1 = 1 ']'
++ ../maxtime 1800 make test
+[175 of 175] Compiling Main ( test.hs, tmp/Main.o )
+test.hs:175:17:
+Not in scope: data constructor `Types.Backend.KeySource'
+test.hs:175:43:
+`Types.Backend.keyFilename' is not a (visible) constructor field name
+test.hs:175:76:
+`Types.Backend.contentLocation' is not a (visible) constructor field name
+** failed to build the test suite
+make: *** [test] Error 1
++ exit 4
+
+ +this issue seems to got introduced at commit e0fdfb2e706da2cb1451193c658dc676b0530968 + From ebe8ee675366da190b5c462f2cc66dd528412db4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" Date: Thu, 21 Jun 2012 07:08:33 +0000 Subject: [PATCH 3880/8313] Added a comment --- .../comment_8_b282757537cda863d3dc6d0bbfd6b656._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment diff --git a/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment new file mode 100644 index 0000000000..ff1de6b2e8 --- /dev/null +++ b/doc/bugs/git_annex_unused_aborts_due_to_filename_encoding_problems/comment_8_b282757537cda863d3dc6d0bbfd6b656._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" + nickname="Kristian" + subject="comment 8" + date="2012-06-21T07:08:22Z" + content=""" +Confirmed. I built the newest version of git-annex and it solved the issue :) +"""]] From 46d1f5a9d05f69a1933bd79e1e0ec82893934e0f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Jun 2012 13:04:24 -0400 Subject: [PATCH 3881/8313] fix test suite build --- ...now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn | 1 + test.hs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn b/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn index 1426f34882..be6db378cf 100644 --- a/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn +++ b/doc/bugs/The_tests_are_failing_to_build_now_on_commit_e0fdfb2e706da2cb1451193c658dc676b0530968.mdwn @@ -20,3 +20,4 @@ make: *** [test] Error 1 this issue seems to got introduced at commit e0fdfb2e706da2cb1451193c658dc676b0530968 +> [[fixed|done]] --[[Joey]] diff --git a/test.hs b/test.hs index 1a7c382c01..089c86bfb6 100644 --- a/test.hs +++ b/test.hs @@ -28,6 +28,7 @@ import qualified Backend import qualified Git.CurrentRepo import qualified Git.Filename import qualified Locations +import qualified Types.KeySource import qualified Types.Backend import qualified Types import qualified GitAnnex @@ -172,7 +173,7 @@ test_reinject = "git-annex reinject/fromkey" ~: TestCase $ intmpclonerepo $ do git_annex "drop" ["--force", sha1annexedfile] @? "drop failed" writeFile tmp $ content sha1annexedfile r <- annexeval $ Types.Backend.getKey backendSHA1 $ - Types.Backend.KeySource { Types.Backend.keyFilename = tmp, Types.Backend.contentLocation = tmp } + Types.KeySource.KeySource { Types.KeySource.keyFilename = tmp, Types.KeySource.contentLocation = tmp } let key = show $ fromJust r git_annex "reinject" [tmp, sha1annexedfile] @? "reinject failed" git_annex "fromkey" [key, sha1annexedfiledup] @? "fromkey failed" From 3138a49084dcf8227674e87de516194ad822b896 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Jun 2012 13:11:22 -0400 Subject: [PATCH 3882/8313] update --- doc/design/assistant/inotify.mdwn | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn index bea21bff7d..47b8c84a34 100644 --- a/doc/design/assistant/inotify.mdwn +++ b/doc/design/assistant/inotify.mdwn @@ -1,7 +1,6 @@ -Finish "git annex watch" command, which runs, in the background, watching via -inotify for changes, and automatically annexing new files, etc. - -There is a `watch` branch in git that adds the command. +"git annex watch" command, which runs, in the background, watching via +inotify for changes, and automatically annexing new files, etc. Now +available! [[!toc]] From 8a0d6d83f4e241f0cc18269e62e7289fec060e4e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Jun 2012 13:14:31 -0400 Subject: [PATCH 3883/8313] remove unused and slightly indefensible Eq and Ord instances --- Types/KeySource.hs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Types/KeySource.hs b/Types/KeySource.hs index 9d1fa173f5..f4885767a5 100644 --- a/Types/KeySource.hs +++ b/Types/KeySource.hs @@ -7,8 +7,6 @@ module Types.KeySource where -import Data.Ord - {- When content is in the process of being added to the annex, - and a Key generated from it, this data type is used. - @@ -22,12 +20,3 @@ data KeySource = KeySource , contentLocation :: FilePath } deriving (Show) - -{- KeySources are assumed to be equal when the same filename is associated - - with the key. The contentLocation can be a random temp file. - -} -instance Eq KeySource where - x == y = keyFilename x == keyFilename y - -instance Ord KeySource where - compare = comparing keyFilename From f27da7a1cc095dcaf9ce0cc2170fe98d3b050336 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 21 Jun 2012 20:02:00 -0400 Subject: [PATCH 3884/8313] blog for the day and design update --- .../blog/day_14__thinking_about_syncing.mdwn | 44 +++++++++++++++++++ doc/design/assistant/syncing.mdwn | 14 ++++-- 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn diff --git a/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn b/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn new file mode 100644 index 0000000000..c4a700d139 --- /dev/null +++ b/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn @@ -0,0 +1,44 @@ +Pondering [[syncing]] today. I will be doing syncing of the git repository +first, and working on syncing of file data later. + +The former seems straightforward enough, since we just want to push all +changes to everywhere. Indeed, git-annex already has a [[sync]] command +that uses a smart technique to allow syncing between clones without a +central bare repository. (Props to Joachim Breitner for that.) + +But it's not all easy. Syncing should happen as fast as possible, so +changes show up without delay. Eventually it'll need to support syncing +between nodes that cannot directly contact one-another. Syncing needs to +deal with nodes coming and going; one example of that is a USB drive being +plugged in, which should immediatly be synced, but network can also come +and go, so it should periodically retry nodes it failed to sync with. To +start with, I'll be focusing on fast syncing between directly connected +nodes, but I have to keep this wider problem space in mind. + +One problem with `git annex sync` is that it has to be run in both clones +in order for changes to fully propigate. This is because git doesn't allow +pushing changes into a non-bare repository; so instead it drops off a new +branch in `.git/refs/remotes/$foo/synced/master`. Then when it's run locally +it merges that new branch into `master`. + +So, how to trigger a clone to run `git annex sync` when syncing to it? +Well, I just realized I have spent two weeks developing something that can +be repurposed to do that! [[Inotify]] can watch for changes to +`.git/refs/remotes`, and the instant a change is made, the local sync +process can be started. This avoids needing to make another ssh connection +to trigger the sync, so is faster and allows the data to be transferred +over another protocol than ssh, which may come in handy later. + +So, in summary, here's what will happen when a new file is created: + +1. inotify event causes the file to be added to the annex, and + immediately committed. +2. new branch is pushed to remotes (probably in parallel) +3. remotes notice new sync branch and merge it +4. (data sync, TBD later) +5. file is fully synced and available + +Steps 1, 2, and 3 should all be able to be accomplished in under a second. +The speed of `git push` making a ssh connection will be the main limit +to making it fast. (Perhaps I should also reuse git-annex's existing ssh +connection caching code?) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 0813b8b70b..56c9692e36 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -3,13 +3,21 @@ all the other git clones, at both the git level and the key/value level. ## git syncing -1. At regular intervals, just run `git annex sync`, which already handles - bidirectional syncing. +1. Can use `git annex sync`, which already handles bidirectional syncing. + When a change is committed, launch the part of `git annex sync` that pushes + out changes. +1. Watch `.git/refs/remotes/` for changes (which would be pushed in from + another node via `git annex sync`), and run the part of `git annex sync` + that merges in received changes, and follow it by the part that pushes out + changes (sending them to any other remotes). + [The watching can be done with the existing inotify code! This avoids needing + any special mechanism to notify a remote that it's been synced to.] 2. Use a git merge driver that adds both conflicting files, so conflicts never break a sync. 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of signaling a change out of band. -4. Add a hook, so when there's a change to sync, a program can be run. +4. Add a hook, so when there's a change to sync, a program can be run + and do its own signaling. ## data syncing From cf9bf920e8052edbcee9dd31bfb1c9b421296d14 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E" Date: Fri, 22 Jun 2012 04:26:06 +0000 Subject: [PATCH 3885/8313] Added a comment: Homebrew instead of MacPorts --- ...comment_8_22b818e1a2a825efb78139271a14f944._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment diff --git a/doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment b/doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment new file mode 100644 index 0000000000..57f354e494 --- /dev/null +++ b/doc/design/assistant/comment_8_22b818e1a2a825efb78139271a14f944._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E" + nickname="Matt" + subject="Homebrew instead of MacPorts" + date="2012-06-22T04:26:02Z" + content=""" +[Homebrew] is a much better package manager than MacPorts IMO. + +[Homebrew]: http://mxcl.github.com/homebrew/ +"""]] From 3ee44cf8feb11fc439c02eb0eb8f12d290b01120 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Jun 2012 13:04:03 -0400 Subject: [PATCH 3886/8313] add assistant command like watch, but more magic --- Assistant.hs | 6 +++--- Command/Assistant.hs | 18 ++++++++++++++++++ Command/Watch.hs | 18 +++++++++--------- GitAnnex.hs | 2 ++ doc/git-annex.mdwn | 4 ++++ 5 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 Command/Assistant.hs diff --git a/Assistant.hs b/Assistant.hs index e924d94777..33c7cef36c 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -55,10 +55,10 @@ import Utility.LogFile import Control.Concurrent -startDaemon :: Bool -> Annex () -startDaemon foreground +startDaemon :: Bool -> Bool -> Annex () +startDaemon assistant foreground | foreground = do - showStart "watch" "." + showStart (if assistant then "assistant" else "watch") "." go id | otherwise = do logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile diff --git a/Command/Assistant.hs b/Command/Assistant.hs new file mode 100644 index 0000000000..60eac5d219 --- /dev/null +++ b/Command/Assistant.hs @@ -0,0 +1,18 @@ +{- git-annex assistant + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Command.Assistant where + +import Command +import qualified Command.Watch + +def :: [Command] +def = [withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $ + command "assistant" paramNothing seek "automatically handle changes"] + +seek :: [CommandSeek] +seek = Command.Watch.mkSeek True diff --git a/Command/Watch.hs b/Command/Watch.hs index 5681b38619..744844c4dc 100644 --- a/Command/Watch.hs +++ b/Command/Watch.hs @@ -1,6 +1,3 @@ -{-# LANGUAGE CPP #-} -{-# LANGUAGE BangPatterns #-} - {- git-annex watch command - - Copyright 2012 Joey Hess @@ -19,10 +16,13 @@ def :: [Command] def = [withOptions [foregroundOption, stopOption] $ command "watch" paramNothing seek "watch for changes"] -seek :: [CommandSeek] -seek = [withFlag stopOption $ \stopdaemon -> +mkSeek :: Bool -> [CommandSeek] +mkSeek assistant = [withFlag stopOption $ \stopdaemon -> withFlag foregroundOption $ \foreground -> - withNothing $ start foreground stopdaemon] + withNothing $ start assistant foreground stopdaemon] + +seek :: [CommandSeek] +seek = mkSeek False foregroundOption :: Option foregroundOption = Option.flag [] "foreground" "do not daemonize" @@ -30,9 +30,9 @@ foregroundOption = Option.flag [] "foreground" "do not daemonize" stopOption :: Option stopOption = Option.flag [] "stop" "stop daemon" -start :: Bool -> Bool -> CommandStart -start foreground stopdaemon = notBareRepo $ do +start :: Bool -> Bool -> Bool -> CommandStart +start assistant foreground stopdaemon = notBareRepo $ do if stopdaemon then stopDaemon - else startDaemon foreground -- does not return + else startDaemon assistant foreground -- does not return stop diff --git a/GitAnnex.hs b/GitAnnex.hs index a4c5eb8490..ee451352f4 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -59,6 +59,7 @@ import qualified Command.Map import qualified Command.Upgrade import qualified Command.Version import qualified Command.Watch +import qualified Command.Assistant cmds :: [Command] cmds = concat @@ -101,6 +102,7 @@ cmds = concat , Command.Upgrade.def , Command.Version.def , Command.Watch.def + , Command.Assistant.def ] options :: [Option] diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 39fad04882..965a07f0d7 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -180,6 +180,10 @@ subdirectories). To not daemonize, run with --foreground ; to stop a running daemon, run with --stop +* assistant + + Like watch, but also automatically syncs changes to other remotes. + # REPOSITORY SETUP COMMANDS * init [description] From 28e28bc0436cb0a33e570b1a1f678e80a770a21a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Jun 2012 13:39:44 -0400 Subject: [PATCH 3887/8313] stub syncer thread and commit channel --- Assistant.hs | 17 +++++++++-------- Assistant/Changes.hs | 25 +++++++++--------------- Assistant/Commits.hs | 32 +++++++++++++++++++++++++++++++ Assistant/Committer.hs | 6 ++++-- Assistant/Syncer.hs | 29 ++++++++++++++++++++++++++++ Assistant/ThreadedMonad.hs | 3 ++- Assistant/Watcher.hs | 4 +--- Utility/TSet.hs | 39 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 125 insertions(+), 30 deletions(-) create mode 100644 Assistant/Commits.hs create mode 100644 Assistant/Syncer.hs create mode 100644 Utility/TSet.hs diff --git a/Assistant.hs b/Assistant.hs index 33c7cef36c..5a3fa5a9d4 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -22,9 +22,11 @@ - Thread 5: committer - Waits for changes to occur, and runs the git queue to update its - index, then commits. - - Thread 6: status logger + - Thread 6: syncer + - Waits for commits to be made, and syncs the git repo to remotes. + - Thread 7: status logger - Wakes up periodically and records the daemon's status to disk. - - Thread 7: sanity checker + - Thread 8: sanity checker - Wakes up periodically (rarely) and does sanity checks. - - ThreadState: (MVar) @@ -47,8 +49,10 @@ import Common.Annex import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.Changes +import Assistant.Commits import Assistant.Watcher import Assistant.Committer +import Assistant.Syncer import Assistant.SanityChecker import qualified Utility.Daemon import Utility.LogFile @@ -70,12 +74,9 @@ startDaemon assistant foreground dstatus <- startDaemonStatus liftIO $ a $ do changechan <- newChangeChan - -- The commit thread is started early, - -- so that the user can immediately - -- begin adding files and having them - -- committed, even while the startup scan - -- is taking place. - _ <- forkIO $ commitThread st changechan + commitchan <- newCommitChan + _ <- forkIO $ commitThread st changechan commitchan + _ <- forkIO $ syncThread st commitchan _ <- forkIO $ daemonStatusThread st dstatus _ <- forkIO $ sanityCheckerThread st dstatus changechan -- Does not return. diff --git a/Assistant/Changes.hs b/Assistant/Changes.hs index 173ba19220..47eae83ef6 100644 --- a/Assistant/Changes.hs +++ b/Assistant/Changes.hs @@ -8,14 +8,14 @@ module Assistant.Changes where import Common.Annex import qualified Annex.Queue import Types.KeySource +import Utility.TSet -import Control.Concurrent.STM import Data.Time.Clock data ChangeType = AddChange | LinkChange | RmChange | RmDirChange deriving (Show, Eq) -type ChangeChan = TChan Change +type ChangeChan = TSet Change data Change = Change @@ -29,11 +29,8 @@ data Change } deriving (Show) -runChangeChan :: STM a -> IO a -runChangeChan = atomically - newChangeChan :: IO ChangeChan -newChangeChan = atomically newTChan +newChangeChan = newTSet {- Handlers call this when they made a change that needs to get committed. -} madeChange :: FilePath -> ChangeType -> Annex (Maybe Change) @@ -65,17 +62,13 @@ finishedChange c = c {- Gets all unhandled changes. - Blocks until at least one change is made. -} getChanges :: ChangeChan -> IO [Change] -getChanges chan = runChangeChan $ do - c <- readTChan chan - go [c] - where - go l = do - v <- tryReadTChan chan - case v of - Nothing -> return l - Just c -> go (c:l) +getChanges = getTSet {- Puts unhandled changes back into the channel. - Note: Original order is not preserved. -} refillChanges :: ChangeChan -> [Change] -> IO () -refillChanges chan cs = runChangeChan $ mapM_ (writeTChan chan) cs +refillChanges = putTSet + +{- Records a change in the channel. -} +recordChange :: ChangeChan -> Change -> IO () +recordChange = putTSet1 diff --git a/Assistant/Commits.hs b/Assistant/Commits.hs new file mode 100644 index 0000000000..152544e7ce --- /dev/null +++ b/Assistant/Commits.hs @@ -0,0 +1,32 @@ +{- git-annex assistant commit tracking + - + - Copyright 2012 Joey Hess + -} + +module Assistant.Commits where + +import Utility.TSet + +import Data.Time.Clock + +type CommitChan = TSet Commit + +data Commit = Commit UTCTime + deriving (Show) + +newCommitChan :: IO CommitChan +newCommitChan = newTSet + +{- Gets all unhandled commits. + - Blocks until at least one commit is made. -} +getCommits :: CommitChan -> IO [Commit] +getCommits = getTSet + +{- Puts unhandled commits back into the channel. + - Note: Original order is not preserved. -} +refillCommits :: CommitChan -> [Commit] -> IO () +refillCommits = putTSet + +{- Records a commit in the channel. -} +recordCommit :: CommitChan -> Commit -> IO () +recordCommit = putTSet1 diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs index 63df8cafcc..acdee1408d 100644 --- a/Assistant/Committer.hs +++ b/Assistant/Committer.hs @@ -7,6 +7,7 @@ module Assistant.Committer where import Common.Annex import Assistant.Changes +import Assistant.Commits import Assistant.ThreadedMonad import Assistant.Watcher import qualified Annex @@ -26,8 +27,8 @@ import qualified Data.Set as S import Data.Either {- This thread makes git commits at appropriate times. -} -commitThread :: ThreadState -> ChangeChan -> IO () -commitThread st changechan = runEvery (Seconds 1) $ do +commitThread :: ThreadState -> ChangeChan -> CommitChan -> IO () +commitThread st changechan commitchan = runEvery (Seconds 1) $ do -- We already waited one second as a simple rate limiter. -- Next, wait until at least one change is available for -- processing. @@ -40,6 +41,7 @@ commitThread st changechan = runEvery (Seconds 1) $ do if shouldCommit time readychanges then do void $ tryIO $ runThreadState st commitStaged + recordCommit commitchan (Commit time) else refillChanges changechan readychanges else refillChanges changechan changes diff --git a/Assistant/Syncer.hs b/Assistant/Syncer.hs new file mode 100644 index 0000000000..059859c073 --- /dev/null +++ b/Assistant/Syncer.hs @@ -0,0 +1,29 @@ +{- git-annex assistant git syncing thread + - + - Copyright 2012 Joey Hess + -} + +module Assistant.Syncer where + +import Assistant.Commits +import Assistant.ThreadedMonad +import qualified Command.Sync +import Utility.ThreadScheduler + +{- This thread syncs git commits out to remotes. -} +syncThread :: ThreadState -> CommitChan -> IO () +syncThread st commitchan = runEvery (Seconds 2) $ do + -- We already waited two seconds as a simple rate limiter. + -- Next, wait until at least one commit has been made + commits <- getCommits commitchan + -- Now see if now's a good time to sync. + if shouldSync commits + then syncToRemotes + else refillCommits commitchan commits + +{- Decide if now is a good time to sync commits to remotes. -} +shouldSync :: [Commit] -> Bool +shouldSync commits = not (null commits) + +syncToRemotes :: IO () +syncToRemotes = return () -- TOOD diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 51f579d07f..91a311deeb 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -32,7 +32,8 @@ withThreadState a = do {- Runs an Annex action, using the state from the MVar. - - - This serializes calls by threads. -} + - This serializes calls by threads; only one thread can run in Annex at a + - time. -} runThreadState :: ThreadState -> Annex a -> IO a runThreadState mvar a = do startstate <- takeMVar mvar diff --git a/Assistant/Watcher.hs b/Assistant/Watcher.hs index db58f01e8b..78330c8d0a 100644 --- a/Assistant/Watcher.hs +++ b/Assistant/Watcher.hs @@ -27,7 +27,6 @@ import Annex.Content import Annex.CatFile import Git.Types -import Control.Concurrent.STM import Data.Bits.Utils import qualified Data.ByteString.Lazy as L @@ -96,8 +95,7 @@ runHandler st dstatus changechan handler file filestatus = void $ do case r of Left e -> print e Right Nothing -> noop - Right (Just change) -> void $ - runChangeChan $ writeTChan changechan change + Right (Just change) -> recordChange changechan change where go = runThreadState st $ handler file filestatus dstatus diff --git a/Utility/TSet.hs b/Utility/TSet.hs new file mode 100644 index 0000000000..24d345477c --- /dev/null +++ b/Utility/TSet.hs @@ -0,0 +1,39 @@ +{- Transactional sets + - + - Copyright 2012 Joey Hess + -} + +module Utility.TSet where + +import Common + +import Control.Concurrent.STM + +type TSet = TChan + +runTSet :: STM a -> IO a +runTSet = atomically + +newTSet :: IO (TSet a) +newTSet = atomically newTChan + +{- Gets the contents of the TSet. Blocks until at least one item is + - present. -} +getTSet :: TSet a -> IO [a] +getTSet tset = runTSet $ do + c <- readTChan tset + go [c] + where + go l = do + v <- tryReadTChan tset + case v of + Nothing -> return l + Just c -> go (c:l) + +{- Puts items into a TSet. -} +putTSet :: TSet a -> [a] -> IO () +putTSet tset vs = runTSet $ mapM_ (writeTChan tset) vs + +{- Put a single item into a TSet. -} +putTSet1 :: TSet a -> a -> IO () +putTSet1 tset v = void $ runTSet $ writeTChan tset v From 153942cc6eb4e1ab5ae730261aa3266fd1b41721 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Jun 2012 15:47:02 -0400 Subject: [PATCH 3888/8313] update --- doc/design/assistant/syncing.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 56c9692e36..05c7483dde 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -5,7 +5,7 @@ all the other git clones, at both the git level and the key/value level. 1. Can use `git annex sync`, which already handles bidirectional syncing. When a change is committed, launch the part of `git annex sync` that pushes - out changes. + out changes. **done**; changes are pushed out to all remotes in parallel 1. Watch `.git/refs/remotes/` for changes (which would be pushed in from another node via `git annex sync`), and run the part of `git annex sync` that merges in received changes, and follow it by the part that pushes out From e9630e90decac4fe0c999af88131bd4b7c9d979f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Jun 2012 15:46:21 -0400 Subject: [PATCH 3889/8313] the syncer now pushes out changes to remotes, in parallel Note that, since this always pushes branch synced/master to the remote, it assumes that master has already gotten all the commits that are on the remote merged in. Otherwise, fast-forward prevention may prevent the push. That's probably ok, because the next stage is to automatically detect incoming pushes and merge. --- Assistant/Syncer.hs | 64 ++++++++++++++++++++++++++++++-------- Command/Sync.hs | 43 ++++++++++++++++--------- Utility/Parallel.hs | 20 ++++++++++++ Utility/ThreadScheduler.hs | 6 ++++ 4 files changed, 105 insertions(+), 28 deletions(-) create mode 100644 Utility/Parallel.hs diff --git a/Assistant/Syncer.hs b/Assistant/Syncer.hs index 059859c073..c579c1c280 100644 --- a/Assistant/Syncer.hs +++ b/Assistant/Syncer.hs @@ -5,25 +5,63 @@ module Assistant.Syncer where +import Common.Annex import Assistant.Commits import Assistant.ThreadedMonad import qualified Command.Sync import Utility.ThreadScheduler +import Utility.Parallel + +import Data.Time.Clock + +data FailedSync = FailedSync + { failedRemote :: Remote + , failedTimeStamp :: UTCTime + } {- This thread syncs git commits out to remotes. -} syncThread :: ThreadState -> CommitChan -> IO () -syncThread st commitchan = runEvery (Seconds 2) $ do - -- We already waited two seconds as a simple rate limiter. - -- Next, wait until at least one commit has been made - commits <- getCommits commitchan - -- Now see if now's a good time to sync. - if shouldSync commits - then syncToRemotes - else refillCommits commitchan commits +syncThread st commitchan = do + remotes <- runThreadState st $ Command.Sync.syncRemotes [] + runEveryWith (Seconds 2) [] $ \failedsyncs -> do + -- We already waited two seconds as a simple rate limiter. + -- Next, wait until at least one commit has been made + commits <- getCommits commitchan + -- Now see if now's a good time to sync. + time <- getCurrentTime + if shouldSync time commits failedsyncs + then syncToRemotes time st remotes + else do + refillCommits commitchan commits + return failedsyncs -{- Decide if now is a good time to sync commits to remotes. -} -shouldSync :: [Commit] -> Bool -shouldSync commits = not (null commits) +{- Decide if now is a good time to sync to remotes. + - + - Current strategy: Immediately sync all commits. The commit machinery + - already determines batches of changes, so we can't easily determine + - batches better. + - + - TODO: FailedSyncs are only retried the next time there's a commit. + - Should retry them periodically, or when a remote that was not available + - becomes available. + -} +shouldSync :: UTCTime -> [Commit] -> [FailedSync] -> Bool +shouldSync _now commits _failedremotes + | not (null commits) = True + | otherwise = False -syncToRemotes :: IO () -syncToRemotes = return () -- TOOD +{- Updates the local sync branch, then pushes it to all remotes, in + - parallel. + - + - Avoids running possibly long-duration commands in the Annex monad, so + - as not to block other threads. -} +syncToRemotes :: UTCTime -> ThreadState -> [Remote] -> IO [FailedSync] +syncToRemotes now st remotes = do + (g, branch) <- runThreadState st $ + (,) <$> fromRepo id <*> Command.Sync.currentBranch + Command.Sync.updateBranch (Command.Sync.syncBranch branch) g + map (`FailedSync` now) <$> inParallel (go g branch) remotes + where + go g branch remote = + ifM (Command.Sync.pushBranch remote branch g) + ( exitSuccess, exitFailure) diff --git a/Command/Sync.hs b/Command/Sync.hs index 5fb49d30c5..110cf2a6c7 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -31,7 +31,7 @@ def = [command "sync" (paramOptional (paramRepeating paramRemote)) -- syncing involves several operations, any of which can independently fail seek :: CommandSeek seek rs = do - !branch <- fromMaybe nobranch <$> inRepo Git.Branch.current + branch <- currentBranch remotes <- syncRemotes rs return $ concat [ [ commit ] @@ -41,6 +41,11 @@ seek rs = do , [ pushLocal branch ] , [ pushRemote remote branch | remote <- remotes ] ] + +currentBranch :: Annex Git.Ref +currentBranch = do + !branch <- fromMaybe nobranch <$> inRepo Git.Branch.current + return branch where nobranch = error "no branch is checked out" @@ -90,7 +95,7 @@ mergeLocal branch = go =<< needmerge syncbranch = syncBranch branch needmerge = do unlessM (inRepo $ Git.Ref.exists syncbranch) $ - updateBranch syncbranch + inRepo $ updateBranch syncbranch inRepo $ Git.Branch.changed branch syncbranch go False = stop go True = do @@ -99,17 +104,17 @@ mergeLocal branch = go =<< needmerge pushLocal :: Git.Ref -> CommandStart pushLocal branch = do - updateBranch $ syncBranch branch + inRepo $ updateBranch $ syncBranch branch stop -updateBranch :: Git.Ref -> Annex () -updateBranch syncbranch = +updateBranch :: Git.Ref -> Git.Repo -> IO () +updateBranch syncbranch g = unlessM go $ error $ "failed to update " ++ show syncbranch where - go = inRepo $ Git.Command.runBool "branch" + go = Git.Command.runBool "branch" [ Param "-f" , Param $ show $ Git.Ref.base syncbranch - ] + ] g pullRemote :: Remote -> Git.Ref -> CommandStart pullRemote remote branch = do @@ -135,19 +140,27 @@ mergeRemote remote branch = all id <$> (mapM merge =<< tomerge) pushRemote :: Remote -> Git.Ref -> CommandStart pushRemote remote branch = go =<< needpush where - needpush = anyM (newer remote) [syncbranch, Annex.Branch.name] + needpush = anyM (newer remote) [syncBranch branch, Annex.Branch.name] go False = stop go True = do showStart "push" (Remote.name remote) next $ next $ do showOutput - inRepo $ Git.Command.runBool "push" - [ Param (Remote.name remote) - , Param (show Annex.Branch.name) - , Param refspec - ] - refspec = show (Git.Ref.base branch) ++ ":" ++ show (Git.Ref.base syncbranch) - syncbranch = syncBranch branch + inRepo $ pushBranch remote branch + +pushBranch :: Remote -> Git.Ref -> Git.Repo -> IO Bool +pushBranch remote branch g = + Git.Command.runBool "push" + [ Param (Remote.name remote) + , Param (show Annex.Branch.name) + , Param refspec + ] g + where + refspec = concat + [ show $ Git.Ref.base branch + , ":" + , show $ Git.Ref.base $ syncBranch branch + ] mergeAnnex :: CommandStart mergeAnnex = do diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs new file mode 100644 index 0000000000..a512a6d306 --- /dev/null +++ b/Utility/Parallel.hs @@ -0,0 +1,20 @@ +{- parallel processes + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Parallel where + +import Common + +{- Runs an action in parallel with a set of values. + - Returns values that caused the action to fail. -} +inParallel :: (v -> IO ()) -> [v] -> IO [v] +inParallel a v = do + pids <- mapM (forkProcess . a) v + statuses <- mapM (getProcessStatus True False) pids + return $ map fst $ filter failed $ zip v statuses + where + failed (_, status) = status /= Just (Exited ExitSuccess) diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs index 6557398fd7..07a7401600 100644 --- a/Utility/ThreadScheduler.hs +++ b/Utility/ThreadScheduler.hs @@ -24,6 +24,12 @@ runEvery n a = forever $ do threadDelaySeconds n a +runEveryWith :: Seconds -> a -> (a -> IO a) -> IO () +runEveryWith n val a = do + threadDelaySeconds n + val' <- a val + runEveryWith n val' a + threadDelaySeconds :: Seconds -> IO () threadDelaySeconds (Seconds n) = unboundDelay (fromIntegral n * oneSecond) where From e699ce18417729abbb9606f6a011628ad6616a64 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Jun 2012 17:01:08 -0400 Subject: [PATCH 3890/8313] added a merger thread Wow! I can create a file in repo a, and it instantly* shows up in repo b! * under 1 second anyway --- Assistant.hs | 23 +++++++--- Assistant/Merger.hs | 72 ++++++++++++++++++++++++++++++ Assistant/{Syncer.hs => Pusher.hs} | 40 ++++++++--------- Utility/Types/DirWatcher.hs | 3 ++ 4 files changed, 112 insertions(+), 26 deletions(-) create mode 100644 Assistant/Merger.hs rename Assistant/{Syncer.hs => Pusher.hs} (58%) diff --git a/Assistant.hs b/Assistant.hs index 5a3fa5a9d4..ce230533cf 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -22,11 +22,17 @@ - Thread 5: committer - Waits for changes to occur, and runs the git queue to update its - index, then commits. - - Thread 6: syncer - - Waits for commits to be made, and syncs the git repo to remotes. - - Thread 7: status logger + - Thread 6: pusher + - Waits for commits to be made, and pushes updated branches to remotes, + - in parallel. (Forks a process for each git push.) + - Thread 7: merger + - Waits for pushes to be received from remotes, and merges the + - updated branches into the current branch. This uses inotify + - on .git/refs/heads, so there are additional inotify threads + - associated with it, too. + - Thread 8: status logger - Wakes up periodically and records the daemon's status to disk. - - Thread 8: sanity checker + - Thread 9: sanity checker - Wakes up periodically (rarely) and does sanity checks. - - ThreadState: (MVar) @@ -41,6 +47,9 @@ - ChangeChan: (STM TChan) - Changes are indicated by writing to this channel. The committer - reads from it. + - CommitChan: (STM TChan) + - Commits are indicated by writing to this channel. The pusher reads + - from it. -} module Assistant where @@ -52,7 +61,8 @@ import Assistant.Changes import Assistant.Commits import Assistant.Watcher import Assistant.Committer -import Assistant.Syncer +import Assistant.Pusher +import Assistant.Merger import Assistant.SanityChecker import qualified Utility.Daemon import Utility.LogFile @@ -76,7 +86,8 @@ startDaemon assistant foreground changechan <- newChangeChan commitchan <- newCommitChan _ <- forkIO $ commitThread st changechan commitchan - _ <- forkIO $ syncThread st commitchan + _ <- forkIO $ pushThread st commitchan + _ <- forkIO $ mergeThread st _ <- forkIO $ daemonStatusThread st dstatus _ <- forkIO $ sanityCheckerThread st dstatus changechan -- Does not return. diff --git a/Assistant/Merger.hs b/Assistant/Merger.hs new file mode 100644 index 0000000000..660636842b --- /dev/null +++ b/Assistant/Merger.hs @@ -0,0 +1,72 @@ +{- git-annex assistant git merge thread + - + - Copyright 2012 Joey Hess + -} + +module Assistant.Merger where + +import Common.Annex +import Assistant.ThreadedMonad +import Utility.DirWatcher +import Utility.Types.DirWatcher +import qualified Git +import qualified Git.Command +import qualified Git.Branch +import qualified Command.Sync + +{- This thread watches for changes to .git/refs/heads/synced/*, + - which indicate incoming pushes. It merges those pushes into the + - currently checked out branch. -} +mergeThread :: ThreadState -> IO () +mergeThread st = do + g <- runThreadState st $ fromRepo id + let dir = Git.localGitDir g "refs" "heads" "synced" + createDirectoryIfMissing True dir + let hook a = Just $ runHandler g a + let hooks = mkWatchHooks + { addHook = hook onAdd + , errHook = hook onErr + } + watchDir dir (const False) hooks id + where + +type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO () + +{- Runs an action handler. + - + - Exceptions are ignored, otherwise a whole thread could be crashed. + -} +runHandler :: Git.Repo -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler g handler file filestatus = void $ do + either print (const noop) =<< tryIO go + where + go = handler g file filestatus + +{- Called when there's an error with inotify. -} +onErr :: Handler +onErr _ msg _ = error msg + +{- Called when a new branch ref is written. + - + - This relies on git's atomic method of updating branch ref files, + - which is to first write the new file to .lock, and then rename it + - over the old file. So, ignore .lock files, and the rename ensures + - the watcher sees a new file being added on each update. + - + - At startup, synthetic add events fire, causing this to run, but that's + - ok; it ensures that any changes pushed since the last time the assistant + - ran are merged in. + -} +onAdd :: Handler +onAdd g file _ + | ".lock" `isSuffixOf` file = noop + | otherwise = do + let branch = Git.Ref $ "refs" "heads" takeFileName file + current <- Git.Branch.current g + print (branch, current) + when (Just branch == current) $ + void $ mergeBranch branch g + +mergeBranch :: Git.Ref -> Git.Repo -> IO Bool +mergeBranch branch = Git.Command.runBool "merge" + [Param $ show $ Command.Sync.syncBranch branch] diff --git a/Assistant/Syncer.hs b/Assistant/Pusher.hs similarity index 58% rename from Assistant/Syncer.hs rename to Assistant/Pusher.hs index c579c1c280..119575b92a 100644 --- a/Assistant/Syncer.hs +++ b/Assistant/Pusher.hs @@ -1,9 +1,9 @@ -{- git-annex assistant git syncing thread +{- git-annex assistant git pushing thread - - Copyright 2012 Joey Hess -} -module Assistant.Syncer where +module Assistant.Pusher where import Common.Annex import Assistant.Commits @@ -14,39 +14,39 @@ import Utility.Parallel import Data.Time.Clock -data FailedSync = FailedSync +data FailedPush = FailedPush { failedRemote :: Remote , failedTimeStamp :: UTCTime } -{- This thread syncs git commits out to remotes. -} -syncThread :: ThreadState -> CommitChan -> IO () -syncThread st commitchan = do +{- This thread pushes git commits out to remotes. -} +pushThread :: ThreadState -> CommitChan -> IO () +pushThread st commitchan = do remotes <- runThreadState st $ Command.Sync.syncRemotes [] - runEveryWith (Seconds 2) [] $ \failedsyncs -> do + runEveryWith (Seconds 2) [] $ \failedpushes -> do -- We already waited two seconds as a simple rate limiter. -- Next, wait until at least one commit has been made commits <- getCommits commitchan - -- Now see if now's a good time to sync. + -- Now see if now's a good time to push. time <- getCurrentTime - if shouldSync time commits failedsyncs - then syncToRemotes time st remotes + if shouldPush time commits failedpushes + then pushToRemotes time st remotes else do refillCommits commitchan commits - return failedsyncs + return failedpushes -{- Decide if now is a good time to sync to remotes. +{- Decide if now is a good time to push to remotes. - - - Current strategy: Immediately sync all commits. The commit machinery + - Current strategy: Immediately push all commits. The commit machinery - already determines batches of changes, so we can't easily determine - batches better. - - - TODO: FailedSyncs are only retried the next time there's a commit. + - TODO: FailedPushs are only retried the next time there's a commit. - Should retry them periodically, or when a remote that was not available - becomes available. -} -shouldSync :: UTCTime -> [Commit] -> [FailedSync] -> Bool -shouldSync _now commits _failedremotes +shouldPush :: UTCTime -> [Commit] -> [FailedPush] -> Bool +shouldPush _now commits _failedremotes | not (null commits) = True | otherwise = False @@ -55,13 +55,13 @@ shouldSync _now commits _failedremotes - - Avoids running possibly long-duration commands in the Annex monad, so - as not to block other threads. -} -syncToRemotes :: UTCTime -> ThreadState -> [Remote] -> IO [FailedSync] -syncToRemotes now st remotes = do +pushToRemotes :: UTCTime -> ThreadState -> [Remote] -> IO [FailedPush] +pushToRemotes now st remotes = do (g, branch) <- runThreadState st $ (,) <$> fromRepo id <*> Command.Sync.currentBranch Command.Sync.updateBranch (Command.Sync.syncBranch branch) g - map (`FailedSync` now) <$> inParallel (go g branch) remotes + map (`FailedPush` now) <$> inParallel (push g branch) remotes where - go g branch remote = + push g branch remote = ifM (Command.Sync.pushBranch remote branch g) ( exitSuccess, exitFailure) diff --git a/Utility/Types/DirWatcher.hs b/Utility/Types/DirWatcher.hs index c828a05938..ba7eae6a16 100644 --- a/Utility/Types/DirWatcher.hs +++ b/Utility/Types/DirWatcher.hs @@ -20,3 +20,6 @@ data WatchHooks = WatchHooks , delDirHook :: Hook FilePath , errHook :: Hook String -- error message } + +mkWatchHooks :: WatchHooks +mkWatchHooks = WatchHooks Nothing Nothing Nothing Nothing Nothing From 264dd38c6547406041adad4fea2c603d5a146a97 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 22 Jun 2012 17:17:41 -0400 Subject: [PATCH 3891/8313] blog for the day --- .../assistant/blog/day_15__its_aliiive.mdwn | 33 +++++++++++++++++++ doc/design/assistant/syncing.mdwn | 3 +- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 doc/design/assistant/blog/day_15__its_aliiive.mdwn diff --git a/doc/design/assistant/blog/day_15__its_aliiive.mdwn b/doc/design/assistant/blog/day_15__its_aliiive.mdwn new file mode 100644 index 0000000000..10ef4ffe56 --- /dev/null +++ b/doc/design/assistant/blog/day_15__its_aliiive.mdwn @@ -0,0 +1,33 @@ +Syncing works! I have two clones, and any file I create in the first +is immediately visible in the second. Delete that file from the second, and +it's immediately removed from the first. + +Most of my work today felt like stitching existing limbs onto a pre-existing +monster. Took the committer thread, that waits for changes and commits them, +and refashioned it into a pusher thread, that waits for commits and pushes +them. Took the watcher thread, that watches for files being made, +and refashioned it into a merger thread, that watches for git refs being +updated. Pulled in bits of the `git annex sync` command to reanimate this. + +It may be a shambling hulk, but it works. + +Actually, it's not much of a shambling hulk; I refactored my code after +copying it. ;) + +I think I'm up to 11 threads now in the new +`git annex assistant` command, each with its own job, and each needing +to avoid stepping on the other's toes. I did see one MVar deadlock error +today, which I have not managed to reproduce after some changes. I think +the committer thread was triggering the merger thread, which probably +then waited on the Annex state MVar the committer thread had held. + +Anyway, it even pushes to remotes in parallel, and keeps track of remotes +it failed to push to, although as of yet it doesn't do any attempt at +periodically retrying. + +One bug I need to deal with is that the push code assumes any change +made to the remote has already been pushed back to it. When it hasn't, +the push will fail due to not being a fast-forward. I need to make it +detect this case and pull before pushing. + +(I've pushed this work out in a new `assistant branch`.) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 05c7483dde..4f29df9453 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -11,7 +11,8 @@ all the other git clones, at both the git level and the key/value level. that merges in received changes, and follow it by the part that pushes out changes (sending them to any other remotes). [The watching can be done with the existing inotify code! This avoids needing - any special mechanism to notify a remote that it's been synced to.] + any special mechanism to notify a remote that it's been synced to.] + **done** 2. Use a git merge driver that adds both conflicting files, so conflicts never break a sync. 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of From 3606e94336abafe2bf01de6beaa778a4d3e87cf8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlc1og3PqIGudOMkFNrCCNg66vB7s-jLpc" Date: Fri, 22 Jun 2012 22:10:22 +0000 Subject: [PATCH 3892/8313] Added a comment: how is this different than rsync? --- ...mment_1_e8a53592adb13f7d7f212a2eb5a18a31._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/special_remotes/directory/comment_1_e8a53592adb13f7d7f212a2eb5a18a31._comment diff --git a/doc/special_remotes/directory/comment_1_e8a53592adb13f7d7f212a2eb5a18a31._comment b/doc/special_remotes/directory/comment_1_e8a53592adb13f7d7f212a2eb5a18a31._comment new file mode 100644 index 0000000000..b2a041c539 --- /dev/null +++ b/doc/special_remotes/directory/comment_1_e8a53592adb13f7d7f212a2eb5a18a31._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlc1og3PqIGudOMkFNrCCNg66vB7s-jLpc" + nickname="Paul" + subject="how is this different than rsync?" + date="2012-06-22T22:10:19Z" + content=""" +Thanks for this great tool! I was wondering what the differences are between using `type=directory`, `type=rsync`, or a bare git repo for directories? + +I guess I can't use just a regular repo because my USB drive is formatted as `vfat` -- which threw me for a loop the first time I heard about `git-annex` about a year ago, because I followed the walkthrough, and it didn't work as expected and gave up (now I know it was just a case of PEBKAC). It might be worth adding a note about [vfat](http://git-annex.branchable.com/bugs/fat_support/) to the \"Adding a remote\" section of the [walkthrough](http://git-annex.branchable.com/walkthrough/), since the unstated assumption there is that the USB drive is formatted as a filesystem that supports symlinks. + +Thanks again, my scientific data management just got a lot more sane! +"""]] From 3118eeb63fbd5ae6565c49c1e8d14f61b6f8f810 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 01:19:33 -0400 Subject: [PATCH 3893/8313] update --- doc/design/assistant/syncing.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 4f29df9453..8173457c55 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -13,6 +13,8 @@ all the other git clones, at both the git level and the key/value level. [The watching can be done with the existing inotify code! This avoids needing any special mechanism to notify a remote that it's been synced to.] **done** +1. Periodically retry pushes that failed. Also, detect if a push failed + due to not being up-to-date, pull, and repush. 2. Use a git merge driver that adds both conflicting files, so conflicts never break a sync. 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of From 38df4ed44f8deaba50b509a36814e59cf0388425 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 01:20:40 -0400 Subject: [PATCH 3894/8313] license --- Assistant/Changes.hs | 2 ++ Assistant/Commits.hs | 2 ++ Assistant/Committer.hs | 2 ++ Assistant/DaemonStatus.hs | 2 ++ Assistant/Merger.hs | 2 ++ Assistant/Pusher.hs | 2 ++ Assistant/SanityChecker.hs | 2 ++ Assistant/ThreadedMonad.hs | 2 ++ 8 files changed, 16 insertions(+) diff --git a/Assistant/Changes.hs b/Assistant/Changes.hs index 47eae83ef6..eca922109d 100644 --- a/Assistant/Changes.hs +++ b/Assistant/Changes.hs @@ -1,6 +1,8 @@ {- git-annex assistant change tracking - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.Changes where diff --git a/Assistant/Commits.hs b/Assistant/Commits.hs index 152544e7ce..86fd7599fd 100644 --- a/Assistant/Commits.hs +++ b/Assistant/Commits.hs @@ -1,6 +1,8 @@ {- git-annex assistant commit tracking - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.Commits where diff --git a/Assistant/Committer.hs b/Assistant/Committer.hs index acdee1408d..0c69995912 100644 --- a/Assistant/Committer.hs +++ b/Assistant/Committer.hs @@ -1,6 +1,8 @@ {- git-annex assistant commit thread - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.Committer where diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index e5ba3d1512..c7713e7d56 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -1,6 +1,8 @@ {- git-annex assistant daemon status - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.DaemonStatus where diff --git a/Assistant/Merger.hs b/Assistant/Merger.hs index 660636842b..48cf02ae59 100644 --- a/Assistant/Merger.hs +++ b/Assistant/Merger.hs @@ -1,6 +1,8 @@ {- git-annex assistant git merge thread - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.Merger where diff --git a/Assistant/Pusher.hs b/Assistant/Pusher.hs index 119575b92a..7504d44c20 100644 --- a/Assistant/Pusher.hs +++ b/Assistant/Pusher.hs @@ -1,6 +1,8 @@ {- git-annex assistant git pushing thread - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.Pusher where diff --git a/Assistant/SanityChecker.hs b/Assistant/SanityChecker.hs index e2ca9da740..b74c9fe5d2 100644 --- a/Assistant/SanityChecker.hs +++ b/Assistant/SanityChecker.hs @@ -1,6 +1,8 @@ {- git-annex assistant sanity checker - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} module Assistant.SanityChecker ( diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 91a311deeb..6d3d25778e 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -1,6 +1,8 @@ {- making the Annex monad available across threads - - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. -} {-# LANGUAGE BangPatterns #-} From a71e7161fc4aad9ec5dfbce219d9d25703a9e3a6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 01:33:10 -0400 Subject: [PATCH 3895/8313] golfing --- Utility/Parallel.hs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs index a512a6d306..6e4671c057 100644 --- a/Utility/Parallel.hs +++ b/Utility/Parallel.hs @@ -12,9 +12,9 @@ import Common {- Runs an action in parallel with a set of values. - Returns values that caused the action to fail. -} inParallel :: (v -> IO ()) -> [v] -> IO [v] -inParallel a v = do - pids <- mapM (forkProcess . a) v +inParallel a l = do + pids <- mapM (forkProcess . a) l statuses <- mapM (getProcessStatus True False) pids - return $ map fst $ filter failed $ zip v statuses + return $ map fst $ filter (failed . snd) $ zip l statuses where - failed (_, status) = status /= Just (Exited ExitSuccess) + failed v = v /= Just (Exited ExitSuccess) From d707f2dce5c3ed1a56c7bf3d65be7bef105f5dd1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 01:35:40 -0400 Subject: [PATCH 3896/8313] cleanup --- Assistant/Merger.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Assistant/Merger.hs b/Assistant/Merger.hs index 48cf02ae59..fbdf9ca015 100644 --- a/Assistant/Merger.hs +++ b/Assistant/Merger.hs @@ -65,7 +65,6 @@ onAdd g file _ | otherwise = do let branch = Git.Ref $ "refs" "heads" takeFileName file current <- Git.Branch.current g - print (branch, current) when (Just branch == current) $ void $ mergeBranch branch g From f0617ece7c2a56d27a57136e7289a69751b8f98e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sat, 23 Jun 2012 07:02:16 +0000 Subject: [PATCH 3897/8313] --- ...e_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn diff --git a/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn new file mode 100644 index 0000000000..c3f915e7dd --- /dev/null +++ b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__.mdwn @@ -0,0 +1 @@ +Logging to file would be nice when running git-annex as a daemon so when something fails or when certain events happens one can look at the logs to see what has failed, unless I'm missing something this probably should be on the wishlist or roadmap. From c6daf4e6c22629fa6adacf3101129e2d3c8f89a3 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Sat, 23 Jun 2012 07:24:23 +0000 Subject: [PATCH 3898/8313] --- ...40__merge_command_now_asks_for_a_commit_message__34__.mdwn | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn diff --git a/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn b/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn new file mode 100644 index 0000000000..2dd7e51b16 --- /dev/null +++ b/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn @@ -0,0 +1,4 @@ +running 'git annex sync' doesn't merge the branches as expected (from the limited testing I have done) with git 1.7.10, the behaviour of merge has changed, it now asks for a commit message. I would expect setting _GIT_MERGE_AUTOEDIT=no_ should resolve this issue. + +I had to manually do a merge (or set that variable) to get the branches back in sync again, this confused me a bit when git-annex watch was running in the background on a remote and it did not pick up the changes. + From c79e3b67e9e55817992d2d0c72fd6f01a6a403ec Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 10:22:56 -0400 Subject: [PATCH 3899/8313] sync: Avoid recent git's interactive merge. --- Command/Sync.hs | 3 ++- Git/Merge.hs | 17 +++++++++++++++++ debian/changelog | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Git/Merge.hs diff --git a/Command/Sync.hs b/Command/Sync.hs index 5fb49d30c5..1da6b0b812 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -16,6 +16,7 @@ import qualified Remote import qualified Annex import qualified Annex.Branch import qualified Git.Command +import qualified Git.Merge import qualified Git.Branch import qualified Git.Ref import qualified Git @@ -157,7 +158,7 @@ mergeAnnex = do mergeFrom :: Git.Ref -> CommandCleanup mergeFrom branch = do showOutput - inRepo $ Git.Command.runBool "merge" [Param $ show branch] + inRepo $ Git.Merge.mergeNonInteractive branch changed :: Remote -> Git.Ref -> Annex Bool changed remote b = do diff --git a/Git/Merge.hs b/Git/Merge.hs new file mode 100644 index 0000000000..08fc6fb487 --- /dev/null +++ b/Git/Merge.hs @@ -0,0 +1,17 @@ +{- git merging + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Git.Merge where + +import Common +import Git +import Git.Command + +{- Avoids recent git's interactive merge. -} +mergeNonInteractive :: Ref -> Repo -> IO Bool +mergeNonInteractive branch = runBool "merge" + [Param "--no-edit", Param $ show branch] diff --git a/debian/changelog b/debian/changelog index 5d8d973c67..a580f88bc1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,7 @@ git-annex (3.20120616) UNRELEASED; urgency=low Available on Linux, BSDs, and OSX! * Enable diskfree on kfreebsd, using kqueue. * unused: Fix crash when key names contain invalid utf8. + * sync: Avoid recent git's interactive merge. -- Joey Hess Tue, 12 Jun 2012 11:35:59 -0400 From 325053e8de109329eac31017271134b1c5faef04 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 10:27:04 -0400 Subject: [PATCH 3900/8313] fixed --- ...nd_now_asks_for_a_commit_message__34__.mdwn | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn b/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn index 2dd7e51b16..05024ffe9f 100644 --- a/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn +++ b/doc/bugs/Possible_issues_with_git_1.7.10_and_newer___40__merge_command_now_asks_for_a_commit_message__34__.mdwn @@ -1,4 +1,18 @@ -running 'git annex sync' doesn't merge the branches as expected (from the limited testing I have done) with git 1.7.10, the behaviour of merge has changed, it now asks for a commit message. I would expect setting _GIT_MERGE_AUTOEDIT=no_ should resolve this issue. +running 'git annex sync' doesn't merge the branches as expected (from the +limited testing I have done) with git 1.7.10, the behaviour of merge has +changed, it now asks for a commit message. I would expect setting +_GIT_MERGE_AUTOEDIT=no_ should resolve this issue. -I had to manually do a merge (or set that variable) to get the branches back in sync again, this confused me a bit when git-annex watch was running in the background on a remote and it did not pick up the changes. +I had to manually do a merge (or set that variable) to get the branches +back in sync again, this confused me a bit when git-annex watch was running +in the background on a remote and it did not pick up the changes. +> Yeah, I tend to miss these since the first thing I did when this +> misfeature was being posted was to write a mail discouraging them from +> doing it (sadly ignored), and then set in ~/.environment: + + # My time is more valuable than git's new, bad default + GIT_MERGE_AUTOEDIT=no + export GIT_MERGE_AUTOEDIT + +> Anyway, I've made sync run merge with --no-edit now. [[done]] --[[Joey]] From 19eee6a1df2a6c724e6d6dbe842b40dc1c17f65b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 23 Jun 2012 10:29:46 -0400 Subject: [PATCH 3901/8313] noninteractive merge --- Assistant/Merger.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Assistant/Merger.hs b/Assistant/Merger.hs index fbdf9ca015..988cbd8a63 100644 --- a/Assistant/Merger.hs +++ b/Assistant/Merger.hs @@ -12,7 +12,7 @@ import Assistant.ThreadedMonad import Utility.DirWatcher import Utility.Types.DirWatcher import qualified Git -import qualified Git.Command +import qualified Git.Merge import qualified Git.Branch import qualified Command.Sync @@ -69,5 +69,4 @@ onAdd g file _ void $ mergeBranch branch g mergeBranch :: Git.Ref -> Git.Repo -> IO Bool -mergeBranch branch = Git.Command.runBool "merge" - [Param $ show $ Command.Sync.syncBranch branch] +mergeBranch = Git.Merge.mergeNonInteractive . Command.Sync.syncBranch From d233537451ffd5d6fcdaf38f30acee2969c1c934 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Sat, 23 Jun 2012 14:30:22 +0000 Subject: [PATCH 3902/8313] Added a comment --- .../comment_1_42aa2b61b880f4048d874210212aa63b._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment diff --git a/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment new file mode 100644 index 0000000000..c0746a5a7c --- /dev/null +++ b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_1_42aa2b61b880f4048d874210212aa63b._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.14.60" + subject="comment 1" + date="2012-06-23T14:30:22Z" + content=""" +The logging format could be improved, but the daemon already logs to .git/annex/daemon.log. It also automatically rotates the log file. +"""]] From a0952dd0f9179e978ab007e162a43b9e6a6ef162 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 24 Jun 2012 12:51:18 -0400 Subject: [PATCH 3903/8313] releasing version 3.20120624 --- debian/changelog | 4 ++-- git-annex.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index a580f88bc1..42f37e572e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (3.20120616) UNRELEASED; urgency=low +git-annex (3.20120624) unstable; urgency=low * watch: New subcommand, a daemon which notices changes to files and automatically annexes new files, etc, so you don't @@ -8,7 +8,7 @@ git-annex (3.20120616) UNRELEASED; urgency=low * unused: Fix crash when key names contain invalid utf8. * sync: Avoid recent git's interactive merge. - -- Joey Hess Tue, 12 Jun 2012 11:35:59 -0400 + -- Joey Hess Sun, 24 Jun 2012 12:36:50 -0400 git-annex (3.20120614) unstable; urgency=medium diff --git a/git-annex.cabal b/git-annex.cabal index 1416a381dd..3902a077b5 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 3.20120616 +Version: 3.20120624 Cabal-Version: >= 1.8 License: GPL Maintainer: Joey Hess From fbc16360d578517ebbef8aad1332877111370bfb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 24 Jun 2012 12:51:36 -0400 Subject: [PATCH 3904/8313] add news item for git-annex 3.20120624 --- doc/news/version_3.20120511.mdwn | 13 ------------- doc/news/version_3.20120522.mdwn | 7 ------- doc/news/version_3.20120624.mdwn | 9 +++++++++ 3 files changed, 9 insertions(+), 20 deletions(-) delete mode 100644 doc/news/version_3.20120511.mdwn delete mode 100644 doc/news/version_3.20120522.mdwn create mode 100644 doc/news/version_3.20120624.mdwn diff --git a/doc/news/version_3.20120511.mdwn b/doc/news/version_3.20120511.mdwn deleted file mode 100644 index 19e8355224..0000000000 --- a/doc/news/version_3.20120511.mdwn +++ /dev/null @@ -1,13 +0,0 @@ -git-annex 3.20120511 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Rsync special remotes can be configured with shellescape=no - to avoid shell quoting that is normally done when using rsync over ssh. - This is known to be needed for certian rsync hosting providers - (specificially hidrive.strato.com) that use rsync over ssh but do not - pass it through the shell. - * dropunused: Allow specifying ranges to drop. - * addunused: New command, the opposite of dropunused, it relinks unused - content into the git repository. - * Fix use of several config settings: annex.ssh-options, - annex.rsync-options, annex.bup-split-options. (And adjust types to avoid - the bugs that broke several config settings.)"""]] \ No newline at end of file diff --git a/doc/news/version_3.20120522.mdwn b/doc/news/version_3.20120522.mdwn deleted file mode 100644 index 55c45900c1..0000000000 --- a/doc/news/version_3.20120522.mdwn +++ /dev/null @@ -1,7 +0,0 @@ -git-annex 3.20120522 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Pass -a to cp even when it supports --reflink=auto, to preserve - permissions. - * Clean up handling of git directory and git worktree. - * Add support for core.worktree, and fix support for GIT\_WORK\_TREE and - GIT\_DIR."""]] \ No newline at end of file diff --git a/doc/news/version_3.20120624.mdwn b/doc/news/version_3.20120624.mdwn new file mode 100644 index 0000000000..8b619228e1 --- /dev/null +++ b/doc/news/version_3.20120624.mdwn @@ -0,0 +1,9 @@ +git-annex 3.20120624 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * watch: New subcommand, a daemon which notices changes to + files and automatically annexes new files, etc, so you don't + need to manually run git commands when manipulating files. + Available on Linux, BSDs, and OSX! + * Enable diskfree on kfreebsd, using kqueue. + * unused: Fix crash when key names contain invalid utf8. + * sync: Avoid recent git's interactive merge."""]] \ No newline at end of file From 38b3d8da1b4f1afab9a70ad6e0482251a6c1f563 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 24 Jun 2012 17:51:08 -0400 Subject: [PATCH 3905/8313] typo --- doc/git-annex-shell.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 89a05b1d68..00c68ff3a5 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -46,7 +46,7 @@ first "/~/" or "/~user/" is expanded to the specified home directory. This runs rsync in server mode to transfer out the content of a key. -* commit +* commit directory This commits any staged changes to the git-annex branch. It also runs the annex-content hook. From 4f04745c6bbf9d82f238bf4745c28cd088cf1e8f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" Date: Mon, 25 Jun 2012 00:29:02 +0000 Subject: [PATCH 3906/8313] Added a comment: The fact that the keys changed causes merge conflicts --- ...mment_1_20f9b7b75786075de666b2146dc13a60._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment diff --git a/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment b/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment new file mode 100644 index 0000000000..7b6be15321 --- /dev/null +++ b/doc/upgrades/SHA_size/comment_1_20f9b7b75786075de666b2146dc13a60._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkjvjLHW9Omza7x1VEzIFQ8Z5honhRB90I" + nickname="Asheesh" + subject="The fact that the keys changed causes merge conflicts" + date="2012-06-25T00:28:59Z" + content=""" +FYI, I have run into a problem where if you 'git annex sync' between various 'git annex v3' repositories, if the different repositories are using different encodings of the SHA1 information (one including size, one not), then the 'git merge' will declare that they conflict. + +There's no indication that 'git annex migrate' is the right tool to run, except from perusing the 'git annex' man page. In my opinion this is a major user interface problem. + +-- Asheesh. +"""]] From 7c754c8f669b6aa3714c4f0411233f82d415f24f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkwR9uOA38yi5kEUvcEWNtRiZwpxXskayE" Date: Mon, 25 Jun 2012 02:21:40 +0000 Subject: [PATCH 3907/8313] Added a comment: Installation not working on OS X 10.6.8 --- ..._a93ad4b67c5df4243268bcf32562f6be._comment | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment diff --git a/doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment b/doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment new file mode 100644 index 0000000000..cd128a6f13 --- /dev/null +++ b/doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment @@ -0,0 +1,39 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkwR9uOA38yi5kEUvcEWNtRiZwpxXskayE" + nickname="Agustin" + subject="Installation not working on OS X 10.6.8" + date="2012-06-25T02:21:40Z" + content=""" +I try installing with brew because I already had brew setup in my machine, but all run ok but when I try to run cabal install git-annex I got an error with the hinotify-0.3.2 library complaining about a header file. + +Full trace: + +~~~ +sudo cabal install git-annex +Resolving dependencies... +Configuring hinotify-0.3.2... +Building hinotify-0.3.2... +Preprocessing library hinotify-0.3.2... +INotify.hsc:35:25: error: sys/inotify.h: No such file or directory +INotify.hsc: In function ‘main’: +INotify.hsc:259: error: invalid use of undefined type ‘struct inotify_event’ +INotify.hsc:260: error: invalid use of undefined type ‘struct inotify_event’ +INotify.hsc:261: error: invalid use of undefined type ‘struct inotify_event’ +INotify.hsc:262: error: invalid use of undefined type ‘struct inotify_event’ +INotify.hsc:265: error: invalid use of undefined type ‘struct inotify_event’ +INotify.hsc:266: error: invalid application of ‘sizeof’ to incomplete type ‘struct inotify_event’ +compiling dist/build/System/INotify_hsc_make.c failed (exit code 1) +command was: /usr/bin/gcc -c dist/build/System/INotify_hsc_make.c -o dist/build/System/INotify_hsc_make.o -m64 -fno-stack-protector -m64 -D__GLASGOW_HASKELL__=704 -Ddarwin_BUILD_OS -Ddarwin_HOST_OS -Dx86_64_BUILD_ARCH -Dx86_64_HOST_ARCH -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/directory-1.1.0.2/include -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/unix-2.5.1.0/include -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/old-time-1.1.0.0/include -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/bytestring-0.9.2.1/include -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/base-4.5.0.0/include -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/include -Idist/build/autogen -include dist/build/autogen/cabal_macros.h -I/usr/local/Cellar/ghc/7.4.1/lib/ghc-7.4.1/include/ +cabal: Error: some packages failed to install: +git-annex-3.20120624 depends on hinotify-0.3.2 which failed to install. +hinotify-0.3.2 failed during the building phase. The exception was: +ExitFailure 1 +~~~ + +Anyone has an idea how can I solve this. + +Thanks for the time! + +Agustin + +"""]] From fa5c88b8f1591e06eeeb7a9d70e3da5bde82d0ac Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkwR9uOA38yi5kEUvcEWNtRiZwpxXskayE" Date: Mon, 25 Jun 2012 02:51:11 +0000 Subject: [PATCH 3908/8313] Added a comment: For the moment --- ...ent_9_ae3ed5345bc84f57e44251d2e6c39342._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment diff --git a/doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment b/doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment new file mode 100644 index 0000000000..70fbc6c3c6 --- /dev/null +++ b/doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkwR9uOA38yi5kEUvcEWNtRiZwpxXskayE" + nickname="Agustin" + subject="For the moment" + date="2012-06-25T02:51:10Z" + content=""" +Hi Joey! I just comment that I could not install it but the issue is with the last version (the one you just release today, so no problem!! man on sunday?? you're awesome!!!) so I installed the previous one and no problem at all + +Thanks for all the efford and if you need me to try os whatever, feel free to ask! + +Thanks again + +Agustin +"""]] From fb415526e9bb7db0c571591276decce675416787 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Mon, 25 Jun 2012 13:37:36 +0000 Subject: [PATCH 3909/8313] --- ...g_weirdness_with_the_assistant_branch_on_OSX.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn diff --git a/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn new file mode 100644 index 0000000000..63431aae68 --- /dev/null +++ b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn @@ -0,0 +1,12 @@ +Running the 'assistant' branch, I occassionally get + +To myhost1:/Users/jtang/annex + ! [rejected] master -> synced/master (non-fast-forward) +error: failed to push some refs to 'myhost1:/Users/jtang/annex' +hint: Updates were rejected because a pushed branch tip is behind its remote +hint: counterpart. Check out this branch and merge the remote changes +hint: (e.g. 'git pull') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +(Recording state in git...) + +manually running a 'git annex sync' usually fixes it, I guess once the sync command runs periodically this problem will go away, is this even OSX specific? I don't quite get the behaviour that is described in [[design/assistant/blog/day_15__its_aliiive]]. From 4884f189224bbf96e4ca56e5644e9a25a9a52cb8 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Mon, 25 Jun 2012 15:29:30 +0000 Subject: [PATCH 3910/8313] Added a comment --- ...comment_2_d949edad6a330079f9e15f703f9091e3._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/special_remotes/directory/comment_2_d949edad6a330079f9e15f703f9091e3._comment diff --git a/doc/special_remotes/directory/comment_2_d949edad6a330079f9e15f703f9091e3._comment b/doc/special_remotes/directory/comment_2_d949edad6a330079f9e15f703f9091e3._comment new file mode 100644 index 0000000000..77b4c4d220 --- /dev/null +++ b/doc/special_remotes/directory/comment_2_d949edad6a330079f9e15f703f9091e3._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 2" + date="2012-06-25T15:29:29Z" + content=""" +The directory and rsync special remotes intentionally use the same layout. So the same directory could be set up as both types of special remotes. + +The main reason to use this rather than a bare git repo is that it supports encryption. +"""]] From 2e92f95edc7fd7ca418dd43d71faf1df8358b052 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Mon, 25 Jun 2012 15:30:00 +0000 Subject: [PATCH 3911/8313] --- doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn new file mode 100644 index 0000000000..6c84e351d9 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn @@ -0,0 +1,11 @@ +I was dumping ~gigs of files of approximately 3-6megs a pop (my music collection) so I could track the files that I want to listen to when I'm on the go. I had the git watch command running from the assistant branch. + +I was getting something along the lines of... + + /Users/jtang/annex/.git/annex/tmp/: openTempFile: resource exhausted (Too many open files) + +and + + git-annex: createPipe: resource exhausted (Too many open files) + +I also noticed that I somehow ended up with 256 ssh-agent's running on one of my machines, I'm not sure if the two issues are related or not, I had not noticed this type of behaviour up until recently. From 0b04ff643131dd37fffe25e295cf5b2c227421f4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Mon, 25 Jun 2012 15:32:59 +0000 Subject: [PATCH 3912/8313] --- doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn index 6c84e351d9..a919477fc2 100644 --- a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn @@ -9,3 +9,9 @@ and git-annex: createPipe: resource exhausted (Too many open files) I also noticed that I somehow ended up with 256 ssh-agent's running on one of my machines, I'm not sure if the two issues are related or not, I had not noticed this type of behaviour up until recently. + +Also this was appearing in the logs + + x00:annex jtang$ tail -f .git/annex/daemon.log + (scanning...) Already up-to-date. + kqueue: Too many open files From cede7bdcdee25adcc635a0496ece6d7078a094d5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Jun 2012 11:38:42 -0400 Subject: [PATCH 3913/8313] cabal: Only try to use inotify on Linux. --- debian/changelog | 6 ++++++ git-annex.cabal | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 42f37e572e..2c289667d5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +git-annex (3.20120625) UNRELEASED; urgency=low + + * cabal: Only try to use inotify on Linux. + + -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 + git-annex (3.20120624) unstable; urgency=low * watch: New subcommand, a daemon which notices changes to diff --git a/git-annex.cabal b/git-annex.cabal index 3902a077b5..b437057939 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -47,7 +47,7 @@ Executable git-annex Build-Depends: hS3 CPP-Options: -DWITH_S3 - if flag(Inotify) + if os(linux) && flag(Inotify) Build-Depends: hinotify CPP-Options: -DWITH_INOTIFY From 07c15d9a171c136b61f027505b98e3011fb6d3d3 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Mon, 25 Jun 2012 15:38:44 +0000 Subject: [PATCH 3914/8313] Added a comment --- ...omment_10_798000aab19af2944b6e44dbc550c6fe._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment diff --git a/doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment b/doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment new file mode 100644 index 0000000000..675a90eee8 --- /dev/null +++ b/doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 10" + date="2012-06-25T15:38:44Z" + content=""" +@Agustin you should be able to work around that with: cabal install git-annex --flags=-Inotify + +I've fixed it properly for the next release, it should only be using that library on Linux. +"""]] From 7e8a6e76b192e30edd98ec83eb7c594777a00770 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Mon, 25 Jun 2012 15:38:57 +0000 Subject: [PATCH 3915/8313] --- doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn index a919477fc2..6c1891314a 100644 --- a/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits.mdwn @@ -15,3 +15,5 @@ Also this was appearing in the logs x00:annex jtang$ tail -f .git/annex/daemon.log (scanning...) Already up-to-date. kqueue: Too many open files + +To be precise, I suspect that the kqueue limit is 256, I had 325 files in the 'queue', I ended up doing a _git annex add_ manually and all was fine. From 61091915f615fd4e72cc248bc75ce551b5076d12 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Mon, 25 Jun 2012 15:42:48 +0000 Subject: [PATCH 3916/8313] Added a comment --- ...mment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment new file mode 100644 index 0000000000..d30cddca50 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_1_5fc1eedb5231edc37c87a2d9b91313b9._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-25T15:42:48Z" + content=""" +Yes, this is a known problem with kqueue, it has to keep every directory in the tree open. On [[design/assistant/inotify]] I have a note that it may need to fork off extra watcher processes to deal with this. Of course that adds significant complication. + +In the meantime, you may be able to increase your system's maximum allowed number of open files per process somehow. + +(I doubt that the ssh-agent is related; git-annex does not use ssh-agent directly anyway..) +"""]] From e3163bf41669a2fd670b3b429975355351a03e24 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Mon, 25 Jun 2012 15:45:18 +0000 Subject: [PATCH 3917/8313] Added a comment --- .../comment_1_377525e70640751e1ead445aeed15efa._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment diff --git a/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment new file mode 100644 index 0000000000..77481789ce --- /dev/null +++ b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX/comment_1_377525e70640751e1ead445aeed15efa._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-25T15:45:18Z" + content=""" +This is indeed the problem I've planned to work on today, as I see it too when things don't start off perfectly in sync. +"""]] From 0b146f9ecc36545478c4a2218981b376828c61db Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Jun 2012 16:10:10 -0400 Subject: [PATCH 3918/8313] reorg threads --- Assistant.hs | 10 +++++----- Assistant/{ => Threads}/Committer.hs | 4 ++-- Assistant/{ => Threads}/Merger.hs | 2 +- Assistant/{ => Threads}/Pusher.hs | 2 +- Assistant/{ => Threads}/SanityChecker.hs | 8 ++++---- Assistant/{ => Threads}/Watcher.hs | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) rename Assistant/{ => Threads}/Committer.hs (98%) rename Assistant/{ => Threads}/Merger.hs (98%) rename Assistant/{ => Threads}/Pusher.hs (98%) rename Assistant/{ => Threads}/SanityChecker.hs (93%) rename Assistant/{ => Threads}/Watcher.hs (99%) diff --git a/Assistant.hs b/Assistant.hs index ce230533cf..ec46894a5b 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -59,11 +59,11 @@ import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.Changes import Assistant.Commits -import Assistant.Watcher -import Assistant.Committer -import Assistant.Pusher -import Assistant.Merger -import Assistant.SanityChecker +import Assistant.Threads.Watcher +import Assistant.Threads.Committer +import Assistant.Threads.Pusher +import Assistant.Threads.Merger +import Assistant.Threads.SanityChecker import qualified Utility.Daemon import Utility.LogFile diff --git a/Assistant/Committer.hs b/Assistant/Threads/Committer.hs similarity index 98% rename from Assistant/Committer.hs rename to Assistant/Threads/Committer.hs index 0c69995912..488056fa2b 100644 --- a/Assistant/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -5,13 +5,13 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Assistant.Committer where +module Assistant.Threads.Committer where import Common.Annex import Assistant.Changes import Assistant.Commits import Assistant.ThreadedMonad -import Assistant.Watcher +import Assistant.Threads.Watcher import qualified Annex import qualified Annex.Queue import qualified Git.Command diff --git a/Assistant/Merger.hs b/Assistant/Threads/Merger.hs similarity index 98% rename from Assistant/Merger.hs rename to Assistant/Threads/Merger.hs index 988cbd8a63..d2c8b9b766 100644 --- a/Assistant/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Assistant.Merger where +module Assistant.Threads.Merger where import Common.Annex import Assistant.ThreadedMonad diff --git a/Assistant/Pusher.hs b/Assistant/Threads/Pusher.hs similarity index 98% rename from Assistant/Pusher.hs rename to Assistant/Threads/Pusher.hs index 7504d44c20..de90d4e64c 100644 --- a/Assistant/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Assistant.Pusher where +module Assistant.Threads.Pusher where import Common.Annex import Assistant.Commits diff --git a/Assistant/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs similarity index 93% rename from Assistant/SanityChecker.hs rename to Assistant/Threads/SanityChecker.hs index b74c9fe5d2..4db2a61b22 100644 --- a/Assistant/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -5,7 +5,7 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Assistant.SanityChecker ( +module Assistant.Threads.SanityChecker ( sanityCheckerThread ) where @@ -15,7 +15,7 @@ import Assistant.DaemonStatus import Assistant.ThreadedMonad import Assistant.Changes import Utility.ThreadScheduler -import qualified Assistant.Watcher +import qualified Assistant.Threads.Watcher as Watcher import Data.Time.Clock.POSIX @@ -79,5 +79,5 @@ check st status changechan = do insanity m = runThreadState st $ warning m addsymlink file s = do insanity $ "found unstaged symlink: " ++ file - Assistant.Watcher.runHandler st status changechan - Assistant.Watcher.onAddSymlink file s + Watcher.runHandler st status changechan + Watcher.onAddSymlink file s diff --git a/Assistant/Watcher.hs b/Assistant/Threads/Watcher.hs similarity index 99% rename from Assistant/Watcher.hs rename to Assistant/Threads/Watcher.hs index 78330c8d0a..1b6ec15f18 100644 --- a/Assistant/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -7,7 +7,7 @@ {-# LANGUAGE CPP #-} -module Assistant.Watcher where +module Assistant.Threads.Watcher where import Common.Annex import Assistant.ThreadedMonad From 5cfe91f06d5eaab217f1b289810d96fee0144c31 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Jun 2012 16:38:12 -0400 Subject: [PATCH 3919/8313] add a push retry thread --- Assistant.hs | 14 +++++--- Assistant/Pushes.hs | 36 +++++++++++++++++++ Assistant/Threads/Pusher.hs | 57 ++++++++++++++++++------------- Utility/ThreadScheduler.hs | 6 ---- doc/design/assistant/syncing.mdwn | 5 +-- 5 files changed, 82 insertions(+), 36 deletions(-) create mode 100644 Assistant/Pushes.hs diff --git a/Assistant.hs b/Assistant.hs index ec46894a5b..c054dafd3d 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -25,14 +25,17 @@ - Thread 6: pusher - Waits for commits to be made, and pushes updated branches to remotes, - in parallel. (Forks a process for each git push.) - - Thread 7: merger + - Thread 7: push retryer + - Runs every 30 minutes when there are failed pushes, and retries + - them. + - Thread 8: merger - Waits for pushes to be received from remotes, and merges the - updated branches into the current branch. This uses inotify - on .git/refs/heads, so there are additional inotify threads - associated with it, too. - - Thread 8: status logger + - Thread 9: status logger - Wakes up periodically and records the daemon's status to disk. - - Thread 9: sanity checker + - Thread 10: sanity checker - Wakes up periodically (rarely) and does sanity checks. - - ThreadState: (MVar) @@ -59,6 +62,7 @@ import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.Changes import Assistant.Commits +import Assistant.Pushes import Assistant.Threads.Watcher import Assistant.Threads.Committer import Assistant.Threads.Pusher @@ -85,8 +89,10 @@ startDaemon assistant foreground liftIO $ a $ do changechan <- newChangeChan commitchan <- newCommitChan + pushchan <- newFailedPushChan _ <- forkIO $ commitThread st changechan commitchan - _ <- forkIO $ pushThread st commitchan + _ <- forkIO $ pushThread st commitchan pushchan + _ <- forkIO $ pushRetryThread st pushchan _ <- forkIO $ mergeThread st _ <- forkIO $ daemonStatusThread st dstatus _ <- forkIO $ sanityCheckerThread st dstatus changechan diff --git a/Assistant/Pushes.hs b/Assistant/Pushes.hs new file mode 100644 index 0000000000..f3bffbf792 --- /dev/null +++ b/Assistant/Pushes.hs @@ -0,0 +1,36 @@ +{- git-annex assistant push tracking + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Pushes where + +import Common.Annex +import Utility.TSet + +import Data.Time.Clock + +type FailedPushChan = TSet FailedPush + +data FailedPush = FailedPush + { failedRemote :: Remote + , failedTimeStamp :: UTCTime + } + +newFailedPushChan :: IO FailedPushChan +newFailedPushChan = newTSet + +{- Gets all failed pushes. Blocks until there is at least one failed push. -} +getFailedPushes :: FailedPushChan -> IO [FailedPush] +getFailedPushes = getTSet + +{- Puts failed pushes back into the channel. + - Note: Original order is not preserved. -} +refillFailedPushes :: FailedPushChan -> [FailedPush] -> IO () +refillFailedPushes = putTSet + +{- Records a failed push in the channel. -} +recordFailedPush :: FailedPushChan -> FailedPush -> IO () +recordFailedPush = putTSet1 diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs index de90d4e64c..6a4ae7838d 100644 --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -1,4 +1,4 @@ -{- git-annex assistant git pushing thread +{- git-annex assistant git pushing threads - - Copyright 2012 Joey Hess - @@ -9,6 +9,7 @@ module Assistant.Threads.Pusher where import Common.Annex import Assistant.Commits +import Assistant.Pushes import Assistant.ThreadedMonad import qualified Command.Sync import Utility.ThreadScheduler @@ -16,39 +17,45 @@ import Utility.Parallel import Data.Time.Clock -data FailedPush = FailedPush - { failedRemote :: Remote - , failedTimeStamp :: UTCTime - } +{- This thread retries pushes that failed before. -} +pushRetryThread :: ThreadState -> FailedPushChan -> IO () +pushRetryThread st pushchan = runEvery (Seconds halfhour) $ do + -- We already waited half an hour, now wait until there are failed + -- pushes to retry. + pushes <- getFailedPushes pushchan + -- Check times, to avoid repushing a push that's too new. + now <- getCurrentTime + let (newpushes, oldpushes) = partition (toorecent now . failedTimeStamp) pushes + unless (null newpushes) $ + refillFailedPushes pushchan newpushes + unless (null oldpushes) $ + pushToRemotes now st pushchan $ map failedRemote oldpushes + where + halfhour = 1800 + toorecent now time = now `diffUTCTime` time < fromIntegral halfhour -{- This thread pushes git commits out to remotes. -} -pushThread :: ThreadState -> CommitChan -> IO () -pushThread st commitchan = do +{- This thread pushes git commits out to remotes soon after they are made. -} +pushThread :: ThreadState -> CommitChan -> FailedPushChan -> IO () +pushThread st commitchan pushchan = do remotes <- runThreadState st $ Command.Sync.syncRemotes [] - runEveryWith (Seconds 2) [] $ \failedpushes -> do + runEvery (Seconds 2) $ do -- We already waited two seconds as a simple rate limiter. -- Next, wait until at least one commit has been made commits <- getCommits commitchan -- Now see if now's a good time to push. - time <- getCurrentTime - if shouldPush time commits failedpushes - then pushToRemotes time st remotes - else do - refillCommits commitchan commits - return failedpushes + now <- getCurrentTime + if shouldPush now commits + then pushToRemotes now st pushchan remotes + else refillCommits commitchan commits {- Decide if now is a good time to push to remotes. - - Current strategy: Immediately push all commits. The commit machinery - already determines batches of changes, so we can't easily determine - batches better. - - - - TODO: FailedPushs are only retried the next time there's a commit. - - Should retry them periodically, or when a remote that was not available - - becomes available. -} -shouldPush :: UTCTime -> [Commit] -> [FailedPush] -> Bool -shouldPush _now commits _failedremotes +shouldPush :: UTCTime -> [Commit] -> Bool +shouldPush _now commits | not (null commits) = True | otherwise = False @@ -57,12 +64,14 @@ shouldPush _now commits _failedremotes - - Avoids running possibly long-duration commands in the Annex monad, so - as not to block other threads. -} -pushToRemotes :: UTCTime -> ThreadState -> [Remote] -> IO [FailedPush] -pushToRemotes now st remotes = do +pushToRemotes :: UTCTime -> ThreadState -> FailedPushChan -> [Remote] -> IO () +pushToRemotes now st pushchan remotes = do (g, branch) <- runThreadState st $ (,) <$> fromRepo id <*> Command.Sync.currentBranch Command.Sync.updateBranch (Command.Sync.syncBranch branch) g - map (`FailedPush` now) <$> inParallel (push g branch) remotes + failed <- map (`FailedPush` now) <$> inParallel (push g branch) remotes + unless (null failed) $ + refillFailedPushes pushchan failed where push g branch remote = ifM (Command.Sync.pushBranch remote branch g) diff --git a/Utility/ThreadScheduler.hs b/Utility/ThreadScheduler.hs index 07a7401600..6557398fd7 100644 --- a/Utility/ThreadScheduler.hs +++ b/Utility/ThreadScheduler.hs @@ -24,12 +24,6 @@ runEvery n a = forever $ do threadDelaySeconds n a -runEveryWith :: Seconds -> a -> (a -> IO a) -> IO () -runEveryWith n val a = do - threadDelaySeconds n - val' <- a val - runEveryWith n val' a - threadDelaySeconds :: Seconds -> IO () threadDelaySeconds (Seconds n) = unboundDelay (fromIntegral n * oneSecond) where diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 8173457c55..a2b80eebc4 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -13,8 +13,9 @@ all the other git clones, at both the git level and the key/value level. [The watching can be done with the existing inotify code! This avoids needing any special mechanism to notify a remote that it's been synced to.] **done** -1. Periodically retry pushes that failed. Also, detect if a push failed - due to not being up-to-date, pull, and repush. +1. Periodically retry pushes that failed. **done** (every half an hour) +1. Also, detect if a push failed due to not being up-to-date, pull, + and repush. 2. Use a git merge driver that adds both conflicting files, so conflicts never break a sync. 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of From 05c4dfb9411e54c21aa4c0d49e3662117ac2f549 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Jun 2012 20:16:30 -0400 Subject: [PATCH 3920/8313] fixup merges now done when needed --- Assistant/Threads/Merger.hs | 8 ++++++++ Assistant/Threads/Pusher.hs | 17 +++++++++++++---- doc/design/assistant/syncing.mdwn | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index d2c8b9b766..77bf9f416e 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -70,3 +70,11 @@ onAdd g file _ mergeBranch :: Git.Ref -> Git.Repo -> IO Bool mergeBranch = Git.Merge.mergeNonInteractive . Command.Sync.syncBranch + +{- Manually pull from remotes and merge their branches. Called by the pusher + - when a push fails, which can happen due to a remote not having pushed + - changes to us. That could be because it doesn't have us as a remote, or + - because the assistant is not running there, or other reasons. -} +manualPull :: Git.Ref -> [Remote] -> Annex () +manualPull currentbranch remotes = forM_ remotes $ \r -> + Command.Sync.mergeRemote r currentbranch diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs index 6a4ae7838d..82c37de5f8 100644 --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -11,6 +11,7 @@ import Common.Annex import Assistant.Commits import Assistant.Pushes import Assistant.ThreadedMonad +import Assistant.Threads.Merger import qualified Command.Sync import Utility.ThreadScheduler import Utility.Parallel @@ -68,11 +69,19 @@ pushToRemotes :: UTCTime -> ThreadState -> FailedPushChan -> [Remote] -> IO () pushToRemotes now st pushchan remotes = do (g, branch) <- runThreadState st $ (,) <$> fromRepo id <*> Command.Sync.currentBranch - Command.Sync.updateBranch (Command.Sync.syncBranch branch) g - failed <- map (`FailedPush` now) <$> inParallel (push g branch) remotes - unless (null failed) $ - refillFailedPushes pushchan failed + go True branch g remotes where + go shouldretry branch g rs = do + Command.Sync.updateBranch (Command.Sync.syncBranch branch) g + failed <- inParallel (push g branch) rs + unless (null failed) $ + if shouldretry + then retry branch g rs + else refillFailedPushes pushchan $ + map (`FailedPush` now) failed push g branch remote = ifM (Command.Sync.pushBranch remote branch g) ( exitSuccess, exitFailure) + retry branch g rs = do + runThreadState st $ manualPull branch rs + go False branch g rs diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index a2b80eebc4..3ece69638c 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -15,7 +15,7 @@ all the other git clones, at both the git level and the key/value level. **done** 1. Periodically retry pushes that failed. **done** (every half an hour) 1. Also, detect if a push failed due to not being up-to-date, pull, - and repush. + and repush. **done** 2. Use a git merge driver that adds both conflicting files, so conflicts never break a sync. 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of From 336cee671e07164c37f01f26573bb9d584368a98 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 25 Jun 2012 20:40:58 -0400 Subject: [PATCH 3921/8313] blog for the day --- ...ness_with_the_assistant_branch_on_OSX.mdwn | 3 ++ .../blog/day_16__more_robust_syncing.mdwn | 44 +++++++++++++++++++ doc/design/assistant/syncing.mdwn | 7 +-- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 doc/design/assistant/blog/day_16__more_robust_syncing.mdwn diff --git a/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn index 63431aae68..00f4253073 100644 --- a/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn +++ b/doc/bugs/More_sync__39__ing_weirdness_with_the_assistant_branch_on_OSX.mdwn @@ -10,3 +10,6 @@ hint: See the 'Note about fast-forwards' in 'git push --help' for details. (Recording state in git...) manually running a 'git annex sync' usually fixes it, I guess once the sync command runs periodically this problem will go away, is this even OSX specific? I don't quite get the behaviour that is described in [[design/assistant/blog/day_15__its_aliiive]]. + +> With my changes today, I've seen it successfully recover from this +> situation. [[done]] --[[Joey]] diff --git a/doc/design/assistant/blog/day_16__more_robust_syncing.mdwn b/doc/design/assistant/blog/day_16__more_robust_syncing.mdwn new file mode 100644 index 0000000000..e9343f846d --- /dev/null +++ b/doc/design/assistant/blog/day_16__more_robust_syncing.mdwn @@ -0,0 +1,44 @@ +I released a version of git-annex over the weekend that includes the `git +annex watch` command. There's a minor issue installing it from cabal on +OSX, which I've fixed in my tree. Nice timing: At least the watch command +should be shipped in the next Debian release, which freezes at the end of +the month. + +Jimmy found out how kqueue [[blows +up|bugs/Issue_on_OSX_with_some_system_limits]] when there are too many +directories to keep all open. I'm not surprised this happens, but it's nice +to see exactly how. Odd that it happened to him at just 512 directories; +I'd have guessed more. I have plans to fork watcher programs that each +watch 512 directories (or whatever the ulimit is), to deal with this. What +a pitiful interface is kqueue.. I have not thought yet about how the watcher +programs would communicate back to the main program. + +---- + +Back on the assistant front, I've worked today on making git syncing more +robust. Now when a push fails, it tries a pull, and a merge, and repushes. +That ensures that the push is, almost always, a fast-forward. Unless +something else gets in a push first, anyway! + +If a push still fails, there's Yet Another Thread, added today, that will +wake up after 30 minutes and retry the push. It currently keeps retrying +every 30 minutes until the push finally gets though. This will deal, to +some degree, with those situations where a remote is only sometimes +available. + +I need to refine the code a bit, to avoid it keeping an ever-growing queue +of failed pushes, if a remote is just dead. And to clear old failed pushes +from the queue when a later push succeeds. + +I also need to write a git merge driver that handles conflicts in the tree. +If two conflicting versions of a file `foo` are saved, this would merge +them, renaming them to `foo.X` and `foo.Y`. Probably X and Y are the +git-annex keys for the content of the files; this way all clones will +resolve the conflict in a way that leads to the same tree. It's also +possible to get a conflict by one repo deleting a file, and another +modifying it. In this case, renaming the deleted file to `foo.Y` may +be the right approach, I am not sure. + +I glanced through some Haskell dbus bindings today. I belive there are dbus +events available to detect when drives are mounted, and on Linux this would +let git-annex notice and sync to usb drives, etc. diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 8173457c55..3e90e6b105 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -13,8 +13,9 @@ all the other git clones, at both the git level and the key/value level. [The watching can be done with the existing inotify code! This avoids needing any special mechanism to notify a remote that it's been synced to.] **done** -1. Periodically retry pushes that failed. Also, detect if a push failed - due to not being up-to-date, pull, and repush. +1. Periodically retry pushes that failed. **done** (every half an hour) +1. Also, detect if a push failed due to not being up-to-date, pull, + and repush. **done** 2. Use a git merge driver that adds both conflicting files, so conflicts never break a sync. 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of @@ -41,7 +42,7 @@ This probably will need lots of refinements to get working well. ## other considerations It would be nice if, when a USB drive is connected, -syncing starts automatically. +syncing starts automatically. Use dbus on Linux? This assumes the network is connected. It's often not, so the [[cloud]] needs to be used to bridge between LANs. From f72dd3e7e5340dab028b7b0631dc1b92d5839536 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmURXBzaYE1gmVc-X9eLAyDat_6rHPl670" Date: Tue, 26 Jun 2012 08:31:48 +0000 Subject: [PATCH 3922/8313] --- ...ding_git-annex_3.20120624_using_cabal.mdwn | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn diff --git a/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn new file mode 100644 index 0000000000..169fa2338b --- /dev/null +++ b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn @@ -0,0 +1,155 @@ +What steps will reproduce the problem? +I am trying to install git-annex 3.20120624 using cabal. My currently installed version of git-annex is 3.20120615. After a "cabal update", the build of git-annex fails: + + bram@falafel% cabal install git-annex + Resolving dependencies... + [1 of 4] Compiling Utility.SafeCommand ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Utility/SafeCommand.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Utility/SafeCommand.o ) + [2 of 4] Compiling Build.TestConfig ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Build/TestConfig.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Build/TestConfig.o ) + [3 of 4] Compiling Build.Configure ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Build/Configure.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Build/Configure.o ) + [4 of 4] Compiling Main ( /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/Setup.hs, /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/Main.o ) + Linking /tmp/git-annex-3.20120624-4173/git-annex-3.20120624/dist/setup/setup ... + checking version... 3.20120624 + checking git... yes + checking git version... 1.7.9.5 + checking cp -a... yes + checking cp -p... yes + checking cp --reflink=auto... yes + checking uuid generator... uuid + checking xargs -0... yes + checking rsync... yes + checking curl... no + checking wget... yes + checking bup... no + checking gpg... yes + checking lsof... yes + checking ssh connection caching... yes + checking sha1... sha1sum + checking sha512... sha512sum + checking sha224... sha224sum + checking sha384... sha384sum + checking sha256... sha256sum + Configuring git-annex-3.20120624... + Building git-annex-3.20120624... + Preprocessing executable 'git-annex' for git-annex-3.20120624... + [ 1 of 183] Compiling Utility.Percentage ( Utility/Percentage.hs, dist/build/git-annex/git-annex-tmp/Utility/Percentage.o ) + [ 2 of 183] Compiling Utility.Dot ( Utility/Dot.hs, dist/build/git-annex/git-annex-tmp/Utility/Dot.o ) + [ 3 of 183] Compiling Utility.ThreadLock ( Utility/ThreadLock.hs, dist/build/git-annex/git-annex-tmp/Utility/ThreadLock.o ) + [ 4 of 183] Compiling Utility.Base64 ( Utility/Base64.hs, dist/build/git-annex/git-annex-tmp/Utility/Base64.o ) + [ 5 of 183] Compiling Utility.DataUnits ( Utility/DataUnits.hs, dist/build/git-annex/git-annex-tmp/Utility/DataUnits.o ) + [ 6 of 183] Compiling Utility.JSONStream ( Utility/JSONStream.hs, dist/build/git-annex/git-annex-tmp/Utility/JSONStream.o ) + [ 7 of 183] Compiling Messages.JSON ( Messages/JSON.hs, dist/build/git-annex/git-annex-tmp/Messages/JSON.o ) + [ 8 of 183] Compiling Build.SysConfig ( Build/SysConfig.hs, dist/build/git-annex/git-annex-tmp/Build/SysConfig.o ) + [ 9 of 183] Compiling Types.KeySource ( Types/KeySource.hs, dist/build/git-annex/git-annex-tmp/Types/KeySource.o ) + [ 10 of 183] Compiling Types.UUID ( Types/UUID.hs, dist/build/git-annex/git-annex-tmp/Types/UUID.o ) + [ 11 of 183] Compiling Utility.State ( Utility/State.hs, dist/build/git-annex/git-annex-tmp/Utility/State.o ) + [ 12 of 183] Compiling Types.Messages ( Types/Messages.hs, dist/build/git-annex/git-annex-tmp/Types/Messages.o ) + [ 13 of 183] Compiling Types.TrustLevel ( Types/TrustLevel.hs, dist/build/git-annex/git-annex-tmp/Types/TrustLevel.o ) + [ 14 of 183] Compiling Types.BranchState ( Types/BranchState.hs, dist/build/git-annex/git-annex-tmp/Types/BranchState.o ) + [ 15 of 183] Compiling Git.Index ( Git/Index.hs, dist/build/git-annex/git-annex-tmp/Git/Index.o ) + [ 16 of 183] Compiling Utility.PartialPrelude ( Utility/PartialPrelude.hs, dist/build/git-annex/git-annex-tmp/Utility/PartialPrelude.o ) + [ 17 of 183] Compiling Utility.Format ( Utility/Format.hs, dist/build/git-annex/git-annex-tmp/Utility/Format.o ) + [ 18 of 183] Compiling Utility.FileSystemEncoding ( Utility/FileSystemEncoding.hs, dist/build/git-annex/git-annex-tmp/Utility/FileSystemEncoding.o ) + [ 19 of 183] Compiling Utility.Touch ( dist/build/git-annex/git-annex-tmp/Utility/Touch.hs, dist/build/git-annex/git-annex-tmp/Utility/Touch.o ) + [ 20 of 183] Compiling Utility.Monad ( Utility/Monad.hs, dist/build/git-annex/git-annex-tmp/Utility/Monad.o ) + [ 21 of 183] Compiling Utility.Path ( Utility/Path.hs, dist/build/git-annex/git-annex-tmp/Utility/Path.o ) + [ 22 of 183] Compiling Utility.SafeCommand ( Utility/SafeCommand.hs, dist/build/git-annex/git-annex-tmp/Utility/SafeCommand.o ) + [ 23 of 183] Compiling Utility.RsyncFile ( Utility/RsyncFile.hs, dist/build/git-annex/git-annex-tmp/Utility/RsyncFile.o ) + [ 24 of 183] Compiling Utility.Exception ( Utility/Exception.hs, dist/build/git-annex/git-annex-tmp/Utility/Exception.o ) + [ 25 of 183] Compiling Utility.TempFile ( Utility/TempFile.hs, dist/build/git-annex/git-annex-tmp/Utility/TempFile.o ) + [ 26 of 183] Compiling Utility.Directory ( Utility/Directory.hs, dist/build/git-annex/git-annex-tmp/Utility/Directory.o ) + [ 27 of 183] Compiling Utility.Misc ( Utility/Misc.hs, dist/build/git-annex/git-annex-tmp/Utility/Misc.o ) + [ 28 of 183] Compiling Git.Types ( Git/Types.hs, dist/build/git-annex/git-annex-tmp/Git/Types.o ) + [ 29 of 183] Compiling Common ( Common.hs, dist/build/git-annex/git-annex-tmp/Common.o ) + [ 30 of 183] Compiling Utility.FileMode ( Utility/FileMode.hs, dist/build/git-annex/git-annex-tmp/Utility/FileMode.o ) + [ 31 of 183] Compiling Git ( Git.hs, dist/build/git-annex/git-annex-tmp/Git.o ) + [ 32 of 183] Compiling Git.Command ( Git/Command.hs, dist/build/git-annex/git-annex-tmp/Git/Command.o ) + [ 33 of 183] Compiling Git.Ref ( Git/Ref.hs, dist/build/git-annex/git-annex-tmp/Git/Ref.o ) + [ 34 of 183] Compiling Git.FilePath ( Git/FilePath.hs, dist/build/git-annex/git-annex-tmp/Git/FilePath.o ) + [ 35 of 183] Compiling Utility.Matcher ( Utility/Matcher.hs, dist/build/git-annex/git-annex-tmp/Utility/Matcher.o ) + [ 36 of 183] Compiling Utility.Gpg ( Utility/Gpg.hs, dist/build/git-annex/git-annex-tmp/Utility/Gpg.o ) + [ 37 of 183] Compiling Types.Crypto ( Types/Crypto.hs, dist/build/git-annex/git-annex-tmp/Types/Crypto.o ) + [ 38 of 183] Compiling Types.Key ( Types/Key.hs, dist/build/git-annex/git-annex-tmp/Types/Key.o ) + [ 39 of 183] Compiling Types.Backend ( Types/Backend.hs, dist/build/git-annex/git-annex-tmp/Types/Backend.o ) + [ 40 of 183] Compiling Types.Remote ( Types/Remote.hs, dist/build/git-annex/git-annex-tmp/Types/Remote.o ) + [ 41 of 183] Compiling Git.Sha ( Git/Sha.hs, dist/build/git-annex/git-annex-tmp/Git/Sha.o ) + [ 42 of 183] Compiling Git.Branch ( Git/Branch.hs, dist/build/git-annex/git-annex-tmp/Git/Branch.o ) + [ 43 of 183] Compiling Git.UpdateIndex ( Git/UpdateIndex.hs, dist/build/git-annex/git-annex-tmp/Git/UpdateIndex.o ) + [ 44 of 183] Compiling Git.Queue ( Git/Queue.hs, dist/build/git-annex/git-annex-tmp/Git/Queue.o ) + [ 45 of 183] Compiling Git.Url ( Git/Url.hs, dist/build/git-annex/git-annex-tmp/Git/Url.o ) + [ 46 of 183] Compiling Git.Construct ( Git/Construct.hs, dist/build/git-annex/git-annex-tmp/Git/Construct.o ) + [ 47 of 183] Compiling Git.Config ( Git/Config.hs, dist/build/git-annex/git-annex-tmp/Git/Config.o ) + [ 48 of 183] Compiling Git.SharedRepository ( Git/SharedRepository.hs, dist/build/git-annex/git-annex-tmp/Git/SharedRepository.o ) + [ 49 of 183] Compiling Git.Version ( Git/Version.hs, dist/build/git-annex/git-annex-tmp/Git/Version.o ) + [ 50 of 183] Compiling Utility.CoProcess ( Utility/CoProcess.hs, dist/build/git-annex/git-annex-tmp/Utility/CoProcess.o ) + [ 51 of 183] Compiling Git.HashObject ( Git/HashObject.hs, dist/build/git-annex/git-annex-tmp/Git/HashObject.o ) + [ 52 of 183] Compiling Git.CatFile ( Git/CatFile.hs, dist/build/git-annex/git-annex-tmp/Git/CatFile.o ) + [ 53 of 183] Compiling Git.UnionMerge ( Git/UnionMerge.hs, dist/build/git-annex/git-annex-tmp/Git/UnionMerge.o ) + [ 54 of 183] Compiling Git.CheckAttr ( Git/CheckAttr.hs, dist/build/git-annex/git-annex-tmp/Git/CheckAttr.o ) + [ 55 of 183] Compiling Annex ( Annex.hs, dist/build/git-annex/git-annex-tmp/Annex.o ) + [ 56 of 183] Compiling Types.Option ( Types/Option.hs, dist/build/git-annex/git-annex-tmp/Types/Option.o ) + [ 57 of 183] Compiling Types ( Types.hs, dist/build/git-annex/git-annex-tmp/Types.o ) + [ 58 of 183] Compiling Messages ( Messages.hs, dist/build/git-annex/git-annex-tmp/Messages.o ) + [ 59 of 183] Compiling Types.Command ( Types/Command.hs, dist/build/git-annex/git-annex-tmp/Types/Command.o ) + [ 60 of 183] Compiling Locations ( Locations.hs, dist/build/git-annex/git-annex-tmp/Locations.o ) + [ 61 of 183] Compiling Common.Annex ( Common/Annex.hs, dist/build/git-annex/git-annex-tmp/Common/Annex.o ) + [ 62 of 183] Compiling Annex.Exception ( Annex/Exception.hs, dist/build/git-annex/git-annex-tmp/Annex/Exception.o ) + [ 63 of 183] Compiling Annex.BranchState ( Annex/BranchState.hs, dist/build/git-annex/git-annex-tmp/Annex/BranchState.o ) + [ 64 of 183] Compiling Annex.CatFile ( Annex/CatFile.hs, dist/build/git-annex/git-annex-tmp/Annex/CatFile.o ) + [ 65 of 183] Compiling Annex.Perms ( Annex/Perms.hs, dist/build/git-annex/git-annex-tmp/Annex/Perms.o ) + [ 66 of 183] Compiling Annex.Journal ( Annex/Journal.hs, dist/build/git-annex/git-annex-tmp/Annex/Journal.o ) + [ 67 of 183] Compiling Annex.Branch ( Annex/Branch.hs, dist/build/git-annex/git-annex-tmp/Annex/Branch.o ) + [ 68 of 183] Compiling Crypto ( Crypto.hs, dist/build/git-annex/git-annex-tmp/Crypto.o ) + [ 69 of 183] Compiling Usage ( Usage.hs, dist/build/git-annex/git-annex-tmp/Usage.o ) + [ 70 of 183] Compiling Annex.CheckAttr ( Annex/CheckAttr.hs, dist/build/git-annex/git-annex-tmp/Annex/CheckAttr.o ) + [ 71 of 183] Compiling Remote.Helper.Special ( Remote/Helper/Special.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Special.o ) + [ 72 of 183] Compiling Logs.Presence ( Logs/Presence.hs, dist/build/git-annex/git-annex-tmp/Logs/Presence.o ) + [ 73 of 183] Compiling Logs.Location ( Logs/Location.hs, dist/build/git-annex/git-annex-tmp/Logs/Location.o ) + [ 74 of 183] Compiling Logs.Web ( Logs/Web.hs, dist/build/git-annex/git-annex-tmp/Logs/Web.o ) + [ 75 of 183] Compiling Annex.LockPool ( Annex/LockPool.hs, dist/build/git-annex/git-annex-tmp/Annex/LockPool.o ) + [ 76 of 183] Compiling Backend.SHA ( Backend/SHA.hs, dist/build/git-annex/git-annex-tmp/Backend/SHA.o ) + [ 77 of 183] Compiling Backend.WORM ( Backend/WORM.hs, dist/build/git-annex/git-annex-tmp/Backend/WORM.o ) + [ 78 of 183] Compiling Backend.URL ( Backend/URL.hs, dist/build/git-annex/git-annex-tmp/Backend/URL.o ) + [ 79 of 183] Compiling Assistant.ThreadedMonad ( Assistant/ThreadedMonad.hs, dist/build/git-annex/git-annex-tmp/Assistant/ThreadedMonad.o ) + [ 80 of 183] Compiling Logs.UUIDBased ( Logs/UUIDBased.hs, dist/build/git-annex/git-annex-tmp/Logs/UUIDBased.o ) + [ 81 of 183] Compiling Logs.Remote ( Logs/Remote.hs, dist/build/git-annex/git-annex-tmp/Logs/Remote.o ) + [ 82 of 183] Compiling Utility.DiskFree ( Utility/DiskFree.hs, dist/build/git-annex/git-annex-tmp/Utility/DiskFree.o ) + [ 83 of 183] Compiling Utility.Url ( Utility/Url.hs, dist/build/git-annex/git-annex-tmp/Utility/Url.o ) + [ 84 of 183] Compiling Utility.CopyFile ( Utility/CopyFile.hs, dist/build/git-annex/git-annex-tmp/Utility/CopyFile.o ) + [ 85 of 183] Compiling Git.LsFiles ( Git/LsFiles.hs, dist/build/git-annex/git-annex-tmp/Git/LsFiles.o ) + [ 86 of 183] Compiling Git.AutoCorrect ( Git/AutoCorrect.hs, dist/build/git-annex/git-annex-tmp/Git/AutoCorrect.o ) + [ 87 of 183] Compiling Git.CurrentRepo ( Git/CurrentRepo.hs, dist/build/git-annex/git-annex-tmp/Git/CurrentRepo.o ) + [ 88 of 183] Compiling Utility.Daemon ( Utility/Daemon.hs, dist/build/git-annex/git-annex-tmp/Utility/Daemon.o ) + [ 89 of 183] Compiling Utility.LogFile ( Utility/LogFile.hs, dist/build/git-annex/git-annex-tmp/Utility/LogFile.o ) + [ 90 of 183] Compiling Utility.ThreadScheduler ( Utility/ThreadScheduler.hs, dist/build/git-annex/git-annex-tmp/Utility/ThreadScheduler.o ) + [ 91 of 183] Compiling Assistant.DaemonStatus ( Assistant/DaemonStatus.hs, dist/build/git-annex/git-annex-tmp/Assistant/DaemonStatus.o ) + [ 92 of 183] Compiling Utility.Types.DirWatcher ( Utility/Types/DirWatcher.hs, dist/build/git-annex/git-annex-tmp/Utility/Types/DirWatcher.o ) + [ 93 of 183] Compiling Utility.INotify ( Utility/INotify.hs, dist/build/git-annex/git-annex-tmp/Utility/INotify.o ) + [ 94 of 183] Compiling Utility.DirWatcher ( Utility/DirWatcher.hs, dist/build/git-annex/git-annex-tmp/Utility/DirWatcher.o ) + [ 95 of 183] Compiling Utility.Lsof ( Utility/Lsof.hs, dist/build/git-annex/git-annex-tmp/Utility/Lsof.o ) + [ 96 of 183] Compiling Git.Merge ( Git/Merge.hs, dist/build/git-annex/git-annex-tmp/Git/Merge.o ) + [ 97 of 183] Compiling Git.Filename ( Git/Filename.hs, dist/build/git-annex/git-annex-tmp/Git/Filename.o ) + [ 98 of 183] Compiling Git.LsTree ( Git/LsTree.hs, dist/build/git-annex/git-annex-tmp/Git/LsTree.o ) + [ 99 of 183] Compiling Config ( Config.hs, dist/build/git-annex/git-annex-tmp/Config.o ) + [100 of 183] Compiling Annex.UUID ( Annex/UUID.hs, dist/build/git-annex/git-annex-tmp/Annex/UUID.o ) + [101 of 183] Compiling Logs.UUID ( Logs/UUID.hs, dist/build/git-annex/git-annex-tmp/Logs/UUID.o ) + [102 of 183] Compiling Backend ( Backend.hs, dist/build/git-annex/git-annex-tmp/Backend.o ) + [103 of 183] Compiling Remote.Helper.Hooks ( Remote/Helper/Hooks.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Hooks.o ) + [104 of 183] Compiling Remote.Helper.Encryptable ( Remote/Helper/Encryptable.hs, dist/build/git-annex/git-annex-tmp/Remote/Helper/Encryptable.o ) + [105 of 183] Compiling Annex.Queue ( Annex/Queue.hs, dist/build/git-annex/git-annex-tmp/Annex/Queue.o ) + [106 of 183] Compiling Annex.Content ( Annex/Content.hs, dist/build/git-annex/git-annex-tmp/Annex/Content.o ) + [107 of 183] Compiling Remote.S3 ( Remote/S3.hs, dist/build/git-annex/git-annex-tmp/Remote/S3.o ) + [108 of 183] Compiling Remote.Directory ( Remote/Directory.hs, dist/build/git-annex/git-annex-tmp/Remote/Directory.o ) + [109 of 183] Compiling Remote.Rsync ( Remote/Rsync.hs, dist/build/git-annex/git-annex-tmp/Remote/Rsync.o ) + [110 of 183] Compiling Remote.Web ( Remote/Web.hs, dist/build/git-annex/git-annex-tmp/Remote/Web.o ) + [111 of 183] Compiling Remote.Hook ( Remote/Hook.hs, dist/build/git-annex/git-annex-tmp/Remote/Hook.o ) + [112 of 183] Compiling Upgrade.V2 ( Upgrade/V2.hs, dist/build/git-annex/git-annex-tmp/Upgrade/V2.o ) + [113 of 183] Compiling Assistant.Changes ( Assistant/Changes.hs, dist/build/git-annex/git-annex-tmp/Assistant/Changes.o ) + + Assistant/Changes.hs:73:30: + Not in scope: `tryReadTChan' + Perhaps you meant `readTChan' (imported from Control.Concurrent.STM) + cabal: Error: some packages failed to install: + git-annex-3.20120624 failed during the building phase. The exception was: + ExitFailure 1 + +This is using haskell-platform 2012.1.0.0~debian1 on Ubuntu 12.04. From 6168795c1a92c509bbd16e29717d9a02afe8414c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmURXBzaYE1gmVc-X9eLAyDat_6rHPl670" Date: Tue, 26 Jun 2012 08:32:16 +0000 Subject: [PATCH 3923/8313] --- doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn index 169fa2338b..63dc23c2df 100644 --- a/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn +++ b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn @@ -1,4 +1,3 @@ -What steps will reproduce the problem? I am trying to install git-annex 3.20120624 using cabal. My currently installed version of git-annex is 3.20120615. After a "cabal update", the build of git-annex fails: bram@falafel% cabal install git-annex From 41fcb3d852d756ed218f3458af1ff9c7ae6d0e1d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 09:15:47 -0400 Subject: [PATCH 3924/8313] Version build dependency on STM, and allow building without it, which disables the watch command. --- GitAnnex.hs | 6 ++++++ Makefile | 2 +- debian/changelog | 2 ++ debian/control | 2 +- ...r_building_git-annex_3.20120624_using_cabal.mdwn | 5 +++++ doc/install.mdwn | 3 ++- git-annex.cabal | 13 ++++++++++--- 7 files changed, 27 insertions(+), 6 deletions(-) diff --git a/GitAnnex.hs b/GitAnnex.hs index a4c5eb8490..8dba5a3a7a 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -5,6 +5,8 @@ - Licensed under the GNU GPL version 3 or higher. -} +{-# LANGUAGE CPP #-} + module GitAnnex where import System.Console.GetOpt @@ -58,7 +60,9 @@ import qualified Command.Import import qualified Command.Map import qualified Command.Upgrade import qualified Command.Version +#ifdef WITH_ASSISTANT import qualified Command.Watch +#endif cmds :: [Command] cmds = concat @@ -100,7 +104,9 @@ cmds = concat , Command.Map.def , Command.Upgrade.def , Command.Version.def +#ifdef WITH_ASSISTANT , Command.Watch.def +#endif ] options :: [Option] diff --git a/Makefile b/Makefile index 73fbc4140d..4d56287468 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ endif PREFIX=/usr IGNORE=-ignore-package monads-fd -ignore-package monads-tf -BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_S3 $(BASEFLAGS_OPTS) +BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 $(BASEFLAGS_OPTS) GHCFLAGS=-O2 $(BASEFLAGS) CFLAGS=-Wall diff --git a/debian/changelog b/debian/changelog index 2c289667d5..92e992ed75 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,8 @@ git-annex (3.20120625) UNRELEASED; urgency=low * cabal: Only try to use inotify on Linux. + * Version build dependency on STM, and allow building without it, + which disables the watch command. -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 diff --git a/debian/control b/debian/control index bcecbec3d9..79702ed29a 100644 --- a/debian/control +++ b/debian/control @@ -21,7 +21,7 @@ Build-Depends: libghc-bloomfilter-dev, libghc-edit-distance-dev, libghc-hinotify-dev [linux-any], - libghc-stm-dev, + libghc-stm-dev (>= 2.3), ikiwiki, perlmagick, git, diff --git a/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn index 63dc23c2df..df83f4e4ed 100644 --- a/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn +++ b/doc/bugs/error_building_git-annex_3.20120624_using_cabal.mdwn @@ -152,3 +152,8 @@ I am trying to install git-annex 3.20120624 using cabal. My currently installed ExitFailure 1 This is using haskell-platform 2012.1.0.0~debian1 on Ubuntu 12.04. + +> Turns out it needs version 2.3 of the STM library. (libghc-stm-dev +> package). I've made cabal detect an older version and skip building +> the new `git annex watch` command, so you'll be able to build the next +> release. [[done]] --[[Joey]] diff --git a/doc/install.mdwn b/doc/install.mdwn index a009ee00d3..3168976f47 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -37,12 +37,13 @@ To build and use git-annex, you will need: * [TestPack](http://hackage.haskell.org/cgi-bin/hackage-scripts/package/testpack) * [QuickCheck 2](http://hackage.haskell.org/package/QuickCheck) * [HTTP](http://hackage.haskell.org/package/HTTP) - * [hS3](http://hackage.haskell.org/package/hS3) (optional) * [json](http://hackage.haskell.org/package/json) * [IfElse](http://hackage.haskell.org/package/IfElse) * [bloomfilter](http://hackage.haskell.org/package/bloomfilter) * [edit-distance](http://hackage.haskell.org/package/edit-distance) + * [hS3](http://hackage.haskell.org/package/hS3) (optional) * [stm](http://hackage.haskell.org/package/stm) + (optional; version 2.3 or newer) * [hinotify](http://hackage.haskell.org/package/hinotify) (optional; Linux only) * Shell commands diff --git a/git-annex.cabal b/git-annex.cabal index b437057939..f559406959 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 3.20120624 +Version: 3.20120625 Cabal-Version: >= 1.8 License: GPL Maintainer: Joey Hess @@ -31,13 +31,16 @@ Flag S3 Flag Inotify Description: Enable inotify support +Flag Assistant + Description: Enable git-annex assistant and watch command + Executable git-annex Main-Is: git-annex.hs Build-Depends: MissingH, hslogger, directory, filepath, unix, containers, utf8-string, network, mtl, bytestring, old-locale, time, pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP, base == 4.5.*, monad-control, transformers-base, lifted-base, - IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, stm + IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance -- Need to list this because it's generated from a .hsc file. Other-Modules: Utility.Touch C-Sources: Utility/libdiskfree.c @@ -47,6 +50,10 @@ Executable git-annex Build-Depends: hS3 CPP-Options: -DWITH_S3 + if flag(Assistant) + Build-Depends: stm >= 2.3 + CPP-Options: -DWITH_ASSISTANT + if os(linux) && flag(Inotify) Build-Depends: hinotify CPP-Options: -DWITH_INOTIFY @@ -58,7 +65,7 @@ Test-Suite test unix, containers, utf8-string, network, mtl, bytestring, old-locale, time, pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP, base == 4.5.*, monad-control, transformers-base, lifted-base, - IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, stm + IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance Other-Modules: Utility.Touch C-Sources: Utility/libdiskfree.c Extensions: CPP From f2bce89055da1a453d420a7f46b5f86ab0c4c1ff Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 12:36:42 -0400 Subject: [PATCH 3925/8313] better data type for push records Not yet plumbed thru --- Assistant/Pushes.hs | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/Assistant/Pushes.hs b/Assistant/Pushes.hs index f3bffbf792..61d2b798b3 100644 --- a/Assistant/Pushes.hs +++ b/Assistant/Pushes.hs @@ -8,29 +8,30 @@ module Assistant.Pushes where import Common.Annex -import Utility.TSet +import Control.Concurrent.SampleVar import Data.Time.Clock +import qualified Data.Map as M -type FailedPushChan = TSet FailedPush - -data FailedPush = FailedPush - { failedRemote :: Remote - , failedTimeStamp :: UTCTime - } +{- Track the most recent push failure for each remote. -} +type PushMap = M.Map Remote UTCTime +type FailedPushes = SampleVar PushMap newFailedPushChan :: IO FailedPushChan -newFailedPushChan = newTSet +newFailedPushChan = newEmptySampleVar -{- Gets all failed pushes. Blocks until there is at least one failed push. -} -getFailedPushes :: FailedPushChan -> IO [FailedPush] -getFailedPushes = getTSet +{- Gets all failed pushes. Blocks until set. -} +getFailedPushes :: FailedPushChan -> IO PushMap +getFailedPushes = readSampleVar -{- Puts failed pushes back into the channel. - - Note: Original order is not preserved. -} -refillFailedPushes :: FailedPushChan -> [FailedPush] -> IO () -refillFailedPushes = putTSet +{- Sets all failed pushes to passed PushMap -} +setFailedPushes :: FailedPushChan -> PushMap -> IO () +setFailedPushes = writeSampleVar -{- Records a failed push in the channel. -} -recordFailedPush :: FailedPushChan -> FailedPush -> IO () -recordFailedPush = putTSet1 +{- Indicates a failure to push to a single remote. -} +failedPush :: FailedPushChan -> Remote -> IO () +failedPush c r = + +{- Indicates that a remote was pushed to successfully. -} +successfulPush :: FailedPushChan -> Remote -> IO () +successfulPush c r = From a7b8c807d9ea101e36ca4175781e68d641413bea Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 12:37:04 -0400 Subject: [PATCH 3926/8313] new bug I noticed --- doc/bugs/undefined.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/undefined.mdwn diff --git a/doc/bugs/undefined.mdwn b/doc/bugs/undefined.mdwn new file mode 100644 index 0000000000..3c157424c4 --- /dev/null +++ b/doc/bugs/undefined.mdwn @@ -0,0 +1,3 @@ +Trying to move files from a local remote that is not mounted: + + git-annex: Prelude.undefined From 7e62e57f8c9ea49250399f385ee898667df80800 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 17:15:17 -0400 Subject: [PATCH 3927/8313] Avoid ugly failure mode when moving content from a local repository that is not available. Prelude.undefined error message was introduced by bb4f31a0ee496ffb83d31cc56f8827e47605d763. It seems best to filter out local repositories that cannot be accessed from the list of remotes, rather than keeping them in and making every thing that uses the list have to deal with remotes that may have an unknown location. Besides fixing the error message, this also makes unavailable local remotes' names not be shown in various messages, including in git annex status output. Also, move --to an unavailable local repository now avoids some ugly errors like "changeWorkingDirectory: does not exist". --- Git.hs | 5 +++ Remote.hs | 2 +- Remote/Git.hs | 75 ++++++++++++++++++++++------------------- debian/changelog | 2 ++ doc/bugs/undefined.mdwn | 2 ++ 5 files changed, 51 insertions(+), 35 deletions(-) diff --git a/Git.hs b/Git.hs index 7d64205634..eab33f19d7 100644 --- a/Git.hs +++ b/Git.hs @@ -19,6 +19,7 @@ module Git ( repoIsHttp, repoIsLocal, repoIsLocalBare, + repoIsLocalUnknown, repoDescribe, repoLocation, repoPath, @@ -99,6 +100,10 @@ repoIsLocalBare :: Repo -> Bool repoIsLocalBare Repo { location = Local { worktree = Nothing } } = True repoIsLocalBare _ = False +repoIsLocalUnknown :: Repo -> Bool +repoIsLocalUnknown Repo { location = LocalUnknown { } } = True +repoIsLocalUnknown _ = False + assertLocal :: Repo -> a -> a assertLocal repo action | repoIsUrl repo = error $ unwords diff --git a/Remote.hs b/Remote.hs index 839c6ddb09..e211ef7cb6 100644 --- a/Remote.hs +++ b/Remote.hs @@ -75,7 +75,7 @@ byName' :: String -> Annex (Either String Remote) byName' "" = return $ Left "no remote specified" byName' n = handle . filter matching <$> remoteList where - handle [] = Left $ "there is no git remote named \"" ++ n ++ "\"" + handle [] = Left $ "there is no available git remote named \"" ++ n ++ "\"" handle match = Right $ Prelude.head match matching r = n == name r || toUUID n == uuid r diff --git a/Remote/Git.hs b/Remote/Git.hs index cf7542d74e..df74a769c2 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -42,7 +42,8 @@ remote = RemoteType { list :: Annex [Git.Repo] list = do c <- fromRepo Git.config - mapM (tweakurl c) =<< fromRepo Git.remotes + rs <- mapM (tweakurl c) =<< fromRepo Git.remotes + catMaybes <$> mapM configread rs where annexurl n = "remote." ++ n ++ ".annexurl" tweakurl c r = do @@ -52,41 +53,47 @@ list = do Just url -> inRepo $ \g -> Git.Construct.remoteNamed n $ Git.Construct.fromRemoteLocation url g + {- It's assumed to be cheap to read the config of non-URL + - remotes, so this is done each time git-annex is run + - in a way that uses remotes. + - Conversely, the config of an URL remote is only read + - when there is no cached UUID value. -} + configread r = do + notignored <- repoNotIgnored r + u <- getRepoUUID r + r' <- case (repoCheap r, notignored, u) of + (_, False, _) -> return r + (True, _, _) -> tryGitConfigRead r + (False, _, NoUUID) -> tryGitConfigRead r + _ -> return r + {- A repo with a LocalUnknown location is not currently + - accessible, so skip it. -} + if Git.repoIsLocalUnknown r' + then return Nothing + else return $ Just r' + +repoCheap :: Git.Repo -> Bool +repoCheap = not . Git.repoIsUrl gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote -gen r u _ = do - {- It's assumed to be cheap to read the config of non-URL remotes, - - so this is done each time git-annex is run. Conversely, - - the config of an URL remote is only read when there is no - - cached UUID value. -} - let cheap = not $ Git.repoIsUrl r - notignored <- repoNotIgnored r - r' <- case (cheap, notignored, u) of - (_, False, _) -> return r - (True, _, _) -> tryGitConfigRead r - (False, _, NoUUID) -> tryGitConfigRead r - _ -> return r - - u' <- getRepoUUID r' - - let defcst = if cheap then cheapRemoteCost else expensiveRemoteCost - cst <- remoteCost r' defcst - - return Remote { - uuid = u', - cost = cst, - name = Git.repoDescribe r', - storeKey = copyToRemote r', - retrieveKeyFile = copyFromRemote r', - retrieveKeyFileCheap = copyFromRemoteCheap r', - removeKey = dropKey r', - hasKey = inAnnex r', - hasKeyCheap = cheap, - whereisKey = Nothing, - config = Nothing, - repo = r', - remotetype = remote - } +gen r u _ = new <$> remoteCost r defcst + where + defcst = if repoCheap r then cheapRemoteCost else expensiveRemoteCost + new cst = Remote { + uuid = u, + cost = cst, + name = Git.repoDescribe r, + storeKey = copyToRemote r, + retrieveKeyFile = copyFromRemote r, + retrieveKeyFileCheap = copyFromRemoteCheap r, + removeKey = dropKey r, + hasKey = inAnnex r, + hasKeyCheap = repoCheap r, + whereisKey = Nothing, + config = Nothing, + repo = r, + remotetype = remote + } {- Tries to read the config for a specified remote, updates state, and - returns the updated repo. -} diff --git a/debian/changelog b/debian/changelog index 92e992ed75..59eb63b645 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,8 @@ git-annex (3.20120625) UNRELEASED; urgency=low * cabal: Only try to use inotify on Linux. * Version build dependency on STM, and allow building without it, which disables the watch command. + * Avoid ugly failure mode when moving content from a local repository + that is not available. -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 diff --git a/doc/bugs/undefined.mdwn b/doc/bugs/undefined.mdwn index 3c157424c4..8ee37f034a 100644 --- a/doc/bugs/undefined.mdwn +++ b/doc/bugs/undefined.mdwn @@ -1,3 +1,5 @@ Trying to move files from a local remote that is not mounted: git-annex: Prelude.undefined + +> [[fixed|done]] --[[Joey]] From 67c8ef7de25ad6f433db2fa5d5fc764dd515a5b2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 17:33:34 -0400 Subject: [PATCH 3928/8313] use a TMVar SampleMVar won't work; between getting the current value and changing it, another thread could made a change, which would get lost. TMVar works well; this update situation is handled by atomic transactions. --- Assistant.hs | 9 +++++--- Assistant/Pushes.hs | 45 ++++++++++++++++++++++--------------- Assistant/Threads/Pusher.hs | 44 ++++++++++++++++++------------------ Utility/Parallel.hs | 10 +++++---- 4 files changed, 61 insertions(+), 47 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index c054dafd3d..4f8a868f4f 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -53,6 +53,9 @@ - CommitChan: (STM TChan) - Commits are indicated by writing to this channel. The pusher reads - from it. + - FailedPushMap (STM TMVar) + - Failed pushes are indicated by writing to this TMVar. The push + - retrier blocks until they're available. -} module Assistant where @@ -89,10 +92,10 @@ startDaemon assistant foreground liftIO $ a $ do changechan <- newChangeChan commitchan <- newCommitChan - pushchan <- newFailedPushChan + pushmap <- newFailedPushMap _ <- forkIO $ commitThread st changechan commitchan - _ <- forkIO $ pushThread st commitchan pushchan - _ <- forkIO $ pushRetryThread st pushchan + _ <- forkIO $ pushThread st commitchan pushmap + _ <- forkIO $ pushRetryThread st pushmap _ <- forkIO $ mergeThread st _ <- forkIO $ daemonStatusThread st dstatus _ <- forkIO $ sanityCheckerThread st dstatus changechan diff --git a/Assistant/Pushes.hs b/Assistant/Pushes.hs index 61d2b798b3..f411dda07d 100644 --- a/Assistant/Pushes.hs +++ b/Assistant/Pushes.hs @@ -8,30 +8,39 @@ module Assistant.Pushes where import Common.Annex -import Control.Concurrent.SampleVar +import Control.Concurrent.STM import Data.Time.Clock import qualified Data.Map as M {- Track the most recent push failure for each remote. -} type PushMap = M.Map Remote UTCTime -type FailedPushes = SampleVar PushMap +type FailedPushMap = TMVar PushMap -newFailedPushChan :: IO FailedPushChan -newFailedPushChan = newEmptySampleVar +{- The TMVar starts empty, and is left empty when there are no + - failed pushes. This way we can block until there are some failed pushes. + -} +newFailedPushMap :: IO FailedPushMap +newFailedPushMap = atomically newEmptyTMVar -{- Gets all failed pushes. Blocks until set. -} -getFailedPushes :: FailedPushChan -> IO PushMap -getFailedPushes = readSampleVar +{- Blocks until there are failed pushes. + - Returns Remotes whose pushes failed a given time duration or more ago. + - (This may be an empty list.) -} +getFailedPushesBefore :: FailedPushMap -> NominalDiffTime -> IO [Remote] +getFailedPushesBefore v duration = do + m <- atomically $ readTMVar v + now <- getCurrentTime + return $ M.keys $ M.filter (not . toorecent now) m + where + toorecent now time = now `diffUTCTime` time < duration -{- Sets all failed pushes to passed PushMap -} -setFailedPushes :: FailedPushChan -> PushMap -> IO () -setFailedPushes = writeSampleVar - -{- Indicates a failure to push to a single remote. -} -failedPush :: FailedPushChan -> Remote -> IO () -failedPush c r = - -{- Indicates that a remote was pushed to successfully. -} -successfulPush :: FailedPushChan -> Remote -> IO () -successfulPush c r = +{- Modifies the map. -} +changeFailedPushMap :: FailedPushMap -> (PushMap -> PushMap) -> IO () +changeFailedPushMap v a = atomically $ + store . a . fromMaybe M.empty =<< tryTakeTMVar v + where + {- tryTakeTMVar empties the TMVar; refill it only if + - the modified map is not itself empty -} + store m + | m == M.empty = noop + | otherwise = putTMVar v $! m diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs index 82c37de5f8..04d3435287 100644 --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -17,27 +17,23 @@ import Utility.ThreadScheduler import Utility.Parallel import Data.Time.Clock +import qualified Data.Map as M {- This thread retries pushes that failed before. -} -pushRetryThread :: ThreadState -> FailedPushChan -> IO () -pushRetryThread st pushchan = runEvery (Seconds halfhour) $ do +pushRetryThread :: ThreadState -> FailedPushMap -> IO () +pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do -- We already waited half an hour, now wait until there are failed -- pushes to retry. - pushes <- getFailedPushes pushchan - -- Check times, to avoid repushing a push that's too new. - now <- getCurrentTime - let (newpushes, oldpushes) = partition (toorecent now . failedTimeStamp) pushes - unless (null newpushes) $ - refillFailedPushes pushchan newpushes - unless (null oldpushes) $ - pushToRemotes now st pushchan $ map failedRemote oldpushes + topush <- getFailedPushesBefore pushmap (fromIntegral halfhour) + unless (null topush) $ do + now <- getCurrentTime + pushToRemotes now st pushmap topush where halfhour = 1800 - toorecent now time = now `diffUTCTime` time < fromIntegral halfhour {- This thread pushes git commits out to remotes soon after they are made. -} -pushThread :: ThreadState -> CommitChan -> FailedPushChan -> IO () -pushThread st commitchan pushchan = do +pushThread :: ThreadState -> CommitChan -> FailedPushMap -> IO () +pushThread st commitchan pushmap = do remotes <- runThreadState st $ Command.Sync.syncRemotes [] runEvery (Seconds 2) $ do -- We already waited two seconds as a simple rate limiter. @@ -46,7 +42,7 @@ pushThread st commitchan pushchan = do -- Now see if now's a good time to push. now <- getCurrentTime if shouldPush now commits - then pushToRemotes now st pushchan remotes + then pushToRemotes now st pushmap remotes else refillCommits commitchan commits {- Decide if now is a good time to push to remotes. @@ -65,23 +61,27 @@ shouldPush _now commits - - Avoids running possibly long-duration commands in the Annex monad, so - as not to block other threads. -} -pushToRemotes :: UTCTime -> ThreadState -> FailedPushChan -> [Remote] -> IO () -pushToRemotes now st pushchan remotes = do +pushToRemotes :: UTCTime -> ThreadState -> FailedPushMap -> [Remote] -> IO () +pushToRemotes now st pushmap remotes = do (g, branch) <- runThreadState st $ (,) <$> fromRepo id <*> Command.Sync.currentBranch go True branch g remotes where go shouldretry branch g rs = do Command.Sync.updateBranch (Command.Sync.syncBranch branch) g - failed <- inParallel (push g branch) rs - unless (null failed) $ - if shouldretry - then retry branch g rs - else refillFailedPushes pushchan $ - map (`FailedPush` now) failed + (succeeded, failed) <- inParallel (push g branch) rs + changeFailedPushMap pushmap $ \m -> + M.union (makemap failed) $ + M.difference m (makemap succeeded) + unless (null failed || not shouldretry) $ + retry branch g failed + + makemap l = M.fromList $ zip l (repeat now) + push g branch remote = ifM (Command.Sync.pushBranch remote branch g) ( exitSuccess, exitFailure) + retry branch g rs = do runThreadState st $ manualPull branch rs go False branch g rs diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs index 6e4671c057..9df95ab2b0 100644 --- a/Utility/Parallel.hs +++ b/Utility/Parallel.hs @@ -10,11 +10,13 @@ module Utility.Parallel where import Common {- Runs an action in parallel with a set of values. - - Returns values that caused the action to fail. -} -inParallel :: (v -> IO ()) -> [v] -> IO [v] + - Returns the values partitioned into ones with which the action succeeded, + - and ones with which it failed. -} +inParallel :: (v -> IO ()) -> [v] -> IO ([v], [v]) inParallel a l = do pids <- mapM (forkProcess . a) l statuses <- mapM (getProcessStatus True False) pids - return $ map fst $ filter (failed . snd) $ zip l statuses + return $ reduce $ partition (succeeded . snd) $ zip l statuses where - failed v = v /= Just (Exited ExitSuccess) + succeeded v = v == Just (Exited ExitSuccess) + reduce (x,y) = (map fst x, map fst y) From 1eff74dd44d9124fe0fd9c480e2e36cdd562c0cc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 19:25:46 -0400 Subject: [PATCH 3929/8313] blog for the day --- .../blog/day_17__push_queue_prune.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/design/assistant/blog/day_17__push_queue_prune.mdwn diff --git a/doc/design/assistant/blog/day_17__push_queue_prune.mdwn b/doc/design/assistant/blog/day_17__push_queue_prune.mdwn new file mode 100644 index 0000000000..54ee75fb8d --- /dev/null +++ b/doc/design/assistant/blog/day_17__push_queue_prune.mdwn @@ -0,0 +1,19 @@ +Not much available time today, only a few hours. + +Main thing I did was fixed up the failed push tracking to use a better data +structure. No need for a queue of failed pushes, all it needs is a map of +remotes that have an outstanding failed push, and a timestamp. Now it +won't grow in memory use forever anymore. :) + +Finding the right thread mutex type for this turned out to be a bit of a +challenge. I ended up with a STM TMVar, which is left empty when there are +no pushes to retry, so the thread using it blocks until there are some. And, +it can be updated transactionally, without races. + +I also fixed a bug outside the git-annex assistant code. It was possible to +crash git-annex if a local git repository was configured as a remote, and +the repository was not available on startup. git-annex now ignores such +remotes. This does impact the assistant, since it is a long running process +and git repositories will come and go. Now it ignores any that +were not available when it started up. This will need to be dealt with when +making it support removable drives. From 06aeff9c7db515ecbff1cc76efc904cf01172540 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkWzAq6TusMi9zI3FLkDOETRIAUTtmGZVg" Date: Wed, 27 Jun 2012 00:33:04 +0000 Subject: [PATCH 3930/8313] --- ...not_decode_byte___39____92__xfc__39__.mdwn | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn new file mode 100644 index 0000000000..d7fbbc6c2d --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn @@ -0,0 +1,32 @@ +What steps will reproduce the problem? + + alip@hayalet /tmp/aaa (git)-[master] % git annex init aaa + init aaa ok + (Recording state in git...) + alip@hayalet /tmp/aaa (git)-[master] % git remote add çüş /tmp/çüş + alip@hayalet /tmp/aaa (git)-[master] % git annex sync --debug + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","symbolic-ref","HEAD"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","show-ref","git-annex"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","show-ref","--hash","refs/heads/git-annex"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","log","refs/heads/git-annex..bc45cd9c2cb7c9b0c7a12a4c0210fe6a262abac9","--oneline","-n1"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","log","refs/heads/git-annex..9220bfedd1e13b2d791c918e2d59901af353825f","--oneline","-n1"] + (merging origin/git-annex into git-annex...) + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","cat-file","--batch"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","update-index","-z","--index-info"] + git ["--git-dir=/tmp/aaa/.git","--work-tree=/tmp/aaa","diff-index","--raw","-z","-r","--no-renames","-l0","--cached","9220bfedd1e13b2d791c918e2d59901af353825f"] + git-annex: Cannot decode byte '\xfc': Data.Text.Encoding.decodeUtf8: Invalid UTF-8 stream + 1 alip@hayalet /tmp/aaa (git)-[master] % + +What is the expected output? What do you see instead? + +Syncing a repository under a path with utf-8 characters in its name fails. + +What version of git-annex are you using? On what operating system? + +git-annex version: 3.20120624 + +On Exherbo, linux-3.4 + +Please provide any additional information below. + +'\xfc' is valid UTF-8: 'LATIN SMALL LETTER U WITH DIAERESIS' From 6aee7e5a8b581b342d0c34d25b57fdb60a3c0821 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 22:27:30 -0400 Subject: [PATCH 3931/8313] Better fix for unavailable local remotes Not including such remotes turned out to have other consequences, including annex-truselevel git config being ignored. Instead, add guards before each operation that might try to operate on such a repo. --- Remote/Git.hs | 56 +++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/Remote/Git.hs b/Remote/Git.hs index df74a769c2..60a881803a 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -43,7 +43,7 @@ list :: Annex [Git.Repo] list = do c <- fromRepo Git.config rs <- mapM (tweakurl c) =<< fromRepo Git.remotes - catMaybes <$> mapM configread rs + mapM configread rs where annexurl n = "remote." ++ n ++ ".annexurl" tweakurl c r = do @@ -61,16 +61,11 @@ list = do configread r = do notignored <- repoNotIgnored r u <- getRepoUUID r - r' <- case (repoCheap r, notignored, u) of + case (repoCheap r, notignored, u) of (_, False, _) -> return r (True, _, _) -> tryGitConfigRead r (False, _, NoUUID) -> tryGitConfigRead r _ -> return r - {- A repo with a LocalUnknown location is not currently - - accessible, so skip it. -} - if Git.repoIsLocalUnknown r' - then return Nothing - else return $ Just r' repoCheap :: Git.Repo -> Bool repoCheap = not . Git.repoIsUrl @@ -95,6 +90,21 @@ gen r u _ = new <$> remoteCost r defcst remotetype = remote } +{- Checks relatively inexpensively if a repository is available for use. -} +repoAvail :: Git.Repo -> Annex Bool +repoAvail r + | Git.repoIsHttp r = return True + | Git.repoIsUrl r = return True + | Git.repoIsLocalUnknown r = return False + | otherwise = liftIO $ catchBoolIO $ onLocal r $ return True + +{- Avoids performing an action on a local repository that's not usable. + - Does not check that the repository is still available on disk. -} +guardUsable :: Git.Repo -> a -> Annex a -> Annex a +guardUsable r onerr a + | Git.repoIsLocalUnknown r = return onerr + | otherwise = a + {- Tries to read the config for a specified remote, updates state, and - returns the updated repo. -} tryGitConfigRead :: Git.Repo -> Annex Git.Repo @@ -166,7 +176,7 @@ inAnnex r key dispatch ExitSuccess = Right True dispatch (ExitFailure 1) = Right False dispatch _ = unknown - checklocal = dispatch <$> check + checklocal = guardUsable r unknown $ dispatch <$> check where check = liftIO $ catchMsgIO $ onLocal r $ Annex.Content.inAnnexSafe key @@ -175,13 +185,6 @@ inAnnex r key dispatch (Right Nothing) = unknown unknown = Left $ "unable to check " ++ Git.repoDescribe r -{- Checks inexpensively if a repository is available for use. -} -repoAvail :: Git.Repo -> Annex Bool -repoAvail r - | Git.repoIsHttp r = return True - | Git.repoIsUrl r = return True - | otherwise = liftIO $ catchBoolIO $ onLocal r $ return True - {- Runs an action on a local repository inexpensively, by making an annex - monad using that repository. -} onLocal :: Git.Repo -> Annex a -> IO a @@ -200,14 +203,15 @@ keyUrls r key = map tourl (annexLocations key) dropKey :: Git.Repo -> Key -> Annex Bool dropKey r key - | not $ Git.repoIsUrl r = commitOnCleanup r $ liftIO $ onLocal r $ do - ensureInitialized - whenM (Annex.Content.inAnnex key) $ do - Annex.Content.lockContent key $ - Annex.Content.removeAnnex key - Annex.Content.logStatus key InfoMissing - Annex.Content.saveState True - return True + | not $ Git.repoIsUrl r = + guardUsable r False $ commitOnCleanup r $ liftIO $ onLocal r $ do + ensureInitialized + whenM (Annex.Content.inAnnex key) $ do + Annex.Content.lockContent key $ + Annex.Content.removeAnnex key + Annex.Content.logStatus key InfoMissing + Annex.Content.saveState True + return True | Git.repoIsHttp r = error "dropping from http repo not supported" | otherwise = commitOnCleanup r $ onRemote r (boolSystem, False) "dropkey" [ Params "--quiet --force" @@ -217,7 +221,7 @@ dropKey r key {- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemote r key file - | not $ Git.repoIsUrl r = do + | not $ Git.repoIsUrl r = guardUsable r False $ do params <- rsyncParams r loc <- liftIO $ gitAnnexLocation key r rsyncOrCopyFile params loc file @@ -227,7 +231,7 @@ copyFromRemote r key file copyFromRemoteCheap :: Git.Repo -> Key -> FilePath -> Annex Bool copyFromRemoteCheap r key file - | not $ Git.repoIsUrl r = do + | not $ Git.repoIsUrl r = guardUsable r False $ do loc <- liftIO $ gitAnnexLocation key r liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True | Git.repoIsSsh r = @@ -240,7 +244,7 @@ copyFromRemoteCheap r key file {- Tries to copy a key's content to a remote's annex. -} copyToRemote :: Git.Repo -> Key -> Annex Bool copyToRemote r key - | not $ Git.repoIsUrl r = commitOnCleanup r $ do + | not $ Git.repoIsUrl r = guardUsable r False $ commitOnCleanup r $ do keysrc <- inRepo $ gitAnnexLocation key params <- rsyncParams r -- run copy from perspective of remote From 5faaf5936855407f62b1e9b3e9db5d03a38a9b62 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 27 Jun 2012 02:48:31 +0000 Subject: [PATCH 3932/8313] Added a comment --- ...mment_1_f1a7352b04f395e06e0094c1f51b6fff._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment new file mode 100644 index 0000000000..28faa7b45d --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_1_f1a7352b04f395e06e0094c1f51b6fff._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-27T02:48:31Z" + content=""" +I don't think this has to do with the path name of the repository containing utf-8 at all. + +Your recipe for reproducing this depends on some pre-existing repository that I don't know how to set up to reproduce this bug. All I can guess is that, based on the \"decodeUtf8\" in the error message, it's coming from the one part of the code that still uses that, the union merger. + + +"""]] From 1093d82f6b6fb7844ca11b136f03767ca4460a98 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 22:58:44 -0400 Subject: [PATCH 3933/8313] Got rid of the last place that did utf8 decoding. Probably fixes bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/ although I don't know how to reproduce that bug. --- Git/UnionMerge.hs | 11 +++++++---- debian/changelog | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Git/UnionMerge.hs b/Git/UnionMerge.hs index 0987f9131a..504147e1dc 100644 --- a/Git/UnionMerge.hs +++ b/Git/UnionMerge.hs @@ -10,8 +10,7 @@ module Git.UnionMerge ( mergeIndex ) where -import qualified Data.Text.Lazy as L -import qualified Data.Text.Lazy.Encoding as L +import qualified Data.ByteString.Lazy as L import qualified Data.Set as S import Common @@ -79,10 +78,14 @@ mergeFile info file h repo = case filter (/= nullSha) [Ref asha, Ref bsha] of =<< calcMerge . zip shas <$> mapM getcontents shas where [_colonmode, _bmode, asha, bsha, _status] = words info - getcontents s = map L.unpack . L.lines . - L.decodeUtf8 <$> catObject h s use sha = return $ Just $ updateIndexLine sha FileBlob $ asTopFilePath file + -- We don't know how the file is encoded, but need to + -- split it into lines to union merge. Using the + -- FileSystemEncoding for this is a hack, but ensures there + -- are no decoding errors. Note that this works because + -- streamUpdateIndex sets fileEncoding on its write handle. + getcontents s = lines . encodeW8 . L.unpack <$> catObject h s {- Calculates a union merge between a list of refs, with contents. - diff --git a/debian/changelog b/debian/changelog index 59eb63b645..5968d85602 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,7 @@ git-annex (3.20120625) UNRELEASED; urgency=low which disables the watch command. * Avoid ugly failure mode when moving content from a local repository that is not available. + * Got rid of the last place that did utf8 decoding. -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 From 6f45827fe07262f93f39ae420f4362bfa0246194 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 26 Jun 2012 23:07:11 -0400 Subject: [PATCH 3934/8313] git-config fileEncoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept arbitrarily encoded repository filepaths etc when reading git config output. This fixes support for remotes with unusual characters in their names. For example, a remote with a url of /tmp/çüş was previously skipped, because the filename wasn't encoded right so it didn't think it was available. And when setting the annex-uuid of a remote named "çüş", it used to add it under a mis-encoded form of the remote's name. Both these cases now work ok in my testing. --- Git/Config.hs | 4 ++++ debian/changelog | 3 +++ 2 files changed, 7 insertions(+) diff --git a/Git/Config.hs b/Git/Config.hs index dab1cdf5ea..c9e4f9a2dc 100644 --- a/Git/Config.hs +++ b/Git/Config.hs @@ -54,6 +54,10 @@ read' repo = go repo {- Reads git config from a handle and populates a repo with it. -} hRead :: Repo -> Handle -> IO Repo hRead repo h = do + -- We use the FileSystemEncoding when reading from git-config, + -- because it can contain arbitrary filepaths (and other strings) + -- in any encoding. + fileEncoding h val <- hGetContentsStrict h store val repo diff --git a/debian/changelog b/debian/changelog index 5968d85602..c1ebac8398 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,9 @@ git-annex (3.20120625) UNRELEASED; urgency=low * Avoid ugly failure mode when moving content from a local repository that is not available. * Got rid of the last place that did utf8 decoding. + * Accept arbitrarily encoded repository filepaths etc when reading + git config output. This fixes support for remotes with unusual characters + in their names. -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 From f6f4b5dfaadae6d271cc1017775c3c05e02e9e46 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 27 Jun 2012 03:08:13 +0000 Subject: [PATCH 3935/8313] Added a comment --- .../comment_2_c1890067079cd99667f31cbb4d2e4545._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment new file mode 100644 index 0000000000..3486be7337 --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_2_c1890067079cd99667f31cbb4d2e4545._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 2" + date="2012-06-27T03:08:13Z" + content=""" +Since I can't reproduce it I am not sure, but it may be fixed by the commits I've just made. +"""]] From b356015cf4e04dc2d14d0d239fb0b811bbbedc68 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 27 Jun 2012 06:50:41 +0000 Subject: [PATCH 3936/8313] --- ...t:_getting_the_disk_used_by_a_subtree_of_files.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files.mdwn diff --git a/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files.mdwn b/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files.mdwn new file mode 100644 index 0000000000..7bdd93654f --- /dev/null +++ b/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files.mdwn @@ -0,0 +1,10 @@ +I'm not sure if this _feature_ exists already wrapped or provided as a recipe for users or not yet. But it would be nice to be able to do a + + git annex du [PATH] + +Such that the output that git annex would return is the total disk used locally in the PATH and the theoretical disk used by the PATH if it was fully populated locally. e.g. + + $ git annex du FSL0001_ANALYSIS + $ Local: 1000kb, Annex: 2000kb + +or something along the lines of that? From 38571d8133d1e46c7550816a8dd95e51f4cabbc0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkwR9uOA38yi5kEUvcEWNtRiZwpxXskayE" Date: Wed, 27 Jun 2012 08:54:53 +0000 Subject: [PATCH 3937/8313] Added a comment --- ...omment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment diff --git a/doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment b/doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment new file mode 100644 index 0000000000..69a4f9128d --- /dev/null +++ b/doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkwR9uOA38yi5kEUvcEWNtRiZwpxXskayE" + nickname="Agustin" + subject="comment 11" + date="2012-06-27T08:54:52Z" + content=""" +Hi @joey! Perfect!... I'll do that then! + +Thanks for your time man! +"""]] From 48c0fd4bed945f1e4c85d16d1a092a69dc4dde32 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 27 Jun 2012 12:36:09 +0000 Subject: [PATCH 3938/8313] Added a comment --- .../comment_1_7abb1155081a23ce4829ee69b2064541._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment diff --git a/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment b/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment new file mode 100644 index 0000000000..bff5b2ea7e --- /dev/null +++ b/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_1_7abb1155081a23ce4829ee69b2064541._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.153.2.25" + subject="comment 1" + date="2012-06-27T12:36:08Z" + content=""" +Use `du -L` for the disk space used locally. The other number is not currently available, but it would be nice to have. I also sometimes would like to have data on which backends are used how much, so making this `git annex status --subdir` is tempting. Unfortunatly, it's current implementation scans `.git/annex/objects` +and not the disk tree (better for accurate numbers due to copies), so it would not be a very easy thing to add. Not massively hard, but not something I can pound out before I start work today.. +"""]] From 46c39888df6df29eccd25b8be9d35f6ce7d6124d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkWzAq6TusMi9zI3FLkDOETRIAUTtmGZVg" Date: Wed, 27 Jun 2012 12:56:37 +0000 Subject: [PATCH 3939/8313] Added a comment --- ...ment_3_213c96085c60c8e52cd803df07240158._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment new file mode 100644 index 0000000000..48a3820296 --- /dev/null +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__/comment_3_213c96085c60c8e52cd803df07240158._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkWzAq6TusMi9zI3FLkDOETRIAUTtmGZVg" + nickname="Ali" + subject="comment 3" + date="2012-06-27T12:56:37Z" + content=""" +Yes, the problem is fixed. + +The repository was a normal git repository with path /tmp/çüş (git init) +and with annex description \"çüş\" (git annex init çüş) + +afaict, i can't reproduce the problem anymore either :-) +"""]] From 8e8439a5191e8768edebdcf27668157b70c0ebf7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 09:27:59 -0400 Subject: [PATCH 3940/8313] add ls-files --unmerged support --- Git/LsFiles.hs | 27 ++++++++++++++++++++++++++- Git/Types.hs | 6 ++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs index 06d4b9f44f..540503a28a 100644 --- a/Git/LsFiles.hs +++ b/Git/LsFiles.hs @@ -1,6 +1,6 @@ {- git ls-files interface - - - Copyright 2010 Joey Hess + - Copyright 2010,2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -18,6 +18,8 @@ module Git.LsFiles ( import Common import Git import Git.Command +import Git.Types +import Git.Sha {- Scans for files that are checked into git at the specified locations. -} inRepo :: [FilePath] -> Repo -> IO [FilePath] @@ -75,3 +77,26 @@ typeChanged' ps l repo = do where prefix = [Params "diff --name-only --diff-filter=T -z"] suffix = Param "--" : map File l + +data Unmerged = Unmerged + { unmergedFile :: FilePath + , unmergedBlobType :: BlobType + , unmergedSha :: Sha + } + +{- Returns a list of the files in the specified locations that have + - unresolved merge conflicts. Each unmerged file will have duplicates + - in the list for each unmerged version (typically two). -} +unmerged :: [FilePath] -> Repo -> IO [Unmerged] +unmerged l repo = catMaybes . map parse <$> list repo + where + list = pipeNullSplit $ Params "ls-files --unmerged -z --" : map File l + parse s + | null file || length ws < 2 = Nothing + | otherwise = do + blobtype <- readBlobType (ws !! 0) + sha <- extractSha (ws !! 1) + return $ Unmerged file blobtype sha + where + (metadata, file) = separate (== '\t') s + ws = words metadata diff --git a/Git/Types.hs b/Git/Types.hs index 1df6e343b8..e8cdbb442d 100644 --- a/Git/Types.hs +++ b/Git/Types.hs @@ -71,3 +71,9 @@ instance Show BlobType where show FileBlob = "100644" show ExecutableBlob = "100755" show SymlinkBlob = "120000" + +readBlobType :: String -> Maybe BlobType +readBlobType "100644" = Just FileBlob +readBlobType "100755" = Just ExecutableBlob +readBlobType "120000" = Just SymlinkBlob +readBlobType _ = Nothing From 051c68041b5b7a58e7080403e389d0641691edfd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 12:09:01 -0400 Subject: [PATCH 3941/8313] properly handle deleted files when processing ls-files --unmerged --- Command/Sync.hs | 24 ++++++++++++-- Git/LsFiles.hs | 88 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 94 insertions(+), 18 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index 1da6b0b812..2f38636175 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -155,10 +155,30 @@ mergeAnnex = do Annex.Branch.forceUpdate stop -mergeFrom :: Git.Ref -> CommandCleanup +mergeFrom :: Git.Ref -> Annex Bool mergeFrom branch = do showOutput - inRepo $ Git.Merge.mergeNonInteractive branch + ok <- inRepo $ Git.Merge.mergeNonInteractive branch + if ok + then return ok + else resolveMerge + +{- Resolves a conflicted merge. It's important that any conflicts be + - resolved in a way that itself avoids later merge conflicts, since + - multiple repositories may be doing this concurrently. + - + - Only annexed files are resolved; other files are left for the user to + - handle. + - + - This uses the Keys pointed to by the files to construct new + - filenames. So a conflicted merge of file foo will delete it, + - and add files foo.KEYA and foo.KEYB. + - + - A conflict can also result due to + -} +resolveMerge :: Annex Bool +resolveMerge = do + changed :: Remote -> Git.Ref -> Annex Bool changed remote b = do diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs index 540503a28a..ce7c84aee9 100644 --- a/Git/LsFiles.hs +++ b/Git/LsFiles.hs @@ -13,6 +13,9 @@ module Git.LsFiles ( changedUnstaged, typeChanged, typeChangedStaged, + Conflicting(..), + Unmerged(..), + unmerged, ) where import Common @@ -78,25 +81,78 @@ typeChanged' ps l repo = do prefix = [Params "diff --name-only --diff-filter=T -z"] suffix = Param "--" : map File l +{- A item in conflict has two possible values. + - Either can be Nothing, when that side deleted the file. -} +data Conflicting v = Conflicting + { valUs :: Maybe v + , valThem :: Maybe v + } deriving (Show) + +isConflicting :: Eq a => Conflicting a -> Bool +isConflicting (Conflicting a b) = a /= b + data Unmerged = Unmerged { unmergedFile :: FilePath - , unmergedBlobType :: BlobType - , unmergedSha :: Sha - } + , unmergedBlobType :: Conflicting BlobType + , unmergedSha :: Conflicting Sha + } deriving (Show) {- Returns a list of the files in the specified locations that have - - unresolved merge conflicts. Each unmerged file will have duplicates - - in the list for each unmerged version (typically two). -} + - unresolved merge conflicts. + - + - ls-files outputs multiple lines per conflicting file, each with its own + - stage number: + - 1 = old version, can be ignored + - 2 = us + - 3 = them + - If a line is omitted, that side deleted the file. + -} unmerged :: [FilePath] -> Repo -> IO [Unmerged] -unmerged l repo = catMaybes . map parse <$> list repo +unmerged l repo = reduceUnmerged [] . catMaybes . map parseUnmerged <$> list repo where - list = pipeNullSplit $ Params "ls-files --unmerged -z --" : map File l - parse s - | null file || length ws < 2 = Nothing - | otherwise = do - blobtype <- readBlobType (ws !! 0) - sha <- extractSha (ws !! 1) - return $ Unmerged file blobtype sha - where - (metadata, file) = separate (== '\t') s - ws = words metadata + files = map File l + list = pipeNullSplit $ Params "ls-files --unmerged -z --" : files + +data InternalUnmerged = InternalUnmerged + { isus :: Bool + , ifile :: FilePath + , iblobtype :: Maybe BlobType + , isha :: Maybe Sha + } deriving (Show) + +parseUnmerged :: String -> Maybe InternalUnmerged +parseUnmerged s + | null file || length ws < 3 = Nothing + | otherwise = do + stage <- readish (ws !! 2) + unless (stage == 2 || stage == 3) $ + fail undefined -- skip stage 1 + blobtype <- readBlobType (ws !! 0) + sha <- extractSha (ws !! 1) + return $ InternalUnmerged (stage == 2) file (Just blobtype) (Just sha) + where + (metadata, file) = separate (== '\t') s + ws = words metadata + +reduceUnmerged :: [Unmerged] -> [InternalUnmerged] -> [Unmerged] +reduceUnmerged c [] = c +reduceUnmerged c (i:is) = reduceUnmerged (new:c) rest + where + (rest, sibi) = findsib i is + (blobtypeA, blobtypeB, shaA, shaB) + | isus i = (iblobtype i, iblobtype sibi, isha i, isha sibi) + | otherwise = (iblobtype sibi, iblobtype i, isha sibi, isha i) + new = Unmerged + { unmergedFile = ifile i + , unmergedBlobType = Conflicting blobtypeA blobtypeB + , unmergedSha = Conflicting shaA shaB + } + findsib templatei [] = ([], deleted templatei) + findsib templatei (i:is) + | ifile i == ifile templatei = (is, i) + | otherwise = (i:is, deleted templatei) + deleted templatei = templatei + { isus = not (isus templatei) + , iblobtype = Nothing + , isha = Nothing + } From 048b64024a14feb0d9ed26abe97c542cfacbc8af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 13:08:32 -0400 Subject: [PATCH 3942/8313] sync: Automatically resolves merge conflicts. untested, but it compiles :) --- Command/Sync.hs | 53 +++++++++++++++++++++++++++++++++++++++++----- Git/LsFiles.hs | 11 ++++------ Git/Types.hs | 2 ++ debian/changelog | 1 + doc/git-annex.mdwn | 5 +++++ 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index 2f38636175..a39a2e57f1 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -15,15 +15,21 @@ import Command import qualified Remote import qualified Annex import qualified Annex.Branch +import qualified Annex.Queue +import Annex.Content +import Annex.CatFile import qualified Git.Command +import qualified Git.LsFiles as LsFiles import qualified Git.Merge import qualified Git.Branch import qualified Git.Ref import qualified Git +import Git.Types (BlobType(..)) import qualified Types.Remote import qualified Remote.Git import qualified Data.Map as M +import qualified Data.ByteString.Lazy as L def :: [Command] def = [command "sync" (paramOptional (paramRepeating paramRemote)) @@ -161,7 +167,11 @@ mergeFrom branch = do ok <- inRepo $ Git.Merge.mergeNonInteractive branch if ok then return ok - else resolveMerge + else do + merged <- resolveMerge + when merged $ + showNote "merge conflict automatically resolved" + return merged {- Resolves a conflicted merge. It's important that any conflicts be - resolved in a way that itself avoids later merge conflicts, since @@ -171,15 +181,48 @@ mergeFrom branch = do - handle. - - This uses the Keys pointed to by the files to construct new - - filenames. So a conflicted merge of file foo will delete it, - - and add files foo.KEYA and foo.KEYB. + - filenames. So when both sides modified file foo, + - it will be deleted, and replaced with files foo.KEYA and foo.KEYB. - - - A conflict can also result due to + - On the other hand, when one side deleted foo, and the other modified it, + - it will be deleted, and the modified version stored as file + - foo.KEYA (or KEYB). -} resolveMerge :: Annex Bool resolveMerge = do - + top <- fromRepo Git.repoPath + all id <$> (mapM resolveMerge' =<< inRepo (LsFiles.unmerged [top])) +resolveMerge' :: LsFiles.Unmerged -> Annex Bool +resolveMerge' u + | issymlink LsFiles.valUs && issymlink LsFiles.valThem = do + keyUs <- getkey LsFiles.valUs + keyThem <- getkey LsFiles.valThem + if (keyUs == keyThem) + then makelink keyUs (file ++ "." ++ show keyUs) + else do + void $ liftIO $ tryIO $ removeFile file + Annex.Queue.addCommand "rm" [Params "--quiet -f --"] [file] + makelink keyUs (file ++ "." ++ show keyUs) + makelink keyThem (file ++ "." ++ show keyThem) + return True + | otherwise = return False + where + file = LsFiles.unmergedFile u + issymlink select = any (select (LsFiles.unmergedBlobType u) ==) + [Just SymlinkBlob, Nothing] + makelink (Just key) f = do + l <- calcGitLink file key + liftIO $ createSymbolicLink l f + Annex.Queue.addCommand "add" [Param "--force", Param "--"] [f] + makelink _ _ = noop + getkey select = do + let msha = select $ LsFiles.unmergedSha u + case msha of + Nothing -> return Nothing + Just sha -> fileKey . takeFileName + . encodeW8 . L.unpack <$> catObject sha + changed :: Remote -> Git.Ref -> Annex Bool changed remote b = do let r = remoteBranch remote b diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs index ce7c84aee9..321913334b 100644 --- a/Git/LsFiles.hs +++ b/Git/LsFiles.hs @@ -88,9 +88,6 @@ data Conflicting v = Conflicting , valThem :: Maybe v } deriving (Show) -isConflicting :: Eq a => Conflicting a -> Bool -isConflicting (Conflicting a b) = a /= b - data Unmerged = Unmerged { unmergedFile :: FilePath , unmergedBlobType :: Conflicting BlobType @@ -124,7 +121,7 @@ parseUnmerged :: String -> Maybe InternalUnmerged parseUnmerged s | null file || length ws < 3 = Nothing | otherwise = do - stage <- readish (ws !! 2) + stage <- readish (ws !! 2) :: Maybe Int unless (stage == 2 || stage == 3) $ fail undefined -- skip stage 1 blobtype <- readBlobType (ws !! 0) @@ -148,9 +145,9 @@ reduceUnmerged c (i:is) = reduceUnmerged (new:c) rest , unmergedSha = Conflicting shaA shaB } findsib templatei [] = ([], deleted templatei) - findsib templatei (i:is) - | ifile i == ifile templatei = (is, i) - | otherwise = (i:is, deleted templatei) + findsib templatei (l:ls) + | ifile l == ifile templatei = (ls, l) + | otherwise = (l:ls, deleted templatei) deleted templatei = templatei { isus = not (isus templatei) , iblobtype = Nothing diff --git a/Git/Types.hs b/Git/Types.hs index e8cdbb442d..0c37427c7d 100644 --- a/Git/Types.hs +++ b/Git/Types.hs @@ -51,6 +51,7 @@ type Tag = Ref {- Types of objects that can be stored in git. -} data ObjectType = BlobObject | CommitObject | TreeObject + deriving (Eq) instance Show ObjectType where show BlobObject = "blob" @@ -65,6 +66,7 @@ readObjectType _ = Nothing {- Types of blobs. -} data BlobType = FileBlob | ExecutableBlob | SymlinkBlob + deriving (Eq) {- Git uses magic numbers to denote the type of a blob. -} instance Show BlobType where diff --git a/debian/changelog b/debian/changelog index c1ebac8398..46afb6e4d5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,7 @@ git-annex (3.20120625) UNRELEASED; urgency=low * Accept arbitrarily encoded repository filepaths etc when reading git config output. This fixes support for remotes with unusual characters in their names. + * sync: Automatically resolves merge conflicts. -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index 39fad04882..c52a5f3bf9 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -135,6 +135,11 @@ subdirectories). commands to do each of those steps by hand, or if you don't want to worry about the details, you can use sync. + Merge conflicts are automatically resolved by sync. When two conflicting + versions of a file have been committed, both will be added to the tree, + under different filenames. For example, file "foo" would be replaced + with "foo.somekey" and "foo.otherkey". + Note that syncing with a remote will not update the remote's working tree with changes made to the local repository. However, those changes are pushed to the remote, so can be merged into its working tree From abd36ed33659f9b0b369c6d2510455365a943e3c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 13:35:02 -0400 Subject: [PATCH 3943/8313] don't automerge when the symlinks cannot be parsed as keys --- Command/Sync.hs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index a39a2e57f1..8ac0399435 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -195,19 +195,21 @@ resolveMerge = do resolveMerge' :: LsFiles.Unmerged -> Annex Bool resolveMerge' u - | issymlink LsFiles.valUs && issymlink LsFiles.valThem = do - keyUs <- getkey LsFiles.valUs - keyThem <- getkey LsFiles.valThem - if (keyUs == keyThem) - then makelink keyUs (file ++ "." ++ show keyUs) - else do + | issymlink LsFiles.valUs && issymlink LsFiles.valThem = + withKey LsFiles.valUs $ \keyUs -> + withKey LsFiles.valThem $ \keyThem -> go keyUs keyThem + | otherwise = return False + where + go keyUs keyThem + | keyUs == keyThem = do + makelink keyUs (file ++ "." ++ show keyUs) + return True + | otherwise = do void $ liftIO $ tryIO $ removeFile file Annex.Queue.addCommand "rm" [Params "--quiet -f --"] [file] makelink keyUs (file ++ "." ++ show keyUs) makelink keyThem (file ++ "." ++ show keyThem) - return True - | otherwise = return False - where + return True file = LsFiles.unmergedFile u issymlink select = any (select (LsFiles.unmergedBlobType u) ==) [Just SymlinkBlob, Nothing] @@ -216,12 +218,15 @@ resolveMerge' u liftIO $ createSymbolicLink l f Annex.Queue.addCommand "add" [Param "--force", Param "--"] [f] makelink _ _ = noop - getkey select = do + withKey select a = do let msha = select $ LsFiles.unmergedSha u case msha of - Nothing -> return Nothing - Just sha -> fileKey . takeFileName - . encodeW8 . L.unpack <$> catObject sha + Nothing -> a Nothing + Just sha -> do + key <- fileKey . takeFileName + . encodeW8 . L.unpack + <$> catObject sha + maybe (return False) (a . Just) key changed :: Remote -> Git.Ref -> Annex Bool changed remote b = do From 855eca0cc66675b8b606c087eafb923883baf904 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 13:38:21 -0400 Subject: [PATCH 3944/8313] close --- .../git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn index d7fbbc6c2d..862259422d 100644 --- a/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn +++ b/doc/bugs/git-annex:_Cannot_decode_byte___39____92__xfc__39__.mdwn @@ -30,3 +30,5 @@ On Exherbo, linux-3.4 Please provide any additional information below. '\xfc' is valid UTF-8: 'LATIN SMALL LETTER U WITH DIAERESIS' + +> closing as non-reproducible and presumably fixed. [[done]] --[[Joey]] From 8810e57995f78876d5eb2b5429272d884c5e25c2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 15:00:26 -0400 Subject: [PATCH 3945/8313] fix file name --- Command/Sync.hs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index 8ac0399435..759afed822 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -202,22 +202,23 @@ resolveMerge' u where go keyUs keyThem | keyUs == keyThem = do - makelink keyUs (file ++ "." ++ show keyUs) + makelink keyUs return True | otherwise = do void $ liftIO $ tryIO $ removeFile file Annex.Queue.addCommand "rm" [Params "--quiet -f --"] [file] - makelink keyUs (file ++ "." ++ show keyUs) - makelink keyThem (file ++ "." ++ show keyThem) + makelink keyUs + makelink keyThem return True file = LsFiles.unmergedFile u issymlink select = any (select (LsFiles.unmergedBlobType u) ==) [Just SymlinkBlob, Nothing] - makelink (Just key) f = do - l <- calcGitLink file key - liftIO $ createSymbolicLink l f - Annex.Queue.addCommand "add" [Param "--force", Param "--"] [f] - makelink _ _ = noop + makelink (Just key) = do + let dest = file ++ "." ++ show key + l <- calcGitLink dest key + liftIO $ createSymbolicLink l dest + Annex.Queue.addCommand "add" [Param "--force", Param "--"] [dest] + makelink _ = noop withKey select a = do let msha = select $ LsFiles.unmergedSha u case msha of From 9147ad74931222f05b76102bfea61b1fe177fd32 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 15:03:13 -0400 Subject: [PATCH 3946/8313] commit merge resolution this is necessary so the sync can continue successfully with its push phase --- Command/Sync.hs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index 759afed822..b146379d14 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -167,11 +167,7 @@ mergeFrom branch = do ok <- inRepo $ Git.Merge.mergeNonInteractive branch if ok then return ok - else do - merged <- resolveMerge - when merged $ - showNote "merge conflict automatically resolved" - return merged + else resolveMerge {- Resolves a conflicted merge. It's important that any conflicts be - resolved in a way that itself avoids later merge conflicts, since @@ -191,7 +187,12 @@ mergeFrom branch = do resolveMerge :: Annex Bool resolveMerge = do top <- fromRepo Git.repoPath - all id <$> (mapM resolveMerge' =<< inRepo (LsFiles.unmerged [top])) + merged <- all id <$> (mapM resolveMerge' =<< inRepo (LsFiles.unmerged [top])) + when merged $ do + Annex.Queue.flush + void $ inRepo $ Git.Command.runBool "commit" + [Param "-m", Param "git-annex automatic merge resolution"] + return merged resolveMerge' :: LsFiles.Unmerged -> Annex Bool resolveMerge' u From 054ddda18a48abce03a1c0b50aef4eed714aa320 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 16:03:42 -0400 Subject: [PATCH 3947/8313] better filenames for conflict resolution files --- Command/Sync.hs | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index b146379d14..5e63ee63ad 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -30,6 +30,7 @@ import qualified Remote.Git import qualified Data.Map as M import qualified Data.ByteString.Lazy as L +import Data.Hash.MD5 def :: [Command] def = [command "sync" (paramOptional (paramRepeating paramRemote)) @@ -191,7 +192,7 @@ resolveMerge = do when merged $ do Annex.Queue.flush void $ inRepo $ Git.Command.runBool "commit" - [Param "-m", Param "git-annex automatic merge resolution"] + [Param "-m", Param "git-annex automatic merge conflict fix"] return merged resolveMerge' :: LsFiles.Unmerged -> Annex Bool @@ -206,7 +207,7 @@ resolveMerge' u makelink keyUs return True | otherwise = do - void $ liftIO $ tryIO $ removeFile file + liftIO $ nukeFile file Annex.Queue.addCommand "rm" [Params "--quiet -f --"] [file] makelink keyUs makelink keyThem @@ -215,9 +216,11 @@ resolveMerge' u issymlink select = any (select (LsFiles.unmergedBlobType u) ==) [Just SymlinkBlob, Nothing] makelink (Just key) = do - let dest = file ++ "." ++ show key + let dest = mergeFile file key l <- calcGitLink dest key - liftIO $ createSymbolicLink l dest + liftIO $ do + nukeFile dest + createSymbolicLink l dest Annex.Queue.addCommand "add" [Param "--force", Param "--"] [dest] makelink _ = noop withKey select a = do @@ -229,7 +232,35 @@ resolveMerge' u . encodeW8 . L.unpack <$> catObject sha maybe (return False) (a . Just) key - + +{- The filename to use when resolving a conflicted merge of a file, + - that points to a key. + - + - Something derived from the key needs to be included in the filename, + - but rather than exposing the whole key to the user, a very weak hash + - is used. There is a very real, although still unlikely, chance of + - conflicts using this hash. + - + - In the event that there is a conflict with the filename generated + - for some other key, that conflict will itself be handled by the + - conflicted merge resolution code. That case is detected, and the full + - key is used in the filename. + -} +mergeFile :: FilePath -> Key -> FilePath +mergeFile file key + | doubleconflict = go $ show key + | otherwise = go $ shortHash $ show key + where + vermarker = ".version-" + doubleconflict = vermarker `isSuffixOf` (dropExtension file) + go v = takeDirectory file + dropExtension (takeFileName file) + ++ vermarker ++ v + ++ takeExtension file + +shortHash :: String -> String +shortHash = take 4 . md5s . encodeFilePath + changed :: Remote -> Git.Ref -> Annex Bool changed remote b = do let r = remoteBranch remote b From 36ddb81df6938cd604ecccea52ae758f481fd79b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 16:09:17 -0400 Subject: [PATCH 3948/8313] use "variant" rather than "version" While this word may be less familiar to some users, it avoids the connotation that version 2 is better than version 1, which is wrong when the two variants were conflicting. --- Command/Sync.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index 5e63ee63ad..06e1fd5c92 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -251,11 +251,11 @@ mergeFile file key | doubleconflict = go $ show key | otherwise = go $ shortHash $ show key where - vermarker = ".version-" + varmarker = ".variant-" doubleconflict = vermarker `isSuffixOf` (dropExtension file) go v = takeDirectory file dropExtension (takeFileName file) - ++ vermarker ++ v + ++ varmarker ++ v ++ takeExtension file shortHash :: String -> String From e365b8300d8d301dfec358bf07496d1eeecad300 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 16:14:50 -0400 Subject: [PATCH 3949/8313] blog for the day --- .../assistant/blog/day_18__merging.mdwn | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 doc/design/assistant/blog/day_18__merging.mdwn diff --git a/doc/design/assistant/blog/day_18__merging.mdwn b/doc/design/assistant/blog/day_18__merging.mdwn new file mode 100644 index 0000000000..43b8f49581 --- /dev/null +++ b/doc/design/assistant/blog/day_18__merging.mdwn @@ -0,0 +1,82 @@ +Worked on automatic merge conflict resolution today. I had expected to be +able to use git's merge driver interface for this, but that interface is +not sufficient. There are two problems with it: + +1. The merge program is run when git is in the middle of an operation + that locks the index. So it cannot delete or stage files. I need to + do both as part of my conflict resolution strategy. +2. The merge program is not run at all when the merge conflict is caused + by one side deleting a file, and the other side modifying it. This is + an important case to handle. + +So, instead, git-annex will use a regular `git merge`, and if it fails, it +will fix up the conflicts. + +That presented its own difficully, of finding which files in the tree +conflict. `git ls-files --unmerged` is the way to do that, but its output +is a quite raw form: + + 120000 3594e94c04db171e2767224db355f514b13715c5 1 foo + 120000 35ec3b9d7586b46c0fd3450ba21e30ef666cfcd6 3 foo + 100644 1eabec834c255a127e2e835dadc2d7733742ed9a 2 bar + 100644 36902d4d842a114e8b8912c02d239b2d7059c02b 3 bar + +I had to stare at the rather inpenetrable documentation for hours and +write a lot of parsing and processing code to get from that to these mostly +self expanatory data types: + + data Conflicting v = Conflicting + { valUs :: Maybe v + , valThem :: Maybe v + } deriving (Show) + + data Unmerged = Unmerged + { unmergedFile :: FilePath + , unmergedBlobType :: Conflicting BlobType + , unmergedSha :: Conflicting Sha + } deriving (Show) + +Not the first time I've whined here about time spent parsing unix command +output, is it? :) + +From there, it was relatively easy to write the actual conflict cleanup +code, and make `git annex sync` use it. Here's how it looks: + + $ ls -1 + foo.png + bar.png + $ git annex sync + commit + # On branch master + nothing to commit (working directory clean) + ok + merge synced/master + CONFLICT (modify/delete): passwd deleted in refs/heads/synced/master and modified in HEAD. Version HEAD of passwd left in tree. + Automatic merge failed; fix conflicts and then commit the result. + passwd: needs merge + (Recording state in git...) + [master 0354a67] git-annex automatic merge conflict fix + ok + $ ls -1 + foo.png + bar.variant-a1fe.png + bar.variant-93a1.png + +There are very few options for ways for the conflict resolution code to +name conflicting variants of files. The conflict resolver can only use data +present in git to generate the names, because the same conflict needs to +be resolved the same everywhere. + +So I had to choose between using the full key name in the filenames produced +when resolving a merge, and using a shorter checksum of the key, that would be +more user-friendly, but could theoretically collide with another key. +I chose the checksum, and weakened it horribly by only using 32 bits of it! + +Surprisingly, I think this is a safe choice. The worst that can +happens if such a collision happens is another conflict, and the conflict +resolution code will work on conflicts produced by the conflict resolution +code! In such a case, it does fall back to putting the whole key in +the filename: +"bar.variant-SHA256-s2550--2c09deac21fa93607be0844fefa870b2878a304a7714684c4cc8f800fda5e16b.png" + +Still need to hook this code into `git annex assistant`. From 70329dc9802e28b7831a226277c5f33902f6e8f5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 16:19:23 -0400 Subject: [PATCH 3950/8313] typo --- doc/design/assistant/blog/day_18__merging.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design/assistant/blog/day_18__merging.mdwn b/doc/design/assistant/blog/day_18__merging.mdwn index 43b8f49581..44a79e14ff 100644 --- a/doc/design/assistant/blog/day_18__merging.mdwn +++ b/doc/design/assistant/blog/day_18__merging.mdwn @@ -51,9 +51,9 @@ code, and make `git annex sync` use it. Here's how it looks: nothing to commit (working directory clean) ok merge synced/master - CONFLICT (modify/delete): passwd deleted in refs/heads/synced/master and modified in HEAD. Version HEAD of passwd left in tree. + CONFLICT (modify/delete): bar.png deleted in refs/heads/synced/master and modified in HEAD. Version HEAD of bar.png left in tree. Automatic merge failed; fix conflicts and then commit the result. - passwd: needs merge + bar.png: needs merge (Recording state in git...) [master 0354a67] git-annex automatic merge conflict fix ok From 4e4c2498b801bb25a696d116388159747ea45b8f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 27 Jun 2012 20:51:19 +0000 Subject: [PATCH 3951/8313] --- ...with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn diff --git a/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn new file mode 100644 index 0000000000..3242b10776 --- /dev/null +++ b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn @@ -0,0 +1,11 @@ +Building commit 8baff14054e65ecbe801eb66786a55fa5245cb30 yields this... + + +
+[164 of 189] Compiling Command.Sync ( Command/Sync.hs, tmp/Command/Sync.o )
+Command/Sync.hs:268:34:
+Not in scope: `vermarker'
+Perhaps you meant `varmarker' (line 267)
+make: *** [git-annex] Error 1
+
+ From c6d595674bb1133c81d1c29a4ad637a7c162b3da Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Wed, 27 Jun 2012 20:56:33 +0000 Subject: [PATCH 3952/8313] --- ...ff14054e65ecbe801eb66786a55fa5245cb30.mdwn | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn index 3242b10776..015226f7d2 100644 --- a/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn +++ b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn @@ -9,3 +9,33 @@ Perhaps you meant `varmarker' (line 267) make: *** [git-annex] Error 1 +Supplied fix... + +
+
+From a23a1af99c7a95c316a87f9c6f5f67a6f8ff6937 Mon Sep 17 00:00:00 2001
+From: Jimmy Tang 
+Date: Wed, 27 Jun 2012 21:55:22 +0100
+Subject: [PATCH 14/14] fix build issue introduced in
+ 8baff14054e65ecbe801eb66786a55fa5245cb30
+
+---
+ Command/Sync.hs | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Command/Sync.hs b/Command/Sync.hs
+index b2bf24d..dfaed59 100644
+--- a/Command/Sync.hs
++++ b/Command/Sync.hs
+@@ -265,7 +265,7 @@ mergeFile file key
+        | otherwise = go $ shortHash $ show key
+        where
+                varmarker = ".variant-"
+-               doubleconflict = vermarker `isSuffixOf` (dropExtension file)
++               doubleconflict = varmarker `isSuffixOf` (dropExtension file)
+                go v = takeDirectory file
+                         dropExtension (takeFileName file)
+                        ++ varmarker ++ v
+-- 
+1.7.11.1
+
From 2d7ebc0582026bc1cadb8812861eb9a6467ba41d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 18:08:52 -0400 Subject: [PATCH 3953/8313] typo --- Command/Sync.hs | 2 +- ...ild_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Command/Sync.hs b/Command/Sync.hs index 06e1fd5c92..bdb5d47a76 100644 --- a/Command/Sync.hs +++ b/Command/Sync.hs @@ -252,7 +252,7 @@ mergeFile file key | otherwise = go $ shortHash $ show key where varmarker = ".variant-" - doubleconflict = vermarker `isSuffixOf` (dropExtension file) + doubleconflict = varmarker `isSuffixOf` (dropExtension file) go v = takeDirectory file dropExtension (takeFileName file) ++ varmarker ++ v diff --git a/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn index 015226f7d2..34c2eef5f0 100644 --- a/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn +++ b/doc/bugs/build_issue_with_8baff14054e65ecbe801eb66786a55fa5245cb30.mdwn @@ -39,3 +39,5 @@ index b2bf24d..dfaed59 100644 -- 1.7.11.1 + +[[fixed|done]] From 783bee285fd357b887de818918b181ea4628783a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 20:06:21 -0400 Subject: [PATCH 3954/8313] automatic conflict resolution for assistant --- Assistant/Threads/Merger.hs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index 77bf9f416e..3659588fcd 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -24,7 +24,7 @@ mergeThread st = do g <- runThreadState st $ fromRepo id let dir = Git.localGitDir g "refs" "heads" "synced" createDirectoryIfMissing True dir - let hook a = Just $ runHandler g a + let hook a = Just $ runHandler st g a let hooks = mkWatchHooks { addHook = hook onAdd , errHook = hook onErr @@ -32,21 +32,21 @@ mergeThread st = do watchDir dir (const False) hooks id where -type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO () +type Handler = ThreadState -> Git.Repo -> FilePath -> Maybe FileStatus -> IO () {- Runs an action handler. - - Exceptions are ignored, otherwise a whole thread could be crashed. -} -runHandler :: Git.Repo -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler g handler file filestatus = void $ do +runHandler :: ThreadState -> Git.Repo -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler st g handler file filestatus = void $ do either print (const noop) =<< tryIO go where - go = handler g file filestatus + go = handler st g file filestatus {- Called when there's an error with inotify. -} onErr :: Handler -onErr _ msg _ = error msg +onErr _ _ msg _ = error msg {- Called when a new branch ref is written. - @@ -60,16 +60,21 @@ onErr _ msg _ = error msg - ran are merged in. -} onAdd :: Handler -onAdd g file _ +onAdd st g file _ | ".lock" `isSuffixOf` file = noop | otherwise = do - let branch = Git.Ref $ "refs" "heads" takeFileName file + let changedbranch = Git.Ref $ + "refs" "heads" takeFileName file current <- Git.Branch.current g - when (Just branch == current) $ - void $ mergeBranch branch g + when (Just changedbranch == current) $ + void $ mergeBranch st changedbranch g -mergeBranch :: Git.Ref -> Git.Repo -> IO Bool -mergeBranch = Git.Merge.mergeNonInteractive . Command.Sync.syncBranch +mergeBranch :: ThreadState -> Git.Ref -> Git.Repo -> IO Bool +mergeBranch st branch repo = do + ok <- Git.Merge.mergeNonInteractive (Command.Sync.syncBranch branch) repo + if ok + then return ok + else runThreadState st Command.Sync.resolveMerge {- Manually pull from remotes and merge their branches. Called by the pusher - when a push fails, which can happen due to a remote not having pushed From 59b5266ad13efe465b67b31aba3b750c31fb83cd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 20:30:04 -0400 Subject: [PATCH 3955/8313] actually fetch from remote when doing a manual pull forgot to do this --- Assistant/Threads/Merger.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index 3659588fcd..602bebb5b0 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -12,9 +12,11 @@ import Assistant.ThreadedMonad import Utility.DirWatcher import Utility.Types.DirWatcher import qualified Git +import qualified Git.Command import qualified Git.Merge import qualified Git.Branch import qualified Command.Sync +import qualified Remote {- This thread watches for changes to .git/refs/heads/synced/*, - which indicate incoming pushes. It merges those pushes into the @@ -81,5 +83,6 @@ mergeBranch st branch repo = do - changes to us. That could be because it doesn't have us as a remote, or - because the assistant is not running there, or other reasons. -} manualPull :: Git.Ref -> [Remote] -> Annex () -manualPull currentbranch remotes = forM_ remotes $ \r -> +manualPull currentbranch remotes = forM_ remotes $ \r -> do + void $ inRepo $ Git.Command.runBool "fetch" [Param $ Remote.name r] Command.Sync.mergeRemote r currentbranch From fb51d9995193b2e15f3e5174783347ec14dbaa28 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 20:50:50 -0400 Subject: [PATCH 3956/8313] merge conflict resolution now working Avoid MVar deadlock issue, which I don't understand. Have not taken the time to debug it fully, because it turns out I don't need to resolve merge conflicts when a new branch ref is written... I think. Ensure the git-annex branch is merged when doing a manual pull. Otherwise it can get out of sync, since git-annex normally only merges it once per run. --- Assistant/Threads/Merger.hs | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index 602bebb5b0..d643f16943 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -11,6 +11,7 @@ import Common.Annex import Assistant.ThreadedMonad import Utility.DirWatcher import Utility.Types.DirWatcher +import qualified Annex.Branch import qualified Git import qualified Git.Command import qualified Git.Merge @@ -26,7 +27,10 @@ mergeThread st = do g <- runThreadState st $ fromRepo id let dir = Git.localGitDir g "refs" "heads" "synced" createDirectoryIfMissing True dir - let hook a = Just $ runHandler st g a + let hook a = Just $ runHandler g a + -- XXX: For reasons currently unknown, using the ThreadState + -- inside the watch hooks leads to a MVar deadlock. + -- Luckily, we don't currently need to do that. let hooks = mkWatchHooks { addHook = hook onAdd , errHook = hook onErr @@ -34,21 +38,21 @@ mergeThread st = do watchDir dir (const False) hooks id where -type Handler = ThreadState -> Git.Repo -> FilePath -> Maybe FileStatus -> IO () +type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO () {- Runs an action handler. - - Exceptions are ignored, otherwise a whole thread could be crashed. -} -runHandler :: ThreadState -> Git.Repo -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler st g handler file filestatus = void $ do +runHandler :: Git.Repo -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler g handler file filestatus = void $ do either print (const noop) =<< tryIO go where - go = handler st g file filestatus + go = handler g file filestatus {- Called when there's an error with inotify. -} onErr :: Handler -onErr _ _ msg _ = error msg +onErr _ msg _ = error msg {- Called when a new branch ref is written. - @@ -62,27 +66,26 @@ onErr _ _ msg _ = error msg - ran are merged in. -} onAdd :: Handler -onAdd st g file _ +onAdd g file _ | ".lock" `isSuffixOf` file = noop | otherwise = do let changedbranch = Git.Ref $ "refs" "heads" takeFileName file current <- Git.Branch.current g when (Just changedbranch == current) $ - void $ mergeBranch st changedbranch g + void $ mergeBranch changedbranch g -mergeBranch :: ThreadState -> Git.Ref -> Git.Repo -> IO Bool -mergeBranch st branch repo = do - ok <- Git.Merge.mergeNonInteractive (Command.Sync.syncBranch branch) repo - if ok - then return ok - else runThreadState st Command.Sync.resolveMerge +mergeBranch :: Git.Ref -> Git.Repo -> IO Bool +mergeBranch = Git.Merge.mergeNonInteractive . Command.Sync.syncBranch {- Manually pull from remotes and merge their branches. Called by the pusher - when a push fails, which can happen due to a remote not having pushed - changes to us. That could be because it doesn't have us as a remote, or - because the assistant is not running there, or other reasons. -} manualPull :: Git.Ref -> [Remote] -> Annex () -manualPull currentbranch remotes = forM_ remotes $ \r -> do - void $ inRepo $ Git.Command.runBool "fetch" [Param $ Remote.name r] - Command.Sync.mergeRemote r currentbranch +manualPull currentbranch remotes = do + forM_ remotes $ \r -> + inRepo $ Git.Command.runBool "fetch" [Param $ Remote.name r] + Annex.Branch.forceUpdate + forM_ remotes $ \r -> + Command.Sync.mergeRemote r currentbranch From 1f09ae686ef35f8fd2d973754f8e1efd99161f4a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 27 Jun 2012 21:11:39 -0400 Subject: [PATCH 3957/8313] update --- doc/design/assistant/inotify.mdwn | 11 +++++++++++ doc/design/assistant/syncing.mdwn | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn index 47b8c84a34..f783f9a7df 100644 --- a/doc/design/assistant/inotify.mdwn +++ b/doc/design/assistant/inotify.mdwn @@ -125,6 +125,17 @@ Many races need to be dealt with by this code. Here are some of them. Not a problem; The removal event removes the old file from the index, and the add event adds the new one. +* Symlink appears, but is then deleted before it can be processed. + + Leads to an ugly message, otherwise no problem: + + ./me: readSymbolicLink: does not exist (No such file or directory) + + Here `me` is a file that was in a conflicted merge, which got + removed as part of the resolution. This is probably coming from the watcher + thread, which sees the newly added symlink (created by the git merge), + but finds it deleted (by the conflict resolver) by the time it processes it. + ## done - on startup, add any files that have appeared since last run **done** diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 3e90e6b105..8b681ac100 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -17,7 +17,7 @@ all the other git clones, at both the git level and the key/value level. 1. Also, detect if a push failed due to not being up-to-date, pull, and repush. **done** 2. Use a git merge driver that adds both conflicting files, - so conflicts never break a sync. + so conflicts never break a sync. **done** 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of signaling a change out of band. 4. Add a hook, so when there's a change to sync, a program can be run From b2327f04c6484d47ad2bf7194934af956fe9e953 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmURXBzaYE1gmVc-X9eLAyDat_6rHPl670" Date: Thu, 28 Jun 2012 12:37:29 +0000 Subject: [PATCH 3958/8313] --- doc/bugs/watcher_commits_unlocked_files.mdwn | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 doc/bugs/watcher_commits_unlocked_files.mdwn diff --git a/doc/bugs/watcher_commits_unlocked_files.mdwn b/doc/bugs/watcher_commits_unlocked_files.mdwn new file mode 100644 index 0000000000..b807593763 --- /dev/null +++ b/doc/bugs/watcher_commits_unlocked_files.mdwn @@ -0,0 +1,43 @@ +When having "git annex watch" running, unlocking files causes the watcher to immediately lock/commit them. Observe: + + bram@falafel% git annex unlock + unlock 01 - Crunchy Joe (featuring Sakhile Moleshe).flac (copying...) ok + unlock 02 - Get Busy Living (featuring Emily Bruce).flac (copying...) ok + unlock 03 - Show You How.flac (copying...) ok + unlock 04 - Call Me (featuring Monique Hellenberg).flac (copying...) ok + unlock 05 - Humbug (featuring Sakhile Moleshe).flac (copying...) ok + unlock 06 - Brush Your Hair.flac (copying...) ok + unlock 07 - We Come Together (featuring Sakhile Moleshe).flac (copying...) ok + unlock 08 - In Too Deep (featuring Emily Bruce).flac (copying...) ok + unlock 09 - My Rainbow.flac (copying...) ok + unlock 10 - Big Band Wolf.flac (copying...) ok + (Recording state in git...) + bram@falafel% ls -l 01\ -\ Crunchy\ Joe\ \(featuring\ Sakhile\ Moleshe\).flac + lrwxrwxrwx 1 bram bram 208 Jul 18 2011 01 - Crunchy Joe (featuring Sakhile Moleshe).flac -> ../../.git/annex/objects/KX/15/SHA256E-s23981083--5ffd30042e313f8e10cf51ded59c369dd03a600fa3b8c13962f833694af449b5.flac/SHA256E-s23981083--5ffd30042e313f8e10cf51ded59c369dd03a600fa3b8c13962f833694af449b5.flac + bram@falafel% tail ~/Media/.git/annex/daemon.log + add ./Uncategorized/Goldfish - Get Busy Living (2010)/04 - Call Me (featuring Monique Hellenberg).flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/03 - Show You How.flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/02 - Get Busy Living (featuring Emily Bruce).flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/10 - Big Band Wolf.flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/09 - My Rainbow.flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/08 - In Too Deep (featuring Emily Bruce).flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/07 - We Come Together (featuring Sakhile Moleshe).flac (checksum...) ok + add ./Uncategorized/Goldfish - Get Busy Living (2010)/06 - Brush Your Hair.flac (checksum...) ok + (Recording state in git...) + (Recording state in git...) + bram@falafel% git annex watch --stop + bram@falafel% git annex unlock + unlock 01 - Crunchy Joe (featuring Sakhile Moleshe).flac (copying...) ok + unlock 02 - Get Busy Living (featuring Emily Bruce).flac (copying...) ok + unlock 03 - Show You How.flac (copying...) ok + unlock 04 - Call Me (featuring Monique Hellenberg).flac (copying...) ok + unlock 05 - Humbug (featuring Sakhile Moleshe).flac (copying...) ok + unlock 06 - Brush Your Hair.flac (copying...) ok + unlock 07 - We Come Together (featuring Sakhile Moleshe).flac (copying...) ok + unlock 08 - In Too Deep (featuring Emily Bruce).flac (copying...) ok + unlock 09 - My Rainbow.flac (copying...) ok + unlock 10 - Big Band Wolf.flac (copying...) ok + bram@falafel% ls -l 01\ -\ Crunchy\ Joe\ \(featuring\ Sakhile\ Moleshe\).flac + -rw-r--r-- 1 bram bram 23981083 Jul 18 2011 01 - Crunchy Joe (featuring Sakhile Moleshe).flac + +This is using git-annex 3.20120624 on Ubuntu, compiled with cabal (I upgraded my libghc-stm-dev package, as you mentioned in another bug, to get the watch command working on this version). From e4596a133e1c6781bd8dd369448f11dc602d0d28 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 28 Jun 2012 13:39:19 +0000 Subject: [PATCH 3959/8313] Added a comment --- .../comment_1_f70e1912fde0eee59e208307df06b503._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/watcher_commits_unlocked_files/comment_1_f70e1912fde0eee59e208307df06b503._comment diff --git a/doc/bugs/watcher_commits_unlocked_files/comment_1_f70e1912fde0eee59e208307df06b503._comment b/doc/bugs/watcher_commits_unlocked_files/comment_1_f70e1912fde0eee59e208307df06b503._comment new file mode 100644 index 0000000000..a06b8fe822 --- /dev/null +++ b/doc/bugs/watcher_commits_unlocked_files/comment_1_f70e1912fde0eee59e208307df06b503._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 1" + date="2012-06-28T13:39:18Z" + content=""" +That is a known problem/bug which is listed at [[design/assistant/inotify]] +"""]] From 40f357fdcf07a9b9844e675fe478ab08f5c1bae8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 13:04:02 -0400 Subject: [PATCH 3960/8313] tweak --- Assistant.hs | 2 +- Assistant/Threads/Merger.hs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 4f8a868f4f..b61270613c 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -40,7 +40,7 @@ - - ThreadState: (MVar) - The Annex state is stored here, which allows resuscitating the - - Annex monad in IO actions run by the inotify and committer + - Annex monad in IO actions run by the watcher and committer - threads. Thus, a single state is shared amoung the threads, and - only one at a time can access it. - DaemonStatusHandle: (MVar) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index d643f16943..de172e8da0 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -36,7 +36,6 @@ mergeThread st = do , errHook = hook onErr } watchDir dir (const False) hooks id - where type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO () From 4888c5b0422c8006b4c178503b24bced733931fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 13:37:03 -0400 Subject: [PATCH 3961/8313] improve thread termination handling The reason the DirWatcher had to wait for program termination was because it used withINotify, so when it finished, its watcher threads were killed. But since I have two DirWatcher threads now, that was not good, and could perhaps explain the MVar problem I saw yesterday. In any case, fixed this part of the code by making the DirWatcher return a handle that can be used to stop it, and now the main Assistant thread is the only one calling waitForTermination. --- Assistant.hs | 5 +++-- Assistant/Threads/Merger.hs | 2 +- Assistant/Threads/Watcher.hs | 2 +- Utility/DirWatcher.hs | 35 ++++++++++++++++++++++++++++------- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index b61270613c..2a11741b4d 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -73,6 +73,7 @@ import Assistant.Threads.Merger import Assistant.Threads.SanityChecker import qualified Utility.Daemon import Utility.LogFile +import Utility.ThreadScheduler import Control.Concurrent @@ -99,8 +100,8 @@ startDaemon assistant foreground _ <- forkIO $ mergeThread st _ <- forkIO $ daemonStatusThread st dstatus _ <- forkIO $ sanityCheckerThread st dstatus changechan - -- Does not return. - watchThread st dstatus changechan + _ <- forkIO $ watchThread st dstatus changechan + waitForTermination stopDaemon :: Annex () stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index de172e8da0..5d24d1862b 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -35,7 +35,7 @@ mergeThread st = do { addHook = hook onAdd , errHook = hook onErr } - watchDir dir (const False) hooks id + void $ watchDir dir (const False) hooks id type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO () diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 1b6ec15f18..e250f4b4a6 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -46,7 +46,7 @@ needLsof = error $ unlines ] watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () -watchThread st dstatus changechan = watchDir "." ignored hooks startup +watchThread st dstatus changechan = void $ watchDir "." ignored hooks startup where startup = statupScan st dstatus hook a = Just $ runHandler st dstatus changechan a diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs index 11ce7baef1..93c3ecb026 100644 --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -17,7 +17,6 @@ import Utility.Types.DirWatcher #if WITH_INOTIFY import qualified Utility.INotify as INotify import qualified System.INotify as INotify -import Utility.ThreadScheduler #endif #if WITH_KQUEUE import qualified Utility.Kqueue as Kqueue @@ -72,19 +71,41 @@ closingTracked = undefined #endif #endif +/* Starts a watcher thread. The runStartup action is passed a scanner action + * to run, that will return once the initial directory scan is complete. + * Once runStartup returns, the watcher thread continues running, + * and processing events. Returns a DirWatcherHandle that can be used + * to shutdown later. */ #if WITH_INOTIFY -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO () -watchDir dir prune hooks runstartup = INotify.withINotify $ \i -> do +type DirWatcherHandle = INotify.INotify +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle +watchDir dir prune hooks runstartup = do + i <- INotify.initINotify runstartup $ INotify.watchDir i dir prune hooks - waitForTermination -- Let the inotify thread run. + return i #else #if WITH_KQUEUE -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO () +type DirWatcherHandle = ThreadID +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle watchDir dir ignored hooks runstartup = do kq <- runstartup $ Kqueue.initKqueue dir ignored - Kqueue.runHooks kq hooks + forkIO $ Kqueue.runHooks kq hooks #else -watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO () +type DirWatcherHandle = () +watchDir :: FilePath -> Pruner -> WatchHooks -> (IO () -> IO ()) -> IO DirWatcherHandle watchDir = undefined #endif #endif + +#if WITH_INOTIFY +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = INotify.killINotify +#else +#if WITH_KQUEUE +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = killThread +#else +stopWatchDir :: DirWatcherHandle -> IO () +stopWatchDir = undefined +#endif +#endif From a3636602ab5b33bf25cef760d4780794841bc8e6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 13:47:15 -0400 Subject: [PATCH 3962/8313] MVar deadlock problem seems to be fixed by previous commit --- Assistant/Threads/Merger.hs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index 5d24d1862b..c7da86a8d3 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -28,9 +28,6 @@ mergeThread st = do let dir = Git.localGitDir g "refs" "heads" "synced" createDirectoryIfMissing True dir let hook a = Just $ runHandler g a - -- XXX: For reasons currently unknown, using the ThreadState - -- inside the watch hooks leads to a MVar deadlock. - -- Luckily, we don't currently need to do that. let hooks = mkWatchHooks { addHook = hook onAdd , errHook = hook onErr From 343ecf999a1ecb700ba2973763fc9237576dcc1c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 14:00:25 -0400 Subject: [PATCH 3963/8313] post my current set of ideas for handling unlocking --- doc/bugs/watcher_commits_unlocked_files.mdwn | 67 ++++++++------------ 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/doc/bugs/watcher_commits_unlocked_files.mdwn b/doc/bugs/watcher_commits_unlocked_files.mdwn index b807593763..ef64921f1a 100644 --- a/doc/bugs/watcher_commits_unlocked_files.mdwn +++ b/doc/bugs/watcher_commits_unlocked_files.mdwn @@ -1,43 +1,28 @@ -When having "git annex watch" running, unlocking files causes the watcher to immediately lock/commit them. Observe: +When having "git annex watch" running, unlocking files causes the watcher +to immediately lock/commit them. - bram@falafel% git annex unlock - unlock 01 - Crunchy Joe (featuring Sakhile Moleshe).flac (copying...) ok - unlock 02 - Get Busy Living (featuring Emily Bruce).flac (copying...) ok - unlock 03 - Show You How.flac (copying...) ok - unlock 04 - Call Me (featuring Monique Hellenberg).flac (copying...) ok - unlock 05 - Humbug (featuring Sakhile Moleshe).flac (copying...) ok - unlock 06 - Brush Your Hair.flac (copying...) ok - unlock 07 - We Come Together (featuring Sakhile Moleshe).flac (copying...) ok - unlock 08 - In Too Deep (featuring Emily Bruce).flac (copying...) ok - unlock 09 - My Rainbow.flac (copying...) ok - unlock 10 - Big Band Wolf.flac (copying...) ok - (Recording state in git...) - bram@falafel% ls -l 01\ -\ Crunchy\ Joe\ \(featuring\ Sakhile\ Moleshe\).flac - lrwxrwxrwx 1 bram bram 208 Jul 18 2011 01 - Crunchy Joe (featuring Sakhile Moleshe).flac -> ../../.git/annex/objects/KX/15/SHA256E-s23981083--5ffd30042e313f8e10cf51ded59c369dd03a600fa3b8c13962f833694af449b5.flac/SHA256E-s23981083--5ffd30042e313f8e10cf51ded59c369dd03a600fa3b8c13962f833694af449b5.flac - bram@falafel% tail ~/Media/.git/annex/daemon.log - add ./Uncategorized/Goldfish - Get Busy Living (2010)/04 - Call Me (featuring Monique Hellenberg).flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/03 - Show You How.flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/02 - Get Busy Living (featuring Emily Bruce).flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/10 - Big Band Wolf.flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/09 - My Rainbow.flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/08 - In Too Deep (featuring Emily Bruce).flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/07 - We Come Together (featuring Sakhile Moleshe).flac (checksum...) ok - add ./Uncategorized/Goldfish - Get Busy Living (2010)/06 - Brush Your Hair.flac (checksum...) ok - (Recording state in git...) - (Recording state in git...) - bram@falafel% git annex watch --stop - bram@falafel% git annex unlock - unlock 01 - Crunchy Joe (featuring Sakhile Moleshe).flac (copying...) ok - unlock 02 - Get Busy Living (featuring Emily Bruce).flac (copying...) ok - unlock 03 - Show You How.flac (copying...) ok - unlock 04 - Call Me (featuring Monique Hellenberg).flac (copying...) ok - unlock 05 - Humbug (featuring Sakhile Moleshe).flac (copying...) ok - unlock 06 - Brush Your Hair.flac (copying...) ok - unlock 07 - We Come Together (featuring Sakhile Moleshe).flac (copying...) ok - unlock 08 - In Too Deep (featuring Emily Bruce).flac (copying...) ok - unlock 09 - My Rainbow.flac (copying...) ok - unlock 10 - Big Band Wolf.flac (copying...) ok - bram@falafel% ls -l 01\ -\ Crunchy\ Joe\ \(featuring\ Sakhile\ Moleshe\).flac - -rw-r--r-- 1 bram bram 23981083 Jul 18 2011 01 - Crunchy Joe (featuring Sakhile Moleshe).flac +---- -This is using git-annex 3.20120624 on Ubuntu, compiled with cabal (I upgraded my libghc-stm-dev package, as you mentioned in another bug, to get the watch command working on this version). +Possible approaches: + +* The watcher could detect unlocked files by checking if newly added files + are a typechange of a file already in git. But this would add git overhead + to every file add. +* `git annex unlock` could add some type of flag file, which the assistant + could check. This would work fine, for users who want to use `git annex + unlock` with the assistant. That's probably not simple enough for most + users, though. +* There could be a UI in the assistant to pick a file and unlock it. + The assistant would have its own list of files it knows are unlocked. + But I'm trying to avoid mandatory UI to use the assistant. +* Perhaps instead, have a directory, like "edit". The assistant could notice + when files move into this special directory, and automatically unlock them. + Then when they're moved out, automatically commit them. +* Alternatively, files that are moved out of the repository entirely could be + automatically unlocked, and then when they're moved back in, it would + automatically do the right thing. This may be worth implementing in + combination with the "edit" directory, as different use cases would work + better with one or the other. However, I don't currently get inotify + events when files are moved out of the repository (well, I do, but it + just says "file moved", with no forwarding address, so I don't know + how to find the file to unlock it. From 6cc3eb97dbd665bdaedf0a28f315d62c169dbe9d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 14:06:22 -0400 Subject: [PATCH 3964/8313] update --- doc/design/assistant/inotify.mdwn | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn index f783f9a7df..7b600090ad 100644 --- a/doc/design/assistant/inotify.mdwn +++ b/doc/design/assistant/inotify.mdwn @@ -8,13 +8,15 @@ available! * If a file is checked into git as a normal file and gets modified (or merged, etc), it will be converted into an annexed file. - See [[blog/day_7__bugfixes]] + See [[blog/day_7__bugfixes]]. * When you `git annex unlock` a file, it will immediately be re-locked. + See [[bugs/watcher_commits_unlocked_files]]. * Kqueue has to open every directory it watches, so too many directories will run it out of the max number of open files (typically 1024), and fail. I may need to fork off multiple watcher processes to handle this. + See [[bugs/Issue_on_OSX_with_some_system_limits]]. ## beyond Linux @@ -42,6 +44,8 @@ I'd also like to support OSX and if possible the BSDs. * [man page](http://www.freebsd.org/cgi/man.cgi?query=kqueue&apropos=0&sektion=0&format=html) * (good example program) + *kqueue is now supported* + * hfsevents ([haskell bindings](http://hackage.haskell.org/package/hfsevents)) is OSX specific. @@ -71,9 +75,6 @@ I'd also like to support OSX and if possible the BSDs. - honor .gitignore, not adding files it excludes (difficult, probably needs my own .gitignore parser to avoid excessive running of git commands to check for ignored files) -- Possibly, when a directory is moved out of the annex location, - unannex its contents. (Does inotify tell us where the directory moved - to so we can access it?) ## the races From 421f9ce0e26936e384700c9cb5e202191cc92d1c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 14:13:15 -0400 Subject: [PATCH 3965/8313] fix kqueue build --- Utility/DirWatcher.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs index 93c3ecb026..5e76e780c8 100644 --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -20,6 +20,7 @@ import qualified System.INotify as INotify #endif #if WITH_KQUEUE import qualified Utility.Kqueue as Kqueue +import Control.Concurrent #endif type Pruner = FilePath -> Bool From 638a321ca504e24809c85e24583ae06cd5f7de8f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 14:15:49 -0400 Subject: [PATCH 3966/8313] typo --- Utility/DirWatcher.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility/DirWatcher.hs b/Utility/DirWatcher.hs index 5e76e780c8..213aeb50a7 100644 --- a/Utility/DirWatcher.hs +++ b/Utility/DirWatcher.hs @@ -86,7 +86,7 @@ watchDir dir prune hooks runstartup = do return i #else #if WITH_KQUEUE -type DirWatcherHandle = ThreadID +type DirWatcherHandle = ThreadId watchDir :: FilePath -> Pruner -> WatchHooks -> (IO Kqueue.Kqueue -> IO Kqueue.Kqueue) -> IO DirWatcherHandle watchDir dir ignored hooks runstartup = do kq <- runstartup $ Kqueue.initKqueue dir ignored From c7328556a3d3aa504c68c612f2c19a1dd49a4e25 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 14:48:46 -0400 Subject: [PATCH 3967/8313] update --- doc/design/assistant/syncing.mdwn | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 3e90e6b105..50e6fb4f18 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -17,12 +17,18 @@ all the other git clones, at both the git level and the key/value level. 1. Also, detect if a push failed due to not being up-to-date, pull, and repush. **done** 2. Use a git merge driver that adds both conflicting files, - so conflicts never break a sync. + so conflicts never break a sync. **done** 3. Investigate the XMPP approach like dvcs-autosync does, or other ways of signaling a change out of band. 4. Add a hook, so when there's a change to sync, a program can be run and do its own signaling. +## misc todo + +* --debug will show often unnecessary work being done. Optimise. +* It would be nice if, when a USB drive is connected, + syncing starts automatically. Use dbus on Linux? + ## data syncing There are two parts to data syncing. First, map the network and second, @@ -41,8 +47,5 @@ This probably will need lots of refinements to get working well. ## other considerations -It would be nice if, when a USB drive is connected, -syncing starts automatically. Use dbus on Linux? - This assumes the network is connected. It's often not, so the [[cloud]] needs to be used to bridge between LANs. From 39569e889a5cdef913cd3927ee0383eafac77190 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 18:22:43 -0400 Subject: [PATCH 3968/8313] blog for the day --- .../blog/day_19__random_improvements.mdwn | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 doc/design/assistant/blog/day_19__random_improvements.mdwn diff --git a/doc/design/assistant/blog/day_19__random_improvements.mdwn b/doc/design/assistant/blog/day_19__random_improvements.mdwn new file mode 100644 index 0000000000..93c1296bab --- /dev/null +++ b/doc/design/assistant/blog/day_19__random_improvements.mdwn @@ -0,0 +1,50 @@ +Random improvements day.. + +Got the merge conflict resolution code working in `git annex assistant`. + +Did some more fixes to the pushing and pulling code, covering some cases +I missed earlier. + +Git syncing seems to work well for me now; I've seen it recover +from a variety of error conditions, including merge conflicts and repos +that were temporarily unavailable. + +---- + +There is definitely a MVar deadlock if the merger thread's inotify event +handler tries to run code in the Annex monad. Luckily, it doesn't +currently seem to need to do that, so I have put off debugging what's going +on there. + +Reworked how the inotify thread runs, to avoid the two inotify threads +in the assistant now from both needing to wait for program termination, +in a possibly conflicting manner. + +Hmm, that *seems* to have fixed the MVar deadlock problem. + +---- + +Been thinking about how to fix [[bugs/watcher_commits_unlocked_files]]. +Posted some thoughts there. + +It's about time to move on to data [[syncing]]. While eventually that will +need to build a map of the repo network to efficiently sync data over the +fastest paths, I'm thinking that I'll first write a dumb version. So, two +more threads: + +1. Uploads new data to every configured remote. Triggered by the watcher + thread when it adds content. Easy; just use a `TSet` of Keys to send. + +2. Downloads new data from the cheapest remote that has it. COuld be + triggered by the + merger thread, after it merges in a git sync. Rather hard; how does it + work out what new keys are in the tree without scanning it all? Scan + through the git history to find newly created files? Maybe the watcher + triggers this thread instead, when it sees a new symlink, without data, + appear. + +Both threads will need to be able to be stopped, and restarted, as needed +to control the data transfer. And a lot of other control smarts will +eventually be needed, but my first pass will be to do a straightforward +implementation. Once it's done, the git annex assistant will be basically +usable. From cd0ab91c91e84b726dbc3da39e57893bd1417ee9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 18:22:43 -0400 Subject: [PATCH 3969/8313] blog for the day --- .../blog/day_19__random_improvements.mdwn | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 doc/design/assistant/blog/day_19__random_improvements.mdwn diff --git a/doc/design/assistant/blog/day_19__random_improvements.mdwn b/doc/design/assistant/blog/day_19__random_improvements.mdwn new file mode 100644 index 0000000000..93c1296bab --- /dev/null +++ b/doc/design/assistant/blog/day_19__random_improvements.mdwn @@ -0,0 +1,50 @@ +Random improvements day.. + +Got the merge conflict resolution code working in `git annex assistant`. + +Did some more fixes to the pushing and pulling code, covering some cases +I missed earlier. + +Git syncing seems to work well for me now; I've seen it recover +from a variety of error conditions, including merge conflicts and repos +that were temporarily unavailable. + +---- + +There is definitely a MVar deadlock if the merger thread's inotify event +handler tries to run code in the Annex monad. Luckily, it doesn't +currently seem to need to do that, so I have put off debugging what's going +on there. + +Reworked how the inotify thread runs, to avoid the two inotify threads +in the assistant now from both needing to wait for program termination, +in a possibly conflicting manner. + +Hmm, that *seems* to have fixed the MVar deadlock problem. + +---- + +Been thinking about how to fix [[bugs/watcher_commits_unlocked_files]]. +Posted some thoughts there. + +It's about time to move on to data [[syncing]]. While eventually that will +need to build a map of the repo network to efficiently sync data over the +fastest paths, I'm thinking that I'll first write a dumb version. So, two +more threads: + +1. Uploads new data to every configured remote. Triggered by the watcher + thread when it adds content. Easy; just use a `TSet` of Keys to send. + +2. Downloads new data from the cheapest remote that has it. COuld be + triggered by the + merger thread, after it merges in a git sync. Rather hard; how does it + work out what new keys are in the tree without scanning it all? Scan + through the git history to find newly created files? Maybe the watcher + triggers this thread instead, when it sees a new symlink, without data, + appear. + +Both threads will need to be able to be stopped, and restarted, as needed +to control the data transfer. And a lot of other control smarts will +eventually be needed, but my first pass will be to do a straightforward +implementation. Once it's done, the git annex assistant will be basically +usable. From 247099f6282262cb72027aeeadce2e7bb8073eef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 20:01:03 -0400 Subject: [PATCH 3970/8313] refactor --- Assistant.hs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 2a11741b4d..a077cf10f6 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -87,21 +87,24 @@ startDaemon assistant foreground pidfile <- fromRepo gitAnnexPidFile go $ Utility.Daemon.daemonize logfd (Just pidfile) False where - go a = withThreadState $ \st -> do + go daemonize = withThreadState $ \st -> do checkCanWatch dstatus <- startDaemonStatus - liftIO $ a $ do - changechan <- newChangeChan - commitchan <- newCommitChan - pushmap <- newFailedPushMap - _ <- forkIO $ commitThread st changechan commitchan - _ <- forkIO $ pushThread st commitchan pushmap - _ <- forkIO $ pushRetryThread st pushmap - _ <- forkIO $ mergeThread st - _ <- forkIO $ daemonStatusThread st dstatus - _ <- forkIO $ sanityCheckerThread st dstatus changechan - _ <- forkIO $ watchThread st dstatus changechan - waitForTermination + liftIO $ daemonize $ run dstatus st + run dstatus st = do + changechan <- newChangeChan + commitchan <- newCommitChan + pushmap <- newFailedPushMap + mapM_ (void . forkIO) + [ commitThread st changechan commitchan + , pushThread st commitchan pushmap + , pushRetryThread st pushmap + , mergeThread st + , daemonStatusThread st dstatus + , sanityCheckerThread st dstatus changechan + , watchThread st dstatus changechan + ] + waitForTermination stopDaemon :: Annex () stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile From 397117429c8824bad7e994454a1d9b8e6f4b3b96 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 28 Jun 2012 23:40:16 -0400 Subject: [PATCH 3971/8313] simplify modifyMVar_ catches exceptions, so no need to roll my own --- Assistant/ThreadedMonad.hs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 6d3d25778e..7b915e12c8 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -5,15 +5,13 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE BangPatterns #-} - module Assistant.ThreadedMonad where import Common.Annex import qualified Annex import Control.Concurrent -import Control.Exception (throw) +import Data.Tuple {- The Annex state is stored in a MVar, so that threaded actions can access - it. -} @@ -37,11 +35,4 @@ withThreadState a = do - This serializes calls by threads; only one thread can run in Annex at a - time. -} runThreadState :: ThreadState -> Annex a -> IO a -runThreadState mvar a = do - startstate <- takeMVar mvar - -- catch IO errors and rethrow after restoring the MVar - !(r, newstate) <- catchIO (Annex.run startstate a) $ \e -> do - putMVar mvar startstate - throw e - putMVar mvar newstate - return r +runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a From 6b84f23317b77a4caf923fb9ab907e39e8cc926d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 29 Jun 2012 12:02:49 +0000 Subject: [PATCH 3972/8313] Added a comment --- ...mment_2_b14e697c211843163285aaa8de5bf4c6._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment new file mode 100644 index 0000000000..17dcf76343 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_2_b14e697c211843163285aaa8de5bf4c6._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" + nickname="Jimmy" + subject="comment 2" + date="2012-06-29T12:02:48Z" + content=""" +Doing, + + sudo sysctl -w kern.maxfilesperproc=400000 + +Somewhat works for me, git-annex watch at least starts up and takes a while to scan the directory, but it's not ideal. Also, creating files seems to work okay, when I remove a file the changes don't seem to get pushed across my other repos, running a sync on the remote repo fixes things. +"""]] From 29335bf32685ee665f9ec5acbcfe7f8edabd1b96 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jun 2012 10:00:05 -0400 Subject: [PATCH 3973/8313] pointlessness --- Annex.hs | 2 +- Common.hs | 1 + Logs/Presence.hs | 2 +- Remote/Directory.hs | 2 +- Utility/Applicative.hs | 16 ++++++++++++++++ 5 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 Utility/Applicative.hs diff --git a/Annex.hs b/Annex.hs index 38168334dd..32edeff5c1 100644 --- a/Annex.hs +++ b/Annex.hs @@ -128,7 +128,7 @@ newState gitrepo = AnnexState {- Makes an Annex state object for the specified git repo. - Ensures the config is read, if it was not already. -} new :: Git.Repo -> IO AnnexState -new gitrepo = newState <$> Git.Config.read gitrepo +new = newState <$$> Git.Config.read {- performs an action in the Annex monad -} run :: AnnexState -> Annex a -> IO (a, AnnexState) diff --git a/Common.hs b/Common.hs index 3475024601..7f07781ce9 100644 --- a/Common.hs +++ b/Common.hs @@ -26,6 +26,7 @@ import Utility.SafeCommand as X import Utility.Path as X import Utility.Directory as X import Utility.Monad as X +import Utility.Applicative as X import Utility.FileSystemEncoding as X import Utility.PartialPrelude as X diff --git a/Logs/Presence.hs b/Logs/Presence.hs index 933426718b..e75e1e4e6d 100644 --- a/Logs/Presence.hs +++ b/Logs/Presence.hs @@ -48,7 +48,7 @@ addLog file line = Annex.Branch.change file $ \s -> {- Reads a log file. - Note that the LogLines returned may be in any order. -} readLog :: FilePath -> Annex [LogLine] -readLog file = parseLog <$> Annex.Branch.get file +readLog = parseLog <$$> Annex.Branch.get {- Parses a log file. Unparseable lines are ignored. -} parseLog :: String -> [LogLine] diff --git a/Remote/Directory.hs b/Remote/Directory.hs index a5b0ff2a25..f618f518ed 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -272,7 +272,7 @@ retrieveCheap d _ k f = liftIO $ withStoredFiles Nothing d k go remove :: FilePath -> ChunkSize -> Key -> Annex Bool remove d chunksize k = liftIO $ withStoredFiles chunksize d k go where - go files = all id <$> mapM removefile files + go = all id <$$> mapM removefile removefile file = catchBoolIO $ do let dir = parentDir file allowWrite dir diff --git a/Utility/Applicative.hs b/Utility/Applicative.hs new file mode 100644 index 0000000000..64400c8012 --- /dev/null +++ b/Utility/Applicative.hs @@ -0,0 +1,16 @@ +{- applicative stuff + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Applicative where + +{- Like <$> , but supports one level of currying. + - + - foo v = bar <$> action v == foo = bar <$$> action + -} +(<$$>) :: Functor f => (a -> b) -> (c -> f a) -> c -> f b +f <$$> v = fmap f . v +infixr 4 <$$> From e7182ad1191b42d3431f14ced24d0a87ab91495e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jun 2012 11:59:25 -0400 Subject: [PATCH 3974/8313] further design --- doc/design/assistant/syncing.mdwn | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 8b681ac100..99474928c4 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -39,6 +39,46 @@ and with appropriate rate limiting and control facilities. This probably will need lots of refinements to get working well. +### first pass: flood syncing + +Before mapping the network, the best we can do is flood all files out to every +reachable remote. This is worth doing first, since it's the simplest way to +get the basic functionality of the assistant to work. And we'll need this +anyway. + + data ToTransfer = ToUpload Key | ToDownload Key + type ToTransferChan = TChan [ToTransfer] + +* ToUpload added by the watcher thread when it adds content. +* ToDownload added by the watcher thread when it seens new symlinks + that lack content. + +Transfer threads started/stopped as necessary to move data. +May sometimes want multiple threads downloading, or uploading, or even both. + + data TransferID = TransferThread ThreadID | TransferProcess Pid + data Direction = Uploading | Downloading + data Transfer = Transfer Direction Key TransferID EpochTime Integer + -- add [Transfer] to DaemonStatus + +The assistant needs to find out when `git-annex-shell` is receiving or +sending (triggered by another remote), so it can add data for those too. +This is important to avoid uploading content to a remote that is already +downloading it from us, or vice versa, as well as to in future let the web +app manage transfers as user desires. + +For files being received, it can see the temp file, but other than lsof +there's no good way to find the pid (and I'd rather not kill blindly). + +For files being sent, there's no filesystem indication. So git-annex-shell +(and other git-annex transfer processes) should write a status file to disk. + +Can use file locking on these status files to claim upload/download rights, +which will avoid races. + +This status file can also be updated periodically to show amount of transfer +complete (necessary for tracking uploads). + ## other considerations It would be nice if, when a USB drive is connected, From 61786c52ad128fe39346241ef47e50ac41afb774 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jun 2012 14:03:03 -0400 Subject: [PATCH 3975/8313] releasing version 3.20120629 --- debian/changelog | 4 ++-- git-annex.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index 46afb6e4d5..96d85da278 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (3.20120625) UNRELEASED; urgency=low +git-annex (3.20120629) unstable; urgency=low * cabal: Only try to use inotify on Linux. * Version build dependency on STM, and allow building without it, @@ -11,7 +11,7 @@ git-annex (3.20120625) UNRELEASED; urgency=low in their names. * sync: Automatically resolves merge conflicts. - -- Joey Hess Mon, 25 Jun 2012 11:38:12 -0400 + -- Joey Hess Fri, 29 Jun 2012 10:17:49 -0400 git-annex (3.20120624) unstable; urgency=low diff --git a/git-annex.cabal b/git-annex.cabal index f559406959..0bd35e14fe 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 3.20120625 +Version: 3.20120629 Cabal-Version: >= 1.8 License: GPL Maintainer: Joey Hess From 0ed7db5f3ac87405f56eb27adb9fdaf42bc49125 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jun 2012 14:03:37 -0400 Subject: [PATCH 3976/8313] add news item for git-annex 3.20120629 --- doc/design/assistant/syncing.mdwn | 2 ++ doc/news/version_3.20120605.mdwn | 11 ----------- doc/news/version_3.20120629.mdwn | 12 ++++++++++++ 3 files changed, 14 insertions(+), 11 deletions(-) delete mode 100644 doc/news/version_3.20120605.mdwn create mode 100644 doc/news/version_3.20120629.mdwn diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 99474928c4..7c6ef16d39 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -46,6 +46,8 @@ reachable remote. This is worth doing first, since it's the simplest way to get the basic functionality of the assistant to work. And we'll need this anyway. +### transfer tracking + data ToTransfer = ToUpload Key | ToDownload Key type ToTransferChan = TChan [ToTransfer] diff --git a/doc/news/version_3.20120605.mdwn b/doc/news/version_3.20120605.mdwn deleted file mode 100644 index ed0a091771..0000000000 --- a/doc/news/version_3.20120605.mdwn +++ /dev/null @@ -1,11 +0,0 @@ -git-annex 3.20120605 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * sync: Show a nicer message if a user tries to sync to a special remote. - * lock: Reset unlocked file to index, rather than to branch head. - * import: New subcommand, pulls files from a directory outside the annex - and adds them. - * Fix display of warning message when encountering a file that uses an - unsupported backend. - * Require that the SHA256 backend can be used when building, since it's the - default. - * Preserve parent environment when running hooks of the hook special remote."""]] \ No newline at end of file diff --git a/doc/news/version_3.20120629.mdwn b/doc/news/version_3.20120629.mdwn new file mode 100644 index 0000000000..e6b98ae997 --- /dev/null +++ b/doc/news/version_3.20120629.mdwn @@ -0,0 +1,12 @@ +git-annex 3.20120629 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * cabal: Only try to use inotify on Linux. + * Version build dependency on STM, and allow building without it, + which disables the watch command. + * Avoid ugly failure mode when moving content from a local repository + that is not available. + * Got rid of the last place that did utf8 decoding. + * Accept arbitrarily encoded repository filepaths etc when reading + git config output. This fixes support for remotes with unusual characters + in their names. + * sync: Automatically resolves merge conflicts."""]] \ No newline at end of file From c79625290a9e17e8c9f6f0ed93a0e23a5ef0126c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jun 2012 14:12:16 -0400 Subject: [PATCH 3977/8313] improving transfer data types and design --- doc/design/assistant/syncing.mdwn | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 7c6ef16d39..02811f07ef 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -48,20 +48,24 @@ anyway. ### transfer tracking - data ToTransfer = ToUpload Key | ToDownload Key - type ToTransferChan = TChan [ToTransfer] - -* ToUpload added by the watcher thread when it adds content. -* ToDownload added by the watcher thread when it seens new symlinks +* Upload added to queue by the watcher thread when it adds content. +* Download added to queue by the watcher thread when it seens new symlinks that lack content. - -Transfer threads started/stopped as necessary to move data. -May sometimes want multiple threads downloading, or uploading, or even both. +* Transfer threads started/stopped as necessary to move data. + (May sometimes want multiple threads downloading, or uploading, or even both.) + + type TransferQueue = TChan [Transfer] + data Transfer = Upload Key Remote | Download Key Remote data TransferID = TransferThread ThreadID | TransferProcess Pid - data Direction = Uploading | Downloading - data Transfer = Transfer Direction Key TransferID EpochTime Integer - -- add [Transfer] to DaemonStatus + type AmountComplete = Integer + type StartedTime = EpochTime + data TransferInfo = TransferInfo TransferID StartedTime AmountComplete + -- add (M.Map Transfer TransferInfo) to DaemonStatus + + startTransfer :: Transfer -> Annex TransferID + + stopTransfer :: TransferID -> IO () The assistant needs to find out when `git-annex-shell` is receiving or sending (triggered by another remote), so it can add data for those too. From 660f81d2b2d8393577771c5f51e9da5f0ba00e22 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 29 Jun 2012 15:44:14 -0400 Subject: [PATCH 3978/8313] blog for the day --- .../blog/day_20__data_transfer_design.mdwn | 51 +++++++++++++++++++ doc/design/assistant/progressbars.mdwn | 2 +- doc/design/assistant/syncing.mdwn | 4 +- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 doc/design/assistant/blog/day_20__data_transfer_design.mdwn diff --git a/doc/design/assistant/blog/day_20__data_transfer_design.mdwn b/doc/design/assistant/blog/day_20__data_transfer_design.mdwn new file mode 100644 index 0000000000..2733f09bc4 --- /dev/null +++ b/doc/design/assistant/blog/day_20__data_transfer_design.mdwn @@ -0,0 +1,51 @@ +Today is a planning day. I have only a few days left before I'm off to +Nicaragua for [DebConf](http://debconf12.debconf.org/), where I'll only +have smaller chunks of time without interruptions. So it's important to get +some well-defined smallish chunks designed that I can work on later. See +bulleted action items below. Each should be around 1-2 hours unless it +turns out to be 8 hours... :) + +First, worked on writing down a design, and some data types, for data transfer +tracking (see [[syncing]] page). Found that writing down these simple data +types before I started slinging code has clarified things a lot for me. + +Most importantly, I realized that I will need to modify `git-annex-shell` +to record on disk what transfers it's doing, so the assistant can get that +information and use it to both avoid redundant transfers (potentially a big +problem!), and later to allow the user to control them using the web app. + +So these will be the first steps as I move toward implementing data +transfer tracking and naive flood fill transferring. + +* on-disk transfers in progress information files (read/write/enumerate) +* locking for the files, so redundant transfer races can be detected, + and failed transfers noticed +* update files as transfers proceed. See [[progressbars]] + (updating for downloads is easy; for uploads is hard) +* add Transfer queue TChan +* enqueue Transfers (Uploads) as new files are added to the annex by + Watcher. +* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by + Watcher. +* add TransferInfo Map to DaemonStatus for tracking transfers in progress. +* Poll transfer in progress info files for changes (use inotify again! + wow! hammer, meet nail..), and update the TransferInfo Map +* Write basic Transfer handling thread. Multiple such threads need to be + able to be run at once. Each will need its own independant copy of the + Annex state monad. +* Write transfer control thread, which decides when to launch transfers. +* At startup, and possibly periodically, look for files we have that + location tracking indicates remotes do not, and enqueue Uploads for + them. Also, enqueue Downloads for any files we're missing. + +While eventually the user will be able to use the web app to prioritize +transfers, stop and start, throttle, etc, it's important to get the default +behavior right. So I'm thinking about things like how to prioritize uploads +vs downloads, when it's appropriate to have multiple downloads running at +once, etc. + +* Find a way to probe available outgoing bandwidth, to throttle so + we don't bufferbloat the network to death. +* git-annex needs a simple speed control knob, which can be plumbed + through to, at least, rsync. A good job for an hour in an + airport somewhere. diff --git a/doc/design/assistant/progressbars.mdwn b/doc/design/assistant/progressbars.mdwn index 2ade05aa57..ee73842743 100644 --- a/doc/design/assistant/progressbars.mdwn +++ b/doc/design/assistant/progressbars.mdwn @@ -9,6 +9,6 @@ To get this info for downloads, git-annex can watch the file as it arrives and use its size. TODO: What about uploads? Will i have to parse rsync's progresss output? -Feed it via a named pipe? Ugh. +Feed it via a named pipe? Ugh. Check into librsync. This is one of those potentially hidden but time consuming problems. diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 02811f07ef..ce7f9673b5 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -58,9 +58,9 @@ anyway. data Transfer = Upload Key Remote | Download Key Remote data TransferID = TransferThread ThreadID | TransferProcess Pid - type AmountComplete = Integer + type BytesComplete = Integer type StartedTime = EpochTime - data TransferInfo = TransferInfo TransferID StartedTime AmountComplete + data TransferInfo = TransferInfo TransferID StartedTime BytesComplete -- add (M.Map Transfer TransferInfo) to DaemonStatus startTransfer :: Transfer -> Annex TransferID From 49136c22d052ea83d0c9468e7cce28d08d961923 Mon Sep 17 00:00:00 2001 From: Philipp Kern Date: Sat, 30 Jun 2012 15:00:00 +0200 Subject: [PATCH 3979/8313] doc/download.mdwn: document no-s3 and assistant branches --- doc/download.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/download.mdwn b/doc/download.mdwn index f0f17e141d..242de13c39 100644 --- a/doc/download.mdwn +++ b/doc/download.mdwn @@ -18,6 +18,7 @@ others need some manual work. See [[install]] for details. The git repository has some branches: +* `assistant` contains the new change-tracking daemon * `ghc7.0` supports versions of ghc older than 7.4, which had a major change to filename encoding. * `old-monad-control` is for systems that don't have a newer monad-control @@ -25,6 +26,7 @@ The git repository has some branches: * `no-ifelse` avoids using the IFelse library (merge it into master if you need it) * `no-bloom` avoids using bloom filters. (merge it into master if you need it) +* `no-s3` avoids using the S3 library (merge it into master if you need it) * `debian-stable` contains the latest backport of git-annex to Debian stable. * `tweak-fetch` adds support for the git tweak-fetch hook, which has From 768036f3dd42bb4b733679cfcd8af1ee42dcd70c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c" Date: Sat, 30 Jun 2012 14:34:12 +0000 Subject: [PATCH 3980/8313] Added a comment: sha256 alternative --- .../comment_12_60d13f2c8e008af1041bea565a392c83._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment diff --git a/doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment b/doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment new file mode 100644 index 0000000000..e2e85aaa94 --- /dev/null +++ b/doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c" + nickname="Damien" + subject="sha256 alternative" + date="2012-06-30T14:34:11Z" + content=""" +in reply to comment 6: On my Mac (10.7.4) there's `/usr/bin/shasum -a 256 ` command that will produce the same output as `sha256sum `. +"""]] From edee8ad05b2fe8487c05cdcdacafa19a75151931 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c" Date: Sun, 1 Jul 2012 17:03:58 +0000 Subject: [PATCH 3981/8313] Added a comment: gnu commands --- .../comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment diff --git a/doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment b/doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment new file mode 100644 index 0000000000..e5ce62b138 --- /dev/null +++ b/doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnHrjHxJAm39x8DR4bnbazQO6H0nMNuY9c" + nickname="Damien" + subject="gnu commands" + date="2012-07-01T17:03:57Z" + content=""" +…and another approach to the same problem: apparently git-annex also relies on the GNU coreutils (for instance, when doing `git annex get .`, `cp` complains about `illegal option -- -`). I do have the GNU coreutils installed with Homebrew, but they are all prefixed with `g`. So maybe you should try `gsha256sum` and `gcp` before `sha256sum` and `cp`, that seems like a more general solution. +"""]] From be0e38bcc38405afec3283e31e8628e8c6a494aa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 14:29:00 -0400 Subject: [PATCH 3982/8313] add transfer information files --- Locations.hs | 6 ++ Logs/Transfer.hs | 159 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 Logs/Transfer.hs diff --git a/Locations.hs b/Locations.hs index cd3f55d466..082a72a506 100644 --- a/Locations.hs +++ b/Locations.hs @@ -18,6 +18,7 @@ module Locations ( gitAnnexBadDir, gitAnnexBadLocation, gitAnnexUnusedLog, + gitAnnexTransferDir, gitAnnexJournalDir, gitAnnexJournalLock, gitAnnexIndex, @@ -127,6 +128,11 @@ gitAnnexBadLocation key r = gitAnnexBadDir r keyFile key gitAnnexUnusedLog :: FilePath -> Git.Repo -> FilePath gitAnnexUnusedLog prefix r = gitAnnexDir r (prefix ++ "unused") +{- .git/annex/transfer/ is used is used to record keys currently + - being transferred. -} +gitAnnexTransferDir :: Git.Repo -> FilePath +gitAnnexTransferDir r = addTrailingPathSeparator $ gitAnnexDir r "transfer" + {- .git/annex/journal/ is used to journal changes made to the git-annex - branch -} gitAnnexJournalDir :: Git.Repo -> FilePath diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs new file mode 100644 index 0000000000..ab99304d1f --- /dev/null +++ b/Logs/Transfer.hs @@ -0,0 +1,159 @@ +{- git-annex transfer log files + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Logs.Transfer where + +import Common.Annex +import Types.Remote +import Remote +import Annex.Perms +import Annex.Exception +import qualified Git + +import qualified Data.Map as M +import Control.Concurrent +import System.Posix.Process +import System.Posix.Types +import Data.Time.Clock + +{- Enough information to uniquely identify a transfer, used as the filename + - of the transfer information file. -} +data Transfer = Transfer Direction Remote Key + deriving (Show) + +{- Information about a Transfer, stored in the transfer information file. -} +data TransferInfo = TransferInfo + { transferPid :: Maybe ProcessID + , transferThread :: Maybe ThreadId + , startedTime :: UTCTime + , bytesComplete :: Maybe Integer + , associatedFile :: Maybe FilePath + } + deriving (Show) + +data Direction = Upload | Download + +instance Show Direction where + show Upload = "upload" + show Download = "download" + +readDirection :: String -> Maybe Direction +readDirection "upload" = Just Upload +readDirection "download" = Just Download +readDirection _ = Nothing + +{- Runs a transfer action. Creates and locks the transfer information file + - while the action is running. Will throw an error if the transfer is + - already in progress. + -} +transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a +transfer transfer file a = do + createAnnexDirectory =<< fromRepo gitAnnexTransferDir + tfile <- fromRepo $ transferFile transfer + mode <- annexFileMode + info <- liftIO $ TransferInfo + <$> pure Nothing -- pid not stored in file, so omitted for speed + <*> pure Nothing -- threadid not stored in file, so omitted for speed + <*> getCurrentTime + <*> pure Nothing -- not 0; transfer may be resuming + <*> pure file + bracketIO (setup tfile mode info) (cleanup tfile) a + where + setup tfile mode info = do + fd <- openFd tfile ReadWrite (Just mode) + defaultFileFlags { trunc = True } + locked <- catchMaybeIO $ + setLock fd (WriteLock, AbsoluteSeek, 0, 0) + when (locked == Nothing) $ + error $ "transfer already in progress" + fdWrite fd $ writeTransferInfo info + return fd + cleanup tfile fd = do + removeFile tfile + closeFd fd + +{- If a transfer is still running, returns its TransferInfo. -} +checkTransfer :: Transfer -> Annex (Maybe TransferInfo) +checkTransfer transfer = do + mode <- annexFileMode + tfile <- fromRepo $ transferFile transfer + mfd <- liftIO $ catchMaybeIO $ + openFd tfile ReadOnly (Just mode) defaultFileFlags + case mfd of + Nothing -> return Nothing -- failed to open file; not running + Just fd -> do + locked <- liftIO $ + getLock fd (WriteLock, AbsoluteSeek, 0, 0) + case locked of + Nothing -> do + liftIO $ closeFd fd + return Nothing + Just (pid, _) -> liftIO $ do + handle <- fdToHandle fd + info <- readTransferInfo pid + <$> hGetContentsStrict handle + closeFd fd + return info + +{- Gets all currently running transfers. -} +getTransfers :: Annex [(Transfer, TransferInfo)] +getTransfers = do + uuidmap <- remoteMap id + transfers <- catMaybes . map (parseTransferFile uuidmap) <$> findfiles + infos <- mapM checkTransfer transfers + return $ map (\(t, Just i) -> (t, i)) $ + filter running $ zip transfers infos + where + findfiles = liftIO . dirContentsRecursive + =<< fromRepo gitAnnexTransferDir + running (_, i) = isJust i + +{- The transfer information file to use for a given Transfer. -} +transferFile :: Transfer -> Git.Repo -> FilePath +transferFile (Transfer direction remote key) repo = + gitAnnexTransferDir repo + show direction + show (uuid remote) + keyFile key + +{- Parses a transfer information filename to a Transfer. -} +parseTransferFile :: M.Map UUID Remote -> FilePath -> Maybe Transfer +parseTransferFile uuidmap file = + case drop (length bits - 3) bits of + [direction, uuid, key] -> Transfer + <$> readDirection direction + <*> M.lookup (toUUID uuid) uuidmap + <*> fileKey key + _ -> Nothing + where + bits = splitDirectories file + +writeTransferInfo :: TransferInfo -> String +writeTransferInfo info = unwords + -- transferPid is not included; instead obtained by looking at + -- the process that locks the file. + -- transferThread is not included; not relevant for other processes + [ show $ startedTime info + -- bytesComplete is not included; changes too fast + , fromMaybe "" $ associatedFile info -- comes last, may contain spaces + ] + +readTransferInfo :: ProcessID -> String -> Maybe TransferInfo +readTransferInfo pid s = + case bits of + [time] -> TransferInfo + <$> pure (Just pid) + <*> pure Nothing + <*> readish time + <*> pure Nothing + <*> pure filename + _ -> Nothing + where + (bits, filebits) = splitAt 1 $ split " " s + filename + | null filebits = Nothing + | otherwise = Just $ join " " filebits From 72988bae34030295f029b36e859d28bd45f7dbc1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 15:04:29 -0400 Subject: [PATCH 3983/8313] tested; bugfixes --- Logs/Transfer.hs | 52 +++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index ab99304d1f..ab569aa0d4 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -1,4 +1,4 @@ -{- git-annex transfer log files +{- git-annex transfer information files - - Copyright 2012 Joey Hess - @@ -16,7 +16,6 @@ import qualified Git import qualified Data.Map as M import Control.Concurrent -import System.Posix.Process import System.Posix.Types import Data.Time.Clock @@ -46,14 +45,20 @@ readDirection "upload" = Just Upload readDirection "download" = Just Download readDirection _ = Nothing +upload :: Remote -> Key -> FilePath -> Annex a -> Annex a +upload remote key file a = transfer (Transfer Upload remote key) (Just file) a + +download :: Remote -> Key -> FilePath -> Annex a -> Annex a +download remote key file a = transfer (Transfer Download remote key) (Just file) a + {- Runs a transfer action. Creates and locks the transfer information file - while the action is running. Will throw an error if the transfer is - already in progress. -} transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a -transfer transfer file a = do - createAnnexDirectory =<< fromRepo gitAnnexTransferDir - tfile <- fromRepo $ transferFile transfer +transfer t file a = do + tfile <- fromRepo $ transferFile t + createAnnexDirectory $ takeDirectory tfile mode <- annexFileMode info <- liftIO $ TransferInfo <$> pure Nothing -- pid not stored in file, so omitted for speed @@ -61,16 +66,18 @@ transfer transfer file a = do <*> getCurrentTime <*> pure Nothing -- not 0; transfer may be resuming <*> pure file - bracketIO (setup tfile mode info) (cleanup tfile) a + bracketIO (prep tfile mode info) (cleanup tfile) a where - setup tfile mode info = do + prep tfile mode info = do fd <- openFd tfile ReadWrite (Just mode) defaultFileFlags { trunc = True } locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) when (locked == Nothing) $ error $ "transfer already in progress" - fdWrite fd $ writeTransferInfo info + h <- fdToHandle fd + hPutStr h $ writeTransferInfo info + hFlush h return fd cleanup tfile fd = do removeFile tfile @@ -78,9 +85,9 @@ transfer transfer file a = do {- If a transfer is still running, returns its TransferInfo. -} checkTransfer :: Transfer -> Annex (Maybe TransferInfo) -checkTransfer transfer = do +checkTransfer t = do mode <- annexFileMode - tfile <- fromRepo $ transferFile transfer + tfile <- fromRepo $ transferFile t mfd <- liftIO $ catchMaybeIO $ openFd tfile ReadOnly (Just mode) defaultFileFlags case mfd of @@ -93,9 +100,9 @@ checkTransfer transfer = do liftIO $ closeFd fd return Nothing Just (pid, _) -> liftIO $ do - handle <- fdToHandle fd + h <- fdToHandle fd info <- readTransferInfo pid - <$> hGetContentsStrict handle + <$> hGetContentsStrict h closeFd fd return info @@ -114,32 +121,31 @@ getTransfers = do {- The transfer information file to use for a given Transfer. -} transferFile :: Transfer -> Git.Repo -> FilePath -transferFile (Transfer direction remote key) repo = - gitAnnexTransferDir repo - show direction - show (uuid remote) - keyFile key +transferFile (Transfer direction remote key) r = gitAnnexTransferDir r + show direction + fromUUID (uuid remote) + keyFile key {- Parses a transfer information filename to a Transfer. -} parseTransferFile :: M.Map UUID Remote -> FilePath -> Maybe Transfer parseTransferFile uuidmap file = case drop (length bits - 3) bits of - [direction, uuid, key] -> Transfer + [direction, u, key] -> Transfer <$> readDirection direction - <*> M.lookup (toUUID uuid) uuidmap + <*> M.lookup (toUUID u) uuidmap <*> fileKey key _ -> Nothing where bits = splitDirectories file writeTransferInfo :: TransferInfo -> String -writeTransferInfo info = unwords +writeTransferInfo info = unlines -- transferPid is not included; instead obtained by looking at -- the process that locks the file. -- transferThread is not included; not relevant for other processes [ show $ startedTime info -- bytesComplete is not included; changes too fast - , fromMaybe "" $ associatedFile info -- comes last, may contain spaces + , fromMaybe "" $ associatedFile info -- comes last; arbitrary content ] readTransferInfo :: ProcessID -> String -> Maybe TransferInfo @@ -153,7 +159,7 @@ readTransferInfo pid s = <*> pure filename _ -> Nothing where - (bits, filebits) = splitAt 1 $ split " " s + (bits, filebits) = splitAt 1 $ lines s filename | null filebits = Nothing - | otherwise = Just $ join " " filebits + | otherwise = Just $ unlines filebits From e5fd8b67b7dc3321b13c9b01c36cc7f4d01e1ad8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 15:18:36 -0400 Subject: [PATCH 3984/8313] get, move, copy: Now refuse to do anything when the requested file transfer is already in progress by another process. Note this is per-remote, so trying to get the same file from multiple remotes can still let duplicate downloads run. (And uploading the same file to multiple remotes is not duplicate at all of course.) get, move, and copy are the only git-annex subcommands that transfer files, but there's still git-annex-shell recvkey and sendkey to deal with too. I considered modifying retrieveKeyFile or getViaTmp, but they are called by other code that does not involve expensive file transfers (migrate) or that does file transfers that should not be checked by this (fsck --from). --- Command/Get.hs | 19 ++++++++++--------- Command/Move.hs | 17 +++++++++-------- debian/changelog | 7 +++++++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/Command/Get.hs b/Command/Get.hs index c4ba483126..35e25d9751 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -12,6 +12,7 @@ import Command import qualified Remote import Annex.Content import qualified Command.Move +import Logs.Transfer def :: [Command] def = [withOptions [Command.Move.fromOption] $ command "get" paramPaths seek @@ -25,24 +26,24 @@ start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart start from file (key, _) = stopUnless (not <$> inAnnex key) $ autoCopies file key (<) $ \_numcopies -> case from of - Nothing -> go $ perform key + Nothing -> go $ perform key file Just src -> -- get --from = copy --from stopUnless (Command.Move.fromOk src key) $ - go $ Command.Move.fromPerform src False key + go $ Command.Move.fromPerform src False key file where go a = do showStart "get" file - next a + next a -perform :: Key -> CommandPerform -perform key = stopUnless (getViaTmp key $ getKeyFile key) $ +perform :: Key -> FilePath -> CommandPerform +perform key file = stopUnless (getViaTmp key $ getKeyFile key file) $ next $ return True -- no cleanup needed {- Try to find a copy of the file in one of the remotes, - and copy it to here. -} -getKeyFile :: Key -> FilePath -> Annex Bool -getKeyFile key file = dispatch =<< Remote.keyPossibilities key +getKeyFile :: Key -> FilePath -> FilePath -> Annex Bool +getKeyFile key file dest = dispatch =<< Remote.keyPossibilities key where dispatch [] = do showNote "not available" @@ -64,7 +65,7 @@ getKeyFile key file = dispatch =<< Remote.keyPossibilities key | Remote.hasKeyCheap r = either (const False) id <$> Remote.hasKey r key | otherwise = return True - docopy r continue = do + docopy r continue = download r key file $ do showAction $ "from " ++ Remote.name r - ifM (Remote.retrieveKeyFile r key file) + ifM (Remote.retrieveKeyFile r key dest) ( return True , continue) diff --git a/Command/Move.hs b/Command/Move.hs index 6ec7cd90ab..8bba468783 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -16,6 +16,7 @@ import qualified Remote import Annex.UUID import qualified Option import Logs.Presence +import Logs.Transfer def :: [Command] def = [withOptions options $ command "move" paramPaths seek @@ -68,9 +69,9 @@ toStart dest move file key = do then stop -- not here, so nothing to do else do showMoveAction move file - next $ toPerform dest move key -toPerform :: Remote -> Bool -> Key -> CommandPerform -toPerform dest move key = moveLock move key $ do + next $ toPerform dest move key file +toPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform +toPerform dest move key file = moveLock move key $ do -- Checking the remote is expensive, so not done in the start step. -- In fast mode, location tracking is assumed to be correct, -- and an explicit check is not done, when copying. When moving, @@ -88,7 +89,7 @@ toPerform dest move key = moveLock move key $ do stop Right False -> do showAction $ "to " ++ Remote.name dest - ok <- Remote.storeKey dest key + ok <- upload dest key file $ Remote.storeKey dest key if ok then finish else do @@ -118,7 +119,7 @@ fromStart src move file key where go = stopUnless (fromOk src key) $ do showMoveAction move file - next $ fromPerform src move key + next $ fromPerform src move key file fromOk :: Remote -> Key -> Annex Bool fromOk src key | Remote.hasKeyCheap src = @@ -129,11 +130,11 @@ fromOk src key u <- getUUID remotes <- Remote.keyPossibilities key return $ u /= Remote.uuid src && elem src remotes -fromPerform :: Remote -> Bool -> Key -> CommandPerform -fromPerform src move key = moveLock move key $ +fromPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform +fromPerform src move key file = moveLock move key $ ifM (inAnnex key) ( handle move True - , do + , download src key file $ do showAction $ "from " ++ Remote.name src ok <- getViaTmp key $ Remote.retrieveKeyFile src key handle move ok diff --git a/debian/changelog b/debian/changelog index 96d85da278..babd1786de 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +git-annex (3.20120630) UNRELEASED; urgency=low + + * get, move, copy: Now refuse to do anything when the requested file + transfer is already in progress by another process. + + -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 + git-annex (3.20120629) unstable; urgency=low * cabal: Only try to use inotify on Linux. From 8c10f377146e6599054488f47a3a742f6a7c5ae2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 16:10:00 -0400 Subject: [PATCH 3985/8313] bugfixes fdToHandle seems to close the fd avoid excess trailing newline --- Logs/Transfer.hs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index ab569aa0d4..fe93b90b43 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -21,20 +21,25 @@ import Data.Time.Clock {- Enough information to uniquely identify a transfer, used as the filename - of the transfer information file. -} -data Transfer = Transfer Direction Remote Key - deriving (Show) +data Transfer = Transfer + { transferDirection :: Direction + , transferRemote :: Remote + , transferKey :: Key + } + deriving (Show, Eq, Ord) {- Information about a Transfer, stored in the transfer information file. -} data TransferInfo = TransferInfo - { transferPid :: Maybe ProcessID + { startedTime :: UTCTime + , transferPid :: Maybe ProcessID , transferThread :: Maybe ThreadId - , startedTime :: UTCTime , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath } - deriving (Show) + deriving (Show, Eq, Ord) data Direction = Upload | Download + deriving (Eq, Ord) instance Show Direction where show Upload = "upload" @@ -61,9 +66,9 @@ transfer t file a = do createAnnexDirectory $ takeDirectory tfile mode <- annexFileMode info <- liftIO $ TransferInfo - <$> pure Nothing -- pid not stored in file, so omitted for speed + <$> getCurrentTime + <*> pure Nothing -- pid not stored in file, so omitted for speed <*> pure Nothing -- threadid not stored in file, so omitted for speed - <*> getCurrentTime <*> pure Nothing -- not 0; transfer may be resuming <*> pure file bracketIO (prep tfile mode info) (cleanup tfile) a @@ -103,7 +108,7 @@ checkTransfer t = do h <- fdToHandle fd info <- readTransferInfo pid <$> hGetContentsStrict h - closeFd fd + hClose h return info {- Gets all currently running transfers. -} @@ -152,9 +157,9 @@ readTransferInfo :: ProcessID -> String -> Maybe TransferInfo readTransferInfo pid s = case bits of [time] -> TransferInfo - <$> pure (Just pid) + <$> readish time + <*> pure (Just pid) <*> pure Nothing - <*> readish time <*> pure Nothing <*> pure filename _ -> Nothing @@ -162,4 +167,4 @@ readTransferInfo pid s = (bits, filebits) = splitAt 1 $ lines s filename | null filebits = Nothing - | otherwise = Just $ unlines filebits + | otherwise = Just $ join "\n" filebits From 7225c2bfc0c7149e646fa9af998da983e3fa8bc8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 16:59:54 -0400 Subject: [PATCH 3986/8313] record transfer information on local git remotes In order to record a semi-useful filename associated with the key, this required plumbing the filename all the way through to the remotes' storeKey and retrieveKeyFile. Note that there is potential for deadlock here, narrowly avoided. Suppose the repos are A and B. A sends file foo to B, and at the same time, B gets file foo from A. So, A locks its upload transfer info file, and then locks B's download transfer info file. At the same time, B is taking the two locks in the opposite order. This is only not a deadlock because the lock code does not wait, and aborts. So one of A or B's transfers will be aborted and the other transfer will continue. Whew! --- Command/Fsck.hs | 2 +- Command/Get.hs | 4 ++-- Command/Move.hs | 8 +++++--- Command/Status.hs | 20 ++++++++++++++++++++ Logs/Transfer.hs | 33 +++++++++++++++------------------ Remote/Bup.hs | 10 +++++----- Remote/Directory.hs | 8 ++++---- Remote/Git.hs | 32 ++++++++++++++++++++------------ Remote/Helper/Encryptable.hs | 14 +++++++------- Remote/Helper/Hooks.hs | 4 ++-- Remote/Hook.hs | 8 ++++---- Remote/Rsync.hs | 12 ++++++------ Remote/S3.hs | 10 +++++----- Remote/Web.hs | 10 +++++----- Types/Remote.hs | 7 +++++-- debian/changelog | 1 + 16 files changed, 107 insertions(+), 76 deletions(-) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 7bfc46f4a6..10cca489b1 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -94,7 +94,7 @@ performRemote key file backend numcopies remote = ( return True , ifM (Annex.getState Annex.fast) ( return False - , Remote.retrieveKeyFile remote key tmp + , Remote.retrieveKeyFile remote key Nothing tmp ) ) diff --git a/Command/Get.hs b/Command/Get.hs index 35e25d9751..a5901ba664 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -65,7 +65,7 @@ getKeyFile key file dest = dispatch =<< Remote.keyPossibilities key | Remote.hasKeyCheap r = either (const False) id <$> Remote.hasKey r key | otherwise = return True - docopy r continue = download r key file $ do + docopy r continue = download (Remote.uuid r) key (Just file) $ do showAction $ "from " ++ Remote.name r - ifM (Remote.retrieveKeyFile r key dest) + ifM (Remote.retrieveKeyFile r key (Just file) dest) ( return True , continue) diff --git a/Command/Move.hs b/Command/Move.hs index 8bba468783..e7c11e80d3 100644 --- a/Command/Move.hs +++ b/Command/Move.hs @@ -89,7 +89,8 @@ toPerform dest move key file = moveLock move key $ do stop Right False -> do showAction $ "to " ++ Remote.name dest - ok <- upload dest key file $ Remote.storeKey dest key + ok <- upload (Remote.uuid dest) key (Just file) $ + Remote.storeKey dest key (Just file) if ok then finish else do @@ -134,9 +135,10 @@ fromPerform :: Remote -> Bool -> Key -> FilePath -> CommandPerform fromPerform src move key file = moveLock move key $ ifM (inAnnex key) ( handle move True - , download src key file $ do + , download (Remote.uuid src) key (Just file) $ do showAction $ "from " ++ Remote.name src - ok <- getViaTmp key $ Remote.retrieveKeyFile src key + ok <- getViaTmp key $ + Remote.retrieveKeyFile src key (Just file) handle move ok ) where diff --git a/Command/Status.hs b/Command/Status.hs index 2540a92da8..eff21bb509 100644 --- a/Command/Status.hs +++ b/Command/Status.hs @@ -31,6 +31,7 @@ import Logs.Trust import Remote import Config import Utility.Percentage +import Logs.Transfer -- a named computation that produces a statistic type Stat = StatState (Maybe (String, StatState String)) @@ -70,6 +71,7 @@ fast_stats = , remote_list SemiTrusted "semitrusted" , remote_list UnTrusted "untrusted" , remote_list DeadTrusted "dead" + , transfer_list , disk_size ] slow_stats :: [Stat] @@ -170,6 +172,24 @@ bloom_info = stat "bloom filter size" $ json id $ do return $ size ++ note +transfer_list :: Stat +transfer_list = stat "transfers in progress" $ nojson $ lift $ do + uuidmap <- Remote.remoteMap id + ts <- getTransfers + if null ts + then return "none" + else return $ pp uuidmap "" $ sort ts + where + pp _ c [] = c + pp uuidmap c ((t, i):xs) = "\n\t" ++ line uuidmap t i ++ pp uuidmap c xs + line uuidmap t i = unwords + [ show (transferDirection t) ++ "ing" + , fromMaybe (show $ transferKey t) (associatedFile i) + , if transferDirection t == Upload then "to" else "from" + , maybe (fromUUID $ transferRemote t) Remote.name $ + M.lookup (transferRemote t) uuidmap + ] + disk_size :: Stat disk_size = stat "available local disk space" $ json id $ lift $ calcfree diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index fe93b90b43..526241f935 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -8,13 +8,11 @@ module Logs.Transfer where import Common.Annex -import Types.Remote -import Remote import Annex.Perms import Annex.Exception import qualified Git +import Types.Remote -import qualified Data.Map as M import Control.Concurrent import System.Posix.Types import Data.Time.Clock @@ -23,7 +21,7 @@ import Data.Time.Clock - of the transfer information file. -} data Transfer = Transfer { transferDirection :: Direction - , transferRemote :: Remote + , transferRemote :: UUID , transferKey :: Key } deriving (Show, Eq, Ord) @@ -50,11 +48,11 @@ readDirection "upload" = Just Upload readDirection "download" = Just Download readDirection _ = Nothing -upload :: Remote -> Key -> FilePath -> Annex a -> Annex a -upload remote key file a = transfer (Transfer Upload remote key) (Just file) a +upload :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a +upload u key file a = transfer (Transfer Upload u key) file a -download :: Remote -> Key -> FilePath -> Annex a -> Annex a -download remote key file a = transfer (Transfer Download remote key) (Just file) a +download :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a +download u key file a = transfer (Transfer Download u key) file a {- Runs a transfer action. Creates and locks the transfer information file - while the action is running. Will throw an error if the transfer is @@ -83,10 +81,10 @@ transfer t file a = do h <- fdToHandle fd hPutStr h $ writeTransferInfo info hFlush h - return fd - cleanup tfile fd = do + return h + cleanup tfile h = do removeFile tfile - closeFd fd + hClose h {- If a transfer is still running, returns its TransferInfo. -} checkTransfer :: Transfer -> Annex (Maybe TransferInfo) @@ -114,8 +112,7 @@ checkTransfer t = do {- Gets all currently running transfers. -} getTransfers :: Annex [(Transfer, TransferInfo)] getTransfers = do - uuidmap <- remoteMap id - transfers <- catMaybes . map (parseTransferFile uuidmap) <$> findfiles + transfers <- catMaybes . map parseTransferFile <$> findfiles infos <- mapM checkTransfer transfers return $ map (\(t, Just i) -> (t, i)) $ filter running $ zip transfers infos @@ -126,18 +123,18 @@ getTransfers = do {- The transfer information file to use for a given Transfer. -} transferFile :: Transfer -> Git.Repo -> FilePath -transferFile (Transfer direction remote key) r = gitAnnexTransferDir r +transferFile (Transfer direction u key) r = gitAnnexTransferDir r show direction - fromUUID (uuid remote) + fromUUID u keyFile key {- Parses a transfer information filename to a Transfer. -} -parseTransferFile :: M.Map UUID Remote -> FilePath -> Maybe Transfer -parseTransferFile uuidmap file = +parseTransferFile :: FilePath -> Maybe Transfer +parseTransferFile file = case drop (length bits - 3) bits of [direction, u, key] -> Transfer <$> readDirection direction - <*> M.lookup (toUUID u) uuidmap + <*> pure (toUUID u) <*> fileKey key _ -> Nothing where diff --git a/Remote/Bup.hs b/Remote/Bup.hs index f1a36e468e..0d1b606d3d 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -108,8 +108,8 @@ bupSplitParams r buprepo k src = do return $ bupParams "split" buprepo (os ++ [Param "-n", Param (bupRef k), src]) -store :: Git.Repo -> BupRepo -> Key -> Annex Bool -store r buprepo k = do +store :: Git.Repo -> BupRepo -> Key -> AssociatedFile -> Annex Bool +store r buprepo k _f = do src <- inRepo $ gitAnnexLocation k params <- bupSplitParams r buprepo k (File src) liftIO $ boolSystem "bup" params @@ -122,11 +122,11 @@ storeEncrypted r buprepo (cipher, enck) k = do withEncryptedHandle cipher (L.readFile src) $ \h -> pipeBup params (Just h) Nothing -retrieve :: BupRepo -> Key -> FilePath -> Annex Bool -retrieve buprepo k f = do +retrieve :: BupRepo -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve buprepo k _f d = do let params = bupParams "join" buprepo [Param $ bupRef k] liftIO $ catchBoolIO $ do - tofile <- openFile f WriteMode + tofile <- openFile d WriteMode pipeBup params Nothing (Just tofile) retrieveCheap :: BupRepo -> Key -> FilePath -> Annex Bool diff --git a/Remote/Directory.hs b/Remote/Directory.hs index f618f518ed..6b158730e8 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -122,8 +122,8 @@ withCheckedFiles check (Just _) d k a = go $ locations d k withStoredFiles :: ChunkSize -> FilePath -> Key -> ([FilePath] -> IO Bool) -> IO Bool withStoredFiles = withCheckedFiles doesFileExist -store :: FilePath -> ChunkSize -> Key -> Annex Bool -store d chunksize k = do +store :: FilePath -> ChunkSize -> Key -> AssociatedFile -> Annex Bool +store d chunksize k _f = do src <- inRepo $ gitAnnexLocation k metered k $ \meterupdate -> storeHelper d chunksize k $ \dests -> @@ -242,8 +242,8 @@ storeHelper d chunksize key a = prep <&&> check <&&> go preventWrite dir return (not $ null stored) -retrieve :: FilePath -> ChunkSize -> Key -> FilePath -> Annex Bool -retrieve d chunksize k f = metered k $ \meterupdate -> +retrieve :: FilePath -> ChunkSize -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve d chunksize k _ f = metered k $ \meterupdate -> liftIO $ withStoredFiles chunksize d k $ \files -> catchBoolIO $ do meteredWriteFile' meterupdate f files feeder diff --git a/Remote/Git.hs b/Remote/Git.hs index 60a881803a..0b839c9a5e 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -21,6 +21,7 @@ import qualified Git.Config import qualified Git.Construct import qualified Annex import Logs.Presence +import Logs.Transfer import Annex.UUID import qualified Annex.Content import qualified Annex.BranchState @@ -219,14 +220,19 @@ dropKey r key ] {- Tries to copy a key's content from a remote's annex to a file. -} -copyFromRemote :: Git.Repo -> Key -> FilePath -> Annex Bool -copyFromRemote r key file +copyFromRemote :: Git.Repo -> Key -> AssociatedFile -> FilePath -> Annex Bool +copyFromRemote r key file dest | not $ Git.repoIsUrl r = guardUsable r False $ do params <- rsyncParams r - loc <- liftIO $ gitAnnexLocation key r - rsyncOrCopyFile params loc file - | Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key file - | Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) file + u <- getUUID + -- run copy from perspective of remote + liftIO $ onLocal r $ do + ensureInitialized + loc <- inRepo $ gitAnnexLocation key + upload u key file $ + rsyncOrCopyFile params loc dest + | Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key dest + | Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) dest | otherwise = error "copying from non-ssh, non-http repo not supported" copyFromRemoteCheap :: Git.Repo -> Key -> FilePath -> Annex Bool @@ -236,23 +242,25 @@ copyFromRemoteCheap r key file liftIO $ catchBoolIO $ createSymbolicLink loc file >> return True | Git.repoIsSsh r = ifM (Annex.Content.preseedTmp key file) - ( copyFromRemote r key file + ( copyFromRemote r key Nothing file , return False ) | otherwise = return False {- Tries to copy a key's content to a remote's annex. -} -copyToRemote :: Git.Repo -> Key -> Annex Bool -copyToRemote r key +copyToRemote :: Git.Repo -> Key -> AssociatedFile -> Annex Bool +copyToRemote r key file | not $ Git.repoIsUrl r = guardUsable r False $ commitOnCleanup r $ do keysrc <- inRepo $ gitAnnexLocation key params <- rsyncParams r + u <- getUUID -- run copy from perspective of remote liftIO $ onLocal r $ do ensureInitialized - Annex.Content.saveState True `after` - Annex.Content.getViaTmp key - (rsyncOrCopyFile params keysrc) + download u key file $ + Annex.Content.saveState True `after` + Annex.Content.getViaTmp key + (rsyncOrCopyFile params keysrc) | Git.repoIsSsh r = commitOnCleanup r $ do keysrc <- inRepo $ gitAnnexLocation key rsyncHelper =<< rsyncParamsRemote r False key keysrc diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs index 789a1d9964..6d5405d9e0 100644 --- a/Remote/Helper/Encryptable.hs +++ b/Remote/Helper/Encryptable.hs @@ -59,14 +59,14 @@ encryptableRemote c storeKeyEncrypted retrieveKeyFileEncrypted r = cost = cost r + encryptedRemoteCostAdj } where - store k = cip k >>= maybe - (storeKey r k) + store k f = cip k >>= maybe + (storeKey r k f) (`storeKeyEncrypted` k) - retrieve k f = cip k >>= maybe - (retrieveKeyFile r k f) - (\enck -> retrieveKeyFileEncrypted enck k f) - retrieveCheap k f = cip k >>= maybe - (retrieveKeyFileCheap r k f) + retrieve k f d = cip k >>= maybe + (retrieveKeyFile r k f d) + (\enck -> retrieveKeyFileEncrypted enck k d) + retrieveCheap k d = cip k >>= maybe + (retrieveKeyFileCheap r k d) (\_ -> return False) withkey a k = cip k >>= maybe (a k) (a . snd) cip = cipherKey c diff --git a/Remote/Helper/Hooks.hs b/Remote/Helper/Hooks.hs index d85959062e..0a6b22081e 100644 --- a/Remote/Helper/Hooks.hs +++ b/Remote/Helper/Hooks.hs @@ -27,8 +27,8 @@ addHooks' r Nothing Nothing = r addHooks' r starthook stophook = r' where r' = r - { storeKey = \k -> wrapper $ storeKey r k - , retrieveKeyFile = \k f -> wrapper $ retrieveKeyFile r k f + { storeKey = \k f -> wrapper $ storeKey r k f + , retrieveKeyFile = \k f d -> wrapper $ retrieveKeyFile r k f d , retrieveKeyFileCheap = \k f -> wrapper $ retrieveKeyFileCheap r k f , removeKey = \k -> wrapper $ removeKey r k , hasKey = \k -> wrapper $ hasKey r k diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 5fb793e65f..9e8d3c620d 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -101,8 +101,8 @@ runHook hooktype hook k f a = maybe (return False) run =<< lookupHook hooktype h return False ) -store :: String -> Key -> Annex Bool -store h k = do +store :: String -> Key -> AssociatedFile -> Annex Bool +store h k _f = do src <- inRepo $ gitAnnexLocation k runHook h "store" k (Just src) $ return True @@ -112,8 +112,8 @@ storeEncrypted h (cipher, enck) k = withTmp enck $ \tmp -> do liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp runHook h "store" enck (Just tmp) $ return True -retrieve :: String -> Key -> FilePath -> Annex Bool -retrieve h k f = runHook h "retrieve" k (Just f) $ return True +retrieve :: String -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve h k _f d = runHook h "retrieve" k (Just d) $ return True retrieveCheap :: String -> Key -> FilePath -> Annex Bool retrieveCheap _ _ _ = return False diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 6207e14253..887c68339a 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -99,8 +99,8 @@ rsyncUrls o k = map use annexHashes use h = rsyncUrl o h k rsyncEscape o (f f) f = keyFile k -store :: RsyncOpts -> Key -> Annex Bool -store o k = rsyncSend o k <=< inRepo $ gitAnnexLocation k +store :: RsyncOpts -> Key -> AssociatedFile -> Annex Bool +store o k _f = rsyncSend o k <=< inRepo $ gitAnnexLocation k storeEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> Annex Bool storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do @@ -108,8 +108,8 @@ storeEncrypted o (cipher, enck) k = withTmp enck $ \tmp -> do liftIO $ withEncryptedContent cipher (L.readFile src) $ L.writeFile tmp rsyncSend o enck tmp -retrieve :: RsyncOpts -> Key -> FilePath -> Annex Bool -retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o +retrieve :: RsyncOpts -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve o k _ f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o -- use inplace when retrieving to support resuming [ Param "--inplace" , Param u @@ -117,11 +117,11 @@ retrieve o k f = untilTrue (rsyncUrls o k) $ \u -> rsyncRemote o ] retrieveCheap :: RsyncOpts -> Key -> FilePath -> Annex Bool -retrieveCheap o k f = ifM (preseedTmp k f) ( retrieve o k f , return False ) +retrieveCheap o k f = ifM (preseedTmp k f) ( retrieve o k undefined f , return False ) retrieveEncrypted :: RsyncOpts -> (Cipher, Key) -> Key -> FilePath -> Annex Bool retrieveEncrypted o (cipher, enck) _ f = withTmp enck $ \tmp -> do - ifM (retrieve o enck tmp) + ifM (retrieve o enck undefined tmp) ( liftIO $ catchBoolIO $ do withDecryptedContent cipher (L.readFile tmp) $ L.writeFile f return True diff --git a/Remote/S3.hs b/Remote/S3.hs index 18d4915dcb..dca08fff8b 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -113,8 +113,8 @@ s3Setup u c = handlehost $ M.lookup "host" c -- be human-readable M.delete "bucket" defaults -store :: Remote -> Key -> Annex Bool -store r k = s3Action r False $ \(conn, bucket) -> do +store :: Remote -> Key -> AssociatedFile -> Annex Bool +store r k _f = s3Action r False $ \(conn, bucket) -> do dest <- inRepo $ gitAnnexLocation k res <- liftIO $ storeHelper (conn, bucket) r k dest s3Bool res @@ -149,12 +149,12 @@ storeHelper (conn, bucket) r k file = do xheaders = filter isxheader $ M.assocs $ fromJust $ config r isxheader (h, _) = "x-amz-" `isPrefixOf` h -retrieve :: Remote -> Key -> FilePath -> Annex Bool -retrieve r k f = s3Action r False $ \(conn, bucket) -> do +retrieve :: Remote -> Key -> AssociatedFile -> FilePath -> Annex Bool +retrieve r k _f d = s3Action r False $ \(conn, bucket) -> do res <- liftIO $ getObject conn $ bucketKey r bucket k case res of Right o -> do - liftIO $ L.writeFile f $ obj_data o + liftIO $ L.writeFile d $ obj_data o return True Left e -> s3Warning e diff --git a/Remote/Web.hs b/Remote/Web.hs index 5fc592326c..2516240ab3 100644 --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -51,21 +51,21 @@ gen r _ _ = remotetype = remote } -downloadKey :: Key -> FilePath -> Annex Bool -downloadKey key file = get =<< getUrls key +downloadKey :: Key -> AssociatedFile -> FilePath -> Annex Bool +downloadKey key _file dest = get =<< getUrls key where get [] = do warning "no known url" return False get urls = do showOutput -- make way for download progress bar - downloadUrl urls file + downloadUrl urls dest downloadKeyCheap :: Key -> FilePath -> Annex Bool downloadKeyCheap _ _ = return False -uploadKey :: Key -> Annex Bool -uploadKey _ = do +uploadKey :: Key -> AssociatedFile -> Annex Bool +uploadKey _ _ = do warning "upload to web not supported" return False diff --git a/Types/Remote.hs b/Types/Remote.hs index 9bac2ca0f8..c7628165c7 100644 --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -33,6 +33,9 @@ data RemoteTypeA a = RemoteType { instance Eq (RemoteTypeA a) where x == y = typename x == typename y +{- A filename associated with a Key, for display to user. -} +type AssociatedFile = Maybe FilePath + {- An individual remote. -} data RemoteA a = Remote { -- each Remote has a unique uuid @@ -42,9 +45,9 @@ data RemoteA a = Remote { -- Remotes have a use cost; higher is more expensive cost :: Int, -- Transfers a key to the remote. - storeKey :: Key -> a Bool, + storeKey :: Key -> AssociatedFile -> a Bool, -- retrieves a key's contents to a file - retrieveKeyFile :: Key -> FilePath -> a Bool, + retrieveKeyFile :: Key -> AssociatedFile -> FilePath -> a Bool, -- retrieves a key's contents to a tmp file, if it can be done cheaply retrieveKeyFileCheap :: Key -> FilePath -> a Bool, -- removes a key's contents diff --git a/debian/changelog b/debian/changelog index babd1786de..c279614ca9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ git-annex (3.20120630) UNRELEASED; urgency=low * get, move, copy: Now refuse to do anything when the requested file transfer is already in progress by another process. + * status: Lists transfers that are currently in progress. -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 From c53da2b04a2e802253bfbbfd4e00e02807d6de77 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 17:11:20 -0400 Subject: [PATCH 3987/8313] blog for the day --- .../blog/day_21__transfer_tracking.mdwn | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 doc/design/assistant/blog/day_21__transfer_tracking.mdwn diff --git a/doc/design/assistant/blog/day_21__transfer_tracking.mdwn b/doc/design/assistant/blog/day_21__transfer_tracking.mdwn new file mode 100644 index 0000000000..79c0b64387 --- /dev/null +++ b/doc/design/assistant/blog/day_21__transfer_tracking.mdwn @@ -0,0 +1,28 @@ +Worked today on two action items from my last blog post: + +* on-disk transfers in progress information files (read/write/enumerate) +* locking for the files, so redundant transfer races can be detected, + and failed transfers noticed + +That's all done, and used by the `get`, `copy`, and `move` subcommands. + +Also, I made `git-annex status` use that information to display any +file transfers that are currently in progress: + + joey@gnu:~/lib/sound/misc>git annex status + [...] + transfers in progress: + downloading Vic-303.mp3 from leech + +(Webapp, here we come!) + +However... Files being sent or received by `git-annex-shell` don't yet +have this transfer info recorded. The problem is that to do so, +`git-annex-shell` will need to be run with a `--remote=` parameter. But +old versions will of course fail when run with such an unknown parameter. + +This is a problem I last faced in December 2011 when adding the `--uuid=` +parameter. That time I punted and required the remote `git-annex-shell` be +updated to a new enough version to accept it. But as git-annex gets more widely +used and packaged, that's becoming less an option. I need to find a real +solution to this problem. From 2d2bfe9809f8d8d5862bc12fbe40c2e25b2405a3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 1 Jul 2012 20:55:20 -0400 Subject: [PATCH 3988/8313] reorg --- .../blog/day_20__data_transfer_design.mdwn | 33 +---------------- doc/design/assistant/syncing.mdwn | 37 ++++++++++++++++--- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/doc/design/assistant/blog/day_20__data_transfer_design.mdwn b/doc/design/assistant/blog/day_20__data_transfer_design.mdwn index 2733f09bc4..4f47ae63c4 100644 --- a/doc/design/assistant/blog/day_20__data_transfer_design.mdwn +++ b/doc/design/assistant/blog/day_20__data_transfer_design.mdwn @@ -2,8 +2,8 @@ Today is a planning day. I have only a few days left before I'm off to Nicaragua for [DebConf](http://debconf12.debconf.org/), where I'll only have smaller chunks of time without interruptions. So it's important to get some well-defined smallish chunks designed that I can work on later. See -bulleted action items below. Each should be around 1-2 hours unless it -turns out to be 8 hours... :) +bulleted action items below (now moved to [[syncing]]. Each +should be around 1-2 hours unless it turns out to be 8 hours... :) First, worked on writing down a design, and some data types, for data transfer tracking (see [[syncing]] page). Found that writing down these simple data @@ -14,38 +14,9 @@ to record on disk what transfers it's doing, so the assistant can get that information and use it to both avoid redundant transfers (potentially a big problem!), and later to allow the user to control them using the web app. -So these will be the first steps as I move toward implementing data -transfer tracking and naive flood fill transferring. - -* on-disk transfers in progress information files (read/write/enumerate) -* locking for the files, so redundant transfer races can be detected, - and failed transfers noticed -* update files as transfers proceed. See [[progressbars]] - (updating for downloads is easy; for uploads is hard) -* add Transfer queue TChan -* enqueue Transfers (Uploads) as new files are added to the annex by - Watcher. -* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by - Watcher. -* add TransferInfo Map to DaemonStatus for tracking transfers in progress. -* Poll transfer in progress info files for changes (use inotify again! - wow! hammer, meet nail..), and update the TransferInfo Map -* Write basic Transfer handling thread. Multiple such threads need to be - able to be run at once. Each will need its own independant copy of the - Annex state monad. -* Write transfer control thread, which decides when to launch transfers. -* At startup, and possibly periodically, look for files we have that - location tracking indicates remotes do not, and enqueue Uploads for - them. Also, enqueue Downloads for any files we're missing. - While eventually the user will be able to use the web app to prioritize transfers, stop and start, throttle, etc, it's important to get the default behavior right. So I'm thinking about things like how to prioritize uploads vs downloads, when it's appropriate to have multiple downloads running at once, etc. -* Find a way to probe available outgoing bandwidth, to throttle so - we don't bufferbloat the network to death. -* git-annex needs a simple speed control knob, which can be plumbed - through to, at least, rsync. A good job for an hour in an - airport somewhere. diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index ce7f9673b5..c18badb533 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -1,6 +1,37 @@ Once files are added (or removed or moved), need to send those changes to all the other git clones, at both the git level and the key/value level. +## action items + +* on-disk transfers in progress information files (read/write/enumerate) + **done** +* locking for the files, so redundant transfer races can be detected, + and failed transfers noticed **done** +* transfer info for git-annex-shell (problem: how to add a switch + with the necessary info w/o breaking backwards compatability?) +* update files as transfers proceed. See [[progressbars]] + (updating for downloads is easy; for uploads is hard) +* add Transfer queue TChan +* enqueue Transfers (Uploads) as new files are added to the annex by + Watcher. +* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by + Watcher. +* add TransferInfo Map to DaemonStatus for tracking transfers in progress. +* Poll transfer in progress info files for changes (use inotify again! + wow! hammer, meet nail..), and update the TransferInfo Map +* Write basic Transfer handling thread. Multiple such threads need to be + able to be run at once. Each will need its own independant copy of the + Annex state monad. +* Write transfer control thread, which decides when to launch transfers. +* At startup, and possibly periodically, look for files we have that + location tracking indicates remotes do not, and enqueue Uploads for + them. Also, enqueue Downloads for any files we're missing. +* Find a way to probe available outgoing bandwidth, to throttle so + we don't bufferbloat the network to death. +* git-annex needs a simple speed control knob, which can be plumbed + through to, at least, rsync. A good job for an hour in an + airport somewhere. + ## git syncing 1. Can use `git annex sync`, which already handles bidirectional syncing. @@ -55,12 +86,6 @@ anyway. (May sometimes want multiple threads downloading, or uploading, or even both.) type TransferQueue = TChan [Transfer] - data Transfer = Upload Key Remote | Download Key Remote - - data TransferID = TransferThread ThreadID | TransferProcess Pid - type BytesComplete = Integer - type StartedTime = EpochTime - data TransferInfo = TransferInfo TransferID StartedTime BytesComplete -- add (M.Map Transfer TransferInfo) to DaemonStatus startTransfer :: Transfer -> Annex TransferID From d1f49b0ad032f13adc39d963cc8ceca28215b1d5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 00:53:00 -0400 Subject: [PATCH 3989/8313] add fields to git-annex-shell --- CmdLine.hs | 5 +++-- GitAnnex.hs | 2 +- GitAnnexShell.hs | 29 ++++++++++++++++++++--------- Utility/Misc.hs | 10 ++++++++++ doc/git-annex-shell.mdwn | 8 ++++++++ 5 files changed, 42 insertions(+), 12 deletions(-) diff --git a/CmdLine.hs b/CmdLine.hs index 910f228b60..edbe5e107d 100644 --- a/CmdLine.hs +++ b/CmdLine.hs @@ -30,8 +30,8 @@ type Params = [String] type Flags = [Annex ()] {- Runs the passed command line. -} -dispatch :: Bool -> Params -> [Command] -> [Option] -> String -> IO Git.Repo -> IO () -dispatch fuzzyok allargs allcmds commonoptions header getgitrepo = do +dispatch :: Bool -> Params -> [Command] -> [Option] -> [(String, String)] -> String -> IO Git.Repo -> IO () +dispatch fuzzyok allargs allcmds commonoptions fields header getgitrepo = do setupConsole r <- E.try getgitrepo :: IO (Either E.SomeException Git.Repo) case r of @@ -40,6 +40,7 @@ dispatch fuzzyok allargs allcmds commonoptions header getgitrepo = do state <- Annex.new g (actions, state') <- Annex.run state $ do checkfuzzy + forM_ fields $ \(f, v) -> Annex.setField f v sequence_ flags prepCommand cmd params tryRun state' cmd $ [startup] ++ actions ++ [shutdown $ cmdoneshot cmd] diff --git a/GitAnnex.hs b/GitAnnex.hs index 8dba5a3a7a..bf1f27bfda 100644 --- a/GitAnnex.hs +++ b/GitAnnex.hs @@ -143,4 +143,4 @@ header :: String header = "Usage: git-annex command [option ..]" run :: [String] -> IO () -run args = dispatch True args cmds options header Git.CurrentRepo.get +run args = dispatch True args cmds options [] header Git.CurrentRepo.get diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs index 6633037138..2a9f3c26a8 100644 --- a/GitAnnexShell.hs +++ b/GitAnnexShell.hs @@ -47,7 +47,8 @@ cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly options :: [OptDescr (Annex ())] options = Option.common ++ - [ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "repository uuid" + [ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "local repository uuid" + , Option [] ["remote-uuid"] (ReqArg checkuuid paramUUID) "remote repository uuid" ] where checkuuid expected = getUUID >>= check @@ -83,21 +84,31 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do checkNotReadOnly cmd - dispatch False (cmd : filterparams params) cmds options header $ + let (params', fields) = partitionParams params + dispatch False (cmd : params') cmds options (parseFields fields) header $ Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath external :: [String] -> IO () external params = do checkNotLimited - unlessM (boolSystem "git-shell" $ map Param $ "-c":filterparams params) $ + unlessM (boolSystem "git-shell" $ map Param $ "-c":fst (partitionParams params)) $ error "git-shell failed" --- Drop all args after "--". --- These tend to be passed by rsync and not useful. -filterparams :: [String] -> [String] -filterparams [] = [] -filterparams ("--":_) = [] -filterparams (a:as) = a:filterparams as +{- Parameters between two -- markers are field settings, in the form: + - field=value field=value + - + - Parameters after the last -- are ignored, these tend to be passed by + - rsync and not be useful. + -} +partitionParams :: [String] -> ([String], [String]) +partitionParams params + | length segments < 2 = (segments !! 0, []) + | otherwise = (segments !! 0, segments !! 1) + where + segments = segment (== "--") params + +parseFields :: [String] -> [(String, String)] +parseFields = map (separate (== '=')) failure :: IO () failure = error $ "bad parameters\n\n" ++ usage header cmds options diff --git a/Utility/Misc.hs b/Utility/Misc.hs index 3ac5ca5c0b..3b359139b9 100644 --- a/Utility/Misc.hs +++ b/Utility/Misc.hs @@ -35,3 +35,13 @@ separate c l = unbreak $ break c l {- Breaks out the first line. -} firstLine :: String-> String firstLine = takeWhile (/= '\n') + +{- Splits a list into segments that are delimited by items matching + - a predicate. (The delimiters are not included in the segments.) -} +segment :: (a -> Bool) -> [a] -> [[a]] +segment p l = map reverse $ go [] [] l + where + go c r [] = reverse $ c:r + go c r (i:is) + | p i = go [] (c:r) is + | otherwise = go (i:c) r is diff --git a/doc/git-annex-shell.mdwn b/doc/git-annex-shell.mdwn index 00c68ff3a5..20a9d3d378 100644 --- a/doc/git-annex-shell.mdwn +++ b/doc/git-annex-shell.mdwn @@ -61,6 +61,14 @@ to git-annex-shell are: git-annex uses this to specify the UUID of the repository it was expecting git-annex-shell to access, as a sanity check. +* -- fields=val fields=val.. -- + + Additional fields may be specified this way, to retain compatability with + past versions of git-annex-shell (that ignore these, but would choke + on new dashed options). + + Currently used fields include remoteuuid= and associatedfile= + # HOOK After content is received or dropped from the repository by git-annex-shell, From bea0ac0274861f639ef999b146a719f4300fbfe4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 01:31:10 -0400 Subject: [PATCH 3990/8313] record transfers for git-annex-shell Not yet tested and places git-annex-shell is run need to be modified to pass the new field settings. Note that rsyncServerSend was changed to fork, rather than directly exec rsync, because it needs to keep the transfer lock held, and clean up the transfer log when done. --- Command/RecvKey.hs | 24 +++++++++++++----------- Command/SendKey.hs | 18 +++++++++++------- GitAnnexShell.hs | 18 ++++++++++++++++-- Logs/Transfer.hs | 7 +++++++ Utility/RsyncFile.hs | 11 ++++------- 5 files changed, 51 insertions(+), 27 deletions(-) diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs index 9744a56d4a..ce8bff9975 100644 --- a/Command/RecvKey.hs +++ b/Command/RecvKey.hs @@ -12,6 +12,7 @@ import Command import CmdLine import Annex.Content import Utility.RsyncFile +import Logs.Transfer def :: [Command] def = [oneShot $ command "recvkey" paramKey seek @@ -21,14 +22,15 @@ seek :: [CommandSeek] seek = [withKeys start] start :: Key -> CommandStart -start key = do - whenM (inAnnex key) $ error "key is already present in annex" - - ok <- getViaTmp key (liftIO . rsyncServerReceive) - if ok - then do - -- forcibly quit after receiving one key, - -- and shutdown cleanly - _ <- shutdown True - liftIO exitSuccess - else liftIO exitFailure +start key = ifM (inAnnex key) + ( error "key is already present in annex" + , fieldTransfer Download key $ do + ifM (getViaTmp key $ liftIO . rsyncServerReceive) + ( do + -- forcibly quit after receiving one key, + -- and shutdown cleanly + _ <- shutdown True + liftIO exitSuccess + , liftIO exitFailure + ) + ) diff --git a/Command/SendKey.hs b/Command/SendKey.hs index 686a31caa7..5eca70d24c 100644 --- a/Command/SendKey.hs +++ b/Command/SendKey.hs @@ -1,6 +1,6 @@ {- git-annex command - - - Copyright 2010 Joey Hess + - Copyright 2010,2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -11,6 +11,7 @@ import Common.Annex import Command import Annex.Content import Utility.RsyncFile +import Logs.Transfer def :: [Command] def = [oneShot $ command "sendkey" paramKey seek @@ -20,9 +21,12 @@ seek :: [CommandSeek] seek = [withKeys start] start :: Key -> CommandStart -start key = do - file <- inRepo $ gitAnnexLocation key - whenM (inAnnex key) $ - liftIO $ rsyncServerSend file -- does not return - warning "requested key is not present" - liftIO exitFailure +start key = ifM (inAnnex key) + ( fieldTransfer Upload key $ do + file <- inRepo $ gitAnnexLocation key + liftIO $ ifM (rsyncServerSend file) + ( exitSuccess , exitFailure ) + , do + warning "requested key is not present" + liftIO exitFailure + ) diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs index 2a9f3c26a8..559e302358 100644 --- a/GitAnnexShell.hs +++ b/GitAnnexShell.hs @@ -9,6 +9,7 @@ module GitAnnexShell where import System.Environment import System.Console.GetOpt +import Data.Char import Common.Annex import qualified Git.Construct @@ -84,8 +85,9 @@ builtins = map cmdname cmds builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do checkNotReadOnly cmd - let (params', fields) = partitionParams params - dispatch False (cmd : params') cmds options (parseFields fields) header $ + let (params', fieldparams) = partitionParams params + fields <- filterM checkField $ parseFields fieldparams + dispatch False (cmd : params') cmds options fields header $ Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath external :: [String] -> IO () @@ -110,6 +112,18 @@ partitionParams params parseFields :: [String] -> [(String, String)] parseFields = map (separate (== '=')) +{- Only allow known fields to be set, ignore others. + - Make sure that field values make sense. -} +checkField :: (String, String) -> IO Bool +checkField (field, value) + | field == "remoteuuid" = return $ + -- does it look like a UUID? + all (\c -> isAlphaNum c || c == '-') value + | field == "associatedfile" = + -- is the file located within the current directory? + dirContains <$> getCurrentDirectory <*> pure value + | otherwise = return False + failure :: IO () failure = error $ "bad parameters\n\n" ++ usage header cmds options diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 526241f935..658e18b5d0 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -10,6 +10,7 @@ module Logs.Transfer where import Common.Annex import Annex.Perms import Annex.Exception +import qualified Annex import qualified Git import Types.Remote @@ -54,6 +55,12 @@ upload u key file a = transfer (Transfer Upload u key) file a download :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a download u key file a = transfer (Transfer Download u key) file a +fieldTransfer :: Direction -> Key -> Annex a -> Annex a +fieldTransfer direction key a = do + afile <- Annex.getField "associatedfile" + maybe a (\u -> transfer (Transfer direction (toUUID u) key) afile a) + =<< Annex.getField "remoteuuid" + {- Runs a transfer action. Creates and locks the transfer information file - while the action is running. Will throw an error if the transfer is - already in progress. diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs index db9057843c..075e91d239 100644 --- a/Utility/RsyncFile.hs +++ b/Utility/RsyncFile.hs @@ -22,9 +22,9 @@ rsyncShell command = [Param "-e", Param $ unwords $ map escape (toCommand comman - string is a single quote. -} escape s = "'" ++ join "''" (split "'" s) ++ "'" -{- Runs rsync in server mode to send a file, and exits. -} -rsyncServerSend :: FilePath -> IO () -rsyncServerSend file = rsyncExec $ +{- Runs rsync in server mode to send a file. -} +rsyncServerSend :: FilePath -> IO Bool +rsyncServerSend file = rsync $ rsyncServerParams ++ [Param "--sender", File file] {- Runs rsync in server mode to receive a file. -} @@ -47,11 +47,8 @@ rsyncServerParams = rsync :: [CommandParam] -> IO Bool rsync = boolSystem "rsync" -rsyncExec :: [CommandParam] -> IO () -rsyncExec params = executeFile "rsync" True (toCommand params) Nothing - {- Checks if an rsync url involves the remote shell (ssh or rsh). - - Use of such urls with rsync or rsyncExec requires additional shell + - Use of such urls with rsync requires additional shell - escaping. -} rsyncUrlIsShell :: String -> Bool rsyncUrlIsShell s From bab6dc48d3ee09036fa555fa5ddc1475c9ce87d4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 07:51:55 -0400 Subject: [PATCH 3991/8313] official Fedora builds now available for F16, F17 --- doc/install/Fedora.mdwn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn index 50f1d78180..8aacbb3b09 100644 --- a/doc/install/Fedora.mdwn +++ b/doc/install/Fedora.mdwn @@ -15,4 +15,6 @@ cabal install --bindir=$HOME/bin Note: You can't just use `cabal install git-annex`, because Fedora does not yet ship ghc 7.4. -[Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259) +* [Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259)a +* [Koji build for F17](http://koji.fedoraproject.org/koji/buildinfo?buildID=328654) +* [Koji build for F16](http://koji.fedoraproject.org/koji/buildinfo?buildID=328656) From 9517fbb9488aac6750b9599db358da8d72a2343e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 08:35:15 -0400 Subject: [PATCH 3992/8313] cleanup --- Fields.hs | 32 ++++++++++++++++++++++++++++++++ GitAnnexShell.hs | 11 +++-------- Logs/Transfer.hs | 6 +++--- Option.hs | 1 - 4 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 Fields.hs diff --git a/Fields.hs b/Fields.hs new file mode 100644 index 0000000000..08189cbdfa --- /dev/null +++ b/Fields.hs @@ -0,0 +1,32 @@ +{- git-annex fields + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Fields where + +import Common.Annex +import qualified Annex + +import Data.Char + +{- A field, stored in Annex state, with a value sanity checker. -} +data Field = Field + { fieldName :: String + , fieldCheck :: String -> IO Bool + } + +remoteUUID :: Field +remoteUUID = Field "remoteuuid" $ + -- does it look like a UUID? + return . all (\c -> isAlphaNum c || c == '-') + +associatedFile :: Field +associatedFile = Field "associatedfile" $ \value -> + -- is the file located within the current directory? + dirContains <$> getCurrentDirectory <*> pure value + +getField :: Field -> Annex (Maybe String) +getField = Annex.getField . fieldName diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs index 559e302358..497e4cf8f1 100644 --- a/GitAnnexShell.hs +++ b/GitAnnexShell.hs @@ -9,7 +9,6 @@ module GitAnnexShell where import System.Environment import System.Console.GetOpt -import Data.Char import Common.Annex import qualified Git.Construct @@ -17,6 +16,7 @@ import CmdLine import Command import Annex.UUID import qualified Option +import Fields import qualified Command.ConfigList import qualified Command.InAnnex @@ -49,7 +49,6 @@ cmds = map adddirparam $ cmds_readonly ++ cmds_notreadonly options :: [OptDescr (Annex ())] options = Option.common ++ [ Option [] ["uuid"] (ReqArg checkuuid paramUUID) "local repository uuid" - , Option [] ["remote-uuid"] (ReqArg checkuuid paramUUID) "remote repository uuid" ] where checkuuid expected = getUUID >>= check @@ -116,12 +115,8 @@ parseFields = map (separate (== '=')) - Make sure that field values make sense. -} checkField :: (String, String) -> IO Bool checkField (field, value) - | field == "remoteuuid" = return $ - -- does it look like a UUID? - all (\c -> isAlphaNum c || c == '-') value - | field == "associatedfile" = - -- is the file located within the current directory? - dirContains <$> getCurrentDirectory <*> pure value + | field == fieldName remoteUUID = fieldCheck remoteUUID value + | field == fieldName associatedFile = fieldCheck associatedFile value | otherwise = return False failure :: IO () diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 658e18b5d0..dbd45f1991 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -10,9 +10,9 @@ module Logs.Transfer where import Common.Annex import Annex.Perms import Annex.Exception -import qualified Annex import qualified Git import Types.Remote +import qualified Fields import Control.Concurrent import System.Posix.Types @@ -57,9 +57,9 @@ download u key file a = transfer (Transfer Download u key) file a fieldTransfer :: Direction -> Key -> Annex a -> Annex a fieldTransfer direction key a = do - afile <- Annex.getField "associatedfile" + afile <- Fields.getField Fields.associatedFile maybe a (\u -> transfer (Transfer direction (toUUID u) key) afile a) - =<< Annex.getField "remoteuuid" + =<< Fields.getField Fields.remoteUUID {- Runs a transfer action. Creates and locks the transfer information file - while the action is running. Will throw an error if the transfer is diff --git a/Option.hs b/Option.hs index 1bac2cd050..967cd3e07b 100644 --- a/Option.hs +++ b/Option.hs @@ -76,4 +76,3 @@ field short opt paramdesc description = {- The flag or field name used for an option. -} name :: Option -> String name (Option _ o _ _) = Prelude.head o - From 74f0d67aa3988a71f3a53b88de4344272d924b95 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 10:56:26 -0400 Subject: [PATCH 3993/8313] avoid untrappable exception if dirContentsRecursive is run on a directory that doesn't exist, or cannot be read The problem is its use of unsafeInterleaveIO, which causes its IO code to run when the thunk is forced, outside any exception trapping the caller may do. --- Utility/Directory.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Utility/Directory.hs b/Utility/Directory.hs index 2f2960a9d8..057da60876 100644 --- a/Utility/Directory.hs +++ b/Utility/Directory.hs @@ -35,14 +35,15 @@ dirContents :: FilePath -> IO [FilePath] dirContents d = map (d ) . filter (not . dirCruft) <$> getDirectoryContents d {- Gets files in a directory, and then its subdirectories, recursively, - - and lazily. -} + - and lazily. If the directory does not exist, no exception is thrown, + - instead, [] is returned. -} dirContentsRecursive :: FilePath -> IO [FilePath] dirContentsRecursive topdir = dirContentsRecursive' topdir [""] dirContentsRecursive' :: FilePath -> [FilePath] -> IO [FilePath] dirContentsRecursive' _ [] = return [] dirContentsRecursive' topdir (dir:dirs) = unsafeInterleaveIO $ do - (files, dirs') <- collect [] [] =<< dirContents (topdir dir) + (files, dirs') <- collect [] [] =<< catchDefaultIO (dirContents (topdir dir)) [] files' <- dirContentsRecursive' topdir (dirs' ++ dirs) return (files ++ files') where From 760e028dca493364b2d8277447b91592b99415a3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 10:57:51 -0400 Subject: [PATCH 3994/8313] pass associatedfile and remoteuuid to git-annex-shell This *almost* works. Along the way, I noticed that the --uuid parameter was being accidentially passed after the --, so that has never been actually used by git-annex-shell to verify it's running in the expected repository. Oops. Fixed. --- Command/Map.hs | 2 +- Remote/Git.hs | 35 ++++++++++++++++++++--------------- Remote/Helper/Ssh.hs | 23 ++++++++++++++++------- debian/changelog | 1 + 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/Command/Map.hs b/Command/Map.hs index 86e9609a7e..65e28945f6 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -203,7 +203,7 @@ tryScan r Git.Config.hRead r configlist = - onRemote r (pipedconfig, Nothing) "configlist" [] + onRemote r (pipedconfig, Nothing) "configlist" [] [] manualconfiglist = do sshparams <- sshToRepo r [Param sshcmd] liftIO $ pipedconfig "ssh" sshparams diff --git a/Remote/Git.hs b/Remote/Git.hs index 0b839c9a5e..d80f580fc5 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -31,6 +31,7 @@ import Utility.TempFile import Config import Init import Types.Key +import qualified Fields remote :: RemoteType remote = RemoteType { @@ -111,7 +112,7 @@ guardUsable r onerr a tryGitConfigRead :: Git.Repo -> Annex Git.Repo tryGitConfigRead r | not $ M.null $ Git.config r = return r -- already read - | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] + | Git.repoIsSsh r = store $ onRemote r (pipedconfig, r) "configlist" [] [] | Git.repoIsHttp r = do headers <- getHttpHeaders store $ safely $ geturlconfig headers @@ -171,7 +172,7 @@ inAnnex r key v -> return v checkremote = do showAction $ "checking " ++ Git.repoDescribe r - onRemote r (check, unknown) "inannex" [Param (show key)] + onRemote r (check, unknown) "inannex" [Param (show key)] [] where check c p = dispatch <$> safeSystem c p dispatch ExitSuccess = Right True @@ -218,6 +219,7 @@ dropKey r key [ Params "--quiet --force" , Param $ show key ] + [] {- Tries to copy a key's content from a remote's annex to a file. -} copyFromRemote :: Git.Repo -> Key -> AssociatedFile -> FilePath -> Annex Bool @@ -231,7 +233,7 @@ copyFromRemote r key file dest loc <- inRepo $ gitAnnexLocation key upload u key file $ rsyncOrCopyFile params loc dest - | Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key dest + | Git.repoIsSsh r = rsyncHelper =<< rsyncParamsRemote r True key dest file | Git.repoIsHttp r = Annex.Content.downloadUrl (keyUrls r key) dest | otherwise = error "copying from non-ssh, non-http repo not supported" @@ -263,7 +265,7 @@ copyToRemote r key file (rsyncOrCopyFile params keysrc) | Git.repoIsSsh r = commitOnCleanup r $ do keysrc <- inRepo $ gitAnnexLocation key - rsyncHelper =<< rsyncParamsRemote r False key keysrc + rsyncHelper =<< rsyncParamsRemote r False key keysrc file | otherwise = error "copying to non-ssh repo not supported" rsyncHelper :: [CommandParam] -> Annex Bool @@ -290,23 +292,26 @@ rsyncOrCopyFile rsyncparams src dest = {- Generates rsync parameters that ssh to the remote and asks it - to either receive or send the key's content. -} -rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> Annex [CommandParam] -rsyncParamsRemote r sending key file = do +rsyncParamsRemote :: Git.Repo -> Bool -> Key -> FilePath -> AssociatedFile -> Annex [CommandParam] +rsyncParamsRemote r sending key file afile = do + u <- getUUID + let fields = (Fields.remoteUUID, fromUUID u) + : maybe [] (\f -> [(Fields.associatedFile, f)]) afile Just (shellcmd, shellparams) <- git_annex_shell r (if sending then "sendkey" else "recvkey") - [ Param $ show key - -- Command is terminated with "--", because - -- rsync will tack on its own options afterwards, - -- and they need to be ignored. - , Param "--" - ] + [ Param $ show key ] + fields -- Convert the ssh command into rsync command line. let eparam = rsyncShell (Param shellcmd:shellparams) o <- rsyncParams r if sending - then return $ o ++ eparam ++ [dummy, File file] - else return $ o ++ eparam ++ [File file, dummy] + then return $ o ++ rsyncopts eparam dummy (File file) + else return $ o ++ rsyncopts eparam (File file) dummy where + rsyncopts ps source dest + | end ps == [dashdash] = ps ++ [source, dest] + | otherwise = ps ++ [dashdash, source, dest] + dashdash = Param "--" -- The rsync shell parameter controls where rsync -- goes, so the source/dest parameter can be a dummy value, -- that just enables remote rsync mode. @@ -333,7 +338,7 @@ commitOnCleanup r a = go `after` a Annex.Branch.commit "update" | otherwise = void $ do Just (shellcmd, shellparams) <- - git_annex_shell r "commit" [] + git_annex_shell r "commit" [] [] -- Throw away stderr, since the remote may not -- have a new enough git-annex shell to -- support committing. diff --git a/Remote/Helper/Ssh.hs b/Remote/Helper/Ssh.hs index f6742b89f6..4434bc65db 100644 --- a/Remote/Helper/Ssh.hs +++ b/Remote/Helper/Ssh.hs @@ -1,6 +1,6 @@ {- git-annex remote access with ssh - - - Copyright 2011 Joey Hess + - Copyright 2011.2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -13,6 +13,7 @@ import qualified Git.Url import Config import Annex.UUID import Annex.Ssh +import Fields {- Generates parameters to ssh to a repository's host and run a command. - Caller is responsible for doing any neccessary shellEscaping of the @@ -25,9 +26,9 @@ sshToRepo repo sshcmd = do {- Generates parameters to run a git-annex-shell command on a remote - repository. -} -git_annex_shell :: Git.Repo -> String -> [CommandParam] -> Annex (Maybe (FilePath, [CommandParam])) -git_annex_shell r command params - | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts) +git_annex_shell :: Git.Repo -> String -> [CommandParam] -> [(Field, String)] -> Annex (Maybe (FilePath, [CommandParam])) +git_annex_shell r command params fields + | not $ Git.repoIsUrl r = return $ Just (shellcmd, shellopts ++ fieldopts) | Git.repoIsSsh r = do uuid <- getRepoUUID r sshparams <- sshToRepo r [Param $ sshcmd uuid ] @@ -39,9 +40,16 @@ git_annex_shell r command params shellopts = Param command : File dir : params sshcmd uuid = unwords $ shellcmd : map shellEscape (toCommand shellopts) ++ - uuidcheck uuid + uuidcheck uuid ++ + map shellEscape (toCommand fieldopts) uuidcheck NoUUID = [] uuidcheck (UUID u) = ["--uuid", u] + fieldopts + | null fields = [] + | otherwise = fieldsep : map fieldopt fields ++ [fieldsep] + fieldsep = Param "--" + fieldopt (field, value) = Param $ + fieldName field ++ "=" ++ value {- Uses a supplied function (such as boolSystem) to run a git-annex-shell - command on a remote. @@ -53,9 +61,10 @@ onRemote -> (FilePath -> [CommandParam] -> IO a, a) -> String -> [CommandParam] + -> [(Field, String)] -> Annex a -onRemote r (with, errorval) command params = do - s <- git_annex_shell r command params +onRemote r (with, errorval) command params fields = do + s <- git_annex_shell r command params fields case s of Just (c, ps) -> liftIO $ with c ps Nothing -> return errorval diff --git a/debian/changelog b/debian/changelog index c279614ca9..33c850861b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ git-annex (3.20120630) UNRELEASED; urgency=low * get, move, copy: Now refuse to do anything when the requested file transfer is already in progress by another process. * status: Lists transfers that are currently in progress. + * Fix passing --uuid to git-annex-shell. -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 From 8f6c2e6081d8e162f34ff5406e8d564dc1b5f4a5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 11:02:47 -0400 Subject: [PATCH 3995/8313] fix reading of empty filename from transfer info file --- Logs/Transfer.hs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index dbd45f1991..6bdc750f4b 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -165,10 +165,8 @@ readTransferInfo pid s = <*> pure (Just pid) <*> pure Nothing <*> pure Nothing - <*> pure filename + <*> pure (if null filename then Nothing else Just filename) _ -> Nothing where (bits, filebits) = splitAt 1 $ lines s - filename - | null filebits = Nothing - | otherwise = Just $ join "\n" filebits + filename = join "\n" filebits From bdcabb3cfa0a7d14a35a6bcf34f9379e8900f556 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 11:08:50 -0400 Subject: [PATCH 3996/8313] fix associatedfile sanity check It seems best to require that the file just be relative, and not some ../ trick. git-annex-shell sendkey and recvkey both update transfer information now --- Fields.hs | 10 +++++----- GitAnnexShell.hs | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Fields.hs b/Fields.hs index 08189cbdfa..38427ad057 100644 --- a/Fields.hs +++ b/Fields.hs @@ -15,18 +15,18 @@ import Data.Char {- A field, stored in Annex state, with a value sanity checker. -} data Field = Field { fieldName :: String - , fieldCheck :: String -> IO Bool + , fieldCheck :: String -> Bool } remoteUUID :: Field remoteUUID = Field "remoteuuid" $ -- does it look like a UUID? - return . all (\c -> isAlphaNum c || c == '-') + all (\c -> isAlphaNum c || c == '-') associatedFile :: Field -associatedFile = Field "associatedfile" $ \value -> - -- is the file located within the current directory? - dirContains <$> getCurrentDirectory <*> pure value +associatedFile = Field "associatedfile" $ \f -> + -- is the file a safe relative filename? + not (isAbsolute f) && not ("../" `isPrefixOf` f) getField :: Field -> Annex (Maybe String) getField = Annex.getField . fieldName diff --git a/GitAnnexShell.hs b/GitAnnexShell.hs index 497e4cf8f1..15be51180a 100644 --- a/GitAnnexShell.hs +++ b/GitAnnexShell.hs @@ -85,7 +85,7 @@ builtin :: String -> String -> [String] -> IO () builtin cmd dir params = do checkNotReadOnly cmd let (params', fieldparams) = partitionParams params - fields <- filterM checkField $ parseFields fieldparams + let fields = filter checkField $ parseFields fieldparams dispatch False (cmd : params') cmds options fields header $ Git.Construct.repoAbsPath dir >>= Git.Construct.fromAbsPath @@ -113,11 +113,11 @@ parseFields = map (separate (== '=')) {- Only allow known fields to be set, ignore others. - Make sure that field values make sense. -} -checkField :: (String, String) -> IO Bool +checkField :: (String, String) -> Bool checkField (field, value) | field == fieldName remoteUUID = fieldCheck remoteUUID value | field == fieldName associatedFile = fieldCheck associatedFile value - | otherwise = return False + | otherwise = False failure :: IO () failure = error $ "bad parameters\n\n" ++ usage header cmds options From 7daa2698538fbc311f82ef9a9fb102b7044fdb7b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 13:47:32 -0400 Subject: [PATCH 3997/8313] better pid file locking code --- Utility/Daemon.hs | 55 +++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs index 192340cef7..f36a761d00 100644 --- a/Utility/Daemon.hs +++ b/Utility/Daemon.hs @@ -27,7 +27,7 @@ daemonize logfd pidfile changedirectory a = do _ <- forkProcess child2 out child2 = do - maybe noop (lockPidFile True alreadyrunning) pidfile + maybe noop (lockPidFile alreadyrunning) pidfile when changedirectory $ setCurrentDirectory "/" nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags @@ -42,31 +42,44 @@ daemonize logfd pidfile changedirectory a = do alreadyrunning = error "Daemon is already running." out = exitImmediately ExitSuccess -lockPidFile :: Bool -> IO () -> FilePath -> IO () -lockPidFile write onfailure file = do - fd <- openFd file ReadWrite (Just stdFileMode) - defaultFileFlags { trunc = write } - locked <- catchMaybeIO $ setLock fd (locktype, AbsoluteSeek, 0, 0) - case locked of - Nothing -> onfailure - _ -> when write $ void $ - fdWrite fd =<< show <$> getProcessID +{- Locks the pid file, with an exclusive, non-blocking lock. + - Runs an action on failure. On success, writes the pid to the file, + - fully atomically. -} +lockPidFile :: IO () -> FilePath -> IO () +lockPidFile onfailure file = do + fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags + locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) + fd' <- openFd newfile ReadWrite (Just stdFileMode) defaultFileFlags + { trunc = True } + locked' <- catchMaybeIO $ setLock fd' (WriteLock, AbsoluteSeek, 0, 0) + case (locked, locked') of + (Nothing, _) -> onfailure + (_, Nothing) -> onfailure + _ -> do + _ <- fdWrite fd' =<< show <$> getProcessID + renameFile newfile file + closeFd fd where - locktype - | write = WriteLock - | otherwise = ReadLock + newfile = file ++ ".new" {- Stops the daemon. - - The pid file is used to get the daemon's pid. - - - To guard against a stale pid, try to take a nonblocking shared lock - - of the pid file. If this *fails*, the daemon must be running, - - and have the exclusive lock, so the pid file is trustworthy. + - To guard against a stale pid, check the lock of the pid file, + - and compare the process that has it locked with the file content. -} stopDaemon :: FilePath -> IO () -stopDaemon pidfile = lockPidFile False go pidfile - where - go = do - pid <- readish <$> readFile pidfile - maybe noop (signalProcess sigTERM) pid +stopDaemon pidfile = do + fd <- openFd pidfile ReadOnly (Just stdFileMode) defaultFileFlags + locked <- getLock fd (ReadLock, AbsoluteSeek, 0, 0) + p <- readish <$> readFile pidfile + case (locked, p) of + (Nothing, _) -> noop + (_, Nothing) -> noop + (Just (pid, _), Just pid') + | pid == pid' -> signalProcess sigTERM pid + | otherwise -> error $ + "stale pid in " ++ pidfile ++ + " (got " ++ show pid' ++ + "; expected" ++ show pid ++ " )" From 0c0fd0c54c126268a4867e4dd0d1d42a46621665 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 13:49:27 -0400 Subject: [PATCH 3998/8313] update --- Logs/Transfer.hs | 7 ++++++- doc/design/assistant/syncing.mdwn | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 6bdc750f4b..dc92833069 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -27,7 +27,12 @@ data Transfer = Transfer } deriving (Show, Eq, Ord) -{- Information about a Transfer, stored in the transfer information file. -} +{- Information about a Transfer, stored in the transfer information file. + - + - Note that the associatedFile may not correspond to a file in the local + - git repository. It's some file, possibly relative to some directory, + - of some repository, that was acted on to initiate the transfer. + -} data TransferInfo = TransferInfo { startedTime :: UTCTime , transferPid :: Maybe ProcessID diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index c18badb533..8757c3ae21 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -7,8 +7,7 @@ all the other git clones, at both the git level and the key/value level. **done** * locking for the files, so redundant transfer races can be detected, and failed transfers noticed **done** -* transfer info for git-annex-shell (problem: how to add a switch - with the necessary info w/o breaking backwards compatability?) +* transfer info for git-annex-shell **done** * update files as transfers proceed. See [[progressbars]] (updating for downloads is easy; for uploads is hard) * add Transfer queue TChan From c9d7e9f6bd5adac8a5ff0e925bbac549f962cdb0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 16:06:52 -0400 Subject: [PATCH 3999/8313] startedTime needs to be a Maybe to handle transfers that have not started yet This changes the file format. --- Logs/Transfer.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index dc92833069..f808cb6a44 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -34,7 +34,7 @@ data Transfer = Transfer - of some repository, that was acted on to initiate the transfer. -} data TransferInfo = TransferInfo - { startedTime :: UTCTime + { startedTime :: Maybe UTCTime , transferPid :: Maybe ProcessID , transferThread :: Maybe ThreadId , bytesComplete :: Maybe Integer @@ -76,7 +76,7 @@ transfer t file a = do createAnnexDirectory $ takeDirectory tfile mode <- annexFileMode info <- liftIO $ TransferInfo - <$> getCurrentTime + <$> (Just <$> getCurrentTime) <*> pure Nothing -- pid not stored in file, so omitted for speed <*> pure Nothing -- threadid not stored in file, so omitted for speed <*> pure Nothing -- not 0; transfer may be resuming From ad0b82795742228d3ed9eab7e50f4000f6d78734 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 16:07:39 -0400 Subject: [PATCH 4000/8313] added --- Assistant/TransferQueue.hs | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 Assistant/TransferQueue.hs diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs new file mode 100644 index 0000000000..979cbb80f5 --- /dev/null +++ b/Assistant/TransferQueue.hs @@ -0,0 +1,43 @@ +{- git-annex assistant pending transfer queue + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.TransferQueue where + +import Common.Annex +import Utility.TSet +import Logs.Transfer +import Types.Remote + +import Control.Concurrent.STM + +type TransferQueue = TChan (Transfer, TransferInfo) + +newTransferQueue :: IO TransferQueue +newTransferQueue = atomically newTChan + +stubInfo :: AssociatedFile -> TransferInfo +stubInfo f = TransferInfo + { startedTime = Nothing + , transferPid = Nothing + , transferThread = Nothing + , bytesComplete = Nothing + , associatedFile = f + } + +{- Adds a pending transfer to the end of the queue. -} +queueTransfer :: TransferQueue -> Transfer -> AssociatedFile -> IO () +queueTransfer q transfer f = void $ atomically $ + writeTChan q (transfer, stubInfo f) + +{- Adds a pending transfer to the start of the queue, to be processed next. -} +queueNextTransfer :: TransferQueue -> Transfer -> AssociatedFile -> IO () +queueNextTransfer q transfer f = void $ atomically $ + unGetTChan q (transfer, stubInfo f) + +{- Blocks until a pending transfer is available in the queue. -} +getNextTransfer :: TransferQueue -> IO (Transfer, TransferInfo) +getNextTransfer = atomically . readTChan From 224dac374e685b93eb32f799ad37cedb5b372e3b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 16:08:23 -0400 Subject: [PATCH 4001/8313] update --- doc/design/assistant/syncing.mdwn | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 8757c3ae21..461ec6dc47 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -10,12 +10,13 @@ all the other git clones, at both the git level and the key/value level. * transfer info for git-annex-shell **done** * update files as transfers proceed. See [[progressbars]] (updating for downloads is easy; for uploads is hard) -* add Transfer queue TChan +* add Transfer queue TChan **done** +* add TransferInfo Map to DaemonStatus for tracking transfers in progress. + **done** * enqueue Transfers (Uploads) as new files are added to the annex by Watcher. * enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by Watcher. -* add TransferInfo Map to DaemonStatus for tracking transfers in progress. * Poll transfer in progress info files for changes (use inotify again! wow! hammer, meet nail..), and update the TransferInfo Map * Write basic Transfer handling thread. Multiple such threads need to be @@ -52,6 +53,9 @@ all the other git clones, at both the git level and the key/value level. signaling a change out of band. 4. Add a hook, so when there's a change to sync, a program can be run and do its own signaling. +5. --debug will show often unnecessary work being done. Optimise. +6. It would be nice if, when a USB drive is connected, + syncing starts automatically. Use dbus on Linux? ## data syncing @@ -83,13 +87,12 @@ anyway. that lack content. * Transfer threads started/stopped as necessary to move data. (May sometimes want multiple threads downloading, or uploading, or even both.) - - type TransferQueue = TChan [Transfer] - -- add (M.Map Transfer TransferInfo) to DaemonStatus - startTransfer :: Transfer -> Annex TransferID + startTransfer :: TransferQueue -> Transfer -> Annex () + startTransfer q transfer = error "TODO" - stopTransfer :: TransferID -> IO () + stopTransfer :: TransferQueue -> TransferID -> Annex () + stopTransfer q transfer = error "TODO" The assistant needs to find out when `git-annex-shell` is receiving or sending (triggered by another remote), so it can add data for those too. From 32e5e02e431338a7b04990ab91feaea7b32d6d0e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 16:11:04 -0400 Subject: [PATCH 4002/8313] added currentTransfers --- Assistant/DaemonStatus.hs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index c7713e7d56..10161a96cb 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -17,6 +17,8 @@ import System.Posix.Types import Data.Time.Clock.POSIX import Data.Time import System.Locale +import Logs.Transfer +import qualified Data.Map as M data DaemonStatus = DaemonStatus -- False when the daemon is performing its startup scan @@ -27,6 +29,8 @@ data DaemonStatus = DaemonStatus , sanityCheckRunning :: Bool -- Last time the sanity checker ran , lastSanityCheck :: Maybe POSIXTime + -- Currently running file content transfers + , currentTransfers :: M.Map Transfer TransferInfo } deriving (Show) @@ -38,6 +42,7 @@ newDaemonStatus = DaemonStatus , lastRunning = Nothing , sanityCheckRunning = False , lastSanityCheck = Nothing + , currentTransfers = M.empty } getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus @@ -47,15 +52,17 @@ modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> An modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a) {- Load any previous daemon status file, and store it in the MVar for this - - process to use as its DaemonStatus. -} + - process to use as its DaemonStatus. Also gets current transfer status. -} startDaemonStatus :: Annex DaemonStatusHandle startDaemonStatus = do file <- fromRepo gitAnnexDaemonStatusFile status <- liftIO $ catchDefaultIO (readDaemonStatusFile file) newDaemonStatus + transfers <- M.fromList <$> getTransfers liftIO $ newMVar status { scanComplete = False , sanityCheckRunning = False + , currentTransfers = transfers } {- This thread wakes up periodically and writes the daemon status to disk. -} From 4845b59413e5ae70cd6b78035fbae93e9cd845de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 16:06:52 -0400 Subject: [PATCH 4003/8313] startedTime needs to be a Maybe to handle transfers that have not started yet This changes the file format. --- Logs/Transfer.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index dc92833069..f808cb6a44 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -34,7 +34,7 @@ data Transfer = Transfer - of some repository, that was acted on to initiate the transfer. -} data TransferInfo = TransferInfo - { startedTime :: UTCTime + { startedTime :: Maybe UTCTime , transferPid :: Maybe ProcessID , transferThread :: Maybe ThreadId , bytesComplete :: Maybe Integer @@ -76,7 +76,7 @@ transfer t file a = do createAnnexDirectory $ takeDirectory tfile mode <- annexFileMode info <- liftIO $ TransferInfo - <$> getCurrentTime + <$> (Just <$> getCurrentTime) <*> pure Nothing -- pid not stored in file, so omitted for speed <*> pure Nothing -- threadid not stored in file, so omitted for speed <*> pure Nothing -- not 0; transfer may be resuming From 7cd9a659cd71bbf8c52ec9bcb46da1eb91fcbef9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 2 Jul 2012 16:17:56 -0400 Subject: [PATCH 4004/8313] blog for the day --- .../day_22__horrible_option_parsing_hack.mdwn | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn diff --git a/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn b/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn new file mode 100644 index 0000000000..9f59d1af91 --- /dev/null +++ b/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn @@ -0,0 +1,34 @@ +Well, sometimes you just have to go for the hack. Trying to find a way +to add additional options to git-annex-shell without breaking backwards +compatability, I noticed that it ignores all options after `--`, because +those tend to be random rsync options due to the way rsync runs it. + +So, I've added a new class of options, that come in between, like +`-- opt=val opt=val ... --` + +The parser for these will not choke on unknown options, unlike normal +getopt. So this let me add the additional info I needed to +pass to git-annex-shell to make it record transfer information. And +if I need to pass more info in the future, that's covered too. + +It's ugly, but since only git-annex runs git-annex-shell, this is an +ugliness only I (and now you, dear reader) have to put up with. + +Note to self: Command-line programs are sometimes an API, particularly +if designed to be called remotely, and so it makes sense consider +whether they are, and design expandability into them from day 1. + +--- + +Anyway, we now have full transfer tracking in git-annex! Both sides of +a transfer know what's being transferred, and from where, and have +the info necessary to interrupt the transfer. + +--- + +Also did some basic groundwork, adding a queue of transfers to perform, +and adding to the daemon's status information a map of currently running +transfers. + +Next up: The daemon will use inotify to notice new and deleted transfer +info files, and update its status info. From b4917bd18fa9e2eacb5fbd916828d30e2ac297b4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 Jul 2012 10:58:40 -0400 Subject: [PATCH 4005/8313] add transfer watching thread Worked the 1st try! --- Assistant.hs | 13 ++++- Assistant/Threads/TransferWatcher.hs | 78 ++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 Assistant/Threads/TransferWatcher.hs diff --git a/Assistant.hs b/Assistant.hs index a077cf10f6..40f53d55ee 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -33,9 +33,14 @@ - updated branches into the current branch. This uses inotify - on .git/refs/heads, so there are additional inotify threads - associated with it, too. - - Thread 9: status logger + - Thread 9: transfer watcher + - Watches for transfer information files being created and removed, + - and maintains the DaemonStatus currentTransfers map. This uses + - inotify on .git/annex/transfer/, so there are additional inotify + - threads associated with it, too. + - Thread 10: status logger - Wakes up periodically and records the daemon's status to disk. - - Thread 10: sanity checker + - Thread 11: sanity checker - Wakes up periodically (rarely) and does sanity checks. - - ThreadState: (MVar) @@ -56,6 +61,8 @@ - FailedPushMap (STM TMVar) - Failed pushes are indicated by writing to this TMVar. The push - retrier blocks until they're available. + - TransferQueue (STM TChan) + - Transfers to make are indicated by writing to this channel. -} module Assistant where @@ -70,6 +77,7 @@ import Assistant.Threads.Watcher import Assistant.Threads.Committer import Assistant.Threads.Pusher import Assistant.Threads.Merger +import Assistant.Threads.TransferWatcher import Assistant.Threads.SanityChecker import qualified Utility.Daemon import Utility.LogFile @@ -100,6 +108,7 @@ startDaemon assistant foreground , pushThread st commitchan pushmap , pushRetryThread st pushmap , mergeThread st + , transferWatcherThread st dstatus , daemonStatusThread st dstatus , sanityCheckerThread st dstatus changechan , watchThread st dstatus changechan diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs new file mode 100644 index 0000000000..811b045a82 --- /dev/null +++ b/Assistant/Threads/TransferWatcher.hs @@ -0,0 +1,78 @@ +{- git-annex assistant transfer watching thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferWatcher where + +import Common.Annex +import Assistant.ThreadedMonad +import Assistant.DaemonStatus +import Logs.Transfer +import Utility.DirWatcher +import Utility.Types.DirWatcher + +import Data.Map as M + +{- This thread watches for changes to the gitAnnexTransferDir, + - and updates the DaemonStatus's map of ongoing transfers. -} +transferWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () +transferWatcherThread st dstatus = do + g <- runThreadState st $ fromRepo id + let dir = gitAnnexTransferDir g + createDirectoryIfMissing True dir + let hook a = Just $ runHandler st dstatus a + let hooks = mkWatchHooks + { addHook = hook onAdd + , delHook = hook onDel + , errHook = hook onErr + } + void $ watchDir dir (const False) hooks id + +type Handler = ThreadState -> DaemonStatusHandle -> FilePath -> Maybe FileStatus -> IO () + +{- Runs an action handler. + - + - Exceptions are ignored, otherwise a whole thread could be crashed. + -} +runHandler :: ThreadState -> DaemonStatusHandle -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler st dstatus handler file filestatus = void $ do + either print (const noop) =<< tryIO go + where + go = handler st dstatus file filestatus + +{- Called when there's an error with inotify. -} +onErr :: Handler +onErr _ _ msg _ = error msg + +{- Called when a new transfer information file is written. + - + - When another thread of the assistant writes a transfer info file, + - this will notice that too, but should skip it, because the thread + - will be managing the transfer itself, and will have stored a more + - complete TransferInfo than is stored in the file. + -} +onAdd :: Handler +onAdd st dstatus file _ = case parseTransferFile file of + Nothing -> noop + Just t -> do + minfo <- runThreadState st $ checkTransfer t + pid <- getProcessID + case minfo of + Nothing -> noop -- transfer already finished + Just info + | transferPid info == Just pid -> noop + | otherwise -> adjustTransfers st dstatus + (M.insertWith' const t info) + +{- Called when a transfer information file is removed. -} +onDel :: Handler +onDel st dstatus file _ = case parseTransferFile file of + Nothing -> noop + Just t -> adjustTransfers st dstatus (M.delete t) + +adjustTransfers :: ThreadState -> DaemonStatusHandle -> (M.Map Transfer TransferInfo -> M.Map Transfer TransferInfo) -> IO () +adjustTransfers st dstatus a = runThreadState st $ modifyDaemonStatus dstatus $ + \s -> s { currentTransfers = a (currentTransfers s) } From a6e267eeec9884441e8834c557c38bd0b7c1a5c1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 Jul 2012 11:11:32 -0400 Subject: [PATCH 4006/8313] update --- doc/design/assistant/syncing.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 461ec6dc47..d2248279f7 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -13,12 +13,12 @@ all the other git clones, at both the git level and the key/value level. * add Transfer queue TChan **done** * add TransferInfo Map to DaemonStatus for tracking transfers in progress. **done** +* Poll transfer in progress info files for changes (use inotify again! + wow! hammer, meet nail..), and update the TransferInfo Map **done** * enqueue Transfers (Uploads) as new files are added to the annex by Watcher. * enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by Watcher. -* Poll transfer in progress info files for changes (use inotify again! - wow! hammer, meet nail..), and update the TransferInfo Map * Write basic Transfer handling thread. Multiple such threads need to be able to be run at once. Each will need its own independant copy of the Annex state monad. From f40b78d1254302f0ecab04ca4331e1048b178190 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 3 Jul 2012 18:11:10 -0400 Subject: [PATCH 4007/8313] blog for the day --- .../blog/day_23__transfer_watching.mdwn | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/design/assistant/blog/day_23__transfer_watching.mdwn diff --git a/doc/design/assistant/blog/day_23__transfer_watching.mdwn b/doc/design/assistant/blog/day_23__transfer_watching.mdwn new file mode 100644 index 0000000000..beaf75bc52 --- /dev/null +++ b/doc/design/assistant/blog/day_23__transfer_watching.mdwn @@ -0,0 +1,25 @@ +Starting to travel, so limited time today. + +Yet Another Thread added to the assistant, all it does is watch for changes +to transfer information files, and update the assistant's map of transfers +currently in progress. Now the assistant will know if some other repository +has connected to the local repo and is sending or receiving a file's +content. + +This seemed really simple to write, it's just 78 lines of code. It worked +100% correctly the first time. :) But it's only so easy because I've got +this shiny new inotify hammer that I keep finding places to use in the +assistant. + +Also, the new thread does some things that caused a similar thread (the +merger thread) to go into a MVar deadlock. Luckily, I spent much of +[day 19](day_19__random_improvements) investigating and fixing that +deadlock, even though it was not a problem at the time. + +So, good.. I'm doing things right and getting to a place where rather +nontrivial features can be added easily. + +-- + +Next up: Enough nonsense with tracking tranfers... Time to start actually +transferring content around! From ed87f27db9738df957fcf6256d77cf8d26de549c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnIsjgK_cdfZcIAumVRwTvNXS4cD0zNnaI" Date: Wed, 4 Jul 2012 11:15:30 +0000 Subject: [PATCH 4008/8313] git annex du --- doc/bugs/git_annex_du.mdwn | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git_annex_du.mdwn diff --git a/doc/bugs/git_annex_du.mdwn b/doc/bugs/git_annex_du.mdwn new file mode 100644 index 0000000000..d38bff2fd7 --- /dev/null +++ b/doc/bugs/git_annex_du.mdwn @@ -0,0 +1,10 @@ +We need a way to calculate space taken by certain files. + +Use cases: I want to drop some files from my small disk. I need to figure out things that take most space, and drop them. + +Usage examples: + + git annex du -hs *.mp3 + git annex du -sBm --in=here *.ogg + +Would be nice if it was compatible with standard unix `df`. From 597d16ed9c185e2c2cf5616d63b6eadd187927ea Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 4 Jul 2012 07:32:03 -0400 Subject: [PATCH 4009/8313] response --- doc/bugs/git_annex_du.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/bugs/git_annex_du.mdwn b/doc/bugs/git_annex_du.mdwn index d38bff2fd7..2b1315298c 100644 --- a/doc/bugs/git_annex_du.mdwn +++ b/doc/bugs/git_annex_du.mdwn @@ -8,3 +8,7 @@ Usage examples: git annex du -sBm --in=here *.ogg Would be nice if it was compatible with standard unix `df`. + +> `du -L` works. +> +> See also: [[forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files]] From f6d18ec68dfc2f391d0bcdbff41ad2c780efc328 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 4 Jul 2012 12:32:44 +0000 Subject: [PATCH 4010/8313] Added a comment --- ..._18ddf8b5934dd6fb1676cd6adc7d103b._comment | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment new file mode 100644 index 0000000000..eb886acf6b --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_3_18ddf8b5934dd6fb1676cd6adc7d103b._comment @@ -0,0 +1,19 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 3" + date="2012-07-04T12:32:44Z" + content=""" +Jimmy, sounds like I could use something like this to get the current limit: + + sysctl kern.maxfilesperproc + +Probably prints \"sysctl kern.maxfilesperproc = 256\" or such.. can you verify? +Once I have the limit, I can make the kqueue code use subset of it, and print out a message when it needs to be increased, like the inotify code does. + +(Also, the kqueue code only opens directories, not files, so unless you have 400000 directories, that's +a little high.) + +--- + +On file removal not propigating, does this still happen? When you remove a file does a git commit automatically happen, or is that broken with kqueue? +"""]] From 5153f845466ca1d2ad7abcd6c3172fde710c527b Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 4 Jul 2012 12:43:54 +0000 Subject: [PATCH 4011/8313] Added a comment --- .../comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment diff --git a/doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment b/doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment new file mode 100644 index 0000000000..35e0bb6ed5 --- /dev/null +++ b/doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 14" + date="2012-07-04T12:43:54Z" + content=""" +@Damien, hmm, it should not be using any cp options, unless when it was built there was a cp in the path that supported some option like -p. Can you check with --debug what cp parameters it's trying to use? + + +"""]] From 1da79ea61faaddd71bdb84ee2c5a89bbe04a69ed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 4 Jul 2012 09:08:20 -0400 Subject: [PATCH 4012/8313] When shaNsum commands cannot be found, use the Haskell SHA library (already a dependency) to do the checksumming. This may be slower, but avoids portability problems. Using Crypto's version of the hashes would be another option. I need to benchmark it. The SHA2 library (which provides SHA1 also, confusing name) may be the fastest option, but is not currently in Debian. --- Backend/SHA.hs | 65 +++++++++++++++++++++++++--------------------- Build/Configure.hs | 9 +++---- debian/changelog | 3 +++ 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 838a97ab8b..6ecc78ff29 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -1,6 +1,6 @@ {- git-annex SHA backend - - - Copyright 2011 Joey Hess + - Copyright 2011,2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -12,7 +12,10 @@ import qualified Annex import Types.Backend import Types.Key import Types.KeySource + import qualified Build.SysConfig as SysConfig +import Data.Digest.Pure.SHA +import qualified Data.ByteString.Lazy as L type SHASize = Int @@ -25,32 +28,31 @@ backends :: [Backend] backends = catMaybes $ map genBackend sizes ++ map genBackendE sizes genBackend :: SHASize -> Maybe Backend -genBackend size - | isNothing (shaCommand size) = Nothing - | otherwise = Just b - where - b = Backend - { name = shaName size - , getKey = keyValue size - , fsckKey = Just $ checkKeyChecksum size - } +genBackend size = Just $ Backend + { name = shaName size + , getKey = keyValue size + , fsckKey = Just $ checkKeyChecksum size + } genBackendE :: SHASize -> Maybe Backend -genBackendE size = - case genBackend size of - Nothing -> Nothing - Just b -> Just $ b - { name = shaNameE size - , getKey = keyValueE size - } +genBackendE size = do + b <- genBackend size + return $ b + { name = shaNameE size + , getKey = keyValueE size + } -shaCommand :: SHASize -> Maybe String -shaCommand 1 = SysConfig.sha1 -shaCommand 256 = Just SysConfig.sha256 -shaCommand 224 = SysConfig.sha224 -shaCommand 384 = SysConfig.sha384 -shaCommand 512 = SysConfig.sha512 -shaCommand _ = Nothing +shaCommand :: SHASize -> Either (L.ByteString -> String) String +shaCommand sz + | sz == 1 = use SysConfig.sha1 sha1 + | sz == 256 = use SysConfig.sha256 sha256 + | sz == 224 = use SysConfig.sha224 sha224 + | sz == 384 = use SysConfig.sha384 sha384 + | sz == 512 = use SysConfig.sha512 sha512 + | otherwise = error $ "bad sha size " ++ show sz + where + use Nothing sha = Left $ showDigest . sha + use (Just c) _ = Right c shaName :: SHASize -> String shaName size = "SHA" ++ show size @@ -61,13 +63,16 @@ shaNameE size = shaName size ++ "E" shaN :: SHASize -> FilePath -> Annex String shaN size file = do showAction "checksum" - liftIO $ pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do - sha <- fst . separate (== ' ') <$> hGetLine h - if null sha - then error $ command ++ " parse error" - else return sha + case shaCommand size of + Left sha -> liftIO $ sha <$> L.readFile file + Right command -> liftIO $ runcommand command where - command = fromJust $ shaCommand size + runcommand command = + pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do + sha <- fst . separate (== ' ') <$> hGetLine h + if null sha + then error $ command ++ " parse error" + else return sha {- A key is a checksum of its contents. -} keyValue :: SHASize -> KeySource -> Annex (Maybe Key) diff --git a/Build/Configure.hs b/Build/Configure.hs index 7af53cf10f..24743bf618 100644 --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -28,15 +28,14 @@ tests = , TestCase "gpg" $ testCmd "gpg" "gpg --version >/dev/null" , TestCase "lsof" $ testCmd "lsof" "lsof -v >/dev/null 2>&1" , TestCase "ssh connection caching" getSshConnectionCaching - ] ++ shaTestCases False [1, 512, 224, 384] ++ shaTestCases True [256] + ] ++ shaTestCases [1, 256, 512, 224, 384] -shaTestCases :: Bool -> [Int] -> [TestCase] -shaTestCases required l = map make l +shaTestCases :: [Int] -> [TestCase] +shaTestCases l = map make l where - make n = TestCase key $ selector key (shacmds n) " [x, osxpath x]) $ map (\x -> "sha" ++ show n ++ x) ["", "sum"] -- Max OSX puts GNU tools outside PATH, so look in diff --git a/debian/changelog b/debian/changelog index 33c850861b..ebd34c9440 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ git-annex (3.20120630) UNRELEASED; urgency=low transfer is already in progress by another process. * status: Lists transfers that are currently in progress. * Fix passing --uuid to git-annex-shell. + * When shaNsum commands cannot be found, use the Haskell SHA library + (already a dependency) to do the checksumming. This may be slower, + but avoids portability problems. -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 From b0be7b4effe30051ff921fd5e4ba37e50242d535 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 4 Jul 2012 13:14:00 +0000 Subject: [PATCH 4013/8313] Added a comment --- .../comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment diff --git a/doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment b/doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment new file mode 100644 index 0000000000..0005328c48 --- /dev/null +++ b/doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment @@ -0,0 +1,9 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 15" + date="2012-07-04T13:14:00Z" + content=""" +git-annex will now fall back to slower pure Haskell hashing code if `sha256sum`, etc programs are not in PATH. I'd still recommend installing the coreutils, as they're probably faster. + +(The `shasum` command seems to come from a perl library, so I have not tried to make git-annex use that one.) +"""]] From 40729e7fa21684bfed758479c97e3172ff8777fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 4 Jul 2012 13:04:01 -0400 Subject: [PATCH 4014/8313] Use SHA library for files less than 50 kb in size, at which point it's faster than forking the more optimised external program. --- Backend/SHA.hs | 54 +++++++++++++++++++++++++++--------------------- debian/changelog | 2 ++ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 6ecc78ff29..7abbf8035a 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -42,28 +42,16 @@ genBackendE size = do , getKey = keyValueE size } -shaCommand :: SHASize -> Either (L.ByteString -> String) String -shaCommand sz - | sz == 1 = use SysConfig.sha1 sha1 - | sz == 256 = use SysConfig.sha256 sha256 - | sz == 224 = use SysConfig.sha224 sha224 - | sz == 384 = use SysConfig.sha384 sha384 - | sz == 512 = use SysConfig.sha512 sha512 - | otherwise = error $ "bad sha size " ++ show sz - where - use Nothing sha = Left $ showDigest . sha - use (Just c) _ = Right c - shaName :: SHASize -> String shaName size = "SHA" ++ show size shaNameE :: SHASize -> String shaNameE size = shaName size ++ "E" -shaN :: SHASize -> FilePath -> Annex String -shaN size file = do +shaN :: SHASize -> FilePath -> Integer -> Annex String +shaN shasize file filesize = do showAction "checksum" - case shaCommand size of + case shaCommand shasize filesize of Left sha -> liftIO $ sha <$> L.readFile file Right command -> liftIO $ runcommand command where @@ -74,16 +62,34 @@ shaN size file = do then error $ command ++ " parse error" else return sha +shaCommand :: SHASize -> Integer -> Either (L.ByteString -> String) String +shaCommand shasize filesize + | shasize == 1 = use SysConfig.sha1 sha1 + | shasize == 256 = use SysConfig.sha256 sha256 + | shasize == 224 = use SysConfig.sha224 sha224 + | shasize == 384 = use SysConfig.sha384 sha384 + | shasize == 512 = use SysConfig.sha512 sha512 + | otherwise = error $ "bad sha size " ++ show shasize + where + use Nothing sha = Left $ showDigest . sha + use (Just c) sha + -- use builtin, but slower sha for small files + -- benchmarking indicates it's faster up to + -- and slightly beyond 50 kb files + | filesize < 51200 = use Nothing sha + | otherwise = Right c + {- A key is a checksum of its contents. -} keyValue :: SHASize -> KeySource -> Annex (Maybe Key) -keyValue size source = do +keyValue shasize source = do let file = contentLocation source - s <- shaN size file stat <- liftIO $ getFileStatus file + let filesize = fromIntegral $ fileSize stat + s <- shaN shasize file filesize return $ Just $ stubKey { keyName = s - , keyBackendName = shaName size - , keySize = Just $ fromIntegral $ fileSize stat + , keyBackendName = shaName shasize + , keySize = Just filesize } {- Extension preserving keys. -} @@ -106,10 +112,12 @@ keyValueE size source = keyValue size source >>= maybe (return Nothing) addE checkKeyChecksum :: SHASize -> Key -> FilePath -> Annex Bool checkKeyChecksum size key file = do fast <- Annex.getState Annex.fast - present <- liftIO $ doesFileExist file - if not present || fast - then return True - else check <$> shaN size file + mstat <- liftIO $ catchMaybeIO $ getFileStatus file + case (mstat, fast) of + (Just stat, False) -> do + let filesize = fromIntegral $ fileSize stat + check <$> shaN size file filesize + _ -> return True where check s | s == dropExtension (keyName key) = True diff --git a/debian/changelog b/debian/changelog index ebd34c9440..1c44f59526 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,8 @@ git-annex (3.20120630) UNRELEASED; urgency=low * When shaNsum commands cannot be found, use the Haskell SHA library (already a dependency) to do the checksumming. This may be slower, but avoids portability problems. + * Use SHA library for files less than 50 kb in size, at which point it's + faster than forking the more optimised external program. -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 From 1f3f221b80540832fdd9ce104f47f079367288cc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 4 Jul 2012 13:22:32 -0400 Subject: [PATCH 4015/8313] blog for the day (may be updated later) --- .../blog/day_24__airport_digressions.mdwn | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 doc/design/assistant/blog/day_24__airport_digressions.mdwn diff --git a/doc/design/assistant/blog/day_24__airport_digressions.mdwn b/doc/design/assistant/blog/day_24__airport_digressions.mdwn new file mode 100644 index 0000000000..58d3ed2777 --- /dev/null +++ b/doc/design/assistant/blog/day_24__airport_digressions.mdwn @@ -0,0 +1,86 @@ +In a series of airport layovers all day. Since I woke up at 3:45 am, +didn't feel up to doing serious new work, so instead I worked through some +OSX support backlog. + +git-annex will now use Haskell's SHA library if the `sha256sum` +command is not available. That library is slow, but it's guaranteed to be +available; git-annex already depended on it to calculate HMACs. + +Then I decided to see if it makes sense to use the SHA library +when adding smaller files. At some point, its slower implementation should +win over needing to fork and parse the output of `sha256sum`. This was +the first time I tried out Haskell's +[Criterion](http://hackage.haskell.org/package/criterion) benchmarker, +and I built this simple benchmark in short order. + +[[!format haskell """ +import Data.Digest.Pure.SHA +import Data.ByteString.Lazy as L +import Criterion.Main +import Common + +testfile :: FilePath +testfile = "/tmp/bar" -- on ram disk + +main = defaultMain + [ bgroup "sha256" + [ bench "internal" $ whnfIO internal + , bench "external" $ whnfIO external + ] + ] + +internal :: IO String +internal = showDigest . sha256 <$> L.readFile testfile + +external :: IO String +external = pOpen ReadFromPipe "sha256sum" [testfile] $ \h -> + fst . separate (== ' ') <$> hGetLine h +"""]] + +The nice thing about benchmarking in Airports is when you're running a +benchmark locally, you don't want to do anything else with the computer, +so can alternate people watching, spacing out, and analizing results. + +100 kb file: + + benchmarking sha256/internal + mean: 15.64729 ms, lb 15.29590 ms, ub 16.10119 ms, ci 0.950 + std dev: 2.032476 ms, lb 1.638016 ms, ub 2.527089 ms, ci 0.950 + + benchmarking sha256/external + mean: 8.217700 ms, lb 7.931324 ms, ub 8.568805 ms, ci 0.950 + std dev: 1.614786 ms, lb 1.357791 ms, ub 2.009682 ms, ci 0.950 + +75 kb file: + + benchmarking sha256/internal + mean: 12.16099 ms, lb 11.89566 ms, ub 12.50317 ms, ci 0.950 + std dev: 1.531108 ms, lb 1.232353 ms, ub 1.929141 ms, ci 0.950 + + benchmarking sha256/external + mean: 8.818731 ms, lb 8.425744 ms, ub 9.269550 ms, ci 0.950 + std dev: 2.158530 ms, lb 1.916067 ms, ub 2.487242 ms, ci 0.950 + +50 kb file: + + benchmarking sha256/internal + mean: 7.699274 ms, lb 7.560254 ms, ub 7.876605 ms, ci 0.950 + std dev: 801.5292 us, lb 655.3344 us, ub 990.4117 us, ci 0.950 + + benchmarking sha256/external + mean: 8.715779 ms, lb 8.330540 ms, ub 9.102232 ms, ci 0.950 + std dev: 1.988089 ms, lb 1.821582 ms, ub 2.181676 ms, ci 0.950 + +10 kb file: + + benchmarking sha256/internal + mean: 1.586105 ms, lb 1.574512 ms, ub 1.604922 ms, ci 0.950 + std dev: 74.07235 us, lb 51.71688 us, ub 108.1348 us, ci 0.950 + + benchmarking sha256/external + mean: 6.873742 ms, lb 6.582765 ms, ub 7.252911 ms, ci 0.950 + std dev: 1.689662 ms, lb 1.346310 ms, ub 2.640399 ms, ci 0.950 + +It's possible to get nice graphical reports out of Criterion, but +this is clear enough, so I stopped here. 50 kb seems a reasonable +cutoff point. From 59f8413abe89b9abe5708fc0ab3aba93fa2c0f64 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 4 Jul 2012 13:40:57 -0400 Subject: [PATCH 4016/8313] update --- .../assistant/blog/day_24__airport_digressions.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/design/assistant/blog/day_24__airport_digressions.mdwn b/doc/design/assistant/blog/day_24__airport_digressions.mdwn index 58d3ed2777..6952969748 100644 --- a/doc/design/assistant/blog/day_24__airport_digressions.mdwn +++ b/doc/design/assistant/blog/day_24__airport_digressions.mdwn @@ -84,3 +84,16 @@ so can alternate people watching, spacing out, and analizing results. It's possible to get nice graphical reports out of Criterion, but this is clear enough, so I stopped here. 50 kb seems a reasonable cutoff point. + +I also used this to benchmark the SHA256 in Haskell's Crypto package. +Surprisingly, it's a *lot* slower than even the Pure.SHA code. +On a 50 kb file: + + benchmarking sha256/Crypto + collecting 100 samples, 1 iterations each, in estimated 6.073809 s + mean: 69.89037 ms, lb 69.15831 ms, ub 70.71845 ms, ci 0.950 + std dev: 3.995397 ms, lb 3.435775 ms, ub 4.721952 ms, ci 0.950 + + +There's another Haskell library, [SHA2](http://hackage.haskell.org/package/SHA2), +which I should try some time. From 530c1b092a8658c8dab4261cd6a0de3a40cbfc6c Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnXgp-iIaBK5pnk22xqMVERQb97VyXaejs" Date: Thu, 5 Jul 2012 08:04:54 +0000 Subject: [PATCH 4017/8313] --- doc/forum/Problems_using_submodules_with_git-annex__63__.mdwn | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/forum/Problems_using_submodules_with_git-annex__63__.mdwn diff --git a/doc/forum/Problems_using_submodules_with_git-annex__63__.mdwn b/doc/forum/Problems_using_submodules_with_git-annex__63__.mdwn new file mode 100644 index 0000000000..39ebe855c1 --- /dev/null +++ b/doc/forum/Problems_using_submodules_with_git-annex__63__.mdwn @@ -0,0 +1 @@ +Are there any problems using submodules with git-annex? I have not tried it yet, I'm just asking. From 83c66ccaf88a10e8f4b16fc2162cbed2656b95e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 10:21:22 -0600 Subject: [PATCH 4018/8313] queue Uploads of newly added files to remotes Added knownRemotes to DaemonStatus. This list is not entirely trivial to calculate, and having it here should make it easier to add/remove remotes on the fly later on. It did require plumbing the daemonstatus through to some more threads. --- Assistant.hs | 9 ++++++--- Assistant/DaemonStatus.hs | 8 +++++++- Assistant/Threads/Committer.hs | 28 ++++++++++++++++------------ Assistant/Threads/Pusher.hs | 11 +++++++---- Assistant/TransferQueue.hs | 29 ++++++++++++++++++++++------- 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 40f53d55ee..548850e92d 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -21,7 +21,8 @@ - until this is complete. - Thread 5: committer - Waits for changes to occur, and runs the git queue to update its - - index, then commits. + - index, then commits. Also queues Transfer events to send added + - files to other remotes. - Thread 6: pusher - Waits for commits to be made, and pushes updated branches to remotes, - in parallel. (Forks a process for each git push.) @@ -73,6 +74,7 @@ import Assistant.DaemonStatus import Assistant.Changes import Assistant.Commits import Assistant.Pushes +import Assistant.TransferQueue import Assistant.Threads.Watcher import Assistant.Threads.Committer import Assistant.Threads.Pusher @@ -103,9 +105,10 @@ startDaemon assistant foreground changechan <- newChangeChan commitchan <- newCommitChan pushmap <- newFailedPushMap + transferqueue <- newTransferQueue mapM_ (void . forkIO) - [ commitThread st changechan commitchan - , pushThread st commitchan pushmap + [ commitThread st changechan commitchan transferqueue dstatus + , pushThread st dstatus commitchan pushmap , pushRetryThread st pushmap , mergeThread st , transferWatcherThread st dstatus diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 10161a96cb..a3e909904f 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -11,13 +11,14 @@ import Common.Annex import Assistant.ThreadedMonad import Utility.ThreadScheduler import Utility.TempFile +import Logs.Transfer +import qualified Command.Sync import Control.Concurrent import System.Posix.Types import Data.Time.Clock.POSIX import Data.Time import System.Locale -import Logs.Transfer import qualified Data.Map as M data DaemonStatus = DaemonStatus @@ -31,6 +32,8 @@ data DaemonStatus = DaemonStatus , lastSanityCheck :: Maybe POSIXTime -- Currently running file content transfers , currentTransfers :: M.Map Transfer TransferInfo + -- Ordered list of remotes to talk to. + , knownRemotes :: [Remote] } deriving (Show) @@ -43,6 +46,7 @@ newDaemonStatus = DaemonStatus , sanityCheckRunning = False , lastSanityCheck = Nothing , currentTransfers = M.empty + , knownRemotes = [] } getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus @@ -59,10 +63,12 @@ startDaemonStatus = do status <- liftIO $ catchDefaultIO (readDaemonStatusFile file) newDaemonStatus transfers <- M.fromList <$> getTransfers + remotes <- Command.Sync.syncRemotes [] liftIO $ newMVar status { scanComplete = False , sanityCheckRunning = False , currentTransfers = transfers + , knownRemotes = remotes } {- This thread wakes up periodically and writes the daemon status to disk. -} diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index 488056fa2b..ff5cc9eabc 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -12,6 +12,9 @@ import Assistant.Changes import Assistant.Commits import Assistant.ThreadedMonad import Assistant.Threads.Watcher +import Assistant.TransferQueue +import Assistant.DaemonStatus +import Logs.Transfer import qualified Annex import qualified Annex.Queue import qualified Git.Command @@ -29,8 +32,8 @@ import qualified Data.Set as S import Data.Either {- This thread makes git commits at appropriate times. -} -commitThread :: ThreadState -> ChangeChan -> CommitChan -> IO () -commitThread st changechan commitchan = runEvery (Seconds 1) $ do +commitThread :: ThreadState -> ChangeChan -> CommitChan -> TransferQueue -> DaemonStatusHandle -> IO () +commitThread st changechan commitchan transferqueue dstatus = runEvery (Seconds 1) $ do -- We already waited one second as a simple rate limiter. -- Next, wait until at least one change is available for -- processing. @@ -39,7 +42,7 @@ commitThread st changechan commitchan = runEvery (Seconds 1) $ do time <- getCurrentTime if shouldCommit time changes then do - readychanges <- handleAdds st changechan changes + readychanges <- handleAdds st changechan transferqueue dstatus changes if shouldCommit time readychanges then do void $ tryIO $ runThreadState st commitStaged @@ -97,8 +100,8 @@ shouldCommit now changes - Any pending adds that are not ready yet are put back into the ChangeChan, - where they will be retried later. -} -handleAdds :: ThreadState -> ChangeChan -> [Change] -> IO [Change] -handleAdds st changechan cs = returnWhen (null pendingadds) $ do +handleAdds :: ThreadState -> ChangeChan -> TransferQueue -> DaemonStatusHandle -> [Change] -> IO [Change] +handleAdds st changechan transferqueue dstatus cs = returnWhen (null pendingadds) $ do (postponed, toadd) <- partitionEithers <$> safeToAdd st pendingadds @@ -110,7 +113,7 @@ handleAdds st changechan cs = returnWhen (null pendingadds) $ do if (DirWatcher.eventsCoalesce || null added) then return $ added ++ otherchanges else do - r <- handleAdds st changechan + r <- handleAdds st changechan transferqueue dstatus =<< getChanges changechan return $ r ++ added ++ otherchanges where @@ -121,12 +124,12 @@ handleAdds st changechan cs = returnWhen (null pendingadds) $ do | otherwise = a add :: Change -> IO (Maybe Change) - add change@(PendingAddChange { keySource = ks }) = do - r <- catchMaybeIO $ sanitycheck ks $ runThreadState st $ do - showStart "add" $ keyFilename ks - handle (finishedChange change) (keyFilename ks) - =<< Command.Add.ingest ks - return $ maybeMaybe r + add change@(PendingAddChange { keySource = ks }) = + liftM maybeMaybe $ catchMaybeIO $ + sanitycheck ks $ runThreadState st $ do + showStart "add" $ keyFilename ks + key <- Command.Add.ingest ks + handle (finishedChange change) (keyFilename ks) key add _ = return Nothing maybeMaybe (Just j@(Just _)) = j @@ -141,6 +144,7 @@ handleAdds st changechan cs = returnWhen (null pendingadds) $ do sha <- inRepo $ Git.HashObject.hashObject BlobObject link stageSymlink file sha + queueTransfers transferqueue dstatus key (Just file) Upload showEndOk return $ Just change diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs index 04d3435287..6d6836120e 100644 --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -10,6 +10,7 @@ module Assistant.Threads.Pusher where import Common.Annex import Assistant.Commits import Assistant.Pushes +import Assistant.DaemonStatus import Assistant.ThreadedMonad import Assistant.Threads.Merger import qualified Command.Sync @@ -32,9 +33,8 @@ pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do halfhour = 1800 {- This thread pushes git commits out to remotes soon after they are made. -} -pushThread :: ThreadState -> CommitChan -> FailedPushMap -> IO () -pushThread st commitchan pushmap = do - remotes <- runThreadState st $ Command.Sync.syncRemotes [] +pushThread :: ThreadState -> DaemonStatusHandle -> CommitChan -> FailedPushMap -> IO () +pushThread st daemonstatus commitchan pushmap = do runEvery (Seconds 2) $ do -- We already waited two seconds as a simple rate limiter. -- Next, wait until at least one commit has been made @@ -42,7 +42,10 @@ pushThread st commitchan pushmap = do -- Now see if now's a good time to push. now <- getCurrentTime if shouldPush now commits - then pushToRemotes now st pushmap remotes + then do + remotes <- runThreadState st $ + knownRemotes <$> getDaemonStatus daemonstatus + pushToRemotes now st pushmap remotes else refillCommits commitchan commits {- Decide if now is a good time to push to remotes. diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index 979cbb80f5..fc25b057d3 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -8,9 +8,10 @@ module Assistant.TransferQueue where import Common.Annex -import Utility.TSet +import Assistant.DaemonStatus import Logs.Transfer import Types.Remote +import qualified Remote import Control.Concurrent.STM @@ -28,15 +29,29 @@ stubInfo f = TransferInfo , associatedFile = f } +{- Adds pending transfers to the end of the queue for some of the known + - remotes. (TBD: a smaller set of remotes that are sufficient to transfer to, + - rather than transferring to all.) -} +queueTransfers :: TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex () +queueTransfers q daemonstatus k f direction = + mapM_ (liftIO . queueTransfer q f . gentransfer) + =<< knownRemotes <$> getDaemonStatus daemonstatus + where + gentransfer r = Transfer + { transferDirection = direction + , transferKey = k + , transferRemote = Remote.uuid r + } + {- Adds a pending transfer to the end of the queue. -} -queueTransfer :: TransferQueue -> Transfer -> AssociatedFile -> IO () -queueTransfer q transfer f = void $ atomically $ - writeTChan q (transfer, stubInfo f) +queueTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO () +queueTransfer q f t = void $ atomically $ + writeTChan q (t, stubInfo f) {- Adds a pending transfer to the start of the queue, to be processed next. -} -queueNextTransfer :: TransferQueue -> Transfer -> AssociatedFile -> IO () -queueNextTransfer q transfer f = void $ atomically $ - unGetTChan q (transfer, stubInfo f) +queueNextTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO () +queueNextTransfer q f t = void $ atomically $ + unGetTChan q (t, stubInfo f) {- Blocks until a pending transfer is available in the queue. -} getNextTransfer :: TransferQueue -> IO (Transfer, TransferInfo) From c1728849a9b5d1e7803cf6a36f826a7bafdc667a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 10:34:47 -0600 Subject: [PATCH 4019/8313] update --- doc/design/assistant/syncing.mdwn | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index d2248279f7..caae60a32e 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -16,7 +16,7 @@ all the other git clones, at both the git level and the key/value level. * Poll transfer in progress info files for changes (use inotify again! wow! hammer, meet nail..), and update the TransferInfo Map **done** * enqueue Transfers (Uploads) as new files are added to the annex by - Watcher. + Watcher. **done** * enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by Watcher. * Write basic Transfer handling thread. Multiple such threads need to be @@ -82,11 +82,8 @@ anyway. ### transfer tracking -* Upload added to queue by the watcher thread when it adds content. -* Download added to queue by the watcher thread when it seens new symlinks - that lack content. -* Transfer threads started/stopped as necessary to move data. - (May sometimes want multiple threads downloading, or uploading, or even both.) +Transfer threads started/stopped as necessary to move data. +(May sometimes want multiple threads downloading, or uploading, or even both.) startTransfer :: TransferQueue -> Transfer -> Annex () startTransfer q transfer = error "TODO" From c8135ea0a8aa2b374e45a8bb8c447c5287862838 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 10:44:03 -0600 Subject: [PATCH 4020/8313] split logic for uploads and downloads --- Assistant/TransferQueue.hs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index fc25b057d3..f1f4882bef 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -30,13 +30,26 @@ stubInfo f = TransferInfo } {- Adds pending transfers to the end of the queue for some of the known - - remotes. (TBD: a smaller set of remotes that are sufficient to transfer to, - - rather than transferring to all.) -} + - remotes. -} queueTransfers :: TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex () queueTransfers q daemonstatus k f direction = mapM_ (liftIO . queueTransfer q f . gentransfer) - =<< knownRemotes <$> getDaemonStatus daemonstatus + =<< sufficientremotes . knownRemotes + <$> getDaemonStatus daemonstatus where + sufficientremotes l + -- Queue downloads from all remotes, with the + -- cheapest ones first. More expensive ones will + -- only be tried if downloading from a cheap one + -- fails. + -- TODO: avoid downloading from remotes that don't + -- have the key. + | direction == Download = l + -- TODO: Determine a smaller set of remotes that + -- can be uploaded to, in order to ensure all + -- remotes can access the content. Currently, + -- send to every remote we can. + | otherwise = l gentransfer r = Transfer { transferDirection = direction , transferKey = k From 6af319d8cdefb4589d9cd354dbc49006bb7d68ea Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 10:58:49 -0600 Subject: [PATCH 4021/8313] enqueue Downloads when new symlinks appear to content we don't have --- Assistant.hs | 4 +-- Assistant/Threads/SanityChecker.hs | 13 +++++----- Assistant/Threads/Watcher.hs | 39 ++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 548850e92d..82ac2037e3 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -113,8 +113,8 @@ startDaemon assistant foreground , mergeThread st , transferWatcherThread st dstatus , daemonStatusThread st dstatus - , sanityCheckerThread st dstatus changechan - , watchThread st dstatus changechan + , sanityCheckerThread st dstatus transferqueue changechan + , watchThread st dstatus transferqueue changechan ] waitForTermination diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs index 4db2a61b22..d7b117cd02 100644 --- a/Assistant/Threads/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -14,14 +14,15 @@ import qualified Git.LsFiles import Assistant.DaemonStatus import Assistant.ThreadedMonad import Assistant.Changes +import Assistant.TransferQueue import Utility.ThreadScheduler import qualified Assistant.Threads.Watcher as Watcher import Data.Time.Clock.POSIX {- This thread wakes up occasionally to make sure the tree is in good shape. -} -sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () -sanityCheckerThread st status changechan = forever $ do +sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () +sanityCheckerThread st status transferqueue changechan = forever $ do waitForNextCheck st status runThreadState st $ @@ -29,7 +30,7 @@ sanityCheckerThread st status changechan = forever $ do { sanityCheckRunning = True } now <- getPOSIXTime -- before check started - catchIO (check st status changechan) + catchIO (check st status transferqueue changechan) (runThreadState st . warning . show) runThreadState st $ do @@ -58,8 +59,8 @@ oneDay = 24 * 60 * 60 {- It's important to stay out of the Annex monad as much as possible while - running potentially expensive parts of this check, since remaining in it - will block the watcher. -} -check :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () -check st status changechan = do +check :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () +check st status transferqueue changechan = do g <- runThreadState st $ do showSideAction "Running daily check" fromRepo id @@ -79,5 +80,5 @@ check st status changechan = do insanity m = runThreadState st $ warning m addsymlink file s = do insanity $ "found unstaged symlink: " ++ file - Watcher.runHandler st status changechan + Watcher.runHandler st status transferqueue changechan Watcher.onAddSymlink file s diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index e250f4b4a6..882aab3a78 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -13,6 +13,8 @@ import Common.Annex import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.Changes +import Assistant.TransferQueue +import Logs.Transfer import Utility.DirWatcher import Utility.Types.DirWatcher import qualified Annex @@ -45,11 +47,11 @@ needLsof = error $ unlines , "Be warned: This can corrupt data in the annex, and make fsck complain." ] -watchThread :: ThreadState -> DaemonStatusHandle -> ChangeChan -> IO () -watchThread st dstatus changechan = void $ watchDir "." ignored hooks startup +watchThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () +watchThread st dstatus transferqueue changechan = void $ watchDir "." ignored hooks startup where startup = statupScan st dstatus - hook a = Just $ runHandler st dstatus changechan a + hook a = Just $ runHandler st dstatus transferqueue changechan a hooks = WatchHooks { addHook = hook onAdd , delHook = hook onDel @@ -82,22 +84,22 @@ ignored = ig . takeFileName ig ".gitattributes" = True ig _ = False -type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> Annex (Maybe Change) +type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> TransferQueue -> Annex (Maybe Change) {- Runs an action handler, inside the Annex monad, and if there was a - change, adds it to the ChangeChan. - - Exceptions are ignored, otherwise a whole watcher thread could be crashed. -} -runHandler :: ThreadState -> DaemonStatusHandle -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler st dstatus changechan handler file filestatus = void $ do +runHandler :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler st dstatus transferqueue changechan handler file filestatus = void $ do r <- tryIO go case r of Left e -> print e Right Nothing -> noop Right (Just change) -> recordChange changechan change where - go = runThreadState st $ handler file filestatus dstatus + go = runThreadState st $ handler file filestatus dstatus transferqueue {- During initial directory scan, this will be run for any regular files - that are already checked into git. We don't want to turn those into @@ -118,7 +120,7 @@ runHandler st dstatus changechan handler file filestatus = void $ do - the add. -} onAdd :: Handler -onAdd file filestatus dstatus +onAdd file filestatus dstatus _ | maybe False isRegularFile filestatus = do ifM (scanComplete <$> getDaemonStatus dstatus) ( go @@ -136,12 +138,15 @@ onAdd file filestatus dstatus - before adding it. -} onAddSymlink :: Handler -onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file +onAddSymlink file filestatus dstatus transferqueue = go =<< Backend.lookupFile file where go (Just (key, _)) = do link <- calcGitLink file key ifM ((==) link <$> liftIO (readSymbolicLink file)) - ( ensurestaged link =<< getDaemonStatus dstatus + ( do + s <- getDaemonStatus dstatus + checkcontent key s + ensurestaged link s , do liftIO $ removeFile file liftIO $ createSymbolicLink link file @@ -183,8 +188,16 @@ onAddSymlink file filestatus dstatus = go =<< Backend.lookupFile file stageSymlink file sha madeChange file LinkChange + {- When a new link appears, after the startup scan, + - try to get the key's content. -} + checkcontent key daemonstatus + | scanComplete daemonstatus = unlessM (inAnnex key) $ + queueTransfers transferqueue dstatus + key (Just file) Download + | otherwise = noop + onDel :: Handler -onDel file _ _dstatus = do +onDel file _ _dstatus _ = do Annex.Queue.addUpdateIndex =<< inRepo (Git.UpdateIndex.unstageFile file) madeChange file RmChange @@ -197,14 +210,14 @@ onDel file _ _dstatus = do - command to get the recursive list of files in the directory, so rm is - just as good. -} onDelDir :: Handler -onDelDir dir _ _dstatus = do +onDelDir dir _ _dstatus _ = do Annex.Queue.addCommand "rm" [Params "--quiet -r --cached --ignore-unmatch --"] [dir] madeChange dir RmDirChange {- Called when there's an error with inotify. -} onErr :: Handler -onErr msg _ _dstatus = do +onErr msg _ _dstatus _ = do warning msg return Nothing From bd2c4c982d79d72c67da838c71c869194d26ea4d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 10:34:47 -0600 Subject: [PATCH 4022/8313] update --- doc/design/assistant/syncing.mdwn | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index d2248279f7..d4d89d03b5 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -16,9 +16,9 @@ all the other git clones, at both the git level and the key/value level. * Poll transfer in progress info files for changes (use inotify again! wow! hammer, meet nail..), and update the TransferInfo Map **done** * enqueue Transfers (Uploads) as new files are added to the annex by - Watcher. + Watcher. **done** * enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by - Watcher. + Watcher. **done** * Write basic Transfer handling thread. Multiple such threads need to be able to be run at once. Each will need its own independant copy of the Annex state monad. @@ -82,11 +82,8 @@ anyway. ### transfer tracking -* Upload added to queue by the watcher thread when it adds content. -* Download added to queue by the watcher thread when it seens new symlinks - that lack content. -* Transfer threads started/stopped as necessary to move data. - (May sometimes want multiple threads downloading, or uploading, or even both.) +Transfer threads started/stopped as necessary to move data. +(May sometimes want multiple threads downloading, or uploading, or even both.) startTransfer :: TransferQueue -> Transfer -> Annex () startTransfer q transfer = error "TODO" From 08d6e6903b6a419a62f3d4d7b74268963ad85d00 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Thu, 5 Jul 2012 17:04:34 +0000 Subject: [PATCH 4023/8313] Added a comment --- .../comment_1_c7a927736d419d3c31c912001ff16ee4._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/forum/Problems_using_submodules_with_git-annex__63__/comment_1_c7a927736d419d3c31c912001ff16ee4._comment diff --git a/doc/forum/Problems_using_submodules_with_git-annex__63__/comment_1_c7a927736d419d3c31c912001ff16ee4._comment b/doc/forum/Problems_using_submodules_with_git-annex__63__/comment_1_c7a927736d419d3c31c912001ff16ee4._comment new file mode 100644 index 0000000000..3c2f5addba --- /dev/null +++ b/doc/forum/Problems_using_submodules_with_git-annex__63__/comment_1_c7a927736d419d3c31c912001ff16ee4._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 1" + date="2012-07-05T17:04:34Z" + content=""" +I haven't tried it either, but I think it should work ok, as long as you bear in mind that to git-annex, each submodule will be treated as a separate git repository. +"""]] From 71b5ad8398c4d86d5e9b993e175b48f2c5f0861d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 14:34:20 -0600 Subject: [PATCH 4024/8313] wrote transfer thread finally! --- Assistant.hs | 8 ++- Assistant/DaemonStatus.hs | 9 ++- Assistant/Threads/TransferWatcher.hs | 20 +++--- Assistant/Threads/Transferrer.hs | 102 +++++++++++++++++++++++++++ Assistant/TransferQueue.hs | 8 ++- Command/Status.hs | 4 +- Logs/Transfer.hs | 5 +- 7 files changed, 136 insertions(+), 20 deletions(-) create mode 100644 Assistant/Threads/Transferrer.hs diff --git a/Assistant.hs b/Assistant.hs index 82ac2037e3..e751b4ae8c 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -39,9 +39,11 @@ - and maintains the DaemonStatus currentTransfers map. This uses - inotify on .git/annex/transfer/, so there are additional inotify - threads associated with it, too. - - Thread 10: status logger + - Thread 10: transferrer + - Waits for Transfers to be queued and does them. + - Thread 11: status logger - Wakes up periodically and records the daemon's status to disk. - - Thread 11: sanity checker + - Thread 12: sanity checker - Wakes up periodically (rarely) and does sanity checks. - - ThreadState: (MVar) @@ -80,6 +82,7 @@ import Assistant.Threads.Committer import Assistant.Threads.Pusher import Assistant.Threads.Merger import Assistant.Threads.TransferWatcher +import Assistant.Threads.Transferrer import Assistant.Threads.SanityChecker import qualified Utility.Daemon import Utility.LogFile @@ -114,6 +117,7 @@ startDaemon assistant foreground , transferWatcherThread st dstatus , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan + , transfererThread st dstatus transferqueue , watchThread st dstatus transferqueue changechan ] waitForTermination diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index a3e909904f..40816bb1a7 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -31,12 +31,14 @@ data DaemonStatus = DaemonStatus -- Last time the sanity checker ran , lastSanityCheck :: Maybe POSIXTime -- Currently running file content transfers - , currentTransfers :: M.Map Transfer TransferInfo + , currentTransfers :: TransferMap -- Ordered list of remotes to talk to. , knownRemotes :: [Remote] } deriving (Show) +type TransferMap = M.Map Transfer TransferInfo + type DaemonStatusHandle = MVar DaemonStatus newDaemonStatus :: DaemonStatus @@ -132,3 +134,8 @@ afterLastDaemonRun timestamp status = maybe False (< t) (lastRunning status) tenMinutes :: Int tenMinutes = 10 * 60 + +{- Mutates the transfer map. -} +adjustTransfers :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> Annex () +adjustTransfers dstatus a = modifyDaemonStatus dstatus $ + \s -> s { currentTransfers = a (currentTransfers s) } diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 811b045a82..f18d4e3f86 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -58,21 +58,17 @@ onAdd :: Handler onAdd st dstatus file _ = case parseTransferFile file of Nothing -> noop Just t -> do - minfo <- runThreadState st $ checkTransfer t pid <- getProcessID - case minfo of - Nothing -> noop -- transfer already finished - Just info - | transferPid info == Just pid -> noop - | otherwise -> adjustTransfers st dstatus - (M.insertWith' const t info) + runThreadState st $ go t pid =<< checkTransfer t + where + go _ _ Nothing = noop -- transfer already finished + go t pid (Just info) + | transferPid info == Just pid = noop + | otherwise = adjustTransfers dstatus $ + M.insertWith' const t info {- Called when a transfer information file is removed. -} onDel :: Handler onDel st dstatus file _ = case parseTransferFile file of Nothing -> noop - Just t -> adjustTransfers st dstatus (M.delete t) - -adjustTransfers :: ThreadState -> DaemonStatusHandle -> (M.Map Transfer TransferInfo -> M.Map Transfer TransferInfo) -> IO () -adjustTransfers st dstatus a = runThreadState st $ modifyDaemonStatus dstatus $ - \s -> s { currentTransfers = a (currentTransfers s) } + Just t -> runThreadState st $ adjustTransfers dstatus $ M.delete t diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs new file mode 100644 index 0000000000..0562a607ce --- /dev/null +++ b/Assistant/Threads/Transferrer.hs @@ -0,0 +1,102 @@ +{- git-annex assistant data transferrer thread + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.Transferrer where + +import Common.Annex +import Assistant.ThreadedMonad +import Assistant.DaemonStatus +import Assistant.TransferQueue +import Logs.Transfer +import Annex.Content +import Annex.BranchState +import Command +import qualified Command.Move + +import Control.Exception as E +import Control.Concurrent +import Data.Time.Clock +import qualified Data.Map as M + +{- Dispatches transfers from the queue. + - + - This is currently very simplistic, and runs only one transfer at a time. + -} +transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> IO () +transfererThread st dstatus transferqueue = do + mypid <- getProcessID + mytid <- myThreadId + go mypid mytid + where + go mypid mytid = do + (t, info) <- getNextTransfer transferqueue + + now <- getCurrentTime + let info' = info + { startedTime = Just now + , transferPid = Just mypid + , transferThread = Just mytid + } + + ifM (runThreadState st $ shouldtransfer t info') + ( runTransfer st t info' + , noop + ) + go mypid mytid + + -- Check if the transfer is already running, + -- and if not, add it to the TransferMap. + shouldtransfer t info = do + current <- currentTransfers <$> getDaemonStatus dstatus + if M.member t current + then ifM (validtransfer t) + ( do + adjustTransfers dstatus $ + M.insertWith' const t info + return True + , return False + ) + else return False + + validtransfer t + | transferDirection t == Download = + not <$> inAnnex (transferKey t) + | otherwise = return True + +{- A transfer is run in a separate thread, with a *copy* of the Annex + - state. This is necessary to avoid blocking the rest of the assistant + - on the transfer completing, and also to allow multiple transfers to run + - at once. + - + - However, it means that the transfer threads are responsible + - for doing any necessary shutdown cleanups, and that the parent + - thread's cache must be invalidated, as changes may have been made to the + - git-annex branch. + - + - Currently a minimal shutdown is done; the transfer threads are + - effectively running in oneshot mode, without committing changes to the + - git-annex branch, and transfers should never queue git commands to run. + - + - Note: It is unsafe to call getDaemonStatus inside the transfer thread. + -} +runTransfer :: ThreadState -> Transfer -> TransferInfo -> IO () +runTransfer st t info + | transferDirection t == Download = go Command.Move.fromStart + | otherwise = go Command.Move.toStart + where + go cmd = case (transferRemote info, associatedFile info) of + (Nothing, _) -> noop + (_, Nothing) -> noop + (Just remote, Just file) -> + inthread $ void $ doCommand $ + cmd remote False file (transferKey t) + inthread a = do + mvar <- newEmptyMVar + void $ forkIO $ + runThreadState st a `E.finally` putMVar mvar () + void $ takeMVar mvar -- wait for transfer thread + runThreadState st invalidateCache diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index f1f4882bef..a35815ca16 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -25,6 +25,7 @@ stubInfo f = TransferInfo { startedTime = Nothing , transferPid = Nothing , transferThread = Nothing + , transferRemote = Nothing , bytesComplete = Nothing , associatedFile = f } @@ -33,7 +34,7 @@ stubInfo f = TransferInfo - remotes. -} queueTransfers :: TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex () queueTransfers q daemonstatus k f direction = - mapM_ (liftIO . queueTransfer q f . gentransfer) + mapM_ (\r -> queue r $ gentransfer r) =<< sufficientremotes . knownRemotes <$> getDaemonStatus daemonstatus where @@ -53,8 +54,11 @@ queueTransfers q daemonstatus k f direction = gentransfer r = Transfer { transferDirection = direction , transferKey = k - , transferRemote = Remote.uuid r + , transferUUID = Remote.uuid r } + queue r t = liftIO $ void $ atomically $ do + let info = (stubInfo f) { transferRemote = Just r } + writeTChan q (t, info) {- Adds a pending transfer to the end of the queue. -} queueTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO () diff --git a/Command/Status.hs b/Command/Status.hs index eff21bb509..2d63c525c3 100644 --- a/Command/Status.hs +++ b/Command/Status.hs @@ -186,8 +186,8 @@ transfer_list = stat "transfers in progress" $ nojson $ lift $ do [ show (transferDirection t) ++ "ing" , fromMaybe (show $ transferKey t) (associatedFile i) , if transferDirection t == Upload then "to" else "from" - , maybe (fromUUID $ transferRemote t) Remote.name $ - M.lookup (transferRemote t) uuidmap + , maybe (fromUUID $ transferUUID t) Remote.name $ + M.lookup (transferUUID t) uuidmap ] disk_size :: Stat diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index f808cb6a44..12ab8ff113 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -22,7 +22,7 @@ import Data.Time.Clock - of the transfer information file. -} data Transfer = Transfer { transferDirection :: Direction - , transferRemote :: UUID + , transferUUID :: UUID , transferKey :: Key } deriving (Show, Eq, Ord) @@ -37,6 +37,7 @@ data TransferInfo = TransferInfo { startedTime :: Maybe UTCTime , transferPid :: Maybe ProcessID , transferThread :: Maybe ThreadId + , transferRemote :: Maybe Remote , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath } @@ -80,6 +81,7 @@ transfer t file a = do <*> pure Nothing -- pid not stored in file, so omitted for speed <*> pure Nothing -- threadid not stored in file, so omitted for speed <*> pure Nothing -- not 0; transfer may be resuming + <*> pure Nothing <*> pure file bracketIO (prep tfile mode info) (cleanup tfile) a where @@ -170,6 +172,7 @@ readTransferInfo pid s = <*> pure (Just pid) <*> pure Nothing <*> pure Nothing + <*> pure Nothing <*> pure (if null filename then Nothing else Just filename) _ -> Nothing where From 2136ee4adbeba262dcf184e653e1e07acb02a2bd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 14:45:21 -0600 Subject: [PATCH 4025/8313] logic error --- Assistant/Threads/Transferrer.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 0562a607ce..29cc393f23 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -53,14 +53,14 @@ transfererThread st dstatus transferqueue = do shouldtransfer t info = do current <- currentTransfers <$> getDaemonStatus dstatus if M.member t current - then ifM (validtransfer t) + then return False + else ifM (validtransfer t) ( do adjustTransfers dstatus $ M.insertWith' const t info return True , return False ) - else return False validtransfer t | transferDirection t == Download = From b1629356ce0db4146b26292054bd894ce0eec774 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 14:47:10 -0600 Subject: [PATCH 4026/8313] blog for the day major milestone today! --- .../blog/day_25__transfer_queueing.mdwn | 41 +++++++++++++++++++ doc/design/assistant/syncing.mdwn | 34 ++------------- 2 files changed, 45 insertions(+), 30 deletions(-) create mode 100644 doc/design/assistant/blog/day_25__transfer_queueing.mdwn diff --git a/doc/design/assistant/blog/day_25__transfer_queueing.mdwn b/doc/design/assistant/blog/day_25__transfer_queueing.mdwn new file mode 100644 index 0000000000..35922c0d11 --- /dev/null +++ b/doc/design/assistant/blog/day_25__transfer_queueing.mdwn @@ -0,0 +1,41 @@ +So as not to bury the lead, I've been hard at work on my first day in +Nicaragua, and ** the git-annex assistant fully syncs files (including +their contents) between remotes now !! ** + +Details follow.. + +Made the committer thread queue Upload Transfers when new files +are added to the annex. Currently it tries to transfer the new content +to *every* remote; this innefficiency needs to be addressed later. + +Made the watcher thread queue Download Transfers when new symlinks +appear that point to content we don't have. Typically, that will happen +after an automatic merge from a remote. This needs to be improved as it +currently adds Transfers from every remote, not just those that have the +content. + +This was the second place that needed an ordered list of remotes +to talk to. So I cached such a list in the DaemonStatus state info. +This will also be handy later on, when the webapp is used to add new +remotes, so the assistant can know about them immediately. + +Added YAT (Yet Another Thread), number 15 or so, the transferrer thread +that waits for transfers to be queued and runs them. Currently a naive +implementation, it runs one transfer at a time, and does not do anything +to recover when a transfer fails. + +Actually transferring content requires YAT, so that the transfer +action can run in a copy of the Annex monad, without blocking +all the assistant's other threads from entering that monad while a transfer +is running. This is also necessary to allow multiple concurrent transfers +to run in the future. + +This is a very tricky peice of code, because that thread will modify the +git-annex branch, and its parent thread has to invalidate its cache in +order to see any changes the child thread made. Hopefully that's the extent +of the complication of doing this. The only reason this was possible at all +is that git-annex already support multiple concurrent processes running +and all making independant changes to the git-annex branch, etc. + +After all my groundwork this week, file content transferring is now +fully working! diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index d4d89d03b5..fc2ac3e5ee 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -21,8 +21,11 @@ all the other git clones, at both the git level and the key/value level. Watcher. **done** * Write basic Transfer handling thread. Multiple such threads need to be able to be run at once. Each will need its own independant copy of the - Annex state monad. + Annex state monad. **done** * Write transfer control thread, which decides when to launch transfers. + **done** +* Check that download transfer triggering code works (when a symlink appears + and the remote does *not* upload to us. * At startup, and possibly periodically, look for files we have that location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. @@ -80,35 +83,6 @@ reachable remote. This is worth doing first, since it's the simplest way to get the basic functionality of the assistant to work. And we'll need this anyway. -### transfer tracking - -Transfer threads started/stopped as necessary to move data. -(May sometimes want multiple threads downloading, or uploading, or even both.) - - startTransfer :: TransferQueue -> Transfer -> Annex () - startTransfer q transfer = error "TODO" - - stopTransfer :: TransferQueue -> TransferID -> Annex () - stopTransfer q transfer = error "TODO" - -The assistant needs to find out when `git-annex-shell` is receiving or -sending (triggered by another remote), so it can add data for those too. -This is important to avoid uploading content to a remote that is already -downloading it from us, or vice versa, as well as to in future let the web -app manage transfers as user desires. - -For files being received, it can see the temp file, but other than lsof -there's no good way to find the pid (and I'd rather not kill blindly). - -For files being sent, there's no filesystem indication. So git-annex-shell -(and other git-annex transfer processes) should write a status file to disk. - -Can use file locking on these status files to claim upload/download rights, -which will avoid races. - -This status file can also be updated periodically to show amount of transfer -complete (necessary for tracking uploads). - ## other considerations It would be nice if, when a USB drive is connected, From 9eaba58dd9706fde7e0fb84364a16576db63a7e0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 16:07:49 -0600 Subject: [PATCH 4027/8313] run transfer with copy of annex state This should have made it run concurrently with other annex actions, but I'm still seeing it serialize. Perhaps I need to forkProcess? --- Assistant/ThreadedMonad.hs | 12 ++++++++++++ Assistant/Threads/Transferrer.hs | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 7b915e12c8..4e871ab676 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -36,3 +36,15 @@ withThreadState a = do - time. -} runThreadState :: ThreadState -> Annex a -> IO a runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a + +{- Runs an Annex action, using a copy of the state from the MVar. + - + - The state modified by the action is thrown away, so it's up to the + - action to perform any necessary shutdown tasks in order for state to not + - be lost. And it's up to the caller to resynchronise with any changes + - the action makes to eg, the git-annex branch. + -} +unsafeRunThreadState :: ThreadState -> Annex a -> IO a +unsafeRunThreadState mvar a = do + state <- readMVar mvar + Annex.eval state a diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 29cc393f23..0b47e97812 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -97,6 +97,6 @@ runTransfer st t info inthread a = do mvar <- newEmptyMVar void $ forkIO $ - runThreadState st a `E.finally` putMVar mvar () + unsafeRunThreadState st a `E.finally` putMVar mvar () void $ takeMVar mvar -- wait for transfer thread runThreadState st invalidateCache From 5a753a7b8a46c7326b6431dcf5a6eb755534e80d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 16:24:02 -0600 Subject: [PATCH 4028/8313] SHAnE backends are now smarter about composite extensions, such as .tar.gz Closes: #680450 --- Backend/SHA.hs | 17 +++++++++-------- debian/changelog | 2 ++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 7abbf8035a..95ce4a7701 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -97,16 +97,17 @@ keyValueE :: SHASize -> KeySource -> Annex (Maybe Key) keyValueE size source = keyValue size source >>= maybe (return Nothing) addE where addE k = return $ Just $ k - { keyName = keyName k ++ extension + { keyName = keyName k ++ selectExtension (keyFilename source) , keyBackendName = shaNameE size } - naiveextension = takeExtension $ keyFilename source - extension - -- long or newline containing extensions are - -- probably not really an extension - | length naiveextension > 6 || - '\n' `elem` naiveextension = "" - | otherwise = naiveextension + +selectExtension :: FilePath -> String +selectExtension = join "." . reverse . take 2 . takeWhile shortenough . + reverse . split "." . takeExtensions + where + shortenough e + | '\n' `elem` e = False -- newline in extension?! + | otherwise = length e <= 4 -- long enough for "jpeg" {- A key's checksum is checked during fsck. -} checkKeyChecksum :: SHASize -> Key -> FilePath -> Annex Bool diff --git a/debian/changelog b/debian/changelog index 1c44f59526..5eaf9d52eb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ git-annex (3.20120630) UNRELEASED; urgency=low but avoids portability problems. * Use SHA library for files less than 50 kb in size, at which point it's faster than forking the more optimised external program. + * SHAnE backends are now smarter about composite extensions, such as + .tar.gz Closes: #680450 -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 From f462520bf93997d899cfaa8eace79f962ba66592 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 16:41:17 -0600 Subject: [PATCH 4029/8313] todo --- doc/design/assistant/syncing.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index fc2ac3e5ee..de07ef2d22 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -26,6 +26,7 @@ all the other git clones, at both the git level and the key/value level. **done** * Check that download transfer triggering code works (when a symlink appears and the remote does *not* upload to us. +* Investigate why transfers seem to block other git-annex assistant work. * At startup, and possibly periodically, look for files we have that location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. From f5cb8ed6906aff0f793b9bea9714227ece66e474 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E" Date: Fri, 6 Jul 2012 00:12:16 +0000 Subject: [PATCH 4030/8313] Added a comment: Source code --- ...comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment diff --git a/doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment b/doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment new file mode 100644 index 0000000000..0a5b6c0991 --- /dev/null +++ b/doc/design/assistant/blog/day_25__transfer_queueing/comment_1_59fd4f1ffe96c412f613dc86276e7dbd._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawldKnauegZulM7X6JoHJs7Gd5PnDjcgx-E" + nickname="Matt" + subject="Source code" + date="2012-07-06T00:12:15Z" + content=""" +Hi Joey, + +Is the source code for git-annex assistant available somewhere? +"""]] From c944f50fc103107fb1082355dc804fb3255fe9a9 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Fri, 6 Jul 2012 00:21:43 +0000 Subject: [PATCH 4031/8313] Added a comment --- .../comment_2_93bf768a67117e873af5732ecf08dc78._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment diff --git a/doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment b/doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment new file mode 100644 index 0000000000..6c0ca0781d --- /dev/null +++ b/doc/design/assistant/blog/day_25__transfer_queueing/comment_2_93bf768a67117e873af5732ecf08dc78._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 2" + date="2012-07-06T00:21:43Z" + content=""" +It's in the `assistant` branch of git://git-annex.branchable.com/ +"""]] From a92f5589fcf5549832914fdee34596818bfdc583 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 5 Jul 2012 18:57:06 -0600 Subject: [PATCH 4032/8313] unfinished (and unbuildable) work toward separate transfer processes --- Assistant.hs | 24 ++++--- Assistant/Threads/Transferrer.hs | 103 ++++++++++++++----------------- Logs/Transfer.hs | 5 -- 3 files changed, 63 insertions(+), 69 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index e751b4ae8c..38ed539a1f 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -31,14 +31,15 @@ - them. - Thread 8: merger - Waits for pushes to be received from remotes, and merges the - - updated branches into the current branch. This uses inotify - - on .git/refs/heads, so there are additional inotify threads - - associated with it, too. + - updated branches into the current branch. + - (This uses inotify on .git/refs/heads, so there are additional + - inotify threads associated with it, too.) - Thread 9: transfer watcher - Watches for transfer information files being created and removed, - - and maintains the DaemonStatus currentTransfers map. This uses - - inotify on .git/annex/transfer/, so there are additional inotify - - threads associated with it, too. + - and maintains the DaemonStatus currentTransfers map and the + - TransferSlots QSemN. + - (This uses inotify on .git/annex/transfer/, so there are + - additional inotify threads associated with it, too.) - Thread 10: transferrer - Waits for Transfers to be queued and does them. - Thread 11: status logger @@ -66,6 +67,12 @@ - retrier blocks until they're available. - TransferQueue (STM TChan) - Transfers to make are indicated by writing to this channel. + - TransferSlots (QSemN) + - Count of the number of currently available transfer slots. + - Updated by the transfer watcher, this allows other threads + - to block until a slot is available. + - This MVar should only be manipulated from inside the Annex monad, + - which ensures it's accessed only after the ThreadState MVar. -} module Assistant where @@ -109,15 +116,16 @@ startDaemon assistant foreground commitchan <- newCommitChan pushmap <- newFailedPushMap transferqueue <- newTransferQueue + transferslots <- newTransferSlots mapM_ (void . forkIO) [ commitThread st changechan commitchan transferqueue dstatus , pushThread st dstatus commitchan pushmap , pushRetryThread st pushmap , mergeThread st - , transferWatcherThread st dstatus + , transferWatcherThread st dstatus transferslots + , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan - , transfererThread st dstatus transferqueue , watchThread st dstatus transferqueue changechan ] waitForTermination diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 0b47e97812..249e15cf26 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -14,6 +14,7 @@ import Assistant.TransferQueue import Logs.Transfer import Annex.Content import Annex.BranchState +import Utility.ThreadScheduler import Command import qualified Command.Move @@ -22,68 +23,58 @@ import Control.Concurrent import Data.Time.Clock import qualified Data.Map as M -{- Dispatches transfers from the queue. - - - - This is currently very simplistic, and runs only one transfer at a time. - -} +{- For now only one transfer is run at a time. -} +maxTransfers :: Int +maxTransfers = 1 + +{- Dispatches transfers from the queue. -} transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> IO () -transfererThread st dstatus transferqueue = do - mypid <- getProcessID - mytid <- myThreadId - go mypid mytid +transfererThread st dstatus transferqueue = runEvery (Seconds 1) $ do + (t, info) <- getNextTransfer transferqueue + go =<< runThreadState st $ shouldTransfer t where - go mypid mytid = do - (t, info) <- getNextTransfer transferqueue + go Yes = runTransfer st t + go No = noop + go TooMany = waitTransfer >> go Yes - now <- getCurrentTime - let info' = info - { startedTime = Just now - , transferPid = Just mypid - , transferThread = Just mytid - } +data ShouldTransfer = Yes | Skip | TooMany - ifM (runThreadState st $ shouldtransfer t info') - ( runTransfer st t info' - , noop - ) - go mypid mytid - - -- Check if the transfer is already running, - -- and if not, add it to the TransferMap. - shouldtransfer t info = do - current <- currentTransfers <$> getDaemonStatus dstatus - if M.member t current - then return False - else ifM (validtransfer t) - ( do - adjustTransfers dstatus $ - M.insertWith' const t info - return True - , return False - ) - - validtransfer t +{- Checks if the requested transfer is already running, or + - the file to download is already present. + - + - There also may be too many transfers already running to service this + - transfer yet. -} +shouldTransfer :: DaemonStatusHandle -> Transfer -> Annex ShouldTransfer +shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus + where + go m + | M.member t m = return Skip + | M.size m > maxTransfers = return TooMany | transferDirection t == Download = - not <$> inAnnex (transferKey t) - | otherwise = return True + ifM (inAnnex $ transferKey t) (No, Yes) + | otherwise = return Yes -{- A transfer is run in a separate thread, with a *copy* of the Annex +{- Waits for any of the transfers in the map to complete. -} +waitTransfer :: IO () +waitTransfer = error "TODO" +-- getProcessStatus True False pid +-- runThreadState st invalidateCache + +{- A transfer is run in a separate process, with a *copy* of the Annex - state. This is necessary to avoid blocking the rest of the assistant - on the transfer completing, and also to allow multiple transfers to run - at once. - - - However, it means that the transfer threads are responsible + - However, it means that the transfer processes are responsible - for doing any necessary shutdown cleanups, and that the parent - - thread's cache must be invalidated, as changes may have been made to the - - git-annex branch. + - thread's cache must be invalidated once a transfer completes, as + - changes may have been made to the git-annex branch. - - - Currently a minimal shutdown is done; the transfer threads are + - Currently a minimal shutdown is done; the transfer processes are - effectively running in oneshot mode, without committing changes to the - git-annex branch, and transfers should never queue git commands to run. - - - - Note: It is unsafe to call getDaemonStatus inside the transfer thread. -} -runTransfer :: ThreadState -> Transfer -> TransferInfo -> IO () +runTransfer :: ThreadState -> Transfer -> TransferInfo -> IO ProcessID runTransfer st t info | transferDirection t == Download = go Command.Move.fromStart | otherwise = go Command.Move.toStart @@ -91,12 +82,12 @@ runTransfer st t info go cmd = case (transferRemote info, associatedFile info) of (Nothing, _) -> noop (_, Nothing) -> noop - (Just remote, Just file) -> - inthread $ void $ doCommand $ - cmd remote False file (transferKey t) - inthread a = do - mvar <- newEmptyMVar - void $ forkIO $ - unsafeRunThreadState st a `E.finally` putMVar mvar () - void $ takeMVar mvar -- wait for transfer thread - runThreadState st invalidateCache + (Just remote, Just file) -> do + now <- getCurrentTime + pid <- forkProcess $ unsafeRunThreadState st $ + doCommand $ cmd remote False file (transferKey t) + adjustTransfers dstatus $ + M.insertWith' const t info + { startedTime = Just now + , transferPid = Just pid + } diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 12ab8ff113..54f98da5cb 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -14,7 +14,6 @@ import qualified Git import Types.Remote import qualified Fields -import Control.Concurrent import System.Posix.Types import Data.Time.Clock @@ -36,7 +35,6 @@ data Transfer = Transfer data TransferInfo = TransferInfo { startedTime :: Maybe UTCTime , transferPid :: Maybe ProcessID - , transferThread :: Maybe ThreadId , transferRemote :: Maybe Remote , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath @@ -79,7 +77,6 @@ transfer t file a = do info <- liftIO $ TransferInfo <$> (Just <$> getCurrentTime) <*> pure Nothing -- pid not stored in file, so omitted for speed - <*> pure Nothing -- threadid not stored in file, so omitted for speed <*> pure Nothing -- not 0; transfer may be resuming <*> pure Nothing <*> pure file @@ -158,7 +155,6 @@ writeTransferInfo :: TransferInfo -> String writeTransferInfo info = unlines -- transferPid is not included; instead obtained by looking at -- the process that locks the file. - -- transferThread is not included; not relevant for other processes [ show $ startedTime info -- bytesComplete is not included; changes too fast , fromMaybe "" $ associatedFile info -- comes last; arbitrary content @@ -172,7 +168,6 @@ readTransferInfo pid s = <*> pure (Just pid) <*> pure Nothing <*> pure Nothing - <*> pure Nothing <*> pure (if null filename then Nothing else Just filename) _ -> Nothing where From 4104785bcbc26ebc8976fb4c44246f3539a067d9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo" Date: Fri, 6 Jul 2012 08:10:00 +0000 Subject: [PATCH 4033/8313] --- doc/forum/Wishlist:_mark_remotes_offline.mdwn | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/Wishlist:_mark_remotes_offline.mdwn diff --git a/doc/forum/Wishlist:_mark_remotes_offline.mdwn b/doc/forum/Wishlist:_mark_remotes_offline.mdwn new file mode 100644 index 0000000000..046c62210f --- /dev/null +++ b/doc/forum/Wishlist:_mark_remotes_offline.mdwn @@ -0,0 +1,12 @@ +I have several remotes which are not always accessible. For example they can +be on hosts only accessible by LAN or on a portable hard drive which is not +plugged in. When running sync these remotes are checked as well, leading to +unnecessary error messages and possibly git-annex waiting for a few minutes +on each remote for a timeout. + +In this situation it would be useful to mark some remotes as offline +(`git annex offline `), so that git-annex would not even attempt +to contact them. Then, I could configure my system to automatically, for example, +mark a portable hard disk remote online when plugging it in, and offline when +unplugging it, and similarly marking remotes offline and online depending on +whether I have an internet connection or a connection to a specific network. From 7a5229eb3b2347d7cd00901f20d6909afcea1d60 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 6 Jul 2012 08:48:10 +0000 Subject: [PATCH 4034/8313] --- ...ist:_options_for_syncing_meta-data_and_data.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/Wishlist:_options_for_syncing_meta-data_and_data.mdwn diff --git a/doc/forum/Wishlist:_options_for_syncing_meta-data_and_data.mdwn b/doc/forum/Wishlist:_options_for_syncing_meta-data_and_data.mdwn new file mode 100644 index 0000000000..d1df6628ea --- /dev/null +++ b/doc/forum/Wishlist:_options_for_syncing_meta-data_and_data.mdwn @@ -0,0 +1,13 @@ +Since _transfer queueing_ and syncing of data works now in the assistant branch (been playing with it), there are times when I really don't want to sync the data, I would like to just sync meta-data and manually do a _get_ on files that I would want or selectively sync data in a subtree. + +It would be nice to have the syncing/watch feature to have the option of syncing only *meta-data* or *meta-data and data*, I think this sort of option was already planned? It would also be nice to be able to automatically sync data for only a subtree. + +My use case is, I have a big stash of files somewhere at home or work, and I want to keep what I am actually using on my laptop and be able to selectively just take a subtree or a set of subtree's of files. I would not always want to suck down all the data but still have the functionally to add files and push them upstream and sync meta-data. + +that is... + +> * Site A: big master annex in a server room with lots of disk (or machines), watches a directory and syncs both data and meta-data, it should always try and pull data from all it's child repos. That way I will always have a master copy of my data somewhere, it would be even nicer if I could have clones of the annex, where each annex is on a different machine which is configured to only sync a subtree of files so I can distribute my annex across different systems and disks. +> * Site A: machine A: syncs Folder A +> * Site A: machine B: syncs Folder B +> * and so on with selectively syncing sites and directories +> * Laptop: has a clone of the annex, and watches a directory, syncs meta-data as usual and only uploads files to a remote (all or a designated one) but it never downloads files automatically or it should only occur inside a selected subtree. From 1cae1bf79cb0e1679f71a9acb690b2d09162e7a1 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Fri, 6 Jul 2012 13:04:07 +0000 Subject: [PATCH 4035/8313] Added a comment --- ...ent_1_9e3901f0123abb66034cce95cc5a941a._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/forum/Wishlist:_mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment diff --git a/doc/forum/Wishlist:_mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment b/doc/forum/Wishlist:_mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment new file mode 100644 index 0000000000..c24a786c96 --- /dev/null +++ b/doc/forum/Wishlist:_mark_remotes_offline/comment_1_9e3901f0123abb66034cce95cc5a941a._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 1" + date="2012-07-06T13:04:07Z" + content=""" +You can already do this: + + git config remote.foo.annex-ignore true + +There's no need to do anything for portable drives that are sometimes mounted and sometimes not -- git-annex will automatically avoid using repositories in directories that do not currently exist. + +I thought git-annex also had a way to run a command and use its exit status to control whether a repo was +ignored or not, but it seems I never actually implemented that. It might be worth adding, although the command would necessarily run whenever git-annex is transferring data around. +"""]] From bde355a65bf78e76a966f015af1642c95794b7cf Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 14:21:26 -0400 Subject: [PATCH 4036/8313] reorg, and add a link to a good forum post todo --- doc/design/assistant/syncing.mdwn | 94 ++++++++++++++++--------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index de07ef2d22..6dd1f79b3f 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -3,27 +3,6 @@ all the other git clones, at both the git level and the key/value level. ## action items -* on-disk transfers in progress information files (read/write/enumerate) - **done** -* locking for the files, so redundant transfer races can be detected, - and failed transfers noticed **done** -* transfer info for git-annex-shell **done** -* update files as transfers proceed. See [[progressbars]] - (updating for downloads is easy; for uploads is hard) -* add Transfer queue TChan **done** -* add TransferInfo Map to DaemonStatus for tracking transfers in progress. - **done** -* Poll transfer in progress info files for changes (use inotify again! - wow! hammer, meet nail..), and update the TransferInfo Map **done** -* enqueue Transfers (Uploads) as new files are added to the annex by - Watcher. **done** -* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by - Watcher. **done** -* Write basic Transfer handling thread. Multiple such threads need to be - able to be run at once. Each will need its own independant copy of the - Annex state monad. **done** -* Write transfer control thread, which decides when to launch transfers. - **done** * Check that download transfer triggering code works (when a symlink appears and the remote does *not* upload to us. * Investigate why transfers seem to block other git-annex assistant work. @@ -36,30 +15,21 @@ all the other git clones, at both the git level and the key/value level. through to, at least, rsync. A good job for an hour in an airport somewhere. -## git syncing +## longer-term TODO -1. Can use `git annex sync`, which already handles bidirectional syncing. - When a change is committed, launch the part of `git annex sync` that pushes - out changes. **done**; changes are pushed out to all remotes in parallel -1. Watch `.git/refs/remotes/` for changes (which would be pushed in from - another node via `git annex sync`), and run the part of `git annex sync` - that merges in received changes, and follow it by the part that pushes out - changes (sending them to any other remotes). - [The watching can be done with the existing inotify code! This avoids needing - any special mechanism to notify a remote that it's been synced to.] - **done** -1. Periodically retry pushes that failed. **done** (every half an hour) -1. Also, detect if a push failed due to not being up-to-date, pull, - and repush. **done** -2. Use a git merge driver that adds both conflicting files, - so conflicts never break a sync. **done** -3. Investigate the XMPP approach like dvcs-autosync does, or other ways of +* Investigate the XMPP approach like dvcs-autosync does, or other ways of signaling a change out of band. -4. Add a hook, so when there's a change to sync, a program can be run +* Add a hook, so when there's a change to sync, a program can be run and do its own signaling. -5. --debug will show often unnecessary work being done. Optimise. -6. It would be nice if, when a USB drive is connected, +* --debug will show often unnecessary work being done. Optimise. +* It would be nice if, when a USB drive is connected, syncing starts automatically. Use dbus on Linux? +* This assumes the network is connected. It's often not, so the + [[cloud]] needs to be used to bridge between LANs. +* Configurablity, including only enabling git syncing but not data transfer; + only uploading new files but not downloading, and only downloading + files in some directories and not others. See for use cases: + [[forum/Wishlist:_options_for_syncing_meta-data_and_data]] ## data syncing @@ -84,10 +54,42 @@ reachable remote. This is worth doing first, since it's the simplest way to get the basic functionality of the assistant to work. And we'll need this anyway. -## other considerations +## done -It would be nice if, when a USB drive is connected, -syncing starts automatically. Use dbus on Linux? +1. Can use `git annex sync`, which already handles bidirectional syncing. + When a change is committed, launch the part of `git annex sync` that pushes + out changes. **done**; changes are pushed out to all remotes in parallel +1. Watch `.git/refs/remotes/` for changes (which would be pushed in from + another node via `git annex sync`), and run the part of `git annex sync` + that merges in received changes, and follow it by the part that pushes out + changes (sending them to any other remotes). + [The watching can be done with the existing inotify code! This avoids needing + any special mechanism to notify a remote that it's been synced to.] + **done** +1. Periodically retry pushes that failed. **done** (every half an hour) +1. Also, detect if a push failed due to not being up-to-date, pull, + and repush. **done** +2. Use a git merge driver that adds both conflicting files, + so conflicts never break a sync. **done** -This assumes the network is connected. It's often not, so the -[[cloud]] needs to be used to bridge between LANs. +* on-disk transfers in progress information files (read/write/enumerate) + **done** +* locking for the files, so redundant transfer races can be detected, + and failed transfers noticed **done** +* transfer info for git-annex-shell **done** +* update files as transfers proceed. See [[progressbars]] + (updating for downloads is easy; for uploads is hard) +* add Transfer queue TChan **done** +* add TransferInfo Map to DaemonStatus for tracking transfers in progress. + **done** +* Poll transfer in progress info files for changes (use inotify again! + wow! hammer, meet nail..), and update the TransferInfo Map **done** +* enqueue Transfers (Uploads) as new files are added to the annex by + Watcher. **done** +* enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by + Watcher. **done** +* Write basic Transfer handling thread. Multiple such threads need to be + able to be run at once. Each will need its own independant copy of the + Annex state monad. **done** +* Write transfer control thread, which decides when to launch transfers. + **done** From 721748135b80a20e78ddc780ffedb2c54b74c307 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 14:42:45 -0400 Subject: [PATCH 4037/8313] fix build (almost) --- Assistant/Threads/Transferrer.hs | 33 ++++++++++++++++---------------- Assistant/TransferQueue.hs | 1 - 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 249e15cf26..0d0bc6f6d3 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -13,13 +13,10 @@ import Assistant.DaemonStatus import Assistant.TransferQueue import Logs.Transfer import Annex.Content -import Annex.BranchState import Utility.ThreadScheduler import Command import qualified Command.Move -import Control.Exception as E -import Control.Concurrent import Data.Time.Clock import qualified Data.Map as M @@ -31,11 +28,11 @@ maxTransfers = 1 transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> IO () transfererThread st dstatus transferqueue = runEvery (Seconds 1) $ do (t, info) <- getNextTransfer transferqueue - go =<< runThreadState st $ shouldTransfer t - where - go Yes = runTransfer st t - go No = noop - go TooMany = waitTransfer >> go Yes + c <- runThreadState st $ shouldTransfer dstatus t + case c of + Yes -> void $ runTransfer st dstatus t info + Skip -> noop + TooMany -> void $ waitTransfer >> runTransfer st dstatus t info data ShouldTransfer = Yes | Skip | TooMany @@ -51,7 +48,8 @@ shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus | M.member t m = return Skip | M.size m > maxTransfers = return TooMany | transferDirection t == Download = - ifM (inAnnex $ transferKey t) (No, Yes) + ifM (inAnnex $ transferKey t) + (return Skip, return Yes) | otherwise = return Yes {- Waits for any of the transfers in the map to complete. -} @@ -74,8 +72,8 @@ waitTransfer = error "TODO" - effectively running in oneshot mode, without committing changes to the - git-annex branch, and transfers should never queue git commands to run. -} -runTransfer :: ThreadState -> Transfer -> TransferInfo -> IO ProcessID -runTransfer st t info +runTransfer :: ThreadState -> DaemonStatusHandle -> Transfer -> TransferInfo -> IO () +runTransfer st dstatus t info | transferDirection t == Download = go Command.Move.fromStart | otherwise = go Command.Move.toStart where @@ -84,10 +82,11 @@ runTransfer st t info (_, Nothing) -> noop (Just remote, Just file) -> do now <- getCurrentTime - pid <- forkProcess $ unsafeRunThreadState st $ + pid <- forkProcess $ unsafeRunThreadState st $ void $ doCommand $ cmd remote False file (transferKey t) - adjustTransfers dstatus $ - M.insertWith' const t info - { startedTime = Just now - , transferPid = Just pid - } + runThreadState st $ + adjustTransfers dstatus $ + M.insertWith' const t info + { startedTime = Just now + , transferPid = Just pid + } diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index a35815ca16..bb65dbae5a 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -24,7 +24,6 @@ stubInfo :: AssociatedFile -> TransferInfo stubInfo f = TransferInfo { startedTime = Nothing , transferPid = Nothing - , transferThread = Nothing , transferRemote = Nothing , bytesComplete = Nothing , associatedFile = f From 8795a392c3f283d8e372cac3d82d29ca52c9cfb5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 16:30:55 -0400 Subject: [PATCH 4038/8313] fix --- Assistant/Threads/TransferWatcher.hs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index f18d4e3f86..48c0c79ae5 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -47,13 +47,7 @@ runHandler st dstatus handler file filestatus = void $ do onErr :: Handler onErr _ _ msg _ = error msg -{- Called when a new transfer information file is written. - - - - When another thread of the assistant writes a transfer info file, - - this will notice that too, but should skip it, because the thread - - will be managing the transfer itself, and will have stored a more - - complete TransferInfo than is stored in the file. - -} +{- Called when a new transfer information file is written. -} onAdd :: Handler onAdd st dstatus file _ = case parseTransferFile file of Nothing -> noop @@ -62,10 +56,8 @@ onAdd st dstatus file _ = case parseTransferFile file of runThreadState st $ go t pid =<< checkTransfer t where go _ _ Nothing = noop -- transfer already finished - go t pid (Just info) - | transferPid info == Just pid = noop - | otherwise = adjustTransfers dstatus $ - M.insertWith' const t info + go t pid (Just info) = adjustTransfers dstatus $ + M.insertWith' const t info {- Called when a transfer information file is removed. -} onDel :: Handler From 430ad8ce85835e002a326b68813c51f85c91141e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 16:39:07 -0400 Subject: [PATCH 4039/8313] it builds again Currently nothing waits on transfer processes. (Second drive of the day fried. Not concentrating very well.) --- Assistant.hs | 3 ++- Assistant/Threads/Transferrer.hs | 11 +++++++---- Assistant/TransferSlots.hs | 30 ++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 Assistant/TransferSlots.hs diff --git a/Assistant.hs b/Assistant.hs index 38ed539a1f..06484b0862 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -84,6 +84,7 @@ import Assistant.Changes import Assistant.Commits import Assistant.Pushes import Assistant.TransferQueue +import Assistant.TransferSlots import Assistant.Threads.Watcher import Assistant.Threads.Committer import Assistant.Threads.Pusher @@ -122,7 +123,7 @@ startDaemon assistant foreground , pushThread st dstatus commitchan pushmap , pushRetryThread st pushmap , mergeThread st - , transferWatcherThread st dstatus transferslots + , transferWatcherThread st dstatus , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 0d0bc6f6d3..3e417e7ff5 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -11,6 +11,7 @@ import Common.Annex import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.TransferQueue +import Assistant.TransferSlots import Logs.Transfer import Annex.Content import Utility.ThreadScheduler @@ -25,14 +26,16 @@ maxTransfers :: Int maxTransfers = 1 {- Dispatches transfers from the queue. -} -transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> IO () -transfererThread st dstatus transferqueue = runEvery (Seconds 1) $ do +transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> TransferSlots -> IO () +transfererThread st dstatus transferqueue slots = runEvery (Seconds 1) $ do (t, info) <- getNextTransfer transferqueue c <- runThreadState st $ shouldTransfer dstatus t + let run = void $ inTransferSlot slots $ + runTransfer st dstatus t info case c of - Yes -> void $ runTransfer st dstatus t info + Yes -> run Skip -> noop - TooMany -> void $ waitTransfer >> runTransfer st dstatus t info + TooMany -> waitTransfer >> run data ShouldTransfer = Yes | Skip | TooMany diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs new file mode 100644 index 0000000000..0e2bb98b0c --- /dev/null +++ b/Assistant/TransferSlots.hs @@ -0,0 +1,30 @@ +{- git-annex assistant transfer slots + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.TransferSlots where + +import Control.Exception +import Control.Concurrent + +type TransferSlots = QSemN + +{- Number of concurrent transfers allowed to be run from the assistant. + - + - Transfers launched by other means, including by remote assistants, + - do not currently take up slots. + -} +numSlots :: Int +numSlots = 1 + +newTransferSlots :: IO TransferSlots +newTransferSlots = newQSemN numSlots + +{- Waits until a transfer slot becomes available, and runs a transfer + - action in the slot. + -} +inTransferSlot :: TransferSlots -> IO a -> IO a +inTransferSlot s = bracket_ (waitQSemN s 1) (signalQSemN s 1) From 3d30a45e72418927d55a31a4d3d7aa5cf0c5c365 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 14:54:07 -0600 Subject: [PATCH 4040/8313] simplified background transferrs seem to work now --- Assistant/Threads/Transferrer.hs | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 3e417e7ff5..2d01855b4c 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -29,31 +29,20 @@ maxTransfers = 1 transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> TransferSlots -> IO () transfererThread st dstatus transferqueue slots = runEvery (Seconds 1) $ do (t, info) <- getNextTransfer transferqueue - c <- runThreadState st $ shouldTransfer dstatus t - let run = void $ inTransferSlot slots $ - runTransfer st dstatus t info - case c of - Yes -> run - Skip -> noop - TooMany -> waitTransfer >> run - -data ShouldTransfer = Yes | Skip | TooMany + whenM (runThreadState st $ shouldTransfer dstatus t) $ + void $ inTransferSlot slots $ + runTransfer st dstatus t info {- Checks if the requested transfer is already running, or - - the file to download is already present. - - - - There also may be too many transfers already running to service this - - transfer yet. -} -shouldTransfer :: DaemonStatusHandle -> Transfer -> Annex ShouldTransfer + - the file to download is already present. -} +shouldTransfer :: DaemonStatusHandle -> Transfer -> Annex Bool shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus where go m - | M.member t m = return Skip - | M.size m > maxTransfers = return TooMany + | M.member t m = return False | transferDirection t == Download = - ifM (inAnnex $ transferKey t) - (return Skip, return Yes) - | otherwise = return Yes + inAnnex $ transferKey t + | otherwise = return True {- Waits for any of the transfers in the map to complete. -} waitTransfer :: IO () From 8489419debfdfb3fb58ba447afd6a7e340d99b62 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 14:56:41 -0600 Subject: [PATCH 4041/8313] todo --- doc/design/assistant/syncing.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 6dd1f79b3f..abeb74fb7b 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -14,6 +14,8 @@ all the other git clones, at both the git level and the key/value level. * git-annex needs a simple speed control knob, which can be plumbed through to, at least, rsync. A good job for an hour in an airport somewhere. +* file transfers run in background processes, which means they + probably don't participate in ssh connection caching. Verify this and fix. ## longer-term TODO From 27ac0ec332d4d3cd8ee03d16b7e22d0498157b14 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 14:58:30 -0600 Subject: [PATCH 4042/8313] ssh connection caching is ok, but there is another problem --- doc/design/assistant/syncing.mdwn | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index abeb74fb7b..9c607f992d 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -14,8 +14,7 @@ all the other git clones, at both the git level and the key/value level. * git-annex needs a simple speed control knob, which can be plumbed through to, at least, rsync. A good job for an hour in an airport somewhere. -* file transfers run in background processes, which means they - probably don't participate in ssh connection caching. Verify this and fix. +* file transfer processes are not waited for, contain the zombies. ## longer-term TODO From 4a107951442f30354fa90b0b31200a9fdc86ffca Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 15:07:42 -0600 Subject: [PATCH 4043/8313] logic error --- Assistant/Threads/Transferrer.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 2d01855b4c..5bc47cfa67 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -41,7 +41,7 @@ shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus go m | M.member t m = return False | transferDirection t == Download = - inAnnex $ transferKey t + not <$> inAnnex (transferKey t) | otherwise = return True {- Waits for any of the transfers in the map to complete. -} From 62876502c55958cd8f716d6676eb97825456d9b7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 16:44:13 -0600 Subject: [PATCH 4044/8313] wait on child transfer processes, and invalidate cache There's still a bug; if the child updates its transfer info file, then the data from it will superscede the TransferInfo, losing the info that we should wait on this child. --- Assistant/DaemonStatus.hs | 19 ++++++++++++++++--- Assistant/Threads/SanityChecker.hs | 4 ++-- Assistant/Threads/TransferWatcher.hs | 26 +++++++++++++++++++------- Assistant/Threads/Transferrer.hs | 7 +------ Assistant/Threads/Watcher.hs | 2 +- Assistant/TransferQueue.hs | 1 + Logs/Transfer.hs | 3 +++ 7 files changed, 43 insertions(+), 19 deletions(-) diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 40816bb1a7..64c441ceee 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -54,8 +54,11 @@ newDaemonStatus = DaemonStatus getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus getDaemonStatus = liftIO . readMVar -modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex () -modifyDaemonStatus handle a = liftIO $ modifyMVar_ handle (return . a) +modifyDaemonStatus_ :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex () +modifyDaemonStatus_ handle a = liftIO $ modifyMVar_ handle (return . a) + +modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> Annex b +modifyDaemonStatus handle a = liftIO $ modifyMVar handle (return . a) {- Load any previous daemon status file, and store it in the MVar for this - process to use as its DaemonStatus. Also gets current transfer status. -} @@ -137,5 +140,15 @@ tenMinutes = 10 * 60 {- Mutates the transfer map. -} adjustTransfers :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> Annex () -adjustTransfers dstatus a = modifyDaemonStatus dstatus $ +adjustTransfers dstatus a = modifyDaemonStatus_ dstatus $ \s -> s { currentTransfers = a (currentTransfers s) } + +{- Removes a transfer from the map, and returns its info. -} +removeTransfer :: DaemonStatusHandle -> Transfer -> Annex (Maybe TransferInfo) +removeTransfer dstatus t = modifyDaemonStatus dstatus go + where + go s = + let (info, ts) = M.updateLookupWithKey + (\_k _v -> Nothing) + t (currentTransfers s) + in (s { currentTransfers = ts }, info) diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs index d7b117cd02..c5b99863e3 100644 --- a/Assistant/Threads/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -26,7 +26,7 @@ sanityCheckerThread st status transferqueue changechan = forever $ do waitForNextCheck st status runThreadState st $ - modifyDaemonStatus status $ \s -> s + modifyDaemonStatus_ status $ \s -> s { sanityCheckRunning = True } now <- getPOSIXTime -- before check started @@ -34,7 +34,7 @@ sanityCheckerThread st status transferqueue changechan = forever $ do (runThreadState st . warning . show) runThreadState st $ do - modifyDaemonStatus status $ \s -> s + modifyDaemonStatus_ status $ \s -> s { sanityCheckRunning = False , lastSanityCheck = Just now } diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 48c0c79ae5..4e468a4165 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -13,6 +13,7 @@ import Assistant.DaemonStatus import Logs.Transfer import Utility.DirWatcher import Utility.Types.DirWatcher +import Annex.BranchState import Data.Map as M @@ -51,16 +52,27 @@ onErr _ _ msg _ = error msg onAdd :: Handler onAdd st dstatus file _ = case parseTransferFile file of Nothing -> noop - Just t -> do - pid <- getProcessID - runThreadState st $ go t pid =<< checkTransfer t + Just t -> runThreadState st $ go t =<< checkTransfer t where - go _ _ Nothing = noop -- transfer already finished - go t pid (Just info) = adjustTransfers dstatus $ + go _ Nothing = noop -- transfer already finished + go t (Just info) = adjustTransfers dstatus $ M.insertWith' const t info -{- Called when a transfer information file is removed. -} +{- Called when a transfer information file is removed. + - + - When the transfer process is a child of this process, wait on it + - to avoid zombies. + -} onDel :: Handler onDel st dstatus file _ = case parseTransferFile file of Nothing -> noop - Just t -> runThreadState st $ adjustTransfers dstatus $ M.delete t + Just t -> maybe noop waitchild + =<< runThreadState st (removeTransfer dstatus t) + where + waitchild info + | shouldWait info = case transferPid info of + Nothing -> noop + Just pid -> do + void $ getProcessStatus True False pid + runThreadState st invalidateCache + | otherwise = noop diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 5bc47cfa67..09c0aa0369 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -44,12 +44,6 @@ shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus not <$> inAnnex (transferKey t) | otherwise = return True -{- Waits for any of the transfers in the map to complete. -} -waitTransfer :: IO () -waitTransfer = error "TODO" --- getProcessStatus True False pid --- runThreadState st invalidateCache - {- A transfer is run in a separate process, with a *copy* of the Annex - state. This is necessary to avoid blocking the rest of the assistant - on the transfer completing, and also to allow multiple transfers to run @@ -81,4 +75,5 @@ runTransfer st dstatus t info M.insertWith' const t info { startedTime = Just now , transferPid = Just pid + , shouldWait = True } diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 882aab3a78..9f0eba74e9 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -67,7 +67,7 @@ statupScan st dstatus scanner = do showAction "scanning" r <- scanner runThreadState st $ - modifyDaemonStatus dstatus $ \s -> s { scanComplete = True } + modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True } -- Notice any files that were deleted before watching was started. runThreadState st $ do diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index bb65dbae5a..5e1fad4560 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -27,6 +27,7 @@ stubInfo f = TransferInfo , transferRemote = Nothing , bytesComplete = Nothing , associatedFile = f + , shouldWait = False } {- Adds pending transfers to the end of the queue for some of the known diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 54f98da5cb..494a44c51b 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -38,6 +38,7 @@ data TransferInfo = TransferInfo , transferRemote :: Maybe Remote , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath + , shouldWait :: Bool } deriving (Show, Eq, Ord) @@ -80,6 +81,7 @@ transfer t file a = do <*> pure Nothing -- not 0; transfer may be resuming <*> pure Nothing <*> pure file + <*> pure False bracketIO (prep tfile mode info) (cleanup tfile) a where prep tfile mode info = do @@ -169,6 +171,7 @@ readTransferInfo pid s = <*> pure Nothing <*> pure Nothing <*> pure (if null filename then Nothing else Just filename) + <*> pure False _ -> Nothing where (bits, filebits) = splitAt 1 $ lines s From 2c4b39be4f68b53c2d2bc3647b17789b316fc542 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 17:06:05 -0600 Subject: [PATCH 4045/8313] blog for the day --- .../assistant/blog/day_26__dying_drives.mdwn | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 doc/design/assistant/blog/day_26__dying_drives.mdwn diff --git a/doc/design/assistant/blog/day_26__dying_drives.mdwn b/doc/design/assistant/blog/day_26__dying_drives.mdwn new file mode 100644 index 0000000000..109ceb19ec --- /dev/null +++ b/doc/design/assistant/blog/day_26__dying_drives.mdwn @@ -0,0 +1,28 @@ +My laptop's SSD died this morning. I had some work from yesterday +committed to the git repo on it, but not pushed as it didn't build. +Luckily I was able to get that off the SSD, which is now a read-only +drive -- even mounting it fails with fsck write errors. + +Wish I'd realized the SSD was dying before the day before my trip to +Nicaragua.. +Getting back to a useful laptop used most of my time and energy today. + +I did manage to fix transfers to not block the rest of the assistant's +threads. Problem was that, without Haskell's threaded runtime, waiting +on something like a rsync command blocks all threads. To fix this, +transfers now are run in separate processes. + +Also added code to allow multiple transfers to run at once. Each transfer +takes up a slot, with the number of free slots tracked by a `QSemN`. +This allows the transfer starting thread to block until a slot frees up, +and then run the transfer. + +This needs to be extended to be aware of transfers initiated by remotes. +The transfer watcher thread should detect those starting and stopping +and update the `QSemN` accordingly. It would also be nice if transfers +initiated by remotes would be delayed when there are no free slots for them +... but I have not thought of a good way to do that. + +There's a bug somewhere in the new transfer code, when two transfers are +queued close together, the second one is lost and doesn't happen. +Would debug this, but I'm spent for the day. From 8ad844e45c3f8a65ef5b725e9c6ac0f414b50fa4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 17:22:56 -0600 Subject: [PATCH 4046/8313] fix leading period before two-element extensions --- Backend/SHA.hs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 95ce4a7701..cf61139e00 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -102,9 +102,13 @@ keyValueE size source = keyValue size source >>= maybe (return Nothing) addE } selectExtension :: FilePath -> String -selectExtension = join "." . reverse . take 2 . takeWhile shortenough . - reverse . split "." . takeExtensions +selectExtension f + | null es = "" + | otherwise = join "." ("":es) where + es = filter (not . null) $ reverse $ + take 2 $ takeWhile shortenough $ + reverse $ split "." $ takeExtensions f shortenough e | '\n' `elem` e = False -- newline in extension?! | otherwise = length e <= 4 -- long enough for "jpeg" From d954a0ce5934a877f8df0c683eaccaf8c2b1938e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 18:48:51 -0600 Subject: [PATCH 4047/8313] fixed close-together transfer race The issue involved forking and they trying to read from a MVar. Reading the MVar 1st fixed it. --- Assistant/ThreadedMonad.hs | 17 +++++++++-------- Assistant/Threads/Transferrer.hs | 16 +++++++++------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 4e871ab676..16f3a9dd9f 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -12,6 +12,7 @@ import qualified Annex import Control.Concurrent import Data.Tuple +import System.Posix.Types {- The Annex state is stored in a MVar, so that threaded actions can access - it. -} @@ -37,14 +38,14 @@ withThreadState a = do runThreadState :: ThreadState -> Annex a -> IO a runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a -{- Runs an Annex action, using a copy of the state from the MVar. +{- Runs an Annex action in a separate process, using a copy of the state + - from the MVar. - - - The state modified by the action is thrown away, so it's up to the - - action to perform any necessary shutdown tasks in order for state to not - - be lost. And it's up to the caller to resynchronise with any changes - - the action makes to eg, the git-annex branch. + - It's up to the action to perform any necessary shutdown tasks in order + - for state to not be lost. And it's up to the caller to resynchronise + - with any changes the action makes to eg, the git-annex branch. -} -unsafeRunThreadState :: ThreadState -> Annex a -> IO a -unsafeRunThreadState mvar a = do +unsafeForkProcessThreadState :: ThreadState -> Annex a -> IO ProcessID +unsafeForkProcessThreadState mvar a = do state <- readMVar mvar - Annex.eval state a + forkProcess $ void $ Annex.eval state a diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 09c0aa0369..f40218c08d 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -14,7 +14,6 @@ import Assistant.TransferQueue import Assistant.TransferSlots import Logs.Transfer import Annex.Content -import Utility.ThreadScheduler import Command import qualified Command.Move @@ -27,11 +26,14 @@ maxTransfers = 1 {- Dispatches transfers from the queue. -} transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> TransferSlots -> IO () -transfererThread st dstatus transferqueue slots = runEvery (Seconds 1) $ do - (t, info) <- getNextTransfer transferqueue - whenM (runThreadState st $ shouldTransfer dstatus t) $ - void $ inTransferSlot slots $ - runTransfer st dstatus t info +transfererThread st dstatus transferqueue slots = go + where + go = do + (t, info) <- getNextTransfer transferqueue + whenM (runThreadState st $ shouldTransfer dstatus t) $ + void $ inTransferSlot slots $ + runTransfer st dstatus t info + go {- Checks if the requested transfer is already running, or - the file to download is already present. -} @@ -68,7 +70,7 @@ runTransfer st dstatus t info (_, Nothing) -> noop (Just remote, Just file) -> do now <- getCurrentTime - pid <- forkProcess $ unsafeRunThreadState st $ void $ + pid <- unsafeForkProcessThreadState st $ doCommand $ cmd remote False file (transferKey t) runThreadState st $ adjustTransfers dstatus $ From 9bcfbc40480c2a6a675ff5b9198e64287c6e6881 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 21:17:21 -0600 Subject: [PATCH 4048/8313] todo --- doc/design/assistant/syncing.mdwn | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 9c607f992d..94699aae05 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -31,6 +31,10 @@ all the other git clones, at both the git level and the key/value level. only uploading new files but not downloading, and only downloading files in some directories and not others. See for use cases: [[forum/Wishlist:_options_for_syncing_meta-data_and_data]] +* Running external commands from one thread blocks all of them until + it completes. Try to switch to haskell's threaded runtime, which I + think fixes this. Failing that, make sure all network accessing + commands are run by separate processes or something. ## data syncing From 94b06deb64c7152aef718cb26ed804266902a6a9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 21:45:08 -0600 Subject: [PATCH 4049/8313] fix transfer slots blocking and refilling when transfers are stopped --- Assistant.hs | 2 +- Assistant/Threads/TransferWatcher.hs | 25 ++++++++++++++----------- Assistant/TransferSlots.hs | 13 +++++++++++-- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 06484b0862..91ebf2d2e0 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -123,7 +123,7 @@ startDaemon assistant foreground , pushThread st dstatus commitchan pushmap , pushRetryThread st pushmap , mergeThread st - , transferWatcherThread st dstatus + , transferWatcherThread st dstatus transferslots , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 4e468a4165..aa8b3f6e68 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -10,6 +10,7 @@ module Assistant.Threads.TransferWatcher where import Common.Annex import Assistant.ThreadedMonad import Assistant.DaemonStatus +import Assistant.TransferSlots import Logs.Transfer import Utility.DirWatcher import Utility.Types.DirWatcher @@ -19,12 +20,12 @@ import Data.Map as M {- This thread watches for changes to the gitAnnexTransferDir, - and updates the DaemonStatus's map of ongoing transfers. -} -transferWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () -transferWatcherThread st dstatus = do +transferWatcherThread :: ThreadState -> DaemonStatusHandle -> TransferSlots -> IO () +transferWatcherThread st dstatus transferslots = do g <- runThreadState st $ fromRepo id let dir = gitAnnexTransferDir g createDirectoryIfMissing True dir - let hook a = Just $ runHandler st dstatus a + let hook a = Just $ runHandler st dstatus transferslots a let hooks = mkWatchHooks { addHook = hook onAdd , delHook = hook onDel @@ -32,25 +33,25 @@ transferWatcherThread st dstatus = do } void $ watchDir dir (const False) hooks id -type Handler = ThreadState -> DaemonStatusHandle -> FilePath -> Maybe FileStatus -> IO () +type Handler = ThreadState -> DaemonStatusHandle -> TransferSlots -> FilePath -> Maybe FileStatus -> IO () {- Runs an action handler. - - Exceptions are ignored, otherwise a whole thread could be crashed. -} -runHandler :: ThreadState -> DaemonStatusHandle -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler st dstatus handler file filestatus = void $ do +runHandler :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler st dstatus transferslots handler file filestatus = void $ do either print (const noop) =<< tryIO go where - go = handler st dstatus file filestatus + go = handler st dstatus transferslots file filestatus {- Called when there's an error with inotify. -} onErr :: Handler -onErr _ _ msg _ = error msg +onErr _ _ _ msg _ = error msg {- Called when a new transfer information file is written. -} onAdd :: Handler -onAdd st dstatus file _ = case parseTransferFile file of +onAdd st dstatus _ file _ = case parseTransferFile file of Nothing -> noop Just t -> runThreadState st $ go t =<< checkTransfer t where @@ -64,7 +65,7 @@ onAdd st dstatus file _ = case parseTransferFile file of - to avoid zombies. -} onDel :: Handler -onDel st dstatus file _ = case parseTransferFile file of +onDel st dstatus transferslots file _ = case parseTransferFile file of Nothing -> noop Just t -> maybe noop waitchild =<< runThreadState st (removeTransfer dstatus t) @@ -73,6 +74,8 @@ onDel st dstatus file _ = case parseTransferFile file of | shouldWait info = case transferPid info of Nothing -> noop Just pid -> do - void $ getProcessStatus True False pid + void $ tryIO $ + getProcessStatus True False pid runThreadState st invalidateCache + transferComplete transferslots | otherwise = noop diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs index 0e2bb98b0c..3dc7917e42 100644 --- a/Assistant/TransferSlots.hs +++ b/Assistant/TransferSlots.hs @@ -24,7 +24,16 @@ newTransferSlots :: IO TransferSlots newTransferSlots = newQSemN numSlots {- Waits until a transfer slot becomes available, and runs a transfer - - action in the slot. + - action in the slot. If the action throws an exception, its slot is + - freed here, otherwise it should be freed by the TransferWatcher when + - the transfer is complete. -} inTransferSlot :: TransferSlots -> IO a -> IO a -inTransferSlot s = bracket_ (waitQSemN s 1) (signalQSemN s 1) +inTransferSlot s a = bracketOnError + (waitQSemN s 1) + (const $ signalQSemN s 1) + (const a) + +{- Call when a transfer is complete. -} +transferComplete :: TransferSlots -> IO () +transferComplete s = signalQSemN s 1 From cc6f660752d4eef1e667f1ac859c6140f4da87ca Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 6 Jul 2012 21:45:08 -0600 Subject: [PATCH 4050/8313] fix transfer slots blocking and refilling when transfers are stopped There's a bug, if a transfer process notices it needs to do nothing, it never starts the transfer, so the slot is never freed. --- Assistant.hs | 2 +- Assistant/Threads/TransferWatcher.hs | 25 ++++++++++++++----------- Assistant/TransferSlots.hs | 14 ++++++++++++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 06484b0862..91ebf2d2e0 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -123,7 +123,7 @@ startDaemon assistant foreground , pushThread st dstatus commitchan pushmap , pushRetryThread st pushmap , mergeThread st - , transferWatcherThread st dstatus + , transferWatcherThread st dstatus transferslots , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 4e468a4165..aa8b3f6e68 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -10,6 +10,7 @@ module Assistant.Threads.TransferWatcher where import Common.Annex import Assistant.ThreadedMonad import Assistant.DaemonStatus +import Assistant.TransferSlots import Logs.Transfer import Utility.DirWatcher import Utility.Types.DirWatcher @@ -19,12 +20,12 @@ import Data.Map as M {- This thread watches for changes to the gitAnnexTransferDir, - and updates the DaemonStatus's map of ongoing transfers. -} -transferWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () -transferWatcherThread st dstatus = do +transferWatcherThread :: ThreadState -> DaemonStatusHandle -> TransferSlots -> IO () +transferWatcherThread st dstatus transferslots = do g <- runThreadState st $ fromRepo id let dir = gitAnnexTransferDir g createDirectoryIfMissing True dir - let hook a = Just $ runHandler st dstatus a + let hook a = Just $ runHandler st dstatus transferslots a let hooks = mkWatchHooks { addHook = hook onAdd , delHook = hook onDel @@ -32,25 +33,25 @@ transferWatcherThread st dstatus = do } void $ watchDir dir (const False) hooks id -type Handler = ThreadState -> DaemonStatusHandle -> FilePath -> Maybe FileStatus -> IO () +type Handler = ThreadState -> DaemonStatusHandle -> TransferSlots -> FilePath -> Maybe FileStatus -> IO () {- Runs an action handler. - - Exceptions are ignored, otherwise a whole thread could be crashed. -} -runHandler :: ThreadState -> DaemonStatusHandle -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler st dstatus handler file filestatus = void $ do +runHandler :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler st dstatus transferslots handler file filestatus = void $ do either print (const noop) =<< tryIO go where - go = handler st dstatus file filestatus + go = handler st dstatus transferslots file filestatus {- Called when there's an error with inotify. -} onErr :: Handler -onErr _ _ msg _ = error msg +onErr _ _ _ msg _ = error msg {- Called when a new transfer information file is written. -} onAdd :: Handler -onAdd st dstatus file _ = case parseTransferFile file of +onAdd st dstatus _ file _ = case parseTransferFile file of Nothing -> noop Just t -> runThreadState st $ go t =<< checkTransfer t where @@ -64,7 +65,7 @@ onAdd st dstatus file _ = case parseTransferFile file of - to avoid zombies. -} onDel :: Handler -onDel st dstatus file _ = case parseTransferFile file of +onDel st dstatus transferslots file _ = case parseTransferFile file of Nothing -> noop Just t -> maybe noop waitchild =<< runThreadState st (removeTransfer dstatus t) @@ -73,6 +74,8 @@ onDel st dstatus file _ = case parseTransferFile file of | shouldWait info = case transferPid info of Nothing -> noop Just pid -> do - void $ getProcessStatus True False pid + void $ tryIO $ + getProcessStatus True False pid runThreadState st invalidateCache + transferComplete transferslots | otherwise = noop diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs index 0e2bb98b0c..1859b281bb 100644 --- a/Assistant/TransferSlots.hs +++ b/Assistant/TransferSlots.hs @@ -24,7 +24,17 @@ newTransferSlots :: IO TransferSlots newTransferSlots = newQSemN numSlots {- Waits until a transfer slot becomes available, and runs a transfer - - action in the slot. + - action in the slot. If the action throws an exception, its slot is + - freed here, otherwise it should be freed by the TransferWatcher when + - the transfer is complete. -} inTransferSlot :: TransferSlots -> IO a -> IO a -inTransferSlot s = bracket_ (waitQSemN s 1) (signalQSemN s 1) +inTransferSlot s a = bracketOnError start abort run + where + start = waitQSemN s 1 + abort = const $ transferComplete s + run = const a + +{- Call when a transfer is complete. -} +transferComplete :: TransferSlots -> IO () +transferComplete s = signalQSemN s 1 From 583cfb5667ae2ec9e01e9fe4f91b32d758de6aa4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmWQTrnPloMWiPFg8Y2Y5g-2IYe26D0KKw" Date: Sat, 7 Jul 2012 16:18:08 +0000 Subject: [PATCH 4051/8313] Added a comment --- .../comment_6_1d4fbbd212fa92967abda346323031f4._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment diff --git a/doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment b/doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment new file mode 100644 index 0000000000..294778cbfa --- /dev/null +++ b/doc/forum/fsck_gives_false_positives/comment_6_1d4fbbd212fa92967abda346323031f4._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmWQTrnPloMWiPFg8Y2Y5g-2IYe26D0KKw" + nickname="Jim" + subject="comment 6" + date="2012-07-07T16:18:06Z" + content=""" +It's also possible you got a one-time DRAM corruption. You have to expect those to happen every so often unless you're using ECC memory. +"""]] From cd168c6cba2ce6d938a4533abf783286addb16b0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 10:50:20 -0600 Subject: [PATCH 4052/8313] fix transferrer thread's use of transfer slots and transfer info files Check first if a transfer needs to be done, using the location log only (for speed), and avoid occupying a slot if not. Always write a transfer info file, and keep it open throughout the tranfer process. Now transfers to remotes seem reliable. --- Assistant/Threads/Transferrer.hs | 77 ++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index f40218c08d..aaf654d34b 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -13,9 +13,10 @@ import Assistant.DaemonStatus import Assistant.TransferQueue import Assistant.TransferSlots import Logs.Transfer +import Logs.Presence +import Logs.Location import Annex.Content -import Command -import qualified Command.Move +import qualified Remote import Data.Time.Clock import qualified Data.Map as M @@ -31,8 +32,7 @@ transfererThread st dstatus transferqueue slots = go go = do (t, info) <- getNextTransfer transferqueue whenM (runThreadState st $ shouldTransfer dstatus t) $ - void $ inTransferSlot slots $ - runTransfer st dstatus t info + runTransfer st dstatus slots t info go {- Checks if the requested transfer is already running, or @@ -49,33 +49,54 @@ shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus {- A transfer is run in a separate process, with a *copy* of the Annex - state. This is necessary to avoid blocking the rest of the assistant - on the transfer completing, and also to allow multiple transfers to run - - at once. + - at once. - - However, it means that the transfer processes are responsible - for doing any necessary shutdown cleanups, and that the parent - thread's cache must be invalidated once a transfer completes, as - - changes may have been made to the git-annex branch. - - - - Currently a minimal shutdown is done; the transfer processes are - - effectively running in oneshot mode, without committing changes to the - - git-annex branch, and transfers should never queue git commands to run. + - changes may have been made to the git-annex branch. -} -runTransfer :: ThreadState -> DaemonStatusHandle -> Transfer -> TransferInfo -> IO () -runTransfer st dstatus t info - | transferDirection t == Download = go Command.Move.fromStart - | otherwise = go Command.Move.toStart +runTransfer :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Transfer -> TransferInfo -> IO () +runTransfer st dstatus slots t info = case (transferRemote info, associatedFile info) of + (Nothing, _) -> noop + (_, Nothing) -> noop + (Just remote, Just file) -> whenM (shouldtransfer remote) $ do + pid <- inTransferSlot slots $ + unsafeForkProcessThreadState st $ + transferprocess remote file + now <- getCurrentTime + runThreadState st $ adjustTransfers dstatus $ + M.insertWith' const t info + { startedTime = Just now + , transferPid = Just pid + , shouldWait = True + } where - go cmd = case (transferRemote info, associatedFile info) of - (Nothing, _) -> noop - (_, Nothing) -> noop - (Just remote, Just file) -> do - now <- getCurrentTime - pid <- unsafeForkProcessThreadState st $ - doCommand $ cmd remote False file (transferKey t) - runThreadState st $ - adjustTransfers dstatus $ - M.insertWith' const t info - { startedTime = Just now - , transferPid = Just pid - , shouldWait = True - } + isdownload = transferDirection t == Download + tofrom + | isdownload = "from" + | otherwise = "to" + key = transferKey t + + shouldtransfer remote + | isdownload = return True + | otherwise = runThreadState st $ + {- Trust the location log to check if the + - remote already has the key. This avoids + - a roundtrip to the remote. -} + notElem (Remote.uuid remote) + <$> loggedLocations key + + transferprocess remote file = do + showStart "copy" file + showAction $ tofrom ++ " " ++ Remote.name remote + ok <- transfer t (Just file) $ + if isdownload + then getViaTmp key $ + Remote.retrieveKeyFile remote key (Just file) + else do + ok <- Remote.storeKey remote key $ Just file + when ok $ + Remote.logStatus remote key InfoPresent + return ok + showEndResult ok From eb9063c0d1bf34a339516c8ecbfbff8dd2f83e25 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 10:56:09 -0600 Subject: [PATCH 4053/8313] update --- doc/design/assistant/syncing.mdwn | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 94699aae05..2871ec2164 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -1,23 +1,24 @@ Once files are added (or removed or moved), need to send those changes to all the other git clones, at both the git level and the key/value level. -## action items +## immediate action items * Check that download transfer triggering code works (when a symlink appears and the remote does *not* upload to us. -* Investigate why transfers seem to block other git-annex assistant work. * At startup, and possibly periodically, look for files we have that location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. -* Find a way to probe available outgoing bandwidth, to throttle so - we don't bufferbloat the network to death. -* git-annex needs a simple speed control knob, which can be plumbed - through to, at least, rsync. A good job for an hour in an - airport somewhere. -* file transfer processes are not waited for, contain the zombies. +* The TransferWatcher does not notice ongoing transfers, because inotify is + waiting for the info file to be closed, but that never happens, it's left + open to keep it locked. ## longer-term TODO +* git-annex needs a simple speed control knob, which can be plumbed + through to, at least, rsync. A good job for an hour in an + airport somewhere. +* Find a way to probe available outgoing bandwidth, to throttle so + we don't bufferbloat the network to death. * Investigate the XMPP approach like dvcs-autosync does, or other ways of signaling a change out of band. * Add a hook, so when there's a change to sync, a program can be run From 3247415c56fcb3b7ed7914e44cae640c6443abc0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 11:12:11 -0600 Subject: [PATCH 4054/8313] update; split out hard todo --- doc/design/assistant/inotify.mdwn | 22 +++++++++++----------- doc/design/assistant/syncing.mdwn | 7 ++----- doc/todo/threaded_runtime.mdwn | 29 +++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 doc/todo/threaded_runtime.mdwn diff --git a/doc/design/assistant/inotify.mdwn b/doc/design/assistant/inotify.mdwn index 7b600090ad..fd81366d44 100644 --- a/doc/design/assistant/inotify.mdwn +++ b/doc/design/assistant/inotify.mdwn @@ -18,6 +18,17 @@ available! I may need to fork off multiple watcher processes to handle this. See [[bugs/Issue_on_OSX_with_some_system_limits]]. +## todo + +* Run niced and ioniced? Seems to make sense, this is a background job. +* configurable option to only annex files meeting certian size or + filename criteria +* option to check files not meeting annex criteria into git directly, + automatically +* honor .gitignore, not adding files it excludes (difficult, probably + needs my own .gitignore parser to avoid excessive running of git commands + to check for ignored files) + ## beyond Linux I'd also like to support OSX and if possible the BSDs. @@ -65,17 +76,6 @@ I'd also like to support OSX and if possible the BSDs. * Windows has a Win32 ReadDirectoryChangesW, and perhaps other things. -## todo - -- Run niced and ioniced? Seems to make sense, this is a background job. -- configurable option to only annex files meeting certian size or - filename criteria -- option to check files not meeting annex criteria into git directly, - automatically -- honor .gitignore, not adding files it excludes (difficult, probably - needs my own .gitignore parser to avoid excessive running of git commands - to check for ignored files) - ## the races Many races need to be dealt with by this code. Here are some of them. diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 2871ec2164..98749d2442 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -10,7 +10,8 @@ all the other git clones, at both the git level and the key/value level. them. Also, enqueue Downloads for any files we're missing. * The TransferWatcher does not notice ongoing transfers, because inotify is waiting for the info file to be closed, but that never happens, it's left - open to keep it locked. + open to keep it locked. May need to separate the transfer info files + into an info file, and a lock file. ## longer-term TODO @@ -32,10 +33,6 @@ all the other git clones, at both the git level and the key/value level. only uploading new files but not downloading, and only downloading files in some directories and not others. See for use cases: [[forum/Wishlist:_options_for_syncing_meta-data_and_data]] -* Running external commands from one thread blocks all of them until - it completes. Try to switch to haskell's threaded runtime, which I - think fixes this. Failing that, make sure all network accessing - commands are run by separate processes or something. ## data syncing diff --git a/doc/todo/threaded_runtime.mdwn b/doc/todo/threaded_runtime.mdwn new file mode 100644 index 0000000000..095ffa4359 --- /dev/null +++ b/doc/todo/threaded_runtime.mdwn @@ -0,0 +1,29 @@ +The [[design/assistant]] would be better if git-annex used ghc's threaded +runtime (`ghc -threaded`). + +Currently, whenever the assistant code runs some external command, all +threads are blocked waiting for it to finish. + +For transfers, the assistant works around this problem by forking separate +upload processes, and not waiting on them until it sees an indication that +they have finished the transfer. While this works, it's messy.. threaded +would be better. + +When pulling, pushing, and merging, the assistant runs external git +commands, and this does block all other threads. The threaded runtime would +really help here. + +--- + +Currently, git-annex seems unstable when built with the threaded runtime. +The test suite tends to hang when testing add. `git-annex` occasionally +hangs, apparently in a futex lock. This is not the assistant hanging, and +git-annex does not otherwise use threads, so this is surprising. --[[Joey]] + +--- + +It would be possible to not use the threaded runtime. Instead, we could +have a child process pool, with associated continuations to run after a +child process finishes. Then periodically do a nonblocking waitpid on each +process in the pool in turn (waiting for any child could break anything not +using the pool!). This is probably a last resort... From c8691d76aa9a438c17a1c15ac01495d782fa84db Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 11:17:20 -0600 Subject: [PATCH 4055/8313] bugfix --- Assistant/Threads/TransferWatcher.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index aa8b3f6e68..5be63fce4f 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -57,7 +57,9 @@ onAdd st dstatus _ file _ = case parseTransferFile file of where go _ Nothing = noop -- transfer already finished go t (Just info) = adjustTransfers dstatus $ - M.insertWith' const t info + M.insertWith' merge t info + -- preseve shouldWait flag, which is not written to disk + merge new old = new { shouldWait = shouldWait old } {- Called when a transfer information file is removed. - From c080bdca18f3f9c78d1864c749815e84d4fa067b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo" Date: Sat, 7 Jul 2012 17:45:44 +0000 Subject: [PATCH 4056/8313] Added a comment --- ...mment_2_d10e3d90cf421ae425e64ab266ea811b._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/forum/Wishlist:_mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment diff --git a/doc/forum/Wishlist:_mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment b/doc/forum/Wishlist:_mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment new file mode 100644 index 0000000000..e14cfa8225 --- /dev/null +++ b/doc/forum/Wishlist:_mark_remotes_offline/comment_2_d10e3d90cf421ae425e64ab266ea811b._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawncBlzaDI248OZGjKQMXrLVQIx4XrZrzFo" + nickname="Perttu" + subject="comment 2" + date="2012-07-07T17:45:43Z" + content=""" +Ah, I didn't read the man page carefully enough. My apologies. + +Setting the ignore status based on an exit status would be +even better, since this avoids re-writing a new config file for +each repository each time I enter or exit my LAN. +"""]] From 9379c77fb304a878481ba1366e055dc726ad2954 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 11:47:36 -0600 Subject: [PATCH 4057/8313] split transfer info and lock files Since the lock file has to be kept open, this prevented the TransferWatcher from noticing when it appeared, since inotify (and more importantly kqueue) events happen when a new file is closed. Writing a separate info file fixes that problem. --- Logs/Transfer.hs | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 494a44c51b..8b88041273 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -1,4 +1,4 @@ -{- git-annex transfer information files +{- git-annex transfer information files and lock files - - Copyright 2012 Joey Hess - @@ -66,9 +66,9 @@ fieldTransfer direction key a = do maybe a (\u -> transfer (Transfer direction (toUUID u) key) afile a) =<< Fields.getField Fields.remoteUUID -{- Runs a transfer action. Creates and locks the transfer information file - - while the action is running. Will throw an error if the transfer is - - already in progress. +{- Runs a transfer action. Creates and locks the lock file while the + - action is running, and stores into in the transfer information + - file. Will throw an error if the transfer is already in progress. -} transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a transfer t file a = do @@ -85,19 +85,18 @@ transfer t file a = do bracketIO (prep tfile mode info) (cleanup tfile) a where prep tfile mode info = do - fd <- openFd tfile ReadWrite (Just mode) + fd <- openFd (transferLockFile tfile) ReadWrite (Just mode) defaultFileFlags { trunc = True } locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) when (locked == Nothing) $ error $ "transfer already in progress" - h <- fdToHandle fd - hPutStr h $ writeTransferInfo info - hFlush h - return h - cleanup tfile h = do + writeFile tfile $ writeTransferInfo info + return fd + cleanup tfile fd = do removeFile tfile - hClose h + removeFile $ transferLockFile tfile + closeFd fd {- If a transfer is still running, returns its TransferInfo. -} checkTransfer :: Transfer -> Annex (Maybe TransferInfo) @@ -105,22 +104,19 @@ checkTransfer t = do mode <- annexFileMode tfile <- fromRepo $ transferFile t mfd <- liftIO $ catchMaybeIO $ - openFd tfile ReadOnly (Just mode) defaultFileFlags + openFd (transferLockFile tfile) ReadOnly (Just mode) defaultFileFlags case mfd of Nothing -> return Nothing -- failed to open file; not running Just fd -> do locked <- liftIO $ getLock fd (WriteLock, AbsoluteSeek, 0, 0) + liftIO $ closeFd fd case locked of - Nothing -> do - liftIO $ closeFd fd - return Nothing - Just (pid, _) -> liftIO $ do - h <- fdToHandle fd - info <- readTransferInfo pid - <$> hGetContentsStrict h - hClose h - return info + Nothing -> return Nothing + Just (pid, _) -> liftIO $ + flip catchDefaultIO Nothing $ + readTransferInfo pid + <$> readFile tfile {- Gets all currently running transfers. -} getTransfers :: Annex [(Transfer, TransferInfo)] @@ -141,6 +137,10 @@ transferFile (Transfer direction u key) r = gitAnnexTransferDir r fromUUID u keyFile key +{- The transfer lock file corresponding to a given transfer info file. -} +transferLockFile :: FilePath -> FilePath +transferLockFile infofile = infofile ++ ".lck" + {- Parses a transfer information filename to a Transfer. -} parseTransferFile :: FilePath -> Maybe Transfer parseTransferFile file = From 7e76e49b9f7a7cb10ff6f9cb0faa354d512f7655 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 11:48:59 -0600 Subject: [PATCH 4058/8313] fixed --- doc/design/assistant/syncing.mdwn | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 98749d2442..89799cd3af 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -8,10 +8,6 @@ all the other git clones, at both the git level and the key/value level. * At startup, and possibly periodically, look for files we have that location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. -* The TransferWatcher does not notice ongoing transfers, because inotify is - waiting for the info file to be closed, but that never happens, it's left - open to keep it locked. May need to separate the transfer info files - into an info file, and a lock file. ## longer-term TODO From 1b9ab0ef93641ea06fa5f2fb97e2e03d6f6a95f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 11:56:23 -0600 Subject: [PATCH 4059/8313] todos --- doc/todo/assistant_git_sync_laddering.mdwn | 8 ++++++++ ...eaded_runtime.mdwn => assistant_threaded_runtime.mdwn} | 0 2 files changed, 8 insertions(+) create mode 100644 doc/todo/assistant_git_sync_laddering.mdwn rename doc/todo/{threaded_runtime.mdwn => assistant_threaded_runtime.mdwn} (100%) diff --git a/doc/todo/assistant_git_sync_laddering.mdwn b/doc/todo/assistant_git_sync_laddering.mdwn new file mode 100644 index 0000000000..f210d41f60 --- /dev/null +++ b/doc/todo/assistant_git_sync_laddering.mdwn @@ -0,0 +1,8 @@ +When the [[design/assistant]] is running on a pair of remotes, I've seen +them get out of sync, such that every pull and merge results in a conflict, +that then has to be auto-resolved. + +This seems similar to the laddering problem described in this old bug: +[[bugs/making_annex-merge_try_a_fast-forward]] + +--[[Joey]] diff --git a/doc/todo/threaded_runtime.mdwn b/doc/todo/assistant_threaded_runtime.mdwn similarity index 100% rename from doc/todo/threaded_runtime.mdwn rename to doc/todo/assistant_threaded_runtime.mdwn From d0b027d27bad3f1249b1d44d9974077aead0f52f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 16:50:19 -0600 Subject: [PATCH 4060/8313] blog for the day --- .../blog/day_27__robust_transfers.mdwn | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 doc/design/assistant/blog/day_27__robust_transfers.mdwn diff --git a/doc/design/assistant/blog/day_27__robust_transfers.mdwn b/doc/design/assistant/blog/day_27__robust_transfers.mdwn new file mode 100644 index 0000000000..49ace417b7 --- /dev/null +++ b/doc/design/assistant/blog/day_27__robust_transfers.mdwn @@ -0,0 +1,31 @@ +Spent most of the day making file content transfers robust. There were lots +of bugs, hopefully I've fixed most of them. It seems to work well now, +even when I throw a lot of files at it. + +One of the changes also sped up transfers; it no longer roundtrips to the +remote to verify it has a file. The idea here is that when the assistant is +running, repos should typically be fairly tightly synced to their remotes +by it, so some of the extra checks that the `move` command does are +unnecessary. + +Also spent some time trying to use ghc's threaded runtime, but continue to +be baffled by the random hangs when using it. This needs fixing eventually; +all the assistant's threads can potentially be blocked when it's waiting on +an external command it has run. + +Also changed how transfer info files are locked. The lock file is now +separate from the info file, which allows the TransferWatcher thread to +notice when an info file is created, and thus actually track transfers +initiated by remotes. + +--- + +I'm fairly close now to merging the `assistant` branch into `master`. +The data syncing code is very brute-force, but it will work well enough +for a first cut. + +Next I can either add some repository network mapping, and use graph +analysis to reduce the number of data transfers, or I can move on to the +[[webapp]]. Not sure yet which I'll do. It's likely that since DebConf +begins tomorrow I'll put off either of those big things until after the +conference. From f74f0b46c1d29dfe83b346c35e569e8aa64c752a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 21:42:52 -0600 Subject: [PATCH 4061/8313] update --- doc/design/assistant/syncing.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 89799cd3af..ca6a09bd80 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -29,6 +29,7 @@ all the other git clones, at both the git level and the key/value level. only uploading new files but not downloading, and only downloading files in some directories and not others. See for use cases: [[forum/Wishlist:_options_for_syncing_meta-data_and_data]] +* speed up git syncing by using the cached ssh connection for it too ## data syncing From 88a5cd6493ef76f9808b55fb6972156a042d60c3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 22:07:26 -0600 Subject: [PATCH 4062/8313] update --- doc/design/assistant.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn index 1069cd8c6d..62070e0201 100644 --- a/doc/design/assistant.mdwn +++ b/doc/design/assistant.mdwn @@ -12,12 +12,13 @@ Feel free to chip in with comments! --[[Joey]] * Month 3 "easy setup": [[!traillink configurators]] [[!traillink pairing]] * Month 4 "polishing": [[!traillink cloud]] [[!traillink leftovers]] * Months 5-6 "9k bonus round": [[!traillink Android]] [[!traillink partial_content]] +* Months 7-11: user-driven features and polishing +* Month 12: "Windows purgatory" [[Windows]] ## not yet on the map: * [[desymlink]] * [[deltas]] -* In my overfunded nighmares: [[Windows]] ## blog From 268b4b7d6e34f8255e6e5f8c115939e731e5ff33 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 7 Jul 2012 22:12:37 -0600 Subject: [PATCH 4063/8313] done --- doc/design/assistant/syncing.mdwn | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index ca6a09bd80..8db4f40ef9 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -3,8 +3,6 @@ all the other git clones, at both the git level and the key/value level. ## immediate action items -* Check that download transfer triggering code works (when a symlink appears - and the remote does *not* upload to us. * At startup, and possibly periodically, look for files we have that location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. From 43426b18000633d3fe227ba2aefaac4779efdaf0 Mon Sep 17 00:00:00 2001 From: hannes Date: Sun, 8 Jul 2012 10:18:32 +0000 Subject: [PATCH 4064/8313] --- ...e_is_not_available_from_a_cloned_repo.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn diff --git a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn new file mode 100644 index 0000000000..e3d642844c --- /dev/null +++ b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn @@ -0,0 +1,19 @@ +I have a git annex repository, and a sync to a remote rsync server (with shared encryption), named "rsync". + +When I clone the repository, I thought it would work as described for S3 (http://git-annex.branchable.com/tips/using_Amazon_S3/): + +> $ git annex initremote rsync + +> git-annex: Specify the type of remote with type= + +When I run + +> $ git initremote rsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=shared + +I get a git remote -- but the data available on the rsync server is not accessible locally + +> $ git whereis + +> git-annex: whereis: 19631 failed + +The original is a git annex from cabal, the cloned repository uses 3.20120522. From ce96276c0a5d9f98c57b52dbca93d130139979a5 Mon Sep 17 00:00:00 2001 From: hannes Date: Sun, 8 Jul 2012 11:57:28 +0000 Subject: [PATCH 4065/8313] --- doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn index e3d642844c..9d48e21004 100644 --- a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn +++ b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn @@ -16,4 +16,4 @@ I get a git remote -- but the data available on the rsync server is not accessib > git-annex: whereis: 19631 failed -The original is a git annex from cabal, the cloned repository uses 3.20120522. +The original is a git annex 3.20120629, the cloned repository uses 3.20120522. From 252f12f3c81dd222d1e1794c9e2a0318a819b292 Mon Sep 17 00:00:00 2001 From: hannes Date: Sun, 8 Jul 2012 13:24:30 +0000 Subject: [PATCH 4066/8313] removed --- ...e_is_not_available_from_a_cloned_repo.mdwn | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn diff --git a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn deleted file mode 100644 index 9d48e21004..0000000000 --- a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo.mdwn +++ /dev/null @@ -1,19 +0,0 @@ -I have a git annex repository, and a sync to a remote rsync server (with shared encryption), named "rsync". - -When I clone the repository, I thought it would work as described for S3 (http://git-annex.branchable.com/tips/using_Amazon_S3/): - -> $ git annex initremote rsync - -> git-annex: Specify the type of remote with type= - -When I run - -> $ git initremote rsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=shared - -I get a git remote -- but the data available on the rsync server is not accessible locally - -> $ git whereis - -> git-annex: whereis: 19631 failed - -The original is a git annex 3.20120629, the cloned repository uses 3.20120522. From b71191df8d025ad22bc5692af3461ae3d3fa96b3 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Sun, 8 Jul 2012 18:13:58 +0000 Subject: [PATCH 4067/8313] Added a comment --- ...mment_4_c25a8eb369e546f65e1a72d89f43066f._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment new file mode 100644 index 0000000000..509752bf18 --- /dev/null +++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_4_c25a8eb369e546f65e1a72d89f43066f._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 4" + date="2012-07-08T18:13:58Z" + content=""" +On kFreeBSD, I get this: + + $ sysctl kern.maxfilesperproc + kern.maxfilesperproc: 11095 + +But ulimit still has 1024 limit, so you'd need to adjust both, as root. Messy.. +"""]] From 92ce8c6164fbac872e4d07d7cf6d1fc48e6bdf4d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 8 Jul 2012 12:24:51 -0600 Subject: [PATCH 4068/8313] update --- doc/design/assistant/syncing.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 8db4f40ef9..14e66ec11b 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -6,6 +6,8 @@ all the other git clones, at both the git level and the key/value level. * At startup, and possibly periodically, look for files we have that location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. +* After git sync, identify content that we don't have that is now available + on remotes, and transfer. ## longer-term TODO From 619297e1a7ba89f50fa5be9d7dfdfe5a9510129a Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Sun, 8 Jul 2012 20:53:50 +0200 Subject: [PATCH 4069/8313] Fix typos on blog --- doc/design/assistant/blog/day_10__lsof.mdwn | 4 ++-- doc/design/assistant/blog/day_12__freebsd_redux.mdwn | 4 ++-- .../assistant/blog/day_14__thinking_about_syncing.mdwn | 4 ++-- doc/design/assistant/blog/day_18__merging.mdwn | 6 +++--- doc/design/assistant/blog/day_19__random_improvements.mdwn | 2 +- .../blog/day_22__horrible_option_parsing_hack.mdwn | 2 +- doc/design/assistant/blog/day_23__transfer_watching.mdwn | 2 +- doc/design/assistant/blog/day_25__transfer_queueing.mdwn | 6 +++--- doc/design/assistant/blog/day_2__races.mdwn | 2 +- doc/design/assistant/blog/day_4__speed.mdwn | 2 +- doc/design/assistant/blog/day_5__committing.mdwn | 4 ++-- doc/design/assistant/blog/day_6__polish.mdwn | 2 +- doc/design/assistant/blog/day_7__bugfixes.mdwn | 2 +- doc/design/assistant/blog/day_8__speed.mdwn | 2 +- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/design/assistant/blog/day_10__lsof.mdwn b/doc/design/assistant/blog/day_10__lsof.mdwn index 32b6705714..d4217677f0 100644 --- a/doc/design/assistant/blog/day_10__lsof.mdwn +++ b/doc/design/assistant/blog/day_10__lsof.mdwn @@ -23,7 +23,7 @@ In other words, I was lost in the weeds for a lot of those hours... At one point, something glorious happened, and it was always making exactly one commit for batch mode modifications of a lot of files (like untarring -them). Unfortunatly, I had to lose that gloriousness due to another +them). Unfortunately, I had to lose that gloriousness due to another potential race, which, while unlikely, would have made the program deadlock if it happened. @@ -40,7 +40,7 @@ are still open for write. This works great! Starting up `git annex watch` when processes have files open is no longer a problem, and even if you're evil enough to try having -muliple processes open the same file, it will complain and not annex it +multiple processes open the same file, it will complain and not annex it until all the writers close it. (Well, someone really evil could turn the write bit back on after git annex diff --git a/doc/design/assistant/blog/day_12__freebsd_redux.mdwn b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn index ba397788a5..5ec446c9de 100644 --- a/doc/design/assistant/blog/day_12__freebsd_redux.mdwn +++ b/doc/design/assistant/blog/day_12__freebsd_redux.mdwn @@ -3,13 +3,13 @@ to `kqueue`, and Haskell code to use that library. By now I think I understand kqueue fairly well -- there are some very tricky parts to the interface. -But... it still did't work. After building all this, my code was +But... it still didn't work. After building all this, my code was failing the same way that the [haskell kqueue library failed](https://github.com/hesselink/kqueue/issues/1) yesterday. I filed a [bug report with a testcase](). Then I thought to ask on #haskell. Got sorted out in quick order! The -problem turns out to be that haskell's runtime has a peridic SIGALARM, +problem turns out to be that haskell's runtime has a periodic SIGALARM, that is interrupting my kevent call. It can be worked around with `+RTS -V0`, but I put in a fix to retry to kevent when it's interrupted. diff --git a/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn b/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn index c4a700d139..4173fbf777 100644 --- a/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn +++ b/doc/design/assistant/blog/day_14__thinking_about_syncing.mdwn @@ -10,13 +10,13 @@ But it's not all easy. Syncing should happen as fast as possible, so changes show up without delay. Eventually it'll need to support syncing between nodes that cannot directly contact one-another. Syncing needs to deal with nodes coming and going; one example of that is a USB drive being -plugged in, which should immediatly be synced, but network can also come +plugged in, which should immediately be synced, but network can also come and go, so it should periodically retry nodes it failed to sync with. To start with, I'll be focusing on fast syncing between directly connected nodes, but I have to keep this wider problem space in mind. One problem with `git annex sync` is that it has to be run in both clones -in order for changes to fully propigate. This is because git doesn't allow +in order for changes to fully propagate. This is because git doesn't allow pushing changes into a non-bare repository; so instead it drops off a new branch in `.git/refs/remotes/$foo/synced/master`. Then when it's run locally it merges that new branch into `master`. diff --git a/doc/design/assistant/blog/day_18__merging.mdwn b/doc/design/assistant/blog/day_18__merging.mdwn index 44a79e14ff..f963cf85dd 100644 --- a/doc/design/assistant/blog/day_18__merging.mdwn +++ b/doc/design/assistant/blog/day_18__merging.mdwn @@ -12,7 +12,7 @@ not sufficient. There are two problems with it: So, instead, git-annex will use a regular `git merge`, and if it fails, it will fix up the conflicts. -That presented its own difficully, of finding which files in the tree +That presented its own difficulty, of finding which files in the tree conflict. `git ls-files --unmerged` is the way to do that, but its output is a quite raw form: @@ -21,9 +21,9 @@ is a quite raw form: 100644 1eabec834c255a127e2e835dadc2d7733742ed9a 2 bar 100644 36902d4d842a114e8b8912c02d239b2d7059c02b 3 bar -I had to stare at the rather inpenetrable documentation for hours and +I had to stare at the rather impenetrable documentation for hours and write a lot of parsing and processing code to get from that to these mostly -self expanatory data types: +self explanatory data types: data Conflicting v = Conflicting { valUs :: Maybe v diff --git a/doc/design/assistant/blog/day_19__random_improvements.mdwn b/doc/design/assistant/blog/day_19__random_improvements.mdwn index 93c1296bab..acb30bf934 100644 --- a/doc/design/assistant/blog/day_19__random_improvements.mdwn +++ b/doc/design/assistant/blog/day_19__random_improvements.mdwn @@ -35,7 +35,7 @@ more threads: 1. Uploads new data to every configured remote. Triggered by the watcher thread when it adds content. Easy; just use a `TSet` of Keys to send. -2. Downloads new data from the cheapest remote that has it. COuld be +2. Downloads new data from the cheapest remote that has it. Could be triggered by the merger thread, after it merges in a git sync. Rather hard; how does it work out what new keys are in the tree without scanning it all? Scan diff --git a/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn b/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn index 9f59d1af91..8f6708e597 100644 --- a/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn +++ b/doc/design/assistant/blog/day_22__horrible_option_parsing_hack.mdwn @@ -1,6 +1,6 @@ Well, sometimes you just have to go for the hack. Trying to find a way to add additional options to git-annex-shell without breaking backwards -compatability, I noticed that it ignores all options after `--`, because +compatibility, I noticed that it ignores all options after `--`, because those tend to be random rsync options due to the way rsync runs it. So, I've added a new class of options, that come in between, like diff --git a/doc/design/assistant/blog/day_23__transfer_watching.mdwn b/doc/design/assistant/blog/day_23__transfer_watching.mdwn index beaf75bc52..3e4e27d479 100644 --- a/doc/design/assistant/blog/day_23__transfer_watching.mdwn +++ b/doc/design/assistant/blog/day_23__transfer_watching.mdwn @@ -21,5 +21,5 @@ nontrivial features can be added easily. -- -Next up: Enough nonsense with tracking tranfers... Time to start actually +Next up: Enough nonsense with tracking transfers... Time to start actually transferring content around! diff --git a/doc/design/assistant/blog/day_25__transfer_queueing.mdwn b/doc/design/assistant/blog/day_25__transfer_queueing.mdwn index 35922c0d11..b07e4592e9 100644 --- a/doc/design/assistant/blog/day_25__transfer_queueing.mdwn +++ b/doc/design/assistant/blog/day_25__transfer_queueing.mdwn @@ -6,7 +6,7 @@ Details follow.. Made the committer thread queue Upload Transfers when new files are added to the annex. Currently it tries to transfer the new content -to *every* remote; this innefficiency needs to be addressed later. +to *every* remote; this inefficiency needs to be addressed later. Made the watcher thread queue Download Transfers when new symlinks appear that point to content we don't have. Typically, that will happen @@ -30,12 +30,12 @@ all the assistant's other threads from entering that monad while a transfer is running. This is also necessary to allow multiple concurrent transfers to run in the future. -This is a very tricky peice of code, because that thread will modify the +This is a very tricky piece of code, because that thread will modify the git-annex branch, and its parent thread has to invalidate its cache in order to see any changes the child thread made. Hopefully that's the extent of the complication of doing this. The only reason this was possible at all is that git-annex already support multiple concurrent processes running -and all making independant changes to the git-annex branch, etc. +and all making independent changes to the git-annex branch, etc. After all my groundwork this week, file content transferring is now fully working! diff --git a/doc/design/assistant/blog/day_2__races.mdwn b/doc/design/assistant/blog/day_2__races.mdwn index fadedb5fb2..19f868a712 100644 --- a/doc/design/assistant/blog/day_2__races.mdwn +++ b/doc/design/assistant/blog/day_2__races.mdwn @@ -1,6 +1,6 @@ Last night I got `git annex watch` to also handle deletion of files. This was not as tricky as feared; the key is using `git rm --ignore-unmatch`, -which avoids most problimatic situations (such as a just deleted file +which avoids most problematic situations (such as a just deleted file being added back before git is run). Also fixed some races when `git annex watch` is doing its startup scan of diff --git a/doc/design/assistant/blog/day_4__speed.mdwn b/doc/design/assistant/blog/day_4__speed.mdwn index badc6b7b18..085d9547b1 100644 --- a/doc/design/assistant/blog/day_4__speed.mdwn +++ b/doc/design/assistant/blog/day_4__speed.mdwn @@ -16,7 +16,7 @@ thread that wakes up periodically, flushes the queue, and autocommits. (This will, in fact, be the start of the [[syncing]] phase of my roadmap!) There's lots of room here for smart behavior. Like, if a lot of changes are being made close together, wait for them to die down before committing. Or, -if it's been idle and a single file appears, commit it immediatly, since +if it's been idle and a single file appears, commit it immediately, since this is probably something the user wants synced out right away. I'll start with something stupid and then add the smarts. diff --git a/doc/design/assistant/blog/day_5__committing.mdwn b/doc/design/assistant/blog/day_5__committing.mdwn index 7d6b52199d..5623873805 100644 --- a/doc/design/assistant/blog/day_5__committing.mdwn +++ b/doc/design/assistant/blog/day_5__committing.mdwn @@ -11,7 +11,7 @@ things slow and ugly. This was not unexpected. So next, I added some smarts to it. First, I wanted to stop it waking up every second when there was nothing to do, and instead blocking wait on a -change occuring. Secondly, I wanted it to know when past changes happened, +change occurring. Secondly, I wanted it to know when past changes happened, so it could detect batch mode scenarios, and avoid committing too frequently. @@ -52,6 +52,6 @@ shouldCommit now changetimes thisSecond t = now `diffUTCTime` t <= 1 """]] -Still some polishing to do to eliminate minor innefficiencies and deal +Still some polishing to do to eliminate minor inefficiencies and deal with more races, but this part of the git-annex assistant is now very usable, and will be going out to my beta testers soon! diff --git a/doc/design/assistant/blog/day_6__polish.mdwn b/doc/design/assistant/blog/day_6__polish.mdwn index dd1239626b..ebe8068c33 100644 --- a/doc/design/assistant/blog/day_6__polish.mdwn +++ b/doc/design/assistant/blog/day_6__polish.mdwn @@ -24,7 +24,7 @@ symlinks might have just been deleted and re-added, or changed, and the index still have the old value. Instead, I got creative. :) We can't trust what the index says about the -symlink, but if the index happens to contian a symlink that looks right, +symlink, but if the index happens to contain a symlink that looks right, we can trust that the SHA1 of its blob is the right SHA1, and reuse it when re-staging the symlink. Wham! Massive speedup! diff --git a/doc/design/assistant/blog/day_7__bugfixes.mdwn b/doc/design/assistant/blog/day_7__bugfixes.mdwn index 3704969e30..79f36fe98d 100644 --- a/doc/design/assistant/blog/day_7__bugfixes.mdwn +++ b/doc/design/assistant/blog/day_7__bugfixes.mdwn @@ -10,7 +10,7 @@ own git index parser (or use one from Hackage), this check requires running tree of files is being moved or unpacked into the watched directory. Instead, I made it only do the check during `git annex watch`'s initial -scan of the tree. This should be ok, because once it's running, you +scan of the tree. This should be OK, because once it's running, you won't be adding new files to git anyway, since it'll automatically annex new files. This is good enough for now, but there are at least two problems with it: diff --git a/doc/design/assistant/blog/day_8__speed.mdwn b/doc/design/assistant/blog/day_8__speed.mdwn index 56b1e9c079..d99add97a7 100644 --- a/doc/design/assistant/blog/day_8__speed.mdwn +++ b/doc/design/assistant/blog/day_8__speed.mdwn @@ -16,7 +16,7 @@ quickly is really only important so people don't think it's a resource hog. First impressions are important. :) But what does "made recently" mean exactly? Well, my answer is possibly -overengineered, but most of it is really groundwork for things I'll need +over engineered, but most of it is really groundwork for things I'll need later anyway. I added a new data structure for tracking the status of the daemon, which is periodically written to disk by another thread (thread #6!) to `.git/annex/daemon.status` Currently it looks like this; I anticipate From ac799c3f363e0008b23e9c174e6fedc35e6fa92a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnIsjgK_cdfZcIAumVRwTvNXS4cD0zNnaI" Date: Mon, 9 Jul 2012 14:44:40 +0000 Subject: [PATCH 4070/8313] removed --- doc/bugs/git_annex_du.mdwn | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 doc/bugs/git_annex_du.mdwn diff --git a/doc/bugs/git_annex_du.mdwn b/doc/bugs/git_annex_du.mdwn deleted file mode 100644 index 2b1315298c..0000000000 --- a/doc/bugs/git_annex_du.mdwn +++ /dev/null @@ -1,14 +0,0 @@ -We need a way to calculate space taken by certain files. - -Use cases: I want to drop some files from my small disk. I need to figure out things that take most space, and drop them. - -Usage examples: - - git annex du -hs *.mp3 - git annex du -sBm --in=here *.ogg - -Would be nice if it was compatible with standard unix `df`. - -> `du -L` works. -> -> See also: [[forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files]] From 94c6be4969f86489add0b25dd04d6785894f511e Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Mon, 9 Jul 2012 23:11:58 +0000 Subject: [PATCH 4071/8313] --- ...t_annex_get__34___does_nothing_useful.mdwn | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 doc/bugs/__34__git_annex_get__34___does_nothing_useful.mdwn diff --git a/doc/bugs/__34__git_annex_get__34___does_nothing_useful.mdwn b/doc/bugs/__34__git_annex_get__34___does_nothing_useful.mdwn new file mode 100644 index 0000000000..7cc431aef2 --- /dev/null +++ b/doc/bugs/__34__git_annex_get__34___does_nothing_useful.mdwn @@ -0,0 +1,62 @@ + + +As you can see, I'm running a pretty recent build of git-annex (ac799c3f363e0008b23e9c174e6fedc35e6fa92a), + + $ git annex version + git-annex version: 3.20120630 + local repository version: 3 + default repository version: 3 + supported repository versions: 3 + upgrade supported from repository versions: 0 1 2 + +We have a file here which isn't currently available yet isn't +currently available (the link is shown in red), + + $ ls -l plot.py + lrwxrwxrwx 1 ben ben 77 Jul 6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py + $ + +Yet git-annex should be able to tell us where it is, + + $ git-annex whereis plot.py + $ + +Hmm, well that's strange. What's happening here, + + $ git-annex whereis plot.py -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","--hash","refs/heads/git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","log","refs/heads/git-annex..d5582e05f41011b571a17003934fe9e40859e4be","--oneline","-n1"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","cat-file","--batch"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"] + $ + +Alright, well maybe `git-annex get` will work, + + $ git annex get plot.py -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"] + $ ls -l plot.py + lrwxrwxrwx 1 ben ben 77 Jul 6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py + +Nope, the link is still shown in red. + +Alright, what about `git-annex copy`? + + $ git annex copy plot.py --from=goldnerlab --to=here -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","show-ref","--hash","refs/heads/git-annex"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","log","refs/heads/git-annex..d5582e05f41011b571a17003934fe9e40859e4be","--oneline","-n1"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","cat-file","--batch"] + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","plot.py"] + $ ls -l plot.py + lrwxrwxrwx 1 ben ben 77 Jul 6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py + +Still red. + +Alright, what if I just try to get a non-existent file? + + $ git annex get adsflkah -d + git ["--git-dir=/home/ben/lori/analysis/data/.git","--work-tree=/home/ben/lori/analysis/data","ls-files","--cached","-z","--","adsflkah"] + $ + +Alright, it didn't fail with an error, that's very strange. What is going on here? From 35e1ff29c6b088bfcd94b3d378ac5ca15b1fb807 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Mon, 9 Jul 2012 23:12:23 +0000 Subject: [PATCH 4072/8313] rename bugs/__34__git_annex_get__34___does_nothing_useful.mdwn to bugs/git_annex_does_nothing_useful.mdwn --- ...oes_nothing_useful.mdwn => git_annex_does_nothing_useful.mdwn} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/bugs/{__34__git_annex_get__34___does_nothing_useful.mdwn => git_annex_does_nothing_useful.mdwn} (100%) diff --git a/doc/bugs/__34__git_annex_get__34___does_nothing_useful.mdwn b/doc/bugs/git_annex_does_nothing_useful.mdwn similarity index 100% rename from doc/bugs/__34__git_annex_get__34___does_nothing_useful.mdwn rename to doc/bugs/git_annex_does_nothing_useful.mdwn From 60537c13ff396aa696d43bc393603d53cbdde7c8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Mon, 9 Jul 2012 23:12:37 +0000 Subject: [PATCH 4073/8313] --- doc/bugs/git_annex_does_nothing_useful.mdwn | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/bugs/git_annex_does_nothing_useful.mdwn b/doc/bugs/git_annex_does_nothing_useful.mdwn index 7cc431aef2..35bbb30581 100644 --- a/doc/bugs/git_annex_does_nothing_useful.mdwn +++ b/doc/bugs/git_annex_does_nothing_useful.mdwn @@ -1,5 +1,3 @@ - - As you can see, I'm running a pretty recent build of git-annex (ac799c3f363e0008b23e9c174e6fedc35e6fa92a), $ git annex version From 5e78008493c4f6ddc9e786f424363188a3b23309 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Mon, 9 Jul 2012 23:16:32 +0000 Subject: [PATCH 4074/8313] Added a comment --- .../comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment b/doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment new file mode 100644 index 0000000000..6bf6e96f6f --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_1_fc4f51ddcbc69631e2835b86c3489c8e._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 1" + date="2012-07-09T23:16:32Z" + content=""" +`git ls-files` is not listing your file. Perhaps your file is not checked into git? +"""]] From 07ed4004f4e48e2f7dba5184a78d67005cdd0505 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Mon, 9 Jul 2012 23:31:09 +0000 Subject: [PATCH 4075/8313] Added a comment --- ...ment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment b/doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment new file mode 100644 index 0000000000..3423bfae45 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_2_9bb1647e6c59f1ed7b13b81ecc33f920._comment @@ -0,0 +1,13 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 2" + date="2012-07-09T23:31:08Z" + content=""" +Not really sure what to say about that other than, + + $ git --git-dir=/home/ben/lori/analysis/data/.git --work-tree=/home/ben/lori/analysis/data ls-files --cached -- plot.py + plot.py + $ + +"""]] From 8b75311b6d1738cfacc040aa49459aa51f73efac Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Tue, 10 Jul 2012 00:23:12 +0000 Subject: [PATCH 4076/8313] Added a comment: Remotes? --- ...ent_3_d434f5c614a27b75d73530b5b918b851._comment | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment b/doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment new file mode 100644 index 0000000000..f03aa27459 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_3_d434f5c614a27b75d73530b5b918b851._comment @@ -0,0 +1,14 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="Remotes? " + date="2012-07-10T00:23:11Z" + content=""" +What does + + git-annex status + +Show? + +Do you have any remotes configured? It looks like you don't somehow. +"""]] From f32dc34f67c202b9c63b6afb72a247d1a70f2270 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Tue, 10 Jul 2012 01:37:33 +0000 Subject: [PATCH 4077/8313] Added a comment --- ..._8b3a37225847fd836c84524c5224fc59._comment | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment b/doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment new file mode 100644 index 0000000000..f798c5e84b --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 4" + date="2012-07-10T01:37:33Z" + content=""" +$ git annex status +supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL +supported remote types: git S3 bup directory rsync web hook +trusted repositories: 0 +semitrusted repositories: 3 + 00000000-0000-0000-0000-000000000001 -- web + 02e4ea72-a77c-11e1-bbd7-0749b04e4b59 -- goldnerlab (Data for Goldner) + 3c1fd026-c794-11e1-8ebb-dbe8684e8a73 -- here +untrusted repositories: 0 +dead repositories: 0 +transfers in progress: none +available local disk space: 16 gigabytes (+1 megabyte reserved) +local annex keys: 0 +local annex size: 0 bytes +known annex keys: 0 +known annex size: 0 bytes +bloom filter size: 16 mebibytes (0% full) +backend usage: +$ git remote +goldnerlab +$ git remote show goldnerlab +* remote goldnerlab + Fetch URL: goldnerlab:data + Push URL: goldnerlab:data + HEAD branch (remote HEAD is ambiguous, may be one of the following): + master + synced/master + Remote branches: + git-annex tracked + master tracked + synced/master tracked + Local branch configured for 'git pull': + master merges with remote master + Local refs configured for 'git push': + git-annex pushes to git-annex (up to date) + master pushes to master (up to date) + synced/master pushes to synced/master (up to date) + + +"""]] From 25f500b76f27df3484d8126f86a33cde64905acb Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Tue, 10 Jul 2012 01:46:00 +0000 Subject: [PATCH 4078/8313] removed --- ..._8b3a37225847fd836c84524c5224fc59._comment | 46 ------------------- 1 file changed, 46 deletions(-) delete mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment b/doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment deleted file mode 100644 index f798c5e84b..0000000000 --- a/doc/bugs/git_annex_does_nothing_useful/comment_4_8b3a37225847fd836c84524c5224fc59._comment +++ /dev/null @@ -1,46 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="comment 4" - date="2012-07-10T01:37:33Z" - content=""" -$ git annex status -supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL -supported remote types: git S3 bup directory rsync web hook -trusted repositories: 0 -semitrusted repositories: 3 - 00000000-0000-0000-0000-000000000001 -- web - 02e4ea72-a77c-11e1-bbd7-0749b04e4b59 -- goldnerlab (Data for Goldner) - 3c1fd026-c794-11e1-8ebb-dbe8684e8a73 -- here -untrusted repositories: 0 -dead repositories: 0 -transfers in progress: none -available local disk space: 16 gigabytes (+1 megabyte reserved) -local annex keys: 0 -local annex size: 0 bytes -known annex keys: 0 -known annex size: 0 bytes -bloom filter size: 16 mebibytes (0% full) -backend usage: -$ git remote -goldnerlab -$ git remote show goldnerlab -* remote goldnerlab - Fetch URL: goldnerlab:data - Push URL: goldnerlab:data - HEAD branch (remote HEAD is ambiguous, may be one of the following): - master - synced/master - Remote branches: - git-annex tracked - master tracked - synced/master tracked - Local branch configured for 'git pull': - master merges with remote master - Local refs configured for 'git push': - git-annex pushes to git-annex (up to date) - master pushes to master (up to date) - synced/master pushes to synced/master (up to date) - - -"""]] From 17b9fcf1d5317d894172f686c97dd859f10f8eb8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Tue, 10 Jul 2012 01:46:23 +0000 Subject: [PATCH 4079/8313] Added a comment --- ..._998e33219d29ea41b0b2a5d2955a9862._comment | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment b/doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment new file mode 100644 index 0000000000..fc9f6c30cb --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_4_998e33219d29ea41b0b2a5d2955a9862._comment @@ -0,0 +1,46 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 4" + date="2012-07-10T01:46:23Z" + content=""" + $ git annex status + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 3 + 00000000-0000-0000-0000-000000000001 -- web + 02e4ea72-a77c-11e1-bbd7-0749b04e4b59 -- goldnerlab (Data for Goldner) + 3c1fd026-c794-11e1-8ebb-dbe8684e8a73 -- here + untrusted repositories: 0 + dead repositories: 0 + transfers in progress: none + available local disk space: 16 gigabytes (+1 megabyte reserved) + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 0 + known annex size: 0 bytes + bloom filter size: 16 mebibytes (0% full) + backend usage: + $ git remote + goldnerlab + $ git remote show goldnerlab + * remote goldnerlab + Fetch URL: goldnerlab:data + Push URL: goldnerlab:data + HEAD branch (remote HEAD is ambiguous, may be one of the following): + master + synced/master + Remote branches: + git-annex tracked + master tracked + synced/master tracked + Local branch configured for 'git pull': + master merges with remote master + Local refs configured for 'git push': + git-annex pushes to git-annex (up to date) + master pushes to master (up to date) + synced/master pushes to synced/master (up to date) + + +"""]] From 9c4d666e9f7e5cc0143424d005ea6baa079d1601 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Tue, 10 Jul 2012 03:03:30 +0000 Subject: [PATCH 4080/8313] Added a comment --- ...t_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment b/doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment new file mode 100644 index 0000000000..90159b5b46 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_5_c72e2571e5b8c06bbfa2276a7ad1e8a6._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 5" + date="2012-07-10T03:03:27Z" + content=""" +Well that's odd. You have remotes but no annexed files.. + +Can you post the commands you used to arrive at this situation? I'm not sure how you would have done that.. Maybe you just need a + + git-annex sync + +to get things going? + +I think somehow you cloned the git repo but not the annex stuff. +"""]] From cfbbe475aa58e475eb52e69aa492cebb7e7c14f8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Tue, 10 Jul 2012 03:26:38 +0000 Subject: [PATCH 4081/8313] Added a comment --- ..._bc8b42432ba25de8f972c192bc3cdff6._comment | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment b/doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment new file mode 100644 index 0000000000..ad98e6874e --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_6_bc8b42432ba25de8f972c192bc3cdff6._comment @@ -0,0 +1,44 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 6" + date="2012-07-10T03:26:35Z" + content=""" +I can easily reproduce the issue as follows, + + $ git clone goldnerlab:data + Cloning into 'data'... + remote: Counting objects: 61902, done. + remote: Compressing objects: 100% (61354/61354), done. + remote: Total 61902 (delta 356), reused 61902 (delta 356) + Receiving objects: 100% (61902/61902), 5.50 MiB | 894 KiB/s, done. + Resolving deltas: 100% (356/356), done. + $ cd data + $ git annex sync + (merging origin/git-annex into git-annex...) + commit + (Recording state in git...) + # On branch master + nothing to commit (working directory clean) + ok + pull origin + ok + push origin + Counting objects: 8, done. + Delta compression using up to 2 threads. + Compressing objects: 100% (5/5), done. + Writing objects: 100% (6/6), 726 bytes, done. + Total 6 (delta 1), reused 1 (delta 0) + Auto packing the repository for optimum performance. + warning: There are too many unreachable loose objects; run 'git prune' to remove them. + To goldnerlab:data + d5582e0..aaddf3c git-annex -> git-annex + ok + +Everything looks good so far. I verify that alex/plot.py doesn't exist. Now let's try getting it, + + $ git annex get alex/plot.py -d + git [\"--git-dir=/home/ben/data/.git\",\"--work-tree=/home/ben/data\",\"ls-files\",\"--cached\",\"-z\",\"--\",\"alex/plot.py\"] + +Uh oh. ls confirms that get was unsucessful. +"""]] From fd6149a8a2bc894cb24cac2deed3cb38f8c0ce19 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Tue, 10 Jul 2012 12:37:44 +0000 Subject: [PATCH 4082/8313] Added a comment --- ..._e7469a4c5e45078ade775f5cbdd17cfc._comment | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment b/doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment new file mode 100644 index 0000000000..c40e4e2cf7 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_7_e7469a4c5e45078ade775f5cbdd17cfc._comment @@ -0,0 +1,67 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 7" + date="2012-07-10T12:37:43Z" + content=""" +But how was the goldnerlab:data repository created? That looks to be where the problem is.. + +I have a slightly older version, but in general it should work the same.. +you can see right away, when I do git annex status it shows \"known annex keys: 1\". +if you do git annex status on goldnerlab, does it say you have any annex keys? + + + $ git-annex version + git-annex version: 3.20120614~bpo60+1 + $ mkdir a + $ cd a + $ git init + Initialized empty Git repository in /tmp/a/.git/ + $ git annex init a + init a ok + (Recording state in git...) + $ echo hi > file + $ git annex add file + add file (checksum...) ok + (Recording state in git...) + $ git commit -m added + fatal: No HEAD commit to compare with (yet) + fatal: No HEAD commit to compare with (yet) + [master (root-commit) cfa9049] added + 1 files changed, 1 insertions(+), 0 deletions(-) + create mode 120000 file + $ cd .. + $ git clone a a_clone + Cloning into a_clone... + done. + $ cd a_clone + $ git annex status + (merging origin/git-annex into git-annex...) + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 3 + 00000000-0000-0000-0000-000000000001 -- web + 445d616e-ca8b-11e1-b170-ff8b03c54243 -- origin (a) + 5d3db51c-ca8b-11e1-bbc3-039dd06ab47b -- here + untrusted repositories: 0 + dead repositories: 0 + available local disk space: 63 megabytes (+1 megabyte reserved) + local annex keys: 0 + local annex size: 0 bytes + known annex keys: 1 + known annex size: 3 bytes + backend usage: + SHA256: 1 + (Recording state in git...) + $ ls + file + $ cat file + cat: file: No such file or directory + $ git annex get file + get file (from origin...) ok + (Recording state in git...) + $ cat file + hi + +"""]] From d03ed387f9902577156d952bdb567ca03cb549bd Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Tue, 10 Jul 2012 13:02:37 +0000 Subject: [PATCH 4083/8313] Added a comment --- ..._bc9e6fd284440a59ffe4e4ed1f73f7d7._comment | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment b/doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment new file mode 100644 index 0000000000..85d03f04bf --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_8_bc9e6fd284440a59ffe4e4ed1f73f7d7._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 8" + date="2012-07-10T13:02:37Z" + content=""" +On goldnerlab, + + $ git annex status + supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL + supported remote types: git S3 bup directory rsync web hook + trusted repositories: 0 + semitrusted repositories: 4 + 00000000-0000-0000-0000-000000000001 -- web + 02e4ea72-a77c-11e1-bbd7-0749b04e4b59 -- here (Data for Goldner) + 351f3ddc-ca3e-11e1-a3fc-6338ef4724a7 + 3c1fd026-c794-11e1-8ebb-dbe8684e8a73 + untrusted repositories: 0 + dead repositories: 0 + transfers in progress: none + available local disk space: 2 terabytes (+1 megabyte reserved) + local annex keys: 19101 + local annex size: 41 gigabytes + known annex keys: 19122 + known annex size: 41 gigabytes + bloom filter size: 16 mebibytes (3.8% full) + backend usage: + WORM: 38223 + +"""]] From 85e5d1167596b1508789e443dc316c4b08932fba Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Tue, 10 Jul 2012 14:08:10 +0000 Subject: [PATCH 4084/8313] Added a comment --- ..._9_38a2dbeee3750d79ca9a943a02fceb29._comment | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment b/doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment new file mode 100644 index 0000000000..dc3206ac5c --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_9_38a2dbeee3750d79ca9a943a02fceb29._comment @@ -0,0 +1,17 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 9" + date="2012-07-10T14:08:10Z" + content=""" +Can you run the series of commands I had above on your two machines? I figure there are two possibilities: + +1. There is something wrong with the git-annex versions you are using. +2. There is something wrong with your repository. (\"warning: There are too many unreachable loose objects\"?) + +so if you can make a temp repository on goldnerlab, then clone it on the other machine and see where it fails, that would be helpful. + +after cloning git-annex status should hopefully say that you have 1 known key, not 0. + +Obviously this won't fix the problem, but it will at least narrow it down. +"""]] From 1d8f876ab946f46088714cd84a5642909e973e83 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Tue, 10 Jul 2012 14:17:43 +0000 Subject: [PATCH 4085/8313] Added a comment --- .../comment_10_457354dc0018333002dc5049935c0feb._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment b/doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment new file mode 100644 index 0000000000..266cff3dbb --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_10_457354dc0018333002dc5049935c0feb._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="comment 10" + date="2012-07-10T14:17:42Z" + content=""" +Hmm, the commands above seem to have worked on both machines (both running 3.20120630). I guess I should probably just try rebuilding my data/ repository from scratch, eh? +"""]] From 3a11b0348cff46b9124fb3c2088fc1557b853d6d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Tue, 10 Jul 2012 14:26:07 +0000 Subject: [PATCH 4086/8313] Added a comment --- ...omment_11_8a6d244165dd238ddf9dd629795de2f6._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment b/doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment new file mode 100644 index 0000000000..1de08ae605 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_11_8a6d244165dd238ddf9dd629795de2f6._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 11" + date="2012-07-10T14:26:06Z" + content=""" +I suppose.. joey can probably help you investigate exactly what went wrong. You might want to save an empty clone of the git repository for later.. + +The easiest way to fix the data is probably to run a `git annex uninit` in the old repository which will put the files back how they were before and then `git-annex import` them into a new repository. +"""]] From 073d25bc9c6d954a55ee5938220432f0bbc56d34 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnIsjgK_cdfZcIAumVRwTvNXS4cD0zNnaI" Date: Tue, 10 Jul 2012 16:52:31 +0000 Subject: [PATCH 4087/8313] git annex pull from encrypted remote --- doc/forum/pulling_from_encrypted_remote.mdwn | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/forum/pulling_from_encrypted_remote.mdwn diff --git a/doc/forum/pulling_from_encrypted_remote.mdwn b/doc/forum/pulling_from_encrypted_remote.mdwn new file mode 100644 index 0000000000..4fa5382382 --- /dev/null +++ b/doc/forum/pulling_from_encrypted_remote.mdwn @@ -0,0 +1,13 @@ +Is there a way to pull from an encrypted remote? + +Use case: +1. Have annex in an encrypted public rsync remote +2. Have USB stick with PGP keys (but not the annex repository) +3. Get to a new computer +4. Set up a new annex using the PGP keys I have. + +1-3 work fine :) However, 4'th is the issue: + +How would I do `git pull ` for an encrypted remote? Is it possible? + +Thanks From 375f1b739256b71733fc1a22bee3dcff30991ec8 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnIsjgK_cdfZcIAumVRwTvNXS4cD0zNnaI" Date: Tue, 10 Jul 2012 16:52:55 +0000 Subject: [PATCH 4088/8313] --- doc/forum/pulling_from_encrypted_remote.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/forum/pulling_from_encrypted_remote.mdwn b/doc/forum/pulling_from_encrypted_remote.mdwn index 4fa5382382..6e8b3c8a1c 100644 --- a/doc/forum/pulling_from_encrypted_remote.mdwn +++ b/doc/forum/pulling_from_encrypted_remote.mdwn @@ -1,6 +1,7 @@ Is there a way to pull from an encrypted remote? Use case: + 1. Have annex in an encrypted public rsync remote 2. Have USB stick with PGP keys (but not the annex repository) 3. Get to a new computer From 00dcafef2c9c1f0c66eb9348e24756e8070a0cc9 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnIsjgK_cdfZcIAumVRwTvNXS4cD0zNnaI" Date: Tue, 10 Jul 2012 16:53:21 +0000 Subject: [PATCH 4089/8313] removed "thanks" --- doc/forum/pulling_from_encrypted_remote.mdwn | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/forum/pulling_from_encrypted_remote.mdwn b/doc/forum/pulling_from_encrypted_remote.mdwn index 6e8b3c8a1c..e9486b597d 100644 --- a/doc/forum/pulling_from_encrypted_remote.mdwn +++ b/doc/forum/pulling_from_encrypted_remote.mdwn @@ -10,5 +10,3 @@ Use case: 1-3 work fine :) However, 4'th is the issue: How would I do `git pull ` for an encrypted remote? Is it possible? - -Thanks From aa558f0b00ef012f6c7532d8451de3fb1ed1eb9b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 10 Jul 2012 11:53:45 -0600 Subject: [PATCH 4090/8313] copy, drop: Avoid checking numcopies attribute unnecessarily --- Command.hs | 15 +++++++++++++-- Command/Copy.hs | 2 +- Command/Drop.hs | 2 +- Command/Get.hs | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Command.hs b/Command.hs index 0dff0c862e..cbffd39fa5 100644 --- a/Command.hs +++ b/Command.hs @@ -21,6 +21,7 @@ module Command ( isBareRepo, numCopies, autoCopies, + autoCopiesWith, module ReExported ) where @@ -112,8 +113,18 @@ numCopies file = readish <$> checkAttr "annex.numcopies" file - In auto mode, first checks that the number of known - copies of the key is > or < than the numcopies setting, before running - the action. -} -autoCopies :: FilePath -> Key -> (Int -> Int -> Bool) -> (Maybe Int -> CommandStart) -> CommandStart -autoCopies file key vs a = do +autoCopies :: FilePath -> Key -> (Int -> Int -> Bool) -> CommandStart -> CommandStart +autoCopies file key vs a = Annex.getState Annex.auto >>= go + where + go False = a + go True = do + numcopiesattr <- numCopies file + needed <- getNumCopies numcopiesattr + (_, have) <- trustPartition UnTrusted =<< Remote.keyLocations key + if length have `vs` needed then a else stop + +autoCopiesWith :: FilePath -> Key -> (Int -> Int -> Bool) -> (Maybe Int -> CommandStart) -> CommandStart +autoCopiesWith file key vs a = do numcopiesattr <- numCopies file Annex.getState Annex.auto >>= auto numcopiesattr where diff --git a/Command/Copy.hs b/Command/Copy.hs index a8ec225706..5d92eef2e1 100644 --- a/Command/Copy.hs +++ b/Command/Copy.hs @@ -24,5 +24,5 @@ seek = [withField Command.Move.toOption Remote.byName $ \to -> -- A copy is just a move that does not delete the source file. -- However, --auto mode avoids unnecessary copies. start :: Maybe Remote -> Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart -start to from file (key, backend) = autoCopies file key (<) $ \_numcopies -> +start to from file (key, backend) = autoCopies file key (<) $ Command.Move.start to from False file (key, backend) diff --git a/Command/Drop.hs b/Command/Drop.hs index 28a52d6262..ddf44ab82d 100644 --- a/Command/Drop.hs +++ b/Command/Drop.hs @@ -30,7 +30,7 @@ seek = [withField fromOption Remote.byName $ \from -> withFilesInGit $ whenAnnexed $ start from] start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart -start from file (key, _) = autoCopies file key (>) $ \numcopies -> +start from file (key, _) = autoCopiesWith file key (>) $ \numcopies -> case from of Nothing -> startLocal file numcopies key Just remote -> do diff --git a/Command/Get.hs b/Command/Get.hs index a5901ba664..95a7040bb2 100644 --- a/Command/Get.hs +++ b/Command/Get.hs @@ -24,7 +24,7 @@ seek = [withField Command.Move.fromOption Remote.byName $ \from -> start :: Maybe Remote -> FilePath -> (Key, Backend) -> CommandStart start from file (key, _) = stopUnless (not <$> inAnnex key) $ - autoCopies file key (<) $ \_numcopies -> + autoCopies file key (<) $ case from of Nothing -> go $ perform key file Just src -> From 1c616d486f01e79776038480e4949ac2c99e3df0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Tue, 10 Jul 2012 18:14:31 +0000 Subject: [PATCH 4091/8313] Added a comment --- ...comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment diff --git a/doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment b/doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment new file mode 100644 index 0000000000..2da698f4a9 --- /dev/null +++ b/doc/forum/pulling_from_encrypted_remote/comment_1_e9d6a9a6e01d01edb41a11b0da11d74d._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2012-07-10T18:14:31Z" + content=""" +You just need to `git clone` the existing repository and make sure the `git remote`s are setup. + +the 'rsync remote' is not actually the annex, it's just a collection of encrypted files with obfuscated names. You need a copy of the actual repository to restore the files. +"""]] From bafc50e05e098234d2d22886085d9844fc763e0e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 10 Jul 2012 12:34:29 -0600 Subject: [PATCH 4092/8313] better git version determination --- Build/Configure.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build/Configure.hs b/Build/Configure.hs index 24743bf618..cf6623b226 100644 --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -72,7 +72,7 @@ getVersionString = do getGitVersion :: Test getGitVersion = do (_, s) <- pipeFrom "git" ["--version"] - let version = last $ words $ head $ lines s + let version = unwords $ drop 2 $ words $ head $ lines s return $ Config "gitversion" (StringConfig version) getSshConnectionCaching :: Test From 81b20a581ae6f3ff4217a340b0c1576851938019 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 10 Jul 2012 12:40:31 -0600 Subject: [PATCH 4093/8313] avoid --no-inplace Not available on systems with shoddy getopts. Should not be necessary, as that's rsync's default. --- Remote/Rsync.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 887c68339a..29bceb2db8 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -169,7 +169,7 @@ checkPresent r o k = do - ensure that files are only moved into place once complete -} partialParams :: CommandParam -partialParams = Params "--no-inplace --partial --partial-dir=.rsync-partial" +partialParams = Params "--partial --partial-dir=.rsync-partial" {- Runs an action in an empty scratch directory that can be used to build - up trees for rsync. -} From 2051949aa46c7619f299e51b98c44e9fdc97459e Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Wed, 11 Jul 2012 23:23:04 +0000 Subject: [PATCH 4094/8313] Added a comment --- ...2_30d06bc0f1c37d988a1a31962b57533c._comment | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment diff --git a/doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment b/doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment new file mode 100644 index 0000000000..3de1577529 --- /dev/null +++ b/doc/bugs/git_annex_does_nothing_useful/comment_12_30d06bc0f1c37d988a1a31962b57533c._comment @@ -0,0 +1,18 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="165.98.113.100" + subject="comment 12" + date="2012-07-11T23:23:04Z" + content=""" +Looking at this a leetle more closely, you had: + +
+lrwxrwxrwx 1 ben ben 77 Jul  6 14:01 plot.py -> ../.git/annex/objects/WORM:1301941019:720:plot.py/WORM:1301941019:720:plot.py
+
+ +Well, that is not how a git-annex symlink currently looks, so it ignores it. + +Apparenly this repository was created with an old version of git-annex, possibly version 1, and you've dropped in the current version, but the normal upgrade machinery failed. This could happen if you made a new clone of a version 1 bare repository. + +I suggest you first find out what version of git-annex was originally used to create this repository (ie, version 0, 1, or 2 ... probably 1). Then make a clone, and \"git config annex.version $N\" where N=the version used). Then \"git annex upgrade\" and you should be good to go. Remember to push or sync the upgrade back to the bare repo so you don't need to do this again. +"""]] From bd037c4ca3426e5dbc08bf7033d9771c9883030c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 Jul 2012 12:07:54 -0400 Subject: [PATCH 4095/8313] update --- doc/design/assistant/syncing.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 14e66ec11b..836dac43c5 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -30,6 +30,8 @@ all the other git clones, at both the git level and the key/value level. files in some directories and not others. See for use cases: [[forum/Wishlist:_options_for_syncing_meta-data_and_data]] * speed up git syncing by using the cached ssh connection for it too + (will need to use `GIT_SSH`, which needs to point to a command to run, + not a shell command line) ## data syncing From ee5a813391aceba12027f2f225d3877c487b04e4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:51:17 +0000 Subject: [PATCH 4096/8313] Added a comment: ARM support --- .../comment_1_bab6f6fa720273c0f9700a3765150189._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment diff --git a/doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment b/doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment new file mode 100644 index 0000000000..3e1330f96d --- /dev/null +++ b/doc/design/assistant/webapp/comment_1_bab6f6fa720273c0f9700a3765150189._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:51:15Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2, GHC's linker has enough ARM support to allow a selection of common packages compile on my PandaBoard. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From 5393d9b98d07ab9c4b3b7317c9f5afa050e8241a Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:51:31 +0000 Subject: [PATCH 4097/8313] Added a comment: ARM support --- .../comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment diff --git a/doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment b/doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment new file mode 100644 index 0000000000..9da9f95a79 --- /dev/null +++ b/doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:51:30Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2, GHC's linker has enough ARM support to allow a selection of common packages compile on my PandaBoard. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From 41c3cb2ed37b84462f7eec3d077b7428c71b10ff Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:51:44 +0000 Subject: [PATCH 4098/8313] Added a comment: ARM support --- .../comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment diff --git a/doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment b/doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment new file mode 100644 index 0000000000..4ffea4d83d --- /dev/null +++ b/doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:51:43Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From 0a1e81b3ef493ca75836a47dc7d18c93c30e9728 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:52:01 +0000 Subject: [PATCH 4099/8313] Added a comment: ARM support --- .../comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment diff --git a/doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment b/doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment new file mode 100644 index 0000000000..ec945fa194 --- /dev/null +++ b/doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:52:01Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From 3882b3216af123ce8f6b206cbcbda5a7d02beccc Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:53:27 +0000 Subject: [PATCH 4100/8313] Added a comment: ARM support --- .../comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment diff --git a/doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment b/doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment new file mode 100644 index 0000000000..23733505e6 --- /dev/null +++ b/doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:53:26Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From a1f5b128a92cdaa51fd94961f5b6b4a0d8071038 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:54:24 +0000 Subject: [PATCH 4101/8313] Added a comment: ARM support --- .../comment_6_987be8011f2a644c9532d4738c30237c._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment diff --git a/doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment b/doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment new file mode 100644 index 0000000000..2d83b900db --- /dev/null +++ b/doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:54:23Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From 94ac8ab9bf55fe3370d24e8700ea2361a57dead4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:56:24 +0000 Subject: [PATCH 4102/8313] Added a comment: ARM support --- .../comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment diff --git a/doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment b/doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment new file mode 100644 index 0000000000..e9807db921 --- /dev/null +++ b/doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:56:23Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From 167027f6e472ae30b16d8f1dc48ae653e9757885 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 16:57:30 +0000 Subject: [PATCH 4103/8313] Added a comment: ARM support --- .../comment_8_dacadc3e7e04452e848605c0f0f5752c._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment diff --git a/doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment b/doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment new file mode 100644 index 0000000000..1f05a3a86a --- /dev/null +++ b/doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T16:57:30Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From ec550077406c01a22dc3c88f2a19f75a22e8a818 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 13 Jul 2012 14:05:19 -0400 Subject: [PATCH 4104/8313] remove duplicates --- .../comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment | 8 -------- .../comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment | 8 -------- .../comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment | 8 -------- .../comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment | 8 -------- .../comment_6_987be8011f2a644c9532d4738c30237c._comment | 8 -------- .../comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment | 8 -------- .../comment_8_dacadc3e7e04452e848605c0f0f5752c._comment | 8 -------- 7 files changed, 56 deletions(-) delete mode 100644 doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment delete mode 100644 doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment delete mode 100644 doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment delete mode 100644 doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment delete mode 100644 doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment delete mode 100644 doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment delete mode 100644 doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment diff --git a/doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment b/doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment deleted file mode 100644 index 9da9f95a79..0000000000 --- a/doc/design/assistant/webapp/comment_2_6d4cffaeccbbef39203070cbe67a0d4f._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:51:30Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2, GHC's linker has enough ARM support to allow a selection of common packages compile on my PandaBoard. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] diff --git a/doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment b/doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment deleted file mode 100644 index 4ffea4d83d..0000000000 --- a/doc/design/assistant/webapp/comment_3_0ce0ac82fb437c7b6e9e0240f3e2aada._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:51:43Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] diff --git a/doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment b/doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment deleted file mode 100644 index ec945fa194..0000000000 --- a/doc/design/assistant/webapp/comment_4_68824ec48a48071d217dd61a0b7e0e9a._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:52:01Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] diff --git a/doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment b/doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment deleted file mode 100644 index 23733505e6..0000000000 --- a/doc/design/assistant/webapp/comment_5_dc8928ced23bc4f85f6a18100d7adfe7._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:53:26Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] diff --git a/doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment b/doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment deleted file mode 100644 index 2d83b900db..0000000000 --- a/doc/design/assistant/webapp/comment_6_987be8011f2a644c9532d4738c30237c._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:54:23Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] diff --git a/doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment b/doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment deleted file mode 100644 index e9807db921..0000000000 --- a/doc/design/assistant/webapp/comment_7_afeee95ac6366065a2e3c5f1f7d4842d._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:56:23Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] diff --git a/doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment b/doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment deleted file mode 100644 index 1f05a3a86a..0000000000 --- a/doc/design/assistant/webapp/comment_8_dacadc3e7e04452e848605c0f0f5752c._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T16:57:30Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] From 50a26cce894bd2f8ec59c21e89e7703dd9b1ebf3 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs" Date: Fri, 13 Jul 2012 19:15:16 +0000 Subject: [PATCH 4105/8313] Added a comment: Path problems --- ...t_8_8448e55026d2c2b50d8da41707686bea._comment | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment diff --git a/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment b/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment new file mode 100644 index 0000000000..148f016db7 --- /dev/null +++ b/doc/walkthrough/using_ssh_remotes/comment_8_8448e55026d2c2b50d8da41707686bea._comment @@ -0,0 +1,16 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs" + nickname="Jakub" + subject="Path problems" + date="2012-07-13T19:15:15Z" + content=""" +Hi, + +I have a same 'git-annex-shell command not found' problem as above. I've installed git annex via cabal into my ~/.haskell_bin directory. Then I've added this dir both to ~/.bashrc and ~/.zshrc. I can run git annex or 'git annex-shell' and everything is fine. My guess is that haskell is trying to spawn git-annex-shell with some current $PATH unaware shell like dash maybe? + +I've fixed this behavior by using a really ugly hack - I've symlinked ~/.haskell_bin/git-annex-shell to /usr/bin/git-annex-shell on all my machines and the problem is gone. Somehow haskell (or whatever is trying to call git-annex-shell) is unaware of path modifications from .bashrc/.zshrc + +Here is the path modification I've used: + +export PATH=~/.haskell_bin:$PATH +"""]] From f869624f9c4370dd8a248ab40965db6d3469f503 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs" Date: Fri, 13 Jul 2012 19:27:47 +0000 Subject: [PATCH 4106/8313] Added a comment: Fixed --- ..._61833299a9878f23ac57598fa6da8839._comment | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment diff --git a/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment b/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment new file mode 100644 index 0000000000..ddb96da369 --- /dev/null +++ b/doc/walkthrough/using_ssh_remotes/comment_9_61833299a9878f23ac57598fa6da8839._comment @@ -0,0 +1,23 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmB-gCGEs--zfmvYU-__Hj2FbliUXgxMDs" + nickname="Jakub" + subject="Fixed" + date="2012-07-13T19:27:46Z" + content=""" +Found the problem: + +One should never use ~ in such path: + +WRONG export PATH=~/somedir:$PATH + +Instead one should use $HOME: + +GOOD export PATH=$HOME/somedir:$PATH + +Can I surpress the message that shell failed with status 255 when a repo is unavailible? I've got two repos pointing to one machine - either via vpn or local lan and I keep getting erros if one is unavailible: + +ssh: connect to host 10.9.0.1 port 39882: No route to host +Command ssh [\"-S\",\"/home/pielgrzym/annex/.git/annex/ssh/nas\",\"-o\",\"ControlMaster=auto\",\"-o\",\"ControlPersist=yes\",\"nas\",\"git-annex-shell 'configlist' '/~/annex'\"] failed; exit code 255 + + +"""]] From c07cfaec768ddb0032f8f43b0e73f55627f9a2c0 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 20:14:31 +0000 Subject: [PATCH 4107/8313] Added a comment: ARM support --- .../comment_2_cce84970912c6ca500134fa4fde46252._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment diff --git a/doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment b/doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment new file mode 100644 index 0000000000..baa8667254 --- /dev/null +++ b/doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" + nickname="Ben" + subject="ARM support" + date="2012-07-13T20:14:30Z" + content=""" +The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. +"""]] From e67877088c05655cb204846f7c4eabe8c488257f Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" Date: Fri, 13 Jul 2012 20:14:53 +0000 Subject: [PATCH 4108/8313] removed --- .../comment_2_cce84970912c6ca500134fa4fde46252._comment | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment diff --git a/doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment b/doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment deleted file mode 100644 index baa8667254..0000000000 --- a/doc/design/assistant/webapp/comment_2_cce84970912c6ca500134fa4fde46252._comment +++ /dev/null @@ -1,8 +0,0 @@ -[[!comment format=mdwn - username="https://www.google.com/accounts/o8/id?id=AItOawlup4hyZo4eCjF8T85vfRXMKBxGj9bMdl0" - nickname="Ben" - subject="ARM support" - date="2012-07-13T20:14:30Z" - content=""" -The closure of [this](http://hackage.haskell.org/trac/ghc/ticket/5839) ticket hopefully marks the end of TH issues on ARM. As of 7.4.2 GHC's linker has enough ARM support to allow a selection of common packages compile on my ARM. That being said, it hasn't had a whole lot of testing so it's possible I still need to implement a few relocation types. -"""]] From 06d2b60eef7c73776fd538fe8b26e98b7e0159ae Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 14 Jul 2012 00:45:39 -0400 Subject: [PATCH 4109/8313] catchup blog --- .../day_28-35__threaded_runtime_tarpit.mdwn | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn diff --git a/doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn b/doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn new file mode 100644 index 0000000000..612dfc514c --- /dev/null +++ b/doc/design/assistant/blog/day_28-35__threaded_runtime_tarpit.mdwn @@ -0,0 +1,17 @@ +I didn't plan to work on git-annex much while at DebConf, because the conference +always prevents the kind of concentration I need. But I unexpectedly also had to deal +with [three dead drives](http://joeyh.name/blog/entry/I_am_become_Joey_destroyer_of_drives/) +and illness this week. + +That said, I have been trying to debug a problem with git-annex and Haskell's threaded +runtime all week. It just hangs, randomly. No luck so far isolating why, although I now +have a branch that hangs fairly reliably, and in which I am trying to whittle the entire +git-annex code base (all 18 thousand lines!) into a nice test case. + +This threaded runtime problem doesn't affect the assistant yet, but if I want to use +Yesod in developing the webapp, I'll need the threaded runtime, and using the threaded +runtime in the assistant generally would make it more responsive and less hacky. + +Since this is a task I can work on without much concentration, I'll probably keep beating +on it until I return home. Then I need to spend some quality thinking time on where +to go next in the assistant. From 966c580e38cf209509e479f02608dbe415dc97df Mon Sep 17 00:00:00 2001 From: "http://bergey.dreamwidth.org/" Date: Sat, 14 Jul 2012 15:42:06 +0000 Subject: [PATCH 4110/8313] Added a comment: git-media --- .../comment_1_ab41bec1ccc884e71780cb9458439170._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/not/comment_1_ab41bec1ccc884e71780cb9458439170._comment diff --git a/doc/not/comment_1_ab41bec1ccc884e71780cb9458439170._comment b/doc/not/comment_1_ab41bec1ccc884e71780cb9458439170._comment new file mode 100644 index 0000000000..6a7eed33b0 --- /dev/null +++ b/doc/not/comment_1_ab41bec1ccc884e71780cb9458439170._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="http://bergey.dreamwidth.org/" + ip="66.80.90.109" + subject="git-media" + date="2012-07-14T15:42:05Z" + content=""" +I haven't used git-media, but from the README it looks as though they now support several backends. Might want to update the (very helpful!) comparison. +"""]] From 4fa3cb3e76f4a0c70766c70e906572804fe9b5de Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 14 Jul 2012 16:25:07 -0400 Subject: [PATCH 4111/8313] blog for the day --- doc/design/assistant/blog/day_36__minimal_test_case.mdwn | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/design/assistant/blog/day_36__minimal_test_case.mdwn diff --git a/doc/design/assistant/blog/day_36__minimal_test_case.mdwn b/doc/design/assistant/blog/day_36__minimal_test_case.mdwn new file mode 100644 index 0000000000..b1877f98a7 --- /dev/null +++ b/doc/design/assistant/blog/day_36__minimal_test_case.mdwn @@ -0,0 +1,9 @@ +Managed to find a minimal, 20 line test case for at least one of the ways +git-annex was hanging with GHC's threaded runtime. Sent it off to +haskell-cafe for analysis. +[thread](http://news.gmane.org/gmane.comp.lang.haskell.cafe) + +Further managed to narrow the bug down to MissingH's use of logging code, +that git-annex doesn't use. [bug report](http://bugs.debian.org/681621). +So, I can at least get around this problem with a modified version of +MissingH. Hopefully that was the only thing causing the hangs I was seeing! From b1321745f54c4f9ff5a17aaf43db821feb914d54 Mon Sep 17 00:00:00 2001 From: "http://www.joachim-breitner.de/" Date: Sun, 15 Jul 2012 21:38:32 +0000 Subject: [PATCH 4112/8313] --- ...4__git_annex_watch__34___adds_map.dot.mdwn | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn diff --git a/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn b/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn new file mode 100644 index 0000000000..d6dc3e42de --- /dev/null +++ b/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn @@ -0,0 +1,19 @@ +"git annex watch" will add the file generated by "git annex map", which is probably not intended. Shouldn’t this file be created in /tmp or .git/annex/ or somewhere else? + + /tmp $ cd test/ + /tmp/test $ git init + Initialized empty Git repository in /tmp/test/.git/ + /tmp/test $ git annex init + init ok + (Recording state in git...) + /tmp/test $ git annex watch + /tmp/test $ git annex map + map /tmp/test ok + + running: dot -Tx11 map.dot + + ok + /tmp/test $ ls -l + insgesamt 4 + lrwxrwxrwx 1 jojo jojo 180 Jul 15 23:36 map.dot -> .git/annex/objects/P3/76/SHA256-s208--44199582b5948512ff12cf03de0b86fa1bebf09785dba2827fe52afee0afbe3d/SHA256-s208--44199582b5948512ff12cf03de0b86fa1bebf09785dba2827fe52afee0afbe3d + From dc2e0d0d9bc8d5fd106e7c1c8d871498f8012233 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 11:32:21 -0400 Subject: [PATCH 4113/8313] updates --- doc/design/assistant/syncing.mdwn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 836dac43c5..a9f59f4968 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -7,7 +7,9 @@ all the other git clones, at both the git level and the key/value level. location tracking indicates remotes do not, and enqueue Uploads for them. Also, enqueue Downloads for any files we're missing. * After git sync, identify content that we don't have that is now available - on remotes, and transfer. + on remotes, and transfer. But first, need to ensure that when a remote + receives content, and updates its location log, it syncs that update + out. ## longer-term TODO @@ -90,6 +92,8 @@ anyway. Watcher. **done** * enqueue Tranferrs (Downloads) as new dangling symlinks are noticed by Watcher. **done** + (Note: Needs git-annex branch to be merged before the tree is merged, + so it knows where to download from. Checked and this is the case.) * Write basic Transfer handling thread. Multiple such threads need to be able to be run at once. Each will need its own independant copy of the Annex state monad. **done** From b7d3cefde9a82a7a5bab90eb621690fc969da5ea Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 12:06:35 -0400 Subject: [PATCH 4114/8313] merge two shouldTransfer checks --- Assistant/Threads/Transferrer.hs | 35 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index aaf654d34b..9d3358f546 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -31,20 +31,32 @@ transfererThread st dstatus transferqueue slots = go where go = do (t, info) <- getNextTransfer transferqueue - whenM (runThreadState st $ shouldTransfer dstatus t) $ + whenM (runThreadState st $ shouldTransfer dstatus t info) $ runTransfer st dstatus slots t info go {- Checks if the requested transfer is already running, or - - the file to download is already present. -} -shouldTransfer :: DaemonStatusHandle -> Transfer -> Annex Bool -shouldTransfer dstatus t = go =<< currentTransfers <$> getDaemonStatus dstatus + - the file to download is already present, or the remote + - being uploaded to isn't known to have the file. -} +shouldTransfer :: DaemonStatusHandle -> Transfer -> TransferInfo -> Annex Bool +shouldTransfer dstatus t info = + go =<< currentTransfers <$> getDaemonStatus dstatus where go m | M.member t m = return False | transferDirection t == Download = - not <$> inAnnex (transferKey t) - | otherwise = return True + not <$> inAnnex key + | transferDirection t == Upload = + {- Trust the location log to check if the + - remote already has the key. This avoids + - a roundtrip to the remote. -} + case transferRemote info of + Nothing -> return False + Just remote -> + notElem (Remote.uuid remote) + <$> loggedLocations key + | otherwise = return False + key = transferKey t {- A transfer is run in a separate process, with a *copy* of the Annex - state. This is necessary to avoid blocking the rest of the assistant @@ -60,7 +72,7 @@ runTransfer :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Transfer -> runTransfer st dstatus slots t info = case (transferRemote info, associatedFile info) of (Nothing, _) -> noop (_, Nothing) -> noop - (Just remote, Just file) -> whenM (shouldtransfer remote) $ do + (Just remote, Just file) -> do pid <- inTransferSlot slots $ unsafeForkProcessThreadState st $ transferprocess remote file @@ -78,15 +90,6 @@ runTransfer st dstatus slots t info = case (transferRemote info, associatedFile | otherwise = "to" key = transferKey t - shouldtransfer remote - | isdownload = return True - | otherwise = runThreadState st $ - {- Trust the location log to check if the - - remote already has the key. This avoids - - a roundtrip to the remote. -} - notElem (Remote.uuid remote) - <$> loggedLocations key - transferprocess remote file = do showStart "copy" file showAction $ tofrom ++ " " ++ Remote.name remote From c34d8ae088e284b7585b7e32330945d1e9c922f2 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 12:17:01 -0400 Subject: [PATCH 4115/8313] avoid enqueing downloads from remotes that don't have the key --- Assistant/TransferQueue.hs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index 5e1fad4560..73e73ca0af 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -33,24 +33,24 @@ stubInfo f = TransferInfo {- Adds pending transfers to the end of the queue for some of the known - remotes. -} queueTransfers :: TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex () -queueTransfers q daemonstatus k f direction = +queueTransfers q daemonstatus k f direction = do + rs <- knownRemotes <$> getDaemonStatus daemonstatus mapM_ (\r -> queue r $ gentransfer r) - =<< sufficientremotes . knownRemotes - <$> getDaemonStatus daemonstatus + =<< sufficientremotes rs where sufficientremotes l - -- Queue downloads from all remotes, with the - -- cheapest ones first. More expensive ones will - -- only be tried if downloading from a cheap one - -- fails. - -- TODO: avoid downloading from remotes that don't - -- have the key. - | direction == Download = l + -- Queue downloads from all remotes that + -- have the key, with the cheapest ones first. + -- More expensive ones will only be tried if + -- downloading from a cheap one fails. + | direction == Download = do + uuids <- Remote.keyLocations k + return $ filter (\r -> uuid r `elem` uuids) l -- TODO: Determine a smaller set of remotes that -- can be uploaded to, in order to ensure all -- remotes can access the content. Currently, -- send to every remote we can. - | otherwise = l + | otherwise = return l gentransfer r = Transfer { transferDirection = direction , transferKey = k From f5f88794713ebbbbc177d064f074a80ca23e9b79 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 12:27:06 -0400 Subject: [PATCH 4116/8313] map: Write map.dot to .git/annex, which avoids watch trying to annex it. --- Command/Map.hs | 4 ++-- debian/changelog | 1 + doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn | 6 +++++- test.hs | 3 --- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Command/Map.hs b/Command/Map.hs index 65e28945f6..0773f68283 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -39,6 +39,8 @@ start = do umap <- uuidMap trusted <- trustGet Trusted + + file <- () <$> fromRepo gitAnnexDir <*> pure "map.dot" liftIO $ writeFile file (drawMap rs umap trusted) next $ next $ @@ -49,8 +51,6 @@ start = do showOutput liftIO $ boolSystem "dot" [Param "-Tx11", File file] ) - where - file = "map.dot" {- Generates a graph for dot(1). Each repository, and any other uuids, are - displayed as a node, and each of its remotes is represented as an edge diff --git a/debian/changelog b/debian/changelog index 5eaf9d52eb..f90763acae 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,7 @@ git-annex (3.20120630) UNRELEASED; urgency=low faster than forking the more optimised external program. * SHAnE backends are now smarter about composite extensions, such as .tar.gz Closes: #680450 + * map: Write map.dot to .git/annex, which avoids watch trying to annex it. -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 diff --git a/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn b/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn index d6dc3e42de..94c495735e 100644 --- a/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn +++ b/doc/bugs/__34__git_annex_watch__34___adds_map.dot.mdwn @@ -1,4 +1,8 @@ -"git annex watch" will add the file generated by "git annex map", which is probably not intended. Shouldn’t this file be created in /tmp or .git/annex/ or somewhere else? +"git annex watch" will add the file generated by "git annex map", which is +probably not intended. Shouldn’t this file be created in /tmp or +.git/annex/ or somewhere else? + +> Indeed, so [[done]] --[[Joey]] /tmp $ cd test/ /tmp/test $ git init diff --git a/test.hs b/test.hs index 089c86bfb6..9de73264ee 100644 --- a/test.hs +++ b/test.hs @@ -550,9 +550,6 @@ test_map = "git-annex map" ~: intmpclonerepo $ do git_annex "describe" ["origin", "origin repo"] @? "describe 2 failed" -- --fast avoids it running graphviz, not a build dependency git_annex "map" ["--fast"] @? "map failed" - doesFileExist "map.dot" @? "map.dot not generated" - c <- readFile "map.dot" - ("this repo" `isInfixOf` c && "origin repo" `isInfixOf` c) @? ("map.dot bad content: " ++ c) test_uninit :: Test test_uninit = "git-annex uninit" ~: intmpclonerepo $ do From 1d5582091e9df550a8b42d0a69bada1d15a1825e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 13:37:55 -0400 Subject: [PATCH 4117/8313] attempt at building with -threaded Added a modified System.Cmd.Utils, working around bug #681621 Unfortunatly, the test suite still hangs partway through. Some of the hangs occur within pOpen3 still. Some of the hangs do not seem to occur within System.Cmd.Utils at all, but in some other code. --- Makefile | 2 +- System/Cmd/.Utils.hs.swp | Bin 0 -> 36864 bytes System/Cmd/Utils.hs | 568 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 System/Cmd/.Utils.hs.swp create mode 100644 System/Cmd/Utils.hs diff --git a/Makefile b/Makefile index 4d56287468..0afb10a7bb 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ endif PREFIX=/usr IGNORE=-ignore-package monads-fd -ignore-package monads-tf -BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 $(BASEFLAGS_OPTS) +BASEFLAGS=-threaded -Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 $(BASEFLAGS_OPTS) GHCFLAGS=-O2 $(BASEFLAGS) CFLAGS=-Wall diff --git a/System/Cmd/.Utils.hs.swp b/System/Cmd/.Utils.hs.swp new file mode 100644 index 0000000000000000000000000000000000000000..65e9e77e4437e33e567a27e7f34f6b8301ca5f47 GIT binary patch literal 36864 zcmeI53zTFnT26r%!i0tpEy7@r{c2>Jf1y0>pX zW_A{3A=m@o_TyI7ty_=3>i_Gn>e;ntX11wQ(imGc8bJ`BaIp6md zgV%~hzI!I@f!jeoKMQukC?DkW)8SBful@Xoyq|$u0=*LGl|ZiqdL_^+fnEvpN}yK)y%Ok^K(7RPC2){Rz|A|(iQMad z(9320Uu*#1dA8%+1Yd-|g1>}Uz%Pz;oPU5T;B7Dke+;)Cfu0W^h8*<6orhy9fZO0( za2>o3rr}laLU7=7f8;nfz+Cyc-vNW>EdS+E9U-s%=Ud=A5lhY@0Dd$)BdL&l&)qLRAYF-eg9cRAi1)ifi{|1~@UNv7TL@%8o zl&VeEyy~EN*$}->yYtSfQlXmn3TpT6tvkkNcJG!i)bT04DpkFFs#2=X&2L!0d+X#( zcK77WfV%i%)qX8IJUu?zadr3ZZJF)kJBhn{V4z#J)l{kGZS(V9DxDq}>>7gNI#Esd z^FtHfUazdOzF!Wbr?k)9?O}_O^67rP*gaK!IqNTTaV+pF-7iayPL!NeE^5A1FRG$j z&6mBfYyNA?=0mP7l?}NzK@EgUXGMZ6eYG>AW%t5G71jfC-aWSkKiET)Fip7oh2h{qY+s>U@Roa+DGG3_u07KND}KIF_7YtK zN>$vw+`I&og4XHg-MULwr~psp+;Um&*PpD8eo{C!ijpmQTO`5)UnK*nS}7mbD}_Os z&4ruv4J1({7x$bC+zNdmto9?fya2i3icDMd>ZFz;g+aA2pk6UVVy4-uzH->7&hA^| z)pLF4I?h(4o8m#;s`3`xN{vF(1xwilzH~NsQ3PV%SEc$!#}V1+mg*ENrJZvdp{Ex8 zMi6I(>moVbKqApc=h9B=ldDD*uQ=rv$gScrlrF3SJzqcBjP5A%L=+&Eg-i|`6)&Gw zYRoN_8;Ig*ui{l^yeqG6NHw&V z`MNIOt>?RRI<0Rq7kD-0H|n(pwI8Z{DXf*B^;*d7)|Wv~3EcK{=P| zB`>7xMoraHfqizUobcz;Fy{q*X=ldssEg3wsQNoDryDh@$Qv)y5y~b;|Dcpxx4#q_ zGK0p*bS9e_8J-v*WM>L0$_690>F(*K*yv*tr0-TX92%|& zr=7#o+ZoACG%}&we4gs`tImjDsUSamw2v%~+o783dzhwi6})ffxovDax=Fq*S5K(Y8{Y18(Z?NUJ>QHCEKU!j7^ks zUX`-qZ=)LDzC(>qO$;h#i;}QfuV^OjtiM;!6G!~oV!*Ifm$1zbsgVJ-e(lYm!L zVeKHTF|7Mk$hKjn)~;W-Zr#wj4Qo$PYRAm53V#Y7j=V{z}@C%E6qnt-j@AZ(gIj^)=It22%riW6Aq|3>pq^X;!#%#HiQ??u( zXX$IY&*{-N?@>`WP&?a3r$@F9Z_f^&o|(vGcS&iFWwP5xXJ*vc7AME*2x`NHN1V7bgXF-+qFpxO9Z*1V^(#a6B$#IuH*)R zTdm8JI9ZuvL?f+{_l!{_N;rC@JZRCc$*t&D(a1<|W=twH%1Fu--|I2b*HU5tnX!l( zBIBQJKxwC=j8(IyHz|)=YKE5PCQ~0TD$PL(#Np0b*ysx~69}VF%M>*NtR@u`7I~Q2 z#g1n!NVSKy+Ba#DDzZ5681!r*$Ri_+)=(!*#7YJH>sQj|m+Wm94bj?Hsfpq3<2#1O zN7cyGlv>4<4`J*Ex#CdWos(W=Bw#E)RD~|B3Z=5=i2i>Gdh5H;ZAJed=H<)i`fq{@ zAPc9!bK#puI?gBIeQ*_s4dAWta!A2L==~4E9dJ9m4HjS?)`J7TLFd070vLjypyOW; z&xb?cJLveIfY(C?j)uF?>AwnBLm5s5(e=NN9{=C)b$ADy3n#!Ka4&lNJ#ZU*5k3QN zgbQFB90|YTnSTjCg9qS#xDli*u7ciQuLOD}&?|vn2^{h>qfo2qrc zX2Ner|3-sbE{L6dPl?SfJ)>2NFrAsJV$C_=XlN!=nW@q4@w(nf>`9nKjDakOBL<#Z zHcv;xp_rH^scwH(8@7_Cu=999fHgC(o6Y`1 z%j}xaliFIc|KQBw2%V{7Lx}@7+lhH{wZcE!K2y!swZDkDvLAY5^iWJMh0d7>lT-VQmQ&-5ZmOC_4HEkyTTgnHWM+Xp3Ti%ll%0 zta_*sp(3%afu)R~Ls|I)G( zF=KJL|Cr{v5^=Xl1t@C2>z}^sIQ2yHlpbL1m8gTw>ZZ6?o`}?6OCrtsMeeU4of+1i zBsbIy&!S72tRNWYQCxMkNq_ZnI?Hl5u^5@$4=S3h=xv2k5Z2QU4bG~sQkAqsZS{$4 zsv*WOo>F1w&{{cqZj}!hjoIdmnr%#NtaFBH*W`|Anrv9ZXj?L7oZ~UqiV-$cr3z-; z5^672JZ+X24R$KCB~nv$`y#y^+HTb^R%Y9D`{mxUj7LV5yO(iGn}ztYY1tPGiJES3 zAF)Ab?QoE+m?Dsts+%O{d2C5o!Z9jztf4$UD{yUNv~69sGX#617HF)-6II4MJ3g`y zsP*!`C{%~ceI(e(n66nn9=FbEICQw%$P_QfJlBu%gvxsr%sB1Si9=3|b+ zHrL`D*n2RV>!E1%@p8NFF}Yx(v2b7~2F(_}9|`7|ki_dI==7}<(oQ923NlR){r~H% zAE#M&7X4p*_8&plza4IYcfvd1EwB$RhgZUj;0|>DYvD5R!G+hrVekmL{td7XM&Sr} z0Db@4@Kv}G{t-S0IS{{qemDUh!Vd5td;rdZov;~R4u``JunF7@pM&?nKG*}N!lT#) zZh%W+2ERs%XKxLRub*^d7h#EpH4EKIx_&<@j6~)XHw-niuv1JYwSCZ&x38RJDDT`)VUnXZFnUa^wTH!MFJJ1?e2U_LoG1R&0sPvZNa+<}n{JXf; zTgq$c=*w37G#)@>SPi<=yD^7VTI^xAFKcTVhMl@6@#0(4Xmri0S|y&z>wZB~i;ZGQ zgP2O$_8*Uh_Ki&kSuZybpFe4ov6jsHcnlmm)ln#q%g9$eJ3{d_akW@6%B$8t2eoS1 zpJ@91qWO0Vj2@b1krWPW zKfVlZ7?OnOl<22;28zz86=N1K?H#b}SO>4PqnFf`cxOfr)DhUKkx!(iSn`c|)-;Y- zH^TxUz7WQWDGk#@456O&*&dPiwspGr@sTf0= zQSfXKUH_x-TDTY{;Fa(TGW-enI9v&Dg;_Wao)6E4AEWQz2{*#u!VWkRZb$F`7&M?C z{tI3IYw$jJ4Ll$2MZdouE`?LzA@utn!F_NqTmhHBcGw0RU_Ja0{r~fD12kYQ+>gKh zC!qmr;Y#c-mjXRR{-VZUKG}~wtuhj%PdV##Ppf>Q?aH>Nb-Z-ZGFs-ntuuvW^qyAP z(<}i#v8TPbFy@_)r8#Nt~B0a6Li?7?`p;d|s8E>bI zo%^+)lr^%ogy?Jew1V&~L$^jje_q1u*6O0eHXS?|!j8?5>DJ`1)L}*w?-Og#!qSZO zt7@FvjC+N#zR2b;Z9ZdjjOS0ro5Sde7aQ7x`$g<)+?X%R!*{Zn(N=7wDp|WuFAbJ^mN> zAl#SOghS58utmt0J~7*I-$q%!oVE@lZT=x)W7g(C+=sA}bayz>`oAofeic1b^#2Mk ze}~S0CCtOw@KQ*_bKnqo0G0iFh5 z!8Y(-coT%M4UU03umyYpu7X)O8?rD1zriN(0DKre1ac0*QScLN13!ju!>w=?Tnf8k z3k<;ta2&iCr0$*ocO0`{HWw>HIkC_ear+watL${cq-~a_=rty2QE%7O+la;dADQc& z8s_%o7x?7GUo9^R5`H~w<Xmb@!B=ar9!-ixh&u3)ubWvHiBi&D8eG zbbHUm$0}&sqpU6Hk*(gWFtBfVa{I(CH9VFboz}+oux-CC%NcpG^6G7WvVU`zxRiiqn zlUmv8)q$!J%pJNqavMKxJ-3kgMf0!e2F_Nus)%76&upV*oxr-NuXdU-qIb)9fLlK= zv*MLl@IqX-I$zYJU#-X?Ji2S*HEA_oxreh!ATHOgYleD>FZN2;s}kZVnWeY47)!*B zaR3VS2){5<)t*_P4a#c3bBSY!=oN zj;Qj}?7i+7)XgqOy=E76J*D;|cJxsTxCOH#ReP0+2(atJ%|$x1bjW8pmBEYlIkp`b zz~)A?TDO0BVRH9`ii$(+!g=5v8n4%_R-sua!)Le8IaD_jx7&4G|ATs;BW*?;S+#G0 zlW6ler)bxkBt{CrQ+wWw(jl;vuVtao3NR=c0gIWyp%J(;&|t zDY}8)d$QC#wiyC{ESz;|S@`2t#rIayZ?^QvwSa|yp`pAt+nC#?QiPtBjU$r9v|jv@ zNCQTNOl+GONTy4^_P3o)EiEtu?4IvY~Br{K|}j7;Tw$We<&bI6#+aMybXfty`!!3(1PCha~v}}Uw znv8M62)kahmDTlHX7X{;@MQge9XkI{tp2};_di0%{~){%u7N2y3=W0+(d&Nzx4^%^ z+u;%r-Cy+nqv0;}{SU!Ka6U}KOW>#I`r_|@H8kLK*aRoR1`t1he+Kadkn;d91M&Hn zvjP4T9!1apHQWRrgUcZYa!$Z0uoea&1>Z#fzY$&!TVWg?M#uj&dM^ue>> z>F_1={lAARL3{&tf!F|kh@SshxE#)a{r# zYeDu2eD@Ii6<{A+4kyD);d$^)^m;i5@C|S_oD9c;ocZ?%b@fw_I{UHI+X4OABkAmm zmCJT$-@i&PGGS?t(^NEP^_x1J*y5N~tY-g|?x`wOoHB<#)%{|0tdALd;z@+Q3&)C) zj5aEWdz#5?RgA3lA{ojbG%vAb8)dV8D2q0efz&i6gG7UgrX13`n28qJ9G>gG zb_|a~>4@#&MBkMQEg@U9;kH!bYohsf#Ej=YEpHCY1fiC)LFfXAC)C|+5bCL6gpi_b zEB2Q1v>&VqLaNY)76U}o9qc24*J3(v}^2|0r6Qt4%wI zso$m@M2oga^R#vl{oOL+u3gBP`SYNhk%;f_vi1;U)6ymoxi10u*O*V)DvQu4`?)=O zVr^zg7>F_0W_^qA!`zRuebUWU|p*=yH1T8REXg8q1&=#i}d@8m_+|IdS?LG=88 z1<~{6-2XaE!Ex|Qbo!e>&i&sFvd;f=^!S_L-LM@}a2LA!ZSY-?_5E?^gX7^wboGCN zx5EOQ2M?jEi~jy4kTd_zgemBU=fd~V(XWOx;W=;w91fpFNB^+Z*N@@#QFQbFh3nuN zcrRQ4C&TmLm+0s>!<%6zybN9hvY-DpxE3yiSHsKTKJ@ZiMK=doudl)>@C>*Gz5HDu z=k;Fz=R*!=;c)mByi+JCu9fATpc^yT@+fYVO#IjCvTGU5@dMTtGOUlru3! zNtJU#L`Ov_?`#VP#fKmg>rE-j?l)r=+vbLQl+Q6&9LL?fY~Qt>^d#G{UbQ2WmEad} zgsVbd?{ZQV28GUene93nlhQ7+n6qNu_N+UQBroBjyO_*jWa!9Va-Frd?CzXqRC2eH z+v%I>Jw4`BA-f}JsjNkLvnCy0u;xZzHgB%q*-{hph~5RK`l4@4Y}xrF^VW9R7@u*{ zmYzPgNuPHXZ+w<<#N>e`Hoe$gsmWohQI1wz0^CYsqDi&q#+=(V#*y6E=4C}%=A%*i z(H9a?Oi^~eq*BS}>QznSC;^^k`@FoHe&xzC~gnTu! zbz+O+B;xv_`EI7!=1EgGX%g*hyt>hw67Ph3CxVkGmG}dTU~1MURR~QDiwh0T2=OR?C8ksv^|1Jls@U6*x8ukh#8Od zmF=8;9GLUWwV}+5mLa${Sw*Lf&s#FX_JK<2rgTo+{@^VPw%Ua0aB?`f&kq#H0y{FI zWz%V2;t2Y9HAy>##)pjQ6td0VS$OP;$Pd$yisiP(sZXTMXC5iM2Prrh!( zo+{cgCNd+gX2I6>bAFzH(McAFwOfgHiQ@}hlgUK%NS}&g5kmT6M<|KW?QRK~a9}q~ zL|pEk2r4TRI-GUdM1kbcJc|_y?INb@b9KhBopV5oqz*|UchP^*^a^p)tz5=g`>*1l zWDm77<4*fzOHZQg%2IP4C5@j6rTyMaNLohT$7&F&ZQR@F zt(^!A{Y3zKn_^6Ghh2FCu!^Jq3j)OO=PyvRc$AaYA3#Jo;oxA83TA? kXMxq-cZm2{p|^=Vjz23t9QrXmIpM%msg~Kl`=31fKh*qing9R* literal 0 HcmV?d00001 diff --git a/System/Cmd/Utils.hs b/System/Cmd/Utils.hs new file mode 100644 index 0000000000..23c2bcedfd --- /dev/null +++ b/System/Cmd/Utils.hs @@ -0,0 +1,568 @@ +-- arch-tag: Command utilities main file +{-# LANGUAGE CPP #-} +{- +Copyright (C) 2004-2006 John Goerzen + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +-} + +{- | + Module : System.Cmd.Utils + Copyright : Copyright (C) 2004-2006 John Goerzen + License : GNU GPL, version 2 or above + + Maintainer : John Goerzen + Stability : provisional + Portability: portable to platforms with POSIX process\/signal tools + +Command invocation utilities. + +Written by John Goerzen, jgoerzen\@complete.org + +Please note: Most of this module is not compatible with Hugs. + +Command lines executed will be logged using "System.Log.Logger" at the +DEBUG level. Failure messages will be logged at the WARNING level in addition +to being raised as an exception. Both are logged under +\"System.Cmd.Utils.funcname\" -- for instance, +\"System.Cmd.Utils.safeSystem\". If you wish to suppress these messages +globally, you can simply run: + +> updateGlobalLogger "System.Cmd.Utils.safeSystem" +> (setLevel CRITICAL) + +See also: 'System.Log.Logger.updateGlobalLogger', +"System.Log.Logger". + +It is possible to set up pipelines with these utilities. Example: + +> (pid1, x1) <- pipeFrom "ls" ["/etc"] +> (pid2, x2) <- pipeBoth "grep" ["x"] x1 +> putStr x2 +> ... the grep output is displayed ... +> forceSuccess pid2 +> forceSuccess pid1 + +Remember, when you use the functions that return a String, you must not call +'forceSuccess' until after all data from the String has been consumed. Failure +to wait will cause your program to appear to hang. + +Here is an example of the wrong way to do it: + +> (pid, x) <- pipeFrom "ls" ["/etc"] +> forceSuccess pid -- Hangs; the called program hasn't terminated yet +> processTheData x + +You must instead process the data before calling 'forceSuccess'. + +When using the hPipe family of functions, this is probably more obvious. + +Most of this module will be incompatible with Windows. +-} + + +module System.Cmd.Utils(-- * High-Level Tools + PipeHandle(..), + safeSystem, +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) + forceSuccess, +#ifndef __HUGS__ + posixRawSystem, + forkRawSystem, + -- ** Piping with lazy strings + pipeFrom, + pipeLinesFrom, + pipeTo, + pipeBoth, + -- ** Piping with handles + hPipeFrom, + hPipeTo, + hPipeBoth, +#endif +#endif + -- * Low-Level Tools + PipeMode(..), +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ + pOpen, pOpen3, pOpen3Raw +#endif +#endif + ) +where + +-- FIXME - largely obsoleted by 6.4 - convert to wrappers. + +import System.Exit +import System.Cmd +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +import System.Posix.IO +import System.Posix.Process +import System.Posix.Signals +import qualified System.Posix.Signals +#endif +import System.Posix.Types +import System.IO +import System.IO.Error +import Control.Concurrent(forkIO) +import Control.Exception(finally) + +data PipeMode = ReadFromPipe | WriteToPipe + +logbase :: String +logbase = "System.Cmd.Utils" + +{- | Return value from 'pipeFrom', 'pipeLinesFrom', 'pipeTo', or +'pipeBoth'. Contains both a ProcessID and the original command that was +executed. If you prefer not to use 'forceSuccess' on the result of one +of these pipe calls, you can use (processID ph), assuming ph is your 'PipeHandle', +as a parameter to 'System.Posix.Process.getProcessStatus'. -} +data PipeHandle = + PipeHandle { processID :: ProcessID, + phCommand :: FilePath, + phArgs :: [String], + phCreator :: String -- ^ Function that created it + } + deriving (Eq, Show) + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Like 'pipeFrom', but returns data in lines instead of just a String. +Shortcut for calling lines on the result from 'pipeFrom'. + +Note: this function logs as pipeFrom. + +Not available on Windows. -} +pipeLinesFrom :: FilePath -> [String] -> IO (PipeHandle, [String]) +pipeLinesFrom fp args = + do (pid, c) <- pipeFrom fp args + return $ (pid, lines c) +#endif +#endif + +logRunning :: String -> FilePath -> [String] -> IO () +logRunning func fp args = return () --debugM (logbase ++ "." ++ func) (showCmd fp args) + +warnFail :: [Char] -> FilePath -> [String] -> [Char] -> IO t +warnFail funcname fp args msg = + let m = showCmd fp args ++ ": " ++ msg + in do putStrLn m + fail m + +ddd s a = do + putStrLn $ s ++ " start" + r <- a + putStrLn $ s ++ " end" + return r + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Read data from a pipe. Returns a Handle and a 'PipeHandle'. + +When done, you must hClose the handle, and then use either 'forceSuccess' or +getProcessStatus on the 'PipeHandle'. Zombies will result otherwise. + +This function logs as pipeFrom. + +Not available on Windows or with Hugs. +-} +hPipeFrom :: FilePath -> [String] -> IO (PipeHandle, Handle) +hPipeFrom fp args = + ddd "hPipeFrom" $ do + pipepair <- createPipe + let childstuff = do dupTo (snd pipepair) stdOutput + closeFd (fst pipepair) + executeFile fp True args Nothing + p <- try (forkProcess childstuff) + -- parent + pid <- case p of + Right x -> return x + Left e -> warnFail "pipeFrom" fp args $ + "Error in fork: " ++ show e + closeFd (snd pipepair) + h <- fdToHandle (fst pipepair) + return (PipeHandle pid fp args "pipeFrom", h) +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Read data from a pipe. Returns a lazy string and a 'PipeHandle'. + +ONLY AFTER the string has been read completely, You must call either +'System.Posix.Process.getProcessStatus' or 'forceSuccess' on the 'PipeHandle'. +Zombies will result otherwise. + +Not available on Windows. +-} +pipeFrom :: FilePath -> [String] -> IO (PipeHandle, String) +pipeFrom fp args = + do (pid, h) <- hPipeFrom fp args + c <- hGetContents h + return (pid, c) +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Write data to a pipe. Returns a 'PipeHandle' and a new Handle to write +to. + +When done, you must hClose the handle, and then use either 'forceSuccess' or +getProcessStatus on the 'PipeHandle'. Zombies will result otherwise. + +This function logs as pipeTo. + +Not available on Windows. +-} +hPipeTo :: FilePath -> [String] -> IO (PipeHandle, Handle) +hPipeTo fp args = + ddd "hPipeTo" $ do + pipepair <- createPipe + let childstuff = do dupTo (fst pipepair) stdInput + closeFd (snd pipepair) + executeFile fp True args Nothing + p <- try (forkProcess childstuff) + -- parent + pid <- case p of + Right x -> return x + Left e -> warnFail "pipeTo" fp args $ + "Error in fork: " ++ show e + closeFd (fst pipepair) + h <- fdToHandle (snd pipepair) + return (PipeHandle pid fp args "pipeTo", h) +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Write data to a pipe. Returns a ProcessID. + +You must call either +'System.Posix.Process.getProcessStatus' or 'forceSuccess' on the ProcessID. +Zombies will result otherwise. + +Not available on Windows. +-} +pipeTo :: FilePath -> [String] -> String -> IO PipeHandle +pipeTo fp args message = + do (pid, h) <- hPipeTo fp args + finally (hPutStr h message) + (hClose h) + return pid +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Like a combination of 'hPipeTo' and 'hPipeFrom'; returns +a 3-tuple of ('PipeHandle', Data From Pipe, Data To Pipe). + +When done, you must hClose both handles, and then use either 'forceSuccess' or +getProcessStatus on the 'PipeHandle'. Zombies will result otherwise. + +Hint: you will usually need to ForkIO a thread to handle one of the Handles; +otherwise, deadlock can result. + +This function logs as pipeBoth. + +Not available on Windows. +-} +hPipeBoth :: FilePath -> [String] -> IO (PipeHandle, Handle, Handle) +hPipeBoth fp args = + ddd "hPipeBoth" $ do + frompair <- createPipe + topair <- createPipe + let childstuff = do dupTo (snd frompair) stdOutput + closeFd (fst frompair) + dupTo (fst topair) stdInput + closeFd (snd topair) + executeFile fp True args Nothing + p <- try (forkProcess childstuff) + -- parent + pid <- case p of + Right x -> return x + Left e -> warnFail "pipeBoth" fp args $ + "Error in fork: " ++ show e + closeFd (snd frompair) + closeFd (fst topair) + fromh <- fdToHandle (fst frompair) + toh <- fdToHandle (snd topair) + return (PipeHandle pid fp args "pipeBoth", fromh, toh) +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Like a combination of 'pipeTo' and 'pipeFrom'; forks an IO thread +to send data to the piped program, and simultaneously returns its output +stream. + +The same note about checking the return status applies here as with 'pipeFrom'. + +Not available on Windows. -} +pipeBoth :: FilePath -> [String] -> String -> IO (PipeHandle, String) +pipeBoth fp args message = + do (pid, fromh, toh) <- hPipeBoth fp args + forkIO $ finally (hPutStr toh message) + (hClose toh) + c <- hGetContents fromh + return (pid, c) +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +{- | Uses 'System.Posix.Process.getProcessStatus' to obtain the exit status +of the given process ID. If the process terminated normally, does nothing. +Otherwise, raises an exception with an appropriate error message. + +This call will block waiting for the given pid to terminate. + +Not available on Windows. -} +forceSuccess :: PipeHandle -> IO () +forceSuccess (PipeHandle pid fp args funcname) = + let warnfail = warnFail funcname + in do status <- getProcessStatus True False pid + case status of + Nothing -> warnfail fp args $ "Got no process status" + Just (Exited (ExitSuccess)) -> return () + Just (Exited (ExitFailure fc)) -> + cmdfailed funcname fp args fc + Just (Terminated sig) -> + warnfail fp args $ "Terminated by signal " ++ show sig + Just (Stopped sig) -> + warnfail fp args $ "Stopped by signal " ++ show sig +#endif + +{- | Invokes the specified command in a subprocess, waiting for the result. +If the command terminated successfully, return normally. Otherwise, +raises a userError with the problem. + +Implemented in terms of 'posixRawSystem' where supported, and System.Posix.rawSystem otherwise. +-} +safeSystem :: FilePath -> [String] -> IO () +safeSystem command args = + ddd "safeSystem" $ do +#if defined(__HUGS__) || defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__) + ec <- rawSystem command args + case ec of + ExitSuccess -> return () + ExitFailure fc -> cmdfailed "safeSystem" command args fc +#else + ec <- posixRawSystem command args + case ec of + Exited ExitSuccess -> return () + Exited (ExitFailure fc) -> cmdfailed "safeSystem" command args fc + Terminated s -> cmdsignalled "safeSystem" command args s + Stopped s -> cmdsignalled "safeSystem" command args s +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Invokes the specified command in a subprocess, waiting for the result. +Return the result status. Never raises an exception. Only available +on POSIX platforms. + +Like system(3), this command ignores SIGINT and SIGQUIT and blocks SIGCHLD +during its execution. + +Logs as System.Cmd.Utils.posixRawSystem -} +posixRawSystem :: FilePath -> [String] -> IO ProcessStatus +posixRawSystem program args = + ddd "posixRawSystem" $ do + oldint <- installHandler sigINT Ignore Nothing + oldquit <- installHandler sigQUIT Ignore Nothing + let sigset = addSignal sigCHLD emptySignalSet + oldset <- getSignalMask + blockSignals sigset + childpid <- forkProcess (childaction oldint oldquit oldset) + + mps <- getProcessStatus True False childpid + restoresignals oldint oldquit oldset + let retval = case mps of + Just x -> x + Nothing -> error "Nothing returned from getProcessStatus" + return retval + + where childaction oldint oldquit oldset = + do restoresignals oldint oldquit oldset + executeFile program True args Nothing + restoresignals oldint oldquit oldset = + do installHandler sigINT oldint Nothing + installHandler sigQUIT oldquit Nothing + setSignalMask oldset + +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Invokes the specified command in a subprocess, without waiting for +the result. Returns the PID of the subprocess -- it is YOUR responsibility +to use getProcessStatus or getAnyProcessStatus on that at some point. Failure +to do so will lead to resource leakage (zombie processes). + +This function does nothing with signals. That too is up to you. + +Logs as System.Cmd.Utils.forkRawSystem -} +forkRawSystem :: FilePath -> [String] -> IO ProcessID +forkRawSystem program args = ddd "forkRawSystem" $ + do + forkProcess childaction + where + childaction = executeFile program True args Nothing + +#endif +#endif + +cmdfailed :: String -> FilePath -> [String] -> Int -> IO a +cmdfailed funcname command args failcode = do + let errormsg = "Command " ++ command ++ " " ++ (show args) ++ + " failed; exit code " ++ (show failcode) + let e = userError (errormsg) + putStrLn errormsg + ioError e + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +cmdsignalled :: String -> FilePath -> [String] -> Signal -> IO a +cmdsignalled funcname command args failcode = do + let errormsg = "Command " ++ command ++ " " ++ (show args) ++ + " failed due to signal " ++ (show failcode) + let e = userError (errormsg) + putStrLn errormsg + ioError e +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Open a pipe to the specified command. + +Passes the handle on to the specified function. + +The 'PipeMode' specifies what you will be doing. That is, specifing 'ReadFromPipe' +sets up a pipe from stdin, and 'WriteToPipe' sets up a pipe from stdout. + +Not available on Windows. + -} +pOpen :: PipeMode -> FilePath -> [String] -> + (Handle -> IO a) -> IO a +pOpen pm fp args func = ddd "pOpen" $ + do + pipepair <- createPipe + case pm of + ReadFromPipe -> do + let callfunc _ = do + closeFd (snd pipepair) + h <- fdToHandle (fst pipepair) + x <- func h + hClose h + return $! x + pOpen3 Nothing (Just (snd pipepair)) Nothing fp args + callfunc (closeFd (fst pipepair)) + WriteToPipe -> do + let callfunc _ = do + closeFd (fst pipepair) + h <- fdToHandle (snd pipepair) + x <- func h + hClose h + return $! x + pOpen3 (Just (fst pipepair)) Nothing Nothing fp args + callfunc (closeFd (snd pipepair)) +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Runs a command, redirecting things to pipes. + +Not available on Windows. + +Note that you may not use the same fd on more than one item. If you +want to redirect stdout and stderr, dup it first. +-} +pOpen3 :: Maybe Fd -- ^ Send stdin to this fd + -> Maybe Fd -- ^ Get stdout from this fd + -> Maybe Fd -- ^ Get stderr from this fd + -> FilePath -- ^ Command to run + -> [String] -- ^ Command args + -> (ProcessID -> IO a) -- ^ Action to run in parent + -> IO () -- ^ Action to run in child before execing (if you don't need something, set this to @return ()@) -- IGNORED IN HUGS + -> IO a +pOpen3 pin pout perr fp args func childfunc = ddd "pOpen3" $ + do pid <- pOpen3Raw pin pout perr fp args childfunc + putStrLn "got pid" + retval <- func $! pid + putStrLn "got retval" + let rv = seq retval retval + forceSuccess (PipeHandle (seq retval pid) fp args "pOpen3") + putStrLn "process finished" + return rv +#endif +#endif + +#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) +#ifndef __HUGS__ +{- | Runs a command, redirecting things to pipes. + +Not available on Windows. + +Returns immediately with the PID of the child. Using 'waitProcess' on it +is YOUR responsibility! + +Note that you may not use the same fd on more than one item. If you +want to redirect stdout and stderr, dup it first. +-} +pOpen3Raw :: Maybe Fd -- ^ Send stdin to this fd + -> Maybe Fd -- ^ Get stdout from this fd + -> Maybe Fd -- ^ Get stderr from this fd + -> FilePath -- ^ Command to run + -> [String] -- ^ Command args + -> IO () -- ^ Action to run in child before execing (if you don't need something, set this to @return ()@) -- IGNORED IN HUGS + -> IO ProcessID +pOpen3Raw pin pout perr fp args childfunc = + let mayberedir Nothing _ = return () + mayberedir (Just fromfd) tofd = do + dupTo fromfd tofd + closeFd fromfd + return () + childstuff = do + mayberedir pin stdInput + mayberedir pout stdOutput + mayberedir perr stdError + childfunc + executeFile fp True args Nothing +{- + realfunc p = do + System.Posix.Signals.installHandler + System.Posix.Signals.sigPIPE + System.Posix.Signals.Ignore + Nothing + func p +-} + in + ddd "pOpen3Raw" $ + do + p <- try (forkProcess childstuff) + pid <- case p of + Right x -> return x + Left e -> fail ("Error in fork: " ++ (show e)) + return pid + +#endif +#endif + +showCmd :: FilePath -> [String] -> String +showCmd fp args = fp ++ " " ++ show args From 182526ff68b1ca68952b4dbd32121e46d4a80e85 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 14:40:05 -0400 Subject: [PATCH 4118/8313] add debugging --- Git/Branch.hs | 10 +++++----- Git/Command.hs | 12 +++++++----- Git/HashObject.hs | 8 +++----- Git/Ref.hs | 5 ++++- System/Cmd/Utils.hs | 4 ++-- Utility/Misc.hs | 2 +- Utility/SafeCommand.hs | 4 ++-- git-annex.cabal | 2 +- 8 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Git/Branch.hs b/Git/Branch.hs index 6edc1c306d..6f3d251863 100644 --- a/Git/Branch.hs +++ b/Git/Branch.hs @@ -73,12 +73,12 @@ commit :: String -> Branch -> [Ref] -> Repo -> IO Sha commit message branch parentrefs repo = do tree <- getSha "write-tree" $ pipeRead [Param "write-tree"] repo - sha <- getSha "commit-tree" $ - ignorehandle $ pipeWriteRead - (map Param $ ["commit-tree", show tree] ++ ps) - message repo + sha <- getSha "commit-tree" $ pipeWriteRead + (map Param $ ["commit-tree", show tree] ++ ps) + message repo + print ("got", sha) run "update-ref" [Param $ show branch, Param $ show sha] repo + print ("update-ref done", sha) return sha where - ignorehandle a = snd <$> a ps = concatMap (\r -> ["-p", show r]) parentrefs diff --git a/Git/Command.hs b/Git/Command.hs index 35f0838ba9..9a09300e24 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -57,16 +57,18 @@ pipeWrite params s repo = assertLocal repo $ do hClose h return p -{- Runs a git subcommand, feeding it input, and returning its output. - - You should call either getProcessStatus or forceSuccess on the PipeHandle. -} -pipeWriteRead :: [CommandParam] -> String -> Repo -> IO (PipeHandle, String) +{- Runs a git subcommand, feeding it input, and returning its output, + - which is expected to be fairly small, since it's all read into memory + - strictly. -} +pipeWriteRead :: [CommandParam] -> String -> Repo -> IO String pipeWriteRead params s repo = assertLocal repo $ do (p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo) fileEncoding to fileEncoding from _ <- forkIO $ finally (hPutStr to s) (hClose to) - c <- hGetContents from - return (p, c) + c <- hGetContentsStrict from + forceSuccess p + return c {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it. -} diff --git a/Git/HashObject.hs b/Git/HashObject.hs index 9f37de5ba9..c90c9ec3d6 100644 --- a/Git/HashObject.hs +++ b/Git/HashObject.hs @@ -38,11 +38,9 @@ hashFile h file = CoProcess.query h send receive {- Injects some content into git, returning its Sha. -} hashObject :: ObjectType -> String -> Repo -> IO Sha hashObject objtype content repo = getSha subcmd $ do - (h, s) <- pipeWriteRead (map Param params) content repo - length s `seq` do - forceSuccess h - reap -- XXX unsure why this is needed - return s + s <- pipeWriteRead (map Param params) content repo + reap -- XXX unsure why this is needed, of if it is anymore + return s where subcmd = "hash-object" params = [subcmd, "-t", show objtype, "-w", "--stdin"] diff --git a/Git/Ref.hs b/Git/Ref.hs index ee2f021871..3052d0a6ef 100644 --- a/Git/Ref.hs +++ b/Git/Ref.hs @@ -40,7 +40,10 @@ exists ref = runBool "show-ref" {- Get the sha of a fully qualified git ref, if it exists. -} sha :: Branch -> Repo -> IO (Maybe Sha) -sha branch repo = process <$> showref repo +sha branch repo = do + r <- process <$> showref repo + print r + return r where showref = pipeRead [Param "show-ref", Param "--hash", -- get the hash diff --git a/System/Cmd/Utils.hs b/System/Cmd/Utils.hs index 23c2bcedfd..15544d6846 100644 --- a/System/Cmd/Utils.hs +++ b/System/Cmd/Utils.hs @@ -179,7 +179,7 @@ Not available on Windows or with Hugs. -} hPipeFrom :: FilePath -> [String] -> IO (PipeHandle, Handle) hPipeFrom fp args = - ddd "hPipeFrom" $ do + ddd (show ("hPipeFrom", fp, args)) $ do pipepair <- createPipe let childstuff = do dupTo (snd pipepair) stdOutput closeFd (fst pipepair) @@ -281,7 +281,7 @@ Not available on Windows. -} hPipeBoth :: FilePath -> [String] -> IO (PipeHandle, Handle, Handle) hPipeBoth fp args = - ddd "hPipeBoth" $ do + ddd (show ("hPipeBoth", fp, args)) $ do frompair <- createPipe topair <- createPipe let childstuff = do dupTo (snd frompair) stdOutput diff --git a/Utility/Misc.hs b/Utility/Misc.hs index 3b359139b9..e11586467d 100644 --- a/Utility/Misc.hs +++ b/Utility/Misc.hs @@ -33,7 +33,7 @@ separate c l = unbreak $ break c l | otherwise = (a, tail b) {- Breaks out the first line. -} -firstLine :: String-> String +firstLine :: String -> String firstLine = takeWhile (/= '\n') {- Splits a list into segments that are delimited by items matching diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs index aedf271373..2c6439b452 100644 --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -78,8 +78,8 @@ safeSystemEnv command params env = do {- executeFile with debug logging -} executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO () executeFile c path p e = do - debugM "Utility.SafeCommand.executeFile" $ - "Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e + --debugM "Utility.SafeCommand.executeFile" $ + -- "Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e System.Posix.Process.executeFile c path p e {- Escapes a filename or other parameter to be safely able to be exposed to diff --git a/git-annex.cabal b/git-annex.cabal index 0bd35e14fe..3f237ce70e 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 3.20120629 +Version: 3.20120630 Cabal-Version: >= 1.8 License: GPL Maintainer: Joey Hess From 4db09814e4961fcb55843b9b51ad423560bb6e91 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 14:50:37 -0400 Subject: [PATCH 4119/8313] avoid --no-edit with older git versions --- Git/Merge.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Git/Merge.hs b/Git/Merge.hs index 08fc6fb487..3fda34e6cb 100644 --- a/Git/Merge.hs +++ b/Git/Merge.hs @@ -10,8 +10,10 @@ module Git.Merge where import Common import Git import Git.Command +import Git.Version {- Avoids recent git's interactive merge. -} mergeNonInteractive :: Ref -> Repo -> IO Bool -mergeNonInteractive branch = runBool "merge" - [Param "--no-edit", Param $ show branch] +mergeNonInteractive branch + | older 1.7.7.6 = runBool "merge" [Param $ show branch] + | otherwise = runBool "merge" [Param "--no-edit", Param $ show branch] From 0962d50ad234f444d4e5157457fbeeb45d858d96 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 14:51:42 -0400 Subject: [PATCH 4120/8313] typo --- Git/Merge.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Git/Merge.hs b/Git/Merge.hs index 3fda34e6cb..bad9f8258c 100644 --- a/Git/Merge.hs +++ b/Git/Merge.hs @@ -15,5 +15,5 @@ import Git.Version {- Avoids recent git's interactive merge. -} mergeNonInteractive :: Ref -> Repo -> IO Bool mergeNonInteractive branch - | older 1.7.7.6 = runBool "merge" [Param $ show branch] + | older "1.7.7.6" = runBool "merge" [Param $ show branch] | otherwise = runBool "merge" [Param "--no-edit", Param $ show branch] From e816776a6247daa2adfa73d9029c6f155ae46b4a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 15:57:49 -0400 Subject: [PATCH 4121/8313] add inodes to kqueue's directory cache This is necessary to generate events when a file is deleted and immediately replaced. Otherwise, the cache will have the old file, and so no event would be generated. Use of getFileStatus is suboptimal, it would be faster to use the inode returned by readdir -- but getDirectoryContents does not expose it, so I'd have to copy and modify a lot of low-level code. --- Utility/Kqueue.hs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs index 7e7e653ec3..17a5be5456 100644 --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -22,6 +22,7 @@ module Utility.Kqueue ( import Common import Utility.Types.DirWatcher +import System.Posix.Directory import System.Posix.Types import Foreign.C.Types import Foreign.C.Error @@ -62,15 +63,19 @@ type DirMap = M.Map Fd DirInfo {- A directory, and its last known contents (with filenames relative to it) -} data DirInfo = DirInfo { dirName :: FilePath - , dirCache :: S.Set FilePath + , dirCache :: S.Set (FilePath, FileID) } deriving (Show) getDirInfo :: FilePath -> IO DirInfo getDirInfo dir = do - contents <- S.fromList . filter (not . dirCruft) - <$> getDirectoryContents dir + l <- filter (not . dirCruft) <$> getDirectoryContents dir + contents <- S.fromList <$> mapM addinode l return $ DirInfo dir contents + where + addinode f = do + inode <- fileID <$> getFileStatus (dir f) + return (f, inode) {- Difference between the dirCaches of two DirInfos. -} (//) :: DirInfo -> DirInfo -> [Change] @@ -78,7 +83,7 @@ oldc // newc = deleted ++ added where deleted = calc Deleted oldc newc added = calc Added newc oldc - calc a x y = map a . map (dirName x ) $ + calc a x y = map a . map (dirName x ) . map fst $ S.toList $ S.difference (dirCache x) (dirCache y) {- Builds a map of directories in a tree, possibly pruning some. @@ -99,7 +104,7 @@ scanRecursive topdir prune = M.fromList <$> walk [] [topdir] case mfd of Nothing -> walk c rest Just fd -> do - let subdirs = map (dir ) $ + let subdirs = map (dir ) . map fst $ S.toList $ dirCache info walk ((fd, info):c) (subdirs ++ rest) @@ -123,7 +128,8 @@ removeSubDir dirmap dir = do findDirContents :: DirMap -> FilePath -> [FilePath] findDirContents dirmap dir = concatMap absolutecontents $ search where - absolutecontents i = map (dirName i ) (S.toList $ dirCache i) + absolutecontents i = map (dirName i ) + (map fst $ S.toList $ dirCache i) search = map snd $ M.toList $ M.filter (\i -> dirName i == dir) dirmap From 7d89cf0eb9aeffff25dc1b7db129c47c9f434078 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 16:02:29 -0400 Subject: [PATCH 4122/8313] cleanup --- Utility/Kqueue.hs | 1 - 1 file changed, 1 deletion(-) diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs index 17a5be5456..b475de3a3b 100644 --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -22,7 +22,6 @@ module Utility.Kqueue ( import Common import Utility.Types.DirWatcher -import System.Posix.Directory import System.Posix.Types import Foreign.C.Types import Foreign.C.Error From 30ac6d7be04d677cdc4a1da6d60622657665083f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 16:29:49 -0400 Subject: [PATCH 4123/8313] robustness fix Don't fall over symlinks, and avoid crashing if the file goes away. --- Utility/Kqueue.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs index b475de3a3b..62b06a5323 100644 --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -69,10 +69,10 @@ data DirInfo = DirInfo getDirInfo :: FilePath -> IO DirInfo getDirInfo dir = do l <- filter (not . dirCruft) <$> getDirectoryContents dir - contents <- S.fromList <$> mapM addinode l + contents <- S.fromList . catMaybes <$> mapM addinode l return $ DirInfo dir contents where - addinode f = do + addinode f = catchMaybeIO $ do inode <- fileID <$> getFileStatus (dir f) return (f, inode) From 9ab9ef3ebd931549b41d40c73ceeeba82a8099cc Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 17:16:30 -0400 Subject: [PATCH 4124/8313] change transfer lock filenames to avoid ambiguity foo.lck could be a lock file for a transfer of foo, or a transfer of a key that happened to end in ".lck". Fix this by using "lck.foo" instead. --- Logs/Transfer.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 8b88041273..daec476ef0 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -67,7 +67,7 @@ fieldTransfer direction key a = do =<< Fields.getField Fields.remoteUUID {- Runs a transfer action. Creates and locks the lock file while the - - action is running, and stores into in the transfer information + - action is running, and stores info in the transfer information - file. Will throw an error if the transfer is already in progress. -} transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a @@ -139,7 +139,7 @@ transferFile (Transfer direction u key) r = gitAnnexTransferDir r {- The transfer lock file corresponding to a given transfer info file. -} transferLockFile :: FilePath -> FilePath -transferLockFile infofile = infofile ++ ".lck" +transferLockFile infofile = "lck." ++ infofile {- Parses a transfer information filename to a Transfer. -} parseTransferFile :: FilePath -> Maybe Transfer From b702bae9501676082a1f9388286b3c9855d8a44a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 17:22:00 -0400 Subject: [PATCH 4125/8313] bugfix --- Logs/Transfer.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index daec476ef0..55db855cc3 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -139,7 +139,8 @@ transferFile (Transfer direction u key) r = gitAnnexTransferDir r {- The transfer lock file corresponding to a given transfer info file. -} transferLockFile :: FilePath -> FilePath -transferLockFile infofile = "lck." ++ infofile +transferLockFile infofile = let (d,f) = splitFileName infofile in + combine d ("lck." ++ f) {- Parses a transfer information filename to a Transfer. -} parseTransferFile :: FilePath -> Maybe Transfer From d53f70e2039a00b2ba2b87e26f29705d8f4c629a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 17:26:53 -0400 Subject: [PATCH 4126/8313] avoid parsing lock files as transfer files This seems to happen with kqueue, not inotify. The newly added lck file triggers an add event and was then parsed as a transfer file. --- Logs/Transfer.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 55db855cc3..2605120674 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -144,8 +144,9 @@ transferLockFile infofile = let (d,f) = splitFileName infofile in {- Parses a transfer information filename to a Transfer. -} parseTransferFile :: FilePath -> Maybe Transfer -parseTransferFile file = - case drop (length bits - 3) bits of +parseTransferFile file + | "lck." `isPrefixOf` (takeFileName file) = Nothing + | otherwise = case drop (length bits - 3) bits of [direction, u, key] -> Transfer <$> readDirection direction <*> pure (toUUID u) From 32ac7739348f6bc6aaf0db1e85a395368300dc33 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 18:32:55 -0400 Subject: [PATCH 4127/8313] kqueue: properly call delHook for file deletion, not delDirHook --- Utility/Kqueue.hs | 60 +++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs index 62b06a5323..58fc5a5b7f 100644 --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -14,8 +14,6 @@ module Utility.Kqueue ( waitChange, Change(..), changedFile, - isAdd, - isDelete, runHooks, ) where @@ -34,15 +32,14 @@ import Control.Concurrent data Change = Deleted FilePath + | DeletedDir FilePath | Added FilePath deriving (Show) isAdd :: Change -> Bool isAdd (Added _) = True isAdd (Deleted _) = False - -isDelete :: Change -> Bool -isDelete = not . isAdd +isAdd (DeletedDir _) = False changedFile :: Change -> FilePath changedFile (Added f) = f @@ -59,31 +56,43 @@ type Pruner = FilePath -> Bool type DirMap = M.Map Fd DirInfo -{- A directory, and its last known contents (with filenames relative to it) -} +{- Enough information to uniquely identify a file in a directory, + - but not too much. -} +data DirEnt = DirEnt + { dirEnt :: FilePath -- relative to the parent directory + , _dirInode :: FileID -- included to notice file replacements + , isSubDir :: Bool + } + deriving (Eq, Ord, Show) + +{- A directory, and its last known contents. -} data DirInfo = DirInfo { dirName :: FilePath - , dirCache :: S.Set (FilePath, FileID) + , dirCache :: S.Set DirEnt } deriving (Show) getDirInfo :: FilePath -> IO DirInfo getDirInfo dir = do l <- filter (not . dirCruft) <$> getDirectoryContents dir - contents <- S.fromList . catMaybes <$> mapM addinode l + contents <- S.fromList . catMaybes <$> mapM getDirEnt l return $ DirInfo dir contents where - addinode f = catchMaybeIO $ do - inode <- fileID <$> getFileStatus (dir f) - return (f, inode) + getDirEnt f = catchMaybeIO $ do + s <- getFileStatus (dir f) + return $ DirEnt f (fileID s) (isDirectory s) {- Difference between the dirCaches of two DirInfos. -} (//) :: DirInfo -> DirInfo -> [Change] oldc // newc = deleted ++ added where - deleted = calc Deleted oldc newc - added = calc Added newc oldc - calc a x y = map a . map (dirName x ) . map fst $ - S.toList $ S.difference (dirCache x) (dirCache y) + deleted = calc gendel oldc newc + added = calc genadd newc oldc + gendel x = (if isSubDir x then DeletedDir else Deleted) $ + dirName oldc dirEnt x + genadd x = Added $ dirName newc dirEnt x + calc a x y = map a $ S.toList $ + S.difference (dirCache x) (dirCache y) {- Builds a map of directories in a tree, possibly pruning some. - Opens each directory in the tree, and records its current contents. -} @@ -103,7 +112,7 @@ scanRecursive topdir prune = M.fromList <$> walk [] [topdir] case mfd of Nothing -> walk c rest Just fd -> do - let subdirs = map (dir ) . map fst $ + let subdirs = map (dir ) . map dirEnt $ S.toList $ dirCache info walk ((fd, info):c) (subdirs ++ rest) @@ -128,7 +137,7 @@ findDirContents :: DirMap -> FilePath -> [FilePath] findDirContents dirmap dir = concatMap absolutecontents $ search where absolutecontents i = map (dirName i ) - (map fst $ S.toList $ dirCache i) + (map dirEnt $ S.toList $ dirCache i) search = map snd $ M.toList $ M.filter (\i -> dirName i == dir) dirmap @@ -229,12 +238,14 @@ runHooks kq hooks = do (q', changes) <- waitChange q forM_ changes $ dispatch (kqueueMap q') loop q' - -- Kqueue returns changes for both whole directories - -- being added and deleted, and individual files being - -- added and deleted. - dispatch dirmap change - | isAdd change = withstatus change $ dispatchadd dirmap - | otherwise = callhook delDirHook Nothing change + + dispatch dirmap change@(Deleted _) = + callhook delHook Nothing change + dispatch dirmap change@(DeletedDir _) = + callhook delDirHook Nothing change + dispatch dirmap change@(Added _) = + withstatus change $ dispatchadd dirmap + dispatchadd dirmap change s | Files.isSymbolicLink s = callhook addSymlinkHook (Just s) change @@ -242,12 +253,15 @@ runHooks kq hooks = do | Files.isRegularFile s = callhook addHook (Just s) change | otherwise = noop + recursiveadd dirmap change = do let contents = findDirContents dirmap $ changedFile change forM_ contents $ \f -> withstatus (Added f) $ dispatchadd dirmap + callhook h s change = case h hooks of Nothing -> noop Just a -> a (changedFile change) s + withstatus change a = maybe noop (a change) =<< (catchMaybeIO (getSymbolicLinkStatus (changedFile change))) From 62a35162a0988f8d6e51796debceafda4d35c061 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 18:35:56 -0400 Subject: [PATCH 4128/8313] bugfix --- Utility/Kqueue.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs index 58fc5a5b7f..9013dbe7e2 100644 --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -44,6 +44,7 @@ isAdd (DeletedDir _) = False changedFile :: Change -> FilePath changedFile (Added f) = f changedFile (Deleted f) = f +changedFile (DeletedDir f) = f data Kqueue = Kqueue { kqueueFd :: Fd From fb85d8e563d071d7355c2cc7f8fb68860312e616 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 18:36:51 -0400 Subject: [PATCH 4129/8313] cleanup --- Utility/Kqueue.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs index 9013dbe7e2..c1a0a5cd60 100644 --- a/Utility/Kqueue.hs +++ b/Utility/Kqueue.hs @@ -240,9 +240,9 @@ runHooks kq hooks = do forM_ changes $ dispatch (kqueueMap q') loop q' - dispatch dirmap change@(Deleted _) = + dispatch _ change@(Deleted _) = callhook delHook Nothing change - dispatch dirmap change@(DeletedDir _) = + dispatch _ change@(DeletedDir _) = callhook delDirHook Nothing change dispatch dirmap change@(Added _) = withstatus change $ dispatchadd dirmap From cfdd4d66029915c683a1653c941c05e45205b13a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 17 Jul 2012 18:51:46 -0400 Subject: [PATCH 4130/8313] blog for the day and updates --- doc/design/assistant/blog/day_37__back.mdwn | 64 +++++++++++++++++++++ doc/design/assistant/syncing.mdwn | 7 +++ doc/todo/assistant_threaded_runtime.mdwn | 3 + 3 files changed, 74 insertions(+) create mode 100644 doc/design/assistant/blog/day_37__back.mdwn diff --git a/doc/design/assistant/blog/day_37__back.mdwn b/doc/design/assistant/blog/day_37__back.mdwn new file mode 100644 index 0000000000..7aef0b681a --- /dev/null +++ b/doc/design/assistant/blog/day_37__back.mdwn @@ -0,0 +1,64 @@ +Back home and laptop is fixed.. back to work. + +Warmup exercises: + +* Went in to make it queue transfers when a broken symlink is received, + only to find I'd already written code to do that, and forgotten about it. + Heh. Did check that the git-annex branch is always sent first, + which will ensure that code always knows where to transfer a key from. + I had probably not considered this wrinkle when first writing the code; + it worked by accident. + +* Made the assistant check that a remote is known to have a key before + queueing a download from it. + +* Fixed a bad interaction between the `git annex map` command and the + assistant. + +---- + +Tried using a modified version of `MissingH` that doesn't use `HSLogger` +to make git-annex work with the threaded GHC runtime. Unfortunatly, +I am still seeing hangs in at least 3 separate code paths when +running the test suite. I may have managed to fix one of the hangs, +but have not grokked what's causing the others. + +---- + +I now have access to a Mac OSX system, thanks to Kevin M. I've fixed +some portability problems in git-annex with it before, but today I tested +the assistant on it: + +* Found a problem with the kqueue code that prevents incoming pushes from + being noticed. + + The problem was that the newly added git ref file does not trigger an add + event. The kqueue code saw a generic change event for the refs directory, + but since the old file was being deleted and replaced by the new file, + the kqueue code, which already had the old file in its cache, did not notice + the file had been replaced. + + I fixed that by making the kqueue code also track the inode of each file. + Currently that adds the overhead of a stat of each file, which could be + avoided if haskell exposed the inode returned by `readdir`. Room to + optimise this later... + +* Also noticed that the kqueue code was not separating out file deletions + from directory deletions. IIRC Jimmy had once mentioned a problem with file + deletions not being noticed by the assistant, and this could be responsible + for that, although the directory deletion code seems to handle them ok + normally. It was making the transfer watching thread not notice when + any transfers finished, for sure. I fixed this oversight, looking in the + cache to see if there used to be a file or a directory, and running the + appropriate hook. + +Even with these fixes, the assistant does not yet reliably transfer file +contents on OSX. I think the problem is that with kqueue we're not +guaranteed to get an add event, and a deletion event for a transfer +info file -- if it's created and quickly deleted, the code that +synthensizes those events doesn't run in time to know it existed. +Since the transfer code relies on deletion events to tell when transfers +are complete, it stops sending files after the first transfer, if the +transfer ran so quickly it doesn't get the expected events. + +So, will need to work on OSX support some more... diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index a9f59f4968..3fe27d5ac6 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -10,6 +10,13 @@ all the other git clones, at both the git level and the key/value level. on remotes, and transfer. But first, need to ensure that when a remote receives content, and updates its location log, it syncs that update out. +* Transfer watching has a race on kqueue systems, which makes finished + fast transfers not be noticed by the TransferWatcher. Which in turn + prevents the transfer slot being freed and any further transfers + from happening. So, this approach is too fragile to rely on for + maintaining the TransferSlots. Instead, need [[todo/assistant_threaded_runtime]], + which would allow running something for sure when a transfer thread + finishes. ## longer-term TODO diff --git a/doc/todo/assistant_threaded_runtime.mdwn b/doc/todo/assistant_threaded_runtime.mdwn index 095ffa4359..edfa51669f 100644 --- a/doc/todo/assistant_threaded_runtime.mdwn +++ b/doc/todo/assistant_threaded_runtime.mdwn @@ -20,6 +20,9 @@ The test suite tends to hang when testing add. `git-annex` occasionally hangs, apparently in a futex lock. This is not the assistant hanging, and git-annex does not otherwise use threads, so this is surprising. --[[Joey]] +> I've spent a lot of time debugging this, and trying to fix it, in the +> "threaded" branch. There are still deadlocks. --[[Joey]] + --- It would be possible to not use the threaded runtime. Instead, we could From 05310538ef4f2c0c483bab355083ec2044a12a0a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 13:30:53 -0400 Subject: [PATCH 4131/8313] more debugging --- Annex/Branch.hs | 4 +++- Git/CatFile.hs | 12 +++++++++++- Git/CheckAttr.hs | 4 ++++ System/Cmd/Utils.hs | 2 +- Utility/SafeCommand.hs | 7 ++++++- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Annex/Branch.hs b/Annex/Branch.hs index 8e7f45a4ad..e551bfcd01 100644 --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -164,7 +164,9 @@ get' staleok file = fromcache =<< getCache file fromjournal Nothing | staleok = withIndex frombranch | otherwise = withIndexUpdate $ frombranch >>= cache - frombranch = L.unpack <$> catFile fullname file + frombranch = do + liftIO $ putStrLn $ "frombranch " ++ file + L.unpack <$> catFile fullname file cache content = do setCache file content return content diff --git a/Git/CatFile.hs b/Git/CatFile.hs index e667b20879..e8f362685d 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -50,11 +50,16 @@ catObjectDetails :: CatFileHandle -> Ref -> IO (Maybe (L.ByteString, Sha)) catObjectDetails h object = CoProcess.query h send receive where send to = do + putStrLn "catObjectDetails send start" fileEncoding to hPutStrLn to $ show object + putStrLn $ "catObjectDetails send done " ++ show object receive from = do + putStrLn "catObjectDetails read header start" fileEncoding from + putStrLn "catObjectDetails read header start2" header <- hGetLine from + putStrLn "catObjectDetails read header done" case words header of [sha, objtype, size] | length sha == shaSize && @@ -67,9 +72,14 @@ catObjectDetails h object = CoProcess.query h send receive | header == show object ++ " missing" -> dne | otherwise -> error $ "unknown response from git cat-file " ++ show (header, object) readcontent bytes from sha = do + putStrLn "readcontent start" content <- S.hGet from bytes + putStrLn "readcontent end" c <- hGetChar from + putStrLn "readcontent newline read" when (c /= '\n') $ error "missing newline from git cat-file" return $ Just (L.fromChunks [content], Ref sha) - dne = return Nothing + dne = do + putStrLn "dne" + return Nothing diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs index 6b321f8b8f..7636ea6411 100644 --- a/Git/CheckAttr.hs +++ b/Git/CheckAttr.hs @@ -44,11 +44,15 @@ checkAttr (h, attrs, cwd) want file = do _ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file where send to = do + putStrLn "checkAttr send start" fileEncoding to hPutStr to $ file' ++ "\0" + putStrLn "checkAttr send end" receive from = forM attrs $ \attr -> do + putStrLn "checkAttr receive start" fileEncoding from l <- hGetLine from + putStrLn "checkAttr receive end" return (attr, attrvalue attr l) {- Before git 1.7.7, git check-attr worked best with - absolute filenames; using them worked around some bugs diff --git a/System/Cmd/Utils.hs b/System/Cmd/Utils.hs index 15544d6846..a81126146b 100644 --- a/System/Cmd/Utils.hs +++ b/System/Cmd/Utils.hs @@ -501,7 +501,7 @@ pOpen3 :: Maybe Fd -- ^ Send stdin to this fd -> (ProcessID -> IO a) -- ^ Action to run in parent -> IO () -- ^ Action to run in child before execing (if you don't need something, set this to @return ()@) -- IGNORED IN HUGS -> IO a -pOpen3 pin pout perr fp args func childfunc = ddd "pOpen3" $ +pOpen3 pin pout perr fp args func childfunc = ddd (show ("pOpen3", fp, args)) $ do pid <- pOpen3Raw pin pout perr fp args childfunc putStrLn "got pid" retval <- func $! pid diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs index 2c6439b452..5f6a53e715 100644 --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -54,6 +54,7 @@ safeSystem command params = safeSystemEnv command params Nothing {- SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode safeSystemEnv command params env = do + putStrLn "safeSystemEnv start" -- Going low-level because all the high-level system functions -- block SIGINT etc. We need to block SIGCHLD, but allow -- SIGINT to do its default program termination. @@ -65,7 +66,9 @@ safeSystemEnv command params env = do mps <- getProcessStatus True False childpid restoresignals oldint oldset case mps of - Just (Exited code) -> return code + Just (Exited code) -> do + putStrLn "safeSystemEnv end" + return code _ -> error $ "unknown error running " ++ command where restoresignals oldint oldset = do @@ -78,9 +81,11 @@ safeSystemEnv command params env = do {- executeFile with debug logging -} executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO () executeFile c path p e = do + putStrLn "executeFile start" --debugM "Utility.SafeCommand.executeFile" $ -- "Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e System.Posix.Process.executeFile c path p e + putStrLn "executeFile end" {- Escapes a filename or other parameter to be safely able to be exposed to - the shell. -} From d1da9cf221aeea5c7ac8a313a18b559791a04f12 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 15:30:26 -0400 Subject: [PATCH 4132/8313] switch from System.Cmd.Utils to System.Process Test suite now passes with -threaded! I traced back all the hangs with -threaded to System.Cmd.Utils. It seems it's just crappy/unsafe/outdated, and should not be used. System.Process seems to be the cool new thing, so converted all the code to use it instead. In the process, --debug stopped printing commands it runs. I may try to bring that back later. Note that even SafeSystem was switched to use System.Process. Since that was a modified version of code from System.Cmd.Utils, it needed to be converted too. I also got rid of nearly all calls to forkProcess, and all calls to executeFile, which I'm also doubtful about working well with -threaded. --- Annex/Branch.hs | 4 +- Annex/UUID.hs | 6 +- Backend/SHA.hs | 17 +- Build/Configure.hs | 4 +- Command/Fsck.hs | 2 + Command/Map.hs | 11 +- Common.hs | 3 +- Config.hs | 6 +- Git/Branch.hs | 2 - Git/CatFile.hs | 12 +- Git/CheckAttr.hs | 4 - Git/Command.hs | 29 +- Git/Config.hs | 14 +- Git/Queue.hs | 17 +- Git/Ref.hs | 5 +- Git/UpdateIndex.hs | 7 +- Remote/Bup.hs | 6 +- Remote/Git.hs | 14 +- Remote/Hook.hs | 17 +- Remote/Rsync.hs | 1 + System/Cmd/.Utils.hs.swp | Bin 36864 -> 0 bytes System/Cmd/Utils.hs | 568 ----------------------- Utility/CoProcess.hs | 14 +- Utility/Gpg.hs | 39 +- Utility/INotify.hs | 8 +- Utility/Lsof.hs | 7 +- Utility/Process.hs | 40 ++ Utility/SafeCommand.hs | 49 +- Utility/TempFile.hs | 2 +- doc/todo/assistant_threaded_runtime.mdwn | 3 + git-annex.cabal | 6 +- test.hs | 1 + 32 files changed, 178 insertions(+), 740 deletions(-) delete mode 100644 System/Cmd/.Utils.hs.swp delete mode 100644 System/Cmd/Utils.hs create mode 100644 Utility/Process.hs diff --git a/Annex/Branch.hs b/Annex/Branch.hs index e551bfcd01..8e7f45a4ad 100644 --- a/Annex/Branch.hs +++ b/Annex/Branch.hs @@ -164,9 +164,7 @@ get' staleok file = fromcache =<< getCache file fromjournal Nothing | staleok = withIndex frombranch | otherwise = withIndexUpdate $ frombranch >>= cache - frombranch = do - liftIO $ putStrLn $ "frombranch " ++ file - L.unpack <$> catFile fullname file + frombranch = L.unpack <$> catFile fullname file cache content = do setCache file content return content diff --git a/Annex/UUID.hs b/Annex/UUID.hs index 517840fbad..1d2175bcb6 100644 --- a/Annex/UUID.hs +++ b/Annex/UUID.hs @@ -20,6 +20,8 @@ module Annex.UUID ( removeRepoUUID, ) where +import System.Process + import Common.Annex import qualified Git import qualified Git.Config @@ -32,8 +34,10 @@ configkey = annexConfig "uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} genUUID :: IO UUID -genUUID = pOpen ReadFromPipe command params $ liftM toUUID . hGetLine +genUUID = gen . lines <$> readProcess command params [] where + gen [] = error $ "no output from " ++ command + gen (l:_) = toUUID l command = SysConfig.uuid params -- request a random uuid be generated diff --git a/Backend/SHA.hs b/Backend/SHA.hs index cf61139e00..a1dd1cf648 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -12,6 +12,7 @@ import qualified Annex import Types.Backend import Types.Key import Types.KeySource +import System.Process import qualified Build.SysConfig as SysConfig import Data.Digest.Pure.SHA @@ -53,14 +54,16 @@ shaN shasize file filesize = do showAction "checksum" case shaCommand shasize filesize of Left sha -> liftIO $ sha <$> L.readFile file - Right command -> liftIO $ runcommand command + Right command -> liftIO $ parse command . lines <$> + readProcess command (toCommand [File file]) "" where - runcommand command = - pOpen ReadFromPipe command (toCommand [File file]) $ \h -> do - sha <- fst . separate (== ' ') <$> hGetLine h - if null sha - then error $ command ++ " parse error" - else return sha + parse command [] = bad command + parse command (l:_) + | null sha = bad command + | otherwise = sha + where + sha = fst $ separate (== ' ') l + bad command = error $ command ++ " parse error" shaCommand :: SHASize -> Integer -> Either (L.ByteString -> String) String shaCommand shasize filesize diff --git a/Build/Configure.hs b/Build/Configure.hs index cf6623b226..9468e1704d 100644 --- a/Build/Configure.hs +++ b/Build/Configure.hs @@ -4,7 +4,7 @@ module Build.Configure where import System.Directory import Data.List -import System.Cmd.Utils +import System.Process import Control.Applicative import System.FilePath @@ -71,7 +71,7 @@ getVersionString = do getGitVersion :: Test getGitVersion = do - (_, s) <- pipeFrom "git" ["--version"] + s <- readProcess "git" ["--version"] "" let version = unwords $ drop 2 $ words $ head $ lines s return $ Config "gitversion" (StringConfig version) diff --git a/Command/Fsck.hs b/Command/Fsck.hs index 10cca489b1..0e3cc934c3 100644 --- a/Command/Fsck.hs +++ b/Command/Fsck.hs @@ -7,6 +7,8 @@ module Command.Fsck where +import System.Posix.Process (getProcessID) + import Common.Annex import Command import qualified Annex diff --git a/Command/Map.hs b/Command/Map.hs index 0773f68283..f69b88a5d6 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -9,6 +9,7 @@ module Command.Map where import Control.Exception.Extensible import qualified Data.Map as M +import System.Process import Common.Annex import Command @@ -198,9 +199,13 @@ tryScan r case result of Left _ -> return Nothing Right r' -> return $ Just r' - pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd (toCommand params) $ - Git.Config.hRead r + pipedconfig cmd params = safely $ do + (_, Just h, _, pid) <- + createProcess (proc cmd $ toCommand params) + { std_out = CreatePipe } + r' <- Git.Config.hRead r h + forceSuccessProcess pid cmd $ toCommand params + return r' configlist = onRemote r (pipedconfig, Nothing) "configlist" [] [] diff --git a/Common.hs b/Common.hs index 7f07781ce9..04ec1e044e 100644 --- a/Common.hs +++ b/Common.hs @@ -13,16 +13,15 @@ import Data.String.Utils as X import System.Path as X import System.FilePath as X import System.Directory as X -import System.Cmd.Utils as X hiding (safeSystem) import System.IO as X hiding (FilePath) import System.Posix.Files as X import System.Posix.IO as X -import System.Posix.Process as X hiding (executeFile) import System.Exit as X import Utility.Misc as X import Utility.Exception as X import Utility.SafeCommand as X +import Utility.Process as X import Utility.Path as X import Utility.Directory as X import Utility.Monad as X diff --git a/Config.hs b/Config.hs index e66947e2cc..84f6125c63 100644 --- a/Config.hs +++ b/Config.hs @@ -7,6 +7,8 @@ module Config where +import System.Process + import Common.Annex import qualified Git import qualified Git.Config @@ -56,7 +58,7 @@ remoteCost r def = do cmd <- getRemoteConfig r "cost-command" "" (fromMaybe def . readish) <$> if not $ null cmd - then liftIO $ snd <$> pipeFrom "sh" ["-c", cmd] + then liftIO $ readProcess "sh" ["-c", cmd] "" else getRemoteConfig r "cost" "" cheapRemoteCost :: Int @@ -116,4 +118,4 @@ getHttpHeaders = do cmd <- getConfig (annexConfig "http-headers-command") "" if null cmd then fromRepo $ Git.Config.getList "annex.http-headers" - else lines . snd <$> liftIO (pipeFrom "sh" ["-c", cmd]) + else lines <$> liftIO (readProcess "sh" ["-c", cmd] "") diff --git a/Git/Branch.hs b/Git/Branch.hs index 6f3d251863..4d239d8fc5 100644 --- a/Git/Branch.hs +++ b/Git/Branch.hs @@ -76,9 +76,7 @@ commit message branch parentrefs repo = do sha <- getSha "commit-tree" $ pipeWriteRead (map Param $ ["commit-tree", show tree] ++ ps) message repo - print ("got", sha) run "update-ref" [Param $ show branch, Param $ show sha] repo - print ("update-ref done", sha) return sha where ps = concatMap (\r -> ["-p", show r]) parentrefs diff --git a/Git/CatFile.hs b/Git/CatFile.hs index e8f362685d..e667b20879 100644 --- a/Git/CatFile.hs +++ b/Git/CatFile.hs @@ -50,16 +50,11 @@ catObjectDetails :: CatFileHandle -> Ref -> IO (Maybe (L.ByteString, Sha)) catObjectDetails h object = CoProcess.query h send receive where send to = do - putStrLn "catObjectDetails send start" fileEncoding to hPutStrLn to $ show object - putStrLn $ "catObjectDetails send done " ++ show object receive from = do - putStrLn "catObjectDetails read header start" fileEncoding from - putStrLn "catObjectDetails read header start2" header <- hGetLine from - putStrLn "catObjectDetails read header done" case words header of [sha, objtype, size] | length sha == shaSize && @@ -72,14 +67,9 @@ catObjectDetails h object = CoProcess.query h send receive | header == show object ++ " missing" -> dne | otherwise -> error $ "unknown response from git cat-file " ++ show (header, object) readcontent bytes from sha = do - putStrLn "readcontent start" content <- S.hGet from bytes - putStrLn "readcontent end" c <- hGetChar from - putStrLn "readcontent newline read" when (c /= '\n') $ error "missing newline from git cat-file" return $ Just (L.fromChunks [content], Ref sha) - dne = do - putStrLn "dne" - return Nothing + dne = return Nothing diff --git a/Git/CheckAttr.hs b/Git/CheckAttr.hs index 7636ea6411..6b321f8b8f 100644 --- a/Git/CheckAttr.hs +++ b/Git/CheckAttr.hs @@ -44,15 +44,11 @@ checkAttr (h, attrs, cwd) want file = do _ -> error $ "unable to determine " ++ want ++ " attribute of " ++ file where send to = do - putStrLn "checkAttr send start" fileEncoding to hPutStr to $ file' ++ "\0" - putStrLn "checkAttr send end" receive from = forM attrs $ \attr -> do - putStrLn "checkAttr receive start" fileEncoding from l <- hGetLine from - putStrLn "checkAttr receive end" return (attr, attrvalue attr l) {- Before git 1.7.7, git check-attr worked best with - absolute filenames; using them worked around some bugs diff --git a/Git/Command.hs b/Git/Command.hs index 9a09300e24..038824f268 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -7,10 +7,8 @@ module Git.Command where -import qualified Data.Text.Lazy as L -import qualified Data.Text.Lazy.IO as L -import Control.Concurrent -import Control.Exception (finally) +import System.Process +import System.Posix.Process (getAnyProcessStatus) import Common import Git @@ -44,31 +42,18 @@ run subcommand params repo = assertLocal repo $ -} pipeRead :: [CommandParam] -> Repo -> IO String pipeRead params repo = assertLocal repo $ do - (_, h) <- hPipeFrom "git" $ toCommand $ gitCommandLine params repo + (_, Just h, _, _) <- createProcess + (proc "git" $ toCommand $ gitCommandLine params repo) + { std_out = CreatePipe } fileEncoding h hGetContents h -{- Runs a git subcommand, feeding it input. - - You should call either getProcessStatus or forceSuccess on the PipeHandle. -} -pipeWrite :: [CommandParam] -> L.Text -> Repo -> IO PipeHandle -pipeWrite params s repo = assertLocal repo $ do - (p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo) - L.hPutStr h s - hClose h - return p - {- Runs a git subcommand, feeding it input, and returning its output, - which is expected to be fairly small, since it's all read into memory - strictly. -} pipeWriteRead :: [CommandParam] -> String -> Repo -> IO String -pipeWriteRead params s repo = assertLocal repo $ do - (p, from, to) <- hPipeBoth "git" (toCommand $ gitCommandLine params repo) - fileEncoding to - fileEncoding from - _ <- forkIO $ finally (hPutStr to s) (hClose to) - c <- hGetContentsStrict from - forceSuccess p - return c +pipeWriteRead params s repo = assertLocal repo $ + readProcess "git" (toCommand $ gitCommandLine params repo) s {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it. -} diff --git a/Git/Config.hs b/Git/Config.hs index c9e4f9a2dc..2347501131 100644 --- a/Git/Config.hs +++ b/Git/Config.hs @@ -9,6 +9,7 @@ module Git.Config where import qualified Data.Map as M import Data.Char +import System.Process import Common import Git @@ -39,7 +40,7 @@ reRead :: Repo -> IO Repo reRead = read' {- Cannot use pipeRead because it relies on the config having been already - - read. Instead, chdir to the repo. + - read. Instead, chdir to the repo and run git config. -} read' :: Repo -> IO Repo read' repo = go repo @@ -47,9 +48,14 @@ read' repo = go repo go Repo { location = Local { gitdir = d } } = git_config d go Repo { location = LocalUnknown d } = git_config d go _ = assertLocal repo $ error "internal" - git_config d = bracketCd d $ - pOpen ReadFromPipe "git" ["config", "--null", "--list"] $ - hRead repo + git_config d = do + (_, Just h, _, pid) + <- createProcess (proc "git" params) + { std_out = CreatePipe, cwd = Just d } + repo' <- hRead repo h + forceSuccessProcess pid "git" params + return repo' + params = ["config", "--null", "--list"] {- Reads git config from a handle and populates a repo with it. -} hRead :: Repo -> Handle -> IO Repo diff --git a/Git/Queue.hs b/Git/Queue.hs index ddcf135197..4e6f05c2e0 100644 --- a/Git/Queue.hs +++ b/Git/Queue.hs @@ -19,7 +19,7 @@ module Git.Queue ( import qualified Data.Map as M import System.IO -import System.Cmd.Utils +import System.Process import Data.String.Utils import Utility.SafeCommand @@ -148,11 +148,14 @@ runAction :: Repo -> Action -> IO () runAction repo (UpdateIndexAction streamers) = -- list is stored in reverse order Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers -runAction repo action@(CommandAction {}) = - pOpen WriteToPipe "xargs" ("-0":"git":params) feedxargs +runAction repo action@(CommandAction {}) = do + (Just h, _, _, pid) <- createProcess (proc "xargs" params) + { std_in = CreatePipe } + fileEncoding h + hPutStr h $ join "\0" $ getFiles action + hClose h + forceSuccessProcess pid "xargs" params where - params = toCommand $ gitCommandLine + params = "-0":"git":baseparams + baseparams = toCommand $ gitCommandLine (Param (getSubcommand action):getParams action) repo - feedxargs h = do - fileEncoding h - hPutStr h $ join "\0" $ getFiles action diff --git a/Git/Ref.hs b/Git/Ref.hs index 3052d0a6ef..ee2f021871 100644 --- a/Git/Ref.hs +++ b/Git/Ref.hs @@ -40,10 +40,7 @@ exists ref = runBool "show-ref" {- Get the sha of a fully qualified git ref, if it exists. -} sha :: Branch -> Repo -> IO (Maybe Sha) -sha branch repo = do - r <- process <$> showref repo - print r - return r +sha branch repo = process <$> showref repo where showref = pipeRead [Param "show-ref", Param "--hash", -- get the hash diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs index abdc4bcbe3..6de0c3adab 100644 --- a/Git/UpdateIndex.hs +++ b/Git/UpdateIndex.hs @@ -17,7 +17,7 @@ module Git.UpdateIndex ( stageSymlink ) where -import System.Cmd.Utils +import System.Process import Common import Git @@ -37,12 +37,13 @@ pureStreamer !s = \streamer -> streamer s {- Streams content into update-index from a list of Streamers. -} streamUpdateIndex :: Repo -> [Streamer] -> IO () streamUpdateIndex repo as = do - (p, h) <- hPipeTo "git" (toCommand $ gitCommandLine params repo) + (Just h, _, _, p) <- createProcess (proc "git" ps) { std_in = CreatePipe } fileEncoding h forM_ as (stream h) hClose h - forceSuccess p + forceSuccessProcess p "git" ps where + ps = toCommand $ gitCommandLine params repo params = map Param ["update-index", "-z", "--index-info"] stream h a = a (streamer h) streamer h s = do diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 0d1b606d3d..9da374174b 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -136,9 +136,11 @@ retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool retrieveEncrypted buprepo (cipher, enck) _ f = do let params = bupParams "join" buprepo [Param $ bupRef enck] liftIO $ catchBoolIO $ do - (pid, h) <- hPipeFrom "bup" $ toCommand params + (_, Just h, _, pid) + <- createProcess (proc "bup" $ toCommand params) + { std_out = CreatePipe } withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f - forceSuccess pid + forceSuccessProcess pid "bup" $ toCommand params return True remove :: Key -> Annex Bool diff --git a/Remote/Git.hs b/Remote/Git.hs index d80f580fc5..a9a6d6004e 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -9,6 +9,7 @@ module Remote.Git (remote, repoAvail) where import qualified Data.Map as M import Control.Exception.Extensible +import System.Process import Common.Annex import Utility.CopyFile @@ -126,17 +127,20 @@ tryGitConfigRead r safely a = either (const $ return r) return =<< liftIO (try a :: IO (Either SomeException Git.Repo)) - pipedconfig cmd params = safely $ - pOpen ReadFromPipe cmd (toCommand params) $ - Git.Config.hRead r + pipedconfig cmd params = safely $ do + (_, Just h, _, pid) <- + createProcess (proc cmd $ toCommand params) + { std_out = CreatePipe } + r' <- Git.Config.hRead r h + forceSuccessProcess pid cmd $ toCommand params + return r' geturlconfig headers = do s <- Url.get (Git.repoLocation r ++ "/config") headers withTempFile "git-annex.tmp" $ \tmpfile h -> do hPutStr h s hClose h - pOpen ReadFromPipe "git" ["config", "--null", "--list", "--file", tmpfile] $ - Git.Config.hRead r + pipedconfig "git" [Param "config", Param "--null", Param "--list", Param "--file", File tmpfile] store = observe $ \r' -> do g <- gitRepo diff --git a/Remote/Hook.hs b/Remote/Hook.hs index 9e8d3c620d..cad6e2fc94 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -9,7 +9,6 @@ module Remote.Hook (remote) where import qualified Data.ByteString.Lazy as L import qualified Data.Map as M -import System.Exit import System.Environment import Common.Annex @@ -136,17 +135,5 @@ checkPresent r h k = do findkey s = show k `elem` lines s check Nothing = error "checkpresent hook misconfigured" check (Just hook) = do - (frompipe, topipe) <- createPipe - pid <- forkProcess $ do - _ <- dupTo topipe stdOutput - closeFd frompipe - executeFile "sh" True ["-c", hook] - =<< hookEnv k Nothing - closeFd topipe - fromh <- fdToHandle frompipe - reply <- hGetContentsStrict fromh - hClose fromh - s <- getProcessStatus True False pid - case s of - Just (Exited ExitSuccess) -> return $ findkey reply - _ -> error "checkpresent hook failed" + env <- hookEnv k Nothing + findkey <$> readProcessEnv "sh" ["-c", hook] env diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index 29bceb2db8..ee516a8a59 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -9,6 +9,7 @@ module Remote.Rsync (remote) where import qualified Data.ByteString.Lazy as L import qualified Data.Map as M +import System.Posix.Process (getProcessID) import Common.Annex import Types.Remote diff --git a/System/Cmd/.Utils.hs.swp b/System/Cmd/.Utils.hs.swp deleted file mode 100644 index 65e9e77e4437e33e567a27e7f34f6b8301ca5f47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36864 zcmeI53zTFnT26r%!i0tpEy7@r{c2>Jf1y0>pX zW_A{3A=m@o_TyI7ty_=3>i_Gn>e;ntX11wQ(imGc8bJ`BaIp6md zgV%~hzI!I@f!jeoKMQukC?DkW)8SBful@Xoyq|$u0=*LGl|ZiqdL_^+fnEvpN}yK)y%Ok^K(7RPC2){Rz|A|(iQMad z(9320Uu*#1dA8%+1Yd-|g1>}Uz%Pz;oPU5T;B7Dke+;)Cfu0W^h8*<6orhy9fZO0( za2>o3rr}laLU7=7f8;nfz+Cyc-vNW>EdS+E9U-s%=Ud=A5lhY@0Dd$)BdL&l&)qLRAYF-eg9cRAi1)ifi{|1~@UNv7TL@%8o zl&VeEyy~EN*$}->yYtSfQlXmn3TpT6tvkkNcJG!i)bT04DpkFFs#2=X&2L!0d+X#( zcK77WfV%i%)qX8IJUu?zadr3ZZJF)kJBhn{V4z#J)l{kGZS(V9DxDq}>>7gNI#Esd z^FtHfUazdOzF!Wbr?k)9?O}_O^67rP*gaK!IqNTTaV+pF-7iayPL!NeE^5A1FRG$j z&6mBfYyNA?=0mP7l?}NzK@EgUXGMZ6eYG>AW%t5G71jfC-aWSkKiET)Fip7oh2h{qY+s>U@Roa+DGG3_u07KND}KIF_7YtK zN>$vw+`I&og4XHg-MULwr~psp+;Um&*PpD8eo{C!ijpmQTO`5)UnK*nS}7mbD}_Os z&4ruv4J1({7x$bC+zNdmto9?fya2i3icDMd>ZFz;g+aA2pk6UVVy4-uzH->7&hA^| z)pLF4I?h(4o8m#;s`3`xN{vF(1xwilzH~NsQ3PV%SEc$!#}V1+mg*ENrJZvdp{Ex8 zMi6I(>moVbKqApc=h9B=ldDD*uQ=rv$gScrlrF3SJzqcBjP5A%L=+&Eg-i|`6)&Gw zYRoN_8;Ig*ui{l^yeqG6NHw&V z`MNIOt>?RRI<0Rq7kD-0H|n(pwI8Z{DXf*B^;*d7)|Wv~3EcK{=P| zB`>7xMoraHfqizUobcz;Fy{q*X=ldssEg3wsQNoDryDh@$Qv)y5y~b;|Dcpxx4#q_ zGK0p*bS9e_8J-v*WM>L0$_690>F(*K*yv*tr0-TX92%|& zr=7#o+ZoACG%}&we4gs`tImjDsUSamw2v%~+o783dzhwi6})ffxovDax=Fq*S5K(Y8{Y18(Z?NUJ>QHCEKU!j7^ks zUX`-qZ=)LDzC(>qO$;h#i;}QfuV^OjtiM;!6G!~oV!*Ifm$1zbsgVJ-e(lYm!L zVeKHTF|7Mk$hKjn)~;W-Zr#wj4Qo$PYRAm53V#Y7j=V{z}@C%E6qnt-j@AZ(gIj^)=It22%riW6Aq|3>pq^X;!#%#HiQ??u( zXX$IY&*{-N?@>`WP&?a3r$@F9Z_f^&o|(vGcS&iFWwP5xXJ*vc7AME*2x`NHN1V7bgXF-+qFpxO9Z*1V^(#a6B$#IuH*)R zTdm8JI9ZuvL?f+{_l!{_N;rC@JZRCc$*t&D(a1<|W=twH%1Fu--|I2b*HU5tnX!l( zBIBQJKxwC=j8(IyHz|)=YKE5PCQ~0TD$PL(#Np0b*ysx~69}VF%M>*NtR@u`7I~Q2 z#g1n!NVSKy+Ba#DDzZ5681!r*$Ri_+)=(!*#7YJH>sQj|m+Wm94bj?Hsfpq3<2#1O zN7cyGlv>4<4`J*Ex#CdWos(W=Bw#E)RD~|B3Z=5=i2i>Gdh5H;ZAJed=H<)i`fq{@ zAPc9!bK#puI?gBIeQ*_s4dAWta!A2L==~4E9dJ9m4HjS?)`J7TLFd070vLjypyOW; z&xb?cJLveIfY(C?j)uF?>AwnBLm5s5(e=NN9{=C)b$ADy3n#!Ka4&lNJ#ZU*5k3QN zgbQFB90|YTnSTjCg9qS#xDli*u7ciQuLOD}&?|vn2^{h>qfo2qrc zX2Ner|3-sbE{L6dPl?SfJ)>2NFrAsJV$C_=XlN!=nW@q4@w(nf>`9nKjDakOBL<#Z zHcv;xp_rH^scwH(8@7_Cu=999fHgC(o6Y`1 z%j}xaliFIc|KQBw2%V{7Lx}@7+lhH{wZcE!K2y!swZDkDvLAY5^iWJMh0d7>lT-VQmQ&-5ZmOC_4HEkyTTgnHWM+Xp3Ti%ll%0 zta_*sp(3%afu)R~Ls|I)G( zF=KJL|Cr{v5^=Xl1t@C2>z}^sIQ2yHlpbL1m8gTw>ZZ6?o`}?6OCrtsMeeU4of+1i zBsbIy&!S72tRNWYQCxMkNq_ZnI?Hl5u^5@$4=S3h=xv2k5Z2QU4bG~sQkAqsZS{$4 zsv*WOo>F1w&{{cqZj}!hjoIdmnr%#NtaFBH*W`|Anrv9ZXj?L7oZ~UqiV-$cr3z-; z5^672JZ+X24R$KCB~nv$`y#y^+HTb^R%Y9D`{mxUj7LV5yO(iGn}ztYY1tPGiJES3 zAF)Ab?QoE+m?Dsts+%O{d2C5o!Z9jztf4$UD{yUNv~69sGX#617HF)-6II4MJ3g`y zsP*!`C{%~ceI(e(n66nn9=FbEICQw%$P_QfJlBu%gvxsr%sB1Si9=3|b+ zHrL`D*n2RV>!E1%@p8NFF}Yx(v2b7~2F(_}9|`7|ki_dI==7}<(oQ923NlR){r~H% zAE#M&7X4p*_8&plza4IYcfvd1EwB$RhgZUj;0|>DYvD5R!G+hrVekmL{td7XM&Sr} z0Db@4@Kv}G{t-S0IS{{qemDUh!Vd5td;rdZov;~R4u``JunF7@pM&?nKG*}N!lT#) zZh%W+2ERs%XKxLRub*^d7h#EpH4EKIx_&<@j6~)XHw-niuv1JYwSCZ&x38RJDDT`)VUnXZFnUa^wTH!MFJJ1?e2U_LoG1R&0sPvZNa+<}n{JXf; zTgq$c=*w37G#)@>SPi<=yD^7VTI^xAFKcTVhMl@6@#0(4Xmri0S|y&z>wZB~i;ZGQ zgP2O$_8*Uh_Ki&kSuZybpFe4ov6jsHcnlmm)ln#q%g9$eJ3{d_akW@6%B$8t2eoS1 zpJ@91qWO0Vj2@b1krWPW zKfVlZ7?OnOl<22;28zz86=N1K?H#b}SO>4PqnFf`cxOfr)DhUKkx!(iSn`c|)-;Y- zH^TxUz7WQWDGk#@456O&*&dPiwspGr@sTf0= zQSfXKUH_x-TDTY{;Fa(TGW-enI9v&Dg;_Wao)6E4AEWQz2{*#u!VWkRZb$F`7&M?C z{tI3IYw$jJ4Ll$2MZdouE`?LzA@utn!F_NqTmhHBcGw0RU_Ja0{r~fD12kYQ+>gKh zC!qmr;Y#c-mjXRR{-VZUKG}~wtuhj%PdV##Ppf>Q?aH>Nb-Z-ZGFs-ntuuvW^qyAP z(<}i#v8TPbFy@_)r8#Nt~B0a6Li?7?`p;d|s8E>bI zo%^+)lr^%ogy?Jew1V&~L$^jje_q1u*6O0eHXS?|!j8?5>DJ`1)L}*w?-Og#!qSZO zt7@FvjC+N#zR2b;Z9ZdjjOS0ro5Sde7aQ7x`$g<)+?X%R!*{Zn(N=7wDp|WuFAbJ^mN> zAl#SOghS58utmt0J~7*I-$q%!oVE@lZT=x)W7g(C+=sA}bayz>`oAofeic1b^#2Mk ze}~S0CCtOw@KQ*_bKnqo0G0iFh5 z!8Y(-coT%M4UU03umyYpu7X)O8?rD1zriN(0DKre1ac0*QScLN13!ju!>w=?Tnf8k z3k<;ta2&iCr0$*ocO0`{HWw>HIkC_ear+watL${cq-~a_=rty2QE%7O+la;dADQc& z8s_%o7x?7GUo9^R5`H~w<Xmb@!B=ar9!-ixh&u3)ubWvHiBi&D8eG zbbHUm$0}&sqpU6Hk*(gWFtBfVa{I(CH9VFboz}+oux-CC%NcpG^6G7WvVU`zxRiiqn zlUmv8)q$!J%pJNqavMKxJ-3kgMf0!e2F_Nus)%76&upV*oxr-NuXdU-qIb)9fLlK= zv*MLl@IqX-I$zYJU#-X?Ji2S*HEA_oxreh!ATHOgYleD>FZN2;s}kZVnWeY47)!*B zaR3VS2){5<)t*_P4a#c3bBSY!=oN zj;Qj}?7i+7)XgqOy=E76J*D;|cJxsTxCOH#ReP0+2(atJ%|$x1bjW8pmBEYlIkp`b zz~)A?TDO0BVRH9`ii$(+!g=5v8n4%_R-sua!)Le8IaD_jx7&4G|ATs;BW*?;S+#G0 zlW6ler)bxkBt{CrQ+wWw(jl;vuVtao3NR=c0gIWyp%J(;&|t zDY}8)d$QC#wiyC{ESz;|S@`2t#rIayZ?^QvwSa|yp`pAt+nC#?QiPtBjU$r9v|jv@ zNCQTNOl+GONTy4^_P3o)EiEtu?4IvY~Br{K|}j7;Tw$We<&bI6#+aMybXfty`!!3(1PCha~v}}Uw znv8M62)kahmDTlHX7X{;@MQge9XkI{tp2};_di0%{~){%u7N2y3=W0+(d&Nzx4^%^ z+u;%r-Cy+nqv0;}{SU!Ka6U}KOW>#I`r_|@H8kLK*aRoR1`t1he+Kadkn;d91M&Hn zvjP4T9!1apHQWRrgUcZYa!$Z0uoea&1>Z#fzY$&!TVWg?M#uj&dM^ue>> z>F_1={lAARL3{&tf!F|kh@SshxE#)a{r# zYeDu2eD@Ii6<{A+4kyD);d$^)^m;i5@C|S_oD9c;ocZ?%b@fw_I{UHI+X4OABkAmm zmCJT$-@i&PGGS?t(^NEP^_x1J*y5N~tY-g|?x`wOoHB<#)%{|0tdALd;z@+Q3&)C) zj5aEWdz#5?RgA3lA{ojbG%vAb8)dV8D2q0efz&i6gG7UgrX13`n28qJ9G>gG zb_|a~>4@#&MBkMQEg@U9;kH!bYohsf#Ej=YEpHCY1fiC)LFfXAC)C|+5bCL6gpi_b zEB2Q1v>&VqLaNY)76U}o9qc24*J3(v}^2|0r6Qt4%wI zso$m@M2oga^R#vl{oOL+u3gBP`SYNhk%;f_vi1;U)6ymoxi10u*O*V)DvQu4`?)=O zVr^zg7>F_0W_^qA!`zRuebUWU|p*=yH1T8REXg8q1&=#i}d@8m_+|IdS?LG=88 z1<~{6-2XaE!Ex|Qbo!e>&i&sFvd;f=^!S_L-LM@}a2LA!ZSY-?_5E?^gX7^wboGCN zx5EOQ2M?jEi~jy4kTd_zgemBU=fd~V(XWOx;W=;w91fpFNB^+Z*N@@#QFQbFh3nuN zcrRQ4C&TmLm+0s>!<%6zybN9hvY-DpxE3yiSHsKTKJ@ZiMK=doudl)>@C>*Gz5HDu z=k;Fz=R*!=;c)mByi+JCu9fATpc^yT@+fYVO#IjCvTGU5@dMTtGOUlru3! zNtJU#L`Ov_?`#VP#fKmg>rE-j?l)r=+vbLQl+Q6&9LL?fY~Qt>^d#G{UbQ2WmEad} zgsVbd?{ZQV28GUene93nlhQ7+n6qNu_N+UQBroBjyO_*jWa!9Va-Frd?CzXqRC2eH z+v%I>Jw4`BA-f}JsjNkLvnCy0u;xZzHgB%q*-{hph~5RK`l4@4Y}xrF^VW9R7@u*{ zmYzPgNuPHXZ+w<<#N>e`Hoe$gsmWohQI1wz0^CYsqDi&q#+=(V#*y6E=4C}%=A%*i z(H9a?Oi^~eq*BS}>QznSC;^^k`@FoHe&xzC~gnTu! zbz+O+B;xv_`EI7!=1EgGX%g*hyt>hw67Ph3CxVkGmG}dTU~1MURR~QDiwh0T2=OR?C8ksv^|1Jls@U6*x8ukh#8Od zmF=8;9GLUWwV}+5mLa${Sw*Lf&s#FX_JK<2rgTo+{@^VPw%Ua0aB?`f&kq#H0y{FI zWz%V2;t2Y9HAy>##)pjQ6td0VS$OP;$Pd$yisiP(sZXTMXC5iM2Prrh!( zo+{cgCNd+gX2I6>bAFzH(McAFwOfgHiQ@}hlgUK%NS}&g5kmT6M<|KW?QRK~a9}q~ zL|pEk2r4TRI-GUdM1kbcJc|_y?INb@b9KhBopV5oqz*|UchP^*^a^p)tz5=g`>*1l zWDm77<4*fzOHZQg%2IP4C5@j6rTyMaNLohT$7&F&ZQR@F zt(^!A{Y3zKn_^6Ghh2FCu!^Jq3j)OO=PyvRc$AaYA3#Jo;oxA83TA? kXMxq-cZm2{p|^=Vjz23t9QrXmIpM%msg~Kl`=31fKh*qing9R* diff --git a/System/Cmd/Utils.hs b/System/Cmd/Utils.hs deleted file mode 100644 index a81126146b..0000000000 --- a/System/Cmd/Utils.hs +++ /dev/null @@ -1,568 +0,0 @@ --- arch-tag: Command utilities main file -{-# LANGUAGE CPP #-} -{- -Copyright (C) 2004-2006 John Goerzen - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA --} - -{- | - Module : System.Cmd.Utils - Copyright : Copyright (C) 2004-2006 John Goerzen - License : GNU GPL, version 2 or above - - Maintainer : John Goerzen - Stability : provisional - Portability: portable to platforms with POSIX process\/signal tools - -Command invocation utilities. - -Written by John Goerzen, jgoerzen\@complete.org - -Please note: Most of this module is not compatible with Hugs. - -Command lines executed will be logged using "System.Log.Logger" at the -DEBUG level. Failure messages will be logged at the WARNING level in addition -to being raised as an exception. Both are logged under -\"System.Cmd.Utils.funcname\" -- for instance, -\"System.Cmd.Utils.safeSystem\". If you wish to suppress these messages -globally, you can simply run: - -> updateGlobalLogger "System.Cmd.Utils.safeSystem" -> (setLevel CRITICAL) - -See also: 'System.Log.Logger.updateGlobalLogger', -"System.Log.Logger". - -It is possible to set up pipelines with these utilities. Example: - -> (pid1, x1) <- pipeFrom "ls" ["/etc"] -> (pid2, x2) <- pipeBoth "grep" ["x"] x1 -> putStr x2 -> ... the grep output is displayed ... -> forceSuccess pid2 -> forceSuccess pid1 - -Remember, when you use the functions that return a String, you must not call -'forceSuccess' until after all data from the String has been consumed. Failure -to wait will cause your program to appear to hang. - -Here is an example of the wrong way to do it: - -> (pid, x) <- pipeFrom "ls" ["/etc"] -> forceSuccess pid -- Hangs; the called program hasn't terminated yet -> processTheData x - -You must instead process the data before calling 'forceSuccess'. - -When using the hPipe family of functions, this is probably more obvious. - -Most of this module will be incompatible with Windows. --} - - -module System.Cmd.Utils(-- * High-Level Tools - PipeHandle(..), - safeSystem, -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) - forceSuccess, -#ifndef __HUGS__ - posixRawSystem, - forkRawSystem, - -- ** Piping with lazy strings - pipeFrom, - pipeLinesFrom, - pipeTo, - pipeBoth, - -- ** Piping with handles - hPipeFrom, - hPipeTo, - hPipeBoth, -#endif -#endif - -- * Low-Level Tools - PipeMode(..), -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ - pOpen, pOpen3, pOpen3Raw -#endif -#endif - ) -where - --- FIXME - largely obsoleted by 6.4 - convert to wrappers. - -import System.Exit -import System.Cmd -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -import System.Posix.IO -import System.Posix.Process -import System.Posix.Signals -import qualified System.Posix.Signals -#endif -import System.Posix.Types -import System.IO -import System.IO.Error -import Control.Concurrent(forkIO) -import Control.Exception(finally) - -data PipeMode = ReadFromPipe | WriteToPipe - -logbase :: String -logbase = "System.Cmd.Utils" - -{- | Return value from 'pipeFrom', 'pipeLinesFrom', 'pipeTo', or -'pipeBoth'. Contains both a ProcessID and the original command that was -executed. If you prefer not to use 'forceSuccess' on the result of one -of these pipe calls, you can use (processID ph), assuming ph is your 'PipeHandle', -as a parameter to 'System.Posix.Process.getProcessStatus'. -} -data PipeHandle = - PipeHandle { processID :: ProcessID, - phCommand :: FilePath, - phArgs :: [String], - phCreator :: String -- ^ Function that created it - } - deriving (Eq, Show) - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Like 'pipeFrom', but returns data in lines instead of just a String. -Shortcut for calling lines on the result from 'pipeFrom'. - -Note: this function logs as pipeFrom. - -Not available on Windows. -} -pipeLinesFrom :: FilePath -> [String] -> IO (PipeHandle, [String]) -pipeLinesFrom fp args = - do (pid, c) <- pipeFrom fp args - return $ (pid, lines c) -#endif -#endif - -logRunning :: String -> FilePath -> [String] -> IO () -logRunning func fp args = return () --debugM (logbase ++ "." ++ func) (showCmd fp args) - -warnFail :: [Char] -> FilePath -> [String] -> [Char] -> IO t -warnFail funcname fp args msg = - let m = showCmd fp args ++ ": " ++ msg - in do putStrLn m - fail m - -ddd s a = do - putStrLn $ s ++ " start" - r <- a - putStrLn $ s ++ " end" - return r - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Read data from a pipe. Returns a Handle and a 'PipeHandle'. - -When done, you must hClose the handle, and then use either 'forceSuccess' or -getProcessStatus on the 'PipeHandle'. Zombies will result otherwise. - -This function logs as pipeFrom. - -Not available on Windows or with Hugs. --} -hPipeFrom :: FilePath -> [String] -> IO (PipeHandle, Handle) -hPipeFrom fp args = - ddd (show ("hPipeFrom", fp, args)) $ do - pipepair <- createPipe - let childstuff = do dupTo (snd pipepair) stdOutput - closeFd (fst pipepair) - executeFile fp True args Nothing - p <- try (forkProcess childstuff) - -- parent - pid <- case p of - Right x -> return x - Left e -> warnFail "pipeFrom" fp args $ - "Error in fork: " ++ show e - closeFd (snd pipepair) - h <- fdToHandle (fst pipepair) - return (PipeHandle pid fp args "pipeFrom", h) -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Read data from a pipe. Returns a lazy string and a 'PipeHandle'. - -ONLY AFTER the string has been read completely, You must call either -'System.Posix.Process.getProcessStatus' or 'forceSuccess' on the 'PipeHandle'. -Zombies will result otherwise. - -Not available on Windows. --} -pipeFrom :: FilePath -> [String] -> IO (PipeHandle, String) -pipeFrom fp args = - do (pid, h) <- hPipeFrom fp args - c <- hGetContents h - return (pid, c) -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Write data to a pipe. Returns a 'PipeHandle' and a new Handle to write -to. - -When done, you must hClose the handle, and then use either 'forceSuccess' or -getProcessStatus on the 'PipeHandle'. Zombies will result otherwise. - -This function logs as pipeTo. - -Not available on Windows. --} -hPipeTo :: FilePath -> [String] -> IO (PipeHandle, Handle) -hPipeTo fp args = - ddd "hPipeTo" $ do - pipepair <- createPipe - let childstuff = do dupTo (fst pipepair) stdInput - closeFd (snd pipepair) - executeFile fp True args Nothing - p <- try (forkProcess childstuff) - -- parent - pid <- case p of - Right x -> return x - Left e -> warnFail "pipeTo" fp args $ - "Error in fork: " ++ show e - closeFd (fst pipepair) - h <- fdToHandle (snd pipepair) - return (PipeHandle pid fp args "pipeTo", h) -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Write data to a pipe. Returns a ProcessID. - -You must call either -'System.Posix.Process.getProcessStatus' or 'forceSuccess' on the ProcessID. -Zombies will result otherwise. - -Not available on Windows. --} -pipeTo :: FilePath -> [String] -> String -> IO PipeHandle -pipeTo fp args message = - do (pid, h) <- hPipeTo fp args - finally (hPutStr h message) - (hClose h) - return pid -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Like a combination of 'hPipeTo' and 'hPipeFrom'; returns -a 3-tuple of ('PipeHandle', Data From Pipe, Data To Pipe). - -When done, you must hClose both handles, and then use either 'forceSuccess' or -getProcessStatus on the 'PipeHandle'. Zombies will result otherwise. - -Hint: you will usually need to ForkIO a thread to handle one of the Handles; -otherwise, deadlock can result. - -This function logs as pipeBoth. - -Not available on Windows. --} -hPipeBoth :: FilePath -> [String] -> IO (PipeHandle, Handle, Handle) -hPipeBoth fp args = - ddd (show ("hPipeBoth", fp, args)) $ do - frompair <- createPipe - topair <- createPipe - let childstuff = do dupTo (snd frompair) stdOutput - closeFd (fst frompair) - dupTo (fst topair) stdInput - closeFd (snd topair) - executeFile fp True args Nothing - p <- try (forkProcess childstuff) - -- parent - pid <- case p of - Right x -> return x - Left e -> warnFail "pipeBoth" fp args $ - "Error in fork: " ++ show e - closeFd (snd frompair) - closeFd (fst topair) - fromh <- fdToHandle (fst frompair) - toh <- fdToHandle (snd topair) - return (PipeHandle pid fp args "pipeBoth", fromh, toh) -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Like a combination of 'pipeTo' and 'pipeFrom'; forks an IO thread -to send data to the piped program, and simultaneously returns its output -stream. - -The same note about checking the return status applies here as with 'pipeFrom'. - -Not available on Windows. -} -pipeBoth :: FilePath -> [String] -> String -> IO (PipeHandle, String) -pipeBoth fp args message = - do (pid, fromh, toh) <- hPipeBoth fp args - forkIO $ finally (hPutStr toh message) - (hClose toh) - c <- hGetContents fromh - return (pid, c) -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -{- | Uses 'System.Posix.Process.getProcessStatus' to obtain the exit status -of the given process ID. If the process terminated normally, does nothing. -Otherwise, raises an exception with an appropriate error message. - -This call will block waiting for the given pid to terminate. - -Not available on Windows. -} -forceSuccess :: PipeHandle -> IO () -forceSuccess (PipeHandle pid fp args funcname) = - let warnfail = warnFail funcname - in do status <- getProcessStatus True False pid - case status of - Nothing -> warnfail fp args $ "Got no process status" - Just (Exited (ExitSuccess)) -> return () - Just (Exited (ExitFailure fc)) -> - cmdfailed funcname fp args fc - Just (Terminated sig) -> - warnfail fp args $ "Terminated by signal " ++ show sig - Just (Stopped sig) -> - warnfail fp args $ "Stopped by signal " ++ show sig -#endif - -{- | Invokes the specified command in a subprocess, waiting for the result. -If the command terminated successfully, return normally. Otherwise, -raises a userError with the problem. - -Implemented in terms of 'posixRawSystem' where supported, and System.Posix.rawSystem otherwise. --} -safeSystem :: FilePath -> [String] -> IO () -safeSystem command args = - ddd "safeSystem" $ do -#if defined(__HUGS__) || defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__) - ec <- rawSystem command args - case ec of - ExitSuccess -> return () - ExitFailure fc -> cmdfailed "safeSystem" command args fc -#else - ec <- posixRawSystem command args - case ec of - Exited ExitSuccess -> return () - Exited (ExitFailure fc) -> cmdfailed "safeSystem" command args fc - Terminated s -> cmdsignalled "safeSystem" command args s - Stopped s -> cmdsignalled "safeSystem" command args s -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Invokes the specified command in a subprocess, waiting for the result. -Return the result status. Never raises an exception. Only available -on POSIX platforms. - -Like system(3), this command ignores SIGINT and SIGQUIT and blocks SIGCHLD -during its execution. - -Logs as System.Cmd.Utils.posixRawSystem -} -posixRawSystem :: FilePath -> [String] -> IO ProcessStatus -posixRawSystem program args = - ddd "posixRawSystem" $ do - oldint <- installHandler sigINT Ignore Nothing - oldquit <- installHandler sigQUIT Ignore Nothing - let sigset = addSignal sigCHLD emptySignalSet - oldset <- getSignalMask - blockSignals sigset - childpid <- forkProcess (childaction oldint oldquit oldset) - - mps <- getProcessStatus True False childpid - restoresignals oldint oldquit oldset - let retval = case mps of - Just x -> x - Nothing -> error "Nothing returned from getProcessStatus" - return retval - - where childaction oldint oldquit oldset = - do restoresignals oldint oldquit oldset - executeFile program True args Nothing - restoresignals oldint oldquit oldset = - do installHandler sigINT oldint Nothing - installHandler sigQUIT oldquit Nothing - setSignalMask oldset - -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Invokes the specified command in a subprocess, without waiting for -the result. Returns the PID of the subprocess -- it is YOUR responsibility -to use getProcessStatus or getAnyProcessStatus on that at some point. Failure -to do so will lead to resource leakage (zombie processes). - -This function does nothing with signals. That too is up to you. - -Logs as System.Cmd.Utils.forkRawSystem -} -forkRawSystem :: FilePath -> [String] -> IO ProcessID -forkRawSystem program args = ddd "forkRawSystem" $ - do - forkProcess childaction - where - childaction = executeFile program True args Nothing - -#endif -#endif - -cmdfailed :: String -> FilePath -> [String] -> Int -> IO a -cmdfailed funcname command args failcode = do - let errormsg = "Command " ++ command ++ " " ++ (show args) ++ - " failed; exit code " ++ (show failcode) - let e = userError (errormsg) - putStrLn errormsg - ioError e - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -cmdsignalled :: String -> FilePath -> [String] -> Signal -> IO a -cmdsignalled funcname command args failcode = do - let errormsg = "Command " ++ command ++ " " ++ (show args) ++ - " failed due to signal " ++ (show failcode) - let e = userError (errormsg) - putStrLn errormsg - ioError e -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Open a pipe to the specified command. - -Passes the handle on to the specified function. - -The 'PipeMode' specifies what you will be doing. That is, specifing 'ReadFromPipe' -sets up a pipe from stdin, and 'WriteToPipe' sets up a pipe from stdout. - -Not available on Windows. - -} -pOpen :: PipeMode -> FilePath -> [String] -> - (Handle -> IO a) -> IO a -pOpen pm fp args func = ddd "pOpen" $ - do - pipepair <- createPipe - case pm of - ReadFromPipe -> do - let callfunc _ = do - closeFd (snd pipepair) - h <- fdToHandle (fst pipepair) - x <- func h - hClose h - return $! x - pOpen3 Nothing (Just (snd pipepair)) Nothing fp args - callfunc (closeFd (fst pipepair)) - WriteToPipe -> do - let callfunc _ = do - closeFd (fst pipepair) - h <- fdToHandle (snd pipepair) - x <- func h - hClose h - return $! x - pOpen3 (Just (fst pipepair)) Nothing Nothing fp args - callfunc (closeFd (snd pipepair)) -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Runs a command, redirecting things to pipes. - -Not available on Windows. - -Note that you may not use the same fd on more than one item. If you -want to redirect stdout and stderr, dup it first. --} -pOpen3 :: Maybe Fd -- ^ Send stdin to this fd - -> Maybe Fd -- ^ Get stdout from this fd - -> Maybe Fd -- ^ Get stderr from this fd - -> FilePath -- ^ Command to run - -> [String] -- ^ Command args - -> (ProcessID -> IO a) -- ^ Action to run in parent - -> IO () -- ^ Action to run in child before execing (if you don't need something, set this to @return ()@) -- IGNORED IN HUGS - -> IO a -pOpen3 pin pout perr fp args func childfunc = ddd (show ("pOpen3", fp, args)) $ - do pid <- pOpen3Raw pin pout perr fp args childfunc - putStrLn "got pid" - retval <- func $! pid - putStrLn "got retval" - let rv = seq retval retval - forceSuccess (PipeHandle (seq retval pid) fp args "pOpen3") - putStrLn "process finished" - return rv -#endif -#endif - -#if !(defined(mingw32_HOST_OS) || defined(mingw32_TARGET_OS) || defined(__MINGW32__)) -#ifndef __HUGS__ -{- | Runs a command, redirecting things to pipes. - -Not available on Windows. - -Returns immediately with the PID of the child. Using 'waitProcess' on it -is YOUR responsibility! - -Note that you may not use the same fd on more than one item. If you -want to redirect stdout and stderr, dup it first. --} -pOpen3Raw :: Maybe Fd -- ^ Send stdin to this fd - -> Maybe Fd -- ^ Get stdout from this fd - -> Maybe Fd -- ^ Get stderr from this fd - -> FilePath -- ^ Command to run - -> [String] -- ^ Command args - -> IO () -- ^ Action to run in child before execing (if you don't need something, set this to @return ()@) -- IGNORED IN HUGS - -> IO ProcessID -pOpen3Raw pin pout perr fp args childfunc = - let mayberedir Nothing _ = return () - mayberedir (Just fromfd) tofd = do - dupTo fromfd tofd - closeFd fromfd - return () - childstuff = do - mayberedir pin stdInput - mayberedir pout stdOutput - mayberedir perr stdError - childfunc - executeFile fp True args Nothing -{- - realfunc p = do - System.Posix.Signals.installHandler - System.Posix.Signals.sigPIPE - System.Posix.Signals.Ignore - Nothing - func p --} - in - ddd "pOpen3Raw" $ - do - p <- try (forkProcess childstuff) - pid <- case p of - Right x -> return x - Left e -> fail ("Error in fork: " ++ (show e)) - return pid - -#endif -#endif - -showCmd :: FilePath -> [String] -> String -showCmd fp args = fp ++ " " ++ show args diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs index 9fa8d864fe..d3b0c46efc 100644 --- a/Utility/CoProcess.hs +++ b/Utility/CoProcess.hs @@ -13,23 +13,25 @@ module Utility.CoProcess ( query ) where -import System.Cmd.Utils +import System.Process import Common -type CoProcessHandle = (PipeHandle, Handle, Handle) +type CoProcessHandle = (ProcessHandle, Handle, Handle, FilePath, [String]) start :: FilePath -> [String] -> IO CoProcessHandle -start command params = hPipeBoth command params +start command params = do + (from, to, _err, pid) <- runInteractiveProcess command params Nothing Nothing + return (pid, to, from, command, params) stop :: CoProcessHandle -> IO () -stop (pid, from, to) = do +stop (pid, from, to, command, params) = do hClose to hClose from - forceSuccess pid + forceSuccessProcess pid command params query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b -query (_, from, to) send receive = do +query (_, from, to, _, _) send receive = do _ <- send to hFlush to receive from diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs index e13afe5d48..26ac688e3a 100644 --- a/Utility/Gpg.hs +++ b/Utility/Gpg.hs @@ -11,9 +11,9 @@ import qualified Data.ByteString.Lazy as L import System.Posix.Types import Control.Applicative import Control.Concurrent -import Control.Exception (finally, bracket) -import System.Exit +import Control.Exception (bracket) import System.Posix.Env (setEnv, unsetEnv, getEnv) +import System.Process import Common @@ -39,18 +39,30 @@ stdParams params = do readStrict :: [CommandParam] -> IO String readStrict params = do params' <- stdParams params - pOpen ReadFromPipe "gpg" params' hGetContentsStrict + (_, Just from, _, pid) + <- createProcess (proc "gpg" params') + { std_out = CreatePipe } + hSetBinaryMode from True + r <- hGetContentsStrict from + forceSuccessProcess pid "gpg" params' + return r {- Runs gpg, piping an input value to it, and returning its stdout, - strictly. -} pipeStrict :: [CommandParam] -> String -> IO String pipeStrict params input = do params' <- stdParams params - (pid, fromh, toh) <- hPipeBoth "gpg" params' - _ <- forkIO $ finally (hPutStr toh input) (hClose toh) - output <- hGetContentsStrict fromh - forceSuccess pid - return output + (Just to, Just from, _, pid) + <- createProcess (proc "gpg" params') + { std_in = CreatePipe + , std_out = CreatePipe } + hSetBinaryMode to True + hSetBinaryMode from True + hPutStr to input + hClose to + r <- hGetContentsStrict from + forceSuccessProcess pid "gpg" params' + return r {- Runs gpg with some parameters, first feeding it a passphrase via - --passphrase-fd, then feeding it an input, and passing a handle @@ -70,17 +82,14 @@ passphraseHandle params passphrase a b = do let passphrasefd = [Param "--passphrase-fd", Param $ show pfd] params' <- stdParams $ passphrasefd ++ params - (pid, fromh, toh) <- hPipeBoth "gpg" params' - pid2 <- forkProcess $ do - L.hPut toh =<< a - hClose toh - exitSuccess + (Just toh, Just fromh, _, pid) <- createProcess (proc "gpg" params') + { std_in = CreatePipe, std_out = CreatePipe } + L.hPut toh =<< a hClose toh ret <- b fromh -- cleanup - forceSuccess pid - _ <- getProcessStatus True False pid2 + forceSuccessProcess pid "gpg" params' closeFd frompipe return ret diff --git a/Utility/INotify.hs b/Utility/INotify.hs index bf87f4e71b..55233ef762 100644 --- a/Utility/INotify.hs +++ b/Utility/INotify.hs @@ -10,6 +10,7 @@ module Utility.INotify where import Common hiding (isDirectory) import Utility.ThreadLock import Utility.Types.DirWatcher +import System.Process import System.INotify import qualified System.Posix.Files as Files @@ -160,12 +161,9 @@ tooManyWatches hook dir = do querySysctl :: Read a => [CommandParam] -> IO (Maybe a) querySysctl ps = do - v <- catchMaybeIO $ hPipeFrom "sysctl" $ toCommand ps + v <- catchMaybeIO $ readProcess "sysctl" (toCommand ps) [] case v of Nothing -> return Nothing - Just (pid, h) -> do - val <- parsesysctl <$> hGetContentsStrict h - void $ getProcessStatus True False $ processID pid - return val + Just s -> return $ parsesysctl s where parsesysctl s = readish =<< lastMaybe (words s) diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs index 0061dfe574..ebd273b2e1 100644 --- a/Utility/Lsof.hs +++ b/Utility/Lsof.hs @@ -12,6 +12,7 @@ module Utility.Lsof where import Common import System.Posix.Types +import System.Process data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown deriving (Show, Eq) @@ -34,10 +35,8 @@ queryDir path = query ["+d", path] -} query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)] query opts = do - (pid, s) <- pipeFrom "lsof" ("-F0can" : opts) - let !r = parse s - void $ getProcessStatus True False $ processID pid - return r + (_, s, _) <- readProcessWithExitCode "lsof" ("-F0can" : opts) [] + return $ parse s {- Parsing null-delimited output like: - diff --git a/Utility/Process.hs b/Utility/Process.hs new file mode 100644 index 0000000000..9f79efa813 --- /dev/null +++ b/Utility/Process.hs @@ -0,0 +1,40 @@ +{- System.Process enhancements + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Utility.Process where + +import System.Process +import System.Exit +import System.IO + +import Utility.Misc + +{- Waits for a ProcessHandle, and throws an exception if the process + - did not exit successfully. -} +forceSuccessProcess :: ProcessHandle -> String -> [String] -> IO () +forceSuccessProcess pid cmd args = do + code <- waitForProcess pid + case code of + ExitSuccess -> return () + ExitFailure n -> error $ + cmd ++ " " ++ show args ++ " exited " ++ show n + +{- Like readProcess, but allows specifying the environment, and does + - not mess with stdin. -} +readProcessEnv :: FilePath -> [String] -> Maybe [(String, String)] -> IO String +readProcessEnv cmd args environ = do + (_, Just h, _, pid) + <- createProcess (proc cmd args) + { std_in = Inherit + , std_out = CreatePipe + , std_err = Inherit + , env = environ + } + output <- hGetContentsStrict h + hClose h + forceSuccessProcess pid cmd args + return output diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs index 5f6a53e715..47280a40b1 100644 --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -1,6 +1,6 @@ {- safely running shell commands - - - Copyright 2010-2011 Joey Hess + - Copyright 2010-2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -8,11 +8,8 @@ module Utility.SafeCommand where import System.Exit -import qualified System.Posix.Process -import System.Posix.Process hiding (executeFile) -import System.Posix.Signals +import System.Process import Data.String.Utils -import System.Log.Logger import Control.Applicative {- A type for parameters passed to a shell command. A command can @@ -42,7 +39,7 @@ boolSystem :: FilePath -> [CommandParam] -> IO Bool boolSystem command params = boolSystemEnv command params Nothing boolSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO Bool -boolSystemEnv command params env = dispatch <$> safeSystemEnv command params env +boolSystemEnv command params environ = dispatch <$> safeSystemEnv command params environ where dispatch ExitSuccess = True dispatch _ = False @@ -51,41 +48,13 @@ boolSystemEnv command params env = dispatch <$> safeSystemEnv command params env safeSystem :: FilePath -> [CommandParam] -> IO ExitCode safeSystem command params = safeSystemEnv command params Nothing -{- SIGINT(ctrl-c) is allowed to propigate and will terminate the program. -} +{- Unlike many implementations of system, SIGINT(ctrl-c) is allowed + - to propigate and will terminate the program. -} safeSystemEnv :: FilePath -> [CommandParam] -> Maybe [(String, String)] -> IO ExitCode -safeSystemEnv command params env = do - putStrLn "safeSystemEnv start" - -- Going low-level because all the high-level system functions - -- block SIGINT etc. We need to block SIGCHLD, but allow - -- SIGINT to do its default program termination. - let sigset = addSignal sigCHLD emptySignalSet - oldint <- installHandler sigINT Default Nothing - oldset <- getSignalMask - blockSignals sigset - childpid <- forkProcess $ childaction oldint oldset - mps <- getProcessStatus True False childpid - restoresignals oldint oldset - case mps of - Just (Exited code) -> do - putStrLn "safeSystemEnv end" - return code - _ -> error $ "unknown error running " ++ command - where - restoresignals oldint oldset = do - _ <- installHandler sigINT oldint Nothing - setSignalMask oldset - childaction oldint oldset = do - restoresignals oldint oldset - executeFile command True (toCommand params) env - -{- executeFile with debug logging -} -executeFile :: FilePath -> Bool -> [String] -> Maybe [(String, String)] -> IO () -executeFile c path p e = do - putStrLn "executeFile start" - --debugM "Utility.SafeCommand.executeFile" $ - -- "Running: " ++ c ++ " " ++ show p ++ " " ++ maybe "" show e - System.Posix.Process.executeFile c path p e - putStrLn "executeFile end" +safeSystemEnv command params environ = do + (_, _, _, pid) <- createProcess (proc command $ toCommand params) + { env = environ } + waitForProcess pid {- Escapes a filename or other parameter to be safely able to be exposed to - the shell. -} diff --git a/Utility/TempFile.hs b/Utility/TempFile.hs index 4dcbf1cca4..62e0fc8596 100644 --- a/Utility/TempFile.hs +++ b/Utility/TempFile.hs @@ -9,7 +9,7 @@ module Utility.TempFile where import Control.Exception (bracket) import System.IO -import System.Posix.Process hiding (executeFile) +import System.Posix.Process import System.Directory import Utility.Exception diff --git a/doc/todo/assistant_threaded_runtime.mdwn b/doc/todo/assistant_threaded_runtime.mdwn index edfa51669f..412f52ae81 100644 --- a/doc/todo/assistant_threaded_runtime.mdwn +++ b/doc/todo/assistant_threaded_runtime.mdwn @@ -23,6 +23,9 @@ git-annex does not otherwise use threads, so this is surprising. --[[Joey]] > I've spent a lot of time debugging this, and trying to fix it, in the > "threaded" branch. There are still deadlocks. --[[Joey]] +>> Fixed, by switching from `System.Cmd.Utils` to `System.Process` +>> --[[Joey]] + --- It would be possible to not use the threaded runtime. Instead, we could diff --git a/git-annex.cabal b/git-annex.cabal index 3f237ce70e..e58bd4d957 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -40,11 +40,12 @@ Executable git-annex unix, containers, utf8-string, network, mtl, bytestring, old-locale, time, pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP, base == 4.5.*, monad-control, transformers-base, lifted-base, - IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance + IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process -- Need to list this because it's generated from a .hsc file. Other-Modules: Utility.Touch C-Sources: Utility/libdiskfree.c Extensions: CPP + GHC-Options: -threaded if flag(S3) Build-Depends: hS3 @@ -65,10 +66,11 @@ Test-Suite test unix, containers, utf8-string, network, mtl, bytestring, old-locale, time, pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP, base == 4.5.*, monad-control, transformers-base, lifted-base, - IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance + IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process Other-Modules: Utility.Touch C-Sources: Utility/libdiskfree.c Extensions: CPP + GHC-Options: -threaded source-repository head type: git diff --git a/test.hs b/test.hs index 9de73264ee..a377057c28 100644 --- a/test.hs +++ b/test.hs @@ -14,6 +14,7 @@ import Test.QuickCheck import System.Posix.Directory (changeWorkingDirectory) import System.Posix.Files import System.Posix.Env +import System.Posix.Process import Control.Exception.Extensible import qualified Data.Map as M import System.IO.HVFS (SystemFS(..)) From f520a2c10359257ea6f920d26fe29831db8d07e3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 18:29:33 -0400 Subject: [PATCH 4133/8313] add missing imports --- Assistant/ThreadedMonad.hs | 1 + Assistant/Threads/TransferWatcher.hs | 1 + Utility/Parallel.hs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 16f3a9dd9f..f32adff435 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -13,6 +13,7 @@ import qualified Annex import Control.Concurrent import Data.Tuple import System.Posix.Types +import System.Posix.Process {- The Annex state is stored in a MVar, so that threaded actions can access - it. -} diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 5be63fce4f..766c1f89e8 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -17,6 +17,7 @@ import Utility.Types.DirWatcher import Annex.BranchState import Data.Map as M +import System.Posix.Process {- This thread watches for changes to the gitAnnexTransferDir, - and updates the DaemonStatus's map of ongoing transfers. -} diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs index 9df95ab2b0..f4a79316c7 100644 --- a/Utility/Parallel.hs +++ b/Utility/Parallel.hs @@ -9,6 +9,8 @@ module Utility.Parallel where import Common +import System.Posix.Process + {- Runs an action in parallel with a set of values. - Returns the values partitioned into ones with which the action succeeded, - and ones with which it failed. -} From eea0a3616cd1cbaf033649c11a5c2b650b6b632f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 18:42:41 -0400 Subject: [PATCH 4134/8313] add thread id field to transferinfo Also converted its timestand to posix seconds, like is used in the other log files. --- Assistant/ThreadedMonad.hs | 10 ++++------ Assistant/Threads/Transferrer.hs | 16 ++++++++-------- Assistant/TransferQueue.hs | 1 + Logs/Transfer.hs | 15 ++++++++++++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index f32adff435..2fc5265997 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -12,8 +12,6 @@ import qualified Annex import Control.Concurrent import Data.Tuple -import System.Posix.Types -import System.Posix.Process {- The Annex state is stored in a MVar, so that threaded actions can access - it. -} @@ -39,14 +37,14 @@ withThreadState a = do runThreadState :: ThreadState -> Annex a -> IO a runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a -{- Runs an Annex action in a separate process, using a copy of the state +{- Runs an Annex action in a separate thread, using a copy of the state - from the MVar. - - It's up to the action to perform any necessary shutdown tasks in order - for state to not be lost. And it's up to the caller to resynchronise - with any changes the action makes to eg, the git-annex branch. -} -unsafeForkProcessThreadState :: ThreadState -> Annex a -> IO ProcessID -unsafeForkProcessThreadState mvar a = do +unsafeForkIOThreadState :: ThreadState -> Annex a -> IO ThreadId +unsafeForkIOThreadState mvar a = do state <- readMVar mvar - forkProcess $ void $ Annex.eval state a + forkIO $ void $ Annex.eval state a diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index 9d3358f546..dd63d4d128 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -18,6 +18,7 @@ import Logs.Location import Annex.Content import qualified Remote +import Data.Time.Clock.POSIX import Data.Time.Clock import qualified Data.Map as M @@ -58,12 +59,12 @@ shouldTransfer dstatus t info = | otherwise = return False key = transferKey t -{- A transfer is run in a separate process, with a *copy* of the Annex +{- A transfer is run in a separate thread, with a *copy* of the Annex - state. This is necessary to avoid blocking the rest of the assistant - on the transfer completing, and also to allow multiple transfers to run - - at once. + - at once. This requires GHC's threaded runtime to work! - - - However, it means that the transfer processes are responsible + - The copy of state means that the transfer processes are responsible - for doing any necessary shutdown cleanups, and that the parent - thread's cache must be invalidated once a transfer completes, as - changes may have been made to the git-annex branch. @@ -73,15 +74,14 @@ runTransfer st dstatus slots t info = case (transferRemote info, associatedFile (Nothing, _) -> noop (_, Nothing) -> noop (Just remote, Just file) -> do - pid <- inTransferSlot slots $ - unsafeForkProcessThreadState st $ + tid <- inTransferSlot slots $ + unsafeForkIOThreadState st $ transferprocess remote file now <- getCurrentTime runThreadState st $ adjustTransfers dstatus $ M.insertWith' const t info - { startedTime = Just now - , transferPid = Just pid - , shouldWait = True + { startedTime = Just $ utcTimeToPOSIXSeconds now + , transferTid = Just tid } where isdownload = transferDirection t == Download diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index 73e73ca0af..fb7fa87cdf 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -24,6 +24,7 @@ stubInfo :: AssociatedFile -> TransferInfo stubInfo f = TransferInfo { startedTime = Nothing , transferPid = Nothing + , transferTid = Nothing , transferRemote = Nothing , bytesComplete = Nothing , associatedFile = f diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 2605120674..f74d128dc9 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -16,6 +16,10 @@ import qualified Fields import System.Posix.Types import Data.Time.Clock +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale +import Control.Concurrent {- Enough information to uniquely identify a transfer, used as the filename - of the transfer information file. -} @@ -33,8 +37,9 @@ data Transfer = Transfer - of some repository, that was acted on to initiate the transfer. -} data TransferInfo = TransferInfo - { startedTime :: Maybe UTCTime + { startedTime :: Maybe POSIXTime , transferPid :: Maybe ProcessID + , transferTid :: Maybe ThreadId , transferRemote :: Maybe Remote , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath @@ -76,8 +81,9 @@ transfer t file a = do createAnnexDirectory $ takeDirectory tfile mode <- annexFileMode info <- liftIO $ TransferInfo - <$> (Just <$> getCurrentTime) + <$> (Just . utcTimeToPOSIXSeconds <$> getCurrentTime) <*> pure Nothing -- pid not stored in file, so omitted for speed + <*> pure Nothing -- tid ditto <*> pure Nothing -- not 0; transfer may be resuming <*> pure Nothing <*> pure file @@ -168,13 +174,16 @@ readTransferInfo :: ProcessID -> String -> Maybe TransferInfo readTransferInfo pid s = case bits of [time] -> TransferInfo - <$> readish time + <$> parsetime time <*> pure (Just pid) <*> pure Nothing <*> pure Nothing + <*> pure Nothing <*> pure (if null filename then Nothing else Just filename) <*> pure False _ -> Nothing where (bits, filebits) = splitAt 1 $ lines s filename = join "\n" filebits + parsetime t = Just . utcTimeToPOSIXSeconds + <$> parseTime defaultTimeLocale "%s%Qs" t From cf47bb3f509ae63ad868b66c0b6f2baecb93e4c7 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 19:13:56 -0400 Subject: [PATCH 4135/8313] run file transfers in threads, not processes This should fix OSX/BSD issues with not noticing transfer information files with kqueue. Now that threads are used, the thread can manage the transfer slot allocation and deallocation by itself; much cleaner. --- Assistant.hs | 2 +- Assistant/ThreadedMonad.hs | 9 +++--- Assistant/Threads/TransferWatcher.hs | 46 +++++++++------------------- Assistant/Threads/Transferrer.hs | 5 ++- Assistant/TransferQueue.hs | 1 - Assistant/TransferSlots.hs | 16 +++++----- Logs/Transfer.hs | 3 -- 7 files changed, 29 insertions(+), 53 deletions(-) diff --git a/Assistant.hs b/Assistant.hs index 91ebf2d2e0..06484b0862 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -123,7 +123,7 @@ startDaemon assistant foreground , pushThread st dstatus commitchan pushmap , pushRetryThread st pushmap , mergeThread st - , transferWatcherThread st dstatus transferslots + , transferWatcherThread st dstatus , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan diff --git a/Assistant/ThreadedMonad.hs b/Assistant/ThreadedMonad.hs index 2fc5265997..1decd8e913 100644 --- a/Assistant/ThreadedMonad.hs +++ b/Assistant/ThreadedMonad.hs @@ -37,14 +37,13 @@ withThreadState a = do runThreadState :: ThreadState -> Annex a -> IO a runThreadState mvar a = modifyMVar mvar $ \state -> swap <$> Annex.run state a -{- Runs an Annex action in a separate thread, using a copy of the state - - from the MVar. +{- Runs an Annex action, using a copy of the state from the MVar. - - It's up to the action to perform any necessary shutdown tasks in order - for state to not be lost. And it's up to the caller to resynchronise - with any changes the action makes to eg, the git-annex branch. -} -unsafeForkIOThreadState :: ThreadState -> Annex a -> IO ThreadId -unsafeForkIOThreadState mvar a = do +unsafeRunThreadState :: ThreadState -> Annex a -> IO () +unsafeRunThreadState mvar a = do state <- readMVar mvar - forkIO $ void $ Annex.eval state a + void $ Annex.eval state a diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 766c1f89e8..364ce04689 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -10,23 +10,20 @@ module Assistant.Threads.TransferWatcher where import Common.Annex import Assistant.ThreadedMonad import Assistant.DaemonStatus -import Assistant.TransferSlots import Logs.Transfer import Utility.DirWatcher import Utility.Types.DirWatcher -import Annex.BranchState import Data.Map as M -import System.Posix.Process {- This thread watches for changes to the gitAnnexTransferDir, - and updates the DaemonStatus's map of ongoing transfers. -} -transferWatcherThread :: ThreadState -> DaemonStatusHandle -> TransferSlots -> IO () -transferWatcherThread st dstatus transferslots = do +transferWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () +transferWatcherThread st dstatus = do g <- runThreadState st $ fromRepo id let dir = gitAnnexTransferDir g createDirectoryIfMissing True dir - let hook a = Just $ runHandler st dstatus transferslots a + let hook a = Just $ runHandler st dstatus a let hooks = mkWatchHooks { addHook = hook onAdd , delHook = hook onDel @@ -34,51 +31,36 @@ transferWatcherThread st dstatus transferslots = do } void $ watchDir dir (const False) hooks id -type Handler = ThreadState -> DaemonStatusHandle -> TransferSlots -> FilePath -> Maybe FileStatus -> IO () +type Handler = ThreadState -> DaemonStatusHandle -> FilePath -> Maybe FileStatus -> IO () {- Runs an action handler. - - Exceptions are ignored, otherwise a whole thread could be crashed. -} -runHandler :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler st dstatus transferslots handler file filestatus = void $ do +runHandler :: ThreadState -> DaemonStatusHandle -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler st dstatus handler file filestatus = void $ do either print (const noop) =<< tryIO go where - go = handler st dstatus transferslots file filestatus + go = handler st dstatus file filestatus {- Called when there's an error with inotify. -} onErr :: Handler -onErr _ _ _ msg _ = error msg +onErr _ _ msg _ = error msg {- Called when a new transfer information file is written. -} onAdd :: Handler -onAdd st dstatus _ file _ = case parseTransferFile file of +onAdd st dstatus file _ = case parseTransferFile file of Nothing -> noop Just t -> runThreadState st $ go t =<< checkTransfer t where go _ Nothing = noop -- transfer already finished go t (Just info) = adjustTransfers dstatus $ M.insertWith' merge t info - -- preseve shouldWait flag, which is not written to disk - merge new old = new { shouldWait = shouldWait old } + -- preseve transferTid, which is not written to disk + merge new old = new { transferTid = transferTid old } -{- Called when a transfer information file is removed. - - - - When the transfer process is a child of this process, wait on it - - to avoid zombies. - -} +{- Called when a transfer information file is removed. -} onDel :: Handler -onDel st dstatus transferslots file _ = case parseTransferFile file of +onDel st dstatus file _ = case parseTransferFile file of Nothing -> noop - Just t -> maybe noop waitchild - =<< runThreadState st (removeTransfer dstatus t) - where - waitchild info - | shouldWait info = case transferPid info of - Nothing -> noop - Just pid -> do - void $ tryIO $ - getProcessStatus True False pid - runThreadState st invalidateCache - transferComplete transferslots - | otherwise = noop + Just t -> void $ runThreadState st $ removeTransfer dstatus t diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index dd63d4d128..c439d8b7ed 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -74,9 +74,8 @@ runTransfer st dstatus slots t info = case (transferRemote info, associatedFile (Nothing, _) -> noop (_, Nothing) -> noop (Just remote, Just file) -> do - tid <- inTransferSlot slots $ - unsafeForkIOThreadState st $ - transferprocess remote file + tid <- inTransferSlot slots st $ + transferprocess remote file now <- getCurrentTime runThreadState st $ adjustTransfers dstatus $ M.insertWith' const t info diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index fb7fa87cdf..b0eca96c84 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -28,7 +28,6 @@ stubInfo f = TransferInfo , transferRemote = Nothing , bytesComplete = Nothing , associatedFile = f - , shouldWait = False } {- Adds pending transfers to the end of the queue for some of the known diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs index 1859b281bb..dc077254d9 100644 --- a/Assistant/TransferSlots.hs +++ b/Assistant/TransferSlots.hs @@ -10,6 +10,9 @@ module Assistant.TransferSlots where import Control.Exception import Control.Concurrent +import Common.Annex +import Assistant.ThreadedMonad + type TransferSlots = QSemN {- Number of concurrent transfers allowed to be run from the assistant. @@ -24,16 +27,13 @@ newTransferSlots :: IO TransferSlots newTransferSlots = newQSemN numSlots {- Waits until a transfer slot becomes available, and runs a transfer - - action in the slot. If the action throws an exception, its slot is - - freed here, otherwise it should be freed by the TransferWatcher when - - the transfer is complete. - -} -inTransferSlot :: TransferSlots -> IO a -> IO a -inTransferSlot s a = bracketOnError start abort run + - action in the slot, in its own thread. -} +inTransferSlot :: TransferSlots -> ThreadState -> Annex a -> IO ThreadId +inTransferSlot s st a = forkIO $ bracket_ start done run where start = waitQSemN s 1 - abort = const $ transferComplete s - run = const a + done = transferComplete s + run = unsafeRunThreadState st a {- Call when a transfer is complete. -} transferComplete :: TransferSlots -> IO () diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index f74d128dc9..1e3d0abdb9 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -43,7 +43,6 @@ data TransferInfo = TransferInfo , transferRemote :: Maybe Remote , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath - , shouldWait :: Bool } deriving (Show, Eq, Ord) @@ -87,7 +86,6 @@ transfer t file a = do <*> pure Nothing -- not 0; transfer may be resuming <*> pure Nothing <*> pure file - <*> pure False bracketIO (prep tfile mode info) (cleanup tfile) a where prep tfile mode info = do @@ -180,7 +178,6 @@ readTransferInfo pid s = <*> pure Nothing <*> pure Nothing <*> pure (if null filename then Nothing else Just filename) - <*> pure False _ -> Nothing where (bits, filebits) = splitAt 1 $ lines s From 2edb5d145c66c36d0f5fd90bfb7905989643266a Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 19:25:46 -0400 Subject: [PATCH 4136/8313] rewrote to not use forkProcess That can make the threaded runtime stall.. But it can use threads now! --- Utility/Parallel.hs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs index f4a79316c7..fcab2a90a1 100644 --- a/Utility/Parallel.hs +++ b/Utility/Parallel.hs @@ -1,4 +1,4 @@ -{- parallel processes +{- parallel processing via threads - - Copyright 2012 Joey Hess - @@ -9,16 +9,27 @@ module Utility.Parallel where import Common -import System.Posix.Process +import Control.Concurrent +import Control.Exception -{- Runs an action in parallel with a set of values. +{- Runs an action in parallel with a set of values, in a set of threads. + - In order for the actions to truely run in parallel, requires GHC's + - threaded runtime, + - - Returns the values partitioned into ones with which the action succeeded, - and ones with which it failed. -} inParallel :: (v -> IO ()) -> [v] -> IO ([v], [v]) inParallel a l = do - pids <- mapM (forkProcess . a) l - statuses <- mapM (getProcessStatus True False) pids - return $ reduce $ partition (succeeded . snd) $ zip l statuses + mvars <- mapM thread l + statuses <- mapM takeMVar mvars + return $ reduce $ partition snd $ zip l statuses where - succeeded v = v == Just (Exited ExitSuccess) reduce (x,y) = (map fst x, map fst y) + thread v = do + mvar <- newEmptyMVar + _ <- forkIO $ do + r <- try (a v) :: IO (Either SomeException ()) + case r of + Left _ -> putMVar mvar False + Right _ -> putMVar mvar True + return mvar From 61695f9f41a90b05c61661f9d9d052afcb5783df Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 19:42:29 -0400 Subject: [PATCH 4137/8313] blog for the day --- .../blog/day_39__twice_is_enemy_action.mdwn | 66 +++++++++++++++++++ doc/design/assistant/syncing.mdwn | 14 ++-- doc/todo/assistant_threaded_runtime.mdwn | 5 ++ 3 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn diff --git a/doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn b/doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn new file mode 100644 index 0000000000..14896fcb1b --- /dev/null +++ b/doc/design/assistant/blog/day_39__twice_is_enemy_action.mdwn @@ -0,0 +1,66 @@ +Beating my head against the threaded runtime some more. I can reproduce +one of the hangs consistently by running 1000 git annex add commands +in a loop. It hangs around 1% of the time, reading from `git cat-file`. + +Interestingly, `git cat-file` is not yet running at this point -- +git-annex has forked a child process, but the child has not yet exec'd it. +Stracing the child git-annex, I see it stuck in a futex. Adding tracing, +I see the child never manages to run any code at all. + +This really looks like the problem is once again in MissingH, which uses +`forkProcess`. Which happens to come with a big warning about being very +unsafe, in very subtle ways. Looking at the C code that the newer `process` +library uses when sparning a pipe to a process, it messes around with lots of +things; blocking signals, stopping a timer, etc. Hundreds of lines of C +code to safely start a child process, all doing things that MissingH omits. + +That's the second time I've seemingly isolated a hang in the GHC threaded +runtime to MissingH. + +And so I've started converting git-annex to use the new `process` library, +for running all its external commands. John Goerzen had mentioned `process` +to me once before when I found a nasty bug in MissingH, as the cool new +thing that would probably eliminate the `System.Cmd.Utils` part of MissingH, +but I'd not otherwise heard much about it. (It also seems to have the +benefit of supporting Windows.) + +This is a big change and it's early days, but each time I see a hang, I'm +converting the code to use `process`, and so far the hangs have just gone +away when I do that. + +--- + +Hours later... I've converted *all* of git-annex to use `process`. + +In the er, process, the `--debug` switch stopped printing all the commands +it runs. I may try to restore that later. + +I've not tested everything, but the test suite passes, even when +using the threaded runtime. **MILESTONE** + +Looking forward to getting out of these weeds and back to useful work.. + +--- + +Hours later yet.... The `assistant` branch in git now uses the threaded +runtime. It works beautifully, using proper threads to run file transfers +in. + +That should fix the problem I was seeing on OSX yesterday. Too tired to +test it now. + +-- + +Amazingly, all the assistant's own dozen or so threads and thread +synch variables etc all work great under the threaded runtime. I had +assumed I'd see yet more concurrency problems there when switching to it, +but it all looks good. (Or whatever problems there are are subtle ones?) + +I'm very relieved. The threaded logjam is broken! I had been getting +increasingly worried that not having the threaded runtime available would +make it very difficult to make the assistant perform really well, and cause +problems with the webapp, perhaps preventing me from using Yesod. + +Now it looks like smooth sailing ahead. Still some hard problems, but +it feels like with inotify and kqueue and the threaded runtime all +dealt with, the really hard infrastructure-level problems are behind me. diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 3fe27d5ac6..9b3e3b08e8 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -10,13 +10,6 @@ all the other git clones, at both the git level and the key/value level. on remotes, and transfer. But first, need to ensure that when a remote receives content, and updates its location log, it syncs that update out. -* Transfer watching has a race on kqueue systems, which makes finished - fast transfers not be noticed by the TransferWatcher. Which in turn - prevents the transfer slot being freed and any further transfers - from happening. So, this approach is too fragile to rely on for - maintaining the TransferSlots. Instead, need [[todo/assistant_threaded_runtime]], - which would allow running something for sure when a transfer thread - finishes. ## longer-term TODO @@ -106,3 +99,10 @@ anyway. Annex state monad. **done** * Write transfer control thread, which decides when to launch transfers. **done** +* Transfer watching has a race on kqueue systems, which makes finished + fast transfers not be noticed by the TransferWatcher. Which in turn + prevents the transfer slot being freed and any further transfers + from happening. So, this approach is too fragile to rely on for + maintaining the TransferSlots. Instead, need [[todo/assistant_threaded_runtime]], + which would allow running something for sure when a transfer thread + finishes. **done** diff --git a/doc/todo/assistant_threaded_runtime.mdwn b/doc/todo/assistant_threaded_runtime.mdwn index edfa51669f..3953cf062f 100644 --- a/doc/todo/assistant_threaded_runtime.mdwn +++ b/doc/todo/assistant_threaded_runtime.mdwn @@ -13,6 +13,11 @@ When pulling, pushing, and merging, the assistant runs external git commands, and this does block all other threads. The threaded runtime would really help here. +[[done]]; the assistant now builds with the threaded runtime. +Some work still remains to run certian long-running external git commands +in their own threads to prevent them blocking things, but that is easy to +do, now. --[[Joey]] + --- Currently, git-annex seems unstable when built with the threaded runtime. From 6d70002233d54445eb3e81755531d2f90e75d8f6 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 19:52:07 -0400 Subject: [PATCH 4138/8313] show all --- doc/design/assistant/blog.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/assistant/blog.mdwn b/doc/design/assistant/blog.mdwn index ca9ba59193..5e9b6c62f3 100644 --- a/doc/design/assistant/blog.mdwn +++ b/doc/design/assistant/blog.mdwn @@ -2,4 +2,4 @@ The git-annex assistant is being [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/). I'll be blogging about my progress here on a semi-daily basis. -[[!inline pages="page(design/assistant/blog/*)" show=30]] +[[!inline pages="page(design/assistant/blog/*)" show=0]] From 549f8619998eab17aaf2122f11f036fac7e6cc40 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 20:48:08 -0400 Subject: [PATCH 4139/8313] fix parsing of startedTime --- Logs/Transfer.hs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index 1e3d0abdb9..b6962262d1 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -118,7 +118,7 @@ checkTransfer t = do case locked of Nothing -> return Nothing Just (pid, _) -> liftIO $ - flip catchDefaultIO Nothing $ + flip catchDefaultIO Nothing $ do readTransferInfo pid <$> readFile tfile @@ -163,7 +163,7 @@ writeTransferInfo :: TransferInfo -> String writeTransferInfo info = unlines -- transferPid is not included; instead obtained by looking at -- the process that locks the file. - [ show $ startedTime info + [ maybe "" show $ startedTime info -- bytesComplete is not included; changes too fast , fromMaybe "" $ associatedFile info -- comes last; arbitrary content ] @@ -172,7 +172,7 @@ readTransferInfo :: ProcessID -> String -> Maybe TransferInfo readTransferInfo pid s = case bits of [time] -> TransferInfo - <$> parsetime time + <$> (Just <$> parsePOSIXTime time) <*> pure (Just pid) <*> pure Nothing <*> pure Nothing @@ -182,5 +182,7 @@ readTransferInfo pid s = where (bits, filebits) = splitAt 1 $ lines s filename = join "\n" filebits - parsetime t = Just . utcTimeToPOSIXSeconds - <$> parseTime defaultTimeLocale "%s%Qs" t + +parsePOSIXTime :: String -> Maybe POSIXTime +parsePOSIXTime s = utcTimeToPOSIXSeconds + <$> parseTime defaultTimeLocale "%s%Qs" s From 21d35f88d88676f71ad7669b9dfe398e57d7731c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 18 Jul 2012 21:45:41 -0400 Subject: [PATCH 4140/8313] pull in transfer log code from assistant branch New log file format. --- Command/Status.hs | 4 +-- Logs/Transfer.hs | 75 +++++++++++++++++++++++++++-------------------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/Command/Status.hs b/Command/Status.hs index eff21bb509..2d63c525c3 100644 --- a/Command/Status.hs +++ b/Command/Status.hs @@ -186,8 +186,8 @@ transfer_list = stat "transfers in progress" $ nojson $ lift $ do [ show (transferDirection t) ++ "ing" , fromMaybe (show $ transferKey t) (associatedFile i) , if transferDirection t == Upload then "to" else "from" - , maybe (fromUUID $ transferRemote t) Remote.name $ - M.lookup (transferRemote t) uuidmap + , maybe (fromUUID $ transferUUID t) Remote.name $ + M.lookup (transferUUID t) uuidmap ] disk_size :: Stat diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs index f808cb6a44..b6962262d1 100644 --- a/Logs/Transfer.hs +++ b/Logs/Transfer.hs @@ -1,4 +1,4 @@ -{- git-annex transfer information files +{- git-annex transfer information files and lock files - - Copyright 2012 Joey Hess - @@ -14,15 +14,18 @@ import qualified Git import Types.Remote import qualified Fields -import Control.Concurrent import System.Posix.Types import Data.Time.Clock +import Data.Time.Clock.POSIX +import Data.Time +import System.Locale +import Control.Concurrent {- Enough information to uniquely identify a transfer, used as the filename - of the transfer information file. -} data Transfer = Transfer { transferDirection :: Direction - , transferRemote :: UUID + , transferUUID :: UUID , transferKey :: Key } deriving (Show, Eq, Ord) @@ -34,9 +37,10 @@ data Transfer = Transfer - of some repository, that was acted on to initiate the transfer. -} data TransferInfo = TransferInfo - { startedTime :: Maybe UTCTime + { startedTime :: Maybe POSIXTime , transferPid :: Maybe ProcessID - , transferThread :: Maybe ThreadId + , transferTid :: Maybe ThreadId + , transferRemote :: Maybe Remote , bytesComplete :: Maybe Integer , associatedFile :: Maybe FilePath } @@ -66,9 +70,9 @@ fieldTransfer direction key a = do maybe a (\u -> transfer (Transfer direction (toUUID u) key) afile a) =<< Fields.getField Fields.remoteUUID -{- Runs a transfer action. Creates and locks the transfer information file - - while the action is running. Will throw an error if the transfer is - - already in progress. +{- Runs a transfer action. Creates and locks the lock file while the + - action is running, and stores info in the transfer information + - file. Will throw an error if the transfer is already in progress. -} transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a transfer t file a = do @@ -76,27 +80,27 @@ transfer t file a = do createAnnexDirectory $ takeDirectory tfile mode <- annexFileMode info <- liftIO $ TransferInfo - <$> (Just <$> getCurrentTime) + <$> (Just . utcTimeToPOSIXSeconds <$> getCurrentTime) <*> pure Nothing -- pid not stored in file, so omitted for speed - <*> pure Nothing -- threadid not stored in file, so omitted for speed + <*> pure Nothing -- tid ditto <*> pure Nothing -- not 0; transfer may be resuming + <*> pure Nothing <*> pure file bracketIO (prep tfile mode info) (cleanup tfile) a where prep tfile mode info = do - fd <- openFd tfile ReadWrite (Just mode) + fd <- openFd (transferLockFile tfile) ReadWrite (Just mode) defaultFileFlags { trunc = True } locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0) when (locked == Nothing) $ error $ "transfer already in progress" - h <- fdToHandle fd - hPutStr h $ writeTransferInfo info - hFlush h - return h - cleanup tfile h = do + writeFile tfile $ writeTransferInfo info + return fd + cleanup tfile fd = do removeFile tfile - hClose h + removeFile $ transferLockFile tfile + closeFd fd {- If a transfer is still running, returns its TransferInfo. -} checkTransfer :: Transfer -> Annex (Maybe TransferInfo) @@ -104,22 +108,19 @@ checkTransfer t = do mode <- annexFileMode tfile <- fromRepo $ transferFile t mfd <- liftIO $ catchMaybeIO $ - openFd tfile ReadOnly (Just mode) defaultFileFlags + openFd (transferLockFile tfile) ReadOnly (Just mode) defaultFileFlags case mfd of Nothing -> return Nothing -- failed to open file; not running Just fd -> do locked <- liftIO $ getLock fd (WriteLock, AbsoluteSeek, 0, 0) + liftIO $ closeFd fd case locked of - Nothing -> do - liftIO $ closeFd fd - return Nothing - Just (pid, _) -> liftIO $ do - h <- fdToHandle fd - info <- readTransferInfo pid - <$> hGetContentsStrict h - hClose h - return info + Nothing -> return Nothing + Just (pid, _) -> liftIO $ + flip catchDefaultIO Nothing $ do + readTransferInfo pid + <$> readFile tfile {- Gets all currently running transfers. -} getTransfers :: Annex [(Transfer, TransferInfo)] @@ -140,10 +141,16 @@ transferFile (Transfer direction u key) r = gitAnnexTransferDir r fromUUID u keyFile key +{- The transfer lock file corresponding to a given transfer info file. -} +transferLockFile :: FilePath -> FilePath +transferLockFile infofile = let (d,f) = splitFileName infofile in + combine d ("lck." ++ f) + {- Parses a transfer information filename to a Transfer. -} parseTransferFile :: FilePath -> Maybe Transfer -parseTransferFile file = - case drop (length bits - 3) bits of +parseTransferFile file + | "lck." `isPrefixOf` (takeFileName file) = Nothing + | otherwise = case drop (length bits - 3) bits of [direction, u, key] -> Transfer <$> readDirection direction <*> pure (toUUID u) @@ -156,8 +163,7 @@ writeTransferInfo :: TransferInfo -> String writeTransferInfo info = unlines -- transferPid is not included; instead obtained by looking at -- the process that locks the file. - -- transferThread is not included; not relevant for other processes - [ show $ startedTime info + [ maybe "" show $ startedTime info -- bytesComplete is not included; changes too fast , fromMaybe "" $ associatedFile info -- comes last; arbitrary content ] @@ -166,12 +172,17 @@ readTransferInfo :: ProcessID -> String -> Maybe TransferInfo readTransferInfo pid s = case bits of [time] -> TransferInfo - <$> readish time + <$> (Just <$> parsePOSIXTime time) <*> pure (Just pid) <*> pure Nothing <*> pure Nothing + <*> pure Nothing <*> pure (if null filename then Nothing else Just filename) _ -> Nothing where (bits, filebits) = splitAt 1 $ lines s filename = join "\n" filebits + +parsePOSIXTime :: String -> Maybe POSIXTime +parsePOSIXTime s = utcTimeToPOSIXSeconds + <$> parseTime defaultTimeLocale "%s%Qs" s From 1db7d27a451f552dbae8760e83c73b90da8114d5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 00:43:36 -0400 Subject: [PATCH 4141/8313] add back debug logging Make Utility.Process wrap the parts of System.Process that I use, and add debug logging to them. Also wrote some higher-level code that allows running an action with handles to a processes stdin or stdout (or both), and checking its exit status, all in a single function call. As a bonus, the debug logging now indicates whether the process is being run to read from it, feed it data, chat with it (writing and reading), or just call it for its side effect. --- Annex/UUID.hs | 2 - Backend/SHA.hs | 1 - Command/Map.hs | 13 +-- Config.hs | 2 - Git/Command.hs | 13 ++- Git/Config.hs | 15 ++- Git/Queue.hs | 13 +-- Git/UpdateIndex.hs | 13 +-- Remote/Bup.hs | 12 +-- Remote/Git.hs | 13 +-- Seek.hs | 4 +- Utility/CoProcess.hs | 12 +-- Utility/Gpg.hs | 45 ++++----- Utility/INotify.hs | 1 - Utility/Lsof.hs | 9 +- Utility/Process.hs | 206 ++++++++++++++++++++++++++++++++++++----- Utility/SafeCommand.hs | 3 +- 17 files changed, 251 insertions(+), 126 deletions(-) diff --git a/Annex/UUID.hs b/Annex/UUID.hs index 1d2175bcb6..13cee865d5 100644 --- a/Annex/UUID.hs +++ b/Annex/UUID.hs @@ -20,8 +20,6 @@ module Annex.UUID ( removeRepoUUID, ) where -import System.Process - import Common.Annex import qualified Git import qualified Git.Config diff --git a/Backend/SHA.hs b/Backend/SHA.hs index a1dd1cf648..04b3e362aa 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -12,7 +12,6 @@ import qualified Annex import Types.Backend import Types.Key import Types.KeySource -import System.Process import qualified Build.SysConfig as SysConfig import Data.Digest.Pure.SHA diff --git a/Command/Map.hs b/Command/Map.hs index f69b88a5d6..3dbdadbd6c 100644 --- a/Command/Map.hs +++ b/Command/Map.hs @@ -9,7 +9,6 @@ module Command.Map where import Control.Exception.Extensible import qualified Data.Map as M -import System.Process import Common.Annex import Command @@ -199,13 +198,11 @@ tryScan r case result of Left _ -> return Nothing Right r' -> return $ Just r' - pipedconfig cmd params = safely $ do - (_, Just h, _, pid) <- - createProcess (proc cmd $ toCommand params) - { std_out = CreatePipe } - r' <- Git.Config.hRead r h - forceSuccessProcess pid cmd $ toCommand params - return r' + pipedconfig cmd params = safely $ + withHandle StdoutHandle createProcessSuccess p $ + Git.Config.hRead r + where + p = proc cmd $ toCommand params configlist = onRemote r (pipedconfig, Nothing) "configlist" [] [] diff --git a/Config.hs b/Config.hs index 84f6125c63..1aa5a4ac50 100644 --- a/Config.hs +++ b/Config.hs @@ -7,8 +7,6 @@ module Config where -import System.Process - import Common.Annex import qualified Git import qualified Git.Config diff --git a/Git/Command.hs b/Git/Command.hs index 038824f268..d7c9830649 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -7,7 +7,6 @@ module Git.Command where -import System.Process import System.Posix.Process (getAnyProcessStatus) import Common @@ -41,12 +40,12 @@ run subcommand params repo = assertLocal repo $ - result unless reap is called. -} pipeRead :: [CommandParam] -> Repo -> IO String -pipeRead params repo = assertLocal repo $ do - (_, Just h, _, _) <- createProcess - (proc "git" $ toCommand $ gitCommandLine params repo) - { std_out = CreatePipe } - fileEncoding h - hGetContents h +pipeRead params repo = assertLocal repo $ + withHandle StdoutHandle createBackgroundProcess p $ \h -> do + fileEncoding h + hGetContents h + where + p = proc "git" $ toCommand $ gitCommandLine params repo {- Runs a git subcommand, feeding it input, and returning its output, - which is expected to be fairly small, since it's all read into memory diff --git a/Git/Config.hs b/Git/Config.hs index 2347501131..c82d6bb1b0 100644 --- a/Git/Config.hs +++ b/Git/Config.hs @@ -9,7 +9,7 @@ module Git.Config where import qualified Data.Map as M import Data.Char -import System.Process +import System.Process (cwd) import Common import Git @@ -48,14 +48,11 @@ read' repo = go repo go Repo { location = Local { gitdir = d } } = git_config d go Repo { location = LocalUnknown d } = git_config d go _ = assertLocal repo $ error "internal" - git_config d = do - (_, Just h, _, pid) - <- createProcess (proc "git" params) - { std_out = CreatePipe, cwd = Just d } - repo' <- hRead repo h - forceSuccessProcess pid "git" params - return repo' - params = ["config", "--null", "--list"] + git_config d = withHandle StdoutHandle createProcessSuccess p $ + hRead repo + where + params = ["config", "--null", "--list"] + p = (proc "git" params) { cwd = Just d } {- Reads git config from a handle and populates a repo with it. -} hRead :: Repo -> Handle -> IO Repo diff --git a/Git/Queue.hs b/Git/Queue.hs index 4e6f05c2e0..f515ad1045 100644 --- a/Git/Queue.hs +++ b/Git/Queue.hs @@ -19,7 +19,6 @@ module Git.Queue ( import qualified Data.Map as M import System.IO -import System.Process import Data.String.Utils import Utility.SafeCommand @@ -148,13 +147,11 @@ runAction :: Repo -> Action -> IO () runAction repo (UpdateIndexAction streamers) = -- list is stored in reverse order Git.UpdateIndex.streamUpdateIndex repo $ reverse streamers -runAction repo action@(CommandAction {}) = do - (Just h, _, _, pid) <- createProcess (proc "xargs" params) - { std_in = CreatePipe } - fileEncoding h - hPutStr h $ join "\0" $ getFiles action - hClose h - forceSuccessProcess pid "xargs" params +runAction repo action@(CommandAction {}) = + withHandle StdinHandle createProcessSuccess (proc "xargs" params) $ \h -> do + fileEncoding h + hPutStr h $ join "\0" $ getFiles action + hClose h where params = "-0":"git":baseparams baseparams = toCommand $ gitCommandLine diff --git a/Git/UpdateIndex.hs b/Git/UpdateIndex.hs index 6de0c3adab..9294487298 100644 --- a/Git/UpdateIndex.hs +++ b/Git/UpdateIndex.hs @@ -17,8 +17,6 @@ module Git.UpdateIndex ( stageSymlink ) where -import System.Process - import Common import Git import Git.Types @@ -36,12 +34,11 @@ pureStreamer !s = \streamer -> streamer s {- Streams content into update-index from a list of Streamers. -} streamUpdateIndex :: Repo -> [Streamer] -> IO () -streamUpdateIndex repo as = do - (Just h, _, _, p) <- createProcess (proc "git" ps) { std_in = CreatePipe } - fileEncoding h - forM_ as (stream h) - hClose h - forceSuccessProcess p "git" ps +streamUpdateIndex repo as = + withHandle StdinHandle createProcessSuccess (proc "git" ps) $ \h -> do + fileEncoding h + forM_ as (stream h) + hClose h where ps = toCommand $ gitCommandLine params repo params = map Param ["update-index", "-z", "--index-info"] diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 9da374174b..8a2c1afefe 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -133,15 +133,13 @@ retrieveCheap :: BupRepo -> Key -> FilePath -> Annex Bool retrieveCheap _ _ _ = return False retrieveEncrypted :: BupRepo -> (Cipher, Key) -> Key -> FilePath -> Annex Bool -retrieveEncrypted buprepo (cipher, enck) _ f = do - let params = bupParams "join" buprepo [Param $ bupRef enck] - liftIO $ catchBoolIO $ do - (_, Just h, _, pid) - <- createProcess (proc "bup" $ toCommand params) - { std_out = CreatePipe } +retrieveEncrypted buprepo (cipher, enck) _ f = liftIO $ catchBoolIO $ + withHandle StdoutHandle createProcessSuccess p $ \h -> do withDecryptedContent cipher (L.hGetContents h) $ L.writeFile f - forceSuccessProcess pid "bup" $ toCommand params return True + where + params = bupParams "join" buprepo [Param $ bupRef enck] + p = proc "bup" $ toCommand params remove :: Key -> Annex Bool remove _ = do diff --git a/Remote/Git.hs b/Remote/Git.hs index a9a6d6004e..3412de89b4 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -9,7 +9,6 @@ module Remote.Git (remote, repoAvail) where import qualified Data.Map as M import Control.Exception.Extensible -import System.Process import Common.Annex import Utility.CopyFile @@ -127,13 +126,11 @@ tryGitConfigRead r safely a = either (const $ return r) return =<< liftIO (try a :: IO (Either SomeException Git.Repo)) - pipedconfig cmd params = safely $ do - (_, Just h, _, pid) <- - createProcess (proc cmd $ toCommand params) - { std_out = CreatePipe } - r' <- Git.Config.hRead r h - forceSuccessProcess pid cmd $ toCommand params - return r' + pipedconfig cmd params = safely $ + withHandle StdoutHandle createProcessSuccess p $ + Git.Config.hRead r + where + p = proc cmd $ toCommand params geturlconfig headers = do s <- Url.get (Git.repoLocation r ++ "/config") headers diff --git a/Seek.hs b/Seek.hs index 2cf0d8d460..3306a02fcd 100644 --- a/Seek.hs +++ b/Seek.hs @@ -108,9 +108,9 @@ withNothing _ _ = error "This command takes no parameters." prepFiltered :: (FilePath -> CommandStart) -> Annex [FilePath] -> Annex [CommandStart] prepFiltered a fs = do matcher <- Limit.getMatcher - map (proc matcher) <$> fs + map (process matcher) <$> fs where - proc matcher f = do + process matcher f = do ok <- matcher f if ok then a f else return Nothing diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs index d3b0c46efc..67f861bb32 100644 --- a/Utility/CoProcess.hs +++ b/Utility/CoProcess.hs @@ -13,25 +13,23 @@ module Utility.CoProcess ( query ) where -import System.Process - import Common -type CoProcessHandle = (ProcessHandle, Handle, Handle, FilePath, [String]) +type CoProcessHandle = (ProcessHandle, Handle, Handle, CreateProcess) start :: FilePath -> [String] -> IO CoProcessHandle start command params = do (from, to, _err, pid) <- runInteractiveProcess command params Nothing Nothing - return (pid, to, from, command, params) + return (pid, to, from, proc command params) stop :: CoProcessHandle -> IO () -stop (pid, from, to, command, params) = do +stop (pid, from, to, p) = do hClose to hClose from - forceSuccessProcess pid command params + forceSuccessProcess p pid query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b -query (_, from, to, _, _) send receive = do +query (_, from, to, _) send receive = do _ <- send to hFlush to receive from diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs index 26ac688e3a..eed77805cb 100644 --- a/Utility/Gpg.hs +++ b/Utility/Gpg.hs @@ -13,7 +13,6 @@ import Control.Applicative import Control.Concurrent import Control.Exception (bracket) import System.Posix.Env (setEnv, unsetEnv, getEnv) -import System.Process import Common @@ -39,30 +38,21 @@ stdParams params = do readStrict :: [CommandParam] -> IO String readStrict params = do params' <- stdParams params - (_, Just from, _, pid) - <- createProcess (proc "gpg" params') - { std_out = CreatePipe } - hSetBinaryMode from True - r <- hGetContentsStrict from - forceSuccessProcess pid "gpg" params' - return r + withHandle StdoutHandle createProcessSuccess (proc "gpg" params') $ \h -> do + hSetBinaryMode h True + hGetContentsStrict h {- Runs gpg, piping an input value to it, and returning its stdout, - strictly. -} pipeStrict :: [CommandParam] -> String -> IO String pipeStrict params input = do params' <- stdParams params - (Just to, Just from, _, pid) - <- createProcess (proc "gpg" params') - { std_in = CreatePipe - , std_out = CreatePipe } - hSetBinaryMode to True - hSetBinaryMode from True - hPutStr to input - hClose to - r <- hGetContentsStrict from - forceSuccessProcess pid "gpg" params' - return r + withBothHandles createProcessSuccess (proc "gpg" params') $ \(to, from) -> do + hSetBinaryMode to True + hSetBinaryMode from True + hPutStr to input + hClose to + hGetContentsStrict from {- Runs gpg with some parameters, first feeding it a passphrase via - --passphrase-fd, then feeding it an input, and passing a handle @@ -82,16 +72,13 @@ passphraseHandle params passphrase a b = do let passphrasefd = [Param "--passphrase-fd", Param $ show pfd] params' <- stdParams $ passphrasefd ++ params - (Just toh, Just fromh, _, pid) <- createProcess (proc "gpg" params') - { std_in = CreatePipe, std_out = CreatePipe } - L.hPut toh =<< a - hClose toh - ret <- b fromh - - -- cleanup - forceSuccessProcess pid "gpg" params' - closeFd frompipe - return ret + closeFd frompipe `after` + withBothHandles createProcessSuccess (proc "gpg" params') go + where + go (to, from) = do + L.hPut to =<< a + hClose to + b from {- Finds gpg public keys matching some string. (Could be an email address, - a key id, or a name. -} diff --git a/Utility/INotify.hs b/Utility/INotify.hs index 55233ef762..66c0ab23df 100644 --- a/Utility/INotify.hs +++ b/Utility/INotify.hs @@ -10,7 +10,6 @@ module Utility.INotify where import Common hiding (isDirectory) import Utility.ThreadLock import Utility.Types.DirWatcher -import System.Process import System.INotify import qualified System.Posix.Files as Files diff --git a/Utility/Lsof.hs b/Utility/Lsof.hs index ebd273b2e1..ce6a162832 100644 --- a/Utility/Lsof.hs +++ b/Utility/Lsof.hs @@ -12,7 +12,6 @@ module Utility.Lsof where import Common import System.Posix.Types -import System.Process data LsofOpenMode = OpenReadWrite | OpenReadOnly | OpenWriteOnly | OpenUnknown deriving (Show, Eq) @@ -34,9 +33,11 @@ queryDir path = query ["+d", path] - Note: If lsof is not available, this always returns [] ! -} query :: [String] -> IO [(FilePath, LsofOpenMode, ProcessInfo)] -query opts = do - (_, s, _) <- readProcessWithExitCode "lsof" ("-F0can" : opts) [] - return $ parse s +query opts = + withHandle StdoutHandle (createProcessChecked checkSuccessProcess) p $ \h -> do + parse <$> hGetContentsStrict h + where + p = proc "lsof" ("-F0can" : opts) {- Parsing null-delimited output like: - diff --git a/Utility/Process.hs b/Utility/Process.hs index 9f79efa813..9b57c3b7ab 100644 --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -1,40 +1,202 @@ -{- System.Process enhancements +{- System.Process enhancements, including additional ways of running + - processes, and logging. - - Copyright 2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} -module Utility.Process where +{-# LANGUAGE Rank2Types #-} -import System.Process +module Utility.Process ( + module X, + CreateProcess, + StdHandle(..), + readProcessEnv, + forceSuccessProcess, + checkSuccessProcess, + createProcessSuccess, + createProcessChecked, + createBackgroundProcess, + withHandle, + withBothHandles, + createProcess, + runInteractiveProcess, + readProcess +) where + +import qualified System.Process +import System.Process as X hiding (CreateProcess(..), createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode, system, rawSystem, runInteractiveCommand, runProcess) +import System.Process hiding (createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode) import System.Exit import System.IO +import System.Log.Logger import Utility.Misc -{- Waits for a ProcessHandle, and throws an exception if the process - - did not exit successfully. -} -forceSuccessProcess :: ProcessHandle -> String -> [String] -> IO () -forceSuccessProcess pid cmd args = do - code <- waitForProcess pid - case code of - ExitSuccess -> return () - ExitFailure n -> error $ - cmd ++ " " ++ show args ++ " exited " ++ show n +type CreateProcessRunner = forall a. CreateProcess -> ((Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> IO a) -> IO a + +data StdHandle = StdinHandle | StdoutHandle | StderrHandle + deriving (Eq) {- Like readProcess, but allows specifying the environment, and does - not mess with stdin. -} readProcessEnv :: FilePath -> [String] -> Maybe [(String, String)] -> IO String -readProcessEnv cmd args environ = do - (_, Just h, _, pid) - <- createProcess (proc cmd args) - { std_in = Inherit - , std_out = CreatePipe - , std_err = Inherit +readProcessEnv cmd args environ = + withHandle StdoutHandle createProcessSuccess p $ \h -> do + output <- hGetContentsStrict h + hClose h + return output + where + p = (proc cmd args) + { std_out = CreatePipe , env = environ } - output <- hGetContentsStrict h - hClose h - forceSuccessProcess pid cmd args - return output + +{- Waits for a ProcessHandle, and throws an exception if the process + - did not exit successfully. -} +forceSuccessProcess :: CreateProcess -> ProcessHandle -> IO () +forceSuccessProcess p pid = do + code <- waitForProcess pid + case code of + ExitSuccess -> return () + ExitFailure n -> error $ showCmd p ++ " exited " ++ show n + +{- Waits for a ProcessHandle and returns True if it exited successfully. -} +checkSuccessProcess :: ProcessHandle -> IO Bool +checkSuccessProcess pid = do + code <- waitForProcess pid + return $ code == ExitSuccess + +{- Runs createProcess, then an action on its handles, and then + - forceSuccessProcess. -} +createProcessSuccess :: CreateProcessRunner +createProcessSuccess p a = createProcessChecked (forceSuccessProcess p) p a + +{- Runs createProcess, then an action on its handles, and then + - an action on its exit code. -} +createProcessChecked :: (ProcessHandle -> IO b) -> CreateProcessRunner +createProcessChecked checker p a = do + t@(_, _, _, pid) <- createProcess p + r <- a t + _ <- checker pid + return r + +{- Leaves the process running, suitable for lazy streaming. + - Note: Zombies will result, and must be waited on. -} +createBackgroundProcess :: CreateProcessRunner +createBackgroundProcess p a = a =<< createProcess p + +{- Runs a CreateProcessRunner, on a CreateProcess structure, that + - is adjusted to pipe only from/to a single StdHandle, and passes + - the resulting Handle to an action. -} +withHandle + :: StdHandle + -> CreateProcessRunner + -> CreateProcess + -> (Handle -> IO a) + -> IO a +withHandle h creator p a = creator p' $ a . select + where + base = p + { std_in = Inherit + , std_out = Inherit + , std_err = Inherit + } + (select, p') + | h == StdinHandle = + (stdinHandle, base { std_in = CreatePipe }) + | h == StdoutHandle = + (stdoutHandle, base { std_out = CreatePipe }) + | h == StderrHandle = + (stderrHandle, base { std_err = CreatePipe }) + +{- Like withHandle, but passes (stdin, stdout) handles to the action. -} +withBothHandles + :: CreateProcessRunner + -> CreateProcess + -> ((Handle, Handle) -> IO a) + -> IO a +withBothHandles creator p a = creator p' $ a . bothHandles + where + p' = p + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = Inherit + } + +{- Extract a desired handle from createProcess's tuple. + - These partial functions are safe as long as createProcess is run + - with appropriate parameters to set up the desired handle. + - Get it wrong and the runtime crash will always happen, so should be + - easily noticed. -} +type HandleExtractor = (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> Handle +stdinHandle :: HandleExtractor +stdinHandle (Just h, _, _, _) = h +stdinHandle _ = error "expected stdinHandle" +stdoutHandle :: HandleExtractor +stdoutHandle (_, Just h, _, _) = h +stdoutHandle _ = error "expected stdoutHandle" +stderrHandle :: HandleExtractor +stderrHandle (_, _, Just h, _) = h +stderrHandle _ = error "expected stderrHandle" +bothHandles :: (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) -> (Handle, Handle) +bothHandles (Just hin, Just hout, _, _) = (hin, hout) +bothHandles _ = error "expected bothHandles" + +{- Debugging trace for a CreateProcess. -} +debugProcess :: CreateProcess -> IO () +debugProcess p = do + debugM "Utility.Process" $ unwords + [ action ++ ":" + , showCmd p + , maybe "" show (env p) + ] + where + action + | piped (std_in p) && piped (std_out p) = "chat" + | piped (std_in p) = "feed" + | piped (std_out p) = "read" + | otherwise = "call" + piped Inherit = False + piped _ = True + +{- Shows the command that a CreateProcess will run. -} +showCmd :: CreateProcess -> String +showCmd = go . cmdspec + where + go (ShellCommand s) = s + go (RawCommand c ps) = c ++ " " ++ show ps + +{- Wrappers for System.Process functions that do debug logging. + - + - More could be added, but these are the only ones I usually need. + -} + +createProcess :: CreateProcess -> IO (Maybe Handle, Maybe Handle, Maybe Handle, ProcessHandle) +createProcess p = do + debugProcess p + System.Process.createProcess p + +runInteractiveProcess + :: FilePath + -> [String] + -> Maybe FilePath + -> Maybe [(String, String)] + -> IO (Handle, Handle, Handle, ProcessHandle) +runInteractiveProcess f args c e = do + debugProcess $ (proc f args) + { std_in = CreatePipe + , std_out = CreatePipe + , std_err = CreatePipe + } + System.Process.runInteractiveProcess f args c e + +readProcess + :: FilePath + -> [String] + -> String + -> IO String +readProcess f args input = do + debugProcess $ (proc f args) { std_out = CreatePipe } + System.Process.readProcess f args input diff --git a/Utility/SafeCommand.hs b/Utility/SafeCommand.hs index 47280a40b1..19dd707b8d 100644 --- a/Utility/SafeCommand.hs +++ b/Utility/SafeCommand.hs @@ -8,7 +8,8 @@ module Utility.SafeCommand where import System.Exit -import System.Process +import Utility.Process +import System.Process (env) import Data.String.Utils import Control.Applicative From 9fc94d780b7331da13597208ba37a9f4d4ab6531 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 00:57:40 -0400 Subject: [PATCH 4142/8313] better readProcess --- Annex/UUID.hs | 2 +- Backend/SHA.hs | 2 +- Config.hs | 4 ++-- Git/Command.hs | 2 +- Utility/INotify.hs | 2 +- Utility/Process.hs | 18 +++++++++++++++--- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Annex/UUID.hs b/Annex/UUID.hs index 13cee865d5..09862f9fc0 100644 --- a/Annex/UUID.hs +++ b/Annex/UUID.hs @@ -32,7 +32,7 @@ configkey = annexConfig "uuid" {- Generates a UUID. There is a library for this, but it's not packaged, - so use the command line tool. -} genUUID :: IO UUID -genUUID = gen . lines <$> readProcess command params [] +genUUID = gen . lines <$> readProcess command params where gen [] = error $ "no output from " ++ command gen (l:_) = toUUID l diff --git a/Backend/SHA.hs b/Backend/SHA.hs index 04b3e362aa..bb400a768b 100644 --- a/Backend/SHA.hs +++ b/Backend/SHA.hs @@ -54,7 +54,7 @@ shaN shasize file filesize = do case shaCommand shasize filesize of Left sha -> liftIO $ sha <$> L.readFile file Right command -> liftIO $ parse command . lines <$> - readProcess command (toCommand [File file]) "" + readProcess command (toCommand [File file]) where parse command [] = bad command parse command (l:_) diff --git a/Config.hs b/Config.hs index 1aa5a4ac50..2c26adc736 100644 --- a/Config.hs +++ b/Config.hs @@ -56,7 +56,7 @@ remoteCost r def = do cmd <- getRemoteConfig r "cost-command" "" (fromMaybe def . readish) <$> if not $ null cmd - then liftIO $ readProcess "sh" ["-c", cmd] "" + then liftIO $ readProcess "sh" ["-c", cmd] else getRemoteConfig r "cost" "" cheapRemoteCost :: Int @@ -116,4 +116,4 @@ getHttpHeaders = do cmd <- getConfig (annexConfig "http-headers-command") "" if null cmd then fromRepo $ Git.Config.getList "annex.http-headers" - else lines <$> liftIO (readProcess "sh" ["-c", cmd] "") + else lines <$> liftIO (readProcess "sh" ["-c", cmd]) diff --git a/Git/Command.hs b/Git/Command.hs index d7c9830649..cd6c98d339 100644 --- a/Git/Command.hs +++ b/Git/Command.hs @@ -52,7 +52,7 @@ pipeRead params repo = assertLocal repo $ - strictly. -} pipeWriteRead :: [CommandParam] -> String -> Repo -> IO String pipeWriteRead params s repo = assertLocal repo $ - readProcess "git" (toCommand $ gitCommandLine params repo) s + writeReadProcess "git" (toCommand $ gitCommandLine params repo) s {- Reads null terminated output of a git command (as enabled by the -z - parameter), and splits it. -} diff --git a/Utility/INotify.hs b/Utility/INotify.hs index 66c0ab23df..6af0228195 100644 --- a/Utility/INotify.hs +++ b/Utility/INotify.hs @@ -160,7 +160,7 @@ tooManyWatches hook dir = do querySysctl :: Read a => [CommandParam] -> IO (Maybe a) querySysctl ps = do - v <- catchMaybeIO $ readProcess "sysctl" (toCommand ps) [] + v <- catchMaybeIO $ readProcess "sysctl" (toCommand ps) case v of Nothing -> return Nothing Just s -> return $ parsesysctl s diff --git a/Utility/Process.hs b/Utility/Process.hs index 9b57c3b7ab..3b293df4f5 100644 --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -22,6 +22,7 @@ module Utility.Process ( withBothHandles, createProcess, runInteractiveProcess, + writeReadProcess, readProcess ) where @@ -192,11 +193,22 @@ runInteractiveProcess f args c e = do } System.Process.runInteractiveProcess f args c e -readProcess +{- I think this is a more descriptive name than System.Process.readProcess. -} +writeReadProcess :: FilePath -> [String] -> String -> IO String -readProcess f args input = do - debugProcess $ (proc f args) { std_out = CreatePipe } +writeReadProcess f args input = do + debugProcess $ (proc f args) { std_out = CreatePipe, std_in = CreatePipe } System.Process.readProcess f args input + +{- Normally, when reading from a process, it does not need to be fed any + - input. -} +readProcess + :: FilePath + -> [String] + -> IO String +readProcess f args = do + debugProcess $ (proc f args) { std_out = CreatePipe } + System.Process.readProcess f args [] From 1e0b7dda8cd66aca89ce3eb608dd2c568a77b141 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 01:02:22 -0400 Subject: [PATCH 4143/8313] foo --- Utility/Process.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utility/Process.hs b/Utility/Process.hs index 3b293df4f5..5c29bbdfb9 100644 --- a/Utility/Process.hs +++ b/Utility/Process.hs @@ -28,7 +28,7 @@ module Utility.Process ( import qualified System.Process import System.Process as X hiding (CreateProcess(..), createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode, system, rawSystem, runInteractiveCommand, runProcess) -import System.Process hiding (createProcess, runInteractiveProcess, readProcess, readProcessWithExitCode) +import System.Process hiding (createProcess, runInteractiveProcess, readProcess) import System.Exit import System.IO import System.Log.Logger From 2771187f0184699a611e0231ecb4858050150d80 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnbBRfl5F8gKRr1ko8Ai6FbEZStXXNF1S4" Date: Thu, 19 Jul 2012 13:56:03 +0000 Subject: [PATCH 4144/8313] --- ...time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn diff --git a/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn new file mode 100644 index 0000000000..a5492dc861 --- /dev/null +++ b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__.mdwn @@ -0,0 +1,5 @@ +git-annex is seriously cool, however I havent figured out an important detail: +in the short presentation Richard Hartmann mentions the Nomad use case and says "... next time Im online I would like to have file x y z ... " +How can this be achieved exactly? If I do a git annex copy/get then it will want to do it instantly, not queuing it. +thanks a lot! +Aron From c6e6521209bda076545c08d06af2d70c741aa510 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" Date: Thu, 19 Jul 2012 14:41:40 +0000 Subject: [PATCH 4145/8313] Added a comment --- ...omment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment diff --git a/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment new file mode 100644 index 0000000000..0789316dc7 --- /dev/null +++ b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_1_bfeb1446dee4d2f52ef25fabfb8cc8f6._comment @@ -0,0 +1,11 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo" + nickname="Justin" + subject="comment 1" + date="2012-07-19T14:41:39Z" + content=""" +What I do for this is maintain a `todo` directory and `cp -a` + `git annex add` files I want to download into this directory. +the `cp -a` maintains the symlinks and `git annex add` fixes them if needed. + +Also this works the same way if the situation is reversed and the machine you want to download the files onto is not online. You can add files to the `todo` directory on the server, then once the client machine is online do a `git annex sync` + `git annex get todo`. +"""]] From e2c86a4b582bf222a51e9bb9066edce204d68ac8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 12:51:55 -0400 Subject: [PATCH 4146/8313] extacted Mounts.hsc from hsshellscript Converted from using c2hs to using hsc2hs, just because other code in git-annex uses hsc2hs. Various cleanups. This code is LGPLed, so I had to include that licence. --- .gitignore | 1 + Utility/Mounts.hsc | 81 ++++++++ debian/copyright | 7 + doc/LGPL | 502 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 591 insertions(+) create mode 100644 Utility/Mounts.hsc create mode 100644 doc/LGPL diff --git a/.gitignore b/.gitignore index afb5f314e4..4a18e7f26a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ html *.tix .hpc Utility/Touch.hs +Utility/Mounts.hs Utility/*.o dist # Sandboxed builds diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc new file mode 100644 index 0000000000..622ac877a2 --- /dev/null +++ b/Utility/Mounts.hsc @@ -0,0 +1,81 @@ +{- Interface to mtab (and fstab) + - + - Derived from hsshellscript, originally written by + - Volker Wysk + - + - Licensed under the GNU LGPL version 2.1 or higher. + -} + +{-# LANGUAGE ForeignFunctionInterface #-} + +module Utility.Mounts ( + Mntent(..), + read_mtab, + read_fstab, +) where + +import Control.Monad +import Foreign +import Foreign.C +import GHC.IO hiding (finally, bracket) +import Prelude hiding (catch) + +#include +#include + +data Mntent = Mntent + { mnt_fsname :: String + , mnt_dir :: String + , mnt_type :: String + , mnt_opts :: String + , mnt_freq :: Int + , mnt_passno :: Int + } deriving (Read, Show, Eq) + +read_mounts :: String -> IO [Mntent] +read_mounts path = do + h <- withCString path $ \cpath -> + withCString "r" $ \r -> + c_setmntent cpath r + when (h == nullPtr) $ + throwErrno "setmntent" + mntent <- getmntent h [] + _ <- c_endmntent h + return mntent + + where + getmntent h l = do + ptr <- c_getmntent h + if (ptr == nullPtr) + then return $ reverse l + else do + mnt_fsname_str <- #{peek struct mntent, mnt_fsname} ptr >>= peekCString + mnt_dir_str <- #{peek struct mntent, mnt_dir} ptr >>= peekCString + mnt_type_str <- #{peek struct mntent, mnt_type} ptr >>= peekCString + mnt_opts_str <- #{peek struct mntent, mnt_opts} ptr >>= peekCString + mnt_freq_int <- #{peek struct mntent, mnt_freq} ptr + mnt_passno_int <- #{peek struct mntent, mnt_passno} ptr + let ent = Mntent + { mnt_fsname = mnt_fsname_str + , mnt_dir = mnt_dir_str + , mnt_type = mnt_type_str + , mnt_opts = mnt_opts_str + , mnt_freq = mnt_freq_int + , mnt_passno = mnt_passno_int + } + getmntent h (ent:l) + +read_mtab :: IO [Mntent] +read_mtab = read_mounts "/etc/mtab" + +read_fstab :: IO [Mntent] +read_fstab = read_mounts "/etc/fstab" + +foreign import ccall safe "setmntent" + c_setmntent :: ((Ptr CChar) -> ((Ptr CChar) -> (IO (Ptr ())))) + +foreign import ccall safe "endmntent" + c_endmntent :: ((Ptr ()) -> (IO CInt)) + +foreign import ccall safe "getmntent" + c_getmntent :: ((Ptr ()) -> (IO (Ptr ()))) diff --git a/debian/copyright b/debian/copyright index de1e08e1cd..dcfbaf3e35 100644 --- a/debian/copyright +++ b/debian/copyright @@ -8,6 +8,13 @@ License: GPL-3+ this package's source, or in /usr/share/common-licenses/GPL-3 on Debian systems. +Files: Utility/Mtab.hcs +Copyright: Volker Wysk +License: LGPL-2.1+ + the full text of version 2.1 of the LGPL is distributed as doc/LGPL + in this package's source, or in /usr/share/common-licences/LGPL-2.1 + on Debian systems. + Files: doc/logo.png doc/logo_small.png doc/favicon.png Copyright: 2007 Henrik Nyh 2010 Joey Hess diff --git a/doc/LGPL b/doc/LGPL new file mode 100644 index 0000000000..4362b49151 --- /dev/null +++ b/doc/LGPL @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! From f20a40f9d4a4574c9f88dac8fd02b73d7f594b8b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 13:01:41 -0400 Subject: [PATCH 4147/8313] MountWatcher thread Currently only prints mount points when mounts happen. --- Assistant.hs | 7 +++ Assistant/Threads/MountWatcher.hs | 89 +++++++++++++++++++++++++++++++ Assistant/Threads/Watcher.hs | 2 - Makefile | 4 +- debian/control | 1 + git-annex.cabal | 11 +++- 6 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 Assistant/Threads/MountWatcher.hs diff --git a/Assistant.hs b/Assistant.hs index 06484b0862..51639584c9 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -46,6 +46,11 @@ - Wakes up periodically and records the daemon's status to disk. - Thread 12: sanity checker - Wakes up periodically (rarely) and does sanity checks. + - Thread 13: mount watcher + - Either uses dbus to watch for drive mount events, or, when + - there's no dbus, polls to find newly mounted filesystems. + - Once a filesystem that contains a remote is mounted, syncs + - with it. - - ThreadState: (MVar) - The Annex state is stored here, which allows resuscitating the @@ -92,6 +97,7 @@ import Assistant.Threads.Merger import Assistant.Threads.TransferWatcher import Assistant.Threads.Transferrer import Assistant.Threads.SanityChecker +import Assistant.Threads.MountWatcher import qualified Utility.Daemon import Utility.LogFile import Utility.ThreadScheduler @@ -127,6 +133,7 @@ startDaemon assistant foreground , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan + , mountWatcherThread st dstatus , watchThread st dstatus transferqueue changechan ] waitForTermination diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs new file mode 100644 index 0000000000..f3b9c0a3a7 --- /dev/null +++ b/Assistant/Threads/MountWatcher.hs @@ -0,0 +1,89 @@ +{- git-annex assistant mount watcher, using either dbus or mtab polling + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +{-# LANGUAGE CPP #-} +{-# LANGUAGE OverloadedStrings #-} + +module Assistant.Threads.MountWatcher where + +import Common.Annex +import Assistant.ThreadedMonad +import Assistant.DaemonStatus +import Utility.ThreadScheduler +import Utility.Mounts + +import Control.Concurrent +import qualified Data.Set as S + +#if WITH_DBUS +import DBus.Client +#else +#warning Building without dbus support; will use mtab polling +#endif + +mountWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () +mountWatcherThread st handle = +#if WITH_DBUS + dbusThread st handle +#else + pollingThread st handle +#endif + +#if WITH_DBUS +dbusThread :: ThreadState -> DaemonStatusHandle -> IO () +dbusThread st handle = do + r <- tryIO connectSession + case r of + Left e -> do + print $ "Failed to connect to dbus; falling back to mtab polling (" ++ show e ++ ")" + pollingThread st handle + Right client -> do + {- Store the current mount points in an mvar, + - to be compared later. We could in theory work + - out the mount point from the dbus message, but + - this is easier. -} + mvar <- newMVar =<< currentMountPoints + -- Spawn a listener thread, and returns. + listen client mountadded (go mvar) + where + mountadded = matchAny + { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" + , matchMember = Just "MountAdded" + } + go mvar event = do + nowmounted <- currentMountPoints + wasmounted <- swapMVar mvar nowmounted + handleMounts st handle wasmounted nowmounted + +#endif + +pollingThread :: ThreadState -> DaemonStatusHandle -> IO () +pollingThread st handle = go =<< currentMountPoints + where + go wasmounted = do + threadDelaySeconds (Seconds 10) + nowmounted <- currentMountPoints + handleMounts st handle wasmounted nowmounted + go nowmounted + +handleMounts :: ThreadState -> DaemonStatusHandle -> MountPoints -> MountPoints -> IO () +handleMounts st handle wasmounted nowmounted = mapM_ (handleMount st handle) $ + S.toList $ newMountPoints wasmounted nowmounted + +handleMount :: ThreadState -> DaemonStatusHandle -> FilePath -> IO () +handleMount st handle mountpoint = do + putStrLn $ "mounted: " ++ mountpoint + +type MountPoints = S.Set FilePath + +{- Reads mtab, getting the current set of mount points. -} +currentMountPoints :: IO MountPoints +currentMountPoints = S.fromList . map mnt_dir <$> read_mtab + +{- Finds new mount points, given an old and a new set. -} +newMountPoints :: MountPoints -> MountPoints -> MountPoints +newMountPoints old new = S.difference new old diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index 9f0eba74e9..ae4fafb78f 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -5,8 +5,6 @@ - Licensed under the GNU GPL version 3 or higher. -} -{-# LANGUAGE CPP #-} - module Assistant.Threads.Watcher where import Common.Annex diff --git a/Makefile b/Makefile index 0afb10a7bb..1791d43396 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ bins=git-annex mans=git-annex.1 git-annex-shell.1 -sources=Build/SysConfig.hs Utility/Touch.hs +sources=Build/SysConfig.hs Utility/Touch.hs Utility/Mounts.hs all=$(bins) $(mans) docs OS:=$(shell uname | sed 's/[-_].*//') ifeq ($(OS),Linux) -BASEFLAGS_OPTS+=-DWITH_INOTIFY +BASEFLAGS_OPTS+=-DWITH_INOTIFY -DWITH_DBUS clibs=Utility/libdiskfree.o else BASEFLAGS_OPTS+=-DWITH_KQUEUE diff --git a/debian/control b/debian/control index 79702ed29a..35cbfde054 100644 --- a/debian/control +++ b/debian/control @@ -22,6 +22,7 @@ Build-Depends: libghc-edit-distance-dev, libghc-hinotify-dev [linux-any], libghc-stm-dev (>= 2.3), + libghc-dbus-dev, ikiwiki, perlmagick, git, diff --git a/git-annex.cabal b/git-annex.cabal index e58bd4d957..00f57319d6 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -31,6 +31,9 @@ Flag S3 Flag Inotify Description: Enable inotify support +Flag Dbus + Description: Enable dbus support + Flag Assistant Description: Enable git-annex assistant and watch command @@ -41,8 +44,8 @@ Executable git-annex pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP, base == 4.5.*, monad-control, transformers-base, lifted-base, IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process - -- Need to list this because it's generated from a .hsc file. - Other-Modules: Utility.Touch + -- Need to list these because they're generated from .hsc files. + Other-Modules: Utility.Touch Utility.Mounts C-Sources: Utility/libdiskfree.c Extensions: CPP GHC-Options: -threaded @@ -59,6 +62,10 @@ Executable git-annex Build-Depends: hinotify CPP-Options: -DWITH_INOTIFY + if flag(Dbus) + Build-Depends: dbus + CPP-Options: -DWITH_DBUS + Test-Suite test Type: exitcode-stdio-1.0 Main-Is: test.hs From c99710a31009820a80d99c8502ba2470b924dc1d Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 13:19:11 -0400 Subject: [PATCH 4148/8313] blog for the day --- doc/design/assistant/blog/day_40__dbus.mdwn | 100 ++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 doc/design/assistant/blog/day_40__dbus.mdwn diff --git a/doc/design/assistant/blog/day_40__dbus.mdwn b/doc/design/assistant/blog/day_40__dbus.mdwn new file mode 100644 index 0000000000..049c3ffe0d --- /dev/null +++ b/doc/design/assistant/blog/day_40__dbus.mdwn @@ -0,0 +1,100 @@ +Really productive day today, now that I'm out of the threaded runtime +tarpit! + +First, brought back `--debug` logging, better than before! As part of that, I +wrote some 250 lines of code to provide a IMHO more pleasant interface to +`System.Process` (itself only 650 lines of code) that avoids all the +low-level setup, cleanup, and tuple unpacking. Now I can do things like +write to a pipe to a process, and ensure it exits nonzero, this easily: + + withHandle StdinHandle createProcessSuccess (proc "git" ["hash-object", "--stdin"]) $ \h -> + hHutStr h objectdata + +My interface also makes it easy to run nasty background processes, +reading their output lazily. + + lazystring <- withHandle StdoutHandle createBackgroundProcess (proc "find" ["/"]) hGetContents + +Any true Haskellers are shuddering here, I really should be using +conduits or pipes, or something. One day.. + +---- + +The assistant needs to detect when removable drives are attached, and +sync with them. This is a reasonable thing to be working on at this point, +because it'll make the currently incomplete data transfer code fully usable +for the sneakernet use case, and firming that up will probably be a good +step toward handing other use cases involving data transfer over the +network, including cases where network remotes are transientely available. + +So I've been playing with using dbus to detect mount events. +There's a very nice Haskell library to use dbus. + +This simple program will detect removable drives being mounted, and +works on Xfce (as long as you have automounting enabled in its +configuration), and should also work on Gnome, and, probably, KDE: + +[[!format haskell """ +{-# LANGUAGE OverloadedStrings #-} + +import Data.List (sort) +import DBus +import DBus.Client +import Control.Monad + +main = do + client <- connectSession + + listen client mountadded $ \s -> + putStrLn (show s) + + forever $ getLine -- let listener thread run forever + + where + mountadded = matchAny + { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" + , matchMember = Just "MountAdded" + } +"""]] + +(Yeah... "org.gtk.Private.RemoteVolumeMonitor". There are so +many things wrong with that string. What does gtk have to do with +mounting a drive? Why is it Private? Bleagh. Should I only match +the "MountAdded" member and not the interface? Seems everyone who does +this relies on google to find other people who have cargo-culted it, +or just runs `dbus-monitor` and picks out things. +There seems to be no canonical list of events. Bleagh.) + +---- + +Spent a while shaving a yak of needing a `getmntent` interface in Haskell. +Found one in a hsshellscript library; since that library is not packaged +in Debian, and I don't really want to depend on it, I extracted just +the mtab and fstab parts of it into a little library in git-annex. + +---- + +I've started putting together a MountWatcher thread. On systems without +dbus (do OSX or the BSDs have dbus?), or if dbus is not running, it polls +`/etc/mtab` every 10 seconds for new mounts. When dbus is available, +it doesn't need the polling, and should notice mounts more quickly. + +Open question: Should it still poll even when dbus is available? Some of us +like to mount our own drives, by hand and may have automounting disabled. It'd +be good if the assistant supported that. This might need a +`annex.no-dbus` setting, but I'd rather avoid needing such manual +configuration. + +One idea is to do polling in addition to dbus, if `/etc/fstab` contains +mount points that seem to be removable drives, on which git remotes lives. +Or it could always do polling in addition to dbus, which is just some extra +work. Or, it could try to introspect dbus to see if mount events will +be generated. + +The MountWatcher so far only detects new mounts and prints out what +happened. Next up: Do something in response to them. + +This will involve manipulating the Annex state to belatedly add the Remote +on the mount point.. tricky. And then, for Git Remotes, it should pull/push +the Remote to sync git data. Finally, for all remotes, it will need to +queue Transfers of file contents from/to the newly available Remote. From f65f62c1e523fbf7c7e256bc244cf92a804843f4 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" Date: Thu, 19 Jul 2012 17:38:40 +0000 Subject: [PATCH 4149/8313] Added a comment: dbus vs polling --- ...mment_1_43ed2a79629868b018ec9f54a32bcacc._comment | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment diff --git a/doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment b/doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment new file mode 100644 index 0000000000..3670e7dd55 --- /dev/null +++ b/doc/design/assistant/blog/day_40__dbus/comment_1_43ed2a79629868b018ec9f54a32bcacc._comment @@ -0,0 +1,12 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U" + nickname="Richard" + subject="dbus vs polling" + date="2012-07-19T17:38:40Z" + content=""" +I am running KDE, dbus, and I like to mount drives by hand; all that without bothering to set up /etc/fstab... Often from CLI so I don't even see the notifications for new drives. + +Especially while on the road, I will use no KDE, but just a tty or three. Still, dbus will be running. + +Long story short, I would need polling for the assistant to work flawlessly in all use cases. +"""]] From 896a8fa803fb87010b1f016f23f0b382eb1f08a1 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Thu, 19 Jul 2012 18:53:23 +0000 Subject: [PATCH 4150/8313] --- ...t_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn diff --git a/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn b/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn new file mode 100644 index 0000000000..3efc9186d5 --- /dev/null +++ b/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn @@ -0,0 +1,3 @@ +commit f20a40f breaks on OSX as mntent.h doesn't exist, the closet thing available to what mntent.h provides is getmntinfo(), it looks yet another bunch of ifdef's might be needed to work around OSX. This problem maybe similarly true with FreeBSD, libfam seems to have worked around the issue - + +hope the above report helps. From d5051ec088a443d0fbc0979d0421e62c60ec13f8 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 16:29:22 -0400 Subject: [PATCH 4151/8313] update --- doc/install.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/install.mdwn b/doc/install.mdwn index 3168976f47..4eb7b179b6 100644 --- a/doc/install.mdwn +++ b/doc/install.mdwn @@ -46,6 +46,8 @@ To build and use git-annex, you will need: (optional; version 2.3 or newer) * [hinotify](http://hackage.haskell.org/package/hinotify) (optional; Linux only) + * [dbus](http://hackage.haskell.org/package/dbus) + (optional) * Shell commands * [git](http://git-scm.com/) * [uuid](http://www.ossp.org/pkg/lib/uuid/) From 55da0fbf627bdc661fd2590967326fce79bfec9d Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnbBRfl5F8gKRr1ko8Ai6FbEZStXXNF1S4" Date: Thu, 19 Jul 2012 22:41:54 +0000 Subject: [PATCH 4152/8313] Added a comment --- .../comment_2_e60f2bbc1c058993472fd920edbc75fc._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment diff --git a/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment new file mode 100644 index 0000000000..b0a8066ee8 --- /dev/null +++ b/doc/forum/__34__next_time_Im_online_I_would_like_to_have_file_x_y_z__34__/comment_2_e60f2bbc1c058993472fd920edbc75fc._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnbBRfl5F8gKRr1ko8Ai6FbEZStXXNF1S4" + nickname="Áron" + subject="comment 2" + date="2012-07-19T22:41:53Z" + content=""" +hmm not bad workaround but still not very comfortable, thanks though. +"""]] From 107a7b9388077a2b7fe9ce107da3a4a5fa396e2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 20:38:58 -0400 Subject: [PATCH 4153/8313] try to make Utility.Mounts portable This is an unholy mashup, but it just might work. It works on Linux, that's all I've tested. :) --- Assistant/Threads/MountWatcher.hs | 2 +- Makefile | 4 +- Utility/Mounts.hsc | 58 +++++++---------- Utility/libmounts.c | 105 ++++++++++++++++++++++++++++++ Utility/libmounts.h | 48 ++++++++++++++ debian/copyright | 13 +++- git-annex.cabal | 2 +- 7 files changed, 190 insertions(+), 42 deletions(-) create mode 100644 Utility/libmounts.c create mode 100644 Utility/libmounts.h diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index f3b9c0a3a7..cc62c294d2 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -82,7 +82,7 @@ type MountPoints = S.Set FilePath {- Reads mtab, getting the current set of mount points. -} currentMountPoints :: IO MountPoints -currentMountPoints = S.fromList . map mnt_dir <$> read_mtab +currentMountPoints = S.fromList . map mnt_dir <$> getMounts {- Finds new mount points, given an old and a new set. -} newMountPoints :: MountPoints -> MountPoints -> MountPoints diff --git a/Makefile b/Makefile index 1791d43396..a6030efa1d 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,10 @@ all=$(bins) $(mans) docs OS:=$(shell uname | sed 's/[-_].*//') ifeq ($(OS),Linux) BASEFLAGS_OPTS+=-DWITH_INOTIFY -DWITH_DBUS -clibs=Utility/libdiskfree.o +clibs=Utility/libdiskfree.o Utility/libmounts.o else BASEFLAGS_OPTS+=-DWITH_KQUEUE -clibs=Utility/libdiskfree.o Utility/libkqueue.o +clibs=Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o endif PREFIX=/usr diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc index 622ac877a2..6bcb03f2c4 100644 --- a/Utility/Mounts.hsc +++ b/Utility/Mounts.hsc @@ -2,6 +2,9 @@ - - Derived from hsshellscript, originally written by - Volker Wysk + - + - Modified to support BSD and Mac OS X by + - Joey Hess - - Licensed under the GNU LGPL version 2.1 or higher. -} @@ -10,8 +13,7 @@ module Utility.Mounts ( Mntent(..), - read_mtab, - read_fstab, + getMounts ) where import Control.Monad @@ -20,62 +22,46 @@ import Foreign.C import GHC.IO hiding (finally, bracket) import Prelude hiding (catch) -#include -#include +#include "libmounts.h" +{- This is a stripped down mntent, containing only + - fields available everywhere. -} data Mntent = Mntent { mnt_fsname :: String , mnt_dir :: String , mnt_type :: String - , mnt_opts :: String - , mnt_freq :: Int - , mnt_passno :: Int } deriving (Read, Show, Eq) -read_mounts :: String -> IO [Mntent] -read_mounts path = do - h <- withCString path $ \cpath -> - withCString "r" $ \r -> - c_setmntent cpath r +getMounts :: IO [Mntent] +getMounts = do + h <- c_mounts_start when (h == nullPtr) $ - throwErrno "setmntent" + throwErrno "getMounts" mntent <- getmntent h [] - _ <- c_endmntent h + _ <- c_mounts_end h return mntent where - getmntent h l = do - ptr <- c_getmntent h + getmntent h c = do + ptr <- c_mounts_next h if (ptr == nullPtr) - then return $ reverse l + then return $ reverse c else do mnt_fsname_str <- #{peek struct mntent, mnt_fsname} ptr >>= peekCString mnt_dir_str <- #{peek struct mntent, mnt_dir} ptr >>= peekCString mnt_type_str <- #{peek struct mntent, mnt_type} ptr >>= peekCString - mnt_opts_str <- #{peek struct mntent, mnt_opts} ptr >>= peekCString - mnt_freq_int <- #{peek struct mntent, mnt_freq} ptr - mnt_passno_int <- #{peek struct mntent, mnt_passno} ptr let ent = Mntent { mnt_fsname = mnt_fsname_str , mnt_dir = mnt_dir_str , mnt_type = mnt_type_str - , mnt_opts = mnt_opts_str - , mnt_freq = mnt_freq_int - , mnt_passno = mnt_passno_int } - getmntent h (ent:l) + getmntent h (ent:c) -read_mtab :: IO [Mntent] -read_mtab = read_mounts "/etc/mtab" +foreign import ccall unsafe "libmounts.h mounts_start" c_mounts_start + :: IO (Ptr ()) -read_fstab :: IO [Mntent] -read_fstab = read_mounts "/etc/fstab" +foreign import ccall unsafe "libmounts.h mounts_next" c_mounts_next + :: Ptr () -> IO (Ptr ()) -foreign import ccall safe "setmntent" - c_setmntent :: ((Ptr CChar) -> ((Ptr CChar) -> (IO (Ptr ())))) - -foreign import ccall safe "endmntent" - c_endmntent :: ((Ptr ()) -> (IO CInt)) - -foreign import ccall safe "getmntent" - c_getmntent :: ((Ptr ()) -> (IO (Ptr ()))) +foreign import ccall unsafe "libmounts.h mounts_end" c_mounts_end + :: Ptr () -> IO CInt diff --git a/Utility/libmounts.c b/Utility/libmounts.c new file mode 100644 index 0000000000..00755c577d --- /dev/null +++ b/Utility/libmounts.c @@ -0,0 +1,105 @@ +/* mounted filesystems, C mini-library + * + * Copyright (c) 1980, 1989, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * Copyright (c) 2001 + * David Rufino + * Copyright 2012 + * Joey Hess + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "libmounts.h" + +#include +#include + +#ifdef GETMNTENT +/* direct passthrough the getmntent */ +FILE *mounts_start (void) { + return setmntent("/etc/mtab", "r"); +} +int mounts_end (FILE *fp) { + return endmntent(fp); +} +struct mntent *mounts_next (FILE *fp) { + return getmntent(fp); +} +#endif + +#ifdef GETMNTINFOCALL +/* getmntent emulation using getmntinfo */ +FILE *mounts_start (void) { + return ((FILE *)0x1) /* dummy non-NULL FILE pointer, not used */ +} +int mounts_end (FILE *fp) { + return 1; +} + +static struct mntent _mntent; + +static struct mntent *statfs_to_mntent (struct MNTINFOSTRUCT *mntbuf) { + _mntent.mnt_fsname = mntbuf->f_mntfromname; + _mntent.mnt_dir = mntbuf->f_mntonname; + _mntent.mnt_type = mntbuf->f_fstypename; + + _mntent.mnt_opts = '\0'; + _mntent.mnt_freq = 0; + _mntent.mnt_passno = 0; + + return (&_mntent); +} + +static int pos = -1; +static int mntsize = -1; + +struct mntent *mounts_next (FILE *fp) { + struct MNTINFOSTRUCT *mntbuf; + + if (pos == -1 || mntsize == -1) + mntsize = GETMNTINFOCALL(&mntbuf, MNT_NOWAIT); + ++pos; + if (pos == mntsize) { + pos = mntsize = -1; + return NULL; + } + + return (statfs_to_mntent(&mntbuf[pos])); +} +#endif + +#ifdef UNKNOWN +/* dummy, do-nothing version */ +FILE *mounts_start (void) { + return ((FILE *)0x1); +} +int mounts_end (FILE *fp) { + return 1; +} +struct mntent *mounts_next (FILE *fp) { + return NULL; +} +#endif diff --git a/Utility/libmounts.h b/Utility/libmounts.h new file mode 100644 index 0000000000..0bd52e3230 --- /dev/null +++ b/Utility/libmounts.h @@ -0,0 +1,48 @@ +/* Include appropriate headers for the OS, and define what will be used. */ +#if defined(__APPLE__) +# include +# include +# include +/* In newer OSX versions, statfs64 is deprecated, in favor of statfs, + * which is 64 bit only with a build option -- but statfs64 still works, + * and this keeps older OSX also supported. */ +# define GETMNTINFOCALL getmntinfo64 +# define MNTINFOSTRUCT statfs64 +#else +#if defined (__FreeBSD__) +# include +# include +# include +# define GETMNTINFOCALL getmntinfo64 +# define MNTINFOSTRUCT statfs64 +#else +#if defined (__linux__) || defined (__FreeBSD_kernel__) +/* Linux or Debian kFreeBSD */ +#include +# define GETMNTENT +#else +# warning mounts listing code not available for this OS +# define UNKNOWN +#endif +#endif +#endif + +#include +#include +#include + +#ifndef GETMNTENT +#warning "boo" +struct mntent { + char *mnt_fsname; + char *mnt_dir; + char *mnt_type; + char *mnt_opts; /* not filled in */ + int mnt_freq; /* not filled in */ + int mnt_passno; /* not filled in */ +}; +#endif + +FILE *mounts_start (void); +int mounts_end (FILE *fp); +struct mntent *mounts_next (FILE *fp); diff --git a/debian/copyright b/debian/copyright index dcfbaf3e35..26a559cc51 100644 --- a/debian/copyright +++ b/debian/copyright @@ -8,13 +8,22 @@ License: GPL-3+ this package's source, or in /usr/share/common-licenses/GPL-3 on Debian systems. -Files: Utility/Mtab.hcs +Files: Utility/Mounts.hsc Copyright: Volker Wysk License: LGPL-2.1+ - the full text of version 2.1 of the LGPL is distributed as doc/LGPL + The full text of version 2.1 of the LGPL is distributed as doc/LGPL in this package's source, or in /usr/share/common-licences/LGPL-2.1 on Debian systems. +Files: Utility/libmounts.c +Copyright: 1980, 1989, 1993, 1994 The Regents of the University of California + 2001 David Rufino + 2012 Joey Hess +License: BSD-3-clause + The full test of the 3 clause BSD license is distributed inside + Utility/libmounts.c in this package's source, or in + /usr/share/common-licenses/BSD on Debian systems. + Files: doc/logo.png doc/logo_small.png doc/favicon.png Copyright: 2007 Henrik Nyh 2010 Joey Hess diff --git a/git-annex.cabal b/git-annex.cabal index 00f57319d6..be752f844e 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -46,7 +46,7 @@ Executable git-annex IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process -- Need to list these because they're generated from .hsc files. Other-Modules: Utility.Touch Utility.Mounts - C-Sources: Utility/libdiskfree.c + C-Sources: Utility/libdiskfree.c Utility/libmounts.c Extensions: CPP GHC-Options: -threaded From f768cddf3a7048334f76fff1a2c895b33fea18af Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 20:44:58 -0400 Subject: [PATCH 4154/8313] fix build on OSX --- Utility/libmounts.c | 5 +---- Utility/libmounts.h | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Utility/libmounts.c b/Utility/libmounts.c index 00755c577d..d29c6ee6a4 100644 --- a/Utility/libmounts.c +++ b/Utility/libmounts.c @@ -34,9 +34,6 @@ #include "libmounts.h" -#include -#include - #ifdef GETMNTENT /* direct passthrough the getmntent */ FILE *mounts_start (void) { @@ -53,7 +50,7 @@ struct mntent *mounts_next (FILE *fp) { #ifdef GETMNTINFOCALL /* getmntent emulation using getmntinfo */ FILE *mounts_start (void) { - return ((FILE *)0x1) /* dummy non-NULL FILE pointer, not used */ + return ((FILE *)0x1); /* dummy non-NULL FILE pointer, not used */ } int mounts_end (FILE *fp) { return 1; diff --git a/Utility/libmounts.h b/Utility/libmounts.h index 0bd52e3230..12f5564b5b 100644 --- a/Utility/libmounts.h +++ b/Utility/libmounts.h @@ -27,12 +27,9 @@ #endif #endif -#include -#include -#include +#include #ifndef GETMNTENT -#warning "boo" struct mntent { char *mnt_fsname; char *mnt_dir; From 4bcc92abd72e060e073eaf0a9d988ae3a015d39c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 21:19:29 -0400 Subject: [PATCH 4155/8313] now working on OSX While this seems to work fine when used in a simple program, when I load it in ghci, it segfaults about half the time. Don't know why, and seems ghci specific, but if I get reports of crashes, I'll need to look into that. --- Utility/libmounts.c | 12 ++++++++---- Utility/libmounts.h | 16 ++-------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/Utility/libmounts.c b/Utility/libmounts.c index d29c6ee6a4..be18fe0c28 100644 --- a/Utility/libmounts.c +++ b/Utility/libmounts.c @@ -34,6 +34,9 @@ #include "libmounts.h" +#include +#include + #ifdef GETMNTENT /* direct passthrough the getmntent */ FILE *mounts_start (void) { @@ -47,7 +50,7 @@ struct mntent *mounts_next (FILE *fp) { } #endif -#ifdef GETMNTINFOCALL +#ifdef GETMNTINFO /* getmntent emulation using getmntinfo */ FILE *mounts_start (void) { return ((FILE *)0x1); /* dummy non-NULL FILE pointer, not used */ @@ -58,7 +61,7 @@ int mounts_end (FILE *fp) { static struct mntent _mntent; -static struct mntent *statfs_to_mntent (struct MNTINFOSTRUCT *mntbuf) { +static struct mntent *statfs_to_mntent (struct statfs *mntbuf) { _mntent.mnt_fsname = mntbuf->f_mntfromname; _mntent.mnt_dir = mntbuf->f_mntonname; _mntent.mnt_type = mntbuf->f_fstypename; @@ -72,15 +75,16 @@ static struct mntent *statfs_to_mntent (struct MNTINFOSTRUCT *mntbuf) { static int pos = -1; static int mntsize = -1; +struct statfs *mntbuf = NULL; struct mntent *mounts_next (FILE *fp) { - struct MNTINFOSTRUCT *mntbuf; if (pos == -1 || mntsize == -1) - mntsize = GETMNTINFOCALL(&mntbuf, MNT_NOWAIT); + mntsize = getmntinfo(&mntbuf, MNT_NOWAIT); ++pos; if (pos == mntsize) { pos = mntsize = -1; + mntbuf = NULL; return NULL; } diff --git a/Utility/libmounts.h b/Utility/libmounts.h index 12f5564b5b..b659786291 100644 --- a/Utility/libmounts.h +++ b/Utility/libmounts.h @@ -1,20 +1,9 @@ /* Include appropriate headers for the OS, and define what will be used. */ -#if defined(__APPLE__) +#if defined (__FreeBSD__) || defined (__APPLE__) # include # include # include -/* In newer OSX versions, statfs64 is deprecated, in favor of statfs, - * which is 64 bit only with a build option -- but statfs64 still works, - * and this keeps older OSX also supported. */ -# define GETMNTINFOCALL getmntinfo64 -# define MNTINFOSTRUCT statfs64 -#else -#if defined (__FreeBSD__) -# include -# include -# include -# define GETMNTINFOCALL getmntinfo64 -# define MNTINFOSTRUCT statfs64 +# define GETMNTINFO #else #if defined (__linux__) || defined (__FreeBSD_kernel__) /* Linux or Debian kFreeBSD */ @@ -25,7 +14,6 @@ # define UNKNOWN #endif #endif -#endif #include From ac044de486331462ce2a815db45f62399fd2cf2e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 21:20:38 -0400 Subject: [PATCH 4156/8313] cleanup --- Utility/libmounts.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Utility/libmounts.c b/Utility/libmounts.c index be18fe0c28..8669f33ea9 100644 --- a/Utility/libmounts.c +++ b/Utility/libmounts.c @@ -34,9 +34,6 @@ #include "libmounts.h" -#include -#include - #ifdef GETMNTENT /* direct passthrough the getmntent */ FILE *mounts_start (void) { From 0496a3971d4679e6a482a5eb277091980383f831 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 21:25:26 -0400 Subject: [PATCH 4157/8313] store whole Mntents This way, if a mount point was already mounted, but something else gets mounted there, it'll be seen as a new mount. --- Assistant/Threads/MountWatcher.hs | 10 +++++----- Utility/Mounts.hsc | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index cc62c294d2..b55e3284bc 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -74,15 +74,15 @@ handleMounts :: ThreadState -> DaemonStatusHandle -> MountPoints -> MountPoints handleMounts st handle wasmounted nowmounted = mapM_ (handleMount st handle) $ S.toList $ newMountPoints wasmounted nowmounted -handleMount :: ThreadState -> DaemonStatusHandle -> FilePath -> IO () -handleMount st handle mountpoint = do - putStrLn $ "mounted: " ++ mountpoint +handleMount :: ThreadState -> DaemonStatusHandle -> Mntent -> IO () +handleMount st handle mntent = do + putStrLn $ "mounted: " ++ mnt_dir mntent -type MountPoints = S.Set FilePath +type MountPoints = S.Set Mntent {- Reads mtab, getting the current set of mount points. -} currentMountPoints :: IO MountPoints -currentMountPoints = S.fromList . map mnt_dir <$> getMounts +currentMountPoints = S.fromList <$> getMounts {- Finds new mount points, given an old and a new set. -} newMountPoints :: MountPoints -> MountPoints -> MountPoints diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc index 6bcb03f2c4..4994c5e180 100644 --- a/Utility/Mounts.hsc +++ b/Utility/Mounts.hsc @@ -30,7 +30,7 @@ data Mntent = Mntent { mnt_fsname :: String , mnt_dir :: String , mnt_type :: String - } deriving (Read, Show, Eq) + } deriving (Read, Show, Eq, Ord) getMounts :: IO [Mntent] getMounts = do From bd7b5cd415fc87196779bb6bdbd30354bc0cdbeb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 21:31:04 -0400 Subject: [PATCH 4158/8313] fixed --- ...f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn b/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn index 3efc9186d5..dfd3ffb829 100644 --- a/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn +++ b/doc/bugs/commit_f20a40f_breaks_on_OSX_as_mntent.h_doesn__39__t_exist.mdwn @@ -1,3 +1,8 @@ commit f20a40f breaks on OSX as mntent.h doesn't exist, the closet thing available to what mntent.h provides is getmntinfo(), it looks yet another bunch of ifdef's might be needed to work around OSX. This problem maybe similarly true with FreeBSD, libfam seems to have worked around the issue - hope the above report helps. + +> Thanks, that was a very useful pointer. I couldn't figure out how to +> use Haskell's FFI to loop over the list of statfs structs returned by +> getmntinfo, so I incorporated that C code into a little library, +> and it seems to work ok. [[done]] --[[Joey]] From 6b4fe507f68427e0cb37e22f278c375151e8e89f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 23:34:33 -0400 Subject: [PATCH 4159/8313] only use dbus when there's a client connected we know will send mount events --- Assistant/Threads/MountWatcher.hs | 66 ++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index b55e3284bc..f1e33a99fa 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -21,6 +21,7 @@ import qualified Data.Set as S #if WITH_DBUS import DBus.Client +import DBus #else #warning Building without dbus support; will use mtab polling #endif @@ -34,30 +35,51 @@ mountWatcherThread st handle = #endif #if WITH_DBUS + dbusThread :: ThreadState -> DaemonStatusHandle -> IO () -dbusThread st handle = do - r <- tryIO connectSession - case r of - Left e -> do - print $ "Failed to connect to dbus; falling back to mtab polling (" ++ show e ++ ")" - pollingThread st handle - Right client -> do - {- Store the current mount points in an mvar, - - to be compared later. We could in theory work - - out the mount point from the dbus message, but - - this is easier. -} - mvar <- newMVar =<< currentMountPoints - -- Spawn a listener thread, and returns. - listen client mountadded (go mvar) +dbusThread st handle = (go =<< connectSession) `catchIO` onerr where - mountadded = matchAny - { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" - , matchMember = Just "MountAdded" - } - go mvar event = do - nowmounted <- currentMountPoints - wasmounted <- swapMVar mvar nowmounted - handleMounts st handle wasmounted nowmounted + go client = ifM (checkMountMonitor client) + ( do + {- Store the current mount points in an mvar, + - to be compared later. We could in theory + - work out the mount point from the dbus + - message, but this is easier. -} + mvar <- newMVar =<< currentMountPoints + listen client mountAdded $ \_event -> do + nowmounted <- currentMountPoints + wasmounted <- swapMVar mvar nowmounted + handleMounts st handle wasmounted nowmounted + , do + runThreadState st $ + warning "No known volume monitor available through dbus; falling back to mtab polling" + pollinstead + ) + onerr e = do + runThreadState st $ + warning $ "Failed to use dbus; falling back to mtab polling (" ++ show e ++ ")" + pollinstead + pollinstead = pollingThread st handle + +listClientNames :: Client -> IO [String] +listClientNames client = do + reply <- call_ client (methodCall "/org/freedesktop/DBus" "org.freedesktop.DBus" "ListNames") + { methodCallDestination = Just "org.freedesktop.DBus" } + return $ fromMaybe [] $ fromVariant (methodReturnBody reply !! 0) + +{- Examine the list of clients connected to dbus, to see if there + - are any we can use to monitor mounts. -} +checkMountMonitor :: Client -> IO Bool +checkMountMonitor client = any (`elem` knownclients) <$> listClientNames client + where + knownclients = ["org.gtk.Private.GduVolumeMonitor"] + +{- Filter matching events recieved when drives are mounted. -} +mountAdded ::MatchRule +mountAdded = matchAny + { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" + , matchMember = Just "MountAdded" + } #endif From e8ccf0bd1127fe487dc279cf764dfbc0d4787275 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 19 Jul 2012 23:46:24 -0400 Subject: [PATCH 4160/8313] update --- doc/design/assistant/syncing.mdwn | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 9b3e3b08e8..52ebcd08b6 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -10,9 +10,15 @@ all the other git clones, at both the git level and the key/value level. on remotes, and transfer. But first, need to ensure that when a remote receives content, and updates its location log, it syncs that update out. +* When MountWatcher detects a newly mounted drive, rescan git remotes + in order to get ones on the drive, and do a git sync and file transfers + to sync any repositories on it. ## longer-term TODO +* Test MountWatcher on KDE, and add whatever dbus events KDE emits when + drives are mounted. +* Test MountWatcher on Gnome (should work ok) and LXDE (dunno). * git-annex needs a simple speed control knob, which can be plumbed through to, at least, rsync. A good job for an hour in an airport somewhere. @@ -23,8 +29,6 @@ all the other git clones, at both the git level and the key/value level. * Add a hook, so when there's a change to sync, a program can be run and do its own signaling. * --debug will show often unnecessary work being done. Optimise. -* It would be nice if, when a USB drive is connected, - syncing starts automatically. Use dbus on Linux? * This assumes the network is connected. It's often not, so the [[cloud]] needs to be used to bridge between LANs. * Configurablity, including only enabling git syncing but not data transfer; From d9f26115c32c8df6865afc291d55b83b142c8428 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 01:59:21 -0400 Subject: [PATCH 4161/8313] use dbus to activate GduVolumeMonitor if it's not already running --- Assistant/Threads/MountWatcher.hs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index f1e33a99fa..a6c15540a6 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -22,6 +22,7 @@ import qualified Data.Set as S #if WITH_DBUS import DBus.Client import DBus +import Data.Word (Word32) #else #warning Building without dbus support; will use mtab polling #endif @@ -63,16 +64,34 @@ dbusThread st handle = (go =<< connectSession) `catchIO` onerr listClientNames :: Client -> IO [String] listClientNames client = do - reply <- call_ client (methodCall "/org/freedesktop/DBus" "org.freedesktop.DBus" "ListNames") - { methodCallDestination = Just "org.freedesktop.DBus" } + reply <- callDBus client "ListNames" [] return $ fromMaybe [] $ fromVariant (methodReturnBody reply !! 0) +callDBus :: Client -> MemberName -> [Variant] -> IO MethodReturn +callDBus client name params = call_ client $ + (methodCall "/org/freedesktop/DBus" "org.freedesktop.DBus" name) + { methodCallDestination = Just "org.freedesktop.DBus" + , methodCallBody = params + } + {- Examine the list of clients connected to dbus, to see if there - - are any we can use to monitor mounts. -} + - are any we can use to monitor mounts. If not, will attempt to start one. -} checkMountMonitor :: Client -> IO Bool -checkMountMonitor client = any (`elem` knownclients) <$> listClientNames client +checkMountMonitor client = ifM isrunning + ( return True + , startclient knownclients + ) where + isrunning = any (`elem` knownclients) <$> listClientNames client knownclients = ["org.gtk.Private.GduVolumeMonitor"] + startclient [] = return False + startclient (c:cs) = do + _ <- callDBus client "StartServiceByName" + [toVariant c, toVariant (0 :: Word32)] + ifM isrunning + ( return True + , startclient cs + ) {- Filter matching events recieved when drives are mounted. -} mountAdded ::MatchRule From 2fce3940b506a7671f622e872e049008df8ef4ad Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 02:16:09 -0400 Subject: [PATCH 4162/8313] catch all errors --- Assistant/Threads/MountWatcher.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index a6c15540a6..1cf854d0aa 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -17,6 +17,7 @@ import Utility.ThreadScheduler import Utility.Mounts import Control.Concurrent +import qualified Control.Exception as E import qualified Data.Set as S #if WITH_DBUS @@ -38,7 +39,7 @@ mountWatcherThread st handle = #if WITH_DBUS dbusThread :: ThreadState -> DaemonStatusHandle -> IO () -dbusThread st handle = (go =<< connectSession) `catchIO` onerr +dbusThread st handle = E.catch (go =<< connectSession) onerr where go client = ifM (checkMountMonitor client) ( do @@ -56,6 +57,7 @@ dbusThread st handle = (go =<< connectSession) `catchIO` onerr warning "No known volume monitor available through dbus; falling back to mtab polling" pollinstead ) + onerr :: E.SomeException -> IO () onerr e = do runThreadState st $ warning $ "Failed to use dbus; falling back to mtab polling (" ++ show e ++ ")" From 3e3b34ad10ae0332bade541a29a3fc448a88b0da Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 20 Jul 2012 07:31:14 +0000 Subject: [PATCH 4163/8313] --- ..._command_on_OSX_--_hangs_with_a_small_repo.mdwn | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn diff --git a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn new file mode 100644 index 0000000000..1b1acf3a98 --- /dev/null +++ b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn @@ -0,0 +1,14 @@ +I had started a fresh repo to test out the watch command again on OSX and noticed that it's borked, I'm not sure when it was broken. + +The snippet of the log message and command is + +
+$ git annex watch --foreground -d -v
+watch . read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","show-ref","git-annex"] 
+read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","show-ref","--hash","refs/heads/git-annex"] 
+read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","log","refs/heads/git-annex..6702e5361146450800ae5af0b63e97bd9c55d70b","--oneline","-n1"] 
+chat: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","cat-file","--batch"] 
+(scanning...) call: git ["--git-dir=/Users/jtan
+
+ +I had run git-annex with a new repo with just doing a git init and git annex init, I just threw in one or two small text files to see if it was working. It just hangs and does nothing. I had also tried it out on one of my bigger repos and it does the same thing it just hangs at _(scanning...)_ There isn't much to go on, I wonder if it's hitting the [[Issue on OSX with some system limits]] or if its just a thread/fork issue on OSX. From e99f700c90fb30a602bd38103ee6cb0756b1f552 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 20 Jul 2012 07:38:08 +0000 Subject: [PATCH 4164/8313] --- ...and_on_OSX_--_hangs_with_a_small_repo.mdwn | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn index 1b1acf3a98..81914d9cc2 100644 --- a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn +++ b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn @@ -12,3 +12,25 @@ chat: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang I had run git-annex with a new repo with just doing a git init and git annex init, I just threw in one or two small text files to see if it was working. It just hangs and does nothing. I had also tried it out on one of my bigger repos and it does the same thing it just hangs at _(scanning...)_ There isn't much to go on, I wonder if it's hitting the [[Issue on OSX with some system limits]] or if its just a thread/fork issue on OSX. + +It still hangs on the small repo even if I do + + $ sudo sysctl -w kern.maxfilesperproc=400000 + $ ulimit -n 2000 + +Also, just in case if you need it still (on a clean OSX 10.7 system) + +
+$ ulimit -a
+core file size          (blocks, -c) 0
+data seg size           (kbytes, -d) unlimited
+file size               (blocks, -f) unlimited
+max locked memory       (kbytes, -l) unlimited
+max memory size         (kbytes, -m) unlimited
+open files                      (-n) 256
+pipe size            (512 bytes, -p) 1
+stack size              (kbytes, -s) 8192
+cpu time               (seconds, -t) unlimited
+max user processes              (-u) 709
+virtual memory          (kbytes, -v) unlimited
+
From 55c512ff2cdd420bda5da2a0cdc222ed40376849 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus" Date: Fri, 20 Jul 2012 07:39:14 +0000 Subject: [PATCH 4165/8313] --- doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn index 81914d9cc2..664773633d 100644 --- a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn +++ b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn @@ -34,3 +34,5 @@ cpu time (seconds, -t) unlimited max user processes (-u) 709 virtual memory (kbytes, -v) unlimited + +Please close or merge this report if it's a duplicate. From 4726639f68f7a5d3b43e682746fb945f1c07e765 Mon Sep 17 00:00:00 2001 From: "http://mildred.fr/" Date: Fri, 20 Jul 2012 11:05:08 +0000 Subject: [PATCH 4166/8313] added pcre-devel dependency --- doc/install/Fedora.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn index 8aacbb3b09..73e9f9a5d9 100644 --- a/doc/install/Fedora.mdwn +++ b/doc/install/Fedora.mdwn @@ -1,7 +1,7 @@ Installation recipe for Fedora 14 thruough 17. 
-sudo yum install ghc cabal-install
+sudo yum install ghc cabal-install pcre-devel
 git clone git://git-annex.branchable.com/ git-annex
 cd git-annex
 git checkout ghc7.0

From 833bd24a33f2f13051439865464918821ba5c65c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jul 2012 12:01:28 -0400
Subject: [PATCH 4167/8313] tweak

---
 Assistant.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant.hs b/Assistant.hs
index 51639584c9..dc36957da7 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -124,7 +124,7 @@ startDaemon assistant foreground
 			pushmap <- newFailedPushMap
 			transferqueue <- newTransferQueue
 			transferslots <- newTransferSlots
-			mapM_ (void . forkIO)
+			mapM_ forkIO
 				[ commitThread st changechan commitchan transferqueue dstatus
 				, pushThread st dstatus commitchan pushmap
 				, pushRetryThread st pushmap

From aa1c64cb4b39b156560cfcd67341dd0f37c38ebb Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Fri, 20 Jul 2012 18:17:04 +0000
Subject: [PATCH 4168/8313] Added a comment

---
 ...ent_1_63f04694610839db0c2381005b15da35._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_1_63f04694610839db0c2381005b15da35._comment

diff --git a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_1_63f04694610839db0c2381005b15da35._comment b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_1_63f04694610839db0c2381005b15da35._comment
new file mode 100644
index 0000000000..1b9fe9a4af
--- /dev/null
+++ b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_1_63f04694610839db0c2381005b15da35._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="98.65.205.24"
+ subject="comment 1"
+ date="2012-07-20T18:17:04Z"
+ content="""
+I see this too, on OSX, and it's another one of the hangs with the threaded runtime I've been battling recently. Removing -threaded from the Makefile or git-annex.cabal (whichever you're using) makes it work.
+
+Seems likely this one is specific to my kqueue code. Indeed, I see it entering the kqueue code and then hanging, when it should be noticing changes. Interestingly, all threads seem to get blocked, too.
+
+AHA! I had the FFI functions marked unsafe. Marking safe seems to fix that. I need to go re-read up on the FFI and when it's safe to mark functions threadsafe.
+
+Thanks as always for the OSX eyes Jimmy!
+"""]]

From da4c506d61115236f3e43dd0bd17f30cd54df950 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jul 2012 15:03:24 -0400
Subject: [PATCH 4169/8313] use safe FFI imports

This avoids blocking all threads when calling waitchange_kqueue,
which blocks.
---
 Utility/Kqueue.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Utility/Kqueue.hs b/Utility/Kqueue.hs
index c1a0a5cd60..f44893195c 100644
--- a/Utility/Kqueue.hs
+++ b/Utility/Kqueue.hs
@@ -142,11 +142,11 @@ findDirContents dirmap dir = concatMap absolutecontents $ search
 		search = map snd $ M.toList $
 			M.filter (\i -> dirName i == dir) dirmap
 
-foreign import ccall unsafe "libkqueue.h init_kqueue" c_init_kqueue
+foreign import ccall safe "libkqueue.h init_kqueue" c_init_kqueue
 	:: IO Fd
-foreign import ccall unsafe "libkqueue.h addfds_kqueue" c_addfds_kqueue
+foreign import ccall safe "libkqueue.h addfds_kqueue" c_addfds_kqueue
 	:: Fd -> CInt -> Ptr Fd -> IO ()
-foreign import ccall unsafe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
+foreign import ccall safe "libkqueue.h waitchange_kqueue" c_waitchange_kqueue
 	:: Fd -> IO Fd
 
 {- Initializes a Kqueue to watch a directory, and all its subdirectories. -}

From 9d26b532abfd04bc97f9b922ddb7c14e545a0073 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jul 2012 15:03:58 -0400
Subject: [PATCH 4170/8313] use safe FFI imports for diskfree

There's a minor performance overhead to doing this, but this way I don't
have to worry about a situation where statfs might block for a long time.
For example, when it's on a network filesystem.
---
 Utility/DiskFree.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/DiskFree.hs b/Utility/DiskFree.hs
index ff70705621..18c7f2ee61 100644
--- a/Utility/DiskFree.hs
+++ b/Utility/DiskFree.hs
@@ -15,7 +15,7 @@ import Foreign.C.Types
 import Foreign.C.String
 import Foreign.C.Error
 
-foreign import ccall unsafe "libdiskfree.h diskfree" c_diskfree
+foreign import ccall safe "libdiskfree.h diskfree" c_diskfree
 	:: CString -> IO CULLong
 
 getDiskFree :: FilePath -> IO (Maybe Integer)

From f6d4786b860c92dffd855a90a070212324ff69dc Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jul 2012 15:07:48 -0400
Subject: [PATCH 4171/8313] left unsafe imports here; added a comment with a
 rationalle

---
 Utility/Mounts.hsc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc
index 4994c5e180..6b69e844a3 100644
--- a/Utility/Mounts.hsc
+++ b/Utility/Mounts.hsc
@@ -57,11 +57,13 @@ getMounts = do
 						}
 					getmntent h (ent:c)
 
+{- Using unsafe imports because the C functions are belived to never block.
+ - Note that getmntinfo is called with MNT_NOWAIT to avoid possibly blocking;
+ - while getmntent only accesses a file in /etc (or /proc) that should not
+ - block. -}
 foreign import ccall unsafe "libmounts.h mounts_start" c_mounts_start
         :: IO (Ptr ())
-
 foreign import ccall unsafe "libmounts.h mounts_next" c_mounts_next
         :: Ptr () -> IO (Ptr ())
-
 foreign import ccall unsafe "libmounts.h mounts_end" c_mounts_end
         :: Ptr () -> IO CInt

From aa72a6e51eba0b67dc2fc7c908712b3739369a5e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 20 Jul 2012 15:09:00 -0400
Subject: [PATCH 4172/8313] fixed

---
 doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn
index 664773633d..c598bb4d7d 100644
--- a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn
+++ b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo.mdwn
@@ -36,3 +36,6 @@ virtual memory          (kbytes, -v) unlimited
Please close or merge this report if it's a duplicate. + +> I've fixed this, it works in my (so far limited) tests. [[done]] +> --[[Joey]] From c42bf7e32e1bb1e91af569b9d72d8d210a74198f Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 16:31:06 -0400 Subject: [PATCH 4173/8313] add --- doc/design/assistant/thanks.mdwn | 233 +++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 doc/design/assistant/thanks.mdwn diff --git a/doc/design/assistant/thanks.mdwn b/doc/design/assistant/thanks.mdwn new file mode 100644 index 0000000000..35ee653208 --- /dev/null +++ b/doc/design/assistant/thanks.mdwn @@ -0,0 +1,233 @@ +The development of the git-annex assistant was made possible by the +generous donations of many people. I want to say "Thank You!" to each of +you individually, but until I meet all 951 of you, this page will have to +do. You have my most sincere thanks. --[[Joey]] + +(If I got your name wrong, or you don't want it publically posted here, +email .) + +## Major Backers + +These people are just inspiring in their enthusiasm and generosity to this +project. + +* Jason Scott +* strager + +## Beta Testers + +Whole weeks of my time were made possible thanks to each of these +people, and their testing is invaluable to the development of +the git-annex assistant. + +* Jimmy Tang +* David Pollak +* Pater +* Francois Marier +* Paul Sherwood +* Fred Epma +* Robert Ristroph +* Josh Triplett +* David Haslem +* AJ Ashton +* Svenne Krap +* Drew Hess + +## Prioritizers + +These forward-thinking people contributed generously just to help +set my priorities in which parts of the git-annex assistant were most +important to develop. + +Paul C. Bryan, Paul Tötterman, Don Marti, Dean Thompson, Djoume, David Johnston +Asokan Pichai, Anders Østhus, Dominik Wagenknecht, Charlie Fox, Yazz D. Atlas, +fenchel, Erik Penninga, Richard Hartmann, Graham, Stan Yamane, Ben Skelton, +Ian McEwen, asc, Paul Tagliamonte, Sherif Abouseda, Igor Támara, Anne Wind, +Mesar Hameed, Brandur K. Holm Petersen, Takahiro Inoue, Kai Hendry, +Stephen Youndt, Lee Roberson, Ben Strawbridge, Andrew Greenberg, Alfred Adams +Andrew, Aaron De Vries, Monti Knazze, Jorge Canseco, Hamish, Mark Eichin + +And special thanks to Kevin McKenzie, who also gave me a login to a Mac OSX +machine, which has proven invaluable. + +## Other Backers + +Most of the success of the Kickstarter is thanks to these folks. Some of +them spent significant amounts of money in the guise of getting some +swag. For others, being listed here, and being crucial to making the +git-annex assistant happen was reward enough. Large or small, these +contributions are, literally, my bread and butter this year. + +Amitai Schlair, mvime, Romain Lenglet, James Petts, Jouni Uuksulainen, +Wichert Akkerman, Robert Bellus, Kasper Souren, rob, Michiel Buddingh', +Kevin, Rob Avina, Alon Levy, Vikash, Michael Alan Dorman, Harley Pig, +Andreas Olsson, Pietpiet, Christine Spang, Liz Young, Oleg Kosorukov, +Allard Hoeve, Valentin Haenel, Joost Baaij, Nathan Yergler, Nathan Howell, +Frédéric Schütz, Matti Eskelinen, Neil McGovern, Lane Lillquist, db48x, +Stuart Prescott, Mark Matienzo, KarlTheGood, leonm, Drew Slininger, +Andreas Fuchs, Conrad Parker, Johannes Engelke, Battlegarden, Justin Kelly, +Robin Wagner, Thad Ward, crenquis, Trudy Goold, Mike Cochrane, Adam Venturella, +Russell Foo, furankupan, Giorgio Occhioni, andy, mind, Mike Linksvayer, +Stefan Strahl, Jelmer Vernooij, Markus Fix, David Hicks, Justin Azoff, +Iain Nicol, Bob Ippolito, Thomas Lundstrøm, Jason Mandel, federico2, +Edd Cochran, Jose Ortega, Emmett Agnew, Rudy Garcia, Kodi, Nick Rusnov, +Michael Rubin, Tom de Grunt, Richard Murray, Peter, Suzanne Pierce, Jared +Marcotte, folk, Eamon, Jeff Richards, Leo Sutedja, dann frazier, Mikkel +kristiansen, Matt Thomas, Kilian Evang, Gergely Risko, Kristian Rumberg, +Peter Kropf, Mark Hepburn, greymont, D. Joe Anderson, Jeremy Zunker, ctebo, +Manuel Roumain, Jason Walsh, np, Shawn, Johan Tibell, Branden Tyree, Dinyar +Rabady, Andrew Mason, damond armstead, Ethan Aubin, TomTom Tommie, Jimmy +Kaplowitz, Steven Zakulec, mike smith, Jacob Kirkwood, Mark Hymers, Nathan +Collins, Asbjørn Sloth Tønnesen, Misty De Meo, +Jim Paris, Adam Sjøgren, miniBill, Taneli, Kumar Appaiah, Greg Grossmeier, +Sten Turpin, Otavio Salvador, Teemu Hukkanen, Brian Stengaard, bob walker, +bibeneus, andrelo, Yaroslav Halchenko, hesfalling, Tommy L, jlargentaye, +Serafeim Zanikolas, Don Armstrong, Chris Cormack, shayne.oneill, Radu +Raduta, Josh S, Robin Sheat, Henrik Mygind, kodx, Christian, Geoff +Crompton, Brian May, Olivier Berger, Filippo Gadotti, Daniel Curto-Millet, +Eskild Hustvedt, Douglas Soares de Andrade, Tom L, Michael Nacos, Michaël +P., William Roe, Joshua Honeycutt, Brian Kelly, Nathan Rasch, jorge, Martin +Galese, alex cox, Avery Brooks, David Whittington, Dan Martinez, Forrest +Sutton, Jouni K. Seppänen, Arnold Cano, Robert Beaty, Daniel, Kevin Savetz, +Randy, Ernie Braganza, Aaron Haviland, Brian Brunswick, asmw, sean, Michael +Williams, Alexander, Dougal Campbell, Robert Bacchas, Michael Lewis, Collin +Price, Wes Frazier, Matt Wall, Brandon Barclay, Derek van Vliet, Martin +DeMello, kitt hodsden, Stephen Kitt, Leif Bergman, Simon Lilburn, Michael +Prokop, Christiaan Conover, Nick Coombe, Tim Dysinger, Brandon Robinson, +Philip Newborough, keith, Mike Fullerton, Kyle, Phil Windley, Tyler Head, +George V. Reilly, Matthew, Ali Gündüz, Vasyl Diakonov, Paolo Capriotti, +allanfranta, Martin Haeberli, msingle, Vincent Sanders, Steven King, Dmitry +Gribanov, Brandon High, Ben Hughes, Mike Dank, JohnE, Diggory Hardy, +Michael Hanke, valhalla, Samuli Tuomola, Jeff Rau, Benjamin Lebsanft, John +Drago, James, Aidan Cooper, rondie, Paul Kohler, Matthew Knights, Aaron +Junod, Patrick R McDonald, Christopher Browne, Daniel Engel, John SJ +Anderson, Peter Sarossy, Mike Prasad, Christoph Ender, Jan Dittberner, +Zohar, Alexander Jelinek, stefan, Danny O'Brien, Matthew Thode, Nicole +Aptekar, maurice gaston, Chris Adams, Mike Klemencic, Reedy, Subito, Tobias +Gruetzmacher, Ole-Morten Duesund, André Koot, mp, gdop, Cole Scott, Blaker, +Matt Sottile, W. Craig Trader, Louis-Philippe Dubrule, Brian Boucheron, +Duncan Smith, Brenton Buchbach, Kyle Stevenson, Eliot Lash, Egon Elbre, +Praveen, williamji, Thomas Schreiber, Neil Ford, Ryan Pratt, Joshua Brand, +Peter Cox, Markus Engstrom, John Sutherland, Dean Bailey, Ed Summers, +Hillel Arnold, David Fiander, Kurt Yoder, Trevor Muñoz, keri, Ivan +Sergeyenko, Shad Bolling, Tal Kelrich, Steve Presley, gerald ocker, Essex +Taylor, Josh Thomson, Trevor Bramble, Lance Higgins, Frank Motta, Dirk +Kraft, soundray, Joe Haskins, nmjk, Apurva Desai, Colin Dean, docwhat, +Joseph Costello, Jst, flamsmark, Alex Lang, Bill Traynor, Anthony David, +Marc-André Lureau, AlNapp, Giovanni Moretti, John Lawrence, João Paulo +Pizani Flor, Jim Ray, Gregory Thrush, Alistair McGann, Andrew Wied, +Koutarou Furukawa, Xiscu Ignacio, Aaron Sachs, Matt, Quirijn, Chet +Williams, Chris Gray, Bruce Segal, Tom Conder, Louis Tovar, Alex Duryee, +booltox, d8uv, Decklin Foster, Rafael Cervantes, Micah R Ledbetter, Kevin +Sjöberg, Johan Strombom, Zachary Cohen, Jason Lewis, Yves Bilgeri, Ville +Aine, Mark Hurley, Marco Bonetti, Maximilian Haack, Hynek Schlawack, +Michael Leinartas, Andreas Liebschner, Duotrig, Nat Fairbanks, David +Deutsch, Colin Hayhurst, calca, Kyle Goodrick, Marc Bobillier, Robert +Snook, James Kim, Olivier Serres, Jon Redfern, Itai Tavor, Michael +Fladischer, Rob, Jan Schmid, Thomas H., Anders Söderbäck, Abhishek +Dasgupta, Jeff Goeke-Smith, Tommy Thorn, bonuswavepilot, Philipp Edelmann, +Nick, Alejandro Navarro Fulleda, Yann Lallemant, andrew brennan, +Dave Allen Barker Jr, Fabian, Lukas Anzinger, Carl Witty, Andy Taylor, +Andre Klärner, Andrew Chilton, Adam Gibbins, Alexander Wauck, Shane O'Dea, +Paul Waite, Iain McLaren, Maggie Ellen Robin Hess, Willard Korfhage, +Nicolas, Eric Nikolaisen, Magnus Enger, Philipp Kern, Andrew Alderwick, +Raphael Wimmer, Benjamin Schötz, Ana Guerrero, Pete, Pieter van der Eems, +Aaron Suggs, Fred Benenson, Cedric Howe, Lance Ivy, Tieg Zaharia, Kevin +Cooney, Jon Williams, Anton Kudris, Roman Komarov, Brad Hilton, Rick Dakan, +Adam Whitcomb, Paul Casagrande, Evgueni Baldin, Robert Sanders, Kagan +Kayal, Dean Gardiner, micah altman, Cameron Banga, Ross Mcnairn, Oscar +Vilaplana, Robin Graham, Dan Gervais, Jon Åslund, Ragan Webber, Noble Hays, +stephen brown, Sean True, Maciek Swiech, faser, eikenberry, Kai Laborenz, +Sergey Fedoseev, Chris Fournier, Svend Sorensen, Greg K, wojtek, Johan +Ribenfors, Anton, Benjamin, Oleg Tsarev, PsychoHazard, John Cochrane, +Kasper Lauritzen, Patrick Naish, Rob, Keith Nasman, zenmaster, David Royer, +Max Woolf, Dan Gabber, martin rhoads, Martin Schmidinger, Paul +Scott-Wilson, Tom Gromak, Andy Webster, Dale Webb, Jim Watson, Stephen +Hansen, Mircea, Dan Goodenberger, Matthias Fink, Andy Gott, Daniel, Jai +Nelson, Shrayas Rajagopal, Vladimir Rutsky, Alexander, Thorben Westerhuys, +hiruki, Tao Neuendorffer Flaherty, Elline, Marco Hegenberg, robert, Balda, +Brennen Bearnes, Richard Parkins, David Gwilliam, Mark Johnson, Jeff Eaton, +Reddawn90, Heather Pusey, Chris Heinz, Colin, Phatsaphong Thadabusapa, +valunthar, Michael Martinez, redlukas, Yury V. Zaytsev, Blake, Tobias +"betabrain" A., Leon, sopyer, Steve Burnett, bessarabov, sarble, krsch.com, +Jack Self, Jeff Welch, Sam Pettiford, Jimmy Stridh, Diego Barberá, David +Steele, Oscar Ciudad, John Braman, Jacob, Nick Jenkins, Ben Sullivan, Brian +Cleary, James Brosnahan, Darryn Ten, Alex Brem, Jonathan Hitchcock, Jan +Schmidle, Wolfrzx99, Steve Pomeroy, Matthew Sitton, Finkregh, Derek Reeve, +GDR!, Cory Chapman, Marc Olivier Chouinard, Andreas Ryland, Justin, Andreas +Persenius, Games That Teach, Walter Somerville, Bill Haecker, Brandon +Phenix, Justin Shultz, Colin Scroggins, Tim Goddard, Ben Margolin, Michael +Martinez, David Hobbs, Andre Le, Jason Roberts, Bob Lopez, Gert Van Gool, +Robert Carnel, Anders Lundqvist, Aniruddha Sathaye, Marco Gillies, Basti +von Bejga, Esko Arajärvi, Dominik Deobald, Pavel Yudaev, Fionn Behrens, +Davide Favargiotti, Perttu Luukko, Silvan Jegen, Marcelo Bittencourt, +Leonard Peris, smercer, Alexandre Dupas, Solomon Matthews, Peter Hogg, +Richard E. Varian, Ian Oswald, James W. Sarvey, Ed Grether, Frederic +Savard, Sebastian Nerz, Hans-Chr. Jehg, Matija Nalis, Josh DiMauro, Jason +Harris, Adam Byrtek, Tellef, Magnus, Bart Schuurmans, Giel van Schijndel, +Ryan, kiodos, Richard 'maddog' Collins, PawZubr, Jason Gassel, Alex +Boisvert, Richard Thompson, maddi bolton, csights, Aaron Bryson, Jason Chu, +Maxime Côté, Kineteka Systems, Joe Cabezas, Mike Czepiel, Rami Nasra, +Christian Simonsen, Wouter Beugelsdijk, Adam Gibson, Gal Buki, James +Marble, Alan Chambers, Bernd Wiehle, Simon Korzun, Daniel Glassey, Eero af +Heurlin, Mikael, Timo Engelhardt, Wesley Faulkner, Jay Wo, Mike Belle, +David Fowlkes Jr., Jimmy Tang, Karl-Heinz Strass, Ed Mullins, Sam Flint, +Hendrica, Mark Emer Anderson, Joshua Cole, Jan Gondol, Henrik Lindhe, +Albert Delgado, Patrick, Alexa Avellar, Chris, sebsn1349, Maxim Kachur, +Andrew Marshall, Navjot Narula, Alwin Mathew, Christian Mangelsdorf, Avi +Shevin, Kevin S., Guillermo Sanchez Estrada, Alex Krieger, Luca Meier, Will +Jessop, Nick Ruest, Lani Aung, Ulf Benjaminsson, Rudi Engelbrecht, Miles +Matton, Cpt_Threepwood, Adam Kuyper, reacocard, David Kilsheimer, Peter +Olson, Bill Fischofer, Prashant Shah, Simon Bonnick, Alexander Grudtsov, +Antoine Boegli, Richard Warren, Sebastian Rust, AlmostHuman, Timmy +Crawford, PC, Marek Belski, pontus, Douglas S Butts, Eric Wallmander, Joe +Pelar, VIjay, Trahloc, Vernon Vantucci, Matthew baya, Viktor Štujber, +Stephen Akiki, Daniil Zamoldinov, Atley, Chris Thomson, Jacob Briggs, Falko +Richter, Andy Schmitz, Sergi Sagas, Peder Refsnes, Jonatan, Ben, Bill +Niblock, Agustin Acuna, Jeff Curl, Tim Humphrey, bib, James Zarbock, +Lachlan Devantier, Michal Berg, Jeff Lucas, Sid Irish, Franklyn, Jared +Dickson, Olli Jarva, Adam Gibson, Lukas Loesche, Jukka Määttä, Alexander +Lin, Dao Tran, Kirk, briankb, Ryan Villasenor, Daniel Wong, barista, Tomas +Jungwirth, Jesper Hansen, Nivin Singh, Alessandro Tieghi, Billy Roessler, +Peter Fetterer, Pallav Laskar, jcherney, Tyler Wang, Steve, Gigahost, Beat +Wolf, Hannibal Skorepa, aktiveradio, Mark Nunnikhoven, Bret Comnes, Alan +Ruttenberg, Anthony DiSanti, Adam Warkiewicz, Brian Bowman, Jonathan, Mark +Filley, Tobias Mohr, Christian St. Cyr, j. faceless user, Karl Miller, +Thomas Taimre, Vikram, Jason Mountcastle, Jason, Paul Elliott, Alexander, +Stephen Farmer, rayslava, Peter Leurs, Sky Kruse, JP Reeves, John J Schatz, +Martin Sandmair, Will Thompson, John Hergenroeder, Thomas, Christophe +Ponsart, Wolfdog, Eagertolearn, LukasM, Federico Hernandez, Vincent Bernat, +Christian Schmidt, Cameron Colby Thomson, Josh Duff, James Brown, Theron +Trowbridge, Falke, Don Meares, tauu, Greg Cornford, Max Fenton, Kenneth +Reitz, Bruce Bensetler, Mark Booth, Herb Mann, Sindre Sorhus, Chris +Knadler, Daniel Digerås, Derek, Sin Valentine, Ben Gamari, david +lampenscherf, fardles, Richard Burdeniuk, Tobias Kienzler, Dawid Humbla, +Bruno Barbaroxa, D Malt, krivar, James Valleroy, Peter, Tim Geddings, +Matthias Holzinger, Hanen, Petr Vacek, Raymond, Griff Maloney, Andreas +Helveg Rudolph, Nelson Blaha, Colonel Fubar, Skyjacker Captain Gavin +Phoenix, shaun, Michael, Kari Salminen, Rodrigo Miranda, Alan Chan, Justin +Eugene Evans, Isaac, Ben Staffin, Matthew Loar, Magos, Roderik, Eugenio +Piasini, Nico B, Scott Walter, Lior Amsalem, Thongrop Rodsavas, Alberto de +Paola, Shawn Poulen, John Swiderski, lluks, Waelen, Mark Slosarek, Jim +Cristol, mikesol, Bilal Quadri, LuP, Allan Nicolson, Kevin Washington, +Isaac Wedin, Paul Anguiano, ldacruz, Jason Manheim, Sawyer, Jason +Woofenden, Joe Danziger, Declan Morahan, KaptainUfolog, Vladron, bart, Jeff +McNeill, Christian Schlotter, Ben McQuillan, Anthony, Julian, Martin O, +altruism, Eric Solheim, MarkS, ndrwc, Matthew, David Lehn, Matthew +Cisneros, Mike Skoglund, Kristy Carey + +## Also thanks to + +* The Kickstarter team, who have unleashed much good on the world. +* The Haskell developers, who toiled for 20 years in obscurity + before most of us noticed them, and on whose giant shoulders I now stand, + in awe of the view. +* The Git developers, for obvious reasons. +* All of git-annex's early adopters, who turned it from a personal + toy project into something much more, and showed me the interest was there. +* Anna and Mark, for the loan of the video camera; as well as the rest of + my family, for your support. Even when I couldn't explain what I was + working on. +* The Hodges, for providing such a congenial place for me to live and work + on these first world problems, while you're off helping people in the + third world. From 8b458fe564132926c54e40111b1c4fa5d10634fa Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 16:37:30 -0400 Subject: [PATCH 4174/8313] link --- doc/design/assistant.mdwn | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/design/assistant.mdwn b/doc/design/assistant.mdwn index 62070e0201..0e49844d06 100644 --- a/doc/design/assistant.mdwn +++ b/doc/design/assistant.mdwn @@ -1,5 +1,6 @@ The git-annex assistant is being [crowd funded on Kickstarter](http://www.kickstarter.com/projects/joeyh/git-annex-assistant-like-dropbox-but-with-your-own/). +[[Thanks]] to all my backers. This is my design and plan for developing it. Still being fleshed out, still many ideas and use cases to add. From 42e73537d1977eac2da2760647e9131f5c9b9eed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 18:14:57 -0400 Subject: [PATCH 4175/8313] detect KDE automounting Best dbus events I could find were setupDone from org.kde.Solid.Device. There may be some spurious events, but that's ok, the code will only check to see if new mounts are available. It does not try to auto-start this service if it's not running. --- Assistant/Threads/MountWatcher.hs | 83 +++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index 1cf854d0aa..8636533517 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -19,6 +19,7 @@ import Utility.Mounts import Control.Concurrent import qualified Control.Exception as E import qualified Data.Set as S +import System.Log.Logger #if WITH_DBUS import DBus.Client @@ -48,10 +49,11 @@ dbusThread st handle = E.catch (go =<< connectSession) onerr - work out the mount point from the dbus - message, but this is easier. -} mvar <- newMVar =<< currentMountPoints - listen client mountAdded $ \_event -> do - nowmounted <- currentMountPoints - wasmounted <- swapMVar mvar nowmounted - handleMounts st handle wasmounted nowmounted + forM_ mountAdded $ \matcher -> + listen client matcher $ \_event -> do + nowmounted <- currentMountPoints + wasmounted <- swapMVar mvar nowmounted + handleMounts st handle wasmounted nowmounted , do runThreadState st $ warning "No known volume monitor available through dbus; falling back to mtab polling" @@ -64,8 +66,10 @@ dbusThread st handle = E.catch (go =<< connectSession) onerr pollinstead pollinstead = pollingThread st handle -listClientNames :: Client -> IO [String] -listClientNames client = do +type ServiceName = String + +listServiceNames :: Client -> IO [ServiceName] +listServiceNames client = do reply <- callDBus client "ListNames" [] return $ fromMaybe [] $ fromVariant (methodReturnBody reply !! 0) @@ -76,31 +80,53 @@ callDBus client name params = call_ client $ , methodCallBody = params } -{- Examine the list of clients connected to dbus, to see if there +{- Examine the list of services connected to dbus, to see if there - are any we can use to monitor mounts. If not, will attempt to start one. -} checkMountMonitor :: Client -> IO Bool -checkMountMonitor client = ifM isrunning - ( return True - , startclient knownclients - ) +checkMountMonitor client = do + running <- filter (`elem` usableservices) + <$> listServiceNames client + if null running + then startOneService client startableservices + else do + myDebug [ "Using running DBUS service" + , Prelude.head running + , "to monitor mount events." + ] + return True where - isrunning = any (`elem` knownclients) <$> listClientNames client - knownclients = ["org.gtk.Private.GduVolumeMonitor"] - startclient [] = return False - startclient (c:cs) = do - _ <- callDBus client "StartServiceByName" - [toVariant c, toVariant (0 :: Word32)] - ifM isrunning - ( return True - , startclient cs - ) + startableservices = [gvfs] + usableservices = startableservices ++ [kde] + gvfs = "org.gtk.Private.GduVolumeMonitor" + kde = "org.kde.DeviceNotifications" + +startOneService :: Client -> [ServiceName] -> IO Bool +startOneService _ [] = return False +startOneService client (x:xs) = do + _ <- callDBus client "StartServiceByName" + [toVariant x, toVariant (0 :: Word32)] + ifM (elem x <$> listServiceNames client) + ( do + myDebug [ "Started DBUS service" + , x + , "to monitor mount events." + ] + return True + , startOneService client xs + ) {- Filter matching events recieved when drives are mounted. -} -mountAdded ::MatchRule -mountAdded = matchAny - { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" - , matchMember = Just "MountAdded" - } +mountAdded :: [MatchRule] +mountAdded = [gvfs, kde] + where + gvfs = matchAny + { matchInterface = Just "org.gtk.Private.RemoteVolumeMonitor" + , matchMember = Just "MountAdded" + } + kde = matchAny + { matchInterface = Just "org.kde.Solid.Device" + , matchMember = Just "setupDone" + } #endif @@ -119,7 +145,7 @@ handleMounts st handle wasmounted nowmounted = mapM_ (handleMount st handle) $ handleMount :: ThreadState -> DaemonStatusHandle -> Mntent -> IO () handleMount st handle mntent = do - putStrLn $ "mounted: " ++ mnt_dir mntent + myDebug ["detected mount of", mnt_dir mntent] type MountPoints = S.Set Mntent @@ -130,3 +156,6 @@ currentMountPoints = S.fromList <$> getMounts {- Finds new mount points, given an old and a new set. -} newMountPoints :: MountPoints -> MountPoints -> MountPoints newMountPoints old new = S.difference new old + +myDebug :: [String] -> IO () +myDebug ms = debugM "MountWatcher" $ unwords ("MountWatcher:":ms) From 9c7475eb4637b682cfea70d02b22147b46b6e6eb Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 18:17:44 -0400 Subject: [PATCH 4176/8313] update --- doc/design/assistant/syncing.mdwn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index 52ebcd08b6..c8fb9882aa 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -16,8 +16,6 @@ all the other git clones, at both the git level and the key/value level. ## longer-term TODO -* Test MountWatcher on KDE, and add whatever dbus events KDE emits when - drives are mounted. * Test MountWatcher on Gnome (should work ok) and LXDE (dunno). * git-annex needs a simple speed control knob, which can be plumbed through to, at least, rsync. A good job for an hour in an @@ -110,3 +108,5 @@ anyway. maintaining the TransferSlots. Instead, need [[todo/assistant_threaded_runtime]], which would allow running something for sure when a transfer thread finishes. **done** +* Test MountWatcher on KDE, and add whatever dbus events KDE emits when + drives are mounted. **done** From b48d7747a3ac8bea7d58e8fff8faf791f98699c0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 19:29:59 -0400 Subject: [PATCH 4177/8313] debugging improvements add timestamps to debug messages Add lots of debug output in the assistant's threads. --- Assistant.hs | 4 ++- Assistant/Common.hs | 21 +++++++++++++ Assistant/Threads/Committer.hs | 23 ++++++++++++-- Assistant/Threads/Merger.hs | 17 ++++++++-- Assistant/Threads/MountWatcher.hs | 15 +++++---- Assistant/Threads/Pusher.hs | 28 +++++++++++++++-- Assistant/Threads/SanityChecker.hs | 15 +++++++-- Assistant/Threads/TransferWatcher.hs | 25 ++++++++++++--- Assistant/Threads/Transferrer.hs | 13 ++++++-- Assistant/Threads/Watcher.hs | 46 +++++++++++++++++++--------- Option.hs | 12 ++++++-- 11 files changed, 175 insertions(+), 44 deletions(-) create mode 100644 Assistant/Common.hs diff --git a/Assistant.hs b/Assistant.hs index dc36957da7..4bb1ed4cea 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -82,7 +82,7 @@ module Assistant where -import Common.Annex +import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.Changes @@ -136,6 +136,8 @@ startDaemon assistant foreground , mountWatcherThread st dstatus , watchThread st dstatus transferqueue changechan ] + debug "assistant" + ["all git-annex assistant threads started"] waitForTermination stopDaemon :: Annex () diff --git a/Assistant/Common.hs b/Assistant/Common.hs new file mode 100644 index 0000000000..c1a346e75c --- /dev/null +++ b/Assistant/Common.hs @@ -0,0 +1,21 @@ +{- Common infrastructure for the git-annex assistant threads. + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Common ( + module X, + ThreadName, + debug +) where + +import Common.Annex as X + +import System.Log.Logger + +type ThreadName = String + +debug :: ThreadName -> [String] -> IO () +debug threadname ws = debugM threadname $ unwords $ (threadname ++ ":") : ws diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs index ff5cc9eabc..ffb2494041 100644 --- a/Assistant/Threads/Committer.hs +++ b/Assistant/Threads/Committer.hs @@ -7,7 +7,7 @@ module Assistant.Threads.Committer where -import Common.Annex +import Assistant.Common import Assistant.Changes import Assistant.Commits import Assistant.ThreadedMonad @@ -31,6 +31,9 @@ import Data.Tuple.Utils import qualified Data.Set as S import Data.Either +thisThread :: ThreadName +thisThread = "Committer" + {- This thread makes git commits at appropriate times. -} commitThread :: ThreadState -> ChangeChan -> CommitChan -> TransferQueue -> DaemonStatusHandle -> IO () commitThread st changechan commitchan transferqueue dstatus = runEvery (Seconds 1) $ do @@ -45,10 +48,24 @@ commitThread st changechan commitchan transferqueue dstatus = runEvery (Seconds readychanges <- handleAdds st changechan transferqueue dstatus changes if shouldCommit time readychanges then do + debug thisThread + [ "committing" + , show (length readychanges) + , "changes" + ] void $ tryIO $ runThreadState st commitStaged recordCommit commitchan (Commit time) - else refillChanges changechan readychanges - else refillChanges changechan changes + else refill readychanges + else refill changes + where + refill cs = do + debug thisThread + [ "delaying commit of" + , show (length cs) + , "changes" + ] + refillChanges changechan cs + commitStaged :: Annex () commitStaged = do diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs index c7da86a8d3..10ea34692b 100644 --- a/Assistant/Threads/Merger.hs +++ b/Assistant/Threads/Merger.hs @@ -5,9 +5,12 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Assistant.Threads.Merger where +module Assistant.Threads.Merger ( + mergeThread, + manualPull, +) where -import Common.Annex +import Assistant.Common import Assistant.ThreadedMonad import Utility.DirWatcher import Utility.Types.DirWatcher @@ -19,6 +22,9 @@ import qualified Git.Branch import qualified Command.Sync import qualified Remote +thisThread :: ThreadName +thisThread = "Merger" + {- This thread watches for changes to .git/refs/heads/synced/*, - which indicate incoming pushes. It merges those pushes into the - currently checked out branch. -} @@ -33,6 +39,7 @@ mergeThread st = do , errHook = hook onErr } void $ watchDir dir (const False) hooks id + debug thisThread ["watching", dir] type Handler = Git.Repo -> FilePath -> Maybe FileStatus -> IO () @@ -68,7 +75,11 @@ onAdd g file _ let changedbranch = Git.Ref $ "refs" "heads" takeFileName file current <- Git.Branch.current g - when (Just changedbranch == current) $ + when (Just changedbranch == current) $ do + liftIO $ debug thisThread + [ "merging changes into" + , show current + ] void $ mergeBranch changedbranch g mergeBranch :: Git.Ref -> Git.Repo -> IO Bool diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index 8636533517..52614c32a4 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -10,7 +10,7 @@ module Assistant.Threads.MountWatcher where -import Common.Annex +import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus import Utility.ThreadScheduler @@ -19,7 +19,6 @@ import Utility.Mounts import Control.Concurrent import qualified Control.Exception as E import qualified Data.Set as S -import System.Log.Logger #if WITH_DBUS import DBus.Client @@ -29,6 +28,9 @@ import Data.Word (Word32) #warning Building without dbus support; will use mtab polling #endif +thisThread :: ThreadName +thisThread = "MountWatcher" + mountWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () mountWatcherThread st handle = #if WITH_DBUS @@ -89,7 +91,7 @@ checkMountMonitor client = do if null running then startOneService client startableservices else do - myDebug [ "Using running DBUS service" + debug thisThread [ "Using running DBUS service" , Prelude.head running , "to monitor mount events." ] @@ -107,7 +109,7 @@ startOneService client (x:xs) = do [toVariant x, toVariant (0 :: Word32)] ifM (elem x <$> listServiceNames client) ( do - myDebug [ "Started DBUS service" + debug thisThread [ "Started DBUS service" , x , "to monitor mount events." ] @@ -145,7 +147,7 @@ handleMounts st handle wasmounted nowmounted = mapM_ (handleMount st handle) $ handleMount :: ThreadState -> DaemonStatusHandle -> Mntent -> IO () handleMount st handle mntent = do - myDebug ["detected mount of", mnt_dir mntent] + debug thisThread ["detected mount of", mnt_dir mntent] type MountPoints = S.Set Mntent @@ -156,6 +158,3 @@ currentMountPoints = S.fromList <$> getMounts {- Finds new mount points, given an old and a new set. -} newMountPoints :: MountPoints -> MountPoints -> MountPoints newMountPoints old new = S.difference new old - -myDebug :: [String] -> IO () -myDebug ms = debugM "MountWatcher" $ unwords ("MountWatcher:":ms) diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs index 6d6836120e..e5191109cb 100644 --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -7,7 +7,7 @@ module Assistant.Threads.Pusher where -import Common.Annex +import Assistant.Common import Assistant.Commits import Assistant.Pushes import Assistant.DaemonStatus @@ -20,6 +20,9 @@ import Utility.Parallel import Data.Time.Clock import qualified Data.Map as M +thisThread :: ThreadName +thisThread = "Pusher" + {- This thread retries pushes that failed before. -} pushRetryThread :: ThreadState -> FailedPushMap -> IO () pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do @@ -27,6 +30,11 @@ pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do -- pushes to retry. topush <- getFailedPushesBefore pushmap (fromIntegral halfhour) unless (null topush) $ do + debug thisThread + [ "retrying" + , show (length topush) + , "failed pushes" + ] now <- getCurrentTime pushToRemotes now st pushmap topush where @@ -46,7 +54,13 @@ pushThread st daemonstatus commitchan pushmap = do remotes <- runThreadState st $ knownRemotes <$> getDaemonStatus daemonstatus pushToRemotes now st pushmap remotes - else refillCommits commitchan commits + else do + debug thisThread + [ "delaying push of" + , show (length commits) + , "commits" + ] + refillCommits commitchan commits {- Decide if now is a good time to push to remotes. - @@ -71,11 +85,20 @@ pushToRemotes now st pushmap remotes = do go True branch g remotes where go shouldretry branch g rs = do + debug thisThread + [ "pushing to" + , show rs + ] Command.Sync.updateBranch (Command.Sync.syncBranch branch) g (succeeded, failed) <- inParallel (push g branch) rs changeFailedPushMap pushmap $ \m -> M.union (makemap failed) $ M.difference m (makemap succeeded) + unless (null failed) $ + debug thisThread + [ "failed to push to" + , show failed + ] unless (null failed || not shouldretry) $ retry branch g failed @@ -86,5 +109,6 @@ pushToRemotes now st pushmap remotes = do ( exitSuccess, exitFailure) retry branch g rs = do + debug thisThread [ "trying manual pull to resolve failed pushes" ] runThreadState st $ manualPull branch rs go False branch g rs diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs index c5b99863e3..09aee0797c 100644 --- a/Assistant/Threads/SanityChecker.hs +++ b/Assistant/Threads/SanityChecker.hs @@ -9,22 +9,27 @@ module Assistant.Threads.SanityChecker ( sanityCheckerThread ) where -import Common.Annex -import qualified Git.LsFiles +import Assistant.Common import Assistant.DaemonStatus import Assistant.ThreadedMonad import Assistant.Changes import Assistant.TransferQueue +import qualified Git.LsFiles import Utility.ThreadScheduler import qualified Assistant.Threads.Watcher as Watcher import Data.Time.Clock.POSIX +thisThread :: ThreadName +thisThread = "SanityChecker" + {- This thread wakes up occasionally to make sure the tree is in good shape. -} sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () sanityCheckerThread st status transferqueue changechan = forever $ do waitForNextCheck st status + debug thisThread ["starting sanity check"] + runThreadState st $ modifyDaemonStatus_ status $ \s -> s { sanityCheckRunning = True } @@ -38,6 +43,9 @@ sanityCheckerThread st status transferqueue changechan = forever $ do { sanityCheckRunning = False , lastSanityCheck = Just now } + + debug thisThread ["sanity check complete"] + {- Only run one check per day, from the time of the last check. -} waitForNextCheck :: ThreadState -> DaemonStatusHandle -> IO () @@ -80,5 +88,6 @@ check st status transferqueue changechan = do insanity m = runThreadState st $ warning m addsymlink file s = do insanity $ "found unstaged symlink: " ++ file - Watcher.runHandler st status transferqueue changechan + Watcher.runHandler thisThread st status + transferqueue changechan Watcher.onAddSymlink file s diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs index 364ce04689..be520aaf93 100644 --- a/Assistant/Threads/TransferWatcher.hs +++ b/Assistant/Threads/TransferWatcher.hs @@ -7,7 +7,7 @@ module Assistant.Threads.TransferWatcher where -import Common.Annex +import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus import Logs.Transfer @@ -16,6 +16,9 @@ import Utility.Types.DirWatcher import Data.Map as M +thisThread :: ThreadName +thisThread = "TransferWatcher" + {- This thread watches for changes to the gitAnnexTransferDir, - and updates the DaemonStatus's map of ongoing transfers. -} transferWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () @@ -30,6 +33,7 @@ transferWatcherThread st dstatus = do , errHook = hook onErr } void $ watchDir dir (const False) hooks id + debug thisThread ["watching for transfers"] type Handler = ThreadState -> DaemonStatusHandle -> FilePath -> Maybe FileStatus -> IO () @@ -51,11 +55,17 @@ onErr _ _ msg _ = error msg onAdd :: Handler onAdd st dstatus file _ = case parseTransferFile file of Nothing -> noop - Just t -> runThreadState st $ go t =<< checkTransfer t + Just t -> do + runThreadState st $ go t =<< checkTransfer t where go _ Nothing = noop -- transfer already finished - go t (Just info) = adjustTransfers dstatus $ - M.insertWith' merge t info + go t (Just info) = do + liftIO $ debug thisThread + [ "transfer starting:" + , show t + ] + adjustTransfers dstatus $ + M.insertWith' merge t info -- preseve transferTid, which is not written to disk merge new old = new { transferTid = transferTid old } @@ -63,4 +73,9 @@ onAdd st dstatus file _ = case parseTransferFile file of onDel :: Handler onDel st dstatus file _ = case parseTransferFile file of Nothing -> noop - Just t -> void $ runThreadState st $ removeTransfer dstatus t + Just t -> do + debug thisThread + [ "transfer finishing:" + , show t + ] + void $ runThreadState st $ removeTransfer dstatus t diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs index c439d8b7ed..4ee5290e1c 100644 --- a/Assistant/Threads/Transferrer.hs +++ b/Assistant/Threads/Transferrer.hs @@ -7,7 +7,7 @@ module Assistant.Threads.Transferrer where -import Common.Annex +import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.TransferQueue @@ -22,6 +22,9 @@ import Data.Time.Clock.POSIX import Data.Time.Clock import qualified Data.Map as M +thisThread :: ThreadName +thisThread = "Transferrer" + {- For now only one transfer is run at a time. -} maxTransfers :: Int maxTransfers = 1 @@ -32,8 +35,12 @@ transfererThread st dstatus transferqueue slots = go where go = do (t, info) <- getNextTransfer transferqueue - whenM (runThreadState st $ shouldTransfer dstatus t info) $ - runTransfer st dstatus slots t info + ifM (runThreadState st $ shouldTransfer dstatus t info) + ( do + debug thisThread [ "Transferring:" , show t ] + runTransfer st dstatus slots t info + , debug thisThread [ "Skipping unnecessary transfer:" , show t ] + ) go {- Checks if the requested transfer is already running, or diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs index ae4fafb78f..617e6d77c5 100644 --- a/Assistant/Threads/Watcher.hs +++ b/Assistant/Threads/Watcher.hs @@ -5,9 +5,16 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Assistant.Threads.Watcher where +module Assistant.Threads.Watcher ( + watchThread, + checkCanWatch, + needLsof, + stageSymlink, + onAddSymlink, + runHandler, +) where -import Common.Annex +import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus import Assistant.Changes @@ -30,6 +37,9 @@ import Git.Types import Data.Bits.Utils import qualified Data.ByteString.Lazy as L +thisThread :: ThreadName +thisThread = "Watcher" + checkCanWatch :: Annex () checkCanWatch | canWatch = @@ -46,10 +56,12 @@ needLsof = error $ unlines ] watchThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () -watchThread st dstatus transferqueue changechan = void $ watchDir "." ignored hooks startup +watchThread st dstatus transferqueue changechan = do + void $ watchDir "." ignored hooks startup + debug thisThread [ "watching", "."] where startup = statupScan st dstatus - hook a = Just $ runHandler st dstatus transferqueue changechan a + hook a = Just $ runHandler thisThread st dstatus transferqueue changechan a hooks = WatchHooks { addHook = hook onAdd , delHook = hook onDel @@ -82,22 +94,22 @@ ignored = ig . takeFileName ig ".gitattributes" = True ig _ = False -type Handler = FilePath -> Maybe FileStatus -> DaemonStatusHandle -> TransferQueue -> Annex (Maybe Change) +type Handler = ThreadName -> FilePath -> Maybe FileStatus -> DaemonStatusHandle -> TransferQueue -> Annex (Maybe Change) {- Runs an action handler, inside the Annex monad, and if there was a - change, adds it to the ChangeChan. - - Exceptions are ignored, otherwise a whole watcher thread could be crashed. -} -runHandler :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO () -runHandler st dstatus transferqueue changechan handler file filestatus = void $ do +runHandler :: ThreadName -> ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> Handler -> FilePath -> Maybe FileStatus -> IO () +runHandler threadname st dstatus transferqueue changechan handler file filestatus = void $ do r <- tryIO go case r of Left e -> print e Right Nothing -> noop Right (Just change) -> recordChange changechan change where - go = runThreadState st $ handler file filestatus dstatus transferqueue + go = runThreadState st $ handler threadname file filestatus dstatus transferqueue {- During initial directory scan, this will be run for any regular files - that are already checked into git. We don't want to turn those into @@ -118,7 +130,7 @@ runHandler st dstatus transferqueue changechan handler file filestatus = void $ - the add. -} onAdd :: Handler -onAdd file filestatus dstatus _ +onAdd threadname file filestatus dstatus _ | maybe False isRegularFile filestatus = do ifM (scanComplete <$> getDaemonStatus dstatus) ( go @@ -129,14 +141,16 @@ onAdd file filestatus dstatus _ ) | otherwise = noChange where - go = pendingAddChange =<< Command.Add.lockDown file + go = do + liftIO $ debug threadname ["file added", file] + pendingAddChange =<< Command.Add.lockDown file {- A symlink might be an arbitrary symlink, which is just added. - Or, if it is a git-annex symlink, ensure it points to the content - before adding it. -} onAddSymlink :: Handler -onAddSymlink file filestatus dstatus transferqueue = go =<< Backend.lookupFile file +onAddSymlink threadname file filestatus dstatus transferqueue = go =<< Backend.lookupFile file where go (Just (key, _)) = do link <- calcGitLink file key @@ -146,6 +160,7 @@ onAddSymlink file filestatus dstatus transferqueue = go =<< Backend.lookupFile f checkcontent key s ensurestaged link s , do + liftIO $ debug threadname ["fix symlink", file] liftIO $ removeFile file liftIO $ createSymbolicLink link file addlink link @@ -175,6 +190,7 @@ onAddSymlink file filestatus dstatus transferqueue = go =<< Backend.lookupFile f {- For speed, tries to reuse the existing blob for - the symlink target. -} addlink link = do + liftIO $ debug threadname ["add symlink", file] v <- catObjectDetails $ Ref $ ':':file case v of Just (currlink, sha) @@ -195,7 +211,8 @@ onAddSymlink file filestatus dstatus transferqueue = go =<< Backend.lookupFile f | otherwise = noop onDel :: Handler -onDel file _ _dstatus _ = do +onDel threadname file _ _dstatus _ = do + liftIO $ debug threadname ["file deleted", file] Annex.Queue.addUpdateIndex =<< inRepo (Git.UpdateIndex.unstageFile file) madeChange file RmChange @@ -208,14 +225,15 @@ onDel file _ _dstatus _ = do - command to get the recursive list of files in the directory, so rm is - just as good. -} onDelDir :: Handler -onDelDir dir _ _dstatus _ = do +onDelDir threadname dir _ _dstatus _ = do + liftIO $ debug threadname ["directory deleted", dir] Annex.Queue.addCommand "rm" [Params "--quiet -r --cached --ignore-unmatch --"] [dir] madeChange dir RmDirChange {- Called when there's an error with inotify. -} onErr :: Handler -onErr msg _ _dstatus _ = do +onErr _ msg _ _dstatus _ = do warning msg return Nothing diff --git a/Option.hs b/Option.hs index 967cd3e07b..ff70fb6859 100644 --- a/Option.hs +++ b/Option.hs @@ -17,6 +17,9 @@ module Option ( import System.Console.GetOpt import System.Log.Logger +import System.Log.Formatter +import System.Log.Handler (setFormatter, LogHandler) +import System.Log.Handler.Simple import Common.Annex import qualified Annex @@ -48,8 +51,13 @@ common = setfast v = Annex.changeState $ \s -> s { Annex.fast = v } setauto v = Annex.changeState $ \s -> s { Annex.auto = v } setforcebackend v = Annex.changeState $ \s -> s { Annex.forcebackend = Just v } - setdebug = liftIO $ updateGlobalLogger rootLoggerName $ - setLevel DEBUG + setdebug = liftIO $ do + s <- simpledebug + updateGlobalLogger rootLoggerName + (setLevel DEBUG . setHandlers [s]) + simpledebug = setFormatter + <$> streamHandler stderr DEBUG + <*> pure (simpleLogFormatter "[$time] $msg") matcher :: [Option] matcher = From 163dd88ee2c200ce54b4dce3d4ebf9b6fd78c429 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 19:33:31 -0400 Subject: [PATCH 4178/8313] blog for the day --- doc/design/assistant/blog/day_41__foo.mdwn | 44 ++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 doc/design/assistant/blog/day_41__foo.mdwn diff --git a/doc/design/assistant/blog/day_41__foo.mdwn b/doc/design/assistant/blog/day_41__foo.mdwn new file mode 100644 index 0000000000..892e0aaee5 --- /dev/null +++ b/doc/design/assistant/blog/day_41__foo.mdwn @@ -0,0 +1,44 @@ +I made the MountWatcher only use dbus if it sees a client connected to dbus +that it knows will send mount events, or if it can start up such a client +via dbus. (Fancy!) Otherwise it falls back to polling. This should be enough +to support users who manually mount things -- if they have gvfs +installed, it'll be used to detect their manual mounts, even when a desktop +is not running, and if they don't have gvfs, they get polling. + +Support for KDE still needs to be added. I think it uses something other +than gvfs. If someone with KDE can run `dbus-monitor`, plug in a drive, get +it mounted, and send me a transcript, I can probably add it quickly. + +Of course, it'd also be nice to support anything similar on OSX that can +provide mount event notifications. Not a priority though, since the polling +code will work. + +--- + +Some OS X fixes today.. + +* Jimmy pointed out that my `getmntent` code broke the build on OSX again. + Sorry about that.. I keep thinking Unix portability nightmares are a 80's + thing, not a 2010's thing. Anyway, adapted a lot of hackish C code + to emulate `getmntent` on BSD systems, and it seems to work. (I actually + think the BSD interface to this is saner than Linux's, but I'd rather have + either one than both, sigh..) +* Kqueue was blocking all the threads on OSX. This is fixed, and the + assistant seems to be working on OSX again. + +---- + +I put together a preliminary page thanking everyone who contributed to the +git-annex Kickstarter. [[thanks]] The wall-o-names is scary crazy humbling. + +---- + +Improved `--debug` mode for the assistant, now every thread says whenever +it's doing anything interesting, and also there are timestamps. + +---- + +Had been meaning to get on with syncing to drives when they're mounted, but +got sidetracked with the above. Maybe tomorrow. I did think through it +in some detail as I was waking up this morning, and think I have a pretty +good handle on it. From b8cf9f2a7e84e4431b3fcbefd18c3f52ac7fedc0 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 20 Jul 2012 19:38:31 -0400 Subject: [PATCH 4179/8313] oops, I made it support KDE too --- doc/design/assistant/blog/day_41__foo.mdwn | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/design/assistant/blog/day_41__foo.mdwn b/doc/design/assistant/blog/day_41__foo.mdwn index 892e0aaee5..9115efd62a 100644 --- a/doc/design/assistant/blog/day_41__foo.mdwn +++ b/doc/design/assistant/blog/day_41__foo.mdwn @@ -5,9 +5,11 @@ to support users who manually mount things -- if they have gvfs installed, it'll be used to detect their manual mounts, even when a desktop is not running, and if they don't have gvfs, they get polling. -Support for KDE still needs to be added. I think it uses something other -than gvfs. If someone with KDE can run `dbus-monitor`, plug in a drive, get -it mounted, and send me a transcript, I can probably add it quickly. +Also, I got the MountWatcher to work with KDE. Found a dbus event that's +emitted when KDE mounts a drive, and this is also used. If anyone with +some other desktop environment wants me to add support for it, and it uses +dbus, it should be easy: Run `dbus-monitor`, plug in a drive, get +it mounted, and send me a transcript. Of course, it'd also be nice to support anything similar on OSX that can provide mount event notifications. Not a priority though, since the polling From 7c1f16b31a591f35b817985f4e97c7c574dfb5d6 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawlUbH3eytydcwlWqv8oauE2Jg4NwcV9uA0" Date: Fri, 20 Jul 2012 23:45:15 +0000 Subject: [PATCH 4180/8313] Added a comment: Special --- .../comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment diff --git a/doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment b/doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment new file mode 100644 index 0000000000..77a0873ee4 --- /dev/null +++ b/doc/design/assistant/thanks/comment_1_8b08b5c30e5aea3fc4599f856fd25df5._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawlUbH3eytydcwlWqv8oauE2Jg4NwcV9uA0" + nickname="Anna" + subject="Special" + date="2012-07-20T23:45:15Z" + content=""" +I feel pretty special getting an individualized thank you! Btw, the good news is that your video finally explained what you were working on so that I understood it. :-) +"""]] From 33e12c718304421e06d4d1c4ffb5487f5a5edb71 Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawnYD2ZzaOz-0anQDrN-Hg8Tvh5_C7wtStk" Date: Sat, 21 Jul 2012 20:31:33 +0000 Subject: [PATCH 4181/8313] Added a comment: Portability --- ...comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment diff --git a/doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment b/doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment new file mode 100644 index 0000000000..e27f7904ed --- /dev/null +++ b/doc/design/assistant/blog/day_41__foo/comment_1_ace21fa257a4c2fd412b6ff2944a23e8._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawnYD2ZzaOz-0anQDrN-Hg8Tvh5_C7wtStk" + nickname="roucaries" + subject="Portability" + date="2012-07-21T20:31:32Z" + content=""" +For portability why not using gnulib ? It will ease porting to windows BTW. + +Bastien +"""]] From b902a2960c74ded853bb2420f1d168f5c04d9dcd Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 21 Jul 2012 17:01:19 -0400 Subject: [PATCH 4182/8313] releasing version 3.20120721 --- debian/changelog | 4 ++-- git-annex.cabal | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/changelog b/debian/changelog index f90763acae..387dacd53f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -git-annex (3.20120630) UNRELEASED; urgency=low +git-annex (3.20120721) unstable; urgency=low * get, move, copy: Now refuse to do anything when the requested file transfer is already in progress by another process. @@ -13,7 +13,7 @@ git-annex (3.20120630) UNRELEASED; urgency=low .tar.gz Closes: #680450 * map: Write map.dot to .git/annex, which avoids watch trying to annex it. - -- Joey Hess Sun, 01 Jul 2012 15:04:37 -0400 + -- Joey Hess Sat, 21 Jul 2012 16:52:48 -0400 git-annex (3.20120629) unstable; urgency=low diff --git a/git-annex.cabal b/git-annex.cabal index 0bd35e14fe..15ac1e3d53 100644 --- a/git-annex.cabal +++ b/git-annex.cabal @@ -1,5 +1,5 @@ Name: git-annex -Version: 3.20120629 +Version: 3.20120721 Cabal-Version: >= 1.8 License: GPL Maintainer: Joey Hess From 898611ce5af106da840e5cebb3ed0e81a77ee184 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 21 Jul 2012 17:05:17 -0400 Subject: [PATCH 4183/8313] add news item for git-annex 3.20120721 --- doc/news/version_3.20120611.mdwn | 6 ------ doc/news/version_3.20120721.mdwn | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) delete mode 100644 doc/news/version_3.20120611.mdwn create mode 100644 doc/news/version_3.20120721.mdwn diff --git a/doc/news/version_3.20120611.mdwn b/doc/news/version_3.20120611.mdwn deleted file mode 100644 index e17456e236..0000000000 --- a/doc/news/version_3.20120611.mdwn +++ /dev/null @@ -1,6 +0,0 @@ -git-annex 3.20120611 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * add: Prevent (most) modifications from being made to a file while it - is being added to the annex. - * initremote: Automatically describe a remote when creating it. - * uninit: Refuse to run in a subdirectory. Closes: #[677076](http://bugs.debian.org/677076)"""]] \ No newline at end of file diff --git a/doc/news/version_3.20120721.mdwn b/doc/news/version_3.20120721.mdwn new file mode 100644 index 0000000000..7e424452e2 --- /dev/null +++ b/doc/news/version_3.20120721.mdwn @@ -0,0 +1,14 @@ +git-annex 3.20120721 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * get, move, copy: Now refuse to do anything when the requested file + transfer is already in progress by another process. + * status: Lists transfers that are currently in progress. + * Fix passing --uuid to git-annex-shell. + * When shaNsum commands cannot be found, use the Haskell SHA library + (already a dependency) to do the checksumming. This may be slower, + but avoids portability problems. + * Use SHA library for files less than 50 kb in size, at which point it's + faster than forking the more optimised external program. + * SHAnE backends are now smarter about composite extensions, such as + .tar.gz Closes: #[680450](http://bugs.debian.org/680450) + * map: Write map.dot to .git/annex, which avoids watch trying to annex it."""]] \ No newline at end of file From a77064cf65a2b375e4c7ac46e6e65680c5066996 Mon Sep 17 00:00:00 2001 From: hamish Date: Sun, 22 Jul 2012 07:13:37 +0000 Subject: [PATCH 4184/8313] Added a comment: dbus vs polling --- ...comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment diff --git a/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment b/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment new file mode 100644 index 0000000000..832be854a3 --- /dev/null +++ b/doc/design/assistant/blog/day_40__dbus/comment_2_6799f2baf6a6ce14b1fa76a8402840c0._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="hamish" + ip="203.0.139.24" + subject="dbus vs polling " + date="2012-07-22T07:13:37Z" + content=""" +I, too, am running a dbus but like to hand mount my filesystems. However, I'd imagine that I am both a minority and that my minority could like the extra control, so perhaps even a \"re-read the mtab /now/\" command that can be manually run after something is manually mounted would suffice + +Is it not possible to use inotify on the mtab? +"""]] From 345806b2dd94ffcc61ecc7e9b7d89a53d935acb8 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Sun, 22 Jul 2012 16:03:52 +0000 Subject: [PATCH 4185/8313] Added a comment --- ...comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment diff --git a/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment b/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment new file mode 100644 index 0000000000..a372670b86 --- /dev/null +++ b/doc/design/assistant/blog/day_40__dbus/comment_3_fa1d7444bdafcb990cacf2ace7ee6ef1._comment @@ -0,0 +1,10 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + ip="4.154.4.169" + subject="comment 3" + date="2012-07-22T16:03:52Z" + content=""" +How did I not think about using my favorite hammer on this problem too? But, no, /proc/mounts cannot be watched with inotify it seems, and of course the BSDs don't seem to have a file at all. + +I think the dbus stuff is sorted out for manual users, see later blog entries. +"""]] From 4ec9244f1af85b95d014103d93de913026b20fe3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 22 Jul 2012 13:48:50 -0400 Subject: [PATCH 4186/8313] add a path field to remotes Also broke out some helper functions around constructing remotes, to be used later. --- Remote/Bup.hs | 33 +++++++++++---------- Remote/Directory.hs | 1 + Remote/Git.hs | 70 +++++++++++++++++++++++++------------------- Remote/Hook.hs | 1 + Remote/List.hs | 14 +++++---- Remote/Rsync.hs | 33 +++++++++++---------- Remote/S3.hs | 1 + Remote/Web.hs | 1 + Types/Remote.hs | 2 ++ Utility/RsyncFile.hs | 6 ++++ 10 files changed, 97 insertions(+), 65 deletions(-) diff --git a/Remote/Bup.hs b/Remote/Bup.hs index 8a2c1afefe..83739a3e15 100644 --- a/Remote/Bup.hs +++ b/Remote/Bup.hs @@ -46,21 +46,24 @@ gen r u c = do return $ encryptableRemote c (storeEncrypted r buprepo) (retrieveEncrypted buprepo) - Remote { - uuid = u', - cost = cst, - name = Git.repoDescribe r, - storeKey = store r buprepo, - retrieveKeyFile = retrieve buprepo, - retrieveKeyFileCheap = retrieveCheap buprepo, - removeKey = remove, - hasKey = checkPresent r bupr', - hasKeyCheap = bupLocal buprepo, - whereisKey = Nothing, - config = c, - repo = r, - remotetype = remote - } + Remote + { uuid = u' + , cost = cst + , name = Git.repoDescribe r + , storeKey = store r buprepo + , retrieveKeyFile = retrieve buprepo + , retrieveKeyFileCheap = retrieveCheap buprepo + , removeKey = remove + , hasKey = checkPresent r bupr' + , hasKeyCheap = bupLocal buprepo + , whereisKey = Nothing + , config = c + , repo = r + , path = if bupLocal buprepo && not (null buprepo) + then Just buprepo + else Nothing + , remotetype = remote + } bupSetup :: UUID -> RemoteConfig -> Annex RemoteConfig bupSetup u c = do diff --git a/Remote/Directory.hs b/Remote/Directory.hs index 6b158730e8..1b75b937f7 100644 --- a/Remote/Directory.hs +++ b/Remote/Directory.hs @@ -53,6 +53,7 @@ gen r u c = do whereisKey = Nothing, config = Nothing, repo = r, + path = Just dir, remotetype = remote } where diff --git a/Remote/Git.hs b/Remote/Git.hs index 3412de89b4..f42a1d5366 100644 --- a/Remote/Git.hs +++ b/Remote/Git.hs @@ -5,7 +5,11 @@ - Licensed under the GNU GPL version 3 or higher. -} -module Remote.Git (remote, repoAvail) where +module Remote.Git ( + remote, + configRead, + repoAvail, +) where import qualified Data.Map as M import Control.Exception.Extensible @@ -45,7 +49,7 @@ list :: Annex [Git.Repo] list = do c <- fromRepo Git.config rs <- mapM (tweakurl c) =<< fromRepo Git.remotes - mapM configread rs + mapM configRead rs where annexurl n = "remote." ++ n ++ ".annexurl" tweakurl c r = do @@ -55,19 +59,21 @@ list = do Just url -> inRepo $ \g -> Git.Construct.remoteNamed n $ Git.Construct.fromRemoteLocation url g - {- It's assumed to be cheap to read the config of non-URL - - remotes, so this is done each time git-annex is run - - in a way that uses remotes. - - Conversely, the config of an URL remote is only read - - when there is no cached UUID value. -} - configread r = do - notignored <- repoNotIgnored r - u <- getRepoUUID r - case (repoCheap r, notignored, u) of - (_, False, _) -> return r - (True, _, _) -> tryGitConfigRead r - (False, _, NoUUID) -> tryGitConfigRead r - _ -> return r + +{- It's assumed to be cheap to read the config of non-URL remotes, so this is + - done each time git-annex is run in a way that uses remotes. + - + - Conversely, the config of an URL remote is only read when there is no + - cached UUID value. -} +configRead :: Git.Repo -> Annex Git.Repo +configRead r = do + notignored <- repoNotIgnored r + u <- getRepoUUID r + case (repoCheap r, notignored, u) of + (_, False, _) -> return r + (True, _, _) -> tryGitConfigRead r + (False, _, NoUUID) -> tryGitConfigRead r + _ -> return r repoCheap :: Git.Repo -> Bool repoCheap = not . Git.repoIsUrl @@ -76,21 +82,25 @@ gen :: Git.Repo -> UUID -> Maybe RemoteConfig -> Annex Remote gen r u _ = new <$> remoteCost r defcst where defcst = if repoCheap r then cheapRemoteCost else expensiveRemoteCost - new cst = Remote { - uuid = u, - cost = cst, - name = Git.repoDescribe r, - storeKey = copyToRemote r, - retrieveKeyFile = copyFromRemote r, - retrieveKeyFileCheap = copyFromRemoteCheap r, - removeKey = dropKey r, - hasKey = inAnnex r, - hasKeyCheap = repoCheap r, - whereisKey = Nothing, - config = Nothing, - repo = r, - remotetype = remote - } + new cst = Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = copyToRemote r + , retrieveKeyFile = copyFromRemote r + , retrieveKeyFileCheap = copyFromRemoteCheap r + , removeKey = dropKey r + , hasKey = inAnnex r + , hasKeyCheap = repoCheap r + , whereisKey = Nothing + , config = Nothing + , path = if Git.repoIsLocal r || Git.repoIsLocalUnknown r + then Just $ Git.repoPath r + else Nothing + , repo = r + , remotetype = remote + } + {- Checks relatively inexpensively if a repository is available for use. -} repoAvail :: Git.Repo -> Annex Bool diff --git a/Remote/Hook.hs b/Remote/Hook.hs index cad6e2fc94..9af851d149 100644 --- a/Remote/Hook.hs +++ b/Remote/Hook.hs @@ -47,6 +47,7 @@ gen r u c = do hasKeyCheap = False, whereisKey = Nothing, config = Nothing, + path = Nothing, repo = r, remotetype = remote } diff --git a/Remote/List.hs b/Remote/List.hs index 14a1771b48..4127cf24b0 100644 --- a/Remote/List.hs +++ b/Remote/List.hs @@ -2,7 +2,7 @@ {- git-annex remote list - - - Copyright 2011 Joey Hess + - Copyright 2011,2012 Joey Hess - - Licensed under the GNU GPL version 3 or higher. -} @@ -18,6 +18,7 @@ import Types.Remote import Annex.UUID import Config import Remote.Helper.Hooks +import qualified Git import qualified Remote.Git #ifdef WITH_S3 @@ -55,10 +56,13 @@ remoteList = do return rs' else return rs where - process m t = enumerate t >>= mapM (gen m t) - gen m t r = do - u <- getRepoUUID r - addHooks =<< generate t r u (M.lookup u m) + process m t = enumerate t >>= mapM (remoteGen m t) + +{- Generates a Remote. -} +remoteGen :: (M.Map UUID RemoteConfig) -> RemoteType -> Git.Repo -> Annex Remote +remoteGen m t r = do + u <- getRepoUUID r + addHooks =<< generate t r u (M.lookup u m) {- All remotes that are not ignored. -} enabledRemoteList :: Annex [Remote] diff --git a/Remote/Rsync.hs b/Remote/Rsync.hs index ee516a8a59..1ed73e119f 100644 --- a/Remote/Rsync.hs +++ b/Remote/Rsync.hs @@ -45,21 +45,24 @@ gen r u c = do return $ encryptableRemote c (storeEncrypted o) (retrieveEncrypted o) - Remote { - uuid = u, - cost = cst, - name = Git.repoDescribe r, - storeKey = store o, - retrieveKeyFile = retrieve o, - retrieveKeyFileCheap = retrieveCheap o, - removeKey = remove o, - hasKey = checkPresent r o, - hasKeyCheap = False, - whereisKey = Nothing, - config = Nothing, - repo = r, - remotetype = remote - } + Remote + { uuid = u + , cost = cst + , name = Git.repoDescribe r + , storeKey = store o + , retrieveKeyFile = retrieve o + , retrieveKeyFileCheap = retrieveCheap o + , removeKey = remove o + , hasKey = checkPresent r o + , hasKeyCheap = False + , whereisKey = Nothing + , config = Nothing + , repo = r + , path = if rsyncUrlIsPath $ rsyncUrl o + then Just $ rsyncUrl o + else Nothing + , remotetype = remote + } genRsyncOpts :: Git.Repo -> Maybe RemoteConfig -> Annex RsyncOpts genRsyncOpts r c = do diff --git a/Remote/S3.hs b/Remote/S3.hs index dca08fff8b..6e249ec4d5 100644 --- a/Remote/S3.hs +++ b/Remote/S3.hs @@ -60,6 +60,7 @@ gen' r u c cst = whereisKey = Nothing, config = c, repo = r, + path = Nothing, remotetype = remote } diff --git a/Remote/Web.hs b/Remote/Web.hs index 2516240ab3..02a2b5ab44 100644 --- a/Remote/Web.hs +++ b/Remote/Web.hs @@ -47,6 +47,7 @@ gen r _ _ = hasKeyCheap = False, whereisKey = Just getUrls, config = Nothing, + path = Nothing, repo = r, remotetype = remote } diff --git a/Types/Remote.hs b/Types/Remote.hs index c7628165c7..814be9febd 100644 --- a/Types/Remote.hs +++ b/Types/Remote.hs @@ -64,6 +64,8 @@ data RemoteA a = Remote { config :: Maybe RemoteConfig, -- git configuration for the remote repo :: Git.Repo, + -- a Remote can be assocated with a specific filesystem path + path :: Maybe FilePath, -- the type of the remote remotetype :: RemoteTypeA a } diff --git a/Utility/RsyncFile.hs b/Utility/RsyncFile.hs index 075e91d239..5a9a256a98 100644 --- a/Utility/RsyncFile.hs +++ b/Utility/RsyncFile.hs @@ -61,3 +61,9 @@ rsyncUrlIsShell s | c == '/' = False -- got to directory with no colon | c == ':' = not $ ":" `isPrefixOf` cs | otherwise = go cs + +{- Checks if a rsync url is really just a local path. -} +rsyncUrlIsPath :: String -> Bool +rsyncUrlIsPath s + | rsyncUrlIsShell s = False + | otherwise = ':' `notElem` s From e4f714d1be7b4341d08e10f1305b24c25da6d70e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 22 Jul 2012 15:06:18 -0400 Subject: [PATCH 4187/8313] pull from newly mounted git remotes --- Assistant/DaemonStatus.hs | 8 ++++ Assistant/Threads/MountWatcher.hs | 70 ++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs index 64c441ceee..88306a6363 100644 --- a/Assistant/DaemonStatus.hs +++ b/Assistant/DaemonStatus.hs @@ -60,6 +60,14 @@ modifyDaemonStatus_ handle a = liftIO $ modifyMVar_ handle (return . a) modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> Annex b modifyDaemonStatus handle a = liftIO $ modifyMVar handle (return . a) +{- Updates the cached ordered list of remotes from the list in Annex + - state. -} +updateKnownRemotes :: DaemonStatusHandle -> Annex () +updateKnownRemotes dstatus = do + remotes <- Command.Sync.syncRemotes [] + modifyDaemonStatus_ dstatus $ + \s -> s { knownRemotes = remotes } + {- Load any previous daemon status file, and store it in the MVar for this - process to use as its DaemonStatus. Also gets current transfer status. -} startDaemonStatus :: Annex DaemonStatusHandle diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index 52614c32a4..f32e043147 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -13,8 +13,16 @@ module Assistant.Threads.MountWatcher where import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus +import qualified Annex +import qualified Git import Utility.ThreadScheduler import Utility.Mounts +import Remote.List +import qualified Types.Remote as Remote +import qualified Remote.Git +import qualified Command.Sync +import Assistant.Threads.Merger +import Logs.Remote import Control.Concurrent import qualified Control.Exception as E @@ -42,7 +50,7 @@ mountWatcherThread st handle = #if WITH_DBUS dbusThread :: ThreadState -> DaemonStatusHandle -> IO () -dbusThread st handle = E.catch (go =<< connectSession) onerr +dbusThread st dstatus = E.catch (go =<< connectSession) onerr where go client = ifM (checkMountMonitor client) ( do @@ -55,7 +63,7 @@ dbusThread st handle = E.catch (go =<< connectSession) onerr listen client matcher $ \_event -> do nowmounted <- currentMountPoints wasmounted <- swapMVar mvar nowmounted - handleMounts st handle wasmounted nowmounted + handleMounts st dstatus wasmounted nowmounted , do runThreadState st $ warning "No known volume monitor available through dbus; falling back to mtab polling" @@ -66,7 +74,7 @@ dbusThread st handle = E.catch (go =<< connectSession) onerr runThreadState st $ warning $ "Failed to use dbus; falling back to mtab polling (" ++ show e ++ ")" pollinstead - pollinstead = pollingThread st handle + pollinstead = pollingThread st dstatus type ServiceName = String @@ -133,28 +141,70 @@ mountAdded = [gvfs, kde] #endif pollingThread :: ThreadState -> DaemonStatusHandle -> IO () -pollingThread st handle = go =<< currentMountPoints +pollingThread st dstatus = go =<< currentMountPoints where go wasmounted = do threadDelaySeconds (Seconds 10) nowmounted <- currentMountPoints - handleMounts st handle wasmounted nowmounted + handleMounts st dstatus wasmounted nowmounted go nowmounted handleMounts :: ThreadState -> DaemonStatusHandle -> MountPoints -> MountPoints -> IO () -handleMounts st handle wasmounted nowmounted = mapM_ (handleMount st handle) $ +handleMounts st dstatus wasmounted nowmounted = mapM_ (handleMount st dstatus) $ S.toList $ newMountPoints wasmounted nowmounted handleMount :: ThreadState -> DaemonStatusHandle -> Mntent -> IO () -handleMount st handle mntent = do - debug thisThread ["detected mount of", mnt_dir mntent] +handleMount st dstatus mntent = do + debug thisThread ["detected mount of", mnt_dir mntent] + rs <- remotesUnder st dstatus mntent + unless (null rs) $ do + branch <- runThreadState st $ Command.Sync.currentBranch + debug thisThread ["pulling from", show rs] + runThreadState st $ manualPull branch rs + -- TODO queue transfers for new files in both directions + where + +{- Finds remotes located underneath the mount point. + - + - Updates state to include the remotes. + - + - The config of git remotes is re-read, as it may not have been available + - at startup time, or may have changed (it could even be a different + - repository at the same remote location..) + -} +remotesUnder :: ThreadState -> DaemonStatusHandle -> Mntent -> IO [Remote] +remotesUnder st dstatus mntent = runThreadState st $ do + repotop <- fromRepo Git.repoPath + rs <- remoteList + pairs <- mapM (checkremote repotop) rs + let (waschanged, rs') = unzip pairs + when (any id waschanged) $ do + Annex.changeState $ \s -> s { Annex.remotes = rs' } + updateKnownRemotes dstatus + return $ map snd $ filter fst pairs + where + checkremote repotop r = case Remote.path r of + Just p | under mntent (absPathFrom repotop p) -> + (,) <$> pure True <*> updateremote r + _ -> return (False, r) + updateremote r = do + liftIO $ debug thisThread ["updating", show r] + m <- readRemoteLog + repo <- updaterepo $ Remote.repo r + remoteGen m (Remote.remotetype r) repo + updaterepo repo + | Git.repoIsLocal repo || Git.repoIsLocalUnknown repo = + Remote.Git.configRead repo + | otherwise = return repo type MountPoints = S.Set Mntent -{- Reads mtab, getting the current set of mount points. -} currentMountPoints :: IO MountPoints currentMountPoints = S.fromList <$> getMounts -{- Finds new mount points, given an old and a new set. -} newMountPoints :: MountPoints -> MountPoints -> MountPoints newMountPoints old new = S.difference new old + +{- Checks if a mount point contains a path. The path must be absolute. -} +under :: Mntent -> FilePath -> Bool +under = dirContains . mnt_dir From 26e4e65307436e4cc9a2db448141652b79d0f582 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 22 Jul 2012 15:09:40 -0400 Subject: [PATCH 4188/8313] filter out special remotes when pulling --- Assistant/Threads/MountWatcher.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index f32e043147..bfdfe0ebbd 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -159,8 +159,9 @@ handleMount st dstatus mntent = do rs <- remotesUnder st dstatus mntent unless (null rs) $ do branch <- runThreadState st $ Command.Sync.currentBranch - debug thisThread ["pulling from", show rs] - runThreadState st $ manualPull branch rs + let pullrs = filter Git.repoIsLocal rs + debug thisThread ["pulling from", show pullrs] + runThreadState st $ manualPull branch pullrs -- TODO queue transfers for new files in both directions where From 522f568450a005ae81b24f63bb37e75320b51219 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 22 Jul 2012 23:16:56 -0400 Subject: [PATCH 4189/8313] add TransferScanner thread Efficiently finding transfers that need to be done to get two repos back in sync seems like an interesting problem. --- Assistant.hs | 22 ++++++++++---- Assistant/ScanRemotes.hs | 41 +++++++++++++++++++++++++ Assistant/Threads/MountWatcher.hs | 45 +++++++++++++++------------- Assistant/Threads/Pusher.hs | 25 +++++++++------- Assistant/Threads/TransferScanner.hs | 34 +++++++++++++++++++++ Assistant/TransferQueue.hs | 14 ++++----- 6 files changed, 138 insertions(+), 43 deletions(-) create mode 100644 Assistant/ScanRemotes.hs create mode 100644 Assistant/Threads/TransferScanner.hs diff --git a/Assistant.hs b/Assistant.hs index 4bb1ed4cea..0049d31777 100644 --- a/Assistant.hs +++ b/Assistant.hs @@ -36,8 +36,7 @@ - inotify threads associated with it, too.) - Thread 9: transfer watcher - Watches for transfer information files being created and removed, - - and maintains the DaemonStatus currentTransfers map and the - - TransferSlots QSemN. + - and maintains the DaemonStatus currentTransfers map. - (This uses inotify on .git/annex/transfer/, so there are - additional inotify threads associated with it, too.) - Thread 10: transferrer @@ -49,8 +48,14 @@ - Thread 13: mount watcher - Either uses dbus to watch for drive mount events, or, when - there's no dbus, polls to find newly mounted filesystems. - - Once a filesystem that contains a remote is mounted, syncs - - with it. + - Once a filesystem that contains a remote is mounted, updates + - state about that remote, pulls from it, and queues a push to it, + - as well as an update, and queues it onto the + - ConnectedRemoteChan + - Thread 14: transfer scanner + - Does potentially expensive checks to find data that needs to be + - transferred from or to remotes, and queues Transfers. + - Uses the ScanRemotes map. - - ThreadState: (MVar) - The Annex state is stored here, which allows resuscitating the @@ -78,6 +83,9 @@ - to block until a slot is available. - This MVar should only be manipulated from inside the Annex monad, - which ensures it's accessed only after the ThreadState MVar. + - ScanRemotes (STM TMVar) + - Remotes that have been disconnected, and should be scanned + - are indicated by writing to this TMVar. -} module Assistant where @@ -88,6 +96,7 @@ import Assistant.DaemonStatus import Assistant.Changes import Assistant.Commits import Assistant.Pushes +import Assistant.ScanRemotes import Assistant.TransferQueue import Assistant.TransferSlots import Assistant.Threads.Watcher @@ -98,6 +107,7 @@ import Assistant.Threads.TransferWatcher import Assistant.Threads.Transferrer import Assistant.Threads.SanityChecker import Assistant.Threads.MountWatcher +import Assistant.Threads.TransferScanner import qualified Utility.Daemon import Utility.LogFile import Utility.ThreadScheduler @@ -124,6 +134,7 @@ startDaemon assistant foreground pushmap <- newFailedPushMap transferqueue <- newTransferQueue transferslots <- newTransferSlots + scanremotes <- newScanRemoteMap mapM_ forkIO [ commitThread st changechan commitchan transferqueue dstatus , pushThread st dstatus commitchan pushmap @@ -133,7 +144,8 @@ startDaemon assistant foreground , transfererThread st dstatus transferqueue transferslots , daemonStatusThread st dstatus , sanityCheckerThread st dstatus transferqueue changechan - , mountWatcherThread st dstatus + , mountWatcherThread st dstatus scanremotes + , transferScannerThread st scanremotes transferqueue , watchThread st dstatus transferqueue changechan ] debug "assistant" diff --git a/Assistant/ScanRemotes.hs b/Assistant/ScanRemotes.hs new file mode 100644 index 0000000000..05b2a2ca9f --- /dev/null +++ b/Assistant/ScanRemotes.hs @@ -0,0 +1,41 @@ +{- git-annex assistant remotes needing scanning + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.ScanRemotes where + +import Common.Annex +import Data.Function + +import Control.Concurrent.STM +import Data.Time.Clock +import qualified Data.Map as M + +type ScanRemoteMap = TMVar (M.Map Remote UTCTime) + +{- The TMVar starts empty, and is left empty when there are no remotes + - to scan. -} +newScanRemoteMap :: IO ScanRemoteMap +newScanRemoteMap = atomically newEmptyTMVar + +{- Blocks until there is a remote that needs to be scanned. + - Processes remotes added most recently first. -} +getScanRemote :: ScanRemoteMap -> IO Remote +getScanRemote v = atomically $ do + m <- takeTMVar v + let newest = Prelude.head $ reverse $ + map fst $ sortBy (compare `on` snd) $ M.toList m + putTMVar v $ M.delete newest m + return newest + +{- Adds new remotes that need scanning to the map. -} +addScanRemotes :: ScanRemoteMap -> [Remote] -> IO () +addScanRemotes _ [] = return () +addScanRemotes v rs = do + now <- getCurrentTime + atomically $ do + m <- fromMaybe M.empty <$> tryTakeTMVar v + putTMVar v $ foldr (`M.insert` now) m rs diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs index bfdfe0ebbd..853d96d51c 100644 --- a/Assistant/Threads/MountWatcher.hs +++ b/Assistant/Threads/MountWatcher.hs @@ -13,6 +13,8 @@ module Assistant.Threads.MountWatcher where import Assistant.Common import Assistant.ThreadedMonad import Assistant.DaemonStatus +import Assistant.ScanRemotes +import Assistant.Threads.Pusher (pushToRemotes) import qualified Annex import qualified Git import Utility.ThreadScheduler @@ -27,6 +29,7 @@ import Logs.Remote import Control.Concurrent import qualified Control.Exception as E import qualified Data.Set as S +import Data.Time.Clock #if WITH_DBUS import DBus.Client @@ -39,18 +42,18 @@ import Data.Word (Word32) thisThread :: ThreadName thisThread = "MountWatcher" -mountWatcherThread :: ThreadState -> DaemonStatusHandle -> IO () -mountWatcherThread st handle = +mountWatcherThread :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> IO () +mountWatcherThread st handle scanremotes = #if WITH_DBUS - dbusThread st handle + dbusThread st handle scanremotes #else - pollingThread st handle + pollingThread st handle scanremotes #endif #if WITH_DBUS -dbusThread :: ThreadState -> DaemonStatusHandle -> IO () -dbusThread st dstatus = E.catch (go =<< connectSession) onerr +dbusThread :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> IO () +dbusThread st dstatus scanremotes = E.catch (go =<< connectSession) onerr where go client = ifM (checkMountMonitor client) ( do @@ -63,7 +66,7 @@ dbusThread st dstatus = E.catch (go =<< connectSession) onerr listen client matcher $ \_event -> do nowmounted <- currentMountPoints wasmounted <- swapMVar mvar nowmounted - handleMounts st dstatus wasmounted nowmounted + handleMounts st dstatus scanremotes wasmounted nowmounted , do runThreadState st $ warning "No known volume monitor available through dbus; falling back to mtab polling" @@ -74,7 +77,7 @@ dbusThread st dstatus = E.catch (go =<< connectSession) onerr runThreadState st $ warning $ "Failed to use dbus; falling back to mtab polling (" ++ show e ++ ")" pollinstead - pollinstead = pollingThread st dstatus + pollinstead = pollingThread st dstatus scanremotes type ServiceName = String @@ -140,30 +143,32 @@ mountAdded = [gvfs, kde] #endif -pollingThread :: ThreadState -> DaemonStatusHandle -> IO () -pollingThread st dstatus = go =<< currentMountPoints +pollingThread :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> IO () +pollingThread st dstatus scanremotes = go =<< currentMountPoints where go wasmounted = do threadDelaySeconds (Seconds 10) nowmounted <- currentMountPoints - handleMounts st dstatus wasmounted nowmounted + handleMounts st dstatus scanremotes wasmounted nowmounted go nowmounted -handleMounts :: ThreadState -> DaemonStatusHandle -> MountPoints -> MountPoints -> IO () -handleMounts st dstatus wasmounted nowmounted = mapM_ (handleMount st dstatus) $ +handleMounts :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> MountPoints -> MountPoints -> IO () +handleMounts st dstatus scanremotes wasmounted nowmounted = mapM_ (handleMount st dstatus scanremotes) $ S.toList $ newMountPoints wasmounted nowmounted -handleMount :: ThreadState -> DaemonStatusHandle -> Mntent -> IO () -handleMount st dstatus mntent = do +handleMount :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> Mntent -> IO () +handleMount st dstatus scanremotes mntent = do debug thisThread ["detected mount of", mnt_dir mntent] rs <- remotesUnder st dstatus mntent unless (null rs) $ do branch <- runThreadState st $ Command.Sync.currentBranch - let pullrs = filter Git.repoIsLocal rs - debug thisThread ["pulling from", show pullrs] - runThreadState st $ manualPull branch pullrs - -- TODO queue transfers for new files in both directions - where + let nonspecial = filter (Git.repoIsLocal . Remote.repo) rs + unless (null nonspecial) $ do + debug thisThread ["pulling from", show nonspecial] + runThreadState st $ manualPull branch nonspecial + now <- getCurrentTime + pushToRemotes thisThread now st Nothing nonspecial + addScanRemotes scanremotes rs {- Finds remotes located underneath the mount point. - diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs index e5191109cb..cba53af233 100644 --- a/Assistant/Threads/Pusher.hs +++ b/Assistant/Threads/Pusher.hs @@ -1,4 +1,4 @@ -{- git-annex assistant git pushing threads +{- git-annex assistant git pushing thread - - Copyright 2012 Joey Hess - @@ -36,7 +36,7 @@ pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do , "failed pushes" ] now <- getCurrentTime - pushToRemotes now st pushmap topush + pushToRemotes thisThread now st (Just pushmap) topush where halfhour = 1800 @@ -53,7 +53,7 @@ pushThread st daemonstatus commitchan pushmap = do then do remotes <- runThreadState st $ knownRemotes <$> getDaemonStatus daemonstatus - pushToRemotes now st pushmap remotes + pushToRemotes thisThread now st (Just pushmap) remotes else do debug thisThread [ "delaying push of" @@ -78,24 +78,27 @@ shouldPush _now commits - - Avoids running possibly long-duration commands in the Annex monad, so - as not to block other threads. -} -pushToRemotes :: UTCTime -> ThreadState -> FailedPushMap -> [Remote] -> IO () -pushToRemotes now st pushmap remotes = do +pushToRemotes :: ThreadName -> UTCTime -> ThreadState -> (Maybe FailedPushMap) -> [Remote] -> IO () +pushToRemotes threadname now st mpushmap remotes = do (g, branch) <- runThreadState st $ (,) <$> fromRepo id <*> Command.Sync.currentBranch go True branch g remotes where go shouldretry branch g rs = do - debug thisThread + debug threadname [ "pushing to" , show rs ] Command.Sync.updateBranch (Command.Sync.syncBranch branch) g (succeeded, failed) <- inParallel (push g branch) rs - changeFailedPushMap pushmap $ \m -> - M.union (makemap failed) $ - M.difference m (makemap succeeded) + case mpushmap of + Nothing -> noop + Just pushmap -> + changeFailedPushMap pushmap $ \m -> + M.union (makemap failed) $ + M.difference m (makemap succeeded) unless (null failed) $ - debug thisThread + debug threadname [ "failed to push to" , show failed ] @@ -109,6 +112,6 @@ pushToRemotes now st pushmap remotes = do ( exitSuccess, exitFailure) retry branch g rs = do - debug thisThread [ "trying manual pull to resolve failed pushes" ] + debug threadname [ "trying manual pull to resolve failed pushes" ] runThreadState st $ manualPull branch rs go False branch g rs diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs new file mode 100644 index 0000000000..0a40f7ead2 --- /dev/null +++ b/Assistant/Threads/TransferScanner.hs @@ -0,0 +1,34 @@ +{- git-annex assistant thread to scan remotes to find needed transfers + - + - Copyright 2012 Joey Hess + - + - Licensed under the GNU GPL version 3 or higher. + -} + +module Assistant.Threads.TransferScanner where + +import Assistant.Common +import Assistant.ScanRemotes +import Assistant.TransferQueue +import Assistant.ThreadedMonad +import Logs.Transfer +import Types.Remote +import Utility.ThreadScheduler + +thisThread :: ThreadName +thisThread = "TransferScanner" + +{- This thread scans remotes, to find transfers that need to be made to + - keep their data in sync. The transfers are queued with lot priority. -} +transferScannerThread :: ThreadState -> ScanRemoteMap -> TransferQueue -> IO () +transferScannerThread st scanremotes transferqueue = do + runEvery (Seconds 2) $ do + r <- getScanRemote scanremotes + needtransfer <- scan st r + forM_ needtransfer $ \(f, t) -> + queueLaterTransfer transferqueue f t + +scan :: ThreadState -> Remote -> IO [(AssociatedFile, Transfer)] +scan st r = do + debug thisThread ["scanning", show r] + return [] -- TODO diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs index b0eca96c84..f8104914c1 100644 --- a/Assistant/TransferQueue.hs +++ b/Assistant/TransferQueue.hs @@ -38,19 +38,19 @@ queueTransfers q daemonstatus k f direction = do mapM_ (\r -> queue r $ gentransfer r) =<< sufficientremotes rs where - sufficientremotes l + sufficientremotes rs -- Queue downloads from all remotes that -- have the key, with the cheapest ones first. -- More expensive ones will only be tried if -- downloading from a cheap one fails. | direction == Download = do uuids <- Remote.keyLocations k - return $ filter (\r -> uuid r `elem` uuids) l + return $ filter (\r -> uuid r `elem` uuids) rs -- TODO: Determine a smaller set of remotes that -- can be uploaded to, in order to ensure all -- remotes can access the content. Currently, -- send to every remote we can. - | otherwise = return l + | otherwise = return rs gentransfer r = Transfer { transferDirection = direction , transferKey = k @@ -60,12 +60,12 @@ queueTransfers q daemonstatus k f direction = do let info = (stubInfo f) { transferRemote = Just r } writeTChan q (t, info) -{- Adds a pending transfer to the end of the queue. -} -queueTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO () -queueTransfer q f t = void $ atomically $ +{- Adds a transfer to the end of the queue, to be processed later. -} +queueLaterTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO () +queueLaterTransfer q f t = void $ atomically $ writeTChan q (t, stubInfo f) -{- Adds a pending transfer to the start of the queue, to be processed next. -} +{- Adds a transfer to the start of the queue, to be processed next. -} queueNextTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO () queueNextTransfer q f t = void $ atomically $ unGetTChan q (t, stubInfo f) From 892f1e6abefefee06dd3d2a3de8e9682f1848d88 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 22 Jul 2012 23:49:52 -0400 Subject: [PATCH 4190/8313] TransferScanner design thoughts --- doc/design/assistant/syncing.mdwn | 53 +++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index c8fb9882aa..a0e8d9d055 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -3,16 +3,55 @@ all the other git clones, at both the git level and the key/value level. ## immediate action items -* At startup, and possibly periodically, look for files we have that - location tracking indicates remotes do not, and enqueue Uploads for - them. Also, enqueue Downloads for any files we're missing. +* At startup, and possibly periodically, or when the network connection + changes, or some heuristic suggests that a remote was disconnected from + us for a while, queue remotes for processing by the TransferScanner, + to queue Transfers of files it or we're missing. * After git sync, identify content that we don't have that is now available - on remotes, and transfer. But first, need to ensure that when a remote + on remotes, and transfer. (Needed when we have a uni-directional connection + to a remote, so it won't be uploading content to us.) + But first, need to ensure that when a remote receives content, and updates its location log, it syncs that update out. -* When MountWatcher detects a newly mounted drive, rescan git remotes - in order to get ones on the drive, and do a git sync and file transfers - to sync any repositories on it. + +## TransferScanner + +The TransferScanner thread needs to find keys that need to be Uploaded +to a remote, or Downloaded from it. + +How to find the keys to transfer? I'd like to avoid potentially +expensive traversals of the whole git working copy if I can. + +One way would be to do a git diff between the (unmerged) git-annex branches +of the git repo, and its remote. Parse that for lines that add a key to +either, and queue transfers. That should work fairly efficiently when the +remote is a git repository. Indeed, git-annex already does such a diff +when it's doing a union merge of data into the git-annex branch. It +might even be possible to have the union merge and scan use the same +git diff data. + +But that approach has several problems: + +1. The list of keys it would generate wouldn't have associated git + filenames, so the UI couldn't show the user what files were being + transferred. +2. Worse, without filenames, any later features to exclude + files/directories from being transferred wouldn't work. +3. Looking at a git diff of the git-annex branches would find keys + that were added to either side while the two repos were disconnected. + But if the two repos' keys were not fully in sync before they + disconnected (which is quite possible; transfers could be incomplete), + the diff would not show those older out of sync keys. + +The remote could also be a special remote. In this case, I have to either +traverse the git working copy, or perhaps traverse the whole git-annex +branch (which would have the same problems with filesnames not being +available). + +If a traversal is done, should check all remotes, not just +one. Probably worth handling the case where a remote is connected +while in the middle of such a scan, so part of the scan needs to be +redone to check it. ## longer-term TODO From d7dc457cc0c22d8d0842765ae9ec0dbdd6ba0387 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sun, 22 Jul 2012 23:50:00 -0400 Subject: [PATCH 4191/8313] blog for the day --- .../assistant/blog/day_42__the_answer.mdwn | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 doc/design/assistant/blog/day_42__the_answer.mdwn diff --git a/doc/design/assistant/blog/day_42__the_answer.mdwn b/doc/design/assistant/blog/day_42__the_answer.mdwn new file mode 100644 index 0000000000..fd7c4ebb5a --- /dev/null +++ b/doc/design/assistant/blog/day_42__the_answer.mdwn @@ -0,0 +1,27 @@ +Made the MountWatcher update state for remotes located in a drive that +gets mounted. This was tricky code. First I had to make remotes declare +when they're located in a local directory. Then it has to rescan git +configs of git remotes (because the git repo mounted at a mount point may +change), and update all the state that a newly available remote can affect. + +And it works: I plug in a drive containing one of my git remotes, and the +assistant automatically notices it and syncs the git repositories. + +--- + +But, data isn't transferred yet. When a disconnected remote becomes +connected, keys should be transferred in both directions to get back into +sync. + +To that end, added Yet Another Thread; the TransferScanner thread +will scan newly available remotes to find keys, and queue low priority +transfers to get them fully in sync. + +(Later, this will probably also be used for network remotes that become +available when moving between networks. I think network-manager sends +dbus events it could use..) + +This new thread is missing a crucial peice, it doesn't yet have a way to +find the keys that need to be transferred. Doing that efficiently (without +scanning the whole git working copy) is Hard. I'm considering design +possibilities.. From 5e6e48c7f2693481922a902e57b78f96bc124c8b Mon Sep 17 00:00:00 2001 From: "https://www.google.com/accounts/o8/id?id=AItOawkZRoTRyW3tox-FD2DQWxskgI6_tkEtHL4" Date: Mon, 23 Jul 2012 16:12:01 +0000 Subject: [PATCH 4192/8313] Added a comment --- .../comment_6_a3b6000330c9c376611c228d746a1d55._comment | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment diff --git a/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment new file mode 100644 index 0000000000..c7defd13ef --- /dev/null +++ b/doc/bugs/fat_support/comment_6_a3b6000330c9c376611c228d746a1d55._comment @@ -0,0 +1,8 @@ +[[!comment format=mdwn + username="https://www.google.com/accounts/o8/id?id=AItOawkZRoTRyW3tox-FD2DQWxskgI6_tkEtHL4" + nickname="Ben" + subject="comment 6" + date="2012-07-23T16:11:52Z" + content=""" +The above would work fine for me but the files in my annex (e.g. .git/annex/objects/xx/yy/blah.ogg) don't have extensions like that, so my media player doesn't recognize them as media files. How do I get the files under \"objects\" to keep the extensions of the original files like in Joey's example? +"""]] From 25581b01c8ac604ef172f19d6e90479efa9b071c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 23 Jul 2012 19:55:26 -0400 Subject: [PATCH 4193/8313] works on Gnome 3 --- doc/design/assistant/syncing.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn index a0e8d9d055..f04f20218b 100644 --- a/doc/design/assistant/syncing.mdwn +++ b/doc/design/assistant/syncing.mdwn @@ -55,7 +55,7 @@ redone to check it. ## longer-term TODO -* Test MountWatcher on Gnome (should work ok) and LXDE (dunno). +* Test MountWatcher on LXDE. * git-annex needs a simple speed control knob, which can be plumbed through to, at least, rsync. A good job for an hour in an airport somewhere. From 9e0eb501c6c5146d0a94ed1c25e82b5763bb0434 Mon Sep 17 00:00:00 2001 From: "https://a-or-b.myopenid.com/" Date: Tue, 24 Jul 2012 03:26:47 +0000 Subject: [PATCH 4194/8313] Added a comment: Compiling git-annex on OSX (with 32 bit Haskell) --- ..._af6fe3540032cdf4400478de87771058._comment | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 doc/install/OSX/comment_16_af6fe3540032cdf4400478de87771058._comment diff --git a/doc/install/OSX/comment_16_af6fe3540032cdf4400478de87771058._comment b/doc/install/OSX/comment_16_af6fe3540032cdf4400478de87771058._comment new file mode 100644 index 0000000000..5da4b22c63 --- /dev/null +++ b/doc/install/OSX/comment_16_af6fe3540032cdf4400478de87771058._comment @@ -0,0 +1,30 @@ +[[!comment format=mdwn + username="https://a-or-b.myopenid.com/" + ip="203.45.2.230" + subject="Compiling git-annex on OSX (with 32 bit Haskell)" + date="2012-07-24T03:26:45Z" + content=""" +I came across an issue when following the instructions here: + + +I'm compiling the 'assistant' branch (522f568450a005ae81b24f63bb37e75320b51219). + + +The pre-compiled version of Haskell for OSX recommends the 32 bit installer, however git-annex compiles + +> Utility/libdiskfree.o Utility/libkqueue.o Utility/libmounts.o + +as 64 bit. The 'make' command fails on linking 32- and 64-bit code. + +So... I made a small change to the Makefile + +> CFLAGS=-Wall + +becomes + +> CFLAGS=-Wall -m32 + +I don't know if there is an easy way to programmatically check for this, or even if you'd want to spend time doing it, but it might help someone else out. + + +"""]] From 7929ff044d14cbba5b01ba0d8b2f3c31551e4df1 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Tue, 24 Jul 2012 14:51:50 +0000 Subject: [PATCH 4195/8313] Added a comment --- .../comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment diff --git a/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment new file mode 100644 index 0000000000..11668615e6 --- /dev/null +++ b/doc/bugs/fat_support/comment_7_a0ac7f2c44efc8116940c7b94b35e9d0._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 7" + date="2012-07-24T14:51:50Z" + content=""" +You can get the extensions by migrating to the SHA1E (or SHA256E) backend. +"""]] From 927b53088fa111c95082715555f3c385193f00d9 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 24 Jul 2012 11:01:15 -0400 Subject: [PATCH 4196/8313] force C compiler to build 32 bit on OSX when 32 bit ghc is being used --- Makefile | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4d56287468..9997ed3a03 100644 --- a/Makefile +++ b/Makefile @@ -3,20 +3,29 @@ mans=git-annex.1 git-annex-shell.1 sources=Build/SysConfig.hs Utility/Touch.hs all=$(bins) $(mans) docs +CFLAGS=-Wall + OS:=$(shell uname | sed 's/[-_].*//') ifeq ($(OS),Linux) -BASEFLAGS_OPTS+=-DWITH_INOTIFY +BASEFLAGS_OPTS=-DWITH_INOTIFY clibs=Utility/libdiskfree.o else -BASEFLAGS_OPTS+=-DWITH_KQUEUE +# BSD system +BASEFLAGS_OPTS=-DWITH_KQUEUE clibs=Utility/libdiskfree.o Utility/libkqueue.o +ifeq ($(OS),Darwin) +# Ensure OSX compiler builds for 32 bit when using 32 bit ghc +GHCARCH:=$(shell ghc -e 'print System.Info.arch') +ifeq ($(GHCARCH),i386) +CFLAGS=-Wall -m32 +endif +endif endif PREFIX=/usr IGNORE=-ignore-package monads-fd -ignore-package monads-tf BASEFLAGS=-Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 $(BASEFLAGS_OPTS) GHCFLAGS=-O2 $(BASEFLAGS) -CFLAGS=-Wall ifdef PROFILE GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS) From 9f165d35cadea4ba22a2273cdeb9d50ff27ed7b4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 24 Jul 2012 11:03:02 -0400 Subject: [PATCH 4197/8313] close --- ...orm___40__if_you_have_the_32bit_version_installed__41__.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn index 0414823f8b..3a50256db3 100644 --- a/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn +++ b/doc/bugs/subtle_build_issue_on_OSX_10.7_and_Haskell_Platform___40__if_you_have_the_32bit_version_installed__41__.mdwn @@ -46,3 +46,5 @@ Or just telling users to use the 64bit version of the haskell platform? It may also be possible to get osx's c compiler to output a universal binary to give you everything, but that be going down the _being too platform specific route_. + +> [[done]], it'll detect this and force -m32. --[[Joey]] From 9671527ee34bc551bd79914582d06e5200bf8621 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Tue, 24 Jul 2012 15:03:49 +0000 Subject: [PATCH 4198/8313] Added a comment --- .../comment_17_8d3a0596db67108041728b20f2790f31._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/install/OSX/comment_17_8d3a0596db67108041728b20f2790f31._comment diff --git a/doc/install/OSX/comment_17_8d3a0596db67108041728b20f2790f31._comment b/doc/install/OSX/comment_17_8d3a0596db67108041728b20f2790f31._comment new file mode 100644 index 0000000000..782f2fd759 --- /dev/null +++ b/doc/install/OSX/comment_17_8d3a0596db67108041728b20f2790f31._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 17" + date="2012-07-24T15:03:49Z" + content=""" +The instructions say to use cabal for a reason -- it's more likely to work. But I have made the Makefile detect the mismatched GHC and C compiler and force the C compiler to 32 bit. +"""]] From 078bad2d0a1e545ca83b9753465a5676feec4987 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 24 Jul 2012 11:08:36 -0400 Subject: [PATCH 4199/8313] move old OSX comments off the OSX page --- doc/install/OSX/old_comments.mdwn | 1 + .../comment_10_798000aab19af2944b6e44dbc550c6fe._comment | 0 .../comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment | 0 .../comment_12_60d13f2c8e008af1041bea565a392c83._comment | 0 .../comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment | 0 .../comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment | 0 .../comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment | 0 .../comment_16_af6fe3540032cdf4400478de87771058._comment | 0 .../comment_17_8d3a0596db67108041728b20f2790f31._comment | 0 .../comment_2_0327c64b15249596add635d26f4ce67f._comment | 0 .../comment_3_47c682a779812dda77601c24a619923c._comment | 0 .../comment_4_e6109a964064a2a799768a370e57801d._comment | 0 .../comment_5_50777853f808d57b957f8ce9a0f84b3d._comment | 0 .../comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment | 0 .../comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment | 0 .../comment_8_a93ad4b67c5df4243268bcf32562f6be._comment | 0 .../comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment | 0 17 files changed, 1 insertion(+) create mode 100644 doc/install/OSX/old_comments.mdwn rename doc/install/OSX/{ => old_comments}/comment_10_798000aab19af2944b6e44dbc550c6fe._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_12_60d13f2c8e008af1041bea565a392c83._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_16_af6fe3540032cdf4400478de87771058._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_17_8d3a0596db67108041728b20f2790f31._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_2_0327c64b15249596add635d26f4ce67f._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_3_47c682a779812dda77601c24a619923c._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_4_e6109a964064a2a799768a370e57801d._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment (100%) rename doc/install/OSX/{ => old_comments}/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment (100%) diff --git a/doc/install/OSX/old_comments.mdwn b/doc/install/OSX/old_comments.mdwn new file mode 100644 index 0000000000..ccb6785fd7 --- /dev/null +++ b/doc/install/OSX/old_comments.mdwn @@ -0,0 +1 @@ +Moved a bunch of outdated comments here, AFAIK all these issues are fixed. diff --git a/doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment b/doc/install/OSX/old_comments/comment_10_798000aab19af2944b6e44dbc550c6fe._comment similarity index 100% rename from doc/install/OSX/comment_10_798000aab19af2944b6e44dbc550c6fe._comment rename to doc/install/OSX/old_comments/comment_10_798000aab19af2944b6e44dbc550c6fe._comment diff --git a/doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment b/doc/install/OSX/old_comments/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment similarity index 100% rename from doc/install/OSX/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment rename to doc/install/OSX/old_comments/comment_11_707a1a27a15b2de8dfc8d1a30420ab4c._comment diff --git a/doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment b/doc/install/OSX/old_comments/comment_12_60d13f2c8e008af1041bea565a392c83._comment similarity index 100% rename from doc/install/OSX/comment_12_60d13f2c8e008af1041bea565a392c83._comment rename to doc/install/OSX/old_comments/comment_12_60d13f2c8e008af1041bea565a392c83._comment diff --git a/doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment b/doc/install/OSX/old_comments/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment similarity index 100% rename from doc/install/OSX/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment rename to doc/install/OSX/old_comments/comment_13_a6f48c87c2d6eabe379d6e10a6cac453._comment diff --git a/doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment b/doc/install/OSX/old_comments/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment similarity index 100% rename from doc/install/OSX/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment rename to doc/install/OSX/old_comments/comment_14_6ef2ddb7b11ce6ad54578ae118ed346e._comment diff --git a/doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment b/doc/install/OSX/old_comments/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment similarity index 100% rename from doc/install/OSX/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment rename to doc/install/OSX/old_comments/comment_15_6fd1fad5b6d9f36620e5a0e99edd2f89._comment diff --git a/doc/install/OSX/comment_16_af6fe3540032cdf4400478de87771058._comment b/doc/install/OSX/old_comments/comment_16_af6fe3540032cdf4400478de87771058._comment similarity index 100% rename from doc/install/OSX/comment_16_af6fe3540032cdf4400478de87771058._comment rename to doc/install/OSX/old_comments/comment_16_af6fe3540032cdf4400478de87771058._comment diff --git a/doc/install/OSX/comment_17_8d3a0596db67108041728b20f2790f31._comment b/doc/install/OSX/old_comments/comment_17_8d3a0596db67108041728b20f2790f31._comment similarity index 100% rename from doc/install/OSX/comment_17_8d3a0596db67108041728b20f2790f31._comment rename to doc/install/OSX/old_comments/comment_17_8d3a0596db67108041728b20f2790f31._comment diff --git a/doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment b/doc/install/OSX/old_comments/comment_2_0327c64b15249596add635d26f4ce67f._comment similarity index 100% rename from doc/install/OSX/comment_2_0327c64b15249596add635d26f4ce67f._comment rename to doc/install/OSX/old_comments/comment_2_0327c64b15249596add635d26f4ce67f._comment diff --git a/doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment b/doc/install/OSX/old_comments/comment_3_47c682a779812dda77601c24a619923c._comment similarity index 100% rename from doc/install/OSX/comment_3_47c682a779812dda77601c24a619923c._comment rename to doc/install/OSX/old_comments/comment_3_47c682a779812dda77601c24a619923c._comment diff --git a/doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment b/doc/install/OSX/old_comments/comment_4_e6109a964064a2a799768a370e57801d._comment similarity index 100% rename from doc/install/OSX/comment_4_e6109a964064a2a799768a370e57801d._comment rename to doc/install/OSX/old_comments/comment_4_e6109a964064a2a799768a370e57801d._comment diff --git a/doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment b/doc/install/OSX/old_comments/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment similarity index 100% rename from doc/install/OSX/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment rename to doc/install/OSX/old_comments/comment_5_50777853f808d57b957f8ce9a0f84b3d._comment diff --git a/doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment b/doc/install/OSX/old_comments/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment similarity index 100% rename from doc/install/OSX/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment rename to doc/install/OSX/old_comments/comment_6_18a8df794aa0ddd294dbf17d3d4c7fe2._comment diff --git a/doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment b/doc/install/OSX/old_comments/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment similarity index 100% rename from doc/install/OSX/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment rename to doc/install/OSX/old_comments/comment_7_2ce7acab15403d3f993cec94ec7f3bc6._comment diff --git a/doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment b/doc/install/OSX/old_comments/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment similarity index 100% rename from doc/install/OSX/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment rename to doc/install/OSX/old_comments/comment_8_a93ad4b67c5df4243268bcf32562f6be._comment diff --git a/doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment b/doc/install/OSX/old_comments/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment similarity index 100% rename from doc/install/OSX/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment rename to doc/install/OSX/old_comments/comment_9_ae3ed5345bc84f57e44251d2e6c39342._comment From 395d4d1026f851300e169ce901a941f482fc3b50 Mon Sep 17 00:00:00 2001 From: "http://joeyh.name/" Date: Tue, 24 Jul 2012 15:09:29 +0000 Subject: [PATCH 4200/8313] Added a comment --- .../comment_2_25552ff2942048fafe97d653757f1ad6._comment | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment diff --git a/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment b/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment new file mode 100644 index 0000000000..62ee109109 --- /dev/null +++ b/doc/install/OSX/comment_2_25552ff2942048fafe97d653757f1ad6._comment @@ -0,0 +1,7 @@ +[[!comment format=mdwn + username="http://joeyh.name/" + subject="comment 2" + date="2012-07-24T15:09:29Z" + content=""" +I've moved some outdated comments about installing on OSX to [[old_comments]]. +"""]] From 2cdc602eb9a4858fedec7b179a249b5006611eec Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 24 Jul 2012 11:14:36 -0400 Subject: [PATCH 4201/8313] cleanup --- doc/install/Fedora.mdwn | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/install/Fedora.mdwn b/doc/install/Fedora.mdwn index 73e9f9a5d9..c1905e01bf 100644 --- a/doc/install/Fedora.mdwn +++ b/doc/install/Fedora.mdwn @@ -1,4 +1,10 @@ -Installation recipe for Fedora 14 thruough 17. +git-annex is recently finding its way into Fedora. + +* [Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259) +* [Koji build for F17](http://koji.fedoraproject.org/koji/buildinfo?buildID=328654) +* [Koji build for F16](http://koji.fedoraproject.org/koji/buildinfo?buildID=328656) + +Installation recipe for Fedora 14 thruough 15. 
 sudo yum install ghc cabal-install pcre-devel
@@ -15,6 +21,3 @@ cabal install --bindir=$HOME/bin
 Note: You can't just use `cabal install git-annex`, because Fedora does
 not yet ship ghc 7.4.
 
-* [Status of getting a Fedora package](https://bugzilla.redhat.com/show_bug.cgi?id=662259)a
-* [Koji build for F17](http://koji.fedoraproject.org/koji/buildinfo?buildID=328654)
-* [Koji build for F16](http://koji.fedoraproject.org/koji/buildinfo?buildID=328656)

From 928be45bac84a144e52fcf9b8ea6cf0152be7c8f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Tue, 24 Jul 2012 11:15:39 -0400
Subject: [PATCH 4202/8313] reorg

---
 doc/install.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/install.mdwn b/doc/install.mdwn
index 3168976f47..1c58b729e3 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -9,8 +9,8 @@
 * [[ArchLinux]]
 * [[NixOS]]
 * [[Gentoo]]
-* Windows: [[sorry, not possible yet|todo/windows_support]]
 * [[ScientificLinux5]] - This should cover RHEL5 clones such as CentOS5 and so on
+* Windows: [[sorry, not possible yet|todo/windows_support]]
 
 ## Using cabal
 

From 1abc228008031fc48011f6cebf8f6e1f0438bf56 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawk4YX0PWICfWGRLuncCPufMPDctT7KAYJA"
 
Date: Tue, 24 Jul 2012 15:27:08 +0000
Subject: [PATCH 4203/8313] Added a comment: selective data syncing

---
 ..._c70156174ff19b503978d623bd2df36f._comment | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment

diff --git a/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment b/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment
new file mode 100644
index 0000000000..019490e619
--- /dev/null
+++ b/doc/design/assistant/syncing/comment_1_c70156174ff19b503978d623bd2df36f._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawk4YX0PWICfWGRLuncCPufMPDctT7KAYJA"
+ nickname="betabrain"
+ subject="selective data syncing"
+ date="2012-07-24T15:27:08Z"
+ content="""
+How will the assistant know which files' data to distribute between the repos?
+
+I'm using git-annex and it's numcopies attribute to maintain a redundant archive spread over different computers and usb drives. Not all drives should get a copy of everything, e.g. the usb drive I take to work should not automatically get a copy of family pictures.
+
+How about .gitattributes?
+
+* \* annex.auto-sync-data = false # don't automatically sync the data
+* archive/ annex.auto-push-repos = NAS # everything added to archive/ in any repo goes automatically to the NAS remote.
+* work/ annex.auto-synced-repos = LAPTOP WORKUSB # everything added to work/ in LAPTOP or WORKUSB gets synced to WORKUSB and LAPTOP
+* work/ annex.auto-push-repos = LAPTOP WORKUSB # stuff added to work/ anywhere gets synced to LAPTOP and WORKUSB
+* important/ annex.auto-sync-data = true # push data to all repos
+* webserver_logs/ annex.remote.WEBSERVER.auto-push-repos = S3 # only the assistant running in WEBSERVER pushes webserver_logs/ to S3 remote
+"""]]

From 47ad76d1105eda38ccc9917ce7bc1886d5aa89d0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 
Date: Tue, 24 Jul 2012 19:08:11 +0000
Subject: [PATCH 4204/8313]

---
 doc/forum/Fixing_up_corrupt_annexes.mdwn | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/forum/Fixing_up_corrupt_annexes.mdwn

diff --git a/doc/forum/Fixing_up_corrupt_annexes.mdwn b/doc/forum/Fixing_up_corrupt_annexes.mdwn
new file mode 100644
index 0000000000..be6beeca8f
--- /dev/null
+++ b/doc/forum/Fixing_up_corrupt_annexes.mdwn
@@ -0,0 +1,10 @@
+I was wondering how does one recover from...
+
+
+(Recording state in git...)
+error: invalid object 100644 8f154c946adc039af5240cc650a0a95c840e6fa6 for '041/5a4/SHA256-s6148--7ddcf853e4b16e77ab8c3c855c46867e6ed61c7089c334edf98bbdd3fb3a89ba.log'
+fatal: git-write-tree: error building trees
+git-annex: failed to read sha from git write-tree
+

+
+The above was caught when i ran a "git annex fsck --fast" to check stash of files"

From b5b0ae6f3e1c9edead1e697f401f7670a93d710a Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Tue, 24 Jul 2012 22:00:36 +0000
Subject: [PATCH 4205/8313] Added a comment

---
 .../comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment

diff --git a/doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment b/doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment
new file mode 100644
index 0000000000..335cbb51d2
--- /dev/null
+++ b/doc/forum/Fixing_up_corrupt_annexes/comment_1_cea21f96bcfb56aaab7ea03c1c804d2d._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-07-24T22:00:35Z"
+ content="""
+This is a corrupt git repository. See [[tips/what_to_do_when_a_repository_is_corrupted]]
+"""]]

From 6107328a6b981ec8130e4154be1ebe7bc11979df Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 12:07:30 -0400
Subject: [PATCH 4206/8313] avoid spawning new transfer thread until a slot
 becomes available

---
 Assistant/Threads/TransferScanner.hs | 2 +-
 Assistant/TransferSlots.hs           | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index 0a40f7ead2..485506e7d3 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -19,7 +19,7 @@ thisThread :: ThreadName
 thisThread = "TransferScanner"
 
 {- This thread scans remotes, to find transfers that need to be made to
- - keep their data in sync. The transfers are queued with lot priority. -}
+ - keep their data in sync. The transfers are queued with low priority. -}
 transferScannerThread :: ThreadState -> ScanRemoteMap -> TransferQueue -> IO ()
 transferScannerThread st scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs
index dc077254d9..710a188840 100644
--- a/Assistant/TransferSlots.hs
+++ b/Assistant/TransferSlots.hs
@@ -29,9 +29,10 @@ newTransferSlots = newQSemN numSlots
 {- Waits until a transfer slot becomes available, and runs a transfer
  - action in the slot, in its own thread. -}
 inTransferSlot :: TransferSlots -> ThreadState -> Annex a -> IO ThreadId
-inTransferSlot s st a = forkIO $ bracket_ start done run
+inTransferSlot s st a = do
+	waitQSemN s 1
+	forkIO $ bracket_ noop done run	
 	where
-		start = waitQSemN s 1
 		done = transferComplete s
 		run = unsafeRunThreadState st a
 

From a9dbfdf28d6c97c636e58be85f68d2a3f6efef77 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 13:12:34 -0400
Subject: [PATCH 4207/8313] better transfer queue management

Allow transfers to be added with blocking until the queue is sufficiently
small.

Better control over which end of the queue to add a transfer to.
---
 Assistant/Threads/Committer.hs       |  2 +-
 Assistant/Threads/TransferScanner.hs | 13 ++++--
 Assistant/Threads/Watcher.hs         |  2 +-
 Assistant/TransferQueue.hs           | 60 ++++++++++++++++++----------
 4 files changed, 52 insertions(+), 25 deletions(-)

diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs
index ffb2494041..33b92c7e53 100644
--- a/Assistant/Threads/Committer.hs
+++ b/Assistant/Threads/Committer.hs
@@ -161,7 +161,7 @@ handleAdds st changechan transferqueue dstatus cs = returnWhen (null pendingadds
 				sha <- inRepo $
 					Git.HashObject.hashObject BlobObject link
 				stageSymlink file sha
-			queueTransfers transferqueue dstatus key (Just file) Upload
+			queueTransfers Next transferqueue dstatus key (Just file) Upload
 			showEndOk
 			return $ Just change
 
diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index 485506e7d3..3c2e8dfabc 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -18,16 +18,23 @@ import Utility.ThreadScheduler
 thisThread :: ThreadName
 thisThread = "TransferScanner"
 
-{- This thread scans remotes, to find transfers that need to be made to
- - keep their data in sync. The transfers are queued with low priority. -}
+{- This thread waits until a remote needs to be scanned, to find transfers
+ - that need to be made, to keep data in sync.
+ -
+ - Remotes are scanned in the background; the scan is blocked when the
+ - transfer queue gets too large.
+ -}
 transferScannerThread :: ThreadState -> ScanRemoteMap -> TransferQueue -> IO ()
 transferScannerThread st scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
 		needtransfer <- scan st r
 		forM_ needtransfer $ \(f, t) ->
-			queueLaterTransfer transferqueue f t
+			queueTransferAt smallsize Later transferqueue f t
+	where
+		smallsize = 10
 
+{-  -}
 scan :: ThreadState -> Remote -> IO [(AssociatedFile, Transfer)]
 scan st r = do
 	debug thisThread ["scanning", show r]
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 617e6d77c5..31025361be 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -206,7 +206,7 @@ onAddSymlink threadname file filestatus dstatus transferqueue = go =<< Backend.l
 		 - try to get the key's content. -}
 		checkcontent key daemonstatus
 			| scanComplete daemonstatus = unlessM (inAnnex key) $
-				queueTransfers transferqueue dstatus
+				queueTransfers Next transferqueue dstatus
 					key (Just file) Download
 			| otherwise = noop
 
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index f8104914c1..1fb0bfa37f 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -15,10 +15,18 @@ import qualified Remote
 
 import Control.Concurrent.STM
 
-type TransferQueue = TChan (Transfer, TransferInfo)
+{- The transfer queue consists of a channel listing the transfers to make;
+ - the size of the queue is also tracked -}
+data TransferQueue = TransferQueue
+	{ queue :: TChan (Transfer, TransferInfo)
+	, queuesize :: TVar Integer
+	}
+
+data Schedule = Next | Later
+	deriving (Eq)
 
 newTransferQueue :: IO TransferQueue
-newTransferQueue = atomically newTChan
+newTransferQueue = atomically $ TransferQueue <$> newTChan <*> newTVar 0
 
 stubInfo :: AssociatedFile -> TransferInfo
 stubInfo f = TransferInfo
@@ -30,13 +38,11 @@ stubInfo f = TransferInfo
 	, associatedFile = f
 	}
 
-{- Adds pending transfers to the end of the queue for some of the known
- - remotes. -}
-queueTransfers :: TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex ()
-queueTransfers q daemonstatus k f direction = do
+{- Adds pending transfers to queue for some of the known remotes. -}
+queueTransfers :: Schedule -> TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex ()
+queueTransfers schedule q daemonstatus k f direction = do
 	rs <- knownRemotes <$> getDaemonStatus daemonstatus
-	mapM_ (\r -> queue r $ gentransfer r)
-		=<< sufficientremotes rs
+	mapM_ go =<< sufficientremotes rs
 	where
 		sufficientremotes rs
 			-- Queue downloads from all remotes that
@@ -56,20 +62,34 @@ queueTransfers q daemonstatus k f direction = do
 			, transferKey = k
 			, transferUUID = Remote.uuid r
 			}
-		queue r t = liftIO $ void $ atomically $ do
+		go r = liftIO $ atomically $ do
 			let info = (stubInfo f) { transferRemote = Just r }
-			writeTChan q (t, info)
+			enqueue schedule q (gentransfer r) info
 
-{- Adds a transfer to the end of the queue, to be processed later. -}
-queueLaterTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO ()
-queueLaterTransfer q f t = void $ atomically $
-	writeTChan q (t, stubInfo f)
+enqueue :: Schedule -> TransferQueue -> Transfer -> TransferInfo -> STM ()
+enqueue schedule q t info
+	| schedule == Next = go unGetTChan
+	| otherwise = go writeTChan
+	where
+		go a = do
+			void $ a (queue q) (t, info)
+			void $ modifyTVar' (queuesize q) succ
 
-{- Adds a transfer to the start of the queue, to be processed next. -}
-queueNextTransfer :: TransferQueue -> AssociatedFile -> Transfer -> IO ()
-queueNextTransfer q f t = void $ atomically $
-	unGetTChan q (t, stubInfo f)
+{- Adds a transfer to the queue. -}
+queueTransfer :: Schedule -> TransferQueue -> AssociatedFile -> Transfer -> IO ()
+queueTransfer schedule q f t = atomically $ enqueue schedule q t (stubInfo f)
 
-{- Blocks until a pending transfer is available in the queue. -}
+{- Blocks until the queue is no larger than a given size, and then adds a
+ - transfer to the queue. -}
+queueTransferAt :: Integer -> Schedule -> TransferQueue -> AssociatedFile -> Transfer -> IO ()
+queueTransferAt wantsz schedule q f t = atomically $ do
+	sz <- readTVar (queuesize q)
+	if sz <= wantsz
+		then enqueue schedule q t (stubInfo f)
+		else retry -- blocks until queuesize changes
+
+{- Blocks until a pending transfer is available from the queue. -}
 getNextTransfer :: TransferQueue -> IO (Transfer, TransferInfo)
-getNextTransfer = atomically . readTChan
+getNextTransfer q = atomically $ do
+	void $ modifyTVar' (queuesize q) pred
+	readTChan (queue q)

From 2b7f9c8442aea97d93011814b7ce6b05e0d576b6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 14:02:50 -0400
Subject: [PATCH 4208/8313] fix including of remote in TransferInfo when
 queueing new transfers

---
 Assistant/TransferQueue.hs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index 1fb0bfa37f..a01c85405a 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -28,17 +28,17 @@ data Schedule = Next | Later
 newTransferQueue :: IO TransferQueue
 newTransferQueue = atomically $ TransferQueue <$> newTChan <*> newTVar 0
 
-stubInfo :: AssociatedFile -> TransferInfo
-stubInfo f = TransferInfo
+stubInfo :: AssociatedFile -> Remote -> TransferInfo
+stubInfo f r = TransferInfo
 	{ startedTime = Nothing
 	, transferPid = Nothing
 	, transferTid = Nothing
-	, transferRemote = Nothing
+	, transferRemote = Just r
 	, bytesComplete = Nothing
 	, associatedFile = f
 	}
 
-{- Adds pending transfers to queue for some of the known remotes. -}
+{- Adds transfers to queue for some of the known remotes. -}
 queueTransfers :: Schedule -> TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex ()
 queueTransfers schedule q daemonstatus k f direction = do
 	rs <- knownRemotes <$> getDaemonStatus daemonstatus
@@ -62,9 +62,8 @@ queueTransfers schedule q daemonstatus k f direction = do
 			, transferKey = k
 			, transferUUID = Remote.uuid r
 			}
-		go r = liftIO $ atomically $ do
-			let info = (stubInfo f) { transferRemote = Just r }
-			enqueue schedule q (gentransfer r) info
+		go r = liftIO $ atomically $
+			enqueue schedule q (gentransfer r) (stubInfo f r)
 
 enqueue :: Schedule -> TransferQueue -> Transfer -> TransferInfo -> STM ()
 enqueue schedule q t info
@@ -76,16 +75,17 @@ enqueue schedule q t info
 			void $ modifyTVar' (queuesize q) succ
 
 {- Adds a transfer to the queue. -}
-queueTransfer :: Schedule -> TransferQueue -> AssociatedFile -> Transfer -> IO ()
-queueTransfer schedule q f t = atomically $ enqueue schedule q t (stubInfo f)
+queueTransfer :: Schedule -> TransferQueue -> AssociatedFile -> Transfer -> Remote -> IO ()
+queueTransfer schedule q f t remote = atomically $
+	enqueue schedule q t (stubInfo f remote)
 
 {- Blocks until the queue is no larger than a given size, and then adds a
  - transfer to the queue. -}
-queueTransferAt :: Integer -> Schedule -> TransferQueue -> AssociatedFile -> Transfer -> IO ()
-queueTransferAt wantsz schedule q f t = atomically $ do
+queueTransferAt :: Integer -> Schedule -> TransferQueue -> AssociatedFile -> Transfer -> Remote -> IO ()
+queueTransferAt wantsz schedule q f t remote = atomically $ do
 	sz <- readTVar (queuesize q)
 	if sz <= wantsz
-		then enqueue schedule q t (stubInfo f)
+		then enqueue schedule q t (stubInfo f remote)
 		else retry -- blocks until queuesize changes
 
 {- Blocks until a pending transfer is available from the queue. -}

From b665ffe36f83587624e98dfe58cb75ac068525b7 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 14:15:09 -0400
Subject: [PATCH 4209/8313] implement simple working copy based scan

Works.. could be more efficient.
---
 Assistant/Threads/TransferScanner.hs | 44 +++++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 7 deletions(-)

diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index 3c2e8dfabc..c2685ae825 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -12,8 +12,13 @@ import Assistant.ScanRemotes
 import Assistant.TransferQueue
 import Assistant.ThreadedMonad
 import Logs.Transfer
+import Logs.Location
 import Types.Remote
+import qualified Remote
 import Utility.ThreadScheduler
+import qualified Git.LsFiles as LsFiles
+import Command
+import Annex.Content
 
 thisThread :: ThreadName
 thisThread = "TransferScanner"
@@ -28,14 +33,39 @@ transferScannerThread :: ThreadState -> ScanRemoteMap -> TransferQueue -> IO ()
 transferScannerThread st scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
-		needtransfer <- scan st r
+		liftIO $ debug thisThread ["starting scan of", show r]
+		needtransfer <- runThreadState st $ scan r
 		forM_ needtransfer $ \(f, t) ->
-			queueTransferAt smallsize Later transferqueue f t
+			queueTransferAt smallsize Later transferqueue f t r
+		liftIO $ debug thisThread ["finished scan of", show r]
 	where
 		smallsize = 10
 
-{-  -}
-scan :: ThreadState -> Remote -> IO [(AssociatedFile, Transfer)]
-scan st r = do
-	debug thisThread ["scanning", show r]
-	return [] -- TODO 
+{- This is a naive scan through the git work tree. -}
+scan :: Remote -> Annex [(AssociatedFile, Transfer)]
+scan r = do
+	files <- inRepo $ LsFiles.inRepo []
+	catMaybes <$> forM files (whenAnnexed go)
+	where
+		u = Remote.uuid r
+
+		go file (key, _) =
+			ifM (inAnnex key)
+				( check Upload False =<< remotehas key
+				, check Download True =<< remotehas key
+				)
+			where
+				check direction x y
+					| x == y = return $
+						Just (Just file, Transfer direction u key)
+					| otherwise = return Nothing
+
+		{- Look directly in remote for the key when it's cheap;
+		 - otherwise rely on the location log. -}
+		remotehas key
+			| Remote.hasKeyCheap r = (==)
+				<$> pure (Right True)
+				<*> Remote.hasKey r key
+			| otherwise = elem
+				<$> pure u
+				<*> loggedLocations key

From 95c80b644046f6fabe445972de68be40285f1841 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 14:54:09 -0400
Subject: [PATCH 4210/8313] laziness fix

Now scanning runs fully interleaved with transferring.
---
 Assistant/Threads/TransferScanner.hs | 50 ++++++++++++++++------------
 Assistant/TransferQueue.hs           | 10 +++++-
 2 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index c2685ae825..e76cbe81d3 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -13,7 +13,6 @@ import Assistant.TransferQueue
 import Assistant.ThreadedMonad
 import Logs.Transfer
 import Logs.Location
-import Types.Remote
 import qualified Remote
 import Utility.ThreadScheduler
 import qualified Git.LsFiles as LsFiles
@@ -25,40 +24,47 @@ thisThread = "TransferScanner"
 
 {- This thread waits until a remote needs to be scanned, to find transfers
  - that need to be made, to keep data in sync.
- -
- - Remotes are scanned in the background; the scan is blocked when the
- - transfer queue gets too large.
  -}
 transferScannerThread :: ThreadState -> ScanRemoteMap -> TransferQueue -> IO ()
 transferScannerThread st scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
 		liftIO $ debug thisThread ["starting scan of", show r]
-		needtransfer <- runThreadState st $ scan r
-		forM_ needtransfer $ \(f, t) ->
-			queueTransferAt smallsize Later transferqueue f t r
+		scan st transferqueue r
 		liftIO $ debug thisThread ["finished scan of", show r]
 	where
-		smallsize = 10
 
-{- This is a naive scan through the git work tree. -}
-scan :: Remote -> Annex [(AssociatedFile, Transfer)]
-scan r = do
-	files <- inRepo $ LsFiles.inRepo []
-	catMaybes <$> forM files (whenAnnexed go)
+{- This is a naive scan through the git work tree.
+ - 
+ - The scan is blocked when the transfer queue gets too large. -}
+scan :: ThreadState -> TransferQueue -> Remote -> IO ()
+scan st transferqueue r = do
+	g <- runThreadState st $ fromRepo id
+	files <- LsFiles.inRepo [] g
+	go files
 	where
-		u = Remote.uuid r
-
-		go file (key, _) =
-			ifM (inAnnex key)
-				( check Upload False =<< remotehas key
-				, check Download True =<< remotehas key
-				)
+		go [] = return ()
+		go (f:fs) = do
+			v <- runThreadState st $ whenAnnexed check f
+			case v of
+				Nothing -> noop
+				Just t -> do
+					debug thisThread ["queuing", show t]
+					enqueue f t
+			go fs
 			where
-				check direction x y
+				check _ (key, _) = ifM (inAnnex key)
+					( helper key Upload False =<< remotehas key
+					, helper key Download True =<< remotehas key
+					)
+				helper key direction x y
 					| x == y = return $
-						Just (Just file, Transfer direction u key)
+						Just $ Transfer direction u key
 					| otherwise = return Nothing
+				
+		u = Remote.uuid r
+		enqueue f t = queueTransferAt smallsize Later transferqueue (Just f) t r
+		smallsize = 10
 
 		{- Look directly in remote for the key when it's cheap;
 		 - otherwise rely on the location log. -}
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index a01c85405a..9f0ea5cbe1 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -5,7 +5,15 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Assistant.TransferQueue where
+module Assistant.TransferQueue (
+	TransferQueue,
+	Schedule(..),
+	newTransferQueue,
+	queueTransfers,
+	queueTransfer,
+	queueTransferAt,
+	getNextTransfer
+) where
 
 import Common.Annex
 import Assistant.DaemonStatus

From bd2b388fd8c668ed6fd031d0ed8a7edf3c7b67ee Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 15:07:41 -0400
Subject: [PATCH 4211/8313] update

---
 doc/design/assistant/syncing.mdwn | 114 ++++++++++++++++--------------
 1 file changed, 61 insertions(+), 53 deletions(-)

diff --git a/doc/design/assistant/syncing.mdwn b/doc/design/assistant/syncing.mdwn
index f04f20218b..3aeb76afc1 100644
--- a/doc/design/assistant/syncing.mdwn
+++ b/doc/design/assistant/syncing.mdwn
@@ -5,14 +5,66 @@ all the other git clones, at both the git level and the key/value level.
 
 * At startup, and possibly periodically, or when the network connection
   changes, or some heuristic suggests that a remote was disconnected from
-  us for a while, queue remotes for processing by the TransferScanner,
-  to queue Transfers of files it or we're missing.
-* After git sync, identify content that we don't have that is now available
+  us for a while, queue remotes for processing by the TransferScanner.
+* Ensure that when a remote receives content, and updates its location log,
+  it syncs that update back out. Prerequisite for:
+* After git sync, identify new content that we don't have that is now available
   on remotes, and transfer. (Needed when we have a uni-directional connection
-  to a remote, so it won't be uploading content to us.) 
-  But first, need to ensure that when a remote
-  receives content, and updates its location log, it syncs that update
-  out.
+  to a remote, so it won't be uploading content to us.) Note: Does not
+  need to use the TransferScanner, if we get and check a list of the changed
+  files.
+
+## longer-term TODO
+
+* Test MountWatcher on LXDE.
+* git-annex needs a simple speed control knob, which can be plumbed
+  through to, at least, rsync. A good job for an hour in an
+  airport somewhere.
+* Find a way to probe available outgoing bandwidth, to throttle so
+  we don't bufferbloat the network to death.
+* Investigate the XMPP approach like dvcs-autosync does, or other ways of
+   signaling a change out of band.
+* Add a hook, so when there's a change to sync, a program can be run
+   and do its own signaling.
+* --debug will show often unnecessary work being done. Optimise.
+* This assumes the network is connected. It's often not, so the
+  [[cloud]] needs to be used to bridge between LANs.
+* Configurablity, including only enabling git syncing but not data transfer;
+  only uploading new files but not downloading, and only downloading
+  files in some directories and not others. See for use cases:
+  [[forum/Wishlist:_options_for_syncing_meta-data_and_data]]
+* speed up git syncing by using the cached ssh connection for it too
+  (will need to use `GIT_SSH`, which needs to point to a command to run,
+  not a shell command line)
+* Map the network of git repos, and use that map to calculate
+  optimal transfers to keep the data in sync. Currently a naive flood fill
+  is done instead.
+* Find a more efficient way for the TransferScanner to find the transfers
+  that need to be done to sync with a remote. Currently it walks the git
+  working copy and checks each file.
+
+## data syncing
+
+There are two parts to data syncing. First, map the network and second,
+decide what to sync when.
+
+Mapping the network can reuse code in `git annex map`. Once the map is
+built, we want to find paths through the network that reach all nodes
+eventually, with the least cost. This is a minimum spanning tree problem,
+except with a directed graph, so really a Arborescence problem.
+
+With the map, we can determine which nodes to push new content to. Then we
+need to control those data transfers, sending to the cheapest nodes first,
+and with appropriate rate limiting and control facilities.
+
+This probably will need lots of refinements to get working well.
+
+### first pass: flood syncing
+
+Before mapping the network, the best we can do is flood all files out to every
+reachable remote. This is worth doing first, since it's the simplest way to
+get the basic functionality of the assistant to work. And we'll need this
+anyway.
 
 ## TransferScanner
 
@@ -21,6 +73,8 @@ to a remote, or Downloaded from it.
 
 How to find the keys to transfer? I'd like to avoid potentially
 expensive traversals of the whole git working copy if I can.
+(Currently, the TransferScanner does do the naive and possibly expensive
+scan of the git working copy.)
 
 One way would be to do a git diff between the (unmerged) git-annex branches
 of the git repo, and its remote. Parse that for lines that add a key to
@@ -53,52 +107,6 @@ one. Probably worth handling the case where a remote is connected
 while in the middle of such a scan, so part of the scan needs to be
 redone to check it.
 
-## longer-term TODO
-
-* Test MountWatcher on LXDE.
-* git-annex needs a simple speed control knob, which can be plumbed
-  through to, at least, rsync. A good job for an hour in an
-  airport somewhere.
-* Find a way to probe available outgoing bandwidth, to throttle so
-  we don't bufferbloat the network to death.
-* Investigate the XMPP approach like dvcs-autosync does, or other ways of
-   signaling a change out of band.
-* Add a hook, so when there's a change to sync, a program can be run
-   and do its own signaling.
-* --debug will show often unnecessary work being done. Optimise.
-* This assumes the network is connected. It's often not, so the
-  [[cloud]] needs to be used to bridge between LANs.
-* Configurablity, including only enabling git syncing but not data transfer;
-  only uploading new files but not downloading, and only downloading
-  files in some directories and not others. See for use cases:
-  [[forum/Wishlist:_options_for_syncing_meta-data_and_data]]
-* speed up git syncing by using the cached ssh connection for it too
-  (will need to use `GIT_SSH`, which needs to point to a command to run,
-  not a shell command line)
-
-## data syncing
-
-There are two parts to data syncing. First, map the network and second,
-decide what to sync when.
-
-Mapping the network can reuse code in `git annex map`. Once the map is
-built, we want to find paths through the network that reach all nodes
-eventually, with the least cost. This is a minimum spanning tree problem,
-except with a directed graph, so really a Arborescence problem.
-
-With the map, we can determine which nodes to push new content to. Then we
-need to control those data transfers, sending to the cheapest nodes first,
-and with appropriate rate limiting and control facilities.
-
-This probably will need lots of refinements to get working well.
-
-### first pass: flood syncing
-
-Before mapping the network, the best we can do is flood all files out to every
-reachable remote. This is worth doing first, since it's the simplest way to
-get the basic functionality of the assistant to work. And we'll need this
-anyway.
-
 ## done
 
 1. Can use `git annex sync`, which already handles bidirectional syncing.

From 2e085c6383f096a58d1e9b52ae457f9491850c7f Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 15:31:26 -0400
Subject: [PATCH 4212/8313] blog for the day

---
 .../blog/day_43__simple_scanner.mdwn          | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_43__simple_scanner.mdwn

diff --git a/doc/design/assistant/blog/day_43__simple_scanner.mdwn b/doc/design/assistant/blog/day_43__simple_scanner.mdwn
new file mode 100644
index 0000000000..11ee3cca49
--- /dev/null
+++ b/doc/design/assistant/blog/day_43__simple_scanner.mdwn
@@ -0,0 +1,37 @@
+Milestone: I can run `git annex assistant`, plug in a USB drive, and it
+automatically transfers files to get the USB drive and current repo back in
+sync.
+
+I decided to implement the naive scan, to find files needing to be
+transferred. So it walks through `git ls-files` and checks each file
+in turn. I've deferred less expensive, more sophisticated approaches to later.
+
+I did some work on the TransferQueue, which now keeps track of the length
+of the queue, and can block attempts to add Transfers to it if it gets too
+long. This was a nice use of STM, which let me implement that without using
+any locking.
+
+[[!format haskell """
+atomically $ do
+        sz <- readTVar (queuesize q)
+        if sz <= wantsz
+                then enqueue schedule q t (stubInfo f remote)
+                else retry -- blocks until queuesize changes
+"""]]
+
+Anyway, the point was that, as the scan finds Transfers to do,
+it doesn't build up a really long TransferQueue, but instead is blocked
+from running further until some of the files get transferred. The resulting
+interleaving of the scan thread with transfer threads means that transfers
+start fairly quickly upon a USB drive being plugged in, and kind of hides
+the innefficiencies of the scanner, which will most of the time be
+swamped out by the IO bound large data transfers.
+
+---
+
+At this point, the assistant should do a good job of keeping repositories
+in sync, as long as they're all interconnected, or on removable media
+like USB drives. There's lots more work to be done to handle use cases
+where repositories are not well-connected, but since the assistant's
+[[syncing]] now covers at least a couple of use cases, I'm ready to move
+on to the next phase. [[Webapp]], here we come!

From 3a02c7b635fc1017c05874b8a6f54a91a587651d Mon Sep 17 00:00:00 2001
From: jtang 
Date: Wed, 25 Jul 2012 20:12:16 +0000
Subject: [PATCH 4213/8313] fix example to match current command in git-annex
 semitrust

---
 doc/tips/what_to_do_when_you_lose_a_repository.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
index 3be13b8abd..363eeea4e0 100644
--- a/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
+++ b/doc/tips/what_to_do_when_you_lose_a_repository.mdwn
@@ -16,4 +16,4 @@ are present.
 If you later found the drive, you could let git-annex know it's found
 like so:
 
-	git annex semitrusted usbdrive
+	git annex semitrust usbdrive

From 32d3cffc4cf075d7c20fee8addc556f402e94cd2 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 21:26:13 -0400
Subject: [PATCH 4214/8313] run yesod, and launch webapp on startup

---
 Assistant.hs                     |   8 +++
 Assistant/Threads/WebApp.hs      |  43 +++++++++++++
 Makefile                         |  20 +++---
 Utility/WebApp.hs                | 104 +++++++++++++++++++++++++++++++
 debian/control                   |   1 +
 doc/design/assistant/webapp.mdwn |   2 +-
 doc/git-annex.mdwn               |   6 ++
 doc/install.mdwn                 |   2 +
 git-annex.cabal                  |  13 ++++
 9 files changed, 189 insertions(+), 10 deletions(-)
 create mode 100644 Assistant/Threads/WebApp.hs
 create mode 100644 Utility/WebApp.hs

diff --git a/Assistant.hs b/Assistant.hs
index 0049d31777..de996aa741 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -88,6 +88,8 @@
  - 	are indicated by writing to this TMVar.
  -}
 
+{-# LANGUAGE CPP #-}
+
 module Assistant where
 
 import Assistant.Common
@@ -108,6 +110,9 @@ import Assistant.Threads.Transferrer
 import Assistant.Threads.SanityChecker
 import Assistant.Threads.MountWatcher
 import Assistant.Threads.TransferScanner
+#ifdef WITH_WEBAPP
+import Assistant.Threads.WebApp
+#endif
 import qualified Utility.Daemon
 import Utility.LogFile
 import Utility.ThreadScheduler
@@ -146,6 +151,9 @@ startDaemon assistant foreground
 				, sanityCheckerThread st dstatus transferqueue changechan
 				, mountWatcherThread st dstatus scanremotes
 				, transferScannerThread st scanremotes transferqueue
+#ifdef WITH_WEBAPP
+				, webAppThread dstatus
+#endif
 				, watchThread st dstatus transferqueue changechan
 				]
 			debug "assistant"
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
new file mode 100644
index 0000000000..1d9d3cc2fc
--- /dev/null
+++ b/Assistant/Threads/WebApp.hs
@@ -0,0 +1,43 @@
+{- git-annex assistant webapp
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings #-}
+
+module Assistant.Threads.WebApp where
+
+import Assistant.Common
+import Assistant.DaemonStatus
+import Utility.WebApp
+
+import Yesod
+
+data WebApp = WebApp DaemonStatusHandle
+
+mkYesod "WebApp" [parseRoutes|
+/ HomeR GET
+/config ConfigR GET
+|]
+
+instance Yesod WebApp
+
+getHomeR :: Handler RepHtml
+getHomeR = defaultLayout [whamlet|Hello, World
config|]
+
+getConfigR :: Handler RepHtml
+getConfigR = defaultLayout [whamlet|main|]
+
+webAppThread :: DaemonStatusHandle -> IO ()
+webAppThread dstatus = do
+	app <- toWaiApp (WebApp dstatus)
+	app' <- ifM debugEnabled
+		( return $ httpDebugLogger app
+		, return app
+		)
+	runWebApp app' browser
+	where
+		browser p = void $
+			runBrowser $ "http://" ++ localhost ++ ":" ++ show p
diff --git a/Makefile b/Makefile
index 8b9b35bdd6..9f312dc49c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,19 +1,23 @@
+CFLAGS=-Wall
+IGNORE=-ignore-package monads-fd -ignore-package monads-tf
+BASEFLAGS=-threaded -Wall $(IGNORE) -outputdir tmp -IUtility
+FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP
+
 bins=git-annex
 mans=git-annex.1 git-annex-shell.1
 sources=Build/SysConfig.hs Utility/Touch.hs Utility/Mounts.hs
 all=$(bins) $(mans) docs
 
-CFLAGS=-Wall
-
 OS:=$(shell uname | sed 's/[-_].*//')
 ifeq ($(OS),Linux)
-BASEFLAGS_OPTS=-DWITH_INOTIFY -DWITH_DBUS
+OPTFLAGS=-DWITH_INOTIFY -DWITH_DBUS
 clibs=Utility/libdiskfree.o Utility/libmounts.o
 else
 # BSD system
-BASEFLAGS_OPTS=-DWITH_KQUEUE
+OPTFLAGS=-DWITH_KQUEUE
 clibs=Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
 ifeq ($(OS),Darwin)
+OPTFLAGS=-DWITH_KQUEUE -DOSX
 # Ensure OSX compiler builds for 32 bit when using 32 bit ghc
 GHCARCH:=$(shell ghc -e 'print System.Info.arch')
 ifeq ($(GHCARCH),i386)
@@ -23,12 +27,10 @@ endif
 endif
 
 PREFIX=/usr
-IGNORE=-ignore-package monads-fd -ignore-package monads-tf
-BASEFLAGS=-threaded -Wall $(IGNORE) -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 $(BASEFLAGS_OPTS)
-GHCFLAGS=-O2 $(BASEFLAGS)
+GHCFLAGS=-O2 $(BASEFLAGS) $(FEATURES)
 
 ifdef PROFILE
-GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS)
+GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS) $(FEATURES) $(OPTFLAGS)
 endif
 
 GHCMAKE=ghc $(GHCFLAGS) --make
@@ -43,7 +45,7 @@ all: $(all)
 sources: $(sources)
 
 # Disables optimisation. Not for production use.
-fast: GHCFLAGS=$(BASEFLAGS)
+fast: GHCFLAGS=$(BASEFLAGS) $(FEATURES) $(OPTFLAGS)
 fast: $(bins)
 
 Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
new file mode 100644
index 0000000000..614a57cea5
--- /dev/null
+++ b/Utility/WebApp.hs
@@ -0,0 +1,104 @@
+{- WAI webapp
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE OverloadedStrings, CPP #-}
+
+module Utility.WebApp where
+
+import Common
+
+import Network.Wai
+import Network.Wai.Handler.Warp
+import Network.Wai.Logger
+import Control.Monad.IO.Class
+import Network.HTTP.Types
+import System.Log.Logger
+import Data.ByteString.Lazy.UTF8
+import Data.ByteString.Lazy
+import Data.CaseInsensitive as CI
+import Network.Socket
+import Control.Exception
+
+localhost :: String
+localhost = "localhost"
+
+{- Runs a web browser on a given url.
+ -
+ - Note: The url *will* be visible to an attacker. -}
+runBrowser :: String -> IO Bool
+runBrowser url = boolSystem cmd [Param url]
+	where
+#if MAC
+		cmd = "open"
+#else
+		cmd = "xdg-open"
+#endif
+
+{- Binds to a socket on localhost, and runs a webapp on it.
+ -
+ - An IO action can also be run, to do something with the port number,
+ - such as start a web browser to view the webapp.
+  -}
+runWebApp :: Application -> (PortNumber -> IO ()) -> IO ()
+runWebApp app observer = do
+	sock <- localSocket
+	observer =<< socketPort sock
+	runSettingsSocket defaultSettings sock app
+
+{- Binds to a local socket, selecting any free port.
+ -
+ - As a (very weak) form of security, only connections from 
+ - localhost are accepted. -}
+localSocket :: IO Socket
+localSocket = do
+	addrs <- getAddrInfo (Just hints) (Just localhost) Nothing
+	go $ Prelude.head addrs
+	where
+		hints = defaultHints
+			{ addrFlags = [AI_ADDRCONFIG, AI_NUMERICSERV]
+			, addrSocketType = Stream
+			}
+		go addr = bracketOnError (open addr) close (use addr)
+		open addr = socket (addrFamily addr) (addrSocketType addr) (addrProtocol addr)
+		close = sClose
+		use addr sock = do
+			setSocketOption sock ReuseAddr 1
+			bindSocket sock (addrAddress addr)
+			listen sock maxListenQueue
+			return sock
+
+{- Checks if debugging is actually enabled. -}
+debugEnabled :: IO Bool
+debugEnabled = do
+	l <- getRootLogger
+	return $ getLevel l <= Just DEBUG
+
+{- WAI middleware that logs using System.Log.Logger at debug level.
+ -
+ - Recommend only inserting this middleware when debugging is actually
+ - enabled, as it's not optimised at all.
+ -}
+httpDebugLogger :: Middleware
+httpDebugLogger waiApp req = do
+	logRequest req
+	waiApp req
+
+logRequest :: MonadIO m => Request -> m ()
+logRequest req = do
+	liftIO $ debugM "WebApp" $ unwords
+		[ showSockAddr $ remoteHost req
+		, frombs $ requestMethod req
+		, frombs $ rawPathInfo req
+		--, show $ httpVersion req
+		--, frombs $ lookupRequestField "referer" req
+		, frombs $ lookupRequestField "user-agent" req
+		]
+	where
+		frombs v = toString $ fromChunks [v]
+
+lookupRequestField :: CI Ascii -> Request -> Ascii
+lookupRequestField k req = fromMaybe "" . lookup k $ requestHeaders req
diff --git a/debian/control b/debian/control
index 35cbfde054..c7531cd3f1 100644
--- a/debian/control
+++ b/debian/control
@@ -23,6 +23,7 @@ Build-Depends:
 	libghc-hinotify-dev [linux-any],
 	libghc-stm-dev (>= 2.3),
 	libghc-dbus-dev,
+	libghc-yesod-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 598c1ff3a4..cec766c579 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -2,7 +2,7 @@ The webapp is a web server that displays a shiny interface.
 
 ## security
 
-* Listen only to localhost.
+* Listen only to localhost. **done**
 * Instruct the user's web browser to open an url that contains a secret
   token. This guards against other users on the same system.
 * I would like to avoid passwords or other authentication methods,
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 85a5a18f00..282b1fda50 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -188,6 +188,12 @@ subdirectories).
 * assistant
 
   Like watch, but also automatically syncs changes to other remotes.
+  Typically started at boot, or when you log in.
+
+* webapp
+
+  Opens a web browser, viewing the git-annex assistant's web app.
+  (If the assistant is not already running, it will be automatically started.)
 
 # REPOSITORY SETUP COMMANDS
 
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 54d6ecb6b5..058c3cf6e1 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -48,6 +48,8 @@ To build and use git-annex, you will need:
     (optional; Linux only)
   * [dbus](http://hackage.haskell.org/package/dbus)
     (optional)
+  * [yesod](http://hackage.haskell.org/package/yesod)
+    (optional; for webapp)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 2e312d4c38..11412d19ab 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -37,6 +37,9 @@ Flag Dbus
 Flag Assistant
   Description: Enable git-annex assistant and watch command
 
+Flag Webapp
+  Description: Enable git-annex webapp
+
 Executable git-annex
   Main-Is: git-annex.hs
   Build-Depends: MissingH, hslogger, directory, filepath,
@@ -61,11 +64,21 @@ Executable git-annex
   if os(linux) && flag(Inotify)
     Build-Depends: hinotify
     CPP-Options: -DWITH_INOTIFY
+  else
+    if (! os(windows))
+      CPP-Options: -DWITH_KQUEUE
 
   if flag(Dbus)
     Build-Depends: dbus
     CPP-Options: -DWITH_DBUS
 
+  if flag(Webapp)
+    Build-Depends: yesod
+    CPP-Options: -DWITH_WEBAPP
+
+  if (os(darwin))
+    CPP-Options: -DOSX
+
 Test-Suite test
   Type: exitcode-stdio-1.0
   Main-Is: test.hs

From e6ce54de82c19999fb5adcd5fd1ea4001fd2059e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 22:04:10 -0400
Subject: [PATCH 4215/8313] build fixes

---
 Assistant/Threads/Merger.hs |  2 +-
 debian/control              |  6 ++++++
 doc/install.mdwn            | 13 +++++++++----
 git-annex.cabal             |  5 ++++-
 4 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/Assistant/Threads/Merger.hs b/Assistant/Threads/Merger.hs
index 10ea34692b..c79566349d 100644
--- a/Assistant/Threads/Merger.hs
+++ b/Assistant/Threads/Merger.hs
@@ -25,7 +25,7 @@ import qualified Remote
 thisThread :: ThreadName
 thisThread = "Merger"
 
-{- This thread watches for changes to .git/refs/heads/synced/*,
+{- This thread watches for changes to .git/refs/heads/synced/,
  - which indicate incoming pushes. It merges those pushes into the
  - currently checked out branch. -}
 mergeThread :: ThreadState -> IO ()
diff --git a/debian/control b/debian/control
index c7531cd3f1..65c666cde4 100644
--- a/debian/control
+++ b/debian/control
@@ -24,6 +24,12 @@ Build-Depends:
 	libghc-stm-dev (>= 2.3),
 	libghc-dbus-dev,
 	libghc-yesod-dev,
+	libghc-case-insensitive-dev,
+	libghc-http-types-dev,
+	libghc-transformers-dev,
+	libghc-wai-dev,
+	libghc-wai-logger-dev,
+	libghc-warp-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 058c3cf6e1..619d5fa11d 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -42,14 +42,19 @@ To build and use git-annex, you will need:
   * [bloomfilter](http://hackage.haskell.org/package/bloomfilter)
   * [edit-distance](http://hackage.haskell.org/package/edit-distance)
   * [hS3](http://hackage.haskell.org/package/hS3) (optional)
+* Optional haskell stuff, used by the assistant and webapp (edit Makefile to disable)
   * [stm](http://hackage.haskell.org/package/stm)
-    (optional; version 2.3 or newer)
+    (version 2.3 or newer)
   * [hinotify](http://hackage.haskell.org/package/hinotify)
-    (optional; Linux only)
+    (Linux only)
   * [dbus](http://hackage.haskell.org/package/dbus)
-    (optional)
   * [yesod](http://hackage.haskell.org/package/yesod)
-    (optional; for webapp)
+  * [case-insensitive](http://hackage.haskell.org/package/case-insensitive)
+  * [http-types](http://hackage.haskell.org/package/http-types)
+  * [transformers](http://hackage.haskell.org/package/transformers)
+  * [wai](http://hackage.haskell.org/package/wai)
+  * [wai-logger](http://hackage.haskell.org/package/wai-logger)
+  * [warp](http://hackage.haskell.org/package/warp)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 11412d19ab..086df31d29 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -46,9 +46,11 @@ Executable git-annex
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,
+   case-insensitive, http-types, transformers, wai, wai-logger, warp
   -- Need to list these because they're generated from .hsc files.
   Other-Modules: Utility.Touch Utility.Mounts
+  Include-Dirs: Utility
   C-Sources: Utility/libdiskfree.c Utility/libmounts.c
   Extensions: CPP
   GHC-Options: -threaded
@@ -88,6 +90,7 @@ Test-Suite test
    base == 4.5.*, monad-control, transformers-base, lifted-base,
    IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process
   Other-Modules: Utility.Touch
+  Include-Dirs: Utility
   C-Sources: Utility/libdiskfree.c
   Extensions: CPP
   GHC-Options: -threaded

From 1ffef3ad75e51b7f66c4ffdd0935a0495042e5ae Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 23:13:01 -0400
Subject: [PATCH 4216/8313] git annex webapp now opens a browser to the webapp

Also, starts the assistant if it wasn't already running.
---
 Assistant.hs                |  2 +-
 Assistant/Threads/WebApp.hs | 31 ++++++++++++++++----
 Command/WebApp.hs           | 58 +++++++++++++++++++++++++++++++++++++
 GitAnnex.hs                 |  6 ++++
 Locations.hs                |  5 ++++
 Utility/Daemon.hs           | 43 ++++++++++++++++-----------
 6 files changed, 122 insertions(+), 23 deletions(-)
 create mode 100644 Command/WebApp.hs

diff --git a/Assistant.hs b/Assistant.hs
index de996aa741..c867529fdf 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -152,7 +152,7 @@ startDaemon assistant foreground
 				, mountWatcherThread st dstatus scanremotes
 				, transferScannerThread st scanremotes transferqueue
 #ifdef WITH_WEBAPP
-				, webAppThread dstatus
+				, webAppThread st dstatus
 #endif
 				, watchThread st dstatus transferqueue changechan
 				]
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 1d9d3cc2fc..f3f13c5a09 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -10,10 +10,12 @@
 module Assistant.Threads.WebApp where
 
 import Assistant.Common
+import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Utility.WebApp
 
 import Yesod
+import Network.Socket (PortNumber)
 
 data WebApp = WebApp DaemonStatusHandle
 
@@ -30,14 +32,33 @@ getHomeR = defaultLayout [whamlet|Hello, World
config|]
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout [whamlet|main|]
 
-webAppThread :: DaemonStatusHandle -> IO ()
-webAppThread dstatus = do
+webAppThread :: ThreadState -> DaemonStatusHandle -> IO ()
+webAppThread st dstatus = do
 	app <- toWaiApp (WebApp dstatus)
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
 		)
-	runWebApp app' browser
+	runWebApp app' $ \p -> runThreadState st $ writeHtmlShim p
+
+{- Creates a html shim file that's used to redirect into the webapp. -}
+writeHtmlShim :: PortNumber -> Annex ()
+writeHtmlShim port = do
+	htmlshim <- fromRepo gitAnnexHtmlShim
+	liftIO $ writeFile htmlshim $ genHtmlShim port
+
+{- TODO: generate this static file using Yesod. -}
+genHtmlShim :: PortNumber -> String
+genHtmlShim port = unlines
+	[ ""
+	, ""
+	, ""
+	, ""
+	, ""
+	, "
"
+	, "Starting webapp..."
+	, "
"
+	, ""
+	]
 	where
-		browser p = void $
-			runBrowser $ "http://" ++ localhost ++ ":" ++ show p
+		url = "http://localhost:" ++ show port ++ "/"
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
new file mode 100644
index 0000000000..616a6512a8
--- /dev/null
+++ b/Command/WebApp.hs
@@ -0,0 +1,58 @@
+{- git-annex webapp launcher
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Command.WebApp where
+
+import Common.Annex
+import Command
+import Assistant
+import Utility.WebApp
+import Utility.Daemon
+import qualified Annex
+
+import Control.Concurrent
+import System.Posix.Process
+
+def :: [Command]
+def = [command "webapp" paramNothing seek "launch webapp"]
+
+seek :: [CommandSeek]
+seek = [withNothing start]
+
+start :: CommandStart
+start = notBareRepo $ do
+	r <- checkpid
+	when (r == Nothing) $
+		startassistant
+	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
+	let url = "file://" ++ f
+	ifM (liftIO $ runBrowser url)
+		( stop
+		, error $ "failed to start web browser on url " ++ url
+		)
+	where
+		checkpid = do
+			pidfile <- fromRepo gitAnnexPidFile
+			liftIO $ checkDaemon pidfile
+		startassistant = do
+			{- Fork a separate process to run the assistant,
+			 - with a copy of the Annex state. -}
+			state <- Annex.getState id
+			liftIO $ void $ forkProcess $
+				Annex.eval state $ startDaemon True False
+			waitdaemon (100 :: Int)
+		waitdaemon 0 = error "failed to start git-annex assistant"
+		waitdaemon n = do
+			r <- checkpid
+			case r of
+				Just _ -> return ()
+				Nothing -> do
+					liftIO $
+						threadDelay 100000 -- 0.1 seconds
+					
+
+waitdaemon (n - 1)
diff --git a/GitAnnex.hs b/GitAnnex.hs
index 7b1fa59868..ce7a41a40f 100644
--- a/GitAnnex.hs
+++ b/GitAnnex.hs
@@ -63,6 +63,9 @@ import qualified Command.Version
 #ifdef WITH_ASSISTANT
 import qualified Command.Watch
 import qualified Command.Assistant
+#ifdef WITH_WEBAPP
+import qualified Command.WebApp
+#endif
 #endif
 
 cmds :: [Command]
@@ -108,6 +111,9 @@ cmds = concat
 #ifdef WITH_ASSISTANT
 	, Command.Watch.def
 	, Command.Assistant.def
+#ifdef WITH_WEBAPP
+	, Command.WebApp.def
+#endif
 #endif
 	]
 
diff --git a/Locations.hs b/Locations.hs
index 082a72a506..cbd1e11ae0 100644
--- a/Locations.hs
+++ b/Locations.hs
@@ -27,6 +27,7 @@ module Locations (
 	gitAnnexPidFile,
 	gitAnnexDaemonStatusFile,
 	gitAnnexLogFile,
+	gitAnnexHtmlShim,
 	gitAnnexSshDir,
 	gitAnnexRemotesDir,
 	isLinkToAnnex,
@@ -166,6 +167,10 @@ gitAnnexDaemonStatusFile r = gitAnnexDir r  "daemon.status"
 gitAnnexLogFile :: Git.Repo -> FilePath
 gitAnnexLogFile r = gitAnnexDir r  "daemon.log"
 
+{- Html shim file used to launch the webapp. -}
+gitAnnexHtmlShim :: Git.Repo -> FilePath
+gitAnnexHtmlShim r = gitAnnexDir r  "webapp.html"
+
 {- .git/annex/ssh/ is used for ssh connection caching -}
 gitAnnexSshDir :: Git.Repo -> FilePath
 gitAnnexSshDir r = addTrailingPathSeparator $ gitAnnexDir r  "ssh"
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index f36a761d00..8aa70d155c 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -62,24 +62,33 @@ lockPidFile onfailure file = do
 	where
 		newfile = file ++ ".new"
 
-{- Stops the daemon.
+{- Checks if the daemon is running, by checking that the pid file
+ - is locked by the same process that is listed in the pid file.
  -
- - The pid file is used to get the daemon's pid.
- -
- - To guard against a stale pid, check the lock of the pid file,
- - and compare the process that has it locked with the file content.
- -}
-stopDaemon :: FilePath -> IO ()
-stopDaemon pidfile = do
-	fd <- openFd pidfile ReadOnly (Just stdFileMode) defaultFileFlags
-	locked <- getLock fd (ReadLock, AbsoluteSeek, 0, 0)
-	p <- readish <$> readFile pidfile
-	case (locked, p) of
-		(Nothing, _) -> noop
-		(_, Nothing) -> noop
-		(Just (pid, _), Just pid')
-			| pid == pid' -> signalProcess sigTERM pid
-			| otherwise -> error $
+ - If it's running, returns its pid. -}
+checkDaemon :: FilePath -> IO (Maybe ProcessID)
+checkDaemon pidfile = do
+	v <- catchMaybeIO $
+		openFd pidfile ReadOnly (Just stdFileMode) defaultFileFlags
+	case v of
+		Just fd -> do
+			locked <- getLock fd (ReadLock, AbsoluteSeek, 0, 0)
+			p <- readish <$> readFile pidfile
+			return $ check locked p
+		Nothing -> return Nothing
+	where
+		check Nothing _ = Nothing
+		check _ Nothing = Nothing
+		check (Just (pid, _)) (Just pid')
+			| pid == pid' = Just pid
+			| otherwise = error $
 				"stale pid in " ++ pidfile ++ 
 				" (got " ++ show pid' ++ 
 				"; expected" ++ show pid ++ " )"
+
+{- Stops the daemon, safely. -}
+stopDaemon :: FilePath -> IO ()
+stopDaemon pidfile = go =<< checkDaemon pidfile
+	where
+		go Nothing = noop
+		go (Just pid) = signalProcess sigTERM pid

From 81b40cf882e50be4d996fd40d045039de94784ef Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 23:50:14 -0400
Subject: [PATCH 4217/8313] fix editor damage

---
 Command/WebApp.hs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 616a6512a8..0e01a07cd3 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -51,8 +51,6 @@ start = notBareRepo $ do
 			case r of
 				Just _ -> return ()
 				Nothing -> do
-					liftIO $
-						threadDelay 100000 -- 0.1 seconds
-					
-
-waitdaemon (n - 1)
+					-- wait 0.1 seconds before retry
+					liftIO $ threadDelay 100000
+					waitdaemon (n - 1)

From 805d50c69d40be97baa28735371778df63b5fed6 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 25 Jul 2012 23:50:29 -0400
Subject: [PATCH 4218/8313] use hamlet at least for the static html

---
 Assistant/Threads/WebApp.hs | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index f3f13c5a09..d475865dca 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -15,7 +15,10 @@ import Assistant.DaemonStatus
 import Utility.WebApp
 
 import Yesod
+import Text.Hamlet
 import Network.Socket (PortNumber)
+import Text.Blaze.Renderer.Utf8
+import Data.ByteString.Lazy as L
 
 data WebApp = WebApp DaemonStatusHandle
 
@@ -45,20 +48,18 @@ webAppThread st dstatus = do
 writeHtmlShim :: PortNumber -> Annex ()
 writeHtmlShim port = do
 	htmlshim <- fromRepo gitAnnexHtmlShim
-	liftIO $ writeFile htmlshim $ genHtmlShim port
+	liftIO $ L.writeFile htmlshim $ genHtmlShim port
 
 {- TODO: generate this static file using Yesod. -}
-genHtmlShim :: PortNumber -> String
-genHtmlShim port = unlines
-	[ ""
-	, ""
-	, ""
-	, ""
-	, ""
-	, "
"
-	, "Starting webapp..."
-	, "
"
-	, ""
-	]
+genHtmlShim :: PortNumber -> L.ByteString
+genHtmlShim port = renderHtml [shamlet|
+!!!
+
+  
+    
+  
+    

+      Starting webapp...
+|]
 	where
 		url = "http://localhost:" ++ show port ++ "/"

From 6a8540c1a2ae61d81e06eae2865ac00c6c759ed5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 26 Jul 2012 00:39:25 -0400
Subject: [PATCH 4219/8313] tweak

---
 Assistant/Threads/WebApp.hs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index d475865dca..d9b648831a 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -37,12 +37,14 @@ getConfigR = defaultLayout [whamlet|main|]
 
 webAppThread :: ThreadState -> DaemonStatusHandle -> IO ()
 webAppThread st dstatus = do
-	app <- toWaiApp (WebApp dstatus)
+	app <- toWaiApp webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
 		)
-	runWebApp app' $ \p -> runThreadState st $ writeHtmlShim p
+	runWebApp app' $ \port -> runThreadState st $ writeHtmlShim port
+	where
+		webapp = WebApp dstatus
 
 {- Creates a html shim file that's used to redirect into the webapp. -}
 writeHtmlShim :: PortNumber -> Annex ()
@@ -53,13 +55,13 @@ writeHtmlShim port = do
 {- TODO: generate this static file using Yesod. -}
 genHtmlShim :: PortNumber -> L.ByteString
 genHtmlShim port = renderHtml [shamlet|
-!!!
+$doctype 5
 
   
     
   
     

-      Starting webapp...
+      Starting webapp...
 |]
 	where
 		url = "http://localhost:" ++ show port ++ "/"

From 3ac2cf09e56cb1918312a31e0884d56829a14c32 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 26 Jul 2012 02:45:01 -0400
Subject: [PATCH 4220/8313] added a custom defaultLayout, static site, and
 favicon

Broke hamlet out into standalone files.

I don't like the favicon display; it should be served from /favicon.ico,
but I could only get the static site to serve /static/favicon.ico, so
I had to use a  to pull it in. I looked at
Yesod.Default.Handlers.getFaviconR, but it doesn't seem to embed
the favicon into the binary?
---
 Assistant/Threads/WebApp.hs     |  53 +++++++++++++++++++++++---------
 Utility/Yesod.hs                |  18 +++++++++++
 static/favicon.ico              | Bin 0 -> 405 bytes
 templates/default-layout.hamlet |  11 +++++++
 templates/htmlshim.hamlet       |   7 +++++
 5 files changed, 74 insertions(+), 15 deletions(-)
 create mode 100644 Utility/Yesod.hs
 create mode 100644 static/favicon.ico
 create mode 100644 templates/default-layout.hamlet
 create mode 100644 templates/htmlshim.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index d9b648831a..4e6fea6b11 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -13,38 +13,69 @@ import Assistant.Common
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Utility.WebApp
+import Utility.Yesod
+import Git
 
 import Yesod
+import Yesod.Static
 import Text.Hamlet
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.Utf8
 import Data.ByteString.Lazy as L
 
-data WebApp = WebApp DaemonStatusHandle
+data WebApp = WebApp
+	{ daemonStatus :: DaemonStatusHandle
+	, baseTitle :: String
+	, getStatic :: Static
+	}
+
+staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
+/static StaticR Static getStatic
 / HomeR GET
 /config ConfigR GET
 |]
 
-instance Yesod WebApp
+instance Yesod WebApp where
+	defaultLayout contents = do
+		page <- widgetToPageContent contents
+		mmsg <- getMessage
+		webapp <- getYesod
+		hamletToRepHtml $(hamletFile $ hamletTemplate "default-layout")
 
 getHomeR :: Handler RepHtml
-getHomeR = defaultLayout [whamlet|Hello, World
config|]
+getHomeR = defaultLayout $ do
+	[whamlet|Hello, World
config|]
 
 getConfigR :: Handler RepHtml
-getConfigR = defaultLayout [whamlet|main|]
+getConfigR = defaultLayout $ do
+	setTitle "configuration"
+	[whamlet|main|]
 
 webAppThread :: ThreadState -> DaemonStatusHandle -> IO ()
 webAppThread st dstatus = do
+	webapp <- mkWebApp st dstatus
 	app <- toWaiApp webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
 		)
 	runWebApp app' $ \port -> runThreadState st $ writeHtmlShim port
-	where
-		webapp = WebApp dstatus
+
+mkWebApp :: ThreadState -> DaemonStatusHandle -> IO WebApp
+mkWebApp st dstatus = do
+	dir <- absPath =<< runThreadState st (fromRepo repoPath)
+	home <- myHomeDir
+	let reldir = if dirContains home dir
+		then relPathDirToFile home dir
+		else dir
+	let s = $(embed "static")
+	return $ WebApp 
+		{ daemonStatus = dstatus
+		, baseTitle = reldir
+		, getStatic = s
+		}
 
 {- Creates a html shim file that's used to redirect into the webapp. -}
 writeHtmlShim :: PortNumber -> Annex ()
@@ -54,14 +85,6 @@ writeHtmlShim port = do
 
 {- TODO: generate this static file using Yesod. -}
 genHtmlShim :: PortNumber -> L.ByteString
-genHtmlShim port = renderHtml [shamlet|
-$doctype 5
-
-  
-    
-  
-    

-      Starting webapp...
-|]
+genHtmlShim port = renderHtml $(shamletFile $ hamletTemplate "htmlshim")
 	where
 		url = "http://localhost:" ++ show port ++ "/"
diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs
new file mode 100644
index 0000000000..05f684490a
--- /dev/null
+++ b/Utility/Yesod.hs
@@ -0,0 +1,18 @@
+{- Yesod stuff
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.Yesod where
+
+import System.FilePath
+
+{- Filename of a template, in the templates/ directory. -}
+template :: FilePath -> FilePath
+template f = "templates"  f
+
+{- A hamlet template file. -}
+hamletTemplate :: FilePath -> FilePath
+hamletTemplate f = template f ++ ".hamlet"
diff --git a/static/favicon.ico b/static/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..5bb405931fcc8f8194694e5d6cb728e50658891e
GIT binary patch
literal 405
zcmex=bI)35R>*e`CsU}f<4#mC*t-skh=9nPFAvL)f4AHU1X%imZ2>GMq7
z9Qkz8nG&C%#d}NA9x5}4Y=}=u%=tanUZi)Dv@7qlRhOEXZ#_NRE_mW`HTUk>SB@)n
MY&muAb@%_9027yGs{jB1

literal 0
HcmV?d00001

diff --git a/templates/default-layout.hamlet b/templates/default-layout.hamlet
new file mode 100644
index 0000000000..e07addc8e0
--- /dev/null
+++ b/templates/default-layout.hamlet
@@ -0,0 +1,11 @@
+$doctype 5
+
+  
+    #{baseTitle webapp} #{pageTitle page}
+    <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
+
+    ^{pageHead page}
+  <body>
+      $maybe msg <- mmsg
+        <div #message>#{msg}
+      ^{pageBody page}
diff --git a/templates/htmlshim.hamlet b/templates/htmlshim.hamlet
new file mode 100644
index 0000000000..c10042c999
--- /dev/null
+++ b/templates/htmlshim.hamlet
@@ -0,0 +1,7 @@
+$doctype 5
+<html>
+  <head>
+    <meta http-equiv="refresh" content="0; URL=#{url}">
+  <body>
+    <p>
+      <a href=#{url}">Starting webapp...

From b36804d6486b342bee7f5b4b621228bc193c4844 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 03:38:20 -0400
Subject: [PATCH 4221/8313] generate random token and launch webapp using it

---
 Assistant/Threads/WebApp.hs | 31 ++++++++++++++++++++-----------
 Utility/WebApp.hs           | 13 +++++++++++++
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 4e6fea6b11..06909fd531 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -14,17 +14,19 @@ import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Utility.WebApp
 import Utility.Yesod
+import Utility.FileMode
+import Utility.TempFile
 import Git
 
 import Yesod
 import Yesod.Static
 import Text.Hamlet
 import Network.Socket (PortNumber)
-import Text.Blaze.Renderer.Utf8
-import Data.ByteString.Lazy as L
+import Text.Blaze.Renderer.String
 
 data WebApp = WebApp
 	{ daemonStatus :: DaemonStatusHandle
+	, secretToken :: String
 	, baseTitle :: String
 	, getStatic :: Static
 	}
@@ -61,7 +63,7 @@ webAppThread st dstatus = do
 		( return $ httpDebugLogger app
 		, return app
 		)
-	runWebApp app' $ \port -> runThreadState st $ writeHtmlShim port
+	runWebApp app' $ \port -> runThreadState st $ writeHtmlShim webapp port
 
 mkWebApp :: ThreadState -> DaemonStatusHandle -> IO WebApp
 mkWebApp st dstatus = do
@@ -70,21 +72,28 @@ mkWebApp st dstatus = do
 	let reldir = if dirContains home dir
 		then relPathDirToFile home dir
 		else dir
-	let s = $(embed "static")
+	token <- genRandomToken 
 	return $ WebApp 
 		{ daemonStatus = dstatus
+		, secretToken = token
 		, baseTitle = reldir
-		, getStatic = s
+		, getStatic = $(embed "static")
 		}
 
 {- Creates a html shim file that's used to redirect into the webapp. -}
-writeHtmlShim :: PortNumber -> Annex ()
-writeHtmlShim port = do
+writeHtmlShim :: WebApp -> PortNumber -> Annex ()
+writeHtmlShim webapp port = do
 	htmlshim <- fromRepo gitAnnexHtmlShim
-	liftIO $ L.writeFile htmlshim $ genHtmlShim port
+	liftIO $ viaTmp go htmlshim $ genHtmlShim webapp port
+	where
+		go file content = do
+			h <- openFile file WriteMode
+			modifyFileMode file $ removeModes [groupReadMode, otherReadMode]
+			hPutStr h content
+			hClose h
 
 {- TODO: generate this static file using Yesod. -}
-genHtmlShim :: PortNumber -> L.ByteString
-genHtmlShim port = renderHtml $(shamletFile $ hamletTemplate "htmlshim")
+genHtmlShim :: WebApp -> PortNumber -> String
+genHtmlShim webapp port = renderHtml $(shamletFile $ hamletTemplate "htmlshim")
 	where
-		url = "http://localhost:" ++ show port ++ "/"
+		url = "http://localhost:" ++ show port ++ "/?" ++ secretToken webapp
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 614a57cea5..cded83229e 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -22,6 +22,9 @@ import Data.ByteString.Lazy
 import Data.CaseInsensitive as CI
 import Network.Socket
 import Control.Exception
+import Crypto.Random
+import Data.Digest.Pure.SHA
+import Data.ByteString.Lazy as L
 
 localhost :: String
 localhost = "localhost"
@@ -102,3 +105,13 @@ logRequest req = do
 
 lookupRequestField :: CI Ascii -> Request -> Ascii
 lookupRequestField k req = fromMaybe "" . lookup k $ requestHeaders req
+
+{- Generates a 512 byte random token, suitable to be used for an
+ - authentication secret. -}
+genRandomToken :: IO String
+genRandomToken = do
+	g <- newGenIO :: IO SystemRandom
+	return $
+		case genBytes 512 g of
+			Left e -> error $ "failed to generate secret token: " ++ show e
+			Right (s, _) -> showDigest $ sha512 $ L.fromChunks [s]

From 9d6b59d0e21e5917d098a84b7b1654bd8d07efb3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 04:50:09 -0400
Subject: [PATCH 4222/8313] use the secret token for authentication, and add to
 all dynamic urls

---
 Assistant/Threads/WebApp.hs | 25 +++++++++++--
 Utility/WebApp.hs           | 73 +++++++++++++++++++++++++++++--------
 templates/htmlshim.hamlet   |  2 +-
 3 files changed, 80 insertions(+), 20 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 06909fd531..50add37354 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -23,10 +23,14 @@ import Yesod.Static
 import Text.Hamlet
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
+import Data.Text
+
+thisThread :: String
+thisThread = "WebApp"
 
 data WebApp = WebApp
 	{ daemonStatus :: DaemonStatusHandle
-	, secretToken :: String
+	, secretToken :: Text
 	, baseTitle :: String
 	, getStatic :: Static
 	}
@@ -46,6 +50,16 @@ instance Yesod WebApp where
 		webapp <- getYesod
 		hamletToRepHtml $(hamletFile $ hamletTemplate "default-layout")
 
+	{- Require an auth token be set when accessing any (non-static route) -}
+	isAuthorized _ _ = checkAuthToken secretToken
+
+	{- Add the auth token to every url generated, except static subsite
+         - urls (which can show up in Permission Denied pages). -}
+	joinPath = insertAuthToken secretToken excludeStatic
+		where
+			excludeStatic [] = True
+			excludeStatic (p:_) = p /= "static"
+
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
 	[whamlet|Hello, World<p><a href=@{ConfigR}>config|]
@@ -75,14 +89,16 @@ mkWebApp st dstatus = do
 	token <- genRandomToken 
 	return $ WebApp 
 		{ daemonStatus = dstatus
-		, secretToken = token
+		, secretToken = pack token
 		, baseTitle = reldir
 		, getStatic = $(embed "static")
 		}
 
-{- Creates a html shim file that's used to redirect into the webapp. -}
+{- Creates a html shim file that's used to redirect into the webapp,
+ - to avoid exposing the secretToken when launching the web browser. -}
 writeHtmlShim :: WebApp -> PortNumber -> Annex ()
 writeHtmlShim webapp port = do
+	liftIO $ debug thisThread ["running on port", show port]
 	htmlshim <- fromRepo gitAnnexHtmlShim
 	liftIO $ viaTmp go htmlshim $ genHtmlShim webapp port
 	where
@@ -96,4 +112,5 @@ writeHtmlShim webapp port = do
 genHtmlShim :: WebApp -> PortNumber -> String
 genHtmlShim webapp port = renderHtml $(shamletFile $ hamletTemplate "htmlshim")
 	where
-		url = "http://localhost:" ++ show port ++ "/?" ++ secretToken webapp
+		url = "http://localhost:" ++ show port ++
+			"/?auth=" ++ unpack (secretToken webapp)
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index cded83229e..fb82c20507 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -1,30 +1,37 @@
-{- WAI webapp
+{- Yesod webapp
  -
  - Copyright 2012 Joey Hess <joey@kitenet.net>
  -
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE OverloadedStrings, CPP #-}
+{-# LANGUAGE OverloadedStrings, CPP, RankNTypes #-}
 
 module Utility.WebApp where
 
 import Common
 
-import Network.Wai
+import Yesod
+import qualified Network.Wai as Wai
 import Network.Wai.Handler.Warp
 import Network.Wai.Logger
 import Control.Monad.IO.Class
 import Network.HTTP.Types
 import System.Log.Logger
 import Data.ByteString.Lazy.UTF8
-import Data.ByteString.Lazy
-import Data.CaseInsensitive as CI
+import qualified Data.CaseInsensitive as CI
 import Network.Socket
 import Control.Exception
 import Crypto.Random
 import Data.Digest.Pure.SHA
-import Data.ByteString.Lazy as L
+import qualified Data.ByteString.Lazy as L
+import Data.AssocList
+import qualified Data.Text as T
+import qualified Data.Text.Encoding as TE
+import Blaze.ByteString.Builder.Char.Utf8 (fromText)
+import Blaze.ByteString.Builder (Builder)
+import Data.Monoid
+import Control.Arrow ((***))
 
 localhost :: String
 localhost = "localhost"
@@ -85,26 +92,26 @@ debugEnabled = do
  - Recommend only inserting this middleware when debugging is actually
  - enabled, as it's not optimised at all.
  -}
-httpDebugLogger :: Middleware
+httpDebugLogger :: Wai.Middleware
 httpDebugLogger waiApp req = do
 	logRequest req
 	waiApp req
 
-logRequest :: MonadIO m => Request -> m ()
+logRequest :: MonadIO m => Wai.Request -> m ()
 logRequest req = do
 	liftIO $ debugM "WebApp" $ unwords
-		[ showSockAddr $ remoteHost req
-		, frombs $ requestMethod req
-		, frombs $ rawPathInfo req
-		--, show $ httpVersion req
+		[ showSockAddr $ Wai.remoteHost req
+		, frombs $ Wai.requestMethod req
+		, frombs $ Wai.rawPathInfo req
+		--, show $ Wai.httpVersion req
 		--, frombs $ lookupRequestField "referer" req
 		, frombs $ lookupRequestField "user-agent" req
 		]
 	where
-		frombs v = toString $ fromChunks [v]
+		frombs v = toString $ L.fromChunks [v]
 
-lookupRequestField :: CI Ascii -> Request -> Ascii
-lookupRequestField k req = fromMaybe "" . lookup k $ requestHeaders req
+lookupRequestField :: CI.CI Ascii -> Wai.Request -> Ascii
+lookupRequestField k req = fromMaybe "" . lookup k $ Wai.requestHeaders req
 
 {- Generates a 512 byte random token, suitable to be used for an
  - authentication secret. -}
@@ -115,3 +122,39 @@ genRandomToken = do
 		case genBytes 512 g of
 			Left e -> error $ "failed to generate secret token: " ++ show e
 			Right (s, _) -> showDigest $ sha512 $ L.fromChunks [s]
+
+{- A Yesod isAuthorized method, which checks the auth cgi parameter
+ - against a token extracted from the Yesod application. -}
+checkAuthToken :: forall t sub. (t -> T.Text) -> GHandler sub t AuthResult
+checkAuthToken extractToken = do
+	webapp <- getYesod
+	req <- getRequest
+	let params = reqGetParams req
+	if lookupDef "" "auth" params == extractToken webapp
+		then return Authorized
+		else return AuthenticationRequired
+
+{- A Yesod joinPath method, which adds an auth cgi parameter to every
+ - url matching a predicate, containing a token extracted from the
+ - Yesod application.
+ - 
+ - A typical predicate would exclude files under /static.
+ -}
+insertAuthToken :: forall y. (y -> T.Text)
+	-> ([T.Text] -> Bool)
+	-> y
+	-> T.Text
+	-> [T.Text]
+	-> [(T.Text, T.Text)]
+	-> Builder
+insertAuthToken extractToken predicate webapp root pathbits params =
+	fromText root `mappend` encodePath pathbits' encodedparams
+	where
+		pathbits' = if null pathbits then [T.empty] else pathbits
+		encodedparams = map (TE.encodeUtf8 *** go) params'
+		go "" = Nothing
+		go x = Just $ TE.encodeUtf8 x
+		authparam = (T.pack "auth", extractToken webapp)
+		params'
+			| predicate pathbits = authparam:params
+			| otherwise = params
diff --git a/templates/htmlshim.hamlet b/templates/htmlshim.hamlet
index c10042c999..073b69c1bd 100644
--- a/templates/htmlshim.hamlet
+++ b/templates/htmlshim.hamlet
@@ -4,4 +4,4 @@ $doctype 5
     <meta http-equiv="refresh" content="0; URL=#{url}">
   <body>
     <p>
-      <a href=#{url}">Starting webapp...
+      <a href="#{url}">Starting webapp...

From 6cecc26206c4a539999b04664136c6f785211a41 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 05:13:27 -0400
Subject: [PATCH 4223/8313] update build deps

---
 Utility/WebApp.hs | 4 ++--
 debian/control    | 5 +++++
 doc/install.mdwn  | 8 +++++++-
 git-annex.cabal   | 7 ++++---
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index fb82c20507..6936c6699e 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -25,7 +25,7 @@ import Control.Exception
 import Crypto.Random
 import Data.Digest.Pure.SHA
 import qualified Data.ByteString.Lazy as L
-import Data.AssocList
+import Data.List
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as TE
 import Blaze.ByteString.Builder.Char.Utf8 (fromText)
@@ -130,7 +130,7 @@ checkAuthToken extractToken = do
 	webapp <- getYesod
 	req <- getRequest
 	let params = reqGetParams req
-	if lookupDef "" "auth" params == extractToken webapp
+	if lookup "auth" params == Just (extractToken webapp)
 		then return Authorized
 		else return AuthenticationRequired
 
diff --git a/debian/control b/debian/control
index 65c666cde4..58645259ef 100644
--- a/debian/control
+++ b/debian/control
@@ -24,12 +24,17 @@ Build-Depends:
 	libghc-stm-dev (>= 2.3),
 	libghc-dbus-dev,
 	libghc-yesod-dev,
+	libghc-yesod-static-dev,
 	libghc-case-insensitive-dev,
 	libghc-http-types-dev,
 	libghc-transformers-dev,
 	libghc-wai-dev,
 	libghc-wai-logger-dev,
 	libghc-warp-dev,
+	libghc-blaze-builder-dev,
+	libghc-blaze-html-dev,
+	libghc-crypto-api-dev,
+	libghc-hamlet-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 619d5fa11d..38039fbb91 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -23,7 +23,8 @@ it yourself and [[manually_build_with_cabal|install/cabal]].
 
 ## Installation by hand
 
-To build and use git-annex, you will need:
+This is not recommended, it's easier to let cabal pull in the many haskell
+libraries. To build and use git-annex by hand, you will need:
 
 * Haskell stuff
   * [The Haskell Platform](http://haskell.org/platform/) (GHC 7.4 or newer)
@@ -49,12 +50,17 @@ To build and use git-annex, you will need:
     (Linux only)
   * [dbus](http://hackage.haskell.org/package/dbus)
   * [yesod](http://hackage.haskell.org/package/yesod)
+  * [yesod-static](http://hackage.haskell.org/package/yesod-static)
   * [case-insensitive](http://hackage.haskell.org/package/case-insensitive)
   * [http-types](http://hackage.haskell.org/package/http-types)
   * [transformers](http://hackage.haskell.org/package/transformers)
   * [wai](http://hackage.haskell.org/package/wai)
   * [wai-logger](http://hackage.haskell.org/package/wai-logger)
   * [warp](http://hackage.haskell.org/package/warp)
+  * [blaze-builder](http://hackage.haskell.org/package/blaze-builder)
+  * [blaze-html](http://hackage.haskell.org/package/blaze-html)
+  * [crypto-api](http://hackage.haskell.org/package/crypto-api)
+  * [hamlet](http://hackage.haskell.org/package/hamlet)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 086df31d29..c7d9bf5707 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -46,8 +46,7 @@ Executable git-annex
    unix, containers, utf8-string, network, mtl, bytestring, old-locale, time,
    pcre-light, extensible-exceptions, dataenc, SHA, process, json, HTTP,
    base == 4.5.*, monad-control, transformers-base, lifted-base,
-   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process,
-   case-insensitive, http-types, transformers, wai, wai-logger, warp
+   IfElse, text, QuickCheck >= 2.1, bloomfilter, edit-distance, process
   -- Need to list these because they're generated from .hsc files.
   Other-Modules: Utility.Touch Utility.Mounts
   Include-Dirs: Utility
@@ -75,7 +74,9 @@ Executable git-annex
     CPP-Options: -DWITH_DBUS
 
   if flag(Webapp)
-    Build-Depends: yesod
+    Build-Depends: yesod, yesod-static, case-insensitive, http-types,
+     transformers, wai, wai-logger, warp, blaze-builder, blaze-html,
+     crypto-api, hamlet
     CPP-Options: -DWITH_WEBAPP
 
   if (os(darwin))

From b89b8015677febfb905bf1fd50546dc981d83ded Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 05:20:52 -0400
Subject: [PATCH 4224/8313] update

---
 Assistant/Threads/WebApp.hs      | 6 +++---
 doc/design/assistant/webapp.mdwn | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 50add37354..3e53828af8 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -38,9 +38,9 @@ data WebApp = WebApp
 staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
-/static StaticR Static getStatic
 / HomeR GET
 /config ConfigR GET
+/static StaticR Static getStatic
 |]
 
 instance Yesod WebApp where
@@ -62,12 +62,12 @@ instance Yesod WebApp where
 
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
-	[whamlet|Hello, World<p><a href=@{ConfigR}>config|]
+	[whamlet|Hello, World<p><a href="@{ConfigR}">config|]
 
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout $ do
 	setTitle "configuration"
-	[whamlet|<a href=@{HomeR}>main|]
+	[whamlet|<a href="@{HomeR}">main|]
 
 webAppThread :: ThreadState -> DaemonStatusHandle -> IO ()
 webAppThread st dstatus = do
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index cec766c579..0f6051d6d6 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -4,7 +4,7 @@ The webapp is a web server that displays a shiny interface.
 
 * Listen only to localhost. **done**
 * Instruct the user's web browser to open an url that contains a secret
-  token. This guards against other users on the same system.
+  token. This guards against other users on the same system. **done**
 * I would like to avoid passwords or other authentication methods,
   it's your local system.
 * Alternative for Linux at least would be to write a small program using

From 1290534ba164e67e555fc1fc7048f0a783c1df9d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 05:21:05 -0400
Subject: [PATCH 4225/8313] updae

---
 doc/design/assistant/webapp.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 598c1ff3a4..0f6051d6d6 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -2,9 +2,9 @@ The webapp is a web server that displays a shiny interface.
 
 ## security
 
-* Listen only to localhost.
+* Listen only to localhost. **done**
 * Instruct the user's web browser to open an url that contains a secret
-  token. This guards against other users on the same system.
+  token. This guards against other users on the same system. **done**
 * I would like to avoid passwords or other authentication methods,
   it's your local system.
 * Alternative for Linux at least would be to write a small program using

From c3131aafc44c424bebd401b1c57f5650ff5d9113 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 05:22:17 -0400
Subject: [PATCH 4226/8313] updae

---
 doc/design/assistant/webapp.mdwn | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 0f6051d6d6..8cdbd270f4 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -5,8 +5,8 @@ The webapp is a web server that displays a shiny interface.
 * Listen only to localhost. **done**
 * Instruct the user's web browser to open an url that contains a secret
   token. This guards against other users on the same system. **done**
-* I would like to avoid passwords or other authentication methods,
-  it's your local system.
+  (I would like to avoid passwords or other authentication methods,
+  it's your local system.)
 * Alternative for Linux at least would be to write a small program using
   GTK+ Webkit, that runs the webapp, and can know what user ran it, avoiding
   needing authentication.

From bdf4e05669615ea37e6c6a0d27ffcb93088eb56a Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 09:27:35 +0000
Subject: [PATCH 4227/8313]

---
 ...__40__or_this_a_general_problem__41__.mdwn | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
new file mode 100644
index 0000000000..4712f5f609
--- /dev/null
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -0,0 +1,28 @@
+Building commit 805d50c69d40be97baa28735371778df63b5fed6
+
+<pre>
+x00:git-annex jtang$ cabal install
+Resolving dependencies...
+Configuring dbus-0.10...
+Building dbus-0.10...
+Preprocessing library dbus-0.10...
+[1 of 9] Compiling DBus.Types       ( lib/DBus/Types.hs, dist/build/DBus/Types.o )
+[2 of 9] Compiling DBus.Message     ( lib/DBus/Message.hs, dist/build/DBus/Message.o )
+[3 of 9] Compiling DBus.Wire        ( lib/DBus/Wire.hs, dist/build/DBus/Wire.o )
+[4 of 9] Compiling DBus.Address     ( lib/DBus/Address.hs, dist/build/DBus/Address.o )
+[5 of 9] Compiling DBus             ( lib/DBus.hs, dist/build/DBus.o )
+[6 of 9] Compiling DBus.Introspection ( lib/DBus/Introspection.hs, dist/build/DBus/Introspection.o )
+[7 of 9] Compiling DBus.Transport   ( lib/DBus/Transport.hs, dist/build/DBus/Transport.o )
+
+lib/DBus/Transport.hs:196:72: Not in scope: `getPeerCred'
+cabal: Error: some packages failed to install:
+dbus-0.10 failed during the building phase. The exception was:
+ExitFailure 1
+git-annex-3.20120721 depends on dbus-0.10 which failed to install.
+</pre>
+
+The above isn't a git-annex problem but a dbus problem, at first I thought I didn't have the network package installed, but did. I should probably report this problem to the haskell dbus author.
+
+On a slightly different note, based on the makefile DBUS is not enabled on OSX/BSD so I did not expect the cabal file to use dbus as well, I'm currently interested in poking at the webapp stuff ;)
+
+Although DBUS is available on OSX from macports I get the feeling that the haskell-dbus package might need some poking before it works properly.

From 3c7cd590d85767d581acfb44fe037bdb959bc21c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 09:35:34 +0000
Subject: [PATCH 4228/8313]

---
 ...__40__or_this_a_general_problem__41__.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index 4712f5f609..f3ba0ea082 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -26,3 +26,22 @@ The above isn't a git-annex problem but a dbus problem, at first I thought I did
 On a slightly different note, based on the makefile DBUS is not enabled on OSX/BSD so I did not expect the cabal file to use dbus as well, I'm currently interested in poking at the webapp stuff ;)
 
 Although DBUS is available on OSX from macports I get the feeling that the haskell-dbus package might need some poking before it works properly.
+
+To continue, pulling, installing the dependancies (dbus is still boned) and building commit 6cecc26206c4a539999b04664136c6f785211a41
+
+<pre>
+[ 92 of 205] Compiling Utility.Url      ( Utility/Url.hs, tmp/Utility/Url.o )
+
+Utility/Url.hs:39:14: Not in scope: `parseURI'
+
+Utility/Url.hs:73:14: Not in scope: `parseURI'
+
+Utility/Url.hs:88:12: Not in scope: type constructor or class `URI'
+
+Utility/Url.hs:91:30: Not in scope: type constructor or class `URI'
+
+Utility/Url.hs:107:38: Not in scope: `parseURIReference'
+
+Utility/Url.hs:111:95: Not in scope: `relativeTo'
+make: *** [git-annex] Error 1
+</pre>

From 69ea2ccf2f136abc756dec072b6a8ceeff89dcfd Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 09:50:46 +0000
Subject: [PATCH 4229/8313]

---
 ...__40__or_this_a_general_problem__41__.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index f3ba0ea082..cab5529f0c 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -45,3 +45,22 @@ Utility/Url.hs:107:38: Not in scope: `parseURIReference'
 Utility/Url.hs:111:95: Not in scope: `relativeTo'
 make: *** [git-annex] Error 1
 </pre>
+
+Which then lead me to doing a "cabal install -f-DBus" which spits out the following when trying to link the binary
+
+<pre>
+[206 of 206] Compiling Main             ( git-annex.hs, dist/build/git-annex/git-annex-tmp/Main.o )
+Linking dist/build/git-annex/git-annex ...
+Undefined symbols for architecture x86_64:
+  "_addfds_kqueue", referenced from:
+      _s16v6_info in Kqueue.o
+  "_init_kqueue", referenced from:
+      _s16v3_info in Kqueue.o
+  "_waitchange_kqueue", referenced from:
+      _UtilityziKqueue_zdwa1_info in Kqueue.o
+ld: symbol(s) not found for architecture x86_64
+collect2: ld returned 1 exit status
+cabal: Error: some packages failed to install:
+git-annex-3.20120721 failed during the building phase. The exception was:
+ExitFailure 1
+</pre>

From fa8c41d251d82a23b01571562941bae2226f83a9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 09:55:25 +0000
Subject: [PATCH 4230/8313]

---
 ...__40__or_this_a_general_problem__41__.mdwn | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index cab5529f0c..86ab7ff317 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -64,3 +64,25 @@ cabal: Error: some packages failed to install:
 git-annex-3.20120721 failed during the building phase. The exception was:
 ExitFailure 1
 </pre>
+
+I then just tried to build commit with 6cecc26206c4a539999b04664136c6f785211a41 (i have the needed dependancies installed), gives me this...
+
+<pre>
+x00:git-annex jtang$ make
+ghc -O2 -threaded -Wall -ignore-package monads-fd -ignore-package monads-tf -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP --make git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+
+Assistant/Threads/MountWatcher.hs:39:0:
+     warning: #warning Building without dbus support; will use mtab polling
+[ 92 of 205] Compiling Utility.Url      ( Utility/Url.hs, tmp/Utility/Url.o )
+
+Utility/Url.hs:98:65:
+    Couldn't match expected type `network-2.3.0.13:Network.URI.URI'
+                with actual type `URI'
+    In the second argument of `mkRequest', namely `u'
+    In the expression: mkRequest requesttype u :: Request_String
+    In an equation for `req':
+        req = mkRequest requesttype u :: Request_String
+make: *** [git-annex] Error 1
+</pre>
+
+The latest version of the network package in hackage is network-2.3.0.14 which I have installed, this might also be the reason why dbus is broken.

From c633739f468fd77d7bd7f48d948b08255849fc17 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 10:02:14 +0000
Subject: [PATCH 4231/8313]

---
 ...cc26206c4a539999b04664136c6f785211a41.mdwn | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn

diff --git a/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn b/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn
new file mode 100644
index 0000000000..a0e9ed845c
--- /dev/null
+++ b/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn
@@ -0,0 +1,31 @@
+Seems commit 6cecc26206c4a539999b04664136c6f785211a41 missed on dependancy, that is blaze-markup
+
+<pre>
+Assistant/Threads/WebApp.hs:25:8:
+    Could not find module `Text.Blaze.Renderer.String'
+    It is a member of the hidden package `blaze-markup-0.5.1.0'.
+    Perhaps you need to add `blaze-markup' to the build-depends in your .cabal file.
+    Use -v to see a list of the files searched for.
+cabal: Error: some packages failed to install:
+git-annex-3.20120721 failed during the building phase. The exception was:
+ExitFailure 1
+</pre>
+
+This should fix it
+
+<pre>
+x00:git-annex jtang$ git diff
+diff --git a/git-annex.cabal b/git-annex.cabal
+index c7d9bf5..4f98d2a 100644
+--- a/git-annex.cabal
++++ b/git-annex.cabal
+@@ -76,7 +76,7 @@ Executable git-annex
+   if flag(Webapp)
+     Build-Depends: yesod, yesod-static, case-insensitive, http-types,
+      transformers, wai, wai-logger, warp, blaze-builder, blaze-html,
+-     crypto-api, hamlet
++     blaze-markup, crypto-api, hamlet
+     CPP-Options: -DWITH_WEBAPP
+ 
+   if (os(darwin))
+</pre>

From c7ae36403bbd851c15986a8200dc489ee42ebff9 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 10:18:50 +0000
Subject: [PATCH 4232/8313]

---
 ...s_on_OSX___40__or_this_a_general_problem__41__.mdwn | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index 86ab7ff317..43048ecba6 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -85,4 +85,12 @@ Utility/Url.hs:98:65:
 make: *** [git-annex] Error 1
 </pre>
 
-The latest version of the network package in hackage is network-2.3.0.14 which I have installed, this might also be the reason why dbus is broken.
+The latest version of the network package in hackage is network-2.3.0.14 which I have installed, this might also be the reason why dbus is broken. removing network-2.3.0.14 at least makes it happy again.
+
+to remove the network-2.3.0.14 package
+
+<pre>
+ghc-pkg unregister network-2.3.0.14
+</pre>
+
+Hope the above isn't too random of bug/issue report.

From e09c57ab5c3f579a06ee6fe5c759a72e436edf26 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 10:49:27 +0000
Subject: [PATCH 4233/8313]

---
 ...9b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn

diff --git a/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn b/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
new file mode 100644
index 0000000000..6084641346
--- /dev/null
+++ b/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
@@ -0,0 +1 @@
+The recent commit 6cecc26206c4a539999b04664136c6f785211a41 seems to have disabled the watch command on OSX, this certainly is the case when I try to run the webapp.

From 83932493250cf451a03dfbc6ca63197f7eaa6059 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 11:08:24 +0000
Subject: [PATCH 4234/8313]

---
 ...problems_on_OSX___40__or_this_a_general_problem__41__.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index 43048ecba6..4a05080ebe 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -94,3 +94,7 @@ ghc-pkg unregister network-2.3.0.14
 </pre>
 
 Hope the above isn't too random of bug/issue report.
+
+----
+
+going through <http://hackage.haskell.org/packages/archive/network/2.3.0.14/doc/html/Network-Socket.html> shows that getPeerCred is only available on systems where SO_PEERCRED is supported, *sigh* OSX isn't supported and thus haskell-dbus is broken. Apparently getpeerid is more portable but it isnt supported in the network package. It looks like dbus support on OSX isn't really going to work too well till haskell-dbus gets fixed on OSX (or BSD?)

From 587fe279515a4811b68726d3d754c470d948e5e2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 11:19:10 +0000
Subject: [PATCH 4235/8313]

---
 ...a41_disables_the_watch_command_on_OSX.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn b/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
index 6084641346..b067b9e37b 100644
--- a/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
+++ b/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
@@ -1 +1,20 @@
 The recent commit 6cecc26206c4a539999b04664136c6f785211a41 seems to have disabled the watch command on OSX, this certainly is the case when I try to run the webapp.
+
+The following fixes the makefile
+
+<pre>
+x00:git-annex jtang$ git diff
+diff --git a/Makefile b/Makefile
+index 9f312dc..4a74e71 100644
+--- a/Makefile
++++ b/Makefile
+@@ -27,7 +27,7 @@ endif
+ endif
+ 
+ PREFIX=/usr
+-GHCFLAGS=-O2 $(BASEFLAGS) $(FEATURES)
++GHCFLAGS=-O2 $(BASEFLAGS) $(FEATURES) $(OPTFLAGS)
+ 
+ ifdef PROFILE
+ GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS) $(FEATURES) $(OPTFLAGS)
+</pre>

From 5457303c1c5f1155d3d380e40400b24990423c88 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 11:24:10 +0000
Subject: [PATCH 4236/8313]

---
 ...dg-open_on_linux__47__bsd__63____41__.mdwn | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn

diff --git a/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
new file mode 100644
index 0000000000..db38d91843
--- /dev/null
+++ b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
@@ -0,0 +1,19 @@
+Utility/WebApp.hs, didn't quite have the right definition to use 'open' instead of 'xdg-open' on OSX, the follow fixes that
+
+<pre>
+diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
+index 6936c66..0593dda 100644
+--- a/Utility/WebApp.hs
++++ b/Utility/WebApp.hs
+@@ -42,7 +42,7 @@ localhost = "localhost"
+ runBrowser :: String -> IO Bool
+ runBrowser url = boolSystem cmd [Param url]
+        where
+-#if MAC
++#if OSX
+                cmd = "open"
+ #else
+                cmd = "xdg-open"
+</pre>
+
+I guess I should really for the repo and submit a stream of minor changes :P, @joeyh please let me know if you're getting annoyed with copy and pasting the small fixes from the bug/forums section.

From ffb67f99460b3e7165f7878ec0f524e4b29cdf1f Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 11:24:28 +0000
Subject: [PATCH 4237/8313]

---
 ...n_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
index db38d91843..29537cfa31 100644
--- a/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
+++ b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
@@ -16,4 +16,4 @@ index 6936c66..0593dda 100644
                 cmd = "xdg-open"
 </pre>
 
-I guess I should really for the repo and submit a stream of minor changes :P, @joeyh please let me know if you're getting annoyed with copy and pasting the small fixes from the bug/forums section.
+I guess I should really clone the repo and submit a stream of minor changes :P, @joeyh please let me know if you're getting annoyed with copy and pasting the small fixes from the bug/forums section.

From 430f1a4cd7e41577cdaff6ecaf8572654931a69c Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 11:31:56 +0000
Subject: [PATCH 4238/8313]

---
 ...4a539999b04664136c6f785211a41_segfaults.mdwn | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
new file mode 100644
index 0000000000..96bfe4c52c
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
@@ -0,0 +1,17 @@
+After fixing a few things - see [[bugs/the tip at commit 6cecc26206c4a539999b04664136c6f785211a41 disables the watch command on OSX]], [[bugs/Missing dependancy in commit 6cecc26206c4a539999b04664136c6f785211a41]] and [[bugs/Fix for opening a browser on a mac (or xdg-open on linux/bsd?)]] I tried the watch command on my ~180gig annex of stuff. This might be yet again related to the issue of [[bugs/Issue on OSX with some system limits]]
+
+the watch command segfaults
+
+<pre>
+x00:annex jtang$ git annex watch --foreground -d
+watch . [2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","git-annex"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","--hash","refs/heads/git-annex"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..38d3f769ef004b96b6d640cfb59a45f7b4edf5f6","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..ebabe9c92516c350a30126037173080648f5930b","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..d36d8d88847decc2320f0be22892ad94a8abe594","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..aaa62a8191b3c964fdf546077049f626e8561b22","--oneline","-n1"] 
+[2012-07-26 12:27:16 IST] chat: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","cat-file","--batch"] 
+(scanning...) error: git-annex died of signal 11
+</pre>
+
+The above was done on the usual OSX 10.7 system that I have.

From 82cc8d6691270cdc7b3d2ec41b7a3e45ddb885ab Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Thu, 26 Jul 2012 15:42:28 +0000
Subject: [PATCH 4239/8313]

---
 ...needs_some___34__error_checking__34__.mdwn | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn

diff --git a/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn b/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn
new file mode 100644
index 0000000000..3521038d74
--- /dev/null
+++ b/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn
@@ -0,0 +1,62 @@
+_git annex initremote_ without a complete command set still adds an entry to the uuid.log etc... and thus clutters up the state of the annex. I would not have expected this behaviour as a user.
+
+_initremote_ should fail and not do anything if the commands that it has been given are incomplete or incorrect. I was initialising a few rsync repos and noticed that i ended up having mutiple rsync remotes with the same name but different uuid's. I know its hard if not impossible to remove these uuid's so I have just marked them as "dead" in my live annexes.
+
+Here's a transcript of the problem
+
+<pre>
+x00:sandbox jtang$ mkdir atest
+x00:sandbox jtang$ cd atest/
+x00:atest jtang$ git init
+Initialized empty Git repository in /Users/jtang/sandbox/atest/.git/
+x00:atest jtang$ git annex init 
+init  ok
+(Recording state in git...)
+x00:atest jtang$ git annex status
+supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+supported remote types: git S3 bup directory rsync web hook
+trusted repositories: 0
+semitrusted repositories: 2
+	00000000-0000-0000-0000-000000000001 -- web
+ 	cbb58e1c-d737-11e1-b682-83239d5ff2e0 -- here
+untrusted repositories: 0
+dead repositories: 0
+transfers in progress: none
+available local disk space: 185 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+known annex keys: 0
+known annex size: 0 bytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+x00:atest jtang$ git annex initremote foo
+git-annex: Specify the type of remote with type=
+x00:atest jtang$ git annex initremote foo type=rsync
+(Recording state in git...)
+initremote foo git-annex: Specify encryption=key or encryption=none or encryption=shared
+x00:atest jtang$ git annex initremote foo type=rsync
+(Recording state in git...)
+initremote foo git-annex: Specify encryption=key or encryption=none or encryption=shared
+x00:atest jtang$ git annex status
+supported backends: SHA256 SHA1 SHA512 SHA224 SHA384 SHA256E SHA1E SHA512E SHA224E SHA384E WORM URL
+supported remote types: git S3 bup directory rsync web hook
+trusted repositories: (Recording state in git...)
+0
+semitrusted repositories: 5
+	00000000-0000-0000-0000-000000000001 -- web
+ 	cbb58e1c-d737-11e1-b682-83239d5ff2e0 -- here
+ 	d3adfcd0-d737-11e1-b15b-b7032388f8aa -- foo
+ 	d6d8e1e0-d737-11e1-956a-0b3d3451226a -- foo
+ 	d78d795c-d737-11e1-ac98-4fe3d6fdfd54 -- foo
+untrusted repositories: 0
+dead repositories: 0
+transfers in progress: none
+available local disk space: 185 gigabytes (+1 megabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+known annex keys: 0
+known annex size: 0 bytes
+bloom filter size: 16 mebibytes (0% full)
+backend usage: 
+x00:atest jtang$ 
+</pre>

From cccdb448749de70d1769ecef79605e4f389f9c5d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 11:52:49 -0400
Subject: [PATCH 4240/8313] git annex webapp --force forces a restart of the
 daemon

Useful for testing..
---
 Command/WebApp.hs | 16 ++++++++++++----
 Utility/Daemon.hs |  2 +-
 Utility/WebApp.hs |  1 -
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 0e01a07cd3..e94338a2fd 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -11,7 +11,7 @@ import Common.Annex
 import Command
 import Assistant
 import Utility.WebApp
-import Utility.Daemon
+import Utility.Daemon (checkDaemon)
 import qualified Annex
 
 import Control.Concurrent
@@ -25,9 +25,17 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = notBareRepo $ do
-	r <- checkpid
-	when (r == Nothing) $
-		startassistant
+	ifM (Annex.getState Annex.force)
+		( do
+			stopDaemon
+			liftIO . catchMaybeIO . removeFile
+				=<< fromRepo gitAnnexPidFile
+			startassistant
+		, do
+			r <- checkpid
+			when (r == Nothing) $
+				startassistant
+		)
 	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 	let url = "file://" ++ f
 	ifM (liftIO $ runBrowser url)
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index 8aa70d155c..3386ea4434 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -84,7 +84,7 @@ checkDaemon pidfile = do
 			| otherwise = error $
 				"stale pid in " ++ pidfile ++ 
 				" (got " ++ show pid' ++ 
-				"; expected" ++ show pid ++ " )"
+				"; expected " ++ show pid ++ " )"
 
 {- Stops the daemon, safely. -}
 stopDaemon :: FilePath -> IO ()
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 6936c6699e..8a1887678a 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -25,7 +25,6 @@ import Control.Exception
 import Crypto.Random
 import Data.Digest.Pure.SHA
 import qualified Data.ByteString.Lazy as L
-import Data.List
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as TE
 import Blaze.ByteString.Builder.Char.Utf8 (fromText)

From d9e9840c3f465a9a1f84b98db8d9cd29d276a8a3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 11:53:18 -0400
Subject: [PATCH 4241/8313] update

---
 doc/design/assistant/webapp.mdwn | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 8cdbd270f4..66561ab6f8 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -28,11 +28,12 @@ The webapp is a web server that displays a shiny interface.
 
 ## implementation
 
-Hope to use Yesod.
-
-TODO: Ensure that Yesod will work on arm. Necessary for later Android port.
-Will its template haskell cause a problem? Does new GHC support TH on ARM?
-Will it use too much memory or be too slow?
-
-Hopefully Yesod comes with some good UI widgets. Otherwise, need to use
-Jquery or similar.
+* use `addStaticContent` to make /favicon.ico work. Return `Right (route, query)`
+  and I think the route can be `favicon_ico`.
+* perhaps define a custom `errorHandler`, which could avoid the potential
+  of leaking auth tokens on error pages
+* possibly lose the ugly auth= token past the first page,
+  and use a client-side session. It could be encrypted using the token
+  as the `encryptKey`. Note: Would need to set the session duration
+  to infinite (how?)
+* look up "server-sent events" sent using `sendWaiResponse`

From a453be4195c36d8214ab44eafbb403ed55bc1d87 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 12:10:53 -0400
Subject: [PATCH 4242/8313] disable client session cookie

---
 Assistant/Threads/WebApp.hs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 3e53828af8..6e2296d5c1 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -60,6 +60,9 @@ instance Yesod WebApp where
 			excludeStatic [] = True
 			excludeStatic (p:_) = p /= "static"
 
+	{- Sessions are overkill for a local webapp with 1 user. -}
+	makeSessionBackend _ = return Nothing
+
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
 	[whamlet|Hello, World<p><a href="@{ConfigR}">config|]

From 3c117685ebaceb1b33ba2058255ef55518a0f850 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 12:17:28 -0400
Subject: [PATCH 4243/8313] on second thought, let's use --restart rather than
 --force

--force could enable other, unwanted behavior
---
 Command/WebApp.hs | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index e94338a2fd..3730e14198 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -13,29 +13,33 @@ import Assistant
 import Utility.WebApp
 import Utility.Daemon (checkDaemon)
 import qualified Annex
+import Option
 
 import Control.Concurrent
 import System.Posix.Process
 
 def :: [Command]
-def = [command "webapp" paramNothing seek "launch webapp"]
+def = [withOptions [restartOption] $
+        command "webapp" paramNothing seek "launch webapp"]
+
+restartOption :: Option
+restartOption = Option.flag [] "restart" "restart the assistant daemon"
 
 seek :: [CommandSeek]
-seek = [withNothing start]
+seek = [withFlag restartOption $ \restart -> withNothing $ start restart]
 
-start :: CommandStart
-start = notBareRepo $ do
-	ifM (Annex.getState Annex.force)
-		( do
+start :: Bool -> CommandStart
+start restart = notBareRepo $ do
+	if restart
+		then do
 			stopDaemon
-			liftIO . catchMaybeIO . removeFile
+			void $ liftIO . catchMaybeIO . removeFile
 				=<< fromRepo gitAnnexPidFile
 			startassistant
-		, do
+		else do
 			r <- checkpid
 			when (r == Nothing) $
 				startassistant
-		)
 	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 	let url = "file://" ++ f
 	ifM (liftIO $ runBrowser url)

From 95f4b192f0accbdaaa4e5c985b4e1c1a17b8aec7 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 12:41:20 -0400
Subject: [PATCH 4244/8313] on second thought, the session cookie is still
 useful to support setMessage

---
 Assistant/Threads/WebApp.hs |  3 +--
 Utility/WebApp.hs           | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 6e2296d5c1..d663b0cd52 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -60,8 +60,7 @@ instance Yesod WebApp where
 			excludeStatic [] = True
 			excludeStatic (p:_) = p /= "static"
 
-	{- Sessions are overkill for a local webapp with 1 user. -}
-	makeSessionBackend _ = return Nothing
+	makeSessionBackend = webAppSessionBackend
 
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 8a1887678a..23e00ba62a 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -24,6 +24,7 @@ import Network.Socket
 import Control.Exception
 import Crypto.Random
 import Data.Digest.Pure.SHA
+import qualified Web.ClientSession as CS
 import qualified Data.ByteString.Lazy as L
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as TE
@@ -112,7 +113,19 @@ logRequest req = do
 lookupRequestField :: CI.CI Ascii -> Wai.Request -> Ascii
 lookupRequestField k req = fromMaybe "" . lookup k $ Wai.requestHeaders req
 
-{- Generates a 512 byte random token, suitable to be used for an
+{- Rather than storing a session key on disk, use a random key
+ - that will only be valid for this run of the webapp. -}
+webAppSessionBackend :: Yesod y => y -> IO (Maybe (SessionBackend y))
+webAppSessionBackend _ = do
+	g <- newGenIO :: IO SystemRandom
+	case genBytes 96 g of
+		Left e -> error $ "failed to generate random key: " ++ show e
+		Right (s, _) -> case CS.initKey s of
+			Left e -> error $ "failed to initialize key: " ++ show e
+			Right key -> return $ Just $
+				clientSessionBackend key 120
+
+{- Generates a random sha512 string, suitable to be used for an
  - authentication secret. -}
 genRandomToken :: IO String
 genRandomToken = do

From 510247ce05f6cf9a573a344bfe942e1a4a7447e6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 12:57:41 -0400
Subject: [PATCH 4245/8313] blog for the day

---
 .../assistant/blog/day_44__webapp_basics.mdwn | 83 +++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_44__webapp_basics.mdwn

diff --git a/doc/design/assistant/blog/day_44__webapp_basics.mdwn b/doc/design/assistant/blog/day_44__webapp_basics.mdwn
new file mode 100644
index 0000000000..a8fdb131af
--- /dev/null
+++ b/doc/design/assistant/blog/day_44__webapp_basics.mdwn
@@ -0,0 +1,83 @@
+After an all-nighter, I have `git annex webapp` launching a WebApp!
+
+It doesn't do anything useful yet, just uses Yesod to display a couple of
+hyperlinked pages and a favicon, securely.
+
+The binary size grew rather alarmingly, BTW. :) Indeed, it's been growing
+for months..
+
+	-rwxr-xr-x 1 root root 9.4M Jul 21 16:59 git-annex-no-assistant-stripped
+	-rwxr-xr-x 1 joey joey  12M Jul 25 20:54 git-annex-no-webapp-stripped
+	-rwxr-xr-x 1 joey joey  17M Jul 25 20:52 git-annex-with-webapp-stripped
+
+----
+
+Along the way, some Not Invented Here occurred:
+
+I didn't use the yesod scaffolded site, because it's a lot of what
+seems mostly to be cruft in this use case. And because I don't like
+code generated from templates that people are then expected to edit. Ugh.
+That's my least favorite part of Yesod. This added some pain, since
+I had to do everything the hard way.
+
+I didn't use [wai-handler-launch](http://hackage.haskell.org/package/wai-handler-launch)
+because:
+
+  * It seems broken on IPv6 capable machines (it always opens
+    `http://127.0.0.1:port/` even though it apparently doesn't always
+    listen there.. I think it was listening on my machine's ipv6 address
+    instead. I know, I know; I should file a bug about this..)
+  * It always uses port 4587, which is **insane**. What if you have two
+    webapps?
+  * It requires javascript in the web browser, which
+    is used to ping the server, and shut it down when the web browser closes
+    (which behavior is wrong for git-annex anyway, since the daemon should
+    stay running across browser closes).
+  * It opens the webapp on web server startup, which is wrong for git-annex; 
+    instead the command `git annex webapp` will open the webapp,
+    after `git annex assistant` started the web server.
+
+Instead, I rolled my own WAI webapp laucher, that binds to any free port
+on localhost, It does use `xdg-open` to launch the web browser, 
+like wai-handler-launch (or just `open` on OS X).
+
+Also, I wrote my own WAI logger, which logs using System.Log.Logger,
+instead of to stdout, like `runDebug` does.
+
+----
+
+The webapp only listens for connections from localhost, but that's
+not sufficient "security". Instead, I added a secret token to
+every url in the webapp, that only `git annex webapp` knows about.
+
+But, if that token is passed to `xdg-open` on its command line,
+it will be briefly visible to local attackers in the parameters of
+`xdg-open`.. And if the web browser's not already running, it'll run
+with it as a parameter, and be *very* visible.
+
+So instead, I used a nasty hack. On startup, the assistant
+will create a html file, readably only by the user, that redirects
+the user to the real site url. Then `git annex webapp` will run
+xdg-open on that file.
+
+----
+
+Making Yesod check the `auth=` parameter (to verify that the secret token
+is right) is when using Yesod started to pay off. Yesod has a simple
+`isAuthorized` method that can be overridden to do your own authentication
+like this.
+
+But Yesod really started to shine when I went to add the `auth=` parameter
+to every url in the webapp. There's a `joinPath` method can can be used
+to override the default url builder. And every type-safe url in the
+application goes through there, so it's perfect for this.
+
+I just had to be careful to make it not add `auth=` to the url for the
+favicon, which is included in the "Permission Denied" error page. That'd be
+an amusing security hole..
+
+----
+
+Next up: Doing some AJAX to get a dynamic view of the state of the daemon,
+including currently running transfers, in the webapp. AKA stuff I've never
+done before, and that, unlike all this heavy Haskell Yesod, scares me. :)

From 2f775ef3e3aaea9ab15f99db3a48390a903919ae Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 12:59:40 -0400
Subject: [PATCH 4246/8313] fix OSX open call

---
 Utility/WebApp.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 23e00ba62a..517251a7a9 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -42,7 +42,7 @@ localhost = "localhost"
 runBrowser :: String -> IO Bool
 runBrowser url = boolSystem cmd [Param url]
 	where
-#if MAC
+#if OSX
 		cmd = "open"
 #else
 		cmd = "xdg-open"

From 0a652440aedfcda8238bfa1d3654a22c2979accd Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:00:29 -0400
Subject: [PATCH 4247/8313] fixed

---
 ...___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
index 29537cfa31..428d62ab17 100644
--- a/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
+++ b/doc/bugs/Fix_for_opening_a_browser_on_a_mac___40__or_xdg-open_on_linux__47__bsd__63____41__.mdwn
@@ -16,4 +16,11 @@ index 6936c66..0593dda 100644
                 cmd = "xdg-open"
 </pre>
 
-I guess I should really clone the repo and submit a stream of minor changes :P, @joeyh please let me know if you're getting annoyed with copy and pasting the small fixes from the bug/forums section.
+> [[done]], thanks
+
+I guess I should really clone the repo and submit a stream of minor changes
+:P, @joeyh please let me know if you're getting annoyed with copy and
+pasting the small fixes from the bug/forums section.
+
+> If you're going to be writing some patches, a git repo I can pull from
+> would make my life easier. --[[Joey]]

From 8e49206af5bed49999c0b2d55b17cd32b1ef666e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:02:00 -0400
Subject: [PATCH 4248/8313] fix normal build to include OPTFLAGS

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9f312dc49c..4a74e712f1 100644
--- a/Makefile
+++ b/Makefile
@@ -27,7 +27,7 @@ endif
 endif
 
 PREFIX=/usr
-GHCFLAGS=-O2 $(BASEFLAGS) $(FEATURES)
+GHCFLAGS=-O2 $(BASEFLAGS) $(FEATURES) $(OPTFLAGS)
 
 ifdef PROFILE
 GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS) $(FEATURES) $(OPTFLAGS)

From 1f040ad058b3e82524b4b0fc19bd6cc84560bc7e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:02:22 -0400
Subject: [PATCH 4249/8313] fixed

---
 ...b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn b/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
index b067b9e37b..1ee4781d10 100644
--- a/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
+++ b/doc/bugs/the_tip_at_commit_6cecc26206c4a539999b04664136c6f785211a41_disables_the_watch_command_on_OSX.mdwn
@@ -18,3 +18,5 @@ index 9f312dc..4a74e71 100644
  ifdef PROFILE
  GHCFLAGS=-prof -auto-all -rtsopts -caf-all -fforce-recomp $(BASEFLAGS) $(FEATURES) $(OPTFLAGS)
 </pre>
+
+> [[done]] --[[Joey]] 

From 78002d4976ded84cd2b0edbd78f9bb27371befe7 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:05:52 -0400
Subject: [PATCH 4250/8313] only enable dbus on linux for now

---
 git-annex.cabal | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index c7d9bf5707..3aa32ba5a0 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -69,7 +69,7 @@ Executable git-annex
     if (! os(windows))
       CPP-Options: -DWITH_KQUEUE
 
-  if flag(Dbus)
+  if os(linux) && flag(Dbus)
     Build-Depends: dbus
     CPP-Options: -DWITH_DBUS
 
@@ -79,7 +79,7 @@ Executable git-annex
      crypto-api, hamlet
     CPP-Options: -DWITH_WEBAPP
 
-  if (os(darwin))
+  if os(darwin)
     CPP-Options: -DOSX
 
 Test-Suite test

From bad5dff96e5d64f5fd203e8e09464f8338b1fe87 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:06:47 -0400
Subject: [PATCH 4251/8313] response

---
 ...ems_on_OSX___40__or_this_a_general_problem__41__.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index 4a05080ebe..7299d90e44 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -98,3 +98,12 @@ Hope the above isn't too random of bug/issue report.
 ----
 
 going through <http://hackage.haskell.org/packages/archive/network/2.3.0.14/doc/html/Network-Socket.html> shows that getPeerCred is only available on systems where SO_PEERCRED is supported, *sigh* OSX isn't supported and thus haskell-dbus is broken. Apparently getpeerid is more portable but it isnt supported in the network package. It looks like dbus support on OSX isn't really going to work too well till haskell-dbus gets fixed on OSX (or BSD?)
+
+> Does OSX acually come with dbus by default, and can you
+> use something like `dbus-monitor` to see events when
+> plugging in removable drives? If so, this might be worth spending time
+> on.
+> 
+> Currently though, dbus is not supposed to be built on non-Linux systems.
+> (Well, it might work on Freebsd or something, but I've not tried it.)
+> I've fixed the cabal file to only enable it on Linux.

From d788920ff7118e7f6929ddab925c6ee511e2de51 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Thu, 26 Jul 2012 17:09:09 +0000
Subject: [PATCH 4252/8313]

---
 ...problems_on_OSX___40__or_this_a_general_problem__41__.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
index 7299d90e44..c62dacc346 100644
--- a/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
+++ b/doc/bugs/haskell-dbus_problems_on_OSX___40__or_this_a_general_problem__41__.mdwn
@@ -103,7 +103,11 @@ going through <http://hackage.haskell.org/packages/archive/network/2.3.0.14/doc/
 > use something like `dbus-monitor` to see events when
 > plugging in removable drives? If so, this might be worth spending time
 > on.
+
+>> No OSX does not come with dbus by default, the user must install it
+
 > 
 > Currently though, dbus is not supposed to be built on non-Linux systems.
 > (Well, it might work on Freebsd or something, but I've not tried it.)
 > I've fixed the cabal file to only enable it on Linux.
+

From a2e3b93fd55cde546f035ef38fda4a05d97e865b Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 17:09:16 +0000
Subject: [PATCH 4253/8313] Added a comment: not good..

---
 ...comment_1_5cad24007f819e4be193123dab0d511a._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment
new file mode 100644
index 0000000000..23a4165a40
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_1_5cad24007f819e4be193123dab0d511a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="not good.."
+ date="2012-07-26T17:09:16Z"
+ content="""
+Do you see the segfault in a repo with fewer files?
+
+
+"""]]

From cadf331a77908f5e1551d0270ade9b87a698fb06 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 17:09:59 +0000
Subject: [PATCH 4254/8313] Added a comment

---
 .../comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment
new file mode 100644
index 0000000000..04b7ed1ae6
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_2_d449bf656a59d424833f9ab5a7fb4e82._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 2"
+ date="2012-07-26T17:09:59Z"
+ content="""
+Also, you might try bisecting to find whatever commit it first started segfaulting on.
+"""]]

From df00c6166c55e7287914706fed9323ed3bf3ac1a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:13:23 -0400
Subject: [PATCH 4255/8313] update deps

Note that here I don't need blaze-markup for cabal to succeed, but Jimmy
reports he does. Seems like Text.Blaze.Renderer.String moved from blaze to
blaze-markup in some version.
---
 debian/control   | 2 ++
 doc/install.mdwn | 2 ++
 git-annex.cabal  | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/debian/control b/debian/control
index 58645259ef..1e7cb19e2c 100644
--- a/debian/control
+++ b/debian/control
@@ -33,8 +33,10 @@ Build-Depends:
 	libghc-warp-dev,
 	libghc-blaze-builder-dev,
 	libghc-blaze-html-dev,
+	libghc-blaze-markup-dev,
 	libghc-crypto-api-dev,
 	libghc-hamlet-dev,
+	libghc-clientsession-dev,
 	ikiwiki,
 	perlmagick,
 	git,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index 38039fbb91..e529952611 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -59,8 +59,10 @@ libraries. To build and use git-annex by hand, you will need:
   * [warp](http://hackage.haskell.org/package/warp)
   * [blaze-builder](http://hackage.haskell.org/package/blaze-builder)
   * [blaze-html](http://hackage.haskell.org/package/blaze-html)
+  * [blaze-markup](http://hackage.haskell.org/package/blaze-markup)
   * [crypto-api](http://hackage.haskell.org/package/crypto-api)
   * [hamlet](http://hackage.haskell.org/package/hamlet)
+  * [clientsession](http://hackage.haskell.org/package/clientsession)
 * Shell commands
   * [git](http://git-scm.com/)
   * [uuid](http://www.ossp.org/pkg/lib/uuid/)
diff --git a/git-annex.cabal b/git-annex.cabal
index 3aa32ba5a0..aa71dacb6f 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -76,7 +76,7 @@ Executable git-annex
   if flag(Webapp)
     Build-Depends: yesod, yesod-static, case-insensitive, http-types,
      transformers, wai, wai-logger, warp, blaze-builder, blaze-html,
-     crypto-api, hamlet
+     blaze-markup, crypto-api, hamlet, clientsession
     CPP-Options: -DWITH_WEBAPP
 
   if os(darwin)

From 16b26c837b53e33e81df29b2987ab52b637e4029 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:15:24 -0400
Subject: [PATCH 4256/8313] close

---
 ...cy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn b/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn
index a0e9ed845c..2253c0f524 100644
--- a/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn
+++ b/doc/bugs/Missing_dependancy_in_commit_6cecc26206c4a539999b04664136c6f785211a41.mdwn
@@ -29,3 +29,7 @@ index c7d9bf5..4f98d2a 100644
  
    if (os(darwin))
 </pre>
+
+> [[done]].. interestingly, cabal had not complained about there here,
+> as in my version, it's in blaze, not blaze-markup. Added it anyway.
+> --[[Joey]] 

From efadf5215a9b9822052dcd6183f459c386caf9aa Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Thu, 26 Jul 2012 17:17:10 +0000
Subject: [PATCH 4257/8313]

---
 ...it_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
index 96bfe4c52c..656cc05e79 100644
--- a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
@@ -15,3 +15,7 @@ watch . [2012-07-26 12:27:16 IST] read: git ["--git-dir=/Users/jtang/annex/.git"
 </pre>
 
 The above was done on the usual OSX 10.7 system that I have.
+
+---
+
+I'll try and bisect it and find out where the problem first appeared, does the tests currently test the watch command? (also my comments seem to get moderated whether i use my openid account with google or with the native ikiwiki account, so some comments might be hidden)

From 8c3491f50c84059c765811a9c80a6db29c53d978 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 17:21:26 +0000
Subject: [PATCH 4258/8313] Comment moderation

---
 ..._6407a3e7aa0316cba2994bfef0e3c633._comment | 37 +++++++++
 ..._f01887695e8b8386e125464c6d401565._comment |  8 ++
 ..._ffb1ce41477ad60840abd7a89a133067._comment |  8 ++
 ..._8afe4720e301cf7ccf11ff0a23261936._comment |  8 ++
 ..._5da32cf53f1de27bfe6cec2d294db3e1._comment |  8 ++
 ..._696d6e22034acf5bb60d80124b72ef2f._comment |  8 ++
 ..._23e7a90429e4431f90787cd016ebe188._comment |  8 ++
 ..._3399ddad951c1a950281bb6941fc3f6f._comment |  8 ++
 ..._faafd1266301997b1822d215ec8e8d8c._comment |  8 ++
 ..._f2233fad55c20686cf299bf6788f1f23._comment | 10 +++
 ..._d052e2142da8b4838fb1edf791ea23ae._comment | 10 +++
 ..._0cdd3046d90ad2012025d846ece0731e._comment |  8 ++
 ..._e197d5d0d853572ec1f2e5985762e60d._comment |  9 +++
 ..._eb992b5b2c7a5ce23443e2a6007e5ff9._comment |  8 ++
 ..._e1b5e8a24556de16d1cacd27ee0c1bd1._comment | 80 +++++++++++++++++++
 ..._3cf0cf460c7869d0cc22940fcc84aec4._comment | 10 +++
 ..._1d70ff052d00f33c34fd45730ea13040._comment | 12 +++
 ..._5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment |  8 ++
 ..._2db02a94dffd525885c9d7fc6c5fa464._comment | 12 +++
 ..._429ec656e0ac02f98843f8d7f3c02d6a._comment | 11 +++
 ..._384813dd022dfd9c1ef14e0f1479a123._comment | 18 +++++
 ..._69ee17959a92bb8359c0fd7b2a9d8dfb._comment | 10 +++
 ..._017f4bac57a040c496e0c9d068dcfd9e._comment | 41 ++++++++++
 ..._b4c6ebada7526263e04c70eac312fda9._comment | 18 +++++
 ..._3e201039fa0e611554171ee30e69a414._comment |  8 ++
 ..._d1074724c44f3296cb438b2d526d8728._comment |  8 ++
 ..._7e88f815e8d9652ef18ea6d54b118962._comment |  8 ++
 ..._8d0db2ff65ce935c6e68044a3e0721a8._comment | 16 ++++
 ..._2e340c5a6473f165dc06cc35db38e5c0._comment |  8 ++
 ..._fc092412e99cf4c5f095b0ef710bc4de._comment |  8 ++
 ..._81cf735c143db13bd9f9e489a31e619c._comment | 19 +++++
 ..._44abd811ef79a85e557418e17a3927be._comment | 10 +++
 ..._1a383c30df4fb1767f13d8c670b0c0b5._comment | 10 +++
 33 files changed, 461 insertions(+)
 create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment
 create mode 100644 doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment
 create mode 100644 doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_2_8afe4720e301cf7ccf11ff0a23261936._comment
 create mode 100644 doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment
 create mode 100644 doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment
 create mode 100644 doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment
 create mode 100644 doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment
 create mode 100644 doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment
 create mode 100644 doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment
 create mode 100644 doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment
 create mode 100644 doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment
 create mode 100644 doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment
 create mode 100644 doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment
 create mode 100644 doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment
 create mode 100644 doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment
 create mode 100644 doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment
 create mode 100644 doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment
 create mode 100644 doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment
 create mode 100644 doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment
 create mode 100644 doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment
 create mode 100644 doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment
 create mode 100644 doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment
 create mode 100644 doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment
 create mode 100644 doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment
 create mode 100644 doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment
 create mode 100644 doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment
 create mode 100644 doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment
 create mode 100644 doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment
 create mode 100644 doc/install/OSX/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment
 create mode 100644 doc/news/version_3.20120624/comment_1_81cf735c143db13bd9f9e489a31e619c._comment
 create mode 100644 doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment
 create mode 100644 doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment

diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment
new file mode 100644
index 0000000000..30ea6b310e
--- /dev/null
+++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_5_6407a3e7aa0316cba2994bfef0e3c633._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 4"
+ date="2012-07-04T13:17:05Z"
+ content="""
+In relation to the system limits,
+
+    laplace:~ jtang$ sysctl kern.maxfilesperproc
+    kern.maxfilesperproc: 10240
+
+Also, the maxfiles for the whole system is
+
+    laplace:~ jtang$ sysctl kern.maxfiles
+    kern.maxfiles: 12288
+
+the above was the defaults as far as I recall. What you probably would be interested is the ulimits that the user see
+
+    laplace:~ jtang$ ulimit -a
+    core file size          (blocks, -c) 0
+    data seg size           (kbytes, -d) unlimited
+    file size               (blocks, -f) unlimited
+    max locked memory       (kbytes, -l) unlimited
+    max memory size         (kbytes, -m) unlimited
+    open files                      (-n) 256
+    pipe size            (512 bytes, -p) 1
+    stack size              (kbytes, -s) 8192
+    cpu time               (seconds, -t) unlimited
+    max user processes              (-u) 709
+    virtual memory          (kbytes, -v) unlimited
+
+I would imagine the limit that you are looking for is 256. Hope this helps.
+
+----
+
+On the point about deletions not being propagated, it does do a commit. I suspect that the kqueue code is just not picking up the changes and pushing the changes out. The watch command on a single annex with no remotes functions as expected.
+"""]]
diff --git a/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment
new file mode 100644
index 0000000000..cd5c73a7a5
--- /dev/null
+++ b/doc/bugs/Issue_on_OSX_with_some_system_limits/comment_6_f01887695e8b8386e125464c6d401565._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-06-25T22:36:39Z"
+ content="""
+On the system limits side, I think if you want to make it more approachable by more users then adjusting system limits might scare users away. On the note of the ssh-agents spawning like no tomorrow on my machine, it turned out that i had a symlink from my .bashrc to .bash_profile, I guess I should not be too lazy and have two seperate files.
+"""]]
diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment
new file mode 100644
index 0000000000..42f59d1bc4
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_3_ffb1ce41477ad60840abd7a89a133067._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 3"
+ date="2012-07-26T17:11:55Z"
+ content="""
+It fails on repos with either no files or smaller repos.
+"""]]
diff --git a/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_2_8afe4720e301cf7ccf11ff0a23261936._comment b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_2_8afe4720e301cf7ccf11ff0a23261936._comment
new file mode 100644
index 0000000000..1c97338f1d
--- /dev/null
+++ b/doc/bugs/watch_command_on_OSX_--_hangs_with_a_small_repo/comment_2_8afe4720e301cf7ccf11ff0a23261936._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-20T19:31:11Z"
+ content="""
+Glad that I can help ;) Alas if it weren't for the learning curve of haskell, I'd fix it myself.
+"""]]
diff --git a/doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment b/doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment
new file mode 100644
index 0000000000..253af9e7c5
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux/comment_1_5da32cf53f1de27bfe6cec2d294db3e1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-19T06:53:26Z"
+ content="""
+heh, yea, it's detecting changes on OSX ;)
+"""]]
diff --git a/doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment b/doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment
new file mode 100644
index 0000000000..9f3e34adb9
--- /dev/null
+++ b/doc/design/assistant/blog/day_12__freebsd_redux/comment_2_696d6e22034acf5bb60d80124b72ef2f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-19T07:01:26Z"
+ content="""
+issues with the watch command on OSX, it seems that there is a race condition somewhere. I dumped a few iso's into an annex and it only annexed the smaller files (checksums) and the bigger ones (the iso's) just got made read only. also do you want these bugs to be logged here or in the bugs section?
+"""]]
diff --git a/doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment
new file mode 100644
index 0000000000..fece5c9aff
--- /dev/null
+++ b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_1_23e7a90429e4431f90787cd016ebe188._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-06-27T12:46:31Z"
+ content="""
+can X and Y be the names of the git-annex remotes?
+"""]]
diff --git a/doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment b/doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment
new file mode 100644
index 0000000000..9007850e46
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_4_3399ddad951c1a950281bb6941fc3f6f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-19T18:43:30Z"
+ content="""
+Joey, yes dbus is available from macports and homebrew, it's not installed by default (or as a dependancy) for most packages in macports.
+"""]]
diff --git a/doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment b/doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment
new file mode 100644
index 0000000000..074a3a82c8
--- /dev/null
+++ b/doc/design/assistant/cloud/comment_3_faafd1266301997b1822d215ec8e8d8c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawn7Oyqusvn0oONFtVhCx5gRAcvPjyRMcBI"
+ nickname="Michaël"
+ subject="is ftp an option?"
+ date="2012-05-30T10:44:12Z"
+ content="""
+for people only having ftp-access to there storage.
+"""]]
diff --git a/doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment b/doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment
new file mode 100644
index 0000000000..f24357fb64
--- /dev/null
+++ b/doc/design/assistant/comment_10_f2233fad55c20686cf299bf6788f1f23._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://www.klomp.eu/"
+ ip="95.91.241.82"
+ subject="Watch also possible with git?"
+ date="2012-06-15T17:25:30Z"
+ content="""
+Hi,
+
+it seems that you put a lot of efforts in handling race conditions. Thats great. I wonder if the watch can also be used with git (i.e. changes are commited into git and not as annex)? I know that other projects follow this idea but why using different tools if the git-annex assistant could handle both...
+"""]]
diff --git a/doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment b/doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment
new file mode 100644
index 0000000000..5e955c2b6c
--- /dev/null
+++ b/doc/design/assistant/comment_9_d052e2142da8b4838fb1edf791ea23ae._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://wiggy.net/"
+ nickname="Wichert"
+ subject="macports"
+ date="2012-06-12T13:00:34Z"
+ content="""
+The average OSX user has a) no idea what macports is, and b) will not be able to install it. Anything that requires a user to do anything with a commandline (or really anything other than using a GUI installer) is effectively a dealbreaker. For our use cases OSX is definitely a requirement, but it must only use standard OSX installation methods in order to be usable. Being in the appstore would be ideal, but standard dmg/pkg installers are still common enough that they are also acceptable.
+
+FWIW this is the same reason many git GUIs were not usable for our OSX users: they required separate installation of the git commandline tools.
+"""]]
diff --git a/doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment b/doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment
new file mode 100644
index 0000000000..8b075c36f0
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_5_0cdd3046d90ad2012025d846ece0731e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-06-17T21:42:59Z"
+ content="""
+okay, I've gotten gitbuilder to poll the git repo every minute for changes, gitbuilder doesn't build every commit. It doesn't work like that, it checks out the master and builds that. If there is a failure it automatically bisects to find out where the problem first got introduced. Hope the change to the builder helps!
+"""]]
diff --git a/doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment b/doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment
new file mode 100644
index 0000000000..76716ddda5
--- /dev/null
+++ b/doc/design/assistant/inotify/comment_6_e197d5d0d853572ec1f2e5985762e60d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnV2c63kDc6X21a1H81me1mIenUCScd2Gs"
+ nickname="Emanuele"
+ subject="watch branch?"
+ date="2012-06-01T19:19:17Z"
+ content="""
+Hello there? Where can I find more info about this git watch branch? 
+Keep up the good work!
+"""]]
diff --git a/doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment b/doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment
new file mode 100644
index 0000000000..a4609d7e19
--- /dev/null
+++ b/doc/design/assistant/syncing/comment_2_eb992b5b2c7a5ce23443e2a6007e5ff9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawnBl7cA6wLDxVNUyLIHvAyCkf8ir3alYpk"
+ nickname="Tyson"
+ subject="Bridging LANs"
+ date="2012-07-10T10:20:59Z"
+ content="""
+Why rely on the cloud when you can instead use XMPP and jingle to perform NAT traversal for you? AFAIKT, it also means that traffic won't leave your router if the two endpoints are behind the same router.
+"""]]
diff --git a/doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment b/doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment
new file mode 100644
index 0000000000..c9118595c9
--- /dev/null
+++ b/doc/design/assistant/syncing/comment_3_e1b5e8a24556de16d1cacd27ee0c1bd1._comment
@@ -0,0 +1,80 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-07-03T08:26:43Z"
+ content="""
+On \"git syncing\" point number 9, on OSX you could potentially do this on a semi-regular basis
+
+<pre>
+system_profiler SPNetworkVolumeDataType
+Volumes:
+
+    net:
+
+      Type: autofs
+      Mount Point: /net
+      Mounted From: map -hosts
+      Automounted: Yes
+
+    home:
+
+      Type: autofs
+      Mount Point: /home
+      Mounted From: map auto_home
+      Automounted: Yes
+</pre>
+
+and
+
+<pre>
+x00:~ jtang$ system_profiler SPUSBDataType
+USB:
+
+    USB High-Speed Bus:
+
+      Host Controller Location: Built-in USB
+      Host Controller Driver: AppleUSBEHCI
+      PCI Device ID: 0x0aa9 
+      PCI Revision ID: 0x00b1 
+      PCI Vendor ID: 0x10de 
+      Bus Number: 0x26 
+
+        Hub:
+
+          Product ID: 0x2504
+          Vendor ID: 0x0424  (SMSC)
+          Version: 0.01
+          Speed: Up to 480 Mb/sec
+          Location ID: 0x26200000 / 3
+          Current Available (mA): 500
+          Current Required (mA): 2
+
+            USB to ATA/ATAPI Bridge:
+
+              Capacity: 750.16 GB (750,156,374,016 bytes)
+              Removable Media: Yes
+              Detachable Drive: Yes
+              BSD Name: disk1
+              Product ID: 0x2338
+              Vendor ID: 0x152d  (JMicron Technology Corp.)
+              Version: 1.00
+              Serial Number: 313541813001
+              Speed: Up to 480 Mb/sec
+              Manufacturer: JMicron
+              Location ID: 0x26240000 / 5
+              Current Available (mA): 500
+              Current Required (mA): 2
+              Partition Map Type: MBR (Master Boot Record)
+              S.M.A.R.T. status: Not Supported
+              Volumes:
+                Porta-Disk:
+                  Capacity: 750.16 GB (750,156,341,760 bytes)
+                  Available: 668.42 GB (668,424,208,384 bytes)
+                  Writable: Yes
+                  File System: ExFAT
+....
+</pre>
+
+I think its possible to programatically get this information either from the CLI (it dumps out XML output if required) or some development library. There is also DBUS in macports, but I have never had much interaction with it, so I don't know if its good or bad on OSX.
+"""]]
diff --git a/doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment b/doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment
new file mode 100644
index 0000000000..60d678aba4
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_2_3cf0cf460c7869d0cc22940fcc84aec4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yatesa"
+ ip="171.25.193.21"
+ subject="Secret URL token"
+ date="2012-06-19T03:41:16Z"
+ content="""
+> Instruct the user's web browser to open an url that contains a secret token. This guards against other users on the same system.
+
+How will you implement that? Running \"sensible-browser URL\" would be the obvious way, but the secret URL would show up in a well timed ps listing. (And depending on the browser, ps may show the URL the entire time it's running.)
+"""]]
diff --git a/doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment b/doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment
new file mode 100644
index 0000000000..017b34b0de
--- /dev/null
+++ b/doc/forum/Debugging_Git_Annex/comment_2_1d70ff052d00f33c34fd45730ea13040._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="comment 2"
+ date="2012-06-05T17:19:16Z"
+ content="""
+Hi,
+
+I want to replace rsync with aspera-rsync. Whenever there is file transfer between 2 repositories which are in two different hosts, git-annex will use rsync protocol. I am trying to replace that rsync call with aspera-rsync so that transfer can be more faster. Since I am new to Haskell I am finding difficulties to understand the flow of execution. Is there any way I can debug so that I can get the flow?
+
+Thanks
+"""]]
diff --git a/doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment b/doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment
new file mode 100644
index 0000000000..4692338af6
--- /dev/null
+++ b/doc/forum/Fixing_up_corrupt_annexes/comment_2_5cdd2fcfa61b3f6255e5ad63a3ab00ce._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-25T06:52:32Z"
+ content="""
+Ah I was looking at the walkthrough on how to fix the issue, I had not thought about looking at the tips section. That tip fixed the issue for me, thanks.
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment b/doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment
new file mode 100644
index 0000000000..4b1dbd271b
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_2_2db02a94dffd525885c9d7fc6c5fa464._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/IAg3idYGk.joxsJb2WCxl20gig.0.8hS#d5165"
+ nickname="Kelly"
+ subject="comment 2"
+ date="2012-05-10T15:01:15Z"
+ content="""
+I think my comment a couple days ago got caught in the spam filter, so I'm reposting. 
+What were the ideas to avoid parameterisation? What were the problems of parameterisation, other than just the current hardcoded assumptions?
+
+Speaking of hash insecurity, http://static.usenix.org/events/hotos03/tech/full_papers/henson/henson_html/node8.html says compare-by-hash is a bad idea. As I understand, git doesn't have an option of verifying content matches when the hash matches when adding data to the object store (like zfs's \"dedup=verify\" option, which you can use even when using sha256), because the assumption is that the risk of collision (or at least just the risk of accidental collision) is negligible. Would it be worthwhile to add this option to git-annex?
+
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment b/doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment
new file mode 100644
index 0000000000..41b7570e12
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_3_429ec656e0ac02f98843f8d7f3c02d6a._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/IAg3idYGk.joxsJb2WCxl20gig.0.8hS#d5165"
+ nickname="Kelly"
+ subject="comment 2"
+ date="2012-05-09T01:22:13Z"
+ content="""
+What were the ideas to avoid parameterisation? What were the problems of parameterisation, other than just the current hardcoded assumptions?
+
+Speaking of hash insecurity, http://static.usenix.org/events/hotos03/tech/full_papers/henson/henson_html/node8.html says compare-by-hash is a bad idea. As I understand, git doesn't have an option of verifying content matches when the hash matches when adding data to the object store (like zfs's \"dedup=verify\" option, which you can use even when using sha256), because the assumption is that the risk of collision (or at least just the risk of accidental collision) is negligible. Would it be worthwhile to add this option to git-annex?
+
+"""]]
diff --git a/doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment b/doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment
new file mode 100644
index 0000000000..40f0e02b0a
--- /dev/null
+++ b/doc/forum/Making_git-annex_less_necessary/comment_4_384813dd022dfd9c1ef14e0f1479a123._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://christian.amsuess.com/"
+ nickname="chrysn"
+ subject="comment 2"
+ date="2012-05-11T17:40:20Z"
+ content="""
+from my layman's standpoint, i think it would be feasible. i've suggested this previously, but not pushed it too much. quoting from [[my user page|users/chrysn]]:
+
+* **would like git-annex to**: not be required any more as git itself learns to use cow filesystems to avoid abundant disk usage and gets better with sparser checkouts (git-annex might then still be a simpler tool that watches over what can be safely dropped for a sparser checkout)
+
+*concerning hash sizes or parameterized hashes*: the problems with hash sizes could be avoided if instead of putting the objects in the \"normal\" object dir, barefiles would be managed in a similar way as packs are. when a new files gets added, they'd be cow-copied to ``.git/objects/bare/${HA}/${SH}``, and ``.git/objects/bareprefix/${HA}/${SH}`` would contain the \"blob ${SIZE}\0\" prefix that gets concatenated to the object body to form the object itself.
+
+(maybe it'd even be sufficient to *just store the size* in the bareprefix, as all those objects would be blobs, but then again, some flexibility won't hurt.)
+
+if the *pack file format* is flexible enough, the bareprefix files can get packed too. for the adventerous user who modifies bigfiles, the pack file mechanisms should be made aware of their presence, and be able to store deltas between them. the operations for applying those deltas would be difficult to optimize, and could be added at a later stage. a typical example could be storing a pdf file -- the pdf file format is designed for appending, so chances are the new version is just the old version plus several k at the end.
+
+neither of that would affect git's *wire protocol*, so no compatibility problems. (it would be advisable to find a reasonable way to do sparse checkouts, though; something like \"server, pack and send your master, but make it sparse and don't include blobs >1mb\").
+"""]]
diff --git a/doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment b/doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment
new file mode 100644
index 0000000000..f4293d9c11
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict/comment_2_69ee17959a92bb8359c0fd7b2a9d8dfb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Resolving conflict"
+ date="2012-04-24T03:59:31Z"
+ content="""
+Hi,
+Now I am able to resolve the conflict.
+Thank you.
+"""]]
diff --git a/doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment b/doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment
new file mode 100644
index 0000000000..a0d3ded394
--- /dev/null
+++ b/doc/forum/What_can_be_done_in_case_of_conflict/comment_3_017f4bac57a040c496e0c9d068dcfd9e._comment
@@ -0,0 +1,41 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlYu7QmD7wrbHWkoxuriaA9XcijM-g5vrQ"
+ nickname="Royal"
+ subject="Resolving conflict"
+ date="2012-04-23T15:49:30Z"
+ content="""
+Thanks for the reply.
+
+I am executing the following commands.
+
+git init main
+cd main
+git annex init main
+echo a > a
+git annex add a
+git commit -m Initial
+git annex unlock a
+echo aa > a
+git annex add a
+git commit -m first
+git annex unlock a
+echo aaa > a
+git annex add a
+git commit -m second
+git log
+git cherry-pick <Hash of first commit>
+
+--------------------
+Error: 
+
+error: could not apply 2be8f38... first
+hint: after resolving the conflicts, mark the corrected paths
+hint: with 'git add <paths>' or 'git rm <paths>'
+hint: and commit the result with 'git commit'
+
+How can resolve the the above conflict.
+If I see the content of the file I will get the content of second commit.
+Is there any way I can get the content for first commit(Like in git we have 'theirs' option.)
+
+Thank you.
+"""]]
diff --git a/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment b/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment
new file mode 100644
index 0000000000..551c685d4e
--- /dev/null
+++ b/doc/forum/Wishlist:_getting_the_disk_used_by_a_subtree_of_files/comment_2_b4c6ebada7526263e04c70eac312fda9._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 2"
+ date="2012-06-27T12:45:42Z"
+ content="""
+I have a hacked up version of sharebox that does this.. I need to fix it up and push it to github..
+
+the short of it is that you can do
+
+    def calculate_size(path):
+        annexfile = os.path.basename(os.readlink(path))
+        #SHA256-s2007550713--....
+        size = annexfile.split(\"-\")[1]
+        return int(size[1:])
+
+to get the size of files.. a 'git-annex du' should be pretty straightforward...
+"""]]
diff --git a/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment
new file mode 100644
index 0000000000..b1a76ce2d3
--- /dev/null
+++ b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_2_3e201039fa0e611554171ee30e69a414._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-06-23T08:00:12Z"
+ content="""
+actually, scratch that, i found it. it was in _.git/annex/daemon.log_ along with the other bits and pieces
+"""]]
diff --git a/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment
new file mode 100644
index 0000000000..eb18628290
--- /dev/null
+++ b/doc/forum/Wishlist:_logging_to_file_when_running_as_a_daemon___40__for_the_assistant__41__/comment_3_d1074724c44f3296cb438b2d526d8728._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 2"
+ date="2012-07-03T14:48:16Z"
+ content="""
+Adding a date and timestamp would be a nice start to improving things.
+"""]]
diff --git a/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment b/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment
new file mode 100644
index 0000000000..82380cd602
--- /dev/null
+++ b/doc/forum/autobuilders_for_git-annex_to_aid_development/comment_1_7e88f815e8d9652ef18ea6d54b118962._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 1"
+ date="2012-07-02T16:25:55Z"
+ content="""
+I've some binaries for OSX which can be found at <http://www.sgenomics.org/~jtang/gitbuilder-git-annex-x00-x86_64-apple-darwin10.8.0-binary/dist/> its just the master branch, and it's built on a system that runs macports. Binaries are built and updated whenever there are changes made to the master branch of git-annex.
+"""]]
diff --git a/doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment b/doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment
new file mode 100644
index 0000000000..8fd73cd92c
--- /dev/null
+++ b/doc/forum/pulling_from_encrypted_remote/comment_2_8d0db2ff65ce935c6e68044a3e0721a8._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmLB39PC89rfGaA8SwrsnB6tbumezj-aC0"
+ nickname="Tobias"
+ subject="Thats a fair solution"
+ date="2012-07-22T13:51:25Z"
+ content="""
+Until you don't have(access to) an existing repository to clone from.
+
+I really hope you revisit this when you come to the encryption part of the assistant.
+
+Btw, I also run FreeBSD if you need a tester on that at some point.
+
+Sincerely
+Tobias Ussing
+
+"""]]
diff --git a/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment
new file mode 100644
index 0000000000..2b6ff45df9
--- /dev/null
+++ b/doc/forum/rsync_remote_is_not_available_from_a_cloned_repo/comment_1_2e340c5a6473f165dc06cc35db38e5c0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="hannes"
+ ip="130.226.142.243"
+ subject="original repo git annex version"
+ date="2012-07-08T11:55:42Z"
+ content="""
+is 3.20120629
+"""]]
diff --git a/doc/install/OSX/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment b/doc/install/OSX/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment
new file mode 100644
index 0000000000..47ad9feafc
--- /dev/null
+++ b/doc/install/OSX/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 17"
+ date="2012-07-24T06:33:13Z"
+ content="""
+@a-or-b that issue is logged here [[bugs/subtle build issue on OSX 10.7 and Haskell Platform (if you have the 32bit version installed)]], you can use cabal to build and install git-annex and it will detect if its 32 or 64bit automatically.
+"""]]
diff --git a/doc/news/version_3.20120624/comment_1_81cf735c143db13bd9f9e489a31e619c._comment b/doc/news/version_3.20120624/comment_1_81cf735c143db13bd9f9e489a31e619c._comment
new file mode 100644
index 0000000000..bb048c8148
--- /dev/null
+++ b/doc/news/version_3.20120624/comment_1_81cf735c143db13bd9f9e489a31e619c._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmURXBzaYE1gmVc-X9eLAyDat_6rHPl670"
+ nickname="Bram"
+ subject="Build failure using Cabal"
+ date="2012-06-24T19:46:13Z"
+ content="""
+Unfortunately I get a build failure when building this version using Cabal:
+
+    [113 of 183] Compiling Assistant.Changes ( Assistant/Changes.hs, dist/build/git-annex/git-annex-tmp/Assistant/Changes.o )
+
+    Assistant/Changes.hs:73:30:
+        Not in scope: `tryReadTChan'
+        Perhaps you meant `readTChan' (imported from Control.Concurrent.STM)
+    cabal: Error: some packages failed to install:
+    git-annex-3.20120624 failed during the building phase. The exception was:
+    ExitFailure 1
+
+The complete build log is at <http://paste.debian.net/176125/>.  I'm looking forward to trying out the new watch behaviour!
+"""]]
diff --git a/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment b/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment
new file mode 100644
index 0000000000..d253feb5b8
--- /dev/null
+++ b/doc/tips/assume-unstaged/comment_1_44abd811ef79a85e557418e17a3927be._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://me.yahoo.com/a/2djv2EYwk43rfJIAQXjYt_vfuOU-#a11a6"
+ nickname="Olivier R"
+ subject="It doesn't work 100%"
+ date="2012-05-03T21:42:54Z"
+ content="""
+When you remove tracked files... it doesn't show the new status. it's like if the file was ignored.
+
+
+"""]]
diff --git a/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment
new file mode 100644
index 0000000000..5569ff94a4
--- /dev/null
+++ b/doc/todo/wishlist:_special-case_handling_of_Youtube_URLs_in_Web_special_remote/comment_1_1a383c30df4fb1767f13d8c670b0c0b5._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://rmunn.myopenid.com/"
+ nickname="rmunn"
+ subject="comment 1"
+ date="2012-06-12T15:52:35Z"
+ content="""
+* One way to handle the configuration might be with regular expressions. If the URL matches regex A, handle it with downloader A' (with option set A''). If the URL matches regex B, handle it with downloader B' and option set B''. And so on. Then if nothing is matched, the default downloader is wget/curl.
+
+* In my experience, youtube-dl breakages are fixed relatively quickly; a much more serious problem from a trust standpoint is that Youtube videos often disappear. Sometimes due to a legitimate copyright claim, sometimes due to illegitimate copyright claims. (I've seen both happen). Or because the video uploader decided to upload *other* videos that violated copyright, and Youtube closed his/her account, thereby removing *all* his/her videos from the Web. Youtube is definitely an untrustworthy repository as far as \"the file will still be there later on\" is concerned. Perhaps a default trust relationship could go along with the regexes? URLs matching regex A are semitrusted, while URLs matching regex B are untrusted.
+"""]]

From fbf5b7b090311af779dda063942b772f24fc9e9c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:24:01 -0400
Subject: [PATCH 4259/8313] reply

---
 ..._6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
index 656cc05e79..f7259d6bfc 100644
--- a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults.mdwn
@@ -19,3 +19,9 @@ The above was done on the usual OSX 10.7 system that I have.
 ---
 
 I'll try and bisect it and find out where the problem first appeared, does the tests currently test the watch command? (also my comments seem to get moderated whether i use my openid account with google or with the native ikiwiki account, so some comments might be hidden)
+
+> The test suite does not currently test the watch command, unfortunatly.
+>
+> Wow, I had not noticed the 30 pending moderated comments.. Let them all
+> thru, and I guess I'll turn off comment spam filtering for now, since
+> there has apparently been none. --[[Joey]] 

From f0e683bbd4ae5c7a2e29c44f140f826d5865e1cf Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 17:27:39 +0000
Subject: [PATCH 4260/8313] Added a comment

---
 ...comment_2_8e7e7cd27791bb47625e60a284e9c802._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment

diff --git a/doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment
new file mode 100644
index 0000000000..e47b03d48d
--- /dev/null
+++ b/doc/design/assistant/blog/day_16__more_robust_syncing/comment_2_8e7e7cd27791bb47625e60a284e9c802._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 2"
+ date="2012-07-26T17:27:39Z"
+ content="""
+That's a good question. Unfortunatly they cannot; X and Y need to be stable across repositories, and git remotes can have different names in different repositories.
+
+Even using the description that git-annex stores for each repository for X and Y is problimatic, since that description can change, and so could be different in two repos that are each trying to resolve the same merge conflict.
+"""]]

From b77eb6b47e73bf32b4e2dc3d806f4dee0a0ecf31 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Thu, 26 Jul 2012 17:35:18 +0000
Subject: [PATCH 4261/8313] Added a comment

---
 .../comment_3_428e153135f7a64215730719207d82c4._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment

diff --git a/doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment b/doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment
new file mode 100644
index 0000000000..7a73799aa4
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_3_428e153135f7a64215730719207d82c4._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 3"
+ date="2012-07-26T17:35:18Z"
+ content="""
+Using twitter-bootstrap for the webapp - this might be a wishlist item, but would it be possible to ensure that the webapp's css uses twitter-bootstrap classes. It would make theming much easier in the long run and it would give you a nice modern look with a low amount of effort.
+"""]]

From 756c80b67acb3a5a7e3e3cc1b3c290421462dc75 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:36:28 -0400
Subject: [PATCH 4262/8313] move to old

---
 .../comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment           | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/install/OSX/{ => old_comments}/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment (100%)

diff --git a/doc/install/OSX/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment b/doc/install/OSX/old_comments/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment
similarity index 100%
rename from doc/install/OSX/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment
rename to doc/install/OSX/old_comments/comment_3_fc092412e99cf4c5f095b0ef710bc4de._comment

From ae3d7cfe6870ca47f2e7f578426c8cec6d637184 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 17:45:28 +0000
Subject: [PATCH 4263/8313] Added a comment

---
 ...comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment

diff --git a/doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment b/doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment
new file mode 100644
index 0000000000..d713e3e8fc
--- /dev/null
+++ b/doc/design/assistant/webapp/comment_4_f4068a7abbb77ba6a3297cbcf1e503e9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 4"
+ date="2012-07-26T17:45:28Z"
+ content="""
+So, Yesod's scaffolded site actually does use bootstrap, but I didn't use the scaffolded site so don't have it. I am not quite to the point of doing any theming of the webapp, but I do have this nice example of how to put in bootstrap right here..
+
+By the way, if anyone would like to play with the html templates for the webapp, the main html template is `templates/default-layout.hamlet`. Uses a slightly weird template markup, but plain html will also work. And there's also the `static/` directory; every file in there will be compiled directly into the git-annex binary, and is available at `http://localhost:port/static/$file` in the webapp. See the favicon link in `default-layout.hamlet` of how to construct a type-safe link to a static file: `href=@{StaticR favicon_ico}`. That's all you really need to theme the webapp, without doing any real programming!
+"""]]

From f3efc6dc93b4d4e5054f8a874bd4657245ffb885 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 13:47:41 -0400
Subject: [PATCH 4264/8313] update

---
 doc/design/assistant/webapp.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 66561ab6f8..fe910c1976 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -7,6 +7,9 @@ The webapp is a web server that displays a shiny interface.
   token. This guards against other users on the same system. **done**
   (I would like to avoid passwords or other authentication methods,
   it's your local system.)
+* Don't pass the url with secret token directly to the web browser,
+  as that exposes it to `ps`. Instead, write a html file only the user can read,
+  that redirects to the webapp. **done**
 * Alternative for Linux at least would be to write a small program using
   GTK+ Webkit, that runs the webapp, and can know what user ran it, avoiding
   needing authentication.

From 04a940434b77c7f73228840d8b941d54199cfebc Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Thu, 26 Jul 2012 17:50:42 +0000
Subject: [PATCH 4265/8313] Added a comment

---
 .../comment_4_cebbc138c6861c086bb7937b54f5adbc._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment
new file mode 100644
index 0000000000..71e810291e
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_4_cebbc138c6861c086bb7937b54f5adbc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 4"
+ date="2012-07-26T17:50:42Z"
+ content="""
+It just occurred to me that bisecting won't help much, as the watch command was disabled accidentally in earlier commits and doing a script for bisecting is going to be as much work as just stepping through and debugging the issue with a debugger (i might need to fire up gdb on a mac (this wont be fun))
+"""]]

From a7dfeaf3ebb2db1d490b2e945216035f7a84ece0 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Thu, 26 Jul 2012 18:06:30 +0000
Subject: [PATCH 4266/8313]

---
 doc/forum/Looking_at_the_webapp_on_OSX.mdwn | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX.mdwn

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX.mdwn b/doc/forum/Looking_at_the_webapp_on_OSX.mdwn
new file mode 100644
index 0000000000..b2c514d835
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX.mdwn
@@ -0,0 +1,18 @@
+Not logging this in the bugs section, but poking at the new webapp feature...
+
+<pre>
+laplace:annex jtang$ git annex webapp -d
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","git-annex"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","show-ref","--hash","refs/heads/git-annex"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..731005d121426a38b206c4544da02cdb3b974974","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..d36d8d88847decc2320f0be22892ad94a8abe594","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..57bcddc14d03b61028f7002e2dabcc5181d74f3d","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] read: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","log","refs/heads/git-annex..372aceaf49b60ebe31cc3fe2e52ba4fbe99c134f","--oneline","-n1"] 
+[2012-07-26 19:02:50 IST] chat: git ["--git-dir=/Users/jtang/annex/.git","--work-tree=/Users/jtang/annex","cat-file","--batch"] 
+[2012-07-26 19:02:50 IST] read: git ["config","--null","--list"] 
+[2012-07-26 19:02:50 IST] call: open ["file:///Users/jtang/annex/.git/annex/webapp.html"] 
+The file /Users/jtang/annex/.git/annex/webapp.html does not exist.
+git-annex: failed to start web browser on url file:///Users/jtang/annex/.git/annex/webapp.html
+</pre>
+
+I would have expected the open command to open up http://localhost:port/ instead of a file on my machine. Anyway, its just an observation on the current state of the webapp feature, not expecting it to work enough for me to test it ;)

From 2a91034e8b868d513b47b9ea1a847e61f6dba676 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 18:17:40 +0000
Subject: [PATCH 4267/8313] Added a comment

---
 ...mment_1_68820f2f469356633c1abb18a47e0c59._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment
new file mode 100644
index 0000000000..9aa05c8f0a
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_1_68820f2f469356633c1abb18a47e0c59._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 1"
+ date="2012-07-26T18:17:40Z"
+ content="""
+This could happen if you already had an old version of git-annex assistant running in that repository.
+
+Try:
+
+git annex assistant --stop ; git annex webapp
+"""]]

From 860415aa5b1d1c367a8a4810ef29e4a9417b0989 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 14:26:35 -0400
Subject: [PATCH 4268/8313] webapp: check that the shim exists, and restart the
 assistant if not

---
 Command/WebApp.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 3730e14198..6533634400 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -30,7 +30,9 @@ seek = [withFlag restartOption $ \restart -> withNothing $ start restart]
 
 start :: Bool -> CommandStart
 start restart = notBareRepo $ do
-	if restart
+	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
+	ok <- liftIO $ doesFileExist f
+	if restart || not ok
 		then do
 			stopDaemon
 			void $ liftIO . catchMaybeIO . removeFile
@@ -40,7 +42,6 @@ start restart = notBareRepo $ do
 			r <- checkpid
 			when (r == Nothing) $
 				startassistant
-	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 	let url = "file://" ++ f
 	ifM (liftIO $ runBrowser url)
 		( stop

From e79198aacbb7891b0b7a4d156160a1524038e18c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 15:28:08 -0400
Subject: [PATCH 4269/8313] when starting the assistant, wait for it to create
 the shim file, as well as the pid file

fixes a possible race
---
 Command/WebApp.hs | 39 +++++++++++++++++----------------------
 1 file changed, 17 insertions(+), 22 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 6533634400..5fcaad6fda 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -31,39 +31,34 @@ seek = [withFlag restartOption $ \restart -> withNothing $ start restart]
 start :: Bool -> CommandStart
 start restart = notBareRepo $ do
 	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
-	ok <- liftIO $ doesFileExist f
-	if restart || not ok
+	if restart
 		then do
 			stopDaemon
-			void $ liftIO . catchMaybeIO . removeFile
-				=<< fromRepo gitAnnexPidFile
-			startassistant
-		else do
-			r <- checkpid
-			when (r == Nothing) $
-				startassistant
+			nuke =<< fromRepo gitAnnexPidFile
+			startassistant f
+		else unlessM (checkpid f) $
+			startassistant f
 	let url = "file://" ++ f
 	ifM (liftIO $ runBrowser url)
 		( stop
 		, error $ "failed to start web browser on url " ++ url
 		)
 	where
-		checkpid = do
+		nuke f = void $ liftIO $ catchMaybeIO $ removeFile f
+		checkpid f = do
 			pidfile <- fromRepo gitAnnexPidFile
-			liftIO $ checkDaemon pidfile
-		startassistant = do
+			liftIO $
+				doesFileExist f <&&> (isJust <$> checkDaemon pidfile)
+		startassistant f = do
+			nuke f
 			{- Fork a separate process to run the assistant,
 			 - with a copy of the Annex state. -}
 			state <- Annex.getState id
 			liftIO $ void $ forkProcess $
 				Annex.eval state $ startDaemon True False
-			waitdaemon (100 :: Int)
-		waitdaemon 0 = error "failed to start git-annex assistant"
-		waitdaemon n = do
-			r <- checkpid
-			case r of
-				Just _ -> return ()
-				Nothing -> do
-					-- wait 0.1 seconds before retry
-					liftIO $ threadDelay 100000
-					waitdaemon (n - 1)
+			waitdaemon f (100 :: Int)
+		waitdaemon _ 0 = error "failed to start git-annex assistant"
+		waitdaemon f n = unlessM (checkpid f) $ do
+			-- wait 0.1 seconds before retry
+			liftIO $ threadDelay 100000
+			waitdaemon f (n - 1)

From af31df77a0634ff5fc7cda6dc4531cf3345f0ef8 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Thu, 26 Jul 2012 20:42:37 +0000
Subject: [PATCH 4270/8313] Added a comment

---
 ..._4ce86546d8a135df9cfab46b4612fa0b._comment | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment
new file mode 100644
index 0000000000..b24956b8c7
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_2_4ce86546d8a135df9cfab46b4612fa0b._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 2"
+ date="2012-07-26T20:42:37Z"
+ content="""
+The last few commits fixed the webapp startup quirk, it doesn't work yet for me as the watch command is probably failing
+
+<pre>
+laplace:annex jtang$ git annex webapp  -d
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"show-ref\",\"git-annex\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..731005d121426a38b206c4544da02cdb3b974974\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..d36d8d88847decc2320f0be22892ad94a8abe594\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..57bcddc14d03b61028f7002e2dabcc5181d74f3d\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] read: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"log\",\"refs/heads/git-annex..372aceaf49b60ebe31cc3fe2e52ba4fbe99c134f\",\"--oneline\",\"-n1\"] 
+[2012-07-26 21:41:15 IST] chat: git [\"--git-dir=/Users/jtang/annex/.git\",\"--work-tree=/Users/jtang/annex\",\"cat-file\",\"--batch\"] 
+[2012-07-26 21:41:15 IST] read: git [\"config\",\"--null\",\"--list\"] 
+git-annex: failed to start git-annex assistant
+</pre>
+
+it doesn't segfault or anything, it just states that the assistant command failed to start :P
+"""]]

From 9fd03c65f9ebee437317a21e27afb600d9815209 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 17:56:24 -0400
Subject: [PATCH 4271/8313] webapp now does long polling

The webapp is now a constantly updating clock! I accomplished this amazing
feat using "long polling", with some jquery and a little custom java
script.

There are more modern techniques, but this one works everywhere.
---
 Assistant/Threads/WebApp.hs     | 20 +++++++++++++++++++-
 Utility/Yesod.hs                |  4 ++++
 templates/default-layout.hamlet |  1 -
 templates/longpolling.julius    | 25 +++++++++++++++++++++++++
 templates/poll.hamlet           |  2 ++
 5 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 templates/longpolling.julius
 create mode 100644 templates/poll.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index d663b0cd52..2d78609e85 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -21,9 +21,11 @@ import Git
 import Yesod
 import Yesod.Static
 import Text.Hamlet
+import Text.Julius
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text
+import Data.Time.Clock
 
 thisThread :: String
 thisThread = "WebApp"
@@ -39,6 +41,7 @@ staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
 / HomeR GET
+/poll PollR GET
 /config ConfigR GET
 /static StaticR Static getStatic
 |]
@@ -61,10 +64,25 @@ instance Yesod WebApp where
 			excludeStatic (p:_) = p /= "static"
 
 	makeSessionBackend = webAppSessionBackend
+	jsLoader _ = BottomOfHeadBlocking
 
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
-	[whamlet|Hello, World<p><a href="@{ConfigR}">config|]
+	[whamlet|<div id="poll">Starting ...|]
+	addScriptRemote "http://ajax.googleapis.com/ajax/libs/jquery/1.6.2/jquery.min.js"
+	toWidgetBody $(juliusFile $ juliusTemplate "longpolling")
+	[whamlet|<p><a href="@{ConfigR}">config|]
+
+{- Called by client to poll for a new webapp status display.
+ -
+ - Should block until the status has changed, and then return a div
+ - containing the new status, which will be inserted into the calling page.
+ -}
+getPollR :: Handler RepHtml
+getPollR = do
+	webapp <- getYesod
+	time <- show <$> liftIO getCurrentTime
+	hamletToRepHtml $(hamletFile $ hamletTemplate "poll")
 
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout $ do
diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs
index 05f684490a..a0dd3bdd2f 100644
--- a/Utility/Yesod.hs
+++ b/Utility/Yesod.hs
@@ -16,3 +16,7 @@ template f = "templates" </> f
 {- A hamlet template file. -}
 hamletTemplate :: FilePath -> FilePath
 hamletTemplate f = template f ++ ".hamlet"
+
+{- A julius template file. -}
+juliusTemplate :: FilePath -> FilePath
+juliusTemplate f = template f ++ ".julius"
diff --git a/templates/default-layout.hamlet b/templates/default-layout.hamlet
index e07addc8e0..bd16969f93 100644
--- a/templates/default-layout.hamlet
+++ b/templates/default-layout.hamlet
@@ -3,7 +3,6 @@ $doctype 5
   <head>
     <title>#{baseTitle webapp} #{pageTitle page}
     <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
-
     ^{pageHead page}
   <body>
       $maybe msg <- mmsg
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
new file mode 100644
index 0000000000..38ecbc77d2
--- /dev/null
+++ b/templates/longpolling.julius
@@ -0,0 +1,25 @@
+// Uses long-polling to update a div with id=poll.
+// The PollR route should return a new div, also with id=poll.
+
+(function( $ ) {
+
+$.LongPoll = (function() {
+	return {
+		send : function() {           
+			$.ajax({
+				'url': '@{PollR}',
+				'dataType': 'html',
+				'success': function(data, status, jqxhr) {
+					$('#poll').replaceWith(data);
+					setTimeout($.LongPoll.send, 3000);
+				},
+			});
+		}
+	}
+}());
+
+$(document).bind('ready.app', function() {
+	setTimeout($.LongPoll.send, 40);
+});
+
+})( jQuery );
diff --git a/templates/poll.hamlet b/templates/poll.hamlet
new file mode 100644
index 0000000000..fcdd705b65
--- /dev/null
+++ b/templates/poll.hamlet
@@ -0,0 +1,2 @@
+<div id="poll">
+  polled at #{time}

From ef31dd08eebcbe506ae24cd2be7d3ab4d01a8d11 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 17:58:44 -0400
Subject: [PATCH 4272/8313] update

---
 doc/design/assistant/webapp.mdwn | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index fe910c1976..fda70cbdca 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -31,12 +31,15 @@ The webapp is a web server that displays a shiny interface.
 
 ## implementation
 
+* Include jquery into the webapp, preferably minimised at build time.
+  Currently the webapp needs an internet connection to load jquery, which
+  is not ideal.
 * use `addStaticContent` to make /favicon.ico work. Return `Right (route, query)`
-  and I think the route can be `favicon_ico`.
+  and I think the route can be `favicon_ico`. (Tried this; couldn't seem
+  to make it work.)
 * perhaps define a custom `errorHandler`, which could avoid the potential
   of leaking auth tokens on error pages
 * possibly lose the ugly auth= token past the first page,
   and use a client-side session. It could be encrypted using the token
   as the `encryptKey`. Note: Would need to set the session duration
   to infinite (how?)
-* look up "server-sent events" sent using `sendWaiResponse`

From 9b2eec2e7af34e107bcb438a501286996fe0eb41 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 18:04:09 -0400
Subject: [PATCH 4273/8313] increase timeout from 10 to 100 seconds

I've seen 10 be too short under load.
---
 Command/WebApp.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 5fcaad6fda..7d0a310d40 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -56,7 +56,7 @@ start restart = notBareRepo $ do
 			state <- Annex.getState id
 			liftIO $ void $ forkProcess $
 				Annex.eval state $ startDaemon True False
-			waitdaemon f (100 :: Int)
+			waitdaemon f (1000 :: Int)
 		waitdaemon _ 0 = error "failed to start git-annex assistant"
 		waitdaemon f n = unlessM (checkpid f) $ do
 			-- wait 0.1 seconds before retry

From f1cbc7d72580a0ec4dab9c7cc8d741710313027f Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 22:07:25 +0000
Subject: [PATCH 4274/8313] Added a comment

---
 ...mment_3_6d398a2cceff14a1b774b85ee1725073._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment
new file mode 100644
index 0000000000..a139537def
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_3_6d398a2cceff14a1b774b85ee1725073._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 3"
+ date="2012-07-26T22:07:25Z"
+ content="""
+I've been improving this code path, so make sure you're current. I just made it wait for longer than the 10 seconds it was waiting before giving up. 
+
+If it still fails, try running `git-annex assistant --debug --foreground`
+in one terminal, wait for it to start up, and then run `git annex webapp` in another.
+If that still fails, check if `.git/annex/webapp.html` exists.
+"""]]

From 576224430a22cc3bf3c86e6626e391a973d8001c Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Thu, 26 Jul 2012 22:08:53 +0000
Subject: [PATCH 4275/8313] Added a comment

---
 .../comment_4_5e503787a4b1d3534c5e20da5480b763._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment
new file mode 100644
index 0000000000..fde217e63c
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_4_5e503787a4b1d3534c5e20da5480b763._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.189"
+ subject="comment 4"
+ date="2012-07-26T22:08:53Z"
+ content="""
+Oh, wait, you're right, if the assistant process that watch forks off is segfaulting, that error message is about the best we can hope for. You can try the manual startup of the assistant to confirm it's crashing..
+"""]]

From f5ef46d01eb7bbaac45eec162267bcbf2500d511 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 21:03:46 -0400
Subject: [PATCH 4276/8313] cleaned up refreshing code into a widget

Very happy to have a reusable autoUpdate widget that can make any Yesod
widget automatically refresh!

Also added support for non-javascript browsers, falling back to meta
refresh.

Also, the home page is now rendered with the webapp status on it, before
any refreshing is done.
---
 Assistant/Threads/WebApp.hs              | 58 ++++++++++++++++++++----
 templates/longpolling.julius             | 16 ++++---
 templates/metarefresh.hamlet             |  2 +
 templates/{poll.hamlet => status.hamlet} |  2 +-
 4 files changed, 62 insertions(+), 16 deletions(-)
 create mode 100644 templates/metarefresh.hamlet
 rename templates/{poll.hamlet => status.hamlet} (51%)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 2d78609e85..0e1f9ba95f 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -41,7 +41,7 @@ staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
 / HomeR GET
-/poll PollR GET
+/status StatusR GET
 /config ConfigR GET
 /static StaticR Static getStatic
 |]
@@ -66,23 +66,63 @@ instance Yesod WebApp where
 	makeSessionBackend = webAppSessionBackend
 	jsLoader _ = BottomOfHeadBlocking
 
+{- Add to any widget to make it auto-update.
+ -
+ - The widget should have a html element with id=poll, which will be
+ - replaced when it's updated.
+ - 
+ - Updating is done by getting html from the gethtml route.
+ - Or, the home route is used if the whole page has to be refreshed to
+ - update.
+ -
+ - ms_delay is how long to delay between updates.
+ - ms_startdelay is how long to delay before updating the widget at the
+ - state.
+ -}
+autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Widget
+autoUpdate poll gethtml home ms_delay ms_startdelay = do
+	{- Fallback refreshing is provided for non-javascript browsers. -}
+	let delayseconds = show $ ms_to_seconds ms_delay
+	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
+
+	{- Use long polling to update the status display. -}
+	let delay = show ms_delay
+	let startdelay = show ms_startdelay
+	addScriptRemote "http://ajax.googleapis.com/ajax/libs/jquery/1.6.2/jquery.min.js"
+	toWidgetHead $(juliusFile $ juliusTemplate "longpolling")
+	where
+		ms_to_seconds :: Int -> Int
+		ms_to_seconds ms = ceiling ((fromIntegral ms :: Double) / 1000)
+
+{- Continually updating status display. -}
+statusDisplay :: Widget
+statusDisplay = do
+	webapp <- lift getYesod
+	time <- show <$> liftIO getCurrentTime
+	
+	poll <- lift newIdent
+	$(whamletFile $ hamletTemplate "status")
+	
+	autoUpdate poll StatusR HomeR (3000 :: Int) (40 :: Int)
+
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
-	[whamlet|<div id="poll">Starting ...|]
-	addScriptRemote "http://ajax.googleapis.com/ajax/libs/jquery/1.6.2/jquery.min.js"
-	toWidgetBody $(juliusFile $ juliusTemplate "longpolling")
+	statusDisplay
 	[whamlet|<p><a href="@{ConfigR}">config|]
 
 {- Called by client to poll for a new webapp status display.
  -
  - Should block until the status has changed, and then return a div
  - containing the new status, which will be inserted into the calling page.
+ -
+ - Note that the head of the widget is not included, only its
+ - body is. To get the widget head content, the widget is also 
+ - inserted onto the getHomeR page.
  -}
-getPollR :: Handler RepHtml
-getPollR = do
-	webapp <- getYesod
-	time <- show <$> liftIO getCurrentTime
-	hamletToRepHtml $(hamletFile $ hamletTemplate "poll")
+getStatusR :: Handler RepHtml
+getStatusR = do
+	page <- widgetToPageContent statusDisplay
+	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout $ do
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 38ecbc77d2..26356f5e93 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,5 +1,9 @@
-// Uses long-polling to update a div with id=poll.
-// The PollR route should return a new div, also with id=poll.
+
+// Uses long-polling to update a div with id=#{poll}
+// The gethtml route should return a new div, with the same id.
+//
+// Maximum update frequency is controlled by #{startdelay}
+// and #{delay}, both in milliseconds.
 
 (function( $ ) {
 
@@ -7,11 +11,11 @@ $.LongPoll = (function() {
 	return {
 		send : function() {           
 			$.ajax({
-				'url': '@{PollR}',
+				'url': '@{gethtml}',
 				'dataType': 'html',
 				'success': function(data, status, jqxhr) {
-					$('#poll').replaceWith(data);
-					setTimeout($.LongPoll.send, 3000);
+					$('##{poll}').replaceWith(data);
+					setTimeout($.LongPoll.send, #{delay});
 				},
 			});
 		}
@@ -19,7 +23,7 @@ $.LongPoll = (function() {
 }());
 
 $(document).bind('ready.app', function() {
-	setTimeout($.LongPoll.send, 40);
+	setTimeout($.LongPoll.send, #{startdelay});
 });
 
 })( jQuery );
diff --git a/templates/metarefresh.hamlet b/templates/metarefresh.hamlet
new file mode 100644
index 0000000000..be22aa8992
--- /dev/null
+++ b/templates/metarefresh.hamlet
@@ -0,0 +1,2 @@
+<noscript>
+  <meta http-equiv="refresh" content="#{delayseconds}; URL=@{home}">
diff --git a/templates/poll.hamlet b/templates/status.hamlet
similarity index 51%
rename from templates/poll.hamlet
rename to templates/status.hamlet
index fcdd705b65..1f975b35f4 100644
--- a/templates/poll.hamlet
+++ b/templates/status.hamlet
@@ -1,2 +1,2 @@
-<div id="poll">
+<div id="#{poll}">
   polled at #{time}

From 1d5971fa7e8b716ed0eaddec698b3a28e37972a6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 21:32:08 -0400
Subject: [PATCH 4277/8313] update

---
 doc/design/assistant/webapp.mdwn | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index fda70cbdca..732d517cc0 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -27,7 +27,8 @@ The webapp is a web server that displays a shiny interface.
 * there could be a UI to export a file, which would make it be served up
   over http by the web app
 * Display any relevant warning messages. One is the `inotify max_user_watches`
-  exceeded message.
+  exceeded message. Need to lift such messages into DaemonStatus
+  so the WebApp can include them in its rendering of DaemonStatus.
 
 ## implementation
 

From 94526aef899fc312fd922341b50a46353764a3c2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 21:40:24 -0400
Subject: [PATCH 4278/8313] todo

---
 doc/design/assistant/webapp.mdwn | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 732d517cc0..439e013c19 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -44,3 +44,6 @@ The webapp is a web server that displays a shiny interface.
   and use a client-side session. It could be encrypted using the token
   as the `encryptKey`. Note: Would need to set the session duration
   to infinite (how?)
+* When long polling fails, retry a time or two, and then give up, and
+  either display an error message, or, possibly, close the browser window.
+  (Currently the display just stops updating.)

From 77c3bf7f887933cf953d5983b849796ea075bfca Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 21:51:47 -0400
Subject: [PATCH 4279/8313] update thread list

---
 Assistant.hs | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index c867529fdf..b539b27bc4 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -10,7 +10,7 @@
  - 	The initial thread run, double forks to background, starts other
  - 	threads, and then stops, waiting for them to terminate,
  - 	or for a ctrl-c.
- - Thread 2: watcher
+ - Thread 2: Watcher
  - 	Notices new files, and calls handlers for events, queuing changes.
  - Thread 3: inotify internal
  - 	Used by haskell inotify library to ensure inotify event buffer is
@@ -19,43 +19,46 @@
  - 	Scans the tree and registers inotify watches for each directory.
  -	A MVar lock is used to prevent other inotify handlers from running
  -	until this is complete.
- - Thread 5: committer
+ - Thread 5: Committer
  - 	Waits for changes to occur, and runs the git queue to update its
  - 	index, then commits. Also queues Transfer events to send added
  - 	files to other remotes.
- - Thread 6: pusher
+ - Thread 6: Pusher
  - 	Waits for commits to be made, and pushes updated branches to remotes,
  - 	in parallel. (Forks a process for each git push.)
- - Thread 7: push retryer
+ - Thread 7: PushRetryer
  - 	Runs every 30 minutes when there are failed pushes, and retries
  - 	them.
- - Thread 8: merger
+ - Thread 8: Merger
  - 	Waits for pushes to be received from remotes, and merges the
  - 	updated branches into the current branch.
  - 	(This uses inotify on .git/refs/heads, so there are additional
  - 	inotify threads associated with it, too.)
- - Thread 9: transfer watcher
+ - Thread 9: TransferWatcher
  - 	Watches for transfer information files being created and removed,
  - 	and maintains the DaemonStatus currentTransfers map.
  - 	(This uses inotify on .git/annex/transfer/, so there are
  - 	additional inotify threads associated with it, too.)
- - Thread 10: transferrer
+ - Thread 10: Transferrer
  - 	Waits for Transfers to be queued and does them.
- - Thread 11: status logger
+ - Thread 11: StatusLogger
  - 	Wakes up periodically and records the daemon's status to disk.
- - Thread 12: sanity checker
+ - Thread 12: SanityChecker
  - 	Wakes up periodically (rarely) and does sanity checks.
- - Thread 13: mount watcher
+ - Thread 13: MountWatcher
  - 	Either uses dbus to watch for drive mount events, or, when
  - 	there's no dbus, polls to find newly mounted filesystems.
  - 	Once a filesystem that contains a remote is mounted, updates
  - 	state about that remote, pulls from it, and queues a push to it,
  - 	as well as an update, and queues it onto the
  - 	ConnectedRemoteChan
- - Thread 14: transfer scanner
+ - Thread 14: TransferScanner
  - 	Does potentially expensive checks to find data that needs to be
  - 	transferred from or to remotes, and queues Transfers.
  - 	Uses the ScanRemotes map.
+ - Thread 15: WebApp
+ - 	Spawns more threads as necessary to handle clients.
+ - 	Displays the DaemonStatus.
  -
  - ThreadState: (MVar)
  - 	The Annex state is stored here, which allows resuscitating the

From e40f94cbcdcacebb8215ee16b5c575ca865ad810 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 21:51:56 -0400
Subject: [PATCH 4280/8313] add threadState member, will need this later to
 access the daemonStatus

---
 Assistant/Threads/WebApp.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 0e1f9ba95f..593c43eae3 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -31,7 +31,8 @@ thisThread :: String
 thisThread = "WebApp"
 
 data WebApp = WebApp
-	{ daemonStatus :: DaemonStatusHandle
+	{ threadState :: ThreadState
+	, daemonStatus :: DaemonStatusHandle
 	, secretToken :: Text
 	, baseTitle :: String
 	, getStatic :: Static
@@ -148,7 +149,8 @@ mkWebApp st dstatus = do
 		else dir
 	token <- genRandomToken 
 	return $ WebApp 
-		{ daemonStatus = dstatus
+		{ threadState = st
+		, daemonStatus = dstatus
 		, secretToken = pack token
 		, baseTitle = reldir
 		, getStatic = $(embed "static")

From 615dc09ffc321ce04d3014b3a9f07e7f04b69c80 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 22:54:31 -0400
Subject: [PATCH 4281/8313] use widgetFile

---
 Assistant/Threads/WebApp.hs |  5 ++---
 Utility/Yesod.hs            | 17 ++++++-----------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 593c43eae3..addb49f79d 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -21,7 +21,6 @@ import Git
 import Yesod
 import Yesod.Static
 import Text.Hamlet
-import Text.Julius
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text
@@ -90,7 +89,7 @@ autoUpdate poll gethtml home ms_delay ms_startdelay = do
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
 	addScriptRemote "http://ajax.googleapis.com/ajax/libs/jquery/1.6.2/jquery.min.js"
-	toWidgetHead $(juliusFile $ juliusTemplate "longpolling")
+	$(widgetFile "longpolling")
 	where
 		ms_to_seconds :: Int -> Int
 		ms_to_seconds ms = ceiling ((fromIntegral ms :: Double) / 1000)
@@ -102,7 +101,7 @@ statusDisplay = do
 	time <- show <$> liftIO getCurrentTime
 	
 	poll <- lift newIdent
-	$(whamletFile $ hamletTemplate "status")
+	$(widgetFile "status")
 	
 	autoUpdate poll StatusR HomeR (3000 :: Int) (40 :: Int)
 
diff --git a/Utility/Yesod.hs b/Utility/Yesod.hs
index a0dd3bdd2f..2d2c6c3436 100644
--- a/Utility/Yesod.hs
+++ b/Utility/Yesod.hs
@@ -1,4 +1,4 @@
-{- Yesod stuff
+{- Yesod stuff, that's typically found in the scaffolded site.
  -
  - Copyright 2012 Joey Hess <joey@kitenet.net>
  -
@@ -7,16 +7,11 @@
 
 module Utility.Yesod where
 
-import System.FilePath
+import Yesod.Default.Util
+import Language.Haskell.TH.Syntax
 
-{- Filename of a template, in the templates/ directory. -}
-template :: FilePath -> FilePath
-template f = "templates" </> f
+widgetFile :: String -> Q Exp
+widgetFile = widgetFileNoReload
 
-{- A hamlet template file. -}
 hamletTemplate :: FilePath -> FilePath
-hamletTemplate f = template f ++ ".hamlet"
-
-{- A julius template file. -}
-juliusTemplate :: FilePath -> FilePath
-juliusTemplate f = template f ++ ".julius"
+hamletTemplate f = globFile "hamlet" f

From ff4ab6d6da60ecf9081f743fc13b5785ebe79d12 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 22:59:14 -0400
Subject: [PATCH 4282/8313] work around GHC not knowing to rebuild files using
 template haskell when things they include change

---
 Makefile | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 4a74e712f1..a6fdab7ca7 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,7 @@ FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP
 bins=git-annex
 mans=git-annex.1 git-annex-shell.1
 sources=Build/SysConfig.hs Utility/Touch.hs Utility/Mounts.hs
+thfiles=Assistant/Threads/WebApp.hs
 all=$(bins) $(mans) docs
 
 OS:=$(shell uname | sed 's/[-_].*//')
@@ -55,8 +56,13 @@ Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs
 %.hs: %.hsc
 	hsc2hs $<
 
+# Force GHC to rebuild template haskell files whenever includes
+# change
+$(thfiles): $(shell echo templates/* static/*)
+$(thfiles):
+	touch $(thfiles)
 
-git-annex: $(sources) $(clibs)
+git-annex: $(sources) $(clibs) $(thfiles)
 	$(GHCMAKE) $@ $(clibs)
 
 git-annex.1: doc/git-annex.mdwn

From 1983ca2852461ca2082504dd22de07638030665b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Thu, 26 Jul 2012 23:55:51 -0400
Subject: [PATCH 4283/8313] added jquery to static site

Had to switch to toWaiAppPlain to avoid a seeming bug in toWaiApp;
chromium only received a partial copy of jquery. Always the same length
each time, which makes me think it's a bug in the compression, although
a bug in the autohead middleware is also a possibility.

Anyway, there's little need for compression for a local webapp. Not wasting
time compressing things is probably a net gain.

Similarly, I've not worried about minifying this yet. Although that would
avoid bloating the git-annex binary quite so much.
---
 Assistant/Threads/WebApp.hs |    4 +-
 debian/copyright            |   28 +-
 static/jquery.full.js       | 8981 +++++++++++++++++++++++++++++++++++
 3 files changed, 9010 insertions(+), 3 deletions(-)
 create mode 100644 static/jquery.full.js

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index addb49f79d..f82a1fb6b9 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -88,7 +88,7 @@ autoUpdate poll gethtml home ms_delay ms_startdelay = do
 	{- Use long polling to update the status display. -}
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
-	addScriptRemote "http://ajax.googleapis.com/ajax/libs/jquery/1.6.2/jquery.min.js"
+	addScript $ StaticR jquery_full_js
 	$(widgetFile "longpolling")
 	where
 		ms_to_seconds :: Int -> Int
@@ -132,7 +132,7 @@ getConfigR = defaultLayout $ do
 webAppThread :: ThreadState -> DaemonStatusHandle -> IO ()
 webAppThread st dstatus = do
 	webapp <- mkWebApp st dstatus
-	app <- toWaiApp webapp
+	app <- toWaiAppPlain webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
diff --git a/debian/copyright b/debian/copyright
index 26a559cc51..72b6333671 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -24,8 +24,34 @@ License: BSD-3-clause
  Utility/libmounts.c in this package's source, or in
  /usr/share/common-licenses/BSD on Debian systems.
 
-Files: doc/logo.png doc/logo_small.png doc/favicon.png
+Files: doc/logo.png doc/logo_small.png */favicon.ico
 Copyright: 2007 Henrik Nyh <http://henrik.nyh.se/>
            2010 Joey Hess <joey@kitenet.net>
 License: other
   Free to modify and redistribute with due credit, and obviously free to use.
+
+Files: static/jquery*
+Copyright: © 2005-2011 by John Resig, Branden Aaron & Jörn Zaefferer
+           © 2011 The Dojo Foundation
+License: MIT or GPL-2
+ The full text of version 2 of the GPL is distributed in
+ /usr/share/common-licenses/GPL-2 on Debian systems.
+ .
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+ .
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/static/jquery.full.js b/static/jquery.full.js
new file mode 100644
index 0000000000..f3201aacb6
--- /dev/null
+++ b/static/jquery.full.js
@@ -0,0 +1,8981 @@
+/*!
+ * jQuery JavaScript Library v1.6.2
+ * http://jquery.com/
+ *
+ * Copyright 2011, John Resig
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * Includes Sizzle.js
+ * http://sizzlejs.com/
+ * Copyright 2011, The Dojo Foundation
+ * Released under the MIT, BSD, and GPL Licenses.
+ *
+ * Date: Thu Jun 30 14:16:56 2011 -0400
+ */
+(function( window, undefined ) {
+
+// Use the correct document accordingly with window argument (sandbox)
+var document = window.document,
+	navigator = window.navigator,
+	location = window.location;
+var jQuery = (function() {
+
+// Define a local copy of jQuery
+var jQuery = function( selector, context ) {
+		// The jQuery object is actually just the init constructor 'enhanced'
+		return new jQuery.fn.init( selector, context, rootjQuery );
+	},
+
+	// Map over jQuery in case of overwrite
+	_jQuery = window.jQuery,
+
+	// Map over the $ in case of overwrite
+	_$ = window.$,
+
+	// A central reference to the root jQuery(document)
+	rootjQuery,
+
+	// A simple way to check for HTML strings or ID strings
+	// (both of which we optimize for)
+	quickExpr = /^(?:[^<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,
+
+	// Check if a string has a non-whitespace character in it
+	rnotwhite = /\S/,
+
+	// Used for trimming whitespace
+	trimLeft = /^\s+/,
+	trimRight = /\s+$/,
+
+	// Check for digits
+	rdigit = /\d/,
+
+	// Match a standalone tag
+	rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>)?$/,
+
+	// JSON RegExp
+	rvalidchars = /^[\],:{}\s]*$/,
+	rvalidescape = /\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,
+	rvalidtokens = /"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,
+	rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g,
+
+	// Useragent RegExp
+	rwebkit = /(webkit)[ \/]([\w.]+)/,
+	ropera = /(opera)(?:.*version)?[ \/]([\w.]+)/,
+	rmsie = /(msie) ([\w.]+)/,
+	rmozilla = /(mozilla)(?:.*? rv:([\w.]+))?/,
+
+	// Matches dashed string for camelizing
+	rdashAlpha = /-([a-z])/ig,
+
+	// Used by jQuery.camelCase as callback to replace()
+	fcamelCase = function( all, letter ) {
+		return letter.toUpperCase();
+	},
+
+	// Keep a UserAgent string for use with jQuery.browser
+	userAgent = navigator.userAgent,
+
+	// For matching the engine and version of the browser
+	browserMatch,
+
+	// The deferred used on DOM ready
+	readyList,
+
+	// The ready event handler
+	DOMContentLoaded,
+
+	// Save a reference to some core methods
+	toString = Object.prototype.toString,
+	hasOwn = Object.prototype.hasOwnProperty,
+	push = Array.prototype.push,
+	slice = Array.prototype.slice,
+	trim = String.prototype.trim,
+	indexOf = Array.prototype.indexOf,
+
+	// [[Class]] -> type pairs
+	class2type = {};
+
+jQuery.fn = jQuery.prototype = {
+	constructor: jQuery,
+	init: function( selector, context, rootjQuery ) {
+		var match, elem, ret, doc;
+
+		// Handle $(""), $(null), or $(undefined)
+		if ( !selector ) {
+			return this;
+		}
+
+		// Handle $(DOMElement)
+		if ( selector.nodeType ) {
+			this.context = this[0] = selector;
+			this.length = 1;
+			return this;
+		}
+
+		// The body element only exists once, optimize finding it
+		if ( selector === "body" && !context && document.body ) {
+			this.context = document;
+			this[0] = document.body;
+			this.selector = selector;
+			this.length = 1;
+			return this;
+		}
+
+		// Handle HTML strings
+		if ( typeof selector === "string" ) {
+			// Are we dealing with HTML string or an ID?
+			if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) {
+				// Assume that strings that start and end with <> are HTML and skip the regex check
+				match = [ null, selector, null ];
+
+			} else {
+				match = quickExpr.exec( selector );
+			}
+
+			// Verify a match, and that no context was specified for #id
+			if ( match && (match[1] || !context) ) {
+
+				// HANDLE: $(html) -> $(array)
+				if ( match[1] ) {
+					context = context instanceof jQuery ? context[0] : context;
+					doc = (context ? context.ownerDocument || context : document);
+
+					// If a single string is passed in and it's a single tag
+					// just do a createElement and skip the rest
+					ret = rsingleTag.exec( selector );
+
+					if ( ret ) {
+						if ( jQuery.isPlainObject( context ) ) {
+							selector = [ document.createElement( ret[1] ) ];
+							jQuery.fn.attr.call( selector, context, true );
+
+						} else {
+							selector = [ doc.createElement( ret[1] ) ];
+						}
+
+					} else {
+						ret = jQuery.buildFragment( [ match[1] ], [ doc ] );
+						selector = (ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment).childNodes;
+					}
+
+					return jQuery.merge( this, selector );
+
+				// HANDLE: $("#id")
+				} else {
+					elem = document.getElementById( match[2] );
+
+					// Check parentNode to catch when Blackberry 4.6 returns
+					// nodes that are no longer in the document #6963
+					if ( elem && elem.parentNode ) {
+						// Handle the case where IE and Opera return items
+						// by name instead of ID
+						if ( elem.id !== match[2] ) {
+							return rootjQuery.find( selector );
+						}
+
+						// Otherwise, we inject the element directly into the jQuery object
+						this.length = 1;
+						this[0] = elem;
+					}
+
+					this.context = document;
+					this.selector = selector;
+					return this;
+				}
+
+			// HANDLE: $(expr, $(...))
+			} else if ( !context || context.jquery ) {
+				return (context || rootjQuery).find( selector );
+
+			// HANDLE: $(expr, context)
+			// (which is just equivalent to: $(context).find(expr)
+			} else {
+				return this.constructor( context ).find( selector );
+			}
+
+		// HANDLE: $(function)
+		// Shortcut for document ready
+		} else if ( jQuery.isFunction( selector ) ) {
+			return rootjQuery.ready( selector );
+		}
+
+		if (selector.selector !== undefined) {
+			this.selector = selector.selector;
+			this.context = selector.context;
+		}
+
+		return jQuery.makeArray( selector, this );
+	},
+
+	// Start with an empty selector
+	selector: "",
+
+	// The current version of jQuery being used
+	jquery: "1.6.2",
+
+	// The default length of a jQuery object is 0
+	length: 0,
+
+	// The number of elements contained in the matched element set
+	size: function() {
+		return this.length;
+	},
+
+	toArray: function() {
+		return slice.call( this, 0 );
+	},
+
+	// Get the Nth element in the matched element set OR
+	// Get the whole matched element set as a clean array
+	get: function( num ) {
+		return num == null ?
+
+			// Return a 'clean' array
+			this.toArray() :
+
+			// Return just the object
+			( num < 0 ? this[ this.length + num ] : this[ num ] );
+	},
+
+	// Take an array of elements and push it onto the stack
+	// (returning the new matched element set)
+	pushStack: function( elems, name, selector ) {
+		// Build a new jQuery matched element set
+		var ret = this.constructor();
+
+		if ( jQuery.isArray( elems ) ) {
+			push.apply( ret, elems );
+
+		} else {
+			jQuery.merge( ret, elems );
+		}
+
+		// Add the old object onto the stack (as a reference)
+		ret.prevObject = this;
+
+		ret.context = this.context;
+
+		if ( name === "find" ) {
+			ret.selector = this.selector + (this.selector ? " " : "") + selector;
+		} else if ( name ) {
+			ret.selector = this.selector + "." + name + "(" + selector + ")";
+		}
+
+		// Return the newly-formed element set
+		return ret;
+	},
+
+	// Execute a callback for every element in the matched set.
+	// (You can seed the arguments with an array of args, but this is
+	// only used internally.)
+	each: function( callback, args ) {
+		return jQuery.each( this, callback, args );
+	},
+
+	ready: function( fn ) {
+		// Attach the listeners
+		jQuery.bindReady();
+
+		// Add the callback
+		readyList.done( fn );
+
+		return this;
+	},
+
+	eq: function( i ) {
+		return i === -1 ?
+			this.slice( i ) :
+			this.slice( i, +i + 1 );
+	},
+
+	first: function() {
+		return this.eq( 0 );
+	},
+
+	last: function() {
+		return this.eq( -1 );
+	},
+
+	slice: function() {
+		return this.pushStack( slice.apply( this, arguments ),
+			"slice", slice.call(arguments).join(",") );
+	},
+
+	map: function( callback ) {
+		return this.pushStack( jQuery.map(this, function( elem, i ) {
+			return callback.call( elem, i, elem );
+		}));
+	},
+
+	end: function() {
+		return this.prevObject || this.constructor(null);
+	},
+
+	// For internal use only.
+	// Behaves like an Array's method, not like a jQuery method.
+	push: push,
+	sort: [].sort,
+	splice: [].splice
+};
+
+// Give the init function the jQuery prototype for later instantiation
+jQuery.fn.init.prototype = jQuery.fn;
+
+jQuery.extend = jQuery.fn.extend = function() {
+	var options, name, src, copy, copyIsArray, clone,
+		target = arguments[0] || {},
+		i = 1,
+		length = arguments.length,
+		deep = false;
+
+	// Handle a deep copy situation
+	if ( typeof target === "boolean" ) {
+		deep = target;
+		target = arguments[1] || {};
+		// skip the boolean and the target
+		i = 2;
+	}
+
+	// Handle case when target is a string or something (possible in deep copy)
+	if ( typeof target !== "object" && !jQuery.isFunction(target) ) {
+		target = {};
+	}
+
+	// extend jQuery itself if only one argument is passed
+	if ( length === i ) {
+		target = this;
+		--i;
+	}
+
+	for ( ; i < length; i++ ) {
+		// Only deal with non-null/undefined values
+		if ( (options = arguments[ i ]) != null ) {
+			// Extend the base object
+			for ( name in options ) {
+				src = target[ name ];
+				copy = options[ name ];
+
+				// Prevent never-ending loop
+				if ( target === copy ) {
+					continue;
+				}
+
+				// Recurse if we're merging plain objects or arrays
+				if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) {
+					if ( copyIsArray ) {
+						copyIsArray = false;
+						clone = src && jQuery.isArray(src) ? src : [];
+
+					} else {
+						clone = src && jQuery.isPlainObject(src) ? src : {};
+					}
+
+					// Never move original objects, clone them
+					target[ name ] = jQuery.extend( deep, clone, copy );
+
+				// Don't bring in undefined values
+				} else if ( copy !== undefined ) {
+					target[ name ] = copy;
+				}
+			}
+		}
+	}
+
+	// Return the modified object
+	return target;
+};
+
+jQuery.extend({
+	noConflict: function( deep ) {
+		if ( window.$ === jQuery ) {
+			window.$ = _$;
+		}
+
+		if ( deep && window.jQuery === jQuery ) {
+			window.jQuery = _jQuery;
+		}
+
+		return jQuery;
+	},
+
+	// Is the DOM ready to be used? Set to true once it occurs.
+	isReady: false,
+
+	// A counter to track how many items to wait for before
+	// the ready event fires. See #6781
+	readyWait: 1,
+
+	// Hold (or release) the ready event
+	holdReady: function( hold ) {
+		if ( hold ) {
+			jQuery.readyWait++;
+		} else {
+			jQuery.ready( true );
+		}
+	},
+
+	// Handle when the DOM is ready
+	ready: function( wait ) {
+		// Either a released hold or an DOMready/load event and not yet ready
+		if ( (wait === true && !--jQuery.readyWait) || (wait !== true && !jQuery.isReady) ) {
+			// Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
+			if ( !document.body ) {
+				return setTimeout( jQuery.ready, 1 );
+			}
+
+			// Remember that the DOM is ready
+			jQuery.isReady = true;
+
+			// If a normal DOM Ready event fired, decrement, and wait if need be
+			if ( wait !== true && --jQuery.readyWait > 0 ) {
+				return;
+			}
+
+			// If there are functions bound, to execute
+			readyList.resolveWith( document, [ jQuery ] );
+
+			// Trigger any bound ready events
+			if ( jQuery.fn.trigger ) {
+				jQuery( document ).trigger( "ready" ).unbind( "ready" );
+			}
+		}
+	},
+
+	bindReady: function() {
+		if ( readyList ) {
+			return;
+		}
+
+		readyList = jQuery._Deferred();
+
+		// Catch cases where $(document).ready() is called after the
+		// browser event has already occurred.
+		if ( document.readyState === "complete" ) {
+			// Handle it asynchronously to allow scripts the opportunity to delay ready
+			return setTimeout( jQuery.ready, 1 );
+		}
+
+		// Mozilla, Opera and webkit nightlies currently support this event
+		if ( document.addEventListener ) {
+			// Use the handy event callback
+			document.addEventListener( "DOMContentLoaded", DOMContentLoaded, false );
+
+			// A fallback to window.onload, that will always work
+			window.addEventListener( "load", jQuery.ready, false );
+
+		// If IE event model is used
+		} else if ( document.attachEvent ) {
+			// ensure firing before onload,
+			// maybe late but safe also for iframes
+			document.attachEvent( "onreadystatechange", DOMContentLoaded );
+
+			// A fallback to window.onload, that will always work
+			window.attachEvent( "onload", jQuery.ready );
+
+			// If IE and not a frame
+			// continually check to see if the document is ready
+			var toplevel = false;
+
+			try {
+				toplevel = window.frameElement == null;
+			} catch(e) {}
+
+			if ( document.documentElement.doScroll && toplevel ) {
+				doScrollCheck();
+			}
+		}
+	},
+
+	// See test/unit/core.js for details concerning isFunction.
+	// Since version 1.3, DOM methods and functions like alert
+	// aren't supported. They return false on IE (#2968).
+	isFunction: function( obj ) {
+		return jQuery.type(obj) === "function";
+	},
+
+	isArray: Array.isArray || function( obj ) {
+		return jQuery.type(obj) === "array";
+	},
+
+	// A crude way of determining if an object is a window
+	isWindow: function( obj ) {
+		return obj && typeof obj === "object" && "setInterval" in obj;
+	},
+
+	isNaN: function( obj ) {
+		return obj == null || !rdigit.test( obj ) || isNaN( obj );
+	},
+
+	type: function( obj ) {
+		return obj == null ?
+			String( obj ) :
+			class2type[ toString.call(obj) ] || "object";
+	},
+
+	isPlainObject: function( obj ) {
+		// Must be an Object.
+		// Because of IE, we also have to check the presence of the constructor property.
+		// Make sure that DOM nodes and window objects don't pass through, as well
+		if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) {
+			return false;
+		}
+
+		// Not own constructor property must be Object
+		if ( obj.constructor &&
+			!hasOwn.call(obj, "constructor") &&
+			!hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) {
+			return false;
+		}
+
+		// Own properties are enumerated firstly, so to speed up,
+		// if last one is own, then all properties are own.
+
+		var key;
+		for ( key in obj ) {}
+
+		return key === undefined || hasOwn.call( obj, key );
+	},
+
+	isEmptyObject: function( obj ) {
+		for ( var name in obj ) {
+			return false;
+		}
+		return true;
+	},
+
+	error: function( msg ) {
+		throw msg;
+	},
+
+	parseJSON: function( data ) {
+		if ( typeof data !== "string" || !data ) {
+			return null;
+		}
+
+		// Make sure leading/trailing whitespace is removed (IE can't handle it)
+		data = jQuery.trim( data );
+
+		// Attempt to parse using the native JSON parser first
+		if ( window.JSON && window.JSON.parse ) {
+			return window.JSON.parse( data );
+		}
+
+		// Make sure the incoming data is actual JSON
+		// Logic borrowed from http://json.org/json2.js
+		if ( rvalidchars.test( data.replace( rvalidescape, "@" )
+			.replace( rvalidtokens, "]" )
+			.replace( rvalidbraces, "")) ) {
+
+			return (new Function( "return " + data ))();
+
+		}
+		jQuery.error( "Invalid JSON: " + data );
+	},
+
+	// Cross-browser xml parsing
+	// (xml & tmp used internally)
+	parseXML: function( data , xml , tmp ) {
+
+		if ( window.DOMParser ) { // Standard
+			tmp = new DOMParser();
+			xml = tmp.parseFromString( data , "text/xml" );
+		} else { // IE
+			xml = new ActiveXObject( "Microsoft.XMLDOM" );
+			xml.async = "false";
+			xml.loadXML( data );
+		}
+
+		tmp = xml.documentElement;
+
+		if ( ! tmp || ! tmp.nodeName || tmp.nodeName === "parsererror" ) {
+			jQuery.error( "Invalid XML: " + data );
+		}
+
+		return xml;
+	},
+
+	noop: function() {},
+
+	// Evaluates a script in a global context
+	// Workarounds based on findings by Jim Driscoll
+	// http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context
+	globalEval: function( data ) {
+		if ( data && rnotwhite.test( data ) ) {
+			// We use execScript on Internet Explorer
+			// We use an anonymous function so that context is window
+			// rather than jQuery in Firefox
+			( window.execScript || function( data ) {
+				window[ "eval" ].call( window, data );
+			} )( data );
+		}
+	},
+
+	// Converts a dashed string to camelCased string;
+	// Used by both the css and data modules
+	camelCase: function( string ) {
+		return string.replace( rdashAlpha, fcamelCase );
+	},
+
+	nodeName: function( elem, name ) {
+		return elem.nodeName && elem.nodeName.toUpperCase() === name.toUpperCase();
+	},
+
+	// args is for internal usage only
+	each: function( object, callback, args ) {
+		var name, i = 0,
+			length = object.length,
+			isObj = length === undefined || jQuery.isFunction( object );
+
+		if ( args ) {
+			if ( isObj ) {
+				for ( name in object ) {
+					if ( callback.apply( object[ name ], args ) === false ) {
+						break;
+					}
+				}
+			} else {
+				for ( ; i < length; ) {
+					if ( callback.apply( object[ i++ ], args ) === false ) {
+						break;
+					}
+				}
+			}
+
+		// A special, fast, case for the most common use of each
+		} else {
+			if ( isObj ) {
+				for ( name in object ) {
+					if ( callback.call( object[ name ], name, object[ name ] ) === false ) {
+						break;
+					}
+				}
+			} else {
+				for ( ; i < length; ) {
+					if ( callback.call( object[ i ], i, object[ i++ ] ) === false ) {
+						break;
+					}
+				}
+			}
+		}
+
+		return object;
+	},
+
+	// Use native String.trim function wherever possible
+	trim: trim ?
+		function( text ) {
+			return text == null ?
+				"" :
+				trim.call( text );
+		} :
+
+		// Otherwise use our own trimming functionality
+		function( text ) {
+			return text == null ?
+				"" :
+				text.toString().replace( trimLeft, "" ).replace( trimRight, "" );
+		},
+
+	// results is for internal usage only
+	makeArray: function( array, results ) {
+		var ret = results || [];
+
+		if ( array != null ) {
+			// The window, strings (and functions) also have 'length'
+			// The extra typeof function check is to prevent crashes
+			// in Safari 2 (See: #3039)
+			// Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930
+			var type = jQuery.type( array );
+
+			if ( array.length == null || type === "string" || type === "function" || type === "regexp" || jQuery.isWindow( array ) ) {
+				push.call( ret, array );
+			} else {
+				jQuery.merge( ret, array );
+			}
+		}
+
+		return ret;
+	},
+
+	inArray: function( elem, array ) {
+
+		if ( indexOf ) {
+			return indexOf.call( array, elem );
+		}
+
+		for ( var i = 0, length = array.length; i < length; i++ ) {
+			if ( array[ i ] === elem ) {
+				return i;
+			}
+		}
+
+		return -1;
+	},
+
+	merge: function( first, second ) {
+		var i = first.length,
+			j = 0;
+
+		if ( typeof second.length === "number" ) {
+			for ( var l = second.length; j < l; j++ ) {
+				first[ i++ ] = second[ j ];
+			}
+
+		} else {
+			while ( second[j] !== undefined ) {
+				first[ i++ ] = second[ j++ ];
+			}
+		}
+
+		first.length = i;
+
+		return first;
+	},
+
+	grep: function( elems, callback, inv ) {
+		var ret = [], retVal;
+		inv = !!inv;
+
+		// Go through the array, only saving the items
+		// that pass the validator function
+		for ( var i = 0, length = elems.length; i < length; i++ ) {
+			retVal = !!callback( elems[ i ], i );
+			if ( inv !== retVal ) {
+				ret.push( elems[ i ] );
+			}
+		}
+
+		return ret;
+	},
+
+	// arg is for internal usage only
+	map: function( elems, callback, arg ) {
+		var value, key, ret = [],
+			i = 0,
+			length = elems.length,
+			// jquery objects are treated as arrays
+			isArray = elems instanceof jQuery || length !== undefined && typeof length === "number" && ( ( length > 0 && elems[ 0 ] && elems[ length -1 ] ) || length === 0 || jQuery.isArray( elems ) ) ;
+
+		// Go through the array, translating each of the items to their
+		if ( isArray ) {
+			for ( ; i < length; i++ ) {
+				value = callback( elems[ i ], i, arg );
+
+				if ( value != null ) {
+					ret[ ret.length ] = value;
+				}
+			}
+
+		// Go through every key on the object,
+		} else {
+			for ( key in elems ) {
+				value = callback( elems[ key ], key, arg );
+
+				if ( value != null ) {
+					ret[ ret.length ] = value;
+				}
+			}
+		}
+
+		// Flatten any nested arrays
+		return ret.concat.apply( [], ret );
+	},
+
+	// A global GUID counter for objects
+	guid: 1,
+
+	// Bind a function to a context, optionally partially applying any
+	// arguments.
+	proxy: function( fn, context ) {
+		if ( typeof context === "string" ) {
+			var tmp = fn[ context ];
+			context = fn;
+			fn = tmp;
+		}
+
+		// Quick check to determine if target is callable, in the spec
+		// this throws a TypeError, but we will just return undefined.
+		if ( !jQuery.isFunction( fn ) ) {
+			return undefined;
+		}
+
+		// Simulated bind
+		var args = slice.call( arguments, 2 ),
+			proxy = function() {
+				return fn.apply( context, args.concat( slice.call( arguments ) ) );
+			};
+
+		// Set the guid of unique handler to the same of original handler, so it can be removed
+		proxy.guid = fn.guid = fn.guid || proxy.guid || jQuery.guid++;
+
+		return proxy;
+	},
+
+	// Mutifunctional method to get and set values to a collection
+	// The value/s can optionally be executed if it's a function
+	access: function( elems, key, value, exec, fn, pass ) {
+		var length = elems.length;
+
+		// Setting many attributes
+		if ( typeof key === "object" ) {
+			for ( var k in key ) {
+				jQuery.access( elems, k, key[k], exec, fn, value );
+			}
+			return elems;
+		}
+
+		// Setting one attribute
+		if ( value !== undefined ) {
+			// Optionally, function values get executed if exec is true
+			exec = !pass && exec && jQuery.isFunction(value);
+
+			for ( var i = 0; i < length; i++ ) {
+				fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass );
+			}
+
+			return elems;
+		}
+
+		// Getting an attribute
+		return length ? fn( elems[0], key ) : undefined;
+	},
+
+	now: function() {
+		return (new Date()).getTime();
+	},
+
+	// Use of jQuery.browser is frowned upon.
+	// More details: http://docs.jquery.com/Utilities/jQuery.browser
+	uaMatch: function( ua ) {
+		ua = ua.toLowerCase();
+
+		var match = rwebkit.exec( ua ) ||
+			ropera.exec( ua ) ||
+			rmsie.exec( ua ) ||
+			ua.indexOf("compatible") < 0 && rmozilla.exec( ua ) ||
+			[];
+
+		return { browser: match[1] || "", version: match[2] || "0" };
+	},
+
+	sub: function() {
+		function jQuerySub( selector, context ) {
+			return new jQuerySub.fn.init( selector, context );
+		}
+		jQuery.extend( true, jQuerySub, this );
+		jQuerySub.superclass = this;
+		jQuerySub.fn = jQuerySub.prototype = this();
+		jQuerySub.fn.constructor = jQuerySub;
+		jQuerySub.sub = this.sub;
+		jQuerySub.fn.init = function init( selector, context ) {
+			if ( context && context instanceof jQuery && !(context instanceof jQuerySub) ) {
+				context = jQuerySub( context );
+			}
+
+			return jQuery.fn.init.call( this, selector, context, rootjQuerySub );
+		};
+		jQuerySub.fn.init.prototype = jQuerySub.fn;
+		var rootjQuerySub = jQuerySub(document);
+		return jQuerySub;
+	},
+
+	browser: {}
+});
+
+// Populate the class2type map
+jQuery.each("Boolean Number String Function Array Date RegExp Object".split(" "), function(i, name) {
+	class2type[ "[object " + name + "]" ] = name.toLowerCase();
+});
+
+browserMatch = jQuery.uaMatch( userAgent );
+if ( browserMatch.browser ) {
+	jQuery.browser[ browserMatch.browser ] = true;
+	jQuery.browser.version = browserMatch.version;
+}
+
+// Deprecated, use jQuery.browser.webkit instead
+if ( jQuery.browser.webkit ) {
+	jQuery.browser.safari = true;
+}
+
+// IE doesn't match non-breaking spaces with \s
+if ( rnotwhite.test( "\xA0" ) ) {
+	trimLeft = /^[\s\xA0]+/;
+	trimRight = /[\s\xA0]+$/;
+}
+
+// All jQuery objects should point back to these
+rootjQuery = jQuery(document);
+
+// Cleanup functions for the document ready method
+if ( document.addEventListener ) {
+	DOMContentLoaded = function() {
+		document.removeEventListener( "DOMContentLoaded", DOMContentLoaded, false );
+		jQuery.ready();
+	};
+
+} else if ( document.attachEvent ) {
+	DOMContentLoaded = function() {
+		// Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443).
+		if ( document.readyState === "complete" ) {
+			document.detachEvent( "onreadystatechange", DOMContentLoaded );
+			jQuery.ready();
+		}
+	};
+}
+
+// The DOM ready check for Internet Explorer
+function doScrollCheck() {
+	if ( jQuery.isReady ) {
+		return;
+	}
+
+	try {
+		// If IE is used, use the trick by Diego Perini
+		// http://javascript.nwbox.com/IEContentLoaded/
+		document.documentElement.doScroll("left");
+	} catch(e) {
+		setTimeout( doScrollCheck, 1 );
+		return;
+	}
+
+	// and execute any waiting functions
+	jQuery.ready();
+}
+
+return jQuery;
+
+})();
+
+
+var // Promise methods
+	promiseMethods = "done fail isResolved isRejected promise then always pipe".split( " " ),
+	// Static reference to slice
+	sliceDeferred = [].slice;
+
+jQuery.extend({
+	// Create a simple deferred (one callbacks list)
+	_Deferred: function() {
+		var // callbacks list
+			callbacks = [],
+			// stored [ context , args ]
+			fired,
+			// to avoid firing when already doing so
+			firing,
+			// flag to know if the deferred has been cancelled
+			cancelled,
+			// the deferred itself
+			deferred  = {
+
+				// done( f1, f2, ...)
+				done: function() {
+					if ( !cancelled ) {
+						var args = arguments,
+							i,
+							length,
+							elem,
+							type,
+							_fired;
+						if ( fired ) {
+							_fired = fired;
+							fired = 0;
+						}
+						for ( i = 0, length = args.length; i < length; i++ ) {
+							elem = args[ i ];
+							type = jQuery.type( elem );
+							if ( type === "array" ) {
+								deferred.done.apply( deferred, elem );
+							} else if ( type === "function" ) {
+								callbacks.push( elem );
+							}
+						}
+						if ( _fired ) {
+							deferred.resolveWith( _fired[ 0 ], _fired[ 1 ] );
+						}
+					}
+					return this;
+				},
+
+				// resolve with given context and args
+				resolveWith: function( context, args ) {
+					if ( !cancelled && !fired && !firing ) {
+						// make sure args are available (#8421)
+						args = args || [];
+						firing = 1;
+						try {
+							while( callbacks[ 0 ] ) {
+								callbacks.shift().apply( context, args );
+							}
+						}
+						finally {
+							fired = [ context, args ];
+							firing = 0;
+						}
+					}
+					return this;
+				},
+
+				// resolve with this as context and given arguments
+				resolve: function() {
+					deferred.resolveWith( this, arguments );
+					return this;
+				},
+
+				// Has this deferred been resolved?
+				isResolved: function() {
+					return !!( firing || fired );
+				},
+
+				// Cancel
+				cancel: function() {
+					cancelled = 1;
+					callbacks = [];
+					return this;
+				}
+			};
+
+		return deferred;
+	},
+
+	// Full fledged deferred (two callbacks list)
+	Deferred: function( func ) {
+		var deferred = jQuery._Deferred(),
+			failDeferred = jQuery._Deferred(),
+			promise;
+		// Add errorDeferred methods, then and promise
+		jQuery.extend( deferred, {
+			then: function( doneCallbacks, failCallbacks ) {
+				deferred.done( doneCallbacks ).fail( failCallbacks );
+				return this;
+			},
+			always: function() {
+				return deferred.done.apply( deferred, arguments ).fail.apply( this, arguments );
+			},
+			fail: failDeferred.done,
+			rejectWith: failDeferred.resolveWith,
+			reject: failDeferred.resolve,
+			isRejected: failDeferred.isResolved,
+			pipe: function( fnDone, fnFail ) {
+				return jQuery.Deferred(function( newDefer ) {
+					jQuery.each( {
+						done: [ fnDone, "resolve" ],
+						fail: [ fnFail, "reject" ]
+					}, function( handler, data ) {
+						var fn = data[ 0 ],
+							action = data[ 1 ],
+							returned;
+						if ( jQuery.isFunction( fn ) ) {
+							deferred[ handler ](function() {
+								returned = fn.apply( this, arguments );
+								if ( returned && jQuery.isFunction( returned.promise ) ) {
+									returned.promise().then( newDefer.resolve, newDefer.reject );
+								} else {
+									newDefer[ action ]( returned );
+								}
+							});
+						} else {
+							deferred[ handler ]( newDefer[ action ] );
+						}
+					});
+				}).promise();
+			},
+			// Get a promise for this deferred
+			// If obj is provided, the promise aspect is added to the object
+			promise: function( obj ) {
+				if ( obj == null ) {
+					if ( promise ) {
+						return promise;
+					}
+					promise = obj = {};
+				}
+				var i = promiseMethods.length;
+				while( i-- ) {
+					obj[ promiseMethods[i] ] = deferred[ promiseMethods[i] ];
+				}
+				return obj;
+			}
+		});
+		// Make sure only one callback list will be used
+		deferred.done( failDeferred.cancel ).fail( deferred.cancel );
+		// Unexpose cancel
+		delete deferred.cancel;
+		// Call given func if any
+		if ( func ) {
+			func.call( deferred, deferred );
+		}
+		return deferred;
+	},
+
+	// Deferred helper
+	when: function( firstParam ) {
+		var args = arguments,
+			i = 0,
+			length = args.length,
+			count = length,
+			deferred = length <= 1 && firstParam && jQuery.isFunction( firstParam.promise ) ?
+				firstParam :
+				jQuery.Deferred();
+		function resolveFunc( i ) {
+			return function( value ) {
+				args[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
+				if ( !( --count ) ) {
+					// Strange bug in FF4:
+					// Values changed onto the arguments object sometimes end up as undefined values
+					// outside the $.when method. Cloning the object into a fresh array solves the issue
+					deferred.resolveWith( deferred, sliceDeferred.call( args, 0 ) );
+				}
+			};
+		}
+		if ( length > 1 ) {
+			for( ; i < length; i++ ) {
+				if ( args[ i ] && jQuery.isFunction( args[ i ].promise ) ) {
+					args[ i ].promise().then( resolveFunc(i), deferred.reject );
+				} else {
+					--count;
+				}
+			}
+			if ( !count ) {
+				deferred.resolveWith( deferred, args );
+			}
+		} else if ( deferred !== firstParam ) {
+			deferred.resolveWith( deferred, length ? [ firstParam ] : [] );
+		}
+		return deferred.promise();
+	}
+});
+
+
+
+jQuery.support = (function() {
+
+	var div = document.createElement( "div" ),
+		documentElement = document.documentElement,
+		all,
+		a,
+		select,
+		opt,
+		input,
+		marginDiv,
+		support,
+		fragment,
+		body,
+		testElementParent,
+		testElement,
+		testElementStyle,
+		tds,
+		events,
+		eventName,
+		i,
+		isSupported;
+
+	// Preliminary tests
+	div.setAttribute("className", "t");
+	div.innerHTML = "   <link/><table></table><a href='/a' style='top:1px;float:left;opacity:.55;'>a</a><input type='checkbox'/>";
+
+	all = div.getElementsByTagName( "*" );
+	a = div.getElementsByTagName( "a" )[ 0 ];
+
+	// Can't get basic test support
+	if ( !all || !all.length || !a ) {
+		return {};
+	}
+
+	// First batch of supports tests
+	select = document.createElement( "select" );
+	opt = select.appendChild( document.createElement("option") );
+	input = div.getElementsByTagName( "input" )[ 0 ];
+
+	support = {
+		// IE strips leading whitespace when .innerHTML is used
+		leadingWhitespace: ( div.firstChild.nodeType === 3 ),
+
+		// Make sure that tbody elements aren't automatically inserted
+		// IE will insert them into empty tables
+		tbody: !div.getElementsByTagName( "tbody" ).length,
+
+		// Make sure that link elements get serialized correctly by innerHTML
+		// This requires a wrapper element in IE
+		htmlSerialize: !!div.getElementsByTagName( "link" ).length,
+
+		// Get the style information from getAttribute
+		// (IE uses .cssText instead)
+		style: /top/.test( a.getAttribute("style") ),
+
+		// Make sure that URLs aren't manipulated
+		// (IE normalizes it by default)
+		hrefNormalized: ( a.getAttribute( "href" ) === "/a" ),
+
+		// Make sure that element opacity exists
+		// (IE uses filter instead)
+		// Use a regex to work around a WebKit issue. See #5145
+		opacity: /^0.55$/.test( a.style.opacity ),
+
+		// Verify style float existence
+		// (IE uses styleFloat instead of cssFloat)
+		cssFloat: !!a.style.cssFloat,
+
+		// Make sure that if no value is specified for a checkbox
+		// that it defaults to "on".
+		// (WebKit defaults to "" instead)
+		checkOn: ( input.value === "on" ),
+
+		// Make sure that a selected-by-default option has a working selected property.
+		// (WebKit defaults to false instead of true, IE too, if it's in an optgroup)
+		optSelected: opt.selected,
+
+		// Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7)
+		getSetAttribute: div.className !== "t",
+
+		// Will be defined later
+		submitBubbles: true,
+		changeBubbles: true,
+		focusinBubbles: false,
+		deleteExpando: true,
+		noCloneEvent: true,
+		inlineBlockNeedsLayout: false,
+		shrinkWrapBlocks: false,
+		reliableMarginRight: true
+	};
+
+	// Make sure checked status is properly cloned
+	input.checked = true;
+	support.noCloneChecked = input.cloneNode( true ).checked;
+
+	// Make sure that the options inside disabled selects aren't marked as disabled
+	// (WebKit marks them as disabled)
+	select.disabled = true;
+	support.optDisabled = !opt.disabled;
+
+	// Test to see if it's possible to delete an expando from an element
+	// Fails in Internet Explorer
+	try {
+		delete div.test;
+	} catch( e ) {
+		support.deleteExpando = false;
+	}
+
+	if ( !div.addEventListener && div.attachEvent && div.fireEvent ) {
+		div.attachEvent( "onclick", function() {
+			// Cloning a node shouldn't copy over any
+			// bound event handlers (IE does this)
+			support.noCloneEvent = false;
+		});
+		div.cloneNode( true ).fireEvent( "onclick" );
+	}
+
+	// Check if a radio maintains it's value
+	// after being appended to the DOM
+	input = document.createElement("input");
+	input.value = "t";
+	input.setAttribute("type", "radio");
+	support.radioValue = input.value === "t";
+
+	input.setAttribute("checked", "checked");
+	div.appendChild( input );
+	fragment = document.createDocumentFragment();
+	fragment.appendChild( div.firstChild );
+
+	// WebKit doesn't clone checked state correctly in fragments
+	support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked;
+
+	div.innerHTML = "";
+
+	// Figure out if the W3C box model works as expected
+	div.style.width = div.style.paddingLeft = "1px";
+
+	body = document.getElementsByTagName( "body" )[ 0 ];
+	// We use our own, invisible, body unless the body is already present
+	// in which case we use a div (#9239)
+	testElement = document.createElement( body ? "div" : "body" );
+	testElementStyle = {
+		visibility: "hidden",
+		width: 0,
+		height: 0,
+		border: 0,
+		margin: 0
+	};
+	if ( body ) {
+		jQuery.extend( testElementStyle, {
+			position: "absolute",
+			left: -1000,
+			top: -1000
+		});
+	}
+	for ( i in testElementStyle ) {
+		testElement.style[ i ] = testElementStyle[ i ];
+	}
+	testElement.appendChild( div );
+	testElementParent = body || documentElement;
+	testElementParent.insertBefore( testElement, testElementParent.firstChild );
+
+	// Check if a disconnected checkbox will retain its checked
+	// value of true after appended to the DOM (IE6/7)
+	support.appendChecked = input.checked;
+
+	support.boxModel = div.offsetWidth === 2;
+
+	if ( "zoom" in div.style ) {
+		// Check if natively block-level elements act like inline-block
+		// elements when setting their display to 'inline' and giving
+		// them layout
+		// (IE < 8 does this)
+		div.style.display = "inline";
+		div.style.zoom = 1;
+		support.inlineBlockNeedsLayout = ( div.offsetWidth === 2 );
+
+		// Check if elements with layout shrink-wrap their children
+		// (IE 6 does this)
+		div.style.display = "";
+		div.innerHTML = "<div style='width:4px;'></div>";
+		support.shrinkWrapBlocks = ( div.offsetWidth !== 2 );
+	}
+
+	div.innerHTML = "<table><tr><td style='padding:0;border:0;display:none'></td><td>t</td></tr></table>";
+	tds = div.getElementsByTagName( "td" );
+
+	// Check if table cells still have offsetWidth/Height when they are set
+	// to display:none and there are still other visible table cells in a
+	// table row; if so, offsetWidth/Height are not reliable for use when
+	// determining if an element has been hidden directly using
+	// display:none (it is still safe to use offsets if a parent element is
+	// hidden; don safety goggles and see bug #4512 for more information).
+	// (only IE 8 fails this test)
+	isSupported = ( tds[ 0 ].offsetHeight === 0 );
+
+	tds[ 0 ].style.display = "";
+	tds[ 1 ].style.display = "none";
+
+	// Check if empty table cells still have offsetWidth/Height
+	// (IE < 8 fail this test)
+	support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 );
+	div.innerHTML = "";
+
+	// Check if div with explicit width and no margin-right incorrectly
+	// gets computed margin-right based on width of container. For more
+	// info see bug #3333
+	// Fails in WebKit before Feb 2011 nightlies
+	// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+	if ( document.defaultView && document.defaultView.getComputedStyle ) {
+		marginDiv = document.createElement( "div" );
+		marginDiv.style.width = "0";
+		marginDiv.style.marginRight = "0";
+		div.appendChild( marginDiv );
+		support.reliableMarginRight =
+			( parseInt( ( document.defaultView.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0;
+	}
+
+	// Remove the body element we added
+	testElement.innerHTML = "";
+	testElementParent.removeChild( testElement );
+
+	// Technique from Juriy Zaytsev
+	// http://thinkweb2.com/projects/prototype/detecting-event-support-without-browser-sniffing/
+	// We only care about the case where non-standard event systems
+	// are used, namely in IE. Short-circuiting here helps us to
+	// avoid an eval call (in setAttribute) which can cause CSP
+	// to go haywire. See: https://developer.mozilla.org/en/Security/CSP
+	if ( div.attachEvent ) {
+		for( i in {
+			submit: 1,
+			change: 1,
+			focusin: 1
+		} ) {
+			eventName = "on" + i;
+			isSupported = ( eventName in div );
+			if ( !isSupported ) {
+				div.setAttribute( eventName, "return;" );
+				isSupported = ( typeof div[ eventName ] === "function" );
+			}
+			support[ i + "Bubbles" ] = isSupported;
+		}
+	}
+
+	// Null connected elements to avoid leaks in IE
+	testElement = fragment = select = opt = body = marginDiv = div = input = null;
+
+	return support;
+})();
+
+// Keep track of boxModel
+jQuery.boxModel = jQuery.support.boxModel;
+
+
+
+
+var rbrace = /^(?:\{.*\}|\[.*\])$/,
+	rmultiDash = /([a-z])([A-Z])/g;
+
+jQuery.extend({
+	cache: {},
+
+	// Please use with caution
+	uuid: 0,
+
+	// Unique for each copy of jQuery on the page
+	// Non-digits removed to match rinlinejQuery
+	expando: "jQuery" + ( jQuery.fn.jquery + Math.random() ).replace( /\D/g, "" ),
+
+	// The following elements throw uncatchable exceptions if you
+	// attempt to add expando properties to them.
+	noData: {
+		"embed": true,
+		// Ban all objects except for Flash (which handle expandos)
+		"object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",
+		"applet": true
+	},
+
+	hasData: function( elem ) {
+		elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ];
+
+		return !!elem && !isEmptyDataObject( elem );
+	},
+
+	data: function( elem, name, data, pvt /* Internal Use Only */ ) {
+		if ( !jQuery.acceptData( elem ) ) {
+			return;
+		}
+
+		var internalKey = jQuery.expando, getByName = typeof name === "string", thisCache,
+
+			// We have to handle DOM nodes and JS objects differently because IE6-7
+			// can't GC object references properly across the DOM-JS boundary
+			isNode = elem.nodeType,
+
+			// Only DOM nodes need the global jQuery cache; JS object data is
+			// attached directly to the object so GC can occur automatically
+			cache = isNode ? jQuery.cache : elem,
+
+			// Only defining an ID for JS objects if its cache already exists allows
+			// the code to shortcut on the same path as a DOM node with no cache
+			id = isNode ? elem[ jQuery.expando ] : elem[ jQuery.expando ] && jQuery.expando;
+
+		// Avoid doing any more work than we need to when trying to get data on an
+		// object that has no data at all
+		if ( (!id || (pvt && id && !cache[ id ][ internalKey ])) && getByName && data === undefined ) {
+			return;
+		}
+
+		if ( !id ) {
+			// Only DOM nodes need a new unique ID for each element since their data
+			// ends up in the global cache
+			if ( isNode ) {
+				elem[ jQuery.expando ] = id = ++jQuery.uuid;
+			} else {
+				id = jQuery.expando;
+			}
+		}
+
+		if ( !cache[ id ] ) {
+			cache[ id ] = {};
+
+			// TODO: This is a hack for 1.5 ONLY. Avoids exposing jQuery
+			// metadata on plain JS objects when the object is serialized using
+			// JSON.stringify
+			if ( !isNode ) {
+				cache[ id ].toJSON = jQuery.noop;
+			}
+		}
+
+		// An object can be passed to jQuery.data instead of a key/value pair; this gets
+		// shallow copied over onto the existing cache
+		if ( typeof name === "object" || typeof name === "function" ) {
+			if ( pvt ) {
+				cache[ id ][ internalKey ] = jQuery.extend(cache[ id ][ internalKey ], name);
+			} else {
+				cache[ id ] = jQuery.extend(cache[ id ], name);
+			}
+		}
+
+		thisCache = cache[ id ];
+
+		// Internal jQuery data is stored in a separate object inside the object's data
+		// cache in order to avoid key collisions between internal data and user-defined
+		// data
+		if ( pvt ) {
+			if ( !thisCache[ internalKey ] ) {
+				thisCache[ internalKey ] = {};
+			}
+
+			thisCache = thisCache[ internalKey ];
+		}
+
+		if ( data !== undefined ) {
+			thisCache[ jQuery.camelCase( name ) ] = data;
+		}
+
+		// TODO: This is a hack for 1.5 ONLY. It will be removed in 1.6. Users should
+		// not attempt to inspect the internal events object using jQuery.data, as this
+		// internal data object is undocumented and subject to change.
+		if ( name === "events" && !thisCache[name] ) {
+			return thisCache[ internalKey ] && thisCache[ internalKey ].events;
+		}
+
+		return getByName ? 
+			// Check for both converted-to-camel and non-converted data property names
+			thisCache[ jQuery.camelCase( name ) ] || thisCache[ name ] :
+			thisCache;
+	},
+
+	removeData: function( elem, name, pvt /* Internal Use Only */ ) {
+		if ( !jQuery.acceptData( elem ) ) {
+			return;
+		}
+
+		var internalKey = jQuery.expando, isNode = elem.nodeType,
+
+			// See jQuery.data for more information
+			cache = isNode ? jQuery.cache : elem,
+
+			// See jQuery.data for more information
+			id = isNode ? elem[ jQuery.expando ] : jQuery.expando;
+
+		// If there is already no cache entry for this object, there is no
+		// purpose in continuing
+		if ( !cache[ id ] ) {
+			return;
+		}
+
+		if ( name ) {
+			var thisCache = pvt ? cache[ id ][ internalKey ] : cache[ id ];
+
+			if ( thisCache ) {
+				delete thisCache[ name ];
+
+				// If there is no data left in the cache, we want to continue
+				// and let the cache object itself get destroyed
+				if ( !isEmptyDataObject(thisCache) ) {
+					return;
+				}
+			}
+		}
+
+		// See jQuery.data for more information
+		if ( pvt ) {
+			delete cache[ id ][ internalKey ];
+
+			// Don't destroy the parent cache unless the internal data object
+			// had been the only thing left in it
+			if ( !isEmptyDataObject(cache[ id ]) ) {
+				return;
+			}
+		}
+
+		var internalCache = cache[ id ][ internalKey ];
+
+		// Browsers that fail expando deletion also refuse to delete expandos on
+		// the window, but it will allow it on all other JS objects; other browsers
+		// don't care
+		if ( jQuery.support.deleteExpando || cache != window ) {
+			delete cache[ id ];
+		} else {
+			cache[ id ] = null;
+		}
+
+		// We destroyed the entire user cache at once because it's faster than
+		// iterating through each key, but we need to continue to persist internal
+		// data if it existed
+		if ( internalCache ) {
+			cache[ id ] = {};
+			// TODO: This is a hack for 1.5 ONLY. Avoids exposing jQuery
+			// metadata on plain JS objects when the object is serialized using
+			// JSON.stringify
+			if ( !isNode ) {
+				cache[ id ].toJSON = jQuery.noop;
+			}
+
+			cache[ id ][ internalKey ] = internalCache;
+
+		// Otherwise, we need to eliminate the expando on the node to avoid
+		// false lookups in the cache for entries that no longer exist
+		} else if ( isNode ) {
+			// IE does not allow us to delete expando properties from nodes,
+			// nor does it have a removeAttribute function on Document nodes;
+			// we must handle all of these cases
+			if ( jQuery.support.deleteExpando ) {
+				delete elem[ jQuery.expando ];
+			} else if ( elem.removeAttribute ) {
+				elem.removeAttribute( jQuery.expando );
+			} else {
+				elem[ jQuery.expando ] = null;
+			}
+		}
+	},
+
+	// For internal use only.
+	_data: function( elem, name, data ) {
+		return jQuery.data( elem, name, data, true );
+	},
+
+	// A method for determining if a DOM node can handle the data expando
+	acceptData: function( elem ) {
+		if ( elem.nodeName ) {
+			var match = jQuery.noData[ elem.nodeName.toLowerCase() ];
+
+			if ( match ) {
+				return !(match === true || elem.getAttribute("classid") !== match);
+			}
+		}
+
+		return true;
+	}
+});
+
+jQuery.fn.extend({
+	data: function( key, value ) {
+		var data = null;
+
+		if ( typeof key === "undefined" ) {
+			if ( this.length ) {
+				data = jQuery.data( this[0] );
+
+				if ( this[0].nodeType === 1 ) {
+			    var attr = this[0].attributes, name;
+					for ( var i = 0, l = attr.length; i < l; i++ ) {
+						name = attr[i].name;
+
+						if ( name.indexOf( "data-" ) === 0 ) {
+							name = jQuery.camelCase( name.substring(5) );
+
+							dataAttr( this[0], name, data[ name ] );
+						}
+					}
+				}
+			}
+
+			return data;
+
+		} else if ( typeof key === "object" ) {
+			return this.each(function() {
+				jQuery.data( this, key );
+			});
+		}
+
+		var parts = key.split(".");
+		parts[1] = parts[1] ? "." + parts[1] : "";
+
+		if ( value === undefined ) {
+			data = this.triggerHandler("getData" + parts[1] + "!", [parts[0]]);
+
+			// Try to fetch any internally stored data first
+			if ( data === undefined && this.length ) {
+				data = jQuery.data( this[0], key );
+				data = dataAttr( this[0], key, data );
+			}
+
+			return data === undefined && parts[1] ?
+				this.data( parts[0] ) :
+				data;
+
+		} else {
+			return this.each(function() {
+				var $this = jQuery( this ),
+					args = [ parts[0], value ];
+
+				$this.triggerHandler( "setData" + parts[1] + "!", args );
+				jQuery.data( this, key, value );
+				$this.triggerHandler( "changeData" + parts[1] + "!", args );
+			});
+		}
+	},
+
+	removeData: function( key ) {
+		return this.each(function() {
+			jQuery.removeData( this, key );
+		});
+	}
+});
+
+function dataAttr( elem, key, data ) {
+	// If nothing was found internally, try to fetch any
+	// data from the HTML5 data-* attribute
+	if ( data === undefined && elem.nodeType === 1 ) {
+		var name = "data-" + key.replace( rmultiDash, "$1-$2" ).toLowerCase();
+
+		data = elem.getAttribute( name );
+
+		if ( typeof data === "string" ) {
+			try {
+				data = data === "true" ? true :
+				data === "false" ? false :
+				data === "null" ? null :
+				!jQuery.isNaN( data ) ? parseFloat( data ) :
+					rbrace.test( data ) ? jQuery.parseJSON( data ) :
+					data;
+			} catch( e ) {}
+
+			// Make sure we set the data so it isn't changed later
+			jQuery.data( elem, key, data );
+
+		} else {
+			data = undefined;
+		}
+	}
+
+	return data;
+}
+
+// TODO: This is a hack for 1.5 ONLY to allow objects with a single toJSON
+// property to be considered empty objects; this property always exists in
+// order to make sure JSON.stringify does not expose internal metadata
+function isEmptyDataObject( obj ) {
+	for ( var name in obj ) {
+		if ( name !== "toJSON" ) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+
+
+
+function handleQueueMarkDefer( elem, type, src ) {
+	var deferDataKey = type + "defer",
+		queueDataKey = type + "queue",
+		markDataKey = type + "mark",
+		defer = jQuery.data( elem, deferDataKey, undefined, true );
+	if ( defer &&
+		( src === "queue" || !jQuery.data( elem, queueDataKey, undefined, true ) ) &&
+		( src === "mark" || !jQuery.data( elem, markDataKey, undefined, true ) ) ) {
+		// Give room for hard-coded callbacks to fire first
+		// and eventually mark/queue something else on the element
+		setTimeout( function() {
+			if ( !jQuery.data( elem, queueDataKey, undefined, true ) &&
+				!jQuery.data( elem, markDataKey, undefined, true ) ) {
+				jQuery.removeData( elem, deferDataKey, true );
+				defer.resolve();
+			}
+		}, 0 );
+	}
+}
+
+jQuery.extend({
+
+	_mark: function( elem, type ) {
+		if ( elem ) {
+			type = (type || "fx") + "mark";
+			jQuery.data( elem, type, (jQuery.data(elem,type,undefined,true) || 0) + 1, true );
+		}
+	},
+
+	_unmark: function( force, elem, type ) {
+		if ( force !== true ) {
+			type = elem;
+			elem = force;
+			force = false;
+		}
+		if ( elem ) {
+			type = type || "fx";
+			var key = type + "mark",
+				count = force ? 0 : ( (jQuery.data( elem, key, undefined, true) || 1 ) - 1 );
+			if ( count ) {
+				jQuery.data( elem, key, count, true );
+			} else {
+				jQuery.removeData( elem, key, true );
+				handleQueueMarkDefer( elem, type, "mark" );
+			}
+		}
+	},
+
+	queue: function( elem, type, data ) {
+		if ( elem ) {
+			type = (type || "fx") + "queue";
+			var q = jQuery.data( elem, type, undefined, true );
+			// Speed up dequeue by getting out quickly if this is just a lookup
+			if ( data ) {
+				if ( !q || jQuery.isArray(data) ) {
+					q = jQuery.data( elem, type, jQuery.makeArray(data), true );
+				} else {
+					q.push( data );
+				}
+			}
+			return q || [];
+		}
+	},
+
+	dequeue: function( elem, type ) {
+		type = type || "fx";
+
+		var queue = jQuery.queue( elem, type ),
+			fn = queue.shift(),
+			defer;
+
+		// If the fx queue is dequeued, always remove the progress sentinel
+		if ( fn === "inprogress" ) {
+			fn = queue.shift();
+		}
+
+		if ( fn ) {
+			// Add a progress sentinel to prevent the fx queue from being
+			// automatically dequeued
+			if ( type === "fx" ) {
+				queue.unshift("inprogress");
+			}
+
+			fn.call(elem, function() {
+				jQuery.dequeue(elem, type);
+			});
+		}
+
+		if ( !queue.length ) {
+			jQuery.removeData( elem, type + "queue", true );
+			handleQueueMarkDefer( elem, type, "queue" );
+		}
+	}
+});
+
+jQuery.fn.extend({
+	queue: function( type, data ) {
+		if ( typeof type !== "string" ) {
+			data = type;
+			type = "fx";
+		}
+
+		if ( data === undefined ) {
+			return jQuery.queue( this[0], type );
+		}
+		return this.each(function() {
+			var queue = jQuery.queue( this, type, data );
+
+			if ( type === "fx" && queue[0] !== "inprogress" ) {
+				jQuery.dequeue( this, type );
+			}
+		});
+	},
+	dequeue: function( type ) {
+		return this.each(function() {
+			jQuery.dequeue( this, type );
+		});
+	},
+	// Based off of the plugin by Clint Helfers, with permission.
+	// http://blindsignals.com/index.php/2009/07/jquery-delay/
+	delay: function( time, type ) {
+		time = jQuery.fx ? jQuery.fx.speeds[time] || time : time;
+		type = type || "fx";
+
+		return this.queue( type, function() {
+			var elem = this;
+			setTimeout(function() {
+				jQuery.dequeue( elem, type );
+			}, time );
+		});
+	},
+	clearQueue: function( type ) {
+		return this.queue( type || "fx", [] );
+	},
+	// Get a promise resolved when queues of a certain type
+	// are emptied (fx is the type by default)
+	promise: function( type, object ) {
+		if ( typeof type !== "string" ) {
+			object = type;
+			type = undefined;
+		}
+		type = type || "fx";
+		var defer = jQuery.Deferred(),
+			elements = this,
+			i = elements.length,
+			count = 1,
+			deferDataKey = type + "defer",
+			queueDataKey = type + "queue",
+			markDataKey = type + "mark",
+			tmp;
+		function resolve() {
+			if ( !( --count ) ) {
+				defer.resolveWith( elements, [ elements ] );
+			}
+		}
+		while( i-- ) {
+			if (( tmp = jQuery.data( elements[ i ], deferDataKey, undefined, true ) ||
+					( jQuery.data( elements[ i ], queueDataKey, undefined, true ) ||
+						jQuery.data( elements[ i ], markDataKey, undefined, true ) ) &&
+					jQuery.data( elements[ i ], deferDataKey, jQuery._Deferred(), true ) )) {
+				count++;
+				tmp.done( resolve );
+			}
+		}
+		resolve();
+		return defer.promise();
+	}
+});
+
+
+
+
+var rclass = /[\n\t\r]/g,
+	rspace = /\s+/,
+	rreturn = /\r/g,
+	rtype = /^(?:button|input)$/i,
+	rfocusable = /^(?:button|input|object|select|textarea)$/i,
+	rclickable = /^a(?:rea)?$/i,
+	rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,
+	rinvalidChar = /\:|^on/,
+	formHook, boolHook;
+
+jQuery.fn.extend({
+	attr: function( name, value ) {
+		return jQuery.access( this, name, value, true, jQuery.attr );
+	},
+
+	removeAttr: function( name ) {
+		return this.each(function() {
+			jQuery.removeAttr( this, name );
+		});
+	},
+	
+	prop: function( name, value ) {
+		return jQuery.access( this, name, value, true, jQuery.prop );
+	},
+	
+	removeProp: function( name ) {
+		name = jQuery.propFix[ name ] || name;
+		return this.each(function() {
+			// try/catch handles cases where IE balks (such as removing a property on window)
+			try {
+				this[ name ] = undefined;
+				delete this[ name ];
+			} catch( e ) {}
+		});
+	},
+
+	addClass: function( value ) {
+		var classNames, i, l, elem,
+			setClass, c, cl;
+
+		if ( jQuery.isFunction( value ) ) {
+			return this.each(function( j ) {
+				jQuery( this ).addClass( value.call(this, j, this.className) );
+			});
+		}
+
+		if ( value && typeof value === "string" ) {
+			classNames = value.split( rspace );
+
+			for ( i = 0, l = this.length; i < l; i++ ) {
+				elem = this[ i ];
+
+				if ( elem.nodeType === 1 ) {
+					if ( !elem.className && classNames.length === 1 ) {
+						elem.className = value;
+
+					} else {
+						setClass = " " + elem.className + " ";
+
+						for ( c = 0, cl = classNames.length; c < cl; c++ ) {
+							if ( !~setClass.indexOf( " " + classNames[ c ] + " " ) ) {
+								setClass += classNames[ c ] + " ";
+							}
+						}
+						elem.className = jQuery.trim( setClass );
+					}
+				}
+			}
+		}
+
+		return this;
+	},
+
+	removeClass: function( value ) {
+		var classNames, i, l, elem, className, c, cl;
+
+		if ( jQuery.isFunction( value ) ) {
+			return this.each(function( j ) {
+				jQuery( this ).removeClass( value.call(this, j, this.className) );
+			});
+		}
+
+		if ( (value && typeof value === "string") || value === undefined ) {
+			classNames = (value || "").split( rspace );
+
+			for ( i = 0, l = this.length; i < l; i++ ) {
+				elem = this[ i ];
+
+				if ( elem.nodeType === 1 && elem.className ) {
+					if ( value ) {
+						className = (" " + elem.className + " ").replace( rclass, " " );
+						for ( c = 0, cl = classNames.length; c < cl; c++ ) {
+							className = className.replace(" " + classNames[ c ] + " ", " ");
+						}
+						elem.className = jQuery.trim( className );
+
+					} else {
+						elem.className = "";
+					}
+				}
+			}
+		}
+
+		return this;
+	},
+
+	toggleClass: function( value, stateVal ) {
+		var type = typeof value,
+			isBool = typeof stateVal === "boolean";
+
+		if ( jQuery.isFunction( value ) ) {
+			return this.each(function( i ) {
+				jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal );
+			});
+		}
+
+		return this.each(function() {
+			if ( type === "string" ) {
+				// toggle individual class names
+				var className,
+					i = 0,
+					self = jQuery( this ),
+					state = stateVal,
+					classNames = value.split( rspace );
+
+				while ( (className = classNames[ i++ ]) ) {
+					// check each className given, space seperated list
+					state = isBool ? state : !self.hasClass( className );
+					self[ state ? "addClass" : "removeClass" ]( className );
+				}
+
+			} else if ( type === "undefined" || type === "boolean" ) {
+				if ( this.className ) {
+					// store className if set
+					jQuery._data( this, "__className__", this.className );
+				}
+
+				// toggle whole className
+				this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || "";
+			}
+		});
+	},
+
+	hasClass: function( selector ) {
+		var className = " " + selector + " ";
+		for ( var i = 0, l = this.length; i < l; i++ ) {
+			if ( (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) {
+				return true;
+			}
+		}
+
+		return false;
+	},
+
+	val: function( value ) {
+		var hooks, ret,
+			elem = this[0];
+		
+		if ( !arguments.length ) {
+			if ( elem ) {
+				hooks = jQuery.valHooks[ elem.nodeName.toLowerCase() ] || jQuery.valHooks[ elem.type ];
+
+				if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) {
+					return ret;
+				}
+
+				ret = elem.value;
+
+				return typeof ret === "string" ? 
+					// handle most common string cases
+					ret.replace(rreturn, "") : 
+					// handle cases where value is null/undef or number
+					ret == null ? "" : ret;
+			}
+
+			return undefined;
+		}
+
+		var isFunction = jQuery.isFunction( value );
+
+		return this.each(function( i ) {
+			var self = jQuery(this), val;
+
+			if ( this.nodeType !== 1 ) {
+				return;
+			}
+
+			if ( isFunction ) {
+				val = value.call( this, i, self.val() );
+			} else {
+				val = value;
+			}
+
+			// Treat null/undefined as ""; convert numbers to string
+			if ( val == null ) {
+				val = "";
+			} else if ( typeof val === "number" ) {
+				val += "";
+			} else if ( jQuery.isArray( val ) ) {
+				val = jQuery.map(val, function ( value ) {
+					return value == null ? "" : value + "";
+				});
+			}
+
+			hooks = jQuery.valHooks[ this.nodeName.toLowerCase() ] || jQuery.valHooks[ this.type ];
+
+			// If set returns undefined, fall back to normal setting
+			if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) {
+				this.value = val;
+			}
+		});
+	}
+});
+
+jQuery.extend({
+	valHooks: {
+		option: {
+			get: function( elem ) {
+				// attributes.value is undefined in Blackberry 4.7 but
+				// uses .value. See #6932
+				var val = elem.attributes.value;
+				return !val || val.specified ? elem.value : elem.text;
+			}
+		},
+		select: {
+			get: function( elem ) {
+				var value,
+					index = elem.selectedIndex,
+					values = [],
+					options = elem.options,
+					one = elem.type === "select-one";
+
+				// Nothing was selected
+				if ( index < 0 ) {
+					return null;
+				}
+
+				// Loop through all the selected options
+				for ( var i = one ? index : 0, max = one ? index + 1 : options.length; i < max; i++ ) {
+					var option = options[ i ];
+
+					// Don't return options that are disabled or in a disabled optgroup
+					if ( option.selected && (jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null) &&
+							(!option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" )) ) {
+
+						// Get the specific value for the option
+						value = jQuery( option ).val();
+
+						// We don't need an array for one selects
+						if ( one ) {
+							return value;
+						}
+
+						// Multi-Selects return an array
+						values.push( value );
+					}
+				}
+
+				// Fixes Bug #2551 -- select.val() broken in IE after form.reset()
+				if ( one && !values.length && options.length ) {
+					return jQuery( options[ index ] ).val();
+				}
+
+				return values;
+			},
+
+			set: function( elem, value ) {
+				var values = jQuery.makeArray( value );
+
+				jQuery(elem).find("option").each(function() {
+					this.selected = jQuery.inArray( jQuery(this).val(), values ) >= 0;
+				});
+
+				if ( !values.length ) {
+					elem.selectedIndex = -1;
+				}
+				return values;
+			}
+		}
+	},
+
+	attrFn: {
+		val: true,
+		css: true,
+		html: true,
+		text: true,
+		data: true,
+		width: true,
+		height: true,
+		offset: true
+	},
+	
+	attrFix: {
+		// Always normalize to ensure hook usage
+		tabindex: "tabIndex"
+	},
+	
+	attr: function( elem, name, value, pass ) {
+		var nType = elem.nodeType;
+		
+		// don't get/set attributes on text, comment and attribute nodes
+		if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
+			return undefined;
+		}
+
+		if ( pass && name in jQuery.attrFn ) {
+			return jQuery( elem )[ name ]( value );
+		}
+
+		// Fallback to prop when attributes are not supported
+		if ( !("getAttribute" in elem) ) {
+			return jQuery.prop( elem, name, value );
+		}
+
+		var ret, hooks,
+			notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+
+		// Normalize the name if needed
+		if ( notxml ) {
+			name = jQuery.attrFix[ name ] || name;
+
+			hooks = jQuery.attrHooks[ name ];
+
+			if ( !hooks ) {
+				// Use boolHook for boolean attributes
+				if ( rboolean.test( name ) ) {
+
+					hooks = boolHook;
+
+				// Use formHook for forms and if the name contains certain characters
+				} else if ( formHook && name !== "className" &&
+					(jQuery.nodeName( elem, "form" ) || rinvalidChar.test( name )) ) {
+
+					hooks = formHook;
+				}
+			}
+		}
+
+		if ( value !== undefined ) {
+
+			if ( value === null ) {
+				jQuery.removeAttr( elem, name );
+				return undefined;
+
+			} else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) {
+				return ret;
+
+			} else {
+				elem.setAttribute( name, "" + value );
+				return value;
+			}
+
+		} else if ( hooks && "get" in hooks && notxml && (ret = hooks.get( elem, name )) !== null ) {
+			return ret;
+
+		} else {
+
+			ret = elem.getAttribute( name );
+
+			// Non-existent attributes return null, we normalize to undefined
+			return ret === null ?
+				undefined :
+				ret;
+		}
+	},
+
+	removeAttr: function( elem, name ) {
+		var propName;
+		if ( elem.nodeType === 1 ) {
+			name = jQuery.attrFix[ name ] || name;
+		
+			if ( jQuery.support.getSetAttribute ) {
+				// Use removeAttribute in browsers that support it
+				elem.removeAttribute( name );
+			} else {
+				jQuery.attr( elem, name, "" );
+				elem.removeAttributeNode( elem.getAttributeNode( name ) );
+			}
+
+			// Set corresponding property to false for boolean attributes
+			if ( rboolean.test( name ) && (propName = jQuery.propFix[ name ] || name) in elem ) {
+				elem[ propName ] = false;
+			}
+		}
+	},
+
+	attrHooks: {
+		type: {
+			set: function( elem, value ) {
+				// We can't allow the type property to be changed (since it causes problems in IE)
+				if ( rtype.test( elem.nodeName ) && elem.parentNode ) {
+					jQuery.error( "type property can't be changed" );
+				} else if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) {
+					// Setting the type on a radio button after the value resets the value in IE6-9
+					// Reset value to it's default in case type is set after value
+					// This is for element creation
+					var val = elem.value;
+					elem.setAttribute( "type", value );
+					if ( val ) {
+						elem.value = val;
+					}
+					return value;
+				}
+			}
+		},
+		tabIndex: {
+			get: function( elem ) {
+				// elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set
+				// http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/
+				var attributeNode = elem.getAttributeNode("tabIndex");
+
+				return attributeNode && attributeNode.specified ?
+					parseInt( attributeNode.value, 10 ) :
+					rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ?
+						0 :
+						undefined;
+			}
+		},
+		// Use the value property for back compat
+		// Use the formHook for button elements in IE6/7 (#1954)
+		value: {
+			get: function( elem, name ) {
+				if ( formHook && jQuery.nodeName( elem, "button" ) ) {
+					return formHook.get( elem, name );
+				}
+				return name in elem ?
+					elem.value :
+					null;
+			},
+			set: function( elem, value, name ) {
+				if ( formHook && jQuery.nodeName( elem, "button" ) ) {
+					return formHook.set( elem, value, name );
+				}
+				// Does not return so that setAttribute is also used
+				elem.value = value;
+			}
+		}
+	},
+
+	propFix: {
+		tabindex: "tabIndex",
+		readonly: "readOnly",
+		"for": "htmlFor",
+		"class": "className",
+		maxlength: "maxLength",
+		cellspacing: "cellSpacing",
+		cellpadding: "cellPadding",
+		rowspan: "rowSpan",
+		colspan: "colSpan",
+		usemap: "useMap",
+		frameborder: "frameBorder",
+		contenteditable: "contentEditable"
+	},
+	
+	prop: function( elem, name, value ) {
+		var nType = elem.nodeType;
+
+		// don't get/set properties on text, comment and attribute nodes
+		if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
+			return undefined;
+		}
+
+		var ret, hooks,
+			notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+
+		if ( notxml ) {
+			// Fix name and attach hooks
+			name = jQuery.propFix[ name ] || name;
+			hooks = jQuery.propHooks[ name ];
+		}
+
+		if ( value !== undefined ) {
+			if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) {
+				return ret;
+
+			} else {
+				return (elem[ name ] = value);
+			}
+
+		} else {
+			if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== undefined ) {
+				return ret;
+
+			} else {
+				return elem[ name ];
+			}
+		}
+	},
+	
+	propHooks: {}
+});
+
+// Hook for boolean attributes
+boolHook = {
+	get: function( elem, name ) {
+		// Align boolean attributes with corresponding properties
+		return jQuery.prop( elem, name ) ?
+			name.toLowerCase() :
+			undefined;
+	},
+	set: function( elem, value, name ) {
+		var propName;
+		if ( value === false ) {
+			// Remove boolean attributes when set to false
+			jQuery.removeAttr( elem, name );
+		} else {
+			// value is true since we know at this point it's type boolean and not false
+			// Set boolean attributes to the same name and set the DOM property
+			propName = jQuery.propFix[ name ] || name;
+			if ( propName in elem ) {
+				// Only set the IDL specifically if it already exists on the element
+				elem[ propName ] = true;
+			}
+
+			elem.setAttribute( name, name.toLowerCase() );
+		}
+		return name;
+	}
+};
+
+// IE6/7 do not support getting/setting some attributes with get/setAttribute
+if ( !jQuery.support.getSetAttribute ) {
+
+	// propFix is more comprehensive and contains all fixes
+	jQuery.attrFix = jQuery.propFix;
+	
+	// Use this for any attribute on a form in IE6/7
+	formHook = jQuery.attrHooks.name = jQuery.attrHooks.title = jQuery.valHooks.button = {
+		get: function( elem, name ) {
+			var ret;
+			ret = elem.getAttributeNode( name );
+			// Return undefined if nodeValue is empty string
+			return ret && ret.nodeValue !== "" ?
+				ret.nodeValue :
+				undefined;
+		},
+		set: function( elem, value, name ) {
+			// Check form objects in IE (multiple bugs related)
+			// Only use nodeValue if the attribute node exists on the form
+			var ret = elem.getAttributeNode( name );
+			if ( ret ) {
+				ret.nodeValue = value;
+				return value;
+			}
+		}
+	};
+
+	// Set width and height to auto instead of 0 on empty string( Bug #8150 )
+	// This is for removals
+	jQuery.each([ "width", "height" ], function( i, name ) {
+		jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
+			set: function( elem, value ) {
+				if ( value === "" ) {
+					elem.setAttribute( name, "auto" );
+					return value;
+				}
+			}
+		});
+	});
+}
+
+
+// Some attributes require a special call on IE
+if ( !jQuery.support.hrefNormalized ) {
+	jQuery.each([ "href", "src", "width", "height" ], function( i, name ) {
+		jQuery.attrHooks[ name ] = jQuery.extend( jQuery.attrHooks[ name ], {
+			get: function( elem ) {
+				var ret = elem.getAttribute( name, 2 );
+				return ret === null ? undefined : ret;
+			}
+		});
+	});
+}
+
+if ( !jQuery.support.style ) {
+	jQuery.attrHooks.style = {
+		get: function( elem ) {
+			// Return undefined in the case of empty string
+			// Normalize to lowercase since IE uppercases css property names
+			return elem.style.cssText.toLowerCase() || undefined;
+		},
+		set: function( elem, value ) {
+			return (elem.style.cssText = "" + value);
+		}
+	};
+}
+
+// Safari mis-reports the default selected property of an option
+// Accessing the parent's selectedIndex property fixes it
+if ( !jQuery.support.optSelected ) {
+	jQuery.propHooks.selected = jQuery.extend( jQuery.propHooks.selected, {
+		get: function( elem ) {
+			var parent = elem.parentNode;
+
+			if ( parent ) {
+				parent.selectedIndex;
+
+				// Make sure that it also works with optgroups, see #5701
+				if ( parent.parentNode ) {
+					parent.parentNode.selectedIndex;
+				}
+			}
+		}
+	});
+}
+
+// Radios and checkboxes getter/setter
+if ( !jQuery.support.checkOn ) {
+	jQuery.each([ "radio", "checkbox" ], function() {
+		jQuery.valHooks[ this ] = {
+			get: function( elem ) {
+				// Handle the case where in Webkit "" is returned instead of "on" if a value isn't specified
+				return elem.getAttribute("value") === null ? "on" : elem.value;
+			}
+		};
+	});
+}
+jQuery.each([ "radio", "checkbox" ], function() {
+	jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], {
+		set: function( elem, value ) {
+			if ( jQuery.isArray( value ) ) {
+				return (elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0);
+			}
+		}
+	});
+});
+
+
+
+
+var rnamespaces = /\.(.*)$/,
+	rformElems = /^(?:textarea|input|select)$/i,
+	rperiod = /\./g,
+	rspaces = / /g,
+	rescape = /[^\w\s.|`]/g,
+	fcleanup = function( nm ) {
+		return nm.replace(rescape, "\\$&");
+	};
+
+/*
+ * A number of helper functions used for managing events.
+ * Many of the ideas behind this code originated from
+ * Dean Edwards' addEvent library.
+ */
+jQuery.event = {
+
+	// Bind an event to an element
+	// Original by Dean Edwards
+	add: function( elem, types, handler, data ) {
+		if ( elem.nodeType === 3 || elem.nodeType === 8 ) {
+			return;
+		}
+
+		if ( handler === false ) {
+			handler = returnFalse;
+		} else if ( !handler ) {
+			// Fixes bug #7229. Fix recommended by jdalton
+			return;
+		}
+
+		var handleObjIn, handleObj;
+
+		if ( handler.handler ) {
+			handleObjIn = handler;
+			handler = handleObjIn.handler;
+		}
+
+		// Make sure that the function being executed has a unique ID
+		if ( !handler.guid ) {
+			handler.guid = jQuery.guid++;
+		}
+
+		// Init the element's event structure
+		var elemData = jQuery._data( elem );
+
+		// If no elemData is found then we must be trying to bind to one of the
+		// banned noData elements
+		if ( !elemData ) {
+			return;
+		}
+
+		var events = elemData.events,
+			eventHandle = elemData.handle;
+
+		if ( !events ) {
+			elemData.events = events = {};
+		}
+
+		if ( !eventHandle ) {
+			elemData.handle = eventHandle = function( e ) {
+				// Discard the second event of a jQuery.event.trigger() and
+				// when an event is called after a page has unloaded
+				return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ?
+					jQuery.event.handle.apply( eventHandle.elem, arguments ) :
+					undefined;
+			};
+		}
+
+		// Add elem as a property of the handle function
+		// This is to prevent a memory leak with non-native events in IE.
+		eventHandle.elem = elem;
+
+		// Handle multiple events separated by a space
+		// jQuery(...).bind("mouseover mouseout", fn);
+		types = types.split(" ");
+
+		var type, i = 0, namespaces;
+
+		while ( (type = types[ i++ ]) ) {
+			handleObj = handleObjIn ?
+				jQuery.extend({}, handleObjIn) :
+				{ handler: handler, data: data };
+
+			// Namespaced event handlers
+			if ( type.indexOf(".") > -1 ) {
+				namespaces = type.split(".");
+				type = namespaces.shift();
+				handleObj.namespace = namespaces.slice(0).sort().join(".");
+
+			} else {
+				namespaces = [];
+				handleObj.namespace = "";
+			}
+
+			handleObj.type = type;
+			if ( !handleObj.guid ) {
+				handleObj.guid = handler.guid;
+			}
+
+			// Get the current list of functions bound to this event
+			var handlers = events[ type ],
+				special = jQuery.event.special[ type ] || {};
+
+			// Init the event handler queue
+			if ( !handlers ) {
+				handlers = events[ type ] = [];
+
+				// Check for a special event handler
+				// Only use addEventListener/attachEvent if the special
+				// events handler returns false
+				if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) {
+					// Bind the global event handler to the element
+					if ( elem.addEventListener ) {
+						elem.addEventListener( type, eventHandle, false );
+
+					} else if ( elem.attachEvent ) {
+						elem.attachEvent( "on" + type, eventHandle );
+					}
+				}
+			}
+
+			if ( special.add ) {
+				special.add.call( elem, handleObj );
+
+				if ( !handleObj.handler.guid ) {
+					handleObj.handler.guid = handler.guid;
+				}
+			}
+
+			// Add the function to the element's handler list
+			handlers.push( handleObj );
+
+			// Keep track of which events have been used, for event optimization
+			jQuery.event.global[ type ] = true;
+		}
+
+		// Nullify elem to prevent memory leaks in IE
+		elem = null;
+	},
+
+	global: {},
+
+	// Detach an event or set of events from an element
+	remove: function( elem, types, handler, pos ) {
+		// don't do events on text and comment nodes
+		if ( elem.nodeType === 3 || elem.nodeType === 8 ) {
+			return;
+		}
+
+		if ( handler === false ) {
+			handler = returnFalse;
+		}
+
+		var ret, type, fn, j, i = 0, all, namespaces, namespace, special, eventType, handleObj, origType,
+			elemData = jQuery.hasData( elem ) && jQuery._data( elem ),
+			events = elemData && elemData.events;
+
+		if ( !elemData || !events ) {
+			return;
+		}
+
+		// types is actually an event object here
+		if ( types && types.type ) {
+			handler = types.handler;
+			types = types.type;
+		}
+
+		// Unbind all events for the element
+		if ( !types || typeof types === "string" && types.charAt(0) === "." ) {
+			types = types || "";
+
+			for ( type in events ) {
+				jQuery.event.remove( elem, type + types );
+			}
+
+			return;
+		}
+
+		// Handle multiple events separated by a space
+		// jQuery(...).unbind("mouseover mouseout", fn);
+		types = types.split(" ");
+
+		while ( (type = types[ i++ ]) ) {
+			origType = type;
+			handleObj = null;
+			all = type.indexOf(".") < 0;
+			namespaces = [];
+
+			if ( !all ) {
+				// Namespaced event handlers
+				namespaces = type.split(".");
+				type = namespaces.shift();
+
+				namespace = new RegExp("(^|\\.)" +
+					jQuery.map( namespaces.slice(0).sort(), fcleanup ).join("\\.(?:.*\\.)?") + "(\\.|$)");
+			}
+
+			eventType = events[ type ];
+
+			if ( !eventType ) {
+				continue;
+			}
+
+			if ( !handler ) {
+				for ( j = 0; j < eventType.length; j++ ) {
+					handleObj = eventType[ j ];
+
+					if ( all || namespace.test( handleObj.namespace ) ) {
+						jQuery.event.remove( elem, origType, handleObj.handler, j );
+						eventType.splice( j--, 1 );
+					}
+				}
+
+				continue;
+			}
+
+			special = jQuery.event.special[ type ] || {};
+
+			for ( j = pos || 0; j < eventType.length; j++ ) {
+				handleObj = eventType[ j ];
+
+				if ( handler.guid === handleObj.guid ) {
+					// remove the given handler for the given type
+					if ( all || namespace.test( handleObj.namespace ) ) {
+						if ( pos == null ) {
+							eventType.splice( j--, 1 );
+						}
+
+						if ( special.remove ) {
+							special.remove.call( elem, handleObj );
+						}
+					}
+
+					if ( pos != null ) {
+						break;
+					}
+				}
+			}
+
+			// remove generic event handler if no more handlers exist
+			if ( eventType.length === 0 || pos != null && eventType.length === 1 ) {
+				if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) {
+					jQuery.removeEvent( elem, type, elemData.handle );
+				}
+
+				ret = null;
+				delete events[ type ];
+			}
+		}
+
+		// Remove the expando if it's no longer used
+		if ( jQuery.isEmptyObject( events ) ) {
+			var handle = elemData.handle;
+			if ( handle ) {
+				handle.elem = null;
+			}
+
+			delete elemData.events;
+			delete elemData.handle;
+
+			if ( jQuery.isEmptyObject( elemData ) ) {
+				jQuery.removeData( elem, undefined, true );
+			}
+		}
+	},
+	
+	// Events that are safe to short-circuit if no handlers are attached.
+	// Native DOM events should not be added, they may have inline handlers.
+	customEvent: {
+		"getData": true,
+		"setData": true,
+		"changeData": true
+	},
+
+	trigger: function( event, data, elem, onlyHandlers ) {
+		// Event object or event type
+		var type = event.type || event,
+			namespaces = [],
+			exclusive;
+
+		if ( type.indexOf("!") >= 0 ) {
+			// Exclusive events trigger only for the exact event (no namespaces)
+			type = type.slice(0, -1);
+			exclusive = true;
+		}
+
+		if ( type.indexOf(".") >= 0 ) {
+			// Namespaced trigger; create a regexp to match event type in handle()
+			namespaces = type.split(".");
+			type = namespaces.shift();
+			namespaces.sort();
+		}
+
+		if ( (!elem || jQuery.event.customEvent[ type ]) && !jQuery.event.global[ type ] ) {
+			// No jQuery handlers for this event type, and it can't have inline handlers
+			return;
+		}
+
+		// Caller can pass in an Event, Object, or just an event type string
+		event = typeof event === "object" ?
+			// jQuery.Event object
+			event[ jQuery.expando ] ? event :
+			// Object literal
+			new jQuery.Event( type, event ) :
+			// Just the event type (string)
+			new jQuery.Event( type );
+
+		event.type = type;
+		event.exclusive = exclusive;
+		event.namespace = namespaces.join(".");
+		event.namespace_re = new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)");
+		
+		// triggerHandler() and global events don't bubble or run the default action
+		if ( onlyHandlers || !elem ) {
+			event.preventDefault();
+			event.stopPropagation();
+		}
+
+		// Handle a global trigger
+		if ( !elem ) {
+			// TODO: Stop taunting the data cache; remove global events and always attach to document
+			jQuery.each( jQuery.cache, function() {
+				// internalKey variable is just used to make it easier to find
+				// and potentially change this stuff later; currently it just
+				// points to jQuery.expando
+				var internalKey = jQuery.expando,
+					internalCache = this[ internalKey ];
+				if ( internalCache && internalCache.events && internalCache.events[ type ] ) {
+					jQuery.event.trigger( event, data, internalCache.handle.elem );
+				}
+			});
+			return;
+		}
+
+		// Don't do events on text and comment nodes
+		if ( elem.nodeType === 3 || elem.nodeType === 8 ) {
+			return;
+		}
+
+		// Clean up the event in case it is being reused
+		event.result = undefined;
+		event.target = elem;
+
+		// Clone any incoming data and prepend the event, creating the handler arg list
+		data = data != null ? jQuery.makeArray( data ) : [];
+		data.unshift( event );
+
+		var cur = elem,
+			// IE doesn't like method names with a colon (#3533, #8272)
+			ontype = type.indexOf(":") < 0 ? "on" + type : "";
+
+		// Fire event on the current element, then bubble up the DOM tree
+		do {
+			var handle = jQuery._data( cur, "handle" );
+
+			event.currentTarget = cur;
+			if ( handle ) {
+				handle.apply( cur, data );
+			}
+
+			// Trigger an inline bound script
+			if ( ontype && jQuery.acceptData( cur ) && cur[ ontype ] && cur[ ontype ].apply( cur, data ) === false ) {
+				event.result = false;
+				event.preventDefault();
+			}
+
+			// Bubble up to document, then to window
+			cur = cur.parentNode || cur.ownerDocument || cur === event.target.ownerDocument && window;
+		} while ( cur && !event.isPropagationStopped() );
+
+		// If nobody prevented the default action, do it now
+		if ( !event.isDefaultPrevented() ) {
+			var old,
+				special = jQuery.event.special[ type ] || {};
+
+			if ( (!special._default || special._default.call( elem.ownerDocument, event ) === false) &&
+				!(type === "click" && jQuery.nodeName( elem, "a" )) && jQuery.acceptData( elem ) ) {
+
+				// Call a native DOM method on the target with the same name name as the event.
+				// Can't use an .isFunction)() check here because IE6/7 fails that test.
+				// IE<9 dies on focus to hidden element (#1486), may want to revisit a try/catch.
+				try {
+					if ( ontype && elem[ type ] ) {
+						// Don't re-trigger an onFOO event when we call its FOO() method
+						old = elem[ ontype ];
+
+						if ( old ) {
+							elem[ ontype ] = null;
+						}
+
+						jQuery.event.triggered = type;
+						elem[ type ]();
+					}
+				} catch ( ieError ) {}
+
+				if ( old ) {
+					elem[ ontype ] = old;
+				}
+
+				jQuery.event.triggered = undefined;
+			}
+		}
+		
+		return event.result;
+	},
+
+	handle: function( event ) {
+		event = jQuery.event.fix( event || window.event );
+		// Snapshot the handlers list since a called handler may add/remove events.
+		var handlers = ((jQuery._data( this, "events" ) || {})[ event.type ] || []).slice(0),
+			run_all = !event.exclusive && !event.namespace,
+			args = Array.prototype.slice.call( arguments, 0 );
+
+		// Use the fix-ed Event rather than the (read-only) native event
+		args[0] = event;
+		event.currentTarget = this;
+
+		for ( var j = 0, l = handlers.length; j < l; j++ ) {
+			var handleObj = handlers[ j ];
+
+			// Triggered event must 1) be non-exclusive and have no namespace, or
+			// 2) have namespace(s) a subset or equal to those in the bound event.
+			if ( run_all || event.namespace_re.test( handleObj.namespace ) ) {
+				// Pass in a reference to the handler function itself
+				// So that we can later remove it
+				event.handler = handleObj.handler;
+				event.data = handleObj.data;
+				event.handleObj = handleObj;
+
+				var ret = handleObj.handler.apply( this, args );
+
+				if ( ret !== undefined ) {
+					event.result = ret;
+					if ( ret === false ) {
+						event.preventDefault();
+						event.stopPropagation();
+					}
+				}
+
+				if ( event.isImmediatePropagationStopped() ) {
+					break;
+				}
+			}
+		}
+		return event.result;
+	},
+
+	props: "altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),
+
+	fix: function( event ) {
+		if ( event[ jQuery.expando ] ) {
+			return event;
+		}
+
+		// store a copy of the original event object
+		// and "clone" to set read-only properties
+		var originalEvent = event;
+		event = jQuery.Event( originalEvent );
+
+		for ( var i = this.props.length, prop; i; ) {
+			prop = this.props[ --i ];
+			event[ prop ] = originalEvent[ prop ];
+		}
+
+		// Fix target property, if necessary
+		if ( !event.target ) {
+			// Fixes #1925 where srcElement might not be defined either
+			event.target = event.srcElement || document;
+		}
+
+		// check if target is a textnode (safari)
+		if ( event.target.nodeType === 3 ) {
+			event.target = event.target.parentNode;
+		}
+
+		// Add relatedTarget, if necessary
+		if ( !event.relatedTarget && event.fromElement ) {
+			event.relatedTarget = event.fromElement === event.target ? event.toElement : event.fromElement;
+		}
+
+		// Calculate pageX/Y if missing and clientX/Y available
+		if ( event.pageX == null && event.clientX != null ) {
+			var eventDocument = event.target.ownerDocument || document,
+				doc = eventDocument.documentElement,
+				body = eventDocument.body;
+
+			event.pageX = event.clientX + (doc && doc.scrollLeft || body && body.scrollLeft || 0) - (doc && doc.clientLeft || body && body.clientLeft || 0);
+			event.pageY = event.clientY + (doc && doc.scrollTop  || body && body.scrollTop  || 0) - (doc && doc.clientTop  || body && body.clientTop  || 0);
+		}
+
+		// Add which for key events
+		if ( event.which == null && (event.charCode != null || event.keyCode != null) ) {
+			event.which = event.charCode != null ? event.charCode : event.keyCode;
+		}
+
+		// Add metaKey to non-Mac browsers (use ctrl for PC's and Meta for Macs)
+		if ( !event.metaKey && event.ctrlKey ) {
+			event.metaKey = event.ctrlKey;
+		}
+
+		// Add which for click: 1 === left; 2 === middle; 3 === right
+		// Note: button is not normalized, so don't use it
+		if ( !event.which && event.button !== undefined ) {
+			event.which = (event.button & 1 ? 1 : ( event.button & 2 ? 3 : ( event.button & 4 ? 2 : 0 ) ));
+		}
+
+		return event;
+	},
+
+	// Deprecated, use jQuery.guid instead
+	guid: 1E8,
+
+	// Deprecated, use jQuery.proxy instead
+	proxy: jQuery.proxy,
+
+	special: {
+		ready: {
+			// Make sure the ready event is setup
+			setup: jQuery.bindReady,
+			teardown: jQuery.noop
+		},
+
+		live: {
+			add: function( handleObj ) {
+				jQuery.event.add( this,
+					liveConvert( handleObj.origType, handleObj.selector ),
+					jQuery.extend({}, handleObj, {handler: liveHandler, guid: handleObj.handler.guid}) );
+			},
+
+			remove: function( handleObj ) {
+				jQuery.event.remove( this, liveConvert( handleObj.origType, handleObj.selector ), handleObj );
+			}
+		},
+
+		beforeunload: {
+			setup: function( data, namespaces, eventHandle ) {
+				// We only want to do this special case on windows
+				if ( jQuery.isWindow( this ) ) {
+					this.onbeforeunload = eventHandle;
+				}
+			},
+
+			teardown: function( namespaces, eventHandle ) {
+				if ( this.onbeforeunload === eventHandle ) {
+					this.onbeforeunload = null;
+				}
+			}
+		}
+	}
+};
+
+jQuery.removeEvent = document.removeEventListener ?
+	function( elem, type, handle ) {
+		if ( elem.removeEventListener ) {
+			elem.removeEventListener( type, handle, false );
+		}
+	} :
+	function( elem, type, handle ) {
+		if ( elem.detachEvent ) {
+			elem.detachEvent( "on" + type, handle );
+		}
+	};
+
+jQuery.Event = function( src, props ) {
+	// Allow instantiation without the 'new' keyword
+	if ( !this.preventDefault ) {
+		return new jQuery.Event( src, props );
+	}
+
+	// Event object
+	if ( src && src.type ) {
+		this.originalEvent = src;
+		this.type = src.type;
+
+		// Events bubbling up the document may have been marked as prevented
+		// by a handler lower down the tree; reflect the correct value.
+		this.isDefaultPrevented = (src.defaultPrevented || src.returnValue === false ||
+			src.getPreventDefault && src.getPreventDefault()) ? returnTrue : returnFalse;
+
+	// Event type
+	} else {
+		this.type = src;
+	}
+
+	// Put explicitly provided properties onto the event object
+	if ( props ) {
+		jQuery.extend( this, props );
+	}
+
+	// timeStamp is buggy for some events on Firefox(#3843)
+	// So we won't rely on the native value
+	this.timeStamp = jQuery.now();
+
+	// Mark it as fixed
+	this[ jQuery.expando ] = true;
+};
+
+function returnFalse() {
+	return false;
+}
+function returnTrue() {
+	return true;
+}
+
+// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding
+// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html
+jQuery.Event.prototype = {
+	preventDefault: function() {
+		this.isDefaultPrevented = returnTrue;
+
+		var e = this.originalEvent;
+		if ( !e ) {
+			return;
+		}
+
+		// if preventDefault exists run it on the original event
+		if ( e.preventDefault ) {
+			e.preventDefault();
+
+		// otherwise set the returnValue property of the original event to false (IE)
+		} else {
+			e.returnValue = false;
+		}
+	},
+	stopPropagation: function() {
+		this.isPropagationStopped = returnTrue;
+
+		var e = this.originalEvent;
+		if ( !e ) {
+			return;
+		}
+		// if stopPropagation exists run it on the original event
+		if ( e.stopPropagation ) {
+			e.stopPropagation();
+		}
+		// otherwise set the cancelBubble property of the original event to true (IE)
+		e.cancelBubble = true;
+	},
+	stopImmediatePropagation: function() {
+		this.isImmediatePropagationStopped = returnTrue;
+		this.stopPropagation();
+	},
+	isDefaultPrevented: returnFalse,
+	isPropagationStopped: returnFalse,
+	isImmediatePropagationStopped: returnFalse
+};
+
+// Checks if an event happened on an element within another element
+// Used in jQuery.event.special.mouseenter and mouseleave handlers
+var withinElement = function( event ) {
+
+	// Check if mouse(over|out) are still within the same parent element
+	var related = event.relatedTarget,
+		inside = false,
+		eventType = event.type;
+
+	event.type = event.data;
+
+	if ( related !== this ) {
+
+		if ( related ) {
+			inside = jQuery.contains( this, related );
+		}
+
+		if ( !inside ) {
+
+			jQuery.event.handle.apply( this, arguments );
+
+			event.type = eventType;
+		}
+	}
+},
+
+// In case of event delegation, we only need to rename the event.type,
+// liveHandler will take care of the rest.
+delegate = function( event ) {
+	event.type = event.data;
+	jQuery.event.handle.apply( this, arguments );
+};
+
+// Create mouseenter and mouseleave events
+jQuery.each({
+	mouseenter: "mouseover",
+	mouseleave: "mouseout"
+}, function( orig, fix ) {
+	jQuery.event.special[ orig ] = {
+		setup: function( data ) {
+			jQuery.event.add( this, fix, data && data.selector ? delegate : withinElement, orig );
+		},
+		teardown: function( data ) {
+			jQuery.event.remove( this, fix, data && data.selector ? delegate : withinElement );
+		}
+	};
+});
+
+// submit delegation
+if ( !jQuery.support.submitBubbles ) {
+
+	jQuery.event.special.submit = {
+		setup: function( data, namespaces ) {
+			if ( !jQuery.nodeName( this, "form" ) ) {
+				jQuery.event.add(this, "click.specialSubmit", function( e ) {
+					var elem = e.target,
+						type = elem.type;
+
+					if ( (type === "submit" || type === "image") && jQuery( elem ).closest("form").length ) {
+						trigger( "submit", this, arguments );
+					}
+				});
+
+				jQuery.event.add(this, "keypress.specialSubmit", function( e ) {
+					var elem = e.target,
+						type = elem.type;
+
+					if ( (type === "text" || type === "password") && jQuery( elem ).closest("form").length && e.keyCode === 13 ) {
+						trigger( "submit", this, arguments );
+					}
+				});
+
+			} else {
+				return false;
+			}
+		},
+
+		teardown: function( namespaces ) {
+			jQuery.event.remove( this, ".specialSubmit" );
+		}
+	};
+
+}
+
+// change delegation, happens here so we have bind.
+if ( !jQuery.support.changeBubbles ) {
+
+	var changeFilters,
+
+	getVal = function( elem ) {
+		var type = elem.type, val = elem.value;
+
+		if ( type === "radio" || type === "checkbox" ) {
+			val = elem.checked;
+
+		} else if ( type === "select-multiple" ) {
+			val = elem.selectedIndex > -1 ?
+				jQuery.map( elem.options, function( elem ) {
+					return elem.selected;
+				}).join("-") :
+				"";
+
+		} else if ( jQuery.nodeName( elem, "select" ) ) {
+			val = elem.selectedIndex;
+		}
+
+		return val;
+	},
+
+	testChange = function testChange( e ) {
+		var elem = e.target, data, val;
+
+		if ( !rformElems.test( elem.nodeName ) || elem.readOnly ) {
+			return;
+		}
+
+		data = jQuery._data( elem, "_change_data" );
+		val = getVal(elem);
+
+		// the current data will be also retrieved by beforeactivate
+		if ( e.type !== "focusout" || elem.type !== "radio" ) {
+			jQuery._data( elem, "_change_data", val );
+		}
+
+		if ( data === undefined || val === data ) {
+			return;
+		}
+
+		if ( data != null || val ) {
+			e.type = "change";
+			e.liveFired = undefined;
+			jQuery.event.trigger( e, arguments[1], elem );
+		}
+	};
+
+	jQuery.event.special.change = {
+		filters: {
+			focusout: testChange,
+
+			beforedeactivate: testChange,
+
+			click: function( e ) {
+				var elem = e.target, type = jQuery.nodeName( elem, "input" ) ? elem.type : "";
+
+				if ( type === "radio" || type === "checkbox" || jQuery.nodeName( elem, "select" ) ) {
+					testChange.call( this, e );
+				}
+			},
+
+			// Change has to be called before submit
+			// Keydown will be called before keypress, which is used in submit-event delegation
+			keydown: function( e ) {
+				var elem = e.target, type = jQuery.nodeName( elem, "input" ) ? elem.type : "";
+
+				if ( (e.keyCode === 13 && !jQuery.nodeName( elem, "textarea" ) ) ||
+					(e.keyCode === 32 && (type === "checkbox" || type === "radio")) ||
+					type === "select-multiple" ) {
+					testChange.call( this, e );
+				}
+			},
+
+			// Beforeactivate happens also before the previous element is blurred
+			// with this event you can't trigger a change event, but you can store
+			// information
+			beforeactivate: function( e ) {
+				var elem = e.target;
+				jQuery._data( elem, "_change_data", getVal(elem) );
+			}
+		},
+
+		setup: function( data, namespaces ) {
+			if ( this.type === "file" ) {
+				return false;
+			}
+
+			for ( var type in changeFilters ) {
+				jQuery.event.add( this, type + ".specialChange", changeFilters[type] );
+			}
+
+			return rformElems.test( this.nodeName );
+		},
+
+		teardown: function( namespaces ) {
+			jQuery.event.remove( this, ".specialChange" );
+
+			return rformElems.test( this.nodeName );
+		}
+	};
+
+	changeFilters = jQuery.event.special.change.filters;
+
+	// Handle when the input is .focus()'d
+	changeFilters.focus = changeFilters.beforeactivate;
+}
+
+function trigger( type, elem, args ) {
+	// Piggyback on a donor event to simulate a different one.
+	// Fake originalEvent to avoid donor's stopPropagation, but if the
+	// simulated event prevents default then we do the same on the donor.
+	// Don't pass args or remember liveFired; they apply to the donor event.
+	var event = jQuery.extend( {}, args[ 0 ] );
+	event.type = type;
+	event.originalEvent = {};
+	event.liveFired = undefined;
+	jQuery.event.handle.call( elem, event );
+	if ( event.isDefaultPrevented() ) {
+		args[ 0 ].preventDefault();
+	}
+}
+
+// Create "bubbling" focus and blur events
+if ( !jQuery.support.focusinBubbles ) {
+	jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) {
+
+		// Attach a single capturing handler while someone wants focusin/focusout
+		var attaches = 0;
+
+		jQuery.event.special[ fix ] = {
+			setup: function() {
+				if ( attaches++ === 0 ) {
+					document.addEventListener( orig, handler, true );
+				}
+			},
+			teardown: function() {
+				if ( --attaches === 0 ) {
+					document.removeEventListener( orig, handler, true );
+				}
+			}
+		};
+
+		function handler( donor ) {
+			// Donor event is always a native one; fix it and switch its type.
+			// Let focusin/out handler cancel the donor focus/blur event.
+			var e = jQuery.event.fix( donor );
+			e.type = fix;
+			e.originalEvent = {};
+			jQuery.event.trigger( e, null, e.target );
+			if ( e.isDefaultPrevented() ) {
+				donor.preventDefault();
+			}
+		}
+	});
+}
+
+jQuery.each(["bind", "one"], function( i, name ) {
+	jQuery.fn[ name ] = function( type, data, fn ) {
+		var handler;
+
+		// Handle object literals
+		if ( typeof type === "object" ) {
+			for ( var key in type ) {
+				this[ name ](key, data, type[key], fn);
+			}
+			return this;
+		}
+
+		if ( arguments.length === 2 || data === false ) {
+			fn = data;
+			data = undefined;
+		}
+
+		if ( name === "one" ) {
+			handler = function( event ) {
+				jQuery( this ).unbind( event, handler );
+				return fn.apply( this, arguments );
+			};
+			handler.guid = fn.guid || jQuery.guid++;
+		} else {
+			handler = fn;
+		}
+
+		if ( type === "unload" && name !== "one" ) {
+			this.one( type, data, fn );
+
+		} else {
+			for ( var i = 0, l = this.length; i < l; i++ ) {
+				jQuery.event.add( this[i], type, handler, data );
+			}
+		}
+
+		return this;
+	};
+});
+
+jQuery.fn.extend({
+	unbind: function( type, fn ) {
+		// Handle object literals
+		if ( typeof type === "object" && !type.preventDefault ) {
+			for ( var key in type ) {
+				this.unbind(key, type[key]);
+			}
+
+		} else {
+			for ( var i = 0, l = this.length; i < l; i++ ) {
+				jQuery.event.remove( this[i], type, fn );
+			}
+		}
+
+		return this;
+	},
+
+	delegate: function( selector, types, data, fn ) {
+		return this.live( types, data, fn, selector );
+	},
+
+	undelegate: function( selector, types, fn ) {
+		if ( arguments.length === 0 ) {
+			return this.unbind( "live" );
+
+		} else {
+			return this.die( types, null, fn, selector );
+		}
+	},
+
+	trigger: function( type, data ) {
+		return this.each(function() {
+			jQuery.event.trigger( type, data, this );
+		});
+	},
+
+	triggerHandler: function( type, data ) {
+		if ( this[0] ) {
+			return jQuery.event.trigger( type, data, this[0], true );
+		}
+	},
+
+	toggle: function( fn ) {
+		// Save reference to arguments for access in closure
+		var args = arguments,
+			guid = fn.guid || jQuery.guid++,
+			i = 0,
+			toggler = function( event ) {
+				// Figure out which function to execute
+				var lastToggle = ( jQuery.data( this, "lastToggle" + fn.guid ) || 0 ) % i;
+				jQuery.data( this, "lastToggle" + fn.guid, lastToggle + 1 );
+
+				// Make sure that clicks stop
+				event.preventDefault();
+
+				// and execute the function
+				return args[ lastToggle ].apply( this, arguments ) || false;
+			};
+
+		// link all the functions, so any of them can unbind this click handler
+		toggler.guid = guid;
+		while ( i < args.length ) {
+			args[ i++ ].guid = guid;
+		}
+
+		return this.click( toggler );
+	},
+
+	hover: function( fnOver, fnOut ) {
+		return this.mouseenter( fnOver ).mouseleave( fnOut || fnOver );
+	}
+});
+
+var liveMap = {
+	focus: "focusin",
+	blur: "focusout",
+	mouseenter: "mouseover",
+	mouseleave: "mouseout"
+};
+
+jQuery.each(["live", "die"], function( i, name ) {
+	jQuery.fn[ name ] = function( types, data, fn, origSelector /* Internal Use Only */ ) {
+		var type, i = 0, match, namespaces, preType,
+			selector = origSelector || this.selector,
+			context = origSelector ? this : jQuery( this.context );
+
+		if ( typeof types === "object" && !types.preventDefault ) {
+			for ( var key in types ) {
+				context[ name ]( key, data, types[key], selector );
+			}
+
+			return this;
+		}
+
+		if ( name === "die" && !types &&
+					origSelector && origSelector.charAt(0) === "." ) {
+
+			context.unbind( origSelector );
+
+			return this;
+		}
+
+		if ( data === false || jQuery.isFunction( data ) ) {
+			fn = data || returnFalse;
+			data = undefined;
+		}
+
+		types = (types || "").split(" ");
+
+		while ( (type = types[ i++ ]) != null ) {
+			match = rnamespaces.exec( type );
+			namespaces = "";
+
+			if ( match )  {
+				namespaces = match[0];
+				type = type.replace( rnamespaces, "" );
+			}
+
+			if ( type === "hover" ) {
+				types.push( "mouseenter" + namespaces, "mouseleave" + namespaces );
+				continue;
+			}
+
+			preType = type;
+
+			if ( liveMap[ type ] ) {
+				types.push( liveMap[ type ] + namespaces );
+				type = type + namespaces;
+
+			} else {
+				type = (liveMap[ type ] || type) + namespaces;
+			}
+
+			if ( name === "live" ) {
+				// bind live handler
+				for ( var j = 0, l = context.length; j < l; j++ ) {
+					jQuery.event.add( context[j], "live." + liveConvert( type, selector ),
+						{ data: data, selector: selector, handler: fn, origType: type, origHandler: fn, preType: preType } );
+				}
+
+			} else {
+				// unbind live handler
+				context.unbind( "live." + liveConvert( type, selector ), fn );
+			}
+		}
+
+		return this;
+	};
+});
+
+function liveHandler( event ) {
+	var stop, maxLevel, related, match, handleObj, elem, j, i, l, data, close, namespace, ret,
+		elems = [],
+		selectors = [],
+		events = jQuery._data( this, "events" );
+
+	// Make sure we avoid non-left-click bubbling in Firefox (#3861) and disabled elements in IE (#6911)
+	if ( event.liveFired === this || !events || !events.live || event.target.disabled || event.button && event.type === "click" ) {
+		return;
+	}
+
+	if ( event.namespace ) {
+		namespace = new RegExp("(^|\\.)" + event.namespace.split(".").join("\\.(?:.*\\.)?") + "(\\.|$)");
+	}
+
+	event.liveFired = this;
+
+	var live = events.live.slice(0);
+
+	for ( j = 0; j < live.length; j++ ) {
+		handleObj = live[j];
+
+		if ( handleObj.origType.replace( rnamespaces, "" ) === event.type ) {
+			selectors.push( handleObj.selector );
+
+		} else {
+			live.splice( j--, 1 );
+		}
+	}
+
+	match = jQuery( event.target ).closest( selectors, event.currentTarget );
+
+	for ( i = 0, l = match.length; i < l; i++ ) {
+		close = match[i];
+
+		for ( j = 0; j < live.length; j++ ) {
+			handleObj = live[j];
+
+			if ( close.selector === handleObj.selector && (!namespace || namespace.test( handleObj.namespace )) && !close.elem.disabled ) {
+				elem = close.elem;
+				related = null;
+
+				// Those two events require additional checking
+				if ( handleObj.preType === "mouseenter" || handleObj.preType === "mouseleave" ) {
+					event.type = handleObj.preType;
+					related = jQuery( event.relatedTarget ).closest( handleObj.selector )[0];
+
+					// Make sure not to accidentally match a child element with the same selector
+					if ( related && jQuery.contains( elem, related ) ) {
+						related = elem;
+					}
+				}
+
+				if ( !related || related !== elem ) {
+					elems.push({ elem: elem, handleObj: handleObj, level: close.level });
+				}
+			}
+		}
+	}
+
+	for ( i = 0, l = elems.length; i < l; i++ ) {
+		match = elems[i];
+
+		if ( maxLevel && match.level > maxLevel ) {
+			break;
+		}
+
+		event.currentTarget = match.elem;
+		event.data = match.handleObj.data;
+		event.handleObj = match.handleObj;
+
+		ret = match.handleObj.origHandler.apply( match.elem, arguments );
+
+		if ( ret === false || event.isPropagationStopped() ) {
+			maxLevel = match.level;
+
+			if ( ret === false ) {
+				stop = false;
+			}
+			if ( event.isImmediatePropagationStopped() ) {
+				break;
+			}
+		}
+	}
+
+	return stop;
+}
+
+function liveConvert( type, selector ) {
+	return (type && type !== "*" ? type + "." : "") + selector.replace(rperiod, "`").replace(rspaces, "&");
+}
+
+jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " +
+	"mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " +
+	"change select submit keydown keypress keyup error").split(" "), function( i, name ) {
+
+	// Handle event binding
+	jQuery.fn[ name ] = function( data, fn ) {
+		if ( fn == null ) {
+			fn = data;
+			data = null;
+		}
+
+		return arguments.length > 0 ?
+			this.bind( name, data, fn ) :
+			this.trigger( name );
+	};
+
+	if ( jQuery.attrFn ) {
+		jQuery.attrFn[ name ] = true;
+	}
+});
+
+
+
+/*!
+ * Sizzle CSS Selector Engine
+ *  Copyright 2011, The Dojo Foundation
+ *  Released under the MIT, BSD, and GPL Licenses.
+ *  More information: http://sizzlejs.com/
+ */
+(function(){
+
+var chunker = /((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,
+	done = 0,
+	toString = Object.prototype.toString,
+	hasDuplicate = false,
+	baseHasDuplicate = true,
+	rBackslash = /\\/g,
+	rNonWord = /\W/;
+
+// Here we check if the JavaScript engine is using some sort of
+// optimization where it does not always call our comparision
+// function. If that is the case, discard the hasDuplicate value.
+//   Thus far that includes Google Chrome.
+[0, 0].sort(function() {
+	baseHasDuplicate = false;
+	return 0;
+});
+
+var Sizzle = function( selector, context, results, seed ) {
+	results = results || [];
+	context = context || document;
+
+	var origContext = context;
+
+	if ( context.nodeType !== 1 && context.nodeType !== 9 ) {
+		return [];
+	}
+	
+	if ( !selector || typeof selector !== "string" ) {
+		return results;
+	}
+
+	var m, set, checkSet, extra, ret, cur, pop, i,
+		prune = true,
+		contextXML = Sizzle.isXML( context ),
+		parts = [],
+		soFar = selector;
+	
+	// Reset the position of the chunker regexp (start from head)
+	do {
+		chunker.exec( "" );
+		m = chunker.exec( soFar );
+
+		if ( m ) {
+			soFar = m[3];
+		
+			parts.push( m[1] );
+		
+			if ( m[2] ) {
+				extra = m[3];
+				break;
+			}
+		}
+	} while ( m );
+
+	if ( parts.length > 1 && origPOS.exec( selector ) ) {
+
+		if ( parts.length === 2 && Expr.relative[ parts[0] ] ) {
+			set = posProcess( parts[0] + parts[1], context );
+
+		} else {
+			set = Expr.relative[ parts[0] ] ?
+				[ context ] :
+				Sizzle( parts.shift(), context );
+
+			while ( parts.length ) {
+				selector = parts.shift();
+
+				if ( Expr.relative[ selector ] ) {
+					selector += parts.shift();
+				}
+				
+				set = posProcess( selector, set );
+			}
+		}
+
+	} else {
+		// Take a shortcut and set the context if the root selector is an ID
+		// (but not if it'll be faster if the inner selector is an ID)
+		if ( !seed && parts.length > 1 && context.nodeType === 9 && !contextXML &&
+				Expr.match.ID.test(parts[0]) && !Expr.match.ID.test(parts[parts.length - 1]) ) {
+
+			ret = Sizzle.find( parts.shift(), context, contextXML );
+			context = ret.expr ?
+				Sizzle.filter( ret.expr, ret.set )[0] :
+				ret.set[0];
+		}
+
+		if ( context ) {
+			ret = seed ?
+				{ expr: parts.pop(), set: makeArray(seed) } :
+				Sizzle.find( parts.pop(), parts.length === 1 && (parts[0] === "~" || parts[0] === "+") && context.parentNode ? context.parentNode : context, contextXML );
+
+			set = ret.expr ?
+				Sizzle.filter( ret.expr, ret.set ) :
+				ret.set;
+
+			if ( parts.length > 0 ) {
+				checkSet = makeArray( set );
+
+			} else {
+				prune = false;
+			}
+
+			while ( parts.length ) {
+				cur = parts.pop();
+				pop = cur;
+
+				if ( !Expr.relative[ cur ] ) {
+					cur = "";
+				} else {
+					pop = parts.pop();
+				}
+
+				if ( pop == null ) {
+					pop = context;
+				}
+
+				Expr.relative[ cur ]( checkSet, pop, contextXML );
+			}
+
+		} else {
+			checkSet = parts = [];
+		}
+	}
+
+	if ( !checkSet ) {
+		checkSet = set;
+	}
+
+	if ( !checkSet ) {
+		Sizzle.error( cur || selector );
+	}
+
+	if ( toString.call(checkSet) === "[object Array]" ) {
+		if ( !prune ) {
+			results.push.apply( results, checkSet );
+
+		} else if ( context && context.nodeType === 1 ) {
+			for ( i = 0; checkSet[i] != null; i++ ) {
+				if ( checkSet[i] && (checkSet[i] === true || checkSet[i].nodeType === 1 && Sizzle.contains(context, checkSet[i])) ) {
+					results.push( set[i] );
+				}
+			}
+
+		} else {
+			for ( i = 0; checkSet[i] != null; i++ ) {
+				if ( checkSet[i] && checkSet[i].nodeType === 1 ) {
+					results.push( set[i] );
+				}
+			}
+		}
+
+	} else {
+		makeArray( checkSet, results );
+	}
+
+	if ( extra ) {
+		Sizzle( extra, origContext, results, seed );
+		Sizzle.uniqueSort( results );
+	}
+
+	return results;
+};
+
+Sizzle.uniqueSort = function( results ) {
+	if ( sortOrder ) {
+		hasDuplicate = baseHasDuplicate;
+		results.sort( sortOrder );
+
+		if ( hasDuplicate ) {
+			for ( var i = 1; i < results.length; i++ ) {
+				if ( results[i] === results[ i - 1 ] ) {
+					results.splice( i--, 1 );
+				}
+			}
+		}
+	}
+
+	return results;
+};
+
+Sizzle.matches = function( expr, set ) {
+	return Sizzle( expr, null, null, set );
+};
+
+Sizzle.matchesSelector = function( node, expr ) {
+	return Sizzle( expr, null, null, [node] ).length > 0;
+};
+
+Sizzle.find = function( expr, context, isXML ) {
+	var set;
+
+	if ( !expr ) {
+		return [];
+	}
+
+	for ( var i = 0, l = Expr.order.length; i < l; i++ ) {
+		var match,
+			type = Expr.order[i];
+		
+		if ( (match = Expr.leftMatch[ type ].exec( expr )) ) {
+			var left = match[1];
+			match.splice( 1, 1 );
+
+			if ( left.substr( left.length - 1 ) !== "\\" ) {
+				match[1] = (match[1] || "").replace( rBackslash, "" );
+				set = Expr.find[ type ]( match, context, isXML );
+
+				if ( set != null ) {
+					expr = expr.replace( Expr.match[ type ], "" );
+					break;
+				}
+			}
+		}
+	}
+
+	if ( !set ) {
+		set = typeof context.getElementsByTagName !== "undefined" ?
+			context.getElementsByTagName( "*" ) :
+			[];
+	}
+
+	return { set: set, expr: expr };
+};
+
+Sizzle.filter = function( expr, set, inplace, not ) {
+	var match, anyFound,
+		old = expr,
+		result = [],
+		curLoop = set,
+		isXMLFilter = set && set[0] && Sizzle.isXML( set[0] );
+
+	while ( expr && set.length ) {
+		for ( var type in Expr.filter ) {
+			if ( (match = Expr.leftMatch[ type ].exec( expr )) != null && match[2] ) {
+				var found, item,
+					filter = Expr.filter[ type ],
+					left = match[1];
+
+				anyFound = false;
+
+				match.splice(1,1);
+
+				if ( left.substr( left.length - 1 ) === "\\" ) {
+					continue;
+				}
+
+				if ( curLoop === result ) {
+					result = [];
+				}
+
+				if ( Expr.preFilter[ type ] ) {
+					match = Expr.preFilter[ type ]( match, curLoop, inplace, result, not, isXMLFilter );
+
+					if ( !match ) {
+						anyFound = found = true;
+
+					} else if ( match === true ) {
+						continue;
+					}
+				}
+
+				if ( match ) {
+					for ( var i = 0; (item = curLoop[i]) != null; i++ ) {
+						if ( item ) {
+							found = filter( item, match, i, curLoop );
+							var pass = not ^ !!found;
+
+							if ( inplace && found != null ) {
+								if ( pass ) {
+									anyFound = true;
+
+								} else {
+									curLoop[i] = false;
+								}
+
+							} else if ( pass ) {
+								result.push( item );
+								anyFound = true;
+							}
+						}
+					}
+				}
+
+				if ( found !== undefined ) {
+					if ( !inplace ) {
+						curLoop = result;
+					}
+
+					expr = expr.replace( Expr.match[ type ], "" );
+
+					if ( !anyFound ) {
+						return [];
+					}
+
+					break;
+				}
+			}
+		}
+
+		// Improper expression
+		if ( expr === old ) {
+			if ( anyFound == null ) {
+				Sizzle.error( expr );
+
+			} else {
+				break;
+			}
+		}
+
+		old = expr;
+	}
+
+	return curLoop;
+};
+
+Sizzle.error = function( msg ) {
+	throw "Syntax error, unrecognized expression: " + msg;
+};
+
+var Expr = Sizzle.selectors = {
+	order: [ "ID", "NAME", "TAG" ],
+
+	match: {
+		ID: /#((?:[\w\u00c0-\uFFFF\-]|\\.)+)/,
+		CLASS: /\.((?:[\w\u00c0-\uFFFF\-]|\\.)+)/,
+		NAME: /\[name=['"]*((?:[\w\u00c0-\uFFFF\-]|\\.)+)['"]*\]/,
+		ATTR: /\[\s*((?:[\w\u00c0-\uFFFF\-]|\\.)+)\s*(?:(\S?=)\s*(?:(['"])(.*?)\3|(#?(?:[\w\u00c0-\uFFFF\-]|\\.)*)|)|)\s*\]/,
+		TAG: /^((?:[\w\u00c0-\uFFFF\*\-]|\\.)+)/,
+		CHILD: /:(only|nth|last|first)-child(?:\(\s*(even|odd|(?:[+\-]?\d+|(?:[+\-]?\d*)?n\s*(?:[+\-]\s*\d+)?))\s*\))?/,
+		POS: /:(nth|eq|gt|lt|first|last|even|odd)(?:\((\d*)\))?(?=[^\-]|$)/,
+		PSEUDO: /:((?:[\w\u00c0-\uFFFF\-]|\\.)+)(?:\((['"]?)((?:\([^\)]+\)|[^\(\)]*)+)\2\))?/
+	},
+
+	leftMatch: {},
+
+	attrMap: {
+		"class": "className",
+		"for": "htmlFor"
+	},
+
+	attrHandle: {
+		href: function( elem ) {
+			return elem.getAttribute( "href" );
+		},
+		type: function( elem ) {
+			return elem.getAttribute( "type" );
+		}
+	},
+
+	relative: {
+		"+": function(checkSet, part){
+			var isPartStr = typeof part === "string",
+				isTag = isPartStr && !rNonWord.test( part ),
+				isPartStrNotTag = isPartStr && !isTag;
+
+			if ( isTag ) {
+				part = part.toLowerCase();
+			}
+
+			for ( var i = 0, l = checkSet.length, elem; i < l; i++ ) {
+				if ( (elem = checkSet[i]) ) {
+					while ( (elem = elem.previousSibling) && elem.nodeType !== 1 ) {}
+
+					checkSet[i] = isPartStrNotTag || elem && elem.nodeName.toLowerCase() === part ?
+						elem || false :
+						elem === part;
+				}
+			}
+
+			if ( isPartStrNotTag ) {
+				Sizzle.filter( part, checkSet, true );
+			}
+		},
+
+		">": function( checkSet, part ) {
+			var elem,
+				isPartStr = typeof part === "string",
+				i = 0,
+				l = checkSet.length;
+
+			if ( isPartStr && !rNonWord.test( part ) ) {
+				part = part.toLowerCase();
+
+				for ( ; i < l; i++ ) {
+					elem = checkSet[i];
+
+					if ( elem ) {
+						var parent = elem.parentNode;
+						checkSet[i] = parent.nodeName.toLowerCase() === part ? parent : false;
+					}
+				}
+
+			} else {
+				for ( ; i < l; i++ ) {
+					elem = checkSet[i];
+
+					if ( elem ) {
+						checkSet[i] = isPartStr ?
+							elem.parentNode :
+							elem.parentNode === part;
+					}
+				}
+
+				if ( isPartStr ) {
+					Sizzle.filter( part, checkSet, true );
+				}
+			}
+		},
+
+		"": function(checkSet, part, isXML){
+			var nodeCheck,
+				doneName = done++,
+				checkFn = dirCheck;
+
+			if ( typeof part === "string" && !rNonWord.test( part ) ) {
+				part = part.toLowerCase();
+				nodeCheck = part;
+				checkFn = dirNodeCheck;
+			}
+
+			checkFn( "parentNode", part, doneName, checkSet, nodeCheck, isXML );
+		},
+
+		"~": function( checkSet, part, isXML ) {
+			var nodeCheck,
+				doneName = done++,
+				checkFn = dirCheck;
+
+			if ( typeof part === "string" && !rNonWord.test( part ) ) {
+				part = part.toLowerCase();
+				nodeCheck = part;
+				checkFn = dirNodeCheck;
+			}
+
+			checkFn( "previousSibling", part, doneName, checkSet, nodeCheck, isXML );
+		}
+	},
+
+	find: {
+		ID: function( match, context, isXML ) {
+			if ( typeof context.getElementById !== "undefined" && !isXML ) {
+				var m = context.getElementById(match[1]);
+				// Check parentNode to catch when Blackberry 4.6 returns
+				// nodes that are no longer in the document #6963
+				return m && m.parentNode ? [m] : [];
+			}
+		},
+
+		NAME: function( match, context ) {
+			if ( typeof context.getElementsByName !== "undefined" ) {
+				var ret = [],
+					results = context.getElementsByName( match[1] );
+
+				for ( var i = 0, l = results.length; i < l; i++ ) {
+					if ( results[i].getAttribute("name") === match[1] ) {
+						ret.push( results[i] );
+					}
+				}
+
+				return ret.length === 0 ? null : ret;
+			}
+		},
+
+		TAG: function( match, context ) {
+			if ( typeof context.getElementsByTagName !== "undefined" ) {
+				return context.getElementsByTagName( match[1] );
+			}
+		}
+	},
+	preFilter: {
+		CLASS: function( match, curLoop, inplace, result, not, isXML ) {
+			match = " " + match[1].replace( rBackslash, "" ) + " ";
+
+			if ( isXML ) {
+				return match;
+			}
+
+			for ( var i = 0, elem; (elem = curLoop[i]) != null; i++ ) {
+				if ( elem ) {
+					if ( not ^ (elem.className && (" " + elem.className + " ").replace(/[\t\n\r]/g, " ").indexOf(match) >= 0) ) {
+						if ( !inplace ) {
+							result.push( elem );
+						}
+
+					} else if ( inplace ) {
+						curLoop[i] = false;
+					}
+				}
+			}
+
+			return false;
+		},
+
+		ID: function( match ) {
+			return match[1].replace( rBackslash, "" );
+		},
+
+		TAG: function( match, curLoop ) {
+			return match[1].replace( rBackslash, "" ).toLowerCase();
+		},
+
+		CHILD: function( match ) {
+			if ( match[1] === "nth" ) {
+				if ( !match[2] ) {
+					Sizzle.error( match[0] );
+				}
+
+				match[2] = match[2].replace(/^\+|\s*/g, '');
+
+				// parse equations like 'even', 'odd', '5', '2n', '3n+2', '4n-1', '-n+6'
+				var test = /(-?)(\d*)(?:n([+\-]?\d*))?/.exec(
+					match[2] === "even" && "2n" || match[2] === "odd" && "2n+1" ||
+					!/\D/.test( match[2] ) && "0n+" + match[2] || match[2]);
+
+				// calculate the numbers (first)n+(last) including if they are negative
+				match[2] = (test[1] + (test[2] || 1)) - 0;
+				match[3] = test[3] - 0;
+			}
+			else if ( match[2] ) {
+				Sizzle.error( match[0] );
+			}
+
+			// TODO: Move to normal caching system
+			match[0] = done++;
+
+			return match;
+		},
+
+		ATTR: function( match, curLoop, inplace, result, not, isXML ) {
+			var name = match[1] = match[1].replace( rBackslash, "" );
+			
+			if ( !isXML && Expr.attrMap[name] ) {
+				match[1] = Expr.attrMap[name];
+			}
+
+			// Handle if an un-quoted value was used
+			match[4] = ( match[4] || match[5] || "" ).replace( rBackslash, "" );
+
+			if ( match[2] === "~=" ) {
+				match[4] = " " + match[4] + " ";
+			}
+
+			return match;
+		},
+
+		PSEUDO: function( match, curLoop, inplace, result, not ) {
+			if ( match[1] === "not" ) {
+				// If we're dealing with a complex expression, or a simple one
+				if ( ( chunker.exec(match[3]) || "" ).length > 1 || /^\w/.test(match[3]) ) {
+					match[3] = Sizzle(match[3], null, null, curLoop);
+
+				} else {
+					var ret = Sizzle.filter(match[3], curLoop, inplace, true ^ not);
+
+					if ( !inplace ) {
+						result.push.apply( result, ret );
+					}
+
+					return false;
+				}
+
+			} else if ( Expr.match.POS.test( match[0] ) || Expr.match.CHILD.test( match[0] ) ) {
+				return true;
+			}
+			
+			return match;
+		},
+
+		POS: function( match ) {
+			match.unshift( true );
+
+			return match;
+		}
+	},
+	
+	filters: {
+		enabled: function( elem ) {
+			return elem.disabled === false && elem.type !== "hidden";
+		},
+
+		disabled: function( elem ) {
+			return elem.disabled === true;
+		},
+
+		checked: function( elem ) {
+			return elem.checked === true;
+		},
+		
+		selected: function( elem ) {
+			// Accessing this property makes selected-by-default
+			// options in Safari work properly
+			if ( elem.parentNode ) {
+				elem.parentNode.selectedIndex;
+			}
+			
+			return elem.selected === true;
+		},
+
+		parent: function( elem ) {
+			return !!elem.firstChild;
+		},
+
+		empty: function( elem ) {
+			return !elem.firstChild;
+		},
+
+		has: function( elem, i, match ) {
+			return !!Sizzle( match[3], elem ).length;
+		},
+
+		header: function( elem ) {
+			return (/h\d/i).test( elem.nodeName );
+		},
+
+		text: function( elem ) {
+			var attr = elem.getAttribute( "type" ), type = elem.type;
+			// IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) 
+			// use getAttribute instead to test this case
+			return elem.nodeName.toLowerCase() === "input" && "text" === type && ( attr === type || attr === null );
+		},
+
+		radio: function( elem ) {
+			return elem.nodeName.toLowerCase() === "input" && "radio" === elem.type;
+		},
+
+		checkbox: function( elem ) {
+			return elem.nodeName.toLowerCase() === "input" && "checkbox" === elem.type;
+		},
+
+		file: function( elem ) {
+			return elem.nodeName.toLowerCase() === "input" && "file" === elem.type;
+		},
+
+		password: function( elem ) {
+			return elem.nodeName.toLowerCase() === "input" && "password" === elem.type;
+		},
+
+		submit: function( elem ) {
+			var name = elem.nodeName.toLowerCase();
+			return (name === "input" || name === "button") && "submit" === elem.type;
+		},
+
+		image: function( elem ) {
+			return elem.nodeName.toLowerCase() === "input" && "image" === elem.type;
+		},
+
+		reset: function( elem ) {
+			var name = elem.nodeName.toLowerCase();
+			return (name === "input" || name === "button") && "reset" === elem.type;
+		},
+
+		button: function( elem ) {
+			var name = elem.nodeName.toLowerCase();
+			return name === "input" && "button" === elem.type || name === "button";
+		},
+
+		input: function( elem ) {
+			return (/input|select|textarea|button/i).test( elem.nodeName );
+		},
+
+		focus: function( elem ) {
+			return elem === elem.ownerDocument.activeElement;
+		}
+	},
+	setFilters: {
+		first: function( elem, i ) {
+			return i === 0;
+		},
+
+		last: function( elem, i, match, array ) {
+			return i === array.length - 1;
+		},
+
+		even: function( elem, i ) {
+			return i % 2 === 0;
+		},
+
+		odd: function( elem, i ) {
+			return i % 2 === 1;
+		},
+
+		lt: function( elem, i, match ) {
+			return i < match[3] - 0;
+		},
+
+		gt: function( elem, i, match ) {
+			return i > match[3] - 0;
+		},
+
+		nth: function( elem, i, match ) {
+			return match[3] - 0 === i;
+		},
+
+		eq: function( elem, i, match ) {
+			return match[3] - 0 === i;
+		}
+	},
+	filter: {
+		PSEUDO: function( elem, match, i, array ) {
+			var name = match[1],
+				filter = Expr.filters[ name ];
+
+			if ( filter ) {
+				return filter( elem, i, match, array );
+
+			} else if ( name === "contains" ) {
+				return (elem.textContent || elem.innerText || Sizzle.getText([ elem ]) || "").indexOf(match[3]) >= 0;
+
+			} else if ( name === "not" ) {
+				var not = match[3];
+
+				for ( var j = 0, l = not.length; j < l; j++ ) {
+					if ( not[j] === elem ) {
+						return false;
+					}
+				}
+
+				return true;
+
+			} else {
+				Sizzle.error( name );
+			}
+		},
+
+		CHILD: function( elem, match ) {
+			var type = match[1],
+				node = elem;
+
+			switch ( type ) {
+				case "only":
+				case "first":
+					while ( (node = node.previousSibling) )	 {
+						if ( node.nodeType === 1 ) { 
+							return false; 
+						}
+					}
+
+					if ( type === "first" ) { 
+						return true; 
+					}
+
+					node = elem;
+
+				case "last":
+					while ( (node = node.nextSibling) )	 {
+						if ( node.nodeType === 1 ) { 
+							return false; 
+						}
+					}
+
+					return true;
+
+				case "nth":
+					var first = match[2],
+						last = match[3];
+
+					if ( first === 1 && last === 0 ) {
+						return true;
+					}
+					
+					var doneName = match[0],
+						parent = elem.parentNode;
+	
+					if ( parent && (parent.sizcache !== doneName || !elem.nodeIndex) ) {
+						var count = 0;
+						
+						for ( node = parent.firstChild; node; node = node.nextSibling ) {
+							if ( node.nodeType === 1 ) {
+								node.nodeIndex = ++count;
+							}
+						} 
+
+						parent.sizcache = doneName;
+					}
+					
+					var diff = elem.nodeIndex - last;
+
+					if ( first === 0 ) {
+						return diff === 0;
+
+					} else {
+						return ( diff % first === 0 && diff / first >= 0 );
+					}
+			}
+		},
+
+		ID: function( elem, match ) {
+			return elem.nodeType === 1 && elem.getAttribute("id") === match;
+		},
+
+		TAG: function( elem, match ) {
+			return (match === "*" && elem.nodeType === 1) || elem.nodeName.toLowerCase() === match;
+		},
+		
+		CLASS: function( elem, match ) {
+			return (" " + (elem.className || elem.getAttribute("class")) + " ")
+				.indexOf( match ) > -1;
+		},
+
+		ATTR: function( elem, match ) {
+			var name = match[1],
+				result = Expr.attrHandle[ name ] ?
+					Expr.attrHandle[ name ]( elem ) :
+					elem[ name ] != null ?
+						elem[ name ] :
+						elem.getAttribute( name ),
+				value = result + "",
+				type = match[2],
+				check = match[4];
+
+			return result == null ?
+				type === "!=" :
+				type === "=" ?
+				value === check :
+				type === "*=" ?
+				value.indexOf(check) >= 0 :
+				type === "~=" ?
+				(" " + value + " ").indexOf(check) >= 0 :
+				!check ?
+				value && result !== false :
+				type === "!=" ?
+				value !== check :
+				type === "^=" ?
+				value.indexOf(check) === 0 :
+				type === "$=" ?
+				value.substr(value.length - check.length) === check :
+				type === "|=" ?
+				value === check || value.substr(0, check.length + 1) === check + "-" :
+				false;
+		},
+
+		POS: function( elem, match, i, array ) {
+			var name = match[2],
+				filter = Expr.setFilters[ name ];
+
+			if ( filter ) {
+				return filter( elem, i, match, array );
+			}
+		}
+	}
+};
+
+var origPOS = Expr.match.POS,
+	fescape = function(all, num){
+		return "\\" + (num - 0 + 1);
+	};
+
+for ( var type in Expr.match ) {
+	Expr.match[ type ] = new RegExp( Expr.match[ type ].source + (/(?![^\[]*\])(?![^\(]*\))/.source) );
+	Expr.leftMatch[ type ] = new RegExp( /(^(?:.|\r|\n)*?)/.source + Expr.match[ type ].source.replace(/\\(\d+)/g, fescape) );
+}
+
+var makeArray = function( array, results ) {
+	array = Array.prototype.slice.call( array, 0 );
+
+	if ( results ) {
+		results.push.apply( results, array );
+		return results;
+	}
+	
+	return array;
+};
+
+// Perform a simple check to determine if the browser is capable of
+// converting a NodeList to an array using builtin methods.
+// Also verifies that the returned array holds DOM nodes
+// (which is not the case in the Blackberry browser)
+try {
+	Array.prototype.slice.call( document.documentElement.childNodes, 0 )[0].nodeType;
+
+// Provide a fallback method if it does not work
+} catch( e ) {
+	makeArray = function( array, results ) {
+		var i = 0,
+			ret = results || [];
+
+		if ( toString.call(array) === "[object Array]" ) {
+			Array.prototype.push.apply( ret, array );
+
+		} else {
+			if ( typeof array.length === "number" ) {
+				for ( var l = array.length; i < l; i++ ) {
+					ret.push( array[i] );
+				}
+
+			} else {
+				for ( ; array[i]; i++ ) {
+					ret.push( array[i] );
+				}
+			}
+		}
+
+		return ret;
+	};
+}
+
+var sortOrder, siblingCheck;
+
+if ( document.documentElement.compareDocumentPosition ) {
+	sortOrder = function( a, b ) {
+		if ( a === b ) {
+			hasDuplicate = true;
+			return 0;
+		}
+
+		if ( !a.compareDocumentPosition || !b.compareDocumentPosition ) {
+			return a.compareDocumentPosition ? -1 : 1;
+		}
+
+		return a.compareDocumentPosition(b) & 4 ? -1 : 1;
+	};
+
+} else {
+	sortOrder = function( a, b ) {
+		// The nodes are identical, we can exit early
+		if ( a === b ) {
+			hasDuplicate = true;
+			return 0;
+
+		// Fallback to using sourceIndex (in IE) if it's available on both nodes
+		} else if ( a.sourceIndex && b.sourceIndex ) {
+			return a.sourceIndex - b.sourceIndex;
+		}
+
+		var al, bl,
+			ap = [],
+			bp = [],
+			aup = a.parentNode,
+			bup = b.parentNode,
+			cur = aup;
+
+		// If the nodes are siblings (or identical) we can do a quick check
+		if ( aup === bup ) {
+			return siblingCheck( a, b );
+
+		// If no parents were found then the nodes are disconnected
+		} else if ( !aup ) {
+			return -1;
+
+		} else if ( !bup ) {
+			return 1;
+		}
+
+		// Otherwise they're somewhere else in the tree so we need
+		// to build up a full list of the parentNodes for comparison
+		while ( cur ) {
+			ap.unshift( cur );
+			cur = cur.parentNode;
+		}
+
+		cur = bup;
+
+		while ( cur ) {
+			bp.unshift( cur );
+			cur = cur.parentNode;
+		}
+
+		al = ap.length;
+		bl = bp.length;
+
+		// Start walking down the tree looking for a discrepancy
+		for ( var i = 0; i < al && i < bl; i++ ) {
+			if ( ap[i] !== bp[i] ) {
+				return siblingCheck( ap[i], bp[i] );
+			}
+		}
+
+		// We ended someplace up the tree so do a sibling check
+		return i === al ?
+			siblingCheck( a, bp[i], -1 ) :
+			siblingCheck( ap[i], b, 1 );
+	};
+
+	siblingCheck = function( a, b, ret ) {
+		if ( a === b ) {
+			return ret;
+		}
+
+		var cur = a.nextSibling;
+
+		while ( cur ) {
+			if ( cur === b ) {
+				return -1;
+			}
+
+			cur = cur.nextSibling;
+		}
+
+		return 1;
+	};
+}
+
+// Utility function for retreiving the text value of an array of DOM nodes
+Sizzle.getText = function( elems ) {
+	var ret = "", elem;
+
+	for ( var i = 0; elems[i]; i++ ) {
+		elem = elems[i];
+
+		// Get the text from text nodes and CDATA nodes
+		if ( elem.nodeType === 3 || elem.nodeType === 4 ) {
+			ret += elem.nodeValue;
+
+		// Traverse everything else, except comment nodes
+		} else if ( elem.nodeType !== 8 ) {
+			ret += Sizzle.getText( elem.childNodes );
+		}
+	}
+
+	return ret;
+};
+
+// Check to see if the browser returns elements by name when
+// querying by getElementById (and provide a workaround)
+(function(){
+	// We're going to inject a fake input element with a specified name
+	var form = document.createElement("div"),
+		id = "script" + (new Date()).getTime(),
+		root = document.documentElement;
+
+	form.innerHTML = "<a name='" + id + "'/>";
+
+	// Inject it into the root element, check its status, and remove it quickly
+	root.insertBefore( form, root.firstChild );
+
+	// The workaround has to do additional checks after a getElementById
+	// Which slows things down for other browsers (hence the branching)
+	if ( document.getElementById( id ) ) {
+		Expr.find.ID = function( match, context, isXML ) {
+			if ( typeof context.getElementById !== "undefined" && !isXML ) {
+				var m = context.getElementById(match[1]);
+
+				return m ?
+					m.id === match[1] || typeof m.getAttributeNode !== "undefined" && m.getAttributeNode("id").nodeValue === match[1] ?
+						[m] :
+						undefined :
+					[];
+			}
+		};
+
+		Expr.filter.ID = function( elem, match ) {
+			var node = typeof elem.getAttributeNode !== "undefined" && elem.getAttributeNode("id");
+
+			return elem.nodeType === 1 && node && node.nodeValue === match;
+		};
+	}
+
+	root.removeChild( form );
+
+	// release memory in IE
+	root = form = null;
+})();
+
+(function(){
+	// Check to see if the browser returns only elements
+	// when doing getElementsByTagName("*")
+
+	// Create a fake element
+	var div = document.createElement("div");
+	div.appendChild( document.createComment("") );
+
+	// Make sure no comments are found
+	if ( div.getElementsByTagName("*").length > 0 ) {
+		Expr.find.TAG = function( match, context ) {
+			var results = context.getElementsByTagName( match[1] );
+
+			// Filter out possible comments
+			if ( match[1] === "*" ) {
+				var tmp = [];
+
+				for ( var i = 0; results[i]; i++ ) {
+					if ( results[i].nodeType === 1 ) {
+						tmp.push( results[i] );
+					}
+				}
+
+				results = tmp;
+			}
+
+			return results;
+		};
+	}
+
+	// Check to see if an attribute returns normalized href attributes
+	div.innerHTML = "<a href='#'></a>";
+
+	if ( div.firstChild && typeof div.firstChild.getAttribute !== "undefined" &&
+			div.firstChild.getAttribute("href") !== "#" ) {
+
+		Expr.attrHandle.href = function( elem ) {
+			return elem.getAttribute( "href", 2 );
+		};
+	}
+
+	// release memory in IE
+	div = null;
+})();
+
+if ( document.querySelectorAll ) {
+	(function(){
+		var oldSizzle = Sizzle,
+			div = document.createElement("div"),
+			id = "__sizzle__";
+
+		div.innerHTML = "<p class='TEST'></p>";
+
+		// Safari can't handle uppercase or unicode characters when
+		// in quirks mode.
+		if ( div.querySelectorAll && div.querySelectorAll(".TEST").length === 0 ) {
+			return;
+		}
+	
+		Sizzle = function( query, context, extra, seed ) {
+			context = context || document;
+
+			// Only use querySelectorAll on non-XML documents
+			// (ID selectors don't work in non-HTML documents)
+			if ( !seed && !Sizzle.isXML(context) ) {
+				// See if we find a selector to speed up
+				var match = /^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec( query );
+				
+				if ( match && (context.nodeType === 1 || context.nodeType === 9) ) {
+					// Speed-up: Sizzle("TAG")
+					if ( match[1] ) {
+						return makeArray( context.getElementsByTagName( query ), extra );
+					
+					// Speed-up: Sizzle(".CLASS")
+					} else if ( match[2] && Expr.find.CLASS && context.getElementsByClassName ) {
+						return makeArray( context.getElementsByClassName( match[2] ), extra );
+					}
+				}
+				
+				if ( context.nodeType === 9 ) {
+					// Speed-up: Sizzle("body")
+					// The body element only exists once, optimize finding it
+					if ( query === "body" && context.body ) {
+						return makeArray( [ context.body ], extra );
+						
+					// Speed-up: Sizzle("#ID")
+					} else if ( match && match[3] ) {
+						var elem = context.getElementById( match[3] );
+
+						// Check parentNode to catch when Blackberry 4.6 returns
+						// nodes that are no longer in the document #6963
+						if ( elem && elem.parentNode ) {
+							// Handle the case where IE and Opera return items
+							// by name instead of ID
+							if ( elem.id === match[3] ) {
+								return makeArray( [ elem ], extra );
+							}
+							
+						} else {
+							return makeArray( [], extra );
+						}
+					}
+					
+					try {
+						return makeArray( context.querySelectorAll(query), extra );
+					} catch(qsaError) {}
+
+				// qSA works strangely on Element-rooted queries
+				// We can work around this by specifying an extra ID on the root
+				// and working up from there (Thanks to Andrew Dupont for the technique)
+				// IE 8 doesn't work on object elements
+				} else if ( context.nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) {
+					var oldContext = context,
+						old = context.getAttribute( "id" ),
+						nid = old || id,
+						hasParent = context.parentNode,
+						relativeHierarchySelector = /^\s*[+~]/.test( query );
+
+					if ( !old ) {
+						context.setAttribute( "id", nid );
+					} else {
+						nid = nid.replace( /'/g, "\\$&" );
+					}
+					if ( relativeHierarchySelector && hasParent ) {
+						context = context.parentNode;
+					}
+
+					try {
+						if ( !relativeHierarchySelector || hasParent ) {
+							return makeArray( context.querySelectorAll( "[id='" + nid + "'] " + query ), extra );
+						}
+
+					} catch(pseudoError) {
+					} finally {
+						if ( !old ) {
+							oldContext.removeAttribute( "id" );
+						}
+					}
+				}
+			}
+		
+			return oldSizzle(query, context, extra, seed);
+		};
+
+		for ( var prop in oldSizzle ) {
+			Sizzle[ prop ] = oldSizzle[ prop ];
+		}
+
+		// release memory in IE
+		div = null;
+	})();
+}
+
+(function(){
+	var html = document.documentElement,
+		matches = html.matchesSelector || html.mozMatchesSelector || html.webkitMatchesSelector || html.msMatchesSelector;
+
+	if ( matches ) {
+		// Check to see if it's possible to do matchesSelector
+		// on a disconnected node (IE 9 fails this)
+		var disconnectedMatch = !matches.call( document.createElement( "div" ), "div" ),
+			pseudoWorks = false;
+
+		try {
+			// This should fail with an exception
+			// Gecko does not error, returns false instead
+			matches.call( document.documentElement, "[test!='']:sizzle" );
+	
+		} catch( pseudoError ) {
+			pseudoWorks = true;
+		}
+
+		Sizzle.matchesSelector = function( node, expr ) {
+			// Make sure that attribute selectors are quoted
+			expr = expr.replace(/\=\s*([^'"\]]*)\s*\]/g, "='$1']");
+
+			if ( !Sizzle.isXML( node ) ) {
+				try { 
+					if ( pseudoWorks || !Expr.match.PSEUDO.test( expr ) && !/!=/.test( expr ) ) {
+						var ret = matches.call( node, expr );
+
+						// IE 9's matchesSelector returns false on disconnected nodes
+						if ( ret || !disconnectedMatch ||
+								// As well, disconnected nodes are said to be in a document
+								// fragment in IE 9, so check for that
+								node.document && node.document.nodeType !== 11 ) {
+							return ret;
+						}
+					}
+				} catch(e) {}
+			}
+
+			return Sizzle(expr, null, null, [node]).length > 0;
+		};
+	}
+})();
+
+(function(){
+	var div = document.createElement("div");
+
+	div.innerHTML = "<div class='test e'></div><div class='test'></div>";
+
+	// Opera can't find a second classname (in 9.6)
+	// Also, make sure that getElementsByClassName actually exists
+	if ( !div.getElementsByClassName || div.getElementsByClassName("e").length === 0 ) {
+		return;
+	}
+
+	// Safari caches class attributes, doesn't catch changes (in 3.2)
+	div.lastChild.className = "e";
+
+	if ( div.getElementsByClassName("e").length === 1 ) {
+		return;
+	}
+	
+	Expr.order.splice(1, 0, "CLASS");
+	Expr.find.CLASS = function( match, context, isXML ) {
+		if ( typeof context.getElementsByClassName !== "undefined" && !isXML ) {
+			return context.getElementsByClassName(match[1]);
+		}
+	};
+
+	// release memory in IE
+	div = null;
+})();
+
+function dirNodeCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) {
+	for ( var i = 0, l = checkSet.length; i < l; i++ ) {
+		var elem = checkSet[i];
+
+		if ( elem ) {
+			var match = false;
+
+			elem = elem[dir];
+
+			while ( elem ) {
+				if ( elem.sizcache === doneName ) {
+					match = checkSet[elem.sizset];
+					break;
+				}
+
+				if ( elem.nodeType === 1 && !isXML ){
+					elem.sizcache = doneName;
+					elem.sizset = i;
+				}
+
+				if ( elem.nodeName.toLowerCase() === cur ) {
+					match = elem;
+					break;
+				}
+
+				elem = elem[dir];
+			}
+
+			checkSet[i] = match;
+		}
+	}
+}
+
+function dirCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) {
+	for ( var i = 0, l = checkSet.length; i < l; i++ ) {
+		var elem = checkSet[i];
+
+		if ( elem ) {
+			var match = false;
+			
+			elem = elem[dir];
+
+			while ( elem ) {
+				if ( elem.sizcache === doneName ) {
+					match = checkSet[elem.sizset];
+					break;
+				}
+
+				if ( elem.nodeType === 1 ) {
+					if ( !isXML ) {
+						elem.sizcache = doneName;
+						elem.sizset = i;
+					}
+
+					if ( typeof cur !== "string" ) {
+						if ( elem === cur ) {
+							match = true;
+							break;
+						}
+
+					} else if ( Sizzle.filter( cur, [elem] ).length > 0 ) {
+						match = elem;
+						break;
+					}
+				}
+
+				elem = elem[dir];
+			}
+
+			checkSet[i] = match;
+		}
+	}
+}
+
+if ( document.documentElement.contains ) {
+	Sizzle.contains = function( a, b ) {
+		return a !== b && (a.contains ? a.contains(b) : true);
+	};
+
+} else if ( document.documentElement.compareDocumentPosition ) {
+	Sizzle.contains = function( a, b ) {
+		return !!(a.compareDocumentPosition(b) & 16);
+	};
+
+} else {
+	Sizzle.contains = function() {
+		return false;
+	};
+}
+
+Sizzle.isXML = function( elem ) {
+	// documentElement is verified for cases where it doesn't yet exist
+	// (such as loading iframes in IE - #4833) 
+	var documentElement = (elem ? elem.ownerDocument || elem : 0).documentElement;
+
+	return documentElement ? documentElement.nodeName !== "HTML" : false;
+};
+
+var posProcess = function( selector, context ) {
+	var match,
+		tmpSet = [],
+		later = "",
+		root = context.nodeType ? [context] : context;
+
+	// Position selectors must be done after the filter
+	// And so must :not(positional) so we move all PSEUDOs to the end
+	while ( (match = Expr.match.PSEUDO.exec( selector )) ) {
+		later += match[0];
+		selector = selector.replace( Expr.match.PSEUDO, "" );
+	}
+
+	selector = Expr.relative[selector] ? selector + "*" : selector;
+
+	for ( var i = 0, l = root.length; i < l; i++ ) {
+		Sizzle( selector, root[i], tmpSet );
+	}
+
+	return Sizzle.filter( later, tmpSet );
+};
+
+// EXPOSE
+jQuery.find = Sizzle;
+jQuery.expr = Sizzle.selectors;
+jQuery.expr[":"] = jQuery.expr.filters;
+jQuery.unique = Sizzle.uniqueSort;
+jQuery.text = Sizzle.getText;
+jQuery.isXMLDoc = Sizzle.isXML;
+jQuery.contains = Sizzle.contains;
+
+
+})();
+
+
+var runtil = /Until$/,
+	rparentsprev = /^(?:parents|prevUntil|prevAll)/,
+	// Note: This RegExp should be improved, or likely pulled from Sizzle
+	rmultiselector = /,/,
+	isSimple = /^.[^:#\[\.,]*$/,
+	slice = Array.prototype.slice,
+	POS = jQuery.expr.match.POS,
+	// methods guaranteed to produce a unique set when starting from a unique set
+	guaranteedUnique = {
+		children: true,
+		contents: true,
+		next: true,
+		prev: true
+	};
+
+jQuery.fn.extend({
+	find: function( selector ) {
+		var self = this,
+			i, l;
+
+		if ( typeof selector !== "string" ) {
+			return jQuery( selector ).filter(function() {
+				for ( i = 0, l = self.length; i < l; i++ ) {
+					if ( jQuery.contains( self[ i ], this ) ) {
+						return true;
+					}
+				}
+			});
+		}
+
+		var ret = this.pushStack( "", "find", selector ),
+			length, n, r;
+
+		for ( i = 0, l = this.length; i < l; i++ ) {
+			length = ret.length;
+			jQuery.find( selector, this[i], ret );
+
+			if ( i > 0 ) {
+				// Make sure that the results are unique
+				for ( n = length; n < ret.length; n++ ) {
+					for ( r = 0; r < length; r++ ) {
+						if ( ret[r] === ret[n] ) {
+							ret.splice(n--, 1);
+							break;
+						}
+					}
+				}
+			}
+		}
+
+		return ret;
+	},
+
+	has: function( target ) {
+		var targets = jQuery( target );
+		return this.filter(function() {
+			for ( var i = 0, l = targets.length; i < l; i++ ) {
+				if ( jQuery.contains( this, targets[i] ) ) {
+					return true;
+				}
+			}
+		});
+	},
+
+	not: function( selector ) {
+		return this.pushStack( winnow(this, selector, false), "not", selector);
+	},
+
+	filter: function( selector ) {
+		return this.pushStack( winnow(this, selector, true), "filter", selector );
+	},
+
+	is: function( selector ) {
+		return !!selector && ( typeof selector === "string" ?
+			jQuery.filter( selector, this ).length > 0 :
+			this.filter( selector ).length > 0 );
+	},
+
+	closest: function( selectors, context ) {
+		var ret = [], i, l, cur = this[0];
+		
+		// Array
+		if ( jQuery.isArray( selectors ) ) {
+			var match, selector,
+				matches = {},
+				level = 1;
+
+			if ( cur && selectors.length ) {
+				for ( i = 0, l = selectors.length; i < l; i++ ) {
+					selector = selectors[i];
+
+					if ( !matches[ selector ] ) {
+						matches[ selector ] = POS.test( selector ) ?
+							jQuery( selector, context || this.context ) :
+							selector;
+					}
+				}
+
+				while ( cur && cur.ownerDocument && cur !== context ) {
+					for ( selector in matches ) {
+						match = matches[ selector ];
+
+						if ( match.jquery ? match.index( cur ) > -1 : jQuery( cur ).is( match ) ) {
+							ret.push({ selector: selector, elem: cur, level: level });
+						}
+					}
+
+					cur = cur.parentNode;
+					level++;
+				}
+			}
+
+			return ret;
+		}
+
+		// String
+		var pos = POS.test( selectors ) || typeof selectors !== "string" ?
+				jQuery( selectors, context || this.context ) :
+				0;
+
+		for ( i = 0, l = this.length; i < l; i++ ) {
+			cur = this[i];
+
+			while ( cur ) {
+				if ( pos ? pos.index(cur) > -1 : jQuery.find.matchesSelector(cur, selectors) ) {
+					ret.push( cur );
+					break;
+
+				} else {
+					cur = cur.parentNode;
+					if ( !cur || !cur.ownerDocument || cur === context || cur.nodeType === 11 ) {
+						break;
+					}
+				}
+			}
+		}
+
+		ret = ret.length > 1 ? jQuery.unique( ret ) : ret;
+
+		return this.pushStack( ret, "closest", selectors );
+	},
+
+	// Determine the position of an element within
+	// the matched set of elements
+	index: function( elem ) {
+		if ( !elem || typeof elem === "string" ) {
+			return jQuery.inArray( this[0],
+				// If it receives a string, the selector is used
+				// If it receives nothing, the siblings are used
+				elem ? jQuery( elem ) : this.parent().children() );
+		}
+		// Locate the position of the desired element
+		return jQuery.inArray(
+			// If it receives a jQuery object, the first element is used
+			elem.jquery ? elem[0] : elem, this );
+	},
+
+	add: function( selector, context ) {
+		var set = typeof selector === "string" ?
+				jQuery( selector, context ) :
+				jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ),
+			all = jQuery.merge( this.get(), set );
+
+		return this.pushStack( isDisconnected( set[0] ) || isDisconnected( all[0] ) ?
+			all :
+			jQuery.unique( all ) );
+	},
+
+	andSelf: function() {
+		return this.add( this.prevObject );
+	}
+});
+
+// A painfully simple check to see if an element is disconnected
+// from a document (should be improved, where feasible).
+function isDisconnected( node ) {
+	return !node || !node.parentNode || node.parentNode.nodeType === 11;
+}
+
+jQuery.each({
+	parent: function( elem ) {
+		var parent = elem.parentNode;
+		return parent && parent.nodeType !== 11 ? parent : null;
+	},
+	parents: function( elem ) {
+		return jQuery.dir( elem, "parentNode" );
+	},
+	parentsUntil: function( elem, i, until ) {
+		return jQuery.dir( elem, "parentNode", until );
+	},
+	next: function( elem ) {
+		return jQuery.nth( elem, 2, "nextSibling" );
+	},
+	prev: function( elem ) {
+		return jQuery.nth( elem, 2, "previousSibling" );
+	},
+	nextAll: function( elem ) {
+		return jQuery.dir( elem, "nextSibling" );
+	},
+	prevAll: function( elem ) {
+		return jQuery.dir( elem, "previousSibling" );
+	},
+	nextUntil: function( elem, i, until ) {
+		return jQuery.dir( elem, "nextSibling", until );
+	},
+	prevUntil: function( elem, i, until ) {
+		return jQuery.dir( elem, "previousSibling", until );
+	},
+	siblings: function( elem ) {
+		return jQuery.sibling( elem.parentNode.firstChild, elem );
+	},
+	children: function( elem ) {
+		return jQuery.sibling( elem.firstChild );
+	},
+	contents: function( elem ) {
+		return jQuery.nodeName( elem, "iframe" ) ?
+			elem.contentDocument || elem.contentWindow.document :
+			jQuery.makeArray( elem.childNodes );
+	}
+}, function( name, fn ) {
+	jQuery.fn[ name ] = function( until, selector ) {
+		var ret = jQuery.map( this, fn, until ),
+			// The variable 'args' was introduced in
+			// https://github.com/jquery/jquery/commit/52a0238
+			// to work around a bug in Chrome 10 (Dev) and should be removed when the bug is fixed.
+			// http://code.google.com/p/v8/issues/detail?id=1050
+			args = slice.call(arguments);
+
+		if ( !runtil.test( name ) ) {
+			selector = until;
+		}
+
+		if ( selector && typeof selector === "string" ) {
+			ret = jQuery.filter( selector, ret );
+		}
+
+		ret = this.length > 1 && !guaranteedUnique[ name ] ? jQuery.unique( ret ) : ret;
+
+		if ( (this.length > 1 || rmultiselector.test( selector )) && rparentsprev.test( name ) ) {
+			ret = ret.reverse();
+		}
+
+		return this.pushStack( ret, name, args.join(",") );
+	};
+});
+
+jQuery.extend({
+	filter: function( expr, elems, not ) {
+		if ( not ) {
+			expr = ":not(" + expr + ")";
+		}
+
+		return elems.length === 1 ?
+			jQuery.find.matchesSelector(elems[0], expr) ? [ elems[0] ] : [] :
+			jQuery.find.matches(expr, elems);
+	},
+
+	dir: function( elem, dir, until ) {
+		var matched = [],
+			cur = elem[ dir ];
+
+		while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) {
+			if ( cur.nodeType === 1 ) {
+				matched.push( cur );
+			}
+			cur = cur[dir];
+		}
+		return matched;
+	},
+
+	nth: function( cur, result, dir, elem ) {
+		result = result || 1;
+		var num = 0;
+
+		for ( ; cur; cur = cur[dir] ) {
+			if ( cur.nodeType === 1 && ++num === result ) {
+				break;
+			}
+		}
+
+		return cur;
+	},
+
+	sibling: function( n, elem ) {
+		var r = [];
+
+		for ( ; n; n = n.nextSibling ) {
+			if ( n.nodeType === 1 && n !== elem ) {
+				r.push( n );
+			}
+		}
+
+		return r;
+	}
+});
+
+// Implement the identical functionality for filter and not
+function winnow( elements, qualifier, keep ) {
+
+	// Can't pass null or undefined to indexOf in Firefox 4
+	// Set to 0 to skip string check
+	qualifier = qualifier || 0;
+
+	if ( jQuery.isFunction( qualifier ) ) {
+		return jQuery.grep(elements, function( elem, i ) {
+			var retVal = !!qualifier.call( elem, i, elem );
+			return retVal === keep;
+		});
+
+	} else if ( qualifier.nodeType ) {
+		return jQuery.grep(elements, function( elem, i ) {
+			return (elem === qualifier) === keep;
+		});
+
+	} else if ( typeof qualifier === "string" ) {
+		var filtered = jQuery.grep(elements, function( elem ) {
+			return elem.nodeType === 1;
+		});
+
+		if ( isSimple.test( qualifier ) ) {
+			return jQuery.filter(qualifier, filtered, !keep);
+		} else {
+			qualifier = jQuery.filter( qualifier, filtered );
+		}
+	}
+
+	return jQuery.grep(elements, function( elem, i ) {
+		return (jQuery.inArray( elem, qualifier ) >= 0) === keep;
+	});
+}
+
+
+
+
+var rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g,
+	rleadingWhitespace = /^\s+/,
+	rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,
+	rtagName = /<([\w:]+)/,
+	rtbody = /<tbody/i,
+	rhtml = /<|&#?\w+;/,
+	rnocache = /<(?:script|object|embed|option|style)/i,
+	// checked="checked" or checked
+	rchecked = /checked\s*(?:[^=]|=\s*.checked.)/i,
+	rscriptType = /\/(java|ecma)script/i,
+	rcleanScript = /^\s*<!(?:\[CDATA\[|\-\-)/,
+	wrapMap = {
+		option: [ 1, "<select multiple='multiple'>", "</select>" ],
+		legend: [ 1, "<fieldset>", "</fieldset>" ],
+		thead: [ 1, "<table>", "</table>" ],
+		tr: [ 2, "<table><tbody>", "</tbody></table>" ],
+		td: [ 3, "<table><tbody><tr>", "</tr></tbody></table>" ],
+		col: [ 2, "<table><tbody></tbody><colgroup>", "</colgroup></table>" ],
+		area: [ 1, "<map>", "</map>" ],
+		_default: [ 0, "", "" ]
+	};
+
+wrapMap.optgroup = wrapMap.option;
+wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead;
+wrapMap.th = wrapMap.td;
+
+// IE can't serialize <link> and <script> tags normally
+if ( !jQuery.support.htmlSerialize ) {
+	wrapMap._default = [ 1, "div<div>", "</div>" ];
+}
+
+jQuery.fn.extend({
+	text: function( text ) {
+		if ( jQuery.isFunction(text) ) {
+			return this.each(function(i) {
+				var self = jQuery( this );
+
+				self.text( text.call(this, i, self.text()) );
+			});
+		}
+
+		if ( typeof text !== "object" && text !== undefined ) {
+			return this.empty().append( (this[0] && this[0].ownerDocument || document).createTextNode( text ) );
+		}
+
+		return jQuery.text( this );
+	},
+
+	wrapAll: function( html ) {
+		if ( jQuery.isFunction( html ) ) {
+			return this.each(function(i) {
+				jQuery(this).wrapAll( html.call(this, i) );
+			});
+		}
+
+		if ( this[0] ) {
+			// The elements to wrap the target around
+			var wrap = jQuery( html, this[0].ownerDocument ).eq(0).clone(true);
+
+			if ( this[0].parentNode ) {
+				wrap.insertBefore( this[0] );
+			}
+
+			wrap.map(function() {
+				var elem = this;
+
+				while ( elem.firstChild && elem.firstChild.nodeType === 1 ) {
+					elem = elem.firstChild;
+				}
+
+				return elem;
+			}).append( this );
+		}
+
+		return this;
+	},
+
+	wrapInner: function( html ) {
+		if ( jQuery.isFunction( html ) ) {
+			return this.each(function(i) {
+				jQuery(this).wrapInner( html.call(this, i) );
+			});
+		}
+
+		return this.each(function() {
+			var self = jQuery( this ),
+				contents = self.contents();
+
+			if ( contents.length ) {
+				contents.wrapAll( html );
+
+			} else {
+				self.append( html );
+			}
+		});
+	},
+
+	wrap: function( html ) {
+		return this.each(function() {
+			jQuery( this ).wrapAll( html );
+		});
+	},
+
+	unwrap: function() {
+		return this.parent().each(function() {
+			if ( !jQuery.nodeName( this, "body" ) ) {
+				jQuery( this ).replaceWith( this.childNodes );
+			}
+		}).end();
+	},
+
+	append: function() {
+		return this.domManip(arguments, true, function( elem ) {
+			if ( this.nodeType === 1 ) {
+				this.appendChild( elem );
+			}
+		});
+	},
+
+	prepend: function() {
+		return this.domManip(arguments, true, function( elem ) {
+			if ( this.nodeType === 1 ) {
+				this.insertBefore( elem, this.firstChild );
+			}
+		});
+	},
+
+	before: function() {
+		if ( this[0] && this[0].parentNode ) {
+			return this.domManip(arguments, false, function( elem ) {
+				this.parentNode.insertBefore( elem, this );
+			});
+		} else if ( arguments.length ) {
+			var set = jQuery(arguments[0]);
+			set.push.apply( set, this.toArray() );
+			return this.pushStack( set, "before", arguments );
+		}
+	},
+
+	after: function() {
+		if ( this[0] && this[0].parentNode ) {
+			return this.domManip(arguments, false, function( elem ) {
+				this.parentNode.insertBefore( elem, this.nextSibling );
+			});
+		} else if ( arguments.length ) {
+			var set = this.pushStack( this, "after", arguments );
+			set.push.apply( set, jQuery(arguments[0]).toArray() );
+			return set;
+		}
+	},
+
+	// keepData is for internal use only--do not document
+	remove: function( selector, keepData ) {
+		for ( var i = 0, elem; (elem = this[i]) != null; i++ ) {
+			if ( !selector || jQuery.filter( selector, [ elem ] ).length ) {
+				if ( !keepData && elem.nodeType === 1 ) {
+					jQuery.cleanData( elem.getElementsByTagName("*") );
+					jQuery.cleanData( [ elem ] );
+				}
+
+				if ( elem.parentNode ) {
+					elem.parentNode.removeChild( elem );
+				}
+			}
+		}
+
+		return this;
+	},
+
+	empty: function() {
+		for ( var i = 0, elem; (elem = this[i]) != null; i++ ) {
+			// Remove element nodes and prevent memory leaks
+			if ( elem.nodeType === 1 ) {
+				jQuery.cleanData( elem.getElementsByTagName("*") );
+			}
+
+			// Remove any remaining nodes
+			while ( elem.firstChild ) {
+				elem.removeChild( elem.firstChild );
+			}
+		}
+
+		return this;
+	},
+
+	clone: function( dataAndEvents, deepDataAndEvents ) {
+		dataAndEvents = dataAndEvents == null ? false : dataAndEvents;
+		deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents;
+
+		return this.map( function () {
+			return jQuery.clone( this, dataAndEvents, deepDataAndEvents );
+		});
+	},
+
+	html: function( value ) {
+		if ( value === undefined ) {
+			return this[0] && this[0].nodeType === 1 ?
+				this[0].innerHTML.replace(rinlinejQuery, "") :
+				null;
+
+		// See if we can take a shortcut and just use innerHTML
+		} else if ( typeof value === "string" && !rnocache.test( value ) &&
+			(jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value )) &&
+			!wrapMap[ (rtagName.exec( value ) || ["", ""])[1].toLowerCase() ] ) {
+
+			value = value.replace(rxhtmlTag, "<$1></$2>");
+
+			try {
+				for ( var i = 0, l = this.length; i < l; i++ ) {
+					// Remove element nodes and prevent memory leaks
+					if ( this[i].nodeType === 1 ) {
+						jQuery.cleanData( this[i].getElementsByTagName("*") );
+						this[i].innerHTML = value;
+					}
+				}
+
+			// If using innerHTML throws an exception, use the fallback method
+			} catch(e) {
+				this.empty().append( value );
+			}
+
+		} else if ( jQuery.isFunction( value ) ) {
+			this.each(function(i){
+				var self = jQuery( this );
+
+				self.html( value.call(this, i, self.html()) );
+			});
+
+		} else {
+			this.empty().append( value );
+		}
+
+		return this;
+	},
+
+	replaceWith: function( value ) {
+		if ( this[0] && this[0].parentNode ) {
+			// Make sure that the elements are removed from the DOM before they are inserted
+			// this can help fix replacing a parent with child elements
+			if ( jQuery.isFunction( value ) ) {
+				return this.each(function(i) {
+					var self = jQuery(this), old = self.html();
+					self.replaceWith( value.call( this, i, old ) );
+				});
+			}
+
+			if ( typeof value !== "string" ) {
+				value = jQuery( value ).detach();
+			}
+
+			return this.each(function() {
+				var next = this.nextSibling,
+					parent = this.parentNode;
+
+				jQuery( this ).remove();
+
+				if ( next ) {
+					jQuery(next).before( value );
+				} else {
+					jQuery(parent).append( value );
+				}
+			});
+		} else {
+			return this.length ?
+				this.pushStack( jQuery(jQuery.isFunction(value) ? value() : value), "replaceWith", value ) :
+				this;
+		}
+	},
+
+	detach: function( selector ) {
+		return this.remove( selector, true );
+	},
+
+	domManip: function( args, table, callback ) {
+		var results, first, fragment, parent,
+			value = args[0],
+			scripts = [];
+
+		// We can't cloneNode fragments that contain checked, in WebKit
+		if ( !jQuery.support.checkClone && arguments.length === 3 && typeof value === "string" && rchecked.test( value ) ) {
+			return this.each(function() {
+				jQuery(this).domManip( args, table, callback, true );
+			});
+		}
+
+		if ( jQuery.isFunction(value) ) {
+			return this.each(function(i) {
+				var self = jQuery(this);
+				args[0] = value.call(this, i, table ? self.html() : undefined);
+				self.domManip( args, table, callback );
+			});
+		}
+
+		if ( this[0] ) {
+			parent = value && value.parentNode;
+
+			// If we're in a fragment, just use that instead of building a new one
+			if ( jQuery.support.parentNode && parent && parent.nodeType === 11 && parent.childNodes.length === this.length ) {
+				results = { fragment: parent };
+
+			} else {
+				results = jQuery.buildFragment( args, this, scripts );
+			}
+
+			fragment = results.fragment;
+
+			if ( fragment.childNodes.length === 1 ) {
+				first = fragment = fragment.firstChild;
+			} else {
+				first = fragment.firstChild;
+			}
+
+			if ( first ) {
+				table = table && jQuery.nodeName( first, "tr" );
+
+				for ( var i = 0, l = this.length, lastIndex = l - 1; i < l; i++ ) {
+					callback.call(
+						table ?
+							root(this[i], first) :
+							this[i],
+						// Make sure that we do not leak memory by inadvertently discarding
+						// the original fragment (which might have attached data) instead of
+						// using it; in addition, use the original fragment object for the last
+						// item instead of first because it can end up being emptied incorrectly
+						// in certain situations (Bug #8070).
+						// Fragments from the fragment cache must always be cloned and never used
+						// in place.
+						results.cacheable || (l > 1 && i < lastIndex) ?
+							jQuery.clone( fragment, true, true ) :
+							fragment
+					);
+				}
+			}
+
+			if ( scripts.length ) {
+				jQuery.each( scripts, evalScript );
+			}
+		}
+
+		return this;
+	}
+});
+
+function root( elem, cur ) {
+	return jQuery.nodeName(elem, "table") ?
+		(elem.getElementsByTagName("tbody")[0] ||
+		elem.appendChild(elem.ownerDocument.createElement("tbody"))) :
+		elem;
+}
+
+function cloneCopyEvent( src, dest ) {
+
+	if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) {
+		return;
+	}
+
+	var internalKey = jQuery.expando,
+		oldData = jQuery.data( src ),
+		curData = jQuery.data( dest, oldData );
+
+	// Switch to use the internal data object, if it exists, for the next
+	// stage of data copying
+	if ( (oldData = oldData[ internalKey ]) ) {
+		var events = oldData.events;
+				curData = curData[ internalKey ] = jQuery.extend({}, oldData);
+
+		if ( events ) {
+			delete curData.handle;
+			curData.events = {};
+
+			for ( var type in events ) {
+				for ( var i = 0, l = events[ type ].length; i < l; i++ ) {
+					jQuery.event.add( dest, type + ( events[ type ][ i ].namespace ? "." : "" ) + events[ type ][ i ].namespace, events[ type ][ i ], events[ type ][ i ].data );
+				}
+			}
+		}
+	}
+}
+
+function cloneFixAttributes( src, dest ) {
+	var nodeName;
+
+	// We do not need to do anything for non-Elements
+	if ( dest.nodeType !== 1 ) {
+		return;
+	}
+
+	// clearAttributes removes the attributes, which we don't want,
+	// but also removes the attachEvent events, which we *do* want
+	if ( dest.clearAttributes ) {
+		dest.clearAttributes();
+	}
+
+	// mergeAttributes, in contrast, only merges back on the
+	// original attributes, not the events
+	if ( dest.mergeAttributes ) {
+		dest.mergeAttributes( src );
+	}
+
+	nodeName = dest.nodeName.toLowerCase();
+
+	// IE6-8 fail to clone children inside object elements that use
+	// the proprietary classid attribute value (rather than the type
+	// attribute) to identify the type of content to display
+	if ( nodeName === "object" ) {
+		dest.outerHTML = src.outerHTML;
+
+	} else if ( nodeName === "input" && (src.type === "checkbox" || src.type === "radio") ) {
+		// IE6-8 fails to persist the checked state of a cloned checkbox
+		// or radio button. Worse, IE6-7 fail to give the cloned element
+		// a checked appearance if the defaultChecked value isn't also set
+		if ( src.checked ) {
+			dest.defaultChecked = dest.checked = src.checked;
+		}
+
+		// IE6-7 get confused and end up setting the value of a cloned
+		// checkbox/radio button to an empty string instead of "on"
+		if ( dest.value !== src.value ) {
+			dest.value = src.value;
+		}
+
+	// IE6-8 fails to return the selected option to the default selected
+	// state when cloning options
+	} else if ( nodeName === "option" ) {
+		dest.selected = src.defaultSelected;
+
+	// IE6-8 fails to set the defaultValue to the correct value when
+	// cloning other types of input fields
+	} else if ( nodeName === "input" || nodeName === "textarea" ) {
+		dest.defaultValue = src.defaultValue;
+	}
+
+	// Event data gets referenced instead of copied if the expando
+	// gets copied too
+	dest.removeAttribute( jQuery.expando );
+}
+
+jQuery.buildFragment = function( args, nodes, scripts ) {
+	var fragment, cacheable, cacheresults, doc;
+
+  // nodes may contain either an explicit document object,
+  // a jQuery collection or context object.
+  // If nodes[0] contains a valid object to assign to doc
+  if ( nodes && nodes[0] ) {
+    doc = nodes[0].ownerDocument || nodes[0];
+  }
+
+  // Ensure that an attr object doesn't incorrectly stand in as a document object
+	// Chrome and Firefox seem to allow this to occur and will throw exception
+	// Fixes #8950
+	if ( !doc.createDocumentFragment ) {
+		doc = document;
+	}
+
+	// Only cache "small" (1/2 KB) HTML strings that are associated with the main document
+	// Cloning options loses the selected state, so don't cache them
+	// IE 6 doesn't like it when you put <object> or <embed> elements in a fragment
+	// Also, WebKit does not clone 'checked' attributes on cloneNode, so don't cache
+	if ( args.length === 1 && typeof args[0] === "string" && args[0].length < 512 && doc === document &&
+		args[0].charAt(0) === "<" && !rnocache.test( args[0] ) && (jQuery.support.checkClone || !rchecked.test( args[0] )) ) {
+
+		cacheable = true;
+
+		cacheresults = jQuery.fragments[ args[0] ];
+		if ( cacheresults && cacheresults !== 1 ) {
+			fragment = cacheresults;
+		}
+	}
+
+	if ( !fragment ) {
+		fragment = doc.createDocumentFragment();
+		jQuery.clean( args, doc, fragment, scripts );
+	}
+
+	if ( cacheable ) {
+		jQuery.fragments[ args[0] ] = cacheresults ? fragment : 1;
+	}
+
+	return { fragment: fragment, cacheable: cacheable };
+};
+
+jQuery.fragments = {};
+
+jQuery.each({
+	appendTo: "append",
+	prependTo: "prepend",
+	insertBefore: "before",
+	insertAfter: "after",
+	replaceAll: "replaceWith"
+}, function( name, original ) {
+	jQuery.fn[ name ] = function( selector ) {
+		var ret = [],
+			insert = jQuery( selector ),
+			parent = this.length === 1 && this[0].parentNode;
+
+		if ( parent && parent.nodeType === 11 && parent.childNodes.length === 1 && insert.length === 1 ) {
+			insert[ original ]( this[0] );
+			return this;
+
+		} else {
+			for ( var i = 0, l = insert.length; i < l; i++ ) {
+				var elems = (i > 0 ? this.clone(true) : this).get();
+				jQuery( insert[i] )[ original ]( elems );
+				ret = ret.concat( elems );
+			}
+
+			return this.pushStack( ret, name, insert.selector );
+		}
+	};
+});
+
+function getAll( elem ) {
+	if ( "getElementsByTagName" in elem ) {
+		return elem.getElementsByTagName( "*" );
+
+	} else if ( "querySelectorAll" in elem ) {
+		return elem.querySelectorAll( "*" );
+
+	} else {
+		return [];
+	}
+}
+
+// Used in clean, fixes the defaultChecked property
+function fixDefaultChecked( elem ) {
+	if ( elem.type === "checkbox" || elem.type === "radio" ) {
+		elem.defaultChecked = elem.checked;
+	}
+}
+// Finds all inputs and passes them to fixDefaultChecked
+function findInputs( elem ) {
+	if ( jQuery.nodeName( elem, "input" ) ) {
+		fixDefaultChecked( elem );
+	} else if ( "getElementsByTagName" in elem ) {
+		jQuery.grep( elem.getElementsByTagName("input"), fixDefaultChecked );
+	}
+}
+
+jQuery.extend({
+	clone: function( elem, dataAndEvents, deepDataAndEvents ) {
+		var clone = elem.cloneNode(true),
+				srcElements,
+				destElements,
+				i;
+
+		if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) &&
+				(elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) {
+			// IE copies events bound via attachEvent when using cloneNode.
+			// Calling detachEvent on the clone will also remove the events
+			// from the original. In order to get around this, we use some
+			// proprietary methods to clear the events. Thanks to MooTools
+			// guys for this hotness.
+
+			cloneFixAttributes( elem, clone );
+
+			// Using Sizzle here is crazy slow, so we use getElementsByTagName
+			// instead
+			srcElements = getAll( elem );
+			destElements = getAll( clone );
+
+			// Weird iteration because IE will replace the length property
+			// with an element if you are cloning the body and one of the
+			// elements on the page has a name or id of "length"
+			for ( i = 0; srcElements[i]; ++i ) {
+				cloneFixAttributes( srcElements[i], destElements[i] );
+			}
+		}
+
+		// Copy the events from the original to the clone
+		if ( dataAndEvents ) {
+			cloneCopyEvent( elem, clone );
+
+			if ( deepDataAndEvents ) {
+				srcElements = getAll( elem );
+				destElements = getAll( clone );
+
+				for ( i = 0; srcElements[i]; ++i ) {
+					cloneCopyEvent( srcElements[i], destElements[i] );
+				}
+			}
+		}
+
+		srcElements = destElements = null;
+
+		// Return the cloned set
+		return clone;
+	},
+
+	clean: function( elems, context, fragment, scripts ) {
+		var checkScriptType;
+
+		context = context || document;
+
+		// !context.createElement fails in IE with an error but returns typeof 'object'
+		if ( typeof context.createElement === "undefined" ) {
+			context = context.ownerDocument || context[0] && context[0].ownerDocument || document;
+		}
+
+		var ret = [], j;
+
+		for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) {
+			if ( typeof elem === "number" ) {
+				elem += "";
+			}
+
+			if ( !elem ) {
+				continue;
+			}
+
+			// Convert html string into DOM nodes
+			if ( typeof elem === "string" ) {
+				if ( !rhtml.test( elem ) ) {
+					elem = context.createTextNode( elem );
+				} else {
+					// Fix "XHTML"-style tags in all browsers
+					elem = elem.replace(rxhtmlTag, "<$1></$2>");
+
+					// Trim whitespace, otherwise indexOf won't work as expected
+					var tag = (rtagName.exec( elem ) || ["", ""])[1].toLowerCase(),
+						wrap = wrapMap[ tag ] || wrapMap._default,
+						depth = wrap[0],
+						div = context.createElement("div");
+
+					// Go to html and back, then peel off extra wrappers
+					div.innerHTML = wrap[1] + elem + wrap[2];
+
+					// Move to the right depth
+					while ( depth-- ) {
+						div = div.lastChild;
+					}
+
+					// Remove IE's autoinserted <tbody> from table fragments
+					if ( !jQuery.support.tbody ) {
+
+						// String was a <table>, *may* have spurious <tbody>
+						var hasBody = rtbody.test(elem),
+							tbody = tag === "table" && !hasBody ?
+								div.firstChild && div.firstChild.childNodes :
+
+								// String was a bare <thead> or <tfoot>
+								wrap[1] === "<table>" && !hasBody ?
+									div.childNodes :
+									[];
+
+						for ( j = tbody.length - 1; j >= 0 ; --j ) {
+							if ( jQuery.nodeName( tbody[ j ], "tbody" ) && !tbody[ j ].childNodes.length ) {
+								tbody[ j ].parentNode.removeChild( tbody[ j ] );
+							}
+						}
+					}
+
+					// IE completely kills leading whitespace when innerHTML is used
+					if ( !jQuery.support.leadingWhitespace && rleadingWhitespace.test( elem ) ) {
+						div.insertBefore( context.createTextNode( rleadingWhitespace.exec(elem)[0] ), div.firstChild );
+					}
+
+					elem = div.childNodes;
+				}
+			}
+
+			// Resets defaultChecked for any radios and checkboxes
+			// about to be appended to the DOM in IE 6/7 (#8060)
+			var len;
+			if ( !jQuery.support.appendChecked ) {
+				if ( elem[0] && typeof (len = elem.length) === "number" ) {
+					for ( j = 0; j < len; j++ ) {
+						findInputs( elem[j] );
+					}
+				} else {
+					findInputs( elem );
+				}
+			}
+
+			if ( elem.nodeType ) {
+				ret.push( elem );
+			} else {
+				ret = jQuery.merge( ret, elem );
+			}
+		}
+
+		if ( fragment ) {
+			checkScriptType = function( elem ) {
+				return !elem.type || rscriptType.test( elem.type );
+			};
+			for ( i = 0; ret[i]; i++ ) {
+				if ( scripts && jQuery.nodeName( ret[i], "script" ) && (!ret[i].type || ret[i].type.toLowerCase() === "text/javascript") ) {
+					scripts.push( ret[i].parentNode ? ret[i].parentNode.removeChild( ret[i] ) : ret[i] );
+
+				} else {
+					if ( ret[i].nodeType === 1 ) {
+						var jsTags = jQuery.grep( ret[i].getElementsByTagName( "script" ), checkScriptType );
+
+						ret.splice.apply( ret, [i + 1, 0].concat( jsTags ) );
+					}
+					fragment.appendChild( ret[i] );
+				}
+			}
+		}
+
+		return ret;
+	},
+
+	cleanData: function( elems ) {
+		var data, id, cache = jQuery.cache, internalKey = jQuery.expando, special = jQuery.event.special,
+			deleteExpando = jQuery.support.deleteExpando;
+
+		for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) {
+			if ( elem.nodeName && jQuery.noData[elem.nodeName.toLowerCase()] ) {
+				continue;
+			}
+
+			id = elem[ jQuery.expando ];
+
+			if ( id ) {
+				data = cache[ id ] && cache[ id ][ internalKey ];
+
+				if ( data && data.events ) {
+					for ( var type in data.events ) {
+						if ( special[ type ] ) {
+							jQuery.event.remove( elem, type );
+
+						// This is a shortcut to avoid jQuery.event.remove's overhead
+						} else {
+							jQuery.removeEvent( elem, type, data.handle );
+						}
+					}
+
+					// Null the DOM reference to avoid IE6/7/8 leak (#7054)
+					if ( data.handle ) {
+						data.handle.elem = null;
+					}
+				}
+
+				if ( deleteExpando ) {
+					delete elem[ jQuery.expando ];
+
+				} else if ( elem.removeAttribute ) {
+					elem.removeAttribute( jQuery.expando );
+				}
+
+				delete cache[ id ];
+			}
+		}
+	}
+});
+
+function evalScript( i, elem ) {
+	if ( elem.src ) {
+		jQuery.ajax({
+			url: elem.src,
+			async: false,
+			dataType: "script"
+		});
+	} else {
+		jQuery.globalEval( ( elem.text || elem.textContent || elem.innerHTML || "" ).replace( rcleanScript, "/*$0*/" ) );
+	}
+
+	if ( elem.parentNode ) {
+		elem.parentNode.removeChild( elem );
+	}
+}
+
+
+
+var ralpha = /alpha\([^)]*\)/i,
+	ropacity = /opacity=([^)]*)/,
+	// fixed for IE9, see #8346
+	rupper = /([A-Z]|^ms)/g,
+	rnumpx = /^-?\d+(?:px)?$/i,
+	rnum = /^-?\d/,
+	rrelNum = /^[+\-]=/,
+	rrelNumFilter = /[^+\-\.\de]+/g,
+
+	cssShow = { position: "absolute", visibility: "hidden", display: "block" },
+	cssWidth = [ "Left", "Right" ],
+	cssHeight = [ "Top", "Bottom" ],
+	curCSS,
+
+	getComputedStyle,
+	currentStyle;
+
+jQuery.fn.css = function( name, value ) {
+	// Setting 'undefined' is a no-op
+	if ( arguments.length === 2 && value === undefined ) {
+		return this;
+	}
+
+	return jQuery.access( this, name, value, true, function( elem, name, value ) {
+		return value !== undefined ?
+			jQuery.style( elem, name, value ) :
+			jQuery.css( elem, name );
+	});
+};
+
+jQuery.extend({
+	// Add in style property hooks for overriding the default
+	// behavior of getting and setting a style property
+	cssHooks: {
+		opacity: {
+			get: function( elem, computed ) {
+				if ( computed ) {
+					// We should always get a number back from opacity
+					var ret = curCSS( elem, "opacity", "opacity" );
+					return ret === "" ? "1" : ret;
+
+				} else {
+					return elem.style.opacity;
+				}
+			}
+		}
+	},
+
+	// Exclude the following css properties to add px
+	cssNumber: {
+		"fillOpacity": true,
+		"fontWeight": true,
+		"lineHeight": true,
+		"opacity": true,
+		"orphans": true,
+		"widows": true,
+		"zIndex": true,
+		"zoom": true
+	},
+
+	// Add in properties whose names you wish to fix before
+	// setting or getting the value
+	cssProps: {
+		// normalize float css property
+		"float": jQuery.support.cssFloat ? "cssFloat" : "styleFloat"
+	},
+
+	// Get and set the style property on a DOM Node
+	style: function( elem, name, value, extra ) {
+		// Don't set styles on text and comment nodes
+		if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) {
+			return;
+		}
+
+		// Make sure that we're working with the right name
+		var ret, type, origName = jQuery.camelCase( name ),
+			style = elem.style, hooks = jQuery.cssHooks[ origName ];
+
+		name = jQuery.cssProps[ origName ] || origName;
+
+		// Check if we're setting a value
+		if ( value !== undefined ) {
+			type = typeof value;
+
+			// Make sure that NaN and null values aren't set. See: #7116
+			if ( type === "number" && isNaN( value ) || value == null ) {
+				return;
+			}
+
+			// convert relative number strings (+= or -=) to relative numbers. #7345
+			if ( type === "string" && rrelNum.test( value ) ) {
+				value = +value.replace( rrelNumFilter, "" ) + parseFloat( jQuery.css( elem, name ) );
+				// Fixes bug #9237
+				type = "number";
+			}
+
+			// If a number was passed in, add 'px' to the (except for certain CSS properties)
+			if ( type === "number" && !jQuery.cssNumber[ origName ] ) {
+				value += "px";
+			}
+
+			// If a hook was provided, use that value, otherwise just set the specified value
+			if ( !hooks || !("set" in hooks) || (value = hooks.set( elem, value )) !== undefined ) {
+				// Wrapped to prevent IE from throwing errors when 'invalid' values are provided
+				// Fixes bug #5509
+				try {
+					style[ name ] = value;
+				} catch(e) {}
+			}
+
+		} else {
+			// If a hook was provided get the non-computed value from there
+			if ( hooks && "get" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) {
+				return ret;
+			}
+
+			// Otherwise just get the value from the style object
+			return style[ name ];
+		}
+	},
+
+	css: function( elem, name, extra ) {
+		var ret, hooks;
+
+		// Make sure that we're working with the right name
+		name = jQuery.camelCase( name );
+		hooks = jQuery.cssHooks[ name ];
+		name = jQuery.cssProps[ name ] || name;
+
+		// cssFloat needs a special treatment
+		if ( name === "cssFloat" ) {
+			name = "float";
+		}
+
+		// If a hook was provided get the computed value from there
+		if ( hooks && "get" in hooks && (ret = hooks.get( elem, true, extra )) !== undefined ) {
+			return ret;
+
+		// Otherwise, if a way to get the computed value exists, use that
+		} else if ( curCSS ) {
+			return curCSS( elem, name );
+		}
+	},
+
+	// A method for quickly swapping in/out CSS properties to get correct calculations
+	swap: function( elem, options, callback ) {
+		var old = {};
+
+		// Remember the old values, and insert the new ones
+		for ( var name in options ) {
+			old[ name ] = elem.style[ name ];
+			elem.style[ name ] = options[ name ];
+		}
+
+		callback.call( elem );
+
+		// Revert the old values
+		for ( name in options ) {
+			elem.style[ name ] = old[ name ];
+		}
+	}
+});
+
+// DEPRECATED, Use jQuery.css() instead
+jQuery.curCSS = jQuery.css;
+
+jQuery.each(["height", "width"], function( i, name ) {
+	jQuery.cssHooks[ name ] = {
+		get: function( elem, computed, extra ) {
+			var val;
+
+			if ( computed ) {
+				if ( elem.offsetWidth !== 0 ) {
+					return getWH( elem, name, extra );
+				} else {
+					jQuery.swap( elem, cssShow, function() {
+						val = getWH( elem, name, extra );
+					});
+				}
+
+				return val;
+			}
+		},
+
+		set: function( elem, value ) {
+			if ( rnumpx.test( value ) ) {
+				// ignore negative width and height values #1599
+				value = parseFloat( value );
+
+				if ( value >= 0 ) {
+					return value + "px";
+				}
+
+			} else {
+				return value;
+			}
+		}
+	};
+});
+
+if ( !jQuery.support.opacity ) {
+	jQuery.cssHooks.opacity = {
+		get: function( elem, computed ) {
+			// IE uses filters for opacity
+			return ropacity.test( (computed && elem.currentStyle ? elem.currentStyle.filter : elem.style.filter) || "" ) ?
+				( parseFloat( RegExp.$1 ) / 100 ) + "" :
+				computed ? "1" : "";
+		},
+
+		set: function( elem, value ) {
+			var style = elem.style,
+				currentStyle = elem.currentStyle;
+
+			// IE has trouble with opacity if it does not have layout
+			// Force it by setting the zoom level
+			style.zoom = 1;
+
+			// Set the alpha filter to set the opacity
+			var opacity = jQuery.isNaN( value ) ?
+				"" :
+				"alpha(opacity=" + value * 100 + ")",
+				filter = currentStyle && currentStyle.filter || style.filter || "";
+
+			style.filter = ralpha.test( filter ) ?
+				filter.replace( ralpha, opacity ) :
+				filter + " " + opacity;
+		}
+	};
+}
+
+jQuery(function() {
+	// This hook cannot be added until DOM ready because the support test
+	// for it is not run until after DOM ready
+	if ( !jQuery.support.reliableMarginRight ) {
+		jQuery.cssHooks.marginRight = {
+			get: function( elem, computed ) {
+				// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+				// Work around by temporarily setting element display to inline-block
+				var ret;
+				jQuery.swap( elem, { "display": "inline-block" }, function() {
+					if ( computed ) {
+						ret = curCSS( elem, "margin-right", "marginRight" );
+					} else {
+						ret = elem.style.marginRight;
+					}
+				});
+				return ret;
+			}
+		};
+	}
+});
+
+if ( document.defaultView && document.defaultView.getComputedStyle ) {
+	getComputedStyle = function( elem, name ) {
+		var ret, defaultView, computedStyle;
+
+		name = name.replace( rupper, "-$1" ).toLowerCase();
+
+		if ( !(defaultView = elem.ownerDocument.defaultView) ) {
+			return undefined;
+		}
+
+		if ( (computedStyle = defaultView.getComputedStyle( elem, null )) ) {
+			ret = computedStyle.getPropertyValue( name );
+			if ( ret === "" && !jQuery.contains( elem.ownerDocument.documentElement, elem ) ) {
+				ret = jQuery.style( elem, name );
+			}
+		}
+
+		return ret;
+	};
+}
+
+if ( document.documentElement.currentStyle ) {
+	currentStyle = function( elem, name ) {
+		var left,
+			ret = elem.currentStyle && elem.currentStyle[ name ],
+			rsLeft = elem.runtimeStyle && elem.runtimeStyle[ name ],
+			style = elem.style;
+
+		// From the awesome hack by Dean Edwards
+		// http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291
+
+		// If we're not dealing with a regular pixel number
+		// but a number that has a weird ending, we need to convert it to pixels
+		if ( !rnumpx.test( ret ) && rnum.test( ret ) ) {
+			// Remember the original values
+			left = style.left;
+
+			// Put in the new values to get a computed value out
+			if ( rsLeft ) {
+				elem.runtimeStyle.left = elem.currentStyle.left;
+			}
+			style.left = name === "fontSize" ? "1em" : (ret || 0);
+			ret = style.pixelLeft + "px";
+
+			// Revert the changed values
+			style.left = left;
+			if ( rsLeft ) {
+				elem.runtimeStyle.left = rsLeft;
+			}
+		}
+
+		return ret === "" ? "auto" : ret;
+	};
+}
+
+curCSS = getComputedStyle || currentStyle;
+
+function getWH( elem, name, extra ) {
+
+	// Start with offset property
+	var val = name === "width" ? elem.offsetWidth : elem.offsetHeight,
+		which = name === "width" ? cssWidth : cssHeight;
+
+	if ( val > 0 ) {
+		if ( extra !== "border" ) {
+			jQuery.each( which, function() {
+				if ( !extra ) {
+					val -= parseFloat( jQuery.css( elem, "padding" + this ) ) || 0;
+				}
+				if ( extra === "margin" ) {
+					val += parseFloat( jQuery.css( elem, extra + this ) ) || 0;
+				} else {
+					val -= parseFloat( jQuery.css( elem, "border" + this + "Width" ) ) || 0;
+				}
+			});
+		}
+
+		return val + "px";
+	}
+
+	// Fall back to computed then uncomputed css if necessary
+	val = curCSS( elem, name, name );
+	if ( val < 0 || val == null ) {
+		val = elem.style[ name ] || 0;
+	}
+	// Normalize "", auto, and prepare for extra
+	val = parseFloat( val ) || 0;
+
+	// Add padding, border, margin
+	if ( extra ) {
+		jQuery.each( which, function() {
+			val += parseFloat( jQuery.css( elem, "padding" + this ) ) || 0;
+			if ( extra !== "padding" ) {
+				val += parseFloat( jQuery.css( elem, "border" + this + "Width" ) ) || 0;
+			}
+			if ( extra === "margin" ) {
+				val += parseFloat( jQuery.css( elem, extra + this ) ) || 0;
+			}
+		});
+	}
+
+	return val + "px";
+}
+
+if ( jQuery.expr && jQuery.expr.filters ) {
+	jQuery.expr.filters.hidden = function( elem ) {
+		var width = elem.offsetWidth,
+			height = elem.offsetHeight;
+
+		return (width === 0 && height === 0) || (!jQuery.support.reliableHiddenOffsets && (elem.style.display || jQuery.css( elem, "display" )) === "none");
+	};
+
+	jQuery.expr.filters.visible = function( elem ) {
+		return !jQuery.expr.filters.hidden( elem );
+	};
+}
+
+
+
+
+var r20 = /%20/g,
+	rbracket = /\[\]$/,
+	rCRLF = /\r?\n/g,
+	rhash = /#.*$/,
+	rheaders = /^(.*?):[ \t]*([^\r\n]*)\r?$/mg, // IE leaves an \r character at EOL
+	rinput = /^(?:color|date|datetime|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,
+	// #7653, #8125, #8152: local protocol detection
+	rlocalProtocol = /^(?:about|app|app\-storage|.+\-extension|file|widget):$/,
+	rnoContent = /^(?:GET|HEAD)$/,
+	rprotocol = /^\/\//,
+	rquery = /\?/,
+	rscript = /<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,
+	rselectTextarea = /^(?:select|textarea)/i,
+	rspacesAjax = /\s+/,
+	rts = /([?&])_=[^&]*/,
+	rurl = /^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+))?)?/,
+
+	// Keep a copy of the old load method
+	_load = jQuery.fn.load,
+
+	/* Prefilters
+	 * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example)
+	 * 2) These are called:
+	 *    - BEFORE asking for a transport
+	 *    - AFTER param serialization (s.data is a string if s.processData is true)
+	 * 3) key is the dataType
+	 * 4) the catchall symbol "*" can be used
+	 * 5) execution will start with transport dataType and THEN continue down to "*" if needed
+	 */
+	prefilters = {},
+
+	/* Transports bindings
+	 * 1) key is the dataType
+	 * 2) the catchall symbol "*" can be used
+	 * 3) selection will start with transport dataType and THEN go to "*" if needed
+	 */
+	transports = {},
+
+	// Document location
+	ajaxLocation,
+
+	// Document location segments
+	ajaxLocParts;
+
+// #8138, IE may throw an exception when accessing
+// a field from window.location if document.domain has been set
+try {
+	ajaxLocation = location.href;
+} catch( e ) {
+	// Use the href attribute of an A element
+	// since IE will modify it given document.location
+	ajaxLocation = document.createElement( "a" );
+	ajaxLocation.href = "";
+	ajaxLocation = ajaxLocation.href;
+}
+
+// Segment location into parts
+ajaxLocParts = rurl.exec( ajaxLocation.toLowerCase() ) || [];
+
+// Base "constructor" for jQuery.ajaxPrefilter and jQuery.ajaxTransport
+function addToPrefiltersOrTransports( structure ) {
+
+	// dataTypeExpression is optional and defaults to "*"
+	return function( dataTypeExpression, func ) {
+
+		if ( typeof dataTypeExpression !== "string" ) {
+			func = dataTypeExpression;
+			dataTypeExpression = "*";
+		}
+
+		if ( jQuery.isFunction( func ) ) {
+			var dataTypes = dataTypeExpression.toLowerCase().split( rspacesAjax ),
+				i = 0,
+				length = dataTypes.length,
+				dataType,
+				list,
+				placeBefore;
+
+			// For each dataType in the dataTypeExpression
+			for(; i < length; i++ ) {
+				dataType = dataTypes[ i ];
+				// We control if we're asked to add before
+				// any existing element
+				placeBefore = /^\+/.test( dataType );
+				if ( placeBefore ) {
+					dataType = dataType.substr( 1 ) || "*";
+				}
+				list = structure[ dataType ] = structure[ dataType ] || [];
+				// then we add to the structure accordingly
+				list[ placeBefore ? "unshift" : "push" ]( func );
+			}
+		}
+	};
+}
+
+// Base inspection function for prefilters and transports
+function inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR,
+		dataType /* internal */, inspected /* internal */ ) {
+
+	dataType = dataType || options.dataTypes[ 0 ];
+	inspected = inspected || {};
+
+	inspected[ dataType ] = true;
+
+	var list = structure[ dataType ],
+		i = 0,
+		length = list ? list.length : 0,
+		executeOnly = ( structure === prefilters ),
+		selection;
+
+	for(; i < length && ( executeOnly || !selection ); i++ ) {
+		selection = list[ i ]( options, originalOptions, jqXHR );
+		// If we got redirected to another dataType
+		// we try there if executing only and not done already
+		if ( typeof selection === "string" ) {
+			if ( !executeOnly || inspected[ selection ] ) {
+				selection = undefined;
+			} else {
+				options.dataTypes.unshift( selection );
+				selection = inspectPrefiltersOrTransports(
+						structure, options, originalOptions, jqXHR, selection, inspected );
+			}
+		}
+	}
+	// If we're only executing or nothing was selected
+	// we try the catchall dataType if not done already
+	if ( ( executeOnly || !selection ) && !inspected[ "*" ] ) {
+		selection = inspectPrefiltersOrTransports(
+				structure, options, originalOptions, jqXHR, "*", inspected );
+	}
+	// unnecessary when only executing (prefilters)
+	// but it'll be ignored by the caller in that case
+	return selection;
+}
+
+jQuery.fn.extend({
+	load: function( url, params, callback ) {
+		if ( typeof url !== "string" && _load ) {
+			return _load.apply( this, arguments );
+
+		// Don't do a request if no elements are being requested
+		} else if ( !this.length ) {
+			return this;
+		}
+
+		var off = url.indexOf( " " );
+		if ( off >= 0 ) {
+			var selector = url.slice( off, url.length );
+			url = url.slice( 0, off );
+		}
+
+		// Default to a GET request
+		var type = "GET";
+
+		// If the second parameter was provided
+		if ( params ) {
+			// If it's a function
+			if ( jQuery.isFunction( params ) ) {
+				// We assume that it's the callback
+				callback = params;
+				params = undefined;
+
+			// Otherwise, build a param string
+			} else if ( typeof params === "object" ) {
+				params = jQuery.param( params, jQuery.ajaxSettings.traditional );
+				type = "POST";
+			}
+		}
+
+		var self = this;
+
+		// Request the remote document
+		jQuery.ajax({
+			url: url,
+			type: type,
+			dataType: "html",
+			data: params,
+			// Complete callback (responseText is used internally)
+			complete: function( jqXHR, status, responseText ) {
+				// Store the response as specified by the jqXHR object
+				responseText = jqXHR.responseText;
+				// If successful, inject the HTML into all the matched elements
+				if ( jqXHR.isResolved() ) {
+					// #4825: Get the actual response in case
+					// a dataFilter is present in ajaxSettings
+					jqXHR.done(function( r ) {
+						responseText = r;
+					});
+					// See if a selector was specified
+					self.html( selector ?
+						// Create a dummy div to hold the results
+						jQuery("<div>")
+							// inject the contents of the document in, removing the scripts
+							// to avoid any 'Permission Denied' errors in IE
+							.append(responseText.replace(rscript, ""))
+
+							// Locate the specified elements
+							.find(selector) :
+
+						// If not, just inject the full result
+						responseText );
+				}
+
+				if ( callback ) {
+					self.each( callback, [ responseText, status, jqXHR ] );
+				}
+			}
+		});
+
+		return this;
+	},
+
+	serialize: function() {
+		return jQuery.param( this.serializeArray() );
+	},
+
+	serializeArray: function() {
+		return this.map(function(){
+			return this.elements ? jQuery.makeArray( this.elements ) : this;
+		})
+		.filter(function(){
+			return this.name && !this.disabled &&
+				( this.checked || rselectTextarea.test( this.nodeName ) ||
+					rinput.test( this.type ) );
+		})
+		.map(function( i, elem ){
+			var val = jQuery( this ).val();
+
+			return val == null ?
+				null :
+				jQuery.isArray( val ) ?
+					jQuery.map( val, function( val, i ){
+						return { name: elem.name, value: val.replace( rCRLF, "\r\n" ) };
+					}) :
+					{ name: elem.name, value: val.replace( rCRLF, "\r\n" ) };
+		}).get();
+	}
+});
+
+// Attach a bunch of functions for handling common AJAX events
+jQuery.each( "ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split( " " ), function( i, o ){
+	jQuery.fn[ o ] = function( f ){
+		return this.bind( o, f );
+	};
+});
+
+jQuery.each( [ "get", "post" ], function( i, method ) {
+	jQuery[ method ] = function( url, data, callback, type ) {
+		// shift arguments if data argument was omitted
+		if ( jQuery.isFunction( data ) ) {
+			type = type || callback;
+			callback = data;
+			data = undefined;
+		}
+
+		return jQuery.ajax({
+			type: method,
+			url: url,
+			data: data,
+			success: callback,
+			dataType: type
+		});
+	};
+});
+
+jQuery.extend({
+
+	getScript: function( url, callback ) {
+		return jQuery.get( url, undefined, callback, "script" );
+	},
+
+	getJSON: function( url, data, callback ) {
+		return jQuery.get( url, data, callback, "json" );
+	},
+
+	// Creates a full fledged settings object into target
+	// with both ajaxSettings and settings fields.
+	// If target is omitted, writes into ajaxSettings.
+	ajaxSetup: function ( target, settings ) {
+		if ( !settings ) {
+			// Only one parameter, we extend ajaxSettings
+			settings = target;
+			target = jQuery.extend( true, jQuery.ajaxSettings, settings );
+		} else {
+			// target was provided, we extend into it
+			jQuery.extend( true, target, jQuery.ajaxSettings, settings );
+		}
+		// Flatten fields we don't want deep extended
+		for( var field in { context: 1, url: 1 } ) {
+			if ( field in settings ) {
+				target[ field ] = settings[ field ];
+			} else if( field in jQuery.ajaxSettings ) {
+				target[ field ] = jQuery.ajaxSettings[ field ];
+			}
+		}
+		return target;
+	},
+
+	ajaxSettings: {
+		url: ajaxLocation,
+		isLocal: rlocalProtocol.test( ajaxLocParts[ 1 ] ),
+		global: true,
+		type: "GET",
+		contentType: "application/x-www-form-urlencoded",
+		processData: true,
+		async: true,
+		/*
+		timeout: 0,
+		data: null,
+		dataType: null,
+		username: null,
+		password: null,
+		cache: null,
+		traditional: false,
+		headers: {},
+		*/
+
+		accepts: {
+			xml: "application/xml, text/xml",
+			html: "text/html",
+			text: "text/plain",
+			json: "application/json, text/javascript",
+			"*": "*/*"
+		},
+
+		contents: {
+			xml: /xml/,
+			html: /html/,
+			json: /json/
+		},
+
+		responseFields: {
+			xml: "responseXML",
+			text: "responseText"
+		},
+
+		// List of data converters
+		// 1) key format is "source_type destination_type" (a single space in-between)
+		// 2) the catchall symbol "*" can be used for source_type
+		converters: {
+
+			// Convert anything to text
+			"* text": window.String,
+
+			// Text to html (true = no transformation)
+			"text html": true,
+
+			// Evaluate text as a json expression
+			"text json": jQuery.parseJSON,
+
+			// Parse text as xml
+			"text xml": jQuery.parseXML
+		}
+	},
+
+	ajaxPrefilter: addToPrefiltersOrTransports( prefilters ),
+	ajaxTransport: addToPrefiltersOrTransports( transports ),
+
+	// Main method
+	ajax: function( url, options ) {
+
+		// If url is an object, simulate pre-1.5 signature
+		if ( typeof url === "object" ) {
+			options = url;
+			url = undefined;
+		}
+
+		// Force options to be an object
+		options = options || {};
+
+		var // Create the final options object
+			s = jQuery.ajaxSetup( {}, options ),
+			// Callbacks context
+			callbackContext = s.context || s,
+			// Context for global events
+			// It's the callbackContext if one was provided in the options
+			// and if it's a DOM node or a jQuery collection
+			globalEventContext = callbackContext !== s &&
+				( callbackContext.nodeType || callbackContext instanceof jQuery ) ?
+						jQuery( callbackContext ) : jQuery.event,
+			// Deferreds
+			deferred = jQuery.Deferred(),
+			completeDeferred = jQuery._Deferred(),
+			// Status-dependent callbacks
+			statusCode = s.statusCode || {},
+			// ifModified key
+			ifModifiedKey,
+			// Headers (they are sent all at once)
+			requestHeaders = {},
+			requestHeadersNames = {},
+			// Response headers
+			responseHeadersString,
+			responseHeaders,
+			// transport
+			transport,
+			// timeout handle
+			timeoutTimer,
+			// Cross-domain detection vars
+			parts,
+			// The jqXHR state
+			state = 0,
+			// To know if global events are to be dispatched
+			fireGlobals,
+			// Loop variable
+			i,
+			// Fake xhr
+			jqXHR = {
+
+				readyState: 0,
+
+				// Caches the header
+				setRequestHeader: function( name, value ) {
+					if ( !state ) {
+						var lname = name.toLowerCase();
+						name = requestHeadersNames[ lname ] = requestHeadersNames[ lname ] || name;
+						requestHeaders[ name ] = value;
+					}
+					return this;
+				},
+
+				// Raw string
+				getAllResponseHeaders: function() {
+					return state === 2 ? responseHeadersString : null;
+				},
+
+				// Builds headers hashtable if needed
+				getResponseHeader: function( key ) {
+					var match;
+					if ( state === 2 ) {
+						if ( !responseHeaders ) {
+							responseHeaders = {};
+							while( ( match = rheaders.exec( responseHeadersString ) ) ) {
+								responseHeaders[ match[1].toLowerCase() ] = match[ 2 ];
+							}
+						}
+						match = responseHeaders[ key.toLowerCase() ];
+					}
+					return match === undefined ? null : match;
+				},
+
+				// Overrides response content-type header
+				overrideMimeType: function( type ) {
+					if ( !state ) {
+						s.mimeType = type;
+					}
+					return this;
+				},
+
+				// Cancel the request
+				abort: function( statusText ) {
+					statusText = statusText || "abort";
+					if ( transport ) {
+						transport.abort( statusText );
+					}
+					done( 0, statusText );
+					return this;
+				}
+			};
+
+		// Callback for when everything is done
+		// It is defined here because jslint complains if it is declared
+		// at the end of the function (which would be more logical and readable)
+		function done( status, statusText, responses, headers ) {
+
+			// Called once
+			if ( state === 2 ) {
+				return;
+			}
+
+			// State is "done" now
+			state = 2;
+
+			// Clear timeout if it exists
+			if ( timeoutTimer ) {
+				clearTimeout( timeoutTimer );
+			}
+
+			// Dereference transport for early garbage collection
+			// (no matter how long the jqXHR object will be used)
+			transport = undefined;
+
+			// Cache response headers
+			responseHeadersString = headers || "";
+
+			// Set readyState
+			jqXHR.readyState = status ? 4 : 0;
+
+			var isSuccess,
+				success,
+				error,
+				response = responses ? ajaxHandleResponses( s, jqXHR, responses ) : undefined,
+				lastModified,
+				etag;
+
+			// If successful, handle type chaining
+			if ( status >= 200 && status < 300 || status === 304 ) {
+
+				// Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode.
+				if ( s.ifModified ) {
+
+					if ( ( lastModified = jqXHR.getResponseHeader( "Last-Modified" ) ) ) {
+						jQuery.lastModified[ ifModifiedKey ] = lastModified;
+					}
+					if ( ( etag = jqXHR.getResponseHeader( "Etag" ) ) ) {
+						jQuery.etag[ ifModifiedKey ] = etag;
+					}
+				}
+
+				// If not modified
+				if ( status === 304 ) {
+
+					statusText = "notmodified";
+					isSuccess = true;
+
+				// If we have data
+				} else {
+
+					try {
+						success = ajaxConvert( s, response );
+						statusText = "success";
+						isSuccess = true;
+					} catch(e) {
+						// We have a parsererror
+						statusText = "parsererror";
+						error = e;
+					}
+				}
+			} else {
+				// We extract error from statusText
+				// then normalize statusText and status for non-aborts
+				error = statusText;
+				if( !statusText || status ) {
+					statusText = "error";
+					if ( status < 0 ) {
+						status = 0;
+					}
+				}
+			}
+
+			// Set data for the fake xhr object
+			jqXHR.status = status;
+			jqXHR.statusText = statusText;
+
+			// Success/Error
+			if ( isSuccess ) {
+				deferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] );
+			} else {
+				deferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] );
+			}
+
+			// Status-dependent callbacks
+			jqXHR.statusCode( statusCode );
+			statusCode = undefined;
+
+			if ( fireGlobals ) {
+				globalEventContext.trigger( "ajax" + ( isSuccess ? "Success" : "Error" ),
+						[ jqXHR, s, isSuccess ? success : error ] );
+			}
+
+			// Complete
+			completeDeferred.resolveWith( callbackContext, [ jqXHR, statusText ] );
+
+			if ( fireGlobals ) {
+				globalEventContext.trigger( "ajaxComplete", [ jqXHR, s] );
+				// Handle the global AJAX counter
+				if ( !( --jQuery.active ) ) {
+					jQuery.event.trigger( "ajaxStop" );
+				}
+			}
+		}
+
+		// Attach deferreds
+		deferred.promise( jqXHR );
+		jqXHR.success = jqXHR.done;
+		jqXHR.error = jqXHR.fail;
+		jqXHR.complete = completeDeferred.done;
+
+		// Status-dependent callbacks
+		jqXHR.statusCode = function( map ) {
+			if ( map ) {
+				var tmp;
+				if ( state < 2 ) {
+					for( tmp in map ) {
+						statusCode[ tmp ] = [ statusCode[tmp], map[tmp] ];
+					}
+				} else {
+					tmp = map[ jqXHR.status ];
+					jqXHR.then( tmp, tmp );
+				}
+			}
+			return this;
+		};
+
+		// Remove hash character (#7531: and string promotion)
+		// Add protocol if not provided (#5866: IE7 issue with protocol-less urls)
+		// We also use the url parameter if available
+		s.url = ( ( url || s.url ) + "" ).replace( rhash, "" ).replace( rprotocol, ajaxLocParts[ 1 ] + "//" );
+
+		// Extract dataTypes list
+		s.dataTypes = jQuery.trim( s.dataType || "*" ).toLowerCase().split( rspacesAjax );
+
+		// Determine if a cross-domain request is in order
+		if ( s.crossDomain == null ) {
+			parts = rurl.exec( s.url.toLowerCase() );
+			s.crossDomain = !!( parts &&
+				( parts[ 1 ] != ajaxLocParts[ 1 ] || parts[ 2 ] != ajaxLocParts[ 2 ] ||
+					( parts[ 3 ] || ( parts[ 1 ] === "http:" ? 80 : 443 ) ) !=
+						( ajaxLocParts[ 3 ] || ( ajaxLocParts[ 1 ] === "http:" ? 80 : 443 ) ) )
+			);
+		}
+
+		// Convert data if not already a string
+		if ( s.data && s.processData && typeof s.data !== "string" ) {
+			s.data = jQuery.param( s.data, s.traditional );
+		}
+
+		// Apply prefilters
+		inspectPrefiltersOrTransports( prefilters, s, options, jqXHR );
+
+		// If request was aborted inside a prefiler, stop there
+		if ( state === 2 ) {
+			return false;
+		}
+
+		// We can fire global events as of now if asked to
+		fireGlobals = s.global;
+
+		// Uppercase the type
+		s.type = s.type.toUpperCase();
+
+		// Determine if request has content
+		s.hasContent = !rnoContent.test( s.type );
+
+		// Watch for a new set of requests
+		if ( fireGlobals && jQuery.active++ === 0 ) {
+			jQuery.event.trigger( "ajaxStart" );
+		}
+
+		// More options handling for requests with no content
+		if ( !s.hasContent ) {
+
+			// If data is available, append data to url
+			if ( s.data ) {
+				s.url += ( rquery.test( s.url ) ? "&" : "?" ) + s.data;
+			}
+
+			// Get ifModifiedKey before adding the anti-cache parameter
+			ifModifiedKey = s.url;
+
+			// Add anti-cache in url if needed
+			if ( s.cache === false ) {
+
+				var ts = jQuery.now(),
+					// try replacing _= if it is there
+					ret = s.url.replace( rts, "$1_=" + ts );
+
+				// if nothing was replaced, add timestamp to the end
+				s.url = ret + ( (ret === s.url ) ? ( rquery.test( s.url ) ? "&" : "?" ) + "_=" + ts : "" );
+			}
+		}
+
+		// Set the correct header, if data is being sent
+		if ( s.data && s.hasContent && s.contentType !== false || options.contentType ) {
+			jqXHR.setRequestHeader( "Content-Type", s.contentType );
+		}
+
+		// Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode.
+		if ( s.ifModified ) {
+			ifModifiedKey = ifModifiedKey || s.url;
+			if ( jQuery.lastModified[ ifModifiedKey ] ) {
+				jqXHR.setRequestHeader( "If-Modified-Since", jQuery.lastModified[ ifModifiedKey ] );
+			}
+			if ( jQuery.etag[ ifModifiedKey ] ) {
+				jqXHR.setRequestHeader( "If-None-Match", jQuery.etag[ ifModifiedKey ] );
+			}
+		}
+
+		// Set the Accepts header for the server, depending on the dataType
+		jqXHR.setRequestHeader(
+			"Accept",
+			s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[0] ] ?
+				s.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== "*" ? ", */*; q=0.01" : "" ) :
+				s.accepts[ "*" ]
+		);
+
+		// Check for headers option
+		for ( i in s.headers ) {
+			jqXHR.setRequestHeader( i, s.headers[ i ] );
+		}
+
+		// Allow custom headers/mimetypes and early abort
+		if ( s.beforeSend && ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || state === 2 ) ) {
+				// Abort if not done already
+				jqXHR.abort();
+				return false;
+
+		}
+
+		// Install callbacks on deferreds
+		for ( i in { success: 1, error: 1, complete: 1 } ) {
+			jqXHR[ i ]( s[ i ] );
+		}
+
+		// Get transport
+		transport = inspectPrefiltersOrTransports( transports, s, options, jqXHR );
+
+		// If no transport, we auto-abort
+		if ( !transport ) {
+			done( -1, "No Transport" );
+		} else {
+			jqXHR.readyState = 1;
+			// Send global event
+			if ( fireGlobals ) {
+				globalEventContext.trigger( "ajaxSend", [ jqXHR, s ] );
+			}
+			// Timeout
+			if ( s.async && s.timeout > 0 ) {
+				timeoutTimer = setTimeout( function(){
+					jqXHR.abort( "timeout" );
+				}, s.timeout );
+			}
+
+			try {
+				state = 1;
+				transport.send( requestHeaders, done );
+			} catch (e) {
+				// Propagate exception as error if not done
+				if ( status < 2 ) {
+					done( -1, e );
+				// Simply rethrow otherwise
+				} else {
+					jQuery.error( e );
+				}
+			}
+		}
+
+		return jqXHR;
+	},
+
+	// Serialize an array of form elements or a set of
+	// key/values into a query string
+	param: function( a, traditional ) {
+		var s = [],
+			add = function( key, value ) {
+				// If value is a function, invoke it and return its value
+				value = jQuery.isFunction( value ) ? value() : value;
+				s[ s.length ] = encodeURIComponent( key ) + "=" + encodeURIComponent( value );
+			};
+
+		// Set traditional to true for jQuery <= 1.3.2 behavior.
+		if ( traditional === undefined ) {
+			traditional = jQuery.ajaxSettings.traditional;
+		}
+
+		// If an array was passed in, assume that it is an array of form elements.
+		if ( jQuery.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) {
+			// Serialize the form elements
+			jQuery.each( a, function() {
+				add( this.name, this.value );
+			});
+
+		} else {
+			// If traditional, encode the "old" way (the way 1.3.2 or older
+			// did it), otherwise encode params recursively.
+			for ( var prefix in a ) {
+				buildParams( prefix, a[ prefix ], traditional, add );
+			}
+		}
+
+		// Return the resulting serialization
+		return s.join( "&" ).replace( r20, "+" );
+	}
+});
+
+function buildParams( prefix, obj, traditional, add ) {
+	if ( jQuery.isArray( obj ) ) {
+		// Serialize array item.
+		jQuery.each( obj, function( i, v ) {
+			if ( traditional || rbracket.test( prefix ) ) {
+				// Treat each array item as a scalar.
+				add( prefix, v );
+
+			} else {
+				// If array item is non-scalar (array or object), encode its
+				// numeric index to resolve deserialization ambiguity issues.
+				// Note that rack (as of 1.0.0) can't currently deserialize
+				// nested arrays properly, and attempting to do so may cause
+				// a server error. Possible fixes are to modify rack's
+				// deserialization algorithm or to provide an option or flag
+				// to force array serialization to be shallow.
+				buildParams( prefix + "[" + ( typeof v === "object" || jQuery.isArray(v) ? i : "" ) + "]", v, traditional, add );
+			}
+		});
+
+	} else if ( !traditional && obj != null && typeof obj === "object" ) {
+		// Serialize object item.
+		for ( var name in obj ) {
+			buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add );
+		}
+
+	} else {
+		// Serialize scalar item.
+		add( prefix, obj );
+	}
+}
+
+// This is still on the jQuery object... for now
+// Want to move this to jQuery.ajax some day
+jQuery.extend({
+
+	// Counter for holding the number of active queries
+	active: 0,
+
+	// Last-Modified header cache for next request
+	lastModified: {},
+	etag: {}
+
+});
+
+/* Handles responses to an ajax request:
+ * - sets all responseXXX fields accordingly
+ * - finds the right dataType (mediates between content-type and expected dataType)
+ * - returns the corresponding response
+ */
+function ajaxHandleResponses( s, jqXHR, responses ) {
+
+	var contents = s.contents,
+		dataTypes = s.dataTypes,
+		responseFields = s.responseFields,
+		ct,
+		type,
+		finalDataType,
+		firstDataType;
+
+	// Fill responseXXX fields
+	for( type in responseFields ) {
+		if ( type in responses ) {
+			jqXHR[ responseFields[type] ] = responses[ type ];
+		}
+	}
+
+	// Remove auto dataType and get content-type in the process
+	while( dataTypes[ 0 ] === "*" ) {
+		dataTypes.shift();
+		if ( ct === undefined ) {
+			ct = s.mimeType || jqXHR.getResponseHeader( "content-type" );
+		}
+	}
+
+	// Check if we're dealing with a known content-type
+	if ( ct ) {
+		for ( type in contents ) {
+			if ( contents[ type ] && contents[ type ].test( ct ) ) {
+				dataTypes.unshift( type );
+				break;
+			}
+		}
+	}
+
+	// Check to see if we have a response for the expected dataType
+	if ( dataTypes[ 0 ] in responses ) {
+		finalDataType = dataTypes[ 0 ];
+	} else {
+		// Try convertible dataTypes
+		for ( type in responses ) {
+			if ( !dataTypes[ 0 ] || s.converters[ type + " " + dataTypes[0] ] ) {
+				finalDataType = type;
+				break;
+			}
+			if ( !firstDataType ) {
+				firstDataType = type;
+			}
+		}
+		// Or just use first one
+		finalDataType = finalDataType || firstDataType;
+	}
+
+	// If we found a dataType
+	// We add the dataType to the list if needed
+	// and return the corresponding response
+	if ( finalDataType ) {
+		if ( finalDataType !== dataTypes[ 0 ] ) {
+			dataTypes.unshift( finalDataType );
+		}
+		return responses[ finalDataType ];
+	}
+}
+
+// Chain conversions given the request and the original response
+function ajaxConvert( s, response ) {
+
+	// Apply the dataFilter if provided
+	if ( s.dataFilter ) {
+		response = s.dataFilter( response, s.dataType );
+	}
+
+	var dataTypes = s.dataTypes,
+		converters = {},
+		i,
+		key,
+		length = dataTypes.length,
+		tmp,
+		// Current and previous dataTypes
+		current = dataTypes[ 0 ],
+		prev,
+		// Conversion expression
+		conversion,
+		// Conversion function
+		conv,
+		// Conversion functions (transitive conversion)
+		conv1,
+		conv2;
+
+	// For each dataType in the chain
+	for( i = 1; i < length; i++ ) {
+
+		// Create converters map
+		// with lowercased keys
+		if ( i === 1 ) {
+			for( key in s.converters ) {
+				if( typeof key === "string" ) {
+					converters[ key.toLowerCase() ] = s.converters[ key ];
+				}
+			}
+		}
+
+		// Get the dataTypes
+		prev = current;
+		current = dataTypes[ i ];
+
+		// If current is auto dataType, update it to prev
+		if( current === "*" ) {
+			current = prev;
+		// If no auto and dataTypes are actually different
+		} else if ( prev !== "*" && prev !== current ) {
+
+			// Get the converter
+			conversion = prev + " " + current;
+			conv = converters[ conversion ] || converters[ "* " + current ];
+
+			// If there is no direct converter, search transitively
+			if ( !conv ) {
+				conv2 = undefined;
+				for( conv1 in converters ) {
+					tmp = conv1.split( " " );
+					if ( tmp[ 0 ] === prev || tmp[ 0 ] === "*" ) {
+						conv2 = converters[ tmp[1] + " " + current ];
+						if ( conv2 ) {
+							conv1 = converters[ conv1 ];
+							if ( conv1 === true ) {
+								conv = conv2;
+							} else if ( conv2 === true ) {
+								conv = conv1;
+							}
+							break;
+						}
+					}
+				}
+			}
+			// If we found no converter, dispatch an error
+			if ( !( conv || conv2 ) ) {
+				jQuery.error( "No conversion from " + conversion.replace(" "," to ") );
+			}
+			// If found converter is not an equivalence
+			if ( conv !== true ) {
+				// Convert with 1 or 2 converters accordingly
+				response = conv ? conv( response ) : conv2( conv1(response) );
+			}
+		}
+	}
+	return response;
+}
+
+
+
+
+var jsc = jQuery.now(),
+	jsre = /(\=)\?(&|$)|\?\?/i;
+
+// Default jsonp settings
+jQuery.ajaxSetup({
+	jsonp: "callback",
+	jsonpCallback: function() {
+		return jQuery.expando + "_" + ( jsc++ );
+	}
+});
+
+// Detect, normalize options and install callbacks for jsonp requests
+jQuery.ajaxPrefilter( "json jsonp", function( s, originalSettings, jqXHR ) {
+
+	var inspectData = s.contentType === "application/x-www-form-urlencoded" &&
+		( typeof s.data === "string" );
+
+	if ( s.dataTypes[ 0 ] === "jsonp" ||
+		s.jsonp !== false && ( jsre.test( s.url ) ||
+				inspectData && jsre.test( s.data ) ) ) {
+
+		var responseContainer,
+			jsonpCallback = s.jsonpCallback =
+				jQuery.isFunction( s.jsonpCallback ) ? s.jsonpCallback() : s.jsonpCallback,
+			previous = window[ jsonpCallback ],
+			url = s.url,
+			data = s.data,
+			replace = "$1" + jsonpCallback + "$2";
+
+		if ( s.jsonp !== false ) {
+			url = url.replace( jsre, replace );
+			if ( s.url === url ) {
+				if ( inspectData ) {
+					data = data.replace( jsre, replace );
+				}
+				if ( s.data === data ) {
+					// Add callback manually
+					url += (/\?/.test( url ) ? "&" : "?") + s.jsonp + "=" + jsonpCallback;
+				}
+			}
+		}
+
+		s.url = url;
+		s.data = data;
+
+		// Install callback
+		window[ jsonpCallback ] = function( response ) {
+			responseContainer = [ response ];
+		};
+
+		// Clean-up function
+		jqXHR.always(function() {
+			// Set callback back to previous value
+			window[ jsonpCallback ] = previous;
+			// Call if it was a function and we have a response
+			if ( responseContainer && jQuery.isFunction( previous ) ) {
+				window[ jsonpCallback ]( responseContainer[ 0 ] );
+			}
+		});
+
+		// Use data converter to retrieve json after script execution
+		s.converters["script json"] = function() {
+			if ( !responseContainer ) {
+				jQuery.error( jsonpCallback + " was not called" );
+			}
+			return responseContainer[ 0 ];
+		};
+
+		// force json dataType
+		s.dataTypes[ 0 ] = "json";
+
+		// Delegate to script
+		return "script";
+	}
+});
+
+
+
+
+// Install script dataType
+jQuery.ajaxSetup({
+	accepts: {
+		script: "text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"
+	},
+	contents: {
+		script: /javascript|ecmascript/
+	},
+	converters: {
+		"text script": function( text ) {
+			jQuery.globalEval( text );
+			return text;
+		}
+	}
+});
+
+// Handle cache's special case and global
+jQuery.ajaxPrefilter( "script", function( s ) {
+	if ( s.cache === undefined ) {
+		s.cache = false;
+	}
+	if ( s.crossDomain ) {
+		s.type = "GET";
+		s.global = false;
+	}
+});
+
+// Bind script tag hack transport
+jQuery.ajaxTransport( "script", function(s) {
+
+	// This transport only deals with cross domain requests
+	if ( s.crossDomain ) {
+
+		var script,
+			head = document.head || document.getElementsByTagName( "head" )[0] || document.documentElement;
+
+		return {
+
+			send: function( _, callback ) {
+
+				script = document.createElement( "script" );
+
+				script.async = "async";
+
+				if ( s.scriptCharset ) {
+					script.charset = s.scriptCharset;
+				}
+
+				script.src = s.url;
+
+				// Attach handlers for all browsers
+				script.onload = script.onreadystatechange = function( _, isAbort ) {
+
+					if ( isAbort || !script.readyState || /loaded|complete/.test( script.readyState ) ) {
+
+						// Handle memory leak in IE
+						script.onload = script.onreadystatechange = null;
+
+						// Remove the script
+						if ( head && script.parentNode ) {
+							head.removeChild( script );
+						}
+
+						// Dereference the script
+						script = undefined;
+
+						// Callback if not abort
+						if ( !isAbort ) {
+							callback( 200, "success" );
+						}
+					}
+				};
+				// Use insertBefore instead of appendChild  to circumvent an IE6 bug.
+				// This arises when a base node is used (#2709 and #4378).
+				head.insertBefore( script, head.firstChild );
+			},
+
+			abort: function() {
+				if ( script ) {
+					script.onload( 0, 1 );
+				}
+			}
+		};
+	}
+});
+
+
+
+
+var // #5280: Internet Explorer will keep connections alive if we don't abort on unload
+	xhrOnUnloadAbort = window.ActiveXObject ? function() {
+		// Abort all pending requests
+		for ( var key in xhrCallbacks ) {
+			xhrCallbacks[ key ]( 0, 1 );
+		}
+	} : false,
+	xhrId = 0,
+	xhrCallbacks;
+
+// Functions to create xhrs
+function createStandardXHR() {
+	try {
+		return new window.XMLHttpRequest();
+	} catch( e ) {}
+}
+
+function createActiveXHR() {
+	try {
+		return new window.ActiveXObject( "Microsoft.XMLHTTP" );
+	} catch( e ) {}
+}
+
+// Create the request object
+// (This is still attached to ajaxSettings for backward compatibility)
+jQuery.ajaxSettings.xhr = window.ActiveXObject ?
+	/* Microsoft failed to properly
+	 * implement the XMLHttpRequest in IE7 (can't request local files),
+	 * so we use the ActiveXObject when it is available
+	 * Additionally XMLHttpRequest can be disabled in IE7/IE8 so
+	 * we need a fallback.
+	 */
+	function() {
+		return !this.isLocal && createStandardXHR() || createActiveXHR();
+	} :
+	// For all other browsers, use the standard XMLHttpRequest object
+	createStandardXHR;
+
+// Determine support properties
+(function( xhr ) {
+	jQuery.extend( jQuery.support, {
+		ajax: !!xhr,
+		cors: !!xhr && ( "withCredentials" in xhr )
+	});
+})( jQuery.ajaxSettings.xhr() );
+
+// Create transport if the browser can provide an xhr
+if ( jQuery.support.ajax ) {
+
+	jQuery.ajaxTransport(function( s ) {
+		// Cross domain only allowed if supported through XMLHttpRequest
+		if ( !s.crossDomain || jQuery.support.cors ) {
+
+			var callback;
+
+			return {
+				send: function( headers, complete ) {
+
+					// Get a new xhr
+					var xhr = s.xhr(),
+						handle,
+						i;
+
+					// Open the socket
+					// Passing null username, generates a login popup on Opera (#2865)
+					if ( s.username ) {
+						xhr.open( s.type, s.url, s.async, s.username, s.password );
+					} else {
+						xhr.open( s.type, s.url, s.async );
+					}
+
+					// Apply custom fields if provided
+					if ( s.xhrFields ) {
+						for ( i in s.xhrFields ) {
+							xhr[ i ] = s.xhrFields[ i ];
+						}
+					}
+
+					// Override mime type if needed
+					if ( s.mimeType && xhr.overrideMimeType ) {
+						xhr.overrideMimeType( s.mimeType );
+					}
+
+					// X-Requested-With header
+					// For cross-domain requests, seeing as conditions for a preflight are
+					// akin to a jigsaw puzzle, we simply never set it to be sure.
+					// (it can always be set on a per-request basis or even using ajaxSetup)
+					// For same-domain requests, won't change header if already provided.
+					if ( !s.crossDomain && !headers["X-Requested-With"] ) {
+						headers[ "X-Requested-With" ] = "XMLHttpRequest";
+					}
+
+					// Need an extra try/catch for cross domain requests in Firefox 3
+					try {
+						for ( i in headers ) {
+							xhr.setRequestHeader( i, headers[ i ] );
+						}
+					} catch( _ ) {}
+
+					// Do send the request
+					// This may raise an exception which is actually
+					// handled in jQuery.ajax (so no try/catch here)
+					xhr.send( ( s.hasContent && s.data ) || null );
+
+					// Listener
+					callback = function( _, isAbort ) {
+
+						var status,
+							statusText,
+							responseHeaders,
+							responses,
+							xml;
+
+						// Firefox throws exceptions when accessing properties
+						// of an xhr when a network error occured
+						// http://helpful.knobs-dials.com/index.php/Component_returned_failure_code:_0x80040111_(NS_ERROR_NOT_AVAILABLE)
+						try {
+
+							// Was never called and is aborted or complete
+							if ( callback && ( isAbort || xhr.readyState === 4 ) ) {
+
+								// Only called once
+								callback = undefined;
+
+								// Do not keep as active anymore
+								if ( handle ) {
+									xhr.onreadystatechange = jQuery.noop;
+									if ( xhrOnUnloadAbort ) {
+										delete xhrCallbacks[ handle ];
+									}
+								}
+
+								// If it's an abort
+								if ( isAbort ) {
+									// Abort it manually if needed
+									if ( xhr.readyState !== 4 ) {
+										xhr.abort();
+									}
+								} else {
+									status = xhr.status;
+									responseHeaders = xhr.getAllResponseHeaders();
+									responses = {};
+									xml = xhr.responseXML;
+
+									// Construct response list
+									if ( xml && xml.documentElement /* #4958 */ ) {
+										responses.xml = xml;
+									}
+									responses.text = xhr.responseText;
+
+									// Firefox throws an exception when accessing
+									// statusText for faulty cross-domain requests
+									try {
+										statusText = xhr.statusText;
+									} catch( e ) {
+										// We normalize with Webkit giving an empty statusText
+										statusText = "";
+									}
+
+									// Filter status for non standard behaviors
+
+									// If the request is local and we have data: assume a success
+									// (success with no data won't get notified, that's the best we
+									// can do given current implementations)
+									if ( !status && s.isLocal && !s.crossDomain ) {
+										status = responses.text ? 200 : 404;
+									// IE - #1450: sometimes returns 1223 when it should be 204
+									} else if ( status === 1223 ) {
+										status = 204;
+									}
+								}
+							}
+						} catch( firefoxAccessException ) {
+							if ( !isAbort ) {
+								complete( -1, firefoxAccessException );
+							}
+						}
+
+						// Call complete if needed
+						if ( responses ) {
+							complete( status, statusText, responses, responseHeaders );
+						}
+					};
+
+					// if we're in sync mode or it's in cache
+					// and has been retrieved directly (IE6 & IE7)
+					// we need to manually fire the callback
+					if ( !s.async || xhr.readyState === 4 ) {
+						callback();
+					} else {
+						handle = ++xhrId;
+						if ( xhrOnUnloadAbort ) {
+							// Create the active xhrs callbacks list if needed
+							// and attach the unload handler
+							if ( !xhrCallbacks ) {
+								xhrCallbacks = {};
+								jQuery( window ).unload( xhrOnUnloadAbort );
+							}
+							// Add to list of active xhrs callbacks
+							xhrCallbacks[ handle ] = callback;
+						}
+						xhr.onreadystatechange = callback;
+					}
+				},
+
+				abort: function() {
+					if ( callback ) {
+						callback(0,1);
+					}
+				}
+			};
+		}
+	});
+}
+
+
+
+
+var elemdisplay = {},
+	iframe, iframeDoc,
+	rfxtypes = /^(?:toggle|show|hide)$/,
+	rfxnum = /^([+\-]=)?([\d+.\-]+)([a-z%]*)$/i,
+	timerId,
+	fxAttrs = [
+		// height animations
+		[ "height", "marginTop", "marginBottom", "paddingTop", "paddingBottom" ],
+		// width animations
+		[ "width", "marginLeft", "marginRight", "paddingLeft", "paddingRight" ],
+		// opacity animations
+		[ "opacity" ]
+	],
+	fxNow,
+	requestAnimationFrame = window.webkitRequestAnimationFrame ||
+		window.mozRequestAnimationFrame ||
+		window.oRequestAnimationFrame;
+
+jQuery.fn.extend({
+	show: function( speed, easing, callback ) {
+		var elem, display;
+
+		if ( speed || speed === 0 ) {
+			return this.animate( genFx("show", 3), speed, easing, callback);
+
+		} else {
+			for ( var i = 0, j = this.length; i < j; i++ ) {
+				elem = this[i];
+
+				if ( elem.style ) {
+					display = elem.style.display;
+
+					// Reset the inline display of this element to learn if it is
+					// being hidden by cascaded rules or not
+					if ( !jQuery._data(elem, "olddisplay") && display === "none" ) {
+						display = elem.style.display = "";
+					}
+
+					// Set elements which have been overridden with display: none
+					// in a stylesheet to whatever the default browser style is
+					// for such an element
+					if ( display === "" && jQuery.css( elem, "display" ) === "none" ) {
+						jQuery._data(elem, "olddisplay", defaultDisplay(elem.nodeName));
+					}
+				}
+			}
+
+			// Set the display of most of the elements in a second loop
+			// to avoid the constant reflow
+			for ( i = 0; i < j; i++ ) {
+				elem = this[i];
+
+				if ( elem.style ) {
+					display = elem.style.display;
+
+					if ( display === "" || display === "none" ) {
+						elem.style.display = jQuery._data(elem, "olddisplay") || "";
+					}
+				}
+			}
+
+			return this;
+		}
+	},
+
+	hide: function( speed, easing, callback ) {
+		if ( speed || speed === 0 ) {
+			return this.animate( genFx("hide", 3), speed, easing, callback);
+
+		} else {
+			for ( var i = 0, j = this.length; i < j; i++ ) {
+				if ( this[i].style ) {
+					var display = jQuery.css( this[i], "display" );
+
+					if ( display !== "none" && !jQuery._data( this[i], "olddisplay" ) ) {
+						jQuery._data( this[i], "olddisplay", display );
+					}
+				}
+			}
+
+			// Set the display of the elements in a second loop
+			// to avoid the constant reflow
+			for ( i = 0; i < j; i++ ) {
+				if ( this[i].style ) {
+					this[i].style.display = "none";
+				}
+			}
+
+			return this;
+		}
+	},
+
+	// Save the old toggle function
+	_toggle: jQuery.fn.toggle,
+
+	toggle: function( fn, fn2, callback ) {
+		var bool = typeof fn === "boolean";
+
+		if ( jQuery.isFunction(fn) && jQuery.isFunction(fn2) ) {
+			this._toggle.apply( this, arguments );
+
+		} else if ( fn == null || bool ) {
+			this.each(function() {
+				var state = bool ? fn : jQuery(this).is(":hidden");
+				jQuery(this)[ state ? "show" : "hide" ]();
+			});
+
+		} else {
+			this.animate(genFx("toggle", 3), fn, fn2, callback);
+		}
+
+		return this;
+	},
+
+	fadeTo: function( speed, to, easing, callback ) {
+		return this.filter(":hidden").css("opacity", 0).show().end()
+					.animate({opacity: to}, speed, easing, callback);
+	},
+
+	animate: function( prop, speed, easing, callback ) {
+		var optall = jQuery.speed(speed, easing, callback);
+
+		if ( jQuery.isEmptyObject( prop ) ) {
+			return this.each( optall.complete, [ false ] );
+		}
+
+		// Do not change referenced properties as per-property easing will be lost
+		prop = jQuery.extend( {}, prop );
+
+		return this[ optall.queue === false ? "each" : "queue" ](function() {
+			// XXX 'this' does not always have a nodeName when running the
+			// test suite
+
+			if ( optall.queue === false ) {
+				jQuery._mark( this );
+			}
+
+			var opt = jQuery.extend( {}, optall ),
+				isElement = this.nodeType === 1,
+				hidden = isElement && jQuery(this).is(":hidden"),
+				name, val, p,
+				display, e,
+				parts, start, end, unit;
+
+			// will store per property easing and be used to determine when an animation is complete
+			opt.animatedProperties = {};
+
+			for ( p in prop ) {
+
+				// property name normalization
+				name = jQuery.camelCase( p );
+				if ( p !== name ) {
+					prop[ name ] = prop[ p ];
+					delete prop[ p ];
+				}
+
+				val = prop[ name ];
+
+				// easing resolution: per property > opt.specialEasing > opt.easing > 'swing' (default)
+				if ( jQuery.isArray( val ) ) {
+					opt.animatedProperties[ name ] = val[ 1 ];
+					val = prop[ name ] = val[ 0 ];
+				} else {
+					opt.animatedProperties[ name ] = opt.specialEasing && opt.specialEasing[ name ] || opt.easing || 'swing';
+				}
+
+				if ( val === "hide" && hidden || val === "show" && !hidden ) {
+					return opt.complete.call( this );
+				}
+
+				if ( isElement && ( name === "height" || name === "width" ) ) {
+					// Make sure that nothing sneaks out
+					// Record all 3 overflow attributes because IE does not
+					// change the overflow attribute when overflowX and
+					// overflowY are set to the same value
+					opt.overflow = [ this.style.overflow, this.style.overflowX, this.style.overflowY ];
+
+					// Set display property to inline-block for height/width
+					// animations on inline elements that are having width/height
+					// animated
+					if ( jQuery.css( this, "display" ) === "inline" &&
+							jQuery.css( this, "float" ) === "none" ) {
+						if ( !jQuery.support.inlineBlockNeedsLayout ) {
+							this.style.display = "inline-block";
+
+						} else {
+							display = defaultDisplay( this.nodeName );
+
+							// inline-level elements accept inline-block;
+							// block-level elements need to be inline with layout
+							if ( display === "inline" ) {
+								this.style.display = "inline-block";
+
+							} else {
+								this.style.display = "inline";
+								this.style.zoom = 1;
+							}
+						}
+					}
+				}
+			}
+
+			if ( opt.overflow != null ) {
+				this.style.overflow = "hidden";
+			}
+
+			for ( p in prop ) {
+				e = new jQuery.fx( this, opt, p );
+				val = prop[ p ];
+
+				if ( rfxtypes.test(val) ) {
+					e[ val === "toggle" ? hidden ? "show" : "hide" : val ]();
+
+				} else {
+					parts = rfxnum.exec( val );
+					start = e.cur();
+
+					if ( parts ) {
+						end = parseFloat( parts[2] );
+						unit = parts[3] || ( jQuery.cssNumber[ p ] ? "" : "px" );
+
+						// We need to compute starting value
+						if ( unit !== "px" ) {
+							jQuery.style( this, p, (end || 1) + unit);
+							start = ((end || 1) / e.cur()) * start;
+							jQuery.style( this, p, start + unit);
+						}
+
+						// If a +=/-= token was provided, we're doing a relative animation
+						if ( parts[1] ) {
+							end = ( (parts[ 1 ] === "-=" ? -1 : 1) * end ) + start;
+						}
+
+						e.custom( start, end, unit );
+
+					} else {
+						e.custom( start, val, "" );
+					}
+				}
+			}
+
+			// For JS strict compliance
+			return true;
+		});
+	},
+
+	stop: function( clearQueue, gotoEnd ) {
+		if ( clearQueue ) {
+			this.queue([]);
+		}
+
+		this.each(function() {
+			var timers = jQuery.timers,
+				i = timers.length;
+			// clear marker counters if we know they won't be
+			if ( !gotoEnd ) {
+				jQuery._unmark( true, this );
+			}
+			while ( i-- ) {
+				if ( timers[i].elem === this ) {
+					if (gotoEnd) {
+						// force the next step to be the last
+						timers[i](true);
+					}
+
+					timers.splice(i, 1);
+				}
+			}
+		});
+
+		// start the next in the queue if the last step wasn't forced
+		if ( !gotoEnd ) {
+			this.dequeue();
+		}
+
+		return this;
+	}
+
+});
+
+// Animations created synchronously will run synchronously
+function createFxNow() {
+	setTimeout( clearFxNow, 0 );
+	return ( fxNow = jQuery.now() );
+}
+
+function clearFxNow() {
+	fxNow = undefined;
+}
+
+// Generate parameters to create a standard animation
+function genFx( type, num ) {
+	var obj = {};
+
+	jQuery.each( fxAttrs.concat.apply([], fxAttrs.slice(0,num)), function() {
+		obj[ this ] = type;
+	});
+
+	return obj;
+}
+
+// Generate shortcuts for custom animations
+jQuery.each({
+	slideDown: genFx("show", 1),
+	slideUp: genFx("hide", 1),
+	slideToggle: genFx("toggle", 1),
+	fadeIn: { opacity: "show" },
+	fadeOut: { opacity: "hide" },
+	fadeToggle: { opacity: "toggle" }
+}, function( name, props ) {
+	jQuery.fn[ name ] = function( speed, easing, callback ) {
+		return this.animate( props, speed, easing, callback );
+	};
+});
+
+jQuery.extend({
+	speed: function( speed, easing, fn ) {
+		var opt = speed && typeof speed === "object" ? jQuery.extend({}, speed) : {
+			complete: fn || !fn && easing ||
+				jQuery.isFunction( speed ) && speed,
+			duration: speed,
+			easing: fn && easing || easing && !jQuery.isFunction(easing) && easing
+		};
+
+		opt.duration = jQuery.fx.off ? 0 : typeof opt.duration === "number" ? opt.duration :
+			opt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[opt.duration] : jQuery.fx.speeds._default;
+
+		// Queueing
+		opt.old = opt.complete;
+		opt.complete = function( noUnmark ) {
+			if ( jQuery.isFunction( opt.old ) ) {
+				opt.old.call( this );
+			}
+
+			if ( opt.queue !== false ) {
+				jQuery.dequeue( this );
+			} else if ( noUnmark !== false ) {
+				jQuery._unmark( this );
+			}
+		};
+
+		return opt;
+	},
+
+	easing: {
+		linear: function( p, n, firstNum, diff ) {
+			return firstNum + diff * p;
+		},
+		swing: function( p, n, firstNum, diff ) {
+			return ((-Math.cos(p*Math.PI)/2) + 0.5) * diff + firstNum;
+		}
+	},
+
+	timers: [],
+
+	fx: function( elem, options, prop ) {
+		this.options = options;
+		this.elem = elem;
+		this.prop = prop;
+
+		options.orig = options.orig || {};
+	}
+
+});
+
+jQuery.fx.prototype = {
+	// Simple function for setting a style value
+	update: function() {
+		if ( this.options.step ) {
+			this.options.step.call( this.elem, this.now, this );
+		}
+
+		(jQuery.fx.step[this.prop] || jQuery.fx.step._default)( this );
+	},
+
+	// Get the current size
+	cur: function() {
+		if ( this.elem[this.prop] != null && (!this.elem.style || this.elem.style[this.prop] == null) ) {
+			return this.elem[ this.prop ];
+		}
+
+		var parsed,
+			r = jQuery.css( this.elem, this.prop );
+		// Empty strings, null, undefined and "auto" are converted to 0,
+		// complex values such as "rotate(1rad)" are returned as is,
+		// simple values such as "10px" are parsed to Float.
+		return isNaN( parsed = parseFloat( r ) ) ? !r || r === "auto" ? 0 : r : parsed;
+	},
+
+	// Start an animation from one number to another
+	custom: function( from, to, unit ) {
+		var self = this,
+			fx = jQuery.fx,
+			raf;
+
+		this.startTime = fxNow || createFxNow();
+		this.start = from;
+		this.end = to;
+		this.unit = unit || this.unit || ( jQuery.cssNumber[ this.prop ] ? "" : "px" );
+		this.now = this.start;
+		this.pos = this.state = 0;
+
+		function t( gotoEnd ) {
+			return self.step(gotoEnd);
+		}
+
+		t.elem = this.elem;
+
+		if ( t() && jQuery.timers.push(t) && !timerId ) {
+			// Use requestAnimationFrame instead of setInterval if available
+			if ( requestAnimationFrame ) {
+				timerId = true;
+				raf = function() {
+					// When timerId gets set to null at any point, this stops
+					if ( timerId ) {
+						requestAnimationFrame( raf );
+						fx.tick();
+					}
+				};
+				requestAnimationFrame( raf );
+			} else {
+				timerId = setInterval( fx.tick, fx.interval );
+			}
+		}
+	},
+
+	// Simple 'show' function
+	show: function() {
+		// Remember where we started, so that we can go back to it later
+		this.options.orig[this.prop] = jQuery.style( this.elem, this.prop );
+		this.options.show = true;
+
+		// Begin the animation
+		// Make sure that we start at a small width/height to avoid any
+		// flash of content
+		this.custom(this.prop === "width" || this.prop === "height" ? 1 : 0, this.cur());
+
+		// Start by showing the element
+		jQuery( this.elem ).show();
+	},
+
+	// Simple 'hide' function
+	hide: function() {
+		// Remember where we started, so that we can go back to it later
+		this.options.orig[this.prop] = jQuery.style( this.elem, this.prop );
+		this.options.hide = true;
+
+		// Begin the animation
+		this.custom(this.cur(), 0);
+	},
+
+	// Each step of an animation
+	step: function( gotoEnd ) {
+		var t = fxNow || createFxNow(),
+			done = true,
+			elem = this.elem,
+			options = this.options,
+			i, n;
+
+		if ( gotoEnd || t >= options.duration + this.startTime ) {
+			this.now = this.end;
+			this.pos = this.state = 1;
+			this.update();
+
+			options.animatedProperties[ this.prop ] = true;
+
+			for ( i in options.animatedProperties ) {
+				if ( options.animatedProperties[i] !== true ) {
+					done = false;
+				}
+			}
+
+			if ( done ) {
+				// Reset the overflow
+				if ( options.overflow != null && !jQuery.support.shrinkWrapBlocks ) {
+
+					jQuery.each( [ "", "X", "Y" ], function (index, value) {
+						elem.style[ "overflow" + value ] = options.overflow[index];
+					});
+				}
+
+				// Hide the element if the "hide" operation was done
+				if ( options.hide ) {
+					jQuery(elem).hide();
+				}
+
+				// Reset the properties, if the item has been hidden or shown
+				if ( options.hide || options.show ) {
+					for ( var p in options.animatedProperties ) {
+						jQuery.style( elem, p, options.orig[p] );
+					}
+				}
+
+				// Execute the complete function
+				options.complete.call( elem );
+			}
+
+			return false;
+
+		} else {
+			// classical easing cannot be used with an Infinity duration
+			if ( options.duration == Infinity ) {
+				this.now = t;
+			} else {
+				n = t - this.startTime;
+				this.state = n / options.duration;
+
+				// Perform the easing function, defaults to swing
+				this.pos = jQuery.easing[ options.animatedProperties[ this.prop ] ]( this.state, n, 0, 1, options.duration );
+				this.now = this.start + ((this.end - this.start) * this.pos);
+			}
+			// Perform the next step of the animation
+			this.update();
+		}
+
+		return true;
+	}
+};
+
+jQuery.extend( jQuery.fx, {
+	tick: function() {
+		for ( var timers = jQuery.timers, i = 0 ; i < timers.length ; ++i ) {
+			if ( !timers[i]() ) {
+				timers.splice(i--, 1);
+			}
+		}
+
+		if ( !timers.length ) {
+			jQuery.fx.stop();
+		}
+	},
+
+	interval: 13,
+
+	stop: function() {
+		clearInterval( timerId );
+		timerId = null;
+	},
+
+	speeds: {
+		slow: 600,
+		fast: 200,
+		// Default speed
+		_default: 400
+	},
+
+	step: {
+		opacity: function( fx ) {
+			jQuery.style( fx.elem, "opacity", fx.now );
+		},
+
+		_default: function( fx ) {
+			if ( fx.elem.style && fx.elem.style[ fx.prop ] != null ) {
+				fx.elem.style[ fx.prop ] = (fx.prop === "width" || fx.prop === "height" ? Math.max(0, fx.now) : fx.now) + fx.unit;
+			} else {
+				fx.elem[ fx.prop ] = fx.now;
+			}
+		}
+	}
+});
+
+if ( jQuery.expr && jQuery.expr.filters ) {
+	jQuery.expr.filters.animated = function( elem ) {
+		return jQuery.grep(jQuery.timers, function( fn ) {
+			return elem === fn.elem;
+		}).length;
+	};
+}
+
+// Try to restore the default display value of an element
+function defaultDisplay( nodeName ) {
+
+	if ( !elemdisplay[ nodeName ] ) {
+
+		var body = document.body,
+			elem = jQuery( "<" + nodeName + ">" ).appendTo( body ),
+			display = elem.css( "display" );
+
+		elem.remove();
+
+		// If the simple way fails,
+		// get element's real default display by attaching it to a temp iframe
+		if ( display === "none" || display === "" ) {
+			// No iframe to use yet, so create it
+			if ( !iframe ) {
+				iframe = document.createElement( "iframe" );
+				iframe.frameBorder = iframe.width = iframe.height = 0;
+			}
+
+			body.appendChild( iframe );
+
+			// Create a cacheable copy of the iframe document on first call.
+			// IE and Opera will allow us to reuse the iframeDoc without re-writing the fake HTML
+			// document to it; WebKit & Firefox won't allow reusing the iframe document.
+			if ( !iframeDoc || !iframe.createElement ) {
+				iframeDoc = ( iframe.contentWindow || iframe.contentDocument ).document;
+				iframeDoc.write( ( document.compatMode === "CSS1Compat" ? "<!doctype html>" : "" ) + "<html><body>" );
+				iframeDoc.close();
+			}
+
+			elem = iframeDoc.createElement( nodeName );
+
+			iframeDoc.body.appendChild( elem );
+
+			display = jQuery.css( elem, "display" );
+
+			body.removeChild( iframe );
+		}
+
+		// Store the correct default display
+		elemdisplay[ nodeName ] = display;
+	}
+
+	return elemdisplay[ nodeName ];
+}
+
+
+
+
+var rtable = /^t(?:able|d|h)$/i,
+	rroot = /^(?:body|html)$/i;
+
+if ( "getBoundingClientRect" in document.documentElement ) {
+	jQuery.fn.offset = function( options ) {
+		var elem = this[0], box;
+
+		if ( options ) {
+			return this.each(function( i ) {
+				jQuery.offset.setOffset( this, options, i );
+			});
+		}
+
+		if ( !elem || !elem.ownerDocument ) {
+			return null;
+		}
+
+		if ( elem === elem.ownerDocument.body ) {
+			return jQuery.offset.bodyOffset( elem );
+		}
+
+		try {
+			box = elem.getBoundingClientRect();
+		} catch(e) {}
+
+		var doc = elem.ownerDocument,
+			docElem = doc.documentElement;
+
+		// Make sure we're not dealing with a disconnected DOM node
+		if ( !box || !jQuery.contains( docElem, elem ) ) {
+			return box ? { top: box.top, left: box.left } : { top: 0, left: 0 };
+		}
+
+		var body = doc.body,
+			win = getWindow(doc),
+			clientTop  = docElem.clientTop  || body.clientTop  || 0,
+			clientLeft = docElem.clientLeft || body.clientLeft || 0,
+			scrollTop  = win.pageYOffset || jQuery.support.boxModel && docElem.scrollTop  || body.scrollTop,
+			scrollLeft = win.pageXOffset || jQuery.support.boxModel && docElem.scrollLeft || body.scrollLeft,
+			top  = box.top  + scrollTop  - clientTop,
+			left = box.left + scrollLeft - clientLeft;
+
+		return { top: top, left: left };
+	};
+
+} else {
+	jQuery.fn.offset = function( options ) {
+		var elem = this[0];
+
+		if ( options ) {
+			return this.each(function( i ) {
+				jQuery.offset.setOffset( this, options, i );
+			});
+		}
+
+		if ( !elem || !elem.ownerDocument ) {
+			return null;
+		}
+
+		if ( elem === elem.ownerDocument.body ) {
+			return jQuery.offset.bodyOffset( elem );
+		}
+
+		jQuery.offset.initialize();
+
+		var computedStyle,
+			offsetParent = elem.offsetParent,
+			prevOffsetParent = elem,
+			doc = elem.ownerDocument,
+			docElem = doc.documentElement,
+			body = doc.body,
+			defaultView = doc.defaultView,
+			prevComputedStyle = defaultView ? defaultView.getComputedStyle( elem, null ) : elem.currentStyle,
+			top = elem.offsetTop,
+			left = elem.offsetLeft;
+
+		while ( (elem = elem.parentNode) && elem !== body && elem !== docElem ) {
+			if ( jQuery.offset.supportsFixedPosition && prevComputedStyle.position === "fixed" ) {
+				break;
+			}
+
+			computedStyle = defaultView ? defaultView.getComputedStyle(elem, null) : elem.currentStyle;
+			top  -= elem.scrollTop;
+			left -= elem.scrollLeft;
+
+			if ( elem === offsetParent ) {
+				top  += elem.offsetTop;
+				left += elem.offsetLeft;
+
+				if ( jQuery.offset.doesNotAddBorder && !(jQuery.offset.doesAddBorderForTableAndCells && rtable.test(elem.nodeName)) ) {
+					top  += parseFloat( computedStyle.borderTopWidth  ) || 0;
+					left += parseFloat( computedStyle.borderLeftWidth ) || 0;
+				}
+
+				prevOffsetParent = offsetParent;
+				offsetParent = elem.offsetParent;
+			}
+
+			if ( jQuery.offset.subtractsBorderForOverflowNotVisible && computedStyle.overflow !== "visible" ) {
+				top  += parseFloat( computedStyle.borderTopWidth  ) || 0;
+				left += parseFloat( computedStyle.borderLeftWidth ) || 0;
+			}
+
+			prevComputedStyle = computedStyle;
+		}
+
+		if ( prevComputedStyle.position === "relative" || prevComputedStyle.position === "static" ) {
+			top  += body.offsetTop;
+			left += body.offsetLeft;
+		}
+
+		if ( jQuery.offset.supportsFixedPosition && prevComputedStyle.position === "fixed" ) {
+			top  += Math.max( docElem.scrollTop, body.scrollTop );
+			left += Math.max( docElem.scrollLeft, body.scrollLeft );
+		}
+
+		return { top: top, left: left };
+	};
+}
+
+jQuery.offset = {
+	initialize: function() {
+		var body = document.body, container = document.createElement("div"), innerDiv, checkDiv, table, td, bodyMarginTop = parseFloat( jQuery.css(body, "marginTop") ) || 0,
+			html = "<div style='position:absolute;top:0;left:0;margin:0;border:5px solid #000;padding:0;width:1px;height:1px;'><div></div></div><table style='position:absolute;top:0;left:0;margin:0;border:5px solid #000;padding:0;width:1px;height:1px;' cellpadding='0' cellspacing='0'><tr><td></td></tr></table>";
+
+		jQuery.extend( container.style, { position: "absolute", top: 0, left: 0, margin: 0, border: 0, width: "1px", height: "1px", visibility: "hidden" } );
+
+		container.innerHTML = html;
+		body.insertBefore( container, body.firstChild );
+		innerDiv = container.firstChild;
+		checkDiv = innerDiv.firstChild;
+		td = innerDiv.nextSibling.firstChild.firstChild;
+
+		this.doesNotAddBorder = (checkDiv.offsetTop !== 5);
+		this.doesAddBorderForTableAndCells = (td.offsetTop === 5);
+
+		checkDiv.style.position = "fixed";
+		checkDiv.style.top = "20px";
+
+		// safari subtracts parent border width here which is 5px
+		this.supportsFixedPosition = (checkDiv.offsetTop === 20 || checkDiv.offsetTop === 15);
+		checkDiv.style.position = checkDiv.style.top = "";
+
+		innerDiv.style.overflow = "hidden";
+		innerDiv.style.position = "relative";
+
+		this.subtractsBorderForOverflowNotVisible = (checkDiv.offsetTop === -5);
+
+		this.doesNotIncludeMarginInBodyOffset = (body.offsetTop !== bodyMarginTop);
+
+		body.removeChild( container );
+		jQuery.offset.initialize = jQuery.noop;
+	},
+
+	bodyOffset: function( body ) {
+		var top = body.offsetTop,
+			left = body.offsetLeft;
+
+		jQuery.offset.initialize();
+
+		if ( jQuery.offset.doesNotIncludeMarginInBodyOffset ) {
+			top  += parseFloat( jQuery.css(body, "marginTop") ) || 0;
+			left += parseFloat( jQuery.css(body, "marginLeft") ) || 0;
+		}
+
+		return { top: top, left: left };
+	},
+
+	setOffset: function( elem, options, i ) {
+		var position = jQuery.css( elem, "position" );
+
+		// set position first, in-case top/left are set even on static elem
+		if ( position === "static" ) {
+			elem.style.position = "relative";
+		}
+
+		var curElem = jQuery( elem ),
+			curOffset = curElem.offset(),
+			curCSSTop = jQuery.css( elem, "top" ),
+			curCSSLeft = jQuery.css( elem, "left" ),
+			calculatePosition = (position === "absolute" || position === "fixed") && jQuery.inArray("auto", [curCSSTop, curCSSLeft]) > -1,
+			props = {}, curPosition = {}, curTop, curLeft;
+
+		// need to be able to calculate position if either top or left is auto and position is either absolute or fixed
+		if ( calculatePosition ) {
+			curPosition = curElem.position();
+			curTop = curPosition.top;
+			curLeft = curPosition.left;
+		} else {
+			curTop = parseFloat( curCSSTop ) || 0;
+			curLeft = parseFloat( curCSSLeft ) || 0;
+		}
+
+		if ( jQuery.isFunction( options ) ) {
+			options = options.call( elem, i, curOffset );
+		}
+
+		if (options.top != null) {
+			props.top = (options.top - curOffset.top) + curTop;
+		}
+		if (options.left != null) {
+			props.left = (options.left - curOffset.left) + curLeft;
+		}
+
+		if ( "using" in options ) {
+			options.using.call( elem, props );
+		} else {
+			curElem.css( props );
+		}
+	}
+};
+
+
+jQuery.fn.extend({
+	position: function() {
+		if ( !this[0] ) {
+			return null;
+		}
+
+		var elem = this[0],
+
+		// Get *real* offsetParent
+		offsetParent = this.offsetParent(),
+
+		// Get correct offsets
+		offset       = this.offset(),
+		parentOffset = rroot.test(offsetParent[0].nodeName) ? { top: 0, left: 0 } : offsetParent.offset();
+
+		// Subtract element margins
+		// note: when an element has margin: auto the offsetLeft and marginLeft
+		// are the same in Safari causing offset.left to incorrectly be 0
+		offset.top  -= parseFloat( jQuery.css(elem, "marginTop") ) || 0;
+		offset.left -= parseFloat( jQuery.css(elem, "marginLeft") ) || 0;
+
+		// Add offsetParent borders
+		parentOffset.top  += parseFloat( jQuery.css(offsetParent[0], "borderTopWidth") ) || 0;
+		parentOffset.left += parseFloat( jQuery.css(offsetParent[0], "borderLeftWidth") ) || 0;
+
+		// Subtract the two offsets
+		return {
+			top:  offset.top  - parentOffset.top,
+			left: offset.left - parentOffset.left
+		};
+	},
+
+	offsetParent: function() {
+		return this.map(function() {
+			var offsetParent = this.offsetParent || document.body;
+			while ( offsetParent && (!rroot.test(offsetParent.nodeName) && jQuery.css(offsetParent, "position") === "static") ) {
+				offsetParent = offsetParent.offsetParent;
+			}
+			return offsetParent;
+		});
+	}
+});
+
+
+// Create scrollLeft and scrollTop methods
+jQuery.each( ["Left", "Top"], function( i, name ) {
+	var method = "scroll" + name;
+
+	jQuery.fn[ method ] = function( val ) {
+		var elem, win;
+
+		if ( val === undefined ) {
+			elem = this[ 0 ];
+
+			if ( !elem ) {
+				return null;
+			}
+
+			win = getWindow( elem );
+
+			// Return the scroll offset
+			return win ? ("pageXOffset" in win) ? win[ i ? "pageYOffset" : "pageXOffset" ] :
+				jQuery.support.boxModel && win.document.documentElement[ method ] ||
+					win.document.body[ method ] :
+				elem[ method ];
+		}
+
+		// Set the scroll offset
+		return this.each(function() {
+			win = getWindow( this );
+
+			if ( win ) {
+				win.scrollTo(
+					!i ? val : jQuery( win ).scrollLeft(),
+					 i ? val : jQuery( win ).scrollTop()
+				);
+
+			} else {
+				this[ method ] = val;
+			}
+		});
+	};
+});
+
+function getWindow( elem ) {
+	return jQuery.isWindow( elem ) ?
+		elem :
+		elem.nodeType === 9 ?
+			elem.defaultView || elem.parentWindow :
+			false;
+}
+
+
+
+
+// Create width, height, innerHeight, innerWidth, outerHeight and outerWidth methods
+jQuery.each([ "Height", "Width" ], function( i, name ) {
+
+	var type = name.toLowerCase();
+
+	// innerHeight and innerWidth
+	jQuery.fn[ "inner" + name ] = function() {
+		var elem = this[0];
+		return elem && elem.style ?
+			parseFloat( jQuery.css( elem, type, "padding" ) ) :
+			null;
+	};
+
+	// outerHeight and outerWidth
+	jQuery.fn[ "outer" + name ] = function( margin ) {
+		var elem = this[0];
+		return elem && elem.style ?
+			parseFloat( jQuery.css( elem, type, margin ? "margin" : "border" ) ) :
+			null;
+	};
+
+	jQuery.fn[ type ] = function( size ) {
+		// Get window width or height
+		var elem = this[0];
+		if ( !elem ) {
+			return size == null ? null : this;
+		}
+
+		if ( jQuery.isFunction( size ) ) {
+			return this.each(function( i ) {
+				var self = jQuery( this );
+				self[ type ]( size.call( this, i, self[ type ]() ) );
+			});
+		}
+
+		if ( jQuery.isWindow( elem ) ) {
+			// Everyone else use document.documentElement or document.body depending on Quirks vs Standards mode
+			// 3rd condition allows Nokia support, as it supports the docElem prop but not CSS1Compat
+			var docElemProp = elem.document.documentElement[ "client" + name ];
+			return elem.document.compatMode === "CSS1Compat" && docElemProp ||
+				elem.document.body[ "client" + name ] || docElemProp;
+
+		// Get document width or height
+		} else if ( elem.nodeType === 9 ) {
+			// Either scroll[Width/Height] or offset[Width/Height], whichever is greater
+			return Math.max(
+				elem.documentElement["client" + name],
+				elem.body["scroll" + name], elem.documentElement["scroll" + name],
+				elem.body["offset" + name], elem.documentElement["offset" + name]
+			);
+
+		// Get or set width or height on the element
+		} else if ( size === undefined ) {
+			var orig = jQuery.css( elem, type ),
+				ret = parseFloat( orig );
+
+			return jQuery.isNaN( ret ) ? orig : ret;
+
+		// Set the width or height on the element (default to pixels if value is unitless)
+		} else {
+			return this.css( type, typeof size === "string" ? size : size + "px" );
+		}
+	};
+
+});
+
+
+// Expose jQuery to the global object
+window.jQuery = window.$ = jQuery;
+})(window);

From 6d4d68b7b48f5a5469d08bd65c208a446583c667 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 00:03:55 -0400
Subject: [PATCH 4284/8313] cleanup done things etc

---
 doc/design/assistant/webapp.mdwn | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 439e013c19..e999e009bf 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -32,14 +32,9 @@ The webapp is a web server that displays a shiny interface.
 
 ## implementation
 
-* Include jquery into the webapp, preferably minimised at build time.
-  Currently the webapp needs an internet connection to load jquery, which
-  is not ideal.
-* use `addStaticContent` to make /favicon.ico work. Return `Right (route, query)`
-  and I think the route can be `favicon_ico`. (Tried this; couldn't seem
-  to make it work.)
 * perhaps define a custom `errorHandler`, which could avoid the potential
-  of leaking auth tokens on error pages
+  of leaking auth tokens on error pages. Or make the test suite test for
+  leakage.
 * possibly lose the ugly auth= token past the first page,
   and use a client-side session. It could be encrypted using the token
   as the `encryptKey`. Note: Would need to set the session duration

From f1c7c475f6eab1a74dcf5b11aaeac9dd6ad986f8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 00:17:04 -0400
Subject: [PATCH 4285/8313] automatically close the webapp window after 3
 errors

It's unlikely an error would occur unless the server is stopped. But
retrying a few times seems like a good idea anyway.
---
 templates/longpolling.julius | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 26356f5e93..351f2f8c60 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -7,6 +7,8 @@
 
 (function( $ ) {
 
+numerrs=0;
+
 $.LongPoll = (function() {
 	return {
 		send : function() {           
@@ -16,6 +18,16 @@ $.LongPoll = (function() {
 				'success': function(data, status, jqxhr) {
 					$('##{poll}').replaceWith(data);
 					setTimeout($.LongPoll.send, #{delay});
+					numerrs=0;
+				},
+				'error': function(jqxhr, msg, e) {
+					numerrs=numerrs+1;
+					if (numerrs > 3) {
+						window.close();
+					}
+					else {
+						setTimeout($.LongPoll.send, #{delay});
+					}
 				},
 			});
 		}

From 5db2605f3e54e90b745eb1215bb139672805272b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 00:23:17 -0400
Subject: [PATCH 4286/8313] update

---
 doc/design/assistant/webapp.mdwn | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index e999e009bf..018d70886b 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -39,6 +39,3 @@ The webapp is a web server that displays a shiny interface.
   and use a client-side session. It could be encrypted using the token
   as the `encryptKey`. Note: Would need to set the session duration
   to infinite (how?)
-* When long polling fails, retry a time or two, and then give up, and
-  either display an error message, or, possibly, close the browser window.
-  (Currently the display just stops updating.)

From 1f47c1f6d8f9b799fc85cfb12bf73030fe455167 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 00:35:09 -0400
Subject: [PATCH 4287/8313] update

---
 debian/copyright | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/debian/copyright b/debian/copyright
index 72b6333671..1b00d07e77 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -20,7 +20,7 @@ Copyright: 1980, 1989, 1993, 1994 The Regents of the University of California
            2001 David Rufino <daverufino@btinternet.com>
            2012 Joey Hess <joey@kitenet.net>
 License: BSD-3-clause
- The full test of the 3 clause BSD license is distributed inside
+ The full text of the 3 clause BSD license is distributed inside
  Utility/libmounts.c in this package's source, or in
  /usr/share/common-licenses/BSD on Debian systems.
 
@@ -35,7 +35,8 @@ Copyright: © 2005-2011 by John Resig, Branden Aaron & Jörn Zaefferer
            © 2011 The Dojo Foundation
 License: MIT or GPL-2
  The full text of version 2 of the GPL is distributed in
- /usr/share/common-licenses/GPL-2 on Debian systems.
+ /usr/share/common-licenses/GPL-2 on Debian systems. The text of the MIT
+ license follows:
  .
  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the

From 2ba02d260b4a2c4df523079e76fcba8342befa7b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 00:40:18 -0400
Subject: [PATCH 4288/8313] add README

---
 templates/README | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 templates/README

diff --git a/templates/README b/templates/README
new file mode 100644
index 0000000000..eb0ca037b6
--- /dev/null
+++ b/templates/README
@@ -0,0 +1,7 @@
+These are template files for the git-annex webapp. They use the
+Shakespearean template library, which is documented here:
+http://www.yesodweb.com/book/shakespearean-templates
+
+Note that for most of the templates, it will use files ending in
+.hamlet, .julius, and .cassius if they exist. So if you need to add CSS,
+or javascript, you can create the missing template files.

From d92f5ff44c6c20af50c2363e754a1ecae3df8d6a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 01:36:01 -0400
Subject: [PATCH 4289/8313] add bootstrap

---
 debian/copyright                      |  398 +++
 static/bootstrap.css                  | 3990 +++++++++++++++++++++++++
 static/glyphicons-halflings-white.png |  Bin 0 -> 4352 bytes
 static/glyphicons-halflings.png       |  Bin 0 -> 4352 bytes
 4 files changed, 4388 insertions(+)
 create mode 100644 static/bootstrap.css
 create mode 100644 static/glyphicons-halflings-white.png
 create mode 100644 static/glyphicons-halflings.png

diff --git a/debian/copyright b/debian/copyright
index 1b00d07e77..7f906a64aa 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -56,3 +56,401 @@ License: MIT or GPL-2
  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: static/bootstrap.css
+Copyright: 2011-2012 Twitter, Inc.
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+        http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ The complete text of the Apache License is distributed in
+ /usr/share/common-licenses/Apache-2.0 on Debian systems.
+
+Files: static/glyphicons*
+Copyright: 2010-2012 Jan Kovarik <glyphicons@gmail.com>
+License: CC-BY-3.0
+ Creative Commons Attribution 3.0 License
+ .
+ THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS
+ OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR
+ "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER
+ APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
+ AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS
+ PROHIBITED.
+ .
+ BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU
+ ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE.
+ TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A
+ CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE
+ IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND
+ CONDITIONS.
+ .
+ 1.   Definitions
+ .
+   a) "Adaptation" means a work based upon
+      the Work, or upon the Work and other pre-existing works,
+      such as a translation, adaptation, derivative work,
+      arrangement of music or other alterations of a literary
+      or artistic work, or phonogram or performance and
+      includes cinematographic adaptations or any other form in
+      which the Work may be recast, transformed, or adapted
+      including in any form recognizably derived from the
+      original, except that a work that constitutes a
+      Collection will not be considered an Adaptation for the
+      purpose of this License. For the avoidance of doubt,
+      where the Work is a musical work, performance or
+      phonogram, the synchronization of the Work in
+      timed-relation with a moving image ("synching") will be
+      considered an Adaptation for the purpose of this
+      License.
+ .
+   b) "Collection"</strong> means a collection of
+      literary or artistic works, such as encyclopedias and
+      anthologies, or performances, phonograms or broadcasts,
+      or other works or subject matter other than works listed
+      in Section 1(f) below, which, by reason of the selection
+      and arrangement of their contents, constitute
+      intellectual creations, in which the Work is included in
+      its entirety in unmodified form along with one or more
+      other contributions, each constituting separate and
+      independent works in themselves, which together are
+      assembled into a collective whole. A work that
+      constitutes a Collection will not be considered an
+      Adaptation (as defined above) for the purposes of this
+      License.
+ .
+   c) "Distribute" means to make available
+      to the public the original and copies of the Work or
+      Adaptation, as appropriate, through sale or other
+      transfer of ownership.
+ .
+   d) "Licensor" means the individual,
+      individuals, entity or entities that offer(s) the Work
+      under the terms of this License.
+ .
+   e) "Original Author" means, in the case
+      of a literary or artistic work, the individual,
+      individuals, entity or entities who created the Work or
+      if no individual or entity can be identified, the
+      publisher; and in addition (i) in the case of a
+      performance the actors, singers, musicians, dancers, and
+      other persons who act, sing, deliver, declaim, play in,
+      interpret or otherwise perform literary or artistic works
+      or expressions of folklore; (ii) in the case of a
+      phonogram the producer being the person or legal entity
+      who first fixes the sounds of a performance or other
+      sounds; and, (iii) in the case of broadcasts, the
+      organization that transmits the broadcast.
+ .
+   f) "Work" means the literary and/or
+      artistic work offered under the terms of this License
+      including without limitation any production in the
+      literary, scientific and artistic domain, whatever may be
+      the mode or form of its expression including digital
+      form, such as a book, pamphlet and other writing; a
+      lecture, address, sermon or other work of the same
+      nature; a dramatic or dramatico-musical work; a
+      choreographic work or entertainment in dumb show; a
+      musical composition with or without words; a
+      cinematographic work to which are assimilated works
+      expressed by a process analogous to cinematography; a
+      work of drawing, painting, architecture, sculpture,
+      engraving or lithography; a photographic work to which
+      are assimilated works expressed by a process analogous to
+      photography; a work of applied art; an illustration, map,
+      plan, sketch or three-dimensional work relative to
+      geography, topography, architecture or science; a
+      performance; a broadcast; a phonogram; a compilation of
+      data to the extent it is protected as a copyrightable
+      work; or a work performed by a variety or circus
+      performer to the extent it is not otherwise considered a
+      literary or artistic work.
+ .
+   g) "You"</strong> means an individual or entity
+      exercising rights under this License who has not
+      previously violated the terms of this License with
+      respect to the Work, or who has received express
+      permission from the Licensor to exercise rights under
+      this License despite a previous violation.
+ .
+   h) "Publicly Perform" means to perform
+      public recitations of the Work and to communicate to the
+      public those public recitations, by any means or process,
+      including by wire or wireless means or public digital
+      performances; to make available to the public Works in
+      such a way that members of the public may access these
+      Works from a place and at a place individually chosen by
+      them; to perform the Work to the public by any means or
+      process and the communication to the public of the
+      performances of the Work, including by public digital
+      performance; to broadcast and rebroadcast the Work by any
+      means including signs, sounds or images.
+ .
+   i) "Reproduce" means to make copies of
+      the Work by any means including without limitation by
+      sound or visual recordings and the right of fixation and
+      reproducing fixations of the Work, including storage of a
+      protected performance or phonogram in digital form or
+      other electronic medium.
+ .
+ 2.   Fair Dealing Rights. Nothing in this
+      License is intended to reduce, limit, or restrict any uses
+      free from copyright or rights arising from limitations or
+      exceptions that are provided for in connection with the
+      copyright protection under copyright law or other
+      applicable laws.
+ .
+ 3.   License Grant. Subject to the terms
+      and conditions of this License, Licensor hereby grants You
+      a worldwide, royalty-free, non-exclusive, perpetual (for
+      the duration of the applicable copyright) license to
+      exercise the rights in the Work as stated below:</p>
+ .
+   a) to Reproduce the Work, to incorporate the Work into
+      one or more Collections, and to Reproduce the Work as
+      incorporated in the Collections;
+ .
+   b) to create and Reproduce Adaptations provided that any
+      such Adaptation, including any translation in any medium,
+      takes reasonable steps to clearly label, demarcate or
+      otherwise identify that changes were made to the original
+      Work. For example, a translation could be marked "The
+      original work was translated from English to Spanish," or
+      a modification could indicate "The original work has been
+      modified.";
+ .
+   c) to Distribute and Publicly Perform the Work including
+      as incorporated in Collections; and,
+ .
+   d) to Distribute and Publicly Perform Adaptations.
+ .
+   e) For the avoidance of doubt:
+ .
+      i) Non-waivable Compulsory License
+         Schemes. In those jurisdictions in which the
+         right to collect royalties through any statutory or
+         compulsory licensing scheme cannot be waived, the
+         Licensor reserves the exclusive right to collect such
+         royalties for any exercise by You of the rights
+         granted under this License;
+ .
+     ii) Waivable Compulsory License
+         Schemes. In those jurisdictions in which the
+         right to collect royalties through any statutory or
+         compulsory licensing scheme can be waived, the
+         Licensor waives the exclusive right to collect such
+         royalties for any exercise by You of the rights
+         granted under this License; and,
+ .
+    iii) Voluntary License Schemes. The
+         Licensor waives the right to collect royalties,
+         whether individually or, in the event that the
+         Licensor is a member of a collecting society that
+         administers voluntary licensing schemes, via that
+         society, from any exercise by You of the rights
+         granted under this License.
+ .
+      The above rights may be exercised in all media and
+      formats whether now known or hereafter devised. The above
+      rights include the right to make such modifications as are
+      technically necessary to exercise the rights in other media
+      and formats. Subject to Section 8(f), all rights not
+      expressly granted by Licensor are hereby reserved.
+ .
+ 4.   Restrictions. The license granted in
+      Section 3 above is expressly made subject to and limited by
+      the following restrictions:
+ .
+   a) You may Distribute or Publicly Perform the Work only
+      under the terms of this License. You must include a copy
+      of, or the Uniform Resource Identifier (URI) for, this
+      License with every copy of the Work You Distribute or
+      Publicly Perform. You may not offer or impose any terms
+      on the Work that restrict the terms of this License or
+      the ability of the recipient of the Work to exercise the
+      rights granted to that recipient under the terms of the
+      License. You may not sublicense the Work. You must keep
+      intact all notices that refer to this License and to the
+      disclaimer of warranties with every copy of the Work You
+      Distribute or Publicly Perform. When You Distribute or
+      Publicly Perform the Work, You may not impose any
+      effective technological measures on the Work that
+      restrict the ability of a recipient of the Work from You
+      to exercise the rights granted to that recipient under
+      the terms of the License. This Section 4(a) applies to
+      the Work as incorporated in a Collection, but this does
+      not require the Collection apart from the Work itself to
+      be made subject to the terms of this License. If You
+      create a Collection, upon notice from any Licensor You
+      must, to the extent practicable, remove from the
+      Collection any credit as required by Section 4(b), as
+      requested. If You create an Adaptation, upon notice from
+      any Licensor You must, to the extent practicable, remove
+      from the Adaptation any credit as required by Section
+      4(b), as requested.
+ .
+   b) If You Distribute, or Publicly Perform the Work or
+      any Adaptations or Collections, You must, unless a
+      request has been made pursuant to Section 4(a), keep
+      intact all copyright notices for the Work and provide,
+      reasonable to the medium or means You are utilizing: (i)
+      the name of the Original Author (or pseudonym, if
+      applicable) if supplied, and/or if the Original Author
+      and/or Licensor designate another party or parties (e.g.,
+      a sponsor institute, publishing entity, journal) for
+      attribution ("Attribution Parties") in Licensor's
+      copyright notice, terms of service or by other reasonable
+      means, the name of such party or parties; (ii) the title
+      of the Work if supplied; (iii) to the extent reasonably
+      practicable, the URI, if any, that Licensor specifies to
+      be associated with the Work, unless such URI does not
+      refer to the copyright notice or licensing information
+      for the Work; and (iv) , consistent with Section 3(b), in
+      the case of an Adaptation, a credit identifying the use
+      of the Work in the Adaptation (e.g., "French translation
+      of the Work by Original Author," or "Screenplay based on
+      original Work by Original Author"). The credit required
+      by this Section 4 (b) may be implemented in any
+      reasonable manner; provided, however, that in the case of
+      a Adaptation or Collection, at a minimum such credit will
+      appear, if a credit for all contributing authors of the
+      Adaptation or Collection appears, then as part of these
+      credits and in a manner at least as prominent as the
+      credits for the other contributing authors. For the
+      avoidance of doubt, You may only use the credit required
+      by this Section for the purpose of attribution in the
+      manner set out above and, by exercising Your rights under
+      this License, You may not implicitly or explicitly assert
+      or imply any connection with, sponsorship or endorsement
+      by the Original Author, Licensor and/or Attribution
+      Parties, as appropriate, of You or Your use of the Work,
+      without the separate, express prior written permission of
+      the Original Author, Licensor and/or Attribution
+      Parties.
+ .
+   c) Except as otherwise agreed in writing by the Licensor
+      or as may be otherwise permitted by applicable law, if
+      You Reproduce, Distribute or Publicly Perform the Work
+      either by itself or as part of any Adaptations or
+      Collections, You must not distort, mutilate, modify or
+      take other derogatory action in relation to the Work
+      which would be prejudicial to the Original Author's honor
+      or reputation. Licensor agrees that in those
+      jurisdictions (e.g. Japan), in which any exercise of the
+      right granted in Section 3(b) of this License (the right
+      to make Adaptations) would be deemed to be a distortion,
+      mutilation, modification or other derogatory action
+      prejudicial to the Original Author's honor and
+      reputation, the Licensor will waive or not assert, as
+      appropriate, this Section, to the fullest extent
+      permitted by the applicable national law, to enable You
+      to reasonably exercise Your right under Section 3(b) of
+      this License (right to make Adaptations) but not
+      otherwise.
+ .
+ 5.   Representations, Warranties and
+      Disclaimer
+ .
+      UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN
+      WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO
+      REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE
+      WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING,
+      WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,
+      FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE
+      ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE
+      PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE.
+      SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED
+      WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+ .
+ 6.   Limitation on Liability. EXCEPT TO
+      THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL
+      LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY
+      SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY
+      DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK,
+      EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF
+      SUCH DAMAGES.
+ .
+ 7.   Termination
+ .
+   a) This License and the rights granted hereunder will
+      terminate automatically upon any breach by You of the
+      terms of this License. Individuals or entities who have
+      received Adaptations or Collections from You under this
+      License, however, will not have their licenses terminated
+      provided such individuals or entities remain in full
+      compliance with those licenses. Sections 1, 2, 5, 6, 7,
+      and 8 will survive any termination of this License.</li>
+ .
+   b) Subject to the above terms and conditions, the
+      license granted here is perpetual (for the duration of
+      the applicable copyright in the Work). Notwithstanding
+      the above, Licensor reserves the right to release the
+      Work under different license terms or to stop
+      distributing the Work at any time; provided, however that
+      any such election will not serve to withdraw this License
+      (or any other license that has been, or is required to
+      be, granted under the terms of this License), and this
+      License will continue in full force and effect unless
+      terminated as stated above.
+ .
+ 8.   Miscellaneous
+ .
+   a) Each time You Distribute or Publicly Perform the Work
+      or a Collection, the Licensor offers to the recipient a
+      license to the Work on the same terms and conditions as
+      the license granted to You under this License.
+ .
+   b) Each time You Distribute or Publicly Perform an
+      Adaptation, Licensor offers to the recipient a license to
+      the original Work on the same terms and conditions as the
+      license granted to You under this License.
+ .
+   c) If any provision of this License is invalid or
+      unenforceable under applicable law, it shall not affect
+      the validity or enforceability of the remainder of the
+      terms of this License, and without further action by the
+      parties to this agreement, such provision shall be
+      reformed to the minimum extent necessary to make such
+      provision valid and enforceable.
+ .
+   d) No term or provision of this License shall be deemed
+      waived and no breach consented to unless such waiver or
+      consent shall be in writing and signed by the party to be
+      charged with such waiver or consent.
+ .
+   e) This License constitutes the entire agreement between
+      the parties with respect to the Work licensed here. There
+      are no understandings, agreements or representations with
+      respect to the Work not specified here. Licensor shall
+      not be bound by any additional provisions that may appear
+      in any communication from You. This License may not be
+      modified without the mutual written agreement of the
+      Licensor and You.
+ .
+   f) The rights granted under, and the subject matter
+      referenced, in this License were drafted utilizing the
+      terminology of the Berne Convention for the Protection of
+      Literary and Artistic Works (as amended on September 28,
+      1979), the Rome Convention of 1961, the WIPO Copyright
+      Treaty of 1996, the WIPO Performances and Phonograms
+      Treaty of 1996 and the Universal Copyright Convention (as
+      revised on July 24, 1971). These rights and subject
+      matter take effect in the relevant jurisdiction in which
+      the License terms are sought to be enforced according to
+      the corresponding provisions of the implementation of
+      those treaty provisions in the applicable national law.
+      If the standard suite of rights granted under applicable
+      copyright law includes additional rights not granted
+      under this License, such additional rights are deemed to
+      be included in the License; this License is not intended
+      to restrict the license of any rights under applicable
+      law.
diff --git a/static/bootstrap.css b/static/bootstrap.css
new file mode 100644
index 0000000000..495188af7f
--- /dev/null
+++ b/static/bootstrap.css
@@ -0,0 +1,3990 @@
+/*!
+ * Bootstrap v2.0.2
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */
+article,
+aside,
+details,
+figcaption,
+figure,
+footer,
+header,
+hgroup,
+nav,
+section {
+  display: block;
+}
+audio,
+canvas,
+video {
+  display: inline-block;
+  *display: inline;
+  *zoom: 1;
+}
+audio:not([controls]) {
+  display: none;
+}
+html {
+  font-size: 100%;
+  -webkit-text-size-adjust: 100%;
+  -ms-text-size-adjust: 100%;
+}
+a:focus {
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
+}
+a:hover,
+a:active {
+  outline: 0;
+}
+sub,
+sup {
+  position: relative;
+  font-size: 75%;
+  line-height: 0;
+  vertical-align: baseline;
+}
+sup {
+  top: -0.5em;
+}
+sub {
+  bottom: -0.25em;
+}
+img {
+  height: auto;
+  border: 0;
+  -ms-interpolation-mode: bicubic;
+  vertical-align: middle;
+}
+button,
+input,
+select,
+textarea {
+  margin: 0;
+  font-size: 100%;
+  vertical-align: middle;
+}
+button,
+input {
+  *overflow: visible;
+  line-height: normal;
+}
+button::-moz-focus-inner,
+input::-moz-focus-inner {
+  padding: 0;
+  border: 0;
+}
+button,
+input[type="button"],
+input[type="reset"],
+input[type="submit"] {
+  cursor: pointer;
+  -webkit-appearance: button;
+}
+input[type="search"] {
+  -webkit-appearance: textfield;
+  -webkit-box-sizing: content-box;
+  -moz-box-sizing: content-box;
+  box-sizing: content-box;
+}
+input[type="search"]::-webkit-search-decoration,
+input[type="search"]::-webkit-search-cancel-button {
+  -webkit-appearance: none;
+}
+textarea {
+  overflow: auto;
+  vertical-align: top;
+}
+.clearfix {
+  *zoom: 1;
+}
+.clearfix:before,
+.clearfix:after {
+  display: table;
+  content: "";
+}
+.clearfix:after {
+  clear: both;
+}
+.hide-text {
+  overflow: hidden;
+  text-indent: 100%;
+  white-space: nowrap;
+}
+.input-block-level {
+  display: block;
+  width: 100%;
+  min-height: 28px;
+  /* Make inputs at least the height of their button counterpart */
+
+  /* Makes inputs behave like true block-level elements */
+
+  -webkit-box-sizing: border-box;
+  -moz-box-sizing: border-box;
+  -ms-box-sizing: border-box;
+  box-sizing: border-box;
+}
+body {
+  margin: 0;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  font-size: 13px;
+  line-height: 18px;
+  color: #333333;
+  background-color: #ffffff;
+}
+a {
+  color: #0088cc;
+  text-decoration: none;
+}
+a:hover {
+  color: #005580;
+  text-decoration: underline;
+}
+.row {
+  margin-left: -20px;
+  *zoom: 1;
+}
+.row:before,
+.row:after {
+  display: table;
+  content: "";
+}
+.row:after {
+  clear: both;
+}
+[class*="span"] {
+  float: left;
+  margin-left: 20px;
+}
+.container,
+.navbar-fixed-top .container,
+.navbar-fixed-bottom .container {
+  width: 940px;
+}
+.span12 {
+  width: 940px;
+}
+.span11 {
+  width: 860px;
+}
+.span10 {
+  width: 780px;
+}
+.span9 {
+  width: 700px;
+}
+.span8 {
+  width: 620px;
+}
+.span7 {
+  width: 540px;
+}
+.span6 {
+  width: 460px;
+}
+.span5 {
+  width: 380px;
+}
+.span4 {
+  width: 300px;
+}
+.span3 {
+  width: 220px;
+}
+.span2 {
+  width: 140px;
+}
+.span1 {
+  width: 60px;
+}
+.offset12 {
+  margin-left: 980px;
+}
+.offset11 {
+  margin-left: 900px;
+}
+.offset10 {
+  margin-left: 820px;
+}
+.offset9 {
+  margin-left: 740px;
+}
+.offset8 {
+  margin-left: 660px;
+}
+.offset7 {
+  margin-left: 580px;
+}
+.offset6 {
+  margin-left: 500px;
+}
+.offset5 {
+  margin-left: 420px;
+}
+.offset4 {
+  margin-left: 340px;
+}
+.offset3 {
+  margin-left: 260px;
+}
+.offset2 {
+  margin-left: 180px;
+}
+.offset1 {
+  margin-left: 100px;
+}
+.row-fluid {
+  width: 100%;
+  *zoom: 1;
+}
+.row-fluid:before,
+.row-fluid:after {
+  display: table;
+  content: "";
+}
+.row-fluid:after {
+  clear: both;
+}
+.row-fluid > [class*="span"] {
+  float: left;
+  margin-left: 2.127659574%;
+}
+.row-fluid > [class*="span"]:first-child {
+  margin-left: 0;
+}
+.row-fluid > .span12 {
+  width: 99.99999998999999%;
+}
+.row-fluid > .span11 {
+  width: 91.489361693%;
+}
+.row-fluid > .span10 {
+  width: 82.97872339599999%;
+}
+.row-fluid > .span9 {
+  width: 74.468085099%;
+}
+.row-fluid > .span8 {
+  width: 65.95744680199999%;
+}
+.row-fluid > .span7 {
+  width: 57.446808505%;
+}
+.row-fluid > .span6 {
+  width: 48.93617020799999%;
+}
+.row-fluid > .span5 {
+  width: 40.425531911%;
+}
+.row-fluid > .span4 {
+  width: 31.914893614%;
+}
+.row-fluid > .span3 {
+  width: 23.404255317%;
+}
+.row-fluid > .span2 {
+  width: 14.89361702%;
+}
+.row-fluid > .span1 {
+  width: 6.382978723%;
+}
+.container {
+  margin-left: auto;
+  margin-right: auto;
+  *zoom: 1;
+}
+.container:before,
+.container:after {
+  display: table;
+  content: "";
+}
+.container:after {
+  clear: both;
+}
+.container-fluid {
+  padding-left: 20px;
+  padding-right: 20px;
+  *zoom: 1;
+}
+.container-fluid:before,
+.container-fluid:after {
+  display: table;
+  content: "";
+}
+.container-fluid:after {
+  clear: both;
+}
+p {
+  margin: 0 0 9px;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  font-size: 13px;
+  line-height: 18px;
+}
+p small {
+  font-size: 11px;
+  color: #999999;
+}
+.lead {
+  margin-bottom: 18px;
+  font-size: 20px;
+  font-weight: 200;
+  line-height: 27px;
+}
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  margin: 0;
+  font-family: inherit;
+  font-weight: bold;
+  color: inherit;
+  text-rendering: optimizelegibility;
+}
+h1 small,
+h2 small,
+h3 small,
+h4 small,
+h5 small,
+h6 small {
+  font-weight: normal;
+  color: #999999;
+}
+h1 {
+  font-size: 30px;
+  line-height: 36px;
+}
+h1 small {
+  font-size: 18px;
+}
+h2 {
+  font-size: 24px;
+  line-height: 36px;
+}
+h2 small {
+  font-size: 18px;
+}
+h3 {
+  line-height: 27px;
+  font-size: 18px;
+}
+h3 small {
+  font-size: 14px;
+}
+h4,
+h5,
+h6 {
+  line-height: 18px;
+}
+h4 {
+  font-size: 14px;
+}
+h4 small {
+  font-size: 12px;
+}
+h5 {
+  font-size: 12px;
+}
+h6 {
+  font-size: 11px;
+  color: #999999;
+  text-transform: uppercase;
+}
+.page-header {
+  padding-bottom: 17px;
+  margin: 18px 0;
+  border-bottom: 1px solid #eeeeee;
+}
+.page-header h1 {
+  line-height: 1;
+}
+ul,
+ol {
+  padding: 0;
+  margin: 0 0 9px 25px;
+}
+ul ul,
+ul ol,
+ol ol,
+ol ul {
+  margin-bottom: 0;
+}
+ul {
+  list-style: disc;
+}
+ol {
+  list-style: decimal;
+}
+li {
+  line-height: 18px;
+}
+ul.unstyled,
+ol.unstyled {
+  margin-left: 0;
+  list-style: none;
+}
+dl {
+  margin-bottom: 18px;
+}
+dt,
+dd {
+  line-height: 18px;
+}
+dt {
+  font-weight: bold;
+  line-height: 17px;
+}
+dd {
+  margin-left: 9px;
+}
+.dl-horizontal dt {
+  float: left;
+  clear: left;
+  width: 120px;
+  text-align: right;
+}
+.dl-horizontal dd {
+  margin-left: 130px;
+}
+hr {
+  margin: 18px 0;
+  border: 0;
+  border-top: 1px solid #eeeeee;
+  border-bottom: 1px solid #ffffff;
+}
+strong {
+  font-weight: bold;
+}
+em {
+  font-style: italic;
+}
+.muted {
+  color: #999999;
+}
+abbr[title] {
+  border-bottom: 1px dotted #ddd;
+  cursor: help;
+}
+abbr.initialism {
+  font-size: 90%;
+  text-transform: uppercase;
+}
+blockquote {
+  padding: 0 0 0 15px;
+  margin: 0 0 18px;
+  border-left: 5px solid #eeeeee;
+}
+blockquote p {
+  margin-bottom: 0;
+  font-size: 16px;
+  font-weight: 300;
+  line-height: 22.5px;
+}
+blockquote small {
+  display: block;
+  line-height: 18px;
+  color: #999999;
+}
+blockquote small:before {
+  content: '\2014 \00A0';
+}
+blockquote.pull-right {
+  float: right;
+  padding-left: 0;
+  padding-right: 15px;
+  border-left: 0;
+  border-right: 5px solid #eeeeee;
+}
+blockquote.pull-right p,
+blockquote.pull-right small {
+  text-align: right;
+}
+q:before,
+q:after,
+blockquote:before,
+blockquote:after {
+  content: "";
+}
+address {
+  display: block;
+  margin-bottom: 18px;
+  line-height: 18px;
+  font-style: normal;
+}
+small {
+  font-size: 100%;
+}
+cite {
+  font-style: normal;
+}
+code,
+pre {
+  padding: 0 3px 2px;
+  font-family: Menlo, Monaco, "Courier New", monospace;
+  font-size: 12px;
+  color: #333333;
+  -webkit-border-radius: 3px;
+  -moz-border-radius: 3px;
+  border-radius: 3px;
+}
+code {
+  padding: 2px 4px;
+  color: #d14;
+  background-color: #f7f7f9;
+  border: 1px solid #e1e1e8;
+}
+pre {
+  display: block;
+  padding: 8.5px;
+  margin: 0 0 9px;
+  font-size: 12.025px;
+  line-height: 18px;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border: 1px solid rgba(0, 0, 0, 0.15);
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+  white-space: pre;
+  white-space: pre-wrap;
+  word-break: break-all;
+  word-wrap: break-word;
+}
+pre.prettyprint {
+  margin-bottom: 18px;
+}
+pre code {
+  padding: 0;
+  color: inherit;
+  background-color: transparent;
+  border: 0;
+}
+.pre-scrollable {
+  max-height: 340px;
+  overflow-y: scroll;
+}
+form {
+  margin: 0 0 18px;
+}
+fieldset {
+  padding: 0;
+  margin: 0;
+  border: 0;
+}
+legend {
+  display: block;
+  width: 100%;
+  padding: 0;
+  margin-bottom: 27px;
+  font-size: 19.5px;
+  line-height: 36px;
+  color: #333333;
+  border: 0;
+  border-bottom: 1px solid #eee;
+}
+legend small {
+  font-size: 13.5px;
+  color: #999999;
+}
+label,
+input,
+button,
+select,
+textarea {
+  font-size: 13px;
+  font-weight: normal;
+  line-height: 18px;
+}
+input,
+button,
+select,
+textarea {
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+label {
+  display: block;
+  margin-bottom: 5px;
+  color: #333333;
+}
+input,
+textarea,
+select,
+.uneditable-input {
+  display: inline-block;
+  width: 210px;
+  height: 18px;
+  padding: 4px;
+  margin-bottom: 9px;
+  font-size: 13px;
+  line-height: 18px;
+  color: #555555;
+  border: 1px solid #cccccc;
+  -webkit-border-radius: 3px;
+  -moz-border-radius: 3px;
+  border-radius: 3px;
+}
+.uneditable-textarea {
+  width: auto;
+  height: auto;
+}
+label input,
+label textarea,
+label select {
+  display: block;
+}
+input[type="image"],
+input[type="checkbox"],
+input[type="radio"] {
+  width: auto;
+  height: auto;
+  padding: 0;
+  margin: 3px 0;
+  *margin-top: 0;
+  /* IE7 */
+
+  line-height: normal;
+  cursor: pointer;
+  -webkit-border-radius: 0;
+  -moz-border-radius: 0;
+  border-radius: 0;
+  border: 0 \9;
+  /* IE9 and down */
+
+}
+input[type="image"] {
+  border: 0;
+}
+input[type="file"] {
+  width: auto;
+  padding: initial;
+  line-height: initial;
+  border: initial;
+  background-color: #ffffff;
+  background-color: initial;
+  -webkit-box-shadow: none;
+  -moz-box-shadow: none;
+  box-shadow: none;
+}
+input[type="button"],
+input[type="reset"],
+input[type="submit"] {
+  width: auto;
+  height: auto;
+}
+select,
+input[type="file"] {
+  height: 28px;
+  /* In IE7, the height of the select element cannot be changed by height, only font-size */
+
+  *margin-top: 4px;
+  /* For IE7, add top margin to align select with labels */
+
+  line-height: 28px;
+}
+input[type="file"] {
+  line-height: 18px \9;
+}
+select {
+  width: 220px;
+  background-color: #ffffff;
+}
+select[multiple],
+select[size] {
+  height: auto;
+}
+input[type="image"] {
+  -webkit-box-shadow: none;
+  -moz-box-shadow: none;
+  box-shadow: none;
+}
+textarea {
+  height: auto;
+}
+input[type="hidden"] {
+  display: none;
+}
+.radio,
+.checkbox {
+  padding-left: 18px;
+}
+.radio input[type="radio"],
+.checkbox input[type="checkbox"] {
+  float: left;
+  margin-left: -18px;
+}
+.controls > .radio:first-child,
+.controls > .checkbox:first-child {
+  padding-top: 5px;
+}
+.radio.inline,
+.checkbox.inline {
+  display: inline-block;
+  padding-top: 5px;
+  margin-bottom: 0;
+  vertical-align: middle;
+}
+.radio.inline + .radio.inline,
+.checkbox.inline + .checkbox.inline {
+  margin-left: 10px;
+}
+input,
+textarea {
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+  -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+  -webkit-transition: border linear 0.2s, box-shadow linear 0.2s;
+  -moz-transition: border linear 0.2s, box-shadow linear 0.2s;
+  -ms-transition: border linear 0.2s, box-shadow linear 0.2s;
+  -o-transition: border linear 0.2s, box-shadow linear 0.2s;
+  transition: border linear 0.2s, box-shadow linear 0.2s;
+}
+input:focus,
+textarea:focus {
+  border-color: rgba(82, 168, 236, 0.8);
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+  -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+  outline: 0;
+  outline: thin dotted \9;
+  /* IE6-9 */
+
+}
+input[type="file"]:focus,
+input[type="radio"]:focus,
+input[type="checkbox"]:focus,
+select:focus {
+  -webkit-box-shadow: none;
+  -moz-box-shadow: none;
+  box-shadow: none;
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
+}
+.input-mini {
+  width: 60px;
+}
+.input-small {
+  width: 90px;
+}
+.input-medium {
+  width: 150px;
+}
+.input-large {
+  width: 210px;
+}
+.input-xlarge {
+  width: 270px;
+}
+.input-xxlarge {
+  width: 530px;
+}
+input[class*="span"],
+select[class*="span"],
+textarea[class*="span"],
+.uneditable-input {
+  float: none;
+  margin-left: 0;
+}
+input,
+textarea,
+.uneditable-input {
+  margin-left: 0;
+}
+input.span12, textarea.span12, .uneditable-input.span12 {
+  width: 930px;
+}
+input.span11, textarea.span11, .uneditable-input.span11 {
+  width: 850px;
+}
+input.span10, textarea.span10, .uneditable-input.span10 {
+  width: 770px;
+}
+input.span9, textarea.span9, .uneditable-input.span9 {
+  width: 690px;
+}
+input.span8, textarea.span8, .uneditable-input.span8 {
+  width: 610px;
+}
+input.span7, textarea.span7, .uneditable-input.span7 {
+  width: 530px;
+}
+input.span6, textarea.span6, .uneditable-input.span6 {
+  width: 450px;
+}
+input.span5, textarea.span5, .uneditable-input.span5 {
+  width: 370px;
+}
+input.span4, textarea.span4, .uneditable-input.span4 {
+  width: 290px;
+}
+input.span3, textarea.span3, .uneditable-input.span3 {
+  width: 210px;
+}
+input.span2, textarea.span2, .uneditable-input.span2 {
+  width: 130px;
+}
+input.span1, textarea.span1, .uneditable-input.span1 {
+  width: 50px;
+}
+input[disabled],
+select[disabled],
+textarea[disabled],
+input[readonly],
+select[readonly],
+textarea[readonly] {
+  background-color: #eeeeee;
+  border-color: #ddd;
+  cursor: not-allowed;
+}
+.control-group.warning > label,
+.control-group.warning .help-block,
+.control-group.warning .help-inline {
+  color: #c09853;
+}
+.control-group.warning input,
+.control-group.warning select,
+.control-group.warning textarea {
+  color: #c09853;
+  border-color: #c09853;
+}
+.control-group.warning input:focus,
+.control-group.warning select:focus,
+.control-group.warning textarea:focus {
+  border-color: #a47e3c;
+  -webkit-box-shadow: 0 0 6px #dbc59e;
+  -moz-box-shadow: 0 0 6px #dbc59e;
+  box-shadow: 0 0 6px #dbc59e;
+}
+.control-group.warning .input-prepend .add-on,
+.control-group.warning .input-append .add-on {
+  color: #c09853;
+  background-color: #fcf8e3;
+  border-color: #c09853;
+}
+.control-group.error > label,
+.control-group.error .help-block,
+.control-group.error .help-inline {
+  color: #b94a48;
+}
+.control-group.error input,
+.control-group.error select,
+.control-group.error textarea {
+  color: #b94a48;
+  border-color: #b94a48;
+}
+.control-group.error input:focus,
+.control-group.error select:focus,
+.control-group.error textarea:focus {
+  border-color: #953b39;
+  -webkit-box-shadow: 0 0 6px #d59392;
+  -moz-box-shadow: 0 0 6px #d59392;
+  box-shadow: 0 0 6px #d59392;
+}
+.control-group.error .input-prepend .add-on,
+.control-group.error .input-append .add-on {
+  color: #b94a48;
+  background-color: #f2dede;
+  border-color: #b94a48;
+}
+.control-group.success > label,
+.control-group.success .help-block,
+.control-group.success .help-inline {
+  color: #468847;
+}
+.control-group.success input,
+.control-group.success select,
+.control-group.success textarea {
+  color: #468847;
+  border-color: #468847;
+}
+.control-group.success input:focus,
+.control-group.success select:focus,
+.control-group.success textarea:focus {
+  border-color: #356635;
+  -webkit-box-shadow: 0 0 6px #7aba7b;
+  -moz-box-shadow: 0 0 6px #7aba7b;
+  box-shadow: 0 0 6px #7aba7b;
+}
+.control-group.success .input-prepend .add-on,
+.control-group.success .input-append .add-on {
+  color: #468847;
+  background-color: #dff0d8;
+  border-color: #468847;
+}
+input:focus:required:invalid,
+textarea:focus:required:invalid,
+select:focus:required:invalid {
+  color: #b94a48;
+  border-color: #ee5f5b;
+}
+input:focus:required:invalid:focus,
+textarea:focus:required:invalid:focus,
+select:focus:required:invalid:focus {
+  border-color: #e9322d;
+  -webkit-box-shadow: 0 0 6px #f8b9b7;
+  -moz-box-shadow: 0 0 6px #f8b9b7;
+  box-shadow: 0 0 6px #f8b9b7;
+}
+.form-actions {
+  padding: 17px 20px 18px;
+  margin-top: 18px;
+  margin-bottom: 18px;
+  background-color: #eeeeee;
+  border-top: 1px solid #ddd;
+  *zoom: 1;
+}
+.form-actions:before,
+.form-actions:after {
+  display: table;
+  content: "";
+}
+.form-actions:after {
+  clear: both;
+}
+.uneditable-input {
+  display: block;
+  background-color: #ffffff;
+  border-color: #eee;
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+  -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+  cursor: not-allowed;
+}
+:-moz-placeholder {
+  color: #999999;
+}
+::-webkit-input-placeholder {
+  color: #999999;
+}
+.help-block,
+.help-inline {
+  color: #555555;
+}
+.help-block {
+  display: block;
+  margin-bottom: 9px;
+}
+.help-inline {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+  vertical-align: middle;
+  padding-left: 5px;
+}
+.input-prepend,
+.input-append {
+  margin-bottom: 5px;
+}
+.input-prepend input,
+.input-append input,
+.input-prepend select,
+.input-append select,
+.input-prepend .uneditable-input,
+.input-append .uneditable-input {
+  *margin-left: 0;
+  -webkit-border-radius: 0 3px 3px 0;
+  -moz-border-radius: 0 3px 3px 0;
+  border-radius: 0 3px 3px 0;
+}
+.input-prepend input:focus,
+.input-append input:focus,
+.input-prepend select:focus,
+.input-append select:focus,
+.input-prepend .uneditable-input:focus,
+.input-append .uneditable-input:focus {
+  position: relative;
+  z-index: 2;
+}
+.input-prepend .uneditable-input,
+.input-append .uneditable-input {
+  border-left-color: #ccc;
+}
+.input-prepend .add-on,
+.input-append .add-on {
+  display: inline-block;
+  width: auto;
+  min-width: 16px;
+  height: 18px;
+  padding: 4px 5px;
+  font-weight: normal;
+  line-height: 18px;
+  text-align: center;
+  text-shadow: 0 1px 0 #ffffff;
+  vertical-align: middle;
+  background-color: #eeeeee;
+  border: 1px solid #ccc;
+}
+.input-prepend .add-on,
+.input-append .add-on,
+.input-prepend .btn,
+.input-append .btn {
+  -webkit-border-radius: 3px 0 0 3px;
+  -moz-border-radius: 3px 0 0 3px;
+  border-radius: 3px 0 0 3px;
+}
+.input-prepend .active,
+.input-append .active {
+  background-color: #a9dba9;
+  border-color: #46a546;
+}
+.input-prepend .add-on,
+.input-prepend .btn {
+  margin-right: -1px;
+}
+.input-append input,
+.input-append select .uneditable-input {
+  -webkit-border-radius: 3px 0 0 3px;
+  -moz-border-radius: 3px 0 0 3px;
+  border-radius: 3px 0 0 3px;
+}
+.input-append .uneditable-input {
+  border-left-color: #eee;
+  border-right-color: #ccc;
+}
+.input-append .add-on,
+.input-append .btn {
+  margin-left: -1px;
+  -webkit-border-radius: 0 3px 3px 0;
+  -moz-border-radius: 0 3px 3px 0;
+  border-radius: 0 3px 3px 0;
+}
+.input-prepend.input-append input,
+.input-prepend.input-append select,
+.input-prepend.input-append .uneditable-input {
+  -webkit-border-radius: 0;
+  -moz-border-radius: 0;
+  border-radius: 0;
+}
+.input-prepend.input-append .add-on:first-child,
+.input-prepend.input-append .btn:first-child {
+  margin-right: -1px;
+  -webkit-border-radius: 3px 0 0 3px;
+  -moz-border-radius: 3px 0 0 3px;
+  border-radius: 3px 0 0 3px;
+}
+.input-prepend.input-append .add-on:last-child,
+.input-prepend.input-append .btn:last-child {
+  margin-left: -1px;
+  -webkit-border-radius: 0 3px 3px 0;
+  -moz-border-radius: 0 3px 3px 0;
+  border-radius: 0 3px 3px 0;
+}
+.search-query {
+  padding-left: 14px;
+  padding-right: 14px;
+  margin-bottom: 0;
+  -webkit-border-radius: 14px;
+  -moz-border-radius: 14px;
+  border-radius: 14px;
+}
+.form-search input,
+.form-inline input,
+.form-horizontal input,
+.form-search textarea,
+.form-inline textarea,
+.form-horizontal textarea,
+.form-search select,
+.form-inline select,
+.form-horizontal select,
+.form-search .help-inline,
+.form-inline .help-inline,
+.form-horizontal .help-inline,
+.form-search .uneditable-input,
+.form-inline .uneditable-input,
+.form-horizontal .uneditable-input,
+.form-search .input-prepend,
+.form-inline .input-prepend,
+.form-horizontal .input-prepend,
+.form-search .input-append,
+.form-inline .input-append,
+.form-horizontal .input-append {
+  display: inline-block;
+  margin-bottom: 0;
+}
+.form-search .hide,
+.form-inline .hide,
+.form-horizontal .hide {
+  display: none;
+}
+.form-search label,
+.form-inline label {
+  display: inline-block;
+}
+.form-search .input-append,
+.form-inline .input-append,
+.form-search .input-prepend,
+.form-inline .input-prepend {
+  margin-bottom: 0;
+}
+.form-search .radio,
+.form-search .checkbox,
+.form-inline .radio,
+.form-inline .checkbox {
+  padding-left: 0;
+  margin-bottom: 0;
+  vertical-align: middle;
+}
+.form-search .radio input[type="radio"],
+.form-search .checkbox input[type="checkbox"],
+.form-inline .radio input[type="radio"],
+.form-inline .checkbox input[type="checkbox"] {
+  float: left;
+  margin-left: 0;
+  margin-right: 3px;
+}
+.control-group {
+  margin-bottom: 9px;
+}
+legend + .control-group {
+  margin-top: 18px;
+  -webkit-margin-top-collapse: separate;
+}
+.form-horizontal .control-group {
+  margin-bottom: 18px;
+  *zoom: 1;
+}
+.form-horizontal .control-group:before,
+.form-horizontal .control-group:after {
+  display: table;
+  content: "";
+}
+.form-horizontal .control-group:after {
+  clear: both;
+}
+.form-horizontal .control-label {
+  float: left;
+  width: 140px;
+  padding-top: 5px;
+  text-align: right;
+}
+.form-horizontal .controls {
+  margin-left: 160px;
+  /* Super jank IE7 fix to ensure the inputs in .input-append and input-prepend don't inherit the margin of the parent, in this case .controls */
+
+  *display: inline-block;
+  *margin-left: 0;
+  *padding-left: 20px;
+}
+.form-horizontal .help-block {
+  margin-top: 9px;
+  margin-bottom: 0;
+}
+.form-horizontal .form-actions {
+  padding-left: 160px;
+}
+table {
+  max-width: 100%;
+  border-collapse: collapse;
+  border-spacing: 0;
+  background-color: transparent;
+}
+.table {
+  width: 100%;
+  margin-bottom: 18px;
+}
+.table th,
+.table td {
+  padding: 8px;
+  line-height: 18px;
+  text-align: left;
+  vertical-align: top;
+  border-top: 1px solid #dddddd;
+}
+.table th {
+  font-weight: bold;
+}
+.table thead th {
+  vertical-align: bottom;
+}
+.table colgroup + thead tr:first-child th,
+.table colgroup + thead tr:first-child td,
+.table thead:first-child tr:first-child th,
+.table thead:first-child tr:first-child td {
+  border-top: 0;
+}
+.table tbody + tbody {
+  border-top: 2px solid #dddddd;
+}
+.table-condensed th,
+.table-condensed td {
+  padding: 4px 5px;
+}
+.table-bordered {
+  border: 1px solid #dddddd;
+  border-left: 0;
+  border-collapse: separate;
+  *border-collapse: collapsed;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.table-bordered th,
+.table-bordered td {
+  border-left: 1px solid #dddddd;
+}
+.table-bordered thead:first-child tr:first-child th,
+.table-bordered tbody:first-child tr:first-child th,
+.table-bordered tbody:first-child tr:first-child td {
+  border-top: 0;
+}
+.table-bordered thead:first-child tr:first-child th:first-child,
+.table-bordered tbody:first-child tr:first-child td:first-child {
+  -webkit-border-radius: 4px 0 0 0;
+  -moz-border-radius: 4px 0 0 0;
+  border-radius: 4px 0 0 0;
+}
+.table-bordered thead:first-child tr:first-child th:last-child,
+.table-bordered tbody:first-child tr:first-child td:last-child {
+  -webkit-border-radius: 0 4px 0 0;
+  -moz-border-radius: 0 4px 0 0;
+  border-radius: 0 4px 0 0;
+}
+.table-bordered thead:last-child tr:last-child th:first-child,
+.table-bordered tbody:last-child tr:last-child td:first-child {
+  -webkit-border-radius: 0 0 0 4px;
+  -moz-border-radius: 0 0 0 4px;
+  border-radius: 0 0 0 4px;
+}
+.table-bordered thead:last-child tr:last-child th:last-child,
+.table-bordered tbody:last-child tr:last-child td:last-child {
+  -webkit-border-radius: 0 0 4px 0;
+  -moz-border-radius: 0 0 4px 0;
+  border-radius: 0 0 4px 0;
+}
+.table-striped tbody tr:nth-child(odd) td,
+.table-striped tbody tr:nth-child(odd) th {
+  background-color: #f9f9f9;
+}
+.table tbody tr:hover td,
+.table tbody tr:hover th {
+  background-color: #f5f5f5;
+}
+table .span1 {
+  float: none;
+  width: 44px;
+  margin-left: 0;
+}
+table .span2 {
+  float: none;
+  width: 124px;
+  margin-left: 0;
+}
+table .span3 {
+  float: none;
+  width: 204px;
+  margin-left: 0;
+}
+table .span4 {
+  float: none;
+  width: 284px;
+  margin-left: 0;
+}
+table .span5 {
+  float: none;
+  width: 364px;
+  margin-left: 0;
+}
+table .span6 {
+  float: none;
+  width: 444px;
+  margin-left: 0;
+}
+table .span7 {
+  float: none;
+  width: 524px;
+  margin-left: 0;
+}
+table .span8 {
+  float: none;
+  width: 604px;
+  margin-left: 0;
+}
+table .span9 {
+  float: none;
+  width: 684px;
+  margin-left: 0;
+}
+table .span10 {
+  float: none;
+  width: 764px;
+  margin-left: 0;
+}
+table .span11 {
+  float: none;
+  width: 844px;
+  margin-left: 0;
+}
+table .span12 {
+  float: none;
+  width: 924px;
+  margin-left: 0;
+}
+table .span13 {
+  float: none;
+  width: 1004px;
+  margin-left: 0;
+}
+table .span14 {
+  float: none;
+  width: 1084px;
+  margin-left: 0;
+}
+table .span15 {
+  float: none;
+  width: 1164px;
+  margin-left: 0;
+}
+table .span16 {
+  float: none;
+  width: 1244px;
+  margin-left: 0;
+}
+table .span17 {
+  float: none;
+  width: 1324px;
+  margin-left: 0;
+}
+table .span18 {
+  float: none;
+  width: 1404px;
+  margin-left: 0;
+}
+table .span19 {
+  float: none;
+  width: 1484px;
+  margin-left: 0;
+}
+table .span20 {
+  float: none;
+  width: 1564px;
+  margin-left: 0;
+}
+table .span21 {
+  float: none;
+  width: 1644px;
+  margin-left: 0;
+}
+table .span22 {
+  float: none;
+  width: 1724px;
+  margin-left: 0;
+}
+table .span23 {
+  float: none;
+  width: 1804px;
+  margin-left: 0;
+}
+table .span24 {
+  float: none;
+  width: 1884px;
+  margin-left: 0;
+}
+[class^="icon-"],
+[class*=" icon-"] {
+  display: inline-block;
+  width: 14px;
+  height: 14px;
+  line-height: 14px;
+  vertical-align: text-top;
+  background-image: url("../img/glyphicons-halflings.png");
+  background-position: 14px 14px;
+  background-repeat: no-repeat;
+  *margin-right: .3em;
+}
+[class^="icon-"]:last-child,
+[class*=" icon-"]:last-child {
+  *margin-left: 0;
+}
+.icon-white {
+  background-image: url("../img/glyphicons-halflings-white.png");
+}
+.icon-glass {
+  background-position: 0      0;
+}
+.icon-music {
+  background-position: -24px 0;
+}
+.icon-search {
+  background-position: -48px 0;
+}
+.icon-envelope {
+  background-position: -72px 0;
+}
+.icon-heart {
+  background-position: -96px 0;
+}
+.icon-star {
+  background-position: -120px 0;
+}
+.icon-star-empty {
+  background-position: -144px 0;
+}
+.icon-user {
+  background-position: -168px 0;
+}
+.icon-film {
+  background-position: -192px 0;
+}
+.icon-th-large {
+  background-position: -216px 0;
+}
+.icon-th {
+  background-position: -240px 0;
+}
+.icon-th-list {
+  background-position: -264px 0;
+}
+.icon-ok {
+  background-position: -288px 0;
+}
+.icon-remove {
+  background-position: -312px 0;
+}
+.icon-zoom-in {
+  background-position: -336px 0;
+}
+.icon-zoom-out {
+  background-position: -360px 0;
+}
+.icon-off {
+  background-position: -384px 0;
+}
+.icon-signal {
+  background-position: -408px 0;
+}
+.icon-cog {
+  background-position: -432px 0;
+}
+.icon-trash {
+  background-position: -456px 0;
+}
+.icon-home {
+  background-position: 0 -24px;
+}
+.icon-file {
+  background-position: -24px -24px;
+}
+.icon-time {
+  background-position: -48px -24px;
+}
+.icon-road {
+  background-position: -72px -24px;
+}
+.icon-download-alt {
+  background-position: -96px -24px;
+}
+.icon-download {
+  background-position: -120px -24px;
+}
+.icon-upload {
+  background-position: -144px -24px;
+}
+.icon-inbox {
+  background-position: -168px -24px;
+}
+.icon-play-circle {
+  background-position: -192px -24px;
+}
+.icon-repeat {
+  background-position: -216px -24px;
+}
+.icon-refresh {
+  background-position: -240px -24px;
+}
+.icon-list-alt {
+  background-position: -264px -24px;
+}
+.icon-lock {
+  background-position: -287px -24px;
+}
+.icon-flag {
+  background-position: -312px -24px;
+}
+.icon-headphones {
+  background-position: -336px -24px;
+}
+.icon-volume-off {
+  background-position: -360px -24px;
+}
+.icon-volume-down {
+  background-position: -384px -24px;
+}
+.icon-volume-up {
+  background-position: -408px -24px;
+}
+.icon-qrcode {
+  background-position: -432px -24px;
+}
+.icon-barcode {
+  background-position: -456px -24px;
+}
+.icon-tag {
+  background-position: 0 -48px;
+}
+.icon-tags {
+  background-position: -25px -48px;
+}
+.icon-book {
+  background-position: -48px -48px;
+}
+.icon-bookmark {
+  background-position: -72px -48px;
+}
+.icon-print {
+  background-position: -96px -48px;
+}
+.icon-camera {
+  background-position: -120px -48px;
+}
+.icon-font {
+  background-position: -144px -48px;
+}
+.icon-bold {
+  background-position: -167px -48px;
+}
+.icon-italic {
+  background-position: -192px -48px;
+}
+.icon-text-height {
+  background-position: -216px -48px;
+}
+.icon-text-width {
+  background-position: -240px -48px;
+}
+.icon-align-left {
+  background-position: -264px -48px;
+}
+.icon-align-center {
+  background-position: -288px -48px;
+}
+.icon-align-right {
+  background-position: -312px -48px;
+}
+.icon-align-justify {
+  background-position: -336px -48px;
+}
+.icon-list {
+  background-position: -360px -48px;
+}
+.icon-indent-left {
+  background-position: -384px -48px;
+}
+.icon-indent-right {
+  background-position: -408px -48px;
+}
+.icon-facetime-video {
+  background-position: -432px -48px;
+}
+.icon-picture {
+  background-position: -456px -48px;
+}
+.icon-pencil {
+  background-position: 0 -72px;
+}
+.icon-map-marker {
+  background-position: -24px -72px;
+}
+.icon-adjust {
+  background-position: -48px -72px;
+}
+.icon-tint {
+  background-position: -72px -72px;
+}
+.icon-edit {
+  background-position: -96px -72px;
+}
+.icon-share {
+  background-position: -120px -72px;
+}
+.icon-check {
+  background-position: -144px -72px;
+}
+.icon-move {
+  background-position: -168px -72px;
+}
+.icon-step-backward {
+  background-position: -192px -72px;
+}
+.icon-fast-backward {
+  background-position: -216px -72px;
+}
+.icon-backward {
+  background-position: -240px -72px;
+}
+.icon-play {
+  background-position: -264px -72px;
+}
+.icon-pause {
+  background-position: -288px -72px;
+}
+.icon-stop {
+  background-position: -312px -72px;
+}
+.icon-forward {
+  background-position: -336px -72px;
+}
+.icon-fast-forward {
+  background-position: -360px -72px;
+}
+.icon-step-forward {
+  background-position: -384px -72px;
+}
+.icon-eject {
+  background-position: -408px -72px;
+}
+.icon-chevron-left {
+  background-position: -432px -72px;
+}
+.icon-chevron-right {
+  background-position: -456px -72px;
+}
+.icon-plus-sign {
+  background-position: 0 -96px;
+}
+.icon-minus-sign {
+  background-position: -24px -96px;
+}
+.icon-remove-sign {
+  background-position: -48px -96px;
+}
+.icon-ok-sign {
+  background-position: -72px -96px;
+}
+.icon-question-sign {
+  background-position: -96px -96px;
+}
+.icon-info-sign {
+  background-position: -120px -96px;
+}
+.icon-screenshot {
+  background-position: -144px -96px;
+}
+.icon-remove-circle {
+  background-position: -168px -96px;
+}
+.icon-ok-circle {
+  background-position: -192px -96px;
+}
+.icon-ban-circle {
+  background-position: -216px -96px;
+}
+.icon-arrow-left {
+  background-position: -240px -96px;
+}
+.icon-arrow-right {
+  background-position: -264px -96px;
+}
+.icon-arrow-up {
+  background-position: -289px -96px;
+}
+.icon-arrow-down {
+  background-position: -312px -96px;
+}
+.icon-share-alt {
+  background-position: -336px -96px;
+}
+.icon-resize-full {
+  background-position: -360px -96px;
+}
+.icon-resize-small {
+  background-position: -384px -96px;
+}
+.icon-plus {
+  background-position: -408px -96px;
+}
+.icon-minus {
+  background-position: -433px -96px;
+}
+.icon-asterisk {
+  background-position: -456px -96px;
+}
+.icon-exclamation-sign {
+  background-position: 0 -120px;
+}
+.icon-gift {
+  background-position: -24px -120px;
+}
+.icon-leaf {
+  background-position: -48px -120px;
+}
+.icon-fire {
+  background-position: -72px -120px;
+}
+.icon-eye-open {
+  background-position: -96px -120px;
+}
+.icon-eye-close {
+  background-position: -120px -120px;
+}
+.icon-warning-sign {
+  background-position: -144px -120px;
+}
+.icon-plane {
+  background-position: -168px -120px;
+}
+.icon-calendar {
+  background-position: -192px -120px;
+}
+.icon-random {
+  background-position: -216px -120px;
+}
+.icon-comment {
+  background-position: -240px -120px;
+}
+.icon-magnet {
+  background-position: -264px -120px;
+}
+.icon-chevron-up {
+  background-position: -288px -120px;
+}
+.icon-chevron-down {
+  background-position: -313px -119px;
+}
+.icon-retweet {
+  background-position: -336px -120px;
+}
+.icon-shopping-cart {
+  background-position: -360px -120px;
+}
+.icon-folder-close {
+  background-position: -384px -120px;
+}
+.icon-folder-open {
+  background-position: -408px -120px;
+}
+.icon-resize-vertical {
+  background-position: -432px -119px;
+}
+.icon-resize-horizontal {
+  background-position: -456px -118px;
+}
+.dropdown {
+  position: relative;
+}
+.dropdown-toggle {
+  *margin-bottom: -3px;
+}
+.dropdown-toggle:active,
+.open .dropdown-toggle {
+  outline: 0;
+}
+.caret {
+  display: inline-block;
+  width: 0;
+  height: 0;
+  vertical-align: top;
+  border-left: 4px solid transparent;
+  border-right: 4px solid transparent;
+  border-top: 4px solid #000000;
+  opacity: 0.3;
+  filter: alpha(opacity=30);
+  content: "";
+}
+.dropdown .caret {
+  margin-top: 8px;
+  margin-left: 2px;
+}
+.dropdown:hover .caret,
+.open.dropdown .caret {
+  opacity: 1;
+  filter: alpha(opacity=100);
+}
+.dropdown-menu {
+  position: absolute;
+  top: 100%;
+  left: 0;
+  z-index: 1000;
+  float: left;
+  display: none;
+  min-width: 160px;
+  padding: 4px 0;
+  margin: 0;
+  list-style: none;
+  background-color: #ffffff;
+  border-color: #ccc;
+  border-color: rgba(0, 0, 0, 0.2);
+  border-style: solid;
+  border-width: 1px;
+  -webkit-border-radius: 0 0 5px 5px;
+  -moz-border-radius: 0 0 5px 5px;
+  border-radius: 0 0 5px 5px;
+  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+  -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+  box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+  -webkit-background-clip: padding-box;
+  -moz-background-clip: padding;
+  background-clip: padding-box;
+  *border-right-width: 2px;
+  *border-bottom-width: 2px;
+}
+.dropdown-menu.pull-right {
+  right: 0;
+  left: auto;
+}
+.dropdown-menu .divider {
+  height: 1px;
+  margin: 8px 1px;
+  overflow: hidden;
+  background-color: #e5e5e5;
+  border-bottom: 1px solid #ffffff;
+  *width: 100%;
+  *margin: -5px 0 5px;
+}
+.dropdown-menu a {
+  display: block;
+  padding: 3px 15px;
+  clear: both;
+  font-weight: normal;
+  line-height: 18px;
+  color: #333333;
+  white-space: nowrap;
+}
+.dropdown-menu li > a:hover,
+.dropdown-menu .active > a,
+.dropdown-menu .active > a:hover {
+  color: #ffffff;
+  text-decoration: none;
+  background-color: #0088cc;
+}
+.dropdown.open {
+  *z-index: 1000;
+}
+.dropdown.open .dropdown-toggle {
+  color: #ffffff;
+  background: #ccc;
+  background: rgba(0, 0, 0, 0.3);
+}
+.dropdown.open .dropdown-menu {
+  display: block;
+}
+.pull-right .dropdown-menu {
+  left: auto;
+  right: 0;
+}
+.dropup .caret,
+.navbar-fixed-bottom .dropdown .caret {
+  border-top: 0;
+  border-bottom: 4px solid #000000;
+  content: "\2191";
+}
+.dropup .dropdown-menu,
+.navbar-fixed-bottom .dropdown .dropdown-menu {
+  top: auto;
+  bottom: 100%;
+  margin-bottom: 1px;
+}
+.typeahead {
+  margin-top: 2px;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.well {
+  min-height: 20px;
+  padding: 19px;
+  margin-bottom: 20px;
+  background-color: #f5f5f5;
+  border: 1px solid #eee;
+  border: 1px solid rgba(0, 0, 0, 0.05);
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+}
+.well blockquote {
+  border-color: #ddd;
+  border-color: rgba(0, 0, 0, 0.15);
+}
+.well-large {
+  padding: 24px;
+  -webkit-border-radius: 6px;
+  -moz-border-radius: 6px;
+  border-radius: 6px;
+}
+.well-small {
+  padding: 9px;
+  -webkit-border-radius: 3px;
+  -moz-border-radius: 3px;
+  border-radius: 3px;
+}
+.fade {
+  -webkit-transition: opacity 0.15s linear;
+  -moz-transition: opacity 0.15s linear;
+  -ms-transition: opacity 0.15s linear;
+  -o-transition: opacity 0.15s linear;
+  transition: opacity 0.15s linear;
+  opacity: 0;
+}
+.fade.in {
+  opacity: 1;
+}
+.collapse {
+  -webkit-transition: height 0.35s ease;
+  -moz-transition: height 0.35s ease;
+  -ms-transition: height 0.35s ease;
+  -o-transition: height 0.35s ease;
+  transition: height 0.35s ease;
+  position: relative;
+  overflow: hidden;
+  height: 0;
+}
+.collapse.in {
+  height: auto;
+}
+.close {
+  float: right;
+  font-size: 20px;
+  font-weight: bold;
+  line-height: 18px;
+  color: #000000;
+  text-shadow: 0 1px 0 #ffffff;
+  opacity: 0.2;
+  filter: alpha(opacity=20);
+}
+.close:hover {
+  color: #000000;
+  text-decoration: none;
+  opacity: 0.4;
+  filter: alpha(opacity=40);
+  cursor: pointer;
+}
+.btn {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+  padding: 4px 10px 4px;
+  margin-bottom: 0;
+  font-size: 13px;
+  line-height: 18px;
+  color: #333333;
+  text-align: center;
+  text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
+  vertical-align: middle;
+  background-color: #f5f5f5;
+  background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: -ms-linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));
+  background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: -o-linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: linear-gradient(top, #ffffff, #e6e6e6);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#e6e6e6', GradientType=0);
+  border-color: #e6e6e6 #e6e6e6 #bfbfbf;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+  border: 1px solid #cccccc;
+  border-bottom-color: #b3b3b3;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  cursor: pointer;
+  *margin-left: .3em;
+}
+.btn:hover,
+.btn:active,
+.btn.active,
+.btn.disabled,
+.btn[disabled] {
+  background-color: #e6e6e6;
+}
+.btn:active,
+.btn.active {
+  background-color: #cccccc \9;
+}
+.btn:first-child {
+  *margin-left: 0;
+}
+.btn:hover {
+  color: #333333;
+  text-decoration: none;
+  background-color: #e6e6e6;
+  background-position: 0 -15px;
+  -webkit-transition: background-position 0.1s linear;
+  -moz-transition: background-position 0.1s linear;
+  -ms-transition: background-position 0.1s linear;
+  -o-transition: background-position 0.1s linear;
+  transition: background-position 0.1s linear;
+}
+.btn:focus {
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
+}
+.btn.active,
+.btn:active {
+  background-image: none;
+  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+  box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+  background-color: #e6e6e6;
+  background-color: #d9d9d9 \9;
+  outline: 0;
+}
+.btn.disabled,
+.btn[disabled] {
+  cursor: default;
+  background-image: none;
+  background-color: #e6e6e6;
+  opacity: 0.65;
+  filter: alpha(opacity=65);
+  -webkit-box-shadow: none;
+  -moz-box-shadow: none;
+  box-shadow: none;
+}
+.btn-large {
+  padding: 9px 14px;
+  font-size: 15px;
+  line-height: normal;
+  -webkit-border-radius: 5px;
+  -moz-border-radius: 5px;
+  border-radius: 5px;
+}
+.btn-large [class^="icon-"] {
+  margin-top: 1px;
+}
+.btn-small {
+  padding: 5px 9px;
+  font-size: 11px;
+  line-height: 16px;
+}
+.btn-small [class^="icon-"] {
+  margin-top: -1px;
+}
+.btn-mini {
+  padding: 2px 6px;
+  font-size: 11px;
+  line-height: 14px;
+}
+.btn-primary,
+.btn-primary:hover,
+.btn-warning,
+.btn-warning:hover,
+.btn-danger,
+.btn-danger:hover,
+.btn-success,
+.btn-success:hover,
+.btn-info,
+.btn-info:hover,
+.btn-inverse,
+.btn-inverse:hover {
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  color: #ffffff;
+}
+.btn-primary.active,
+.btn-warning.active,
+.btn-danger.active,
+.btn-success.active,
+.btn-info.active,
+.btn-inverse.active {
+  color: rgba(255, 255, 255, 0.75);
+}
+.btn-primary {
+  background-color: #0074cc;
+  background-image: -moz-linear-gradient(top, #0088cc, #0055cc);
+  background-image: -ms-linear-gradient(top, #0088cc, #0055cc);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0055cc));
+  background-image: -webkit-linear-gradient(top, #0088cc, #0055cc);
+  background-image: -o-linear-gradient(top, #0088cc, #0055cc);
+  background-image: linear-gradient(top, #0088cc, #0055cc);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#0088cc', endColorstr='#0055cc', GradientType=0);
+  border-color: #0055cc #0055cc #003580;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+.btn-primary:hover,
+.btn-primary:active,
+.btn-primary.active,
+.btn-primary.disabled,
+.btn-primary[disabled] {
+  background-color: #0055cc;
+}
+.btn-primary:active,
+.btn-primary.active {
+  background-color: #004099 \9;
+}
+.btn-warning {
+  background-color: #faa732;
+  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
+  background-image: -ms-linear-gradient(top, #fbb450, #f89406);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
+  background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
+  background-image: -o-linear-gradient(top, #fbb450, #f89406);
+  background-image: linear-gradient(top, #fbb450, #f89406);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fbb450', endColorstr='#f89406', GradientType=0);
+  border-color: #f89406 #f89406 #ad6704;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+.btn-warning:hover,
+.btn-warning:active,
+.btn-warning.active,
+.btn-warning.disabled,
+.btn-warning[disabled] {
+  background-color: #f89406;
+}
+.btn-warning:active,
+.btn-warning.active {
+  background-color: #c67605 \9;
+}
+.btn-danger {
+  background-color: #da4f49;
+  background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: -ms-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));
+  background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: -o-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: linear-gradient(top, #ee5f5b, #bd362f);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ee5f5b', endColorstr='#bd362f', GradientType=0);
+  border-color: #bd362f #bd362f #802420;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+.btn-danger:hover,
+.btn-danger:active,
+.btn-danger.active,
+.btn-danger.disabled,
+.btn-danger[disabled] {
+  background-color: #bd362f;
+}
+.btn-danger:active,
+.btn-danger.active {
+  background-color: #942a25 \9;
+}
+.btn-success {
+  background-color: #5bb75b;
+  background-image: -moz-linear-gradient(top, #62c462, #51a351);
+  background-image: -ms-linear-gradient(top, #62c462, #51a351);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));
+  background-image: -webkit-linear-gradient(top, #62c462, #51a351);
+  background-image: -o-linear-gradient(top, #62c462, #51a351);
+  background-image: linear-gradient(top, #62c462, #51a351);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#62c462', endColorstr='#51a351', GradientType=0);
+  border-color: #51a351 #51a351 #387038;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+.btn-success:hover,
+.btn-success:active,
+.btn-success.active,
+.btn-success.disabled,
+.btn-success[disabled] {
+  background-color: #51a351;
+}
+.btn-success:active,
+.btn-success.active {
+  background-color: #408140 \9;
+}
+.btn-info {
+  background-color: #49afcd;
+  background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: -ms-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));
+  background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: -o-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: linear-gradient(top, #5bc0de, #2f96b4);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#5bc0de', endColorstr='#2f96b4', GradientType=0);
+  border-color: #2f96b4 #2f96b4 #1f6377;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+.btn-info:hover,
+.btn-info:active,
+.btn-info.active,
+.btn-info.disabled,
+.btn-info[disabled] {
+  background-color: #2f96b4;
+}
+.btn-info:active,
+.btn-info.active {
+  background-color: #24748c \9;
+}
+.btn-inverse {
+  background-color: #414141;
+  background-image: -moz-linear-gradient(top, #555555, #222222);
+  background-image: -ms-linear-gradient(top, #555555, #222222);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#555555), to(#222222));
+  background-image: -webkit-linear-gradient(top, #555555, #222222);
+  background-image: -o-linear-gradient(top, #555555, #222222);
+  background-image: linear-gradient(top, #555555, #222222);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#555555', endColorstr='#222222', GradientType=0);
+  border-color: #222222 #222222 #000000;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+.btn-inverse:hover,
+.btn-inverse:active,
+.btn-inverse.active,
+.btn-inverse.disabled,
+.btn-inverse[disabled] {
+  background-color: #222222;
+}
+.btn-inverse:active,
+.btn-inverse.active {
+  background-color: #080808 \9;
+}
+button.btn,
+input[type="submit"].btn {
+  *padding-top: 2px;
+  *padding-bottom: 2px;
+}
+button.btn::-moz-focus-inner,
+input[type="submit"].btn::-moz-focus-inner {
+  padding: 0;
+  border: 0;
+}
+button.btn.btn-large,
+input[type="submit"].btn.btn-large {
+  *padding-top: 7px;
+  *padding-bottom: 7px;
+}
+button.btn.btn-small,
+input[type="submit"].btn.btn-small {
+  *padding-top: 3px;
+  *padding-bottom: 3px;
+}
+button.btn.btn-mini,
+input[type="submit"].btn.btn-mini {
+  *padding-top: 1px;
+  *padding-bottom: 1px;
+}
+.btn-group {
+  position: relative;
+  *zoom: 1;
+  *margin-left: .3em;
+}
+.btn-group:before,
+.btn-group:after {
+  display: table;
+  content: "";
+}
+.btn-group:after {
+  clear: both;
+}
+.btn-group:first-child {
+  *margin-left: 0;
+}
+.btn-group + .btn-group {
+  margin-left: 5px;
+}
+.btn-toolbar {
+  margin-top: 9px;
+  margin-bottom: 9px;
+}
+.btn-toolbar .btn-group {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+}
+.btn-group .btn {
+  position: relative;
+  float: left;
+  margin-left: -1px;
+  -webkit-border-radius: 0;
+  -moz-border-radius: 0;
+  border-radius: 0;
+}
+.btn-group .btn:first-child {
+  margin-left: 0;
+  -webkit-border-top-left-radius: 4px;
+  -moz-border-radius-topleft: 4px;
+  border-top-left-radius: 4px;
+  -webkit-border-bottom-left-radius: 4px;
+  -moz-border-radius-bottomleft: 4px;
+  border-bottom-left-radius: 4px;
+}
+.btn-group .btn:last-child,
+.btn-group .dropdown-toggle {
+  -webkit-border-top-right-radius: 4px;
+  -moz-border-radius-topright: 4px;
+  border-top-right-radius: 4px;
+  -webkit-border-bottom-right-radius: 4px;
+  -moz-border-radius-bottomright: 4px;
+  border-bottom-right-radius: 4px;
+}
+.btn-group .btn.large:first-child {
+  margin-left: 0;
+  -webkit-border-top-left-radius: 6px;
+  -moz-border-radius-topleft: 6px;
+  border-top-left-radius: 6px;
+  -webkit-border-bottom-left-radius: 6px;
+  -moz-border-radius-bottomleft: 6px;
+  border-bottom-left-radius: 6px;
+}
+.btn-group .btn.large:last-child,
+.btn-group .large.dropdown-toggle {
+  -webkit-border-top-right-radius: 6px;
+  -moz-border-radius-topright: 6px;
+  border-top-right-radius: 6px;
+  -webkit-border-bottom-right-radius: 6px;
+  -moz-border-radius-bottomright: 6px;
+  border-bottom-right-radius: 6px;
+}
+.btn-group .btn:hover,
+.btn-group .btn:focus,
+.btn-group .btn:active,
+.btn-group .btn.active {
+  z-index: 2;
+}
+.btn-group .dropdown-toggle:active,
+.btn-group.open .dropdown-toggle {
+  outline: 0;
+}
+.btn-group .dropdown-toggle {
+  padding-left: 8px;
+  padding-right: 8px;
+  -webkit-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  *padding-top: 3px;
+  *padding-bottom: 3px;
+}
+.btn-group .btn-mini.dropdown-toggle {
+  padding-left: 5px;
+  padding-right: 5px;
+  *padding-top: 1px;
+  *padding-bottom: 1px;
+}
+.btn-group .btn-small.dropdown-toggle {
+  *padding-top: 4px;
+  *padding-bottom: 4px;
+}
+.btn-group .btn-large.dropdown-toggle {
+  padding-left: 12px;
+  padding-right: 12px;
+}
+.btn-group.open {
+  *z-index: 1000;
+}
+.btn-group.open .dropdown-menu {
+  display: block;
+  margin-top: 1px;
+  -webkit-border-radius: 5px;
+  -moz-border-radius: 5px;
+  border-radius: 5px;
+}
+.btn-group.open .dropdown-toggle {
+  background-image: none;
+  -webkit-box-shadow: inset 0 1px 6px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: inset 0 1px 6px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+  box-shadow: inset 0 1px 6px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+.btn .caret {
+  margin-top: 7px;
+  margin-left: 0;
+}
+.btn:hover .caret,
+.open.btn-group .caret {
+  opacity: 1;
+  filter: alpha(opacity=100);
+}
+.btn-mini .caret {
+  margin-top: 5px;
+}
+.btn-small .caret {
+  margin-top: 6px;
+}
+.btn-large .caret {
+  margin-top: 6px;
+  border-left: 5px solid transparent;
+  border-right: 5px solid transparent;
+  border-top: 5px solid #000000;
+}
+.btn-primary .caret,
+.btn-warning .caret,
+.btn-danger .caret,
+.btn-info .caret,
+.btn-success .caret,
+.btn-inverse .caret {
+  border-top-color: #ffffff;
+  border-bottom-color: #ffffff;
+  opacity: 0.75;
+  filter: alpha(opacity=75);
+}
+.alert {
+  padding: 8px 35px 8px 14px;
+  margin-bottom: 18px;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+  background-color: #fcf8e3;
+  border: 1px solid #fbeed5;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+  color: #c09853;
+}
+.alert-heading {
+  color: inherit;
+}
+.alert .close {
+  position: relative;
+  top: -2px;
+  right: -21px;
+  line-height: 18px;
+}
+.alert-success {
+  background-color: #dff0d8;
+  border-color: #d6e9c6;
+  color: #468847;
+}
+.alert-danger,
+.alert-error {
+  background-color: #f2dede;
+  border-color: #eed3d7;
+  color: #b94a48;
+}
+.alert-info {
+  background-color: #d9edf7;
+  border-color: #bce8f1;
+  color: #3a87ad;
+}
+.alert-block {
+  padding-top: 14px;
+  padding-bottom: 14px;
+}
+.alert-block > p,
+.alert-block > ul {
+  margin-bottom: 0;
+}
+.alert-block p + p {
+  margin-top: 5px;
+}
+.nav {
+  margin-left: 0;
+  margin-bottom: 18px;
+  list-style: none;
+}
+.nav > li > a {
+  display: block;
+}
+.nav > li > a:hover {
+  text-decoration: none;
+  background-color: #eeeeee;
+}
+.nav .nav-header {
+  display: block;
+  padding: 3px 15px;
+  font-size: 11px;
+  font-weight: bold;
+  line-height: 18px;
+  color: #999999;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+  text-transform: uppercase;
+}
+.nav li + .nav-header {
+  margin-top: 9px;
+}
+.nav-list {
+  padding-left: 15px;
+  padding-right: 15px;
+  margin-bottom: 0;
+}
+.nav-list > li > a,
+.nav-list .nav-header {
+  margin-left: -15px;
+  margin-right: -15px;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+}
+.nav-list > li > a {
+  padding: 3px 15px;
+}
+.nav-list > .active > a,
+.nav-list > .active > a:hover {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
+  background-color: #0088cc;
+}
+.nav-list [class^="icon-"] {
+  margin-right: 2px;
+}
+.nav-list .divider {
+  height: 1px;
+  margin: 8px 1px;
+  overflow: hidden;
+  background-color: #e5e5e5;
+  border-bottom: 1px solid #ffffff;
+  *width: 100%;
+  *margin: -5px 0 5px;
+}
+.nav-tabs,
+.nav-pills {
+  *zoom: 1;
+}
+.nav-tabs:before,
+.nav-pills:before,
+.nav-tabs:after,
+.nav-pills:after {
+  display: table;
+  content: "";
+}
+.nav-tabs:after,
+.nav-pills:after {
+  clear: both;
+}
+.nav-tabs > li,
+.nav-pills > li {
+  float: left;
+}
+.nav-tabs > li > a,
+.nav-pills > li > a {
+  padding-right: 12px;
+  padding-left: 12px;
+  margin-right: 2px;
+  line-height: 14px;
+}
+.nav-tabs {
+  border-bottom: 1px solid #ddd;
+}
+.nav-tabs > li {
+  margin-bottom: -1px;
+}
+.nav-tabs > li > a {
+  padding-top: 8px;
+  padding-bottom: 8px;
+  line-height: 18px;
+  border: 1px solid transparent;
+  -webkit-border-radius: 4px 4px 0 0;
+  -moz-border-radius: 4px 4px 0 0;
+  border-radius: 4px 4px 0 0;
+}
+.nav-tabs > li > a:hover {
+  border-color: #eeeeee #eeeeee #dddddd;
+}
+.nav-tabs > .active > a,
+.nav-tabs > .active > a:hover {
+  color: #555555;
+  background-color: #ffffff;
+  border: 1px solid #ddd;
+  border-bottom-color: transparent;
+  cursor: default;
+}
+.nav-pills > li > a {
+  padding-top: 8px;
+  padding-bottom: 8px;
+  margin-top: 2px;
+  margin-bottom: 2px;
+  -webkit-border-radius: 5px;
+  -moz-border-radius: 5px;
+  border-radius: 5px;
+}
+.nav-pills > .active > a,
+.nav-pills > .active > a:hover {
+  color: #ffffff;
+  background-color: #0088cc;
+}
+.nav-stacked > li {
+  float: none;
+}
+.nav-stacked > li > a {
+  margin-right: 0;
+}
+.nav-tabs.nav-stacked {
+  border-bottom: 0;
+}
+.nav-tabs.nav-stacked > li > a {
+  border: 1px solid #ddd;
+  -webkit-border-radius: 0;
+  -moz-border-radius: 0;
+  border-radius: 0;
+}
+.nav-tabs.nav-stacked > li:first-child > a {
+  -webkit-border-radius: 4px 4px 0 0;
+  -moz-border-radius: 4px 4px 0 0;
+  border-radius: 4px 4px 0 0;
+}
+.nav-tabs.nav-stacked > li:last-child > a {
+  -webkit-border-radius: 0 0 4px 4px;
+  -moz-border-radius: 0 0 4px 4px;
+  border-radius: 0 0 4px 4px;
+}
+.nav-tabs.nav-stacked > li > a:hover {
+  border-color: #ddd;
+  z-index: 2;
+}
+.nav-pills.nav-stacked > li > a {
+  margin-bottom: 3px;
+}
+.nav-pills.nav-stacked > li:last-child > a {
+  margin-bottom: 1px;
+}
+.nav-tabs .dropdown-menu,
+.nav-pills .dropdown-menu {
+  margin-top: 1px;
+  border-width: 1px;
+}
+.nav-pills .dropdown-menu {
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.nav-tabs .dropdown-toggle .caret,
+.nav-pills .dropdown-toggle .caret {
+  border-top-color: #0088cc;
+  border-bottom-color: #0088cc;
+  margin-top: 6px;
+}
+.nav-tabs .dropdown-toggle:hover .caret,
+.nav-pills .dropdown-toggle:hover .caret {
+  border-top-color: #005580;
+  border-bottom-color: #005580;
+}
+.nav-tabs .active .dropdown-toggle .caret,
+.nav-pills .active .dropdown-toggle .caret {
+  border-top-color: #333333;
+  border-bottom-color: #333333;
+}
+.nav > .dropdown.active > a:hover {
+  color: #000000;
+  cursor: pointer;
+}
+.nav-tabs .open .dropdown-toggle,
+.nav-pills .open .dropdown-toggle,
+.nav > .open.active > a:hover {
+  color: #ffffff;
+  background-color: #999999;
+  border-color: #999999;
+}
+.nav .open .caret,
+.nav .open.active .caret,
+.nav .open a:hover .caret {
+  border-top-color: #ffffff;
+  border-bottom-color: #ffffff;
+  opacity: 1;
+  filter: alpha(opacity=100);
+}
+.tabs-stacked .open > a:hover {
+  border-color: #999999;
+}
+.tabbable {
+  *zoom: 1;
+}
+.tabbable:before,
+.tabbable:after {
+  display: table;
+  content: "";
+}
+.tabbable:after {
+  clear: both;
+}
+.tab-content {
+  display: table;
+  width: 100%;
+}
+.tabs-below .nav-tabs,
+.tabs-right .nav-tabs,
+.tabs-left .nav-tabs {
+  border-bottom: 0;
+}
+.tab-content > .tab-pane,
+.pill-content > .pill-pane {
+  display: none;
+}
+.tab-content > .active,
+.pill-content > .active {
+  display: block;
+}
+.tabs-below .nav-tabs {
+  border-top: 1px solid #ddd;
+}
+.tabs-below .nav-tabs > li {
+  margin-top: -1px;
+  margin-bottom: 0;
+}
+.tabs-below .nav-tabs > li > a {
+  -webkit-border-radius: 0 0 4px 4px;
+  -moz-border-radius: 0 0 4px 4px;
+  border-radius: 0 0 4px 4px;
+}
+.tabs-below .nav-tabs > li > a:hover {
+  border-bottom-color: transparent;
+  border-top-color: #ddd;
+}
+.tabs-below .nav-tabs .active > a,
+.tabs-below .nav-tabs .active > a:hover {
+  border-color: transparent #ddd #ddd #ddd;
+}
+.tabs-left .nav-tabs > li,
+.tabs-right .nav-tabs > li {
+  float: none;
+}
+.tabs-left .nav-tabs > li > a,
+.tabs-right .nav-tabs > li > a {
+  min-width: 74px;
+  margin-right: 0;
+  margin-bottom: 3px;
+}
+.tabs-left .nav-tabs {
+  float: left;
+  margin-right: 19px;
+  border-right: 1px solid #ddd;
+}
+.tabs-left .nav-tabs > li > a {
+  margin-right: -1px;
+  -webkit-border-radius: 4px 0 0 4px;
+  -moz-border-radius: 4px 0 0 4px;
+  border-radius: 4px 0 0 4px;
+}
+.tabs-left .nav-tabs > li > a:hover {
+  border-color: #eeeeee #dddddd #eeeeee #eeeeee;
+}
+.tabs-left .nav-tabs .active > a,
+.tabs-left .nav-tabs .active > a:hover {
+  border-color: #ddd transparent #ddd #ddd;
+  *border-right-color: #ffffff;
+}
+.tabs-right .nav-tabs {
+  float: right;
+  margin-left: 19px;
+  border-left: 1px solid #ddd;
+}
+.tabs-right .nav-tabs > li > a {
+  margin-left: -1px;
+  -webkit-border-radius: 0 4px 4px 0;
+  -moz-border-radius: 0 4px 4px 0;
+  border-radius: 0 4px 4px 0;
+}
+.tabs-right .nav-tabs > li > a:hover {
+  border-color: #eeeeee #eeeeee #eeeeee #dddddd;
+}
+.tabs-right .nav-tabs .active > a,
+.tabs-right .nav-tabs .active > a:hover {
+  border-color: #ddd #ddd #ddd transparent;
+  *border-left-color: #ffffff;
+}
+.navbar {
+  *position: relative;
+  *z-index: 2;
+  overflow: visible;
+  margin-bottom: 18px;
+}
+.navbar-inner {
+  padding-left: 20px;
+  padding-right: 20px;
+  background-color: #2c2c2c;
+  background-image: -moz-linear-gradient(top, #333333, #222222);
+  background-image: -ms-linear-gradient(top, #333333, #222222);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#333333), to(#222222));
+  background-image: -webkit-linear-gradient(top, #333333, #222222);
+  background-image: -o-linear-gradient(top, #333333, #222222);
+  background-image: linear-gradient(top, #333333, #222222);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#333333', endColorstr='#222222', GradientType=0);
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+  -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
+  -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
+  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
+}
+.navbar .container {
+  width: auto;
+}
+.btn-navbar {
+  display: none;
+  float: right;
+  padding: 7px 10px;
+  margin-left: 5px;
+  margin-right: 5px;
+  background-color: #2c2c2c;
+  background-image: -moz-linear-gradient(top, #333333, #222222);
+  background-image: -ms-linear-gradient(top, #333333, #222222);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#333333), to(#222222));
+  background-image: -webkit-linear-gradient(top, #333333, #222222);
+  background-image: -o-linear-gradient(top, #333333, #222222);
+  background-image: linear-gradient(top, #333333, #222222);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#333333', endColorstr='#222222', GradientType=0);
+  border-color: #222222 #222222 #000000;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+  -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+}
+.btn-navbar:hover,
+.btn-navbar:active,
+.btn-navbar.active,
+.btn-navbar.disabled,
+.btn-navbar[disabled] {
+  background-color: #222222;
+}
+.btn-navbar:active,
+.btn-navbar.active {
+  background-color: #080808 \9;
+}
+.btn-navbar .icon-bar {
+  display: block;
+  width: 18px;
+  height: 2px;
+  background-color: #f5f5f5;
+  -webkit-border-radius: 1px;
+  -moz-border-radius: 1px;
+  border-radius: 1px;
+  -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+  -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+  box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+}
+.btn-navbar .icon-bar + .icon-bar {
+  margin-top: 3px;
+}
+.nav-collapse.collapse {
+  height: auto;
+}
+.navbar {
+  color: #999999;
+}
+.navbar .brand:hover {
+  text-decoration: none;
+}
+.navbar .brand {
+  float: left;
+  display: block;
+  padding: 8px 20px 12px;
+  margin-left: -20px;
+  font-size: 20px;
+  font-weight: 200;
+  line-height: 1;
+  color: #ffffff;
+}
+.navbar .navbar-text {
+  margin-bottom: 0;
+  line-height: 40px;
+}
+.navbar .btn,
+.navbar .btn-group {
+  margin-top: 5px;
+}
+.navbar .btn-group .btn {
+  margin-top: 0;
+}
+.navbar-form {
+  margin-bottom: 0;
+  *zoom: 1;
+}
+.navbar-form:before,
+.navbar-form:after {
+  display: table;
+  content: "";
+}
+.navbar-form:after {
+  clear: both;
+}
+.navbar-form input,
+.navbar-form select,
+.navbar-form .radio,
+.navbar-form .checkbox {
+  margin-top: 5px;
+}
+.navbar-form input,
+.navbar-form select {
+  display: inline-block;
+  margin-bottom: 0;
+}
+.navbar-form input[type="image"],
+.navbar-form input[type="checkbox"],
+.navbar-form input[type="radio"] {
+  margin-top: 3px;
+}
+.navbar-form .input-append,
+.navbar-form .input-prepend {
+  margin-top: 6px;
+  white-space: nowrap;
+}
+.navbar-form .input-append input,
+.navbar-form .input-prepend input {
+  margin-top: 0;
+}
+.navbar-search {
+  position: relative;
+  float: left;
+  margin-top: 6px;
+  margin-bottom: 0;
+}
+.navbar-search .search-query {
+  padding: 4px 9px;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  font-size: 13px;
+  font-weight: normal;
+  line-height: 1;
+  color: #ffffff;
+  background-color: #626262;
+  border: 1px solid #151515;
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0px rgba(255, 255, 255, 0.15);
+  -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0px rgba(255, 255, 255, 0.15);
+  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0px rgba(255, 255, 255, 0.15);
+  -webkit-transition: none;
+  -moz-transition: none;
+  -ms-transition: none;
+  -o-transition: none;
+  transition: none;
+}
+.navbar-search .search-query:-moz-placeholder {
+  color: #cccccc;
+}
+.navbar-search .search-query::-webkit-input-placeholder {
+  color: #cccccc;
+}
+.navbar-search .search-query:focus,
+.navbar-search .search-query.focused {
+  padding: 5px 10px;
+  color: #333333;
+  text-shadow: 0 1px 0 #ffffff;
+  background-color: #ffffff;
+  border: 0;
+  -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+  -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+  box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+  outline: 0;
+}
+.navbar-fixed-top,
+.navbar-fixed-bottom {
+  position: fixed;
+  right: 0;
+  left: 0;
+  z-index: 1030;
+  margin-bottom: 0;
+}
+.navbar-fixed-top .navbar-inner,
+.navbar-fixed-bottom .navbar-inner {
+  padding-left: 0;
+  padding-right: 0;
+  -webkit-border-radius: 0;
+  -moz-border-radius: 0;
+  border-radius: 0;
+}
+.navbar-fixed-top .container,
+.navbar-fixed-bottom .container {
+  width: 940px;
+}
+.navbar-fixed-top {
+  top: 0;
+}
+.navbar-fixed-bottom {
+  bottom: 0;
+}
+.navbar .nav {
+  position: relative;
+  left: 0;
+  display: block;
+  float: left;
+  margin: 0 10px 0 0;
+}
+.navbar .nav.pull-right {
+  float: right;
+}
+.navbar .nav > li {
+  display: block;
+  float: left;
+}
+.navbar .nav > li > a {
+  float: none;
+  padding: 10px 10px 11px;
+  line-height: 19px;
+  color: #999999;
+  text-decoration: none;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+}
+.navbar .nav > li > a:hover {
+  background-color: transparent;
+  color: #ffffff;
+  text-decoration: none;
+}
+.navbar .nav .active > a,
+.navbar .nav .active > a:hover {
+  color: #ffffff;
+  text-decoration: none;
+  background-color: #222222;
+}
+.navbar .divider-vertical {
+  height: 40px;
+  width: 1px;
+  margin: 0 9px;
+  overflow: hidden;
+  background-color: #222222;
+  border-right: 1px solid #333333;
+}
+.navbar .nav.pull-right {
+  margin-left: 10px;
+  margin-right: 0;
+}
+.navbar .dropdown-menu {
+  margin-top: 1px;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.navbar .dropdown-menu:before {
+  content: '';
+  display: inline-block;
+  border-left: 7px solid transparent;
+  border-right: 7px solid transparent;
+  border-bottom: 7px solid #ccc;
+  border-bottom-color: rgba(0, 0, 0, 0.2);
+  position: absolute;
+  top: -7px;
+  left: 9px;
+}
+.navbar .dropdown-menu:after {
+  content: '';
+  display: inline-block;
+  border-left: 6px solid transparent;
+  border-right: 6px solid transparent;
+  border-bottom: 6px solid #ffffff;
+  position: absolute;
+  top: -6px;
+  left: 10px;
+}
+.navbar-fixed-bottom .dropdown-menu:before {
+  border-top: 7px solid #ccc;
+  border-top-color: rgba(0, 0, 0, 0.2);
+  border-bottom: 0;
+  bottom: -7px;
+  top: auto;
+}
+.navbar-fixed-bottom .dropdown-menu:after {
+  border-top: 6px solid #ffffff;
+  border-bottom: 0;
+  bottom: -6px;
+  top: auto;
+}
+.navbar .nav .dropdown-toggle .caret,
+.navbar .nav .open.dropdown .caret {
+  border-top-color: #ffffff;
+  border-bottom-color: #ffffff;
+}
+.navbar .nav .active .caret {
+  opacity: 1;
+  filter: alpha(opacity=100);
+}
+.navbar .nav .open > .dropdown-toggle,
+.navbar .nav .active > .dropdown-toggle,
+.navbar .nav .open.active > .dropdown-toggle {
+  background-color: transparent;
+}
+.navbar .nav .active > .dropdown-toggle:hover {
+  color: #ffffff;
+}
+.navbar .nav.pull-right .dropdown-menu,
+.navbar .nav .dropdown-menu.pull-right {
+  left: auto;
+  right: 0;
+}
+.navbar .nav.pull-right .dropdown-menu:before,
+.navbar .nav .dropdown-menu.pull-right:before {
+  left: auto;
+  right: 12px;
+}
+.navbar .nav.pull-right .dropdown-menu:after,
+.navbar .nav .dropdown-menu.pull-right:after {
+  left: auto;
+  right: 13px;
+}
+.breadcrumb {
+  padding: 7px 14px;
+  margin: 0 0 18px;
+  list-style: none;
+  background-color: #fbfbfb;
+  background-image: -moz-linear-gradient(top, #ffffff, #f5f5f5);
+  background-image: -ms-linear-gradient(top, #ffffff, #f5f5f5);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f5f5f5));
+  background-image: -webkit-linear-gradient(top, #ffffff, #f5f5f5);
+  background-image: -o-linear-gradient(top, #ffffff, #f5f5f5);
+  background-image: linear-gradient(top, #ffffff, #f5f5f5);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#f5f5f5', GradientType=0);
+  border: 1px solid #ddd;
+  -webkit-border-radius: 3px;
+  -moz-border-radius: 3px;
+  border-radius: 3px;
+  -webkit-box-shadow: inset 0 1px 0 #ffffff;
+  -moz-box-shadow: inset 0 1px 0 #ffffff;
+  box-shadow: inset 0 1px 0 #ffffff;
+}
+.breadcrumb li {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+  text-shadow: 0 1px 0 #ffffff;
+}
+.breadcrumb .divider {
+  padding: 0 5px;
+  color: #999999;
+}
+.breadcrumb .active a {
+  color: #333333;
+}
+.pagination {
+  height: 36px;
+  margin: 18px 0;
+}
+.pagination ul {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+  margin-left: 0;
+  margin-bottom: 0;
+  -webkit-border-radius: 3px;
+  -moz-border-radius: 3px;
+  border-radius: 3px;
+  -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+  box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+.pagination li {
+  display: inline;
+}
+.pagination a {
+  float: left;
+  padding: 0 14px;
+  line-height: 34px;
+  text-decoration: none;
+  border: 1px solid #ddd;
+  border-left-width: 0;
+}
+.pagination a:hover,
+.pagination .active a {
+  background-color: #f5f5f5;
+}
+.pagination .active a {
+  color: #999999;
+  cursor: default;
+}
+.pagination .disabled span,
+.pagination .disabled a,
+.pagination .disabled a:hover {
+  color: #999999;
+  background-color: transparent;
+  cursor: default;
+}
+.pagination li:first-child a {
+  border-left-width: 1px;
+  -webkit-border-radius: 3px 0 0 3px;
+  -moz-border-radius: 3px 0 0 3px;
+  border-radius: 3px 0 0 3px;
+}
+.pagination li:last-child a {
+  -webkit-border-radius: 0 3px 3px 0;
+  -moz-border-radius: 0 3px 3px 0;
+  border-radius: 0 3px 3px 0;
+}
+.pagination-centered {
+  text-align: center;
+}
+.pagination-right {
+  text-align: right;
+}
+.pager {
+  margin-left: 0;
+  margin-bottom: 18px;
+  list-style: none;
+  text-align: center;
+  *zoom: 1;
+}
+.pager:before,
+.pager:after {
+  display: table;
+  content: "";
+}
+.pager:after {
+  clear: both;
+}
+.pager li {
+  display: inline;
+}
+.pager a {
+  display: inline-block;
+  padding: 5px 14px;
+  background-color: #fff;
+  border: 1px solid #ddd;
+  -webkit-border-radius: 15px;
+  -moz-border-radius: 15px;
+  border-radius: 15px;
+}
+.pager a:hover {
+  text-decoration: none;
+  background-color: #f5f5f5;
+}
+.pager .next a {
+  float: right;
+}
+.pager .previous a {
+  float: left;
+}
+.pager .disabled a,
+.pager .disabled a:hover {
+  color: #999999;
+  background-color: #fff;
+  cursor: default;
+}
+.modal-open .dropdown-menu {
+  z-index: 2050;
+}
+.modal-open .dropdown.open {
+  *z-index: 2050;
+}
+.modal-open .popover {
+  z-index: 2060;
+}
+.modal-open .tooltip {
+  z-index: 2070;
+}
+.modal-backdrop {
+  position: fixed;
+  top: 0;
+  right: 0;
+  bottom: 0;
+  left: 0;
+  z-index: 1040;
+  background-color: #000000;
+}
+.modal-backdrop.fade {
+  opacity: 0;
+}
+.modal-backdrop,
+.modal-backdrop.fade.in {
+  opacity: 0.8;
+  filter: alpha(opacity=80);
+}
+.modal {
+  position: fixed;
+  top: 50%;
+  left: 50%;
+  z-index: 1050;
+  overflow: auto;
+  width: 560px;
+  margin: -250px 0 0 -280px;
+  background-color: #ffffff;
+  border: 1px solid #999;
+  border: 1px solid rgba(0, 0, 0, 0.3);
+  *border: 1px solid #999;
+  /* IE6-7 */
+
+  -webkit-border-radius: 6px;
+  -moz-border-radius: 6px;
+  border-radius: 6px;
+  -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+  -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+  box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+  -webkit-background-clip: padding-box;
+  -moz-background-clip: padding-box;
+  background-clip: padding-box;
+}
+.modal.fade {
+  -webkit-transition: opacity .3s linear, top .3s ease-out;
+  -moz-transition: opacity .3s linear, top .3s ease-out;
+  -ms-transition: opacity .3s linear, top .3s ease-out;
+  -o-transition: opacity .3s linear, top .3s ease-out;
+  transition: opacity .3s linear, top .3s ease-out;
+  top: -25%;
+}
+.modal.fade.in {
+  top: 50%;
+}
+.modal-header {
+  padding: 9px 15px;
+  border-bottom: 1px solid #eee;
+}
+.modal-header .close {
+  margin-top: 2px;
+}
+.modal-body {
+  overflow-y: auto;
+  max-height: 400px;
+  padding: 15px;
+}
+.modal-form {
+  margin-bottom: 0;
+}
+.modal-footer {
+  padding: 14px 15px 15px;
+  margin-bottom: 0;
+  text-align: right;
+  background-color: #f5f5f5;
+  border-top: 1px solid #ddd;
+  -webkit-border-radius: 0 0 6px 6px;
+  -moz-border-radius: 0 0 6px 6px;
+  border-radius: 0 0 6px 6px;
+  -webkit-box-shadow: inset 0 1px 0 #ffffff;
+  -moz-box-shadow: inset 0 1px 0 #ffffff;
+  box-shadow: inset 0 1px 0 #ffffff;
+  *zoom: 1;
+}
+.modal-footer:before,
+.modal-footer:after {
+  display: table;
+  content: "";
+}
+.modal-footer:after {
+  clear: both;
+}
+.modal-footer .btn + .btn {
+  margin-left: 5px;
+  margin-bottom: 0;
+}
+.modal-footer .btn-group .btn + .btn {
+  margin-left: -1px;
+}
+.tooltip {
+  position: absolute;
+  z-index: 1020;
+  display: block;
+  visibility: visible;
+  padding: 5px;
+  font-size: 11px;
+  opacity: 0;
+  filter: alpha(opacity=0);
+}
+.tooltip.in {
+  opacity: 0.8;
+  filter: alpha(opacity=80);
+}
+.tooltip.top {
+  margin-top: -2px;
+}
+.tooltip.right {
+  margin-left: 2px;
+}
+.tooltip.bottom {
+  margin-top: 2px;
+}
+.tooltip.left {
+  margin-left: -2px;
+}
+.tooltip.top .tooltip-arrow {
+  bottom: 0;
+  left: 50%;
+  margin-left: -5px;
+  border-left: 5px solid transparent;
+  border-right: 5px solid transparent;
+  border-top: 5px solid #000000;
+}
+.tooltip.left .tooltip-arrow {
+  top: 50%;
+  right: 0;
+  margin-top: -5px;
+  border-top: 5px solid transparent;
+  border-bottom: 5px solid transparent;
+  border-left: 5px solid #000000;
+}
+.tooltip.bottom .tooltip-arrow {
+  top: 0;
+  left: 50%;
+  margin-left: -5px;
+  border-left: 5px solid transparent;
+  border-right: 5px solid transparent;
+  border-bottom: 5px solid #000000;
+}
+.tooltip.right .tooltip-arrow {
+  top: 50%;
+  left: 0;
+  margin-top: -5px;
+  border-top: 5px solid transparent;
+  border-bottom: 5px solid transparent;
+  border-right: 5px solid #000000;
+}
+.tooltip-inner {
+  max-width: 200px;
+  padding: 3px 8px;
+  color: #ffffff;
+  text-align: center;
+  text-decoration: none;
+  background-color: #000000;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.tooltip-arrow {
+  position: absolute;
+  width: 0;
+  height: 0;
+}
+.popover {
+  position: absolute;
+  top: 0;
+  left: 0;
+  z-index: 1010;
+  display: none;
+  padding: 5px;
+}
+.popover.top {
+  margin-top: -5px;
+}
+.popover.right {
+  margin-left: 5px;
+}
+.popover.bottom {
+  margin-top: 5px;
+}
+.popover.left {
+  margin-left: -5px;
+}
+.popover.top .arrow {
+  bottom: 0;
+  left: 50%;
+  margin-left: -5px;
+  border-left: 5px solid transparent;
+  border-right: 5px solid transparent;
+  border-top: 5px solid #000000;
+}
+.popover.right .arrow {
+  top: 50%;
+  left: 0;
+  margin-top: -5px;
+  border-top: 5px solid transparent;
+  border-bottom: 5px solid transparent;
+  border-right: 5px solid #000000;
+}
+.popover.bottom .arrow {
+  top: 0;
+  left: 50%;
+  margin-left: -5px;
+  border-left: 5px solid transparent;
+  border-right: 5px solid transparent;
+  border-bottom: 5px solid #000000;
+}
+.popover.left .arrow {
+  top: 50%;
+  right: 0;
+  margin-top: -5px;
+  border-top: 5px solid transparent;
+  border-bottom: 5px solid transparent;
+  border-left: 5px solid #000000;
+}
+.popover .arrow {
+  position: absolute;
+  width: 0;
+  height: 0;
+}
+.popover-inner {
+  padding: 3px;
+  width: 280px;
+  overflow: hidden;
+  background: #000000;
+  background: rgba(0, 0, 0, 0.8);
+  -webkit-border-radius: 6px;
+  -moz-border-radius: 6px;
+  border-radius: 6px;
+  -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+  -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+  box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+}
+.popover-title {
+  padding: 9px 15px;
+  line-height: 1;
+  background-color: #f5f5f5;
+  border-bottom: 1px solid #eee;
+  -webkit-border-radius: 3px 3px 0 0;
+  -moz-border-radius: 3px 3px 0 0;
+  border-radius: 3px 3px 0 0;
+}
+.popover-content {
+  padding: 14px;
+  background-color: #ffffff;
+  -webkit-border-radius: 0 0 3px 3px;
+  -moz-border-radius: 0 0 3px 3px;
+  border-radius: 0 0 3px 3px;
+  -webkit-background-clip: padding-box;
+  -moz-background-clip: padding-box;
+  background-clip: padding-box;
+}
+.popover-content p,
+.popover-content ul,
+.popover-content ol {
+  margin-bottom: 0;
+}
+.thumbnails {
+  margin-left: -20px;
+  list-style: none;
+  *zoom: 1;
+}
+.thumbnails:before,
+.thumbnails:after {
+  display: table;
+  content: "";
+}
+.thumbnails:after {
+  clear: both;
+}
+.thumbnails > li {
+  float: left;
+  margin: 0 0 18px 20px;
+}
+.thumbnail {
+  display: block;
+  padding: 4px;
+  line-height: 1;
+  border: 1px solid #ddd;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+  -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
+  -moz-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
+  box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+a.thumbnail:hover {
+  border-color: #0088cc;
+  -webkit-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+  -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+  box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+}
+.thumbnail > img {
+  display: block;
+  max-width: 100%;
+  margin-left: auto;
+  margin-right: auto;
+}
+.thumbnail .caption {
+  padding: 9px;
+}
+.label {
+  padding: 1px 4px 2px;
+  font-size: 10.998px;
+  font-weight: bold;
+  line-height: 13px;
+  color: #ffffff;
+  vertical-align: middle;
+  white-space: nowrap;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #999999;
+  -webkit-border-radius: 3px;
+  -moz-border-radius: 3px;
+  border-radius: 3px;
+}
+.label:hover {
+  color: #ffffff;
+  text-decoration: none;
+}
+.label-important {
+  background-color: #b94a48;
+}
+.label-important:hover {
+  background-color: #953b39;
+}
+.label-warning {
+  background-color: #f89406;
+}
+.label-warning:hover {
+  background-color: #c67605;
+}
+.label-success {
+  background-color: #468847;
+}
+.label-success:hover {
+  background-color: #356635;
+}
+.label-info {
+  background-color: #3a87ad;
+}
+.label-info:hover {
+  background-color: #2d6987;
+}
+.label-inverse {
+  background-color: #333333;
+}
+.label-inverse:hover {
+  background-color: #1a1a1a;
+}
+.badge {
+  padding: 1px 9px 2px;
+  font-size: 12.025px;
+  font-weight: bold;
+  white-space: nowrap;
+  color: #ffffff;
+  background-color: #999999;
+  -webkit-border-radius: 9px;
+  -moz-border-radius: 9px;
+  border-radius: 9px;
+}
+.badge:hover {
+  color: #ffffff;
+  text-decoration: none;
+  cursor: pointer;
+}
+.badge-error {
+  background-color: #b94a48;
+}
+.badge-error:hover {
+  background-color: #953b39;
+}
+.badge-warning {
+  background-color: #f89406;
+}
+.badge-warning:hover {
+  background-color: #c67605;
+}
+.badge-success {
+  background-color: #468847;
+}
+.badge-success:hover {
+  background-color: #356635;
+}
+.badge-info {
+  background-color: #3a87ad;
+}
+.badge-info:hover {
+  background-color: #2d6987;
+}
+.badge-inverse {
+  background-color: #333333;
+}
+.badge-inverse:hover {
+  background-color: #1a1a1a;
+}
+@-webkit-keyframes progress-bar-stripes {
+  from {
+    background-position: 0 0;
+  }
+  to {
+    background-position: 40px 0;
+  }
+}
+@-moz-keyframes progress-bar-stripes {
+  from {
+    background-position: 0 0;
+  }
+  to {
+    background-position: 40px 0;
+  }
+}
+@-ms-keyframes progress-bar-stripes {
+  from {
+    background-position: 0 0;
+  }
+  to {
+    background-position: 40px 0;
+  }
+}
+@keyframes progress-bar-stripes {
+  from {
+    background-position: 0 0;
+  }
+  to {
+    background-position: 40px 0;
+  }
+}
+.progress {
+  overflow: hidden;
+  height: 18px;
+  margin-bottom: 18px;
+  background-color: #f7f7f7;
+  background-image: -moz-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: -ms-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f5f5f5), to(#f9f9f9));
+  background-image: -webkit-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: -o-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#f5f5f5', endColorstr='#f9f9f9', GradientType=0);
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+  -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.progress .bar {
+  width: 0%;
+  height: 18px;
+  color: #ffffff;
+  font-size: 12px;
+  text-align: center;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #0e90d2;
+  background-image: -moz-linear-gradient(top, #149bdf, #0480be);
+  background-image: -ms-linear-gradient(top, #149bdf, #0480be);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be));
+  background-image: -webkit-linear-gradient(top, #149bdf, #0480be);
+  background-image: -o-linear-gradient(top, #149bdf, #0480be);
+  background-image: linear-gradient(top, #149bdf, #0480be);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#149bdf', endColorstr='#0480be', GradientType=0);
+  -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+  -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+  box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+  -webkit-box-sizing: border-box;
+  -moz-box-sizing: border-box;
+  -ms-box-sizing: border-box;
+  box-sizing: border-box;
+  -webkit-transition: width 0.6s ease;
+  -moz-transition: width 0.6s ease;
+  -ms-transition: width 0.6s ease;
+  -o-transition: width 0.6s ease;
+  transition: width 0.6s ease;
+}
+.progress-striped .bar {
+  background-color: #149bdf;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -ms-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  -webkit-background-size: 40px 40px;
+  -moz-background-size: 40px 40px;
+  -o-background-size: 40px 40px;
+  background-size: 40px 40px;
+}
+.progress.active .bar {
+  -webkit-animation: progress-bar-stripes 2s linear infinite;
+  -moz-animation: progress-bar-stripes 2s linear infinite;
+  animation: progress-bar-stripes 2s linear infinite;
+}
+.progress-danger .bar {
+  background-color: #dd514c;
+  background-image: -moz-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: -ms-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#c43c35));
+  background-image: -webkit-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: -o-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: linear-gradient(top, #ee5f5b, #c43c35);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ee5f5b', endColorstr='#c43c35', GradientType=0);
+}
+.progress-danger.progress-striped .bar {
+  background-color: #ee5f5b;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -ms-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+.progress-success .bar {
+  background-color: #5eb95e;
+  background-image: -moz-linear-gradient(top, #62c462, #57a957);
+  background-image: -ms-linear-gradient(top, #62c462, #57a957);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#57a957));
+  background-image: -webkit-linear-gradient(top, #62c462, #57a957);
+  background-image: -o-linear-gradient(top, #62c462, #57a957);
+  background-image: linear-gradient(top, #62c462, #57a957);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#62c462', endColorstr='#57a957', GradientType=0);
+}
+.progress-success.progress-striped .bar {
+  background-color: #62c462;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -ms-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+.progress-info .bar {
+  background-color: #4bb1cf;
+  background-image: -moz-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: -ms-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#339bb9));
+  background-image: -webkit-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: -o-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: linear-gradient(top, #5bc0de, #339bb9);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#5bc0de', endColorstr='#339bb9', GradientType=0);
+}
+.progress-info.progress-striped .bar {
+  background-color: #5bc0de;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -ms-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+.progress-warning .bar {
+  background-color: #faa732;
+  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
+  background-image: -ms-linear-gradient(top, #fbb450, #f89406);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
+  background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
+  background-image: -o-linear-gradient(top, #fbb450, #f89406);
+  background-image: linear-gradient(top, #fbb450, #f89406);
+  background-repeat: repeat-x;
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fbb450', endColorstr='#f89406', GradientType=0);
+}
+.progress-warning.progress-striped .bar {
+  background-color: #fbb450;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -ms-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+.accordion {
+  margin-bottom: 18px;
+}
+.accordion-group {
+  margin-bottom: 2px;
+  border: 1px solid #e5e5e5;
+  -webkit-border-radius: 4px;
+  -moz-border-radius: 4px;
+  border-radius: 4px;
+}
+.accordion-heading {
+  border-bottom: 0;
+}
+.accordion-heading .accordion-toggle {
+  display: block;
+  padding: 8px 15px;
+}
+.accordion-inner {
+  padding: 9px 15px;
+  border-top: 1px solid #e5e5e5;
+}
+.carousel {
+  position: relative;
+  margin-bottom: 18px;
+  line-height: 1;
+}
+.carousel-inner {
+  overflow: hidden;
+  width: 100%;
+  position: relative;
+}
+.carousel .item {
+  display: none;
+  position: relative;
+  -webkit-transition: 0.6s ease-in-out left;
+  -moz-transition: 0.6s ease-in-out left;
+  -ms-transition: 0.6s ease-in-out left;
+  -o-transition: 0.6s ease-in-out left;
+  transition: 0.6s ease-in-out left;
+}
+.carousel .item > img {
+  display: block;
+  line-height: 1;
+}
+.carousel .active,
+.carousel .next,
+.carousel .prev {
+  display: block;
+}
+.carousel .active {
+  left: 0;
+}
+.carousel .next,
+.carousel .prev {
+  position: absolute;
+  top: 0;
+  width: 100%;
+}
+.carousel .next {
+  left: 100%;
+}
+.carousel .prev {
+  left: -100%;
+}
+.carousel .next.left,
+.carousel .prev.right {
+  left: 0;
+}
+.carousel .active.left {
+  left: -100%;
+}
+.carousel .active.right {
+  left: 100%;
+}
+.carousel-control {
+  position: absolute;
+  top: 40%;
+  left: 15px;
+  width: 40px;
+  height: 40px;
+  margin-top: -20px;
+  font-size: 60px;
+  font-weight: 100;
+  line-height: 30px;
+  color: #ffffff;
+  text-align: center;
+  background: #222222;
+  border: 3px solid #ffffff;
+  -webkit-border-radius: 23px;
+  -moz-border-radius: 23px;
+  border-radius: 23px;
+  opacity: 0.5;
+  filter: alpha(opacity=50);
+}
+.carousel-control.right {
+  left: auto;
+  right: 15px;
+}
+.carousel-control:hover {
+  color: #ffffff;
+  text-decoration: none;
+  opacity: 0.9;
+  filter: alpha(opacity=90);
+}
+.carousel-caption {
+  position: absolute;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  padding: 10px 15px 5px;
+  background: #333333;
+  background: rgba(0, 0, 0, 0.75);
+}
+.carousel-caption h4,
+.carousel-caption p {
+  color: #ffffff;
+}
+.hero-unit {
+  padding: 60px;
+  margin-bottom: 30px;
+  background-color: #eeeeee;
+  -webkit-border-radius: 6px;
+  -moz-border-radius: 6px;
+  border-radius: 6px;
+}
+.hero-unit h1 {
+  margin-bottom: 0;
+  font-size: 60px;
+  line-height: 1;
+  color: inherit;
+  letter-spacing: -1px;
+}
+.hero-unit p {
+  font-size: 18px;
+  font-weight: 200;
+  line-height: 27px;
+  color: inherit;
+}
+.pull-right {
+  float: right;
+}
+.pull-left {
+  float: left;
+}
+.hide {
+  display: none;
+}
+.show {
+  display: block;
+}
+.invisible {
+  visibility: hidden;
+}
diff --git a/static/glyphicons-halflings-white.png b/static/glyphicons-halflings-white.png
new file mode 100644
index 0000000000000000000000000000000000000000..a20760bfde58d1c92cee95116059fba03c68d689
GIT binary patch
literal 4352
zcmd6r_dnEu|G?izMxtxU%uI5!l8nr<BF?zWUS(u;&WdwZC0F)1B-!J<$%*WB$U3Xi
z$ta3LaXK6#>)ZF&&*%FGe4jtO*5mbhJzhV&et11z&&^B?xH$MZ007{+ZK!Jj01(PQ
zJBFS4pH$0DefCd1HM@h*JNkcsi%oOXzj>qsEle$eQ7ApHL(XYdn5Y$Lk_3-J9p9d)
zFeVfl3J47_g1XaoDXWsnBp9ZzZ74CI9RN-Nw{>+8A&#rBpZgc9WX2H3Ssv6doZP?t
zS!g}lGvW1<9%?dj_G_x}3WUMN(8(x{a6_pd0yiUsf^67GGS50uSB*ORe5x6}qAf1z
z@Q;2y4G{Lb?f21p)uTpChN&4q%^blZ2IsusUOhk)pe0<chGtjyTP-b6%vl?4F2xqG
zOU>yxPD6oHKXWSj<y;3B&r^tK>v8&2pMdnegiQUtoXt1U0MmWAWu2&>3j$eb^qKNV
z_(`JQZP&mXLT@U%-2rPy!7r|*Y1oAdlarltaUyq+yq^|d{B9_>t@Rd#@_KW9w_6P$
z^Dv8(Hi8pDJK{r0Iqq*va$cL=isZh0=1)wIoQ^vYPs$<T2#x2Kj^?$few0Pe4I~zZ
zeAYbg0c0)2OtIx}d)C`Mw&~<64nQ!Uk8$^SW6e!?j1HfU4$&%i_`y~2R>(rBz$+DY
z`y}1}`M%-da686<lVV-dk8h2*Tn8V7;-njKI(p4zUJy$ofY$z#INdRf(>`}zw_w>8
z!BcqxVTim*F)-}$segV$ON*!Zl~dhX@Rz^K2Xu<c1P8u4bp<yQO?OQj^dKZcE}xh_
z<z&gNJz{ZTTu3nGIcR;qG9;?^M0kG|PuThGH1+;j!xXDN6I_*@xL=@r$xRBuVh{MN
zIUGEgxYJ(DFHKoLGF3_xPSW_^TT*1w(&gCNFdnv^AMnNFK6+ia>rh<1-vjImult%O
z!-WXvkA_agVuhluW};J;#r>)?^uHS;G?a?j;(z?Y^FTwOA?tzLFvQDf&X8}9s7Wh<
znEfd_vPyF_V`?>kR`w_h@+%59oKa;NPVGUo52QjisO-|$cYE(VNmm#+`#T5a;gh|Z
z8A0^l3UwQMn0J3x<h`4-5?ApmemDp`8K)X6T0efPN*-~cf<tL>XWL7tY~Ox<iRkdJ
zU|072zio5s?pAI0%Yx0uJh1f5i7VKWaFIaB;45=yji!1nH9<de2OLj_y{&41?nyPO
zUrZT8xW#w*TQ5)($;JeSp2Pgrams&!r<Pe}#(LDg-blL{ESlmQ?a5Th4_;WRJR+4E
zw6tQreDz+4bser4GB#?<roQ`hsw<hwcyHa9dkP0IO=6)WWkTxg{$NTm-b*c?j2_ul
zyuRy=77P?tF`%S2aa=XEJa>Au=_hGvp@_%SZKA)ec-h-dfwIhS3jGBLL6e6Os;1LR
zRDG&3TF`HV*n{&*H!oTSsLq!U5xV5!Yr6I_!*VhmwC3a2BOYfWH13AtVY|n5jv49e
zcb0xCCZnt0i$>-S$k9J@-c!8wG#siu(Lg<MtkAtqhD8bV`jR^%b&>y_r1nfy+}!<h
zAF+SdUhcuD`9zF%pRIHymB_I~)P%%~M=eQ#Ic#<Zr+NPzGTI`9;4khM^2h2PqMd?5
zGH>W9g-ucwp=&Hs1=Vs4i_q;dQL$8~Uq2BVA4o4uY!6}S`xH(Qec+{mJD~qgg@6W8
zipi@Z!ZR+Kr_)u&G);pG$tg$8#KPrsl&N3(m($NAU&9ogH9rVfW<4Mw>^7$&96g<9
zHQzekG9T5SS7DVm7EFY%CjChhfRyap4+d;+^0ng^B)~xKFG^7d2oOo|R8uY&S|X0@
znAGMb^rFQwGPTzsFQ8ZK4S@WO(8`6T+$Yt9{jGMd?jrTeb|_!Un`n9xDZu-fW+_aJ
z4Uyy_$)`Ot!~doWUHW`(?F!iYvc5+g-(W9X<-tX*h%6(f;+A(OQ@w{WYSiq&pjKnN
z)tSH~5g)03sKk)U+&GyP*?86fusX1ttpH1ng8ruC6UOddM~t>0wvZh}1cW%&7{tT$
zze(TwkA~V|_~nL{6YE#^RUC__Mx26zo*w(EfK2Q@R6xo`VkJKs^Eax`&*O*bw~*ap
zyaqA_p(~(POY{H5+NIgewtB{|(%ML_wR8o);^XGTQ|{*J>74v>{_iyU;U*NTN}A%`
z`8ltg(&furYlb!j%1ra!KPSiG<VRTwPDN9f5*7>mJ>f4c!bkAtjb_qmQ+aVB(QohO
zRo@%)1krVtMPgkT6&3T*u`XO8pE&-!!u((3qVnraj|gN5aDxvqtrPs*MCZcO3i^Qt
zI7$&BFr)50exhv11)82?u`ab0FgUSw;dpbnAtmz4k^&Nx`xMQ$5(JW}ry%)ry+DV>
zS)TWjtXz7V6iK5$ghFuPiT>;;fAp)oy%%7grs4UwqU5+Ms96%`wU=YU5W-UGw(6iq
z2GhB=Zw49;Yu<#7=soc@tZvYFIVNfkRPsCT&;76cYOONM<!9yYT8XS_j|<f)GAw6X
z_w&Wq9xu5;px-$u*_Z^YS22HQpD*L|Z1fb)`d&qCQ^smb{5_5>wv!v*e#(X?l7eB-
z&pWvVcaO;IKDg7C8bZ-+Hm`g>n_WC6%BL=CZlc``M{0T;%eYQ4t}V%m20okR=HET)
z@)@WU_}tJOqiH7w2K%l<a?3NQ^6bZPnFJ<Mk`|jLP2*o$M^nx2160!F+h^quABnz;
zAF6)v=cSvmebPJaPi4k%(nh|zGG@U(va!x`)nhbzOU0MkhuA%7v6ZH!EaE%H>pe0P
z^FhhCX$ufUPCq4?C1A8ZSrVz=$~!VZ>;=kb8eaI;S1TKb|E9j*muthJe2||9pYYI$
zR@lkEo?K76^_v{llrL+?Swi1koJYJqG_-g!v?$ITb=q4#Rk--)f<yZAd%OCYe=RDW
z4aV9=2rZm-rEPrLKA|1kuMv{%I=`DA(f6L;GQJ=_TAoYWBDl;}XZ0E+YfGjvp>ABD
zh4Ibu7+f~5HEzy@7xo<qj_3c_D9C_vmh4{K98*=04-QLt1~2F@dBZe-l2GMsk#;A`
zYHOcLf#^)Gn+{G3Q4YowOIK^&zQ|LTx89&c{IWvimdkFT8nJ{0X1}p;P(C>P^f$=}
z+D3gYZ3W>%>m=U)p#UNOPPd&2cD&<J9<&QiV~vk5R%jVK^J1%HQ}`fxWs9c=2}L>;
zxb{vXTzpCjcJAOEA_~=RX^_BM+_BYW*T{zzM(3TosvFOmf6Kp0IerP4`MuBgFdrkZ
zf9X~m0O$toCckMn8klZDxWKr2%FHNk1VLQE)$!{Hz9{*a@TaZjC7kKsC1dIUx*6AQ
zJFZc8p~!CewW(VvE@yaTPFt-6n+dZ@TM582m7=-#9JoDOH#zYPe{)-Lza89t+w#Zd
zvQ3k$)Q)mPF)g)_+v$Gqgq~*RwGeBn{vhp!IPgkixW8WY)H`S{&~om!keO$Sum=oY
zTatGW#*O^aVU<^!#et91z~$IYa;_C@J7+V)`<1b_lh`8FHOAgc=Az}lf)k%5xTMrv
zr6uV%eKaU~wvi7pU)MeB7<DU@<PM)Ua&x<*j67UgrpKP|!tXx2R%YzH<LQn0XK>HK
z2D;27Dik%)-q@hK-!I|N(cl`lAF^EIv0C-t$d1qtFnKIkcMW<4b%Lzf3Y+~~qB7`<
zj);HTQS0Oex%zA170>?kRVA_m_*O?rZRpS3v{+O+cifN7Eb&>$Z==vGKh1V)C`qGu
z_u8y<#N3Wp&$V^@T??GnE&RN^IyXM)r0h(gS3;b2pt0O!eNIt4{;3H~V5Ln7vs>8{
ziqqZL4Nwlvj4CtEv0>;Fw~D>LB_+-ecI)tiR%a!^GI3BawvNQGz4#b|_d<K40`zom
zmZ%w0mYHcNzK(Ivg#;79zJA3Qs(emYQh|-Y&A>f&`e||2k;K}WnvU!Dx=0#ue(=U#
zK&pYNNf5RQZOveUm+;dQ*FIA0&#`?@z*bBhUgr(n9_FpoHPB2pI8iMpW|sF*D{+75
z-k;nba~m^}=b7P$<BGu%3I<`>FAF1)S!oDKtNG-`%h{XQi6=SMH5GZ%8j?ugqt~!K
zw<hNaHlewKU9pKh0n@^4X=DQ<4~UnDj4@h3>vA_m(*=EI<IgUo)z0l9R@mb|@QOas
zWU>ssFVW0EZ;o=u#R5gBB$CUL+->U32;2PM2O(drij20XBy|hH+=bu!0*KIKBj%c+
z^{)B`3$NB2yp-IHf02C#Fw!(;S&rR%2P<?W3i)a{Hv71$$mqNwIwWJTc5XCVCY(ZM
zZEUT%{m1IMAyv+ZxJdeiWsFSau%`*Ji4gu)?i`XAkA6AeCLD>q(!<`Q=u&+_V4eCe
z?!d0m@n<F6bnzf#{rI&DDtbzb{#Q?q`iI`Fv^=Q#{GVsrKi@5H!=Yk{`KU+uXc?t@
zxGi_IMbsNpVL63R9MI#c?&2tT**S1&xk6UXV{?VrG2Vb8uwy$l2i~-P)jArRJvd+p
zAMPX_jhyzm3a}Qc-9M){f2vD<`B3X9uKLW{DLodF&IsV$kXKT%@Qtp6|3s@S0+S=%
zV+#X9n<D=<XzlauBx&tS1|?-doY!<IKSZPJ`vt2XRD)VP6|a+O3xDEZOZR$X3e5-S
zuOL@^Te?HwRm63Ch16HfZ|^W=1@ax6$xAQ(4$2J*D69D!1&Ss_Wp=KanXxf%3)jB=
zyl{(zRa6B4dz*qTVGFnQ#lf#G^~(Orm6*fvz@t#mixM85R=piy5ZZ)?<t2uZj~#Q1
z%87M&!_4Xmtg&aKmcnz`(+k~CS_9jg?1HcPF4&*jQGA1B5O}@9G995LTJuL|d}-c#
zRi6~5UoNF~Ng3*RH>dhMu%QZ`ERBCD+uU~%h<WLJg$(5L-k}}ce*Ymz9%AWcG8~o)
zrgMWKP5N71i-Vz&u9fBxjTT}~QT7=y$EdDt>>+E^Qd;Cz=IlGV(IwUrOz(+1Gkd7O
z$HME|^+mAGBc4k(2jEj5$g30r-BUoK@Nn!*Td)5USoe+IZ-x9)#yd)sD}2Z?2{4@)
zb|)xsK&pqOpB;+H#gbf^Pto29M<2Y>dU5pAF4p{+j=oBZ$2EXA*xI~AM@g20H7o_x
z{2-Kc;SRpcxLXzU)a53ZoX%ndB^i8=>Sf&{i6CYkGSkvLj0<@C-!VKm#iX8dws__S
zKp`T~rIAfaogJ!tV(~rs5)ctD#A};YXgPNI`<5=nWQjnIf<=1Pzn2y$C8yUkFKhwM
z@%Ah?L`DM^@d<2evu->Oo=SVaiR<1GjYwe^G2)XY`l$Q%4H`|PpFA($N_8=6uOr0s
zj+)C5x<cICx<i}#5D8LZ3LNFG7uU}%Q5<kbowYRV6Bs|^frDu{l2XM2Lj-Yh_!|?f
z+a6@mRKb9j3p<Zh$+a4#UQQYhPF@-a9mWMpS)m;R6VEWV!i;mbS?{`eur*GS8_tX$
jEfLfZC2@~9k9g`Sv9u1yERTOL1mL!wsczLx=g0p8M%V6I

literal 0
HcmV?d00001

diff --git a/static/glyphicons-halflings.png b/static/glyphicons-halflings.png
new file mode 100644
index 0000000000000000000000000000000000000000..92d4445dfd0af414835467132cf50c7c38a303af
GIT binary patch
literal 4352
zcmc(jSvb@I*TDbCl4#mw&6;FnOR{7wTf|sqB70@Y*4Sf=t&%DGGDxy7<4<NH`!>in
zwn`&QQOr<`27|~lU*GNfe)r$+;%v`3=Q$VW;ymZMrG+ssw-7e~0K7L%46Ffwh5XNs
z<6`?KHS^P-{ZmgZZ@~?jOs2~JH%~nY@PG5j1zTI#0Amn(L8qe2oETm=+B^jogFL!D
zS!ISRHW3ybWQ6o&?2=byQi)JhfBSH9PzL~<0B#!S!^50cUq25lRnLyYPq06zWw>~J
z`$KJG?wJet%MCZ1y81U)c?UzG;{mBi?no2aAHvt8L__Xy66K$DAupSD_4^VSeG;vA
zGhrY7dmCA}Zg<=d*dvUYvYMo40k!iu>o|-n)q^ld6Q(6yBtUWr1GY<4vK2?uoeS|r
zT(a}}&NC3;#Lv8{0Y$f=#j|95fZYUrx?foCUQ)KvUf$-LSb+6D%%)z#|1KO+ZTgw~
zNbE_n|4p~xYoc$edOQF-XOS;%<r!#dmHF{5#RTzN2!T(FFMc;x+SmC=Km>evzdNi3
zk@(r9h#R5FpacG)j3VDRRz>g49u-o5A=@X`M=nQQ@W&MqFu3+}8)vIJ<N(sT_Zk8X
zMLcU+@C0(GanqcI+g=S0kMDE|mIlGu&c=CozOm!OJr0PRi4D`Obo#+t^;Xncwa7ai
zdh<9v@cF*U;YbI{iHB@gJAiGAx>yezf?(vDF#3iq72Yg1rU0$uCw``L1fzH6tU=MT
zJ)FP#7~BMLoosB<>)Y`BnyxN?%PW`qwa_nrmk;P<^+|3lA$<ii8%uY~81lp=vs^qj
zbHc@jPjA%v+=Qq|+H=tEesx$(-?7Mn9R7&2+ZkE|b0%rF{pOe&2PywxrLyZ}ob?{?
zS!X14D0!!&R7j4z&a_ZX7E96o-d9#)-rDWl);JNdUKsSn^M1g(Sofin)_^D2A6fsk
zK%N&KDHIEtjz^2e+RnzNElc?mvW&i)!L?OOITL^U!tz7HAYtpl+few+Se~$Nk0>cC
z!KnRdI-*8rENgl-h*t3^hviocbR?_BCX&(%?-)#H*`RRAUES@w^(0ey@bvFIq^EE0
zYIYPpa4Xz>{9(cUIq~=IuByDHtJskc@OXkoyhOvqjT$BRxhihe#hq<$(TaV?g(bYx
zzk*$b_y4xdrKd-u!#@W)7x%!%FE62JOZu)fTpnAUKW94KXQKo9lR9BoI`nN#BV<pu
zN$Y&tINUw4JJ60wNhX=$oO%xnS0~8-36@e}lO69__j)7adZ<L2U#u5vwGeo2shUYF
zH*Rf1efjD`jdvTV8a6X+&!xQkbfP<z!gz1jlz##Puuh>NL^WLc-2PBnDb`!FkQ6Yw
zt8#VMCqN`vOx>8A-pqa3!sg7$vF4w|C29%3h5O_{d+D-|gED!U;S&A}5QU_Uz%?vp
zmMBIPvj7qQQG74PJJYIU8KAgcJcJvNO0O6=%8w|@chXvpUX6O34cERMj)m?X)jwit
zWYksusgx8zcrOv1Kd4Cm%yUoW#?wfM-ee=?*pXt7dU<wL(ECgNtn7KAcQSgjF$wec
z&wNDxS$nw$r4-^(dj0lt)f7DU5?+TTQ7UFh7R=*xI5;Wr=aA7JB?^0SzgQ^V;4r`?
zBrN-yp=!hwMFq0PE?Y!UWLSr9S4c>vyZrhI*Zx3!VQzm2&D<yRh#LVfjXeOPcj~wR
z5UG;7Ix04MSLbA=`nZloXfY{`*@7=#K0}`VWppv~RO%H}$!V*DNHvZFBHUqfI?CD0
zbx!B4^9`#pqXl&iB{Pv^*lNn33}KeCeaE4o?M=ZBEL9o|=KG==a{vbsI4@1LO3@|!
z=#F_^eo|k6WLCD`I?D^lB}ZeRa3j2$+MNG{fZN~d@a`$my6AxPBfp8Irx1gDj_8y_
zB{|_Ko-%@Zv$H~Px&z9e&#zqq`(8HmN?{uv#cDixDOJ%G_;k$j?o_(Za8|9R0~pd~
zhP6EvoFeWkI{=X$R-d0BaUhyb8w0in4s%stxoODXOl;El?W^#yR`?d86Ax#>k2i(z
zv;J?=_W|Z`2Nb*9*m`XJ^1ixr>GY^eNXXM8UzHKbJ%`E&g=n<QM>C-&t%U{b2>k}4
zM^eC8z9@VJ)NO6~zgW94x7psn_*GsP&AXPV>|c7+3V*`GDl?NuNHOr8_5jSBY+FrJ
zxxFy&omakmacj-wPLUexLeI~s2^i^7j<QS1^o69wqChX$OY6u{tW}exT*h=kf_UuP
z+XaMs<6dAuy-kT^H%eXIYHYk4Q!FTjJ*&L%*Q})SYV#u;NgCV`gwN=QJ~!7t_q2+B
zpbd_ZMR9D%dyk)}nec)ZXV~q^?S+kxZJj&X5@|w?zO<x_02M#Q3a*5JM8Y&n;d~#^
zX?>diy$lDh;U-ze^bf8Wq&_j48xx9sRj~I0?AI|l`&NRKa0xj_M7{QQP8x>W$llZ#
z^2}mA)Bep^+iA@Qw-LK1wT3nbnW#j??18HOX9M~EwO_4MW54*U(nB|yBja(g7FnMC
zblZNR)Y{`EcNWNZ9&#=!$@W#;-?`_@7{fb;%BTG<Be%)pb!CB`M;1FsO>aNt!jg%h
zP{`+<{G!`T5|=OLq>Z*{Z2O&8zMn16ACVB$Qm``DYk?tjJdb2uC7aci<-`J?E%OU+
zGrN5UtA#%|w#4Z;NP?k$>n!<|SrjF%qnK<QD=|fvQ-`MgFRin=cJ~1?W$Nv-%7>36
z-X#tb9{hRfZswTsPVZBN8H~75sHKLYIz~6u+pKzy#crwlQTpM#$E~+Abk)TD#sz#v
zXX8Go`ZaF>B8Zu%M9U<U?k5{O<y&QE7KlDa9?QUr-S}#I$LYUm81UoWaH<><;>RXE
zbfFb@39Y9#&~E%DMKl*GIPjFwcNZ7nuMbVEpA0WbvBjM9QA!sp{YiDoe131&NawG0
z)w7{^`zTTBX*b%&r|n~U@dMgnxo!))g;D+Qg=`Xw5@VHk^{hiH?Dbc#u;gsXHzn0i
z2)8o6*&Kl>6tpGG-xYv<M}QNBKQ@Z8AUtKe=QqbSl$Amw(w@PJ2Fl*B4kD#B1X|@h
zND6v-C(>B-r`9coW<<#c<0|E=wQpY(XerrkkfVOt!t*N?wvbI|9F@&~JQ7q2jXe2H
zCW^MvkWX8I-=%fo@BdI{A^py@pAB`s<yjfB3(la;jxJW|8b9;qtmahyAaiMpzZU^P
zxD>hd&A{*amKE*X!a7A2Yu?Z%f;af$36@t#hgGI$UAqZQr>(vfUM3&C0L=d07kpTV
z65hXXqa6SYLUvQ%beIm#w8HN~d3!4?$?iB2Owr|ut8l>>rMSqaZB}JGncrpN>H)eX
z?`{XC$$(nou>9J>y&RJ_GCHrPS%%Jr+GeZ-p;^lV`1YLmyxKN-u#7+}dnx}N%zgXH
z$CV1rQyi4eN)t(4&9Ix9{_jMeW*4;LYis@>9EQ2Es^gfy-VKyn0lc8i{7q3yuQV}F
zD6Fom;2?qz@ukzYpge~g8?BAWbC}{;E82F=WrGc<q3x&8B^qmty_aky%YQ{CKTGq<
zYP!kE(69SylMU^oAELgld(|`QSIDWIzU`!z4rh5Kn4EmCqu{yU{SIwx=mqDK8w<~1
zUPFy^`6*;La<HSR(O`c-+NrbAEnz=wxz=;F=D$%Gr5~UQ*wG%^^eW0ENv91u_qR-a
z$S)u&@oi_Fi#yBZUxTms_h&AvlAOS|`l_14f97W-V>0;?er)DQ&9VG84bSn{>9B(k
zwM%!e%*jQ~?@0DuS;yYC#^~O_E+}d7VN;GP%ockmCFlj4DNZ%yl_X-Hn$v_=+Er1z
z)xF^ugN@xFweaki3bVXB3?uwjsn55R<b|OgeId~Hv@}>D1&YMi6B+jBAEU6|0Y1ne
zLxbyOnkM9BHX2f}bHa<7WG>P_pz=aP(B)D(uo1i&yvId9DaA3GTsK?WdG%g5Q5z-%
zUfT;wH`Xu@LDvM>F<4<`LiFUdk7UO)oS&1>Rnv!81;V#S1gZ^;byAIw5fmjY3m)nw
z?+@SmlmBCWV>bFM8|-jGB{WLeI3o9DaWo<)11@8`kh*v=cN0DNB+st4sz6R#2I0qi
z4c&8ZcAexDoiEyzoZJ((D9)8bG%^Z+MCs@_Q)++#Uvn&7#CI<7^ioFM{2qLTEAfMX
z#1kD>oACS6EsTK8F}{R&pahvhyt|}$lX5-EzVP=!*jL*U(=7^7%UUF#`g>m(9)4uh
zN+-O*&B&PgYQ520)x+!;$#)PXM`Kgq-o1CQLPsDGuSVi?k7|gIEtmv^WewHMkLAio
zl1Us*ZM8T5*j_cED4OCIiNDZ{(dj&{3{g&T+~4Y*L((GimlI~v8Q&*2;zNurHxdEX
zDgWY5T-u#~Rw6AH53<&eUOA_3sJa+<`S@61`0Z+&gPPC(dA9xY-3vCHs+QQ8y<*H|
zq`~2~B6ACGIIhlq0<JP>$V=$vE_&HDcwxCpLD6$_1>ZT*h{SQByL1NMw0+fOj?Wz&
zFvJdbQkbJBeJ=wX#hUle7%rUXR$4yPWhM|#t(`DrC+d#^K8*!sRn%{Eee5S%bqSan
z?Gaxb6y6;Dw^4Ura3@7~UnV3ahsAZxfc!%uwqZbo@PGj7@>ji1sVn}8fiB(aiz~Jo
zTDXK*@oVh~gVo^Iu~o8PQNMj6)RalL?o3^H@pnjZNLWoX&@@;gDJHvX&C-&SZCkAF
z?Pux@B3eZQ037cWb&FZMuP+XLz1yG`s8)?SoCs!ygWlxG$PB`Eka2i37Fv)TK{|58
zJti;S=?xo)8?eTei(HD#<H{`dIBo}QZ*qye7Ch&8W=G`>f`Jq8j>vX~5NRzRU9sf_
z>oxtdr~$>ax+OJ;^X)vsSztp0JYJsoQlX{)JP`NN^%4mv6u3oW-hBTdM2W@5-Fze>
z9n9nd!<vn@x)+DSqur{lShQR5c_q20z&z9X_VI@tF-sZITiJ8(=%yDq%20jBZq4o?
zgCC2nZ#R@cyO{hJ?i_$meOX?m;pkq%(#414r`r1hpFn%A^?fTAk~P~=C0`Omj7x)=
z_=sB}!Gp5B>;qg7R6d&M#&&}CPAvA|mF^4XPltG`XZl9!t)5o^flxcEGJRDAZjOjF
zQ0Iea%DG$E3bP&!(93|2RCY3l5t3s3J*JOik0=hGeaJ@3@H8tD7<k9<<dKwp&eQ6Z
z9|U0$hb)b5lItCim6MC_Nf&^qL{S0zjAEPdi{G~l$mUBpQVcZOtKq$za5*WnwuQO{
zxF$NXUlSh-TEr%CuFbjgKX@wV^CqEZM<ObXOWagY0q?8j*FR)BnR)!IQXA#2X-7RS
zQDDqU9@ib_?%osL+z(HZl~m@gbUVL(W{K>CVRqHg&`+R3j0a8@kqB}PI}{$m!yRab
zvul5lL(>3*TF>n~)*#hsmwUTtKRAA2Fnk0PENdI!9GrZLu@zyKzs+&m-IKFviqv>&
kg1Lm#gqI~e;$iYPkmG5c&N-g{UI@TVLkokN>#mRg2V?7pi2wiq

literal 0
HcmV?d00001


From b5c3743c423534a1259baf80e88a14c30cd9348c Mon Sep 17 00:00:00 2001
From: "http://jasonwoof.com/" <JasonWoof@web>
Date: Fri, 27 Jul 2012 06:37:03 +0000
Subject: [PATCH 4290/8313] Added a comment: Your security solution is fine

---
 ...1_d5fb67f373038e9f583cb2e1992bef67._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment

diff --git a/doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment b/doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment
new file mode 100644
index 0000000000..bc1f259e85
--- /dev/null
+++ b/doc/design/assistant/blog/day_44__webapp_basics/comment_1_d5fb67f373038e9f583cb2e1992bef67._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="Your security solution is fine"
+ date="2012-07-27T06:37:03Z"
+ content="""
+I think making the html file redirect is just fine. desktopcouch does something like this. They talk about it being there to help you find the port number, but it contains the security token too.
+
+They also have some javascript to delay the redirect for a while so you can bookmark the redirect file. You can click to set a cookie to skip the delay in future.
+
+It's been a while since I used desktopcouch, but I found an old redirect file laying around. Here it is:
+
+http://jasonwoof.com/downloads/desktopcouch-redirect.txt
+
+The code is pretty short, you might find something useful there.
+
+      - Jason
+"""]]

From 3f1c96800d0d4fb03e02d52996b8fea4bd604dc4 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Fri, 27 Jul 2012 06:47:46 +0000
Subject: [PATCH 4291/8313]

---
 doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn

diff --git a/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn b/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn
new file mode 100644
index 0000000000..6dc9795f34
--- /dev/null
+++ b/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn
@@ -0,0 +1,9 @@
+Seems like there is a needed dependancy, yesod-default is needed on OSX.
+
+<pre>
+Utility/Yesod.hs:10:8:
+    Could not find module `Yesod.Default.Util'
+    Use -v to see a list of the files searched for.
+make: *** [git-annex] Error 1
+</pre>
+

From 1192e305c7fa7ba0b6572cc8c450127d6458b0af Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 04:37:29 -0400
Subject: [PATCH 4292/8313] update to newer version needed by bootstrap

---
 static/jquery.full.js | 4435 ++++++++++++++++++++++-------------------
 1 file changed, 2429 insertions(+), 2006 deletions(-)

diff --git a/static/jquery.full.js b/static/jquery.full.js
index f3201aacb6..3774ff9861 100644
--- a/static/jquery.full.js
+++ b/static/jquery.full.js
@@ -1,5 +1,5 @@
 /*!
- * jQuery JavaScript Library v1.6.2
+ * jQuery JavaScript Library v1.7.2
  * http://jquery.com/
  *
  * Copyright 2011, John Resig
@@ -11,7 +11,7 @@
  * Copyright 2011, The Dojo Foundation
  * Released under the MIT, BSD, and GPL Licenses.
  *
- * Date: Thu Jun 30 14:16:56 2011 -0400
+ * Date: Wed Mar 21 12:46:34 2012 -0700
  */
 (function( window, undefined ) {
 
@@ -37,8 +37,8 @@ var jQuery = function( selector, context ) {
 	rootjQuery,
 
 	// A simple way to check for HTML strings or ID strings
-	// (both of which we optimize for)
-	quickExpr = /^(?:[^<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,
+	// Prioritize #id over <tag> to avoid XSS via location.hash (#9521)
+	quickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,
 
 	// Check if a string has a non-whitespace character in it
 	rnotwhite = /\S/,
@@ -47,9 +47,6 @@ var jQuery = function( selector, context ) {
 	trimLeft = /^\s+/,
 	trimRight = /\s+$/,
 
-	// Check for digits
-	rdigit = /\d/,
-
 	// Match a standalone tag
 	rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>)?$/,
 
@@ -66,11 +63,12 @@ var jQuery = function( selector, context ) {
 	rmozilla = /(mozilla)(?:.*? rv:([\w.]+))?/,
 
 	// Matches dashed string for camelizing
-	rdashAlpha = /-([a-z])/ig,
+	rdashAlpha = /-([a-z]|[0-9])/ig,
+	rmsPrefix = /^-ms-/,
 
 	// Used by jQuery.camelCase as callback to replace()
 	fcamelCase = function( all, letter ) {
-		return letter.toUpperCase();
+		return ( letter + "" ).toUpperCase();
 	},
 
 	// Keep a UserAgent string for use with jQuery.browser
@@ -139,7 +137,7 @@ jQuery.fn = jQuery.prototype = {
 				// HANDLE: $(html) -> $(array)
 				if ( match[1] ) {
 					context = context instanceof jQuery ? context[0] : context;
-					doc = (context ? context.ownerDocument || context : document);
+					doc = ( context ? context.ownerDocument || context : document );
 
 					// If a single string is passed in and it's a single tag
 					// just do a createElement and skip the rest
@@ -156,7 +154,7 @@ jQuery.fn = jQuery.prototype = {
 
 					} else {
 						ret = jQuery.buildFragment( [ match[1] ], [ doc ] );
-						selector = (ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment).childNodes;
+						selector = ( ret.cacheable ? jQuery.clone(ret.fragment) : ret.fragment ).childNodes;
 					}
 
 					return jQuery.merge( this, selector );
@@ -186,7 +184,7 @@ jQuery.fn = jQuery.prototype = {
 
 			// HANDLE: $(expr, $(...))
 			} else if ( !context || context.jquery ) {
-				return (context || rootjQuery).find( selector );
+				return ( context || rootjQuery ).find( selector );
 
 			// HANDLE: $(expr, context)
 			// (which is just equivalent to: $(context).find(expr)
@@ -200,7 +198,7 @@ jQuery.fn = jQuery.prototype = {
 			return rootjQuery.ready( selector );
 		}
 
-		if (selector.selector !== undefined) {
+		if ( selector.selector !== undefined ) {
 			this.selector = selector.selector;
 			this.context = selector.context;
 		}
@@ -212,7 +210,7 @@ jQuery.fn = jQuery.prototype = {
 	selector: "",
 
 	// The current version of jQuery being used
-	jquery: "1.6.2",
+	jquery: "1.7.2",
 
 	// The default length of a jQuery object is 0
 	length: 0,
@@ -257,7 +255,7 @@ jQuery.fn = jQuery.prototype = {
 		ret.context = this.context;
 
 		if ( name === "find" ) {
-			ret.selector = this.selector + (this.selector ? " " : "") + selector;
+			ret.selector = this.selector + ( this.selector ? " " : "" ) + selector;
 		} else if ( name ) {
 			ret.selector = this.selector + "." + name + "(" + selector + ")";
 		}
@@ -278,15 +276,16 @@ jQuery.fn = jQuery.prototype = {
 		jQuery.bindReady();
 
 		// Add the callback
-		readyList.done( fn );
+		readyList.add( fn );
 
 		return this;
 	},
 
 	eq: function( i ) {
+		i = +i;
 		return i === -1 ?
 			this.slice( i ) :
-			this.slice( i, +i + 1 );
+			this.slice( i, i + 1 );
 	},
 
 	first: function() {
@@ -433,11 +432,11 @@ jQuery.extend({
 			}
 
 			// If there are functions bound, to execute
-			readyList.resolveWith( document, [ jQuery ] );
+			readyList.fireWith( document, [ jQuery ] );
 
 			// Trigger any bound ready events
 			if ( jQuery.fn.trigger ) {
-				jQuery( document ).trigger( "ready" ).unbind( "ready" );
+				jQuery( document ).trigger( "ready" ).off( "ready" );
 			}
 		}
 	},
@@ -447,7 +446,7 @@ jQuery.extend({
 			return;
 		}
 
-		readyList = jQuery._Deferred();
+		readyList = jQuery.Callbacks( "once memory" );
 
 		// Catch cases where $(document).ready() is called after the
 		// browser event has already occurred.
@@ -498,13 +497,12 @@ jQuery.extend({
 		return jQuery.type(obj) === "array";
 	},
 
-	// A crude way of determining if an object is a window
 	isWindow: function( obj ) {
-		return obj && typeof obj === "object" && "setInterval" in obj;
+		return obj != null && obj == obj.window;
 	},
 
-	isNaN: function( obj ) {
-		return obj == null || !rdigit.test( obj ) || isNaN( obj );
+	isNumeric: function( obj ) {
+		return !isNaN( parseFloat(obj) ) && isFinite( obj );
 	},
 
 	type: function( obj ) {
@@ -521,10 +519,15 @@ jQuery.extend({
 			return false;
 		}
 
-		// Not own constructor property must be Object
-		if ( obj.constructor &&
-			!hasOwn.call(obj, "constructor") &&
-			!hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) {
+		try {
+			// Not own constructor property must be Object
+			if ( obj.constructor &&
+				!hasOwn.call(obj, "constructor") &&
+				!hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) {
+				return false;
+			}
+		} catch ( e ) {
+			// IE8,9 Will throw exceptions on certain host objects #9897
 			return false;
 		}
 
@@ -545,7 +548,7 @@ jQuery.extend({
 	},
 
 	error: function( msg ) {
-		throw msg;
+		throw new Error( msg );
 	},
 
 	parseJSON: function( data ) {
@@ -567,31 +570,33 @@ jQuery.extend({
 			.replace( rvalidtokens, "]" )
 			.replace( rvalidbraces, "")) ) {
 
-			return (new Function( "return " + data ))();
+			return ( new Function( "return " + data ) )();
 
 		}
 		jQuery.error( "Invalid JSON: " + data );
 	},
 
 	// Cross-browser xml parsing
-	// (xml & tmp used internally)
-	parseXML: function( data , xml , tmp ) {
-
-		if ( window.DOMParser ) { // Standard
-			tmp = new DOMParser();
-			xml = tmp.parseFromString( data , "text/xml" );
-		} else { // IE
-			xml = new ActiveXObject( "Microsoft.XMLDOM" );
-			xml.async = "false";
-			xml.loadXML( data );
+	parseXML: function( data ) {
+		if ( typeof data !== "string" || !data ) {
+			return null;
 		}
-
-		tmp = xml.documentElement;
-
-		if ( ! tmp || ! tmp.nodeName || tmp.nodeName === "parsererror" ) {
+		var xml, tmp;
+		try {
+			if ( window.DOMParser ) { // Standard
+				tmp = new DOMParser();
+				xml = tmp.parseFromString( data , "text/xml" );
+			} else { // IE
+				xml = new ActiveXObject( "Microsoft.XMLDOM" );
+				xml.async = "false";
+				xml.loadXML( data );
+			}
+		} catch( e ) {
+			xml = undefined;
+		}
+		if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) {
 			jQuery.error( "Invalid XML: " + data );
 		}
-
 		return xml;
 	},
 
@@ -611,10 +616,10 @@ jQuery.extend({
 		}
 	},
 
-	// Converts a dashed string to camelCased string;
-	// Used by both the css and data modules
+	// Convert dashed to camelCase; used by the css and data modules
+	// Microsoft forgot to hump their vendor prefix (#9572)
 	camelCase: function( string ) {
-		return string.replace( rdashAlpha, fcamelCase );
+		return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase );
 	},
 
 	nodeName: function( elem, name ) {
@@ -683,8 +688,6 @@ jQuery.extend({
 
 		if ( array != null ) {
 			// The window, strings (and functions) also have 'length'
-			// The extra typeof function check is to prevent crashes
-			// in Safari 2 (See: #3039)
 			// Tweaked logic slightly to handle Blackberry 4.7 RegExp issues #6930
 			var type = jQuery.type( array );
 
@@ -698,15 +701,22 @@ jQuery.extend({
 		return ret;
 	},
 
-	inArray: function( elem, array ) {
+	inArray: function( elem, array, i ) {
+		var len;
 
-		if ( indexOf ) {
-			return indexOf.call( array, elem );
-		}
+		if ( array ) {
+			if ( indexOf ) {
+				return indexOf.call( array, elem, i );
+			}
 
-		for ( var i = 0, length = array.length; i < length; i++ ) {
-			if ( array[ i ] === elem ) {
-				return i;
+			len = array.length;
+			i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0;
+
+			for ( ; i < len; i++ ) {
+				// Skip accessing in sparse arrays
+				if ( i in array && array[ i ] === elem ) {
+					return i;
+				}
 			}
 		}
 
@@ -814,35 +824,59 @@ jQuery.extend({
 
 	// Mutifunctional method to get and set values to a collection
 	// The value/s can optionally be executed if it's a function
-	access: function( elems, key, value, exec, fn, pass ) {
-		var length = elems.length;
+	access: function( elems, fn, key, value, chainable, emptyGet, pass ) {
+		var exec,
+			bulk = key == null,
+			i = 0,
+			length = elems.length;
 
-		// Setting many attributes
-		if ( typeof key === "object" ) {
-			for ( var k in key ) {
-				jQuery.access( elems, k, key[k], exec, fn, value );
+		// Sets many values
+		if ( key && typeof key === "object" ) {
+			for ( i in key ) {
+				jQuery.access( elems, fn, i, key[i], 1, emptyGet, value );
 			}
-			return elems;
-		}
+			chainable = 1;
 
-		// Setting one attribute
-		if ( value !== undefined ) {
+		// Sets one value
+		} else if ( value !== undefined ) {
 			// Optionally, function values get executed if exec is true
-			exec = !pass && exec && jQuery.isFunction(value);
+			exec = pass === undefined && jQuery.isFunction( value );
 
-			for ( var i = 0; i < length; i++ ) {
-				fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass );
+			if ( bulk ) {
+				// Bulk operations only iterate when executing function values
+				if ( exec ) {
+					exec = fn;
+					fn = function( elem, key, value ) {
+						return exec.call( jQuery( elem ), value );
+					};
+
+				// Otherwise they run against the entire set
+				} else {
+					fn.call( elems, value );
+					fn = null;
+				}
 			}
 
-			return elems;
+			if ( fn ) {
+				for (; i < length; i++ ) {
+					fn( elems[i], key, exec ? value.call( elems[i], i, fn( elems[i], key ) ) : value, pass );
+				}
+			}
+
+			chainable = 1;
 		}
 
-		// Getting an attribute
-		return length ? fn( elems[0], key ) : undefined;
+		return chainable ?
+			elems :
+
+			// Gets
+			bulk ?
+				fn.call( elems ) :
+				length ? fn( elems[0], key ) : emptyGet;
 	},
 
 	now: function() {
-		return (new Date()).getTime();
+		return ( new Date() ).getTime();
 	},
 
 	// Use of jQuery.browser is frowned upon.
@@ -949,188 +983,363 @@ return jQuery;
 })();
 
 
-var // Promise methods
-	promiseMethods = "done fail isResolved isRejected promise then always pipe".split( " " ),
-	// Static reference to slice
+// String to Object flags format cache
+var flagsCache = {};
+
+// Convert String-formatted flags into Object-formatted ones and store in cache
+function createFlags( flags ) {
+	var object = flagsCache[ flags ] = {},
+		i, length;
+	flags = flags.split( /\s+/ );
+	for ( i = 0, length = flags.length; i < length; i++ ) {
+		object[ flags[i] ] = true;
+	}
+	return object;
+}
+
+/*
+ * Create a callback list using the following parameters:
+ *
+ *	flags:	an optional list of space-separated flags that will change how
+ *			the callback list behaves
+ *
+ * By default a callback list will act like an event callback list and can be
+ * "fired" multiple times.
+ *
+ * Possible flags:
+ *
+ *	once:			will ensure the callback list can only be fired once (like a Deferred)
+ *
+ *	memory:			will keep track of previous values and will call any callback added
+ *					after the list has been fired right away with the latest "memorized"
+ *					values (like a Deferred)
+ *
+ *	unique:			will ensure a callback can only be added once (no duplicate in the list)
+ *
+ *	stopOnFalse:	interrupt callings when a callback returns false
+ *
+ */
+jQuery.Callbacks = function( flags ) {
+
+	// Convert flags from String-formatted to Object-formatted
+	// (we check in cache first)
+	flags = flags ? ( flagsCache[ flags ] || createFlags( flags ) ) : {};
+
+	var // Actual callback list
+		list = [],
+		// Stack of fire calls for repeatable lists
+		stack = [],
+		// Last fire value (for non-forgettable lists)
+		memory,
+		// Flag to know if list was already fired
+		fired,
+		// Flag to know if list is currently firing
+		firing,
+		// First callback to fire (used internally by add and fireWith)
+		firingStart,
+		// End of the loop when firing
+		firingLength,
+		// Index of currently firing callback (modified by remove if needed)
+		firingIndex,
+		// Add one or several callbacks to the list
+		add = function( args ) {
+			var i,
+				length,
+				elem,
+				type,
+				actual;
+			for ( i = 0, length = args.length; i < length; i++ ) {
+				elem = args[ i ];
+				type = jQuery.type( elem );
+				if ( type === "array" ) {
+					// Inspect recursively
+					add( elem );
+				} else if ( type === "function" ) {
+					// Add if not in unique mode and callback is not in
+					if ( !flags.unique || !self.has( elem ) ) {
+						list.push( elem );
+					}
+				}
+			}
+		},
+		// Fire callbacks
+		fire = function( context, args ) {
+			args = args || [];
+			memory = !flags.memory || [ context, args ];
+			fired = true;
+			firing = true;
+			firingIndex = firingStart || 0;
+			firingStart = 0;
+			firingLength = list.length;
+			for ( ; list && firingIndex < firingLength; firingIndex++ ) {
+				if ( list[ firingIndex ].apply( context, args ) === false && flags.stopOnFalse ) {
+					memory = true; // Mark as halted
+					break;
+				}
+			}
+			firing = false;
+			if ( list ) {
+				if ( !flags.once ) {
+					if ( stack && stack.length ) {
+						memory = stack.shift();
+						self.fireWith( memory[ 0 ], memory[ 1 ] );
+					}
+				} else if ( memory === true ) {
+					self.disable();
+				} else {
+					list = [];
+				}
+			}
+		},
+		// Actual Callbacks object
+		self = {
+			// Add a callback or a collection of callbacks to the list
+			add: function() {
+				if ( list ) {
+					var length = list.length;
+					add( arguments );
+					// Do we need to add the callbacks to the
+					// current firing batch?
+					if ( firing ) {
+						firingLength = list.length;
+					// With memory, if we're not firing then
+					// we should call right away, unless previous
+					// firing was halted (stopOnFalse)
+					} else if ( memory && memory !== true ) {
+						firingStart = length;
+						fire( memory[ 0 ], memory[ 1 ] );
+					}
+				}
+				return this;
+			},
+			// Remove a callback from the list
+			remove: function() {
+				if ( list ) {
+					var args = arguments,
+						argIndex = 0,
+						argLength = args.length;
+					for ( ; argIndex < argLength ; argIndex++ ) {
+						for ( var i = 0; i < list.length; i++ ) {
+							if ( args[ argIndex ] === list[ i ] ) {
+								// Handle firingIndex and firingLength
+								if ( firing ) {
+									if ( i <= firingLength ) {
+										firingLength--;
+										if ( i <= firingIndex ) {
+											firingIndex--;
+										}
+									}
+								}
+								// Remove the element
+								list.splice( i--, 1 );
+								// If we have some unicity property then
+								// we only need to do this once
+								if ( flags.unique ) {
+									break;
+								}
+							}
+						}
+					}
+				}
+				return this;
+			},
+			// Control if a given callback is in the list
+			has: function( fn ) {
+				if ( list ) {
+					var i = 0,
+						length = list.length;
+					for ( ; i < length; i++ ) {
+						if ( fn === list[ i ] ) {
+							return true;
+						}
+					}
+				}
+				return false;
+			},
+			// Remove all callbacks from the list
+			empty: function() {
+				list = [];
+				return this;
+			},
+			// Have the list do nothing anymore
+			disable: function() {
+				list = stack = memory = undefined;
+				return this;
+			},
+			// Is it disabled?
+			disabled: function() {
+				return !list;
+			},
+			// Lock the list in its current state
+			lock: function() {
+				stack = undefined;
+				if ( !memory || memory === true ) {
+					self.disable();
+				}
+				return this;
+			},
+			// Is it locked?
+			locked: function() {
+				return !stack;
+			},
+			// Call all callbacks with the given context and arguments
+			fireWith: function( context, args ) {
+				if ( stack ) {
+					if ( firing ) {
+						if ( !flags.once ) {
+							stack.push( [ context, args ] );
+						}
+					} else if ( !( flags.once && memory ) ) {
+						fire( context, args );
+					}
+				}
+				return this;
+			},
+			// Call all the callbacks with the given arguments
+			fire: function() {
+				self.fireWith( this, arguments );
+				return this;
+			},
+			// To know if the callbacks have already been called at least once
+			fired: function() {
+				return !!fired;
+			}
+		};
+
+	return self;
+};
+
+
+
+
+var // Static reference to slice
 	sliceDeferred = [].slice;
 
 jQuery.extend({
-	// Create a simple deferred (one callbacks list)
-	_Deferred: function() {
-		var // callbacks list
-			callbacks = [],
-			// stored [ context , args ]
-			fired,
-			// to avoid firing when already doing so
-			firing,
-			// flag to know if the deferred has been cancelled
-			cancelled,
-			// the deferred itself
-			deferred  = {
 
-				// done( f1, f2, ...)
-				done: function() {
-					if ( !cancelled ) {
-						var args = arguments,
-							i,
-							length,
-							elem,
-							type,
-							_fired;
-						if ( fired ) {
-							_fired = fired;
-							fired = 0;
-						}
-						for ( i = 0, length = args.length; i < length; i++ ) {
-							elem = args[ i ];
-							type = jQuery.type( elem );
-							if ( type === "array" ) {
-								deferred.done.apply( deferred, elem );
-							} else if ( type === "function" ) {
-								callbacks.push( elem );
-							}
-						}
-						if ( _fired ) {
-							deferred.resolveWith( _fired[ 0 ], _fired[ 1 ] );
-						}
-					}
-					return this;
-				},
-
-				// resolve with given context and args
-				resolveWith: function( context, args ) {
-					if ( !cancelled && !fired && !firing ) {
-						// make sure args are available (#8421)
-						args = args || [];
-						firing = 1;
-						try {
-							while( callbacks[ 0 ] ) {
-								callbacks.shift().apply( context, args );
-							}
-						}
-						finally {
-							fired = [ context, args ];
-							firing = 0;
-						}
-					}
-					return this;
-				},
-
-				// resolve with this as context and given arguments
-				resolve: function() {
-					deferred.resolveWith( this, arguments );
-					return this;
-				},
-
-				// Has this deferred been resolved?
-				isResolved: function() {
-					return !!( firing || fired );
-				},
-
-				// Cancel
-				cancel: function() {
-					cancelled = 1;
-					callbacks = [];
-					return this;
-				}
-			};
-
-		return deferred;
-	},
-
-	// Full fledged deferred (two callbacks list)
 	Deferred: function( func ) {
-		var deferred = jQuery._Deferred(),
-			failDeferred = jQuery._Deferred(),
-			promise;
-		// Add errorDeferred methods, then and promise
-		jQuery.extend( deferred, {
-			then: function( doneCallbacks, failCallbacks ) {
-				deferred.done( doneCallbacks ).fail( failCallbacks );
-				return this;
+		var doneList = jQuery.Callbacks( "once memory" ),
+			failList = jQuery.Callbacks( "once memory" ),
+			progressList = jQuery.Callbacks( "memory" ),
+			state = "pending",
+			lists = {
+				resolve: doneList,
+				reject: failList,
+				notify: progressList
 			},
-			always: function() {
-				return deferred.done.apply( deferred, arguments ).fail.apply( this, arguments );
-			},
-			fail: failDeferred.done,
-			rejectWith: failDeferred.resolveWith,
-			reject: failDeferred.resolve,
-			isRejected: failDeferred.isResolved,
-			pipe: function( fnDone, fnFail ) {
-				return jQuery.Deferred(function( newDefer ) {
-					jQuery.each( {
-						done: [ fnDone, "resolve" ],
-						fail: [ fnFail, "reject" ]
-					}, function( handler, data ) {
-						var fn = data[ 0 ],
-							action = data[ 1 ],
-							returned;
-						if ( jQuery.isFunction( fn ) ) {
-							deferred[ handler ](function() {
-								returned = fn.apply( this, arguments );
-								if ( returned && jQuery.isFunction( returned.promise ) ) {
-									returned.promise().then( newDefer.resolve, newDefer.reject );
-								} else {
-									newDefer[ action ]( returned );
-								}
-							});
-						} else {
-							deferred[ handler ]( newDefer[ action ] );
+			promise = {
+				done: doneList.add,
+				fail: failList.add,
+				progress: progressList.add,
+
+				state: function() {
+					return state;
+				},
+
+				// Deprecated
+				isResolved: doneList.fired,
+				isRejected: failList.fired,
+
+				then: function( doneCallbacks, failCallbacks, progressCallbacks ) {
+					deferred.done( doneCallbacks ).fail( failCallbacks ).progress( progressCallbacks );
+					return this;
+				},
+				always: function() {
+					deferred.done.apply( deferred, arguments ).fail.apply( deferred, arguments );
+					return this;
+				},
+				pipe: function( fnDone, fnFail, fnProgress ) {
+					return jQuery.Deferred(function( newDefer ) {
+						jQuery.each( {
+							done: [ fnDone, "resolve" ],
+							fail: [ fnFail, "reject" ],
+							progress: [ fnProgress, "notify" ]
+						}, function( handler, data ) {
+							var fn = data[ 0 ],
+								action = data[ 1 ],
+								returned;
+							if ( jQuery.isFunction( fn ) ) {
+								deferred[ handler ](function() {
+									returned = fn.apply( this, arguments );
+									if ( returned && jQuery.isFunction( returned.promise ) ) {
+										returned.promise().then( newDefer.resolve, newDefer.reject, newDefer.notify );
+									} else {
+										newDefer[ action + "With" ]( this === deferred ? newDefer : this, [ returned ] );
+									}
+								});
+							} else {
+								deferred[ handler ]( newDefer[ action ] );
+							}
+						});
+					}).promise();
+				},
+				// Get a promise for this deferred
+				// If obj is provided, the promise aspect is added to the object
+				promise: function( obj ) {
+					if ( obj == null ) {
+						obj = promise;
+					} else {
+						for ( var key in promise ) {
+							obj[ key ] = promise[ key ];
 						}
-					});
-				}).promise();
-			},
-			// Get a promise for this deferred
-			// If obj is provided, the promise aspect is added to the object
-			promise: function( obj ) {
-				if ( obj == null ) {
-					if ( promise ) {
-						return promise;
 					}
-					promise = obj = {};
+					return obj;
 				}
-				var i = promiseMethods.length;
-				while( i-- ) {
-					obj[ promiseMethods[i] ] = deferred[ promiseMethods[i] ];
-				}
-				return obj;
-			}
-		});
-		// Make sure only one callback list will be used
-		deferred.done( failDeferred.cancel ).fail( deferred.cancel );
-		// Unexpose cancel
-		delete deferred.cancel;
+			},
+			deferred = promise.promise({}),
+			key;
+
+		for ( key in lists ) {
+			deferred[ key ] = lists[ key ].fire;
+			deferred[ key + "With" ] = lists[ key ].fireWith;
+		}
+
+		// Handle state
+		deferred.done( function() {
+			state = "resolved";
+		}, failList.disable, progressList.lock ).fail( function() {
+			state = "rejected";
+		}, doneList.disable, progressList.lock );
+
 		// Call given func if any
 		if ( func ) {
 			func.call( deferred, deferred );
 		}
+
+		// All done!
 		return deferred;
 	},
 
 	// Deferred helper
 	when: function( firstParam ) {
-		var args = arguments,
+		var args = sliceDeferred.call( arguments, 0 ),
 			i = 0,
 			length = args.length,
+			pValues = new Array( length ),
 			count = length,
+			pCount = length,
 			deferred = length <= 1 && firstParam && jQuery.isFunction( firstParam.promise ) ?
 				firstParam :
-				jQuery.Deferred();
+				jQuery.Deferred(),
+			promise = deferred.promise();
 		function resolveFunc( i ) {
 			return function( value ) {
 				args[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
 				if ( !( --count ) ) {
-					// Strange bug in FF4:
-					// Values changed onto the arguments object sometimes end up as undefined values
-					// outside the $.when method. Cloning the object into a fresh array solves the issue
-					deferred.resolveWith( deferred, sliceDeferred.call( args, 0 ) );
+					deferred.resolveWith( deferred, args );
 				}
 			};
 		}
+		function progressFunc( i ) {
+			return function( value ) {
+				pValues[ i ] = arguments.length > 1 ? sliceDeferred.call( arguments, 0 ) : value;
+				deferred.notifyWith( promise, pValues );
+			};
+		}
 		if ( length > 1 ) {
-			for( ; i < length; i++ ) {
-				if ( args[ i ] && jQuery.isFunction( args[ i ].promise ) ) {
-					args[ i ].promise().then( resolveFunc(i), deferred.reject );
+			for ( ; i < length; i++ ) {
+				if ( args[ i ] && args[ i ].promise && jQuery.isFunction( args[ i ].promise ) ) {
+					args[ i ].promise().then( resolveFunc(i), deferred.reject, progressFunc(i) );
 				} else {
 					--count;
 				}
@@ -1141,33 +1350,29 @@ jQuery.extend({
 		} else if ( deferred !== firstParam ) {
 			deferred.resolveWith( deferred, length ? [ firstParam ] : [] );
 		}
-		return deferred.promise();
+		return promise;
 	}
 });
 
 
 
+
 jQuery.support = (function() {
 
-	var div = document.createElement( "div" ),
-		documentElement = document.documentElement,
+	var support,
 		all,
 		a,
 		select,
 		opt,
 		input,
-		marginDiv,
-		support,
 		fragment,
-		body,
-		testElementParent,
-		testElement,
-		testElementStyle,
 		tds,
 		events,
 		eventName,
 		i,
-		isSupported;
+		isSupported,
+		div = document.createElement( "div" ),
+		documentElement = document.documentElement;
 
 	// Preliminary tests
 	div.setAttribute("className", "t");
@@ -1192,11 +1397,11 @@ jQuery.support = (function() {
 
 		// Make sure that tbody elements aren't automatically inserted
 		// IE will insert them into empty tables
-		tbody: !div.getElementsByTagName( "tbody" ).length,
+		tbody: !div.getElementsByTagName("tbody").length,
 
 		// Make sure that link elements get serialized correctly by innerHTML
 		// This requires a wrapper element in IE
-		htmlSerialize: !!div.getElementsByTagName( "link" ).length,
+		htmlSerialize: !!div.getElementsByTagName("link").length,
 
 		// Get the style information from getAttribute
 		// (IE uses .cssText instead)
@@ -1204,12 +1409,12 @@ jQuery.support = (function() {
 
 		// Make sure that URLs aren't manipulated
 		// (IE normalizes it by default)
-		hrefNormalized: ( a.getAttribute( "href" ) === "/a" ),
+		hrefNormalized: ( a.getAttribute("href") === "/a" ),
 
 		// Make sure that element opacity exists
 		// (IE uses filter instead)
 		// Use a regex to work around a WebKit issue. See #5145
-		opacity: /^0.55$/.test( a.style.opacity ),
+		opacity: /^0.55/.test( a.style.opacity ),
 
 		// Verify style float existence
 		// (IE uses styleFloat instead of cssFloat)
@@ -1227,6 +1432,13 @@ jQuery.support = (function() {
 		// Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7)
 		getSetAttribute: div.className !== "t",
 
+		// Tests for enctype support on a form(#6743)
+		enctype: !!document.createElement("form").enctype,
+
+		// Makes sure cloning an html5 element does not cause problems
+		// Where outerHTML is undefined, this still works
+		html5Clone: document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav></:nav>",
+
 		// Will be defined later
 		submitBubbles: true,
 		changeBubbles: true,
@@ -1235,9 +1447,13 @@ jQuery.support = (function() {
 		noCloneEvent: true,
 		inlineBlockNeedsLayout: false,
 		shrinkWrapBlocks: false,
-		reliableMarginRight: true
+		reliableMarginRight: true,
+		pixelMargin: true
 	};
 
+	// jQuery.boxModel DEPRECATED in 1.3, use jQuery.support.boxModel instead
+	jQuery.boxModel = support.boxModel = (document.compatMode === "CSS1Compat");
+
 	// Make sure checked status is properly cloned
 	input.checked = true;
 	support.noCloneChecked = input.cloneNode( true ).checked;
@@ -1264,7 +1480,7 @@ jQuery.support = (function() {
 		div.cloneNode( true ).fireEvent( "onclick" );
 	}
 
-	// Check if a radio maintains it's value
+	// Check if a radio maintains its value
 	// after being appended to the DOM
 	input = document.createElement("input");
 	input.value = "t";
@@ -1272,115 +1488,36 @@ jQuery.support = (function() {
 	support.radioValue = input.value === "t";
 
 	input.setAttribute("checked", "checked");
+
+	// #11217 - WebKit loses check when the name is after the checked attribute
+	input.setAttribute( "name", "t" );
+
 	div.appendChild( input );
 	fragment = document.createDocumentFragment();
-	fragment.appendChild( div.firstChild );
+	fragment.appendChild( div.lastChild );
 
 	// WebKit doesn't clone checked state correctly in fragments
 	support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked;
 
-	div.innerHTML = "";
-
-	// Figure out if the W3C box model works as expected
-	div.style.width = div.style.paddingLeft = "1px";
-
-	body = document.getElementsByTagName( "body" )[ 0 ];
-	// We use our own, invisible, body unless the body is already present
-	// in which case we use a div (#9239)
-	testElement = document.createElement( body ? "div" : "body" );
-	testElementStyle = {
-		visibility: "hidden",
-		width: 0,
-		height: 0,
-		border: 0,
-		margin: 0
-	};
-	if ( body ) {
-		jQuery.extend( testElementStyle, {
-			position: "absolute",
-			left: -1000,
-			top: -1000
-		});
-	}
-	for ( i in testElementStyle ) {
-		testElement.style[ i ] = testElementStyle[ i ];
-	}
-	testElement.appendChild( div );
-	testElementParent = body || documentElement;
-	testElementParent.insertBefore( testElement, testElementParent.firstChild );
-
 	// Check if a disconnected checkbox will retain its checked
 	// value of true after appended to the DOM (IE6/7)
 	support.appendChecked = input.checked;
 
-	support.boxModel = div.offsetWidth === 2;
-
-	if ( "zoom" in div.style ) {
-		// Check if natively block-level elements act like inline-block
-		// elements when setting their display to 'inline' and giving
-		// them layout
-		// (IE < 8 does this)
-		div.style.display = "inline";
-		div.style.zoom = 1;
-		support.inlineBlockNeedsLayout = ( div.offsetWidth === 2 );
-
-		// Check if elements with layout shrink-wrap their children
-		// (IE 6 does this)
-		div.style.display = "";
-		div.innerHTML = "<div style='width:4px;'></div>";
-		support.shrinkWrapBlocks = ( div.offsetWidth !== 2 );
-	}
-
-	div.innerHTML = "<table><tr><td style='padding:0;border:0;display:none'></td><td>t</td></tr></table>";
-	tds = div.getElementsByTagName( "td" );
-
-	// Check if table cells still have offsetWidth/Height when they are set
-	// to display:none and there are still other visible table cells in a
-	// table row; if so, offsetWidth/Height are not reliable for use when
-	// determining if an element has been hidden directly using
-	// display:none (it is still safe to use offsets if a parent element is
-	// hidden; don safety goggles and see bug #4512 for more information).
-	// (only IE 8 fails this test)
-	isSupported = ( tds[ 0 ].offsetHeight === 0 );
-
-	tds[ 0 ].style.display = "";
-	tds[ 1 ].style.display = "none";
-
-	// Check if empty table cells still have offsetWidth/Height
-	// (IE < 8 fail this test)
-	support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 );
-	div.innerHTML = "";
-
-	// Check if div with explicit width and no margin-right incorrectly
-	// gets computed margin-right based on width of container. For more
-	// info see bug #3333
-	// Fails in WebKit before Feb 2011 nightlies
-	// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
-	if ( document.defaultView && document.defaultView.getComputedStyle ) {
-		marginDiv = document.createElement( "div" );
-		marginDiv.style.width = "0";
-		marginDiv.style.marginRight = "0";
-		div.appendChild( marginDiv );
-		support.reliableMarginRight =
-			( parseInt( ( document.defaultView.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0;
-	}
-
-	// Remove the body element we added
-	testElement.innerHTML = "";
-	testElementParent.removeChild( testElement );
+	fragment.removeChild( input );
+	fragment.appendChild( div );
 
 	// Technique from Juriy Zaytsev
-	// http://thinkweb2.com/projects/prototype/detecting-event-support-without-browser-sniffing/
+	// http://perfectionkills.com/detecting-event-support-without-browser-sniffing/
 	// We only care about the case where non-standard event systems
 	// are used, namely in IE. Short-circuiting here helps us to
 	// avoid an eval call (in setAttribute) which can cause CSP
 	// to go haywire. See: https://developer.mozilla.org/en/Security/CSP
 	if ( div.attachEvent ) {
-		for( i in {
+		for ( i in {
 			submit: 1,
 			change: 1,
 			focusin: 1
-		} ) {
+		}) {
 			eventName = "on" + i;
 			isSupported = ( eventName in div );
 			if ( !isSupported ) {
@@ -1391,20 +1528,143 @@ jQuery.support = (function() {
 		}
 	}
 
-	// Null connected elements to avoid leaks in IE
-	testElement = fragment = select = opt = body = marginDiv = div = input = null;
+	fragment.removeChild( div );
+
+	// Null elements to avoid leaks in IE
+	fragment = select = opt = div = input = null;
+
+	// Run tests that need a body at doc ready
+	jQuery(function() {
+		var container, outer, inner, table, td, offsetSupport,
+			marginDiv, conMarginTop, style, html, positionTopLeftWidthHeight,
+			paddingMarginBorderVisibility, paddingMarginBorder,
+			body = document.getElementsByTagName("body")[0];
+
+		if ( !body ) {
+			// Return for frameset docs that don't have a body
+			return;
+		}
+
+		conMarginTop = 1;
+		paddingMarginBorder = "padding:0;margin:0;border:";
+		positionTopLeftWidthHeight = "position:absolute;top:0;left:0;width:1px;height:1px;";
+		paddingMarginBorderVisibility = paddingMarginBorder + "0;visibility:hidden;";
+		style = "style='" + positionTopLeftWidthHeight + paddingMarginBorder + "5px solid #000;";
+		html = "<div " + style + "display:block;'><div style='" + paddingMarginBorder + "0;display:block;overflow:hidden;'></div></div>" +
+			"<table " + style + "' cellpadding='0' cellspacing='0'>" +
+			"<tr><td></td></tr></table>";
+
+		container = document.createElement("div");
+		container.style.cssText = paddingMarginBorderVisibility + "width:0;height:0;position:static;top:0;margin-top:" + conMarginTop + "px";
+		body.insertBefore( container, body.firstChild );
+
+		// Construct the test element
+		div = document.createElement("div");
+		container.appendChild( div );
+
+		// Check if table cells still have offsetWidth/Height when they are set
+		// to display:none and there are still other visible table cells in a
+		// table row; if so, offsetWidth/Height are not reliable for use when
+		// determining if an element has been hidden directly using
+		// display:none (it is still safe to use offsets if a parent element is
+		// hidden; don safety goggles and see bug #4512 for more information).
+		// (only IE 8 fails this test)
+		div.innerHTML = "<table><tr><td style='" + paddingMarginBorder + "0;display:none'></td><td>t</td></tr></table>";
+		tds = div.getElementsByTagName( "td" );
+		isSupported = ( tds[ 0 ].offsetHeight === 0 );
+
+		tds[ 0 ].style.display = "";
+		tds[ 1 ].style.display = "none";
+
+		// Check if empty table cells still have offsetWidth/Height
+		// (IE <= 8 fail this test)
+		support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 );
+
+		// Check if div with explicit width and no margin-right incorrectly
+		// gets computed margin-right based on width of container. For more
+		// info see bug #3333
+		// Fails in WebKit before Feb 2011 nightlies
+		// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+		if ( window.getComputedStyle ) {
+			div.innerHTML = "";
+			marginDiv = document.createElement( "div" );
+			marginDiv.style.width = "0";
+			marginDiv.style.marginRight = "0";
+			div.style.width = "2px";
+			div.appendChild( marginDiv );
+			support.reliableMarginRight =
+				( parseInt( ( window.getComputedStyle( marginDiv, null ) || { marginRight: 0 } ).marginRight, 10 ) || 0 ) === 0;
+		}
+
+		if ( typeof div.style.zoom !== "undefined" ) {
+			// Check if natively block-level elements act like inline-block
+			// elements when setting their display to 'inline' and giving
+			// them layout
+			// (IE < 8 does this)
+			div.innerHTML = "";
+			div.style.width = div.style.padding = "1px";
+			div.style.border = 0;
+			div.style.overflow = "hidden";
+			div.style.display = "inline";
+			div.style.zoom = 1;
+			support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 );
+
+			// Check if elements with layout shrink-wrap their children
+			// (IE 6 does this)
+			div.style.display = "block";
+			div.style.overflow = "visible";
+			div.innerHTML = "<div style='width:5px;'></div>";
+			support.shrinkWrapBlocks = ( div.offsetWidth !== 3 );
+		}
+
+		div.style.cssText = positionTopLeftWidthHeight + paddingMarginBorderVisibility;
+		div.innerHTML = html;
+
+		outer = div.firstChild;
+		inner = outer.firstChild;
+		td = outer.nextSibling.firstChild.firstChild;
+
+		offsetSupport = {
+			doesNotAddBorder: ( inner.offsetTop !== 5 ),
+			doesAddBorderForTableAndCells: ( td.offsetTop === 5 )
+		};
+
+		inner.style.position = "fixed";
+		inner.style.top = "20px";
+
+		// safari subtracts parent border width here which is 5px
+		offsetSupport.fixedPosition = ( inner.offsetTop === 20 || inner.offsetTop === 15 );
+		inner.style.position = inner.style.top = "";
+
+		outer.style.overflow = "hidden";
+		outer.style.position = "relative";
+
+		offsetSupport.subtractsBorderForOverflowNotVisible = ( inner.offsetTop === -5 );
+		offsetSupport.doesNotIncludeMarginInBodyOffset = ( body.offsetTop !== conMarginTop );
+
+		if ( window.getComputedStyle ) {
+			div.style.marginTop = "1%";
+			support.pixelMargin = ( window.getComputedStyle( div, null ) || { marginTop: 0 } ).marginTop !== "1%";
+		}
+
+		if ( typeof container.style.zoom !== "undefined" ) {
+			container.style.zoom = 1;
+		}
+
+		body.removeChild( container );
+		marginDiv = div = container = null;
+
+		jQuery.extend( support, offsetSupport );
+	});
 
 	return support;
 })();
 
-// Keep track of boxModel
-jQuery.boxModel = jQuery.support.boxModel;
-
 
 
 
 var rbrace = /^(?:\{.*\}|\[.*\])$/,
-	rmultiDash = /([a-z])([A-Z])/g;
+	rmultiDash = /([A-Z])/g;
 
 jQuery.extend({
 	cache: {},
@@ -1427,7 +1687,6 @@ jQuery.extend({
 
 	hasData: function( elem ) {
 		elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ];
-
 		return !!elem && !isEmptyDataObject( elem );
 	},
 
@@ -1436,7 +1695,9 @@ jQuery.extend({
 			return;
 		}
 
-		var internalKey = jQuery.expando, getByName = typeof name === "string", thisCache,
+		var privateCache, thisCache, ret,
+			internalKey = jQuery.expando,
+			getByName = typeof name === "string",
 
 			// We have to handle DOM nodes and JS objects differently because IE6-7
 			// can't GC object references properly across the DOM-JS boundary
@@ -1448,11 +1709,12 @@ jQuery.extend({
 
 			// Only defining an ID for JS objects if its cache already exists allows
 			// the code to shortcut on the same path as a DOM node with no cache
-			id = isNode ? elem[ jQuery.expando ] : elem[ jQuery.expando ] && jQuery.expando;
+			id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey,
+			isEvents = name === "events";
 
 		// Avoid doing any more work than we need to when trying to get data on an
 		// object that has no data at all
-		if ( (!id || (pvt && id && !cache[ id ][ internalKey ])) && getByName && data === undefined ) {
+		if ( (!id || !cache[id] || (!isEvents && !pvt && !cache[id].data)) && getByName && data === undefined ) {
 			return;
 		}
 
@@ -1460,18 +1722,17 @@ jQuery.extend({
 			// Only DOM nodes need a new unique ID for each element since their data
 			// ends up in the global cache
 			if ( isNode ) {
-				elem[ jQuery.expando ] = id = ++jQuery.uuid;
+				elem[ internalKey ] = id = ++jQuery.uuid;
 			} else {
-				id = jQuery.expando;
+				id = internalKey;
 			}
 		}
 
 		if ( !cache[ id ] ) {
 			cache[ id ] = {};
 
-			// TODO: This is a hack for 1.5 ONLY. Avoids exposing jQuery
-			// metadata on plain JS objects when the object is serialized using
-			// JSON.stringify
+			// Avoids exposing jQuery metadata on plain JS objects when the object
+			// is serialized using JSON.stringify
 			if ( !isNode ) {
 				cache[ id ].toJSON = jQuery.noop;
 			}
@@ -1481,40 +1742,53 @@ jQuery.extend({
 		// shallow copied over onto the existing cache
 		if ( typeof name === "object" || typeof name === "function" ) {
 			if ( pvt ) {
-				cache[ id ][ internalKey ] = jQuery.extend(cache[ id ][ internalKey ], name);
+				cache[ id ] = jQuery.extend( cache[ id ], name );
 			} else {
-				cache[ id ] = jQuery.extend(cache[ id ], name);
+				cache[ id ].data = jQuery.extend( cache[ id ].data, name );
 			}
 		}
 
-		thisCache = cache[ id ];
+		privateCache = thisCache = cache[ id ];
 
-		// Internal jQuery data is stored in a separate object inside the object's data
+		// jQuery data() is stored in a separate object inside the object's internal data
 		// cache in order to avoid key collisions between internal data and user-defined
-		// data
-		if ( pvt ) {
-			if ( !thisCache[ internalKey ] ) {
-				thisCache[ internalKey ] = {};
+		// data.
+		if ( !pvt ) {
+			if ( !thisCache.data ) {
+				thisCache.data = {};
 			}
 
-			thisCache = thisCache[ internalKey ];
+			thisCache = thisCache.data;
 		}
 
 		if ( data !== undefined ) {
 			thisCache[ jQuery.camelCase( name ) ] = data;
 		}
 
-		// TODO: This is a hack for 1.5 ONLY. It will be removed in 1.6. Users should
-		// not attempt to inspect the internal events object using jQuery.data, as this
-		// internal data object is undocumented and subject to change.
-		if ( name === "events" && !thisCache[name] ) {
-			return thisCache[ internalKey ] && thisCache[ internalKey ].events;
+		// Users should not attempt to inspect the internal events object using jQuery.data,
+		// it is undocumented and subject to change. But does anyone listen? No.
+		if ( isEvents && !thisCache[ name ] ) {
+			return privateCache.events;
 		}
 
-		return getByName ? 
-			// Check for both converted-to-camel and non-converted data property names
-			thisCache[ jQuery.camelCase( name ) ] || thisCache[ name ] :
-			thisCache;
+		// Check for both converted-to-camel and non-converted data property names
+		// If a data property was specified
+		if ( getByName ) {
+
+			// First Try to find as-is property data
+			ret = thisCache[ name ];
+
+			// Test for null|undefined property data
+			if ( ret == null ) {
+
+				// Try to find the camelCased property
+				ret = thisCache[ jQuery.camelCase( name ) ];
+			}
+		} else {
+			ret = thisCache;
+		}
+
+		return ret;
 	},
 
 	removeData: function( elem, name, pvt /* Internal Use Only */ ) {
@@ -1522,13 +1796,18 @@ jQuery.extend({
 			return;
 		}
 
-		var internalKey = jQuery.expando, isNode = elem.nodeType,
+		var thisCache, i, l,
+
+			// Reference to internal data cache key
+			internalKey = jQuery.expando,
+
+			isNode = elem.nodeType,
 
 			// See jQuery.data for more information
 			cache = isNode ? jQuery.cache : elem,
 
 			// See jQuery.data for more information
-			id = isNode ? elem[ jQuery.expando ] : jQuery.expando;
+			id = isNode ? elem[ internalKey ] : internalKey;
 
 		// If there is already no cache entry for this object, there is no
 		// purpose in continuing
@@ -1537,22 +1816,44 @@ jQuery.extend({
 		}
 
 		if ( name ) {
-			var thisCache = pvt ? cache[ id ][ internalKey ] : cache[ id ];
+
+			thisCache = pvt ? cache[ id ] : cache[ id ].data;
 
 			if ( thisCache ) {
-				delete thisCache[ name ];
+
+				// Support array or space separated string names for data keys
+				if ( !jQuery.isArray( name ) ) {
+
+					// try the string as a key before any manipulation
+					if ( name in thisCache ) {
+						name = [ name ];
+					} else {
+
+						// split the camel cased version by spaces unless a key with the spaces exists
+						name = jQuery.camelCase( name );
+						if ( name in thisCache ) {
+							name = [ name ];
+						} else {
+							name = name.split( " " );
+						}
+					}
+				}
+
+				for ( i = 0, l = name.length; i < l; i++ ) {
+					delete thisCache[ name[i] ];
+				}
 
 				// If there is no data left in the cache, we want to continue
 				// and let the cache object itself get destroyed
-				if ( !isEmptyDataObject(thisCache) ) {
+				if ( !( pvt ? isEmptyDataObject : jQuery.isEmptyObject )( thisCache ) ) {
 					return;
 				}
 			}
 		}
 
 		// See jQuery.data for more information
-		if ( pvt ) {
-			delete cache[ id ][ internalKey ];
+		if ( !pvt ) {
+			delete cache[ id ].data;
 
 			// Don't destroy the parent cache unless the internal data object
 			// had been the only thing left in it
@@ -1561,43 +1862,28 @@ jQuery.extend({
 			}
 		}
 
-		var internalCache = cache[ id ][ internalKey ];
-
 		// Browsers that fail expando deletion also refuse to delete expandos on
 		// the window, but it will allow it on all other JS objects; other browsers
 		// don't care
-		if ( jQuery.support.deleteExpando || cache != window ) {
+		// Ensure that `cache` is not a window object #10080
+		if ( jQuery.support.deleteExpando || !cache.setInterval ) {
 			delete cache[ id ];
 		} else {
 			cache[ id ] = null;
 		}
 
-		// We destroyed the entire user cache at once because it's faster than
-		// iterating through each key, but we need to continue to persist internal
-		// data if it existed
-		if ( internalCache ) {
-			cache[ id ] = {};
-			// TODO: This is a hack for 1.5 ONLY. Avoids exposing jQuery
-			// metadata on plain JS objects when the object is serialized using
-			// JSON.stringify
-			if ( !isNode ) {
-				cache[ id ].toJSON = jQuery.noop;
-			}
-
-			cache[ id ][ internalKey ] = internalCache;
-
-		// Otherwise, we need to eliminate the expando on the node to avoid
+		// We destroyed the cache and need to eliminate the expando on the node to avoid
 		// false lookups in the cache for entries that no longer exist
-		} else if ( isNode ) {
+		if ( isNode ) {
 			// IE does not allow us to delete expando properties from nodes,
 			// nor does it have a removeAttribute function on Document nodes;
 			// we must handle all of these cases
 			if ( jQuery.support.deleteExpando ) {
-				delete elem[ jQuery.expando ];
+				delete elem[ internalKey ];
 			} else if ( elem.removeAttribute ) {
-				elem.removeAttribute( jQuery.expando );
+				elem.removeAttribute( internalKey );
 			} else {
-				elem[ jQuery.expando ] = null;
+				elem[ internalKey ] = null;
 			}
 		}
 	},
@@ -1623,60 +1909,70 @@ jQuery.extend({
 
 jQuery.fn.extend({
 	data: function( key, value ) {
-		var data = null;
+		var parts, part, attr, name, l,
+			elem = this[0],
+			i = 0,
+			data = null;
 
-		if ( typeof key === "undefined" ) {
+		// Gets all values
+		if ( key === undefined ) {
 			if ( this.length ) {
-				data = jQuery.data( this[0] );
+				data = jQuery.data( elem );
 
-				if ( this[0].nodeType === 1 ) {
-			    var attr = this[0].attributes, name;
-					for ( var i = 0, l = attr.length; i < l; i++ ) {
+				if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) {
+					attr = elem.attributes;
+					for ( l = attr.length; i < l; i++ ) {
 						name = attr[i].name;
 
 						if ( name.indexOf( "data-" ) === 0 ) {
 							name = jQuery.camelCase( name.substring(5) );
 
-							dataAttr( this[0], name, data[ name ] );
+							dataAttr( elem, name, data[ name ] );
 						}
 					}
+					jQuery._data( elem, "parsedAttrs", true );
 				}
 			}
 
 			return data;
+		}
 
-		} else if ( typeof key === "object" ) {
+		// Sets multiple values
+		if ( typeof key === "object" ) {
 			return this.each(function() {
 				jQuery.data( this, key );
 			});
 		}
 
-		var parts = key.split(".");
+		parts = key.split( ".", 2 );
 		parts[1] = parts[1] ? "." + parts[1] : "";
+		part = parts[1] + "!";
 
-		if ( value === undefined ) {
-			data = this.triggerHandler("getData" + parts[1] + "!", [parts[0]]);
+		return jQuery.access( this, function( value ) {
 
-			// Try to fetch any internally stored data first
-			if ( data === undefined && this.length ) {
-				data = jQuery.data( this[0], key );
-				data = dataAttr( this[0], key, data );
+			if ( value === undefined ) {
+				data = this.triggerHandler( "getData" + part, [ parts[0] ] );
+
+				// Try to fetch any internally stored data first
+				if ( data === undefined && elem ) {
+					data = jQuery.data( elem, key );
+					data = dataAttr( elem, key, data );
+				}
+
+				return data === undefined && parts[1] ?
+					this.data( parts[0] ) :
+					data;
 			}
 
-			return data === undefined && parts[1] ?
-				this.data( parts[0] ) :
-				data;
+			parts[1] = value;
+			this.each(function() {
+				var self = jQuery( this );
 
-		} else {
-			return this.each(function() {
-				var $this = jQuery( this ),
-					args = [ parts[0], value ];
-
-				$this.triggerHandler( "setData" + parts[1] + "!", args );
+				self.triggerHandler( "setData" + part, parts );
 				jQuery.data( this, key, value );
-				$this.triggerHandler( "changeData" + parts[1] + "!", args );
+				self.triggerHandler( "changeData" + part, parts );
 			});
-		}
+		}, null, value, arguments.length > 1, null, false );
 	},
 
 	removeData: function( key ) {
@@ -1690,7 +1986,8 @@ function dataAttr( elem, key, data ) {
 	// If nothing was found internally, try to fetch any
 	// data from the HTML5 data-* attribute
 	if ( data === undefined && elem.nodeType === 1 ) {
-		var name = "data-" + key.replace( rmultiDash, "$1-$2" ).toLowerCase();
+
+		var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase();
 
 		data = elem.getAttribute( name );
 
@@ -1699,7 +1996,7 @@ function dataAttr( elem, key, data ) {
 				data = data === "true" ? true :
 				data === "false" ? false :
 				data === "null" ? null :
-				!jQuery.isNaN( data ) ? parseFloat( data ) :
+				jQuery.isNumeric( data ) ? +data :
 					rbrace.test( data ) ? jQuery.parseJSON( data ) :
 					data;
 			} catch( e ) {}
@@ -1715,11 +2012,14 @@ function dataAttr( elem, key, data ) {
 	return data;
 }
 
-// TODO: This is a hack for 1.5 ONLY to allow objects with a single toJSON
-// property to be considered empty objects; this property always exists in
-// order to make sure JSON.stringify does not expose internal metadata
+// checks a cache object for emptiness
 function isEmptyDataObject( obj ) {
 	for ( var name in obj ) {
+
+		// if the public data object is empty, the private is still empty
+		if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) {
+			continue;
+		}
 		if ( name !== "toJSON" ) {
 			return false;
 		}
@@ -1735,17 +2035,17 @@ function handleQueueMarkDefer( elem, type, src ) {
 	var deferDataKey = type + "defer",
 		queueDataKey = type + "queue",
 		markDataKey = type + "mark",
-		defer = jQuery.data( elem, deferDataKey, undefined, true );
+		defer = jQuery._data( elem, deferDataKey );
 	if ( defer &&
-		( src === "queue" || !jQuery.data( elem, queueDataKey, undefined, true ) ) &&
-		( src === "mark" || !jQuery.data( elem, markDataKey, undefined, true ) ) ) {
+		( src === "queue" || !jQuery._data(elem, queueDataKey) ) &&
+		( src === "mark" || !jQuery._data(elem, markDataKey) ) ) {
 		// Give room for hard-coded callbacks to fire first
 		// and eventually mark/queue something else on the element
 		setTimeout( function() {
-			if ( !jQuery.data( elem, queueDataKey, undefined, true ) &&
-				!jQuery.data( elem, markDataKey, undefined, true ) ) {
+			if ( !jQuery._data( elem, queueDataKey ) &&
+				!jQuery._data( elem, markDataKey ) ) {
 				jQuery.removeData( elem, deferDataKey, true );
-				defer.resolve();
+				defer.fire();
 			}
 		}, 0 );
 	}
@@ -1755,8 +2055,8 @@ jQuery.extend({
 
 	_mark: function( elem, type ) {
 		if ( elem ) {
-			type = (type || "fx") + "mark";
-			jQuery.data( elem, type, (jQuery.data(elem,type,undefined,true) || 0) + 1, true );
+			type = ( type || "fx" ) + "mark";
+			jQuery._data( elem, type, (jQuery._data( elem, type ) || 0) + 1 );
 		}
 	},
 
@@ -1769,9 +2069,9 @@ jQuery.extend({
 		if ( elem ) {
 			type = type || "fx";
 			var key = type + "mark",
-				count = force ? 0 : ( (jQuery.data( elem, key, undefined, true) || 1 ) - 1 );
+				count = force ? 0 : ( (jQuery._data( elem, key ) || 1) - 1 );
 			if ( count ) {
-				jQuery.data( elem, key, count, true );
+				jQuery._data( elem, key, count );
 			} else {
 				jQuery.removeData( elem, key, true );
 				handleQueueMarkDefer( elem, type, "mark" );
@@ -1780,13 +2080,15 @@ jQuery.extend({
 	},
 
 	queue: function( elem, type, data ) {
+		var q;
 		if ( elem ) {
-			type = (type || "fx") + "queue";
-			var q = jQuery.data( elem, type, undefined, true );
+			type = ( type || "fx" ) + "queue";
+			q = jQuery._data( elem, type );
+
 			// Speed up dequeue by getting out quickly if this is just a lookup
 			if ( data ) {
 				if ( !q || jQuery.isArray(data) ) {
-					q = jQuery.data( elem, type, jQuery.makeArray(data), true );
+					q = jQuery._data( elem, type, jQuery.makeArray(data) );
 				} else {
 					q.push( data );
 				}
@@ -1800,7 +2102,7 @@ jQuery.extend({
 
 		var queue = jQuery.queue( elem, type ),
 			fn = queue.shift(),
-			defer;
+			hooks = {};
 
 		// If the fx queue is dequeued, always remove the progress sentinel
 		if ( fn === "inprogress" ) {
@@ -1811,16 +2113,17 @@ jQuery.extend({
 			// Add a progress sentinel to prevent the fx queue from being
 			// automatically dequeued
 			if ( type === "fx" ) {
-				queue.unshift("inprogress");
+				queue.unshift( "inprogress" );
 			}
 
-			fn.call(elem, function() {
-				jQuery.dequeue(elem, type);
-			});
+			jQuery._data( elem, type + ".run", hooks );
+			fn.call( elem, function() {
+				jQuery.dequeue( elem, type );
+			}, hooks );
 		}
 
 		if ( !queue.length ) {
-			jQuery.removeData( elem, type + "queue", true );
+			jQuery.removeData( elem, type + "queue " + type + ".run", true );
 			handleQueueMarkDefer( elem, type, "queue" );
 		}
 	}
@@ -1828,21 +2131,27 @@ jQuery.extend({
 
 jQuery.fn.extend({
 	queue: function( type, data ) {
+		var setter = 2;
+
 		if ( typeof type !== "string" ) {
 			data = type;
 			type = "fx";
+			setter--;
 		}
 
-		if ( data === undefined ) {
+		if ( arguments.length < setter ) {
 			return jQuery.queue( this[0], type );
 		}
-		return this.each(function() {
-			var queue = jQuery.queue( this, type, data );
 
-			if ( type === "fx" && queue[0] !== "inprogress" ) {
-				jQuery.dequeue( this, type );
-			}
-		});
+		return data === undefined ?
+			this :
+			this.each(function() {
+				var queue = jQuery.queue( this, type, data );
+
+				if ( type === "fx" && queue[0] !== "inprogress" ) {
+					jQuery.dequeue( this, type );
+				}
+			});
 	},
 	dequeue: function( type ) {
 		return this.each(function() {
@@ -1852,14 +2161,14 @@ jQuery.fn.extend({
 	// Based off of the plugin by Clint Helfers, with permission.
 	// http://blindsignals.com/index.php/2009/07/jquery-delay/
 	delay: function( time, type ) {
-		time = jQuery.fx ? jQuery.fx.speeds[time] || time : time;
+		time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time;
 		type = type || "fx";
 
-		return this.queue( type, function() {
-			var elem = this;
-			setTimeout(function() {
-				jQuery.dequeue( elem, type );
-			}, time );
+		return this.queue( type, function( next, hooks ) {
+			var timeout = setTimeout( next, time );
+			hooks.stop = function() {
+				clearTimeout( timeout );
+			};
 		});
 	},
 	clearQueue: function( type ) {
@@ -1890,13 +2199,13 @@ jQuery.fn.extend({
 			if (( tmp = jQuery.data( elements[ i ], deferDataKey, undefined, true ) ||
 					( jQuery.data( elements[ i ], queueDataKey, undefined, true ) ||
 						jQuery.data( elements[ i ], markDataKey, undefined, true ) ) &&
-					jQuery.data( elements[ i ], deferDataKey, jQuery._Deferred(), true ) )) {
+					jQuery.data( elements[ i ], deferDataKey, jQuery.Callbacks( "once memory" ), true ) )) {
 				count++;
-				tmp.done( resolve );
+				tmp.add( resolve );
 			}
 		}
 		resolve();
-		return defer.promise();
+		return defer.promise( object );
 	}
 });
 
@@ -1910,12 +2219,12 @@ var rclass = /[\n\t\r]/g,
 	rfocusable = /^(?:button|input|object|select|textarea)$/i,
 	rclickable = /^a(?:rea)?$/i,
 	rboolean = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,
-	rinvalidChar = /\:|^on/,
-	formHook, boolHook;
+	getSetAttribute = jQuery.support.getSetAttribute,
+	nodeHook, boolHook, fixSpecified;
 
 jQuery.fn.extend({
 	attr: function( name, value ) {
-		return jQuery.access( this, name, value, true, jQuery.attr );
+		return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 );
 	},
 
 	removeAttr: function( name ) {
@@ -1923,11 +2232,11 @@ jQuery.fn.extend({
 			jQuery.removeAttr( this, name );
 		});
 	},
-	
+
 	prop: function( name, value ) {
-		return jQuery.access( this, name, value, true, jQuery.prop );
+		return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 );
 	},
-	
+
 	removeProp: function( name ) {
 		name = jQuery.propFix[ name ] || name;
 		return this.each(function() {
@@ -1986,7 +2295,7 @@ jQuery.fn.extend({
 		}
 
 		if ( (value && typeof value === "string") || value === undefined ) {
-			classNames = (value || "").split( rspace );
+			classNames = ( value || "" ).split( rspace );
 
 			for ( i = 0, l = this.length; i < l; i++ ) {
 				elem = this[ i ];
@@ -2047,9 +2356,11 @@ jQuery.fn.extend({
 	},
 
 	hasClass: function( selector ) {
-		var className = " " + selector + " ";
-		for ( var i = 0, l = this.length; i < l; i++ ) {
-			if ( (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) {
+		var className = " " + selector + " ",
+			i = 0,
+			l = this.length;
+		for ( ; i < l; i++ ) {
+			if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) > -1 ) {
 				return true;
 			}
 		}
@@ -2058,12 +2369,12 @@ jQuery.fn.extend({
 	},
 
 	val: function( value ) {
-		var hooks, ret,
+		var hooks, ret, isFunction,
 			elem = this[0];
-		
+
 		if ( !arguments.length ) {
 			if ( elem ) {
-				hooks = jQuery.valHooks[ elem.nodeName.toLowerCase() ] || jQuery.valHooks[ elem.type ];
+				hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ];
 
 				if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) {
 					return ret;
@@ -2071,17 +2382,17 @@ jQuery.fn.extend({
 
 				ret = elem.value;
 
-				return typeof ret === "string" ? 
+				return typeof ret === "string" ?
 					// handle most common string cases
-					ret.replace(rreturn, "") : 
+					ret.replace(rreturn, "") :
 					// handle cases where value is null/undef or number
 					ret == null ? "" : ret;
 			}
 
-			return undefined;
+			return;
 		}
 
-		var isFunction = jQuery.isFunction( value );
+		isFunction = jQuery.isFunction( value );
 
 		return this.each(function( i ) {
 			var self = jQuery(this), val;
@@ -2107,7 +2418,7 @@ jQuery.fn.extend({
 				});
 			}
 
-			hooks = jQuery.valHooks[ this.nodeName.toLowerCase() ] || jQuery.valHooks[ this.type ];
+			hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ];
 
 			// If set returns undefined, fall back to normal setting
 			if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) {
@@ -2129,7 +2440,7 @@ jQuery.extend({
 		},
 		select: {
 			get: function( elem ) {
-				var value,
+				var value, i, max, option,
 					index = elem.selectedIndex,
 					values = [],
 					options = elem.options,
@@ -2141,8 +2452,10 @@ jQuery.extend({
 				}
 
 				// Loop through all the selected options
-				for ( var i = one ? index : 0, max = one ? index + 1 : options.length; i < max; i++ ) {
-					var option = options[ i ];
+				i = one ? index : 0;
+				max = one ? index + 1 : options.length;
+				for ( ; i < max; i++ ) {
+					option = options[ i ];
 
 					// Don't return options that are disabled or in a disabled optgroup
 					if ( option.selected && (jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null) &&
@@ -2194,18 +2507,14 @@ jQuery.extend({
 		height: true,
 		offset: true
 	},
-	
-	attrFix: {
-		// Always normalize to ensure hook usage
-		tabindex: "tabIndex"
-	},
-	
+
 	attr: function( elem, name, value, pass ) {
-		var nType = elem.nodeType;
-		
+		var ret, hooks, notxml,
+			nType = elem.nodeType;
+
 		// don't get/set attributes on text, comment and attribute nodes
 		if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
-			return undefined;
+			return;
 		}
 
 		if ( pass && name in jQuery.attrFn ) {
@@ -2213,39 +2522,24 @@ jQuery.extend({
 		}
 
 		// Fallback to prop when attributes are not supported
-		if ( !("getAttribute" in elem) ) {
+		if ( typeof elem.getAttribute === "undefined" ) {
 			return jQuery.prop( elem, name, value );
 		}
 
-		var ret, hooks,
-			notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+		notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
 
-		// Normalize the name if needed
+		// All attributes are lowercase
+		// Grab necessary hook if one is defined
 		if ( notxml ) {
-			name = jQuery.attrFix[ name ] || name;
-
-			hooks = jQuery.attrHooks[ name ];
-
-			if ( !hooks ) {
-				// Use boolHook for boolean attributes
-				if ( rboolean.test( name ) ) {
-
-					hooks = boolHook;
-
-				// Use formHook for forms and if the name contains certain characters
-				} else if ( formHook && name !== "className" &&
-					(jQuery.nodeName( elem, "form" ) || rinvalidChar.test( name )) ) {
-
-					hooks = formHook;
-				}
-			}
+			name = name.toLowerCase();
+			hooks = jQuery.attrHooks[ name ] || ( rboolean.test( name ) ? boolHook : nodeHook );
 		}
 
 		if ( value !== undefined ) {
 
 			if ( value === null ) {
 				jQuery.removeAttr( elem, name );
-				return undefined;
+				return;
 
 			} else if ( hooks && "set" in hooks && notxml && (ret = hooks.set( elem, value, name )) !== undefined ) {
 				return ret;
@@ -2269,22 +2563,33 @@ jQuery.extend({
 		}
 	},
 
-	removeAttr: function( elem, name ) {
-		var propName;
-		if ( elem.nodeType === 1 ) {
-			name = jQuery.attrFix[ name ] || name;
-		
-			if ( jQuery.support.getSetAttribute ) {
-				// Use removeAttribute in browsers that support it
-				elem.removeAttribute( name );
-			} else {
-				jQuery.attr( elem, name, "" );
-				elem.removeAttributeNode( elem.getAttributeNode( name ) );
-			}
+	removeAttr: function( elem, value ) {
+		var propName, attrNames, name, l, isBool,
+			i = 0;
 
-			// Set corresponding property to false for boolean attributes
-			if ( rboolean.test( name ) && (propName = jQuery.propFix[ name ] || name) in elem ) {
-				elem[ propName ] = false;
+		if ( value && elem.nodeType === 1 ) {
+			attrNames = value.toLowerCase().split( rspace );
+			l = attrNames.length;
+
+			for ( ; i < l; i++ ) {
+				name = attrNames[ i ];
+
+				if ( name ) {
+					propName = jQuery.propFix[ name ] || name;
+					isBool = rboolean.test( name );
+
+					// See #9699 for explanation of this approach (setting first, then removal)
+					// Do not do this for boolean attributes (see #10870)
+					if ( !isBool ) {
+						jQuery.attr( elem, name, "" );
+					}
+					elem.removeAttribute( getSetAttribute ? name : propName );
+
+					// Set corresponding property to false for boolean attributes
+					if ( isBool && propName in elem ) {
+						elem[ propName ] = false;
+					}
+				}
 			}
 		}
 	},
@@ -2308,33 +2613,20 @@ jQuery.extend({
 				}
 			}
 		},
-		tabIndex: {
-			get: function( elem ) {
-				// elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set
-				// http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/
-				var attributeNode = elem.getAttributeNode("tabIndex");
-
-				return attributeNode && attributeNode.specified ?
-					parseInt( attributeNode.value, 10 ) :
-					rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ?
-						0 :
-						undefined;
-			}
-		},
 		// Use the value property for back compat
-		// Use the formHook for button elements in IE6/7 (#1954)
+		// Use the nodeHook for button elements in IE6/7 (#1954)
 		value: {
 			get: function( elem, name ) {
-				if ( formHook && jQuery.nodeName( elem, "button" ) ) {
-					return formHook.get( elem, name );
+				if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
+					return nodeHook.get( elem, name );
 				}
 				return name in elem ?
 					elem.value :
 					null;
 			},
 			set: function( elem, value, name ) {
-				if ( formHook && jQuery.nodeName( elem, "button" ) ) {
-					return formHook.set( elem, value, name );
+				if ( nodeHook && jQuery.nodeName( elem, "button" ) ) {
+					return nodeHook.set( elem, value, name );
 				}
 				// Does not return so that setAttribute is also used
 				elem.value = value;
@@ -2356,17 +2648,17 @@ jQuery.extend({
 		frameborder: "frameBorder",
 		contenteditable: "contentEditable"
 	},
-	
+
 	prop: function( elem, name, value ) {
-		var nType = elem.nodeType;
+		var ret, hooks, notxml,
+			nType = elem.nodeType;
 
 		// don't get/set properties on text, comment and attribute nodes
 		if ( !elem || nType === 3 || nType === 8 || nType === 2 ) {
-			return undefined;
+			return;
 		}
 
-		var ret, hooks,
-			notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
+		notxml = nType !== 1 || !jQuery.isXMLDoc( elem );
 
 		if ( notxml ) {
 			// Fix name and attach hooks
@@ -2379,11 +2671,11 @@ jQuery.extend({
 				return ret;
 
 			} else {
-				return (elem[ name ] = value);
+				return ( elem[ name ] = value );
 			}
 
 		} else {
-			if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== undefined ) {
+			if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) {
 				return ret;
 
 			} else {
@@ -2391,15 +2683,35 @@ jQuery.extend({
 			}
 		}
 	},
-	
-	propHooks: {}
+
+	propHooks: {
+		tabIndex: {
+			get: function( elem ) {
+				// elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set
+				// http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/
+				var attributeNode = elem.getAttributeNode("tabindex");
+
+				return attributeNode && attributeNode.specified ?
+					parseInt( attributeNode.value, 10 ) :
+					rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ?
+						0 :
+						undefined;
+			}
+		}
+	}
 });
 
+// Add the tabIndex propHook to attrHooks for back-compat (different case is intentional)
+jQuery.attrHooks.tabindex = jQuery.propHooks.tabIndex;
+
 // Hook for boolean attributes
 boolHook = {
 	get: function( elem, name ) {
 		// Align boolean attributes with corresponding properties
-		return jQuery.prop( elem, name ) ?
+		// Fall back to attribute presence where some booleans are not supported
+		var attrNode,
+			property = jQuery.prop( elem, name );
+		return property === true || typeof property !== "boolean" && ( attrNode = elem.getAttributeNode(name) ) && attrNode.nodeValue !== false ?
 			name.toLowerCase() :
 			undefined;
 	},
@@ -2424,32 +2736,38 @@ boolHook = {
 };
 
 // IE6/7 do not support getting/setting some attributes with get/setAttribute
-if ( !jQuery.support.getSetAttribute ) {
+if ( !getSetAttribute ) {
 
-	// propFix is more comprehensive and contains all fixes
-	jQuery.attrFix = jQuery.propFix;
-	
-	// Use this for any attribute on a form in IE6/7
-	formHook = jQuery.attrHooks.name = jQuery.attrHooks.title = jQuery.valHooks.button = {
+	fixSpecified = {
+		name: true,
+		id: true,
+		coords: true
+	};
+
+	// Use this for any attribute in IE6/7
+	// This fixes almost every IE6/7 issue
+	nodeHook = jQuery.valHooks.button = {
 		get: function( elem, name ) {
 			var ret;
 			ret = elem.getAttributeNode( name );
-			// Return undefined if nodeValue is empty string
-			return ret && ret.nodeValue !== "" ?
+			return ret && ( fixSpecified[ name ] ? ret.nodeValue !== "" : ret.specified ) ?
 				ret.nodeValue :
 				undefined;
 		},
 		set: function( elem, value, name ) {
-			// Check form objects in IE (multiple bugs related)
-			// Only use nodeValue if the attribute node exists on the form
+			// Set the existing or create a new attribute node
 			var ret = elem.getAttributeNode( name );
-			if ( ret ) {
-				ret.nodeValue = value;
-				return value;
+			if ( !ret ) {
+				ret = document.createAttribute( name );
+				elem.setAttributeNode( ret );
 			}
+			return ( ret.nodeValue = value + "" );
 		}
 	};
 
+	// Apply the nodeHook to tabindex
+	jQuery.attrHooks.tabindex.set = nodeHook.set;
+
 	// Set width and height to auto instead of 0 on empty string( Bug #8150 )
 	// This is for removals
 	jQuery.each([ "width", "height" ], function( i, name ) {
@@ -2462,6 +2780,18 @@ if ( !jQuery.support.getSetAttribute ) {
 			}
 		});
 	});
+
+	// Set contenteditable to false on removals(#10429)
+	// Setting to empty string throws an error as an invalid value
+	jQuery.attrHooks.contenteditable = {
+		get: nodeHook.get,
+		set: function( elem, value, name ) {
+			if ( value === "" ) {
+				value = "false";
+			}
+			nodeHook.set( elem, value, name );
+		}
+	};
 }
 
 
@@ -2485,7 +2815,7 @@ if ( !jQuery.support.style ) {
 			return elem.style.cssText.toLowerCase() || undefined;
 		},
 		set: function( elem, value ) {
-			return (elem.style.cssText = "" + value);
+			return ( elem.style.cssText = "" + value );
 		}
 	};
 }
@@ -2505,10 +2835,16 @@ if ( !jQuery.support.optSelected ) {
 					parent.parentNode.selectedIndex;
 				}
 			}
+			return null;
 		}
 	});
 }
 
+// IE6/7 call enctype encoding
+if ( !jQuery.support.enctype ) {
+	jQuery.propFix.enctype = "encoding";
+}
+
 // Radios and checkboxes getter/setter
 if ( !jQuery.support.checkOn ) {
 	jQuery.each([ "radio", "checkbox" ], function() {
@@ -2524,7 +2860,7 @@ jQuery.each([ "radio", "checkbox" ], function() {
 	jQuery.valHooks[ this ] = jQuery.extend( jQuery.valHooks[ this ], {
 		set: function( elem, value ) {
 			if ( jQuery.isArray( value ) ) {
-				return (elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0);
+				return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 );
 			}
 		}
 	});
@@ -2533,116 +2869,119 @@ jQuery.each([ "radio", "checkbox" ], function() {
 
 
 
-var rnamespaces = /\.(.*)$/,
-	rformElems = /^(?:textarea|input|select)$/i,
-	rperiod = /\./g,
-	rspaces = / /g,
-	rescape = /[^\w\s.|`]/g,
-	fcleanup = function( nm ) {
-		return nm.replace(rescape, "\\$&");
+var rformElems = /^(?:textarea|input|select)$/i,
+	rtypenamespace = /^([^\.]*)?(?:\.(.+))?$/,
+	rhoverHack = /(?:^|\s)hover(\.\S+)?\b/,
+	rkeyEvent = /^key/,
+	rmouseEvent = /^(?:mouse|contextmenu)|click/,
+	rfocusMorph = /^(?:focusinfocus|focusoutblur)$/,
+	rquickIs = /^(\w*)(?:#([\w\-]+))?(?:\.([\w\-]+))?$/,
+	quickParse = function( selector ) {
+		var quick = rquickIs.exec( selector );
+		if ( quick ) {
+			//   0  1    2   3
+			// [ _, tag, id, class ]
+			quick[1] = ( quick[1] || "" ).toLowerCase();
+			quick[3] = quick[3] && new RegExp( "(?:^|\\s)" + quick[3] + "(?:\\s|$)" );
+		}
+		return quick;
+	},
+	quickIs = function( elem, m ) {
+		var attrs = elem.attributes || {};
+		return (
+			(!m[1] || elem.nodeName.toLowerCase() === m[1]) &&
+			(!m[2] || (attrs.id || {}).value === m[2]) &&
+			(!m[3] || m[3].test( (attrs[ "class" ] || {}).value ))
+		);
+	},
+	hoverHack = function( events ) {
+		return jQuery.event.special.hover ? events : events.replace( rhoverHack, "mouseenter$1 mouseleave$1" );
 	};
 
 /*
- * A number of helper functions used for managing events.
- * Many of the ideas behind this code originated from
- * Dean Edwards' addEvent library.
+ * Helper functions for managing events -- not part of the public interface.
+ * Props to Dean Edwards' addEvent library for many of the ideas.
  */
 jQuery.event = {
 
-	// Bind an event to an element
-	// Original by Dean Edwards
-	add: function( elem, types, handler, data ) {
-		if ( elem.nodeType === 3 || elem.nodeType === 8 ) {
+	add: function( elem, types, handler, data, selector ) {
+
+		var elemData, eventHandle, events,
+			t, tns, type, namespaces, handleObj,
+			handleObjIn, quick, handlers, special;
+
+		// Don't attach events to noData or text/comment nodes (allow plain objects tho)
+		if ( elem.nodeType === 3 || elem.nodeType === 8 || !types || !handler || !(elemData = jQuery._data( elem )) ) {
 			return;
 		}
 
-		if ( handler === false ) {
-			handler = returnFalse;
-		} else if ( !handler ) {
-			// Fixes bug #7229. Fix recommended by jdalton
-			return;
-		}
-
-		var handleObjIn, handleObj;
-
+		// Caller can pass in an object of custom data in lieu of the handler
 		if ( handler.handler ) {
 			handleObjIn = handler;
 			handler = handleObjIn.handler;
+			selector = handleObjIn.selector;
 		}
 
-		// Make sure that the function being executed has a unique ID
+		// Make sure that the handler has a unique ID, used to find/remove it later
 		if ( !handler.guid ) {
 			handler.guid = jQuery.guid++;
 		}
 
-		// Init the element's event structure
-		var elemData = jQuery._data( elem );
-
-		// If no elemData is found then we must be trying to bind to one of the
-		// banned noData elements
-		if ( !elemData ) {
-			return;
-		}
-
-		var events = elemData.events,
-			eventHandle = elemData.handle;
-
+		// Init the element's event structure and main handler, if this is the first
+		events = elemData.events;
 		if ( !events ) {
 			elemData.events = events = {};
 		}
-
+		eventHandle = elemData.handle;
 		if ( !eventHandle ) {
 			elemData.handle = eventHandle = function( e ) {
 				// Discard the second event of a jQuery.event.trigger() and
 				// when an event is called after a page has unloaded
 				return typeof jQuery !== "undefined" && (!e || jQuery.event.triggered !== e.type) ?
-					jQuery.event.handle.apply( eventHandle.elem, arguments ) :
+					jQuery.event.dispatch.apply( eventHandle.elem, arguments ) :
 					undefined;
 			};
+			// Add elem as a property of the handle fn to prevent a memory leak with IE non-native events
+			eventHandle.elem = elem;
 		}
 
-		// Add elem as a property of the handle function
-		// This is to prevent a memory leak with non-native events in IE.
-		eventHandle.elem = elem;
-
 		// Handle multiple events separated by a space
 		// jQuery(...).bind("mouseover mouseout", fn);
-		types = types.split(" ");
+		types = jQuery.trim( hoverHack(types) ).split( " " );
+		for ( t = 0; t < types.length; t++ ) {
 
-		var type, i = 0, namespaces;
+			tns = rtypenamespace.exec( types[t] ) || [];
+			type = tns[1];
+			namespaces = ( tns[2] || "" ).split( "." ).sort();
 
-		while ( (type = types[ i++ ]) ) {
-			handleObj = handleObjIn ?
-				jQuery.extend({}, handleObjIn) :
-				{ handler: handler, data: data };
+			// If event changes its type, use the special event handlers for the changed type
+			special = jQuery.event.special[ type ] || {};
 
-			// Namespaced event handlers
-			if ( type.indexOf(".") > -1 ) {
-				namespaces = type.split(".");
-				type = namespaces.shift();
-				handleObj.namespace = namespaces.slice(0).sort().join(".");
+			// If selector defined, determine special event api type, otherwise given type
+			type = ( selector ? special.delegateType : special.bindType ) || type;
 
-			} else {
-				namespaces = [];
-				handleObj.namespace = "";
-			}
+			// Update special based on newly reset type
+			special = jQuery.event.special[ type ] || {};
 
-			handleObj.type = type;
-			if ( !handleObj.guid ) {
-				handleObj.guid = handler.guid;
-			}
+			// handleObj is passed to all event handlers
+			handleObj = jQuery.extend({
+				type: type,
+				origType: tns[1],
+				data: data,
+				handler: handler,
+				guid: handler.guid,
+				selector: selector,
+				quick: selector && quickParse( selector ),
+				namespace: namespaces.join(".")
+			}, handleObjIn );
 
-			// Get the current list of functions bound to this event
-			var handlers = events[ type ],
-				special = jQuery.event.special[ type ] || {};
-
-			// Init the event handler queue
+			// Init the event handler queue if we're the first
+			handlers = events[ type ];
 			if ( !handlers ) {
 				handlers = events[ type ] = [];
+				handlers.delegateCount = 0;
 
-				// Check for a special event handler
-				// Only use addEventListener/attachEvent if the special
-				// events handler returns false
+				// Only use addEventListener/attachEvent if the special events handler returns false
 				if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) {
 					// Bind the global event handler to the element
 					if ( elem.addEventListener ) {
@@ -2662,10 +3001,14 @@ jQuery.event = {
 				}
 			}
 
-			// Add the function to the element's handler list
-			handlers.push( handleObj );
+			// Add to the element's handler list, delegates in front
+			if ( selector ) {
+				handlers.splice( handlers.delegateCount++, 0, handleObj );
+			} else {
+				handlers.push( handleObj );
+			}
 
-			// Keep track of which events have been used, for event optimization
+			// Keep track of which events have ever been used, for event optimization
 			jQuery.event.global[ type ] = true;
 		}
 
@@ -2676,129 +3019,80 @@ jQuery.event = {
 	global: {},
 
 	// Detach an event or set of events from an element
-	remove: function( elem, types, handler, pos ) {
-		// don't do events on text and comment nodes
-		if ( elem.nodeType === 3 || elem.nodeType === 8 ) {
+	remove: function( elem, types, handler, selector, mappedTypes ) {
+
+		var elemData = jQuery.hasData( elem ) && jQuery._data( elem ),
+			t, tns, type, origType, namespaces, origCount,
+			j, events, special, handle, eventType, handleObj;
+
+		if ( !elemData || !(events = elemData.events) ) {
 			return;
 		}
 
-		if ( handler === false ) {
-			handler = returnFalse;
-		}
+		// Once for each type.namespace in types; type may be omitted
+		types = jQuery.trim( hoverHack( types || "" ) ).split(" ");
+		for ( t = 0; t < types.length; t++ ) {
+			tns = rtypenamespace.exec( types[t] ) || [];
+			type = origType = tns[1];
+			namespaces = tns[2];
 
-		var ret, type, fn, j, i = 0, all, namespaces, namespace, special, eventType, handleObj, origType,
-			elemData = jQuery.hasData( elem ) && jQuery._data( elem ),
-			events = elemData && elemData.events;
-
-		if ( !elemData || !events ) {
-			return;
-		}
-
-		// types is actually an event object here
-		if ( types && types.type ) {
-			handler = types.handler;
-			types = types.type;
-		}
-
-		// Unbind all events for the element
-		if ( !types || typeof types === "string" && types.charAt(0) === "." ) {
-			types = types || "";
-
-			for ( type in events ) {
-				jQuery.event.remove( elem, type + types );
-			}
-
-			return;
-		}
-
-		// Handle multiple events separated by a space
-		// jQuery(...).unbind("mouseover mouseout", fn);
-		types = types.split(" ");
-
-		while ( (type = types[ i++ ]) ) {
-			origType = type;
-			handleObj = null;
-			all = type.indexOf(".") < 0;
-			namespaces = [];
-
-			if ( !all ) {
-				// Namespaced event handlers
-				namespaces = type.split(".");
-				type = namespaces.shift();
-
-				namespace = new RegExp("(^|\\.)" +
-					jQuery.map( namespaces.slice(0).sort(), fcleanup ).join("\\.(?:.*\\.)?") + "(\\.|$)");
-			}
-
-			eventType = events[ type ];
-
-			if ( !eventType ) {
-				continue;
-			}
-
-			if ( !handler ) {
-				for ( j = 0; j < eventType.length; j++ ) {
-					handleObj = eventType[ j ];
-
-					if ( all || namespace.test( handleObj.namespace ) ) {
-						jQuery.event.remove( elem, origType, handleObj.handler, j );
-						eventType.splice( j--, 1 );
-					}
+			// Unbind all events (on this namespace, if provided) for the element
+			if ( !type ) {
+				for ( type in events ) {
+					jQuery.event.remove( elem, type + types[ t ], handler, selector, true );
 				}
-
 				continue;
 			}
 
 			special = jQuery.event.special[ type ] || {};
+			type = ( selector? special.delegateType : special.bindType ) || type;
+			eventType = events[ type ] || [];
+			origCount = eventType.length;
+			namespaces = namespaces ? new RegExp("(^|\\.)" + namespaces.split(".").sort().join("\\.(?:.*\\.)?") + "(\\.|$)") : null;
 
-			for ( j = pos || 0; j < eventType.length; j++ ) {
+			// Remove matching events
+			for ( j = 0; j < eventType.length; j++ ) {
 				handleObj = eventType[ j ];
 
-				if ( handler.guid === handleObj.guid ) {
-					// remove the given handler for the given type
-					if ( all || namespace.test( handleObj.namespace ) ) {
-						if ( pos == null ) {
-							eventType.splice( j--, 1 );
-						}
+				if ( ( mappedTypes || origType === handleObj.origType ) &&
+					 ( !handler || handler.guid === handleObj.guid ) &&
+					 ( !namespaces || namespaces.test( handleObj.namespace ) ) &&
+					 ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) {
+					eventType.splice( j--, 1 );
 
-						if ( special.remove ) {
-							special.remove.call( elem, handleObj );
-						}
+					if ( handleObj.selector ) {
+						eventType.delegateCount--;
 					}
-
-					if ( pos != null ) {
-						break;
+					if ( special.remove ) {
+						special.remove.call( elem, handleObj );
 					}
 				}
 			}
 
-			// remove generic event handler if no more handlers exist
-			if ( eventType.length === 0 || pos != null && eventType.length === 1 ) {
+			// Remove generic event handler if we removed something and no more handlers exist
+			// (avoids potential for endless recursion during removal of special event handlers)
+			if ( eventType.length === 0 && origCount !== eventType.length ) {
 				if ( !special.teardown || special.teardown.call( elem, namespaces ) === false ) {
 					jQuery.removeEvent( elem, type, elemData.handle );
 				}
 
-				ret = null;
 				delete events[ type ];
 			}
 		}
 
 		// Remove the expando if it's no longer used
 		if ( jQuery.isEmptyObject( events ) ) {
-			var handle = elemData.handle;
+			handle = elemData.handle;
 			if ( handle ) {
 				handle.elem = null;
 			}
 
-			delete elemData.events;
-			delete elemData.handle;
-
-			if ( jQuery.isEmptyObject( elemData ) ) {
-				jQuery.removeData( elem, undefined, true );
-			}
+			// removeData also checks for emptiness and clears the expando if empty
+			// so use it instead of delete
+			jQuery.removeData( elem, [ "events", "handle" ], true );
 		}
 	},
-	
+
 	// Events that are safe to short-circuit if no handlers are attached.
 	// Native DOM events should not be added, they may have inline handlers.
 	customEvent: {
@@ -2808,18 +3102,28 @@ jQuery.event = {
 	},
 
 	trigger: function( event, data, elem, onlyHandlers ) {
+		// Don't do events on text and comment nodes
+		if ( elem && (elem.nodeType === 3 || elem.nodeType === 8) ) {
+			return;
+		}
+
 		// Event object or event type
 		var type = event.type || event,
 			namespaces = [],
-			exclusive;
+			cache, exclusive, i, cur, old, ontype, special, handle, eventPath, bubbleType;
 
-		if ( type.indexOf("!") >= 0 ) {
+		// focus/blur morphs to focusin/out; ensure we're not firing them right now
+		if ( rfocusMorph.test( type + jQuery.event.triggered ) ) {
+			return;
+		}
+
+		if ( type.indexOf( "!" ) >= 0 ) {
 			// Exclusive events trigger only for the exact event (no namespaces)
 			type = type.slice(0, -1);
 			exclusive = true;
 		}
 
-		if ( type.indexOf(".") >= 0 ) {
+		if ( type.indexOf( "." ) >= 0 ) {
 			// Namespaced trigger; create a regexp to match event type in handle()
 			namespaces = type.split(".");
 			type = namespaces.shift();
@@ -2841,230 +3145,314 @@ jQuery.event = {
 			new jQuery.Event( type );
 
 		event.type = type;
+		event.isTrigger = true;
 		event.exclusive = exclusive;
-		event.namespace = namespaces.join(".");
-		event.namespace_re = new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)");
-		
-		// triggerHandler() and global events don't bubble or run the default action
-		if ( onlyHandlers || !elem ) {
-			event.preventDefault();
-			event.stopPropagation();
-		}
+		event.namespace = namespaces.join( "." );
+		event.namespace_re = event.namespace? new RegExp("(^|\\.)" + namespaces.join("\\.(?:.*\\.)?") + "(\\.|$)") : null;
+		ontype = type.indexOf( ":" ) < 0 ? "on" + type : "";
 
 		// Handle a global trigger
 		if ( !elem ) {
-			// TODO: Stop taunting the data cache; remove global events and always attach to document
-			jQuery.each( jQuery.cache, function() {
-				// internalKey variable is just used to make it easier to find
-				// and potentially change this stuff later; currently it just
-				// points to jQuery.expando
-				var internalKey = jQuery.expando,
-					internalCache = this[ internalKey ];
-				if ( internalCache && internalCache.events && internalCache.events[ type ] ) {
-					jQuery.event.trigger( event, data, internalCache.handle.elem );
-				}
-			});
-			return;
-		}
 
-		// Don't do events on text and comment nodes
-		if ( elem.nodeType === 3 || elem.nodeType === 8 ) {
+			// TODO: Stop taunting the data cache; remove global events and always attach to document
+			cache = jQuery.cache;
+			for ( i in cache ) {
+				if ( cache[ i ].events && cache[ i ].events[ type ] ) {
+					jQuery.event.trigger( event, data, cache[ i ].handle.elem, true );
+				}
+			}
 			return;
 		}
 
 		// Clean up the event in case it is being reused
 		event.result = undefined;
-		event.target = elem;
+		if ( !event.target ) {
+			event.target = elem;
+		}
 
 		// Clone any incoming data and prepend the event, creating the handler arg list
 		data = data != null ? jQuery.makeArray( data ) : [];
 		data.unshift( event );
 
-		var cur = elem,
-			// IE doesn't like method names with a colon (#3533, #8272)
-			ontype = type.indexOf(":") < 0 ? "on" + type : "";
+		// Allow special events to draw outside the lines
+		special = jQuery.event.special[ type ] || {};
+		if ( special.trigger && special.trigger.apply( elem, data ) === false ) {
+			return;
+		}
 
-		// Fire event on the current element, then bubble up the DOM tree
-		do {
-			var handle = jQuery._data( cur, "handle" );
+		// Determine event propagation path in advance, per W3C events spec (#9951)
+		// Bubble up to document, then to window; watch for a global ownerDocument var (#9724)
+		eventPath = [[ elem, special.bindType || type ]];
+		if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) {
 
-			event.currentTarget = cur;
+			bubbleType = special.delegateType || type;
+			cur = rfocusMorph.test( bubbleType + type ) ? elem : elem.parentNode;
+			old = null;
+			for ( ; cur; cur = cur.parentNode ) {
+				eventPath.push([ cur, bubbleType ]);
+				old = cur;
+			}
+
+			// Only add window if we got to document (e.g., not plain obj or detached DOM)
+			if ( old && old === elem.ownerDocument ) {
+				eventPath.push([ old.defaultView || old.parentWindow || window, bubbleType ]);
+			}
+		}
+
+		// Fire handlers on the event path
+		for ( i = 0; i < eventPath.length && !event.isPropagationStopped(); i++ ) {
+
+			cur = eventPath[i][0];
+			event.type = eventPath[i][1];
+
+			handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" );
 			if ( handle ) {
 				handle.apply( cur, data );
 			}
-
-			// Trigger an inline bound script
-			if ( ontype && jQuery.acceptData( cur ) && cur[ ontype ] && cur[ ontype ].apply( cur, data ) === false ) {
-				event.result = false;
+			// Note that this is a bare JS function and not a jQuery handler
+			handle = ontype && cur[ ontype ];
+			if ( handle && jQuery.acceptData( cur ) && handle.apply( cur, data ) === false ) {
 				event.preventDefault();
 			}
-
-			// Bubble up to document, then to window
-			cur = cur.parentNode || cur.ownerDocument || cur === event.target.ownerDocument && window;
-		} while ( cur && !event.isPropagationStopped() );
+		}
+		event.type = type;
 
 		// If nobody prevented the default action, do it now
-		if ( !event.isDefaultPrevented() ) {
-			var old,
-				special = jQuery.event.special[ type ] || {};
+		if ( !onlyHandlers && !event.isDefaultPrevented() ) {
 
-			if ( (!special._default || special._default.call( elem.ownerDocument, event ) === false) &&
+			if ( (!special._default || special._default.apply( elem.ownerDocument, data ) === false) &&
 				!(type === "click" && jQuery.nodeName( elem, "a" )) && jQuery.acceptData( elem ) ) {
 
 				// Call a native DOM method on the target with the same name name as the event.
-				// Can't use an .isFunction)() check here because IE6/7 fails that test.
-				// IE<9 dies on focus to hidden element (#1486), may want to revisit a try/catch.
-				try {
-					if ( ontype && elem[ type ] ) {
-						// Don't re-trigger an onFOO event when we call its FOO() method
-						old = elem[ ontype ];
+				// Can't use an .isFunction() check here because IE6/7 fails that test.
+				// Don't do default actions on window, that's where global variables be (#6170)
+				// IE<9 dies on focus/blur to hidden element (#1486)
+				if ( ontype && elem[ type ] && ((type !== "focus" && type !== "blur") || event.target.offsetWidth !== 0) && !jQuery.isWindow( elem ) ) {
 
-						if ( old ) {
-							elem[ ontype ] = null;
-						}
+					// Don't re-trigger an onFOO event when we call its FOO() method
+					old = elem[ ontype ];
 
-						jQuery.event.triggered = type;
-						elem[ type ]();
+					if ( old ) {
+						elem[ ontype ] = null;
 					}
-				} catch ( ieError ) {}
 
-				if ( old ) {
-					elem[ ontype ] = old;
+					// Prevent re-triggering of the same event, since we already bubbled it above
+					jQuery.event.triggered = type;
+					elem[ type ]();
+					jQuery.event.triggered = undefined;
+
+					if ( old ) {
+						elem[ ontype ] = old;
+					}
 				}
-
-				jQuery.event.triggered = undefined;
 			}
 		}
-		
+
 		return event.result;
 	},
 
-	handle: function( event ) {
+	dispatch: function( event ) {
+
+		// Make a writable jQuery.Event from the native event object
 		event = jQuery.event.fix( event || window.event );
-		// Snapshot the handlers list since a called handler may add/remove events.
-		var handlers = ((jQuery._data( this, "events" ) || {})[ event.type ] || []).slice(0),
+
+		var handlers = ( (jQuery._data( this, "events" ) || {} )[ event.type ] || []),
+			delegateCount = handlers.delegateCount,
+			args = [].slice.call( arguments, 0 ),
 			run_all = !event.exclusive && !event.namespace,
-			args = Array.prototype.slice.call( arguments, 0 );
+			special = jQuery.event.special[ event.type ] || {},
+			handlerQueue = [],
+			i, j, cur, jqcur, ret, selMatch, matched, matches, handleObj, sel, related;
 
-		// Use the fix-ed Event rather than the (read-only) native event
+		// Use the fix-ed jQuery.Event rather than the (read-only) native event
 		args[0] = event;
-		event.currentTarget = this;
+		event.delegateTarget = this;
 
-		for ( var j = 0, l = handlers.length; j < l; j++ ) {
-			var handleObj = handlers[ j ];
+		// Call the preDispatch hook for the mapped type, and let it bail if desired
+		if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) {
+			return;
+		}
 
-			// Triggered event must 1) be non-exclusive and have no namespace, or
-			// 2) have namespace(s) a subset or equal to those in the bound event.
-			if ( run_all || event.namespace_re.test( handleObj.namespace ) ) {
-				// Pass in a reference to the handler function itself
-				// So that we can later remove it
-				event.handler = handleObj.handler;
-				event.data = handleObj.data;
-				event.handleObj = handleObj;
+		// Determine handlers that should run if there are delegated events
+		// Avoid non-left-click bubbling in Firefox (#3861)
+		if ( delegateCount && !(event.button && event.type === "click") ) {
 
-				var ret = handleObj.handler.apply( this, args );
+			// Pregenerate a single jQuery object for reuse with .is()
+			jqcur = jQuery(this);
+			jqcur.context = this.ownerDocument || this;
 
-				if ( ret !== undefined ) {
-					event.result = ret;
-					if ( ret === false ) {
-						event.preventDefault();
-						event.stopPropagation();
+			for ( cur = event.target; cur != this; cur = cur.parentNode || this ) {
+
+				// Don't process events on disabled elements (#6911, #8165)
+				if ( cur.disabled !== true ) {
+					selMatch = {};
+					matches = [];
+					jqcur[0] = cur;
+					for ( i = 0; i < delegateCount; i++ ) {
+						handleObj = handlers[ i ];
+						sel = handleObj.selector;
+
+						if ( selMatch[ sel ] === undefined ) {
+							selMatch[ sel ] = (
+								handleObj.quick ? quickIs( cur, handleObj.quick ) : jqcur.is( sel )
+							);
+						}
+						if ( selMatch[ sel ] ) {
+							matches.push( handleObj );
+						}
+					}
+					if ( matches.length ) {
+						handlerQueue.push({ elem: cur, matches: matches });
 					}
-				}
-
-				if ( event.isImmediatePropagationStopped() ) {
-					break;
 				}
 			}
 		}
+
+		// Add the remaining (directly-bound) handlers
+		if ( handlers.length > delegateCount ) {
+			handlerQueue.push({ elem: this, matches: handlers.slice( delegateCount ) });
+		}
+
+		// Run delegates first; they may want to stop propagation beneath us
+		for ( i = 0; i < handlerQueue.length && !event.isPropagationStopped(); i++ ) {
+			matched = handlerQueue[ i ];
+			event.currentTarget = matched.elem;
+
+			for ( j = 0; j < matched.matches.length && !event.isImmediatePropagationStopped(); j++ ) {
+				handleObj = matched.matches[ j ];
+
+				// Triggered event must either 1) be non-exclusive and have no namespace, or
+				// 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace).
+				if ( run_all || (!event.namespace && !handleObj.namespace) || event.namespace_re && event.namespace_re.test( handleObj.namespace ) ) {
+
+					event.data = handleObj.data;
+					event.handleObj = handleObj;
+
+					ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler )
+							.apply( matched.elem, args );
+
+					if ( ret !== undefined ) {
+						event.result = ret;
+						if ( ret === false ) {
+							event.preventDefault();
+							event.stopPropagation();
+						}
+					}
+				}
+			}
+		}
+
+		// Call the postDispatch hook for the mapped type
+		if ( special.postDispatch ) {
+			special.postDispatch.call( this, event );
+		}
+
 		return event.result;
 	},
 
-	props: "altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),
+	// Includes some event props shared by KeyEvent and MouseEvent
+	// *** attrChange attrName relatedNode srcElement  are not normalized, non-W3C, deprecated, will be removed in 1.8 ***
+	props: "attrChange attrName relatedNode srcElement altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),
+
+	fixHooks: {},
+
+	keyHooks: {
+		props: "char charCode key keyCode".split(" "),
+		filter: function( event, original ) {
+
+			// Add which for key events
+			if ( event.which == null ) {
+				event.which = original.charCode != null ? original.charCode : original.keyCode;
+			}
+
+			return event;
+		}
+	},
+
+	mouseHooks: {
+		props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),
+		filter: function( event, original ) {
+			var eventDoc, doc, body,
+				button = original.button,
+				fromElement = original.fromElement;
+
+			// Calculate pageX/Y if missing and clientX/Y available
+			if ( event.pageX == null && original.clientX != null ) {
+				eventDoc = event.target.ownerDocument || document;
+				doc = eventDoc.documentElement;
+				body = eventDoc.body;
+
+				event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 );
+				event.pageY = original.clientY + ( doc && doc.scrollTop  || body && body.scrollTop  || 0 ) - ( doc && doc.clientTop  || body && body.clientTop  || 0 );
+			}
+
+			// Add relatedTarget, if necessary
+			if ( !event.relatedTarget && fromElement ) {
+				event.relatedTarget = fromElement === event.target ? original.toElement : fromElement;
+			}
+
+			// Add which for click: 1 === left; 2 === middle; 3 === right
+			// Note: button is not normalized, so don't use it
+			if ( !event.which && button !== undefined ) {
+				event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) );
+			}
+
+			return event;
+		}
+	},
 
 	fix: function( event ) {
 		if ( event[ jQuery.expando ] ) {
 			return event;
 		}
 
-		// store a copy of the original event object
-		// and "clone" to set read-only properties
-		var originalEvent = event;
+		// Create a writable copy of the event object and normalize some properties
+		var i, prop,
+			originalEvent = event,
+			fixHook = jQuery.event.fixHooks[ event.type ] || {},
+			copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props;
+
 		event = jQuery.Event( originalEvent );
 
-		for ( var i = this.props.length, prop; i; ) {
-			prop = this.props[ --i ];
+		for ( i = copy.length; i; ) {
+			prop = copy[ --i ];
 			event[ prop ] = originalEvent[ prop ];
 		}
 
-		// Fix target property, if necessary
+		// Fix target property, if necessary (#1925, IE 6/7/8 & Safari2)
 		if ( !event.target ) {
-			// Fixes #1925 where srcElement might not be defined either
-			event.target = event.srcElement || document;
+			event.target = originalEvent.srcElement || document;
 		}
 
-		// check if target is a textnode (safari)
+		// Target should not be a text node (#504, Safari)
 		if ( event.target.nodeType === 3 ) {
 			event.target = event.target.parentNode;
 		}
 
-		// Add relatedTarget, if necessary
-		if ( !event.relatedTarget && event.fromElement ) {
-			event.relatedTarget = event.fromElement === event.target ? event.toElement : event.fromElement;
-		}
-
-		// Calculate pageX/Y if missing and clientX/Y available
-		if ( event.pageX == null && event.clientX != null ) {
-			var eventDocument = event.target.ownerDocument || document,
-				doc = eventDocument.documentElement,
-				body = eventDocument.body;
-
-			event.pageX = event.clientX + (doc && doc.scrollLeft || body && body.scrollLeft || 0) - (doc && doc.clientLeft || body && body.clientLeft || 0);
-			event.pageY = event.clientY + (doc && doc.scrollTop  || body && body.scrollTop  || 0) - (doc && doc.clientTop  || body && body.clientTop  || 0);
-		}
-
-		// Add which for key events
-		if ( event.which == null && (event.charCode != null || event.keyCode != null) ) {
-			event.which = event.charCode != null ? event.charCode : event.keyCode;
-		}
-
-		// Add metaKey to non-Mac browsers (use ctrl for PC's and Meta for Macs)
-		if ( !event.metaKey && event.ctrlKey ) {
+		// For mouse/key events; add metaKey if it's not there (#3368, IE6/7/8)
+		if ( event.metaKey === undefined ) {
 			event.metaKey = event.ctrlKey;
 		}
 
-		// Add which for click: 1 === left; 2 === middle; 3 === right
-		// Note: button is not normalized, so don't use it
-		if ( !event.which && event.button !== undefined ) {
-			event.which = (event.button & 1 ? 1 : ( event.button & 2 ? 3 : ( event.button & 4 ? 2 : 0 ) ));
-		}
-
-		return event;
+		return fixHook.filter? fixHook.filter( event, originalEvent ) : event;
 	},
 
-	// Deprecated, use jQuery.guid instead
-	guid: 1E8,
-
-	// Deprecated, use jQuery.proxy instead
-	proxy: jQuery.proxy,
-
 	special: {
 		ready: {
 			// Make sure the ready event is setup
-			setup: jQuery.bindReady,
-			teardown: jQuery.noop
+			setup: jQuery.bindReady
 		},
 
-		live: {
-			add: function( handleObj ) {
-				jQuery.event.add( this,
-					liveConvert( handleObj.origType, handleObj.selector ),
-					jQuery.extend({}, handleObj, {handler: liveHandler, guid: handleObj.handler.guid}) );
-			},
+		load: {
+			// Prevent triggered image.load events from bubbling to window.load
+			noBubble: true
+		},
 
-			remove: function( handleObj ) {
-				jQuery.event.remove( this, liveConvert( handleObj.origType, handleObj.selector ), handleObj );
-			}
+		focus: {
+			delegateType: "focusin"
+		},
+		blur: {
+			delegateType: "focusout"
 		},
 
 		beforeunload: {
@@ -3081,9 +3469,35 @@ jQuery.event = {
 				}
 			}
 		}
+	},
+
+	simulate: function( type, elem, event, bubble ) {
+		// Piggyback on a donor event to simulate a different one.
+		// Fake originalEvent to avoid donor's stopPropagation, but if the
+		// simulated event prevents default then we do the same on the donor.
+		var e = jQuery.extend(
+			new jQuery.Event(),
+			event,
+			{ type: type,
+				isSimulated: true,
+				originalEvent: {}
+			}
+		);
+		if ( bubble ) {
+			jQuery.event.trigger( e, null, elem );
+		} else {
+			jQuery.event.dispatch.call( elem, e );
+		}
+		if ( e.isDefaultPrevented() ) {
+			event.preventDefault();
+		}
 	}
 };
 
+// Some plugins are using, but it's undocumented/deprecated and will be removed.
+// The 1.7 special event interface should provide all the hooks needed now.
+jQuery.event.handle = jQuery.event.dispatch;
+
 jQuery.removeEvent = document.removeEventListener ?
 	function( elem, type, handle ) {
 		if ( elem.removeEventListener ) {
@@ -3098,7 +3512,7 @@ jQuery.removeEvent = document.removeEventListener ?
 
 jQuery.Event = function( src, props ) {
 	// Allow instantiation without the 'new' keyword
-	if ( !this.preventDefault ) {
+	if ( !(this instanceof jQuery.Event) ) {
 		return new jQuery.Event( src, props );
 	}
 
@@ -3109,8 +3523,8 @@ jQuery.Event = function( src, props ) {
 
 		// Events bubbling up the document may have been marked as prevented
 		// by a handler lower down the tree; reflect the correct value.
-		this.isDefaultPrevented = (src.defaultPrevented || src.returnValue === false ||
-			src.getPreventDefault && src.getPreventDefault()) ? returnTrue : returnFalse;
+		this.isDefaultPrevented = ( src.defaultPrevented || src.returnValue === false ||
+			src.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse;
 
 	// Event type
 	} else {
@@ -3122,9 +3536,8 @@ jQuery.Event = function( src, props ) {
 		jQuery.extend( this, props );
 	}
 
-	// timeStamp is buggy for some events on Firefox(#3843)
-	// So we won't rely on the native value
-	this.timeStamp = jQuery.now();
+	// Create a timestamp if incoming event doesn't have one
+	this.timeStamp = src && src.timeStamp || jQuery.now();
 
 	// Mark it as fixed
 	this[ jQuery.expando ] = true;
@@ -3180,214 +3593,137 @@ jQuery.Event.prototype = {
 	isImmediatePropagationStopped: returnFalse
 };
 
-// Checks if an event happened on an element within another element
-// Used in jQuery.event.special.mouseenter and mouseleave handlers
-var withinElement = function( event ) {
-
-	// Check if mouse(over|out) are still within the same parent element
-	var related = event.relatedTarget,
-		inside = false,
-		eventType = event.type;
-
-	event.type = event.data;
-
-	if ( related !== this ) {
-
-		if ( related ) {
-			inside = jQuery.contains( this, related );
-		}
-
-		if ( !inside ) {
-
-			jQuery.event.handle.apply( this, arguments );
-
-			event.type = eventType;
-		}
-	}
-},
-
-// In case of event delegation, we only need to rename the event.type,
-// liveHandler will take care of the rest.
-delegate = function( event ) {
-	event.type = event.data;
-	jQuery.event.handle.apply( this, arguments );
-};
-
-// Create mouseenter and mouseleave events
+// Create mouseenter/leave events using mouseover/out and event-time checks
 jQuery.each({
 	mouseenter: "mouseover",
 	mouseleave: "mouseout"
 }, function( orig, fix ) {
 	jQuery.event.special[ orig ] = {
-		setup: function( data ) {
-			jQuery.event.add( this, fix, data && data.selector ? delegate : withinElement, orig );
-		},
-		teardown: function( data ) {
-			jQuery.event.remove( this, fix, data && data.selector ? delegate : withinElement );
+		delegateType: fix,
+		bindType: fix,
+
+		handle: function( event ) {
+			var target = this,
+				related = event.relatedTarget,
+				handleObj = event.handleObj,
+				selector = handleObj.selector,
+				ret;
+
+			// For mousenter/leave call the handler if related is outside the target.
+			// NB: No relatedTarget if the mouse left/entered the browser window
+			if ( !related || (related !== target && !jQuery.contains( target, related )) ) {
+				event.type = handleObj.origType;
+				ret = handleObj.handler.apply( this, arguments );
+				event.type = fix;
+			}
+			return ret;
 		}
 	};
 });
 
-// submit delegation
+// IE submit delegation
 if ( !jQuery.support.submitBubbles ) {
 
 	jQuery.event.special.submit = {
-		setup: function( data, namespaces ) {
-			if ( !jQuery.nodeName( this, "form" ) ) {
-				jQuery.event.add(this, "click.specialSubmit", function( e ) {
-					var elem = e.target,
-						type = elem.type;
-
-					if ( (type === "submit" || type === "image") && jQuery( elem ).closest("form").length ) {
-						trigger( "submit", this, arguments );
-					}
-				});
-
-				jQuery.event.add(this, "keypress.specialSubmit", function( e ) {
-					var elem = e.target,
-						type = elem.type;
-
-					if ( (type === "text" || type === "password") && jQuery( elem ).closest("form").length && e.keyCode === 13 ) {
-						trigger( "submit", this, arguments );
-					}
-				});
-
-			} else {
+		setup: function() {
+			// Only need this for delegated form submit events
+			if ( jQuery.nodeName( this, "form" ) ) {
 				return false;
 			}
+
+			// Lazy-add a submit handler when a descendant form may potentially be submitted
+			jQuery.event.add( this, "click._submit keypress._submit", function( e ) {
+				// Node name check avoids a VML-related crash in IE (#9807)
+				var elem = e.target,
+					form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined;
+				if ( form && !form._submit_attached ) {
+					jQuery.event.add( form, "submit._submit", function( event ) {
+						event._submit_bubble = true;
+					});
+					form._submit_attached = true;
+				}
+			});
+			// return undefined since we don't need an event listener
+		},
+		
+		postDispatch: function( event ) {
+			// If form was submitted by the user, bubble the event up the tree
+			if ( event._submit_bubble ) {
+				delete event._submit_bubble;
+				if ( this.parentNode && !event.isTrigger ) {
+					jQuery.event.simulate( "submit", this.parentNode, event, true );
+				}
+			}
 		},
 
-		teardown: function( namespaces ) {
-			jQuery.event.remove( this, ".specialSubmit" );
+		teardown: function() {
+			// Only need this for delegated form submit events
+			if ( jQuery.nodeName( this, "form" ) ) {
+				return false;
+			}
+
+			// Remove delegated handlers; cleanData eventually reaps submit handlers attached above
+			jQuery.event.remove( this, "._submit" );
 		}
 	};
-
 }
 
-// change delegation, happens here so we have bind.
+// IE change delegation and checkbox/radio fix
 if ( !jQuery.support.changeBubbles ) {
 
-	var changeFilters,
-
-	getVal = function( elem ) {
-		var type = elem.type, val = elem.value;
-
-		if ( type === "radio" || type === "checkbox" ) {
-			val = elem.checked;
-
-		} else if ( type === "select-multiple" ) {
-			val = elem.selectedIndex > -1 ?
-				jQuery.map( elem.options, function( elem ) {
-					return elem.selected;
-				}).join("-") :
-				"";
-
-		} else if ( jQuery.nodeName( elem, "select" ) ) {
-			val = elem.selectedIndex;
-		}
-
-		return val;
-	},
-
-	testChange = function testChange( e ) {
-		var elem = e.target, data, val;
-
-		if ( !rformElems.test( elem.nodeName ) || elem.readOnly ) {
-			return;
-		}
-
-		data = jQuery._data( elem, "_change_data" );
-		val = getVal(elem);
-
-		// the current data will be also retrieved by beforeactivate
-		if ( e.type !== "focusout" || elem.type !== "radio" ) {
-			jQuery._data( elem, "_change_data", val );
-		}
-
-		if ( data === undefined || val === data ) {
-			return;
-		}
-
-		if ( data != null || val ) {
-			e.type = "change";
-			e.liveFired = undefined;
-			jQuery.event.trigger( e, arguments[1], elem );
-		}
-	};
-
 	jQuery.event.special.change = {
-		filters: {
-			focusout: testChange,
 
-			beforedeactivate: testChange,
+		setup: function() {
 
-			click: function( e ) {
-				var elem = e.target, type = jQuery.nodeName( elem, "input" ) ? elem.type : "";
-
-				if ( type === "radio" || type === "checkbox" || jQuery.nodeName( elem, "select" ) ) {
-					testChange.call( this, e );
+			if ( rformElems.test( this.nodeName ) ) {
+				// IE doesn't fire change on a check/radio until blur; trigger it on click
+				// after a propertychange. Eat the blur-change in special.change.handle.
+				// This still fires onchange a second time for check/radio after blur.
+				if ( this.type === "checkbox" || this.type === "radio" ) {
+					jQuery.event.add( this, "propertychange._change", function( event ) {
+						if ( event.originalEvent.propertyName === "checked" ) {
+							this._just_changed = true;
+						}
+					});
+					jQuery.event.add( this, "click._change", function( event ) {
+						if ( this._just_changed && !event.isTrigger ) {
+							this._just_changed = false;
+							jQuery.event.simulate( "change", this, event, true );
+						}
+					});
 				}
-			},
-
-			// Change has to be called before submit
-			// Keydown will be called before keypress, which is used in submit-event delegation
-			keydown: function( e ) {
-				var elem = e.target, type = jQuery.nodeName( elem, "input" ) ? elem.type : "";
-
-				if ( (e.keyCode === 13 && !jQuery.nodeName( elem, "textarea" ) ) ||
-					(e.keyCode === 32 && (type === "checkbox" || type === "radio")) ||
-					type === "select-multiple" ) {
-					testChange.call( this, e );
-				}
-			},
-
-			// Beforeactivate happens also before the previous element is blurred
-			// with this event you can't trigger a change event, but you can store
-			// information
-			beforeactivate: function( e ) {
-				var elem = e.target;
-				jQuery._data( elem, "_change_data", getVal(elem) );
-			}
-		},
-
-		setup: function( data, namespaces ) {
-			if ( this.type === "file" ) {
 				return false;
 			}
+			// Delegated event; lazy-add a change handler on descendant inputs
+			jQuery.event.add( this, "beforeactivate._change", function( e ) {
+				var elem = e.target;
 
-			for ( var type in changeFilters ) {
-				jQuery.event.add( this, type + ".specialChange", changeFilters[type] );
-			}
-
-			return rformElems.test( this.nodeName );
+				if ( rformElems.test( elem.nodeName ) && !elem._change_attached ) {
+					jQuery.event.add( elem, "change._change", function( event ) {
+						if ( this.parentNode && !event.isSimulated && !event.isTrigger ) {
+							jQuery.event.simulate( "change", this.parentNode, event, true );
+						}
+					});
+					elem._change_attached = true;
+				}
+			});
 		},
 
-		teardown: function( namespaces ) {
-			jQuery.event.remove( this, ".specialChange" );
+		handle: function( event ) {
+			var elem = event.target;
+
+			// Swallow native change events from checkbox/radio, we already triggered them above
+			if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) {
+				return event.handleObj.handler.apply( this, arguments );
+			}
+		},
+
+		teardown: function() {
+			jQuery.event.remove( this, "._change" );
 
 			return rformElems.test( this.nodeName );
 		}
 	};
-
-	changeFilters = jQuery.event.special.change.filters;
-
-	// Handle when the input is .focus()'d
-	changeFilters.focus = changeFilters.beforeactivate;
-}
-
-function trigger( type, elem, args ) {
-	// Piggyback on a donor event to simulate a different one.
-	// Fake originalEvent to avoid donor's stopPropagation, but if the
-	// simulated event prevents default then we do the same on the donor.
-	// Don't pass args or remember liveFired; they apply to the donor event.
-	var event = jQuery.extend( {}, args[ 0 ] );
-	event.type = type;
-	event.originalEvent = {};
-	event.liveFired = undefined;
-	jQuery.event.handle.call( elem, event );
-	if ( event.isDefaultPrevented() ) {
-		args[ 0 ].preventDefault();
-	}
 }
 
 // Create "bubbling" focus and blur events
@@ -3395,7 +3731,10 @@ if ( !jQuery.support.focusinBubbles ) {
 	jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) {
 
 		// Attach a single capturing handler while someone wants focusin/focusout
-		var attaches = 0;
+		var attaches = 0,
+			handler = function( event ) {
+				jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true );
+			};
 
 		jQuery.event.special[ fix ] = {
 			setup: function() {
@@ -3409,89 +3748,120 @@ if ( !jQuery.support.focusinBubbles ) {
 				}
 			}
 		};
-
-		function handler( donor ) {
-			// Donor event is always a native one; fix it and switch its type.
-			// Let focusin/out handler cancel the donor focus/blur event.
-			var e = jQuery.event.fix( donor );
-			e.type = fix;
-			e.originalEvent = {};
-			jQuery.event.trigger( e, null, e.target );
-			if ( e.isDefaultPrevented() ) {
-				donor.preventDefault();
-			}
-		}
 	});
 }
 
-jQuery.each(["bind", "one"], function( i, name ) {
-	jQuery.fn[ name ] = function( type, data, fn ) {
-		var handler;
+jQuery.fn.extend({
 
-		// Handle object literals
-		if ( typeof type === "object" ) {
-			for ( var key in type ) {
-				this[ name ](key, data, type[key], fn);
+	on: function( types, selector, data, fn, /*INTERNAL*/ one ) {
+		var origFn, type;
+
+		// Types can be a map of types/handlers
+		if ( typeof types === "object" ) {
+			// ( types-Object, selector, data )
+			if ( typeof selector !== "string" ) { // && selector != null
+				// ( types-Object, data )
+				data = data || selector;
+				selector = undefined;
+			}
+			for ( type in types ) {
+				this.on( type, selector, data, types[ type ], one );
 			}
 			return this;
 		}
 
-		if ( arguments.length === 2 || data === false ) {
-			fn = data;
-			data = undefined;
+		if ( data == null && fn == null ) {
+			// ( types, fn )
+			fn = selector;
+			data = selector = undefined;
+		} else if ( fn == null ) {
+			if ( typeof selector === "string" ) {
+				// ( types, selector, fn )
+				fn = data;
+				data = undefined;
+			} else {
+				// ( types, data, fn )
+				fn = data;
+				data = selector;
+				selector = undefined;
+			}
+		}
+		if ( fn === false ) {
+			fn = returnFalse;
+		} else if ( !fn ) {
+			return this;
 		}
 
-		if ( name === "one" ) {
-			handler = function( event ) {
-				jQuery( this ).unbind( event, handler );
-				return fn.apply( this, arguments );
+		if ( one === 1 ) {
+			origFn = fn;
+			fn = function( event ) {
+				// Can use an empty set, since event contains the info
+				jQuery().off( event );
+				return origFn.apply( this, arguments );
 			};
-			handler.guid = fn.guid || jQuery.guid++;
-		} else {
-			handler = fn;
+			// Use same guid so caller can remove using origFn
+			fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ );
 		}
-
-		if ( type === "unload" && name !== "one" ) {
-			this.one( type, data, fn );
-
-		} else {
-			for ( var i = 0, l = this.length; i < l; i++ ) {
-				jQuery.event.add( this[i], type, handler, data );
+		return this.each( function() {
+			jQuery.event.add( this, types, fn, data, selector );
+		});
+	},
+	one: function( types, selector, data, fn ) {
+		return this.on( types, selector, data, fn, 1 );
+	},
+	off: function( types, selector, fn ) {
+		if ( types && types.preventDefault && types.handleObj ) {
+			// ( event )  dispatched jQuery.Event
+			var handleObj = types.handleObj;
+			jQuery( types.delegateTarget ).off(
+				handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType,
+				handleObj.selector,
+				handleObj.handler
+			);
+			return this;
+		}
+		if ( typeof types === "object" ) {
+			// ( types-object [, selector] )
+			for ( var type in types ) {
+				this.off( type, selector, types[ type ] );
 			}
+			return this;
 		}
+		if ( selector === false || typeof selector === "function" ) {
+			// ( types [, fn] )
+			fn = selector;
+			selector = undefined;
+		}
+		if ( fn === false ) {
+			fn = returnFalse;
+		}
+		return this.each(function() {
+			jQuery.event.remove( this, types, fn, selector );
+		});
+	},
 
+	bind: function( types, data, fn ) {
+		return this.on( types, null, data, fn );
+	},
+	unbind: function( types, fn ) {
+		return this.off( types, null, fn );
+	},
+
+	live: function( types, data, fn ) {
+		jQuery( this.context ).on( types, this.selector, data, fn );
 		return this;
-	};
-});
-
-jQuery.fn.extend({
-	unbind: function( type, fn ) {
-		// Handle object literals
-		if ( typeof type === "object" && !type.preventDefault ) {
-			for ( var key in type ) {
-				this.unbind(key, type[key]);
-			}
-
-		} else {
-			for ( var i = 0, l = this.length; i < l; i++ ) {
-				jQuery.event.remove( this[i], type, fn );
-			}
-		}
-
+	},
+	die: function( types, fn ) {
+		jQuery( this.context ).off( types, this.selector || "**", fn );
 		return this;
 	},
 
 	delegate: function( selector, types, data, fn ) {
-		return this.live( types, data, fn, selector );
+		return this.on( types, selector, data, fn );
 	},
-
 	undelegate: function( selector, types, fn ) {
-		if ( arguments.length === 0 ) {
-			return this.unbind( "live" );
-
-		} else {
-			return this.die( types, null, fn, selector );
-		}
+		// ( namespace ) or ( selector, types [, fn] )
+		return arguments.length == 1? this.off( selector, "**" ) : this.off( types, selector, fn );
 	},
 
 	trigger: function( type, data ) {
@@ -3499,7 +3869,6 @@ jQuery.fn.extend({
 			jQuery.event.trigger( type, data, this );
 		});
 	},
-
 	triggerHandler: function( type, data ) {
 		if ( this[0] ) {
 			return jQuery.event.trigger( type, data, this[0], true );
@@ -3513,8 +3882,8 @@ jQuery.fn.extend({
 			i = 0,
 			toggler = function( event ) {
 				// Figure out which function to execute
-				var lastToggle = ( jQuery.data( this, "lastToggle" + fn.guid ) || 0 ) % i;
-				jQuery.data( this, "lastToggle" + fn.guid, lastToggle + 1 );
+				var lastToggle = ( jQuery._data( this, "lastToggle" + fn.guid ) || 0 ) % i;
+				jQuery._data( this, "lastToggle" + fn.guid, lastToggle + 1 );
 
 				// Make sure that clicks stop
 				event.preventDefault();
@@ -3537,178 +3906,9 @@ jQuery.fn.extend({
 	}
 });
 
-var liveMap = {
-	focus: "focusin",
-	blur: "focusout",
-	mouseenter: "mouseover",
-	mouseleave: "mouseout"
-};
-
-jQuery.each(["live", "die"], function( i, name ) {
-	jQuery.fn[ name ] = function( types, data, fn, origSelector /* Internal Use Only */ ) {
-		var type, i = 0, match, namespaces, preType,
-			selector = origSelector || this.selector,
-			context = origSelector ? this : jQuery( this.context );
-
-		if ( typeof types === "object" && !types.preventDefault ) {
-			for ( var key in types ) {
-				context[ name ]( key, data, types[key], selector );
-			}
-
-			return this;
-		}
-
-		if ( name === "die" && !types &&
-					origSelector && origSelector.charAt(0) === "." ) {
-
-			context.unbind( origSelector );
-
-			return this;
-		}
-
-		if ( data === false || jQuery.isFunction( data ) ) {
-			fn = data || returnFalse;
-			data = undefined;
-		}
-
-		types = (types || "").split(" ");
-
-		while ( (type = types[ i++ ]) != null ) {
-			match = rnamespaces.exec( type );
-			namespaces = "";
-
-			if ( match )  {
-				namespaces = match[0];
-				type = type.replace( rnamespaces, "" );
-			}
-
-			if ( type === "hover" ) {
-				types.push( "mouseenter" + namespaces, "mouseleave" + namespaces );
-				continue;
-			}
-
-			preType = type;
-
-			if ( liveMap[ type ] ) {
-				types.push( liveMap[ type ] + namespaces );
-				type = type + namespaces;
-
-			} else {
-				type = (liveMap[ type ] || type) + namespaces;
-			}
-
-			if ( name === "live" ) {
-				// bind live handler
-				for ( var j = 0, l = context.length; j < l; j++ ) {
-					jQuery.event.add( context[j], "live." + liveConvert( type, selector ),
-						{ data: data, selector: selector, handler: fn, origType: type, origHandler: fn, preType: preType } );
-				}
-
-			} else {
-				// unbind live handler
-				context.unbind( "live." + liveConvert( type, selector ), fn );
-			}
-		}
-
-		return this;
-	};
-});
-
-function liveHandler( event ) {
-	var stop, maxLevel, related, match, handleObj, elem, j, i, l, data, close, namespace, ret,
-		elems = [],
-		selectors = [],
-		events = jQuery._data( this, "events" );
-
-	// Make sure we avoid non-left-click bubbling in Firefox (#3861) and disabled elements in IE (#6911)
-	if ( event.liveFired === this || !events || !events.live || event.target.disabled || event.button && event.type === "click" ) {
-		return;
-	}
-
-	if ( event.namespace ) {
-		namespace = new RegExp("(^|\\.)" + event.namespace.split(".").join("\\.(?:.*\\.)?") + "(\\.|$)");
-	}
-
-	event.liveFired = this;
-
-	var live = events.live.slice(0);
-
-	for ( j = 0; j < live.length; j++ ) {
-		handleObj = live[j];
-
-		if ( handleObj.origType.replace( rnamespaces, "" ) === event.type ) {
-			selectors.push( handleObj.selector );
-
-		} else {
-			live.splice( j--, 1 );
-		}
-	}
-
-	match = jQuery( event.target ).closest( selectors, event.currentTarget );
-
-	for ( i = 0, l = match.length; i < l; i++ ) {
-		close = match[i];
-
-		for ( j = 0; j < live.length; j++ ) {
-			handleObj = live[j];
-
-			if ( close.selector === handleObj.selector && (!namespace || namespace.test( handleObj.namespace )) && !close.elem.disabled ) {
-				elem = close.elem;
-				related = null;
-
-				// Those two events require additional checking
-				if ( handleObj.preType === "mouseenter" || handleObj.preType === "mouseleave" ) {
-					event.type = handleObj.preType;
-					related = jQuery( event.relatedTarget ).closest( handleObj.selector )[0];
-
-					// Make sure not to accidentally match a child element with the same selector
-					if ( related && jQuery.contains( elem, related ) ) {
-						related = elem;
-					}
-				}
-
-				if ( !related || related !== elem ) {
-					elems.push({ elem: elem, handleObj: handleObj, level: close.level });
-				}
-			}
-		}
-	}
-
-	for ( i = 0, l = elems.length; i < l; i++ ) {
-		match = elems[i];
-
-		if ( maxLevel && match.level > maxLevel ) {
-			break;
-		}
-
-		event.currentTarget = match.elem;
-		event.data = match.handleObj.data;
-		event.handleObj = match.handleObj;
-
-		ret = match.handleObj.origHandler.apply( match.elem, arguments );
-
-		if ( ret === false || event.isPropagationStopped() ) {
-			maxLevel = match.level;
-
-			if ( ret === false ) {
-				stop = false;
-			}
-			if ( event.isImmediatePropagationStopped() ) {
-				break;
-			}
-		}
-	}
-
-	return stop;
-}
-
-function liveConvert( type, selector ) {
-	return (type && type !== "*" ? type + "." : "") + selector.replace(rperiod, "`").replace(rspaces, "&");
-}
-
 jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblclick " +
 	"mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave " +
-	"change select submit keydown keypress keyup error").split(" "), function( i, name ) {
+	"change select submit keydown keypress keyup error contextmenu").split(" "), function( i, name ) {
 
 	// Handle event binding
 	jQuery.fn[ name ] = function( data, fn ) {
@@ -3718,13 +3918,21 @@ jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblcl
 		}
 
 		return arguments.length > 0 ?
-			this.bind( name, data, fn ) :
+			this.on( name, null, data, fn ) :
 			this.trigger( name );
 	};
 
 	if ( jQuery.attrFn ) {
 		jQuery.attrFn[ name ] = true;
 	}
+
+	if ( rkeyEvent.test( name ) ) {
+		jQuery.event.fixHooks[ name ] = jQuery.event.keyHooks;
+	}
+
+	if ( rmouseEvent.test( name ) ) {
+		jQuery.event.fixHooks[ name ] = jQuery.event.mouseHooks;
+	}
 });
 
 
@@ -3738,11 +3946,13 @@ jQuery.each( ("blur focus focusin focusout load resize scroll unload click dblcl
 (function(){
 
 var chunker = /((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,
+	expando = "sizcache" + (Math.random() + '').replace('.', ''),
 	done = 0,
 	toString = Object.prototype.toString,
 	hasDuplicate = false,
 	baseHasDuplicate = true,
 	rBackslash = /\\/g,
+	rReturn = /\r\n/g,
 	rNonWord = /\W/;
 
 // Here we check if the JavaScript engine is using some sort of
@@ -3763,7 +3973,7 @@ var Sizzle = function( selector, context, results, seed ) {
 	if ( context.nodeType !== 1 && context.nodeType !== 9 ) {
 		return [];
 	}
-	
+
 	if ( !selector || typeof selector !== "string" ) {
 		return results;
 	}
@@ -3773,7 +3983,7 @@ var Sizzle = function( selector, context, results, seed ) {
 		contextXML = Sizzle.isXML( context ),
 		parts = [],
 		soFar = selector;
-	
+
 	// Reset the position of the chunker regexp (start from head)
 	do {
 		chunker.exec( "" );
@@ -3781,9 +3991,9 @@ var Sizzle = function( selector, context, results, seed ) {
 
 		if ( m ) {
 			soFar = m[3];
-		
+
 			parts.push( m[1] );
-		
+
 			if ( m[2] ) {
 				extra = m[3];
 				break;
@@ -3794,7 +4004,7 @@ var Sizzle = function( selector, context, results, seed ) {
 	if ( parts.length > 1 && origPOS.exec( selector ) ) {
 
 		if ( parts.length === 2 && Expr.relative[ parts[0] ] ) {
-			set = posProcess( parts[0] + parts[1], context );
+			set = posProcess( parts[0] + parts[1], context, seed );
 
 		} else {
 			set = Expr.relative[ parts[0] ] ?
@@ -3807,8 +4017,8 @@ var Sizzle = function( selector, context, results, seed ) {
 				if ( Expr.relative[ selector ] ) {
 					selector += parts.shift();
 				}
-				
-				set = posProcess( selector, set );
+
+				set = posProcess( selector, set, seed );
 			}
 		}
 
@@ -3927,18 +4137,17 @@ Sizzle.matchesSelector = function( node, expr ) {
 };
 
 Sizzle.find = function( expr, context, isXML ) {
-	var set;
+	var set, i, len, match, type, left;
 
 	if ( !expr ) {
 		return [];
 	}
 
-	for ( var i = 0, l = Expr.order.length; i < l; i++ ) {
-		var match,
-			type = Expr.order[i];
-		
+	for ( i = 0, len = Expr.order.length; i < len; i++ ) {
+		type = Expr.order[i];
+
 		if ( (match = Expr.leftMatch[ type ].exec( expr )) ) {
-			var left = match[1];
+			left = match[1];
 			match.splice( 1, 1 );
 
 			if ( left.substr( left.length - 1 ) !== "\\" ) {
@@ -3964,17 +4173,18 @@ Sizzle.find = function( expr, context, isXML ) {
 
 Sizzle.filter = function( expr, set, inplace, not ) {
 	var match, anyFound,
+		type, found, item, filter, left,
+		i, pass,
 		old = expr,
 		result = [],
 		curLoop = set,
 		isXMLFilter = set && set[0] && Sizzle.isXML( set[0] );
 
 	while ( expr && set.length ) {
-		for ( var type in Expr.filter ) {
+		for ( type in Expr.filter ) {
 			if ( (match = Expr.leftMatch[ type ].exec( expr )) != null && match[2] ) {
-				var found, item,
-					filter = Expr.filter[ type ],
-					left = match[1];
+				filter = Expr.filter[ type ];
+				left = match[1];
 
 				anyFound = false;
 
@@ -4000,10 +4210,10 @@ Sizzle.filter = function( expr, set, inplace, not ) {
 				}
 
 				if ( match ) {
-					for ( var i = 0; (item = curLoop[i]) != null; i++ ) {
+					for ( i = 0; (item = curLoop[i]) != null; i++ ) {
 						if ( item ) {
 							found = filter( item, match, i, curLoop );
-							var pass = not ^ !!found;
+							pass = not ^ found;
 
 							if ( inplace && found != null ) {
 								if ( pass ) {
@@ -4054,7 +4264,46 @@ Sizzle.filter = function( expr, set, inplace, not ) {
 };
 
 Sizzle.error = function( msg ) {
-	throw "Syntax error, unrecognized expression: " + msg;
+	throw new Error( "Syntax error, unrecognized expression: " + msg );
+};
+
+/**
+ * Utility function for retreiving the text value of an array of DOM nodes
+ * @param {Array|Element} elem
+ */
+var getText = Sizzle.getText = function( elem ) {
+    var i, node,
+		nodeType = elem.nodeType,
+		ret = "";
+
+	if ( nodeType ) {
+		if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) {
+			// Use textContent || innerText for elements
+			if ( typeof elem.textContent === 'string' ) {
+				return elem.textContent;
+			} else if ( typeof elem.innerText === 'string' ) {
+				// Replace IE's carriage returns
+				return elem.innerText.replace( rReturn, '' );
+			} else {
+				// Traverse it's children
+				for ( elem = elem.firstChild; elem; elem = elem.nextSibling) {
+					ret += getText( elem );
+				}
+			}
+		} else if ( nodeType === 3 || nodeType === 4 ) {
+			return elem.nodeValue;
+		}
+	} else {
+
+		// If no nodeType, this is expected to be an array
+		for ( i = 0; (node = elem[i]); i++ ) {
+			// Do not traverse comment nodes
+			if ( node.nodeType !== 8 ) {
+				ret += getText( node );
+			}
+		}
+	}
+	return ret;
 };
 
 var Expr = Sizzle.selectors = {
@@ -4268,7 +4517,7 @@ var Expr = Sizzle.selectors = {
 
 		ATTR: function( match, curLoop, inplace, result, not, isXML ) {
 			var name = match[1] = match[1].replace( rBackslash, "" );
-			
+
 			if ( !isXML && Expr.attrMap[name] ) {
 				match[1] = Expr.attrMap[name];
 			}
@@ -4302,7 +4551,7 @@ var Expr = Sizzle.selectors = {
 			} else if ( Expr.match.POS.test( match[0] ) || Expr.match.CHILD.test( match[0] ) ) {
 				return true;
 			}
-			
+
 			return match;
 		},
 
@@ -4312,7 +4561,7 @@ var Expr = Sizzle.selectors = {
 			return match;
 		}
 	},
-	
+
 	filters: {
 		enabled: function( elem ) {
 			return elem.disabled === false && elem.type !== "hidden";
@@ -4325,14 +4574,14 @@ var Expr = Sizzle.selectors = {
 		checked: function( elem ) {
 			return elem.checked === true;
 		},
-		
+
 		selected: function( elem ) {
 			// Accessing this property makes selected-by-default
 			// options in Safari work properly
 			if ( elem.parentNode ) {
 				elem.parentNode.selectedIndex;
 			}
-			
+
 			return elem.selected === true;
 		},
 
@@ -4354,7 +4603,7 @@ var Expr = Sizzle.selectors = {
 
 		text: function( elem ) {
 			var attr = elem.getAttribute( "type" ), type = elem.type;
-			// IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) 
+			// IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc)
 			// use getAttribute instead to test this case
 			return elem.nodeName.toLowerCase() === "input" && "text" === type && ( attr === type || attr === null );
 		},
@@ -4444,7 +4693,7 @@ var Expr = Sizzle.selectors = {
 				return filter( elem, i, match, array );
 
 			} else if ( name === "contains" ) {
-				return (elem.textContent || elem.innerText || Sizzle.getText([ elem ]) || "").indexOf(match[3]) >= 0;
+				return (elem.textContent || elem.innerText || getText([ elem ]) || "").indexOf(match[3]) >= 0;
 
 			} else if ( name === "not" ) {
 				var not = match[3];
@@ -4463,57 +4712,61 @@ var Expr = Sizzle.selectors = {
 		},
 
 		CHILD: function( elem, match ) {
-			var type = match[1],
+			var first, last,
+				doneName, parent, cache,
+				count, diff,
+				type = match[1],
 				node = elem;
 
 			switch ( type ) {
 				case "only":
 				case "first":
-					while ( (node = node.previousSibling) )	 {
-						if ( node.nodeType === 1 ) { 
-							return false; 
+					while ( (node = node.previousSibling) ) {
+						if ( node.nodeType === 1 ) {
+							return false;
 						}
 					}
 
-					if ( type === "first" ) { 
-						return true; 
+					if ( type === "first" ) {
+						return true;
 					}
 
 					node = elem;
 
+					/* falls through */
 				case "last":
-					while ( (node = node.nextSibling) )	 {
-						if ( node.nodeType === 1 ) { 
-							return false; 
+					while ( (node = node.nextSibling) ) {
+						if ( node.nodeType === 1 ) {
+							return false;
 						}
 					}
 
 					return true;
 
 				case "nth":
-					var first = match[2],
-						last = match[3];
+					first = match[2];
+					last = match[3];
 
 					if ( first === 1 && last === 0 ) {
 						return true;
 					}
-					
-					var doneName = match[0],
-						parent = elem.parentNode;
-	
-					if ( parent && (parent.sizcache !== doneName || !elem.nodeIndex) ) {
-						var count = 0;
-						
+
+					doneName = match[0];
+					parent = elem.parentNode;
+
+					if ( parent && (parent[ expando ] !== doneName || !elem.nodeIndex) ) {
+						count = 0;
+
 						for ( node = parent.firstChild; node; node = node.nextSibling ) {
 							if ( node.nodeType === 1 ) {
 								node.nodeIndex = ++count;
 							}
-						} 
+						}
 
-						parent.sizcache = doneName;
+						parent[ expando ] = doneName;
 					}
-					
-					var diff = elem.nodeIndex - last;
+
+					diff = elem.nodeIndex - last;
 
 					if ( first === 0 ) {
 						return diff === 0;
@@ -4529,9 +4782,9 @@ var Expr = Sizzle.selectors = {
 		},
 
 		TAG: function( elem, match ) {
-			return (match === "*" && elem.nodeType === 1) || elem.nodeName.toLowerCase() === match;
+			return (match === "*" && elem.nodeType === 1) || !!elem.nodeName && elem.nodeName.toLowerCase() === match;
 		},
-		
+
 		CLASS: function( elem, match ) {
 			return (" " + (elem.className || elem.getAttribute("class")) + " ")
 				.indexOf( match ) > -1;
@@ -4539,7 +4792,9 @@ var Expr = Sizzle.selectors = {
 
 		ATTR: function( elem, match ) {
 			var name = match[1],
-				result = Expr.attrHandle[ name ] ?
+				result = Sizzle.attr ?
+					Sizzle.attr( elem, name ) :
+					Expr.attrHandle[ name ] ?
 					Expr.attrHandle[ name ]( elem ) :
 					elem[ name ] != null ?
 						elem[ name ] :
@@ -4550,6 +4805,8 @@ var Expr = Sizzle.selectors = {
 
 			return result == null ?
 				type === "!=" :
+				!type && Sizzle.attr ?
+				result != null :
 				type === "=" ?
 				value === check :
 				type === "*=" ?
@@ -4589,6 +4846,9 @@ for ( var type in Expr.match ) {
 	Expr.match[ type ] = new RegExp( Expr.match[ type ].source + (/(?![^\[]*\])(?![^\(]*\))/.source) );
 	Expr.leftMatch[ type ] = new RegExp( /(^(?:.|\r|\n)*?)/.source + Expr.match[ type ].source.replace(/\\(\d+)/g, fescape) );
 }
+// Expose origPOS
+// "global" as in regardless of relation to brackets/parens
+Expr.match.globalPOS = origPOS;
 
 var makeArray = function( array, results ) {
 	array = Array.prototype.slice.call( array, 0 );
@@ -4597,7 +4857,7 @@ var makeArray = function( array, results ) {
 		results.push.apply( results, array );
 		return results;
 	}
-	
+
 	return array;
 };
 
@@ -4730,26 +4990,6 @@ if ( document.documentElement.compareDocumentPosition ) {
 	};
 }
 
-// Utility function for retreiving the text value of an array of DOM nodes
-Sizzle.getText = function( elems ) {
-	var ret = "", elem;
-
-	for ( var i = 0; elems[i]; i++ ) {
-		elem = elems[i];
-
-		// Get the text from text nodes and CDATA nodes
-		if ( elem.nodeType === 3 || elem.nodeType === 4 ) {
-			ret += elem.nodeValue;
-
-		// Traverse everything else, except comment nodes
-		} else if ( elem.nodeType !== 8 ) {
-			ret += Sizzle.getText( elem.childNodes );
-		}
-	}
-
-	return ret;
-};
-
 // Check to see if the browser returns elements by name when
 // querying by getElementById (and provide a workaround)
 (function(){
@@ -4849,7 +5089,7 @@ if ( document.querySelectorAll ) {
 		if ( div.querySelectorAll && div.querySelectorAll(".TEST").length === 0 ) {
 			return;
 		}
-	
+
 		Sizzle = function( query, context, extra, seed ) {
 			context = context || document;
 
@@ -4858,24 +5098,24 @@ if ( document.querySelectorAll ) {
 			if ( !seed && !Sizzle.isXML(context) ) {
 				// See if we find a selector to speed up
 				var match = /^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec( query );
-				
+
 				if ( match && (context.nodeType === 1 || context.nodeType === 9) ) {
 					// Speed-up: Sizzle("TAG")
 					if ( match[1] ) {
 						return makeArray( context.getElementsByTagName( query ), extra );
-					
+
 					// Speed-up: Sizzle(".CLASS")
 					} else if ( match[2] && Expr.find.CLASS && context.getElementsByClassName ) {
 						return makeArray( context.getElementsByClassName( match[2] ), extra );
 					}
 				}
-				
+
 				if ( context.nodeType === 9 ) {
 					// Speed-up: Sizzle("body")
 					// The body element only exists once, optimize finding it
 					if ( query === "body" && context.body ) {
 						return makeArray( [ context.body ], extra );
-						
+
 					// Speed-up: Sizzle("#ID")
 					} else if ( match && match[3] ) {
 						var elem = context.getElementById( match[3] );
@@ -4888,12 +5128,12 @@ if ( document.querySelectorAll ) {
 							if ( elem.id === match[3] ) {
 								return makeArray( [ elem ], extra );
 							}
-							
+
 						} else {
 							return makeArray( [], extra );
 						}
 					}
-					
+
 					try {
 						return makeArray( context.querySelectorAll(query), extra );
 					} catch(qsaError) {}
@@ -4931,7 +5171,7 @@ if ( document.querySelectorAll ) {
 					}
 				}
 			}
-		
+
 			return oldSizzle(query, context, extra, seed);
 		};
 
@@ -4958,7 +5198,7 @@ if ( document.querySelectorAll ) {
 			// This should fail with an exception
 			// Gecko does not error, returns false instead
 			matches.call( document.documentElement, "[test!='']:sizzle" );
-	
+
 		} catch( pseudoError ) {
 			pseudoWorks = true;
 		}
@@ -4968,7 +5208,7 @@ if ( document.querySelectorAll ) {
 			expr = expr.replace(/\=\s*([^'"\]]*)\s*\]/g, "='$1']");
 
 			if ( !Sizzle.isXML( node ) ) {
-				try { 
+				try {
 					if ( pseudoWorks || !Expr.match.PSEUDO.test( expr ) && !/!=/.test( expr ) ) {
 						var ret = matches.call( node, expr );
 
@@ -5005,7 +5245,7 @@ if ( document.querySelectorAll ) {
 	if ( div.getElementsByClassName("e").length === 1 ) {
 		return;
 	}
-	
+
 	Expr.order.splice(1, 0, "CLASS");
 	Expr.find.CLASS = function( match, context, isXML ) {
 		if ( typeof context.getElementsByClassName !== "undefined" && !isXML ) {
@@ -5027,13 +5267,13 @@ function dirNodeCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) {
 			elem = elem[dir];
 
 			while ( elem ) {
-				if ( elem.sizcache === doneName ) {
+				if ( elem[ expando ] === doneName ) {
 					match = checkSet[elem.sizset];
 					break;
 				}
 
 				if ( elem.nodeType === 1 && !isXML ){
-					elem.sizcache = doneName;
+					elem[ expando ] = doneName;
 					elem.sizset = i;
 				}
 
@@ -5056,18 +5296,18 @@ function dirCheck( dir, cur, doneName, checkSet, nodeCheck, isXML ) {
 
 		if ( elem ) {
 			var match = false;
-			
+
 			elem = elem[dir];
 
 			while ( elem ) {
-				if ( elem.sizcache === doneName ) {
+				if ( elem[ expando ] === doneName ) {
 					match = checkSet[elem.sizset];
 					break;
 				}
 
 				if ( elem.nodeType === 1 ) {
 					if ( !isXML ) {
-						elem.sizcache = doneName;
+						elem[ expando ] = doneName;
 						elem.sizset = i;
 					}
 
@@ -5109,13 +5349,13 @@ if ( document.documentElement.contains ) {
 
 Sizzle.isXML = function( elem ) {
 	// documentElement is verified for cases where it doesn't yet exist
-	// (such as loading iframes in IE - #4833) 
+	// (such as loading iframes in IE - #4833)
 	var documentElement = (elem ? elem.ownerDocument || elem : 0).documentElement;
 
 	return documentElement ? documentElement.nodeName !== "HTML" : false;
 };
 
-var posProcess = function( selector, context ) {
+var posProcess = function( selector, context, seed ) {
 	var match,
 		tmpSet = [],
 		later = "",
@@ -5131,13 +5371,16 @@ var posProcess = function( selector, context ) {
 	selector = Expr.relative[selector] ? selector + "*" : selector;
 
 	for ( var i = 0, l = root.length; i < l; i++ ) {
-		Sizzle( selector, root[i], tmpSet );
+		Sizzle( selector, root[i], tmpSet, seed );
 	}
 
 	return Sizzle.filter( later, tmpSet );
 };
 
 // EXPOSE
+// Override sizzle attribute retrieval
+Sizzle.attr = jQuery.attr;
+Sizzle.selectors.attrMap = {};
 jQuery.find = Sizzle;
 jQuery.expr = Sizzle.selectors;
 jQuery.expr[":"] = jQuery.expr.filters;
@@ -5156,7 +5399,7 @@ var runtil = /Until$/,
 	rmultiselector = /,/,
 	isSimple = /^.[^:#\[\.,]*$/,
 	slice = Array.prototype.slice,
-	POS = jQuery.expr.match.POS,
+	POS = jQuery.expr.match.globalPOS,
 	// methods guaranteed to produce a unique set when starting from a unique set
 	guaranteedUnique = {
 		children: true,
@@ -5223,43 +5466,33 @@ jQuery.fn.extend({
 	},
 
 	is: function( selector ) {
-		return !!selector && ( typeof selector === "string" ?
-			jQuery.filter( selector, this ).length > 0 :
-			this.filter( selector ).length > 0 );
+		return !!selector && (
+			typeof selector === "string" ?
+				// If this is a positional selector, check membership in the returned set
+				// so $("p:first").is("p:last") won't return true for a doc with two "p".
+				POS.test( selector ) ?
+					jQuery( selector, this.context ).index( this[0] ) >= 0 :
+					jQuery.filter( selector, this ).length > 0 :
+				this.filter( selector ).length > 0 );
 	},
 
 	closest: function( selectors, context ) {
 		var ret = [], i, l, cur = this[0];
-		
-		// Array
+
+		// Array (deprecated as of jQuery 1.7)
 		if ( jQuery.isArray( selectors ) ) {
-			var match, selector,
-				matches = {},
-				level = 1;
+			var level = 1;
 
-			if ( cur && selectors.length ) {
-				for ( i = 0, l = selectors.length; i < l; i++ ) {
-					selector = selectors[i];
+			while ( cur && cur.ownerDocument && cur !== context ) {
+				for ( i = 0; i < selectors.length; i++ ) {
 
-					if ( !matches[ selector ] ) {
-						matches[ selector ] = POS.test( selector ) ?
-							jQuery( selector, context || this.context ) :
-							selector;
+					if ( jQuery( cur ).is( selectors[ i ] ) ) {
+						ret.push({ selector: selectors[ i ], elem: cur, level: level });
 					}
 				}
 
-				while ( cur && cur.ownerDocument && cur !== context ) {
-					for ( selector in matches ) {
-						match = matches[ selector ];
-
-						if ( match.jquery ? match.index( cur ) > -1 : jQuery( cur ).is( match ) ) {
-							ret.push({ selector: selector, elem: cur, level: level });
-						}
-					}
-
-					cur = cur.parentNode;
-					level++;
-				}
+				cur = cur.parentNode;
+				level++;
 			}
 
 			return ret;
@@ -5295,12 +5528,17 @@ jQuery.fn.extend({
 	// Determine the position of an element within
 	// the matched set of elements
 	index: function( elem ) {
-		if ( !elem || typeof elem === "string" ) {
-			return jQuery.inArray( this[0],
-				// If it receives a string, the selector is used
-				// If it receives nothing, the siblings are used
-				elem ? jQuery( elem ) : this.parent().children() );
+
+		// No argument, return index in parent
+		if ( !elem ) {
+			return ( this[0] && this[0].parentNode ) ? this.prevAll().length : -1;
 		}
+
+		// index in selector
+		if ( typeof elem === "string" ) {
+			return jQuery.inArray( this[0], jQuery( elem ) );
+		}
+
 		// Locate the position of the desired element
 		return jQuery.inArray(
 			// If it receives a jQuery object, the first element is used
@@ -5359,7 +5597,7 @@ jQuery.each({
 		return jQuery.dir( elem, "previousSibling", until );
 	},
 	siblings: function( elem ) {
-		return jQuery.sibling( elem.parentNode.firstChild, elem );
+		return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem );
 	},
 	children: function( elem ) {
 		return jQuery.sibling( elem.firstChild );
@@ -5371,12 +5609,7 @@ jQuery.each({
 	}
 }, function( name, fn ) {
 	jQuery.fn[ name ] = function( until, selector ) {
-		var ret = jQuery.map( this, fn, until ),
-			// The variable 'args' was introduced in
-			// https://github.com/jquery/jquery/commit/52a0238
-			// to work around a bug in Chrome 10 (Dev) and should be removed when the bug is fixed.
-			// http://code.google.com/p/v8/issues/detail?id=1050
-			args = slice.call(arguments);
+		var ret = jQuery.map( this, fn, until );
 
 		if ( !runtil.test( name ) ) {
 			selector = until;
@@ -5392,7 +5625,7 @@ jQuery.each({
 			ret = ret.reverse();
 		}
 
-		return this.pushStack( ret, name, args.join(",") );
+		return this.pushStack( ret, name, slice.call( arguments ).join(",") );
 	};
 });
 
@@ -5461,7 +5694,7 @@ function winnow( elements, qualifier, keep ) {
 
 	} else if ( qualifier.nodeType ) {
 		return jQuery.grep(elements, function( elem, i ) {
-			return (elem === qualifier) === keep;
+			return ( elem === qualifier ) === keep;
 		});
 
 	} else if ( typeof qualifier === "string" ) {
@@ -5477,20 +5710,38 @@ function winnow( elements, qualifier, keep ) {
 	}
 
 	return jQuery.grep(elements, function( elem, i ) {
-		return (jQuery.inArray( elem, qualifier ) >= 0) === keep;
+		return ( jQuery.inArray( elem, qualifier ) >= 0 ) === keep;
 	});
 }
 
 
 
 
-var rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g,
+function createSafeFragment( document ) {
+	var list = nodeNames.split( "|" ),
+	safeFrag = document.createDocumentFragment();
+
+	if ( safeFrag.createElement ) {
+		while ( list.length ) {
+			safeFrag.createElement(
+				list.pop()
+			);
+		}
+	}
+	return safeFrag;
+}
+
+var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" +
+		"header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",
+	rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g,
 	rleadingWhitespace = /^\s+/,
 	rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,
 	rtagName = /<([\w:]+)/,
 	rtbody = /<tbody/i,
 	rhtml = /<|&#?\w+;/,
+	rnoInnerhtml = /<(?:script|style)/i,
 	rnocache = /<(?:script|object|embed|option|style)/i,
+	rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"),
 	// checked="checked" or checked
 	rchecked = /checked\s*(?:[^=]|=\s*.checked.)/i,
 	rscriptType = /\/(java|ecma)script/i,
@@ -5504,7 +5755,8 @@ var rinlinejQuery = / jQuery\d+="(?:\d+|null)"/g,
 		col: [ 2, "<table><tbody></tbody><colgroup>", "</colgroup></table>" ],
 		area: [ 1, "<map>", "</map>" ],
 		_default: [ 0, "", "" ]
-	};
+	},
+	safeFragment = createSafeFragment( document );
 
 wrapMap.optgroup = wrapMap.option;
 wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead;
@@ -5516,20 +5768,12 @@ if ( !jQuery.support.htmlSerialize ) {
 }
 
 jQuery.fn.extend({
-	text: function( text ) {
-		if ( jQuery.isFunction(text) ) {
-			return this.each(function(i) {
-				var self = jQuery( this );
-
-				self.text( text.call(this, i, self.text()) );
-			});
-		}
-
-		if ( typeof text !== "object" && text !== undefined ) {
-			return this.empty().append( (this[0] && this[0].ownerDocument || document).createTextNode( text ) );
-		}
-
-		return jQuery.text( this );
+	text: function( value ) {
+		return jQuery.access( this, function( value ) {
+			return value === undefined ?
+				jQuery.text( this ) :
+				this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) );
+		}, null, value, arguments.length );
 	},
 
 	wrapAll: function( html ) {
@@ -5582,8 +5826,10 @@ jQuery.fn.extend({
 	},
 
 	wrap: function( html ) {
-		return this.each(function() {
-			jQuery( this ).wrapAll( html );
+		var isFunction = jQuery.isFunction( html );
+
+		return this.each(function(i) {
+			jQuery( this ).wrapAll( isFunction ? html.call(this, i) : html );
 		});
 	},
 
@@ -5617,7 +5863,7 @@ jQuery.fn.extend({
 				this.parentNode.insertBefore( elem, this );
 			});
 		} else if ( arguments.length ) {
-			var set = jQuery(arguments[0]);
+			var set = jQuery.clean( arguments );
 			set.push.apply( set, this.toArray() );
 			return this.pushStack( set, "before", arguments );
 		}
@@ -5630,7 +5876,7 @@ jQuery.fn.extend({
 			});
 		} else if ( arguments.length ) {
 			var set = this.pushStack( this, "after", arguments );
-			set.push.apply( set, jQuery(arguments[0]).toArray() );
+			set.push.apply( set, jQuery.clean(arguments) );
 			return set;
 		}
 	},
@@ -5679,44 +5925,44 @@ jQuery.fn.extend({
 	},
 
 	html: function( value ) {
-		if ( value === undefined ) {
-			return this[0] && this[0].nodeType === 1 ?
-				this[0].innerHTML.replace(rinlinejQuery, "") :
-				null;
+		return jQuery.access( this, function( value ) {
+			var elem = this[0] || {},
+				i = 0,
+				l = this.length;
 
-		// See if we can take a shortcut and just use innerHTML
-		} else if ( typeof value === "string" && !rnocache.test( value ) &&
-			(jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value )) &&
-			!wrapMap[ (rtagName.exec( value ) || ["", ""])[1].toLowerCase() ] ) {
-
-			value = value.replace(rxhtmlTag, "<$1></$2>");
-
-			try {
-				for ( var i = 0, l = this.length; i < l; i++ ) {
-					// Remove element nodes and prevent memory leaks
-					if ( this[i].nodeType === 1 ) {
-						jQuery.cleanData( this[i].getElementsByTagName("*") );
-						this[i].innerHTML = value;
-					}
-				}
-
-			// If using innerHTML throws an exception, use the fallback method
-			} catch(e) {
-				this.empty().append( value );
+			if ( value === undefined ) {
+				return elem.nodeType === 1 ?
+					elem.innerHTML.replace( rinlinejQuery, "" ) :
+					null;
 			}
 
-		} else if ( jQuery.isFunction( value ) ) {
-			this.each(function(i){
-				var self = jQuery( this );
 
-				self.html( value.call(this, i, self.html()) );
-			});
+			if ( typeof value === "string" && !rnoInnerhtml.test( value ) &&
+				( jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value ) ) &&
+				!wrapMap[ ( rtagName.exec( value ) || ["", ""] )[1].toLowerCase() ] ) {
 
-		} else {
-			this.empty().append( value );
-		}
+				value = value.replace( rxhtmlTag, "<$1></$2>" );
 
-		return this;
+				try {
+					for (; i < l; i++ ) {
+						// Remove element nodes and prevent memory leaks
+						elem = this[i] || {};
+						if ( elem.nodeType === 1 ) {
+							jQuery.cleanData( elem.getElementsByTagName( "*" ) );
+							elem.innerHTML = value;
+						}
+					}
+
+					elem = 0;
+
+				// If using innerHTML throws an exception, use the fallback method
+				} catch(e) {}
+			}
+
+			if ( elem ) {
+				this.empty().append( value );
+			}
+		}, null, value, arguments.length );
 	},
 
 	replaceWith: function( value ) {
@@ -5811,7 +6057,7 @@ jQuery.fn.extend({
 						// in certain situations (Bug #8070).
 						// Fragments from the fragment cache must always be cloned and never used
 						// in place.
-						results.cacheable || (l > 1 && i < lastIndex) ?
+						results.cacheable || ( l > 1 && i < lastIndex ) ?
 							jQuery.clone( fragment, true, true ) :
 							fragment
 					);
@@ -5819,7 +6065,23 @@ jQuery.fn.extend({
 			}
 
 			if ( scripts.length ) {
-				jQuery.each( scripts, evalScript );
+				jQuery.each( scripts, function( i, elem ) {
+					if ( elem.src ) {
+						jQuery.ajax({
+							type: "GET",
+							global: false,
+							url: elem.src,
+							async: false,
+							dataType: "script"
+						});
+					} else {
+						jQuery.globalEval( ( elem.text || elem.textContent || elem.innerHTML || "" ).replace( rcleanScript, "/*$0*/" ) );
+					}
+
+					if ( elem.parentNode ) {
+						elem.parentNode.removeChild( elem );
+					}
+				});
 			}
 		}
 
@@ -5840,27 +6102,26 @@ function cloneCopyEvent( src, dest ) {
 		return;
 	}
 
-	var internalKey = jQuery.expando,
-		oldData = jQuery.data( src ),
-		curData = jQuery.data( dest, oldData );
+	var type, i, l,
+		oldData = jQuery._data( src ),
+		curData = jQuery._data( dest, oldData ),
+		events = oldData.events;
 
-	// Switch to use the internal data object, if it exists, for the next
-	// stage of data copying
-	if ( (oldData = oldData[ internalKey ]) ) {
-		var events = oldData.events;
-				curData = curData[ internalKey ] = jQuery.extend({}, oldData);
+	if ( events ) {
+		delete curData.handle;
+		curData.events = {};
 
-		if ( events ) {
-			delete curData.handle;
-			curData.events = {};
-
-			for ( var type in events ) {
-				for ( var i = 0, l = events[ type ].length; i < l; i++ ) {
-					jQuery.event.add( dest, type + ( events[ type ][ i ].namespace ? "." : "" ) + events[ type ][ i ].namespace, events[ type ][ i ], events[ type ][ i ].data );
-				}
+		for ( type in events ) {
+			for ( i = 0, l = events[ type ].length; i < l; i++ ) {
+				jQuery.event.add( dest, type, events[ type ][ i ] );
 			}
 		}
 	}
+
+	// make the cloned public data object a copy from the original
+	if ( curData.data ) {
+		curData.data = jQuery.extend( {}, curData.data );
+	}
 }
 
 function cloneFixAttributes( src, dest ) {
@@ -5914,24 +6175,34 @@ function cloneFixAttributes( src, dest ) {
 	// cloning other types of input fields
 	} else if ( nodeName === "input" || nodeName === "textarea" ) {
 		dest.defaultValue = src.defaultValue;
+
+	// IE blanks contents when cloning scripts
+	} else if ( nodeName === "script" && dest.text !== src.text ) {
+		dest.text = src.text;
 	}
 
 	// Event data gets referenced instead of copied if the expando
 	// gets copied too
 	dest.removeAttribute( jQuery.expando );
+
+	// Clear flags for bubbling special change/submit events, they must
+	// be reattached when the newly cloned events are first activated
+	dest.removeAttribute( "_submit_attached" );
+	dest.removeAttribute( "_change_attached" );
 }
 
 jQuery.buildFragment = function( args, nodes, scripts ) {
-	var fragment, cacheable, cacheresults, doc;
+	var fragment, cacheable, cacheresults, doc,
+	first = args[ 0 ];
 
-  // nodes may contain either an explicit document object,
-  // a jQuery collection or context object.
-  // If nodes[0] contains a valid object to assign to doc
-  if ( nodes && nodes[0] ) {
-    doc = nodes[0].ownerDocument || nodes[0];
-  }
+	// nodes may contain either an explicit document object,
+	// a jQuery collection or context object.
+	// If nodes[0] contains a valid object to assign to doc
+	if ( nodes && nodes[0] ) {
+		doc = nodes[0].ownerDocument || nodes[0];
+	}
 
-  // Ensure that an attr object doesn't incorrectly stand in as a document object
+	// Ensure that an attr object doesn't incorrectly stand in as a document object
 	// Chrome and Firefox seem to allow this to occur and will throw exception
 	// Fixes #8950
 	if ( !doc.createDocumentFragment ) {
@@ -5942,12 +6213,15 @@ jQuery.buildFragment = function( args, nodes, scripts ) {
 	// Cloning options loses the selected state, so don't cache them
 	// IE 6 doesn't like it when you put <object> or <embed> elements in a fragment
 	// Also, WebKit does not clone 'checked' attributes on cloneNode, so don't cache
-	if ( args.length === 1 && typeof args[0] === "string" && args[0].length < 512 && doc === document &&
-		args[0].charAt(0) === "<" && !rnocache.test( args[0] ) && (jQuery.support.checkClone || !rchecked.test( args[0] )) ) {
+	// Lastly, IE6,7,8 will not correctly reuse cached fragments that were created from unknown elems #10501
+	if ( args.length === 1 && typeof first === "string" && first.length < 512 && doc === document &&
+		first.charAt(0) === "<" && !rnocache.test( first ) &&
+		(jQuery.support.checkClone || !rchecked.test( first )) &&
+		(jQuery.support.html5Clone || !rnoshimcache.test( first )) ) {
 
 		cacheable = true;
 
-		cacheresults = jQuery.fragments[ args[0] ];
+		cacheresults = jQuery.fragments[ first ];
 		if ( cacheresults && cacheresults !== 1 ) {
 			fragment = cacheresults;
 		}
@@ -5959,7 +6233,7 @@ jQuery.buildFragment = function( args, nodes, scripts ) {
 	}
 
 	if ( cacheable ) {
-		jQuery.fragments[ args[0] ] = cacheresults ? fragment : 1;
+		jQuery.fragments[ first ] = cacheresults ? fragment : 1;
 	}
 
 	return { fragment: fragment, cacheable: cacheable };
@@ -5985,7 +6259,7 @@ jQuery.each({
 
 		} else {
 			for ( var i = 0, l = insert.length; i < l; i++ ) {
-				var elems = (i > 0 ? this.clone(true) : this).get();
+				var elems = ( i > 0 ? this.clone(true) : this ).get();
 				jQuery( insert[i] )[ original ]( elems );
 				ret = ret.concat( elems );
 			}
@@ -5996,10 +6270,10 @@ jQuery.each({
 });
 
 function getAll( elem ) {
-	if ( "getElementsByTagName" in elem ) {
+	if ( typeof elem.getElementsByTagName !== "undefined" ) {
 		return elem.getElementsByTagName( "*" );
 
-	} else if ( "querySelectorAll" in elem ) {
+	} else if ( typeof elem.querySelectorAll !== "undefined" ) {
 		return elem.querySelectorAll( "*" );
 
 	} else {
@@ -6015,19 +6289,33 @@ function fixDefaultChecked( elem ) {
 }
 // Finds all inputs and passes them to fixDefaultChecked
 function findInputs( elem ) {
-	if ( jQuery.nodeName( elem, "input" ) ) {
+	var nodeName = ( elem.nodeName || "" ).toLowerCase();
+	if ( nodeName === "input" ) {
 		fixDefaultChecked( elem );
-	} else if ( "getElementsByTagName" in elem ) {
+	// Skip scripts, get other children
+	} else if ( nodeName !== "script" && typeof elem.getElementsByTagName !== "undefined" ) {
 		jQuery.grep( elem.getElementsByTagName("input"), fixDefaultChecked );
 	}
 }
 
+// Derived From: http://www.iecss.com/shimprove/javascript/shimprove.1-0-1.js
+function shimCloneNode( elem ) {
+	var div = document.createElement( "div" );
+	safeFragment.appendChild( div );
+
+	div.innerHTML = elem.outerHTML;
+	return div.firstChild;
+}
+
 jQuery.extend({
 	clone: function( elem, dataAndEvents, deepDataAndEvents ) {
-		var clone = elem.cloneNode(true),
-				srcElements,
-				destElements,
-				i;
+		var srcElements,
+			destElements,
+			i,
+			// IE<=8 does not properly clone detached, unknown element nodes
+			clone = jQuery.support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ?
+				elem.cloneNode( true ) :
+				shimCloneNode( elem );
 
 		if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) &&
 				(elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) {
@@ -6039,8 +6327,7 @@ jQuery.extend({
 
 			cloneFixAttributes( elem, clone );
 
-			// Using Sizzle here is crazy slow, so we use getElementsByTagName
-			// instead
+			// Using Sizzle here is crazy slow, so we use getElementsByTagName instead
 			srcElements = getAll( elem );
 			destElements = getAll( clone );
 
@@ -6048,7 +6335,10 @@ jQuery.extend({
 			// with an element if you are cloning the body and one of the
 			// elements on the page has a name or id of "length"
 			for ( i = 0; srcElements[i]; ++i ) {
-				cloneFixAttributes( srcElements[i], destElements[i] );
+				// Ensure that the destination node is not null; Fixes #9587
+				if ( destElements[i] ) {
+					cloneFixAttributes( srcElements[i], destElements[i] );
+				}
 			}
 		}
 
@@ -6073,7 +6363,8 @@ jQuery.extend({
 	},
 
 	clean: function( elems, context, fragment, scripts ) {
-		var checkScriptType;
+		var checkScriptType, script, j,
+				ret = [];
 
 		context = context || document;
 
@@ -6082,8 +6373,6 @@ jQuery.extend({
 			context = context.ownerDocument || context[0] && context[0].ownerDocument || document;
 		}
 
-		var ret = [], j;
-
 		for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) {
 			if ( typeof elem === "number" ) {
 				elem += "";
@@ -6102,10 +6391,21 @@ jQuery.extend({
 					elem = elem.replace(rxhtmlTag, "<$1></$2>");
 
 					// Trim whitespace, otherwise indexOf won't work as expected
-					var tag = (rtagName.exec( elem ) || ["", ""])[1].toLowerCase(),
+					var tag = ( rtagName.exec( elem ) || ["", ""] )[1].toLowerCase(),
 						wrap = wrapMap[ tag ] || wrapMap._default,
 						depth = wrap[0],
-						div = context.createElement("div");
+						div = context.createElement("div"),
+						safeChildNodes = safeFragment.childNodes,
+						remove;
+
+					// Append wrapper element to unknown element safe doc fragment
+					if ( context === document ) {
+						// Use the fragment we've already created for this document
+						safeFragment.appendChild( div );
+					} else {
+						// Use a fragment created with the owner document
+						createSafeFragment( context ).appendChild( div );
+					}
 
 					// Go to html and back, then peel off extra wrappers
 					div.innerHTML = wrap[1] + elem + wrap[2];
@@ -6141,6 +6441,21 @@ jQuery.extend({
 					}
 
 					elem = div.childNodes;
+
+					// Clear elements from DocumentFragment (safeFragment or otherwise)
+					// to avoid hoarding elements. Fixes #11356
+					if ( div ) {
+						div.parentNode.removeChild( div );
+
+						// Guard against -1 index exceptions in FF3.6
+						if ( safeChildNodes.length > 0 ) {
+							remove = safeChildNodes[ safeChildNodes.length - 1 ];
+
+							if ( remove && remove.parentNode ) {
+								remove.parentNode.removeChild( remove );
+							}
+						}
+					}
 				}
 			}
 
@@ -6169,16 +6484,17 @@ jQuery.extend({
 				return !elem.type || rscriptType.test( elem.type );
 			};
 			for ( i = 0; ret[i]; i++ ) {
-				if ( scripts && jQuery.nodeName( ret[i], "script" ) && (!ret[i].type || ret[i].type.toLowerCase() === "text/javascript") ) {
-					scripts.push( ret[i].parentNode ? ret[i].parentNode.removeChild( ret[i] ) : ret[i] );
+				script = ret[i];
+				if ( scripts && jQuery.nodeName( script, "script" ) && (!script.type || rscriptType.test( script.type )) ) {
+					scripts.push( script.parentNode ? script.parentNode.removeChild( script ) : script );
 
 				} else {
-					if ( ret[i].nodeType === 1 ) {
-						var jsTags = jQuery.grep( ret[i].getElementsByTagName( "script" ), checkScriptType );
+					if ( script.nodeType === 1 ) {
+						var jsTags = jQuery.grep( script.getElementsByTagName( "script" ), checkScriptType );
 
 						ret.splice.apply( ret, [i + 1, 0].concat( jsTags ) );
 					}
-					fragment.appendChild( ret[i] );
+					fragment.appendChild( script );
 				}
 			}
 		}
@@ -6187,7 +6503,9 @@ jQuery.extend({
 	},
 
 	cleanData: function( elems ) {
-		var data, id, cache = jQuery.cache, internalKey = jQuery.expando, special = jQuery.event.special,
+		var data, id,
+			cache = jQuery.cache,
+			special = jQuery.event.special,
 			deleteExpando = jQuery.support.deleteExpando;
 
 		for ( var i = 0, elem; (elem = elems[i]) != null; i++ ) {
@@ -6198,7 +6516,7 @@ jQuery.extend({
 			id = elem[ jQuery.expando ];
 
 			if ( id ) {
-				data = cache[ id ] && cache[ id ][ internalKey ];
+				data = cache[ id ];
 
 				if ( data && data.events ) {
 					for ( var type in data.events ) {
@@ -6230,21 +6548,6 @@ jQuery.extend({
 	}
 });
 
-function evalScript( i, elem ) {
-	if ( elem.src ) {
-		jQuery.ajax({
-			url: elem.src,
-			async: false,
-			dataType: "script"
-		});
-	} else {
-		jQuery.globalEval( ( elem.text || elem.textContent || elem.innerHTML || "" ).replace( rcleanScript, "/*$0*/" ) );
-	}
-
-	if ( elem.parentNode ) {
-		elem.parentNode.removeChild( elem );
-	}
-}
 
 
 
@@ -6252,30 +6555,27 @@ var ralpha = /alpha\([^)]*\)/i,
 	ropacity = /opacity=([^)]*)/,
 	// fixed for IE9, see #8346
 	rupper = /([A-Z]|^ms)/g,
-	rnumpx = /^-?\d+(?:px)?$/i,
-	rnum = /^-?\d/,
-	rrelNum = /^[+\-]=/,
-	rrelNumFilter = /[^+\-\.\de]+/g,
+	rnum = /^[\-+]?(?:\d*\.)?\d+$/i,
+	rnumnonpx = /^-?(?:\d*\.)?\d+(?!px)[^\d\s]+$/i,
+	rrelNum = /^([\-+])=([\-+.\de]+)/,
+	rmargin = /^margin/,
 
 	cssShow = { position: "absolute", visibility: "hidden", display: "block" },
-	cssWidth = [ "Left", "Right" ],
-	cssHeight = [ "Top", "Bottom" ],
+
+	// order is important!
+	cssExpand = [ "Top", "Right", "Bottom", "Left" ],
+
 	curCSS,
 
 	getComputedStyle,
 	currentStyle;
 
 jQuery.fn.css = function( name, value ) {
-	// Setting 'undefined' is a no-op
-	if ( arguments.length === 2 && value === undefined ) {
-		return this;
-	}
-
-	return jQuery.access( this, name, value, true, function( elem, name, value ) {
+	return jQuery.access( this, function( elem, name, value ) {
 		return value !== undefined ?
 			jQuery.style( elem, name, value ) :
 			jQuery.css( elem, name );
-	});
+	}, name, value, arguments.length > 1 );
 };
 
 jQuery.extend({
@@ -6286,7 +6586,7 @@ jQuery.extend({
 			get: function( elem, computed ) {
 				if ( computed ) {
 					// We should always get a number back from opacity
-					var ret = curCSS( elem, "opacity", "opacity" );
+					var ret = curCSS( elem, "opacity" );
 					return ret === "" ? "1" : ret;
 
 				} else {
@@ -6332,18 +6632,18 @@ jQuery.extend({
 		if ( value !== undefined ) {
 			type = typeof value;
 
-			// Make sure that NaN and null values aren't set. See: #7116
-			if ( type === "number" && isNaN( value ) || value == null ) {
-				return;
-			}
-
 			// convert relative number strings (+= or -=) to relative numbers. #7345
-			if ( type === "string" && rrelNum.test( value ) ) {
-				value = +value.replace( rrelNumFilter, "" ) + parseFloat( jQuery.css( elem, name ) );
+			if ( type === "string" && (ret = rrelNum.exec( value )) ) {
+				value = ( +( ret[1] + 1) * +ret[2] ) + parseFloat( jQuery.css( elem, name ) );
 				// Fixes bug #9237
 				type = "number";
 			}
 
+			// Make sure that NaN and null values aren't set. See: #7116
+			if ( value == null || type === "number" && isNaN( value ) ) {
+				return;
+			}
+
 			// If a number was passed in, add 'px' to the (except for certain CSS properties)
 			if ( type === "number" && !jQuery.cssNumber[ origName ] ) {
 				value += "px";
@@ -6394,154 +6694,87 @@ jQuery.extend({
 
 	// A method for quickly swapping in/out CSS properties to get correct calculations
 	swap: function( elem, options, callback ) {
-		var old = {};
+		var old = {},
+			ret, name;
 
 		// Remember the old values, and insert the new ones
-		for ( var name in options ) {
+		for ( name in options ) {
 			old[ name ] = elem.style[ name ];
 			elem.style[ name ] = options[ name ];
 		}
 
-		callback.call( elem );
+		ret = callback.call( elem );
 
 		// Revert the old values
 		for ( name in options ) {
 			elem.style[ name ] = old[ name ];
 		}
+
+		return ret;
 	}
 });
 
-// DEPRECATED, Use jQuery.css() instead
+// DEPRECATED in 1.3, Use jQuery.css() instead
 jQuery.curCSS = jQuery.css;
 
-jQuery.each(["height", "width"], function( i, name ) {
-	jQuery.cssHooks[ name ] = {
-		get: function( elem, computed, extra ) {
-			var val;
-
-			if ( computed ) {
-				if ( elem.offsetWidth !== 0 ) {
-					return getWH( elem, name, extra );
-				} else {
-					jQuery.swap( elem, cssShow, function() {
-						val = getWH( elem, name, extra );
-					});
-				}
-
-				return val;
-			}
-		},
-
-		set: function( elem, value ) {
-			if ( rnumpx.test( value ) ) {
-				// ignore negative width and height values #1599
-				value = parseFloat( value );
-
-				if ( value >= 0 ) {
-					return value + "px";
-				}
-
-			} else {
-				return value;
-			}
-		}
-	};
-});
-
-if ( !jQuery.support.opacity ) {
-	jQuery.cssHooks.opacity = {
-		get: function( elem, computed ) {
-			// IE uses filters for opacity
-			return ropacity.test( (computed && elem.currentStyle ? elem.currentStyle.filter : elem.style.filter) || "" ) ?
-				( parseFloat( RegExp.$1 ) / 100 ) + "" :
-				computed ? "1" : "";
-		},
-
-		set: function( elem, value ) {
-			var style = elem.style,
-				currentStyle = elem.currentStyle;
-
-			// IE has trouble with opacity if it does not have layout
-			// Force it by setting the zoom level
-			style.zoom = 1;
-
-			// Set the alpha filter to set the opacity
-			var opacity = jQuery.isNaN( value ) ?
-				"" :
-				"alpha(opacity=" + value * 100 + ")",
-				filter = currentStyle && currentStyle.filter || style.filter || "";
-
-			style.filter = ralpha.test( filter ) ?
-				filter.replace( ralpha, opacity ) :
-				filter + " " + opacity;
-		}
-	};
-}
-
-jQuery(function() {
-	// This hook cannot be added until DOM ready because the support test
-	// for it is not run until after DOM ready
-	if ( !jQuery.support.reliableMarginRight ) {
-		jQuery.cssHooks.marginRight = {
-			get: function( elem, computed ) {
-				// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
-				// Work around by temporarily setting element display to inline-block
-				var ret;
-				jQuery.swap( elem, { "display": "inline-block" }, function() {
-					if ( computed ) {
-						ret = curCSS( elem, "margin-right", "marginRight" );
-					} else {
-						ret = elem.style.marginRight;
-					}
-				});
-				return ret;
-			}
-		};
-	}
-});
-
 if ( document.defaultView && document.defaultView.getComputedStyle ) {
 	getComputedStyle = function( elem, name ) {
-		var ret, defaultView, computedStyle;
+		var ret, defaultView, computedStyle, width,
+			style = elem.style;
 
 		name = name.replace( rupper, "-$1" ).toLowerCase();
 
-		if ( !(defaultView = elem.ownerDocument.defaultView) ) {
-			return undefined;
-		}
+		if ( (defaultView = elem.ownerDocument.defaultView) &&
+				(computedStyle = defaultView.getComputedStyle( elem, null )) ) {
 
-		if ( (computedStyle = defaultView.getComputedStyle( elem, null )) ) {
 			ret = computedStyle.getPropertyValue( name );
 			if ( ret === "" && !jQuery.contains( elem.ownerDocument.documentElement, elem ) ) {
 				ret = jQuery.style( elem, name );
 			}
 		}
 
+		// A tribute to the "awesome hack by Dean Edwards"
+		// WebKit uses "computed value (percentage if specified)" instead of "used value" for margins
+		// which is against the CSSOM draft spec: http://dev.w3.org/csswg/cssom/#resolved-values
+		if ( !jQuery.support.pixelMargin && computedStyle && rmargin.test( name ) && rnumnonpx.test( ret ) ) {
+			width = style.width;
+			style.width = ret;
+			ret = computedStyle.width;
+			style.width = width;
+		}
+
 		return ret;
 	};
 }
 
 if ( document.documentElement.currentStyle ) {
 	currentStyle = function( elem, name ) {
-		var left,
+		var left, rsLeft, uncomputed,
 			ret = elem.currentStyle && elem.currentStyle[ name ],
-			rsLeft = elem.runtimeStyle && elem.runtimeStyle[ name ],
 			style = elem.style;
 
+		// Avoid setting ret to empty string here
+		// so we don't default to auto
+		if ( ret == null && style && (uncomputed = style[ name ]) ) {
+			ret = uncomputed;
+		}
+
 		// From the awesome hack by Dean Edwards
 		// http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291
 
 		// If we're not dealing with a regular pixel number
 		// but a number that has a weird ending, we need to convert it to pixels
-		if ( !rnumpx.test( ret ) && rnum.test( ret ) ) {
+		if ( rnumnonpx.test( ret ) ) {
+
 			// Remember the original values
 			left = style.left;
+			rsLeft = elem.runtimeStyle && elem.runtimeStyle.left;
 
 			// Put in the new values to get a computed value out
 			if ( rsLeft ) {
 				elem.runtimeStyle.left = elem.currentStyle.left;
 			}
-			style.left = name === "fontSize" ? "1em" : (ret || 0);
+			style.left = name === "fontSize" ? "1em" : ret;
 			ret = style.pixelLeft + "px";
 
 			// Revert the changed values
@@ -6557,59 +6790,149 @@ if ( document.documentElement.currentStyle ) {
 
 curCSS = getComputedStyle || currentStyle;
 
-function getWH( elem, name, extra ) {
+function getWidthOrHeight( elem, name, extra ) {
 
 	// Start with offset property
 	var val = name === "width" ? elem.offsetWidth : elem.offsetHeight,
-		which = name === "width" ? cssWidth : cssHeight;
+		i = name === "width" ? 1 : 0,
+		len = 4;
 
 	if ( val > 0 ) {
 		if ( extra !== "border" ) {
-			jQuery.each( which, function() {
+			for ( ; i < len; i += 2 ) {
 				if ( !extra ) {
-					val -= parseFloat( jQuery.css( elem, "padding" + this ) ) || 0;
+					val -= parseFloat( jQuery.css( elem, "padding" + cssExpand[ i ] ) ) || 0;
 				}
 				if ( extra === "margin" ) {
-					val += parseFloat( jQuery.css( elem, extra + this ) ) || 0;
+					val += parseFloat( jQuery.css( elem, extra + cssExpand[ i ] ) ) || 0;
 				} else {
-					val -= parseFloat( jQuery.css( elem, "border" + this + "Width" ) ) || 0;
+					val -= parseFloat( jQuery.css( elem, "border" + cssExpand[ i ] + "Width" ) ) || 0;
 				}
-			});
+			}
 		}
 
 		return val + "px";
 	}
 
 	// Fall back to computed then uncomputed css if necessary
-	val = curCSS( elem, name, name );
+	val = curCSS( elem, name );
 	if ( val < 0 || val == null ) {
-		val = elem.style[ name ] || 0;
+		val = elem.style[ name ];
 	}
+
+	// Computed unit is not pixels. Stop here and return.
+	if ( rnumnonpx.test(val) ) {
+		return val;
+	}
+
 	// Normalize "", auto, and prepare for extra
 	val = parseFloat( val ) || 0;
 
 	// Add padding, border, margin
 	if ( extra ) {
-		jQuery.each( which, function() {
-			val += parseFloat( jQuery.css( elem, "padding" + this ) ) || 0;
+		for ( ; i < len; i += 2 ) {
+			val += parseFloat( jQuery.css( elem, "padding" + cssExpand[ i ] ) ) || 0;
 			if ( extra !== "padding" ) {
-				val += parseFloat( jQuery.css( elem, "border" + this + "Width" ) ) || 0;
+				val += parseFloat( jQuery.css( elem, "border" + cssExpand[ i ] + "Width" ) ) || 0;
 			}
 			if ( extra === "margin" ) {
-				val += parseFloat( jQuery.css( elem, extra + this ) ) || 0;
+				val += parseFloat( jQuery.css( elem, extra + cssExpand[ i ]) ) || 0;
 			}
-		});
+		}
 	}
 
 	return val + "px";
 }
 
+jQuery.each([ "height", "width" ], function( i, name ) {
+	jQuery.cssHooks[ name ] = {
+		get: function( elem, computed, extra ) {
+			if ( computed ) {
+				if ( elem.offsetWidth !== 0 ) {
+					return getWidthOrHeight( elem, name, extra );
+				} else {
+					return jQuery.swap( elem, cssShow, function() {
+						return getWidthOrHeight( elem, name, extra );
+					});
+				}
+			}
+		},
+
+		set: function( elem, value ) {
+			return rnum.test( value ) ?
+				value + "px" :
+				value;
+		}
+	};
+});
+
+if ( !jQuery.support.opacity ) {
+	jQuery.cssHooks.opacity = {
+		get: function( elem, computed ) {
+			// IE uses filters for opacity
+			return ropacity.test( (computed && elem.currentStyle ? elem.currentStyle.filter : elem.style.filter) || "" ) ?
+				( parseFloat( RegExp.$1 ) / 100 ) + "" :
+				computed ? "1" : "";
+		},
+
+		set: function( elem, value ) {
+			var style = elem.style,
+				currentStyle = elem.currentStyle,
+				opacity = jQuery.isNumeric( value ) ? "alpha(opacity=" + value * 100 + ")" : "",
+				filter = currentStyle && currentStyle.filter || style.filter || "";
+
+			// IE has trouble with opacity if it does not have layout
+			// Force it by setting the zoom level
+			style.zoom = 1;
+
+			// if setting opacity to 1, and no other filters exist - attempt to remove filter attribute #6652
+			if ( value >= 1 && jQuery.trim( filter.replace( ralpha, "" ) ) === "" ) {
+
+				// Setting style.filter to null, "" & " " still leave "filter:" in the cssText
+				// if "filter:" is present at all, clearType is disabled, we want to avoid this
+				// style.removeAttribute is IE Only, but so apparently is this code path...
+				style.removeAttribute( "filter" );
+
+				// if there there is no filter style applied in a css rule, we are done
+				if ( currentStyle && !currentStyle.filter ) {
+					return;
+				}
+			}
+
+			// otherwise, set new filter values
+			style.filter = ralpha.test( filter ) ?
+				filter.replace( ralpha, opacity ) :
+				filter + " " + opacity;
+		}
+	};
+}
+
+jQuery(function() {
+	// This hook cannot be added until DOM ready because the support test
+	// for it is not run until after DOM ready
+	if ( !jQuery.support.reliableMarginRight ) {
+		jQuery.cssHooks.marginRight = {
+			get: function( elem, computed ) {
+				// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right
+				// Work around by temporarily setting element display to inline-block
+				return jQuery.swap( elem, { "display": "inline-block" }, function() {
+					if ( computed ) {
+						return curCSS( elem, "margin-right" );
+					} else {
+						return elem.style.marginRight;
+					}
+				});
+			}
+		};
+	}
+});
+
 if ( jQuery.expr && jQuery.expr.filters ) {
 	jQuery.expr.filters.hidden = function( elem ) {
 		var width = elem.offsetWidth,
 			height = elem.offsetHeight;
 
-		return (width === 0 && height === 0) || (!jQuery.support.reliableHiddenOffsets && (elem.style.display || jQuery.css( elem, "display" )) === "none");
+		return ( width === 0 && height === 0 ) || (!jQuery.support.reliableHiddenOffsets && ((elem.style && elem.style.display) || jQuery.css( elem, "display" )) === "none");
 	};
 
 	jQuery.expr.filters.visible = function( elem ) {
@@ -6617,6 +6940,31 @@ if ( jQuery.expr && jQuery.expr.filters ) {
 	};
 }
 
+// These hooks are used by animate to expand properties
+jQuery.each({
+	margin: "",
+	padding: "",
+	border: "Width"
+}, function( prefix, suffix ) {
+
+	jQuery.cssHooks[ prefix + suffix ] = {
+		expand: function( value ) {
+			var i,
+
+				// assumes a single number if not a string
+				parts = typeof value === "string" ? value.split(" ") : [ value ],
+				expanded = {};
+
+			for ( i = 0; i < 4; i++ ) {
+				expanded[ prefix + cssExpand[ i ] + suffix ] =
+					parts[ i ] || parts[ i - 2 ] || parts[ 0 ];
+			}
+
+			return expanded;
+		}
+	};
+});
+
 
 
 
@@ -6625,9 +6973,9 @@ var r20 = /%20/g,
 	rCRLF = /\r?\n/g,
 	rhash = /#.*$/,
 	rheaders = /^(.*?):[ \t]*([^\r\n]*)\r?$/mg, // IE leaves an \r character at EOL
-	rinput = /^(?:color|date|datetime|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,
+	rinput = /^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,
 	// #7653, #8125, #8152: local protocol detection
-	rlocalProtocol = /^(?:about|app|app\-storage|.+\-extension|file|widget):$/,
+	rlocalProtocol = /^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/,
 	rnoContent = /^(?:GET|HEAD)$/,
 	rprotocol = /^\/\//,
 	rquery = /\?/,
@@ -6662,7 +7010,10 @@ var r20 = /%20/g,
 	ajaxLocation,
 
 	// Document location segments
-	ajaxLocParts;
+	ajaxLocParts,
+
+	// Avoid comment-prolog char sequence (#10098); must appease lint and evade compression
+	allTypes = ["*/"] + ["*"];
 
 // #8138, IE may throw an exception when accessing
 // a field from window.location if document.domain has been set
@@ -6699,7 +7050,7 @@ function addToPrefiltersOrTransports( structure ) {
 				placeBefore;
 
 			// For each dataType in the dataTypeExpression
-			for(; i < length; i++ ) {
+			for ( ; i < length; i++ ) {
 				dataType = dataTypes[ i ];
 				// We control if we're asked to add before
 				// any existing element
@@ -6730,7 +7081,7 @@ function inspectPrefiltersOrTransports( structure, options, originalOptions, jqX
 		executeOnly = ( structure === prefilters ),
 		selection;
 
-	for(; i < length && ( executeOnly || !selection ); i++ ) {
+	for ( ; i < length && ( executeOnly || !selection ); i++ ) {
 		selection = list[ i ]( options, originalOptions, jqXHR );
 		// If we got redirected to another dataType
 		// we try there if executing only and not done already
@@ -6755,6 +7106,22 @@ function inspectPrefiltersOrTransports( structure, options, originalOptions, jqX
 	return selection;
 }
 
+// A special extend for ajax options
+// that takes "flat" options (not to be deep extended)
+// Fixes #9887
+function ajaxExtend( target, src ) {
+	var key, deep,
+		flatOptions = jQuery.ajaxSettings.flatOptions || {};
+	for ( key in src ) {
+		if ( src[ key ] !== undefined ) {
+			( flatOptions[ key ] ? target : ( deep || ( deep = {} ) ) )[ key ] = src[ key ];
+		}
+	}
+	if ( deep ) {
+		jQuery.extend( true, target, deep );
+	}
+}
+
 jQuery.fn.extend({
 	load: function( url, params, callback ) {
 		if ( typeof url !== "string" && _load ) {
@@ -6862,7 +7229,7 @@ jQuery.fn.extend({
 // Attach a bunch of functions for handling common AJAX events
 jQuery.each( "ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split( " " ), function( i, o ){
 	jQuery.fn[ o ] = function( f ){
-		return this.bind( o, f );
+		return this.on( o, f );
 	};
 });
 
@@ -6898,23 +7265,16 @@ jQuery.extend({
 	// Creates a full fledged settings object into target
 	// with both ajaxSettings and settings fields.
 	// If target is omitted, writes into ajaxSettings.
-	ajaxSetup: function ( target, settings ) {
-		if ( !settings ) {
-			// Only one parameter, we extend ajaxSettings
-			settings = target;
-			target = jQuery.extend( true, jQuery.ajaxSettings, settings );
+	ajaxSetup: function( target, settings ) {
+		if ( settings ) {
+			// Building a settings object
+			ajaxExtend( target, jQuery.ajaxSettings );
 		} else {
-			// target was provided, we extend into it
-			jQuery.extend( true, target, jQuery.ajaxSettings, settings );
-		}
-		// Flatten fields we don't want deep extended
-		for( var field in { context: 1, url: 1 } ) {
-			if ( field in settings ) {
-				target[ field ] = settings[ field ];
-			} else if( field in jQuery.ajaxSettings ) {
-				target[ field ] = jQuery.ajaxSettings[ field ];
-			}
+			// Extending ajaxSettings
+			settings = target;
+			target = jQuery.ajaxSettings;
 		}
+		ajaxExtend( target, settings );
 		return target;
 	},
 
@@ -6923,7 +7283,7 @@ jQuery.extend({
 		isLocal: rlocalProtocol.test( ajaxLocParts[ 1 ] ),
 		global: true,
 		type: "GET",
-		contentType: "application/x-www-form-urlencoded",
+		contentType: "application/x-www-form-urlencoded; charset=UTF-8",
 		processData: true,
 		async: true,
 		/*
@@ -6942,7 +7302,7 @@ jQuery.extend({
 			html: "text/html",
 			text: "text/plain",
 			json: "application/json, text/javascript",
-			"*": "*/*"
+			"*": allTypes
 		},
 
 		contents: {
@@ -6972,6 +7332,15 @@ jQuery.extend({
 
 			// Parse text as xml
 			"text xml": jQuery.parseXML
+		},
+
+		// For options that shouldn't be deep extended:
+		// you can add your own custom options here if
+		// and when you create one that shouldn't be
+		// deep extended (see ajaxExtend)
+		flatOptions: {
+			context: true,
+			url: true
 		}
 	},
 
@@ -7002,7 +7371,7 @@ jQuery.extend({
 						jQuery( callbackContext ) : jQuery.event,
 			// Deferreds
 			deferred = jQuery.Deferred(),
-			completeDeferred = jQuery._Deferred(),
+			completeDeferred = jQuery.Callbacks( "once memory" ),
 			// Status-dependent callbacks
 			statusCode = s.statusCode || {},
 			// ifModified key
@@ -7082,7 +7451,7 @@ jQuery.extend({
 		// Callback for when everything is done
 		// It is defined here because jslint complains if it is declared
 		// at the end of the function (which would be more logical and readable)
-		function done( status, statusText, responses, headers ) {
+		function done( status, nativeStatusText, responses, headers ) {
 
 			// Called once
 			if ( state === 2 ) {
@@ -7105,11 +7474,12 @@ jQuery.extend({
 			responseHeadersString = headers || "";
 
 			// Set readyState
-			jqXHR.readyState = status ? 4 : 0;
+			jqXHR.readyState = status > 0 ? 4 : 0;
 
 			var isSuccess,
 				success,
 				error,
+				statusText = nativeStatusText,
 				response = responses ? ajaxHandleResponses( s, jqXHR, responses ) : undefined,
 				lastModified,
 				etag;
@@ -7151,7 +7521,7 @@ jQuery.extend({
 				// We extract error from statusText
 				// then normalize statusText and status for non-aborts
 				error = statusText;
-				if( !statusText || status ) {
+				if ( !statusText || status ) {
 					statusText = "error";
 					if ( status < 0 ) {
 						status = 0;
@@ -7161,7 +7531,7 @@ jQuery.extend({
 
 			// Set data for the fake xhr object
 			jqXHR.status = status;
-			jqXHR.statusText = statusText;
+			jqXHR.statusText = "" + ( nativeStatusText || statusText );
 
 			// Success/Error
 			if ( isSuccess ) {
@@ -7180,10 +7550,10 @@ jQuery.extend({
 			}
 
 			// Complete
-			completeDeferred.resolveWith( callbackContext, [ jqXHR, statusText ] );
+			completeDeferred.fireWith( callbackContext, [ jqXHR, statusText ] );
 
 			if ( fireGlobals ) {
-				globalEventContext.trigger( "ajaxComplete", [ jqXHR, s] );
+				globalEventContext.trigger( "ajaxComplete", [ jqXHR, s ] );
 				// Handle the global AJAX counter
 				if ( !( --jQuery.active ) ) {
 					jQuery.event.trigger( "ajaxStop" );
@@ -7195,14 +7565,14 @@ jQuery.extend({
 		deferred.promise( jqXHR );
 		jqXHR.success = jqXHR.done;
 		jqXHR.error = jqXHR.fail;
-		jqXHR.complete = completeDeferred.done;
+		jqXHR.complete = completeDeferred.add;
 
 		// Status-dependent callbacks
 		jqXHR.statusCode = function( map ) {
 			if ( map ) {
 				var tmp;
 				if ( state < 2 ) {
-					for( tmp in map ) {
+					for ( tmp in map ) {
 						statusCode[ tmp ] = [ statusCode[tmp], map[tmp] ];
 					}
 				} else {
@@ -7239,7 +7609,7 @@ jQuery.extend({
 		// Apply prefilters
 		inspectPrefiltersOrTransports( prefilters, s, options, jqXHR );
 
-		// If request was aborted inside a prefiler, stop there
+		// If request was aborted inside a prefilter, stop there
 		if ( state === 2 ) {
 			return false;
 		}
@@ -7264,6 +7634,8 @@ jQuery.extend({
 			// If data is available, append data to url
 			if ( s.data ) {
 				s.url += ( rquery.test( s.url ) ? "&" : "?" ) + s.data;
+				// #9682: remove data so that it's not used in an eventual retry
+				delete s.data;
 			}
 
 			// Get ifModifiedKey before adding the anti-cache parameter
@@ -7277,7 +7649,7 @@ jQuery.extend({
 					ret = s.url.replace( rts, "$1_=" + ts );
 
 				// if nothing was replaced, add timestamp to the end
-				s.url = ret + ( (ret === s.url ) ? ( rquery.test( s.url ) ? "&" : "?" ) + "_=" + ts : "" );
+				s.url = ret + ( ( ret === s.url ) ? ( rquery.test( s.url ) ? "&" : "?" ) + "_=" + ts : "" );
 			}
 		}
 
@@ -7301,7 +7673,7 @@ jQuery.extend({
 		jqXHR.setRequestHeader(
 			"Accept",
 			s.dataTypes[ 0 ] && s.accepts[ s.dataTypes[0] ] ?
-				s.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== "*" ? ", */*; q=0.01" : "" ) :
+				s.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== "*" ? ", " + allTypes + "; q=0.01" : "" ) :
 				s.accepts[ "*" ]
 		);
 
@@ -7347,11 +7719,11 @@ jQuery.extend({
 				transport.send( requestHeaders, done );
 			} catch (e) {
 				// Propagate exception as error if not done
-				if ( status < 2 ) {
+				if ( state < 2 ) {
 					done( -1, e );
 				// Simply rethrow otherwise
 				} else {
-					jQuery.error( e );
+					throw e;
 				}
 			}
 		}
@@ -7410,11 +7782,11 @@ function buildParams( prefix, obj, traditional, add ) {
 				// a server error. Possible fixes are to modify rack's
 				// deserialization algorithm or to provide an option or flag
 				// to force array serialization to be shallow.
-				buildParams( prefix + "[" + ( typeof v === "object" || jQuery.isArray(v) ? i : "" ) + "]", v, traditional, add );
+				buildParams( prefix + "[" + ( typeof v === "object" ? i : "" ) + "]", v, traditional, add );
 			}
 		});
 
-	} else if ( !traditional && obj != null && typeof obj === "object" ) {
+	} else if ( !traditional && jQuery.type( obj ) === "object" ) {
 		// Serialize object item.
 		for ( var name in obj ) {
 			buildParams( prefix + "[" + name + "]", obj[ name ], traditional, add );
@@ -7455,7 +7827,7 @@ function ajaxHandleResponses( s, jqXHR, responses ) {
 		firstDataType;
 
 	// Fill responseXXX fields
-	for( type in responseFields ) {
+	for ( type in responseFields ) {
 		if ( type in responses ) {
 			jqXHR[ responseFields[type] ] = responses[ type ];
 		}
@@ -7534,13 +7906,13 @@ function ajaxConvert( s, response ) {
 		conv2;
 
 	// For each dataType in the chain
-	for( i = 1; i < length; i++ ) {
+	for ( i = 1; i < length; i++ ) {
 
 		// Create converters map
 		// with lowercased keys
 		if ( i === 1 ) {
-			for( key in s.converters ) {
-				if( typeof key === "string" ) {
+			for ( key in s.converters ) {
+				if ( typeof key === "string" ) {
 					converters[ key.toLowerCase() ] = s.converters[ key ];
 				}
 			}
@@ -7551,7 +7923,7 @@ function ajaxConvert( s, response ) {
 		current = dataTypes[ i ];
 
 		// If current is auto dataType, update it to prev
-		if( current === "*" ) {
+		if ( current === "*" ) {
 			current = prev;
 		// If no auto and dataTypes are actually different
 		} else if ( prev !== "*" && prev !== current ) {
@@ -7563,7 +7935,7 @@ function ajaxConvert( s, response ) {
 			// If there is no direct converter, search transitively
 			if ( !conv ) {
 				conv2 = undefined;
-				for( conv1 in converters ) {
+				for ( conv1 in converters ) {
 					tmp = conv1.split( " " );
 					if ( tmp[ 0 ] === prev || tmp[ 0 ] === "*" ) {
 						conv2 = converters[ tmp[1] + " " + current ];
@@ -7610,8 +7982,7 @@ jQuery.ajaxSetup({
 // Detect, normalize options and install callbacks for jsonp requests
 jQuery.ajaxPrefilter( "json jsonp", function( s, originalSettings, jqXHR ) {
 
-	var inspectData = s.contentType === "application/x-www-form-urlencoded" &&
-		( typeof s.data === "string" );
+	var inspectData = ( typeof s.data === "string" ) && /^application\/x\-www\-form\-urlencoded/.test( s.contentType );
 
 	if ( s.dataTypes[ 0 ] === "jsonp" ||
 		s.jsonp !== false && ( jsre.test( s.url ) ||
@@ -7912,7 +8283,13 @@ if ( jQuery.support.ajax ) {
 									if ( xml && xml.documentElement /* #4958 */ ) {
 										responses.xml = xml;
 									}
-									responses.text = xhr.responseText;
+
+									// When requesting binary data, IE6-9 will throw an exception
+									// on any attempt to access responseText (#11426)
+									try {
+										responses.text = xhr.responseText;
+									} catch( _ ) {
+									}
 
 									// Firefox throws an exception when accessing
 									// statusText for faulty cross-domain requests
@@ -7995,21 +8372,18 @@ var elemdisplay = {},
 		// opacity animations
 		[ "opacity" ]
 	],
-	fxNow,
-	requestAnimationFrame = window.webkitRequestAnimationFrame ||
-		window.mozRequestAnimationFrame ||
-		window.oRequestAnimationFrame;
+	fxNow;
 
 jQuery.fn.extend({
 	show: function( speed, easing, callback ) {
 		var elem, display;
 
 		if ( speed || speed === 0 ) {
-			return this.animate( genFx("show", 3), speed, easing, callback);
+			return this.animate( genFx("show", 3), speed, easing, callback );
 
 		} else {
 			for ( var i = 0, j = this.length; i < j; i++ ) {
-				elem = this[i];
+				elem = this[ i ];
 
 				if ( elem.style ) {
 					display = elem.style.display;
@@ -8023,8 +8397,9 @@ jQuery.fn.extend({
 					// Set elements which have been overridden with display: none
 					// in a stylesheet to whatever the default browser style is
 					// for such an element
-					if ( display === "" && jQuery.css( elem, "display" ) === "none" ) {
-						jQuery._data(elem, "olddisplay", defaultDisplay(elem.nodeName));
+					if ( (display === "" && jQuery.css(elem, "display") === "none") ||
+						!jQuery.contains( elem.ownerDocument.documentElement, elem ) ) {
+						jQuery._data( elem, "olddisplay", defaultDisplay(elem.nodeName) );
 					}
 				}
 			}
@@ -8032,13 +8407,13 @@ jQuery.fn.extend({
 			// Set the display of most of the elements in a second loop
 			// to avoid the constant reflow
 			for ( i = 0; i < j; i++ ) {
-				elem = this[i];
+				elem = this[ i ];
 
 				if ( elem.style ) {
 					display = elem.style.display;
 
 					if ( display === "" || display === "none" ) {
-						elem.style.display = jQuery._data(elem, "olddisplay") || "";
+						elem.style.display = jQuery._data( elem, "olddisplay" ) || "";
 					}
 				}
 			}
@@ -8052,12 +8427,17 @@ jQuery.fn.extend({
 			return this.animate( genFx("hide", 3), speed, easing, callback);
 
 		} else {
-			for ( var i = 0, j = this.length; i < j; i++ ) {
-				if ( this[i].style ) {
-					var display = jQuery.css( this[i], "display" );
+			var elem, display,
+				i = 0,
+				j = this.length;
 
-					if ( display !== "none" && !jQuery._data( this[i], "olddisplay" ) ) {
-						jQuery._data( this[i], "olddisplay", display );
+			for ( ; i < j; i++ ) {
+				elem = this[i];
+				if ( elem.style ) {
+					display = jQuery.css( elem, "display" );
+
+					if ( display !== "none" && !jQuery._data( elem, "olddisplay" ) ) {
+						jQuery._data( elem, "olddisplay", display );
 					}
 				}
 			}
@@ -8102,7 +8482,7 @@ jQuery.fn.extend({
 	},
 
 	animate: function( prop, speed, easing, callback ) {
-		var optall = jQuery.speed(speed, easing, callback);
+		var optall = jQuery.speed( speed, easing, callback );
 
 		if ( jQuery.isEmptyObject( prop ) ) {
 			return this.each( optall.complete, [ false ] );
@@ -8111,7 +8491,7 @@ jQuery.fn.extend({
 		// Do not change referenced properties as per-property easing will be lost
 		prop = jQuery.extend( {}, prop );
 
-		return this[ optall.queue === false ? "each" : "queue" ](function() {
+		function doAnimation() {
 			// XXX 'this' does not always have a nodeName when running the
 			// test suite
 
@@ -8122,24 +8502,37 @@ jQuery.fn.extend({
 			var opt = jQuery.extend( {}, optall ),
 				isElement = this.nodeType === 1,
 				hidden = isElement && jQuery(this).is(":hidden"),
-				name, val, p,
-				display, e,
-				parts, start, end, unit;
+				name, val, p, e, hooks, replace,
+				parts, start, end, unit,
+				method;
 
 			// will store per property easing and be used to determine when an animation is complete
 			opt.animatedProperties = {};
 
+			// first pass over propertys to expand / normalize
 			for ( p in prop ) {
-
-				// property name normalization
 				name = jQuery.camelCase( p );
 				if ( p !== name ) {
 					prop[ name ] = prop[ p ];
 					delete prop[ p ];
 				}
 
-				val = prop[ name ];
+				if ( ( hooks = jQuery.cssHooks[ name ] ) && "expand" in hooks ) {
+					replace = hooks.expand( prop[ name ] );
+					delete prop[ name ];
 
+					// not quite $.extend, this wont overwrite keys already present.
+					// also - reusing 'p' from above because we have the correct "name"
+					for ( p in replace ) {
+						if ( ! ( p in prop ) ) {
+							prop[ p ] = replace[ p ];
+						}
+					}
+				}
+			}
+
+			for ( name in prop ) {
+				val = prop[ name ];
 				// easing resolution: per property > opt.specialEasing > opt.easing > 'swing' (default)
 				if ( jQuery.isArray( val ) ) {
 					opt.animatedProperties[ name ] = val[ 1 ];
@@ -8160,25 +8553,17 @@ jQuery.fn.extend({
 					opt.overflow = [ this.style.overflow, this.style.overflowX, this.style.overflowY ];
 
 					// Set display property to inline-block for height/width
-					// animations on inline elements that are having width/height
-					// animated
+					// animations on inline elements that are having width/height animated
 					if ( jQuery.css( this, "display" ) === "inline" &&
 							jQuery.css( this, "float" ) === "none" ) {
-						if ( !jQuery.support.inlineBlockNeedsLayout ) {
+
+						// inline-level elements accept inline-block;
+						// block-level elements need to be inline with layout
+						if ( !jQuery.support.inlineBlockNeedsLayout || defaultDisplay( this.nodeName ) === "inline" ) {
 							this.style.display = "inline-block";
 
 						} else {
-							display = defaultDisplay( this.nodeName );
-
-							// inline-level elements accept inline-block;
-							// block-level elements need to be inline with layout
-							if ( display === "inline" ) {
-								this.style.display = "inline-block";
-
-							} else {
-								this.style.display = "inline";
-								this.style.zoom = 1;
-							}
+							this.style.zoom = 1;
 						}
 					}
 				}
@@ -8192,8 +8577,17 @@ jQuery.fn.extend({
 				e = new jQuery.fx( this, opt, p );
 				val = prop[ p ];
 
-				if ( rfxtypes.test(val) ) {
-					e[ val === "toggle" ? hidden ? "show" : "hide" : val ]();
+				if ( rfxtypes.test( val ) ) {
+
+					// Tracks whether to show or hide based on private
+					// data attached to the element
+					method = jQuery._data( this, "toggle" + p ) || ( val === "toggle" ? hidden ? "show" : "hide" : 0 );
+					if ( method ) {
+						jQuery._data( this, "toggle" + p, method === "show" ? "hide" : "show" );
+						e[ method ]();
+					} else {
+						e[ val ]();
+					}
 
 				} else {
 					parts = rfxnum.exec( val );
@@ -8206,7 +8600,7 @@ jQuery.fn.extend({
 						// We need to compute starting value
 						if ( unit !== "px" ) {
 							jQuery.style( this, p, (end || 1) + unit);
-							start = ((end || 1) / e.cur()) * start;
+							start = ( (end || 1) / e.cur() ) * start;
 							jQuery.style( this, p, start + unit);
 						}
 
@@ -8225,39 +8619,71 @@ jQuery.fn.extend({
 
 			// For JS strict compliance
 			return true;
-		});
-	},
-
-	stop: function( clearQueue, gotoEnd ) {
-		if ( clearQueue ) {
-			this.queue([]);
 		}
 
-		this.each(function() {
-			var timers = jQuery.timers,
-				i = timers.length;
+		return optall.queue === false ?
+			this.each( doAnimation ) :
+			this.queue( optall.queue, doAnimation );
+	},
+
+	stop: function( type, clearQueue, gotoEnd ) {
+		if ( typeof type !== "string" ) {
+			gotoEnd = clearQueue;
+			clearQueue = type;
+			type = undefined;
+		}
+		if ( clearQueue && type !== false ) {
+			this.queue( type || "fx", [] );
+		}
+
+		return this.each(function() {
+			var index,
+				hadTimers = false,
+				timers = jQuery.timers,
+				data = jQuery._data( this );
+
 			// clear marker counters if we know they won't be
 			if ( !gotoEnd ) {
 				jQuery._unmark( true, this );
 			}
-			while ( i-- ) {
-				if ( timers[i].elem === this ) {
-					if (gotoEnd) {
-						// force the next step to be the last
-						timers[i](true);
-					}
 
-					timers.splice(i, 1);
+			function stopQueue( elem, data, index ) {
+				var hooks = data[ index ];
+				jQuery.removeData( elem, index, true );
+				hooks.stop( gotoEnd );
+			}
+
+			if ( type == null ) {
+				for ( index in data ) {
+					if ( data[ index ] && data[ index ].stop && index.indexOf(".run") === index.length - 4 ) {
+						stopQueue( this, data, index );
+					}
+				}
+			} else if ( data[ index = type + ".run" ] && data[ index ].stop ){
+				stopQueue( this, data, index );
+			}
+
+			for ( index = timers.length; index--; ) {
+				if ( timers[ index ].elem === this && (type == null || timers[ index ].queue === type) ) {
+					if ( gotoEnd ) {
+
+						// force the next step to be the last
+						timers[ index ]( true );
+					} else {
+						timers[ index ].saveState();
+					}
+					hadTimers = true;
+					timers.splice( index, 1 );
 				}
 			}
+
+			// start the next in the queue if the last step wasn't forced
+			// timers currently will call their complete callbacks, which will dequeue
+			// but only if they were gotoEnd
+			if ( !( gotoEnd && hadTimers ) ) {
+				jQuery.dequeue( this, type );
+			}
 		});
-
-		// start the next in the queue if the last step wasn't forced
-		if ( !gotoEnd ) {
-			this.dequeue();
-		}
-
-		return this;
 	}
 
 });
@@ -8276,7 +8702,7 @@ function clearFxNow() {
 function genFx( type, num ) {
 	var obj = {};
 
-	jQuery.each( fxAttrs.concat.apply([], fxAttrs.slice(0,num)), function() {
+	jQuery.each( fxAttrs.concat.apply([], fxAttrs.slice( 0, num )), function() {
 		obj[ this ] = type;
 	});
 
@@ -8285,9 +8711,9 @@ function genFx( type, num ) {
 
 // Generate shortcuts for custom animations
 jQuery.each({
-	slideDown: genFx("show", 1),
-	slideUp: genFx("hide", 1),
-	slideToggle: genFx("toggle", 1),
+	slideDown: genFx( "show", 1 ),
+	slideUp: genFx( "hide", 1 ),
+	slideToggle: genFx( "toggle", 1 ),
 	fadeIn: { opacity: "show" },
 	fadeOut: { opacity: "hide" },
 	fadeToggle: { opacity: "toggle" }
@@ -8299,25 +8725,31 @@ jQuery.each({
 
 jQuery.extend({
 	speed: function( speed, easing, fn ) {
-		var opt = speed && typeof speed === "object" ? jQuery.extend({}, speed) : {
+		var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : {
 			complete: fn || !fn && easing ||
 				jQuery.isFunction( speed ) && speed,
 			duration: speed,
-			easing: fn && easing || easing && !jQuery.isFunction(easing) && easing
+			easing: fn && easing || easing && !jQuery.isFunction( easing ) && easing
 		};
 
 		opt.duration = jQuery.fx.off ? 0 : typeof opt.duration === "number" ? opt.duration :
-			opt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[opt.duration] : jQuery.fx.speeds._default;
+			opt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[ opt.duration ] : jQuery.fx.speeds._default;
+
+		// normalize opt.queue - true/undefined/null -> "fx"
+		if ( opt.queue == null || opt.queue === true ) {
+			opt.queue = "fx";
+		}
 
 		// Queueing
 		opt.old = opt.complete;
+
 		opt.complete = function( noUnmark ) {
 			if ( jQuery.isFunction( opt.old ) ) {
 				opt.old.call( this );
 			}
 
-			if ( opt.queue !== false ) {
-				jQuery.dequeue( this );
+			if ( opt.queue ) {
+				jQuery.dequeue( this, opt.queue );
 			} else if ( noUnmark !== false ) {
 				jQuery._unmark( this );
 			}
@@ -8327,11 +8759,11 @@ jQuery.extend({
 	},
 
 	easing: {
-		linear: function( p, n, firstNum, diff ) {
-			return firstNum + diff * p;
+		linear: function( p ) {
+			return p;
 		},
-		swing: function( p, n, firstNum, diff ) {
-			return ((-Math.cos(p*Math.PI)/2) + 0.5) * diff + firstNum;
+		swing: function( p ) {
+			return ( -Math.cos( p*Math.PI ) / 2 ) + 0.5;
 		}
 	},
 
@@ -8354,12 +8786,12 @@ jQuery.fx.prototype = {
 			this.options.step.call( this.elem, this.now, this );
 		}
 
-		(jQuery.fx.step[this.prop] || jQuery.fx.step._default)( this );
+		( jQuery.fx.step[ this.prop ] || jQuery.fx.step._default )( this );
 	},
 
 	// Get the current size
 	cur: function() {
-		if ( this.elem[this.prop] != null && (!this.elem.style || this.elem.style[this.prop] == null) ) {
+		if ( this.elem[ this.prop ] != null && (!this.elem.style || this.elem.style[ this.prop ] == null) ) {
 			return this.elem[ this.prop ];
 		}
 
@@ -8374,50 +8806,51 @@ jQuery.fx.prototype = {
 	// Start an animation from one number to another
 	custom: function( from, to, unit ) {
 		var self = this,
-			fx = jQuery.fx,
-			raf;
+			fx = jQuery.fx;
 
 		this.startTime = fxNow || createFxNow();
-		this.start = from;
 		this.end = to;
-		this.unit = unit || this.unit || ( jQuery.cssNumber[ this.prop ] ? "" : "px" );
-		this.now = this.start;
+		this.now = this.start = from;
 		this.pos = this.state = 0;
+		this.unit = unit || this.unit || ( jQuery.cssNumber[ this.prop ] ? "" : "px" );
 
 		function t( gotoEnd ) {
-			return self.step(gotoEnd);
+			return self.step( gotoEnd );
 		}
 
+		t.queue = this.options.queue;
 		t.elem = this.elem;
+		t.saveState = function() {
+			if ( jQuery._data( self.elem, "fxshow" + self.prop ) === undefined ) {
+				if ( self.options.hide ) {
+					jQuery._data( self.elem, "fxshow" + self.prop, self.start );
+				} else if ( self.options.show ) {
+					jQuery._data( self.elem, "fxshow" + self.prop, self.end );
+				}
+			}
+		};
 
 		if ( t() && jQuery.timers.push(t) && !timerId ) {
-			// Use requestAnimationFrame instead of setInterval if available
-			if ( requestAnimationFrame ) {
-				timerId = true;
-				raf = function() {
-					// When timerId gets set to null at any point, this stops
-					if ( timerId ) {
-						requestAnimationFrame( raf );
-						fx.tick();
-					}
-				};
-				requestAnimationFrame( raf );
-			} else {
-				timerId = setInterval( fx.tick, fx.interval );
-			}
+			timerId = setInterval( fx.tick, fx.interval );
 		}
 	},
 
 	// Simple 'show' function
 	show: function() {
+		var dataShow = jQuery._data( this.elem, "fxshow" + this.prop );
+
 		// Remember where we started, so that we can go back to it later
-		this.options.orig[this.prop] = jQuery.style( this.elem, this.prop );
+		this.options.orig[ this.prop ] = dataShow || jQuery.style( this.elem, this.prop );
 		this.options.show = true;
 
 		// Begin the animation
-		// Make sure that we start at a small width/height to avoid any
-		// flash of content
-		this.custom(this.prop === "width" || this.prop === "height" ? 1 : 0, this.cur());
+		// Make sure that we start at a small width/height to avoid any flash of content
+		if ( dataShow !== undefined ) {
+			// This show is picking up where a previous hide or show left off
+			this.custom( this.cur(), dataShow );
+		} else {
+			this.custom( this.prop === "width" || this.prop === "height" ? 1 : 0, this.cur() );
+		}
 
 		// Start by showing the element
 		jQuery( this.elem ).show();
@@ -8426,20 +8859,20 @@ jQuery.fx.prototype = {
 	// Simple 'hide' function
 	hide: function() {
 		// Remember where we started, so that we can go back to it later
-		this.options.orig[this.prop] = jQuery.style( this.elem, this.prop );
+		this.options.orig[ this.prop ] = jQuery._data( this.elem, "fxshow" + this.prop ) || jQuery.style( this.elem, this.prop );
 		this.options.hide = true;
 
 		// Begin the animation
-		this.custom(this.cur(), 0);
+		this.custom( this.cur(), 0 );
 	},
 
 	// Each step of an animation
 	step: function( gotoEnd ) {
-		var t = fxNow || createFxNow(),
+		var p, n, complete,
+			t = fxNow || createFxNow(),
 			done = true,
 			elem = this.elem,
-			options = this.options,
-			i, n;
+			options = this.options;
 
 		if ( gotoEnd || t >= options.duration + this.startTime ) {
 			this.now = this.end;
@@ -8448,8 +8881,8 @@ jQuery.fx.prototype = {
 
 			options.animatedProperties[ this.prop ] = true;
 
-			for ( i in options.animatedProperties ) {
-				if ( options.animatedProperties[i] !== true ) {
+			for ( p in options.animatedProperties ) {
+				if ( options.animatedProperties[ p ] !== true ) {
 					done = false;
 				}
 			}
@@ -8458,25 +8891,36 @@ jQuery.fx.prototype = {
 				// Reset the overflow
 				if ( options.overflow != null && !jQuery.support.shrinkWrapBlocks ) {
 
-					jQuery.each( [ "", "X", "Y" ], function (index, value) {
-						elem.style[ "overflow" + value ] = options.overflow[index];
+					jQuery.each( [ "", "X", "Y" ], function( index, value ) {
+						elem.style[ "overflow" + value ] = options.overflow[ index ];
 					});
 				}
 
 				// Hide the element if the "hide" operation was done
 				if ( options.hide ) {
-					jQuery(elem).hide();
+					jQuery( elem ).hide();
 				}
 
 				// Reset the properties, if the item has been hidden or shown
 				if ( options.hide || options.show ) {
-					for ( var p in options.animatedProperties ) {
-						jQuery.style( elem, p, options.orig[p] );
+					for ( p in options.animatedProperties ) {
+						jQuery.style( elem, p, options.orig[ p ] );
+						jQuery.removeData( elem, "fxshow" + p, true );
+						// Toggle data is no longer needed
+						jQuery.removeData( elem, "toggle" + p, true );
 					}
 				}
 
 				// Execute the complete function
-				options.complete.call( elem );
+				// in the event that the complete function throws an exception
+				// we must ensure it won't be called twice. #5684
+
+				complete = options.complete;
+				if ( complete ) {
+
+					options.complete = false;
+					complete.call( elem );
+				}
 			}
 
 			return false;
@@ -8490,8 +8934,8 @@ jQuery.fx.prototype = {
 				this.state = n / options.duration;
 
 				// Perform the easing function, defaults to swing
-				this.pos = jQuery.easing[ options.animatedProperties[ this.prop ] ]( this.state, n, 0, 1, options.duration );
-				this.now = this.start + ((this.end - this.start) * this.pos);
+				this.pos = jQuery.easing[ options.animatedProperties[this.prop] ]( this.state, n, 0, 1, options.duration );
+				this.now = this.start + ( (this.end - this.start) * this.pos );
 			}
 			// Perform the next step of the animation
 			this.update();
@@ -8503,9 +8947,15 @@ jQuery.fx.prototype = {
 
 jQuery.extend( jQuery.fx, {
 	tick: function() {
-		for ( var timers = jQuery.timers, i = 0 ; i < timers.length ; ++i ) {
-			if ( !timers[i]() ) {
-				timers.splice(i--, 1);
+		var timer,
+			timers = jQuery.timers,
+			i = 0;
+
+		for ( ; i < timers.length; i++ ) {
+			timer = timers[ i ];
+			// Checks the timer has not already been removed
+			if ( !timer() && timers[ i ] === timer ) {
+				timers.splice( i--, 1 );
 			}
 		}
 
@@ -8535,7 +8985,7 @@ jQuery.extend( jQuery.fx, {
 
 		_default: function( fx ) {
 			if ( fx.elem.style && fx.elem.style[ fx.prop ] != null ) {
-				fx.elem.style[ fx.prop ] = (fx.prop === "width" || fx.prop === "height" ? Math.max(0, fx.now) : fx.now) + fx.unit;
+				fx.elem.style[ fx.prop ] = fx.now + fx.unit;
 			} else {
 				fx.elem[ fx.prop ] = fx.now;
 			}
@@ -8543,6 +8993,16 @@ jQuery.extend( jQuery.fx, {
 	}
 });
 
+// Ensure props that can't be negative don't go there on undershoot easing
+jQuery.each( fxAttrs.concat.apply( [], fxAttrs ), function( i, prop ) {
+	// exclude marginTop, marginLeft, marginBottom and marginRight from this list
+	if ( prop.indexOf( "margin" ) ) {
+		jQuery.fx.step[ prop ] = function( fx ) {
+			jQuery.style( fx.elem, prop, Math.max(0, fx.now) + fx.unit );
+		};
+	}
+});
+
 if ( jQuery.expr && jQuery.expr.filters ) {
 	jQuery.expr.filters.animated = function( elem ) {
 		return jQuery.grep(jQuery.timers, function( fn ) {
@@ -8559,7 +9019,6 @@ function defaultDisplay( nodeName ) {
 		var body = document.body,
 			elem = jQuery( "<" + nodeName + ">" ).appendTo( body ),
 			display = elem.css( "display" );
-
 		elem.remove();
 
 		// If the simple way fails,
@@ -8578,7 +9037,7 @@ function defaultDisplay( nodeName ) {
 			// document to it; WebKit & Firefox won't allow reusing the iframe document.
 			if ( !iframeDoc || !iframe.createElement ) {
 				iframeDoc = ( iframe.contentWindow || iframe.contentDocument ).document;
-				iframeDoc.write( ( document.compatMode === "CSS1Compat" ? "<!doctype html>" : "" ) + "<html><body>" );
+				iframeDoc.write( ( jQuery.support.boxModel ? "<!doctype html>" : "" ) + "<html><body>" );
 				iframeDoc.close();
 			}
 
@@ -8587,7 +9046,6 @@ function defaultDisplay( nodeName ) {
 			iframeDoc.body.appendChild( elem );
 
 			display = jQuery.css( elem, "display" );
-
 			body.removeChild( iframe );
 		}
 
@@ -8601,41 +9059,23 @@ function defaultDisplay( nodeName ) {
 
 
 
-var rtable = /^t(?:able|d|h)$/i,
+var getOffset,
+	rtable = /^t(?:able|d|h)$/i,
 	rroot = /^(?:body|html)$/i;
 
 if ( "getBoundingClientRect" in document.documentElement ) {
-	jQuery.fn.offset = function( options ) {
-		var elem = this[0], box;
-
-		if ( options ) {
-			return this.each(function( i ) {
-				jQuery.offset.setOffset( this, options, i );
-			});
-		}
-
-		if ( !elem || !elem.ownerDocument ) {
-			return null;
-		}
-
-		if ( elem === elem.ownerDocument.body ) {
-			return jQuery.offset.bodyOffset( elem );
-		}
-
+	getOffset = function( elem, doc, docElem, box ) {
 		try {
 			box = elem.getBoundingClientRect();
 		} catch(e) {}
 
-		var doc = elem.ownerDocument,
-			docElem = doc.documentElement;
-
 		// Make sure we're not dealing with a disconnected DOM node
 		if ( !box || !jQuery.contains( docElem, elem ) ) {
 			return box ? { top: box.top, left: box.left } : { top: 0, left: 0 };
 		}
 
 		var body = doc.body,
-			win = getWindow(doc),
+			win = getWindow( doc ),
 			clientTop  = docElem.clientTop  || body.clientTop  || 0,
 			clientLeft = docElem.clientLeft || body.clientLeft || 0,
 			scrollTop  = win.pageYOffset || jQuery.support.boxModel && docElem.scrollTop  || body.scrollTop,
@@ -8647,30 +9087,10 @@ if ( "getBoundingClientRect" in document.documentElement ) {
 	};
 
 } else {
-	jQuery.fn.offset = function( options ) {
-		var elem = this[0];
-
-		if ( options ) {
-			return this.each(function( i ) {
-				jQuery.offset.setOffset( this, options, i );
-			});
-		}
-
-		if ( !elem || !elem.ownerDocument ) {
-			return null;
-		}
-
-		if ( elem === elem.ownerDocument.body ) {
-			return jQuery.offset.bodyOffset( elem );
-		}
-
-		jQuery.offset.initialize();
-
+	getOffset = function( elem, doc, docElem ) {
 		var computedStyle,
 			offsetParent = elem.offsetParent,
 			prevOffsetParent = elem,
-			doc = elem.ownerDocument,
-			docElem = doc.documentElement,
 			body = doc.body,
 			defaultView = doc.defaultView,
 			prevComputedStyle = defaultView ? defaultView.getComputedStyle( elem, null ) : elem.currentStyle,
@@ -8678,7 +9098,7 @@ if ( "getBoundingClientRect" in document.documentElement ) {
 			left = elem.offsetLeft;
 
 		while ( (elem = elem.parentNode) && elem !== body && elem !== docElem ) {
-			if ( jQuery.offset.supportsFixedPosition && prevComputedStyle.position === "fixed" ) {
+			if ( jQuery.support.fixedPosition && prevComputedStyle.position === "fixed" ) {
 				break;
 			}
 
@@ -8690,7 +9110,7 @@ if ( "getBoundingClientRect" in document.documentElement ) {
 				top  += elem.offsetTop;
 				left += elem.offsetLeft;
 
-				if ( jQuery.offset.doesNotAddBorder && !(jQuery.offset.doesAddBorderForTableAndCells && rtable.test(elem.nodeName)) ) {
+				if ( jQuery.support.doesNotAddBorder && !(jQuery.support.doesAddBorderForTableAndCells && rtable.test(elem.nodeName)) ) {
 					top  += parseFloat( computedStyle.borderTopWidth  ) || 0;
 					left += parseFloat( computedStyle.borderLeftWidth ) || 0;
 				}
@@ -8699,7 +9119,7 @@ if ( "getBoundingClientRect" in document.documentElement ) {
 				offsetParent = elem.offsetParent;
 			}
 
-			if ( jQuery.offset.subtractsBorderForOverflowNotVisible && computedStyle.overflow !== "visible" ) {
+			if ( jQuery.support.subtractsBorderForOverflowNotVisible && computedStyle.overflow !== "visible" ) {
 				top  += parseFloat( computedStyle.borderTopWidth  ) || 0;
 				left += parseFloat( computedStyle.borderLeftWidth ) || 0;
 			}
@@ -8712,7 +9132,7 @@ if ( "getBoundingClientRect" in document.documentElement ) {
 			left += body.offsetLeft;
 		}
 
-		if ( jQuery.offset.supportsFixedPosition && prevComputedStyle.position === "fixed" ) {
+		if ( jQuery.support.fixedPosition && prevComputedStyle.position === "fixed" ) {
 			top  += Math.max( docElem.scrollTop, body.scrollTop );
 			left += Math.max( docElem.scrollLeft, body.scrollLeft );
 		}
@@ -8721,47 +9141,36 @@ if ( "getBoundingClientRect" in document.documentElement ) {
 	};
 }
 
+jQuery.fn.offset = function( options ) {
+	if ( arguments.length ) {
+		return options === undefined ?
+			this :
+			this.each(function( i ) {
+				jQuery.offset.setOffset( this, options, i );
+			});
+	}
+
+	var elem = this[0],
+		doc = elem && elem.ownerDocument;
+
+	if ( !doc ) {
+		return null;
+	}
+
+	if ( elem === doc.body ) {
+		return jQuery.offset.bodyOffset( elem );
+	}
+
+	return getOffset( elem, doc, doc.documentElement );
+};
+
 jQuery.offset = {
-	initialize: function() {
-		var body = document.body, container = document.createElement("div"), innerDiv, checkDiv, table, td, bodyMarginTop = parseFloat( jQuery.css(body, "marginTop") ) || 0,
-			html = "<div style='position:absolute;top:0;left:0;margin:0;border:5px solid #000;padding:0;width:1px;height:1px;'><div></div></div><table style='position:absolute;top:0;left:0;margin:0;border:5px solid #000;padding:0;width:1px;height:1px;' cellpadding='0' cellspacing='0'><tr><td></td></tr></table>";
-
-		jQuery.extend( container.style, { position: "absolute", top: 0, left: 0, margin: 0, border: 0, width: "1px", height: "1px", visibility: "hidden" } );
-
-		container.innerHTML = html;
-		body.insertBefore( container, body.firstChild );
-		innerDiv = container.firstChild;
-		checkDiv = innerDiv.firstChild;
-		td = innerDiv.nextSibling.firstChild.firstChild;
-
-		this.doesNotAddBorder = (checkDiv.offsetTop !== 5);
-		this.doesAddBorderForTableAndCells = (td.offsetTop === 5);
-
-		checkDiv.style.position = "fixed";
-		checkDiv.style.top = "20px";
-
-		// safari subtracts parent border width here which is 5px
-		this.supportsFixedPosition = (checkDiv.offsetTop === 20 || checkDiv.offsetTop === 15);
-		checkDiv.style.position = checkDiv.style.top = "";
-
-		innerDiv.style.overflow = "hidden";
-		innerDiv.style.position = "relative";
-
-		this.subtractsBorderForOverflowNotVisible = (checkDiv.offsetTop === -5);
-
-		this.doesNotIncludeMarginInBodyOffset = (body.offsetTop !== bodyMarginTop);
-
-		body.removeChild( container );
-		jQuery.offset.initialize = jQuery.noop;
-	},
 
 	bodyOffset: function( body ) {
 		var top = body.offsetTop,
 			left = body.offsetLeft;
 
-		jQuery.offset.initialize();
-
-		if ( jQuery.offset.doesNotIncludeMarginInBodyOffset ) {
+		if ( jQuery.support.doesNotIncludeMarginInBodyOffset ) {
 			top  += parseFloat( jQuery.css(body, "marginTop") ) || 0;
 			left += parseFloat( jQuery.css(body, "marginLeft") ) || 0;
 		}
@@ -8781,7 +9190,7 @@ jQuery.offset = {
 			curOffset = curElem.offset(),
 			curCSSTop = jQuery.css( elem, "top" ),
 			curCSSLeft = jQuery.css( elem, "left" ),
-			calculatePosition = (position === "absolute" || position === "fixed") && jQuery.inArray("auto", [curCSSTop, curCSSLeft]) > -1,
+			calculatePosition = ( position === "absolute" || position === "fixed" ) && jQuery.inArray("auto", [curCSSTop, curCSSLeft]) > -1,
 			props = {}, curPosition = {}, curTop, curLeft;
 
 		// need to be able to calculate position if either top or left is auto and position is either absolute or fixed
@@ -8798,11 +9207,11 @@ jQuery.offset = {
 			options = options.call( elem, i, curOffset );
 		}
 
-		if (options.top != null) {
-			props.top = (options.top - curOffset.top) + curTop;
+		if ( options.top != null ) {
+			props.top = ( options.top - curOffset.top ) + curTop;
 		}
-		if (options.left != null) {
-			props.left = (options.left - curOffset.left) + curLeft;
+		if ( options.left != null ) {
+			props.left = ( options.left - curOffset.left ) + curLeft;
 		}
 
 		if ( "using" in options ) {
@@ -8815,6 +9224,7 @@ jQuery.offset = {
 
 
 jQuery.fn.extend({
+
 	position: function() {
 		if ( !this[0] ) {
 			return null;
@@ -8859,42 +9269,30 @@ jQuery.fn.extend({
 
 
 // Create scrollLeft and scrollTop methods
-jQuery.each( ["Left", "Top"], function( i, name ) {
-	var method = "scroll" + name;
+jQuery.each( {scrollLeft: "pageXOffset", scrollTop: "pageYOffset"}, function( method, prop ) {
+	var top = /Y/.test( prop );
 
 	jQuery.fn[ method ] = function( val ) {
-		var elem, win;
+		return jQuery.access( this, function( elem, method, val ) {
+			var win = getWindow( elem );
 
-		if ( val === undefined ) {
-			elem = this[ 0 ];
-
-			if ( !elem ) {
-				return null;
+			if ( val === undefined ) {
+				return win ? (prop in win) ? win[ prop ] :
+					jQuery.support.boxModel && win.document.documentElement[ method ] ||
+						win.document.body[ method ] :
+					elem[ method ];
 			}
 
-			win = getWindow( elem );
-
-			// Return the scroll offset
-			return win ? ("pageXOffset" in win) ? win[ i ? "pageYOffset" : "pageXOffset" ] :
-				jQuery.support.boxModel && win.document.documentElement[ method ] ||
-					win.document.body[ method ] :
-				elem[ method ];
-		}
-
-		// Set the scroll offset
-		return this.each(function() {
-			win = getWindow( this );
-
 			if ( win ) {
 				win.scrollTo(
-					!i ? val : jQuery( win ).scrollLeft(),
-					 i ? val : jQuery( win ).scrollTop()
+					!top ? val : jQuery( win ).scrollLeft(),
+					 top ? val : jQuery( win ).scrollTop()
 				);
 
 			} else {
-				this[ method ] = val;
+				elem[ method ] = val;
 			}
-		});
+		}, method, val, arguments.length, null );
 	};
 });
 
@@ -8910,72 +9308,97 @@ function getWindow( elem ) {
 
 
 // Create width, height, innerHeight, innerWidth, outerHeight and outerWidth methods
-jQuery.each([ "Height", "Width" ], function( i, name ) {
-
-	var type = name.toLowerCase();
+jQuery.each( { Height: "height", Width: "width" }, function( name, type ) {
+	var clientProp = "client" + name,
+		scrollProp = "scroll" + name,
+		offsetProp = "offset" + name;
 
 	// innerHeight and innerWidth
 	jQuery.fn[ "inner" + name ] = function() {
 		var elem = this[0];
-		return elem && elem.style ?
+		return elem ?
+			elem.style ?
 			parseFloat( jQuery.css( elem, type, "padding" ) ) :
+			this[ type ]() :
 			null;
 	};
 
 	// outerHeight and outerWidth
 	jQuery.fn[ "outer" + name ] = function( margin ) {
 		var elem = this[0];
-		return elem && elem.style ?
+		return elem ?
+			elem.style ?
 			parseFloat( jQuery.css( elem, type, margin ? "margin" : "border" ) ) :
+			this[ type ]() :
 			null;
 	};
 
-	jQuery.fn[ type ] = function( size ) {
-		// Get window width or height
-		var elem = this[0];
-		if ( !elem ) {
-			return size == null ? null : this;
-		}
+	jQuery.fn[ type ] = function( value ) {
+		return jQuery.access( this, function( elem, type, value ) {
+			var doc, docElemProp, orig, ret;
 
-		if ( jQuery.isFunction( size ) ) {
-			return this.each(function( i ) {
-				var self = jQuery( this );
-				self[ type ]( size.call( this, i, self[ type ]() ) );
-			});
-		}
+			if ( jQuery.isWindow( elem ) ) {
+				// 3rd condition allows Nokia support, as it supports the docElem prop but not CSS1Compat
+				doc = elem.document;
+				docElemProp = doc.documentElement[ clientProp ];
+				return jQuery.support.boxModel && docElemProp ||
+					doc.body && doc.body[ clientProp ] || docElemProp;
+			}
 
-		if ( jQuery.isWindow( elem ) ) {
-			// Everyone else use document.documentElement or document.body depending on Quirks vs Standards mode
-			// 3rd condition allows Nokia support, as it supports the docElem prop but not CSS1Compat
-			var docElemProp = elem.document.documentElement[ "client" + name ];
-			return elem.document.compatMode === "CSS1Compat" && docElemProp ||
-				elem.document.body[ "client" + name ] || docElemProp;
+			// Get document width or height
+			if ( elem.nodeType === 9 ) {
+				// Either scroll[Width/Height] or offset[Width/Height], whichever is greater
+				doc = elem.documentElement;
 
-		// Get document width or height
-		} else if ( elem.nodeType === 9 ) {
-			// Either scroll[Width/Height] or offset[Width/Height], whichever is greater
-			return Math.max(
-				elem.documentElement["client" + name],
-				elem.body["scroll" + name], elem.documentElement["scroll" + name],
-				elem.body["offset" + name], elem.documentElement["offset" + name]
-			);
+				// when a window > document, IE6 reports a offset[Width/Height] > client[Width/Height]
+				// so we can't use max, as it'll choose the incorrect offset[Width/Height]
+				// instead we use the correct client[Width/Height]
+				// support:IE6
+				if ( doc[ clientProp ] >= doc[ scrollProp ] ) {
+					return doc[ clientProp ];
+				}
 
-		// Get or set width or height on the element
-		} else if ( size === undefined ) {
-			var orig = jQuery.css( elem, type ),
+				return Math.max(
+					elem.body[ scrollProp ], doc[ scrollProp ],
+					elem.body[ offsetProp ], doc[ offsetProp ]
+				);
+			}
+
+			// Get width or height on the element
+			if ( value === undefined ) {
+				orig = jQuery.css( elem, type );
 				ret = parseFloat( orig );
+				return jQuery.isNumeric( ret ) ? ret : orig;
+			}
 
-			return jQuery.isNaN( ret ) ? orig : ret;
-
-		// Set the width or height on the element (default to pixels if value is unitless)
-		} else {
-			return this.css( type, typeof size === "string" ? size : size + "px" );
-		}
+			// Set the width or height on the element
+			jQuery( elem ).css( type, value );
+		}, type, value, arguments.length, null );
 	};
-
 });
 
 
+
+
 // Expose jQuery to the global object
 window.jQuery = window.$ = jQuery;
-})(window);
+
+// Expose jQuery as an AMD module, but only for AMD loaders that
+// understand the issues with loading multiple versions of jQuery
+// in a page that all might call define(). The loader will indicate
+// they have special allowances for multiple jQuery versions by
+// specifying define.amd.jQuery = true. Register as a named module,
+// since jQuery can be concatenated with other files that may use define,
+// but not use a proper concatenation script that understands anonymous
+// AMD modules. A named AMD is safest and most robust way to register.
+// Lowercase jquery is used because AMD module names are derived from
+// file names, and jQuery is normally delivered in a lowercase file name.
+// Do this after creating the global so that if an AMD module wants to call
+// noConflict to hide this version of jQuery, it will work.
+if ( typeof define === "function" && define.amd && define.amd.jQuery ) {
+	define( "jquery", [], function () { return jQuery; } );
+}
+
+
+
+})( window );

From 7e3c1e008d6b2eff10d412993df293fce2156151 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 04:48:50 -0400
Subject: [PATCH 4293/8313] webapp now uses twitter bootstrap

mocked up the main screen, and am actually pretty happy with it!
---
 Assistant/Threads/WebApp.hs                   |  24 +-
 debian/copyright                              | 382 +---------
 .../blog/day_45__long_polling/full.png        | Bin 0 -> 55185 bytes
 .../blog/day_45__long_polling/phone.png       | Bin 0 -> 41602 bytes
 static/css/bootstrap-responsive.css           | 686 ++++++++++++++++++
 static/{ => css}/bootstrap.css                |   4 +-
 static/glyphicons-halflings-white.png         | Bin 4352 -> 0 bytes
 static/glyphicons-halflings.png               | Bin 4352 -> 0 bytes
 static/js/bootstrap-dropdown.js               |  92 +++
 templates/bootstrap.hamlet                    |  52 ++
 templates/default-layout.hamlet               |  13 +-
 templates/longpolling.julius                  |   4 +-
 templates/status.hamlet                       |  28 +-
 13 files changed, 877 insertions(+), 408 deletions(-)
 create mode 100644 doc/design/assistant/blog/day_45__long_polling/full.png
 create mode 100644 doc/design/assistant/blog/day_45__long_polling/phone.png
 create mode 100644 static/css/bootstrap-responsive.css
 rename static/{ => css}/bootstrap.css (99%)
 delete mode 100644 static/glyphicons-halflings-white.png
 delete mode 100644 static/glyphicons-halflings.png
 create mode 100644 static/js/bootstrap-dropdown.js
 create mode 100644 templates/bootstrap.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index f82a1fb6b9..050d62cf17 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -47,11 +47,16 @@ mkYesod "WebApp" [parseRoutes|
 |]
 
 instance Yesod WebApp where
-	defaultLayout contents = do
-		page <- widgetToPageContent contents
+	defaultLayout widget = do
 		mmsg <- getMessage
 		webapp <- getYesod
-		hamletToRepHtml $(hamletFile $ hamletTemplate "default-layout")
+		page <- widgetToPageContent $ do
+			addStylesheet $ StaticR css_bootstrap_css
+			addStylesheet $ StaticR css_bootstrap_responsive_css
+			addScript $ StaticR jquery_full_js
+			addScript $ StaticR js_bootstrap_dropdown_js
+			$(widgetFile "default-layout")
+		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
 
 	{- Require an auth token be set when accessing any (non-static route) -}
 	isAuthorized _ _ = checkAuthToken secretToken
@@ -68,7 +73,7 @@ instance Yesod WebApp where
 
 {- Add to any widget to make it auto-update.
  -
- - The widget should have a html element with id=poll, which will be
+ - The widget should have a html element with id=updating, which will be
  - replaced when it's updated.
  - 
  - Updating is done by getting html from the gethtml route.
@@ -80,7 +85,7 @@ instance Yesod WebApp where
  - state.
  -}
 autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Widget
-autoUpdate poll gethtml home ms_delay ms_startdelay = do
+autoUpdate updating gethtml home ms_delay ms_startdelay = do
 	{- Fallback refreshing is provided for non-javascript browsers. -}
 	let delayseconds = show $ ms_to_seconds ms_delay
 	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
@@ -88,7 +93,6 @@ autoUpdate poll gethtml home ms_delay ms_startdelay = do
 	{- Use long polling to update the status display. -}
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
-	addScript $ StaticR jquery_full_js
 	$(widgetFile "longpolling")
 	where
 		ms_to_seconds :: Int -> Int
@@ -100,15 +104,13 @@ statusDisplay = do
 	webapp <- lift getYesod
 	time <- show <$> liftIO getCurrentTime
 	
-	poll <- lift newIdent
+	updating <- lift newIdent
 	$(widgetFile "status")
 	
-	autoUpdate poll StatusR HomeR (3000 :: Int) (40 :: Int)
+	autoUpdate updating StatusR HomeR (3000 :: Int) (40 :: Int)
 
 getHomeR :: Handler RepHtml
-getHomeR = defaultLayout $ do
-	statusDisplay
-	[whamlet|<p><a href="@{ConfigR}">config|]
+getHomeR = defaultLayout statusDisplay
 
 {- Called by client to poll for a new webapp status display.
  -
diff --git a/debian/copyright b/debian/copyright
index 7f906a64aa..4cab3a048f 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -57,7 +57,7 @@ License: MIT or GPL-2
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-Files: static/bootstrap.css
+Files: static/*/bootstrap*
 Copyright: 2011-2012 Twitter, Inc.
 License: Apache-2.0
  Licensed under the Apache License, Version 2.0 (the "License");
@@ -74,383 +74,3 @@ License: Apache-2.0
  .
  The complete text of the Apache License is distributed in
  /usr/share/common-licenses/Apache-2.0 on Debian systems.
-
-Files: static/glyphicons*
-Copyright: 2010-2012 Jan Kovarik <glyphicons@gmail.com>
-License: CC-BY-3.0
- Creative Commons Attribution 3.0 License
- .
- THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS
- OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR
- "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER
- APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS
- AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS
- PROHIBITED.
- .
- BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU
- ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE.
- TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A
- CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE
- IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND
- CONDITIONS.
- .
- 1.   Definitions
- .
-   a) "Adaptation" means a work based upon
-      the Work, or upon the Work and other pre-existing works,
-      such as a translation, adaptation, derivative work,
-      arrangement of music or other alterations of a literary
-      or artistic work, or phonogram or performance and
-      includes cinematographic adaptations or any other form in
-      which the Work may be recast, transformed, or adapted
-      including in any form recognizably derived from the
-      original, except that a work that constitutes a
-      Collection will not be considered an Adaptation for the
-      purpose of this License. For the avoidance of doubt,
-      where the Work is a musical work, performance or
-      phonogram, the synchronization of the Work in
-      timed-relation with a moving image ("synching") will be
-      considered an Adaptation for the purpose of this
-      License.
- .
-   b) "Collection"</strong> means a collection of
-      literary or artistic works, such as encyclopedias and
-      anthologies, or performances, phonograms or broadcasts,
-      or other works or subject matter other than works listed
-      in Section 1(f) below, which, by reason of the selection
-      and arrangement of their contents, constitute
-      intellectual creations, in which the Work is included in
-      its entirety in unmodified form along with one or more
-      other contributions, each constituting separate and
-      independent works in themselves, which together are
-      assembled into a collective whole. A work that
-      constitutes a Collection will not be considered an
-      Adaptation (as defined above) for the purposes of this
-      License.
- .
-   c) "Distribute" means to make available
-      to the public the original and copies of the Work or
-      Adaptation, as appropriate, through sale or other
-      transfer of ownership.
- .
-   d) "Licensor" means the individual,
-      individuals, entity or entities that offer(s) the Work
-      under the terms of this License.
- .
-   e) "Original Author" means, in the case
-      of a literary or artistic work, the individual,
-      individuals, entity or entities who created the Work or
-      if no individual or entity can be identified, the
-      publisher; and in addition (i) in the case of a
-      performance the actors, singers, musicians, dancers, and
-      other persons who act, sing, deliver, declaim, play in,
-      interpret or otherwise perform literary or artistic works
-      or expressions of folklore; (ii) in the case of a
-      phonogram the producer being the person or legal entity
-      who first fixes the sounds of a performance or other
-      sounds; and, (iii) in the case of broadcasts, the
-      organization that transmits the broadcast.
- .
-   f) "Work" means the literary and/or
-      artistic work offered under the terms of this License
-      including without limitation any production in the
-      literary, scientific and artistic domain, whatever may be
-      the mode or form of its expression including digital
-      form, such as a book, pamphlet and other writing; a
-      lecture, address, sermon or other work of the same
-      nature; a dramatic or dramatico-musical work; a
-      choreographic work or entertainment in dumb show; a
-      musical composition with or without words; a
-      cinematographic work to which are assimilated works
-      expressed by a process analogous to cinematography; a
-      work of drawing, painting, architecture, sculpture,
-      engraving or lithography; a photographic work to which
-      are assimilated works expressed by a process analogous to
-      photography; a work of applied art; an illustration, map,
-      plan, sketch or three-dimensional work relative to
-      geography, topography, architecture or science; a
-      performance; a broadcast; a phonogram; a compilation of
-      data to the extent it is protected as a copyrightable
-      work; or a work performed by a variety or circus
-      performer to the extent it is not otherwise considered a
-      literary or artistic work.
- .
-   g) "You"</strong> means an individual or entity
-      exercising rights under this License who has not
-      previously violated the terms of this License with
-      respect to the Work, or who has received express
-      permission from the Licensor to exercise rights under
-      this License despite a previous violation.
- .
-   h) "Publicly Perform" means to perform
-      public recitations of the Work and to communicate to the
-      public those public recitations, by any means or process,
-      including by wire or wireless means or public digital
-      performances; to make available to the public Works in
-      such a way that members of the public may access these
-      Works from a place and at a place individually chosen by
-      them; to perform the Work to the public by any means or
-      process and the communication to the public of the
-      performances of the Work, including by public digital
-      performance; to broadcast and rebroadcast the Work by any
-      means including signs, sounds or images.
- .
-   i) "Reproduce" means to make copies of
-      the Work by any means including without limitation by
-      sound or visual recordings and the right of fixation and
-      reproducing fixations of the Work, including storage of a
-      protected performance or phonogram in digital form or
-      other electronic medium.
- .
- 2.   Fair Dealing Rights. Nothing in this
-      License is intended to reduce, limit, or restrict any uses
-      free from copyright or rights arising from limitations or
-      exceptions that are provided for in connection with the
-      copyright protection under copyright law or other
-      applicable laws.
- .
- 3.   License Grant. Subject to the terms
-      and conditions of this License, Licensor hereby grants You
-      a worldwide, royalty-free, non-exclusive, perpetual (for
-      the duration of the applicable copyright) license to
-      exercise the rights in the Work as stated below:</p>
- .
-   a) to Reproduce the Work, to incorporate the Work into
-      one or more Collections, and to Reproduce the Work as
-      incorporated in the Collections;
- .
-   b) to create and Reproduce Adaptations provided that any
-      such Adaptation, including any translation in any medium,
-      takes reasonable steps to clearly label, demarcate or
-      otherwise identify that changes were made to the original
-      Work. For example, a translation could be marked "The
-      original work was translated from English to Spanish," or
-      a modification could indicate "The original work has been
-      modified.";
- .
-   c) to Distribute and Publicly Perform the Work including
-      as incorporated in Collections; and,
- .
-   d) to Distribute and Publicly Perform Adaptations.
- .
-   e) For the avoidance of doubt:
- .
-      i) Non-waivable Compulsory License
-         Schemes. In those jurisdictions in which the
-         right to collect royalties through any statutory or
-         compulsory licensing scheme cannot be waived, the
-         Licensor reserves the exclusive right to collect such
-         royalties for any exercise by You of the rights
-         granted under this License;
- .
-     ii) Waivable Compulsory License
-         Schemes. In those jurisdictions in which the
-         right to collect royalties through any statutory or
-         compulsory licensing scheme can be waived, the
-         Licensor waives the exclusive right to collect such
-         royalties for any exercise by You of the rights
-         granted under this License; and,
- .
-    iii) Voluntary License Schemes. The
-         Licensor waives the right to collect royalties,
-         whether individually or, in the event that the
-         Licensor is a member of a collecting society that
-         administers voluntary licensing schemes, via that
-         society, from any exercise by You of the rights
-         granted under this License.
- .
-      The above rights may be exercised in all media and
-      formats whether now known or hereafter devised. The above
-      rights include the right to make such modifications as are
-      technically necessary to exercise the rights in other media
-      and formats. Subject to Section 8(f), all rights not
-      expressly granted by Licensor are hereby reserved.
- .
- 4.   Restrictions. The license granted in
-      Section 3 above is expressly made subject to and limited by
-      the following restrictions:
- .
-   a) You may Distribute or Publicly Perform the Work only
-      under the terms of this License. You must include a copy
-      of, or the Uniform Resource Identifier (URI) for, this
-      License with every copy of the Work You Distribute or
-      Publicly Perform. You may not offer or impose any terms
-      on the Work that restrict the terms of this License or
-      the ability of the recipient of the Work to exercise the
-      rights granted to that recipient under the terms of the
-      License. You may not sublicense the Work. You must keep
-      intact all notices that refer to this License and to the
-      disclaimer of warranties with every copy of the Work You
-      Distribute or Publicly Perform. When You Distribute or
-      Publicly Perform the Work, You may not impose any
-      effective technological measures on the Work that
-      restrict the ability of a recipient of the Work from You
-      to exercise the rights granted to that recipient under
-      the terms of the License. This Section 4(a) applies to
-      the Work as incorporated in a Collection, but this does
-      not require the Collection apart from the Work itself to
-      be made subject to the terms of this License. If You
-      create a Collection, upon notice from any Licensor You
-      must, to the extent practicable, remove from the
-      Collection any credit as required by Section 4(b), as
-      requested. If You create an Adaptation, upon notice from
-      any Licensor You must, to the extent practicable, remove
-      from the Adaptation any credit as required by Section
-      4(b), as requested.
- .
-   b) If You Distribute, or Publicly Perform the Work or
-      any Adaptations or Collections, You must, unless a
-      request has been made pursuant to Section 4(a), keep
-      intact all copyright notices for the Work and provide,
-      reasonable to the medium or means You are utilizing: (i)
-      the name of the Original Author (or pseudonym, if
-      applicable) if supplied, and/or if the Original Author
-      and/or Licensor designate another party or parties (e.g.,
-      a sponsor institute, publishing entity, journal) for
-      attribution ("Attribution Parties") in Licensor's
-      copyright notice, terms of service or by other reasonable
-      means, the name of such party or parties; (ii) the title
-      of the Work if supplied; (iii) to the extent reasonably
-      practicable, the URI, if any, that Licensor specifies to
-      be associated with the Work, unless such URI does not
-      refer to the copyright notice or licensing information
-      for the Work; and (iv) , consistent with Section 3(b), in
-      the case of an Adaptation, a credit identifying the use
-      of the Work in the Adaptation (e.g., "French translation
-      of the Work by Original Author," or "Screenplay based on
-      original Work by Original Author"). The credit required
-      by this Section 4 (b) may be implemented in any
-      reasonable manner; provided, however, that in the case of
-      a Adaptation or Collection, at a minimum such credit will
-      appear, if a credit for all contributing authors of the
-      Adaptation or Collection appears, then as part of these
-      credits and in a manner at least as prominent as the
-      credits for the other contributing authors. For the
-      avoidance of doubt, You may only use the credit required
-      by this Section for the purpose of attribution in the
-      manner set out above and, by exercising Your rights under
-      this License, You may not implicitly or explicitly assert
-      or imply any connection with, sponsorship or endorsement
-      by the Original Author, Licensor and/or Attribution
-      Parties, as appropriate, of You or Your use of the Work,
-      without the separate, express prior written permission of
-      the Original Author, Licensor and/or Attribution
-      Parties.
- .
-   c) Except as otherwise agreed in writing by the Licensor
-      or as may be otherwise permitted by applicable law, if
-      You Reproduce, Distribute or Publicly Perform the Work
-      either by itself or as part of any Adaptations or
-      Collections, You must not distort, mutilate, modify or
-      take other derogatory action in relation to the Work
-      which would be prejudicial to the Original Author's honor
-      or reputation. Licensor agrees that in those
-      jurisdictions (e.g. Japan), in which any exercise of the
-      right granted in Section 3(b) of this License (the right
-      to make Adaptations) would be deemed to be a distortion,
-      mutilation, modification or other derogatory action
-      prejudicial to the Original Author's honor and
-      reputation, the Licensor will waive or not assert, as
-      appropriate, this Section, to the fullest extent
-      permitted by the applicable national law, to enable You
-      to reasonably exercise Your right under Section 3(b) of
-      this License (right to make Adaptations) but not
-      otherwise.
- .
- 5.   Representations, Warranties and
-      Disclaimer
- .
-      UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN
-      WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO
-      REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE
-      WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING,
-      WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,
-      FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE
-      ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE
-      PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE.
-      SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED
-      WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
- .
- 6.   Limitation on Liability. EXCEPT TO
-      THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL
-      LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY
-      SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY
-      DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK,
-      EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF
-      SUCH DAMAGES.
- .
- 7.   Termination
- .
-   a) This License and the rights granted hereunder will
-      terminate automatically upon any breach by You of the
-      terms of this License. Individuals or entities who have
-      received Adaptations or Collections from You under this
-      License, however, will not have their licenses terminated
-      provided such individuals or entities remain in full
-      compliance with those licenses. Sections 1, 2, 5, 6, 7,
-      and 8 will survive any termination of this License.</li>
- .
-   b) Subject to the above terms and conditions, the
-      license granted here is perpetual (for the duration of
-      the applicable copyright in the Work). Notwithstanding
-      the above, Licensor reserves the right to release the
-      Work under different license terms or to stop
-      distributing the Work at any time; provided, however that
-      any such election will not serve to withdraw this License
-      (or any other license that has been, or is required to
-      be, granted under the terms of this License), and this
-      License will continue in full force and effect unless
-      terminated as stated above.
- .
- 8.   Miscellaneous
- .
-   a) Each time You Distribute or Publicly Perform the Work
-      or a Collection, the Licensor offers to the recipient a
-      license to the Work on the same terms and conditions as
-      the license granted to You under this License.
- .
-   b) Each time You Distribute or Publicly Perform an
-      Adaptation, Licensor offers to the recipient a license to
-      the original Work on the same terms and conditions as the
-      license granted to You under this License.
- .
-   c) If any provision of this License is invalid or
-      unenforceable under applicable law, it shall not affect
-      the validity or enforceability of the remainder of the
-      terms of this License, and without further action by the
-      parties to this agreement, such provision shall be
-      reformed to the minimum extent necessary to make such
-      provision valid and enforceable.
- .
-   d) No term or provision of this License shall be deemed
-      waived and no breach consented to unless such waiver or
-      consent shall be in writing and signed by the party to be
-      charged with such waiver or consent.
- .
-   e) This License constitutes the entire agreement between
-      the parties with respect to the Work licensed here. There
-      are no understandings, agreements or representations with
-      respect to the Work not specified here. Licensor shall
-      not be bound by any additional provisions that may appear
-      in any communication from You. This License may not be
-      modified without the mutual written agreement of the
-      Licensor and You.
- .
-   f) The rights granted under, and the subject matter
-      referenced, in this License were drafted utilizing the
-      terminology of the Berne Convention for the Protection of
-      Literary and Artistic Works (as amended on September 28,
-      1979), the Rome Convention of 1961, the WIPO Copyright
-      Treaty of 1996, the WIPO Performances and Phonograms
-      Treaty of 1996 and the Universal Copyright Convention (as
-      revised on July 24, 1971). These rights and subject
-      matter take effect in the relevant jurisdiction in which
-      the License terms are sought to be enforced according to
-      the corresponding provisions of the implementation of
-      those treaty provisions in the applicable national law.
-      If the standard suite of rights granted under applicable
-      copyright law includes additional rights not granted
-      under this License, such additional rights are deemed to
-      be included in the License; this License is not intended
-      to restrict the license of any rights under applicable
-      law.
diff --git a/doc/design/assistant/blog/day_45__long_polling/full.png b/doc/design/assistant/blog/day_45__long_polling/full.png
new file mode 100644
index 0000000000000000000000000000000000000000..3963ae1dcb26d2590c08ab036d9ec68fe266bbe0
GIT binary patch
literal 55185
zcmYg%1z1yW-1cZCML{|Qm6Glj>6Y#ikZwjdQc^1Ah@nV#cL~CP0i(McM%U<l)BpQk
z?>8=XYR@^(Ip-JmbN}uq=DnIc0WKvj001CRRFKgG05DOj==zV)Q40@u3{TVo(?&{F
z3IM1|#JjV=LT#gYXv)6>ln+zyqBiI)6*N@=0KeA&K*%Qm;0CoTWCs8MasdE)<^X`m
zcL0FQHM3bm40QnegR;B~0QvXtM_WM>YR_X=1p^P%QsLjfT)%?^syRSWMoQafZhz6+
zSmy$Xb3EOY(MD{LBh(q1o5S2Fzr|cP5owrjU@*%?Ri9=cK;2R8VwVR>*-{!xiNPRC
zXC|a3BgT{xAUhO+BOnp*2!BrF$ehz4iGYcw>|5b%;BZrb#CE{2u*&iYEJq-R{!tG7
zX)7~s4)YXcHlI3vH&@)JuS%~_Ukd5XLZ6<zd-UQDw!sjMp9YDn)QT{p!-R)ic&A%f
zryvm2@1(@=RmqA6^o~WZFqa35@hcbp_(TN5zg9YZ_78?6m{Wy<w)}0s7_obLWV{nx
zHCZy62R8(C$|qmFD%hX%KHs?Z%&W;T{`nzyET4nq#owkyVKKow6#mOGVVzajfd;!*
z#P?typSaF5bX7D4{wJz>ai5NKADY;>4L0{UlGyAxp8Rdr=aiSv1_<k{!8UR9jAEjt
z?LHHOxp@)+ic*+AYcVAZ5A=&>yO}VgDB1zuK0>u{7p<BXeufpgtsh<<@ZK18elf^n
zboI92wMUT_YyL76o||Hj&tW4IJ^y<3LU5)Ho&J%S!wAGHerJ)<@QIH`>u5Zl%e;ma
z7Pi<+bn}pU5>{aqK^*1Wks4Pooa84j8FJ%{bJ{)Yi%JxoX(BDGcnkt6{GSribR?U+
zOz9sYLlgQH%D>msydXcUkgEwY&VR4CqlEtVatfA};(#ZzsLm}FQt_8|?0$Hg;(;^-
zYbd^u4b30v8HxaUPyO(`PXz*c<NZcdS2?0`jkWgIw;;z)ZFh^Lh6ua4+?yP2JHSP3
zf~QI;GOX6*j{#FZ4o2DMX8UJPay|V_f}}M3<RVqdzp?CKd!pX+r!SG!sEcBTS`rXD
zh55^PZwZ&*4AN=fqP3Dd6@)ofvfQP7kP9v<6Adxx;#aRKjIGT|%U~i>z<Qgz-N`44
z->r~0X1dtyxKPDeAj6r$ma7@V{5hWtYbRa~lJrf<njOQ?$JZ}YMYx+gPNm#-Z-Dsk
zRTpL*)k~iEB=GJ);>q9OhQ${lHKRo^m`*pjz|10uGc&lJBNE=(lZ-*8rHfkI?wHDE
z7SSVQWID|m29!Jm8!t<-6u|g&u{?d#eyz)ompyKvijiSR8Hgl`-o*C{-8T1A{jP1Q
zuWel5iK(&SWBT*GXWV`#1%avw`@h>31;?`ia_EoW;Dy2$8Rxb<BxkGLn_80MLV2yK
zOliV<Z#K?e-6l{2Mk@4LD<&8**)!?Ca%;-5SSv$a?6EFi@54oZWN*UrWqx-m*p~MV
z<xErI;^ARn7^O>>n?O%4FEv?9YK6PGG>dY*5rs#3$ex-~m3;G;PeH~S@AY@q*G;BX
zurxf~yqs6JlgzYP^@`J*3I5vXyF+t<a=;^WSWP6@gz?o=ITA;Fz(zaAd!^ivmQbRw
zm%Tdz3hVszFzdHqD1E$M+$j5_r`{NxSU-~c0!`jb2kzFnRBV%~e!sIQpj9t7J>v|S
zk+>%;ySJ5teW#`Q<<-hN`u=LU0MRQayHw+9Y!=$)Y{*ux-XXp$hWyh>5$LBnf4tDf
zU}mqz^}^yP<l$b$?ehNkG_Kup9cRdPPq!_D^!anJu)n~?9o9HRIc=f<mdV1wQI)`w
zqC|@8YvscXqg3Jik~*-_^Xe5fWRlTLiX=P?UHDgd5mm_F;Ic8S8wj9JKSc6V($acl
zGk+GyzAdtRuUQ_U@g+vdn5D7WnKQ#xV0eYb=J?~YVN-(!GCf?YoTqX>bB#+Zi%VOM
z&p;1BrVc^Ah`}m4x-70flB*@c>z3QZfZ(ecWh3Q$@l)XbUfZoYo!QZx_o8~q5F3XO
zUH$&@5rW3_A>4CjuVPy-S$E_>)U(`dnKnN$8%$_W>wW+5iBxs5*t@2)Gxf!b7a$NQ
zZG2o6Iag2MGTOB)^7`+~bR0ZJD;=djCiVWNkayaZlXGLup4NYNe`m|^FC-~Q>#L$Y
zl?7YsxGP3^l{qauIDAI$BXL)-=5p`R^SIo<lAo~Mq$UbUQKS&GIU3=DJxBY}Zxtua
zN$7J1()g{USgA=&Mm1Znl@=<Y1W3a17WUh_x*8Cw5aW^Q1$34q65P8VmO@OiLU`q$
z-HRelwR9G2j&CfbvtGtwP<+FV*%>h{(l&0+#ePCTOiWE}#Y@@E#RKB2HgRZ@*$~!!
ztrupI>n>{{^|K?}O#GBohP%@Gt?7e-AN7CzZ_}asYkU+0(zl*b%ya%n6(r`lJW?yK
zcw%f+ri-Lp6z=Xk$v95MTu6N6iabwT&{!~{m70uUlMl3o^oY+7nX&IEH+~iyr|!fF
zZhgkQ6#_kf7m!)L<)8|E{P-4!aNi-}*^XjOw@Y0*9gOoUDhMhfI1UoC+8Fdi6kk0p
zQYmL)o3yVR9tQf57lM@X+&w%-Yf80WyX+M|`xiUD7N0N4Xlw{4@KVYV?lMK^m~WrA
zX<{-)$EgfJD_67GEAA^_LiNut+J`FcPwwN8xBGDR;OD~1Lc#+CA}lMcWlK_4-)}N=
zIft9|tG)z|_B5;|*IZ)*+}TW;bK5-?X()(Ci!33$HZaq+bZgH1-8^&Y+a7nh@awQ|
zDnDCOlZ-k_u{0c4k04b*pD)b+Qbm=OIDku}#A>xnT2?CL{#sOwjokMIFQ?$DkLHee
zR*ETmE_>iQB>wQMGdf`Od%^VSm*UJ!3Sw#_abhOK^UWqExZwy8Y4no39JjYrvK>Sh
zyo?EWrKdG+uw3r6#|!#2F<<8(obQYL{aY$Dl%0{0VK^^;YnyB)&e8A1+m@-HxD-Ca
zwzL2e5)yL!i~T?Tetw_D*hKi6;{LkaCv80-pwmOlfts33tNV@)AV!p;)LZRn;cRN>
z{IE_WZfN{v{g*33qY>ugs^s%_r>>R_9}|$5X9#M&xqh%1m+*7H%)W1yak@M3hTySM
z-CL-1B>rkq72D#yElZ=r{!-~pm-Wzb(c0FQ)9h(S^ZertIW@JXx@T=mOGew((-@i8
zNxd?%r==PjY+IiFBx=*{?t6NG$K5{uzK^NrC)lO_#y)rXW;db`=FeiVD(rjdUr~=y
zSj7ugxfd^IV-r!Lw7a?XV)C^5sfmNlay>#4Jt3T)y)9E`ZS-B26l=K33$n7U+d&E{
z37dVtGe|cM0+dF^e%IA{x5N01F3>9HdVPX?Sv@hp2Bg*nZVN9MtO5}>`6eq_*V#N=
z-P}w(S@tA0A3T1*Nyyd_YErOrkF?lO(lHJC*^s73z=-27(>Xm|ppeCp*MR=d5+@e_
z<q+4>-}8&J%P@Ho%+5o=+s#rg@qY1I>-X@^m6T;<E|fE3_)!+O#khcF5yxv*!`EK{
zuI}HagYaAfN&mD?H+XAn3DOru><WYcf+h8sgdvnNKR+aXVM!*Nv@ffdfL$PtkDp*A
zd_?<~IVYrhNC6roz6qQ7H&sLgQMP{fX^TrKnC7`Fz8b~J1nu5e+3=}Bv(t+U_%xQ3
ztA_^_pOvd^OTD@4y<dBFN5!AfZxgcv1@nSHIB^sgK4TIT8-kdrW{S}h;(q!V6)Kv-
zp7ro<4@Y2P`TF@4YXg0%3)Bk8;QoTb|3C^a<d5VzJ|J|PpqWQfujiYu3-bm~LdouH
z=*kyDr95q$m1HGQ-w&07TX;o$=!@TPpmM;y{Uf*w7rAwu%+D{}3<Cq_O~Q7nCaPn9
zgc8`^u!2O9R0M1qB27xUq!rFw;$ErS`Wh=V>hWI{ta0;<rPKpo{ey_S6s)h2<nhg?
z<>I2n$)D5xk%01#;ELyC1x9+~Vdcz7C5<30z0V6?FQ3L_WGLTGy@*mX{W3t}F4wQ{
z67ceA(L0@NGW37m@a5(!U`79zhThbR(#`M0(wIN<=Yt;?(M4L2xL<!?#R$I_k8kEl
z;O7Iz{7cE{EFg`FNfJ|{)3bn;r3WeYd*6hz)l0K?dr=nTF)~k=L^j$(tAZWpbhMMV
z_jc$12N2sOd{1ygT<k4;Jnbf`cNqV?Eezh?U1i;P&3#~K12Z$Mahs1hqfj-W{)b7n
zqMJNdRZxNdUmu^i$1b!Mcs~7m=*c^0Q9c0u-^cJGm*Oiw5G`7ok8&txT|T{;<lkQJ
z8vkYlDpWe`X)bv30UCKTg@QJ~hi(6N{23(ioQf;ANZD<C6tJT9bTPgc@UN+$GB3zg
z3ISN~%wZ-WA;rxZnZWuN(_ZA7VuWbMM{fSn>9}{c!Yb4CPEvaPSwYNG!RE;&je_I3
z;4@obGaL)cplQ0?%DR|$tisBAi}d-<Icn$t6j|)yyU06-ZW=mu^R5YTuro@Rzt6Ka
zUKP50T}S%g6$p#Z06!XYWqx`pN`Owcs9cT~>Xccv7IMEGsH?OhA8^LU#$Kxh>=%%-
z`TlgnMiU8tuve>>VU68$SI?e#2*+L!bv%x4cMG43lqc6e{E%*#<<{hK*Vz|6tXIuL
z^*wFLyzx7Dsz(0EKga)kZ<Og`>{XU&a8i2gf6rKc;(#_J=i5^zrUqV{;RHI#pv7iM
zn?awg?nKG&4}X#~G^%OEKOhbl{g(p~PV@B#S>8**NB0c{s#&ZlTm*`|9fj|$zR!RI
zNpT}aZAak1{F+nurmpVpb?^=?HRNCsSrHpsGHT1r!C|LhN*VupeXj0jcq9wSz}VOa
zK|zFt^x=G!hmr03FHupqL$=)i>_9YY)K<*%yhF3hRMR)zkYr^m*V8*SHF@Lr!ZQ-t
zChesn2TpZ$nBn-{vbuwMRvY<b)#A7oL=brIj0YqKkfbCRYM0v#X9gbs#+fcR;q}^Y
z+=Qlyi=3=xXjfz0KBEn?8Z!%aq7-(YNT3mw@IkIOECpHip6~M5i60p;v$D8(dBGQ0
z25k9A$jJJwG-G6n-!oH0qK23S<B!pgHZ~TR>MK}+q(AO&kdlz3JYN=ceA)f>7t2>A
zYf9Cal9Ks8VuZ_lyN=BW)>VV;pZYQQMHCU8nwlC12Zum}kDs5Myga10w^+MaZZP`q
zd)UYq@x2+uDVXnHXvG3!(0^We!Eb*`*5L2)wuH@`A|$*|%Id|o*S`PO(rv+Xruj!*
zXM4l?j3{q=Lod|2!u$2}-%p8XB~si|*M$IK^nKRVGc(_v5uQ#9Q;h+_fd|=e<Si%2
zf}D=-s6B99F1{7k(~}q4FER(>VCB$YH5mqr+Z^6k=r`qfA1#%hd|brfvO4|&lgJjd
z9a&*Fs>i1iaD*h#WCw@^?;_i`uWlApjmjtOFK&jjay&)KOe^$QQ_^{c`ja>QNZ&qO
zdvAkkSYGA5!i~_Zn5K&N18<|bzR~OWW@BYCNkxDlq<Hb@odHPh=leYZzle7Ry7|PI
zK^xJd)`nR(r<=X@h3Dtz5qM+|w?`es#Khu$=Q}f1>)*L885kG{=_FcRSG!O@$f)-H
zFEE5vpSmY}rM&Q)GUrNv*acE2AF842iwj}s2lhysx=f@@rmqy$km)GtMAet+C~f9J
z39?q4P7iOhsYNn3wzjrbS62@Ymx^^th#BQ6<L(A@I{2L(FtM;AK7GRe_syJAyx8(g
z;>IV)*Vh*#31|y=uWg*8-IOV+mxoyX<?o$3>FeXe=Pd4$tkNZR{pZ=4#J%mv52-Eg
z&Vn<E+mE}Nsq66MWaiI$)#bgHalDlAZyb{Qa}^<pFKI;Ir17vDUESW^6my!*UBfP=
z<p~vEK1Csq9)N!NFa|*Ci6*CsC-7{4*}L+$`3SrYH-hKlZnmS|Q7pqx?sb~OfKr{v
z!N~7!lZAzE>IIM9Nz)M|gTW7HFiBBSQ3wRm(%g)VgCj31i-m(jL?;o@8%F^KgC&u7
zyzyxNFh{``t~iYX{ym^v)^OoCrKe$=8pIB25RO)=W(8y@!6=E)fTryTS6%k6+Gveq
zMO#K9h-Ik7gX&jbiKV#DEiRgn$d>75piY{S0^A+TLnYo4ZB{zEj?ByjIY)vNC1H;(
z-lKMevu(+Hl-vIJ@#EC};F6|=^1I`KRtRovY>HA^&GxYP`HD1Y8<JEKWrA6jK#8TL
zcztOH&BtdGwRUvDcdJG&Z8g4UTN=OQ4i=g$9}4hHv(5%ex3iO=xrF&z<>NQkS5T{J
zR9K042MXAe3^$z?&iRE(MQ3D3E#9>SQA_~vwrX3j)RRULLV$j%cvkvco`?3W7qOue
z1>}SWw0}G*jfFMskBAn|@+&VaEVM}ki#Sm@@SPu$2H)%#Nh``*9`5G9pE#4VZ!<-r
zj8k6b?*|nlJg9Bbk1f;jw+{|zXlMe@cSe_kk@tCNrqy7)T+O1-v9VM_@_m_)y170C
zW=|#GS*V=PN4+R&mwMKIf1@3fYn<$<YmY6-Ga<}{{Z?aO*{D8i$Rey+)aKIvR0E_{
zIrM#Vv$^7VMY)2)2wv;fP*Vk)Rou`dnz0a&k6&Jrp?&>pb{gH-l@Q)F5y8R!vZZzw
z>$e)jZAg%Ms09F(Mo^tawuiHwHjh1HYZujjIU4&}!t`baLYFEWHHVM+#lsPjjWp#*
ziEO`Hc+ZrwAX3+rWYpj&nKkqYGY0hQ5WYsr-*q=Ri;*#X_~5cPQS|fYPk_v*tKAuq
z=<p7{>)!Z48RAzWhCT}V1`%V(5t{e?0Jr%As~Xs;H_WwW=B)&vmM!<}hH`l%r6pdg
zu#_lHYh!`q@qAT0Lc)4e=tuuK$2_@GpDZs@c)Ts|h`#$tcJ@>@cciE8M?%oemj&?!
zQ}eRPhL1HAPS{hGY*Q&fwpr1)jvo!-a(64?L`(ih;xluTP9G<Z7bmy8ytzh97oD6x
zH~{UNBVF&jyZta`*{re>Aoh@mizVkXTl+T&N1q9<Ep1#Hsz5E_p_i1)->$z@f%1pw
zFf-(%tod=o5$VE#;c844zEV$Q-ae64`w}u4D~F*^BNRz&@YM!-(K<~3Ie<Md@vdua
ze<AF0^`|M=+2S|*a0%}zUTk2SI90V<gVXY>G`80&-$y!vkq*=4Lm*z8>(fnu#KYBk
zV`Jk+Qb8mRzEdfU;*(D~T>-&n6%g7$&y%%RtSJK!$oAW{RUAmXokDdTK0u`n=}}aa
z+J<yqY(B|c8XrG9q7n6j^fJ9Z>~Oi_Dw65K>)I*{MkK^5i`60@zA{>CMv3fLTU)c*
zG^BKtem|eFYHJ@{oGjD+^_49%$na!+0+P)~Sql@ppNOJsLtGjifevwInmiz3u#N!h
zOJ&XuA?R=Jw;CF5RuB#`nBvQ^yRf@Vhv^&<AWyIZALRgaKqiz-_Af(9A|pj28)^x6
zB?>|1Z7&|*<IbNb+RL6V4<94)cln|y#R{0igd%eH`6U15;4BStpL{0x$8X!dHt-@#
zb!<z??ecuXWnyfJ8K;=AFhwUS(xcMyN8JAY{;&O3DA0k`=|&B4qY?GuTcmwLMKvFC
zzp=G<vikVmSmPs5m^EOo6itaGgoY(euiOAJsLUOi<FM=~CVst{X4Eu*R7EA#t%|Bd
zSS`hPPtR_aU;$`MxZdS<7M}C2cgBe3t-Zay%+c*<AEN30iq^f?l4d_lE{DPdTb7|*
zguob;hcwrNtoEEwMQ3Gu)?pqG_b`~_?i=f4=y2}cg(yTN-I@1E=W@TEoVwo+9dqRj
z#Tc>M4pm6+U;zKv^ydCW^-=rnX}p>=MG<w#3*;I&wzZ(A@qQ(>j%kKNy?qI0(3QLE
zVyy4K*UycHhQ{v_E4NJG=*;84S=C={{Zz1`y4>(^BnCqc`nmm9(%)g$sv;%*(}i3u
zMWfe6{tt_!El+$UArIi<kpmt0q`Ixsu*7BPBd;M2%1Z=rGjO8YRA`o?r&^ka;wal&
z1pF2coYt@fOr!^PuZ|)wu&edtA%b}qZW>N54ZeK*sUin|-D2D45I~JzJAIX)H@;PY
zS(=bwKiT^AjrqtfIRz(fEB(7CzaQgs?qK~QEKOY{*D!~AzH<Zp;E$%(3Iw={UGED@
z-VJ|$c+UI2FNKvXt?#l7AXD!Ge8Ik+mDs^wjU5f3|Geg*F1{e%Y?)PGp_KQ*pj@X!
z7hr5`tbzwXM@Qdqzg}fj>^^e(gw}Z&jM&)ROr+)S>gnk*A0Z>%ta$_9*g(ZqC1qu*
z?>?#+A#2~BMl4(}G<gt-^DQmhc5H+FeU?_>2W{5335tno!mcZwmWiqc*HCq$ILanp
z=oE_<Sj^{mWj!hTZnwaiNLYXmHT-%_D|S8_br$q+T`)6a3fRCc=#EN!bG2WB0*@Da
zlUlD+1_QQ0L=pjrMQ@3j*6WI|jPj^dbXLMi)_%T*S7YX@$Wd|6ASb1P8ZbAE+9oUI
zf%JG$)EIdzhoQ{v<-E7^WO53q87UZv1{BT?ydFsLk6PG<!;K6L8B&xku)FIIjP3&i
z=jw}u(7L%)w71pBI85dm?beU|qm=W;;s(!Getc;^xmN=#9g=EgA8Nb1yL)=pD&_H*
zmA)_1&Jy)b?Y9C5C7hj~zhPrLLm(=A0j}2TaF|%zp3t~|HME0n-(Zzeg!1iS=(UTA
zab*PE9AOb#M{vo({d!0IQT3I3%Tvo)HVY;*r5QHwn8MWIRbR`3!0!HmL}b%>j921d
z4XC=)6!}r%+@@&My$P}LWrB!^CPT<6o<6$$U<>10DX&Lnik!j*{x86joKo_Yx@W^C
zo(=@`j1_|rSY1j9_1lViQ?pVSRoQf(w(}G>do=PsZDISNh`*Er7aOR%TWBu%7()^e
zQnB^M7(<nb^;>GNf8z|Klgi(J>yH|0&~dwWaU&c1yzSuz4#1SU+DKAg^4-p=%NPb+
zIi8?}_+B(g0Ll-Mf#Vud9|plfd$dO!;SMv&v>er|9Hjp4*<6ze(5L>PB}lOpU)3|X
z3zH$HamJ;7!)XbQcbTxel<g$0o?IM!bt<1T*L{@#W#4yPPofX780QyX2GF7zTI}vG
ziJyU&PN7H@J6y->rDuBurl6fS7JyofNxsbb*{)3r9IrCfR{Br=j4liJ_pPFy)y`~M
zgSEjyvwj1z{iYO>)xY$%{hjKIhg3f%0(;nvKJI-Y6eyXRmNsC|vOlmhVf8PwGVz8#
z$sqOSOTS31t0Q*8P9-x@XM0W;gO5H^VR_6H?AH7ZU23HaLvVkfxLQ=WUaBl()ayr<
z$>`U7kK7bTT6zKAx&~rAt$O4&Lim@Mr{{Tm#%!rKaGqk)+8cP+=*l(n@vEb=Hxm&Z
zJh1;H-{X#}fx`CIySrFSK=8}ZJCPtULpriaLaz+@r0|rwDu~FXmSe>o0n$=0u%QYO
z)minHh-_wI+liXTi>LaU&7#bZe`pn*Z4V#2vXve;j2@RkAG|87y8->3f+wd4Qe$R)
z77LZZV8bn+RG&5-8U}i~c>sX~_Pcl?7B2zKGWFc|i1Q{WKt==b1<2k}TDouV<!pij
zz(?^5)gOn_r%nA2V3NqzfNOVOUt!o;fm+t*xVU5mN;rSZH<an40NK(DX>1<+B%kBT
zmJCrqjLe6<U49u=ICMh|x&N@8M`mVPvz1t732~aOekF1PzH80AJeWh@c}C^E(iyti
z9py4p+3xK~9b(#cLoMdxdUd=CpRS0FKUg<^tXpb)(htclH)=Q>m8bK+%@WMcI+-}7
zOAEg4nTAi5W$@XZW=@;>`cC1wgU)9QvV%4!(}IjI&(isW<0$$4ZW9h_%mNSQ^N@(O
zGhT<GoKj=9r2e`&cNAFt3L1UIGj?i3<ei9i5=b?ZFzxPfe}B)Pt_f1nH8yUwp5Kc0
zIta(WD3Ntot}j!{Gi(T2JYMZyG+rDVo1_KPPM@8@)9lpLhEY4wzS}$$qN*u+dgBc!
z7%@F9C@I;YnjwHgNL`s0EdbX(&ovV)<Rq|NJ>PPPqN5h}k{{p?Txu%x?=Z~FywnmU
zQj<yjio>;^y#Zon39-Y|<Mi2%Z8BpoNu7wS6t=0pxyv$iMTSgm(}HYxH#-Cm=11ID
zCfhpD`vh<S5Bd@oi=XAsINL4eGW04;=j)y3!Q%euW5yl!_nUFhOfu2?6GgxRMwim*
zXZ#48<fSb4;eBG+s>;Se$<85>{Mier4W3Menv>x~<&Dlo#E;|4Et|o;ih|Oc`{NzE
zaaFX{L`f<}OMs(Dru*@7c;Hdn@<-)_;2+~S{lV=LY)b(f*=KS7&R%^z0gHRt?LqBO
zb3g~jZ5@5bkGan%Mmre<smgJE=sT=TkS_jH`6%H0XCd<bXas}E6YtTZm5T`I6xeU$
z+Q5=z%u&3<akMM#qqmLKa8iEH{qm{g;!HQC?8ycb9na*NcGt(WVQC}(rSC4OX=ZOc
z+}IWM>o-eB<uxT%Lcxaztf!03Yh-FG{#X6Y5(1`M69ZfIE4A0l5AMh9HfEITWh64T
zm{pPW%{z<R5qe#`bqfeQGMMwC>eiZ`bcvxePsV1Ifg3cnzb)0e!hTY}+`8sYz^K3Z
zP{63WdCi<$K8IVgXo%gRjo8U~_HK=blZb5m;f=-0=Aq_$&EB(C;1BByc&!x8Z@^Sb
z4x{R}x#fWIT`AF%43KF<9kGGxAtD1NaXD7{#AY`27a5?gY2z|T#gP_VcUTS-;n*Y5
zFH;a`@K=6i>0#zxxl6swwm(^AXgMe5<8Z}ZtgC>ND4zk5tqWfi2}8w)gsm&}6W}es
z(+Zs^T>Q@#9XbS~jB0u7MfXH&bBevZe4TwHg52rz*6u39sPVQnEd%T^IwlG?7JVBh
zA@RseTJ;0f)*8QICF2*-C<q^bRTy%Vy+gNl&-+T+WrUMRJ~n^KM&|<CE|9b}e7Mb)
zPU9_yZVc$Y2rXnJ&XHj_IXMAwTgq8j{8Y;GyIPG>O*BV)GGAh2@~pPV`)I~#-V0Ty
z4ZMAd=HufdBrJ^fM0RwxXLBGWPPc4N5?Gdx25=Z5+?PC#qH7Ocizz~3b>H(*$W*z>
zd^1o#Na|-rA|mup+p}KD8`MzhVt>{FnyR|qyAXW55-Lv%s`b0r`~LlVhk(oXtC8@8
zWb@P9e&vo^uYnCn$JN>yq(BVKeD9eS@(7tKcJgQ9Iy+D#a83)LpI~-6)UX`f;0Rwf
zl8+;A@;x&P$0i7Z9|$ge=Y@QILyk=i;f|9*ab7B~qY2tBrqUDByr=3M1@)%S*>)d4
zZt?eoAHvryc|Y2^Rx0H!+)d|8xVim&QO1!ed33O;16e3`K)E0a;JUh2HX&TTWoHxp
zC@syW+GsdXO`uqCc#FpbpcAKt^Qju8r?uT?TV~;0WX%j?gv3d)<&lI~?Cuzt(P{D0
z;Kki!mJ66#A1{4c0?5f+iRr&SmJ1(BHu|&Vy;_v$CAm<425mz>ynb33^s{vwV?A&~
zFmibPb^NnX_I?ASw6e_8E;#$brZ{cTDm?qLB?hU;xNe*YJB}OtsKP}WbLYJ#<JsVx
zpZ(Imq#!@vYi9(SxtmRexS)}&k2ULQeIsFaQ7>%}5tp61J%GN)W#I6(z#MQ8koCaz
znE}SDxobI2SK>x+($Nvz`m$sD`f9ii3%#>P86@X)UG5Sr7|7RE@uP2Lo!!y2*?A^8
z%@4ErrF9yz&U187xW>~L^w|F7+D?aKt};xfWHb|xS`erW*NTWl-fqE#Ipz8n@1rDz
z8C6)UWltK!Z8%Dek$vNw^Ny>rdvB0aWg`A;XbR*$3Lwh4;$UqDz>DGPAIDFCeri!_
zLHxDFo3Am<O=vSn5f>iib@p}6JaElfmA$bPhsnL5VO<A(J>w^s@;Na%<|a}$51%jO
zCblwb2sd7&tn^l7Vu}&55b5LS&SvdZOVPYIwMMxzkYk(rQB?z(0o~smUqkHiZmu5J
zEftNNoF*t>KK|SMHFs6u%c<s;solmxE%Q2t*@Z)X+BbEAS?m50Z6dnJ#Y!D8CD9CP
zgy*qOt*`Nf--?jF0o=A|oBE!m{AWJFou!@=ksaNtRhB3K2ZN%h1h{Ud)Zo@Pv*5(4
zS229EQ>d$YCeG$wv{B;FV)DvaF7z3e1_>qwlq_t~V6S<IX{0b}iJANy-DYFpVo;_e
z#(va9L?uN*uVTK=_x2%~ciYTo&%$P#Rnp9`EU2HTqrk@0+uPg!{I`FLm%UK|oghFd
zugi8Lfm-<V9<fYDLh>kVWqTWeVze5Z<}Xh-q4`mY0+|^l&~PI0M$215GBUDpgWxj?
zxhdPbwGvr%N&YgA?Pik}t`a-I%I2oQd+mufe*|5y;pLxo^@mgFXT_xaQj?a~Z{ED=
z_+h>gh`7JJEncc}GpcuN4L)ZE5G_dBPZpE5`9qxMY7c+AS<<`-mC{@4FzfJM4!-SY
z0$at46mE!>98FgoQMt6Ed>J3iTbX%R5)s?}U_Y8bGk<#l@9>3g{SL>5W!?5tRlqcu
zsnfyCX-Wig;o7XkvZIqdI=@C6g@oth?5t*!c?)W)YudzGo4pR!4)?aq+2`t<JT~5c
z_&B8l-Zn8d2}$qm)RbdL>3@287|vXa`*?Ts0DR*l#%pG_9334!UZ9S8FuJdknwp!_
zhx4N%B2rRPChx-H+#4Hzes~SS&*ZJ$j5DHG--=;?+S(xQ7mt|kNAae+bjE_~T{4k9
zNCT!#V__^^+{!Rp-p%jkX_nLKtsdpB%V|vLUS2aEuT*mic{KO-Gy^kuI-|=&>0Eg1
zp|!)#<<fIaL?H2}AsQa{<Lk%gCv{}cLON5yVo48wVxJAUIhJvlHhR__MMn*~;Tlxv
zf=N5feDydUe8Mug9TL*w6kC%0R#r5$P1X5`Wm_Okt*7be7{@ux7`hL~4(*eqA!Mdm
zENM^x_w~`{t<szXLP2sjei8D~94ox=5g{<x$~{F&*6U-wY2F_34m$no5$kLv;DO=E
zW}N%_MagnNJ^9<1pBpfp;3dJE_Ee`i9(>+hEj)n6(tu1bf=&39)8f4mTONP}pI4xM
za{5+TR)bF+RO4vD_)GieKt`EUeM$_v+*_=-kHd*SQ+_aX;ensdls3N7Q#Z@dk&bsv
z9r&m@1#BQhrRh*&n$NFsr$|-A3hS4mBPgkirkI3kTXP@f0OZg$nAupoE{<2Rev4FY
zu})NisNL9G`uy}fb9bp7RRB{s(L*XGZH6;5(eH&6SzoKClMR-lQZZdj!hZQRMH;mj
z#~!(Un?gCbeJZ1|QO0uH+^ukzdV$S;N#_mK_||VD<Ka1BbEk>Xuj?l=>GA6f)yh^w
z#MGswKt4$DXQ#u@TiN-McU2|zQ8NqaPh@d0Fv6Yo&jg4lGAOXH=V<{mck3SvR6Pd2
zb#)1BV!2m^koi19hv2<|u@a1ln^)KgRcUwaVd_FWL^L)&W#+)Ae1QiO0*!gg-jXhj
z-@Pt2o3exXT@qqzl6Fhn%MMeT@O(7JnsttO)#ENs%3<SgMCIf%m4w++lx$$&`@@!_
z=BB3m8+Zj7rzt)zF8UL?=@wc$!*yThnUUY+<>m7YM_iQnVz7+J=k84lJv|ZDvlZ>8
zVs>NQ;n;+Q@@sX2nx`kHPcbbK&f=-aM#sCW7&hHIzM#8L(6n)3Fw2`a)c(jFK=6^9
zB`w$D5nGZHlNcYz%fg=SXT4AwaOTG0;Z|<4IC5}&n6)XYMHGMvK7wSE`)Q`9t{z@)
zZufie2(#ci+uO^7laoL}mj{QDPAQkRi<`fGEHZVA^FzCkFI_WaRj6L8pl||@%t`JU
zrp3|2y@`l?2OoSgG(3`8+;8y2OPfW<q{TCpYY^iX#4GXeWVs&+FowaFA7JGvC3+QC
z7t`5#WplMa2WL~-kWcrwk<0#O%Z)-zOa)7Aek`Pfswp+kpFH`Jo^Ee=d2tb+kkIbA
ztLg538j68aZrX<1IXTHMAfQ_@J?$!Q=!v`~y!TM?cM=Sd?r1w$@N!i)YXD0h%_Cgw
z`>!Vf^pw7VbFx|f0kL7ODI}etk{w>TJ_1?-_6aw<5t1_3`42O^(yYdf9{}{;ms_=@
zutA>G!|ii2v`3CC1-z4LtwDYzgTilj_i#g|OVFaj4p$QR5B$%)zj0rFKvxh4?aww_
z;vuM7K40uz>v1+=*J~Ku>13Z&#x~gWl!l}0QtTC9eB%c24Ab&pu_oi1YM(X-0H!Ae
zFP{}HD4IL?@f{@>(E4u<8B9Ka{E$rb9Lmgu!G~WTPN@V7XYgBJFt-NIrF=UrWh7PA
z`-b^;b5XNkboUH~K4PLiYIn|B5z;6jzBL%=HbqtQv5gx#K7aO8kx14n=G^nEf0XBN
zhYvZxsF<V8bZ2i_X#?uBt^;@y;ed1ko;Y=L)f)*3wJxSWeIvt208f0svQ^LOzD`qr
z1z4%20PU5gTCDQ_DXs5!Ooe>xVV^BEEb}iNLgny*=EJEM0&Iuod>N#*O{g*Yjeipt
z*r^;mi${N^5;j<nl^B&Y^cK1|Z5J-mwN<{OFfKf!B=8r3&n-lwSvXbXnGs)W0%pnB
z5<ZAZzV=F#k@wPrB*r#l&M|)*QguZ6dF@|6cf_7^R6&gfJUZu?bHgFOhSs~PH=X2M
zE32nQsgW%81`l%Ae{(j9$zBf?_=~zNZGcpcv^0~&b_s(DR<GoK{$YMj*+Y#HJlSc=
z?M@L^Ra%)jv>t~gS3lDC+|BVZB-NcqG!gCM<kAP8!eqr0QBY9Ogdpndne87R*Ltnh
zdNJ_q0)CI7psb8T|5agOana0oGo=9Rw}&cpTpS;BMeHxswFLDWjwe4y9Fm?*>Y0fl
zyUsWV{Z_+?B%@<uxXJN(K!Z?d35vQwnf`22WdkkC^pYy#;&Paid4IGGQoSiDCJdab
zrVIWq0K7@B3;u)|WekHN5NAO8G`|nR53#5qaCNbEFyDYGYPrmwPOtoy-fq9|U6?lW
zZbcCNi0Cj|`}s!$5zoQKc5!y$Ktb-g-|=iQr$x6-Sz63}Umuc?HQ$if7$AGFwx4dM
z-|5ymxEd9Y&dkh;kB4`5d3oaieGP~rqyr0`XaskBZu&{anYG+*Igi08zeP%d62Wcn
z@3-z&s1n#^@9gY+`t%761N-|ST4gKM)mWbL_4W1W=%`<tgs`x0X9(Km(->TLaP^Iq
z&$LbR`l?I=OUcs*%#q*n*K@<qEHJjuw@n*6J#WmfVTi=IV7t+{#4c(5?eh6s@zn$~
z2X^pKa`61&{S==1rSu0}x~C#e4l*ro0n+)%nF`<$v+Szi-U85wXTDzNazExn*|;~9
zqlHTFjKR~Hn@GjKYeLrS#Kzfo3tL=!IO`~Iuf0BG@jF264q?{N$eo&3Z7R-x(P}pa
zp8ZD0sRCYr^4Z#x(Kg>JAlhQvpKhR$8OfG>K@UM|CdxjC3kJ)H#B9$mI+#|eBDf_v
z?q0ktrn+8pD2#<SO$)xV#$tBY9hL-i_6!}YZktt8*fynB)Oe0efMSsfk2WPZqdc1Q
z_Mz&(=Vhk?2L~gd4UsDqDxdzuW$Ye&KyU@Gtm5b*YXy&wU==TZKxekm#YiK1t`4It
z!3!1G8usULG=h0)ZO<>=zV)j5Uc$4zA|D5D^9EtikyDKh%Nc>FS@&G;pszrs)zEw1
z023?IRWD23c`I4U2%xW{ZtGMr+RW{Aud<CMpWJE%sbo>^H>PDDsC$Z+my$Jakv;He
z!qlGh!>_e^-CCNmDcq_32YE=v&2DyQS+)(EsO0%ZI6r2(aJ!U~z%SsBHj~lNe2+rz
zitQ~!I{HhVx+dc8qpWtn(R2=0OLzSqF~^J+KLYt2#ukSbSNTLd2!_+t;@0bW-^!WN
zui^91EmMn{1Ock?rQZ&1q5G4gRqIvG5aldFG`MZn-1KU}OmBtr<MCrjuc?;!p#>WJ
z+Sx1ig^j7GxHfx3B>jw-NU@GbNY55CKj2BNSi!n*c;~LKE@N~v`mfaOKkkvdxS3Kb
z@nOs%4amXE^?n;)a0?SF`r9U5#CY+A7E`>3n^!Xv>jy-;B_s(pb+KetQ(`dd?ChMw
z9Qo;!af{c``Abib%IimA85u@%n`{@#Q~A}^W7XA~!DgHQ>XK}slNVA?zJ2?KVz#5A
zaJsoV?&~j61UPSYmPUaUNSSO{=qYE%kC1f`qkN)@zxz$GSup5iVm$cnME?EzAF)YY
zknMc2cd_eNSZ0Au$8h9)yQP1C@g0gji^8W6K$J}{`<)GmTs){gEH=38tXLc#9)>ad
z_+L|txbIk<20L|kqblWi=w`;6?x*LwUQTlabP_y_XVs6AQ<D!DGr0EJSAheY1!h6M
z$i>9Zv3MPZSDUpEI^=3Ww#3a!Qdn33cw0rC+_2VXb&!Jh;iv^Ul#TG+nq*Uca5*Uj
z)h++QH#h5&M0g&NE|#TP|1hmEYjm~H*PrPol61rs_S*mM+uKo5;j%rPg{to&&d)tF
zj?m~|Lm!UdQTeb_?MaJVc~nXLOU6sLqleg<;0;+RC>2nwgiId;wukv5PhPhQ4wQl%
z%W0U)f=Ojoc5jz_+_p*t#;*?x>g3fkgM;RzS@ZUxNTZ+AHLNKCmuHD^f(0ZyvSwSH
z1`HBe@+->@(qC}t{6hH|aZMJRnVIPWMqFIbpo$rLlO?a7qO|9z_;`eRC!kzjhYq8R
zw4`mT$)nTDv!IWM=gudi1deo^F3Zb7MwXNqkHUV8@0Oc&EY&%r*?_@^J`4$J6DJoZ
z{ZQBuDj*X{21o|pXrXwFEl`0?5V9M2xI0%DI$3E0T@FUko;j#1UF<$<@P(R5N8$T>
z96GqbvWya!Bttl8L=PA00=urF>I@g_GWTb|ZH%IxDs5ky1y6ny6xf(UQxnzG_YM!i
zz=LAbiZm!Gm7o*(L`SfR$EuJTiEP5UiVN`;YR$rmz>L#a2p(aOUjcBXW_!2cQ%}oD
zRd3Y{zuV3ZUTSn(M^T<StU8Ib;wWWE=``G$$H3L~xHq0kOG_);XYKLc{j~Yb8K<Dw
zKk*<Rt{O?Xhr8NJZL187*N+w(%}{rOgzB7{bB7|76jP1FCnS`9Ca@YBlZ-dax|uS#
zFDZDK%!Wd7^^zII4;!)8MDY|`rJ7)cruQ9dAllxe)m3iC8Lo(Ts|N>J+1Z?*QF=|K
zyi?c_a(}MQVWsbF#Jf?3+=!AEhu$9isXp{Zcj5aC^@oOl`7!Kzbq$LL#zYl>IVA>)
zlB%;EUEAnSMyarXdnCK&3_zc_!t9Epi`Kit#TJycA@kLP?NQ<z(D_cX0jC6V)&GdE
z^0k1hJVHT1VQ0Q!IbW43lCTYd5(l)MO&OuGbp|eq1G<LHNy^1jzj_pQbBe{Op6xt}
zK<OEvXXp?&O4a_N@%~}8Vb+v!RBv-cya{nj&n$oHCt%1VSPG|_RG*6y=REqlE`-_w
zqh^g*H*r2^4`WWp#-N`6fd&YvtgK`L`>Y|BgVj}4QSj7xxjoR&PXrM0?qtz>8O>)c
zhN-Ko%efc(-|QGWPo<{n?--n1n_}mgg=CnFoVLQh@^dqv`Tp*F)XIFCLmy&6pEM`Y
zV^BW13w*-{$#=RHZ*8RuJYKq*DaRqqHg0qNyYH1I1+04d4DUV=E*5WnNL8bES1rQH
z8H_5?X%<aD$1Tl~KE!>4t<BcZ)I0b!SrR9BAzN&#zbeTLgugPes>Qh^i5F6kXrmaU
z_=`^i^%Cw8iG!`_pp2xF_sk@?5g8d7*LlzcS5H)x@`HxP*CeGp-RzizcAI`Ad(cA@
zS^L%LSwX%k=bJa@r>7tG9xk=~eZV#bUPnvfS_NwJjdtBXLc>vHe~NHL>$i-g^`j%E
zG-zHzn`L9Vf3sY+s;WuT>Gs!Oa<@LW_Dag-W-#t*xkw9?Xj6st?DDj~ukXqmc33$w
zQf|^b!QwXFmuM-hK_U#kz3TSm0Z~8E@RbT#*ViD~hTW=g=zZ@NPFVLfW%$aMdx3h`
zrhdDx#^FIIosT0fN+cB*|M!#Uf&43UYzp^`Is7J$SVEOc<*Wc2pE_f63erks^7>f%
zEII+`n9(`RdVTUsqH`Y^IEK~A-Wh$|eqM)5`2A+5Z)61kIH@v`4mY~1^>g^>W$Ie!
zBJPI-kVpD{_#6)z={euQHlIGt@+(tgNLfH$-OmTj*TZ31W@Wd{K#0H|ZyN3y9#vI}
z(A4o-Y|V3a!_15%wnts#>agR>%Qatl*c(w1z=-VtiIR11O5Bu1St7OgT!{^gLR~%h
zV=r%0(=18}MyU#FY5Ao-SV{`RyT-y|l*dQjz&VfF5beHa65#7|-aX~L%Y*q%GLoy?
z>uDE&ITty)Q9X$eO4C?px+HOcn#k~&@Z$7bVCQ0U?)2_1tXp0nCB<GuP7V{hcCf!+
zN4O|cn>BCjr~El8Hcr}$i;JmcE4q1mLz}&HNLZIxOVyOu^$*v2Vl6Bz(kcpNf3C&W
zLemx(7d=I?l9IIU?XB`Pi?H#@-*;!V|EEgDBqK|6OB6ZhC`u2!+8Qq+8)KY{GQf_+
z#%V>&@xhPfUcLo9856J?A5djV1Y<r~5ZGA*`3(zIXT;L))TNV|Wuo|ugo=s^<6c`R
zl-jI~kO)p9TO}QJxw$!KAsv;RoE&g`DEbCkk4xaV^4QS<H$se=Ys8jM;`UtY)vH$(
zt$Zb|t*u54{xTZJr>CW6ruikfa=%6w7u#$Q-%&@#Sy=oLhm%zJ_~amz%#rmjAfSsu
zv{+lIO$x>pC&T(xF%JdYzb-Y=KN4wc2kOsMl<1|T+W%1s+1Xv4ZFibCX?b8_Vd0L5
zW{}Z1zdUc=2HUnRy=B^c{8!*gnbl_cFoYU7aJ9MW{?x8SF?4+BUj|$TjBHgc0Vq_F
zz-vnY%+N87@<*A{SEams-g`AQ*0W$2>_`ihf~EVr05l;{QGPC|BJI7sy<*KG{9$ae
zw%^^0Z92BhP&af~hCmEz+Fq_EiXlhk5!?ur0-H63Cu;d4YM7F#T`XW>@j^<9K?eS4
z@i3;`RGuq8j?()|=AUvf9hVaqpcJc=wU}knwg5;bNjVm@007=T!3>u&OC*JJ0UGi8
zP&55hw$|3hc3712GzC$g2DuCz*?;X1g<AVC6z0<S_J3MPm4(O2Sq_|#XssMbj~&1K
z_k%|-@X6K8g+uQ<)Ke9#$9@yxgOPg$*S@~_Z?PMoxrtjqtbg-?P=Mi2_atKlb0XQb
zG8t$50dMaq8w(?WqsF%U;PT6a_Q&7(*yH6q)~>$hbNrJg_bu+Dw9S+}zit<Tw>yGt
z6jy;%>UZa-7k-;b245FN_<*Enlz{&vzGI~$;{OSqKk+gCe<Erc;q-|K>;E*>pB5NY
zP$J)qX8c?)Sy@?ClTWWv#OdUO?7zcJ^L;WJjy|lP7Vv95{^_y#M4EB_r|Ta-VgKLR
zv7#5+ZMR?kwS7BJ&;Px<kDtl`U*^dUGXMERd_wvE_AG9M6yScFla=(hr@5nxkM#f~
zfj@Ro^C|!9)`l~^!yfvC^X#Y#no44V^Iq70x8x3K!N><(ynR|ci2e8z1AcF~stJet
z_~L^o(_gFWquC~GL_c;M^n@m2m<60vF-55#&D9k_|9fwJS=*OQ=uaNWI&=coecxip
z61{zF$q*vLO$hiS(@F5T5Q0q@hbH4o2grYPdHr^j@;y2>c}?g()iU!M03#0a=Clo;
zMCjbf`X#dn;P5R=+-^ud4_Cgs3A8bP?<HDq##BgaPjxL?i~<dwV>0^mx6*FW0zh=U
z#z}^@)<1#se4mpWR%aPx9?`-3f`e<huzx9;vT=JIMpk~kFhnz+lLL4oSx?egTUc1h
zA#wC3kf+z=vH5j}I%QlWD14s)un-U#{2XBwka=Wn79oex@MnK&KJB4o_Keb8o0&v8
zk$nip8n`_xOB5hxXa2pnFm5hrpz=PC>8E1R_58rbx?k)4EC%-@?c8m~{R+#0KZ&N&
zziA`78eI=8(>t%Gqa<U(2Dn4i7rG9VNoR3PDYI=`?e2raM{wdW0T|&EJ{qi?pJYrE
zWs<)!CSVFEg;FsYn8GN<ePeJsp*_Y$xWI!yXL4Xt44nbvtQRASp!psCe{&d`%D7Oa
zdgr;P%w5<Ib-r$2UI2%Z5?`4JsFBltmvlM^lsW`7d+wsszk1bEyN@lya=ds#0)TlQ
zq0^751&i2^<<ObYeaeNk2l`K16BxEwY(bLnxho_uFVoLe=|E$AfS4kj>ZPC)?RvK@
z<!&Q&{o~pCq!QB%ZFs}`Pl4ytTijCDI9p=1ELAJ*%nee@_jT;ch&q4y?(Y&T0|yso
zh=9PiXX`Zx6zzL5mgt|~!z}6VxKgZ@?$_{5x1>zCi_^X?_na#Pj*U1-3-fMmd7Ae8
z?FZTw(C}AYo$Om}@nf<&KX)IW%^u7m9dq`Q`NySR?u#T+h9F1P%&aU2J2LPl#t~c~
zSG7=?7^0|d;@#2e)Di}B{lS$&O7c=7@byIkkMs2dqKdcI`6@_y61ZCp_-oUL0;)*z
z5ykP>IA~O4LARqU>s|z4<BY|d3HJ<c{^#NkGQSp4(A%4~Pm_{RjMru;adzg{@5EW3
ztpnA=P6$lGJmD862dCN1wJ)mWRzSuh^RXZ02?3(6V}smM3OClPHS>7&4#yvIwmLcS
zj?Q{I6~_UkB6-5s&{$5$Z6rZ3&D)vdX_Ry=`);!*^8A5j6z2*hSPVKu%#7m5!<LXM
zW*JOMbBQHa2{r~*Wm0iAJ@qSWhN>%Zr87-DU-+3AT;nw2xIc?yCNvTE)yQ#2lFq+d
zVI3p!8GiDCWL;SL5oUX2T-*mbunt9!k5+l<%oKBPhz-@|@A9$dCfStJ3_ByRBFTy&
z)7#rG)n(z4J|N}3y-b2gdfjbDj-S6rG|h#>e@6UX;jE+v#Q4MIEyae*7Km0x5GiEI
zm{Yc{g5-ve-q>)}wiZz({0$=;&mC^iIs;E=0Hxxt1Ju<4LmdPvX9p08?78h(ODrX3
zk+-85pBpXHnE@u@$*H4LFZ0u%bCB5rwC~3b8^&M%LDO$%>8JLX%zp_;I*Jtx3~;hw
z;BYZMKEafo$;F~m@7#>%3^F}y!;QDhCT`)oV#n)#NK5Uw%9>vC874WTMfSJc>LNj6
zw@Kab1)M+&+;h6y?p8)+qzZ~8ZwC%sc@M?Z`W?ev5(iCETlvnvB6)GJ<NSiJzcyu4
zn-<t3qFH2~a$W6yVlVk-Bbikx`&6XMSq}Hk^#gE_YPnnqOb}v+=wphSW|!od_SngY
z*W7G(6EELazY-nt8I4Y&yi5H8{0LDT*K5_XMeJ(P`u%i4Equ}h@;lvN>Yi|(5BoJ3
zbUv$SX=;pUasGU`Zzqz$0KYb_QIhzxoe5-7zhySf&eTb}8>eb3*8G1@HP!AWBZM(v
z5iQqxGXu|^j~`lVZcuN(*?OWutoKS76==1?-w~f3c7iO|4AX8{t5dj`o$``&?77&<
z_fDOr>Ae9=XCl$3Tv}KCVy{97iQDrZ8DO{a!@Jc5AjszHi$@zBBJl1zqoqE)se}8A
zx}z-&K+s($8$63xGP{pTknb+d#anX3jw56UFUrN2{$?90yN><o?&dd?h@eE@%{i^i
zDDsf<IaO`IYP<imF6J<yc(e5mlNQ|p|Is+i6jb@V)8T9>?smK9sC{1W0u&UK-Oe9;
zN;pqJ!M%X-tvWjx-aypZs-EqDIE$4mTS5%{x(~5&D3dJ{wP|^uSSMNm`bF_ZKzCu~
z!NP7nfd(3VNW3Rjd-PLv7cf^ye`DHutZe90IX|<i6)fIy-&cKM3C;jI&iZM{*jgt@
zM_!(`HNLvm@BJb1)uKB3wWQoXFhM!$<o1jU5Z@C3MpvbAPPyOhKpF&P&NcCdbgouZ
zJbZ|A*$u_;*t+vK&?*ikju!T8CdRv4-0I)bWChUoFRsn2rg2}tWN2R7P;~4FP&OLd
zSOEV1<?j1C8iX^@C1ZV8&i<aW@X<MJzQy7^^_k?|DPqcrThtVr^^n}O(pE1WyT!{o
zOKVZ#AQ*m`Z}+A3=Fjl-JCzsDu<HYNHoNU8Gj9)%v-_R4=V6aJ!+=rad;r&A#21Tk
zLK{0*=fzcsslzAx;t&RQhqHIOX|CCI%hse6UxVK`-0@08v33*DD*iv(-ZCn#VA~en
zjRyh*C%7cR-Q9u&cXxujJ2Xyk65QS0-4fgq+$C6WcYB@W+#~m$fA9Oo=pLg-@4ahR
zt+Kgh&00mnOhzYjp7s)8zQKQH3x9d`VC|?Hi<YvnS5=fPQ}y`A4mK$TolyE@VQ2W!
z1Ijnba5FxzByi&8Uvb))e&8et><8$5mZa&hLn>pF3F%}5hK#?O;Jy?qMZiMx3(yVw
zwHIC{(BhIl=ddO<|Lo%uzBrk0?D>mDW_ZWL>*3DSh`Y6vu8_0+aSfxhW0}WxICoJh
z^M3nMz^-R1B&4`dx7^VD)<f`f{*P36SR%m`F;s~OR*!RuuO?!^#2L+m5=f(<{_HYp
zD(5Qw3k}Xz24jLc%!`6kFd`)@F6cvccTtT*$#)W{wUPooYa!L<qk6-1k>0;3%aQdF
z3lK3+nBW_rtEzu@-$s?^{GL$vjBO*Mc*al18>DYE5efh78X9I>2DMX|eqlYyGDs(!
z+ibrIg{i=(Tj7Qe)(T<ZPxgOUmgBuyAJ_8lxG{t@P7Yzmr|ri4UET&tyPBqAN4bMV
z3q+lIFUgs-qy4Du{pPN@jSv#Mm;qH2XI-8RGuOfZIn)>0wR>yO>D#k06(h}9g))g;
zjZh;xhv>;ll!dKi=WS?aEV(6AyT)U5oYqkzr6Re?O(!MXlsR%UpCV(1NJ1eBz?08|
zL(=Yk_CZB{Bgl4UZ3cb;%CNc=&R4DG_<#1F<(Wf@44%qc)`6GXeZ}(5+(?wz>3d*u
z0jzihK)^UB&4zk1*&6%_Z}I_u_0xFrWSf0)wvQTcWvk6~6`#R(l=^U7qb9T7p@5<L
z^z>*AFH7Whi3VV-=uAu#X7WFE|4=+Fop;pJsJ4A*m6~bum^A&2D=oKRk7|D2m3OO@
zG#f(C90Z@K^(eph1uwv3ZPLC=CEz)C=z;B=-M8W6hwY9)Hi`-O45KNw!X?Lvb5EZY
zjQfGQk+085!c(5U`=<wup3ldNPvc|C$B*T~5ww%dK3AFa(O#Z6iCfV9s7`re^+#zG
zC=@8(W<y~Ur<#{kMasH_Mr0xtPmQ;FKv1F*lp#@**Dv`+{&u8Er82pzuTNOuvwbx$
zp2!hidot>VFXaDd9s#V6O5mrPb^NT0W{v_%vK8>6w;zU225zkKBx3#!aMRZYT4lP*
zNn1QY26x+fp*G&X;&lK1**QQmF<JE`!3EJ099v%3u<8?}3#@n<2WmZ_H!T;2x^1+-
zJ&tzfh~T;d0joEOtHfw=SJ(HQZ?>S-6XG)K?<cRtVQ%|6y}sa|LHhUjbMnK(nD^Jl
zh}&CVRQ@KKQp2?K)Ben3%~_lCgr=t4%;U$Sjr7)F=os3Jmi==7QSpq0dgU6MvE1$V
zv>0VKod(BQ!{Hpqp8C5=#o+=zi<k{->9A5Ew#!~0dwTfy1$?Z9kY8&Q+;+OIJe?L#
zjouT5wzlC0w<A|NN=A`do)5kDt}Wo5y&UZS`Yz~A!a;>wpz_a*Z}8mONoU%#3I;n@
zN2~%tyCuB~YKL@bbB&MP8OBN(Q3zv_O@msIf2Mv_BQkPwh_|Eeka@hO_RjRyWt3s`
zT4sUsIslp`Dh>8~wUIbrI{x%B%c!41I(8L*lgEB#SYs&b;h{`CxWkyG8v)kmiFt+3
zXXHabrLdz$^L;c<hh#cGr`>_)_!d<_jTD>C;}Y#;S%OqOLAi+G(n&9k!jO;=1C7?b
zFh*Rx?={Dw$#W?aHKu^A&cvh)i_Ndn;`s5$=(FDA)&zo4HD0DO*r{ukF1Uc~ZG=b6
zcqULi+O10GVLlin!QO-OtEh{g9C%N?2EPf~iQqt=H?6nQ{FN_QEGi9HgY{vj#6C_Z
z#Tus+NH6=5tFE&{LQ`;dA=xGR!=Jg~mi2_YDX$>-t14=*mojKE*m)RNl_<Z9*TGuS
z^|0^j2}0U$oc=r5ibz9tI4LD|BcE88Hxa<#f|bcx5gQ#TqI+U+UN55xwm|_X>Q30L
z)N62N;4ghC=#lb*FE*jg`@%j56ks8Pi}lQt3ykS!xHLm3!R&}PP;ex}p$9`<8Ho6>
z;R=E;2-tEmmVLX_S7W~t%4Ey4^2U0Edi`2I|J1usc7Aww*<PtKcHMbowU_W`j5KGJ
z@x`!v_>6@X8%tBj>&$fzh76M9)xa3x<Ko3BSY`G)4?5idIYv1Vi_Won><9NZguDer
zUVWPIyzM#9&VZn_o6_T7D{m)`o)&6bpSgehBhrnzv?4KaW&vF{k5D3UhihQneyB)4
zmt96!DAlp=trdQvV<!LkP%l}>`tB+rjF8zL87t^hp81i20#1#_pgC9nI1yiLdbVX0
zogBn)F|D|h*&-XfG1V(eU|X@9D=(k-DYoN^P4KU4mXw}vJKgVno1_azZ6V00IQ%=z
zrh~f@&YxY+n>JtTY*#yqhXFuHFa}C83sZtcc=81En4X}Y%^58PEIo6fH1(%$j+h&z
z;AxNsa#)`J_|<_+A7!42fZq-##093Cys(9O_l;<Q>n{;Qr~~@zQ|G<TrU5puME|<V
z61llFz~9GN$srE=e|O*i<>50Y82=#&M$glm{>cTr<Kby-YfIa?EujDNG6=SQ_gOL@
zvfGszBF$>nmWBRDu8wRJi<PtG843Dd_zOmB{mv-l-lIMU(L`oYwEx?5kT5{!LUN$L
z@suEW1^w}U5J*Q6kbys!5zNKR{suD=hM4AmQcgkpC*2Y89r$n5#6NeB|9_TbO#wYX
z>WUlC4dEHxGHm~n6L}gjlKGPPkaDuC`bSWP$>>lCAHDP^iHna{n9clM-o^}NYG-HX
zcnLg%Clp9@r*8^V_Y}X&t+ejmQC3!_@Hq!kXq0Sof#2pa*W?|?e;FKHC;}-e!31SL
z{vR8OKnfY76(almI}k(OAe&^!ztb@PrK}PndNA)ooAQ3(RBOMcV0G3(sP0yIclvpF
zp{4kZv=UCxpQ>R0o`vK0smpI459ibSx3Lp6L4${%pzEM2F#_Fu8-rhh!otE*TB!fj
z8-z}@&__g0fy$$w9)8^y@A16J{tkNSts>XgBB@Im9qivNF{WTP3nWN7>@HDsE_VbF
zp=*#1vpG}!smT=G05||oHfEyD!hs@&`TKY4zX@MplKg)3m2e8y|68Tn90K6s(x-9e
zC0(z6C7Y!r72LggRimvH+*?(BYfLarJr+hi%pXE4hUYsSAuytspV2AF_uQX%EATEk
zfu_e-I_25ubNfwH;ZhS5U4DUfokmZ~x|=6k`#d#^u@WlLjl=Y<#cIW}OAocl^z{CU
z;UCj%UcG6JPmataf?wOP<$!kc^dNw2ivr^lDnf`b!EF^ZUk9G>`uepYkl;2Hr<z=|
z50eZ<gR#$aVrhQY)?rJG_X=hq8B~c$Av8?tyo!l3VioODscU<;YMOUqGqgjv;s=s*
zkQx9m6NBN^tmc_rX6wJk*?#ajGA(o-m)TRUw-WsXyS35G+x3EZ_FT}|1w4WgqsgX;
z)&-IrSlVxT{L@b*CR_8N!6*cJ*2%Ru?9UVP_3#d$<y`0<-X3nK8^>ETpmH2HdsLRM
zN`Ovu2MR%F$qD*ie8^i%t3W$Csfc&+QEb|HAG2fWv**eHD5+EOu@6H94zN0nhi)Bz
zD71%Sa5`uGh8p$R6Hcv^t#ojuiR(5+fs1*ebqGh>d}!sXCv!dk*Ra|8GAd+bUK!n#
zf_I^HbOCy{YrpRr)j0BDA@|k+cZ3eQg4X!Bn=cv2JEfbq3ApE+Ak?ebq6YiU=3Tn=
ztP(UCYN>ugJ69{-$PL)Y)d6Yq_*!fz5Dr69xv-kAZY};HO_$*NbTgBiKkH^(lw&2m
zs=pHx)#&j2cLYLql_~`6vH5%|ERKl<{of3jn`P#T(avYuAbx;V1Oa;1QupOBI>ZN)
zdwx%ySod`NS)~aMe1=e*w)UspZC2_2yFIYfmHO(MDe^TnU4@6xk6KZCjWxiqOGZP*
z<w9V%Z(?_|Ugi5l_JnnKl(Z+wQLgL#xBIjEr8G;v;&Q1{4#FWYAEXMiy`#?5YdDM&
z-;5ZUJt&YT9E=yKUw_}NoJuRxD|77Bm8wASM420yT_B&_0C?5wu|<eNnN{d&EJLX_
z$9u9?zT9)8M(R0G&Muj~BR(E=#>0B<n(7d>f9&M`#`T%%PjQ>emnLWcHhiL=>(srv
zf;#P!C~U_tzZM8qCtCvaq6f=$RQO+6QT%k1_AFCEC~C9&;kQwG=?>8)br9#r-a*a8
zufh`0VQ17!GM;L3f%~;u@!WjIGw|1aOTt0CM9O`@hpvLU9vg3~Ix)CtE<#>y(KmpF
zj;Jxa4kZjD`IJr^8QP7O4zHcyp$dzRbFpf;ybvQSqO_<I-%4o^>P^zCtv4MWh;PSy
z?2F-hxI?a9n%h^)u-<U9)Bnl8xkcj-pi2kk$Cz4xSTAJTPf#?ycA{PrHfj5n`Z4ye
zxpqIMbvs>cW5gNLHl0b?gXPT2)*qkZx#y+ikpOu{`Juc7$j)`Z>q-lc1mwm1(nafQ
zy5E5EEqf+B&|EguuW8?4(X9pRA^9G=ip$Q!#jDc&9&w4Iz*VV8{2Ez|1Ttp-?&Zy2
zm&BmKc(U&VF!Y;z71w-QWLu|O;sQMR`C?kDj6{qy^);ZiIx+c|)14#x2TmV(n$I&C
zR*fkVUaq|#)88z;n#`A7b4HboQW@A(eByqe-#%0IgIST{g!@O9@hGuO$<ol$4Sr5#
znf{tYv~tIN3+r;>qKhwM<Ugik*Cd_|QGLsX4Bh4jd4bXSTcLSC5|bY}`i0=p0W68Y
zqnDV6m^GCIP}K_=SL`Cypxlz0$pkzR?Vtb=*?P_=At8Wy%5-{<b@*f~mJo2$=0;14
zuko3W-Qd|fDH<J%jRmH5cbKPkMu-ib{YvU%?y7n%aBD0hZyx(5)y6W+%13>AZw=wJ
z_tVh+Q8h9Suw8|HukeTanlhdb$GnWrotx_jbkhr7zNk+0hi0<HWzFi3@*VR^(Eo)!
zt9n#sTwm6X>EH70W4`vCnd525m(slN<$A}@=t?_Qv=^;~-j#k6<ezY`ZQJ-Q4Z+4~
zpP;$v=fRaIyW~ciAnx<mSA{?QxA9amiCbuQhR>mM;7|}9s@MQttXYQ&CYQnNVcN9`
z{;Oh6;!ra1c@@XnX?47Q&@P6DcinAW8iTi~LsmKg4fe-&!_`G!{xw05;vVPk-&Zb|
z|Aa;mmiS`uUZL!IsMS$QWHYR#BAz}G3c<)dm6Sa(9oGy~8g3jwh!3j}Q(#z-t({3y
z>U_rUl4B7A>=H;s$bLW@<k-QYtM<%E3B5C=7s^5Z1}<rAr_OI0k#Nf@rvwHJ-;{+W
zO#ftTqK`Ght-}rqM}N_kWI+M?=<rVFV}xMQdnZFtRi`30+F4`pOvFHSj((x_=%JD9
zw>vGY0IN^m5MWOHjbZld)7}o4yn%hWK^NAYBkv==44pQJ2j5uq+=}#PzJ~j^`C5h%
zVAOG1I|!W9N>G+nL${JqbY*;3+RVGr=m<V5d>tS|st5)(Ehy>(R3xe^%$5w6lQJDM
z_Ey`5vJILFgyzV@&hAT1I&&@7qwCt!u>0sKpKBuY9qE*Gtk`?4qyDzdW+YZG;5&>G
zxCu|>fDVamC2dcC#C3K{81H+tnx`Yeu{vT-x{qT$*5TzR>uo~@4ROt{vCh`Pi{bjU
zf$Xy_8+VU?^89VW`T{dcYpr<Ko;uX}y^!3p`X6X8LCaPN67-CM&$8V!{Eb`!Uri{d
z%mQFbH}pRb1}+UAMO^=5ok<uE_Fv#JWeb2z{1_RtW*?^MVq12!IxosuEc9n0=RW^=
zax=j!3L0F$C_DEuvr}?e{loO&1MwFd3-8h0alP1&$EA1Gi=R1~#s1u!w_-C7*{kZH
znnY8jK33}0)P}81lvRF*Wm%o00wY!6)C6S*tCFdz2S<-eSsI}lr!J2|kE*Rv57Q@S
zV;YNTMTg@z+-kqD=%Wa9#jAd@;t>Xc&sBu2*`a9nwgd@FnF}-D#ZM<W=hkn!pQ~2r
zKH@c!{2j$_^B^%DeA?AIv~5meBndxh7{2PfHP*{4B9Yi!s|#0qb5)}K<j~tKo4!yI
z0n78u-Eyj~k$k7<mPD~RwpQgft&1wM8|Hg{|2{-8RgF(rMXzFMQ8W9|+;lAVi<$^l
zuZ@o%pe{3Z71_`JtfuOuyUacJp~OL}TE)8~@6BZd5xS_EVElE@Qxz>~hbIqko_NSc
zr%u0<DbV54AAd+8#jH5l!-wpE+KKm>dXTr>XYsb2f_0C@yqs@HIRi8;IlQf>QNO;~
zF`0r8eRjB)!&Go!HGMi9@43ihu6EGm@@sQiUesPgcTA4Q(Xzs*zKxx*(@gStmG$vN
z!go=F!$Qi2+lOf-K&OF@=F&;?o+No`ep2_)CSTFdcB7>1de4K8PaxrU<B%C294z*u
zTFRjTG9-6ZDgkE?dcgUv9;5Ajx(~9T#a(@=E(mgEA+Q!cAU8z%(36qVV?X;GAo==&
z+=Z+DHB&tUfR9$fZT&6tt<WRIwKEZ3=carGmEGaf`3LJ()Z_(@X6x(vdj=@nGeJ!7
z>Es4kYp}8uX}v=?v-9~`zo%omSg!zMy{&@wb0vyB8%6!*D>eS2XK1DSgx+4?dY<8y
zIvSKboyemVUx!thw;kJpsm{FC-)!^gJmrq|d0C%Ip_@D71E(~_Jy&p7dv#|b7e52+
z@%LAmOV<`oBSX;UY*T;e_rN8?Ml*l}>NGt7W3mQ#quYi(RUq@T*An<VQPJy)g5?)j
zx>cKdLu55}K*>+qKG*hhgRm|4J<%1px<{K@IMG@rvVv}DTH|N8dM}qcHPl#N(gm8|
z^Srze@ZE3Y5w}`ah0^{iT#E0NipP57E0;U+A8ZtCTd_6_uy{FhX==j5ELQE#H@}>h
z4<ssy*2Kl#QlSsog&VP%ekZ*RJxN%U%jego&#*6MI|xti<q6It+&x`&))}oQ|I?=$
zC(1q1)K^vMwhy0e5!GDY)E)#;*-NSFru4)Uj?<>{44}TM@5&LNA0}A`d#&`hB@`<5
zKmiA$Rhv|38bD9vE4nq(_$qLrvSEsw&EEH!jY$3RsAD!;J-_?D3me4CUme{D2kIh<
zE4zh0E1BsAPR56MeNbb}FOY61Z6mr5*!?fW+6jtt`Pv@K@Y~<E$D=!nvCaO&ypoMJ
zz;{I|W}heVs0>vY7$<u}%nlIbx5)#P=Fod0AP!abs6w7QU?=|}GF6hUC98p49o;H$
z@MSP_^Tc!(U8#|=yWkL>&BrM<RYto!aq3lXos^?V>sEoJnd;zpoP&`QVrD0aDoN1c
z&amrZ7#S08okM=DkfnR7Re9uhO6R&{XY$66q8|f{Hy8eAFqpSlHWG9v;U{z2+U>as
z)mMG*_3^~E5NU(lUxwndR--wcfJ>c-dx7iTj}71S2Ho}<1a{bZL@615kcf-q*{3X3
z1FURC;s$kOk%r~zJlWgbLj3_*E+3<kyK8(oJuc6jdiHy)o$%Lo<y^&HovI8?#h$no
zq5X;=3bP93@U>n1EdIooA}&Kg!yNfhRricOsTv#}QUw`GUt`iej^cdf1w|}=OCuqY
zklFH9C_rBxzmx8TkGGaP^smJ}*I}PdA?pVKtt;<oUGIqw^@ff{e3$oQ=<ViR<R>y0
zGWD}+-Ha&#!$G|ou%iaoQFg6(BYXNn27mUfc&gm+?S$()_yt<iN}qOqS<W5uQRKbl
z7hz$w>ovO7$A<#Mb|me}>)d8$wn|y;c?JFABTqE*ItB-vBsHmRH_Ogy{9ZZn#)0<G
zt+ex>@nRgxb)IEo@GQ%6BR!@y#es8QX%C}5gwlE|)8zp=s!N`YWrK|r7Os;d7hmy^
z|EYpqT!pg5_NoMzorK^4tf>A(z}(>jC(Q{MQ-1iZwR-oIPCSSk_6sQ^KbdBjzbpWD
zYG#cnZ$g?jfssmx9UXRn>qUGQVk(@U^F@eNMqQ0*AVDHR5YT%YU%dMKI2nNk#3>fN
z{zO8C-t$@vz3&C^ACo@)R9xI=KTYb`PiAz8crU(y3Xs*_`iBWcy;nqhAKIhwK@_{5
z-G9URW4$PgYW@Rju3mlpf$?$^T{8U_C(J-)v7ab)5Z%kK@7Qhg%o-bO*hiPT?L|eu
zR1=eakD%Ye5);*+m74OQAwFmLqL{IYEonS}E?DPwsa%dm@L|3#LEc%nb$1IkVHbin
zP0P5RCnMIkF#$W3FQ1_zR6u24m<#p81_4Daj^Px=4XgMWU2;y{{`41xj#@#oHitIL
z5mz-fWeKd6Z)+_|QySMs#r{$|OR#wSKvTXFSiZhhqkNxAec4mf5=LCHp*g`B{$e@u
z_VE6qTGp@Bo^I}VI7JQfke527F-*hqcj^t$xxpV`l!#`U7e`vEH7D8<oPV;}3jKi#
z4oFLTpf2nE;@$RzPf)9G3JYxguX4K1i=KN4|K5<80}>3nO0#Qpdh+vV|MizsVufb~
ziUj#g{@gM_u+;KTjFCd{FN|Sx3V_Sq_{wkJ*0a%0p{iL_ThSB;7wBEH;wR+>Y!%o4
zbe1TIZTK5yu=#Om2n3{hoOC6YjzL4bXlq0W>7aNSG*r1~df0%mZS0tmfS1%U3W;vC
zxz>pMkzTGG%axaEKk`o|gpcqbLz4XG%=>wBo#z*apoq++F-*6PGp%#~TN*oe0JQ?p
z7K-m5b<D>io!KdD&iUr=VCTLzL_JxS(Gz(AvJ+i9D@o&Nl&sPWEGbQubjUpc0J<h#
zLCYuYQ#v(yKJtn>k0b4s$nZB<C8}R&itv&lN+oD4*ES0k4n)1TllZ+&sTs+CG*uH3
zOD&+T&va^_ht_w<U75qtbKxdWd&4RpX2&u#9~N(WOFvhp%do@N+^VToay8vFp3jbD
z-Z_X~o=?&=7{}u>`NrRM?HVc#)Snk7sLFJwL87s2(D+DYbngA!V+*NIHZ7MjWbt=*
zW`7<ETSZ}bI`sPBu@~uF=gu=r2q9y7)AvyXJ#$yne0L9iu8vjS99FSEhf(FdIltIr
z6W^(9Y5Ljt_3bVN2>kV@%7<ccS+C~39QR&y=nm~z7+EAB&E+(Ve0z&Q?M*x`_0Wby
zFJWcMTRIAzmUqo$N*J=OPNP(G6z!kx5&wcPbDL#gEq2!)AW9>GCz<wPJy$D?ys3Rt
z^t{<ICbE%8FvJF1!5QPyEF%3p=<PdvE@179;xPcN0QAX4hacb>BHq?A7P(Gij_mrj
zam<=4KR_S>cJJl$UQA>mcp*Yx*SpuRuJp(9z1&f2@weHwq5p+Z@NwY&0|petybWnO
z6M7_0T4<!De=1$p_%{$!=`Jdle^H_a=H|lzb3ZP;(9MzUgd<^lT!3IAxqQ~?^)Mb?
zHYs{$OxQ-b_81eC4_w|Av}NT;|G-h3V@~^b_Jo`(7kd4fP0wzX*!*jrfs_;~SC`Sc
zv))V#!|-js*Vt65riUW1-El!(f>jcvePpn;=%$@bQ_u$KBA&r@O>^(?+T;rn#4Go)
zqt6_miSgFk3(uz9d#2BBq1|oS3uQ2$*Y|$L#>WyzoLoY$l1>=Skfu|893o#P>s8FH
z=H$kV){HwmMOon|8&~zg^t&1B{$!5Rv)CZp=0;Z}llm=uwr4SzB3~!`n83j2hiy9b
zk;Jlr0{8=yP^UF7&W~zn5_q*MEr)IvjN@$Amw#g_XG1CyTzgX)!26aAsVm2I2r5JI
z1riL;A|;n?e2_Du$$vo*g$T53v|p;g(Pw=sm~>UYqKBC!I7u?EnL<2i-frJuk^U23
zl$z`n$0;N2E|&DRJwD5<4@G@UXlHzDGdLur)Ks&PZ0rbs3o_*Uol&}a`vkML9*FjL
zuEx6mSvE5fCIs@A9V^ry`o5WEd4`3{%31F%T!3nY(@Y%pG)EAkw=pcVxJTaE@@)Pt
z|8m29fH~nlZg$Qaxbm&?_91u*S2fD??sLVC5dsUwxWV`eIFaP$?*D}}X64rsBx6s{
z1spEw$VjrnF>+6kIUO!oH*mfyzMHWlwn4^n%{xlqM8oh5@x9)OLWO#K;?3aM$~N13
z{A<JJqbeLf_q)5&@pOOE`}Ynhhq!+sg-iJd@6OYU_*x20%UV7AjwH}*i`dBpvV5jg
zFe%SgkFlfvK709Af0PU5_=PN=$3x}h_Q{kZglbS4Pd`INWDMH8ulMMdS^jD#%A0`5
zDdgCz%k|&UPU-a%VMP<Ub7etlA?qr`$IFH{I)bJ28BX`dgcR_7Ul`wYyHylR{T?}E
z3S7f0ftxAI^}e&B4xL0R0gRZ>i!HVkX|?vA`xh_T`4_x2#}YeOxW1bpZo_I+&Rt(+
zqH@OQme>)p)-^^JR1;<!Uj8r8AmjJtF2wh?k$0`Jj3EC!hBUr#I^V~%jmJvc@f|$T
z<WVI;8WAzmedR~jpz00qfX&%Y*IAw0lTWHDNM2vF*nPCnAMP5?;J!CT9w$8d1S}-&
zJl}9?+l!Z)w0DKMy?JMKyFF6a-(a+R@l2S1u-V9iV=Xb{7b^-z+O}Gno_hP6nv%dU
zdSCrS*s|Q5PwEh~aXPs*!I)6F@4r<enKP=deJP0Vy}Oxt^~h@UzMfY3WzAc3@>g9*
z>`IIboKi8P5{S5cT9^T%*-I7^5u*qkl078iWK^(Z@7K%T|Hd63!Lr>S5h9{P6LPXx
z%E|s6J|F;+a2Cby-{sU(n!S;ROJdiKhtm4|pFl`R@bYX*+rGcl#1Z3TLHgsbrl*ik
z%imH=ahCi3g**nbHB1{*ar$1$&~eZO1N5yfbmn{LkWC!Te6PE4gz@yzdIA^#pcqx(
z{_$BXi|Y-%VkfmdK{pQcT?v<_NxJD}eEQiGX+Hta{IUT==8Tur1L5}vsBM+YC;hk1
zGDacMta;pT8#z55ell>a=tb`vriYtHuN~&;zovx66|8$!q}fN9DRoqVehdCKL(N<o
zF+%#1q+YULcXGN)+YvG^XkNmKplHS1&o@|li=p6RW;u8{uG*w*09{F}PMg}3KWn8{
zuN`{3z?|o<j3s`|U07kS=8|5hS_Wzr5edv+sWxl*KzR5(d#s}9f(KXQuA9?%kk@Wd
zU~MrkI0_B%`pR2|5ZPPR;q3IKAl@>I1lB8LFvIwVrE~%who&4^@3XTDrF*6D%QGJ%
zxW&6OJ4M9;QG<hoLRs{git?20^pbUGuogFLXz8r~mctz%Q|eXP^*Rf4x=6<B`2`bt
zuShDp2Ezp-!tU+81V8_2<fbbk-;XG0gl_NAce<GMv|!*%Rjt=^SDDXAQFmu;C9))&
zjO3m<<Z5Xkk*Co!y@|WJbXtKd#?#FSSe`FKjzjB*{suZ*Oblaw*IoFQGkEJWlIoqE
z6~D{+VJ?kZ|K)Fllt<lh;bvj*NmS5WOj5W->)Tp}A_?i~FCdZSNJs%dp(ssnMNNbX
ziMAzXWX9|2-P{NZO=`J(^7DQ%NY6yx!)Ki)KC_c1*+TR=M0!}zbF27ETY=5?V!!nx
zCW+;E$k1d)D78@%P`qZ1iOfp;xb5lW!Jw9vP8m1DtxwjNtP~0*t3eA54AFa^joiP$
znq?uA&q%S0IdpVbffs=5;+e=ssHe2N7>2#P8Ej~j{PqhJsl^Uyct{`f9tsg58?c`s
z&Fw8)sgkmG6B0mD;ysl7ttpEVaMxV-Bf$YuQ4oQM-d$5fZLz+)*`ziiFDgJ{8H5F#
zaJhRlWSWxZ0VSDnT+VzX4TaC!CDG#5Y8lkhEU~nvl=B%q$}fMBIzwxxw5+F&5MDvr
z6v(d);&v~;oVxE6<+c+h5pRFynP@B5)1*)3KT>aAnrgOXI+$+8PY8VJiPhEZV5QMK
zfmu6)z(&QBd~rw?g4n+^_c(w99tY`J)k`wn5M(X+<ZQ94ZdR^JU?2#hbg7NE=Kuxm
zO9)TgH~ZkzcUF+Wy!r6tZhvt1?w71kf}i`_1;CbL(g9qkrJFtT5Dz5=YkL>8sT7ro
z7#*d$?jfqSm{Lt*xjOc3l<J<n@Z+0<z*BB;;ApWwIIvR*gS}9*5{+=nmT$c&_$=vg
zv8|K$@n*f8>QgFeVgW^b$rp5);k~choLjFURW8tCd;a=z?C`rOoAV+^GLIGHa3J`m
zg$zeIJhYuIZ7K)PFa>1|cJHMay9vy*naw7@iX@iipB3Osc$yx|NPV@j1Y6Vg6ilKQ
zMxPITvC(Ia8*i(5`A%<UzP?Bue`71hxcw%TeCp{-`MARIO1<rQ^2TEH9!pe800zhL
zv?-xL%cEP*HcLT7L>sI9xdATS%9;V~a^r1PY+{dkT6`0Yy`{s?nYgrXI#`8ekfrMi
zm(v?xezYG)Dmy1tV5zM-qIgLQAF1M{a#3?t2XoaL$Pf3iUg;SM=@cKg^YyP<sQYw2
zb))nTt`8AXPTx<u4Zgw`dbr5RNQ!Os&V~Mfw$hEScXusrKT{tvj@8D)<n*dyH96_d
z8~G?B5o*qL+b1&drRI?r9u*0npL3dJd+JAAfzACp$B^Hv609*TML$hhc)O+_^nm2}
z^?I?YogfV!#qd`URsKPK14D^VGgDuiwslz4?Z#2w!FQqD#pdD($DwPA%-6haWV}?1
z3?J>vh+`b6b05qkl{>R6K0(e<8AI5^K-(UN3Xv=J3k&a`r@f&a&jKw6Y+}8p%A-UL
z9gvhmwpg9Sq@dwbA%2w|BkcX))}nQNas&cobq0P=JmbuG3o6+{f_pcA`rS3o-h4H(
zY~hl$uzy!2<1`+aI|wXXM99iMH5xUp4}Q=}H<EURJz!qO(x+l8&08HHk;4L-#9<B=
zJs6=7VxZ!m%i)V3S-$9y`7dE1AUbHEUhJwjTiGu(8701&yo9|j$_gA6-}A!nUiOF3
z(X2{{xS*h(r}v4P<jRi6Ki?KRb`r9qkwTl!v?{|HhsTU8^B?v=4roF0_el9_`8X&$
zI`cB})LsDqn-*62Rz8?MQRgUy`c9L6C=6vUzGsK8KSq%N@w2hjDs}z<#y(=B;958(
z?f@r9JJ-iUNc%qLjWDEvtm$zZgF<Kq^aI7el+U@qFp9o{WB9}Qh&Vu-F6yTxOaW>^
zELLW0FzfjZzq>?P=iX*yE0k3l1vA2lvdYB;pNgW$W1$@+Mkw16VwOWk=T=Q}XLyfx
zlBTTX^0Z9=2uOra&NO0@A}xJ3B6vXxZ0fFC#o`HSL@lt9)G%(OaV>QtTty%X4*X2k
zN_ZRD+W6GF&)&59w7FlhZ0;8}6>4K}63reod-uiK-FdFi!gaMj|L$^=M6TC9F>8XL
z-c#`YY=3u)UPved5r6}IXr+h98UWrb1HyZAvcMP~Rn6=|Eqi-mi|SpYo^9p$T56Z1
z&f3Pz0>)vMEI#m0e_D=l{6qw=JV3AGUfIN{6RjXq2R6b*Zb&MwtwX)!45n*8oK#_^
zGL0>+P}a@cPkC#yt7i!|<%tLkS$^ljb%Z#-GCJzQM{Z2e?^3YQeQ#lmJyD)&vNYYw
z(Ja1CV=;7KDp}S^V$EW^Jw|Hv_o%?;Zg{msM$jE`vKl;#IyP}dJ!V>Q*_Jg4%&F^c
z>9VHcBrl3PN%O@+b+EF%&G&x4<=Ogn>@c`Lr6Au0??>bDuV1N+q>z5?^4)a_{EG>C
zWb%|;4AqQ<6I)Q?{Y8*cF=Z$C3<*a`@hi8A`}u^kGvYo5+Fmh0V&7q}26nz%Ax%G<
z_+l^u=I5iiEOM1ETp0egqs&{<-6pT)PLSwB4eP-c@tO-K$^)j(c5mC3x$yxJP(#E8
z@^7`DZmIPwyW>TED+2n)14b~knMkB)DT&9oK(aB-P=FeYUBU-6VGB0*p{{@z@CtD!
zy<<ysJUc49zTK8{n^e$*5%MhJnE>@Qr3$eN>I-<Sj0}g3WZr&P;^<JjQ&fa=r@?s`
z4oA1U;<%HdR}!0CIEvO!A6x6ql+qG@VPJ!*lBtsGw^!c{cu(5PJ?A}|<2AzjQ~QXt
ziEtM)quz%EK~u_h#>8%*Gm=!y<Uy=7rs7Q2k$G!WNRs$?*UjTv2*`^f+u-Qio5yoJ
zB@T(B+L+!=Rd{acfU5b*C!?cK1k&;p^Gir$6Yib%aEv+vhL7IP42eVECSyx8*cKP_
zVsBL%YfR*jWO?2lDOYXbHJ;2hX;xUR_S)>Gf~A<{Y?m{?7&zHb(cw=v28pYy0gwf^
z`@+}cfM9B)<KidGNK2f?8iQ-+kXlquTHVX*Dv8zd9Zo&s>p;ubgqSO*Ke{2G+ns1z
zK248EwP5O@z6)d*l-*x$OcLi)1gkJFUlMXJ)>T=~MvD8t!#3=m__FmuOI{Zt5lCz@
zu$9@z+d%FoXI=V1>wR^~V&fmI=6?5HJdco{-$$ANyrFS7vdLmoCdS};zCTfHT7=lO
z;C!#y>ol8}<Wm!1xY1L-`XN1Ee1g9|l|4MMwaN9<fHXHmr0N6ARLI~8_<*@mw)%5%
zD)hMA+4C&lTiu6A4fgV)e{oQroqly=xC5r=`?}~kbN!ADGLTMgo=n@0w2z5cq?{Z1
z!Jn1%(XN3?%}hMHE*S0Qb|Wbs)OU<EUP>kW_#L+7)|x&Q8_}%_oO*%cgCzKM6~;>n
z9w4s#VDqC=$xCXw(_-|J?**|mJ}4a2wa|H;ar%h>f6?LVx+Rlwq8Q4N(OrX<)i|k+
zRujw5#Z_x}&uud~)f0nni5<_a<2TtlZ7R<Nmf=xpzMjIh?UAXRCGIG|XX@j%$YPJY
z1W<Bcn3_uvjTNtB`3wTG+K32h5g=zCuV*oAhXL<fGYmpT<ys>ze{!@nxm7^LJU@x+
zabPuo33Eqqy;+5W_-d}z!Ki@EE@l>0v{N{yRAk@JU-G;b&b1Q%7iq8_?Rs*B<}L&S
zk+m=|z5@Sm=7F~z7yJs}@Ze%e$8P|S=dxbRjC0s=Kf&8+-|?ux*4u&y=NPI<qLf1Y
z@a2~+@%ZwP?^J*<_bx=52AS+3Hv8wILH+G3)c<t)sUc3k_Et~rhUe&_+FA!wd#W&B
zi!CFRIpUv_r4UH#f6HxPc~~AoUJ|00DGJ`-o-M1c(h#RD>0~>>jZ57!I~I*P-v8g;
zJnR1mx?CDBn6hqt%609Ot{Jt_qVu6-ziU@NG&rZuoEeGp#F=^&67fCFn&a<FdCwWL
zy_^R+R^9G~<}u3a=PNn&MykAy5E9}4@`j;=jTeYr7dN5vzHeJ;A_(E%yL`dvY&{wZ
zk*kBEHo2GHBYxUiJp-@SIK+*K1SgIET1`S?^%ky-X3tDG&p(z%iMtH*wtc<@L+|Dz
z&I*Zmrfxt4R6ZLoJX}%#&|{<x%M*nF)$5{s^yJW)oUB;J*ZK=~!Y$f(eP{pSMq&V6
z4|Kpkm$YkP<|FdJ25q8D>X<_`J1>fjC+wnJ+a&UC%`*N<0|#`rQ9Kc2gx=#)go06G
zkPJOT3BD(6$uwR$fF~$jz-D3khELdF-9TanU=36b5F>Pe5#qEy{_tx>`y;{&e5E*A
z?t@X)>`JdnuXQyGTdL(#db#b+Gko>)51a8yn09#&qQl!%9r|9in#Q6#o&8<C5o<dC
zl2|6Bb@n+V=q%rhaM0EJfzI0>79wJV93m;49V3yl8~p;}<}_RCP#NoSn@r6<79{bs
zyGq=yT;4(s{J1NC@T9LqV|UyGy0et2Q+IH|5Qwr=|K#eIzap^ZskRLXZLNF^QFub4
zJ_@~x5F_umA9V!26YNV%u*7DUpHsVvAICsm4RIXd{v1BI%?n$UteP6Loz*4Z81lg0
zxI((+>AE9@K+5(-;?3`t!)iHN{yPK9x9GwW<+#BHxCO;hKUjaOD9v5kMTV}A#u6c<
z!6@DZQmH<!U*D@u*{*QA_#YJ(T8{gKiXE;(9ivxygb+`a=_PrXb1J>3tC@UaQNE<^
zyQxP~^1vmBEwW?sLX0r>rFckkjq}@(mw!V;#sdOdOP&0)(A+$p>6@+X1&h1(VMv$K
zi3oIc{wD1DdzUo9R92=YT<E!@lI2AZtzO@ce87RnA^t-<)`X1>J*!bx%JvLTp4aga
zq|AYw71^B)=x~R>zEP99{^LT8*Dgf%Dv_Jz32`q+dv#VJcc-fSPI@owH~wSzSMd>w
z;USVjhgn6*(h$~vSt`#5VvGTu(puAa=(grd6spx$9zBTml0*SBbq5(RyOHS%j9HwS
zJjAN~{L(}A{rMlWxOO;UFp}o2yJZyqZYVYwKdB%0hz5J|(((=*Dn`^h-wW&Hx72uG
zljN2?$-x(RQLW;gUsk%#1itDyZzDkvGva$@UV%;7{Thu>T$+=UQ$YXc9882id<E1p
z^q(U;|9@AxYCzB8h>|uQgw3d1tkuit;0RmEOQxq|9nooPF}*}~{Ua6Z3e8bJju#EZ
zIg9#`k65hS{k<2)K?|Mqq%C$Q^6=4vQGYL8RzX2QN$ro26h>E97g;)6sQqfGhUO)C
z><1Ny#<h_S4n0}9m+>>?zkid5MzzkUlO^(SIAb9cr)wXTu_dtnyFe69R%+*IkkHTX
zTi1_&2t=p6f9qu-R07xko0j*zODVHqw+>Vq2~sJ-nuDKk7jn(oP>hoxyoQ#b4G;1W
z0Wd&T%CNX0k=9b87#toGq!E_@n~0*(sx_V07>vDy)oMME-dfFt*O>mHHUZj*ULQs~
zJ2a4~m`oiFI_pl{p?3Ka*(aQ4E_8m{%WTQ!vSr`(jqnjAWQkrrr?&<oq`T*oArM%_
zthvv`8r;=L7j6rQ_8P}L>kmGz!2VSrciZ!IY&fV>`-%`eKTccHV1?g5LsWP~JNa;#
zd4BEE_7k7`+l7ix-wL7WWi)bUDjcwI{t3w?%{ApR?{MrfakzuUHmc9sbiyh={lR4?
z2l)dRV-12`cUoktNpza!-QF;S!BM`KuxV&*>yGTx((V=;sn{0TD3~3d%C0;ZolRNR
z=^&rhLy1TRvdln%?9K)fEDqstK*reFu+2IS;uHpQvwQ<k$1_nY2_8-k`r52Guw+1a
z(<n&cigfLi5DsL)Cc*+WBw2=ncHLy%NNgYbbYPum{mXvNNnk8-3Bqac*x`U~Batvs
z;<9(Lm`Z1TxoN5>J{}3~E(A~nM|k3VSMkY2s(8h0o7EfHHc~ivn9j_h16S-|@NGpC
zcCA9~lZIo~2=PfV6?tjb=Z8tXe9W;V-eFy)k#c)bl{YBwP840(;&j^+UvXF_oV0LA
zXB#MdW&C<wSt+rqxw#d8$!+*|1*Mlfoz4(a8NdE?EaORx-F~ZCTdK+n6^C`?#;_i(
z^nnWqKf%UjqvzFM;_YfJMiNU2(qaW(`4O%2p}s90H_bTu-XQ8z(E=0W>1-MvWol9w
z9C#%?%!`~Q?v{oCZuRgOe?A<J1O37oa3*4gXC{VcF=TtTezFve>bAb&oA$_QF1+??
zXTSiL$z{=d3=9|WA3^HR(V-`>=?VWT9|*#LySg^xLt%>(qJ|$&`5d8a=h-67mtIqF
z&Nn%psZM^#%%+&VK^<mqaXmAQQ%ASJvxCy%>c3QqUy|{z(S3;b`Jsj|q}IXhTHpBS
zR|j9ZNdLT!l&Agi)fPp#R-HU{1kv0i*NM-)-;Rr|j=k7q&KIqtwb;qEr7>Nx89%9K
zNvNvpXzS%nua21>ORn2W-!eCgpTivN*lH(ViA#x7Z!*s5Qq9B;hQLfXRj!{05pwY;
zp|!ISK-zR2Ogi<hL1gHZWxPsQx~~ggiLqr>AyiAVKvUM7mlO8o4Xp@d=x~n?V*%Tb
zr&Z*Z3!@bY)Je%ZgziE=t`h15NAdGbP~v9XUQpH~*(g-fDJlib+Ga9|^T5Xmp+UiP
zxuG*cIj0aPb{-n1dlK?!&77-Sjak`OxfGbm3;w825QLxd$y1K|HpPi0bjc&@tJiT6
zXM9<zHYzr}NTn+h<qvlp7^T)p)CrAMn$(cH%i3u?ys>nQWJ1SLpMwK7Y8LgSRT-yw
zcw=&x=xi0Dx!a8+viG70k7rBF0HzY8E(lfZIW4@`od^Y$X9Jc}N<6o5+;1^yUvS}%
zi`dwUL$QX>lo9}~W%F<NEP6bY-MyDE8{gP11YKAxg1WjNE!B#gt3?D1{ob~&xL*w{
z{Aw~K7%_g#IDEEv#dA^rwGHKFoWSQ~Ed|R$=Q{g%kW{N>KVAH~@=)N!I7wD#|GVO^
z4c3FFM=cYa11YcE%O<=blEfj64l)^(B(&+D8v@HbFco^KM^9HGbetVOTQBb$v=M!M
zoTU2?KbT5gRi?T5a&X$gZq)KoRd3)#U)V3E>ek(?W!eVztY|J?m-AttJv?r36Aew#
z%4l<Wwk>yEda|b>xqCQS&5nl1)qKJaLG_;h-A*<#J<iW<3;0i_v`_Nuu~$TcbJpO;
zGgp_hCt;KZcH@wZ%a2@Rh1!gHURRrp465JmjRk5u(*sV{ejrqFn25WJKE2|eZ(pQv
z4y38asX`i&f}P|0+c@D@ph38iZjPJU0EkrA5xX<~W-wJtwgypOJ%abByq9uh>2$Cy
zP?>TQ7iVX}dCXNFW=X>F)}QagR<h`cOLq}&Vg*A7?Ciq5%k~GTe-hX}nd)e`Mz-7F
z$AbfvYzx0(MAx>)^JpasM*y2t_jiZCSa%0sk6QR?a!AS$P#=an7JBixD%iZ{fAmxn
zX$W10tK8sMD))A9X}i5YGfEZ>Md~iP*3&-O%jqElZk+b%gL3ROvx1k9T-%y!n6C{S
zGs~5x%Xg~PLpXI?ZE+6dDXcE9dbBJXps-fyyi8n7-r>KRHXTcJ#s{Xn@^D#Pyv{~&
zjA?uNI|Lj}wp_}N?LCGJUr(D{#-+11J``~#CUJ#BIBSJAI6~OSDhzJjM~?@TB(>A4
zo-_vCB`#Zq9K03!BwjvhePz*SA0c6+fqO7p)Wc6Y+~J0?^q2ZS?vzRkX3cA7k6O4d
z4}5WWTr78piIxXGaHR25?A)wF0y(#nsC<|cboDlq=4c9pS_@~$BNj-_w}*bUS@t6Q
zkjxp?cJJqk3S<qHWMZ69ah2tDB8YwWOoLJr#bWCzb?RDud)~7<W3UM*w$01rd2jGF
z-wB9@HD>Ix*AKrlbV|!@CKwxoJyEx0AU|%ICG#4-l>!Qa?>;FKz`1Bk9t)iGHHv7$
zH%s<Dc5Q{R+*GN-nI>yn#?pR0Cx_#I_;Jh-;qLb0>-l!Z(fY^|>M`<}nE4!^&@th7
z`Y-i$sq|b!rB&4MuzjM1OT|?4S3?KAOR*XE>oFv@aaT6IX_l8|R0X45kIBu%C{u2Z
z;+o>s=e{z;HUi}PT_|2uV90kza9wn5J?Y5T<|bF9Pyx)+v{X+)FXQPJjF4^b`bLq$
z6T4l&)|4I~fng*X9@XifybQ0AYvF&O0^)S;Hj}S0!ldoV|8t>1F}tX#ta6O%%;Wrv
zfT9S94IBVJkAvfFfAq0$Y{^a#85<sBX!TkXlVdV&yIC&7OCWoRY&7sJc4xceVuOL>
z>0-eiQDM6nX8Q$ny)<1B6Hlv373E3uC6b9s+Nc!>HyR7eTD>Aet%1bku8^Y`)?%KL
z&7rEUk0IeLW{~m)6@Hbd6`Nc8)$8ypD&FR;L58Sg`n-&x2nmnNMI)@@Zod`Z_J%~2
zqpj_w9r{wGL%C^e#li*Hm-vmO1~g$O2hDGe2RMPg9w{vr4PYHweA;<<$|)}{{eE+r
z0i`BJobP^Sq$w;$j8Aqnq5XQ0xX`BVLAx>^+SJEFr!ANO7J2o>zVhG3Dugz~zq-l?
z0II4V-U}(~K3=ISz{G?(9B4RMs0?@<S8M<yI=r<6j8II7xRG;+z<}|-cSh^T%G_=a
zckWc>S5fB$YfeBymN#wdH9ixk;9GI7@VKe8w6Dc{@F7qVyqY7ysEDWzi9j(<#SXy|
zF-IC45V?!Z52uW|(6E;<PA+i&>()@G|HQ_hqz?HwAzReP<tH0C<Qkj;AU_1%rB?cU
z>);IaD%sp@-|bz&O0!~5>u{-ZIT#)YN>h_`I~Qj`<zk3c-Ys0930P_l*LL5ymW45F
z|E5hcBZXOt{Z*t7qDhSS(W5X)(N>pru*)&4cW*qJxT2;uqb;L!x8*-q;E<-zaFV+E
zoZjhh&Cz*0yhCm|Fu?tN>ew~!+IFH(gYptD&C#3(ycstteX_<rHg#`n817OcgEUV6
zY<TeVnB*8#85{nYdd%SvvNUudUe{Ztwqv7?&U~N(zz<g&8^<qKruE9s+E+fE>fFp-
z%aE=fDN_VVSHINgI8QP+7_Koc5GY;8*cn}}`i5y>NiTc!&^|LUws@DnRkJ1TNiZ_j
zg18OEFt-cqPE2C)^XIYH*nLIL@(gjSJDvQwSaEVdzRYJ-&(lO=y~mAKI2$y}%N5g+
z!`icJx`HSBaD&RRN23oWvn}Z-Wvf>;Bg<3!NUv}AmXM{GTgS@8$)~bp>+!KM5Zw{L
zfqXnNq!eG!0pLsl8t+tri>tGf>s}x2yW!kV)V}0x-sQFpr@|b`;pdw3q|x_y4w$T7
z6lHps`xiX=3G<h|kQK2y+$(ATTm9Cvx51`VPz}})Q&7WjY)nqtkJP(h$`+r(yA{EB
zr|he{x4p}YMHY7I*1(<1UTnQhfcIv9trFU6dJCToE)$v`p0ZrKG>h!pL#2O*gR}QE
zEzp7Mg>qv!=TIF(NO1W2$e#pup;h1R8pxaj6;zVJT@$=IA}SCrLVV<QH?b_@`7vO#
zdvGx4vdoZwxRlcLTCj3P2ly5&s?$>CX!O%$vJvYvuVd}JU)o)T-Z8;}3M{U}Y!6pQ
zre;x=R&1MGU`DAwN=!F5SAGy@Fayn3;v5>8WA;>O%=9u?ySXMvDAjW#*a*-Eaaj)3
zYg}+@NOmfe)1QpnNYdPT?b96dvW-1y62Tp+G&?c^QLXCk`{jpSOiCt0@f~X#RIZSH
z>jyhW^6MbGt^n^~TCMwuMZZ}5sC-^-pOlRNc!w`>Uy&%GUY+%S$?Zc4)nMx#W?QO1
zQ{D^?{8A9q_^w~Hg-qXJsVrR&Q1?CAEJ`fNW7(NEU|I?F;xt)+1N6&<nQ3;8D&>ke
z&)Hm+HHyfnywZMP8azfB=~(g<zRs;`*M{z|tyNp&3fE)K8Bs5YQb=R%@gl155R06H
zVzZ4);Zk3~F^$4kcPFYP)%vE{e*N0<o`6T~$9XCXS4Am@HU(-@YSq*!)Re7fvZH_-
zWReZpAk$grYaTCs5qS$!%RN1Y^^3`3sTw)_>J0$_2L^zS9tA?!WumR~yEapwt6iJk
zDyCgKd-};`P4lJQy?-}+GPrl%;@09H?aK9%@R^X&({e#@SeS^nxI$B`jq%ERQg&lv
zr<cpo!qWSkk3!{bkoddN6gJ0|aD<}I!m_~&<)Z3fL>syu2j(fELmF<)d-mgFqGjr>
zxL};2*7x@T;zP*}2wOTWH%%e^AV%!G0^-h?TxZZ)<wV`?jMzGHyccIJXJr$ac?T{O
zkN$%jE>aB&MFCFBiC~hcyIH0><pb!A&B`iYgN%iPP_^#fyB|(_L`8^u8tqGCe)&`{
zN}>p}PvDiQN!rfPLp!Imw`jc!G<RpaW4!}7j}lOToPYpT5Im9-zzqiv?Nt~v1AIp0
z1b&qJON)$05%y=u+;)pRrbnk<KR+$tqfgLSZRVnwtm9y&CSKYWcmyOk*1SBPv+tRY
zvz?saNRl;kJ>bmVb~=b(pWQ+NwJ9!=GeB!0bNd>K?sx{Lah=NPmkL2t#;Xs*op*3z
z>0DQtqxC)uRog3`ZCW80UMpOe=5=KT<*L?4GW-q91+ym)2hZ#TKLjrATJfN^`mtS_
zOeK8oVgp&c=9+d6jX(0Z)-E90PK-*O<?5{pH24%E;ZDs<^@OoLcYYF$@Md!UdJk+L
z(Oxe!v-KJ8&w+G7ES-#_Ln=5b7pXna+M~~~F&fJ40@G|$_<}`^Il(bu&NR<YbV<B&
z3ijvZK$`L*Avb&A#{4{uUEZnog#SE`Tuo=<s`|1fdBb0SB!T_iLmz8!-!d8Uys5MU
z%`nF6HH)b2CJ$Q%3`ZuJx6x%rNA)F^hkcQFc2-|=rpV&9(wvYH<hTpq7%h~S#Z9h~
zO%|H6cu^?)!tN1mR_yU6XIzHWCS!85)_DxtOOJ1k!5SaIJvA|^#_y@g0Jg5EA#F_E
zcOm=jY?%62FpIO4Ly=n+*n1kD9okC9I|_xS4ZEz6)sPcAXGwo|AZD1XnU(URo4kwe
zZFhh#Pvbs@i6;L;qj}C~_eTEb&~5_|JwRW8^O5ku^(3h%X3lJq0jZKf;=>%ni`Ot~
zJ-+>uIysBiFXMsYVvU8o&Cn(Tphlx=H^a}n^?opWjCMJ(&M0Xb$kJM!d%PdNDf%L^
zq~0kX?<v946bP+2%-PCOm>|Vg9YAKW%p{S%GywvKq+KK6s(jA9LO{5TJAqoG*-7(0
zp5FdH?7ekRn^E^J8fXhOpg?hJgL`p@7J`-HR=l`taSK%(io1KUqQNQd?i$=e&|ra(
zlYZxS&zvhWcg~%;f8Cp5@+M^a+IzpxyPmbzvi5%X-A}=LD&@Mwbp^`j9hh((oj%6>
zIC;@ajr%VE@W<b&5Y9X$CA^k%EcnUQOg%XHUC@4;n`WxfD&MeP>NUYLVMrFAK!x)5
zYiBWKKTBa}G3db;0)htcK2uRpjiSgA%6diWFT!D*7G;cb7^9P%oca>LBvUmgN@9Cx
zPj|fZ;*YaQNwp<9$2l3>vEt$+s3ROA3cnX9c}p?vyhlDIo<WKIV3cW_<pB;Jh4a(e
zl$epXpJxtVN>(povxgB}P}iPSqNaMIvRBdYwQnh|S>MsFc3tm^y<Wcduom%j;`FV$
zL#aEXY$%km=(E-4kJVVIAD1wZ5GGa)E-dQUB<ug%123MDa!7pSv43D=>dfXrUmx_1
zs^E2jN1b*9W|m#dTOXuK%jo}|SdO%;lP%N3oAE04<jJYvS`QeU!Hpxj1bK3@V*@I&
zcs!D?^_z=zb5=EsGn)b@zPi$Bp{J(y`bx}`jwsz<e%zWF&-ha@Q`vVMB&+V|2kwgH
z45jF6OTLt63R^W1-EB_A0T5JHN$~J~9)l>MLnwtS2jc~eO@0}4ULx3WD+w+(RqZ|+
zo4nH27B8t(0{v>ne)o<G_C=D0J&D~K<W9XjW%3&rO!ES}^Upp5;5ylN$KTd&($vQr
z+D0e|(5QKjjC_8kg%#un`rB5HJ7RRfHDM~rh9S|tpcb$#(}w10XYXlQ<)UzmqmO+N
zC+pE}Pi}XD@VW?M9$y!`gNyVn26P}jjlILhF9T7)iy)!iRTH+D=4a`z?37<St9UC6
z^)EltdBL8Uef*>=*3@)=;~B>b2A-xrI8(20c0N~Pj`iw0E@*HI)n5=t$Ma0Su?YsX
zKYUXtocSho(jrJFLS-#NI?`5D1E)qP%dcEt!J2ucPxI&2+0_pPJ!|uxj2=FEBQ7uh
zcQ1M#+uX{wJT%rUKhZScxl#!%`xWC8YWT)o|6MIa$M47e`IA<@ANR&UI+g<j&<+nl
zdXt#&MUW5UD!n3g#4NeJ&f$yBVyj!_Q7iUKARfT!!79t$*=luX!>_nrMlge+26q7I
z-|Fwf>dsg}%k1f#8V^{8)SGcYq*huVUMKj?ppDr-ryiqE6FUU}a`X|*Sh_4ugEGsM
znJa_eQVM@mg}1{NJFo9SxXz1};267AmPq48%xsGMGuGewmsBs`rCQ~Y&EDnj%-hDM
za^r!Ii8(HKoGlgtba_X0RDzz%Ke_RbJ_vu*`iTF^?C+n&(agCKI+N9AH>JH<*_uS!
zVH!`TaQiKm<+IIceT*jM=kfPC-S*Q}0|yTAqLcdhDML0BJd2ixf8Oe)+kY5t#iDNq
z1Z5nYi3k*0IXi8scnoj!@Wg0aM_F3tKX3|0w|%5oOH9b5rBvquxaa~Bn3p({l>PZj
z_WLwH4`xJutv%r1ce_~sribPF04T0}iia((O;YOnJ0B-}UNT`_;Bewd!~H70b4nH}
zdG|qOyrX-hs}vvb_2tRoR(n8eqA|lngVVY)W>=9?pG)MvAa-7+Skf4;-&R6_&Tng8
z0mS=$oThrV$Z0Q}<BTVAI~ptu)f(1YJoJC)NM5<_-NsqT)5lY^5H@#0%{-l0NN*V=
z`>E~)sf*Bo*a>BVx^pJ(*i&ipN?ZDP+ebO&+Q)Q3DlY;aIpVfWtB$84p_j88022GX
zzxH<z%iF56j6P>q%4Uy5IN+p~TGfyzD&qovYC2q2wjYU{sI`Y;48N2Qea>er;|e)b
zPwYM%K}-<E!n*p#H*dF-u}CtCx=keCEo6Rc=O`vMCNR2<5a48)ZKpF|CXPi6AVWVB
zXQn6I44kE@cl@-r&a4(&u`IIRF82ngC40w(X`Rd2A|?foNNukriHRsQQ#!F<9w}ZI
zIBA|xHMDj%UzViaer=k8GIk@-aqVL=>roj)rDScibyPxs*w@@X7}YYr#)_0c;|2-6
zvD9KP?8M^G|MQqdW)n_g$xN2(qAKxz=D**}%4f19+$#1>x|YCWXOCExKA%K~Ny}*+
zeHpSEyS3;n6Ote06l?s$vA+~HfA$NxeYP=t&<Y0Z+v?Ha;DwP({6$m_Wxw0s*TZLO
zka{gkBiQeJ^5=x`+Y(-Ri!5<(L|!l-%4`Ia;}^>agzOw?rV#JJB4pyw2LJd7<abV)
zQ*H8l><xGuyVYgr7UN3602Rix`sD>6w>)^v#WNAk_>oh^XsS#gaOBI^a=-9KO5LhC
zO2N76lr~IA4Zdpp+6Bo=(j7Ds4PWu|*Sg5KMqgR*^1T+qUgv%BAch5pP=O}fkTj%P
z4)V5izG6)$$4=I0yxyj{_xL_&<?G5*I3!#9#;!o0yEcgI@3w~0Q@lc^j$h;y()9E^
z<c}MFS;RJu9Y-rN#ihgz;1TuvNYIHd@@3r^u4eI>@3dG;?xshVGBVj7Ar0gS&b&oT
z(P4j>Zzgi@I0${VPQ6gjj(kq<PKYZM1)(01WUE`CB1m8Q{OQ|*t|-HK+pY6{$45y5
zNq4{d3ICFj(M99L`T&{6<B$>2v&<e<!>A{hfeo*1G1easfM@;RxOLvU4R1A<$sULG
zryf@tU9ER;c}|x9>TG985?@rh4MW^Z)VK@;aFfyyxjKofwe9}>SshaYYN|SFNvhP{
ziM=%$mKqO4<QdTg*zE+|96P;;DT7;<ED6o#uh2IcZ8#VRPkvjVxQG?CH(i*3@94)Q
zZy6N2qZi9EDA;0s(BE*4hX=Wm{4U$QSKz7_mqt7c%622bv%hhE@`J0KxXmcCJRvj*
z;$C@vAHlsI5D$(Ms^(M()oaM#7L~Hpq#Y5mzPZe4KbyYwyY}Km<hw%Znz+omm%Hn^
zp^>3T-i!&y^4VVZz|S*mMW*V@B`g{04l{|GDls-yi5XHJgG5=p%sqt{CTMvvcbl_r
zHl-ZbW#LnAw&4PqI3`bt!(4hs^N88+TUrycp;P)sR-v>DlXoulmS@7BJCjd|2-S6+
zLaBTL=9*=7U7~did7-!Kaa9}27lH!5!pL?F@$>mw=HNDc%)m8%wgdv{HRH&vBx@SB
z`I}*8mT;vp@v9AK`&HVSIytilo*KR^AD2N7M>?h)jO#j$$;sW)XXQS@!*$!a;R(|b
z&9rH#@$Xz*_IvK#NBDD!!O@cE-9O7LSjX7uMAj>c5<igVLVpQp*$qZfYI0A}K1Sz$
zpF`PjpR!NhUJZBJ_R2)JSp=f+ssvLtt)*fTPSbFY5A+$5)-LpgE%$Pmjd|NAd698?
zDO7YO+_Q3ouENyF(F*DFJsF6&F_rzu0{w<CAI5#?*s$?=zX+_6l8`W127%-nR#_Kl
zvUlz6&0CVSu<PwS;y|Y?Uj$!qrhBekJg=H{@UJ^7FrfSwV0EN<Bhxx_<$hVr?FdxP
zRX;0S!0soxXX<><zK2Tta|;$~LV5i?&WVH>8#c=iezs6JE0aHNZ@mad7SX0A3Lqk@
zjr{BkuHGJO{21eCv)DJ0mF>C?6~ZAYEG*F0nvS5zV&N6f{%GolsJ)@vP6YX=h~&we
zNCj*t7;S&~W=WQz{*9h3IvJ&ljz@^>@9&p1`lze#@P>}-6ElxyB<4>$?#U3wB&S(h
z(3OU8Nt9m9FB7Vd67KU^MVv@jrt^s15-TBgt!shxIjmEKURQDxd*2I%lmt}L(cZt-
ziA<!ibSo7IopIYa4WFdK1&HN_4_(g}7Z+(Ea4&1ns0auocfg2<w#idy%-iPTEkSyQ
z3!LkZ8Ju&A!{3L72!jSOu77gL6lay@&>!3v!Ct|z_kY_^n@|yJkR1)L26yjJBQ9on
zX0)XmMO?bI04JBOK12%JsqpJlnKLuME~*634KP|L+}j+V*8e#67^~-6RXAUrL62K5
zW-}^rim%ywdqFOSH&_AY^xSrJ;4pqu1hLd7)WQz5H}Utn^e}40wz)?0OXHpeixylk
z!cLnz(QJDKrGCb)8xr@GZ+%h0N=@fpdAML)r)IbywCfu#flgSwm_<mx$Lr&&)x$3*
z2P;H*j$jEILP*FA=c#^)dwA7FV^n?V%ErsksrL~f%K0k$Qvt;~2w?qeGc%*jl5xo2
z3TZmz&7v>J9q78~S(&Gu-@!=;Jk<|v$``W+kn>t7aQnT&3eqnY!D{&G?m`G7ZIy%`
z{hqtJEgF*-M<D2#2M0$52J=)$cvXgk5rb4zmx<vDXIqcjmk@b@fx!r0go+i}@6%t?
z@#TU3elgl+8hlVc*0q=8fy*SiH>EC&k3RsAVx-UCwyFX?Vn572c6TSV&r&`%qM?H)
zsX~vl+UR)v28rd*b;E_DorP05<PGNJidN6}%uz&R1uInR)@0oi^9{%fygE+nC(nKM
zLgl#~-=G@kDpn`364uB7#gEznhEML=W7!WjC?l}Sj;~aq(eo_TR$%3cONSD1u2H#A
zo&zR}W%OP=qdDS#lxn!J-T)iD*l>t{T|%MC;bZ05JT~>O(YKgJH^A#>+9JQ<3SCq%
zzx{$t1T?BBSoD`S7@8<<37E38ka)JDl_<b(i<3&Ug_xG|K2_*5VEhl{X>6s-Py?%g
z1HStQ)mNpUIqO}6q{;?SYTddw%v*rgx!Z?pY|Rx1g}TO1IhQZiVe9M;zNoy4OXzm0
z1Tp!6rCmV-E|=*Wz__1W46AQSVbYTaM|<rCHRsx!d*HQ{LgyP?!xW?-mP^J9TcRZN
zX!y*A=xdFYwozTowW3n`scNV~iF8)t^0VD;4{5_)eRkm<kN1W}ApMFF|6No#HjEts
z{as$j-CrO0_4egl=xk?p#b_uD)cj29)_O%!YkQ#HV@XuUru0v9DT?dqLYL`REQ}5L
zQ=2(VDA&zivNDr>Ag)JUy;9&Ua%>ZB4Mo=6q(PHem6;`dk0E|%Y*c91Ps?@YvPmpJ
zvM~oRx{Kh7PRtKkBV&%66;8>k2TCV%TV1#bZYRm2k!m>jw6HI$iDvvWCv-*4Q7)#5
zT#yY7QKn6(Wd=iQ8-7F3kiFv{bj+~!2pZR#5Qn8EtDR{uVazjT6Q3(^G&KhYw-6m=
zl3FDdb&FQ7$>;O>tRlI&mYeo0B+|A_^d5OXc7}~yzB?Gx#1(Q{GEmd^Uu#mYYMSQ(
zZaA=rULT=v-?Ud>Ze{f`;mzPvk_f}VTKqUae+w^+ECovHm|P(5f#{F#h@VI3R0T`i
z-0FL(@6jF!C*S*`FBp@WeY`&qI9Ia(1=GsgtpNbQt8X|SHz&;x0<NFE&8PP2&nXsj
z^7|ASawv*$ayko(z@6P<s+>=%>C@fr$y0%q!^p({6uMq6j)3pGb+32UquJW3&a#e7
zYIbuw;oH1zH6K7ncc!y=`=9+8k9EdU1JSAPQtVB{^$*!a7W8kx_$2b2hRthV1Uzcy
zoFxyQ!$(YNx?h8Tr|rP3&to>EjMJ}F;kO*FLw+;+Xm@mKnDZTCBt4+5(<u1rWq!r5
z_a|lbgFz$t%SR)kELuGSiNaee;WgNOPcyQ!%m@OX`Za8R!gMQ}C?J4hyuj8xSg}=e
zOW~#w-Bf^aH+MXz;(5QJJVXNox*fMh=lis~h)QMg>zu!ZaWbEc=}KL}^SzY=+!h<U
z8arsBP6b2z@s50><@_O8tygJ6cvI79y?b=e8*OC+5L*;Gb*ypLX}uqA`WkS1rpTG5
z{glg5A$Nyq#KE=)&Q8!>ufqJu*=ZB-W=K)g?CpEEp)u4PTv4V#a+^xg{^{t-RNyRa
z<wK0@^uTxbRnpA!IyK*D{a;?W$ai4+@RkuNibYOTa2jyB%5c^x2A)vN*WCKrY{{yU
zyFDXCeS=`X?f=WYn*rpBp)67`i)X5pfx1?*yF4!MVg=Iv=voZEzmOk8U1;4^eB1vi
z#hol>B*%?UW@lj_6&665*K$9f@1gOHUO*-0yMl7uWZ455?R1B|k2%EgqrNA)6s
zPjN8qV4IkG^7bzwVomHA{K1->$*E}eqONjJ_cl7J!BMGKpvt#V4H<_186w);s&B1O
zD-<v@^7`vY6h~&Wi$lrn#8rF4^-!zS%@`fmkr}4u#lv0qpD~_(bqenGdbKOM^X^>P
zYK=a)aq!-&Q<{qxRs>$9U`)Nw#ZjIV<5inZbK7VrRcbY{>izGNZ&KG!?SDd>fcF7$
zo7r2(FDRJsM1Z{B$@}4BCS<o&v2no(x05O^v-f(3eWEmoIxqCcz>OLEwCIY6)tiKf
zgYB0pf%h@<(HVDl7wd9yl@^G31JyvJUlkqu4dN=VZ<SIKGp#)BzpneKEq6&QIb&Kv
z{swtX7F7R(MTvLEc~z|c+Q+z{!y6_A9)nO?E~Fn@v|j1AbfgL%XR^ka++Stk-|s%C
zX%84`Z*A}@swzHditv+NqF&su_6xiVUjdIdj^6iGr#%RMJnytHK6<@BM$m3A?%%Nb
z#%|RSO(-bc6S(L|hG*Qen*kXVU>#~}1$A=)m3Z4w8@JygvUat5$NcV=D1amHl(8`)
zE8F-8XC{Ah`bUE0Xg%ud{t^VU{{fK7S*uUxb0bw<A?e&ytdseC@qP%I)+6;*tF$?G
zpl{?(c*w`{IluC`#TPm3WL?5u-`?ewOh31K0mn0w$O%<+6?*7lKi;pWTq^^V^PzkY
zeByj7gd=M+I+#tVy!E0IMrrGG<}DVcR+aS^PQop5iZnM+dH*53LEblw+L(|qL!n&q
zAl)FT%tKDhq2kxhmmD?~9oMqII$tJ!9=~%X@OnAC?c-&n*MM<x4Il^jf|=JOl>xw|
zEDy@pM6Wj-CB|k<`gC&%C+cVC!fZf>uO+T+WkKC5ZC}A+f5zvu$Fqs*LpLtNk}|z-
zVxXX(?%Qec%%4gE)Q0cXNSK!<8orc&#m1kK*`viL9uAVn#*7J0&?++^py?sY-uz0Q
zyY%n&D4|bFEAI%1>c1+Lyt(VBV0Vy$Lzr25iA77YMV$(Y?&9D!%Tb;9P#MU-n~L2w
zsaJwc<m(?ZK;UBc?D6w}o_lGqVu4LCq;#r)2L^KJ_=bG?piX=5Ri;zKM+#o~gacS9
z0FPW1eXCmF!HEX|{GA!H8yFg5AG=<?!X8|{C|Jl!TLHS6r%DFiQA2qk<+*?UtVM(f
z2KZ{k^B94aX9v@xpIUP?$Uw`@-AW7gLK;U4)BLQP>CbZ0zoxQ5Y%cC&BNCqup1Lbt
zR?x$Di;Woo67Ctznc;g6070c0<<%_5{%#5JxTLdLtfTea1tVD%Bm<*24D^KzlD5?$
zTcnAyX-DH>Tcn3WAAfNHi=IA1*iJQhA-LUyo8MS1LNtgG@3s?^N2eo1c%JaQ+otK6
z;?zd9mR3^#Tb-pe@@N3y6M#k3Ih{v>wfN7xA&=|X^(jv*B|R~7G`AomA6HvB6Oo^+
zsK}$OOiSoGt@r-7`J|@&cbX2Sqw@_(XnlR3ksik30g+v7ZNfY44E$goIs54_w8^#i
zqN{FLgF&Ag1d3m#!+i9f&>nnzecz2a1quKhBuM&vm(2fjV5W&27lzjJysl^{96-k#
zQ<uw6A8mZiokldOyBRMpdRjla7WvKY_L)O~$0;WMAuL&s4<rTjgDh;u91@I7)^`^D
znA)7)ch4!J_#WDKd~!rzh^Xqp9BYX7&_F(RRdNm07N1p&BihuozCHLjwv`$VD_3#U
za5p+wrXWgS_zn6uwX%(U3#5w_KESFoi&@HR>I}(*s@rnamUF|YY|F*9k3I~K>$`bn
z#e~$?U}GoHvj6GMkIs^0lJ0h>uo-0kkd~2B&yr_1mo-Uh$r1rWU@X+1DLK$W{3g4L
zlcy=#0AgXx<03^bXPFtHG4Az)zKFfxiMg@&r;fXU-tu-z87(zkal2#WTpEV%y-3B=
ztwuKGf9Mqhgz#-2B#uWYWn~(z4R9eMlvKGn_@Ul4C4c^Km@WEeu+l^R5-rXey1F8}
z^W#}4h#9A*A_@}7Wc5ieA6A^$^7-qyejo5&y30A|i`HVAN!b1o&D@_~o6qB$lK67!
zRcxR+%=0fJp>ukCIwhlduEph6KKH0Vi2mube`5N&w~Sw^Lq`vEN>c{9w@nW87Uw_W
z1Az>TT|Op>n}eoLnZ~?g%|KhIyr<BiC5hM<Gj7RQFAD1XL@6%BC=@x{74|H@vocyA
zS3N3O-*`e)NoLzBuC`VW4IyupBJar})XH@R|3jCAt{Fsgtb|1&e4myo^$nRpzn6q+
zLtx(<+76`Gb!lE^61mntd|~wp)2NwnTG5uA!cK^ht`T9<c)f;{I(w)YCkvaZkn$Og
zF{~%YODtY2>`jBCN1HMP6W{82#&bj0kIS69Px};&;yhi7PTj+obXFoi;Z#VX9(_88
zaj#$yV{_wf#?e6!^aEjx(GA{4h4J0U%gyd!5GYl}zxL{i5s#j^HzTcm?OkY_F^DVO
z-xBO3TOf8%+0`ifxlle@Z&`i4hi~bMhV_0h(`Gu;K-g<?%^t1%4}W^Xq1x~!tJ8?r
zVDY>CO{qI&`3IN{sW_^+dgbRV3vj@8zd)8wIhvSIMB70{F^g_~!(Xq8kH_)DS7Q(`
zcWLzLW4Pb{Nv-iSdYgOa+7>UEShF|0f%8#OFD9g6LU42uj6=4kBrc9oT364G)`1!d
zZEk}iY_Zigeo4(Jr+J{K%}E5(uF3>ej8k0OOR{r!deBC#PPrld6*9+}3dU77z40rJ
zeHkLhE3Qk?vRfF`huQCT9)-Gs>!zI~n>M9-736<#(|SZTc%Z(jyMj@DHyA*<0KPwg
zg0@Dw=rZWgn_`8l*M03YtghgLaFFXivDFQ%I+7B`theDoEq$1ndZ?>Lq)ybULO{xy
z;8vcs&O4_*o*J=$z@em~xwJxSum%fo)9M2OTwFpzK!}@d(2$8x-+rUJaq5nF1z-2M
z)#M?9UL;ZYoWkq7q}@jP0xRd{&+wv=6P0>;IUOqH#OUFTTEXsu(c}y^yP3INz_NXl
zZF1o%zZ^Qdf2>W!HM#J9hX;vkFt#nM5qQ+@WJ`X4yseANX!o9S3l0gHmRHR*HBRI8
zska@D2S@m+iAU0qtoj|(+pcn_kB%`%&WMjMTHoIs;6YX1PfrsFt!QZwIsK=jr$&*?
z<pDW54D<p!{)5lU70hlu5KiTi0UZwLo96o(Y@&ptM~THrH#KnwD~iC_Y{%w}@$?>w
zunq=U#f`goQLnIyfMC-N!U9PW$nLVpTyeqyL!2gq3Oi4(aX<#(d8oUvZ1*N%%<IK^
z$1J88velb#6<`LCY?;{q2LTEeop}o@xi-R`!UG$>@oI`NURF2#pD?H9({$hy4Cd??
zn21@&#cY3SROWp`r!+Z;c)@%{v-u@}d?@IL)?+>!t-i6S$6{)q*n1qIa%Mu#`r@xH
zJh0dHKQ+(%@8P74r^V^=4|AoJ=r(jukI4~g0DSM_D*vN>5$xXWXuw+$Iy3ZRQa95M
zu4xU?g*^|&!z=0{!0c~;=ia)r0f649W}_rD*x6=KMf|3;li+@h-5<Upm6+vyvAC1R
zp63<FnAiDo_IbO9?A<L>-@5_mYu*o(ZXk=3KZ+*)_lfh-G-yQL$%9aZ+o6a5@Y4fj
zIyCm1*LRn(5jSa9_`HFcy0&jtkz-I|*U%8pUR7}5wbAB!*TqlubEjJZ6XhO^Pm8v=
zi>m9Q(x<zd}&p4HPVTokcKwpVUau(s1l(K4ERX>u(~7woU#5vY3L=l?Miqyu(yQ
z{)gYV64Do``cHLaYBL8js6AypLcUpw0V<2iG|Ed&!)Ui1>wQ{*ye!h6b#P{EfWh8I
za|)m1in^p*<<4b6WwWr>S$oHj@q}>G^yZH264tqhs!&6FDb#qW6efBM)SdrVNWAT)
zra@jOV9KE~<&BwDblTkP?B&&SOam<Xf4y!!%uxknHH;b^KW&B+g}N;|zi8I&o<%y;
z^oe~){Tc2po7zb#+Oqo6>sw{{Dx91fKdJxA71;Sb9k1`!*4-VNM%*FFUHn(b@{1QF
zm@?m-U^a}kgU)*S{jE<B4d<{Q_Z2PWOD2VAo3R`(;uZO!yj_V}m$ndz;kVSN2N)3X
z35LN+7p?LdKo#`bZDeYHjDDY5@W{k1Ek+(erL%uk0jc7v6O;Lbrwu>yz{MjeIJ?i7
zsZ9iwl!c%u%hr#0NtDoYDGtkvicRR`Rf7lsV3adqgYXRibtia83`_Ez5Cir<RlRt#
z7NB>0r{2~zi$VjCr+X$K%t;KS*)aexJqWvKT5)P#ae6~Oo${a<h=p;7x~;LRjln~}
zZ#Z~Ohs`*EY%}A^xy@5yYJ5zL$I*vxBR}J4ZG6y#q#<r6qyT`VG)%BPyE;2#?EFdF
z2X<<X$E!w?!bn^|u{;R$@Cg8bQH;<5ve_sjG3gLqFHw2APl5!28u`0IL^#L@U2z;o
zcd)$c_68;u;9_q^@$ad67ZuziCMLZqm_1|3GFpHElTEjn9lmE$2kpZ^{baekQ{S0u
zBhSh}EOH95*-ek8Wx>i$WsfiHUv<T~bZh!x)D&X%?dFX{Mz;R|+nO6}h8sXRVVGqj
zYArFf)nN141O1i$@3wVFM3ewSL<B`1VVf>~@4x=AO)uR7Z1@R#=i~1)9|bx6YdyeY
zL+(=$rO|*qIvpdrJu1xmTh5BDJTL&@(inq5XPy7aib37aKjJvLJo#5?&_m2jcY1Gg
zMJN34>$Oj-u}3E0RXXMj<|p=E7~NRSlA&?q@2Go`C**)}++`=BCxD<YOd>kBrOu!`
zXv<(rjgD?!o^?h<=!~Gy<(nYB$Ri9bwIXzj3&>V+f-L=$iNu`+H9*Vvls>!J&ffl3
z?B5PS@gJq(8a3u4;H|8}zYBom^}j_9(E9!_0YHrqV|Dg_`y<eAVHI{E2sYZU7Hoze
zv}W*DSm=L~<h2^lR+o#En}Swh%0Ng(&cWb5_LY!Pe>Q2V*D-|DM(cxH7rs5)rwtNM
zZ0D4?R}1Ru>QEB@R>^;F+TILNcH!VTQO?9(Ej4O0tYE^1U$h)#lgc)B<;8(XhN{HO
zk<tmuoT<yJmqDYld$kpE_F#ET7sk=P34jJTe%o!I_Pe|Zm(-`2fp#Hzg~{|*`+uRU
z`C~+YB4MR&AJOb=jdaEh4gwYt&e7P}RK|cNDs1`ZG)=f2EQ0&KQeT?4pU$Zl>rTAH
zR56_&O$3l{jlaA8F-4VimE@0FJboW|C*3n!ejkhZ5BJUKv$j-5slXiWE9te69GR<E
z<ULGA$Gbz&`wH$NC>6~QUlL{0pwC-%cQDsSj}kV7xuV2f)73B}1PwMIsEv3*sA~@w
zM)}i7*W@(XK8=lWk!4a#9g(1!zYg+<J<r|4QPT9Vl&!Iv_ds?u`GWbZOF~rK=Wr0y
z45O7Gko#N+$V=nsw%16=IK4;~CzPwP&aF<ET^8gn7Xu<JA=qPa5o_qyn_Nr_InpOI
zviE6JXjNzdhal^(aUYT4^o~=$6&sz>CfV&3RRR&oTE8TWdGD$d&ZXWx-+_NQ!h#91
z_P+e(=CO^&AYnb7!s}^TRjj0*gGYL8k=B?Xo3IO0HV0AfKkPJuv}pJjXiJ(dD}E8V
z`BuplKF-DX3O*L#EgdZH;oBD8!yRWGn(MwEV%cxXa!dP~Dug2}aaS;GSP3zY(5Q{D
zIs>rF>7MztzV;8v-TttcEO3Su+AfPZK?nqn4lW91ig`(c-03TvDUsib^a*4VhLkXp
z0Od#AM3vliY5wd-R;ezNnWpcHUKr1a;T{}B0!P2+%+Md-I&}*`!c>@5tc&7=V=klz
zA<8C!AL%O&s-P+q{JVsNuKxErJz}!TBcb1b?3H@Fm^uT&cZoElty)nxS7P*)DN_}B
z$k<AgYa^pb9#fm`Vz9~U)6EB2cN-9}m!P8JY{xoeggV9+p+$#ALMrFxs5Up*<xwuS
z-(BroK3)_g7?@N@SVvH5W%xrQ`eDZ}343(HPT5J><jHx})QTmtu({H5N7+9d=>)4*
z`ph|8I5LsxnP)!T&zT~`0_c)M8cJnBwV}YpATw}Id%i|(*V4?s#+3(9aDtKq^E_U<
zM(U;QLR=NvY}`%1_%NoJR=JipUp3*NIW8zzIfDZ@(-#KHueeSQ!5lhl25MZ{ehJYc
zBMa^SQ(}9M+^wytW4~ARs7YTA?ALV*+!fm~A0dRIQ!a0%#Pqg)`|oC)Y+nF9xJ1aV
zIN?`djfGc<aJsPmP%`|Fql%q;V<5Eqx9z*FxFr|}lsEOr|9mB<$?E2;^Dw5}u*j%t
zw54WScy~)B81@BEvE6vr@XUnSc<4g+{`XNKCdC|rTk6K__DgnPo4>jBOt*YQyRojo
zX-rtgF|LHzS&`I>n5|v3>K)Bz(AE(ASXb4^^6sK3wWPFd`QFq<i{!ZFbKnNsX62m7
z(UOaswb9Uve-l=hLo4Eqdv>;(pZFt0Lh7uC7(RI7CtS%Ny{PS;{z5)ep&+ET;&OnP
z&~>Yh7B!T|Et!1L8;kGvc;`i*RCf$>)Fv<z)k3wLkW{8f(1abt9^Tx>FXrWJdbHY!
zoWwlunUs|0zVUKaIa)O|uryVYu-{&9{ekZ#6n$TwnES0U*b6rcM=xIU7s^=u(y0b*
zax3Sv&>U48T|Im@(bm=cjNq87%D+~%%-Z+OOt$1q8Rp*hLQ#R1qsi2R(IiB;H2gh-
zZ@lZquc;=}8Ae9MjWTu-)+W;@15Y+CuMeU#lgp-j#=`mQxiW6pAX=b`G~xH|Ztu1n
zI&3!s)&~P(i;VI-1|P*dEBf6R%HQ-e)G%b_d}=;Li0OXJF~(IWWQC+;sxh~jKit(z
zCG3o!NrgOq(~h<AayOc>LDkQJ`KL<630YWw@&a+Lc%*BM{4Tmmv^efFGh2IA%fzQ!
zze9idQ@*0OMJXzKXVRFCk;pcv^l!<^yIy*8yrbUar9RT5@)kjc{IJ|XXL0`7H7Dx8
zy!xZUzV6kRh1xcEVL!qMf8xJ<X%}4-B)L{B?!ML-DPk2G8FPE2N>$?>7h()rZ7j0v
z8Lgcq5lgtu3f!H!JJV0e;CMDCdAc0eUQ$9r=;L|ZQDl&G6{0YjXD_>z_zhRa@3BtY
z=ewC*{B6A<drO-Lj&xGK?NDn5Pkk^v?9fP_x9zG`X_CPYBC@H{_5s_4q*O{X-7!=t
zP0ENpHsf4T?t`NLqy^ysCr7y8N)E{Y=X_GtD<hB!3F+Qc;eNBv9r%CK8hxG1))5rZ
z<lE_H=>9HPrA@k>p;{5^sY^UCQeIuKoGQv>3>s!km}$&P-ih8nvM5D*=$fb_xv*Y9
zn4Mxv*5?D4N|up>g&m)rNQ#(8E$sSNoW(ry8}&Av1aG46WkE$=O1GQuo0UKe(sLS4
zULG2&lhJeVTQ_LZWYpcn;^)fNTfHM!Wv7m{{;%=P%tPt&KpfR5n**CCYRzr|-={W7
zT<d#muMr8=UQbg~BQq~Y5Z}5t6Z#b*h4SSgAwKXddeAOmRWs8=2^ikBiHT)~&gN;D
zaj7f&(cJ7)+9Dy9)D}~NAH9_yi^)y*-oDGZ^nSYcpk43Q%+n_|M6WCsm3tGpCDCge
z&S=r2GIA<G;4pmEx*t9`b(VSO80a{p>D%gDr>gV}J(#63d&i&o5&d`ZFcFG3*=;vE
zTF-^qiu0hYuaOnhqC#Qa#*0Q1-w1-DQu8?n8)~bt-qNuP>V8NU7n|{-mh|y=RUAFc
z9{_61Y~846M1-_}r|E%FqDqP$!(S=rJ3V&{a_Ms;o%M1zE6S=xcA2I_biS%!yNgoy
zk~>Cdh^o7<3Y#NCR!r*ITLSMRK#oI;dF%UKQp=V?Kk5<XYEKOt^IK|5RJxfKg$aKk
zWEhceuwGC|2IzNHp-kZZs-T~$DL`OXQNGaDk<=!8%IZB;$N1*^l{c=&FYN33&~N-3
zqtA2u_K655@JM4$=d_K^!%GD4&^mf*nh?}LEL>J|xBvUhR4XN#CC@N{F{U@PO+KnJ
zN(td%nv3*@L33#XJA_}W&}(0m>f8=V%npXw<n`$<+*?>?bqur{QV>cQrbj)Vo6CUL
z?<H1nv3W8Aze$h{(Qm}bcEdPGR<2#jTi(;KykwIr7FN@B7ky;t$*gfaw%BqVse74p
zyut!>J`_EfuI<U0F$P;lAMb%JcH#&my>Cmok75tWP-9N>&8+qvj`#42i00h{U4MCW
z$(2Kc^*}AWayP!U+4ESWar>{BCh2`~TyJFGxiORvUS%yx$7G{ty(Z&T7oy4VzV-<`
zZQA1!>l2BWjEl^b&qS2XzNS6@`+|O#A3uIO4q4D_R1gzRFkJ__VFj_jpXn(?<hZsu
znoAqHD#xK&TZQ$>9yW$Q3)7g|l^gJJwO`60S=%kH=pU(??rxCPASyGr8hL>VIb5-d
zmamrL4_6pJE>MZqESh3Xw|W&T=>yu!8Y0BPJrT9H{BCMv^ElXxj;A1+lgrM<uPkm-
zKExnQscfT4B{iNrOLR%M&zEV=(9qe#E-n|Qu{-*!m_fZjvXyZD7gelm{z0h~Aw{~N
z90;eVAWY#xmy3|XP1Egq=iBZwJrJ))tgO%Tn(im-RTZhf4d?1>;I~Pukp3i02rR^J
zG==&_uTbwlt=0rFXe*pqFe9*K+s5j;9aX$RAXebk&QEjYe^JC&*(zaoBV(Fd9i0L0
z8owf@BCwO3$2_zDHb_h|8Wv$uz9IX(`sTyqQFV;#C>yEki?ss>nbAr+!M2?75~r_4
zn1Z&$`?CkmxoPh*__yb1?p82gWMI9?N5wZMUX$_8V)q*R-1<86UW;tvJ&#FnpFXea
z(|k+keb5$RV76Vt$O*<ek4j?I8In?;)dyd4%4~<;@?vLY6gopB%AeuZHdh-2zV6qb
z_YRljws#%c$8Emc^_r}98*`=u^KyGgFL}s*yMOdpe|8^iQ@8c<aAaiFNTY#!X`(4Y
z8&qcP_Utmi&D%?H;Y)y7sT%o;j^5=&M_04b)5`!evGdS&Ve2asN0LWbhwS~UMjKT9
z>$5wDqa^ZwwcDsKg1vS{m-StzwYR8;7YveXAPu~`pN7cH0WOTLVcoS{u+XqcWRk?*
zhTN>~OU&QFhMYdTYJwQCW^Ll~O5+#nM-SN~bkvCAGTr;`;>C8-h(GMKNZX=P_#M^d
z_Av9Ns828RcA%|ux%JU(b=Mc>B9F6$jSBWrPKC^}lLvqQqlu1eGjH=;yvy_1{((0z
z&<5|Uxfk`Z4rU%6`_O3KCSs~T!Q6vty}eb=y2s9*YxXD!Pq##m7evj38-Q{VT<~PE
zW$U)-0^=ew^x3b!+B}X+b0~XX)2T6)BjG*5iP7r?543MaL%Qi;09)Dl!DRqT7pajS
z8c3MrHxO-aETVr6Qx#}pqws*%5sVR?&f-S!2F|X<ono{e;A8n&^oWR!_jImxWfsx4
zqaJPEi+PVLF3I{+l9lY<jw@~oGvR<cGTb6@A$j*WZl)E~3>clG7Z;1%YAuI_=%*U`
zne2p)e8WM(k4Glx<xwXlGVNbfjcS^%b)7-NV=BI$2X$8~9&S_F@4&Zy=YM>Nb;bxJ
zE!*la`1WpvHiQ2x*z3j@mYuK>rOl!hq2`Kl*6|N(eR*R3Gfd%km8FQXBd*fwXkF%}
zSO+;fP_y?rTpGF_5AnQS4xmC{f!CswGM+(QW;Sm}>SB>;4@yn6?tcbO`8l>z;BZcy
zO5L#>A0``+8PSnhmekU+a-@Lem3&qg)(6h5pOdk2iJGJGz0a1{oP&lR{jiR;Dkz9>
z!LU3Dp|y7}17=x|!oKG}*Nm;OeATK`)?52DT*UUy-6Z2__=44kDf@!5?7u|T?*%Ai
z^NVXIb-X!x;wOesPXI~L^>Bp`y#=ac_xWm|US$lIa5oew1XWLl2>GTSmj!)m!_XTk
z>|CQpzv5po0*SJ$#9OXxQbW|)Cz0(PKgZaj6sQ9X;zwBJpAK0m7)!&F5bMHhBl42A
z3IoY;M3vWf7-qq{m4UJ6kSN9DAG6T8OU29>eey^|gw^i~+U)hZ1uI{b7;rk6k8T($
ziJWd+F>}65_;5ewXQ*F33McX#a$V1dN^k-vr^#!}y+o|V(-+rT)#-^!tNiqJ;a*xt
zahK&=y+2g<TAf<Ec24{2^0re=EIt~cIWRnWML$z;--^@A=q^6f+`^kAsJ*8xp0R2~
z-;ta5BDTHV^F>-g(aw{*Xnteo^wI^NDG<yE`pXKIpQJ<JDPYoa)8~5%5=oOd@>JgN
zxGX_Kr39LsL(hD18XcT;PXc+FL~+n=jRyQ?_LM~C%{O2tXE)A}NOxN*$JJlkqW7!M
zjaA+4ox}bMp#P6agVibwLLh@GKXN&uR8kl@PwZ*zt>;!+1I|f5&pPuR$7~tU!cMb2
zyZX1zqKPUoz6B|bGWb_aR`+p(ZXCKCr-;lEK;RQ%&HJ-$9<-({ah!ausv=WiBp=;s
z2LCEUE{SW}$TsmV7g0>?S!%=*^=Z?4LW;}-Nw+_2Jq!Ss9}|r<I^~jbpk=`{P_5xh
z*J>U%+m7di6NzNhvy5ZpgdQ#MGQf}aHP2`UP#yI?GsqU6sUn9`k<%EG|7t9ktl@S^
zYw&APFJEVfdEp0M$!Ff)J}-ARw-)1kk5=%1u{r&t+yE(;3rFl%M3#^nuH^%%$JARx
z53%2Gq*9^?9zbNfr?$dKl0qJ~R*JHb5k{}j(CawEG@T$qxgE6Jo+c350(C=`%2C_%
z)_2$hqnoLet*@78Awu;FY(S@&$4Pv~>Bcb)R;30Eqqca@2cX@sgb=yDPp%;$8;Vu5
z5Kv>90T7{J@h#xbd@Bv-n(l#eRf5?+Bp$#<@bblfkodUw4(KqMHPTw8Ry>NL#86cT
zlwJp%{HRB2pRV?K$D6xv22asrHZin+5k9tO9=YKXve}?ewkIZXVV9#GD<oCFo6yn&
zx$;M(?T$%kcuOASam&F*Y=w&YL)=Y#is%4e+C!P^qxpH5uIKji9|pYppb=oH*W%i8
z(b7X17ayNZM%XNWhMzfUR|ni6wROQi-X47zq{QXBck(ke?BS+2VM%F4h?0##Nmen5
ztAP|7n`+iACzP%`s0LdaM7~@v>ZtWK1Vd~QneJJvg>{*wb1kwEhQ>89!Pis{mPU8|
zwQQ7zlKCWa(#U{zeU7EGa*Un={a;E|$FN=yI>Xvkl%yeIZ2JxVU!9f1AvJ@mT+=bX
zm}nkRU$@f;JmDkUmtLw+ZdW?>66&OF2T~)iDi4J+<5Fcrma@(zKBVy~Rq3C7PxKHF
zaTaG52(E@+D|7c{@WQyDKLh6$ZKGrcd|+NHaS5ouo7*rGc#C71_-S5+31{tg-&06A
zytBDPeJ0$Bd9~$}+)5<Gl1^ls`)Y3g`oY^>&)PC$w~tFk1G=Z*to;`wcXG{K=hDK%
z`-)cPH7E%KP&9o0;U>(1_M_>t`>o3<0G9#AqJkvSlpcMy*-6cwo%ql=>?{crox=|w
zsN%B8VnX&XT&?siMXGq!pax46pIMDovJ^!sKhIqQn5R!SOO!(}e14GQk&pS5b*;p1
zg+t>|7+9wJPBC-DW>}$WKj6pF`<P5ay>SgKj(T3#^b!t~#h)rz5+8*P5=j=+_Ojs*
zu396MR&@nD_H*{Otb4f)bkk<^ENS`5Y~nTHZtsI_KYRxkDiNhE!#VjU7i%(*UAmE^
z#Y2o?we6BneE3J%?*ABbyBGNARsw!g8JZ+gWu>#Jlom9=cMKoVk>nTjM7$|v=+<iE
zcD;8~Qg=PrvIrg5PD~|{0&K`dmEHoGc^H+u^^6Ty_pdi&H=x`K<S|}b;XfOW8S2VD
z(4>m<{1*2Dz5gA8VeQAXddc~k{&bQ6kmKF?$@KL%xRPw7F~tMtl|ndod;6Z$e9cq4
zCdYp6ttA&_K@?Lmw~DH&BO~Eqyh)1RXH$h;kNDAz+JjPoDybQP41Gk_lABTE^2+Ia
zVg-)NvuR)Pt{ktq=(PN~m_$20w;YyGomR_<FG&0;Ce||?<Ri)iQ#t5TdTbp-lrngl
zr?llC%)V6td?mdE{N~pKfr`Lpn|xG4hcVUE6tUtL9k*mE<ck&3r9Y-3f@uQ=?vPYw
zj{HTVfk5_6A%fL{anpuaPlC$uggS1C^X2~w>uMz*<>9KSo%@1phKlQWF}Bw{0Zm$@
zwc<RkM;)E94dz7-O$}93y!ua@Pq5zDBDd&+%{PWgAzVy>ll&z%@Hf|SW!pK%77mN~
zEt^@)s*2~T!xrxc2ExNc)tG3#jNAwF_YY9{SS~%2VHWwl>8dF?7v(ok)@Bc;$c8Vo
zI$=`muhjRG#zC#aSs7Z#Dnv049hX>5K9gFrM5VoC3_aI8SY~bdg>O(nGu;+wBh~cl
z4K|F$X{_;cg*y9;B2$q<x=xQ5){6;mKiBtGaO<6Z@(Zr4&Y2#|I)X;l^1cy!dwaj-
zdb;hp%x*Dd?gZdAekjQk@k{)&S3sb^MzP3*jblW{ud=zm*}KgOKUz%q-Q;h$P|e?T
zhD7@BK-54>eA|~3;S8_>ND`@r#x7}^LcGFP9kvxmvW&+ZSEmvN@Wc~3fKCh!<XST2
zjt1wF8ZpkKv77j?{CG#mk+|}6xco&HKOjiS=0D_<=_@og;MEDsGK<-CaOo>xaOS9}
zx{hasI&FYmqN{#y&YZy!c8s`)OjV#w=*9>Z0$vj}^!g<K&tG%IcwCSF7?Y<bkzPay
z8e*BJgYha9to}Z!X$sQrgR!~M#l~p&9jlAPUHQRvA?MdW^9I~%J<2T)B%&|x^wTkP
zNZ2Gq7t1Ztz3xx=m~o*bR-Lci!1n>w`**Q)bq7`{met)B2Zc`FCo>}u6R(u^zqkNE
zK!GZ8P%K^hcuyynq+bg1p%%N2#S~lydcCp{d0f8q$7^awO7f!5x`>ma>mM2NKt78L
z>-QBZt47P>*Iw)rNxir>mj*Wx^j@z>ST#IkuYzRa;$2qIAZsQkMzxOfvrQ8Bug(ec
z5PrBZTi@MnM$Fvs6aT@}8v~N#mjVLJ_+e!&Rs%0l@wWlnBaM|w?IxS9+R-S_!EBLe
zgmM<9h`&M8b6(j0M>S%R%8`2m?ffqU%wv=h?Xgj32);=&8Kb=)@>nE+4U2^%%TzI$
zKfOJGnE-l+Ghb}mMx;rIqN8WtsT$p$_Yo}@Nz@a|Fw1Q<EDrXqh_DV$B@3IXeOli>
zBF5w+|Anzy948=()&<fF<Ur{7{E_b$#2zV+$f>HuJud2F-#{QNv;O{FJN2f<mv=t8
z!Ug>OoA=*oI*>`_k49+(h9Iuekpodl%)XD=%Px|HA|b`WCT~<irWCYMzh@j(-97i*
zM34`00MUKskR^J0`a1n`xxD>j7XY9E=-p}zS#pFd;o*D}{T$S$0^r)ArsnnQ>`9mT
z`J^zr=+Jyul>!+E+U#F!zmu^@)}o8)7717^ssXtZlhl7Hja!08g@v63biKn6+d<KD
zunUy*kz3<*3IG95_xSgyU~~uVQ=xfJ(TupCoqU4vhk3Nx(<i7+1FksS=w_554Mbma
zhhb;oEwmv55-n)7mT@XH2nh*&TQe)sSb)d*6E?N_<*ArxLB2J@4VQeMEh2JY7gV@C
z&oKyc8h<jJd2r#-H9Ul0uCG^6AWY%A&Qx~vtaqzxg_X5MzII-;eLXS@ugoU}Sw{Dg
z3R?Cj@=M46w5o6Ya8`CMeDiX1|B;Tb;^$y>3l6%QMO(jK%lIzHWHA9K2`fXKh5VMt
zkPwvW4Lkd63w3~}C~1Wq+Xn!MfsO5Vy9{PcJhiM@K#G0O+zY@SFWLooKe!hv{*wof
z%542lYK*8|>R_<AAS?t$q}*qL`|2lvx6pAL-#XHrgMvv-I!pq92Zlh!y#5EvHi`wn
z;lR8BUOoT!{*NWbkcF5R+hpm(SJI{0o0UurPcg(``8z079nn?x9|F&%XI~pTTPfs9
z)DR`{DEqT<kr=OFVSC<QbZ8ugea!BLUHrb(tkz}5lBbe4EGxYdT_K|AX#Cm0iz%sc
zijy`}Flz7kBhvJUHkz3?_A?42gdNXjMSw3TFI2U;qVX(!Xz{Rm;nt5U2-`9K9T}Za
zzp#c@O5P&evvQ7#kT7S)?pBNx0H}EUUo2zPJ$^nEaasB<QjS<+a2{F%7=JB*{(@;t
zPQ%j|nEqq~8UA%EDD2n23xG}fpN$xx_%Vk5{r`Xb|IQ$cGwPSu|65G}J1<3i1qDD5
z7Z3S!6o%Ras5L9Oy~Rum_t1_{Yx0;B<I%PHoKFDbMP}ya1DM6}wtd{LuBxgkDWClx
zqkOy%&?UwgCP0=}oV>vQjBd0#u6odnhLIosYi=n8VM_UbL~Q>r@WlTseEI)N#R6}|
z0H0n}(~)$Qc~TiUPx_3+nKk5m+Z_#C0mja^b?Q|abpHbKivA(1BKsauJa5?PVJWqE
z<g?BC-F@jor^D2sJ5c9&o~%X_rKv$_wlq4Eh1Aw2;En!sR=B=Llh8{x{*cxNRQ}6m
zyBB^XA;!<+M8g>lVMNH$(}aictu-U~vw>GXPl2x305#(Oh=x4qi&f8b3HR<Uu9JJK
zVo`lrgWt@Mn9LA7aYanm8ENk^cr85CWq%ySS58D|%d*;dFyJs$91&l6yjrcdp&#{X
z?8v0~C`IB6J)00g{`5>(1PSMuC|Ahl&^RlL=aMaZPZebt#L?jP(t0M(>3uDSNwc`q
z<XJ^+b=C^{^0)P~-<n@9gP?^ZSBa$Idz&DJ&+0oXrN^X%&6c}Mr4aw+)78f{%@`^U
zQR?A*%RKW=mn^En=+x8GFt?-HZ6w#UM#qa<k-1W7PpnUOvObRL8N=+>lE28UC84cw
zYTf6liLVDQTRW<3?iL~h>@7da>8|<#g0MQDWrsV67*~$i4_z$|!?T3pb=%&pMW|?H
zCe87mm^-I0drLiaB_uZ>%c_!7D&I=&M|wS>qio9g2V;Y%IZx5zjnsl?Nv%4ym8;EX
z-=+wx>k;+ls6orh?ZeV5$vgc~Tq<4UQ`kB5ZlE_kn~)?8SL2YaZ}!Z2(_YoO!k@D?
z!3>g7uR(3U-I>BJB^*%vpJLd_Mjg;i?(=r2>S4>Y^Q#%ZBu7a^2Qne2oRr(Jq04YP
ze7ClId!UHSn#-?dPp#E!8~?m3g-?e5(A#UK(y*Sx1<K}`&~A>JjJ&<n_5oK{nk`IT
zm;(0^9!~;xuixK^Em-j0Kw6ERweQnX_{h*(YN$5f<1~`hAFy*FNmL;jAe)H+@K(1x
z-S+o9<eoWYXx4hPjC*mNy;hsr*YUI^;XJXm-a8YHkTl7W##T!>-<kAb-{tiCyf5yo
z9jIh*9(9p&9vItLb}(+$^Dvvu8rAzFg<#B}Uh*@>EwO-cr5CCTrpz?ebxwP~R!8ua
zP3rEQcEZ0AhdEx{uO?cJ^6P40?tbU{zYCu2-aHj*)6=4JTPvOy|7E7jc`y>b%Q;-B
zF>kxx1aC;#_{bDjvZ_P3ID;t;ANso!10Py%cBYgb;BTYKd3Q(3eiNvy)2`~8(0Mw$
z^H)v<8yhqR$~T|HH|aO>^Fa;Q>$aC1c5-LU1GW`J{ftE414MlrcGiE`du~BmAAi9F
zE|~rj7xbk9IsbYrX>@S8s@#1)(OSsyf<zL1nD~H#iu7nTF{5hz#v0-|?K#w>?DxT*
z>}a*Vuhw&^vi|yMR)D9${v=+CSiNIi@=F7W#*!-JzEIN9s!prh(x#pt$YNrLgHW#?
zy{)K3_DDO>XX1}35U7DVo$XE|MLtz3rO6`>4(;Dl?|R?vgdmahKh7VQg@%nhVD;Rt
z@q?Y`k(Kk~X(mf2J9X7+P_v&F7Fb_dD9U(Ve5ng9w7ef2x@@Nwlt$Tzumkg+;}y(A
zjqBh=Mm;FiscZ^>)i)p4WvxJoF5#}Oz{y_$o~k-)A2UXV+ALv9LrIXdLNnb*1v6EN
zkeasE?VIEKPc4?#4gZFziM@FZwten&CKUm@c7F=^`?g2%?^_`b1JU!J!YS@o%Fuwe
zMcwM#O>T(en}Y5EydmWwXN*HKTs*J9yY2cSI4d$wh@-9&d1O5#3)=c?ZE`$sMoGgg
z#K8~uHO`s&4G~O!PFj^TI{a$^V|l<t0RmZ3h(?S%`Pjo3^Vm%ShfIj9ZEm57MEh=^
zp`rQZ)=@t(#F?2?vxnDy@5+B`@7jZ!KJqwdZEX<&&&aD(<b_F$D|=cTj;yF)9T$
zAw&`o0;G8fB|HR4p~oZiqyjY_<t3MfmWBX<gkUHHjYxPulvhH+$fH0^f<TQtLhO&3
z&duB(H*+(c{&RDG?e6S;cfPyxo!#$e_p{#>zVi*A$hNxc%hCCEg{|?IRt~>!B_PPh
zS6_WQE?ceaXtR)=T&eEM;-ahP5noNq{xm_ya7s!M_kzMayeTi3>L&DH44G&`*wDz^
z$LMX5az*%5c&v%;n<xElw~;BSS)znIefu*vYL`kV1l3~h37&IXB?13z*q!!cumk=d
zRj;=Jp9^@<@Ui<8w9ozsYj&T3T4W0knb~)w2vA+!63LRv(xzc~4<W0-G457It~3rO
z-@jk_<Pd3MRh>DKb1*lNu|0?R+Q*R8BO5=YFdwRFJG(FbI(6~v8c6sC>yI6%09A8l
z)VmqgeYU$A^Y0_`gR`68V>qVNs%gZodSy~_OwRxbp%cwFDSNc%Id_SbSb=I|C2hY0
zBpk9DANQ3XCM+S-7TK@Yf=AKDLEvdIdBM2XP&@C)=J_rNXuV7};xMI~t`30D=uzwM
zYCfA(%u@4;1I<d?x3^TLksk(UpuLpw<$tLU;^RCBR!M%AB7it|!P$71=%F&g!XizA
zE}R%^oM1jOVSe%GQ$}wxDHJgF++?cqr{!bZ?3+LX3m(p+X5#0SW65ZwHnb&5K4NF<
zX{LQN+=U|9z50bnbd@1T7XxmKUkc#<kaRM`j&?b@^pyNfBfaKAK)A;HPI*a*#l&TR
zs!IW3_5f=tiZGj#&9IJd{IhF3iV$Uq0o%NqBzht1HY1)&99th3gjlKP9H%?o1G>iz
zsG&v;mWPw(72h_w+Z9qaUIE;hHfihI^|75A@$}0%ZGmXySb@7+sDCuS3^bg}_>N^~
z1=a9OnaxfaSdS7g?>TmnSDkuz`Q2-r7|T{T;4|&GM}7|n$RiEnh}f|oN?p>AM#U<_
z8^8z~9bLaw%X9U*=H!y09BC0&md#I87HvmAEEcbV918~5NwG{f^_!GIc|Lcb0QVDY
zUL(f~4I1td)|~p1g+*I8Wzx}f>&uf=v~ZxJkT-Vaq>gpnN44dc67ytaFo{}}vt6lC
zM)zEkg$UTb9p~QZkF}ODdyXd`4S89DXA;sQ95T5FTd$HppTs+FAHV!6)0rUkPP0dx
z>nCv(o_jix=+_iR?|{o`HAK<2RNV*cJ{fmtx;aeHd5Pg`VciE_O?dop{_Ycds6Ho5
zCz*dm-(>R+E(p#n%-_uoE2_kA;jsLVIIOyu7>JYdL7<J!lGJa57kOQ9u+w}j?i(#_
z<|-VR<dHjMp9f1DwXv2yZLq$yb!9G!EZdyS-sXH4DK1kGCX<$%YY`2NdyQ^@o_hnE
z(#3L5+RCQcdAgwH$^jG=d+W97+ZNs4PDB(LQ3GlCd&PsuwYH|5jnC?&gY+{uW*7H#
z-%N<jKPo@quF?uP$dVcVIqpmjH;58gs0s|K&+vLNsb^ub&P|0+HPA+7k6S#RK6OHv
zcyp8w*0^dgwVc14noz79ZXv(P(LLu{vF{CYryc9*#O;P*$t9R{mXlP_7)`(B_j7Nl
zCELnB?34sIbi6vWaEvH%f|;ptJSU5i)5Ojy)~RGZjLo7*gn)Ts;w8<s<P)Zq03#%<
zJ#u4p%(Vm+UHkIlS>9-l+Jt^fw-KyUsn7@<*0om*gql?CD2>KkwmA=GEpQd-^~-6x
z@<Ut9QR??s=p=@%I9(#Cipri2H;lKj>YS6%E9=_eV`iZTE(TH2H@D*a0typ1MHBEb
zynESHBe+h)_oO^qobPp?R%o67))A3D_5oR{<h!y(9g%sQ_7iJM-@AHl(u|Fu_5jEY
zKTT6T)9dxSS|8`X1$ok(S0+H@S7~5P0U9vyO|`rKTy0(_grSjF@DZ%M?xDJv`@32f
z;f~gGS)UMy<;6+y{0eht<1gQKv3_h0rF@!_sjoT-Gdn1|I`qCD?HPF;X?J^GJ_@+C
z`!qRwW!-1F`I;GXYieAw?(+yvaZ=5pd0rN_z5@m8)4um@`E^vgWg@>y0Z;M$aH;3T
z$gp`N)*g4uPf9U1`oS20Ay?bqx-Vt{fMO8@Ky?uicoG5>HZ8@#P)3)21jwsjI34~C
z$pH%$Q1>sC|KD(F{~&De|89R5!})(``V91rSm3f^e_sP&npTPM(8MSznT)3-0#T=L
zoJz0<+xx{OP{{-u8RJXaG2f#*ZK8p%;XR`g@IYeZ0{zGZ40Q*)xL<*~c|y;4o`!*U
kGSF07n&ICB$kb>;OzM9Y*n3>!3kVnm!Y}#P`9@^@3Ik%-k^lez

literal 0
HcmV?d00001

diff --git a/doc/design/assistant/blog/day_45__long_polling/phone.png b/doc/design/assistant/blog/day_45__long_polling/phone.png
new file mode 100644
index 0000000000000000000000000000000000000000..389334d95a8f616818fdb63dd26c273331a689d1
GIT binary patch
literal 41602
zcmc$_WmH^2*DlytAV>lPcY<3WK!BhL!QI^w+zIZDgy0gK;2vmPgF^_x-L(nUc;nVU
z&&m6K_pUYf$IM#uXMpaas?MpitIn2Z@7fiqrXq`nO@R#pf$-$zq%=Sv)I#7RiG>QZ
z_-*Tx0R_7Cd*$~aP;ETU{YMO-j^d^vD+#I^quK*1bmnpz${>(0GYAy)1q8YSnu2yg
zAP*i8Xx|J35>5w!NL{j8)kT4hC!Z8$r9jAkzj+;{NkGd}7dd@5pp^U1M`qAo9Jn1M
zFZEu_d+Bi1%U^T*5%XkxaL{gA%qBln5~pFLPIl4oy)+eFR1{s@fybtH`{T-HYcPik
zG1ZR&8T4d&jVsndHEc^F1;zYgKg;Y(1OrpisQmY5Sh3LSVbUB?pRsfQTv*oDGG8K3
z=&>KR$2TeUWGQ)kM)_pMva!1@c;rX2uj)LU5Hq3}$-l=gSoB{&K~p|Esf_Z3(!qjm
zRt(Mpk)qsDA#LW&&8%S|+N>!GjWWe^bqa3^)p&3>kMpqTx8N#!_qOsxZ|f(Es&{kA
zBt*-+KEb@XEB~y%PBPR=SxZR#-sA!6F)yv>U+ZLVO#P_ug(;0i5B3`A_RDp@$YP8V
zwdU1Hg?b^@Wm#15G2R!|U;lnO==w~~=%>Z(oLHFwtwgcHhtkuh^uIf@sHdX~x-)Pe
z2t`DNeM0E4P{*e|d^K3)6HAm^83e1>-LJGUAQo)XFC-%j-};lLN~y2EP0F|P)^JwG
zN6#UZ4xSSIjM2-3=I8N-yc|6zNosQSFTUge^n0(luR4)XQmtw8&0`0lw0c6(0ersp
zGc}Wn1>HM(ROa2`eC;0B(wva6Z=ub!dBHytx_Q~9cUqE;X_#AlfMsM)K|^?|u6f<8
zY!m8~)zQqITfMLrP`GR<OH!|?&?wfjCw6xV!k46w5}#H~WIf3;6NsS|`|av&x@lv=
z&0|2q8O0vu(J4GO-iEquFHrG>=kvW-r=0{bkv4I0PiKBiUQo{QEOPlHnM|=p9K{NY
z`CxU348%<Tt2$_1>CQxqu%n~h*m=l|pNJ&I!qS>U+o1US@e`&m3BF0=7X_?mS!|t-
z7S+GUP{-%RZMkdl>1)((o)|qzV>aTnjDc<3gB*yf(dC$o7d(SIQBb?1f~34mkw-kZ
zy>$*y8sos#cE0M7f$|wnNCM;FS+HN>;-Rl5C&YoAhJ<V#%*mB)O;)UMblgKADaXeE
zmeC^eUbD#ZMqtSkS-u*tPrzk1AuKl&H*noYK<Dmj_L^6%r#*9(5g(_Wp~$8ko53>n
zbRT)(vIVBQqW8ViDn){`AY2SmLd>=I&Cgfth_Eq++54Xz^=HNuboZWh2v?N<ENU=0
zib(v86Jk(#oIVjOn7JaxDT!qpXc4!^1iZ{=FQHw!1&96nKZn>;a?=un1(P38MPyc}
zjaZY9L~AI!*E`WZ=pk9Z_`J`z(39zP-PRU#2Z<zlJ);m_xI*YNPX1=y<%h?9;{P5M
z9c|cXXUo;%%EZmes#aL`8iw0jQ&9jOuQKI+P+k`*))-|aQb=k2@l!}h=tpOERASNj
z)iWHtSDb9RwhhNQXaHR*0A14R_(GbwPEWBh7$SKyOo$6YWqMEpndwoZ)~uj^iGx4j
zxC4K^9H@So0StL3d`~$+v693E^~Tq?x;v&^tS#Vuk!)YFCigy5&>bFGyYqG%AL8P8
zw)S90kU1kN-e0K1bpFe+sR|XNt1g%QB_)qHLU&wL*kLEAb9(;KJccpZkWxuy?X~-W
znPPS}w4KLoVihfX-2J-3w{Bx&<06TLPbIfjRMhL$xs<Q3t!+kta;oL5owlSyLw5$J
zY2^UBcCn?U=(u2>se0nemj<U+p=~SxTYt}3$}p*0iLn+?&T+|Pl&_<O@bR!hrg}Nq
znebkb7#nqjDvD=VU@P!qd>&-su38I@mR68QtJl*h7M!^kp5c}VY?ZcIkD^6#KTI@h
z9@9Fp`McHcBR#2ae3AJu!Bl1NdUT+o*P-xd@7VFYStwUy)y?2ux->YT!wtFNyfCTQ
zb@M~mV6w&yTGkWlt4|Z?{nBnM?oC@F*yZ2kbSe#Gq@}mFwu}ab>h4KNNhwL@eswbU
z0_rb|$CK4sAX4&jm4*|-{))|3Z=SulPTGbwyO#wJpIQ+=Zh6#kTkWW4M@s`M(bi?C
zth4Mj3hB2~iayJNNH*<V?RHR*W?f9RmzulMJ3XOf1*Xj@%o9^TSN|dzf--fMOT~zT
z8B4vIC*3my#?NRZ{GpA)+~PbMgP`u==W%xqSM|?3aY1{S*A2TXf?9?F+}&L?RNKXQ
z+y-5LH$)odZDwF&mZ)+O@$oS+36+0%Vl)Ps8|(`;<WR@iv?+LSAsVEl#LZZEj_<k(
z7#r=#P1L*-oqn-=!JnTU17!ciYWfF?7?*?-L*|}-(8rVlR@UUGptaQ!>GpCgyf;~I
zIj}<XiO6DQ=3nA+qRCOD3=Pab^sx~H&N{CMzoGnF-O*IRlU|*>6N`IQ$0>#3XUslh
zn3|O2g-0ncU{I<JGVbYE%mR-X=c@riVV!mru41BP+pY(gjB<AN;Y;6+W-l>ud+Xv^
zyMO=>nK<(lcCY8z0Mmq1I8P@sJ{29$RNj<%M#3cT?((kcTXzIup#=uh++)k%6z84o
z?~0uZEsrVNrsMZ;aOXlz`ESD>h?*sDP0({gkr(`j*P{w<<2u|%;t-rN@rVYKA{N2v
z7Pkyw#h8Ehd+jg0e%KmROmn54ZSapC`_c2@T|ej>C090QG&t<6H($UQ)`+#;;_I&x
zk1zSSbI8}RXb6dMe8Cm@k}B7iRA`bj;w{GQV7O*Mo7Wur*IKR*PgL5?yN#Pk6?RYy
zO+b*#Gb_duqa&I2`(Nc}@w0*xv&c+_@2?_vR<=z8_)nS;D9Oe{0~?a9jgA98W>aRm
zRa^R1qp~tG4R*7<^x}_NGBO`Oesi!K#ynVVjMU+vKe4he;bv#&&7erHL_<Yo3@+rh
zV*Y|pIy-WK%I&g4Am5Rl)rv`nHDQ-R)lUo~O@@(*yTX&F3fj#YO>35%cf_9ef0Z?T
z3;q_2mDg@mtu-qiLM!&L+HYd;=BMJkDOi8UH*ZK}ZBOi`<2OyxGmNf_JDemGM)-m}
z=7pfCnwr`CE90f#zx8NjWMte`l`hWDJ+w^o)mT#;;8jmity91355HDuoKI<Q|B8`k
zmY&!B?i*}H?8O)SDN&;?6>j-Nx|6IyZ#Z?Kv^<e?c)yZN{o=CWQQ<85{=I%j9w%SJ
z>9feTbCeTNsiu2Lrwnj<2qxr<MrGom%ItWni09m@eZ})@_^PdO&iQuf+&0^DrSE6m
zj0e589UsI+YbKhqw~j<)L`4;ws9iU0_`e%?ErowU7b$aoTanA-IdI&|GHW6nrqRa}
zqfp4j%J~c%*Zqtogn<yBkkBYGqi^bZn?N!Gr;2vkFfdkpV7yJ*riL*ps8^}_QpZr7
zfO?g;Kp8T?WD2%s<qYq^4SnmgO1tCw0bUiU5QZ9*mbU-rAwgdqE{J&v-#>oJ=uu--
z{@j^hf3tYwZPI1wQL^3W;)qg5Cw*@G_4Rd{RF29z07f{<<M){43BBV4$>E(|oQ%WQ
z+d}@7+8|t<_X&EjuD{c3Av)DJHfyB1)h5K!lk@XlHv<HdM;qPi`?J3$NazKRF$Ypb
zC)ZZ%WsQCT;$KYlx@P|C5nISov^pz*KFVj>8H;m;iHQja@bU9z=Qcy#o`kB~?N(-g
z!9Vlr#cjWq2Nlqz<EG?`@B7fUm+P?ktSwTjep9FBr<gO_ctcVAJ8#PD=BU}!P+HoR
zYQ9kuMncb}#9W`K{$o}_M$)qQwFJDqEnI#j26nrVShq+Upxwj9Zz7J}bjN|unY&Nq
zUDYXJ3DL6Gvb$4bqW?0Z9JzmfWmTVFZkQWq5WD2_9n*yk=3beIBG#%;g3A;}s))TR
z#_Bq(&PvqT*?Q$za78WB%Rb94)5}T(BEr_GbR57ucuN1}ryNQUY{cq=Wq5k(3nyB(
zsIlWdhN6GK6{7Xx&$DO;0y6=vGpc|>F8VJkj0*MYrt1E0Uh=P3z5ejhpb6i5Y_P-o
z3qNiz4|dqgCgmB&-a9VS=EkC;ZWg-VtM3T_?1wu2datQ7ds!vhU$w2N5+j%tJ=n>X
zSunzKUfAbuCCU^j<SV@VUz~{QcC6IeW={770FAM(o8suQ0BFX)E>uzlN^f4v%}hGM
z_tMgRlf9TL)EGch^064=?rsht(vnF%bTQfsbhKU{9vrh`(f=-_67x4ZnfP`bxAC=5
z37W-tm1hBrqrBtnT9sTi4RuLmBuOEPo{B|O(n<o?=YO@hJBz}OOAEZS%*c2_d1Yu(
zmB4to*gS4V=ykj{y_I`1Pl>A=9p!pnZ8S1EI5O6no~fy3WWhthB+pBRYdvnAGu*0H
zJTKVEkOLdHwua0soiCF5+{t?Y!wVXB@CDfFXBZSJiltsHL6B>89vqN%^)7-=`|%tR
zcsuO=_B@GM>}K`3;uq$-IFrypZ#H;ch_>g^RKU9~`rhz3a<Pr8_wv1)pEi}Lz*qiO
zR##!~(ZZR2dv&H~P7id(Fi$U5oCo+`!gCfN&ebU7urczOKa~8&z-mTR+!>d0vvHiC
zohhqI_&@`$D@{7=c1BpBKJX**cyWJ*mj|;DXdJbOQ`=FK#N*9otl*mOc_mmiS0GlK
ze?MFSM=^@&>)yO|f<dJ~rm{?7k84(?M%lx|LmSK0hEHIBRhPRkZT+H_fx%pQCp$ZP
zU9}PU8~+A_N;Yop%gakjv9qR8!^eoYh?DE<HUd5wEc(&7MTa%76S<bPA3r_U*&<Ia
zr*3@eEiEx>I=2-QqqIHt=LG5K^2P*^*Q-ODEaRR>3&)YR*DZej&0l;i?h@;GE9Y3m
z9=61;SL$82quSs(KfO;2*pir`c>|tsg`E5D4_yyUPAfl8GX<Rk5tY{_$ZH6n^@v&>
zUVC$1QN_sea<f}$l~sLNLxWVA1u-BLb3C{VQ3X9__|iSY!+!6&-lyKhzFjm?)6&u~
z@m=2ij2jvi#ha?Fqm_Jna}(G0FeV}3$e)h;m+RMem#P?<&Bq6XrlzK%q9PX)3JOYO
zWaPu$<+R;isqs_#<|=cRDX93Q${r;nwkSwG1T<%)F4Vkg|H04{y&ojXNJEH8+~osH
zGC$7x+uoD4avyV^7Vr?tJ7zjII2s>i^IdRR%<r03Rb=p76r@h1tzg*|*x~06pKGZ9
z{8w^rNZUS+O5T!;SxCrcBCqIl(`kCoJ=S4vwnAUSpk#7aB}dd!`CNkYHXn?9%(|5^
zE~S0@=>{zRXm)c#Kq=rrCgyvwGnxqn!wWHxi0`BnuJ(p0Ce?xzB3}D;JiD<ng))U@
zKPrS$Q(LW}V-m=v#6aH4igu;SIRMzyFR}t|e$FN(iI3+oQ{f`N)z(n~tB#J&k6DVs
z!5=#I>>M0Y>SSIB>J;A(J7GA=Z`Aem^-TkBw@twh#I&?fh;B8&dn)NX^y1=y$j1ku
z%Th;Y4sTHuFndJokaF*@ZzyB41}&QIt@%$TZhRuwW7d_|V`h<0W$N?m#-7CIib7^>
zM1D6;+VR6)@n5~H=tFi~K6XB^b#`_>K0XEp238t2cLuBoCF$#(O>--YP!bRjsH;!@
z=Ubg3)^oXO%L|J5QC?pDZL~FcPf=l*-(a;&^Uaihb&lBEv2{2c+OXM9lM&)`I4LV}
zbF%ukH{3AkA)NLe2Eo7)GWnsanaF57(lJx5nz-n&BUG<3SVAS9;A+zqWohuW(T0F9
z(dX|<s)XQ?NR4W~8i<vf8>9R5@*jY-$`U!)q~x)fz)Bjl;NPFMQ~tA;)a7_{X(7|9
zQC$9mbDr{|P$p*f_O2vH?nU39wS?T1l*@-}sGpzTb1I>U{(cr--a7mFs&C)EnfJ$K
zzjd>E_pbXI3RWG=!FL_j;E7qx6Qsnm3hi;2$i($6?CCy$2qitudcT27Xbmb2*XK?q
z_bEHZ`N_gpCnX1{eS9V=QUAJA0vc73zc3i!|C>~g4-E~Cj*595uY7-F-(cYA;$mZB
z68K{5s}98HZjS{CA4NtxQB&jN;{(g8T?}pTv1iDvL>vZuwk?w>6!F=g>9}4o%-#Xt
zbXt$+h+1Pk!?ycHuw+g=vN>Jab-gB1-SznA?((2Np0>+!cqd-`ersEwTO2VWp`|d_
z=UI+>H?dBlP+ZsEu2nf#L?iKumqNJTt3fpL^WCAN`7zeXa}XU?2-9eE!N5G<v>TPz
zONK&4Sk=*T-@#>yC^u`$&;#Z*F|qLx6mlIIqshd?1Zc|FyBu%p%fQfJ78M$3nrn^M
zHI1DI8r+VbB6c{HD?0w)4>uRrw$GYt)tW0c2P-S<>({62J)uCa-{URpNOjY`acOF5
zN+xHmS>P9sQYaYX^=0;2By#P!^by{7QDpR{WfZ&oq`v0sjbQb)N?$Zqc|<@jEz5a&
zM9$jVP}<Cbi>TyEeQnDd|Kq9suTq7qTI~V|Q#&SxHsK(?{^wR(<j%AUr`^}l*V^cQ
z(P7td`zr<WH@<j`&i(U$LuoO4LEZ<I&1=2~T%@{0xZAQ|y`RleACqU$P-7oxGAb_O
zQn}C*yll?PyIv*NPjcUzC#UIHLJdx9V<`o)e~w5=%e~!nJ|JAU^gPgH@v1g$IT~oN
zoh~RWM3GC}Te))<D5X?7p_o`|MkdRGIQ44sYQCTpqVl+U=RO9s2`!Pla{bIs<;c6}
zy#<3>Sv62h2HIHba4mvgAwuNS#RGgTEQ%fs>dx6j6qSWee2>_#DAk)Lj6AQ#;d4cq
zg~Il(GAeDl+oT6SYR3L9CuPelO0u*Sa3%HRt)r_GF}+J>G|>v;Q?Pm=gy#MAn~?l^
zc}z$Tqr4=wGjCX5aoTzGQoju5lbb(MU$e(XUU9y{dy4H6;Cu8?YYR~*?nFK!!g0x6
zw}(>n>#ev<+R3{8;2mkn$;rNlHw!1~v9{1u)+naFFfK_{+ALnn;}svkx(Nw+U*|kr
zD`icjTOABsr1FE=8F~dquXiUutj7Xsy~}xRr3>0E=I6O8;K>hB!z%b0xCwtiFdZ#6
zAB=t8Os-JO_*qw{ad6bO)lW;~>ry_cw+GuaG*Qpj@C&%^ZJws_8F84Bxc$8eZ%6EO
znS!TXM+1jbM|1t}7o)kwkiXo<Thk8}6%<|wdlI0V<eT9{W1m+U?JbohUFriSR$HGf
zGK5h+kufrnH4^w1)r~*6u&TeEsGT9`m1)Fv)>l^_<1r(1SZxYJY4zIs!=n^mwK_GW
z>h$P#UPIrjWTp8yJDg3}dUJd8&}qAwP*n7_czI?<3-p3nNyvRSp_q}ITeVE<PhTI(
zx=#-57EsLwul`;i%vTd224iOrvJ3w#CQcorYaU)$hui6aGN)g0>et>4!u&E7s{Ahx
zBFnhjCzfO{bq&Vuh`UxF2Bz!YcnDVunq)fJ!Y0O)S*jK6m^N)ZK;-1)rw`5?1or_Y
zoi}yW`|jTYCv|h%=iG36e_pIvpLT6=IO8S~DRsapD~URA;Ql*%scW2}w9Q3CRg1iT
z^U3SNK!K_q-u)lE8l7e}n%LymADfRSzLc7(U`=MX8??CmwXXK0=&NA|mHet-9>+!z
z0O!G1YtC5bw`;+8S2{ef*m>0yvB!lkCnqkj<$^&kv_<6|RvP^#zIy8i)vFJ~v~O&8
zsfQB7kRA=GoU-*>B>x!LWqZnlppf?a?CY0rAEx##IatR1d(=N>_cpVL#9oD|cuE@*
zn(>m671KqAu)a_3j*g+Dkgi9`%K1w!zr~LC$+q~1puDK2$TY2r*zK&~FxhZ)T(Cj$
z<i7jUBSSPf8|dB0?V;dWrt^+c0_a_05m;2b#rjjf?`Vq-WLv1y<6sWtE}U(N^3L1)
zr$X@(n9lMroq6D5`EkiaP0hRiX4ykHd(eWDjcrS4Y<O4^GMk!~CRq2`I-|3pLaFjh
z+~c1%Ub9v{Uv2XuMc0&5oA|1nN&Efm<ao7>*8k)$lSH7`{LSYBK?nbvzx`LotB8S_
z{!WC`WJlElUcu?vV!ao%4eoyen4rspdD0+-qed_?SoQJzcB3!)?&_$}f>>4Jc54}a
z4@GvIUfo@G#Zn3aEY6h+zS=$?e>g8D&DAa27w1yf(x4(Dvk=JCPaSPZUVX`*T<gOf
z=L`*;PK_l_Nl6t2i$UjMSeL*0qRnO2NC*ivOicdNRZLpknNR;Aq3`+ZXwF4U_H}Qm
zWjezR7HE8Zu<+U(7;9`WDkg%0-{qusvn;?*a4UfC&6_u`U#mlA4OAs;8kfehML1Fv
zK%(QA=;&HHI!!i{zkEu%ZVtL~{5MK_m5^nvU2AKPp{f_upffM<#eR{h09N4=z4_pL
zb^odl^gh^l$HYe0s>=WM&cgYih*s;3!O<ad{dmvxbf~-+LL$0!9Q@qEkfa4-8mn}#
zDpR09%>RacGdm!hS=p36N<*$K9whB_$LSnPj!oSA14YWV9GPg(k|TVu{N$bIdrOxe
zs`j+RL7kt2bF0{X_Uh%&+%3&F&{Mxn6kZ>*RnbY4(zJ5Ay%|u$2vq~Ulk1beX{e);
zE#b`R>f&EuOh;gDU1kEnJ&Y{&&s(>h`vX%XDqYkD0^WuEL+M;$l=f_0!p#e9WEe>X
zTA!-2Ja`s`eZDQf*@o#s{?|hD?jYIRW7h91i0o(w2U|8wQI~qWMdS3GTekOCwGy~S
zcDA^~P?t)Gp~)#Wx>@ybd)X6TFJ<cm@y9WcuZ&A6Ba{`8xWwO;hV^?_h}?yfO(dsd
zl+kQlVMctH(4b)t$OhtZ<yC)(t7{2gYjqtbe=UH8@d8doHItw8V5D!CgFNq#?f0NK
z-U!$fe&R{PAOfk&93emiFOQ8hAVdb9IVN<K=!i7#OpCM2)>}w;A_y0Q+YA6LI^-y|
z_^Uh-0#hu;Bzy_NJtn(-bZ*bDkR!rC{Ax6Gvp-a+a#TauCUd*ja2WVzoJ@699fc?M
zSfH{*B(g>&vIf5(d%wY$kL{mgqoYOfJ5iymL4)^m;;tqFbU3x`@0?v7AFnDkj1R&s
z1WhpIS8hXJl)eHbFcQ2wPWc63B*nDzgUmg0z#sn97e!p%el}FD4S8ksQ%zf&3G3JC
zX8#L9LZYaWJ5=GmRIcU6JNy2DZ@pw+0g39$t)>NqNq+QEbjGzl;LtV!J0Z?*VJ3RD
zpKe#ZR99*~e)#n1`d3K%)zJnBcQ>uU@A|mbyuZr0tpwb~2ns+vTWIrwFVvdPSLm-+
zLqVhrHy;O|!@I8k?$1I>m3&U1&h1w-BmJb{wwrp@F1-%l%MZ_$voD9DxkVAGDTx95
zkt`<ux3|uu0bQT^&Q3Ss3k^Qgirihi;2v<U-_D+J+mDJVjuiHePKC-j3z&tMyj)Gd
z9CRGK+x<Q^tDoccmdl{w#}8Ir_LSn{V#-H7T4+^(1w^N@yfHp{+O82YUqK_{Sy3Gi
zzpYRLcJ3fyVR#4p+}zHG3$>sMmZL_4N&s(d>sRVf0)nomx6tIc3@9iOJE>eh(bN3g
zd8-^sik-_PZUmg%>w2ew<4=gDalgN~;N`v7(Y?>p7AutQT?x(HxjIk@xaA^AP(1tL
zw!lESGBzV6M`Zmvra&aT>Fs4VW*rx@(+Ree+WYG0hP>Od9?R@@1EGuLDLy%bq;~l1
zO{IMP{5jz6qPC%Br}PbJ9rsD_N{lQG>N+WnrjFVtc9nT_k0C|qD|wC2wwBQ@rikx4
zy4+cuiKrrtcy4uf#Up$KwMUh-fB8+65w?quA|>NBM8Cz)w49mH%vK*8I>1NN{5vX(
z8SU^Y73b2byZL&w%IbSf6nUgK+rC5o$2~7o(P>Ncpj^@iBP~5G`@T|BUjy5RP;Ek0
z+69v7+8xr*%S0+I%?HTGW_VSb@0aj!oIe}V9Ptt-BlRz44lN67Y{t^YyJnKqeI@wC
z_PU~VZ(R`_GsT*;ZS%ZQn}Y`)*|LpKe+HKQ$g9yRJ7-fgY4}_8R!{xSMU{BqQ8{Au
zrZ>Kw^#WVl)9*yQSl52hvF88>hWvTmRGO2EUqry96_J?KB23A@p3D7bXZN<cD+{5K
zNuyx=BT2t;)uQfn;Mt0&)minHFogkEll|ZyZV|;R^R#R{L~!Uwqq09W6^E&f(1<$2
zrkV!c2~+FETw+|*2X|@9ln}ZF(xqozCX_S8xMp1(27a|VI4ARL>{ZwbVEJ`vf&x}~
zLdai^C<ULjVfV&FuIfaE@*?a|3!ZnB6b6vAIrYk9_SlRU=F3uR;&oj|nmzAP#z%hk
z*WDv>5w9#vwafV{3k~vf80CO?6#(>hMl$;P`V1;P&$b3rDRWUS4wg$h@rL%HT~J!E
zgO$D~Y)lMv|M1Y!-VTyW+^U>5Zckrr^>7cR^h(`(n-sW^@Db{Nepwi=V%Xx^m(2FO
z%BZDIIWzrN!A8P*-m0JH3K*JfIUKk*T?+q-cfhRj(+$41`dsv6VyajXkl%;pyT+vB
zj{X#^5^;Wyo?o}mRRgxBRV4zq{>q{#`MEEFks|O(+0uZE+~?LxU@A~Q&Uxo|<mE4E
zN-)$LakM#CjdTPY_-Z6#R@ZvewHW6s%J?~MVFIIkvDR7ycM0Az+@TKF#rb}ggHJlH
zXH6zzzF*NXhv#DAVrJ~#9Dof4>DbtW%P%vgOG8o7C>Q;VjEsP|G=SRQj2aS=kf8pk
zaQAH9rxnrgi{BODAHnHvOG5Z4&!>?ZtzE(L$9gf9?O@N}4dyj|#V|8)>904X8nO^3
z$%yq?wdf>VA4MS0eu|0T<S3}~WLg+LJMdc0=!dQfysbLN3FvJDKba9{THdQdglNwr
zWpZvNq@FG$d~f?anHY||t>%%?t{z0*Z?qYAP7w&u%fqr?@hB{QH!G)n>w01HDV`?P
z0cLX<6BF}{x(Zp+U2f|2tQCIzDt|Uj>q~Mm_;8rZO7arbl-wl#fb)ZhG^!2hM&k;n
zd`NfAVJIJsd+S&Ct!wA;vQ`Pc8aLxGkjs@(oW5zJ$0P6X2R_MY1*M+%8+RBGS&4i9
zYMs{PVeKxi^z(}fYdnm%gBeqN)Am_k|4M~*7HnuiBB%c}zcq2VgX@}(_*OxNs{0oD
zt^U~PsU!w2<^);|e#gx5e((8p+-j%EX1dia;Ym<Pu*9%X287;t85D7*BlBCv(rVjd
zmuTNY$5D_Rgz&b&q#@ilhE43RSX`+O3V0lEq!^e>FbPs#p^-{Yrr0GiE<K+x{Atrg
zaOqQeh=WmsZ3Ir!e23qS9Te~}cC}0#5THag+b)i!t55l#h;vOBcM4D5M7J|4B(k!<
z%_mz)b<LFsG0ceN`7WKLzBTHf@-4%{>rA3nG29m0b8o-$K=4RGQQ~2|pT*B_K0>ov
za13FkuQR4CApvE(j@g@It1>AB<xTz=wR-FhMgzh)9_s4fSfldu^Pym5fRWKk7!E0C
z?yQ}tDebD*^;*5pv{kl>_1m~+8a@}R^U~aLpN-SKY>|1m!<UqLC0a5vH7$*)dh2oL
zF2jpmhWLk@&F9a>E>4D0I6i&)gdH9AXVr6=dpK|(%5B^G)fC=tE*jK!Ooaqw<pQ4D
z;Iz8$$Gmp*37z@FY(h^c5?%?GxY{+(S_D-ptjC06KKH$BOtV(XWjkSx(+zms&f*r&
zt@k-E=?M+5fYt>5J|U>L?z-FWvMQT>W&A<AwqnrXlYSVZytr^~R1|i0TC;;PUw?6N
z?m&nC+X;yp!{%3k;#H5|KeqjB!n`^f9PD7k3K^5d6l&8L=Ec@2Q(CKik+>ktUalK|
zjJNEqLr+hCdV0#Dk`)mht;wDuE-s$BxtX4xJ~X5NGv!yu$mTBRMrG^q`(WC4qr=;Y
z!dhGD=?a5~cJa}Ox-sbtKep`6$BREy2WYO%SC!X+SQP5GuH$B?TYWELHTy&)L?}9o
zd#|El#QPLrLXYX$wGvzxzh)hHCEo$Z<yxmW!}4vczbzHWt#9`Ea|Sx0-|EVsm%4$R
zVh58jX8nVK%2(NMy=*6rt_==|+=467%8^#=KfSM(3-3p)tufd$&SAWsaIpw`Yb-9t
z!M_Ja9!H!gXjq9w=&AnaZuzUPA0$iz4T?2THTL1rUa5T#o54MYm#eZ(HQ(hmWMvRy
zJdJ-$pQpLG#uzwO>j;B74EH?k8g;9ZO$MrXO*<}T!8@p|d?}T8wzH02D|2;@RJ}Y5
zB4H@I6h({Ezu=!t^s`K(1t>^@zJEMEp1+GW-cDrHul8IVtI3$OQQs7ywS)~x;_>BS
z<OL7nHYR<)${Z1=Sqxr2dy4gqnj$MPR-uUvQM;Tr@XW}9F|Ay_-~n`fINAV>=p*Je
z6cFNVEw4aN#F9agui)gkJbg|gHQ70Fi=XAdtZUR_^a+rmYYiQvm8NnzCgP_BAoVSb
zZ!d&mBUhgtD`0S^a0rDCkfX2B$+xHLy3TeMI%RLXH2nJNKJaMqJk2Ei;S@$YGUel^
zpHn_XX_EC2NyoO-5a)k3*%lf~arUt`?&)d&v`QT(VoH(w0g8Oct*(ZTOCbHP4i_UL
zus~=@cY=A!ej(lQG;e<O_ghbwj9Mkh<D(Bl@!3-pN~b?lUEh+r(fZn)oqYU2IpIxh
zjSNLY!-<aeyV`BMS(c7h&V1~B4i!5aXdvd$X>g)8HZt<R&h`SO3;nIyZ_>MZ^kYFy
z_Xmr0rVfmclXml|ilcxGh(Qt(5+EQHAqt8k6-RCib9bh<*-n?d4}DT;id>E$U;qT(
zwEFSDlwy8L6clS<sIGntJLo?*DpfIVc6j)36>Tr2zEW#HpU~5L+~%d=Io78PcA$Fe
zpB`;&^JdA(SQut9Mh)&?Wn$`M3%`!15kIs0`1jjPp>8#aUeyEUlP6aP(=43){66zz
z01##8<P>thQA2j(A$?j;PEG)wpPo(z0>lasrLr<0z)0}U$S7+Yzss~MFjK(M)m2b)
zBVpMj!2j#xkN`ZD;p*Yc8UB-i>w`ZcruC?APc<#uWFPtwo3Y-k)qQSVkf(xGXBc&%
zV;?8)creXz(dctcnC`N3>sLAGmF=>DsM{nF^cg#x&Lp?`A{u1&3R<id&e3=zRb{lr
z&C#)evbP@@s{nV2jeid6NguQ`XPT_>I*+>Q`0C9@F8c<>f=Ut3X8QFA+_&KhLTVy>
zFkN0b$NX(9t6~~mD_4hWgMU)|?i~{v9b!v72qSOq2ZlTY8Mc0P0wg26f}@U|`x`{V
zhUzi+=43N2G0EM1vw+(2Gt%2EV*gH=u`X<0`gaqw#>rg!7{yr16tq)E)ChKVR8#yA
z)}!aGK{++)@@7=+<IIz<EYdQ}c)OPR_G|4vUaKBy+s}^jevE}#U(0&XjHOtonsQy)
zP8txhOXnwR%SsYfQ!W0$u^r|)<D(*9b~<mxD=FYtb|IAuVN4z#OX&;S9IO@lb9Q%_
zHL~(?lZDiz<?x49r7|;R{9(7FHS@dDI$DElzY2%@I%N#I1q1WuTs#?a8wnSgo}U_c
zxaJbwMJOE>f&~urFD9-u?&1c<dR?cIQhJU;9y+Lq4QA~_TO~bpV{n@?P(YwUwbTsh
zl;7#ngPd40!Gcl#DXdx~x}OQTb%F$=veY+}3mD}TFmvm0o}Gao`ebwc_sqejm2XDT
zCVc!b(9x5%$xOkglfD?v2luy}AndST<Kyr?Qqu+^&)tb0jXKMNCcu8ouK#h=PENam
zFZ*5Ws*{tKl7D^r^yy+vU~e!QCO?I;nHk-1onl&*8JX3>1pTFB%kjZI{rmS}(tRDi
zR|X9?nf)corWZ$o)1{+&wXQx|7BjOUzew`wUtLVSqvH0zyCpRZeC;(|qKt<5Y^COJ
zGXaCl=;Jk{{vc@w<x3;hYQ6LL&GPY&3eM+vPaPN8HzvLniV?QWwkJsnyX>j~f&Jy?
z_%4^%ny@2HJ)_^}AyeDqXDF=fY;lp1PqDG@9eagAh%0z5zpX;_#C{k1SdK`C?K!Pi
zETyiR78U98*z&U9s`CK_p9HWoH8?o<A^cgRQE79>8~93#TUb~a&?-~-`pp}U-HH4-
ziam<I4<}RRwW~RYk%NBzw2y2)#6iz`zUwwo#gYryV<E@+luDaP!C=_W;PLT{TPFu1
z*sA07y6nS=TlU|<ru*TNym{p9g4v<`i-Lku=NRXC?Ct?_=rgD3E=cPQWaNI9(sg5l
zKj1D;3*P85i6L?B`#z+n`Pz7zwsC8NR%csxyW)-H!K%;5W^*6PGka7a-z~ZLqNh#j
zi`6;bJ}<d;jM}X=TbHOD+20=gFm|{oK@;;Dxi!)X%XQ5DAx!~#vitTbKex*uWt}P8
zN$^kWr{$a%oeyW3E^|D`ExP-SY}Gra#%YO^Rbz%{lTILEQ|REhD7T?an@)tFi2u^G
zTN;b(IjwF_8+#2DC!RsB^ziN9J#JFxm|xCsBlg8k_vo^F({7EE;0AZAaXMXKe~iDz
z$B(k!L9IWDaJt?Sbt4@T_1xZdMLxgw;P34*ahU2=4LDs*B|E*AWOiRL_Udm=%HW6&
zV4{8o7T^w_c5iXBicldZJE#GqNJk^TDPGod-QyWvcN{cOEG<XpBMBJsSA+}VeK<Ss
zsA~(OV_z(>i|x?#o2!ztT6g6b(uKGMjjg`j?#%5()X)Bj=e8GyA{N9{uq%zAd3Dn^
zrn@$}4Hswo!}Ahc`c9~H`xt9<C(Pdjv5mr5$)iSmgVV~%);Q0Y)p?>5v3gFebDx*y
z5TSSa&3l8ieS5tmDl@Joq_UfJud7a%J!yoqeCs*|H}s8%It~vGIOH|7v}RH`bUAZb
zb=ZL%sJpwnoGzEzkE2876`a)-oX!M<SV6FuB<59)s80$CQ8I<6*Voo$J87mmIl>-y
zd!<%$5dEE%40qjqViMw0RJZl@;Jx_Kn>`q<wfN)jv+ZGqmj-+7Gd(?mTbh@m-2P{`
z)~11{P~}WPJSB>spEvKmG02Vw0%ob}%AjMaz0Xv{D~{sz()hS)^6$Lv^#IH(YDF<Z
zQcB9Xf-qKWAkD5{CCg^MX?3A?v(D|6+&dqAAkaOy=}c)@ZPJN2?QedB13~kR@veJ^
zJBE1w^Hnc+=R=^ndOqDrD&h_{@^H47!G~-+0n-MYd>$)~K7SOsUMxN6EX3~~+mdLz
zJ~TYI-iwvlOfwB6fANCP$Mw>SkXG1zz*k$Gk+H}PHU`8)hlhp|A4J7!^)8^?$JwV2
zK}~`E9A6TcYpwgM>bV0r<O)o9DAC7QU73UVB0d>ZuV~~#3W5nn5(nf_SrocoG<V#s
zOmh!6)t|JbV8#Qk#$AU&>{&hX(Tv~KpmM+1n&$QQ8{{jK*;2o7mhDb7WSjHOGaBJ3
zd2d><a6b?l#{ZI#kO1f*PUBXsd^Nemna%(N5F9NnEvh#9Sw~mhJ$#HOz{<ZnxX9;=
ztK_hCkNza={%jaqv98(wGh!B^+vag_w)KL~$ocoI-}Q%(P#kO=zstkRQ9;x3^wXP=
zFjKMH-_xex^RDC7?>C0vN`d=Fgmv!NV#8iM3R>cv^Nr)oz-0Ak;5M}0T99rOZFj16
zGqp1FxXZcD#QA5USBq)x<5Oy3tE|C=T5IQb@7_UDxoCZCzUSte9;YbE6si|%AX}Y|
zPUtViKm0mxXOtHmH|#Wxhxy2dWbg%?Zl{eup6d^5g1mE*=uT1RgR1}5Tee_a{0oI%
z&YSkfQA1|!{4V!__;q06<PR-&1_lQ3!{t1HFl1>1lBe2V@a6dQtkUD3UH2(&Xq)s!
zx0|SuVf<n3W|SP4CoX1Y;Es@y*^>H}CWT_QzU3z!Uy^+~XZLEy$mH^XBt>Sq(h!kE
zo&g89?yfy8>}dR6XlqSDlC%Bc2-uV^l#SPc%)SRKNLU!Kk5lkt(ncqsquj9Bsq#;w
z24wc5`bY6V@kQ(Uw|toE8(|gIYe?vZ1PQ-uph6T5t@nB$DAfYKy-iF);(4fN;Col5
zl1*sjP(+hEg@gCB#XxNV@TY)nRcH)(JogP$_qnpq%_1zv2R155@s?0OGO{BvDG3vg
zqTOkANxkq7u<w{6Xaer|XuC$}uV&VLmNMKQPsttiX(x;?7^MfWMAz_TReY%&7zg`M
zT&~>Z)!Cd~hfbBzWTP7_Lg(_4=Sly0(C>4q@z3WZoXK?Z3Env{pEy!)AbmX!={`m!
zm&kM=cmuht1Ux#LfQw1X;Z#ct3r1$<CTHl{*=n<0qwn8eVIZJOeK^ct5~~HnR_nV4
zO~`=OQ=YR#b)wObAFxqkOYL^X2xb`~M)?FO0$emE4)BCVu^<^9Rq3x*3C~;1C(13>
zBi(_ko^XjK_sbm*@S#SHLhxo42}v_n*){p@Vs+<5Z@eN7Hg?)8<DohSvmj&*JsJ}o
z3vB8u9wl})Nsh9ZpM&oHm;U3NyA3T+|1~Z6c+46F9UU{7SYRJhy%Jb_c)Od?SaH^U
zyFU=L)PV#91XP+z?PBZlbuh{UTgxT}j#gH_K(y8enGNh<Ki=+H(@Ka3$N*tQ07GaZ
z;TUdG_~>z)o`(zKj!PG&^Oe-%4!?)1+l3DLNYBsD3>vVs%d{%&p`E&l*G`cve7vcg
zw}C4+A|auY>_fv<CZl|AR$o67VTUDZKf!onZEcNg3@}u3cW-`Wyg&PMTU9h?lSzI9
z<FUg_T#a2j=!<>$jp(?1;>9E;QoU?u&NT;y@m`IUC^n-5@Iqm^;I%VpZ^Sjcndk{R
z`o+O~eBf{#zH9VB;==aX;cGxu#{32)q61m`LGNh8bh34m`-4!VBK`%LbYdyp!W|fd
zovY_2xwv6z4RN@+0+vSZ7D7S653SO&Ujv%m&BwMw26hvk6#0N~dqwYRMq(Q}{4PN?
z9;YOQYk6kGUgEAW+OGRrkJ;35UGQXy`SQqxTVGh8vXG*vweN9@HEbj!GdGPpYU73K
z2rf1@5xQ<XvGM2yF>#7rEHmPhXJ)zp<};eZ<<jvH7Aw)W=Y#k6!+^gH^rsr`W`3cI
z2$PoHbfQJ{k><vdyDekVwqIYoP_@b#c7S#oe^ghGo*=;+zF<x&(;iNJ$z5U6!OO$?
z6L6HYEZoiBB@EhBOz8qLb|g_#Ro7l%F6j?BN%ir$HQ=sH(}7Id7}**TNq3mguEW7E
z#YV@+1+Ryg<SQOKJ4Vg{_LuOxa!^tl35(81)O+}Si$i*U?~nH+G%ZcHSH=R!*bRS1
zy}#8Y0&x4AJ@?i~O{2RrV%VC`QDh9C{^^mq15P1BFVx?%HNfWuAw6nDRJ1{}#cFM6
zKS=72be6L7dLL?dM?`_T>FJ8wxJgSQDIsBj2Jc;YD|l{vXjTahg8_y){Miwt{Gw1c
zmXm|S4}n};7pn4bhse+*E@?uv2`UXTMt4ecyxS`(4*OuamF4C8lZ9=Jk-@<zcn8C-
zIo%Edh}+xCy~Vb?ymcvqC15u=hWQ1WOqtm%g@=uci;|>lrH(m2Ir&%h8f>Ovz%A9C
zs$q?KQdq03G%~ITX6LR>s6r_MA%m_ye>QA>)Ys9mV*AD@mq<i}ztG`VFlkfJ>jWh5
zi^}fZ|0(tXrB;8^w4ce!;L98Z>CNKZ1_~(F0pMZaoVlYcP&e@4b9T;0vQ$g1{$mpp
zgM%(X)Lx#{(7lNXe9+pONvuPHr9Le!ZKaO=nZRBQGgzW)ZgP@pJUYZ|kr7)a-%LM6
zqfDc0#;)<-tK+W%FV{%spG?^;|M@K?EhU$5`|F-&^@V*rK}>OB$%9BFkcg<vAonup
zEYa0j!a4VyDaew^@)zE~1xQMrKO+WDueKJN32g_C?i6d3P49hgw@kGJv_U{X<{{w|
z`Y#EFxpYrK8lELFonG!znOOCUS)M}52`l%R;f?&W@hA8|W+0dwm&|0#`x^p|IM-tm
z*{C9*?YRH=@qKV{LQkb@=M~)JKk1MB#9&%vNFwTc!m(^Iuj`<vO{%rH%j^|vw6mS>
zjUM7eH%y3$iDjSDx_A+N!QW5^@*tPfAyuVuS&tjBK7G2D;%cO1gf|<AapsHN^<#R}
z<UJt&6MtN0{y(LvOhyhs&8fu@8m8t7H5$+*=|ofl<mw#91^*-Svc?al1ENIU@JV!a
zt7q5LOnh#RMGqK?13GQvIv*a>`e_2XQ-mELcSr$_f48r&a<<PFA&$%I4X%H`0`Amg
zlTXBPBm@TA()qhp)DfWEdi~#r^3Lx8Y3$pj*Z=Z8H%#mrmzI~?UUSWemk4GkCGP-)
zCK!HMbP&i;s+zgKLYP_%6Kg0x3GG=8efcj})Z<BLzD`|JwYmIx-C#}?a6So0FU)id
zX;!`Z&ny$HM#LNE+4G&p{?YZlR=7(dJnny{4bA!hJalEBBB^38{lC<>XLp*QXIeIW
zj;(o>Cg^i7*nk5JC>UC(I=)<}Wcr->GYG_k{e(H}Hbz4mHLUz&tPK&S5vTLnjk68u
zzjcc{ZH}+wo;kQ|RCrWa`;gH|qhq9p{Z75J3pXE}6>EtIWz$3CEn3fdONHh0r6
z^2{&(R#saZWM(+6e*2~_e6vo>z#e;kKRO7#aXw0O?iE1ez3~LK&n0jij;9hH>iFSw
zdx|@E@#N;qYZg3MAR6c!tc6FN8qbDkr(Qu@oJawSYCJr8|0>9WtAYO|HdXa58rkf|
z=NfkV&6ndR4%|;H+4n{oyXd8QDu}hRK{ZCg5CI()k3|}kB%z>>N}Rd$tZk5t0DO!%
zTb3TEq0pP|)faMD^hF<c<JvRrO39(Hja9)%Qq1<VeF@GmrHkaj%x913ciZu$omuUB
zbT0w~LCV#-H1gs%=KISMC6BOgqnH_<(-pA|?VFXzlEAm{-6vgtofibw{?^8mP#d58
zd8~}ujZ7spArLcLdz6;#GIn?Gsd0`wQxahjL)H8IaZeDTdmVULn5$umnupc)kV0)p
zN^OjfT#!A^c?-NCr*|2h!2(pH=q~d$Lc_hcrdM~Wum|#Kr>^_0BjjF;Dk&I!y+1d{
zhvgaOaU1LxgZWzIa_x;oWhF$_QsAswDPuienpVtd`S2m7%LT6N*N(AvQ*-AW(FVP7
z*+U2Kbl9%=O?Og40@g#HXJ25Mfq0rQn28_id`~(+Cb#!})&<=if=7=9@KrJEfBYY`
zH&>Z?v<XCzy~5oOhDJuv@eKpA4M#@W%>vSY@cVIWG#7?FSsKxd=GxB45D6OaOHIo*
zs_lRk6ZbJ;s^TOTf9r{?-DWPzX<f@}T(S5SteIi?E`C7|G`N?cZg8O-6ZRBS#d3%2
zbwg4wjpLI1KqU{FP-4d3^9$vOS(_OVP1VKUK+1E`;>qP@V}Ov`j#4WF{Hx}^0ZPzm
z`S#!2BGueSo2lu?w1yJJIbld{J0iy`*Hp|l;`7v(fIm;2J6oPY%HwC;sPEsW7}6V%
z+*hYa*`nl~7_@BX7XlA_z1*Z}^JGNEYn&VqEJ3xI+YjEq<3Qiy_uim(4`x$xNprwd
zecLg3nW0T)H`8-=+)kG?eBPt|3}3e&t#5d^bRMi7vX!IPxKh^sc3*bUo3*3AyLi^6
z6F4Xz@2Z@Ng0BxQ_EPj3rvwzh6Fa5467xG>K6$jsm-}VSP!wL$z_Z$79u-5-mrh-G
z)p(I7ptK;ldaK{lQ1LafS}4vn*WKfIND@*q`s~3&V9oMEf{`Bm-A?d+7yVWB?N4ZO
zOSWt~hU4W27O`^W!A6Ra#je-o(Bz-XP~4L$_v;Y?NvzLktqj#BS7->`V)1&*gN3KW
z$j9THQ_6AOTt?{WF;RMx!y?Z*d3B57qZr;cB}sLc*hvyg|4wI{hj!Z~1e5N3HNI5L
z*70{!&%tmBHD7Pw?G^QtG?KtG84nro=4$LVWeKo#e}3*z;@{gKxVE}mB-<wN`a0H2
zy~ce{#3bJP0rjvZAnM;SN|fnFd)?*!loa<Z*o4{T<+c)^t7DD^+UhLGWd;{SJqt>(
zlR?LaRH;L!22|?r&jR^cA?hF_#0ylAfAt&hshFr%-v-C&m+$c;)_;&c9OeMpbBgok
z=4L_Me<P9wkZj6d@^SoVm<~D09Zgv{Ti>Y#UbgvnSEXUMNXm*WVfqZ6Y66@sN)sSU
z1H5rpNFjWH0ceO29lH2eu>7x7{r^=JIRg||f}yeI+BezAT)<Sc6!Y`GnIF4QU<&y4
zPusSH{kKjL`_)okxDGf}CAp%K$e3#RhC<n#;@d6X`@%{7I(0z2zpT96jS~g_`JFUb
zdI;3<|F_NhuPOWARrmi8w=uZ;Kl=Rt?9TqLn!oT9z@tMY?B>6Vi2#Aj97+C>$}E%7
znS-9Rc*tEF9ApxP8fLIq<IkNgWnzaBCRy0|iC-3VqP+U2pToapz_+~f#mA6eyjpwT
zYu?+ai>on=#=;TZj2x(A&Sbxmg*9Ow&rO3QA+^&8cN{gc>MFyZ7d@~iAiWN=#zxwA
z1?B<3e3V0Y1H`<gnN`|ENJu@8R(yXtYxLWS$yA#kx7L_rfZh=)t!KD^6xecSyAx+(
z|LDYo=+^2A(WCO%FQC@g6}~+Sc0vIOJl5cJuVJ)RU(;h-t(k_TDSQrl!N5ph;Mm#L
zuM8*w@<Nfb{f`6P#|G=o57xYpqJjtV=f7oe&%^>u5<w>guUzjL#LQ8!`d``A1tQ*;
zabliyL!Ogy)CM5__7rX~-Z(G2s0}Qhuh*k-;&}!Xjb~siKA3oey1#RGg6Q@?g{yqW
z&#PSWqbmirrYoCTv;F}Cvb$tsv4&VIkV{z})jN}xli_*8)XFX!ANIh71x#UEVv#uo
zk-4?`_lrIwdWt^wd7;mJe~il+>Ud0T<kX6Riu^l<&MGYP-%Q*YkoNw0cK8Bh+rM=D
zr;xf~e)u0O_?lmTeVK;=O8>GS?h|73plPc%KL={E+zFNqf4vKR{nC8-l6ac;TuD?I
zg+N>ZL|WZwW?ZwsAdLRb)hd|KLC`Bc4Pac%JS9_ZnRgt#3SqxL5DAF~Sw<1F69<8q
zK)cEX#U}4YeYgZ$mV&qApLRS2!H#<X5Oa#|e$`16<;sh)8}bq*=kgitT|7B`A#HhJ
z95)(W-5TzJW55EP4Cp1qLNB{5ToGX07yk^ni$ApLqk64zqSJEi<M{6$G{j9_aeAFb
zB511!9=9*>>c=S7u=R%Pl-IIK#lf)U+tP=^R*XjSLbJ;&kAEzBR{%`e<sz|jyR}dL
zLtB$iohHXt3iWM(n`_Q?nM9JRMBeLFiE<7cGQPR{j4ThwVyet1`zCAGFjI4Q+A-~R
zQj#m7ocnbIuzvi0N`MElLj8g71=dYH4=M(#1N_G{D4!EVgjHjb7XJ~2PSM39WAk28
zFSdHawR7uoJS_(!CKoIYU1b1{&AHn9TrFDnL&b+WjjHG8*>5>~o>NcH8$MNgIgp$g
zr0BV=FZqxfu0;M~f!FSVTNV|y0TlJB=%?RLKz_ap4qY^p^i@HHV~!{u-xVd2-ryWG
zJ>={iM*#EWcXpRutF)Xa-mHZ`ds2&e`&}31-}Jxco8K9#yj6R}-$@Qg4|?ACfAC&T
zEUy@#6E+4}|67eY{*Pu7C8z$M`v1}WKh^)OQeOYx@BTlo{(l+lzmPL0=6^tC$(GOB
z>T7`i>9=ImkD*;%<>d<a=w7lw!<Pi%8cxXc-dvUO!5zCowo0i+*?%c+7XNn?*rP*7
zecxS}o143p8#pqF)pGe7w@_^cP9wrJT*o=wD&qQYkQID)`2Qol{|EX1KUb5T4h@rC
zs($JHI!$TcJ@)x`z{_Z(Pz$|0`E#cM#^$k_(PonpH;NxNyE}GRwIulu*K7WN%UAU`
z*tNc<H1Crc4%F(~!K&w#c>T(cPh_+h2V?cvz$kKJY$y^KkWc9&u7z#ppCgxf177zd
zI9P`P7r-|4o_Od}oR=5TtIvid)$7tt`dOF%9(`DY%l>^(+3Nq(8h%$Tf#KL8tqqTV
zbRKXCI*w8(;Tb^6e4V2LA)k3sPPSd&sLTGX9PyNjrHUvl*`FW4%Xzn)rZMfg`i1YQ
z5%l2@&6D@b6HI>w&z&a?J|CJNFMS~P>z6j=kxdx38l%k9;0w(iSqaO)3&$Z@Xo+4b
zWVlQDc6XUb_R`q7#nyglNKS-<^5ervNa43Nn|a}F-mBxs!_w)-9OK)5^|tSCbgPju
zEony)6mRimm1l=z*LMFe_TDloj<#zPZJgi)3l0hHf#6PpdvJFN4Z)oT5-dT3y9bBH
zU4sUP;10pHahFp(@ArN)YtEcm-<oySnLp=G*Q&0nyQ=mrwfA*xnZ6XgrS2J5!(4Zu
zBmDzV^P`uwM&8+uf(sGL$rf7fBBZ=7`+Dm|&Z|&fi+0uDWm$7Rl!p+m7*GqZpRm8*
zs0=*F5ab|yb}Z=+KhUvvtF|y#T83hT1d=0PHE7-po~pU;DaI&k)h|J6Gu}@gEMRpl
zX3ny7M;41aJkpik1oizYcZ)>QdPr>egWYgRD((Z9<<I%b2N%Q;00-w&bU4JPKLaP<
zZV`0z#BXxl_deaGSF@_1a8INSbo}+t_B)3%K`~-bg<Gs9{INf>Tzg77>S|c|D7F!z
z#qk9$V*sem%hZlOl--{b<nt*a+5KH?f}2e*GN=&I-v1B!aV2H+{dE0O9CrdZa>P+>
zhNndKGn0+2J3|2Yv-_MZYHM910%%fLs__tgwdI1BVWatTRkD!X(tyeoSz8yYb@QDL
zZ}+Y)=Jc|Zlei`}YgX^0o81}r_{c$p#PF2Cd>hvGfbUb$gOgY~d&!=M{;P)l_a%+0
zh{t&|f~^$60haxk6SkZX6vM(=+dJ2;plv1SVmkJ#e2j>bs9jX!slN!INA5GXht2S%
z=8*zo7ysFw84g>SuS`E9>;^Flzk~fM+jO1h+JZ)THZpn6Kq9w{XH(MPmTHGQB6oXv
z(T{6L;s?0KBz!p~OjK3Vg^i(u?NH@b)9G+|jN%hSql`MUr%N)zj*0|{L|laI^STdz
z4}%yV7NCm|q_1mj)Z=M7+MX3ENU^S#9a{N^v(3UNAi)aS8MAIsh`k2SmR$qlYB|BS
zxA1(Y?sXSsqlKjpFi1PPe}DNd0)nQAJnoAl(7!s`#~Z6MQy%=OV_AB9z7~NpSdN-v
z?YbE#9=)teUUnFlT(MISmq<a3mdESyut<p|<ZX9TP5IGM=v}W7!|Z9_cnw-6TU>
z{r(G>@p00f+Q3fWkBEZnL<-EHWvGN==0hB3p7LOP2xi6ih?3t6r*|h90m>~}n>%mQ
zHQ>q)q|KWz%2JoBmh2J_rMG_;Ri)vPx^G&fw(3>I;~@v#^tqI@E}^rtJ*>MB`%n;@
zS1)b$Ue75rMx93iuRp-K4br6G;it^aRV+OVbC`9F9r?M5)a+tZB+3M9E(>s%0ZZq`
zE<Kczc4k~K-cX^lLl>QjRgRm>qh|grX&I&%=0@kwCzqfwX#Hwx+F#IegPLDv>awHE
z7;2XI@*S5a*Ee%I%!F9Vk5-G8%0TuW9~0c^L8}+&nq+<(k0#5*bY~vlES)bi>SYgo
zhTmaR>ENz$+jNL?Z5<9&#hHVMLH?spd{6SkO-o!^7Y6Tr88OqF7gS0%ktY!ynzoF}
z3&?AX`3b&&w`puyW!UsS;*+`0{76_wVQ1{jVWmHiZsoi=fW;qEEV@TQ%j|BBmgq%B
zH(F7yG_a2DZX_Rn|H7M=i6r?^YQ6h>Tevefb9;^M>nIYBv2f4ks*;%{_rGTfhWrhg
zSF3GjY@rM8gXdt_7h+k^Q4xP=w7Y9925YunT%Ngx<A<lBfEH=Yw0iaxflq|6!{1jV
zqQ}rr0dg_HK5gZ_$QWZ4iSS)q)u(XgmInMx44l<a(C^<(PK+EvVqH)Lkw#ul$Bi0s
z?>Bh7M}n9jIWlgu60L=W0!(-+wd{O6%TlJWS0B0R;+vHUx-ZE1?Nn^^N8_7E1|omE
z?d#DKyA59EI{%q$f%r@vQ@bmYswclYKC^4Q9Dl61yU**+c%P4@t3_y9&GkF{B#^j0
zuD)jIX1n>!k1?#+IXtq(%ZNzYfcH=VUR#TR(|j*W{(EcUfe6gprK;HonXF2LmUIKV
zdDLiU%=`DGcsqU#AN8bReUY0bJwa(na~s5BTh8*S=*Fp{@XR89?X<&yGKDq$C2jN}
z*GO7@!q0R*4Pv~4+;@Hxp1<RwB2KSNDiMB{rl&_UdcXzOh1A#Y;>?3>NGa_&$nI&r
z>~oV>+Ed@p7k$HOQQ3c3ZTtS+JxA!5cJ~dc-j{&9ziMunM~xqmR-<`D=)`B+9=vXf
zhdzR$KB#kwp;a9DI7Y#7R!6fMYohp3zUUXr&-UWjAHhdRB0i+AoJF|L&=Ih;UV2}h
zCSAlGYOPv>AF6PnF8qg=<&^}CVfeip?5D+Ywk&b8nSk`wcQnBT9^B50{&@c{>2I#m
zMDa*xEJx><aqSk4_`PX)*mySY<G-Y@(^B1hoRaPeq+%<#(^^Z<%Ug$*UCV@I8OyGc
zX;oN!FH2Ok^=8w6yrbigc{xvm+~(F`6T0|K-&|CHUQzg7EDEt`(Uwgyycb_QzcxS%
z)431jR^nSE_PlRa)7To#c>~XO2Ps?lc(mqM2ELu9eyh`r>mMqp<S*|tm#iE_1n8SH
zoiD{HT5PV*BRh;yS-pbu(n}56f&S5Angrj?kH+Ud^xgi`J5&B&nmOHO5sy`1-;8{P
z305i*vfzzw<ZZ@dW1HlucvC~ny+jGf4u!k=u<!TFxnLQM#RAT>8Lg+rPaL!9QN{L(
z4U0s6Wk`4K44G%r?gs+~mY09E*`+b};6OIWBo+~ax)$)D`|ZRte|Xdp9{w~;QRI)q
zD;O;JAhEpClM1DlMFzy?PEb%cT$Su;(cP_YMfGnO;3>a1B#11TqY;6Z5uh6aso4MB
z<+J%EZNN_??VWvpW5~@w7k_+pNC<okMN;+s*OzD%4EWH@8O$+;1~4A29-NZd1j1wK
zR|I(NpyglxAzCC0@+aS!P1|bh?JRDrV@)PLDsAV7($p?l<MOD%__qSD`Hlc-a?0dZ
zxNZ2jRD3EhT<8&zk@q0OVRS!gc;UV!!2I9K&*`zsrlZ4vxVfxuIW)8*iFtdZE_~l(
zqx+Mjxq_ZOl5){w{@D)&*v9dGzr#d2cCCQvP7UkCk2?3ekmj@YgW1^>9=?%o?$RLa
z`b|Szj-xi}^!valB70XA`4`78?K#o8@i_O9emRUz`r71$5<n&X@TFiTC!4fVy`qWD
zs1~UybTk4|9DNfX$T3;J=L*ay86_3DH(OYk04pfUM|20)@i`m^3JA4?5#?gN5Op|4
z#3Nb$$oK-3)o5j-T-!_fwF52QX{leUkeMg1U%lwhb<%nKg%2w#XfPS-4v3)Er;4_@
z=eO#cH?=l9qJ}Ms(9llvKPB&$=~@p_ruNbT>4<Y}XB35Er{Wds;`f)oUc4qUzZhd0
zkSCd!s1@LKHz3%ngdA=Uj6a%DIvB8*cwsNbS1$aeRoJ6!c9`0#*9(FQq&*$CevjK)
z%9+zIu&*E>Rc6s?v3Ka{k9y69z@Uc0+eQb9cc=32ES;FsnRBhp0(Hb27EW~%h$9$t
zI;*;#@~t9)@>_IWbw7Ln{T>lYPWD``YjKu90X=R<N^Rwr>A0*OZ+ZD#7dT76`wx|>
z%_SF<Vy|7Fg}NFCH2zUYVw=%9**8+w7xJu$csvbTwyHk-3{T|$<z!r52_HK=YJ&zt
z;oJS@hm($LnZY~N)H*x$5G=RCffoaO9@o6dr09F4RLnW93Q~KjNo)mq6cN_s>wDj{
z%R5WgISiBpeI87U?{~=FX}p@h&1_3<RUpcHPwY>`#5P@wgYz=OlZhaI@RJo3#-qqD
z#dqIO4aid@mbie^q$$mD&m-*Hac?-(5MDypJVO9$Vl%gBdlUR_d_><W$ji>?uCdtp
zsq)|B(3@2leztS0FwgJjt4=-l7g>5oOSyC86DWeL(K<Zz7yMPM+fx@>8ny)Rg2F<a
z1FGn3lP#aG#|%@S6rPl;iljfL4d{*T@=LRRaM?{ybvs(R$b%`AjY~xRdWbscPr3j*
zT3=m4LMl-%(~|G@r>y9M7N(Qm0b${_*FKqVv)#cR#t-1Tlb_$_ED!tc*a`!SJCC7V
z!O{(Uq#z1q8~%kw`{g!tvMlfl?Bu7h%uD|Xhx0_m3jKg7%f*$er3-p>i5pTiJ^r0a
zFY&qhmDFaX5ZDNl)*S}rl#!*}Lhxr9Eb>VK)Q+q=YEri#EPFjj<ViMlW#$7E?TMCG
zd0p7aT>8#~Ouf)d)YDq#zGWqDt-$H*KF6pjH^RaT94RBU@(Z#eFG4W1C4CM<tlcqp
ziA+g(%kh}E^6_DEEAl;_SE1aK{Y8&Eg`J(A$=?dLGamE60)tPyKk>AyC*Dq1_E_|N
zBgv#fg;wv=Wytcp6E5s3XZ~}xD*7-Id9m{KLbst!$XBJ_Gxbl`#OVUYUZ5K1RSNgo
z$U&BbLu-BcqCWqu--c1UlA-3b2B0OEystJ|r4bqvv^=>@qd)rGHxAdD$`P}X(YTUe
zM3MJ##=VS%0G%or&s^R-vGtXDVY2NPIGkZQ5zBst{(2p&;ZH_SSgXy+^$5qBRC?JQ
z@NZ*0aM6^0@!5zMnU08O6XYGI)I+Vc^^Ni2ap=E;oQRR=vn|R5FrZBM1Q_!&e|3W3
zjahQlKyn1mhNeSeH4>+PX$IvBN9HOF65-}KUXIG<1T@x;MAhQnS=C~&G2>fO>EqZj
z&z5hNby!HLc5{fw;4^;La4SG1=<tkxoq&hrkHo1FT9;;Bl!u66L#Jo{m9{4C!{BgG
z-BQ2i5AvYHs!;KwlzzM<Yt2(m*_NemHg=G~{F#d1=J+{*h~IX@Uk4MBM~tYdksv3R
zN-$<e&}cFw*>c$3kPy^hoZ^Q%Q5n8eW-4wjbbhH%`KEw9`9^KtoeO2bUULqb>PA-`
z7aEz9^o!x%pF{PX?8O4mBNYbIOy_gqQMlG%mdx9w5!itU!E~TtwQbOTbVD0tb4$+c
zDH`G_;HNX!yY90J#a48~+ej0=IWk%2jrEebXz9C|PV7pS4xD;m+$1%5yuVqur?fDF
z&djt?Z<lWf53@|@DTyCr$&s-#Lk;q9Hrd1*wPQ;fIL;E){I{Bu->}fKbbx#eSUEuG
zGBbceGFiJ>ngAi6Gb|mRMw^NUM6K0v;eB<mJ*_)3qhAp<_9xB`X|9&VEIJ)+Rs%Yf
zZeeOSQ&#<znN@GPxVG`I9!bl~WGi%HWtdD44BaD3&mAsq(VVDaJ)oa<Yn}<69;ay#
z>1FMwKh={~wp*uf*0E^*T%243&8taeCaO1XCA3YfFJD<;9%rtQoc+~h<u`<WYide#
zXPDw(Rab7@-W0Yl;|o2wqRd+tBs7?lO~YQi$euFf*mRV<dAeC2@OOM45&~XxOp(vG
zx3q$J7SKi}NETcrHE5qbX>JE#!WQdWYd<)YA@!dykJT-X-v!ySN2a)1@aXGk)h((b
z&7oS%6rb_<(Z5|#woNC{|D@x_ddK$!o9VnnK%T8v+1??;oNLO-UFzy-Eit+CTdaWQ
z66EAz-z}3LIw^9-PS+lgx;vszgMYd4mw3Zm7nD)Z=lGXZl_SA#7Uc1chAs)rwcIW4
z1UzfyBKLn^?px~=mV6C+Us8K~#Lge2B8f>MR28pz-hgmx`7_zVi`UCXDu(c*p6t7(
z%gT3uWhpYp>(gn_GOBhL<a4N2VQ$0c$<|Xh(#@Kj3Vh7S$+QzIcK1gd7E(r(S?M$h
z-UHx*<QxT9@ns9BW9NwDn_kVfq_S6mO$QOOhd&c;85O7h@2<(~t#9Z8$2o}adXxoX
zq_TnY+qYBSRA!aIh=q1XYc0PR^3|h`J_$YJTBq*+rlW_2CIqiD*c($^ioQK_AgX60
z=TJZI{Q=R)CP8fgBI?FW(a7H({6#cNkv?-z02aRf3Ko5`EHVN)tfF@$9xxWe8-X6@
zVN%DmwJFbRvCzlr#;Z1#?e1O!&pAt2f-PX5nmoQ;vJom{|6kakKaATsxd6M{+1PZW
z<3ncWL<lbhOZ#i<oGHMxl=PCGdI)oU2Bf&AY$JEK57$eAu^YkjSpz@6WZDmwT^tN-
z!~({Qv=V6zVA-s<O@OkxRk`ZCYFi=mS%Bq)<ggO{Gw`!2Xn7$T#_hXmFo7ITgL&ga
z3Bdqb{YDSw={v_39eX3r#G5LQm;1KoNUgFzu~>wJ4$;xmt{Dx+!b<L5b{~x;{wsz`
z4m9?K=mYX=l`Zm=xJto%XP3;vifTJN>aoOk&&ahlfHI-{r!q)?Rr=rQkWYV1{iRIG
z?1er2_S)Jg4_Hp`xJyC=&LYL5ttG8s0A}n_Y8ksLIv2BKx=|P%t;>B*$-ZoP)v}dQ
zBs@i8j|$JMt}Aa5HI2d;794RlisJ<pvA;f!va?q%cy?syHsem+KM%?-d)iH%8+1_>
zh=cRU{+n}HZ8u-1N-i*)%|y?zzPh%e^t*Hv**Pzn!vOjCqMV*BLug*g??w_<J$Kw_
z*mt44{SV3ip_)HG^MC#U!&J{K^s|;Zuxc;q^y?p<@Q5M<NR@V962IOD$cllV1Z`aV
z+uHal`*vRUbgY$h4d^AYNO61Z2!Oq{-C7qmqsTp@(6mx9i)RZF3#Ws*>NYd`;8*5V
zXnBD=G1}S2RljOBdsqlayIz3e{KbdO!xSn46Jo4!;5u$uK&%0SXhFoNL?7&t*tkMp
z2fv8<DI;6j<Q6Kb*57-2j`s@aR{fagb@iq(Z~x|OJE|QL2Wd1cFV{HLI?b?jR<Mu3
z=?Z&k3{uaTq<D$l?f7+5M37=EDG%!h3js^%C_IN8D!~U3d=?Qd##_W6{!FQC0>|)>
z4ZtDeTgWG#WC|YMQTdq!JG^RZ_a}q08dg4SH}z)<?gTWv^$(F2_zq;@^*n`QRr8;T
zHzDHx<!<r<XKRYdPM_&BZ=m)Sk<#LfPuEQ^j<vQ}F1pPt0T}$%>a;I@6>ps!I)5HK
zuk_UoV9@+_j#LA*oq|WfK!g;)(0lu7i9Aex@$w24fhOqIQu9?9e|C(oR`B!GS<5vK
zn6$+<B40m}(cc*1!bU?rOzEfm09+5uE1@I0XY^L4OXa`vwEydO`QJLqxHMZ&^Zh2C
z=wiz$Hp~4==ozAB8?RViwrS_coW__CicDBa*rk=RLc^{uP0=WBjnj1>eRie!`|^b;
zt(KvtM+3x~9XY(Yt0)yfs(m*lgH;)H22TEw4>B5U34x?Y`>APzGH?c*KW4i<K-&9Q
z#~i+|xA1ChRLDDZ(}F!qz^qVx*~>Y(Ik%AK@lrJbsnu}HreOj4HpxTy{S^W*!#`!b
zVo>loYv><Jci#Y5Z0esUpU11OfNN1dy{zAv^9KG3TtsyFmuHc^%jJJKzWlSjxy$+I
ze~&85=w1Hff(|PK_SLgrnecx7`84SFjQ-BE$bM+|i~w@H#~V-bHm!`973J{&GyQ}6
zm|*Jrg<;2u-<KL@(gK@%y3qQSxD%tMHSm;K1twZBj?)BpfPztepH&zkn^E(aZq?x^
zrHEdQ!^Bq4bj&80$hXegBzYaiacw)^IZDV{W0cV6>^f5<y1?W2c;;eqiM6j5G$-i$
zcy>OVGM@V0&uf(4IkD88leCubGyGiJ^nF{mXV7SZt#d6T5zG^ASIBG4I>Rg7^%Zzs
zpJA#A6ps?N>)uBVbALukP}WLXyf^af4>gQ7GLV9#9|19N4B4ZEg`gu7$&auh<7O0O
zwUGc`X9uD+zf_mJnV`E?n?m_}KSxqPrvEK_^!SDR6-8Mj<tFnL#Ua<dNb1zHm`=gx
zw9Wc&u34JZ$RnqjiVY?_sYCa6enTrtkGr=&^~lB^IpPN&Zv7C=Q@yJRu`pOTL#zV$
zz_7!ZJV^54h}9u`P1@^$a@=r57#>AiZe*pAb89~B@BPEc&(*j1_<qw+#yQ3E`n2gP
zOby?Pnbg+l#0Ooaf|4!#x#5afV;ehDdNcQi!@cEl{n>%EA*f-=8;M8DlwYk^FUOec
zM%yNbCiv+Ex`PO|tP<dgJ2Le9*U|=wRmUMC+i9RNW*%Ai4kLlehju~dI1qCEU%{v}
zB#VkLaz_H`>8H#08U60IS8m)Ht7po#Po`E_piO_x;O=O4G2$1i)spC~Ee?Kve$)z1
z`2W2N!920&!c{Ug*1#!VId}=UZm|KWhs$;rA-%h~%(HA0FDH@P1j5%Bc-Vw<vB9u(
zQPZJ^;tq<0HbXC`NI)<aC7lytJh0JWy!+SOH|`jF>*T`YYr419f;_Aunk0nzj7?_R
zlrT+q0XCspyLIec0+V!~GwJYDESOuQKoz_Gu^yT~`QX#e%`BHIa{0D~DErgei7dPR
zinh-<?_i<`->#zlKE;69V@%K+0XkcePac<*h;y?8Rf+cV@oxS2WFuQql#4y>!vUa%
zPGM#qDEs@_fB>^Y-rLdkg1F?9xfr{L<}2bcVGOb5-OL#n`&wYmcZgZ1;4rA-^Y}Kq
z2!G%Btep;c#GuMS?qg={UAgW&T*a0OlI?uj-BKrG7jo&W^BiU35ryEmAd9T;#6q|L
zAI%>g^|@cbCC86#OEY9lm6sNxV$5@dwsoOP7=l>6Zmj}+%SjId{hEf=o{G4X{FO$l
zq3P0@5TD50NYoor8|1!o&~2|PvI8F%W=BzJF6fAy1Gl4pC>Mlfl@1!?g(CR3HfxRy
zzVpEJl;q!t_tXiVL|`_6Sqxt)rZTT`Z^S<|Rn~2KUqO=+KdE4O_JDsgqgP#%Hy1Am
zA&_wyC-QZG&-X)yv@4vIT_GrM3$+Q1`S(E-Npsu>Gc2NPe!~MS?a2mR9^hAkykwCO
z%N;X&iYSQXFjR56t<+pu=HALX#qC~iox;2N-R?u9H$OetZobh;Ym)0drFXT<@g4eQ
z96zIeHRSNIkUm)Catqk-9L}x&d~?@%N8yLrVGuRyM(gp6sa~atKprNqzSq>om%%3I
z9;u7F-*VM~`y#y#6s0R!X9qPf{rncq>^13Hxvxe85;`;8KMx*04t0dAEUHZRx9w-U
z3&R_Cu2&1vU{%GHvXYE#ukp4@wbvTkr{r~e=}5XA)Rl5i3~TGmFZ>x<H2B0k5R6ee
zXW8U$6!sED<lgf0m6q>vb^ls^J+W(<iUmsQ=id1$xH`xj_Ly89QfgwrKYgZ|Ze~B^
zyJ^mBWc$<!+*{wV*1HKrPmX)aJBbmw3r7=p`vp^Wv<>9INTejO<m3TvH=6zNo}a$Y
zUw%{sfro~A5<8~hQlHQt<xLgwDn|wNnXPF|N$Wv-8uscho-dZatU?rTf+w~?3Sn6B
z+6O2+nS&&p2e;L47Cu~8{9#*_xOmG?*0WlsKDR8>f!cZ4Mc3YZkcme(YKACY6&xM=
zG?Mg<Dz48Z%Hz!8k!I#A`_<7r`+8XtR8}~*IUOuWZkm^yl3IPbLQISsc=}kDtX7s;
zH5tdc7!O=p5lZzamI_^z7PX3SU5tj9pk$MYp=77IQ+vEfT5R4ft{oI~A;I<t7&Y=d
zUp(*UqWxBDBZ9LcfVi0!Fq^O7$!RDs>PX*QV@c6`>E^d%GfwX=b^Ufz^~BEosC>z_
z=4X93Bj;4Q;_d=}AR~;pEQ4(ldT>jFko4zFSFoe~)(y>Mz<bj64Ep(`Ea}vZTyJPz
z@5pCDPl{Ep0#n1SV*0ua=#XPCMHE);-y;>d#(UhCy$4!Kb*CuqQixoe*}_1~%gkFu
ziO0_=()l4Yy|qV<NB0SxgDA?0Bm#68M3QgW>iQbF^w8L0jiyri3lu*-fHncQs<v}w
zu@^KONi;Kpl|M})B*vI*()Ji$g2c*R+EVJ?lmzNR_(H`umn1EkGOlT|jIz!}fgke(
zo4~blb3Q7u{)R^`S*|0$$5?W!hiO~I$$S3K4wN-1JB9hR5T&^JoEUfA>#3@CbrmPE
z>>tk)KtdiJ`qMM4ZrRjx^;?T`YUes#etwzNYw4N`!b2_WNzSzO*{{`vPc{2Hv})SJ
z%G_cFz>TyAXRnxNdx(Fh@Pl$7kHvj72&n3Y_R@934rS&M!#DZ$k@L^1&ioay)&Oyf
z+^_D4-GIznz5iNhuv9q7a6BKpqPl=k821SUPT{d~gHh-0dfUZn3b7R)1JHTcuGi5}
zg3(oJy^x+k6y_HdzO}VP4LSP+WrX|viSN8wiPuimDraTShZvGW`S#zFa=!Q_jxoAC
zRWdD@s}ocDG#_ljxmohggbpIIYNs{=$23CYZbb0orvj-^+XeAjk?Z!L$ny1H(kNw@
zmV|a$KiN(47yjF$5_2Q6Yk?t|nx7=799aB%WThj$-5EVVLJp(;(P5e*PcY3FVC+1_
zM*AjP!pda3<XMrGminGT&-ZU2lMF)D*3-2qOtD2Wc`tZ&4Q|zwcCHzTf*DsLZBwNN
zHF*f;Y%OjS*_)dJehGxF^bn>ERfHYRa9>ItYK_!8N%jN@goyoKQ9Nr<!lru53E#2&
z8o@s@IvOM(Sqt~$V>g0~ExAe2bmjUI=$?y6%HYxWA!k&L_vy%sC>!?yv|$bv_Jj6X
z<q5d8P<Gsu*75@|2&KRpLvSXMU!%`SqHFMY@x}3kw4eX3%<VX(ZY_nO!9Xy+;Wcso
zo86EN(fZ4!c>?|d!uq!dkY4W#+8UQC&HXyFkJzkVc4gLrBv(r34_Z`QcLuR;H&!Py
z3mz|HV<R`b>x-E^!YICk=8TCC^0K`P#D<Y+_hm4W>N)Nl)89{DytD7g)(}Hwx&$3#
z?|MFNyiAJxnYqlkLYUeSfB#}4T7XxPysNZeg1-Kj%f@*>ZT6aQx}6tF_5fC$A%e^D
z*8>b*+hzSZHz-UCtIjF$j6Si`Wn*^SZbT6N^N|)GjuW~yV9`D@{Sx)@(J{>T^KkK(
z!1;nh03+v=GD<q6q}VUVVF|&Si1BLsgm&#Sn#q)1!QeBWwFZI119e<3rr&$Ecn2>x
zJFspYG476e^Qo9TsEpP8WV27PWSu>uNAal7<Yp~mHFo=xpZiK_@Jg@Q;dD%psPqg?
zG&yFQcVkX41RfdGirjvmc{ms_hgXKI5Tv^W5XV;o-hwm1utgo{Cw}wEk_3slS~_g<
zx9+(!b(w0R?{EX86)V!dx_*4+%So$@C*rdK0ZxfDM=yKtjN>CwhNKj8bjan?Z{M81
z0uiL|H*{+h(W<Hx=G=kKwfk-t6opc97Nws4ZDy`-JJJ4=-bJBb;plpOGTD%=)<JSW
z3NinKPFeOO+hgQ=5lw+r_v!0|NoPNkQbN=nkORv06Bgdt@AOz!?{5eB^#(o9Q+D7|
z!`Yt)!F?VOx*%Hb2^(wm;_nAJo`VHs(K_P^_O(9c8j5oqrMAB=sMogrb%_+Yu{Rs`
zfzdy`^!q<c6hM#8{vb|)<4t=@^s`U#Zm5cGUvFsy)n};Qe%Jq@{(uDq;mcVhZ1lk4
zqE$z7>HKje+()63Vkob*mB-ZA@X+G4#W>$y#pFm3nC2nQ;x@zaG<2EIXMU?~QR!PN
zr!MwgH<i6GM|)(!wx)%@2tO1(Tq*9S$Br^*)2r+p&0|BfEzXuuVvDS9>CP-7T4^vN
zT&rB{n_>jo?09nyxsT~V;xS{Vy4kkURYB9){lO-XX*XoHbiKRrwa{`p(bvs-^V4w=
zio+b+1R|w0wb8{)_{C(_^JoMl!Tn|}OdW1wQ;dy86C4dg9;w;ZjGFUqCY0F+92Rfl
z?E2hpz6Xu6dU`E?+VP<dl=T{nAgM}Sc5__-KIF^`iHEw}ZXN}!|9mmX?C9n;rH-D5
zvm+Ls9pZcv+mQDC@>53Idf^*Ta;Z`2(qQHfWR9dzJl%75`I)or(fY^Y2-Z8J2EZ#$
z*?Hg`fI<J%riPo#g8(QLgkuNA)g3W7b{nh$eq2lK(EK+fQeAsNsS<ZG`2IOtO?4&A
z>BvT49`l*18l#f@{h%uHIimSXYf7m@qzOkT!{J$k^ljvmv9)}}0hS7QwW^Y_JYJgz
z&c0rUTru5ZeL6I`eqYP&z@aJh;1mx%^KK>9ISz}iW%{!jX*D-6as>8f+{$#}1V8=q
z4NgaV)0|Ak%4^vkgX!I8Z*;DJfq}a3wKGO3U+!{!z3iP1qk*n4%YDNerk&tTt-ku<
zof!kEkKZutZ3(cD#+)uzTjAL+g|(=}HAj7SZwk$w`(Yqo2bn>)5UgU4OQ)1Zr+jEB
zM{WA{3x;jKNh^2i4w)ipIMb<GLX6~LPSgC6E&w-jth7+nm_|h!^^j#Vg}aeag*C0(
z9@*)(EYbS42;AZd`@(jlRb^tN;N#nMS^KMVnG8nF)f?5|HVlnS5B$;oeJgJFJ4!R`
zBoRW@BX=t|ev#UjS~>H0g0_R`G~vl9<!CbhK0cf?oEs%M!qUC({cqMFPT@vc7aS&<
ztf~`CZahaQqyWb;+AvO6d=!)ek~<5Wlt|84eEInK6AZ?)oGNDRgBR#gXnE%O44wQx
zLMV-_6V+&W!--?h7{Ty&MAa@Y-nR7A4!h)KWzvB&DR=B=DCQID7hX^B-i33~XTSlD
za%)02j?a)o?&3OG5xA^-nC6?1Jq|6~P(kfK&TS&fqyC?W7P;D)^z7NVPH%g8frt#i
zjwi{hkD{YeAR6{}bZ236+~(cp2!Fju-@C*m1|0X`{CAkm|KJGy|Jkk_?<K2j%$gj9
zkp+%|<-Z{j&0NjBmKNP$E}+(tw*KEX1$J2rVk?4Dc)6gD1&;rLu0+#(FE3>p<Risd
z!5XHXYd$<^6@f9~W8|8q4z4@{L#Uu&lUD@jNV%iUd(X|(DU51qGOCw0hX7z>;JZMH
zp3``ZqsY!$m8GdlW6p_jA;1UIGSnmqz;K9{yq{0<;QY6{`oD0G|Ml)asEEVvD%&jn
z#us^%L^CxfIQ`q&lwDq4fmDja=FegrRV59}k%u*lf60P$%c<WPH|?bjk&e>=>Goy0
zexGkS{cJy;3Q~Rovo-=@ue<C%@pl(9(=q6okGu~-<t*G2M%R0m!Bz{okQq!=DK~<d
zg+{)og;f6gUI%=e*5EP!-C}1>`wyuAuEYjl8YK@J1k{e6jmSG^K-sLt^~8ZMX=;+I
z10XS@xI}Ov>UXufl7TNygEouFBn5CFG3+CZ1}^`$qtm@Nu2_r7DIrOcGZ{=(LT$^r
z!|(_4(0VuIRa=39B3Iu|x<|6#_4?+dVwj6TUl)CVw?2guYyxk6x~igjh(}We<aUEu
ztDNFaAIQ*624iF&?Z#x9%vIPM|MPH1_fxuBvZ%Y`^~QsZRcd_?-od2oX`7&Y3+3H`
zwc_dm7xFueGyyECpM7F7Q{Y$F<wJ&UWDKF6)=5nZ5KzZv{Qb4;BO|Do@bFxG!cv@I
z^QRCXA$++HD^pzCSJfmIp8MPROr3XUR(`D7b$K;HWUKIBLp$@AOTLj|#iQvj!43*6
z{Ro_`M#I{P0@!UJR^c;Q^l(YaY{Ld_`u3=oAGaixj`EK#E4Uv^HFRt?B%jQ_ayqF*
z1P@G<`&`(4a`X^WF&y+0f@{C5Q_jfr^PhMFxttzTeb5WXA_|K%NN)exBD6TlUcR}G
z&CuyRg;jE|usrj`+>bruMIfnoSYXQw=YODEbuvgeAZ<OPTBbrOf><!IX&sDtm$*^N
zOla5Db^?UNebajvVCDs)Zf$~R5O5+B`!tUWdV95mo^`%s0+RgH2N!rAT0cE%6+27+
zXkgy4ySczhtEI(0$Y-h|bk2U6?UybxvdyZu)zU)7uo(^W7Q7n^^CbAK`zYwIW@Oht
zAl<QF?%jSWK0O63NKAw`R%a5^*v65=Y|u`^zdTtuz_Yh175U64)H**TdMVybWbYii
z!hf;k3*MB^OYai#)$>~j+fKAEni}b$)FN{6;=B*5M;8!IZuT-Lv+6QTMyA?pf4U?U
zMD6I=+z<77-~x_^c|V-}JzY7>9Wbanuqp5h_B*Nb+(RkQO4M|xu1^5+!UR>}&bzBK
zPa<Rgx^dx8SqsyOkpKrn@aAa(aq6upl0~QnD2spmIX?qwxd@Rjvvvt0Nf3OD%fNET
z+bzgOd+`meV%mO+k%?ynU2L;f4VrkF35PRNWMONH>OUC5&c|N=ZI7IIp@`PMvsx0w
zbb|vDAvOHY_S7`!Csmsa5e4PR^?1yr@50rjkI`zE9{gCf*eH>oZ@F@j&=>!4A|_E_
zE9}>5>GDycLYp{qviMr!Q0@Gt;Pj=GIx;CXO?csusOJ7DqK^VH=rLHX%o<t6fs@@t
zVc$5rsw-(4$;x8HcblRdI#%>%;dZ!yH7IhrD$1x%g-7Bp(k5trm6|V;O`3~EFoAUG
zd2Nt%=JUoD!+|uPm-1E3dARv`W-R-SIXL9lk7f|u@sf@nalTqz$fZXd*EPr^EVVji
zRS*I(s84OuwMCxpxzYS`YG4C_Iy3y3pPFmwFDcC~DKE7sA{z^sk%fznVZ~#D0^UZs
z`emyhUaUg<iujutoaanhsr5@6Q|8fg5c6W94|MVU!|NZ_#c1K_M6#!=V}{!k2AC3X
zX$y6vex%QU2|g&D1{pdvB|!eizDqM{tX$mT0jkKcrpV@_NZg%6`KoUI>>0rYy6q!V
z)##b_JPyw##E7(lC+qyYHG}+_KWIc-Lv~sgzcS<TQ-IVTLGyhCAABcE8vg!j>FUWF
zG*UA1ZJD8>X_=I+dc1`e(`F_@F<|DN=4~z*%Z^Gp0yh_*4~NP-@qCE!<DSlF!HwGS
z1;<!KC8r?!=ICo_$bERHAby(8FQX#xd`n3xQ!JoiwQDEj9*4nGIa+1ez-(D%p>W9(
z039pnB$@u+fyW@yQrnk>w*mP0I)-gIUQ&n|hcJMU5-~vc>$z~(eSPo@P2C8n2V|Q*
zu215zAoZVR#R63Ww_~!6l2dCdC4V&GjzeKUu7)%tw)-<tRy5y(WnW&<h#*|Y&9+GM
z@qtysZ8o+@=G}zNhF$PulE){DOZWB~AD7_RLWMGE7gSv${l@qkI|7`qIR*)*Rl<7N
zqUv?q(uxwJx+ObakrsY;o$U-d)!S!(mHpyAsmD`o0nOlm%As!t>U_~3vj3E=c)60B
z7-`Ko+n&HyB>aAR*sdi<6_<aVX-sgg!X0okC$$L(S@mfm8+J%fDOFq;otv;adO>|O
z2C$Y>HsS={D2bj*1Pi*DaQ>RZ)#GMC2VupteB7m0Zg$GnMn;j%qHi^x$|p+dLO}~K
zBgd=0FWYHxdK;RUj7#T&5=Yi&$K)V(1>bCs@#RVONX}yTnzVV#Esz*f{chd^=xM^&
zp8UZwy~>Aoo1nOLy~}N>=Htb#esag(d<CJ?-PiG2c~4!j2;4-Gkeb>B`hcuGT|&pv
zJa<1%j<=QU%8hO>vb^GX)TA?0_BU(q{C^3SZmwS~P|EbS-F%$i7G-J}G!QwMvfNuL
z5IsAchjid@(yZm-`d4aN$b+?(g^W)lQN8a!%Y(6|W74(%)%8`FwZGmwh!~u59T?L$
z;@ahpF3<1ADqRi&(Jd3tz9`O-wDg>D0*bAgZ$H(Lm8ie?+G}1B_lnz4Xs{U3(5zSq
z6u&G2@;Q&MO@L!)i6@8&hVyw9Ywb)QZ+p{bm4W=@eE5lpK)05ra9BzeagZ~uzA01-
zevHvGZ<z({W0_$bQ*YMPD-X>#-v!3IQzgO1(*n~LNQ3Hyt;DjYo0w@Ib$SXRm>Thh
z0)ERXy<SDsHv9n$liEQhR6CRLU*(v;8@8yfg6&0+&;thKLF;yt%=jGp@4O6mb2sMX
zVjuGB>hhyMz`$kwP+HP<+icswQ}VgMf2}?@#0ltk$hA60o9k9Tbm#&-Ky3r~v;1u*
zhwnOodNltL*HeBG^!EB+Cd_on+D8N0m?ciKI6kI&^V`R#^wHU=at)n2#W^S~%3vLw
zJkL|-FD51AzYG%F3$x=md*PWyR)Ltyufr#P2|TZN2R}FIY^z7w0JT`lx&M54>p+-n
zrQoozJN`Cu5KrH>yJumNCNa^%Ud}ylB3Q%C5w+^AJao$4QI~`#P@q8GNmr-om)-On
z5uo{64>L}zC>XRsFhS=o(0G6vJCjN_%qVYr>4F24nE82=tH2)R1ZaFXqhq!r%FoK)
z59F->xt0fTY)FHD4D6rDI3Q46J7RuGwV~S%C?mK^pS<hd^2~b5zoch?xB%KDEx>Os
zudz}|{6%z!kh23f_yYn`xL8uWZt2o%5F|`&fSsLVtP@dM{T|#d!Dt!Z_>ZWUP8)6}
zQKRx>4i1HNz2+B-H^clGT^{uq?pGMkmO@7OQR{z!iv8cBFaJv<@BbSuGcJ*X6zG6j
zCNB|xE&Y=`z|~ksGZ)CWK;n+_5&LKesCfbuhxr%@yy+i()UR1H`-*h2K<;OWaiNj0
zW-d9v=mI+7jW`ZaH)e@EsAMr=cQ+jH3;D4nkAoQ7zj4H%|93<Ic2xQ71AsZB{}Q04
zg~`XBV}7Pn<X?j+-0_};`P1azv0(p)<DdTj$9Nvv3cE)jU;^TNv?ujl*)vuA$LNwd
zn<$fSVV;W2vl#n_mj1svBaFM=BL5EHr{q8oQK4@Td<8VsO+0XZq2_7(n7|KKJ2l?I
zfp(P|8MIzR1HJBu4jen1PX%)ovxKSkmAF(dl4Ap*m&w%l0hslSxK#LXAX1D9o}Ot5
zuq?&w7>TPsbsT(eitxOff$>#bfXCjrSw_H-%mk3>fpRMj{E0%DHHl(JZJj@f>x*0m
z${s^UpYLW9)8YqAj|hI<AVX;2yV{(g$(lDrA<OP7weZ>Z3a9-TaAJKPwMKX12^%&;
zii*N%K*S2<b<Ns$P+)P3@WRKt2+|U$)@j**KpS)JWFd(Vwf<2*P?&@))tIbTjlCc2
zHf0;TC>FQ7zn14%T{h^Q;f3B&QKko%<uO``lxZtlG*BY(11q<=$V-$LUAzlgu45IH
zS^`)8R=R|yeDsyT7huRMN;8P2sp?y*sbr)lq-~Ol=3t}K!1(#p6OV|-2|6Ap6<`@=
zI>t|Hitqbq;rUX%w8>YReLqpmCZ`qeL%ePM=N}o(mQUhJRFQsHvbx|ax8gT=r#xxy
zEZ=$YMMCQar?8nvTc0-c*y*+GiaH&;>9tHQ4UdNVM8r;t_KICK3*TGBV*mW-p}JSz
z*7SG1*?_SK9)9B~4zrWhmHJ!&)aTj9DR~cUCpe4GK7TP(2p@C(K1Q%33EKKodj8tL
z)E2#9n3P&S$bo=ElL*xlNeXHu=7^YTWpsBvb?ggyZ9^asT7XQAjRU=)oz$-@(?p}j
zE`)E4W|)}iLLn8cHmDO?Y%zCqdI1WI@b$c$_f*X_LawMsBTZnboGh*W$`#-Dack6(
zk&k+G?R_??1V!M-W{o|c2#x+y2L0A$IvlLQ{kbFI&Qk#t&{?fk!ws@6o`g`SaZ5^y
zg~EaQn?}O8-L3#1*1<KnHl!Y}eKnh;sjr*yv_Hv3C{mLOd#3i7cP@t{8wKkRbTaFl
zA-Zm-gh}P_GKQJdp)!xcZ#*Hv4<z3W^sEoj(!0%SogbjZrCoQt`G-2~?Ly3XLR=P6
z7W~fbwMrV7k6o!)NBWr=jTU&}jYk>YK)ib!g;xofMUMiiC}PRz39<RZK{=F?V(!^s
zY47TpPCz!W!0|MfsAh_1tQ7q;Ds`G-P=#GmbxF3|AnFP6+RHb0HY~1{SSo*OX1G;X
zCOA_-x$H8Q1R2w9v3kHVJ6a1!#iB1)u6HpfoE6@`>eBax7&zaIT-*)AqdR{($|&>Z
z$v>J)N~pJD4s^niAQSYymT~aOF&dtLk!3ZzNg0+{G=evoRw_;xCr=mE<L$LImgmUW
z_=1P$+kKxNBkm%2*xgFFq_GP2U%u&${6;MJ`c0OVlpGT@Jt3^!tnlE5KN;^fa=kKs
zl)6?ZXKEJL7KZDs=Jb&{Cx>OQD6!1-BU=b^_$zWQw+~WcuyvG)Md7QAaOecDrlstK
zF9qq5*#nW-NbRWrGvDc`6|YqsmD@-{BH@lXv(F{ie&4CAdVe}(;Uk+gc6)AmIS+HW
zjh4;rx5}wa#>DrzQs?MMe<H24$QAYl8ZKWng?AOr;UfGs({<>0Tu<(hl)tgSlK3pv
zJ(hu6ycapiQrx-jT+R7<{;G1ae|Uf+xUWRVoI)gRbAhYQiqC)D>NLDa+SCin^@{M#
z2B8JpupGNe_~G_JFVXbYf%`(YNI!}iT+Lx@Qp&AXABCBk8jsgioqpqrO{0@N^P76x
z<A!e_R?u}%2yC%|mjc!53;2l#q_=O|62W$Y>udAj!_G7aLBLRtuv}|rmUh6Yx1${V
z;q(^aqHR)H9~dhAg(k?8HhtTL`=u&_NsVWmmZOxyFI2lSK0Bz>DQc^a)UCQz;^LNt
zzP{BS-1mvF(1*U764Wk0_V-tM>t`Tj3SnwfQx8k=^L7cfPpx!@e7MU1p`#KL^QR-P
zz)=Yoc<eA#XEI@nS(jFT!Qu*|r0ukTBhnwN0;%r4ogZ51!a+h}aCJyoFNZWrby37P
zNU~LQ_S`XNq{8=6{D=8l)Ci`^gD#T{TbLzKjF$|P5njZ81KGLCVUJbOjmKx1_xGuV
zWs)?wskI$eX`rXilPHk)3IB9tnBYxd;z(fbM^s5p+{MdKNrMdN9+@IZ_lCT&Ihr6P
z%u^tnI6n3Yf7QZ#QS7#Gqq!G;P4U(`+M2QajN8)tD_eu5A@=l?NFEVxmPlzB!|32S
zy1kJrcZnPkE^ZdG@yqC{3zoqu5MtI>^j2BDpdXeyK92tOpFTNB6>XZF-eQ#1*RAXj
zE(7l7J&kZ{hj4L(o43gNqC41@aM?BsCu!Zk>h%nvHc8|Zmdx+IR>n;sgK)z0RIEt|
z3t~O?BsmBx=U1hW-|LGfw^e#!r`Nb2qJS>%uOE{N@?j5-m9>jKXi6?;;s<A~`=xFX
zb51&n9ER<b^qPA3vH1ElHNStC$t_>p?RdXr-DGR&g+9%Fh88;pL>+jxU7nD0McN<Y
zkz_<COpEw>xE6IbFU|VelwpAo;Y@pm7dQgn@dnUg)v^(l>Ja<&sSHM_tDbzfinQNf
zM+aF=aGJi9o`t${b48#>Mq#NSsCb0LZ|6(+5AY7PJ!BOsA?XyY*)2%mJ9CX7O1cya
z((Mn|B_Q#%jCklLx(WfMAMo*uTeyfH*AAEXFES!IA!dpYkJ;soRtDKe7x~9vR!a4i
zRuVq1y{<0(($%z*m5SU`_R&Qc5dk`siTUt0{WzqaJThvg7B%X{*k)Nkd2P$cOp?9!
z!k;tmuV#PY*yvdcMjjWWuFC|;#$H>@+@P&AzrJS)5?4*H;lI=C?uKU={xTlY1Oib@
z%YG177l=(SK&MP`QRBs?9+n;s+svL0Nq_Cq9R8-xRvsbiP$0*kVOW;umk-T*8M5(_
z(?Ww$u-4JtMWwyy+qZ6hp_-}DFOc*I7C$-I?=SIVDc2b^Q;SYk8Q+{@Se?vv8}(Ja
zE@f!3ny$le&>(25e2CfO>ysr3h^FahB?sz`HFzIjzO|W@PY5d0EBE{RS^2GqjzcMv
z1wwIXHu?;0gfGOaep%e1Rbridh}%#k?TA?ub>4rq`}*t-dxe+!MaP=HnUYNtyOxRi
zO|aYUB)u-W@#bE7nfFzxOm$1|>s#8M-~~tw-ORVR^yIc%eet}r{g;`sPS<VT$Ozhv
zoHuQE!MY`LB0?Tpdp&0KxlKJ+U?T`mxq3%NoY3Kv!Z(Q`caKDo%0mPauWf8<KYC9I
zpoke1&6u@d-wstwd^klC-dJmR05uMvuR09d%a_~O7dUL^G-D&Ad3@(A1|_0FSjQIB
zelw2FIImvqB=~NRX0ANzcid5V7ApDWnQ`Fyi3_M_QPa!cZ}r;t2rF%pqw*(pvQOlY
zUcs^AbWD4EnoH1YxF#uqXF@FWH?9NnfDc?|<1!H7baY;hB6F!QiP>(kTS}ltPqA<c
zr?-}8E@OSH!Tz!O&U*%E#x^R!F1GSPwPtf}wJ8$n>K|FQWL(z%U`J`QDN>S)tUG1`
zSc47N5NE7sx6J=c7V%#)5O&VaFJcXTL0kuX`u$C{{Gy?{mi#NLN@fovZOMf|9ynO#
zg8eZsJ<ea9=K{J@Xh`Iq%M8ytoGJg+J$1Ee;Z<n8#BH{(y>7d1SzR_O?rHrSpq?`-
zU94>GxJp&K__NmXG~y7qRkR{-cX4`Z;{Xo4Uy_NT3C#g&N_u8+K43~r=wi>^dX+ov
z|D@cz(%etPf%kvThqrt1fGNqPhb@_Z`#_@NtROlG8{SRCF_q6&QOR$2w6?U5+e-2U
zo^pbn`X?d4nFKq!<5PqIEsm2-U|)DU-_i#DNCmQHfW3y_KB%-S&a#uI09GwVm(0fp
z=#Q5~<8L8&%@)9^`<K4YI3KX?=KW`jfeq*uw{#Q#*%A(K&i=V#E`L7ojK~-AK#sXj
zpe7~1^)a9kYr@@-TUYsZK>luVyQC%moV0ZgBuJ@;2Xyx=6L8|YzVr7F%W1%UHVA=L
zkwO4vG70*hH~VQwBbyU;&Wp7at%a>zqa~VZ%6a-S#p+qIO6||te=cZI;Vo`SLx99O
zA8sl#-L-s@V-lSjL(AXsmPT7sPOHpyfU21HpK==~VTv#!>#5bL7dOqgh`&xtYLBp<
zO_1_T9=Lh_TvMRVaATf^hDM_xB9M$D^52O*{!8QK|CHGmcZB7?i0H|&jbI7bD~L&I
z#bHyeaFf0=pWj7g9;XU*&>f82$ggXXCicN(e(tK|Upd70ZP_PR^Uz{jVyy`b-Ma3}
zuUqP6+}iGIf@&#ZNQb-V7j4`65o|k;(%RpBzHVSizSEtDRtsXUM3b;#Z;^piLg2=a
zu(mYClb5?%FSq8ynFdWKPneciR{5IdHTjN?P@g^vd3O%;Li`nHmrmxf?U$MJIK4+x
zCsw{H0xdP`uHj}?d7P#84@MPiz>%4Darm0rUUy^rQ@NvT`2ypFj&#eXV>~m$^1s<(
zTf~Decqt3<{QrXeyn78x)G(asvg#Ns?YKx89FI#8{+s!R0HS2cJQzs9YA!KD)tr_`
z{-V;i6i4jPrkdtVt$W6w)xD+gDWL9K^$Yu15wkKkw8B6EG}MrrSH{SDu7>p{3xjP}
zUTr1wnO_rx84v&NuuNqIH|dcFu@&CMe<c5I;Zz((aw%X?=kd#NVQq#YWoSqcwft@Q
zS1DK^6uTV`hA>s4U6;GrvE;VpXfsDY%2qdIO6C9lhIWw1+*jq^7e<^p<xKHULMhH|
z3wBRWc_&{>5s(Xd*I56_+PY{<n;;>Kziu^_l~Kb}1ND!I3$F%j+mhP)6{woHJR%j7
zN{Go}dU58e1#qG=OCx!;q!)jFtB=2J>-!Y2fl;WDl=O3-2TVP=HnX6A-+viboc0oN
zm!qv7y5L;iRdB@XxPK!I`nQ#ts94N)aJcg`bNz@6GuyHwz9)5MvFiT1Qpo?kI#sW7
zctplM9F9w_W!6Z<-*EYH5#GDCo9gWit-oC>^*okgd}+sCM_jdA^!8`qK|;y_M)8mM
zz3mr^^z5UIY;gYAyq`5nQZ13w&J9~`YH4S@D24BeyfBv@@oOO8_j6O9`nQHQBE-LV
zKzgLIZX4M%eHiIO&|w0`(^eOQ=OadpTL;51(VI`SA+yaZUbnm8h05&miFKaALRX2l
z(<IleKKfm+(G+yo8U>FCFM@Z!ly+{9NsmR04QLr~szo552T`0a&0oue6mc~LN0lpW
zWYUC)bA!@67YoX}^cx*7)j`hFWKW-@T6vnSOlf4D<okf6zDE?^?`@wp`myWG;Q_}0
zt{>Q|6T7WViU9u31X#hdYW-7h!caIt7-MUP_S>%+;5rIbb3r@muP8|ygx;W5ybIcv
z>dO#FXr&dM*FBwHYK_ULtMS}>2+2@{7LRW=a0opeM$H(RVTF@i#0=|nKtXk%ABJ8c
z_ydKDb>CNslvWE$zpp^{u^CPsWt&9@p&|^4z3|7TyVw(6?;?GN;A)FspdVi6Uc<)E
zg{oq|)Ku|S@5AR#(uPs(?Skvs;1I)%E)8DF6=sezl^0sMVfjgL^sS<or)kB(mxkUv
z2Xa6po!tXB;~cuBGD?Ndk)HSqS<q0=cCVrrTF25T<oNu0`2wwXu4_QeUgYv%`DI~~
z$i;*eUVsCFQ9*1IGMlI`WXo!02|PPNIUh2Uq$R%Y-g%4lJou+iF45G<nZDP$WW)R3
zo)=|(#)C`a%0=TqXr1CnW$ue;CIw|JG+oRzS!LnEBOusY-$f2(ini+|dT7~D$PZ>I
z&rOY{?ajQ*(7oSXawd|PLjSV9^;5k)z}B#8_SEJD>BH$k?OBdgJBG160S}9g>4FSF
z2&)L5kzpIh<xxEg?ODBf{s&=Oy(*7E>cP%n$uO4k%vo8=Aw$8@k0xd!k#;(oRyCBi
z_d{-Tm)URF+^j?q!5x{lF9K~qW3GP|gVsf~wT11$SW6B6S~K3x<4^5u``%w}ZlaSm
zN*-`~WLA!h##oTD6tUZ$4^}6R4(@8UzfTIbV_#TiwTG#$8)me+87kEmkaE0GngqM#
zdP~12K%(-zBb4v21X6qW+uV=QwTKJ<UzJ^VR8viq4<MjOFVYcFia_W^x>OOQNR^Is
z2t_&}BoskGnuvmcB=p{tUP2HAq*v(>={<A;p>4iz&z`gU$L=}%{&{of+<V`fncv*G
zGv~c~y{19JyFh|=jUgx`XVs@O9+K(#a`Nl>{d%@f9(j0+(zO>!!3skzke2y0DM&+h
zTdGEciT;Lh_m$N9e`A>+-zX>H$F6vQ(FRPCe;40*Ia8@P?`;>7g{QoQ7p*47qaZGO
z2oB2Qn3Ki;N^g%t(Z2R=!(`eJYDLM-X&{2|RW)|%Y#mf1zFj<meEU(P5qn$<Xq*1B
zVkxI(o@{y5eZ6z=Wn}B~RE(2$lw<~#Kda5N8swkDKHY|e%-k^N0q8=Uman6e*NLcj
z6%F9y&>u|E9p|=l4e!BD7X&1DsS5OrCw`jil%)j^%=O{ot@<-F!{jF7pUzvu(>-@B
zsj~3acIINieli&G?epKaS;a)wx^Tliyr*oPdqfZ%j|g(v{kGc!;Z>vn^z^(3sGYE|
z&;P#QbH~rl)lO!%EC>iw)&Lq<Xqepe9ec*xGY|rd2Qx^(c-{B+twQft1w(Nnvz%Ct
zzB1;A*I=wgN~VVN-$t|Qt(){?HN3mVdyQEan`kArJxwWT1F}N4r4|$<L@qb=fTn|D
zm6@h-6xi?AcR365PFbDnH#h+!%5J>ef3~e>N0icV;Y~Q*M}Q|zjck+SmV1LQ+wQCx
zK1Q_EtTNS?ZQ-^~)8CMM^P*q>XQu9)h=0^+R%fXmdbMH`%llxi+kC*EV!`^}ga|N+
zcf1-a`zO$kuA54sCO}u?4*yZYd1QA1TkkCqScqZkd}tN-{V+=7=TAtCY%WmVm17Iu
zhvGKc_SVGl=sT?PA(Lp*GTQb(b;>iH(d55~(g4zm&bsdTWQz7Ijd$b$jyN3nVlxkG
zpL*Y>nK3Ox`{Va~G7G5`dDD}Lvq@BKdBHno{Ze+6Z;2IOF*ngcU2RE2lZneW*=Aq*
z{d7G=v#NSrqyY8W#<Gge6Cxqc&s}QU4;d}TJd;Rv#7{i3MaBQR?XC4{3`A5fHcK=u
zj3_ou5g4aI3_sQL`jrY2gtcWOmxm*pTI2dDjE$K=YawC>y9hUh>6J>uN!>2VEo0j;
zQ~Uq2AX3@A5YTQ51>^@6R2B-R)LJkX0X=XG#`}x(kxN5eM)!Et*~Nz};2}c3KMeu*
zxYy%C*#So3-z}f>g!Q((K<q~@?HH!i;%g`{BQt9jeKw{Sksa>u&YAua)nYWrs}AOY
z(;4<=4G>Jb=j*k49-XF<2Qp|r9S#qClX#|_jl%zGxtl*RsRXbzBi*pD5`ttvn`K)0
zX^fC!s=-AJF46ONo^BG>6sUc;@M(!3aG?5Z(qG<EsZWWu+ClZWO9pSXGn8=SB#6j^
zhRsSt$bVw8405J}jfbr*8E$cADCpn=K*sn2NR4>Tk1+~7A#;U*D(-5x37xN3794nY
zS83J;0v=HvTc5k}*}FVPoR%o!id9Y)<{&O7%`5Qx=x3{^NF|s-X$wIUMb(@#*-3hR
za7O*J{1%kc=*~(vd;lDJ5YJ@0++1yKl94^T<h$F-1mIineXHWr`Z8U9reK~c9_^e%
z8*Bqwq@(xJyZ#PK?O|<1+BH)YO<(Yyu%d)|4{z-I@JGCBz*rC|I&4ITZGvK$y;pD=
z{w=OR(VvR*c2H0Vq2G4^en?4+)W&uhzI3;$9h{P={rS&DLH7O4Xv1{M4U5vvEMsUy
z?=B?Ol$1HtgUa%Flynb$vjbXoGUd4VA?674_b@hEh8zR%v(j00+_3NHdum7WPz5v~
z_#{*LU>RO5_yQWlM0kTjUc|dD_a6bSJ0Hp!#uXC~m<U$4He^4tj25PMh<fOwtv&Y6
zB|~$5B@g81_Rxy9xJS#?#{Dn{$=KIAb?j%hISL;)`E%Oo8IZqY;u`D`?5a;49jm<(
zHhR_`yxOt;I}E_cv^B--D||`HN<I}P3Wq5zb?yhr4|%~4&ZNbM#O>w6?oK{ikRJzs
z2i*en&e+JE?v#K+md|2G9O2D7up|BTn85|F@NI*WUyI_6uYT5=nwm^<WIj3Qx<H+|
z7|c>%kOi&ZM-RNz?DP8UpBMnG|2?B5&i>4XXdED9iERbv9d%qxTRk;jK6O(H7R(R8
zCcIeqvj5@&G!o;RI?tP6*&sGXSsNS=avpB`QBM>Gq8(0$&m#}Z6Q}t}XA+SFS$(mn
zkQ2p;>tepxd7%@W#pHtgXyb>E6F(Tsu2dVAwy1vHOki?yaRaKEjFr!x0bJw~x-=%6
z0}(r)_JR9&e%)`el)z5$Zl6O1vo}E}igN3El1St0B<#`l!-t-LqPnlLGeh?i;=$mO
zTYIoK*OPmJnh8Tzqqk~EJhU&)r2~{Cl`w$^M1rP#F~Fl-${z1A$<3&yTlX%q4T0T(
z4ngWZoh;Smo>9bb`Ot!+Z+EQU*9UiJ*C%55#{A#{-I2|cwmmQJ3Zh=7q(W=9Thu46
z(>7hi=H$=Ax9brq>E34MgWsxcLYDQNlFB+xx2CLMNByJ~RQ9zf|B9>jO&gVT0&21-
z34!=4MeybHSTeW4(E3xCpuq0R!TJ_oJ2jRxK$!k*79A2$suL@_tUGEL!rUf)n&4hB
zC+%>(2V&Qm3<bTH@Lqg)Lpn4lZ*1@*MXxjI2-S4CjkaI8&TDcQt&&SlvDx1<KaHpj
zy<L5?@EZr=Ts(^k1>ejME$%9(x$gDY3uB_y&rc5ud{+@ol_84EUIcxM2}kcIZ=O_A
zAN0tSijv1g0<z4L*TR#<!=&k<d5;9sKNIjIoN<2og_KD947g2z(LYXti@5#>TH5%~
zH&oa3f-s>@FUR{`cn@`nq{zw-;uJKWfad^t?m*|gpXN&V0G~Q@h$JVifUgqipJGsK
zvb0crtAZhNv*lC5H%gJrZF%7Y4XhvHzx3{R4KlZ}6D|!jZ0rh@S>b{_t!3xWc>#N^
zH-#yMV#uGK=T`RK>M{C(-O|$~N(?dOS;_@KL_nu0lF@d+#`5<X#GV07)T2jsNVGuR
z8&-G^+BGpsSZYYEH~6|h_|1cEpUT5^LFrbgR7+O|>WMdcmm}?GPd%0;eMeg9vgRnA
zF|2s8t$XiEehBeVa<a931}NoAM~A5{6sFP~HlMk<R2~f~lb2Et_lg>@Jy@ofj*0kv
zjE~wiHF)jb_)H&(U2mUy@9Vrg>%0uwnSmByXRDu{c{*3#K~}u_zWTez5FFzSwXi?0
zwcwNx_dV5LcQ}4t#U}^jS$%K`cK@J)CNtUy(zTgl>szq;wzz6YJbQLJea7M(7TB83
z)G$YUV-@L~C0P_Wm6$mPt<gwM4VtKzOtEQ>5`VAq-uj%oU7W9aYKTn%{mJKCK2hi8
zhQ({~pv9=_k(f%<f+b4FlzRQR|5#1yFR6yg2tq&VdG!Q(c-~#irr8Azp5d_Mcj+Tf
z@2UK^QF?i|NZP7;S_zJR1T=q{y{+Vv0sP5$Z$VE_O)bv>uSHEwtLw%8hX~Q&pcyIP
zQssDS`^xAEB|wFk^AQ6t867w5_Ls*;4o1#^+d^d$=3m`m@3@zGax`vo{PQed7oatj
ze-qpee%c@S$Tf;nkLdMN;%o`Y-<&ItISG|XU5?m!lRiD?c;B&Di^91Xq{*LTZOf&S
z+d5|!)YM6)TfS8+tPW+OE*sDou_flzzJ{~v&W2y5mwl>SVcrNJj@W`$=H)Gf!>?mR
zncFXYU3R}Sk9H@2sSubi-qw`)^#bLsF<-HVA>W;=M)ukb1PaF9y1Kqr918Ll;fgNV
zpBS~(8RKvP0e***!msy2#$*9qORm)(srqg`femWC^v6_BG<59uc^Gvmr~<%#{vHE_
z!+(5dE0Csc?ttNSKSpT`V_568@GfZ?;N7>Y^rnKHQTaEeeZ?NZ(>XS04&WR2DLbz}
zTZ|lY!VG#QU5E4IO8h&8Hi6>WNRbr|cu7d`@7<ojNY^J298TTB(c<DXb+&i*lxoYp
z{^FViTD%_6W)G`y3e5nAX3SRAS5Btw*~ZX=Y{zXuQbK%ee&<`PZlKyKmzQXLxs4C(
zq;UfpWg5FexdHA{gq-yIbF7o*K;ym%>PUGqu;2BMmYw@|-?5(1Y5YXlPQ2lB7$zej
zh(WC`KYwsRAY+~SO*T{B0`TMh$HV!3vTk3T|5}6J_syx-pLbsA3y~60p{VLP+&5Zf
zACVG86eXhmzSmD1olYshYDdUZ08~WvSrEM643x{d1%lJqAE%;ouU>x_lBS8^h$YvF
zq?$Rgt5KnFW1&!d9Kc>#QQoK$9c5r#iNYx*h182C*_pAH+`Dm_ox)a6_*nVPWKB}&
z_VZYea?<N%;sd9JbvEh!;&XVAzpnT|0<*n+@WREhtt{2;bSA5Kt7+?l+VDx#-6O_z
z`}U+N^Kxr>dfT)hT2HH8&zft8H@4;pn$2a#A4P-oRMci_OLN9YX)~X(V0<ZbpTGY3
zimhs=T3I5(=WQCSRwbdPld_Mp8=rkycBpMQ$r5uvV7p|1gr>=@I&{jT%p)$PvOy`b
z%L;x?dTj;_<M$4pg?przkT@fxMeaod%_%u)>+o2fnP+;6-^OUnN~QYBQ`7sEY~s4g
zbH0OL&vjZ9k~IIyl*=wARHLc-RXJtMogl5!nl*{cKimwFR!FkTzjB=r$%?m0VpM<N
z9alMFtatwHWA=ox_Voj=hIXMKTm_}7B-$6T^D`5cIAK0(nM-IXgf8nan;f+c999fi
zP4p+-b={(P*1n*)j<i)v3H%dl9t&joYjE(0^-hP3qr|SXB3p7W1jV<|I?0xaxv&du
z!44f%jgC8&-XbA=mp}WK;^OP3Z&VuW%i#46*0nGH;|gU56t@Nm)$BR;OboO@vbRaa
zb}CGL`F$*ap+a66mNZFKE8n&KEsoi~D%kx(l03k|%0$O@e`m`i4c<Z9EE@Lyb)*8W
zygdGkTLaRtp3SCKxc3-h?lC`)<)70{7P2&Zn37(n3NjPvUF$MVee_}$UfLHpOLrA?
z=ea4W-YlcpQ;xtLR41E>-kGaLZ#J0B(eF0i$j<m8`d2StXo6hpTVbQJ9pxaTf3#Jy
zbl&bv<(ynm$rLVvi9C09v+iehA|jzM-Z|XLzsrCkPGgtBc5m>^Z&dPUX7dZ<5nGlx
zE7Kdf?Fv{gllT=s`&9?TwGSyRj0|mDoE|jniSl_9vU2gpH~*H*E5pa=Ov=8W-pX-S
z6c(`1sp$zz?pqHM32ELkBj$<2K5TeC)@x*gx@m(zR>Ub&@%F~A1{k;iM&?>d&B;9O
z+Qs^fsMjt{Fc*o9<1>FOW}1apYoKlgyhT~pXqgKu%ymPDhe#`pBz`cbd>)~3_bDYw
zw!t%PR`Q&KFp99hyQqAr>5^Z~)kz4JT7crVj&H#gUqlci`*F^`OMXHFVkZ7Oh@8bp
zG96MeFJo&a1Si&#e0u5k04$rWRr>r^@@YGuoY4tq>^7PwjKX;02E5V(KIhTn-YJAt
z?8RmWg%7SDjE{GUT^(MicGmQb?Hy`&W=DT)ve&}2S|hSMtqT(eQT?8lG{QDqZU8=p
z;UdPYNfHncd)c)BFTF4nN4_M2LjR<iI)OKIYLY(m2^8{%1giqW+8Xo<5qmn7)Bv+4
z(Z97Ah>T)CPtA&Jotw7Z$B%5}Piq~cK0xbjUEXmL4nGhPkZ-h;H+%kV-+`XIoIw0a
zR?fGi*x_aNdSz=NV%CNM71K$58_Z@iF)r!!qYYD`44#a2QY|hbX%GJ{+)4t7eCLG)
z3wWHGUtPXdu50!dlL06*Ds)j3fEq)%X;H!tpOB9*L7&IkMq)wKs|h=LY1HI^7yc8Z
zMAGU001=4|F^kzxXwfYWa?|lY8<GH{&LahEY^s%K$^D}z6ti^?Xb&1WKC^D8Ca$uY
zx{e)}llcNUA-MN2yLncijUv1sOhN*r;3*ovCavJ5s6F=#=C&`eagH5p5sW7_GQQwn
z(<`K4V5y(s)LxFLv+E>Ih^l8Br-*SQidcqci|CX0D9B2zJeI6af2*<tQRW*8kk*wN
zhE{fQZtC1CQjg5{t}kqmJw3Isg1_?+g6aT`O2Q^{OBdjKQalD_6{SLer;dJHvP`WD
zMZ1)BRw&WiPu*Q+JvEztT2an>_e|7HSvMj@7ks@y)-7*J|BS9vtEaN^E0+|ZCqFY0
zn5Id6wE|9HI1&dX1xqFDprBY8lRck<{!+`==Ee%1F2{3k9ti4e3)bCJ%zyYEZ!iYe
zGb+l_jakD6rt3#K<Kre}4NC4FZp|ZsdU51YvGtx_6_q9zm7%-Wih_#8hk?y&d!e;;
z=Hd(SV>^Fl6zVGET9-(vqBQkQVix&J2ktcsOxl7rn|2>xd%glhtgaQTrH9n>CE2NW
zw+S<y@B;#31IFFyXzl%$9F*-9hCeasrEWA<YD%|PsJyMaxBS`tRu<#h#i2RX>==nX
z{mT(2%HK}@TZE}Gq?;*e0Zk`gpd<v#U0otsbY~j_bNuNU8GYKF$;rv*)>sNLZdCz6
zwhvrHpW+M%U|`tzw!zr47_FQt_3gavNCCzyf{3=y<awB`o>=osFO_2?%~Ohv$j=<m
zJ6}_>@&6;~;5co#a$5KV1p2g*_`Gp5NP%3rYz$>nkS@YEe{-~iT3AoFNn3~azGSTS
zJQZE2vE}hr#Cd8kWCp9@O0Zi3>xhVsK1|3hjgF4~9G3=<j@In?x*79-6WB3l=OEe3
zJ*?^tmWW4X8abkH67=VbxfY{mfk5bxssxuobWRReyhg6U0SIXNb+dXOWw?k4WhLu1
zLwQ5T<ng(xbGM>3@2st@IhetYH%3x`7GHOKBhwV`-W+hYFptWI@xp_+(DBM@HI#TJ
zt|U!9`rykmQE_NOm07ije8!1|PtsaUG<|MSktX#mYvj=5I`?~B_Z_jIr!S7w)Y9~e
zjrxxpJ5mN13cy@B<oFC!_-j>;tkbSw=nlS$Yv$2cqX{%V_Z!D0ZBjcu1H-_8<D{w<
z^ODS!?DDic4UGqX|M7N=E51jhSe<bG9TIp?HiMe3A2;<^Vh9$@Wx#YgwoW;;IY}^t
ze-W24y8;`5e--!R+gqjn>Q(!{5^Unod)Fq2QOk3!BT#W_v<n>`0QS64%zlm7-5bkN
zncX1z5c&5Zh2VtW54pIL9MjSi1MwPc`Dw0C_Ka@U_T@OV5}j2_Ei`pv_B;CdiJ0Ke
zFI_MbA<`r?BeEYIWpJgn=ag#rQL8?Z4nA1AyB6|{;Q6ORc0w8&29nb;IvSeC%;E|(
zG)bRC|2yHW#z8~#a<NfvqxX%jQ!p1-PWur~=}MKS8hY8Y7j*K!LgC>d9xG&1bTfyY
zQA(P0_%8ZKEko5x9=tB{9|RRuaTlI;#vuMJ|5c~ET#}&r=K)-fni*?9|J@O||GPaP
zE@)~JJCaHJ2XAEk;i>SOvt)Gnli6;>C8kY#h;d+GV3oTUpAooaTw6`8#=5t+_t3!W
zO#p3M?agInQquSQwc?5j`TbG+@ydMydXr2w71#*?0ATV{f8}TI<>BGz=?C!hcq8n?
z3FK6D@$vF-_4lw)@h8OdaR0;O@Y>H&*51bve*@q@<QD^qNCU;BjYK45MPy_pq=5ev
kL~79x+y5cp;pO1!1peOzenoW5;ROKC)pgY>RBR*u1FBO?7ytkO

literal 0
HcmV?d00001

diff --git a/static/css/bootstrap-responsive.css b/static/css/bootstrap-responsive.css
new file mode 100644
index 0000000000..0bc6de916b
--- /dev/null
+++ b/static/css/bootstrap-responsive.css
@@ -0,0 +1,686 @@
+/*!
+ * Bootstrap Responsive v2.0.2
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */
+.clearfix {
+  *zoom: 1;
+}
+.clearfix:before,
+.clearfix:after {
+  display: table;
+  content: "";
+}
+.clearfix:after {
+  clear: both;
+}
+.hide-text {
+  overflow: hidden;
+  text-indent: 100%;
+  white-space: nowrap;
+}
+.input-block-level {
+  display: block;
+  width: 100%;
+  min-height: 28px;
+  /* Make inputs at least the height of their button counterpart */
+
+  /* Makes inputs behave like true block-level elements */
+
+  -webkit-box-sizing: border-box;
+  -moz-box-sizing: border-box;
+  -ms-box-sizing: border-box;
+  box-sizing: border-box;
+}
+.hidden {
+  display: none;
+  visibility: hidden;
+}
+.visible-phone {
+  display: none;
+}
+.visible-tablet {
+  display: none;
+}
+.visible-desktop {
+  display: block;
+}
+.hidden-phone {
+  display: block;
+}
+.hidden-tablet {
+  display: block;
+}
+.hidden-desktop {
+  display: none;
+}
+@media (max-width: 767px) {
+  .visible-phone {
+    display: block;
+  }
+  .hidden-phone {
+    display: none;
+  }
+  .hidden-desktop {
+    display: block;
+  }
+  .visible-desktop {
+    display: none;
+  }
+}
+@media (min-width: 768px) and (max-width: 979px) {
+  .visible-tablet {
+    display: block;
+  }
+  .hidden-tablet {
+    display: none;
+  }
+  .hidden-desktop {
+    display: block;
+  }
+  .visible-desktop {
+    display: none;
+  }
+}
+@media (max-width: 480px) {
+  .nav-collapse {
+    -webkit-transform: translate3d(0, 0, 0);
+  }
+  .page-header h1 small {
+    display: block;
+    line-height: 18px;
+  }
+  input[type="checkbox"],
+  input[type="radio"] {
+    border: 1px solid #ccc;
+  }
+  .form-horizontal .control-group > label {
+    float: none;
+    width: auto;
+    padding-top: 0;
+    text-align: left;
+  }
+  .form-horizontal .controls {
+    margin-left: 0;
+  }
+  .form-horizontal .control-list {
+    padding-top: 0;
+  }
+  .form-horizontal .form-actions {
+    padding-left: 10px;
+    padding-right: 10px;
+  }
+  .modal {
+    position: absolute;
+    top: 10px;
+    left: 10px;
+    right: 10px;
+    width: auto;
+    margin: 0;
+  }
+  .modal.fade.in {
+    top: auto;
+  }
+  .modal-header .close {
+    padding: 10px;
+    margin: -10px;
+  }
+  .carousel-caption {
+    position: static;
+  }
+}
+@media (max-width: 767px) {
+  body {
+    padding-left: 20px;
+    padding-right: 20px;
+  }
+  .navbar-fixed-top {
+    margin-left: -20px;
+    margin-right: -20px;
+  }
+  .container {
+    width: auto;
+  }
+  .row-fluid {
+    width: 100%;
+  }
+  .row {
+    margin-left: 0;
+  }
+  .row > [class*="span"],
+  .row-fluid > [class*="span"] {
+    float: none;
+    display: block;
+    width: auto;
+    margin: 0;
+  }
+  .thumbnails [class*="span"] {
+    width: auto;
+  }
+  input[class*="span"],
+  select[class*="span"],
+  textarea[class*="span"],
+  .uneditable-input {
+    display: block;
+    width: 100%;
+    min-height: 28px;
+    /* Make inputs at least the height of their button counterpart */
+  
+    /* Makes inputs behave like true block-level elements */
+  
+    -webkit-box-sizing: border-box;
+    -moz-box-sizing: border-box;
+    -ms-box-sizing: border-box;
+    box-sizing: border-box;
+  }
+  .input-prepend input[class*="span"],
+  .input-append input[class*="span"] {
+    width: auto;
+  }
+}
+@media (min-width: 768px) and (max-width: 979px) {
+  .row {
+    margin-left: -20px;
+    *zoom: 1;
+  }
+  .row:before,
+  .row:after {
+    display: table;
+    content: "";
+  }
+  .row:after {
+    clear: both;
+  }
+  [class*="span"] {
+    float: left;
+    margin-left: 20px;
+  }
+  .container,
+  .navbar-fixed-top .container,
+  .navbar-fixed-bottom .container {
+    width: 724px;
+  }
+  .span12 {
+    width: 724px;
+  }
+  .span11 {
+    width: 662px;
+  }
+  .span10 {
+    width: 600px;
+  }
+  .span9 {
+    width: 538px;
+  }
+  .span8 {
+    width: 476px;
+  }
+  .span7 {
+    width: 414px;
+  }
+  .span6 {
+    width: 352px;
+  }
+  .span5 {
+    width: 290px;
+  }
+  .span4 {
+    width: 228px;
+  }
+  .span3 {
+    width: 166px;
+  }
+  .span2 {
+    width: 104px;
+  }
+  .span1 {
+    width: 42px;
+  }
+  .offset12 {
+    margin-left: 764px;
+  }
+  .offset11 {
+    margin-left: 702px;
+  }
+  .offset10 {
+    margin-left: 640px;
+  }
+  .offset9 {
+    margin-left: 578px;
+  }
+  .offset8 {
+    margin-left: 516px;
+  }
+  .offset7 {
+    margin-left: 454px;
+  }
+  .offset6 {
+    margin-left: 392px;
+  }
+  .offset5 {
+    margin-left: 330px;
+  }
+  .offset4 {
+    margin-left: 268px;
+  }
+  .offset3 {
+    margin-left: 206px;
+  }
+  .offset2 {
+    margin-left: 144px;
+  }
+  .offset1 {
+    margin-left: 82px;
+  }
+  .row-fluid {
+    width: 100%;
+    *zoom: 1;
+  }
+  .row-fluid:before,
+  .row-fluid:after {
+    display: table;
+    content: "";
+  }
+  .row-fluid:after {
+    clear: both;
+  }
+  .row-fluid > [class*="span"] {
+    float: left;
+    margin-left: 2.762430939%;
+  }
+  .row-fluid > [class*="span"]:first-child {
+    margin-left: 0;
+  }
+  .row-fluid > .span12 {
+    width: 99.999999993%;
+  }
+  .row-fluid > .span11 {
+    width: 91.436464082%;
+  }
+  .row-fluid > .span10 {
+    width: 82.87292817100001%;
+  }
+  .row-fluid > .span9 {
+    width: 74.30939226%;
+  }
+  .row-fluid > .span8 {
+    width: 65.74585634900001%;
+  }
+  .row-fluid > .span7 {
+    width: 57.182320438000005%;
+  }
+  .row-fluid > .span6 {
+    width: 48.618784527%;
+  }
+  .row-fluid > .span5 {
+    width: 40.055248616%;
+  }
+  .row-fluid > .span4 {
+    width: 31.491712705%;
+  }
+  .row-fluid > .span3 {
+    width: 22.928176794%;
+  }
+  .row-fluid > .span2 {
+    width: 14.364640883%;
+  }
+  .row-fluid > .span1 {
+    width: 5.801104972%;
+  }
+  input,
+  textarea,
+  .uneditable-input {
+    margin-left: 0;
+  }
+  input.span12, textarea.span12, .uneditable-input.span12 {
+    width: 714px;
+  }
+  input.span11, textarea.span11, .uneditable-input.span11 {
+    width: 652px;
+  }
+  input.span10, textarea.span10, .uneditable-input.span10 {
+    width: 590px;
+  }
+  input.span9, textarea.span9, .uneditable-input.span9 {
+    width: 528px;
+  }
+  input.span8, textarea.span8, .uneditable-input.span8 {
+    width: 466px;
+  }
+  input.span7, textarea.span7, .uneditable-input.span7 {
+    width: 404px;
+  }
+  input.span6, textarea.span6, .uneditable-input.span6 {
+    width: 342px;
+  }
+  input.span5, textarea.span5, .uneditable-input.span5 {
+    width: 280px;
+  }
+  input.span4, textarea.span4, .uneditable-input.span4 {
+    width: 218px;
+  }
+  input.span3, textarea.span3, .uneditable-input.span3 {
+    width: 156px;
+  }
+  input.span2, textarea.span2, .uneditable-input.span2 {
+    width: 94px;
+  }
+  input.span1, textarea.span1, .uneditable-input.span1 {
+    width: 32px;
+  }
+}
+@media (max-width: 979px) {
+  body {
+    padding-top: 0;
+  }
+  .navbar-fixed-top {
+    position: static;
+    margin-bottom: 18px;
+  }
+  .navbar-fixed-top .navbar-inner {
+    padding: 5px;
+  }
+  .navbar .container {
+    width: auto;
+    padding: 0;
+  }
+  .navbar .brand {
+    padding-left: 10px;
+    padding-right: 10px;
+    margin: 0 0 0 -5px;
+  }
+  .navbar .nav-collapse {
+    clear: left;
+  }
+  .navbar .nav {
+    float: none;
+    margin: 0 0 9px;
+  }
+  .navbar .nav > li {
+    float: none;
+  }
+  .navbar .nav > li > a {
+    margin-bottom: 2px;
+  }
+  .navbar .nav > .divider-vertical {
+    display: none;
+  }
+  .navbar .nav .nav-header {
+    color: #999999;
+    text-shadow: none;
+  }
+  .navbar .nav > li > a,
+  .navbar .dropdown-menu a {
+    padding: 6px 15px;
+    font-weight: bold;
+    color: #999999;
+    -webkit-border-radius: 3px;
+    -moz-border-radius: 3px;
+    border-radius: 3px;
+  }
+  .navbar .dropdown-menu li + li a {
+    margin-bottom: 2px;
+  }
+  .navbar .nav > li > a:hover,
+  .navbar .dropdown-menu a:hover {
+    background-color: #222222;
+  }
+  .navbar .dropdown-menu {
+    position: static;
+    top: auto;
+    left: auto;
+    float: none;
+    display: block;
+    max-width: none;
+    margin: 0 15px;
+    padding: 0;
+    background-color: transparent;
+    border: none;
+    -webkit-border-radius: 0;
+    -moz-border-radius: 0;
+    border-radius: 0;
+    -webkit-box-shadow: none;
+    -moz-box-shadow: none;
+    box-shadow: none;
+  }
+  .navbar .dropdown-menu:before,
+  .navbar .dropdown-menu:after {
+    display: none;
+  }
+  .navbar .dropdown-menu .divider {
+    display: none;
+  }
+  .navbar-form,
+  .navbar-search {
+    float: none;
+    padding: 9px 15px;
+    margin: 9px 0;
+    border-top: 1px solid #222222;
+    border-bottom: 1px solid #222222;
+    -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+    -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+    box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+  }
+  .navbar .nav.pull-right {
+    float: none;
+    margin-left: 0;
+  }
+  .navbar-static .navbar-inner {
+    padding-left: 10px;
+    padding-right: 10px;
+  }
+  .btn-navbar {
+    display: block;
+  }
+  .nav-collapse {
+    overflow: hidden;
+    height: 0;
+  }
+}
+@media (min-width: 980px) {
+  .nav-collapse.collapse {
+    height: auto !important;
+    overflow: visible !important;
+  }
+}
+@media (min-width: 1200px) {
+  .row {
+    margin-left: -30px;
+    *zoom: 1;
+  }
+  .row:before,
+  .row:after {
+    display: table;
+    content: "";
+  }
+  .row:after {
+    clear: both;
+  }
+  [class*="span"] {
+    float: left;
+    margin-left: 30px;
+  }
+  .container,
+  .navbar-fixed-top .container,
+  .navbar-fixed-bottom .container {
+    width: 1170px;
+  }
+  .span12 {
+    width: 1170px;
+  }
+  .span11 {
+    width: 1070px;
+  }
+  .span10 {
+    width: 970px;
+  }
+  .span9 {
+    width: 870px;
+  }
+  .span8 {
+    width: 770px;
+  }
+  .span7 {
+    width: 670px;
+  }
+  .span6 {
+    width: 570px;
+  }
+  .span5 {
+    width: 470px;
+  }
+  .span4 {
+    width: 370px;
+  }
+  .span3 {
+    width: 270px;
+  }
+  .span2 {
+    width: 170px;
+  }
+  .span1 {
+    width: 70px;
+  }
+  .offset12 {
+    margin-left: 1230px;
+  }
+  .offset11 {
+    margin-left: 1130px;
+  }
+  .offset10 {
+    margin-left: 1030px;
+  }
+  .offset9 {
+    margin-left: 930px;
+  }
+  .offset8 {
+    margin-left: 830px;
+  }
+  .offset7 {
+    margin-left: 730px;
+  }
+  .offset6 {
+    margin-left: 630px;
+  }
+  .offset5 {
+    margin-left: 530px;
+  }
+  .offset4 {
+    margin-left: 430px;
+  }
+  .offset3 {
+    margin-left: 330px;
+  }
+  .offset2 {
+    margin-left: 230px;
+  }
+  .offset1 {
+    margin-left: 130px;
+  }
+  .row-fluid {
+    width: 100%;
+    *zoom: 1;
+  }
+  .row-fluid:before,
+  .row-fluid:after {
+    display: table;
+    content: "";
+  }
+  .row-fluid:after {
+    clear: both;
+  }
+  .row-fluid > [class*="span"] {
+    float: left;
+    margin-left: 2.564102564%;
+  }
+  .row-fluid > [class*="span"]:first-child {
+    margin-left: 0;
+  }
+  .row-fluid > .span12 {
+    width: 100%;
+  }
+  .row-fluid > .span11 {
+    width: 91.45299145300001%;
+  }
+  .row-fluid > .span10 {
+    width: 82.905982906%;
+  }
+  .row-fluid > .span9 {
+    width: 74.358974359%;
+  }
+  .row-fluid > .span8 {
+    width: 65.81196581200001%;
+  }
+  .row-fluid > .span7 {
+    width: 57.264957265%;
+  }
+  .row-fluid > .span6 {
+    width: 48.717948718%;
+  }
+  .row-fluid > .span5 {
+    width: 40.170940171000005%;
+  }
+  .row-fluid > .span4 {
+    width: 31.623931624%;
+  }
+  .row-fluid > .span3 {
+    width: 23.076923077%;
+  }
+  .row-fluid > .span2 {
+    width: 14.529914530000001%;
+  }
+  .row-fluid > .span1 {
+    width: 5.982905983%;
+  }
+  input,
+  textarea,
+  .uneditable-input {
+    margin-left: 0;
+  }
+  input.span12, textarea.span12, .uneditable-input.span12 {
+    width: 1160px;
+  }
+  input.span11, textarea.span11, .uneditable-input.span11 {
+    width: 1060px;
+  }
+  input.span10, textarea.span10, .uneditable-input.span10 {
+    width: 960px;
+  }
+  input.span9, textarea.span9, .uneditable-input.span9 {
+    width: 860px;
+  }
+  input.span8, textarea.span8, .uneditable-input.span8 {
+    width: 760px;
+  }
+  input.span7, textarea.span7, .uneditable-input.span7 {
+    width: 660px;
+  }
+  input.span6, textarea.span6, .uneditable-input.span6 {
+    width: 560px;
+  }
+  input.span5, textarea.span5, .uneditable-input.span5 {
+    width: 460px;
+  }
+  input.span4, textarea.span4, .uneditable-input.span4 {
+    width: 360px;
+  }
+  input.span3, textarea.span3, .uneditable-input.span3 {
+    width: 260px;
+  }
+  input.span2, textarea.span2, .uneditable-input.span2 {
+    width: 160px;
+  }
+  input.span1, textarea.span1, .uneditable-input.span1 {
+    width: 60px;
+  }
+  .thumbnails {
+    margin-left: -30px;
+  }
+  .thumbnails > li {
+    margin-left: 30px;
+  }
+}
diff --git a/static/bootstrap.css b/static/css/bootstrap.css
similarity index 99%
rename from static/bootstrap.css
rename to static/css/bootstrap.css
index 495188af7f..dee87331f3 100644
--- a/static/bootstrap.css
+++ b/static/css/bootstrap.css
@@ -1395,7 +1395,7 @@ table .span24 {
   height: 14px;
   line-height: 14px;
   vertical-align: text-top;
-  background-image: url("../img/glyphicons-halflings.png");
+  background-image: url("/static/img/glyphicons-halflings.png");
   background-position: 14px 14px;
   background-repeat: no-repeat;
   *margin-right: .3em;
@@ -1405,7 +1405,7 @@ table .span24 {
   *margin-left: 0;
 }
 .icon-white {
-  background-image: url("../img/glyphicons-halflings-white.png");
+  background-image: url("/static/img/glyphicons-halflings-white.png");
 }
 .icon-glass {
   background-position: 0      0;
diff --git a/static/glyphicons-halflings-white.png b/static/glyphicons-halflings-white.png
deleted file mode 100644
index a20760bfde58d1c92cee95116059fba03c68d689..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4352
zcmd6r_dnEu|G?izMxtxU%uI5!l8nr<BF?zWUS(u;&WdwZC0F)1B-!J<$%*WB$U3Xi
z$ta3LaXK6#>)ZF&&*%FGe4jtO*5mbhJzhV&et11z&&^B?xH$MZ007{+ZK!Jj01(PQ
zJBFS4pH$0DefCd1HM@h*JNkcsi%oOXzj>qsEle$eQ7ApHL(XYdn5Y$Lk_3-J9p9d)
zFeVfl3J47_g1XaoDXWsnBp9ZzZ74CI9RN-Nw{>+8A&#rBpZgc9WX2H3Ssv6doZP?t
zS!g}lGvW1<9%?dj_G_x}3WUMN(8(x{a6_pd0yiUsf^67GGS50uSB*ORe5x6}qAf1z
z@Q;2y4G{Lb?f21p)uTpChN&4q%^blZ2IsusUOhk)pe0<chGtjyTP-b6%vl?4F2xqG
zOU>yxPD6oHKXWSj<y;3B&r^tK>v8&2pMdnegiQUtoXt1U0MmWAWu2&>3j$eb^qKNV
z_(`JQZP&mXLT@U%-2rPy!7r|*Y1oAdlarltaUyq+yq^|d{B9_>t@Rd#@_KW9w_6P$
z^Dv8(Hi8pDJK{r0Iqq*va$cL=isZh0=1)wIoQ^vYPs$<T2#x2Kj^?$few0Pe4I~zZ
zeAYbg0c0)2OtIx}d)C`Mw&~<64nQ!Uk8$^SW6e!?j1HfU4$&%i_`y~2R>(rBz$+DY
z`y}1}`M%-da686<lVV-dk8h2*Tn8V7;-njKI(p4zUJy$ofY$z#INdRf(>`}zw_w>8
z!BcqxVTim*F)-}$segV$ON*!Zl~dhX@Rz^K2Xu<c1P8u4bp<yQO?OQj^dKZcE}xh_
z<z&gNJz{ZTTu3nGIcR;qG9;?^M0kG|PuThGH1+;j!xXDN6I_*@xL=@r$xRBuVh{MN
zIUGEgxYJ(DFHKoLGF3_xPSW_^TT*1w(&gCNFdnv^AMnNFK6+ia>rh<1-vjImult%O
z!-WXvkA_agVuhluW};J;#r>)?^uHS;G?a?j;(z?Y^FTwOA?tzLFvQDf&X8}9s7Wh<
znEfd_vPyF_V`?>kR`w_h@+%59oKa;NPVGUo52QjisO-|$cYE(VNmm#+`#T5a;gh|Z
z8A0^l3UwQMn0J3x<h`4-5?ApmemDp`8K)X6T0efPN*-~cf<tL>XWL7tY~Ox<iRkdJ
zU|072zio5s?pAI0%Yx0uJh1f5i7VKWaFIaB;45=yji!1nH9<de2OLj_y{&41?nyPO
zUrZT8xW#w*TQ5)($;JeSp2Pgrams&!r<Pe}#(LDg-blL{ESlmQ?a5Th4_;WRJR+4E
zw6tQreDz+4bser4GB#?<roQ`hsw<hwcyHa9dkP0IO=6)WWkTxg{$NTm-b*c?j2_ul
zyuRy=77P?tF`%S2aa=XEJa>Au=_hGvp@_%SZKA)ec-h-dfwIhS3jGBLL6e6Os;1LR
zRDG&3TF`HV*n{&*H!oTSsLq!U5xV5!Yr6I_!*VhmwC3a2BOYfWH13AtVY|n5jv49e
zcb0xCCZnt0i$>-S$k9J@-c!8wG#siu(Lg<MtkAtqhD8bV`jR^%b&>y_r1nfy+}!<h
zAF+SdUhcuD`9zF%pRIHymB_I~)P%%~M=eQ#Ic#<Zr+NPzGTI`9;4khM^2h2PqMd?5
zGH>W9g-ucwp=&Hs1=Vs4i_q;dQL$8~Uq2BVA4o4uY!6}S`xH(Qec+{mJD~qgg@6W8
zipi@Z!ZR+Kr_)u&G);pG$tg$8#KPrsl&N3(m($NAU&9ogH9rVfW<4Mw>^7$&96g<9
zHQzekG9T5SS7DVm7EFY%CjChhfRyap4+d;+^0ng^B)~xKFG^7d2oOo|R8uY&S|X0@
znAGMb^rFQwGPTzsFQ8ZK4S@WO(8`6T+$Yt9{jGMd?jrTeb|_!Un`n9xDZu-fW+_aJ
z4Uyy_$)`Ot!~doWUHW`(?F!iYvc5+g-(W9X<-tX*h%6(f;+A(OQ@w{WYSiq&pjKnN
z)tSH~5g)03sKk)U+&GyP*?86fusX1ttpH1ng8ruC6UOddM~t>0wvZh}1cW%&7{tT$
zze(TwkA~V|_~nL{6YE#^RUC__Mx26zo*w(EfK2Q@R6xo`VkJKs^Eax`&*O*bw~*ap
zyaqA_p(~(POY{H5+NIgewtB{|(%ML_wR8o);^XGTQ|{*J>74v>{_iyU;U*NTN}A%`
z`8ltg(&furYlb!j%1ra!KPSiG<VRTwPDN9f5*7>mJ>f4c!bkAtjb_qmQ+aVB(QohO
zRo@%)1krVtMPgkT6&3T*u`XO8pE&-!!u((3qVnraj|gN5aDxvqtrPs*MCZcO3i^Qt
zI7$&BFr)50exhv11)82?u`ab0FgUSw;dpbnAtmz4k^&Nx`xMQ$5(JW}ry%)ry+DV>
zS)TWjtXz7V6iK5$ghFuPiT>;;fAp)oy%%7grs4UwqU5+Ms96%`wU=YU5W-UGw(6iq
z2GhB=Zw49;Yu<#7=soc@tZvYFIVNfkRPsCT&;76cYOONM<!9yYT8XS_j|<f)GAw6X
z_w&Wq9xu5;px-$u*_Z^YS22HQpD*L|Z1fb)`d&qCQ^smb{5_5>wv!v*e#(X?l7eB-
z&pWvVcaO;IKDg7C8bZ-+Hm`g>n_WC6%BL=CZlc``M{0T;%eYQ4t}V%m20okR=HET)
z@)@WU_}tJOqiH7w2K%l<a?3NQ^6bZPnFJ<Mk`|jLP2*o$M^nx2160!F+h^quABnz;
zAF6)v=cSvmebPJaPi4k%(nh|zGG@U(va!x`)nhbzOU0MkhuA%7v6ZH!EaE%H>pe0P
z^FhhCX$ufUPCq4?C1A8ZSrVz=$~!VZ>;=kb8eaI;S1TKb|E9j*muthJe2||9pYYI$
zR@lkEo?K76^_v{llrL+?Swi1koJYJqG_-g!v?$ITb=q4#Rk--)f<yZAd%OCYe=RDW
z4aV9=2rZm-rEPrLKA|1kuMv{%I=`DA(f6L;GQJ=_TAoYWBDl;}XZ0E+YfGjvp>ABD
zh4Ibu7+f~5HEzy@7xo<qj_3c_D9C_vmh4{K98*=04-QLt1~2F@dBZe-l2GMsk#;A`
zYHOcLf#^)Gn+{G3Q4YowOIK^&zQ|LTx89&c{IWvimdkFT8nJ{0X1}p;P(C>P^f$=}
z+D3gYZ3W>%>m=U)p#UNOPPd&2cD&<J9<&QiV~vk5R%jVK^J1%HQ}`fxWs9c=2}L>;
zxb{vXTzpCjcJAOEA_~=RX^_BM+_BYW*T{zzM(3TosvFOmf6Kp0IerP4`MuBgFdrkZ
zf9X~m0O$toCckMn8klZDxWKr2%FHNk1VLQE)$!{Hz9{*a@TaZjC7kKsC1dIUx*6AQ
zJFZc8p~!CewW(VvE@yaTPFt-6n+dZ@TM582m7=-#9JoDOH#zYPe{)-Lza89t+w#Zd
zvQ3k$)Q)mPF)g)_+v$Gqgq~*RwGeBn{vhp!IPgkixW8WY)H`S{&~om!keO$Sum=oY
zTatGW#*O^aVU<^!#et91z~$IYa;_C@J7+V)`<1b_lh`8FHOAgc=Az}lf)k%5xTMrv
zr6uV%eKaU~wvi7pU)MeB7<DU@<PM)Ua&x<*j67UgrpKP|!tXx2R%YzH<LQn0XK>HK
z2D;27Dik%)-q@hK-!I|N(cl`lAF^EIv0C-t$d1qtFnKIkcMW<4b%Lzf3Y+~~qB7`<
zj);HTQS0Oex%zA170>?kRVA_m_*O?rZRpS3v{+O+cifN7Eb&>$Z==vGKh1V)C`qGu
z_u8y<#N3Wp&$V^@T??GnE&RN^IyXM)r0h(gS3;b2pt0O!eNIt4{;3H~V5Ln7vs>8{
ziqqZL4Nwlvj4CtEv0>;Fw~D>LB_+-ecI)tiR%a!^GI3BawvNQGz4#b|_d<K40`zom
zmZ%w0mYHcNzK(Ivg#;79zJA3Qs(emYQh|-Y&A>f&`e||2k;K}WnvU!Dx=0#ue(=U#
zK&pYNNf5RQZOveUm+;dQ*FIA0&#`?@z*bBhUgr(n9_FpoHPB2pI8iMpW|sF*D{+75
z-k;nba~m^}=b7P$<BGu%3I<`>FAF1)S!oDKtNG-`%h{XQi6=SMH5GZ%8j?ugqt~!K
zw<hNaHlewKU9pKh0n@^4X=DQ<4~UnDj4@h3>vA_m(*=EI<IgUo)z0l9R@mb|@QOas
zWU>ssFVW0EZ;o=u#R5gBB$CUL+->U32;2PM2O(drij20XBy|hH+=bu!0*KIKBj%c+
z^{)B`3$NB2yp-IHf02C#Fw!(;S&rR%2P<?W3i)a{Hv71$$mqNwIwWJTc5XCVCY(ZM
zZEUT%{m1IMAyv+ZxJdeiWsFSau%`*Ji4gu)?i`XAkA6AeCLD>q(!<`Q=u&+_V4eCe
z?!d0m@n<F6bnzf#{rI&DDtbzb{#Q?q`iI`Fv^=Q#{GVsrKi@5H!=Yk{`KU+uXc?t@
zxGi_IMbsNpVL63R9MI#c?&2tT**S1&xk6UXV{?VrG2Vb8uwy$l2i~-P)jArRJvd+p
zAMPX_jhyzm3a}Qc-9M){f2vD<`B3X9uKLW{DLodF&IsV$kXKT%@Qtp6|3s@S0+S=%
zV+#X9n<D=<XzlauBx&tS1|?-doY!<IKSZPJ`vt2XRD)VP6|a+O3xDEZOZR$X3e5-S
zuOL@^Te?HwRm63Ch16HfZ|^W=1@ax6$xAQ(4$2J*D69D!1&Ss_Wp=KanXxf%3)jB=
zyl{(zRa6B4dz*qTVGFnQ#lf#G^~(Orm6*fvz@t#mixM85R=piy5ZZ)?<t2uZj~#Q1
z%87M&!_4Xmtg&aKmcnz`(+k~CS_9jg?1HcPF4&*jQGA1B5O}@9G995LTJuL|d}-c#
zRi6~5UoNF~Ng3*RH>dhMu%QZ`ERBCD+uU~%h<WLJg$(5L-k}}ce*Ymz9%AWcG8~o)
zrgMWKP5N71i-Vz&u9fBxjTT}~QT7=y$EdDt>>+E^Qd;Cz=IlGV(IwUrOz(+1Gkd7O
z$HME|^+mAGBc4k(2jEj5$g30r-BUoK@Nn!*Td)5USoe+IZ-x9)#yd)sD}2Z?2{4@)
zb|)xsK&pqOpB;+H#gbf^Pto29M<2Y>dU5pAF4p{+j=oBZ$2EXA*xI~AM@g20H7o_x
z{2-Kc;SRpcxLXzU)a53ZoX%ndB^i8=>Sf&{i6CYkGSkvLj0<@C-!VKm#iX8dws__S
zKp`T~rIAfaogJ!tV(~rs5)ctD#A};YXgPNI`<5=nWQjnIf<=1Pzn2y$C8yUkFKhwM
z@%Ah?L`DM^@d<2evu->Oo=SVaiR<1GjYwe^G2)XY`l$Q%4H`|PpFA($N_8=6uOr0s
zj+)C5x<cICx<i}#5D8LZ3LNFG7uU}%Q5<kbowYRV6Bs|^frDu{l2XM2Lj-Yh_!|?f
z+a6@mRKb9j3p<Zh$+a4#UQQYhPF@-a9mWMpS)m;R6VEWV!i;mbS?{`eur*GS8_tX$
jEfLfZC2@~9k9g`Sv9u1yERTOL1mL!wsczLx=g0p8M%V6I

diff --git a/static/glyphicons-halflings.png b/static/glyphicons-halflings.png
deleted file mode 100644
index 92d4445dfd0af414835467132cf50c7c38a303af..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4352
zcmc(jSvb@I*TDbCl4#mw&6;FnOR{7wTf|sqB70@Y*4Sf=t&%DGGDxy7<4<NH`!>in
zwn`&QQOr<`27|~lU*GNfe)r$+;%v`3=Q$VW;ymZMrG+ssw-7e~0K7L%46Ffwh5XNs
z<6`?KHS^P-{ZmgZZ@~?jOs2~JH%~nY@PG5j1zTI#0Amn(L8qe2oETm=+B^jogFL!D
zS!ISRHW3ybWQ6o&?2=byQi)JhfBSH9PzL~<0B#!S!^50cUq25lRnLyYPq06zWw>~J
z`$KJG?wJet%MCZ1y81U)c?UzG;{mBi?no2aAHvt8L__Xy66K$DAupSD_4^VSeG;vA
zGhrY7dmCA}Zg<=d*dvUYvYMo40k!iu>o|-n)q^ld6Q(6yBtUWr1GY<4vK2?uoeS|r
zT(a}}&NC3;#Lv8{0Y$f=#j|95fZYUrx?foCUQ)KvUf$-LSb+6D%%)z#|1KO+ZTgw~
zNbE_n|4p~xYoc$edOQF-XOS;%<r!#dmHF{5#RTzN2!T(FFMc;x+SmC=Km>evzdNi3
zk@(r9h#R5FpacG)j3VDRRz>g49u-o5A=@X`M=nQQ@W&MqFu3+}8)vIJ<N(sT_Zk8X
zMLcU+@C0(GanqcI+g=S0kMDE|mIlGu&c=CozOm!OJr0PRi4D`Obo#+t^;Xncwa7ai
zdh<9v@cF*U;YbI{iHB@gJAiGAx>yezf?(vDF#3iq72Yg1rU0$uCw``L1fzH6tU=MT
zJ)FP#7~BMLoosB<>)Y`BnyxN?%PW`qwa_nrmk;P<^+|3lA$<ii8%uY~81lp=vs^qj
zbHc@jPjA%v+=Qq|+H=tEesx$(-?7Mn9R7&2+ZkE|b0%rF{pOe&2PywxrLyZ}ob?{?
zS!X14D0!!&R7j4z&a_ZX7E96o-d9#)-rDWl);JNdUKsSn^M1g(Sofin)_^D2A6fsk
zK%N&KDHIEtjz^2e+RnzNElc?mvW&i)!L?OOITL^U!tz7HAYtpl+few+Se~$Nk0>cC
z!KnRdI-*8rENgl-h*t3^hviocbR?_BCX&(%?-)#H*`RRAUES@w^(0ey@bvFIq^EE0
zYIYPpa4Xz>{9(cUIq~=IuByDHtJskc@OXkoyhOvqjT$BRxhihe#hq<$(TaV?g(bYx
zzk*$b_y4xdrKd-u!#@W)7x%!%FE62JOZu)fTpnAUKW94KXQKo9lR9BoI`nN#BV<pu
zN$Y&tINUw4JJ60wNhX=$oO%xnS0~8-36@e}lO69__j)7adZ<L2U#u5vwGeo2shUYF
zH*Rf1efjD`jdvTV8a6X+&!xQkbfP<z!gz1jlz##Puuh>NL^WLc-2PBnDb`!FkQ6Yw
zt8#VMCqN`vOx>8A-pqa3!sg7$vF4w|C29%3h5O_{d+D-|gED!U;S&A}5QU_Uz%?vp
zmMBIPvj7qQQG74PJJYIU8KAgcJcJvNO0O6=%8w|@chXvpUX6O34cERMj)m?X)jwit
zWYksusgx8zcrOv1Kd4Cm%yUoW#?wfM-ee=?*pXt7dU<wL(ECgNtn7KAcQSgjF$wec
z&wNDxS$nw$r4-^(dj0lt)f7DU5?+TTQ7UFh7R=*xI5;Wr=aA7JB?^0SzgQ^V;4r`?
zBrN-yp=!hwMFq0PE?Y!UWLSr9S4c>vyZrhI*Zx3!VQzm2&D<yRh#LVfjXeOPcj~wR
z5UG;7Ix04MSLbA=`nZloXfY{`*@7=#K0}`VWppv~RO%H}$!V*DNHvZFBHUqfI?CD0
zbx!B4^9`#pqXl&iB{Pv^*lNn33}KeCeaE4o?M=ZBEL9o|=KG==a{vbsI4@1LO3@|!
z=#F_^eo|k6WLCD`I?D^lB}ZeRa3j2$+MNG{fZN~d@a`$my6AxPBfp8Irx1gDj_8y_
zB{|_Ko-%@Zv$H~Px&z9e&#zqq`(8HmN?{uv#cDixDOJ%G_;k$j?o_(Za8|9R0~pd~
zhP6EvoFeWkI{=X$R-d0BaUhyb8w0in4s%stxoODXOl;El?W^#yR`?d86Ax#>k2i(z
zv;J?=_W|Z`2Nb*9*m`XJ^1ixr>GY^eNXXM8UzHKbJ%`E&g=n<QM>C-&t%U{b2>k}4
zM^eC8z9@VJ)NO6~zgW94x7psn_*GsP&AXPV>|c7+3V*`GDl?NuNHOr8_5jSBY+FrJ
zxxFy&omakmacj-wPLUexLeI~s2^i^7j<QS1^o69wqChX$OY6u{tW}exT*h=kf_UuP
z+XaMs<6dAuy-kT^H%eXIYHYk4Q!FTjJ*&L%*Q})SYV#u;NgCV`gwN=QJ~!7t_q2+B
zpbd_ZMR9D%dyk)}nec)ZXV~q^?S+kxZJj&X5@|w?zO<x_02M#Q3a*5JM8Y&n;d~#^
zX?>diy$lDh;U-ze^bf8Wq&_j48xx9sRj~I0?AI|l`&NRKa0xj_M7{QQP8x>W$llZ#
z^2}mA)Bep^+iA@Qw-LK1wT3nbnW#j??18HOX9M~EwO_4MW54*U(nB|yBja(g7FnMC
zblZNR)Y{`EcNWNZ9&#=!$@W#;-?`_@7{fb;%BTG<Be%)pb!CB`M;1FsO>aNt!jg%h
zP{`+<{G!`T5|=OLq>Z*{Z2O&8zMn16ACVB$Qm``DYk?tjJdb2uC7aci<-`J?E%OU+
zGrN5UtA#%|w#4Z;NP?k$>n!<|SrjF%qnK<QD=|fvQ-`MgFRin=cJ~1?W$Nv-%7>36
z-X#tb9{hRfZswTsPVZBN8H~75sHKLYIz~6u+pKzy#crwlQTpM#$E~+Abk)TD#sz#v
zXX8Go`ZaF>B8Zu%M9U<U?k5{O<y&QE7KlDa9?QUr-S}#I$LYUm81UoWaH<><;>RXE
zbfFb@39Y9#&~E%DMKl*GIPjFwcNZ7nuMbVEpA0WbvBjM9QA!sp{YiDoe131&NawG0
z)w7{^`zTTBX*b%&r|n~U@dMgnxo!))g;D+Qg=`Xw5@VHk^{hiH?Dbc#u;gsXHzn0i
z2)8o6*&Kl>6tpGG-xYv<M}QNBKQ@Z8AUtKe=QqbSl$Amw(w@PJ2Fl*B4kD#B1X|@h
zND6v-C(>B-r`9coW<<#c<0|E=wQpY(XerrkkfVOt!t*N?wvbI|9F@&~JQ7q2jXe2H
zCW^MvkWX8I-=%fo@BdI{A^py@pAB`s<yjfB3(la;jxJW|8b9;qtmahyAaiMpzZU^P
zxD>hd&A{*amKE*X!a7A2Yu?Z%f;af$36@t#hgGI$UAqZQr>(vfUM3&C0L=d07kpTV
z65hXXqa6SYLUvQ%beIm#w8HN~d3!4?$?iB2Owr|ut8l>>rMSqaZB}JGncrpN>H)eX
z?`{XC$$(nou>9J>y&RJ_GCHrPS%%Jr+GeZ-p;^lV`1YLmyxKN-u#7+}dnx}N%zgXH
z$CV1rQyi4eN)t(4&9Ix9{_jMeW*4;LYis@>9EQ2Es^gfy-VKyn0lc8i{7q3yuQV}F
zD6Fom;2?qz@ukzYpge~g8?BAWbC}{;E82F=WrGc<q3x&8B^qmty_aky%YQ{CKTGq<
zYP!kE(69SylMU^oAELgld(|`QSIDWIzU`!z4rh5Kn4EmCqu{yU{SIwx=mqDK8w<~1
zUPFy^`6*;La<HSR(O`c-+NrbAEnz=wxz=;F=D$%Gr5~UQ*wG%^^eW0ENv91u_qR-a
z$S)u&@oi_Fi#yBZUxTms_h&AvlAOS|`l_14f97W-V>0;?er)DQ&9VG84bSn{>9B(k
zwM%!e%*jQ~?@0DuS;yYC#^~O_E+}d7VN;GP%ockmCFlj4DNZ%yl_X-Hn$v_=+Er1z
z)xF^ugN@xFweaki3bVXB3?uwjsn55R<b|OgeId~Hv@}>D1&YMi6B+jBAEU6|0Y1ne
zLxbyOnkM9BHX2f}bHa<7WG>P_pz=aP(B)D(uo1i&yvId9DaA3GTsK?WdG%g5Q5z-%
zUfT;wH`Xu@LDvM>F<4<`LiFUdk7UO)oS&1>Rnv!81;V#S1gZ^;byAIw5fmjY3m)nw
z?+@SmlmBCWV>bFM8|-jGB{WLeI3o9DaWo<)11@8`kh*v=cN0DNB+st4sz6R#2I0qi
z4c&8ZcAexDoiEyzoZJ((D9)8bG%^Z+MCs@_Q)++#Uvn&7#CI<7^ioFM{2qLTEAfMX
z#1kD>oACS6EsTK8F}{R&pahvhyt|}$lX5-EzVP=!*jL*U(=7^7%UUF#`g>m(9)4uh
zN+-O*&B&PgYQ520)x+!;$#)PXM`Kgq-o1CQLPsDGuSVi?k7|gIEtmv^WewHMkLAio
zl1Us*ZM8T5*j_cED4OCIiNDZ{(dj&{3{g&T+~4Y*L((GimlI~v8Q&*2;zNurHxdEX
zDgWY5T-u#~Rw6AH53<&eUOA_3sJa+<`S@61`0Z+&gPPC(dA9xY-3vCHs+QQ8y<*H|
zq`~2~B6ACGIIhlq0<JP>$V=$vE_&HDcwxCpLD6$_1>ZT*h{SQByL1NMw0+fOj?Wz&
zFvJdbQkbJBeJ=wX#hUle7%rUXR$4yPWhM|#t(`DrC+d#^K8*!sRn%{Eee5S%bqSan
z?Gaxb6y6;Dw^4Ura3@7~UnV3ahsAZxfc!%uwqZbo@PGj7@>ji1sVn}8fiB(aiz~Jo
zTDXK*@oVh~gVo^Iu~o8PQNMj6)RalL?o3^H@pnjZNLWoX&@@;gDJHvX&C-&SZCkAF
z?Pux@B3eZQ037cWb&FZMuP+XLz1yG`s8)?SoCs!ygWlxG$PB`Eka2i37Fv)TK{|58
zJti;S=?xo)8?eTei(HD#<H{`dIBo}QZ*qye7Ch&8W=G`>f`Jq8j>vX~5NRzRU9sf_
z>oxtdr~$>ax+OJ;^X)vsSztp0JYJsoQlX{)JP`NN^%4mv6u3oW-hBTdM2W@5-Fze>
z9n9nd!<vn@x)+DSqur{lShQR5c_q20z&z9X_VI@tF-sZITiJ8(=%yDq%20jBZq4o?
zgCC2nZ#R@cyO{hJ?i_$meOX?m;pkq%(#414r`r1hpFn%A^?fTAk~P~=C0`Omj7x)=
z_=sB}!Gp5B>;qg7R6d&M#&&}CPAvA|mF^4XPltG`XZl9!t)5o^flxcEGJRDAZjOjF
zQ0Iea%DG$E3bP&!(93|2RCY3l5t3s3J*JOik0=hGeaJ@3@H8tD7<k9<<dKwp&eQ6Z
z9|U0$hb)b5lItCim6MC_Nf&^qL{S0zjAEPdi{G~l$mUBpQVcZOtKq$za5*WnwuQO{
zxF$NXUlSh-TEr%CuFbjgKX@wV^CqEZM<ObXOWagY0q?8j*FR)BnR)!IQXA#2X-7RS
zQDDqU9@ib_?%osL+z(HZl~m@gbUVL(W{K>CVRqHg&`+R3j0a8@kqB}PI}{$m!yRab
zvul5lL(>3*TF>n~)*#hsmwUTtKRAA2Fnk0PENdI!9GrZLu@zyKzs+&m-IKFviqv>&
kg1Lm#gqI~e;$iYPkmG5c&N-g{UI@TVLkokN>#mRg2V?7pi2wiq

diff --git a/static/js/bootstrap-dropdown.js b/static/js/bootstrap-dropdown.js
new file mode 100644
index 0000000000..2bf8858749
--- /dev/null
+++ b/static/js/bootstrap-dropdown.js
@@ -0,0 +1,92 @@
+/* ============================================================
+ * bootstrap-dropdown.js v2.0.2
+ * http://twitter.github.com/bootstrap/javascript.html#dropdowns
+ * ============================================================
+ * Copyright 2012 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============================================================ */
+
+
+!function( $ ){
+
+  "use strict"
+
+ /* DROPDOWN CLASS DEFINITION
+  * ========================= */
+
+  var toggle = '[data-toggle="dropdown"]'
+    , Dropdown = function ( element ) {
+        var $el = $(element).on('click.dropdown.data-api', this.toggle)
+        $('html').on('click.dropdown.data-api', function () {
+          $el.parent().removeClass('open')
+        })
+      }
+
+  Dropdown.prototype = {
+
+    constructor: Dropdown
+
+  , toggle: function ( e ) {
+      var $this = $(this)
+        , selector = $this.attr('data-target')
+        , $parent
+        , isActive
+
+      if (!selector) {
+        selector = $this.attr('href')
+        selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7
+      }
+
+      $parent = $(selector)
+      $parent.length || ($parent = $this.parent())
+
+      isActive = $parent.hasClass('open')
+
+      clearMenus()
+      !isActive && $parent.toggleClass('open')
+
+      return false
+    }
+
+  }
+
+  function clearMenus() {
+    $(toggle).parent().removeClass('open')
+  }
+
+
+  /* DROPDOWN PLUGIN DEFINITION
+   * ========================== */
+
+  $.fn.dropdown = function ( option ) {
+    return this.each(function () {
+      var $this = $(this)
+        , data = $this.data('dropdown')
+      if (!data) $this.data('dropdown', (data = new Dropdown(this)))
+      if (typeof option == 'string') data[option].call($this)
+    })
+  }
+
+  $.fn.dropdown.Constructor = Dropdown
+
+
+  /* APPLY TO STANDARD DROPDOWN ELEMENTS
+   * =================================== */
+
+  $(function () {
+    $('html').on('click.dropdown.data-api', clearMenus)
+    $('body').on('click.dropdown.data-api', toggle, Dropdown.prototype.toggle)
+  })
+
+}( window.jQuery );
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
new file mode 100644
index 0000000000..c03c459a64
--- /dev/null
+++ b/templates/bootstrap.hamlet
@@ -0,0 +1,52 @@
+$doctype 5
+<html>
+  <head>
+    <title>#{baseTitle webapp} #{pageTitle page}
+    <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <style type="text/css">
+      body {
+        padding-top: 60px;
+        padding-bottom: 40px;
+      }
+      .sidebar-nav {
+        padding: 9px 0;
+      }
+    ^{pageHead page}
+  <body>
+
+    <div class="navbar navbar-fixed-top">
+      <div class="navbar-inner">
+        <div class="container">
+          <a class="brand" href="#">
+            git-annex
+          <ul class="nav">
+            <li class="active">
+              <a href="#">Dashboard</a>
+            <li>
+              <a href="@{ConfigR}">Config</a>
+          <ul class="nav pull-right">
+            <li class="dropdown" id="menu1">
+              <a class="dropdown-toggle" data-toggle="dropdown" href="#menu1">
+                Current Repository: #{baseTitle webapp}
+                <b class="caret"></b>
+              <ul class="dropdown-menu">
+                 <li><a href="#">#{baseTitle webapp}</a></li>
+                 <li class="divider"></li>
+                 <li><a href="#">Add new repository</a></li>
+
+    <div class="container-fluid">
+      <div class="row-fluid">
+        <div class="span3">
+          <div class="sidebar-nav">
+            <div class="alert alert-info">
+              <b>This is just a demo.</b> If this were not just a demo,
+              I'd not be filling this sidebar with silly alerts.
+            <div class="alert alert-success">
+              <b>Well done!</b>
+              You successfully read this important alert message.
+            <div class="alert alert-error">
+              <b>Whoops!</b>
+              Unable to connect to blah blah..
+        <div class="span9">
+          ^{pageBody page}
diff --git a/templates/default-layout.hamlet b/templates/default-layout.hamlet
index bd16969f93..3701e3c42f 100644
--- a/templates/default-layout.hamlet
+++ b/templates/default-layout.hamlet
@@ -1,10 +1,3 @@
-$doctype 5
-<html>
-  <head>
-    <title>#{baseTitle webapp} #{pageTitle page}
-    <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
-    ^{pageHead page}
-  <body>
-      $maybe msg <- mmsg
-        <div #message>#{msg}
-      ^{pageBody page}
+$maybe msg <- mmsg
+  <div #message>#{msg}
+^{widget}
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 351f2f8c60..eff8d3f444 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,5 +1,5 @@
 
-// Uses long-polling to update a div with id=#{poll}
+// Uses long-polling to update a div with id=#{updating}
 // The gethtml route should return a new div, with the same id.
 //
 // Maximum update frequency is controlled by #{startdelay}
@@ -16,7 +16,7 @@ $.LongPoll = (function() {
 				'url': '@{gethtml}',
 				'dataType': 'html',
 				'success': function(data, status, jqxhr) {
-					$('##{poll}').replaceWith(data);
+					$('##{updating}').replaceWith(data);
 					setTimeout($.LongPoll.send, #{delay});
 					numerrs=0;
 				},
diff --git a/templates/status.hamlet b/templates/status.hamlet
index 1f975b35f4..1da189d1f3 100644
--- a/templates/status.hamlet
+++ b/templates/status.hamlet
@@ -1,2 +1,26 @@
-<div id="#{poll}">
-  polled at #{time}
+<span id="#{updating}">
+  <div class="hero-unit">
+    <div class="row-fluid">
+      <h3>
+        foo ←
+        <small>usb drive</small>
+        <small class="pull-right">40% of 10 mb</small>
+    <div class="progress progress-striped">
+      <div class="bar" style="width: 40%;">
+    <div class="row-fluid">
+      <h3>
+        some_filenames_are_long_and_ugly_like_this_one.mp3 →
+        <small>Amazon S3</small>
+        <small class="pull-right">10% of 50 mb</small>
+    <div class="progress progress-striped">
+      <div class="bar" style="width: 10%;">
+    <div class="row-fluid">
+      <h3>
+        bigfile ←
+        <small>usb drive</small>
+        <small class="pull-right">0% of 512 mb</small>
+    <div class="progress progress-striped">
+      <div class="bar" style="width: 0%;">
+  <footer>
+    <span>
+      polled at #{time}

From 4d15a9d1baaf09ede8cf07b8ddc716a7f9431433 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 04:50:27 -0400
Subject: [PATCH 4294/8313] add

---
 .../blog/day_45__long_polling/full.png          | Bin 0 -> 55185 bytes
 .../blog/day_45__long_polling/phone.png         | Bin 0 -> 41602 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 doc/design/assistant/blog/day_45__long_polling/full.png
 create mode 100644 doc/design/assistant/blog/day_45__long_polling/phone.png

diff --git a/doc/design/assistant/blog/day_45__long_polling/full.png b/doc/design/assistant/blog/day_45__long_polling/full.png
new file mode 100644
index 0000000000000000000000000000000000000000..3963ae1dcb26d2590c08ab036d9ec68fe266bbe0
GIT binary patch
literal 55185
zcmYg%1z1yW-1cZCML{|Qm6Glj>6Y#ikZwjdQc^1Ah@nV#cL~CP0i(McM%U<l)BpQk
z?>8=XYR@^(Ip-JmbN}uq=DnIc0WKvj001CRRFKgG05DOj==zV)Q40@u3{TVo(?&{F
z3IM1|#JjV=LT#gYXv)6>ln+zyqBiI)6*N@=0KeA&K*%Qm;0CoTWCs8MasdE)<^X`m
zcL0FQHM3bm40QnegR;B~0QvXtM_WM>YR_X=1p^P%QsLjfT)%?^syRSWMoQafZhz6+
zSmy$Xb3EOY(MD{LBh(q1o5S2Fzr|cP5owrjU@*%?Ri9=cK;2R8VwVR>*-{!xiNPRC
zXC|a3BgT{xAUhO+BOnp*2!BrF$ehz4iGYcw>|5b%;BZrb#CE{2u*&iYEJq-R{!tG7
zX)7~s4)YXcHlI3vH&@)JuS%~_Ukd5XLZ6<zd-UQDw!sjMp9YDn)QT{p!-R)ic&A%f
zryvm2@1(@=RmqA6^o~WZFqa35@hcbp_(TN5zg9YZ_78?6m{Wy<w)}0s7_obLWV{nx
zHCZy62R8(C$|qmFD%hX%KHs?Z%&W;T{`nzyET4nq#owkyVKKow6#mOGVVzajfd;!*
z#P?typSaF5bX7D4{wJz>ai5NKADY;>4L0{UlGyAxp8Rdr=aiSv1_<k{!8UR9jAEjt
z?LHHOxp@)+ic*+AYcVAZ5A=&>yO}VgDB1zuK0>u{7p<BXeufpgtsh<<@ZK18elf^n
zboI92wMUT_YyL76o||Hj&tW4IJ^y<3LU5)Ho&J%S!wAGHerJ)<@QIH`>u5Zl%e;ma
z7Pi<+bn}pU5>{aqK^*1Wks4Pooa84j8FJ%{bJ{)Yi%JxoX(BDGcnkt6{GSribR?U+
zOz9sYLlgQH%D>msydXcUkgEwY&VR4CqlEtVatfA};(#ZzsLm}FQt_8|?0$Hg;(;^-
zYbd^u4b30v8HxaUPyO(`PXz*c<NZcdS2?0`jkWgIw;;z)ZFh^Lh6ua4+?yP2JHSP3
zf~QI;GOX6*j{#FZ4o2DMX8UJPay|V_f}}M3<RVqdzp?CKd!pX+r!SG!sEcBTS`rXD
zh55^PZwZ&*4AN=fqP3Dd6@)ofvfQP7kP9v<6Adxx;#aRKjIGT|%U~i>z<Qgz-N`44
z->r~0X1dtyxKPDeAj6r$ma7@V{5hWtYbRa~lJrf<njOQ?$JZ}YMYx+gPNm#-Z-Dsk
zRTpL*)k~iEB=GJ);>q9OhQ${lHKRo^m`*pjz|10uGc&lJBNE=(lZ-*8rHfkI?wHDE
z7SSVQWID|m29!Jm8!t<-6u|g&u{?d#eyz)ompyKvijiSR8Hgl`-o*C{-8T1A{jP1Q
zuWel5iK(&SWBT*GXWV`#1%avw`@h>31;?`ia_EoW;Dy2$8Rxb<BxkGLn_80MLV2yK
zOliV<Z#K?e-6l{2Mk@4LD<&8**)!?Ca%;-5SSv$a?6EFi@54oZWN*UrWqx-m*p~MV
z<xErI;^ARn7^O>>n?O%4FEv?9YK6PGG>dY*5rs#3$ex-~m3;G;PeH~S@AY@q*G;BX
zurxf~yqs6JlgzYP^@`J*3I5vXyF+t<a=;^WSWP6@gz?o=ITA;Fz(zaAd!^ivmQbRw
zm%Tdz3hVszFzdHqD1E$M+$j5_r`{NxSU-~c0!`jb2kzFnRBV%~e!sIQpj9t7J>v|S
zk+>%;ySJ5teW#`Q<<-hN`u=LU0MRQayHw+9Y!=$)Y{*ux-XXp$hWyh>5$LBnf4tDf
zU}mqz^}^yP<l$b$?ehNkG_Kup9cRdPPq!_D^!anJu)n~?9o9HRIc=f<mdV1wQI)`w
zqC|@8YvscXqg3Jik~*-_^Xe5fWRlTLiX=P?UHDgd5mm_F;Ic8S8wj9JKSc6V($acl
zGk+GyzAdtRuUQ_U@g+vdn5D7WnKQ#xV0eYb=J?~YVN-(!GCf?YoTqX>bB#+Zi%VOM
z&p;1BrVc^Ah`}m4x-70flB*@c>z3QZfZ(ecWh3Q$@l)XbUfZoYo!QZx_o8~q5F3XO
zUH$&@5rW3_A>4CjuVPy-S$E_>)U(`dnKnN$8%$_W>wW+5iBxs5*t@2)Gxf!b7a$NQ
zZG2o6Iag2MGTOB)^7`+~bR0ZJD;=djCiVWNkayaZlXGLup4NYNe`m|^FC-~Q>#L$Y
zl?7YsxGP3^l{qauIDAI$BXL)-=5p`R^SIo<lAo~Mq$UbUQKS&GIU3=DJxBY}Zxtua
zN$7J1()g{USgA=&Mm1Znl@=<Y1W3a17WUh_x*8Cw5aW^Q1$34q65P8VmO@OiLU`q$
z-HRelwR9G2j&CfbvtGtwP<+FV*%>h{(l&0+#ePCTOiWE}#Y@@E#RKB2HgRZ@*$~!!
ztrupI>n>{{^|K?}O#GBohP%@Gt?7e-AN7CzZ_}asYkU+0(zl*b%ya%n6(r`lJW?yK
zcw%f+ri-Lp6z=Xk$v95MTu6N6iabwT&{!~{m70uUlMl3o^oY+7nX&IEH+~iyr|!fF
zZhgkQ6#_kf7m!)L<)8|E{P-4!aNi-}*^XjOw@Y0*9gOoUDhMhfI1UoC+8Fdi6kk0p
zQYmL)o3yVR9tQf57lM@X+&w%-Yf80WyX+M|`xiUD7N0N4Xlw{4@KVYV?lMK^m~WrA
zX<{-)$EgfJD_67GEAA^_LiNut+J`FcPwwN8xBGDR;OD~1Lc#+CA}lMcWlK_4-)}N=
zIft9|tG)z|_B5;|*IZ)*+}TW;bK5-?X()(Ci!33$HZaq+bZgH1-8^&Y+a7nh@awQ|
zDnDCOlZ-k_u{0c4k04b*pD)b+Qbm=OIDku}#A>xnT2?CL{#sOwjokMIFQ?$DkLHee
zR*ETmE_>iQB>wQMGdf`Od%^VSm*UJ!3Sw#_abhOK^UWqExZwy8Y4no39JjYrvK>Sh
zyo?EWrKdG+uw3r6#|!#2F<<8(obQYL{aY$Dl%0{0VK^^;YnyB)&e8A1+m@-HxD-Ca
zwzL2e5)yL!i~T?Tetw_D*hKi6;{LkaCv80-pwmOlfts33tNV@)AV!p;)LZRn;cRN>
z{IE_WZfN{v{g*33qY>ugs^s%_r>>R_9}|$5X9#M&xqh%1m+*7H%)W1yak@M3hTySM
z-CL-1B>rkq72D#yElZ=r{!-~pm-Wzb(c0FQ)9h(S^ZertIW@JXx@T=mOGew((-@i8
zNxd?%r==PjY+IiFBx=*{?t6NG$K5{uzK^NrC)lO_#y)rXW;db`=FeiVD(rjdUr~=y
zSj7ugxfd^IV-r!Lw7a?XV)C^5sfmNlay>#4Jt3T)y)9E`ZS-B26l=K33$n7U+d&E{
z37dVtGe|cM0+dF^e%IA{x5N01F3>9HdVPX?Sv@hp2Bg*nZVN9MtO5}>`6eq_*V#N=
z-P}w(S@tA0A3T1*Nyyd_YErOrkF?lO(lHJC*^s73z=-27(>Xm|ppeCp*MR=d5+@e_
z<q+4>-}8&J%P@Ho%+5o=+s#rg@qY1I>-X@^m6T;<E|fE3_)!+O#khcF5yxv*!`EK{
zuI}HagYaAfN&mD?H+XAn3DOru><WYcf+h8sgdvnNKR+aXVM!*Nv@ffdfL$PtkDp*A
zd_?<~IVYrhNC6roz6qQ7H&sLgQMP{fX^TrKnC7`Fz8b~J1nu5e+3=}Bv(t+U_%xQ3
ztA_^_pOvd^OTD@4y<dBFN5!AfZxgcv1@nSHIB^sgK4TIT8-kdrW{S}h;(q!V6)Kv-
zp7ro<4@Y2P`TF@4YXg0%3)Bk8;QoTb|3C^a<d5VzJ|J|PpqWQfujiYu3-bm~LdouH
z=*kyDr95q$m1HGQ-w&07TX;o$=!@TPpmM;y{Uf*w7rAwu%+D{}3<Cq_O~Q7nCaPn9
zgc8`^u!2O9R0M1qB27xUq!rFw;$ErS`Wh=V>hWI{ta0;<rPKpo{ey_S6s)h2<nhg?
z<>I2n$)D5xk%01#;ELyC1x9+~Vdcz7C5<30z0V6?FQ3L_WGLTGy@*mX{W3t}F4wQ{
z67ceA(L0@NGW37m@a5(!U`79zhThbR(#`M0(wIN<=Yt;?(M4L2xL<!?#R$I_k8kEl
z;O7Iz{7cE{EFg`FNfJ|{)3bn;r3WeYd*6hz)l0K?dr=nTF)~k=L^j$(tAZWpbhMMV
z_jc$12N2sOd{1ygT<k4;Jnbf`cNqV?Eezh?U1i;P&3#~K12Z$Mahs1hqfj-W{)b7n
zqMJNdRZxNdUmu^i$1b!Mcs~7m=*c^0Q9c0u-^cJGm*Oiw5G`7ok8&txT|T{;<lkQJ
z8vkYlDpWe`X)bv30UCKTg@QJ~hi(6N{23(ioQf;ANZD<C6tJT9bTPgc@UN+$GB3zg
z3ISN~%wZ-WA;rxZnZWuN(_ZA7VuWbMM{fSn>9}{c!Yb4CPEvaPSwYNG!RE;&je_I3
z;4@obGaL)cplQ0?%DR|$tisBAi}d-<Icn$t6j|)yyU06-ZW=mu^R5YTuro@Rzt6Ka
zUKP50T}S%g6$p#Z06!XYWqx`pN`Owcs9cT~>Xccv7IMEGsH?OhA8^LU#$Kxh>=%%-
z`TlgnMiU8tuve>>VU68$SI?e#2*+L!bv%x4cMG43lqc6e{E%*#<<{hK*Vz|6tXIuL
z^*wFLyzx7Dsz(0EKga)kZ<Og`>{XU&a8i2gf6rKc;(#_J=i5^zrUqV{;RHI#pv7iM
zn?awg?nKG&4}X#~G^%OEKOhbl{g(p~PV@B#S>8**NB0c{s#&ZlTm*`|9fj|$zR!RI
zNpT}aZAak1{F+nurmpVpb?^=?HRNCsSrHpsGHT1r!C|LhN*VupeXj0jcq9wSz}VOa
zK|zFt^x=G!hmr03FHupqL$=)i>_9YY)K<*%yhF3hRMR)zkYr^m*V8*SHF@Lr!ZQ-t
zChesn2TpZ$nBn-{vbuwMRvY<b)#A7oL=brIj0YqKkfbCRYM0v#X9gbs#+fcR;q}^Y
z+=Qlyi=3=xXjfz0KBEn?8Z!%aq7-(YNT3mw@IkIOECpHip6~M5i60p;v$D8(dBGQ0
z25k9A$jJJwG-G6n-!oH0qK23S<B!pgHZ~TR>MK}+q(AO&kdlz3JYN=ceA)f>7t2>A
zYf9Cal9Ks8VuZ_lyN=BW)>VV;pZYQQMHCU8nwlC12Zum}kDs5Myga10w^+MaZZP`q
zd)UYq@x2+uDVXnHXvG3!(0^We!Eb*`*5L2)wuH@`A|$*|%Id|o*S`PO(rv+Xruj!*
zXM4l?j3{q=Lod|2!u$2}-%p8XB~si|*M$IK^nKRVGc(_v5uQ#9Q;h+_fd|=e<Si%2
zf}D=-s6B99F1{7k(~}q4FER(>VCB$YH5mqr+Z^6k=r`qfA1#%hd|brfvO4|&lgJjd
z9a&*Fs>i1iaD*h#WCw@^?;_i`uWlApjmjtOFK&jjay&)KOe^$QQ_^{c`ja>QNZ&qO
zdvAkkSYGA5!i~_Zn5K&N18<|bzR~OWW@BYCNkxDlq<Hb@odHPh=leYZzle7Ry7|PI
zK^xJd)`nR(r<=X@h3Dtz5qM+|w?`es#Khu$=Q}f1>)*L885kG{=_FcRSG!O@$f)-H
zFEE5vpSmY}rM&Q)GUrNv*acE2AF842iwj}s2lhysx=f@@rmqy$km)GtMAet+C~f9J
z39?q4P7iOhsYNn3wzjrbS62@Ymx^^th#BQ6<L(A@I{2L(FtM;AK7GRe_syJAyx8(g
z;>IV)*Vh*#31|y=uWg*8-IOV+mxoyX<?o$3>FeXe=Pd4$tkNZR{pZ=4#J%mv52-Eg
z&Vn<E+mE}Nsq66MWaiI$)#bgHalDlAZyb{Qa}^<pFKI;Ir17vDUESW^6my!*UBfP=
z<p~vEK1Csq9)N!NFa|*Ci6*CsC-7{4*}L+$`3SrYH-hKlZnmS|Q7pqx?sb~OfKr{v
z!N~7!lZAzE>IIM9Nz)M|gTW7HFiBBSQ3wRm(%g)VgCj31i-m(jL?;o@8%F^KgC&u7
zyzyxNFh{``t~iYX{ym^v)^OoCrKe$=8pIB25RO)=W(8y@!6=E)fTryTS6%k6+Gveq
zMO#K9h-Ik7gX&jbiKV#DEiRgn$d>75piY{S0^A+TLnYo4ZB{zEj?ByjIY)vNC1H;(
z-lKMevu(+Hl-vIJ@#EC};F6|=^1I`KRtRovY>HA^&GxYP`HD1Y8<JEKWrA6jK#8TL
zcztOH&BtdGwRUvDcdJG&Z8g4UTN=OQ4i=g$9}4hHv(5%ex3iO=xrF&z<>NQkS5T{J
zR9K042MXAe3^$z?&iRE(MQ3D3E#9>SQA_~vwrX3j)RRULLV$j%cvkvco`?3W7qOue
z1>}SWw0}G*jfFMskBAn|@+&VaEVM}ki#Sm@@SPu$2H)%#Nh``*9`5G9pE#4VZ!<-r
zj8k6b?*|nlJg9Bbk1f;jw+{|zXlMe@cSe_kk@tCNrqy7)T+O1-v9VM_@_m_)y170C
zW=|#GS*V=PN4+R&mwMKIf1@3fYn<$<YmY6-Ga<}{{Z?aO*{D8i$Rey+)aKIvR0E_{
zIrM#Vv$^7VMY)2)2wv;fP*Vk)Rou`dnz0a&k6&Jrp?&>pb{gH-l@Q)F5y8R!vZZzw
z>$e)jZAg%Ms09F(Mo^tawuiHwHjh1HYZujjIU4&}!t`baLYFEWHHVM+#lsPjjWp#*
ziEO`Hc+ZrwAX3+rWYpj&nKkqYGY0hQ5WYsr-*q=Ri;*#X_~5cPQS|fYPk_v*tKAuq
z=<p7{>)!Z48RAzWhCT}V1`%V(5t{e?0Jr%As~Xs;H_WwW=B)&vmM!<}hH`l%r6pdg
zu#_lHYh!`q@qAT0Lc)4e=tuuK$2_@GpDZs@c)Ts|h`#$tcJ@>@cciE8M?%oemj&?!
zQ}eRPhL1HAPS{hGY*Q&fwpr1)jvo!-a(64?L`(ih;xluTP9G<Z7bmy8ytzh97oD6x
zH~{UNBVF&jyZta`*{re>Aoh@mizVkXTl+T&N1q9<Ep1#Hsz5E_p_i1)->$z@f%1pw
zFf-(%tod=o5$VE#;c844zEV$Q-ae64`w}u4D~F*^BNRz&@YM!-(K<~3Ie<Md@vdua
ze<AF0^`|M=+2S|*a0%}zUTk2SI90V<gVXY>G`80&-$y!vkq*=4Lm*z8>(fnu#KYBk
zV`Jk+Qb8mRzEdfU;*(D~T>-&n6%g7$&y%%RtSJK!$oAW{RUAmXokDdTK0u`n=}}aa
z+J<yqY(B|c8XrG9q7n6j^fJ9Z>~Oi_Dw65K>)I*{MkK^5i`60@zA{>CMv3fLTU)c*
zG^BKtem|eFYHJ@{oGjD+^_49%$na!+0+P)~Sql@ppNOJsLtGjifevwInmiz3u#N!h
zOJ&XuA?R=Jw;CF5RuB#`nBvQ^yRf@Vhv^&<AWyIZALRgaKqiz-_Af(9A|pj28)^x6
zB?>|1Z7&|*<IbNb+RL6V4<94)cln|y#R{0igd%eH`6U15;4BStpL{0x$8X!dHt-@#
zb!<z??ecuXWnyfJ8K;=AFhwUS(xcMyN8JAY{;&O3DA0k`=|&B4qY?GuTcmwLMKvFC
zzp=G<vikVmSmPs5m^EOo6itaGgoY(euiOAJsLUOi<FM=~CVst{X4Eu*R7EA#t%|Bd
zSS`hPPtR_aU;$`MxZdS<7M}C2cgBe3t-Zay%+c*<AEN30iq^f?l4d_lE{DPdTb7|*
zguob;hcwrNtoEEwMQ3Gu)?pqG_b`~_?i=f4=y2}cg(yTN-I@1E=W@TEoVwo+9dqRj
z#Tc>M4pm6+U;zKv^ydCW^-=rnX}p>=MG<w#3*;I&wzZ(A@qQ(>j%kKNy?qI0(3QLE
zVyy4K*UycHhQ{v_E4NJG=*;84S=C={{Zz1`y4>(^BnCqc`nmm9(%)g$sv;%*(}i3u
zMWfe6{tt_!El+$UArIi<kpmt0q`Ixsu*7BPBd;M2%1Z=rGjO8YRA`o?r&^ka;wal&
z1pF2coYt@fOr!^PuZ|)wu&edtA%b}qZW>N54ZeK*sUin|-D2D45I~JzJAIX)H@;PY
zS(=bwKiT^AjrqtfIRz(fEB(7CzaQgs?qK~QEKOY{*D!~AzH<Zp;E$%(3Iw={UGED@
z-VJ|$c+UI2FNKvXt?#l7AXD!Ge8Ik+mDs^wjU5f3|Geg*F1{e%Y?)PGp_KQ*pj@X!
z7hr5`tbzwXM@Qdqzg}fj>^^e(gw}Z&jM&)ROr+)S>gnk*A0Z>%ta$_9*g(ZqC1qu*
z?>?#+A#2~BMl4(}G<gt-^DQmhc5H+FeU?_>2W{5335tno!mcZwmWiqc*HCq$ILanp
z=oE_<Sj^{mWj!hTZnwaiNLYXmHT-%_D|S8_br$q+T`)6a3fRCc=#EN!bG2WB0*@Da
zlUlD+1_QQ0L=pjrMQ@3j*6WI|jPj^dbXLMi)_%T*S7YX@$Wd|6ASb1P8ZbAE+9oUI
zf%JG$)EIdzhoQ{v<-E7^WO53q87UZv1{BT?ydFsLk6PG<!;K6L8B&xku)FIIjP3&i
z=jw}u(7L%)w71pBI85dm?beU|qm=W;;s(!Getc;^xmN=#9g=EgA8Nb1yL)=pD&_H*
zmA)_1&Jy)b?Y9C5C7hj~zhPrLLm(=A0j}2TaF|%zp3t~|HME0n-(Zzeg!1iS=(UTA
zab*PE9AOb#M{vo({d!0IQT3I3%Tvo)HVY;*r5QHwn8MWIRbR`3!0!HmL}b%>j921d
z4XC=)6!}r%+@@&My$P}LWrB!^CPT<6o<6$$U<>10DX&Lnik!j*{x86joKo_Yx@W^C
zo(=@`j1_|rSY1j9_1lViQ?pVSRoQf(w(}G>do=PsZDISNh`*Er7aOR%TWBu%7()^e
zQnB^M7(<nb^;>GNf8z|Klgi(J>yH|0&~dwWaU&c1yzSuz4#1SU+DKAg^4-p=%NPb+
zIi8?}_+B(g0Ll-Mf#Vud9|plfd$dO!;SMv&v>er|9Hjp4*<6ze(5L>PB}lOpU)3|X
z3zH$HamJ;7!)XbQcbTxel<g$0o?IM!bt<1T*L{@#W#4yPPofX780QyX2GF7zTI}vG
ziJyU&PN7H@J6y->rDuBurl6fS7JyofNxsbb*{)3r9IrCfR{Br=j4liJ_pPFy)y`~M
zgSEjyvwj1z{iYO>)xY$%{hjKIhg3f%0(;nvKJI-Y6eyXRmNsC|vOlmhVf8PwGVz8#
z$sqOSOTS31t0Q*8P9-x@XM0W;gO5H^VR_6H?AH7ZU23HaLvVkfxLQ=WUaBl()ayr<
z$>`U7kK7bTT6zKAx&~rAt$O4&Lim@Mr{{Tm#%!rKaGqk)+8cP+=*l(n@vEb=Hxm&Z
zJh1;H-{X#}fx`CIySrFSK=8}ZJCPtULpriaLaz+@r0|rwDu~FXmSe>o0n$=0u%QYO
z)minHh-_wI+liXTi>LaU&7#bZe`pn*Z4V#2vXve;j2@RkAG|87y8->3f+wd4Qe$R)
z77LZZV8bn+RG&5-8U}i~c>sX~_Pcl?7B2zKGWFc|i1Q{WKt==b1<2k}TDouV<!pij
zz(?^5)gOn_r%nA2V3NqzfNOVOUt!o;fm+t*xVU5mN;rSZH<an40NK(DX>1<+B%kBT
zmJCrqjLe6<U49u=ICMh|x&N@8M`mVPvz1t732~aOekF1PzH80AJeWh@c}C^E(iyti
z9py4p+3xK~9b(#cLoMdxdUd=CpRS0FKUg<^tXpb)(htclH)=Q>m8bK+%@WMcI+-}7
zOAEg4nTAi5W$@XZW=@;>`cC1wgU)9QvV%4!(}IjI&(isW<0$$4ZW9h_%mNSQ^N@(O
zGhT<GoKj=9r2e`&cNAFt3L1UIGj?i3<ei9i5=b?ZFzxPfe}B)Pt_f1nH8yUwp5Kc0
zIta(WD3Ntot}j!{Gi(T2JYMZyG+rDVo1_KPPM@8@)9lpLhEY4wzS}$$qN*u+dgBc!
z7%@F9C@I;YnjwHgNL`s0EdbX(&ovV)<Rq|NJ>PPPqN5h}k{{p?Txu%x?=Z~FywnmU
zQj<yjio>;^y#Zon39-Y|<Mi2%Z8BpoNu7wS6t=0pxyv$iMTSgm(}HYxH#-Cm=11ID
zCfhpD`vh<S5Bd@oi=XAsINL4eGW04;=j)y3!Q%euW5yl!_nUFhOfu2?6GgxRMwim*
zXZ#48<fSb4;eBG+s>;Se$<85>{Mier4W3Menv>x~<&Dlo#E;|4Et|o;ih|Oc`{NzE
zaaFX{L`f<}OMs(Dru*@7c;Hdn@<-)_;2+~S{lV=LY)b(f*=KS7&R%^z0gHRt?LqBO
zb3g~jZ5@5bkGan%Mmre<smgJE=sT=TkS_jH`6%H0XCd<bXas}E6YtTZm5T`I6xeU$
z+Q5=z%u&3<akMM#qqmLKa8iEH{qm{g;!HQC?8ycb9na*NcGt(WVQC}(rSC4OX=ZOc
z+}IWM>o-eB<uxT%Lcxaztf!03Yh-FG{#X6Y5(1`M69ZfIE4A0l5AMh9HfEITWh64T
zm{pPW%{z<R5qe#`bqfeQGMMwC>eiZ`bcvxePsV1Ifg3cnzb)0e!hTY}+`8sYz^K3Z
zP{63WdCi<$K8IVgXo%gRjo8U~_HK=blZb5m;f=-0=Aq_$&EB(C;1BByc&!x8Z@^Sb
z4x{R}x#fWIT`AF%43KF<9kGGxAtD1NaXD7{#AY`27a5?gY2z|T#gP_VcUTS-;n*Y5
zFH;a`@K=6i>0#zxxl6swwm(^AXgMe5<8Z}ZtgC>ND4zk5tqWfi2}8w)gsm&}6W}es
z(+Zs^T>Q@#9XbS~jB0u7MfXH&bBevZe4TwHg52rz*6u39sPVQnEd%T^IwlG?7JVBh
zA@RseTJ;0f)*8QICF2*-C<q^bRTy%Vy+gNl&-+T+WrUMRJ~n^KM&|<CE|9b}e7Mb)
zPU9_yZVc$Y2rXnJ&XHj_IXMAwTgq8j{8Y;GyIPG>O*BV)GGAh2@~pPV`)I~#-V0Ty
z4ZMAd=HufdBrJ^fM0RwxXLBGWPPc4N5?Gdx25=Z5+?PC#qH7Ocizz~3b>H(*$W*z>
zd^1o#Na|-rA|mup+p}KD8`MzhVt>{FnyR|qyAXW55-Lv%s`b0r`~LlVhk(oXtC8@8
zWb@P9e&vo^uYnCn$JN>yq(BVKeD9eS@(7tKcJgQ9Iy+D#a83)LpI~-6)UX`f;0Rwf
zl8+;A@;x&P$0i7Z9|$ge=Y@QILyk=i;f|9*ab7B~qY2tBrqUDByr=3M1@)%S*>)d4
zZt?eoAHvryc|Y2^Rx0H!+)d|8xVim&QO1!ed33O;16e3`K)E0a;JUh2HX&TTWoHxp
zC@syW+GsdXO`uqCc#FpbpcAKt^Qju8r?uT?TV~;0WX%j?gv3d)<&lI~?Cuzt(P{D0
z;Kki!mJ66#A1{4c0?5f+iRr&SmJ1(BHu|&Vy;_v$CAm<425mz>ynb33^s{vwV?A&~
zFmibPb^NnX_I?ASw6e_8E;#$brZ{cTDm?qLB?hU;xNe*YJB}OtsKP}WbLYJ#<JsVx
zpZ(Imq#!@vYi9(SxtmRexS)}&k2ULQeIsFaQ7>%}5tp61J%GN)W#I6(z#MQ8koCaz
znE}SDxobI2SK>x+($Nvz`m$sD`f9ii3%#>P86@X)UG5Sr7|7RE@uP2Lo!!y2*?A^8
z%@4ErrF9yz&U187xW>~L^w|F7+D?aKt};xfWHb|xS`erW*NTWl-fqE#Ipz8n@1rDz
z8C6)UWltK!Z8%Dek$vNw^Ny>rdvB0aWg`A;XbR*$3Lwh4;$UqDz>DGPAIDFCeri!_
zLHxDFo3Am<O=vSn5f>iib@p}6JaElfmA$bPhsnL5VO<A(J>w^s@;Na%<|a}$51%jO
zCblwb2sd7&tn^l7Vu}&55b5LS&SvdZOVPYIwMMxzkYk(rQB?z(0o~smUqkHiZmu5J
zEftNNoF*t>KK|SMHFs6u%c<s;solmxE%Q2t*@Z)X+BbEAS?m50Z6dnJ#Y!D8CD9CP
zgy*qOt*`Nf--?jF0o=A|oBE!m{AWJFou!@=ksaNtRhB3K2ZN%h1h{Ud)Zo@Pv*5(4
zS229EQ>d$YCeG$wv{B;FV)DvaF7z3e1_>qwlq_t~V6S<IX{0b}iJANy-DYFpVo;_e
z#(va9L?uN*uVTK=_x2%~ciYTo&%$P#Rnp9`EU2HTqrk@0+uPg!{I`FLm%UK|oghFd
zugi8Lfm-<V9<fYDLh>kVWqTWeVze5Z<}Xh-q4`mY0+|^l&~PI0M$215GBUDpgWxj?
zxhdPbwGvr%N&YgA?Pik}t`a-I%I2oQd+mufe*|5y;pLxo^@mgFXT_xaQj?a~Z{ED=
z_+h>gh`7JJEncc}GpcuN4L)ZE5G_dBPZpE5`9qxMY7c+AS<<`-mC{@4FzfJM4!-SY
z0$at46mE!>98FgoQMt6Ed>J3iTbX%R5)s?}U_Y8bGk<#l@9>3g{SL>5W!?5tRlqcu
zsnfyCX-Wig;o7XkvZIqdI=@C6g@oth?5t*!c?)W)YudzGo4pR!4)?aq+2`t<JT~5c
z_&B8l-Zn8d2}$qm)RbdL>3@287|vXa`*?Ts0DR*l#%pG_9334!UZ9S8FuJdknwp!_
zhx4N%B2rRPChx-H+#4Hzes~SS&*ZJ$j5DHG--=;?+S(xQ7mt|kNAae+bjE_~T{4k9
zNCT!#V__^^+{!Rp-p%jkX_nLKtsdpB%V|vLUS2aEuT*mic{KO-Gy^kuI-|=&>0Eg1
zp|!)#<<fIaL?H2}AsQa{<Lk%gCv{}cLON5yVo48wVxJAUIhJvlHhR__MMn*~;Tlxv
zf=N5feDydUe8Mug9TL*w6kC%0R#r5$P1X5`Wm_Okt*7be7{@ux7`hL~4(*eqA!Mdm
zENM^x_w~`{t<szXLP2sjei8D~94ox=5g{<x$~{F&*6U-wY2F_34m$no5$kLv;DO=E
zW}N%_MagnNJ^9<1pBpfp;3dJE_Ee`i9(>+hEj)n6(tu1bf=&39)8f4mTONP}pI4xM
za{5+TR)bF+RO4vD_)GieKt`EUeM$_v+*_=-kHd*SQ+_aX;ensdls3N7Q#Z@dk&bsv
z9r&m@1#BQhrRh*&n$NFsr$|-A3hS4mBPgkirkI3kTXP@f0OZg$nAupoE{<2Rev4FY
zu})NisNL9G`uy}fb9bp7RRB{s(L*XGZH6;5(eH&6SzoKClMR-lQZZdj!hZQRMH;mj
z#~!(Un?gCbeJZ1|QO0uH+^ukzdV$S;N#_mK_||VD<Ka1BbEk>Xuj?l=>GA6f)yh^w
z#MGswKt4$DXQ#u@TiN-McU2|zQ8NqaPh@d0Fv6Yo&jg4lGAOXH=V<{mck3SvR6Pd2
zb#)1BV!2m^koi19hv2<|u@a1ln^)KgRcUwaVd_FWL^L)&W#+)Ae1QiO0*!gg-jXhj
z-@Pt2o3exXT@qqzl6Fhn%MMeT@O(7JnsttO)#ENs%3<SgMCIf%m4w++lx$$&`@@!_
z=BB3m8+Zj7rzt)zF8UL?=@wc$!*yThnUUY+<>m7YM_iQnVz7+J=k84lJv|ZDvlZ>8
zVs>NQ;n;+Q@@sX2nx`kHPcbbK&f=-aM#sCW7&hHIzM#8L(6n)3Fw2`a)c(jFK=6^9
zB`w$D5nGZHlNcYz%fg=SXT4AwaOTG0;Z|<4IC5}&n6)XYMHGMvK7wSE`)Q`9t{z@)
zZufie2(#ci+uO^7laoL}mj{QDPAQkRi<`fGEHZVA^FzCkFI_WaRj6L8pl||@%t`JU
zrp3|2y@`l?2OoSgG(3`8+;8y2OPfW<q{TCpYY^iX#4GXeWVs&+FowaFA7JGvC3+QC
z7t`5#WplMa2WL~-kWcrwk<0#O%Z)-zOa)7Aek`Pfswp+kpFH`Jo^Ee=d2tb+kkIbA
ztLg538j68aZrX<1IXTHMAfQ_@J?$!Q=!v`~y!TM?cM=Sd?r1w$@N!i)YXD0h%_Cgw
z`>!Vf^pw7VbFx|f0kL7ODI}etk{w>TJ_1?-_6aw<5t1_3`42O^(yYdf9{}{;ms_=@
zutA>G!|ii2v`3CC1-z4LtwDYzgTilj_i#g|OVFaj4p$QR5B$%)zj0rFKvxh4?aww_
z;vuM7K40uz>v1+=*J~Ku>13Z&#x~gWl!l}0QtTC9eB%c24Ab&pu_oi1YM(X-0H!Ae
zFP{}HD4IL?@f{@>(E4u<8B9Ka{E$rb9Lmgu!G~WTPN@V7XYgBJFt-NIrF=UrWh7PA
z`-b^;b5XNkboUH~K4PLiYIn|B5z;6jzBL%=HbqtQv5gx#K7aO8kx14n=G^nEf0XBN
zhYvZxsF<V8bZ2i_X#?uBt^;@y;ed1ko;Y=L)f)*3wJxSWeIvt208f0svQ^LOzD`qr
z1z4%20PU5gTCDQ_DXs5!Ooe>xVV^BEEb}iNLgny*=EJEM0&Iuod>N#*O{g*Yjeipt
z*r^;mi${N^5;j<nl^B&Y^cK1|Z5J-mwN<{OFfKf!B=8r3&n-lwSvXbXnGs)W0%pnB
z5<ZAZzV=F#k@wPrB*r#l&M|)*QguZ6dF@|6cf_7^R6&gfJUZu?bHgFOhSs~PH=X2M
zE32nQsgW%81`l%Ae{(j9$zBf?_=~zNZGcpcv^0~&b_s(DR<GoK{$YMj*+Y#HJlSc=
z?M@L^Ra%)jv>t~gS3lDC+|BVZB-NcqG!gCM<kAP8!eqr0QBY9Ogdpndne87R*Ltnh
zdNJ_q0)CI7psb8T|5agOana0oGo=9Rw}&cpTpS;BMeHxswFLDWjwe4y9Fm?*>Y0fl
zyUsWV{Z_+?B%@<uxXJN(K!Z?d35vQwnf`22WdkkC^pYy#;&Paid4IGGQoSiDCJdab
zrVIWq0K7@B3;u)|WekHN5NAO8G`|nR53#5qaCNbEFyDYGYPrmwPOtoy-fq9|U6?lW
zZbcCNi0Cj|`}s!$5zoQKc5!y$Ktb-g-|=iQr$x6-Sz63}Umuc?HQ$if7$AGFwx4dM
z-|5ymxEd9Y&dkh;kB4`5d3oaieGP~rqyr0`XaskBZu&{anYG+*Igi08zeP%d62Wcn
z@3-z&s1n#^@9gY+`t%761N-|ST4gKM)mWbL_4W1W=%`<tgs`x0X9(Km(->TLaP^Iq
z&$LbR`l?I=OUcs*%#q*n*K@<qEHJjuw@n*6J#WmfVTi=IV7t+{#4c(5?eh6s@zn$~
z2X^pKa`61&{S==1rSu0}x~C#e4l*ro0n+)%nF`<$v+Szi-U85wXTDzNazExn*|;~9
zqlHTFjKR~Hn@GjKYeLrS#Kzfo3tL=!IO`~Iuf0BG@jF264q?{N$eo&3Z7R-x(P}pa
zp8ZD0sRCYr^4Z#x(Kg>JAlhQvpKhR$8OfG>K@UM|CdxjC3kJ)H#B9$mI+#|eBDf_v
z?q0ktrn+8pD2#<SO$)xV#$tBY9hL-i_6!}YZktt8*fynB)Oe0efMSsfk2WPZqdc1Q
z_Mz&(=Vhk?2L~gd4UsDqDxdzuW$Ye&KyU@Gtm5b*YXy&wU==TZKxekm#YiK1t`4It
z!3!1G8usULG=h0)ZO<>=zV)j5Uc$4zA|D5D^9EtikyDKh%Nc>FS@&G;pszrs)zEw1
z023?IRWD23c`I4U2%xW{ZtGMr+RW{Aud<CMpWJE%sbo>^H>PDDsC$Z+my$Jakv;He
z!qlGh!>_e^-CCNmDcq_32YE=v&2DyQS+)(EsO0%ZI6r2(aJ!U~z%SsBHj~lNe2+rz
zitQ~!I{HhVx+dc8qpWtn(R2=0OLzSqF~^J+KLYt2#ukSbSNTLd2!_+t;@0bW-^!WN
zui^91EmMn{1Ock?rQZ&1q5G4gRqIvG5aldFG`MZn-1KU}OmBtr<MCrjuc?;!p#>WJ
z+Sx1ig^j7GxHfx3B>jw-NU@GbNY55CKj2BNSi!n*c;~LKE@N~v`mfaOKkkvdxS3Kb
z@nOs%4amXE^?n;)a0?SF`r9U5#CY+A7E`>3n^!Xv>jy-;B_s(pb+KetQ(`dd?ChMw
z9Qo;!af{c``Abib%IimA85u@%n`{@#Q~A}^W7XA~!DgHQ>XK}slNVA?zJ2?KVz#5A
zaJsoV?&~j61UPSYmPUaUNSSO{=qYE%kC1f`qkN)@zxz$GSup5iVm$cnME?EzAF)YY
zknMc2cd_eNSZ0Au$8h9)yQP1C@g0gji^8W6K$J}{`<)GmTs){gEH=38tXLc#9)>ad
z_+L|txbIk<20L|kqblWi=w`;6?x*LwUQTlabP_y_XVs6AQ<D!DGr0EJSAheY1!h6M
z$i>9Zv3MPZSDUpEI^=3Ww#3a!Qdn33cw0rC+_2VXb&!Jh;iv^Ul#TG+nq*Uca5*Uj
z)h++QH#h5&M0g&NE|#TP|1hmEYjm~H*PrPol61rs_S*mM+uKo5;j%rPg{to&&d)tF
zj?m~|Lm!UdQTeb_?MaJVc~nXLOU6sLqleg<;0;+RC>2nwgiId;wukv5PhPhQ4wQl%
z%W0U)f=Ojoc5jz_+_p*t#;*?x>g3fkgM;RzS@ZUxNTZ+AHLNKCmuHD^f(0ZyvSwSH
z1`HBe@+->@(qC}t{6hH|aZMJRnVIPWMqFIbpo$rLlO?a7qO|9z_;`eRC!kzjhYq8R
zw4`mT$)nTDv!IWM=gudi1deo^F3Zb7MwXNqkHUV8@0Oc&EY&%r*?_@^J`4$J6DJoZ
z{ZQBuDj*X{21o|pXrXwFEl`0?5V9M2xI0%DI$3E0T@FUko;j#1UF<$<@P(R5N8$T>
z96GqbvWya!Bttl8L=PA00=urF>I@g_GWTb|ZH%IxDs5ky1y6ny6xf(UQxnzG_YM!i
zz=LAbiZm!Gm7o*(L`SfR$EuJTiEP5UiVN`;YR$rmz>L#a2p(aOUjcBXW_!2cQ%}oD
zRd3Y{zuV3ZUTSn(M^T<StU8Ib;wWWE=``G$$H3L~xHq0kOG_);XYKLc{j~Yb8K<Dw
zKk*<Rt{O?Xhr8NJZL187*N+w(%}{rOgzB7{bB7|76jP1FCnS`9Ca@YBlZ-dax|uS#
zFDZDK%!Wd7^^zII4;!)8MDY|`rJ7)cruQ9dAllxe)m3iC8Lo(Ts|N>J+1Z?*QF=|K
zyi?c_a(}MQVWsbF#Jf?3+=!AEhu$9isXp{Zcj5aC^@oOl`7!Kzbq$LL#zYl>IVA>)
zlB%;EUEAnSMyarXdnCK&3_zc_!t9Epi`Kit#TJycA@kLP?NQ<z(D_cX0jC6V)&GdE
z^0k1hJVHT1VQ0Q!IbW43lCTYd5(l)MO&OuGbp|eq1G<LHNy^1jzj_pQbBe{Op6xt}
zK<OEvXXp?&O4a_N@%~}8Vb+v!RBv-cya{nj&n$oHCt%1VSPG|_RG*6y=REqlE`-_w
zqh^g*H*r2^4`WWp#-N`6fd&YvtgK`L`>Y|BgVj}4QSj7xxjoR&PXrM0?qtz>8O>)c
zhN-Ko%efc(-|QGWPo<{n?--n1n_}mgg=CnFoVLQh@^dqv`Tp*F)XIFCLmy&6pEM`Y
zV^BW13w*-{$#=RHZ*8RuJYKq*DaRqqHg0qNyYH1I1+04d4DUV=E*5WnNL8bES1rQH
z8H_5?X%<aD$1Tl~KE!>4t<BcZ)I0b!SrR9BAzN&#zbeTLgugPes>Qh^i5F6kXrmaU
z_=`^i^%Cw8iG!`_pp2xF_sk@?5g8d7*LlzcS5H)x@`HxP*CeGp-RzizcAI`Ad(cA@
zS^L%LSwX%k=bJa@r>7tG9xk=~eZV#bUPnvfS_NwJjdtBXLc>vHe~NHL>$i-g^`j%E
zG-zHzn`L9Vf3sY+s;WuT>Gs!Oa<@LW_Dag-W-#t*xkw9?Xj6st?DDj~ukXqmc33$w
zQf|^b!QwXFmuM-hK_U#kz3TSm0Z~8E@RbT#*ViD~hTW=g=zZ@NPFVLfW%$aMdx3h`
zrhdDx#^FIIosT0fN+cB*|M!#Uf&43UYzp^`Is7J$SVEOc<*Wc2pE_f63erks^7>f%
zEII+`n9(`RdVTUsqH`Y^IEK~A-Wh$|eqM)5`2A+5Z)61kIH@v`4mY~1^>g^>W$Ie!
zBJPI-kVpD{_#6)z={euQHlIGt@+(tgNLfH$-OmTj*TZ31W@Wd{K#0H|ZyN3y9#vI}
z(A4o-Y|V3a!_15%wnts#>agR>%Qatl*c(w1z=-VtiIR11O5Bu1St7OgT!{^gLR~%h
zV=r%0(=18}MyU#FY5Ao-SV{`RyT-y|l*dQjz&VfF5beHa65#7|-aX~L%Y*q%GLoy?
z>uDE&ITty)Q9X$eO4C?px+HOcn#k~&@Z$7bVCQ0U?)2_1tXp0nCB<GuP7V{hcCf!+
zN4O|cn>BCjr~El8Hcr}$i;JmcE4q1mLz}&HNLZIxOVyOu^$*v2Vl6Bz(kcpNf3C&W
zLemx(7d=I?l9IIU?XB`Pi?H#@-*;!V|EEgDBqK|6OB6ZhC`u2!+8Qq+8)KY{GQf_+
z#%V>&@xhPfUcLo9856J?A5djV1Y<r~5ZGA*`3(zIXT;L))TNV|Wuo|ugo=s^<6c`R
zl-jI~kO)p9TO}QJxw$!KAsv;RoE&g`DEbCkk4xaV^4QS<H$se=Ys8jM;`UtY)vH$(
zt$Zb|t*u54{xTZJr>CW6ruikfa=%6w7u#$Q-%&@#Sy=oLhm%zJ_~amz%#rmjAfSsu
zv{+lIO$x>pC&T(xF%JdYzb-Y=KN4wc2kOsMl<1|T+W%1s+1Xv4ZFibCX?b8_Vd0L5
zW{}Z1zdUc=2HUnRy=B^c{8!*gnbl_cFoYU7aJ9MW{?x8SF?4+BUj|$TjBHgc0Vq_F
zz-vnY%+N87@<*A{SEams-g`AQ*0W$2>_`ihf~EVr05l;{QGPC|BJI7sy<*KG{9$ae
zw%^^0Z92BhP&af~hCmEz+Fq_EiXlhk5!?ur0-H63Cu;d4YM7F#T`XW>@j^<9K?eS4
z@i3;`RGuq8j?()|=AUvf9hVaqpcJc=wU}knwg5;bNjVm@007=T!3>u&OC*JJ0UGi8
zP&55hw$|3hc3712GzC$g2DuCz*?;X1g<AVC6z0<S_J3MPm4(O2Sq_|#XssMbj~&1K
z_k%|-@X6K8g+uQ<)Ke9#$9@yxgOPg$*S@~_Z?PMoxrtjqtbg-?P=Mi2_atKlb0XQb
zG8t$50dMaq8w(?WqsF%U;PT6a_Q&7(*yH6q)~>$hbNrJg_bu+Dw9S+}zit<Tw>yGt
z6jy;%>UZa-7k-;b245FN_<*Enlz{&vzGI~$;{OSqKk+gCe<Erc;q-|K>;E*>pB5NY
zP$J)qX8c?)Sy@?ClTWWv#OdUO?7zcJ^L;WJjy|lP7Vv95{^_y#M4EB_r|Ta-VgKLR
zv7#5+ZMR?kwS7BJ&;Px<kDtl`U*^dUGXMERd_wvE_AG9M6yScFla=(hr@5nxkM#f~
zfj@Ro^C|!9)`l~^!yfvC^X#Y#no44V^Iq70x8x3K!N><(ynR|ci2e8z1AcF~stJet
z_~L^o(_gFWquC~GL_c;M^n@m2m<60vF-55#&D9k_|9fwJS=*OQ=uaNWI&=coecxip
z61{zF$q*vLO$hiS(@F5T5Q0q@hbH4o2grYPdHr^j@;y2>c}?g()iU!M03#0a=Clo;
zMCjbf`X#dn;P5R=+-^ud4_Cgs3A8bP?<HDq##BgaPjxL?i~<dwV>0^mx6*FW0zh=U
z#z}^@)<1#se4mpWR%aPx9?`-3f`e<huzx9;vT=JIMpk~kFhnz+lLL4oSx?egTUc1h
zA#wC3kf+z=vH5j}I%QlWD14s)un-U#{2XBwka=Wn79oex@MnK&KJB4o_Keb8o0&v8
zk$nip8n`_xOB5hxXa2pnFm5hrpz=PC>8E1R_58rbx?k)4EC%-@?c8m~{R+#0KZ&N&
zziA`78eI=8(>t%Gqa<U(2Dn4i7rG9VNoR3PDYI=`?e2raM{wdW0T|&EJ{qi?pJYrE
zWs<)!CSVFEg;FsYn8GN<ePeJsp*_Y$xWI!yXL4Xt44nbvtQRASp!psCe{&d`%D7Oa
zdgr;P%w5<Ib-r$2UI2%Z5?`4JsFBltmvlM^lsW`7d+wsszk1bEyN@lya=ds#0)TlQ
zq0^751&i2^<<ObYeaeNk2l`K16BxEwY(bLnxho_uFVoLe=|E$AfS4kj>ZPC)?RvK@
z<!&Q&{o~pCq!QB%ZFs}`Pl4ytTijCDI9p=1ELAJ*%nee@_jT;ch&q4y?(Y&T0|yso
zh=9PiXX`Zx6zzL5mgt|~!z}6VxKgZ@?$_{5x1>zCi_^X?_na#Pj*U1-3-fMmd7Ae8
z?FZTw(C}AYo$Om}@nf<&KX)IW%^u7m9dq`Q`NySR?u#T+h9F1P%&aU2J2LPl#t~c~
zSG7=?7^0|d;@#2e)Di}B{lS$&O7c=7@byIkkMs2dqKdcI`6@_y61ZCp_-oUL0;)*z
z5ykP>IA~O4LARqU>s|z4<BY|d3HJ<c{^#NkGQSp4(A%4~Pm_{RjMru;adzg{@5EW3
ztpnA=P6$lGJmD862dCN1wJ)mWRzSuh^RXZ02?3(6V}smM3OClPHS>7&4#yvIwmLcS
zj?Q{I6~_UkB6-5s&{$5$Z6rZ3&D)vdX_Ry=`);!*^8A5j6z2*hSPVKu%#7m5!<LXM
zW*JOMbBQHa2{r~*Wm0iAJ@qSWhN>%Zr87-DU-+3AT;nw2xIc?yCNvTE)yQ#2lFq+d
zVI3p!8GiDCWL;SL5oUX2T-*mbunt9!k5+l<%oKBPhz-@|@A9$dCfStJ3_ByRBFTy&
z)7#rG)n(z4J|N}3y-b2gdfjbDj-S6rG|h#>e@6UX;jE+v#Q4MIEyae*7Km0x5GiEI
zm{Yc{g5-ve-q>)}wiZz({0$=;&mC^iIs;E=0Hxxt1Ju<4LmdPvX9p08?78h(ODrX3
zk+-85pBpXHnE@u@$*H4LFZ0u%bCB5rwC~3b8^&M%LDO$%>8JLX%zp_;I*Jtx3~;hw
z;BYZMKEafo$;F~m@7#>%3^F}y!;QDhCT`)oV#n)#NK5Uw%9>vC874WTMfSJc>LNj6
zw@Kab1)M+&+;h6y?p8)+qzZ~8ZwC%sc@M?Z`W?ev5(iCETlvnvB6)GJ<NSiJzcyu4
zn-<t3qFH2~a$W6yVlVk-Bbikx`&6XMSq}Hk^#gE_YPnnqOb}v+=wphSW|!od_SngY
z*W7G(6EELazY-nt8I4Y&yi5H8{0LDT*K5_XMeJ(P`u%i4Equ}h@;lvN>Yi|(5BoJ3
zbUv$SX=;pUasGU`Zzqz$0KYb_QIhzxoe5-7zhySf&eTb}8>eb3*8G1@HP!AWBZM(v
z5iQqxGXu|^j~`lVZcuN(*?OWutoKS76==1?-w~f3c7iO|4AX8{t5dj`o$``&?77&<
z_fDOr>Ae9=XCl$3Tv}KCVy{97iQDrZ8DO{a!@Jc5AjszHi$@zBBJl1zqoqE)se}8A
zx}z-&K+s($8$63xGP{pTknb+d#anX3jw56UFUrN2{$?90yN><o?&dd?h@eE@%{i^i
zDDsf<IaO`IYP<imF6J<yc(e5mlNQ|p|Is+i6jb@V)8T9>?smK9sC{1W0u&UK-Oe9;
zN;pqJ!M%X-tvWjx-aypZs-EqDIE$4mTS5%{x(~5&D3dJ{wP|^uSSMNm`bF_ZKzCu~
z!NP7nfd(3VNW3Rjd-PLv7cf^ye`DHutZe90IX|<i6)fIy-&cKM3C;jI&iZM{*jgt@
zM_!(`HNLvm@BJb1)uKB3wWQoXFhM!$<o1jU5Z@C3MpvbAPPyOhKpF&P&NcCdbgouZ
zJbZ|A*$u_;*t+vK&?*ikju!T8CdRv4-0I)bWChUoFRsn2rg2}tWN2R7P;~4FP&OLd
zSOEV1<?j1C8iX^@C1ZV8&i<aW@X<MJzQy7^^_k?|DPqcrThtVr^^n}O(pE1WyT!{o
zOKVZ#AQ*m`Z}+A3=Fjl-JCzsDu<HYNHoNU8Gj9)%v-_R4=V6aJ!+=rad;r&A#21Tk
zLK{0*=fzcsslzAx;t&RQhqHIOX|CCI%hse6UxVK`-0@08v33*DD*iv(-ZCn#VA~en
zjRyh*C%7cR-Q9u&cXxujJ2Xyk65QS0-4fgq+$C6WcYB@W+#~m$fA9Oo=pLg-@4ahR
zt+Kgh&00mnOhzYjp7s)8zQKQH3x9d`VC|?Hi<YvnS5=fPQ}y`A4mK$TolyE@VQ2W!
z1Ijnba5FxzByi&8Uvb))e&8et><8$5mZa&hLn>pF3F%}5hK#?O;Jy?qMZiMx3(yVw
zwHIC{(BhIl=ddO<|Lo%uzBrk0?D>mDW_ZWL>*3DSh`Y6vu8_0+aSfxhW0}WxICoJh
z^M3nMz^-R1B&4`dx7^VD)<f`f{*P36SR%m`F;s~OR*!RuuO?!^#2L+m5=f(<{_HYp
zD(5Qw3k}Xz24jLc%!`6kFd`)@F6cvccTtT*$#)W{wUPooYa!L<qk6-1k>0;3%aQdF
z3lK3+nBW_rtEzu@-$s?^{GL$vjBO*Mc*al18>DYE5efh78X9I>2DMX|eqlYyGDs(!
z+ibrIg{i=(Tj7Qe)(T<ZPxgOUmgBuyAJ_8lxG{t@P7Yzmr|ri4UET&tyPBqAN4bMV
z3q+lIFUgs-qy4Du{pPN@jSv#Mm;qH2XI-8RGuOfZIn)>0wR>yO>D#k06(h}9g))g;
zjZh;xhv>;ll!dKi=WS?aEV(6AyT)U5oYqkzr6Re?O(!MXlsR%UpCV(1NJ1eBz?08|
zL(=Yk_CZB{Bgl4UZ3cb;%CNc=&R4DG_<#1F<(Wf@44%qc)`6GXeZ}(5+(?wz>3d*u
z0jzihK)^UB&4zk1*&6%_Z}I_u_0xFrWSf0)wvQTcWvk6~6`#R(l=^U7qb9T7p@5<L
z^z>*AFH7Whi3VV-=uAu#X7WFE|4=+Fop;pJsJ4A*m6~bum^A&2D=oKRk7|D2m3OO@
zG#f(C90Z@K^(eph1uwv3ZPLC=CEz)C=z;B=-M8W6hwY9)Hi`-O45KNw!X?Lvb5EZY
zjQfGQk+085!c(5U`=<wup3ldNPvc|C$B*T~5ww%dK3AFa(O#Z6iCfV9s7`re^+#zG
zC=@8(W<y~Ur<#{kMasH_Mr0xtPmQ;FKv1F*lp#@**Dv`+{&u8Er82pzuTNOuvwbx$
zp2!hidot>VFXaDd9s#V6O5mrPb^NT0W{v_%vK8>6w;zU225zkKBx3#!aMRZYT4lP*
zNn1QY26x+fp*G&X;&lK1**QQmF<JE`!3EJ099v%3u<8?}3#@n<2WmZ_H!T;2x^1+-
zJ&tzfh~T;d0joEOtHfw=SJ(HQZ?>S-6XG)K?<cRtVQ%|6y}sa|LHhUjbMnK(nD^Jl
zh}&CVRQ@KKQp2?K)Ben3%~_lCgr=t4%;U$Sjr7)F=os3Jmi==7QSpq0dgU6MvE1$V
zv>0VKod(BQ!{Hpqp8C5=#o+=zi<k{->9A5Ew#!~0dwTfy1$?Z9kY8&Q+;+OIJe?L#
zjouT5wzlC0w<A|NN=A`do)5kDt}Wo5y&UZS`Yz~A!a;>wpz_a*Z}8mONoU%#3I;n@
zN2~%tyCuB~YKL@bbB&MP8OBN(Q3zv_O@msIf2Mv_BQkPwh_|Eeka@hO_RjRyWt3s`
zT4sUsIslp`Dh>8~wUIbrI{x%B%c!41I(8L*lgEB#SYs&b;h{`CxWkyG8v)kmiFt+3
zXXHabrLdz$^L;c<hh#cGr`>_)_!d<_jTD>C;}Y#;S%OqOLAi+G(n&9k!jO;=1C7?b
zFh*Rx?={Dw$#W?aHKu^A&cvh)i_Ndn;`s5$=(FDA)&zo4HD0DO*r{ukF1Uc~ZG=b6
zcqULi+O10GVLlin!QO-OtEh{g9C%N?2EPf~iQqt=H?6nQ{FN_QEGi9HgY{vj#6C_Z
z#Tus+NH6=5tFE&{LQ`;dA=xGR!=Jg~mi2_YDX$>-t14=*mojKE*m)RNl_<Z9*TGuS
z^|0^j2}0U$oc=r5ibz9tI4LD|BcE88Hxa<#f|bcx5gQ#TqI+U+UN55xwm|_X>Q30L
z)N62N;4ghC=#lb*FE*jg`@%j56ks8Pi}lQt3ykS!xHLm3!R&}PP;ex}p$9`<8Ho6>
z;R=E;2-tEmmVLX_S7W~t%4Ey4^2U0Edi`2I|J1usc7Aww*<PtKcHMbowU_W`j5KGJ
z@x`!v_>6@X8%tBj>&$fzh76M9)xa3x<Ko3BSY`G)4?5idIYv1Vi_Won><9NZguDer
zUVWPIyzM#9&VZn_o6_T7D{m)`o)&6bpSgehBhrnzv?4KaW&vF{k5D3UhihQneyB)4
zmt96!DAlp=trdQvV<!LkP%l}>`tB+rjF8zL87t^hp81i20#1#_pgC9nI1yiLdbVX0
zogBn)F|D|h*&-XfG1V(eU|X@9D=(k-DYoN^P4KU4mXw}vJKgVno1_azZ6V00IQ%=z
zrh~f@&YxY+n>JtTY*#yqhXFuHFa}C83sZtcc=81En4X}Y%^58PEIo6fH1(%$j+h&z
z;AxNsa#)`J_|<_+A7!42fZq-##093Cys(9O_l;<Q>n{;Qr~~@zQ|G<TrU5puME|<V
z61llFz~9GN$srE=e|O*i<>50Y82=#&M$glm{>cTr<Kby-YfIa?EujDNG6=SQ_gOL@
zvfGszBF$>nmWBRDu8wRJi<PtG843Dd_zOmB{mv-l-lIMU(L`oYwEx?5kT5{!LUN$L
z@suEW1^w}U5J*Q6kbys!5zNKR{suD=hM4AmQcgkpC*2Y89r$n5#6NeB|9_TbO#wYX
z>WUlC4dEHxGHm~n6L}gjlKGPPkaDuC`bSWP$>>lCAHDP^iHna{n9clM-o^}NYG-HX
zcnLg%Clp9@r*8^V_Y}X&t+ejmQC3!_@Hq!kXq0Sof#2pa*W?|?e;FKHC;}-e!31SL
z{vR8OKnfY76(almI}k(OAe&^!ztb@PrK}PndNA)ooAQ3(RBOMcV0G3(sP0yIclvpF
zp{4kZv=UCxpQ>R0o`vK0smpI459ibSx3Lp6L4${%pzEM2F#_Fu8-rhh!otE*TB!fj
z8-z}@&__g0fy$$w9)8^y@A16J{tkNSts>XgBB@Im9qivNF{WTP3nWN7>@HDsE_VbF
zp=*#1vpG}!smT=G05||oHfEyD!hs@&`TKY4zX@MplKg)3m2e8y|68Tn90K6s(x-9e
zC0(z6C7Y!r72LggRimvH+*?(BYfLarJr+hi%pXE4hUYsSAuytspV2AF_uQX%EATEk
zfu_e-I_25ubNfwH;ZhS5U4DUfokmZ~x|=6k`#d#^u@WlLjl=Y<#cIW}OAocl^z{CU
z;UCj%UcG6JPmataf?wOP<$!kc^dNw2ivr^lDnf`b!EF^ZUk9G>`uepYkl;2Hr<z=|
z50eZ<gR#$aVrhQY)?rJG_X=hq8B~c$Av8?tyo!l3VioODscU<;YMOUqGqgjv;s=s*
zkQx9m6NBN^tmc_rX6wJk*?#ajGA(o-m)TRUw-WsXyS35G+x3EZ_FT}|1w4WgqsgX;
z)&-IrSlVxT{L@b*CR_8N!6*cJ*2%Ru?9UVP_3#d$<y`0<-X3nK8^>ETpmH2HdsLRM
zN`Ovu2MR%F$qD*ie8^i%t3W$Csfc&+QEb|HAG2fWv**eHD5+EOu@6H94zN0nhi)Bz
zD71%Sa5`uGh8p$R6Hcv^t#ojuiR(5+fs1*ebqGh>d}!sXCv!dk*Ra|8GAd+bUK!n#
zf_I^HbOCy{YrpRr)j0BDA@|k+cZ3eQg4X!Bn=cv2JEfbq3ApE+Ak?ebq6YiU=3Tn=
ztP(UCYN>ugJ69{-$PL)Y)d6Yq_*!fz5Dr69xv-kAZY};HO_$*NbTgBiKkH^(lw&2m
zs=pHx)#&j2cLYLql_~`6vH5%|ERKl<{of3jn`P#T(avYuAbx;V1Oa;1QupOBI>ZN)
zdwx%ySod`NS)~aMe1=e*w)UspZC2_2yFIYfmHO(MDe^TnU4@6xk6KZCjWxiqOGZP*
z<w9V%Z(?_|Ugi5l_JnnKl(Z+wQLgL#xBIjEr8G;v;&Q1{4#FWYAEXMiy`#?5YdDM&
z-;5ZUJt&YT9E=yKUw_}NoJuRxD|77Bm8wASM420yT_B&_0C?5wu|<eNnN{d&EJLX_
z$9u9?zT9)8M(R0G&Muj~BR(E=#>0B<n(7d>f9&M`#`T%%PjQ>emnLWcHhiL=>(srv
zf;#P!C~U_tzZM8qCtCvaq6f=$RQO+6QT%k1_AFCEC~C9&;kQwG=?>8)br9#r-a*a8
zufh`0VQ17!GM;L3f%~;u@!WjIGw|1aOTt0CM9O`@hpvLU9vg3~Ix)CtE<#>y(KmpF
zj;Jxa4kZjD`IJr^8QP7O4zHcyp$dzRbFpf;ybvQSqO_<I-%4o^>P^zCtv4MWh;PSy
z?2F-hxI?a9n%h^)u-<U9)Bnl8xkcj-pi2kk$Cz4xSTAJTPf#?ycA{PrHfj5n`Z4ye
zxpqIMbvs>cW5gNLHl0b?gXPT2)*qkZx#y+ikpOu{`Juc7$j)`Z>q-lc1mwm1(nafQ
zy5E5EEqf+B&|EguuW8?4(X9pRA^9G=ip$Q!#jDc&9&w4Iz*VV8{2Ez|1Ttp-?&Zy2
zm&BmKc(U&VF!Y;z71w-QWLu|O;sQMR`C?kDj6{qy^);ZiIx+c|)14#x2TmV(n$I&C
zR*fkVUaq|#)88z;n#`A7b4HboQW@A(eByqe-#%0IgIST{g!@O9@hGuO$<ol$4Sr5#
znf{tYv~tIN3+r;>qKhwM<Ugik*Cd_|QGLsX4Bh4jd4bXSTcLSC5|bY}`i0=p0W68Y
zqnDV6m^GCIP}K_=SL`Cypxlz0$pkzR?Vtb=*?P_=At8Wy%5-{<b@*f~mJo2$=0;14
zuko3W-Qd|fDH<J%jRmH5cbKPkMu-ib{YvU%?y7n%aBD0hZyx(5)y6W+%13>AZw=wJ
z_tVh+Q8h9Suw8|HukeTanlhdb$GnWrotx_jbkhr7zNk+0hi0<HWzFi3@*VR^(Eo)!
zt9n#sTwm6X>EH70W4`vCnd525m(slN<$A}@=t?_Qv=^;~-j#k6<ezY`ZQJ-Q4Z+4~
zpP;$v=fRaIyW~ciAnx<mSA{?QxA9amiCbuQhR>mM;7|}9s@MQttXYQ&CYQnNVcN9`
z{;Oh6;!ra1c@@XnX?47Q&@P6DcinAW8iTi~LsmKg4fe-&!_`G!{xw05;vVPk-&Zb|
z|Aa;mmiS`uUZL!IsMS$QWHYR#BAz}G3c<)dm6Sa(9oGy~8g3jwh!3j}Q(#z-t({3y
z>U_rUl4B7A>=H;s$bLW@<k-QYtM<%E3B5C=7s^5Z1}<rAr_OI0k#Nf@rvwHJ-;{+W
zO#ftTqK`Ght-}rqM}N_kWI+M?=<rVFV}xMQdnZFtRi`30+F4`pOvFHSj((x_=%JD9
zw>vGY0IN^m5MWOHjbZld)7}o4yn%hWK^NAYBkv==44pQJ2j5uq+=}#PzJ~j^`C5h%
zVAOG1I|!W9N>G+nL${JqbY*;3+RVGr=m<V5d>tS|st5)(Ehy>(R3xe^%$5w6lQJDM
z_Ey`5vJILFgyzV@&hAT1I&&@7qwCt!u>0sKpKBuY9qE*Gtk`?4qyDzdW+YZG;5&>G
zxCu|>fDVamC2dcC#C3K{81H+tnx`Yeu{vT-x{qT$*5TzR>uo~@4ROt{vCh`Pi{bjU
zf$Xy_8+VU?^89VW`T{dcYpr<Ko;uX}y^!3p`X6X8LCaPN67-CM&$8V!{Eb`!Uri{d
z%mQFbH}pRb1}+UAMO^=5ok<uE_Fv#JWeb2z{1_RtW*?^MVq12!IxosuEc9n0=RW^=
zax=j!3L0F$C_DEuvr}?e{loO&1MwFd3-8h0alP1&$EA1Gi=R1~#s1u!w_-C7*{kZH
znnY8jK33}0)P}81lvRF*Wm%o00wY!6)C6S*tCFdz2S<-eSsI}lr!J2|kE*Rv57Q@S
zV;YNTMTg@z+-kqD=%Wa9#jAd@;t>Xc&sBu2*`a9nwgd@FnF}-D#ZM<W=hkn!pQ~2r
zKH@c!{2j$_^B^%DeA?AIv~5meBndxh7{2PfHP*{4B9Yi!s|#0qb5)}K<j~tKo4!yI
z0n78u-Eyj~k$k7<mPD~RwpQgft&1wM8|Hg{|2{-8RgF(rMXzFMQ8W9|+;lAVi<$^l
zuZ@o%pe{3Z71_`JtfuOuyUacJp~OL}TE)8~@6BZd5xS_EVElE@Qxz>~hbIqko_NSc
zr%u0<DbV54AAd+8#jH5l!-wpE+KKm>dXTr>XYsb2f_0C@yqs@HIRi8;IlQf>QNO;~
zF`0r8eRjB)!&Go!HGMi9@43ihu6EGm@@sQiUesPgcTA4Q(Xzs*zKxx*(@gStmG$vN
z!go=F!$Qi2+lOf-K&OF@=F&;?o+No`ep2_)CSTFdcB7>1de4K8PaxrU<B%C294z*u
zTFRjTG9-6ZDgkE?dcgUv9;5Ajx(~9T#a(@=E(mgEA+Q!cAU8z%(36qVV?X;GAo==&
z+=Z+DHB&tUfR9$fZT&6tt<WRIwKEZ3=carGmEGaf`3LJ()Z_(@X6x(vdj=@nGeJ!7
z>Es4kYp}8uX}v=?v-9~`zo%omSg!zMy{&@wb0vyB8%6!*D>eS2XK1DSgx+4?dY<8y
zIvSKboyemVUx!thw;kJpsm{FC-)!^gJmrq|d0C%Ip_@D71E(~_Jy&p7dv#|b7e52+
z@%LAmOV<`oBSX;UY*T;e_rN8?Ml*l}>NGt7W3mQ#quYi(RUq@T*An<VQPJy)g5?)j
zx>cKdLu55}K*>+qKG*hhgRm|4J<%1px<{K@IMG@rvVv}DTH|N8dM}qcHPl#N(gm8|
z^Srze@ZE3Y5w}`ah0^{iT#E0NipP57E0;U+A8ZtCTd_6_uy{FhX==j5ELQE#H@}>h
z4<ssy*2Kl#QlSsog&VP%ekZ*RJxN%U%jego&#*6MI|xti<q6It+&x`&))}oQ|I?=$
zC(1q1)K^vMwhy0e5!GDY)E)#;*-NSFru4)Uj?<>{44}TM@5&LNA0}A`d#&`hB@`<5
zKmiA$Rhv|38bD9vE4nq(_$qLrvSEsw&EEH!jY$3RsAD!;J-_?D3me4CUme{D2kIh<
zE4zh0E1BsAPR56MeNbb}FOY61Z6mr5*!?fW+6jtt`Pv@K@Y~<E$D=!nvCaO&ypoMJ
zz;{I|W}heVs0>vY7$<u}%nlIbx5)#P=Fod0AP!abs6w7QU?=|}GF6hUC98p49o;H$
z@MSP_^Tc!(U8#|=yWkL>&BrM<RYto!aq3lXos^?V>sEoJnd;zpoP&`QVrD0aDoN1c
z&amrZ7#S08okM=DkfnR7Re9uhO6R&{XY$66q8|f{Hy8eAFqpSlHWG9v;U{z2+U>as
z)mMG*_3^~E5NU(lUxwndR--wcfJ>c-dx7iTj}71S2Ho}<1a{bZL@615kcf-q*{3X3
z1FURC;s$kOk%r~zJlWgbLj3_*E+3<kyK8(oJuc6jdiHy)o$%Lo<y^&HovI8?#h$no
zq5X;=3bP93@U>n1EdIooA}&Kg!yNfhRricOsTv#}QUw`GUt`iej^cdf1w|}=OCuqY
zklFH9C_rBxzmx8TkGGaP^smJ}*I}PdA?pVKtt;<oUGIqw^@ff{e3$oQ=<ViR<R>y0
zGWD}+-Ha&#!$G|ou%iaoQFg6(BYXNn27mUfc&gm+?S$()_yt<iN}qOqS<W5uQRKbl
z7hz$w>ovO7$A<#Mb|me}>)d8$wn|y;c?JFABTqE*ItB-vBsHmRH_Ogy{9ZZn#)0<G
zt+ex>@nRgxb)IEo@GQ%6BR!@y#es8QX%C}5gwlE|)8zp=s!N`YWrK|r7Os;d7hmy^
z|EYpqT!pg5_NoMzorK^4tf>A(z}(>jC(Q{MQ-1iZwR-oIPCSSk_6sQ^KbdBjzbpWD
zYG#cnZ$g?jfssmx9UXRn>qUGQVk(@U^F@eNMqQ0*AVDHR5YT%YU%dMKI2nNk#3>fN
z{zO8C-t$@vz3&C^ACo@)R9xI=KTYb`PiAz8crU(y3Xs*_`iBWcy;nqhAKIhwK@_{5
z-G9URW4$PgYW@Rju3mlpf$?$^T{8U_C(J-)v7ab)5Z%kK@7Qhg%o-bO*hiPT?L|eu
zR1=eakD%Ye5);*+m74OQAwFmLqL{IYEonS}E?DPwsa%dm@L|3#LEc%nb$1IkVHbin
zP0P5RCnMIkF#$W3FQ1_zR6u24m<#p81_4Daj^Px=4XgMWU2;y{{`41xj#@#oHitIL
z5mz-fWeKd6Z)+_|QySMs#r{$|OR#wSKvTXFSiZhhqkNxAec4mf5=LCHp*g`B{$e@u
z_VE6qTGp@Bo^I}VI7JQfke527F-*hqcj^t$xxpV`l!#`U7e`vEH7D8<oPV;}3jKi#
z4oFLTpf2nE;@$RzPf)9G3JYxguX4K1i=KN4|K5<80}>3nO0#Qpdh+vV|MizsVufb~
ziUj#g{@gM_u+;KTjFCd{FN|Sx3V_Sq_{wkJ*0a%0p{iL_ThSB;7wBEH;wR+>Y!%o4
zbe1TIZTK5yu=#Om2n3{hoOC6YjzL4bXlq0W>7aNSG*r1~df0%mZS0tmfS1%U3W;vC
zxz>pMkzTGG%axaEKk`o|gpcqbLz4XG%=>wBo#z*apoq++F-*6PGp%#~TN*oe0JQ?p
z7K-m5b<D>io!KdD&iUr=VCTLzL_JxS(Gz(AvJ+i9D@o&Nl&sPWEGbQubjUpc0J<h#
zLCYuYQ#v(yKJtn>k0b4s$nZB<C8}R&itv&lN+oD4*ES0k4n)1TllZ+&sTs+CG*uH3
zOD&+T&va^_ht_w<U75qtbKxdWd&4RpX2&u#9~N(WOFvhp%do@N+^VToay8vFp3jbD
z-Z_X~o=?&=7{}u>`NrRM?HVc#)Snk7sLFJwL87s2(D+DYbngA!V+*NIHZ7MjWbt=*
zW`7<ETSZ}bI`sPBu@~uF=gu=r2q9y7)AvyXJ#$yne0L9iu8vjS99FSEhf(FdIltIr
z6W^(9Y5Ljt_3bVN2>kV@%7<ccS+C~39QR&y=nm~z7+EAB&E+(Ve0z&Q?M*x`_0Wby
zFJWcMTRIAzmUqo$N*J=OPNP(G6z!kx5&wcPbDL#gEq2!)AW9>GCz<wPJy$D?ys3Rt
z^t{<ICbE%8FvJF1!5QPyEF%3p=<PdvE@179;xPcN0QAX4hacb>BHq?A7P(Gij_mrj
zam<=4KR_S>cJJl$UQA>mcp*Yx*SpuRuJp(9z1&f2@weHwq5p+Z@NwY&0|petybWnO
z6M7_0T4<!De=1$p_%{$!=`Jdle^H_a=H|lzb3ZP;(9MzUgd<^lT!3IAxqQ~?^)Mb?
zHYs{$OxQ-b_81eC4_w|Av}NT;|G-h3V@~^b_Jo`(7kd4fP0wzX*!*jrfs_;~SC`Sc
zv))V#!|-js*Vt65riUW1-El!(f>jcvePpn;=%$@bQ_u$KBA&r@O>^(?+T;rn#4Go)
zqt6_miSgFk3(uz9d#2BBq1|oS3uQ2$*Y|$L#>WyzoLoY$l1>=Skfu|893o#P>s8FH
z=H$kV){HwmMOon|8&~zg^t&1B{$!5Rv)CZp=0;Z}llm=uwr4SzB3~!`n83j2hiy9b
zk;Jlr0{8=yP^UF7&W~zn5_q*MEr)IvjN@$Amw#g_XG1CyTzgX)!26aAsVm2I2r5JI
z1riL;A|;n?e2_Du$$vo*g$T53v|p;g(Pw=sm~>UYqKBC!I7u?EnL<2i-frJuk^U23
zl$z`n$0;N2E|&DRJwD5<4@G@UXlHzDGdLur)Ks&PZ0rbs3o_*Uol&}a`vkML9*FjL
zuEx6mSvE5fCIs@A9V^ry`o5WEd4`3{%31F%T!3nY(@Y%pG)EAkw=pcVxJTaE@@)Pt
z|8m29fH~nlZg$Qaxbm&?_91u*S2fD??sLVC5dsUwxWV`eIFaP$?*D}}X64rsBx6s{
z1spEw$VjrnF>+6kIUO!oH*mfyzMHWlwn4^n%{xlqM8oh5@x9)OLWO#K;?3aM$~N13
z{A<JJqbeLf_q)5&@pOOE`}Ynhhq!+sg-iJd@6OYU_*x20%UV7AjwH}*i`dBpvV5jg
zFe%SgkFlfvK709Af0PU5_=PN=$3x}h_Q{kZglbS4Pd`INWDMH8ulMMdS^jD#%A0`5
zDdgCz%k|&UPU-a%VMP<Ub7etlA?qr`$IFH{I)bJ28BX`dgcR_7Ul`wYyHylR{T?}E
z3S7f0ftxAI^}e&B4xL0R0gRZ>i!HVkX|?vA`xh_T`4_x2#}YeOxW1bpZo_I+&Rt(+
zqH@OQme>)p)-^^JR1;<!Uj8r8AmjJtF2wh?k$0`Jj3EC!hBUr#I^V~%jmJvc@f|$T
z<WVI;8WAzmedR~jpz00qfX&%Y*IAw0lTWHDNM2vF*nPCnAMP5?;J!CT9w$8d1S}-&
zJl}9?+l!Z)w0DKMy?JMKyFF6a-(a+R@l2S1u-V9iV=Xb{7b^-z+O}Gno_hP6nv%dU
zdSCrS*s|Q5PwEh~aXPs*!I)6F@4r<enKP=deJP0Vy}Oxt^~h@UzMfY3WzAc3@>g9*
z>`IIboKi8P5{S5cT9^T%*-I7^5u*qkl078iWK^(Z@7K%T|Hd63!Lr>S5h9{P6LPXx
z%E|s6J|F;+a2Cby-{sU(n!S;ROJdiKhtm4|pFl`R@bYX*+rGcl#1Z3TLHgsbrl*ik
z%imH=ahCi3g**nbHB1{*ar$1$&~eZO1N5yfbmn{LkWC!Te6PE4gz@yzdIA^#pcqx(
z{_$BXi|Y-%VkfmdK{pQcT?v<_NxJD}eEQiGX+Hta{IUT==8Tur1L5}vsBM+YC;hk1
zGDacMta;pT8#z55ell>a=tb`vriYtHuN~&;zovx66|8$!q}fN9DRoqVehdCKL(N<o
zF+%#1q+YULcXGN)+YvG^XkNmKplHS1&o@|li=p6RW;u8{uG*w*09{F}PMg}3KWn8{
zuN`{3z?|o<j3s`|U07kS=8|5hS_Wzr5edv+sWxl*KzR5(d#s}9f(KXQuA9?%kk@Wd
zU~MrkI0_B%`pR2|5ZPPR;q3IKAl@>I1lB8LFvIwVrE~%who&4^@3XTDrF*6D%QGJ%
zxW&6OJ4M9;QG<hoLRs{git?20^pbUGuogFLXz8r~mctz%Q|eXP^*Rf4x=6<B`2`bt
zuShDp2Ezp-!tU+81V8_2<fbbk-;XG0gl_NAce<GMv|!*%Rjt=^SDDXAQFmu;C9))&
zjO3m<<Z5Xkk*Co!y@|WJbXtKd#?#FSSe`FKjzjB*{suZ*Oblaw*IoFQGkEJWlIoqE
z6~D{+VJ?kZ|K)Fllt<lh;bvj*NmS5WOj5W->)Tp}A_?i~FCdZSNJs%dp(ssnMNNbX
ziMAzXWX9|2-P{NZO=`J(^7DQ%NY6yx!)Ki)KC_c1*+TR=M0!}zbF27ETY=5?V!!nx
zCW+;E$k1d)D78@%P`qZ1iOfp;xb5lW!Jw9vP8m1DtxwjNtP~0*t3eA54AFa^joiP$
znq?uA&q%S0IdpVbffs=5;+e=ssHe2N7>2#P8Ej~j{PqhJsl^Uyct{`f9tsg58?c`s
z&Fw8)sgkmG6B0mD;ysl7ttpEVaMxV-Bf$YuQ4oQM-d$5fZLz+)*`ziiFDgJ{8H5F#
zaJhRlWSWxZ0VSDnT+VzX4TaC!CDG#5Y8lkhEU~nvl=B%q$}fMBIzwxxw5+F&5MDvr
z6v(d);&v~;oVxE6<+c+h5pRFynP@B5)1*)3KT>aAnrgOXI+$+8PY8VJiPhEZV5QMK
zfmu6)z(&QBd~rw?g4n+^_c(w99tY`J)k`wn5M(X+<ZQ94ZdR^JU?2#hbg7NE=Kuxm
zO9)TgH~ZkzcUF+Wy!r6tZhvt1?w71kf}i`_1;CbL(g9qkrJFtT5Dz5=YkL>8sT7ro
z7#*d$?jfqSm{Lt*xjOc3l<J<n@Z+0<z*BB;;ApWwIIvR*gS}9*5{+=nmT$c&_$=vg
zv8|K$@n*f8>QgFeVgW^b$rp5);k~choLjFURW8tCd;a=z?C`rOoAV+^GLIGHa3J`m
zg$zeIJhYuIZ7K)PFa>1|cJHMay9vy*naw7@iX@iipB3Osc$yx|NPV@j1Y6Vg6ilKQ
zMxPITvC(Ia8*i(5`A%<UzP?Bue`71hxcw%TeCp{-`MARIO1<rQ^2TEH9!pe800zhL
zv?-xL%cEP*HcLT7L>sI9xdATS%9;V~a^r1PY+{dkT6`0Yy`{s?nYgrXI#`8ekfrMi
zm(v?xezYG)Dmy1tV5zM-qIgLQAF1M{a#3?t2XoaL$Pf3iUg;SM=@cKg^YyP<sQYw2
zb))nTt`8AXPTx<u4Zgw`dbr5RNQ!Os&V~Mfw$hEScXusrKT{tvj@8D)<n*dyH96_d
z8~G?B5o*qL+b1&drRI?r9u*0npL3dJd+JAAfzACp$B^Hv609*TML$hhc)O+_^nm2}
z^?I?YogfV!#qd`URsKPK14D^VGgDuiwslz4?Z#2w!FQqD#pdD($DwPA%-6haWV}?1
z3?J>vh+`b6b05qkl{>R6K0(e<8AI5^K-(UN3Xv=J3k&a`r@f&a&jKw6Y+}8p%A-UL
z9gvhmwpg9Sq@dwbA%2w|BkcX))}nQNas&cobq0P=JmbuG3o6+{f_pcA`rS3o-h4H(
zY~hl$uzy!2<1`+aI|wXXM99iMH5xUp4}Q=}H<EURJz!qO(x+l8&08HHk;4L-#9<B=
zJs6=7VxZ!m%i)V3S-$9y`7dE1AUbHEUhJwjTiGu(8701&yo9|j$_gA6-}A!nUiOF3
z(X2{{xS*h(r}v4P<jRi6Ki?KRb`r9qkwTl!v?{|HhsTU8^B?v=4roF0_el9_`8X&$
zI`cB})LsDqn-*62Rz8?MQRgUy`c9L6C=6vUzGsK8KSq%N@w2hjDs}z<#y(=B;958(
z?f@r9JJ-iUNc%qLjWDEvtm$zZgF<Kq^aI7el+U@qFp9o{WB9}Qh&Vu-F6yTxOaW>^
zELLW0FzfjZzq>?P=iX*yE0k3l1vA2lvdYB;pNgW$W1$@+Mkw16VwOWk=T=Q}XLyfx
zlBTTX^0Z9=2uOra&NO0@A}xJ3B6vXxZ0fFC#o`HSL@lt9)G%(OaV>QtTty%X4*X2k
zN_ZRD+W6GF&)&59w7FlhZ0;8}6>4K}63reod-uiK-FdFi!gaMj|L$^=M6TC9F>8XL
z-c#`YY=3u)UPved5r6}IXr+h98UWrb1HyZAvcMP~Rn6=|Eqi-mi|SpYo^9p$T56Z1
z&f3Pz0>)vMEI#m0e_D=l{6qw=JV3AGUfIN{6RjXq2R6b*Zb&MwtwX)!45n*8oK#_^
zGL0>+P}a@cPkC#yt7i!|<%tLkS$^ljb%Z#-GCJzQM{Z2e?^3YQeQ#lmJyD)&vNYYw
z(Ja1CV=;7KDp}S^V$EW^Jw|Hv_o%?;Zg{msM$jE`vKl;#IyP}dJ!V>Q*_Jg4%&F^c
z>9VHcBrl3PN%O@+b+EF%&G&x4<=Ogn>@c`Lr6Au0??>bDuV1N+q>z5?^4)a_{EG>C
zWb%|;4AqQ<6I)Q?{Y8*cF=Z$C3<*a`@hi8A`}u^kGvYo5+Fmh0V&7q}26nz%Ax%G<
z_+l^u=I5iiEOM1ETp0egqs&{<-6pT)PLSwB4eP-c@tO-K$^)j(c5mC3x$yxJP(#E8
z@^7`DZmIPwyW>TED+2n)14b~knMkB)DT&9oK(aB-P=FeYUBU-6VGB0*p{{@z@CtD!
zy<<ysJUc49zTK8{n^e$*5%MhJnE>@Qr3$eN>I-<Sj0}g3WZr&P;^<JjQ&fa=r@?s`
z4oA1U;<%HdR}!0CIEvO!A6x6ql+qG@VPJ!*lBtsGw^!c{cu(5PJ?A}|<2AzjQ~QXt
ziEtM)quz%EK~u_h#>8%*Gm=!y<Uy=7rs7Q2k$G!WNRs$?*UjTv2*`^f+u-Qio5yoJ
zB@T(B+L+!=Rd{acfU5b*C!?cK1k&;p^Gir$6Yib%aEv+vhL7IP42eVECSyx8*cKP_
zVsBL%YfR*jWO?2lDOYXbHJ;2hX;xUR_S)>Gf~A<{Y?m{?7&zHb(cw=v28pYy0gwf^
z`@+}cfM9B)<KidGNK2f?8iQ-+kXlquTHVX*Dv8zd9Zo&s>p;ubgqSO*Ke{2G+ns1z
zK248EwP5O@z6)d*l-*x$OcLi)1gkJFUlMXJ)>T=~MvD8t!#3=m__FmuOI{Zt5lCz@
zu$9@z+d%FoXI=V1>wR^~V&fmI=6?5HJdco{-$$ANyrFS7vdLmoCdS};zCTfHT7=lO
z;C!#y>ol8}<Wm!1xY1L-`XN1Ee1g9|l|4MMwaN9<fHXHmr0N6ARLI~8_<*@mw)%5%
zD)hMA+4C&lTiu6A4fgV)e{oQroqly=xC5r=`?}~kbN!ADGLTMgo=n@0w2z5cq?{Z1
z!Jn1%(XN3?%}hMHE*S0Qb|Wbs)OU<EUP>kW_#L+7)|x&Q8_}%_oO*%cgCzKM6~;>n
z9w4s#VDqC=$xCXw(_-|J?**|mJ}4a2wa|H;ar%h>f6?LVx+Rlwq8Q4N(OrX<)i|k+
zRujw5#Z_x}&uud~)f0nni5<_a<2TtlZ7R<Nmf=xpzMjIh?UAXRCGIG|XX@j%$YPJY
z1W<Bcn3_uvjTNtB`3wTG+K32h5g=zCuV*oAhXL<fGYmpT<ys>ze{!@nxm7^LJU@x+
zabPuo33Eqqy;+5W_-d}z!Ki@EE@l>0v{N{yRAk@JU-G;b&b1Q%7iq8_?Rs*B<}L&S
zk+m=|z5@Sm=7F~z7yJs}@Ze%e$8P|S=dxbRjC0s=Kf&8+-|?ux*4u&y=NPI<qLf1Y
z@a2~+@%ZwP?^J*<_bx=52AS+3Hv8wILH+G3)c<t)sUc3k_Et~rhUe&_+FA!wd#W&B
zi!CFRIpUv_r4UH#f6HxPc~~AoUJ|00DGJ`-o-M1c(h#RD>0~>>jZ57!I~I*P-v8g;
zJnR1mx?CDBn6hqt%609Ot{Jt_qVu6-ziU@NG&rZuoEeGp#F=^&67fCFn&a<FdCwWL
zy_^R+R^9G~<}u3a=PNn&MykAy5E9}4@`j;=jTeYr7dN5vzHeJ;A_(E%yL`dvY&{wZ
zk*kBEHo2GHBYxUiJp-@SIK+*K1SgIET1`S?^%ky-X3tDG&p(z%iMtH*wtc<@L+|Dz
z&I*Zmrfxt4R6ZLoJX}%#&|{<x%M*nF)$5{s^yJW)oUB;J*ZK=~!Y$f(eP{pSMq&V6
z4|Kpkm$YkP<|FdJ25q8D>X<_`J1>fjC+wnJ+a&UC%`*N<0|#`rQ9Kc2gx=#)go06G
zkPJOT3BD(6$uwR$fF~$jz-D3khELdF-9TanU=36b5F>Pe5#qEy{_tx>`y;{&e5E*A
z?t@X)>`JdnuXQyGTdL(#db#b+Gko>)51a8yn09#&qQl!%9r|9in#Q6#o&8<C5o<dC
zl2|6Bb@n+V=q%rhaM0EJfzI0>79wJV93m;49V3yl8~p;}<}_RCP#NoSn@r6<79{bs
zyGq=yT;4(s{J1NC@T9LqV|UyGy0et2Q+IH|5Qwr=|K#eIzap^ZskRLXZLNF^QFub4
zJ_@~x5F_umA9V!26YNV%u*7DUpHsVvAICsm4RIXd{v1BI%?n$UteP6Loz*4Z81lg0
zxI((+>AE9@K+5(-;?3`t!)iHN{yPK9x9GwW<+#BHxCO;hKUjaOD9v5kMTV}A#u6c<
z!6@DZQmH<!U*D@u*{*QA_#YJ(T8{gKiXE;(9ivxygb+`a=_PrXb1J>3tC@UaQNE<^
zyQxP~^1vmBEwW?sLX0r>rFckkjq}@(mw!V;#sdOdOP&0)(A+$p>6@+X1&h1(VMv$K
zi3oIc{wD1DdzUo9R92=YT<E!@lI2AZtzO@ce87RnA^t-<)`X1>J*!bx%JvLTp4aga
zq|AYw71^B)=x~R>zEP99{^LT8*Dgf%Dv_Jz32`q+dv#VJcc-fSPI@owH~wSzSMd>w
z;USVjhgn6*(h$~vSt`#5VvGTu(puAa=(grd6spx$9zBTml0*SBbq5(RyOHS%j9HwS
zJjAN~{L(}A{rMlWxOO;UFp}o2yJZyqZYVYwKdB%0hz5J|(((=*Dn`^h-wW&Hx72uG
zljN2?$-x(RQLW;gUsk%#1itDyZzDkvGva$@UV%;7{Thu>T$+=UQ$YXc9882id<E1p
z^q(U;|9@AxYCzB8h>|uQgw3d1tkuit;0RmEOQxq|9nooPF}*}~{Ua6Z3e8bJju#EZ
zIg9#`k65hS{k<2)K?|Mqq%C$Q^6=4vQGYL8RzX2QN$ro26h>E97g;)6sQqfGhUO)C
z><1Ny#<h_S4n0}9m+>>?zkid5MzzkUlO^(SIAb9cr)wXTu_dtnyFe69R%+*IkkHTX
zTi1_&2t=p6f9qu-R07xko0j*zODVHqw+>Vq2~sJ-nuDKk7jn(oP>hoxyoQ#b4G;1W
z0Wd&T%CNX0k=9b87#toGq!E_@n~0*(sx_V07>vDy)oMME-dfFt*O>mHHUZj*ULQs~
zJ2a4~m`oiFI_pl{p?3Ka*(aQ4E_8m{%WTQ!vSr`(jqnjAWQkrrr?&<oq`T*oArM%_
zthvv`8r;=L7j6rQ_8P}L>kmGz!2VSrciZ!IY&fV>`-%`eKTccHV1?g5LsWP~JNa;#
zd4BEE_7k7`+l7ix-wL7WWi)bUDjcwI{t3w?%{ApR?{MrfakzuUHmc9sbiyh={lR4?
z2l)dRV-12`cUoktNpza!-QF;S!BM`KuxV&*>yGTx((V=;sn{0TD3~3d%C0;ZolRNR
z=^&rhLy1TRvdln%?9K)fEDqstK*reFu+2IS;uHpQvwQ<k$1_nY2_8-k`r52Guw+1a
z(<n&cigfLi5DsL)Cc*+WBw2=ncHLy%NNgYbbYPum{mXvNNnk8-3Bqac*x`U~Batvs
z;<9(Lm`Z1TxoN5>J{}3~E(A~nM|k3VSMkY2s(8h0o7EfHHc~ivn9j_h16S-|@NGpC
zcCA9~lZIo~2=PfV6?tjb=Z8tXe9W;V-eFy)k#c)bl{YBwP840(;&j^+UvXF_oV0LA
zXB#MdW&C<wSt+rqxw#d8$!+*|1*Mlfoz4(a8NdE?EaORx-F~ZCTdK+n6^C`?#;_i(
z^nnWqKf%UjqvzFM;_YfJMiNU2(qaW(`4O%2p}s90H_bTu-XQ8z(E=0W>1-MvWol9w
z9C#%?%!`~Q?v{oCZuRgOe?A<J1O37oa3*4gXC{VcF=TtTezFve>bAb&oA$_QF1+??
zXTSiL$z{=d3=9|WA3^HR(V-`>=?VWT9|*#LySg^xLt%>(qJ|$&`5d8a=h-67mtIqF
z&Nn%psZM^#%%+&VK^<mqaXmAQQ%ASJvxCy%>c3QqUy|{z(S3;b`Jsj|q}IXhTHpBS
zR|j9ZNdLT!l&Agi)fPp#R-HU{1kv0i*NM-)-;Rr|j=k7q&KIqtwb;qEr7>Nx89%9K
zNvNvpXzS%nua21>ORn2W-!eCgpTivN*lH(ViA#x7Z!*s5Qq9B;hQLfXRj!{05pwY;
zp|!ISK-zR2Ogi<hL1gHZWxPsQx~~ggiLqr>AyiAVKvUM7mlO8o4Xp@d=x~n?V*%Tb
zr&Z*Z3!@bY)Je%ZgziE=t`h15NAdGbP~v9XUQpH~*(g-fDJlib+Ga9|^T5Xmp+UiP
zxuG*cIj0aPb{-n1dlK?!&77-Sjak`OxfGbm3;w825QLxd$y1K|HpPi0bjc&@tJiT6
zXM9<zHYzr}NTn+h<qvlp7^T)p)CrAMn$(cH%i3u?ys>nQWJ1SLpMwK7Y8LgSRT-yw
zcw=&x=xi0Dx!a8+viG70k7rBF0HzY8E(lfZIW4@`od^Y$X9Jc}N<6o5+;1^yUvS}%
zi`dwUL$QX>lo9}~W%F<NEP6bY-MyDE8{gP11YKAxg1WjNE!B#gt3?D1{ob~&xL*w{
z{Aw~K7%_g#IDEEv#dA^rwGHKFoWSQ~Ed|R$=Q{g%kW{N>KVAH~@=)N!I7wD#|GVO^
z4c3FFM=cYa11YcE%O<=blEfj64l)^(B(&+D8v@HbFco^KM^9HGbetVOTQBb$v=M!M
zoTU2?KbT5gRi?T5a&X$gZq)KoRd3)#U)V3E>ek(?W!eVztY|J?m-AttJv?r36Aew#
z%4l<Wwk>yEda|b>xqCQS&5nl1)qKJaLG_;h-A*<#J<iW<3;0i_v`_Nuu~$TcbJpO;
zGgp_hCt;KZcH@wZ%a2@Rh1!gHURRrp465JmjRk5u(*sV{ejrqFn25WJKE2|eZ(pQv
z4y38asX`i&f}P|0+c@D@ph38iZjPJU0EkrA5xX<~W-wJtwgypOJ%abByq9uh>2$Cy
zP?>TQ7iVX}dCXNFW=X>F)}QagR<h`cOLq}&Vg*A7?Ciq5%k~GTe-hX}nd)e`Mz-7F
z$AbfvYzx0(MAx>)^JpasM*y2t_jiZCSa%0sk6QR?a!AS$P#=an7JBixD%iZ{fAmxn
zX$W10tK8sMD))A9X}i5YGfEZ>Md~iP*3&-O%jqElZk+b%gL3ROvx1k9T-%y!n6C{S
zGs~5x%Xg~PLpXI?ZE+6dDXcE9dbBJXps-fyyi8n7-r>KRHXTcJ#s{Xn@^D#Pyv{~&
zjA?uNI|Lj}wp_}N?LCGJUr(D{#-+11J``~#CUJ#BIBSJAI6~OSDhzJjM~?@TB(>A4
zo-_vCB`#Zq9K03!BwjvhePz*SA0c6+fqO7p)Wc6Y+~J0?^q2ZS?vzRkX3cA7k6O4d
z4}5WWTr78piIxXGaHR25?A)wF0y(#nsC<|cboDlq=4c9pS_@~$BNj-_w}*bUS@t6Q
zkjxp?cJJqk3S<qHWMZ69ah2tDB8YwWOoLJr#bWCzb?RDud)~7<W3UM*w$01rd2jGF
z-wB9@HD>Ix*AKrlbV|!@CKwxoJyEx0AU|%ICG#4-l>!Qa?>;FKz`1Bk9t)iGHHv7$
zH%s<Dc5Q{R+*GN-nI>yn#?pR0Cx_#I_;Jh-;qLb0>-l!Z(fY^|>M`<}nE4!^&@th7
z`Y-i$sq|b!rB&4MuzjM1OT|?4S3?KAOR*XE>oFv@aaT6IX_l8|R0X45kIBu%C{u2Z
z;+o>s=e{z;HUi}PT_|2uV90kza9wn5J?Y5T<|bF9Pyx)+v{X+)FXQPJjF4^b`bLq$
z6T4l&)|4I~fng*X9@XifybQ0AYvF&O0^)S;Hj}S0!ldoV|8t>1F}tX#ta6O%%;Wrv
zfT9S94IBVJkAvfFfAq0$Y{^a#85<sBX!TkXlVdV&yIC&7OCWoRY&7sJc4xceVuOL>
z>0-eiQDM6nX8Q$ny)<1B6Hlv373E3uC6b9s+Nc!>HyR7eTD>Aet%1bku8^Y`)?%KL
z&7rEUk0IeLW{~m)6@Hbd6`Nc8)$8ypD&FR;L58Sg`n-&x2nmnNMI)@@Zod`Z_J%~2
zqpj_w9r{wGL%C^e#li*Hm-vmO1~g$O2hDGe2RMPg9w{vr4PYHweA;<<$|)}{{eE+r
z0i`BJobP^Sq$w;$j8Aqnq5XQ0xX`BVLAx>^+SJEFr!ANO7J2o>zVhG3Dugz~zq-l?
z0II4V-U}(~K3=ISz{G?(9B4RMs0?@<S8M<yI=r<6j8II7xRG;+z<}|-cSh^T%G_=a
zckWc>S5fB$YfeBymN#wdH9ixk;9GI7@VKe8w6Dc{@F7qVyqY7ysEDWzi9j(<#SXy|
zF-IC45V?!Z52uW|(6E;<PA+i&>()@G|HQ_hqz?HwAzReP<tH0C<Qkj;AU_1%rB?cU
z>);IaD%sp@-|bz&O0!~5>u{-ZIT#)YN>h_`I~Qj`<zk3c-Ys0930P_l*LL5ymW45F
z|E5hcBZXOt{Z*t7qDhSS(W5X)(N>pru*)&4cW*qJxT2;uqb;L!x8*-q;E<-zaFV+E
zoZjhh&Cz*0yhCm|Fu?tN>ew~!+IFH(gYptD&C#3(ycstteX_<rHg#`n817OcgEUV6
zY<TeVnB*8#85{nYdd%SvvNUudUe{Ztwqv7?&U~N(zz<g&8^<qKruE9s+E+fE>fFp-
z%aE=fDN_VVSHINgI8QP+7_Koc5GY;8*cn}}`i5y>NiTc!&^|LUws@DnRkJ1TNiZ_j
zg18OEFt-cqPE2C)^XIYH*nLIL@(gjSJDvQwSaEVdzRYJ-&(lO=y~mAKI2$y}%N5g+
z!`icJx`HSBaD&RRN23oWvn}Z-Wvf>;Bg<3!NUv}AmXM{GTgS@8$)~bp>+!KM5Zw{L
zfqXnNq!eG!0pLsl8t+tri>tGf>s}x2yW!kV)V}0x-sQFpr@|b`;pdw3q|x_y4w$T7
z6lHps`xiX=3G<h|kQK2y+$(ATTm9Cvx51`VPz}})Q&7WjY)nqtkJP(h$`+r(yA{EB
zr|he{x4p}YMHY7I*1(<1UTnQhfcIv9trFU6dJCToE)$v`p0ZrKG>h!pL#2O*gR}QE
zEzp7Mg>qv!=TIF(NO1W2$e#pup;h1R8pxaj6;zVJT@$=IA}SCrLVV<QH?b_@`7vO#
zdvGx4vdoZwxRlcLTCj3P2ly5&s?$>CX!O%$vJvYvuVd}JU)o)T-Z8;}3M{U}Y!6pQ
zre;x=R&1MGU`DAwN=!F5SAGy@Fayn3;v5>8WA;>O%=9u?ySXMvDAjW#*a*-Eaaj)3
zYg}+@NOmfe)1QpnNYdPT?b96dvW-1y62Tp+G&?c^QLXCk`{jpSOiCt0@f~X#RIZSH
z>jyhW^6MbGt^n^~TCMwuMZZ}5sC-^-pOlRNc!w`>Uy&%GUY+%S$?Zc4)nMx#W?QO1
zQ{D^?{8A9q_^w~Hg-qXJsVrR&Q1?CAEJ`fNW7(NEU|I?F;xt)+1N6&<nQ3;8D&>ke
z&)Hm+HHyfnywZMP8azfB=~(g<zRs;`*M{z|tyNp&3fE)K8Bs5YQb=R%@gl155R06H
zVzZ4);Zk3~F^$4kcPFYP)%vE{e*N0<o`6T~$9XCXS4Am@HU(-@YSq*!)Re7fvZH_-
zWReZpAk$grYaTCs5qS$!%RN1Y^^3`3sTw)_>J0$_2L^zS9tA?!WumR~yEapwt6iJk
zDyCgKd-};`P4lJQy?-}+GPrl%;@09H?aK9%@R^X&({e#@SeS^nxI$B`jq%ERQg&lv
zr<cpo!qWSkk3!{bkoddN6gJ0|aD<}I!m_~&<)Z3fL>syu2j(fELmF<)d-mgFqGjr>
zxL};2*7x@T;zP*}2wOTWH%%e^AV%!G0^-h?TxZZ)<wV`?jMzGHyccIJXJr$ac?T{O
zkN$%jE>aB&MFCFBiC~hcyIH0><pb!A&B`iYgN%iPP_^#fyB|(_L`8^u8tqGCe)&`{
zN}>p}PvDiQN!rfPLp!Imw`jc!G<RpaW4!}7j}lOToPYpT5Im9-zzqiv?Nt~v1AIp0
z1b&qJON)$05%y=u+;)pRrbnk<KR+$tqfgLSZRVnwtm9y&CSKYWcmyOk*1SBPv+tRY
zvz?saNRl;kJ>bmVb~=b(pWQ+NwJ9!=GeB!0bNd>K?sx{Lah=NPmkL2t#;Xs*op*3z
z>0DQtqxC)uRog3`ZCW80UMpOe=5=KT<*L?4GW-q91+ym)2hZ#TKLjrATJfN^`mtS_
zOeK8oVgp&c=9+d6jX(0Z)-E90PK-*O<?5{pH24%E;ZDs<^@OoLcYYF$@Md!UdJk+L
z(Oxe!v-KJ8&w+G7ES-#_Ln=5b7pXna+M~~~F&fJ40@G|$_<}`^Il(bu&NR<YbV<B&
z3ijvZK$`L*Avb&A#{4{uUEZnog#SE`Tuo=<s`|1fdBb0SB!T_iLmz8!-!d8Uys5MU
z%`nF6HH)b2CJ$Q%3`ZuJx6x%rNA)F^hkcQFc2-|=rpV&9(wvYH<hTpq7%h~S#Z9h~
zO%|H6cu^?)!tN1mR_yU6XIzHWCS!85)_DxtOOJ1k!5SaIJvA|^#_y@g0Jg5EA#F_E
zcOm=jY?%62FpIO4Ly=n+*n1kD9okC9I|_xS4ZEz6)sPcAXGwo|AZD1XnU(URo4kwe
zZFhh#Pvbs@i6;L;qj}C~_eTEb&~5_|JwRW8^O5ku^(3h%X3lJq0jZKf;=>%ni`Ot~
zJ-+>uIysBiFXMsYVvU8o&Cn(Tphlx=H^a}n^?opWjCMJ(&M0Xb$kJM!d%PdNDf%L^
zq~0kX?<v946bP+2%-PCOm>|Vg9YAKW%p{S%GywvKq+KK6s(jA9LO{5TJAqoG*-7(0
zp5FdH?7ekRn^E^J8fXhOpg?hJgL`p@7J`-HR=l`taSK%(io1KUqQNQd?i$=e&|ra(
zlYZxS&zvhWcg~%;f8Cp5@+M^a+IzpxyPmbzvi5%X-A}=LD&@Mwbp^`j9hh((oj%6>
zIC;@ajr%VE@W<b&5Y9X$CA^k%EcnUQOg%XHUC@4;n`WxfD&MeP>NUYLVMrFAK!x)5
zYiBWKKTBa}G3db;0)htcK2uRpjiSgA%6diWFT!D*7G;cb7^9P%oca>LBvUmgN@9Cx
zPj|fZ;*YaQNwp<9$2l3>vEt$+s3ROA3cnX9c}p?vyhlDIo<WKIV3cW_<pB;Jh4a(e
zl$epXpJxtVN>(povxgB}P}iPSqNaMIvRBdYwQnh|S>MsFc3tm^y<Wcduom%j;`FV$
zL#aEXY$%km=(E-4kJVVIAD1wZ5GGa)E-dQUB<ug%123MDa!7pSv43D=>dfXrUmx_1
zs^E2jN1b*9W|m#dTOXuK%jo}|SdO%;lP%N3oAE04<jJYvS`QeU!Hpxj1bK3@V*@I&
zcs!D?^_z=zb5=EsGn)b@zPi$Bp{J(y`bx}`jwsz<e%zWF&-ha@Q`vVMB&+V|2kwgH
z45jF6OTLt63R^W1-EB_A0T5JHN$~J~9)l>MLnwtS2jc~eO@0}4ULx3WD+w+(RqZ|+
zo4nH27B8t(0{v>ne)o<G_C=D0J&D~K<W9XjW%3&rO!ES}^Upp5;5ylN$KTd&($vQr
z+D0e|(5QKjjC_8kg%#un`rB5HJ7RRfHDM~rh9S|tpcb$#(}w10XYXlQ<)UzmqmO+N
zC+pE}Pi}XD@VW?M9$y!`gNyVn26P}jjlILhF9T7)iy)!iRTH+D=4a`z?37<St9UC6
z^)EltdBL8Uef*>=*3@)=;~B>b2A-xrI8(20c0N~Pj`iw0E@*HI)n5=t$Ma0Su?YsX
zKYUXtocSho(jrJFLS-#NI?`5D1E)qP%dcEt!J2ucPxI&2+0_pPJ!|uxj2=FEBQ7uh
zcQ1M#+uX{wJT%rUKhZScxl#!%`xWC8YWT)o|6MIa$M47e`IA<@ANR&UI+g<j&<+nl
zdXt#&MUW5UD!n3g#4NeJ&f$yBVyj!_Q7iUKARfT!!79t$*=luX!>_nrMlge+26q7I
z-|Fwf>dsg}%k1f#8V^{8)SGcYq*huVUMKj?ppDr-ryiqE6FUU}a`X|*Sh_4ugEGsM
znJa_eQVM@mg}1{NJFo9SxXz1};267AmPq48%xsGMGuGewmsBs`rCQ~Y&EDnj%-hDM
za^r!Ii8(HKoGlgtba_X0RDzz%Ke_RbJ_vu*`iTF^?C+n&(agCKI+N9AH>JH<*_uS!
zVH!`TaQiKm<+IIceT*jM=kfPC-S*Q}0|yTAqLcdhDML0BJd2ixf8Oe)+kY5t#iDNq
z1Z5nYi3k*0IXi8scnoj!@Wg0aM_F3tKX3|0w|%5oOH9b5rBvquxaa~Bn3p({l>PZj
z_WLwH4`xJutv%r1ce_~sribPF04T0}iia((O;YOnJ0B-}UNT`_;Bewd!~H70b4nH}
zdG|qOyrX-hs}vvb_2tRoR(n8eqA|lngVVY)W>=9?pG)MvAa-7+Skf4;-&R6_&Tng8
z0mS=$oThrV$Z0Q}<BTVAI~ptu)f(1YJoJC)NM5<_-NsqT)5lY^5H@#0%{-l0NN*V=
z`>E~)sf*Bo*a>BVx^pJ(*i&ipN?ZDP+ebO&+Q)Q3DlY;aIpVfWtB$84p_j88022GX
zzxH<z%iF56j6P>q%4Uy5IN+p~TGfyzD&qovYC2q2wjYU{sI`Y;48N2Qea>er;|e)b
zPwYM%K}-<E!n*p#H*dF-u}CtCx=keCEo6Rc=O`vMCNR2<5a48)ZKpF|CXPi6AVWVB
zXQn6I44kE@cl@-r&a4(&u`IIRF82ngC40w(X`Rd2A|?foNNukriHRsQQ#!F<9w}ZI
zIBA|xHMDj%UzViaer=k8GIk@-aqVL=>roj)rDScibyPxs*w@@X7}YYr#)_0c;|2-6
zvD9KP?8M^G|MQqdW)n_g$xN2(qAKxz=D**}%4f19+$#1>x|YCWXOCExKA%K~Ny}*+
zeHpSEyS3;n6Ote06l?s$vA+~HfA$NxeYP=t&<Y0Z+v?Ha;DwP({6$m_Wxw0s*TZLO
zka{gkBiQeJ^5=x`+Y(-Ri!5<(L|!l-%4`Ia;}^>agzOw?rV#JJB4pyw2LJd7<abV)
zQ*H8l><xGuyVYgr7UN3602Rix`sD>6w>)^v#WNAk_>oh^XsS#gaOBI^a=-9KO5LhC
zO2N76lr~IA4Zdpp+6Bo=(j7Ds4PWu|*Sg5KMqgR*^1T+qUgv%BAch5pP=O}fkTj%P
z4)V5izG6)$$4=I0yxyj{_xL_&<?G5*I3!#9#;!o0yEcgI@3w~0Q@lc^j$h;y()9E^
z<c}MFS;RJu9Y-rN#ihgz;1TuvNYIHd@@3r^u4eI>@3dG;?xshVGBVj7Ar0gS&b&oT
z(P4j>Zzgi@I0${VPQ6gjj(kq<PKYZM1)(01WUE`CB1m8Q{OQ|*t|-HK+pY6{$45y5
zNq4{d3ICFj(M99L`T&{6<B$>2v&<e<!>A{hfeo*1G1easfM@;RxOLvU4R1A<$sULG
zryf@tU9ER;c}|x9>TG985?@rh4MW^Z)VK@;aFfyyxjKofwe9}>SshaYYN|SFNvhP{
ziM=%$mKqO4<QdTg*zE+|96P;;DT7;<ED6o#uh2IcZ8#VRPkvjVxQG?CH(i*3@94)Q
zZy6N2qZi9EDA;0s(BE*4hX=Wm{4U$QSKz7_mqt7c%622bv%hhE@`J0KxXmcCJRvj*
z;$C@vAHlsI5D$(Ms^(M()oaM#7L~Hpq#Y5mzPZe4KbyYwyY}Km<hw%Znz+omm%Hn^
zp^>3T-i!&y^4VVZz|S*mMW*V@B`g{04l{|GDls-yi5XHJgG5=p%sqt{CTMvvcbl_r
zHl-ZbW#LnAw&4PqI3`bt!(4hs^N88+TUrycp;P)sR-v>DlXoulmS@7BJCjd|2-S6+
zLaBTL=9*=7U7~did7-!Kaa9}27lH!5!pL?F@$>mw=HNDc%)m8%wgdv{HRH&vBx@SB
z`I}*8mT;vp@v9AK`&HVSIytilo*KR^AD2N7M>?h)jO#j$$;sW)XXQS@!*$!a;R(|b
z&9rH#@$Xz*_IvK#NBDD!!O@cE-9O7LSjX7uMAj>c5<igVLVpQp*$qZfYI0A}K1Sz$
zpF`PjpR!NhUJZBJ_R2)JSp=f+ssvLtt)*fTPSbFY5A+$5)-LpgE%$Pmjd|NAd698?
zDO7YO+_Q3ouENyF(F*DFJsF6&F_rzu0{w<CAI5#?*s$?=zX+_6l8`W127%-nR#_Kl
zvUlz6&0CVSu<PwS;y|Y?Uj$!qrhBekJg=H{@UJ^7FrfSwV0EN<Bhxx_<$hVr?FdxP
zRX;0S!0soxXX<><zK2Tta|;$~LV5i?&WVH>8#c=iezs6JE0aHNZ@mad7SX0A3Lqk@
zjr{BkuHGJO{21eCv)DJ0mF>C?6~ZAYEG*F0nvS5zV&N6f{%GolsJ)@vP6YX=h~&we
zNCj*t7;S&~W=WQz{*9h3IvJ&ljz@^>@9&p1`lze#@P>}-6ElxyB<4>$?#U3wB&S(h
z(3OU8Nt9m9FB7Vd67KU^MVv@jrt^s15-TBgt!shxIjmEKURQDxd*2I%lmt}L(cZt-
ziA<!ibSo7IopIYa4WFdK1&HN_4_(g}7Z+(Ea4&1ns0auocfg2<w#idy%-iPTEkSyQ
z3!LkZ8Ju&A!{3L72!jSOu77gL6lay@&>!3v!Ct|z_kY_^n@|yJkR1)L26yjJBQ9on
zX0)XmMO?bI04JBOK12%JsqpJlnKLuME~*634KP|L+}j+V*8e#67^~-6RXAUrL62K5
zW-}^rim%ywdqFOSH&_AY^xSrJ;4pqu1hLd7)WQz5H}Utn^e}40wz)?0OXHpeixylk
z!cLnz(QJDKrGCb)8xr@GZ+%h0N=@fpdAML)r)IbywCfu#flgSwm_<mx$Lr&&)x$3*
z2P;H*j$jEILP*FA=c#^)dwA7FV^n?V%ErsksrL~f%K0k$Qvt;~2w?qeGc%*jl5xo2
z3TZmz&7v>J9q78~S(&Gu-@!=;Jk<|v$``W+kn>t7aQnT&3eqnY!D{&G?m`G7ZIy%`
z{hqtJEgF*-M<D2#2M0$52J=)$cvXgk5rb4zmx<vDXIqcjmk@b@fx!r0go+i}@6%t?
z@#TU3elgl+8hlVc*0q=8fy*SiH>EC&k3RsAVx-UCwyFX?Vn572c6TSV&r&`%qM?H)
zsX~vl+UR)v28rd*b;E_DorP05<PGNJidN6}%uz&R1uInR)@0oi^9{%fygE+nC(nKM
zLgl#~-=G@kDpn`364uB7#gEznhEML=W7!WjC?l}Sj;~aq(eo_TR$%3cONSD1u2H#A
zo&zR}W%OP=qdDS#lxn!J-T)iD*l>t{T|%MC;bZ05JT~>O(YKgJH^A#>+9JQ<3SCq%
zzx{$t1T?BBSoD`S7@8<<37E38ka)JDl_<b(i<3&Ug_xG|K2_*5VEhl{X>6s-Py?%g
z1HStQ)mNpUIqO}6q{;?SYTddw%v*rgx!Z?pY|Rx1g}TO1IhQZiVe9M;zNoy4OXzm0
z1Tp!6rCmV-E|=*Wz__1W46AQSVbYTaM|<rCHRsx!d*HQ{LgyP?!xW?-mP^J9TcRZN
zX!y*A=xdFYwozTowW3n`scNV~iF8)t^0VD;4{5_)eRkm<kN1W}ApMFF|6No#HjEts
z{as$j-CrO0_4egl=xk?p#b_uD)cj29)_O%!YkQ#HV@XuUru0v9DT?dqLYL`REQ}5L
zQ=2(VDA&zivNDr>Ag)JUy;9&Ua%>ZB4Mo=6q(PHem6;`dk0E|%Y*c91Ps?@YvPmpJ
zvM~oRx{Kh7PRtKkBV&%66;8>k2TCV%TV1#bZYRm2k!m>jw6HI$iDvvWCv-*4Q7)#5
zT#yY7QKn6(Wd=iQ8-7F3kiFv{bj+~!2pZR#5Qn8EtDR{uVazjT6Q3(^G&KhYw-6m=
zl3FDdb&FQ7$>;O>tRlI&mYeo0B+|A_^d5OXc7}~yzB?Gx#1(Q{GEmd^Uu#mYYMSQ(
zZaA=rULT=v-?Ud>Ze{f`;mzPvk_f}VTKqUae+w^+ECovHm|P(5f#{F#h@VI3R0T`i
z-0FL(@6jF!C*S*`FBp@WeY`&qI9Ia(1=GsgtpNbQt8X|SHz&;x0<NFE&8PP2&nXsj
z^7|ASawv*$ayko(z@6P<s+>=%>C@fr$y0%q!^p({6uMq6j)3pGb+32UquJW3&a#e7
zYIbuw;oH1zH6K7ncc!y=`=9+8k9EdU1JSAPQtVB{^$*!a7W8kx_$2b2hRthV1Uzcy
zoFxyQ!$(YNx?h8Tr|rP3&to>EjMJ}F;kO*FLw+;+Xm@mKnDZTCBt4+5(<u1rWq!r5
z_a|lbgFz$t%SR)kELuGSiNaee;WgNOPcyQ!%m@OX`Za8R!gMQ}C?J4hyuj8xSg}=e
zOW~#w-Bf^aH+MXz;(5QJJVXNox*fMh=lis~h)QMg>zu!ZaWbEc=}KL}^SzY=+!h<U
z8arsBP6b2z@s50><@_O8tygJ6cvI79y?b=e8*OC+5L*;Gb*ypLX}uqA`WkS1rpTG5
z{glg5A$Nyq#KE=)&Q8!>ufqJu*=ZB-W=K)g?CpEEp)u4PTv4V#a+^xg{^{t-RNyRa
z<wK0@^uTxbRnpA!IyK*D{a;?W$ai4+@RkuNibYOTa2jyB%5c^x2A)vN*WCKrY{{yU
zyFDXCeS=`X?f=WYn*rpBp)67`i)X5pfx1?*yF4!MVg=Iv=voZEzmOk8U1;4^eB1vi
z#hol>B*%?UW@lj_6&665*K$9f@1gOHUO*-0yMl7uWZ455?R1B|k2%EgqrNA)6s
zPjN8qV4IkG^7bzwVomHA{K1->$*E}eqONjJ_cl7J!BMGKpvt#V4H<_186w);s&B1O
zD-<v@^7`vY6h~&Wi$lrn#8rF4^-!zS%@`fmkr}4u#lv0qpD~_(bqenGdbKOM^X^>P
zYK=a)aq!-&Q<{qxRs>$9U`)Nw#ZjIV<5inZbK7VrRcbY{>izGNZ&KG!?SDd>fcF7$
zo7r2(FDRJsM1Z{B$@}4BCS<o&v2no(x05O^v-f(3eWEmoIxqCcz>OLEwCIY6)tiKf
zgYB0pf%h@<(HVDl7wd9yl@^G31JyvJUlkqu4dN=VZ<SIKGp#)BzpneKEq6&QIb&Kv
z{swtX7F7R(MTvLEc~z|c+Q+z{!y6_A9)nO?E~Fn@v|j1AbfgL%XR^ka++Stk-|s%C
zX%84`Z*A}@swzHditv+NqF&su_6xiVUjdIdj^6iGr#%RMJnytHK6<@BM$m3A?%%Nb
z#%|RSO(-bc6S(L|hG*Qen*kXVU>#~}1$A=)m3Z4w8@JygvUat5$NcV=D1amHl(8`)
zE8F-8XC{Ah`bUE0Xg%ud{t^VU{{fK7S*uUxb0bw<A?e&ytdseC@qP%I)+6;*tF$?G
zpl{?(c*w`{IluC`#TPm3WL?5u-`?ewOh31K0mn0w$O%<+6?*7lKi;pWTq^^V^PzkY
zeByj7gd=M+I+#tVy!E0IMrrGG<}DVcR+aS^PQop5iZnM+dH*53LEblw+L(|qL!n&q
zAl)FT%tKDhq2kxhmmD?~9oMqII$tJ!9=~%X@OnAC?c-&n*MM<x4Il^jf|=JOl>xw|
zEDy@pM6Wj-CB|k<`gC&%C+cVC!fZf>uO+T+WkKC5ZC}A+f5zvu$Fqs*LpLtNk}|z-
zVxXX(?%Qec%%4gE)Q0cXNSK!<8orc&#m1kK*`viL9uAVn#*7J0&?++^py?sY-uz0Q
zyY%n&D4|bFEAI%1>c1+Lyt(VBV0Vy$Lzr25iA77YMV$(Y?&9D!%Tb;9P#MU-n~L2w
zsaJwc<m(?ZK;UBc?D6w}o_lGqVu4LCq;#r)2L^KJ_=bG?piX=5Ri;zKM+#o~gacS9
z0FPW1eXCmF!HEX|{GA!H8yFg5AG=<?!X8|{C|Jl!TLHS6r%DFiQA2qk<+*?UtVM(f
z2KZ{k^B94aX9v@xpIUP?$Uw`@-AW7gLK;U4)BLQP>CbZ0zoxQ5Y%cC&BNCqup1Lbt
zR?x$Di;Woo67Ctznc;g6070c0<<%_5{%#5JxTLdLtfTea1tVD%Bm<*24D^KzlD5?$
zTcnAyX-DH>Tcn3WAAfNHi=IA1*iJQhA-LUyo8MS1LNtgG@3s?^N2eo1c%JaQ+otK6
z;?zd9mR3^#Tb-pe@@N3y6M#k3Ih{v>wfN7xA&=|X^(jv*B|R~7G`AomA6HvB6Oo^+
zsK}$OOiSoGt@r-7`J|@&cbX2Sqw@_(XnlR3ksik30g+v7ZNfY44E$goIs54_w8^#i
zqN{FLgF&Ag1d3m#!+i9f&>nnzecz2a1quKhBuM&vm(2fjV5W&27lzjJysl^{96-k#
zQ<uw6A8mZiokldOyBRMpdRjla7WvKY_L)O~$0;WMAuL&s4<rTjgDh;u91@I7)^`^D
znA)7)ch4!J_#WDKd~!rzh^Xqp9BYX7&_F(RRdNm07N1p&BihuozCHLjwv`$VD_3#U
za5p+wrXWgS_zn6uwX%(U3#5w_KESFoi&@HR>I}(*s@rnamUF|YY|F*9k3I~K>$`bn
z#e~$?U}GoHvj6GMkIs^0lJ0h>uo-0kkd~2B&yr_1mo-Uh$r1rWU@X+1DLK$W{3g4L
zlcy=#0AgXx<03^bXPFtHG4Az)zKFfxiMg@&r;fXU-tu-z87(zkal2#WTpEV%y-3B=
ztwuKGf9Mqhgz#-2B#uWYWn~(z4R9eMlvKGn_@Ul4C4c^Km@WEeu+l^R5-rXey1F8}
z^W#}4h#9A*A_@}7Wc5ieA6A^$^7-qyejo5&y30A|i`HVAN!b1o&D@_~o6qB$lK67!
zRcxR+%=0fJp>ukCIwhlduEph6KKH0Vi2mube`5N&w~Sw^Lq`vEN>c{9w@nW87Uw_W
z1Az>TT|Op>n}eoLnZ~?g%|KhIyr<BiC5hM<Gj7RQFAD1XL@6%BC=@x{74|H@vocyA
zS3N3O-*`e)NoLzBuC`VW4IyupBJar})XH@R|3jCAt{Fsgtb|1&e4myo^$nRpzn6q+
zLtx(<+76`Gb!lE^61mntd|~wp)2NwnTG5uA!cK^ht`T9<c)f;{I(w)YCkvaZkn$Og
zF{~%YODtY2>`jBCN1HMP6W{82#&bj0kIS69Px};&;yhi7PTj+obXFoi;Z#VX9(_88
zaj#$yV{_wf#?e6!^aEjx(GA{4h4J0U%gyd!5GYl}zxL{i5s#j^HzTcm?OkY_F^DVO
z-xBO3TOf8%+0`ifxlle@Z&`i4hi~bMhV_0h(`Gu;K-g<?%^t1%4}W^Xq1x~!tJ8?r
zVDY>CO{qI&`3IN{sW_^+dgbRV3vj@8zd)8wIhvSIMB70{F^g_~!(Xq8kH_)DS7Q(`
zcWLzLW4Pb{Nv-iSdYgOa+7>UEShF|0f%8#OFD9g6LU42uj6=4kBrc9oT364G)`1!d
zZEk}iY_Zigeo4(Jr+J{K%}E5(uF3>ej8k0OOR{r!deBC#PPrld6*9+}3dU77z40rJ
zeHkLhE3Qk?vRfF`huQCT9)-Gs>!zI~n>M9-736<#(|SZTc%Z(jyMj@DHyA*<0KPwg
zg0@Dw=rZWgn_`8l*M03YtghgLaFFXivDFQ%I+7B`theDoEq$1ndZ?>Lq)ybULO{xy
z;8vcs&O4_*o*J=$z@em~xwJxSum%fo)9M2OTwFpzK!}@d(2$8x-+rUJaq5nF1z-2M
z)#M?9UL;ZYoWkq7q}@jP0xRd{&+wv=6P0>;IUOqH#OUFTTEXsu(c}y^yP3INz_NXl
zZF1o%zZ^Qdf2>W!HM#J9hX;vkFt#nM5qQ+@WJ`X4yseANX!o9S3l0gHmRHR*HBRI8
zska@D2S@m+iAU0qtoj|(+pcn_kB%`%&WMjMTHoIs;6YX1PfrsFt!QZwIsK=jr$&*?
z<pDW54D<p!{)5lU70hlu5KiTi0UZwLo96o(Y@&ptM~THrH#KnwD~iC_Y{%w}@$?>w
zunq=U#f`goQLnIyfMC-N!U9PW$nLVpTyeqyL!2gq3Oi4(aX<#(d8oUvZ1*N%%<IK^
z$1J88velb#6<`LCY?;{q2LTEeop}o@xi-R`!UG$>@oI`NURF2#pD?H9({$hy4Cd??
zn21@&#cY3SROWp`r!+Z;c)@%{v-u@}d?@IL)?+>!t-i6S$6{)q*n1qIa%Mu#`r@xH
zJh0dHKQ+(%@8P74r^V^=4|AoJ=r(jukI4~g0DSM_D*vN>5$xXWXuw+$Iy3ZRQa95M
zu4xU?g*^|&!z=0{!0c~;=ia)r0f649W}_rD*x6=KMf|3;li+@h-5<Upm6+vyvAC1R
zp63<FnAiDo_IbO9?A<L>-@5_mYu*o(ZXk=3KZ+*)_lfh-G-yQL$%9aZ+o6a5@Y4fj
zIyCm1*LRn(5jSa9_`HFcy0&jtkz-I|*U%8pUR7}5wbAB!*TqlubEjJZ6XhO^Pm8v=
zi>m9Q(x<zd}&p4HPVTokcKwpVUau(s1l(K4ERX>u(~7woU#5vY3L=l?Miqyu(yQ
z{)gYV64Do``cHLaYBL8js6AypLcUpw0V<2iG|Ed&!)Ui1>wQ{*ye!h6b#P{EfWh8I
za|)m1in^p*<<4b6WwWr>S$oHj@q}>G^yZH264tqhs!&6FDb#qW6efBM)SdrVNWAT)
zra@jOV9KE~<&BwDblTkP?B&&SOam<Xf4y!!%uxknHH;b^KW&B+g}N;|zi8I&o<%y;
z^oe~){Tc2po7zb#+Oqo6>sw{{Dx91fKdJxA71;Sb9k1`!*4-VNM%*FFUHn(b@{1QF
zm@?m-U^a}kgU)*S{jE<B4d<{Q_Z2PWOD2VAo3R`(;uZO!yj_V}m$ndz;kVSN2N)3X
z35LN+7p?LdKo#`bZDeYHjDDY5@W{k1Ek+(erL%uk0jc7v6O;Lbrwu>yz{MjeIJ?i7
zsZ9iwl!c%u%hr#0NtDoYDGtkvicRR`Rf7lsV3adqgYXRibtia83`_Ez5Cir<RlRt#
z7NB>0r{2~zi$VjCr+X$K%t;KS*)aexJqWvKT5)P#ae6~Oo${a<h=p;7x~;LRjln~}
zZ#Z~Ohs`*EY%}A^xy@5yYJ5zL$I*vxBR}J4ZG6y#q#<r6qyT`VG)%BPyE;2#?EFdF
z2X<<X$E!w?!bn^|u{;R$@Cg8bQH;<5ve_sjG3gLqFHw2APl5!28u`0IL^#L@U2z;o
zcd)$c_68;u;9_q^@$ad67ZuziCMLZqm_1|3GFpHElTEjn9lmE$2kpZ^{baekQ{S0u
zBhSh}EOH95*-ek8Wx>i$WsfiHUv<T~bZh!x)D&X%?dFX{Mz;R|+nO6}h8sXRVVGqj
zYArFf)nN141O1i$@3wVFM3ewSL<B`1VVf>~@4x=AO)uR7Z1@R#=i~1)9|bx6YdyeY
zL+(=$rO|*qIvpdrJu1xmTh5BDJTL&@(inq5XPy7aib37aKjJvLJo#5?&_m2jcY1Gg
zMJN34>$Oj-u}3E0RXXMj<|p=E7~NRSlA&?q@2Go`C**)}++`=BCxD<YOd>kBrOu!`
zXv<(rjgD?!o^?h<=!~Gy<(nYB$Ri9bwIXzj3&>V+f-L=$iNu`+H9*Vvls>!J&ffl3
z?B5PS@gJq(8a3u4;H|8}zYBom^}j_9(E9!_0YHrqV|Dg_`y<eAVHI{E2sYZU7Hoze
zv}W*DSm=L~<h2^lR+o#En}Swh%0Ng(&cWb5_LY!Pe>Q2V*D-|DM(cxH7rs5)rwtNM
zZ0D4?R}1Ru>QEB@R>^;F+TILNcH!VTQO?9(Ej4O0tYE^1U$h)#lgc)B<;8(XhN{HO
zk<tmuoT<yJmqDYld$kpE_F#ET7sk=P34jJTe%o!I_Pe|Zm(-`2fp#Hzg~{|*`+uRU
z`C~+YB4MR&AJOb=jdaEh4gwYt&e7P}RK|cNDs1`ZG)=f2EQ0&KQeT?4pU$Zl>rTAH
zR56_&O$3l{jlaA8F-4VimE@0FJboW|C*3n!ejkhZ5BJUKv$j-5slXiWE9te69GR<E
z<ULGA$Gbz&`wH$NC>6~QUlL{0pwC-%cQDsSj}kV7xuV2f)73B}1PwMIsEv3*sA~@w
zM)}i7*W@(XK8=lWk!4a#9g(1!zYg+<J<r|4QPT9Vl&!Iv_ds?u`GWbZOF~rK=Wr0y
z45O7Gko#N+$V=nsw%16=IK4;~CzPwP&aF<ET^8gn7Xu<JA=qPa5o_qyn_Nr_InpOI
zviE6JXjNzdhal^(aUYT4^o~=$6&sz>CfV&3RRR&oTE8TWdGD$d&ZXWx-+_NQ!h#91
z_P+e(=CO^&AYnb7!s}^TRjj0*gGYL8k=B?Xo3IO0HV0AfKkPJuv}pJjXiJ(dD}E8V
z`BuplKF-DX3O*L#EgdZH;oBD8!yRWGn(MwEV%cxXa!dP~Dug2}aaS;GSP3zY(5Q{D
zIs>rF>7MztzV;8v-TttcEO3Su+AfPZK?nqn4lW91ig`(c-03TvDUsib^a*4VhLkXp
z0Od#AM3vliY5wd-R;ezNnWpcHUKr1a;T{}B0!P2+%+Md-I&}*`!c>@5tc&7=V=klz
zA<8C!AL%O&s-P+q{JVsNuKxErJz}!TBcb1b?3H@Fm^uT&cZoElty)nxS7P*)DN_}B
z$k<AgYa^pb9#fm`Vz9~U)6EB2cN-9}m!P8JY{xoeggV9+p+$#ALMrFxs5Up*<xwuS
z-(BroK3)_g7?@N@SVvH5W%xrQ`eDZ}343(HPT5J><jHx})QTmtu({H5N7+9d=>)4*
z`ph|8I5LsxnP)!T&zT~`0_c)M8cJnBwV}YpATw}Id%i|(*V4?s#+3(9aDtKq^E_U<
zM(U;QLR=NvY}`%1_%NoJR=JipUp3*NIW8zzIfDZ@(-#KHueeSQ!5lhl25MZ{ehJYc
zBMa^SQ(}9M+^wytW4~ARs7YTA?ALV*+!fm~A0dRIQ!a0%#Pqg)`|oC)Y+nF9xJ1aV
zIN?`djfGc<aJsPmP%`|Fql%q;V<5Eqx9z*FxFr|}lsEOr|9mB<$?E2;^Dw5}u*j%t
zw54WScy~)B81@BEvE6vr@XUnSc<4g+{`XNKCdC|rTk6K__DgnPo4>jBOt*YQyRojo
zX-rtgF|LHzS&`I>n5|v3>K)Bz(AE(ASXb4^^6sK3wWPFd`QFq<i{!ZFbKnNsX62m7
z(UOaswb9Uve-l=hLo4Eqdv>;(pZFt0Lh7uC7(RI7CtS%Ny{PS;{z5)ep&+ET;&OnP
z&~>Yh7B!T|Et!1L8;kGvc;`i*RCf$>)Fv<z)k3wLkW{8f(1abt9^Tx>FXrWJdbHY!
zoWwlunUs|0zVUKaIa)O|uryVYu-{&9{ekZ#6n$TwnES0U*b6rcM=xIU7s^=u(y0b*
zax3Sv&>U48T|Im@(bm=cjNq87%D+~%%-Z+OOt$1q8Rp*hLQ#R1qsi2R(IiB;H2gh-
zZ@lZquc;=}8Ae9MjWTu-)+W;@15Y+CuMeU#lgp-j#=`mQxiW6pAX=b`G~xH|Ztu1n
zI&3!s)&~P(i;VI-1|P*dEBf6R%HQ-e)G%b_d}=;Li0OXJF~(IWWQC+;sxh~jKit(z
zCG3o!NrgOq(~h<AayOc>LDkQJ`KL<630YWw@&a+Lc%*BM{4Tmmv^efFGh2IA%fzQ!
zze9idQ@*0OMJXzKXVRFCk;pcv^l!<^yIy*8yrbUar9RT5@)kjc{IJ|XXL0`7H7Dx8
zy!xZUzV6kRh1xcEVL!qMf8xJ<X%}4-B)L{B?!ML-DPk2G8FPE2N>$?>7h()rZ7j0v
z8Lgcq5lgtu3f!H!JJV0e;CMDCdAc0eUQ$9r=;L|ZQDl&G6{0YjXD_>z_zhRa@3BtY
z=ewC*{B6A<drO-Lj&xGK?NDn5Pkk^v?9fP_x9zG`X_CPYBC@H{_5s_4q*O{X-7!=t
zP0ENpHsf4T?t`NLqy^ysCr7y8N)E{Y=X_GtD<hB!3F+Qc;eNBv9r%CK8hxG1))5rZ
z<lE_H=>9HPrA@k>p;{5^sY^UCQeIuKoGQv>3>s!km}$&P-ih8nvM5D*=$fb_xv*Y9
zn4Mxv*5?D4N|up>g&m)rNQ#(8E$sSNoW(ry8}&Av1aG46WkE$=O1GQuo0UKe(sLS4
zULG2&lhJeVTQ_LZWYpcn;^)fNTfHM!Wv7m{{;%=P%tPt&KpfR5n**CCYRzr|-={W7
zT<d#muMr8=UQbg~BQq~Y5Z}5t6Z#b*h4SSgAwKXddeAOmRWs8=2^ikBiHT)~&gN;D
zaj7f&(cJ7)+9Dy9)D}~NAH9_yi^)y*-oDGZ^nSYcpk43Q%+n_|M6WCsm3tGpCDCge
z&S=r2GIA<G;4pmEx*t9`b(VSO80a{p>D%gDr>gV}J(#63d&i&o5&d`ZFcFG3*=;vE
zTF-^qiu0hYuaOnhqC#Qa#*0Q1-w1-DQu8?n8)~bt-qNuP>V8NU7n|{-mh|y=RUAFc
z9{_61Y~846M1-_}r|E%FqDqP$!(S=rJ3V&{a_Ms;o%M1zE6S=xcA2I_biS%!yNgoy
zk~>Cdh^o7<3Y#NCR!r*ITLSMRK#oI;dF%UKQp=V?Kk5<XYEKOt^IK|5RJxfKg$aKk
zWEhceuwGC|2IzNHp-kZZs-T~$DL`OXQNGaDk<=!8%IZB;$N1*^l{c=&FYN33&~N-3
zqtA2u_K655@JM4$=d_K^!%GD4&^mf*nh?}LEL>J|xBvUhR4XN#CC@N{F{U@PO+KnJ
zN(td%nv3*@L33#XJA_}W&}(0m>f8=V%npXw<n`$<+*?>?bqur{QV>cQrbj)Vo6CUL
z?<H1nv3W8Aze$h{(Qm}bcEdPGR<2#jTi(;KykwIr7FN@B7ky;t$*gfaw%BqVse74p
zyut!>J`_EfuI<U0F$P;lAMb%JcH#&my>Cmok75tWP-9N>&8+qvj`#42i00h{U4MCW
z$(2Kc^*}AWayP!U+4ESWar>{BCh2`~TyJFGxiORvUS%yx$7G{ty(Z&T7oy4VzV-<`
zZQA1!>l2BWjEl^b&qS2XzNS6@`+|O#A3uIO4q4D_R1gzRFkJ__VFj_jpXn(?<hZsu
znoAqHD#xK&TZQ$>9yW$Q3)7g|l^gJJwO`60S=%kH=pU(??rxCPASyGr8hL>VIb5-d
zmamrL4_6pJE>MZqESh3Xw|W&T=>yu!8Y0BPJrT9H{BCMv^ElXxj;A1+lgrM<uPkm-
zKExnQscfT4B{iNrOLR%M&zEV=(9qe#E-n|Qu{-*!m_fZjvXyZD7gelm{z0h~Aw{~N
z90;eVAWY#xmy3|XP1Egq=iBZwJrJ))tgO%Tn(im-RTZhf4d?1>;I~Pukp3i02rR^J
zG==&_uTbwlt=0rFXe*pqFe9*K+s5j;9aX$RAXebk&QEjYe^JC&*(zaoBV(Fd9i0L0
z8owf@BCwO3$2_zDHb_h|8Wv$uz9IX(`sTyqQFV;#C>yEki?ss>nbAr+!M2?75~r_4
zn1Z&$`?CkmxoPh*__yb1?p82gWMI9?N5wZMUX$_8V)q*R-1<86UW;tvJ&#FnpFXea
z(|k+keb5$RV76Vt$O*<ek4j?I8In?;)dyd4%4~<;@?vLY6gopB%AeuZHdh-2zV6qb
z_YRljws#%c$8Emc^_r}98*`=u^KyGgFL}s*yMOdpe|8^iQ@8c<aAaiFNTY#!X`(4Y
z8&qcP_Utmi&D%?H;Y)y7sT%o;j^5=&M_04b)5`!evGdS&Ve2asN0LWbhwS~UMjKT9
z>$5wDqa^ZwwcDsKg1vS{m-StzwYR8;7YveXAPu~`pN7cH0WOTLVcoS{u+XqcWRk?*
zhTN>~OU&QFhMYdTYJwQCW^Ll~O5+#nM-SN~bkvCAGTr;`;>C8-h(GMKNZX=P_#M^d
z_Av9Ns828RcA%|ux%JU(b=Mc>B9F6$jSBWrPKC^}lLvqQqlu1eGjH=;yvy_1{((0z
z&<5|Uxfk`Z4rU%6`_O3KCSs~T!Q6vty}eb=y2s9*YxXD!Pq##m7evj38-Q{VT<~PE
zW$U)-0^=ew^x3b!+B}X+b0~XX)2T6)BjG*5iP7r?543MaL%Qi;09)Dl!DRqT7pajS
z8c3MrHxO-aETVr6Qx#}pqws*%5sVR?&f-S!2F|X<ono{e;A8n&^oWR!_jImxWfsx4
zqaJPEi+PVLF3I{+l9lY<jw@~oGvR<cGTb6@A$j*WZl)E~3>clG7Z;1%YAuI_=%*U`
zne2p)e8WM(k4Glx<xwXlGVNbfjcS^%b)7-NV=BI$2X$8~9&S_F@4&Zy=YM>Nb;bxJ
zE!*la`1WpvHiQ2x*z3j@mYuK>rOl!hq2`Kl*6|N(eR*R3Gfd%km8FQXBd*fwXkF%}
zSO+;fP_y?rTpGF_5AnQS4xmC{f!CswGM+(QW;Sm}>SB>;4@yn6?tcbO`8l>z;BZcy
zO5L#>A0``+8PSnhmekU+a-@Lem3&qg)(6h5pOdk2iJGJGz0a1{oP&lR{jiR;Dkz9>
z!LU3Dp|y7}17=x|!oKG}*Nm;OeATK`)?52DT*UUy-6Z2__=44kDf@!5?7u|T?*%Ai
z^NVXIb-X!x;wOesPXI~L^>Bp`y#=ac_xWm|US$lIa5oew1XWLl2>GTSmj!)m!_XTk
z>|CQpzv5po0*SJ$#9OXxQbW|)Cz0(PKgZaj6sQ9X;zwBJpAK0m7)!&F5bMHhBl42A
z3IoY;M3vWf7-qq{m4UJ6kSN9DAG6T8OU29>eey^|gw^i~+U)hZ1uI{b7;rk6k8T($
ziJWd+F>}65_;5ewXQ*F33McX#a$V1dN^k-vr^#!}y+o|V(-+rT)#-^!tNiqJ;a*xt
zahK&=y+2g<TAf<Ec24{2^0re=EIt~cIWRnWML$z;--^@A=q^6f+`^kAsJ*8xp0R2~
z-;ta5BDTHV^F>-g(aw{*Xnteo^wI^NDG<yE`pXKIpQJ<JDPYoa)8~5%5=oOd@>JgN
zxGX_Kr39LsL(hD18XcT;PXc+FL~+n=jRyQ?_LM~C%{O2tXE)A}NOxN*$JJlkqW7!M
zjaA+4ox}bMp#P6agVibwLLh@GKXN&uR8kl@PwZ*zt>;!+1I|f5&pPuR$7~tU!cMb2
zyZX1zqKPUoz6B|bGWb_aR`+p(ZXCKCr-;lEK;RQ%&HJ-$9<-({ah!ausv=WiBp=;s
z2LCEUE{SW}$TsmV7g0>?S!%=*^=Z?4LW;}-Nw+_2Jq!Ss9}|r<I^~jbpk=`{P_5xh
z*J>U%+m7di6NzNhvy5ZpgdQ#MGQf}aHP2`UP#yI?GsqU6sUn9`k<%EG|7t9ktl@S^
zYw&APFJEVfdEp0M$!Ff)J}-ARw-)1kk5=%1u{r&t+yE(;3rFl%M3#^nuH^%%$JARx
z53%2Gq*9^?9zbNfr?$dKl0qJ~R*JHb5k{}j(CawEG@T$qxgE6Jo+c350(C=`%2C_%
z)_2$hqnoLet*@78Awu;FY(S@&$4Pv~>Bcb)R;30Eqqca@2cX@sgb=yDPp%;$8;Vu5
z5Kv>90T7{J@h#xbd@Bv-n(l#eRf5?+Bp$#<@bblfkodUw4(KqMHPTw8Ry>NL#86cT
zlwJp%{HRB2pRV?K$D6xv22asrHZin+5k9tO9=YKXve}?ewkIZXVV9#GD<oCFo6yn&
zx$;M(?T$%kcuOASam&F*Y=w&YL)=Y#is%4e+C!P^qxpH5uIKji9|pYppb=oH*W%i8
z(b7X17ayNZM%XNWhMzfUR|ni6wROQi-X47zq{QXBck(ke?BS+2VM%F4h?0##Nmen5
ztAP|7n`+iACzP%`s0LdaM7~@v>ZtWK1Vd~QneJJvg>{*wb1kwEhQ>89!Pis{mPU8|
zwQQ7zlKCWa(#U{zeU7EGa*Un={a;E|$FN=yI>Xvkl%yeIZ2JxVU!9f1AvJ@mT+=bX
zm}nkRU$@f;JmDkUmtLw+ZdW?>66&OF2T~)iDi4J+<5Fcrma@(zKBVy~Rq3C7PxKHF
zaTaG52(E@+D|7c{@WQyDKLh6$ZKGrcd|+NHaS5ouo7*rGc#C71_-S5+31{tg-&06A
zytBDPeJ0$Bd9~$}+)5<Gl1^ls`)Y3g`oY^>&)PC$w~tFk1G=Z*to;`wcXG{K=hDK%
z`-)cPH7E%KP&9o0;U>(1_M_>t`>o3<0G9#AqJkvSlpcMy*-6cwo%ql=>?{crox=|w
zsN%B8VnX&XT&?siMXGq!pax46pIMDovJ^!sKhIqQn5R!SOO!(}e14GQk&pS5b*;p1
zg+t>|7+9wJPBC-DW>}$WKj6pF`<P5ay>SgKj(T3#^b!t~#h)rz5+8*P5=j=+_Ojs*
zu396MR&@nD_H*{Otb4f)bkk<^ENS`5Y~nTHZtsI_KYRxkDiNhE!#VjU7i%(*UAmE^
z#Y2o?we6BneE3J%?*ABbyBGNARsw!g8JZ+gWu>#Jlom9=cMKoVk>nTjM7$|v=+<iE
zcD;8~Qg=PrvIrg5PD~|{0&K`dmEHoGc^H+u^^6Ty_pdi&H=x`K<S|}b;XfOW8S2VD
z(4>m<{1*2Dz5gA8VeQAXddc~k{&bQ6kmKF?$@KL%xRPw7F~tMtl|ndod;6Z$e9cq4
zCdYp6ttA&_K@?Lmw~DH&BO~Eqyh)1RXH$h;kNDAz+JjPoDybQP41Gk_lABTE^2+Ia
zVg-)NvuR)Pt{ktq=(PN~m_$20w;YyGomR_<FG&0;Ce||?<Ri)iQ#t5TdTbp-lrngl
zr?llC%)V6td?mdE{N~pKfr`Lpn|xG4hcVUE6tUtL9k*mE<ck&3r9Y-3f@uQ=?vPYw
zj{HTVfk5_6A%fL{anpuaPlC$uggS1C^X2~w>uMz*<>9KSo%@1phKlQWF}Bw{0Zm$@
zwc<RkM;)E94dz7-O$}93y!ua@Pq5zDBDd&+%{PWgAzVy>ll&z%@Hf|SW!pK%77mN~
zEt^@)s*2~T!xrxc2ExNc)tG3#jNAwF_YY9{SS~%2VHWwl>8dF?7v(ok)@Bc;$c8Vo
zI$=`muhjRG#zC#aSs7Z#Dnv049hX>5K9gFrM5VoC3_aI8SY~bdg>O(nGu;+wBh~cl
z4K|F$X{_;cg*y9;B2$q<x=xQ5){6;mKiBtGaO<6Z@(Zr4&Y2#|I)X;l^1cy!dwaj-
zdb;hp%x*Dd?gZdAekjQk@k{)&S3sb^MzP3*jblW{ud=zm*}KgOKUz%q-Q;h$P|e?T
zhD7@BK-54>eA|~3;S8_>ND`@r#x7}^LcGFP9kvxmvW&+ZSEmvN@Wc~3fKCh!<XST2
zjt1wF8ZpkKv77j?{CG#mk+|}6xco&HKOjiS=0D_<=_@og;MEDsGK<-CaOo>xaOS9}
zx{hasI&FYmqN{#y&YZy!c8s`)OjV#w=*9>Z0$vj}^!g<K&tG%IcwCSF7?Y<bkzPay
z8e*BJgYha9to}Z!X$sQrgR!~M#l~p&9jlAPUHQRvA?MdW^9I~%J<2T)B%&|x^wTkP
zNZ2Gq7t1Ztz3xx=m~o*bR-Lci!1n>w`**Q)bq7`{met)B2Zc`FCo>}u6R(u^zqkNE
zK!GZ8P%K^hcuyynq+bg1p%%N2#S~lydcCp{d0f8q$7^awO7f!5x`>ma>mM2NKt78L
z>-QBZt47P>*Iw)rNxir>mj*Wx^j@z>ST#IkuYzRa;$2qIAZsQkMzxOfvrQ8Bug(ec
z5PrBZTi@MnM$Fvs6aT@}8v~N#mjVLJ_+e!&Rs%0l@wWlnBaM|w?IxS9+R-S_!EBLe
zgmM<9h`&M8b6(j0M>S%R%8`2m?ffqU%wv=h?Xgj32);=&8Kb=)@>nE+4U2^%%TzI$
zKfOJGnE-l+Ghb}mMx;rIqN8WtsT$p$_Yo}@Nz@a|Fw1Q<EDrXqh_DV$B@3IXeOli>
zBF5w+|Anzy948=()&<fF<Ur{7{E_b$#2zV+$f>HuJud2F-#{QNv;O{FJN2f<mv=t8
z!Ug>OoA=*oI*>`_k49+(h9Iuekpodl%)XD=%Px|HA|b`WCT~<irWCYMzh@j(-97i*
zM34`00MUKskR^J0`a1n`xxD>j7XY9E=-p}zS#pFd;o*D}{T$S$0^r)ArsnnQ>`9mT
z`J^zr=+Jyul>!+E+U#F!zmu^@)}o8)7717^ssXtZlhl7Hja!08g@v63biKn6+d<KD
zunUy*kz3<*3IG95_xSgyU~~uVQ=xfJ(TupCoqU4vhk3Nx(<i7+1FksS=w_554Mbma
zhhb;oEwmv55-n)7mT@XH2nh*&TQe)sSb)d*6E?N_<*ArxLB2J@4VQeMEh2JY7gV@C
z&oKyc8h<jJd2r#-H9Ul0uCG^6AWY%A&Qx~vtaqzxg_X5MzII-;eLXS@ugoU}Sw{Dg
z3R?Cj@=M46w5o6Ya8`CMeDiX1|B;Tb;^$y>3l6%QMO(jK%lIzHWHA9K2`fXKh5VMt
zkPwvW4Lkd63w3~}C~1Wq+Xn!MfsO5Vy9{PcJhiM@K#G0O+zY@SFWLooKe!hv{*wof
z%542lYK*8|>R_<AAS?t$q}*qL`|2lvx6pAL-#XHrgMvv-I!pq92Zlh!y#5EvHi`wn
z;lR8BUOoT!{*NWbkcF5R+hpm(SJI{0o0UurPcg(``8z079nn?x9|F&%XI~pTTPfs9
z)DR`{DEqT<kr=OFVSC<QbZ8ugea!BLUHrb(tkz}5lBbe4EGxYdT_K|AX#Cm0iz%sc
zijy`}Flz7kBhvJUHkz3?_A?42gdNXjMSw3TFI2U;qVX(!Xz{Rm;nt5U2-`9K9T}Za
zzp#c@O5P&evvQ7#kT7S)?pBNx0H}EUUo2zPJ$^nEaasB<QjS<+a2{F%7=JB*{(@;t
zPQ%j|nEqq~8UA%EDD2n23xG}fpN$xx_%Vk5{r`Xb|IQ$cGwPSu|65G}J1<3i1qDD5
z7Z3S!6o%Ras5L9Oy~Rum_t1_{Yx0;B<I%PHoKFDbMP}ya1DM6}wtd{LuBxgkDWClx
zqkOy%&?UwgCP0=}oV>vQjBd0#u6odnhLIosYi=n8VM_UbL~Q>r@WlTseEI)N#R6}|
z0H0n}(~)$Qc~TiUPx_3+nKk5m+Z_#C0mja^b?Q|abpHbKivA(1BKsauJa5?PVJWqE
z<g?BC-F@jor^D2sJ5c9&o~%X_rKv$_wlq4Eh1Aw2;En!sR=B=Llh8{x{*cxNRQ}6m
zyBB^XA;!<+M8g>lVMNH$(}aictu-U~vw>GXPl2x305#(Oh=x4qi&f8b3HR<Uu9JJK
zVo`lrgWt@Mn9LA7aYanm8ENk^cr85CWq%ySS58D|%d*;dFyJs$91&l6yjrcdp&#{X
z?8v0~C`IB6J)00g{`5>(1PSMuC|Ahl&^RlL=aMaZPZebt#L?jP(t0M(>3uDSNwc`q
z<XJ^+b=C^{^0)P~-<n@9gP?^ZSBa$Idz&DJ&+0oXrN^X%&6c}Mr4aw+)78f{%@`^U
zQR?A*%RKW=mn^En=+x8GFt?-HZ6w#UM#qa<k-1W7PpnUOvObRL8N=+>lE28UC84cw
zYTf6liLVDQTRW<3?iL~h>@7da>8|<#g0MQDWrsV67*~$i4_z$|!?T3pb=%&pMW|?H
zCe87mm^-I0drLiaB_uZ>%c_!7D&I=&M|wS>qio9g2V;Y%IZx5zjnsl?Nv%4ym8;EX
z-=+wx>k;+ls6orh?ZeV5$vgc~Tq<4UQ`kB5ZlE_kn~)?8SL2YaZ}!Z2(_YoO!k@D?
z!3>g7uR(3U-I>BJB^*%vpJLd_Mjg;i?(=r2>S4>Y^Q#%ZBu7a^2Qne2oRr(Jq04YP
ze7ClId!UHSn#-?dPp#E!8~?m3g-?e5(A#UK(y*Sx1<K}`&~A>JjJ&<n_5oK{nk`IT
zm;(0^9!~;xuixK^Em-j0Kw6ERweQnX_{h*(YN$5f<1~`hAFy*FNmL;jAe)H+@K(1x
z-S+o9<eoWYXx4hPjC*mNy;hsr*YUI^;XJXm-a8YHkTl7W##T!>-<kAb-{tiCyf5yo
z9jIh*9(9p&9vItLb}(+$^Dvvu8rAzFg<#B}Uh*@>EwO-cr5CCTrpz?ebxwP~R!8ua
zP3rEQcEZ0AhdEx{uO?cJ^6P40?tbU{zYCu2-aHj*)6=4JTPvOy|7E7jc`y>b%Q;-B
zF>kxx1aC;#_{bDjvZ_P3ID;t;ANso!10Py%cBYgb;BTYKd3Q(3eiNvy)2`~8(0Mw$
z^H)v<8yhqR$~T|HH|aO>^Fa;Q>$aC1c5-LU1GW`J{ftE414MlrcGiE`du~BmAAi9F
zE|~rj7xbk9IsbYrX>@S8s@#1)(OSsyf<zL1nD~H#iu7nTF{5hz#v0-|?K#w>?DxT*
z>}a*Vuhw&^vi|yMR)D9${v=+CSiNIi@=F7W#*!-JzEIN9s!prh(x#pt$YNrLgHW#?
zy{)K3_DDO>XX1}35U7DVo$XE|MLtz3rO6`>4(;Dl?|R?vgdmahKh7VQg@%nhVD;Rt
z@q?Y`k(Kk~X(mf2J9X7+P_v&F7Fb_dD9U(Ve5ng9w7ef2x@@Nwlt$Tzumkg+;}y(A
zjqBh=Mm;FiscZ^>)i)p4WvxJoF5#}Oz{y_$o~k-)A2UXV+ALv9LrIXdLNnb*1v6EN
zkeasE?VIEKPc4?#4gZFziM@FZwten&CKUm@c7F=^`?g2%?^_`b1JU!J!YS@o%Fuwe
zMcwM#O>T(en}Y5EydmWwXN*HKTs*J9yY2cSI4d$wh@-9&d1O5#3)=c?ZE`$sMoGgg
z#K8~uHO`s&4G~O!PFj^TI{a$^V|l<t0RmZ3h(?S%`Pjo3^Vm%ShfIj9ZEm57MEh=^
zp`rQZ)=@t(#F?2?vxnDy@5+B`@7jZ!KJqwdZEX<&&&aD(<b_F$D|=cTj;yF)9T$
zAw&`o0;G8fB|HR4p~oZiqyjY_<t3MfmWBX<gkUHHjYxPulvhH+$fH0^f<TQtLhO&3
z&duB(H*+(c{&RDG?e6S;cfPyxo!#$e_p{#>zVi*A$hNxc%hCCEg{|?IRt~>!B_PPh
zS6_WQE?ceaXtR)=T&eEM;-ahP5noNq{xm_ya7s!M_kzMayeTi3>L&DH44G&`*wDz^
z$LMX5az*%5c&v%;n<xElw~;BSS)znIefu*vYL`kV1l3~h37&IXB?13z*q!!cumk=d
zRj;=Jp9^@<@Ui<8w9ozsYj&T3T4W0knb~)w2vA+!63LRv(xzc~4<W0-G457It~3rO
z-@jk_<Pd3MRh>DKb1*lNu|0?R+Q*R8BO5=YFdwRFJG(FbI(6~v8c6sC>yI6%09A8l
z)VmqgeYU$A^Y0_`gR`68V>qVNs%gZodSy~_OwRxbp%cwFDSNc%Id_SbSb=I|C2hY0
zBpk9DANQ3XCM+S-7TK@Yf=AKDLEvdIdBM2XP&@C)=J_rNXuV7};xMI~t`30D=uzwM
zYCfA(%u@4;1I<d?x3^TLksk(UpuLpw<$tLU;^RCBR!M%AB7it|!P$71=%F&g!XizA
zE}R%^oM1jOVSe%GQ$}wxDHJgF++?cqr{!bZ?3+LX3m(p+X5#0SW65ZwHnb&5K4NF<
zX{LQN+=U|9z50bnbd@1T7XxmKUkc#<kaRM`j&?b@^pyNfBfaKAK)A;HPI*a*#l&TR
zs!IW3_5f=tiZGj#&9IJd{IhF3iV$Uq0o%NqBzht1HY1)&99th3gjlKP9H%?o1G>iz
zsG&v;mWPw(72h_w+Z9qaUIE;hHfihI^|75A@$}0%ZGmXySb@7+sDCuS3^bg}_>N^~
z1=a9OnaxfaSdS7g?>TmnSDkuz`Q2-r7|T{T;4|&GM}7|n$RiEnh}f|oN?p>AM#U<_
z8^8z~9bLaw%X9U*=H!y09BC0&md#I87HvmAEEcbV918~5NwG{f^_!GIc|Lcb0QVDY
zUL(f~4I1td)|~p1g+*I8Wzx}f>&uf=v~ZxJkT-Vaq>gpnN44dc67ytaFo{}}vt6lC
zM)zEkg$UTb9p~QZkF}ODdyXd`4S89DXA;sQ95T5FTd$HppTs+FAHV!6)0rUkPP0dx
z>nCv(o_jix=+_iR?|{o`HAK<2RNV*cJ{fmtx;aeHd5Pg`VciE_O?dop{_Ycds6Ho5
zCz*dm-(>R+E(p#n%-_uoE2_kA;jsLVIIOyu7>JYdL7<J!lGJa57kOQ9u+w}j?i(#_
z<|-VR<dHjMp9f1DwXv2yZLq$yb!9G!EZdyS-sXH4DK1kGCX<$%YY`2NdyQ^@o_hnE
z(#3L5+RCQcdAgwH$^jG=d+W97+ZNs4PDB(LQ3GlCd&PsuwYH|5jnC?&gY+{uW*7H#
z-%N<jKPo@quF?uP$dVcVIqpmjH;58gs0s|K&+vLNsb^ub&P|0+HPA+7k6S#RK6OHv
zcyp8w*0^dgwVc14noz79ZXv(P(LLu{vF{CYryc9*#O;P*$t9R{mXlP_7)`(B_j7Nl
zCELnB?34sIbi6vWaEvH%f|;ptJSU5i)5Ojy)~RGZjLo7*gn)Ts;w8<s<P)Zq03#%<
zJ#u4p%(Vm+UHkIlS>9-l+Jt^fw-KyUsn7@<*0om*gql?CD2>KkwmA=GEpQd-^~-6x
z@<Ut9QR??s=p=@%I9(#Cipri2H;lKj>YS6%E9=_eV`iZTE(TH2H@D*a0typ1MHBEb
zynESHBe+h)_oO^qobPp?R%o67))A3D_5oR{<h!y(9g%sQ_7iJM-@AHl(u|Fu_5jEY
zKTT6T)9dxSS|8`X1$ok(S0+H@S7~5P0U9vyO|`rKTy0(_grSjF@DZ%M?xDJv`@32f
z;f~gGS)UMy<;6+y{0eht<1gQKv3_h0rF@!_sjoT-Gdn1|I`qCD?HPF;X?J^GJ_@+C
z`!qRwW!-1F`I;GXYieAw?(+yvaZ=5pd0rN_z5@m8)4um@`E^vgWg@>y0Z;M$aH;3T
z$gp`N)*g4uPf9U1`oS20Ay?bqx-Vt{fMO8@Ky?uicoG5>HZ8@#P)3)21jwsjI34~C
z$pH%$Q1>sC|KD(F{~&De|89R5!})(``V91rSm3f^e_sP&npTPM(8MSznT)3-0#T=L
zoJz0<+xx{OP{{-u8RJXaG2f#*ZK8p%;XR`g@IYeZ0{zGZ40Q*)xL<*~c|y;4o`!*U
kGSF07n&ICB$kb>;OzM9Y*n3>!3kVnm!Y}#P`9@^@3Ik%-k^lez

literal 0
HcmV?d00001

diff --git a/doc/design/assistant/blog/day_45__long_polling/phone.png b/doc/design/assistant/blog/day_45__long_polling/phone.png
new file mode 100644
index 0000000000000000000000000000000000000000..389334d95a8f616818fdb63dd26c273331a689d1
GIT binary patch
literal 41602
zcmc$_WmH^2*DlytAV>lPcY<3WK!BhL!QI^w+zIZDgy0gK;2vmPgF^_x-L(nUc;nVU
z&&m6K_pUYf$IM#uXMpaas?MpitIn2Z@7fiqrXq`nO@R#pf$-$zq%=Sv)I#7RiG>QZ
z_-*Tx0R_7Cd*$~aP;ETU{YMO-j^d^vD+#I^quK*1bmnpz${>(0GYAy)1q8YSnu2yg
zAP*i8Xx|J35>5w!NL{j8)kT4hC!Z8$r9jAkzj+;{NkGd}7dd@5pp^U1M`qAo9Jn1M
zFZEu_d+Bi1%U^T*5%XkxaL{gA%qBln5~pFLPIl4oy)+eFR1{s@fybtH`{T-HYcPik
zG1ZR&8T4d&jVsndHEc^F1;zYgKg;Y(1OrpisQmY5Sh3LSVbUB?pRsfQTv*oDGG8K3
z=&>KR$2TeUWGQ)kM)_pMva!1@c;rX2uj)LU5Hq3}$-l=gSoB{&K~p|Esf_Z3(!qjm
zRt(Mpk)qsDA#LW&&8%S|+N>!GjWWe^bqa3^)p&3>kMpqTx8N#!_qOsxZ|f(Es&{kA
zBt*-+KEb@XEB~y%PBPR=SxZR#-sA!6F)yv>U+ZLVO#P_ug(;0i5B3`A_RDp@$YP8V
zwdU1Hg?b^@Wm#15G2R!|U;lnO==w~~=%>Z(oLHFwtwgcHhtkuh^uIf@sHdX~x-)Pe
z2t`DNeM0E4P{*e|d^K3)6HAm^83e1>-LJGUAQo)XFC-%j-};lLN~y2EP0F|P)^JwG
zN6#UZ4xSSIjM2-3=I8N-yc|6zNosQSFTUge^n0(luR4)XQmtw8&0`0lw0c6(0ersp
zGc}Wn1>HM(ROa2`eC;0B(wva6Z=ub!dBHytx_Q~9cUqE;X_#AlfMsM)K|^?|u6f<8
zY!m8~)zQqITfMLrP`GR<OH!|?&?wfjCw6xV!k46w5}#H~WIf3;6NsS|`|av&x@lv=
z&0|2q8O0vu(J4GO-iEquFHrG>=kvW-r=0{bkv4I0PiKBiUQo{QEOPlHnM|=p9K{NY
z`CxU348%<Tt2$_1>CQxqu%n~h*m=l|pNJ&I!qS>U+o1US@e`&m3BF0=7X_?mS!|t-
z7S+GUP{-%RZMkdl>1)((o)|qzV>aTnjDc<3gB*yf(dC$o7d(SIQBb?1f~34mkw-kZ
zy>$*y8sos#cE0M7f$|wnNCM;FS+HN>;-Rl5C&YoAhJ<V#%*mB)O;)UMblgKADaXeE
zmeC^eUbD#ZMqtSkS-u*tPrzk1AuKl&H*noYK<Dmj_L^6%r#*9(5g(_Wp~$8ko53>n
zbRT)(vIVBQqW8ViDn){`AY2SmLd>=I&Cgfth_Eq++54Xz^=HNuboZWh2v?N<ENU=0
zib(v86Jk(#oIVjOn7JaxDT!qpXc4!^1iZ{=FQHw!1&96nKZn>;a?=un1(P38MPyc}
zjaZY9L~AI!*E`WZ=pk9Z_`J`z(39zP-PRU#2Z<zlJ);m_xI*YNPX1=y<%h?9;{P5M
z9c|cXXUo;%%EZmes#aL`8iw0jQ&9jOuQKI+P+k`*))-|aQb=k2@l!}h=tpOERASNj
z)iWHtSDb9RwhhNQXaHR*0A14R_(GbwPEWBh7$SKyOo$6YWqMEpndwoZ)~uj^iGx4j
zxC4K^9H@So0StL3d`~$+v693E^~Tq?x;v&^tS#Vuk!)YFCigy5&>bFGyYqG%AL8P8
zw)S90kU1kN-e0K1bpFe+sR|XNt1g%QB_)qHLU&wL*kLEAb9(;KJccpZkWxuy?X~-W
znPPS}w4KLoVihfX-2J-3w{Bx&<06TLPbIfjRMhL$xs<Q3t!+kta;oL5owlSyLw5$J
zY2^UBcCn?U=(u2>se0nemj<U+p=~SxTYt}3$}p*0iLn+?&T+|Pl&_<O@bR!hrg}Nq
znebkb7#nqjDvD=VU@P!qd>&-su38I@mR68QtJl*h7M!^kp5c}VY?ZcIkD^6#KTI@h
z9@9Fp`McHcBR#2ae3AJu!Bl1NdUT+o*P-xd@7VFYStwUy)y?2ux->YT!wtFNyfCTQ
zb@M~mV6w&yTGkWlt4|Z?{nBnM?oC@F*yZ2kbSe#Gq@}mFwu}ab>h4KNNhwL@eswbU
z0_rb|$CK4sAX4&jm4*|-{))|3Z=SulPTGbwyO#wJpIQ+=Zh6#kTkWW4M@s`M(bi?C
zth4Mj3hB2~iayJNNH*<V?RHR*W?f9RmzulMJ3XOf1*Xj@%o9^TSN|dzf--fMOT~zT
z8B4vIC*3my#?NRZ{GpA)+~PbMgP`u==W%xqSM|?3aY1{S*A2TXf?9?F+}&L?RNKXQ
z+y-5LH$)odZDwF&mZ)+O@$oS+36+0%Vl)Ps8|(`;<WR@iv?+LSAsVEl#LZZEj_<k(
z7#r=#P1L*-oqn-=!JnTU17!ciYWfF?7?*?-L*|}-(8rVlR@UUGptaQ!>GpCgyf;~I
zIj}<XiO6DQ=3nA+qRCOD3=Pab^sx~H&N{CMzoGnF-O*IRlU|*>6N`IQ$0>#3XUslh
zn3|O2g-0ncU{I<JGVbYE%mR-X=c@riVV!mru41BP+pY(gjB<AN;Y;6+W-l>ud+Xv^
zyMO=>nK<(lcCY8z0Mmq1I8P@sJ{29$RNj<%M#3cT?((kcTXzIup#=uh++)k%6z84o
z?~0uZEsrVNrsMZ;aOXlz`ESD>h?*sDP0({gkr(`j*P{w<<2u|%;t-rN@rVYKA{N2v
z7Pkyw#h8Ehd+jg0e%KmROmn54ZSapC`_c2@T|ej>C090QG&t<6H($UQ)`+#;;_I&x
zk1zSSbI8}RXb6dMe8Cm@k}B7iRA`bj;w{GQV7O*Mo7Wur*IKR*PgL5?yN#Pk6?RYy
zO+b*#Gb_duqa&I2`(Nc}@w0*xv&c+_@2?_vR<=z8_)nS;D9Oe{0~?a9jgA98W>aRm
zRa^R1qp~tG4R*7<^x}_NGBO`Oesi!K#ynVVjMU+vKe4he;bv#&&7erHL_<Yo3@+rh
zV*Y|pIy-WK%I&g4Am5Rl)rv`nHDQ-R)lUo~O@@(*yTX&F3fj#YO>35%cf_9ef0Z?T
z3;q_2mDg@mtu-qiLM!&L+HYd;=BMJkDOi8UH*ZK}ZBOi`<2OyxGmNf_JDemGM)-m}
z=7pfCnwr`CE90f#zx8NjWMte`l`hWDJ+w^o)mT#;;8jmity91355HDuoKI<Q|B8`k
zmY&!B?i*}H?8O)SDN&;?6>j-Nx|6IyZ#Z?Kv^<e?c)yZN{o=CWQQ<85{=I%j9w%SJ
z>9feTbCeTNsiu2Lrwnj<2qxr<MrGom%ItWni09m@eZ})@_^PdO&iQuf+&0^DrSE6m
zj0e589UsI+YbKhqw~j<)L`4;ws9iU0_`e%?ErowU7b$aoTanA-IdI&|GHW6nrqRa}
zqfp4j%J~c%*Zqtogn<yBkkBYGqi^bZn?N!Gr;2vkFfdkpV7yJ*riL*ps8^}_QpZr7
zfO?g;Kp8T?WD2%s<qYq^4SnmgO1tCw0bUiU5QZ9*mbU-rAwgdqE{J&v-#>oJ=uu--
z{@j^hf3tYwZPI1wQL^3W;)qg5Cw*@G_4Rd{RF29z07f{<<M){43BBV4$>E(|oQ%WQ
z+d}@7+8|t<_X&EjuD{c3Av)DJHfyB1)h5K!lk@XlHv<HdM;qPi`?J3$NazKRF$Ypb
zC)ZZ%WsQCT;$KYlx@P|C5nISov^pz*KFVj>8H;m;iHQja@bU9z=Qcy#o`kB~?N(-g
z!9Vlr#cjWq2Nlqz<EG?`@B7fUm+P?ktSwTjep9FBr<gO_ctcVAJ8#PD=BU}!P+HoR
zYQ9kuMncb}#9W`K{$o}_M$)qQwFJDqEnI#j26nrVShq+Upxwj9Zz7J}bjN|unY&Nq
zUDYXJ3DL6Gvb$4bqW?0Z9JzmfWmTVFZkQWq5WD2_9n*yk=3beIBG#%;g3A;}s))TR
z#_Bq(&PvqT*?Q$za78WB%Rb94)5}T(BEr_GbR57ucuN1}ryNQUY{cq=Wq5k(3nyB(
zsIlWdhN6GK6{7Xx&$DO;0y6=vGpc|>F8VJkj0*MYrt1E0Uh=P3z5ejhpb6i5Y_P-o
z3qNiz4|dqgCgmB&-a9VS=EkC;ZWg-VtM3T_?1wu2datQ7ds!vhU$w2N5+j%tJ=n>X
zSunzKUfAbuCCU^j<SV@VUz~{QcC6IeW={770FAM(o8suQ0BFX)E>uzlN^f4v%}hGM
z_tMgRlf9TL)EGch^064=?rsht(vnF%bTQfsbhKU{9vrh`(f=-_67x4ZnfP`bxAC=5
z37W-tm1hBrqrBtnT9sTi4RuLmBuOEPo{B|O(n<o?=YO@hJBz}OOAEZS%*c2_d1Yu(
zmB4to*gS4V=ykj{y_I`1Pl>A=9p!pnZ8S1EI5O6no~fy3WWhthB+pBRYdvnAGu*0H
zJTKVEkOLdHwua0soiCF5+{t?Y!wVXB@CDfFXBZSJiltsHL6B>89vqN%^)7-=`|%tR
zcsuO=_B@GM>}K`3;uq$-IFrypZ#H;ch_>g^RKU9~`rhz3a<Pr8_wv1)pEi}Lz*qiO
zR##!~(ZZR2dv&H~P7id(Fi$U5oCo+`!gCfN&ebU7urczOKa~8&z-mTR+!>d0vvHiC
zohhqI_&@`$D@{7=c1BpBKJX**cyWJ*mj|;DXdJbOQ`=FK#N*9otl*mOc_mmiS0GlK
ze?MFSM=^@&>)yO|f<dJ~rm{?7k84(?M%lx|LmSK0hEHIBRhPRkZT+H_fx%pQCp$ZP
zU9}PU8~+A_N;Yop%gakjv9qR8!^eoYh?DE<HUd5wEc(&7MTa%76S<bPA3r_U*&<Ia
zr*3@eEiEx>I=2-QqqIHt=LG5K^2P*^*Q-ODEaRR>3&)YR*DZej&0l;i?h@;GE9Y3m
z9=61;SL$82quSs(KfO;2*pir`c>|tsg`E5D4_yyUPAfl8GX<Rk5tY{_$ZH6n^@v&>
zUVC$1QN_sea<f}$l~sLNLxWVA1u-BLb3C{VQ3X9__|iSY!+!6&-lyKhzFjm?)6&u~
z@m=2ij2jvi#ha?Fqm_Jna}(G0FeV}3$e)h;m+RMem#P?<&Bq6XrlzK%q9PX)3JOYO
zWaPu$<+R;isqs_#<|=cRDX93Q${r;nwkSwG1T<%)F4Vkg|H04{y&ojXNJEH8+~osH
zGC$7x+uoD4avyV^7Vr?tJ7zjII2s>i^IdRR%<r03Rb=p76r@h1tzg*|*x~06pKGZ9
z{8w^rNZUS+O5T!;SxCrcBCqIl(`kCoJ=S4vwnAUSpk#7aB}dd!`CNkYHXn?9%(|5^
zE~S0@=>{zRXm)c#Kq=rrCgyvwGnxqn!wWHxi0`BnuJ(p0Ce?xzB3}D;JiD<ng))U@
zKPrS$Q(LW}V-m=v#6aH4igu;SIRMzyFR}t|e$FN(iI3+oQ{f`N)z(n~tB#J&k6DVs
z!5=#I>>M0Y>SSIB>J;A(J7GA=Z`Aem^-TkBw@twh#I&?fh;B8&dn)NX^y1=y$j1ku
z%Th;Y4sTHuFndJokaF*@ZzyB41}&QIt@%$TZhRuwW7d_|V`h<0W$N?m#-7CIib7^>
zM1D6;+VR6)@n5~H=tFi~K6XB^b#`_>K0XEp238t2cLuBoCF$#(O>--YP!bRjsH;!@
z=Ubg3)^oXO%L|J5QC?pDZL~FcPf=l*-(a;&^Uaihb&lBEv2{2c+OXM9lM&)`I4LV}
zbF%ukH{3AkA)NLe2Eo7)GWnsanaF57(lJx5nz-n&BUG<3SVAS9;A+zqWohuW(T0F9
z(dX|<s)XQ?NR4W~8i<vf8>9R5@*jY-$`U!)q~x)fz)Bjl;NPFMQ~tA;)a7_{X(7|9
zQC$9mbDr{|P$p*f_O2vH?nU39wS?T1l*@-}sGpzTb1I>U{(cr--a7mFs&C)EnfJ$K
zzjd>E_pbXI3RWG=!FL_j;E7qx6Qsnm3hi;2$i($6?CCy$2qitudcT27Xbmb2*XK?q
z_bEHZ`N_gpCnX1{eS9V=QUAJA0vc73zc3i!|C>~g4-E~Cj*595uY7-F-(cYA;$mZB
z68K{5s}98HZjS{CA4NtxQB&jN;{(g8T?}pTv1iDvL>vZuwk?w>6!F=g>9}4o%-#Xt
zbXt$+h+1Pk!?ycHuw+g=vN>Jab-gB1-SznA?((2Np0>+!cqd-`ersEwTO2VWp`|d_
z=UI+>H?dBlP+ZsEu2nf#L?iKumqNJTt3fpL^WCAN`7zeXa}XU?2-9eE!N5G<v>TPz
zONK&4Sk=*T-@#>yC^u`$&;#Z*F|qLx6mlIIqshd?1Zc|FyBu%p%fQfJ78M$3nrn^M
zHI1DI8r+VbB6c{HD?0w)4>uRrw$GYt)tW0c2P-S<>({62J)uCa-{URpNOjY`acOF5
zN+xHmS>P9sQYaYX^=0;2By#P!^by{7QDpR{WfZ&oq`v0sjbQb)N?$Zqc|<@jEz5a&
zM9$jVP}<Cbi>TyEeQnDd|Kq9suTq7qTI~V|Q#&SxHsK(?{^wR(<j%AUr`^}l*V^cQ
z(P7td`zr<WH@<j`&i(U$LuoO4LEZ<I&1=2~T%@{0xZAQ|y`RleACqU$P-7oxGAb_O
zQn}C*yll?PyIv*NPjcUzC#UIHLJdx9V<`o)e~w5=%e~!nJ|JAU^gPgH@v1g$IT~oN
zoh~RWM3GC}Te))<D5X?7p_o`|MkdRGIQ44sYQCTpqVl+U=RO9s2`!Pla{bIs<;c6}
zy#<3>Sv62h2HIHba4mvgAwuNS#RGgTEQ%fs>dx6j6qSWee2>_#DAk)Lj6AQ#;d4cq
zg~Il(GAeDl+oT6SYR3L9CuPelO0u*Sa3%HRt)r_GF}+J>G|>v;Q?Pm=gy#MAn~?l^
zc}z$Tqr4=wGjCX5aoTzGQoju5lbb(MU$e(XUU9y{dy4H6;Cu8?YYR~*?nFK!!g0x6
zw}(>n>#ev<+R3{8;2mkn$;rNlHw!1~v9{1u)+naFFfK_{+ALnn;}svkx(Nw+U*|kr
zD`icjTOABsr1FE=8F~dquXiUutj7Xsy~}xRr3>0E=I6O8;K>hB!z%b0xCwtiFdZ#6
zAB=t8Os-JO_*qw{ad6bO)lW;~>ry_cw+GuaG*Qpj@C&%^ZJws_8F84Bxc$8eZ%6EO
znS!TXM+1jbM|1t}7o)kwkiXo<Thk8}6%<|wdlI0V<eT9{W1m+U?JbohUFriSR$HGf
zGK5h+kufrnH4^w1)r~*6u&TeEsGT9`m1)Fv)>l^_<1r(1SZxYJY4zIs!=n^mwK_GW
z>h$P#UPIrjWTp8yJDg3}dUJd8&}qAwP*n7_czI?<3-p3nNyvRSp_q}ITeVE<PhTI(
zx=#-57EsLwul`;i%vTd224iOrvJ3w#CQcorYaU)$hui6aGN)g0>et>4!u&E7s{Ahx
zBFnhjCzfO{bq&Vuh`UxF2Bz!YcnDVunq)fJ!Y0O)S*jK6m^N)ZK;-1)rw`5?1or_Y
zoi}yW`|jTYCv|h%=iG36e_pIvpLT6=IO8S~DRsapD~URA;Ql*%scW2}w9Q3CRg1iT
z^U3SNK!K_q-u)lE8l7e}n%LymADfRSzLc7(U`=MX8??CmwXXK0=&NA|mHet-9>+!z
z0O!G1YtC5bw`;+8S2{ef*m>0yvB!lkCnqkj<$^&kv_<6|RvP^#zIy8i)vFJ~v~O&8
zsfQB7kRA=GoU-*>B>x!LWqZnlppf?a?CY0rAEx##IatR1d(=N>_cpVL#9oD|cuE@*
zn(>m671KqAu)a_3j*g+Dkgi9`%K1w!zr~LC$+q~1puDK2$TY2r*zK&~FxhZ)T(Cj$
z<i7jUBSSPf8|dB0?V;dWrt^+c0_a_05m;2b#rjjf?`Vq-WLv1y<6sWtE}U(N^3L1)
zr$X@(n9lMroq6D5`EkiaP0hRiX4ykHd(eWDjcrS4Y<O4^GMk!~CRq2`I-|3pLaFjh
z+~c1%Ub9v{Uv2XuMc0&5oA|1nN&Efm<ao7>*8k)$lSH7`{LSYBK?nbvzx`LotB8S_
z{!WC`WJlElUcu?vV!ao%4eoyen4rspdD0+-qed_?SoQJzcB3!)?&_$}f>>4Jc54}a
z4@GvIUfo@G#Zn3aEY6h+zS=$?e>g8D&DAa27w1yf(x4(Dvk=JCPaSPZUVX`*T<gOf
z=L`*;PK_l_Nl6t2i$UjMSeL*0qRnO2NC*ivOicdNRZLpknNR;Aq3`+ZXwF4U_H}Qm
zWjezR7HE8Zu<+U(7;9`WDkg%0-{qusvn;?*a4UfC&6_u`U#mlA4OAs;8kfehML1Fv
zK%(QA=;&HHI!!i{zkEu%ZVtL~{5MK_m5^nvU2AKPp{f_upffM<#eR{h09N4=z4_pL
zb^odl^gh^l$HYe0s>=WM&cgYih*s;3!O<ad{dmvxbf~-+LL$0!9Q@qEkfa4-8mn}#
zDpR09%>RacGdm!hS=p36N<*$K9whB_$LSnPj!oSA14YWV9GPg(k|TVu{N$bIdrOxe
zs`j+RL7kt2bF0{X_Uh%&+%3&F&{Mxn6kZ>*RnbY4(zJ5Ay%|u$2vq~Ulk1beX{e);
zE#b`R>f&EuOh;gDU1kEnJ&Y{&&s(>h`vX%XDqYkD0^WuEL+M;$l=f_0!p#e9WEe>X
zTA!-2Ja`s`eZDQf*@o#s{?|hD?jYIRW7h91i0o(w2U|8wQI~qWMdS3GTekOCwGy~S
zcDA^~P?t)Gp~)#Wx>@ybd)X6TFJ<cm@y9WcuZ&A6Ba{`8xWwO;hV^?_h}?yfO(dsd
zl+kQlVMctH(4b)t$OhtZ<yC)(t7{2gYjqtbe=UH8@d8doHItw8V5D!CgFNq#?f0NK
z-U!$fe&R{PAOfk&93emiFOQ8hAVdb9IVN<K=!i7#OpCM2)>}w;A_y0Q+YA6LI^-y|
z_^Uh-0#hu;Bzy_NJtn(-bZ*bDkR!rC{Ax6Gvp-a+a#TauCUd*ja2WVzoJ@699fc?M
zSfH{*B(g>&vIf5(d%wY$kL{mgqoYOfJ5iymL4)^m;;tqFbU3x`@0?v7AFnDkj1R&s
z1WhpIS8hXJl)eHbFcQ2wPWc63B*nDzgUmg0z#sn97e!p%el}FD4S8ksQ%zf&3G3JC
zX8#L9LZYaWJ5=GmRIcU6JNy2DZ@pw+0g39$t)>NqNq+QEbjGzl;LtV!J0Z?*VJ3RD
zpKe#ZR99*~e)#n1`d3K%)zJnBcQ>uU@A|mbyuZr0tpwb~2ns+vTWIrwFVvdPSLm-+
zLqVhrHy;O|!@I8k?$1I>m3&U1&h1w-BmJb{wwrp@F1-%l%MZ_$voD9DxkVAGDTx95
zkt`<ux3|uu0bQT^&Q3Ss3k^Qgirihi;2v<U-_D+J+mDJVjuiHePKC-j3z&tMyj)Gd
z9CRGK+x<Q^tDoccmdl{w#}8Ir_LSn{V#-H7T4+^(1w^N@yfHp{+O82YUqK_{Sy3Gi
zzpYRLcJ3fyVR#4p+}zHG3$>sMmZL_4N&s(d>sRVf0)nomx6tIc3@9iOJE>eh(bN3g
zd8-^sik-_PZUmg%>w2ew<4=gDalgN~;N`v7(Y?>p7AutQT?x(HxjIk@xaA^AP(1tL
zw!lESGBzV6M`Zmvra&aT>Fs4VW*rx@(+Ree+WYG0hP>Od9?R@@1EGuLDLy%bq;~l1
zO{IMP{5jz6qPC%Br}PbJ9rsD_N{lQG>N+WnrjFVtc9nT_k0C|qD|wC2wwBQ@rikx4
zy4+cuiKrrtcy4uf#Up$KwMUh-fB8+65w?quA|>NBM8Cz)w49mH%vK*8I>1NN{5vX(
z8SU^Y73b2byZL&w%IbSf6nUgK+rC5o$2~7o(P>Ncpj^@iBP~5G`@T|BUjy5RP;Ek0
z+69v7+8xr*%S0+I%?HTGW_VSb@0aj!oIe}V9Ptt-BlRz44lN67Y{t^YyJnKqeI@wC
z_PU~VZ(R`_GsT*;ZS%ZQn}Y`)*|LpKe+HKQ$g9yRJ7-fgY4}_8R!{xSMU{BqQ8{Au
zrZ>Kw^#WVl)9*yQSl52hvF88>hWvTmRGO2EUqry96_J?KB23A@p3D7bXZN<cD+{5K
zNuyx=BT2t;)uQfn;Mt0&)minHFogkEll|ZyZV|;R^R#R{L~!Uwqq09W6^E&f(1<$2
zrkV!c2~+FETw+|*2X|@9ln}ZF(xqozCX_S8xMp1(27a|VI4ARL>{ZwbVEJ`vf&x}~
zLdai^C<ULjVfV&FuIfaE@*?a|3!ZnB6b6vAIrYk9_SlRU=F3uR;&oj|nmzAP#z%hk
z*WDv>5w9#vwafV{3k~vf80CO?6#(>hMl$;P`V1;P&$b3rDRWUS4wg$h@rL%HT~J!E
zgO$D~Y)lMv|M1Y!-VTyW+^U>5Zckrr^>7cR^h(`(n-sW^@Db{Nepwi=V%Xx^m(2FO
z%BZDIIWzrN!A8P*-m0JH3K*JfIUKk*T?+q-cfhRj(+$41`dsv6VyajXkl%;pyT+vB
zj{X#^5^;Wyo?o}mRRgxBRV4zq{>q{#`MEEFks|O(+0uZE+~?LxU@A~Q&Uxo|<mE4E
zN-)$LakM#CjdTPY_-Z6#R@ZvewHW6s%J?~MVFIIkvDR7ycM0Az+@TKF#rb}ggHJlH
zXH6zzzF*NXhv#DAVrJ~#9Dof4>DbtW%P%vgOG8o7C>Q;VjEsP|G=SRQj2aS=kf8pk
zaQAH9rxnrgi{BODAHnHvOG5Z4&!>?ZtzE(L$9gf9?O@N}4dyj|#V|8)>904X8nO^3
z$%yq?wdf>VA4MS0eu|0T<S3}~WLg+LJMdc0=!dQfysbLN3FvJDKba9{THdQdglNwr
zWpZvNq@FG$d~f?anHY||t>%%?t{z0*Z?qYAP7w&u%fqr?@hB{QH!G)n>w01HDV`?P
z0cLX<6BF}{x(Zp+U2f|2tQCIzDt|Uj>q~Mm_;8rZO7arbl-wl#fb)ZhG^!2hM&k;n
zd`NfAVJIJsd+S&Ct!wA;vQ`Pc8aLxGkjs@(oW5zJ$0P6X2R_MY1*M+%8+RBGS&4i9
zYMs{PVeKxi^z(}fYdnm%gBeqN)Am_k|4M~*7HnuiBB%c}zcq2VgX@}(_*OxNs{0oD
zt^U~PsU!w2<^);|e#gx5e((8p+-j%EX1dia;Ym<Pu*9%X287;t85D7*BlBCv(rVjd
zmuTNY$5D_Rgz&b&q#@ilhE43RSX`+O3V0lEq!^e>FbPs#p^-{Yrr0GiE<K+x{Atrg
zaOqQeh=WmsZ3Ir!e23qS9Te~}cC}0#5THag+b)i!t55l#h;vOBcM4D5M7J|4B(k!<
z%_mz)b<LFsG0ceN`7WKLzBTHf@-4%{>rA3nG29m0b8o-$K=4RGQQ~2|pT*B_K0>ov
za13FkuQR4CApvE(j@g@It1>AB<xTz=wR-FhMgzh)9_s4fSfldu^Pym5fRWKk7!E0C
z?yQ}tDebD*^;*5pv{kl>_1m~+8a@}R^U~aLpN-SKY>|1m!<UqLC0a5vH7$*)dh2oL
zF2jpmhWLk@&F9a>E>4D0I6i&)gdH9AXVr6=dpK|(%5B^G)fC=tE*jK!Ooaqw<pQ4D
z;Iz8$$Gmp*37z@FY(h^c5?%?GxY{+(S_D-ptjC06KKH$BOtV(XWjkSx(+zms&f*r&
zt@k-E=?M+5fYt>5J|U>L?z-FWvMQT>W&A<AwqnrXlYSVZytr^~R1|i0TC;;PUw?6N
z?m&nC+X;yp!{%3k;#H5|KeqjB!n`^f9PD7k3K^5d6l&8L=Ec@2Q(CKik+>ktUalK|
zjJNEqLr+hCdV0#Dk`)mht;wDuE-s$BxtX4xJ~X5NGv!yu$mTBRMrG^q`(WC4qr=;Y
z!dhGD=?a5~cJa}Ox-sbtKep`6$BREy2WYO%SC!X+SQP5GuH$B?TYWELHTy&)L?}9o
zd#|El#QPLrLXYX$wGvzxzh)hHCEo$Z<yxmW!}4vczbzHWt#9`Ea|Sx0-|EVsm%4$R
zVh58jX8nVK%2(NMy=*6rt_==|+=467%8^#=KfSM(3-3p)tufd$&SAWsaIpw`Yb-9t
z!M_Ja9!H!gXjq9w=&AnaZuzUPA0$iz4T?2THTL1rUa5T#o54MYm#eZ(HQ(hmWMvRy
zJdJ-$pQpLG#uzwO>j;B74EH?k8g;9ZO$MrXO*<}T!8@p|d?}T8wzH02D|2;@RJ}Y5
zB4H@I6h({Ezu=!t^s`K(1t>^@zJEMEp1+GW-cDrHul8IVtI3$OQQs7ywS)~x;_>BS
z<OL7nHYR<)${Z1=Sqxr2dy4gqnj$MPR-uUvQM;Tr@XW}9F|Ay_-~n`fINAV>=p*Je
z6cFNVEw4aN#F9agui)gkJbg|gHQ70Fi=XAdtZUR_^a+rmYYiQvm8NnzCgP_BAoVSb
zZ!d&mBUhgtD`0S^a0rDCkfX2B$+xHLy3TeMI%RLXH2nJNKJaMqJk2Ei;S@$YGUel^
zpHn_XX_EC2NyoO-5a)k3*%lf~arUt`?&)d&v`QT(VoH(w0g8Oct*(ZTOCbHP4i_UL
zus~=@cY=A!ej(lQG;e<O_ghbwj9Mkh<D(Bl@!3-pN~b?lUEh+r(fZn)oqYU2IpIxh
zjSNLY!-<aeyV`BMS(c7h&V1~B4i!5aXdvd$X>g)8HZt<R&h`SO3;nIyZ_>MZ^kYFy
z_Xmr0rVfmclXml|ilcxGh(Qt(5+EQHAqt8k6-RCib9bh<*-n?d4}DT;id>E$U;qT(
zwEFSDlwy8L6clS<sIGntJLo?*DpfIVc6j)36>Tr2zEW#HpU~5L+~%d=Io78PcA$Fe
zpB`;&^JdA(SQut9Mh)&?Wn$`M3%`!15kIs0`1jjPp>8#aUeyEUlP6aP(=43){66zz
z01##8<P>thQA2j(A$?j;PEG)wpPo(z0>lasrLr<0z)0}U$S7+Yzss~MFjK(M)m2b)
zBVpMj!2j#xkN`ZD;p*Yc8UB-i>w`ZcruC?APc<#uWFPtwo3Y-k)qQSVkf(xGXBc&%
zV;?8)creXz(dctcnC`N3>sLAGmF=>DsM{nF^cg#x&Lp?`A{u1&3R<id&e3=zRb{lr
z&C#)evbP@@s{nV2jeid6NguQ`XPT_>I*+>Q`0C9@F8c<>f=Ut3X8QFA+_&KhLTVy>
zFkN0b$NX(9t6~~mD_4hWgMU)|?i~{v9b!v72qSOq2ZlTY8Mc0P0wg26f}@U|`x`{V
zhUzi+=43N2G0EM1vw+(2Gt%2EV*gH=u`X<0`gaqw#>rg!7{yr16tq)E)ChKVR8#yA
z)}!aGK{++)@@7=+<IIz<EYdQ}c)OPR_G|4vUaKBy+s}^jevE}#U(0&XjHOtonsQy)
zP8txhOXnwR%SsYfQ!W0$u^r|)<D(*9b~<mxD=FYtb|IAuVN4z#OX&;S9IO@lb9Q%_
zHL~(?lZDiz<?x49r7|;R{9(7FHS@dDI$DElzY2%@I%N#I1q1WuTs#?a8wnSgo}U_c
zxaJbwMJOE>f&~urFD9-u?&1c<dR?cIQhJU;9y+Lq4QA~_TO~bpV{n@?P(YwUwbTsh
zl;7#ngPd40!Gcl#DXdx~x}OQTb%F$=veY+}3mD}TFmvm0o}Gao`ebwc_sqejm2XDT
zCVc!b(9x5%$xOkglfD?v2luy}AndST<Kyr?Qqu+^&)tb0jXKMNCcu8ouK#h=PENam
zFZ*5Ws*{tKl7D^r^yy+vU~e!QCO?I;nHk-1onl&*8JX3>1pTFB%kjZI{rmS}(tRDi
zR|X9?nf)corWZ$o)1{+&wXQx|7BjOUzew`wUtLVSqvH0zyCpRZeC;(|qKt<5Y^COJ
zGXaCl=;Jk{{vc@w<x3;hYQ6LL&GPY&3eM+vPaPN8HzvLniV?QWwkJsnyX>j~f&Jy?
z_%4^%ny@2HJ)_^}AyeDqXDF=fY;lp1PqDG@9eagAh%0z5zpX;_#C{k1SdK`C?K!Pi
zETyiR78U98*z&U9s`CK_p9HWoH8?o<A^cgRQE79>8~93#TUb~a&?-~-`pp}U-HH4-
ziam<I4<}RRwW~RYk%NBzw2y2)#6iz`zUwwo#gYryV<E@+luDaP!C=_W;PLT{TPFu1
z*sA07y6nS=TlU|<ru*TNym{p9g4v<`i-Lku=NRXC?Ct?_=rgD3E=cPQWaNI9(sg5l
zKj1D;3*P85i6L?B`#z+n`Pz7zwsC8NR%csxyW)-H!K%;5W^*6PGka7a-z~ZLqNh#j
zi`6;bJ}<d;jM}X=TbHOD+20=gFm|{oK@;;Dxi!)X%XQ5DAx!~#vitTbKex*uWt}P8
zN$^kWr{$a%oeyW3E^|D`ExP-SY}Gra#%YO^Rbz%{lTILEQ|REhD7T?an@)tFi2u^G
zTN;b(IjwF_8+#2DC!RsB^ziN9J#JFxm|xCsBlg8k_vo^F({7EE;0AZAaXMXKe~iDz
z$B(k!L9IWDaJt?Sbt4@T_1xZdMLxgw;P34*ahU2=4LDs*B|E*AWOiRL_Udm=%HW6&
zV4{8o7T^w_c5iXBicldZJE#GqNJk^TDPGod-QyWvcN{cOEG<XpBMBJsSA+}VeK<Ss
zsA~(OV_z(>i|x?#o2!ztT6g6b(uKGMjjg`j?#%5()X)Bj=e8GyA{N9{uq%zAd3Dn^
zrn@$}4Hswo!}Ahc`c9~H`xt9<C(Pdjv5mr5$)iSmgVV~%);Q0Y)p?>5v3gFebDx*y
z5TSSa&3l8ieS5tmDl@Joq_UfJud7a%J!yoqeCs*|H}s8%It~vGIOH|7v}RH`bUAZb
zb=ZL%sJpwnoGzEzkE2876`a)-oX!M<SV6FuB<59)s80$CQ8I<6*Voo$J87mmIl>-y
zd!<%$5dEE%40qjqViMw0RJZl@;Jx_Kn>`q<wfN)jv+ZGqmj-+7Gd(?mTbh@m-2P{`
z)~11{P~}WPJSB>spEvKmG02Vw0%ob}%AjMaz0Xv{D~{sz()hS)^6$Lv^#IH(YDF<Z
zQcB9Xf-qKWAkD5{CCg^MX?3A?v(D|6+&dqAAkaOy=}c)@ZPJN2?QedB13~kR@veJ^
zJBE1w^Hnc+=R=^ndOqDrD&h_{@^H47!G~-+0n-MYd>$)~K7SOsUMxN6EX3~~+mdLz
zJ~TYI-iwvlOfwB6fANCP$Mw>SkXG1zz*k$Gk+H}PHU`8)hlhp|A4J7!^)8^?$JwV2
zK}~`E9A6TcYpwgM>bV0r<O)o9DAC7QU73UVB0d>ZuV~~#3W5nn5(nf_SrocoG<V#s
zOmh!6)t|JbV8#Qk#$AU&>{&hX(Tv~KpmM+1n&$QQ8{{jK*;2o7mhDb7WSjHOGaBJ3
zd2d><a6b?l#{ZI#kO1f*PUBXsd^Nemna%(N5F9NnEvh#9Sw~mhJ$#HOz{<ZnxX9;=
ztK_hCkNza={%jaqv98(wGh!B^+vag_w)KL~$ocoI-}Q%(P#kO=zstkRQ9;x3^wXP=
zFjKMH-_xex^RDC7?>C0vN`d=Fgmv!NV#8iM3R>cv^Nr)oz-0Ak;5M}0T99rOZFj16
zGqp1FxXZcD#QA5USBq)x<5Oy3tE|C=T5IQb@7_UDxoCZCzUSte9;YbE6si|%AX}Y|
zPUtViKm0mxXOtHmH|#Wxhxy2dWbg%?Zl{eup6d^5g1mE*=uT1RgR1}5Tee_a{0oI%
z&YSkfQA1|!{4V!__;q06<PR-&1_lQ3!{t1HFl1>1lBe2V@a6dQtkUD3UH2(&Xq)s!
zx0|SuVf<n3W|SP4CoX1Y;Es@y*^>H}CWT_QzU3z!Uy^+~XZLEy$mH^XBt>Sq(h!kE
zo&g89?yfy8>}dR6XlqSDlC%Bc2-uV^l#SPc%)SRKNLU!Kk5lkt(ncqsquj9Bsq#;w
z24wc5`bY6V@kQ(Uw|toE8(|gIYe?vZ1PQ-uph6T5t@nB$DAfYKy-iF);(4fN;Col5
zl1*sjP(+hEg@gCB#XxNV@TY)nRcH)(JogP$_qnpq%_1zv2R155@s?0OGO{BvDG3vg
zqTOkANxkq7u<w{6Xaer|XuC$}uV&VLmNMKQPsttiX(x;?7^MfWMAz_TReY%&7zg`M
zT&~>Z)!Cd~hfbBzWTP7_Lg(_4=Sly0(C>4q@z3WZoXK?Z3Env{pEy!)AbmX!={`m!
zm&kM=cmuht1Ux#LfQw1X;Z#ct3r1$<CTHl{*=n<0qwn8eVIZJOeK^ct5~~HnR_nV4
zO~`=OQ=YR#b)wObAFxqkOYL^X2xb`~M)?FO0$emE4)BCVu^<^9Rq3x*3C~;1C(13>
zBi(_ko^XjK_sbm*@S#SHLhxo42}v_n*){p@Vs+<5Z@eN7Hg?)8<DohSvmj&*JsJ}o
z3vB8u9wl})Nsh9ZpM&oHm;U3NyA3T+|1~Z6c+46F9UU{7SYRJhy%Jb_c)Od?SaH^U
zyFU=L)PV#91XP+z?PBZlbuh{UTgxT}j#gH_K(y8enGNh<Ki=+H(@Ka3$N*tQ07GaZ
z;TUdG_~>z)o`(zKj!PG&^Oe-%4!?)1+l3DLNYBsD3>vVs%d{%&p`E&l*G`cve7vcg
zw}C4+A|auY>_fv<CZl|AR$o67VTUDZKf!onZEcNg3@}u3cW-`Wyg&PMTU9h?lSzI9
z<FUg_T#a2j=!<>$jp(?1;>9E;QoU?u&NT;y@m`IUC^n-5@Iqm^;I%VpZ^Sjcndk{R
z`o+O~eBf{#zH9VB;==aX;cGxu#{32)q61m`LGNh8bh34m`-4!VBK`%LbYdyp!W|fd
zovY_2xwv6z4RN@+0+vSZ7D7S653SO&Ujv%m&BwMw26hvk6#0N~dqwYRMq(Q}{4PN?
z9;YOQYk6kGUgEAW+OGRrkJ;35UGQXy`SQqxTVGh8vXG*vweN9@HEbj!GdGPpYU73K
z2rf1@5xQ<XvGM2yF>#7rEHmPhXJ)zp<};eZ<<jvH7Aw)W=Y#k6!+^gH^rsr`W`3cI
z2$PoHbfQJ{k><vdyDekVwqIYoP_@b#c7S#oe^ghGo*=;+zF<x&(;iNJ$z5U6!OO$?
z6L6HYEZoiBB@EhBOz8qLb|g_#Ro7l%F6j?BN%ir$HQ=sH(}7Id7}**TNq3mguEW7E
z#YV@+1+Ryg<SQOKJ4Vg{_LuOxa!^tl35(81)O+}Si$i*U?~nH+G%ZcHSH=R!*bRS1
zy}#8Y0&x4AJ@?i~O{2RrV%VC`QDh9C{^^mq15P1BFVx?%HNfWuAw6nDRJ1{}#cFM6
zKS=72be6L7dLL?dM?`_T>FJ8wxJgSQDIsBj2Jc;YD|l{vXjTahg8_y){Miwt{Gw1c
zmXm|S4}n};7pn4bhse+*E@?uv2`UXTMt4ecyxS`(4*OuamF4C8lZ9=Jk-@<zcn8C-
zIo%Edh}+xCy~Vb?ymcvqC15u=hWQ1WOqtm%g@=uci;|>lrH(m2Ir&%h8f>Ovz%A9C
zs$q?KQdq03G%~ITX6LR>s6r_MA%m_ye>QA>)Ys9mV*AD@mq<i}ztG`VFlkfJ>jWh5
zi^}fZ|0(tXrB;8^w4ce!;L98Z>CNKZ1_~(F0pMZaoVlYcP&e@4b9T;0vQ$g1{$mpp
zgM%(X)Lx#{(7lNXe9+pONvuPHr9Le!ZKaO=nZRBQGgzW)ZgP@pJUYZ|kr7)a-%LM6
zqfDc0#;)<-tK+W%FV{%spG?^;|M@K?EhU$5`|F-&^@V*rK}>OB$%9BFkcg<vAonup
zEYa0j!a4VyDaew^@)zE~1xQMrKO+WDueKJN32g_C?i6d3P49hgw@kGJv_U{X<{{w|
z`Y#EFxpYrK8lELFonG!znOOCUS)M}52`l%R;f?&W@hA8|W+0dwm&|0#`x^p|IM-tm
z*{C9*?YRH=@qKV{LQkb@=M~)JKk1MB#9&%vNFwTc!m(^Iuj`<vO{%rH%j^|vw6mS>
zjUM7eH%y3$iDjSDx_A+N!QW5^@*tPfAyuVuS&tjBK7G2D;%cO1gf|<AapsHN^<#R}
z<UJt&6MtN0{y(LvOhyhs&8fu@8m8t7H5$+*=|ofl<mw#91^*-Svc?al1ENIU@JV!a
zt7q5LOnh#RMGqK?13GQvIv*a>`e_2XQ-mELcSr$_f48r&a<<PFA&$%I4X%H`0`Amg
zlTXBPBm@TA()qhp)DfWEdi~#r^3Lx8Y3$pj*Z=Z8H%#mrmzI~?UUSWemk4GkCGP-)
zCK!HMbP&i;s+zgKLYP_%6Kg0x3GG=8efcj})Z<BLzD`|JwYmIx-C#}?a6So0FU)id
zX;!`Z&ny$HM#LNE+4G&p{?YZlR=7(dJnny{4bA!hJalEBBB^38{lC<>XLp*QXIeIW
zj;(o>Cg^i7*nk5JC>UC(I=)<}Wcr->GYG_k{e(H}Hbz4mHLUz&tPK&S5vTLnjk68u
zzjcc{ZH}+wo;kQ|RCrWa`;gH|qhq9p{Z75J3pXE}6>EtIWz$3CEn3fdONHh0r6
z^2{&(R#saZWM(+6e*2~_e6vo>z#e;kKRO7#aXw0O?iE1ez3~LK&n0jij;9hH>iFSw
zdx|@E@#N;qYZg3MAR6c!tc6FN8qbDkr(Qu@oJawSYCJr8|0>9WtAYO|HdXa58rkf|
z=NfkV&6ndR4%|;H+4n{oyXd8QDu}hRK{ZCg5CI()k3|}kB%z>>N}Rd$tZk5t0DO!%
zTb3TEq0pP|)faMD^hF<c<JvRrO39(Hja9)%Qq1<VeF@GmrHkaj%x913ciZu$omuUB
zbT0w~LCV#-H1gs%=KISMC6BOgqnH_<(-pA|?VFXzlEAm{-6vgtofibw{?^8mP#d58
zd8~}ujZ7spArLcLdz6;#GIn?Gsd0`wQxahjL)H8IaZeDTdmVULn5$umnupc)kV0)p
zN^OjfT#!A^c?-NCr*|2h!2(pH=q~d$Lc_hcrdM~Wum|#Kr>^_0BjjF;Dk&I!y+1d{
zhvgaOaU1LxgZWzIa_x;oWhF$_QsAswDPuienpVtd`S2m7%LT6N*N(AvQ*-AW(FVP7
z*+U2Kbl9%=O?Og40@g#HXJ25Mfq0rQn28_id`~(+Cb#!})&<=if=7=9@KrJEfBYY`
zH&>Z?v<XCzy~5oOhDJuv@eKpA4M#@W%>vSY@cVIWG#7?FSsKxd=GxB45D6OaOHIo*
zs_lRk6ZbJ;s^TOTf9r{?-DWPzX<f@}T(S5SteIi?E`C7|G`N?cZg8O-6ZRBS#d3%2
zbwg4wjpLI1KqU{FP-4d3^9$vOS(_OVP1VKUK+1E`;>qP@V}Ov`j#4WF{Hx}^0ZPzm
z`S#!2BGueSo2lu?w1yJJIbld{J0iy`*Hp|l;`7v(fIm;2J6oPY%HwC;sPEsW7}6V%
z+*hYa*`nl~7_@BX7XlA_z1*Z}^JGNEYn&VqEJ3xI+YjEq<3Qiy_uim(4`x$xNprwd
zecLg3nW0T)H`8-=+)kG?eBPt|3}3e&t#5d^bRMi7vX!IPxKh^sc3*bUo3*3AyLi^6
z6F4Xz@2Z@Ng0BxQ_EPj3rvwzh6Fa5467xG>K6$jsm-}VSP!wL$z_Z$79u-5-mrh-G
z)p(I7ptK;ldaK{lQ1LafS}4vn*WKfIND@*q`s~3&V9oMEf{`Bm-A?d+7yVWB?N4ZO
zOSWt~hU4W27O`^W!A6Ra#je-o(Bz-XP~4L$_v;Y?NvzLktqj#BS7->`V)1&*gN3KW
z$j9THQ_6AOTt?{WF;RMx!y?Z*d3B57qZr;cB}sLc*hvyg|4wI{hj!Z~1e5N3HNI5L
z*70{!&%tmBHD7Pw?G^QtG?KtG84nro=4$LVWeKo#e}3*z;@{gKxVE}mB-<wN`a0H2
zy~ce{#3bJP0rjvZAnM;SN|fnFd)?*!loa<Z*o4{T<+c)^t7DD^+UhLGWd;{SJqt>(
zlR?LaRH;L!22|?r&jR^cA?hF_#0ylAfAt&hshFr%-v-C&m+$c;)_;&c9OeMpbBgok
z=4L_Me<P9wkZj6d@^SoVm<~D09Zgv{Ti>Y#UbgvnSEXUMNXm*WVfqZ6Y66@sN)sSU
z1H5rpNFjWH0ceO29lH2eu>7x7{r^=JIRg||f}yeI+BezAT)<Sc6!Y`GnIF4QU<&y4
zPusSH{kKjL`_)okxDGf}CAp%K$e3#RhC<n#;@d6X`@%{7I(0z2zpT96jS~g_`JFUb
zdI;3<|F_NhuPOWARrmi8w=uZ;Kl=Rt?9TqLn!oT9z@tMY?B>6Vi2#Aj97+C>$}E%7
znS-9Rc*tEF9ApxP8fLIq<IkNgWnzaBCRy0|iC-3VqP+U2pToapz_+~f#mA6eyjpwT
zYu?+ai>on=#=;TZj2x(A&Sbxmg*9Ow&rO3QA+^&8cN{gc>MFyZ7d@~iAiWN=#zxwA
z1?B<3e3V0Y1H`<gnN`|ENJu@8R(yXtYxLWS$yA#kx7L_rfZh=)t!KD^6xecSyAx+(
z|LDYo=+^2A(WCO%FQC@g6}~+Sc0vIOJl5cJuVJ)RU(;h-t(k_TDSQrl!N5ph;Mm#L
zuM8*w@<Nfb{f`6P#|G=o57xYpqJjtV=f7oe&%^>u5<w>guUzjL#LQ8!`d``A1tQ*;
zabliyL!Ogy)CM5__7rX~-Z(G2s0}Qhuh*k-;&}!Xjb~siKA3oey1#RGg6Q@?g{yqW
z&#PSWqbmirrYoCTv;F}Cvb$tsv4&VIkV{z})jN}xli_*8)XFX!ANIh71x#UEVv#uo
zk-4?`_lrIwdWt^wd7;mJe~il+>Ud0T<kX6Riu^l<&MGYP-%Q*YkoNw0cK8Bh+rM=D
zr;xf~e)u0O_?lmTeVK;=O8>GS?h|73plPc%KL={E+zFNqf4vKR{nC8-l6ac;TuD?I
zg+N>ZL|WZwW?ZwsAdLRb)hd|KLC`Bc4Pac%JS9_ZnRgt#3SqxL5DAF~Sw<1F69<8q
zK)cEX#U}4YeYgZ$mV&qApLRS2!H#<X5Oa#|e$`16<;sh)8}bq*=kgitT|7B`A#HhJ
z95)(W-5TzJW55EP4Cp1qLNB{5ToGX07yk^ni$ApLqk64zqSJEi<M{6$G{j9_aeAFb
zB511!9=9*>>c=S7u=R%Pl-IIK#lf)U+tP=^R*XjSLbJ;&kAEzBR{%`e<sz|jyR}dL
zLtB$iohHXt3iWM(n`_Q?nM9JRMBeLFiE<7cGQPR{j4ThwVyet1`zCAGFjI4Q+A-~R
zQj#m7ocnbIuzvi0N`MElLj8g71=dYH4=M(#1N_G{D4!EVgjHjb7XJ~2PSM39WAk28
zFSdHawR7uoJS_(!CKoIYU1b1{&AHn9TrFDnL&b+WjjHG8*>5>~o>NcH8$MNgIgp$g
zr0BV=FZqxfu0;M~f!FSVTNV|y0TlJB=%?RLKz_ap4qY^p^i@HHV~!{u-xVd2-ryWG
zJ>={iM*#EWcXpRutF)Xa-mHZ`ds2&e`&}31-}Jxco8K9#yj6R}-$@Qg4|?ACfAC&T
zEUy@#6E+4}|67eY{*Pu7C8z$M`v1}WKh^)OQeOYx@BTlo{(l+lzmPL0=6^tC$(GOB
z>T7`i>9=ImkD*;%<>d<a=w7lw!<Pi%8cxXc-dvUO!5zCowo0i+*?%c+7XNn?*rP*7
zecxS}o143p8#pqF)pGe7w@_^cP9wrJT*o=wD&qQYkQID)`2Qol{|EX1KUb5T4h@rC
zs($JHI!$TcJ@)x`z{_Z(Pz$|0`E#cM#^$k_(PonpH;NxNyE}GRwIulu*K7WN%UAU`
z*tNc<H1Crc4%F(~!K&w#c>T(cPh_+h2V?cvz$kKJY$y^KkWc9&u7z#ppCgxf177zd
zI9P`P7r-|4o_Od}oR=5TtIvid)$7tt`dOF%9(`DY%l>^(+3Nq(8h%$Tf#KL8tqqTV
zbRKXCI*w8(;Tb^6e4V2LA)k3sPPSd&sLTGX9PyNjrHUvl*`FW4%Xzn)rZMfg`i1YQ
z5%l2@&6D@b6HI>w&z&a?J|CJNFMS~P>z6j=kxdx38l%k9;0w(iSqaO)3&$Z@Xo+4b
zWVlQDc6XUb_R`q7#nyglNKS-<^5ervNa43Nn|a}F-mBxs!_w)-9OK)5^|tSCbgPju
zEony)6mRimm1l=z*LMFe_TDloj<#zPZJgi)3l0hHf#6PpdvJFN4Z)oT5-dT3y9bBH
zU4sUP;10pHahFp(@ArN)YtEcm-<oySnLp=G*Q&0nyQ=mrwfA*xnZ6XgrS2J5!(4Zu
zBmDzV^P`uwM&8+uf(sGL$rf7fBBZ=7`+Dm|&Z|&fi+0uDWm$7Rl!p+m7*GqZpRm8*
zs0=*F5ab|yb}Z=+KhUvvtF|y#T83hT1d=0PHE7-po~pU;DaI&k)h|J6Gu}@gEMRpl
zX3ny7M;41aJkpik1oizYcZ)>QdPr>egWYgRD((Z9<<I%b2N%Q;00-w&bU4JPKLaP<
zZV`0z#BXxl_deaGSF@_1a8INSbo}+t_B)3%K`~-bg<Gs9{INf>Tzg77>S|c|D7F!z
z#qk9$V*sem%hZlOl--{b<nt*a+5KH?f}2e*GN=&I-v1B!aV2H+{dE0O9CrdZa>P+>
zhNndKGn0+2J3|2Yv-_MZYHM910%%fLs__tgwdI1BVWatTRkD!X(tyeoSz8yYb@QDL
zZ}+Y)=Jc|Zlei`}YgX^0o81}r_{c$p#PF2Cd>hvGfbUb$gOgY~d&!=M{;P)l_a%+0
zh{t&|f~^$60haxk6SkZX6vM(=+dJ2;plv1SVmkJ#e2j>bs9jX!slN!INA5GXht2S%
z=8*zo7ysFw84g>SuS`E9>;^Flzk~fM+jO1h+JZ)THZpn6Kq9w{XH(MPmTHGQB6oXv
z(T{6L;s?0KBz!p~OjK3Vg^i(u?NH@b)9G+|jN%hSql`MUr%N)zj*0|{L|laI^STdz
z4}%yV7NCm|q_1mj)Z=M7+MX3ENU^S#9a{N^v(3UNAi)aS8MAIsh`k2SmR$qlYB|BS
zxA1(Y?sXSsqlKjpFi1PPe}DNd0)nQAJnoAl(7!s`#~Z6MQy%=OV_AB9z7~NpSdN-v
z?YbE#9=)teUUnFlT(MISmq<a3mdESyut<p|<ZX9TP5IGM=v}W7!|Z9_cnw-6TU>
z{r(G>@p00f+Q3fWkBEZnL<-EHWvGN==0hB3p7LOP2xi6ih?3t6r*|h90m>~}n>%mQ
zHQ>q)q|KWz%2JoBmh2J_rMG_;Ri)vPx^G&fw(3>I;~@v#^tqI@E}^rtJ*>MB`%n;@
zS1)b$Ue75rMx93iuRp-K4br6G;it^aRV+OVbC`9F9r?M5)a+tZB+3M9E(>s%0ZZq`
zE<Kczc4k~K-cX^lLl>QjRgRm>qh|grX&I&%=0@kwCzqfwX#Hwx+F#IegPLDv>awHE
z7;2XI@*S5a*Ee%I%!F9Vk5-G8%0TuW9~0c^L8}+&nq+<(k0#5*bY~vlES)bi>SYgo
zhTmaR>ENz$+jNL?Z5<9&#hHVMLH?spd{6SkO-o!^7Y6Tr88OqF7gS0%ktY!ynzoF}
z3&?AX`3b&&w`puyW!UsS;*+`0{76_wVQ1{jVWmHiZsoi=fW;qEEV@TQ%j|BBmgq%B
zH(F7yG_a2DZX_Rn|H7M=i6r?^YQ6h>Tevefb9;^M>nIYBv2f4ks*;%{_rGTfhWrhg
zSF3GjY@rM8gXdt_7h+k^Q4xP=w7Y9925YunT%Ngx<A<lBfEH=Yw0iaxflq|6!{1jV
zqQ}rr0dg_HK5gZ_$QWZ4iSS)q)u(XgmInMx44l<a(C^<(PK+EvVqH)Lkw#ul$Bi0s
z?>Bh7M}n9jIWlgu60L=W0!(-+wd{O6%TlJWS0B0R;+vHUx-ZE1?Nn^^N8_7E1|omE
z?d#DKyA59EI{%q$f%r@vQ@bmYswclYKC^4Q9Dl61yU**+c%P4@t3_y9&GkF{B#^j0
zuD)jIX1n>!k1?#+IXtq(%ZNzYfcH=VUR#TR(|j*W{(EcUfe6gprK;HonXF2LmUIKV
zdDLiU%=`DGcsqU#AN8bReUY0bJwa(na~s5BTh8*S=*Fp{@XR89?X<&yGKDq$C2jN}
z*GO7@!q0R*4Pv~4+;@Hxp1<RwB2KSNDiMB{rl&_UdcXzOh1A#Y;>?3>NGa_&$nI&r
z>~oV>+Ed@p7k$HOQQ3c3ZTtS+JxA!5cJ~dc-j{&9ziMunM~xqmR-<`D=)`B+9=vXf
zhdzR$KB#kwp;a9DI7Y#7R!6fMYohp3zUUXr&-UWjAHhdRB0i+AoJF|L&=Ih;UV2}h
zCSAlGYOPv>AF6PnF8qg=<&^}CVfeip?5D+Ywk&b8nSk`wcQnBT9^B50{&@c{>2I#m
zMDa*xEJx><aqSk4_`PX)*mySY<G-Y@(^B1hoRaPeq+%<#(^^Z<%Ug$*UCV@I8OyGc
zX;oN!FH2Ok^=8w6yrbigc{xvm+~(F`6T0|K-&|CHUQzg7EDEt`(Uwgyycb_QzcxS%
z)431jR^nSE_PlRa)7To#c>~XO2Ps?lc(mqM2ELu9eyh`r>mMqp<S*|tm#iE_1n8SH
zoiD{HT5PV*BRh;yS-pbu(n}56f&S5Angrj?kH+Ud^xgi`J5&B&nmOHO5sy`1-;8{P
z305i*vfzzw<ZZ@dW1HlucvC~ny+jGf4u!k=u<!TFxnLQM#RAT>8Lg+rPaL!9QN{L(
z4U0s6Wk`4K44G%r?gs+~mY09E*`+b};6OIWBo+~ax)$)D`|ZRte|Xdp9{w~;QRI)q
zD;O;JAhEpClM1DlMFzy?PEb%cT$Su;(cP_YMfGnO;3>a1B#11TqY;6Z5uh6aso4MB
z<+J%EZNN_??VWvpW5~@w7k_+pNC<okMN;+s*OzD%4EWH@8O$+;1~4A29-NZd1j1wK
zR|I(NpyglxAzCC0@+aS!P1|bh?JRDrV@)PLDsAV7($p?l<MOD%__qSD`Hlc-a?0dZ
zxNZ2jRD3EhT<8&zk@q0OVRS!gc;UV!!2I9K&*`zsrlZ4vxVfxuIW)8*iFtdZE_~l(
zqx+Mjxq_ZOl5){w{@D)&*v9dGzr#d2cCCQvP7UkCk2?3ekmj@YgW1^>9=?%o?$RLa
z`b|Szj-xi}^!valB70XA`4`78?K#o8@i_O9emRUz`r71$5<n&X@TFiTC!4fVy`qWD
zs1~UybTk4|9DNfX$T3;J=L*ay86_3DH(OYk04pfUM|20)@i`m^3JA4?5#?gN5Op|4
z#3Nb$$oK-3)o5j-T-!_fwF52QX{leUkeMg1U%lwhb<%nKg%2w#XfPS-4v3)Er;4_@
z=eO#cH?=l9qJ}Ms(9llvKPB&$=~@p_ruNbT>4<Y}XB35Er{Wds;`f)oUc4qUzZhd0
zkSCd!s1@LKHz3%ngdA=Uj6a%DIvB8*cwsNbS1$aeRoJ6!c9`0#*9(FQq&*$CevjK)
z%9+zIu&*E>Rc6s?v3Ka{k9y69z@Uc0+eQb9cc=32ES;FsnRBhp0(Hb27EW~%h$9$t
zI;*;#@~t9)@>_IWbw7Ln{T>lYPWD``YjKu90X=R<N^Rwr>A0*OZ+ZD#7dT76`wx|>
z%_SF<Vy|7Fg}NFCH2zUYVw=%9**8+w7xJu$csvbTwyHk-3{T|$<z!r52_HK=YJ&zt
z;oJS@hm($LnZY~N)H*x$5G=RCffoaO9@o6dr09F4RLnW93Q~KjNo)mq6cN_s>wDj{
z%R5WgISiBpeI87U?{~=FX}p@h&1_3<RUpcHPwY>`#5P@wgYz=OlZhaI@RJo3#-qqD
z#dqIO4aid@mbie^q$$mD&m-*Hac?-(5MDypJVO9$Vl%gBdlUR_d_><W$ji>?uCdtp
zsq)|B(3@2leztS0FwgJjt4=-l7g>5oOSyC86DWeL(K<Zz7yMPM+fx@>8ny)Rg2F<a
z1FGn3lP#aG#|%@S6rPl;iljfL4d{*T@=LRRaM?{ybvs(R$b%`AjY~xRdWbscPr3j*
zT3=m4LMl-%(~|G@r>y9M7N(Qm0b${_*FKqVv)#cR#t-1Tlb_$_ED!tc*a`!SJCC7V
z!O{(Uq#z1q8~%kw`{g!tvMlfl?Bu7h%uD|Xhx0_m3jKg7%f*$er3-p>i5pTiJ^r0a
zFY&qhmDFaX5ZDNl)*S}rl#!*}Lhxr9Eb>VK)Q+q=YEri#EPFjj<ViMlW#$7E?TMCG
zd0p7aT>8#~Ouf)d)YDq#zGWqDt-$H*KF6pjH^RaT94RBU@(Z#eFG4W1C4CM<tlcqp
ziA+g(%kh}E^6_DEEAl;_SE1aK{Y8&Eg`J(A$=?dLGamE60)tPyKk>AyC*Dq1_E_|N
zBgv#fg;wv=Wytcp6E5s3XZ~}xD*7-Id9m{KLbst!$XBJ_Gxbl`#OVUYUZ5K1RSNgo
z$U&BbLu-BcqCWqu--c1UlA-3b2B0OEystJ|r4bqvv^=>@qd)rGHxAdD$`P}X(YTUe
zM3MJ##=VS%0G%or&s^R-vGtXDVY2NPIGkZQ5zBst{(2p&;ZH_SSgXy+^$5qBRC?JQ
z@NZ*0aM6^0@!5zMnU08O6XYGI)I+Vc^^Ni2ap=E;oQRR=vn|R5FrZBM1Q_!&e|3W3
zjahQlKyn1mhNeSeH4>+PX$IvBN9HOF65-}KUXIG<1T@x;MAhQnS=C~&G2>fO>EqZj
z&z5hNby!HLc5{fw;4^;La4SG1=<tkxoq&hrkHo1FT9;;Bl!u66L#Jo{m9{4C!{BgG
z-BQ2i5AvYHs!;KwlzzM<Yt2(m*_NemHg=G~{F#d1=J+{*h~IX@Uk4MBM~tYdksv3R
zN-$<e&}cFw*>c$3kPy^hoZ^Q%Q5n8eW-4wjbbhH%`KEw9`9^KtoeO2bUULqb>PA-`
z7aEz9^o!x%pF{PX?8O4mBNYbIOy_gqQMlG%mdx9w5!itU!E~TtwQbOTbVD0tb4$+c
zDH`G_;HNX!yY90J#a48~+ej0=IWk%2jrEebXz9C|PV7pS4xD;m+$1%5yuVqur?fDF
z&djt?Z<lWf53@|@DTyCr$&s-#Lk;q9Hrd1*wPQ;fIL;E){I{Bu->}fKbbx#eSUEuG
zGBbceGFiJ>ngAi6Gb|mRMw^NUM6K0v;eB<mJ*_)3qhAp<_9xB`X|9&VEIJ)+Rs%Yf
zZeeOSQ&#<znN@GPxVG`I9!bl~WGi%HWtdD44BaD3&mAsq(VVDaJ)oa<Yn}<69;ay#
z>1FMwKh={~wp*uf*0E^*T%243&8taeCaO1XCA3YfFJD<;9%rtQoc+~h<u`<WYide#
zXPDw(Rab7@-W0Yl;|o2wqRd+tBs7?lO~YQi$euFf*mRV<dAeC2@OOM45&~XxOp(vG
zx3q$J7SKi}NETcrHE5qbX>JE#!WQdWYd<)YA@!dykJT-X-v!ySN2a)1@aXGk)h((b
z&7oS%6rb_<(Z5|#woNC{|D@x_ddK$!o9VnnK%T8v+1??;oNLO-UFzy-Eit+CTdaWQ
z66EAz-z}3LIw^9-PS+lgx;vszgMYd4mw3Zm7nD)Z=lGXZl_SA#7Uc1chAs)rwcIW4
z1UzfyBKLn^?px~=mV6C+Us8K~#Lge2B8f>MR28pz-hgmx`7_zVi`UCXDu(c*p6t7(
z%gT3uWhpYp>(gn_GOBhL<a4N2VQ$0c$<|Xh(#@Kj3Vh7S$+QzIcK1gd7E(r(S?M$h
z-UHx*<QxT9@ns9BW9NwDn_kVfq_S6mO$QOOhd&c;85O7h@2<(~t#9Z8$2o}adXxoX
zq_TnY+qYBSRA!aIh=q1XYc0PR^3|h`J_$YJTBq*+rlW_2CIqiD*c($^ioQK_AgX60
z=TJZI{Q=R)CP8fgBI?FW(a7H({6#cNkv?-z02aRf3Ko5`EHVN)tfF@$9xxWe8-X6@
zVN%DmwJFbRvCzlr#;Z1#?e1O!&pAt2f-PX5nmoQ;vJom{|6kakKaATsxd6M{+1PZW
z<3ncWL<lbhOZ#i<oGHMxl=PCGdI)oU2Bf&AY$JEK57$eAu^YkjSpz@6WZDmwT^tN-
z!~({Qv=V6zVA-s<O@OkxRk`ZCYFi=mS%Bq)<ggO{Gw`!2Xn7$T#_hXmFo7ITgL&ga
z3Bdqb{YDSw={v_39eX3r#G5LQm;1KoNUgFzu~>wJ4$;xmt{Dx+!b<L5b{~x;{wsz`
z4m9?K=mYX=l`Zm=xJto%XP3;vifTJN>aoOk&&ahlfHI-{r!q)?Rr=rQkWYV1{iRIG
z?1er2_S)Jg4_Hp`xJyC=&LYL5ttG8s0A}n_Y8ksLIv2BKx=|P%t;>B*$-ZoP)v}dQ
zBs@i8j|$JMt}Aa5HI2d;794RlisJ<pvA;f!va?q%cy?syHsem+KM%?-d)iH%8+1_>
zh=cRU{+n}HZ8u-1N-i*)%|y?zzPh%e^t*Hv**Pzn!vOjCqMV*BLug*g??w_<J$Kw_
z*mt44{SV3ip_)HG^MC#U!&J{K^s|;Zuxc;q^y?p<@Q5M<NR@V962IOD$cllV1Z`aV
z+uHal`*vRUbgY$h4d^AYNO61Z2!Oq{-C7qmqsTp@(6mx9i)RZF3#Ws*>NYd`;8*5V
zXnBD=G1}S2RljOBdsqlayIz3e{KbdO!xSn46Jo4!;5u$uK&%0SXhFoNL?7&t*tkMp
z2fv8<DI;6j<Q6Kb*57-2j`s@aR{fagb@iq(Z~x|OJE|QL2Wd1cFV{HLI?b?jR<Mu3
z=?Z&k3{uaTq<D$l?f7+5M37=EDG%!h3js^%C_IN8D!~U3d=?Qd##_W6{!FQC0>|)>
z4ZtDeTgWG#WC|YMQTdq!JG^RZ_a}q08dg4SH}z)<?gTWv^$(F2_zq;@^*n`QRr8;T
zHzDHx<!<r<XKRYdPM_&BZ=m)Sk<#LfPuEQ^j<vQ}F1pPt0T}$%>a;I@6>ps!I)5HK
zuk_UoV9@+_j#LA*oq|WfK!g;)(0lu7i9Aex@$w24fhOqIQu9?9e|C(oR`B!GS<5vK
zn6$+<B40m}(cc*1!bU?rOzEfm09+5uE1@I0XY^L4OXa`vwEydO`QJLqxHMZ&^Zh2C
z=wiz$Hp~4==ozAB8?RViwrS_coW__CicDBa*rk=RLc^{uP0=WBjnj1>eRie!`|^b;
zt(KvtM+3x~9XY(Yt0)yfs(m*lgH;)H22TEw4>B5U34x?Y`>APzGH?c*KW4i<K-&9Q
z#~i+|xA1ChRLDDZ(}F!qz^qVx*~>Y(Ik%AK@lrJbsnu}HreOj4HpxTy{S^W*!#`!b
zVo>loYv><Jci#Y5Z0esUpU11OfNN1dy{zAv^9KG3TtsyFmuHc^%jJJKzWlSjxy$+I
ze~&85=w1Hff(|PK_SLgrnecx7`84SFjQ-BE$bM+|i~w@H#~V-bHm!`973J{&GyQ}6
zm|*Jrg<;2u-<KL@(gK@%y3qQSxD%tMHSm;K1twZBj?)BpfPztepH&zkn^E(aZq?x^
zrHEdQ!^Bq4bj&80$hXegBzYaiacw)^IZDV{W0cV6>^f5<y1?W2c;;eqiM6j5G$-i$
zcy>OVGM@V0&uf(4IkD88leCubGyGiJ^nF{mXV7SZt#d6T5zG^ASIBG4I>Rg7^%Zzs
zpJA#A6ps?N>)uBVbALukP}WLXyf^af4>gQ7GLV9#9|19N4B4ZEg`gu7$&auh<7O0O
zwUGc`X9uD+zf_mJnV`E?n?m_}KSxqPrvEK_^!SDR6-8Mj<tFnL#Ua<dNb1zHm`=gx
zw9Wc&u34JZ$RnqjiVY?_sYCa6enTrtkGr=&^~lB^IpPN&Zv7C=Q@yJRu`pOTL#zV$
zz_7!ZJV^54h}9u`P1@^$a@=r57#>AiZe*pAb89~B@BPEc&(*j1_<qw+#yQ3E`n2gP
zOby?Pnbg+l#0Ooaf|4!#x#5afV;ehDdNcQi!@cEl{n>%EA*f-=8;M8DlwYk^FUOec
zM%yNbCiv+Ex`PO|tP<dgJ2Le9*U|=wRmUMC+i9RNW*%Ai4kLlehju~dI1qCEU%{v}
zB#VkLaz_H`>8H#08U60IS8m)Ht7po#Po`E_piO_x;O=O4G2$1i)spC~Ee?Kve$)z1
z`2W2N!920&!c{Ug*1#!VId}=UZm|KWhs$;rA-%h~%(HA0FDH@P1j5%Bc-Vw<vB9u(
zQPZJ^;tq<0HbXC`NI)<aC7lytJh0JWy!+SOH|`jF>*T`YYr419f;_Aunk0nzj7?_R
zlrT+q0XCspyLIec0+V!~GwJYDESOuQKoz_Gu^yT~`QX#e%`BHIa{0D~DErgei7dPR
zinh-<?_i<`->#zlKE;69V@%K+0XkcePac<*h;y?8Rf+cV@oxS2WFuQql#4y>!vUa%
zPGM#qDEs@_fB>^Y-rLdkg1F?9xfr{L<}2bcVGOb5-OL#n`&wYmcZgZ1;4rA-^Y}Kq
z2!G%Btep;c#GuMS?qg={UAgW&T*a0OlI?uj-BKrG7jo&W^BiU35ryEmAd9T;#6q|L
zAI%>g^|@cbCC86#OEY9lm6sNxV$5@dwsoOP7=l>6Zmj}+%SjId{hEf=o{G4X{FO$l
zq3P0@5TD50NYoor8|1!o&~2|PvI8F%W=BzJF6fAy1Gl4pC>Mlfl@1!?g(CR3HfxRy
zzVpEJl;q!t_tXiVL|`_6Sqxt)rZTT`Z^S<|Rn~2KUqO=+KdE4O_JDsgqgP#%Hy1Am
zA&_wyC-QZG&-X)yv@4vIT_GrM3$+Q1`S(E-Npsu>Gc2NPe!~MS?a2mR9^hAkykwCO
z%N;X&iYSQXFjR56t<+pu=HALX#qC~iox;2N-R?u9H$OetZobh;Ym)0drFXT<@g4eQ
z96zIeHRSNIkUm)Catqk-9L}x&d~?@%N8yLrVGuRyM(gp6sa~atKprNqzSq>om%%3I
z9;u7F-*VM~`y#y#6s0R!X9qPf{rncq>^13Hxvxe85;`;8KMx*04t0dAEUHZRx9w-U
z3&R_Cu2&1vU{%GHvXYE#ukp4@wbvTkr{r~e=}5XA)Rl5i3~TGmFZ>x<H2B0k5R6ee
zXW8U$6!sED<lgf0m6q>vb^ls^J+W(<iUmsQ=id1$xH`xj_Ly89QfgwrKYgZ|Ze~B^
zyJ^mBWc$<!+*{wV*1HKrPmX)aJBbmw3r7=p`vp^Wv<>9INTejO<m3TvH=6zNo}a$Y
zUw%{sfro~A5<8~hQlHQt<xLgwDn|wNnXPF|N$Wv-8uscho-dZatU?rTf+w~?3Sn6B
z+6O2+nS&&p2e;L47Cu~8{9#*_xOmG?*0WlsKDR8>f!cZ4Mc3YZkcme(YKACY6&xM=
zG?Mg<Dz48Z%Hz!8k!I#A`_<7r`+8XtR8}~*IUOuWZkm^yl3IPbLQISsc=}kDtX7s;
zH5tdc7!O=p5lZzamI_^z7PX3SU5tj9pk$MYp=77IQ+vEfT5R4ft{oI~A;I<t7&Y=d
zUp(*UqWxBDBZ9LcfVi0!Fq^O7$!RDs>PX*QV@c6`>E^d%GfwX=b^Ufz^~BEosC>z_
z=4X93Bj;4Q;_d=}AR~;pEQ4(ldT>jFko4zFSFoe~)(y>Mz<bj64Ep(`Ea}vZTyJPz
z@5pCDPl{Ep0#n1SV*0ua=#XPCMHE);-y;>d#(UhCy$4!Kb*CuqQixoe*}_1~%gkFu
ziO0_=()l4Yy|qV<NB0SxgDA?0Bm#68M3QgW>iQbF^w8L0jiyri3lu*-fHncQs<v}w
zu@^KONi;Kpl|M})B*vI*()Ji$g2c*R+EVJ?lmzNR_(H`umn1EkGOlT|jIz!}fgke(
zo4~blb3Q7u{)R^`S*|0$$5?W!hiO~I$$S3K4wN-1JB9hR5T&^JoEUfA>#3@CbrmPE
z>>tk)KtdiJ`qMM4ZrRjx^;?T`YUes#etwzNYw4N`!b2_WNzSzO*{{`vPc{2Hv})SJ
z%G_cFz>TyAXRnxNdx(Fh@Pl$7kHvj72&n3Y_R@934rS&M!#DZ$k@L^1&ioay)&Oyf
z+^_D4-GIznz5iNhuv9q7a6BKpqPl=k821SUPT{d~gHh-0dfUZn3b7R)1JHTcuGi5}
zg3(oJy^x+k6y_HdzO}VP4LSP+WrX|viSN8wiPuimDraTShZvGW`S#zFa=!Q_jxoAC
zRWdD@s}ocDG#_ljxmohggbpIIYNs{=$23CYZbb0orvj-^+XeAjk?Z!L$ny1H(kNw@
zmV|a$KiN(47yjF$5_2Q6Yk?t|nx7=799aB%WThj$-5EVVLJp(;(P5e*PcY3FVC+1_
zM*AjP!pda3<XMrGminGT&-ZU2lMF)D*3-2qOtD2Wc`tZ&4Q|zwcCHzTf*DsLZBwNN
zHF*f;Y%OjS*_)dJehGxF^bn>ERfHYRa9>ItYK_!8N%jN@goyoKQ9Nr<!lru53E#2&
z8o@s@IvOM(Sqt~$V>g0~ExAe2bmjUI=$?y6%HYxWA!k&L_vy%sC>!?yv|$bv_Jj6X
z<q5d8P<Gsu*75@|2&KRpLvSXMU!%`SqHFMY@x}3kw4eX3%<VX(ZY_nO!9Xy+;Wcso
zo86EN(fZ4!c>?|d!uq!dkY4W#+8UQC&HXyFkJzkVc4gLrBv(r34_Z`QcLuR;H&!Py
z3mz|HV<R`b>x-E^!YICk=8TCC^0K`P#D<Y+_hm4W>N)Nl)89{DytD7g)(}Hwx&$3#
z?|MFNyiAJxnYqlkLYUeSfB#}4T7XxPysNZeg1-Kj%f@*>ZT6aQx}6tF_5fC$A%e^D
z*8>b*+hzSZHz-UCtIjF$j6Si`Wn*^SZbT6N^N|)GjuW~yV9`D@{Sx)@(J{>T^KkK(
z!1;nh03+v=GD<q6q}VUVVF|&Si1BLsgm&#Sn#q)1!QeBWwFZI119e<3rr&$Ecn2>x
zJFspYG476e^Qo9TsEpP8WV27PWSu>uNAal7<Yp~mHFo=xpZiK_@Jg@Q;dD%psPqg?
zG&yFQcVkX41RfdGirjvmc{ms_hgXKI5Tv^W5XV;o-hwm1utgo{Cw}wEk_3slS~_g<
zx9+(!b(w0R?{EX86)V!dx_*4+%So$@C*rdK0ZxfDM=yKtjN>CwhNKj8bjan?Z{M81
z0uiL|H*{+h(W<Hx=G=kKwfk-t6opc97Nws4ZDy`-JJJ4=-bJBb;plpOGTD%=)<JSW
z3NinKPFeOO+hgQ=5lw+r_v!0|NoPNkQbN=nkORv06Bgdt@AOz!?{5eB^#(o9Q+D7|
z!`Yt)!F?VOx*%Hb2^(wm;_nAJo`VHs(K_P^_O(9c8j5oqrMAB=sMogrb%_+Yu{Rs`
zfzdy`^!q<c6hM#8{vb|)<4t=@^s`U#Zm5cGUvFsy)n};Qe%Jq@{(uDq;mcVhZ1lk4
zqE$z7>HKje+()63Vkob*mB-ZA@X+G4#W>$y#pFm3nC2nQ;x@zaG<2EIXMU?~QR!PN
zr!MwgH<i6GM|)(!wx)%@2tO1(Tq*9S$Br^*)2r+p&0|BfEzXuuVvDS9>CP-7T4^vN
zT&rB{n_>jo?09nyxsT~V;xS{Vy4kkURYB9){lO-XX*XoHbiKRrwa{`p(bvs-^V4w=
zio+b+1R|w0wb8{)_{C(_^JoMl!Tn|}OdW1wQ;dy86C4dg9;w;ZjGFUqCY0F+92Rfl
z?E2hpz6Xu6dU`E?+VP<dl=T{nAgM}Sc5__-KIF^`iHEw}ZXN}!|9mmX?C9n;rH-D5
zvm+Ls9pZcv+mQDC@>53Idf^*Ta;Z`2(qQHfWR9dzJl%75`I)or(fY^Y2-Z8J2EZ#$
z*?Hg`fI<J%riPo#g8(QLgkuNA)g3W7b{nh$eq2lK(EK+fQeAsNsS<ZG`2IOtO?4&A
z>BvT49`l*18l#f@{h%uHIimSXYf7m@qzOkT!{J$k^ljvmv9)}}0hS7QwW^Y_JYJgz
z&c0rUTru5ZeL6I`eqYP&z@aJh;1mx%^KK>9ISz}iW%{!jX*D-6as>8f+{$#}1V8=q
z4NgaV)0|Ak%4^vkgX!I8Z*;DJfq}a3wKGO3U+!{!z3iP1qk*n4%YDNerk&tTt-ku<
zof!kEkKZutZ3(cD#+)uzTjAL+g|(=}HAj7SZwk$w`(Yqo2bn>)5UgU4OQ)1Zr+jEB
zM{WA{3x;jKNh^2i4w)ipIMb<GLX6~LPSgC6E&w-jth7+nm_|h!^^j#Vg}aeag*C0(
z9@*)(EYbS42;AZd`@(jlRb^tN;N#nMS^KMVnG8nF)f?5|HVlnS5B$;oeJgJFJ4!R`
zBoRW@BX=t|ev#UjS~>H0g0_R`G~vl9<!CbhK0cf?oEs%M!qUC({cqMFPT@vc7aS&<
ztf~`CZahaQqyWb;+AvO6d=!)ek~<5Wlt|84eEInK6AZ?)oGNDRgBR#gXnE%O44wQx
zLMV-_6V+&W!--?h7{Ty&MAa@Y-nR7A4!h)KWzvB&DR=B=DCQID7hX^B-i33~XTSlD
za%)02j?a)o?&3OG5xA^-nC6?1Jq|6~P(kfK&TS&fqyC?W7P;D)^z7NVPH%g8frt#i
zjwi{hkD{YeAR6{}bZ236+~(cp2!Fju-@C*m1|0X`{CAkm|KJGy|Jkk_?<K2j%$gj9
zkp+%|<-Z{j&0NjBmKNP$E}+(tw*KEX1$J2rVk?4Dc)6gD1&;rLu0+#(FE3>p<Risd
z!5XHXYd$<^6@f9~W8|8q4z4@{L#Uu&lUD@jNV%iUd(X|(DU51qGOCw0hX7z>;JZMH
zp3``ZqsY!$m8GdlW6p_jA;1UIGSnmqz;K9{yq{0<;QY6{`oD0G|Ml)asEEVvD%&jn
z#us^%L^CxfIQ`q&lwDq4fmDja=FegrRV59}k%u*lf60P$%c<WPH|?bjk&e>=>Goy0
zexGkS{cJy;3Q~Rovo-=@ue<C%@pl(9(=q6okGu~-<t*G2M%R0m!Bz{okQq!=DK~<d
zg+{)og;f6gUI%=e*5EP!-C}1>`wyuAuEYjl8YK@J1k{e6jmSG^K-sLt^~8ZMX=;+I
z10XS@xI}Ov>UXufl7TNygEouFBn5CFG3+CZ1}^`$qtm@Nu2_r7DIrOcGZ{=(LT$^r
z!|(_4(0VuIRa=39B3Iu|x<|6#_4?+dVwj6TUl)CVw?2guYyxk6x~igjh(}We<aUEu
ztDNFaAIQ*624iF&?Z#x9%vIPM|MPH1_fxuBvZ%Y`^~QsZRcd_?-od2oX`7&Y3+3H`
zwc_dm7xFueGyyECpM7F7Q{Y$F<wJ&UWDKF6)=5nZ5KzZv{Qb4;BO|Do@bFxG!cv@I
z^QRCXA$++HD^pzCSJfmIp8MPROr3XUR(`D7b$K;HWUKIBLp$@AOTLj|#iQvj!43*6
z{Ro_`M#I{P0@!UJR^c;Q^l(YaY{Ld_`u3=oAGaixj`EK#E4Uv^HFRt?B%jQ_ayqF*
z1P@G<`&`(4a`X^WF&y+0f@{C5Q_jfr^PhMFxttzTeb5WXA_|K%NN)exBD6TlUcR}G
z&CuyRg;jE|usrj`+>bruMIfnoSYXQw=YODEbuvgeAZ<OPTBbrOf><!IX&sDtm$*^N
zOla5Db^?UNebajvVCDs)Zf$~R5O5+B`!tUWdV95mo^`%s0+RgH2N!rAT0cE%6+27+
zXkgy4ySczhtEI(0$Y-h|bk2U6?UybxvdyZu)zU)7uo(^W7Q7n^^CbAK`zYwIW@Oht
zAl<QF?%jSWK0O63NKAw`R%a5^*v65=Y|u`^zdTtuz_Yh175U64)H**TdMVybWbYii
z!hf;k3*MB^OYai#)$>~j+fKAEni}b$)FN{6;=B*5M;8!IZuT-Lv+6QTMyA?pf4U?U
zMD6I=+z<77-~x_^c|V-}JzY7>9Wbanuqp5h_B*Nb+(RkQO4M|xu1^5+!UR>}&bzBK
zPa<Rgx^dx8SqsyOkpKrn@aAa(aq6upl0~QnD2spmIX?qwxd@Rjvvvt0Nf3OD%fNET
z+bzgOd+`meV%mO+k%?ynU2L;f4VrkF35PRNWMONH>OUC5&c|N=ZI7IIp@`PMvsx0w
zbb|vDAvOHY_S7`!Csmsa5e4PR^?1yr@50rjkI`zE9{gCf*eH>oZ@F@j&=>!4A|_E_
zE9}>5>GDycLYp{qviMr!Q0@Gt;Pj=GIx;CXO?csusOJ7DqK^VH=rLHX%o<t6fs@@t
zVc$5rsw-(4$;x8HcblRdI#%>%;dZ!yH7IhrD$1x%g-7Bp(k5trm6|V;O`3~EFoAUG
zd2Nt%=JUoD!+|uPm-1E3dARv`W-R-SIXL9lk7f|u@sf@nalTqz$fZXd*EPr^EVVji
zRS*I(s84OuwMCxpxzYS`YG4C_Iy3y3pPFmwFDcC~DKE7sA{z^sk%fznVZ~#D0^UZs
z`emyhUaUg<iujutoaanhsr5@6Q|8fg5c6W94|MVU!|NZ_#c1K_M6#!=V}{!k2AC3X
zX$y6vex%QU2|g&D1{pdvB|!eizDqM{tX$mT0jkKcrpV@_NZg%6`KoUI>>0rYy6q!V
z)##b_JPyw##E7(lC+qyYHG}+_KWIc-Lv~sgzcS<TQ-IVTLGyhCAABcE8vg!j>FUWF
zG*UA1ZJD8>X_=I+dc1`e(`F_@F<|DN=4~z*%Z^Gp0yh_*4~NP-@qCE!<DSlF!HwGS
z1;<!KC8r?!=ICo_$bERHAby(8FQX#xd`n3xQ!JoiwQDEj9*4nGIa+1ez-(D%p>W9(
z039pnB$@u+fyW@yQrnk>w*mP0I)-gIUQ&n|hcJMU5-~vc>$z~(eSPo@P2C8n2V|Q*
zu215zAoZVR#R63Ww_~!6l2dCdC4V&GjzeKUu7)%tw)-<tRy5y(WnW&<h#*|Y&9+GM
z@qtysZ8o+@=G}zNhF$PulE){DOZWB~AD7_RLWMGE7gSv${l@qkI|7`qIR*)*Rl<7N
zqUv?q(uxwJx+ObakrsY;o$U-d)!S!(mHpyAsmD`o0nOlm%As!t>U_~3vj3E=c)60B
z7-`Ko+n&HyB>aAR*sdi<6_<aVX-sgg!X0okC$$L(S@mfm8+J%fDOFq;otv;adO>|O
z2C$Y>HsS={D2bj*1Pi*DaQ>RZ)#GMC2VupteB7m0Zg$GnMn;j%qHi^x$|p+dLO}~K
zBgd=0FWYHxdK;RUj7#T&5=Yi&$K)V(1>bCs@#RVONX}yTnzVV#Esz*f{chd^=xM^&
zp8UZwy~>Aoo1nOLy~}N>=Htb#esag(d<CJ?-PiG2c~4!j2;4-Gkeb>B`hcuGT|&pv
zJa<1%j<=QU%8hO>vb^GX)TA?0_BU(q{C^3SZmwS~P|EbS-F%$i7G-J}G!QwMvfNuL
z5IsAchjid@(yZm-`d4aN$b+?(g^W)lQN8a!%Y(6|W74(%)%8`FwZGmwh!~u59T?L$
z;@ahpF3<1ADqRi&(Jd3tz9`O-wDg>D0*bAgZ$H(Lm8ie?+G}1B_lnz4Xs{U3(5zSq
z6u&G2@;Q&MO@L!)i6@8&hVyw9Ywb)QZ+p{bm4W=@eE5lpK)05ra9BzeagZ~uzA01-
zevHvGZ<z({W0_$bQ*YMPD-X>#-v!3IQzgO1(*n~LNQ3Hyt;DjYo0w@Ib$SXRm>Thh
z0)ERXy<SDsHv9n$liEQhR6CRLU*(v;8@8yfg6&0+&;thKLF;yt%=jGp@4O6mb2sMX
zVjuGB>hhyMz`$kwP+HP<+icswQ}VgMf2}?@#0ltk$hA60o9k9Tbm#&-Ky3r~v;1u*
zhwnOodNltL*HeBG^!EB+Cd_on+D8N0m?ciKI6kI&^V`R#^wHU=at)n2#W^S~%3vLw
zJkL|-FD51AzYG%F3$x=md*PWyR)Ltyufr#P2|TZN2R}FIY^z7w0JT`lx&M54>p+-n
zrQoozJN`Cu5KrH>yJumNCNa^%Ud}ylB3Q%C5w+^AJao$4QI~`#P@q8GNmr-om)-On
z5uo{64>L}zC>XRsFhS=o(0G6vJCjN_%qVYr>4F24nE82=tH2)R1ZaFXqhq!r%FoK)
z59F->xt0fTY)FHD4D6rDI3Q46J7RuGwV~S%C?mK^pS<hd^2~b5zoch?xB%KDEx>Os
zudz}|{6%z!kh23f_yYn`xL8uWZt2o%5F|`&fSsLVtP@dM{T|#d!Dt!Z_>ZWUP8)6}
zQKRx>4i1HNz2+B-H^clGT^{uq?pGMkmO@7OQR{z!iv8cBFaJv<@BbSuGcJ*X6zG6j
zCNB|xE&Y=`z|~ksGZ)CWK;n+_5&LKesCfbuhxr%@yy+i()UR1H`-*h2K<;OWaiNj0
zW-d9v=mI+7jW`ZaH)e@EsAMr=cQ+jH3;D4nkAoQ7zj4H%|93<Ic2xQ71AsZB{}Q04
zg~`XBV}7Pn<X?j+-0_};`P1azv0(p)<DdTj$9Nvv3cE)jU;^TNv?ujl*)vuA$LNwd
zn<$fSVV;W2vl#n_mj1svBaFM=BL5EHr{q8oQK4@Td<8VsO+0XZq2_7(n7|KKJ2l?I
zfp(P|8MIzR1HJBu4jen1PX%)ovxKSkmAF(dl4Ap*m&w%l0hslSxK#LXAX1D9o}Ot5
zuq?&w7>TPsbsT(eitxOff$>#bfXCjrSw_H-%mk3>fpRMj{E0%DHHl(JZJj@f>x*0m
z${s^UpYLW9)8YqAj|hI<AVX;2yV{(g$(lDrA<OP7weZ>Z3a9-TaAJKPwMKX12^%&;
zii*N%K*S2<b<Ns$P+)P3@WRKt2+|U$)@j**KpS)JWFd(Vwf<2*P?&@))tIbTjlCc2
zHf0;TC>FQ7zn14%T{h^Q;f3B&QKko%<uO``lxZtlG*BY(11q<=$V-$LUAzlgu45IH
zS^`)8R=R|yeDsyT7huRMN;8P2sp?y*sbr)lq-~Ol=3t}K!1(#p6OV|-2|6Ap6<`@=
zI>t|Hitqbq;rUX%w8>YReLqpmCZ`qeL%ePM=N}o(mQUhJRFQsHvbx|ax8gT=r#xxy
zEZ=$YMMCQar?8nvTc0-c*y*+GiaH&;>9tHQ4UdNVM8r;t_KICK3*TGBV*mW-p}JSz
z*7SG1*?_SK9)9B~4zrWhmHJ!&)aTj9DR~cUCpe4GK7TP(2p@C(K1Q%33EKKodj8tL
z)E2#9n3P&S$bo=ElL*xlNeXHu=7^YTWpsBvb?ggyZ9^asT7XQAjRU=)oz$-@(?p}j
zE`)E4W|)}iLLn8cHmDO?Y%zCqdI1WI@b$c$_f*X_LawMsBTZnboGh*W$`#-Dack6(
zk&k+G?R_??1V!M-W{o|c2#x+y2L0A$IvlLQ{kbFI&Qk#t&{?fk!ws@6o`g`SaZ5^y
zg~EaQn?}O8-L3#1*1<KnHl!Y}eKnh;sjr*yv_Hv3C{mLOd#3i7cP@t{8wKkRbTaFl
zA-Zm-gh}P_GKQJdp)!xcZ#*Hv4<z3W^sEoj(!0%SogbjZrCoQt`G-2~?Ly3XLR=P6
z7W~fbwMrV7k6o!)NBWr=jTU&}jYk>YK)ib!g;xofMUMiiC}PRz39<RZK{=F?V(!^s
zY47TpPCz!W!0|MfsAh_1tQ7q;Ds`G-P=#GmbxF3|AnFP6+RHb0HY~1{SSo*OX1G;X
zCOA_-x$H8Q1R2w9v3kHVJ6a1!#iB1)u6HpfoE6@`>eBax7&zaIT-*)AqdR{($|&>Z
z$v>J)N~pJD4s^niAQSYymT~aOF&dtLk!3ZzNg0+{G=evoRw_;xCr=mE<L$LImgmUW
z_=1P$+kKxNBkm%2*xgFFq_GP2U%u&${6;MJ`c0OVlpGT@Jt3^!tnlE5KN;^fa=kKs
zl)6?ZXKEJL7KZDs=Jb&{Cx>OQD6!1-BU=b^_$zWQw+~WcuyvG)Md7QAaOecDrlstK
zF9qq5*#nW-NbRWrGvDc`6|YqsmD@-{BH@lXv(F{ie&4CAdVe}(;Uk+gc6)AmIS+HW
zjh4;rx5}wa#>DrzQs?MMe<H24$QAYl8ZKWng?AOr;UfGs({<>0Tu<(hl)tgSlK3pv
zJ(hu6ycapiQrx-jT+R7<{;G1ae|Uf+xUWRVoI)gRbAhYQiqC)D>NLDa+SCin^@{M#
z2B8JpupGNe_~G_JFVXbYf%`(YNI!}iT+Lx@Qp&AXABCBk8jsgioqpqrO{0@N^P76x
z<A!e_R?u}%2yC%|mjc!53;2l#q_=O|62W$Y>udAj!_G7aLBLRtuv}|rmUh6Yx1${V
z;q(^aqHR)H9~dhAg(k?8HhtTL`=u&_NsVWmmZOxyFI2lSK0Bz>DQc^a)UCQz;^LNt
zzP{BS-1mvF(1*U764Wk0_V-tM>t`Tj3SnwfQx8k=^L7cfPpx!@e7MU1p`#KL^QR-P
zz)=Yoc<eA#XEI@nS(jFT!Qu*|r0ukTBhnwN0;%r4ogZ51!a+h}aCJyoFNZWrby37P
zNU~LQ_S`XNq{8=6{D=8l)Ci`^gD#T{TbLzKjF$|P5njZ81KGLCVUJbOjmKx1_xGuV
zWs)?wskI$eX`rXilPHk)3IB9tnBYxd;z(fbM^s5p+{MdKNrMdN9+@IZ_lCT&Ihr6P
z%u^tnI6n3Yf7QZ#QS7#Gqq!G;P4U(`+M2QajN8)tD_eu5A@=l?NFEVxmPlzB!|32S
zy1kJrcZnPkE^ZdG@yqC{3zoqu5MtI>^j2BDpdXeyK92tOpFTNB6>XZF-eQ#1*RAXj
zE(7l7J&kZ{hj4L(o43gNqC41@aM?BsCu!Zk>h%nvHc8|Zmdx+IR>n;sgK)z0RIEt|
z3t~O?BsmBx=U1hW-|LGfw^e#!r`Nb2qJS>%uOE{N@?j5-m9>jKXi6?;;s<A~`=xFX
zb51&n9ER<b^qPA3vH1ElHNStC$t_>p?RdXr-DGR&g+9%Fh88;pL>+jxU7nD0McN<Y
zkz_<COpEw>xE6IbFU|VelwpAo;Y@pm7dQgn@dnUg)v^(l>Ja<&sSHM_tDbzfinQNf
zM+aF=aGJi9o`t${b48#>Mq#NSsCb0LZ|6(+5AY7PJ!BOsA?XyY*)2%mJ9CX7O1cya
z((Mn|B_Q#%jCklLx(WfMAMo*uTeyfH*AAEXFES!IA!dpYkJ;soRtDKe7x~9vR!a4i
zRuVq1y{<0(($%z*m5SU`_R&Qc5dk`siTUt0{WzqaJThvg7B%X{*k)Nkd2P$cOp?9!
z!k;tmuV#PY*yvdcMjjWWuFC|;#$H>@+@P&AzrJS)5?4*H;lI=C?uKU={xTlY1Oib@
z%YG177l=(SK&MP`QRBs?9+n;s+svL0Nq_Cq9R8-xRvsbiP$0*kVOW;umk-T*8M5(_
z(?Ww$u-4JtMWwyy+qZ6hp_-}DFOc*I7C$-I?=SIVDc2b^Q;SYk8Q+{@Se?vv8}(Ja
zE@f!3ny$le&>(25e2CfO>ysr3h^FahB?sz`HFzIjzO|W@PY5d0EBE{RS^2GqjzcMv
z1wwIXHu?;0gfGOaep%e1Rbridh}%#k?TA?ub>4rq`}*t-dxe+!MaP=HnUYNtyOxRi
zO|aYUB)u-W@#bE7nfFzxOm$1|>s#8M-~~tw-ORVR^yIc%eet}r{g;`sPS<VT$Ozhv
zoHuQE!MY`LB0?Tpdp&0KxlKJ+U?T`mxq3%NoY3Kv!Z(Q`caKDo%0mPauWf8<KYC9I
zpoke1&6u@d-wstwd^klC-dJmR05uMvuR09d%a_~O7dUL^G-D&Ad3@(A1|_0FSjQIB
zelw2FIImvqB=~NRX0ANzcid5V7ApDWnQ`Fyi3_M_QPa!cZ}r;t2rF%pqw*(pvQOlY
zUcs^AbWD4EnoH1YxF#uqXF@FWH?9NnfDc?|<1!H7baY;hB6F!QiP>(kTS}ltPqA<c
zr?-}8E@OSH!Tz!O&U*%E#x^R!F1GSPwPtf}wJ8$n>K|FQWL(z%U`J`QDN>S)tUG1`
zSc47N5NE7sx6J=c7V%#)5O&VaFJcXTL0kuX`u$C{{Gy?{mi#NLN@fovZOMf|9ynO#
zg8eZsJ<ea9=K{J@Xh`Iq%M8ytoGJg+J$1Ee;Z<n8#BH{(y>7d1SzR_O?rHrSpq?`-
zU94>GxJp&K__NmXG~y7qRkR{-cX4`Z;{Xo4Uy_NT3C#g&N_u8+K43~r=wi>^dX+ov
z|D@cz(%etPf%kvThqrt1fGNqPhb@_Z`#_@NtROlG8{SRCF_q6&QOR$2w6?U5+e-2U
zo^pbn`X?d4nFKq!<5PqIEsm2-U|)DU-_i#DNCmQHfW3y_KB%-S&a#uI09GwVm(0fp
z=#Q5~<8L8&%@)9^`<K4YI3KX?=KW`jfeq*uw{#Q#*%A(K&i=V#E`L7ojK~-AK#sXj
zpe7~1^)a9kYr@@-TUYsZK>luVyQC%moV0ZgBuJ@;2Xyx=6L8|YzVr7F%W1%UHVA=L
zkwO4vG70*hH~VQwBbyU;&Wp7at%a>zqa~VZ%6a-S#p+qIO6||te=cZI;Vo`SLx99O
zA8sl#-L-s@V-lSjL(AXsmPT7sPOHpyfU21HpK==~VTv#!>#5bL7dOqgh`&xtYLBp<
zO_1_T9=Lh_TvMRVaATf^hDM_xB9M$D^52O*{!8QK|CHGmcZB7?i0H|&jbI7bD~L&I
z#bHyeaFf0=pWj7g9;XU*&>f82$ggXXCicN(e(tK|Upd70ZP_PR^Uz{jVyy`b-Ma3}
zuUqP6+}iGIf@&#ZNQb-V7j4`65o|k;(%RpBzHVSizSEtDRtsXUM3b;#Z;^piLg2=a
zu(mYClb5?%FSq8ynFdWKPneciR{5IdHTjN?P@g^vd3O%;Li`nHmrmxf?U$MJIK4+x
zCsw{H0xdP`uHj}?d7P#84@MPiz>%4Darm0rUUy^rQ@NvT`2ypFj&#eXV>~m$^1s<(
zTf~Decqt3<{QrXeyn78x)G(asvg#Ns?YKx89FI#8{+s!R0HS2cJQzs9YA!KD)tr_`
z{-V;i6i4jPrkdtVt$W6w)xD+gDWL9K^$Yu15wkKkw8B6EG}MrrSH{SDu7>p{3xjP}
zUTr1wnO_rx84v&NuuNqIH|dcFu@&CMe<c5I;Zz((aw%X?=kd#NVQq#YWoSqcwft@Q
zS1DK^6uTV`hA>s4U6;GrvE;VpXfsDY%2qdIO6C9lhIWw1+*jq^7e<^p<xKHULMhH|
z3wBRWc_&{>5s(Xd*I56_+PY{<n;;>Kziu^_l~Kb}1ND!I3$F%j+mhP)6{woHJR%j7
zN{Go}dU58e1#qG=OCx!;q!)jFtB=2J>-!Y2fl;WDl=O3-2TVP=HnX6A-+viboc0oN
zm!qv7y5L;iRdB@XxPK!I`nQ#ts94N)aJcg`bNz@6GuyHwz9)5MvFiT1Qpo?kI#sW7
zctplM9F9w_W!6Z<-*EYH5#GDCo9gWit-oC>^*okgd}+sCM_jdA^!8`qK|;y_M)8mM
zz3mr^^z5UIY;gYAyq`5nQZ13w&J9~`YH4S@D24BeyfBv@@oOO8_j6O9`nQHQBE-LV
zKzgLIZX4M%eHiIO&|w0`(^eOQ=OadpTL;51(VI`SA+yaZUbnm8h05&miFKaALRX2l
z(<IleKKfm+(G+yo8U>FCFM@Z!ly+{9NsmR04QLr~szo552T`0a&0oue6mc~LN0lpW
zWYUC)bA!@67YoX}^cx*7)j`hFWKW-@T6vnSOlf4D<okf6zDE?^?`@wp`myWG;Q_}0
zt{>Q|6T7WViU9u31X#hdYW-7h!caIt7-MUP_S>%+;5rIbb3r@muP8|ygx;W5ybIcv
z>dO#FXr&dM*FBwHYK_ULtMS}>2+2@{7LRW=a0opeM$H(RVTF@i#0=|nKtXk%ABJ8c
z_ydKDb>CNslvWE$zpp^{u^CPsWt&9@p&|^4z3|7TyVw(6?;?GN;A)FspdVi6Uc<)E
zg{oq|)Ku|S@5AR#(uPs(?Skvs;1I)%E)8DF6=sezl^0sMVfjgL^sS<or)kB(mxkUv
z2Xa6po!tXB;~cuBGD?Ndk)HSqS<q0=cCVrrTF25T<oNu0`2wwXu4_QeUgYv%`DI~~
z$i;*eUVsCFQ9*1IGMlI`WXo!02|PPNIUh2Uq$R%Y-g%4lJou+iF45G<nZDP$WW)R3
zo)=|(#)C`a%0=TqXr1CnW$ue;CIw|JG+oRzS!LnEBOusY-$f2(ini+|dT7~D$PZ>I
z&rOY{?ajQ*(7oSXawd|PLjSV9^;5k)z}B#8_SEJD>BH$k?OBdgJBG160S}9g>4FSF
z2&)L5kzpIh<xxEg?ODBf{s&=Oy(*7E>cP%n$uO4k%vo8=Aw$8@k0xd!k#;(oRyCBi
z_d{-Tm)URF+^j?q!5x{lF9K~qW3GP|gVsf~wT11$SW6B6S~K3x<4^5u``%w}ZlaSm
zN*-`~WLA!h##oTD6tUZ$4^}6R4(@8UzfTIbV_#TiwTG#$8)me+87kEmkaE0GngqM#
zdP~12K%(-zBb4v21X6qW+uV=QwTKJ<UzJ^VR8viq4<MjOFVYcFia_W^x>OOQNR^Is
z2t_&}BoskGnuvmcB=p{tUP2HAq*v(>={<A;p>4iz&z`gU$L=}%{&{of+<V`fncv*G
zGv~c~y{19JyFh|=jUgx`XVs@O9+K(#a`Nl>{d%@f9(j0+(zO>!!3skzke2y0DM&+h
zTdGEciT;Lh_m$N9e`A>+-zX>H$F6vQ(FRPCe;40*Ia8@P?`;>7g{QoQ7p*47qaZGO
z2oB2Qn3Ki;N^g%t(Z2R=!(`eJYDLM-X&{2|RW)|%Y#mf1zFj<meEU(P5qn$<Xq*1B
zVkxI(o@{y5eZ6z=Wn}B~RE(2$lw<~#Kda5N8swkDKHY|e%-k^N0q8=Uman6e*NLcj
z6%F9y&>u|E9p|=l4e!BD7X&1DsS5OrCw`jil%)j^%=O{ot@<-F!{jF7pUzvu(>-@B
zsj~3acIINieli&G?epKaS;a)wx^Tliyr*oPdqfZ%j|g(v{kGc!;Z>vn^z^(3sGYE|
z&;P#QbH~rl)lO!%EC>iw)&Lq<Xqepe9ec*xGY|rd2Qx^(c-{B+twQft1w(Nnvz%Ct
zzB1;A*I=wgN~VVN-$t|Qt(){?HN3mVdyQEan`kArJxwWT1F}N4r4|$<L@qb=fTn|D
zm6@h-6xi?AcR365PFbDnH#h+!%5J>ef3~e>N0icV;Y~Q*M}Q|zjck+SmV1LQ+wQCx
zK1Q_EtTNS?ZQ-^~)8CMM^P*q>XQu9)h=0^+R%fXmdbMH`%llxi+kC*EV!`^}ga|N+
zcf1-a`zO$kuA54sCO}u?4*yZYd1QA1TkkCqScqZkd}tN-{V+=7=TAtCY%WmVm17Iu
zhvGKc_SVGl=sT?PA(Lp*GTQb(b;>iH(d55~(g4zm&bsdTWQz7Ijd$b$jyN3nVlxkG
zpL*Y>nK3Ox`{Va~G7G5`dDD}Lvq@BKdBHno{Ze+6Z;2IOF*ngcU2RE2lZneW*=Aq*
z{d7G=v#NSrqyY8W#<Gge6Cxqc&s}QU4;d}TJd;Rv#7{i3MaBQR?XC4{3`A5fHcK=u
zj3_ou5g4aI3_sQL`jrY2gtcWOmxm*pTI2dDjE$K=YawC>y9hUh>6J>uN!>2VEo0j;
zQ~Uq2AX3@A5YTQ51>^@6R2B-R)LJkX0X=XG#`}x(kxN5eM)!Et*~Nz};2}c3KMeu*
zxYy%C*#So3-z}f>g!Q((K<q~@?HH!i;%g`{BQt9jeKw{Sksa>u&YAua)nYWrs}AOY
z(;4<=4G>Jb=j*k49-XF<2Qp|r9S#qClX#|_jl%zGxtl*RsRXbzBi*pD5`ttvn`K)0
zX^fC!s=-AJF46ONo^BG>6sUc;@M(!3aG?5Z(qG<EsZWWu+ClZWO9pSXGn8=SB#6j^
zhRsSt$bVw8405J}jfbr*8E$cADCpn=K*sn2NR4>Tk1+~7A#;U*D(-5x37xN3794nY
zS83J;0v=HvTc5k}*}FVPoR%o!id9Y)<{&O7%`5Qx=x3{^NF|s-X$wIUMb(@#*-3hR
za7O*J{1%kc=*~(vd;lDJ5YJ@0++1yKl94^T<h$F-1mIineXHWr`Z8U9reK~c9_^e%
z8*Bqwq@(xJyZ#PK?O|<1+BH)YO<(Yyu%d)|4{z-I@JGCBz*rC|I&4ITZGvK$y;pD=
z{w=OR(VvR*c2H0Vq2G4^en?4+)W&uhzI3;$9h{P={rS&DLH7O4Xv1{M4U5vvEMsUy
z?=B?Ol$1HtgUa%Flynb$vjbXoGUd4VA?674_b@hEh8zR%v(j00+_3NHdum7WPz5v~
z_#{*LU>RO5_yQWlM0kTjUc|dD_a6bSJ0Hp!#uXC~m<U$4He^4tj25PMh<fOwtv&Y6
zB|~$5B@g81_Rxy9xJS#?#{Dn{$=KIAb?j%hISL;)`E%Oo8IZqY;u`D`?5a;49jm<(
zHhR_`yxOt;I}E_cv^B--D||`HN<I}P3Wq5zb?yhr4|%~4&ZNbM#O>w6?oK{ikRJzs
z2i*en&e+JE?v#K+md|2G9O2D7up|BTn85|F@NI*WUyI_6uYT5=nwm^<WIj3Qx<H+|
z7|c>%kOi&ZM-RNz?DP8UpBMnG|2?B5&i>4XXdED9iERbv9d%qxTRk;jK6O(H7R(R8
zCcIeqvj5@&G!o;RI?tP6*&sGXSsNS=avpB`QBM>Gq8(0$&m#}Z6Q}t}XA+SFS$(mn
zkQ2p;>tepxd7%@W#pHtgXyb>E6F(Tsu2dVAwy1vHOki?yaRaKEjFr!x0bJw~x-=%6
z0}(r)_JR9&e%)`el)z5$Zl6O1vo}E}igN3El1St0B<#`l!-t-LqPnlLGeh?i;=$mO
zTYIoK*OPmJnh8Tzqqk~EJhU&)r2~{Cl`w$^M1rP#F~Fl-${z1A$<3&yTlX%q4T0T(
z4ngWZoh;Smo>9bb`Ot!+Z+EQU*9UiJ*C%55#{A#{-I2|cwmmQJ3Zh=7q(W=9Thu46
z(>7hi=H$=Ax9brq>E34MgWsxcLYDQNlFB+xx2CLMNByJ~RQ9zf|B9>jO&gVT0&21-
z34!=4MeybHSTeW4(E3xCpuq0R!TJ_oJ2jRxK$!k*79A2$suL@_tUGEL!rUf)n&4hB
zC+%>(2V&Qm3<bTH@Lqg)Lpn4lZ*1@*MXxjI2-S4CjkaI8&TDcQt&&SlvDx1<KaHpj
zy<L5?@EZr=Ts(^k1>ejME$%9(x$gDY3uB_y&rc5ud{+@ol_84EUIcxM2}kcIZ=O_A
zAN0tSijv1g0<z4L*TR#<!=&k<d5;9sKNIjIoN<2og_KD947g2z(LYXti@5#>TH5%~
zH&oa3f-s>@FUR{`cn@`nq{zw-;uJKWfad^t?m*|gpXN&V0G~Q@h$JVifUgqipJGsK
zvb0crtAZhNv*lC5H%gJrZF%7Y4XhvHzx3{R4KlZ}6D|!jZ0rh@S>b{_t!3xWc>#N^
zH-#yMV#uGK=T`RK>M{C(-O|$~N(?dOS;_@KL_nu0lF@d+#`5<X#GV07)T2jsNVGuR
z8&-G^+BGpsSZYYEH~6|h_|1cEpUT5^LFrbgR7+O|>WMdcmm}?GPd%0;eMeg9vgRnA
zF|2s8t$XiEehBeVa<a931}NoAM~A5{6sFP~HlMk<R2~f~lb2Et_lg>@Jy@ofj*0kv
zjE~wiHF)jb_)H&(U2mUy@9Vrg>%0uwnSmByXRDu{c{*3#K~}u_zWTez5FFzSwXi?0
zwcwNx_dV5LcQ}4t#U}^jS$%K`cK@J)CNtUy(zTgl>szq;wzz6YJbQLJea7M(7TB83
z)G$YUV-@L~C0P_Wm6$mPt<gwM4VtKzOtEQ>5`VAq-uj%oU7W9aYKTn%{mJKCK2hi8
zhQ({~pv9=_k(f%<f+b4FlzRQR|5#1yFR6yg2tq&VdG!Q(c-~#irr8Azp5d_Mcj+Tf
z@2UK^QF?i|NZP7;S_zJR1T=q{y{+Vv0sP5$Z$VE_O)bv>uSHEwtLw%8hX~Q&pcyIP
zQssDS`^xAEB|wFk^AQ6t867w5_Ls*;4o1#^+d^d$=3m`m@3@zGax`vo{PQed7oatj
ze-qpee%c@S$Tf;nkLdMN;%o`Y-<&ItISG|XU5?m!lRiD?c;B&Di^91Xq{*LTZOf&S
z+d5|!)YM6)TfS8+tPW+OE*sDou_flzzJ{~v&W2y5mwl>SVcrNJj@W`$=H)Gf!>?mR
zncFXYU3R}Sk9H@2sSubi-qw`)^#bLsF<-HVA>W;=M)ukb1PaF9y1Kqr918Ll;fgNV
zpBS~(8RKvP0e***!msy2#$*9qORm)(srqg`femWC^v6_BG<59uc^Gvmr~<%#{vHE_
z!+(5dE0Csc?ttNSKSpT`V_568@GfZ?;N7>Y^rnKHQTaEeeZ?NZ(>XS04&WR2DLbz}
zTZ|lY!VG#QU5E4IO8h&8Hi6>WNRbr|cu7d`@7<ojNY^J298TTB(c<DXb+&i*lxoYp
z{^FViTD%_6W)G`y3e5nAX3SRAS5Btw*~ZX=Y{zXuQbK%ee&<`PZlKyKmzQXLxs4C(
zq;UfpWg5FexdHA{gq-yIbF7o*K;ym%>PUGqu;2BMmYw@|-?5(1Y5YXlPQ2lB7$zej
zh(WC`KYwsRAY+~SO*T{B0`TMh$HV!3vTk3T|5}6J_syx-pLbsA3y~60p{VLP+&5Zf
zACVG86eXhmzSmD1olYshYDdUZ08~WvSrEM643x{d1%lJqAE%;ouU>x_lBS8^h$YvF
zq?$Rgt5KnFW1&!d9Kc>#QQoK$9c5r#iNYx*h182C*_pAH+`Dm_ox)a6_*nVPWKB}&
z_VZYea?<N%;sd9JbvEh!;&XVAzpnT|0<*n+@WREhtt{2;bSA5Kt7+?l+VDx#-6O_z
z`}U+N^Kxr>dfT)hT2HH8&zft8H@4;pn$2a#A4P-oRMci_OLN9YX)~X(V0<ZbpTGY3
zimhs=T3I5(=WQCSRwbdPld_Mp8=rkycBpMQ$r5uvV7p|1gr>=@I&{jT%p)$PvOy`b
z%L;x?dTj;_<M$4pg?przkT@fxMeaod%_%u)>+o2fnP+;6-^OUnN~QYBQ`7sEY~s4g
zbH0OL&vjZ9k~IIyl*=wARHLc-RXJtMogl5!nl*{cKimwFR!FkTzjB=r$%?m0VpM<N
z9alMFtatwHWA=ox_Voj=hIXMKTm_}7B-$6T^D`5cIAK0(nM-IXgf8nan;f+c999fi
zP4p+-b={(P*1n*)j<i)v3H%dl9t&joYjE(0^-hP3qr|SXB3p7W1jV<|I?0xaxv&du
z!44f%jgC8&-XbA=mp}WK;^OP3Z&VuW%i#46*0nGH;|gU56t@Nm)$BR;OboO@vbRaa
zb}CGL`F$*ap+a66mNZFKE8n&KEsoi~D%kx(l03k|%0$O@e`m`i4c<Z9EE@Lyb)*8W
zygdGkTLaRtp3SCKxc3-h?lC`)<)70{7P2&Zn37(n3NjPvUF$MVee_}$UfLHpOLrA?
z=ea4W-YlcpQ;xtLR41E>-kGaLZ#J0B(eF0i$j<m8`d2StXo6hpTVbQJ9pxaTf3#Jy
zbl&bv<(ynm$rLVvi9C09v+iehA|jzM-Z|XLzsrCkPGgtBc5m>^Z&dPUX7dZ<5nGlx
zE7Kdf?Fv{gllT=s`&9?TwGSyRj0|mDoE|jniSl_9vU2gpH~*H*E5pa=Ov=8W-pX-S
z6c(`1sp$zz?pqHM32ELkBj$<2K5TeC)@x*gx@m(zR>Ub&@%F~A1{k;iM&?>d&B;9O
z+Qs^fsMjt{Fc*o9<1>FOW}1apYoKlgyhT~pXqgKu%ymPDhe#`pBz`cbd>)~3_bDYw
zw!t%PR`Q&KFp99hyQqAr>5^Z~)kz4JT7crVj&H#gUqlci`*F^`OMXHFVkZ7Oh@8bp
zG96MeFJo&a1Si&#e0u5k04$rWRr>r^@@YGuoY4tq>^7PwjKX;02E5V(KIhTn-YJAt
z?8RmWg%7SDjE{GUT^(MicGmQb?Hy`&W=DT)ve&}2S|hSMtqT(eQT?8lG{QDqZU8=p
z;UdPYNfHncd)c)BFTF4nN4_M2LjR<iI)OKIYLY(m2^8{%1giqW+8Xo<5qmn7)Bv+4
z(Z97Ah>T)CPtA&Jotw7Z$B%5}Piq~cK0xbjUEXmL4nGhPkZ-h;H+%kV-+`XIoIw0a
zR?fGi*x_aNdSz=NV%CNM71K$58_Z@iF)r!!qYYD`44#a2QY|hbX%GJ{+)4t7eCLG)
z3wWHGUtPXdu50!dlL06*Ds)j3fEq)%X;H!tpOB9*L7&IkMq)wKs|h=LY1HI^7yc8Z
zMAGU001=4|F^kzxXwfYWa?|lY8<GH{&LahEY^s%K$^D}z6ti^?Xb&1WKC^D8Ca$uY
zx{e)}llcNUA-MN2yLncijUv1sOhN*r;3*ovCavJ5s6F=#=C&`eagH5p5sW7_GQQwn
z(<`K4V5y(s)LxFLv+E>Ih^l8Br-*SQidcqci|CX0D9B2zJeI6af2*<tQRW*8kk*wN
zhE{fQZtC1CQjg5{t}kqmJw3Isg1_?+g6aT`O2Q^{OBdjKQalD_6{SLer;dJHvP`WD
zMZ1)BRw&WiPu*Q+JvEztT2an>_e|7HSvMj@7ks@y)-7*J|BS9vtEaN^E0+|ZCqFY0
zn5Id6wE|9HI1&dX1xqFDprBY8lRck<{!+`==Ee%1F2{3k9ti4e3)bCJ%zyYEZ!iYe
zGb+l_jakD6rt3#K<Kre}4NC4FZp|ZsdU51YvGtx_6_q9zm7%-Wih_#8hk?y&d!e;;
z=Hd(SV>^Fl6zVGET9-(vqBQkQVix&J2ktcsOxl7rn|2>xd%glhtgaQTrH9n>CE2NW
zw+S<y@B;#31IFFyXzl%$9F*-9hCeasrEWA<YD%|PsJyMaxBS`tRu<#h#i2RX>==nX
z{mT(2%HK}@TZE}Gq?;*e0Zk`gpd<v#U0otsbY~j_bNuNU8GYKF$;rv*)>sNLZdCz6
zwhvrHpW+M%U|`tzw!zr47_FQt_3gavNCCzyf{3=y<awB`o>=osFO_2?%~Ohv$j=<m
zJ6}_>@&6;~;5co#a$5KV1p2g*_`Gp5NP%3rYz$>nkS@YEe{-~iT3AoFNn3~azGSTS
zJQZE2vE}hr#Cd8kWCp9@O0Zi3>xhVsK1|3hjgF4~9G3=<j@In?x*79-6WB3l=OEe3
zJ*?^tmWW4X8abkH67=VbxfY{mfk5bxssxuobWRReyhg6U0SIXNb+dXOWw?k4WhLu1
zLwQ5T<ng(xbGM>3@2st@IhetYH%3x`7GHOKBhwV`-W+hYFptWI@xp_+(DBM@HI#TJ
zt|U!9`rykmQE_NOm07ije8!1|PtsaUG<|MSktX#mYvj=5I`?~B_Z_jIr!S7w)Y9~e
zjrxxpJ5mN13cy@B<oFC!_-j>;tkbSw=nlS$Yv$2cqX{%V_Z!D0ZBjcu1H-_8<D{w<
z^ODS!?DDic4UGqX|M7N=E51jhSe<bG9TIp?HiMe3A2;<^Vh9$@Wx#YgwoW;;IY}^t
ze-W24y8;`5e--!R+gqjn>Q(!{5^Unod)Fq2QOk3!BT#W_v<n>`0QS64%zlm7-5bkN
zncX1z5c&5Zh2VtW54pIL9MjSi1MwPc`Dw0C_Ka@U_T@OV5}j2_Ei`pv_B;CdiJ0Ke
zFI_MbA<`r?BeEYIWpJgn=ag#rQL8?Z4nA1AyB6|{;Q6ORc0w8&29nb;IvSeC%;E|(
zG)bRC|2yHW#z8~#a<NfvqxX%jQ!p1-PWur~=}MKS8hY8Y7j*K!LgC>d9xG&1bTfyY
zQA(P0_%8ZKEko5x9=tB{9|RRuaTlI;#vuMJ|5c~ET#}&r=K)-fni*?9|J@O||GPaP
zE@)~JJCaHJ2XAEk;i>SOvt)Gnli6;>C8kY#h;d+GV3oTUpAooaTw6`8#=5t+_t3!W
zO#p3M?agInQquSQwc?5j`TbG+@ydMydXr2w71#*?0ATV{f8}TI<>BGz=?C!hcq8n?
z3FK6D@$vF-_4lw)@h8OdaR0;O@Y>H&*51bve*@q@<QD^qNCU;BjYK45MPy_pq=5ev
kL~79x+y5cp;pO1!1peOzenoW5;ROKC)pgY>RBR*u1FBO?7ytkO

literal 0
HcmV?d00001


From a8de617953720e19e13778ef838734d23aab5b46 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Fri, 27 Jul 2012 12:20:08 +0000
Subject: [PATCH 4295/8313] Added a comment

---
 .../comment_5_5e27737a5bb0e9e46c98708700318e67._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment
new file mode 100644
index 0000000000..8f781ed97b
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_5_5e27737a5bb0e9e46c98708700318e67._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-07-27T12:20:07Z"
+ content="""
+After some debugging, I looked at the Utility/libkqueue.c and used it as a test, it seems to be hanging/segfaulting around the call to that library. Annoyingly I get segfaults from the library every so often on OSX, it's pretty a random event.
+"""]]

From 0f6292920ac360f78c3c4a3b9d883b758900c063 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 11:47:34 -0400
Subject: [PATCH 4296/8313] webapp now displays the real running and queued
 transfers

yowza!!!
---
 Assistant.hs                     |  2 +-
 Assistant/Threads/Transferrer.hs |  8 ++---
 Assistant/Threads/WebApp.hs      | 54 ++++++++++++++++++++------------
 Assistant/TransferQueue.hs       | 25 +++++++++++----
 Logs/Transfer.hs                 | 17 +++++++---
 templates/status.hamlet          | 48 +++++++++++++++-------------
 6 files changed, 96 insertions(+), 58 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index b539b27bc4..072aa3be3b 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -155,7 +155,7 @@ startDaemon assistant foreground
 				, mountWatcherThread st dstatus scanremotes
 				, transferScannerThread st scanremotes transferqueue
 #ifdef WITH_WEBAPP
-				, webAppThread st dstatus
+				, webAppThread st dstatus transferqueue
 #endif
 				, watchThread st dstatus transferqueue changechan
 				]
diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs
index 4ee5290e1c..d8a1469484 100644
--- a/Assistant/Threads/Transferrer.hs
+++ b/Assistant/Threads/Transferrer.hs
@@ -38,7 +38,7 @@ transfererThread st dstatus transferqueue slots = go
 			ifM (runThreadState st $ shouldTransfer dstatus t info)
 				( do
 					debug thisThread [ "Transferring:" , show t ]
-					runTransfer st dstatus slots t info
+					transferThread st dstatus slots t info
 				, debug thisThread [ "Skipping unnecessary transfer:" , show t ]
 				)
 			go
@@ -76,8 +76,8 @@ shouldTransfer dstatus t info =
  - thread's cache must be invalidated once a transfer completes, as
  - changes may have been made to the git-annex branch.
  -}
-runTransfer :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Transfer -> TransferInfo -> IO ()
-runTransfer st dstatus slots t info = case (transferRemote info, associatedFile info) of
+transferThread :: ThreadState -> DaemonStatusHandle -> TransferSlots -> Transfer -> TransferInfo -> IO ()
+transferThread st dstatus slots t info = case (transferRemote info, associatedFile info) of
 	(Nothing, _) -> noop
 	(_, Nothing) -> noop
 	(Just remote, Just file) -> do
@@ -99,7 +99,7 @@ runTransfer st dstatus slots t info = case (transferRemote info, associatedFile
 		transferprocess remote file = do
 			showStart "copy" file
 			showAction $ tofrom ++ " " ++ Remote.name remote
-			ok <- transfer t (Just file) $
+			ok <- runTransfer t (Just file) $
 				if isdownload
 					then getViaTmp key $
 						Remote.retrieveKeyFile remote key (Just file)
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 050d62cf17..171c7fd9c4 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -12,19 +12,26 @@ module Assistant.Threads.WebApp where
 import Assistant.Common
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
+import Assistant.TransferQueue
 import Utility.WebApp
 import Utility.Yesod
 import Utility.FileMode
 import Utility.TempFile
 import Git
+import Logs.Transfer
+import Utility.Percentage
+import Utility.DataUnits
+import Types.Key
+import qualified Remote
 
 import Yesod
 import Yesod.Static
 import Text.Hamlet
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
-import Data.Text
+import Data.Text (Text, pack, unpack)
 import Data.Time.Clock
+import qualified Data.Map as M
 
 thisThread :: String
 thisThread = "WebApp"
@@ -32,6 +39,7 @@ thisThread = "WebApp"
 data WebApp = WebApp
 	{ threadState :: ThreadState
 	, daemonStatus :: DaemonStatusHandle
+	, transferQueue :: TransferQueue
 	, secretToken :: Text
 	, baseTitle :: String
 	, getStatic :: Static
@@ -104,6 +112,12 @@ statusDisplay = do
 	webapp <- lift getYesod
 	time <- show <$> liftIO getCurrentTime
 	
+	current <- liftIO $ runThreadState (threadState webapp) $
+		M.toList . currentTransfers
+			<$> getDaemonStatus (daemonStatus webapp)
+	queued <- liftIO $ getTransferQueue $ transferQueue webapp
+	let transfers = current ++ queued
+
 	updating <- lift newIdent
 	$(widgetFile "status")
 	
@@ -131,31 +145,31 @@ getConfigR = defaultLayout $ do
 	setTitle "configuration"
 	[whamlet|<a href="@{HomeR}">main|]
 
-webAppThread :: ThreadState -> DaemonStatusHandle -> IO ()
-webAppThread st dstatus = do
-	webapp <- mkWebApp st dstatus
+webAppThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> IO ()
+webAppThread st dstatus transferqueue = do
+	webapp <- mkWebApp
 	app <- toWaiAppPlain webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
 		)
 	runWebApp app' $ \port -> runThreadState st $ writeHtmlShim webapp port
-
-mkWebApp :: ThreadState -> DaemonStatusHandle -> IO WebApp
-mkWebApp st dstatus = do
-	dir <- absPath =<< runThreadState st (fromRepo repoPath)
-	home <- myHomeDir
-	let reldir = if dirContains home dir
-		then relPathDirToFile home dir
-		else dir
-	token <- genRandomToken 
-	return $ WebApp 
-		{ threadState = st
-		, daemonStatus = dstatus
-		, secretToken = pack token
-		, baseTitle = reldir
-		, getStatic = $(embed "static")
-		}
+	where
+		mkWebApp = do
+			dir <- absPath =<< runThreadState st (fromRepo repoPath)
+			home <- myHomeDir
+			let reldir = if dirContains home dir
+				then relPathDirToFile home dir
+				else dir
+			token <- genRandomToken 
+			return $ WebApp 
+				{ threadState = st
+				, daemonStatus = dstatus
+				, transferQueue = transferqueue
+				, secretToken = pack token
+				, baseTitle = reldir
+				, getStatic = $(embed "static")
+				}
 
 {- Creates a html shim file that's used to redirect into the webapp,
  - to avoid exposing the secretToken when launching the web browser. -}
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index 9f0ea5cbe1..414a1f9bee 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -9,6 +9,7 @@ module Assistant.TransferQueue (
 	TransferQueue,
 	Schedule(..),
 	newTransferQueue,
+	getTransferQueue,
 	queueTransfers,
 	queueTransfer,
 	queueTransferAt,
@@ -24,17 +25,26 @@ import qualified Remote
 import Control.Concurrent.STM
 
 {- The transfer queue consists of a channel listing the transfers to make;
- - the size of the queue is also tracked -}
+ - the size of the queue is also tracked, and a list is maintained
+ - in parallel to allow for reading. -}
 data TransferQueue = TransferQueue
 	{ queue :: TChan (Transfer, TransferInfo)
 	, queuesize :: TVar Integer
+	, queuelist :: TVar [(Transfer, TransferInfo)]
 	}
 
 data Schedule = Next | Later
 	deriving (Eq)
 
 newTransferQueue :: IO TransferQueue
-newTransferQueue = atomically $ TransferQueue <$> newTChan <*> newTVar 0
+newTransferQueue = atomically $ TransferQueue
+	<$> newTChan
+	<*> newTVar 0
+	<*> newTVar []
+
+{- Reads the queue's content without blocking or changing it. -}
+getTransferQueue :: TransferQueue -> IO [(Transfer, TransferInfo)]
+getTransferQueue q = atomically $ readTVar $ queuelist q
 
 stubInfo :: AssociatedFile -> Remote -> TransferInfo
 stubInfo f r = TransferInfo
@@ -75,12 +85,14 @@ queueTransfers schedule q daemonstatus k f direction = do
 
 enqueue :: Schedule -> TransferQueue -> Transfer -> TransferInfo -> STM ()
 enqueue schedule q t info
-	| schedule == Next = go unGetTChan
-	| otherwise = go writeTChan
+	| schedule == Next = go unGetTChan (new:)
+	| otherwise = go writeTChan (\l -> l++[new])
 	where
-		go a = do
-			void $ a (queue q) (t, info)
+		new = (t, info)
+		go modqueue modlist = do
+			void $ modqueue (queue q) new
 			void $ modifyTVar' (queuesize q) succ
+			void $ modifyTVar' (queuelist q) modlist
 
 {- Adds a transfer to the queue. -}
 queueTransfer :: Schedule -> TransferQueue -> AssociatedFile -> Transfer -> Remote -> IO ()
@@ -100,4 +112,5 @@ queueTransferAt wantsz schedule q f t remote = atomically $ do
 getNextTransfer :: TransferQueue -> IO (Transfer, TransferInfo)
 getNextTransfer q = atomically $ do
 	void $ modifyTVar' (queuesize q) pred
+	void $ modifyTVar' (queuelist q) (drop 1)
 	readTChan (queue q)
diff --git a/Logs/Transfer.hs b/Logs/Transfer.hs
index b6962262d1..b0e21481cc 100644
--- a/Logs/Transfer.hs
+++ b/Logs/Transfer.hs
@@ -12,7 +12,9 @@ import Annex.Perms
 import Annex.Exception
 import qualified Git
 import Types.Remote
+import Types.Key
 import qualified Fields
+import Utility.Percentage
 
 import System.Posix.Types
 import Data.Time.Clock
@@ -58,24 +60,29 @@ readDirection "upload" = Just Upload
 readDirection "download" = Just Download
 readDirection _ = Nothing
 
+percentComplete :: Transfer -> TransferInfo -> Maybe Percentage
+percentComplete (Transfer { transferKey = key }) (TransferInfo { bytesComplete = Just complete }) =
+	(\size -> percentage size complete) <$> keySize key
+percentComplete _ _ = Nothing
+
 upload :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a
-upload u key file a = transfer (Transfer Upload u key) file a
+upload u key file a = runTransfer (Transfer Upload u key) file a
 
 download :: UUID -> Key -> AssociatedFile -> Annex a -> Annex a
-download u key file a = transfer (Transfer Download u key) file a
+download u key file a = runTransfer (Transfer Download u key) file a
 
 fieldTransfer :: Direction -> Key -> Annex a -> Annex a
 fieldTransfer direction key a = do
 	afile <- Fields.getField Fields.associatedFile
-	maybe a (\u -> transfer (Transfer direction (toUUID u) key) afile a)
+	maybe a (\u -> runTransfer (Transfer direction (toUUID u) key) afile a)
 		=<< Fields.getField Fields.remoteUUID
 
 {- Runs a transfer action. Creates and locks the lock file while the
  - action is running, and stores info in the transfer information
  - file. Will throw an error if the transfer is already in progress.
  -}
-transfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a
-transfer t file a = do
+runTransfer :: Transfer -> Maybe FilePath -> Annex a -> Annex a
+runTransfer t file a = do
 	tfile <- fromRepo $ transferFile t
 	createAnnexDirectory $ takeDirectory tfile
 	mode <- annexFileMode
diff --git a/templates/status.hamlet b/templates/status.hamlet
index 1da189d1f3..9b9b0f7d18 100644
--- a/templates/status.hamlet
+++ b/templates/status.hamlet
@@ -1,26 +1,30 @@
 <span id="#{updating}">
-  <div class="hero-unit">
-    <div class="row-fluid">
-      <h3>
-        foo ←
-        <small>usb drive</small>
-        <small class="pull-right">40% of 10 mb</small>
-    <div class="progress progress-striped">
-      <div class="bar" style="width: 40%;">
-    <div class="row-fluid">
-      <h3>
-        some_filenames_are_long_and_ugly_like_this_one.mp3 →
-        <small>Amazon S3</small>
-        <small class="pull-right">10% of 50 mb</small>
-    <div class="progress progress-striped">
-      <div class="bar" style="width: 10%;">
-    <div class="row-fluid">
-      <h3>
-        bigfile ←
-        <small>usb drive</small>
-        <small class="pull-right">0% of 512 mb</small>
-    <div class="progress progress-striped">
-      <div class="bar" style="width: 0%;">
+  <div class="span9">
+    $if null transfers
+      <h2>No current transfers
+    $else
+      <h2>Transfers
+      $forall (transfer, info) <- transfers
+        $with percent <- maybe "unknown" (showPercentage 0) $ percentComplete transfer info
+          <div class="row-fluid">
+            <h3>
+              $maybe file <- associatedFile info
+                #{file}
+              $nothing
+                #{show $ transferKey transfer}
+              $case transferDirection transfer
+                $of Upload
+                  →
+                $of Download
+                  ←
+              <small>#{maybe "unknown" Remote.name $ transferRemote info}</small>
+                $with size <- maybe "unknown" (roughSize dataUnits True) $ keySize $ transferKey transfer
+                  $if isJust $ startedTime info
+                      <small class="pull-right"><b>#{percent} of #{size}</b></small>
+                  $else
+                      <small class="pull-right">queued (#{size})</small>
+          <div class="progress progress-striped">
+            <div class="bar" style="width: #{percent};">
   <footer>
     <span>
       polled at #{time}

From 904bbd713fc6bd2bc1a6a478bc8276e6b96220f0 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Fri, 27 Jul 2012 15:53:50 +0000
Subject: [PATCH 4297/8313] Added a comment

---
 .../comment_5_c735841bc230efc61594ea013fc2902b._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment
new file mode 100644
index 0000000000..47f7be6033
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_5_c735841bc230efc61594ea013fc2902b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 5"
+ date="2012-07-27T15:53:50Z"
+ content="""
+I can confirm that the watch command is crashing when i start it up manually.
+"""]]

From 6dd4863f831c2031628d3e6ebf963bee7ff47b8a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 11:58:10 -0400
Subject: [PATCH 4298/8313] blog for the "day"

Or rather, for yesterday evening up until 6 am last night, and 3 hours this
morning. I need to fix my sleep schedule.
---
 .../assistant/blog/day_45__long_polling.mdwn  | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_45__long_polling.mdwn

diff --git a/doc/design/assistant/blog/day_45__long_polling.mdwn b/doc/design/assistant/blog/day_45__long_polling.mdwn
new file mode 100644
index 0000000000..380c55c127
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling.mdwn
@@ -0,0 +1,66 @@
+The webapp now displays actual progress bars, for the actual transfers
+that the assistant is making! And it's seriously shiny.
+
+[[!img full.png]]
+
+Yes, I used Bootstrap. I can see why so many people are using it,
+that the common complaint is everything looks the same. I spent a few hours
+mocking up the transfer display part of the WebApp using Bootstrap, and
+arrived at something that doesn't entirely suck remarkably quickly.
+
+The really sweet thing about Bootstrap is that when I resized my browser to
+the shape of a cell phone, it magically redrew the WebApp like so:
+
+[[!img phone.png]]
+
+---
+
+To update the display, the WebApp uses two techniques. On noscript
+browsers, it just uses a meta refresh, which is about the best I can do. I
+welcome feedback; it might be better to just have an "Update" button in
+this case.
+
+With javascript enabled, it uses long polling, done over AJAX. There are
+some other options I considered, including websockets, and server-sent
+events. Websockets seem too new, and while there's a WAI module supporting
+server-sent events, and even an example of them in the Yesod book, the
+module is not packaged for Debian yet. Anyway, long polling is the most
+widely supported, so a good starting place. It seems to work fine too, I
+don't really anticipate needing the more sophisticated methods.
+
+(Incidentially, this's the first time I've ever written code that uses AJAX.)
+
+Currently the status display is rendered in html by the web server, and
+just updated into place by javascript. I like this approach since it
+keeps the javascript code to a minimum and the pure haskell code to a
+maximum. But who knows, I may have to switch to JSON that gets rendered by
+javascript, for some reason, later on.
+
+---
+
+I was very happy with Yesod when I managed to factor out a
+general purpose widget that adds long-polling and meta-refresh to any
+other widget. I was less happy with Yesod when I tried to include
+jquery on my static site and it kept serving up a truncated version of it.
+Eventually worked around what's seemingly a bug in the default WAI
+middleware, by disabling that middleware.
+
+----
+
+Also yesterday I realized there were about 30 comments stuck in moderation on
+this website. I thought I had a feed of those, but obviously I didn't. I've
+posted them all, and also read them all.
+
+----
+
+Next up is probably some cleanup of bugs and minor todos. Including
+figuring out why `watch` has started to segfault on OSX when it was
+working fine before.
+
+After that, I need to build a way to block the long polling request
+until the DaemonStatus and/or TransferQueue change from the version
+previously displayed by the WebApp. An interesting concurrency problem..
+
+Once I have that working, I can reduce the current 3 second delay between
+refreshes to a very short delay, and the WebApp will update in
+near-realtime as changes come in.

From 3305e019db5938385172429216c96a9a1f328642 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 12:09:34 -0400
Subject: [PATCH 4299/8313] add yesod-default

another dependency cabal works without here, oddly
---
 debian/control   | 1 +
 doc/install.mdwn | 1 +
 git-annex.cabal  | 6 +++---
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/debian/control b/debian/control
index 1e7cb19e2c..89ca892834 100644
--- a/debian/control
+++ b/debian/control
@@ -25,6 +25,7 @@ Build-Depends:
 	libghc-dbus-dev,
 	libghc-yesod-dev,
 	libghc-yesod-static-dev,
+	libghc-yesod-default-dev,
 	libghc-case-insensitive-dev,
 	libghc-http-types-dev,
 	libghc-transformers-dev,
diff --git a/doc/install.mdwn b/doc/install.mdwn
index e529952611..76bffa00cf 100644
--- a/doc/install.mdwn
+++ b/doc/install.mdwn
@@ -51,6 +51,7 @@ libraries. To build and use git-annex by hand, you will need:
   * [dbus](http://hackage.haskell.org/package/dbus)
   * [yesod](http://hackage.haskell.org/package/yesod)
   * [yesod-static](http://hackage.haskell.org/package/yesod-static)
+  * [yesod-default](http://hackage.haskell.org/package/yesod-default)
   * [case-insensitive](http://hackage.haskell.org/package/case-insensitive)
   * [http-types](http://hackage.haskell.org/package/http-types)
   * [transformers](http://hackage.haskell.org/package/transformers)
diff --git a/git-annex.cabal b/git-annex.cabal
index aa71dacb6f..24e0df9c99 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -74,9 +74,9 @@ Executable git-annex
     CPP-Options: -DWITH_DBUS
 
   if flag(Webapp)
-    Build-Depends: yesod, yesod-static, case-insensitive, http-types,
-     transformers, wai, wai-logger, warp, blaze-builder, blaze-html,
-     blaze-markup, crypto-api, hamlet, clientsession
+    Build-Depends: yesod, yesod-static, yesod-default, case-insensitive,
+     http-types, transformers, wai, wai-logger, warp, blaze-builder,
+     blaze-html, blaze-markup, crypto-api, hamlet, clientsession
     CPP-Options: -DWITH_WEBAPP
 
   if os(darwin)

From 1407a3b82944ff1bf0ec6560affab042522d7317 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 12:10:25 -0400
Subject: [PATCH 4300/8313] close

---
 doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn b/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn
index 6dc9795f34..28727f97ff 100644
--- a/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn
+++ b/doc/bugs/yesod-default_is_needed_as_a_dependancy.mdwn
@@ -7,3 +7,4 @@ Utility/Yesod.hs:10:8:
 make: *** [git-annex] Error 1
 </pre>
 
+> Only on OSX apparently. Weird. Added. [[done]] --[[Joey]] 

From f1f90cb30b5318b3d13c1d401bd81a0747ec7b40 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 12:14:57 -0400
Subject: [PATCH 4301/8313] add warning when building without the webapp

---
 Assistant.hs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Assistant.hs b/Assistant.hs
index 072aa3be3b..6b155a4a67 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -115,6 +115,8 @@ import Assistant.Threads.MountWatcher
 import Assistant.Threads.TransferScanner
 #ifdef WITH_WEBAPP
 import Assistant.Threads.WebApp
+#else
+#warning Building without the webapp. You probably need to install Yesod..
 #endif
 import qualified Utility.Daemon
 import Utility.LogFile

From 0774dff546284fb4162d937c35449edf01ad3d60 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Fri, 27 Jul 2012 16:34:45 +0000
Subject: [PATCH 4302/8313] Added a comment

---
 ..._1f92da712232d050e085a4f39063d7a6._comment | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment
new file mode 100644
index 0000000000..2aa866410a
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_6_1f92da712232d050e085a4f39063d7a6._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.20"
+ subject="comment 6"
+ date="2012-07-27T16:34:45Z"
+ content="""
+Are you seeing libkqueue crash when it's called from a debugger or C program, rather than from Haskell?
+
+Are you building for 32 or 64 bit? You might try getting the 32 bit version of GCC (or The Haskell Platform) and see if it does better. There is a known GCC crashes on 64 bit OSX involving C libraries,
+although this bug report doesn't seem to apply, since we're not using ghci <http://hackage.haskell.org/trac/ghc/ticket/7040>.
+
+Are you building with cabal, or using the Makefile?
+
+You might try reverting git commit da4c506d61115236f3e43dd0bd17f30cd54df950
+
+You might try disabling the -threaded option in the cabal file or Makefile.
+
+I ssh'd to the OSX box I have an account on, and confirmed that git-annex watch still works there as of the current head of the `assistant` branch. That's a 64 bit GHC system, FWIW.
+
+Do you see the crash when building from the `master` branch, or only `assistant`? Master has the watch command, but it's much out of date, so this will tell if the problem was introduced recently... and you might still have to bisect it since I can't reproduce it. :(
+"""]]

From faaf0e7990ea99f0b2ee5e4295c65a2879010d01 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Fri, 27 Jul 2012 17:23:24 +0000
Subject: [PATCH 4303/8313] Added a comment

---
 ...nt_7_4153dc8029c545f8e86584a38bd536fb._comment | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment
new file mode 100644
index 0000000000..ccb4593ec8
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_7_4153dc8029c545f8e86584a38bd536fb._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
+ nickname="Jimmy"
+ subject="comment 7"
+ date="2012-07-27T17:23:24Z"
+ content="""
+I'm using make and a 64bit version of haskell-platform, it's fine on the master branch. It's just crashing on the assistant branch, I'm just thinking out loud, but could I share the binaries that I have with you (I'd like to grab your binaries too) to see see if its just some silly problem with my build environment.
+
+I'm seeing the crash when I'm running git-annex (in haskell), when I run libkqueue in a debugger it behaves randomly, is mostly succeeds, but every so often it fails. A back trace reveals nothing so I am a bit at a loss.
+
+I've tried disabling the threaded option, but it still crashes, I will give it another try later when I get home. The problem seems to occur on my desktop mac in work and my home mac, however it is fine on my linux machines.
+
+Could I ask which version of OSX do you have access to? is it 10.6 or 10.7 ?
+
+"""]]

From c93b546ebdd4693e2ce32150ca14b5a90e8b09ba Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 14:06:06 -0400
Subject: [PATCH 4304/8313] remove bogus AI_NUMERICSERV

Not needed, and causes a segfault on OSX when it tries to dereference the
NULL servicename. (Linux handles this case better.)
---
 Utility/WebApp.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 517251a7a9..69864dc6d3 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -69,7 +69,7 @@ localSocket = do
 	go $ Prelude.head addrs
 	where
 		hints = defaultHints
-			{ addrFlags = [AI_ADDRCONFIG, AI_NUMERICSERV]
+			{ addrFlags = [AI_ADDRCONFIG]
 			, addrSocketType = Stream
 			}
 		go addr = bracketOnError (open addr) close (use addr)

From 9506aae871f4ffae89313f78374786638ec379ee Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Fri, 27 Jul 2012 18:10:06 +0000
Subject: [PATCH 4305/8313] Added a comment

---
 .../comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment
new file mode 100644
index 0000000000..cbbd2933cf
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_8_f85b6eb5bfd28ffc6973fb4ab0fe4337._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.20"
+ subject="comment 8"
+ date="2012-07-27T18:10:06Z"
+ content="""
+I've reproduced a crash on OSX, involving not kqueue, but the WebApp's use of getaddrinfo. I've fixed that, but several things you've said in this bug report don't 100% add up to this being the same crash you've been seeing (for one thing, this can't affect `git annex watch`), so I'll wait for you to confirm.
+"""]]

From db0cdee88440fc33e1ea8e17c99b32016e65e812 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Fri, 27 Jul 2012 18:17:45 +0000
Subject: [PATCH 4306/8313] Added a comment

---
 ...comment_9_c747c488461c98cd285b51d3afc2c3eb._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment
new file mode 100644
index 0000000000..b23ae0125c
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_9_c747c488461c98cd285b51d3afc2c3eb._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 9"
+ date="2012-07-27T18:17:45Z"
+ content="""
+Ah, it's alive, I'm testing on my home machine right now and it's functioning as expected. I've tested on my work machine as well and the watch command works as expected! (short of the existing system limits which fails for my bigger annexes).
+
+Apologies for sending you on the wrong path with the kqueue path. I reckon if I give this a good testing over the weekend is a good idea before closing this bug, I will report back in a few days on this after giving a thrashing of files!
+"""]]

From 3a861a22765dfa065771888be427eb13afd25943 Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Fri, 27 Jul 2012 18:21:17 +0000
Subject: [PATCH 4307/8313] Added a comment

---
 .../comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment

diff --git a/doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment b/doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment
new file mode 100644
index 0000000000..9014923867
--- /dev/null
+++ b/doc/forum/Looking_at_the_webapp_on_OSX/comment_6_0e489fbfc89d282e9eb47f1b814ff70c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 6"
+ date="2012-07-27T18:21:17Z"
+ content="""
+FYI, the webapp starts up and sends me to a web browser with the correct page ;) this is after the other issue of [[Watch command as of commit 6cecc26206c4a539999b04664136c6f785211a41 segfaults]] got fixed.
+"""]]

From bc5b1516175f143f42bda2d12f512768d2df7c9e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 14:45:19 -0400
Subject: [PATCH 4308/8313] update to newer version of Bootstrap

This does fix some UI issues I was having.
---
 debian/copyright                    |    2 +-
 static/css/bootstrap-responsive.css |  563 +++---
 static/css/bootstrap.css            | 2473 +++++++++++++++++++--------
 3 files changed, 2080 insertions(+), 958 deletions(-)

diff --git a/debian/copyright b/debian/copyright
index 4cab3a048f..4e7345fd87 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -58,7 +58,7 @@ License: MIT or GPL-2
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 Files: static/*/bootstrap*
-Copyright: 2011-2012 Twitter, Inc.
+Copyright: 2012 Twitter, Inc.
 License: Apache-2.0
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
diff --git a/static/css/bootstrap-responsive.css b/static/css/bootstrap-responsive.css
index 0bc6de916b..06e55c0b45 100644
--- a/static/css/bootstrap-responsive.css
+++ b/static/css/bootstrap-responsive.css
@@ -1,5 +1,5 @@
 /*!
- * Bootstrap Responsive v2.0.2
+ * Bootstrap Responsive v2.0.4
  *
  * Copyright 2012 Twitter, Inc
  * Licensed under the Apache License v2.0
@@ -7,85 +7,86 @@
  *
  * Designed and built with all the love in the world @twitter by @mdo and @fat.
  */
+
 .clearfix {
   *zoom: 1;
 }
+
 .clearfix:before,
 .clearfix:after {
   display: table;
   content: "";
 }
+
 .clearfix:after {
   clear: both;
 }
+
 .hide-text {
-  overflow: hidden;
-  text-indent: 100%;
-  white-space: nowrap;
+  font: 0/0 a;
+  color: transparent;
+  text-shadow: none;
+  background-color: transparent;
+  border: 0;
 }
+
 .input-block-level {
   display: block;
   width: 100%;
   min-height: 28px;
-  /* Make inputs at least the height of their button counterpart */
-
-  /* Makes inputs behave like true block-level elements */
-
   -webkit-box-sizing: border-box;
-  -moz-box-sizing: border-box;
-  -ms-box-sizing: border-box;
-  box-sizing: border-box;
+     -moz-box-sizing: border-box;
+      -ms-box-sizing: border-box;
+          box-sizing: border-box;
 }
+
 .hidden {
   display: none;
   visibility: hidden;
 }
+
 .visible-phone {
-  display: none;
+  display: none !important;
 }
+
 .visible-tablet {
-  display: none;
-}
-.visible-desktop {
-  display: block;
-}
-.hidden-phone {
-  display: block;
-}
-.hidden-tablet {
-  display: block;
+  display: none !important;
 }
+
 .hidden-desktop {
-  display: none;
+  display: none !important;
 }
+
 @media (max-width: 767px) {
   .visible-phone {
-    display: block;
+    display: inherit !important;
   }
   .hidden-phone {
-    display: none;
+    display: none !important;
   }
   .hidden-desktop {
-    display: block;
+    display: inherit !important;
   }
   .visible-desktop {
-    display: none;
+    display: none !important;
   }
 }
+
 @media (min-width: 768px) and (max-width: 979px) {
   .visible-tablet {
-    display: block;
+    display: inherit !important;
   }
   .hidden-tablet {
-    display: none;
+    display: none !important;
   }
   .hidden-desktop {
-    display: block;
+    display: inherit !important;
   }
   .visible-desktop {
-    display: none;
+    display: none !important ;
   }
 }
+
 @media (max-width: 480px) {
   .nav-collapse {
     -webkit-transform: translate3d(0, 0, 0);
@@ -111,14 +112,14 @@
     padding-top: 0;
   }
   .form-horizontal .form-actions {
-    padding-left: 10px;
     padding-right: 10px;
+    padding-left: 10px;
   }
   .modal {
     position: absolute;
     top: 10px;
-    left: 10px;
     right: 10px;
+    left: 10px;
     width: auto;
     margin: 0;
   }
@@ -133,14 +134,28 @@
     position: static;
   }
 }
+
 @media (max-width: 767px) {
   body {
-    padding-left: 20px;
     padding-right: 20px;
+    padding-left: 20px;
   }
-  .navbar-fixed-top {
-    margin-left: -20px;
+  .navbar-fixed-top,
+  .navbar-fixed-bottom {
     margin-right: -20px;
+    margin-left: -20px;
+  }
+  .container-fluid {
+    padding: 0;
+  }
+  .dl-horizontal dt {
+    float: none;
+    width: auto;
+    clear: none;
+    text-align: left;
+  }
+  .dl-horizontal dd {
+    margin-left: 0;
   }
   .container {
     width: auto;
@@ -148,19 +163,20 @@
   .row-fluid {
     width: 100%;
   }
-  .row {
+  .row,
+  .thumbnails {
     margin-left: 0;
   }
-  .row > [class*="span"],
-  .row-fluid > [class*="span"] {
-    float: none;
+  [class*="span"],
+  .row-fluid [class*="span"] {
     display: block;
+    float: none;
     width: auto;
-    margin: 0;
-  }
-  .thumbnails [class*="span"] {
-    width: auto;
+    margin-left: 0;
   }
+  .input-large,
+  .input-xlarge,
+  .input-xxlarge,
   input[class*="span"],
   select[class*="span"],
   textarea[class*="span"],
@@ -168,20 +184,20 @@
     display: block;
     width: 100%;
     min-height: 28px;
-    /* Make inputs at least the height of their button counterpart */
-  
-    /* Makes inputs behave like true block-level elements */
-  
     -webkit-box-sizing: border-box;
-    -moz-box-sizing: border-box;
-    -ms-box-sizing: border-box;
-    box-sizing: border-box;
+       -moz-box-sizing: border-box;
+        -ms-box-sizing: border-box;
+            box-sizing: border-box;
   }
+  .input-prepend input,
+  .input-append input,
   .input-prepend input[class*="span"],
   .input-append input[class*="span"] {
+    display: inline-block;
     width: auto;
   }
 }
+
 @media (min-width: 768px) and (max-width: 979px) {
   .row {
     margin-left: -20px;
@@ -288,205 +304,136 @@
   .row-fluid:after {
     clear: both;
   }
-  .row-fluid > [class*="span"] {
+  .row-fluid [class*="span"] {
+    display: block;
     float: left;
+    width: 100%;
+    min-height: 28px;
     margin-left: 2.762430939%;
+    *margin-left: 2.709239449638298%;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+        -ms-box-sizing: border-box;
+            box-sizing: border-box;
   }
-  .row-fluid > [class*="span"]:first-child {
+  .row-fluid [class*="span"]:first-child {
     margin-left: 0;
   }
-  .row-fluid > .span12 {
+  .row-fluid .span12 {
     width: 99.999999993%;
+    *width: 99.9468085036383%;
   }
-  .row-fluid > .span11 {
+  .row-fluid .span11 {
     width: 91.436464082%;
+    *width: 91.38327259263829%;
   }
-  .row-fluid > .span10 {
+  .row-fluid .span10 {
     width: 82.87292817100001%;
+    *width: 82.8197366816383%;
   }
-  .row-fluid > .span9 {
+  .row-fluid .span9 {
     width: 74.30939226%;
+    *width: 74.25620077063829%;
   }
-  .row-fluid > .span8 {
+  .row-fluid .span8 {
     width: 65.74585634900001%;
+    *width: 65.6926648596383%;
   }
-  .row-fluid > .span7 {
+  .row-fluid .span7 {
     width: 57.182320438000005%;
+    *width: 57.129128948638304%;
   }
-  .row-fluid > .span6 {
+  .row-fluid .span6 {
     width: 48.618784527%;
+    *width: 48.5655930376383%;
   }
-  .row-fluid > .span5 {
+  .row-fluid .span5 {
     width: 40.055248616%;
+    *width: 40.0020571266383%;
   }
-  .row-fluid > .span4 {
+  .row-fluid .span4 {
     width: 31.491712705%;
+    *width: 31.4385212156383%;
   }
-  .row-fluid > .span3 {
+  .row-fluid .span3 {
     width: 22.928176794%;
+    *width: 22.874985304638297%;
   }
-  .row-fluid > .span2 {
+  .row-fluid .span2 {
     width: 14.364640883%;
+    *width: 14.311449393638298%;
   }
-  .row-fluid > .span1 {
+  .row-fluid .span1 {
     width: 5.801104972%;
+    *width: 5.747913482638298%;
   }
   input,
   textarea,
   .uneditable-input {
     margin-left: 0;
   }
-  input.span12, textarea.span12, .uneditable-input.span12 {
+  input.span12,
+  textarea.span12,
+  .uneditable-input.span12 {
     width: 714px;
   }
-  input.span11, textarea.span11, .uneditable-input.span11 {
+  input.span11,
+  textarea.span11,
+  .uneditable-input.span11 {
     width: 652px;
   }
-  input.span10, textarea.span10, .uneditable-input.span10 {
+  input.span10,
+  textarea.span10,
+  .uneditable-input.span10 {
     width: 590px;
   }
-  input.span9, textarea.span9, .uneditable-input.span9 {
+  input.span9,
+  textarea.span9,
+  .uneditable-input.span9 {
     width: 528px;
   }
-  input.span8, textarea.span8, .uneditable-input.span8 {
+  input.span8,
+  textarea.span8,
+  .uneditable-input.span8 {
     width: 466px;
   }
-  input.span7, textarea.span7, .uneditable-input.span7 {
+  input.span7,
+  textarea.span7,
+  .uneditable-input.span7 {
     width: 404px;
   }
-  input.span6, textarea.span6, .uneditable-input.span6 {
+  input.span6,
+  textarea.span6,
+  .uneditable-input.span6 {
     width: 342px;
   }
-  input.span5, textarea.span5, .uneditable-input.span5 {
+  input.span5,
+  textarea.span5,
+  .uneditable-input.span5 {
     width: 280px;
   }
-  input.span4, textarea.span4, .uneditable-input.span4 {
+  input.span4,
+  textarea.span4,
+  .uneditable-input.span4 {
     width: 218px;
   }
-  input.span3, textarea.span3, .uneditable-input.span3 {
+  input.span3,
+  textarea.span3,
+  .uneditable-input.span3 {
     width: 156px;
   }
-  input.span2, textarea.span2, .uneditable-input.span2 {
+  input.span2,
+  textarea.span2,
+  .uneditable-input.span2 {
     width: 94px;
   }
-  input.span1, textarea.span1, .uneditable-input.span1 {
+  input.span1,
+  textarea.span1,
+  .uneditable-input.span1 {
     width: 32px;
   }
 }
-@media (max-width: 979px) {
-  body {
-    padding-top: 0;
-  }
-  .navbar-fixed-top {
-    position: static;
-    margin-bottom: 18px;
-  }
-  .navbar-fixed-top .navbar-inner {
-    padding: 5px;
-  }
-  .navbar .container {
-    width: auto;
-    padding: 0;
-  }
-  .navbar .brand {
-    padding-left: 10px;
-    padding-right: 10px;
-    margin: 0 0 0 -5px;
-  }
-  .navbar .nav-collapse {
-    clear: left;
-  }
-  .navbar .nav {
-    float: none;
-    margin: 0 0 9px;
-  }
-  .navbar .nav > li {
-    float: none;
-  }
-  .navbar .nav > li > a {
-    margin-bottom: 2px;
-  }
-  .navbar .nav > .divider-vertical {
-    display: none;
-  }
-  .navbar .nav .nav-header {
-    color: #999999;
-    text-shadow: none;
-  }
-  .navbar .nav > li > a,
-  .navbar .dropdown-menu a {
-    padding: 6px 15px;
-    font-weight: bold;
-    color: #999999;
-    -webkit-border-radius: 3px;
-    -moz-border-radius: 3px;
-    border-radius: 3px;
-  }
-  .navbar .dropdown-menu li + li a {
-    margin-bottom: 2px;
-  }
-  .navbar .nav > li > a:hover,
-  .navbar .dropdown-menu a:hover {
-    background-color: #222222;
-  }
-  .navbar .dropdown-menu {
-    position: static;
-    top: auto;
-    left: auto;
-    float: none;
-    display: block;
-    max-width: none;
-    margin: 0 15px;
-    padding: 0;
-    background-color: transparent;
-    border: none;
-    -webkit-border-radius: 0;
-    -moz-border-radius: 0;
-    border-radius: 0;
-    -webkit-box-shadow: none;
-    -moz-box-shadow: none;
-    box-shadow: none;
-  }
-  .navbar .dropdown-menu:before,
-  .navbar .dropdown-menu:after {
-    display: none;
-  }
-  .navbar .dropdown-menu .divider {
-    display: none;
-  }
-  .navbar-form,
-  .navbar-search {
-    float: none;
-    padding: 9px 15px;
-    margin: 9px 0;
-    border-top: 1px solid #222222;
-    border-bottom: 1px solid #222222;
-    -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
-    -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
-    box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
-  }
-  .navbar .nav.pull-right {
-    float: none;
-    margin-left: 0;
-  }
-  .navbar-static .navbar-inner {
-    padding-left: 10px;
-    padding-right: 10px;
-  }
-  .btn-navbar {
-    display: block;
-  }
-  .nav-collapse {
-    overflow: hidden;
-    height: 0;
-  }
-}
-@media (min-width: 980px) {
-  .nav-collapse.collapse {
-    height: auto !important;
-    overflow: visible !important;
-  }
-}
+
 @media (min-width: 1200px) {
   .row {
     margin-left: -30px;
@@ -593,88 +540,132 @@
   .row-fluid:after {
     clear: both;
   }
-  .row-fluid > [class*="span"] {
+  .row-fluid [class*="span"] {
+    display: block;
     float: left;
+    width: 100%;
+    min-height: 28px;
     margin-left: 2.564102564%;
+    *margin-left: 2.510911074638298%;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+        -ms-box-sizing: border-box;
+            box-sizing: border-box;
   }
-  .row-fluid > [class*="span"]:first-child {
+  .row-fluid [class*="span"]:first-child {
     margin-left: 0;
   }
-  .row-fluid > .span12 {
+  .row-fluid .span12 {
     width: 100%;
+    *width: 99.94680851063829%;
   }
-  .row-fluid > .span11 {
+  .row-fluid .span11 {
     width: 91.45299145300001%;
+    *width: 91.3997999636383%;
   }
-  .row-fluid > .span10 {
+  .row-fluid .span10 {
     width: 82.905982906%;
+    *width: 82.8527914166383%;
   }
-  .row-fluid > .span9 {
+  .row-fluid .span9 {
     width: 74.358974359%;
+    *width: 74.30578286963829%;
   }
-  .row-fluid > .span8 {
+  .row-fluid .span8 {
     width: 65.81196581200001%;
+    *width: 65.7587743226383%;
   }
-  .row-fluid > .span7 {
+  .row-fluid .span7 {
     width: 57.264957265%;
+    *width: 57.2117657756383%;
   }
-  .row-fluid > .span6 {
+  .row-fluid .span6 {
     width: 48.717948718%;
+    *width: 48.6647572286383%;
   }
-  .row-fluid > .span5 {
+  .row-fluid .span5 {
     width: 40.170940171000005%;
+    *width: 40.117748681638304%;
   }
-  .row-fluid > .span4 {
+  .row-fluid .span4 {
     width: 31.623931624%;
+    *width: 31.5707401346383%;
   }
-  .row-fluid > .span3 {
+  .row-fluid .span3 {
     width: 23.076923077%;
+    *width: 23.0237315876383%;
   }
-  .row-fluid > .span2 {
+  .row-fluid .span2 {
     width: 14.529914530000001%;
+    *width: 14.4767230406383%;
   }
-  .row-fluid > .span1 {
+  .row-fluid .span1 {
     width: 5.982905983%;
+    *width: 5.929714493638298%;
   }
   input,
   textarea,
   .uneditable-input {
     margin-left: 0;
   }
-  input.span12, textarea.span12, .uneditable-input.span12 {
+  input.span12,
+  textarea.span12,
+  .uneditable-input.span12 {
     width: 1160px;
   }
-  input.span11, textarea.span11, .uneditable-input.span11 {
+  input.span11,
+  textarea.span11,
+  .uneditable-input.span11 {
     width: 1060px;
   }
-  input.span10, textarea.span10, .uneditable-input.span10 {
+  input.span10,
+  textarea.span10,
+  .uneditable-input.span10 {
     width: 960px;
   }
-  input.span9, textarea.span9, .uneditable-input.span9 {
+  input.span9,
+  textarea.span9,
+  .uneditable-input.span9 {
     width: 860px;
   }
-  input.span8, textarea.span8, .uneditable-input.span8 {
+  input.span8,
+  textarea.span8,
+  .uneditable-input.span8 {
     width: 760px;
   }
-  input.span7, textarea.span7, .uneditable-input.span7 {
+  input.span7,
+  textarea.span7,
+  .uneditable-input.span7 {
     width: 660px;
   }
-  input.span6, textarea.span6, .uneditable-input.span6 {
+  input.span6,
+  textarea.span6,
+  .uneditable-input.span6 {
     width: 560px;
   }
-  input.span5, textarea.span5, .uneditable-input.span5 {
+  input.span5,
+  textarea.span5,
+  .uneditable-input.span5 {
     width: 460px;
   }
-  input.span4, textarea.span4, .uneditable-input.span4 {
+  input.span4,
+  textarea.span4,
+  .uneditable-input.span4 {
     width: 360px;
   }
-  input.span3, textarea.span3, .uneditable-input.span3 {
+  input.span3,
+  textarea.span3,
+  .uneditable-input.span3 {
     width: 260px;
   }
-  input.span2, textarea.span2, .uneditable-input.span2 {
+  input.span2,
+  textarea.span2,
+  .uneditable-input.span2 {
     width: 160px;
   }
-  input.span1, textarea.span1, .uneditable-input.span1 {
+  input.span1,
+  textarea.span1,
+  .uneditable-input.span1 {
     width: 60px;
   }
   .thumbnails {
@@ -683,4 +674,142 @@
   .thumbnails > li {
     margin-left: 30px;
   }
+  .row-fluid .thumbnails {
+    margin-left: 0;
+  }
+}
+
+@media (max-width: 979px) {
+  body {
+    padding-top: 0;
+  }
+  .navbar-fixed-top,
+  .navbar-fixed-bottom {
+    position: static;
+  }
+  .navbar-fixed-top {
+    margin-bottom: 18px;
+  }
+  .navbar-fixed-bottom {
+    margin-top: 18px;
+  }
+  .navbar-fixed-top .navbar-inner,
+  .navbar-fixed-bottom .navbar-inner {
+    padding: 5px;
+  }
+  .navbar .container {
+    width: auto;
+    padding: 0;
+  }
+  .navbar .brand {
+    padding-right: 10px;
+    padding-left: 10px;
+    margin: 0 0 0 -5px;
+  }
+  .nav-collapse {
+    clear: both;
+  }
+  .nav-collapse .nav {
+    float: none;
+    margin: 0 0 9px;
+  }
+  .nav-collapse .nav > li {
+    float: none;
+  }
+  .nav-collapse .nav > li > a {
+    margin-bottom: 2px;
+  }
+  .nav-collapse .nav > .divider-vertical {
+    display: none;
+  }
+  .nav-collapse .nav .nav-header {
+    color: #999999;
+    text-shadow: none;
+  }
+  .nav-collapse .nav > li > a,
+  .nav-collapse .dropdown-menu a {
+    padding: 6px 15px;
+    font-weight: bold;
+    color: #999999;
+    -webkit-border-radius: 3px;
+       -moz-border-radius: 3px;
+            border-radius: 3px;
+  }
+  .nav-collapse .btn {
+    padding: 4px 10px 4px;
+    font-weight: normal;
+    -webkit-border-radius: 4px;
+       -moz-border-radius: 4px;
+            border-radius: 4px;
+  }
+  .nav-collapse .dropdown-menu li + li a {
+    margin-bottom: 2px;
+  }
+  .nav-collapse .nav > li > a:hover,
+  .nav-collapse .dropdown-menu a:hover {
+    background-color: #222222;
+  }
+  .nav-collapse.in .btn-group {
+    padding: 0;
+    margin-top: 5px;
+  }
+  .nav-collapse .dropdown-menu {
+    position: static;
+    top: auto;
+    left: auto;
+    display: block;
+    float: none;
+    max-width: none;
+    padding: 0;
+    margin: 0 15px;
+    background-color: transparent;
+    border: none;
+    -webkit-border-radius: 0;
+       -moz-border-radius: 0;
+            border-radius: 0;
+    -webkit-box-shadow: none;
+       -moz-box-shadow: none;
+            box-shadow: none;
+  }
+  .nav-collapse .dropdown-menu:before,
+  .nav-collapse .dropdown-menu:after {
+    display: none;
+  }
+  .nav-collapse .dropdown-menu .divider {
+    display: none;
+  }
+  .nav-collapse .navbar-form,
+  .nav-collapse .navbar-search {
+    float: none;
+    padding: 9px 15px;
+    margin: 9px 0;
+    border-top: 1px solid #222222;
+    border-bottom: 1px solid #222222;
+    -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+       -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+            box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+  }
+  .navbar .nav-collapse .nav.pull-right {
+    float: none;
+    margin-left: 0;
+  }
+  .nav-collapse,
+  .nav-collapse.collapse {
+    height: 0;
+    overflow: hidden;
+  }
+  .navbar .btn-navbar {
+    display: block;
+  }
+  .navbar-static .navbar-inner {
+    padding-right: 10px;
+    padding-left: 10px;
+  }
+}
+
+@media (min-width: 980px) {
+  .nav-collapse.collapse {
+    height: auto !important;
+    overflow: visible !important;
+  }
 }
diff --git a/static/css/bootstrap.css b/static/css/bootstrap.css
index dee87331f3..bb40c85f7d 100644
--- a/static/css/bootstrap.css
+++ b/static/css/bootstrap.css
@@ -1,5 +1,5 @@
 /*!
- * Bootstrap v2.0.2
+ * Bootstrap v2.0.4
  *
  * Copyright 2012 Twitter, Inc
  * Licensed under the Apache License v2.0
@@ -7,6 +7,7 @@
  *
  * Designed and built with all the love in the world @twitter by @mdo and @fat.
  */
+
 article,
 aside,
 details,
@@ -19,6 +20,7 @@ nav,
 section {
   display: block;
 }
+
 audio,
 canvas,
 video {
@@ -26,23 +28,28 @@ video {
   *display: inline;
   *zoom: 1;
 }
+
 audio:not([controls]) {
   display: none;
 }
+
 html {
   font-size: 100%;
   -webkit-text-size-adjust: 100%;
-  -ms-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
 }
+
 a:focus {
   outline: thin dotted #333;
   outline: 5px auto -webkit-focus-ring-color;
   outline-offset: -2px;
 }
+
 a:hover,
 a:active {
   outline: 0;
 }
+
 sub,
 sup {
   position: relative;
@@ -50,18 +57,26 @@ sup {
   line-height: 0;
   vertical-align: baseline;
 }
+
 sup {
   top: -0.5em;
 }
+
 sub {
   bottom: -0.25em;
 }
+
 img {
-  height: auto;
+  max-width: 100%;
+  vertical-align: middle;
   border: 0;
   -ms-interpolation-mode: bicubic;
-  vertical-align: middle;
 }
+
+#map_canvas img {
+  max-width: none;
+}
+
 button,
 input,
 select,
@@ -70,16 +85,19 @@ textarea {
   font-size: 100%;
   vertical-align: middle;
 }
+
 button,
 input {
   *overflow: visible;
   line-height: normal;
 }
+
 button::-moz-focus-inner,
 input::-moz-focus-inner {
   padding: 0;
   border: 0;
 }
+
 button,
 input[type="button"],
 input[type="reset"],
@@ -87,49 +105,56 @@ input[type="submit"] {
   cursor: pointer;
   -webkit-appearance: button;
 }
+
 input[type="search"] {
-  -webkit-appearance: textfield;
   -webkit-box-sizing: content-box;
-  -moz-box-sizing: content-box;
-  box-sizing: content-box;
+     -moz-box-sizing: content-box;
+          box-sizing: content-box;
+  -webkit-appearance: textfield;
 }
+
 input[type="search"]::-webkit-search-decoration,
 input[type="search"]::-webkit-search-cancel-button {
   -webkit-appearance: none;
 }
+
 textarea {
   overflow: auto;
   vertical-align: top;
 }
+
 .clearfix {
   *zoom: 1;
 }
+
 .clearfix:before,
 .clearfix:after {
   display: table;
   content: "";
 }
+
 .clearfix:after {
   clear: both;
 }
+
 .hide-text {
-  overflow: hidden;
-  text-indent: 100%;
-  white-space: nowrap;
+  font: 0/0 a;
+  color: transparent;
+  text-shadow: none;
+  background-color: transparent;
+  border: 0;
 }
+
 .input-block-level {
   display: block;
   width: 100%;
   min-height: 28px;
-  /* Make inputs at least the height of their button counterpart */
-
-  /* Makes inputs behave like true block-level elements */
-
   -webkit-box-sizing: border-box;
-  -moz-box-sizing: border-box;
-  -ms-box-sizing: border-box;
-  box-sizing: border-box;
+     -moz-box-sizing: border-box;
+      -ms-box-sizing: border-box;
+          box-sizing: border-box;
 }
+
 body {
   margin: 0;
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
@@ -138,204 +163,279 @@ body {
   color: #333333;
   background-color: #ffffff;
 }
+
 a {
   color: #0088cc;
   text-decoration: none;
 }
+
 a:hover {
   color: #005580;
   text-decoration: underline;
 }
+
 .row {
   margin-left: -20px;
   *zoom: 1;
 }
+
 .row:before,
 .row:after {
   display: table;
   content: "";
 }
+
 .row:after {
   clear: both;
 }
+
 [class*="span"] {
   float: left;
   margin-left: 20px;
 }
+
 .container,
 .navbar-fixed-top .container,
 .navbar-fixed-bottom .container {
   width: 940px;
 }
+
 .span12 {
   width: 940px;
 }
+
 .span11 {
   width: 860px;
 }
+
 .span10 {
   width: 780px;
 }
+
 .span9 {
   width: 700px;
 }
+
 .span8 {
   width: 620px;
 }
+
 .span7 {
   width: 540px;
 }
+
 .span6 {
   width: 460px;
 }
+
 .span5 {
   width: 380px;
 }
+
 .span4 {
   width: 300px;
 }
+
 .span3 {
   width: 220px;
 }
+
 .span2 {
   width: 140px;
 }
+
 .span1 {
   width: 60px;
 }
+
 .offset12 {
   margin-left: 980px;
 }
+
 .offset11 {
   margin-left: 900px;
 }
+
 .offset10 {
   margin-left: 820px;
 }
+
 .offset9 {
   margin-left: 740px;
 }
+
 .offset8 {
   margin-left: 660px;
 }
+
 .offset7 {
   margin-left: 580px;
 }
+
 .offset6 {
   margin-left: 500px;
 }
+
 .offset5 {
   margin-left: 420px;
 }
+
 .offset4 {
   margin-left: 340px;
 }
+
 .offset3 {
   margin-left: 260px;
 }
+
 .offset2 {
   margin-left: 180px;
 }
+
 .offset1 {
   margin-left: 100px;
 }
+
 .row-fluid {
   width: 100%;
   *zoom: 1;
 }
+
 .row-fluid:before,
 .row-fluid:after {
   display: table;
   content: "";
 }
+
 .row-fluid:after {
   clear: both;
 }
-.row-fluid > [class*="span"] {
+
+.row-fluid [class*="span"] {
+  display: block;
   float: left;
+  width: 100%;
+  min-height: 28px;
   margin-left: 2.127659574%;
+  *margin-left: 2.0744680846382977%;
+  -webkit-box-sizing: border-box;
+     -moz-box-sizing: border-box;
+      -ms-box-sizing: border-box;
+          box-sizing: border-box;
 }
-.row-fluid > [class*="span"]:first-child {
+
+.row-fluid [class*="span"]:first-child {
   margin-left: 0;
 }
-.row-fluid > .span12 {
+
+.row-fluid .span12 {
   width: 99.99999998999999%;
+  *width: 99.94680850063828%;
 }
-.row-fluid > .span11 {
+
+.row-fluid .span11 {
   width: 91.489361693%;
+  *width: 91.4361702036383%;
 }
-.row-fluid > .span10 {
+
+.row-fluid .span10 {
   width: 82.97872339599999%;
+  *width: 82.92553190663828%;
 }
-.row-fluid > .span9 {
+
+.row-fluid .span9 {
   width: 74.468085099%;
+  *width: 74.4148936096383%;
 }
-.row-fluid > .span8 {
+
+.row-fluid .span8 {
   width: 65.95744680199999%;
+  *width: 65.90425531263828%;
 }
-.row-fluid > .span7 {
+
+.row-fluid .span7 {
   width: 57.446808505%;
+  *width: 57.3936170156383%;
 }
-.row-fluid > .span6 {
+
+.row-fluid .span6 {
   width: 48.93617020799999%;
+  *width: 48.88297871863829%;
 }
-.row-fluid > .span5 {
+
+.row-fluid .span5 {
   width: 40.425531911%;
+  *width: 40.3723404216383%;
 }
-.row-fluid > .span4 {
+
+.row-fluid .span4 {
   width: 31.914893614%;
+  *width: 31.8617021246383%;
 }
-.row-fluid > .span3 {
+
+.row-fluid .span3 {
   width: 23.404255317%;
+  *width: 23.3510638276383%;
 }
-.row-fluid > .span2 {
+
+.row-fluid .span2 {
   width: 14.89361702%;
+  *width: 14.8404255306383%;
 }
-.row-fluid > .span1 {
+
+.row-fluid .span1 {
   width: 6.382978723%;
+  *width: 6.329787233638298%;
 }
+
 .container {
-  margin-left: auto;
   margin-right: auto;
+  margin-left: auto;
   *zoom: 1;
 }
+
 .container:before,
 .container:after {
   display: table;
   content: "";
 }
+
 .container:after {
   clear: both;
 }
+
 .container-fluid {
-  padding-left: 20px;
   padding-right: 20px;
+  padding-left: 20px;
   *zoom: 1;
 }
+
 .container-fluid:before,
 .container-fluid:after {
   display: table;
   content: "";
 }
+
 .container-fluid:after {
   clear: both;
 }
+
 p {
   margin: 0 0 9px;
-  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
-  font-size: 13px;
-  line-height: 18px;
 }
+
 p small {
   font-size: 11px;
   color: #999999;
 }
+
 .lead {
   margin-bottom: 18px;
   font-size: 20px;
   font-weight: 200;
   line-height: 27px;
 }
+
 h1,
 h2,
 h3,
@@ -348,6 +448,7 @@ h6 {
   color: inherit;
   text-rendering: optimizelegibility;
 }
+
 h1 small,
 h2 small,
 h3 small,
@@ -357,227 +458,280 @@ h6 small {
   font-weight: normal;
   color: #999999;
 }
+
 h1 {
   font-size: 30px;
   line-height: 36px;
 }
+
 h1 small {
   font-size: 18px;
 }
+
 h2 {
   font-size: 24px;
   line-height: 36px;
 }
+
 h2 small {
   font-size: 18px;
 }
+
 h3 {
-  line-height: 27px;
   font-size: 18px;
+  line-height: 27px;
 }
+
 h3 small {
   font-size: 14px;
 }
+
 h4,
 h5,
 h6 {
   line-height: 18px;
 }
+
 h4 {
   font-size: 14px;
 }
+
 h4 small {
   font-size: 12px;
 }
+
 h5 {
   font-size: 12px;
 }
+
 h6 {
   font-size: 11px;
   color: #999999;
   text-transform: uppercase;
 }
+
 .page-header {
   padding-bottom: 17px;
   margin: 18px 0;
   border-bottom: 1px solid #eeeeee;
 }
+
 .page-header h1 {
   line-height: 1;
 }
+
 ul,
 ol {
   padding: 0;
   margin: 0 0 9px 25px;
 }
+
 ul ul,
 ul ol,
 ol ol,
 ol ul {
   margin-bottom: 0;
 }
+
 ul {
   list-style: disc;
 }
+
 ol {
   list-style: decimal;
 }
+
 li {
   line-height: 18px;
 }
+
 ul.unstyled,
 ol.unstyled {
   margin-left: 0;
   list-style: none;
 }
+
 dl {
   margin-bottom: 18px;
 }
+
 dt,
 dd {
   line-height: 18px;
 }
+
 dt {
   font-weight: bold;
   line-height: 17px;
 }
+
 dd {
   margin-left: 9px;
 }
+
 .dl-horizontal dt {
   float: left;
-  clear: left;
   width: 120px;
+  overflow: hidden;
+  clear: left;
   text-align: right;
+  text-overflow: ellipsis;
+  white-space: nowrap;
 }
+
 .dl-horizontal dd {
   margin-left: 130px;
 }
+
 hr {
   margin: 18px 0;
   border: 0;
   border-top: 1px solid #eeeeee;
   border-bottom: 1px solid #ffffff;
 }
+
 strong {
   font-weight: bold;
 }
+
 em {
   font-style: italic;
 }
+
 .muted {
   color: #999999;
 }
+
 abbr[title] {
-  border-bottom: 1px dotted #ddd;
   cursor: help;
+  border-bottom: 1px dotted #999999;
 }
+
 abbr.initialism {
   font-size: 90%;
   text-transform: uppercase;
 }
+
 blockquote {
   padding: 0 0 0 15px;
   margin: 0 0 18px;
   border-left: 5px solid #eeeeee;
 }
+
 blockquote p {
   margin-bottom: 0;
   font-size: 16px;
   font-weight: 300;
   line-height: 22.5px;
 }
+
 blockquote small {
   display: block;
   line-height: 18px;
   color: #999999;
 }
+
 blockquote small:before {
   content: '\2014 \00A0';
 }
+
 blockquote.pull-right {
   float: right;
-  padding-left: 0;
   padding-right: 15px;
-  border-left: 0;
+  padding-left: 0;
   border-right: 5px solid #eeeeee;
+  border-left: 0;
 }
+
 blockquote.pull-right p,
 blockquote.pull-right small {
   text-align: right;
 }
+
 q:before,
 q:after,
 blockquote:before,
 blockquote:after {
   content: "";
 }
+
 address {
   display: block;
   margin-bottom: 18px;
-  line-height: 18px;
   font-style: normal;
+  line-height: 18px;
 }
+
 small {
   font-size: 100%;
 }
+
 cite {
   font-style: normal;
 }
+
 code,
 pre {
   padding: 0 3px 2px;
-  font-family: Menlo, Monaco, "Courier New", monospace;
+  font-family: Menlo, Monaco, Consolas, "Courier New", monospace;
   font-size: 12px;
   color: #333333;
   -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
 }
+
 code {
   padding: 2px 4px;
   color: #d14;
   background-color: #f7f7f9;
   border: 1px solid #e1e1e8;
 }
+
 pre {
   display: block;
   padding: 8.5px;
   margin: 0 0 9px;
   font-size: 12.025px;
   line-height: 18px;
+  word-break: break-all;
+  word-wrap: break-word;
+  white-space: pre;
+  white-space: pre-wrap;
   background-color: #f5f5f5;
   border: 1px solid #ccc;
   border: 1px solid rgba(0, 0, 0, 0.15);
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
-  white-space: pre;
-  white-space: pre-wrap;
-  word-break: break-all;
-  word-wrap: break-word;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 pre.prettyprint {
   margin-bottom: 18px;
 }
+
 pre code {
   padding: 0;
   color: inherit;
   background-color: transparent;
   border: 0;
 }
+
 .pre-scrollable {
   max-height: 340px;
   overflow-y: scroll;
 }
+
 form {
   margin: 0 0 18px;
 }
+
 fieldset {
   padding: 0;
   margin: 0;
   border: 0;
 }
+
 legend {
   display: block;
   width: 100%;
@@ -587,12 +741,14 @@ legend {
   line-height: 36px;
   color: #333333;
   border: 0;
-  border-bottom: 1px solid #eee;
+  border-bottom: 1px solid #e5e5e5;
 }
+
 legend small {
   font-size: 13.5px;
   color: #999999;
 }
+
 label,
 input,
 button,
@@ -602,82 +758,134 @@ textarea {
   font-weight: normal;
   line-height: 18px;
 }
+
 input,
 button,
 select,
 textarea {
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
 }
+
 label {
   display: block;
   margin-bottom: 5px;
-  color: #333333;
 }
-input,
-textarea,
+
 select,
+textarea,
+input[type="text"],
+input[type="password"],
+input[type="datetime"],
+input[type="datetime-local"],
+input[type="date"],
+input[type="month"],
+input[type="time"],
+input[type="week"],
+input[type="number"],
+input[type="email"],
+input[type="url"],
+input[type="search"],
+input[type="tel"],
+input[type="color"],
 .uneditable-input {
   display: inline-block;
-  width: 210px;
   height: 18px;
   padding: 4px;
   margin-bottom: 9px;
   font-size: 13px;
   line-height: 18px;
   color: #555555;
+}
+
+input,
+textarea {
+  width: 210px;
+}
+
+textarea {
+  height: auto;
+}
+
+textarea,
+input[type="text"],
+input[type="password"],
+input[type="datetime"],
+input[type="datetime-local"],
+input[type="date"],
+input[type="month"],
+input[type="time"],
+input[type="week"],
+input[type="number"],
+input[type="email"],
+input[type="url"],
+input[type="search"],
+input[type="tel"],
+input[type="color"],
+.uneditable-input {
+  background-color: #ffffff;
   border: 1px solid #cccccc;
   -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+  -webkit-transition: border linear 0.2s, box-shadow linear 0.2s;
+     -moz-transition: border linear 0.2s, box-shadow linear 0.2s;
+      -ms-transition: border linear 0.2s, box-shadow linear 0.2s;
+       -o-transition: border linear 0.2s, box-shadow linear 0.2s;
+          transition: border linear 0.2s, box-shadow linear 0.2s;
 }
-.uneditable-textarea {
-  width: auto;
-  height: auto;
+
+textarea:focus,
+input[type="text"]:focus,
+input[type="password"]:focus,
+input[type="datetime"]:focus,
+input[type="datetime-local"]:focus,
+input[type="date"]:focus,
+input[type="month"]:focus,
+input[type="time"]:focus,
+input[type="week"]:focus,
+input[type="number"]:focus,
+input[type="email"]:focus,
+input[type="url"]:focus,
+input[type="search"]:focus,
+input[type="tel"]:focus,
+input[type="color"]:focus,
+.uneditable-input:focus {
+  border-color: rgba(82, 168, 236, 0.8);
+  outline: 0;
+  outline: thin dotted \9;
+  /* IE6-9 */
+
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
 }
-label input,
-label textarea,
-label select {
-  display: block;
-}
-input[type="image"],
-input[type="checkbox"],
-input[type="radio"] {
-  width: auto;
-  height: auto;
-  padding: 0;
+
+input[type="radio"],
+input[type="checkbox"] {
   margin: 3px 0;
   *margin-top: 0;
   /* IE7 */
 
   line-height: normal;
   cursor: pointer;
-  -webkit-border-radius: 0;
-  -moz-border-radius: 0;
-  border-radius: 0;
-  border: 0 \9;
-  /* IE9 and down */
+}
 
-}
-input[type="image"] {
-  border: 0;
-}
-input[type="file"] {
-  width: auto;
-  padding: initial;
-  line-height: initial;
-  border: initial;
-  background-color: #ffffff;
-  background-color: initial;
-  -webkit-box-shadow: none;
-  -moz-box-shadow: none;
-  box-shadow: none;
-}
-input[type="button"],
+input[type="submit"],
 input[type="reset"],
-input[type="submit"] {
+input[type="button"],
+input[type="radio"],
+input[type="checkbox"] {
+  width: auto;
+}
+
+.uneditable-textarea {
   width: auto;
   height: auto;
 }
+
 select,
 input[type="file"] {
   height: 28px;
@@ -688,41 +896,43 @@ input[type="file"] {
 
   line-height: 28px;
 }
-input[type="file"] {
-  line-height: 18px \9;
-}
+
 select {
   width: 220px;
-  background-color: #ffffff;
+  border: 1px solid #bbb;
 }
+
 select[multiple],
 select[size] {
   height: auto;
 }
-input[type="image"] {
-  -webkit-box-shadow: none;
-  -moz-box-shadow: none;
-  box-shadow: none;
-}
-textarea {
-  height: auto;
-}
-input[type="hidden"] {
-  display: none;
+
+select:focus,
+input[type="file"]:focus,
+input[type="radio"]:focus,
+input[type="checkbox"]:focus {
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
 }
+
 .radio,
 .checkbox {
+  min-height: 18px;
   padding-left: 18px;
 }
+
 .radio input[type="radio"],
 .checkbox input[type="checkbox"] {
   float: left;
   margin-left: -18px;
 }
+
 .controls > .radio:first-child,
 .controls > .checkbox:first-child {
   padding-top: 5px;
 }
+
 .radio.inline,
 .checkbox.inline {
   display: inline-block;
@@ -730,290 +940,368 @@ input[type="hidden"] {
   margin-bottom: 0;
   vertical-align: middle;
 }
+
 .radio.inline + .radio.inline,
 .checkbox.inline + .checkbox.inline {
   margin-left: 10px;
 }
-input,
-textarea {
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-  -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
-  -webkit-transition: border linear 0.2s, box-shadow linear 0.2s;
-  -moz-transition: border linear 0.2s, box-shadow linear 0.2s;
-  -ms-transition: border linear 0.2s, box-shadow linear 0.2s;
-  -o-transition: border linear 0.2s, box-shadow linear 0.2s;
-  transition: border linear 0.2s, box-shadow linear 0.2s;
-}
-input:focus,
-textarea:focus {
-  border-color: rgba(82, 168, 236, 0.8);
-  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
-  -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
-  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
-  outline: 0;
-  outline: thin dotted \9;
-  /* IE6-9 */
 
-}
-input[type="file"]:focus,
-input[type="radio"]:focus,
-input[type="checkbox"]:focus,
-select:focus {
-  -webkit-box-shadow: none;
-  -moz-box-shadow: none;
-  box-shadow: none;
-  outline: thin dotted #333;
-  outline: 5px auto -webkit-focus-ring-color;
-  outline-offset: -2px;
-}
 .input-mini {
   width: 60px;
 }
+
 .input-small {
   width: 90px;
 }
+
 .input-medium {
   width: 150px;
 }
+
 .input-large {
   width: 210px;
 }
+
 .input-xlarge {
   width: 270px;
 }
+
 .input-xxlarge {
   width: 530px;
 }
+
 input[class*="span"],
 select[class*="span"],
 textarea[class*="span"],
-.uneditable-input {
+.uneditable-input[class*="span"],
+.row-fluid input[class*="span"],
+.row-fluid select[class*="span"],
+.row-fluid textarea[class*="span"],
+.row-fluid .uneditable-input[class*="span"] {
   float: none;
   margin-left: 0;
 }
+
+.input-append input[class*="span"],
+.input-append .uneditable-input[class*="span"],
+.input-prepend input[class*="span"],
+.input-prepend .uneditable-input[class*="span"],
+.row-fluid .input-prepend [class*="span"],
+.row-fluid .input-append [class*="span"] {
+  display: inline-block;
+}
+
 input,
 textarea,
 .uneditable-input {
   margin-left: 0;
 }
-input.span12, textarea.span12, .uneditable-input.span12 {
+
+input.span12,
+textarea.span12,
+.uneditable-input.span12 {
   width: 930px;
 }
-input.span11, textarea.span11, .uneditable-input.span11 {
+
+input.span11,
+textarea.span11,
+.uneditable-input.span11 {
   width: 850px;
 }
-input.span10, textarea.span10, .uneditable-input.span10 {
+
+input.span10,
+textarea.span10,
+.uneditable-input.span10 {
   width: 770px;
 }
-input.span9, textarea.span9, .uneditable-input.span9 {
+
+input.span9,
+textarea.span9,
+.uneditable-input.span9 {
   width: 690px;
 }
-input.span8, textarea.span8, .uneditable-input.span8 {
+
+input.span8,
+textarea.span8,
+.uneditable-input.span8 {
   width: 610px;
 }
-input.span7, textarea.span7, .uneditable-input.span7 {
+
+input.span7,
+textarea.span7,
+.uneditable-input.span7 {
   width: 530px;
 }
-input.span6, textarea.span6, .uneditable-input.span6 {
+
+input.span6,
+textarea.span6,
+.uneditable-input.span6 {
   width: 450px;
 }
-input.span5, textarea.span5, .uneditable-input.span5 {
+
+input.span5,
+textarea.span5,
+.uneditable-input.span5 {
   width: 370px;
 }
-input.span4, textarea.span4, .uneditable-input.span4 {
+
+input.span4,
+textarea.span4,
+.uneditable-input.span4 {
   width: 290px;
 }
-input.span3, textarea.span3, .uneditable-input.span3 {
+
+input.span3,
+textarea.span3,
+.uneditable-input.span3 {
   width: 210px;
 }
-input.span2, textarea.span2, .uneditable-input.span2 {
+
+input.span2,
+textarea.span2,
+.uneditable-input.span2 {
   width: 130px;
 }
-input.span1, textarea.span1, .uneditable-input.span1 {
+
+input.span1,
+textarea.span1,
+.uneditable-input.span1 {
   width: 50px;
 }
+
 input[disabled],
 select[disabled],
 textarea[disabled],
 input[readonly],
 select[readonly],
 textarea[readonly] {
+  cursor: not-allowed;
   background-color: #eeeeee;
   border-color: #ddd;
-  cursor: not-allowed;
 }
+
+input[type="radio"][disabled],
+input[type="checkbox"][disabled],
+input[type="radio"][readonly],
+input[type="checkbox"][readonly] {
+  background-color: transparent;
+}
+
 .control-group.warning > label,
 .control-group.warning .help-block,
 .control-group.warning .help-inline {
   color: #c09853;
 }
+
+.control-group.warning .checkbox,
+.control-group.warning .radio,
 .control-group.warning input,
 .control-group.warning select,
 .control-group.warning textarea {
   color: #c09853;
   border-color: #c09853;
 }
+
+.control-group.warning .checkbox:focus,
+.control-group.warning .radio:focus,
 .control-group.warning input:focus,
 .control-group.warning select:focus,
 .control-group.warning textarea:focus {
   border-color: #a47e3c;
   -webkit-box-shadow: 0 0 6px #dbc59e;
-  -moz-box-shadow: 0 0 6px #dbc59e;
-  box-shadow: 0 0 6px #dbc59e;
+     -moz-box-shadow: 0 0 6px #dbc59e;
+          box-shadow: 0 0 6px #dbc59e;
 }
+
 .control-group.warning .input-prepend .add-on,
 .control-group.warning .input-append .add-on {
   color: #c09853;
   background-color: #fcf8e3;
   border-color: #c09853;
 }
+
 .control-group.error > label,
 .control-group.error .help-block,
 .control-group.error .help-inline {
   color: #b94a48;
 }
+
+.control-group.error .checkbox,
+.control-group.error .radio,
 .control-group.error input,
 .control-group.error select,
 .control-group.error textarea {
   color: #b94a48;
   border-color: #b94a48;
 }
+
+.control-group.error .checkbox:focus,
+.control-group.error .radio:focus,
 .control-group.error input:focus,
 .control-group.error select:focus,
 .control-group.error textarea:focus {
   border-color: #953b39;
   -webkit-box-shadow: 0 0 6px #d59392;
-  -moz-box-shadow: 0 0 6px #d59392;
-  box-shadow: 0 0 6px #d59392;
+     -moz-box-shadow: 0 0 6px #d59392;
+          box-shadow: 0 0 6px #d59392;
 }
+
 .control-group.error .input-prepend .add-on,
 .control-group.error .input-append .add-on {
   color: #b94a48;
   background-color: #f2dede;
   border-color: #b94a48;
 }
+
 .control-group.success > label,
 .control-group.success .help-block,
 .control-group.success .help-inline {
   color: #468847;
 }
+
+.control-group.success .checkbox,
+.control-group.success .radio,
 .control-group.success input,
 .control-group.success select,
 .control-group.success textarea {
   color: #468847;
   border-color: #468847;
 }
+
+.control-group.success .checkbox:focus,
+.control-group.success .radio:focus,
 .control-group.success input:focus,
 .control-group.success select:focus,
 .control-group.success textarea:focus {
   border-color: #356635;
   -webkit-box-shadow: 0 0 6px #7aba7b;
-  -moz-box-shadow: 0 0 6px #7aba7b;
-  box-shadow: 0 0 6px #7aba7b;
+     -moz-box-shadow: 0 0 6px #7aba7b;
+          box-shadow: 0 0 6px #7aba7b;
 }
+
 .control-group.success .input-prepend .add-on,
 .control-group.success .input-append .add-on {
   color: #468847;
   background-color: #dff0d8;
   border-color: #468847;
 }
+
 input:focus:required:invalid,
 textarea:focus:required:invalid,
 select:focus:required:invalid {
   color: #b94a48;
   border-color: #ee5f5b;
 }
+
 input:focus:required:invalid:focus,
 textarea:focus:required:invalid:focus,
 select:focus:required:invalid:focus {
   border-color: #e9322d;
   -webkit-box-shadow: 0 0 6px #f8b9b7;
-  -moz-box-shadow: 0 0 6px #f8b9b7;
-  box-shadow: 0 0 6px #f8b9b7;
+     -moz-box-shadow: 0 0 6px #f8b9b7;
+          box-shadow: 0 0 6px #f8b9b7;
 }
+
 .form-actions {
   padding: 17px 20px 18px;
   margin-top: 18px;
   margin-bottom: 18px;
-  background-color: #eeeeee;
-  border-top: 1px solid #ddd;
+  background-color: #f5f5f5;
+  border-top: 1px solid #e5e5e5;
   *zoom: 1;
 }
+
 .form-actions:before,
 .form-actions:after {
   display: table;
   content: "";
 }
+
 .form-actions:after {
   clear: both;
 }
+
 .uneditable-input {
-  display: block;
+  overflow: hidden;
+  white-space: nowrap;
+  cursor: not-allowed;
   background-color: #ffffff;
   border-color: #eee;
   -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
-  -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
-  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
-  cursor: not-allowed;
+     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
 }
+
 :-moz-placeholder {
   color: #999999;
 }
+
+:-ms-input-placeholder {
+  color: #999999;
+}
+
 ::-webkit-input-placeholder {
   color: #999999;
 }
+
 .help-block,
 .help-inline {
   color: #555555;
 }
+
 .help-block {
   display: block;
   margin-bottom: 9px;
 }
+
 .help-inline {
   display: inline-block;
   *display: inline;
-  /* IE7 inline-block hack */
-
-  *zoom: 1;
-  vertical-align: middle;
   padding-left: 5px;
+  vertical-align: middle;
+  *zoom: 1;
 }
+
 .input-prepend,
 .input-append {
   margin-bottom: 5px;
 }
+
 .input-prepend input,
 .input-append input,
 .input-prepend select,
 .input-append select,
 .input-prepend .uneditable-input,
 .input-append .uneditable-input {
+  position: relative;
+  margin-bottom: 0;
   *margin-left: 0;
+  vertical-align: middle;
   -webkit-border-radius: 0 3px 3px 0;
-  -moz-border-radius: 0 3px 3px 0;
-  border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
 }
+
 .input-prepend input:focus,
 .input-append input:focus,
 .input-prepend select:focus,
 .input-append select:focus,
 .input-prepend .uneditable-input:focus,
 .input-append .uneditable-input:focus {
-  position: relative;
   z-index: 2;
 }
+
 .input-prepend .uneditable-input,
 .input-append .uneditable-input {
   border-left-color: #ccc;
 }
+
 .input-prepend .add-on,
 .input-append .add-on {
   display: inline-block;
   width: auto;
-  min-width: 16px;
   height: 18px;
+  min-width: 16px;
   padding: 4px 5px;
   font-weight: normal;
   line-height: 18px;
@@ -1023,69 +1311,92 @@ select:focus:required:invalid:focus {
   background-color: #eeeeee;
   border: 1px solid #ccc;
 }
+
 .input-prepend .add-on,
 .input-append .add-on,
 .input-prepend .btn,
 .input-append .btn {
-  -webkit-border-radius: 3px 0 0 3px;
-  -moz-border-radius: 3px 0 0 3px;
-  border-radius: 3px 0 0 3px;
+  margin-left: -1px;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
 }
+
 .input-prepend .active,
 .input-append .active {
   background-color: #a9dba9;
   border-color: #46a546;
 }
+
 .input-prepend .add-on,
 .input-prepend .btn {
   margin-right: -1px;
 }
-.input-append input,
-.input-append select .uneditable-input {
+
+.input-prepend .add-on:first-child,
+.input-prepend .btn:first-child {
   -webkit-border-radius: 3px 0 0 3px;
-  -moz-border-radius: 3px 0 0 3px;
-  border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
 }
+
+.input-append input,
+.input-append select,
+.input-append .uneditable-input {
+  -webkit-border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
+}
+
 .input-append .uneditable-input {
-  border-left-color: #eee;
   border-right-color: #ccc;
+  border-left-color: #eee;
 }
-.input-append .add-on,
-.input-append .btn {
-  margin-left: -1px;
+
+.input-append .add-on:last-child,
+.input-append .btn:last-child {
   -webkit-border-radius: 0 3px 3px 0;
-  -moz-border-radius: 0 3px 3px 0;
-  border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
 }
+
 .input-prepend.input-append input,
 .input-prepend.input-append select,
 .input-prepend.input-append .uneditable-input {
   -webkit-border-radius: 0;
-  -moz-border-radius: 0;
-  border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
 }
+
 .input-prepend.input-append .add-on:first-child,
 .input-prepend.input-append .btn:first-child {
   margin-right: -1px;
   -webkit-border-radius: 3px 0 0 3px;
-  -moz-border-radius: 3px 0 0 3px;
-  border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
 }
+
 .input-prepend.input-append .add-on:last-child,
 .input-prepend.input-append .btn:last-child {
   margin-left: -1px;
   -webkit-border-radius: 0 3px 3px 0;
-  -moz-border-radius: 0 3px 3px 0;
-  border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
 }
+
 .search-query {
-  padding-left: 14px;
   padding-right: 14px;
+  padding-right: 4px \9;
+  padding-left: 14px;
+  padding-left: 4px \9;
+  /* IE7-8 doesn't have border-radius, so don't indent the padding */
+
   margin-bottom: 0;
   -webkit-border-radius: 14px;
-  -moz-border-radius: 14px;
-  border-radius: 14px;
+     -moz-border-radius: 14px;
+          border-radius: 14px;
 }
+
 .form-search input,
 .form-inline input,
 .form-horizontal input,
@@ -1108,23 +1419,29 @@ select:focus:required:invalid:focus {
 .form-inline .input-append,
 .form-horizontal .input-append {
   display: inline-block;
+  *display: inline;
   margin-bottom: 0;
+  *zoom: 1;
 }
+
 .form-search .hide,
 .form-inline .hide,
 .form-horizontal .hide {
   display: none;
 }
+
 .form-search label,
 .form-inline label {
   display: inline-block;
 }
+
 .form-search .input-append,
 .form-inline .input-append,
 .form-search .input-prepend,
 .form-inline .input-prepend {
   margin-bottom: 0;
 }
+
 .form-search .radio,
 .form-search .checkbox,
 .form-inline .radio,
@@ -1133,64 +1450,79 @@ select:focus:required:invalid:focus {
   margin-bottom: 0;
   vertical-align: middle;
 }
+
 .form-search .radio input[type="radio"],
 .form-search .checkbox input[type="checkbox"],
 .form-inline .radio input[type="radio"],
 .form-inline .checkbox input[type="checkbox"] {
   float: left;
-  margin-left: 0;
   margin-right: 3px;
+  margin-left: 0;
 }
+
 .control-group {
   margin-bottom: 9px;
 }
+
 legend + .control-group {
   margin-top: 18px;
   -webkit-margin-top-collapse: separate;
 }
+
 .form-horizontal .control-group {
   margin-bottom: 18px;
   *zoom: 1;
 }
+
 .form-horizontal .control-group:before,
 .form-horizontal .control-group:after {
   display: table;
   content: "";
 }
+
 .form-horizontal .control-group:after {
   clear: both;
 }
+
 .form-horizontal .control-label {
   float: left;
   width: 140px;
   padding-top: 5px;
   text-align: right;
 }
-.form-horizontal .controls {
-  margin-left: 160px;
-  /* Super jank IE7 fix to ensure the inputs in .input-append and input-prepend don't inherit the margin of the parent, in this case .controls */
 
+.form-horizontal .controls {
   *display: inline-block;
-  *margin-left: 0;
   *padding-left: 20px;
+  margin-left: 160px;
+  *margin-left: 0;
 }
+
+.form-horizontal .controls:first-child {
+  *padding-left: 160px;
+}
+
 .form-horizontal .help-block {
   margin-top: 9px;
   margin-bottom: 0;
 }
+
 .form-horizontal .form-actions {
   padding-left: 160px;
 }
+
 table {
   max-width: 100%;
+  background-color: transparent;
   border-collapse: collapse;
   border-spacing: 0;
-  background-color: transparent;
 }
+
 .table {
   width: 100%;
   margin-bottom: 18px;
 }
+
 .table th,
 .table td {
   padding: 8px;
@@ -1199,646 +1531,907 @@ table {
   vertical-align: top;
   border-top: 1px solid #dddddd;
 }
+
 .table th {
   font-weight: bold;
 }
+
 .table thead th {
   vertical-align: bottom;
 }
+
+.table caption + thead tr:first-child th,
+.table caption + thead tr:first-child td,
 .table colgroup + thead tr:first-child th,
 .table colgroup + thead tr:first-child td,
 .table thead:first-child tr:first-child th,
 .table thead:first-child tr:first-child td {
   border-top: 0;
 }
+
 .table tbody + tbody {
   border-top: 2px solid #dddddd;
 }
+
 .table-condensed th,
 .table-condensed td {
   padding: 4px 5px;
 }
+
 .table-bordered {
   border: 1px solid #dddddd;
-  border-left: 0;
   border-collapse: separate;
   *border-collapse: collapsed;
+  border-left: 0;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 .table-bordered th,
 .table-bordered td {
   border-left: 1px solid #dddddd;
 }
+
+.table-bordered caption + thead tr:first-child th,
+.table-bordered caption + tbody tr:first-child th,
+.table-bordered caption + tbody tr:first-child td,
+.table-bordered colgroup + thead tr:first-child th,
+.table-bordered colgroup + tbody tr:first-child th,
+.table-bordered colgroup + tbody tr:first-child td,
 .table-bordered thead:first-child tr:first-child th,
 .table-bordered tbody:first-child tr:first-child th,
 .table-bordered tbody:first-child tr:first-child td {
   border-top: 0;
 }
+
 .table-bordered thead:first-child tr:first-child th:first-child,
 .table-bordered tbody:first-child tr:first-child td:first-child {
-  -webkit-border-radius: 4px 0 0 0;
-  -moz-border-radius: 4px 0 0 0;
-  border-radius: 4px 0 0 0;
+  -webkit-border-top-left-radius: 4px;
+          border-top-left-radius: 4px;
+  -moz-border-radius-topleft: 4px;
 }
+
 .table-bordered thead:first-child tr:first-child th:last-child,
 .table-bordered tbody:first-child tr:first-child td:last-child {
-  -webkit-border-radius: 0 4px 0 0;
-  -moz-border-radius: 0 4px 0 0;
-  border-radius: 0 4px 0 0;
+  -webkit-border-top-right-radius: 4px;
+          border-top-right-radius: 4px;
+  -moz-border-radius-topright: 4px;
 }
+
 .table-bordered thead:last-child tr:last-child th:first-child,
 .table-bordered tbody:last-child tr:last-child td:first-child {
   -webkit-border-radius: 0 0 0 4px;
-  -moz-border-radius: 0 0 0 4px;
-  border-radius: 0 0 0 4px;
+     -moz-border-radius: 0 0 0 4px;
+          border-radius: 0 0 0 4px;
+  -webkit-border-bottom-left-radius: 4px;
+          border-bottom-left-radius: 4px;
+  -moz-border-radius-bottomleft: 4px;
 }
+
 .table-bordered thead:last-child tr:last-child th:last-child,
 .table-bordered tbody:last-child tr:last-child td:last-child {
-  -webkit-border-radius: 0 0 4px 0;
-  -moz-border-radius: 0 0 4px 0;
-  border-radius: 0 0 4px 0;
+  -webkit-border-bottom-right-radius: 4px;
+          border-bottom-right-radius: 4px;
+  -moz-border-radius-bottomright: 4px;
 }
+
 .table-striped tbody tr:nth-child(odd) td,
 .table-striped tbody tr:nth-child(odd) th {
   background-color: #f9f9f9;
 }
+
 .table tbody tr:hover td,
 .table tbody tr:hover th {
   background-color: #f5f5f5;
 }
+
 table .span1 {
   float: none;
   width: 44px;
   margin-left: 0;
 }
+
 table .span2 {
   float: none;
   width: 124px;
   margin-left: 0;
 }
+
 table .span3 {
   float: none;
   width: 204px;
   margin-left: 0;
 }
+
 table .span4 {
   float: none;
   width: 284px;
   margin-left: 0;
 }
+
 table .span5 {
   float: none;
   width: 364px;
   margin-left: 0;
 }
+
 table .span6 {
   float: none;
   width: 444px;
   margin-left: 0;
 }
+
 table .span7 {
   float: none;
   width: 524px;
   margin-left: 0;
 }
+
 table .span8 {
   float: none;
   width: 604px;
   margin-left: 0;
 }
+
 table .span9 {
   float: none;
   width: 684px;
   margin-left: 0;
 }
+
 table .span10 {
   float: none;
   width: 764px;
   margin-left: 0;
 }
+
 table .span11 {
   float: none;
   width: 844px;
   margin-left: 0;
 }
+
 table .span12 {
   float: none;
   width: 924px;
   margin-left: 0;
 }
+
 table .span13 {
   float: none;
   width: 1004px;
   margin-left: 0;
 }
+
 table .span14 {
   float: none;
   width: 1084px;
   margin-left: 0;
 }
+
 table .span15 {
   float: none;
   width: 1164px;
   margin-left: 0;
 }
+
 table .span16 {
   float: none;
   width: 1244px;
   margin-left: 0;
 }
+
 table .span17 {
   float: none;
   width: 1324px;
   margin-left: 0;
 }
+
 table .span18 {
   float: none;
   width: 1404px;
   margin-left: 0;
 }
+
 table .span19 {
   float: none;
   width: 1484px;
   margin-left: 0;
 }
+
 table .span20 {
   float: none;
   width: 1564px;
   margin-left: 0;
 }
+
 table .span21 {
   float: none;
   width: 1644px;
   margin-left: 0;
 }
+
 table .span22 {
   float: none;
   width: 1724px;
   margin-left: 0;
 }
+
 table .span23 {
   float: none;
   width: 1804px;
   margin-left: 0;
 }
+
 table .span24 {
   float: none;
   width: 1884px;
   margin-left: 0;
 }
+
 [class^="icon-"],
 [class*=" icon-"] {
   display: inline-block;
   width: 14px;
   height: 14px;
+  *margin-right: .3em;
   line-height: 14px;
   vertical-align: text-top;
-  background-image: url("/static/img/glyphicons-halflings.png");
+  background-image: url("../img/glyphicons-halflings.png");
   background-position: 14px 14px;
   background-repeat: no-repeat;
-  *margin-right: .3em;
 }
+
 [class^="icon-"]:last-child,
 [class*=" icon-"]:last-child {
   *margin-left: 0;
 }
+
 .icon-white {
-  background-image: url("/static/img/glyphicons-halflings-white.png");
+  background-image: url("../img/glyphicons-halflings-white.png");
 }
+
 .icon-glass {
   background-position: 0      0;
 }
+
 .icon-music {
   background-position: -24px 0;
 }
+
 .icon-search {
   background-position: -48px 0;
 }
+
 .icon-envelope {
   background-position: -72px 0;
 }
+
 .icon-heart {
   background-position: -96px 0;
 }
+
 .icon-star {
   background-position: -120px 0;
 }
+
 .icon-star-empty {
   background-position: -144px 0;
 }
+
 .icon-user {
   background-position: -168px 0;
 }
+
 .icon-film {
   background-position: -192px 0;
 }
+
 .icon-th-large {
   background-position: -216px 0;
 }
+
 .icon-th {
   background-position: -240px 0;
 }
+
 .icon-th-list {
   background-position: -264px 0;
 }
+
 .icon-ok {
   background-position: -288px 0;
 }
+
 .icon-remove {
   background-position: -312px 0;
 }
+
 .icon-zoom-in {
   background-position: -336px 0;
 }
+
 .icon-zoom-out {
   background-position: -360px 0;
 }
+
 .icon-off {
   background-position: -384px 0;
 }
+
 .icon-signal {
   background-position: -408px 0;
 }
+
 .icon-cog {
   background-position: -432px 0;
 }
+
 .icon-trash {
   background-position: -456px 0;
 }
+
 .icon-home {
   background-position: 0 -24px;
 }
+
 .icon-file {
   background-position: -24px -24px;
 }
+
 .icon-time {
   background-position: -48px -24px;
 }
+
 .icon-road {
   background-position: -72px -24px;
 }
+
 .icon-download-alt {
   background-position: -96px -24px;
 }
+
 .icon-download {
   background-position: -120px -24px;
 }
+
 .icon-upload {
   background-position: -144px -24px;
 }
+
 .icon-inbox {
   background-position: -168px -24px;
 }
+
 .icon-play-circle {
   background-position: -192px -24px;
 }
+
 .icon-repeat {
   background-position: -216px -24px;
 }
+
 .icon-refresh {
   background-position: -240px -24px;
 }
+
 .icon-list-alt {
   background-position: -264px -24px;
 }
+
 .icon-lock {
   background-position: -287px -24px;
 }
+
 .icon-flag {
   background-position: -312px -24px;
 }
+
 .icon-headphones {
   background-position: -336px -24px;
 }
+
 .icon-volume-off {
   background-position: -360px -24px;
 }
+
 .icon-volume-down {
   background-position: -384px -24px;
 }
+
 .icon-volume-up {
   background-position: -408px -24px;
 }
+
 .icon-qrcode {
   background-position: -432px -24px;
 }
+
 .icon-barcode {
   background-position: -456px -24px;
 }
+
 .icon-tag {
   background-position: 0 -48px;
 }
+
 .icon-tags {
   background-position: -25px -48px;
 }
+
 .icon-book {
   background-position: -48px -48px;
 }
+
 .icon-bookmark {
   background-position: -72px -48px;
 }
+
 .icon-print {
   background-position: -96px -48px;
 }
+
 .icon-camera {
   background-position: -120px -48px;
 }
+
 .icon-font {
   background-position: -144px -48px;
 }
+
 .icon-bold {
   background-position: -167px -48px;
 }
+
 .icon-italic {
   background-position: -192px -48px;
 }
+
 .icon-text-height {
   background-position: -216px -48px;
 }
+
 .icon-text-width {
   background-position: -240px -48px;
 }
+
 .icon-align-left {
   background-position: -264px -48px;
 }
+
 .icon-align-center {
   background-position: -288px -48px;
 }
+
 .icon-align-right {
   background-position: -312px -48px;
 }
+
 .icon-align-justify {
   background-position: -336px -48px;
 }
+
 .icon-list {
   background-position: -360px -48px;
 }
+
 .icon-indent-left {
   background-position: -384px -48px;
 }
+
 .icon-indent-right {
   background-position: -408px -48px;
 }
+
 .icon-facetime-video {
   background-position: -432px -48px;
 }
+
 .icon-picture {
   background-position: -456px -48px;
 }
+
 .icon-pencil {
   background-position: 0 -72px;
 }
+
 .icon-map-marker {
   background-position: -24px -72px;
 }
+
 .icon-adjust {
   background-position: -48px -72px;
 }
+
 .icon-tint {
   background-position: -72px -72px;
 }
+
 .icon-edit {
   background-position: -96px -72px;
 }
+
 .icon-share {
   background-position: -120px -72px;
 }
+
 .icon-check {
   background-position: -144px -72px;
 }
+
 .icon-move {
   background-position: -168px -72px;
 }
+
 .icon-step-backward {
   background-position: -192px -72px;
 }
+
 .icon-fast-backward {
   background-position: -216px -72px;
 }
+
 .icon-backward {
   background-position: -240px -72px;
 }
+
 .icon-play {
   background-position: -264px -72px;
 }
+
 .icon-pause {
   background-position: -288px -72px;
 }
+
 .icon-stop {
   background-position: -312px -72px;
 }
+
 .icon-forward {
   background-position: -336px -72px;
 }
+
 .icon-fast-forward {
   background-position: -360px -72px;
 }
+
 .icon-step-forward {
   background-position: -384px -72px;
 }
+
 .icon-eject {
   background-position: -408px -72px;
 }
+
 .icon-chevron-left {
   background-position: -432px -72px;
 }
+
 .icon-chevron-right {
   background-position: -456px -72px;
 }
+
 .icon-plus-sign {
   background-position: 0 -96px;
 }
+
 .icon-minus-sign {
   background-position: -24px -96px;
 }
+
 .icon-remove-sign {
   background-position: -48px -96px;
 }
+
 .icon-ok-sign {
   background-position: -72px -96px;
 }
+
 .icon-question-sign {
   background-position: -96px -96px;
 }
+
 .icon-info-sign {
   background-position: -120px -96px;
 }
+
 .icon-screenshot {
   background-position: -144px -96px;
 }
+
 .icon-remove-circle {
   background-position: -168px -96px;
 }
+
 .icon-ok-circle {
   background-position: -192px -96px;
 }
+
 .icon-ban-circle {
   background-position: -216px -96px;
 }
+
 .icon-arrow-left {
   background-position: -240px -96px;
 }
+
 .icon-arrow-right {
   background-position: -264px -96px;
 }
+
 .icon-arrow-up {
   background-position: -289px -96px;
 }
+
 .icon-arrow-down {
   background-position: -312px -96px;
 }
+
 .icon-share-alt {
   background-position: -336px -96px;
 }
+
 .icon-resize-full {
   background-position: -360px -96px;
 }
+
 .icon-resize-small {
   background-position: -384px -96px;
 }
+
 .icon-plus {
   background-position: -408px -96px;
 }
+
 .icon-minus {
   background-position: -433px -96px;
 }
+
 .icon-asterisk {
   background-position: -456px -96px;
 }
+
 .icon-exclamation-sign {
   background-position: 0 -120px;
 }
+
 .icon-gift {
   background-position: -24px -120px;
 }
+
 .icon-leaf {
   background-position: -48px -120px;
 }
+
 .icon-fire {
   background-position: -72px -120px;
 }
+
 .icon-eye-open {
   background-position: -96px -120px;
 }
+
 .icon-eye-close {
   background-position: -120px -120px;
 }
+
 .icon-warning-sign {
   background-position: -144px -120px;
 }
+
 .icon-plane {
   background-position: -168px -120px;
 }
+
 .icon-calendar {
   background-position: -192px -120px;
 }
+
 .icon-random {
   background-position: -216px -120px;
 }
+
 .icon-comment {
   background-position: -240px -120px;
 }
+
 .icon-magnet {
   background-position: -264px -120px;
 }
+
 .icon-chevron-up {
   background-position: -288px -120px;
 }
+
 .icon-chevron-down {
   background-position: -313px -119px;
 }
+
 .icon-retweet {
   background-position: -336px -120px;
 }
+
 .icon-shopping-cart {
   background-position: -360px -120px;
 }
+
 .icon-folder-close {
   background-position: -384px -120px;
 }
+
 .icon-folder-open {
   background-position: -408px -120px;
 }
+
 .icon-resize-vertical {
   background-position: -432px -119px;
 }
+
 .icon-resize-horizontal {
   background-position: -456px -118px;
 }
+
+.icon-hdd {
+  background-position: 0 -144px;
+}
+
+.icon-bullhorn {
+  background-position: -24px -144px;
+}
+
+.icon-bell {
+  background-position: -48px -144px;
+}
+
+.icon-certificate {
+  background-position: -72px -144px;
+}
+
+.icon-thumbs-up {
+  background-position: -96px -144px;
+}
+
+.icon-thumbs-down {
+  background-position: -120px -144px;
+}
+
+.icon-hand-right {
+  background-position: -144px -144px;
+}
+
+.icon-hand-left {
+  background-position: -168px -144px;
+}
+
+.icon-hand-up {
+  background-position: -192px -144px;
+}
+
+.icon-hand-down {
+  background-position: -216px -144px;
+}
+
+.icon-circle-arrow-right {
+  background-position: -240px -144px;
+}
+
+.icon-circle-arrow-left {
+  background-position: -264px -144px;
+}
+
+.icon-circle-arrow-up {
+  background-position: -288px -144px;
+}
+
+.icon-circle-arrow-down {
+  background-position: -312px -144px;
+}
+
+.icon-globe {
+  background-position: -336px -144px;
+}
+
+.icon-wrench {
+  background-position: -360px -144px;
+}
+
+.icon-tasks {
+  background-position: -384px -144px;
+}
+
+.icon-filter {
+  background-position: -408px -144px;
+}
+
+.icon-briefcase {
+  background-position: -432px -144px;
+}
+
+.icon-fullscreen {
+  background-position: -456px -144px;
+}
+
+.dropup,
 .dropdown {
   position: relative;
 }
+
 .dropdown-toggle {
   *margin-bottom: -3px;
 }
+
 .dropdown-toggle:active,
 .open .dropdown-toggle {
   outline: 0;
 }
+
 .caret {
   display: inline-block;
   width: 0;
   height: 0;
   vertical-align: top;
-  border-left: 4px solid transparent;
-  border-right: 4px solid transparent;
   border-top: 4px solid #000000;
+  border-right: 4px solid transparent;
+  border-left: 4px solid transparent;
+  content: "";
   opacity: 0.3;
   filter: alpha(opacity=30);
-  content: "";
 }
+
 .dropdown .caret {
   margin-top: 8px;
   margin-left: 2px;
 }
+
 .dropdown:hover .caret,
-.open.dropdown .caret {
+.open .caret {
   opacity: 1;
   filter: alpha(opacity=100);
 }
+
 .dropdown-menu {
   position: absolute;
   top: 100%;
   left: 0;
   z-index: 1000;
-  float: left;
   display: none;
+  float: left;
   min-width: 160px;
   padding: 4px 0;
-  margin: 0;
+  margin: 1px 0 0;
   list-style: none;
   background-color: #ffffff;
-  border-color: #ccc;
-  border-color: rgba(0, 0, 0, 0.2);
-  border-style: solid;
-  border-width: 1px;
-  -webkit-border-radius: 0 0 5px 5px;
-  -moz-border-radius: 0 0 5px 5px;
-  border-radius: 0 0 5px 5px;
-  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-  -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-  box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
-  -webkit-background-clip: padding-box;
-  -moz-background-clip: padding;
-  background-clip: padding-box;
+  border: 1px solid #ccc;
+  border: 1px solid rgba(0, 0, 0, 0.2);
   *border-right-width: 2px;
   *border-bottom-width: 2px;
+  -webkit-border-radius: 5px;
+     -moz-border-radius: 5px;
+          border-radius: 5px;
+  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+     -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+          box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+  -webkit-background-clip: padding-box;
+     -moz-background-clip: padding;
+          background-clip: padding-box;
 }
+
 .dropdown-menu.pull-right {
   right: 0;
   left: auto;
 }
+
 .dropdown-menu .divider {
+  *width: 100%;
   height: 1px;
   margin: 8px 1px;
+  *margin: -5px 0 5px;
   overflow: hidden;
   background-color: #e5e5e5;
   border-bottom: 1px solid #ffffff;
-  *width: 100%;
-  *margin: -5px 0 5px;
 }
+
 .dropdown-menu a {
   display: block;
   padding: 3px 15px;
@@ -1848,6 +2441,7 @@ table .span24 {
   color: #333333;
   white-space: nowrap;
 }
+
 .dropdown-menu li > a:hover,
 .dropdown-menu .active > a,
 .dropdown-menu .active > a:hover {
@@ -1855,39 +2449,41 @@ table .span24 {
   text-decoration: none;
   background-color: #0088cc;
 }
-.dropdown.open {
+
+.open {
   *z-index: 1000;
 }
-.dropdown.open .dropdown-toggle {
-  color: #ffffff;
-  background: #ccc;
-  background: rgba(0, 0, 0, 0.3);
-}
-.dropdown.open .dropdown-menu {
+
+.open > .dropdown-menu {
   display: block;
 }
-.pull-right .dropdown-menu {
-  left: auto;
+
+.pull-right > .dropdown-menu {
   right: 0;
+  left: auto;
 }
+
 .dropup .caret,
 .navbar-fixed-bottom .dropdown .caret {
   border-top: 0;
   border-bottom: 4px solid #000000;
   content: "\2191";
 }
+
 .dropup .dropdown-menu,
 .navbar-fixed-bottom .dropdown .dropdown-menu {
   top: auto;
   bottom: 100%;
   margin-bottom: 1px;
 }
+
 .typeahead {
   margin-top: 2px;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 .well {
   min-height: 20px;
   padding: 19px;
@@ -1896,52 +2492,60 @@ table .span24 {
   border: 1px solid #eee;
   border: 1px solid rgba(0, 0, 0, 0.05);
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
   -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
-  -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
-  box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
 }
+
 .well blockquote {
   border-color: #ddd;
   border-color: rgba(0, 0, 0, 0.15);
 }
+
 .well-large {
   padding: 24px;
   -webkit-border-radius: 6px;
-  -moz-border-radius: 6px;
-  border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
 }
+
 .well-small {
   padding: 9px;
   -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
 }
+
 .fade {
-  -webkit-transition: opacity 0.15s linear;
-  -moz-transition: opacity 0.15s linear;
-  -ms-transition: opacity 0.15s linear;
-  -o-transition: opacity 0.15s linear;
-  transition: opacity 0.15s linear;
   opacity: 0;
+  -webkit-transition: opacity 0.15s linear;
+     -moz-transition: opacity 0.15s linear;
+      -ms-transition: opacity 0.15s linear;
+       -o-transition: opacity 0.15s linear;
+          transition: opacity 0.15s linear;
 }
+
 .fade.in {
   opacity: 1;
 }
+
 .collapse {
-  -webkit-transition: height 0.35s ease;
-  -moz-transition: height 0.35s ease;
-  -ms-transition: height 0.35s ease;
-  -o-transition: height 0.35s ease;
-  transition: height 0.35s ease;
   position: relative;
-  overflow: hidden;
   height: 0;
+  overflow: hidden;
+  -webkit-transition: height 0.35s ease;
+     -moz-transition: height 0.35s ease;
+      -ms-transition: height 0.35s ease;
+       -o-transition: height 0.35s ease;
+          transition: height 0.35s ease;
 }
+
 .collapse.in {
   height: auto;
 }
+
 .close {
   float: right;
   font-size: 20px;
@@ -1952,125 +2556,153 @@ table .span24 {
   opacity: 0.2;
   filter: alpha(opacity=20);
 }
+
 .close:hover {
   color: #000000;
   text-decoration: none;
+  cursor: pointer;
   opacity: 0.4;
   filter: alpha(opacity=40);
-  cursor: pointer;
 }
+
+button.close {
+  padding: 0;
+  cursor: pointer;
+  background: transparent;
+  border: 0;
+  -webkit-appearance: none;
+}
+
 .btn {
   display: inline-block;
   *display: inline;
-  /* IE7 inline-block hack */
-
-  *zoom: 1;
   padding: 4px 10px 4px;
   margin-bottom: 0;
+  *margin-left: .3em;
   font-size: 13px;
   line-height: 18px;
+  *line-height: 20px;
   color: #333333;
   text-align: center;
   text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
   vertical-align: middle;
+  cursor: pointer;
   background-color: #f5f5f5;
-  background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
+  *background-color: #e6e6e6;
   background-image: -ms-linear-gradient(top, #ffffff, #e6e6e6);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));
   background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6);
   background-image: -o-linear-gradient(top, #ffffff, #e6e6e6);
   background-image: linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#e6e6e6', GradientType=0);
-  border-color: #e6e6e6 #e6e6e6 #bfbfbf;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
   border: 1px solid #cccccc;
+  *border: 0;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  border-color: #e6e6e6 #e6e6e6 #bfbfbf;
   border-bottom-color: #b3b3b3;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffff', endColorstr='#e6e6e6', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+  *zoom: 1;
   -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-  -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-  cursor: pointer;
-  *margin-left: .3em;
+     -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
 }
+
 .btn:hover,
 .btn:active,
 .btn.active,
 .btn.disabled,
 .btn[disabled] {
   background-color: #e6e6e6;
+  *background-color: #d9d9d9;
 }
+
 .btn:active,
 .btn.active {
   background-color: #cccccc \9;
 }
+
 .btn:first-child {
   *margin-left: 0;
 }
+
 .btn:hover {
   color: #333333;
   text-decoration: none;
   background-color: #e6e6e6;
+  *background-color: #d9d9d9;
+  /* Buttons in IE7 don't get borders, so darken on hover */
+
   background-position: 0 -15px;
   -webkit-transition: background-position 0.1s linear;
-  -moz-transition: background-position 0.1s linear;
-  -ms-transition: background-position 0.1s linear;
-  -o-transition: background-position 0.1s linear;
-  transition: background-position 0.1s linear;
+     -moz-transition: background-position 0.1s linear;
+      -ms-transition: background-position 0.1s linear;
+       -o-transition: background-position 0.1s linear;
+          transition: background-position 0.1s linear;
 }
+
 .btn:focus {
   outline: thin dotted #333;
   outline: 5px auto -webkit-focus-ring-color;
   outline-offset: -2px;
 }
+
 .btn.active,
 .btn:active {
-  background-image: none;
-  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-  -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-  box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
   background-color: #e6e6e6;
   background-color: #d9d9d9 \9;
+  background-image: none;
   outline: 0;
+  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
 }
+
 .btn.disabled,
 .btn[disabled] {
   cursor: default;
-  background-image: none;
   background-color: #e6e6e6;
+  background-image: none;
   opacity: 0.65;
   filter: alpha(opacity=65);
   -webkit-box-shadow: none;
-  -moz-box-shadow: none;
-  box-shadow: none;
+     -moz-box-shadow: none;
+          box-shadow: none;
 }
+
 .btn-large {
   padding: 9px 14px;
   font-size: 15px;
   line-height: normal;
   -webkit-border-radius: 5px;
-  -moz-border-radius: 5px;
-  border-radius: 5px;
+     -moz-border-radius: 5px;
+          border-radius: 5px;
 }
+
 .btn-large [class^="icon-"] {
   margin-top: 1px;
 }
+
 .btn-small {
   padding: 5px 9px;
   font-size: 11px;
   line-height: 16px;
 }
+
 .btn-small [class^="icon-"] {
   margin-top: -1px;
 }
+
 .btn-mini {
   padding: 2px 6px;
   font-size: 11px;
   line-height: 14px;
 }
+
 .btn-primary,
 .btn-primary:hover,
 .btn-warning,
@@ -2083,9 +2715,10 @@ table .span24 {
 .btn-info:hover,
 .btn-inverse,
 .btn-inverse:hover {
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
   color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
 }
+
 .btn-primary.active,
 .btn-warning.active,
 .btn-danger.active,
@@ -2094,204 +2727,251 @@ table .span24 {
 .btn-inverse.active {
   color: rgba(255, 255, 255, 0.75);
 }
+
+.btn {
+  border-color: #ccc;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+}
+
 .btn-primary {
   background-color: #0074cc;
-  background-image: -moz-linear-gradient(top, #0088cc, #0055cc);
+  *background-color: #0055cc;
   background-image: -ms-linear-gradient(top, #0088cc, #0055cc);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0055cc));
   background-image: -webkit-linear-gradient(top, #0088cc, #0055cc);
   background-image: -o-linear-gradient(top, #0088cc, #0055cc);
+  background-image: -moz-linear-gradient(top, #0088cc, #0055cc);
   background-image: linear-gradient(top, #0088cc, #0055cc);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#0088cc', endColorstr='#0055cc', GradientType=0);
   border-color: #0055cc #0055cc #003580;
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#0088cc', endColorstr='#0055cc', GradientType=0);
   filter: progid:dximagetransform.microsoft.gradient(enabled=false);
 }
+
 .btn-primary:hover,
 .btn-primary:active,
 .btn-primary.active,
 .btn-primary.disabled,
 .btn-primary[disabled] {
   background-color: #0055cc;
+  *background-color: #004ab3;
 }
+
 .btn-primary:active,
 .btn-primary.active {
   background-color: #004099 \9;
 }
+
 .btn-warning {
   background-color: #faa732;
-  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
+  *background-color: #f89406;
   background-image: -ms-linear-gradient(top, #fbb450, #f89406);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
   background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
   background-image: -o-linear-gradient(top, #fbb450, #f89406);
+  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
   background-image: linear-gradient(top, #fbb450, #f89406);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fbb450', endColorstr='#f89406', GradientType=0);
   border-color: #f89406 #f89406 #ad6704;
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450', endColorstr='#f89406', GradientType=0);
   filter: progid:dximagetransform.microsoft.gradient(enabled=false);
 }
+
 .btn-warning:hover,
 .btn-warning:active,
 .btn-warning.active,
 .btn-warning.disabled,
 .btn-warning[disabled] {
   background-color: #f89406;
+  *background-color: #df8505;
 }
+
 .btn-warning:active,
 .btn-warning.active {
   background-color: #c67605 \9;
 }
+
 .btn-danger {
   background-color: #da4f49;
-  background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
+  *background-color: #bd362f;
   background-image: -ms-linear-gradient(top, #ee5f5b, #bd362f);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));
   background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f);
   background-image: -o-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
   background-image: linear-gradient(top, #ee5f5b, #bd362f);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ee5f5b', endColorstr='#bd362f', GradientType=0);
   border-color: #bd362f #bd362f #802420;
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ee5f5b', endColorstr='#bd362f', GradientType=0);
   filter: progid:dximagetransform.microsoft.gradient(enabled=false);
 }
+
 .btn-danger:hover,
 .btn-danger:active,
 .btn-danger.active,
 .btn-danger.disabled,
 .btn-danger[disabled] {
   background-color: #bd362f;
+  *background-color: #a9302a;
 }
+
 .btn-danger:active,
 .btn-danger.active {
   background-color: #942a25 \9;
 }
+
 .btn-success {
   background-color: #5bb75b;
-  background-image: -moz-linear-gradient(top, #62c462, #51a351);
+  *background-color: #51a351;
   background-image: -ms-linear-gradient(top, #62c462, #51a351);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));
   background-image: -webkit-linear-gradient(top, #62c462, #51a351);
   background-image: -o-linear-gradient(top, #62c462, #51a351);
+  background-image: -moz-linear-gradient(top, #62c462, #51a351);
   background-image: linear-gradient(top, #62c462, #51a351);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#62c462', endColorstr='#51a351', GradientType=0);
   border-color: #51a351 #51a351 #387038;
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#62c462', endColorstr='#51a351', GradientType=0);
   filter: progid:dximagetransform.microsoft.gradient(enabled=false);
 }
+
 .btn-success:hover,
 .btn-success:active,
 .btn-success.active,
 .btn-success.disabled,
 .btn-success[disabled] {
   background-color: #51a351;
+  *background-color: #499249;
 }
+
 .btn-success:active,
 .btn-success.active {
   background-color: #408140 \9;
 }
+
 .btn-info {
   background-color: #49afcd;
-  background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
+  *background-color: #2f96b4;
   background-image: -ms-linear-gradient(top, #5bc0de, #2f96b4);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));
   background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4);
   background-image: -o-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
   background-image: linear-gradient(top, #5bc0de, #2f96b4);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#5bc0de', endColorstr='#2f96b4', GradientType=0);
   border-color: #2f96b4 #2f96b4 #1f6377;
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de', endColorstr='#2f96b4', GradientType=0);
   filter: progid:dximagetransform.microsoft.gradient(enabled=false);
 }
+
 .btn-info:hover,
 .btn-info:active,
 .btn-info.active,
 .btn-info.disabled,
 .btn-info[disabled] {
   background-color: #2f96b4;
+  *background-color: #2a85a0;
 }
+
 .btn-info:active,
 .btn-info.active {
   background-color: #24748c \9;
 }
+
 .btn-inverse {
   background-color: #414141;
-  background-image: -moz-linear-gradient(top, #555555, #222222);
+  *background-color: #222222;
   background-image: -ms-linear-gradient(top, #555555, #222222);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#555555), to(#222222));
   background-image: -webkit-linear-gradient(top, #555555, #222222);
   background-image: -o-linear-gradient(top, #555555, #222222);
+  background-image: -moz-linear-gradient(top, #555555, #222222);
   background-image: linear-gradient(top, #555555, #222222);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#555555', endColorstr='#222222', GradientType=0);
   border-color: #222222 #222222 #000000;
   border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#555555', endColorstr='#222222', GradientType=0);
   filter: progid:dximagetransform.microsoft.gradient(enabled=false);
 }
+
 .btn-inverse:hover,
 .btn-inverse:active,
 .btn-inverse.active,
 .btn-inverse.disabled,
 .btn-inverse[disabled] {
   background-color: #222222;
+  *background-color: #151515;
 }
+
 .btn-inverse:active,
 .btn-inverse.active {
   background-color: #080808 \9;
 }
+
 button.btn,
 input[type="submit"].btn {
   *padding-top: 2px;
   *padding-bottom: 2px;
 }
+
 button.btn::-moz-focus-inner,
 input[type="submit"].btn::-moz-focus-inner {
   padding: 0;
   border: 0;
 }
+
 button.btn.btn-large,
 input[type="submit"].btn.btn-large {
   *padding-top: 7px;
   *padding-bottom: 7px;
 }
+
 button.btn.btn-small,
 input[type="submit"].btn.btn-small {
   *padding-top: 3px;
   *padding-bottom: 3px;
 }
+
 button.btn.btn-mini,
 input[type="submit"].btn.btn-mini {
   *padding-top: 1px;
   *padding-bottom: 1px;
 }
+
 .btn-group {
   position: relative;
-  *zoom: 1;
   *margin-left: .3em;
+  *zoom: 1;
 }
+
 .btn-group:before,
 .btn-group:after {
   display: table;
   content: "";
 }
+
 .btn-group:after {
   clear: both;
 }
+
 .btn-group:first-child {
   *margin-left: 0;
 }
+
 .btn-group + .btn-group {
   margin-left: 5px;
 }
+
 .btn-toolbar {
   margin-top: 9px;
   margin-bottom: 9px;
 }
+
 .btn-toolbar .btn-group {
   display: inline-block;
   *display: inline;
@@ -2299,120 +2979,159 @@ input[type="submit"].btn.btn-mini {
 
   *zoom: 1;
 }
-.btn-group .btn {
+
+.btn-group > .btn {
   position: relative;
   float: left;
   margin-left: -1px;
   -webkit-border-radius: 0;
-  -moz-border-radius: 0;
-  border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
 }
-.btn-group .btn:first-child {
+
+.btn-group > .btn:first-child {
   margin-left: 0;
-  -webkit-border-top-left-radius: 4px;
-  -moz-border-radius-topleft: 4px;
-  border-top-left-radius: 4px;
   -webkit-border-bottom-left-radius: 4px;
+          border-bottom-left-radius: 4px;
+  -webkit-border-top-left-radius: 4px;
+          border-top-left-radius: 4px;
   -moz-border-radius-bottomleft: 4px;
-  border-bottom-left-radius: 4px;
+  -moz-border-radius-topleft: 4px;
 }
-.btn-group .btn:last-child,
-.btn-group .dropdown-toggle {
+
+.btn-group > .btn:last-child,
+.btn-group > .dropdown-toggle {
   -webkit-border-top-right-radius: 4px;
-  -moz-border-radius-topright: 4px;
-  border-top-right-radius: 4px;
+          border-top-right-radius: 4px;
   -webkit-border-bottom-right-radius: 4px;
+          border-bottom-right-radius: 4px;
+  -moz-border-radius-topright: 4px;
   -moz-border-radius-bottomright: 4px;
-  border-bottom-right-radius: 4px;
 }
-.btn-group .btn.large:first-child {
+
+.btn-group > .btn.large:first-child {
   margin-left: 0;
-  -webkit-border-top-left-radius: 6px;
-  -moz-border-radius-topleft: 6px;
-  border-top-left-radius: 6px;
   -webkit-border-bottom-left-radius: 6px;
+          border-bottom-left-radius: 6px;
+  -webkit-border-top-left-radius: 6px;
+          border-top-left-radius: 6px;
   -moz-border-radius-bottomleft: 6px;
-  border-bottom-left-radius: 6px;
+  -moz-border-radius-topleft: 6px;
 }
-.btn-group .btn.large:last-child,
-.btn-group .large.dropdown-toggle {
+
+.btn-group > .btn.large:last-child,
+.btn-group > .large.dropdown-toggle {
   -webkit-border-top-right-radius: 6px;
-  -moz-border-radius-topright: 6px;
-  border-top-right-radius: 6px;
+          border-top-right-radius: 6px;
   -webkit-border-bottom-right-radius: 6px;
+          border-bottom-right-radius: 6px;
+  -moz-border-radius-topright: 6px;
   -moz-border-radius-bottomright: 6px;
-  border-bottom-right-radius: 6px;
 }
-.btn-group .btn:hover,
-.btn-group .btn:focus,
-.btn-group .btn:active,
-.btn-group .btn.active {
+
+.btn-group > .btn:hover,
+.btn-group > .btn:focus,
+.btn-group > .btn:active,
+.btn-group > .btn.active {
   z-index: 2;
 }
+
 .btn-group .dropdown-toggle:active,
 .btn-group.open .dropdown-toggle {
   outline: 0;
 }
-.btn-group .dropdown-toggle {
-  padding-left: 8px;
+
+.btn-group > .dropdown-toggle {
+  *padding-top: 4px;
   padding-right: 8px;
+  *padding-bottom: 4px;
+  padding-left: 8px;
   -webkit-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-  -moz-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-  box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
-  *padding-top: 3px;
-  *padding-bottom: 3px;
+     -moz-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
 }
-.btn-group .btn-mini.dropdown-toggle {
-  padding-left: 5px;
+
+.btn-group > .btn-mini.dropdown-toggle {
   padding-right: 5px;
-  *padding-top: 1px;
-  *padding-bottom: 1px;
+  padding-left: 5px;
 }
-.btn-group .btn-small.dropdown-toggle {
+
+.btn-group > .btn-small.dropdown-toggle {
   *padding-top: 4px;
   *padding-bottom: 4px;
 }
-.btn-group .btn-large.dropdown-toggle {
-  padding-left: 12px;
+
+.btn-group > .btn-large.dropdown-toggle {
   padding-right: 12px;
+  padding-left: 12px;
 }
-.btn-group.open {
-  *z-index: 1000;
-}
-.btn-group.open .dropdown-menu {
-  display: block;
-  margin-top: 1px;
-  -webkit-border-radius: 5px;
-  -moz-border-radius: 5px;
-  border-radius: 5px;
-}
+
 .btn-group.open .dropdown-toggle {
   background-image: none;
-  -webkit-box-shadow: inset 0 1px 6px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-  -moz-box-shadow: inset 0 1px 6px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
-  box-shadow: inset 0 1px 6px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
 }
+
+.btn-group.open .btn.dropdown-toggle {
+  background-color: #e6e6e6;
+}
+
+.btn-group.open .btn-primary.dropdown-toggle {
+  background-color: #0055cc;
+}
+
+.btn-group.open .btn-warning.dropdown-toggle {
+  background-color: #f89406;
+}
+
+.btn-group.open .btn-danger.dropdown-toggle {
+  background-color: #bd362f;
+}
+
+.btn-group.open .btn-success.dropdown-toggle {
+  background-color: #51a351;
+}
+
+.btn-group.open .btn-info.dropdown-toggle {
+  background-color: #2f96b4;
+}
+
+.btn-group.open .btn-inverse.dropdown-toggle {
+  background-color: #222222;
+}
+
 .btn .caret {
   margin-top: 7px;
   margin-left: 0;
 }
+
 .btn:hover .caret,
 .open.btn-group .caret {
   opacity: 1;
   filter: alpha(opacity=100);
 }
+
 .btn-mini .caret {
   margin-top: 5px;
 }
+
 .btn-small .caret {
   margin-top: 6px;
 }
+
 .btn-large .caret {
   margin-top: 6px;
-  border-left: 5px solid transparent;
-  border-right: 5px solid transparent;
-  border-top: 5px solid #000000;
+  border-top-width: 5px;
+  border-right-width: 5px;
+  border-left-width: 5px;
 }
+
+.dropup .btn-large .caret {
+  border-top: 0;
+  border-bottom: 5px solid #000000;
+}
+
 .btn-primary .caret,
 .btn-warning .caret,
 .btn-danger .caret,
@@ -2424,65 +3143,82 @@ input[type="submit"].btn.btn-mini {
   opacity: 0.75;
   filter: alpha(opacity=75);
 }
+
 .alert {
   padding: 8px 35px 8px 14px;
   margin-bottom: 18px;
+  color: #c09853;
   text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
   background-color: #fcf8e3;
   border: 1px solid #fbeed5;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
-  color: #c09853;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 .alert-heading {
   color: inherit;
 }
+
 .alert .close {
   position: relative;
   top: -2px;
   right: -21px;
   line-height: 18px;
 }
+
 .alert-success {
+  color: #468847;
   background-color: #dff0d8;
   border-color: #d6e9c6;
-  color: #468847;
 }
+
 .alert-danger,
 .alert-error {
+  color: #b94a48;
   background-color: #f2dede;
   border-color: #eed3d7;
-  color: #b94a48;
 }
+
 .alert-info {
+  color: #3a87ad;
   background-color: #d9edf7;
   border-color: #bce8f1;
-  color: #3a87ad;
 }
+
 .alert-block {
   padding-top: 14px;
   padding-bottom: 14px;
 }
+
 .alert-block > p,
 .alert-block > ul {
   margin-bottom: 0;
 }
+
 .alert-block p + p {
   margin-top: 5px;
 }
+
 .nav {
-  margin-left: 0;
   margin-bottom: 18px;
+  margin-left: 0;
   list-style: none;
 }
+
 .nav > li > a {
   display: block;
 }
+
 .nav > li > a:hover {
   text-decoration: none;
   background-color: #eeeeee;
 }
+
+.nav > .pull-right {
+  float: right;
+}
+
 .nav .nav-header {
   display: block;
   padding: 3px 15px;
@@ -2493,45 +3229,54 @@ input[type="submit"].btn.btn-mini {
   text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
   text-transform: uppercase;
 }
+
 .nav li + .nav-header {
   margin-top: 9px;
 }
+
 .nav-list {
-  padding-left: 15px;
   padding-right: 15px;
+  padding-left: 15px;
   margin-bottom: 0;
 }
+
 .nav-list > li > a,
 .nav-list .nav-header {
-  margin-left: -15px;
   margin-right: -15px;
+  margin-left: -15px;
   text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
 }
+
 .nav-list > li > a {
   padding: 3px 15px;
 }
+
 .nav-list > .active > a,
 .nav-list > .active > a:hover {
   color: #ffffff;
   text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
   background-color: #0088cc;
 }
+
 .nav-list [class^="icon-"] {
   margin-right: 2px;
 }
+
 .nav-list .divider {
+  *width: 100%;
   height: 1px;
   margin: 8px 1px;
+  *margin: -5px 0 5px;
   overflow: hidden;
   background-color: #e5e5e5;
   border-bottom: 1px solid #ffffff;
-  *width: 100%;
-  *margin: -5px 0 5px;
 }
+
 .nav-tabs,
 .nav-pills {
   *zoom: 1;
 }
+
 .nav-tabs:before,
 .nav-pills:before,
 .nav-tabs:after,
@@ -2539,14 +3284,17 @@ input[type="submit"].btn.btn-mini {
   display: table;
   content: "";
 }
+
 .nav-tabs:after,
 .nav-pills:after {
   clear: both;
 }
+
 .nav-tabs > li,
 .nav-pills > li {
   float: left;
 }
+
 .nav-tabs > li > a,
 .nav-pills > li > a {
   padding-right: 12px;
@@ -2554,234 +3302,283 @@ input[type="submit"].btn.btn-mini {
   margin-right: 2px;
   line-height: 14px;
 }
+
 .nav-tabs {
   border-bottom: 1px solid #ddd;
 }
+
 .nav-tabs > li {
   margin-bottom: -1px;
 }
+
 .nav-tabs > li > a {
   padding-top: 8px;
   padding-bottom: 8px;
   line-height: 18px;
   border: 1px solid transparent;
   -webkit-border-radius: 4px 4px 0 0;
-  -moz-border-radius: 4px 4px 0 0;
-  border-radius: 4px 4px 0 0;
+     -moz-border-radius: 4px 4px 0 0;
+          border-radius: 4px 4px 0 0;
 }
+
 .nav-tabs > li > a:hover {
   border-color: #eeeeee #eeeeee #dddddd;
 }
+
 .nav-tabs > .active > a,
 .nav-tabs > .active > a:hover {
   color: #555555;
+  cursor: default;
   background-color: #ffffff;
   border: 1px solid #ddd;
   border-bottom-color: transparent;
-  cursor: default;
 }
+
 .nav-pills > li > a {
   padding-top: 8px;
   padding-bottom: 8px;
   margin-top: 2px;
   margin-bottom: 2px;
   -webkit-border-radius: 5px;
-  -moz-border-radius: 5px;
-  border-radius: 5px;
+     -moz-border-radius: 5px;
+          border-radius: 5px;
 }
+
 .nav-pills > .active > a,
 .nav-pills > .active > a:hover {
   color: #ffffff;
   background-color: #0088cc;
 }
+
 .nav-stacked > li {
   float: none;
 }
+
 .nav-stacked > li > a {
   margin-right: 0;
 }
+
 .nav-tabs.nav-stacked {
   border-bottom: 0;
 }
+
 .nav-tabs.nav-stacked > li > a {
   border: 1px solid #ddd;
   -webkit-border-radius: 0;
-  -moz-border-radius: 0;
-  border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
 }
+
 .nav-tabs.nav-stacked > li:first-child > a {
   -webkit-border-radius: 4px 4px 0 0;
-  -moz-border-radius: 4px 4px 0 0;
-  border-radius: 4px 4px 0 0;
+     -moz-border-radius: 4px 4px 0 0;
+          border-radius: 4px 4px 0 0;
 }
+
 .nav-tabs.nav-stacked > li:last-child > a {
   -webkit-border-radius: 0 0 4px 4px;
-  -moz-border-radius: 0 0 4px 4px;
-  border-radius: 0 0 4px 4px;
+     -moz-border-radius: 0 0 4px 4px;
+          border-radius: 0 0 4px 4px;
 }
+
 .nav-tabs.nav-stacked > li > a:hover {
-  border-color: #ddd;
   z-index: 2;
+  border-color: #ddd;
 }
+
 .nav-pills.nav-stacked > li > a {
   margin-bottom: 3px;
 }
+
 .nav-pills.nav-stacked > li:last-child > a {
   margin-bottom: 1px;
 }
-.nav-tabs .dropdown-menu,
-.nav-pills .dropdown-menu {
-  margin-top: 1px;
-  border-width: 1px;
+
+.nav-tabs .dropdown-menu {
+  -webkit-border-radius: 0 0 5px 5px;
+     -moz-border-radius: 0 0 5px 5px;
+          border-radius: 0 0 5px 5px;
 }
+
 .nav-pills .dropdown-menu {
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 .nav-tabs .dropdown-toggle .caret,
 .nav-pills .dropdown-toggle .caret {
+  margin-top: 6px;
   border-top-color: #0088cc;
   border-bottom-color: #0088cc;
-  margin-top: 6px;
 }
+
 .nav-tabs .dropdown-toggle:hover .caret,
 .nav-pills .dropdown-toggle:hover .caret {
   border-top-color: #005580;
   border-bottom-color: #005580;
 }
+
 .nav-tabs .active .dropdown-toggle .caret,
 .nav-pills .active .dropdown-toggle .caret {
   border-top-color: #333333;
   border-bottom-color: #333333;
 }
+
 .nav > .dropdown.active > a:hover {
   color: #000000;
   cursor: pointer;
 }
+
 .nav-tabs .open .dropdown-toggle,
 .nav-pills .open .dropdown-toggle,
-.nav > .open.active > a:hover {
+.nav > li.dropdown.open.active > a:hover {
   color: #ffffff;
   background-color: #999999;
   border-color: #999999;
 }
-.nav .open .caret,
-.nav .open.active .caret,
-.nav .open a:hover .caret {
+
+.nav li.dropdown.open .caret,
+.nav li.dropdown.open.active .caret,
+.nav li.dropdown.open a:hover .caret {
   border-top-color: #ffffff;
   border-bottom-color: #ffffff;
   opacity: 1;
   filter: alpha(opacity=100);
 }
+
 .tabs-stacked .open > a:hover {
   border-color: #999999;
 }
+
 .tabbable {
   *zoom: 1;
 }
+
 .tabbable:before,
 .tabbable:after {
   display: table;
   content: "";
 }
+
 .tabbable:after {
   clear: both;
 }
+
 .tab-content {
-  display: table;
-  width: 100%;
+  overflow: auto;
 }
-.tabs-below .nav-tabs,
-.tabs-right .nav-tabs,
-.tabs-left .nav-tabs {
+
+.tabs-below > .nav-tabs,
+.tabs-right > .nav-tabs,
+.tabs-left > .nav-tabs {
   border-bottom: 0;
 }
+
 .tab-content > .tab-pane,
 .pill-content > .pill-pane {
   display: none;
 }
+
 .tab-content > .active,
 .pill-content > .active {
   display: block;
 }
-.tabs-below .nav-tabs {
+
+.tabs-below > .nav-tabs {
   border-top: 1px solid #ddd;
 }
-.tabs-below .nav-tabs > li {
+
+.tabs-below > .nav-tabs > li {
   margin-top: -1px;
   margin-bottom: 0;
 }
-.tabs-below .nav-tabs > li > a {
+
+.tabs-below > .nav-tabs > li > a {
   -webkit-border-radius: 0 0 4px 4px;
-  -moz-border-radius: 0 0 4px 4px;
-  border-radius: 0 0 4px 4px;
+     -moz-border-radius: 0 0 4px 4px;
+          border-radius: 0 0 4px 4px;
 }
-.tabs-below .nav-tabs > li > a:hover {
-  border-bottom-color: transparent;
+
+.tabs-below > .nav-tabs > li > a:hover {
   border-top-color: #ddd;
+  border-bottom-color: transparent;
 }
-.tabs-below .nav-tabs .active > a,
-.tabs-below .nav-tabs .active > a:hover {
+
+.tabs-below > .nav-tabs > .active > a,
+.tabs-below > .nav-tabs > .active > a:hover {
   border-color: transparent #ddd #ddd #ddd;
 }
-.tabs-left .nav-tabs > li,
-.tabs-right .nav-tabs > li {
+
+.tabs-left > .nav-tabs > li,
+.tabs-right > .nav-tabs > li {
   float: none;
 }
-.tabs-left .nav-tabs > li > a,
-.tabs-right .nav-tabs > li > a {
+
+.tabs-left > .nav-tabs > li > a,
+.tabs-right > .nav-tabs > li > a {
   min-width: 74px;
   margin-right: 0;
   margin-bottom: 3px;
 }
-.tabs-left .nav-tabs {
+
+.tabs-left > .nav-tabs {
   float: left;
   margin-right: 19px;
   border-right: 1px solid #ddd;
 }
-.tabs-left .nav-tabs > li > a {
+
+.tabs-left > .nav-tabs > li > a {
   margin-right: -1px;
   -webkit-border-radius: 4px 0 0 4px;
-  -moz-border-radius: 4px 0 0 4px;
-  border-radius: 4px 0 0 4px;
+     -moz-border-radius: 4px 0 0 4px;
+          border-radius: 4px 0 0 4px;
 }
-.tabs-left .nav-tabs > li > a:hover {
+
+.tabs-left > .nav-tabs > li > a:hover {
   border-color: #eeeeee #dddddd #eeeeee #eeeeee;
 }
-.tabs-left .nav-tabs .active > a,
-.tabs-left .nav-tabs .active > a:hover {
+
+.tabs-left > .nav-tabs .active > a,
+.tabs-left > .nav-tabs .active > a:hover {
   border-color: #ddd transparent #ddd #ddd;
   *border-right-color: #ffffff;
 }
-.tabs-right .nav-tabs {
+
+.tabs-right > .nav-tabs {
   float: right;
   margin-left: 19px;
   border-left: 1px solid #ddd;
 }
-.tabs-right .nav-tabs > li > a {
+
+.tabs-right > .nav-tabs > li > a {
   margin-left: -1px;
   -webkit-border-radius: 0 4px 4px 0;
-  -moz-border-radius: 0 4px 4px 0;
-  border-radius: 0 4px 4px 0;
+     -moz-border-radius: 0 4px 4px 0;
+          border-radius: 0 4px 4px 0;
 }
-.tabs-right .nav-tabs > li > a:hover {
+
+.tabs-right > .nav-tabs > li > a:hover {
   border-color: #eeeeee #eeeeee #eeeeee #dddddd;
 }
-.tabs-right .nav-tabs .active > a,
-.tabs-right .nav-tabs .active > a:hover {
+
+.tabs-right > .nav-tabs .active > a,
+.tabs-right > .nav-tabs .active > a:hover {
   border-color: #ddd #ddd #ddd transparent;
   *border-left-color: #ffffff;
 }
+
 .navbar {
   *position: relative;
   *z-index: 2;
-  overflow: visible;
   margin-bottom: 18px;
+  overflow: visible;
 }
+
 .navbar-inner {
-  padding-left: 20px;
+  min-height: 40px;
   padding-right: 20px;
+  padding-left: 20px;
   background-color: #2c2c2c;
   background-image: -moz-linear-gradient(top, #333333, #222222);
   background-image: -ms-linear-gradient(top, #333333, #222222);
@@ -2790,138 +3587,116 @@ input[type="submit"].btn.btn-mini {
   background-image: -o-linear-gradient(top, #333333, #222222);
   background-image: linear-gradient(top, #333333, #222222);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#333333', endColorstr='#222222', GradientType=0);
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#333333', endColorstr='#222222', GradientType=0);
   -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
-  -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
-  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
+     -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
+          box-shadow: 0 1px 3px rgba(0, 0, 0, 0.25), inset 0 -1px 0 rgba(0, 0, 0, 0.1);
 }
+
 .navbar .container {
   width: auto;
 }
-.btn-navbar {
-  display: none;
-  float: right;
-  padding: 7px 10px;
-  margin-left: 5px;
-  margin-right: 5px;
-  background-color: #2c2c2c;
-  background-image: -moz-linear-gradient(top, #333333, #222222);
-  background-image: -ms-linear-gradient(top, #333333, #222222);
-  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#333333), to(#222222));
-  background-image: -webkit-linear-gradient(top, #333333, #222222);
-  background-image: -o-linear-gradient(top, #333333, #222222);
-  background-image: linear-gradient(top, #333333, #222222);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#333333', endColorstr='#222222', GradientType=0);
-  border-color: #222222 #222222 #000000;
-  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
-  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
-  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
-  -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
-  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
-}
-.btn-navbar:hover,
-.btn-navbar:active,
-.btn-navbar.active,
-.btn-navbar.disabled,
-.btn-navbar[disabled] {
-  background-color: #222222;
-}
-.btn-navbar:active,
-.btn-navbar.active {
-  background-color: #080808 \9;
-}
-.btn-navbar .icon-bar {
-  display: block;
-  width: 18px;
-  height: 2px;
-  background-color: #f5f5f5;
-  -webkit-border-radius: 1px;
-  -moz-border-radius: 1px;
-  border-radius: 1px;
-  -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-  -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-  box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
-}
-.btn-navbar .icon-bar + .icon-bar {
-  margin-top: 3px;
-}
+
 .nav-collapse.collapse {
   height: auto;
 }
+
 .navbar {
   color: #999999;
 }
+
 .navbar .brand:hover {
   text-decoration: none;
 }
+
 .navbar .brand {
-  float: left;
   display: block;
+  float: left;
   padding: 8px 20px 12px;
   margin-left: -20px;
   font-size: 20px;
   font-weight: 200;
   line-height: 1;
-  color: #ffffff;
+  color: #999999;
 }
+
 .navbar .navbar-text {
   margin-bottom: 0;
   line-height: 40px;
 }
+
+.navbar .navbar-link {
+  color: #999999;
+}
+
+.navbar .navbar-link:hover {
+  color: #ffffff;
+}
+
 .navbar .btn,
 .navbar .btn-group {
   margin-top: 5px;
 }
+
 .navbar .btn-group .btn {
-  margin-top: 0;
+  margin: 0;
 }
+
 .navbar-form {
   margin-bottom: 0;
   *zoom: 1;
 }
+
 .navbar-form:before,
 .navbar-form:after {
   display: table;
   content: "";
 }
+
 .navbar-form:after {
   clear: both;
 }
+
 .navbar-form input,
 .navbar-form select,
 .navbar-form .radio,
 .navbar-form .checkbox {
   margin-top: 5px;
 }
+
 .navbar-form input,
 .navbar-form select {
   display: inline-block;
   margin-bottom: 0;
 }
+
 .navbar-form input[type="image"],
 .navbar-form input[type="checkbox"],
 .navbar-form input[type="radio"] {
   margin-top: 3px;
 }
+
 .navbar-form .input-append,
 .navbar-form .input-prepend {
   margin-top: 6px;
   white-space: nowrap;
 }
+
 .navbar-form .input-append input,
 .navbar-form .input-prepend input {
   margin-top: 0;
 }
+
 .navbar-search {
   position: relative;
   float: left;
   margin-top: 6px;
   margin-bottom: 0;
 }
+
 .navbar-search .search-query {
   padding: 4px 9px;
   font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
@@ -2931,21 +3706,28 @@ input[type="submit"].btn.btn-mini {
   color: #ffffff;
   background-color: #626262;
   border: 1px solid #151515;
-  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0px rgba(255, 255, 255, 0.15);
-  -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0px rgba(255, 255, 255, 0.15);
-  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0px rgba(255, 255, 255, 0.15);
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
+     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
+          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
   -webkit-transition: none;
-  -moz-transition: none;
-  -ms-transition: none;
-  -o-transition: none;
-  transition: none;
+     -moz-transition: none;
+      -ms-transition: none;
+       -o-transition: none;
+          transition: none;
 }
+
 .navbar-search .search-query:-moz-placeholder {
   color: #cccccc;
 }
+
+.navbar-search .search-query:-ms-input-placeholder {
+  color: #cccccc;
+}
+
 .navbar-search .search-query::-webkit-input-placeholder {
   color: #cccccc;
 }
+
 .navbar-search .search-query:focus,
 .navbar-search .search-query.focused {
   padding: 5px 10px;
@@ -2953,11 +3735,12 @@ input[type="submit"].btn.btn-mini {
   text-shadow: 0 1px 0 #ffffff;
   background-color: #ffffff;
   border: 0;
-  -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
-  -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
-  box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
   outline: 0;
+  -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+     -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+          box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
 }
+
 .navbar-fixed-top,
 .navbar-fixed-bottom {
   position: fixed;
@@ -2966,24 +3749,29 @@ input[type="submit"].btn.btn-mini {
   z-index: 1030;
   margin-bottom: 0;
 }
+
 .navbar-fixed-top .navbar-inner,
 .navbar-fixed-bottom .navbar-inner {
-  padding-left: 0;
   padding-right: 0;
+  padding-left: 0;
   -webkit-border-radius: 0;
-  -moz-border-radius: 0;
-  border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
 }
+
 .navbar-fixed-top .container,
 .navbar-fixed-bottom .container {
   width: 940px;
 }
+
 .navbar-fixed-top {
   top: 0;
 }
+
 .navbar-fixed-bottom {
   bottom: 0;
 }
+
 .navbar .nav {
   position: relative;
   left: 0;
@@ -2991,116 +3779,196 @@ input[type="submit"].btn.btn-mini {
   float: left;
   margin: 0 10px 0 0;
 }
+
 .navbar .nav.pull-right {
   float: right;
 }
+
 .navbar .nav > li {
   display: block;
   float: left;
 }
+
 .navbar .nav > li > a {
   float: none;
-  padding: 10px 10px 11px;
+  padding: 9px 10px 11px;
   line-height: 19px;
   color: #999999;
   text-decoration: none;
   text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
 }
+
+.navbar .btn {
+  display: inline-block;
+  padding: 4px 10px 4px;
+  margin: 5px 5px 6px;
+  line-height: 18px;
+}
+
+.navbar .btn-group {
+  padding: 5px 5px 6px;
+  margin: 0;
+}
+
 .navbar .nav > li > a:hover {
-  background-color: transparent;
   color: #ffffff;
   text-decoration: none;
+  background-color: transparent;
 }
+
 .navbar .nav .active > a,
 .navbar .nav .active > a:hover {
   color: #ffffff;
   text-decoration: none;
   background-color: #222222;
 }
+
 .navbar .divider-vertical {
-  height: 40px;
   width: 1px;
+  height: 40px;
   margin: 0 9px;
   overflow: hidden;
   background-color: #222222;
   border-right: 1px solid #333333;
 }
+
 .navbar .nav.pull-right {
-  margin-left: 10px;
   margin-right: 0;
+  margin-left: 10px;
 }
-.navbar .dropdown-menu {
-  margin-top: 1px;
-  -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+
+.navbar .btn-navbar {
+  display: none;
+  float: right;
+  padding: 7px 10px;
+  margin-right: 5px;
+  margin-left: 5px;
+  background-color: #2c2c2c;
+  *background-color: #222222;
+  background-image: -ms-linear-gradient(top, #333333, #222222);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#333333), to(#222222));
+  background-image: -webkit-linear-gradient(top, #333333, #222222);
+  background-image: -o-linear-gradient(top, #333333, #222222);
+  background-image: linear-gradient(top, #333333, #222222);
+  background-image: -moz-linear-gradient(top, #333333, #222222);
+  background-repeat: repeat-x;
+  border-color: #222222 #222222 #000000;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#333333', endColorstr='#222222', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+     -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
 }
+
+.navbar .btn-navbar:hover,
+.navbar .btn-navbar:active,
+.navbar .btn-navbar.active,
+.navbar .btn-navbar.disabled,
+.navbar .btn-navbar[disabled] {
+  background-color: #222222;
+  *background-color: #151515;
+}
+
+.navbar .btn-navbar:active,
+.navbar .btn-navbar.active {
+  background-color: #080808 \9;
+}
+
+.navbar .btn-navbar .icon-bar {
+  display: block;
+  width: 18px;
+  height: 2px;
+  background-color: #f5f5f5;
+  -webkit-border-radius: 1px;
+     -moz-border-radius: 1px;
+          border-radius: 1px;
+  -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+     -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+          box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+}
+
+.btn-navbar .icon-bar + .icon-bar {
+  margin-top: 3px;
+}
+
 .navbar .dropdown-menu:before {
-  content: '';
-  display: inline-block;
-  border-left: 7px solid transparent;
-  border-right: 7px solid transparent;
-  border-bottom: 7px solid #ccc;
-  border-bottom-color: rgba(0, 0, 0, 0.2);
   position: absolute;
   top: -7px;
   left: 9px;
-}
-.navbar .dropdown-menu:after {
-  content: '';
   display: inline-block;
-  border-left: 6px solid transparent;
-  border-right: 6px solid transparent;
-  border-bottom: 6px solid #ffffff;
+  border-right: 7px solid transparent;
+  border-bottom: 7px solid #ccc;
+  border-left: 7px solid transparent;
+  border-bottom-color: rgba(0, 0, 0, 0.2);
+  content: '';
+}
+
+.navbar .dropdown-menu:after {
   position: absolute;
   top: -6px;
   left: 10px;
+  display: inline-block;
+  border-right: 6px solid transparent;
+  border-bottom: 6px solid #ffffff;
+  border-left: 6px solid transparent;
+  content: '';
 }
+
 .navbar-fixed-bottom .dropdown-menu:before {
-  border-top: 7px solid #ccc;
-  border-top-color: rgba(0, 0, 0, 0.2);
-  border-bottom: 0;
-  bottom: -7px;
   top: auto;
+  bottom: -7px;
+  border-top: 7px solid #ccc;
+  border-bottom: 0;
+  border-top-color: rgba(0, 0, 0, 0.2);
 }
+
 .navbar-fixed-bottom .dropdown-menu:after {
+  top: auto;
+  bottom: -6px;
   border-top: 6px solid #ffffff;
   border-bottom: 0;
-  bottom: -6px;
-  top: auto;
 }
-.navbar .nav .dropdown-toggle .caret,
-.navbar .nav .open.dropdown .caret {
+
+.navbar .nav li.dropdown .dropdown-toggle .caret,
+.navbar .nav li.dropdown.open .caret {
   border-top-color: #ffffff;
   border-bottom-color: #ffffff;
 }
-.navbar .nav .active .caret {
+
+.navbar .nav li.dropdown.active .caret {
   opacity: 1;
   filter: alpha(opacity=100);
 }
-.navbar .nav .open > .dropdown-toggle,
-.navbar .nav .active > .dropdown-toggle,
-.navbar .nav .open.active > .dropdown-toggle {
+
+.navbar .nav li.dropdown.open > .dropdown-toggle,
+.navbar .nav li.dropdown.active > .dropdown-toggle,
+.navbar .nav li.dropdown.open.active > .dropdown-toggle {
   background-color: transparent;
 }
-.navbar .nav .active > .dropdown-toggle:hover {
+
+.navbar .nav li.dropdown.active > .dropdown-toggle:hover {
   color: #ffffff;
 }
-.navbar .nav.pull-right .dropdown-menu,
-.navbar .nav .dropdown-menu.pull-right {
-  left: auto;
+
+.navbar .pull-right .dropdown-menu,
+.navbar .dropdown-menu.pull-right {
   right: 0;
-}
-.navbar .nav.pull-right .dropdown-menu:before,
-.navbar .nav .dropdown-menu.pull-right:before {
   left: auto;
+}
+
+.navbar .pull-right .dropdown-menu:before,
+.navbar .dropdown-menu.pull-right:before {
   right: 12px;
-}
-.navbar .nav.pull-right .dropdown-menu:after,
-.navbar .nav .dropdown-menu.pull-right:after {
   left: auto;
-  right: 13px;
 }
+
+.navbar .pull-right .dropdown-menu:after,
+.navbar .dropdown-menu.pull-right:after {
+  right: 13px;
+  left: auto;
+}
+
 .breadcrumb {
   padding: 7px 14px;
   margin: 0 0 18px;
@@ -3113,52 +3981,55 @@ input[type="submit"].btn.btn-mini {
   background-image: -o-linear-gradient(top, #ffffff, #f5f5f5);
   background-image: linear-gradient(top, #ffffff, #f5f5f5);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#f5f5f5', GradientType=0);
   border: 1px solid #ddd;
   -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffff', endColorstr='#f5f5f5', GradientType=0);
   -webkit-box-shadow: inset 0 1px 0 #ffffff;
-  -moz-box-shadow: inset 0 1px 0 #ffffff;
-  box-shadow: inset 0 1px 0 #ffffff;
+     -moz-box-shadow: inset 0 1px 0 #ffffff;
+          box-shadow: inset 0 1px 0 #ffffff;
 }
+
 .breadcrumb li {
   display: inline-block;
   *display: inline;
-  /* IE7 inline-block hack */
-
-  *zoom: 1;
   text-shadow: 0 1px 0 #ffffff;
+  *zoom: 1;
 }
+
 .breadcrumb .divider {
   padding: 0 5px;
   color: #999999;
 }
+
 .breadcrumb .active a {
   color: #333333;
 }
+
 .pagination {
   height: 36px;
   margin: 18px 0;
 }
+
 .pagination ul {
   display: inline-block;
   *display: inline;
-  /* IE7 inline-block hack */
-
-  *zoom: 1;
-  margin-left: 0;
   margin-bottom: 0;
+  margin-left: 0;
   -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+  *zoom: 1;
   -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-  -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-  box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
 }
+
 .pagination li {
   display: inline;
 }
+
 .pagination a {
   float: left;
   padding: 0 14px;
@@ -3167,93 +4038,114 @@ input[type="submit"].btn.btn-mini {
   border: 1px solid #ddd;
   border-left-width: 0;
 }
+
 .pagination a:hover,
 .pagination .active a {
   background-color: #f5f5f5;
 }
+
 .pagination .active a {
   color: #999999;
   cursor: default;
 }
+
 .pagination .disabled span,
 .pagination .disabled a,
 .pagination .disabled a:hover {
   color: #999999;
-  background-color: transparent;
   cursor: default;
+  background-color: transparent;
 }
+
 .pagination li:first-child a {
   border-left-width: 1px;
   -webkit-border-radius: 3px 0 0 3px;
-  -moz-border-radius: 3px 0 0 3px;
-  border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
 }
+
 .pagination li:last-child a {
   -webkit-border-radius: 0 3px 3px 0;
-  -moz-border-radius: 0 3px 3px 0;
-  border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
 }
+
 .pagination-centered {
   text-align: center;
 }
+
 .pagination-right {
   text-align: right;
 }
+
 .pager {
-  margin-left: 0;
   margin-bottom: 18px;
-  list-style: none;
+  margin-left: 0;
   text-align: center;
+  list-style: none;
   *zoom: 1;
 }
+
 .pager:before,
 .pager:after {
   display: table;
   content: "";
 }
+
 .pager:after {
   clear: both;
 }
+
 .pager li {
   display: inline;
 }
+
 .pager a {
   display: inline-block;
   padding: 5px 14px;
   background-color: #fff;
   border: 1px solid #ddd;
   -webkit-border-radius: 15px;
-  -moz-border-radius: 15px;
-  border-radius: 15px;
+     -moz-border-radius: 15px;
+          border-radius: 15px;
 }
+
 .pager a:hover {
   text-decoration: none;
   background-color: #f5f5f5;
 }
+
 .pager .next a {
   float: right;
 }
+
 .pager .previous a {
   float: left;
 }
+
 .pager .disabled a,
 .pager .disabled a:hover {
   color: #999999;
-  background-color: #fff;
   cursor: default;
+  background-color: #fff;
 }
+
 .modal-open .dropdown-menu {
   z-index: 2050;
 }
+
 .modal-open .dropdown.open {
   *z-index: 2050;
 }
+
 .modal-open .popover {
   z-index: 2060;
 }
+
 .modal-open .tooltip {
   z-index: 2070;
 }
+
 .modal-backdrop {
   position: fixed;
   top: 0;
@@ -3263,64 +4155,72 @@ input[type="submit"].btn.btn-mini {
   z-index: 1040;
   background-color: #000000;
 }
+
 .modal-backdrop.fade {
   opacity: 0;
 }
+
 .modal-backdrop,
 .modal-backdrop.fade.in {
   opacity: 0.8;
   filter: alpha(opacity=80);
 }
+
 .modal {
   position: fixed;
   top: 50%;
   left: 50%;
   z-index: 1050;
-  overflow: auto;
   width: 560px;
   margin: -250px 0 0 -280px;
+  overflow: auto;
   background-color: #ffffff;
   border: 1px solid #999;
   border: 1px solid rgba(0, 0, 0, 0.3);
   *border: 1px solid #999;
-  /* IE6-7 */
-
   -webkit-border-radius: 6px;
-  -moz-border-radius: 6px;
-  border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
   -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-  -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-  box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+     -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+          box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
   -webkit-background-clip: padding-box;
-  -moz-background-clip: padding-box;
-  background-clip: padding-box;
+     -moz-background-clip: padding-box;
+          background-clip: padding-box;
 }
+
 .modal.fade {
-  -webkit-transition: opacity .3s linear, top .3s ease-out;
-  -moz-transition: opacity .3s linear, top .3s ease-out;
-  -ms-transition: opacity .3s linear, top .3s ease-out;
-  -o-transition: opacity .3s linear, top .3s ease-out;
-  transition: opacity .3s linear, top .3s ease-out;
   top: -25%;
+  -webkit-transition: opacity 0.3s linear, top 0.3s ease-out;
+     -moz-transition: opacity 0.3s linear, top 0.3s ease-out;
+      -ms-transition: opacity 0.3s linear, top 0.3s ease-out;
+       -o-transition: opacity 0.3s linear, top 0.3s ease-out;
+          transition: opacity 0.3s linear, top 0.3s ease-out;
 }
+
 .modal.fade.in {
   top: 50%;
 }
+
 .modal-header {
   padding: 9px 15px;
   border-bottom: 1px solid #eee;
 }
+
 .modal-header .close {
   margin-top: 2px;
 }
+
 .modal-body {
-  overflow-y: auto;
   max-height: 400px;
   padding: 15px;
+  overflow-y: auto;
 }
+
 .modal-form {
   margin-bottom: 0;
 }
+
 .modal-footer {
   padding: 14px 15px 15px;
   margin-bottom: 0;
@@ -3328,62 +4228,74 @@ input[type="submit"].btn.btn-mini {
   background-color: #f5f5f5;
   border-top: 1px solid #ddd;
   -webkit-border-radius: 0 0 6px 6px;
-  -moz-border-radius: 0 0 6px 6px;
-  border-radius: 0 0 6px 6px;
-  -webkit-box-shadow: inset 0 1px 0 #ffffff;
-  -moz-box-shadow: inset 0 1px 0 #ffffff;
-  box-shadow: inset 0 1px 0 #ffffff;
+     -moz-border-radius: 0 0 6px 6px;
+          border-radius: 0 0 6px 6px;
   *zoom: 1;
+  -webkit-box-shadow: inset 0 1px 0 #ffffff;
+     -moz-box-shadow: inset 0 1px 0 #ffffff;
+          box-shadow: inset 0 1px 0 #ffffff;
 }
+
 .modal-footer:before,
 .modal-footer:after {
   display: table;
   content: "";
 }
+
 .modal-footer:after {
   clear: both;
 }
+
 .modal-footer .btn + .btn {
-  margin-left: 5px;
   margin-bottom: 0;
+  margin-left: 5px;
 }
+
 .modal-footer .btn-group .btn + .btn {
   margin-left: -1px;
 }
+
 .tooltip {
   position: absolute;
   z-index: 1020;
   display: block;
-  visibility: visible;
   padding: 5px;
   font-size: 11px;
   opacity: 0;
   filter: alpha(opacity=0);
+  visibility: visible;
 }
+
 .tooltip.in {
   opacity: 0.8;
   filter: alpha(opacity=80);
 }
+
 .tooltip.top {
   margin-top: -2px;
 }
+
 .tooltip.right {
   margin-left: 2px;
 }
+
 .tooltip.bottom {
   margin-top: 2px;
 }
+
 .tooltip.left {
   margin-left: -2px;
 }
+
 .tooltip.top .tooltip-arrow {
   bottom: 0;
   left: 50%;
   margin-left: -5px;
-  border-left: 5px solid transparent;
-  border-right: 5px solid transparent;
   border-top: 5px solid #000000;
+  border-right: 5px solid transparent;
+  border-left: 5px solid transparent;
 }
+
 .tooltip.left .tooltip-arrow {
   top: 50%;
   right: 0;
@@ -3392,22 +4304,25 @@ input[type="submit"].btn.btn-mini {
   border-bottom: 5px solid transparent;
   border-left: 5px solid #000000;
 }
+
 .tooltip.bottom .tooltip-arrow {
   top: 0;
   left: 50%;
   margin-left: -5px;
-  border-left: 5px solid transparent;
   border-right: 5px solid transparent;
   border-bottom: 5px solid #000000;
+  border-left: 5px solid transparent;
 }
+
 .tooltip.right .tooltip-arrow {
   top: 50%;
   left: 0;
   margin-top: -5px;
   border-top: 5px solid transparent;
-  border-bottom: 5px solid transparent;
   border-right: 5px solid #000000;
+  border-bottom: 5px solid transparent;
 }
+
 .tooltip-inner {
   max-width: 200px;
   padding: 3px 8px;
@@ -3416,14 +4331,16 @@ input[type="submit"].btn.btn-mini {
   text-decoration: none;
   background-color: #000000;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 .tooltip-arrow {
   position: absolute;
   width: 0;
   height: 0;
 }
+
 .popover {
   position: absolute;
   top: 0;
@@ -3432,42 +4349,50 @@ input[type="submit"].btn.btn-mini {
   display: none;
   padding: 5px;
 }
+
 .popover.top {
   margin-top: -5px;
 }
+
 .popover.right {
   margin-left: 5px;
 }
+
 .popover.bottom {
   margin-top: 5px;
 }
+
 .popover.left {
   margin-left: -5px;
 }
+
 .popover.top .arrow {
   bottom: 0;
   left: 50%;
   margin-left: -5px;
-  border-left: 5px solid transparent;
-  border-right: 5px solid transparent;
   border-top: 5px solid #000000;
+  border-right: 5px solid transparent;
+  border-left: 5px solid transparent;
 }
+
 .popover.right .arrow {
   top: 50%;
   left: 0;
   margin-top: -5px;
   border-top: 5px solid transparent;
-  border-bottom: 5px solid transparent;
   border-right: 5px solid #000000;
+  border-bottom: 5px solid transparent;
 }
+
 .popover.bottom .arrow {
   top: 0;
   left: 50%;
   margin-left: -5px;
-  border-left: 5px solid transparent;
   border-right: 5px solid transparent;
   border-bottom: 5px solid #000000;
+  border-left: 5px solid transparent;
 }
+
 .popover.left .arrow {
   top: 50%;
   right: 0;
@@ -3476,211 +4401,222 @@ input[type="submit"].btn.btn-mini {
   border-bottom: 5px solid transparent;
   border-left: 5px solid #000000;
 }
+
 .popover .arrow {
   position: absolute;
   width: 0;
   height: 0;
 }
+
 .popover-inner {
-  padding: 3px;
   width: 280px;
+  padding: 3px;
   overflow: hidden;
   background: #000000;
   background: rgba(0, 0, 0, 0.8);
   -webkit-border-radius: 6px;
-  -moz-border-radius: 6px;
-  border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
   -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-  -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
-  box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+     -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+          box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
 }
+
 .popover-title {
   padding: 9px 15px;
   line-height: 1;
   background-color: #f5f5f5;
   border-bottom: 1px solid #eee;
   -webkit-border-radius: 3px 3px 0 0;
-  -moz-border-radius: 3px 3px 0 0;
-  border-radius: 3px 3px 0 0;
+     -moz-border-radius: 3px 3px 0 0;
+          border-radius: 3px 3px 0 0;
 }
+
 .popover-content {
   padding: 14px;
   background-color: #ffffff;
   -webkit-border-radius: 0 0 3px 3px;
-  -moz-border-radius: 0 0 3px 3px;
-  border-radius: 0 0 3px 3px;
+     -moz-border-radius: 0 0 3px 3px;
+          border-radius: 0 0 3px 3px;
   -webkit-background-clip: padding-box;
-  -moz-background-clip: padding-box;
-  background-clip: padding-box;
+     -moz-background-clip: padding-box;
+          background-clip: padding-box;
 }
+
 .popover-content p,
 .popover-content ul,
 .popover-content ol {
   margin-bottom: 0;
 }
+
 .thumbnails {
   margin-left: -20px;
   list-style: none;
   *zoom: 1;
 }
+
 .thumbnails:before,
 .thumbnails:after {
   display: table;
   content: "";
 }
+
 .thumbnails:after {
   clear: both;
 }
+
+.row-fluid .thumbnails {
+  margin-left: 0;
+}
+
 .thumbnails > li {
   float: left;
-  margin: 0 0 18px 20px;
+  margin-bottom: 18px;
+  margin-left: 20px;
 }
+
 .thumbnail {
   display: block;
   padding: 4px;
   line-height: 1;
   border: 1px solid #ddd;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
   -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
-  -moz-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
-  box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: 0 1px 1px rgba(0, 0, 0, 0.075);
 }
+
 a.thumbnail:hover {
   border-color: #0088cc;
   -webkit-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
-  -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
-  box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+     -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+          box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
 }
+
 .thumbnail > img {
   display: block;
   max-width: 100%;
-  margin-left: auto;
   margin-right: auto;
+  margin-left: auto;
 }
+
 .thumbnail .caption {
   padding: 9px;
 }
-.label {
-  padding: 1px 4px 2px;
+
+.label,
+.badge {
   font-size: 10.998px;
   font-weight: bold;
-  line-height: 13px;
+  line-height: 14px;
   color: #ffffff;
-  vertical-align: middle;
-  white-space: nowrap;
   text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  white-space: nowrap;
+  vertical-align: baseline;
   background-color: #999999;
+}
+
+.label {
+  padding: 1px 4px 2px;
   -webkit-border-radius: 3px;
-  -moz-border-radius: 3px;
-  border-radius: 3px;
-}
-.label:hover {
-  color: #ffffff;
-  text-decoration: none;
-}
-.label-important {
-  background-color: #b94a48;
-}
-.label-important:hover {
-  background-color: #953b39;
-}
-.label-warning {
-  background-color: #f89406;
-}
-.label-warning:hover {
-  background-color: #c67605;
-}
-.label-success {
-  background-color: #468847;
-}
-.label-success:hover {
-  background-color: #356635;
-}
-.label-info {
-  background-color: #3a87ad;
-}
-.label-info:hover {
-  background-color: #2d6987;
-}
-.label-inverse {
-  background-color: #333333;
-}
-.label-inverse:hover {
-  background-color: #1a1a1a;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
 }
+
 .badge {
   padding: 1px 9px 2px;
-  font-size: 12.025px;
-  font-weight: bold;
-  white-space: nowrap;
-  color: #ffffff;
-  background-color: #999999;
   -webkit-border-radius: 9px;
-  -moz-border-radius: 9px;
-  border-radius: 9px;
+     -moz-border-radius: 9px;
+          border-radius: 9px;
 }
-.badge:hover {
+
+a.label:hover,
+a.badge:hover {
   color: #ffffff;
   text-decoration: none;
   cursor: pointer;
 }
-.badge-error {
+
+.label-important,
+.badge-important {
   background-color: #b94a48;
 }
-.badge-error:hover {
+
+.label-important[href],
+.badge-important[href] {
   background-color: #953b39;
 }
+
+.label-warning,
 .badge-warning {
   background-color: #f89406;
 }
-.badge-warning:hover {
+
+.label-warning[href],
+.badge-warning[href] {
   background-color: #c67605;
 }
+
+.label-success,
 .badge-success {
   background-color: #468847;
 }
-.badge-success:hover {
+
+.label-success[href],
+.badge-success[href] {
   background-color: #356635;
 }
+
+.label-info,
 .badge-info {
   background-color: #3a87ad;
 }
-.badge-info:hover {
+
+.label-info[href],
+.badge-info[href] {
   background-color: #2d6987;
 }
+
+.label-inverse,
 .badge-inverse {
   background-color: #333333;
 }
-.badge-inverse:hover {
+
+.label-inverse[href],
+.badge-inverse[href] {
   background-color: #1a1a1a;
 }
+
 @-webkit-keyframes progress-bar-stripes {
   from {
-    background-position: 0 0;
-  }
-  to {
     background-position: 40px 0;
   }
+  to {
+    background-position: 0 0;
+  }
 }
+
 @-moz-keyframes progress-bar-stripes {
   from {
-    background-position: 0 0;
-  }
-  to {
     background-position: 40px 0;
   }
+  to {
+    background-position: 0 0;
+  }
 }
+
 @-ms-keyframes progress-bar-stripes {
   from {
-    background-position: 0 0;
-  }
-  to {
     background-position: 40px 0;
   }
+  to {
+    background-position: 0 0;
+  }
 }
-@keyframes progress-bar-stripes {
+
+@-o-keyframes progress-bar-stripes {
   from {
     background-position: 0 0;
   }
@@ -3688,10 +4624,20 @@ a.thumbnail:hover {
     background-position: 40px 0;
   }
 }
+
+@keyframes progress-bar-stripes {
+  from {
+    background-position: 40px 0;
+  }
+  to {
+    background-position: 0 0;
+  }
+}
+
 .progress {
-  overflow: hidden;
   height: 18px;
   margin-bottom: 18px;
+  overflow: hidden;
   background-color: #f7f7f7;
   background-image: -moz-linear-gradient(top, #f5f5f5, #f9f9f9);
   background-image: -ms-linear-gradient(top, #f5f5f5, #f9f9f9);
@@ -3700,61 +4646,67 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(top, #f5f5f5, #f9f9f9);
   background-image: linear-gradient(top, #f5f5f5, #f9f9f9);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#f5f5f5', endColorstr='#f9f9f9', GradientType=0);
-  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
-  -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
-  box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#f5f5f5', endColorstr='#f9f9f9', GradientType=0);
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
 }
+
 .progress .bar {
-  width: 0%;
+  width: 0;
   height: 18px;
-  color: #ffffff;
   font-size: 12px;
+  color: #ffffff;
   text-align: center;
   text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
   background-color: #0e90d2;
   background-image: -moz-linear-gradient(top, #149bdf, #0480be);
-  background-image: -ms-linear-gradient(top, #149bdf, #0480be);
   background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be));
   background-image: -webkit-linear-gradient(top, #149bdf, #0480be);
   background-image: -o-linear-gradient(top, #149bdf, #0480be);
   background-image: linear-gradient(top, #149bdf, #0480be);
+  background-image: -ms-linear-gradient(top, #149bdf, #0480be);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#149bdf', endColorstr='#0480be', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#149bdf', endColorstr='#0480be', GradientType=0);
   -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-  -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
-  box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+     -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+          box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
   -webkit-box-sizing: border-box;
-  -moz-box-sizing: border-box;
-  -ms-box-sizing: border-box;
-  box-sizing: border-box;
+     -moz-box-sizing: border-box;
+      -ms-box-sizing: border-box;
+          box-sizing: border-box;
   -webkit-transition: width 0.6s ease;
-  -moz-transition: width 0.6s ease;
-  -ms-transition: width 0.6s ease;
-  -o-transition: width 0.6s ease;
-  transition: width 0.6s ease;
+     -moz-transition: width 0.6s ease;
+      -ms-transition: width 0.6s ease;
+       -o-transition: width 0.6s ease;
+          transition: width 0.6s ease;
 }
+
 .progress-striped .bar {
   background-color: #149bdf;
-  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: -webkit-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: -moz-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: -ms-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
-  background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
   background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   -webkit-background-size: 40px 40px;
-  -moz-background-size: 40px 40px;
-  -o-background-size: 40px 40px;
-  background-size: 40px 40px;
+     -moz-background-size: 40px 40px;
+       -o-background-size: 40px 40px;
+          background-size: 40px 40px;
 }
+
 .progress.active .bar {
   -webkit-animation: progress-bar-stripes 2s linear infinite;
-  -moz-animation: progress-bar-stripes 2s linear infinite;
-  animation: progress-bar-stripes 2s linear infinite;
+     -moz-animation: progress-bar-stripes 2s linear infinite;
+      -ms-animation: progress-bar-stripes 2s linear infinite;
+       -o-animation: progress-bar-stripes 2s linear infinite;
+          animation: progress-bar-stripes 2s linear infinite;
 }
+
 .progress-danger .bar {
   background-color: #dd514c;
   background-image: -moz-linear-gradient(top, #ee5f5b, #c43c35);
@@ -3764,8 +4716,9 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(top, #ee5f5b, #c43c35);
   background-image: linear-gradient(top, #ee5f5b, #c43c35);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ee5f5b', endColorstr='#c43c35', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ee5f5b', endColorstr='#c43c35', GradientType=0);
 }
+
 .progress-danger.progress-striped .bar {
   background-color: #ee5f5b;
   background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
@@ -3775,6 +4728,7 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
 }
+
 .progress-success .bar {
   background-color: #5eb95e;
   background-image: -moz-linear-gradient(top, #62c462, #57a957);
@@ -3784,8 +4738,9 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(top, #62c462, #57a957);
   background-image: linear-gradient(top, #62c462, #57a957);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#62c462', endColorstr='#57a957', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#62c462', endColorstr='#57a957', GradientType=0);
 }
+
 .progress-success.progress-striped .bar {
   background-color: #62c462;
   background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
@@ -3795,6 +4750,7 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
 }
+
 .progress-info .bar {
   background-color: #4bb1cf;
   background-image: -moz-linear-gradient(top, #5bc0de, #339bb9);
@@ -3804,8 +4760,9 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(top, #5bc0de, #339bb9);
   background-image: linear-gradient(top, #5bc0de, #339bb9);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#5bc0de', endColorstr='#339bb9', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#5bc0de', endColorstr='#339bb9', GradientType=0);
 }
+
 .progress-info.progress-striped .bar {
   background-color: #5bc0de;
   background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
@@ -3815,6 +4772,7 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
 }
+
 .progress-warning .bar {
   background-color: #faa732;
   background-image: -moz-linear-gradient(top, #fbb450, #f89406);
@@ -3824,8 +4782,9 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(top, #fbb450, #f89406);
   background-image: linear-gradient(top, #fbb450, #f89406);
   background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fbb450', endColorstr='#f89406', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fbb450', endColorstr='#f89406', GradientType=0);
 }
+
 .progress-warning.progress-striped .bar {
   background-color: #fbb450;
   background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
@@ -3835,80 +4794,102 @@ a.thumbnail:hover {
   background-image: -o-linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
   background-image: linear-gradient(-45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
 }
+
 .accordion {
   margin-bottom: 18px;
 }
+
 .accordion-group {
   margin-bottom: 2px;
   border: 1px solid #e5e5e5;
   -webkit-border-radius: 4px;
-  -moz-border-radius: 4px;
-  border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
 }
+
 .accordion-heading {
   border-bottom: 0;
 }
+
 .accordion-heading .accordion-toggle {
   display: block;
   padding: 8px 15px;
 }
+
+.accordion-toggle {
+  cursor: pointer;
+}
+
 .accordion-inner {
   padding: 9px 15px;
   border-top: 1px solid #e5e5e5;
 }
+
 .carousel {
   position: relative;
   margin-bottom: 18px;
   line-height: 1;
 }
+
 .carousel-inner {
-  overflow: hidden;
+  position: relative;
   width: 100%;
-  position: relative;
+  overflow: hidden;
 }
+
 .carousel .item {
-  display: none;
   position: relative;
+  display: none;
   -webkit-transition: 0.6s ease-in-out left;
-  -moz-transition: 0.6s ease-in-out left;
-  -ms-transition: 0.6s ease-in-out left;
-  -o-transition: 0.6s ease-in-out left;
-  transition: 0.6s ease-in-out left;
+     -moz-transition: 0.6s ease-in-out left;
+      -ms-transition: 0.6s ease-in-out left;
+       -o-transition: 0.6s ease-in-out left;
+          transition: 0.6s ease-in-out left;
 }
+
 .carousel .item > img {
   display: block;
   line-height: 1;
 }
+
 .carousel .active,
 .carousel .next,
 .carousel .prev {
   display: block;
 }
+
 .carousel .active {
   left: 0;
 }
+
 .carousel .next,
 .carousel .prev {
   position: absolute;
   top: 0;
   width: 100%;
 }
+
 .carousel .next {
   left: 100%;
 }
+
 .carousel .prev {
   left: -100%;
 }
+
 .carousel .next.left,
 .carousel .prev.right {
   left: 0;
 }
+
 .carousel .active.left {
   left: -100%;
 }
+
 .carousel .active.right {
   left: 100%;
 }
+
 .carousel-control {
   position: absolute;
   top: 40%;
@@ -3924,67 +4905,79 @@ a.thumbnail:hover {
   background: #222222;
   border: 3px solid #ffffff;
   -webkit-border-radius: 23px;
-  -moz-border-radius: 23px;
-  border-radius: 23px;
+     -moz-border-radius: 23px;
+          border-radius: 23px;
   opacity: 0.5;
   filter: alpha(opacity=50);
 }
+
 .carousel-control.right {
-  left: auto;
   right: 15px;
+  left: auto;
 }
+
 .carousel-control:hover {
   color: #ffffff;
   text-decoration: none;
   opacity: 0.9;
   filter: alpha(opacity=90);
 }
+
 .carousel-caption {
   position: absolute;
-  left: 0;
   right: 0;
   bottom: 0;
+  left: 0;
   padding: 10px 15px 5px;
   background: #333333;
   background: rgba(0, 0, 0, 0.75);
 }
+
 .carousel-caption h4,
 .carousel-caption p {
   color: #ffffff;
 }
+
 .hero-unit {
   padding: 60px;
   margin-bottom: 30px;
   background-color: #eeeeee;
   -webkit-border-radius: 6px;
-  -moz-border-radius: 6px;
-  border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
 }
+
 .hero-unit h1 {
   margin-bottom: 0;
   font-size: 60px;
   line-height: 1;
-  color: inherit;
   letter-spacing: -1px;
+  color: inherit;
 }
+
 .hero-unit p {
   font-size: 18px;
   font-weight: 200;
   line-height: 27px;
   color: inherit;
 }
+
 .pull-right {
   float: right;
 }
+
 .pull-left {
   float: left;
 }
+
 .hide {
   display: none;
 }
+
 .show {
   display: block;
 }
+
 .invisible {
   visibility: hidden;
 }

From e8081ad81262761f6d06dd9ae15cb61f92baeb01 Mon Sep 17 00:00:00 2001
From: "http://jasonwoof.com/" <JasonWoof@web>
Date: Fri, 27 Jul 2012 18:52:19 +0000
Subject: [PATCH 4309/8313] Added a comment: auth token length

---
 .../comment_1_994bec0978324e268666073e8ff4f6ae._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment

diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment b/doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment
new file mode 100644
index 0000000000..20d3a64e65
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_1_994bec0978324e268666073e8ff4f6ae._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://jasonwoof.com/"
+ nickname="JasonWoof"
+ subject="auth token length"
+ date="2012-07-27T18:52:19Z"
+ content="""
+Your auth token looks a little short. Aren't you worried about people brute-forcing it? ;)
+"""]]

From 1b8a934e8be28ac8632f9345f867d68203d84cd1 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Fri, 27 Jul 2012 18:55:51 +0000
Subject: [PATCH 4310/8313] Added a comment

---
 ...comment_2_dfa164c86290899139491acccddd8b2b._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment

diff --git a/doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment b/doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment
new file mode 100644
index 0000000000..836fe89e9a
--- /dev/null
+++ b/doc/design/assistant/blog/day_45__long_polling/comment_2_dfa164c86290899139491acccddd8b2b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.2.20"
+ subject="comment 2"
+ date="2012-07-27T18:55:51Z"
+ content="""
+Heh, I consider it overflowing most address fields a bonus, as you don't have to worry when making screenshots. :)
+
+Of course, it changes each app run too..
+"""]]

From 02ec8ea01254637facb30f77b7cb74be3b735c0d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 15:33:24 -0400
Subject: [PATCH 4311/8313] much better webapp startup of the assistant

This avoids forking another process, avoids polling, fixes a race,
and avoids a rare forkProcess thread hang that I saw once time
when starting the webapp.
---
 Assistant.hs                |  9 ++++----
 Assistant/Threads/WebApp.hs |  8 ++++---
 Command/Watch.hs            |  2 +-
 Command/WebApp.hs           | 43 ++++++++++++-------------------------
 Utility/WebApp.hs           |  5 +++--
 5 files changed, 27 insertions(+), 40 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 6b155a4a67..ca428988fe 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -124,8 +124,8 @@ import Utility.ThreadScheduler
 
 import Control.Concurrent
 
-startDaemon :: Bool -> Bool -> Annex ()
-startDaemon assistant foreground
+startDaemon :: Bool -> Bool -> Maybe (IO ()) -> Annex ()
+startDaemon assistant foreground webappwaiter
 	| foreground = do
 		showStart (if assistant then "assistant" else "watch") "."
 		go id
@@ -157,12 +157,11 @@ startDaemon assistant foreground
 				, mountWatcherThread st dstatus scanremotes
 				, transferScannerThread st scanremotes transferqueue
 #ifdef WITH_WEBAPP
-				, webAppThread st dstatus transferqueue
+				, webAppThread st dstatus transferqueue webappwaiter
 #endif
 				, watchThread st dstatus transferqueue changechan
 				]
-			debug "assistant"
-				["all git-annex assistant threads started"]
+			debug "Assistant" ["all threads started"]
 			waitForTermination
 
 stopDaemon :: Annex ()
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 171c7fd9c4..f0acaeb22b 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -145,15 +145,17 @@ getConfigR = defaultLayout $ do
 	setTitle "configuration"
 	[whamlet|<a href="@{HomeR}">main|]
 
-webAppThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> IO ()
-webAppThread st dstatus transferqueue = do
+webAppThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Maybe (IO ()) -> IO ()
+webAppThread st dstatus transferqueue onstartup = do
 	webapp <- mkWebApp
 	app <- toWaiAppPlain webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
 		)
-	runWebApp app' $ \port -> runThreadState st $ writeHtmlShim webapp port
+	runWebApp app' $ \port -> do
+		runThreadState st $ writeHtmlShim webapp port
+		maybe noop id onstartup
 	where
 		mkWebApp = do
 			dir <- absPath =<< runThreadState st (fromRepo repoPath)
diff --git a/Command/Watch.hs b/Command/Watch.hs
index 744844c4dc..61c859106f 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -34,5 +34,5 @@ start :: Bool -> Bool -> Bool -> CommandStart
 start assistant foreground stopdaemon = notBareRepo $ do
 	if stopdaemon
 		then stopDaemon
-		else startDaemon assistant foreground -- does not return
+		else startDaemon assistant foreground Nothing -- does not return
 	stop
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 7d0a310d40..1635ac0442 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -12,12 +12,8 @@ import Command
 import Assistant
 import Utility.WebApp
 import Utility.Daemon (checkDaemon)
-import qualified Annex
 import Option
 
-import Control.Concurrent
-import System.Posix.Process
-
 def :: [Command]
 def = [withOptions [restartOption] $
         command "webapp" paramNothing seek "launch webapp"]
@@ -34,31 +30,20 @@ start restart = notBareRepo $ do
 	if restart
 		then do
 			stopDaemon
-			nuke =<< fromRepo gitAnnexPidFile
+			void $ liftIO . nukeFile =<< fromRepo gitAnnexPidFile
 			startassistant f
-		else unlessM (checkpid f) $
-			startassistant f
-	let url = "file://" ++ f
-	ifM (liftIO $ runBrowser url)
-		( stop
-		, error $ "failed to start web browser on url " ++ url
-		)
+		else ifM (checkpid <&&> checkshim f) $
+			( liftIO $ go f 
+			, startassistant f
+			)
+	stop
 	where
-		nuke f = void $ liftIO $ catchMaybeIO $ removeFile f
-		checkpid f = do
+		checkpid = do
 			pidfile <- fromRepo gitAnnexPidFile
-			liftIO $
-				doesFileExist f <&&> (isJust <$> checkDaemon pidfile)
-		startassistant f = do
-			nuke f
-			{- Fork a separate process to run the assistant,
-			 - with a copy of the Annex state. -}
-			state <- Annex.getState id
-			liftIO $ void $ forkProcess $
-				Annex.eval state $ startDaemon True False
-			waitdaemon f (1000 :: Int)
-		waitdaemon _ 0 = error "failed to start git-annex assistant"
-		waitdaemon f n = unlessM (checkpid f) $ do
-			-- wait 0.1 seconds before retry
-			liftIO $ threadDelay 100000
-			waitdaemon f (n - 1)
+			liftIO $ isJust <$> checkDaemon pidfile
+		checkshim f = liftIO $ doesFileExist f
+		startassistant = startDaemon True False . Just . go
+		go f = unlessM (runBrowser url) $
+			error $ "failed to start web browser on url " ++ url
+			where
+				url = "file://" ++ f
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 69864dc6d3..75e8dde9ec 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -32,6 +32,7 @@ import Blaze.ByteString.Builder.Char.Utf8 (fromText)
 import Blaze.ByteString.Builder (Builder)
 import Data.Monoid
 import Control.Arrow ((***))
+import Control.Concurrent
 
 localhost :: String
 localhost = "localhost"
@@ -52,12 +53,12 @@ runBrowser url = boolSystem cmd [Param url]
  -
  - An IO action can also be run, to do something with the port number,
  - such as start a web browser to view the webapp.
-  -}
+ -}
 runWebApp :: Application -> (PortNumber -> IO ()) -> IO ()
 runWebApp app observer = do
 	sock <- localSocket
+	void $ forkIO $ runSettingsSocket defaultSettings sock app
 	observer =<< socketPort sock
-	runSettingsSocket defaultSettings sock app
 
 {- Binds to a local socket, selecting any free port.
  -

From adae40a292cf3192659f0edef486756431cf97da Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 15:40:52 -0400
Subject: [PATCH 4312/8313] now the webapp has the same options as the
 assistant

---
 Command/WebApp.hs | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 1635ac0442..ee1274f97d 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -12,37 +12,33 @@ import Command
 import Assistant
 import Utility.WebApp
 import Utility.Daemon (checkDaemon)
-import Option
+import qualified Command.Watch
 
 def :: [Command]
-def = [withOptions [restartOption] $
+def = [withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $
         command "webapp" paramNothing seek "launch webapp"]
 
-restartOption :: Option
-restartOption = Option.flag [] "restart" "restart the assistant daemon"
-
 seek :: [CommandSeek]
-seek = [withFlag restartOption $ \restart -> withNothing $ start restart]
+seek = [withFlag Command.Watch.stopOption $ \stopdaemon ->
+	withFlag Command.Watch.foregroundOption $ \foreground ->
+	withNothing $ start foreground stopdaemon]
 
-start :: Bool -> CommandStart
-start restart = notBareRepo $ do
-	f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
-	if restart
-		then do
-			stopDaemon
-			void $ liftIO . nukeFile =<< fromRepo gitAnnexPidFile
-			startassistant f
-		else ifM (checkpid <&&> checkshim f) $
-			( liftIO $ go f 
-			, startassistant f
-			)
+start :: Bool -> Bool -> CommandStart
+start foreground stopdaemon = notBareRepo $ do
+	if stopdaemon
+		then stopDaemon
+		else do
+			f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
+			ifM (checkpid <&&> checkshim f) $
+				( liftIO $ go f 
+				, startDaemon True foreground $ Just $ go f
+				)
 	stop
 	where
 		checkpid = do
 			pidfile <- fromRepo gitAnnexPidFile
 			liftIO $ isJust <$> checkDaemon pidfile
 		checkshim f = liftIO $ doesFileExist f
-		startassistant = startDaemon True False . Just . go
 		go f = unlessM (runBrowser url) $
 			error $ "failed to start web browser on url " ++ url
 			where

From 7717501fee57b614e0e86d4d46356ba26f5f5247 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 16:28:00 -0400
Subject: [PATCH 4313/8313] add alert close buttons

---
 Assistant/Threads/WebApp.hs  |  1 +
 static/js/bootstrap-alert.js | 94 ++++++++++++++++++++++++++++++++++++
 templates/bootstrap.hamlet   |  3 ++
 3 files changed, 98 insertions(+)
 create mode 100644 static/js/bootstrap-alert.js

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index f0acaeb22b..71ce317583 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -63,6 +63,7 @@ instance Yesod WebApp where
 			addStylesheet $ StaticR css_bootstrap_responsive_css
 			addScript $ StaticR jquery_full_js
 			addScript $ StaticR js_bootstrap_dropdown_js
+			addScript $ StaticR js_bootstrap_alert_js
 			$(widgetFile "default-layout")
 		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
 
diff --git a/static/js/bootstrap-alert.js b/static/js/bootstrap-alert.js
new file mode 100644
index 0000000000..d17f44e150
--- /dev/null
+++ b/static/js/bootstrap-alert.js
@@ -0,0 +1,94 @@
+/* ==========================================================
+ * bootstrap-alert.js v2.0.2
+ * http://twitter.github.com/bootstrap/javascript.html#alerts
+ * ==========================================================
+ * Copyright 2012 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ========================================================== */
+
+
+!function( $ ){
+
+  "use strict"
+
+ /* ALERT CLASS DEFINITION
+  * ====================== */
+
+  var dismiss = '[data-dismiss="alert"]'
+    , Alert = function ( el ) {
+        $(el).on('click', dismiss, this.close)
+      }
+
+  Alert.prototype = {
+
+    constructor: Alert
+
+  , close: function ( e ) {
+      var $this = $(this)
+        , selector = $this.attr('data-target')
+        , $parent
+
+      if (!selector) {
+        selector = $this.attr('href')
+        selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7
+      }
+
+      $parent = $(selector)
+      $parent.trigger('close')
+
+      e && e.preventDefault()
+
+      $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent())
+
+      $parent
+        .trigger('close')
+        .removeClass('in')
+
+      function removeElement() {
+        $parent
+          .trigger('closed')
+          .remove()
+      }
+
+      $.support.transition && $parent.hasClass('fade') ?
+        $parent.on($.support.transition.end, removeElement) :
+        removeElement()
+    }
+
+  }
+
+
+ /* ALERT PLUGIN DEFINITION
+  * ======================= */
+
+  $.fn.alert = function ( option ) {
+    return this.each(function () {
+      var $this = $(this)
+        , data = $this.data('alert')
+      if (!data) $this.data('alert', (data = new Alert(this)))
+      if (typeof option == 'string') data[option].call($this)
+    })
+  }
+
+  $.fn.alert.Constructor = Alert
+
+
+ /* ALERT DATA-API
+  * ============== */
+
+  $(function () {
+    $('body').on('click.alert.data-api', dismiss, Alert.prototype.close)
+  })
+
+}( window.jQuery );
\ No newline at end of file
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
index c03c459a64..6697283b4f 100644
--- a/templates/bootstrap.hamlet
+++ b/templates/bootstrap.hamlet
@@ -40,12 +40,15 @@ $doctype 5
         <div class="span3">
           <div class="sidebar-nav">
             <div class="alert alert-info">
+              <a class="close" data-dismiss="alert" href="#">×</a>
               <b>This is just a demo.</b> If this were not just a demo,
               I'd not be filling this sidebar with silly alerts.
             <div class="alert alert-success">
+              <a class="close" data-dismiss="alert" href="#">×</a>
               <b>Well done!</b>
               You successfully read this important alert message.
             <div class="alert alert-error">
+              <a class="close" data-dismiss="alert" href="#">×</a>
               <b>Whoops!</b>
               Unable to connect to blah blah..
         <div class="span9">

From ecd63e29818c009bfc96d7a011bb8a2f1304e0ae Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 20:25:28 -0400
Subject: [PATCH 4314/8313] template cleanup

use julius's nice #id and .class things
---
 Assistant/Threads/WebApp.hs |  1 +
 static/css/webapp.css       |  7 +++++
 templates/bootstrap.hamlet  | 54 ++++++++++++++++---------------------
 templates/status.hamlet     | 14 +++++-----
 4 files changed, 38 insertions(+), 38 deletions(-)
 create mode 100644 static/css/webapp.css

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 71ce317583..5407ed1954 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -61,6 +61,7 @@ instance Yesod WebApp where
 		page <- widgetToPageContent $ do
 			addStylesheet $ StaticR css_bootstrap_css
 			addStylesheet $ StaticR css_bootstrap_responsive_css
+			addStylesheet $ StaticR css_webapp_css
 			addScript $ StaticR jquery_full_js
 			addScript $ StaticR js_bootstrap_dropdown_js
 			addScript $ StaticR js_bootstrap_alert_js
diff --git a/static/css/webapp.css b/static/css/webapp.css
new file mode 100644
index 0000000000..ba309effa6
--- /dev/null
+++ b/static/css/webapp.css
@@ -0,0 +1,7 @@
+body {
+	padding-top: 60px;
+	padding-bottom: 40px;
+}
+.sidebar-nav {
+	padding: 9px 0;
+}
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
index 6697283b4f..360b3d686d 100644
--- a/templates/bootstrap.hamlet
+++ b/templates/bootstrap.hamlet
@@ -4,52 +4,44 @@ $doctype 5
     <title>#{baseTitle webapp} #{pageTitle page}
     <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
     <meta name="viewport" content="width=device-width,initial-scale=1.0">
-    <style type="text/css">
-      body {
-        padding-top: 60px;
-        padding-bottom: 40px;
-      }
-      .sidebar-nav {
-        padding: 9px 0;
-      }
     ^{pageHead page}
   <body>
 
-    <div class="navbar navbar-fixed-top">
-      <div class="navbar-inner">
-        <div class="container">
-          <a class="brand" href="#">
+    <div .navbar .navbar-fixed-top>
+      <div .navbar-inner>
+        <div .container>
+          <a .brand href="#">
             git-annex
-          <ul class="nav">
-            <li class="active">
+          <ul .nav>
+            <li .active>
               <a href="#">Dashboard</a>
             <li>
               <a href="@{ConfigR}">Config</a>
-          <ul class="nav pull-right">
-            <li class="dropdown" id="menu1">
-              <a class="dropdown-toggle" data-toggle="dropdown" href="#menu1">
+          <ul .nav .pull-right>
+            <li .dropdown #menu1>
+              <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
                 Current Repository: #{baseTitle webapp}
-                <b class="caret"></b>
-              <ul class="dropdown-menu">
+                <b .caret></b>
+              <ul .dropdown-menu>
                  <li><a href="#">#{baseTitle webapp}</a></li>
-                 <li class="divider"></li>
+                 <li .divider></li>
                  <li><a href="#">Add new repository</a></li>
 
-    <div class="container-fluid">
-      <div class="row-fluid">
-        <div class="span3">
-          <div class="sidebar-nav">
-            <div class="alert alert-info">
-              <a class="close" data-dismiss="alert" href="#">×</a>
+    <div .container-fluid>
+      <div .row-fluid>
+        <div .span3>
+          <div .sidebar-nav>
+            <div .alert .alert-info>
+              <a .close data-dismiss="alert" href="#">×</a>
               <b>This is just a demo.</b> If this were not just a demo,
               I'd not be filling this sidebar with silly alerts.
-            <div class="alert alert-success">
-              <a class="close" data-dismiss="alert" href="#">×</a>
+            <div .alert .alert-success>
+              <a .close data-dismiss="alert" href="#">×</a>
               <b>Well done!</b>
               You successfully read this important alert message.
-            <div class="alert alert-error">
-              <a class="close" data-dismiss="alert" href="#">×</a>
+            <div .alert .alert-error>
+              <a .close data-dismiss="alert" href="#">×</a>
               <b>Whoops!</b>
               Unable to connect to blah blah..
-        <div class="span9">
+        <div .span9>
           ^{pageBody page}
diff --git a/templates/status.hamlet b/templates/status.hamlet
index 9b9b0f7d18..2ccea1f1af 100644
--- a/templates/status.hamlet
+++ b/templates/status.hamlet
@@ -1,12 +1,12 @@
-<span id="#{updating}">
-  <div class="span9">
+<span ##{updating}>
+  <div .span9>
     $if null transfers
       <h2>No current transfers
     $else
       <h2>Transfers
       $forall (transfer, info) <- transfers
         $with percent <- maybe "unknown" (showPercentage 0) $ percentComplete transfer info
-          <div class="row-fluid">
+          <div .row-fluid>
             <h3>
               $maybe file <- associatedFile info
                 #{file}
@@ -20,11 +20,11 @@
               <small>#{maybe "unknown" Remote.name $ transferRemote info}</small>
                 $with size <- maybe "unknown" (roughSize dataUnits True) $ keySize $ transferKey transfer
                   $if isJust $ startedTime info
-                      <small class="pull-right"><b>#{percent} of #{size}</b></small>
+                      <small .pull-right><b>#{percent} of #{size}</b></small>
                   $else
-                      <small class="pull-right">queued (#{size})</small>
-          <div class="progress progress-striped">
-            <div class="bar" style="width: #{percent};">
+                      <small .pull-right>queued (#{size})</small>
+          <div .progress .progress-striped>
+            <div .bar style="width: #{percent};">
   <footer>
     <span>
       polled at #{time}

From c0ca6f44ac2cbe6d1cbb82bc73e2d2ead7695770 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 20:47:48 -0400
Subject: [PATCH 4315/8313] template reorg

---
 Assistant/Threads/WebApp.hs     |  3 +--
 static/css/webapp.css           |  7 ------
 templates/bootstrap.hamlet      | 40 +------------------------------
 templates/default-layout.hamlet |  3 ---
 templates/page.cassius          |  5 ++++
 templates/page.hamlet           | 42 +++++++++++++++++++++++++++++++++
 6 files changed, 49 insertions(+), 51 deletions(-)
 delete mode 100644 static/css/webapp.css
 delete mode 100644 templates/default-layout.hamlet
 create mode 100644 templates/page.cassius
 create mode 100644 templates/page.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 5407ed1954..92f7ff2535 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -61,11 +61,10 @@ instance Yesod WebApp where
 		page <- widgetToPageContent $ do
 			addStylesheet $ StaticR css_bootstrap_css
 			addStylesheet $ StaticR css_bootstrap_responsive_css
-			addStylesheet $ StaticR css_webapp_css
 			addScript $ StaticR jquery_full_js
 			addScript $ StaticR js_bootstrap_dropdown_js
 			addScript $ StaticR js_bootstrap_alert_js
-			$(widgetFile "default-layout")
+			$(widgetFile "page")
 		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
 
 	{- Require an auth token be set when accessing any (non-static route) -}
diff --git a/static/css/webapp.css b/static/css/webapp.css
deleted file mode 100644
index ba309effa6..0000000000
--- a/static/css/webapp.css
+++ /dev/null
@@ -1,7 +0,0 @@
-body {
-	padding-top: 60px;
-	padding-bottom: 40px;
-}
-.sidebar-nav {
-	padding: 9px 0;
-}
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
index 360b3d686d..389895df74 100644
--- a/templates/bootstrap.hamlet
+++ b/templates/bootstrap.hamlet
@@ -6,42 +6,4 @@ $doctype 5
     <meta name="viewport" content="width=device-width,initial-scale=1.0">
     ^{pageHead page}
   <body>
-
-    <div .navbar .navbar-fixed-top>
-      <div .navbar-inner>
-        <div .container>
-          <a .brand href="#">
-            git-annex
-          <ul .nav>
-            <li .active>
-              <a href="#">Dashboard</a>
-            <li>
-              <a href="@{ConfigR}">Config</a>
-          <ul .nav .pull-right>
-            <li .dropdown #menu1>
-              <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
-                Current Repository: #{baseTitle webapp}
-                <b .caret></b>
-              <ul .dropdown-menu>
-                 <li><a href="#">#{baseTitle webapp}</a></li>
-                 <li .divider></li>
-                 <li><a href="#">Add new repository</a></li>
-
-    <div .container-fluid>
-      <div .row-fluid>
-        <div .span3>
-          <div .sidebar-nav>
-            <div .alert .alert-info>
-              <a .close data-dismiss="alert" href="#">×</a>
-              <b>This is just a demo.</b> If this were not just a demo,
-              I'd not be filling this sidebar with silly alerts.
-            <div .alert .alert-success>
-              <a .close data-dismiss="alert" href="#">×</a>
-              <b>Well done!</b>
-              You successfully read this important alert message.
-            <div .alert .alert-error>
-              <a .close data-dismiss="alert" href="#">×</a>
-              <b>Whoops!</b>
-              Unable to connect to blah blah..
-        <div .span9>
-          ^{pageBody page}
+    ^{pageBody page}
diff --git a/templates/default-layout.hamlet b/templates/default-layout.hamlet
deleted file mode 100644
index 3701e3c42f..0000000000
--- a/templates/default-layout.hamlet
+++ /dev/null
@@ -1,3 +0,0 @@
-$maybe msg <- mmsg
-  <div #message>#{msg}
-^{widget}
diff --git a/templates/page.cassius b/templates/page.cassius
new file mode 100644
index 0000000000..39decd03a4
--- /dev/null
+++ b/templates/page.cassius
@@ -0,0 +1,5 @@
+body
+  padding-top: 60px
+  padding-bottom: 40px
+.sidebar-nav
+  padding: 9px 0
diff --git a/templates/page.hamlet b/templates/page.hamlet
new file mode 100644
index 0000000000..ae80bb05d5
--- /dev/null
+++ b/templates/page.hamlet
@@ -0,0 +1,42 @@
+<div .navbar .navbar-fixed-top>
+  <div .navbar-inner>
+    <div .container>
+      <a .brand href="#">
+        git-annex
+      <ul .nav>
+        <li .active>
+          <a href="#">Dashboard</a>
+        <li>
+          <a href="@{ConfigR}">Config</a>
+      <ul .nav .pull-right>
+        <li .dropdown #menu1>
+          <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
+            Current Repository: #{baseTitle webapp}
+            <b .caret></b>
+          <ul .dropdown-menu>
+             <li><a href="#">#{baseTitle webapp}</a></li>
+             <li .divider></li>
+             <li><a href="#">Add new repository</a></li>
+
+<div .container-fluid>
+  <div .row-fluid>
+    <div .span3>
+      <div .sidebar-nav>
+        <div .alert .alert-info>
+          <a .close data-dismiss="alert" href="#">×</a>
+          <b>This is just a demo.</b> If this were not just a demo,
+          I'd not be filling this sidebar with silly alerts.
+        <div .alert .alert-success>
+          <a .close data-dismiss="alert" href="#">×</a>
+          <b>Well done!</b>
+          You successfully read this important alert message.
+        <div .alert .alert-error>
+          <a .close data-dismiss="alert" href="#">×</a>
+          <b>Whoops!</b>
+          Unable to connect to blah blah..
+    <div .span9>
+      $maybe msg <- mmsg
+        <div .alert .alert-info>
+          <a .close data-dismiss="alert" href="#">×</a>
+          #{msg}
+      ^{widget}

From 13e9b275dd96673e5a3421b64d672abdd29e9a26 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Fri, 27 Jul 2012 21:05:27 -0400
Subject: [PATCH 4316/8313] initremote: Avoid recording remote's description
 before checking that its config is valid.

---
 Command/InitRemote.hs                              | 14 +++++++-------
 debian/changelog                                   |  7 +++++++
 ...mote_needs_some___34__error_checking__34__.mdwn |  3 +++
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/Command/InitRemote.hs b/Command/InitRemote.hs
index a78505a197..ad93529cc2 100644
--- a/Command/InitRemote.hs
+++ b/Command/InitRemote.hs
@@ -38,18 +38,19 @@ start (name:ws) = do
 	t <- findType fullconfig
 
 	showStart "initremote" name
-	next $ perform t u $ M.union config c
+	next $ perform t u name $ M.union config c
 
 	where
 		config = Logs.Remote.keyValToConfig ws
 
-perform :: RemoteType -> UUID -> R.RemoteConfig -> CommandPerform
-perform t u c = do
+perform :: RemoteType -> UUID -> String -> R.RemoteConfig -> CommandPerform
+perform t u name c = do
 	c' <- R.setup t u c
-	next $ cleanup u c'
+	next $ cleanup u name c'
 
-cleanup :: UUID -> R.RemoteConfig -> CommandCleanup
-cleanup u c = do
+cleanup :: UUID -> String -> R.RemoteConfig -> CommandCleanup
+cleanup u name c = do
+	describeUUID u name
 	Logs.Remote.configSet u c
         return True
 
@@ -61,7 +62,6 @@ findByName name = do
 	where
 		generate = do
 			uuid <- liftIO genUUID
-			describeUUID uuid name
 			return (uuid, M.insert nameKey name M.empty)
 
 findByName' :: String ->  M.Map UUID R.RemoteConfig -> Maybe (UUID, R.RemoteConfig)
diff --git a/debian/changelog b/debian/changelog
index 387dacd53f..82bf5009a1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,10 @@
+git-annex (3.20120722) UNRELEASED; urgency=low
+
+  * initremote: Avoid recording remote's description before checking
+    that its config is valid.
+
+ -- Joey Hess <joeyh@debian.org>  Fri, 27 Jul 2012 21:04:47 -0400
+
 git-annex (3.20120721) unstable; urgency=low
 
   * get, move, copy: Now refuse to do anything when the requested file
diff --git a/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn b/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn
index 3521038d74..6b57f8ce5f 100644
--- a/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn
+++ b/doc/bugs/git_annex_initremote_needs_some___34__error_checking__34__.mdwn
@@ -60,3 +60,6 @@ bloom filter size: 16 mebibytes (0% full)
 backend usage: 
 x00:atest jtang$ 
 </pre>
+
+> Indeed, I broke that in June by making it record the name in a much too
+> early stage. Now fixed. [[done]] --[[Joey]]

From c8cc5cc8da5d4467a12b8723a700769d2ffb1666 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkpEY8WTFDhjIVTWG38Ph7ppmuXUTJAHAg"
 <Justin@web>
Date: Sat, 28 Jul 2012 04:08:00 +0000
Subject: [PATCH 4317/8313] Added a comment: selective sync

---
 ...1_58c4faa321a5bb71adf9fdee079849f4._comment | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment

diff --git a/doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment b/doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment
new file mode 100644
index 0000000000..aa65cd6b9a
--- /dev/null
+++ b/doc/design/assistant/partial_content/comment_1_58c4faa321a5bb71adf9fdee079849f4._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkpEY8WTFDhjIVTWG38Ph7ppmuXUTJAHAg"
+ nickname="Justin"
+ subject="selective sync"
+ date="2012-07-28T04:08:00Z"
+ content="""
+hey joey
+
+great work!!
+
+will partial content work like selective sync in dropbox
+
+use case: on desktop i have photos/mp3s/docs, but would only want to sync the docs to my netbook
+
+cheers
+
+justin
+"""]]

From ca478b7bcb48fee0d1a97340e6ea5da8e97074f0 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 15:41:49 -0400
Subject: [PATCH 4318/8313] Focus today was writing a notification broadcaster.

This is a way to send a notification to a set of clients, any of which can be
blocked waiting for a new notification to arrive. A complication is that any
number of clients may be be dead, and we don't want stale notifications for
those clients to pile up and leak memory.

It took me 3 tries to find the solution, which turns out to be simple: An array
of SampleVars, one per client. Using SampleVars means that clients only see the
most recent notification, but when the notification is just "the assistant's
state changed somehow; display a refreshed rendering of it", that's sufficient.
---
 Utility/NotificationBroadcaster.hs | 75 ++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)
 create mode 100644 Utility/NotificationBroadcaster.hs

diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs
new file mode 100644
index 0000000000..51b321752b
--- /dev/null
+++ b/Utility/NotificationBroadcaster.hs
@@ -0,0 +1,75 @@
+{- notification broadcaster
+ -
+ - This is used to allow clients to block until there is a new notification
+ - that some thing occurred. It does not communicate what the change is,
+ - it only provides blocking reads to wait on notifications.
+ -
+ - Multiple clients are supported. Each has a unique id.
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Assistant.NotificationBroadCaster (
+	NotificationBroadCaster,
+	NotificationHandle,
+	newNotificationBroadCaster,
+	newNotificationHandle,
+	notificationHandleToId,
+	notificationHandleFromId,
+	sendNotification,
+	waitNotification,
+) where
+
+import Common
+
+import Control.Concurrent.STM
+import Control.Concurrent.SampleVar
+
+{- One SampleVar per client. The TMVar is never empty, so never blocks. -}
+type NotificationBroadCaster = TMVar [SampleVar ()]
+
+{- Handle given out to an individual client. -}
+data NotificationHandle = NotificationHandle NotificationBroadCaster Int
+
+newNotificationBroadCaster :: IO NotificationBroadCaster
+newNotificationBroadCaster = atomically (newTMVar [])
+
+{- Allocates a notification handle for a client to use. -}
+newNotificationHandle :: NotificationBroadCaster -> IO NotificationHandle
+newNotificationHandle b = NotificationHandle
+	<$> pure b
+	<*> addclient b
+	where
+		addclient b = do
+			s <- newEmptySampleVar
+			atomically $ do
+				l <- readTMVar b
+				putTMVar b $ l ++ [s]
+				return $ length l
+
+{- Extracts the Int identifier from a notification handle.
+ - This can be used to eg, pass the identifier through to a WebApp. -}
+notificationHandleToId :: NotificationHandle -> Int
+notificationHandleToId (NotificationHandle _ i) = i
+
+{- Given a NotificationBroadCaster, and an Int identifier, recreates the
+ - NotificationHandle. -}
+notificationHandleFromId :: NotificationBroadCaster -> Int -> NotificationHandle
+notificationHandleFromId = NotificationHandle
+
+{- Sends a notification to all clients. -}
+sendNotification :: NotificationBroadCaster -> IO ()
+sendNotification b = do
+	l <- atomically $ readTMVar b
+	mapM_ notify l
+	where
+		notify s = writeSampleVar s ()
+
+{- Used by a client to block until a new notification is available since
+ - the last time it tried. -}
+waitNotification :: NotificationHandle -> IO ()
+waitNotification (NotificationHandle b i) = do
+	l <- atomically $ readTMVar b
+	readSampleVar (l !! i)

From a17fde22fabdb706086ac945bc331e32527b58bd Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 16:01:50 -0400
Subject: [PATCH 4319/8313] add a NotificationBroadcaster to DaemonStatus

First use of it is to make the status checkpointer thread block until
there is really a change to the status.
---
 Assistant/DaemonStatus.hs          | 54 ++++++++++++++++++++----------
 Utility/NotificationBroadcaster.hs | 26 +++++++-------
 2 files changed, 50 insertions(+), 30 deletions(-)

diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 88306a6363..84a3662f0c 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -11,6 +11,7 @@ import Common.Annex
 import Assistant.ThreadedMonad
 import Utility.ThreadScheduler
 import Utility.TempFile
+import Utility.NotificationBroadcaster
 import Logs.Transfer
 import qualified Command.Sync
 
@@ -34,31 +35,43 @@ data DaemonStatus = DaemonStatus
 	, currentTransfers :: TransferMap
 	-- Ordered list of remotes to talk to.
 	, knownRemotes :: [Remote]
+	-- Clients can use this to wait on changes to the DaemonStatus
+	, notificationBroadcaster :: NotificationBroadcaster
 	}
-	deriving (Show)
 
 type TransferMap = M.Map Transfer TransferInfo
 
 type DaemonStatusHandle = MVar DaemonStatus
 
-newDaemonStatus :: DaemonStatus
-newDaemonStatus = DaemonStatus
-	{ scanComplete = False
-	, lastRunning = Nothing
-	, sanityCheckRunning = False
-	, lastSanityCheck = Nothing
-	, currentTransfers = M.empty
-	, knownRemotes = []
-	}
+newDaemonStatus :: IO DaemonStatus
+newDaemonStatus = do
+	nb <- newNotificationBroadcaster
+	return $ DaemonStatus
+		{ scanComplete = False
+		, lastRunning = Nothing
+		, sanityCheckRunning = False
+		, lastSanityCheck = Nothing
+		, currentTransfers = M.empty
+		, knownRemotes = []
+		, notificationBroadcaster = nb
+		}
 
 getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
 getDaemonStatus = liftIO . readMVar
 
 modifyDaemonStatus_ :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex ()
-modifyDaemonStatus_ handle a = liftIO $ modifyMVar_ handle (return . a)
+modifyDaemonStatus_ handle a = do
+	nb <- liftIO $ modifyMVar handle $ \s -> return
+		(a s, notificationBroadcaster s)
+	liftIO $ sendNotification nb
 
 modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> Annex b
-modifyDaemonStatus handle a = liftIO $ modifyMVar handle (return . a)
+modifyDaemonStatus handle a = do
+	(b, nb) <- liftIO $ modifyMVar handle $ \s -> do
+		let (s', b) = a s
+		return $ (s', (b, notificationBroadcaster s))
+	liftIO $ sendNotification nb
+	return b
 
 {- Updates the cached ordered list of remotes from the list in Annex
  - state. -}
@@ -74,7 +87,7 @@ startDaemonStatus :: Annex DaemonStatusHandle
 startDaemonStatus = do
 	file <- fromRepo gitAnnexDaemonStatusFile
 	status <- liftIO $
-		catchDefaultIO (readDaemonStatusFile file) newDaemonStatus
+		catchDefaultIO (readDaemonStatusFile file) =<< newDaemonStatus
 	transfers <- M.fromList <$> getTransfers
 	remotes <- Command.Sync.syncRemotes []
 	liftIO $ newMVar status
@@ -84,11 +97,18 @@ startDaemonStatus = do
 		, knownRemotes = remotes
 		}
 
-{- This thread wakes up periodically and writes the daemon status to disk. -}
+{- This writes the daemon status to disk, when it changes, but no more
+ - frequently than once every ten minutes.
+ -}
 daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
 daemonStatusThread st handle = do
+	bhandle <- runThreadState st $ 
+		liftIO . newNotificationHandle
+			=<< notificationBroadcaster <$> getDaemonStatus handle
 	checkpoint
-	runEvery (Seconds tenMinutes) checkpoint
+	runEvery (Seconds tenMinutes) $ do
+		liftIO $ waitNotification bhandle
+		checkpoint
 	where
 		checkpoint = runThreadState st $ do
 			file <- fromRepo gitAnnexDaemonStatusFile
@@ -109,9 +129,9 @@ writeDaemonStatusFile file status =
 			]
 
 readDaemonStatusFile :: FilePath -> IO DaemonStatus
-readDaemonStatusFile file = parse <$> readFile file
+readDaemonStatusFile file = parse <$> newDaemonStatus <*> readFile file
 	where
-		parse = foldr parseline newDaemonStatus . lines
+		parse status = foldr parseline status . lines
 		parseline line status
 			| key == "lastRunning" = parseval readtime $ \v ->
 				status { lastRunning = Just v }
diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs
index 51b321752b..caa13bbacf 100644
--- a/Utility/NotificationBroadcaster.hs
+++ b/Utility/NotificationBroadcaster.hs
@@ -11,10 +11,10 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Assistant.NotificationBroadCaster (
-	NotificationBroadCaster,
+module Utility.NotificationBroadcaster (
+	NotificationBroadcaster,
 	NotificationHandle,
-	newNotificationBroadCaster,
+	newNotificationBroadcaster,
 	newNotificationHandle,
 	notificationHandleToId,
 	notificationHandleFromId,
@@ -28,21 +28,21 @@ import Control.Concurrent.STM
 import Control.Concurrent.SampleVar
 
 {- One SampleVar per client. The TMVar is never empty, so never blocks. -}
-type NotificationBroadCaster = TMVar [SampleVar ()]
+type NotificationBroadcaster = TMVar [SampleVar ()]
 
 {- Handle given out to an individual client. -}
-data NotificationHandle = NotificationHandle NotificationBroadCaster Int
+data NotificationHandle = NotificationHandle NotificationBroadcaster Int
 
-newNotificationBroadCaster :: IO NotificationBroadCaster
-newNotificationBroadCaster = atomically (newTMVar [])
+newNotificationBroadcaster :: IO NotificationBroadcaster
+newNotificationBroadcaster = atomically (newTMVar [])
 
 {- Allocates a notification handle for a client to use. -}
-newNotificationHandle :: NotificationBroadCaster -> IO NotificationHandle
+newNotificationHandle :: NotificationBroadcaster -> IO NotificationHandle
 newNotificationHandle b = NotificationHandle
 	<$> pure b
-	<*> addclient b
+	<*> addclient
 	where
-		addclient b = do
+		addclient = do
 			s <- newEmptySampleVar
 			atomically $ do
 				l <- readTMVar b
@@ -54,13 +54,13 @@ newNotificationHandle b = NotificationHandle
 notificationHandleToId :: NotificationHandle -> Int
 notificationHandleToId (NotificationHandle _ i) = i
 
-{- Given a NotificationBroadCaster, and an Int identifier, recreates the
+{- Given a NotificationBroadcaster, and an Int identifier, recreates the
  - NotificationHandle. -}
-notificationHandleFromId :: NotificationBroadCaster -> Int -> NotificationHandle
+notificationHandleFromId :: NotificationBroadcaster -> Int -> NotificationHandle
 notificationHandleFromId = NotificationHandle
 
 {- Sends a notification to all clients. -}
-sendNotification :: NotificationBroadCaster -> IO ()
+sendNotification :: NotificationBroadcaster -> IO ()
 sendNotification b = do
 	l <- atomically $ readTMVar b
 	mapM_ notify l

From d7e696f38c73dfb351df187fb783d54acf275634 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl_x1UcmuSOmX7WkBTJPQwOP1FgUPdWEWM"
 <Philipp@web>
Date: Sat, 28 Jul 2012 21:09:23 +0000
Subject: [PATCH 4320/8313] fixed typo: repositry

---
 doc/sync.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sync.mdwn b/doc/sync.mdwn
index 765c1e43fd..057dcb355e 100644
--- a/doc/sync.mdwn
+++ b/doc/sync.mdwn
@@ -1,7 +1,7 @@
 The `git annex sync` command provides an easy way to keep several
 repositories in sync. 
 
-Often git is used in a centralized fashion with a central bare repositry
+Often git is used in a centralized fashion with a central bare repository
 which changes are pulled and pushed to using normal git commands.
 That works fine, if you don't mind having a central repository.
 

From 3cc18857936e5a09e033439971dc9c43e6ccbaa2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 18:02:11 -0400
Subject: [PATCH 4321/8313] move DaemonStatus manipulation out of the Annex
 monad to IO

I've convinced myself that nothing in DaemonStatus can deadlock,
as it always keepts the TMVar full. That was the only reason it was in the
Annex monad.
---
 Assistant.hs                         |  6 ++--
 Assistant/DaemonStatus.hs            | 50 +++++++++++++---------------
 Assistant/Threads/Pusher.hs          |  3 +-
 Assistant/Threads/SanityChecker.hs   | 24 ++++++-------
 Assistant/Threads/TransferWatcher.hs |  9 +++--
 Assistant/Threads/Transferrer.hs     |  4 +--
 Assistant/Threads/Watcher.hs         |  9 +++--
 Assistant/Threads/WebApp.hs          |  2 +-
 Assistant/TransferQueue.hs           |  2 +-
 9 files changed, 49 insertions(+), 60 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index ca428988fe..6b25c3c6f1 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -65,10 +65,8 @@
  - 	Annex monad in IO actions run by the watcher and committer
  - 	threads. Thus, a single state is shared amoung the threads, and
  - 	only one at a time can access it.
- - DaemonStatusHandle: (MVar)
- - 	The daemon's current status. This MVar should only be manipulated
- - 	from inside the Annex monad, which ensures it's accessed only
- - 	after the ThreadState MVar.
+ - DaemonStatusHandle: (STM TMVar)
+ - 	The daemon's current status.
  - ChangeChan: (STM TChan)
  - 	Changes are indicated by writing to this channel. The committer
  - 	reads from it.
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 84a3662f0c..52165138e6 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -15,7 +15,7 @@ import Utility.NotificationBroadcaster
 import Logs.Transfer
 import qualified Command.Sync
 
-import Control.Concurrent
+import Control.Concurrent.STM
 import System.Posix.Types
 import Data.Time.Clock.POSIX
 import Data.Time
@@ -41,7 +41,8 @@ data DaemonStatus = DaemonStatus
 
 type TransferMap = M.Map Transfer TransferInfo
 
-type DaemonStatusHandle = MVar DaemonStatus
+{- This TMVar is never left empty, so accessing it will never block. -}
+type DaemonStatusHandle = TMVar DaemonStatus
 
 newDaemonStatus :: IO DaemonStatus
 newDaemonStatus = do
@@ -56,21 +57,19 @@ newDaemonStatus = do
 		, notificationBroadcaster = nb
 		}
 
-getDaemonStatus :: DaemonStatusHandle -> Annex DaemonStatus
-getDaemonStatus = liftIO . readMVar
+getDaemonStatus :: DaemonStatusHandle -> IO DaemonStatus
+getDaemonStatus = atomically . readTMVar
 
-modifyDaemonStatus_ :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> Annex ()
-modifyDaemonStatus_ handle a = do
-	nb <- liftIO $ modifyMVar handle $ \s -> return
-		(a s, notificationBroadcaster s)
-	liftIO $ sendNotification nb
+modifyDaemonStatus_ :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> IO ()
+modifyDaemonStatus_ handle a = modifyDaemonStatus handle $ \s -> (a s, ())
 
-modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> Annex b
+modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> IO b
 modifyDaemonStatus handle a = do
-	(b, nb) <- liftIO $ modifyMVar handle $ \s -> do
-		let (s', b) = a s
-		return $ (s', (b, notificationBroadcaster s))
-	liftIO $ sendNotification nb
+	(b, nb) <- atomically $ do
+		(s, b) <- a <$> takeTMVar handle
+		putTMVar handle s
+		return $ (b, notificationBroadcaster s)
+	sendNotification nb
 	return b
 
 {- Updates the cached ordered list of remotes from the list in Annex
@@ -78,10 +77,10 @@ modifyDaemonStatus handle a = do
 updateKnownRemotes :: DaemonStatusHandle -> Annex ()
 updateKnownRemotes dstatus = do
 	remotes <- Command.Sync.syncRemotes []
-	modifyDaemonStatus_ dstatus $
+	liftIO $ modifyDaemonStatus_ dstatus $
 		\s -> s { knownRemotes = remotes }
 
-{- Load any previous daemon status file, and store it in the MVar for this
+{- Load any previous daemon status file, and store it in a MVar for this
  - process to use as its DaemonStatus. Also gets current transfer status. -}
 startDaemonStatus :: Annex DaemonStatusHandle
 startDaemonStatus = do
@@ -90,7 +89,7 @@ startDaemonStatus = do
 		catchDefaultIO (readDaemonStatusFile file) =<< newDaemonStatus
 	transfers <- M.fromList <$> getTransfers
 	remotes <- Command.Sync.syncRemotes []
-	liftIO $ newMVar status
+	liftIO $ atomically $ newTMVar status
 		{ scanComplete = False
 		, sanityCheckRunning = False
 		, currentTransfers = transfers
@@ -102,18 +101,17 @@ startDaemonStatus = do
  -}
 daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
 daemonStatusThread st handle = do
-	bhandle <- runThreadState st $ 
-		liftIO . newNotificationHandle
-			=<< notificationBroadcaster <$> getDaemonStatus handle
+	bhandle <- newNotificationHandle
+		=<< notificationBroadcaster <$> getDaemonStatus handle
 	checkpoint
 	runEvery (Seconds tenMinutes) $ do
-		liftIO $ waitNotification bhandle
+		waitNotification bhandle
 		checkpoint
 	where
-		checkpoint = runThreadState st $ do
-			file <- fromRepo gitAnnexDaemonStatusFile
+		checkpoint = do
 			status <- getDaemonStatus handle
-			liftIO $ writeDaemonStatusFile file status
+			file <- runThreadState st $ fromRepo gitAnnexDaemonStatusFile
+			writeDaemonStatusFile file status
 
 {- Don't just dump out the structure, because it will change over time,
  - and parts of it are not relevant. -}
@@ -167,12 +165,12 @@ tenMinutes :: Int
 tenMinutes = 10 * 60
 
 {- Mutates the transfer map. -}
-adjustTransfers :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> Annex ()
+adjustTransfers :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> IO ()
 adjustTransfers dstatus a = modifyDaemonStatus_ dstatus $
 	\s -> s { currentTransfers = a (currentTransfers s) }
 
 {- Removes a transfer from the map, and returns its info. -}
-removeTransfer :: DaemonStatusHandle -> Transfer -> Annex (Maybe TransferInfo)
+removeTransfer :: DaemonStatusHandle -> Transfer -> IO (Maybe TransferInfo)
 removeTransfer dstatus t = modifyDaemonStatus dstatus go
 	where
 		go s =
diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs
index cba53af233..3762c48368 100644
--- a/Assistant/Threads/Pusher.hs
+++ b/Assistant/Threads/Pusher.hs
@@ -51,8 +51,7 @@ pushThread st daemonstatus commitchan pushmap = do
 		now <- getCurrentTime
 		if shouldPush now commits
 			then do
-				remotes <- runThreadState st $
-					knownRemotes <$> getDaemonStatus daemonstatus
+				remotes <- knownRemotes <$> getDaemonStatus daemonstatus
 				pushToRemotes thisThread now st (Just pushmap) remotes
 			else do
 				debug thisThread
diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs
index 09aee0797c..5e27246a02 100644
--- a/Assistant/Threads/SanityChecker.hs
+++ b/Assistant/Threads/SanityChecker.hs
@@ -26,32 +26,28 @@ thisThread = "SanityChecker"
 {- This thread wakes up occasionally to make sure the tree is in good shape. -}
 sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO ()
 sanityCheckerThread st status transferqueue changechan = forever $ do
-	waitForNextCheck st status
+	waitForNextCheck status
 
 	debug thisThread ["starting sanity check"]
 
-	runThreadState st $
-		modifyDaemonStatus_ status $ \s -> s
-				{ sanityCheckRunning = True }
+	modifyDaemonStatus_ status $ \s -> s
+		{ sanityCheckRunning = True }
 
 	now <- getPOSIXTime -- before check started
 	catchIO (check st status transferqueue changechan)
 		(runThreadState st . warning . show)
 
-	runThreadState st $ do
-		modifyDaemonStatus_ status $ \s -> s
-			{ sanityCheckRunning = False
-			, lastSanityCheck = Just now
-			}
+	modifyDaemonStatus_ status $ \s -> s
+		{ sanityCheckRunning = False
+		, lastSanityCheck = Just now
+		}
 	
 	debug thisThread ["sanity check complete"]
 
-
 {- Only run one check per day, from the time of the last check. -}
-waitForNextCheck :: ThreadState -> DaemonStatusHandle -> IO ()
-waitForNextCheck st status = do
-	v <- runThreadState st $
-		lastSanityCheck <$> getDaemonStatus status
+waitForNextCheck :: DaemonStatusHandle -> IO ()
+waitForNextCheck status = do
+	v <- lastSanityCheck <$> getDaemonStatus status
 	now <- getPOSIXTime
 	threadDelaySeconds $ Seconds $ calcdelay now v
 	where
diff --git a/Assistant/Threads/TransferWatcher.hs b/Assistant/Threads/TransferWatcher.hs
index be520aaf93..447ff2264f 100644
--- a/Assistant/Threads/TransferWatcher.hs
+++ b/Assistant/Threads/TransferWatcher.hs
@@ -55,12 +55,11 @@ onErr _ _ msg _ = error msg
 onAdd :: Handler
 onAdd st dstatus file _ = case parseTransferFile file of
 	Nothing -> noop
-	Just t -> do
-		runThreadState st $ go t =<< checkTransfer t
+	Just t -> go t =<< runThreadState st (checkTransfer t)
 	where
 		go _ Nothing = noop -- transfer already finished
 		go t (Just info) = do
-			liftIO $ debug thisThread
+			debug thisThread
 				[ "transfer starting:"
 				, show t
 				]
@@ -71,11 +70,11 @@ onAdd st dstatus file _ = case parseTransferFile file of
 
 {- Called when a transfer information file is removed. -}
 onDel :: Handler
-onDel st dstatus file _ = case parseTransferFile file of
+onDel _ dstatus file _ = case parseTransferFile file of
 	Nothing -> noop
 	Just t -> do
 		debug thisThread
 			[ "transfer finishing:"
 			, show t
 			]
-		void $ runThreadState st $ removeTransfer dstatus t
+		void $ removeTransfer dstatus t
diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs
index d8a1469484..30802f7428 100644
--- a/Assistant/Threads/Transferrer.hs
+++ b/Assistant/Threads/Transferrer.hs
@@ -48,7 +48,7 @@ transfererThread st dstatus transferqueue slots = go
  - being uploaded to isn't known to have the file. -}
 shouldTransfer :: DaemonStatusHandle -> Transfer -> TransferInfo -> Annex Bool
 shouldTransfer dstatus t info =
-	go =<< currentTransfers <$> getDaemonStatus dstatus
+	go =<< currentTransfers <$> liftIO (getDaemonStatus dstatus)
 	where
 		go m
 			| M.member t m = return False
@@ -84,7 +84,7 @@ transferThread st dstatus slots t info = case (transferRemote info, associatedFi
 		tid <- inTransferSlot slots st $
 			transferprocess remote file
 		now <- getCurrentTime
-		runThreadState st $ adjustTransfers dstatus $
+		adjustTransfers dstatus $
 			M.insertWith' const t info
 				{ startedTime = Just $ utcTimeToPOSIXSeconds now
 				, transferTid = Just tid
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 31025361be..ab57bf04a0 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -76,8 +76,7 @@ statupScan st dstatus scanner = do
 	runThreadState st $
 		showAction "scanning"
 	r <- scanner
-	runThreadState st $
-		modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
+	modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
 
 	-- Notice any files that were deleted before watching was started.
 	runThreadState st $ do
@@ -132,7 +131,7 @@ runHandler threadname st dstatus transferqueue changechan handler file filestatu
 onAdd :: Handler
 onAdd threadname file filestatus dstatus _
 	| maybe False isRegularFile filestatus = do
-		ifM (scanComplete <$> getDaemonStatus dstatus)
+		ifM (scanComplete <$> liftIO (getDaemonStatus dstatus))
 			( go
 			, ifM (null <$> inRepo (Git.LsFiles.notInRepo False [file]))
 				( noChange
@@ -156,7 +155,7 @@ onAddSymlink threadname file filestatus dstatus transferqueue = go =<< Backend.l
 			link <- calcGitLink file key
 			ifM ((==) link <$> liftIO (readSymbolicLink file))
 				( do
-					s <- getDaemonStatus dstatus
+					s <- liftIO $ getDaemonStatus dstatus
 					checkcontent key s
 					ensurestaged link s
 				, do
@@ -167,7 +166,7 @@ onAddSymlink threadname file filestatus dstatus transferqueue = go =<< Backend.l
 				)
 		go Nothing = do -- other symlink
 			link <- liftIO (readSymbolicLink file)
-			ensurestaged link =<< getDaemonStatus dstatus
+			ensurestaged link =<< liftIO (getDaemonStatus dstatus)
 
 		{- This is often called on symlinks that are already
 		 - staged correctly. A symlink may have been deleted
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 92f7ff2535..6e895ccf63 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -115,7 +115,7 @@ statusDisplay = do
 	
 	current <- liftIO $ runThreadState (threadState webapp) $
 		M.toList . currentTransfers
-			<$> getDaemonStatus (daemonStatus webapp)
+			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
 	queued <- liftIO $ getTransferQueue $ transferQueue webapp
 	let transfers = current ++ queued
 
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index 414a1f9bee..2f09813eb0 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -59,7 +59,7 @@ stubInfo f r = TransferInfo
 {- Adds transfers to queue for some of the known remotes. -}
 queueTransfers :: Schedule -> TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex ()
 queueTransfers schedule q daemonstatus k f direction = do
-	rs <- knownRemotes <$> getDaemonStatus daemonstatus
+	rs <- knownRemotes <$> liftIO (getDaemonStatus daemonstatus)
 	mapM_ go =<< sufficientremotes rs
 	where
 		sufficientremotes rs

From e31277d38aa5d9b07395d05a6f1646b5eb3d48c2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 18:47:24 -0400
Subject: [PATCH 4322/8313] send notifications when the TransferQueue is
 changed

The fun part was making it move things from TransferQueue to currentTransfers
entirely atomically. Which will avoid inconsistent display if the WebApp
renders the current status at just the wrong time. STM to the rescue!
---
 Assistant.hs                         |  2 +-
 Assistant/DaemonStatus.hs            | 18 ++++++++-
 Assistant/Threads/TransferScanner.hs | 13 ++++---
 Assistant/Threads/Transferrer.hs     | 10 ++++-
 Assistant/TransferQueue.hs           | 56 +++++++++++++++++-----------
 5 files changed, 67 insertions(+), 32 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 6b25c3c6f1..1f41a9398f 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -153,7 +153,7 @@ startDaemon assistant foreground webappwaiter
 				, daemonStatusThread st dstatus
 				, sanityCheckerThread st dstatus transferqueue changechan
 				, mountWatcherThread st dstatus scanremotes
-				, transferScannerThread st scanremotes transferqueue
+				, transferScannerThread st dstatus scanremotes transferqueue
 #ifdef WITH_WEBAPP
 				, webAppThread st dstatus transferqueue webappwaiter
 #endif
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 52165138e6..3610c2fdad 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -36,6 +36,7 @@ data DaemonStatus = DaemonStatus
 	-- Ordered list of remotes to talk to.
 	, knownRemotes :: [Remote]
 	-- Clients can use this to wait on changes to the DaemonStatus
+	-- and other related things like the TransferQueue.
 	, notificationBroadcaster :: NotificationBroadcaster
 	}
 
@@ -72,6 +73,12 @@ modifyDaemonStatus handle a = do
 	sendNotification nb
 	return b
 
+{- Can be used to send a notification that the daemon status, or other
+ - associated thing, like the TransferQueue, has changed. -}
+notifyDaemonStatusChange :: DaemonStatusHandle -> IO ()
+notifyDaemonStatusChange handle = sendNotification
+	=<< notificationBroadcaster <$> atomically (readTMVar handle)
+
 {- Updates the cached ordered list of remotes from the list in Annex
  - state. -}
 updateKnownRemotes :: DaemonStatusHandle -> Annex ()
@@ -164,7 +171,16 @@ afterLastDaemonRun timestamp status = maybe False (< t) (lastRunning status)
 tenMinutes :: Int
 tenMinutes = 10 * 60
 
-{- Mutates the transfer map. -}
+{- Mutates the transfer map. Runs in STM so that the transfer map can
+ - be modified in the same transaction that modifies the transfer queue.
+ - Note that this does not send a notification of the change; that's left
+ - to the caller. -}
+adjustTransfersSTM :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> STM ()
+adjustTransfersSTM dstatus a = do
+	s <- takeTMVar dstatus
+	putTMVar dstatus $ s { currentTransfers = a (currentTransfers s) }
+
+{- Variant that does send notifications. -}
 adjustTransfers :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> IO ()
 adjustTransfers dstatus a = modifyDaemonStatus_ dstatus $
 	\s -> s { currentTransfers = a (currentTransfers s) }
diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index e76cbe81d3..e6a078907b 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -11,6 +11,7 @@ import Assistant.Common
 import Assistant.ScanRemotes
 import Assistant.TransferQueue
 import Assistant.ThreadedMonad
+import Assistant.DaemonStatus
 import Logs.Transfer
 import Logs.Location
 import qualified Remote
@@ -25,20 +26,20 @@ thisThread = "TransferScanner"
 {- This thread waits until a remote needs to be scanned, to find transfers
  - that need to be made, to keep data in sync.
  -}
-transferScannerThread :: ThreadState -> ScanRemoteMap -> TransferQueue -> IO ()
-transferScannerThread st scanremotes transferqueue = do
+transferScannerThread :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> TransferQueue -> IO ()
+transferScannerThread st dstatus scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
 		liftIO $ debug thisThread ["starting scan of", show r]
-		scan st transferqueue r
+		scan st dstatus transferqueue r
 		liftIO $ debug thisThread ["finished scan of", show r]
 	where
 
 {- This is a naive scan through the git work tree.
  - 
  - The scan is blocked when the transfer queue gets too large. -}
-scan :: ThreadState -> TransferQueue -> Remote -> IO ()
-scan st transferqueue r = do
+scan :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Remote -> IO ()
+scan st dstatus transferqueue r = do
 	g <- runThreadState st $ fromRepo id
 	files <- LsFiles.inRepo [] g
 	go files
@@ -63,7 +64,7 @@ scan st transferqueue r = do
 					| otherwise = return Nothing
 				
 		u = Remote.uuid r
-		enqueue f t = queueTransferAt smallsize Later transferqueue (Just f) t r
+		enqueue f t = queueTransferAt smallsize Later transferqueue dstatus (Just f) t r
 		smallsize = 10
 
 		{- Look directly in remote for the key when it's cheap;
diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs
index 30802f7428..f011ff0363 100644
--- a/Assistant/Threads/Transferrer.hs
+++ b/Assistant/Threads/Transferrer.hs
@@ -34,12 +34,18 @@ transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Transf
 transfererThread st dstatus transferqueue slots = go
 	where
 		go = do
-			(t, info) <- getNextTransfer transferqueue
+			(t, info) <- getNextTransfer transferqueue dstatus
 			ifM (runThreadState st $ shouldTransfer dstatus t info)
 				( do
 					debug thisThread [ "Transferring:" , show t ]
+					notifyDaemonStatusChange dstatus
 					transferThread st dstatus slots t info
-				, debug thisThread [ "Skipping unnecessary transfer:" , show t ]
+				, do
+					debug thisThread [ "Skipping unnecessary transfer:" , show t ]
+					-- getNextTransfer added t to the
+					-- daemonstatus's transfer map.
+					void $ removeTransfer dstatus t
+					notifyDaemonStatusChange dstatus
 				)
 			go
 
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index 2f09813eb0..51ed5c9c78 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -23,6 +23,7 @@ import Types.Remote
 import qualified Remote
 
 import Control.Concurrent.STM
+import qualified Data.Map as M
 
 {- The transfer queue consists of a channel listing the transfers to make;
  - the size of the queue is also tracked, and a list is maintained
@@ -58,8 +59,8 @@ stubInfo f r = TransferInfo
 
 {- Adds transfers to queue for some of the known remotes. -}
 queueTransfers :: Schedule -> TransferQueue -> DaemonStatusHandle -> Key -> AssociatedFile -> Direction -> Annex ()
-queueTransfers schedule q daemonstatus k f direction = do
-	rs <- knownRemotes <$> liftIO (getDaemonStatus daemonstatus)
+queueTransfers schedule q dstatus k f direction = do
+	rs <- knownRemotes <$> liftIO (getDaemonStatus dstatus)
 	mapM_ go =<< sufficientremotes rs
 	where
 		sufficientremotes rs
@@ -80,37 +81,48 @@ queueTransfers schedule q daemonstatus k f direction = do
 			, transferKey = k
 			, transferUUID = Remote.uuid r
 			}
-		go r = liftIO $ atomically $
-			enqueue schedule q (gentransfer r) (stubInfo f r)
+		go r = liftIO $
+			enqueue schedule q dstatus (gentransfer r) (stubInfo f r)
 
-enqueue :: Schedule -> TransferQueue -> Transfer -> TransferInfo -> STM ()
-enqueue schedule q t info
+enqueue :: Schedule -> TransferQueue -> DaemonStatusHandle -> Transfer -> TransferInfo -> IO ()
+enqueue schedule q dstatus t info
 	| schedule == Next = go unGetTChan (new:)
 	| otherwise = go writeTChan (\l -> l++[new])
 	where
 		new = (t, info)
 		go modqueue modlist = do
-			void $ modqueue (queue q) new
-			void $ modifyTVar' (queuesize q) succ
-			void $ modifyTVar' (queuelist q) modlist
+			atomically $ do
+				void $ modqueue (queue q) new
+				void $ modifyTVar' (queuesize q) succ
+				void $ modifyTVar' (queuelist q) modlist
+			void $ notifyDaemonStatusChange dstatus
 
 {- Adds a transfer to the queue. -}
-queueTransfer :: Schedule -> TransferQueue -> AssociatedFile -> Transfer -> Remote -> IO ()
-queueTransfer schedule q f t remote = atomically $
-	enqueue schedule q t (stubInfo f remote)
+queueTransfer :: Schedule -> TransferQueue -> DaemonStatusHandle -> AssociatedFile -> Transfer -> Remote -> IO ()
+queueTransfer schedule q dstatus f t remote =
+	enqueue schedule q dstatus t (stubInfo f remote)
 
 {- Blocks until the queue is no larger than a given size, and then adds a
  - transfer to the queue. -}
-queueTransferAt :: Integer -> Schedule -> TransferQueue -> AssociatedFile -> Transfer -> Remote -> IO ()
-queueTransferAt wantsz schedule q f t remote = atomically $ do
-	sz <- readTVar (queuesize q)
-	if sz <= wantsz
-		then enqueue schedule q t (stubInfo f remote)
-		else retry -- blocks until queuesize changes
+queueTransferAt :: Integer -> Schedule -> TransferQueue -> DaemonStatusHandle -> AssociatedFile -> Transfer -> Remote -> IO ()
+queueTransferAt wantsz schedule q dstatus f t remote = do
+	atomically $ do
+		sz <- readTVar (queuesize q)
+		if sz <= wantsz
+			then return ()
+			else retry -- blocks until queuesize changes
+	enqueue schedule q dstatus t (stubInfo f remote)
 
-{- Blocks until a pending transfer is available from the queue. -}
-getNextTransfer :: TransferQueue -> IO (Transfer, TransferInfo)
-getNextTransfer q = atomically $ do
+{- Blocks until a pending transfer is available from the queue.
+ - The transfer is removed from the transfer queue, and added to
+ - the daemon status currentTransfers map. This is done in a single STM
+ - transaction, so there is no window where an observer sees an
+ - inconsistent status. -}
+getNextTransfer :: TransferQueue -> DaemonStatusHandle -> IO (Transfer, TransferInfo)
+getNextTransfer q dstatus = atomically $ do
 	void $ modifyTVar' (queuesize q) pred
 	void $ modifyTVar' (queuelist q) (drop 1)
-	readTChan (queue q)
+	r@(t, info) <- readTChan (queue q)
+	adjustTransfersSTM dstatus $
+		M.insertWith' const t info
+	return r

From 109dc122da2c2fb1efdb83e5e361fe95fc4f1b16 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 20:30:46 -0400
Subject: [PATCH 4323/8313] add a newtype

---
 Utility/NotificationBroadcaster.hs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs
index caa13bbacf..1b05329e29 100644
--- a/Utility/NotificationBroadcaster.hs
+++ b/Utility/NotificationBroadcaster.hs
@@ -30,8 +30,13 @@ import Control.Concurrent.SampleVar
 {- One SampleVar per client. The TMVar is never empty, so never blocks. -}
 type NotificationBroadcaster = TMVar [SampleVar ()]
 
+newtype NotificationId = NotificationId Int
+
+instance Show NotificationId where
+	show (NotificationId i) = show i
+
 {- Handle given out to an individual client. -}
-data NotificationHandle = NotificationHandle NotificationBroadcaster Int
+data NotificationHandle = NotificationHandle NotificationBroadcaster NotificationId
 
 newNotificationBroadcaster :: IO NotificationBroadcaster
 newNotificationBroadcaster = atomically (newTMVar [])
@@ -47,16 +52,14 @@ newNotificationHandle b = NotificationHandle
 			atomically $ do
 				l <- readTMVar b
 				putTMVar b $ l ++ [s]
-				return $ length l
+				return $ NotificationId $ length l
 
-{- Extracts the Int identifier from a notification handle.
+{- Extracts the identifier from a notification handle.
  - This can be used to eg, pass the identifier through to a WebApp. -}
-notificationHandleToId :: NotificationHandle -> Int
+notificationHandleToId :: NotificationHandle -> NotificationId
 notificationHandleToId (NotificationHandle _ i) = i
 
-{- Given a NotificationBroadcaster, and an Int identifier, recreates the
- - NotificationHandle. -}
-notificationHandleFromId :: NotificationBroadcaster -> Int -> NotificationHandle
+notificationHandleFromId :: NotificationBroadcaster -> NotificationId -> NotificationHandle
 notificationHandleFromId = NotificationHandle
 
 {- Sends a notification to all clients. -}
@@ -70,6 +73,6 @@ sendNotification b = do
 {- Used by a client to block until a new notification is available since
  - the last time it tried. -}
 waitNotification :: NotificationHandle -> IO ()
-waitNotification (NotificationHandle b i) = do
+waitNotification (NotificationHandle b (NotificationId i)) = do
 	l <- atomically $ readTMVar b
 	readSampleVar (l !! i)

From 5be5cb219f1d277fbc7c8b0a33a9012fcd219a00 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 21:11:40 -0400
Subject: [PATCH 4324/8313] add derives needed for use with Yesod, and fix a
 bug

---
 Utility/NotificationBroadcaster.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs
index 1b05329e29..c811152ff5 100644
--- a/Utility/NotificationBroadcaster.hs
+++ b/Utility/NotificationBroadcaster.hs
@@ -14,6 +14,7 @@
 module Utility.NotificationBroadcaster (
 	NotificationBroadcaster,
 	NotificationHandle,
+	NotificationId,
 	newNotificationBroadcaster,
 	newNotificationHandle,
 	notificationHandleToId,
@@ -31,6 +32,7 @@ import Control.Concurrent.SampleVar
 type NotificationBroadcaster = TMVar [SampleVar ()]
 
 newtype NotificationId = NotificationId Int
+	deriving (Read, Eq, Ord)
 
 instance Show NotificationId where
 	show (NotificationId i) = show i
@@ -39,7 +41,7 @@ instance Show NotificationId where
 data NotificationHandle = NotificationHandle NotificationBroadcaster NotificationId
 
 newNotificationBroadcaster :: IO NotificationBroadcaster
-newNotificationBroadcaster = atomically (newTMVar [])
+newNotificationBroadcaster = atomically $ newTMVar []
 
 {- Allocates a notification handle for a client to use. -}
 newNotificationHandle :: NotificationBroadcaster -> IO NotificationHandle
@@ -50,7 +52,7 @@ newNotificationHandle b = NotificationHandle
 		addclient = do
 			s <- newEmptySampleVar
 			atomically $ do
-				l <- readTMVar b
+				l <- takeTMVar b
 				putTMVar b $ l ++ [s]
 				return $ NotificationId $ length l
 

From 6a9abf652612af149be806ba8055879141929475 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 21:21:22 -0400
Subject: [PATCH 4325/8313] add NotificationID to StatusR, and use it to block

---
 Assistant/Threads/WebApp.hs        | 27 ++++++++++++++++++++++-----
 Utility/NotificationBroadcaster.hs |  5 +----
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 6e895ccf63..430e6f50cb 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -6,6 +6,7 @@
  -}
 
 {-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
 
 module Assistant.Threads.WebApp where
 
@@ -13,6 +14,7 @@ import Assistant.Common
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.TransferQueue
+import Utility.NotificationBroadcaster
 import Utility.WebApp
 import Utility.Yesod
 import Utility.FileMode
@@ -49,11 +51,15 @@ staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
 / HomeR GET
-/status StatusR GET
+/status/#NotificationId StatusR GET
 /config ConfigR GET
 /static StaticR Static getStatic
 |]
 
+instance PathPiece NotificationId where
+    toPathPiece = pack . show
+    fromPathPiece = readish . unpack
+
 instance Yesod WebApp where
 	defaultLayout widget = do
 		mmsg <- getMessage
@@ -107,7 +113,7 @@ autoUpdate updating gethtml home ms_delay ms_startdelay = do
 		ms_to_seconds :: Int -> Int
 		ms_to_seconds ms = ceiling ((fromIntegral ms :: Double) / 1000)
 
-{- Continually updating status display. -}
+{- A dynamically updating status display. -}
 statusDisplay :: Widget
 statusDisplay = do
 	webapp <- lift getYesod
@@ -122,7 +128,13 @@ statusDisplay = do
 	updating <- lift newIdent
 	$(widgetFile "status")
 	
-	autoUpdate updating StatusR HomeR (3000 :: Int) (40 :: Int)
+	nid <- liftIO $ notificationHandleToId <$>
+		(newNotificationHandle =<< getNotificationBroadcaster webapp)
+	autoUpdate updating (StatusR nid) HomeR (3000 :: Int) (40 :: Int)
+
+getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
+getNotificationBroadcaster webapp = notificationBroadcaster
+		<$> getDaemonStatus (daemonStatus webapp)
 
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout statusDisplay
@@ -136,8 +148,13 @@ getHomeR = defaultLayout statusDisplay
  - body is. To get the widget head content, the widget is also 
  - inserted onto the getHomeR page.
  -}
-getStatusR :: Handler RepHtml
-getStatusR = do
+getStatusR :: NotificationId -> Handler RepHtml
+getStatusR nid = do
+	{- Block until there is an updated status to display. -}
+	webapp <- getYesod
+	b <- liftIO $ getNotificationBroadcaster webapp
+	liftIO $ waitNotification $ notificationHandleFromId b nid
+
 	page <- widgetToPageContent statusDisplay
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
diff --git a/Utility/NotificationBroadcaster.hs b/Utility/NotificationBroadcaster.hs
index c811152ff5..accc35fe18 100644
--- a/Utility/NotificationBroadcaster.hs
+++ b/Utility/NotificationBroadcaster.hs
@@ -32,10 +32,7 @@ import Control.Concurrent.SampleVar
 type NotificationBroadcaster = TMVar [SampleVar ()]
 
 newtype NotificationId = NotificationId Int
-	deriving (Read, Eq, Ord)
-
-instance Show NotificationId where
-	show (NotificationId i) = show i
+	deriving (Read, Show, Eq, Ord)
 
 {- Handle given out to an individual client. -}
 data NotificationHandle = NotificationHandle NotificationBroadcaster NotificationId

From 9b18dc2a394560d6a6f39b61e1155b8bb512caec Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 21:25:56 -0400
Subject: [PATCH 4326/8313] tune javascript refresh delays

WebApp now shows changes with no delay. Comparing a running git-annex get
and the webapp side-by-side, they both show each new transfer at the same
time.
---
 Assistant/Threads/WebApp.hs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 430e6f50cb..28d1a48827 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -95,14 +95,14 @@ instance Yesod WebApp where
  - Or, the home route is used if the whole page has to be refreshed to
  - update.
  -
- - ms_delay is how long to delay between updates.
- - ms_startdelay is how long to delay before updating the widget at the
- - state.
+ - ms_delay is how long to delay between AJAX updates
+ - ms_startdelay is how long to delay before updating with AJAX at the start
+ - ms_refreshdelay is how long to delay between refreshes, when not using AJAX
  -}
-autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Widget
-autoUpdate updating gethtml home ms_delay ms_startdelay = do
+autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Int -> Widget
+autoUpdate updating gethtml home ms_delay ms_startdelay ms_refreshdelay = do
 	{- Fallback refreshing is provided for non-javascript browsers. -}
-	let delayseconds = show $ ms_to_seconds ms_delay
+	let delayseconds = show $ ms_to_seconds ms_refreshdelay
 	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
 
 	{- Use long polling to update the status display. -}
@@ -130,7 +130,7 @@ statusDisplay = do
 	
 	nid <- liftIO $ notificationHandleToId <$>
 		(newNotificationHandle =<< getNotificationBroadcaster webapp)
-	autoUpdate updating (StatusR nid) HomeR (3000 :: Int) (40 :: Int)
+	autoUpdate updating (StatusR nid) HomeR (10 :: Int) (10 :: Int) (3000 :: Int)
 
 getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
 getNotificationBroadcaster webapp = notificationBroadcaster

From 238641261c72a6850a067d970163c9213fe3efef Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 21:53:04 -0400
Subject: [PATCH 4327/8313] blog for the day

Wow, what a nice productive day this was!
---
 .../blog/day_46__notification_pools.mdwn      | 68 +++++++++++++++++++
 doc/design/assistant/webapp.mdwn              |  2 +-
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 doc/design/assistant/blog/day_46__notification_pools.mdwn

diff --git a/doc/design/assistant/blog/day_46__notification_pools.mdwn b/doc/design/assistant/blog/day_46__notification_pools.mdwn
new file mode 100644
index 0000000000..d6283ddba2
--- /dev/null
+++ b/doc/design/assistant/blog/day_46__notification_pools.mdwn
@@ -0,0 +1,68 @@
+Focus today was writing a notification broadcaster library. This is a way to
+send a notification to a set of clients, any of which can be blocked
+waiting for a new notification to arrive. A complication is that any number
+of clients may be be dead, and we don't want stale notifications for those
+clients to pile up and leak memory.
+
+It took me 3 tries to find the solution, which turns out to be head-smackingly
+simple: An array of SampleVars, one per client.
+
+Using SampleVars means that clients only see the most recent notification,
+but when the notification is just "the assistant's state changed somehow;
+display a refreshed rendering of it", that's sufficient.
+
+----
+
+First use of that was to make the thread that woke up every 10 minutes
+and checkpointed the daemon status to disk also wait for a notification
+that it changed. So that'll be more current, and use less IO.
+
+----
+
+Second use, of course, was to make the WebApp block long polling clients
+until there is really a change since the last time the client polled.
+
+To do that, I made one change to my Yesod routes:
+
+[[!format diff """
+	-/status StatusR GET
+	+/status/#NotificationId StatusR GET
+"""]]
+
+Now I find another reason to love Yesod, because after doing that,
+I hit "make".. and fixed the type error. And hit make.. and fixed
+the type error. And then it just freaking worked! Html was generated with
+all urls to /status including a `NotificationId`, and the handler for
+that route got it and was able to use it:
+
+[[!format haskell """
+	{- Block until there is an updated status to display. -}
+	b <- liftIO $ getNotificationBroadcaster webapp
+	liftIO $ waitNotification $ notificationHandleFromId b nid
+""]]
+
+And now the WebApp is able to display transfers in realtime!
+When I have both the WebApp and `git annex get` running on the same screen,
+the WebApp displays files that git-annex is transferring about as fast
+as the terminal updates.
+
+The [[progressbars]] still need to be sorted out, but otherwise
+the WebApp is a nice live view of file transfers.
+
+---
+
+I also had some fun with Software Transactional Memory. Now when the
+assistant moves a transfer from its queue of transfers to do, to its map of
+transfers that are currently running, it does so in an atomic transaction.
+This will avoid the transfer seeming to go missing (or be listed twice) if
+the webapp refreshes at just the wrong point in time. I'm really starting
+to get into STM.
+
+----
+
+Next up, I will be making the WebApp maintain a list of notices, displayed
+on its sidebar, scrolling new notices into view, and removing ones the user
+closes, and ones that expire. This will be used for displaying errors, as
+well as other communication with the user (such as displaying a notice
+while a git sync is in progress with a remote, etc). Seems worth doing now,
+so the basic UI of the WebApp is complete with no placeholders.
diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 018d70886b..1fc32282a2 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -16,7 +16,7 @@ The webapp is a web server that displays a shiny interface.
 
 ## interface
 
-* list of files uploading and downloading
+* list of files uploading and downloading **done**
 * progress bars for each file
 * drag and drop to reorder
 * cancel and pause

From ff9aeda585c2e9ad80c1f6a4e74e46fe804e5dbd Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 21:56:37 -0400
Subject: [PATCH 4328/8313] format

---
 doc/design/assistant/blog/day_46__notification_pools.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_46__notification_pools.mdwn b/doc/design/assistant/blog/day_46__notification_pools.mdwn
index d6283ddba2..483ad95b1c 100644
--- a/doc/design/assistant/blog/day_46__notification_pools.mdwn
+++ b/doc/design/assistant/blog/day_46__notification_pools.mdwn
@@ -39,7 +39,7 @@ that route got it and was able to use it:
 	{- Block until there is an updated status to display. -}
 	b <- liftIO $ getNotificationBroadcaster webapp
 	liftIO $ waitNotification $ notificationHandleFromId b nid
-""]]
+"""]]
 
 And now the WebApp is able to display transfers in realtime!
 When I have both the WebApp and `git annex get` running on the same screen,

From a498be7f98927370ad29221a170530a6de01b928 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sat, 28 Jul 2012 23:55:41 -0400
Subject: [PATCH 4329/8313] renamed /status to /transfers

Also fixed a bug; the ident for the div was regnerated each time
/status was called. This only was the same as the original ident due to
luck.
---
 Assistant/Threads/WebApp.hs                   | 49 ++++++++++---------
 git-annex.cabal                               |  2 +-
 templates/longpolling.julius                  |  4 +-
 templates/{status.hamlet => transfers.hamlet} |  5 +-
 4 files changed, 29 insertions(+), 31 deletions(-)
 rename templates/{status.hamlet => transfers.hamlet} (94%)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 28d1a48827..4da48ae046 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -32,7 +32,6 @@ import Text.Hamlet
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text (Text, pack, unpack)
-import Data.Time.Clock
 import qualified Data.Map as M
 
 thisThread :: String
@@ -51,7 +50,7 @@ staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
 / HomeR GET
-/status/#NotificationId StatusR GET
+/transfers/#NotificationId TransfersR GET
 /config ConfigR GET
 /static StaticR Static getStatic
 |]
@@ -88,7 +87,7 @@ instance Yesod WebApp where
 
 {- Add to any widget to make it auto-update.
  -
- - The widget should have a html element with id=updating, which will be
+ - The widget should have a html element with an id=ident, which will be
  - replaced when it's updated.
  - 
  - Updating is done by getting html from the gethtml route.
@@ -100,12 +99,12 @@ instance Yesod WebApp where
  - ms_refreshdelay is how long to delay between refreshes, when not using AJAX
  -}
 autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Int -> Widget
-autoUpdate updating gethtml home ms_delay ms_startdelay ms_refreshdelay = do
+autoUpdate ident gethtml home ms_delay ms_startdelay ms_refreshdelay = do
 	{- Fallback refreshing is provided for non-javascript browsers. -}
 	let delayseconds = show $ ms_to_seconds ms_refreshdelay
 	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
 
-	{- Use long polling to update the status display. -}
+	{- Use long polling to update the transfers display. -}
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
 	$(widgetFile "longpolling")
@@ -113,49 +112,51 @@ autoUpdate updating gethtml home ms_delay ms_startdelay ms_refreshdelay = do
 		ms_to_seconds :: Int -> Int
 		ms_to_seconds ms = ceiling ((fromIntegral ms :: Double) / 1000)
 
-{- A dynamically updating status display. -}
-statusDisplay :: Widget
-statusDisplay = do
+{- A display of currently running and queued transfers. -}
+transfersDisplay :: Widget
+transfersDisplay = do
 	webapp <- lift getYesod
-	time <- show <$> liftIO getCurrentTime
-	
 	current <- liftIO $ runThreadState (threadState webapp) $
 		M.toList . currentTransfers
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
 	queued <- liftIO $ getTransferQueue $ transferQueue webapp
 	let transfers = current ++ queued
+	let ident = transfersDisplayIdent
+	$(widgetFile "transfers")
 
-	updating <- lift newIdent
-	$(widgetFile "status")
-	
-	nid <- liftIO $ notificationHandleToId <$>
-		(newNotificationHandle =<< getNotificationBroadcaster webapp)
-	autoUpdate updating (StatusR nid) HomeR (10 :: Int) (10 :: Int) (3000 :: Int)
+transfersDisplayIdent :: Text
+transfersDisplayIdent = "transfers"
 
 getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
 getNotificationBroadcaster webapp = notificationBroadcaster
 		<$> getDaemonStatus (daemonStatus webapp)
 
 getHomeR :: Handler RepHtml
-getHomeR = defaultLayout statusDisplay
+getHomeR = defaultLayout $ do
+	{- Set up automatic updates for the transfers display. -}
+	webapp <- lift getYesod
+	nid <- liftIO $ notificationHandleToId <$>
+		(newNotificationHandle =<< getNotificationBroadcaster webapp)
+	autoUpdate transfersDisplayIdent (TransfersR nid) HomeR
+		(10 :: Int) (10 :: Int) (3000 :: Int)
+	transfersDisplay
 
-{- Called by client to poll for a new webapp status display.
+{- Called by client to get a display of currently in process transfers.
  -
- - Should block until the status has changed, and then return a div
- - containing the new status, which will be inserted into the calling page.
+ - Returns a div, which will be inserted into the calling page.
  -
  - Note that the head of the widget is not included, only its
  - body is. To get the widget head content, the widget is also 
  - inserted onto the getHomeR page.
  -}
-getStatusR :: NotificationId -> Handler RepHtml
-getStatusR nid = do
-	{- Block until there is an updated status to display. -}
+getTransfersR :: NotificationId -> Handler RepHtml
+getTransfersR nid = do
+	{- Block until there is a change from last time. -}
 	webapp <- getYesod
 	b <- liftIO $ getNotificationBroadcaster webapp
 	liftIO $ waitNotification $ notificationHandleFromId b nid
 
-	page <- widgetToPageContent statusDisplay
+	page <- widgetToPageContent transfersDisplay
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
 getConfigR :: Handler RepHtml
diff --git a/git-annex.cabal b/git-annex.cabal
index 24e0df9c99..afa8814253 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,5 +1,5 @@
 Name: git-annex
-Version: 3.20120721
+Version: 3.20120722
 Cabal-Version: >= 1.8
 License: GPL
 Maintainer: Joey Hess <joey@kitenet.net>
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index eff8d3f444..945ef1251f 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,5 +1,5 @@
 
-// Uses long-polling to update a div with id=#{updating}
+// Uses long-polling to update a div with id=#{ident}
 // The gethtml route should return a new div, with the same id.
 //
 // Maximum update frequency is controlled by #{startdelay}
@@ -16,7 +16,7 @@ $.LongPoll = (function() {
 				'url': '@{gethtml}',
 				'dataType': 'html',
 				'success': function(data, status, jqxhr) {
-					$('##{updating}').replaceWith(data);
+					$('##{ident}').replaceWith(data);
 					setTimeout($.LongPoll.send, #{delay});
 					numerrs=0;
 				},
diff --git a/templates/status.hamlet b/templates/transfers.hamlet
similarity index 94%
rename from templates/status.hamlet
rename to templates/transfers.hamlet
index 2ccea1f1af..154e8d58b3 100644
--- a/templates/status.hamlet
+++ b/templates/transfers.hamlet
@@ -1,4 +1,4 @@
-<span ##{updating}>
+<span ##{ident}>
   <div .span9>
     $if null transfers
       <h2>No current transfers
@@ -25,6 +25,3 @@
                       <small .pull-right>queued (#{size})</small>
           <div .progress .progress-striped>
             <div .bar style="width: #{percent};">
-  <footer>
-    <span>
-      polled at #{time}

From 376f8443c1786a1acbaaf24fc7c4f8a662f0ef38 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 00:08:14 -0400
Subject: [PATCH 4330/8313] add a separate page for noscript browsers

This may be customised differently than the main page later on, but
for now the important thing is that this constantly refreshed page does not
allocate a new NotificationHandle each time it's loaded.
---
 Assistant/Threads/WebApp.hs  | 18 ++++++++++++++++--
 templates/metarefresh.hamlet |  2 +-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 4da48ae046..d9d98e1bf3 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -50,6 +50,7 @@ staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
 / HomeR GET
+/noscript NoScriptR GET
 /transfers/#NotificationId TransfersR GET
 /config ConfigR GET
 /static StaticR Static getStatic
@@ -101,7 +102,7 @@ instance Yesod WebApp where
 autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Int -> Widget
 autoUpdate ident gethtml home ms_delay ms_startdelay ms_refreshdelay = do
 	{- Fallback refreshing is provided for non-javascript browsers. -}
-	let delayseconds = show $ ms_to_seconds ms_refreshdelay
+	let delayseconds = ms_to_seconds ms_refreshdelay
 	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
 
 	{- Use long polling to update the transfers display. -}
@@ -131,6 +132,9 @@ getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
 getNotificationBroadcaster webapp = notificationBroadcaster
 		<$> getDaemonStatus (daemonStatus webapp)
 
+dashboard :: Widget
+dashboard = transfersDisplay
+
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
 	{- Set up automatic updates for the transfers display. -}
@@ -139,7 +143,17 @@ getHomeR = defaultLayout $ do
 		(newNotificationHandle =<< getNotificationBroadcaster webapp)
 	autoUpdate transfersDisplayIdent (TransfersR nid) HomeR
 		(10 :: Int) (10 :: Int) (3000 :: Int)
-	transfersDisplay
+	
+	dashboard
+
+{- Same as HomeR, except with no javascript, so it doesn't allocate
+ - new resources each time the page is refreshed. -}
+getNoScriptR :: Handler RepHtml
+getNoScriptR = defaultLayout $ do
+	let ident = NoScriptR
+	let delayseconds = 3 :: Int
+	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
+	dashboard
 
 {- Called by client to get a display of currently in process transfers.
  -
diff --git a/templates/metarefresh.hamlet b/templates/metarefresh.hamlet
index be22aa8992..ddbd225fe2 100644
--- a/templates/metarefresh.hamlet
+++ b/templates/metarefresh.hamlet
@@ -1,2 +1,2 @@
 <noscript>
-  <meta http-equiv="refresh" content="#{delayseconds}; URL=@{home}">
+  <meta http-equiv="refresh" content="#{show delayseconds}; URL=@{NoScriptR}">

From 38ade1af70a08d278a56bcec4f7a9e32b09f4336 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 00:55:22 -0400
Subject: [PATCH 4331/8313] better noscript UI

---
 Assistant/Threads/WebApp.hs  | 44 +++++++++++++++++-------------------
 templates/longpolling.julius |  6 ++---
 templates/metarefresh.hamlet |  2 +-
 templates/transfers.hamlet   | 13 +++++++++++
 4 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index d9d98e1bf3..e47ee9fdac 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -51,6 +51,7 @@ staticFiles "static"
 mkYesod "WebApp" [parseRoutes|
 / HomeR GET
 /noscript NoScriptR GET
+/noscriptauto NoScriptAutoR GET
 /transfers/#NotificationId TransfersR GET
 /config ConfigR GET
 /static StaticR Static getStatic
@@ -86,7 +87,7 @@ instance Yesod WebApp where
 	makeSessionBackend = webAppSessionBackend
 	jsLoader _ = BottomOfHeadBlocking
 
-{- Add to any widget to make it auto-update.
+{- Add to any widget to make it auto-update using long polling.
  -
  - The widget should have a html element with an id=ident, which will be
  - replaced when it's updated.
@@ -97,25 +98,16 @@ instance Yesod WebApp where
  -
  - ms_delay is how long to delay between AJAX updates
  - ms_startdelay is how long to delay before updating with AJAX at the start
- - ms_refreshdelay is how long to delay between refreshes, when not using AJAX
  -}
-autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Int -> Widget
-autoUpdate ident gethtml home ms_delay ms_startdelay ms_refreshdelay = do
-	{- Fallback refreshing is provided for non-javascript browsers. -}
-	let delayseconds = ms_to_seconds ms_refreshdelay
-	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
-
-	{- Use long polling to update the transfers display. -}
+autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Widget
+autoUpdate ident gethtml home ms_delay ms_startdelay = do
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
 	$(widgetFile "longpolling")
-	where
-		ms_to_seconds :: Int -> Int
-		ms_to_seconds ms = ceiling ((fromIntegral ms :: Double) / 1000)
 
 {- A display of currently running and queued transfers. -}
-transfersDisplay :: Widget
-transfersDisplay = do
+transfersDisplay :: Bool -> Widget
+transfersDisplay warnNoScript = do
 	webapp <- lift getYesod
 	current <- liftIO $ runThreadState (threadState webapp) $
 		M.toList . currentTransfers
@@ -132,8 +124,8 @@ getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
 getNotificationBroadcaster webapp = notificationBroadcaster
 		<$> getDaemonStatus (daemonStatus webapp)
 
-dashboard :: Widget
-dashboard = transfersDisplay
+dashboard :: Bool -> Widget
+dashboard warnNoScript = transfersDisplay warnNoScript
 
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
@@ -142,18 +134,24 @@ getHomeR = defaultLayout $ do
 	nid <- liftIO $ notificationHandleToId <$>
 		(newNotificationHandle =<< getNotificationBroadcaster webapp)
 	autoUpdate transfersDisplayIdent (TransfersR nid) HomeR
-		(10 :: Int) (10 :: Int) (3000 :: Int)
+		(10 :: Int) (10 :: Int)
 	
-	dashboard
+	dashboard True
 
 {- Same as HomeR, except with no javascript, so it doesn't allocate
- - new resources each time the page is refreshed. -}
-getNoScriptR :: Handler RepHtml
-getNoScriptR = defaultLayout $ do
+ - new resources each time the page is refreshed, and with autorefreshing
+ - via meta refresh. -}
+getNoScriptAutoR :: Handler RepHtml
+getNoScriptAutoR = defaultLayout $ do
 	let ident = NoScriptR
 	let delayseconds = 3 :: Int
+	let this = NoScriptAutoR
 	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
-	dashboard
+	dashboard False
+
+getNoScriptR :: Handler RepHtml
+getNoScriptR = defaultLayout $
+	dashboard True
 
 {- Called by client to get a display of currently in process transfers.
  -
@@ -170,7 +168,7 @@ getTransfersR nid = do
 	b <- liftIO $ getNotificationBroadcaster webapp
 	liftIO $ waitNotification $ notificationHandleFromId b nid
 
-	page <- widgetToPageContent transfersDisplay
+	page <- widgetToPageContent $ transfersDisplay False
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
 getConfigR :: Handler RepHtml
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 945ef1251f..926249a35d 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -17,7 +17,7 @@ $.LongPoll = (function() {
 				'dataType': 'html',
 				'success': function(data, status, jqxhr) {
 					$('##{ident}').replaceWith(data);
-					setTimeout($.LongPoll.send, #{delay});
+					setTimeout($.LongPoll.send, #{show delay});
 					numerrs=0;
 				},
 				'error': function(jqxhr, msg, e) {
@@ -26,7 +26,7 @@ $.LongPoll = (function() {
 						window.close();
 					}
 					else {
-						setTimeout($.LongPoll.send, #{delay});
+						setTimeout($.LongPoll.send, #{show delay});
 					}
 				},
 			});
@@ -35,7 +35,7 @@ $.LongPoll = (function() {
 }());
 
 $(document).bind('ready.app', function() {
-	setTimeout($.LongPoll.send, #{startdelay});
+	setTimeout($.LongPoll.send, #{show startdelay});
 });
 
 })( jQuery );
diff --git a/templates/metarefresh.hamlet b/templates/metarefresh.hamlet
index ddbd225fe2..89a2e0b2c6 100644
--- a/templates/metarefresh.hamlet
+++ b/templates/metarefresh.hamlet
@@ -1,2 +1,2 @@
 <noscript>
-  <meta http-equiv="refresh" content="#{show delayseconds}; URL=@{NoScriptR}">
+  <meta http-equiv="refresh" content="#{show delayseconds}; URL=@{this}">
diff --git a/templates/transfers.hamlet b/templates/transfers.hamlet
index 154e8d58b3..5df4533154 100644
--- a/templates/transfers.hamlet
+++ b/templates/transfers.hamlet
@@ -1,4 +1,17 @@
 <span ##{ident}>
+  $if warnNoScript
+    <noscript>
+      <div .alert .alert-block>
+        <h4 .alert-heading>Javascript is disabled
+        <p>
+          This display cannot update in real-time without Javascript. #
+          Can you turn it on?
+        <p>
+          Otherwise, there are two options:
+        <p>
+          <div .btn-group>
+            <a .btn href="@{NoScriptAutoR}">Auto-refresh every 3 seconds #
+            <a .btn href="@{NoScriptR}">Manually refresh
   <div .span9>
     $if null transfers
       <h2>No current transfers

From e96107caf3745d658a36e0ad7716dd07a57657a2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 01:09:29 -0400
Subject: [PATCH 4332/8313] move noscript UI to bottom navbar

---
 templates/transfers.hamlet | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/templates/transfers.hamlet b/templates/transfers.hamlet
index 5df4533154..417ba3d62a 100644
--- a/templates/transfers.hamlet
+++ b/templates/transfers.hamlet
@@ -1,17 +1,4 @@
 <span ##{ident}>
-  $if warnNoScript
-    <noscript>
-      <div .alert .alert-block>
-        <h4 .alert-heading>Javascript is disabled
-        <p>
-          This display cannot update in real-time without Javascript. #
-          Can you turn it on?
-        <p>
-          Otherwise, there are two options:
-        <p>
-          <div .btn-group>
-            <a .btn href="@{NoScriptAutoR}">Auto-refresh every 3 seconds #
-            <a .btn href="@{NoScriptR}">Manually refresh
   <div .span9>
     $if null transfers
       <h2>No current transfers
@@ -38,3 +25,12 @@
                       <small .pull-right>queued (#{size})</small>
           <div .progress .progress-striped>
             <div .bar style="width: #{percent};">
+  $if warnNoScript
+    <noscript>
+      <div .navbar .navbar-fixed-bottom>
+        <div .navbar-inner>
+          <div .container>
+            Javascript is disabled; cannot update in real-time.
+            <div .btn-group>
+              <a .btn .btn-primary href="@{NoScriptAutoR}">Auto-refresh every 3 seconds #
+              <a .btn .btn-primary href="@{NoScriptR}">Manually refresh

From 62dac858807da8fb62ce55adbed84cfe582367b2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 03:23:17 -0400
Subject: [PATCH 4333/8313] update the sidebar by long polling

Needs to use a different NotificationBroadcaster, and not replace the
whole sidebar div, but instead add in new content. However, it's 3:30 am.
---
 Assistant/Threads/WebApp.hs  | 110 ++++++++++++++++++++++-------------
 templates/longpolling.julius |   8 +--
 templates/page.hamlet        |  21 +------
 templates/sidebar.hamlet     |  18 ++++++
 templates/transfers.hamlet   |  53 +++++++++--------
 5 files changed, 120 insertions(+), 90 deletions(-)
 create mode 100644 templates/sidebar.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index e47ee9fdac..500297693b 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -33,6 +33,7 @@ import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text (Text, pack, unpack)
 import qualified Data.Map as M
+import Data.Time.Clock
 
 thisThread :: String
 thisThread = "WebApp"
@@ -46,6 +47,10 @@ data WebApp = WebApp
 	, getStatic :: Static
 	}
 
+getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
+getNotificationBroadcaster webapp = notificationBroadcaster
+		<$> getDaemonStatus (daemonStatus webapp)
+
 staticFiles "static"
 
 mkYesod "WebApp" [parseRoutes|
@@ -53,6 +58,7 @@ mkYesod "WebApp" [parseRoutes|
 /noscript NoScriptR GET
 /noscriptauto NoScriptAutoR GET
 /transfers/#NotificationId TransfersR GET
+/sidebar/#NotificationId SideBarR GET
 /config ConfigR GET
 /static StaticR Static getStatic
 |]
@@ -62,8 +68,7 @@ instance PathPiece NotificationId where
     fromPathPiece = readish . unpack
 
 instance Yesod WebApp where
-	defaultLayout widget = do
-		mmsg <- getMessage
+	defaultLayout content = do
 		webapp <- getYesod
 		page <- widgetToPageContent $ do
 			addStylesheet $ StaticR css_bootstrap_css
@@ -93,14 +98,12 @@ instance Yesod WebApp where
  - replaced when it's updated.
  - 
  - Updating is done by getting html from the gethtml route.
- - Or, the home route is used if the whole page has to be refreshed to
- - update.
  -
  - ms_delay is how long to delay between AJAX updates
  - ms_startdelay is how long to delay before updating with AJAX at the start
  -}
-autoUpdate :: Text -> Route WebApp -> Route WebApp -> Int -> Int -> Widget
-autoUpdate ident gethtml home ms_delay ms_startdelay = do
+autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget
+autoUpdate ident gethtml ms_delay ms_startdelay = do
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
 	$(widgetFile "longpolling")
@@ -120,39 +123,6 @@ transfersDisplay warnNoScript = do
 transfersDisplayIdent :: Text
 transfersDisplayIdent = "transfers"
 
-getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
-getNotificationBroadcaster webapp = notificationBroadcaster
-		<$> getDaemonStatus (daemonStatus webapp)
-
-dashboard :: Bool -> Widget
-dashboard warnNoScript = transfersDisplay warnNoScript
-
-getHomeR :: Handler RepHtml
-getHomeR = defaultLayout $ do
-	{- Set up automatic updates for the transfers display. -}
-	webapp <- lift getYesod
-	nid <- liftIO $ notificationHandleToId <$>
-		(newNotificationHandle =<< getNotificationBroadcaster webapp)
-	autoUpdate transfersDisplayIdent (TransfersR nid) HomeR
-		(10 :: Int) (10 :: Int)
-	
-	dashboard True
-
-{- Same as HomeR, except with no javascript, so it doesn't allocate
- - new resources each time the page is refreshed, and with autorefreshing
- - via meta refresh. -}
-getNoScriptAutoR :: Handler RepHtml
-getNoScriptAutoR = defaultLayout $ do
-	let ident = NoScriptR
-	let delayseconds = 3 :: Int
-	let this = NoScriptAutoR
-	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
-	dashboard False
-
-getNoScriptR :: Handler RepHtml
-getNoScriptR = defaultLayout $
-	dashboard True
-
 {- Called by client to get a display of currently in process transfers.
  -
  - Returns a div, which will be inserted into the calling page.
@@ -171,8 +141,70 @@ getTransfersR nid = do
 	page <- widgetToPageContent $ transfersDisplay False
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
+sideBarDisplay :: Bool -> Widget
+sideBarDisplay noScript = do
+	date <- liftIO $ show <$> getCurrentTime
+	ident <- lift newIdent
+	mmsg <- lift getMessage
+	$(widgetFile "sidebar")
+	unless noScript $ do
+		{- Set up automatic updates of the sidebar. -}
+		webapp <- lift getYesod
+		nid <- liftIO $ notificationHandleToId <$>
+			(newNotificationHandle =<< getNotificationBroadcaster webapp)
+		autoUpdate ident (SideBarR nid) (10 :: Int) (10 :: Int)
+
+{- Called by client to get a sidebar display.
+ -
+ - Returns a div, which will be inserted into the calling page.
+ -
+ - Note that the head of the widget is not included, only its
+ - body is. To get the widget head content, the widget is also 
+ - inserted onto all pages.
+ -}
+getSideBarR :: NotificationId -> Handler RepHtml
+getSideBarR nid = do
+	{- Block until there is a change from last time. -}
+	webapp <- getYesod
+	b <- liftIO $ getNotificationBroadcaster webapp
+	liftIO $ waitNotification $ notificationHandleFromId b nid
+
+	page <- widgetToPageContent $ sideBarDisplay True
+	hamletToRepHtml $ [hamlet|^{pageBody page}|]
+
+dashboard :: Bool -> Bool -> Widget
+dashboard noScript warnNoScript = do
+	sideBarDisplay noScript
+	transfersDisplay warnNoScript
+
+getHomeR :: Handler RepHtml
+getHomeR = defaultLayout $ do
+	{- Set up automatic updates for the transfers display. -}
+	webapp <- lift getYesod
+	nid <- liftIO $ notificationHandleToId <$>
+		(newNotificationHandle =<< getNotificationBroadcaster webapp)
+	autoUpdate transfersDisplayIdent (TransfersR nid) (10 :: Int) (10 :: Int)
+	
+	dashboard False True
+
+{- Same as HomeR, except with no javascript, so it doesn't allocate
+ - new resources each time the page is refreshed, and with autorefreshing
+ - via meta refresh. -}
+getNoScriptAutoR :: Handler RepHtml
+getNoScriptAutoR = defaultLayout $ do
+	let ident = NoScriptR
+	let delayseconds = 3 :: Int
+	let this = NoScriptAutoR
+	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
+	dashboard True False
+
+getNoScriptR :: Handler RepHtml
+getNoScriptR = defaultLayout $
+	dashboard True True
+
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout $ do
+	sideBarDisplay False
 	setTitle "configuration"
 	[whamlet|<a href="@{HomeR}">main|]
 
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 926249a35d..95425d615a 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -9,7 +9,7 @@
 
 numerrs=0;
 
-$.LongPoll = (function() {
+$.LongPoll#{ident} = (function() {
 	return {
 		send : function() {           
 			$.ajax({
@@ -17,7 +17,7 @@ $.LongPoll = (function() {
 				'dataType': 'html',
 				'success': function(data, status, jqxhr) {
 					$('##{ident}').replaceWith(data);
-					setTimeout($.LongPoll.send, #{show delay});
+					setTimeout($.LongPoll#{ident}.send, #{show delay});
 					numerrs=0;
 				},
 				'error': function(jqxhr, msg, e) {
@@ -26,7 +26,7 @@ $.LongPoll = (function() {
 						window.close();
 					}
 					else {
-						setTimeout($.LongPoll.send, #{show delay});
+						setTimeout($.LongPoll#{ident}.send, #{show delay});
 					}
 				},
 			});
@@ -35,7 +35,7 @@ $.LongPoll = (function() {
 }());
 
 $(document).bind('ready.app', function() {
-	setTimeout($.LongPoll.send, #{show startdelay});
+	setTimeout($.LongPoll#{ident}.send, #{show startdelay});
 });
 
 })( jQuery );
diff --git a/templates/page.hamlet b/templates/page.hamlet
index ae80bb05d5..c397d248c2 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -20,23 +20,4 @@
 
 <div .container-fluid>
   <div .row-fluid>
-    <div .span3>
-      <div .sidebar-nav>
-        <div .alert .alert-info>
-          <a .close data-dismiss="alert" href="#">×</a>
-          <b>This is just a demo.</b> If this were not just a demo,
-          I'd not be filling this sidebar with silly alerts.
-        <div .alert .alert-success>
-          <a .close data-dismiss="alert" href="#">×</a>
-          <b>Well done!</b>
-          You successfully read this important alert message.
-        <div .alert .alert-error>
-          <a .close data-dismiss="alert" href="#">×</a>
-          <b>Whoops!</b>
-          Unable to connect to blah blah..
-    <div .span9>
-      $maybe msg <- mmsg
-        <div .alert .alert-info>
-          <a .close data-dismiss="alert" href="#">×</a>
-          #{msg}
-      ^{widget}
+    ^{content}
diff --git a/templates/sidebar.hamlet b/templates/sidebar.hamlet
new file mode 100644
index 0000000000..3b5048151b
--- /dev/null
+++ b/templates/sidebar.hamlet
@@ -0,0 +1,18 @@
+<div .span3 ##{ident}>
+  <div .sidebar-nav>
+    $maybe msg <- mmsg
+      <div .alert .alert-info>
+        <a .close data-dismiss="alert" href="#">×</a>
+          #{msg}
+    <div .alert .alert-info>
+      <a .close data-dismiss="alert" href="#">×</a>
+      <b>This is just a demo.</b> If this were not just a demo,
+      I'd not be filling this sidebar with silly alerts.
+    <div .alert .alert-success>
+      <a .close data-dismiss="alert" href="#">×</a>
+      <b>Well done!</b>
+      You successfully read this important alert message.
+    <div .alert .alert-error>
+      <a .close data-dismiss="alert" href="#">×</a>
+      <b>Whoops!</b>
+      Unable to connect to blah blah.. #{date}
diff --git a/templates/transfers.hamlet b/templates/transfers.hamlet
index 417ba3d62a..bc69d7f876 100644
--- a/templates/transfers.hamlet
+++ b/templates/transfers.hamlet
@@ -1,30 +1,29 @@
-<span ##{ident}>
-  <div .span9>
-    $if null transfers
-      <h2>No current transfers
-    $else
-      <h2>Transfers
-      $forall (transfer, info) <- transfers
-        $with percent <- maybe "unknown" (showPercentage 0) $ percentComplete transfer info
-          <div .row-fluid>
-            <h3>
-              $maybe file <- associatedFile info
-                #{file}
-              $nothing
-                #{show $ transferKey transfer}
-              $case transferDirection transfer
-                $of Upload
-                  →
-                $of Download
-                  ←
-              <small>#{maybe "unknown" Remote.name $ transferRemote info}</small>
-                $with size <- maybe "unknown" (roughSize dataUnits True) $ keySize $ transferKey transfer
-                  $if isJust $ startedTime info
-                      <small .pull-right><b>#{percent} of #{size}</b></small>
-                  $else
-                      <small .pull-right>queued (#{size})</small>
-          <div .progress .progress-striped>
-            <div .bar style="width: #{percent};">
+<div .span9 ##{ident}>
+  $if null transfers
+    <h2>No current transfers
+  $else
+    <h2>Transfers
+    $forall (transfer, info) <- transfers
+      $with percent <- maybe "unknown" (showPercentage 0) $ percentComplete transfer info
+        <div .row-fluid>
+          <h3>
+            $maybe file <- associatedFile info
+              #{file}
+            $nothing
+              #{show $ transferKey transfer}
+            $case transferDirection transfer
+              $of Upload
+                →
+              $of Download
+                ←
+            <small>#{maybe "unknown" Remote.name $ transferRemote info}</small>
+              $with size <- maybe "unknown" (roughSize dataUnits True) $ keySize $ transferKey transfer
+                $if isJust $ startedTime info
+                    <small .pull-right><b>#{percent} of #{size}</b></small>
+                $else
+                    <small .pull-right>queued (#{size})</small>
+        <div .progress .progress-striped>
+          <div .bar style="width: #{percent};">
   $if warnNoScript
     <noscript>
       <div .navbar .navbar-fixed-bottom>

From 69c46c1004e70cc159dd23bdacacc0727a621acf Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Sun, 29 Jul 2012 10:49:05 +0000
Subject: [PATCH 4334/8313] Added a comment

---
 .../comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment

diff --git a/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment
new file mode 100644
index 0000000000..952ffadc4d
--- /dev/null
+++ b/doc/bugs/Watch_command_as_of_commit_6cecc26206c4a539999b04664136c6f785211a41_segfaults/comment_10_6c872dff4fcc63c16bf69d1e96891c89._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jtang"
+ ip="79.97.135.214"
+ subject="comment 10"
+ date="2012-07-29T10:49:05Z"
+ content="""
+This is looking good, no more segfaulting.
+"""]]

From fb2ef6090a613b6b76d5a91477e1bcf682aee1cd Mon Sep 17 00:00:00 2001
From: jtang <jtang@web>
Date: Sun, 29 Jul 2012 11:20:00 +0000
Subject: [PATCH 4335/8313]

---
 ..._committer_thread_loops_occassionally.mdwn | 49 +++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn

diff --git a/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn b/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn
new file mode 100644
index 0000000000..eb8a0ead0a
--- /dev/null
+++ b/doc/bugs/lsof__47__committer_thread_loops_occassionally.mdwn
@@ -0,0 +1,49 @@
+I've noticed this to occur occassionally 
+
+<pre>
+laplace:atest jtang$ ls
+1@  2@	3@  4@	5@  readme.txt@
+laplace:atest jtang$ git annex watch -d --foreground
+watch . [2012-07-29 11:49:26 IST] read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","show-ref","git-annex"] 
+[2012-07-29 11:49:26 IST] read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","show-ref","--hash","refs/heads/git-annex"] 
+[2012-07-29 11:49:26 IST] read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","log","refs/heads/git-annex..f85faa60e73efabc2e92f837b19c3918d3ab030f","--oneline","-n1"] 
+[2012-07-29 11:49:26 IST] chat: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","cat-file","--batch"] 
+(scanning...) [2012-07-29 11:49:26 IST] Assistant: all threads started
+[2012-07-29 11:49:26 IST] Merger: watching /Users/jtang/sandbox/atest/.git/refs/heads/synced
+[2012-07-29 11:49:26 IST] TransferWatcher: watching for transfers
+[2012-07-29 11:49:26 IST] read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","symbolic-ref","HEAD"] 
+[2012-07-29 11:49:26 IST] call: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","add","--update"] 
+[2012-07-29 11:49:26 IST] Merger: merging changes into Just refs/heads/master
+[2012-07-29 11:49:26 IST] call: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","merge","--no-edit","refs/heads/synced/master"] 
+(started...) [2012-07-29 11:49:26 IST] Watcher: watching .
+[2012-07-29 11:49:26 IST] WebApp: running on port 60042
+Already up-to-date.
+[2012-07-29 11:49:26 IST] Watcher: add symlink ./1
+[2012-07-29 11:49:26 IST] chat: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","cat-file","--batch"] 
+[2012-07-29 11:49:26 IST] Watcher: add symlink ./2
+[2012-07-29 11:49:26 IST] Watcher: add symlink ./3
+[2012-07-29 11:49:26 IST] Watcher: add symlink ./4
+[2012-07-29 11:49:26 IST] Watcher: add symlink ./5
+[2012-07-29 11:49:26 IST] Watcher: add symlink ./readme.txt
+[2012-07-29 11:49:27 IST] Committer: committing 6 changes
+(Recording state in git...)
+[2012-07-29 11:49:27 IST] feed: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","update-index","-z","--index-info"] 
+[2012-07-29 11:49:27 IST] call: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","commit","--allow-empty-message","-m","","--allow-empty","--quiet"] 
+[2012-07-29 11:49:28 IST] read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","symbolic-ref","HEAD"] 
+[2012-07-29 11:49:28 IST] Pusher: pushing to []
+[2012-07-29 11:49:28 IST] call: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","branch","-f","synced/master"] 
+[2012-07-29 11:49:28 IST] read: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","symbolic-ref","HEAD"] 
+[2012-07-29 11:49:28 IST] Merger: merging changes into Just refs/heads/master
+[2012-07-29 11:49:28 IST] call: git ["--git-dir=/Users/jtang/sandbox/atest/.git","--work-tree=/Users/jtang/sandbox/atest","merge","--no-edit","refs/heads/synced/master"] 
+Already up-to-date.
+[2012-07-29 11:49:43 IST] Watcher: file added ./.gitignore
+[2012-07-29 11:49:43 IST] read: lsof ["-F0can","+d","/Users/jtang/sandbox/atest/.git/annex/tmp/"] 
+[2012-07-29 11:49:44 IST] Committer: delaying commit of 0 changes
+[2012-07-29 11:49:45 IST] read: lsof ["-F0can","+d","/Users/jtang/sandbox/atest/.git/annex/tmp/"] 
+[2012-07-29 11:49:46 IST] Committer: delaying commit of 0 changes
+[2012-07-29 11:49:47 IST] read: lsof ["-F0can","+d","/Users/jtang/sandbox/atest/.git/annex/tmp/"] 
+[2012-07-29 11:49:48 IST] Committer: delaying commit of 0 changes
+[2012-07-29 11:49:49 IST] read: lsof ["-F0can","+d","/Users/jtang/sandbox/atest/.git/annex/tmp/"] 
+</pre>
+
+I ran " git annex watch -d --foreground" to watch what was going one, and just created a .gitignore file and the the commiter/lsof thread just loops over and over.... I only noticed as my laptop battery had drained at somepoint when git-annex was running in the background.

From 57203e39811e4e769a6feb576a8779707664c40d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 08:52:57 -0400
Subject: [PATCH 4336/8313] refactor

---
 Assistant/DaemonStatus.hs        | 47 ++++++++++++++++++--------------
 Assistant/Threads/Transferrer.hs |  3 +-
 Assistant/Threads/WebApp.hs      | 37 +++++++++++++------------
 Assistant/TransferQueue.hs       |  2 +-
 4 files changed, 47 insertions(+), 42 deletions(-)

diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 3610c2fdad..958a816c0b 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -35,9 +35,10 @@ data DaemonStatus = DaemonStatus
 	, currentTransfers :: TransferMap
 	-- Ordered list of remotes to talk to.
 	, knownRemotes :: [Remote]
-	-- Clients can use this to wait on changes to the DaemonStatus
-	-- and other related things like the TransferQueue.
-	, notificationBroadcaster :: NotificationBroadcaster
+	-- Broadcasts notifications about all changes to the DaemonStatus
+	, changeNotifier :: NotificationBroadcaster
+	-- Broadcasts notifications when queued or running transfers change.
+	, transferNotifier :: NotificationBroadcaster
 	}
 
 type TransferMap = M.Map Transfer TransferInfo
@@ -47,7 +48,8 @@ type DaemonStatusHandle = TMVar DaemonStatus
 
 newDaemonStatus :: IO DaemonStatus
 newDaemonStatus = do
-	nb <- newNotificationBroadcaster
+	cn <- newNotificationBroadcaster
+	tn <- newNotificationBroadcaster
 	return $ DaemonStatus
 		{ scanComplete = False
 		, lastRunning = Nothing
@@ -55,7 +57,8 @@ newDaemonStatus = do
 		, lastSanityCheck = Nothing
 		, currentTransfers = M.empty
 		, knownRemotes = []
-		, notificationBroadcaster = nb
+		, changeNotifier = cn
+		, transferNotifier = tn
 		}
 
 getDaemonStatus :: DaemonStatusHandle -> IO DaemonStatus
@@ -66,19 +69,13 @@ modifyDaemonStatus_ handle a = modifyDaemonStatus handle $ \s -> (a s, ())
 
 modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> IO b
 modifyDaemonStatus handle a = do
-	(b, nb) <- atomically $ do
-		(s, b) <- a <$> takeTMVar handle
+	(s, b) <- atomically $ do
+		r@(s, _) <- a <$> takeTMVar handle
 		putTMVar handle s
-		return $ (b, notificationBroadcaster s)
-	sendNotification nb
+		return r
+	sendNotification $ changeNotifier s
 	return b
 
-{- Can be used to send a notification that the daemon status, or other
- - associated thing, like the TransferQueue, has changed. -}
-notifyDaemonStatusChange :: DaemonStatusHandle -> IO ()
-notifyDaemonStatusChange handle = sendNotification
-	=<< notificationBroadcaster <$> atomically (readTMVar handle)
-
 {- Updates the cached ordered list of remotes from the list in Annex
  - state. -}
 updateKnownRemotes :: DaemonStatusHandle -> Annex ()
@@ -108,11 +105,11 @@ startDaemonStatus = do
  -}
 daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
 daemonStatusThread st handle = do
-	bhandle <- newNotificationHandle
-		=<< notificationBroadcaster <$> getDaemonStatus handle
+	notifier <- newNotificationHandle
+		=<< changeNotifier <$> getDaemonStatus handle
 	checkpoint
 	runEvery (Seconds tenMinutes) $ do
-		waitNotification bhandle
+		waitNotification notifier
 		checkpoint
 	where
 		checkpoint = do
@@ -182,15 +179,23 @@ adjustTransfersSTM dstatus a = do
 
 {- Variant that does send notifications. -}
 adjustTransfers :: DaemonStatusHandle -> (TransferMap -> TransferMap) -> IO ()
-adjustTransfers dstatus a = modifyDaemonStatus_ dstatus $
-	\s -> s { currentTransfers = a (currentTransfers s) }
+adjustTransfers dstatus a = 
+	notifyTransfer dstatus `after` modifyDaemonStatus_ dstatus go
+	where
+		go s = s { currentTransfers = a (currentTransfers s) }
 
 {- Removes a transfer from the map, and returns its info. -}
 removeTransfer :: DaemonStatusHandle -> Transfer -> IO (Maybe TransferInfo)
-removeTransfer dstatus t = modifyDaemonStatus dstatus go
+removeTransfer dstatus t = 
+	notifyTransfer dstatus `after` modifyDaemonStatus dstatus go
 	where
 		go s =
 			let (info, ts) = M.updateLookupWithKey
 				(\_k _v -> Nothing)
 				t (currentTransfers s)
 			in (s { currentTransfers = ts }, info)
+
+{- Send a notification when a transfer is changed. -}
+notifyTransfer :: DaemonStatusHandle -> IO ()
+notifyTransfer handle = sendNotification
+	=<< transferNotifier <$> atomically (readTMVar handle)
diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs
index f011ff0363..a801556dbd 100644
--- a/Assistant/Threads/Transferrer.hs
+++ b/Assistant/Threads/Transferrer.hs
@@ -38,14 +38,13 @@ transfererThread st dstatus transferqueue slots = go
 			ifM (runThreadState st $ shouldTransfer dstatus t info)
 				( do
 					debug thisThread [ "Transferring:" , show t ]
-					notifyDaemonStatusChange dstatus
+					notifyTransfer dstatus
 					transferThread st dstatus slots t info
 				, do
 					debug thisThread [ "Skipping unnecessary transfer:" , show t ]
 					-- getNextTransfer added t to the
 					-- daemonstatus's transfer map.
 					void $ removeTransfer dstatus t
-					notifyDaemonStatusChange dstatus
 				)
 			go
 
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 500297693b..3db5f368c7 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings #-}
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
 {-# OPTIONS_GHC -fno-warn-orphans #-}
 
 module Assistant.Threads.WebApp where
@@ -47,9 +47,20 @@ data WebApp = WebApp
 	, getStatic :: Static
 	}
 
-getNotificationBroadcaster :: WebApp -> IO NotificationBroadcaster
-getNotificationBroadcaster webapp = notificationBroadcaster
-		<$> getDaemonStatus (daemonStatus webapp)
+waitNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> NotificationId -> GHandler sub WebApp ()
+waitNotifier selector nid = do
+	notifier <- getNotifier selector
+	liftIO $ waitNotification $ notificationHandleFromId notifier nid
+
+newNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> GHandler sub WebApp NotificationId
+newNotifier selector = do
+	notifier <- getNotifier selector
+	liftIO $ notificationHandleToId <$> newNotificationHandle notifier
+
+getNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> GHandler sub WebApp NotificationBroadcaster
+getNotifier selector = do
+	webapp <- getYesod
+	liftIO $ selector <$> getDaemonStatus (daemonStatus webapp)
 
 staticFiles "static"
 
@@ -133,10 +144,7 @@ transfersDisplayIdent = "transfers"
  -}
 getTransfersR :: NotificationId -> Handler RepHtml
 getTransfersR nid = do
-	{- Block until there is a change from last time. -}
-	webapp <- getYesod
-	b <- liftIO $ getNotificationBroadcaster webapp
-	liftIO $ waitNotification $ notificationHandleFromId b nid
+	waitNotifier transferNotifier nid
 
 	page <- widgetToPageContent $ transfersDisplay False
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
@@ -149,9 +157,7 @@ sideBarDisplay noScript = do
 	$(widgetFile "sidebar")
 	unless noScript $ do
 		{- Set up automatic updates of the sidebar. -}
-		webapp <- lift getYesod
-		nid <- liftIO $ notificationHandleToId <$>
-			(newNotificationHandle =<< getNotificationBroadcaster webapp)
+		nid <- lift $ newNotifier transferNotifier
 		autoUpdate ident (SideBarR nid) (10 :: Int) (10 :: Int)
 
 {- Called by client to get a sidebar display.
@@ -164,10 +170,7 @@ sideBarDisplay noScript = do
  -}
 getSideBarR :: NotificationId -> Handler RepHtml
 getSideBarR nid = do
-	{- Block until there is a change from last time. -}
-	webapp <- getYesod
-	b <- liftIO $ getNotificationBroadcaster webapp
-	liftIO $ waitNotification $ notificationHandleFromId b nid
+	waitNotifier transferNotifier nid
 
 	page <- widgetToPageContent $ sideBarDisplay True
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
@@ -180,9 +183,7 @@ dashboard noScript warnNoScript = do
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ do
 	{- Set up automatic updates for the transfers display. -}
-	webapp <- lift getYesod
-	nid <- liftIO $ notificationHandleToId <$>
-		(newNotificationHandle =<< getNotificationBroadcaster webapp)
+	nid <- lift $ newNotifier transferNotifier
 	autoUpdate transfersDisplayIdent (TransfersR nid) (10 :: Int) (10 :: Int)
 	
 	dashboard False True
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index 51ed5c9c78..01c159b08a 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -95,7 +95,7 @@ enqueue schedule q dstatus t info
 				void $ modqueue (queue q) new
 				void $ modifyTVar' (queuesize q) succ
 				void $ modifyTVar' (queuelist q) modlist
-			void $ notifyDaemonStatusChange dstatus
+			void $ notifyTransfer dstatus
 
 {- Adds a transfer to the queue. -}
 queueTransfer :: Schedule -> TransferQueue -> DaemonStatusHandle -> AssociatedFile -> Transfer -> Remote -> IO ()

From 5271d699d22f9addb35f2374a2a70da59897bb1d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 09:35:01 -0400
Subject: [PATCH 4337/8313] add alerts to DaemonStatus

---
 Assistant/Alert.hs        | 25 ++++++++++++
 Assistant/DaemonStatus.hs | 80 ++++++++++++++++++++++++++++-----------
 2 files changed, 82 insertions(+), 23 deletions(-)
 create mode 100644 Assistant/Alert.hs

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
new file mode 100644
index 0000000000..c8bfa48fd7
--- /dev/null
+++ b/Assistant/Alert.hs
@@ -0,0 +1,25 @@
+{- git-annex assistant alerts
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE RankNTypes #-}
+
+module Assistant.Alert where
+
+import Yesod
+
+type Widget = forall sub master. GWidget sub master ()
+
+{- Different classes of alerts are displayed differently. -}
+data AlertClass = Activity | Warning | Error | Message
+
+{- An alert can be a simple message, or a Yesod Widget -}
+data AlertMessage = StringAlert String | WidgetAlert Widget
+
+data Alert = Alert
+	{ alertClass :: AlertClass
+	, alertMessage :: AlertMessage
+	}
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 958a816c0b..62cf2ea2ac 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -9,6 +9,7 @@ module Assistant.DaemonStatus where
 
 import Common.Annex
 import Assistant.ThreadedMonad
+import Assistant.Alert
 import Utility.ThreadScheduler
 import Utility.TempFile
 import Utility.NotificationBroadcaster
@@ -21,6 +22,7 @@ import Data.Time.Clock.POSIX
 import Data.Time
 import System.Locale
 import qualified Data.Map as M
+import Control.Exception
 
 data DaemonStatus = DaemonStatus
 	-- False when the daemon is performing its startup scan
@@ -33,45 +35,52 @@ data DaemonStatus = DaemonStatus
 	, lastSanityCheck :: Maybe POSIXTime
 	-- Currently running file content transfers
 	, currentTransfers :: TransferMap
+	-- Messages to display to the user.
+	, alertMap :: AlertMap
+	, alertMax :: AlertId
 	-- Ordered list of remotes to talk to.
 	, knownRemotes :: [Remote]
 	-- Broadcasts notifications about all changes to the DaemonStatus
 	, changeNotifier :: NotificationBroadcaster
-	-- Broadcasts notifications when queued or running transfers change.
+	-- Broadcasts notifications when queued or current transfers change.
 	, transferNotifier :: NotificationBroadcaster
+	-- Broadcasts notifications when there's a change to the alerts
+	, alertNotifier :: NotificationBroadcaster
 	}
 
 type TransferMap = M.Map Transfer TransferInfo
 
+type AlertMap = M.Map AlertId Alert
+type AlertId = Integer
+
 {- This TMVar is never left empty, so accessing it will never block. -}
 type DaemonStatusHandle = TMVar DaemonStatus
 
 newDaemonStatus :: IO DaemonStatus
-newDaemonStatus = do
-	cn <- newNotificationBroadcaster
-	tn <- newNotificationBroadcaster
-	return $ DaemonStatus
-		{ scanComplete = False
-		, lastRunning = Nothing
-		, sanityCheckRunning = False
-		, lastSanityCheck = Nothing
-		, currentTransfers = M.empty
-		, knownRemotes = []
-		, changeNotifier = cn
-		, transferNotifier = tn
-		}
+newDaemonStatus = DaemonStatus
+	<$> pure False
+	<*> pure Nothing
+	<*> pure False
+	<*> pure Nothing
+	<*> pure M.empty
+	<*> pure M.empty
+	<*> pure 0
+	<*> pure []
+	<*> newNotificationBroadcaster
+	<*> newNotificationBroadcaster
+	<*> newNotificationBroadcaster
 
 getDaemonStatus :: DaemonStatusHandle -> IO DaemonStatus
 getDaemonStatus = atomically . readTMVar
 
 modifyDaemonStatus_ :: DaemonStatusHandle -> (DaemonStatus -> DaemonStatus) -> IO ()
-modifyDaemonStatus_ handle a = modifyDaemonStatus handle $ \s -> (a s, ())
+modifyDaemonStatus_ dstatus a = modifyDaemonStatus dstatus $ \s -> (a s, ())
 
 modifyDaemonStatus :: DaemonStatusHandle -> (DaemonStatus -> (DaemonStatus, b)) -> IO b
-modifyDaemonStatus handle a = do
+modifyDaemonStatus dstatus a = do
 	(s, b) <- atomically $ do
-		r@(s, _) <- a <$> takeTMVar handle
-		putTMVar handle s
+		r@(s, _) <- a <$> takeTMVar dstatus
+		putTMVar dstatus s
 		return r
 	sendNotification $ changeNotifier s
 	return b
@@ -104,16 +113,16 @@ startDaemonStatus = do
  - frequently than once every ten minutes.
  -}
 daemonStatusThread :: ThreadState -> DaemonStatusHandle -> IO ()
-daemonStatusThread st handle = do
+daemonStatusThread st dstatus = do
 	notifier <- newNotificationHandle
-		=<< changeNotifier <$> getDaemonStatus handle
+		=<< changeNotifier <$> getDaemonStatus dstatus
 	checkpoint
 	runEvery (Seconds tenMinutes) $ do
 		waitNotification notifier
 		checkpoint
 	where
 		checkpoint = do
-			status <- getDaemonStatus handle
+			status <- getDaemonStatus dstatus
 			file <- runThreadState st $ fromRepo gitAnnexDaemonStatusFile
 			writeDaemonStatusFile file status
 
@@ -197,5 +206,30 @@ removeTransfer dstatus t =
 
 {- Send a notification when a transfer is changed. -}
 notifyTransfer :: DaemonStatusHandle -> IO ()
-notifyTransfer handle = sendNotification
-	=<< transferNotifier <$> atomically (readTMVar handle)
+notifyTransfer dstatus = sendNotification
+	=<< transferNotifier <$> atomically (readTMVar dstatus)
+
+{- Send a notification when alerts are changed. -}
+notifyAlert :: DaemonStatusHandle -> IO ()
+notifyAlert dstatus = sendNotification
+	=<< alertNotifier <$> atomically (readTMVar dstatus)
+
+{- Returns the alert's identifier, which can be used to remove it. -}
+addAlert :: DaemonStatusHandle -> Alert -> IO AlertId
+addAlert dstatus alert = notifyAlert dstatus `after` modifyDaemonStatus dstatus go
+	where
+		go s = (s { alertMax = i, alertMap = m }, i)
+			where
+				i = alertMax s + 1
+				m = M.insertWith' const i alert (alertMap s)
+
+removeAlert :: DaemonStatusHandle -> AlertId -> IO ()
+removeAlert dstatus i = notifyAlert dstatus `after` modifyDaemonStatus_ dstatus go
+	where
+		go s = s { alertMap = M.delete i (alertMap s) }
+
+{- Displays an alert while performing an activity, then removes it. -}
+alertWhile :: DaemonStatusHandle -> Alert -> IO a -> IO a
+alertWhile dstatus alert a = do
+	let alert' = alert { alertClass = Activity }
+	bracket (addAlert dstatus alert') (removeAlert dstatus) (const a)

From c2f3e66d8c65e46046f83221996b3a180bd49657 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 11:31:06 -0400
Subject: [PATCH 4338/8313] show alerts in the sidebar

This has a bug -- it seems long polling can only wait on one page at a
time. Need to re-unify the notifiers.
---
 Assistant/Alert.hs           | 15 ++++++++++--
 Assistant/Threads/Watcher.hs | 10 ++++----
 Assistant/Threads/WebApp.hs  | 44 ++++++++++++++++++++++++++++++++----
 templates/sidebar.hamlet     | 17 +-------------
 4 files changed, 59 insertions(+), 27 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index c8bfa48fd7..f4220eea9a 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -14,12 +14,23 @@ import Yesod
 type Widget = forall sub master. GWidget sub master ()
 
 {- Different classes of alerts are displayed differently. -}
-data AlertClass = Activity | Warning | Error | Message
+data AlertClass = Activity | Warning | Error | Success | Message
+	deriving (Eq)
 
-{- An alert can be a simple message, or a Yesod Widget -}
+{- An alert can be a simple message, or an arbitrary Yesod Widget -}
 data AlertMessage = StringAlert String | WidgetAlert Widget
 
 data Alert = Alert
 	{ alertClass :: AlertClass
+	, alertHeader :: Maybe String
 	, alertMessage :: AlertMessage
+	, alertBlockDisplay :: Bool
+	}
+
+activityAlert :: Maybe String -> String -> Alert
+activityAlert header message = Alert
+	{ alertClass = Activity
+	, alertHeader = header
+	, alertMessage = StringAlert message
+	, alertBlockDisplay = False
 	}
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index ab57bf04a0..5086f95a23 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -19,6 +19,7 @@ import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.Changes
 import Assistant.TransferQueue
+import Assistant.Alert
 import Logs.Transfer
 import Utility.DirWatcher
 import Utility.Types.DirWatcher
@@ -60,7 +61,7 @@ watchThread st dstatus transferqueue changechan = do
 	void $ watchDir "." ignored hooks startup
 	debug thisThread [ "watching", "."]
 	where
-		startup = statupScan st dstatus
+		startup = startupScan st dstatus
 		hook a = Just $ runHandler thisThread st dstatus transferqueue changechan a
 		hooks = WatchHooks
 			{ addHook = hook onAdd
@@ -71,11 +72,12 @@ watchThread st dstatus transferqueue changechan = do
 			}
 
 {- Initial scartup scan. The action should return once the scan is complete. -}
-statupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
-statupScan st dstatus scanner = do
+startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
+startupScan st dstatus scanner = do
 	runThreadState st $
 		showAction "scanning"
-	r <- scanner
+	let alert = activityAlert Nothing "Performing startup scan"
+	r <- alertWhile dstatus alert scanner
 	modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
 
 	-- Notice any files that were deleted before watching was started.
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 3db5f368c7..132aad22e8 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -14,6 +14,7 @@ import Assistant.Common
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.TransferQueue
+import Assistant.Alert hiding (Widget)
 import Utility.NotificationBroadcaster
 import Utility.WebApp
 import Utility.Yesod
@@ -33,7 +34,7 @@ import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text (Text, pack, unpack)
 import qualified Data.Map as M
-import Data.Time.Clock
+import Data.Function
 
 thisThread :: String
 thisThread = "WebApp"
@@ -151,14 +152,47 @@ getTransfersR nid = do
 
 sideBarDisplay :: Bool -> Widget
 sideBarDisplay noScript = do
-	date <- liftIO $ show <$> getCurrentTime
+	let content = do
+		{- Any yesod message appears as the first alert. -}
+		maybe noop rendermessage =<< lift getMessage
+	
+		{- Add newest 10 alerts to the sidebar. -}
+		webapp <- lift getYesod
+		alerts <- M.toList . alertMap
+			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
+		mapM_ renderalert $
+			take 10 $ reverse $ sortBy (compare `on` fst) alerts
 	ident <- lift newIdent
-	mmsg <- lift getMessage
 	$(widgetFile "sidebar")
+
 	unless noScript $ do
-		{- Set up automatic updates of the sidebar. -}
-		nid <- lift $ newNotifier transferNotifier
+		{- Set up automatic updates of the sidebar
+		 - when alerts come in. -}
+		nid <- lift $ newNotifier alertNotifier
 		autoUpdate ident (SideBarR nid) (10 :: Int) (10 :: Int)
+	where
+		bootstrapclass Activity = "alert-info"
+		bootstrapclass Warning = "alert"
+		bootstrapclass Error = "alert-error"
+		bootstrapclass Success = "alert-success"
+		bootstrapclass Message = "alert-info"
+
+		renderalert (alertid, alert) = addalert
+			(show alertid)
+			-- Activity alerts auto-close
+			(not noScript && alertClass alert /= Activity)
+			(alertBlockDisplay alert)
+			(bootstrapclass $ alertClass alert)
+			(alertHeader alert)
+			$ case alertMessage alert of
+				StringAlert s -> [whamlet|#{s}|]
+				WidgetAlert w -> w
+
+		rendermessage msg = addalert "yesodmessage" True False
+			"alert-info" Nothing [whamlet|#{msg}|]
+
+		addalert :: String -> Bool -> Bool -> Text -> Maybe String -> Widget -> Widget
+		addalert alertid closable block divclass heading widget = $(widgetFile "alert")
 
 {- Called by client to get a sidebar display.
  -
diff --git a/templates/sidebar.hamlet b/templates/sidebar.hamlet
index 3b5048151b..32900b920c 100644
--- a/templates/sidebar.hamlet
+++ b/templates/sidebar.hamlet
@@ -1,18 +1,3 @@
 <div .span3 ##{ident}>
   <div .sidebar-nav>
-    $maybe msg <- mmsg
-      <div .alert .alert-info>
-        <a .close data-dismiss="alert" href="#">×</a>
-          #{msg}
-    <div .alert .alert-info>
-      <a .close data-dismiss="alert" href="#">×</a>
-      <b>This is just a demo.</b> If this were not just a demo,
-      I'd not be filling this sidebar with silly alerts.
-    <div .alert .alert-success>
-      <a .close data-dismiss="alert" href="#">×</a>
-      <b>Well done!</b>
-      You successfully read this important alert message.
-    <div .alert .alert-error>
-      <a .close data-dismiss="alert" href="#">×</a>
-      <b>Whoops!</b>
-      Unable to connect to blah blah.. #{date}
+    ^{content}

From ea05ba893c1e7f56e24115a2641cd517bb3560a5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 12:12:14 -0400
Subject: [PATCH 4339/8313] fix the auto token leak on auth error page issue
 permanantly

---
 Utility/WebApp.hs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 75e8dde9ec..971422e369 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -137,7 +137,11 @@ genRandomToken = do
 			Right (s, _) -> showDigest $ sha512 $ L.fromChunks [s]
 
 {- A Yesod isAuthorized method, which checks the auth cgi parameter
- - against a token extracted from the Yesod application. -}
+ - against a token extracted from the Yesod application.
+ -
+ - Note that the usual Yesod error page is bypassed on error, to avoid
+ - possibly leaking the auth token in urls on that page!
+ -}
 checkAuthToken :: forall t sub. (t -> T.Text) -> GHandler sub t AuthResult
 checkAuthToken extractToken = do
 	webapp <- getYesod
@@ -145,7 +149,7 @@ checkAuthToken extractToken = do
 	let params = reqGetParams req
 	if lookup "auth" params == Just (extractToken webapp)
 		then return Authorized
-		else return AuthenticationRequired
+		else sendResponseStatus unauthorized401 ()
 
 {- A Yesod joinPath method, which adds an auth cgi parameter to every
  - url matching a predicate, containing a token extracted from the

From e1d4bfe6716f409f0ab400f532e48db2dfc98cf1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 12:37:45 -0400
Subject: [PATCH 4340/8313] typo; was waiting on the wrong notifier for the
 sidebar!

---
 Assistant/Threads/WebApp.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 132aad22e8..3d42db8125 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -204,7 +204,7 @@ sideBarDisplay noScript = do
  -}
 getSideBarR :: NotificationId -> Handler RepHtml
 getSideBarR nid = do
-	waitNotifier transferNotifier nid
+	waitNotifier alertNotifier nid
 
 	page <- widgetToPageContent $ sideBarDisplay True
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]

From 2dc5697a0ac36fdfe21da79a721db3f086bce041 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 12:39:10 -0400
Subject: [PATCH 4341/8313] add

---
 templates/alert.hamlet | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 templates/alert.hamlet

diff --git a/templates/alert.hamlet b/templates/alert.hamlet
new file mode 100644
index 0000000000..2d0daf8418
--- /dev/null
+++ b/templates/alert.hamlet
@@ -0,0 +1,9 @@
+<div .alert .fade .in .#{divclass} :block:.alert-block ##{alertid}>
+  $if closable
+    <a .close data-dismiss="alert" href="#">×</a>
+  $maybe h <- heading
+    $if block
+      <h4 class="alert-heading">#{h}</h4>
+    $else
+      <strong>#{h}</strong>
+  ^{widget}

From 09e77a0cf0ca6e6c76ead584f16818dcf04a94b6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 13:22:08 -0400
Subject: [PATCH 4342/8313] add some alerts

---
 Assistant/Threads/MountWatcher.hs    | 25 +++++++++++++++++++------
 Assistant/Threads/TransferScanner.hs | 12 +++++++++++-
 Assistant/Threads/Watcher.hs         |  7 ++++---
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs
index 853d96d51c..7d0ef5ae40 100644
--- a/Assistant/Threads/MountWatcher.hs
+++ b/Assistant/Threads/MountWatcher.hs
@@ -15,6 +15,7 @@ import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.ScanRemotes
 import Assistant.Threads.Pusher (pushToRemotes)
+import Assistant.Alert
 import qualified Annex
 import qualified Git
 import Utility.ThreadScheduler
@@ -158,17 +159,29 @@ handleMounts st dstatus scanremotes wasmounted nowmounted = mapM_ (handleMount s
 
 handleMount :: ThreadState -> DaemonStatusHandle -> ScanRemoteMap -> Mntent -> IO ()
 handleMount st dstatus scanremotes mntent = do
-	debug thisThread ["detected mount of", mnt_dir mntent]
+	debug thisThread ["detected mount of", dir]
 	rs <- remotesUnder st dstatus mntent
 	unless (null rs) $ do
 		branch <- runThreadState st $ Command.Sync.currentBranch
 		let nonspecial = filter (Git.repoIsLocal . Remote.repo) rs
-		unless (null nonspecial) $ do
-			debug thisThread ["pulling from", show nonspecial]
-			runThreadState st $ manualPull branch nonspecial
-			now <- getCurrentTime	
-			pushToRemotes thisThread now st Nothing nonspecial
+		unless (null nonspecial) $
+			alertWhile dstatus (syncalert nonspecial) $ do
+				debug thisThread ["syncing with", show nonspecial]
+				runThreadState st $ manualPull branch nonspecial
+				now <- getCurrentTime	
+				pushToRemotes thisThread now st Nothing nonspecial
 		addScanRemotes scanremotes rs
+	where
+		dir = mnt_dir mntent
+		syncalert rs = Alert
+			{ alertClass = Activity
+			, alertHeader = Just $ "Syncing with " ++ unwords (map Remote.name rs)
+			, alertMessage = StringAlert $ unwords
+				["I noticed you plugged in", dir,
+				 " -- let's get it in sync!"]
+			, alertBlockDisplay = True
+			}
+			
 
 {- Finds remotes located underneath the mount point.
  -
diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index e6a078907b..1bf8b062fc 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -12,6 +12,7 @@ import Assistant.ScanRemotes
 import Assistant.TransferQueue
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
+import Assistant.Alert
 import Logs.Transfer
 import Logs.Location
 import qualified Remote
@@ -31,9 +32,18 @@ transferScannerThread st dstatus scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
 		liftIO $ debug thisThread ["starting scan of", show r]
-		scan st dstatus transferqueue r
+		alertWhile dstatus (scanalert r) $
+			scan st dstatus transferqueue r
 		liftIO $ debug thisThread ["finished scan of", show r]
 	where
+		scanalert r = Alert
+			{ alertClass = Activity
+			, alertHeader = Just $ "Scanning " ++ Remote.name r
+			, alertMessage = StringAlert $ unwords
+				[ "Ensuring that ", Remote.name r
+				, "is fully in sync." ]
+			, alertBlockDisplay = True
+			}
 
 {- This is a naive scan through the git work tree.
  - 
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 5086f95a23..ade26be19b 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -74,9 +74,7 @@ watchThread st dstatus transferqueue changechan = do
 {- Initial scartup scan. The action should return once the scan is complete. -}
 startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
 startupScan st dstatus scanner = do
-	runThreadState st $
-		showAction "scanning"
-	let alert = activityAlert Nothing "Performing startup scan"
+	runThreadState st $ showAction "scanning"
 	r <- alertWhile dstatus alert scanner
 	modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
 
@@ -86,6 +84,9 @@ startupScan st dstatus scanner = do
 		showAction "started"
 
 	return r
+	
+	where
+		alert = activityAlert Nothing "Performing startup scan"
 
 ignored :: FilePath -> Bool
 ignored = ig . takeFileName

From ebd8362d58036a75f8aaf4ad0b69ba57d3c77a0e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 13:37:26 -0400
Subject: [PATCH 4343/8313] fix bug in transfer initiation checking

Putting the transfer on the currentTransfers atomically introduced a bug:
It checks to see if the transfer is in progress, and cancels it.

Fixed by moving that check inside the STM transaction.
---
 Assistant/Threads/Transferrer.hs | 43 ++++++++++++++++----------------
 Assistant/TransferQueue.hs       | 26 +++++++++++--------
 2 files changed, 37 insertions(+), 32 deletions(-)

diff --git a/Assistant/Threads/Transferrer.hs b/Assistant/Threads/Transferrer.hs
index a801556dbd..956e0fc9d1 100644
--- a/Assistant/Threads/Transferrer.hs
+++ b/Assistant/Threads/Transferrer.hs
@@ -33,9 +33,10 @@ maxTransfers = 1
 transfererThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> TransferSlots -> IO ()
 transfererThread st dstatus transferqueue slots = go
 	where
-		go = do
-			(t, info) <- getNextTransfer transferqueue dstatus
-			ifM (runThreadState st $ shouldTransfer dstatus t info)
+		go = getNextTransfer transferqueue dstatus notrunning >>= handle
+		handle Nothing = go
+		handle (Just (t, info)) = do
+			ifM (runThreadState st $ shouldTransfer t info)
 				( do
 					debug thisThread [ "Transferring:" , show t ]
 					notifyTransfer dstatus
@@ -47,28 +48,26 @@ transfererThread st dstatus transferqueue slots = go
 					void $ removeTransfer dstatus t
 				)
 			go
+		{- Skip transfers that are already running. -}
+		notrunning i = startedTime i == Nothing
 
-{- Checks if the requested transfer is already running, or
- - the file to download is already present, or the remote
+{- Checks if the file to download is already present, or the remote
  - being uploaded to isn't known to have the file. -}
-shouldTransfer :: DaemonStatusHandle -> Transfer -> TransferInfo -> Annex Bool
-shouldTransfer dstatus t info =
-	go =<< currentTransfers <$> liftIO (getDaemonStatus dstatus)
+shouldTransfer :: Transfer -> TransferInfo -> Annex Bool
+shouldTransfer t info
+	| transferDirection t == Download =
+		not <$> inAnnex key
+	| transferDirection t == Upload =
+		{- Trust the location log to check if the
+		 - remote already has the key. This avoids
+		 - a roundtrip to the remote. -}
+		case transferRemote info of
+			Nothing -> return False
+			Just remote -> 
+				notElem (Remote.uuid remote)
+					<$> loggedLocations key
+	| otherwise = return False
 	where
-		go m
-			| M.member t m = return False
-			| transferDirection t == Download =
-				not <$> inAnnex key
-			| transferDirection t == Upload =
-				{- Trust the location log to check if the
-				 - remote already has the key. This avoids
-				 - a roundtrip to the remote. -}
-				case transferRemote info of
-					Nothing -> return False
-					Just remote -> 
-						notElem (Remote.uuid remote)
-							<$> loggedLocations key
-			| otherwise = return False
 		key = transferKey t
 
 {- A transfer is run in a separate thread, with a *copy* of the Annex
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index 01c159b08a..40adc35206 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -113,16 +113,22 @@ queueTransferAt wantsz schedule q dstatus f t remote = do
 			else retry -- blocks until queuesize changes
 	enqueue schedule q dstatus t (stubInfo f remote)
 
-{- Blocks until a pending transfer is available from the queue.
- - The transfer is removed from the transfer queue, and added to
- - the daemon status currentTransfers map. This is done in a single STM
- - transaction, so there is no window where an observer sees an
- - inconsistent status. -}
-getNextTransfer :: TransferQueue -> DaemonStatusHandle -> IO (Transfer, TransferInfo)
-getNextTransfer q dstatus = atomically $ do
+{- Blocks until a pending transfer is available from the queue,
+ - and removes it.
+ -
+ - Checks that it's acceptable, before adding it to the
+ - the currentTransfers map. If it's not acceptable, it's discarded.
+ -
+ - This is done in a single STM transaction, so there is no window
+ - where an observer sees an inconsistent status. -}
+getNextTransfer :: TransferQueue -> DaemonStatusHandle -> (TransferInfo -> Bool) -> IO (Maybe (Transfer, TransferInfo))
+getNextTransfer q dstatus acceptable = atomically $ do
 	void $ modifyTVar' (queuesize q) pred
 	void $ modifyTVar' (queuelist q) (drop 1)
 	r@(t, info) <- readTChan (queue q)
-	adjustTransfersSTM dstatus $
-		M.insertWith' const t info
-	return r
+	if acceptable info
+		then do
+			adjustTransfersSTM dstatus $
+				M.insertWith' const t info
+			return $ Just r
+		else return Nothing

From 0b9ecea8ff19eec95263b0b682ec8417a1364587 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 13:45:56 -0400
Subject: [PATCH 4344/8313] update

---
 doc/design/assistant/webapp.mdwn | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 1fc32282a2..ebf1689963 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -27,15 +27,14 @@ The webapp is a web server that displays a shiny interface.
 * there could be a UI to export a file, which would make it be served up
   over http by the web app
 * Display any relevant warning messages. One is the `inotify max_user_watches`
-  exceeded message. Need to lift such messages into DaemonStatus
-  so the WebApp can include them in its rendering of DaemonStatus.
+  exceeded message.
 
 ## implementation
 
-* perhaps define a custom `errorHandler`, which could avoid the potential
-  of leaking auth tokens on error pages. Or make the test suite test for
-  leakage.
 * possibly lose the ugly auth= token past the first page,
   and use a client-side session. It could be encrypted using the token
   as the `encryptKey`. Note: Would need to set the session duration
   to infinite (how?)
+* Fix notification handle leakage on pages other than the main page.
+  The javascript should use AJAX to request handles, that way
+  they won't be allocated at all in noscript.

From 702fadd2837a0d3b982c86a79dae2d12d6683cec Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 13:52:57 -0400
Subject: [PATCH 4345/8313] blog for the day + screencast

---
 .../assistant/blog/day_46__alert_messages.mdwn   | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_46__alert_messages.mdwn

diff --git a/doc/design/assistant/blog/day_46__alert_messages.mdwn b/doc/design/assistant/blog/day_46__alert_messages.mdwn
new file mode 100644
index 0000000000..d65b0b6962
--- /dev/null
+++ b/doc/design/assistant/blog/day_46__alert_messages.mdwn
@@ -0,0 +1,16 @@
+Some days I spend 2 hours chasing red herrings (like "perhaps my JSON ajax
+calls arn't running asynchronoously?") that turn out to be a simple
+one-word typo. This was one of them.
+
+However, I did get the sidebar displaying alert messages, which can be
+easily sent to the user from any part of the assistant. This includes
+transient alerts of things it's doing, which disappear once the action
+finishes, and long-term alerts that are displayed until the user closes
+them. It even supports rendering arbitrary Yesod widgets as alerts, so
+they can also be used for asking questions, etc.
+
+Time for a screencast! For some reason `recordmydesktop` has the sound
+running around 1 second ahead of the video in this, but I think you'll
+still get the idea.
+
+<video controls src="http://joeyh.name/screencasts/git-annex-webapp.ogg"></video>

From c4023f785834bc237e5fcdb69e275bbae10dd40b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 13:55:07 -0400
Subject: [PATCH 4346/8313] probably fixes
 http://git-annex.branchable.com/bugs/lsof__47__committer_thread_loops_occassionally/

---
 Assistant/Threads/Committer.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs
index 33b92c7e53..b3258f7299 100644
--- a/Assistant/Threads/Committer.hs
+++ b/Assistant/Threads/Committer.hs
@@ -58,6 +58,7 @@ commitThread st changechan commitchan transferqueue dstatus = runEvery (Seconds
 				else refill readychanges
 		else refill changes
 	where
+		refill [] = noop
 		refill cs = do
 			debug thisThread
 				[ "delaying commit of"

From 7f75c2574a35081ca1b1446bdf8adbda8afb2fdd Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Sun, 29 Jul 2012 18:10:13 +0000
Subject: [PATCH 4347/8313] Added a comment: probably a kqueue specific
 problem...

---
 ...omment_1_f8d1720aa26c719609720acf0772606e._comment | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/bugs/lsof__47__committer_thread_loops_occassionally/comment_1_f8d1720aa26c719609720acf0772606e._comment

diff --git a/doc/bugs/lsof__47__committer_thread_loops_occassionally/comment_1_f8d1720aa26c719609720acf0772606e._comment b/doc/bugs/lsof__47__committer_thread_loops_occassionally/comment_1_f8d1720aa26c719609720acf0772606e._comment
new file mode 100644
index 0000000000..521d2e0cb6
--- /dev/null
+++ b/doc/bugs/lsof__47__committer_thread_loops_occassionally/comment_1_f8d1720aa26c719609720acf0772606e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.153.8.133"
+ subject="probably a kqueue specific problem..."
+ date="2012-07-29T18:10:12Z"
+ content="""
+I don't think this would eat your battery badly; it's looping for sure, but with a hardcoded 1 second delay I put in to guard against it eating all CPU.
+
+I tried to fix this in commit c4023f785834bc237e5fcdb69e275bbae10dd40b, but I sort of doubt I did.
+I made one more commit that will at least tell us what file it is trying to check over and over with lsof.
+"""]]

From ce7889ba86fc15e2892db8190114e291128e9c62 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 14:10:17 -0400
Subject: [PATCH 4348/8313] debuggery

---
 Assistant/Threads/Committer.hs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs
index b3258f7299..f236159f98 100644
--- a/Assistant/Threads/Committer.hs
+++ b/Assistant/Threads/Committer.hs
@@ -190,6 +190,13 @@ safeToAdd st changes = runThreadState st $
 			tmpdir <- fromRepo gitAnnexTmpDir
 			openfiles <- S.fromList . map fst3 . filter openwrite <$>
 				liftIO (Lsof.queryDir tmpdir)
+
+			liftIO $ debug thisThread
+				[ "checking changes:"
+				, show changes
+				, "vs open files:"
+				, show openfiles
+				]
 			
 			let checked = map (check openfiles) changes
 

From a1f4bb2f2cb35ba8d541c2dad2c43fd316c0fcef Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 14:49:42 -0400
Subject: [PATCH 4349/8313] fix

---
 ...ay_46__alert_messages.mdwn => day_47__alert_messages.mdwn} | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
 rename doc/design/assistant/blog/{day_46__alert_messages.mdwn => day_47__alert_messages.mdwn} (80%)

diff --git a/doc/design/assistant/blog/day_46__alert_messages.mdwn b/doc/design/assistant/blog/day_47__alert_messages.mdwn
similarity index 80%
rename from doc/design/assistant/blog/day_46__alert_messages.mdwn
rename to doc/design/assistant/blog/day_47__alert_messages.mdwn
index d65b0b6962..81551fa95b 100644
--- a/doc/design/assistant/blog/day_46__alert_messages.mdwn
+++ b/doc/design/assistant/blog/day_47__alert_messages.mdwn
@@ -9,8 +9,6 @@ finishes, and long-term alerts that are displayed until the user closes
 them. It even supports rendering arbitrary Yesod widgets as alerts, so
 they can also be used for asking questions, etc.
 
-Time for a screencast! For some reason `recordmydesktop` has the sound
-running around 1 second ahead of the video in this, but I think you'll
-still get the idea.
+Time for a screencast!
 
 <video controls src="http://joeyh.name/screencasts/git-annex-webapp.ogg"></video>

From e4a02f542265709f13a20b421d8b0160642499cd Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 16:25:19 -0400
Subject: [PATCH 4350/8313] something I heard on irc

---
 doc/design/assistant/android.mdwn | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/design/assistant/android.mdwn b/doc/design/assistant/android.mdwn
index f2711a4794..65569e4037 100644
--- a/doc/design/assistant/android.mdwn
+++ b/doc/design/assistant/android.mdwn
@@ -26,6 +26,10 @@ References:
 * <http://stackoverflow.com/questions/5151858/running-a-haskell-program-on-the-android-os>
 * <http://www.reddit.com/r/haskell/comments/ful84/haskell_on_android/>
 
+I've heard anecdoally that ipwnstudios not only has an IPhone GHC port,
+but also Android. Need to get in touch with them.
+<http://ipwnstudios.com/>
+
 ### Android specific features
 
 The app should be aware of power status, and avoid expensive background

From b2dc8fdb06068276869df682b439348aa96e57f5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 17:53:18 -0400
Subject: [PATCH 4351/8313] add more alerts

Nearly all long-running actions now display an alert.
---
 Assistant.hs                       |  2 +-
 Assistant/Threads/Pusher.hs        | 23 ++++++++----
 Assistant/Threads/SanityChecker.hs | 57 +++++++++++++++++++-----------
 3 files changed, 53 insertions(+), 29 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 1f41a9398f..22a87fe8cc 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -146,7 +146,7 @@ startDaemon assistant foreground webappwaiter
 			mapM_ forkIO
 				[ commitThread st changechan commitchan transferqueue dstatus
 				, pushThread st dstatus commitchan pushmap
-				, pushRetryThread st pushmap
+				, pushRetryThread st dstatus pushmap
 				, mergeThread st
 				, transferWatcherThread st dstatus
 				, transfererThread st dstatus transferqueue transferslots
diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs
index 3762c48368..27e95a7344 100644
--- a/Assistant/Threads/Pusher.hs
+++ b/Assistant/Threads/Pusher.hs
@@ -10,12 +10,14 @@ module Assistant.Threads.Pusher where
 import Assistant.Common
 import Assistant.Commits
 import Assistant.Pushes
-import Assistant.DaemonStatus
+import Assistant.Alert
 import Assistant.ThreadedMonad
 import Assistant.Threads.Merger
+import Assistant.DaemonStatus
 import qualified Command.Sync
 import Utility.ThreadScheduler
 import Utility.Parallel
+import qualified Remote
 
 import Data.Time.Clock
 import qualified Data.Map as M
@@ -24,8 +26,8 @@ thisThread :: ThreadName
 thisThread = "Pusher"
 
 {- This thread retries pushes that failed before. -}
-pushRetryThread :: ThreadState -> FailedPushMap -> IO ()
-pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do
+pushRetryThread :: ThreadState -> DaemonStatusHandle -> FailedPushMap -> IO ()
+pushRetryThread st dstatus pushmap = runEvery (Seconds halfhour) $ do
 	-- We already waited half an hour, now wait until there are failed
 	-- pushes to retry.
 	topush <- getFailedPushesBefore pushmap (fromIntegral halfhour)
@@ -36,13 +38,16 @@ pushRetryThread st pushmap = runEvery (Seconds halfhour) $ do
 			, "failed pushes"
 			]
 		now <- getCurrentTime
-		pushToRemotes thisThread now st (Just pushmap) topush
+		alertWhile dstatus (alert topush) $
+			pushToRemotes thisThread now st (Just pushmap) topush
 	where
 		halfhour = 1800
+		alert rs = activityAlert (Just "Retrying sync") $
+			"with " ++ unwords (map Remote.name rs) ++ ", which failed earlier."
 
 {- This thread pushes git commits out to remotes soon after they are made. -}
 pushThread :: ThreadState -> DaemonStatusHandle -> CommitChan -> FailedPushMap -> IO ()
-pushThread st daemonstatus commitchan pushmap = do
+pushThread st dstatus commitchan pushmap = do
 	runEvery (Seconds 2) $ do
 		-- We already waited two seconds as a simple rate limiter.
 		-- Next, wait until at least one commit has been made
@@ -51,8 +56,9 @@ pushThread st daemonstatus commitchan pushmap = do
 		now <- getCurrentTime
 		if shouldPush now commits
 			then do
-				remotes <- knownRemotes <$> getDaemonStatus daemonstatus
-				pushToRemotes thisThread now st (Just pushmap) remotes
+				remotes <- knownRemotes <$> getDaemonStatus dstatus
+				alertWhile dstatus (syncalert remotes) $
+					pushToRemotes thisThread now st (Just pushmap) remotes
 			else do
 				debug thisThread
 					[ "delaying push of"
@@ -60,6 +66,9 @@ pushThread st daemonstatus commitchan pushmap = do
 					, "commits"
 					]
 				refillCommits commitchan commits
+	where
+		syncalert rs = activityAlert Nothing $
+			"Syncing with " ++ unwords (map Remote.name rs)
 
 {- Decide if now is a good time to push to remotes.
  -
diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs
index 5e27246a02..69610c2a7b 100644
--- a/Assistant/Threads/SanityChecker.hs
+++ b/Assistant/Threads/SanityChecker.hs
@@ -13,6 +13,7 @@ import Assistant.Common
 import Assistant.DaemonStatus
 import Assistant.ThreadedMonad
 import Assistant.Changes
+import Assistant.Alert
 import Assistant.TransferQueue
 import qualified Git.LsFiles
 import Utility.ThreadScheduler
@@ -25,29 +26,34 @@ thisThread = "SanityChecker"
 
 {- This thread wakes up occasionally to make sure the tree is in good shape. -}
 sanityCheckerThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO ()
-sanityCheckerThread st status transferqueue changechan = forever $ do
-	waitForNextCheck status
+sanityCheckerThread st dstatus transferqueue changechan = forever $ do
+	waitForNextCheck dstatus
 
 	debug thisThread ["starting sanity check"]
 
-	modifyDaemonStatus_ status $ \s -> s
-		{ sanityCheckRunning = True }
-
-	now <- getPOSIXTime -- before check started
-	catchIO (check st status transferqueue changechan)
-		(runThreadState st . warning . show)
-
-	modifyDaemonStatus_ status $ \s -> s
-		{ sanityCheckRunning = False
-		, lastSanityCheck = Just now
-		}
+	alertWhile dstatus alert go
 	
 	debug thisThread ["sanity check complete"]
+	where
+		go = do
+			modifyDaemonStatus_ dstatus $ \s -> s
+				{ sanityCheckRunning = True }
+
+			now <- getPOSIXTime -- before check started
+			catchIO (check st dstatus transferqueue changechan)
+				(runThreadState st . warning . show)
+
+			modifyDaemonStatus_ dstatus $ \s -> s
+				{ sanityCheckRunning = False
+				, lastSanityCheck = Just now
+				}
+		alert = activityAlert (Just "Running daily sanity check")
+			"to make sure I've not missed anything."
 
 {- Only run one check per day, from the time of the last check. -}
 waitForNextCheck :: DaemonStatusHandle -> IO ()
-waitForNextCheck status = do
-	v <- lastSanityCheck <$> getDaemonStatus status
+waitForNextCheck dstatus = do
+	v <- lastSanityCheck <$> getDaemonStatus dstatus
 	now <- getPOSIXTime
 	threadDelaySeconds $ Seconds $ calcdelay now v
 	where
@@ -64,10 +70,8 @@ oneDay = 24 * 60 * 60
  - running potentially expensive parts of this check, since remaining in it
  - will block the watcher. -}
 check :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () 
-check st status transferqueue changechan = do
-	g <- runThreadState st $ do
-		showSideAction "Running daily check"
-		fromRepo id
+check st dstatus transferqueue changechan = do
+	g <- runThreadState st $ fromRepo id
 	-- Find old unstaged symlinks, and add them to git.
 	unstaged <- Git.LsFiles.notInRepo False ["."] g
 	now <- getPOSIXTime
@@ -81,9 +85,20 @@ check st status transferqueue changechan = do
 	where
 		toonew timestamp now = now < (realToFrac (timestamp + slop) :: POSIXTime)
 		slop = fromIntegral tenMinutes
-		insanity m = runThreadState st $ warning m
+		insanity msg = do
+			runThreadState st $ warning msg
+			void $ addAlert dstatus $ Alert
+			        { alertClass = Warning
+			        , alertHeader = Just "Fixed a problem"
+			        , alertMessage = StringAlert $ unwords
+					[ "The daily sanity check found and fixed a problem:"
+					, msg
+					, "If these problems persist, consider filing a bug report."
+					]
+			        , alertBlockDisplay = True
+		        }
 		addsymlink file s = do
 			insanity $ "found unstaged symlink: " ++ file
-			Watcher.runHandler thisThread st status
+			Watcher.runHandler thisThread st dstatus
 				transferqueue changechan
 				Watcher.onAddSymlink file s

From d52c93242450c0bd01e7d3c1fdae375806aa6e1f Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 18:07:45 -0400
Subject: [PATCH 4352/8313] moved all alert messages into one file

Makes it easier to edit for consistent voice etc.
---
 Assistant/Alert.hs                   | 52 ++++++++++++++++++++++++++++
 Assistant/Threads/MountWatcher.hs    | 11 +-----
 Assistant/Threads/Pusher.hs          | 10 ++----
 Assistant/Threads/SanityChecker.hs   | 17 ++-------
 Assistant/Threads/TransferScanner.hs | 11 +-----
 Assistant/Threads/Watcher.hs         |  5 +--
 6 files changed, 60 insertions(+), 46 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index f4220eea9a..6b0804fd88 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -9,6 +9,9 @@
 
 module Assistant.Alert where
 
+import Common.Annex
+import qualified Remote
+
 import Yesod
 
 type Widget = forall sub master. GWidget sub master ()
@@ -34,3 +37,52 @@ activityAlert header message = Alert
 	, alertMessage = StringAlert message
 	, alertBlockDisplay = False
 	}
+
+startupScanAlert :: Alert
+startupScanAlert = activityAlert Nothing "Performing startup scan"
+
+pushAlert :: [Remote] -> Alert
+pushAlert rs = activityAlert Nothing $
+	"Syncing with " ++ unwords (map Remote.name rs)
+
+pushRetryAlert :: [Remote] -> Alert
+pushRetryAlert rs = activityAlert (Just "Retrying sync") $
+	"with " ++ unwords (map Remote.name rs) ++ ", which failed earlier."
+
+syncMountAlert :: FilePath -> [Remote] -> Alert
+syncMountAlert dir rs = Alert
+	{ alertClass = Activity
+	, alertHeader = Just $ "Syncing with " ++ unwords (map Remote.name rs)
+	, alertMessage = StringAlert $ unwords
+		["I noticed you plugged in"
+		, dir
+		, " -- let's get it in sync!"
+		]
+	, alertBlockDisplay = True
+        }
+
+scanAlert :: Remote -> Alert
+scanAlert r = Alert
+	{ alertClass = Activity
+	, alertHeader = Just $ "Scanning " ++ Remote.name r
+	, alertMessage = StringAlert $ unwords
+		[ "Ensuring that ", Remote.name r
+		, "is fully in sync." ]
+	, alertBlockDisplay = True
+	}
+
+sanityCheckAlert :: Alert
+sanityCheckAlert = activityAlert (Just "Running daily sanity check")
+	"to make sure I've not missed anything."
+
+sanityCheckFixAlert :: String -> Alert
+sanityCheckFixAlert msg = Alert
+	{ alertClass = Warning
+	, alertHeader = Just "Fixed a problem"
+	, alertMessage = StringAlert $ unwords
+		[ "The daily sanity check found and fixed a problem:"
+		, msg
+		, "If these problems persist, consider filing a bug report."
+		]
+	, alertBlockDisplay = True
+	}
diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs
index 7d0ef5ae40..20862dac12 100644
--- a/Assistant/Threads/MountWatcher.hs
+++ b/Assistant/Threads/MountWatcher.hs
@@ -165,7 +165,7 @@ handleMount st dstatus scanremotes mntent = do
 		branch <- runThreadState st $ Command.Sync.currentBranch
 		let nonspecial = filter (Git.repoIsLocal . Remote.repo) rs
 		unless (null nonspecial) $
-			alertWhile dstatus (syncalert nonspecial) $ do
+			alertWhile dstatus (syncMountAlert dir nonspecial) $ do
 				debug thisThread ["syncing with", show nonspecial]
 				runThreadState st $ manualPull branch nonspecial
 				now <- getCurrentTime	
@@ -173,15 +173,6 @@ handleMount st dstatus scanremotes mntent = do
 		addScanRemotes scanremotes rs
 	where
 		dir = mnt_dir mntent
-		syncalert rs = Alert
-			{ alertClass = Activity
-			, alertHeader = Just $ "Syncing with " ++ unwords (map Remote.name rs)
-			, alertMessage = StringAlert $ unwords
-				["I noticed you plugged in", dir,
-				 " -- let's get it in sync!"]
-			, alertBlockDisplay = True
-			}
-			
 
 {- Finds remotes located underneath the mount point.
  -
diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs
index 27e95a7344..1b0420b9b3 100644
--- a/Assistant/Threads/Pusher.hs
+++ b/Assistant/Threads/Pusher.hs
@@ -17,7 +17,6 @@ import Assistant.DaemonStatus
 import qualified Command.Sync
 import Utility.ThreadScheduler
 import Utility.Parallel
-import qualified Remote
 
 import Data.Time.Clock
 import qualified Data.Map as M
@@ -38,12 +37,10 @@ pushRetryThread st dstatus pushmap = runEvery (Seconds halfhour) $ do
 			, "failed pushes"
 			]
 		now <- getCurrentTime
-		alertWhile dstatus (alert topush) $
+		alertWhile dstatus (pushRetryAlert topush) $
 			pushToRemotes thisThread now st (Just pushmap) topush
 	where
 		halfhour = 1800
-		alert rs = activityAlert (Just "Retrying sync") $
-			"with " ++ unwords (map Remote.name rs) ++ ", which failed earlier."
 
 {- This thread pushes git commits out to remotes soon after they are made. -}
 pushThread :: ThreadState -> DaemonStatusHandle -> CommitChan -> FailedPushMap -> IO ()
@@ -57,7 +54,7 @@ pushThread st dstatus commitchan pushmap = do
 		if shouldPush now commits
 			then do
 				remotes <- knownRemotes <$> getDaemonStatus dstatus
-				alertWhile dstatus (syncalert remotes) $
+				alertWhile dstatus (pushAlert remotes) $
 					pushToRemotes thisThread now st (Just pushmap) remotes
 			else do
 				debug thisThread
@@ -66,9 +63,6 @@ pushThread st dstatus commitchan pushmap = do
 					, "commits"
 					]
 				refillCommits commitchan commits
-	where
-		syncalert rs = activityAlert Nothing $
-			"Syncing with " ++ unwords (map Remote.name rs)
 
 {- Decide if now is a good time to push to remotes.
  -
diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs
index 69610c2a7b..cd5dc06446 100644
--- a/Assistant/Threads/SanityChecker.hs
+++ b/Assistant/Threads/SanityChecker.hs
@@ -31,7 +31,7 @@ sanityCheckerThread st dstatus transferqueue changechan = forever $ do
 
 	debug thisThread ["starting sanity check"]
 
-	alertWhile dstatus alert go
+	alertWhile dstatus sanityCheckAlert go
 	
 	debug thisThread ["sanity check complete"]
 	where
@@ -47,8 +47,6 @@ sanityCheckerThread st dstatus transferqueue changechan = forever $ do
 				{ sanityCheckRunning = False
 				, lastSanityCheck = Just now
 				}
-		alert = activityAlert (Just "Running daily sanity check")
-			"to make sure I've not missed anything."
 
 {- Only run one check per day, from the time of the last check. -}
 waitForNextCheck :: DaemonStatusHandle -> IO ()
@@ -87,18 +85,9 @@ check st dstatus transferqueue changechan = do
 		slop = fromIntegral tenMinutes
 		insanity msg = do
 			runThreadState st $ warning msg
-			void $ addAlert dstatus $ Alert
-			        { alertClass = Warning
-			        , alertHeader = Just "Fixed a problem"
-			        , alertMessage = StringAlert $ unwords
-					[ "The daily sanity check found and fixed a problem:"
-					, msg
-					, "If these problems persist, consider filing a bug report."
-					]
-			        , alertBlockDisplay = True
-		        }
+			void $ addAlert dstatus $ sanityCheckFixAlert msg
 		addsymlink file s = do
-			insanity $ "found unstaged symlink: " ++ file
 			Watcher.runHandler thisThread st dstatus
 				transferqueue changechan
 				Watcher.onAddSymlink file s
+			insanity $ "found unstaged symlink: " ++ file
diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index 1bf8b062fc..1d91a65d48 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -32,18 +32,9 @@ transferScannerThread st dstatus scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
 		liftIO $ debug thisThread ["starting scan of", show r]
-		alertWhile dstatus (scanalert r) $
+		alertWhile dstatus (scanAlert r) $
 			scan st dstatus transferqueue r
 		liftIO $ debug thisThread ["finished scan of", show r]
-	where
-		scanalert r = Alert
-			{ alertClass = Activity
-			, alertHeader = Just $ "Scanning " ++ Remote.name r
-			, alertMessage = StringAlert $ unwords
-				[ "Ensuring that ", Remote.name r
-				, "is fully in sync." ]
-			, alertBlockDisplay = True
-			}
 
 {- This is a naive scan through the git work tree.
  - 
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index ade26be19b..1c8d122d52 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -75,7 +75,7 @@ watchThread st dstatus transferqueue changechan = do
 startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
 startupScan st dstatus scanner = do
 	runThreadState st $ showAction "scanning"
-	r <- alertWhile dstatus alert scanner
+	r <- alertWhile dstatus startupScanAlert scanner
 	modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
 
 	-- Notice any files that were deleted before watching was started.
@@ -84,9 +84,6 @@ startupScan st dstatus scanner = do
 		showAction "started"
 
 	return r
-	
-	where
-		alert = activityAlert Nothing "Performing startup scan"
 
 ignored :: FilePath -> Bool
 ignored = ig . takeFileName

From d62b157194248402b566e96bbc92d19b8e1ce6e8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 19:05:51 -0400
Subject: [PATCH 4353/8313] better ordering of alerts

---
 Assistant/Alert.hs           | 41 ++++++++++++++++++++++++++++++++++--
 Assistant/DaemonStatus.hs    |  1 -
 Assistant/Threads/Watcher.hs | 17 +++++++++------
 Assistant/Threads/WebApp.hs  |  8 +++----
 Utility/Misc.hs              |  7 ++++++
 templates/transfers.hamlet   |  1 -
 6 files changed, 60 insertions(+), 15 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 6b0804fd88..648ea58546 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -17,8 +17,8 @@ import Yesod
 type Widget = forall sub master. GWidget sub master ()
 
 {- Different classes of alerts are displayed differently. -}
-data AlertClass = Activity | Warning | Error | Success | Message
-	deriving (Eq)
+data AlertClass = Success | Message | Activity | Warning | Error
+	deriving (Eq, Ord)
 
 {- An alert can be a simple message, or an arbitrary Yesod Widget -}
 data AlertMessage = StringAlert String | WidgetAlert Widget
@@ -28,19 +28,53 @@ data Alert = Alert
 	, alertHeader :: Maybe String
 	, alertMessage :: AlertMessage
 	, alertBlockDisplay :: Bool
+	, alertPriority :: AlertPriority
 	}
 
+{- Higher AlertId indicates a more recent alert. -}
+type AlertId = Integer
+
+type AlertPair = (AlertId, Alert)
+
+data AlertPriority = Low | Medium | High
+	deriving (Eq, Ord)
+
+{- The desired order is the reverse of:
+ -
+ - - High priority alerts, newest first
+ - - Medium priority Activity, newest first (mostly used for Activity)
+ - - Low priority alwerts, newest first
+ - - Ties are broken by the AlertClass, with Errors etc coming first.
+ -}
+compareAlertPairs :: AlertPair -> AlertPair -> Ordering
+compareAlertPairs
+	(aid, Alert {alertClass = aclass, alertPriority = aprio})
+	(bid, Alert {alertClass = bclass, alertPriority = bprio})
+	 = compare aprio bprio
+		`thenOrd` compare aid bid
+			`thenOrd` compare aclass bclass
+
+sortAlertPairs :: [AlertPair] -> [AlertPair]
+sortAlertPairs = reverse . sortBy compareAlertPairs
+
 activityAlert :: Maybe String -> String -> Alert
 activityAlert header message = Alert
 	{ alertClass = Activity
 	, alertHeader = header
 	, alertMessage = StringAlert message
 	, alertBlockDisplay = False
+	, alertPriority = Medium
 	}
 
 startupScanAlert :: Alert
 startupScanAlert = activityAlert Nothing "Performing startup scan"
 
+runningAlert :: Alert
+runningAlert = (activityAlert Nothing "Running")
+	{ alertClass = Success
+	, alertPriority = High -- pin above the other activity alerts
+	}
+
 pushAlert :: [Remote] -> Alert
 pushAlert rs = activityAlert Nothing $
 	"Syncing with " ++ unwords (map Remote.name rs)
@@ -59,6 +93,7 @@ syncMountAlert dir rs = Alert
 		, " -- let's get it in sync!"
 		]
 	, alertBlockDisplay = True
+	, alertPriority = Low
         }
 
 scanAlert :: Remote -> Alert
@@ -69,6 +104,7 @@ scanAlert r = Alert
 		[ "Ensuring that ", Remote.name r
 		, "is fully in sync." ]
 	, alertBlockDisplay = True
+	, alertPriority = Low
 	}
 
 sanityCheckAlert :: Alert
@@ -85,4 +121,5 @@ sanityCheckFixAlert msg = Alert
 		, "If these problems persist, consider filing a bug report."
 		]
 	, alertBlockDisplay = True
+	, alertPriority = High
 	}
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 62cf2ea2ac..f1b3bdb9fe 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -51,7 +51,6 @@ data DaemonStatus = DaemonStatus
 type TransferMap = M.Map Transfer TransferInfo
 
 type AlertMap = M.Map AlertId Alert
-type AlertId = Integer
 
 {- This TMVar is never left empty, so accessing it will never block. -}
 type DaemonStatusHandle = TMVar DaemonStatus
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 1c8d122d52..ddbd51655f 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -75,13 +75,18 @@ watchThread st dstatus transferqueue changechan = do
 startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
 startupScan st dstatus scanner = do
 	runThreadState st $ showAction "scanning"
-	r <- alertWhile dstatus startupScanAlert scanner
-	modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
+	r <- alertWhile dstatus startupScanAlert $ do
+		r <- scanner
+		modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
 
-	-- Notice any files that were deleted before watching was started.
-	runThreadState st $ do
-		inRepo $ Git.Command.run "add" [Param "--update"]
-		showAction "started"
+		-- Notice any files that were deleted before
+		-- watching was started.
+		runThreadState st $ do
+			inRepo $ Git.Command.run "add" [Param "--update"]
+			showAction "started"
+		return r
+	
+	void $ addAlert dstatus runningAlert
 
 	return r
 
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 3d42db8125..4d37a941ad 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -34,7 +34,6 @@ import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text (Text, pack, unpack)
 import qualified Data.Map as M
-import Data.Function
 
 thisThread :: String
 thisThread = "WebApp"
@@ -158,10 +157,9 @@ sideBarDisplay noScript = do
 	
 		{- Add newest 10 alerts to the sidebar. -}
 		webapp <- lift getYesod
-		alerts <- M.toList . alertMap
+		alertpairs <- M.toList . alertMap
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
-		mapM_ renderalert $
-			take 10 $ reverse $ sortBy (compare `on` fst) alerts
+		mapM_ renderalert $ take 10 $ sortAlertPairs alertpairs
 	ident <- lift newIdent
 	$(widgetFile "sidebar")
 
@@ -180,7 +178,7 @@ sideBarDisplay noScript = do
 		renderalert (alertid, alert) = addalert
 			(show alertid)
 			-- Activity alerts auto-close
-			(not noScript && alertClass alert /= Activity)
+			(alertClass alert /= Activity)
 			(alertBlockDisplay alert)
 			(bootstrapclass $ alertClass alert)
 			(alertHeader alert)
diff --git a/Utility/Misc.hs b/Utility/Misc.hs
index e11586467d..77ebb4f3d9 100644
--- a/Utility/Misc.hs
+++ b/Utility/Misc.hs
@@ -45,3 +45,10 @@ segment p l = map reverse $ go [] [] l
 		go c r (i:is)
 			| p i = go [] (c:r) is
 			| otherwise = go (i:c) r is
+
+{- Given two orderings, returns the second if the first is EQ and returns
+ - the first otherwise. -}
+thenOrd :: Ordering -> Ordering -> Ordering
+thenOrd EQ x = x
+thenOrd x _ = x
+{-# INLINE thenOrd #-}
diff --git a/templates/transfers.hamlet b/templates/transfers.hamlet
index bc69d7f876..e79885fb54 100644
--- a/templates/transfers.hamlet
+++ b/templates/transfers.hamlet
@@ -1,6 +1,5 @@
 <div .span9 ##{ident}>
   $if null transfers
-    <h2>No current transfers
   $else
     <h2>Transfers
     $forall (transfer, info) <- transfers

From 0186f06744e6c379d41c482f42374853bd3c5539 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 19:41:17 -0400
Subject: [PATCH 4354/8313] tweak Alert closability and construction

---
 Assistant/Alert.hs          | 35 ++++++++++++++++++++++-------------
 Assistant/Threads/WebApp.hs |  3 +--
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 648ea58546..78771b1ea7 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -28,6 +28,7 @@ data Alert = Alert
 	, alertHeader :: Maybe String
 	, alertMessage :: AlertMessage
 	, alertBlockDisplay :: Bool
+	, alertClosable :: Bool
 	, alertPriority :: AlertPriority
 	}
 
@@ -36,11 +37,12 @@ type AlertId = Integer
 
 type AlertPair = (AlertId, Alert)
 
-data AlertPriority = Low | Medium | High
+data AlertPriority = Low | Medium | High | Pinned
 	deriving (Eq, Ord)
 
 {- The desired order is the reverse of:
  -
+ - - Pinned alerts
  - - High priority alerts, newest first
  - - Medium priority Activity, newest first (mostly used for Activity)
  - - Low priority alwerts, newest first
@@ -57,22 +59,30 @@ compareAlertPairs
 sortAlertPairs :: [AlertPair] -> [AlertPair]
 sortAlertPairs = reverse . sortBy compareAlertPairs
 
-activityAlert :: Maybe String -> String -> Alert
-activityAlert header message = Alert
+baseActivityAlert :: Alert
+baseActivityAlert = Alert
 	{ alertClass = Activity
-	, alertHeader = header
-	, alertMessage = StringAlert message
+	, alertHeader = Nothing
+	, alertMessage = StringAlert ""
 	, alertBlockDisplay = False
+	, alertClosable = False
 	, alertPriority = Medium
 	}
 
+activityAlert :: Maybe String -> String -> Alert
+activityAlert header message = baseActivityAlert
+	{ alertHeader = header
+	, alertMessage = StringAlert message
+	}
+
 startupScanAlert :: Alert
 startupScanAlert = activityAlert Nothing "Performing startup scan"
 
 runningAlert :: Alert
-runningAlert = (activityAlert Nothing "Running")
+runningAlert = baseActivityAlert
 	{ alertClass = Success
-	, alertPriority = High -- pin above the other activity alerts
+	, alertMessage = StringAlert "Running"
+	, alertPriority = Pinned
 	}
 
 pushAlert :: [Remote] -> Alert
@@ -84,9 +94,8 @@ pushRetryAlert rs = activityAlert (Just "Retrying sync") $
 	"with " ++ unwords (map Remote.name rs) ++ ", which failed earlier."
 
 syncMountAlert :: FilePath -> [Remote] -> Alert
-syncMountAlert dir rs = Alert
-	{ alertClass = Activity
-	, alertHeader = Just $ "Syncing with " ++ unwords (map Remote.name rs)
+syncMountAlert dir rs = baseActivityAlert
+	{ alertHeader = Just $ "Syncing with " ++ unwords (map Remote.name rs)
 	, alertMessage = StringAlert $ unwords
 		["I noticed you plugged in"
 		, dir
@@ -97,9 +106,8 @@ syncMountAlert dir rs = Alert
         }
 
 scanAlert :: Remote -> Alert
-scanAlert r = Alert
-	{ alertClass = Activity
-	, alertHeader = Just $ "Scanning " ++ Remote.name r
+scanAlert r = baseActivityAlert
+	{ alertHeader = Just $ "Scanning " ++ Remote.name r
 	, alertMessage = StringAlert $ unwords
 		[ "Ensuring that ", Remote.name r
 		, "is fully in sync." ]
@@ -122,4 +130,5 @@ sanityCheckFixAlert msg = Alert
 		]
 	, alertBlockDisplay = True
 	, alertPriority = High
+	, alertClosable = True
 	}
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 4d37a941ad..84b9bcd204 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -177,8 +177,7 @@ sideBarDisplay noScript = do
 
 		renderalert (alertid, alert) = addalert
 			(show alertid)
-			-- Activity alerts auto-close
-			(alertClass alert /= Activity)
+			(alertClosable alert)
 			(alertBlockDisplay alert)
 			(bootstrapclass $ alertClass alert)
 			(alertHeader alert)

From 326617ad2f6c1708bc2826ba75cb8f9c3064d6dc Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 21:54:23 -0400
Subject: [PATCH 4355/8313] add intro

---
 Assistant/Threads/WebApp.hs | 66 +++++++++++++++++++++++++++++++++----
 Remote.hs                   | 10 ++++++
 templates/bootstrap.hamlet  |  2 +-
 templates/intro.hamlet      | 23 +++++++++++++
 templates/page.hamlet       |  6 ++--
 5 files changed, 97 insertions(+), 10 deletions(-)
 create mode 100644 templates/intro.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 84b9bcd204..daddbc28cb 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -26,6 +26,7 @@ import Utility.Percentage
 import Utility.DataUnits
 import Types.Key
 import qualified Remote
+import Logs.Web (webUUID)
 
 import Yesod
 import Yesod.Static
@@ -34,6 +35,7 @@ import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text (Text, pack, unpack)
 import qualified Data.Map as M
+import Control.Concurrent.STM
 
 thisThread :: String
 thisThread = "WebApp"
@@ -43,10 +45,29 @@ data WebApp = WebApp
 	, daemonStatus :: DaemonStatusHandle
 	, transferQueue :: TransferQueue
 	, secretToken :: Text
-	, baseTitle :: String
+	, relDir :: FilePath
 	, getStatic :: Static
+	, webAppState :: TMVar WebAppState
 	}
 
+data WebAppState = WebAppState
+	{ showIntro :: Bool
+	}
+
+newWebAppState :: IO (TMVar WebAppState)
+newWebAppState = liftIO $ atomically $
+	newTMVar $ WebAppState { showIntro = True }
+
+getWebAppState :: forall sub. GHandler sub WebApp WebAppState
+getWebAppState = liftIO . atomically . readTMVar =<< webAppState <$> getYesod
+
+modifyWebAppState :: forall sub. (WebAppState -> WebAppState) -> GHandler sub WebApp ()
+modifyWebAppState a = go =<< webAppState <$> getYesod
+	where
+		go s = liftIO $ atomically $ do
+			v <- takeTMVar s
+			putTMVar s $ a v
+
 waitNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> NotificationId -> GHandler sub WebApp ()
 waitNotifier selector nid = do
 	notifier <- getNotifier selector
@@ -71,6 +92,7 @@ mkYesod "WebApp" [parseRoutes|
 /transfers/#NotificationId TransfersR GET
 /sidebar/#NotificationId SideBarR GET
 /config ConfigR GET
+/addrepository AddRepositoryR GET
 /static StaticR Static getStatic
 |]
 
@@ -119,7 +141,9 @@ autoUpdate ident gethtml ms_delay ms_startdelay = do
 	let startdelay = show ms_startdelay
 	$(widgetFile "longpolling")
 
-{- A display of currently running and queued transfers. -}
+{- A display of currently running and queued transfers.
+ -
+ - Or, if there have never been any this run, an intro display. -}
 transfersDisplay :: Bool -> Widget
 transfersDisplay warnNoScript = do
 	webapp <- lift getYesod
@@ -127,13 +151,35 @@ transfersDisplay warnNoScript = do
 		M.toList . currentTransfers
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
 	queued <- liftIO $ getTransferQueue $ transferQueue webapp
-	let transfers = current ++ queued
 	let ident = transfersDisplayIdent
-	$(widgetFile "transfers")
+	let transfers = current ++ queued
+	if null transfers
+		then ifM (lift $ showIntro <$> getWebAppState)
+			( introDisplay ident
+			, noop
+			)
+		else do
+			lift $ modifyWebAppState $ \s -> s { showIntro = False }
+			$(widgetFile "transfers")
 
 transfersDisplayIdent :: Text
 transfersDisplayIdent = "transfers"
 
+introDisplay :: Text -> Widget
+introDisplay ident = do
+	webapp <- lift getYesod
+	let reldir = relDir webapp
+	remotelist <- liftIO $ runThreadState (threadState webapp) $
+		Remote.prettyListUUIDs
+			=<< filter (/= webUUID) . nub . map Remote.uuid
+				<$> Remote.remoteList
+	let n = (length remotelist) + 1 -- plus this one
+	let numrepos = show n
+	let notenough = n < 2
+	let barelyenough = n == 2
+	let morethanenough = n > 2
+	$(widgetFile "intro")
+
 {- Called by client to get a display of currently in process transfers.
  -
  - Returns a div, which will be inserted into the calling page.
@@ -237,7 +283,13 @@ getNoScriptR = defaultLayout $
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout $ do
 	sideBarDisplay False
-	setTitle "configuration"
+	setTitle "Configuration"
+	[whamlet|<a href="@{HomeR}">main|]
+
+getAddRepositoryR :: Handler RepHtml
+getAddRepositoryR = defaultLayout $ do
+	sideBarDisplay False
+	setTitle "Add repository"
 	[whamlet|<a href="@{HomeR}">main|]
 
 webAppThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Maybe (IO ()) -> IO ()
@@ -259,13 +311,15 @@ webAppThread st dstatus transferqueue onstartup = do
 				then relPathDirToFile home dir
 				else dir
 			token <- genRandomToken 
+			s <- newWebAppState
 			return $ WebApp 
 				{ threadState = st
 				, daemonStatus = dstatus
 				, transferQueue = transferqueue
 				, secretToken = pack token
-				, baseTitle = reldir
+				, relDir = reldir
 				, getStatic = $(embed "static")
+				, webAppState = s
 				}
 
 {- Creates a html shim file that's used to redirect into the webapp,
diff --git a/Remote.hs b/Remote.hs
index e211ef7cb6..bb582778fd 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -24,6 +24,7 @@ module Remote (
 	uuidDescriptions,
 	byName,
 	prettyPrintUUIDs,
+	prettyListUUIDs,
 	remotesWithUUID,
 	remotesWithoutUUID,
 	keyLocations,
@@ -128,6 +129,15 @@ prettyPrintUUIDs desc uuids = do
 			, ("here", toJSON $ hereu == u)
 			]
 
+{- List of remote names and/or descriptions, for human display. 
+ - Omits the current repisitory. -}
+prettyListUUIDs :: [UUID] -> Annex [String]
+prettyListUUIDs uuids = do
+	hereu <- getUUID
+	m <- uuidDescriptions
+	return $ map (\u -> M.findWithDefault "" u m) $ 
+		filter (/= hereu) uuids
+
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
 remotesWithUUID rs us = filter (\r -> uuid r `elem` us) rs
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
index 389895df74..13aefd486a 100644
--- a/templates/bootstrap.hamlet
+++ b/templates/bootstrap.hamlet
@@ -1,7 +1,7 @@
 $doctype 5
 <html>
   <head>
-    <title>#{baseTitle webapp} #{pageTitle page}
+    <title>#{relDir webapp} #{pageTitle page}
     <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
     <meta name="viewport" content="width=device-width,initial-scale=1.0">
     ^{pageHead page}
diff --git a/templates/intro.hamlet b/templates/intro.hamlet
new file mode 100644
index 0000000000..ef82df79b8
--- /dev/null
+++ b/templates/intro.hamlet
@@ -0,0 +1,23 @@
+<div .span9 ##{ident} .hero-unit>
+  <h2>
+    git-annex is watching over your files in <small><tt>#{reldir}</tt></small>
+  <p>
+    It will automatically notice changes, and keep files in sync between #
+    $if notenough
+      repositories on your devices ...
+      <h2>
+        But no other repositories are set up yet.
+      <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
+    $else
+      these #
+      $if barelyenough
+        <span .badge .badge-warning>#{numrepos}</span>
+      $else
+        <span .badge .badge-success>#{numrepos}</span>
+      \ repositories and devices:
+      <ul>
+        $forall name <- remotelist
+          <li>#{name}
+      <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
+      <div>
+        Or just sit back, watch the magic, and get on with using your files.
diff --git a/templates/page.hamlet b/templates/page.hamlet
index c397d248c2..5004241257 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -11,12 +11,12 @@
       <ul .nav .pull-right>
         <li .dropdown #menu1>
           <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
-            Current Repository: #{baseTitle webapp}
+            Current Repository: #{relDir webapp}
             <b .caret></b>
           <ul .dropdown-menu>
-             <li><a href="#">#{baseTitle webapp}</a></li>
+             <li><a href="#">#{relDir webapp}</a></li>
              <li .divider></li>
-             <li><a href="#">Add new repository</a></li>
+             <li><a href="@{AddRepositoryR}">Add new repository</a></li>
 
 <div .container-fluid>
   <div .row-fluid>

From 675ad9fe226e99ca9ee2defa7ba6b0489123f0dc Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 21:58:01 -0400
Subject: [PATCH 4356/8313] tweak

---
 templates/intro.hamlet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/intro.hamlet b/templates/intro.hamlet
index ef82df79b8..d62bfda9ab 100644
--- a/templates/intro.hamlet
+++ b/templates/intro.hamlet
@@ -9,13 +9,13 @@
         But no other repositories are set up yet.
       <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
     $else
-      these #
       $if barelyenough
         <span .badge .badge-warning>#{numrepos}</span>
       $else
         <span .badge .badge-success>#{numrepos}</span>
       \ repositories and devices:
       <ul>
+        <li>here
         $forall name <- remotelist
           <li>#{name}
       <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>

From 895b068e350977cec1e31d49bd87b13b0a5676d9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 22:11:01 -0400
Subject: [PATCH 4357/8313] tweak intro

---
 Assistant/Threads/WebApp.hs | 11 ++++++-----
 Remote.hs                   | 13 +++++++++----
 templates/intro.hamlet      |  3 +--
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index daddbc28cb..40f8300e53 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -27,6 +27,7 @@ import Utility.DataUnits
 import Types.Key
 import qualified Remote
 import Logs.Web (webUUID)
+import Annex.UUID (getUUID)
 
 import Yesod
 import Yesod.Static
@@ -169,11 +170,11 @@ introDisplay :: Text -> Widget
 introDisplay ident = do
 	webapp <- lift getYesod
 	let reldir = relDir webapp
-	remotelist <- liftIO $ runThreadState (threadState webapp) $
-		Remote.prettyListUUIDs
-			=<< filter (/= webUUID) . nub . map Remote.uuid
-				<$> Remote.remoteList
-	let n = (length remotelist) + 1 -- plus this one
+	remotelist <- liftIO $ runThreadState (threadState webapp) $ do
+		u <- getUUID
+		rs <- map Remote.uuid <$> Remote.remoteList
+		Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs
+	let n = length remotelist
 	let numrepos = show n
 	let notenough = n < 2
 	let barelyenough = n == 2
diff --git a/Remote.hs b/Remote.hs
index bb582778fd..ac9da05365 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -129,14 +129,19 @@ prettyPrintUUIDs desc uuids = do
 			, ("here", toJSON $ hereu == u)
 			]
 
-{- List of remote names and/or descriptions, for human display. 
- - Omits the current repisitory. -}
+{- List of remote names and/or descriptions, for human display.  -}
 prettyListUUIDs :: [UUID] -> Annex [String]
 prettyListUUIDs uuids = do
 	hereu <- getUUID
 	m <- uuidDescriptions
-	return $ map (\u -> M.findWithDefault "" u m) $ 
-		filter (/= hereu) uuids
+	return $ map (\u -> prettify m hereu u) uuids
+	where
+		finddescription m u = M.findWithDefault "" u m
+		prettify m hereu u
+			| u == hereu = addName n "here"
+			| otherwise = n
+			where
+				n = finddescription m u
 
 {- Filters a list of remotes to ones that have the listed uuids. -}
 remotesWithUUID :: [Remote] -> [UUID] -> [Remote]
diff --git a/templates/intro.hamlet b/templates/intro.hamlet
index d62bfda9ab..6660b6a039 100644
--- a/templates/intro.hamlet
+++ b/templates/intro.hamlet
@@ -15,9 +15,8 @@
         <span .badge .badge-success>#{numrepos}</span>
       \ repositories and devices:
       <ul>
-        <li>here
         $forall name <- remotelist
           <li>#{name}
       <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
-      <div>
+      <p>
         Or just sit back, watch the magic, and get on with using your files.

From 74cf65a4dd5f7a59844184a4ece3279adc97d3a7 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Sun, 29 Jul 2012 22:18:58 -0400
Subject: [PATCH 4358/8313] avoid first person

---
 Assistant/Alert.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 78771b1ea7..49ad515ade 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -97,7 +97,7 @@ syncMountAlert :: FilePath -> [Remote] -> Alert
 syncMountAlert dir rs = baseActivityAlert
 	{ alertHeader = Just $ "Syncing with " ++ unwords (map Remote.name rs)
 	, alertMessage = StringAlert $ unwords
-		["I noticed you plugged in"
+		["You plugged in"
 		, dir
 		, " -- let's get it in sync!"
 		]
@@ -117,7 +117,7 @@ scanAlert r = baseActivityAlert
 
 sanityCheckAlert :: Alert
 sanityCheckAlert = activityAlert (Just "Running daily sanity check")
-	"to make sure I've not missed anything."
+	"to make sure everything is ok."
 
 sanityCheckFixAlert :: String -> Alert
 sanityCheckFixAlert msg = Alert

From ec0493fa4d45a8d8f6617c906727d653afb1c50e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 00:31:33 -0400
Subject: [PATCH 4359/8313] filter out dead repos from the intro

---
 Assistant/Threads/WebApp.hs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 40f8300e53..7ad40c3079 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -27,6 +27,7 @@ import Utility.DataUnits
 import Types.Key
 import qualified Remote
 import Logs.Web (webUUID)
+import Logs.Trust
 import Annex.UUID (getUUID)
 
 import Yesod
@@ -173,7 +174,8 @@ introDisplay ident = do
 	remotelist <- liftIO $ runThreadState (threadState webapp) $ do
 		u <- getUUID
 		rs <- map Remote.uuid <$> Remote.remoteList
-		Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs
+		rs' <- snd <$> trustPartition DeadTrusted rs
+		Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs'
 	let n = length remotelist
 	let numrepos = show n
 	let notenough = n < 2

From 3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 02:07:02 -0400
Subject: [PATCH 4360/8313] make old activiy alerts stay visible

They're updated to show whether the activity succeeded or failed.

This adds several TODOs to the code to fix later.
---
 Assistant/Alert.hs                   | 34 ++++++++++++++++++++++++----
 Assistant/DaemonStatus.hs            | 29 +++++++++++++++++++-----
 Assistant/Threads/MountWatcher.hs    |  2 +-
 Assistant/Threads/Pusher.hs          | 17 ++++++++++----
 Assistant/Threads/SanityChecker.hs   | 13 +++++++----
 Assistant/Threads/TransferScanner.hs |  5 ++--
 Assistant/Threads/Watcher.hs         | 13 +++++------
 Assistant/Threads/WebApp.hs          |  2 +-
 8 files changed, 85 insertions(+), 30 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 49ad515ade..23a93b1c16 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -20,8 +20,8 @@ type Widget = forall sub master. GWidget sub master ()
 data AlertClass = Success | Message | Activity | Warning | Error
 	deriving (Eq, Ord)
 
-{- An alert can be a simple message, or an arbitrary Yesod Widget -}
-data AlertMessage = StringAlert String | WidgetAlert Widget
+{- An alert can be a simple message, or an arbitrary Yesod Widget. -}
+data AlertMessage = StringAlert String | WidgetAlert (Alert -> Widget)
 
 data Alert = Alert
 	{ alertClass :: AlertClass
@@ -37,7 +37,7 @@ type AlertId = Integer
 
 type AlertPair = (AlertId, Alert)
 
-data AlertPriority = Low | Medium | High | Pinned
+data AlertPriority = Filler | Low | Medium | High | Pinned
 	deriving (Eq, Ord)
 
 {- The desired order is the reverse of:
@@ -45,7 +45,8 @@ data AlertPriority = Low | Medium | High | Pinned
  - - Pinned alerts
  - - High priority alerts, newest first
  - - Medium priority Activity, newest first (mostly used for Activity)
- - - Low priority alwerts, newest first
+ - - Low priority alerts, newest first
+ - - Filler priorty alerts, newest first
  - - Ties are broken by the AlertClass, with Errors etc coming first.
  -}
 compareAlertPairs :: AlertPair -> AlertPair -> Ordering
@@ -56,6 +57,31 @@ compareAlertPairs
 		`thenOrd` compare aid bid
 			`thenOrd` compare aclass bclass
 
+makeAlertFiller :: Bool -> Alert -> Alert
+makeAlertFiller success alert
+	| alertPriority alert == Filler = alert
+	| otherwise = alert
+		{ alertClass = if c == Activity then c' else c
+		, alertPriority = Filler
+		, alertHeader = finished <$> h
+		, alertMessage = massage m
+		}
+	where
+		h = alertHeader alert
+		m = alertMessage alert
+		c = alertClass alert
+		c'
+			| success = Success
+			| otherwise = Error
+
+		massage (WidgetAlert w) = WidgetAlert w -- renders old on its own
+		massage (StringAlert s) = StringAlert $
+			maybe (finished s) (const s) h
+
+		finished s
+			| success = s ++ ": Succeeded"
+			| otherwise = s ++ ": Failed"
+
 sortAlertPairs :: [AlertPair] -> [AlertPair]
 sortAlertPairs = reverse . sortBy compareAlertPairs
 
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index f1b3bdb9fe..6d05c61528 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -223,12 +223,29 @@ addAlert dstatus alert = notifyAlert dstatus `after` modifyDaemonStatus dstatus
 				m = M.insertWith' const i alert (alertMap s)
 
 removeAlert :: DaemonStatusHandle -> AlertId -> IO ()
-removeAlert dstatus i = notifyAlert dstatus `after` modifyDaemonStatus_ dstatus go
-	where
-		go s = s { alertMap = M.delete i (alertMap s) }
+removeAlert dstatus i = updateAlert dstatus i (const Nothing)
 
-{- Displays an alert while performing an activity, then removes it. -}
-alertWhile :: DaemonStatusHandle -> Alert -> IO a -> IO a
+updateAlert :: DaemonStatusHandle -> AlertId -> (Alert -> Maybe Alert) -> IO ()
+updateAlert dstatus i a = updateAlertMap dstatus $ \m -> M.update a i m
+
+updateAlertMap :: DaemonStatusHandle -> (AlertMap -> AlertMap) -> IO ()
+updateAlertMap dstatus a = notifyAlert dstatus `after` modifyDaemonStatus_ dstatus go
+	where
+		go s = s { alertMap = a (alertMap s) }
+
+{- Displays an alert while performing an activity.
+ -
+ - The alert is left visible afterwards, as filler.
+ - Old filler is pruned, to prevent the map growing too large. -}
+alertWhile :: DaemonStatusHandle -> Alert -> IO Bool -> IO Bool
 alertWhile dstatus alert a = do
 	let alert' = alert { alertClass = Activity }
-	bracket (addAlert dstatus alert') (removeAlert dstatus) (const a)
+	i <- addAlert dstatus alert'
+	r <- bracket_ noop noop a
+	updateAlertMap dstatus $ makeold i (makeAlertFiller r)
+	return r
+	where
+		-- TODO prune old filler
+		makeold i filler m
+			| M.size m < 20 = M.adjust filler i m
+			| otherwise = M.adjust filler i m
diff --git a/Assistant/Threads/MountWatcher.hs b/Assistant/Threads/MountWatcher.hs
index 20862dac12..4baef1d11a 100644
--- a/Assistant/Threads/MountWatcher.hs
+++ b/Assistant/Threads/MountWatcher.hs
@@ -165,7 +165,7 @@ handleMount st dstatus scanremotes mntent = do
 		branch <- runThreadState st $ Command.Sync.currentBranch
 		let nonspecial = filter (Git.repoIsLocal . Remote.repo) rs
 		unless (null nonspecial) $
-			alertWhile dstatus (syncMountAlert dir nonspecial) $ do
+			void $ alertWhile dstatus (syncMountAlert dir nonspecial) $ do
 				debug thisThread ["syncing with", show nonspecial]
 				runThreadState st $ manualPull branch nonspecial
 				now <- getCurrentTime	
diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs
index 1b0420b9b3..0a0edf1d0e 100644
--- a/Assistant/Threads/Pusher.hs
+++ b/Assistant/Threads/Pusher.hs
@@ -37,7 +37,7 @@ pushRetryThread st dstatus pushmap = runEvery (Seconds halfhour) $ do
 			, "failed pushes"
 			]
 		now <- getCurrentTime
-		alertWhile dstatus (pushRetryAlert topush) $
+		void $ alertWhile dstatus (pushRetryAlert topush) $
 			pushToRemotes thisThread now st (Just pushmap) topush
 	where
 		halfhour = 1800
@@ -54,7 +54,7 @@ pushThread st dstatus commitchan pushmap = do
 		if shouldPush now commits
 			then do
 				remotes <- knownRemotes <$> getDaemonStatus dstatus
-				alertWhile dstatus (pushAlert remotes) $
+				void $ alertWhile dstatus (pushAlert remotes) $
 					pushToRemotes thisThread now st (Just pushmap) remotes
 			else do
 				debug thisThread
@@ -80,7 +80,7 @@ shouldPush _now commits
  -
  - Avoids running possibly long-duration commands in the Annex monad, so
  - as not to block other threads. -}
-pushToRemotes :: ThreadName -> UTCTime -> ThreadState -> (Maybe FailedPushMap) -> [Remote] -> IO ()
+pushToRemotes :: ThreadName -> UTCTime -> ThreadState -> (Maybe FailedPushMap) -> [Remote] -> IO Bool
 pushToRemotes threadname now st mpushmap remotes = do
 	(g, branch) <- runThreadState st $
 		(,) <$> fromRepo id <*> Command.Sync.currentBranch
@@ -92,6 +92,11 @@ pushToRemotes threadname now st mpushmap remotes = do
 				, show rs
 				]
 			Command.Sync.updateBranch (Command.Sync.syncBranch branch) g
+			{- TODO git push exits nonzero if the remote
+			 - is already up-to-date. This code does not tell
+			 - the difference between the two. Could perhaps
+			 - be check the refs when it seemed to fail?
+			 - Note bewloe  -}
 			(succeeded, failed) <- inParallel (push g branch) rs
 			case mpushmap of
 				Nothing -> noop
@@ -104,8 +109,10 @@ pushToRemotes threadname now st mpushmap remotes = do
 					[ "failed to push to"
 					, show failed
 					]
-			unless (null failed || not shouldretry) $
-				retry branch g failed
+			if (null failed || not shouldretry)
+				{- TODO see above TODO item -}
+				then return True -- return $ null failed 
+				else retry branch g failed
 
 		makemap l = M.fromList $ zip l (repeat now)
 
diff --git a/Assistant/Threads/SanityChecker.hs b/Assistant/Threads/SanityChecker.hs
index cd5dc06446..a7c2189d80 100644
--- a/Assistant/Threads/SanityChecker.hs
+++ b/Assistant/Threads/SanityChecker.hs
@@ -31,7 +31,7 @@ sanityCheckerThread st dstatus transferqueue changechan = forever $ do
 
 	debug thisThread ["starting sanity check"]
 
-	alertWhile dstatus sanityCheckAlert go
+	void $ alertWhile dstatus sanityCheckAlert go
 	
 	debug thisThread ["sanity check complete"]
 	where
@@ -40,14 +40,18 @@ sanityCheckerThread st dstatus transferqueue changechan = forever $ do
 				{ sanityCheckRunning = True }
 
 			now <- getPOSIXTime -- before check started
-			catchIO (check st dstatus transferqueue changechan)
-				(runThreadState st . warning . show)
+			r <- catchIO (check st dstatus transferqueue changechan)
+				$ \e -> do
+					runThreadState st $ warning $ show e
+					return False
 
 			modifyDaemonStatus_ dstatus $ \s -> s
 				{ sanityCheckRunning = False
 				, lastSanityCheck = Just now
 				}
 
+			return r
+
 {- Only run one check per day, from the time of the last check. -}
 waitForNextCheck :: DaemonStatusHandle -> IO ()
 waitForNextCheck dstatus = do
@@ -67,7 +71,7 @@ oneDay = 24 * 60 * 60
 {- It's important to stay out of the Annex monad as much as possible while
  - running potentially expensive parts of this check, since remaining in it
  - will block the watcher. -}
-check :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO () 
+check :: ThreadState -> DaemonStatusHandle -> TransferQueue -> ChangeChan -> IO Bool
 check st dstatus transferqueue changechan = do
 	g <- runThreadState st $ fromRepo id
 	-- Find old unstaged symlinks, and add them to git.
@@ -80,6 +84,7 @@ check st dstatus transferqueue changechan = do
 				| isSymbolicLink s ->
 					addsymlink file ms
 			_ -> noop
+	return True
 	where
 		toonew timestamp now = now < (realToFrac (timestamp + slop) :: POSIXTime)
 		slop = fromIntegral tenMinutes
diff --git a/Assistant/Threads/TransferScanner.hs b/Assistant/Threads/TransferScanner.hs
index 1d91a65d48..2cba0b2a78 100644
--- a/Assistant/Threads/TransferScanner.hs
+++ b/Assistant/Threads/TransferScanner.hs
@@ -32,18 +32,19 @@ transferScannerThread st dstatus scanremotes transferqueue = do
 	runEvery (Seconds 2) $ do
 		r <- getScanRemote scanremotes
 		liftIO $ debug thisThread ["starting scan of", show r]
-		alertWhile dstatus (scanAlert r) $
+		void $ alertWhile dstatus (scanAlert r) $
 			scan st dstatus transferqueue r
 		liftIO $ debug thisThread ["finished scan of", show r]
 
 {- This is a naive scan through the git work tree.
  - 
  - The scan is blocked when the transfer queue gets too large. -}
-scan :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Remote -> IO ()
+scan :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Remote -> IO Bool
 scan st dstatus transferqueue r = do
 	g <- runThreadState st $ fromRepo id
 	files <- LsFiles.inRepo [] g
 	go files
+	return True
 	where
 		go [] = return ()
 		go (f:fs) = do
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index ddbd51655f..bfeec7630c 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -72,24 +72,23 @@ watchThread st dstatus transferqueue changechan = do
 			}
 
 {- Initial scartup scan. The action should return once the scan is complete. -}
-startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
+startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO ()
 startupScan st dstatus scanner = do
 	runThreadState st $ showAction "scanning"
-	r <- alertWhile dstatus startupScanAlert $ do
-		r <- scanner
-		modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
+	void $ alertWhile dstatus startupScanAlert $ do
+		void $ scanner
 
 		-- Notice any files that were deleted before
 		-- watching was started.
 		runThreadState st $ do
 			inRepo $ Git.Command.run "add" [Param "--update"]
 			showAction "started"
-		return r
+		
+		modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
+		return True
 	
 	void $ addAlert dstatus runningAlert
 
-	return r
-
 ignored :: FilePath -> Bool
 ignored = ig . takeFileName
 	where
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 7ad40c3079..d268559108 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -232,7 +232,7 @@ sideBarDisplay noScript = do
 			(alertHeader alert)
 			$ case alertMessage alert of
 				StringAlert s -> [whamlet|#{s}|]
-				WidgetAlert w -> w
+				WidgetAlert w -> w alert
 
 		rendermessage msg = addalert "yesodmessage" True False
 			"alert-info" Nothing [whamlet|#{msg}|]

From 40c997367544d72c6ab55eb96a1c3344fcf4012c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 11:52:44 -0400
Subject: [PATCH 4361/8313] fix push status, broken when inParallel was adapted
 for -threaded

Before pushing ran in its own process, so exitSuccess was the right thing
to do, but with the threaded code, that's caught as an exception.
---
 Assistant/Threads/Pusher.hs | 17 +++++------------
 Utility/Parallel.hs         |  6 +++---
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs
index 0a0edf1d0e..3fe85673bc 100644
--- a/Assistant/Threads/Pusher.hs
+++ b/Assistant/Threads/Pusher.hs
@@ -92,33 +92,26 @@ pushToRemotes threadname now st mpushmap remotes = do
 				, show rs
 				]
 			Command.Sync.updateBranch (Command.Sync.syncBranch branch) g
-			{- TODO git push exits nonzero if the remote
-			 - is already up-to-date. This code does not tell
-			 - the difference between the two. Could perhaps
-			 - be check the refs when it seemed to fail?
-			 - Note bewloe  -}
 			(succeeded, failed) <- inParallel (push g branch) rs
+			let ok = null failed
 			case mpushmap of
 				Nothing -> noop
 				Just pushmap -> 
 					changeFailedPushMap pushmap $ \m ->
 						M.union (makemap failed) $
 							M.difference m (makemap succeeded)
-			unless (null failed) $
+			unless (ok) $
 				debug threadname
 					[ "failed to push to"
 					, show failed
 					]
-			if (null failed || not shouldretry)
-				{- TODO see above TODO item -}
-				then return True -- return $ null failed 
+			if (ok || not shouldretry)
+				then return ok
 				else retry branch g failed
 
 		makemap l = M.fromList $ zip l (repeat now)
 
-		push g branch remote =
-			ifM (Command.Sync.pushBranch remote branch g)
-				( exitSuccess, exitFailure)
+		push g branch remote = Command.Sync.pushBranch remote branch g
 
 		retry branch g rs = do
 			debug threadname [ "trying manual pull to resolve failed pushes" ]
diff --git a/Utility/Parallel.hs b/Utility/Parallel.hs
index fcab2a90a1..373a0ece54 100644
--- a/Utility/Parallel.hs
+++ b/Utility/Parallel.hs
@@ -18,7 +18,7 @@ import Control.Exception
  -
  - Returns the values partitioned into ones with which the action succeeded,
  - and ones with which it failed. -}
-inParallel :: (v -> IO ()) -> [v] -> IO ([v], [v])
+inParallel :: (v -> IO Bool) -> [v] -> IO ([v], [v])
 inParallel a l = do
 	mvars <- mapM thread l
 	statuses <- mapM takeMVar mvars
@@ -28,8 +28,8 @@ inParallel a l = do
 		thread v = do
 			mvar <- newEmptyMVar
 			_ <- forkIO $ do
-				r <- try (a v) :: IO (Either SomeException ())
+				r <- try (a v) :: IO (Either SomeException Bool)
 				case r of
 					Left _ -> putMVar mvar False
-					Right _ -> putMVar mvar True
+					Right b -> putMVar mvar b
 			return mvar

From 8d2667715b0508c538cf652e2dcfe2b8a47d1aff Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 12:21:53 -0400
Subject: [PATCH 4362/8313] prune old filler alerts

---
 Assistant/Alert.hs          | 45 ++++++++++++++++++++++++++++++++-----
 Assistant/DaemonStatus.hs   |  9 +-------
 Assistant/Threads/WebApp.hs |  5 +++--
 3 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 23a93b1c16..817a1be272 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -12,6 +12,7 @@ module Assistant.Alert where
 import Common.Annex
 import qualified Remote
 
+import qualified Data.Map as M
 import Yesod
 
 type Widget = forall sub master. GWidget sub master ()
@@ -20,6 +21,9 @@ type Widget = forall sub master. GWidget sub master ()
 data AlertClass = Success | Message | Activity | Warning | Error
 	deriving (Eq, Ord)
 
+data AlertPriority = Filler | Low | Medium | High | Pinned
+	deriving (Eq, Ord)
+
 {- An alert can be a simple message, or an arbitrary Yesod Widget. -}
 data AlertMessage = StringAlert String | WidgetAlert (Alert -> Widget)
 
@@ -37,8 +41,19 @@ type AlertId = Integer
 
 type AlertPair = (AlertId, Alert)
 
-data AlertPriority = Filler | Low | Medium | High | Pinned
-	deriving (Eq, Ord)
+type AlertMap = M.Map AlertId Alert
+
+{- This is as many alerts as it makes sense to display at a time.
+ - A display might be smaller ,or larger, the point is to not overwhelm the
+ - user with a ton of alerts. -}
+displayAlerts :: Int
+displayAlerts = 10
+
+{- This is not a hard maximum, but there's no point in keeping a great
+ - many filler alerts in an AlertMap, so when there's more than this many,
+ - they start being pruned, down toward displayAlerts. -}
+maxAlerts :: Int
+maxAlerts = displayAlerts * 2
 
 {- The desired order is the reverse of:
  -
@@ -57,9 +72,12 @@ compareAlertPairs
 		`thenOrd` compare aid bid
 			`thenOrd` compare aclass bclass
 
+sortAlertPairs :: [AlertPair] -> [AlertPair]
+sortAlertPairs = sortBy compareAlertPairs
+
 makeAlertFiller :: Bool -> Alert -> Alert
 makeAlertFiller success alert
-	| alertPriority alert == Filler = alert
+	| isFiller alert = alert
 	| otherwise = alert
 		{ alertClass = if c == Activity then c' else c
 		, alertPriority = Filler
@@ -79,11 +97,26 @@ makeAlertFiller success alert
 			maybe (finished s) (const s) h
 
 		finished s
-			| success = s ++ ": Succeeded"
+			| success = s ++ ": Ok"
 			| otherwise = s ++ ": Failed"
 
-sortAlertPairs :: [AlertPair] -> [AlertPair]
-sortAlertPairs = reverse . sortBy compareAlertPairs
+isFiller :: Alert -> Bool
+isFiller alert = alertPriority alert == Filler
+
+{- Converts a given alert into filler, manipulating it in the AlertMap.
+ -
+ - Old filler alerts are pruned once maxAlerts is reached.
+ -}
+convertToFiller :: AlertId -> Bool -> AlertMap -> AlertMap
+convertToFiller i success m
+	| bloat > 0 = M.fromList $ prune $ M.toList m'
+	| otherwise = m'
+	where
+		bloat = M.size m - maxAlerts
+		m' = M.adjust (\al -> makeAlertFiller success al) i m
+		prune l =
+			let (f, rest) = partition (\(_, al) -> isFiller al) l
+			in drop bloat f ++ rest
 
 baseActivityAlert :: Alert
 baseActivityAlert = Alert
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 6d05c61528..77387deb84 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -50,8 +50,6 @@ data DaemonStatus = DaemonStatus
 
 type TransferMap = M.Map Transfer TransferInfo
 
-type AlertMap = M.Map AlertId Alert
-
 {- This TMVar is never left empty, so accessing it will never block. -}
 type DaemonStatusHandle = TMVar DaemonStatus
 
@@ -242,10 +240,5 @@ alertWhile dstatus alert a = do
 	let alert' = alert { alertClass = Activity }
 	i <- addAlert dstatus alert'
 	r <- bracket_ noop noop a
-	updateAlertMap dstatus $ makeold i (makeAlertFiller r)
+	updateAlertMap dstatus $ convertToFiller i r
 	return r
-	where
-		-- TODO prune old filler
-		makeold i filler m
-			| M.size m < 20 = M.adjust filler i m
-			| otherwise = M.adjust filler i m
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index d268559108..5349ec2a46 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -204,11 +204,12 @@ sideBarDisplay noScript = do
 		{- Any yesod message appears as the first alert. -}
 		maybe noop rendermessage =<< lift getMessage
 	
-		{- Add newest 10 alerts to the sidebar. -}
+		{- Add newest alerts to the sidebar. -}
 		webapp <- lift getYesod
 		alertpairs <- M.toList . alertMap
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
-		mapM_ renderalert $ take 10 $ sortAlertPairs alertpairs
+		mapM_ renderalert $
+			take displayAlerts $ reverse $ sortAlertPairs alertpairs
 	ident <- lift newIdent
 	$(widgetFile "sidebar")
 

From 9c9db6feb8e732887838337e700c33e6033cd192 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 12:23:40 -0400
Subject: [PATCH 4363/8313] make filler closeable

---
 Assistant/Alert.hs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 817a1be272..18f3ffa5d4 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -83,6 +83,7 @@ makeAlertFiller success alert
 		, alertPriority = Filler
 		, alertHeader = finished <$> h
 		, alertMessage = massage m
+		, alertClosable = True
 		}
 	where
 		h = alertHeader alert

From 71a6f3521c4195eebf49f7b3b23431d1155a5a02 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 12:25:15 -0400
Subject: [PATCH 4364/8313] todo

---
 doc/design/assistant/webapp.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index ebf1689963..da8de81329 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -38,3 +38,5 @@ The webapp is a web server that displays a shiny interface.
 * Fix notification handle leakage on pages other than the main page.
   The javascript should use AJAX to request handles, that way
   they won't be allocated at all in noscript.
+* Hook up notificaton close button to a callback that removes the notification
+  from the list. Otherwise reloading brings them back.

From 1f671ee40c7f26d0adb16408ba1cf7fc9ceb3a7a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 13:31:19 -0400
Subject: [PATCH 4365/8313] spruce up display of the repo list

---
 Assistant/Threads/WebApp.hs |  7 +++++--
 templates/intro.hamlet      | 11 ++++++++---
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 5349ec2a46..44fb44f2b7 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -171,17 +171,20 @@ introDisplay :: Text -> Widget
 introDisplay ident = do
 	webapp <- lift getYesod
 	let reldir = relDir webapp
-	remotelist <- liftIO $ runThreadState (threadState webapp) $ do
+	l <- liftIO $ runThreadState (threadState webapp) $ do
 		u <- getUUID
 		rs <- map Remote.uuid <$> Remote.remoteList
 		rs' <- snd <$> trustPartition DeadTrusted rs
 		Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs'
-	let n = length remotelist
+	let remotelist = zip counter l
+	let n = length l
 	let numrepos = show n
 	let notenough = n < 2
 	let barelyenough = n == 2
 	let morethanenough = n > 2
 	$(widgetFile "intro")
+	where
+		counter = map show ([1..] :: [Int])
 
 {- Called by client to get a display of currently in process transfers.
  -
diff --git a/templates/intro.hamlet b/templates/intro.hamlet
index 6660b6a039..ecb15f39cc 100644
--- a/templates/intro.hamlet
+++ b/templates/intro.hamlet
@@ -14,9 +14,14 @@
       $else
         <span .badge .badge-success>#{numrepos}</span>
       \ repositories and devices:
-      <ul>
-        $forall name <- remotelist
-          <li>#{name}
+      <table .table .table-striped .table-condensed>
+        <tbody>
+          $forall (num, name) <- remotelist
+            <tr>
+              <td>
+                #{num}
+              <td>
+                #{name}
       <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
       <p>
         Or just sit back, watch the magic, and get on with using your files.

From e3d11943c4e90a81e3a0792ab16cd535ee899d20 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Mon, 30 Jul 2012 17:34:49 +0000
Subject: [PATCH 4366/8313]

---
 ...e75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn

diff --git a/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn b/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn
new file mode 100644
index 0000000000..6b8a7e3a4c
--- /dev/null
+++ b/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn
@@ -0,0 +1,23 @@
+Here goes...
+
+<pre>
+laplace:git-annex jtang$ make
+ghc -O2 -threaded -Wall -ignore-package monads-fd -ignore-package monads-tf -outputdir tmp -IUtility -DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP -DWITH_KQUEUE -DOSX --make git-annex Utility/libdiskfree.o Utility/libmounts.o Utility/libkqueue.o
+
+Assistant/Threads/MountWatcher.hs:40:0:
+     warning: #warning Building without dbus support; will use mtab polling
+[165 of 208] Compiling Assistant.Alert  ( Assistant/Alert.hs, tmp/Assistant/Alert.o )
+[173 of 208] Compiling Assistant.DaemonStatus ( Assistant/DaemonStatus.hs, tmp/Assistant/DaemonStatus.o )
+[174 of 208] Compiling Assistant.TransferQueue ( Assistant/TransferQueue.hs, tmp/Assistant/TransferQueue.o )
+[175 of 208] Compiling Assistant.Threads.Watcher ( Assistant/Threads/Watcher.hs, tmp/Assistant/Threads/Watcher.o )
+
+Assistant/Threads/Watcher.hs:61:43:
+    Couldn't match expected type `Utility.Kqueue.Kqueue'
+                with actual type `()'
+    Expected type: IO Utility.Kqueue.Kqueue -> IO Utility.Kqueue.Kqueue
+      Actual type: IO Utility.Kqueue.Kqueue -> IO ()
+    In the fourth argument of `watchDir', namely `startup'
+    In the second argument of `($)', namely
+      `watchDir "." ignored hooks startup'
+make: *** [git-annex] Error 1
+</pre>

From a9941308434354046fa51d3327c5e05ff080a247 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 14:08:22 -0400
Subject: [PATCH 4367/8313] implement server-side alert closing

Rather than using bootstrap's client-side closing.
Now closed alerts stay closed.
---
 Assistant/Alert.hs           | 10 +++-
 Assistant/DaemonStatus.hs    |  4 +-
 Assistant/Threads/WebApp.hs  | 23 +++++++--
 static/js/bootstrap-alert.js | 94 ------------------------------------
 templates/alert.hamlet       |  2 +-
 5 files changed, 30 insertions(+), 103 deletions(-)
 delete mode 100644 static/js/bootstrap-alert.js

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 18f3ffa5d4..4a3b2cf720 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -37,7 +37,15 @@ data Alert = Alert
 	}
 
 {- Higher AlertId indicates a more recent alert. -}
-type AlertId = Integer
+newtype AlertId = AlertId Integer
+        deriving (Read, Show, Eq, Ord)
+
+{- Note: This first alert id is used for yesod's message. -}
+firstAlertId :: AlertId
+firstAlertId = AlertId 0
+
+nextAlertId :: AlertId -> AlertId
+nextAlertId (AlertId i) = AlertId $ succ i
 
 type AlertPair = (AlertId, Alert)
 
diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 77387deb84..3c0bfba429 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -61,7 +61,7 @@ newDaemonStatus = DaemonStatus
 	<*> pure Nothing
 	<*> pure M.empty
 	<*> pure M.empty
-	<*> pure 0
+	<*> pure firstAlertId
 	<*> pure []
 	<*> newNotificationBroadcaster
 	<*> newNotificationBroadcaster
@@ -217,7 +217,7 @@ addAlert dstatus alert = notifyAlert dstatus `after` modifyDaemonStatus dstatus
 	where
 		go s = (s { alertMax = i, alertMap = m }, i)
 			where
-				i = alertMax s + 1
+				i = nextAlertId $ alertMax s
 				m = M.insertWith' const i alert (alertMap s)
 
 removeAlert :: DaemonStatusHandle -> AlertId -> IO ()
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 44fb44f2b7..0de4a50a80 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -93,6 +93,7 @@ mkYesod "WebApp" [parseRoutes|
 /noscriptauto NoScriptAutoR GET
 /transfers/#NotificationId TransfersR GET
 /sidebar/#NotificationId SideBarR GET
+/closealert/#AlertId CloseAlert GET
 /config ConfigR GET
 /addrepository AddRepositoryR GET
 /static StaticR Static getStatic
@@ -102,6 +103,10 @@ instance PathPiece NotificationId where
     toPathPiece = pack . show
     fromPathPiece = readish . unpack
 
+instance PathPiece AlertId where
+    toPathPiece = pack . show
+    fromPathPiece = readish . unpack
+
 instance Yesod WebApp where
 	defaultLayout content = do
 		webapp <- getYesod
@@ -110,7 +115,6 @@ instance Yesod WebApp where
 			addStylesheet $ StaticR css_bootstrap_responsive_css
 			addScript $ StaticR jquery_full_js
 			addScript $ StaticR js_bootstrap_dropdown_js
-			addScript $ StaticR js_bootstrap_alert_js
 			$(widgetFile "page")
 		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
 
@@ -229,7 +233,7 @@ sideBarDisplay noScript = do
 		bootstrapclass Message = "alert-info"
 
 		renderalert (alertid, alert) = addalert
-			(show alertid)
+			alertid
 			(alertClosable alert)
 			(alertBlockDisplay alert)
 			(bootstrapclass $ alertClass alert)
@@ -238,11 +242,14 @@ sideBarDisplay noScript = do
 				StringAlert s -> [whamlet|#{s}|]
 				WidgetAlert w -> w alert
 
-		rendermessage msg = addalert "yesodmessage" True False
+		rendermessage msg = addalert firstAlertId True False
 			"alert-info" Nothing [whamlet|#{msg}|]
 
-		addalert :: String -> Bool -> Bool -> Text -> Maybe String -> Widget -> Widget
-		addalert alertid closable block divclass heading widget = $(widgetFile "alert")
+		addalert :: AlertId -> Bool -> Bool -> Text -> Maybe String -> Widget -> Widget
+		addalert i closable block divclass heading widget = do
+			let alertid = show i
+			let closealert = CloseAlert i
+			$(widgetFile "alert")
 
 {- Called by client to get a sidebar display.
  -
@@ -259,6 +266,12 @@ getSideBarR nid = do
 	page <- widgetToPageContent $ sideBarDisplay True
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
+{- Called by the client to close an alert. -}
+getCloseAlert :: AlertId -> Handler ()
+getCloseAlert i = do
+	webapp <- getYesod
+	void $ liftIO $ removeAlert (daemonStatus webapp) i
+
 dashboard :: Bool -> Bool -> Widget
 dashboard noScript warnNoScript = do
 	sideBarDisplay noScript
diff --git a/static/js/bootstrap-alert.js b/static/js/bootstrap-alert.js
deleted file mode 100644
index d17f44e150..0000000000
--- a/static/js/bootstrap-alert.js
+++ /dev/null
@@ -1,94 +0,0 @@
-/* ==========================================================
- * bootstrap-alert.js v2.0.2
- * http://twitter.github.com/bootstrap/javascript.html#alerts
- * ==========================================================
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ========================================================== */
-
-
-!function( $ ){
-
-  "use strict"
-
- /* ALERT CLASS DEFINITION
-  * ====================== */
-
-  var dismiss = '[data-dismiss="alert"]'
-    , Alert = function ( el ) {
-        $(el).on('click', dismiss, this.close)
-      }
-
-  Alert.prototype = {
-
-    constructor: Alert
-
-  , close: function ( e ) {
-      var $this = $(this)
-        , selector = $this.attr('data-target')
-        , $parent
-
-      if (!selector) {
-        selector = $this.attr('href')
-        selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7
-      }
-
-      $parent = $(selector)
-      $parent.trigger('close')
-
-      e && e.preventDefault()
-
-      $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent())
-
-      $parent
-        .trigger('close')
-        .removeClass('in')
-
-      function removeElement() {
-        $parent
-          .trigger('closed')
-          .remove()
-      }
-
-      $.support.transition && $parent.hasClass('fade') ?
-        $parent.on($.support.transition.end, removeElement) :
-        removeElement()
-    }
-
-  }
-
-
- /* ALERT PLUGIN DEFINITION
-  * ======================= */
-
-  $.fn.alert = function ( option ) {
-    return this.each(function () {
-      var $this = $(this)
-        , data = $this.data('alert')
-      if (!data) $this.data('alert', (data = new Alert(this)))
-      if (typeof option == 'string') data[option].call($this)
-    })
-  }
-
-  $.fn.alert.Constructor = Alert
-
-
- /* ALERT DATA-API
-  * ============== */
-
-  $(function () {
-    $('body').on('click.alert.data-api', dismiss, Alert.prototype.close)
-  })
-
-}( window.jQuery );
\ No newline at end of file
diff --git a/templates/alert.hamlet b/templates/alert.hamlet
index 2d0daf8418..bf84482b10 100644
--- a/templates/alert.hamlet
+++ b/templates/alert.hamlet
@@ -1,4 +1,4 @@
-<div .alert .fade .in .#{divclass} :block:.alert-block ##{alertid}>
+<div .alert .fade .in .#{divclass} :block:.alert-block ##{alertid} :closable:onclick="(function( $ ) { $.get('@{closealert}') })( jQuery );">
   $if closable
     <a .close data-dismiss="alert" href="#">×</a>
   $maybe h <- heading

From 78b3dada5b056320922cb9f453343ae4d95b0407 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 14:50:32 -0400
Subject: [PATCH 4368/8313] better connection close handling

Depending on how the webapp was started up and whether the user clicked on
any links in it, window.close() may be disallowed by browser security
policy. Also if that fails, display a modal dialog that nicely blackens out
the webapp.

TODO: avoid Escape closing it. Bootstrap's docs are unclear about how to do
that.
---
 Assistant/Threads/WebApp.hs  |   1 +
 static/js/bootstrap-modal.js | 210 +++++++++++++++++++++++++++++++++++
 templates/longpolling.julius |  13 +--
 templates/page.hamlet        |   7 ++
 4 files changed, 224 insertions(+), 7 deletions(-)
 create mode 100644 static/js/bootstrap-modal.js

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 0de4a50a80..e75870e0d7 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -115,6 +115,7 @@ instance Yesod WebApp where
 			addStylesheet $ StaticR css_bootstrap_responsive_css
 			addScript $ StaticR jquery_full_js
 			addScript $ StaticR js_bootstrap_dropdown_js
+			addScript $ StaticR js_bootstrap_modal_js
 			$(widgetFile "page")
 		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
 
diff --git a/static/js/bootstrap-modal.js b/static/js/bootstrap-modal.js
new file mode 100644
index 0000000000..e929706279
--- /dev/null
+++ b/static/js/bootstrap-modal.js
@@ -0,0 +1,210 @@
+/* =========================================================
+ * bootstrap-modal.js v2.0.2
+ * http://twitter.github.com/bootstrap/javascript.html#modals
+ * =========================================================
+ * Copyright 2012 Twitter, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ========================================================= */
+
+
+!function( $ ){
+
+  "use strict"
+
+ /* MODAL CLASS DEFINITION
+  * ====================== */
+
+  var Modal = function ( content, options ) {
+    this.options = options
+    this.$element = $(content)
+      .delegate('[data-dismiss="modal"]', 'click.dismiss.modal', $.proxy(this.hide, this))
+  }
+
+  Modal.prototype = {
+
+      constructor: Modal
+
+    , toggle: function () {
+        return this[!this.isShown ? 'show' : 'hide']()
+      }
+
+    , show: function () {
+        var that = this
+
+        if (this.isShown) return
+
+        $('body').addClass('modal-open')
+
+        this.isShown = true
+        this.$element.trigger('show')
+
+        escape.call(this)
+        backdrop.call(this, function () {
+          var transition = $.support.transition && that.$element.hasClass('fade')
+
+          !that.$element.parent().length && that.$element.appendTo(document.body) //don't move modals dom position
+
+          that.$element
+            .show()
+
+          if (transition) {
+            that.$element[0].offsetWidth // force reflow
+          }
+
+          that.$element.addClass('in')
+
+          transition ?
+            that.$element.one($.support.transition.end, function () { that.$element.trigger('shown') }) :
+            that.$element.trigger('shown')
+
+        })
+      }
+
+    , hide: function ( e ) {
+        e && e.preventDefault()
+
+        if (!this.isShown) return
+
+        var that = this
+        this.isShown = false
+
+        $('body').removeClass('modal-open')
+
+        escape.call(this)
+
+        this.$element
+          .trigger('hide')
+          .removeClass('in')
+
+        $.support.transition && this.$element.hasClass('fade') ?
+          hideWithTransition.call(this) :
+          hideModal.call(this)
+      }
+
+  }
+
+
+ /* MODAL PRIVATE METHODS
+  * ===================== */
+
+  function hideWithTransition() {
+    var that = this
+      , timeout = setTimeout(function () {
+          that.$element.off($.support.transition.end)
+          hideModal.call(that)
+        }, 500)
+
+    this.$element.one($.support.transition.end, function () {
+      clearTimeout(timeout)
+      hideModal.call(that)
+    })
+  }
+
+  function hideModal( that ) {
+    this.$element
+      .hide()
+      .trigger('hidden')
+
+    backdrop.call(this)
+  }
+
+  function backdrop( callback ) {
+    var that = this
+      , animate = this.$element.hasClass('fade') ? 'fade' : ''
+
+    if (this.isShown && this.options.backdrop) {
+      var doAnimate = $.support.transition && animate
+
+      this.$backdrop = $('<div class="modal-backdrop ' + animate + '" />')
+        .appendTo(document.body)
+
+      if (this.options.backdrop != 'static') {
+        this.$backdrop.click($.proxy(this.hide, this))
+      }
+
+      if (doAnimate) this.$backdrop[0].offsetWidth // force reflow
+
+      this.$backdrop.addClass('in')
+
+      doAnimate ?
+        this.$backdrop.one($.support.transition.end, callback) :
+        callback()
+
+    } else if (!this.isShown && this.$backdrop) {
+      this.$backdrop.removeClass('in')
+
+      $.support.transition && this.$element.hasClass('fade')?
+        this.$backdrop.one($.support.transition.end, $.proxy(removeBackdrop, this)) :
+        removeBackdrop.call(this)
+
+    } else if (callback) {
+      callback()
+    }
+  }
+
+  function removeBackdrop() {
+    this.$backdrop.remove()
+    this.$backdrop = null
+  }
+
+  function escape() {
+    var that = this
+    if (this.isShown && this.options.keyboard) {
+      $(document).on('keyup.dismiss.modal', function ( e ) {
+        e.which == 27 && that.hide()
+      })
+    } else if (!this.isShown) {
+      $(document).off('keyup.dismiss.modal')
+    }
+  }
+
+
+ /* MODAL PLUGIN DEFINITION
+  * ======================= */
+
+  $.fn.modal = function ( option ) {
+    return this.each(function () {
+      var $this = $(this)
+        , data = $this.data('modal')
+        , options = $.extend({}, $.fn.modal.defaults, $this.data(), typeof option == 'object' && option)
+      if (!data) $this.data('modal', (data = new Modal(this, options)))
+      if (typeof option == 'string') data[option]()
+      else if (options.show) data.show()
+    })
+  }
+
+  $.fn.modal.defaults = {
+      backdrop: true
+    , keyboard: true
+    , show: true
+  }
+
+  $.fn.modal.Constructor = Modal
+
+
+ /* MODAL DATA-API
+  * ============== */
+
+  $(function () {
+    $('body').on('click.modal.data-api', '[data-toggle="modal"]', function ( e ) {
+      var $this = $(this), href
+        , $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7
+        , option = $target.data('modal') ? 'toggle' : $.extend({}, $target.data(), $this.data())
+
+      e.preventDefault()
+      $target.modal(option)
+    })
+  })
+
+}( window.jQuery );
\ No newline at end of file
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 95425d615a..a4077c3d50 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -5,9 +5,9 @@
 // Maximum update frequency is controlled by #{startdelay}
 // and #{delay}, both in milliseconds.
 
-(function( $ ) {
+$dead=0;
 
-numerrs=0;
+(function( $ ) {
 
 $.LongPoll#{ident} = (function() {
 	return {
@@ -21,12 +21,11 @@ $.LongPoll#{ident} = (function() {
 					numerrs=0;
 				},
 				'error': function(jqxhr, msg, e) {
-					numerrs=numerrs+1;
-					if (numerrs > 3) {
+					if (! $dead) {
+						$dead=1;
+						// blocked by many browsers
 						window.close();
-					}
-					else {
-						setTimeout($.LongPoll#{ident}.send, #{show delay});
+						$('#lostconnection').modal('show');
 					}
 				},
 			});
diff --git a/templates/page.hamlet b/templates/page.hamlet
index 5004241257..919724f870 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -21,3 +21,10 @@
 <div .container-fluid>
   <div .row-fluid>
     ^{content}
+<div #lostconnection .modal .fade>
+  <div .modal-header>
+     <h3>
+       git-annex has shut down
+     <div .modal-body>
+       <p>
+         You can now close this browser window.

From f4484949eff4af666f3aabd7dc78a8973c444d91 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 14:53:36 -0400
Subject: [PATCH 4369/8313] consistent wording

---
 templates/page.hamlet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/page.hamlet b/templates/page.hamlet
index 919724f870..ea8025d4ff 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -16,7 +16,7 @@
           <ul .dropdown-menu>
              <li><a href="#">#{relDir webapp}</a></li>
              <li .divider></li>
-             <li><a href="@{AddRepositoryR}">Add new repository</a></li>
+             <li><a href="@{AddRepositoryR}">Add another repository</a></li>
 
 <div .container-fluid>
   <div .row-fluid>

From 5469bd6e427ea09e6dea2137f40da74d16f9a0a2 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 15:33:12 -0400
Subject: [PATCH 4370/8313] remove old filler that is effectively the same as
 new filler

---
 Assistant/Alert.hs | 58 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 43 insertions(+), 15 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 4a3b2cf720..54192aae65 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -36,6 +36,10 @@ data Alert = Alert
 	, alertPriority :: AlertPriority
 	}
 
+type AlertPair = (AlertId, Alert)
+
+type AlertMap = M.Map AlertId Alert
+
 {- Higher AlertId indicates a more recent alert. -}
 newtype AlertId = AlertId Integer
         deriving (Read, Show, Eq, Ord)
@@ -47,15 +51,11 @@ firstAlertId = AlertId 0
 nextAlertId :: AlertId -> AlertId
 nextAlertId (AlertId i) = AlertId $ succ i
 
-type AlertPair = (AlertId, Alert)
-
-type AlertMap = M.Map AlertId Alert
-
 {- This is as many alerts as it makes sense to display at a time.
  - A display might be smaller ,or larger, the point is to not overwhelm the
  - user with a ton of alerts. -}
 displayAlerts :: Int
-displayAlerts = 10
+displayAlerts = 6
 
 {- This is not a hard maximum, but there's no point in keeping a great
  - many filler alerts in an AlertMap, so when there's more than this many,
@@ -74,8 +74,8 @@ maxAlerts = displayAlerts * 2
  -}
 compareAlertPairs :: AlertPair -> AlertPair -> Ordering
 compareAlertPairs
-	(aid, Alert {alertClass = aclass, alertPriority = aprio})
-	(bid, Alert {alertClass = bclass, alertPriority = bprio})
+	(aid, Alert { alertClass = aclass, alertPriority = aprio })
+	(bid, Alert { alertClass = bclass, alertPriority = bprio })
 	 = compare aprio bprio
 		`thenOrd` compare aid bid
 			`thenOrd` compare aclass bclass
@@ -83,6 +83,25 @@ compareAlertPairs
 sortAlertPairs :: [AlertPair] -> [AlertPair]
 sortAlertPairs = sortBy compareAlertPairs
 
+{- Checks if two alerts display the same.
+ - Yesod Widgets cannot be compared, as they run code. -}
+effectivelySameAlert :: Alert -> Alert -> Bool
+effectivelySameAlert x y
+	| uncomparable x || uncomparable y = False
+	| otherwise = all id 
+		[ alertClass x == alertClass y
+		, alertHeader x == alertHeader y
+		, extract (alertMessage x) == extract (alertMessage y)
+		, alertBlockDisplay x == alertBlockDisplay y
+		, alertClosable x == alertClosable y
+		, alertPriority x == alertPriority y
+		]
+	where
+		uncomparable (Alert { alertMessage = StringAlert _ }) = False
+		uncomparable _ = True
+		extract (StringAlert s) = s
+		extract _ = ""
+
 makeAlertFiller :: Bool -> Alert -> Alert
 makeAlertFiller success alert
 	| isFiller alert = alert
@@ -113,19 +132,28 @@ isFiller :: Alert -> Bool
 isFiller alert = alertPriority alert == Filler
 
 {- Converts a given alert into filler, manipulating it in the AlertMap.
+ -
+ - Any old filler that looks the same as the reference alert is removed.
  -
  - Old filler alerts are pruned once maxAlerts is reached.
  -}
 convertToFiller :: AlertId -> Bool -> AlertMap -> AlertMap
-convertToFiller i success m
-	| bloat > 0 = M.fromList $ prune $ M.toList m'
-	| otherwise = m'
+convertToFiller i success m = case M.lookup i m of
+	Nothing -> m
+	Just al -> 
+		let al' = makeAlertFiller success al
+		in pruneBloat $ M.filterWithKey (pruneSame al') $
+			M.insertWith' const i al' m
 	where
-		bloat = M.size m - maxAlerts
-		m' = M.adjust (\al -> makeAlertFiller success al) i m
-		prune l =
-			let (f, rest) = partition (\(_, al) -> isFiller al) l
-			in drop bloat f ++ rest
+		pruneSame ref k al = k == i || not (effectivelySameAlert ref al)
+		pruneBloat m'
+			| bloat > 0 = M.fromList $ pruneold $ M.toList m'
+			| otherwise = m'
+			where
+				bloat = M.size m' - maxAlerts
+				pruneold l =
+			 		let (f, rest) = partition (\(_, al) -> isFiller al) l
+					in drop bloat f ++ rest
 
 baseActivityAlert :: Alert
 baseActivityAlert = Alert

From 703f24b83e1788955bf0a35525dc16fb2dbfb39a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 15:39:24 -0400
Subject: [PATCH 4371/8313] remove href

Avoids a reload of the page when closing an alert.
---
 templates/alert.hamlet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/alert.hamlet b/templates/alert.hamlet
index bf84482b10..8fa77eac1d 100644
--- a/templates/alert.hamlet
+++ b/templates/alert.hamlet
@@ -1,6 +1,6 @@
 <div .alert .fade .in .#{divclass} :block:.alert-block ##{alertid} :closable:onclick="(function( $ ) { $.get('@{closealert}') })( jQuery );">
   $if closable
-    <a .close data-dismiss="alert" href="#">×</a>
+    <a .close data-dismiss="alert">×</a>
   $maybe h <- heading
     $if block
       <h4 class="alert-heading">#{h}</h4>

From 2400c6faa34bc696c30253edb219a5ffad0b2a36 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:11:16 -0400
Subject: [PATCH 4372/8313] insert modal dialog html from javascript

This avoids it being visible all the time in noscript
---
 templates/longpolling.julius | 15 +++++++++++++--
 templates/page.hamlet        |  8 +-------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index a4077c3d50..0d18b8435e 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -5,7 +5,17 @@
 // Maximum update frequency is controlled by #{startdelay}
 // and #{delay}, both in milliseconds.
 
-$dead=0;
+$dead = 0;
+
+$connfailed = 
+  '<div id="modal" class="modal fade">' +
+  '  <div .modal-header>' +
+  '    <h3>git-annex has shut down</h2>' +
+  '  </div>' +
+  '  <div class="modal-body">' +
+  '    You can now close this browser window.' +
+  '  </div>' +
+  '</div>' ;
 
 (function( $ ) {
 
@@ -25,7 +35,8 @@ $.LongPoll#{ident} = (function() {
 						$dead=1;
 						// blocked by many browsers
 						window.close();
-						$('#lostconnection').modal('show');
+						$('#modal').replaceWith($connfailed);
+						$('#modal').modal('show');
 					}
 				},
 			});
diff --git a/templates/page.hamlet b/templates/page.hamlet
index ea8025d4ff..67b19aaf71 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -21,10 +21,4 @@
 <div .container-fluid>
   <div .row-fluid>
     ^{content}
-<div #lostconnection .modal .fade>
-  <div .modal-header>
-     <h3>
-       git-annex has shut down
-     <div .modal-body>
-       <p>
-         You can now close this browser window.
+<div #modal></div>

From afacc413e1643a93a71e93cba3f4e1e470fc4581 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:14:27 -0400
Subject: [PATCH 4373/8313] bring back connection failure counter

Otherwise, navigating between pages can, I guess, cause connection failures
that take down the webapp.
---
 templates/longpolling.julius | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 0d18b8435e..dd5705fc95 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -5,7 +5,7 @@
 // Maximum update frequency is controlled by #{startdelay}
 // and #{delay}, both in milliseconds.
 
-$dead = 0;
+$connfails = 0;
 
 $connfailed = 
   '<div id="modal" class="modal fade">' +
@@ -31,8 +31,8 @@ $.LongPoll#{ident} = (function() {
 					numerrs=0;
 				},
 				'error': function(jqxhr, msg, e) {
-					if (! $dead) {
-						$dead=1;
+					$connfails = $confails + 1;
+					if (! $connfails > 3) {
 						// blocked by many browsers
 						window.close();
 						$('#modal').replaceWith($connfailed);

From 453b185c8506e4a6a030788f42eec196386ac78e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:24:55 -0400
Subject: [PATCH 4374/8313] bring back retry on connection fail

---
 templates/longpolling.julius | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index dd5705fc95..76ff94ce35 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -5,9 +5,9 @@
 // Maximum update frequency is controlled by #{startdelay}
 // and #{delay}, both in milliseconds.
 
-$connfails = 0;
+connfails=0;
 
-$connfailed = 
+connfailed=
   '<div id="modal" class="modal fade">' +
   '  <div .modal-header>' +
   '    <h3>git-annex has shut down</h2>' +
@@ -31,13 +31,16 @@ $.LongPoll#{ident} = (function() {
 					numerrs=0;
 				},
 				'error': function(jqxhr, msg, e) {
-					$connfails = $confails + 1;
-					if (! $connfails > 3) {
+					connfails=connfails+1;
+					if (connfails > 3) {
 						// blocked by many browsers
 						window.close();
-						$('#modal').replaceWith($connfailed);
+						$('#modal').replaceWith(connfailed);
 						$('#modal').modal('show');
 					}
+					else {
+						setTimeout($.LongPoll#{ident}.send, #{show delay});
+					}
 				},
 			});
 		}

From b2e359a15d20d2b2b13a1883b451d014ae60db7c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:32:32 -0400
Subject: [PATCH 4375/8313] fix kqueue build

---
 Assistant/DaemonStatus.hs    | 11 ++++++++---
 Assistant/Threads/Watcher.hs | 11 +++++++----
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/Assistant/DaemonStatus.hs b/Assistant/DaemonStatus.hs
index 3c0bfba429..13828d3eee 100644
--- a/Assistant/DaemonStatus.hs
+++ b/Assistant/DaemonStatus.hs
@@ -236,9 +236,14 @@ updateAlertMap dstatus a = notifyAlert dstatus `after` modifyDaemonStatus_ dstat
  - The alert is left visible afterwards, as filler.
  - Old filler is pruned, to prevent the map growing too large. -}
 alertWhile :: DaemonStatusHandle -> Alert -> IO Bool -> IO Bool
-alertWhile dstatus alert a = do
+alertWhile dstatus alert a = alertWhile' dstatus alert $ do
+	r <- a
+	return $ (r, r)
+
+alertWhile' :: DaemonStatusHandle -> Alert -> IO (Bool, a) -> IO a
+alertWhile' dstatus alert a = do
 	let alert' = alert { alertClass = Activity }
 	i <- addAlert dstatus alert'
-	r <- bracket_ noop noop a
-	updateAlertMap dstatus $ convertToFiller i r
+	(ok, r) <- bracket_ noop noop a
+	updateAlertMap dstatus $ convertToFiller i ok
 	return r
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index bfeec7630c..51dc572636 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -72,11 +72,11 @@ watchThread st dstatus transferqueue changechan = do
 			}
 
 {- Initial scartup scan. The action should return once the scan is complete. -}
-startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO ()
+startupScan :: ThreadState -> DaemonStatusHandle -> IO a -> IO a
 startupScan st dstatus scanner = do
 	runThreadState st $ showAction "scanning"
-	void $ alertWhile dstatus startupScanAlert $ do
-		void $ scanner
+	r <- alertWhile' dstatus startupScanAlert $ do
+		r <- scanner
 
 		-- Notice any files that were deleted before
 		-- watching was started.
@@ -85,10 +85,13 @@ startupScan st dstatus scanner = do
 			showAction "started"
 		
 		modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
-		return True
+
+		return (True, r)
 	
 	void $ addAlert dstatus runningAlert
 
+	return r
+
 ignored :: FilePath -> Bool
 ignored = ig . takeFileName
 	where

From 17433ecbead915a4245a3be3a1b8cf4ea6263e8a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:36:29 -0400
Subject: [PATCH 4376/8313] fixes

---
 ...that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn b/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn
index 6b8a7e3a4c..d0551c151b 100644
--- a/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn
+++ b/doc/bugs/build_failure_with_kqueue_code__44___first_commit_that_breaks_is_3dce75fb23fca94ad86c3f0ee816bb0ad2ecb27c.mdwn
@@ -21,3 +21,5 @@ Assistant/Threads/Watcher.hs:61:43:
       `watchDir "." ignored hooks startup'
 make: *** [git-annex] Error 1
 </pre>
+
+> [[fixed|done]] --[[Joey]] 

From 999d337f9cbbb2bc16a25532c66b525d7064b95d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:37:50 -0400
Subject: [PATCH 4377/8313] blog for the day

---
 doc/design/assistant/blog/day_48__intro.mdwn |   8 ++++++++
 doc/design/assistant/screenshot/intro.png    | Bin 0 -> 50730 bytes
 2 files changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_48__intro.mdwn
 create mode 100644 doc/design/assistant/screenshot/intro.png

diff --git a/doc/design/assistant/blog/day_48__intro.mdwn b/doc/design/assistant/blog/day_48__intro.mdwn
new file mode 100644
index 0000000000..2cac85d4d1
--- /dev/null
+++ b/doc/design/assistant/blog/day_48__intro.mdwn
@@ -0,0 +1,8 @@
+Lots of WebApp UI improvements, mostly around the behavior when
+displaying alert messages. Trying to make the alerts informative
+without being intrusively annoying, think I've mostly succeeded now.
+
+Also, added an intro display. Shown is the display with only one repo;
+if there are more repos it also lists them all.
+
+[[!img screenshot/intro.png]]
diff --git a/doc/design/assistant/screenshot/intro.png b/doc/design/assistant/screenshot/intro.png
new file mode 100644
index 0000000000000000000000000000000000000000..23ed49d6776cd9c365d2a834b55265931a61c26e
GIT binary patch
literal 50730
zcmW(-1zb~a7runj($bAemq?CorI9WHkp}7R?odF9AuTQ43_&_3-2)^>*XS7E{@=Fy
z-re`!-R*bZ_nhZB&pCIon(9gfxYW1+01&7s%WDGwdL(L*#YRW<h?a09qgI$UGHNma
zP@9Z*Z}9}RjpnYcBnwoJ)9#`+7_F4G)d0ZnB>;p(0KhG(D`W=%Jox}%?*jmceFFfB
z&pB;cAk>bhmhY70fye)zKRQdlpn7mVD;v6_*2@216o(unQ6C3X<YnIbEFP?Qo9O7@
z0VnY7PcjBLI^S_QL;!LVk<a<MIdqHpPBYJ(Dw<G@`QdR@({Tme;+5`Fxl?1W1czRq
z-o*3rFx}4~EW@AqNEJ!a#Djxb{}9Py%UPP|9S>;uySp8?!gKtO0^Oi=L~`_bc-Tzm
z-rerB1MZ-aGtdB{qdq>IUK3M>C?bSW1&xUj6WxYlKucEXRuJm`wJReZA2e_latG8U
z0%1fcvbFMDo=n*Lfq*};o4i$L4BQvIVGvcoE+I!H616TeyAY{B$QBwxh8>AIlr<ww
zmY<1;2y^{F_Kn*7CM}oToPD|9Rl>nag}<$Gil(wy`{)i@v9RVa&6D6QlkxVT-oCLf
zWO1YB7QC|0>?<iwX_a*R!mB@haNLZExSj{6@@U7_<Hgf!8kst6-{2`}fQP!$cwIdy
zVv8|vghjTA!WsM*#^*rL-SYP;y;#YV0x1@TQ5K>}?q1&~Fyi2$aNj8F<Oi_pwqsL8
zX^&4(cqg{#_$S_fqDlD#ergvR8&Q(NfdT_{v`P9b;yxQbxXl(L)rGt8VrP2U)Xm=v
zEzbY$zfI2dUnbB>`Y<X?J&pThxQvw|fr&;tKu9;QQCvO>(cy4SMZ094RweK!9;j`|
z_mXyOhL>o3GR&~1><CK8`h&PfOuby@;c3@4;9!j0Tot27fRt$Lt^B=8sT5s)7-aMz
zlMh~lmrp>eoyOOV*-tFX@PH4zvSXGr@$!~9BY&fe4r3yGlB^9o+!AONiT<p$?BsCX
zMOv0gOG~bm4qpD_FRnQxUwF}>!=^BW-)n@Vet|QqtL#eaKuxiR7J|iSU^W;MgQ4|>
zQ}MAE8(l0q%HtUh2S*tIWOpjFu3a79?=}}I0n&`9v9qB6^Bfx!=ys)*Ap${@Dboe5
z7hgYp8r{Oc<Jrl^_o&Vet;4Wn#_I2Lqa8t9`N{godxN*JKge{WQA+C*XcJQwHilqI
z5LD9xBx{(UlnCV?9YG5VGqQ)#jM({_A}ztk4N_1uNa4#!W(2!{9F^@`s4zU5j)_qv
zUzsff_sCe&MUb9GBkArcaa{!b$sno1TbqSs`WZFf`=jDtsvjxaItuO&TDjD?$5N}W
z%Kq&99zb9+F&6sE!`PI!LVqp9OB&wD$RO2z-pLHO=Co#J2d$ohfX}p7OA{<}HKu`w
zhg;*-&+Ya5&8A<MFQxClMME1>=>>dHUt|f0Wu&A@%PJ(k-R3EjFmd?dr!njde9OB3
z**-6aiSDFU+}zIl8&^8oW`>Y1j8UF>eJB6ntm(k&g*KWBwTpc|L0gMtZ8gtWj85b2
z;u)@IkVU>GD|y`UNM}T1zwhD?LP7#uJc1I+EDJ-JNYwkQWTENdPBQAMl5!){d>&42
zh<|Cz&!PioPu-ao{-G{6eQ<TH(H|rv7>Vfpm-g=yci7|TqvY+*hH-OTju}2eiZO(U
z$bv~G2zJIO6**?+A?LetR?V|f_KSF=Zrcu66FJ%@D4ycH0Ge*XHKXoU;={zQ{rv~q
zqP$|_4~JX{s;LmHpS#hqmphLDS}VLHH}4uHACu%0k>M4I{^!eRL@oZcKsVY+aJaam
z))F&z80!!+(C^6|C*|S?(Dn()B|zFxLo4Z#Ma3c-c!W)KibPr3?gf2qnm|}B{$N;!
z@Q(b#?>!+niYm5y3LL>&<+K{C{Zv#`nw_<JqZ8*Fy-P!9@gioWm|;OSD!ra+d6$?W
zyX9K$oF}_B4sYMS9c6$+#CZAGem)_v7$4AKqa<eC7k0>yK3HvX3Ve)E4brOo>>{&o
z*ybTB{L2c9m$#;gofJxnd7ijCq}_-=Hh-X?=}kgQOAE5-!5lfqz<NgZ{JD{$b;V-f
z?LyXiY_H!{QfF|Lz4;fXu78EedD*84;hm`#--?eP3N3EW?KI}g3S05@_U&iOX#|e(
zNYIGkOzK$@hI=$qm9$^qT5D?ll4o|dW<k&@O}#hJjINEv@J*_h6nzSiTpMAqklEtw
z<%k{MHRM&Ws=lds5|TqB3?k^UA$fI~snm9cF2k)BQ%R_VBd6$S@TgD&mM}6@FI=q4
z#wZ<?x*7<D{0^@sBn_38PU}IPSm;QEE3GA^D1{L_jI3-r9R|Z4K&<<Ktqf}omL<xU
z`bP943TC~IqZ|jUk~T0gS4FwLexj`498knp!wBh78mD&U8WG?cNwc4?_0N94NwV^K
z8#>CBIyz_ndy7+2Q2pM&Z+M_YW1p7(8DJSE;&YPeuLRp(^Zj;vZ8cF_+n-E=Hc{8k
z<4aE;{O|PJ{5(1aVx!-b-2Yi}w$Ri+<0!_gAieg~9OJIaaPYx4j%RJvmJc|(L>Ns$
z>&;9$OfqQuYuuRtp@v3kB1|R18&^mF>n?#`RO9$UOY0MKG$<lJKmT1OPvgDU?qu;)
ziJG|Uzlg2T3ySCr&u_D@o&T0B<zT_q6J^srTD3>z(jKM%wl^I!^z232!6*c|AcxKM
zt4<Q<|Mn-w91C7HC53m!Y#0PORh5pu$PZ@7CKQ>Vt{UuBP!8e5#u{wLQa|OfsZa+a
zF@UH`h66djr{NGh><CO1W#u-brAi~gx!8s;awO5P(hsBVPh~@(#&d;pH19cd2a@PU
z*}adSbCS8m7MaNrW6Gq3g$5QvZclA8N+SQmKx~D7xasAFr}HrzzTYb<AT{2tZAPrA
zO-_l912{%Chp;L<;J09lo8zCqf`GYMxg}-4z>3n*@ox1Rwjv|Z^z6^b>e*8R%hNP6
zt%u~SrSD1*$%;zr0Wa6&yw(uLKS1~F^PJnufemjvVKi}1TZ6$EIL&iKd7`Vc_4U0Y
z7ad+0m-V1Bd*=Nn_SLu?;bV}VzK`?XlzT$`z1IgN(RFcd-xgg-bb#bXy}XE^5F6Y?
z2--WG&LANqa`0Yp!Kr6!rP-YCCokPn2VC{Vs?4Z59mB?y4F@otk+THZg;)4lSSrbQ
ze)y(YNh$sbKjflrj{57c2()2hWh(h2e1BKz&g&pUkmq8A+vdHhEY{G^O!F7Ygjn#K
zD~P_t!?{jqXeg(C(E0UW_23eH3N1ddqT%I)+p`h7JB{TnlRK{z&7e@`G8@%~3*uRj
zrmeZBt~HXNSg(-a*&!y-9FH4+3_GHo(5*gk=Hq1|N?~Q^;N?c7_zN4*xpYL2_?%J#
z>R`~#A5smVyP~G+7Uq9^u{X1SeY&yz-Rb%0xA|XAgn5JIo#_%=y9*`U;JW&pE3lA7
zL{(L{x%e+6yfV$+jGAXwBk0U7`J_65=W*0qs(59F6jPLAT0Cf$lR)^Vb+!f_QBJBt
z=HL5oe4fXz;Kb;ePVGrE(>t?)53iYsUL{k_L(H#UdiO=O_vhL0J~tD4<Fri+Tw7i!
z(P4;7-fq?dCkqgysC7T51s!?6z&#R9KH|B;#noigIF>tDE+%yKL?cc4Xu<V?K@5z+
z7*gIA^y$$>|GWp#Urz~L0rg-3Kt;R%gX58@-p>@=_9!gnKyiu{j$*sZ@mzsa&B~L@
z&#tw^3bY3bWD5HpZGV3}Lk|tH*on%O-ekc*S6&0*SIWvh)jWEdE3NqwBjgWt>mOak
zxF-prFJ1s7p@T|K)2}j2uV>pP1ujTmBQ>OlGrBHDbu<zv`D}M=YLbzkW}v}#_?+~K
zZv!3<HS%(<bn-3qzAoJ#wm)H(0{%NnNkb{PIlisjieg|TtHZ0*l>sxPp@B5^i@Csf
z{a)mC)hfA-gN23T&^a$55jq9%qa6Ck=YYKJFk)b8cT>-s*B5qP$o1p77=U)&KL=hW
z(a#q8Zz?%-VTSO_VO3tc*_gaP36YU|d<a-QZr(I!mYAICa;T$_c>kVbrg$W?!oi-o
z-SbJv{#?2+0#WFHadvf8pY?@RrPhFqoZjuh>&dIsG;}m}GF?4g0eluRp}UW<8#mqE
zNqsL~9Fixq_pTvb@kjU)K^$+V1PRxQPd*s-dq{63*bG%tvSKnN?o3}~%CXZzNfONq
zsvq&PR^5~GTCY3#fk)G;T&{i9s}{_)R)(&7a1yv~r*F_LYUz6Q*yiRV<`uQAFvH;C
z8{b7t??~m(YnHoh$H?~UhpyV%33K`z-?T1m#?wvTZIjF5SSYWf0q@j2TPZi>thAzl
z6N&)te;dd7x~P#7V-8RW1-6c`tPfq0wB=O&4Mz4Y3X@ZS!rL7e8+bpQBaZtb!R*x%
z^+!{oIB{(`$J`@`fL?9lB2EDENOE(qbUgF-pkLQP&V`=pzDGQ^dXr-H==6RAdY!M*
zA3*zUU>W;<y}iqwLIWdY#@xmIj+3(-Dh)35IuKd3nBRbU9jBBQ@X#-=7G_2#?#<=S
z&L8vRx|mlf>KFy+vyq`mpXM6Vh0Atq;$|P>U@DuM&X8|8gWW2wW~i)wozn9l2u%7<
z9Phv7<+*hAEp88Gh}K#xDGr=3UhhLmf7)wrJ_BXvy1OpFu31>dEp9$SrVKzj&KZ!*
zGWnrZ+rM+InjGifUVLjg=`)i>SFd2trfD$CrtpU}*>V8^wTCfG7i%H>$?E<d*JJzN
zUxU9BQBq<PsO!GdIQ-&s>#t~d^|7KnGBUEb_+U-6|A&_TucN4YV-VCqw~zcQE1R@4
zPxwsRZJ*MN@tXmmI#NUZf!VGf?f$0}z8}2leZYm@$n$s`hzhhnimd`4>lt0nrq<5h
z3KPpi2N!dY?Kf{b?k|DsmX+}wY=8C3Dsiw+_>=UP4z!ZanZa-Zzd$w5nH|?&8u6v-
z7a@~-AO%Y%-q5hLuml|#Pc_b0bYQ1pq_44UJp3v4!$au5a@$S=z2og%=%WNL0%}S#
z``GK&;kQ6MvAwF1N&T&q^wp&VV&Q;6V-!K6=7a3)b_oi&?~CmB)icGpyP0POzO}58
zuGlb0b=tlFPD~Dmq}*5US;qx6oZ%}2QtN1nk{4~qN4bUcc{rfEy>os1p-0svev>jA
z8+G5UrIX{k3!2BRxaJTWkFDl2J9DR51wfPBlzY1MG6`Sk)5S#Jn#3!~bnMS$%SI@O
zEO)Svi;GK;IQ$e8BrosN`vJd1=i~ZjZ7f68qt>VZy0$VArb^s<`^LN)mwJ7CMiShI
zfWVoV#iq{M5k=4^Lq(3~H+OzCY>PfNYQ4S*bICL0x?Bz$849w}A^e}*K}3mktG*+J
zQqbNOi&{xpbl_im^fpCBp-)%CPXC+49fn?J6?CyUOqQotV(z<cnou-7DmqxZxYMc`
zaB_(V{B*=QL`>gL`|A9p^Xr~F+{<oYP4oDKzI9<?Wh0u*bclZtokh$?sB&(i(;qnx
z(eFE9WZ>KAo9JihX2EES2SVV-%Uo>RA9?ZP7(<c~8Ekj`Im4~9Ur6<_JGL(g(7?xV
z-p*wS+`Xs(?#wMT$Ba1mdTRcJ8_6as6+LVMBKJX<3J1Vo#z4g5{u{j!6ivPXhI{9O
zQqkAY7@O*Yk*2z=P>R(yT@^nN9XSAEV5eBY!KtJ2ror<kcoG?bqfoG<@B|>Don4rN
zLQadZ^OaSn1g-gEUd+<Xbtqan(hE5n*x8kxRva80G&Bez9!vlsJjb^U3r$BdrqyOG
zL<|0FbXzYH=OOP>!r(Vi?$nrO;9G6)nVKBv-w(Bzz|nIymmSte%9(d48H07OPv}3?
zOO88M2M7U`hqltAyV_t97sKCCiEex~0>26!JwB4QM?sd;(>R8Y$ujic%X{qxG+#UG
z?pE~_6#V_G8EW_S7RFIIYmJ<rSO2I>$0_c0v{>s-0mOh5+_zP?eY=Ww^r+xs+%FA)
z016FP6`#`mC*E?>CVjC_BiTaW4@DmvYYN`$lJ4wTJ7>?HGiO+zH#Q1dV;@j<3nYv!
z`vCa{m_<ZvC1-l4r!0`OH)6!S=m0k>j<>#o^0&-j03}rP<ged(o2x%dL&Y-xBG9ym
zCHx4kGJx)c)XwqEoWKreurQd``hX@LO#d*HcB8?I4zzf@T&b}MHZR-FsPzU@^DqOz
zOM-(e-$q-q?#6@BbGO}_Ru*~8b2suXF^78%C2gso&F@CdYxg5?m*->&w3EkYTODaM
znd7<QtHL}<mGKZ4DT&1=;jSzwa7tJHGgrfj#>gQcz{d7>+YwU+JM!cEOm${7s6c}z
zD_R(%f~9U`x%KI{jhiGo^x<LN)hi#>eb3X0711Os%97eE>PdV8z|Dm)V@s8Sj@z&O
zR5eB@k#}JMZ||_yO@E4^IVDB2?-vM~sXG&9)_hd>NNMv!1SZ~M6~2;|Od};)pf3o-
zJn2RR_|7J+2Dcl@f2r)&wfQ+oMG1!cUnNO+Z)?6^UPHj^t*oq^4wnK}H-}6M{e4r=
zh0)MFquDhmbjsdScP2-_*YrNeJ^2c2E!Li+Bs(HO5RVO*I#TV8j-ZE(X`_W?#t)h}
zAnq;Geth$1Ax#>!qM!66#Y-S%Ml<hq13~re*;`dV@?=E6x1Bl55ETiSm=I);5p?|x
z22zxZ)9ur1f<vs^PM7D-B1FW+@4^Ws*L)>~guE8&th0rjvOvB92Frd<x|r_lj!jM;
zHD(K<R4bNPAM>EQ<aADzLA|2Ay<2D-=B5{i?6pT5N<Ak2l5YwN6p!IsZx?ZOGH9mj
z#<m13Rw91Z_5b}`XyEI$t!lgZXQDh*JTh{n#Z{+RFEY|IQ@L2!Wt-ypa};PS8AQ&s
z?MW^!#8y^V)t+o5OHh-N$|3Gg(Z`Y>4exGcr5a}$y-7(ui!a@X_4McL9qhi#f2@v+
zk14_@bIE_x86%)ct)|v|qPcd{59?PzCn`$WFbLILBWFMatcjzWuP#>18DtcTWTYs6
z3_?ni03GtIlj0}X{y(C3PdG@wgn}pA>}0q-k9$l~n&VxW=a;thTE|6lGcNB=*5yVK
z*DYDF-)qdQ<@or(GFx?M)-XFp3FXuqT88ISbinOJyj0-J`X9f3nFpjrw-4;T{th3{
zJ)YzJoujK%BScU3U~c_b`{BRfGpwt#?UN}Fk%~!z-0Yhy`&@PS-eOv*d$D#|d%LQ}
zR0i*!%Zq)VQ3Oa%r?%6Nk9J_b!vh}x1l8tdQsKWvi0p0{&;iP*Mu@mkpFKhe5v<@0
zbMyYq8NdDgtnqp_O_miq<6BUM9tA~^>-D8o4Mf85C3*<&Y^4xVJLYKO=C4_2^?}jl
z*1e$>xWN!b`6p!LWm$?z+S$)%tU*vTCLm<7oFum0Wb<XI{jl#^;tRp|nRr`t*H1@g
z1+_UU6m7rQRp8J|Z|<qsag0S4*;go7GSEPjs2x-Lms@oW0<P4RZC}YU4noQ{MaBo9
zmz_HxfQe{uS%P!;U8gV1ZE!I_&qlqhvQqWkJB$$0SL$5GDCY6YqTfSj)VZiTvC*l<
zOzZK+nxVM!i~H^(DX&JuYxo8~ZVZ*aV)8?f_uh)!?s_dwV>;^BN{6?fQ192a4Qs=s
z0Kg-?kJ-D`j=pE&vSGow|4DC8kF&{;cExn9T)#@uB+bIEf`Y;ZRzd^pNN>OH+ScY(
zoepS2Up9=AZaYY3+wTQr6czW^Xdyh?+hz0>-YzSbaaPTJBc&?OAX4!ixJ`R7`-A8R
z6|fb26R7lT%?Iqj+VIZjhbWjRjq9`2V_F&*Ij|pZ_IR`zYTWFcp7GWDc=lmN-&`lA
zThn-d@86yx?%O?HSRLd>$M=rf<M1hv?p-|;p!({go!u*#`Biyt@p6D%*nI^1D=%XS
z>qUt9{r3%V+&E{6%SmlQ!Yip@<o0swLGY--d{~%7M8aG`Av4Pp^xr6UE}B&DAIC*2
z;n7`&&^PQC{K!rA?&x`Dg!j>On65kM_!wR>%4^^2<M<FBW<{l6_GixcH(%wcM6b|<
zpc?Q~H=W!%y}WKX4)qf7RQTRj@ZMCbT(6=W2(h5-a{a(>Vs|kCT4A3zXL;71cF8?p
z?+Y$$Y;7!tQ*VE~$xldQD_T6#&Vt_zXH!{|zv=g2M>8$7@_K9-VPTzImHSvPe~Cv$
z`>*B&dbKr*9_pDR^)DCR5ld{yg<+J7C<M)#f#Du~Hg4{C)X$g}+0%8a@?qsS+hZ6_
zpPqu@C}$m5VF(q^Oh$l8^|#2$GZrej2)o_!q5LR~J?E6tR!+g!@HBUc4kNrH@PP~_
z`J2fi@<zCvV<yqr8b~!5t*iX~qj{<aAfl#j`|vyL1_5`0bx;A5lapL|!cLC<Z)Tg^
zC?<J9dw-J-BfM>6k}GRPFN)vFZLx!`I-htDx&{%fQ$61-^*;1ipK}q_8*qK^q~O;}
zU@@~DoO_>Mtjkq3t}0G8!7?rquDq9D!}Bj2>ZXuykuO|L@so;zfl$T3;AEzQ##&6~
zGnPw*fI-#BnU$4zYSqg|TjQh+(JU*^^f^Mia)&=>Kf05;8Y5hO-}Hh8s5ol2Iq@*U
zddS@fG1j%rQpOOpI>y>&<!*?xZC@}Ewy5H&4^r)lIm*J`Q!FDN{r<rxsop)0>76^-
z2zh#XQSKQ2?uric)hqF9e6X0s-y+v7kZ#Fbt@K+{6C8TVMKMwe!eA#AFL%PMc8EzY
zm9y*g;o;+h&k2hLAU#vn5wQ5Gq`os1iVSSiZRszr${wq=2Vj~v#*Jq-z|I^m&qi{6
z^96k_#`5jOe(vQGdvVsm6ia2*v(v}kBN(@k+3RPxxCisKa_>KWbT%{d^^F|+o=C{-
zy-UXY9DU@QGcIx53vzPur*e1<FE7u|{u$CVQ#F0NpaP2LE|L#_pMs_a)2&#tJ^|Q|
zfcaV|V@%uX_lVv5Z?CY!KG;$lDM@#aQj|Vi_xe7>UZDT*%`_|gf_2p_zR2+W;1zb>
z?F?l0$=hOuTt)W#Bh#hi#KfE3gJY|w!pACaU-QeT<~|%GaSz-SGQ2K$#!tF^fqj7<
zf&-OozgP<I34bK<pd^`XZXUZTTx8c3C&G(KO;#8EyhfNQ1);z?-=&<up%&$C3A}T*
zvx_v~BEeBOqCcXirze2BUY%g$9b0x0&JIA$xl&u&hN+FkWr)zjK6`prX;(a;ix~u2
zlP9CRQwF#3r+fR=uE4yYTlSDn8Pg4auY(3B)tVji%T0V0YI_^b<;v5gMoy(4kHUmS
zKR15C^A0%~*LFpxnq5Teg{`<{Vgh7I&A~XAsS~79W`QY%&LeFJzP?3e;R-KQVRa6f
zG~PbbEE|^pHjJAN#r)cJc+Pn8nSiyI&tn2H!mS1Be$Un$WzGz2AISs1{GqTF|4Whx
z&7H~CyB6PdItet!-PoZrcEg@}<0m{prdSDhXDA8aTj;51@CqwpN<q6OWL#oXfFEe_
zy~+HV86Ww^tu=FcmNO;mKtWJ#*CXp^jJns;3bYVw84hxL{HMAqhMp=n^~b!N6>OK!
z{e9vkUP_8>k0hF}bZY(WlfsnNG_69M{NhoWJP025bFU@5JEeJ-p&I1Y7EYrevi|rl
zI!M`pmm`V?_IN7~D2uxaxX%B5nZf*FY8(phYJm7XSTT2HTu?Xr+#FD?|797~-zOI2
zl6(dPn5xX3nAPIC5S#IOg~&2BcVvsPv1xLo`}wp83qeo`8{DDmou5+_E+<?V===b7
zACGr@e6OsWktTp9!?t-wog3;{Xp_4C(5V21H>PjZKyzQDE87!1&2Tz*8@b3?)xSq<
zILuXRecjEB)MUfQeJ0p%4Ti6LJc?fpVyT1jh52a0Vy%Ai-0V!bJm(G^QS*SnB-?E6
zp0EA_%iUhNdYK-IUfj9*M*ngq=B6sJLj%Kolcx4BtY3J`iA82BVN;9-)venynG2UV
zNg}gqxT+V<zij0=NSaFcEf*c7n&vEK0Kg-14wC1O6op(qBwj!gH(Ijx$_wsk>cLqD
z>pWJr{b#u|8R+ONCZ>Iu-DBBLIFUEO4w85B92qGjIDGinkt48RUa=-<5xY*LY-n|c
zmn-o~%k1KnJ|U%`$4GN_n^)D`saYQ4eDHhxm*{@6_N^2N5LAau*JppcfPCND48|NA
ze`;GeH|w!a;`}OQNlRDPQ|&t{(dh7nWHw%vOU>7}rG3bRD#Pi1kzab{Vk0S(PR#S!
zJ}rzLRN!t<s+p?EN`@OLi^^aLjBg|~zY*L+Vc_~i;39}}F5|v$Vhi}udExJDrd7)0
zgs9GIvSx$);kO6)zM8VN>25DEwXEo=5n(bB#r$Cg{@T6CFgHxJ$T~i(15w$3LkF@7
zq~7a_<vq(EchdEcIsrlJ64Wb*0r=MLZ}*&B&IxV(+EW5-COUL+PyEHR8R5pQ8O?Vx
z-04k^M7pLl40N1i*g$umg2yA0QAOdtjtL|CYM5*qnUI-+aUZZ^B9HIH2jSOb%P`U5
z@N8RrzPY=*JLIlvOkg-iQ&U3F$U=g33R>MX|4ANOY~T3SVuPIM<;;w3k)|W9mWNnY
ziR7bgvDxI|f^V;JUUqr{XI#P;aTDKkb;{YRTKzCG!hhl(<J<nG*PJtQ1|w~lN(A&Z
zDJ!d2k#FRQ(7(4Usd+Yzj-9gzf(k}19P|^X`o|TC<a_TN6KzI`P1}rKRwISl^(XOy
zz)p`{Vw)Zvr?nedd)F|DP&iHNcc?rdozh(gVkb_beeE@OM8^<(YG`N2KHz$Ebi|^V
zh`&pMhWRxk<L>rm84^B`k=fQHNjm)uJB%nUf(2`!m=^l6k0(M}nkYh(^^*@$+u8Z^
zhaZ`lneP`9tbClRz%ILqp+a#s^flYQ>fO^_mp%2ggpGc3RvKbdx@B*Vo|}}-*s0d+
zy@p{}y6U9HR~UaNJ;GgmmPCgB0NoHt%HKsr==r%lPgrpPZKl&xtamPKHFBnZw`i6_
zobKmELI0|P1C-ZSw|%iu!^+oC-qF#d_;CM4M|twnp>;GYtX+lmV^wIXas&u1YjH6z
z7&gZ*5>v9fLoe|VaF74=tupo>OQzr9iiN!FUF!!N%&)8{6^pBl&Q60JY(xL_jroY6
z14&j?!`%WN?vL1onq0)OOWWTY{NXB3bDK1hd017n{p7ybtb%caHTqV!q1=q6EZDM)
zM9Qq%>E%5GbA-S-KIHRm^fR8M-`7mPT^*~`^~>};anH$PBQC@Ffn|<1%XERC$-6;q
zMxMT*ZOaSfvV^pu(rZUN`-|+?(1VI1O)h5E{_M|B(6zZd&+>4(t_Hq$eoR9<4K&EG
z%A&4R{P_fvb+`DWx~zQK-ag0l`FTkB%<eQwCDdLIxrWgWEI{I)<UeP`mgOQ*qZbDa
zLgUM4_!@pMyl7KW-t}zA*Bhj|z0%mEY-?!vTC#K&$`O&^1AsTM!&<&R=tp;dAk^Ts
z$;6iU>}&?%V79nkLxbV+X5<^+N|HxVThmtH9h2P;o}evVG3Zu|EO!LtnEq_5^VIP9
zGtgVC)Qru`%bR6E8TEU$Xh)JCIX<=T&`6!=I!(>=?uf+letzP&3|(R{gj@svZ=mIY
zQD%+FYVGmgR?f(Ao+KRChrJgznP^NAM1J!UL{Jvg8(BipgaK@(h!jNQl|Pf_$n^Ad
zj=2BgLa{V9)2p48W=OmLJDAPQRoLnE@%r+@ROj8Lw--B+>>$o~u7rf3>(}~suB#Rx
zq<HLMI1?4q`CngUMXb}?)4&0QNu*KV>D95*CF=CZOCyqGl=f{K^@uEWA{%QwaIq8#
zgEKuqr5p&BhomN#?%Cx{ot*3x`VQ0DpESpRjl4m%!|%x({Uo4>#>U1uXSN#n>hzvR
zl{s-QpD-?-g5_>v4|<f{Awz^Bs-^@|zHr)7SN@hOH5b}-awL9SOfT^k_jqn~{zgI~
z@ZTRLm*p0Ir^Tt&faXZ2!sf5j)aT7{$Joji9yPG=r}#9fA}{SWTbOWIn0r_V1l<p#
zv<-dS2iMlMQYG-5DH`2O{leFckcmOA82$tO7s)hVy&wFl%!2c>@R%b0t{Ne#87eBu
zO-z<~P229TH<FKGo%?+h=12&k)HlTP=$%$wOAFd7^&FqgRHbKhV&}-akxcGjmrDfn
zOPJn+VTD^`+^ZP0hp3o7%Q%6Zi^hI;y;S9c-Q5Lop(KlMq!5NLL(6Q#nudlmS)Mb~
z(`2wtaX<Mq1Fo{_Y7g%#W$KH;)M+`U&`>a#{;IH#NOngupKTBNI#_rAJgn`D2pGeg
zA~u@SP*?ZqHpXjcys}pEH>8=FncDWvmaw=xk9qtNY^xZ~;O+H5k3AMMuvgAeHhnl*
zJd%vc?W0tp64mc*c2oVOC2A0j7k3+14Z9kW2n6(QAX!7wZF3N7M9SnACeAzlorTrd
zXUy1CcWZACR<9(eK*hm1Bg@22lww&M!HAu-VejA|n&j!~N+<Xs#uYh>&zq_H{$~Xz
zg9^0<k*1E0LRwE&7KgC#)#=7dda;0Cvzjz(a~Z*O?`r1q#60)zZxHF}>HTD=r272)
zJQ~JR<zgp9bT?`i*B``ZO18`^>{PEAJOzB{4H(O^&KgZ&YWUnh7Vsg&CZts653T6U
z*|%8t0~-U6cGly5I&TdHSfaQ>^SdyNrR|h&U{?-!k9Cv<5V9eXswwh8Iv@|m@3cgX
z(pQd`8{svOaZ=3n!zxMvRwmzj7J=`T=?h}iOVjDPB(Vo)TPE&&Naqo6R6f6`?9TJS
zXL^wM@%Cz%m_@OqsEGB&3p!D^kA2Y$HIHYUQBFgNT1L6*rE^yY#HPj-D1Jf`TcoIi
zlI8<$^Iio@Wccl#MV9Yz)EX4b?X$my)^P!Rf>U*A6tJ%1QhY1L#0?Sk!ZPCc7k=Sp
zTp6f9nsy+GL0G1+LKoJtphu9Nm-l!B&tnkx;t~)L;NhwEzdHKzN+T&D0pz(i4bKY(
z=j8>P3lYlhSQJdyQ-u|s`z?O>k??rlh_4JX5&3b^O&KzqjjZ)6*PHqnC7M~I@?z4N
zIJbyo=pwtSQ^fW>d5aSXO5a668VZ`%*NLgA#hmYbF#4Qtkfo@ca(8$4+FHv0+Yhnw
zw@WT|kM}25ReDRqj&yMYd&3;D?=!NEc_{}j4&O~Rg27-XW2pZNQlfgWzw?&TBQ3vW
zgeYrp!<qS`DFpl}>k9ta7tGrd=pYTjj4QCsBEcORtuK9DuO6#h5Dv-9OHMx9zL+*A
zeDuB52$cqanVEDz`t?}S0PVn*DP(*5=#Ar`a`^8bbNhcRRbc5OQl$s&Io>L4Br~?<
zo{eo2ECa*KD^T-}Zr7^LT`?)C$0Zktr>7?!$mdsfwy~zBriX`zrsm)IS}XH_Qw7xA
zbAdcOmlKfvnU0AmGqcAQ$gOMqZe|a^Q7q7jyKrjqDaPMMkm<-&qx62rWUa1Wl*?_;
zzpajdXV*hkSgxQPQUBJ97o)A8CwhCs;l8=*nW)jYyFA>Vt2th%S3;3#|GVcDnGF^W
z%Ef8E2s7kBU2sH1l#j3Plc!JZ8{2x7l9~jTN(L%))6-Ei&&~PBDp~RqZvSfFQnQPB
zRP}`(+v2Y%f>iW>kqh-w_a~1TJb{nhp%_C$LuF-3l1ST`f<Bkg;0yB96c0WIr>gO}
zB@H$*J$?PSzBGhrFx7dZ1WP*&WUSroFqNOQ1JntGSQ}dLZHpw?eFBx(-CCNN)qp2V
z+vWE>sG}?*W!KAI2kX86`Ya(H@-4GMui~!nqGHC*c9`$i<63;CJ8~36y4s#>u-xPS
z@7(X9$!Uq1<-Z3fCnr@Df08;)%`zKV9Lk(u+l)IdH94*KL=s?C7i-&DTU&EtHH>{H
z!kqXA>32sy{<B&Ye}cY5bAHBv<%PVQLqMe%NJ!9u@`G^pXkHjo7{*P(rjp-K;ecIZ
zV_x;cgrIOKd_K!NvqO#MoPe}X*7D)@V#^*Kn;1x-o^bi_Xx497y`3|}Ih#zg{BYi!
zjjTNN+8Sn{S()PHX*DEgmps+&{!0<U2z<CX+ZxH_bx~U?=e<vJ+i~M3o}X*9-Ki{%
z;E~aL9dPm+Yb1jkBjjHtOMB8v<x4rUVC#nX_tSiw*^w&7SBC28(j$0&Z|xN&MHd^b
zI>s#L9nETg!O-SR(xPH7DVmpC9Gxy&kA*(|eBx&YQm$q=Iq=ALh|xV%mP6L89<<6o
zx%x_$=!fJ_RT2T_mrI0<p#l|C*^g7dZiQ&}_Nuu{t0`cDzv^wE&v5HtxR==twQo;5
z)^9^=l1<9rgpB;qyb9Hj*dO$AoL>_{)V5mF9b;r1-XPoS$6nLeJ3J9|xUZO|g}%-K
zY$Y_PLaLucG5TcL*j0h6-kgAz{^b8K(e*7YDsb?LSssz#8hnha`(>iLTX+(tC*pDa
zQ<*jBU+k+W0vfNYqm{mR3YWzO96<8^q&FfW0<j=!g`wE|GWZ?sK$-4VkmOluo}c~U
zB5#p)S#$Fh!)_^ku2}1v1tH?(#NQ&qyiftktKF$+>vI%1L1;w<K6kI3pRY1dWy|AY
zat+XlT#t04;p_$@C0*}*n07Tgrfmha^{=>%A^zLDRDPZPy<?qiC5o(GH{0!FP$}_v
zfgQ7;-N?r~c->V7&ug=H>S_{od#iaJr`tVq6r<jJtQ9lqS}XpwTr5N>TjX1P-p7hp
zI~K@{0(_&yD7XPCdIZCLlHfjIIAt5vTrzvy@UWzm+x5;fG&+0xavXGQ^dw})Tj2bN
zT#iUSZ|56cCJ)2S#o&oWFTvORqI31+pJw#V^blG}5iX{DYnf9Nf&qNKXPcFkTvl&g
zU28Sj5@KQiAPPn4c43|Er;}$PXc$Hl8b>uPkK(1~J$Oz8vy{U%bdY|(nTN9t!`qPu
z!!K~|xU*DKo*BpalOb^l#gUIZ+l7j=L%o^rd}~vP@SIqwrMmbW+@z&LX5!Ay&X}<z
z3qDD#eCntjH1#N!Z9aumzz)Tu7!T;4R|S0cx_V&h2(g_pjV_6+I7WJ}_EFV9==WZV
zdFAhPC*_rb#A|B52aB5Xt$$a^rxO=+AYZSCzj~_-h`6n9XVA5qo7sxfFFBeG%vsvF
zRwP!en~c2b5~~U4ROz0b9zByN`;zzFO1GV%Yk}e0`RKo+P`ga$Ls*mhZF~()X5(T>
zVk6jE@0lJ0@K#adT@K@Wmun5YEY$q9E;;}yU1eQdYU-o8%0Zia#1}^gPRrj2+4KfR
zVDU}QlE`Mj4U({uu%cKD)S+4^Z;KtZCTgTJqH8ab8)}2S?-Ou&zx5c@naFoUjTxgr
zQl|QR<_#H8&=zIZsdbHv-e+6<a37AU)`gt)oOji((79j%DQ}Af$0DMl&_h&TzE%gM
zQP)tJRm@Y&{otEW%m9Q=+)PhTlF2s*wU_eT-cwX<iid|syR4zDtu6igQv5)=uC8UO
zgY$nrS;F~Qf2aS|C&Tb$*;Kdd!PgpN_Z!Jl;2^*4jH*@DxAeh`TWh1=>roI5&gI|F
zb`g@?p67iM8*l8_pZ9nVNnZ`R8w){uK$B+m67}3WFAom<kAoo}?r>CocVwC)*B+6l
z$yZV$pV#208((yEbfg|O5s`Ri{)og#X055xYjXQ^+3Fu`ajZiH#*sl?T@ii6uj?G<
zddsjJ1gWhAyC~AL-J87Ye|#AIsT2?pP+n1?oHk8^AN`Es9C4boL+hM_(h7IL64hpq
zE|<ryTC4svRzriY3}lp`k0*Z*au5$28+7KG3n(ENm5D>;UT)r-aim`!Ea<S2p#!~q
zC>y3u?1w_3K_H&_tIHAIz~Br@5jn4e@Qu~hNL3&tO-eGzX0e>i%Y$NY>1<{)CN{Z-
zX4&Czj7RtDcH*ytNV`|^b;=b1>Qy-ccaQ{4f3LY)cpIp7^3yo2>Fugk?c;Dfc`VuM
zFa+`#T4`7B{h9W7YRd}vefTck^aUBVJSNjq09<-?EDAb`&D3rRvNOVB|F;r`Q3eX>
z3iaSQrpGU`?$~_1-M{}43!UT59Q$+Y{`u+F;oc^rIT~6Q0Icb_Q!R^8YY+<vSBdNH
zI(;Y$HS5Cw2ZPCOb}nw;ILvlDTrQ!ilb7SjIbp$%54m@|Xx)%mB$_mrpEVJ%d>gog
ze4P{0_L{r3xnNHD&RECzdSD`VSHJc1ldo#IA`M|g{&#JWZLz?Q;D~?=a2?Jqb^f%e
z=lV&txf{FIV1&U`6^+!<FngOZqE`cck8v~})Rugy*FQm5TH}_VugT|ZpW|;c^$;r@
zwBpC7^fDiho?+D#=8g|wzEhR5Y5&|hw=7OoN~70Qo8f4X)=%QZ1N*$;HymMQU9#e<
z`R#mAEmZfgBpI(wYe<TU92<~p$Ev#iURG89`Sjd*{#&GVR#a6p%T|fC+(Kp7SQPa?
z_S)J`$42*Fkdt0qVdJZDc5@%AtcnmIWL$f)->Y(!fo#Q3(9t;iMzHBWoDp&rG<V<w
z#@s*yuAl*oF%m8fMCEfzZY3PS#vG$weS05_F?Z+LSFf9ozP6x<*zz*E_%#5?U;{t&
z-`i{i^KQd10Wcg`yIgTgRCM20S8?CqM~jeVMDei!W0%fGlw6oC>|)<&lws}e=2lQp
zfEm{0cazTe=AzkZKs|x_O%7wsd4di~Rt6Y}uoJLgv%i`<G&j%oT_8LrC$~v-OQovc
zv(nN?XqMVLDq9>fu>nd!#`R<=B&E350j$&i=`&>@1o?nv{qbW0owyeiyn^u*-|ZLX
zs?&`#iqRj!I`7I#!9m9l-teyLo|3U_NyCSQqm}kN-%tJjnqfYHNC##n(W~N2m_fs5
zM8Avo!8|wzw0@G>>%!ybuyXsz<>P7grQ)xxCrYp@okG#Do`N1TEv<}IKR=(B&*&$I
zy*iAw|H0w<Iy&5BxG3VJ^NcR$VDimRE2WBsHbr^m;>i_%uVgy$*_E*e#jeSzk@|*?
z4r@+s9v*sB3;=wKE<+h-c6RouaoPL5jSU?$GgzxTTnc$zSgY#M>~5KNV^MSFR{$Al
zm&*0?xc|yRakMw$RyYvQc6p!Z(&i%oS}rjFly5KYLF_m{z0{-Y%w0uhK-&aIMW;nP
z?+X3-JF~t&f7DwwsvEu|c<(fh4*Bk9V;O+8XMJehmueaZ$fL3|hhs{AM6%Qo3h{R4
zY8ZgcBD$0d+LfVwn2#jd$>>4EqvPV1dR{rP{`j|xZ(HAucE?7@J35r$e!})bs-Wu?
zy(p`VfJlA~DIO1;Cn3he6`|{<V}w(??|1_shtm-Y?g=>S(D6hH65!Y4bYy}Rm0Z7S
z!$MlJ#@^}g8&el3gCh*#-brHS`}*3*_b_=i;L^S(6*%ebXW$s-tbCqxwGH~gIp;Ly
z^YN%>2p4BSj$pvw_ItTNql~9Y!H!nIt_T!S{x;F1<ZiEftCV7#(2Kkzrxo`HW=JW0
zJo0Qr>Xq6aDG^cko-U2|Ht1&We#r7H3B1y>jfix)d=XI%d{*P%X%y~>_ZZdBs|l>v
zVaOk)04@U{@K5V%it13`ODfy)x(sfWxvzp;v15Pwh2(n^{RyF_q7TLfS6X_FoGrd`
zN9?biU!tM85gIC_zZ{%uUl+Q-#&($Z%P^#xL-E1HH($xc4;^1?RJ=`3*T=PX*0rP2
z;n9JiJN+vPqBm7Ee4ad!ZhV22?d0$}Z(DC{J7@dyK-?ykzi&k6<xh6&NHA#RGwo+u
zD55B$tnBAUN(#BMsLI(hdv0@`k-6Eh>AAKE*>G`O1(LY!9`PrMG;jW%ak@m#Rk+i?
zQGYz!&lARb7s;$*`|IL&y$a|AJ+ymi=?<lYyf1sNUFLCnzT2N|jP^nLF=%Y;Z%HTy
z7Oj{^1nbb3S5s)+Q&UrYeNO-%uy^_pE-tQ|oSackYp0JYjYn@Z0m?_$fc=kda>czt
z7t>SX-bV-1?3Ep|TN_o^EDwK3lcgG+mNZZVUqfX<%Jb|`LV^@H;Osj?@IAkzBodC{
zUn+6=ljX&?)6s+rn%S|O(}TU4#~`n5ci`mS9r_5DG6!8PBN!g{3q|YA0U*l)a(&!Z
zBjgCW9{KIM(a#`qf`}nz$?Uos7mue9ufq*PaW-t(ZbXMNTL(i;^?3%T^J?eD(RPM;
zN?yjLu}NN+Bn&15$H&1<d;0<_2qwkG#KbCczL(C4+uD)tkQ@^;I5L8Viz|R-Rd{yh
z2K==>SlRd4`Ezrf`at?%3asT#VYQ6T>SnIo9L`mOS)x?;|KR|#=&~<Z0m@m8Z;yr<
zX**^zyYW_zAlc4?*ie9vXYx*qU_kq6w&)$9&*mQ9<GqugJaq03>l}&M?YG=AP&Ck~
zfDZg6hnKM5=-bSR<G~HbLND6CeBw#iXB+7#eDHn9@{Na+7M0Bg=ZaG-l_t*^l<WSR
zH{^{{QO<$~iv{iOWk_gror?jRTU(y{UdJnI?U%W<XV+iNx=P})9k-LCksVdc0iIli
zIJW)ik$>oAfWLhM*5Z8#u6HS$Q|{m<Q5$EN6GS~<f}8{(B3e@+0n-<>?A8QcjvXJ}
zn}2*CHhMww4jU!);Ur-bc#)%bpUj`sM^_?Ry*gti?UJjPKHV#ax<oFnZS3O$y#?ZE
z(!VPB_&^5V&6lg9c>&eHPaK3PZph2uiN>G0E`^BD#nT%dhxTv!2;Nz~p4k2;BwX#w
z@^~?_y<Y&Ud3I!X@dU33I6CY=|6s`I<g62HTs8Muji6%HKu#1kE`(?(CmX}@c=M)}
zu2eE_nO({H8X7ER+`(wHx$eyyT<jS<*f`VVb>`Y(PXKRlgm!r<kferb93!`1el8V>
zLd6Xm5rP%yVvUQ3Dc(wN3H{<;?}$cOHuzam+Xbe-YSM3pKiN!fPo@{Szl(gQZ{}~&
zR^iYIE1uegT>BXZX_ZAs-{~YL#qJWXUnhNHO^soX6AipQ*EbK;)zbRynp0DA+`Jk@
zLrm<7*jC%nd0hoRdg*^&tCUPD$^2y~0sd!ubyZNI3h6QSj(7BEq2AZ#^3&zNT9YH)
ze7&o~C2xc<O`gBk>}=Jl1Y&d)Wm^%Ek?H#H6M{EW^O^#sK1<8K9qx)HO+))I`DgPO
z^60$Q9r1AIEhXj|7yg28+CG|q?lyVrc+AiFx!O$F`#+Pn`Et=6h@%3O5q+e>7>kj7
z2pF@;s}^<JJo%g7`>cqTQq*!Y?Zt5i;?lBqKHtHgIZaUWX&TuH==i03MOj&DYs+HJ
zmx5hX90lugS2|)(j@{i_-`+95-Xoc~InUcwXm+XP;&lIlHs9W@NP@1-rYi1A;=>~-
zB2uk!oZ$p|&A^a1I5-$j!K)}QkE*T{78c%U{bXS9O|h`{@k7`1k4GQc4wx|e$vupE
zOyyeqLl@neMqUL|-6kSOXWG}b&*pmXB+Z;QjItb7JY`5iwtcci2e5|f-a<R`D#UJv
zT0^Ft2k@M+|6ByM{25eTI$tK>*7e(mJR0Evx>4n(tH*vHdsx6`&VFTr)rI)LKk;Gp
zx5eJadS3W$7(i0>c-Pm*<uJR9);FpLtW0Rqszf?ZBZ4md-#`(VX!<p))B;ER2oQ%6
z{hoE)sr8dNp*$<I9EbrAtFsC8j%5RXI<blBG@$UV_2*GSHt@mM4ExvCKEp6g4g;#h
zm|j7-=)=QNla?t+u~U>D_?b1CJRX+WtYK%@8sF{V)B~~@>W&YW1%Y!cSoY#W(NoYv
zF!9NzY1}9&rcQewzN!~al^+RJR*q#9NiY9m6(V)wadzKOyof_zgFe>ne$@TLsqyRu
z&4HY=ndBnJn-f|h^fgg6-Qw(0Z<eL6w)1{M&8gNGrYkE4f?vZA9XU1j-?sYR5MlC{
zM@s?noUFS+^iTS9iJzagSOouCXu3q83`fu1sHxN!%#mJR-z57-{}(MD?Z7YEu}KmP
zU)r7aH~8k<KO98cGOSU>GV5_O#RS<eN|fX^u~rFIk2<|JX7}GAA9jcP1oSC3^o>jp
zY2NA8cKpz>{W}6xJJoC0tj#^rN)@Hx`NDd$o$Z3$V0irO?ftkO%}{OH{v1_%4TO_E
zZDNE*8DywXOH7W99iFVSeEr>BSIIvaFVFm4RdMRPG_NXf`};0S)T<^&#LNEn&aE@I
z^m3QH7PV~QalfET@6|xWK|S<%zLrH$P`+4O?YjsXP@)e0I2_NK=i|r$s0J5M@fx3=
ze8vjVHxI%Aq&d(5LPA2o$9-KE_(63!1(kd}sBerWWS|%LI2ei{l}syYzdPBy>U-xD
zYxSEFV70VUQ)`TF$Ljqn;U`~8FYyuKNlQDBd;M@_Ar~!$oY#p|coI!WNBhRICA=c#
zZ4rN`AIf~heS63njnaKD-%rM96>n_?=St}t8GY(_$(=cZb-oWj-p{_g8`|7d+1j!;
zw5fl3>2j0jh>MN=A2Q+LVE_ya48p?A2sm{A!VidviQxnD5K{83ovil~xR?vX(_IdV
z^NvoeJd8E=4oU_4LEl(@#A3J#DrE4ry6h<Dl=AcP$MuPOv)rb)&p!`gHhGKQ&Co81
zVcT}<w5>XmbLzf(@7X{IX*Gvy9@&R7)#1gMkSA2j?+{}H?=OzgT|+b%Us!0G^!yO9
z(nCqFeMKr?WMyUF8<jMb=$Qu@?oD+zoy$5UnsEs0a`WDG=$9TL=`V0Ur4QCqCsncb
z$#lDWKZ>y-c2$YwZ>k)e>BQf0KYQd=&s6D9^(K7(H;qU**F+hRUf+^JQwE6M{o&%Z
za2Qdc$kVpY@RiZt?3pkMEt<%4eQo|L5R(x1*zIF*T>ihXKkXDJTu?nyS<wppw8_ul
ziZPM8LWgBO3MOKr=cfay7$9S{6ngJ!>o=pX_&OYhk>QI=rPiSjXd&gpWd2#sYAxTt
zJcrqE6>@TE!|1`F+`<ac?WDgiNJ4l!eKdEd8C~5uxoWixv$^f+S-s+H8e7H^@w)TG
zTTbQwZ+@f;ZLh{Pcvw2bR@*$7f5J}+-r30>^T5D7Le1=eprgb{rK8^JC%J`XGohu{
zRFKpA;dd{!F|W8FUK}lO>^37qQ&i;>UgId_|28P}pA8DZ8|*cbf0sOG#O&dSBR$3J
zJ0%<0GgO4~8zs&BY!HNge$%}pY6N}#G|9%-f(0t>%L&TtI$a)dK`u?1+CS?D7~MTj
z&(6xrCwK!lL~YAxgwN1W<sd#jJ{_n1H2r^^SK6p_w3CsSJJTxM<ICD=o?G3>Qx%m$
z#91}!2JMZFFK&!m=etuRvGP2bC|lh~7RL^w1dCL*iV6olEGXr()oAuzBrdh=?i)g(
zu$G(s1t@f(*4_WJ=4U&@*1Vcz3<o==EDQbM+XZ-Dl|;bx$=dI59qk}`QMdE7E12PA
z&Ifz|8Pv5KT=GgvK&G(l^5XRHFF)UYjeDS_)t>n}T*JoN^B7j|FgKX}<n8wg<JR&p
z3xoV#4P7+pNF(y+&r381gVh%ZfnYG|zNtl>psten;KSSWUZrPhD7~#xmYf*YH#6h;
z2)#3&lB}(*wX?H}jg9T!AIK2)GKB<nc65M*g!E8}EU92ge*W58)dCDt8{{`SdW4(#
zH}H`p;;z9I_&a8NrnlGnYdt=izsB^1U%;>3rPvKDUK7MWsorqAxo^F4d592(EH3a^
zPOw;|!R=yYeo9a%!L#pye0A@X0LnKuh`;I>g?;6Z9PhkJ$rR+3Z{N68DKU{==7Nq9
zYFOdxaAxeJJTVO|@A}bJ2y*WW--3Nmg}i5?Pw$+e*(l*ta8buo)!YRJ;OwlLvWD+_
zVC~Ib0@Xtz#G)-CnpSe<>Q9!2s8)C5Ujud%)E^l9IG(o&x<0OEqd>mR<llZGewEvC
zgQ&q6Ts*~t+0T}odohuI8h<#*p5{BsbnEfHE@GEx>+x=X+wec?-ZH3;sB0H&oZ!J7
z5+H=&1a}QC3Bldn-5~^b4-O$5<lyd>;2zxF-5sX$e&4N`sk(orZq=Q-{5j_|yI1eM
zdhgZU&-1L+5?)u7E=N4uL6UEQjujnTLDP{KE7!~TFFU`jA9>U%)Exg@Tuj?4q8dDx
zBB+&mcN{SPXuDTlIy$7)!yS*;#nHQrd!?enM8dgznS)WmKiSl)nD#^}JzEA|ziKqF
z`MRqcT}l#@6OTt*$S9eX&(A2pY3i2!ye|s%cqe#ud$aO5f416wBIPN_c`)8nN9P|k
zUH!GF`m;|uIs!oJyX^M#H;6d=;jY>4*Dw8PK3SW(>S2Bo=;W`x1mDMP@m{@W9|BlU
zc@LVLXBTI!r%u-e&LK$^32t?cbRS`rc7YXg@A9uxh`_2dgYqC7V)S^mGkWigsuFMV
zS}nYoLu7yvUGxqpywDi0-eIn0nBe2Z25Oq-?b=o@9zQCZlrgpDQTg$ux_~(5_HvJj
z>%lddo10tKPGDdlNHAM@!-7DYBk9iZvG0bd$Y52ibY9W4dGV9?CeSf<Z!tl>!OisX
zqw?E0cTZ1j__GN`J}0O0q4WpC>MEKtz1Gb>Z%_q_CYvGX<au4fv+`1L)lHx~-{#|`
zuP>PzF}v{4#ve4qa#*Y=boW^OLMrI!Zr`;Uh=dtQoK$1Hw;;RXQ?JoEyF1&TkSF=h
z!`<DUzaZ8NWZ^~+K|S^NkcWpm2U0B!l_Z}t!;}%7o@g>Xb>{@6!m?sPhqsJ(ah1cG
zgIrqmOfS7c^z<s(zR&%KX4y~o;8psr6U@|cOg4$Uu`SnlxN9NMdmIgB7Sg>t`Kg7V
z_h@$X=pnxng9y-0_#S~8g{c*x>4U3mzh@71_1Vik$V!EVhQ>CuTToC?!0URk&PL1E
zw{3VQCT_gv#rt|T<4*Ul>EkKLekS;?-&on%ntCYoeyL?B-7$xgopD?YYFJ3J&DUn^
z{K7AJ!7zLVFbkZIeDjE~-mYFt&omp+@o>2|`Vw}U@Zw!CGx4QpcUUcZisT1~UQCs{
z)u-6B4&JAi&tS-tniROd=(hHBcL$khfVZV_RFco2GyqI_+_?kqK0uuG;>&h^CW&h@
zUP>c7H`)hpJZ9$92{U_Z`UT0d)4RO_vYuFLZx|;qlE}%Gr|6|^xW9{D&OX{W(oxkV
zpW!+_C~r^uxD#|m^89cvd(n1(KL`@mkGe|LiF&?w##d>-Lom^a6YVLtcE<DKmvzpk
zy=0SXC<)<&%a50oW5|8i&-QqAj+WN>;O@~Q_ooF;ESJMB(LjbXnv9qKVdw%BV%DE5
z|2?<@r_joCD^)OuS51HcVb%`F+_f!4fZve9IlZs%b}sbN?H*A;LH%}snR*jKIZ$of
z5^xIgs7v*BD<Dt@(s`gZqDZ;m(zErStPf=#q$)ua%4t*}q;oc!r@Za>@Ga$Kj*)v=
z;%ipYJR=0p<gse~Rbj~sjwD2Z-;SQ&@_3@xc<}uJK09)Lx3yyE*Dp%WW;1PH<HU(^
zCaxT_9J}|UV&oMZlVnD88s@9ki6~&NFE8a}WUwQ;QG$N9v}Axvzzu5b{<3mc5UX+~
zUHQy_|A6$+?9qUyOaF_G7(lVzesSwgNyjL)GkGXf*-{j|g9ix3sv;sH+S~nHuY+V$
z9K9|>9a$1g`I)y4Hk370m%-tI*}RS;83faQcPnODX*_<N(3kqdC)MdcklERWnwpw7
zV!_8N?QWO5tS3w!2T%F3TKJJsL27-ibtj8i{DP?)+*`v?N9Gg3%2uRnijFGM-z*#)
zDm(<E%}ot5`Ai_v5qdoLvBId@uAzVk(cg?PEq9N3+HN&qC%}v_*uXs4#63FC!t^SQ
zqvN7yc5z+JI33Id5x7}&0@>8T!NG{AsH6E>=-JT^!%Fwx!RWy0qSGi5GYi#sr3h*9
zco{_X=&`CCGk1aB&kO}dg=rrce^+M&tg2_3c|^l~1KwCXzzaj6Q!{^SkgqB3mz2xg
z_s3ZSk^Ve7e0b<xk_5Nu2%~7EF)^y$-LZQOx%Bhd`PN_cR#lyY;7c5}bYsfSsX?HJ
z|IeRVbaZrE`CBt1dV|kTkDD6GrY)>+fTO||!w+r>prxg?ctGo_;BjcO_&f}!7fc-f
zK*#htq8ofrTv}RsMqMi5CXVsQQiD!`jLR1fI?lQq0x+#*pT4~Gd2@w%C@b$v(WWEM
zIQ|JWo1sQBWDkcnTIl8R(C6VMno!VjT+5EZcOxo(-B3tKFHTCF_j;R|zSYQ(d1uRJ
z?)>O>^VKp+HYoJ```0iW0t)hw;YGg`#(IINHROEHTIy0#a^<?i1B<s}k-0?R3P`>?
zeNjaTdJB;6E2IzwZ+kp%B)wpV{n>U7Hc(H-LxwN4XsxKI2tp+w*6RXg7j#BHQ&MI?
zcJK6r@6Gpl@$y++x8%s8lH#}Su>Ly%Mjp$u$M0gpBk03lQ3Bkn{yQFV>g>Kl@ad_2
z+B%|Qyo+?7Ll6=8UnXy3a{z#lpYk1b-%C0LCjJr~D$1FK2n!<cjWR3UV#Bomwm}fE
zcUCUeRimdwz{$>Z-z^Z6bj$Z`dZ<=d;-hEQQA>i(S0oY6SLk;>f*eLNZVFhA<gYCA
z2VBQwm&=yHad&Oyyw|eSK?p&aUA&osP7uF#U*<OUoh<@ACb~u(A|egtJl5rROYhW^
z>B|cVg>1pd?Zaj|L<Q#1bD!j=B+Vr0G`rH37oJ{RS!wWd5~7juIj<6e1aYQgkXuS<
z=xPpDV@g-fjFrp$`FXf<!Hre1Rg<KIE84D-w)R_>xF)H5(3f#TJkb+-&bnDy1k5_8
zr}phWH~6Ee%UAg=4uQF;An*@P^d$zR^NhU+5-vMXggi4n-PF)fQ}W65OA$LO>-dPg
z7V?`(VzyN4ePxW+FI{afUN`wI(7QOD-wZfn-(ZtlIhqYTN!I=0{8vL)pR}&cvilMJ
zb3!|;QZ%{wmv=R}70RI+75dLh&Ht_@Z{Orja6T234UtCml9M0vkGdU9t3an{qR3TL
zRIWOOmJ2L+*?$?)E1nQzM@R&a@qNMNxXC9dE-qeH=hlNxUGGh#@9;D(LQM|XvYeE0
zx7OCi%k8{1H0HWd2^2E8bGZnSDZhil#Mcj=ULg`LDKenKCyNpph7K$Km(qJ@YIR3a
zHRy~R3hKjJ?+FxYb;}F1q!g&lgofrx`jr+Jvq4VzPx;GF<8O}1pfe<=xcF}_Pt6&2
zm)pU+-<}!aFdD<OpLF4{z!zt|Wpop?eqTYyVAfr<;pVq<kq5o-BC*sK{Ftr|acM7J
zLSXS7e%7n2D;Pj6f_6$Xm|7YHQ;NP?+H>?>A}V6WjKyN58E_|a*kZ$g6%T$%y~hm`
zH>_0ukR(No-Mb?B0Sj6C*WVEXgU)~e{#o}km+a-rX&3mehM#p$9D$&O5M|H_P%15R
zy|Q;nXoeF`Y6@?~1nVSlM}yQVqk2W{N&B<c>whX{rFRLn9wXG!P0iq6{alWKC~|IH
z*Es`VoHZop$v{`3Oeyi1U_@8{uFfOORLcS?<PiXtpb`j0rX-9u<-Ql9Zv*Qb^+{6+
z>0W;J&`4#C^G7oIe!j*l^iMfhapru%pqx_4@AWi)MHl!^%U1&1Sn|bGXx=lM``(vn
zQ=5Z>V`n7G^}lqFo;OD;Y0!X1Kb^b7C+4sDoa8p~4}B}@tU|?#TxM#ofelts&?;re
z!GGy04fw%>e9usHzn6nl5BmS>*CBG&j#f4**z9}Bu8SqW?Y=t`A79n8eRgw0_R{yh
zV<!`Amiu6KQ7<&?c`Plt-FaaIYBhJ;W?Q`q&KB#XkuaaaZ=C+=H*bP64fcbb4FCHm
zeN`oAnCHCB)S&IL7vyi0@ZYvG&rlw2PEHODslfIxfu_T0Q!L{Wkiw~XcMn>y&=k1!
z=#K=1<z*%uOS1Sicw|?7qF|CNnvJ`ELvdc=;HRyV`5#@V_86YUZp$77HR&oNOy6+8
zdrRJCWN^ev9cPxHslGe=7UH?UB{(q1A^n&2o55{6KSz6o|0qqzp0FqA|HjQ#F>kK9
zb5tQUBZ(r{Vzy+1I!Y`@!T%Pc{@?$JspA%UC+3gl;41OYC+(FW4WFQw#d5dxYNQPL
zpQpK|5jUm+>`C^~t()NJkm|<fRK#q6Ug&GlLZ6z>i(aAf6|1|-iH?9upXTGIQNIkG
z+Us6TxrEZb@6HUB@+ga{bB=(w*4IbDDcsy?bDtaA=_YFot*R>_?3+7V?Tf6wrFy(t
zm5~=dB=p{mO1@Y}0DqPZK<N_({<>ppBlkq~1_6eYaE5$t81O7QpKaF}=J2E%_udbw
z!c8td1f#*|2ejH$>T>vx<k}sW!u~}PhSdjD(S@gI5T_Zfc*-kz>q(sNO<C*%{jtEB
zZ;0_tpTfUVV%1XOgss+4g_X{j&Q*D)of{%#0VSc#Z}P_$mjd(E(@x$efhtzsCMlWu
z76@{!HniETS}8F^y~N^&t`-RZt9<zukSY_UF0$!--6OIsP+57t*q+9TQ&8}jzkdEO
ziE%WPduHjNZ}`=)TnUrWsIZ^jt=-}A#z8SUNlO2OKo96hY{Y^2_pR?q2Z#3RRU$Hs
z{&|TY!|&&o#+*t9o#sgNJrd&mVn@6$h*$)w^krF}zi>qZ71YuS8GN7BY;Lc1A`3sW
zW^Un%A!^f`mXR&OeGPlEJE+2ZMJmYo0c$bHlC00kE-uJIQrIn!I`wSA5RE)nvTXsb
zsVVjlL_+7JyY{~YE}+aNhZxN|Tu6#s(E;x{+)3NWoi{18R_{Mx{qQ>@^fT0YDy~&n
zbhzjxT3d4wLhVBglV@0#`Z-lp;DA`gq~(=%OwKUuO1k3qbq6ao&7d#iOH6tQM#RYz
zMz!*fCKN~5l{cg4k&rKK^VgeDxEtTv`sXV5zmXZex;J%)$=eo#KEvC-W^Q_b?U)e`
z+l`%#`#qn{8GFE5vO^l;+7!y&mYy%(^m5_DvcfhakNM<L82aFb6l2b_n!Xi5-9V$k
zJ5{3jGK|5-$jmHz@0+%s2m=-UbS-q^Ipd?!y$ho+%Xj@nt^7Z#P`zB2RjXNbmYU^!
z*t8w(f^(W)*@WF6LRv*Gbm9D~)34|ICxlEVCv$V{4>nT+=Pcp+XjqKD4&|JQxsj9o
z9FaBB;6(O6i;X0jXxyH$X`Hfd7t!gtCO+WxHouO`yl&@6cArd0_{m>`5K^zhFy`_K
zOEK+oXv+Ugvh5+uWD233oc#XJ;rVByk>|(E$|a;|A`&KJg=fzO4iJY#gyCO#u?U0r
z=3gD&94RNgbvUP{aeFcYfMf{5689;Aa9HRd+!r)}RIJjN0Vxai{c8s`e_W!EYlPq4
zY{0m`@zSN-ln-MVM!v^UqFXOdGvx1Ob4I4cc&!v9`uPoVUyrKzS)kOrbi8YP96#9J
zF>y8WmOqPoA~xdzQZGL=l%FDEywlwx9FsZ<+X4nY3;L1Lu_woJ_HhTK_(O4}YkIJW
z(a6NV@v##C9TD&}uyIPO<9kK!qk3)2aDbn0lI=iB9-Zb$;8*JRkj7|iX$vh}{M_xz
ziB_vCGVNawTZ=4CLSE#sFTt3JQsImRFzQXJU9Y_1wHy1o_4yaM775-D)E8vmNQ>nt
z{d)asKa(#QI4tKYY?8@QjPyHAENXvhAP0ztwcS%pA770{NLWs!)|BeR=b@wRjDAam
zk5VsWBWI8F`sw09JeL<FPWKxLj*vQ@EkeJa2+bE;ng-CaLUxYB)l%*KU>2AgD5WIs
z6wm_!fI(A>8eV|xt&v3ig*7LnH2F2jzxJ&DIc4r9E)!(b@hE|vqlO7QA``!FDt(Oj
z0mqLczmh6u-p_hfe$w@x-p;@kHqeoxH$IW)V&^aZ)Q9`%y}3R|+#u39M|^uXqy4I*
z@cHw$$9G`Ay1<Xwto*b6rW_3L#}&hVb77?N=_mi{^j&0j`Ci%QQ`t8v_JVm(G~V88
z1ph<q;@ihgkz6*9ls5q72^%$&)EDn(eb-l0_tX5+o#Vu}dz{oh9(mqJ4|c5=;jBwW
zr{R7Xj}9|;TxuEfN34dmtmAeAGxUn(D=yE&ei@65M30zG?{Z^&>y9tuqjqLO_&rXD
zy@k)u<qJ|uxNEiMx1=YMQ3>z9Xb#-$ta#HgN?@^-jpK!_9XJW~v{xTLT-d<^<6HfB
z1+I3yt$(a8n@h`?0W|4Q`dfjc6p~2}lq##$f|`YtXk(IgLmkokk<`0cD>3k+e}I<+
zr=X2@xOJw1&M8FE(<Z*8*B$tA|CJpOA3a0@p7hd9GosE8Sr3DVl(#Y5PM-TFgC^#G
z6(Vz_WF8CnoVD6oq`fm(hfeUKrJjOD_vPV~jKN}E)#P3_Qs9UA-<5{1zkLiYYjkbD
zxbN2n2y6rr<3nOiqF&COFUy1mn!M6SAl7DW_Gz+zLcu=7Wbnb^3Pgr+x=B0KhvjW6
zc<$G}31b%QMtDbz6Xo?&#lxCGS8bS9SC;Uli#pkfBPDrDyQiH+SFQ>;owR~0-8W3S
zm{cohbf(}kI6jpwmv=l8TyC8a2PaesZ%1XTQ67^{qd_$*NnIqA5W<wv6MzPA$I5Hp
zCOV`c%z>lR*4ejVE5JIH-@{%<1qe`%)0C^2-bbO52x%&CJ(^F9lhL~QHNGUPcA&O+
zh)C*UdCQ()0x;yFgK|upDMUiWHHg3|);Fq^>pdbAKqJL2Rn6`AoDm4!1JhO9-?su^
zQ+gx!X->n5J{A)cG8JSnm{*adEmT2Wv)v8dzdLwdAf5Z)^!o5QhrbllY0ad%Zop;7
z<Zn~^@wq$pwf8=q1}wI$#_J-*MEgei9*oNm_NL*ipz$QI-B)I<w6&a5DM$KQZ8azO
z=~Q^_8pS*|t(L|tn>$P3MqYg<U|)G=zhni}@5bo`+I3>VeYl!39Dd<rIAe}h4|w_f
z?zkv!+WQ{&yJ>|x@nJZ+j0=M%Qi0{2YjPJ$wv}4_@oLQR54I|>0#X6jF^qpnl$U!J
zW|7uL;rJ6=dA{~ilaYAu&yg-jM8MdSet}nr`XL#_BKYJ(5cmCKx1H-``u3fO(I@d0
z!T5(^#qs)>OKDO|?c%kz*!q_S2fg<^DOM`jz}Oz&{w0}9Et#`;;?62<ts668dsSv5
zRxkpt?o!k5m&fNgDmkz0VI9Yw+p@*BlM%b;PWq=1+D}dD(|yJKocQHX(-AJd7TfIW
z1AH|#-8v(=pIV(7aKA73?G~FEHMSNtP^f%t7iXISg_<_bDs?1pH#IqBji%l#!ioLb
z*ZL4R++#h<rU|!E-gJyJ9k%0<T@hL)+C$3#7HPpSc>P>_ph;%b*zcZob8$g3FFaGG
zlRg!chR2acUVfb<4G63FoY)pEm`n|;2TnmnXGMYcX3}i-?mN#*Vq1u62DMm(>3LTi
ziF>%|HY4?9{M3L_$pyk++|6(oQQ-6{DHY#%ZMSnH&+NM(t5G4dIVKr@Szuj?)_}*f
zd^s`K38p+DGm$qtLLyWy%-4kWT0x5E=4x6GgFIa5oY&t1-gGK0kIQm(cYLAouUDf_
z<HMgQV(yoX#>x6`QFMxEG46*~%3PL|Zoi9kcBlkxy(NGO%bOw+4gkP;K%^ZObZJI(
zbj$X<FXb$R;0l-K*<jRMqTDPuuGOV|J1<#?X+L~2)A8)tBe6uWT`csFKvv>{7;>ax
z<LzEe1T0vhSdM)7{Zn!!<S0;S;?Vh|$9QvDRe5@%^>=H1*^WwDhgQLDuO9NyU9D~K
z{ex&_sZZx&(On?mp8_8yQ_XLwKrUq7B%4<`G@8Zyz98pT^ehsd%P4JF=dZk$eeS|g
zbL!3olg_KgYgGT)mQT=g_9n?4@BsSUnSAd1hXRd$$WmEbi;@_~(hqxu4#B}3mg*Cc
zB~1{3Td7@x7^;x6?31W=alAFAUlh#fbY-Y}V|&)V(lXZSl_=xU=5<rJVwZI?&$z$e
zEkt752I}O1(8!jXnbh!zh;r@5zjZ!C@p(6D%J}F7d1(&0F8lNej6%m6&G^7#Gn&tO
z+%cP&NN28?42KB|fQof_{>WJ}`-`Y@`%63k)H5)7h?Nl=o_w=IYVaZ`D_XFxz3Jn1
zM(~NVaB-k$dn)f25J*fmaX%T~@pZS;*BDR^f$H6zEG(>Lu>ShSYOfiUweo%j4%v2Q
zcbIh^XZ2GQxv+iSCp(NsPBnOdYe4=^8pf6+<-<(9yZ{knXp4W{W0<LDqzFn5#2g4n
z`^o65>0jBra3a>#WD)_LFU&x$72a5C_8kzgBwMi8M2Y2xQoxm9b;!COETdr@qMDlh
zTjsgHn}6X2C2fBwYO*SfGSMx|gMsJ3wGkXxSfq(Ln&q=GuW&Ujw+e>wBOiI4_%f#s
zL%fvd!T>Cm8H7NY2Y4|H8E>iBU6CCg){oLm#Zd+!6>|)#Q)}uX+U2J;r6EV`roWT>
zTOEwn$0{vK7(OgM%-Fd`c?vQ`CAsV!K3!CH1*c=~hipEOMRuOu1e~_4>dH&cG>I=8
zBDOt0LSjVhp$7M}xvj<kEyknkb6w|U&}?fzKiP9K43??QaxZ6t$Y9HNuN{##GGnF8
ziCnQ7tpX7aE`_?!Uc@tfPLV8YcQiW+<m?dNc8l$!ZXdU7x|xq;NEuH!+=%d9jP+i}
z-e+=pDUm6kTo%uKdVY~pEG#IL<wi2;?GmFXsidgVIKMKmME$ONgKbk*=&RtF3l++v
z#~*0s`I7yvIb=V%%4joZVdol#4nBGZ%uQ=lGH=0?7!5?bCB|xk+x3XBB0|5gW9dAF
zeN|EWE25mLTX@&EVxcv_#2iK%Sc9qabbPWnN{xDOZ%c;DLXg0DfRzpOH+PS)gZT~D
z(PpHUkd3^gNk})TQ~F>sCbbRAr4J2ouAF_aK8!b`Z(%+jJ$cU*dO+=U9+pz~#Fck;
z&R!{*tuUIwZF8KAK41{o46)O*`^>lC#-KG_Q)%R7H06MKv|Ys5q%u6KGtgu*P$QhU
z?It5-c%rSQ`ti4HIr#mpOnC~Cwe{<~^>xFn9gyuic4I-5>@gq|;{ZaJPIOii9}w9a
z6X)7q4(lGx<PIs}fCle}Ww>5Jvab1J8BG;1c{TnPzry~60v@-*WrM{qTQc4=P_^+O
zWFu!76yz*ZURxxnklc16`c`y46|L_0KD$G9QYgw;AxK-#4U~4%<iGQ3o$z56*=_|8
z0D8>F!GpM$tnYEmAb~Ia{%B|MBvlQ4{menpSJiodoj9NWOnGt91@Y|md*eYOt#4KJ
zTiN^+zoV;7-az-q*j_#Jgb&$5nS!2Dl&_x~vZBUhJ)3%l_qlkMZ<9W}hzhq$dvd<5
zSfN&V4%)W)^A|RoeB;Vmd<O<Dy_QncrcdDp5h8=lrZD;mo|c+3Rk7&21#*(7Rv5Zo
zQS>HezV*3l@u$wlrM0-x%1nj*^J%3+0wP;U$kPyC587n)YZ!rdDoY#!_$E#%qykPQ
z6&K<(cy$FW?Lvizi3R-v_m(_c8m<<z8x#3%PD4hHKn~YzgFtD2nr3CO!b#gx*`~I&
zNVQ&nGk2ZdZc+<B&=8-_S8O{U9UmX>irll>=k7AcXcBU|+$OSQx09yu_O}<hy*C0v
zog_mIz^2N8G{CKXI1lxEX)2cr>9r%)Fgk8LekCHZWN2rc8qB5*=Sd?+?!fV4+qp&t
zj9LBpj)BeXusUml@)wg#-u*C0m+GB1_oF<N<YKbyq8!#AiSS$Eimf~Q)~=0m444|@
zN3ryDQnl|taq{v2KhAfxn}Fkaf>q-4yz7OU_~O}Ff8TT5z1`t4z$P~Ff=>HsQTw5%
z|EBBBN%rW=#E-iDCzXTtBGnQ1w~y0V2%~;hW)jIBzIV@F>0B-O-Uff~ZiL_i?VwOJ
zg76~$L`G(eDAky#cSM0D+)uNt6C>Ha3`T#AWtR9fbu+!(SMHi<7<y%GIXE4j!SqxL
z>902js;=kof4G}^-%i#YI@@jaKaDC#6`D2`O(=fr^rozxYkjk!KAqQ(=yGD-n0qg3
z*_4^8XuZO9(VxpeRAIC+d~edo<y4l2-s)c1TPUa&oo&tUv<u-8xXm|cQ}T7p-W#)V
zO2N%o=CTs^yP|s1MlT+ebX%Z{f%9U|xc~Uk!7|3EdB4?sw+JEd+IklKYO1{G$hA77
zOpUD4%2|K-cqQ>VrD~3ipd9y-C@ve?f&etCq28knw!7zdsJ>GW7MGBi%g*!ir?$8+
z(L(HB*4dfeZzk8tqljR-I6XoKK(derV9;(D<3|TjcLdR~SIS*8CyEAy-R4ZQ6lt%{
zj|9(C2B*{%A7QXHZYvsI8z+~Dxy9bv3fec`Cp)e!zDoAcEbX?8Ql)dm&hl{;ua&&S
z^hyd|r~f4x;*j7KGWdhO-A2eccZh?1t()$w82rU!eDA|$C2ALQYM^&CJc;*9Yy%Au
zZkD6w_;LZr`PH!>#QM~&4{$36U<$jjX8ZzxpMO?Sl@McPmj*YD11&bh{aC+h4ZQ*~
zVZQyz`L$ZNgr~7>p<RU3v4^<FDNT)S?@!th2#=o=i69o}%z?UnV3)hunr*(l7mO3B
zi%ngMAK#^w@z909tZ8fYeLi*gSRU4yWjVELez!w)0sxEr(TX4`nOY$ZTIgZ-HSuK)
zl{kJZd%|BZ-$o^1dVIVX4U}S0r~X!|^S9xXnt6?N@>XTDWOWBy_~~m!^rZ5YctZ_u
zXqQ&m%nv6cd4sIFVvJW<Uxh|{9L;ChTup6DGD^?(q2)c7;m*b*ho~O%*%W$AoZny0
zb#7OeM->ioI2s0QO(4e~+^ucxA~g1&AbrwHB$@2)_r`jq3ROPUNA4N-3qifqQyWdL
zCuHP|?1fTWZ90C5-68bNF{CU<uVXC5+_c)2i63B0GJl#H%wYDCOE>V-7d6WeGQtH)
zUfB&8sVOHRQuaGg`E=TEja7ODGPXau+IpIgHN>?ww^8NdoxoR8EGW$QXxXxwF(EWY
znHgqrDrDHkzM%V)pktGi^%d(kCb~SA79c8BW8`bOx?MQ`4%1asY>WR~)a=7y*r#t(
zbfo)T>7in-Ril9bgTz(eoP}K-UFoQ!4Y;C`G-}i+TQc4LR8LGjUn2XPgHx59&#NZu
zfEsz1jg4o2GJw%~qj65#4s>8`P9v{%U^%hYfaxH2L(}z6-_uk)oEnXzNfXlKlPEO}
z07OpTUM(}bYK-{E6*WJq6vKCn>=_fGreosX#v@SsV+eP{qQ-jG@=B(Du-N?DzNJ|*
zJ<GC{48uu5ubT1R6+s`S_`r8Kjv@VSsZD7AroxcyP}9#Jo^oy)Ju$Bi$NRZsbrVLV
ztIc<Q(FkdTr<oi1>8R#r>|jXk-u;`1&w7Xx!mtPBwMc2SGP{qT%kFncAD^d@qxrmI
zPV}llxI7@|avE==>Mru?ts9YfJb6wb52|Ka{u(g}gayd2!@jU7{n-})E0mGpVssIA
zFz=t<E)5iGGzg96fhqmgqw|?ZEhky`j^(+XEDtO7yb9x&cMjRjv|Vo@QoZtz8-3Y1
z$_eKcokl-lWIj$j+q<rfzf?Wc7*Soan7`!CJT!eOh@q~Jfoz;HYt+B{Y8Os1%joSH
z3)96vYbiZ9+I}+BUhK=Q34-NVO)uc%$~4?z%g?KZ^Ax1J6f9xP9MjHK_2b<W481=f
zp>Xtj{KkaS^@aQ&aIaZTbRsSJ4`ugR-+h3N|9<sfKsNtBPT_k7=6_g#O>b`9sKd_V
ze>=mC#(>cJo1Ne8ksy%GLjgkLUKL)H;7qZ<^r4iZ?5KqQ2j`>q5dIGk{Xc;C|FHMp
zd*iU7Z5*;uf=w{_)<8bJMw4-HokCe0Jmx&2bUa9Ofx?hz=C_Ic#&6@0|I>*{RjGK~
zh;AU(=gnxR#CNvTAy8#s{`Md2Zxcj`yrD%O-y3CnTdGh{EyI(X43^-Cr<OKLAq5Zb
zS-5tg0Gf`y_-_{De+-N8|2>u-qyO}oh5Vn<k&rXO|8)JovHbr{yV6>3o%4?-nWH#=
ztqqvsX=VcZtwa(VTUKU>=~u7x&Ah#OTD9AV{1&=pCjCVYrJG6Aq8X1E>}ivq`zpJ$
zi^|<xS(^hq)=~9eRf%V8c^DDo@4pf|9PGd`z0f0LAxqPVAj_~L9M%0*?fy)rLWA8{
zr={^9x`Yb5alf$GRZ9oDUKudHmI~4)xi8Z2$Y^p8OT<Crxc{Hsq51H<zir@nP`-%S
z(LC&k|EbnW=O6u1H&J$Wn-E1Jyx7gn2+P|8H6A;qd+YJS+2=86XE)OIZ*J|L6zgo9
zsrc9HsvBmMv3KM;-KoWtxEpx-`QOtvY+Jk|`2M<M6AnQB4Twk4C}i<;8qOg`0Du!}
z4*qw84(;BL7T}06)2`cm7o$o!pGn@AJWFO!L*rQ?epL>E9^@C7^Nb|n%uf}^A&D+p
zZS-xInVEl{NQ7!H7<IDRd>+gWe5L5x&VSF_adIokfwl^DYEQ0=fFAExsl{!c$7-Yg
zuX_~L7`Is!^?vHxP33fc+!$F_Efu=lMiL;FZ!vU>qfmHOoxi%ObXanGJbR$I)^6rv
z0~gQa_v`51Z0kF8z`yB}S4`pQ4^}g6YX8=H7on;40?f}#wBr2fHgCsA>NWE5=xGr>
zai%~ulWx=G2*<xwnKr$A=6;SU3+NwImh8cpd0rke<hax<-EC1zzyZA+spdOc#2vCy
zV}MB6k&YhN^S;+3jlPc~6<N)W;=z9}?>m1W0`p3Ssm32S9ra^M?IaMz5P-?aMeCjW
zw_x*V^Aq<WfCVTMj`A|z=4yXNl+(GS@fomt27=p}OvubHl05G(n;h;WSss&oj_X~^
zn88tPhPp=gLXe`k{b6ho4KC@4h=_U;8*(F_T}TC&U0HkW5JU4yx66fWgvNX7W6&~d
zpTE8IE--=4nrOR{0LMit3cjjkJ;+J+#X0UhX|_=IC9)B8m?8-F`~&SC8ZUtL3ZCl~
zQSU`uPP1HJ*DNvgZg}Dv$Htfb%~uKWQ1YCFwe1Xf3g5ePVlytje<WLgX5fbbJN*a(
zy?R@n^M-Ns2^Zc2lGK>|{Q83@`PbGQMrUzrvt>CSA!DQjh#Wu;T&O=Wr1Npx;vtN)
zU4H3BXGPsZ@@2QAL@Y~CY`%KnUkss|=OF*+s{EsemQ=Ck&PL?0D{;fCbuRMljDb_U
zBrM^4_4L0bn?N4E<EM*(v*`Uyg#PA|mJT7mAVY`z-^myddmZCKy7Jl?9_WQd{1<aD
zb3)#d!vPET4F4faA+6c}3BO8W|MLK)i5DXsfR(dVe)BJcldJRFWB;vPQk^MwKYVj#
zLg>?+ga<BFOD+Z@`y}_eg?+I7(~>9;_|_MJl|R|S|Dx<Or~4`)BzeQauYSxGq?)sY
z-udsYFV%}{73LLRMwW4CfEGkCk;BHO<kz>DQQ!ER*C*9o`%uEaLAFP=s$c&%PhJvE
zpZ|+1{r@#&{x1Y-TI&RDG{nG8eSMATxhWCQF`7Zt(qZCn<D}+2T1%tA+8Y+^#F2mg
zd5o)q0Xne<wgxj0g~xpJ`bu?h{&V(TywsDY%-N3`d%(a<n)i>4Zx1)o2<#lQg<9G0
z+tz>k=d7bcJDixaP4JpB_EK?;=5*VF^Ka33|IbhH|EUiBUpN>4$L^Y2je6=PY?Aiw
z&)EpmK;V&qRiWwBje%80$ZLc|Y+*P6u%(QKH8*bNgMQSf^{z1Bx1CSlNFk&Z<ngRo
zCg>T75@=S)L%*W5>i%S{>hqEu6-ZjpKdgIkd?sfThyMpaA!1l|!zG?O9@$5meMMnU
z7?YlC^if6mb~};AUtmF0;h@^wtaKrCxcFqH+(F67TC37fIT;@SjDgOad;&OOs)Z$u
zOc=g3IE;--s-Gn9ObQANcWfz}t~6>2u9rgns(VC+t7xEqxLM8bw*L?!C~e!37alze
z;~{Ez^=&w?DoFsqy1qXE03S#qz2dup?SZ3-14eNZSYb=cv@J=f1hEJVE?hoh4zNjy
z0fWW$qx*-}Qg?wD4H}YCjio6LI%t6yU=0mC7&a~A>k}dU>?hIaqSpO~GTNfVa~j>G
zdEF4}AzgDpS9#8QstHB20P6pR^L^vew5vIlW?d4TN!`M89Z3O=)>y8>f$(S9ea;Y4
z%3r=k#3!$EU_R8Bl^Qan)tvd7<UxohuB+x!j$JsNs@WT_A(mgu(a?zpmT7b)Q8Hcu
z&PxTDhKZD_-w1*A&Si!19}rFK92!1cpvHt&{H9xsB!;7yYVb?_jxhm98^VkItugQZ
zfvGy=siq<d*CC078Qu+P^;uw_kVv`ByJ1339fgv%Qs?zEE!KVDC>1Y)&FR#`Fj_bH
zTul)8wv!)K;`2$g9xMQ}cHT|mYR;yXU#2+B0E;CeicHOz_AdIzL{+N1toF)}Mmid*
z6sGBK>BxvdVqF>3^*<1LHs*^}8oMph0ZM+tZJT9ZJGEaZLVl*cal`;6w?$`HcT|rN
zG??u(S!0`xvT87|8Uiu#qBs@>;LyJY(;q=hdN7O-<ThN|<VN3+8qFaosF&c$GRXZj
zK7F+gOsuDRfr{dG0sy;5F};bUD{k7>k?Kp(LKT;d^dM)giX4EJMob=C1OMj@=is98
z1DAEiwq2*GvnJnD1-Fa*Cz;Q#l@?Coboxp^{rV;ww|nj$mlA(wfF;kvud^@uZ<1wq
z$Q9=r91|Qo_m=Ze>LCv*Pigne%N3knQ-fa7P?*&~J?ld@l9^vS5L+bwRsHd<;r3bG
zW~s~0Hoe6^(bI*MrHi}Mb_xN*5$S>oMJee=nO$`dt$(NMYr3nbGljsfGj6Y?pKdoF
z4t)Oxo8fgPPw`OTG0M`j=4+78lmxRo$r%aqTnWwR_dh*uNg&C?X|4f)<yPm{qnyhv
z3Ts{T@Frwhe3FlhA0S`%4s><1ii4#9jxiby8L8A}pCib72PsimQ^fazRfgmHk=Wq-
z4gH12Xox)a<s-hRF)Ia8`@sYKsE|=bMI>J40_1hBve4{{Fo#Ymt7M9=aDYF*?eU^9
zU#&IwwXBX%mG$#J!@Rb`>#_aFpV;c<8OdyJBlB0QrW?p_@dMeoHMJ7qfSIJm*Q^lt
zjQFHXv*mAd8fPwN+#DZ!8}`@EUSZVLJw+AD=+&BgSiT1Qktr9|x38=connTv?)z1-
zw6Y_(|2}gf_M|r=q)%Dc{6vW<(Wv2_0u>Jw!WcE2dB4mjuH!|#l9`{4>$E1F0W3_#
zfqrQ?|HX`Wp0AZAKFVL@sXz`+8igz5UZpjyn0?M`nptKF5QjCl!Dr*9^YGet1^VcI
z?%?f&!yRY7rI42YxQ#=aV2sv-F<m$=Keh>R$mzI#h~FR(C$;ELK2pM%nygd^J%)yy
z-5CvEZA8fRa(upBl#i&hQ?Kw5>!`UJtmCUWDdE5E9|+7=_o<PycyIE=YvdkF%cNQ3
z^ZVx!$)<D!Kq-dm#c9Jn>S}kW2-T^3H|#e)P2<)6qssSV9}<i|w*KI>4P$!hhmj1s
z#VM2VjTzr|?-K%Pt9B%f8e;wNG=^es65!{e{yY)|;f`LVK*$Gv44`-@3K|vg%GgOr
zop^1!1)^JRns7Gzn&0N=+Yw3f$&=wV<U}HF0?_w*{i{T8V)oma14K+%*Va`>0QDj=
zKoS=X*dt2aPIYpM3Y@|s!`afeT384zUTo4o6f(by{@1_OJul>JSm5A-(kCK{jce|i
zlfw$Nu4;mqud}#mj7C<Q$7bSY@Wz+b>%jExwibvL`CaA46#&5k;&C=$zyh3bo2I7f
z#<?=1Y&y(RV9i$j_F^^VDAx^glg0Csl*1>(&go&!2p;}-z-C;}-en|JGG1Tbm+4X;
z%WBI>?cophs2Ip~oY7*8bL(WqduhC5ug0IQS$Z~Xyp$tx;-sB8<<N=p{EZQ;f8gX2
zvM<DSZ**fZ*F6*3-3%G0vmqLX0S@##7!-N_tSIU{UJYkZfjtlOodTFYVb$$~I4q0k
z+Y+H>%s&j|6bG%>hUm0z2o>Kz7=Ki=$YiV{M&$jOz#oDEIE44fl>>*u)V{}naZuCW
zg<dV-omY4GQBV0#Gd~uN;?!|MN{&(I?J8FYg9gkiTNVL$L3Lgn@xTth0rIKI9{k7%
zu1sU&nW?vY6S(e(@|lPL&m1HsvA9AU6`(wiL341bsXem(L*;c%Xf%>zQ7t58PjZYn
zA*O#9p9(lo$}A@0EmqV-+cBh*&V4IAtN~(Es~F|c0m_*4>!-iBJa1xG5hvjEFS-g>
zMXF&aB~kb27GNm<pg-oxGrIC0HSmNSq!PK9g;xFf{@%VpZwX3=ZW}fsD5Het7P$GT
z%sm_iJ3)?&it%c>JYaoMPT36mj<!359r}rJTPi@yUf8^^GuiqKW6D`xYbomUfk5QO
zl*SxHZ*C$1DHW$1_z1z&Osy<Z(NIpErdxLh)Z1J!&~tLxUmMaW5$6<!Ha#%QTE_%*
z4sCkXCsX%)x3*jwDoTZ{%=CZm7Pi=j&}~%iyDOA=5PIIVtbMFv%b`BXi4WF@aE#1l
zavb=(7p2QtCySAT=zr;^M=tnLmox)8{Pfn}f9gRn7Un&%kZE`CkKm=DowihsJ%AE3
zMphwmbQVKGGbg+}0`9G&lYGFOx6IPF>AJ*t150R;^vx$39MMHp@<%B&N~^0L%Q<{&
zpMoW2f`x4b#i~bxEz>w@8BNnJXTX1Rb5owD2#Af#;u>|Iz!KIhrsE93ltICQV=vH4
zl~=U9hg26S<NU3I1B~BK<<P;`V*!1<S*DMiycWe;7gm#o(YOs)q_+L|$4b%WK3tzl
zFIjqwOmqk$P|9y2xZW-1df^^bG*!dxAMb@?!MXStZQmb36O~G8-En_9e13CtY>j4*
zQ}1Y+KkwXa(H)tzT|V5c^g;HRkttYWkf1y(9X*FrhZ8@@@3gZedC823$5&*za|@{8
z4biQBPesax{rKkHC;X%g%IGwM9f(p8;pW=7rHR5T^u%JN66F4YAw>V%-P<3?2>xio
zRxRH%(rtKS(Pdbk49!KXW~`uZC+MWyg<IF>cs4Aidn`3DN>6I$WL%6Q{=g%t4=4P<
zs(ob)%wrROZA=X>ga<w_E?rLaa;u{c@2Q!t!N|MizYkI;Pc0k)UyN{g8To_t4Pehd
zcxmKj>1c>*`QOg5?#Dm*e>mqOG&>>u-HE)7j8V!td!yI(=!)*mF9jB`gVryqF{XIk
zXQ>y~OFpJosP1y#xoH&QQdHc`72<pAFGFjbb-TRPp@Iz<lre~XaDDVJQd0w8ly}4n
z-uYGFFa9Lf;(W*ZMm`k|7+739eLn$1&-6wC4U3daFEJrGD~?FW)YCP359Y^@ABFh!
zbu<@&^wAV?ntr>E{2};mnog~_=pRfKSVwUvgHJ`I2O?X6SO<Pao)02P=@w0Pa=00U
zMm`CRR8IqaovN8&kxWRE1rVE-O#IQ5*Br$uyaoWC$6vh;$Db_hp66kiv+JKeRFdyP
zK6J)(;YXB#Tla#~!OgdHiLN`T1%r7yD}NtOid?q9;tKj7Va8OpwkN9=?-NP+<sj(5
zVqr<M10qm`Y5KXJZavi4ybWOWZfDuQ9b*3C#L-wJqnY}OMgx#x>_{WcQ7uio@7(T5
zCQHOhsUrX=x%fhUMoLnD_Lf5i#Cv=3wtj6H?*QgiNi+Z@YBWY6SUONk%a}KE=7e0>
z1hxStG|jts^V!m)Ac+X`F!2+OtMcUN7ZcVPX-CJuX`mPE`Kbgv$bXS3azJ`%Xt-Kp
zI^uk%7^Gi)PVb<|p%#Fevldnz*wV|&)E$NlLq8j#&j)wg3Oq2*rIDcs+>-aWGbA8c
zO-TSs+CNCJ98Th;EALd7lPwP+12s(|)l;b0kvF6k;aZLhW`oc}iXz5-IDf*Zzr*V=
z*f~`&#+0gyUi0b+t=4nbO*?d^fsP5p`rYvy6y6kbKe@%jL^yg%jpanqlCYG~Sb2(h
z7V1n=4t3xGEw=E(ML%pDJVUgo)W*hcOFZCKvWJ99L{xjIQ(oy;P}e1{(dS`;7_F6g
zwikONEe%&p8RZ7+0c<X!KD={}bhnyORj69As+;D^X(6@=K7dRAQOvMM-B+cA9Ub98
za=BTSl2y<Uy$@!RhlV|WKp@LBJhZ=G8sBEizTNU&5aG{+c&9I17BFRIrlg9pT%`hg
z%fYs`QO8LmZ+)RCPHQl$_Of-}A)=UVA!pC~r-Tz^tQvG$4Dm&g0$N(}ncc6#@1_TO
zjw{h#ryo!DDO79E_`C#><8VQ2Td&A-H8?%So6NS!HR`%D>TKq!_=$lIzp-yuRt156
zZvo@NHzwWj_=z9Ax(B^Xht*j<;vn40XjrhZCH}Qea`o@6M2yWe{)G!xAU0hsF(DHH
zaA1XS<Kn#5=RcWinyNt<zA>t=sAc3d9Li~`lUXuS7?NK}K!}9|F}#SZqJe?J!G(*l
zz;@y54J-qd{66OdnL%LBB41WBjQG1W(UUXA139kidr4Y#{>yF9Az?+za#o760`YOF
zNWgjDpZ5`G_uJFR#?L)31ly3*3ATgUf(h=~_|m!DvA`39$X;n>`R3iGK#y)OUBs!^
zO8MF=x91Qvd8aDYs6dkw4cN4e9GS|G(7u&ReiPdPwlPvQGC#Lr2wH5aXy~>liqg!9
zRZ@Y1B5W*N$GtpJ@ui!H6Ddl)jdmkjU~sEtUl>>GH(IyRaQPm-Vv{&o0$RyXgYkjx
z9)ua=!tw(4htnqV_!$BbV)O-7#=v!(9BCvKneiTp9=oT!bV4|;tSd{HNVAr+R}MYS
ztitxjXuz+jR`TE8LaraR7J5Eq@Cv-@pa3|JhS}=yl3FdF(gJ&03Pzkq4#G88EDC6Q
z1NmPAb!J5$)rP-yfyP6+qEhLz=4@NkI)p4NzqD5OZ#H$VT_MS1woXzoxz^~_%87m#
z-K0PV=qx;Xnlm8^rEd2#i^LK=w8XS^WtxgQE68aZ-E`$PGWn}8$VM1-#`JFQFa5m;
zd!fDJban3-nWc3U;D*%8(e&Dhfkevi^e5?Q$Q_0$wxBtL%}4-XnEw^ISQC#pYhN>-
zX>fb#QDn6Z*O7j3(KS>46*hDOUrrirrP)&uXyJ47NhIAqYKZZp{qRnh-+X*Naw<Y5
zXESw6V=Xoxk!87xj-<xe))5ylFDqcsvhyU_s>!Dyq~IjSc{`$B!Dck94X9VW+j!{P
z){)BMbKC!EO;e~`zcWVkk+UB#2PX0@=5ppx)3|keyCn;jy}ZBT<G7q$kMDlJ{mDOp
zT+Giua?V8CtF^HGDe`kDPD+V_R-=^8j4Dl!q?|ChfSZZq1x1%F4md+$4ApB~&-U-w
zlqj_0i!l_IsNq=L;7%#pjjy`=@2Ply8^NkZd<$S=TTwLT(MF5Df=)}N@m{UoXEfCg
zCSdQI%q|kkXR|taXd()K4kk7oUSdM(gPXhEBKfO2JYyME@RC@4Z0T4*qtz&ui?sNg
zFa}2dX#G3QnpTq6wQ$%MI>$i&9}PrdvN(&7S>O1H9`)UC-O4N4TZ(27*wTI+I?9ga
zTr*q8#Q6E*#kwd}f~|ho#{@`82F}{M;F^r7{h<vH+)cW-+Z=6WJrn7y7k^n{NPcq_
zSP%Si`gmT`P5ICME@(d_U01Zl(xi#_Ck!R{7!=)2HWiUTl_jm0Gt66ts1DZ4-J7R2
zzx2g9xaxVYzqQ?MRVPQ+-}m9m7Hla@9Lf8ik4W9andEU}leOKqKd`X6VdP<i?X_F9
zTFcilrc0Rw%a#@4d$pP?qZ-`Z{rMYwwSq8Z(mr~NM5=UawO6LaiGAcNMK^-|1Cvfr
zCY`C982|ZehRr<bo3pik2h@1jpTyQGtisxVBTE7yl@uPvU%v)QYNzk2$<It?ZT|Wm
zcpyI!7bcCKU2dZqg)RDLV%l!DjTaAPLe~8Kt7yfHbav)tMymtL(kf)&{Xmr!SA9q6
z5n6N&IL8a8-9(-BUh|LEyBpLt)s9~C#2^a0=;WzJ82tgB4Z+%W?A;p$gex*se4ftP
z9@9fjsL`r3fgFI$z-7ezk3gRpIwiOUnC}ttLS1zJqA2EoU*(={RL``@$5xGx>4Vzh
zL@%y#t!&D`oQo_1SD`Gc@aLs^*7Z-Ll*C%h2uf%<Wmfc!gs`N7=&U<*NTFCWgA)n?
zWY#thB;!@3-?}+%<t(3iEVuEkKNV<y3F`&`@k42Gk*~GoggJjI%d8srN;v~m`mL{_
z@2uu=ha1L~BWI}bOP6`yLB`{Gb*b?D<6$dyIU5uA)~l(~A*?o4JG_76$HK{qOEIW0
zNgB16*G*<hwNm(Poqs!LDwriXGB7^95fB>DzOMSB5fA;=O_JO^)yk-oYSHA^D^#^i
z;z4{6Z40mt?-jkS8xbnOCecb6zBC=SW^zg_wd;0GK>?8Mb0AS$`YmVFza+eBGx?7w
z-jQ0^R5ZUPv4XxE+`jFxRK*zHues)F5FyL1?zS;mxuPPG=R&^4BqldHAt>MA_`>cU
z%va2l4r#=*o_0k-(!|!R6sw&wfM24;wBXmp?3bVcOX>^`Y+WP|Axg0156w38Tz)rM
zpClS#q7<Db|EU1&htFXjh@5h%O|R2u`h;{Tbn+&u7ZqPlFHsAAf3j7k??<aNhLW-8
zt7xgL8t3iP*y%DLx9<AXd46<d>DQ3*^c2P;PRdZE-BGjkYtK^<iJ5iQ`cr2?V?jhy
ze2AR}n+p342|bOxof2kcF`Qf+Q?X#!N%reEQh!(v_tVMEqzNt8Qn=jq)R1dmri8Ah
zg}00QE8Br%a0h;E6Zl^sl0JuvBEd?{oez!k(h(;gMf)_Kclc~V37_2%S6(~L{=W5P
z0x|Vnk-+~wB<;%Yr<<5LLjYjQ@li#zCh`ZQgv``PKrA0_Cs$BNVd27SK5J#cOy~aZ
zV~s$2_We_EHGR~-4qhttZm{`RU2D#OFXkEmTAG~QT=PM;RUnjn;LLJ^rFJ-`r_$<z
zn^Ozr7j6ffKddUkD#|a(Rw94cI&8rI5*CZAPPDk{^iwK~_ER&w=cE=L5zqmv1($;q
z3%fT}QR1?KvR~R&)4&YO=|5L_;Z^E>MgV}lC4D;3K?>mWz7dqwaxf+=M#xqn?1aKw
zu)0qNWt=J`QI|K{Ew5(~&l7D=(epc@nN$Y*x~pkg)ekC|E^mYYpNDS@L)*llCKu74
zNx(DqN-gd7_WOQIjg^{}CE6-e08<=FOj+2?HySMs4>|!MJv9p;uA?DbJvIogqyM)V
zPo{0h4}&i;o@#klNq7a(&rJ3%WW2uKPk8dh{C@0;MR5Md0{jAR6Ea#~gRIxo1&B<r
zowjmmoa9M-5Y{veicat!0ZSv4e+>huX?Wq>-Y|2%`NREtN_lUN<}E<vf2|PK`C`63
z=5S1TT{jRlm8{#W4EJ`lkHrqEf}5@Q@{~p{3?rOEWNB**{S?9M9@I@tF!*q^8}tE)
z1^WE+P7)_?$)D*RTKJ@%%-!^o%5rK-xPO5)UKtsg;FKC3G)oKD_w{=BRlhpt?w?7S
z*};Jhy%LB#bMKoJ-}rOl<*Mmhv(BfFEFI6~`?4j2N5K1joZF|l<XxR@Fe=A#rX(h^
zJ<Xu=VeRD*wocr6SN>_6Sn2s^-KGwh@2)Y)oa)_0qMvswUIpm>N)8@0EB{L~$D@WC
z65I_dd{&D}Pz)^cxshX$tXgJ>XsDjgOl1t}Jlx$O!WYV*ghvdDc6G8RD|`U{ba$Ho
zt0ek9y|;5#Q*BSZZr2Ha8MK?eqJgB<yv_i#B%@HYlw8H*E$Cq2@3Lg|EclwyhR=SQ
zl1?|Kp)u$;Z(%5v<eGZjUJk_L&RsR!SMD2?l2TB=0%ZbSHvB46KEf+yzSMv}Var1>
z#RE-q$$<XjetZ>}N7e0x)I?H_6zEp6JNvr9wV;Gv*gty(&%cTd5BQ3^t;d~7#`nI5
zzQQV11_6NaosRkH;T%SeWq2=uwOH<xbpjG#Ull=Ow&zu*!p!MH2rD_Cl-zt4C+c;w
zhMd<Uf|AK<$u?a}h8Wzf+W}2=lGqOkjAb@#U}zBP%nodl4YsJJbio4t(O`1Mbv$gS
zGyCv9{Y6ChKnF+rt^%{25E(k~){gl=`K&M2i)uD%*(Dg{Up-jJ>oW-7J2~UPnHXWH
zdaY@`@_*0E1||u~FKw4t+S@FI!rQCeP8S|0;t>!KKzXl;T_662`TH9J;AYODDcNwY
zuT<Y-6V}{a5(<+u*;Umxy>4AN6s+jO0bxpq`ixw44-ups<fE;r$3uxSkj)@wFC(BU
z^$R3~ay+lhaOHe*p?%q>A!~?^y!^ka`|6;$o-g0QN$?OPxD!0MI|&I+a3{D8?m7`%
zCIokayEC{X!QI{6eb8Zelkab<-fnGe)!W+l{&@4x)UDg+-tNAqPoLADPxldZmi`e0
z{lUFFe;kge9QwoNG#>OpN{R*vWM_%47Tif}AepojkCj3)H&83JY{fQe(g(u=_@lJw
z`d5xN8KTncP1Bd5y~Wy_KmpiIfbX=^pJ!j9%q=1<h*joH9hi3{TVbA|fpS<?2=Hmp
z&W=cRHM>(L5hm!dO>AnnH)4&d<V!~kTx$oalj0&adkd@n%DUu7A?7QME?5UZ2jJgc
z?-2x^F2Y(_$(BSt&*SPX%7h^HoyVd9obitdUO9b45$^%wAY6cynHb~EwLqIbgOCs@
zK<3L=uBE0=DKQ8$`1#A`bo9ZZpcu~5zUW)l<^s52JFb%KZ%|*I#!1NwM$GqTxF>|v
z8v&;htk=Ku?Ezz@^5)?C`vui_<S42afvl3*2Mfonz*o+E@0ZY4e*KCqShr_+_F|MX
zq+|dDkOb_Ju6ZH(GiKsczTDhLa4f{}kH9*F2**XpzFz<MXrZ1MORXQ-=w|%(NP~N^
z>k5$!1ZnNOmwFaOvxWw6Mt5lRrJ_B<i(&IiSHopjSG&NheM<SG90w>74Oi;<n#zpd
zxFs4^T{o<oJq*xF*++<A0({+a^@&U}YH-XDMmMk{1tWL`l86okNC{w?s!96t3ji5_
z3{yt!+RxS&bN#?QGxlyQY2iGe9`x2WO7HJv^ay^<BZt`0`K`phn1vK6h^t2iVPax_
zc`xxO8^%NbyhqSrii8o`Dra8V`j`2kkG|=mxb{{LQ3`mt9!UMq#$nUWf1_gk-=i-D
zspfE~r64drZ@<YW^MyP+JWYdoH>7mqRg9F39v`)Za{Qtf8Xr!)1OwGdqE1>ud8lR&
zSG%Q%>T4=Hgb_!>`MX{fW$X(~qgy&|D=g;O9ZzT<>*pp;Q|eVu3gOAL*w!=v4eD8R
z7)c@@B^OA7I$F-Zs##7+AeJ*6<~t?@9T#fF%M${`F>*Kxe(kuE+r{xTxaKEnIaYhx
zlzLXE4<5vnZPo3Sde+(nD7<?XWzr@rv4lgm`nXuZzff;Y$LveG>9Q;#i<HpDOfQZR
zWj%y|U~7^iky**R)&UXC2qn>cBPC@N@%%Mon7OuN*2+Z9RJ8ABiY@|THm5f-mI~pv
ztwY^X2>>;yqD{Io_6BKo1xgVTOqv22PyjXy0(~!tu*m)Cpf%r^O&<0KsS8(Y68(A6
zkza7#fHZ}9Ms1U4PdSPiK8ArqO^<#7l>Z}=mPKy;G0-yY4z&DZai8kiUKuGLbu2kT
z-V#L(ilKQFZ-Syp04g{~7Sx-ZdD-|W)FHwHK^jPX*WT99JHM*SnjPQ7N(5lX9!28@
zbP3i!G(d_Ew@=kV8#*EiJh@pHKWoi7F;2<gnTrWb6ct(4Lbh#e0p6G4z+dXWuq2yF
zz*?-j+`#^;CXJ245#EA4r$vbM<gen|xEBRn0tFT)^AO<UO8z3AV;Te%u#x8VyM%S6
zmH`OnF(AYR{4%q=m|FY3UmE?`zK)r|z5Vsr516PI_u#XT`gOni7eZHpd2Ahj7{Cx-
z((@IH4mny94qXwC!BhuJ7xj1<(SSR{lGk2?GaES@Io`pZ2Lo?6zH@s<<f(^~S4@=*
zqz-AZ9BS}t#Uf>=|HunVZh$yqKy;MJ$=}i&Uw754Kcnqe6x1y!EYtUZHC}wB>}YJ#
z%bS^YJRcA)w*NR8DJQkf81e@J)qeTy?hcvf5fE(q!;gG$-?M2Cv3`lB0XUDy$wT|#
z$Q?<3Y!$tW0+@n|BLpWREf9ilf1Ty;{19AmRGv*`UE&RTAhzWG#*C`gANfOvUB7aC
z<xu5%pmJcyu$G<QpzO;UpifaHUVhjBYlA7sxlZr%CJK#@bo!<rftI($G;A85-|t-h
ze4{`>*?jfm%U03A@HA}Vpf|w9RQCMmOH#rd!a`;v=%uD{8OC*%(*pI?yJEFC0Eu}L
zC+i!hxPtbi4iw6ExIlIFa~vX5Cz1zvlI3LTLgA68?Gzc_>EpCn@ZuF^7HMJO>8-^O
z1{Oz$*6Xe&cx>^$-cJ0~RT+i4vz;%*QlTO!Plk5H7A~Z<)1$>Z!=_=D$l$p09dY=w
zJ}hTGEn;G}{7vBE8(H&+VIKPm>L4HZFMdrkSjg}*a)B}u=3)IGCH>;a0GbdA5FdXV
zYd42Klw?Dg8p!ooOL2xv5euyU80<YFpQd2CoLYVzY2lWi$T(#v$z%WFAaiZLKdnQJ
zi6f(>fx&H)Rla2Ydm4F>`47PYmd~v1Neab~yq)6BxI;Aa$c1v<ID=MG;KYqdQCw2x
z$k*u7nOUMC&%wHacbcE>2i+PfXP6K>_b56(UHZCAl)d#RUj}9PM<tf`a~N0A?muuT
z{79-CYSMA_JD$-j$C_*yGOkp@A(iz16SDD^Jh5CX^Jkk5|HHOPgTwR>?9|j2fx8E;
z8p6aO$s&oB+aA|CNkOf(jVtlmSvD|66m`1~3j2JOgHd>{Uo2}tH*DG#Nuz)>gVV9Z
z&a<5;k`@C2NDc}lHgc?fukw-gJ-cmMjC>Aus@DJHEE8op%B>;@d1%nmSRL#rPOLL?
zBT_yxi(tAbW@_fMiw+f{Lg|`)ypAT3)%+@*blbR|BHZ(n3NU7|lYp&|nV6#UF-gX9
z=Vf;<j??_YtTo!#*N%5f-?*QX=7JJxLlKK@@l3y)hk2aln+s7*PmYQ;P-uh!#6y`m
z=yNremD^uFOup!SZ7Tgy2(d`{={}0#vbQe%#lu^>nx7Dd<x+IrH{vdh$cai|j4*?&
z_0<hKl?(|jtnV4kz3D&FGE2~t1w8B!_{2~;H?xe`5>Db!6b2~1ET{R)ykpADl+9oP
z@GSsqXC#>npn+og>&Vs)6b&_4jP+NtuVlNsxM%drTP7=o&H!Hls@9$sTb_y_Coq*;
zr>C`rYQFw#rFRNeay(cqUl(u{45m_kL+Byq!i1%w)eF(Ksz*Dh5dIb39-}mCv(^XD
zqIo)J9?l5M8>2I(xx(Nhz4c6@;#lhlD^Z)ci2ZkU0bN5mDcGs~?^W2ltDT1ufSHmE
z-|#1<6Hwddd^>Q1-s{{1c4VFI8t?yk{8`FWsulW2>q}4bflY}!%)FKWz)a4EW9Kd~
zom=K;GlL-da|)QY5O%I33V!W*o=gUG^_82N4al1;Us;@Vo68;%=M!cdnX<7j18IGo
zv0?+!wy>3}4j*vzGWg_qe*MT1+s`-O{QZ)65S}?u{Xv6U;l|rTWhX@cM(B$~<%k;f
zm5;fU`6ktk?&!tKQDjrczRfxPlIz+~5)kVrDl=64QdC5m(dV`U#~Em9)@4zW<@}le
z=LcA*#tygk9$;_k_l1V?#lFlqZIgH)0KE-eD)`+VnPe_!j&h{>tbsAGK{RTn3zNcA
zbikf0N)Mcx8XeC2iKtYbftmL%rjN);gE}nhb@^mu8))%RGSymiB6hIf0Akq&mvXHF
zlzE_A51eSk+-gnhCPVS4x%mXNY_Y{jREu3JpUhCMZ?(}J1T=BCk5tpC#axD1Qd`ht
z$dcSx|8`2n$oDzTs01QFYv8=!L5hP~2_sJmry_~#ikaT3Mw5UoEBc{}Hj_D8c53E@
zE)H~3gDRoDC;PL4NnG*<6u=tCJ!-F-sP*jEUF<c1d?(3t#A}ksFy1;q-qgbF*Rg|}
zBK_~a`WElc!A8Tbxy7P5Xmu<AtF&SMDWZ&<A~q#6?-610>bg3GqX?UtR;ItreckTe
zOY7&KiV5J4)m=G$FVD&BG@lra5l7*X?>OBgzJGg@A;v#59$?|#Y*ja@kD_6ou`q?c
z1@ANJ1@WgY&0+|P#6*qs=<B0t46NmaGFf+P*=Pe<vCtlaTE~kM`z%r&jQV^cEdx}y
z7xIcp%X;;Skph(4dPRX~Vk@>9(L|u)o~?6{GLW^MzKMa!*;7bdW%>yjUi@_3T(d(0
z8D{5D`|?1H7rhK?+3;f``m9>Rpko6SzfZb3$gU&bcz!F-w$=F^zQ9?o4drYw2)mox
z02ZoD_9vLvQ`OsQNSv3qEY&5;P6X&6mh0VV>l=V>!#`qNSU!^O$r5)}65F;Bafhd;
z4Mr=}mk8$+crqyCOE|eR7d{|65UZsPyUd(d1GW)J0o%BR^D9;iod+K`YXQa*w!jLh
zuVhRv#&(X}?9dVgEU74_ijntXYTE%HnLEDMU;xHZ#&wj^w8-}C%ROiL@asj>6Fg=j
ze#6gMgm~xWH47$C-bDPm-It965c)IGraFm|*K-1-&=Ajoq}Vu5*EDV52^6(eo{UG~
z>T+aQlYmavZH0}9iJa#TK!1LvE7swq+n(A%Wfy11tNq30V}ADQxv-=ukOBq<E<Zdh
zd;-2Nxs0uk{Jm-x>^Q-IaajG*ErSJJ2?wtmkoybJY4fHF#PV=QYFZKJ_t>qEpvg#Q
z+zKuok{Lbv!U$Q<e+ry>eBjdp@}QjgEr+KBoNRS@;INb`m{WqXSFxn9N3-S@;RCTJ
zbc=QT3&>b!G44a(s#JH-Bo32ag%i9w-~hm=tYNn9fh7;~^qG^)7eMQ;GiU%{K9bbf
zX}Ba*S@fexLd@zpbh9u^m@{OTPpb3-4$Vt8MRpe77bX)%PoU^xB#gw_q;KeyWUQ!e
z_PcWqL!uM3);6x%h64o~XxV}<;pkkf(}!}+k-t~W(3~b1<ojngdBj0Qm3Zs@^<3Jv
z-*|w~@5|*A+J{qsYz^7`atX;J)=6!hezEAd4?gv1o$0kG<Narvy=U>P@X<ygCyeaM
zdgL=)FIo|n5u8lw>k55ukD+J`Tx8z|HOViYsK@4ykDwLLwYB>YhS;J2#EtEk^}uh;
zV)RCnQbmxr+G+E8Q6lj=Uf9D@29u_*&laPFc{Ikt$nfSs2eCiZhrl#Pq$d)`hM?RU
zVROKX<CpcE4Ac5}$j;c;bYiq4I19|$8E6SHK{+-3M1Q87Npj*Y;|h^%><RZpAPwSh
z##;}(IS0}2*rb>A^tqYokzMtAr33Z`A|X~e5`H6pRp2_71yvs3Qz*n2N!LM{hrYjP
zO|z0@pYyq2BvI>!IEs89xc4^PZD)p4=V|PxIox&lcPR;vzfZrWwHsLYz>cGzrANQi
z&#<Vp*2>|{!={$L3dGYxn`-}36SsvJJ8?z>`iCBeH}_9yjQ6NfU40#YgV)|7WrbZK
zZ$V{49i=;^Wb}JJ^C2kF@Ag_0^R|)BpDYBJd-^vS^?aI`dS-q1nj(4p;re>cstLXY
zhfBJrVW((?)Cmg#y^E=(TXih4Y)Eemt{dN3-z<<ZJ{=avCevl4{C;OK@l08k3lFN&
z=G?_7byW(SE}s#w-?8stT!@Vk%WBQ%T6Ex78qK0~Td+2E{yEY5wDoHrWn`;#;=F;<
zQHW~Nqlv;Zi5?I=u;f<5Ukw)eYG(?mI@S|e>j`0P=m;9(7P$OENT#8D$l~s9PCLi~
zibq3RoBFUAy9--h&eTBB-v8UZ+rx#p9wPYZSGnyq`=D~;B;F8yRUImlbi4i&I|dV@
zx6P49PTa+u41%_-Gl}{6`Cmw4<meO-+x@|qd4{j{2sgdr2G7Jb0n#)yGF)X#`;N!)
zQ|PtgBQrGs&F5+;>YdCofGA|Zx6dF<g7BDDI=TMjuTt+(0c}+WR>HR2?<Es(S?VY2
zWz#TZ=%%Fds}3^Xy#Olg4vf!^i|&w<1?AHDNgKCJEwIe<Xq<(8oQ=`$3-IcVI^K#h
zv|8ceWRtlH|AMLcd@O8y+3yC)GKs5$g*lMmjH~c1SDE3dsPO5r$b{eBMw7!IcFr}q
z!gKEli03{(l}UKd_L4$nszc?;N*j6S_FxjDwl9P<PjD0YBgc3~I*xH5$*WZ}*XHkW
z`41m_eAabdTtT6FStHKC&1iDLi2_);eaXv1?q!smc9Kl-kB-2gJLk_{hAevP{e+*g
zI!c<D1HW!$J6&|fR$L4;3!<wKO_n`gu~7jZQ+6IMb)J4eb@!!C=-Rt2TI`{Xd|;hP
zCgKbkp*M5<6l$~e)LYDNS45U}^k&_6W&8prk;1U9SEQvZZCS4wOe`1G-B^yECm^?X
z3$V?YFBrDN)9I1FLCcS=Z#2(WB=79)7blI4=l5MTRfOLbp`=>56>eu2&uNmKUYZ%o
zblCPa%oTKRPS-VcscLIIhxI4_4)1J5oHYl~r@c9q{&A%@_kg`>cDHeE$EwbFk<$1R
zH9Nv~Nd5!RgOPvEPq9DfDLE|6hR{@;iH`0g)gd<lLF|PtI6N#YzSmbtPA*Su#+;6h
zZo8U_kbr=tlHss=i>qluM7I~)&9c9aGu#Xk9<LE`gA)D}bvTH&A}WE&4OoY_mco7Y
zGS+i=w3h~0X%BzAwn}?1ov#bFj?neAbMBKk*uY4vONnnmB1x%E`Pf|T%pEabqCxfv
z68=&n0xHxk)`g!Pq845+>B5E%AoxNRP7a8~Z}YsF!bVJ8M*w14O|xyzs&mR;Td?zY
zF#M?ZZ~=SYv<dN^7AOE2)jTHv6aT8u^KgD^XoCX(!#rDp4F$f^k77(2Y|Ce&&$&Uj
zr=<n#bNma*EarBYAr!!h@SrA^SK$Lxs>DrZeX-62IV58}K{Iaxa`pB0U$mJ{fBHJb
zqBm=<4+x}rE*&}4@jKmze1tEEZbxZ;<WT(MTm%hWcKo5s)Z*@5#`(9OUxQ4{Ewwi1
zw+cgq1cl~B4{-P1cIG|P)#tD?H{fIUY<<I1i;v=bG#@KpLglyWS6pqG)$b|PJ~h=B
z7GSe9V^Qqn5UF}TC;#rgS0yIKo-bul2|W&~MbLW<u^$**Xim*&8!480)urdgc%6Rm
zh3S!3|7AC!-y;7)%}iV<hUy}U`t%@)@{8z5I+Y`!PU)DZ-P8NiI!SDP*hBe!$<AjB
z)|_@7!?sNgX>polzCgvuw?KVd(WCP#PoXv$QeR4|@*USd>$UjqfOppoC~u--@~xDd
z#;C>3zuWMd!_Eto23ux7+73~vhSX;P4QmLgqMuWadh$)_=&Iu8sd%}2sjA$oww)K~
zb0oCY%ZJ2i)Rgzu)r&hx@`xPa<xJ##t*;k&+*S&IE+0JkkDR`0d#cE#oL7U}M9@XF
z;A9OY)Lrx*aY9)=E<6B{&ttu||4DJi{w==x#}CG%Xn?&vaJF0a`ch|Jpwr;@pg>n4
z38VBZktv>qT2lesbY7hFLlSl_3a@togkRn*-}Vfi`;e`A-`N}W`Y+h+PPx}e?@nDQ
zrK|{VD`#v>YTNV4Km_vrPytB1Ahs8x%ZFHy%z8s_ezW7Flx|b6rv-v%U*4c71)Duy
z3meS}9|r5>larth0Z!1|u|uXZP@uc$1y_Cs?@;<R<#w6yC|{it4HtZNgQB23e%TJb
zPN;eiX<l6BuHG(#{w`g)zoy@e?QfqvPqG5NqTw+2Q=v?7MD!~!i@aI><ZA*|vSOEh
zl}7#OuyK*S3t8+6C89rnM&`r%OSE_UR)$jPhx%DwU4Ca0vUEDQQ9Za8NGF$ikdilf
z4V`ufpC}^W(G4-G#s5>xjVEj^`!4cb8*9nlQWw^-N=tjc?}=8&w#A@sAMadSJKLAA
zj|q<A0>=&f+MOlR_O(QvJ}sgW0p2zYfI0|8)xPQE&wAS+R8k5m!jFL@-3*;Hotlel
zDdE)dDEQE!kb|-Z@!Fr;EIHOprX|4dHX6;nQ#R2Uczim~aO~=+`H-BIFlwUN&^1%b
zqls+P27yacjjTkno;q|@m)H1X6Bf1mrqhTwn4ppx8+$qkAg@_`Es%~LDe#S-#u+Bl
zNHa>SP6t**OD0_Dy7tsgs8o<zFfGm47yw#T{9GfP23{duFFNq}#nQxL0Cta60HJAR
z+BjrvrgnAxO7X*1k$`p<Yu6oNFdOn*8KjOgWEA`sH1p2wEBbLXQ#NF^+-Q|ZTCP~3
zfok>@+a}&W0T8?kqhe4LD_ey+Ea)az1(7%G`hFH->R4n1_#(jt${r<0AtS%((k}2O
z2Dr%rM+CO7Yy_B8ZncWg09e8y5LZt%l}(S3I!y*o$kMrG5}}rT5#kvfT%{%GYt#~7
z_B0ci7TnG?cjtG~ZAZs<Qg1c3Nh6QEx4LMU*NJm+x2HW@=!LsKHRQGy{3?o@kF0a>
z0j!;){Qx4r;i~+V9$qu0ldW{>To%)$<W#JBI(U4J@@9`8z{)2xdz%9JoO{?p7xr3f
zKU1<|jNy9O-42nnF)^7+9|5q)Uf&MV0j3yClr4F*0VVq^;Au5U@on{edu+0zEB<g6
zW4vshS8T2jmO^=D36icZd4`VjI!*R6T=sc=n6yn3-?6BVU0Lo!-=6!hO<_HHL61m%
zF-~8dV3JJpaQ%6uk-(zvRxdp>y6`g+K<)4j*+ImJ-xLuOx7coav@RkNn{Vqg0o!im
zs*GgoQC7(1Oi7h@?@*iqBS)l-g7;e8m^51&Jb<Y(+<-B5uh{$Cro;Mw8Z)!`a%15`
zG|332Obpn9KfAK{#=Ec_goxEjs*&GJCyN<}5}he2AKD*I*DS_Qn*Xv8<Qzz5Ua$IK
z-ss$O@@zD!$=WJJi}yrBxiC7~TEGTVxHsp$*aHVn#ZSkiWk+$l3!z@XbfDtnJ8M%8
z+AQFl6|XAzMn#ws3dyBPqDP-|*g+Oq)@1v#C!yBdTiR{2!9&4Kfexy<xSixlpaQ95
zr=l{z*La_eM@3KZPx)BM;_o7c#)2<BXu;RqRl%rDPbJL<n#?5}U71JM%u1wYtJ6WI
zq=E`SC7)W+xv_bNQI<GTizRh)#^*XPZpCdLOQ}l0omV^|Wny37Qt-KGdA+QxR=Wu6
zfDww!=F_CBXClQ%VfdJ%nWKcrt=T@Z?u_g#l+GHEQA{^Ea(Hk{>q~P#<AX4m5fkOT
zfleF96+TaVfHvCmgcq`HXrR@&#x+I}$?3-<g3*1#SLEEYDQ2BhXKL7Q$glxX=(5<!
zHCbS!9g*C0fy)DK6IU{m55c2YQcmx`83B1J1E(|6F+2ZLgf#dH?4H&g^)^tfyQ?a(
ze4JPp<dtX6^eup87LkRW(O7Mlu+kTCWPM3FA2Q<D#Ty7V`wbnlI)Gi^(4B_(dSMxk
z7CD?)9r>R^Ij9lAe#35jpn`h`+p@0%b&8BL>ZefiB0NyHCyhkgIS8kS>J?cx4D)x)
z`pM)7nh<Le*L#*pl(74I{7h7}e1d_xX<5$(1%q-86p#u|0^}Z6RjOoQRPV2?+B{4l
zPAspo$x^(BW?Ie9!R3cPwitR(P)Bb=eRzs-1wp~stW?nyXfjJCJCSjB4ZPq30FQrR
z6O@f4|I=DWvF`3FZCFV~%+FUg_GH*(DQ$?z8qoU3C`={7?dxu6n~#Sbgc7hudi2&C
z`EPhsdZ0a$;8>ds&~oyU%&yVnF@d$7c1mY2aW8?-zFFtllVMIn%1$GM{zg{i_QI;+
zCw}=}aNU=m$%>{Hb|A7DxTwsJHboBePNk|#m7RHnsUgNx2N}NWVTzzVoy8J^a^Or?
z>3@u*$N<R)4kz65M&*q2U#n*hs7MYD%KBL*fC?QE*bQ&u8;d{PD>ji~fG&1428)X-
z9Ynq>y$uMiIz8&tt>ksKQ9wm*o(Zt#z#$3`;#t5IqyX<X8{)PM5`f-Wo(Rv|8Ir%#
z@5x(?rz*SBW!TiqPbQst0jRYE+q-*KmaBhXR#Y@_^9yZXUN7+!F~9d!T$G+YpA3`R
zzQW9|{yA`bO6?Xk+`9IyM|0BmIpwmht^{3%ef}Y;W@_4UMZbdESI23VwS>SM&0_23
zk!Wblt+DgHdhxPXYk8AhfR5b4v2cFYi{%>#I|m0*t24t3brHCXw;U!&y;BVT#ZPqB
z;Ce4)IjL`$Ho;d0SodAGKmegh5e_mEf$yj7kk9soTMil<T42nw_GXXc_;fBydw)a*
zX#l55J9}&d4gL!xDXzR#zK^J|!TB*PHl4A8;Mhd&aDi=XZMx|BjsBu!(g~q^5#L^a
z*-a-+Ojo}Tl<)mD&Li-X{QBq34@@_G-W!Qo=KzKxF_HAMWnA-u+vF`O!s<@)vR<&{
zx3;#fSw7s~m#Ll_i$3;5liO;*XZW=upvgb}6<k=VuVRf-u{1zB{2QhB)xvBZm+O5E
zeS1s8XB6F^uXmcxI!hVN27d2O4vYBhYq;SZQQ>S7G(v#%`+nxS*00371&fXq_t53T
zAz6^#KK{ukj8!3&?2modn@MtVKBnTFwynAaq(f8$<+&4&a6}X^zVqup`~P4lB3O9u
zQ`RddCpTdJe-twOw?u$CB7oqqOm;3z{5p`z{;fcp1u|OP#LuO{mmlb7DaQ*mbD+0#
zE7wtz{H&xNMcjvMWgKcGH?~AiQ!RKB7d$u&DV<4NCfXwHe!4S!j$1u{JQ^fU(gcO%
zn{DH5#547LbmMsRx@7vc!JX7qbBl}U&DkvOgi#~Zi?cBgYxG=$QdZ9$Vzm3ydvI2~
z8&76_1ze)<z88#pQQrH(Xp@VXTK=s!%7(WWHS*>N=78TrnktblxJ_f^VN>I@{Z*fp
zR;}km&u(kMEt7)bek+%PbzYiu@vM2=_X|0xuhMTk1iS>IQJfo5hKp;hL=O`MjIIFA
zB1XcA17lyvz90eczmWm}KVOVlfW<MnQNuuhpA<!XL03OGwgmBox#elmL9c0~#;lWp
zT2wjsj{q9d?QM=t7?0n}#3X%{u<V4P!l9u^DSR^vpAqa?M)zm|;&``LNEWIpzJ)&J
z^gf#doF4P8m_(o|tHd0g(z2C&`q7oQ>rQ8e)Rd)6J1xpRNl>Km_@n&jzJ{d?ZpPz7
zk&mn+|9+()d%4A{m)g+S4gcM@Mz5Qd`bsJTV}|2K<%O%;$*rLHe(_ibc|SRIv1%_a
zfRUxoVkfz}YD4Yn{xg7OQ6*^mHt=VOa6n3t;NFpkDDnzT4tLR@=~(tiS$S2xlm?eQ
zj7R@F)q6W+%Pw|=;mo8BT_nNclrjowR+thrnCh9?+LckDsso}}w=IVr#Ss7sSB6p=
zqAn2`MG9Jvw#c2n6{@t1s`~8^ME}YTuCLa&7EgfdaY<gThd0IkwDsyDQ5mwG_&K9r
zTuT+Irghsx&jr}uvY7wjW~~HotO0sl(l4v%^iiXmjU}3cKUmt>#Zfmc(eVBt_le=W
z2>2>{EieCh(IVp4iFB6ROOUaN=haeLG1ncFJbFtAAiL~ns#|dScSzJ#pcW$C^jS?n
zK)_6)S7k=vW>0h-OVD8|EA(-*zN@MD7A2uwz{3~vRe*P2)&3*%=$4qkYs_;!6lrd;
zX|JNmSJM3W((HW4p}Ce<N8hE3-rsemX6=cuY%0O%^Uwwe44gxeWpjUu_*pLwD%;8C
zx`*-@d>2OO2nMqX^{GoJAe&;0j@WhdT}X_mpZ*`B8Qy&#O!>BJ_~yuKH5~S>5rcxe
z%L9VgWMJOr+?Wz}0s!!&gqhn`o8i`b2+m}qx!hQi>y6Fa*M;dZs)guwQDXjF?N}Y3
z@P|+zlpqUaifOl5xbismyYesv8NQMWsH6~)pi<z(l7nrI1UDCtXzcoC>#aO;1lC(E
z?O4iZ)<(9juGL72d<^AsOH;~i(3Q`~1y~_h|B=qhHkf~#>D;`*aTrZpU!_i;PnX~?
zuJX7M_c{V;)bvIOkiE1pp)cikPB?Mw=T#_?P~|#j^XX^qY`%D4d%p25R)qO+A%V-o
z=k-haVdf<JG}0r9mCXnMXkf`w+?2-#KkT=}Mr0JVOYfMA+1@smhnlry1!D_Z3QGU0
zTGA_ueHVE^k==Esu=xbm320Bug|6m}TdUW-Ou5e3E1hlH<~KGsv!2FPr2JUwuY3%6
zgWAQBg}ApS`Jtx`nn=!#Uu)aA0M#SsopYJb>OZNrr<@*BMg(0Q-elF!V{aoPf5P^{
z>#>rTM;U|OAht#i`)&3!gZ$%Ic+JtkSfJbvF5{@d;oda;YXB)(YM7AU5>^!+&O#yy
zJEHFxcwL2!wjR(&6$2Ny$Z*~nsrvBK)dC8Z2yq}5;>637@LT%QVaTg@3a=9GAEjcB
zfvWI;0Fzxe`JP@B5M*XzdeVV)%b<_x)3osZt%gR0xh=p9Wa{Ya0j`ja{-Y(MarpVx
zJ;_=?7~ISK;z^FPaye){ueb>rYR7lrd{QKBGK*?zg`^Rq9&WtVK;uRuZ0?!f(0j9Y
z-5@~Cj|sJ_d-d(Hd>B9nnml<wDvND*3Fn2TqVt#W%ycf?T^<vFMkd6@bnE56ij%e4
zO>3QsQ)Ee)c@cm2*>y0ZzCMyp(E94jPsZkauq7QoE!{Fo1tF?Be+fCTWYVzec}_(F
zBIqDs<yBjpp1>b<)6tNHF+NNkE!@po)-&}-8LE(=b^N+aa7!~(1Pmp1=^Cndx%!Un
zeu(N&vgg)vfW&_MSUkJ=YKq~>N}W=g5fEV0?3wo=ow{l87e!r#;p<NVOxG^5ETIm^
zgQwMr?;=GkrTLw+t2yCw>lg3c)%~&VdXBEVH}T3sXyjC3qjs3R%qk<dGmv?k{?jk9
zb$3{M)wWr}P`90IewPxng<Y=KPT{r68qB!iE4bg7Kl{YtBG5NzmgtgO0d3_t(Mzeo
z{`idQ-@Vg5kh>#t=9V56?czs#@h}xFS{Jj|@!NTF=cW9?Wqpx{W*OT{OuRMoFM9Kr
z;Fvj=;cuCY&N29j(|j+Z<9{eTdV3s5ld8)xL_Blyh7`8h_n6G{O--6qR4i!7UEXnP
zmKhfR)UdvlpE^{9rP-G$t<G~Fc-UTAX#UeCFvG|~hQwHczn)V20baRL-bd!Ah<WTw
zlCj)sJ1wk`2su{RU_D)BtJWzngYp!0jF<3G{G{Sri!oNNx@FZ2RtQS)@?-fL5i@fN
z53lx6V+m>)!d%109VNeDu|+Gs4*dIhT9Y|vj7ESw^k3h-t`v~BG6J+}IBsS>2gFT=
z2TeI>Uoug!Fednlt30$5PE{T7Rbuw`w&=s8f==%sI;Y0sBR;QRLts+11s9BkT>_24
z{^lyF!|vkmd%(mVnY~>#!(1J`b?7q!$vP`4R27n6YGcnOOa;(~kZWl>o3dR+((Kb`
zG^~FL(5RwSHn#p?$*4B#A%f0>DPG|)WbPvo(Y#+x%e$RUjd$8YbFV&eb&Q*zbAbHh
zBmoo-+Ad9o?ZK2xkpOl4P_6x^bYRH8lqm{#MyI=SKQ$evemhxbMDqUdJyB3?*mza8
zF?8ld?s~kBKwU7hM@$C6w-(=PmMp4nEozV%50NE{aBP}j1_(<YCl15J<32~CP;%#n
zh7aeK1hn!hY<U@?p~n?FeBxLpv>!#pTd}ed>&?w6*dQptZ!?Wg136u6IgzeV-14$C
z7`_{ibxRu>`shUZg&0#=8Djze+V_RE2)p)SGeevJ!plRqV-R?Z5~V1n4qLx#@y&T_
zV2gQIh=lr>`Fyj4@+Mo-<-LLdaA}PAB7Z`g>zfr4X>Op$0LdE~UnC+_Wr-WNxgx4R
zZpO;f1M>qu9ZpTf179wfo@1V>hh9&Scl(;-_oO&q_Js-Zc~W#%iPCtiYY>w+Z&+BD
zow+n6g7a_cW^{^>Xvor6lM!<#(cL?zQl9|#ttF<~am=D>UF&pNtj_da@qLB#!Ulcq
z_AiaN^xMbM0#d~$qY2|M%?*VL9^k+)iUfi4lF@|p9T)6f8~g2!1rN0r|MNUI3!A%l
zVlL{teQGfEWva!HI<-BisQU1Yb&qmGYLbk!BkvvWRW}Yo%EDQ0Nwk|iDsqK+6M{m@
z-AyodJr3H|u;2an$6yo$v`2;Gi<*sGW#<ghqYm<*orNp+q?2?UM2^liN0rKiao%`H
zcoZEHS-j(+=gPwpK8^e$A%f777Y+~X=NO#3-xhhXvRfA%#>$Q`NF*n=W<R>(Wm&M@
z`L_B4KfzK}zcK%g>-BB!-QW`7t5~uwS+Iza6k1*88o{9xdFAdO(8ppW>>n@Wx>w&o
zFA`ogmBhVeUze)pi2g=|^4z@xbVndp_(pP1_;G0uamY{|aawm{1;<NKe1lxZKo9$;
z@1kNJ$m1V|z}TK1P3EMLrae>XPG8B#&9mGB1Gn&h6Jn|kErPK{Hj}Cn^?)<T7c+SF
zL4XR<cg>>s6w>a`KgpY(dBe1T(L}-)eLm-<?YDpRjtA%g`#d`SdC$W8LhkfCc}dq(
z%eXK`SCfZH`3D~Q(ZWcv6%wC7cLv1gWSr8m-FSF`JL>=&;79{bKiPGI-L3AxGwQ{w
zT)f9-A!w!&F3I{@j>0|_!|^c3{;Afvd3Q`q8W$&qA({FHx6bJRi%+!3YZu$+TX&hV
zl%5?elJG{dFeH?DC!VC^os3t7s;J$W^Z{W}=65B@nBN=8KDIQp?b^szlzLs-(Jo3|
zL^PSx?VSkYej5M{#Ghz-9FwT<7cnL!)V`P<$#McuKloY6t7_7ro8B!lSWZ8f1l9|0
z<I2}u?%-_R{&bW}?-w@jx6<-BwpJC;#C##!mw4H#oi6A4+*tmiw_dy9&+d^pGR;am
zHNKqz$Id`?CjI#*vH_Rd=s{iZbaNuC&(bq?mRzh79McN<@T_vKu;)QEDJuaS)A3WU
z%{79;w*7R*kL)z5Y5Ya7_wbV!@<6?10u(2@hdB59J#J+s{`C!B)U^<|BvAxyo?war
zd#Lik{-p;N8~{+4yZ9Y8Bd}QQ2prm!V;3<f_&h?xOe#EivxmCel-oAa1NuXW=}Qoq
zqZIvIBflOpox|JL$U|RAT!Y5|PF6h`$I;Go!}c+gxH*!+x3t@f;>Q?55M~Yx2Z`ea
z@5GtXZ3R5{xQ3!c2+oi}E1G_t{xzSbS@`aJc<shKD$2dQP5Q(sF&|#I*<mj<FC_Mq
z+-sFvk;*{Qcbg6vf9S1=3|JfP>zAQ-c&|wEyof}#(h@1RG|_QO&<EKUFmI%!6LFUo
zi?ptk)`JjEOJwWH+FvVBudGm|f6z;4`EuGS(n|T^EjYem)G&y#Iq0ZAu(^a{>{;LS
zH7-gZ0pLnzLorf<+eLuuc!XcQlZ%NncIx@Z)nNn_G9v@ocaf%HsC;y=fP8nMLF=MG
zLpFnXS@3u<iyPnh#I2eyJl$SO*+2(4n4IYFQRxQV60ye0?_MZEXx}@Dxgvjmufl!!
zey6b}g^&OV0akw3l|$Wm+XX29-uR^mF)2jD`nKI^pKz#{FLQINfIQ|P)zcM{4mzm!
z=$&}akRplLg#T*!O<fBl$S?<*jLGOZ6n*mSuwHpYxpiWK7Z<S5e2!J9kZt2gLsiK{
zmm~I897GCj@$)5YXj)la_-*E2`4@9*R&8}P>ytdhV_!8_w0D%$9Xs9lPNxeljz3R*
zx!@47z+^pbB{_gZu$|zWR$p5EMYQ(@f5x2E2=+k-aeIkrQI$Vr3^SXe$c&|Zwu(w)
zpEX!fc7NsLmVh2@w#+`Q+Ajq-+z=73rvd&wxTlk|3;s$SUI|-H=t%Ze-Iscg)M4^8
z7ryF!^)q8J9`{?A&0Ch4P09Nf(^W~>QEe7{io^~o3IVE%P&AhkJ8=aZ<{DH6xa}lV
z`d6IqN*22d$;mxq)By+GtCyYnNtWK1U0aXOspbljIFSUls5O%u&p|A8=4VPqZNSFx
z@yh+6bzZyyIKLTPPPC>z)Z5l}pM3Q`{A4{e=nOt=z~04Zdp<H@c61E|J4s%j{aL-8
zQ5`w;GaDqPqNk<XPB0V?yu>(_yge&)T2>c%`Ejb8(ck-WBuz3P$adca0a`9#D^yUK
zKZ|Tg?9ABwL9&V+aTmZ#m*atW87bhzl)DFP`g<dT@$Ahr0w7tw4mhbZ!(3r4QA#fD
z-Tw5Z=$bJBqq8DSj&knR0rw4<CENfRQI1g$+@vGtf>;2lvyHb9Unx>uKkQ__NC})p
zo`b!NNJp%X2t-em_6BE3j%e`)dZK^>>A?Gk<^*#tIwA}2brR3^`TsOwdFOrmTkavC
zF#ikOZ1#Kwr>jD7i1+S((nr}OtD}^9B|k>s!s%kveinC^^O1l6h>NJ<>C1}-**P9z
z*cI;d#{!+o%@MkjPVAJ)xS+^JH*sNufsg%3#Jo@sap-=EWSPNa?2zT;{g<eu3p?6P
zSc?4P1a*MZvGbNkDu?TQ5JDkj5$*hPos;<lEn!{_c)BwYEX!;U8>6UTUmx)BhdC;2
z&KDG(I0u2o{2FV%u_<D<h0bAw@9Q|OZQeL<%NpwaPQW`m_hSPZKT1?A{Z24wyboaK
zFniWm^LW=Q8P2BI=qXxVdUNiFX>E8Nx<AN?q*Jl^S5%~eJwG%F{t-fqc$61mZSanE
zfpwJUv#_=#mnp)}<gY^F%3~AeAZrTj`5LCS1UhEA3%?)D>6&-XVAa@yl~j)xaiJr=
zCL-6tPBFu#y??^@H$ppR;Rey*$Y12%=h3C%J&Xl~-g`fLGbOL@3MCkk8*hJ&%#QdH
zJE1Ic&)i&HqMUk4RQkEjmU|I<dR#wV*5kbe=r*?R;sqXlUl$lSY29IaICL@&IT){&
zd_c>19*0uzG|Z8H>m-SGHntnse%aKU>3=pOiX|%g;6FHM{NOsdl4}BRwl;Nq=o0Ke
zo6Z%5IKA3`7?wqmJn}czX**Q;(|^irK6v)AfVhe>=+Y!hg74*?|MjQk3Q>B=mAm_S
zC-dDHhjC=@J;M84@S~ysRFcG;@S8n@R!-#P)-(4%K``TcXCD3^WCp_(q8ry3WHA9G
zi%F`&t9t1aFu_sxR=4AVqv+3JHg2!md=W_zBHC}=z$dVAjNG63Dy!36Gh+AGWu#ef
z3=_YLZt#xisGO%`Z}%!(sb<?@Qo{)LDDM>b@KAYIfwD1f;xpzybtNip+;tZ9oTQ`q
za>4|5*(on{&sGu{ij8C43b+nBCNb83?s@GGSj#V@hHp8|w!?Q+F-C5)_Tk$}@KtL6
z#-;A=%9Cby&Vcz!aiI5k&ytiXX{TaG0pb7~t~U72D6zTXXc6F&b2jUNGoY3AcRiP-
z0}{6wqv_V5W#wbj=^q2{>h+%)4_UCC?ktN);?8`MEbJKn!}=y8(Lqm&``<CvfluQh
znbat6P;_M$oOuL(5HBi@2JA#;X|#!Fh54%v7rIMg8a?Rl<3xqM4nmBf%Vo)SkH;ct
zWX99w(^|~jT{(5T%|mwpuao%$Gk~Jp?<j@$)VmTzYK)-y)~(rl5HSP}V3R@jBMCRo
zyPu>QuWIe7PZr^#gO{y*#OaO5fQnO2)Egj}Yw@mhVfO)kCprsQTXr%b7rLLHrIgIT
zM>3T$v=z9l>43{^7M}(_Ju(6+N~w+Sx%ewZ1BtVq>YYO2_jWrjh3yREzZGM;9BQA~
ze9yh71&<~Bi8(-A#d1b#Cf{JjM`1CBk5_R{92FKjxD$Uz$kd$s(;><w36<Z&viF<F
z(Wk@F4xaN|u^Fx#aJbDRLJQJ&_5Or<a){R-USF;n$YJNs5@Tc%QZLj)7w7<*rEmP`
zT{f;KR2><~MkU0jy@*fzk8f*7#atA1JngM{-T*q~_B>(IuOU2b&HfJtKMl_Joq`>;
zN+FeXDHGm3mRan*_C`Io1WOrG(2B$Uw}z5U3Ba{tt`Bw2wmy*g(uORx?Xc^O;L{Gb
znvHt<4xe+3hSfXIGvL$e=C7bwiHL2C7qTin0viUcb2P9IR}X3dR9?STZBNCXf;8ab
zv9;Xz2+LR2J3*Wkv`}?9X}|@Y?#Vh`sz1If#A&;JB7aIfGE+iD1w`HojeGl7vrop|
zcc?#U?#ZJB-H+#kUEH`GPL?yO+Y1_+i~v>M_MfI56kl1X_}wEo1}lcXG{Udh8C4pM
z2Qwq;l4|O!NDVqYi9p>3W+OH?Z2^z=yl$bHbJzJ8XEWAlvvD|tBh-cR?FW56JG9K^
zE$FQen|qt8@E-7!i|=UV<3xVPlPP!Qi$TP)`9D)gYm82$pmABy)>z~zYLe=Q1FBBN
zpeu`0lNC@E3Kq3w-OE-;9ah6x<IZQfO{tlE7Tf^GUc#UYRmm0YnC?>WVJScg23V9#
z__QMmaK8LgbWKPdOza7WTrgU63?wy5QTE4Ml#FfE`H*P2y5P!N*9ZUz(uD;)$2)$X
zL2I9r_@OTQkwIo1uPPwab2mB-Yn)=-xCK5te%E3A8pG4ZNq&X$wh4ptQK+qDd;L9*
zFe7BPojmXc9Q5I)mxu&U%qsAl`a%?Scyk&t+nqhAd<-&yCm;@hrK=437OuN-3I#h~
zhU#>{*udt`oQC~P;SH+mJAdXbQe-<G6#IK>l}(^t^btW&M*ljGUh9slkDHsF!-&+Y
z;mh@PPMhpVij7yEXNu5{h%~&loj<CNi_MBo7)c${byc#Dm1pB9KL+7P#g)_RJM@ir
zRp)G~S$CuLw!9W_VzmK{aN^7-uZ}Pd@Wu{`$$i9Kc~B!OCGaVwc{J#bWyznETD+?E
z9<gfIlDJ|t;9P_2<KeK{)co?wr0F)A3)T9oQ&8PE|9=gX%Qbj)JW0^zdTd9_4}qsn
zFG=rnLHE_W6?ys9M%VqrhSdP5$%grP2DfBM)_cRz1WVr%9rQ;he(`I^d9H(sz}ty6
zJjuIKy!Oip!jLB+3_}xfw~hVrC!gbqQ;xRV!YqvJhhM70Pmj5`SF`X7`BPXm{LdVH
z$9*)lrNQ;soV?THNk(qaeb;aeP}R8YdgJhvYt&QRk5#)Zz70+64(}A6H3?qvm-60V
z{#AHSKQSJ7n*?tNJYDz7dQ1}h%v=%ZeKlQb8=HsAY(Q`SCvbZL&-flMOX2`VbNXm>
zLQObcP;F^jd=~KSJwuK=?5-)`;hH|^R{ZEbe(3JTq1T|n83^qP1#tm;yEp)2$f}O=
z_-8A>Gq=NEOlkzUh)O&gTKNr5+Rf7Cbvu|01u-|Ebc(Xj9s`CJ&uPyfTQ#D8MMt(z
zW=5(!*K841{L_9~N9#6TyY&(-<=F_wH*ITMU>$%8qoe^vkM0Ad3gn%3uP0@sPKUNr
z>+KPJ7A$CIVn)+ZP{R9ZG|*=e0=(mtm&?!nFJLiWXd^$^S7{;<e#qr?ceAR&^Ch@D
zRH88Gunvxxs3xg>i7$;|Js77qHB^ML@h#e5#ZzzoQL>7{!vUttAUrsXu#pl*{dBdl
zx@!{ki{o>cKrF&(gAsj(UQV4pNzGe$Qf0I}YxTfF(=vB_CRZ08Z!Eq8Kfg~2Ti!(a
z&t4&7UTF$a`mppO3}JQqeTUlUA`svh!gXXg<dzI(iTRi7r~~fMe<<#_>lK~*=qva<
zpHI=NU4PIihj83V9pL~ECUP8i6lWyE;XlsCUO-+`;<8u`PGoRYgPQ{vt1PhuQRo(4
zD%cCUAPQ#99^a~VKy3{lpCIhUi>?Mk54#w*Uf?HS6NjT-Ye?1UXrSs%$_39~KbS$m
z@yYveru^wsgXa@T$I^E4<6y%h><y=|!Fdam=y--J=-?Ly#Bq7xujP`noi+y;G&rK2
z^~tsv!!wxu$f6ZOWuJ#V84wq`v;WJ<N9wwXKnUm$IoXc2*$?vMqtMdnt4Bad2ebMV
z<fY~0;yX7z{vTn4|6(Hkw_wHpKSUN2a6w^Vd+poamo-DedVe{X0ukp})2>#R@J)8a
zK3D_+^MBfGwh}7DenI8^+rJMVSY~&7JH1IM>bX>p$dws(Qq!Wh6H(r~^cE4XK1@J>
zizg|**iz-%wG4y=;{jHPVQE)??|7|ybV}{C+zkDNL2dZYuN7fWma#JsJksTEugFvQ
zvw5>amvp+Hh#D^o-UcxLZvyH6wEq7AbDr6~Ax2c&hT~LnWFS56FE(wH+j<5&?Mpdc
zvJXPAV*e`@JZuU>w0{YOb$B71n(JHK-{C7KSGWFA(W!^!A9UMG)(*+oAoHKEaPq_Z
zm9at8KX|yipe1|HKV;bj?0;yo0^hVB!eIaQxw=*=1Gf>cPH=;`RX$E5t}F*9CVosH
z{q2cR$CWSu=~J3hu*|7-ntW-1#i>pQ^4b%IGIeOcEI0*`ZWloA{#MuB+zAM@baV%}
z1D&}XExlZ6IA~;TT%CZn9ze^|hyg?~#=pfZKDk?pn7djc{s260xOq6<2y*ZUYw~c5
u@bilBBQE(w-n<cv+Hm_10zfAVTPvUcTHxKd{wP8K@ZqhBT!oD3m;VLw7?KnK

literal 0
HcmV?d00001


From d1358cc96f2e719a90e2192a24e61c57ce9ed50c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:54:38 -0400
Subject: [PATCH 4378/8313] fix html

---
 templates/longpolling.julius | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 76ff94ce35..35205bd04d 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -9,8 +9,8 @@ connfails=0;
 
 connfailed=
   '<div id="modal" class="modal fade">' +
-  '  <div .modal-header>' +
-  '    <h3>git-annex has shut down</h2>' +
+  '  <div class="modal-header">' +
+  '    <h3>git-annex has shut down</h3>' +
   '  </div>' +
   '  <div class="modal-body">' +
   '    You can now close this browser window.' +

From 579972ddfe0c895241d8700df00e80a602f463a6 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 16:56:10 -0400
Subject: [PATCH 4379/8313] update

---
 doc/design/assistant/webapp.mdwn | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index da8de81329..ebf1689963 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -38,5 +38,3 @@ The webapp is a web server that displays a shiny interface.
 * Fix notification handle leakage on pages other than the main page.
   The javascript should use AJAX to request handles, that way
   they won't be allocated at all in noscript.
-* Hook up notificaton close button to a callback that removes the notification
-  from the list. Otherwise reloading brings them back.

From ef796ba6f7b16f7fe8b55b84793740f1b8faaabd Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlHLKeo9MOJ-f0S3EIchJCHhj_YqXDY6vM"
 <Brian@web>
Date: Mon, 30 Jul 2012 21:03:58 +0000
Subject: [PATCH 4380/8313] Added a comment: HTML / CSS Framework

---
 .../comment_1_2965ef966713721220ea5c18868b20fe._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment

diff --git a/doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment b/doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment
new file mode 100644
index 0000000000..f4ced7392e
--- /dev/null
+++ b/doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawlHLKeo9MOJ-f0S3EIchJCHhj_YqXDY6vM"
+ nickname="Brian"
+ subject="HTML / CSS Framework"
+ date="2012-07-30T21:03:58Z"
+ content="""
+Are you using Twitter Boostrap (HTML/CSS Framework)?  If not, consider using it (unless to \"fat\"), because the app will work great on phones and tablets if used correctly.
+"""]]

From 2401d1c37ea62eb83524e2274532f87a7b947345 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlHLKeo9MOJ-f0S3EIchJCHhj_YqXDY6vM"
 <Brian@web>
Date: Mon, 30 Jul 2012 21:04:44 +0000
Subject: [PATCH 4381/8313] removed

---
 .../comment_1_2965ef966713721220ea5c18868b20fe._comment   | 8 --------
 1 file changed, 8 deletions(-)
 delete mode 100644 doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment

diff --git a/doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment b/doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment
deleted file mode 100644
index f4ced7392e..0000000000
--- a/doc/design/assistant/blog/day_48__intro/comment_1_2965ef966713721220ea5c18868b20fe._comment
+++ /dev/null
@@ -1,8 +0,0 @@
-[[!comment format=mdwn
- username="https://www.google.com/accounts/o8/id?id=AItOawlHLKeo9MOJ-f0S3EIchJCHhj_YqXDY6vM"
- nickname="Brian"
- subject="HTML / CSS Framework"
- date="2012-07-30T21:03:58Z"
- content="""
-Are you using Twitter Boostrap (HTML/CSS Framework)?  If not, consider using it (unless to \"fat\"), because the app will work great on phones and tablets if used correctly.
-"""]]

From 502bc5d5f84a26bfd2ca700d8f90d78a81c7b1ac Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 18:01:41 -0400
Subject: [PATCH 4382/8313] rewrote longpolling, trying to avoid duplication

does not work though. stupid JS
---
 Assistant/Threads/WebApp.hs  |  1 +
 static/longpolling.js        | 41 ++++++++++++++++++++++++
 templates/longpolling.julius | 61 ++++++------------------------------
 3 files changed, 51 insertions(+), 52 deletions(-)
 create mode 100644 static/longpolling.js

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index e75870e0d7..7b96665d32 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -146,6 +146,7 @@ autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget
 autoUpdate ident gethtml ms_delay ms_startdelay = do
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
+	addScript $ StaticR longpolling_js
 	$(widgetFile "longpolling")
 
 {- A display of currently running and queued transfers.
diff --git a/static/longpolling.js b/static/longpolling.js
new file mode 100644
index 0000000000..4e5f102f00
--- /dev/null
+++ b/static/longpolling.js
@@ -0,0 +1,41 @@
+// Uses long-polling to update a div with a specified id,
+// by polling an url, which should return a new div, with the same id.
+
+connfails=0;
+
+connfailed=
+  '<div id="modal" class="modal fade">' +
+  '  <div class="modal-header">' +
+  '    <h3>git-annex has shut down</h3>' +
+  '  </div>' +
+  '  <div class="modal-body">' +
+  '    You can now close this browser window.' +
+  '  </div>' +
+  '</div>' ;
+
+function longpoll(url, divid) {
+	(function( $ ) {
+		$.ajax({
+			'url': url,
+			'dataType': 'html',
+			'success': function(data, status, jqxhr) {
+				$('#' + divid).replaceWith(data);
+				connfails=0;
+				return 1;
+			},
+			'error': function(jqxhr, msg, e) {
+				connfails=connfails+1;
+				if (connfails > 3) {
+					// blocked by many browsers
+					window.close();
+					$('#modal').replaceWith(connfailed);
+					$('#modal').modal('show');
+					return 0;
+				}
+				else {
+					return 1;
+				}
+			}
+		});
+	})( jQuery );
+}
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 35205bd04d..29e533c418 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,54 +1,11 @@
-
-// Uses long-polling to update a div with id=#{ident}
-// The gethtml route should return a new div, with the same id.
-//
-// Maximum update frequency is controlled by #{startdelay}
-// and #{delay}, both in milliseconds.
-
-connfails=0;
-
-connfailed=
-  '<div id="modal" class="modal fade">' +
-  '  <div class="modal-header">' +
-  '    <h3>git-annex has shut down</h3>' +
-  '  </div>' +
-  '  <div class="modal-body">' +
-  '    You can now close this browser window.' +
-  '  </div>' +
-  '</div>' ;
-
-(function( $ ) {
-
-$.LongPoll#{ident} = (function() {
-	return {
-		send : function() {           
-			$.ajax({
-				'url': '@{gethtml}',
-				'dataType': 'html',
-				'success': function(data, status, jqxhr) {
-					$('##{ident}').replaceWith(data);
-					setTimeout($.LongPoll#{ident}.send, #{show delay});
-					numerrs=0;
-				},
-				'error': function(jqxhr, msg, e) {
-					connfails=connfails+1;
-					if (connfails > 3) {
-						// blocked by many browsers
-						window.close();
-						$('#modal').replaceWith(connfailed);
-						$('#modal').modal('show');
-					}
-					else {
-						setTimeout($.LongPoll#{ident}.send, #{show delay});
-					}
-				},
-			});
-		}
+// longpolling for #{ident}
+function poller#{ident}() {
+	if (longpoll('@{gethtml}', '#{ident}')) {
+		setTimeout(poller#{ident}, #{delay});
 	}
-}());
-
-$(document).bind('ready.app', function() {
-	setTimeout($.LongPoll#{ident}.send, #{show startdelay});
-});
-
+}
+(function( $ ) {
+	$(document).bind('ready.app', function() {
+		setTimeout(poller#{ident}, #{startdelay});
+	});
 })( jQuery );

From 254c174bba261a14ebd75e9fefae823a8a0f3d88 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 18:18:53 -0400
Subject: [PATCH 4383/8313] fix transfers display logic

---
 Assistant/Threads/WebApp.hs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index e75870e0d7..fac2cf1eab 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -163,11 +163,9 @@ transfersDisplay warnNoScript = do
 	if null transfers
 		then ifM (lift $ showIntro <$> getWebAppState)
 			( introDisplay ident
-			, noop
+			, $(widgetFile "transfers")
 			)
-		else do
-			lift $ modifyWebAppState $ \s -> s { showIntro = False }
-			$(widgetFile "transfers")
+		else $(widgetFile "transfers")
 
 transfersDisplayIdent :: Text
 transfersDisplayIdent = "transfers"
@@ -188,6 +186,7 @@ introDisplay ident = do
 	let barelyenough = n == 2
 	let morethanenough = n > 2
 	$(widgetFile "intro")
+	lift $ modifyWebAppState $ \s -> s { showIntro = False }
 	where
 		counter = map show ([1..] :: [Int])
 

From 5de897e8d70d82effe389fddf4b8be287b5f738f Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 18:26:36 -0400
Subject: [PATCH 4384/8313] remove the "running" alert

---
 Assistant/Alert.hs           | 7 -------
 Assistant/Threads/Watcher.hs | 2 --
 2 files changed, 9 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 54192aae65..9a0bba8ae5 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -174,13 +174,6 @@ activityAlert header message = baseActivityAlert
 startupScanAlert :: Alert
 startupScanAlert = activityAlert Nothing "Performing startup scan"
 
-runningAlert :: Alert
-runningAlert = baseActivityAlert
-	{ alertClass = Success
-	, alertMessage = StringAlert "Running"
-	, alertPriority = Pinned
-	}
-
 pushAlert :: [Remote] -> Alert
 pushAlert rs = activityAlert Nothing $
 	"Syncing with " ++ unwords (map Remote.name rs)
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 51dc572636..8ba015b19a 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -87,8 +87,6 @@ startupScan st dstatus scanner = do
 		modifyDaemonStatus_ dstatus $ \s -> s { scanComplete = True }
 
 		return (True, r)
-	
-	void $ addAlert dstatus runningAlert
 
 	return r
 

From 0b156468edac8e0b146f23c98a0d2366d20d5df2 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmnrN9a3uAu6Ur2SSyE_AiKw7bOon1yJBc"
 <Clayton@web>
Date: Mon, 30 Jul 2012 23:58:45 +0000
Subject: [PATCH 4385/8313]

---
 doc/forum/Wishlist:_Don__39__t_make_files_readonly.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/Wishlist:_Don__39__t_make_files_readonly.mdwn

diff --git a/doc/forum/Wishlist:_Don__39__t_make_files_readonly.mdwn b/doc/forum/Wishlist:_Don__39__t_make_files_readonly.mdwn
new file mode 100644
index 0000000000..4d67f93b1a
--- /dev/null
+++ b/doc/forum/Wishlist:_Don__39__t_make_files_readonly.mdwn
@@ -0,0 +1,3 @@
+Maybe this explained somewhere else, but what is the purpose of marking files readonly. To me this is an annoyance that doesn't make sense. I want to be able to change my files, having to go through an unlock step, to do that seems like unnecessary work. Interestingly, Microsoft's Team Foundation Server source control does the same thing and I don't like it there either.
+
+In addition why replace files with symlinks? Why not just leave the real files in place, or do the reverse and put the symlink to the file in the repository.

From 9648acc7a4edd44a080f40d8e918b1388b24171a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 20:22:10 -0400
Subject: [PATCH 4386/8313] got this JS working

---
 static/longpolling.js        | 45 +++++++++++++++++-------------------
 templates/longpolling.julius | 12 ++++------
 2 files changed, 26 insertions(+), 31 deletions(-)

diff --git a/static/longpolling.js b/static/longpolling.js
index 4e5f102f00..75c6faeaf3 100644
--- a/static/longpolling.js
+++ b/static/longpolling.js
@@ -13,29 +13,26 @@ connfailed=
   '  </div>' +
   '</div>' ;
 
-function longpoll(url, divid) {
-	(function( $ ) {
-		$.ajax({
-			'url': url,
-			'dataType': 'html',
-			'success': function(data, status, jqxhr) {
-				$('#' + divid).replaceWith(data);
-				connfails=0;
-				return 1;
-			},
-			'error': function(jqxhr, msg, e) {
-				connfails=connfails+1;
-				if (connfails > 3) {
-					// blocked by many browsers
-					window.close();
-					$('#modal').replaceWith(connfailed);
-					$('#modal').modal('show');
-					return 0;
-				}
-				else {
-					return 1;
-				}
+function longpoll(url, divid, cont) {
+	$.ajax({
+		'url': url,
+		'dataType': 'html',
+		'success': function(data, status, jqxhr) {
+			$('#' + divid).replaceWith(data);
+			connfails=0;
+			cont();
+		},
+		'error': function(jqxhr, msg, e) {
+			connfails=connfails+1;
+			if (connfails > 3) {
+				// blocked by many browsers
+				window.close();
+				$('#modal').replaceWith(connfailed);
+				$('#modal').modal('show');
 			}
-		});
-	})( jQuery );
+			else {
+				cont();
+			}
+		}
+	});
 }
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 29e533c418..d34d5b47d8 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,11 +1,9 @@
 // longpolling for #{ident}
 function poller#{ident}() {
-	if (longpoll('@{gethtml}', '#{ident}')) {
+	longpoll('@{gethtml}', '#{ident}', function() {
 		setTimeout(poller#{ident}, #{delay});
-	}
-}
-(function( $ ) {
-	$(document).bind('ready.app', function() {
-		setTimeout(poller#{ident}, #{startdelay});
 	});
-})( jQuery );
+}
+$(function() {
+	setTimeout(poller#{ident}, #{startdelay});
+});

From d3413ee5a98367bc2f3c56011db8c9393306d80b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 20:33:23 -0400
Subject: [PATCH 4387/8313] further refactoring my js

---
 static/longpolling.js        | 17 ++---------------
 templates/longpolling.julius |  7 ++++---
 templates/page.julius        | 17 +++++++++++++++++
 3 files changed, 23 insertions(+), 18 deletions(-)
 create mode 100644 templates/page.julius

diff --git a/static/longpolling.js b/static/longpolling.js
index 75c6faeaf3..5b704acec3 100644
--- a/static/longpolling.js
+++ b/static/longpolling.js
@@ -3,17 +3,7 @@
 
 connfails=0;
 
-connfailed=
-  '<div id="modal" class="modal fade">' +
-  '  <div class="modal-header">' +
-  '    <h3>git-annex has shut down</h3>' +
-  '  </div>' +
-  '  <div class="modal-body">' +
-  '    You can now close this browser window.' +
-  '  </div>' +
-  '</div>' ;
-
-function longpoll(url, divid, cont) {
+function longpoll(url, divid, cont, fail) {
 	$.ajax({
 		'url': url,
 		'dataType': 'html',
@@ -25,10 +15,7 @@ function longpoll(url, divid, cont) {
 		'error': function(jqxhr, msg, e) {
 			connfails=connfails+1;
 			if (connfails > 3) {
-				// blocked by many browsers
-				window.close();
-				$('#modal').replaceWith(connfailed);
-				$('#modal').modal('show');
+				fail();
 			}
 			else {
 				cont();
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index d34d5b47d8..520699cd73 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,8 +1,9 @@
 // longpolling for #{ident}
 function poller#{ident}() {
-	longpoll('@{gethtml}', '#{ident}', function() {
-		setTimeout(poller#{ident}, #{delay});
-	});
+	longpoll('@{gethtml}', '#{ident}'
+		, function() { setTimeout(poller#{ident}, #{delay}); }
+		, function() { webapp_disconnected(); }
+	);
 }
 $(function() {
 	setTimeout(poller#{ident}, #{startdelay});
diff --git a/templates/page.julius b/templates/page.julius
new file mode 100644
index 0000000000..a9d0b42239
--- /dev/null
+++ b/templates/page.julius
@@ -0,0 +1,17 @@
+connfailed =
+  '<div id="modal" class="modal fade">' +
+  '  <div class="modal-header">' +
+  '    <h3>git-annex has shut down</h3>' +
+  '  </div>' +
+  '  <div class="modal-body">' +
+  '    You can now close this browser window.' +
+  '  </div>' +
+  '</div>' ;
+
+function webapp_disconnected () {
+	$('#modal').replaceWith(connfailed);
+	$('#modal').modal('show');
+
+	// ideal, but blocked by many browsers
+	window.close();
+}

From 2821f9f976b104bcb107f44a13ae7f2eb61f2d7a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 20:35:30 -0400
Subject: [PATCH 4388/8313] cleanup

---
 templates/longpolling.julius | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 520699cd73..67fe059cf9 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,10 +1,9 @@
-// longpolling for #{ident}
-function poller#{ident}() {
+function longpoll_#{ident}() {
 	longpoll('@{gethtml}', '#{ident}'
-		, function() { setTimeout(poller#{ident}, #{delay}); }
+		, function() { setTimeout(longpoll_#{ident}, #{delay}); }
 		, function() { webapp_disconnected(); }
 	);
 }
 $(function() {
-	setTimeout(poller#{ident}, #{startdelay});
+	setTimeout(longpoll_#{ident}, #{startdelay});
 });

From 6e40aed948c44348c977bb7ed7a9a6a84b9972ba Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 22:24:19 -0400
Subject: [PATCH 4389/8313] fix noscript mode to not allocate notification ids
 on each refresh

Now the javascript does an ajax call at the start to request the url
to use to poll, and the notification id is generated then, once we know
javascript is working.
---
 Assistant/Threads/WebApp.hs  | 82 +++++++++++++++++++++---------------
 static/longpolling.js        |  4 +-
 templates/dashboard.hamlet   | 10 +++++
 templates/longpolling.julius |  7 ++-
 4 files changed, 65 insertions(+), 38 deletions(-)
 create mode 100644 templates/dashboard.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index a9b87ea58c..79a388463b 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -36,6 +36,7 @@ import Text.Hamlet
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
 import Data.Text (Text, pack, unpack)
+import qualified Data.Text as T
 import qualified Data.Map as M
 import Control.Concurrent.STM
 
@@ -93,6 +94,8 @@ mkYesod "WebApp" [parseRoutes|
 /noscriptauto NoScriptAutoR GET
 /transfers/#NotificationId TransfersR GET
 /sidebar/#NotificationId SideBarR GET
+/notifier/transfers NotifierTransfersR GET
+/notifier/sidebar NotifierSideBarR GET
 /closealert/#AlertId CloseAlert GET
 /config ConfigR GET
 /addrepository AddRepositoryR GET
@@ -136,19 +139,40 @@ instance Yesod WebApp where
  -
  - The widget should have a html element with an id=ident, which will be
  - replaced when it's updated.
- - 
- - Updating is done by getting html from the gethtml route.
+ -
+ - The geturl route should return the notifier url to use for polling.
  -
  - ms_delay is how long to delay between AJAX updates
  - ms_startdelay is how long to delay before updating with AJAX at the start
  -}
 autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget
-autoUpdate ident gethtml ms_delay ms_startdelay = do
+autoUpdate ident geturl ms_delay ms_startdelay = do
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
 	addScript $ StaticR longpolling_js
 	$(widgetFile "longpolling")
 
+{- Notifier urls are requested by the javascript, to avoid allocation
+ - of NotificationIds when noscript pages are loaded. This constructs a
+ - notifier url for a given Route and NotificationBroadcaster.
+ -}
+notifierUrl :: (NotificationId -> Route WebApp) -> (DaemonStatus -> NotificationBroadcaster) -> Handler RepPlain
+notifierUrl route selector = do
+	(urlbits, _params) <- renderRoute . route <$> newNotifier selector
+	webapp <- getYesod
+	return $ RepPlain $ toContent $ T.concat
+		[ "/"
+		, T.intercalate "/" urlbits
+		, "?auth="
+		, secretToken webapp
+		]
+
+getNotifierTransfersR :: Handler RepPlain
+getNotifierTransfersR = notifierUrl TransfersR transferNotifier
+
+getNotifierSideBarR :: Handler RepPlain
+getNotifierSideBarR = notifierUrl SideBarR alertNotifier
+
 {- A display of currently running and queued transfers.
  -
  - Or, if there have never been any this run, an intro display. -}
@@ -159,7 +183,8 @@ transfersDisplay warnNoScript = do
 		M.toList . currentTransfers
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
 	queued <- liftIO $ getTransferQueue $ transferQueue webapp
-	let ident = transfersDisplayIdent
+	let ident = "transfers"
+	autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int)
 	let transfers = current ++ queued
 	if null transfers
 		then ifM (lift $ showIntro <$> getWebAppState)
@@ -168,9 +193,7 @@ transfersDisplay warnNoScript = do
 			)
 		else $(widgetFile "transfers")
 
-transfersDisplayIdent :: Text
-transfersDisplayIdent = "transfers"
-
+{- An intro message, and list of repositories. -}
 introDisplay :: Text -> Widget
 introDisplay ident = do
 	webapp <- lift getYesod
@@ -206,8 +229,8 @@ getTransfersR nid = do
 	page <- widgetToPageContent $ transfersDisplay False
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
-sideBarDisplay :: Bool -> Widget
-sideBarDisplay noScript = do
+sideBarDisplay :: Widget
+sideBarDisplay = do
 	let content = do
 		{- Any yesod message appears as the first alert. -}
 		maybe noop rendermessage =<< lift getMessage
@@ -218,14 +241,9 @@ sideBarDisplay noScript = do
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
 		mapM_ renderalert $
 			take displayAlerts $ reverse $ sortAlertPairs alertpairs
-	ident <- lift newIdent
+	let ident = "sidebar"
 	$(widgetFile "sidebar")
-
-	unless noScript $ do
-		{- Set up automatic updates of the sidebar
-		 - when alerts come in. -}
-		nid <- lift $ newNotifier alertNotifier
-		autoUpdate ident (SideBarR nid) (10 :: Int) (10 :: Int)
+	autoUpdate ident NotifierSideBarR (10 :: Int) (10 :: Int)
 	where
 		bootstrapclass Activity = "alert-info"
 		bootstrapclass Warning = "alert"
@@ -264,7 +282,7 @@ getSideBarR :: NotificationId -> Handler RepHtml
 getSideBarR nid = do
 	waitNotifier alertNotifier nid
 
-	page <- widgetToPageContent $ sideBarDisplay True
+	page <- widgetToPageContent sideBarDisplay
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 
 {- Called by the client to close an alert. -}
@@ -273,43 +291,39 @@ getCloseAlert i = do
 	webapp <- getYesod
 	void $ liftIO $ removeAlert (daemonStatus webapp) i
 
-dashboard :: Bool -> Bool -> Widget
-dashboard noScript warnNoScript = do
-	sideBarDisplay noScript
-	transfersDisplay warnNoScript
+{- The main dashboard. -}
+dashboard :: Bool -> Widget
+dashboard warnNoScript = do
+	sideBarDisplay
+	let content = transfersDisplay warnNoScript
+	$(widgetFile "dashboard")
 
 getHomeR :: Handler RepHtml
-getHomeR = defaultLayout $ do
-	{- Set up automatic updates for the transfers display. -}
-	nid <- lift $ newNotifier transferNotifier
-	autoUpdate transfersDisplayIdent (TransfersR nid) (10 :: Int) (10 :: Int)
-	
-	dashboard False True
+getHomeR = defaultLayout $ dashboard True
 
-{- Same as HomeR, except with no javascript, so it doesn't allocate
- - new resources each time the page is refreshed, and with autorefreshing
- - via meta refresh. -}
+{- Same as HomeR, except with autorefreshing via meta refresh. -}
 getNoScriptAutoR :: Handler RepHtml
 getNoScriptAutoR = defaultLayout $ do
 	let ident = NoScriptR
 	let delayseconds = 3 :: Int
 	let this = NoScriptAutoR
 	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
-	dashboard True False
+	dashboard False
 
+{- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
 getNoScriptR :: Handler RepHtml
 getNoScriptR = defaultLayout $
-	dashboard True True
+	dashboard False
 
 getConfigR :: Handler RepHtml
 getConfigR = defaultLayout $ do
-	sideBarDisplay False
+	sideBarDisplay
 	setTitle "Configuration"
 	[whamlet|<a href="@{HomeR}">main|]
 
 getAddRepositoryR :: Handler RepHtml
 getAddRepositoryR = defaultLayout $ do
-	sideBarDisplay False
+	sideBarDisplay
 	setTitle "Add repository"
 	[whamlet|<a href="@{HomeR}">main|]
 
diff --git a/static/longpolling.js b/static/longpolling.js
index 5b704acec3..965c1d18db 100644
--- a/static/longpolling.js
+++ b/static/longpolling.js
@@ -1,5 +1,5 @@
-// Uses long-polling to update a div with a specified id,
-// by polling an url, which should return a new div, with the same id.
+// Updates a div with a specified id, by polling an url,
+// which should return a new div, with the same id.
 
 connfails=0;
 
diff --git a/templates/dashboard.hamlet b/templates/dashboard.hamlet
new file mode 100644
index 0000000000..7bcfce9626
--- /dev/null
+++ b/templates/dashboard.hamlet
@@ -0,0 +1,10 @@
+^{content}
+$if warnNoScript
+  <noscript>
+    <div .navbar .navbar-fixed-bottom>
+      <div .navbar-inner>
+        <div .container>
+          Javascript is disabled; cannot update in real-time.
+          <div .btn-group>
+            <a .btn .btn-primary href="@{NoScriptAutoR}">Auto-refresh every 3 seconds #
+            <a .btn .btn-primary href="@{NoScriptR}">Manually refresh
diff --git a/templates/longpolling.julius b/templates/longpolling.julius
index 67fe059cf9..affa50cc81 100644
--- a/templates/longpolling.julius
+++ b/templates/longpolling.julius
@@ -1,9 +1,12 @@
 function longpoll_#{ident}() {
-	longpoll('@{gethtml}', '#{ident}'
+	longpoll(longpoll_#{ident}_url, '#{ident}'
 		, function() { setTimeout(longpoll_#{ident}, #{delay}); }
 		, function() { webapp_disconnected(); }
 	);
 }
 $(function() {
-	setTimeout(longpoll_#{ident}, #{startdelay});
+	$.get("@{geturl}", function(url){
+		longpoll_#{ident}_url = url;
+		setTimeout(longpoll_#{ident}, #{startdelay});
+	});
 });

From 58496c18d9c37329d36f1c8dc96af55345a19582 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Mon, 30 Jul 2012 22:28:51 -0400
Subject: [PATCH 4390/8313] done

---
 doc/design/assistant/webapp.mdwn | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index ebf1689963..d1eedcb322 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -35,6 +35,3 @@ The webapp is a web server that displays a shiny interface.
   and use a client-side session. It could be encrypted using the token
   as the `encryptKey`. Note: Would need to set the session duration
   to infinite (how?)
-* Fix notification handle leakage on pages other than the main page.
-  The javascript should use AJAX to request handles, that way
-  they won't be allocated at all in noscript.

From 58dfa3fa5b1b8be6f344e9ef5bfb3adda11069ab Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 01:11:32 -0400
Subject: [PATCH 4391/8313] split up webapp files

---
 Assistant/Threads/WebApp.hs       | 304 +-----------------------------
 Assistant/WebApp.hs               | 106 +++++++++++
 Assistant/WebApp/Configurators.hs |  56 ++++++
 Assistant/WebApp/DashBoard.hs     |  89 +++++++++
 Assistant/WebApp/Notifications.hs |  58 ++++++
 Assistant/WebApp/SideBar.hs       |  84 +++++++++
 Assistant/WebApp/routes           |  13 ++
 7 files changed, 413 insertions(+), 297 deletions(-)
 create mode 100644 Assistant/WebApp.hs
 create mode 100644 Assistant/WebApp/Configurators.hs
 create mode 100644 Assistant/WebApp/DashBoard.hs
 create mode 100644 Assistant/WebApp/Notifications.hs
 create mode 100644 Assistant/WebApp/SideBar.hs
 create mode 100644 Assistant/WebApp/routes

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 79a388463b..7b794b6ebf 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -11,321 +11,31 @@
 module Assistant.Threads.WebApp where
 
 import Assistant.Common
+import Assistant.WebApp
+import Assistant.WebApp.DashBoard
+import Assistant.WebApp.SideBar
+import Assistant.WebApp.Notifications
+import Assistant.WebApp.Configurators
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.TransferQueue
-import Assistant.Alert hiding (Widget)
-import Utility.NotificationBroadcaster
 import Utility.WebApp
 import Utility.Yesod
 import Utility.FileMode
 import Utility.TempFile
 import Git
-import Logs.Transfer
-import Utility.Percentage
-import Utility.DataUnits
-import Types.Key
-import qualified Remote
-import Logs.Web (webUUID)
-import Logs.Trust
-import Annex.UUID (getUUID)
 
 import Yesod
 import Yesod.Static
 import Text.Hamlet
 import Network.Socket (PortNumber)
 import Text.Blaze.Renderer.String
-import Data.Text (Text, pack, unpack)
-import qualified Data.Text as T
-import qualified Data.Map as M
-import Control.Concurrent.STM
+import Data.Text (pack, unpack)
 
 thisThread :: String
 thisThread = "WebApp"
 
-data WebApp = WebApp
-	{ threadState :: ThreadState
-	, daemonStatus :: DaemonStatusHandle
-	, transferQueue :: TransferQueue
-	, secretToken :: Text
-	, relDir :: FilePath
-	, getStatic :: Static
-	, webAppState :: TMVar WebAppState
-	}
-
-data WebAppState = WebAppState
-	{ showIntro :: Bool
-	}
-
-newWebAppState :: IO (TMVar WebAppState)
-newWebAppState = liftIO $ atomically $
-	newTMVar $ WebAppState { showIntro = True }
-
-getWebAppState :: forall sub. GHandler sub WebApp WebAppState
-getWebAppState = liftIO . atomically . readTMVar =<< webAppState <$> getYesod
-
-modifyWebAppState :: forall sub. (WebAppState -> WebAppState) -> GHandler sub WebApp ()
-modifyWebAppState a = go =<< webAppState <$> getYesod
-	where
-		go s = liftIO $ atomically $ do
-			v <- takeTMVar s
-			putTMVar s $ a v
-
-waitNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> NotificationId -> GHandler sub WebApp ()
-waitNotifier selector nid = do
-	notifier <- getNotifier selector
-	liftIO $ waitNotification $ notificationHandleFromId notifier nid
-
-newNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> GHandler sub WebApp NotificationId
-newNotifier selector = do
-	notifier <- getNotifier selector
-	liftIO $ notificationHandleToId <$> newNotificationHandle notifier
-
-getNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> GHandler sub WebApp NotificationBroadcaster
-getNotifier selector = do
-	webapp <- getYesod
-	liftIO $ selector <$> getDaemonStatus (daemonStatus webapp)
-
-staticFiles "static"
-
-mkYesod "WebApp" [parseRoutes|
-/ HomeR GET
-/noscript NoScriptR GET
-/noscriptauto NoScriptAutoR GET
-/transfers/#NotificationId TransfersR GET
-/sidebar/#NotificationId SideBarR GET
-/notifier/transfers NotifierTransfersR GET
-/notifier/sidebar NotifierSideBarR GET
-/closealert/#AlertId CloseAlert GET
-/config ConfigR GET
-/addrepository AddRepositoryR GET
-/static StaticR Static getStatic
-|]
-
-instance PathPiece NotificationId where
-    toPathPiece = pack . show
-    fromPathPiece = readish . unpack
-
-instance PathPiece AlertId where
-    toPathPiece = pack . show
-    fromPathPiece = readish . unpack
-
-instance Yesod WebApp where
-	defaultLayout content = do
-		webapp <- getYesod
-		page <- widgetToPageContent $ do
-			addStylesheet $ StaticR css_bootstrap_css
-			addStylesheet $ StaticR css_bootstrap_responsive_css
-			addScript $ StaticR jquery_full_js
-			addScript $ StaticR js_bootstrap_dropdown_js
-			addScript $ StaticR js_bootstrap_modal_js
-			$(widgetFile "page")
-		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
-
-	{- Require an auth token be set when accessing any (non-static route) -}
-	isAuthorized _ _ = checkAuthToken secretToken
-
-	{- Add the auth token to every url generated, except static subsite
-         - urls (which can show up in Permission Denied pages). -}
-	joinPath = insertAuthToken secretToken excludeStatic
-		where
-			excludeStatic [] = True
-			excludeStatic (p:_) = p /= "static"
-
-	makeSessionBackend = webAppSessionBackend
-	jsLoader _ = BottomOfHeadBlocking
-
-{- Add to any widget to make it auto-update using long polling.
- -
- - The widget should have a html element with an id=ident, which will be
- - replaced when it's updated.
- -
- - The geturl route should return the notifier url to use for polling.
- -
- - ms_delay is how long to delay between AJAX updates
- - ms_startdelay is how long to delay before updating with AJAX at the start
- -}
-autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget
-autoUpdate ident geturl ms_delay ms_startdelay = do
-	let delay = show ms_delay
-	let startdelay = show ms_startdelay
-	addScript $ StaticR longpolling_js
-	$(widgetFile "longpolling")
-
-{- Notifier urls are requested by the javascript, to avoid allocation
- - of NotificationIds when noscript pages are loaded. This constructs a
- - notifier url for a given Route and NotificationBroadcaster.
- -}
-notifierUrl :: (NotificationId -> Route WebApp) -> (DaemonStatus -> NotificationBroadcaster) -> Handler RepPlain
-notifierUrl route selector = do
-	(urlbits, _params) <- renderRoute . route <$> newNotifier selector
-	webapp <- getYesod
-	return $ RepPlain $ toContent $ T.concat
-		[ "/"
-		, T.intercalate "/" urlbits
-		, "?auth="
-		, secretToken webapp
-		]
-
-getNotifierTransfersR :: Handler RepPlain
-getNotifierTransfersR = notifierUrl TransfersR transferNotifier
-
-getNotifierSideBarR :: Handler RepPlain
-getNotifierSideBarR = notifierUrl SideBarR alertNotifier
-
-{- A display of currently running and queued transfers.
- -
- - Or, if there have never been any this run, an intro display. -}
-transfersDisplay :: Bool -> Widget
-transfersDisplay warnNoScript = do
-	webapp <- lift getYesod
-	current <- liftIO $ runThreadState (threadState webapp) $
-		M.toList . currentTransfers
-			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
-	queued <- liftIO $ getTransferQueue $ transferQueue webapp
-	let ident = "transfers"
-	autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int)
-	let transfers = current ++ queued
-	if null transfers
-		then ifM (lift $ showIntro <$> getWebAppState)
-			( introDisplay ident
-			, $(widgetFile "transfers")
-			)
-		else $(widgetFile "transfers")
-
-{- An intro message, and list of repositories. -}
-introDisplay :: Text -> Widget
-introDisplay ident = do
-	webapp <- lift getYesod
-	let reldir = relDir webapp
-	l <- liftIO $ runThreadState (threadState webapp) $ do
-		u <- getUUID
-		rs <- map Remote.uuid <$> Remote.remoteList
-		rs' <- snd <$> trustPartition DeadTrusted rs
-		Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs'
-	let remotelist = zip counter l
-	let n = length l
-	let numrepos = show n
-	let notenough = n < 2
-	let barelyenough = n == 2
-	let morethanenough = n > 2
-	$(widgetFile "intro")
-	lift $ modifyWebAppState $ \s -> s { showIntro = False }
-	where
-		counter = map show ([1..] :: [Int])
-
-{- Called by client to get a display of currently in process transfers.
- -
- - Returns a div, which will be inserted into the calling page.
- -
- - Note that the head of the widget is not included, only its
- - body is. To get the widget head content, the widget is also 
- - inserted onto the getHomeR page.
- -}
-getTransfersR :: NotificationId -> Handler RepHtml
-getTransfersR nid = do
-	waitNotifier transferNotifier nid
-
-	page <- widgetToPageContent $ transfersDisplay False
-	hamletToRepHtml $ [hamlet|^{pageBody page}|]
-
-sideBarDisplay :: Widget
-sideBarDisplay = do
-	let content = do
-		{- Any yesod message appears as the first alert. -}
-		maybe noop rendermessage =<< lift getMessage
-	
-		{- Add newest alerts to the sidebar. -}
-		webapp <- lift getYesod
-		alertpairs <- M.toList . alertMap
-			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
-		mapM_ renderalert $
-			take displayAlerts $ reverse $ sortAlertPairs alertpairs
-	let ident = "sidebar"
-	$(widgetFile "sidebar")
-	autoUpdate ident NotifierSideBarR (10 :: Int) (10 :: Int)
-	where
-		bootstrapclass Activity = "alert-info"
-		bootstrapclass Warning = "alert"
-		bootstrapclass Error = "alert-error"
-		bootstrapclass Success = "alert-success"
-		bootstrapclass Message = "alert-info"
-
-		renderalert (alertid, alert) = addalert
-			alertid
-			(alertClosable alert)
-			(alertBlockDisplay alert)
-			(bootstrapclass $ alertClass alert)
-			(alertHeader alert)
-			$ case alertMessage alert of
-				StringAlert s -> [whamlet|#{s}|]
-				WidgetAlert w -> w alert
-
-		rendermessage msg = addalert firstAlertId True False
-			"alert-info" Nothing [whamlet|#{msg}|]
-
-		addalert :: AlertId -> Bool -> Bool -> Text -> Maybe String -> Widget -> Widget
-		addalert i closable block divclass heading widget = do
-			let alertid = show i
-			let closealert = CloseAlert i
-			$(widgetFile "alert")
-
-{- Called by client to get a sidebar display.
- -
- - Returns a div, which will be inserted into the calling page.
- -
- - Note that the head of the widget is not included, only its
- - body is. To get the widget head content, the widget is also 
- - inserted onto all pages.
- -}
-getSideBarR :: NotificationId -> Handler RepHtml
-getSideBarR nid = do
-	waitNotifier alertNotifier nid
-
-	page <- widgetToPageContent sideBarDisplay
-	hamletToRepHtml $ [hamlet|^{pageBody page}|]
-
-{- Called by the client to close an alert. -}
-getCloseAlert :: AlertId -> Handler ()
-getCloseAlert i = do
-	webapp <- getYesod
-	void $ liftIO $ removeAlert (daemonStatus webapp) i
-
-{- The main dashboard. -}
-dashboard :: Bool -> Widget
-dashboard warnNoScript = do
-	sideBarDisplay
-	let content = transfersDisplay warnNoScript
-	$(widgetFile "dashboard")
-
-getHomeR :: Handler RepHtml
-getHomeR = defaultLayout $ dashboard True
-
-{- Same as HomeR, except with autorefreshing via meta refresh. -}
-getNoScriptAutoR :: Handler RepHtml
-getNoScriptAutoR = defaultLayout $ do
-	let ident = NoScriptR
-	let delayseconds = 3 :: Int
-	let this = NoScriptAutoR
-	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
-	dashboard False
-
-{- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
-getNoScriptR :: Handler RepHtml
-getNoScriptR = defaultLayout $
-	dashboard False
-
-getConfigR :: Handler RepHtml
-getConfigR = defaultLayout $ do
-	sideBarDisplay
-	setTitle "Configuration"
-	[whamlet|<a href="@{HomeR}">main|]
-
-getAddRepositoryR :: Handler RepHtml
-getAddRepositoryR = defaultLayout $ do
-	sideBarDisplay
-	setTitle "Add repository"
-	[whamlet|<a href="@{HomeR}">main|]
+mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes")
 
 webAppThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Maybe (IO ()) -> IO ()
 webAppThread st dstatus transferqueue onstartup = do
diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
new file mode 100644
index 0000000000..d3989a68af
--- /dev/null
+++ b/Assistant/WebApp.hs
@@ -0,0 +1,106 @@
+{- git-annex assistant webapp data types
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
+
+module Assistant.WebApp where
+
+import Assistant.Common
+import Assistant.ThreadedMonad
+import Assistant.DaemonStatus
+import Assistant.TransferQueue
+import Assistant.Alert hiding (Widget)
+import Utility.NotificationBroadcaster
+import Utility.WebApp
+import Utility.Yesod
+
+import Yesod
+import Yesod.Static
+import Text.Hamlet
+import Data.Text (Text, pack, unpack)
+import Control.Concurrent.STM
+
+staticFiles "static"
+
+mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes")
+
+data WebApp = WebApp
+	{ threadState :: ThreadState
+	, daemonStatus :: DaemonStatusHandle
+	, transferQueue :: TransferQueue
+	, secretToken :: Text
+	, relDir :: FilePath
+	, getStatic :: Static
+	, webAppState :: TMVar WebAppState
+	}
+
+instance Yesod WebApp where
+	defaultLayout content = do
+		webapp <- getYesod
+		page <- widgetToPageContent $ do
+			addStylesheet $ StaticR css_bootstrap_css
+			addStylesheet $ StaticR css_bootstrap_responsive_css
+			addScript $ StaticR jquery_full_js
+			addScript $ StaticR js_bootstrap_dropdown_js
+			addScript $ StaticR js_bootstrap_modal_js
+			$(widgetFile "page")
+		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
+
+	{- Require an auth token be set when accessing any (non-static route) -}
+	isAuthorized _ _ = checkAuthToken secretToken
+
+	{- Add the auth token to every url generated, except static subsite
+         - urls (which can show up in Permission Denied pages). -}
+	joinPath = insertAuthToken secretToken excludeStatic
+		where
+			excludeStatic [] = True
+			excludeStatic (p:_) = p /= "static"
+
+	makeSessionBackend = webAppSessionBackend
+	jsLoader _ = BottomOfHeadBlocking
+
+data WebAppState = WebAppState
+	{ showIntro :: Bool
+	}
+
+newWebAppState :: IO (TMVar WebAppState)
+newWebAppState = liftIO $ atomically $
+	newTMVar $ WebAppState { showIntro = True }
+
+getWebAppState :: forall sub. GHandler sub WebApp WebAppState
+getWebAppState = liftIO . atomically . readTMVar =<< webAppState <$> getYesod
+
+modifyWebAppState :: forall sub. (WebAppState -> WebAppState) -> GHandler sub WebApp ()
+modifyWebAppState a = go =<< webAppState <$> getYesod
+	where
+		go s = liftIO $ atomically $ do
+			v <- takeTMVar s
+			putTMVar s $ a v
+
+waitNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> NotificationId -> GHandler sub WebApp ()
+waitNotifier selector nid = do
+	notifier <- getNotifier selector
+	liftIO $ waitNotification $ notificationHandleFromId notifier nid
+
+newNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> GHandler sub WebApp NotificationId
+newNotifier selector = do
+	notifier <- getNotifier selector
+	liftIO $ notificationHandleToId <$> newNotificationHandle notifier
+
+getNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> GHandler sub WebApp NotificationBroadcaster
+getNotifier selector = do
+	webapp <- getYesod
+	liftIO $ selector <$> getDaemonStatus (daemonStatus webapp)
+
+instance PathPiece NotificationId where
+    toPathPiece = pack . show
+    fromPathPiece = readish . unpack
+
+instance PathPiece AlertId where
+    toPathPiece = pack . show
+    fromPathPiece = readish . unpack
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
new file mode 100644
index 0000000000..be6f12db3c
--- /dev/null
+++ b/Assistant/WebApp/Configurators.hs
@@ -0,0 +1,56 @@
+{- git-annex assistant webapp configurators
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+
+module Assistant.WebApp.Configurators where
+
+import Assistant.Common
+import Assistant.WebApp
+import Assistant.WebApp.SideBar
+import Assistant.ThreadedMonad
+import Utility.Yesod
+import qualified Remote
+import Logs.Web (webUUID)
+import Logs.Trust
+import Annex.UUID (getUUID)
+
+import Yesod
+import Data.Text (Text)
+
+{- An intro message, list of repositories, and nudge to make more. -}
+introDisplay :: Text -> Widget
+introDisplay ident = do
+	webapp <- lift getYesod
+	let reldir = relDir webapp
+	l <- liftIO $ runThreadState (threadState webapp) $ do
+		u <- getUUID
+		rs <- map Remote.uuid <$> Remote.remoteList
+		rs' <- snd <$> trustPartition DeadTrusted rs
+		Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs'
+	let remotelist = zip counter l
+	let n = length l
+	let numrepos = show n
+	let notenough = n < 2
+	let barelyenough = n == 2
+	let morethanenough = n > 2
+	$(widgetFile "intro")
+	lift $ modifyWebAppState $ \s -> s { showIntro = False }
+	where
+		counter = map show ([1..] :: [Int])
+
+getConfigR :: Handler RepHtml
+getConfigR = defaultLayout $ do
+	sideBarDisplay
+	setTitle "Configuration"
+	[whamlet|<a href="@{HomeR}">main|]
+
+getAddRepositoryR :: Handler RepHtml
+getAddRepositoryR = defaultLayout $ do
+	sideBarDisplay
+	setTitle "Add repository"
+	[whamlet|<a href="@{HomeR}">main|]
diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs
new file mode 100644
index 0000000000..5df68c93b6
--- /dev/null
+++ b/Assistant/WebApp/DashBoard.hs
@@ -0,0 +1,89 @@
+{- git-annex assistant webapp dashboard
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+
+module Assistant.WebApp.DashBoard where
+
+import Assistant.Common
+import Assistant.WebApp
+import Assistant.WebApp.SideBar
+import Assistant.WebApp.Notifications
+import Assistant.WebApp.Configurators
+import Assistant.ThreadedMonad
+import Assistant.DaemonStatus
+import Assistant.TransferQueue
+import Utility.NotificationBroadcaster
+import Utility.Yesod
+import Logs.Transfer
+import Utility.Percentage
+import Utility.DataUnits
+import Types.Key
+import qualified Remote
+
+import Yesod
+import Text.Hamlet
+import qualified Data.Map as M
+
+{- A display of currently running and queued transfers.
+ -
+ - Or, if there have never been any this run, an intro display. -}
+transfersDisplay :: Bool -> Widget
+transfersDisplay warnNoScript = do
+	webapp <- lift getYesod
+	current <- liftIO $ runThreadState (threadState webapp) $
+		M.toList . currentTransfers
+			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
+	queued <- liftIO $ getTransferQueue $ transferQueue webapp
+	let ident = "transfers"
+	autoUpdate ident NotifierTransfersR (10 :: Int) (10 :: Int)
+	let transfers = current ++ queued
+	if null transfers
+		then ifM (lift $ showIntro <$> getWebAppState)
+			( introDisplay ident
+			, $(widgetFile "transfers")
+			)
+		else $(widgetFile "transfers")
+
+{- Called by client to get a display of currently in process transfers.
+ -
+ - Returns a div, which will be inserted into the calling page.
+ -
+ - Note that the head of the widget is not included, only its
+ - body is. To get the widget head content, the widget is also 
+ - inserted onto the getHomeR page.
+ -}
+getTransfersR :: NotificationId -> Handler RepHtml
+getTransfersR nid = do
+	waitNotifier transferNotifier nid
+
+	page <- widgetToPageContent $ transfersDisplay False
+	hamletToRepHtml $ [hamlet|^{pageBody page}|]
+
+{- The main dashboard. -}
+dashboard :: Bool -> Widget
+dashboard warnNoScript = do
+	sideBarDisplay
+	let content = transfersDisplay warnNoScript
+	$(widgetFile "dashboard")
+
+getHomeR :: Handler RepHtml
+getHomeR = defaultLayout $ dashboard True
+
+{- Same as HomeR, except with autorefreshing via meta refresh. -}
+getNoScriptAutoR :: Handler RepHtml
+getNoScriptAutoR = defaultLayout $ do
+	let ident = NoScriptR
+	let delayseconds = 3 :: Int
+	let this = NoScriptAutoR
+	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
+	dashboard False
+
+{- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
+getNoScriptR :: Handler RepHtml
+getNoScriptR = defaultLayout $
+	dashboard False
diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs
new file mode 100644
index 0000000000..1e7c0176a0
--- /dev/null
+++ b/Assistant/WebApp/Notifications.hs
@@ -0,0 +1,58 @@
+{- git-annex assistant webapp notifications
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+
+module Assistant.WebApp.Notifications where
+
+import Assistant.Common
+import Assistant.WebApp
+import Assistant.DaemonStatus
+import Utility.NotificationBroadcaster
+import Utility.Yesod
+
+import Yesod
+import Data.Text (Text)
+import qualified Data.Text as T
+
+{- Add to any widget to make it auto-update using long polling.
+ -
+ - The widget should have a html element with an id=ident, which will be
+ - replaced when it's updated.
+ -
+ - The geturl route should return the notifier url to use for polling.
+ -
+ - ms_delay is how long to delay between AJAX updates
+ - ms_startdelay is how long to delay before updating with AJAX at the start
+ -}
+autoUpdate :: Text -> Route WebApp -> Int -> Int -> Widget
+autoUpdate ident geturl ms_delay ms_startdelay = do
+	let delay = show ms_delay
+	let startdelay = show ms_startdelay
+	addScript $ StaticR longpolling_js
+	$(widgetFile "longpolling")
+
+{- Notifier urls are requested by the javascript, to avoid allocation
+ - of NotificationIds when noscript pages are loaded. This constructs a
+ - notifier url for a given Route and NotificationBroadcaster.
+ -}
+notifierUrl :: (NotificationId -> Route WebApp) -> (DaemonStatus -> NotificationBroadcaster) -> Handler RepPlain
+notifierUrl route selector = do
+	(urlbits, _params) <- renderRoute . route <$> newNotifier selector
+	webapp <- getYesod
+	return $ RepPlain $ toContent $ T.concat
+		[ "/"
+		, T.intercalate "/" urlbits
+		, "?auth="
+		, secretToken webapp
+		]
+
+getNotifierTransfersR :: Handler RepPlain
+getNotifierTransfersR = notifierUrl TransfersR transferNotifier
+
+getNotifierSideBarR :: Handler RepPlain
+getNotifierSideBarR = notifierUrl SideBarR alertNotifier
diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs
new file mode 100644
index 0000000000..4df0c8d550
--- /dev/null
+++ b/Assistant/WebApp/SideBar.hs
@@ -0,0 +1,84 @@
+{- git-annex assistant webapp sidebar
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+
+module Assistant.WebApp.SideBar where
+
+import Assistant.Common
+import Assistant.WebApp
+import Assistant.WebApp.Notifications
+import Assistant.DaemonStatus
+import Assistant.Alert hiding (Widget)
+import Utility.NotificationBroadcaster
+import Utility.Yesod
+
+import Yesod
+import Data.Text (Text)
+import qualified Data.Map as M
+
+sideBarDisplay :: Widget
+sideBarDisplay = do
+	let content = do
+		{- Any yesod message appears as the first alert. -}
+		maybe noop rendermessage =<< lift getMessage
+	
+		{- Add newest alerts to the sidebar. -}
+		webapp <- lift getYesod
+		alertpairs <- M.toList . alertMap
+			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
+		mapM_ renderalert $
+			take displayAlerts $ reverse $ sortAlertPairs alertpairs
+	let ident = "sidebar"
+	$(widgetFile "sidebar")
+	autoUpdate ident NotifierSideBarR (10 :: Int) (10 :: Int)
+	where
+		bootstrapclass Activity = "alert-info"
+		bootstrapclass Warning = "alert"
+		bootstrapclass Error = "alert-error"
+		bootstrapclass Success = "alert-success"
+		bootstrapclass Message = "alert-info"
+
+		renderalert (alertid, alert) = addalert
+			alertid
+			(alertClosable alert)
+			(alertBlockDisplay alert)
+			(bootstrapclass $ alertClass alert)
+			(alertHeader alert)
+			$ case alertMessage alert of
+				StringAlert s -> [whamlet|#{s}|]
+				WidgetAlert w -> w alert
+
+		rendermessage msg = addalert firstAlertId True False
+			"alert-info" Nothing [whamlet|#{msg}|]
+
+		addalert :: AlertId -> Bool -> Bool -> Text -> Maybe String -> Widget -> Widget
+		addalert i closable block divclass heading widget = do
+			let alertid = show i
+			let closealert = CloseAlert i
+			$(widgetFile "alert")
+
+{- Called by client to get a sidebar display.
+ -
+ - Returns a div, which will be inserted into the calling page.
+ -
+ - Note that the head of the widget is not included, only its
+ - body is. To get the widget head content, the widget is also 
+ - inserted onto all pages.
+ -}
+getSideBarR :: NotificationId -> Handler RepHtml
+getSideBarR nid = do
+	waitNotifier alertNotifier nid
+
+	page <- widgetToPageContent sideBarDisplay
+	hamletToRepHtml $ [hamlet|^{pageBody page}|]
+
+{- Called by the client to close an alert. -}
+getCloseAlert :: AlertId -> Handler ()
+getCloseAlert i = do
+	webapp <- getYesod
+	void $ liftIO $ removeAlert (daemonStatus webapp) i
diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes
new file mode 100644
index 0000000000..75f1ad7c78
--- /dev/null
+++ b/Assistant/WebApp/routes
@@ -0,0 +1,13 @@
+/ HomeR GET
+/noscript NoScriptR GET
+/noscriptauto NoScriptAutoR GET
+/config ConfigR GET
+/addrepository AddRepositoryR GET
+
+/transfers/#NotificationId TransfersR GET
+/sidebar/#NotificationId SideBarR GET
+/notifier/transfers NotifierTransfersR GET
+/notifier/sidebar NotifierSideBarR GET
+/closealert/#AlertId CloseAlert GET
+
+/static StaticR Static getStatic

From 5fed026bcdaa0724acd2640193e341bb8358980b Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 01:24:49 -0400
Subject: [PATCH 4392/8313] reorg templates

---
 Assistant/WebApp/Configurators.hs                     | 2 +-
 Assistant/WebApp/DashBoard.hs                         | 8 ++++----
 Assistant/WebApp/Notifications.hs                     | 2 +-
 Assistant/WebApp/SideBar.hs                           | 4 ++--
 templates/{ => configurators}/intro.hamlet            | 0
 templates/{dashboard.hamlet => dashboard/main.hamlet} | 0
 templates/{ => dashboard}/metarefresh.hamlet          | 0
 templates/{ => dashboard}/transfers.hamlet            | 0
 templates/{ => notifications}/longpolling.julius      | 0
 templates/{ => sidebar}/alert.hamlet                  | 0
 templates/{sidebar.hamlet => sidebar/main.hamlet}     | 0
 11 files changed, 8 insertions(+), 8 deletions(-)
 rename templates/{ => configurators}/intro.hamlet (100%)
 rename templates/{dashboard.hamlet => dashboard/main.hamlet} (100%)
 rename templates/{ => dashboard}/metarefresh.hamlet (100%)
 rename templates/{ => dashboard}/transfers.hamlet (100%)
 rename templates/{ => notifications}/longpolling.julius (100%)
 rename templates/{ => sidebar}/alert.hamlet (100%)
 rename templates/{sidebar.hamlet => sidebar/main.hamlet} (100%)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index be6f12db3c..e3f0275d9a 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -38,7 +38,7 @@ introDisplay ident = do
 	let notenough = n < 2
 	let barelyenough = n == 2
 	let morethanenough = n > 2
-	$(widgetFile "intro")
+	$(widgetFile "configurators/intro")
 	lift $ modifyWebAppState $ \s -> s { showIntro = False }
 	where
 		counter = map show ([1..] :: [Int])
diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs
index 5df68c93b6..2961dabd35 100644
--- a/Assistant/WebApp/DashBoard.hs
+++ b/Assistant/WebApp/DashBoard.hs
@@ -45,9 +45,9 @@ transfersDisplay warnNoScript = do
 	if null transfers
 		then ifM (lift $ showIntro <$> getWebAppState)
 			( introDisplay ident
-			, $(widgetFile "transfers")
+			, $(widgetFile "dashboard/transfers")
 			)
-		else $(widgetFile "transfers")
+		else $(widgetFile "dashboard/transfers")
 
 {- Called by client to get a display of currently in process transfers.
  -
@@ -69,7 +69,7 @@ dashboard :: Bool -> Widget
 dashboard warnNoScript = do
 	sideBarDisplay
 	let content = transfersDisplay warnNoScript
-	$(widgetFile "dashboard")
+	$(widgetFile "dashboard/main")
 
 getHomeR :: Handler RepHtml
 getHomeR = defaultLayout $ dashboard True
@@ -80,7 +80,7 @@ getNoScriptAutoR = defaultLayout $ do
 	let ident = NoScriptR
 	let delayseconds = 3 :: Int
 	let this = NoScriptAutoR
-	toWidgetHead $(hamletFile $ hamletTemplate "metarefresh")
+	toWidgetHead $(hamletFile $ hamletTemplate "dashboard/metarefresh")
 	dashboard False
 
 {- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
diff --git a/Assistant/WebApp/Notifications.hs b/Assistant/WebApp/Notifications.hs
index 1e7c0176a0..3aa56424a4 100644
--- a/Assistant/WebApp/Notifications.hs
+++ b/Assistant/WebApp/Notifications.hs
@@ -34,7 +34,7 @@ autoUpdate ident geturl ms_delay ms_startdelay = do
 	let delay = show ms_delay
 	let startdelay = show ms_startdelay
 	addScript $ StaticR longpolling_js
-	$(widgetFile "longpolling")
+	$(widgetFile "notifications/longpolling")
 
 {- Notifier urls are requested by the javascript, to avoid allocation
  - of NotificationIds when noscript pages are loaded. This constructs a
diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs
index 4df0c8d550..2c630324c4 100644
--- a/Assistant/WebApp/SideBar.hs
+++ b/Assistant/WebApp/SideBar.hs
@@ -34,7 +34,7 @@ sideBarDisplay = do
 		mapM_ renderalert $
 			take displayAlerts $ reverse $ sortAlertPairs alertpairs
 	let ident = "sidebar"
-	$(widgetFile "sidebar")
+	$(widgetFile "sidebar/main")
 	autoUpdate ident NotifierSideBarR (10 :: Int) (10 :: Int)
 	where
 		bootstrapclass Activity = "alert-info"
@@ -60,7 +60,7 @@ sideBarDisplay = do
 		addalert i closable block divclass heading widget = do
 			let alertid = show i
 			let closealert = CloseAlert i
-			$(widgetFile "alert")
+			$(widgetFile "sidebar/alert")
 
 {- Called by client to get a sidebar display.
  -
diff --git a/templates/intro.hamlet b/templates/configurators/intro.hamlet
similarity index 100%
rename from templates/intro.hamlet
rename to templates/configurators/intro.hamlet
diff --git a/templates/dashboard.hamlet b/templates/dashboard/main.hamlet
similarity index 100%
rename from templates/dashboard.hamlet
rename to templates/dashboard/main.hamlet
diff --git a/templates/metarefresh.hamlet b/templates/dashboard/metarefresh.hamlet
similarity index 100%
rename from templates/metarefresh.hamlet
rename to templates/dashboard/metarefresh.hamlet
diff --git a/templates/transfers.hamlet b/templates/dashboard/transfers.hamlet
similarity index 100%
rename from templates/transfers.hamlet
rename to templates/dashboard/transfers.hamlet
diff --git a/templates/longpolling.julius b/templates/notifications/longpolling.julius
similarity index 100%
rename from templates/longpolling.julius
rename to templates/notifications/longpolling.julius
diff --git a/templates/alert.hamlet b/templates/sidebar/alert.hamlet
similarity index 100%
rename from templates/alert.hamlet
rename to templates/sidebar/alert.hamlet
diff --git a/templates/sidebar.hamlet b/templates/sidebar/main.hamlet
similarity index 100%
rename from templates/sidebar.hamlet
rename to templates/sidebar/main.hamlet

From 2c8bbdf307899683ea8e2d934ec0ed2bfa3bc3d4 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 02:30:26 -0400
Subject: [PATCH 4393/8313] made navbar work

also added an About page and a stub Config page.
---
 Assistant/Threads/WebApp.hs          |  1 +
 Assistant/WebApp.hs                  | 44 +++++++++++++++++++++-------
 Assistant/WebApp/Configurators.hs    |  7 ++---
 Assistant/WebApp/DashBoard.hs        | 13 ++++----
 Assistant/WebApp/Documentation.hs    | 22 ++++++++++++++
 Assistant/WebApp/routes              |  1 +
 templates/bootstrap.hamlet           |  1 +
 templates/configurators/main.hamlet  |  3 ++
 templates/documentation/about.hamlet | 16 ++++++++++
 templates/page.hamlet                |  8 ++---
 10 files changed, 89 insertions(+), 27 deletions(-)
 create mode 100644 Assistant/WebApp/Documentation.hs
 create mode 100644 templates/configurators/main.hamlet
 create mode 100644 templates/documentation/about.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 7b794b6ebf..ca81498f4a 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -16,6 +16,7 @@ import Assistant.WebApp.DashBoard
 import Assistant.WebApp.SideBar
 import Assistant.WebApp.Notifications
 import Assistant.WebApp.Configurators
+import Assistant.WebApp.Documentation
 import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.TransferQueue
diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index d3989a68af..fc40ca5bfb 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -39,18 +39,40 @@ data WebApp = WebApp
 	, webAppState :: TMVar WebAppState
 	}
 
-instance Yesod WebApp where
-	defaultLayout content = do
-		webapp <- getYesod
-		page <- widgetToPageContent $ do
-			addStylesheet $ StaticR css_bootstrap_css
-			addStylesheet $ StaticR css_bootstrap_responsive_css
-			addScript $ StaticR jquery_full_js
-			addScript $ StaticR js_bootstrap_dropdown_js
-			addScript $ StaticR js_bootstrap_modal_js
-			$(widgetFile "page")
-		hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
+data NavBarItem = DashBoard | Config | About
+	deriving (Eq)
 
+navBarName :: NavBarItem -> Text
+navBarName DashBoard = "Dashboard"
+navBarName Config = "Configuration"
+navBarName About = "About"
+
+navBarRoute :: NavBarItem -> Route WebApp
+navBarRoute DashBoard = HomeR
+navBarRoute Config = ConfigR
+navBarRoute About = AboutR
+
+navBar :: Maybe NavBarItem -> [(Text, Route WebApp, Bool)]
+navBar r = map details [DashBoard, Config, About]
+	where
+		details i = (navBarName i, navBarRoute i, Just i == r)
+
+{- Used instead of defaultContent; highlights the current page if it's
+ - on the navbar. -}
+bootstrap :: Maybe NavBarItem -> Widget -> Handler RepHtml
+bootstrap navbaritem content = do
+	webapp <- getYesod
+	page <- widgetToPageContent $ do
+		addStylesheet $ StaticR css_bootstrap_css
+		addStylesheet $ StaticR css_bootstrap_responsive_css
+		addScript $ StaticR jquery_full_js
+		addScript $ StaticR js_bootstrap_dropdown_js
+		addScript $ StaticR js_bootstrap_modal_js
+		let navbar = navBar navbaritem
+		$(widgetFile "page")
+	hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
+
+instance Yesod WebApp where
 	{- Require an auth token be set when accessing any (non-static route) -}
 	isAuthorized _ _ = checkAuthToken secretToken
 
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index e3f0275d9a..47a9b687e6 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -44,13 +44,12 @@ introDisplay ident = do
 		counter = map show ([1..] :: [Int])
 
 getConfigR :: Handler RepHtml
-getConfigR = defaultLayout $ do
+getConfigR = bootstrap (Just Config) $ do
 	sideBarDisplay
 	setTitle "Configuration"
-	[whamlet|<a href="@{HomeR}">main|]
+	$(widgetFile "configurators/main")
 
 getAddRepositoryR :: Handler RepHtml
-getAddRepositoryR = defaultLayout $ do
+getAddRepositoryR = bootstrap (Just Config) $ do
 	sideBarDisplay
 	setTitle "Add repository"
-	[whamlet|<a href="@{HomeR}">main|]
diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs
index 2961dabd35..f80fb87878 100644
--- a/Assistant/WebApp/DashBoard.hs
+++ b/Assistant/WebApp/DashBoard.hs
@@ -72,18 +72,17 @@ dashboard warnNoScript = do
 	$(widgetFile "dashboard/main")
 
 getHomeR :: Handler RepHtml
-getHomeR = defaultLayout $ dashboard True
+getHomeR = bootstrap (Just DashBoard) $ dashboard True
+
+{- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
+getNoScriptR :: Handler RepHtml
+getNoScriptR = bootstrap (Just DashBoard)  $ dashboard False
 
 {- Same as HomeR, except with autorefreshing via meta refresh. -}
 getNoScriptAutoR :: Handler RepHtml
-getNoScriptAutoR = defaultLayout $ do
+getNoScriptAutoR = bootstrap (Just DashBoard) $ do
 	let ident = NoScriptR
 	let delayseconds = 3 :: Int
 	let this = NoScriptAutoR
 	toWidgetHead $(hamletFile $ hamletTemplate "dashboard/metarefresh")
 	dashboard False
-
-{- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
-getNoScriptR :: Handler RepHtml
-getNoScriptR = defaultLayout $
-	dashboard False
diff --git a/Assistant/WebApp/Documentation.hs b/Assistant/WebApp/Documentation.hs
new file mode 100644
index 0000000000..b0a9e4d98c
--- /dev/null
+++ b/Assistant/WebApp/Documentation.hs
@@ -0,0 +1,22 @@
+{- git-annex assistant webapp documentation
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+
+module Assistant.WebApp.Documentation where
+
+import Assistant.WebApp
+import Assistant.WebApp.SideBar
+import Utility.Yesod
+
+import Yesod
+
+getAboutR :: Handler RepHtml
+getAboutR = bootstrap (Just About) $ do
+	sideBarDisplay
+	setTitle "About git-annex"
+	$(widgetFile "documentation/about")
diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes
index 75f1ad7c78..5a1550b240 100644
--- a/Assistant/WebApp/routes
+++ b/Assistant/WebApp/routes
@@ -3,6 +3,7 @@
 /noscriptauto NoScriptAutoR GET
 /config ConfigR GET
 /addrepository AddRepositoryR GET
+/about AboutR GET
 
 /transfers/#NotificationId TransfersR GET
 /sidebar/#NotificationId SideBarR GET
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
index 13aefd486a..cf686f8433 100644
--- a/templates/bootstrap.hamlet
+++ b/templates/bootstrap.hamlet
@@ -7,3 +7,4 @@ $doctype 5
     ^{pageHead page}
   <body>
     ^{pageBody page}
+    <div #modal></div>
diff --git a/templates/configurators/main.hamlet b/templates/configurators/main.hamlet
new file mode 100644
index 0000000000..150e08981a
--- /dev/null
+++ b/templates/configurators/main.hamlet
@@ -0,0 +1,3 @@
+<div .span9 .hero-unit>
+  <h2>
+    Sorry, no configuration is implemented yet...
diff --git a/templates/documentation/about.hamlet b/templates/documentation/about.hamlet
new file mode 100644
index 0000000000..6236fb22cf
--- /dev/null
+++ b/templates/documentation/about.hamlet
@@ -0,0 +1,16 @@
+<div .span9 .hero-unit>
+  <h2>
+    git-annex watches over your files
+  <p>
+    It will automatically notice changes, and keep files in sync between #
+    repositories and devices.
+  <p>
+    For full details, see #
+    <a href="http://git-annex.branchable.com/">the git-annex website</a>.
+  <hr>
+    git-annex is © 2010-2012 Joey Hess. It is free software, licensed #
+    under the terms of the GNU General Public License, version 3 or above. #
+    Its development was made possible by #
+    <a href="http://git-annex.branchable.com/design/assistant/thanks/">
+      many excellent people
+    .
diff --git a/templates/page.hamlet b/templates/page.hamlet
index 67b19aaf71..d9fee6eafc 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -4,10 +4,9 @@
       <a .brand href="#">
         git-annex
       <ul .nav>
-        <li .active>
-          <a href="#">Dashboard</a>
-        <li>
-          <a href="@{ConfigR}">Config</a>
+        $forall (name, route, isactive) <- navbar
+          <li :isactive:.active>
+            <a href="@{route}">#{name}</a>
       <ul .nav .pull-right>
         <li .dropdown #menu1>
           <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
@@ -21,4 +20,3 @@
 <div .container-fluid>
   <div .row-fluid>
     ^{content}
-<div #modal></div>

From c77411053b1a4cbe865c1da4db70f1be2cbf5346 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 02:33:51 -0400
Subject: [PATCH 4394/8313] remove clickability on brand

---
 templates/page.hamlet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/page.hamlet b/templates/page.hamlet
index d9fee6eafc..8a2df1e4ba 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -1,7 +1,7 @@
 <div .navbar .navbar-fixed-top>
   <div .navbar-inner>
     <div .container>
-      <a .brand href="#">
+      <a .brand>
         git-annex
       <ul .nav>
         $forall (name, route, isactive) <- navbar

From 02b345249b06ec8e03c261a5574401ca8eaa530e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 02:35:01 -0400
Subject: [PATCH 4395/8313] stub

---
 Assistant/WebApp/Configurators.hs            | 1 +
 templates/configurators/addrepository.hamlet | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 templates/configurators/addrepository.hamlet

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 47a9b687e6..0930741e28 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -53,3 +53,4 @@ getAddRepositoryR :: Handler RepHtml
 getAddRepositoryR = bootstrap (Just Config) $ do
 	sideBarDisplay
 	setTitle "Add repository"
+	$(widgetFile "configurators/addrepository")
diff --git a/templates/configurators/addrepository.hamlet b/templates/configurators/addrepository.hamlet
new file mode 100644
index 0000000000..150e08981a
--- /dev/null
+++ b/templates/configurators/addrepository.hamlet
@@ -0,0 +1,3 @@
+<div .span9 .hero-unit>
+  <h2>
+    Sorry, no configuration is implemented yet...

From f0a88e120367fb68f36e316361c14639c338f8c3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 02:36:18 -0400
Subject: [PATCH 4396/8313] change url

---
 Assistant/WebApp/routes | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes
index 5a1550b240..69e6078b02 100644
--- a/Assistant/WebApp/routes
+++ b/Assistant/WebApp/routes
@@ -2,7 +2,7 @@
 /noscript NoScriptR GET
 /noscriptauto NoScriptAutoR GET
 /config ConfigR GET
-/addrepository AddRepositoryR GET
+/config/addrepository AddRepositoryR GET
 /about AboutR GET
 
 /transfers/#NotificationId TransfersR GET

From e9d9d9d5ea36c9d20913470079db9ea8ac0db994 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 03:10:16 -0400
Subject: [PATCH 4397/8313] add icons

---
 Assistant/Alert.hs                        |  16 ++++------------
 Assistant/WebApp/SideBar.hs               |   7 ++++---
 static/img/glyphicons-halflings-white.png | Bin 0 -> 4352 bytes
 static/img/glyphicons-halflings.png       | Bin 0 -> 4352 bytes
 templates/documentation/about.hamlet      |   3 ++-
 templates/sidebar/alert.hamlet            |   4 ++++
 6 files changed, 14 insertions(+), 16 deletions(-)
 create mode 100644 static/img/glyphicons-halflings-white.png
 create mode 100644 static/img/glyphicons-halflings.png

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 9a0bba8ae5..0412dfe519 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -34,6 +34,7 @@ data Alert = Alert
 	, alertBlockDisplay :: Bool
 	, alertClosable :: Bool
 	, alertPriority :: AlertPriority
+	, alertIcon :: Maybe String
 	}
 
 type AlertPair = (AlertId, Alert)
@@ -108,26 +109,15 @@ makeAlertFiller success alert
 	| otherwise = alert
 		{ alertClass = if c == Activity then c' else c
 		, alertPriority = Filler
-		, alertHeader = finished <$> h
-		, alertMessage = massage m
 		, alertClosable = True
+		, alertIcon = Just $ if success then "ok" else "exclamation-sign"
 		}
 	where
-		h = alertHeader alert
-		m = alertMessage alert
 		c = alertClass alert
 		c'
 			| success = Success
 			| otherwise = Error
 
-		massage (WidgetAlert w) = WidgetAlert w -- renders old on its own
-		massage (StringAlert s) = StringAlert $
-			maybe (finished s) (const s) h
-
-		finished s
-			| success = s ++ ": Ok"
-			| otherwise = s ++ ": Failed"
-
 isFiller :: Alert -> Bool
 isFiller alert = alertPriority alert == Filler
 
@@ -163,6 +153,7 @@ baseActivityAlert = Alert
 	, alertBlockDisplay = False
 	, alertClosable = False
 	, alertPriority = Medium
+	, alertIcon = Just "refresh"
 	}
 
 activityAlert :: Maybe String -> String -> Alert
@@ -220,4 +211,5 @@ sanityCheckFixAlert msg = Alert
 	, alertBlockDisplay = True
 	, alertPriority = High
 	, alertClosable = True
+	, alertIcon = Just "exclamation-sign"
 	}
diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs
index 2c630324c4..a4b8378979 100644
--- a/Assistant/WebApp/SideBar.hs
+++ b/Assistant/WebApp/SideBar.hs
@@ -49,15 +49,16 @@ sideBarDisplay = do
 			(alertBlockDisplay alert)
 			(bootstrapclass $ alertClass alert)
 			(alertHeader alert)
+			(alertIcon alert)
 			$ case alertMessage alert of
 				StringAlert s -> [whamlet|#{s}|]
 				WidgetAlert w -> w alert
 
 		rendermessage msg = addalert firstAlertId True False
-			"alert-info" Nothing [whamlet|#{msg}|]
+			"alert-info" Nothing (Just "exclamation-sign") [whamlet|#{msg}|]
 
-		addalert :: AlertId -> Bool -> Bool -> Text -> Maybe String -> Widget -> Widget
-		addalert i closable block divclass heading widget = do
+		addalert :: AlertId -> Bool -> Bool -> Text -> Maybe String -> Maybe String -> Widget -> Widget
+		addalert i closable block divclass heading icon widget = do
 			let alertid = show i
 			let closealert = CloseAlert i
 			$(widgetFile "sidebar/alert")
diff --git a/static/img/glyphicons-halflings-white.png b/static/img/glyphicons-halflings-white.png
new file mode 100644
index 0000000000000000000000000000000000000000..a20760bfde58d1c92cee95116059fba03c68d689
GIT binary patch
literal 4352
zcmd6r_dnEu|G?izMxtxU%uI5!l8nr<BF?zWUS(u;&WdwZC0F)1B-!J<$%*WB$U3Xi
z$ta3LaXK6#>)ZF&&*%FGe4jtO*5mbhJzhV&et11z&&^B?xH$MZ007{+ZK!Jj01(PQ
zJBFS4pH$0DefCd1HM@h*JNkcsi%oOXzj>qsEle$eQ7ApHL(XYdn5Y$Lk_3-J9p9d)
zFeVfl3J47_g1XaoDXWsnBp9ZzZ74CI9RN-Nw{>+8A&#rBpZgc9WX2H3Ssv6doZP?t
zS!g}lGvW1<9%?dj_G_x}3WUMN(8(x{a6_pd0yiUsf^67GGS50uSB*ORe5x6}qAf1z
z@Q;2y4G{Lb?f21p)uTpChN&4q%^blZ2IsusUOhk)pe0<chGtjyTP-b6%vl?4F2xqG
zOU>yxPD6oHKXWSj<y;3B&r^tK>v8&2pMdnegiQUtoXt1U0MmWAWu2&>3j$eb^qKNV
z_(`JQZP&mXLT@U%-2rPy!7r|*Y1oAdlarltaUyq+yq^|d{B9_>t@Rd#@_KW9w_6P$
z^Dv8(Hi8pDJK{r0Iqq*va$cL=isZh0=1)wIoQ^vYPs$<T2#x2Kj^?$few0Pe4I~zZ
zeAYbg0c0)2OtIx}d)C`Mw&~<64nQ!Uk8$^SW6e!?j1HfU4$&%i_`y~2R>(rBz$+DY
z`y}1}`M%-da686<lVV-dk8h2*Tn8V7;-njKI(p4zUJy$ofY$z#INdRf(>`}zw_w>8
z!BcqxVTim*F)-}$segV$ON*!Zl~dhX@Rz^K2Xu<c1P8u4bp<yQO?OQj^dKZcE}xh_
z<z&gNJz{ZTTu3nGIcR;qG9;?^M0kG|PuThGH1+;j!xXDN6I_*@xL=@r$xRBuVh{MN
zIUGEgxYJ(DFHKoLGF3_xPSW_^TT*1w(&gCNFdnv^AMnNFK6+ia>rh<1-vjImult%O
z!-WXvkA_agVuhluW};J;#r>)?^uHS;G?a?j;(z?Y^FTwOA?tzLFvQDf&X8}9s7Wh<
znEfd_vPyF_V`?>kR`w_h@+%59oKa;NPVGUo52QjisO-|$cYE(VNmm#+`#T5a;gh|Z
z8A0^l3UwQMn0J3x<h`4-5?ApmemDp`8K)X6T0efPN*-~cf<tL>XWL7tY~Ox<iRkdJ
zU|072zio5s?pAI0%Yx0uJh1f5i7VKWaFIaB;45=yji!1nH9<de2OLj_y{&41?nyPO
zUrZT8xW#w*TQ5)($;JeSp2Pgrams&!r<Pe}#(LDg-blL{ESlmQ?a5Th4_;WRJR+4E
zw6tQreDz+4bser4GB#?<roQ`hsw<hwcyHa9dkP0IO=6)WWkTxg{$NTm-b*c?j2_ul
zyuRy=77P?tF`%S2aa=XEJa>Au=_hGvp@_%SZKA)ec-h-dfwIhS3jGBLL6e6Os;1LR
zRDG&3TF`HV*n{&*H!oTSsLq!U5xV5!Yr6I_!*VhmwC3a2BOYfWH13AtVY|n5jv49e
zcb0xCCZnt0i$>-S$k9J@-c!8wG#siu(Lg<MtkAtqhD8bV`jR^%b&>y_r1nfy+}!<h
zAF+SdUhcuD`9zF%pRIHymB_I~)P%%~M=eQ#Ic#<Zr+NPzGTI`9;4khM^2h2PqMd?5
zGH>W9g-ucwp=&Hs1=Vs4i_q;dQL$8~Uq2BVA4o4uY!6}S`xH(Qec+{mJD~qgg@6W8
zipi@Z!ZR+Kr_)u&G);pG$tg$8#KPrsl&N3(m($NAU&9ogH9rVfW<4Mw>^7$&96g<9
zHQzekG9T5SS7DVm7EFY%CjChhfRyap4+d;+^0ng^B)~xKFG^7d2oOo|R8uY&S|X0@
znAGMb^rFQwGPTzsFQ8ZK4S@WO(8`6T+$Yt9{jGMd?jrTeb|_!Un`n9xDZu-fW+_aJ
z4Uyy_$)`Ot!~doWUHW`(?F!iYvc5+g-(W9X<-tX*h%6(f;+A(OQ@w{WYSiq&pjKnN
z)tSH~5g)03sKk)U+&GyP*?86fusX1ttpH1ng8ruC6UOddM~t>0wvZh}1cW%&7{tT$
zze(TwkA~V|_~nL{6YE#^RUC__Mx26zo*w(EfK2Q@R6xo`VkJKs^Eax`&*O*bw~*ap
zyaqA_p(~(POY{H5+NIgewtB{|(%ML_wR8o);^XGTQ|{*J>74v>{_iyU;U*NTN}A%`
z`8ltg(&furYlb!j%1ra!KPSiG<VRTwPDN9f5*7>mJ>f4c!bkAtjb_qmQ+aVB(QohO
zRo@%)1krVtMPgkT6&3T*u`XO8pE&-!!u((3qVnraj|gN5aDxvqtrPs*MCZcO3i^Qt
zI7$&BFr)50exhv11)82?u`ab0FgUSw;dpbnAtmz4k^&Nx`xMQ$5(JW}ry%)ry+DV>
zS)TWjtXz7V6iK5$ghFuPiT>;;fAp)oy%%7grs4UwqU5+Ms96%`wU=YU5W-UGw(6iq
z2GhB=Zw49;Yu<#7=soc@tZvYFIVNfkRPsCT&;76cYOONM<!9yYT8XS_j|<f)GAw6X
z_w&Wq9xu5;px-$u*_Z^YS22HQpD*L|Z1fb)`d&qCQ^smb{5_5>wv!v*e#(X?l7eB-
z&pWvVcaO;IKDg7C8bZ-+Hm`g>n_WC6%BL=CZlc``M{0T;%eYQ4t}V%m20okR=HET)
z@)@WU_}tJOqiH7w2K%l<a?3NQ^6bZPnFJ<Mk`|jLP2*o$M^nx2160!F+h^quABnz;
zAF6)v=cSvmebPJaPi4k%(nh|zGG@U(va!x`)nhbzOU0MkhuA%7v6ZH!EaE%H>pe0P
z^FhhCX$ufUPCq4?C1A8ZSrVz=$~!VZ>;=kb8eaI;S1TKb|E9j*muthJe2||9pYYI$
zR@lkEo?K76^_v{llrL+?Swi1koJYJqG_-g!v?$ITb=q4#Rk--)f<yZAd%OCYe=RDW
z4aV9=2rZm-rEPrLKA|1kuMv{%I=`DA(f6L;GQJ=_TAoYWBDl;}XZ0E+YfGjvp>ABD
zh4Ibu7+f~5HEzy@7xo<qj_3c_D9C_vmh4{K98*=04-QLt1~2F@dBZe-l2GMsk#;A`
zYHOcLf#^)Gn+{G3Q4YowOIK^&zQ|LTx89&c{IWvimdkFT8nJ{0X1}p;P(C>P^f$=}
z+D3gYZ3W>%>m=U)p#UNOPPd&2cD&<J9<&QiV~vk5R%jVK^J1%HQ}`fxWs9c=2}L>;
zxb{vXTzpCjcJAOEA_~=RX^_BM+_BYW*T{zzM(3TosvFOmf6Kp0IerP4`MuBgFdrkZ
zf9X~m0O$toCckMn8klZDxWKr2%FHNk1VLQE)$!{Hz9{*a@TaZjC7kKsC1dIUx*6AQ
zJFZc8p~!CewW(VvE@yaTPFt-6n+dZ@TM582m7=-#9JoDOH#zYPe{)-Lza89t+w#Zd
zvQ3k$)Q)mPF)g)_+v$Gqgq~*RwGeBn{vhp!IPgkixW8WY)H`S{&~om!keO$Sum=oY
zTatGW#*O^aVU<^!#et91z~$IYa;_C@J7+V)`<1b_lh`8FHOAgc=Az}lf)k%5xTMrv
zr6uV%eKaU~wvi7pU)MeB7<DU@<PM)Ua&x<*j67UgrpKP|!tXx2R%YzH<LQn0XK>HK
z2D;27Dik%)-q@hK-!I|N(cl`lAF^EIv0C-t$d1qtFnKIkcMW<4b%Lzf3Y+~~qB7`<
zj);HTQS0Oex%zA170>?kRVA_m_*O?rZRpS3v{+O+cifN7Eb&>$Z==vGKh1V)C`qGu
z_u8y<#N3Wp&$V^@T??GnE&RN^IyXM)r0h(gS3;b2pt0O!eNIt4{;3H~V5Ln7vs>8{
ziqqZL4Nwlvj4CtEv0>;Fw~D>LB_+-ecI)tiR%a!^GI3BawvNQGz4#b|_d<K40`zom
zmZ%w0mYHcNzK(Ivg#;79zJA3Qs(emYQh|-Y&A>f&`e||2k;K}WnvU!Dx=0#ue(=U#
zK&pYNNf5RQZOveUm+;dQ*FIA0&#`?@z*bBhUgr(n9_FpoHPB2pI8iMpW|sF*D{+75
z-k;nba~m^}=b7P$<BGu%3I<`>FAF1)S!oDKtNG-`%h{XQi6=SMH5GZ%8j?ugqt~!K
zw<hNaHlewKU9pKh0n@^4X=DQ<4~UnDj4@h3>vA_m(*=EI<IgUo)z0l9R@mb|@QOas
zWU>ssFVW0EZ;o=u#R5gBB$CUL+->U32;2PM2O(drij20XBy|hH+=bu!0*KIKBj%c+
z^{)B`3$NB2yp-IHf02C#Fw!(;S&rR%2P<?W3i)a{Hv71$$mqNwIwWJTc5XCVCY(ZM
zZEUT%{m1IMAyv+ZxJdeiWsFSau%`*Ji4gu)?i`XAkA6AeCLD>q(!<`Q=u&+_V4eCe
z?!d0m@n<F6bnzf#{rI&DDtbzb{#Q?q`iI`Fv^=Q#{GVsrKi@5H!=Yk{`KU+uXc?t@
zxGi_IMbsNpVL63R9MI#c?&2tT**S1&xk6UXV{?VrG2Vb8uwy$l2i~-P)jArRJvd+p
zAMPX_jhyzm3a}Qc-9M){f2vD<`B3X9uKLW{DLodF&IsV$kXKT%@Qtp6|3s@S0+S=%
zV+#X9n<D=<XzlauBx&tS1|?-doY!<IKSZPJ`vt2XRD)VP6|a+O3xDEZOZR$X3e5-S
zuOL@^Te?HwRm63Ch16HfZ|^W=1@ax6$xAQ(4$2J*D69D!1&Ss_Wp=KanXxf%3)jB=
zyl{(zRa6B4dz*qTVGFnQ#lf#G^~(Orm6*fvz@t#mixM85R=piy5ZZ)?<t2uZj~#Q1
z%87M&!_4Xmtg&aKmcnz`(+k~CS_9jg?1HcPF4&*jQGA1B5O}@9G995LTJuL|d}-c#
zRi6~5UoNF~Ng3*RH>dhMu%QZ`ERBCD+uU~%h<WLJg$(5L-k}}ce*Ymz9%AWcG8~o)
zrgMWKP5N71i-Vz&u9fBxjTT}~QT7=y$EdDt>>+E^Qd;Cz=IlGV(IwUrOz(+1Gkd7O
z$HME|^+mAGBc4k(2jEj5$g30r-BUoK@Nn!*Td)5USoe+IZ-x9)#yd)sD}2Z?2{4@)
zb|)xsK&pqOpB;+H#gbf^Pto29M<2Y>dU5pAF4p{+j=oBZ$2EXA*xI~AM@g20H7o_x
z{2-Kc;SRpcxLXzU)a53ZoX%ndB^i8=>Sf&{i6CYkGSkvLj0<@C-!VKm#iX8dws__S
zKp`T~rIAfaogJ!tV(~rs5)ctD#A};YXgPNI`<5=nWQjnIf<=1Pzn2y$C8yUkFKhwM
z@%Ah?L`DM^@d<2evu->Oo=SVaiR<1GjYwe^G2)XY`l$Q%4H`|PpFA($N_8=6uOr0s
zj+)C5x<cICx<i}#5D8LZ3LNFG7uU}%Q5<kbowYRV6Bs|^frDu{l2XM2Lj-Yh_!|?f
z+a6@mRKb9j3p<Zh$+a4#UQQYhPF@-a9mWMpS)m;R6VEWV!i;mbS?{`eur*GS8_tX$
jEfLfZC2@~9k9g`Sv9u1yERTOL1mL!wsczLx=g0p8M%V6I

literal 0
HcmV?d00001

diff --git a/static/img/glyphicons-halflings.png b/static/img/glyphicons-halflings.png
new file mode 100644
index 0000000000000000000000000000000000000000..92d4445dfd0af414835467132cf50c7c38a303af
GIT binary patch
literal 4352
zcmc(jSvb@I*TDbCl4#mw&6;FnOR{7wTf|sqB70@Y*4Sf=t&%DGGDxy7<4<NH`!>in
zwn`&QQOr<`27|~lU*GNfe)r$+;%v`3=Q$VW;ymZMrG+ssw-7e~0K7L%46Ffwh5XNs
z<6`?KHS^P-{ZmgZZ@~?jOs2~JH%~nY@PG5j1zTI#0Amn(L8qe2oETm=+B^jogFL!D
zS!ISRHW3ybWQ6o&?2=byQi)JhfBSH9PzL~<0B#!S!^50cUq25lRnLyYPq06zWw>~J
z`$KJG?wJet%MCZ1y81U)c?UzG;{mBi?no2aAHvt8L__Xy66K$DAupSD_4^VSeG;vA
zGhrY7dmCA}Zg<=d*dvUYvYMo40k!iu>o|-n)q^ld6Q(6yBtUWr1GY<4vK2?uoeS|r
zT(a}}&NC3;#Lv8{0Y$f=#j|95fZYUrx?foCUQ)KvUf$-LSb+6D%%)z#|1KO+ZTgw~
zNbE_n|4p~xYoc$edOQF-XOS;%<r!#dmHF{5#RTzN2!T(FFMc;x+SmC=Km>evzdNi3
zk@(r9h#R5FpacG)j3VDRRz>g49u-o5A=@X`M=nQQ@W&MqFu3+}8)vIJ<N(sT_Zk8X
zMLcU+@C0(GanqcI+g=S0kMDE|mIlGu&c=CozOm!OJr0PRi4D`Obo#+t^;Xncwa7ai
zdh<9v@cF*U;YbI{iHB@gJAiGAx>yezf?(vDF#3iq72Yg1rU0$uCw``L1fzH6tU=MT
zJ)FP#7~BMLoosB<>)Y`BnyxN?%PW`qwa_nrmk;P<^+|3lA$<ii8%uY~81lp=vs^qj
zbHc@jPjA%v+=Qq|+H=tEesx$(-?7Mn9R7&2+ZkE|b0%rF{pOe&2PywxrLyZ}ob?{?
zS!X14D0!!&R7j4z&a_ZX7E96o-d9#)-rDWl);JNdUKsSn^M1g(Sofin)_^D2A6fsk
zK%N&KDHIEtjz^2e+RnzNElc?mvW&i)!L?OOITL^U!tz7HAYtpl+few+Se~$Nk0>cC
z!KnRdI-*8rENgl-h*t3^hviocbR?_BCX&(%?-)#H*`RRAUES@w^(0ey@bvFIq^EE0
zYIYPpa4Xz>{9(cUIq~=IuByDHtJskc@OXkoyhOvqjT$BRxhihe#hq<$(TaV?g(bYx
zzk*$b_y4xdrKd-u!#@W)7x%!%FE62JOZu)fTpnAUKW94KXQKo9lR9BoI`nN#BV<pu
zN$Y&tINUw4JJ60wNhX=$oO%xnS0~8-36@e}lO69__j)7adZ<L2U#u5vwGeo2shUYF
zH*Rf1efjD`jdvTV8a6X+&!xQkbfP<z!gz1jlz##Puuh>NL^WLc-2PBnDb`!FkQ6Yw
zt8#VMCqN`vOx>8A-pqa3!sg7$vF4w|C29%3h5O_{d+D-|gED!U;S&A}5QU_Uz%?vp
zmMBIPvj7qQQG74PJJYIU8KAgcJcJvNO0O6=%8w|@chXvpUX6O34cERMj)m?X)jwit
zWYksusgx8zcrOv1Kd4Cm%yUoW#?wfM-ee=?*pXt7dU<wL(ECgNtn7KAcQSgjF$wec
z&wNDxS$nw$r4-^(dj0lt)f7DU5?+TTQ7UFh7R=*xI5;Wr=aA7JB?^0SzgQ^V;4r`?
zBrN-yp=!hwMFq0PE?Y!UWLSr9S4c>vyZrhI*Zx3!VQzm2&D<yRh#LVfjXeOPcj~wR
z5UG;7Ix04MSLbA=`nZloXfY{`*@7=#K0}`VWppv~RO%H}$!V*DNHvZFBHUqfI?CD0
zbx!B4^9`#pqXl&iB{Pv^*lNn33}KeCeaE4o?M=ZBEL9o|=KG==a{vbsI4@1LO3@|!
z=#F_^eo|k6WLCD`I?D^lB}ZeRa3j2$+MNG{fZN~d@a`$my6AxPBfp8Irx1gDj_8y_
zB{|_Ko-%@Zv$H~Px&z9e&#zqq`(8HmN?{uv#cDixDOJ%G_;k$j?o_(Za8|9R0~pd~
zhP6EvoFeWkI{=X$R-d0BaUhyb8w0in4s%stxoODXOl;El?W^#yR`?d86Ax#>k2i(z
zv;J?=_W|Z`2Nb*9*m`XJ^1ixr>GY^eNXXM8UzHKbJ%`E&g=n<QM>C-&t%U{b2>k}4
zM^eC8z9@VJ)NO6~zgW94x7psn_*GsP&AXPV>|c7+3V*`GDl?NuNHOr8_5jSBY+FrJ
zxxFy&omakmacj-wPLUexLeI~s2^i^7j<QS1^o69wqChX$OY6u{tW}exT*h=kf_UuP
z+XaMs<6dAuy-kT^H%eXIYHYk4Q!FTjJ*&L%*Q})SYV#u;NgCV`gwN=QJ~!7t_q2+B
zpbd_ZMR9D%dyk)}nec)ZXV~q^?S+kxZJj&X5@|w?zO<x_02M#Q3a*5JM8Y&n;d~#^
zX?>diy$lDh;U-ze^bf8Wq&_j48xx9sRj~I0?AI|l`&NRKa0xj_M7{QQP8x>W$llZ#
z^2}mA)Bep^+iA@Qw-LK1wT3nbnW#j??18HOX9M~EwO_4MW54*U(nB|yBja(g7FnMC
zblZNR)Y{`EcNWNZ9&#=!$@W#;-?`_@7{fb;%BTG<Be%)pb!CB`M;1FsO>aNt!jg%h
zP{`+<{G!`T5|=OLq>Z*{Z2O&8zMn16ACVB$Qm``DYk?tjJdb2uC7aci<-`J?E%OU+
zGrN5UtA#%|w#4Z;NP?k$>n!<|SrjF%qnK<QD=|fvQ-`MgFRin=cJ~1?W$Nv-%7>36
z-X#tb9{hRfZswTsPVZBN8H~75sHKLYIz~6u+pKzy#crwlQTpM#$E~+Abk)TD#sz#v
zXX8Go`ZaF>B8Zu%M9U<U?k5{O<y&QE7KlDa9?QUr-S}#I$LYUm81UoWaH<><;>RXE
zbfFb@39Y9#&~E%DMKl*GIPjFwcNZ7nuMbVEpA0WbvBjM9QA!sp{YiDoe131&NawG0
z)w7{^`zTTBX*b%&r|n~U@dMgnxo!))g;D+Qg=`Xw5@VHk^{hiH?Dbc#u;gsXHzn0i
z2)8o6*&Kl>6tpGG-xYv<M}QNBKQ@Z8AUtKe=QqbSl$Amw(w@PJ2Fl*B4kD#B1X|@h
zND6v-C(>B-r`9coW<<#c<0|E=wQpY(XerrkkfVOt!t*N?wvbI|9F@&~JQ7q2jXe2H
zCW^MvkWX8I-=%fo@BdI{A^py@pAB`s<yjfB3(la;jxJW|8b9;qtmahyAaiMpzZU^P
zxD>hd&A{*amKE*X!a7A2Yu?Z%f;af$36@t#hgGI$UAqZQr>(vfUM3&C0L=d07kpTV
z65hXXqa6SYLUvQ%beIm#w8HN~d3!4?$?iB2Owr|ut8l>>rMSqaZB}JGncrpN>H)eX
z?`{XC$$(nou>9J>y&RJ_GCHrPS%%Jr+GeZ-p;^lV`1YLmyxKN-u#7+}dnx}N%zgXH
z$CV1rQyi4eN)t(4&9Ix9{_jMeW*4;LYis@>9EQ2Es^gfy-VKyn0lc8i{7q3yuQV}F
zD6Fom;2?qz@ukzYpge~g8?BAWbC}{;E82F=WrGc<q3x&8B^qmty_aky%YQ{CKTGq<
zYP!kE(69SylMU^oAELgld(|`QSIDWIzU`!z4rh5Kn4EmCqu{yU{SIwx=mqDK8w<~1
zUPFy^`6*;La<HSR(O`c-+NrbAEnz=wxz=;F=D$%Gr5~UQ*wG%^^eW0ENv91u_qR-a
z$S)u&@oi_Fi#yBZUxTms_h&AvlAOS|`l_14f97W-V>0;?er)DQ&9VG84bSn{>9B(k
zwM%!e%*jQ~?@0DuS;yYC#^~O_E+}d7VN;GP%ockmCFlj4DNZ%yl_X-Hn$v_=+Er1z
z)xF^ugN@xFweaki3bVXB3?uwjsn55R<b|OgeId~Hv@}>D1&YMi6B+jBAEU6|0Y1ne
zLxbyOnkM9BHX2f}bHa<7WG>P_pz=aP(B)D(uo1i&yvId9DaA3GTsK?WdG%g5Q5z-%
zUfT;wH`Xu@LDvM>F<4<`LiFUdk7UO)oS&1>Rnv!81;V#S1gZ^;byAIw5fmjY3m)nw
z?+@SmlmBCWV>bFM8|-jGB{WLeI3o9DaWo<)11@8`kh*v=cN0DNB+st4sz6R#2I0qi
z4c&8ZcAexDoiEyzoZJ((D9)8bG%^Z+MCs@_Q)++#Uvn&7#CI<7^ioFM{2qLTEAfMX
z#1kD>oACS6EsTK8F}{R&pahvhyt|}$lX5-EzVP=!*jL*U(=7^7%UUF#`g>m(9)4uh
zN+-O*&B&PgYQ520)x+!;$#)PXM`Kgq-o1CQLPsDGuSVi?k7|gIEtmv^WewHMkLAio
zl1Us*ZM8T5*j_cED4OCIiNDZ{(dj&{3{g&T+~4Y*L((GimlI~v8Q&*2;zNurHxdEX
zDgWY5T-u#~Rw6AH53<&eUOA_3sJa+<`S@61`0Z+&gPPC(dA9xY-3vCHs+QQ8y<*H|
zq`~2~B6ACGIIhlq0<JP>$V=$vE_&HDcwxCpLD6$_1>ZT*h{SQByL1NMw0+fOj?Wz&
zFvJdbQkbJBeJ=wX#hUle7%rUXR$4yPWhM|#t(`DrC+d#^K8*!sRn%{Eee5S%bqSan
z?Gaxb6y6;Dw^4Ura3@7~UnV3ahsAZxfc!%uwqZbo@PGj7@>ji1sVn}8fiB(aiz~Jo
zTDXK*@oVh~gVo^Iu~o8PQNMj6)RalL?o3^H@pnjZNLWoX&@@;gDJHvX&C-&SZCkAF
z?Pux@B3eZQ037cWb&FZMuP+XLz1yG`s8)?SoCs!ygWlxG$PB`Eka2i37Fv)TK{|58
zJti;S=?xo)8?eTei(HD#<H{`dIBo}QZ*qye7Ch&8W=G`>f`Jq8j>vX~5NRzRU9sf_
z>oxtdr~$>ax+OJ;^X)vsSztp0JYJsoQlX{)JP`NN^%4mv6u3oW-hBTdM2W@5-Fze>
z9n9nd!<vn@x)+DSqur{lShQR5c_q20z&z9X_VI@tF-sZITiJ8(=%yDq%20jBZq4o?
zgCC2nZ#R@cyO{hJ?i_$meOX?m;pkq%(#414r`r1hpFn%A^?fTAk~P~=C0`Omj7x)=
z_=sB}!Gp5B>;qg7R6d&M#&&}CPAvA|mF^4XPltG`XZl9!t)5o^flxcEGJRDAZjOjF
zQ0Iea%DG$E3bP&!(93|2RCY3l5t3s3J*JOik0=hGeaJ@3@H8tD7<k9<<dKwp&eQ6Z
z9|U0$hb)b5lItCim6MC_Nf&^qL{S0zjAEPdi{G~l$mUBpQVcZOtKq$za5*WnwuQO{
zxF$NXUlSh-TEr%CuFbjgKX@wV^CqEZM<ObXOWagY0q?8j*FR)BnR)!IQXA#2X-7RS
zQDDqU9@ib_?%osL+z(HZl~m@gbUVL(W{K>CVRqHg&`+R3j0a8@kqB}PI}{$m!yRab
zvul5lL(>3*TF>n~)*#hsmwUTtKRAA2Fnk0PENdI!9GrZLu@zyKzs+&m-IKFviqv>&
kg1Lm#gqI~e;$iYPkmG5c&N-g{UI@TVLkokN>#mRg2V?7pi2wiq

literal 0
HcmV?d00001

diff --git a/templates/documentation/about.hamlet b/templates/documentation/about.hamlet
index 6236fb22cf..af48a0b102 100644
--- a/templates/documentation/about.hamlet
+++ b/templates/documentation/about.hamlet
@@ -10,7 +10,8 @@
   <hr>
     git-annex is © 2010-2012 Joey Hess. It is free software, licensed #
     under the terms of the GNU General Public License, version 3 or above. #
-    Its development was made possible by #
+    <br>
+    <i class="icon-heart"></i> Its development was made possible by #
     <a href="http://git-annex.branchable.com/design/assistant/thanks/">
       many excellent people
     .
diff --git a/templates/sidebar/alert.hamlet b/templates/sidebar/alert.hamlet
index 8fa77eac1d..84126c3810 100644
--- a/templates/sidebar/alert.hamlet
+++ b/templates/sidebar/alert.hamlet
@@ -1,6 +1,10 @@
 <div .alert .fade .in .#{divclass} :block:.alert-block ##{alertid} :closable:onclick="(function( $ ) { $.get('@{closealert}') })( jQuery );">
   $if closable
     <a .close data-dismiss="alert">×</a>
+  $case icon
+    $of Nothing
+    $of Just name
+      <i class="icon-#{name}"></i> #
   $maybe h <- heading
     $if block
       <h4 class="alert-heading">#{h}</h4>

From 2f446439ea4646251f261433af3e0ed7263e7134 Mon Sep 17 00:00:00 2001
From: "http://smcv.pseudorandom.co.uk/" <smcv@web>
Date: Tue, 31 Jul 2012 09:58:13 +0000
Subject: [PATCH 4398/8313] Added a comment

---
 ...mment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment

diff --git a/doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment b/doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment
new file mode 100644
index 0000000000..38916dd8c2
--- /dev/null
+++ b/doc/design/assistant/blog/day_40__dbus/comment_5_40b6b9d741d3081203f0cc94eb8dc3ea._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://smcv.pseudorandom.co.uk/"
+ nickname="smcv"
+ subject="comment 5"
+ date="2012-07-31T09:58:13Z"
+ content="""
+It's `org.gtk` because gvfs, GLib and Gtk are all products of gtk.org (there is no separate glib.org).
+
+It's `Private` because the gvfs developers don't consider its D-Bus interface to be stable API, I believe. The official API for it is the C API in GIO, part of GLib.
+
+To poll for changes to /proc/mounts on Linux, you open it for reading and poll() for POLLERR (not sure why it gets to be different, but it is). See gio/gunixmounts.c in recent GLib.
+"""]]

From ed0ea1960014fbc07aabd5c250829133817e31b8 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkstq9oH1vHXY_VP0nYO9Gg3eKnKerDGRI"
 <Hadi@web>
Date: Tue, 31 Jul 2012 15:13:07 +0000
Subject: [PATCH 4399/8313] Added a comment: ARM

---
 .../comment_3_4d922e11249627634ecc35bba4044d9e._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/install/Debian/comment_3_4d922e11249627634ecc35bba4044d9e._comment

diff --git a/doc/install/Debian/comment_3_4d922e11249627634ecc35bba4044d9e._comment b/doc/install/Debian/comment_3_4d922e11249627634ecc35bba4044d9e._comment
new file mode 100644
index 0000000000..98d65740bc
--- /dev/null
+++ b/doc/install/Debian/comment_3_4d922e11249627634ecc35bba4044d9e._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawkstq9oH1vHXY_VP0nYO9Gg3eKnKerDGRI"
+ nickname="Hadi"
+ subject="ARM"
+ date="2012-07-31T15:13:06Z"
+ content="""
+is there any package for Debian armhf? I'd love to install git-annex on my raspberry pi
+"""]]

From b9b009787662cda4948b3c9706b8897587d05d8a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 11:19:40 -0400
Subject: [PATCH 4400/8313] tweak types so the webapp can run without a
 threadstate when outside an annex

---
 Assistant/Threads/WebApp.hs       |  2 +-
 Assistant/WebApp.hs               | 12 +++++++++++-
 Assistant/WebApp/Configurators.hs |  3 +--
 Assistant/WebApp/DashBoard.hs     |  3 +--
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index ca81498f4a..9cdbae451c 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -59,7 +59,7 @@ webAppThread st dstatus transferqueue onstartup = do
 			token <- genRandomToken 
 			s <- newWebAppState
 			return $ WebApp 
-				{ threadState = st
+				{ threadState = Just st
 				, daemonStatus = dstatus
 				, transferQueue = transferqueue
 				, secretToken = pack token
diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index fc40ca5bfb..f7fb7bb6ef 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -30,7 +30,7 @@ staticFiles "static"
 mkYesodData "WebApp" $(parseRoutesFile "Assistant/WebApp/routes")
 
 data WebApp = WebApp
-	{ threadState :: ThreadState
+	{ threadState :: Maybe ThreadState
 	, daemonStatus :: DaemonStatusHandle
 	, transferQueue :: TransferQueue
 	, secretToken :: Text
@@ -104,6 +104,16 @@ modifyWebAppState a = go =<< webAppState <$> getYesod
 			v <- takeTMVar s
 			putTMVar s $ a v
 
+{- Runs an Annex action from the webapp.
+ -
+ - When the webapp is run outside a git-annex repository, the fallback
+ - value is returned.
+ -}
+runAnnex :: forall sub a. a -> Annex a -> GHandler sub WebApp a
+runAnnex fallback a = maybe (return fallback) go =<< threadState <$> getYesod
+	where
+		go st = liftIO $ runThreadState st a
+
 waitNotifier :: forall sub. (DaemonStatus -> NotificationBroadcaster) -> NotificationId -> GHandler sub WebApp ()
 waitNotifier selector nid = do
 	notifier <- getNotifier selector
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 0930741e28..ee3209ce25 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -12,7 +12,6 @@ module Assistant.WebApp.Configurators where
 import Assistant.Common
 import Assistant.WebApp
 import Assistant.WebApp.SideBar
-import Assistant.ThreadedMonad
 import Utility.Yesod
 import qualified Remote
 import Logs.Web (webUUID)
@@ -27,7 +26,7 @@ introDisplay :: Text -> Widget
 introDisplay ident = do
 	webapp <- lift getYesod
 	let reldir = relDir webapp
-	l <- liftIO $ runThreadState (threadState webapp) $ do
+	l <- lift $ runAnnex [] $ do
 		u <- getUUID
 		rs <- map Remote.uuid <$> Remote.remoteList
 		rs' <- snd <$> trustPartition DeadTrusted rs
diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs
index f80fb87878..9a9fccdaa4 100644
--- a/Assistant/WebApp/DashBoard.hs
+++ b/Assistant/WebApp/DashBoard.hs
@@ -14,7 +14,6 @@ import Assistant.WebApp
 import Assistant.WebApp.SideBar
 import Assistant.WebApp.Notifications
 import Assistant.WebApp.Configurators
-import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.TransferQueue
 import Utility.NotificationBroadcaster
@@ -35,7 +34,7 @@ import qualified Data.Map as M
 transfersDisplay :: Bool -> Widget
 transfersDisplay warnNoScript = do
 	webapp <- lift getYesod
-	current <- liftIO $ runThreadState (threadState webapp) $
+	current <- lift $ runAnnex [] $
 		M.toList . currentTransfers
 			<$> liftIO (getDaemonStatus $ daemonStatus webapp)
 	queued <- liftIO $ getTransferQueue $ transferQueue webapp

From 8102958300d475f2e430ebe4285f4587b0c1665d Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Tue, 31 Jul 2012 15:41:43 +0000
Subject: [PATCH 4401/8313] Added a comment

---
 .../comment_4_2a93ab18b05ccb90e7acc5885866fca2._comment  | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/install/Debian/comment_4_2a93ab18b05ccb90e7acc5885866fca2._comment

diff --git a/doc/install/Debian/comment_4_2a93ab18b05ccb90e7acc5885866fca2._comment b/doc/install/Debian/comment_4_2a93ab18b05ccb90e7acc5885866fca2._comment
new file mode 100644
index 0000000000..8436f0354a
--- /dev/null
+++ b/doc/install/Debian/comment_4_2a93ab18b05ccb90e7acc5885866fca2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 4"
+ date="2012-07-31T15:41:43Z"
+ content="""
+Yes, git-annex is available for every Debian architecture which supports Haskell, including all arm ports:
+
+<pre>git-annex | 3.20120629         | wheezy            | source, amd64, armel, armhf, i386, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc</pre>
+"""]]

From 04794eafc0f0fd09e645247136fe557fd80bfb55 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 12:17:31 -0400
Subject: [PATCH 4402/8313] webapp now starts up when run not in a git repo

---
 Assistant.hs                         |  4 +-
 Assistant/Threads/WebApp.hs          | 55 ++++++++++++++--------------
 Assistant/WebApp.hs                  |  2 +-
 Assistant/WebApp/Configurators.hs    |  1 -
 Command/WebApp.hs                    | 31 ++++++++++++----
 templates/bootstrap.hamlet           |  6 ++-
 templates/configurators/intro.hamlet | 51 +++++++++++++-------------
 templates/page.hamlet                | 21 ++++++-----
 8 files changed, 96 insertions(+), 75 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 22a87fe8cc..4bb85975b8 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -122,7 +122,7 @@ import Utility.ThreadScheduler
 
 import Control.Concurrent
 
-startDaemon :: Bool -> Bool -> Maybe (IO ()) -> Annex ()
+startDaemon :: Bool -> Bool -> Maybe (FilePath -> IO ()) -> Annex ()
 startDaemon assistant foreground webappwaiter
 	| foreground = do
 		showStart (if assistant then "assistant" else "watch") "."
@@ -155,7 +155,7 @@ startDaemon assistant foreground webappwaiter
 				, mountWatcherThread st dstatus scanremotes
 				, transferScannerThread st dstatus scanremotes transferqueue
 #ifdef WITH_WEBAPP
-				, webAppThread st dstatus transferqueue webappwaiter
+				, webAppThread (Just st) dstatus transferqueue webappwaiter
 #endif
 				, watchThread st dstatus transferqueue changechan
 				]
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index 9cdbae451c..ad2bff8923 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -1,4 +1,4 @@
-{- git-annex assistant webapp
+{- git-annex assistant webapp thread
  -
  - Copyright 2012 Joey Hess <joey@kitenet.net>
  -
@@ -38,47 +38,46 @@ thisThread = "WebApp"
 
 mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes")
 
-webAppThread :: ThreadState -> DaemonStatusHandle -> TransferQueue -> Maybe (IO ()) -> IO ()
-webAppThread st dstatus transferqueue onstartup = do
-	webapp <- mkWebApp
+webAppThread :: (Maybe ThreadState) -> DaemonStatusHandle -> TransferQueue -> Maybe (FilePath -> IO ()) -> IO ()
+webAppThread mst dstatus transferqueue onstartup = do
+	webapp <- WebApp
+		<$> pure mst
+		<*> pure dstatus
+		<*> pure transferqueue
+		<*> (pack <$> genRandomToken)
+		<*> getreldir mst
+		<*> pure $(embed "static")
+		<*> newWebAppState
 	app <- toWaiAppPlain webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
 		, return app
 		)
-	runWebApp app' $ \port -> do
-		runThreadState st $ writeHtmlShim webapp port
-		maybe noop id onstartup
+	runWebApp app' $ \port -> case mst of
+		Nothing -> withTempFile "webapp.html" $ \tmpfile _ -> go port webapp tmpfile
+		Just st -> go port webapp =<< runThreadState st (fromRepo gitAnnexHtmlShim)
 	where
-		mkWebApp = do
+		getreldir Nothing = return Nothing
+		getreldir (Just st) = do
 			dir <- absPath =<< runThreadState st (fromRepo repoPath)
 			home <- myHomeDir
-			let reldir = if dirContains home dir
+			return $ Just $ if dirContains home dir
 				then relPathDirToFile home dir
 				else dir
-			token <- genRandomToken 
-			s <- newWebAppState
-			return $ WebApp 
-				{ threadState = Just st
-				, daemonStatus = dstatus
-				, transferQueue = transferqueue
-				, secretToken = pack token
-				, relDir = reldir
-				, getStatic = $(embed "static")
-				, webAppState = s
-				}
+		go port webapp htmlshim = do
+			writeHtmlShim webapp port htmlshim
+			maybe noop (\a -> a htmlshim) onstartup
 
 {- Creates a html shim file that's used to redirect into the webapp,
  - to avoid exposing the secretToken when launching the web browser. -}
-writeHtmlShim :: WebApp -> PortNumber -> Annex ()
-writeHtmlShim webapp port = do
-	liftIO $ debug thisThread ["running on port", show port]
-	htmlshim <- fromRepo gitAnnexHtmlShim
-	liftIO $ viaTmp go htmlshim $ genHtmlShim webapp port
+writeHtmlShim :: WebApp -> PortNumber -> FilePath -> IO ()
+writeHtmlShim webapp port file = do
+	debug thisThread ["running on port", show port]
+	viaTmp go file $ genHtmlShim webapp port
 	where
-		go file content = do
-			h <- openFile file WriteMode
-			modifyFileMode file $ removeModes [groupReadMode, otherReadMode]
+		go tmpfile content = do
+			h <- openFile tmpfile WriteMode
+			modifyFileMode tmpfile $ removeModes [groupReadMode, otherReadMode]
 			hPutStr h content
 			hClose h
 
diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index f7fb7bb6ef..2a1fcb6b42 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -34,7 +34,7 @@ data WebApp = WebApp
 	, daemonStatus :: DaemonStatusHandle
 	, transferQueue :: TransferQueue
 	, secretToken :: Text
-	, relDir :: FilePath
+	, relDir :: Maybe FilePath
 	, getStatic :: Static
 	, webAppState :: TMVar WebAppState
 	}
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index ee3209ce25..66d92ebc04 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -25,7 +25,6 @@ import Data.Text (Text)
 introDisplay :: Text -> Widget
 introDisplay ident = do
 	webapp <- lift getYesod
-	let reldir = relDir webapp
 	l <- lift $ runAnnex [] $ do
 		u <- getUUID
 		rs <- map Remote.uuid <$> Remote.remoteList
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index ee1274f97d..6755763b32 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -10,12 +10,19 @@ module Command.WebApp where
 import Common.Annex
 import Command
 import Assistant
+import Assistant.DaemonStatus
+import Assistant.TransferQueue
+import Assistant.Threads.WebApp
 import Utility.WebApp
+import Utility.ThreadScheduler
 import Utility.Daemon (checkDaemon)
 import qualified Command.Watch
 
+import Control.Concurrent.STM
+
 def :: [Command]
-def = [withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $
+def = [oneShot $ noRepo firstRun $ dontCheck repoExists $
+	withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $
         command "webapp" paramNothing seek "launch webapp"]
 
 seek :: [CommandSeek]
@@ -30,8 +37,8 @@ start foreground stopdaemon = notBareRepo $ do
 		else do
 			f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 			ifM (checkpid <&&> checkshim f) $
-				( liftIO $ go f 
-				, startDaemon True foreground $ Just $ go f
+				( liftIO $ openBrowser f 
+				, startDaemon True foreground $ Just openBrowser
 				)
 	stop
 	where
@@ -39,7 +46,17 @@ start foreground stopdaemon = notBareRepo $ do
 			pidfile <- fromRepo gitAnnexPidFile
 			liftIO $ isJust <$> checkDaemon pidfile
 		checkshim f = liftIO $ doesFileExist f
-		go f = unlessM (runBrowser url) $
-			error $ "failed to start web browser on url " ++ url
-			where
-				url = "file://" ++ f
+
+openBrowser :: FilePath -> IO ()
+openBrowser htmlshim = unlessM (runBrowser url) $
+	error $ "failed to start web browser on url " ++ url
+	where
+		url = "file://" ++ htmlshim
+
+firstRun :: IO ()
+firstRun = do
+	dstatus <- atomically . newTMVar =<< newDaemonStatus
+	transferqueue <- newTransferQueue
+	webAppThread Nothing dstatus transferqueue $ Just $ \f -> do
+		openBrowser f
+		waitForTermination
diff --git a/templates/bootstrap.hamlet b/templates/bootstrap.hamlet
index cf686f8433..f743a0d463 100644
--- a/templates/bootstrap.hamlet
+++ b/templates/bootstrap.hamlet
@@ -1,7 +1,11 @@
 $doctype 5
 <html>
   <head>
-    <title>#{relDir webapp} #{pageTitle page}
+    <title>
+     $maybe reldir <- relDir webapp
+       #{reldir} #{pageTitle page}
+     $nothing
+       #{pageTitle page}
     <link rel="icon" href=@{StaticR favicon_ico} type="image/x-icon">
     <meta name="viewport" content="width=device-width,initial-scale=1.0">
     ^{pageHead page}
diff --git a/templates/configurators/intro.hamlet b/templates/configurators/intro.hamlet
index ecb15f39cc..5062346a85 100644
--- a/templates/configurators/intro.hamlet
+++ b/templates/configurators/intro.hamlet
@@ -1,27 +1,28 @@
 <div .span9 ##{ident} .hero-unit>
-  <h2>
-    git-annex is watching over your files in <small><tt>#{reldir}</tt></small>
-  <p>
-    It will automatically notice changes, and keep files in sync between #
-    $if notenough
-      repositories on your devices ...
-      <h2>
-        But no other repositories are set up yet.
-      <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
-    $else
-      $if barelyenough
-        <span .badge .badge-warning>#{numrepos}</span>
+  $maybe reldir <- relDir webapp
+   <h2>
+      git-annex is watching over your files in <small><tt>#{reldir}</tt></small>
+    <p>
+      It will automatically notice changes, and keep files in sync between #
+      $if notenough
+        repositories on your devices ...
+        <h2>
+          But no other repositories are set up yet.
+        <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
       $else
-        <span .badge .badge-success>#{numrepos}</span>
-      \ repositories and devices:
-      <table .table .table-striped .table-condensed>
-        <tbody>
-          $forall (num, name) <- remotelist
-            <tr>
-              <td>
-                #{num}
-              <td>
-                #{name}
-      <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
-      <p>
-        Or just sit back, watch the magic, and get on with using your files.
+        $if barelyenough
+          <span .badge .badge-warning>#{numrepos}</span>
+        $else
+          <span .badge .badge-success>#{numrepos}</span>
+        \ repositories and devices:
+        <table .table .table-striped .table-condensed>
+          <tbody>
+            $forall (num, name) <- remotelist
+              <tr>
+                <td>
+                  #{num}
+                <td>
+                  #{name}
+        <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
+        <p>
+          Or just sit back, watch the magic, and get on with using your files.
diff --git a/templates/page.hamlet b/templates/page.hamlet
index 8a2df1e4ba..29a091110a 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -7,16 +7,17 @@
         $forall (name, route, isactive) <- navbar
           <li :isactive:.active>
             <a href="@{route}">#{name}</a>
-      <ul .nav .pull-right>
-        <li .dropdown #menu1>
-          <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
-            Current Repository: #{relDir webapp}
-            <b .caret></b>
-          <ul .dropdown-menu>
-             <li><a href="#">#{relDir webapp}</a></li>
-             <li .divider></li>
-             <li><a href="@{AddRepositoryR}">Add another repository</a></li>
-
+      $maybe reldir <- relDir webapp
+        <ul .nav .pull-right>
+          <li .dropdown #menu1>
+            <a .dropdown-toggle data-toggle="dropdown" href="#menu1">
+              Current Repository: #{reldir}
+              <b .caret></b>
+            <ul .dropdown-menu>
+               <li><a href="#">#{reldir}</a></li>
+               <li .divider></li>
+               <li><a href="@{AddRepositoryR}">Add another repository</a></li>
+      $nothing
 <div .container-fluid>
   <div .row-fluid>
     ^{content}

From 1efb0d109ee425b53b76e92ee69c5ec7bf6cf979 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 12:20:31 -0400
Subject: [PATCH 4403/8313] don't start webapp when in a git repo that has not
 been git-annex initted

Maybe later it can prompt if they want to use the repo with git-annex, but
for now this is a sane easy choice.
---
 Command/WebApp.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 6755763b32..a3294911f9 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -21,7 +21,7 @@ import qualified Command.Watch
 import Control.Concurrent.STM
 
 def :: [Command]
-def = [oneShot $ noRepo firstRun $ dontCheck repoExists $
+def = [oneShot $ noRepo firstRun $
 	withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $
         command "webapp" paramNothing seek "launch webapp"]
 

From 0d3686972d9b08b061f86b3e38fb681becf1c833 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 12:33:52 -0400
Subject: [PATCH 4404/8313] form

---
 templates/configurators/intro.hamlet | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/templates/configurators/intro.hamlet b/templates/configurators/intro.hamlet
index 5062346a85..be451a91fe 100644
--- a/templates/configurators/intro.hamlet
+++ b/templates/configurators/intro.hamlet
@@ -26,3 +26,12 @@
         <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
         <p>
           Or just sit back, watch the magic, and get on with using your files.
+  $nothing
+    <h2>
+      Creating a git-annex repository
+    <p>
+      Files in this repository will managed by git-annex, #
+      and kept in sync with your repositories on other devices.
+    <form .form-inline>
+      <input type="text" .input-xlarge placeholder="directory"> #
+      <button type="submit" .btn .btn-primary .btn-large>Make Repository</button>

From c70496dc7f89f07e05bea0257b7d93986dd61d89 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 14:23:17 -0400
Subject: [PATCH 4405/8313] improve first run screen

---
 Assistant/WebApp.hs                          | 19 ++++++++++++++-----
 Assistant/WebApp/Configurators.hs            | 16 ++++++++++++----
 Assistant/WebApp/DashBoard.hs                |  7 +++++--
 templates/configurators/addrepository.hamlet | 17 +++++++++++++++--
 templates/configurators/intro.hamlet         |  9 ---------
 5 files changed, 46 insertions(+), 22 deletions(-)

diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index 2a1fcb6b42..3351aa48fe 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -52,25 +52,34 @@ navBarRoute DashBoard = HomeR
 navBarRoute Config = ConfigR
 navBarRoute About = AboutR
 
-navBar :: Maybe NavBarItem -> [(Text, Route WebApp, Bool)]
-navBar r = map details [DashBoard, Config, About]
-	where
-		details i = (navBarName i, navBarRoute i, Just i == r)
+defaultNavBar :: [NavBarItem]
+defaultNavBar = [DashBoard, Config, About]
+
+firstRunNavBar :: [NavBarItem]
+firstRunNavBar = [Config, About]
+
+selectNavBar :: Handler [NavBarItem]
+selectNavBar = ifM (inFirstRun) (return firstRunNavBar, return defaultNavBar)
+
+inFirstRun :: Handler Bool
+inFirstRun = isNothing . threadState <$> getYesod
 
 {- Used instead of defaultContent; highlights the current page if it's
  - on the navbar. -}
 bootstrap :: Maybe NavBarItem -> Widget -> Handler RepHtml
 bootstrap navbaritem content = do
 	webapp <- getYesod
+	navbar <- map navdetails <$> selectNavBar
 	page <- widgetToPageContent $ do
 		addStylesheet $ StaticR css_bootstrap_css
 		addStylesheet $ StaticR css_bootstrap_responsive_css
 		addScript $ StaticR jquery_full_js
 		addScript $ StaticR js_bootstrap_dropdown_js
 		addScript $ StaticR js_bootstrap_modal_js
-		let navbar = navBar navbaritem
 		$(widgetFile "page")
 	hamletToRepHtml $(hamletFile $ hamletTemplate "bootstrap")
+	where
+		navdetails i = (navBarName i, navBarRoute i, Just i == navbaritem)
 
 instance Yesod WebApp where
 	{- Require an auth token be set when accessing any (non-static route) -}
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 66d92ebc04..9fe10aff92 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -41,14 +41,22 @@ introDisplay ident = do
 	where
 		counter = map show ([1..] :: [Int])
 
+addRepository :: Bool -> Widget
+addRepository firstrun = do
+	setTitle $ if firstrun then "Getting started" else "Add repository"
+	$(widgetFile "configurators/addrepository")
+
 getConfigR :: Handler RepHtml
 getConfigR = bootstrap (Just Config) $ do
 	sideBarDisplay
-	setTitle "Configuration"
-	$(widgetFile "configurators/main")
+	ifM (lift inFirstRun)
+		( addRepository True
+		, do
+			setTitle "Configuration"
+			$(widgetFile "configurators/main")
+		)
 
 getAddRepositoryR :: Handler RepHtml
 getAddRepositoryR = bootstrap (Just Config) $ do
 	sideBarDisplay
-	setTitle "Add repository"
-	$(widgetFile "configurators/addrepository")
+	addRepository False
diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs
index 9a9fccdaa4..f4f56a4763 100644
--- a/Assistant/WebApp/DashBoard.hs
+++ b/Assistant/WebApp/DashBoard.hs
@@ -71,11 +71,14 @@ dashboard warnNoScript = do
 	$(widgetFile "dashboard/main")
 
 getHomeR :: Handler RepHtml
-getHomeR = bootstrap (Just DashBoard) $ dashboard True
+getHomeR = ifM (inFirstRun)
+	( redirect ConfigR
+	, bootstrap (Just DashBoard) $ dashboard True
+	)
 
 {- Same as HomeR, except no autorefresh at all (and no noscript warning). -}
 getNoScriptR :: Handler RepHtml
-getNoScriptR = bootstrap (Just DashBoard)  $ dashboard False
+getNoScriptR = bootstrap (Just DashBoard) $ dashboard False
 
 {- Same as HomeR, except with autorefreshing via meta refresh. -}
 getNoScriptAutoR :: Handler RepHtml
diff --git a/templates/configurators/addrepository.hamlet b/templates/configurators/addrepository.hamlet
index 150e08981a..20ece28067 100644
--- a/templates/configurators/addrepository.hamlet
+++ b/templates/configurators/addrepository.hamlet
@@ -1,3 +1,16 @@
 <div .span9 .hero-unit>
-  <h2>
-    Sorry, no configuration is implemented yet...
+    $if firstrun
+      <h2>
+        Welcome to git-annex!
+        <p>
+          There's just one thing to do before you can start using the power #
+          and convenience of git-annex.
+    <h2>
+      Create a git-annex repository
+    <p>
+      Files in this repository will managed by git-annex, #
+      and kept in sync with your repositories on other devices.
+    <form .form-inline>
+      <i class="icon-folder-open"></i> #
+      <input type="text" .input-xlarge placeholder="directory"> #
+      <button type="submit" .btn .btn-primary .btn-large>Make Repository</button>
diff --git a/templates/configurators/intro.hamlet b/templates/configurators/intro.hamlet
index be451a91fe..5062346a85 100644
--- a/templates/configurators/intro.hamlet
+++ b/templates/configurators/intro.hamlet
@@ -26,12 +26,3 @@
         <a .btn .btn-primary .btn-large href="@{AddRepositoryR}">Add another repository</a>
         <p>
           Or just sit back, watch the magic, and get on with using your files.
-  $nothing
-    <h2>
-      Creating a git-annex repository
-    <p>
-      Files in this repository will managed by git-annex, #
-      and kept in sync with your repositories on other devices.
-    <form .form-inline>
-      <input type="text" .input-xlarge placeholder="directory"> #
-      <button type="submit" .btn .btn-primary .btn-large>Make Repository</button>

From c3effcad8370e6de183324ad40e8e10facc38b30 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 16:13:19 -0400
Subject: [PATCH 4406/8313] update

---
 doc/design/assistant/webapp.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index d1eedcb322..f8867f0e28 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -20,6 +20,7 @@ The webapp is a web server that displays a shiny interface.
 * progress bars for each file
 * drag and drop to reorder
 * cancel and pause
+* button to open file browser on repo (`xdg-open $DIR`)
 * keep it usable w/o javascript, and accessible to blind, etc
 
 ## other features
@@ -29,6 +30,10 @@ The webapp is a web server that displays a shiny interface.
 * Display any relevant warning messages. One is the `inotify max_user_watches`
   exceeded message.
 
+## first start
+
+* make git repo
+
 ## implementation
 
 * possibly lose the ugly auth= token past the first page,

From bcf5c81593f26a253b514224e3326defd6fa0a8d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 16:19:24 -0400
Subject: [PATCH 4407/8313] when run in uninitialized git repo, do firstrun

My $HOME is in git, let's make it work :)
---
 Command/WebApp.hs | 10 ++++++----
 Init.hs           |  5 +++++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index a3294911f9..e2442c37ec 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -16,12 +16,13 @@ import Assistant.Threads.WebApp
 import Utility.WebApp
 import Utility.ThreadScheduler
 import Utility.Daemon (checkDaemon)
+import Init
 import qualified Command.Watch
 
 import Control.Concurrent.STM
 
 def :: [Command]
-def = [oneShot $ noRepo firstRun $
+def = [oneShot $ noRepo firstRun $ dontCheck repoExists $
 	withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $
         command "webapp" paramNothing seek "launch webapp"]
 
@@ -34,14 +35,15 @@ start :: Bool -> Bool -> CommandStart
 start foreground stopdaemon = notBareRepo $ do
 	if stopdaemon
 		then stopDaemon
-		else do
+		else ifM (isInitialized) ( go , liftIO firstRun )
+	stop
+	where
+		go = do
 			f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 			ifM (checkpid <&&> checkshim f) $
 				( liftIO $ openBrowser f 
 				, startDaemon True foreground $ Just openBrowser
 				)
-	stop
-	where
 		checkpid = do
 			pidfile <- fromRepo gitAnnexPidFile
 			liftIO $ isJust <$> checkDaemon pidfile
diff --git a/Init.hs b/Init.hs
index bddcc696e0..8c000cc413 100644
--- a/Init.hs
+++ b/Init.hs
@@ -7,6 +7,7 @@
 
 module Init (
 	ensureInitialized,
+	isInitialized,
 	initialize,
 	uninitialize
 ) where
@@ -45,6 +46,10 @@ ensureInitialized = getVersion >>= maybe needsinit checkVersion
 				, error "First run: git-annex init"
 				)
 
+{- Checks if a repository is initialized. Does not check version for ugrade. -}
+isInitialized :: Annex Bool
+isInitialized = maybe Annex.Branch.hasSibling (const $ return True) =<< getVersion
+
 {- set up a git pre-commit hook, if one is not already present -}
 gitPreCommitHookWrite :: Annex ()
 gitPreCommitHookWrite = unlessBare $ do

From b08da863be6e7c2837d4564ed69426d5427a5f7c Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 17:56:32 -0400
Subject: [PATCH 4408/8313] blog for the day

---
 .../blog/day_49__first_run_experience.mdwn    |  37 ++++++++++++++++++
 doc/design/assistant/screenshot/firstrun.png  | Bin 0 -> 54347 bytes
 2 files changed, 37 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_49__first_run_experience.mdwn
 create mode 100644 doc/design/assistant/screenshot/firstrun.png

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
new file mode 100644
index 0000000000..f026c3951b
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -0,0 +1,37 @@
+Started work on the interface displayed when the webapp is started 
+with no existing git-annex repository. All this needs to do is walk the user
+through setting up a repository, as simply as possible.
+
+A tricky part of this is that most of git-annex runs in the Annex monad,
+which requires a git-annex repository. Luckily, much of the webapp
+does not run in Annex, and it was pretty easy to work around the parts that
+do. Dodged a bullet there.
+
+There will, however, be a tricky transition from this first run webapp,
+to a normally fully running git-annex assistant and webapp. I think the
+first webapp will have to start up all the normal threads once it makes the
+repository, and then redirect the user's web browser to the full webapp.
+
+Anyway, the UI I've made is very simple: A single prompt, for the
+directory where the repository should go. With, eventually, tab completion,
+sanity checking (putting the repository in "/" is not good, and making it
+all of "$HOME" is probably unwise).
+
+[[!img screenshot/firstrun.png]]
+
+Ideally most users will accept the default, which will be something
+like `/home/username/Desktop/Annex`, and be through this step in seconds.
+
+Suggestions for a good default directory name appreciated.. Putting it on a
+folder that will appear on the desktop seems like a good idea, when there's
+a Desktop directory. I'm unsure if I should name it something specific like
+"GitAnnex", or something generic like "Synced".
+
+Time for the first of probably many polls!
+
+What should the default directory name used by the git-annex assistant be?
+
+[[!poll 0 "Annex" 0 "GitAnnex" 0 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
+
+(Note: This is a wiki. You can edit this page to add your own
+[[ikiwiki/directive/poll]] options!)
diff --git a/doc/design/assistant/screenshot/firstrun.png b/doc/design/assistant/screenshot/firstrun.png
new file mode 100644
index 0000000000000000000000000000000000000000..7e9d505b4057e2bf71faeeb0f32ef325fd15700e
GIT binary patch
literal 54347
zcmX7P1z1z>|MqAUkWeWV5Cy){Atf=9R6&&Pl<v-L2uL^q5fG4=iimVfnB-_ANB8I)
zFq#2l+q>WYJ?A>-T<2^jpHJMM`+lCp*RQmhu5(`p002zSbu<hB0Lu5|<rzIC`H12#
z;S}<Q+C@!I4FIT4VYsrVA@5TJ7-&BOR1DqSChzb$>KNz&0O5iFKvXOMa7jKCwFLkK
zNdf>EI{@HG768EEo7?b4nfwE-!%J-q0O^18x8|ZW@{w!4I_3f7t<L`~TD=~s<kJDq
zHPnnkXLr#Mt6M=tx`V0R6mz>w-o}J)S4OYfc$b)Us6>8i=m{6{jxv|0KRJC;{c>NN
z5R5^)S<hyK3oQtkxW5NJ6DjL_@rRE|xb4=9N%A?b=hu70cZaY;kt8#-i|W>+4wGX-
z<oWM)xD3oz2ELz%igf-`znXf%7yBvaUF&gED+4=}?;x5_q4CKYZI>itM*?THS!V)g
z>c5!@zx2*jkOfQV(6H;qM@k>or#<AaL(!4Y$!5~s@K;)Q&N1qYe<dOLj9-m~Iv<gx
zc+*8PKx@HsBMb@`dFlO{Kh^AEs*YK9dJyYNGp&aXoZT}Sn6E$k1w@?<3o3#K{vAja
z#L>~w=sa|AnaZC3#;ElQbUDx5CG@iYRim(S7hCb3*eo<5=E*FSI$z$8)={0XSJ1Ja
z4CPXho}bX%*zNm#C%xr+_0seqRXkGU@}g{D(7LZdawTU;CO-T8py|~DJ9W!<le8?!
zL#&dwyrrl!{QYLw|J{q4BAPEt>`P_K5DAOtm)FlYy^15hj6!2XYU{Cc+Q_%kh_x0a
z*sJUzOHt({)3jIglZUs!W0PpQqvZPklph+*BmsTVT^CX=u6I?WmF-vda(myd>9RB2
zqxvbsNR`ilZ6e4e{o2^=>X$ZEbd}&okAo#v!l1p=3ic;eMTWmDMcJV<x>G67GpW@=
zoi_*^!j)H6VDpDx5_~YBqV!GVKZdXQrp<Ut`m;mdpFu_ycY2ff=sV)y=RXNZ4)UXU
z8o|2#MJ^&WJ;QuJ(6NAs{mZ%FP(7UCxFQ*6FQ}!>PP$5s4Ps5~ls|vu?{b(W^EmlB
zFWZy<vShLcoS}Pjzl|P7{@IC<VBV$UV@u?Wt7Zaz(BKCsqhuH#(6^&;f<svsKSy+9
z(h%%%ZA(kv+SDj+8=F#0qoMTMR>wALkh2x1HivAM+u(>3*L(zhdo3z$xN12%(Z%jv
z0^LW~-!afzldfWH9d^+<bbM42Y<$i?`GaPjyU0VY!a3tobspaAN$Dk=;cK84)f(<f
znsNbJH(gj7j=Pds6Ul8)es*ibw*`CUy8OE-yc}J}(Y>C_7aqI?m6bMO*j=v2Od|}+
zs5y)a;ZH3wbMu1yA`(ic{k+7cmGCpg`ek}bvZbazXxE4Jv$p=<Q|AUraC1rRZc5iC
zs2QwSN3|Z8BS&S;I60&`|5{Y0HumfBc%T06?<2B2^&eHomc`%V2Z53hjm(a$-`@?D
z_$V@V=oEf_9<pH45LBtT!OsRQPqWbNj@zm(KwF{jqDn@F%|ONxbxl_ML8(<!PU80Q
zhtqE#$I{ZpQqysN%$*s<&}NKbq~q*8rKZD$JwaYcgJLA*01Hss!u%0Ngp@10bBz=Q
za4}sI?2KZSy01n-XwERG{)e^E(Ba^e<QKGyszk)kYp=7ag#YaF^j2!&PiX7lkNbz-
z;n(dE+jQqtH_xJrqPh~0P*ZxLtg(%CqUcoSvs&b;3lY%0#Kp;LC*57%P2Q<9wXuGX
znvf|<EP#uQ;yB}2p<Syz$+L|lU#D_4QKv44R6hF2JLyb!H$zPvs^3PzDo6X_Tj$qg
zI&^+|!lvrrp~`+&3eXt`xg44ZxP3XDmYv-a0k^m)O-R#MeIy~l@ztEetG*Sv5DG8B
zNCUDH63^^zZYpn$LIPp(WdiWtrrmX9oxXnd#a5QoJ@$K#35q7`H`L5d`%3ju;iK$X
zb5}Fg%|77F+gnad_&7dE#%|4|KfX!MY4rq~z?IP6_fTVIRmmC3;*rkKhQpvIDu8O8
zhiv_N&QM)<Y5!m1jU(>M#|&FJV?HUZ$AULG*d*;<T<uY!5Gsnad~w6Bk6&tp$Kl^t
zHmKlMEwbx}e$c&+y!=!2=IHGe)oRAFceKv9tHaOWAvbXd9u<=53<|9u>0}r)64xiG
z`X4yvwCe0l3SpBQ<qLhPH0R|Y$;2I#u-8b{{aCp|t0H4?hXy(PiDt6_?T?30h0><g
zX6=%U55BnnkzH62?m1qjd517F`*M#*YPTp2ND4p9+6-fTz<A@DT=w8z<%Ky_`Khvk
zf<ov_jg#1-b)DzTK(-9?y(i=T%O>><p=%@BKeSRN&0_~7kPti?bj+@8ow+xxE6Q4F
zcFMf7Gi6c^R&d%_8h#D%gv#L0c<yYtl>@2i2N!{5r#`1HMbvErlI2SExWWAz%6p=A
zMbhXVKP!-RQV5b*mtHzys;sP17<~3hFz|K48RmA(h+W+uG*fqS(tH{r4yyYlkn~fe
zwuvvH6CgqFU%&ajbmJ*+cAeH@n9xJuk?-2p$RzSD|F3?rQK3O0P<vZ*!lqbUx0Zgr
z$NYJ9u#d0w_K5+r=`|{O9l`(N@u-7}x^W)WOQtBL9{pXRGsf6T5e+u2fq$L98tNj9
z+VGXC7w%Zl4z30G9od*Q6I5@1R-=C=IC;9WiH^}Kf62MD3x3~vhijzV-?FaFZrt@T
z6`jq_*~?5}W8p&G$r8H%*|#Pl1$woJ=e5)oOpJBX;7?3%E*S|F;jui7<_ifpK0U0M
zoP6#i4w~b~^Xd!6fX}P$103r@22tTPsN<&76{)C>cxK?`RskR%LcAd1k)%7a?&EDS
zbY2Vyh8`9QhgQRZhe>zOVd_kj%#sSH%SwtuLsoK$9oFYRd)Vgb7HQ~<RG*XnR+YT+
zz2?L;#v9T8An<qL;e2Yy0x-1qCyV*#ea34IcrNYPM;?d1?2q^yvwV=G?fJw9kKl$B
zo@V$y0O=hw@a$HUJn;g@1qbs=zdoJy@W>BcIjlqy>RVduS1oyE_~qP=_FVczEQyN5
zy5-;<c=)LZvp%EhXIs!aKhQP`^<*@%;H=ZsvL<<wp33~?(VyR7$21D1#Oy|-yY;&S
zIqQ?{l+C&EDm|y&QyG&=ut3-|JDcy@HP5r|mT?`Q?{lS=qVvm2%M~Lm%BQxeV(2LT
zzJ6m9w%xu|_5Fo0K1EXNAse)76w`02obI*g_1HhaMJ2s@ujAUyZt8Q!zdx%PjULe~
z82NSmHqv=)p|^R*2=J3Jzzcqz#SY0l_ZV`DF?D{$osU3nm9?D=|9#iNw%>oTKe_*S
z<1a-`#HI==L1h;k1Pc6{DtH5{dDq3=)s_ARCpm=;I2+<aMfZg0sH8%f3p538;{CAQ
zw5eMk?mmrdHrNp4fb+vg_`gs{r~ZN6m7SVg*9(G@F4{groPT!9gT6@|n2LsOR{#33
zliV!tVju422uySe!c32#d~8mNk}@k@dGk)?Q_Wg-_4UD~X0u1A9^U3TIUAMxXG5~6
zZqEorjZTVYL8$=|pLpt8LReqTKwa(m=JoGuB}g#yb~Mn`-r;9IBB};-VGWI&dZ<m&
zi&Myp7n6r_g^}=+dU;g|WvBG+#vEH4`9+sV;izlXAuVSEfA=g=`79Bg^KvbN!h3U+
z-zq#8`-;BsmIPd&>7{^?j1Sd@79KU|6^!I6gw<5Es76k|1~@rkWL!{M$=vHZbI8HZ
z4%Jysg{<GzoH#0uq&diO@{5j#yT^btTw`=^f2)b_9M4=<R&rUpBIS~+pgQX*m9vKy
zc28=X-07*iW;J=>NjTn}f!95n3?53!(f>-*vOKB}6DGu4qMp9g8q5kqJ`=bNIvQI~
z+9$g7_K#$uY{M&!vJktKEe3DuM+}5o^AJr4x5h{A-7A~zrGr-+Bv06w1kiVUPfP}R
zwP%x^S?SWbw-r@;@afa1`Q1E9A34k)Q!#8ee3M3P*h9F|-}4)6bh09wSs?e3NY?2Z
zfk-3*yrGG%7e&MNpjgmmGso2<P51p<O`v0%{QmQOZ`|J=o8&u39mX9wcP5d)pJ2Hj
z{{cDVTpHyYBes8J$Hf0B`Of#JSQig=?(~j22rI<#vg=b+-48%Yxh%%^GSBR(WFS&(
ziW;rLp7W<t{%#5d+dci1Y2bv&wp&`+_|<GteqdAwu|c=CzKeNwCI8dM@~ntP`dY+c
z$<hYDvqn>CY-w2q9L)R{pzl*S^A_u>RChl1iZ`4l#S<YGoAN3FMm#1>WQ*oZl`*Tr
zAuDi5V#rOOP@(>v9G`-H-LfZBbuZCwuR?7D-b%1hx4BA=PffgPkZ8m9&_`*}=C-sM
z_PwCy8Zyr72d?a4*n5frS^oMaa&nBxd3=YbG(A&a8Oxd`*7DMNURkYm$_ORZ8Uovn
zBK!7}o<mUR>~C^r&Zu*O()YQ=NH4WE1UZk9CZ625Ixh%8T)~rM1!928=M8}Q<8ua8
zUTgy{b%t_L5?`FL42djVO0DL-vdYO;4my+ZQeQ$X@>=_{G$ym(2x2AumNz~O4QRjT
z^+HkMbia~RYx@a$?gK044Hs8L9N4FhGqj2TX`Js}X6kk(M46OMd%Rp*i-(0_D(@rp
zN!Y`=I-V)8q|9NlqP(p#kAm-0YY<3N^Sy32r9!}&r<s{6-tOZs!?(RBj}9w;B2g9t
z{ME7c<dUUi#2t@6Kl$cbtTDRSqitEVKbL47yi(oe5Nu!AqPn0p^tO5$sn*5rT3&ro
z(THt`tlxI9dGqFtw|9lt2iaYwT<m;p#AXE=@H2mOyq?*DL+5$`!Wl%Ohuu_M;E=GA
z`_QD!l!=zLJs@Bi-#P(GgvX0cMv)7*#hoQz3pYfp{N8_+%gCwt-Dv|4t~liV4|Z!|
z#O1eLbK{8Vc6a^n5{WOSw#`pG|1I_Py{yedWttjM)5S~b6^t+C0BO!AdFkqa5~jIz
zmb_}M<JUS0tAJ(mEf*Z9{L-~`R9Bo7OLtg*U$^GHN2;<PGHdm7?>_$FM02v&@Ei1g
z&gd5i$jr#dPCb7|CifkH@33xJF=CMyHf_&xo&I{NP9cZz5CuIW*Y4ONVzt^qi{D94
z@TSY-nwmPLir%;2%ij1*lB4wyhpm3&em>OVu`m6>mwXkF^Qz?>1#PR0zF=d$$gVTt
zBA+<`hox9TFK+f;AQ5x%X)#)V+*<hTFnOsq4xx!%-@j8u(or*}K`C_UaX>>q?_9pX
z@xq@zLG>KwqR#4$1&K13%qMV}GzI`4fX(E3HFa+NM&x=M!Iu7%$pB6h^_<^%9&My{
z-=+opN#~*N+^L8MH~My2<4-pm3^WC36MXbAs?ewDyFtT-apT<_2qL}hKl$F{fM1|0
zppZKzEg6B8ND<QE0)!r%eWyVh%`@cwceR|`>9ha9k!$JDKA%nyjRu!<7b9)aj~0*b
z#I~W5Qs-co_@jF*lN=(y;x?LC4Z>GW4n`a)t}YUvSMV#PxT+c(4Y!DW2FJW=bD3I2
zBt{i!KSDO)d4zT;K=;YfHv6NPh#^%1Ju@@?0iuz!Ly|2~!QmfYu})Q%9$PUZ8T8eB
zE&ZC`<!Hk~d*h(d0Cd#Q<CymSzM#JX>ev}RgAHIR-)s@nPk?^-U{YcAh`xPmq##qm
zHa|603UnFxSijjjgzs94^3`i2qx9!2H}c24Y)Rd`9C)2|suG3}9^z5vv0%04+N52X
zm*iJ<^$hS>zU}c4Yu)lh@qO<Hys)6cI@Y+jq{^$guc@gVO;mJsy`<t9_(+$Ft6>K1
z``BDnkXFG2<8?0V3cnI)!&5U%Q%A=Y`m$^4*U#F%steRB%<A^+h)>Y8Tg$J7VHEnk
zTSYCbXm=$n3Ee4odhO6>PLsdb2Y($=MvkX-ErsbXzuM|}V3P9a?ptCR$GYBI(tfJ{
zJMJp0+5!&72>|N)(LB;EHbuv$$L1GD3}Sthe2=;324ZjhDAHrTbNgD<Z8etIR}H0u
zbqD?#7Nqn<F8$132s5dU>@YQ!C%XotQ%@G53J;3LCYTwR>WJyFXtXY)q{B6or3PE7
zsqVN85yE<L{+Q_B=TPn2^ggwBjR3mu3{Vry;I;tLw@vg<{T@4X&eQn;IKYd|F*Y4E
z;?ppqk$O%l7!PONEPHE%9)|mGIvOYbfW7r(+N}h1%SRZOt|q$G2<vrGTo{i{%Z>AY
ztqDkQ<fhEr`t0A#?$TdGCF_;hGmGBsJm{iDCSIM&gEeAeOJd@&+&vDe=G~OM{JN<x
zA24yl6!~?HuJIQlmB+t`vVmaBDstTd)m+tL5>pO)B^*$D!+dH7!_Cb-Ki}Xtm6imB
zLK_?B?WAiXUqd8nlX}R!h2H$un+F5RKG>z;kuUOGmJ{RSYa1Js3V~b0N9+Cj4Ljl<
z;f4d=u$WV*ScmR)KM8}s8DETP@L%O3!OP1WL}hhxnksv=1w}MKnXT~IILK7junv+H
zunli&dsF=)WMrR1qAFi`#(#(XdsqfJCyxO+++}A+Lgs}6{1mOzP2aqc@@iLkmLrhu
zQTpu^X7D~l`;_%bg&BUMzbTUG)Rg)i-2=*Dvxw{<(-&yty_<7WMS2_ps-NcT+Qozo
zZOy$AFp9_HR_u`kz?O$_&UH*{*YSErrZ_DPvxCMDxXZXQ3b*3}bhrk1z}AaasWh=Y
zV~>?npEO7e-HG6BX!wV%4_Pr*p)xZ|bi&2ceMrvyv#U?v4ztnGY|OI}xJ}d7Sr`3e
zS3Gd~&b{WkvBIB~K`k8DqV3i`KCveJjSxJDZVfre(-&;*WcaAzNkKX+nh6VAY^pkc
zAHodrCth9j8&7n^Gm(Kl@b^ziIk|@VhDV)0O4xUj@7}Yw2SLuIMZ%9f0WsE{fHI-l
z>#%Q8#Lv6xyp-jtD|5T8=*wO{c^?RRJz&z2GO7aeN0hIL;NYXml0au;aEEv@51S-j
z^(jF~bq!9BG9jv8PTUV?bA4>KH!1BfYZ1F(J-7M~=(<R2-_dXNu6Xz%CM<jL+fGYw
zCDo4XYQlkUs!rg_+VlKPB^Avtx(xyCsg6jq0X_9eHk07u8sR`+{n;UgXdejg>C6Y!
zk3C+<*9YwI9OyugrIVax8Hx)_Z(-HSaO1|SmoF{<`YwMFvjmXM$KpY?Eb7eTy0XN9
z;KJx@!!vV`mrCjNmiAsF@ylg#K{Xclp*7oj3kmw@<(gTu0F5QF1^zLev$EvFySKJV
z9o4}@pm@X<%{%$+m)sJh{{pr9FRA0HVd6?fvW;iQpWeq(6Ai*IU&Q?98-=SCxKAI=
z7j!5&X*@mYe&M2hT0Mq&mBUNEkjoXgck$5yyZR}!<|2$!TDsfYA?Ny^{U2wyz0J8A
z37*y)@g25iy4=v`mRWf7>4dH15=iaki}fxBDgsp#=(4m4Pec1Y9j0P!S+G}ypm@}8
zA<AAbGQ5FixXm$?+xT~>UdSvvR|ZOG7P#A^mmRnXBb_$_Nr#GaNTg_X541D5p}IEt
zf8odV!X>A9>@prLE&wX+Z~o^gColKdY#<lr19`VOa>)6yt22PTW#xr27DuDzsMVL=
zvO+-+6kO!RFM%c~L%N3T-c5H+B(d7b)%JB73Ws0mR}#Z|XJ5t>-%91Ki^(!&%ndS`
za>j81gQtxVTk}Z)VNiCKkFvIt#X#C6%yMa^aLxMg25LxCTiZ&S?l$%k`1CiysfUBE
z#RsWN*uIME!`(Ndg~t^i?Mp!E{ybU_j8T65J-g&F<=@c2(o4?_bd5!Y143+Mm4E0C
zUH;%4ZL9BTygHuw)L7#}*1EySt=ai5%M#FcDJtRM;PdDx22&c$QMZtw{Ps1A$h)qA
zYtewG?R^zC&F+tbSH;)9G&MC*L-X9?R3^`^$duwoUX^GLYIq@B`_A?9RX-h_f8+pE
z?z4w(#MzQAIfiEv4&%)uR0n4Nmlb!GSaCq$!`7DojqklA{H7L!BlCkAuH+I&=liRL
zghhApcX&{Cj=l>~IUZMFjhR6WMndpX9p3|4b)$EFKYxC+2lw`h7eE`{7*Nr7zfBEr
zt1RL4kU71-Yp>-<tMk=jLj)q4bhS{7)9mW}a{;G2bz6)C^aj9YYfr!VE*ZlEpN<n4
zw5UU}rfVpr_sO;HcuTt&^v1Q_#THn(Mk;etLohGO+V_rh7*XZkLi9RznU(EHryO&g
zEyUJDFd!f;<d1TPB!8S=KtSmP8E{%R%Cs4Gw;V4lB;M=}$A{WBxt|5zlzsd-vSA-;
z;?g~JCQr<N9ul-rD%vr<SAlg$PMz}I9IY3}@uwwvxDIQ^GGX3UaYuDga&ROLxsDrI
z7nhcRG3)E56(-?uo`td=DrP<6{szhV3CKIahlQ`C)Z^g5RY~+>O2~o8e$_ReiIBI+
z7ocb7m;kM^gE%u;MD0IS?+jib@xR3h_Qk`lhYdcxAL`-HQWbFf0X9~ns3Qz-rg$FF
z>_rYx$)_eK;bN{aXH%KLJhQm+u*0wiC>_I!P^rAc9u0HYTp1ZCjkH&JKiKdB;aR^K
ze`i+x!k>4JQNQ{50UC{jRs9$GkEPM6wm7S~PO@|Xw6k1fs>1h5p(I@4oR~ed@F4tI
zp+Q})-aVG3{gsuK<>iHuBmPt!e%2lht-FNnKXy{;os^%dDl&yd)T1M$^Yl6;g$rw1
z+Wx~9M5J007Jl&2Uw*}KY@qy7d%2Uepk2g(qOlFtN$@u<SDvIl_Y~_qp6y7hZslNw
zrUpGok$c-N`PfAyWFPLX&O-felZmz?qvWMp%rmcuSKGZX5s}LIkVTb`(-|@yn(j~h
zuGV@Dg6Ogj4EGzSX7V(^0?MeVsR@%@6_w4yZ62&Yym$U6dqV8uBEG)9n$dS-pxzil
znro1^39)x_$YykwdH8u<f@Hh4Rn0u4^F>F4DM`@ObbH%SzVEn{^hZ)w)EN^1=fQwX
zI@O~2eD9iqi7P9{+x`+h#RC(Ues71TnD4O7KIhxl@bcPwbZVZR^Mh`uZVGcX?Xc3M
zJb&9?J|=bN>P?4NOT{eV%v~u$5JBdS`PiPDKZ>GlepTT*yG`b#7<g67+U%%50(7;g
znXcd9zO}o1$tHA@y3IjKek+Loz$TxE9bTOFh+o{hY=*IzT8-+8j)`&l5{YmrRIYd(
z*6uk(zyM6AIBzOi@Uv0)24y@UxOllkE?%=fHrbWdG0gQ_>UY!^1&__hHq@F)+{d5d
z-u!)`Cs?2%?AYl*#__=~%9A<e1_lRf8^glG!>>1)yUHv6a<S-gOshV`o|};|8|nS*
zaMgXMB5-v}op`sTE}rZ2q;MS5cb7*jk{@-j*f+MOF3_IXX#q1IrHV=@mke9<a9jGY
zAb%8VhXZS;bgxQU(Js+<v-Hlw-^dGT)G-!|4OxuPAO0jM`8TqDwg>_&B8@J5iD|PO
zC2e(OdN?5pn}~0WUN7bvvGzzYK1h9m&F0JlRkgM^Nb|FjqtPP|_Sl;O0`G`K8yg#O
zarEE!#nmHxfdS68E)(!1QwB;2X?f!EOi?IB5^VumDX+}x;XtWKW}Zprcg3x`i(or^
zfUoN`jCjeM3^rc>(sSH2^S0r&pLpo&-#U>X-W`2!8!}QG6x0g}a%?`!?r~1&I^*qk
zy2+xTD|l0$dr&%C!|t1h=hgXYGZht;uAt#sLMj1&D$lKW?SX^&wFgg&WQ>gSgM)7-
zrezO0C)EXSmy(L*6W>-`pX*^oSq|imILHOzv^fRJbo<T*V;U;*c<<afanxocQ~9~y
zp%MB_&h{fU5Ct;;jU(kwnRu#>V8DoL#o6&TzI1#I@orNriHS+pVvrBQTWmBMCQ0kt
zva%`nsM;~-7r&f)Dax%h56$P_!J4Rn#U6c+p+t&ZYKqzye*DOq2uRRym@E{M=f^pr
z%SbW-S`zi1&j!q(@8v_DnCa=D5;`3uGD67ehK=d|Zo{WLMvm$+XYRG_n$q{38Bxb4
zF*83=Qj8rh3df2bQlH>fxP!=o8`-$D+PS!OyS1+Q+y$!U%jbyOws=aPt}_|Onwk4F
zIdg7kj4)GOQK=|5=UPAT?@%;6gBpE|O}yD;Y?Fst<NxVW<`_hN&>}(I>BA9<^0N=^
zrTY*8czn<u9{E~W{7!<sJ-Mj`;WM6cV<86qd1lB>F2<zrykxEN=Mh+G=h^h<?<lCm
z*x`j^+I%ddcP6cO)=*ALIPlpjYJI5U)y-eKnKy?jsP4zI_?HG;iUDF-p3#2~)uq25
zI}do+jEcUxj@j9PToHf6J=Ey$zmjyG4-LUB5({Gg<5|25z9MdA`K{(zGd!dBi{?cF
z!(Fa?NY~D8rT7RI^cd&}#?JeDr;eyKfFse@C?UqB%QQv6!F-ly|B_0DHhY3jPb(@u
z+-nS0zC0*!POGLvCN9&b^+pzBMvcoS`#S=nU@&Oe9n?A2fLv!+d~bWZLFV^n98wdy
zyeqt~%hyA7oh3PkY#aV1Xi#0^`_SR|^T+U+NNjr;3{23^gfSd4f71EcJn)HpM;`;k
zs-K=$IkXgC{bN;vuud8sBgt$p?8yFtERS~GS8rdXit%v&1KUIKC`QK4*D_N=3Pya5
zOSg-T67R}t2?mhWJpkFJy}SnNk7UTGE5I4M5c|kf))bmdl8pRUwUF*4$&E6~%?ZU@
z+t~Dd<=*>ViM-ClwT2m$*XI$6WjV4OZ4NCfFAoX|0{X42JxEbLT8?joMR+eP604<8
zw`ZmboF^>wnjqiWhPRfN!$^O&%_BmR5<g<ruX)s(JdhlSox|Fhe_8!Vt6nhUNy(Q{
zlb@gOsHL4VxS?(f5y%u#7&0?7oXe>pw^_f0_1C}<{{B7;#(f^G!nU8+0NjN0K_L42
zDLW&M<o3?<y^Grm0glUZY7BBH((qs_q?zUYy!LKSgBSE!emZO?{5~Wbm-WWQA|m8A
zO!>BA<iGv=u+Evi#`}Q}csx8jtgUVdy~>T;`W_#=H>>EV<*(Jf(c|*Vs_1KdV|^ab
z7P~rTTU2WJpR%*^`M)9be$U5yMW6a3zI?B%n;aYjttIFr%S?{D3b2vsTW52w4oPK{
zyqw%oQ`f|VSaAEj$@TK@4K5W{%_Zff*<Tfs>|fj3im<=L?0DS1j&sAQ9L#Luxs#H>
z1Q5Qel99F}^?`QsJtY%ELr1;Cqa$vW@RQhUH?6h;b6G|`G=X_$7h^1mjUNzM6!es$
zECHtzJMm4ocq4BJ(n_k6XEpZ)Na(^Eoq-anYUP>n_BIroUotwzAtG)>bBP<&cCaC)
zuB-|TI?Dj^Hw)Q%W3Nd$_Gp}X`Q$1F1W#ehc)<$Yk{=NWOtR9-=U;jmxW{rc3Qu4!
zEe%D|diC=~t7%q45G@h-Jw(pm?iQ~iz1Mcb-z%AIN}B#l)d^~NQg224*O9W$OQ^6Z
z+EMeqjEkd<v0qF0*WFDU)5e$9iwQc_>`7S%MVvz`{a@uO+gO_R*GR{-wffOQyC_>d
zrEz5&>l{OO6js+F_E*Dfib`F~v)f$l%(GvaqK?0~;n0;Hu0}A$G1GfNXS*}<i;>5r
zl`YqqnNyOJc~m3MPqxQ8IyzGBDjsd*_~ffZoHcmzJo{I!Q8bmqCFLTS^-*i+5;g1^
zH_(w0-wS}V3aXC*q3=9)FAW>c*B)_X_g7V^iA$T$WPeaCUb8hVyFAl**Vx!tR8$ln
zukGu502=$nrPx2Y$9v-CbB@Q;(b8W0-@5OuxGgvvkApOoOzlKum#ZI|kv~~LYG(SM
z>)j)(-PyWHWGe^Fy}Dz{kvUSpAHWO>+BPuMG5L2prAYa35OEQ7ie~7I=F4sL1-E<o
z-R;TN&DG!+KPS4XEwdJ9aV^auAxJPr#!1mOlDIeGqsh-2)zH8V5DY-9N}-Mmor-Q@
zZ9>p~DI>0rIl0YDOfHXmQ{cFKQI-3tFLr{0n39;%u-W_}A;7aSzx@5ZpF40DRLk*@
zlDD__O`%st|J|aXph!(kC9e&C>4stJH^~LXLfGcL>%Fm`Y}lk2r>0xExa+&MDyrM2
zr|n!lAIop)X+6+QiY|U5Dce2{sAx0rkYv!!<Rag)Up-4z;0wnA8YdKC+p)3=3=0jV
zj~*T#_9eF&1BZBeMuwA<Q~lq+p%Hi^6O&VWpTB?qv5RKmrq&^3q;y1{aJZ?RJF1J2
zZ<#Z}DmLZC#bmBX=;aF@@YeD#yDO~HYIl-M4tKM0iK?f4gugsuOUe;>j%h_-!`-+m
zSygAUv%P)V=?Gt(X;`mN!X5;k`-xq4)AH^U*%E(NRKz(s^?ZOxa&v3PF(X<mFOQ3h
z)An&kCzN*8W@T+;#Wz5HwYvB1W$~ZD@qhomtGD0b4s?cV@~7Utz|NJLN-Xl+J2rGn
zG%0NsyR7S`%}bP7PzA5(3qg@$=i<%-W?!5lCV{ju_hn>cl>aw1{rc7IDQBr&RBq~R
zBs@W;5s`Ro<G{dxCX~IF_iwPDUvY8q_IgyE*v&Zm@GB4H9wNKUB`NvoL+`Zw#>1o)
zi&Ptyr1Xi!<LQTpcT)ER{A-I^LX~!?ITQ>`{Pty-U#dK#ppUNJO+%c)Tpzc~%1sl_
zdSIhFcSyBF!ms%-DxYGA+cwPn8r4$f%Ki%xCgz6=GdT;tMdy+DQkkHYZyaEt+um1G
z+wg6Ho!B<H0j3G<#oqI}g}J0kaOmhi;>dHf$oeqp3LDAJNM+ac;kJlL*_K`&h3khi
zCkSy&Q*hY!x6aomFSx>Bd*x}1ku^L0y!l}p+26kL0k*fdCoE_`-fC?QSq#RXzX`_I
z7ew!R|N25ERqG=!j*{zY(9yG|3!`SZ)yABoTK<^UZegaI`7cuc(gdX!ot+oW!1Bu8
zUiH))&ns^FC3-yFyysxrZ-e<A*hn+u6@eE}>O0OvTc)IEz@#iThVt&P45xob+hA%o
zy`B^oRad`xTVn>v_{u>oBN5%p=c|>)ZSTz4_Rvf>Jf?8uKl(4?s@F?oWjRVKZZI(!
zgxAMYpA*Z(D<{WK48u;H<%zq;R0famTLmc#K~vV5;?@>5Z=Y*v(MCb8R(bvW{CYlo
zC_2}}=n=&Zl`AKVaQj7(D~CN(m5WQ{%4%<qptq13|G5Y!I48bREz6mi{-utw+p#CP
zHj9azm|)mN2tGD3EX-Cv@AH4`iK5D+-WGiy7)vH@Z;DwrN01eIub868FH_vq?T>{%
zGwL1Hm6aJeBr03J@hiNnLG(quemxwUm6b~cL#Ii)Ew#Rzm@sW>#t(vzvOgk7A^Ojv
zGvn277W|q0ny_3T$`03gSkYo(9ky+-mcS||Y&X6WNbW&KP3|aWvqWRVLcQRL={|K#
zUcC-Y{@?YY7NagNpTtpgEVX7TGD<od-Rh8Zd{B@KQ>Z6F3D-gdI29cy)}ashwZi^W
zRwYC(TKoB_Fns7qzDFeqx;$xbf9CPR$*E9-{~7}Wm7QG9p!4y;az}aQ7a`rOVJKnk
z@&j=$C9W{~C)!SLYySPAv{V$K2jz*rSZ>lJ+c`Fz^nlbUl*x!gsrV5)2CN^UngM*k
zh^drcZvu#sML7wIUR8ik{noZ%w+h+g_6``VNz^PxDr$6e^kA;&tmZz9M=6Aq#!Sn6
z7hpH~0|*s3OOYt=GL+g$fuak6wqDa$Fj<MDoB>JnUsf!#KGUW}okhwT7On8CDvOAJ
zvwT7WeQ?>UZ!_sNdva;3rR&gX0O-aT(&?KwSwQx{!9#&SAo9zjJ$(IxVEHubV1z$D
zde_9Jr_mUAa1~AA?La%{f4v0m{|lJyF0Ws3HVQc-KHhoH5q*2KGTAh;=6ID$Uu^5A
z`mX!f%ixy?%TI*)U9p)r&=SGHk4W1e=#zbM`S=)>#pfzCt<%&0bVMnAmT3+s2b3L`
znUtsK4XGTRsTY2Fta$`R#=Mb~@mf8^^}Hf(mg{ymj7i%SN^{gJJGe?lC46r~=88iu
zc<(5tja<e|Me|(|XVU!&;zK<2$>B#Td!VhY4Y1~4e@X)Y0)fRw;cKpK>AfJ96yz0V
zDMMreCxjNt>`&5)F|qWL&bG3)wzjguU@+m?VJ@o^vbD8k<K;h3vk$YR9Lby>;>D@?
z(7PVq9ME!mmr#CI9%0)BpxV12A2BIoT44l~^%z=6QT3~$<tPzWSC1FahXnrnbZ~UM
zNs%iC<dE__-xb=>V7%jpIdKbl&+q+gmzay{6X_E$aC4f_WM;-G>K9>AimxqqUgIm>
zWrYhrtSVHSS^2dzAyb&gR^XzMTf?Y;v*t!ZUI+qtjI9f)^NtAFISW}t6TK1LrKnsb
z`HO{Oa0@snsFt7AaqVx4>dcO?qqdfYS-%6Pp#K_uh&-a1cTYBFFgoh!1Dr^t2+?gK
z_p>_8_i7^hZW52mY*$xTurQfCkq$p`-BHY*@jN>^GWhQn0gppyVUu&`@SDXgL;`^a
zEM-(E;xp@Oyx4U45s65#eNnc!Rg)4zF|FWd^cQjjfOX1BwFrXT@R7-7`BLp9;(wLz
zid4;J-wZWvAH_A&(w;BK|F_hOT;O4{+nCQ&mN>a{R5JB#;*vJaZiIo-#aiX{<Vl7%
zy{$tq@A#V^HW3$rNxm+zz$m47y4(hL@3Q8;oRfI45i0lYj-Ug8G1%>R@Adg|yd(KV
zye?zpOC-kM)0_i_Ga+!?DWs*&8Mh(3{S~GU_${VV1sdF`ET5@|pi8axP?N?cWoCYt
zt!F2sTUQ}_N?%uPDMKM4eqDu^IMs8h)n;D6-OytgJDjU^!VyNq!F-6Dw&v?}i?;Rt
z0s!6+xAewaPmz~tuNnX3d$YgwoIkisuC4Uy$1ie2i@BrP?9MlVBsa0-%?l)nun!0Z
z1yZ&Gry$3DIkq{v?0`+=&=qQDzX(|~X^}KkD?#H>{Ehx(^999u&I@YX(}Y3*b=%3C
z$xYng-^ybTi+iU5WLNYS1bkhEk@jIeS9n|Ae7t+3*xZ@r3|Q?<c>htRF4vZzeJh~)
zzBtFpkK`j!<d5UCH`F3O0p0PBpeP%uI~XMl3*q9bnH0WSTWhW~+42T*x${e<$Xa4X
z3{AestanInvPEEgk*Ln3dr#fm%X_Dhc(3u66Lv+0BrQjB$+uuPEH8(-bB7GU8nj+j
zZP`yATd9MZ-xzkdx$D*jktzMNw;NYo!t(Wv`Vi#En+Mw3$#<n9&0Kw3e<!p7v@{uS
zFx}vkik#m1ms(Jub$NmdNNa>4r}vhfXg}&aoUIQKib6Mrz!U?tD#;|-#BB}^az|0N
z(}|8s4Mj!UmV+MP{`~231=@T^-t*6fC|lyg&kL&iHSkjbgDHuYD~C6;3`!x`EU5>I
zN=gwYTT&CI!tpn*QL!l9$L9F7WJ^2s<*S$@^b@Eg!I7R|Ffsj+VNH!{uH^m6Hg4An
z8>WxU?w+2n9~oy>TJbzl_;Yl$UTjE9E`iR^PmqD+G;_epgZ(h_CN)B}@6vJB0Xh>&
ze0C8<K|wBblzk7DI>vZ+Yv}V@)c8~79&eeOnAp5g73Tl3-k8sme|U3$%L!;xQi36K
zhaga%20s}m^%8f}4(_W{(cbWcPY+i~a&Tla!&P&oojx?S6Hd%)i!wLYF_`?@7Ut%P
zPJOokb@uKY931bT*@LFKx(aQP;E9Qe7&>Mmxk7l-_jCE#;g-<iT!A<g7X;dldBCif
zQ%yP-=*al|ua{&2**%$4S01Dk_hknp+TOKX;-{6w4dSMnnu52Ha)@4c{ePctayS6R
znlGPTef!*FGB<cJmU3gC>ys~WI07K<uN-xcD+PRxU7aW*I-qkFArMnDvuQ5`+IdjM
zBVsc9NXEJF(P^=e7ftQ+vWk&MD(n2T01d`;o}<#*CvGun{HEg@-b1RrVbGI9=7kNZ
zwb|yQOjK@@@PFEj-d{(<YzR=u)@xR(0QJ*uuM5LXiLVu(4=FKWZAP{kM0>x!AAeZt
z*PyGLAx)Wz9%^IO+op6vL$GnuXYUhy&ZIIBli$Oee6<fjf)3iM#q@dgf{cAYU`#Ow
zQSzxE@d_`!l)fj{vA#MdlQIvKpSVOzo)lk5vC+1Yt#lKT0qI$z+x4#0=DAyaG>rvH
z<htS`A02+c6QJ*kO74)MP12?|nrf+eaNoMTW!AZiMne3Pj=<ha`zWE!Zi12{qH$YC
z#Zty&sAKZ-)+U&rSJCa0=^XSCegW;gAvU7=`L_^H>R(#h;)?1y)E+}kV|JwG1_|VU
zl&Hx-g?>1*NTB?!G}{-<fcbr-M@`%;u#ICxA_|mMuAaG9x+Z<SDLs+Ucax6XS!sOh
z5?_x%jX4Z^E*M`;A2SreSD>{08$mwe;eK}?`0f18oK5#e@D(<F2hAy!7-);RcX&(8
z>|Fr#_s`~jgSP8(o0ZppT=ugk7THN1D~FP?QuTlpq_*2LSk`7Tapxr3L}Fb2dQy__
z;h~#hH5rJp6@QdDBoklQM%>vKpRm%Ve0|Wr%gZr|78OIVIe!KX-3<--Yb@mcs0aiu
z0Hsni;%a4OWnf@XmS#zz%L%azk@>T8HYXH$HUq9Ui#+$`0qE=No0^&eqWS*zu5S)y
z(9zMIDjW~3RQFDQ@-=O@^danT7r|y~JvLWUfYlyT6=a6Jq|*H|thA+9>169cii<PY
zcg|}S4hO!-Cq4xe<x^D7*6{mC-kUd%4}~I}Rj)ohkUcm!Q7}I;1nz@@c?uOr29yUT
z#UDOEPL5CU#GZXnWF%mGasp&}PT1chsP)wZ8FSqB$NgC0MUq+$-nred4t3<r8g!0h
zy%x=vp;OUQAkv&s!jLK6AactEl{Yv@kPhGd3tm_V&Sbn!@9UQeIy%&@_VZJF*M_{h
zg1;!<b=tPKw=XO#B-f&KcAm!8)=W>Qwm+VrZrp>`yF@lsibNrcbba&YUWcr&WZK?J
zZo(&Lt1DCJa8mbo0zPizBq9r54>BVAxP%LKtsa>XxAp*71{_$Y$5NpHoeL7uC9KaN
zA=zwlNAC~8jy7~xvu~wW+O(Srr&@Xif!U&f8S+q#xu&2qULvE`xatUHHT_gwF+Dp&
z=aMGV>8Su!8tgf@QJEAG5A+dz4owd~L5LrBqyUq7lqSz+tSu~_cut1`ipDuEnXmH{
zf2LrOI=Cexgfh8gzCoY*5-_hKy~N9#{gLTZT0R8eNQa}7e0slJ>Uwln`*h>e0F0HB
zuWKtHo3qtAH742FHo^n9&=I06?U=4VR(Sno8y*IVQ0q7+tDR4f-d4y2URHpe4shz8
zm7#3D4(Lb0`?*Yhvq0ESNR~pxo0F$uZJLCh?HzVe^Xbq+ep4}JpnTIH#&AQKga2j@
z^>R#%5P0V6>GZyhWYF;c_prVp6WTZ$x`yX~haYbch;R?5+8MHkeg21@HO*(WOrczk
z*GHs3?@<Q&z10r84A&_zBp!6n*)}o&6CSLrc?*et?DUS$6)s)kKB1Fywm4p13NZEv
zgy+KNUha_*?(3Nu@+b%IkmrqIvrdT2U0%Mk;JCBGxyjETG)WC7h0oT84xQg|Q0Hf6
ziE?N8V{XC1nAaF3eVayFa}?nVtplF&O~CN+Di@6v98me7chi;^czDd4ujd!wQJW_J
zm{YDPBdl(vCO#g&%B+{n&AM@Z-%cQ{`mT#=_{kQDh(nf|Rvszuudb5mkNw#9@ADJ`
zy2Gb*_v=O0m{qQZi6~tEy8n)1^<GYO;_4*P1HGT0r{w<@r#0*K1lseWaLTwmc>b@S
z<-)?%iD5vUR|KwC$Yp=FR_5WupO=^kH^U<Er+UKKHNTme2;RTIYppTN54{$N*gZrK
z=wu|GSpel*Mf;A5awu1cV7;6QULerY(NWPeaGeC0&&Y?xgYft`=6e>FrkTg<nZ*Gx
z!~e*RQ*!%yip=IvWP{Hl)$`{zHa7;bUp{nc4gJRsrB-|W@Am2GslO>6b$eO&*QyY>
z!T0dTs3F&F**^8<t@P^q_Ax2o98<sLCh`CRk+r8TQ)itYbwpFwc%Cr6W4^)okw@v5
z%N^7&a&lZ=UIr3Q49(24H2B~AcWW|W@Y5%jyjN2fNf+V&&?Ap`CEm?2F@<+q_xz1e
zoAa+uiw8Czdv$~$t*!^LLJlt$Os9yAAQSJwDTl78M2_8!yEpkHnyyy+1-^DvlD0Bs
zc9-hM`0FM(GXD#kdyq|mg>s*#r2M6|j>&2K0s7Hi3TrW8Aoo&<ZrySBxRt-)kQi&9
z6`_4jnPAj=bQzIbTE=v|#T+F+aXU`|A2>U(!4_-VH?-)t*a1+`+MrbLAJzL*BWYm^
zZGCAk{e63;++o9lNi5n9XC|%I_rvWReb)ZM(SSBWR(7+m-p~^K7>MvqG-%Hi7a<6e
zlmLtYdl89zf#+`gx23#Fs!z*tC3AVaB>ThYuTYoSO#kg4lS?aXDz9(wIe)vVg7z1_
zly&JlonJpDU2KK}YP)CAb0PZ85ckgC32M?(fG_sWy7b`HohNE6w`>TrICxQA9Y`%$
z-cZl0x%Bt8J9O_oTF(c;CvWPA?_*|s4QQ{{m$@+&M<7-!b?L^3HPnk>@2&XlL&pHV
zr<t}FJT?miackv8RsDiiTIw)mxzY)ayr#%nv`vj=I6hk3wxVwC?ChmQ$h%~V@y}?R
zqXVw#md!n^KiTn$CtPWFDqp-WWnZAom$wW}>v|~jfw?`fctLbLQ8%fnc*Mc`7PX9R
zar1nO_nQapl(grb;V!7+u}xdE%UPs}M5;W{!+df0yr<AZG}`sGZ);PLOK54h|J=|)
z!U368?s7PUO0=9$A&C=cG@%){pcI5jM<RhyiGBziAL>k$Jz(W$#I5UHo3Q=Ok7Uu<
zA?d`R*hoAxNB0+`xtiE3WQcv0u&=gZ{V^m$Bx8bm@Ne}9m!%vk%o*Fb-$EESMD5Pk
z%P1)56pRdnE`b5PgAcmds?SH_(<pu_&nCr}lW9c$IL0JS>G3Zs7|fIK@!ul_S`@#F
zOO+0~IEn)xMV~~*i}b_(M&v4mY>esSR6<Dv;lj)q&3|T@WSWCHgc&~b*g@m%&^y$w
zYI4lx1zC{*P38Tsk6&n5S3L?i7qXAOmR$F2gH6_b?LCy^GZyxJ!u@&+@N+1xIkU7B
zqHwml-fvna)0O^WJ$BBUJba;-aOrC>8bfAft|KLyK#?d&O)m;n`RC8$hYzWrBEbam
z2dQ@KVWg#ss;ZLQ**+rk{i>v^W4W8Y7Tlnzaint2tnc<q1O+Y<-$tr`kCyoc!+WLu
zBKl#YXJL!DyJrg2Lk`N2k`c!r`xCD@q@sL+kraU{q<`K+R)i3OH68NyjQ+I$+>#wJ
zGobt6{-=iG0VZQMW(dFvJ;gt%zakwgKXJGLWt;MS`uA!~H8BUK`@Je>t2^qDJCfM>
zJu3CuZ6SYw1@$HsYl`2L+9t3kYzjx-hItLx8m$qe8(7<IM5<D)1Elix>uXSpu9R;T
zG>d|T<;h!;_DpwakJCZ{pBeSbJllVtsCXfsqj_J1X<?*okz)6%_VPQ&%Y^9oH7tJG
zq3ORbOv;A2b4TTy25ZulzYbeV1<gPGP<vz#NQYgqvrySNm1%x5=PvE4Ac&VTc!_){
ztY~^J@j2++$?pt)hDzD{vhss_*lf}Dx9aw&Pw49UYSJk!N1?6N)2CLRe}c-H?#L;h
zFU6>-sYPq{Sh{z50;q|j%=k3|tqkL;KxW<kX6Z!69s{N_9kHmp@u|w^48m`Q;jHfI
zBD?FZs|Z3OA>+fWDGv*H9&z#7YkHrz@#U<tVW)RKK$Nz2TEJ-O&M|KPE;lH%G}i6U
z)O}(J#SHJ>n5}{^j+Z&ARLoAramArv4)+;7y2rleQn?qLx%(gLY}Y%PXyb{`muPly
zadEY65OdLbSiBxu-0IuZYA#9(ooUTEZPF|n$szkDGyI)A7d1SULp1tl3_h>PIKQ8G
ze$vEYdmI>&ia1}Us?RCS%ry0Cwx8jJ1z#<>ubynf0c3<3AK$}EI@k7Csj}}Vd!$I$
z68T^K4m2nEcV%fkC{ov9<>%-qj4N(?s?zpMXhu7QC!_c6i$ViYb|i`TD$bs5Wgp``
zWNiz<78w}ZPH3b+7jh3Lb6qyP7mg<9T?|67v#gvfp%*WLATTg!^jV`j26M)%aIibS
z*n9hfp%XGRJe2Hn-iw=f=XugoM0Y4T$*qBMG9RCU^D2glV5q2f#P`4}h2TJ=9g(Z{
zk@oiG4(%KFxWOB_7T90EC#__2#Mlzm;hp6b?y~pq-xtm}THQ?nL`JOr8^}(<;oLqD
ziu%CBNRkWltDNBV5<$_xgq)l?ZC=QD4v|YGJdmt&L`F89Ui$SXT9d^Gat&CZog&A{
zNM<UbDJlE^rRhKrGy5FxwFd9Rrz_)T<^n>fI5))bhC?a$uf4=dla@K0JNrI{@MOvP
zjWlc$dAMEup_f-fMJfk2nG^7wV>(#r7lQpz-ajSpYm8qTB_jVl1>k;p*9Q#mrIu{~
zF(u4!k@oht2z1<6+9|ypRH!e^jQVjykFo;JO^Pb1Sp&eVO#F9-Kyv=wuZ7m6GNarY
zULV=@#q1+Nu1iIg*3pZ<xsW)ImVdZaX5h!99Pd&Ww{6s<h%TVXy2}r~i98H`MNe{*
zT}?Y!`pBwYSit)g>Fs#-bXPo6Qvi7Eil|GijDPIe2j5wkzRf3_)2(~^CDhZ$Yx=0a
z%zU+cF+DDp76cBQ{~knIDFhcqYd0<_0BAYo*L!h9xKg|EdDy?Vb01=Q^cIgcEWs<?
zZf+2aiPP?vAD>@#>)9SScegx^w0hT?wMHWxMmN^JGo$BoJ3M<)=DNd>$Frl|!+WRj
zl%l-<F*3FxZd+^PxT(Fn>+Sa#+!{nJ#@bBk8r9}ZiT=hD+B6(j(|zTYV5+BBZ2r*|
znU{$S3CU=2NTCq-ejo%?k}XTSQIF52*$G*r&9)uu(5uWnXosIEUHDI|lX0ZjKkvCc
z@tHUI@u~}!!7HIEJ&-HxoVdeCE>iS=RUWPho!92QEf=H(h>Yo)ca^#m25)mH*?iw!
zY8BHZy2T~alati+ewBvjOY&wr7_$4b%Yo-6*Nvy-)OU7P=8~`brLjIak&$%dyG-U<
zgF`H82QL^n&}cM`8a*RpQ{eXJ(;K>p8tunTjg9zina%Jo<<?KE^In-wAz^G$GjFpL
zkL~j}8LnS{E1g|bB-A0v8(CHQb8VC67`9_lr#mAdbl1G%5@QU1uXu?bsShwfAQ5}@
zE)axc&kYFaoxkbi9KK=xy_Id{9D)RQG29fnMRqZPd>)(Iw{OG1bw1rOW6h0_w$sv%
zcp-EeS%q(7XnL_tAj^i(WC9C2H)akAL|MAKdqBaLZvU3<lNCRaN`(i2pxU7L8|#94
zd86e93k!8#UA80}!-9*coJtf3Q;b*u<M%~Ot(z^ps<UX+mT2tJm+~CSxOt>J0XH#G
zX`Bm0EHCbGh?_0x48;c^Ic2;fa~6SQAM<IsaMsb$dGTV*@FfESgO@23f4U>@Pj=mb
z0kRA^+f2$b^CIHzFc*oR_-}V}RC=?DGJTw8YAqa9=e^e~y<Z%khjw_@Ahtg|Rb(^o
zz<N&0Fy$)66`D5U;J$XTI>O<8EAv9DyZu_z^Qx>+igWZh@~4=Yr#wv0Xa5r01B1cv
zRZ!idOXQ*~r>?m8C<hGEXZM(-<cRZfj2ndGM0XI277&+67o_#c>}*g)MFl(be@qc>
zJ^)3ojwRR<sns~21d<uJ2l}^ir)%A&GWtaJAVb>g1z!=5gLXSdns<2%OBU&PcmKq_
z-J!~oux+_}Th@JH-{2mqbL3)%(9~RMWU8G41@DxT3V*5Qv}^;9!H6Y@4<SU?h!@B)
z{%fmf74D)^@Tvdh3y4AZqyGMW9H?&p`?gFt%SVcj+^Wi(t(z&TpGSv>LyM1A$k-x<
zT<Dx+j;e=UkaZ*~I>t~;uM1gg6t{ZfBgy&GAb4Lp0{j2Sd&{UO!>C<&h7be=1OXM0
zP`X6Wp*sWw=^DCg=<ZOFMpC-FYk;ApyJKi+7(%-7Jo>)xIqR(Ro<HBe@B0>O0kfXD
z^SNiwzOH@k{j?O%XFoQr)+E3H)HTiO{T_NmW^Y@LHj{DxRE%DefU1$Nh2KCw3b<it
z$WA}}eAsj{ywvQO|1l8=T@g}Hc(3<90pj-GKcJzZogXYhwY08JCq5~^IS}7HVLug&
zk@q^)e136a@YPmjpqu~r<8s)KC&$d3xVI+NlY_}*EHr^%I5hOv$Mgwzt->Bue}TMd
z3f>w*vjsnMI$m$5jQq2@NyDdZZ-22wUN}h0WVzj^bXHWGOqrIpy-9O(bjdDIoOVX1
z|LN06XQz0)i7JySK-g1Z^N$P|S1taTrywSNXE!GtZ{m&aP5d(CSb2kP7-MB2vHFkZ
z*>c#(xF=FbOYY1lQUIszMKlnx@b>l&4FN(|if;|YKL;i{t@$tl8XGpf!j}-61ppW>
zPfnUm{JBY|r%Z2tvwzjf>f4aOS}6;}u}bLC<>T+ycYbN=>K^TwA@N7t3e4JPwcO*z
zt$NBRq$i@?($v$hBz86gTuy(?_k60*aH~dY>*+ynCpaML5K__^okkbQ2=X1Fig}^~
zIaAKR613{R8fxztkci*Ez*r2@ki>1t_-(mP9bSNb_>h2)I9v`8?oNCXN=;1E1ZV{S
zzGjLAX{AT>Rqf@;C4>0yPA8^u2Rk+M=p|?v$C0&EIqJqrn&Hn$(#D|rZ*U)^ZLxR-
zCKi`ZC1c{+8oPx8epnh-hcFG?7cIlqoQo^zqmtJtawbh?3P8$f+|;|tuOtqA>;Ghr
zw2%bTlEmyd)aLoN+C(<6-3^2u+IarykK4Ui_SIt{VQ%IpI!Iyw5fg7!S63IatEcn2
z6m6r7CbkBDob_meLL#5$3seyJCDf_IoXdUsM<k02eOCM}h_{%Fn2Lb_x#l!rFwYmb
zq_|&10`#0^Z5bPz<yr7pc8~St#r2l&pP<_E*7Lmum2k;y6{bW%4=n;vXqOTciYh%8
zygj1^455kTnOVB8i|5BM-Fzb|HxG{x3eRJ8GNUZXbQab{#969^^o3{f_0-IiZi{wb
zA~&jZm9SDg_mqvD{m<@70=0;UVz#6+vm=U>qq>g;I{;~T-?OYH`&vm@_++OFX-$an
zYijCGtc)fUfwYEeu+V%jfHad55>}ay(V)cYBglCS<xY=}linDh-L()==X?r#A+WnT
z-Umd(!rDLqpd}h?U70VvUrL*_`uxCdzX>oz`epLvT9$3-93V3*8DP@lns%_y2)`gJ
zLmPfu=R-viFd!dqEni)Y^ULrN!0rY;`|9oiPqhf+0P>#5dwV&!xI#L!u+ltMs__N+
zD)<2EqlLr#QvWRYtz2y$zGt8H+q_&>?K&H8RTJWolh&I>N2a)OL;KUFfiXAW`@+bi
z>ez83Q1k)C-9h6~9kBn0N?QVu@}gZsH#ax$=HZTxF=!wnX9mK>_5K?Fnceni`);wi
zj&W7HNIMB<`NWR6;dgPGeN9ctZ2J{Ia1-*c_$!v_klMh&A1orn9ybO=JGP69%U|+b
ze=jTZaTC83ZUx81sj`&;KWtTbo=`;FiI<pIU_$=ZC;A9aw|c#LC*2iyRE_1?&V*^q
z)nuVE;NF=H7h<;Tgin|aySdc^u>=GHalS|P709`f_44wfkAY)T=RnpS+>h7r1eCHI
zeLZ(vb6Zm$Q@$8~%fZlQdzHQk|8ywE?*{TW@TY{`6>K-sfWQX#`SPcBZcQCZZcC#I
z>GAO2<GKAlch}Oi9|K#+T7A~d`OkmbkzyFkR%Cj3ivd^_kcX+?pP9t;JtgO=NXU0}
z8E|k_zdS{e^14-wnb&Br7*=Y~#q=4o3{(j)4*=P`AL%0202^#WM(ViHAD13MDde;r
z)oyOw_CefGt^3-|Gf4cAtNr)qVGq6<{M|~H_z$lmFl5uV$R2+cp{71;Zf!0r(=~s9
zt(&GR*6|aM{4xuWnI+L+17~L=mz>kIrd>ZrSp$$XWoKu2-UV=gP(~RV&#hKJlJ|fK
zWXlJtGfSu&e1&q6*4A%a4nvF~Wt2cPh$OUhE3d4K4gN<L|4~OT;CFm{a5`jBr@^Ae
z@RW^>O%js5pnrP$8ygSjZQ>9>5-q-_`I{}COBk%e1bkGxB!E?t`@Ssv@Y=!I{y8}z
z;r1bW+pukN7zHN+M%J!@yE|{o3CJ$zDzN$ecwwo@LCK;!co}L?s|LKIXP}W2$?Qxl
z=JWHb83<`G07pIm_bz=4sI&qYfkle@haW_-sas!!>Vp;R>Zy)?iT@F~C-0&7b6+p9
zog8F-9@lc+2goz2*CjjsI8Nv|ZLcqi9vcQ<rePhrJ0CGmsh4RGJzrpW6<arb#X<c!
z3L2jO^CaXvM4}=#l_~iNU-JgkP?CSNZoUq#OCKSa9?Grm?R^)yOzG^xicQ`5use)w
zIl0)MR_OM&dw`59VSiyRXD|PDt?0`WqadEfU{IIQztM$fA|C=O)2-}(;!^JP^=((o
z{}P@WVTf#dv;CExT6BzNU{)MfvlkVWIn{)fmygsGUIZMt(f}9B(pe9^-fZpkkZXSB
zNFJJ|<hFB31^SFy#zwJY9V{lB2wd!sw~Vwzg;5GEA*bkoHa>xygNbV^(W<KI=05L$
z@mIYFUljhmk`giecS2Emk;U33kktQt5&Z61W4~(nHGE_u?<SR86|k9mObhk=0s^Tz
zn4;JXW3~L8Hk8p8O-aCH8xH31mFuK4k1WTrfND6OqlRlKGi(C?b@e|NN(mI&5d`Ym
zP=7up94h}ia#;efj+K?@r~k%B(ORn1zX~O$V9))Z{Q!xfSK6yo-vUz>fnliE>87!c
z+^(B$@<$q%k&l5@h}sx>ztP=~#5I3sxPw<o81G;P3BUC)|2r*@Li|lf@06DOjr2g9
zR`1jLOwao}TQkUzrd!*Klum)6cC>HFf9AKGtHZx(N8VD{>TEFz*gORUza`{<s2qd1
zu?>CwFhP~A`$_gZ0sRKvL!#td9>4W{x!n|zrHqu7LMuzVsd=688wj3jn-fuj8&PtM
zN?YaBWI`AUu01`MqT54{Yxe$u!%uPfZ<Tf|`a3QU=;Dd!(B9w>c|sFiK0Y(_^cFZi
zB(ybrL3YPNmuF&Vl&xobyv`k|*9t6de2e@=x#`{7t5qpX&+Vb^>v7AA39QzZ!%D`t
z&(E!ALHgadSfD**g$TgS|5?%SO~hzB<uPJ49q54$JIChEM=^@51q3c~*$@t6!<*#K
z23#!<PJP;FNJ1%p{1FyQJqnFHd36x}qGWy0uvIy3Q~9Q<P1X6{#_?R_{-H9>?Ju^m
z_rjVEy4q=tl#lbf3{Pqj6(u8A?hZ{ee~C`~v=O-wILpX6SZEn^jJLrWC1Ejrf3RHW
zEBozCku0o=f*!5qaxHXxt~_ZOo(AFEA66)p2m$-Kl3g6<`~J`ravi*Dj1>6O`NI}k
zK7X9ku)$-JEIywOlUe^$16w1jKB&!`9cZ0EIQO?7Ssog!9oDmN&ff{?-fxgWR|jc1
zh2|>26*XnT^tF~(Ns-={_W_acT4(}rDBkM$eh!Wcq?CQ>oZl0NC~^G+TPAk)m|0){
ztneHyi4<FS!pb}D{3a}~5_QOVPVRZJ7IAzUj<_gfY=79@G1a)l>|v%pg>|@Sz<+yM
zqhNAL0b(JM1-+RSd+&a%ztKJz?drlPE0|756)aO<J>XjUuH~W<D<&vuv1RFgg0HbN
zz4b)Dz}}&s>suTN?y&dKl|r!lLUT;A5VEW8QyL)?NBYg`i7uv58|UVZ*z;FCr41(?
ze3bVGnKyr2?3Y4X-$fGqCPBGZBcEZn6h4#xwlVaGzSB;W%hnX+_IdH<SG7MSZ8P4=
zvTKR0?u<3zPj2e35Y7|=D>=^NO1J~8oQHba(4jI-x5M!5Ljy^JENSe(FX;NB>)Ee<
zhW!+;c2hvzQWQZcS6J&JDYLm-*iC;j{BC09!Cqla8QUIsF3^^2jhp;pF-+1|IXk&j
zE&MAOURzSmGi+w;dnlRR8k}EkEZRF%igM%k(2m7Yi2kEv6Swg+NI+<cnYp#D@ikT~
zB&fz=pmt`TR(w)+a7%FM&&0(U_t38!53gH^;79eYUY}vlA7<R$Z_iA{E0dr5y5hBM
zxBEmi@wuJCp1kDtW%;wF{5YT6OKUilFJ4F`qCL9@%uuw-6uBSzIG#CZ0W8e39<dG5
z+^F=HP}OE{JZ5Lld3oP6aDqR1Slq)$8g+U+hCjY!v1`)m>64J$d^a1~Ixs|nxo9T0
zaJ_cZ9@+Zc-f3skXh$USj;t*n-wf?X;vI`0ro<tX)aWb!`HUFyxm3afq0{nG`eZ`a
zH1}mSn+Yot+q+AjLB01jLgzG~$l&68kITEo9$jtcb+O3C88o+z=EOUJ%A47f$eakP
z#MVG(ltA#z3RObCKtn#3EEDbB*BY+@=iWumP5K4SKlW}B6?q!kf!(>oK9~5eVX~>P
z@Az=KJeEWYXxFvJ_@E*VDgS-Yyi_*^w-!ENXsMkAbOm1bVznJPCRodoyj1^0@gj6O
zI~I1l!#|JChd<zlOG0AEF8tuz2pOntBwN)xBE8x0PB|Nb_B*QWyK7IxqX!CZ$4#(b
zl%LY@UxpV@i|mSm+sxdz(jM~iTrDZ`m7CKq^Wh}LAcIEEe0+Yd&$b2WU5=fb2`DVz
zUVkdq3%Q|zP2_cq3^q<|#hH{3&*4+G6Xj*zRMAs>NLfig+WghmSmAR^WSda%hwuI&
z2!66WRetN!+v09(%iH82eXPTVuR&x4jh**$31dtn*jc(-8SVhVOH_2-u3B7unxgp`
zo#wmQDak9jP>BiY4DPqBYw6s}yjWPMX&$WZ;;S)^wNEbvN=OI0q*^H`9L?IdT4!7E
z<WCX52jJyWk;koO;{tkKmz;j}o(DKQcbWzUSKHfEb<(b0H)EP05mub47Ph->{q;jB
zVw>S2Eb|Zu^?Kg)*{vmI!y~`Evi5j~Uotun8FS9!0VTYQIU151`bRH?_%$q?VkV{a
z;hw=&X;!>laHp+B@XNQLxW|G7%%zNRj|oA7g2#EptlSlcD><6&k;8}ytm~0;3Tt7?
z-x%`hWP5l#{7D4ozj3AHXR?hazAyIt)P>3GRUfqNc#w13Wp-kM@Em9EtxnqN+itdA
z^5{h3{t>X7A<~&N7zzT~ptmL6ka%U|J~1Z3Hx_Gt0E$2kEF2-uq9xV^EekH->|NoK
zf0~F^%??flX^sg`<oL!fXt!#}nIhGPLr$*IKyR2PJ!kT8weY6CkiJaZx`SaM)wDoP
z?+L}QsUf&hhpwwyUhD7d7B6pQ@9$3IeN9XC@Y}q5QzPXm*dS?q7f3TpRx_@jXc&Xg
z*`UXscGKoi)-I5p?)2&+A*-Zbzz9ei@mYCJkZIgbGmG4EI==RUC4#tzh+D@NB%Y6Y
zoW{BY?;ak;>eoq{5l~aKI`nSYpv*5$e#;zu-;@4_$2%uT>nq!7o=pTUUaK5+p{&ON
zJWo=3H$W<aPtR-gIv?JS@okqU2IlGnu1R@FV8Ij9e2CtJZbArHqRz1`f9!zmx!{7?
zQl3iG(Jm;(HeGOISkYl+lwR*;qrSCAp!4C|PeIG?(4&8{da~}*nIos|D9KTr!V!p7
zFggu!pdsoL<$<?)JjERi+tnAbKb|Cp9B%$wv`t*%G#crlO3e0;-$K%2+%VB=7YApl
z>`&f7R1lB9Sts^GpM9TYv433IID8(4B#jTd`Q2z4=)Us#uoctx>jyMY({q}<v)G-b
z$<hwk7^3Ls<-;fLht19s^G}yQF(Z%g1w9Xf*d+`f_)#gfDPdLe>6;JtQwif`zKo6!
z1`18kaG-Z*n`8!bO_nJF`6@9x9)bx-4J<!r^*N<j!Nqz5r^8ku>#LC@ZectSJ^L(M
zMb@%kREuS_uJXf`ropWg%$fW4!<5`BHe>kmWGBaHv1_X<!F9EAegt^F{9>b}`o1=f
zJ;JfoHu-z1>E0i=m8KL-vh>=PT)|V3)7x-K7p;Zc@LjQp`}6pzjhG-Qf%C85Q{V4g
zjOfX}OF8ttKuNmHb&|%<=`d%k-_?xsXm$s^P$os+3csVo0<As480M^+&|*U-BY1jY
zZFaV0Srtjuun4266o~q-_xknD{{D(}WGn_gukwHdO?aO9@XL$FeUU)g!9Q@UcV>|g
z6XUWtz;IW5#_aCW8(8#OS|@>3eAbCfA`<N_&qKL+Z8W{``_yI!mDw(M7(qJ9#GuQk
zH-%nN0e^P*-tzWxbEa8ffJgoKm7kwMHjL56C!0=ykJGK+36-9G(BkX&C`j;T7cHo1
z1n*Il{^X+f@7sO+h$&x)?f5U!)|>V1Y#yXNWdzi5b0eYDw$$BS?mE^xo!t9$Xtu=n
z)<W*&g_Kr_cTP6x;jPh8ycxTXgPY~)SSvDq_^O8!X^Q|}?|SF{5njwZe}2D4Sx#|j
z*GrGIw_k2{F`p1}!21aPeG!y)wSKqq3GWlE!OnGeTC6Ve>}tbpp69BS{O10=QV^ss
zNT%lKbbctw?*=^^>yWFn+VYP3vofk&?Ms>XRLS8|xaFOq_hokeftlGK8;#4UjyGww
zbT8*BUC)jBk*#FYH>@9k#I?yYx3i;wSLmWapX#j!)0itJb&j9C;OXDuUkQwmj0{(&
zI<Y@%trl^_d`!Q;>Atc4B>mULqZX@c_MjI3k1svcoeC-mx_?Do)#JA`9yB3fY5TG}
zmx}?ElO0mT%fzJa_f?%5SJksdZHv-Iq9&$CSLMSv0Z_Yz#SVThA>XStcLBeElWWCY
zF<1P?;yq*Qj)o@P^UZX;8GQL$ThNW(E4%2%_pR4$nEbJq%b@dCOT3{^)a{&4W=zeH
zx^=!s=^Zvw^lZ_=oy(ztl(xEp`+*Zif+_eb3_*K)Jo}u3ylTD%w_dO8!o*rTu)db_
zfmTup54Lj>OiD>znru?Hz1VpZ9Sep7-pi6R3Le_8Hc{z+x?XZ5!AQC5#MZXB@6Ia=
zG_~>%su#BMhn=tFcO`-81;KVp0qMR=R;(vI%$p~NKD2pPQ$Ueb?U|DXd9@c}Lf#Ok
z)gSq$c%x*C>sz=EF6o|b_diHNYY?K^Db@6E?G5{#D7^!%dq{4MxVHw@@CSAe$auXz
zn&)&DJSo07xZjldSm1FfKx8U^R=|m88)Md}#OtYU)qZf%lJvsWL4MUP{IwEZw$x(6
zZn;mNA{jNwvTG!Mn2_)Jl>Pj1y3du7j9r`e#pRxldYG;t5eE0;ludFOTb?Q8IpQv~
z$2H#<o|$zWG(tprcXZsB2lmC|o;zYvgJTv!n^X;L4S?@fqquY4TCjNq$o$1So@JZo
ztQoLWbBjui(8_o##T3A)*8M0hF0MY%3xecopuie_%p}jcsw8}W;8))d`L<$GX*{aA
z_c|pKd+?0!k_v9J`|9g6a}Za2UZLSMF$`bAI?-%EX)a+_P~uTRryV+ru+9lj{4;EV
z`9cl9XND_Hxz$^8WZ{zB9ab^di&qylCspko<>dFR?zU#B2gQsd-qLg^hnEllo;mz2
z(pRAPD}qJSsi_9A^(!}&$45JV)(%+YbS63q;_jb-*wX(X5eJ%%^Qvr#W3=cBW2wZ@
z`0<v`b4JsZs^-q>^N<3&jGx2D`RG&G_|1FZlr6hCWZVw+-4YD|P?cuZri|Dra2_WA
za#%)R-upxHSgZ1_@#^}A6$g<62;Taq>$Xa=qu3t=q962jd0+Z!x!vC*!hXyLeaXXk
z_GHzdFh=L<5~XA2J2tM$2O!rRTPFcs7|7+W<l^!Q3;zh8unlC+YB!v#;*gznWGRZh
zLo1b@pouj(cJ&58eh%KFpX!N#cEelSC&dqUF!m(?I4JAn^EV%roAMu%(0K)*+*9ye
zDA0St_AM|KY<t0w9sm=6I~j9(w}W0o`b6{tJI^&g0S)1p1{whIQC|!n{e%8ox^n+I
zN~Qcah}719G?NChN~=-Ae{)`UUjqu_G1M6e?%fYyM+n9toraa#Y|GVVk!!^O^U|Qg
zl>3jsI3=#s7_73cQ~Tb#;Q;6g5g~u)cuM@$0C6*k6RZL7GhM@FJbxD@n|+fCT>D>)
zz<;iT;pKnKqV-T5xU9`C(IZQt9lZ*WFa!?GuNNVO-iBggV#%@P4k!PB{r_60BL6=I
zBl6#SSH1ob3IEsbkN&qU`hR=VXu-ZNbFP1N6IS;&|FR>^0IoLySUI|zxPg5b|0a<6
zIHC9&*r4NiASDf`ITR+DEt*LX?9;5cz7#Ls4s1uSZDg0>*p5@HX>fB$50@lxD%R_O
zx=PQ&yY))zm%7*VO0$3i)my%o7u@6I0NmMi$llB&N2gXI;O6}QYs3G)dANeBnSV2{
zwGqE-HWmY`qFkBVRk>QvG07&cbslp&pCA-LD7!?=A*(L##;aE6$*FzCbie_vuU)N_
z3{>Sz(9+9iOC|drHGaGpm|SOa*jw2m{~=6X>sV&MRTgj4MmN>QyV<%&Fs=x-TB4gW
zt+I;3!sBj4FQZ3owI~an^6l-CW!MZGqGHs~_l-1zTEz)PeVpsm_|*z$4NVwoEJ0W&
zp>v&UDL=DTH)pN`vT~+TFIia3J=C8UDLJn>md+kY*y<vVNzGWkRJS_L9`2H5|Lu+l
zr~C85icJHmXN+TPnzTzoqI7wTxUksTx1g48yC2%3qoguUhh?Z7o1p9^=5_MT`rZ|N
zI-Ee1*U>7K68&3fIP)OVKkpm#;g|Gx%~d&_lOBE1S*zdbH)VTM@@iyk2QtqBc<qS&
z@budsz#Hin8{$#dXg(C|nh(oxRzM^rgxFQpA12~NF=w$sEBzqW3=A?;C|b&k(OUVv
z92q>WAZAchOw2q2Zyzg&qQKWIiG9D?U#_{67_6Nfpmu2fZsnKV;$96-0Y*M1xA4RF
zy~9ra@0-0#zh=7dmO^n|8>p>ewyhKmgu-U9*Ru>DrNT+$xBdhU9gQ2S_D(9F_noDs
z6~#X)W3w<>URT6lOC}fLfS(dS74LlaSSukRc6Pvjt^Yl#vGG>I52iDP4^PNK>L<k|
zRFYm*H`gBzW`&hDNXVfiuc+ViOtRZs%)1*Svieh;uaLwNaifVDQ8c1oX5E%E)~^(F
zZcN+k7i1^m(SGBqvLS1l-`;c$mgW*`eP;UpR!A%EH%<oQe7)T#dDYyPnKC&!T}7*E
z@IpoWB%@}2&4uevHRR+jCHE8)7U%3ENJVWGFi6q=L(22Qq$Y={lmsqSx|$Z2BqTo%
z)n*$wXxFs0E$evj9Z!R*<zxMTa>5v+^(T%AP7CvV&X`sX<Bm8?AjZ{zsCqX#viP~`
zFBC`utLa#kO;Zr|<IVt9zOpZtWw%50q-y41sy7_4v1kSm%YYI;-FlwgK$npa=DPp)
z>kvbb1m>3~95JOkaI2nRM#h_IVzgdnwyIPZaRQfwmAigUdKP1)!=IuqtCM1o@WacW
zDsg&6LmDa7@8yVMPi3~IM%PO<n`rXUKsmw;ALnjEicXz(uWq*JT3a4lPt;+QdHq3j
zSvl5Bw|aqr(%rX3Az>7%m72`)O@iuA#<Zp4Yu{&NMKcfRLVR_I32+KRYkPHd>UH8u
zhv!?J@XcfgHNUrdd&6l~=Twi=-sRU@*wxpcrEk03@hQ~UAIW-74jsL&sX&9ykhCgQ
z^4IlGE*w}b6VE-8an*&~IMLb;Lz~KXeo8Qa%%^S~PAlD+)oe6zRaI`5rq^L9bu?0;
zcEY&Is+lj4C$1$U-}D1lJ*q0u$!Hi^O$>Bje$4G6{)TK~qC0a&h1|@P&YJC8nw;KV
zNyF~}e?h6R1_dV)P1%ia^m#-@huEnuFSwkeUC*LHHGdqB?aUil7{?LdmYZlCa%`|h
zImBV&mQp4i_gbLp%4e&o?eAQAU)8p)I)BkNQ8aru8q0-cv^-19mvDf512N(gH3NNM
zmspy1HfJ-Eb9@uiQ=XPs>D`s~*s%_#C=bP_TK>GjZZgZ}Fdq;tU#^#|DQI+eUPF<b
z^NSPo!!PVKyx8F*E74ioW+@W>;;f#h_@cSRV;Kdzx0czFzuO;=qtIS>8kU=R!bx&C
z8w)pzSspyy;3G$nFK90EMy0lly$Fjt3MD<9t#Rw2Gg&nRU3nOrSi@PTtLhTU2hMbw
zU^{7#ImhdSpW_NYwqA@!{yTwBa0a!JT<}UuDGqfX0>3={6D^b==GAT7z|W-CcUrCG
zqsDBaHePjx{1M?+Nl~gf(^~BC9VXctI?(UOk;g`vkF0sWGg+mbsH!N9nJ(_ZG^drn
zCt4JS(#&#W#*0^QPAGxR{ZmTz<e4gjm<N({_68f6*PcGa$-yw_8b=Cq<s3c#;y>Mm
zQ^o063(5#fYw*_mXi2d8^utj{yu=g#N=NWh)c1tuY`IN3DlQ%q5sc&<Q7y{}X{Cs{
zrw~t@<j$XjwXo8c5c;nYkLJ%mS;>s-E-P)O<vDZ^DMM(n<kYC8zsMCD6wNk7SZ~Yk
zP8$U8GW4$bZ%>^N(G|bZ{4(W8RAOtg5eL^>!l*cMgnT@gRnQlC#s`7LzZ0m-2*Ty7
zr&cmj$pmc3C_0&Xxqq4YR8d}9SW4qt9#MmVD8RjgghDx9N=aaVl|XbERXG^KijE86
zAtC4lQ(lDH+EI}tV=tn*#kq7Ah8WGYXc@Bzwf)cZl*A{lPTN<b<(Z;VmuqYsR%=6%
z_97uuH@cdQuE_B-+!97jo~von6J<hTRyB1{wUS|SRX?kF`k%1bJmuRT*sUbNPhf<}
z^aW>d9*Q0NI~Y##1J<bMr&hAY(2^y`Az0HnKB&Ubd-pnZK<!YIBu^ob*^iC=k^6pQ
zaV}DN#x$r)3iHgtpa;?!g8A@wI9A0cn1|!=s-XbrLx&lQY_O#!4d4XZkkl=l8G)Ec
zT7Ge;=3aq9LU3+r3~_Lh1tW~sXuzG_5?cpgzkt=~brM(JtFJu_NkY?;L)0EqB~$Q7
z-m<GNh^|uQ{X*s>wt}zMwMhz#o0Fv<7Hc$A%;()_6#ac*@fM5)SA=y%UHKJ@omQ&5
z5{|s%Yu9?ks*X7!=BZ*gj~WNw*9~;@_A!THs>2(oyn9}}V>+Ah$RK(mGGi)u!#JzU
zws-mUw^5~AS9I-3C<r!TVq~%1ABQ{laR#o-ljruaH~dg7tOt~}W_S^|iO#^nl$H`O
zp7)J8JOo>G;vp9(wxAePmhol;u69!*vp+<*yRugh41-&3r7yv|RQBVmb7xbz2Y6Fv
z>4$bsuIpwtub7K;r4k~_gitb7By0;~x1*E7!gv*_%gqa=McgeFugDA?o+VS1GO=1;
z%M+y+gju+rJT3l)rlIzHZs6ss&)T+i=KK+%5y%8Eh*rXKo2itZw&j=NwDJdHTodye
zHJt)1t_{oiz=qYV{e?P&;Ry4h&{foX2^x7Uc$Wg3I*0*eaUz%NfTDksKT#2)Mp=_0
zKXI$HR3|Wzz3G&5xVwzV=9XU@)k>`tn(<gfe6)I_*3B-Ln7mR9!nN56v+$mzUR>$@
zI{MUFjZ`-oL!qWTO19s$LrXx@=DuHrH3I~MqFqC~*9zVcXJFK*(mmZUc}FX$)427T
z?k$~^4t7S4ov*}%X7F34JuQXrH(yc3GGOtYNt{-z1pjtoumezRJ}rm(;&E*mR8x1A
zbo%KUkIvQcb6H{s2GxE{s0-Tf$KeG^q-X^yq!<!%Hi=@n2n)YO2tST@CS+*m!y$@Y
z{lRW6F^m^S*hD&iVuoULj0Lr@d2~{1VK0IM8dZ(wOR=Kd?*2MT%&OOtZQ)1l#5Va{
zOMhzInYk{g&g;X6XV&W5wp~50L=Hp^#9NMFegDzTn8iT~IY+fp(#X;**hxWkl8*rm
z7h-B-v0yt7q#WzHZ%V9#Y~Pq5we@nizuze4WfWk7yDz4&Kn<mJ)u^M2r7~?N3f!Cw
ztIj0kO(jpn5|aA+`UcI5PS)22ccPZK^5aO%%9_i4%~h1upPJgjG$nne;b~k|5{uT3
zrF=<F6rH<Cd^skX5!wm|#6ghHY>rpQN+>ySrNs_Pxx}&Y82B=+tuYbMOMy>bA9~CH
zMR8<-lOh=!+Wv)j+VN50eiet`k4>K0FGC}#=8i|)JhV>~&_0ax&a5|MEQ-XM+2}U&
zE`Ju#dfJoYqU7WtL$iWQW%Mc2S*Ofg?i%4eZTWh{MhimA_@Lhn#D`OhX|<WbPlVu)
zwPn*dTgOAuiY<4pced~d9mqL?X1}I}R>h0D5+v;j7Wx`GDAz32m)E?tddYQ@*~d1|
z&;Cj;$fiQMF5-iCM=85ZRf3qSg5&R8=jT^Z5O2kaIFmqTR1DtX##e?utvG{JMOJ)0
z_-n?nuGJ~o66UgjnRpYun|qao+eiY3z($FlVsNK~RBhAQl*`eqOuOjw7rCmWaP5|j
zT}2=3z16cTD^m5$-9F+knGBO?(|zT~v$-cRG`x(wGF{#5I?0V5mB5{1YV*jqS-+i<
z9)d((S86o*^v!*7Fu!0O8hfy5+zW;>{!WEE?bEahA|~gPH$m9psini8uqwF4DAFqD
zFsyl_CU34kMJT{zwH4#4e_{;Es0ZbhV`hBi@;E?!d5%kAowSH>uCWvC4@hyw1vmZB
zeUa3n^|)fpf^${uqC^LKCHHFQi44oEJB2TO|M$n4fSZ)IRAV-*4If9ee#A38nNZs*
zPsZ2ZLEP^Dg0a6Q61B0bv*C~Utt^~~4a+?nDX522>T(=O^XzWg3+c6Dx56C_<K@oh
z$q}@C*=;J(dPU`*RluzUy_!b@ges+^wVcl8(gEpjE6xy^mBED%Efh=9CXI!rZ3nWP
zc^HY2SHy7=Z+56usTk<E7<pO8*t1}7s`;e)#>9h?jilzH*GXqClW-jL$KzyF&0BsW
zhpy)<eWVx0M`61qEGmUlN|?7<S!P}Ap~{236^-?&F3u{D#!Y&Of(F>`;Kvwvmde3a
zGPjYl<&OXfn<Fh)d%F@|Zcznhs<U++Tr~1MMo+0`YQDs4zqZbtNfOSYgx6mL5)<;+
z`3o?oONO9u!6z)ZQfY)T*bU$}0aT}MCZ%fgYtFV9v<tPxN&eRJl<GwoOi3M;Xxj{K
z+kQC1*UbFCP4;g^l|>xs_Su!-+K{My>uwcVg)U8BiasAk3gUIQtiYk2Z+Hr>+G0|>
z%l6$#Z&TF<B$3k_PH_{a8k9D)LU#JL3H!TEHv4h*6~oeaCdmg1t9}l6bZvF+oX#q0
zb-YbjWC7?QNi(}%`Qe7`E?OAWolj(Pt9np23-ug3Ez<iFWG`OK<#@#<yQ0!4v)2f$
zHn6zgz9`vdN|zVJYSgZS##KV2)`AyHm2QZ$d3DFyhpo%?ulQ>bMdE?9$!ZKqzD^=K
zqg?pIkfy1WUlDUt?u&D{R&{UY*($TE5L3SdW(lnGt!fX=a5_FOu-GhKX|Lx+MdXAr
zttVGFpW&dw-jtaSZ903F9Cp)>IJjUw1W3Gv@vWZq?u7XGs`?zXxfB7*^^qY3G-A2P
z!e}O<gdutCT^JZ3t&B3<KH8WoGL<Y<8S88g7A@Q|$tlJK2Wo!-%5TK}sHrbYBLS!u
z2!2S<x+H0K+xcY1@gx;%!cTArNlejSk|l7Jqk#tlEaJrb5&LRC>q`3{t@r<gv+{WP
z0jnMjq;KSrJW{IJjGC!Q&iW-#W^4;p$15vx<%uF&DtE+Qc#1V=JIwk#vb0f4YW5k3
zT^@?Fa)yrO(|qk@HglN0s~S*T55{Xyo7!%&n1MB(ny8=FM>kufmN9TT%1H)G_Eoi%
z?>S$;zk_XW=-TT<l<RE~T&fCzq^kkHt@YA;vsUP|3q%!JL)+8>c_4gLu9?0g3vRl%
zfVkWliu1PENgcm8Gv4oYzNVhZdy#tKmCkkR;VKd7nzpp(Y&BkeQv3$CAOAx^LeF)C
zwJ5=}%bB;uCsk}^<`(q&J>WUZBd-wqXX6$zpc0$!<MOoiIDVL9-{7r=8DMp=ojy9a
z)v8Y$+!hdyrvAVj>Lmhi_2S53J)ZP0208hmSu^PW`94(anp4A8cDCa@AA9cZuo6;r
zd=ipOqUIOAU4NyTT3d;AH$jsiUu;0X<?{Hj36GrnU4~T=NFBp&e^{NjPz(!Jw@hFq
z(&D$6?&rkM%#7--Ui8lJF1Ktel;wH>3Kf-YL;JRqwP16Q1w-Ke#+=Vf+*m1UlYBP=
zk8$8zR&Sgl9$Gt52zZPdju?}y!;31z+35Eh@T&=3E2a-({cS~ww!H|JM{EIp7Kil=
zqnfs++Aw0T35TbCyc7F13-qz-#}jBWe%btYy(hh3j-rsx8gYq9LJ)aRGZ(C|Up)zK
z-M*=+NzB`D_PH^81KdkIM9-BJ5a2xE7^3kQ?M@--X?d7N#mkaZ3IWGWB@hP|UfWWy
zQt?wy_g$xu4ac{kXLKNT!KCNUp44v>^BrlZADbQ^yxrscL1fuV^uwQ2I$6LVJmBmU
zDaj<1$PZ~I#Hdqas=uHN>0@p@NLRbEs*fldQykH=a+`5p>8~a^m@O(3H&3BVsNuGl
zT6Ico*(cDJa^9}kKX)O5ig5URkO_6EA>+$np8L^URxgY$RA83#`D<=Toj1w+{!~^|
zNU7D^+G#re#NbViw21!wb~cYv8(x>I>;a<HfarLbWnx;h=}>#%ZXBPGDg&>TE$NNr
zO&NnmOlq2P#>0z156LVh>-Ou-sEoovt)%2!w(&1$*v3=Ref}bhR^Yf-d}hpx+`_yp
zHF61QrH)TRwfTfvJnUw*gm8Rn#XthrKxIgglVZMVFCA%<xxCJOvZ83U61Hxj_Q|wU
z8Ncn)Err$Dkzz8u(|Rs?m~L1zC7T(E?`MKEq$=-^O;g>h^N<}sf1JvWFPs^oT3`sa
z)^0JiXbDbk6|}hdnpK~RdtrfZqis@W*v0xFHe~)WTkXfnWcp&I*#{4mTv`O(H?qzp
zw)^phq4l;Dq?FYbvI}`~JJzcyxkJvTijUSU<%*}{RPstcEwqRv*BHVI#qK7Q=%=;|
zw#wg?7W2k#0V2z?;iKgE3Jc|pRk;Cm%E6iW=U=L!LBn%P@+P{|3r=SutricV?+teH
zHpq&}_q!v`5l*aX)-tnIJNhd%SI>8k1EG9}Q*HGH@p2U{2RREb_p;vOVAd!~Cp%89
zFqP*zB$e{y&i#1(B`fsYrKKD`kA5+tnG{3J4Y3$_;JjLO<ynODzPz$UwZeU<#L3xZ
za{Q7td9}{EKU|X^l1($>QQ=f;RP*Z+JT9FHLpaUdkTvrEy$AP6LBbPdvv>Hy<77`G
z`G`0@)hwSSXS9VNIY3po%q2Te&bQ2dK%MC5yQsaOng-5V0QGxXGbj0tUgpG7y=Fk@
z`XH|*2Bz83D)Lou^)&;SMEr>z*QmmW7{Yf=i1s%7;LixZ89Xttb<;<|OPKWU7uJo1
zUSCne4j!>q)Jl+n#ot|wjN6<?s{Y&3c~k-3{M!?7&XAnUaYw)#wYXSASLa4&yPW;^
z^HmNYmLJKnIifNIhDy<m^Wm@yL}DMK$Ok2A)^`uv;3c=io~l!bha3^_GNFy;vVDww
z9vKYu1My1H3E7yYbo3V`%@jxkERy7dZ15)q0}Gi)1gU9<R^H6bZwSV>iBH!43uuFK
z%<4kkQS3j1ar^i=X%>{q(la6iTX<(mBLh(O*j#tW31G`O|85!Ol+$;X5AA59ko+x@
zIJ9q%OF-(NGOG6#Xu>|<5gUj~#Tm8==N7Ne`_H0+&=qWTuyu5Q$ke$U_7kI&J?Fos
zZ6?~5I;s?$jck9T0$tIT&dny@?`V$jwHSX#{8WR1q${7;{dfhdV`Q*URqzrwkF2h$
zVkIYjVQlrm)pP2*1I4lgR3_{=@GW31=i{SO<1kH(MPv0w=H`E?F(?2}-B1BzD^&s{
zO_lKk4GJnd!>-YI5%)%dww}fg)4=+PACq$N6vg5fazSBo#u}4^a~W5>d0>%FX^w~Z
zc7`*hZg;cxMR+Z<YCB;4t0orI4LL~M5=xoqg9C>Sp#P|}-bmZ9A~|3PTd1UAB;;VM
z4EaGTT%I>@&i%<M6;@i&Qeb&kIQ^fC%Bkgs-ih6*L~L$r%=0mQm_mpy_V)vSEe)Wp
z&c*5!jCELz3?PzkpX7&!G)kG-6)?K&lB!aaV5_%W@cjLKJLVrC;i(kW0qK_p)+X+o
z<AZx4gsJRUv}U%>SgzDP_ykfoFl27?SZT+(2RT~m*1`>X%B;ju{gx8E^=H*0_bo!(
zM<k5N@P=6vGig=es;djS(RIbVRTd)So(qX<dI^Ml?i<Q9?3zc`)!BRH)n;ndDF(ev
z4MlCblNL%!awIq~c|1W^gPrv#71~C;mIlkS795N9+x4lwOr{FqqcCZKf&L@(9DLmU
z4lSOt5qy8LUC<v`#?fwCR6jac+&*r4z?p-^P=T;x*Q43XBp}PY!+0t}PCA%*8s+~E
zirc8eBR>Ig@An4VA@VHv<k;6+X0m<D@OeP;Gm~V9{^Ns^xm36;lr*Ps&s~i*0n8o#
z0F=|8c4`S3*jL*$V8W23H`yRTLzL{LjLzmzNksO;8p~a#G3+o`WJ=g4F5=MKyfu|9
z2?(El`-P~spLaL~?M#R^?uE2mGHee~j~0@xf&M(sa$6vJVjm2e#N?S4a^7&ada0vP
zv74*QmVOu1Rh-AZ$z{Ss#VQHT=zlFSQzE4Uqz>xF&Xc;+@YB8YLljuNRdknbVXYjw
z+42BDG)SS)#)AYZZ2(_GxfJiWWGb@a7Mbf7_bj@>44kR`+P?jGd1~o8uzx@&D=%F(
zn$6(oZcl7l^W*2VYD4ONG3>Ks*IyJ4Ug6qcmxEq!{shzmT2`sVa`avj4Sb`@5tLNk
z8s-T)V{0Ao+$G@7H$K7pz5F_)N8tNg=I7WWgR}m}AI+=k$~5D$rCiJ{*%Ov2c_A~O
zn@Wo_`80g&h(XKSNF6o0H@pvp$EK%7Dc1Yn+s<pPo2QKngAmc-qvSQ}TS#GjHA!Bk
zYK6pFdI1XTk7-lR-iSq^RJH7$tQb@D-lZnIq&scoX31Tu?xLkmkWH<CpnH5NRjirm
zf%zHN^h+lrHPe?&oSA*Y!?V~_9PVoP9<_}dF0Kji;a#mX#YfA%pRwZtH~t!V4x?#n
z;l$>56iIiOiN(17McY@?b+NyKLH!Q#0D_IF^Rh}?;0<YuGs{dS7ZOJ3nCJ7kjNn6q
z&Nd0QwOubrR;ds^<U4lPwXx5HTx_~g&r!#lEQ2QPg~WZoETm<lZ+W%qiZ_y3G@~kJ
z*Xwv8e9VwWG$!}kITH(=<wGuC>D}=Sf7xos4o8QN{`SGLrdw>fn5AU{{A&O@%rhJG
zR4i55fmnYskAZ0u@EP6Z37{O9HaF~rzv#_{odrJ>fe%PD@1(2Esb*KF@cAshr+vqm
zRpE7)mQu3H(|{eG-eB$Wt0X1jx<ZZuVd=@t_|V{L_f!Ds8M3u5OiYvrCmfg&E_W;;
zgRXE>5juRa-Hmo<u2OVY2>(2K#Tm19>rpY|?0VvC71PdJ%xv#)5HW7i$5_(wAG^o%
zQL)KmAk8_E*3n@S`z*E6P1<teq+*j@7TQeX^s75yw6x*+TW3w2J(QCu#ML^ruEKm^
zX}!!PQ|<3=Ld~~&)Fbid0T+C`13iU1;t57(53^ORmf1{e9S(Qf(g{#gz3$T(h?tza
zEdh2G)){fUXMPpWEj*-2`8`M&-eEBsLnnu3mvP0z2TwuQyaY>-*OL%$zaEL+JdLpV
zUV>4?4Z=2t3NV*L?k02xECGWnRlYy%kFNf+lANGlv}=b>E*bm*pZv0BfY)j1TppfO
zn+v;MJ}SZCt`L42>W~o<h=y%4FT`3+=9#*dT;7=cNcg;a19cv2yf=@$pO>KD-}lmk
zm1ma~U@!qq7;rH&&}EW5N4}>{9BiVFK|5-h+q;*$NP`;`Ihwbp-|}9pj7j9oV!gf5
zxVOvZ9q^cj8tE%0$e~(f4}{)kJjcTPNu;!AQXsp9%%nk~Rp}?KG8P_gt6!ET=6l0(
z%&9uH%NRO+DN>|NG4U-eRGAlv-%<W}{w>GFT}Cg;TaeqwOrgi%58m0HsWZ07<J-&e
zg`dRNx%Q4`{u5@U^N)IowuW6@nr(a#{N_yW%a^n=zv=KM2ONcm;`t}zb*&{2Ql<@-
zt#V%ur!=$VUNqc&Wz~G^Uij8Y!p6m17Y|F~3)4FviPgPl*Q`I6t~+9oTEav>;7)hz
zZ+3^to>8DdD1ce?X_T+tZnDpwKEz|#%D~iUOUR}Cj3S#BcvDkswYx4U93-5cab#(z
zN6)GEfsBPfa&c<L9KR`el-kZQ^X7lcJNYjW$$yD2{(E)Aflv#k9VV!)o4fI4oxWIq
zyXslaFho!GA$7CF(PiCXmjwXgd`|vcb`DJu=qQ?QCWR4(N1J%-A?KTW=McGZD@+XU
ziH?({fH#-VB8j&fzlxs=#^?<to$lQ-eS|jB($&m=dSrtfZyI{8$>(4jZwu&DUN+LM
z9o4oAqwKN(Xf}60Y)xCw<cCi7Ew85qz-4V;SLXsfOX=C*m>~1E!gk4bTX|shF<v4l
zrbdM#5N(C^<O>dErvgx_x*Ac7283v{rq(|5*jWl=O_SmYY$Kp|qwW+X&66z)4tA=G
z+s{4apv~ELpUh8RQ(|c?ionln4{3n#cd#V7zM}a<4Uh|@62^&%=Ccw48b)w()y1N#
z19f867ba;iXTO!bH-ub}1o0Mm+uE&!%n=(eCT7n$(@}R)*J<-_ayJ3kOEiL(+03+z
zS;Z>D&%;znUt^bUe(PJ=`p|;&%`p=?_`PJVd`f!3H_fau{i(rR#uKy7*X$K+@d=5E
zah2@uHip{$gjO;lYNn>wX(oK<g(@rk?<uwGUbWIhp&P0Y2U`S(6wrdWBRnnUl;X;{
zV90=>V&GXp$20qnOf8wJ<2OwUL!B}~XqwJS4*ClHTEdwB7nG(hW-=7SJfc-^RE!|Z
z{0qQPt7gxE&}!9$BM#RuABB+*DqmPNY;U;X^Y|d%!oaWLnrGX`pX*0+Ovtq2t9*x&
zQUZ5P0JO!(yO+q89M9v(2#~3qT`#vL+>7}Jr)>_AsI314ZXQH_r0Ihg`nE)R^B?Vf
zw8vL-R6{46uk$!8tz=)h>}d+pSI;JmWGDJR@ekFlB#ZP!)Nzk0x_Abz&2BhWi6Vba
zvhpR}$W_}f-QtLsFFfr~;-<@yvT3-QVhJ^*(u>?kE`G}Vz6*woCFO?QM}z=ar4;wS
z#Y;Sh=sPB2=qg!SjJo*%6McDw#)}^@>XUDCJ0aS+h7HF}?m#Sbsz(YoM+m?y-ehhw
z`-n1QAyB+YsDBfavWHsY@-D0NJfx=FxP9|o_4pK0QFg7)I%K(u5n_6@5EjxDUJUY+
z;D{=ISE{A1ln58Jzj*)~52?+aZK7L5*!-Vhi+AU-4)UHy*0Bn0iw=g&kk@|i`$-k(
zJoQ3s%UOX@f%tp<ySk?-n_3(xbg%Mjn3xoEh3N$*OeR90Va*qG>GE2Ons6775g$^_
zZi(DIFCawGKRg2~hcb=K^Fnk|*C4D06M6kz$?@faKgwIDd97qM<FZTyW7OHIpy6qU
ztUAsrQqApE-+)@hQeSEq9OWD-)b<ObBy(A_^*?@yTy!{^rK6+ep~aSF%E_ez_xASJ
z=GiU+&mtR4PfpnjdM%VQl$wbNw^>A|qT|iXs7D(GgPbsPRV$@Aab=2=l<NalLw;;y
zfc|@=mYe*aNNu(e{zLZ}w#8}4m3@rrkCLCe4G%%%)lePmh$I2#cVR&N_?^9HASA%v
zpTabGKqP~5GD_66*c^;~sZKwqVV*U@S~6Y%II_xye%q&B%WOB9C%mue>8<>uo5<%4
zO)wj@sCXEW-_v$-EJ`hxLcMscYSEGKZGjt|9Lu9uA2>?xD8=XE6Y;5FHcHWyB%dpb
zUh}n@4#9`%vjP=nlACiwBX;W{N?Cscn5F#;>-{J7$AMNZPaWF>+d&}Z2h!pqDnY{<
z@q`ksrLrGJ77<6>?`bBlX_&aMg})<q_}{h`JQcQ9nP?n)A_6*=odm5CWt{;rnJ0ez
z(KJL+(QlDXthvJYo%G`9y|t!<8sIXJwurSprr#}%b1{y%vzv+Q&NZghMGIkG0fst8
z8C4i7nu3XC?MJVZT@f4-zZRb6jGVuC>}Y7HhPhSvF%^vhCe??ovZaguXt_|tUeGWq
z>KQ49g}#dm99Fqk!tw!~Xf1$zFEJu%TFLf!Cub_c*t7J4d^;=RX(1GrmotkkHC<H<
z@s!;<bXg)}Ch4^BHiDFC*4m%FD7C!}nXG>a@h1p?{=Tl1iz?c2>KM6xK6!hkn%f$0
z?r9(w7L0?o{>2?r`iVX8`?#$an;6ht$T3P_`tj2-^}=tD-8HARxk2A1;rHKah{trF
zK{eAFxHucL#J2`S`jAk`srI>PCF{gumDI;nT0*s|L%4PPF{m6$-ts`SZ}K09k##n~
zWsgGMZ1+2c6ldk<GHMymyN?u*KRelz6V!ntPOKZJBq%JZ6GiV-@JhdKBEP9wKk}%n
za~e}jJ^xA>T4Q2?A{oYriWgjtlq`C<lM02rtZU<`PeH>D3u9(r`iO=Tf@4Gz1L_p-
ze)`me9*pV!#edA66>IM=Qaj)Fm}`fqmoBMOho_`B@$#r(k}#xk6q><|C~R~nW#l8+
z7wXv9>G{~)6-8{|4?oC9*ft19dgg_<^l#1d!HFVl^GGR&YMSBlM5vDD+{Wf7nn*4b
zO_d?@Pm;SN0m)w5ky5fq9v#}^-{@^STp%jd_s!H7Us{T~8sp@GL4LEJXG@%UR{DWY
z*|@0`_*+D%RvmjsQgjN5t<Fa#e6Ahlta_O*S?&r+%wRHXZYO_MZ#_o8nFLyi0+8QC
ztZTM`63<5u`FsOnCJ?Wji5YI4jdgLdCGURFa2E#Sx_>1S$z$IaT2<``{Ldj0BDj-C
zQo1K0NKuFIM4o4bf_O#^IaPTP!Sf?{sHvs#Wclg165Z+%h>sw5iEc~_qv_kH0+i15
zpBE4_ky8>t1b%~70)+M|tHk$fPiLJEWH}z2O7ML*#yV{14OsEX6Hpcn%|~I~>Gwng
zl=HFD;@UJIXOW2%Ivw|=Bq>so5|6%ighgp0L(*_!h=VKl65H}911N4cPjM(@^Xn|}
z3u4?1B^?Z4X{}taon+h@?s0q_nUC^JC2uSMmUrs9bKT0Z-vi0%9sP;E^gs>XyhF1*
z`Yk@WFz3zu_u5f@k&+Fr%B4Nl=51p(t0AUZMmtTd<LDaW+CGTYP*GkcU0KFd1}}9d
z2M}l)?jIC0q2od#pQAvp=|*bv@&}V1W0;!cWWU9Y)ASsl@NZk!tS&wLTBuQI>n5qz
zvH)551P1xw^m=#H!G_|lzXrY2`zD(h_vo{k>7KAV&=qv1q<`!cAJ<eng0X`L9J|31
zbdW)3do|79aCVx?JhJPsc)6|#s)CdUnhN!Ea=JR5_i_K95G>+UEFqf62y}&bx8ujp
z!_gnVG?2Cv<HZYehE(m#kxI}H!C`b|){fIY%0Zzr=-_Rc_o9xY2RF?~sK-KI942()
zq?nMDJzRlG=F9Ar?D$DB6D}Oh&3;Y^bt3qXdeH$Dq0X$VJ`DXH-XCb$`DHjYj(q0y
zSzbMrOZT;$=k$ik;J}(6PHW9`6cJHCnT}9su-x!+3<X)LxomKVeE)J!cQ>65v^z32
z`N*Ri(Zqx6sA*>@;ozn<9{qYgCJo_suy$)=QLO%%Z9#(o@lKuc`w#eA8!J=MqlnTI
z5)v3P+1mEZit)ckW7W{n^*0hnyR4qmCh>!PimxVJyH{xo4I<9O%$z|hJv3Z`?}RS&
z&Amhi5(O6z6F)STd&c}d4@~oFh-HbH96E`o@#V`gF{=FZq?t#Y2}5itaTBX|R;cd$
z(qXxg9~W6#h2yrfX`^-j4C5>rd7Wvh)(b(``}I5@S=osGk+e$F)ax_Wg`qPIWl<aQ
zFe+>TUCrVt-o|?Y4vX>EAS^s`2CO1(20sP(oidGL7g(5T{`{AJoG@mP@cEqM)A@W2
z3qKq@@|6L&hlep3tL+|E2$M1JS0KR=#)`Gm_h*<M-i0XvKu%;v+vW*Y*~F^HD89}#
zkSiI))JBSJsa~@tF!46`u2Enhe?ZMO7A6JlZ)dX%{BMBU9fFn|2#o+w%3D}#?)n~K
zxlHyyh<nSRxVmm#w1F20o}j@A?t}#QAi>>fECGVMI|L6F+}+(9_u%f(jccHBXk1Re
z-@bKk)!F-=zo)AE&+6(`bB#6TnDcqYGshrP`O~ykZ${P#hj7%WZua7K)c;?~PDJpE
z<^N6D$q$*BYs05E7pl>0VrjqH(oZZz6<sHX#lD5n>E)~pErE8c;NMz)O&&gQpTXOf
z9J|_)cgE2En=&|&g{+j*WcQg$Ia6?B1x6jrH&I0#d-y=;SWhQU&!HJwJv@OuF}9Uk
zs!}vqBa=xkF_l5PyXeX?xC0mVocYoRUw?fY$%fH4s!n8yKzMG}{-P@lS2{zg-p5)N
zqqV-P6?=d3EBN=1=F{hYW$(k$@MPQneKm6eUppc%B328t7%D&|0EVnFSv{*5$ztKv
z3&&o}y8;EaiZVHEO*?hfxt|yc7^Mw#(4r|=bK_|Q-m1#xl?UG&7i4O2LK)T+?>KL_
z8;cf+YT&igAQtZq-`HA<KP<u*4EQ#Fv2t`|3HW-6oEU)iaG9xwC$|tm966^oW!kTn
zrP}bXj#lb1wl(9+g?+VDAx&cy+941wj#$LE&2HwB#sJ9eH6}%jynDy|mv1-uFZx__
zzLjze;nBU|qJ5<C^S0k!8^PWl{;dd6NyO7cQ@Beoi2H-@VRF<^=8uI-ay@_;TxBpu
z|5!J1rVyi^_=i=8PdWA%8F#Jc8VI?DYiOsq!VJCkT>y3_i5QgAmb?Y-*0sh+f-P?*
z5Rx_<0jP|<C<r!@ycmGp-CYoS-NbYbP0_Nwo4J&G6|O!qpf)go65t>OSSt4VPcm;V
zju6%esshQ3k%WQ|lkM16&9ot4Y`W#zMdT!|DJf&Y_T8DswK67DAXCz2IRLwo<L&&R
zTO}f20CL5#$4K(?)K8c#_%GYF3s6u6p*;%;G(a00?glI`Y;7sEJ6#T5Wwaac(2q}1
zuS5lyd3+x;wjt=HX0ttQ(^RvR6xaM-${n}Ul2Xiuyh(J_ZaVjn&wW*oZG}+zbS*ab
zXqJKTss2h{A<2ueFdR!t-zuM)xoSZ=sJc|hSy9O@Ne?M?v*OS0d(S_sbSm5-K2T(S
z$e~m_)Hx>UySBEJdsqFB*UL=u0Vx8&TY~UEX>sKvEZP&Evg&4t=c8S|tPqXB@iax(
z7=DMIKtfmthz%6(G*X<^_LH0ckRO>K0N?S8jQ;A8o<uXDTVcggB~#h>1mG=6I9xoW
zkCCi@pP|fNIgKc0{2iQ6ceCSo=FR}bW-fABnaoP>(7cDBV~pl~{?JDx%e3Vu%K`~?
zFi<s{Y<oKXl`r>K0WK-d6SwKGSGjxwwFmz2*ZuMJTNg-juF*7#GCx85Ii9fj>2l`8
z_CyC+;3%^vIE{{95bzI*9FgsPqnWQf_78i*EChhs;b><LGb337spd^dmB_<sjJ@mM
zwO<!-H9CurlT*hU4KJr0L6@)UTL|@NT_r52SiFE`RL&@~0I4sqIC3iKQJc*$!w4B5
z*F#iX{-kC*xSiQ#nVwsyo$fT9)WL6m(Zvj<%md_Vo{h~6?N{lvnz3o90)Azn05U}2
zKq?khK;xGIHvU>fesrV}xPXWInLtk`u*y2ps7xmtXl8YqVf{S+qq6>r5LONXv8jz%
zUh=@#zXB<~+>o+zmFQ#m)G(K<{o*LByBt`pBqy(%nu4ls>4^NDr{<Te?^@gTiz~~G
zE~;VT%+&#d>+<0GnbHKmn6=!b{>%5Tbt7PH6(a3#Lv6DJW|2FDwsi!<NoPjQ9#t@O
z#W@?a1d;A?Y7=Qqo5f#rHB*|BTv^9XW{)Wx`ap|%ST{bst)CRZ?84`UgOY0HFzQNF
z2>1fsWu8A0FTrj17@(s6O;1I)@4-M&YCR{x+JNpObKFnP);PK~-jRp=yu7^Uc6er;
zotde*D;yiEsU3h3TxY9Z=_2sF&+2;p>m)*zxf2`EGWs*b`l<=A8#ia@Nk{*CqGr>{
z&`Y+0!?2^;!Q15>ln9wh(6@klka34-Dod1D?CYSP?IhSoxc(Cl1IG%$7}m9fz?!w#
z*jwqZ$FK9tAO?T!9b6(d;9Xe|GCO^r@#7C_T0muG{hr{y;Os(HV`)lLCrIl0VGru$
zOJtuxd7!N}a-!RY4))=$bp`%7s&AoSJ(kgyX8Z`RpF=Khj;PaA;9<V~g@^Hrdab9e
z5;8zc#DA~;qg0sf@$>1t)2$Cv!5@3Ohw(oj0W0@OMbLdfF~KAAjlz0Oc&g2W%fxn5
zs~+I@P>oVL0bZVX2)1q-9oQG+N>S%UZ4e4E$$NuXYiAJ3{T_8vKPQtc&r~(9cab|o
zu}IKOqld%SKn+z;!5q>iNbtAR8TYAHix}IMDWK!&8EpjQF}_En=dMai#~@e_#C0-h
zBgjEjE3$vMJUduAd5lt|XWTaL!Yz?gp&G#f)Hj>yc3)2^$Q$7I4l>o!{>(~CMi;}_
zH8lfjk84POp}>)>$<BexZAX|Zvonl+;SEvYATv}-AreOqnzM+HDqlAimoKRAuB<+}
z6;YNEA4FgAyRff#{-frxzI0nf0wGM(RuMod>fUeK{SPu6=Q_{*(4BL}@M@_IGRPt+
zK+gi<2T&}xcwxCnRcM{mb`kKqn>k4i&9s>LdbJ5Q?WUx*G0}~KwMpZj`byX=jz%iU
zppie16|7k`zD})nvC?zDaYQe?`!<#z-m}~a_yrptvn%&evYg{TeptDrC4eiE(fUTs
z8a_lnzuugRR#rWB5#2q$abA^_vz}imb(CuOA`s5I>%w?q3>?rJldt3kxsf-}x~Z9z
z4BAEuW?Z9P6Hg1C+tmS0c*XE`z<1KUrvyq2t9mwHnk$w`E>`u9Wj8HVD2b{KnCzs|
z<&^k73EalgjEIZI#L(7$^f8RhktEAHyY=Z3EsUMSA^+eXIg;>kD<aBO4uSTFjBxre
z2-vI_+F#K+c-$NAd<D~s!#fBz&AGqKJwJ1;IB~6DJ6hRm@h3}O&#Pj7&V;J7x`mzh
zsB9x1U<vOByQk#MaBh{F&D9zB^^AE&>M-ze@KrN<e0jUtwM&}RlGO5P<vN_E<v`Vp
z5r@Mx@I-U3s(+0b=6C0nBM7sai*yUynhgKTg}jzl#oAesaie8NNgN6h*t|A5__CEB
zyt>VXrOD*uEZljBU*<KIIma%DX$C_AeEhVJKlv*|e5ImTj`E)K%RA1P9~GU{9zA?G
zsVK(YmdbnBY!!=NbW1hMu#K#ekwzO0KQW>Fu62$YM7O2{U<#IOW#QkHNQYMTH57Rj
z6ykgdHpL;YiAln^LvYAUF;n+LBbr>Ra8p9P1pz_Lp&OMg=M9sa5!2L!tAEhE&5bhf
z*f+8WXygC$gpHpQ%jD1x^bISRG>X&|$+n=reZd(1v*Dn^&B^f6c;8{Uw)*R8tvbGC
z4fmEo1O-%O(gQ*s-ZRC+QhBdArubqDB`iAR-aNF}tbXFvz$kL=Xgz`^rHf+lMIb#O
zO!^sqqy~diKMp7QtKs)jV|mb!p&6T$974E@Qbha&%c5c|5P{F8KRp^55c>5e<#7~>
z0ZHAvdJoniAJ!?3^VTeYP&Je07uoUwyw0uIp>`dnj*UI?&nJ*2UV<litYHM68DRp+
zaC^FH&lQ*Xn7H_f>-sdw3}qT{qIg_aePZ>x&Y$$L#2uPm{rtHDOLZaC_;nXtOI4eY
zJnM_xC{Okd+76}u^<TbxJi^lmQICO+#;aI5K2DKl+GPXNKA67}jXp6UQ?0|>nlG<e
zAsg4}agkPRW|3uGB8d4}O1)d#PiA#<@pWzNi=S0(#tU$EVf-Sp-=kRkfVEcqB4M-5
zy0E^^vYRSo;H&9s)`iI0^1zm%hGpo(g3)3*85wSZMs^^}efj>PrYXB7W;8Rg`es>~
zPl5;!!2vVM0p&B$RBTX<*uw6#dAVOAfz*qNNR9shR{M)zHuB1gdJzFY5ZUl3AL>Td
z9OXI0s?<G)efwtf8sVg16AX@;V2G>7M|@jS!+7{rWD^vvu+Y0xiy9jY#k!h42xjFH
zFeI-MH-F!BgH)S3K@e(CC%a_En+9%!wRIqNiX5_i-n$<Vx#a^yFO`MgVWaS&QlVDh
zccEBek;e$?166YYfEsHxuW{UFTy6S{<byKbIdy=3f=PM4Aj6fuNerg+hnc2DVmFO_
zwZB$11$A{C*Aux`eC;KH$qzUC4Ffa0Xu9E5VIK5kqv3gJE-@kH=zhLb-L-#C$PhN2
zz!+SlG-GX<y+NJ#16$7PWEfLlL%dBwLMx_Qa*1KbM<=c=k2r{biL>Q)(D(6?-WHgu
zmh(pGwfXi0`+6|CNJV||Z$2xK9_|3fENMeo(0|t;eLTKaPA7fm8Z&EsW^;C$^HuTn
zTHZW<shA0)tDCQ~)BPy*z16Bu?aoR%OEjA*l&E%n`XR1qNKl!i4k!IcYg_Ky+=r0#
zO*4I({p?}tetI+m8gTwOtT}?cO^F8V`JuI<vokt+#}6>fQF9`sMUbg6_py1tgxs72
zNqBXGypG@Q57}{beEp{Dp;_KoJX_~l9#9{d1$2$VG0L*>UVmt&eyGvedu%_*FeM8a
z9T0>&<W?49?(2Tjnh^0GnusVf+9JgvRI68w2^m7zC4kD!w$DcW9x&zKH$hf`XfUi=
ze>Zv@UY;@M^h7iy71HhirGVWQF>b$$*;#6y#1u9WS6qux1WcZOi#?(92>ZC=o!{iT
z9Q$G!Wfj!E)OtVA7sTmI(;<<amjZ4N`+!5y08>isRSYi2q-b$YJ=!AwImGJA;#gFF
zXz?&jBp`?~XizMCt{*rg<oI3rww8z~d@esE+|5wt_=r<ozS%*lsiP%tQsvLKv2~V8
zt*?hyAL}8W8W<Qm|1B-^m>_h2w9s+`P5%6G8^z@OtzFWWNFnlot~g6?h?2rhpLPE(
zn)g_PZ(rJ(Pp2y7s_;p0o-?|Zs27A}>nLOiCD!xk&zG&0r`f`=ke&Z544SHi;}aan
z#-JDs?QY^j>mgHFV~=d*_Vep&7qD5w>7Rd=v^N3|5mxVA85BZmSVlI~=Gw+|91bLI
zL$P{yUV=Dpy=*$RNkn|!IvgN|^l(wpx_Ej%uk6ch6<TL_j?Z@C1k}dh8IM`P+WKDl
z9|8DCbe_xJ-`@6Qe@$?1Q7>(Me~x4j6a=3WDc6caFq@1CTT5IDRC|Pw)%qhEnzhI8
zg0yxVbXKaNGxQ{JEo16>{Fd6@4C8{<;2<M>yAudOTX@Bc5a+5P5)}>~M3=>hn!o=L
zUsvJ+)W74)LC}Xo<-#o3>;Z|}X{SCl-rL4L!<cm2B58GK8s5<G_wYCS+h-nzihA4l
z4UB6zM_H|9Xy{*=_vt!c#<H>qm1>>;X8}D#%v}9%O-ENCMx(a^p%T2@EZ9s?K%nm+
zie?~N)NjoS7;_Md-2Q9%SHq)Vs9RLV$!VEt^As|`iD2HW=&Pb)NIl~NLH4;}EEcII
z1G;j0`^ih)mSYY^D|#tc5yTZMY4p6^JUZ{sNS5Qx%|l7a;9j}UbMm5r6guJJO#j?X
zo+I|oDD{>Ic9ZKMlI^`;JF?%IM>TF<Lbw{}KZbLytj_N`kD4?62{|0s6f0taR9ly-
z*hIX-cKA9IOQndVGb|+K=s2Rfg=Dpz^??<(Bt?LEum0+AmU1Yj)D+mtmqubv4Dt=w
z&nb*+n2<E%^2@~{rC8P1t#Rrn6Dpvz$?V^8IxBs)*QhH`c!aQNs_$<xr6BonI1w;T
z6OxQt!OG*q-*d|~W>;FDt!z_Jq+LcKFKHz2rlc%r5NM8<afj3;Qv`S%yup0<6=*yw
zU8kfxlFEuT=Kjhzj?~h6jx$7+6YX!7DC`o-K66sk(Sy(MM22&@a<?&;x%m-`F`#dH
zqK@A<KFI&ntur72!;qU)Gbyo(DoByRBF(%{P<X<sk94o2S`iqW0USX^<)AyEiz6%|
zx6QX5Fvy-A-8Ps&h*4p3RRHSuX#Ca1$~Z@S9xqX!SK6iB+}{qvC&&i&i8%ydq+(a4
z4T%#tubFt6u+aq+W+8dxEZ0i$CAyxUr$whLbYm%pzX6MOc_+@E3KS}WVh|Z+K}zEy
z?wQ_Z7@C&28gynXX*L4MosO|{hZI-2)fKYu9%hhB)@X%>IkKHm>1?-axyypt=)jVY
zU+qFgN&TEdCstaGiRdb8Rq)*pfbn;2*PnA8G=gH@MKwejFh8EQQt%{8lD66>dU@nM
zlXyu6O!ZScfN+(fH?$&8!+CNG@1)n81{wuvAg9vh>2M3U5(vWyE8YNsNd&YMlQBn!
zXd}Sj@*orSRM52`r~X<Nssge-eo{t>{j#d8`#FRA$V^IG*3RVgwM197Us$qR%*gDQ
z`&8O@9X%!P84!Wv8@^RPY$v+;y**7Z82zDXg*E=%WLAExMf^aLqUS@6iY-^8mMp)x
z^007F^#Qe-VkrB$DsA^aQLKDuM8a&`3Hs+OYM!&~zEq}$(=dJ0nK+Mux2=N7$|Tq<
z#$U{(hjBq<XH!z}8VamDiT|mocwFG~Z#4!}%DsPUPtraz{-@r<iuC=zr4U9$M*ro%
zWNuVN{;i@=AA9L2lu4c`Jh>B6>c4fQ8$NuNK0xM^e^{4lVmPhz`(=I&EkJVgLFU~^
z^4@c+WHD6*vV*a@Z~#qEkZSCKJc?^uk0RO^`YQ7e)ZNzmD550`x+xZ3t8?VpvJ5i*
zR=tQi`kzfzM+RkP911PrtH(D&af$=CCuw#|gs88pW&SP*i0>6|Qvfy$ZHm5&=TK=O
zP-Aobdr1_^g`-M&E@8s&<*a%ImcraM2(0VG^`)!Nars*^5U25>K31Df2ih6*Dqqcf
z0*@X=1Tr1&w=idWLkIN$B9#EPf0lP)&Teo{2hFea9}KN%KGDUVK<Cs@+k#$d^jf-i
zQ-e&TbLg^lYY|}?fMObvU3!ZqhuXje#DvXNG-mfe=aPAoyiF;+tlUZdi*BS{L^#`K
z=PiUbiKT`2jsbNN0rWBdAY|$;F3G!xzt<{W5eKc41e<Aj<X)`f4FVO8jq0xkx-Ton
zpR)pZm)R!~9Gq`WB=_A~Y-}1MBo)!HGbO+dv^Y9_n6miFzdD)Arc53cEo|mPgSQrG
zeP9z)WHKg&eNqL*lI1lIiJ?J~z?gd-Dks60bz2yq@j_(75sx7h{1{p?PnN5xbZ8AH
z2Mo^nW7pQ#EgWkpN;Ti}{=JmDp+Ks@t;p#QRD_LothOy0B)di0G83(~RKF$70On;n
zHx5C!NZfB?9-aW$LJJk_s%N4axacFMmb{SS-OWK5U8&8R<g7V#%Q{E0+u33-Ge0}q
z{&hUIVx}5>KI7k|3uIv_JPRzlrPk|<5RvU`F@yu2pf(F_?K|=zLAQCZh_Q1Sf>wpg
zk@YpY?Ii`?PmV)qEaXVi_3MED$?YXscSC)C&-66Gih?r`GX1;ZzRxU#8b2JB4RTar
z1+s|t4xt0B8>-(1)ae<i5=1T99i5yU+y5nfn?`XF^0gJzyF7OaR~^bFc9j>C*jZH`
zL|Kn=7im63RiUo@j235QSMOZdb@9MJDAg_LEoCniz+|KwPyS<u5Rk!j0);xZsp#Zw
zW1?cQp$c|(35Kx@?7K<oR-uQ2>kFfA1O&!U3fyos9PG-NS<40XVfE3<+;YS*%h7`<
ztRfQM3n3M_hIx*U(L2k7S;gXRrb{DTDzDxGH}eX4m96q)^s+-~<AI6N%=zs#jVFRp
z@kebEdOTIyox>3PxS!Przf8D#<H+o0K_$g!waO)j0%a!xD<enl6!4SA)o-TUxIhay
zkFDt@p$$Xi1h7Yx;xGZd`1z^t__a!pK=6=$)6lhftTJgjcI=KM&|Kr4R#_?f$+Wfs
zUwxgY|6SSO4X3tk%FD51b6%P2wbIJ4mP}N%*@|k<{*8M&y+#!SmU(PsbGjKPI_BfF
zM_-l2Q90j=nYw-XVMaRH(L3gACCrl*-s#E1Mgs<xwl)FX=SeSFVfue4n-{3z@9%ZD
zb!U#GAFTo}ljS?UuyFE)Wz~nvHKQ`qq02}%P~16gq^&8Jwrw3hbxPD85lhmsMLFt&
zc){uqEqn@*$y0&(gReke^Ma<@-E&8`r93-c0r1otC|J4C_RJ-n8`hR!0RL#6^d6lu
zWlXb;{D{=iHJ)8mUps6p<<WF=ZN+xu24Q18o^nb0*tOdfA1NLDwo>04VV`&Bc56xC
zX{&Q4Q8Y+mXb+vwuxiUnkPt>kro|3R43PREQ+e&pt((Je#G4!fFK=wmyfgzL{FPVa
z&#bKu`OQjf&V@SrzKkli%539Xg;iu--Y>;lO%6vs8R{`UN{l7KuP^e&CGGC$rBy3-
z8oj0BwlI+8rlfNBxy|bxS3szoSrMY*0>Lob8O4%JlvIPsV>LBME${)(h406el@ct6
zlY6m6#)qdg@@RmVzN~1NTp@)BNN6H36N~`Kqw+>|G~qk)rgls~o8qEzY!+v0V7gU-
zDsm8d(*m$KkYYrbuz6t{{tE0XKRp1#2Jrpd#($)tfQX}2s}_?r!O|f#2Y`+nNB7om
z8NRFJdVMR>C5>iDOp0}k54LU5a`Q(R2$;$RR_!8BC9Yg|T`T<gU+r|&6$if3@i5Qm
z8KC9P2^}Db<*!g1-YA@X3;<gNeF^E5s+|3JP}+XSn8T?HIIiM(pIFKq02&Q9ZG`MA
zM0_s(MisIpLsC188IdGg^GlRv)?SmuY^yjl`ShLL?4Tlc6D*q0&a!lt@wWo~Ofjb#
zV3XV$vcy%n>~eR$9*=1V!wn0ft@nx5rt~Q1wTf`C(F$JR#Lm?hrpsJn(sn-SsqJqy
z^3XHH+-PYqH<HzKjjP{1FZ$8CYwVJ~X_<|PjM0-KCqbK}$O&?q$eL4<)nltPQezaF
z#wP?g@CB`y=`dEHi2$|K0LC8Ut-L`L@@faGcQ6u7Z$o&?_Qm%<r!o}jcayZN@?#gE
z%wwUbn<UPqa?JWB*rF<$O^z+E4i*t>sDGK&`5uVm)pHSZ)oSxZleR?r3$*-gYH%38
zGxRDV>A%AO*mW1aBjSKMD223R1mt!6!F9y{>XUe4;yijJhu)sR%_Ah1$HbX5>p+Xe
zN0)Y8e%G-5yx7?M{ZYnK8i?_fI=Ms+(X5;P*G8N4${j8}eTRDOH({b|rkCXmmcXw2
z7Um0>eng8eV7g_9wj<98pdvu-m1SdTacXY^CoI&!OQwt=pmP1ZDx`RSGL+o$&7i`>
z`m#a7Nn%+b8{-?~cjN3Y{AT@4XgI5Rys_6_3LTTBce-NcTzc)m&IC89z~n0hoxu_q
zcT=5j@d7^J@?FN7iW=wlhOvbrI=SJ6qFy>|L8>JywGM$F?8q!PCAs0QiZ_e5=_P|5
z_hgqX9xz%G$BX*D-yz<`D%=OZXfIS_m@5)L9^@AXs{~#D>n5qgCI*!}{e}eoS{)l2
ztX8B}kXu_M6^L#@{LQB+S}@nw(bbk3`&A?LF)JDYz65PyS;*zY5ZFcCcwt4trbAuK
zyJ<AOAxyz-%Hs9>uSKknOG|5;l)5V;1S>0N8MjFQh2dLPJJ<wrowRt$Q!!$?T+;-6
zK)C1e&0NDr&SkHv75s`Bu*R9s#toQGwmQLXK&XX$oi`(h7Xh06_-i2#?a{EajILYI
zf$tuo>{e>unyq-y&^acqXnf_jl9M5eQQ#r*{43A*7w+zIA7<@~syCbPi`0b~!5l>(
z&nPXqlBU=sDkDqF=J#W+swr{oe=4%Wb-M2Puv(p(Hh7D+^wFg(L?cL%DzHyxTbT(?
zN25umzDhJ#jn6iAR+;bG92rIHzCRpFu3P~0cp6lSBL`f00jPL6?t+CVMcn^1RB!ev
z<jbj<H&WBho1r8$ZPad9e*&V9H51SdSL`M26}w@y$Ue5TyM>{M>4p^h<^pxTS1zoD
zI3AI`M|CLdRmuMcF!`C^-)t`+_R32q1UF!b`(TB(Ad@9bG{15vi8VQ#X)EnBT_H}G
zM-TOP9AC59itYI84EM?=TPh=MB<w2NECX<~3T^i~Ux9@>W!Xcu#AeH)m9(_U&iW2f
z@bG7^g#;J29cZhXwpt$QW+O0Iv@-z&D!Ew!8sffi`R^IS35ZN{akP0W0>J9as!dqb
zxCx$pQ_tahLJEZ#en6@G&lEEPYggwSa%>I-LFNmAoljcCB|Z+<JTYx`bQc8dZBA8`
zwJ?`Q1Vvdi0Q^V-Wo_Ez9fCpYe8tr#PPXe->X!bk+M5aOm8U*j0K56Q&gpVE|E2DQ
z>Lka!WO5d|Ygf1+(k>Z7E_-Xk4xJ<4A0)y&v$LnM4()-68&Y#CYsZHLtfs*>y@I(M
zdn;?SQ@LM;V(GGtmQfl-=t;<x)u1x})w930iPP1&k~LoOJflNQrC_^%etEY)eq6zb
zZ7r!9*bZuSNI5@F>a(h#1Z8V^gwcoT0d`YPQV-rZEdV-9$j>p0G}sfC{{Y!O!oZtH
zAwxE&cBr|$T4AH7fcfH6r4|K;v<E>!geJ$wnECv4h!xxpv(_5JEOy5fQht_ZvVC*E
zrQ~>cyIUY!a5htm2_RM!Ak=-Q5X%j}#+Z=s6u(CRMNrTo*&@GOCh@j$`qmtj!DXfc
z8m~nPxR(?Z(r$^sD`s(4x{ZjfpKKA<Crs$<^d0Grddo>C0QegXQ76Z~2sn_rX&Wwt
z8|FG=Y#mPDFH%-=sEQRxdMX+m)I!Y8;72aieb3ngugGmlXy{5m<L@OcaJj{L4JiyC
zYv$ZQ@O86?DiYRZ>bXu$s(w|Sip;LrL0YFqe>yWq4K#%cNSx13w{<3=K9_4%teA2u
ze_>i8w<}#J)bO7@cWJ6xoIDjq&GeMpxM)Zk8D00xw6hrjvJFEPI!Z<`mww|Bge*X!
z33|<0(3jbU+8!Wm>fSbFAB)P@oD__7d*s58l-{dW5!QURJF5#&>Yf4f!j;u@*&0oI
zgIj*io@pW-lIlO+Qy7npVc~`<n;Is`Jv9{Pftt!3jKVr=hmKfgmUy(UueD-sV8<&5
ze4ZnWBHP<$?!<Cm>0@C)g#y7U#(7)oWzPCFYRAM1=E%^jtmplgnJhxarlO_^3!9)R
z(vmzf)I{;>;SEiqbljv6Zb+ZH_=7_@D(feS?-iqP%jPI9+mkIVo{qCsSFgP9Xws@u
z{cN*7yLc+(6i+I}`7yh63r;%s$xN^fBTirq`c4Tzz{e8?p$djDS@`1FcP^CUwsm-T
zXfu?9Er=R!I7QGZemqKH{0j+nw_%32U<bg!u=@DUif1dNZyOdF#!Fb<9jvIva|CZ*
z7@1hQK+0nS_{hh{ht|T+7$DAbW0&UO49NXaVE;LlvApynHIU8FJpXQjJo@C>WDSW3
zT@tSzBXv9X$Jr|BM>pAgB|_qs(vm-~_@H6L5LWyEqW3#Ji0P&@?1qE@RJ$k>{@mXo
z@mF@sjmX2~WF46<i8Ru39;3a1@~b-EBB|BIw;jU&$#e<*L%zc0r8&yN1=`^2>HR<%
zZnODMn59^MAJ3u1+LDOJVd0&Sf<1O2mzt_3+FB`Ev^Kcrp|Ic4eq>Dkb>2f-I&3R~
zFJ%W_QGms21gZsbS`}6DGIQNUuVZ1s^Ir)_;(7f8CBmtZbXqAjp9@#~UV)74yx~P%
z2GIN^Oi1wcSRmpkin7~%+?0ZSP0i?cCF=KzXoL9D1Su4yvEp;{c|ro`vpOon1{vL{
zY58o<k{?Uowr*znm<Bug_?PD7av&sV(TaR@g;?;oDQhm1*y#OVPqXY%K@c)!o<q$1
zd7f3WQO3Fu*VP0*_M*7Zv#!0=*pW}$O}(F8^P9}&R1LdLbSVX_$rhu5(&;`hf(1zL
zy}}6tptY-w{wiJl_oxHwuK=R2-`4!d%~)}RE;&c0^ItTFYJ{w_QG|}b4<@9Mr+rId
zM0IU#(J;>tMJ?yNiM}tKMXN>DvuD%m3rez<e1sA`7?RFY3_DWK<=yUVLsvtl6m-JG
z-S7KYOh4SOE6Ee1DmbO0=uqW^Fkg|VCmuPN3LE1tQTs1U2@3ctoMJ^jakmclw~sA{
ztEcM5{nyV#ot1=dd|&@(vf*FNO}y{_%u^(ri4Z<t%y?(ZK$G4250->(7SxVz+E<f3
zS60=I{Ob$Zh8_dobPIWzI(&o#u7)hO_9zA}u!n_Cam3k;SC)G#@svz=*ql<F`PWCm
z^EyIph4x(}D*})RJM|pyCnc`*ljYM7)N84-nzxz9N_p_x3a`DI>gvYaw=~rpO;To(
z8~0lMd_*wrE&t%=ZNSmO^hWe_gb<MxP8$I3`)c~6QvW$XsX#dRT^k|ma2hU9;IK(%
zj6oxZ=FMic+FDYBnu%9QKQDlt+0SJ|iUm?d0MX`)UkuV)t9}$G;^Xa^z(VRZ35xSh
zh?DOlpHu|QIl2kpRc|8j(zWr&i733adP0cKq@`YCoPFZIm`bzXwv{DQMjZM6TcBb`
zE9nLwmzZfL565mE&7*gp=7Av<Pm{ev;5)pA?vlqd$0yBwYl}CsD{y9{6LN%Lfr{F@
z)dL6YhIR%}cKP-Jo+J!AeeLk*i7J&r?}qc`lapvdRA``Hh+`0Yjp-RDM~RhBdEI}Z
z@z(0rDqD-;6`vcY9T08CIT$jYMc7`VgM2{PS#NPxSIWy5`#(c;O$OGw3@>lNN74tB
z;KihC81LyZIqV*R%%fb}Ab;JT@$Sc{Vk+y&4@e@}YD?EYNmw({FfZLYLW7n{m7f2`
z7<`Oh?ho3cgCAyZEHg~Y0YYkTB_l0ISwx*ICyVi_dK5F?Tb+?+t2qZ~@fX!BuyDBo
z_^>%_%e9A9mpZDNeg7N=B2L<(xXx5ppF@gdN*z)0E1894;nOZ(U}w<ZybJkJ)DQS)
zCnBlVpltH!A5BkTeG+xX3mH0Mue|F*OCzJ$xe)RH@B#2PYnrDQ^=!)AT3YxdBr(#}
zK1Ed%pLT6p9Qey$r3~$^xVeH0&G`+dxJg8?2mgOS`sz>eXj@pnU-f?K*0H0s)N@zh
zVO|Vj2;%>Pq@k1LnI%J?M5?$M6J>d}vRdjJ*1xN1lPXcTACX!;tli+*szNlGSL&X=
zN7~YM{P|8wJfR7+i5aFdxgU_DV`+@hWG-hXdzW3_=3r`a^>UsbePU7#W`ADUv6E1T
zT4WB=TgD--N8|}VqJa6cKu}XDZEc2%4PE<-7=wt@^;B$gT{ryN*5|BZG)<^%|H_VP
zMhsk}W(iTBb6+l*bIF=_%Q`lN3F4se4%NR$4)wJV+S1~3)yzDuFXLUfR#e1>1-EIl
z&bJ_u5Ie8h;g@~3dx|5co+Z5S$PaH~8PLrtE$1r!{r6uB1K2al+UEER!5W_R7%+oq
z^3!2+Jk4s29)tX|x|pCVNu?#bu@+!~HuL^<_wbl)S7_+(Mz$dzGWq`vQnysVVwI6B
zN;R~Yr!@=$S1(T;u>dAXGpKy5#pmfJJY!t76BY_>5!s)g=frH2s?{PMnAubo`7<fo
zEp2s+d^PyAP<pH>-BU<GD?@8}beX;`azM8O)b*9{4da=5#t6Q~Vl1iI#Fn~kwj+KM
zCnWBLuLbBR1XdNz3P{4j&$!+Y9Q-3pP2^Gy8NIy!#ti&fov^Hk(nS8qsM<aT$c30<
zxz^~EMxbbPkS6dL6$ZlRImuL(7cFMspZ_wC%4{4i<-=!sjDXGslptJ9<Y_J9gFk64
zP6ac3SWzpiTS*MZi<UhWAHUj)@9yy!Nl3_m87Ywhl<<VEG;)m*?r^IZI8Zi^S5RxJ
zWDbRW%32SX*#TbB=$-@g3H%-TE2iRxmPKEaT=4qc7y@vcaK!@SZOQpGmYPJBf|JZd
zdbsxiaP)f@SY96eA}D_>BBrg2f>y|-)gjd%;|SPG7vCzYk4FUbn<yQU8&)&@?)s^@
zoarTZw=LJ|h2B$ZGPyr0|FhQu9Rt9Z*=tZin@9g%#ZfIir!-I&OwCWh#3_@xU>Wrz
zKb<aGV>_=oRxtz#kc$y*IXem?Kpxi2m~bw!uGTYp*U1)_0`bAZ`e6BQrpSb;$RMq*
zD6h4<&emu{#?mMu4<B6a6avClsSTL4Xmo4#VmSGQ$aauc8>I}jw$eq*aCk%zD|;i7
z^*2naDIHNv<a-0D9P?Kjx7M~sNwn5Bu9iRFP<O$mvPxbi1Hj!IgaBszhVz~ic|%*B
zmdZqe;B?a4)L<zxK%&y)L8Vb{CSKH(Xo*d}3LaLehs}Q;Xxlc9A~x|o?c@>H*Uwv~
z$sg6IB{(XH4ZrZaf2K}i3Ha1G^J&`fAS)pM&ARMZX}gWOt<6wG$LfEgHR;|C{_z5{
z{3OMv72$&C{(=JG%bBhHBSOW64}kxUTnB{g^bktrS&n1gAsM?zS;%7n8jJMW`)vTn
zpm(mi15aJNtS*sinyxo*$K*ynW?l<Op>VO-j%W$Fm$;UF-@!7=AC3OHHZ0aY@xA)v
zX)|!bug7A3!sJ}u#A29QdZ<EuyIg}W#$Mybl^C8?^brFppOaMl7rb=+=RAEpxz9-=
z@Xs~l6l#yBgtn5Sn}dm?Hfs%Qb8LIBB6w)cjDQ4SMv#6Ue~G^K#G_d$<!pj2^7~+@
zhVha)qj7h10l<|hKSq6rW1_bxXinq)^8z4dm))p>hh3VQn4Y5tq#a85f{CG>lo*H0
z5qtgy(J)rtYY&VVqHkcw0|cZ_5Vp4Ilim`f-tF<&!@a(Zk+P)cDJh681IH;FjZURP
z6X5XZ+&@K95v4A|=S3>Lwyb8BsAHa2OcpS92#3-1Mqmwp7bD;*x9gsss4?{L1}Qxr
z7Xbf?!io9|q_)fZ?lxN0$X`tY0HvYY^Ml=G2agW`F%Ru5Oc7}9kN00?Lns2~C_40p
z1TC$-t#Uf|iiTaOe>wtGQY;FCnMHCVXs!Jo;69;D&&jH<srFS)(k~AsV2rq<c~66p
zHX;cc9U8~fJ@k#l?csXxjzcUi1^6H#al1o%6@2N4UX8B>e6EEx_6c(F&m&TG+eqK(
zZjqD@o~rJqXeBe!Ew-9HVpC16&Y?{ac-uqYY_eBGwEuw1Q6@M}d9cQ=A~n$gDoj*U
z93pcj@N4Y9yp=e#v3dj-5vEi~P88?!%TELWO=azP?0EXcFOy1Qw(#N|=&or90R<%l
z>mTg4|D;h!lVrqbG_AeCzjeF)R!>CXtPG4CKy}{Atguw*<Z1ER;>>7=ApDJ9P!lJh
zoR(Z;G&HBTjU!S1!C9!K$*e9jDibkH+>G=w$0F<io!;B~@t8@|Zr&Qf(;LV@#e1~l
zFGS$kS|rSc%Cms6QOiIpF?RLjX!&y}BKe<%lPaOV+D&>B^c9}ya;>Kh5+_+H!We(A
zNvnK!GNLqX;?d7oS9A*B+X}C2tP6tXwk5y4BnP>+|H4JiW#h%Z%@`ZI-l9wY)PD{D
z8rp>eTdP3};FZpUfX~W@f_|V+MT5T@OY;NPoGL52*FH534I-$QgV~0zlaPBgt&ZFl
zc*~&^i{B#$YO1G~PHHJ^dK<Mz33sY<HR~4Eq}w32g15T5aAL2nWl_aE)tM!<!(hTY
zecS<yUsYA7b4@2N9=H5&0R8dT3xNSa-43>GJXSS&&O!}MId#Sr_>%WfKAQi{+`PNh
z-=zdu&eqcQGYMq#O0I&8nJ}cypnow>+@R=JG-clK7~4r+-lJpu{r`sPg4wI0e8?Vc
zSg{?P`pB=~*o>ow%bg2!8|#Xr&-k|LquU5PDVqyG*R5+mZw+9=wz`@0efpjvsVPGE
z3lyx_S{I>*zuuUE1mGt1?v_+=+<c2U{x1UCqAtK|p7e2>^nJ(sZ<jF#2bDq`DRD)v
zT=;&}uq07QX`YZa!x%S=DmswxFkhoYC4Z4F`a}J__{wlvLL(HQ;N*aGFZQL4qnRu8
zt6clAFdGL4$4NaoIe91p&mZ<~SE$x}{8AMNj&q3M$JY<kA1^Us(l=T^3|I0UI2su<
zV;jidd@!|av%dH6q^QZZhs#RT8T7o1@%LbYg!V^zi)`>Fa|6+r7eaTV4f~co1=Ure
zpgIfL)eoE5R`0SpX@Xgi6Jrm0V=%%1`1QZNRAKq<I8CLln=&8?s^H(AV)q^ttex`w
zr+I{H5_!1d;cp&Z{!e)1uUEC_?_M^|Vcb6-S7HL*I^pE|WYTA&rcNV?Q=j`V(p@EH
zpnXtN6;Vq29pMQOx>FI^_oPtx%1cRiD-imv=L)kQVEi*@5ul<)A1ot&&&S<sI!@Db
zC4@uq=KL)Mys@W%POJV!$47+s=a@*uNGQhdhksh8?<0SpMT4_Q=yW@Psee2~06%zo
zFqrtMjydKar<Z?_Fgfs?J_eBy(;#smA)SlM!PB^oiLvT$!5{XKgg%d5H3;qG?3sJ6
zVpH`5q2lR<#prQxV5Uxgvoahs4#DI{5aA6QLGu&C!~J|hjVJy`MMMc1>Ad%OM>jjI
zb>b1Jh1~cF!7thThbM(GqR`Z|$mOU0pcGEbo`a7H*!>5isRW-ydahWsKVsgcUX295
z#T}|28xqMmbBFf~MWyul{Z)6%#xY=s)$|3>kNkejyU#xPGv_PanB4I{#-MWlp-~SW
zRF?OKgaQQ$`%-pxr@psqDG3P)Ct`2Xl*d>oAD>@t*S+m0(z#i*3g;X?5N}|gzW{#x
zxahS-IopEf{P=rK74hflN%S>JTe!4~gNRbEx0Zh#otUQ3^pnxL_gN>T*?vwwmX!{E
zv)6szZrx5VSFaJiCrx^h)TAw(a{x`snAxYc3F20ow{HN3c09G>gS(?|je9=#2C*cH
z?2jSg2svEsPY|$ZsmRgG4_=NAoprzZ!EX_D;3<+AZ|;c_pg>=+^Mf|oKRIXO3spb>
zW*}f{di}!Z>hbydh(>+{{+FwOZ#`6iZ0U2vY8zBZ!ZJu;kBu157-~&tiTA*`;E`JW
zzlK*lr>E+@MUWdD8&FpkcHQa=!N=efnz}+bM-b`x_z5hUZJ!uBf-7;)=ZcBf#qy@e
zG=A8Is^_X6FohvFlqk4Q4r9{&4qpNzP<^LT!?)jUA_+~M_mOKdxsUUX4w**e`H<~i
z-Di956X#4ZJTaw=aTv5Esz$G77%v63qq{x-l<s~DL0=+H#BgJ)Wv(jUMjkXkdQ|yZ
zhkW1DiFNZ$OvK(choR5TvCl{7%1Gy)vG&vY<rHnD#*=QGTAY?2;@vVLNQi8A9HMi7
zj%XWv$*KGOZ=?XA_FI4GL<VnI0w*A6s`*>>yw&cyXX`z6K(RTHNa*1lWZ1&>>M8;O
zU-uOi6S0#&&oTk${7%8GT+ERWv;)1^AMZ0Zi54Rjr$Akn{G$I3!sjm)cTP{r>qAfU
zYscHBbKckH*D-(J0A+=L$71yIq`S2t>etooe|tr#PbyPDQI_!tzxHJ)&|j~m>Ktyz
zS@FjqjBQ1z*(;3zz#ajBdTI+WrbJnuW?t%D`=^~mhg6^HSklQpbWyTH?JS@j;pU66
z;xUMaa*gbrxWe<y0KbF1^syN_X9trF`=^%&?Q}lTb=cPSIZPf)k&VrICCr8TBH$6(
z0Yisi{n^)gl4l+H78?L<BAs&34qPx^qCfpD4L9pB&23?S9U8kKiI01KuY65Y&77!v
zNx^Ai^xvKqjGe}xNVkN`(g2=q$5HBrudB~HIY<TvF9nBlIj9tG*fQ?dSNvZ+ce6Lf
zApxi=p|_t7gd#V-4~8W5QN56SPSDYP>ppaQ(9?`65JKe#l`=Z>e|;+A;LY+Dy7p06
zNz3`R=QZAe%(#EL`GWqLy;1`qr=`JWU$KXjcsFL9TxiN<e}b}8xb-0D6;VUyKH_e`
z+%GgK)_4$MxbAX>|I5@984G2E^AlBm*xY*}aYDAyp)dFM_vTaC1iBxJmR|P0K=2-$
z(PL|Xg6!Gthqv8V*K;}g6*WjbomYc<i!r<#2JLnWFoq+y25S~TW5;yc<NZ{&sJny?
zUeA5x8FfDXdZ)m&)Ai9r6I9!1V_qdp6PgclJq%fd5wX*ji<Y;Xy}<iI!~VY5HwR!9
zshwsO7@aIXpQP*^XWlI}05FF0HabAa?Nd2|uRcNSszPTv5JN%tlQ#baiAyNZ9l8t6
z$(iF=<pvJTWZ;~2TE2>Qw}-%;@Q?@UlCd;+_M4;ebiIA+y3D%5;pDLGU^i!e=;ow-
z$;SSk!q4;8gyY!nZp{c2&NwRRtY3;luar}Ax;;MEBkiz9*@ncL#QMRuPuIZQP+lYd
zo$azY`vu>2x@cm$?AVZY9S_l)R_j-{OFaJuWt}@{j=>X5S`_I?pYa8fLoVXJTY_h(
zi6D3%g5-ZO(_M+P@%Zz=bG+mEEG*psYUuTYC3fTG_`AW?ls<BfmZF`lb+0sK^jb2{
zf!oJG-+%n;)M3r$tzWmMJj`%J9M5~?b42}o#3Ld&Kt6kcuF8n1_b~sb+|%hPVdPk5
zhF9saz@(fzQb6~M>T~Ovl7iDH^}sF>_R|Sc&N@6>vsK}LlaNEke@7EbWCW*zF3c(X
zpVEHv4VbHR?`^6pi?|)dyDuPB7k|#uA$}1%hZdhy!B0bY7}L=i)ay@Q9OaS=y6Lbc
zW98Fo?X~;CCV<vURan`RLjG0y-i>tb_VN%*Z-<eJ16%NOTMF<P{Cr@9Md9VT+II-m
z`!=zNccacL0u*VUZ%t9wT{>ClT>Wm)d^=lqRR=*o3mvS2yF%u`gfu)t__C4RER2<G
z7`zr{M#qTGer|(l4PnK3kO&;Ny0R&EtU6sC+Jmw$Czy`gy~<i`&$phHZ60>bq3%e_
zPjYkp%71a)!YL1GEJtHO`VfuRqr*tYN4L#cy0yisiU2HNGj<2h+x5g9!pRNB*dzEP
zH35Dwue)glYa4aGIejRbOIZ@p9;=thxT{-agXHkNtWjP)dBX$wllM?;4y4}KxkaPj
zaig6hyZV!>@=}N1($kIuQ})6A?F&@jwjbwpmlT-ufR}yHj!phLQ$m7Sd`jtCzO{OB
zHn^s>`TUCc($0S#ImyAYgv^T%B<vw2GfcZ14?W9quzOsL+u6WlzCVpSdwDeCCRJDy
zeXiI)qd;S?<iDQ7c^<3Ke0>aV-*;Ja%&9$R?zws@w)e;Cao(R4^|Vr7-PVc7@p8Nf
zv{g=1YN1@azoN{+Kz(YUB+GdjZ@{U<hl+tk>E$u^OaQ}EQ%g{ou(0rzTE@v`07#$c
z<?661R&%)SwvL<ksh3V`DEc{rvh26X*}in8{aQ=I{{Ch<Z{UH69F~CHv45XYoVV*M
z-piLqh+{LIn!n9ON%IkV*AFBgK#zf_>)}L|@)7O=lZOR?-51vE4PQxhk=66mGqtS6
z*HzwbeD>i^6VLmBMP5;$QTO2o!aG_F093cqY(T*Dsu)J#G;imz{Bm=)-MHkwX4QSM
zYa-&Ho#Qjk>j6XYS(0oMKJ9C%@_4A@UZPkPn10xfMr{^(*7=inry;|Ct9x<r$RKjl
z%IklPK6PdP`dHR1^3eJV-sPDc=zK&P4c>U>hsR%zYGsFB0fh0VhbD=3!;XhP&0r8=
zb!yh0=S!Hq0*`&x45z5=y1*4!dE-}ml`n8|fn;jw$f(lOF@L0~=2XRi>aC0hVb6_i
zPFcs9+uVV?-$Syr&fN|#eMXLM&ExOxmo%7YmjF}czN+6t4XS%ew|869_A^-BD0Vz<
zfWH~xDQ&zPKGj#Z`$oByg&qzlFV9x)?`|hMC$G8>JhOA|7cQK&-9@fi-hS9KsaKf3
zumUXCnS}LkIkc}`sivLPFVsCkT(`PId|QVNDuF$E{20bnnVkf6LQWGcMFNH$nUVUU
z4s)+EtF{sIlt7;~lA8roUj1eRec;6tY08wzTBxh_vZN~t<Ytn4w_1q|$E}I0G272s
zvp-kM&+^KSd{LN#)!z4_soAdTWitH>;na{;MyUja-T<Nwjk&~%vyzRy4#)Fkcbm!U
zo-y6af;}d-5a?b$qRqgm<|ne7y`gK%ihr%2Cirns6}4h#KSqqI*7@M`hNq?VkkM=G
zna=^LU<L;YVxYrH6U}SI6|hFx1EWPHM7*r+$XUCy9Hab&puuR{GyB_nj_Jz~3|1X%
z2gg|I_11-h@hqV!N?X!O&s_#6um64*#P6cs|1K_bWo3o?wq{eD+aoY`qx-v?+s)2x
z&3UM}z};vwHt5yJO!3u-aZ>m>)_=^p3xCBYFM11WfDpCzhK4^-EZ1cMeTsocxyw(^
z;GPbGW7u1eA-`1l)=Q!=Y>u2voRDU82#~u3%}7t*FDi%H>)2O&nXZvsG+#XOZ@6l8
zC41irICSr}iC*r`=-s?<pF5v#yj{~;eJF@TT7DScFTgUg05-ooNj=W7ZO9Xfy2CE-
z!<JGaHA;<O7zS(o_RdiDl?_X)XD3MOLC7A?g1`4k70_0iR8Nn>bm@rBO~xVbjQrcy
z>T=5xJC^A4gf@lOM~`DY@BUl8F^pK-b@GG|b*N^Z(@>c@C6nl1zNL%hhf91G%=q|N
z&+Mg?YX0J<>)37>tQkM&<;Nk3$aefOQTI!K>_Ru_P~E>a-qGN>y&I<_N7)bhh~wD_
zdu@E&OA*bkco75yI1Sytnpf)&-0}PEYdhYOiw-yl-SxlTEy28BuJ%Nqmjb*@{-*t{
z@0(5Ac(odMrhMJ(Bn_Z$RJrK&UUJ$y!trxf)EU0&*y@kzwq+x}@;^FOh6fTyD2wPi
zshysw=pQ!rN-El8F{<+r4w2_*VdakUnJ%ZJ67RO7kiBi;%)yqo^%gVmSpfHco(x;V
z@;0A)C(ruN6F62p9j=FrKA?(}7?ILAFTDpmu;OJP2<8U~x&iS*lp<_c?4Iie5Hy5n
zyYHvXm75wLDAv!HB`)t=owR!m2HKOMNLV%457|C;6*3_;JRJ|!RlZrT(a#k|j|3E$
zOBYWJD{D|~;l}2oDj0W%Hvzzja;qBNEtP)ugFTkv	Y5<Hw`0Ce^x5c=T#}ikrY0
zS!#IuV(TU2XYevCXs@SEQqxk^@<Z6R{{i$ItNRSC)>SSMCf?7yh<49SI=CljXaKsO
z-ltG1+pV=q^WBlLpE7%0T2r1qvb^G$ZTJ(Y^ui>*-KV_h+$@Mbp7+Bxo*yC)?l)ed
z4({{2eU2nHUcn_kqA$x(ju@X4ct|FA^Un`>4@t584Y13(QJ14gCdgzA9s2sz<otAI
z)cw$JUrKQ=|83%@vZyEQiT7@!`(^On9^dCd_S;rmI?b8B<rCfp^fl4{2O_fS1nMIr
zuV?n_Gt@Csx|;aSKg_Rx{j_IxIQ&nn*Ikug?kU$(u8q0jQR>h13nw5|p|}u=Pe@2#
zGkjYOJ6VYKZh99<5#jrEeQF*98ePKHtEpK$v0tXG|FOGc?q+Xi-R^&q3v2V42<=oJ
zf)Gj`#j;b>yF!cSs$R{tcNQftFpR8kKHTnu6*GXNlo%%hLp2W8{gEXc)xNMxcr$>U
zH3<GfFo!VcsI>cHe&%Y5SNKw2-d6%@ncTj3X(ja&JJ1)l82PAd)Gsl`Ekr~Bu@2&8
z=CO&f@)*F&Fd}U{GfXM{On1!;n7qAgbqAfk&N3TAuckVmE-P&JL<-rd@q%JkQg%Ek
zJT!(S{A(WjouM~RL8mxK&GuuapT6-=T^X(14j4QFuXf>0>Nm5MKby3>!MY!6{85Dj
z1O?%QSfu~UmVY+?`o&b1cX!iENX<+CQnz3CLldR4@Oj--_w_(&f5(<B8jld;<H~+9
z3&(4P->V^@&MagDr%AZwYW`{{bG+Zk^6}T(XQJ1O$kzs?yqB${vRL7nWl++=v(VIM
zL7Z-fC+{QsSL263Wg$a<q4`M_yf*Mn*Yc#3z5fAfQr!T}(mx*s1s_AQfU%TMGl3>X
zGHuS7|85DCzKRNd78cGrxl_LyGkUp<efb;vJw<vAleRht70UA$){OLIn|;5@<Kwq}
zVE%IZ&&$$*2-(X3FSbWy|3i2|hBnPhl#dU2%gEN3o5@=@yei`RJMvAGFwGtbn@n4&
zm;Em`&#D9ce%}Nad@lBHV;1YswuyETtyliGI}cQylZ3f*nr|)q<LeZSMAQai;W5tR
zw|DwIWPQfLR~Zp;V6aJ!onyM=o||Zhg|kh=x*R?<Reb4<`@h;d*Qh40Jb+Ii2x?j)
zZxne6SR_Cmtl0pDS5X2{pm-px7=%m~P#{8534%N|RY{v5LIg@8_R#IF2qL%`c^FJ9
z;hAoIfq+2K#1?_@PzXe_6Sg0=AG+If_V{h*!_1jE_kYjKojYe{elx%OZ*FdmUu#`n
z?=)J7qYC?SlMSNLnl0ASiNQHL9D-@?_LR2&kFBS7;g<vQRcF)LVzNt4SyJo-kC`5h
z>-S_R>$}OIYbAMarjIg`L$gxg#bT}8GfL06M+4PeI{)z#SJdWu3PzFi1d}&8{LL+F
z<!i>ef>9qm{c6U>K((1;YSK=2X)CRse%tp*T<E7-J9myiAUxF*+j1fo+9Q1bw)x(V
z!`4F2`{dO5AxYkRSwo{RPZ=McA9QN%)DL;5#BtYS8owSt;<WYO2-}Y~-?jfj*!Dj2
z+XT3_J}}E0BH5G>E*LVk+}w7HuCS7_25U9@10=C&$8Go7Re<mIN&evsK-JN8v{^K3
z_54rEY0-h2rfY6+G#=l5!%3qIy0+7iBDmTi;<hgAIcDyYWLnCrTRQ%TGX(ClPaITF
zvDdom*nY^<^;MOdzeR~^YTOfu5P{!o3wWsv6%||Hdea%z%C1Gt0mAH8K*30>dMM<6
z9h^I^4SD7Erim`ix$fL<CrP!ZOul|yzXursyV8~V&AwK~cr>or05YnKEqSR$Rce2V
z)sfso$Eej>fmU6UZ@n5g9XmWT^2uA%WbG9c<}y!k-f~9+QNyBz25;~5TKVwB3DKIu
zbo!wtRIV4MY@n1}3?WufrR|NCEhzb-WYymunUl>Ny%BQn=TLCq*^GYrN5rPN7mDzA
z0si*~GdKrbz->C*w+X=uUdkzv-}^yz@?uWrk@%E5eqCQE5+m~e?mFTyTDr=!+a)nP
z>-!~%ICv4d6CDi8u~&7CAx<Z0w;8Cq39Zmu4#5TdsW$gZZ=s<vCjsnFBs~b3zIpuF
zfkex+5)1Q909yrd8_zaD9-~yL_WSOJzrwuknwN?Q<WG3)G_l{<k?jyr1h}gGH_yN2
z^(0CcVUtzG6LGaK9QJ&^UhCluJ)%#U&HJ)tz^2^6MCbx7`5aBMvaLM|R=F>7by%h%
zhDdEn3|^To|0%(W1Cyiq7+C_;cutq4D(+22zdpj5r_LHbvrPAa2Pz&y3=0}JB%^H%
z>TA>{$Cn%&S!H&W&yA7s!*YC`nru`AgeQ61q~uffmQut(fB&7a*`lJN{_Bq5JcX)W
z39?)L{xd(#WHp}Asa;=7F;f--Jb)J~)4(`e(f(ae*JnCX@g=DJgRR}nO34-%esKX8
zrY|IR4p@rggd&lstW5enVb`s*sK#7`zlWgj<-$_$0wAaDmB<(F4B&9B>YsspAIXZo
z1|{!b@Rh*X!rJ1YK7=v#0U8iV(FBA@pUN4wF>5Bs$^$kL4_LG#`Tdra<O&Ek_K0@}
zT2#{oJe&!U`hkm~rq`UCS`qIs6$3|3WZNPDVUp9RtOVtXIQ!0v1M{TurS_lo^dx|8
zARmV12r$g_3KDkrsE5}`BoZrv5S%~Tue>neRqEAL1Kf@YCVe6sj`Svl_yqPL<;8_4
zb~dbR7Y&@CBTUH;T9{-b5@#Xf;>yH2MuICqw399F-oAZO{9(tOM|kb@?JLJZEo|Xs
zm@`o|$V+A~ryjwX4g5>issa=%Di7W$tSf%pXQJ;g$Kl0oRpVRGBa_<#E%rgsxNY%y
zFPs-Q9veE>))$wo2iJPqc+`>tj7kd@GO8T&WEGu<Llx&W-|jbZFo`4z4O?O$SI8-7
z=^X6$xI+I4m;V>d=>MU=%f@<7&4lgk##+}gp>rJeWQ@$TPP;%KZnO9Z`Wq`#FgD;N
zh$K?Qna;Wo0!FcyK<sdDx!c@x1(y4!(9l-a(<44HX#&M&1spmwsN>`KWkL){p}Lfv
zS&`ZaD>tT^nvN40x}3gS9d_?ddf8i^#gtR{DMSm<nFVbHd#*O$`?Y!Yo{sf@=!nsv
z53sSX*m&E^cU(3EI<Pi*y@k?y{0pt^U#1?v-y?X}sY|hYE5b&*&m-gDhN&kD99C=m
zLSkoL{B`c3E@|{{8h!tGqe~bgD{2dG;2vG+l$+*}*EYtwDk>p&WC$WQguhvi3~o~`
zblq?Qvo7#Cv@*=U8Z0Yn6oh<7T@{J`8?In1QmwvT*GT$hrP@|I+7mS%e)kzPA75Hg
zS_zfSJ9U|>Z^6%;Jh5FO;jItMbXF$UXf%(Z!Zrn3V<$UUzO?j-1^@s@RxpJ{W2B|g
z(^<d)I*XN(o@CEtC1kSbpuM|&P;w?CE#=d+2*C~-RN7fr8cbl(y=j?rXaQi+7_2)Q
z?~cP9#^AmGfb+(p-O*m&XmpYCFzHtTX$&ys&)L5nuz2_N8z=xE1t0pjA@EqyKLK=F
BYZ(9l

literal 0
HcmV?d00001


From 4b5ffe8f9b84c20912871b0dfe627d041ce2d99f Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 17:57:08 -0400
Subject: [PATCH 4409/8313] implemented the addrepository form

shiny!
---
 Assistant/WebApp.hs                          | 12 ++++++
 Assistant/WebApp/Configurators.hs            | 43 ++++++++++++++++----
 Assistant/WebApp/routes                      |  3 +-
 templates/configurators/addrepository.hamlet |  7 ++--
 4 files changed, 53 insertions(+), 12 deletions(-)

diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index 3351aa48fe..c2a021246e 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -95,6 +95,11 @@ instance Yesod WebApp where
 	makeSessionBackend = webAppSessionBackend
 	jsLoader _ = BottomOfHeadBlocking
 
+instance RenderMessage WebApp FormMessage where
+	renderMessage _ _ = defaultFormMessage
+
+type Form x = Html -> MForm WebApp WebApp (FormResult x, Widget)
+
 data WebAppState = WebAppState
 	{ showIntro :: Bool
 	}
@@ -145,3 +150,10 @@ instance PathPiece NotificationId where
 instance PathPiece AlertId where
     toPathPiece = pack . show
     fromPathPiece = readish . unpack
+
+{- Adds the auth parameter as a hidden field on a form. Must be put into
+ - every form. -}
+webAppFormAuthToken :: Widget
+webAppFormAuthToken = do
+	webapp <- lift getYesod
+	[whamlet|<input type="hidden" name="auth" value="#{secretToken webapp}">|]
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 9fe10aff92..69bf92fdb3 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -19,7 +19,7 @@ import Logs.Trust
 import Annex.UUID (getUUID)
 
 import Yesod
-import Data.Text (Text)
+import Data.Text (Text, pack)
 
 {- An intro message, list of repositories, and nudge to make more. -}
 introDisplay :: Text -> Widget
@@ -41,10 +41,44 @@ introDisplay ident = do
 	where
 		counter = map show ([1..] :: [Int])
 
+data RepositoryPath = RepositoryPath Text
+	deriving Show
+
+addRepositoryForm :: Form RepositoryPath
+addRepositoryForm msg = do
+	cwd <- liftIO $ getCurrentDirectory
+	(pathRes, pathView) <- mreq textField "" (Just $ pack cwd)
+	let widget = do
+		webAppFormAuthToken
+		toWidget [julius|
+$(function() {
+	$('##{fvId pathView}').focus();
+})
+|]
+		[whamlet|
+#{msg}
+<p>
+  <div .input-prepend .input-append>
+    <span .add-on>
+      <i .icon-folder-open></i>
+    ^{fvInput pathView}
+    <button type=submit .btn>
+      Make Repository
+|]
+	return (RepositoryPath <$> pathRes, widget)
+
 addRepository :: Bool -> Widget
 addRepository firstrun = do
 	setTitle $ if firstrun then "Getting started" else "Add repository"
-	$(widgetFile "configurators/addrepository")
+	((res, form), enctype) <- lift $ runFormGet addRepositoryForm
+	case res of
+		FormSuccess (RepositoryPath p) -> error $ "TODO" ++ show p
+		_ -> $(widgetFile "configurators/addrepository")
+
+getAddRepositoryR :: Handler RepHtml
+getAddRepositoryR = bootstrap (Just Config) $ do
+	sideBarDisplay
+	addRepository False
 
 getConfigR :: Handler RepHtml
 getConfigR = bootstrap (Just Config) $ do
@@ -55,8 +89,3 @@ getConfigR = bootstrap (Just Config) $ do
 			setTitle "Configuration"
 			$(widgetFile "configurators/main")
 		)
-
-getAddRepositoryR :: Handler RepHtml
-getAddRepositoryR = bootstrap (Just Config) $ do
-	sideBarDisplay
-	addRepository False
diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes
index 69e6078b02..95813edb6f 100644
--- a/Assistant/WebApp/routes
+++ b/Assistant/WebApp/routes
@@ -1,9 +1,10 @@
 / HomeR GET
 /noscript NoScriptR GET
 /noscriptauto NoScriptAutoR GET
+/about AboutR GET
+
 /config ConfigR GET
 /config/addrepository AddRepositoryR GET
-/about AboutR GET
 
 /transfers/#NotificationId TransfersR GET
 /sidebar/#NotificationId SideBarR GET
diff --git a/templates/configurators/addrepository.hamlet b/templates/configurators/addrepository.hamlet
index 20ece28067..7af450b877 100644
--- a/templates/configurators/addrepository.hamlet
+++ b/templates/configurators/addrepository.hamlet
@@ -10,7 +10,6 @@
     <p>
       Files in this repository will managed by git-annex, #
       and kept in sync with your repositories on other devices.
-    <form .form-inline>
-      <i class="icon-folder-open"></i> #
-      <input type="text" .input-xlarge placeholder="directory"> #
-      <button type="submit" .btn .btn-primary .btn-large>Make Repository</button>
+    <p>
+    <form .form-inline enctype=#{enctype}>
+      ^{form}

From c23650faf983766b5b704f17eea59f2ca44ff923 Mon Sep 17 00:00:00 2001
From: "85.84.137.70" <85.84.137.70@web>
Date: Tue, 31 Jul 2012 22:14:44 +0000
Subject: [PATCH 4410/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index f026c3951b..c77af9e023 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 0 "Annex" 0 "GitAnnex" 0 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
+[[!poll 0 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From b21a6dd081e43bf2cbd01f3bf2d7052f6e92c73e Mon Sep 17 00:00:00 2001
From: "24.59.180.123" <24.59.180.123@web>
Date: Tue, 31 Jul 2012 22:24:19 +0000
Subject: [PATCH 4411/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index c77af9e023..a6c84b35f6 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 0 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
+[[!poll 1 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 5765adc18b3b4d2aa156cf7d1abf3fadf18f6678 Mon Sep 17 00:00:00 2001
From: "95.34.39.27" <95.34.39.27@web>
Date: Tue, 31 Jul 2012 22:28:20 +0000
Subject: [PATCH 4412/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index a6c84b35f6..bb6bc60ac8 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 1 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
+[[!poll 2 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 6e194559d62f393fe7bd8d939b1cf47ab0382a2a Mon Sep 17 00:00:00 2001
From: "95.34.39.27" <95.34.39.27@web>
Date: Tue, 31 Jul 2012 22:28:36 +0000
Subject: [PATCH 4413/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index bb6bc60ac8..f4c8929fe4 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 2 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 0 "something lowercase!"]]
+[[!poll 1 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 1 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From c950e8fba0de95f3a102b2653f70457fc8f19ab3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 18:33:19 -0400
Subject: [PATCH 4414/8313] move out to template

---
 Assistant/WebApp/Configurators.hs             | 20 +++----------------
 Makefile                                      |  4 ++--
 .../configurators/addrepository/form.hamlet   |  8 ++++++++
 3 files changed, 13 insertions(+), 19 deletions(-)
 create mode 100644 templates/configurators/addrepository/form.hamlet

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 69bf92fdb3..6a467692a2 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -48,24 +48,10 @@ addRepositoryForm :: Form RepositoryPath
 addRepositoryForm msg = do
 	cwd <- liftIO $ getCurrentDirectory
 	(pathRes, pathView) <- mreq textField "" (Just $ pack cwd)
-	let widget = do
+	let form = do
 		webAppFormAuthToken
-		toWidget [julius|
-$(function() {
-	$('##{fvId pathView}').focus();
-})
-|]
-		[whamlet|
-#{msg}
-<p>
-  <div .input-prepend .input-append>
-    <span .add-on>
-      <i .icon-folder-open></i>
-    ^{fvInput pathView}
-    <button type=submit .btn>
-      Make Repository
-|]
-	return (RepositoryPath <$> pathRes, widget)
+		$(widgetFile "configurators/addrepository/form")
+	return (RepositoryPath <$> pathRes, form)
 
 addRepository :: Bool -> Widget
 addRepository firstrun = do
diff --git a/Makefile b/Makefile
index a6fdab7ca7..8eef53f30e 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ FEATURES=-DWITH_ASSISTANT -DWITH_S3 -DWITH_WEBAPP
 bins=git-annex
 mans=git-annex.1 git-annex-shell.1
 sources=Build/SysConfig.hs Utility/Touch.hs Utility/Mounts.hs
-thfiles=Assistant/Threads/WebApp.hs
+thfiles=Assistant/Threads/WebApp.hs $(shell find Assistant/WebApp*)
 all=$(bins) $(mans) docs
 
 OS:=$(shell uname | sed 's/[-_].*//')
@@ -58,7 +58,7 @@ Build/SysConfig.hs: configure.hs Build/TestConfig.hs Build/Configure.hs
 
 # Force GHC to rebuild template haskell files whenever includes
 # change
-$(thfiles): $(shell echo templates/* static/*)
+$(thfiles): $(shell find templates static)
 $(thfiles):
 	touch $(thfiles)
 
diff --git a/templates/configurators/addrepository/form.hamlet b/templates/configurators/addrepository/form.hamlet
new file mode 100644
index 0000000000..fa5d07f2d6
--- /dev/null
+++ b/templates/configurators/addrepository/form.hamlet
@@ -0,0 +1,8 @@
+#{msg}
+<p>
+  <div .input-prepend .input-append>
+    <span .add-on>
+      <i .icon-folder-open></i>
+    ^{fvInput pathView}
+    <button type=submit .btn>
+      Make Repository

From dde5937bafcbf93ecd445bcdd7496af330556752 Mon Sep 17 00:00:00 2001
From: "109.43.0.112" <109.43.0.112@web>
Date: Tue, 31 Jul 2012 22:36:03 +0000
Subject: [PATCH 4415/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index f4c8929fe4..f1dab6e6f9 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 1 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 1 "something lowercase!"]]
+[[!poll 1 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From c7c6843b4c8426919b6bcbff276d6e30afc05e3e Mon Sep 17 00:00:00 2001
From: "208.74.181.198" <208.74.181.198@web>
Date: Tue, 31 Jul 2012 22:43:42 +0000
Subject: [PATCH 4416/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index f1dab6e6f9..b74f7e14c8 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 1 "Annex" 0 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 1 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 5b096f86e9ceb98208eae06899f945b1024f0c1c Mon Sep 17 00:00:00 2001
From: "82.32.202.53" <82.32.202.53@web>
Date: Tue, 31 Jul 2012 22:48:08 +0000
Subject: [PATCH 4417/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index b74f7e14c8..a5a06c22d8 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 1 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 2 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 1e0b60f1649cbfaf2a3b48be601fd6dac39013d3 Mon Sep 17 00:00:00 2001
From: "71.171.115.88" <71.171.115.88@web>
Date: Tue, 31 Jul 2012 22:51:18 +0000
Subject: [PATCH 4418/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index a5a06c22d8..75628b156e 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 2 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 3 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From b603363455e4b3d66ae8d04f7405b1257a80d2f3 Mon Sep 17 00:00:00 2001
From: "71.171.115.88" <71.171.115.88@web>
Date: Tue, 31 Jul 2012 22:51:27 +0000
Subject: [PATCH 4419/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 75628b156e..5938c4c0dd 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 3 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 2 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From da9e8417bc7f3c740fac1443761f002c25721ff7 Mon Sep 17 00:00:00 2001
From: "71.171.115.88" <71.171.115.88@web>
Date: Tue, 31 Jul 2012 22:51:38 +0000
Subject: [PATCH 4420/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 5938c4c0dd..75628b156e 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 2 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
+[[!poll 3 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 858a9e5c5406ab61f84374c7f554ebf8cc7841c8 Mon Sep 17 00:00:00 2001
From: "140.107.155.156" <140.107.155.156@web>
Date: Tue, 31 Jul 2012 22:59:42 +0000
Subject: [PATCH 4421/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 75628b156e..ba3845d886 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 3 "Annex" 1 "GitAnnex" 1 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 3 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 96dc0d331b2ef50bc6195f356382b5ffabe01450 Mon Sep 17 00:00:00 2001
From: "75.168.84.33" <75.168.84.33@web>
Date: Tue, 31 Jul 2012 23:01:55 +0000
Subject: [PATCH 4422/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index ba3845d886..5157ddf48a 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 3 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 4 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 72d14af34c167f9549ff1c4ce9866009887637bc Mon Sep 17 00:00:00 2001
From: "69.204.133.212" <69.204.133.212@web>
Date: Tue, 31 Jul 2012 23:55:04 +0000
Subject: [PATCH 4423/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 5157ddf48a..d79c5ed6bf 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 4 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 5 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 01f9d5352d85044ae5d70074b4f7551f953d6749 Mon Sep 17 00:00:00 2001
From: "205.206.58.17" <205.206.58.17@web>
Date: Wed, 1 Aug 2012 00:26:42 +0000
Subject: [PATCH 4424/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index d79c5ed6bf..32444de855 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 5 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 6 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From d5b0777d0711df08fd3d3431684f76858ddc3b57 Mon Sep 17 00:00:00 2001
From: "205.206.58.17" <205.206.58.17@web>
Date: Wed, 1 Aug 2012 00:26:48 +0000
Subject: [PATCH 4425/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 32444de855..6c7d0965ad 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 6 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 7 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From ece6a728700ad58c0e0658a0cc2b755f85b27a81 Mon Sep 17 00:00:00 2001
From: "205.206.58.17" <205.206.58.17@web>
Date: Wed, 1 Aug 2012 00:26:52 +0000
Subject: [PATCH 4426/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 6c7d0965ad..e7f66715f2 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 7 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 8 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From deecf242e1f93d7848e9eb8e27899968f95d373a Mon Sep 17 00:00:00 2001
From: "205.206.58.17" <205.206.58.17@web>
Date: Wed, 1 Aug 2012 00:26:56 +0000
Subject: [PATCH 4427/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index e7f66715f2..ce41cfa207 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 8 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 9 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From e5dae828ba9401b18a08f443cd603b2756fc47e4 Mon Sep 17 00:00:00 2001
From: "205.206.58.17" <205.206.58.17@web>
Date: Wed, 1 Aug 2012 00:27:00 +0000
Subject: [PATCH 4428/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index ce41cfa207..69226adf63 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 9 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 10 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 56636a787b45d2ebfd066f464264cb14c65146ec Mon Sep 17 00:00:00 2001
From: "76.85.184.113" <76.85.184.113@web>
Date: Wed, 1 Aug 2012 00:41:02 +0000
Subject: [PATCH 4429/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 69226adf63..773f3492bf 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 10 "Annex" 1 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 10 "Annex" 2 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 9c869015f43b2f4442012a380cfd3ea16f83915d Mon Sep 17 00:00:00 2001
From: "59.167.133.99" <59.167.133.99@web>
Date: Wed, 1 Aug 2012 00:46:08 +0000
Subject: [PATCH 4430/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 773f3492bf..2c29ca153e 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 10 "Annex" 2 "GitAnnex" 2 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 10 "Annex" 2 "GitAnnex" 3 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From bab80bf24ada54f8dec2a35bbb77219441719f6a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 20:56:10 -0400
Subject: [PATCH 4431/8313] full input validation for repository path

Expands ~ , checks for every crazy input problem I can think of
---
 Assistant/WebApp/Configurators.hs             | 58 ++++++++++++++++++-
 .../configurators/addrepository/form.hamlet   |  3 +
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 6a467692a2..59acb763a6 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -20,6 +20,8 @@ import Annex.UUID (getUUID)
 
 import Yesod
 import Data.Text (Text, pack)
+import qualified Data.Text as T
+import Data.Char
 
 {- An intro message, list of repositories, and nudge to make more. -}
 introDisplay :: Text -> Widget
@@ -44,10 +46,64 @@ introDisplay ident = do
 data RepositoryPath = RepositoryPath Text
 	deriving Show
 
+{- Custom field display for a RepositoryPath, with an icon etc.
+ -
+ - Validates that the path entered is not empty, and is a safe value
+ - to use as a repository. -}
+repositoryPathField :: forall sub. Bool -> Field sub WebApp Text
+repositoryPathField autofocus = Field { fieldParse = parse, fieldView = view }
+	where
+		view idAttr nameAttr attrs val isReq =
+			[whamlet|<input type="text" *{attrs} id="#{idAttr}" name="#{nameAttr}" :isReq:required :autofocus:autofocus value="#{either id id val}">|]
+
+		parse [path]
+			| T.null path = nopath
+			| otherwise = liftIO $ checkRepositoryPath path
+		parse [] = return $ Right Nothing
+		parse _ = nopath
+
+		nopath = return $ Left "Enter a location for the repository"
+
+{- As well as checking the path for a lot of silly things, tilde is
+ - expanded in the returned path. -}
+checkRepositoryPath :: Text -> IO (Either (SomeMessage WebApp) (Maybe Text))
+checkRepositoryPath p = do
+	home <- myHomeDir
+	let basepath = expandTilde home $ T.unpack p
+	path <- absPath basepath
+	let parent = parentDir path
+	problems <- catMaybes <$> mapM runcheck
+		[ (return $ path == "/", "Enter the full path to use for the repository.")
+		, (return $ all isSpace basepath, "A blank path? Seems unlikely.")
+		, (doesFileExist path, "A file already exists with that name.")
+		, (return $ path == home, "Sorry, using git-annex for your whole home directory is not currently supported.")
+		, (not <$> doesDirectoryExist parent, "Parent directory does not exist.")
+		, (cannotWrite path, "Cannot write a repository there.")
+		]
+	return $ 
+		case headMaybe problems of
+			Nothing -> Right $ Just $ T.pack basepath
+			Just prob -> Left prob
+	where
+		runcheck (chk, msg) = ifM (chk)
+			( return $ Just msg
+			, return Nothing
+			)
+		cannotWrite path = do
+			tocheck <- ifM (doesDirectoryExist path)
+				(return path, return $ parentDir path)
+			not <$> (catchBoolIO $ fileAccess tocheck False True False)
+		expandTilde home ('~':path) = home </> path
+		expandTilde _ path = path
+
 addRepositoryForm :: Form RepositoryPath
 addRepositoryForm msg = do
 	cwd <- liftIO $ getCurrentDirectory
-	(pathRes, pathView) <- mreq textField "" (Just $ pack cwd)
+	(pathRes, pathView) <- mreq (repositoryPathField True) "" (Just $ pack $ cwd ++ "/")
+	let (err, errmsg) = case pathRes of
+		FormMissing -> (False, "")
+		FormFailure l -> (True, concat $ map T.unpack l)
+		FormSuccess _ -> (False, "")
 	let form = do
 		webAppFormAuthToken
 		$(widgetFile "configurators/addrepository/form")
diff --git a/templates/configurators/addrepository/form.hamlet b/templates/configurators/addrepository/form.hamlet
index fa5d07f2d6..e72dbcf431 100644
--- a/templates/configurators/addrepository/form.hamlet
+++ b/templates/configurators/addrepository/form.hamlet
@@ -6,3 +6,6 @@
     ^{fvInput pathView}
     <button type=submit .btn>
       Make Repository
+$if err
+  <div .alert .alert-error>
+    #{errmsg}

From e81e8913d9c663cbe680224e6237433e7508e7d3 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 21:06:30 -0400
Subject: [PATCH 4432/8313] default repository location

Unifying poll results, it's Annex in lowercase. :)

When cwd is HOME, use ~/Desktop/annex, unless there's no Desktop directory;
then use use ~/annex

If cwd is not $HOME, use cwd
---
 Assistant/WebApp/Configurators.hs | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 59acb763a6..d646e6fb78 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -19,7 +19,7 @@ import Logs.Trust
 import Annex.UUID (getUUID)
 
 import Yesod
-import Data.Text (Text, pack)
+import Data.Text (Text)
 import qualified Data.Text as T
 import Data.Char
 
@@ -96,10 +96,24 @@ checkRepositoryPath p = do
 		expandTilde home ('~':path) = home </> path
 		expandTilde _ path = path
 
+{- If run in the home directory, default to putting it in ~/Desktop/annex,
+ - when a Desktop directory exists, and ~/annex otherwise.
+ -
+ - If run in another directory, the user probably wants to put it there. -}
+defaultRepositoryPath :: IO FilePath
+defaultRepositoryPath = do
+	cwd <- liftIO $ getCurrentDirectory
+	home <- myHomeDir
+	if home == cwd
+		then ifM (doesDirectoryExist $ home </> "Desktop")
+			(return "~/Desktop/annex", return "~")
+		else return cwd
+
 addRepositoryForm :: Form RepositoryPath
 addRepositoryForm msg = do
-	cwd <- liftIO $ getCurrentDirectory
-	(pathRes, pathView) <- mreq (repositoryPathField True) "" (Just $ pack $ cwd ++ "/")
+	path <- T.pack . addTrailingPathSeparator
+		<$> liftIO defaultRepositoryPath
+	(pathRes, pathView) <- mreq (repositoryPathField True) ""(Just path)
 	let (err, errmsg) = case pathRes of
 		FormMissing -> (False, "")
 		FormFailure l -> (True, concat $ map T.unpack l)

From b9afb7785e84ba03cb44a9b6962dbd01feaa4aae Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 21:12:58 -0400
Subject: [PATCH 4433/8313] typo

---
 Assistant/WebApp/Configurators.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index d646e6fb78..69108e46e2 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -106,7 +106,7 @@ defaultRepositoryPath = do
 	home <- myHomeDir
 	if home == cwd
 		then ifM (doesDirectoryExist $ home </> "Desktop")
-			(return "~/Desktop/annex", return "~")
+			(return "~/Desktop/annex", return "~/annex")
 		else return cwd
 
 addRepositoryForm :: Form RepositoryPath

From 1efe4f3332680be5ad9d5d496939d6757fbd2b0a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Tue, 31 Jul 2012 21:34:29 -0400
Subject: [PATCH 4434/8313] only use smart default on first run

---
 Assistant/WebApp/Configurators.hs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 69108e46e2..b9630b10a5 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -96,15 +96,15 @@ checkRepositoryPath p = do
 		expandTilde home ('~':path) = home </> path
 		expandTilde _ path = path
 
-{- If run in the home directory, default to putting it in ~/Desktop/annex,
- - when a Desktop directory exists, and ~/annex otherwise.
+{- On first run, if run in the home directory, default to putting it in
+ - ~/Desktop/annex, when a Desktop directory exists, and ~/annex otherwise.
  -
  - If run in another directory, the user probably wants to put it there. -}
-defaultRepositoryPath :: IO FilePath
-defaultRepositoryPath = do
+defaultRepositoryPath :: Bool -> IO FilePath
+defaultRepositoryPath firstrun = do
 	cwd <- liftIO $ getCurrentDirectory
 	home <- myHomeDir
-	if home == cwd
+	if home == cwd && firstRun
 		then ifM (doesDirectoryExist $ home </> "Desktop")
 			(return "~/Desktop/annex", return "~/annex")
 		else return cwd
@@ -112,7 +112,7 @@ defaultRepositoryPath = do
 addRepositoryForm :: Form RepositoryPath
 addRepositoryForm msg = do
 	path <- T.pack . addTrailingPathSeparator
-		<$> liftIO defaultRepositoryPath
+		<$> liftIO defaultRepositoryPath =<< lift inFirstRun
 	(pathRes, pathView) <- mreq (repositoryPathField True) ""(Just path)
 	let (err, errmsg) = case pathRes of
 		FormMissing -> (False, "")

From 006652d811b6d09d48c0850730e25e766ffa9bc9 Mon Sep 17 00:00:00 2001
From: "109.129.132.3" <109.129.132.3@web>
Date: Wed, 1 Aug 2012 02:24:43 +0000
Subject: [PATCH 4435/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 2c29ca153e..9076dcc365 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 10 "Annex" 2 "GitAnnex" 3 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 10 "Annex" 3 "GitAnnex" 3 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 3e70950c166830ed2fcc7ca166fd6f032ee91793 Mon Sep 17 00:00:00 2001
From: "140.79.22.5" <140.79.22.5@web>
Date: Wed, 1 Aug 2012 02:50:47 +0000
Subject: [PATCH 4436/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 9076dcc365..513b1ed541 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 10 "Annex" 3 "GitAnnex" 3 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 10 "Annex" 3 "GitAnnex" 4 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 7cc4a298ec04e724919021f846aaac03d4948b7c Mon Sep 17 00:00:00 2001
From: "61.7.176.125" <61.7.176.125@web>
Date: Wed, 1 Aug 2012 03:34:52 +0000
Subject: [PATCH 4437/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 513b1ed541..9ac9c64486 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 10 "Annex" 3 "GitAnnex" 4 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 10 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 8de121108891ef4d83db91d1c1a852985e4d6ac7 Mon Sep 17 00:00:00 2001
From: "37.209.42.124" <37.209.42.124@web>
Date: Wed, 1 Aug 2012 04:18:58 +0000
Subject: [PATCH 4438/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 9ac9c64486..411d36985b 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 10 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 11 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 5c5aa4440182cb01f8013389f115cfe0cdfaa180 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawlwYMdU0H7P7MMlD0v_BcczO-ZkYHY4zuY"
 <Morris@web>
Date: Wed, 1 Aug 2012 04:49:48 +0000
Subject: [PATCH 4439/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 411d36985b..bfcd7f8211 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 11 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 12 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 6de4a37026d99ef1c3ef9be039c537d3903eac12 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Wed, 1 Aug 2012 04:57:15 +0000
Subject: [PATCH 4440/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index bfcd7f8211..16f2b9ac21 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 12 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 13 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 3939b4d0201b8346071998361c89ef84513e7e13 Mon Sep 17 00:00:00 2001
From: "193.23.174.18" <193.23.174.18@web>
Date: Wed, 1 Aug 2012 04:58:05 +0000
Subject: [PATCH 4441/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 16f2b9ac21..068ac13dfb 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 13 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 14 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 51134005700e55d6e0707cfaf6c07cd1e5d423d0 Mon Sep 17 00:00:00 2001
From: "108.202.177.46" <108.202.177.46@web>
Date: Wed, 1 Aug 2012 05:16:15 +0000
Subject: [PATCH 4442/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 068ac13dfb..296d8a3d41 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 14 "Annex" 3 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 14 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 5a8c44c858a2e860d21b6aa8699ef943621c5c32 Mon Sep 17 00:00:00 2001
From: "193.162.34.14" <193.162.34.14@web>
Date: Wed, 1 Aug 2012 06:26:11 +0000
Subject: [PATCH 4443/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 296d8a3d41..ac60f8fd63 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 14 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 15 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 46cf6dba84929c4ef997386a637005941558892d Mon Sep 17 00:00:00 2001
From: "193.162.34.14" <193.162.34.14@web>
Date: Wed, 1 Aug 2012 06:26:19 +0000
Subject: [PATCH 4444/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index ac60f8fd63..fc5e5d832f 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 2 "something lowercase!"]]
+[[!poll 14 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From f33110562795027512ff230b0c484d5c374382fd Mon Sep 17 00:00:00 2001
From: "193.253.219.219" <193.253.219.219@web>
Date: Wed, 1 Aug 2012 06:40:33 +0000
Subject: [PATCH 4445/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index fc5e5d832f..08d05b47d7 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 14 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
+[[!poll 15 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 96a6b16ef998b237fbb328657fbe01654fe76d18 Mon Sep 17 00:00:00 2001
From: "87.194.100.189" <87.194.100.189@web>
Date: Wed, 1 Aug 2012 06:58:25 +0000
Subject: [PATCH 4446/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 08d05b47d7..fb05cc402d 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 4 "GitAnnex" 5 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
+[[!poll 15 "Annex" 4 "GitAnnex" 6 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 1656d86429c30ef3c824582e93a034cc40ccf6a7 Mon Sep 17 00:00:00 2001
From: "109.43.0.112" <109.43.0.112@web>
Date: Wed, 1 Aug 2012 07:32:40 +0000
Subject: [PATCH 4447/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index fb05cc402d..99a1fdc620 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 4 "GitAnnex" 6 "Synced" 0 "AutoSynced" 0 "Shared" 3 "something lowercase!"]]
+[[!poll 15 "Annex" 4 "GitAnnex" 6 "Synced" 0 "AutoSynced" 0 "Shared" 4 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From da5846d600b6364f36abf11e1b9a2d322db63894 Mon Sep 17 00:00:00 2001
From: "http://nico.kaiser.me/" <Nico_Kaiser@web>
Date: Wed, 1 Aug 2012 07:51:36 +0000
Subject: [PATCH 4448/8313] Added a comment: Directory location

---
 .../comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment

diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment
new file mode 100644
index 0000000000..9f1d74e677
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_1_e146cf06c8dd6303dd6a991f152a73fe._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://nico.kaiser.me/"
+ nickname="Nico Kaiser"
+ subject="Directory location"
+ date="2012-08-01T07:51:36Z"
+ content="""
+I don't think the Annex directory (or whatever it will be called – \"Annex\" seems like a good choice in my opinion) should be inside the \"Desktop\" folder by default. All other xdg-user-dirs (Documents, Downloads, etc.) are in $HOME, and i think Desktop should not be cluttered with folders.
+"""]]

From 4bd6b3129bbba2e5b0b6fe27a196ae0b2c6dd514 Mon Sep 17 00:00:00 2001
From: "193.29.115.250" <193.29.115.250@web>
Date: Wed, 1 Aug 2012 07:56:15 +0000
Subject: [PATCH 4449/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 99a1fdc620..14208c83d1 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 4 "GitAnnex" 6 "Synced" 0 "AutoSynced" 0 "Shared" 4 "something lowercase!"]]
+[[!poll 15 "Annex" 4 "GitAnnex" 6 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 05ea639c92d8b850f43e6c096d318706603760fe Mon Sep 17 00:00:00 2001
From: "80.187.96.149" <80.187.96.149@web>
Date: Wed, 1 Aug 2012 07:56:25 +0000
Subject: [PATCH 4450/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 14208c83d1..c37837ef82 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 4 "GitAnnex" 6 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
+[[!poll 15 "Annex" 4 "GitAnnex" 7 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From a62d959340dd24ace2cf8aaea8ef4786b42c5a33 Mon Sep 17 00:00:00 2001
From: "129.187.210.106" <129.187.210.106@web>
Date: Wed, 1 Aug 2012 08:12:58 +0000
Subject: [PATCH 4451/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index c37837ef82..77669a8c11 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 4 "GitAnnex" 7 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
+[[!poll 15 "Annex" 5 "GitAnnex" 7 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 87f39f2b1263d2c9f7b7115d89eb3c8ad9cfdbf4 Mon Sep 17 00:00:00 2001
From: "84.48.82.161" <84.48.82.161@web>
Date: Wed, 1 Aug 2012 08:13:10 +0000
Subject: [PATCH 4452/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 77669a8c11..238a1a6be0 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 15 "Annex" 5 "GitAnnex" 7 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
+[[!poll 16 "Annex" 5 "GitAnnex" 7 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 2b2c05152013a961ef7e0aea0068a14e7da4536b Mon Sep 17 00:00:00 2001
From: "137.226.146.228" <137.226.146.228@web>
Date: Wed, 1 Aug 2012 08:36:11 +0000
Subject: [PATCH 4453/8313] poll vote (Shared)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 238a1a6be0..607652ac37 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 16 "Annex" 5 "GitAnnex" 7 "Synced" 0 "AutoSynced" 0 "Shared" 5 "something lowercase!"]]
+[[!poll 16 "Annex" 5 "GitAnnex" 7 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 9a4c31fb7997d9d360022e91fca2b6620abb19a1 Mon Sep 17 00:00:00 2001
From: "134.169.117.246" <134.169.117.246@web>
Date: Wed, 1 Aug 2012 08:38:42 +0000
Subject: [PATCH 4454/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 607652ac37..a928d3fea9 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 16 "Annex" 5 "GitAnnex" 7 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 16 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 58c18d4a88dae16f50c8b15054cf899cf2385c84 Mon Sep 17 00:00:00 2001
From: "89.171.198.10" <89.171.198.10@web>
Date: Wed, 1 Aug 2012 08:53:04 +0000
Subject: [PATCH 4455/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index a928d3fea9..0a9e2bb885 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 16 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 17 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 2b7a2a58f82ec53f001cb6524bb3a2c72edc865e Mon Sep 17 00:00:00 2001
From: "130.226.133.41" <130.226.133.41@web>
Date: Wed, 1 Aug 2012 09:48:29 +0000
Subject: [PATCH 4456/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 0a9e2bb885..263577ef55 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 17 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 18 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From d85355b083750811fd30540383adb83acbc94a7e Mon Sep 17 00:00:00 2001
From: "2001:980:45c4:1:4c53:6a79:d31e:99fd"
 <2001:980:45c4:1:4c53:6a79:d31e:99fd@web>
Date: Wed, 1 Aug 2012 09:54:08 +0000
Subject: [PATCH 4457/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 263577ef55..fd837d30cb 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 18 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 19 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 65bcd9a11770c93d975e2d8f519289ffc32e8d5b Mon Sep 17 00:00:00 2001
From: "87.156.252.180" <87.156.252.180@web>
Date: Wed, 1 Aug 2012 09:56:23 +0000
Subject: [PATCH 4458/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index fd837d30cb..91bad1f19a 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 19 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 33ebda987028b73190f24bceb322126543f305f3 Mon Sep 17 00:00:00 2001
From: "87.156.252.180" <87.156.252.180@web>
Date: Wed, 1 Aug 2012 09:56:32 +0000
Subject: [PATCH 4459/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 91bad1f19a..a69dfaccd1 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 5 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!"]]
+[[!poll 19 "Annex" 6 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From c7a661677f1ed79e47723190a5fa4c2111ec8146 Mon Sep 17 00:00:00 2001
From: "31.164.248.87" <31.164.248.87@web>
Date: Wed, 1 Aug 2012 10:29:53 +0000
Subject: [PATCH 4460/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index a69dfaccd1..843bdaea3e 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 6 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 20 "Annex" 6 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 82f4046c533b27819f290212062bf3e61d4518b0 Mon Sep 17 00:00:00 2001
From: "77.109.139.87" <77.109.139.87@web>
Date: Wed, 1 Aug 2012 10:32:27 +0000
Subject: [PATCH 4461/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 843bdaea3e..63a1777774 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 20 "Annex" 6 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 5 "something lowercase!"]]
+[[!poll 20 "Annex" 6 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 493ee20ffc10fa4a9b90c7c4b141948745d038ba Mon Sep 17 00:00:00 2001
From: "130.183.86.174" <130.183.86.174@web>
Date: Wed, 1 Aug 2012 11:23:42 +0000
Subject: [PATCH 4462/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 63a1777774..00b0f2bb44 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 20 "Annex" 6 "GitAnnex" 8 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!"]]
+[[!poll 20 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From f5c7f80666d7465c767fdf9b63cb234bdaa75d08 Mon Sep 17 00:00:00 2001
From: fmarier <fmarier@web>
Date: Wed, 1 Aug 2012 11:41:38 +0000
Subject: [PATCH 4463/8313]

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 00b0f2bb44..dcbe5c3793 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 20 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!"]]
+[[!poll 20 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 2b75a36ef298b458396b9f0002701c742c844d37 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" <http://joeyh.name/@web>
Date: Wed, 1 Aug 2012 14:04:08 +0000
Subject: [PATCH 4464/8313] Added a comment

---
 ...mment_2_5d6adcf6782c02283bef6189582ee467._comment | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment

diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment
new file mode 100644
index 0000000000..40f75ca0bd
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_2_5d6adcf6782c02283bef6189582ee467._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ ip="4.154.3.236"
+ subject="comment 2"
+ date="2012-08-01T14:04:08Z"
+ content="""
+Hmm, my thought on putting it under Desktop is that'll let me install git-annex and tell my mom (really!) that files dragged to there are shared. Don't the other xdg directories get used as default save/open locations for things like word processors, so not need to be on the desktop?
+
+Although that may not be necessary. I've realized I can, at least on linux, easily make a button on the webapp that pops open the directory in the desktop's file manager.
+
+I guess another way of looking at the question is: Where does dropbox put its folder?
+"""]]

From 8cbb3d3b6899dfe2cd066934944c592a9a785bb9 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 10:08:21 -0400
Subject: [PATCH 4465/8313] corrected a few duplicate votes (doesn't really
 change result)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index dcbe5c3793..ccb3124862 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 20 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 16 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From a0b9153c4e695069275d75935176bf0d4ad3a1eb Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawllHWUwOT-UVe5bGkk8G87bpeRAzy-5T7w"
 <Rick@web>
Date: Wed, 1 Aug 2012 14:18:21 +0000
Subject: [PATCH 4466/8313] Added a comment: Folder location

---
 ...ent_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment

diff --git a/doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment b/doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment
new file mode 100644
index 0000000000..22eff96b65
--- /dev/null
+++ b/doc/design/assistant/blog/day_49__first_run_experience/comment_3_7ac2e34c2a7bc9b57488ca0c91307d32._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawllHWUwOT-UVe5bGkk8G87bpeRAzy-5T7w"
+ nickname="Rick"
+ subject="Folder location"
+ date="2012-08-01T14:18:21Z"
+ content="""
+I'm of two minds about where the Annex folder should be.
+
+For beginners, the desktop is ideal. No doubt about it. And more experienced users can override the default path during the install.
+
+On the other hand, many of the other 'synced folder' services place their folder in the user's home. Both Dropbox and Skydrive do this. It's where I'd put it for consistency, and to keep my desktop free of clutter.
+
+It's a close call, but if push came to shove, I'd probably go with the user home folder.
+"""]]

From 3fab8a6d7dd4d55edefe1c3cff192e1912ffc5f4 Mon Sep 17 00:00:00 2001
From: "173.160.118.149" <173.160.118.149@web>
Date: Wed, 1 Aug 2012 16:04:30 +0000
Subject: [PATCH 4467/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index ccb3124862..91bc691f8c 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 16 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 6 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 16 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 7 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From e652723b195c18d46479fae03205ed8a56af288d Mon Sep 17 00:00:00 2001
From: "173.228.13.253" <173.228.13.253@web>
Date: Wed, 1 Aug 2012 16:39:17 +0000
Subject: [PATCH 4468/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 91bc691f8c..4be6db5f10 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 16 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 7 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 16 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 8 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From c41f0d6cb8cc6c1401b1ec4816e8b4f8f549fd42 Mon Sep 17 00:00:00 2001
From: "131.104.156.128" <131.104.156.128@web>
Date: Wed, 1 Aug 2012 17:16:40 +0000
Subject: [PATCH 4469/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 4be6db5f10..3fcf9015f8 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 16 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 8 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 17 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 8 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 351a6989c3add4557ca4a3c3c96270792e1aae5d Mon Sep 17 00:00:00 2001
From: "85.181.129.49" <85.181.129.49@web>
Date: Wed, 1 Aug 2012 17:36:44 +0000
Subject: [PATCH 4470/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 3fcf9015f8..f0a227d663 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 17 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 8 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 18 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 8 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 435382861a7092c7667860189510519cd8c857b4 Mon Sep 17 00:00:00 2001
From: "95.16.87.239" <95.16.87.239@web>
Date: Wed, 1 Aug 2012 17:45:22 +0000
Subject: [PATCH 4471/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index f0a227d663..e07d88d3e2 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 18 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 8 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 18 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 9 "something lowercase!" 1 "CowboyNeal"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From 5d948adf99107931facf327d530a1625153a28da Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawnb4DPapYuC5mwL1lZPhz18mpJqbr5Wkjw"
 <Javier@web>
Date: Wed, 1 Aug 2012 17:47:03 +0000
Subject: [PATCH 4472/8313] Adding Annexbox option

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index e07d88d3e2..20b8cbebbb 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 18 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 9 "something lowercase!" 1 "CowboyNeal"]]
+[[!poll 18 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 9 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From af36bf6669a907e011c8541dec4c0b312e6a414c Mon Sep 17 00:00:00 2001
From: "128.189.202.237" <128.189.202.237@web>
Date: Wed, 1 Aug 2012 17:56:24 +0000
Subject: [PATCH 4473/8313] poll vote (Annex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 20b8cbebbb..2eb80f9cb0 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 18 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 9 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
+[[!poll 19 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 9 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From b5bf025e6fa7b416a45367211684a6030f36251d Mon Sep 17 00:00:00 2001
From: "2600:3c00::f03c:91ff:fedf:a0a8" <2600:3c00::f03c:91ff:fedf:a0a8@web>
Date: Wed, 1 Aug 2012 18:16:13 +0000
Subject: [PATCH 4474/8313] poll vote (something lowercase!)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 2eb80f9cb0..f450bc3e50 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 9 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
+[[!poll 19 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From ef327ed5449901eab3ada5dc5d13e1ba0ad98aec Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawkSq2FDpK2n66QRUxtqqdbyDuwgbQmUWus"
 <Jimmy@web>
Date: Wed, 1 Aug 2012 18:20:48 +0000
Subject: [PATCH 4475/8313]

---
 doc/bugs/Build_failure_at_commit_1efe4f3.mdwn | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 doc/bugs/Build_failure_at_commit_1efe4f3.mdwn

diff --git a/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn b/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn
new file mode 100644
index 0000000000..a82d6d0456
--- /dev/null
+++ b/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn
@@ -0,0 +1,44 @@
+Applying this
+
+<pre>
+laplace:git-annex jtang$ git diff
+diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
+index b9630b1..bf36e59 100644
+--- a/Assistant/WebApp/Configurators.hs
++++ b/Assistant/WebApp/Configurators.hs
+@@ -101,7 +101,7 @@ checkRepositoryPath p = do
+  -
+  - If run in another directory, the user probably wants to put it there. -}
+ defaultRepositoryPath :: Bool -> IO FilePath
+-defaultRepositoryPath firstrun = do
++defaultRepositoryPath firstRun = do
+        cwd <- liftIO $ getCurrentDirectory
+        home <- myHomeDir
+        if home == cwd && firstRun
+</pre>
+
+Causes this to occur,
+
+<pre>
+Assistant/WebApp/Configurators.hs:114:17:
+    Couldn't match expected type `Control.Monad.Trans.RWS.Lazy.RWST
+                                    (Maybe (Env, FileEnv), WebApp, [Yesod.Form.Types.Lang])
+                                    Enctype
+                                    Ints
+                                    (GHandler WebApp WebApp)
+                                    t0'
+                with actual type `Text'
+    Expected type: String
+                   -> Control.Monad.Trans.RWS.Lazy.RWST
+                        (Maybe (Env, FileEnv), WebApp, [Yesod.Form.Types.Lang])
+                        Enctype
+                        Ints
+                        (GHandler WebApp WebApp)
+                        t0
+      Actual type: String -> Text
+    In the first argument of `(.)', namely `T.pack'
+    In the first argument of `(<$>)', namely
+      `T.pack . addTrailingPathSeparator'
+make: *** [git-annex] Error 1
+</pre>
+

From d2c6d16458dc5b6f400ce7c1e6bc5e06d8c959db Mon Sep 17 00:00:00 2001
From: "83.145.224.122" <83.145.224.122@web>
Date: Wed, 1 Aug 2012 18:28:46 +0000
Subject: [PATCH 4476/8313] poll vote (Synced)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index f450bc3e50..78d122e88a 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 6 "GitAnnex" 9 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
+[[!poll 19 "Annex" 6 "GitAnnex" 10 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From a3671074b54705dfc60fc2634ddfb8826c924568 Mon Sep 17 00:00:00 2001
From: "2001:15c0:66bc:0:f2de:f1ff:fe77:7ecb"
 <2001:15c0:66bc:0:f2de:f1ff:fe77:7ecb@web>
Date: Wed, 1 Aug 2012 19:36:05 +0000
Subject: [PATCH 4477/8313] poll vote (GitAnnex)

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 78d122e88a..6ee3634f31 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 6 "GitAnnex" 10 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
+[[!poll 19 "Annex" 7 "GitAnnex" 10 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From ecc168aba30a0477381bcd2037c8d301368f3449 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 16:10:26 -0400
Subject: [PATCH 4478/8313] implemented firstrun repository creation and
 redirection to full webapp

Some of the trickiest code I've possibly ever written.
---
 Assistant.hs                      | 24 +++++++++-------
 Assistant/Threads/WebApp.hs       | 22 +++++++++++----
 Assistant/WebApp.hs               |  1 +
 Assistant/WebApp/Configurators.hs | 41 ++++++++++++++++++++++++---
 Command/WebApp.hs                 | 47 +++++++++++++++++++++++++++----
 5 files changed, 110 insertions(+), 25 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 4bb85975b8..be84fab55e 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -122,7 +122,10 @@ import Utility.ThreadScheduler
 
 import Control.Concurrent
 
-startDaemon :: Bool -> Bool -> Maybe (FilePath -> IO ()) -> Annex ()
+stopDaemon :: Annex ()
+stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
+
+startDaemon :: Bool -> Bool -> Maybe (Url -> FilePath -> IO ()) -> Annex ()
 startDaemon assistant foreground webappwaiter
 	| foreground = do
 		showStart (if assistant then "assistant" else "watch") "."
@@ -132,10 +135,15 @@ startDaemon assistant foreground webappwaiter
 		pidfile <- fromRepo gitAnnexPidFile
 		go $ Utility.Daemon.daemonize logfd (Just pidfile) False
 	where
-		go daemonize = withThreadState $ \st -> do
-			checkCanWatch
-			dstatus <- startDaemonStatus
-			liftIO $ daemonize $ run dstatus st
+		go d = startAssistant assistant d webappwaiter
+
+startAssistant :: Bool -> (IO () -> IO ()) -> Maybe (Url -> FilePath -> IO ()) -> Annex ()
+startAssistant assistant daemonize webappwaiter = do
+	withThreadState $ \st -> do
+		checkCanWatch
+		dstatus <- startDaemonStatus
+		liftIO $ daemonize $ run dstatus st
+	where
 		run dstatus st = do
 			changechan <- newChangeChan
 			commitchan <- newCommitChan
@@ -155,12 +163,8 @@ startDaemon assistant foreground webappwaiter
 				, mountWatcherThread st dstatus scanremotes
 				, transferScannerThread st dstatus scanremotes transferqueue
 #ifdef WITH_WEBAPP
-				, webAppThread (Just st) dstatus transferqueue webappwaiter
+				, webAppThread (Just st) dstatus transferqueue Nothing webappwaiter
 #endif
 				, watchThread st dstatus transferqueue changechan
 				]
-			debug "Assistant" ["all threads started"]
 			waitForTermination
-
-stopDaemon :: Annex ()
-stopDaemon = liftIO . Utility.Daemon.stopDaemon =<< fromRepo gitAnnexPidFile
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index ad2bff8923..a5484b5bec 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -38,8 +38,16 @@ thisThread = "WebApp"
 
 mkYesodDispatch "WebApp" $(parseRoutesFile "Assistant/WebApp/routes")
 
-webAppThread :: (Maybe ThreadState) -> DaemonStatusHandle -> TransferQueue -> Maybe (FilePath -> IO ()) -> IO ()
-webAppThread mst dstatus transferqueue onstartup = do
+type Url = String
+
+webAppThread 
+	:: (Maybe ThreadState) 
+	-> DaemonStatusHandle
+	-> TransferQueue
+	-> Maybe (IO String)
+	-> Maybe (Url -> FilePath -> IO ())
+	-> IO ()
+webAppThread mst dstatus transferqueue postfirstrun onstartup = do
 	webapp <- WebApp
 		<$> pure mst
 		<*> pure dstatus
@@ -48,6 +56,7 @@ webAppThread mst dstatus transferqueue onstartup = do
 		<*> getreldir mst
 		<*> pure $(embed "static")
 		<*> newWebAppState
+		<*> pure postfirstrun
 	app <- toWaiAppPlain webapp
 	app' <- ifM debugEnabled
 		( return $ httpDebugLogger app
@@ -66,7 +75,7 @@ webAppThread mst dstatus transferqueue onstartup = do
 				else dir
 		go port webapp htmlshim = do
 			writeHtmlShim webapp port htmlshim
-			maybe noop (\a -> a htmlshim) onstartup
+			maybe noop (\a -> a (myUrl webapp port) htmlshim) onstartup
 
 {- Creates a html shim file that's used to redirect into the webapp,
  - to avoid exposing the secretToken when launching the web browser. -}
@@ -85,5 +94,8 @@ writeHtmlShim webapp port file = do
 genHtmlShim :: WebApp -> PortNumber -> String
 genHtmlShim webapp port = renderHtml $(shamletFile $ hamletTemplate "htmlshim")
 	where
-		url = "http://localhost:" ++ show port ++
-			"/?auth=" ++ unpack (secretToken webapp)
+		url = myUrl webapp port
+
+myUrl :: WebApp -> PortNumber -> Url
+myUrl webapp port = "http://localhost:" ++ show port ++
+	"/?auth=" ++ unpack (secretToken webapp)
diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index c2a021246e..1b767c642d 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -37,6 +37,7 @@ data WebApp = WebApp
 	, relDir :: Maybe FilePath
 	, getStatic :: Static
 	, webAppState :: TMVar WebAppState
+	, postFirstRun :: Maybe (IO String)
 	}
 
 data NavBarItem = DashBoard | Config | About
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index b9630b10a5..5c2a1f25ed 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -17,11 +17,16 @@ import qualified Remote
 import Logs.Web (webUUID)
 import Logs.Trust
 import Annex.UUID (getUUID)
+import Init
+import qualified Git.Construct
+import qualified Git.Config
+import qualified Annex
 
 import Yesod
 import Data.Text (Text)
 import qualified Data.Text as T
 import Data.Char
+import System.Posix.Directory
 
 {- An intro message, list of repositories, and nudge to make more. -}
 introDisplay :: Text -> Widget
@@ -104,7 +109,7 @@ defaultRepositoryPath :: Bool -> IO FilePath
 defaultRepositoryPath firstrun = do
 	cwd <- liftIO $ getCurrentDirectory
 	home <- myHomeDir
-	if home == cwd && firstRun
+	if home == cwd && firstrun
 		then ifM (doesDirectoryExist $ home </> "Desktop")
 			(return "~/Desktop/annex", return "~/annex")
 		else return cwd
@@ -112,8 +117,8 @@ defaultRepositoryPath firstrun = do
 addRepositoryForm :: Form RepositoryPath
 addRepositoryForm msg = do
 	path <- T.pack . addTrailingPathSeparator
-		<$> liftIO defaultRepositoryPath =<< lift inFirstRun
-	(pathRes, pathView) <- mreq (repositoryPathField True) ""(Just path)
+		<$> (liftIO . defaultRepositoryPath =<< lift inFirstRun)
+	(pathRes, pathView) <- mreq (repositoryPathField True) "" (Just path)
 	let (err, errmsg) = case pathRes of
 		FormMissing -> (False, "")
 		FormFailure l -> (True, concat $ map T.unpack l)
@@ -128,8 +133,36 @@ addRepository firstrun = do
 	setTitle $ if firstrun then "Getting started" else "Add repository"
 	((res, form), enctype) <- lift $ runFormGet addRepositoryForm
 	case res of
-		FormSuccess (RepositoryPath p) -> error $ "TODO" ++ show p
+		FormSuccess (RepositoryPath p) -> go $ T.unpack p
 		_ -> $(widgetFile "configurators/addrepository")
+	where
+		go path
+			| firstrun = lift $ startFullAssistant path
+			| otherwise = error "TODO"
+
+{- Bootstraps from first run mode to a fully running assistant in a
+ - repository, by running the postFirstRun callback, which returns the
+ - url to the new webapp. -}
+startFullAssistant :: FilePath -> Handler ()
+startFullAssistant path = do
+	webapp <- getYesod
+	url <- liftIO $ do
+		makeRepo path
+		changeWorkingDirectory path
+		putStrLn "pre run"
+		r <- fromJust $ postFirstRun webapp
+		putStrLn $ "got " ++ r
+		return r
+	redirect $ T.pack url
+
+{- Makes a new git-annex repository. -}
+makeRepo :: FilePath -> IO ()
+makeRepo path = do
+	unlessM (boolSystem "git" [Param "init", Param "--quiet", File path]) $
+		error "git init failed!"
+	g <- Git.Config.read =<< Git.Construct.fromPath path
+	state <- Annex.new g
+	Annex.eval state $ initialize $ Just "new repo"
 
 getAddRepositoryR :: Handler RepHtml
 getAddRepositoryR = bootstrap (Just Config) $ do
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index e2442c37ec..0ddf65c589 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -14,11 +14,13 @@ import Assistant.DaemonStatus
 import Assistant.TransferQueue
 import Assistant.Threads.WebApp
 import Utility.WebApp
-import Utility.ThreadScheduler
 import Utility.Daemon (checkDaemon)
 import Init
 import qualified Command.Watch
+import qualified Git.CurrentRepo
+import qualified Annex
 
+import Control.Concurrent
 import Control.Concurrent.STM
 
 def :: [Command]
@@ -42,7 +44,8 @@ start foreground stopdaemon = notBareRepo $ do
 			f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 			ifM (checkpid <&&> checkshim f) $
 				( liftIO $ openBrowser f 
-				, startDaemon True foreground $ Just openBrowser
+				, startDaemon True foreground $ Just $
+					const openBrowser
 				)
 		checkpid = do
 			pidfile <- fromRepo gitAnnexPidFile
@@ -53,12 +56,44 @@ openBrowser :: FilePath -> IO ()
 openBrowser htmlshim = unlessM (runBrowser url) $
 	error $ "failed to start web browser on url " ++ url
 	where
-		url = "file://" ++ htmlshim
+		url = fileUrl htmlshim
 
+fileUrl :: FilePath -> String
+fileUrl file = "file://" ++ file
+
+{- Run the webapp without a repository, which prompts the user, makes one,
+ - changes to it, starts the regular assistant, and redirects the
+ - browser to its url.
+ -
+ - This is a very tricky dance -- The first webapp calls the signaler,
+ - which signals the main thread when it's ok to continue by writing to a
+ - MVar. The main thread starts the second webapp, and uses its callback
+ - to write its url back to the MVar, from where the signaler retrieves it,
+ - returning it to the first webapp, which does the redirect.
+ -
+ - Note that it's important that mainthread never terminates! Much
+ - of this complication is due to needing to keep the mainthread running.
+ -}
 firstRun :: IO ()
 firstRun = do
 	dstatus <- atomically . newTMVar =<< newDaemonStatus
 	transferqueue <- newTransferQueue
-	webAppThread Nothing dstatus transferqueue $ Just $ \f -> do
-		openBrowser f
-		waitForTermination
+	v <- newEmptyMVar
+	let callback a = Just $ a v
+	webAppThread Nothing dstatus transferqueue (callback signaler) (callback mainthread)
+	where
+		signaler v = do
+			putMVar v ""
+			putStrLn "signaler waiting..."
+			r <- takeMVar v
+			putStrLn "signaler got value"
+			return r
+		mainthread v _url htmlshim = do
+			openBrowser htmlshim
+
+			_wait <- takeMVar v
+
+			state <- Annex.new =<< Git.CurrentRepo.get
+			Annex.eval state $
+				startAssistant True id $ Just $ sendurlback v
+		sendurlback v url _htmlshim = putMVar v url

From 8181b38ef6060103953ce464d03e9cfd75c45663 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 16:29:38 -0400
Subject: [PATCH 4479/8313] write pid file even when running in foreground

This prevents multiple runs of the assistant in the foreground, and lets
--stop stop foregrounded runs too.

The webapp firstrun case also now writes a pid file, once it's made the git
repo to put it in.
---
 Assistant.hs      |  1 +
 Command/WebApp.hs |  8 ++++++--
 Utility/Daemon.hs | 16 ++++++++--------
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index be84fab55e..21414e721e 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -129,6 +129,7 @@ startDaemon :: Bool -> Bool -> Maybe (Url -> FilePath -> IO ()) -> Annex ()
 startDaemon assistant foreground webappwaiter
 	| foreground = do
 		showStart (if assistant then "assistant" else "watch") "."
+		liftIO . Utility.Daemon.lockPidFile =<< fromRepo gitAnnexPidFile
 		go id
 	| otherwise = do
 		logfd <- liftIO . openLog =<< fromRepo gitAnnexLogFile
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 0ddf65c589..61de2c2f19 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -14,7 +14,7 @@ import Assistant.DaemonStatus
 import Assistant.TransferQueue
 import Assistant.Threads.WebApp
 import Utility.WebApp
-import Utility.Daemon (checkDaemon)
+import Utility.Daemon (checkDaemon, lockPidFile)
 import Init
 import qualified Command.Watch
 import qualified Git.CurrentRepo
@@ -94,6 +94,10 @@ firstRun = do
 			_wait <- takeMVar v
 
 			state <- Annex.new =<< Git.CurrentRepo.get
-			Annex.eval state $
+			Annex.eval state $ do
+				dummydaemonize
 				startAssistant True id $ Just $ sendurlback v
 		sendurlback v url _htmlshim = putMVar v url
+		{- Set up the pid file in the new repo. -}
+		dummydaemonize = do
+			liftIO . lockPidFile =<< fromRepo gitAnnexPidFile
diff --git a/Utility/Daemon.hs b/Utility/Daemon.hs
index 3386ea4434..ba2b2c9c3c 100644
--- a/Utility/Daemon.hs
+++ b/Utility/Daemon.hs
@@ -27,7 +27,7 @@ daemonize logfd pidfile changedirectory a = do
 			_ <- forkProcess child2
 			out
 		child2 = do
-			maybe noop (lockPidFile alreadyrunning) pidfile 
+			maybe noop lockPidFile pidfile 
 			when changedirectory $
 				setCurrentDirectory "/"
 			nullfd <- openFd "/dev/null" ReadOnly Nothing defaultFileFlags
@@ -39,28 +39,28 @@ daemonize logfd pidfile changedirectory a = do
 		redir newh h = do
 			closeFd h
 			dupTo newh h
-		alreadyrunning = error "Daemon is already running."
 		out = exitImmediately ExitSuccess
 
 {- Locks the pid file, with an exclusive, non-blocking lock.
- - Runs an action on failure. On success, writes the pid to the file,
- - fully atomically. -}
-lockPidFile :: IO () -> FilePath -> IO ()
-lockPidFile onfailure file = do
+ - Writes the pid to the file, fully atomically.
+ - Fails if the pid file is already locked by another process. -}
+lockPidFile :: FilePath -> IO ()
+lockPidFile file = do
 	fd <- openFd file ReadWrite (Just stdFileMode) defaultFileFlags
 	locked <- catchMaybeIO $ setLock fd (WriteLock, AbsoluteSeek, 0, 0)
 	fd' <- openFd newfile ReadWrite (Just stdFileMode) defaultFileFlags
 		{ trunc = True }
 	locked' <- catchMaybeIO $ setLock fd' (WriteLock, AbsoluteSeek, 0, 0)
 	case (locked, locked') of
-		(Nothing, _) -> onfailure
-		(_, Nothing) -> onfailure
+		(Nothing, _) -> alreadyrunning
+		(_, Nothing) -> alreadyrunning
 		_ -> do
 			_ <- fdWrite fd' =<< show <$> getProcessID
 			renameFile newfile file
 			closeFd fd
 	where
 		newfile = file ++ ".new"
+		alreadyrunning = error "Daemon is already running."
 
 {- Checks if the daemon is running, by checking that the pid file
  - is locked by the same process that is listed in the pid file.

From 7606f3e7c1cc1b13445d04d11372c197fb6a3f13 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 16:31:16 -0400
Subject: [PATCH 4480/8313] cleanup

---
 Assistant/WebApp/Configurators.hs | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 5c2a1f25ed..e1fcfcd971 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -149,10 +149,7 @@ startFullAssistant path = do
 	url <- liftIO $ do
 		makeRepo path
 		changeWorkingDirectory path
-		putStrLn "pre run"
-		r <- fromJust $ postFirstRun webapp
-		putStrLn $ "got " ++ r
-		return r
+		fromJust $ postFirstRun webapp
 	redirect $ T.pack url
 
 {- Makes a new git-annex repository. -}

From ca512f1450d919397b6eb4fbc54e32e70677c8f7 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 16:34:17 -0400
Subject: [PATCH 4481/8313] always run webapp in foreground

---
 Command/WebApp.hs | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 61de2c2f19..20b914b7af 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -25,16 +25,15 @@ import Control.Concurrent.STM
 
 def :: [Command]
 def = [oneShot $ noRepo firstRun $ dontCheck repoExists $
-	withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $
+	withOptions [Command.Watch.stopOption] $
         command "webapp" paramNothing seek "launch webapp"]
 
 seek :: [CommandSeek]
 seek = [withFlag Command.Watch.stopOption $ \stopdaemon ->
-	withFlag Command.Watch.foregroundOption $ \foreground ->
-	withNothing $ start foreground stopdaemon]
+	withNothing $ start stopdaemon]
 
-start :: Bool -> Bool -> CommandStart
-start foreground stopdaemon = notBareRepo $ do
+start :: Bool -> CommandStart
+start stopdaemon = notBareRepo $ do
 	if stopdaemon
 		then stopDaemon
 		else ifM (isInitialized) ( go , liftIO firstRun )
@@ -44,7 +43,7 @@ start foreground stopdaemon = notBareRepo $ do
 			f <- liftIO . absPath =<< fromRepo gitAnnexHtmlShim
 			ifM (checkpid <&&> checkshim f) $
 				( liftIO $ openBrowser f 
-				, startDaemon True foreground $ Just $
+				, startDaemon True True $ Just $
 					const openBrowser
 				)
 		checkpid = do
@@ -84,10 +83,7 @@ firstRun = do
 	where
 		signaler v = do
 			putMVar v ""
-			putStrLn "signaler waiting..."
-			r <- takeMVar v
-			putStrLn "signaler got value"
-			return r
+			takeMVar v
 		mainthread v _url htmlshim = do
 			openBrowser htmlshim
 

From d7a2600edd1989934c61ae5a80fdc2ee385d49c5 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 16:40:38 -0400
Subject: [PATCH 4482/8313] webapp is no longer a daemon

Remove --foreground and --stop options from it.
---
 Command/WebApp.hs  | 13 ++++---------
 doc/git-annex.mdwn |  4 ++--
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 20b914b7af..f143d8667b 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -16,7 +16,6 @@ import Assistant.Threads.WebApp
 import Utility.WebApp
 import Utility.Daemon (checkDaemon, lockPidFile)
 import Init
-import qualified Command.Watch
 import qualified Git.CurrentRepo
 import qualified Annex
 
@@ -25,18 +24,14 @@ import Control.Concurrent.STM
 
 def :: [Command]
 def = [oneShot $ noRepo firstRun $ dontCheck repoExists $
-	withOptions [Command.Watch.stopOption] $
         command "webapp" paramNothing seek "launch webapp"]
 
 seek :: [CommandSeek]
-seek = [withFlag Command.Watch.stopOption $ \stopdaemon ->
-	withNothing $ start stopdaemon]
+seek = [withNothing start]
 
-start :: Bool -> CommandStart
-start stopdaemon = notBareRepo $ do
-	if stopdaemon
-		then stopDaemon
-		else ifM (isInitialized) ( go , liftIO firstRun )
+start :: CommandStart
+start = notBareRepo $ do
+	ifM (isInitialized) ( go , liftIO firstRun )
 	stop
 	where
 		go = do
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 282b1fda50..0a6df035be 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -192,8 +192,8 @@ subdirectories).
 
 * webapp
 
-  Opens a web browser, viewing the git-annex assistant's web app.
-  (If the assistant is not already running, it will be automatically started.)
+  Runs a web app, that allows easy setup of a git-annex repository,
+  and control of the git-annex assistant.
 
 # REPOSITORY SETUP COMMANDS
 

From bdd5fe4170afe3e6ca0cd2f72c1522d2cf4f8bb1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 17:01:04 -0400
Subject: [PATCH 4483/8313] fix the watch command to not run the whole
 assistant

heh :)
---
 Assistant.hs | 31 ++++++++++++++++++-------------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 21414e721e..6af6c6639e 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -152,20 +152,25 @@ startAssistant assistant daemonize webappwaiter = do
 			transferqueue <- newTransferQueue
 			transferslots <- newTransferSlots
 			scanremotes <- newScanRemoteMap
-			mapM_ forkIO
-				[ commitThread st changechan commitchan transferqueue dstatus
-				, pushThread st dstatus commitchan pushmap
-				, pushRetryThread st dstatus pushmap
-				, mergeThread st
-				, transferWatcherThread st dstatus
-				, transfererThread st dstatus transferqueue transferslots
-				, daemonStatusThread st dstatus
-				, sanityCheckerThread st dstatus transferqueue changechan
-				, mountWatcherThread st dstatus scanremotes
-				, transferScannerThread st dstatus scanremotes transferqueue
+			mapM_ startthread
+				[ watch $ commitThread st changechan commitchan transferqueue dstatus
+				, assist $ pushThread st dstatus commitchan pushmap
+				, assist $ pushRetryThread st dstatus pushmap
+				, assist $ mergeThread st
+				, assist $ transferWatcherThread st dstatus
+				, assist $ transfererThread st dstatus transferqueue transferslots
+				, assist $ daemonStatusThread st dstatus
+				, assist $ sanityCheckerThread st dstatus transferqueue changechan
+				, assist $ mountWatcherThread st dstatus scanremotes
+				, assist $ transferScannerThread st dstatus scanremotes transferqueue
 #ifdef WITH_WEBAPP
-				, webAppThread (Just st) dstatus transferqueue Nothing webappwaiter
+				, assist $ webAppThread (Just st) dstatus transferqueue Nothing webappwaiter
 #endif
-				, watchThread st dstatus transferqueue changechan
+				, watch $ watchThread st dstatus transferqueue changechan
 				]
 			waitForTermination
+		watch a = (True, a)
+		assist a = (False, a)
+		startthread (watcher, a)
+			| watcher || assistant = void $ forkIO a
+			| otherwise = noop

From 452f7d91f12b623e84e49037858ca2c034d4f14f Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 17:03:17 -0400
Subject: [PATCH 4484/8313] close poll

---
 doc/design/assistant/blog/day_49__first_run_experience.mdwn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/design/assistant/blog/day_49__first_run_experience.mdwn b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
index 6ee3634f31..08ba462d45 100644
--- a/doc/design/assistant/blog/day_49__first_run_experience.mdwn
+++ b/doc/design/assistant/blog/day_49__first_run_experience.mdwn
@@ -31,7 +31,7 @@ Time for the first of probably many polls!
 
 What should the default directory name used by the git-annex assistant be?
 
-[[!poll 19 "Annex" 7 "GitAnnex" 10 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
+[[!poll open=no 19 "Annex" 7 "GitAnnex" 10 "Synced" 0 "AutoSynced" 1 "Shared" 10 "something lowercase!" 1 "CowboyNeal" 1 "Annexbox"]]
 
 (Note: This is a wiki. You can edit this page to add your own
 [[ikiwiki/directive/poll]] options!)

From ed5736ff3085b2ec482548b4fee29c4e03193ca1 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 17:11:28 -0400
Subject: [PATCH 4485/8313] blog for the day

---
 .../blog/day_50__directory_name.mdwn          | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_50__directory_name.mdwn

diff --git a/doc/design/assistant/blog/day_50__directory_name.mdwn b/doc/design/assistant/blog/day_50__directory_name.mdwn
new file mode 100644
index 0000000000..a0e57cae68
--- /dev/null
+++ b/doc/design/assistant/blog/day_50__directory_name.mdwn
@@ -0,0 +1,20 @@
+Based on the results of yesterday's poll, the WebApp defaults to
+`~/Desktop/annex` when run in the home directory. If there's no `Desktop`
+directory, it uses just `~/annex`. And if run from some other place than
+the home directory, it assumes you want to use cwd. Of course, you can
+change this default, but I think it's a good one for most use cases.
+
+----
+
+My work today has all been on making **one second** of the total lifetime
+of the WebApp work. It's the very tricky second in between clicking on
+"Make repository" and being redirected to a WebApp running in your new
+repository. The trickiness involves threads, and MVars, and
+multiple web servers, and I don't want to go into details here.
+I'd rather forget. ;-)
+
+Anyway, it works; you can run "git annex webapp" and be walked right
+through to having a usable repository! Now I need to see about adding
+that to the desktop menus, and making "git annex webapp", when run a second
+time, remembering where your repository is. I'll use
+`~/.config/git-annex/repository` for storing that.

From ef13a3cd25aa1a7c0b18e19da81f67b8568dbfd5 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
 <Richard@web>
Date: Wed, 1 Aug 2012 21:19:03 +0000
Subject: [PATCH 4486/8313] Added a comment

---
 .../comment_1_782cec95a8558a05b2b38a2d2302214d._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment

diff --git a/doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment b/doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment
new file mode 100644
index 0000000000..be55932f13
--- /dev/null
+++ b/doc/design/assistant/blog/day_50__directory_name/comment_1_782cec95a8558a05b2b38a2d2302214d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawl9sYlePmv1xK-VvjBdN-5doOa_Xw-jH4U"
+ nickname="Richard"
+ subject="comment 1"
+ date="2012-08-01T21:19:03Z"
+ content="""
+Please use `$XDG_CONFIG_HOME` and fall back to `~/.config` if that's not defined.
+"""]]

From 31f123de464005c6e605b2eb147cddd88f42d1db Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 17:40:14 -0400
Subject: [PATCH 4487/8313] close

---
 doc/bugs/Build_failure_at_commit_1efe4f3.mdwn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn b/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn
index a82d6d0456..ba87b11915 100644
--- a/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn
+++ b/doc/bugs/Build_failure_at_commit_1efe4f3.mdwn
@@ -42,3 +42,4 @@ Assistant/WebApp/Configurators.hs:114:17:
 make: *** [git-annex] Error 1
 </pre>
 
+> [[fixed|done]] --[[Joey]] 

From a9d3d56b8097010fa53e60e3b608e007601fa28c Mon Sep 17 00:00:00 2001
From: "https://creativecommons.net/greg/identity"
 <https://creativecommons.net/greg/identity@web>
Date: Wed, 1 Aug 2012 22:24:11 +0000
Subject: [PATCH 4488/8313] but reported

---
 .../journal_commit_error_when_using_annex.mdwn  | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 doc/bugs/journal_commit_error_when_using_annex.mdwn

diff --git a/doc/bugs/journal_commit_error_when_using_annex.mdwn b/doc/bugs/journal_commit_error_when_using_annex.mdwn
new file mode 100644
index 0000000000..415d9fec3e
--- /dev/null
+++ b/doc/bugs/journal_commit_error_when_using_annex.mdwn
@@ -0,0 +1,17 @@
+Setup:
+
+* git-annex version: 3.20120629 on Debian unstable.
+* external RAID1 array connected via usb
+
+I did: <code> git-annex add \<a specific album of ogg files\> </code>
+
+When it gets to the fourth song, there is a long delay and a bunch of "journal commit I/O error" from the kernel start appearing. Then GNOME helpfully tells me that my drive was just connected (implying that it thought it was disconnected momentarily)
+
+Two log outputs:
+
+http://paste.mitechie.com/show/745/
+
+and
+
+http://paste.mitechie.com/show/746/
+

From 33f0368c32b25f921eeff7f636c0266c5154ef7c Mon Sep 17 00:00:00 2001
From: "https://creativecommons.net/greg/identity"
 <https://creativecommons.net/greg/identity@web>
Date: Wed, 1 Aug 2012 22:31:04 +0000
Subject: [PATCH 4489/8313] linkify the logs

---
 doc/bugs/journal_commit_error_when_using_annex.mdwn | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/doc/bugs/journal_commit_error_when_using_annex.mdwn b/doc/bugs/journal_commit_error_when_using_annex.mdwn
index 415d9fec3e..9558459dd7 100644
--- a/doc/bugs/journal_commit_error_when_using_annex.mdwn
+++ b/doc/bugs/journal_commit_error_when_using_annex.mdwn
@@ -9,9 +9,5 @@ When it gets to the fourth song, there is a long delay and a bunch of "journal c
 
 Two log outputs:
 
-http://paste.mitechie.com/show/745/
-
-and
-
-http://paste.mitechie.com/show/746/
-
+* <http://paste.mitechie.com/show/745/>
+* <http://paste.mitechie.com/show/746/>

From 6275146b846bdf383dbb418cf35f59d7e818baff Mon Sep 17 00:00:00 2001
From: "https://creativecommons.net/greg/identity"
 <https://creativecommons.net/greg/identity@web>
Date: Wed, 1 Aug 2012 22:41:11 +0000
Subject: [PATCH 4490/8313] more info

---
 doc/bugs/journal_commit_error_when_using_annex.mdwn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/doc/bugs/journal_commit_error_when_using_annex.mdwn b/doc/bugs/journal_commit_error_when_using_annex.mdwn
index 9558459dd7..99228a5495 100644
--- a/doc/bugs/journal_commit_error_when_using_annex.mdwn
+++ b/doc/bugs/journal_commit_error_when_using_annex.mdwn
@@ -7,7 +7,13 @@ I did: <code> git-annex add \<a specific album of ogg files\> </code>
 
 When it gets to the fourth song, there is a long delay and a bunch of "journal commit I/O error" from the kernel start appearing. Then GNOME helpfully tells me that my drive was just connected (implying that it thought it was disconnected momentarily)
 
-Two log outputs:
+Log outputs:
+
+Both trying to add whole album, failing at song 4
 
 * <http://paste.mitechie.com/show/745/>
 * <http://paste.mitechie.com/show/746/>
+
+Just trying to add song 4 individually
+
+* <http://paste.mitechie.com/show/747/> (lots o' journal commit I/O errors that time)

From 45a32f55424761fd5c21e78deaa18241697f72b1 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
 <Justin@web>
Date: Wed, 1 Aug 2012 23:06:03 +0000
Subject: [PATCH 4491/8313] Added a comment

---
 ...comment_1_38f60ca3503ea1530c4bd2cde5c9182f._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/journal_commit_error_when_using_annex/comment_1_38f60ca3503ea1530c4bd2cde5c9182f._comment

diff --git a/doc/bugs/journal_commit_error_when_using_annex/comment_1_38f60ca3503ea1530c4bd2cde5c9182f._comment b/doc/bugs/journal_commit_error_when_using_annex/comment_1_38f60ca3503ea1530c4bd2cde5c9182f._comment
new file mode 100644
index 0000000000..de39249a8b
--- /dev/null
+++ b/doc/bugs/journal_commit_error_when_using_annex/comment_1_38f60ca3503ea1530c4bd2cde5c9182f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://www.google.com/accounts/o8/id?id=AItOawmBUR4O9mofxVbpb8JV9mEbVfIYv670uJo"
+ nickname="Justin"
+ subject="comment 1"
+ date="2012-08-01T23:06:03Z"
+ content="""
+That's not a git-annex error, your drive is dying.
+
+I'm sure if you tried just playing the song you'll get the same errors.
+"""]]

From c77338948219abdb5be2e2dc54c46b5838a79bbb Mon Sep 17 00:00:00 2001
From: "https://creativecommons.net/greg/identity"
 <https://creativecommons.net/greg/identity@web>
Date: Wed, 1 Aug 2012 23:22:03 +0000
Subject: [PATCH 4492/8313] Added a comment

---
 ...comment_2_6de455a67f37d9ee0a307a78123781bf._comment | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/bugs/journal_commit_error_when_using_annex/comment_2_6de455a67f37d9ee0a307a78123781bf._comment

diff --git a/doc/bugs/journal_commit_error_when_using_annex/comment_2_6de455a67f37d9ee0a307a78123781bf._comment b/doc/bugs/journal_commit_error_when_using_annex/comment_2_6de455a67f37d9ee0a307a78123781bf._comment
new file mode 100644
index 0000000000..dd1e1c92b7
--- /dev/null
+++ b/doc/bugs/journal_commit_error_when_using_annex/comment_2_6de455a67f37d9ee0a307a78123781bf._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="https://creativecommons.net/greg/identity"
+ ip="64.79.125.70"
+ subject="comment 2"
+ date="2012-08-01T23:22:03Z"
+ content="""
+Probably right, the song in question is not playing fully and causing same error.
+
+Time to go into crises mode...
+"""]]

From d9ad123668f97ecafcc05701d07361601867ddee Mon Sep 17 00:00:00 2001
From: "https://creativecommons.net/greg/identity"
 <https://creativecommons.net/greg/identity@web>
Date: Wed, 1 Aug 2012 23:35:42 +0000
Subject: [PATCH 4493/8313]

---
 doc/bugs/journal_commit_error_when_using_annex.mdwn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/bugs/journal_commit_error_when_using_annex.mdwn b/doc/bugs/journal_commit_error_when_using_annex.mdwn
index 99228a5495..fe19282292 100644
--- a/doc/bugs/journal_commit_error_when_using_annex.mdwn
+++ b/doc/bugs/journal_commit_error_when_using_annex.mdwn
@@ -1,3 +1,5 @@
+Sorry for the noise - notabug - [[done]]
+
 Setup:
 
 * git-annex version: 3.20120629 on Debian unstable.

From 89ec253a6a02addca9293815966454a9646dcf0d Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 20:27:45 -0400
Subject: [PATCH 4494/8313] implement enough of the fdo specs to be able to
 write desktop menu files to the appropriate system or local user directory

---
 Utility/FreeDesktop.hs | 94 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 Utility/FreeDesktop.hs

diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs
new file mode 100644
index 0000000000..ce3501766c
--- /dev/null
+++ b/Utility/FreeDesktop.hs
@@ -0,0 +1,94 @@
+{- Freedesktop.org specifications
+ -
+ - http://standards.freedesktop.org/basedir-spec/latest/
+ - http://standards.freedesktop.org/desktop-entry-spec/latest/
+ - http://standards.freedesktop.org/menu-spec/latest/
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Utility.DesktopMenu (
+	DesktopEntry,	
+	genDesktopEntry,
+	buildDesktopMenuFile,
+	writeDesktopMenuFile,
+	userDesktopMenuFilePath,
+	systemDesktopMenuFilePath		
+) where
+
+import Utility.Exception
+import Utility.Directory
+import Utility.Path
+
+import System.IO
+import System.Environment
+import System.Directory
+import System.FilePath
+import Data.List
+import Data.String.Utils
+
+type DesktopEntry = [(Key, Value)]
+
+type Key = String
+
+data Value = StringV String | BoolV Bool | NumericV Float | ListV [Value]
+
+toString :: Value -> String
+toString (StringV s) = s
+toString (BoolV b)
+	| b = "true"
+	| otherwise = "false"
+toString(NumericV f) = show f
+toString (ListV l)
+	| null l = ""
+	| otherwise = (intercalate ";" $ map (escapesemi . toString) l) ++ ";"
+	where
+		escapesemi = join "\\;" . split ";"
+
+genDesktopEntry :: String -> String -> Bool -> FilePath -> FilePath -> [String] -> DesktopEntry
+genDesktopEntry name comment terminal program icon categories =
+	[ item "Encoding" StringV "UTF-8"
+	, item "Type" StringV "Application"
+	, item "Version" NumericV 1.0
+	, item "Name" StringV name
+	, item "Comment" StringV comment
+	, item "Terminal" BoolV terminal
+	, item "Exec" StringV program
+	, item "Icon" StringV icon
+	, item "Categories" ListV (map StringV categories)
+	]
+	where
+		item x c y = (x, c y)
+
+buildDesktopMenuFile :: DesktopEntry -> String
+buildDesktopMenuFile d = unlines ("[Desktop Entry]" : map keyvalue d) ++ "\n"
+	where
+		keyvalue (k, v) = k ++ "=" ++ toString v
+
+writeDesktopMenuFile :: DesktopEntry -> String -> IO ()
+writeDesktopMenuFile d file = do
+	createDirectoryIfMissing True (parentDir file)
+	writeFile file $ buildDesktopMenuFile d
+
+userDesktopMenuFilePath :: String -> IO FilePath
+userDesktopMenuFilePath basename = do
+	datadir <- userDataDir
+	return $ datadir </> "applications" </> basename
+
+systemDesktopMenuFilePath :: String -> FilePath
+systemDesktopMenuFilePath basename = "/usr/share/applications" </> basename
+
+userDataDir :: IO FilePath
+userDataDir = do
+	dir <- xdgEnv "DATA_HOME" =<< myHomeDir
+	return $ dir </> ".local" </> "share"
+
+userConfigDir :: IO FilePath
+userConfigDir = do
+	dir <- xdgEnv "DATA_HOME" =<< myHomeDir
+	return $ dir </> ".config"
+
+xdgEnv :: String -> String -> IO String
+xdgEnv envbase def = catchDefaultIO (getEnv $ "XDG_" ++ envbase) def

From 9422e274897eb7581f453cb4374c326f0bf83d4f Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 20:49:02 -0400
Subject: [PATCH 4495/8313] installing desktop file working

Not hooked up to either Makefile or cabal yet
---
 Build/Desktop.hs       | 34 ++++++++++++++++++++++++++++++++++
 Utility/FreeDesktop.hs | 18 ++++++++++--------
 2 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 Build/Desktop.hs

diff --git a/Build/Desktop.hs b/Build/Desktop.hs
new file mode 100644
index 0000000000..b35b0c0104
--- /dev/null
+++ b/Build/Desktop.hs
@@ -0,0 +1,34 @@
+{- Generating and installing a desktop menu entry file.
+ -
+ - Copyright 2012 Joey Hess <joey@kitenet.net>
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Build.Desktop where
+
+import Utility.Exception
+import Utility.FreeDesktop
+
+import Control.Applicative
+import System.Environment
+import System.Posix.User
+
+{- The command can be either just "git-annex", or the full path to use
+ - to run it. -}
+desktop :: FilePath -> DesktopEntry
+desktop command = genDesktopEntry
+	"Git Annex"
+	"Track and sync the files in your Git Annex"
+	False
+	(command ++ " webapp")
+	["Network", "FileTransfer"]
+
+writeDesktop :: DesktopEntry -> IO ()
+writeDesktop d = do
+	destdir <- catchDefaultIO (getEnv "DESTDIR") ""
+	uid <- fromIntegral <$> getRealUserID
+	dest <- if uid /= 0
+		then userDesktopMenuFilePath "git-annex"
+		else return $ systemDesktopMenuFilePath "git-annex"
+	writeDesktopMenuFile d dest
diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs
index ce3501766c..5e38d382d5 100644
--- a/Utility/FreeDesktop.hs
+++ b/Utility/FreeDesktop.hs
@@ -9,7 +9,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Utility.DesktopMenu (
+module Utility.FreeDesktop (
 	DesktopEntry,	
 	genDesktopEntry,
 	buildDesktopMenuFile,
@@ -47,16 +47,14 @@ toString (ListV l)
 	where
 		escapesemi = join "\\;" . split ";"
 
-genDesktopEntry :: String -> String -> Bool -> FilePath -> FilePath -> [String] -> DesktopEntry
-genDesktopEntry name comment terminal program icon categories =
-	[ item "Encoding" StringV "UTF-8"
-	, item "Type" StringV "Application"
+genDesktopEntry :: String -> String -> Bool -> FilePath -> [String] -> DesktopEntry
+genDesktopEntry name comment terminal program categories =
+	[ item "Type" StringV "Application"
 	, item "Version" NumericV 1.0
 	, item "Name" StringV name
 	, item "Comment" StringV comment
 	, item "Terminal" BoolV terminal
 	, item "Exec" StringV program
-	, item "Icon" StringV icon
 	, item "Categories" ListV (map StringV categories)
 	]
 	where
@@ -75,10 +73,14 @@ writeDesktopMenuFile d file = do
 userDesktopMenuFilePath :: String -> IO FilePath
 userDesktopMenuFilePath basename = do
 	datadir <- userDataDir
-	return $ datadir </> "applications" </> basename
+	return $ datadir </> "applications" </> desktopfile basename
 
 systemDesktopMenuFilePath :: String -> FilePath
-systemDesktopMenuFilePath basename = "/usr/share/applications" </> basename
+systemDesktopMenuFilePath basename =
+	"/usr/share/applications" </> desktopfile basename
+
+desktopfile :: FilePath -> FilePath
+desktopfile f = f ++ ".desktop"
 
 userDataDir :: IO FilePath
 userDataDir = do

From ed07546288733a13129a866ce70ea2f94d6259cb Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 21:04:15 -0400
Subject: [PATCH 4496/8313] add template-haskell build-dep for webapp

---
 git-annex.cabal | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/git-annex.cabal b/git-annex.cabal
index afa8814253..de417a4a1e 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -76,7 +76,8 @@ Executable git-annex
   if flag(Webapp)
     Build-Depends: yesod, yesod-static, yesod-default, case-insensitive,
      http-types, transformers, wai, wai-logger, warp, blaze-builder,
-     blaze-html, blaze-markup, crypto-api, hamlet, clientsession
+     blaze-html, blaze-markup, crypto-api, hamlet, clientsession,
+     template-haskell
     CPP-Options: -DWITH_WEBAPP
 
   if os(darwin)

From e78b13c42807f598d9dd7e449a5980c26f731f72 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 21:04:25 -0400
Subject: [PATCH 4497/8313] hook desktop menu file installation into makefile
 and cabal

---
 Build/{Desktop.hs => InstallDesktopFile.hs} | 7 ++++++-
 Makefile                                    | 1 +
 Setup.hs                                    | 8 ++++++++
 3 files changed, 15 insertions(+), 1 deletion(-)
 rename Build/{Desktop.hs => InstallDesktopFile.hs} (84%)

diff --git a/Build/Desktop.hs b/Build/InstallDesktopFile.hs
similarity index 84%
rename from Build/Desktop.hs
rename to Build/InstallDesktopFile.hs
index b35b0c0104..b4a56a2cbd 100644
--- a/Build/Desktop.hs
+++ b/Build/InstallDesktopFile.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-module Build.Desktop where
+module Build.InstallDesktopFile where
 
 import Utility.Exception
 import Utility.FreeDesktop
@@ -32,3 +32,8 @@ writeDesktop d = do
 		then userDesktopMenuFilePath "git-annex"
 		else return $ systemDesktopMenuFilePath "git-annex"
 	writeDesktopMenuFile d dest
+
+main = getArgs >>= go
+	where
+		go [] = error "specify git-annex command"
+		go (command:_) = writeDesktop $ desktop command
diff --git a/Makefile b/Makefile
index 8eef53f30e..bd8aa4f3f6 100644
--- a/Makefile
+++ b/Makefile
@@ -86,6 +86,7 @@ install: all install-docs
 	install -d $(DESTDIR)$(PREFIX)/bin
 	install $(bins) $(DESTDIR)$(PREFIX)/bin
 	ln -sf git-annex $(DESTDIR)$(PREFIX)/bin/git-annex-shell
+	runghc Build/InstallDesktopFile.hs $(PREFIX)/bin/git-annex || true
 
 test: $(sources) $(clibs)
 	@if ! $(GHCMAKE) -O0 test $(clibs); then \
diff --git a/Setup.hs b/Setup.hs
index 80d23cc878..4451e86455 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -10,6 +10,7 @@ import Distribution.PackageDescription (PackageDescription(..))
 import Distribution.Verbosity (Verbosity)
 import System.FilePath
 
+import qualified Build.InstallDesktopFile as InstallDesktopFile
 import qualified Build.Configure as Configure
 
 main = defaultMainWithHooks simpleUserHooks
@@ -25,6 +26,7 @@ myPostInst :: Args -> InstallFlags -> PackageDescription -> LocalBuildInfo -> IO
 myPostInst _ (InstallFlags { installVerbosity }) pkg lbi = do
 	installGitAnnexShell dest verbosity pkg lbi
 	installManpages      dest verbosity pkg lbi
+	installDesktopFile   dest verbosity pkg lbi
 	where
 		dest      = NoCopyDest
 		verbosity = fromFlag installVerbosity
@@ -47,3 +49,9 @@ installManpages copyDest verbosity pkg lbi =
 		srcManpages = zip (repeat srcManDir) manpages
 		srcManDir   = ""
 		manpages    = ["git-annex.1", "git-annex-shell.1"]
+
+installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO ()
+installDesktopFile copyDest verbosity pkg lbi =
+	InstallDesktopFile.writeDesktop $ InstallDesktopFile.desktop $ dstBinDir </> "git-annex"
+	where
+		dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest

From ffeb060002cd33b7db0fc03b1abacc7b2762b88e Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kitenet.net>
Date: Wed, 1 Aug 2012 21:26:36 -0400
Subject: [PATCH 4498/8313] don't use hamlet for htmlshim

This allows me to not build-depend on blaze-markup, which was causing
me some trouble when tring to build with cabal on debian. Seems debian
ships Text.Blaze.Renderer.String in two packages.
---
 Assistant/Threads/WebApp.hs | 26 +++++++++++++++++---------
 git-annex.cabal             |  2 +-
 templates/htmlshim.hamlet   |  7 -------
 3 files changed, 18 insertions(+), 17 deletions(-)
 delete mode 100644 templates/htmlshim.hamlet

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index a5484b5bec..e8de408a13 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -21,16 +21,13 @@ import Assistant.ThreadedMonad
 import Assistant.DaemonStatus
 import Assistant.TransferQueue
 import Utility.WebApp
-import Utility.Yesod
 import Utility.FileMode
 import Utility.TempFile
 import Git
 
 import Yesod
 import Yesod.Static
-import Text.Hamlet
 import Network.Socket (PortNumber)
-import Text.Blaze.Renderer.String
 import Data.Text (pack, unpack)
 
 thisThread :: String
@@ -75,7 +72,7 @@ webAppThread mst dstatus transferqueue postfirstrun onstartup = do
 				else dir
 		go port webapp htmlshim = do
 			writeHtmlShim webapp port htmlshim
-			maybe noop (\a -> a (myUrl webapp port) htmlshim) onstartup
+			maybe noop (\a -> a (myUrl webapp port "/") htmlshim) onstartup
 
 {- Creates a html shim file that's used to redirect into the webapp,
  - to avoid exposing the secretToken when launching the web browser. -}
@@ -92,10 +89,21 @@ writeHtmlShim webapp port file = do
 
 {- TODO: generate this static file using Yesod. -}
 genHtmlShim :: WebApp -> PortNumber -> String
-genHtmlShim webapp port = renderHtml $(shamletFile $ hamletTemplate "htmlshim")
+genHtmlShim webapp port = unlines
+	[ "<html>"
+	, "<head>"
+	, "<title>Starting webapp..."
+	, ""
+	, ""
+	, "
"
+	, "Starting webapp..."
+	, "
"
+	, ""
+	, ""
+	]
 	where
-		url = myUrl webapp port
+		url = myUrl webapp port "/"
 
-myUrl :: WebApp -> PortNumber -> Url
-myUrl webapp port = "http://localhost:" ++ show port ++
-	"/?auth=" ++ unpack (secretToken webapp)
+myUrl :: WebApp -> PortNumber -> FilePath -> Url
+myUrl webapp port page = "http://localhost:" ++ show port ++ page ++
+	"?auth=" ++ unpack (secretToken webapp)
diff --git a/git-annex.cabal b/git-annex.cabal
index de417a4a1e..ec96bdc328 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -76,7 +76,7 @@ Executable git-annex
   if flag(Webapp)
     Build-Depends: yesod, yesod-static, yesod-default, case-insensitive,
      http-types, transformers, wai, wai-logger, warp, blaze-builder,
-     blaze-html, blaze-markup, crypto-api, hamlet, clientsession,
+     blaze-html, crypto-api, hamlet, clientsession,
      template-haskell
     CPP-Options: -DWITH_WEBAPP
 
diff --git a/templates/htmlshim.hamlet b/templates/htmlshim.hamlet
deleted file mode 100644
index 073b69c1bd..0000000000
--- a/templates/htmlshim.hamlet
+++ /dev/null
@@ -1,7 +0,0 @@
-$doctype 5
-
-  
-    
-  
-    

-      Starting webapp...

From 50679952d2956e5fecc585ac751e9199aad1492c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Aug 2012 21:38:12 -0400
Subject: [PATCH 4499/8313] todo

---
 doc/design/assistant/webapp.mdwn | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index f8867f0e28..7df93cf401 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -32,7 +32,11 @@ The webapp is a web server that displays a shiny interface.
 
 ## first start
 
-* make git repo
+* make git repo **done**
+* generate a nice description like "joey@hostname Desktop/annex"
+* record repository that was made, and use it next time run
+* write a pid file, to prevent more than one first-start process running
+  at once
 
 ## implementation
 

From 23fe661d37ceb6c7bf754e9dc8fd5dda89793b63 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Wed, 1 Aug 2012 23:31:53 -0400
Subject: [PATCH 4500/8313] install autostart file too

---
 Build/InstallDesktopFile.hs | 29 +++++++++++++++++++++--------
 Setup.hs                    |  2 +-
 Utility/FreeDesktop.hs      | 29 +++++++++++++++++++----------
 3 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs
index b4a56a2cbd..a08743f3da 100644
--- a/Build/InstallDesktopFile.hs
+++ b/Build/InstallDesktopFile.hs
@@ -1,4 +1,5 @@
-{- Generating and installing a desktop menu entry file.
+{- Generating and installing a desktop menu entry file
+ - and a desktop autostart file.
  -
  - Copyright 2012 Joey Hess 
  -
@@ -24,16 +25,28 @@ desktop command = genDesktopEntry
 	(command ++ " webapp")
 	["Network", "FileTransfer"]
 
-writeDesktop :: DesktopEntry -> IO ()
-writeDesktop d = do
+autostart :: FilePath -> DesktopEntry
+autostart command = genDesktopEntry
+	"Git Annex Assistant"
+	"Autostart"
+	False
+	(command ++ " assistant --autostart")
+	[]
+
+writeDesktop :: String -> IO ()
+writeDesktop command = do
 	destdir <- catchDefaultIO (getEnv "DESTDIR") ""
 	uid <- fromIntegral <$> getRealUserID
-	dest <- if uid /= 0
-		then userDesktopMenuFilePath "git-annex"
-		else return $ systemDesktopMenuFilePath "git-annex"
-	writeDesktopMenuFile d dest
+
+	datadir <- if uid /= 0 then userDataDir else return systemDataDir
+	writeDesktopMenuFile (desktop command) $
+		desktopMenuFilePath "git-annex" datadir
+
+	configdir <- if uid /= 0 then userConfigDir else return systemConfigDir
+	writeDesktopMenuFile (autostart command) $
+		autoStartPath "git-annex" configdir
 
 main = getArgs >>= go
 	where
 		go [] = error "specify git-annex command"
-		go (command:_) = writeDesktop $ desktop command
+		go (command:_) = writeDesktop command
diff --git a/Setup.hs b/Setup.hs
index 4451e86455..06390975bb 100644
--- a/Setup.hs
+++ b/Setup.hs
@@ -52,6 +52,6 @@ installManpages copyDest verbosity pkg lbi =
 
 installDesktopFile :: CopyDest -> Verbosity -> PackageDescription -> LocalBuildInfo -> IO ()
 installDesktopFile copyDest verbosity pkg lbi =
-	InstallDesktopFile.writeDesktop $ InstallDesktopFile.desktop $ dstBinDir  "git-annex"
+	InstallDesktopFile.writeDesktop $ dstBinDir  "git-annex"
 	where
 		dstBinDir = bindir $ absoluteInstallDirs pkg lbi copyDest
diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs
index 5e38d382d5..5bab4950ab 100644
--- a/Utility/FreeDesktop.hs
+++ b/Utility/FreeDesktop.hs
@@ -14,8 +14,12 @@ module Utility.FreeDesktop (
 	genDesktopEntry,
 	buildDesktopMenuFile,
 	writeDesktopMenuFile,
-	userDesktopMenuFilePath,
-	systemDesktopMenuFilePath		
+	desktopMenuFilePath,
+	autoStartPath,
+	systemDataDir,
+	systemConfigDir,
+	userDataDir,
+	userConfigDir	
 ) where
 
 import Utility.Exception
@@ -70,18 +74,23 @@ writeDesktopMenuFile d file = do
 	createDirectoryIfMissing True (parentDir file)
 	writeFile file $ buildDesktopMenuFile d
 
-userDesktopMenuFilePath :: String -> IO FilePath
-userDesktopMenuFilePath basename = do
-	datadir <- userDataDir
-	return $ datadir  "applications"  desktopfile basename
+desktopMenuFilePath :: String -> FilePath -> FilePath
+desktopMenuFilePath basename datadir = 
+	datadir  "applications"  desktopfile basename
 
-systemDesktopMenuFilePath :: String -> FilePath
-systemDesktopMenuFilePath basename =
-	"/usr/share/applications"  desktopfile basename
+autoStartPath :: String -> FilePath -> FilePath
+autoStartPath basename configdir =
+	configdir  "autostart"  desktopfile basename
 
 desktopfile :: FilePath -> FilePath
 desktopfile f = f ++ ".desktop"
 
+systemDataDir :: FilePath
+systemDataDir = "/usr/share"
+
+systemConfigDir :: FilePath
+systemConfigDir = "/etc/xdg"
+
 userDataDir :: IO FilePath
 userDataDir = do
 	dir <- xdgEnv "DATA_HOME" =<< myHomeDir
@@ -89,7 +98,7 @@ userDataDir = do
 
 userConfigDir :: IO FilePath
 userConfigDir = do
-	dir <- xdgEnv "DATA_HOME" =<< myHomeDir
+	dir <- xdgEnv "CONFIG_HOME" =<< myHomeDir
 	return $ dir  ".config"
 
 xdgEnv :: String -> String -> IO String

From 60da0d6ad28bff7c601ba631a8ec65030f940367 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 00:42:33 -0400
Subject: [PATCH 4501/8313] full autostart support

git annex assistant --autostart will start separate daemons in each
listed autostart repo

running the webapp outside any git-annex repo will open it on the
first listed autostart repo
---
 Assistant/WebApp/Configurators.hs |  6 +++-
 Build/InstallDesktopFile.hs       |  4 +++
 Command/Assistant.hs              | 58 +++++++++++++++++++++++++++++--
 Command/Watch.hs                  |  9 ++---
 Command/WebApp.hs                 | 39 ++++++++++++++++-----
 Locations/UserConfig.hs           | 26 ++++++++++++++
 Utility/FreeDesktop.hs            |  2 --
 doc/git-annex.mdwn                | 13 +++++--
 8 files changed, 134 insertions(+), 23 deletions(-)
 create mode 100644 Locations/UserConfig.hs

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index e1fcfcd971..01245b4bc1 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -21,6 +21,7 @@ import Init
 import qualified Git.Construct
 import qualified Git.Config
 import qualified Annex
+import Locations.UserConfig
 
 import Yesod
 import Data.Text (Text)
@@ -159,7 +160,10 @@ makeRepo path = do
 		error "git init failed!"
 	g <- Git.Config.read =<< Git.Construct.fromPath path
 	state <- Annex.new g
-	Annex.eval state $ initialize $ Just "new repo"
+	Annex.eval state $ initialize $ Just "new repo" -- TODO better description
+	autostart <- autoStartFile
+	createDirectoryIfMissing True (parentDir autostart)
+	appendFile autostart $ path ++ "\n"
 
 getAddRepositoryR :: Handler RepHtml
 getAddRepositoryR = bootstrap (Just Config) $ do
diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs
index a08743f3da..3bc796315a 100644
--- a/Build/InstallDesktopFile.hs
+++ b/Build/InstallDesktopFile.hs
@@ -46,6 +46,10 @@ writeDesktop command = do
 	writeDesktopMenuFile (autostart command) $
 		autoStartPath "git-annex" configdir
 
+	programfile <- programFile
+	createDirectoryIfMissing True (parentDir programFile)
+	writeFile programfile command
+
 main = getArgs >>= go
 	where
 		go [] = error "specify git-annex command"
diff --git a/Command/Assistant.hs b/Command/Assistant.hs
index 60eac5d219..24cc3ec6c9 100644
--- a/Command/Assistant.hs
+++ b/Command/Assistant.hs
@@ -7,12 +7,66 @@
 
 module Command.Assistant where
 
+import Common.Annex
 import Command
+import qualified Option
 import qualified Command.Watch
+import Init
+import Locations.UserConfig
+
+import System.Environment
+import System.Posix.Directory
 
 def :: [Command]
-def = [withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption] $ 
+def = [noRepo checkAutoStart $ dontCheck repoExists $
+	withOptions [Command.Watch.foregroundOption, Command.Watch.stopOption, autoStartOption] $ 
 	command "assistant" paramNothing seek "automatically handle changes"]
 
+autoStartOption :: Option
+autoStartOption = Option.flag [] "autostart" "start in known repositories"
+
 seek :: [CommandSeek]
-seek = Command.Watch.mkSeek True
+seek = [withFlag Command.Watch.stopOption $ \stopdaemon ->
+	withFlag Command.Watch.foregroundOption $ \foreground ->
+	withFlag autoStartOption $ \autostart ->
+	withNothing $ start foreground stopdaemon autostart]
+
+start :: Bool -> Bool -> Bool -> CommandStart
+start foreground stopdaemon autostart
+	| autostart = do
+		liftIO $ autoStart
+		stop
+	| otherwise = do
+		ensureInitialized
+		Command.Watch.start True foreground stopdaemon
+
+{- Run outside a git repository. Check to see if any parameter is
+ - --autostart and enter autostart mode. -}
+checkAutoStart :: IO ()
+checkAutoStart = ifM (any (== "--autostart") <$> getArgs)
+	( autoStart
+	, error "Not in a git repository."
+	) 
+
+autoStart :: IO ()
+autoStart = do
+	autostartfile <- autoStartFile
+	let nothing = error $ "Nothing listed in " ++ autostartfile
+	ifM (doesFileExist autostartfile)
+		( do
+			dirs <- lines <$> readFile autostartfile
+			programfile <- programFile
+			program <- catchDefaultIO (readFile programfile) "git-annex"
+			when (null dirs) nothing
+			forM_ dirs $ \d -> do
+				putStrLn $ "git-annex autostart in " ++ d
+				ifM (catchBoolIO $ go program d)
+					( putStrLn "ok"
+					, putStrLn "failed"
+					)
+		, nothing
+		)
+	where
+		go program dir = do
+			changeWorkingDirectory dir
+			boolSystem program [Param "assistant"]
diff --git a/Command/Watch.hs b/Command/Watch.hs
index 61c859106f..eb70ef6b10 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -16,13 +16,10 @@ def :: [Command]
 def = [withOptions [foregroundOption, stopOption] $ 
 	command "watch" paramNothing seek "watch for changes"]
 
-mkSeek :: Bool -> [CommandSeek]
-mkSeek assistant = [withFlag stopOption $ \stopdaemon -> 
-	withFlag foregroundOption $ \foreground ->
-	withNothing $ start assistant foreground stopdaemon]
-
 seek :: [CommandSeek]
-seek = mkSeek False
+seek = [withFlag stopOption $ \stopdaemon -> 
+	withFlag foregroundOption $ \foreground ->
+	withNothing $ start False foreground stopdaemon]
 
 foregroundOption :: Option
 foregroundOption = Option.flag [] "foreground" "do not daemonize"
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index f143d8667b..d3153f6304 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -18,12 +18,14 @@ import Utility.Daemon (checkDaemon, lockPidFile)
 import Init
 import qualified Git.CurrentRepo
 import qualified Annex
+import Locations.UserConfig
 
+import System.Posix.Directory
 import Control.Concurrent
 import Control.Concurrent.STM
 
 def :: [Command]
-def = [oneShot $ noRepo firstRun $ dontCheck repoExists $
+def = [oneShot $ noRepo startNoRepo $ dontCheck repoExists $
         command "webapp" paramNothing seek "launch webapp"]
 
 seek :: [CommandSeek]
@@ -31,7 +33,7 @@ seek = [withNothing start]
 
 start :: CommandStart
 start = notBareRepo $ do
-	ifM (isInitialized) ( go , liftIO firstRun )
+	ifM (isInitialized) ( go , liftIO startNoRepo )
 	stop
 	where
 		go = do
@@ -46,14 +48,24 @@ start = notBareRepo $ do
 			liftIO $ isJust <$> checkDaemon pidfile
 		checkshim f = liftIO $ doesFileExist f
 
-openBrowser :: FilePath -> IO ()
-openBrowser htmlshim = unlessM (runBrowser url) $
-	error $ "failed to start web browser on url " ++ url
-	where
-		url = fileUrl htmlshim
+{- When run without a repo, see if there is an autoStartFile,
+ - and if so, start the first available listed repository.
+ - If not, it's our first time being run! -}
+startNoRepo :: IO ()
+startNoRepo = do
+	autostartfile <- autoStartFile
+	ifM (doesFileExist autostartfile) ( autoStart autostartfile , firstRun )
 
-fileUrl :: FilePath -> String
-fileUrl file = "file://" ++ file
+autoStart :: FilePath -> IO ()
+autoStart autostartfile = do
+	dirs <- lines <$> readFile autostartfile
+	edirs <- filterM doesDirectoryExist dirs
+	case edirs of
+		[] -> firstRun -- what else can I do? Nothing works..
+		(d:_) -> do
+			changeWorkingDirectory d
+			state <- Annex.new =<< Git.CurrentRepo.get
+			void $ Annex.eval state $ doCommand start
 
 {- Run the webapp without a repository, which prompts the user, makes one,
  - changes to it, starts the regular assistant, and redirects the
@@ -92,3 +104,12 @@ firstRun = do
 		{- Set up the pid file in the new repo. -}
 		dummydaemonize = do
 			liftIO . lockPidFile =<< fromRepo gitAnnexPidFile
+
+openBrowser :: FilePath -> IO ()
+openBrowser htmlshim = unlessM (runBrowser url) $
+	error $ "failed to start web browser on url " ++ url
+	where
+		url = fileUrl htmlshim
+
+fileUrl :: FilePath -> String
+fileUrl file = "file://" ++ file
diff --git a/Locations/UserConfig.hs b/Locations/UserConfig.hs
new file mode 100644
index 0000000000..9b04aed619
--- /dev/null
+++ b/Locations/UserConfig.hs
@@ -0,0 +1,26 @@
+{- git-annex user config files
+ -
+ - Copyright 2012 Joey Hess 
+ -
+ - Licensed under the GNU GPL version 3 or higher.
+ -}
+
+module Locations.UserConfig where
+
+import Utility.FreeDesktop
+
+import System.FilePath
+
+{- ~/.config/git-annex/file -}
+userConfigFile :: FilePath -> IO FilePath
+userConfigFile file = do
+	dir <- userConfigDir
+	return $ dir  "git-annex"  file
+
+autoStartFile :: IO FilePath
+autoStartFile = userConfigFile "autostart"
+
+{- The path to git-annex is written here; which is useful when cabal
+ - has installed it to some aweful non-PATH location. -}
+programFile :: IO FilePath
+programFile = userConfigFile "program"
diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs
index 5bab4950ab..784473b275 100644
--- a/Utility/FreeDesktop.hs
+++ b/Utility/FreeDesktop.hs
@@ -23,10 +23,8 @@ module Utility.FreeDesktop (
 ) where
 
 import Utility.Exception
-import Utility.Directory
 import Utility.Path
 
-import System.IO
 import System.Environment
 import System.Directory
 import System.FilePath
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 0a6df035be..6a1c70a4c3 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -809,10 +809,17 @@ For example, this makes two copies be needed for wav files:
 
 # FILES
 
-These files are used by git-annex, in your git repository:
+These files are used by git-annex:
 
-`.git/annex/objects/` contains the annexed file contents that are currently
-available. Annexed files in your git repository symlink to that content.
+`.git/annex/objects/` in your git repository contains the annexed file
+contents that are currently available. Annexed files in your git
+repository symlink to that content.
+
+`.git/annex/` in your git repository contains other run-time information
+used by git-annex.
+
+`~/.config/git-annex/autostart` is a list of git repositories
+to start the git-annex assistant in.
 
 # SEE ALSO
 

From 8e1077cb62a7b177bf0f36b1fedbe35fe38a91e7 Mon Sep 17 00:00:00 2001
From: "http://piotr.ozarowski.pl/" 
Date: Thu, 2 Aug 2012 10:30:55 +0000
Subject: [PATCH 4502/8313] Added a comment: XDG

---
 .../comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment

diff --git a/doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment b/doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment
new file mode 100644
index 0000000000..a09b721be9
--- /dev/null
+++ b/doc/design/assistant/blog/day_50__directory_name/comment_2_2b8ceb0a26f25e8ed2711bcbe7225a58._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://piotr.ozarowski.pl/"
+ nickname="POX"
+ subject="XDG"
+ date="2012-08-02T10:30:55Z"
+ content="""
+same for XDG_DESKTOP_DIR (for pl_PL ~/.config/user-dirs.dirs file contains XDG_DESKTOP_DIR=\"$HOME/Pulpit\" instead of Desktop)
+"""]]

From adf5789c1b6c80549d28284834b91a1cbddc90c9 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 07:47:20 -0400
Subject: [PATCH 4503/8313] fix bugs, add desktop dir

---
 Utility/FreeDesktop.hs | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs
index 784473b275..f2168077a9 100644
--- a/Utility/FreeDesktop.hs
+++ b/Utility/FreeDesktop.hs
@@ -19,7 +19,8 @@ module Utility.FreeDesktop (
 	systemDataDir,
 	systemConfigDir,
 	userDataDir,
-	userConfigDir	
+	userConfigDir,
+	userDesktopDir
 ) where
 
 import Utility.Exception
@@ -72,10 +73,14 @@ writeDesktopMenuFile d file = do
 	createDirectoryIfMissing True (parentDir file)
 	writeFile file $ buildDesktopMenuFile d
 
+{- Path to use for a desktop menu file, in either the systemDataDir or
+ - the userDataDir -}
 desktopMenuFilePath :: String -> FilePath -> FilePath
 desktopMenuFilePath basename datadir = 
 	datadir  "applications"  desktopfile basename
 
+{- Path to use for a desktop autostart file, in either the systemDataDir
+ - or the userDataDir -}
 autoStartPath :: String -> FilePath -> FilePath
 autoStartPath basename configdir =
 	configdir  "autostart"  desktopfile basename
@@ -83,21 +88,27 @@ autoStartPath basename configdir =
 desktopfile :: FilePath -> FilePath
 desktopfile f = f ++ ".desktop"
 
+{- Directory used for installation of system wide data files.. -}
 systemDataDir :: FilePath
 systemDataDir = "/usr/share"
 
+{- Directory used for installation of system wide config files. -}
 systemConfigDir :: FilePath
 systemConfigDir = "/etc/xdg"
 
+{- Directory for user data files. -}
 userDataDir :: IO FilePath
-userDataDir = do
-	dir <- xdgEnv "DATA_HOME" =<< myHomeDir
-	return $ dir  ".local"  "share"
+userDataDir = xdgEnvHome "DATA_HOME" ".local/share"
 
+{- Directory for user config files. -}
 userConfigDir :: IO FilePath
-userConfigDir = do
-	dir <- xdgEnv "CONFIG_HOME" =<< myHomeDir
-	return $ dir  ".config"
+userConfigDir = xdgEnvHome "CONFIG_HOME" ".config"
 
-xdgEnv :: String -> String -> IO String
-xdgEnv envbase def = catchDefaultIO (getEnv $ "XDG_" ++ envbase) def
+{- Directory for the user's Desktop, may be localized. -}
+userDesktopDir :: IO FilePath
+userDesktopDir = xdgEnvHome "DESKTOP_DIR" "Desktop"
+
+xdgEnvHome :: String -> String -> IO String
+xdgEnvHome envbase homedef = do
+	home <- myHomeDir
+	catchDefaultIO (getEnv $ "XDG_" ++ envbase) (home  homedef)

From 112ce4f49c95022d1afe83bd31f4af35a01f877c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 07:47:39 -0400
Subject: [PATCH 4504/8313] support XDG_DATA_DIR

---
 Assistant/WebApp/Configurators.hs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 01245b4bc1..1c7ea7cf41 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -111,8 +111,10 @@ defaultRepositoryPath firstrun = do
 	cwd <- liftIO $ getCurrentDirectory
 	home <- myHomeDir
 	if home == cwd && firstrun
-		then ifM (doesDirectoryExist $ home  "Desktop")
-			(return "~/Desktop/annex", return "~/annex")
+		then do
+			desktop <- userDesktopDir
+			ifM (doesDirectoryExist desktop)
+				(relHome (desktop  "annex"), return "~/annex")
 		else return cwd
 
 addRepositoryForm :: Form RepositoryPath

From 9a038b4a9b62824646bf4e876ed9017a1128aa56 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 07:47:50 -0400
Subject: [PATCH 4505/8313] better ~/ handling

---
 Assistant/Threads/WebApp.hs       | 9 +++------
 Assistant/WebApp/Configurators.hs | 1 +
 Utility/Path.hs                   | 8 ++++++++
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index e8de408a13..7ea7314e0f 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -64,12 +64,9 @@ webAppThread mst dstatus transferqueue postfirstrun onstartup = do
 		Just st -> go port webapp =<< runThreadState st (fromRepo gitAnnexHtmlShim)
 	where
 		getreldir Nothing = return Nothing
-		getreldir (Just st) = do
-			dir <- absPath =<< runThreadState st (fromRepo repoPath)
-			home <- myHomeDir
-			return $ Just $ if dirContains home dir
-				then relPathDirToFile home dir
-				else dir
+		getreldir (Just st) = Just <$>
+			(relHome =<< absPath
+				=<< runThreadState st (fromRepo repoPath))
 		go port webapp htmlshim = do
 			writeHtmlShim webapp port htmlshim
 			maybe noop (\a -> a (myUrl webapp port "/") htmlshim) onstartup
diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 1c7ea7cf41..08a1f60d3e 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -22,6 +22,7 @@ import qualified Git.Construct
 import qualified Git.Config
 import qualified Annex
 import Locations.UserConfig
+import Utility.FreeDesktop
 
 import Yesod
 import Data.Text (Text)
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 76fbc6c4a4..209ff1b0f2 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -132,6 +132,14 @@ runPreserveOrder a files = preserveOrder files <$> a files
 myHomeDir :: IO FilePath
 myHomeDir = homeDirectory <$> (getUserEntryForID =<< getEffectiveUserID)
 
+{- Converts paths in the home directory to use ~/ -}
+relHome :: FilePath -> IO String
+relHome path = do
+	home <- myHomeDir
+	return $ if dirContains home path
+		then "~/" ++ relPathDirToFile home path
+		else path
+
 {- Checks if a command is available in PATH. -}
 inPath :: String -> IO Bool
 inPath command = getSearchPath >>= anyM indir

From 22d7447cede9b8eb43ab552700092b74726e8c60 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 07:55:10 -0400
Subject: [PATCH 4506/8313] fix tilde expansion

---
 Assistant/WebApp/Configurators.hs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs
index 08a1f60d3e..9c308dee14 100644
--- a/Assistant/WebApp/Configurators.hs
+++ b/Assistant/WebApp/Configurators.hs
@@ -100,7 +100,7 @@ checkRepositoryPath p = do
 			tocheck <- ifM (doesDirectoryExist path)
 				(return path, return $ parentDir path)
 			not <$> (catchBoolIO $ fileAccess tocheck False True False)
-		expandTilde home ('~':path) = home  path
+		expandTilde home ('~':'/':path) = home  path
 		expandTilde _ path = path
 
 {- On first run, if run in the home directory, default to putting it in

From 3695cab949ccd6096f3ce1c121a909416851462c Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 09:00:13 -0400
Subject: [PATCH 4507/8313] avoid showing alert when there are no remotes to
 push to

---
 Assistant/Threads/Pusher.hs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Assistant/Threads/Pusher.hs b/Assistant/Threads/Pusher.hs
index 3fe85673bc..ab0274db1d 100644
--- a/Assistant/Threads/Pusher.hs
+++ b/Assistant/Threads/Pusher.hs
@@ -54,8 +54,9 @@ pushThread st dstatus commitchan pushmap = do
 		if shouldPush now commits
 			then do
 				remotes <- knownRemotes <$> getDaemonStatus dstatus
-				void $ alertWhile dstatus (pushAlert remotes) $
-					pushToRemotes thisThread now st (Just pushmap) remotes
+				unless (null remotes) $ 
+					void $ alertWhile dstatus (pushAlert remotes) $
+						pushToRemotes thisThread now st (Just pushmap) remotes
 			else do
 				debug thisThread
 					[ "delaying push of"

From 191ee3b697cfefd4061c2a398b4c6a021895bacd Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 09:03:04 -0400
Subject: [PATCH 4508/8313] awesome alert combining

Now an alert tracks files that have recently been added. As a large file
is added, it will have its own alert, that then combines with the tracker
when dones.

Also used for combining sanity checker alerts, as it could possibly want to
display a lot.
---
 Assistant/Alert.hs             | 72 +++++++++++++++++++++++++++++-----
 Assistant/Threads/Committer.hs | 16 ++++----
 2 files changed, 72 insertions(+), 16 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 0412dfe519..5877ba069c 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -27,6 +27,15 @@ data AlertPriority = Filler | Low | Medium | High | Pinned
 {- An alert can be a simple message, or an arbitrary Yesod Widget. -}
 data AlertMessage = StringAlert String | WidgetAlert (Alert -> Widget)
 
+{- An alert can have an name, which is used to combine it with other similar
+ - alerts. -}
+data AlertName = AddFileAlert | DownloadFailedAlert | SanityCheckFixAlert
+	deriving (Eq)
+
+{- The first alert is the new alert, the second is an old alert.
+ - Should return a modified version of the old alert. -}
+type AlertCombiner = Maybe (Alert -> Alert -> Maybe Alert)
+
 data Alert = Alert
 	{ alertClass :: AlertClass
 	, alertHeader :: Maybe String
@@ -35,6 +44,8 @@ data Alert = Alert
 	, alertClosable :: Bool
 	, alertPriority :: AlertPriority
 	, alertIcon :: Maybe String
+	, alertCombiner :: AlertCombiner
+	, alertName :: Maybe AlertName
 	}
 
 type AlertPair = (AlertId, Alert)
@@ -123,17 +134,21 @@ isFiller alert = alertPriority alert == Filler
 
 {- Converts a given alert into filler, manipulating it in the AlertMap.
  -
- - Any old filler that looks the same as the reference alert is removed.
+ - Any old filler that looks the same as the reference alert is removed,
+ - or, if the input alert has an alertCombine that combines it with
+ - old filler, the old filler is replaced with the result, and the
+ - input alert is removed.
  -
  - Old filler alerts are pruned once maxAlerts is reached.
  -}
 convertToFiller :: AlertId -> Bool -> AlertMap -> AlertMap
 convertToFiller i success m = case M.lookup i m of
 	Nothing -> m
-	Just al -> 
+ 	Just al ->
 		let al' = makeAlertFiller success al
-		in pruneBloat $ M.filterWithKey (pruneSame al') $
-			M.insertWith' const i al' m
+		in case alertCombiner al' of
+			Nothing -> updatePrune al'
+			Just combiner -> updateCombine combiner al'
 	where
 		pruneSame ref k al = k == i || not (effectivelySameAlert ref al)
 		pruneBloat m'
@@ -144,6 +159,13 @@ convertToFiller i success m = case M.lookup i m of
 				pruneold l =
 			 		let (f, rest) = partition (\(_, al) -> isFiller al) l
 					in drop bloat f ++ rest
+		updatePrune al = pruneBloat $ M.filterWithKey (pruneSame al) $
+			M.insertWith' const i al m
+		updateCombine combiner al = 
+			let combined = M.mapMaybe (combiner al) m
+			in if M.null combined
+				then updatePrune al
+				else M.delete i $ M.union combined m
 
 baseActivityAlert :: Alert
 baseActivityAlert = Alert
@@ -154,6 +176,8 @@ baseActivityAlert = Alert
 	, alertClosable = False
 	, alertPriority = Medium
 	, alertIcon = Just "refresh"
+	, alertCombiner = Nothing
+	, alertName = Nothing
 	}
 
 activityAlert :: Maybe String -> String -> Alert
@@ -203,13 +227,43 @@ sanityCheckFixAlert :: String -> Alert
 sanityCheckFixAlert msg = Alert
 	{ alertClass = Warning
 	, alertHeader = Just "Fixed a problem"
-	, alertMessage = StringAlert $ unwords
-		[ "The daily sanity check found and fixed a problem:"
-		, msg
-		, "If these problems persist, consider filing a bug report."
-		]
+	, alertMessage = StringAlert $ unlines [ alerthead, msg, alertfoot ]
 	, alertBlockDisplay = True
 	, alertPriority = High
 	, alertClosable = True
 	, alertIcon = Just "exclamation-sign"
+	, alertName = Just SanityCheckFixAlert
+	, alertCombiner = messageCombiner combinemessage
 	}
+	where
+		alerthead = "The daily sanity check found and fixed a problem:"
+		alertfoot = "If these problems persist, consider filing a bug report."
+		combinemessage (StringAlert new) (StringAlert old) =
+			let newmsg = filter (/= alerthead) $
+				filter (/= alertfoot) $
+				lines old ++ lines new
+			in Just $ StringAlert $
+				unlines $ alerthead : newmsg ++ [alertfoot]
+		combinemessage _ _ = Nothing
+
+addFileAlert :: FilePath -> Alert
+addFileAlert file = (activityAlert (Just "Added") $ takeFileName file)
+	{ alertName = Just AddFileAlert
+	, alertCombiner = messageCombiner combinemessage
+	}
+	where
+		combinemessage (StringAlert new) (StringAlert old) =
+			Just $ StringAlert $
+				unlines $ take 10 $ new : lines old
+		combinemessage _ _ = Nothing
+
+messageCombiner :: (AlertMessage -> AlertMessage -> Maybe AlertMessage) -> AlertCombiner
+messageCombiner combinemessage = Just go
+	where
+		go new old
+			| alertClass new /= alertClass old = Nothing
+			| alertName new == alertName old =
+				case combinemessage (alertMessage new) (alertMessage old) of
+					Nothing -> Nothing
+					Just m -> Just $ old { alertMessage = m }
+			| otherwise = Nothing
diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs
index f236159f98..2ca6a15b93 100644
--- a/Assistant/Threads/Committer.hs
+++ b/Assistant/Threads/Committer.hs
@@ -10,6 +10,7 @@ module Assistant.Threads.Committer where
 import Assistant.Common
 import Assistant.Changes
 import Assistant.Commits
+import Assistant.Alert
 import Assistant.ThreadedMonad
 import Assistant.Threads.Watcher
 import Assistant.TransferQueue
@@ -143,15 +144,16 @@ handleAdds st changechan transferqueue dstatus cs = returnWhen (null pendingadds
 
 		add :: Change -> IO (Maybe Change)
 		add change@(PendingAddChange { keySource = ks }) =
-			liftM maybeMaybe $ catchMaybeIO $
-				sanitycheck ks $ runThreadState st $ do
-					showStart "add" $ keyFilename ks
-					key <- Command.Add.ingest ks
-					handle (finishedChange change) (keyFilename ks) key
+			alertWhile' dstatus (addFileAlert $ keyFilename ks) $
+				liftM maybeMaybe $ catchMaybeIO $
+					sanitycheck ks $ runThreadState st $ do
+						showStart "add" $ keyFilename ks
+						key <- Command.Add.ingest ks
+						handle (finishedChange change) (keyFilename ks) key
 		add _ = return Nothing
 
-		maybeMaybe (Just j@(Just _)) = j
-		maybeMaybe _ = Nothing
+		maybeMaybe (Just j@(Just _)) = (True, j)
+		maybeMaybe _ = (False, Nothing)
 
 		handle _ _ Nothing = do
 			showEndFail

From 7cc5fdbd7b25a32c1e5786abbd419c845711d9d5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 09:08:58 -0400
Subject: [PATCH 4509/8313] layout

---
 templates/sidebar/alert.hamlet | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/templates/sidebar/alert.hamlet b/templates/sidebar/alert.hamlet
index 84126c3810..9c204402bc 100644
--- a/templates/sidebar/alert.hamlet
+++ b/templates/sidebar/alert.hamlet
@@ -7,7 +7,7 @@
        #
   $maybe h <- heading
     $if block
-      
#{h}

+      
#{h}
 #
     $else
-      #{h}
+      #{h} #
   ^{widget}

From 520a0463a7172314298db0ce507fa355dc1238a3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 09:09:06 -0400
Subject: [PATCH 4510/8313] start webapp thread earlier so it opens ASAP

---
 Assistant.hs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Assistant.hs b/Assistant.hs
index 6af6c6639e..b81806ff9c 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -154,6 +154,9 @@ startAssistant assistant daemonize webappwaiter = do
 			scanremotes <- newScanRemoteMap
 			mapM_ startthread
 				[ watch $ commitThread st changechan commitchan transferqueue dstatus
+#ifdef WITH_WEBAPP
+				, assist $ webAppThread (Just st) dstatus transferqueue Nothing webappwaiter
+#endif
 				, assist $ pushThread st dstatus commitchan pushmap
 				, assist $ pushRetryThread st dstatus pushmap
 				, assist $ mergeThread st
@@ -163,9 +166,6 @@ startAssistant assistant daemonize webappwaiter = do
 				, assist $ sanityCheckerThread st dstatus transferqueue changechan
 				, assist $ mountWatcherThread st dstatus scanremotes
 				, assist $ transferScannerThread st dstatus scanremotes transferqueue
-#ifdef WITH_WEBAPP
-				, assist $ webAppThread (Just st) dstatus transferqueue Nothing webappwaiter
-#endif
 				, watch $ watchThread st dstatus transferqueue changechan
 				]
 			waitForTermination

From 1f2127c520b48e35a5fb335c9e711559733dfd23 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 09:15:08 -0400
Subject: [PATCH 4511/8313] trim long filenames (have to fit on the sidebar)

30 characters would mostly work, but 20 is safer due to some wider letters
like 'w'. Of course this is very heuristic based on filesize anyway.

(Bootstrap does a surprisingly bad job at dealing with overlong words
in the sidebar.)
---
 Assistant/Alert.hs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 5877ba069c..6b32fde1d9 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -247,11 +247,18 @@ sanityCheckFixAlert msg = Alert
 		combinemessage _ _ = Nothing
 
 addFileAlert :: FilePath -> Alert
-addFileAlert file = (activityAlert (Just "Added") $ takeFileName file)
+addFileAlert file = (activityAlert (Just "Added") $ trim $ takeFileName file)
 	{ alertName = Just AddFileAlert
 	, alertCombiner = messageCombiner combinemessage
 	}
 	where
+		trim f
+			| len < maxlen = f
+			| otherwise = take half f ++ ".." ++ drop (len - half) f
+			where
+				len = length f
+				maxlen = 20
+				half = (maxlen - 2) `div` 2 
 		combinemessage (StringAlert new) (StringAlert old) =
 			Just $ StringAlert $
 				unlines $ take 10 $ new : lines old

From d2b48cacdbb192e7501985e93e60b64fa57fa72d Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 09:20:21 -0400
Subject: [PATCH 4512/8313] add some strictness annotations

on general principles
---
 Assistant/Alert.hs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 6b32fde1d9..b234d2a5a5 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE RankNTypes, BangPatterns #-}
 
 module Assistant.Alert where
 
@@ -272,5 +272,5 @@ messageCombiner combinemessage = Just go
 			| alertName new == alertName old =
 				case combinemessage (alertMessage new) (alertMessage old) of
 					Nothing -> Nothing
-					Just m -> Just $ old { alertMessage = m }
+					Just !m -> Just $! old { alertMessage = m }
 			| otherwise = Nothing

From 6d0353eaadcedc205bae4481cb5116a17b178cc8 Mon Sep 17 00:00:00 2001
From: 
 "https://www.google.com/accounts/o8/id?id=AItOawmnrN9a3uAu6Ur2SSyE_AiKw7bOon1yJBc"
 
Date: Thu, 2 Aug 2012 14:22:28 +0000
Subject: [PATCH 4513/8313]

---
 doc/forum/git_unannex_speed.mdwn | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/forum/git_unannex_speed.mdwn

diff --git a/doc/forum/git_unannex_speed.mdwn b/doc/forum/git_unannex_speed.mdwn
new file mode 100644
index 0000000000..a5e6ad5719
--- /dev/null
+++ b/doc/forum/git_unannex_speed.mdwn
@@ -0,0 +1 @@
+It was fast to git annex a bunch of files. But git unannex seems a lot slower. Is there a faster way to get files out of git annex? Or to replace the symlinks with real files and then I could just remove the .git directory? I shouldn't have put so many in as a test but they are there now.

From ab6ad94bb9d7e576f2af1398823a63de180e6365 Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 2 Aug 2012 16:58:41 +0000
Subject: [PATCH 4514/8313] Added a comment

---
 .../comment_1_10cf326248f4e89e1f75bf97d7574763._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment

diff --git a/doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment b/doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment
new file mode 100644
index 0000000000..39fc4793cc
--- /dev/null
+++ b/doc/forum/git_unannex_speed/comment_1_10cf326248f4e89e1f75bf97d7574763._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-08-02T16:58:41Z"
+ content="""
+It currently commits once per file, which is slow. `Command/Unannex.hs` explains why this is necessary. If I think of a way to avoid that, I will.
+"""]]

From 6b38227bad8c303470a5b29f50360cea0206d9a1 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 13:12:36 -0400
Subject: [PATCH 4515/8313] only write data file when installing as normal user

---
 Build/InstallDesktopFile.hs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs
index 3bc796315a..ab2773bf11 100644
--- a/Build/InstallDesktopFile.hs
+++ b/Build/InstallDesktopFile.hs
@@ -46,9 +46,10 @@ writeDesktop command = do
 	writeDesktopMenuFile (autostart command) $
 		autoStartPath "git-annex" configdir
 
-	programfile <- programFile
-	createDirectoryIfMissing True (parentDir programFile)
-	writeFile programfile command
+	when (uid /= 0) $ do
+		programfile <- programFile
+		createDirectoryIfMissing True (parentDir programFile)
+		writeFile programfile command
 
 main = getArgs >>= go
 	where

From f294825b891d5d7bdddae1b4da149256b3faf97b Mon Sep 17 00:00:00 2001
From: "http://xolus.net/openid/max" 
Date: Thu, 2 Aug 2012 17:45:10 +0000
Subject: [PATCH 4516/8313] Added a comment: What about NTFS support ?

---
 .../comment_3_bd0a12f4c9b884ab8a06082842381a01._comment   | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment

diff --git a/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment b/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment
new file mode 100644
index 0000000000..0b48db7502
--- /dev/null
+++ b/doc/todo/windows_support/comment_3_bd0a12f4c9b884ab8a06082842381a01._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="http://xolus.net/openid/max"
+ nickname="B0FH"
+ subject="What about NTFS support ?"
+ date="2012-08-02T17:45:10Z"
+ content="""
+Has git-annex been tested with an NTFS-formatted disk under Linux ? NTFS is supposed to be case-sensitive and to allow symlinks, and these are supposed to work with ntfs3g.
+"""]]

From d2f975944380aab57047a9e84af620e15464fa15 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 13:47:26 -0400
Subject: [PATCH 4517/8313] refactor

---
 Assistant/Alert.hs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index b234d2a5a5..16ce222ad5 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -247,18 +247,11 @@ sanityCheckFixAlert msg = Alert
 		combinemessage _ _ = Nothing
 
 addFileAlert :: FilePath -> Alert
-addFileAlert file = (activityAlert (Just "Added") $ trim $ takeFileName file)
+addFileAlert file = (activityAlert (Just "Added") $ shortFile $ takeFileName file)
 	{ alertName = Just AddFileAlert
 	, alertCombiner = messageCombiner combinemessage
 	}
 	where
-		trim f
-			| len < maxlen = f
-			| otherwise = take half f ++ ".." ++ drop (len - half) f
-			where
-				len = length f
-				maxlen = 20
-				half = (maxlen - 2) `div` 2 
 		combinemessage (StringAlert new) (StringAlert old) =
 			Just $ StringAlert $
 				unlines $ take 10 $ new : lines old
@@ -274,3 +267,13 @@ messageCombiner combinemessage = Just go
 					Nothing -> Nothing
 					Just !m -> Just $! old { alertMessage = m }
 			| otherwise = Nothing
+
+shortFile :: FilePath -> String
+shortFile f
+	| len < maxlen = f
+	| otherwise = take half f ++ ".." ++ drop (len - half) f
+	where
+		len = length f
+		maxlen = 20
+		half = (maxlen - 2) `div` 2 
+

From a6e4283fed739207e3fd5a239ec71f7a71789259 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 13:55:38 -0400
Subject: [PATCH 4518/8313] add slight delay in between sidebar updates, to
 avoid excessive churn

Tested and 0.01 seconds is not perceivable as a delay when interacting with
the UI.
---
 Assistant/WebApp/SideBar.hs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs
index a4b8378979..4373b5a5b0 100644
--- a/Assistant/WebApp/SideBar.hs
+++ b/Assistant/WebApp/SideBar.hs
@@ -20,6 +20,7 @@ import Utility.Yesod
 import Yesod
 import Data.Text (Text)
 import qualified Data.Map as M
+import Control.Concurrent
 
 sideBarDisplay :: Widget
 sideBarDisplay = do
@@ -75,6 +76,13 @@ getSideBarR :: NotificationId -> Handler RepHtml
 getSideBarR nid = do
 	waitNotifier alertNotifier nid
 
+	{- This 0.1 second delay avoids very transient notifications from
+	 - being displayed and churning the sidebar unnecesarily. 
+	 -
+	 - This needs to be below the level perceptable by the user,
+	 - to avoid slowing down user actions like closing alerts. -}
+	liftIO $ threadDelay 100000
+
 	page <- widgetToPageContent sideBarDisplay
 	hamletToRepHtml $ [hamlet|^{pageBody page}|]
 

From e21a32627fc600e6f7fa72929c95cfb49390dae3 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 13:57:34 -0400
Subject: [PATCH 4519/8313] avoid bogus alert errors

---
 Assistant/Threads/Committer.hs | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs
index 2ca6a15b93..cce8d5e2b6 100644
--- a/Assistant/Threads/Committer.hs
+++ b/Assistant/Threads/Committer.hs
@@ -145,16 +145,19 @@ handleAdds st changechan transferqueue dstatus cs = returnWhen (null pendingadds
 		add :: Change -> IO (Maybe Change)
 		add change@(PendingAddChange { keySource = ks }) =
 			alertWhile' dstatus (addFileAlert $ keyFilename ks) $
-				liftM maybeMaybe $ catchMaybeIO $
+				liftM ret $ catchMaybeIO $
 					sanitycheck ks $ runThreadState st $ do
 						showStart "add" $ keyFilename ks
 						key <- Command.Add.ingest ks
 						handle (finishedChange change) (keyFilename ks) key
+			where
+				{- Add errors tend to be transient and will
+				 - be automatically dealt with, so don't
+				 - pass to the alert code. -}
+				ret (Just j@(Just _)) = (True, j)
+				ret _ = (True, Nothing)
 		add _ = return Nothing
 
-		maybeMaybe (Just j@(Just _)) = (True, j)
-		maybeMaybe _ = (False, Nothing)
-
 		handle _ _ Nothing = do
 			showEndFail
 			return Nothing

From 74fc9fcbe68ce7802d44499e8b5dc5c5d186ee66 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 14:02:35 -0400
Subject: [PATCH 4520/8313] add alert when committing

---
 Assistant/Alert.hs             | 3 +++
 Assistant/Threads/Committer.hs | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/Assistant/Alert.hs b/Assistant/Alert.hs
index 16ce222ad5..b152c48dca 100644
--- a/Assistant/Alert.hs
+++ b/Assistant/Alert.hs
@@ -189,6 +189,9 @@ activityAlert header message = baseActivityAlert
 startupScanAlert :: Alert
 startupScanAlert = activityAlert Nothing "Performing startup scan"
 
+commitAlert :: Alert
+commitAlert = activityAlert Nothing "Committing changes to git"
+
 pushAlert :: [Remote] -> Alert
 pushAlert rs = activityAlert Nothing $
 	"Syncing with " ++ unwords (map Remote.name rs)
diff --git a/Assistant/Threads/Committer.hs b/Assistant/Threads/Committer.hs
index cce8d5e2b6..095c8feac0 100644
--- a/Assistant/Threads/Committer.hs
+++ b/Assistant/Threads/Committer.hs
@@ -54,7 +54,9 @@ commitThread st changechan commitchan transferqueue dstatus = runEvery (Seconds
 						, show (length readychanges)
 						, "changes"
 						]
-					void $ tryIO $ runThreadState st commitStaged
+					void $ alertWhile dstatus commitAlert $
+						tryIO (runThreadState st commitStaged)
+							>> return True
 					recordCommit commitchan (Commit time)
 				else refill readychanges
 		else refill changes

From e11946796d05a733290a47e9cead0b2a2e58d272 Mon Sep 17 00:00:00 2001
From: Nicolas Pouillard 
Date: Thu, 2 Aug 2012 20:11:41 +0200
Subject: [PATCH 4521/8313] Fix imports and casing in Build.InstallDesktopFile

---
 Build/InstallDesktopFile.hs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Build/InstallDesktopFile.hs b/Build/InstallDesktopFile.hs
index ab2773bf11..891431ebe2 100644
--- a/Build/InstallDesktopFile.hs
+++ b/Build/InstallDesktopFile.hs
@@ -10,8 +10,12 @@ module Build.InstallDesktopFile where
 
 import Utility.Exception
 import Utility.FreeDesktop
+import Utility.Path
+import Locations.UserConfig
 
 import Control.Applicative
+import Control.Monad
+import System.Directory
 import System.Environment
 import System.Posix.User
 
@@ -48,7 +52,7 @@ writeDesktop command = do
 
 	when (uid /= 0) $ do
 		programfile <- programFile
-		createDirectoryIfMissing True (parentDir programFile)
+		createDirectoryIfMissing True (parentDir programfile)
 		writeFile programfile command
 
 main = getArgs >>= go

From 83c65f05f5f54b7af8edc389f386a8ebc0e8acc2 Mon Sep 17 00:00:00 2001
From: "http://nico.kaiser.me/" 
Date: Thu, 2 Aug 2012 22:56:12 +0000
Subject: [PATCH 4522/8313]

---
 doc/forum/DS__95__Store_files_are_not_added.mdwn | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/forum/DS__95__Store_files_are_not_added.mdwn

diff --git a/doc/forum/DS__95__Store_files_are_not_added.mdwn b/doc/forum/DS__95__Store_files_are_not_added.mdwn
new file mode 100644
index 0000000000..f0ccf2023b
--- /dev/null
+++ b/doc/forum/DS__95__Store_files_are_not_added.mdwn
@@ -0,0 +1,3 @@
+The "git annex add" command adds new file to the annex. However ".DS_Store" files are ignored by git-annex. Is there a list of files that are being ignored? 
+
+Maybe sometimes it's useful to add .DS_Store extended attribute data to the annex to ensure a complete sync of Mac files...

From f57becc8bd30fbaa0b221db0ab6ee33bb43588f6 Mon Sep 17 00:00:00 2001
From: "http://nico.kaiser.me/" 
Date: Thu, 2 Aug 2012 23:00:54 +0000
Subject: [PATCH 4523/8313]

---
 doc/forum/Git_repositories_in_the_annex__63__.mdwn | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/forum/Git_repositories_in_the_annex__63__.mdwn

diff --git a/doc/forum/Git_repositories_in_the_annex__63__.mdwn b/doc/forum/Git_repositories_in_the_annex__63__.mdwn
new file mode 100644
index 0000000000..a18af8c2c7
--- /dev/null
+++ b/doc/forum/Git_repositories_in_the_annex__63__.mdwn
@@ -0,0 +1,5 @@
+First of all thanks for the great work. git-annex looks very nice already!
+
+I wonder if it's possible to add Git repositories to the annex. In my current Dropbox setup I drop some small Git repositories into the Dropbox to keep them in sync on two computers. 
+
+This is obviously not possible with git-annex, as the annex itself is a Git repository and sub-repositories (directories with .git folders) are ignored. But in some cases people might want to be able to drop "anything" into the annex, maybe even Git repositories (e.g. the "nomad" usecase)...

From 002e08084643863e63e488216981ad76bb8be77c Mon Sep 17 00:00:00 2001
From: "http://nico.kaiser.me/" 
Date: Thu, 2 Aug 2012 23:01:15 +0000
Subject: [PATCH 4524/8313]

---
 doc/forum/Delete_unused_files__47__metadata.mdwn | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/Delete_unused_files__47__metadata.mdwn

diff --git a/doc/forum/Delete_unused_files__47__metadata.mdwn b/doc/forum/Delete_unused_files__47__metadata.mdwn
new file mode 100644
index 0000000000..5c6b413996
--- /dev/null
+++ b/doc/forum/Delete_unused_files__47__metadata.mdwn
@@ -0,0 +1,7 @@
+After moving some files (about 1G, some big and some small files) in and out the annex, I noticed that the size of the repository has grown quite a bit. My empty repository now is over 100 MB (even after  "git annex dropunused ..." and "git gc").
+
+Most of this size is not Git metadata but many small files in the git-annex branch that seem to hold information about files I deleted (even in the other known repositories).
+
+So is there a way to get rid of these useless but space consuming information?
+
+(Maybe there is no (elegant) way to remove the symlink versions from the Git history (is there?), but it would already be nice if those small git-annex metadata files could be removed)

From 01cda6b34e9f8cd4302b9c82374dee01677e48ea Mon Sep 17 00:00:00 2001
From: "http://joeyh.name/" 
Date: Thu, 2 Aug 2012 23:29:38 +0000
Subject: [PATCH 4525/8313] Added a comment

---
 .../comment_1_30687306da9bd35ec02a806193c5e240._comment    | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment

diff --git a/doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment b/doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment
new file mode 100644
index 0000000000..3402b84c61
--- /dev/null
+++ b/doc/forum/DS__95__Store_files_are_not_added/comment_1_30687306da9bd35ec02a806193c5e240._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="http://joeyh.name/"
+ subject="comment 1"
+ date="2012-08-02T23:29:38Z"
+ content="""
+git-annex ignores whatever git does. See `git-ls-files` and the gitignore configuration.
+"""]]

From a0eee78da20eea52f21a26b55cf72b40c0486897 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 23:09:07 -0400
Subject: [PATCH 4526/8313] blog for the day

---
 .../assistant/blog/day_51__desktop.mdwn       | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 doc/design/assistant/blog/day_51__desktop.mdwn

diff --git a/doc/design/assistant/blog/day_51__desktop.mdwn b/doc/design/assistant/blog/day_51__desktop.mdwn
new file mode 100644
index 0000000000..c079a9a912
--- /dev/null
+++ b/doc/design/assistant/blog/day_51__desktop.mdwn
@@ -0,0 +1,34 @@
+Now installing git-annex automatically generates a freedesktop.org .desktop
+file, and installs it, either system-wide (root) or locally (user). So
+`Menu -> Internet -> Git Annex` will start up the web app.
+
+(I don't entirely like putting it on the Internet menu, but the
+Accessories menu is not any better (and much more crowded here), 
+and there's really no menu where it entirely fits.)
+
+I generated that file by writing a generic library to deal with
+freedesktop.org desktop files and locations. Which seemed like overkill at
+the time, but then I found myself continuing to use that library. Funny how
+that happens.
+
+So, there's also another .desktop file that's used to autostart the
+`git-annex assistant` daemon when the user logs into the desktop.
+
+This even works when git-annex is installed to the ugly non-PATH location
+`.cabal/bin/git-annex` by Cabal! To make that work, it records the path
+the binary is at to a freedesktop.org data file, at install time.
+
+---
+
+That should all work in Gnome, KDE, XFCE, etc. Not Mac OSX I'm guessing...
+
+---
+
+Also today, I added a sidebar notification when the assistant notices new
+files. To make that work well, I implemented merging of related sidebar
+action notifications, so the effect is that there's one notification that
+collectes a list of recently added files, and transient notifications that
+show up if a really big file is taking a while to checksum.
+
+I'm pleased that the notification interface is at a point where I was able
+to  implement all that, entirely in pure functional code.

From 13a7362a1a6264689519a8aa685c908ec5660129 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Thu, 2 Aug 2012 23:51:38 -0400
Subject: [PATCH 4527/8313] fix xdg desktop dir lookup code

had to use xdg-user-dir to query it, since it's in a shell format file.
Fall back to
---
 Utility/FreeDesktop.hs | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/Utility/FreeDesktop.hs b/Utility/FreeDesktop.hs
index f2168077a9..434537a6d2 100644
--- a/Utility/FreeDesktop.hs
+++ b/Utility/FreeDesktop.hs
@@ -25,12 +25,16 @@ module Utility.FreeDesktop (
 
 import Utility.Exception
 import Utility.Path
+import Utility.Process
+import Utility.PartialPrelude
 
 import System.Environment
 import System.Directory
 import System.FilePath
 import Data.List
 import Data.String.Utils
+import Control.Applicative
+import Control.Monad (liftM)
 
 type DesktopEntry = [(Key, Value)]
 
@@ -104,9 +108,18 @@ userDataDir = xdgEnvHome "DATA_HOME" ".local/share"
 userConfigDir :: IO FilePath
 userConfigDir = xdgEnvHome "CONFIG_HOME" ".config"
 
-{- Directory for the user's Desktop, may be localized. -}
+{- Directory for the user's Desktop, may be localized. 
+ -
+ - This is not looked up very fast; the config file is in a shell format
+ - that is best parsed by shell, so xdg-user-dir is used, with a fallback
+ - to ~/Desktop. -}
 userDesktopDir :: IO FilePath
-userDesktopDir = xdgEnvHome "DESKTOP_DIR" "Desktop"
+userDesktopDir = maybe fallback return =<< (parse <$> xdg_user_dir)
+	where
+		parse = maybe Nothing (headMaybe . lines)
+		xdg_user_dir = catchMaybeIO $
+			readProcess "xdg-user-dir" ["DESKTOP"]
+		fallback = xdgEnvHome "DESKTOP_DIR" "Desktop"
 
 xdgEnvHome :: String -> String -> IO String
 xdgEnvHome envbase homedef = do

From 5b87007dad323c02b50d396583861ca8eb0c8d4e Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Aug 2012 09:00:03 -0400
Subject: [PATCH 4528/8313] done

---
 doc/design/assistant/webapp.mdwn | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/design/assistant/webapp.mdwn b/doc/design/assistant/webapp.mdwn
index 7df93cf401..5a206f20fa 100644
--- a/doc/design/assistant/webapp.mdwn
+++ b/doc/design/assistant/webapp.mdwn
@@ -29,6 +29,11 @@ The webapp is a web server that displays a shiny interface.
   over http by the web app
 * Display any relevant warning messages. One is the `inotify max_user_watches`
   exceeded message.
+* possibly add a desktop file to the top of the repository that can be used
+  to open the webapp (rather than using the menus). Would be complicated
+  some by the path to git-annex sometimes needing to be hardcoded and varying
+  across systems, so it would need to be a symlink to `.git/annex/desktop`
+  which would be per-system.
 
 ## first start
 

From 1f89712e6b0a601f3a4685cfbcd4cb5d3180c0e5 Mon Sep 17 00:00:00 2001
From: Joey Hess 
Date: Fri, 3 Aug 2012 09:44:43 -0400
Subject: [PATCH 4529/8313] add a navbar button that opens the repo in the
 desktop's native file browser

This should work on linux (xdg-open) and OSX (open). If the program
is not in $PATH, it falls back to opening a browser window/tab with file:///

The only tricky bit is the javascript code, that handles clicking on the
link. This is to avoid unnecessary page refreshes. Until I added the
return false at the end, the 's normal click event also fired, so two
file browsers opened. I have not checked portability extensively.
---
 Assistant/WebApp.hs           |  2 +-
 Assistant/WebApp/DashBoard.hs | 37 ++++++++++++++++++++++++++++++++++-
 Assistant/WebApp/routes       |  1 +
 templates/page.hamlet         |  3 +++
 4 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/Assistant/WebApp.hs b/Assistant/WebApp.hs
index 1b767c642d..4042d410e0 100644
--- a/Assistant/WebApp.hs
+++ b/Assistant/WebApp.hs
@@ -63,7 +63,7 @@ selectNavBar :: Handler [NavBarItem]
 selectNavBar = ifM (inFirstRun) (return firstRunNavBar, return defaultNavBar)
 
 inFirstRun :: Handler Bool
-inFirstRun = isNothing . threadState <$> getYesod
+inFirstRun = isNothing . relDir <$> getYesod
 
 {- Used instead of defaultContent; highlights the current page if it's
  - on the navbar. -}
diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs
index f4f56a4763..a1e499d702 100644
--- a/Assistant/WebApp/DashBoard.hs
+++ b/Assistant/WebApp/DashBoard.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU GPL version 3 or higher.
  -}
 
-{-# LANGUAGE TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
+{-# LANGUAGE CPP, TypeFamilies, QuasiQuotes, MultiParamTypeClasses, TemplateHaskell, OverloadedStrings, RankNTypes #-}
 
 module Assistant.WebApp.DashBoard where
 
@@ -23,6 +23,7 @@ import Utility.Percentage
 import Utility.DataUnits
 import Types.Key
 import qualified Remote
+import qualified Git
 
 import Yesod
 import Text.Hamlet
@@ -88,3 +89,37 @@ getNoScriptAutoR = bootstrap (Just DashBoard) $ do
 	let this = NoScriptAutoR
 	toWidgetHead $(hamletFile $ hamletTemplate "dashboard/metarefresh")
 	dashboard False
+
+{- The javascript code does a post. -}
+postFileBrowserR :: Handler ()
+postFileBrowserR = void openFileBrowser
+
+{- Used by non-javascript browsers, where clicking on the link actually
+ - opens this page, so we redirect back to the referrer. -}
+getFileBrowserR :: Handler ()
+getFileBrowserR = whenM openFileBrowser $ do
+	clearUltDest
+	setUltDestReferer
+	redirectUltDest HomeR
+
+{- Opens the system file browser on the repo, or, as a fallback,
+ - goes to a file:// url. Returns True if it's ok to redirect away
+ - from the page (ie, the system file browser was opened). -}
+openFileBrowser :: Handler Bool
+openFileBrowser = do
+	path <- runAnnex (error "no configured repository") $
+		fromRepo Git.repoPath
+	ifM (liftIO $ inPath cmd <&&> boolSystem cmd [File path])
+		( return True
+		, do
+			clearUltDest
+			setUltDest $ "file://" ++ path
+			void $ redirectUltDest HomeR
+			return False
+		)
+	where
+#if OSX
+		cmd = "open"
+#else
+		cmd = "xdg-open"
+#endif
diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes
index 95813edb6f..192e1cd6ba 100644
--- a/Assistant/WebApp/routes
+++ b/Assistant/WebApp/routes
@@ -11,5 +11,6 @@
 /notifier/transfers NotifierTransfersR GET
 /notifier/sidebar NotifierSideBarR GET
 /closealert/#AlertId CloseAlert GET
+/filebrowser FileBrowserR GET POST
 
 /static StaticR Static getStatic
diff --git a/templates/page.hamlet b/templates/page.hamlet
index 29a091110a..6321f7a181 100644
--- a/templates/page.hamlet
+++ b/templates/page.hamlet
@@ -9,6 +9,9 @@
             #{name}
       $maybe reldir <- relDir webapp
- $forall (num, name) <- remotelist + $forall (num, name) <- repolist
#{num} diff --git a/templates/configurators/addrepository/form.hamlet b/templates/configurators/localrepositoryform.hamlet similarity index 100% rename from templates/configurators/addrepository/form.hamlet rename to templates/configurators/localrepositoryform.hamlet diff --git a/templates/configurators/main.hamlet b/templates/configurators/main.hamlet index 150e08981a..09fa096b1c 100644 --- a/templates/configurators/main.hamlet +++ b/templates/configurators/main.hamlet @@ -1,3 +1,42 @@ -
-

- Sorry, no configuration is implemented yet... +
+
+
+

+ + Clone to removable drive +

+ Clone this repository to a USB drive, memory stick, or other # + removable media. +

+ For offline archiving, backups, or to # + SneakerNet # + between computers. +

+

+ + Clone to a local computer +

+ Automatically keep files in sync between computers on your # + local network. +

+ For easy sharing with family and friends, or between your devices. +

+
+
+

+ + Store data in the cloud +

+ Store your data on a third-party cloud platform, # + including Amazon S3, Box.com, and Rsync.net. +

+ Pay someone to keep your data safe. With strong encryption to # + protect your privacy. +

+

+ + Clone to a remote server +

+ Set up a repository on a remote Unix server running SSH. +

+ For when you want to run your own cloud. diff --git a/templates/configurators/main/sidebar.hamlet b/templates/configurators/main/sidebar.hamlet new file mode 100644 index 0000000000..3d427f8c56 --- /dev/null +++ b/templates/configurators/main/sidebar.hamlet @@ -0,0 +1,18 @@ +

+

+ git-annex is managing # + $if notenough + only # + #{numrepos} repository. # + $else + $if barelyenough + #{numrepos} repositories. # + $else + #{numrepos} repositories. # + $if notenough + Recommend you add more clones to avoid data loss. + $else + $if barelyenough + Consider adding more. + $else + Adding more can't hurt! diff --git a/templates/documentation/about.hamlet b/templates/documentation/about.hamlet index af48a0b102..e9a2334710 100644 --- a/templates/documentation/about.hamlet +++ b/templates/documentation/about.hamlet @@ -10,8 +10,8 @@
git-annex is © 2010-2012 Joey Hess. It is free software, licensed # under the terms of the GNU General Public License, version 3 or above. # -
- Its development was made possible by # +

+ Its development was made possible by # many excellent people - . + . diff --git a/templates/sidebar/alert.hamlet b/templates/sidebar/alert.hamlet index 9c204402bc..9377518195 100644 --- a/templates/sidebar/alert.hamlet +++ b/templates/sidebar/alert.hamlet @@ -7,7 +7,7 @@ # $maybe h <- heading $if block -

#{h}

# +

#{h}

# $else #{h} # ^{widget} From bbea7757ab6bab7b2993f99ab5ff2727b9f00fd3 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Aug 2012 17:22:35 -0400 Subject: [PATCH 4537/8313] update --- templates/configurators/main.hamlet | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/templates/configurators/main.hamlet b/templates/configurators/main.hamlet index 09fa096b1c..929b22437e 100644 --- a/templates/configurators/main.hamlet +++ b/templates/configurators/main.hamlet @@ -1,9 +1,12 @@
+

+ Add repositories # +

- Clone to removable drive + Clone to a removable drive

Clone this repository to a USB drive, memory stick, or other # removable media. @@ -14,13 +17,20 @@

- Clone to a local computer + Pair with a local computer

Automatically keep files in sync between computers on your # local network.

For easy sharing with family and friends, or between your devices.

+

+ + Connect to your phone +

+ Save photos and recordings from your phone. +

+ Send selected files to your phone.

@@ -30,13 +40,14 @@ Store your data on a third-party cloud platform, # including Amazon S3, Box.com, and Rsync.net.

- Pay someone to keep your data safe. With strong encryption to # - protect your privacy. + Pay someone to keep your data safe and available. # + With strong encryption to protect your privacy.

Clone to a remote server

- Set up a repository on a remote Unix server running SSH. + Set up a repository on a remote server using # + ssh or rsync.

- For when you want to run your own cloud. + To use your own personal cloud. From 1bd2be549f0736340b09cc056ce9d7c1db6b928c Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Aug 2012 20:10:32 -0400 Subject: [PATCH 4538/8313] add extra sidebar widget html is slightly broken --- Assistant/WebApp/SideBar.hs | 7 +++---- templates/sidebar/main.hamlet | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs index e0c31c9498..2fbd4e8c0f 100644 --- a/Assistant/WebApp/SideBar.hs +++ b/Assistant/WebApp/SideBar.hs @@ -24,11 +24,10 @@ import Control.Concurrent sideBarDisplay :: Maybe Widget -> Widget sideBarDisplay onsidebar = do + {- If a widget was passed to include on the sidebar, display + - it above alerts. -} + let perpage = maybe noop id onsidebar let content = do - {- If a widget was passed to include on the sidebar, display - - it above alerts. -} - maybe noop id onsidebar - {- Any yesod message appears as the first alert. -} maybe noop rendermessage =<< lift getMessage diff --git a/templates/sidebar/main.hamlet b/templates/sidebar/main.hamlet index 32900b920c..63f201cf14 100644 --- a/templates/sidebar/main.hamlet +++ b/templates/sidebar/main.hamlet @@ -1,3 +1,4 @@ -

-
+
+ ^{perpage} +
^{content} From e0c3958d9acc97c15a209c287c1d49e859ca4fea Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Aug 2012 20:40:34 -0400 Subject: [PATCH 4539/8313] improved config --- Assistant/WebApp/Configurators.hs | 38 +++++++----- Assistant/WebApp/DashBoard.hs | 2 +- Assistant/WebApp/Documentation.hs | 2 +- Assistant/WebApp/SideBar.hs | 9 +-- Assistant/WebApp/routes | 7 ++- templates/configurators/addrepository.hamlet | 62 ++++++++++++++++--- .../configurators/listrepositories.hamlet | 9 +++ templates/configurators/main.hamlet | 56 +++-------------- templates/configurators/main/sidebar.hamlet | 18 ------ templates/sidebar/main.hamlet | 5 +- 10 files changed, 103 insertions(+), 105 deletions(-) create mode 100644 templates/configurators/listrepositories.hamlet delete mode 100644 templates/configurators/main/sidebar.hamlet diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs index 018f973632..2771a22842 100644 --- a/Assistant/WebApp/Configurators.hs +++ b/Assistant/WebApp/Configurators.hs @@ -35,46 +35,52 @@ getConfigR :: Handler RepHtml getConfigR = ifM (inFirstRun) ( getFirstRepositoryR , bootstrap (Just Config) $ do - sideBarDisplay $ Just sidebar + sideBarDisplay setTitle "Configuration" $(widgetFile "configurators/main") ) - where - sidebar = do - (_repolist, numrepos, notenough, barelyenough, morethanenough) - <- lift repoList - $(widgetFile "configurators/main/sidebar") {- Lists different types of repositories that can be added. -} getAddRepositoryR :: Handler RepHtml getAddRepositoryR = bootstrap (Just Config) $ do - sideBarDisplay Nothing + sideBarDisplay setTitle "Add repository" $(widgetFile "configurators/addrepository") -{- A numbered list of known repositories, including the current one, - - as well as the total number, and whether that is not enough, - - barely enough, or more than enough. -} -repoList :: Handler ([(String, String)], String, Bool, Bool, Bool) +{- Lists known repositories. -} +getListRepositoriesR :: Handler RepHtml +getListRepositoriesR = bootstrap (Just Config) $ do + sideBarDisplay + setTitle "Repository list" + repolist <- lift repoList + $(widgetFile "configurators/listrepositories") + +{- A numbered list of known repositories, including the current one. -} +repoList :: Handler [(String, String)] repoList = do l <- runAnnex [] $ do u <- getUUID rs <- map Remote.uuid <$> Remote.remoteList rs' <- snd <$> trustPartition DeadTrusted rs Remote.prettyListUUIDs $ filter (/= webUUID) $ nub $ u:rs' - let n = length l - return (zip counter l, show (length l), n < enough, n == enough, n > enough) + return $ zip counter l where counter = map show ([1..] :: [Int]) - enough = 2 {- An intro message, list of repositories, and nudge to make more. -} introDisplay :: Text -> Widget introDisplay ident = do webapp <- lift getYesod - (repolist, numrepos, notenough, barelyenough, morethanenough) <- lift repoList + repolist <- lift repoList + let n = length repolist + let numrepos = show n + let notenough = n < enough + let barelyenough = n == enough + let morethanenough = n > enough $(widgetFile "configurators/intro") lift $ modifyWebAppState $ \s -> s { showIntro = False } + where + enough = 2 data RepositoryPath = RepositoryPath Text deriving Show @@ -160,7 +166,7 @@ addLocalRepositoryForm msg = do getFirstRepositoryR :: Handler RepHtml getFirstRepositoryR = bootstrap (Just Config) $ do - sideBarDisplay Nothing + sideBarDisplay setTitle "Getting started" ((res, form), enctype) <- lift $ runFormGet addLocalRepositoryForm case res of diff --git a/Assistant/WebApp/DashBoard.hs b/Assistant/WebApp/DashBoard.hs index 32159c22d9..8e526fb1d9 100644 --- a/Assistant/WebApp/DashBoard.hs +++ b/Assistant/WebApp/DashBoard.hs @@ -68,7 +68,7 @@ getTransfersR nid = do {- The main dashboard. -} dashboard :: Bool -> Widget dashboard warnNoScript = do - sideBarDisplay Nothing + sideBarDisplay let content = transfersDisplay warnNoScript $(widgetFile "dashboard/main") diff --git a/Assistant/WebApp/Documentation.hs b/Assistant/WebApp/Documentation.hs index 530c4ceb35..b0a9e4d98c 100644 --- a/Assistant/WebApp/Documentation.hs +++ b/Assistant/WebApp/Documentation.hs @@ -17,6 +17,6 @@ import Yesod getAboutR :: Handler RepHtml getAboutR = bootstrap (Just About) $ do - sideBarDisplay Nothing + sideBarDisplay setTitle "About git-annex" $(widgetFile "documentation/about") diff --git a/Assistant/WebApp/SideBar.hs b/Assistant/WebApp/SideBar.hs index 2fbd4e8c0f..4373b5a5b0 100644 --- a/Assistant/WebApp/SideBar.hs +++ b/Assistant/WebApp/SideBar.hs @@ -22,11 +22,8 @@ import Data.Text (Text) import qualified Data.Map as M import Control.Concurrent -sideBarDisplay :: Maybe Widget -> Widget -sideBarDisplay onsidebar = do - {- If a widget was passed to include on the sidebar, display - - it above alerts. -} - let perpage = maybe noop id onsidebar +sideBarDisplay :: Widget +sideBarDisplay = do let content = do {- Any yesod message appears as the first alert. -} maybe noop rendermessage =<< lift getMessage @@ -86,7 +83,7 @@ getSideBarR nid = do - to avoid slowing down user actions like closing alerts. -} liftIO $ threadDelay 100000 - page <- widgetToPageContent $ sideBarDisplay Nothing + page <- widgetToPageContent sideBarDisplay hamletToRepHtml $ [hamlet|^{pageBody page}|] {- Called by the client to close an alert. -} diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes index 893ba79feb..12b9564ee0 100644 --- a/Assistant/WebApp/routes +++ b/Assistant/WebApp/routes @@ -1,11 +1,12 @@ / HomeR GET /noscript NoScriptR GET -/noscriptauto NoScriptAutoR GET +/noscript/auto NoScriptAutoR GET /about AboutR GET /config ConfigR GET -/config/addrepository AddRepositoryR GET -/config/firstrepository FirstRepositoryR GET +/config/repository/add AddRepositoryR GET +/config/repository/first FirstRepositoryR GET +/config/repository/list ListRepositoriesR GET /transfers/#NotificationId TransfersR GET /sidebar/#NotificationId SideBarR GET diff --git a/templates/configurators/addrepository.hamlet b/templates/configurators/addrepository.hamlet index 1f793c2a56..d91286ad84 100644 --- a/templates/configurators/addrepository.hamlet +++ b/templates/configurators/addrepository.hamlet @@ -1,10 +1,52 @@ -
-
-

- Clone to removable drive -

- Clone this repository to a USB drive, or other removable media, # - for offline archiving, backups, or to # - SneakerNet # - between computers. - +

+

+ Add repositories +
+
+

+ + Clone to a removable drive +

+ Clone this repository to a USB drive, memory stick, or other # + removable media. +

+ For offline archiving, backups, or to # + SneakerNet # + between computers. +

+

+ + Pair with a local computer +

+ Automatically keep files in sync between computers on your # + local network. +

+ For easy sharing with family and friends, or between your devices. +

+

+ + Connect to your phone +

+ Save photos and recordings from your phone. +

+ Send selected files to your phone. +

+
+

+ + Store data in the cloud +

+ Store your data on a third-party cloud platform, # + including Amazon S3, Box.com, and Rsync.net. +

+ Pay someone to keep your data safe and available. # + With strong encryption to protect your privacy. +

+

+ + Clone to a remote server +

+ Set up a repository on a remote server using # + ssh or rsync. +

+ To use your own personal cloud. diff --git a/templates/configurators/listrepositories.hamlet b/templates/configurators/listrepositories.hamlet new file mode 100644 index 0000000000..22a6910de5 --- /dev/null +++ b/templates/configurators/listrepositories.hamlet @@ -0,0 +1,9 @@ +

+ + + $forall (num, name) <- repolist + +
+ #{num} + + #{name} diff --git a/templates/configurators/main.hamlet b/templates/configurators/main.hamlet index 929b22437e..c974eeb5dd 100644 --- a/templates/configurators/main.hamlet +++ b/templates/configurators/main.hamlet @@ -1,53 +1,15 @@
-

- Add repositories # -

- - Clone to a removable drive -

- Clone this repository to a USB drive, memory stick, or other # - removable media. -

- For offline archiving, backups, or to # - SneakerNet # - between computers. + + Add repositories +

+ Distribute the files in this repository to other devices; + make backups; and more by adding repositories.

- - Pair with a local computer -

- Automatically keep files in sync between computers on your # - local network. -

- For easy sharing with family and friends, or between your devices. -

-

- - Connect to your phone -

- Save photos and recordings from your phone. -

- Send selected files to your phone. -

-
-

- - Store data in the cloud -

- Store your data on a third-party cloud platform, # - including Amazon S3, Box.com, and Rsync.net. -

- Pay someone to keep your data safe and available. # - With strong encryption to protect your privacy. -

-

- - Clone to a remote server -

- Set up a repository on a remote server using # - ssh or rsync. -

- To use your own personal cloud. + + Repository list +

+ An overview of your repositories. diff --git a/templates/configurators/main/sidebar.hamlet b/templates/configurators/main/sidebar.hamlet deleted file mode 100644 index 3d427f8c56..0000000000 --- a/templates/configurators/main/sidebar.hamlet +++ /dev/null @@ -1,18 +0,0 @@ -

-

- git-annex is managing # - $if notenough - only # - #{numrepos} repository. # - $else - $if barelyenough - #{numrepos} repositories. # - $else - #{numrepos} repositories. # - $if notenough - Recommend you add more clones to avoid data loss. - $else - $if barelyenough - Consider adding more. - $else - Adding more can't hurt! diff --git a/templates/sidebar/main.hamlet b/templates/sidebar/main.hamlet index 63f201cf14..32900b920c 100644 --- a/templates/sidebar/main.hamlet +++ b/templates/sidebar/main.hamlet @@ -1,4 +1,3 @@ -
- ^{perpage} -
+
+
^{content} From 98aa929f0271631ba5ee1fc430cd177bd886d9ef Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Aug 2012 20:49:40 -0400 Subject: [PATCH 4540/8313] blog for the day --- .../assistant/blog/day_52__file_browser.mdwn | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/design/assistant/blog/day_52__file_browser.mdwn diff --git a/doc/design/assistant/blog/day_52__file_browser.mdwn b/doc/design/assistant/blog/day_52__file_browser.mdwn new file mode 100644 index 0000000000..a9762cc096 --- /dev/null +++ b/doc/design/assistant/blog/day_52__file_browser.mdwn @@ -0,0 +1,21 @@ +Today I added a "Files" link in the navbar of the WebApp. It looks like a +regular hyperlink, but clicking on it opens up your desktop's native file +manager, to manage the files in the repository! + +Quite fun to be able to do this kind of thing from a web page. :) + +--- + +Made `git annex init` (and the WebApp) automatically generate a description +of the repo when none is provided. + +--- + +Also worked on the configuration pages some. I don't want to get ahead +of myself by diving into the full configuration stage yet, but I am at +least going to add a configuration screen to clone the repo to a removable +drive. + +After that, the list of transfers on the dashboard needs some love. +I'll probably start by adding UI to cancel running transfers, and then +try to get drag and drop reordering of transfers working. From 0ca85a94283a0ec067fc498747f246ef63dce993 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Fri, 3 Aug 2012 23:51:49 -0400 Subject: [PATCH 4541/8313] Revert "init: If no description is provided for a new repository, one will automatically be generated, like "joey@gnu:~/foo"" This reverts commit abde98cda21d5deeb16ed3baf736b06fdc2fce2f. Temporarily dropping from master, since this actually uses stuff that's only currently availble in the assistant branch. Will come back when I merge that, and can wait.. --- Init.hs | 22 +--------------------- debian/changelog | 2 -- doc/git-annex.mdwn | 3 +-- 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/Init.hs b/Init.hs index 291e346f5f..bddcc696e0 100644 --- a/Init.hs +++ b/Init.hs @@ -19,26 +19,6 @@ import Logs.UUID import Annex.Version import Annex.UUID -import System.Posix.User - -genDescription :: Maybe String -> Annex String -genDescription (Just d) = return d -genDescription Nothing = do - hostname <- getHostname - let at = if null hostname then "" else "@" - username <- clicketyclickety - reldir <- liftIO . relHome =<< fromRepo Git.repoPath - return $ concat [username, at, hostname, ":", reldir] - where - {- Haskell lacks uname(2) bindings, except in the - - Bindings.Uname addon. Rather than depend on that, - - use uname -n when available. -} - getHostname = liftIO $ catchDefaultIO uname_node "" - uname_node = takeWhile (/= '\n') <$> - readProcess "uname" ["-n"] - clicketyclickety = liftIO $ userName <$> - (getUserEntryForID =<< getEffectiveUserID) - initialize :: Maybe String -> Annex () initialize mdescription = do prepUUID @@ -46,7 +26,7 @@ initialize mdescription = do setVersion gitPreCommitHookWrite u <- getUUID - describeUUID u =<< genDescription mdescription + maybe (recordUUID u) (describeUUID u) mdescription uninitialize :: Annex () uninitialize = do diff --git a/debian/changelog b/debian/changelog index 081d1ac9f8..82bf5009a1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,8 +2,6 @@ git-annex (3.20120722) UNRELEASED; urgency=low * initremote: Avoid recording remote's description before checking that its config is valid. - * init: If no description is provided for a new repository, one will - automatically be generated, like "joey@gnu:~/foo" -- Joey Hess Fri, 27 Jul 2012 21:04:47 -0400 diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn index a3fc82b55f..c52a5f3bf9 100644 --- a/doc/git-annex.mdwn +++ b/doc/git-annex.mdwn @@ -194,8 +194,7 @@ subdirectories). using it in a repository that was not intended to have an annex. It's useful, but not mandatory, to initialize each new clone - of a repository with its own description. If you don't provide one, - one will be generated. + of a repository with its own description. * describe repository description From e125ce74b87a86b80f1eead371390ce72c58428b Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Sat, 4 Aug 2012 18:17:16 -0400 Subject: [PATCH 4542/8313] work toward adding new repos on removable drives This actually does add a new repo, but it doesn't yet set up remotes, or sync to it. --- Assistant/WebApp/Configurators.hs | 163 ++++++++++++++++-- Assistant/WebApp/routes | 1 + Utility/Mounts.hsc | 2 +- templates/configurators/adddrive.hamlet | 30 ++++ templates/configurators/addrepository.hamlet | 4 +- .../configurators/firstrepository.hamlet | 26 +-- templates/page.hamlet | 2 - 7 files changed, 196 insertions(+), 32 deletions(-) create mode 100644 templates/configurators/adddrive.hamlet diff --git a/Assistant/WebApp/Configurators.hs b/Assistant/WebApp/Configurators.hs index 2771a22842..30c7c93301 100644 --- a/Assistant/WebApp/Configurators.hs +++ b/Assistant/WebApp/Configurators.hs @@ -23,12 +23,17 @@ import qualified Git.Config import qualified Annex import Locations.UserConfig import Utility.FreeDesktop +import Utility.Mounts +import Utility.DiskFree +import Utility.DataUnits import Yesod import Data.Text (Text) import qualified Data.Text as T import Data.Char import System.Posix.Directory +import System.Posix.User +import qualified Control.Exception as E {- The main configuration screen. -} getConfigR :: Handler RepHtml @@ -117,7 +122,8 @@ checkRepositoryPath p = do , (doesFileExist path, "A file already exists with that name.") , (return $ path == home, "Sorry, using git-annex for your whole home directory is not currently supported.") , (not <$> doesDirectoryExist parent, "Parent directory does not exist.") - , (cannotWrite path, "Cannot write a repository there.") + , (not <$> canWrite path, "Cannot write a repository there.") + , (not <$> canMakeSymlink path, "That directory is on a filesystem that does not support symlinks. Try a different location.") ] return $ case headMaybe problems of @@ -128,10 +134,6 @@ checkRepositoryPath p = do ( return $ Just msg , return Nothing ) - cannotWrite path = do - tocheck <- ifM (doesDirectoryExist path) - (return path, return $ parentDir path) - not <$> (catchBoolIO $ fileAccess tocheck False True False) expandTilde home ('~':'/':path) = home path expandTilde _ path = path @@ -150,8 +152,8 @@ defaultRepositoryPath firstrun = do (relHome (desktop "annex"), return "~/annex") else return cwd -addLocalRepositoryForm :: Form RepositoryPath -addLocalRepositoryForm msg = do +localRepositoryForm :: Form RepositoryPath +localRepositoryForm msg = do path <- T.pack . addTrailingPathSeparator <$> (liftIO . defaultRepositoryPath =<< lift inFirstRun) (pathRes, pathView) <- mreq (repositoryPathField True) "" (Just path) @@ -164,16 +166,117 @@ addLocalRepositoryForm msg = do $(widgetFile "configurators/localrepositoryform") return (RepositoryPath <$> pathRes, form) +{- Making the first repository, when starting the webapp for the first time. -} getFirstRepositoryR :: Handler RepHtml getFirstRepositoryR = bootstrap (Just Config) $ do sideBarDisplay setTitle "Getting started" - ((res, form), enctype) <- lift $ runFormGet addLocalRepositoryForm + ((res, form), enctype) <- lift $ runFormGet localRepositoryForm case res of FormSuccess (RepositoryPath p) -> lift $ startFullAssistant $ T.unpack p _ -> $(widgetFile "configurators/firstrepository") +data RemovableDrive = RemovableDrive + { diskFree :: Maybe Integer + , mountPoint :: Text + } + deriving (Show, Eq, Ord) + +selectDriveForm :: [RemovableDrive] -> Maybe RemovableDrive -> Form RemovableDrive +selectDriveForm drives def = renderBootstrap $ RemovableDrive + <$> pure Nothing + <*> areq (selectFieldList pairs) "Select drive:" (mountPoint <$> def) + where + pairs = zip (map describe drives) (map mountPoint drives) + describe drive = case diskFree drive of + Nothing -> mountPoint drive + Just free -> + let sz = roughSize storageUnits True free + in T.unwords + [ mountPoint drive + , T.concat ["(", T.pack sz] + , "free)" + ] + +{- Making the first repository, when starting the webapp for the first time. -} +getAddDriveR :: Handler RepHtml +getAddDriveR = bootstrap (Just Config) $ do + sideBarDisplay + setTitle "Clone to a removable drive" + removabledrives <- liftIO $ driveList + writabledrives <- liftIO $ + filterM (canWrite . T.unpack . mountPoint) removabledrives + ((res, form), enctype) <- lift $ runFormGet $ + selectDriveForm (sort writabledrives) Nothing + case res of + FormSuccess (RemovableDrive { mountPoint = d }) -> lift $ do + liftIO $ go $ T.unpack d "annex" + setMessage $ toHtml $ T.unwords ["Added", d] + redirect ListRepositoriesR + _ -> do + let authtoken = webAppFormAuthToken + $(widgetFile "configurators/adddrive") + where + {- There may already be a git-annex repo on the drive. + - If so, avoid re-initualizing it; this will be the + - case if a user is adding the same removable drive + - to several computers. + - + - Some drives will have FAT or another horrible filesystem + - that does not support symlinks; make a bare repo on those. + - + - Use the basename of the mount point, along with the + - username (but without the hostname as this repo + - travels!), as the repo description, and use the basename + - of the mount point as the git remote name. + -} + go dir = do + r <- E.try getannex :: IO (Either E.SomeException Annex.AnnexState) + state <- case r of + Right state -> return state + Left _e -> do + createDirectoryIfMissing True dir + bare <- not <$> canMakeSymlink dir + makeRepo dir bare + getannex + desc <- getdesc + Annex.eval state $ + unlessM isInitialized $ + initialize $ Just desc + where + getannex = Annex.new =<< Git.Construct.fromAbsPath dir + remotename = takeFileName dir + getdesc = do + username <- userName <$> + (getUserEntryForID =<< getEffectiveUserID) + return $ concat + [ username + , ":" + , remotename + ] + +{- List of removable drives. -} +driveList :: IO [RemovableDrive] +driveList = mapM (gen . mnt_dir) =<< filter sane <$> getMounts + where + gen dir = RemovableDrive + <$> getDiskFree dir + <*> pure (T.pack dir) + -- filter out some things that are surely not removable drives + sane Mntent { mnt_dir = dir, mnt_fsname = dev } + -- We want real disks like /dev/foo, not + -- dummy mount points like proc or tmpfs or + -- gvfs-fuse-daemon + | not ('/' `elem` dev) = False + -- Just in case: These mount points are surely not + -- removable disks. + | dir == "/" = False + | dir == "/tmp" = False + | dir == "/run/shm" = False + | dir == "/run/lock" = False + | otherwise = True + {- Bootstraps from first run mode to a fully running assistant in a - repository, by running the postFirstRun callback, which returns the - url to the new webapp. -} @@ -181,19 +284,53 @@ startFullAssistant :: FilePath -> Handler () startFullAssistant path = do webapp <- getYesod url <- liftIO $ do - makeRepo path + makeRepo path False + initRepo path Nothing + addAutoStart path changeWorkingDirectory path fromJust $ postFirstRun webapp redirect $ T.pack url {- Makes a new git-annex repository. -} -makeRepo :: FilePath -> IO () -makeRepo path = do - unlessM (boolSystem "git" [Param "init", Param "--quiet", File path]) $ +makeRepo :: FilePath -> Bool -> IO () +makeRepo path bare = do + unlessM (boolSystem "git" params) $ error "git init failed!" + where + baseparams = [Param "init", Param "--quiet"] + params + | bare = baseparams ++ [Param "--bare", File path] + | otherwise = baseparams ++ [File path] + +{- Initializes a git-annex repository in a directory with a description. -} +initRepo :: FilePath -> Maybe String -> IO () +initRepo path desc = do g <- Git.Config.read =<< Git.Construct.fromPath path state <- Annex.new g - Annex.eval state $ initialize Nothing + Annex.eval state $ initialize desc + +{- Adds a directory to the autostart file. -} +addAutoStart :: FilePath -> IO () +addAutoStart path = do autostart <- autoStartFile createDirectoryIfMissing True (parentDir autostart) appendFile autostart $ path ++ "\n" + +{- Checks if the user can write to a directory. + - + - The directory may be in the process of being created; if so + - the parent directory is checked instead. -} +canWrite :: FilePath -> IO Bool +canWrite dir = do + tocheck <- ifM (doesDirectoryExist dir) + (return dir, return $ parentDir dir) + catchBoolIO $ fileAccess tocheck False True False + +{- Checks if a directory is on a filesystem that supports symlinks. -} +canMakeSymlink :: FilePath -> IO Bool +canMakeSymlink dir = catchBoolIO $ do + createSymbolicLink link link + removeLink link + return True + where + link = dir "delete.me" diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes index 12b9564ee0..a53caf5efc 100644 --- a/Assistant/WebApp/routes +++ b/Assistant/WebApp/routes @@ -5,6 +5,7 @@ /config ConfigR GET /config/repository/add AddRepositoryR GET +/config/repository/add/drive AddDriveR GET /config/repository/first FirstRepositoryR GET /config/repository/list ListRepositoriesR GET diff --git a/Utility/Mounts.hsc b/Utility/Mounts.hsc index 6b69e844a3..0b14685217 100644 --- a/Utility/Mounts.hsc +++ b/Utility/Mounts.hsc @@ -28,7 +28,7 @@ import Prelude hiding (catch) - fields available everywhere. -} data Mntent = Mntent { mnt_fsname :: String - , mnt_dir :: String + , mnt_dir :: FilePath , mnt_type :: String } deriving (Read, Show, Eq, Ord) diff --git a/templates/configurators/adddrive.hamlet b/templates/configurators/adddrive.hamlet new file mode 100644 index 0000000000..030ef8a4c2 --- /dev/null +++ b/templates/configurators/adddrive.hamlet @@ -0,0 +1,30 @@ +
+

+ Adding a removable drive +

+ Clone this repository to a USB drive, memory stick, or other # + removable media. +

+ $if (null writabledrives) +

+ $if (null removabledrives) +

+ No removable drives found + Please make sure you have a removable drive plugged in and mounted. + $else +

+ No usable removable drives found + Seems you cannot write to any of the removable drives that are # + currently mounted. Try plugging in a removable drive that you can # + write to, or correcting the write permissions. +

+ + Rescan for removable drives + $else + +

+ ^{form} + ^{authtoken} + # + + Rescan for removable drives diff --git a/templates/configurators/addrepository.hamlet b/templates/configurators/addrepository.hamlet index d91286ad84..1ba4f94bf3 100644 --- a/templates/configurators/addrepository.hamlet +++ b/templates/configurators/addrepository.hamlet @@ -1,10 +1,8 @@
-

- Add repositories

- + Clone to a removable drive

Clone this repository to a USB drive, memory stick, or other # diff --git a/templates/configurators/firstrepository.hamlet b/templates/configurators/firstrepository.hamlet index f4ffcf372c..ac28119eba 100644 --- a/templates/configurators/firstrepository.hamlet +++ b/templates/configurators/firstrepository.hamlet @@ -1,14 +1,14 @@

-

- Welcome to git-annex! -

- There's just one thing to do before you can start using the power # - and convenience of git-annex. -

- Create a git-annex repository -

- Files in this repository will managed by git-annex, # - and kept in sync with your repositories on other devices. -

- - ^{form} +

+ Welcome to git-annex! +

+ There's just one thing to do before you can start using the power # + and convenience of git-annex. +

+ Create a git-annex repository +

+ Files in this repository will managed by git-annex, # + and kept in sync with your repositories on other devices. +

+ + ^{form} diff --git a/templates/page.hamlet b/templates/page.hamlet index 6321f7a181..5544221daf 100644 --- a/templates/page.hamlet +++ b/templates/page.hamlet @@ -17,8 +17,6 @@ Current Repository: #{reldir}

#{name} - Add another repository + Add another repository

Or just sit back, watch the magic, and get on with using your files. diff --git a/templates/configurators/listrepositories.hamlet b/templates/configurators/listrepositories.hamlet deleted file mode 100644 index 22a6910de5..0000000000 --- a/templates/configurators/listrepositories.hamlet +++ /dev/null @@ -1,9 +0,0 @@ -

- - - $forall (num, name) <- repolist - -
- #{num} - - #{name} diff --git a/templates/configurators/main.hamlet b/templates/configurators/main.hamlet index c974eeb5dd..1482a904b4 100644 --- a/templates/configurators/main.hamlet +++ b/templates/configurators/main.hamlet @@ -2,14 +2,8 @@

- - Add repositories + + Manage repositories

- Distribute the files in this repository to other devices; - make backups; and more by adding repositories. -

-

- - Repository list -

- An overview of your repositories. + Distribute the files in this repository to other devices, + make backups, and more, by adding repositories. diff --git a/templates/configurators/addrepository.hamlet b/templates/configurators/repositories.hamlet similarity index 66% rename from templates/configurators/addrepository.hamlet rename to templates/configurators/repositories.hamlet index 81f78828b8..8013e64ec2 100644 --- a/templates/configurators/addrepository.hamlet +++ b/templates/configurators/repositories.hamlet @@ -1,9 +1,21 @@

+

+ Your repositories + + + $forall (num, name) <- repolist + +
+ #{num} + + #{name} +

+ Add more repositories

- Add a removable drive + Removable drive

Clone this repository to a USB drive, memory stick, or other # removable media. @@ -13,15 +25,15 @@ between computers.

- Pair with a local computer + Local computer

- Automatically keep files in sync between computers on your # - local network. + Pair with a local computer to automatically keep files in sync # + between computers on your local network.

For easy sharing with family and friends, or between your devices.

- Connect to your phone + Phone

Save photos and recordings from your phone.

@@ -29,7 +41,7 @@

- Store data in the cloud + The cloud

Store your data on a third-party cloud platform, # including Amazon S3, Box.com, and Rsync.net. @@ -38,7 +50,7 @@ With strong encryption to protect your privacy.

- Clone to a remote server + Remote server

Set up a repository on a remote server using # ssh or rsync. diff --git a/templates/page.hamlet b/templates/page.hamlet index 5544221daf..2d33e95619 100644 --- a/templates/page.hamlet +++ b/templates/page.hamlet @@ -17,7 +17,7 @@ Current Repository: #{reldir}

#{name} -

- Add more repositories
-
+
+

+ Add more repositories +

Removable drive @@ -23,7 +24,7 @@ For offline archiving, backups, or to # SneakerNet # between computers. -
+

Local computer

@@ -31,29 +32,37 @@ between computers on your local network.

For easy sharing with family and friends, or between your devices. -

+

Phone

Save photos and recordings from your phone.

Send selected files to your phone. -

-
+ +
+

+ Store your data in the cloud +

- The cloud + + Rsync.net

- Store your data on a third-party cloud platform, # - including Amazon S3, Box.com, and Rsync.net. + Works very well with git-annex. + +

+ Amazon S3

- Pay someone to keep your data safe and available. # - With strong encryption to protect your privacy. -

+ Good choice for professional storage quality and low prices. + +

+ Box.com +

+ Provides free cloud storage for small amounts of data. +

Remote server

Set up a repository on a remote server using # - ssh. -

- To build your own personal cloud. + ssh, to build your own personal cloud. From b584d96c131a23d0f7d6ed424a8915e9e15a6bc4 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 3 Sep 2012 00:39:55 -0400 Subject: [PATCH 4845/8313] rsync.net configurator display Doesn't set up the repo yet. --- Assistant/WebApp/Configurators/Ssh.hs | 27 +++++++++++-- Assistant/WebApp/routes | 1 + templates/configurators/addrsync.net.hamlet | 43 +++++++++++++++++++++ templates/configurators/main.hamlet | 2 +- 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 templates/configurators/addrsync.net.hamlet diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs index b8e2b351a2..334ee08079 100644 --- a/Assistant/WebApp/Configurators/Ssh.hs +++ b/Assistant/WebApp/Configurators/Ssh.hs @@ -42,10 +42,10 @@ data SshServer = SshServer } deriving (Show) -sshServerAForm :: Text -> AForm WebApp WebApp SshServer +sshServerAForm :: (Maybe Text) -> AForm WebApp WebApp SshServer sshServerAForm localusername = SshServer <$> aopt check_hostname "Host name" Nothing - <*> aopt check_username "User name" (Just $ Just localusername) + <*> aopt check_username "User name" (Just localusername) <*> aopt textField "Directory" (Just $ Just $ T.pack gitAnnexAssistantDefaultDir) where check_hostname = checkM (liftIO . checkdns) textField @@ -83,7 +83,7 @@ getAddSshR = sshConfigurator $ do u <- liftIO $ T.pack . userName <$> (getUserEntryForID =<< getEffectiveUserID) ((result, form), enctype) <- lift $ - runFormGet $ renderBootstrap $ sshServerAForm u + runFormGet $ renderBootstrap $ sshServerAForm (Just u) case result of FormSuccess sshserver -> do (status, needspubkey) <- liftIO $ testServer sshserver @@ -320,3 +320,24 @@ genSshKey sshdata = do sshpubkeyfile = sshprivkeyfile ++ ".pub" mangledhost = "git-annex-" ++ T.unpack (sshHostName sshdata) ++ user user = maybe "" (\u -> "-" ++ T.unpack u) (sshUserName sshdata) + +getAddRsyncNetR :: Handler RepHtml +getAddRsyncNetR = bootstrap (Just Config) $ do + sideBarDisplay + setTitle "Add a Rsync.net repository" + ((result, form), enctype) <- lift $ + runFormGet $ renderBootstrap $ sshServerAForm Nothing + let showform status = do + let authtoken = webAppFormAuthToken + $(widgetFile "configurators/addrsync.net") + case result of + FormSuccess sshserver -> do + let host = fromMaybe "" $ hostname sshserver + checkhost host showform $ do + error "TODO" + _ -> showform UntestedServer + where + checkhost host showform a + | ".rsync.net" `T.isSuffixOf` T.toLower host = a + | otherwise = showform $ UnusableServer + "That is not a rsync.net host name." diff --git a/Assistant/WebApp/routes b/Assistant/WebApp/routes index 5f8dfbbc48..7ed1f30d33 100644 --- a/Assistant/WebApp/routes +++ b/Assistant/WebApp/routes @@ -10,6 +10,7 @@ /config/repository/add/ssh/confirm/#SshData ConfirmSshR GET /config/repository/add/ssh/make/git/#SshData MakeSshGitR GET /config/repository/add/ssh/make/rsync/#SshData MakeSshRsyncR GET +/config/repository/add/rsync.net AddRsyncNetR GET /config/repository/first FirstRepositoryR GET /transfers/#NotificationId TransfersR GET diff --git a/templates/configurators/addrsync.net.hamlet b/templates/configurators/addrsync.net.hamlet new file mode 100644 index 0000000000..163d0721ee --- /dev/null +++ b/templates/configurators/addrsync.net.hamlet @@ -0,0 +1,43 @@ +

+

+ Adding a Rsync.net repository +

+ + Rsync.net # + is a well-respected cloud storage provider. Its rsync repositories are # + supported very well by git-annex. # + + pricing details +

+ $case status + $of UnusableServer msg +

+ #{msg} + $of _ + # + All your data will be synced to the Rsync.net repository, so make # + sure it has enough available space. Your data will be encrypted before # + it is sent to Rsync.net. +

+ When you sign up for a Rsync.net account, you receive an # + email from them with a host name and a username. Fill that # + information in below. You also likely don't want to use your whole # + rsync.net repository for git-annex alone, so git-annex will use a # + subdirectory of it, as configured below. +

+ +

+ ^{form} + ^{authtoken} +
+